diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
index 237f45e6f08e0..c44936b19dab9 100644
--- a/.ci/generate_test_report.py
+++ b/.ci/generate_test_report.py
@@ -18,7 +18,7 @@ def junit_from_xml(xml):
 
 class TestReports(unittest.TestCase):
     def test_title_only(self):
-        self.assertEqual(_generate_report("Foo", []), ("", None))
+        self.assertEqual(_generate_report("Foo", []), ("", "success"))
 
     def test_no_tests_in_testsuite(self):
         self.assertEqual(
@@ -336,7 +336,7 @@ def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures
                     )
 
     if not tests_run:
-        return ("", style)
+        return ("", None)
 
     style = "error" if tests_failed else "success"
     report = [f"# {title}", ""]
diff --git a/.ci/metrics/Dockerfile b/.ci/metrics/Dockerfile
new file mode 100644
index 0000000000000..80f1c64e3e61b
--- /dev/null
+++ b/.ci/metrics/Dockerfile
@@ -0,0 +1,7 @@
+FROM docker.io/python:3.12
+
+COPY requirements.lock.txt ./
+RUN pip3 install --no-cache-dir -r requirements.lock.txt
+COPY metrics.py ./
+
+CMD ["python3", "metrics.py"]
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
new file mode 100644
index 0000000000000..deb36bc1689a0
--- /dev/null
+++ b/.ci/metrics/metrics.py
@@ -0,0 +1,182 @@
+import requests
+import time
+import os
+from dataclasses import dataclass
+import sys
+
+import github
+from github import Github
+from github import Auth
+
+GRAFANA_URL = (
+    "https://influx-prod-13-prod-us-east-0.grafana.net/api/v1/push/influx/write"
+)
+GITHUB_PROJECT = "llvm/llvm-project"
+WORKFLOWS_TO_TRACK = ["Check code formatting"]
+SCRAPE_INTERVAL_SECONDS = 5 * 60
+
+
+@dataclass
+class JobMetrics:
+    job_name: str
+    queue_time: int
+    run_time: int
+    status: int
+    created_at_ns: int
+    workflow_id: int
+
+
+def get_metrics(github_repo: github.Repository, workflows_to_track: dict[str, int]):
+    """Gets the metrics for specified Github workflows.
+
+    This function takes in a list of workflows to track, and optionally the
+    workflow ID of the last tracked invocation. It grabs the relevant data
+    from Github, returning it to the caller.
+
+    Args:
+      github_repo: A github repo object to use to query the relevant information.
+      workflows_to_track: A dictionary mapping workflow names to the last
+        invocation ID where metrics have been collected, or None to collect the
+        last five results.
+
+    Returns:
+      Returns a list of JobMetrics objects, containing the relevant metrics about
+      the workflow.
+    """
+    workflow_runs = iter(github_repo.get_workflow_runs())
+
+    workflow_metrics = []
+
+    workflows_to_include = set(workflows_to_track.keys())
+
+    while len(workflows_to_include) > 0:
+        workflow_run = next(workflow_runs)
+        if workflow_run.status != "completed":
+            continue
+
+        # This workflow was already sampled for this run, or is not tracked at
+        # all. Ignoring.
+        if workflow_run.name not in workflows_to_include:
+            continue
+
+        # There were no new workflow invocations since the previous scrape.
+        # The API returns a sorted list with the most recent invocations first,
+        # so we can stop looking for this particular workflow. Continue to grab
+        # information on the other workflows of interest, if present.
+        if workflows_to_track[workflow_run.name] == workflow_run.id:
+            workflows_to_include.remove(workflow_run.name)
+            continue
+
+        workflow_jobs = workflow_run.jobs()
+        if workflow_jobs.totalCount == 0:
+            continue
+        if workflow_jobs.totalCount > 1:
+            raise ValueError(
+                f"Encountered an unexpected number of jobs: {workflow_jobs.totalCount}"
+            )
+
+        created_at = workflow_jobs[0].created_at
+        started_at = workflow_jobs[0].started_at
+        completed_at = workflow_jobs[0].completed_at
+
+        job_result = int(workflow_jobs[0].conclusion == "success")
+
+        queue_time = started_at - created_at
+        run_time = completed_at - started_at
+
+        if run_time.seconds == 0:
+            continue
+
+        if (
+            workflows_to_track[workflow_run.name] is None
+            or workflows_to_track[workflow_run.name] == workflow_run.id
+        ):
+            workflows_to_include.remove(workflow_run.name)
+        if (
+            workflows_to_track[workflow_run.name] is not None
+            and len(workflows_to_include) == 0
+        ):
+            break
+
+        # The timestamp associated with the event is expected by Grafana to be
+        # in nanoseconds.
+        created_at_ns = int(created_at.timestamp()) * 10**9
+
+        workflow_metrics.append(
+            JobMetrics(
+                workflow_run.name,
+                queue_time.seconds,
+                run_time.seconds,
+                job_result,
+                created_at_ns,
+                workflow_run.id,
+            )
+        )
+
+    return workflow_metrics
+
+
+def upload_metrics(workflow_metrics, metrics_userid, api_key):
+    """Upload metrics to Grafana.
+
+    Takes in a list of workflow metrics and then uploads them to Grafana
+    through a REST request.
+
+    Args:
+      workflow_metrics: A list of metrics to upload to Grafana.
+      metrics_userid: The userid to use for the upload.
+      api_key: The API key to use for the upload.
+    """
+    metrics_batch = []
+    for workflow_metric in workflow_metrics:
+        workflow_formatted_name = workflow_metric.job_name.lower().replace(" ", "_")
+        metrics_batch.append(
+            f"{workflow_formatted_name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.created_at_ns}"
+        )
+
+    request_data = "\n".join(metrics_batch)
+    response = requests.post(
+        GRAFANA_URL,
+        headers={"Content-Type": "text/plain"},
+        data=request_data,
+        auth=(metrics_userid, api_key),
+    )
+
+    if response.status_code < 200 or response.status_code >= 300:
+        print(
+            f"Failed to submit data to Grafana: {response.status_code}", file=sys.stderr
+        )
+
+
+def main():
+    # Authenticate with Github
+    auth = Auth.Token(os.environ["GITHUB_TOKEN"])
+    github_object = Github(auth=auth)
+    github_repo = github_object.get_repo("llvm/llvm-project")
+
+    grafana_api_key = os.environ["GRAFANA_API_KEY"]
+    grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
+
+    workflows_to_track = {}
+    for workflow_to_track in WORKFLOWS_TO_TRACK:
+        workflows_to_track[workflow_to_track] = None
+
+    # Enter the main loop. Every five minutes we wake up and dump metrics for
+    # the relevant jobs.
+    while True:
+        current_metrics = get_metrics(github_repo, workflows_to_track)
+        if len(current_metrics) == 0:
+            print("No metrics found to upload.", file=sys.stderr)
+            continue
+
+        upload_metrics(current_metrics, grafana_metrics_userid, grafana_api_key)
+        print(f"Uploaded {len(current_metrics)} metrics", file=sys.stderr)
+
+        for workflow_metric in reversed(current_metrics):
+            workflows_to_track[workflow_metric.job_name] = workflow_metric.workflow_id
+
+        time.sleep(SCRAPE_INTERVAL_SECONDS)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.ci/metrics/requirements.lock.txt b/.ci/metrics/requirements.lock.txt
new file mode 100644
index 0000000000000..8ab76327e0c82
--- /dev/null
+++ b/.ci/metrics/requirements.lock.txt
@@ -0,0 +1,335 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --generate-hashes --output-file=./requirements.lock.txt ./requirements.txt
+#
+certifi==2024.8.30 \
+    --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
+    --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
+    # via requests
+cffi==1.17.1 \
+    --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \
+    --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \
+    --hash=sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1 \
+    --hash=sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15 \
+    --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \
+    --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \
+    --hash=sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8 \
+    --hash=sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36 \
+    --hash=sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17 \
+    --hash=sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf \
+    --hash=sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc \
+    --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \
+    --hash=sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed \
+    --hash=sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702 \
+    --hash=sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1 \
+    --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \
+    --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \
+    --hash=sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6 \
+    --hash=sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d \
+    --hash=sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b \
+    --hash=sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e \
+    --hash=sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be \
+    --hash=sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c \
+    --hash=sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683 \
+    --hash=sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9 \
+    --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \
+    --hash=sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8 \
+    --hash=sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1 \
+    --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \
+    --hash=sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655 \
+    --hash=sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67 \
+    --hash=sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595 \
+    --hash=sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0 \
+    --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \
+    --hash=sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41 \
+    --hash=sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6 \
+    --hash=sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401 \
+    --hash=sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6 \
+    --hash=sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3 \
+    --hash=sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16 \
+    --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \
+    --hash=sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e \
+    --hash=sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4 \
+    --hash=sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964 \
+    --hash=sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c \
+    --hash=sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576 \
+    --hash=sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0 \
+    --hash=sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3 \
+    --hash=sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662 \
+    --hash=sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3 \
+    --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \
+    --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \
+    --hash=sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd \
+    --hash=sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f \
+    --hash=sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5 \
+    --hash=sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14 \
+    --hash=sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d \
+    --hash=sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9 \
+    --hash=sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7 \
+    --hash=sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382 \
+    --hash=sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a \
+    --hash=sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e \
+    --hash=sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a \
+    --hash=sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4 \
+    --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 \
+    --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \
+    --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b
+    # via
+    #   cryptography
+    #   pynacl
+charset-normalizer==3.4.0 \
+    --hash=sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621 \
+    --hash=sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6 \
+    --hash=sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8 \
+    --hash=sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912 \
+    --hash=sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c \
+    --hash=sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b \
+    --hash=sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d \
+    --hash=sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d \
+    --hash=sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95 \
+    --hash=sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e \
+    --hash=sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565 \
+    --hash=sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64 \
+    --hash=sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab \
+    --hash=sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be \
+    --hash=sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e \
+    --hash=sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907 \
+    --hash=sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0 \
+    --hash=sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2 \
+    --hash=sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62 \
+    --hash=sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62 \
+    --hash=sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23 \
+    --hash=sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc \
+    --hash=sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284 \
+    --hash=sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca \
+    --hash=sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455 \
+    --hash=sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858 \
+    --hash=sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b \
+    --hash=sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594 \
+    --hash=sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc \
+    --hash=sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db \
+    --hash=sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b \
+    --hash=sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea \
+    --hash=sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6 \
+    --hash=sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920 \
+    --hash=sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749 \
+    --hash=sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7 \
+    --hash=sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd \
+    --hash=sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99 \
+    --hash=sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242 \
+    --hash=sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee \
+    --hash=sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129 \
+    --hash=sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2 \
+    --hash=sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51 \
+    --hash=sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee \
+    --hash=sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8 \
+    --hash=sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b \
+    --hash=sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613 \
+    --hash=sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742 \
+    --hash=sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe \
+    --hash=sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3 \
+    --hash=sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5 \
+    --hash=sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631 \
+    --hash=sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7 \
+    --hash=sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15 \
+    --hash=sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c \
+    --hash=sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea \
+    --hash=sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417 \
+    --hash=sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250 \
+    --hash=sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88 \
+    --hash=sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca \
+    --hash=sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa \
+    --hash=sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99 \
+    --hash=sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149 \
+    --hash=sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41 \
+    --hash=sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574 \
+    --hash=sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0 \
+    --hash=sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f \
+    --hash=sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d \
+    --hash=sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654 \
+    --hash=sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3 \
+    --hash=sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19 \
+    --hash=sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90 \
+    --hash=sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578 \
+    --hash=sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9 \
+    --hash=sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1 \
+    --hash=sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51 \
+    --hash=sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719 \
+    --hash=sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236 \
+    --hash=sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a \
+    --hash=sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c \
+    --hash=sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade \
+    --hash=sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944 \
+    --hash=sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc \
+    --hash=sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6 \
+    --hash=sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6 \
+    --hash=sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27 \
+    --hash=sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6 \
+    --hash=sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2 \
+    --hash=sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12 \
+    --hash=sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf \
+    --hash=sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114 \
+    --hash=sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7 \
+    --hash=sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf \
+    --hash=sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d \
+    --hash=sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b \
+    --hash=sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed \
+    --hash=sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03 \
+    --hash=sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4 \
+    --hash=sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67 \
+    --hash=sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365 \
+    --hash=sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a \
+    --hash=sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748 \
+    --hash=sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b \
+    --hash=sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079 \
+    --hash=sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482
+    # via requests
+cryptography==43.0.3 \
+    --hash=sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362 \
+    --hash=sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4 \
+    --hash=sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa \
+    --hash=sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83 \
+    --hash=sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff \
+    --hash=sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805 \
+    --hash=sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6 \
+    --hash=sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664 \
+    --hash=sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08 \
+    --hash=sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e \
+    --hash=sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18 \
+    --hash=sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f \
+    --hash=sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73 \
+    --hash=sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5 \
+    --hash=sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984 \
+    --hash=sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd \
+    --hash=sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3 \
+    --hash=sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e \
+    --hash=sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405 \
+    --hash=sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2 \
+    --hash=sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c \
+    --hash=sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995 \
+    --hash=sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73 \
+    --hash=sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16 \
+    --hash=sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7 \
+    --hash=sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd \
+    --hash=sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7
+    # via pyjwt
+deprecated==1.2.15 \
+    --hash=sha256:353bc4a8ac4bfc96800ddab349d89c25dec1079f65fd53acdcc1e0b975b21320 \
+    --hash=sha256:683e561a90de76239796e6b6feac66b99030d2dd3fcf61ef996330f14bbb9b0d
+    # via pygithub
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
+    # via requests
+pycparser==2.22 \
+    --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
+    --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
+    # via cffi
+pygithub==2.5.0 \
+    --hash=sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2 \
+    --hash=sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf
+    # via -r ./requirements.txt
+pyjwt[crypto]==2.10.0 \
+    --hash=sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15 \
+    --hash=sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c
+    # via pygithub
+pynacl==1.5.0 \
+    --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \
+    --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \
+    --hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \
+    --hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \
+    --hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \
+    --hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \
+    --hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \
+    --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \
+    --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
+    --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
+    # via pygithub
+requests==2.32.3 \
+    --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
+    --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+    # via pygithub
+typing-extensions==4.12.2 \
+    --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
+    --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
+    # via pygithub
+urllib3==2.2.3 \
+    --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
+    --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
+    # via
+    #   pygithub
+    #   requests
+wrapt==1.16.0 \
+    --hash=sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc \
+    --hash=sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81 \
+    --hash=sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09 \
+    --hash=sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e \
+    --hash=sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca \
+    --hash=sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0 \
+    --hash=sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb \
+    --hash=sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487 \
+    --hash=sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40 \
+    --hash=sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c \
+    --hash=sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060 \
+    --hash=sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202 \
+    --hash=sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41 \
+    --hash=sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9 \
+    --hash=sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b \
+    --hash=sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664 \
+    --hash=sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d \
+    --hash=sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362 \
+    --hash=sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00 \
+    --hash=sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc \
+    --hash=sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1 \
+    --hash=sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267 \
+    --hash=sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956 \
+    --hash=sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966 \
+    --hash=sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1 \
+    --hash=sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228 \
+    --hash=sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72 \
+    --hash=sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d \
+    --hash=sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292 \
+    --hash=sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0 \
+    --hash=sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0 \
+    --hash=sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36 \
+    --hash=sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c \
+    --hash=sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5 \
+    --hash=sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f \
+    --hash=sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73 \
+    --hash=sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b \
+    --hash=sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2 \
+    --hash=sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593 \
+    --hash=sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39 \
+    --hash=sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389 \
+    --hash=sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf \
+    --hash=sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf \
+    --hash=sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89 \
+    --hash=sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c \
+    --hash=sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c \
+    --hash=sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f \
+    --hash=sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440 \
+    --hash=sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465 \
+    --hash=sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136 \
+    --hash=sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b \
+    --hash=sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8 \
+    --hash=sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3 \
+    --hash=sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8 \
+    --hash=sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6 \
+    --hash=sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e \
+    --hash=sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f \
+    --hash=sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c \
+    --hash=sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e \
+    --hash=sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8 \
+    --hash=sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2 \
+    --hash=sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020 \
+    --hash=sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35 \
+    --hash=sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d \
+    --hash=sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3 \
+    --hash=sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537 \
+    --hash=sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809 \
+    --hash=sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d \
+    --hash=sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a \
+    --hash=sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4
+    # via deprecated
diff --git a/.ci/metrics/requirements.txt b/.ci/metrics/requirements.txt
new file mode 100644
index 0000000000000..ab5ad7d9e469c
--- /dev/null
+++ b/.ci/metrics/requirements.txt
@@ -0,0 +1 @@
+pygithub==2.5.0
diff --git a/.github/new-issues-labeler.yml b/.github/new-issues-labeler.yml
index 860535bbe3083..ee7506c1366ef 100644
--- a/.github/new-issues-labeler.yml
+++ b/.github/new-issues-labeler.yml
@@ -30,3 +30,9 @@
 
 'infra:commit-access-request':
   - '/Request Commit Access/'
+
+'false-positive':
+  - '\bfalse[- ]positive\b'
+
+'false-negative':
+  - '\bfalse[- ]negative\b'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 28fc7de2ee065..f037a91f6e5d0 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -18,44 +18,18 @@ on:
       - '.github/workflows/containers/github-action-ci/**'
 
 jobs:
-  # TODO(boomanaiden154): Switch this back to a single stage build when we can
-  # run this on the self-hosted runners and don't have to do it this way to
-  # avoid timeouts.
-  build-ci-container-stage1:
+  build-ci-container:
     if: github.repository_owner == 'llvm'
-    runs-on: ubuntu-latest
+    runs-on: depot-ubuntu-22.04-16
+    outputs:
+      container-name: ${{ steps.vars.outputs.container-name }}
+      container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
+      container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@v4
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
-      - name: Change podman Root Direcotry
-        run: |
-          mkdir -p ~/.config/containers
-          sudo mkdir -p /mnt/podman
-          sudo chown `whoami`:`whoami` /mnt/podman
-          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
-          podman info
-      - name: Build container stage1
-        working-directory: ./.github/workflows/containers/github-action-ci/
-        run: |
-          podman build -t stage1-toolchain --target stage1-toolchain -f stage1.Dockerfile .
-      - name: Save container image
-        run: |
-          podman save stage1-toolchain > stage1-toolchain.tar
-      - name: Upload container image
-        uses: actions/upload-artifact@v4
-        with:
-          name: stage1-toolchain
-          path: stage1-toolchain.tar
-          retention-days: 1
-  build-ci-container-stage2:
-    if: github.repository_owner == 'llvm'
-    runs-on: ubuntu-latest
-    needs: build-ci-container-stage1
-    permissions:
-      packages: write
-    steps:
       - name: Write Variables
         id: vars
         run: |
@@ -63,38 +37,25 @@ jobs:
           container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04"
           echo "container-name=$container_name" >> $GITHUB_OUTPUT
           echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
-
-      - name: Checkout LLVM
-        uses: actions/checkout@v4
-        with:
-          sparse-checkout: .github/workflows/containers/github-action-ci/
-
-      - name: Change podman Root Direcotry
+          echo "container-filename=$(echo $container_name:$tag  | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
+      - name: Build container
+        working-directory: ./.github/workflows/containers/github-action-ci/
         run: |
-          mkdir -p ~/.config/containers
-          sudo mkdir -p /mnt/podman
-          sudo chown `whoami`:`whoami` /mnt/podman
-          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
-          podman info
+          podman build -t ${{ steps.vars.outputs.container-name-tag }} .
 
-        # Download the container image into /mnt/podman rather than
-        # $GITHUB_WORKSPACE to avoid space limitations on the default drive
-        # and use the permissions setup for /mnt/podman.
-      - name: Download stage1-toolchain
-        uses: actions/download-artifact@v4
-        with:
-          name: stage1-toolchain
-          path: /mnt/podman
-
-      - name: Load stage1-toolchain
+      # Save the container so we have it in case the push fails.  This also
+      # allows us to separate the push step into a different job so we can
+      # maintain minimal permissions while building the container.
+      - name: Save container image
         run: |
-          podman load -i /mnt/podman/stage1-toolchain.tar
+          podman save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
 
-      - name: Build Container
-        working-directory: ./.github/workflows/containers/github-action-ci/
-        run: |
-          podman build -t ${{ steps.vars.outputs.container-name-tag }} -f stage2.Dockerfile .
-          podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
+      - name: Upload container image
+        uses: actions/upload-artifact@v4
+        with:
+          name: container
+          path: ${{ steps.vars.outputs.container-filename }}
+          retention-days: 14
 
       - name: Test Container
         run: |
@@ -102,11 +63,25 @@ jobs:
             podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
           done
 
+  push-ci-container:
+    if: github.event_name == 'push'
+    needs:
+      - build-ci-container
+    permissions:
+      packages: write
+    runs-on: ubuntu-24.04
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Download container
+        uses: actions/download-artifact@v4
+        with:
+          name: container
+
       - name: Push Container
-        if: github.event_name == 'push'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
+          podman load -i ${{ needs.build-ci-container.outptus.container-filename }}
+          podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
           podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
-          podman push ${{ steps.vars.outputs.container-name-tag }}
-          podman push ${{ steps.vars.outputs.container-name }}:latest
+          podman push ${{ needs.build-ci-container.outputs.container-name-tag }}
+          podman push ${{ needs.build-ci-container.outputs.container-name }}:latest
diff --git a/.github/workflows/build-metrics-container.yml b/.github/workflows/build-metrics-container.yml
new file mode 100644
index 0000000000000..751ab679411dc
--- /dev/null
+++ b/.github/workflows/build-metrics-container.yml
@@ -0,0 +1,78 @@
+name: Build Metrics Container
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/build-metrics-container.yml
+      - '.ci/metrics/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/build-metrics-container.yml
+      - '.ci/metrics/**'
+
+jobs:
+  build-metrics-container:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-latest
+    outputs:
+      container-name: ${{ steps.vars.outputs.container-name }}
+      container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
+      container-filename: ${{ steps.vars.outputs.container-filename }}
+    steps:
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .ci/metrics/
+      - name: Write Variables
+        id: vars
+        run: |
+          tag=`date +%s`
+          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/metrics"
+          echo "container-name=$container_name" >> $GITHUB_OUTPUT
+          echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
+          echo "container-filename=$(echo $container_name:$tag  | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
+      - name: Build Container
+        working-directory: ./.ci/metrics
+        run: |
+          podman build -t ${{ steps.vars.outputs.container-name-tag }} -f Dockerfile .
+      # Save the container so we have it in case the push fails.  This also
+      # allows us to separate the push step into a different job so we can
+      # maintain minimal permissions while building the container.
+      - name: Save Container Image
+        run: |
+          podman save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
+      - name: Upload Container Image
+        uses: actions/upload-artifact@v4
+        with:
+          name: container
+          path: ${{ steps.vars.outputs.container-filename }}
+          retention-days: 14
+  
+  push-metrics-container:
+    if: github.event_name == 'push'
+    needs:
+      - build-metrics-container
+    permissions:
+      packages: write
+    runs-on: ubuntu-24.04
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Download Container
+        uses: actions/download-artifact@v4
+        with:
+          name: container
+      - name: Push Container
+        run: |
+          podman load -i ${{ needs.build-metrics-container.outputs.container-filename }}
+          podman tag ${{ needs.build-metrics-container.outputs.container-name-tag }} ${{ needs.build-metrics-container.outputs.container-name }}:latest
+          podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+          podman push ${{ needs.build-metrics-container.outputs.container-name-tag }}
+          podman push ${{ needs.build-metrics-container.outputs.container-name }}:latest
diff --git a/.github/workflows/commit-access-review.py b/.github/workflows/commit-access-review.py
index 8ea9b1fcc2fb0..91d3a61cdcb17 100644
--- a/.github/workflows/commit-access-review.py
+++ b/.github/workflows/commit-access-review.py
@@ -62,57 +62,9 @@ def __repr__(self):
         )
 
 
-def run_graphql_query(
-    query: str, variables: dict, token: str, retry: bool = True
-) -> dict:
-    """
-    This function submits a graphql query and returns the results as a
-    dictionary.
-    """
-    s = requests.Session()
-    retries = requests.adapters.Retry(total=8, backoff_factor=2, status_forcelist=[504])
-    s.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
-
-    headers = {
-        "Authorization": "bearer {}".format(token),
-        # See
-        # https://github.blog/2021-11-16-graphql-global-id-migration-update/
-        "X-Github-Next-Global-ID": "1",
-    }
-    request = s.post(
-        url="https://api.github.com/graphql",
-        json={"query": query, "variables": variables},
-        headers=headers,
-    )
-
-    rate_limit = request.headers.get("X-RateLimit-Remaining")
-    print(rate_limit)
-    if rate_limit and int(rate_limit) < 10:
-        reset_time = int(request.headers["X-RateLimit-Reset"])
-        while reset_time - int(time.time()) > 0:
-            time.sleep(60)
-            print(
-                "Waiting until rate limit reset",
-                reset_time - int(time.time()),
-                "seconds remaining",
-            )
-
-    if request.status_code == 200:
-        if "data" not in request.json():
-            print(request.json())
-            sys.exit(1)
-        return request.json()["data"]
-    elif retry:
-        return run_graphql_query(query, variables, token, False)
-    else:
-        raise Exception(
-            "Failed to run graphql query\nquery: {}\nerror: {}".format(
-                query, request.json()
-            )
-        )
-
-
-def check_manual_requests(start_date: datetime.datetime, token: str) -> list[str]:
+def check_manual_requests(
+    gh: github.Github, start_date: datetime.datetime
+) -> list[str]:
     """
     Return a list of users who have been asked since ``start_date`` if they
     want to keep their commit access.
@@ -137,10 +89,13 @@ def check_manual_requests(start_date: datetime.datetime, token: str) -> list[str
         """
     formatted_start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S")
     variables = {
-        "query": f"type:issue created:>{formatted_start_date} org:llvm repo:llvm-project label:infrastructure:commit-access"
+        "query": f"type:issue created:>{formatted_start_date} org:llvm repo:llvm-project label:infra:commit-access"
     }
 
-    data = run_graphql_query(query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=query, variables=variables
+    )
+    data = res_data["data"]
     users = []
     for issue in data["search"]["nodes"]:
         users.extend([user[1:] for user in re.findall("@[^ ,\n]+", issue["body"])])
@@ -148,7 +103,7 @@ def check_manual_requests(start_date: datetime.datetime, token: str) -> list[str
     return users
 
 
-def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int:
+def get_num_commits(gh: github.Github, user: str, start_date: datetime.datetime) -> int:
     """
     Get number of commits that ``user`` has been made since ``start_date`.
     """
@@ -166,7 +121,10 @@ def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int
         }
     """
 
-    data = run_graphql_query(user_query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=user_query, variables=variables
+    )
+    data = res_data["data"]
     variables["user_id"] = data["user"]["id"]
 
     query = """
@@ -193,7 +151,10 @@ def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int
         }
      """
     count = 0
-    data = run_graphql_query(query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=query, variables=variables
+    )
+    data = res_data["data"]
     for repo in data["organization"]["teams"]["nodes"][0]["repositories"]["nodes"]:
         count += int(repo["ref"]["target"]["history"]["totalCount"])
         if count >= User.THRESHOLD:
@@ -202,7 +163,7 @@ def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int
 
 
 def is_new_committer_query_repo(
-    user: str, start_date: datetime.datetime, token: str
+    gh: github.Github, user: str, start_date: datetime.datetime
 ) -> bool:
     """
     Determine if ``user`` is a new committer.  A new committer can keep their
@@ -220,7 +181,10 @@ def is_new_committer_query_repo(
         }
     """
 
-    data = run_graphql_query(user_query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=user_query, variables=variables
+    )
+    data = res_data["data"]
     variables["owner"] = "llvm"
     variables["user_id"] = data["user"]["id"]
     variables["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%S")
@@ -245,7 +209,10 @@ def is_new_committer_query_repo(
         }
      """
 
-    data = run_graphql_query(query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=query, variables=variables
+    )
+    data = res_data["data"]
     repo = data["organization"]["repository"]
     commits = repo["ref"]["target"]["history"]["nodes"]
     if len(commits) == 0:
@@ -256,18 +223,22 @@ def is_new_committer_query_repo(
     return True
 
 
-def is_new_committer(user: str, start_date: datetime.datetime, token: str) -> bool:
+def is_new_committer(
+    gh: github.Github, user: str, start_date: datetime.datetime
+) -> bool:
     """
     Wrapper around is_new_commiter_query_repo to handle exceptions.
     """
     try:
-        return is_new_committer_query_repo(user, start_date, token)
+        return is_new_committer_query_repo(gh, user, start_date)
     except:
         pass
     return True
 
 
-def get_review_count(user: str, start_date: datetime.datetime, token: str) -> int:
+def get_review_count(
+    gh: github.Github, user: str, start_date: datetime.datetime
+) -> int:
     """
     Return the number of reviews that ``user`` has done since ``start_date``.
     """
@@ -286,11 +257,14 @@ def get_review_count(user: str, start_date: datetime.datetime, token: str) -> in
         "query": f"type:pr commenter:{user} -author:{user} merged:>{formatted_start_date} org:llvm",
     }
 
-    data = run_graphql_query(query, variables, token)
+    res_header, res_data = gh._Github__requester.graphql_query(
+        query=query, variables=variables
+    )
+    data = res_data["data"]
     return int(data["search"]["issueCount"])
 
 
-def count_prs(triage_list: dict, start_date: datetime.datetime, token: str):
+def count_prs(gh: github.Github, triage_list: dict, start_date: datetime.datetime):
     """
     Fetch all the merged PRs for the project since ``start_date`` and update
     ``triage_list`` with the number of PRs merged for each user.
@@ -329,7 +303,10 @@ def count_prs(triage_list: dict, start_date: datetime.datetime, token: str):
         has_next_page = True
         while has_next_page:
             print(variables)
-            data = run_graphql_query(query, variables, token)
+            res_header, res_data = gh._Github__requester.graphql_query(
+                query=query, variables=variables
+            )
+            data = res_data["data"]
             for pr in data["search"]["nodes"]:
                 # Users can be None if the user has been deleted.
                 if not pr["author"]:
@@ -365,14 +342,14 @@ def main():
 
     print("Start:", len(triage_list), "triagers")
     # Step 0 Check if users have requested commit access in the last year.
-    for user in check_manual_requests(one_year_ago, token):
+    for user in check_manual_requests(gh, one_year_ago):
         if user in triage_list:
             print(user, "requested commit access in the last year.")
             del triage_list[user]
     print("After Request Check:", len(triage_list), "triagers")
 
     # Step 1 count all PRs authored or merged
-    count_prs(triage_list, one_year_ago, token)
+    count_prs(gh, triage_list, one_year_ago)
 
     print("After PRs:", len(triage_list), "triagers")
 
@@ -381,7 +358,7 @@ def main():
 
     # Step 2 check for reviews
     for user in list(triage_list.keys()):
-        review_count = get_review_count(user, one_year_ago, token)
+        review_count = get_review_count(gh, user, one_year_ago)
         triage_list[user].add_reviewed(review_count)
 
     print("After Reviews:", len(triage_list), "triagers")
@@ -391,7 +368,7 @@ def main():
 
     # Step 3 check for number of commits
     for user in list(triage_list.keys()):
-        num_commits = get_num_commits(user, one_year_ago, token)
+        num_commits = get_num_commits(gh, user, one_year_ago)
         # Override the total number of commits to not double count commits and
         # authored PRs.
         triage_list[user].set_authored(num_commits)
@@ -401,7 +378,7 @@ def main():
     # Step 4 check for new committers
     for user in list(triage_list.keys()):
         print("Checking", user)
-        if is_new_committer(user, one_year_ago, token):
+        if is_new_committer(gh, user, one_year_ago):
             print("Removing new committer: ", user)
             del triage_list[user]
 
diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile
new file mode 100644
index 0000000000000..32a809ee268ea
--- /dev/null
+++ b/.github/workflows/containers/github-action-ci/Dockerfile
@@ -0,0 +1,62 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM base as stage1-toolchain
+ENV LLVM_VERSION=19.1.2
+
+RUN apt-get update && \
+    apt-get install -y \
+    wget \
+    gcc \
+    g++ \
+    cmake \
+    ninja-build \
+    python3 \
+    git \
+    curl
+
+RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && tar -xf llvmorg-$LLVM_VERSION.tar.gz
+
+WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
+
+# Patch to enable better PGO profile data.
+# TODO: Remove this for llvm 20
+ADD https://github.com/llvm/llvm-project/commit/738250989ce516f02f809bdfde474a039c77e81f.patch .
+
+RUN patch -p1 < 738250989ce516f02f809bdfde474a039c77e81f.patch
+
+RUN cmake -B ./build -G Ninja ./llvm \
+  -C ./clang/cmake/caches/BOLT-PGO.cmake \
+  -DBOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DPGO_INSTRUMENT_LTO=Thin \
+  -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
+  -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
+  -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
+  -DCLANG_DEFAULT_LINKER="lld"
+
+RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution
+
+FROM base
+    
+COPY --from=stage1-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+    
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+# Need git for SPIRV-Tools tests.
+RUN apt-get update && \
+    apt-get install -y \
+    binutils \ 
+    cmake \
+    curl \
+    git \
+    libstdc++-11-dev \
+    ninja-build \
+    nodejs \
+    perl-modules \
+    python3-psutil
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
diff --git a/.github/workflows/containers/github-action-ci/bootstrap.patch b/.github/workflows/containers/github-action-ci/bootstrap.patch
deleted file mode 100644
index 55631c54a396f..0000000000000
--- a/.github/workflows/containers/github-action-ci/bootstrap.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/clang/cmake/caches/BOLT-PGO.cmake b/clang/cmake/caches/BOLT-PGO.cmake
-index 1a04ca9a74e5..d092820e4115 100644
---- a/clang/cmake/caches/BOLT-PGO.cmake
-+++ b/clang/cmake/caches/BOLT-PGO.cmake
-@@ -4,6 +4,8 @@ set(CLANG_BOOTSTRAP_TARGETS
-   stage2-clang-bolt
-   stage2-distribution
-   stage2-install-distribution
-+  clang
-+  lld
-   CACHE STRING "")
- set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
-   clang-bolt
diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile
deleted file mode 100644
index 3e2c1ab11d58b..0000000000000
--- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile
+++ /dev/null
@@ -1,42 +0,0 @@
-FROM docker.io/library/ubuntu:22.04 as base
-ENV LLVM_SYSROOT=/opt/llvm
-
-FROM base as stage1-toolchain
-ENV LLVM_VERSION=19.1.2
-
-RUN apt-get update && \
-    apt-get install -y \
-    wget \
-    gcc \
-    g++ \
-    cmake \
-    ninja-build \
-    python3 \
-    git \
-    curl
-
-RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && tar -xf llvmorg-$LLVM_VERSION.tar.gz
-
-WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
-
-COPY bootstrap.patch /
-
-# TODO(boomanaiden154): Remove the bootstrap patch once we unsplit the build
-# and no longer need to explicitly build the stage2 dependencies.
-RUN cat /bootstrap.patch | patch -p1
-
-RUN mkdir build
-
-RUN cmake -B ./build -G Ninja ./llvm \
-  -C ./clang/cmake/caches/BOLT-PGO.cmake \
-  -DBOOTSTRAP_LLVM_ENABLE_LLD=ON \
-  -DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON \
-  -DPGO_INSTRUMENT_LTO=Thin \
-  -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
-  -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
-  -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
-  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
-  -DCLANG_DEFAULT_LINKER="lld" \
-  -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
-
-RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld
diff --git a/.github/workflows/containers/github-action-ci/stage2.Dockerfile b/.github/workflows/containers/github-action-ci/stage2.Dockerfile
deleted file mode 100644
index 0ca0da87734c4..0000000000000
--- a/.github/workflows/containers/github-action-ci/stage2.Dockerfile
+++ /dev/null
@@ -1,29 +0,0 @@
-FROM docker.io/library/ubuntu:22.04 as base
-ENV LLVM_SYSROOT=/opt/llvm
-
-FROM stage1-toolchain AS stage2-toolchain
-
-RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
-
-FROM base
-
-COPY --from=stage2-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
-
-# Need to install curl for hendrikmuhs/ccache-action
-# Need nodejs for some of the GitHub actions.
-# Need perl-modules for clang analyzer tests.
-# Need git for SPIRV-Tools tests.
-RUN apt-get update && \
-    apt-get install -y \
-    binutils \
-    cmake \
-    curl \
-    git \
-    libstdc++-11-dev \
-    ninja-build \
-    nodejs \
-    perl-modules \
-    python3-psutil
-
-ENV LLVM_SYSROOT=$LLVM_SYSROOT
-ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
diff --git a/.github/workflows/containers/github-action-ci/storage.conf b/.github/workflows/containers/github-action-ci/storage.conf
deleted file mode 100644
index 60f295ff1e969..0000000000000
--- a/.github/workflows/containers/github-action-ci/storage.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-[storage]
-  driver = "overlay"
-  runroot = "/mnt/podman/container"
-  graphroot = "/mnt/podman/image"
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 54f82bd4e4c0b..3b9fca21d63cf 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -112,58 +112,87 @@ jobs:
           sudo apt-get update
           # swig and graphviz are lldb specific dependencies
           sudo apt-get install -y cmake ninja-build swig graphviz libhwloc-dev
+      - name: Setup output folder
+        run: mkdir built-docs
       - name: Build LLVM docs
         if: steps.docs-changed-subprojects.outputs.llvm_any_changed == 'true'
         run: |
           cmake -B llvm-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C llvm-build docs-llvm-html docs-llvm-man
+          mkdir built-docs/llvm
+          cp -r llvm-build/docs/* built-docs/llvm/
       - name: Build Clang docs
         if: steps.docs-changed-subprojects.outputs.clang_any_changed == 'true'
         run: |
           cmake -B clang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C clang-build docs-clang-html docs-clang-man
+          mkdir built-docs/clang
+          cp -r clang-build/docs/* built-docs/clang/
       - name: Build clang-tools-extra docs
         if: steps.docs-changed-subprojects.outputs.clang-tools-extra_any_changed == 'true'
         run: |
           cmake -B clang-tools-extra-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C clang-tools-extra-build docs-clang-tools-html docs-clang-tools-man
+          mkdir built-docs/clang-tools-extra
+          cp -r clang-tools-extra-build/docs/* built-docs/clang-tools-extra/
       - name: Build LLDB docs
         if: steps.docs-changed-subprojects.outputs.lldb_any_changed == 'true'
         run: |
           cmake -B lldb-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lldb" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C lldb-build docs-lldb-html docs-lldb-man
+          mkdir built-docs/lldb
+          cp -r lldb-build/docs/* built-docs/lldb/
       - name: Build libunwind docs
         if: steps.docs-changed-subprojects.outputs.libunwind_any_changed == 'true'
         run: |
           cmake -B libunwind-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libunwind" -DLLVM_ENABLE_SPHINX=ON ./runtimes
           TZ=UTC ninja -C libunwind-build docs-libunwind-html
+          mkdir built-docs/libunwind
+          cp -r libunwind-build/libunwind/docs/* built-docs/libunwind
       - name: Build libcxx docs
         if: steps.docs-changed-subprojects.outputs.libcxx_any_changed == 'true'
         run: |
           cmake -B libcxx-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libcxxabi;libcxx;libunwind" -DLLVM_ENABLE_SPHINX=ON ./runtimes
           TZ=UTC ninja -C libcxx-build docs-libcxx-html
+          mkdir built-docs/libcxx
+          cp -r libcxx-build/libcxx/docs/* built-docs/libcxx/
       - name: Build libc docs
         if: steps.docs-changed-subprojects.outputs.libc_any_changed == 'true'
         run: |
           cmake -B libc-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libc" -DLLVM_ENABLE_SPHINX=ON ./runtimes
           TZ=UTC ninja -C libc-build docs-libc-html
+          mkdir built-docs/libc
+          cp -r libc-build/libc/docs/* built-docs/libc/
       - name: Build LLD docs
         if: steps.docs-changed-subprojects.outputs.lld_any_changed == 'true'
         run: |
           cmake -B lld-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="lld" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C lld-build docs-lld-html
+          mkdir built-docs/lld
+          cp -r lld-build/docs/* built-docs/lld/
       - name: Build OpenMP docs
         if: steps.docs-changed-subprojects.outputs.openmp_any_changed == 'true'
         run: |
           cmake -B openmp-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;openmp" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C openmp-build docs-openmp-html
+          mkdir built-docs/openmp
+          cp -r openmp-build/docs/* built-docs/openmp/
       - name: Build Polly docs
         if: steps.docs-changed-subprojects.outputs.polly_any_changed == 'true'
         run: |
           cmake -B polly-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="polly" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C polly-build docs-polly-html docs-polly-man
+          mkdir built-docs/polly
+          cp -r polly-build/docs/* built-docs/polly/
       - name: Build Flang docs
         if: steps.docs-changed-subprojects.outputs.flang_any_changed == 'true'
         run: |
           cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C flang-build docs-flang-html
+          mkdir built-docs/flang
+          cp -r flang-build/docs/* built-docs/flang/
+      - name: Upload docs
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-output
+          path: built-docs/
diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml
new file mode 100644
index 0000000000000..58e15ce29546e
--- /dev/null
+++ b/.github/workflows/libc-fullbuild-tests.yml
@@ -0,0 +1,89 @@
+# This workflow is for pre-commit testing of the LLVM-libc project.
+name: LLVM-libc Pre-commit Fullbuild Tests
+permissions:
+  contents: read
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'libc/**'
+      - '.github/workflows/libc-fullbuild-tests.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - c_compiler: clang
+            cpp_compiler: clang++
+          # TODO: add back gcc build when it is fixed
+          # - c_compiler: gcc
+          #   cpp_compiler: g++
+    steps:
+    - uses: actions/checkout@v4
+    
+    # Libc's build is relatively small comparing with other components of LLVM.
+    # A fresh fullbuild takes about 190MiB of uncompressed disk space, which can
+    # be compressed into ~40MiB. Limiting the cache size to 1G should be enough.
+    # Prefer sccache as it is more modern.
+    # Do not use direct GHAC access even though it is supported by sccache. GHAC rejects
+    # frequent small object writes.
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1.2
+      with:
+        max-size: 1G
+        key: libc_fullbuild_${{ matrix.c_compiler }}
+        variant: sccache
+    
+    # Notice:
+    # - MPFR is required by some of the mathlib tests.
+    # - Debian has a multilib setup, so we need to symlink the asm directory.
+    #   For more information, see https://wiki.debian.org/Multiarch/LibraryPathOverview
+    - name: Prepare dependencies (Ubuntu)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libmpfr-dev libgmp-dev libmpc-dev ninja-build linux-libc-dev
+        sudo ln -sf /usr/include/$(uname -p)-linux-gnu/asm /usr/include/asm
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "build-output-dir=${{ github.workspace }}/build" >> "$GITHUB_OUTPUT"
+        echo "build-install-dir=${{ github.workspace }}/install" >> "$GITHUB_OUTPUT"
+    
+    # Configure libc fullbuild with scudo.
+    # Use MinSizeRel to reduce the size of the build.
+    - name: Configure CMake
+      run: >
+        cmake -B ${{ steps.strings.outputs.build-output-dir }}
+        -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
+        -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
+        -DCMAKE_BUILD_TYPE=MinSizeRel
+        -DCMAKE_C_COMPILER_LAUNCHER=sccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+        -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }}
+        -DLLVM_ENABLE_RUNTIMES="libc;compiler-rt"
+        -DLLVM_LIBC_FULL_BUILD=ON
+        -DLLVM_LIBC_INCLUDE_SCUDO=ON
+        -DCOMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC=ON
+        -DCOMPILER_RT_BUILD_GWP_ASAN=OFF
+        -DCOMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED=OFF
+        -G Ninja
+        -S ${{ github.workspace }}/runtimes
+
+    - name: Build
+      run: >
+        cmake 
+        --build ${{ steps.strings.outputs.build-output-dir }} 
+        --parallel
+        --target install
+
+    - name: Test
+      run: >
+        cmake 
+        --build ${{ steps.strings.outputs.build-output-dir }} 
+        --parallel
+        --target check-libc
diff --git a/.github/workflows/libc-overlay-tests.yml b/.github/workflows/libc-overlay-tests.yml
new file mode 100644
index 0000000000000..8b59d76aed4a8
--- /dev/null
+++ b/.github/workflows/libc-overlay-tests.yml
@@ -0,0 +1,106 @@
+# This workflow is for pre-commit testing of the LLVM-libc project.
+name: LLVM-libc Pre-commit Overlay Tests
+permissions:
+  contents: read
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'libc/**'
+      - '.github/workflows/libc-overlay-tests.yml'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations.
+      fail-fast: false
+      matrix:
+        include:
+          # TODO: add linux gcc when it is fixed
+          - os: ubuntu-24.04
+            compiler:
+              c_compiler: clang
+              cpp_compiler: clang++
+          - os: windows-2022
+            compiler:
+              c_compiler: clang-cl
+              cpp_compiler: clang-cl
+          - os: macos-14
+            compiler:
+              c_compiler: clang
+              cpp_compiler: clang++
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    # Libc's build is relatively small comparing with other components of LLVM.
+    # A fresh linux overlay takes about 180MiB of uncompressed disk space, which can
+    # be compressed into ~40MiB. MacOS and Windows overlay builds are less than 10MiB
+    # after compression. Limiting the cache size to 1G should be enough.
+    # Prefer sccache as it is modern and it has a guarantee to work with MSVC.
+    # Do not use direct GHAC access even though it is supported by sccache. GHAC rejects
+    # frequent small object writes.
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1
+      with:
+        max-size: 1G
+        key: libc_overlay_build_${{ matrix.os }}_${{ matrix.compiler.c_compiler }}
+        variant: sccache
+    
+    # MPFR is required by some of the mathlib tests.
+    - name: Prepare dependencies (Ubuntu)
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libmpfr-dev libgmp-dev libmpc-dev ninja-build
+    
+    # Chocolatey is shipped with Windows runners. Windows Server 2025 recommends WinGet.
+    # Consider migrating to WinGet when Windows Server 2025 is available.
+    - name: Prepare dependencies (Windows)
+      if: runner.os == 'Windows'
+      run: |
+        choco install ninja
+    
+    - name: Prepare dependencies (macOS)
+      if: runner.os == 'macOS'
+      run: |
+        brew install ninja
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "build-output-dir=${{ github.workspace }}/build" >> "$GITHUB_OUTPUT"
+
+    # Use MinSizeRel to reduce the size of the build.
+    # Notice that CMP0141=NEW and MSVC_DEBUG_INFORMATION_FORMAT=Embedded are required
+    # by the sccache tool.
+    - name: Configure CMake
+      run: >
+        cmake -B ${{ steps.strings.outputs.build-output-dir }}
+        -DCMAKE_CXX_COMPILER=${{ matrix.compiler.cpp_compiler }}
+        -DCMAKE_C_COMPILER=${{ matrix.compiler.c_compiler }}
+        -DCMAKE_BUILD_TYPE=MinSizeRel
+        -DCMAKE_C_COMPILER_LAUNCHER=sccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+        -DCMAKE_POLICY_DEFAULT_CMP0141=NEW
+        -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded
+        -DLLVM_ENABLE_RUNTIMES=libc
+        -G Ninja
+        -S ${{ github.workspace }}/runtimes
+
+    - name: Build
+      run: >
+        cmake 
+        --build ${{ steps.strings.outputs.build-output-dir }} 
+        --parallel 
+        --config MinSizeRel 
+        --target libc
+
+    - name: Test
+      run: >
+        cmake 
+        --build ${{ steps.strings.outputs.build-output-dir }} 
+        --parallel 
+        --target check-libc
diff --git a/.github/workflows/libcxx-restart-preempted-jobs.yaml b/.github/workflows/libcxx-restart-preempted-jobs.yaml
index 21879ce19c27c..82d84c01c92af 100644
--- a/.github/workflows/libcxx-restart-preempted-jobs.yaml
+++ b/.github/workflows/libcxx-restart-preempted-jobs.yaml
@@ -130,3 +130,91 @@ jobs:
                 run_id: context.payload.workflow_run.id
               })
             await create_check_run('success', 'Restarted workflow run due to preempted job')
+
+  restart-test:
+    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled') && github.event.actor.login == 'ldionne' # TESTING ONLY
+    name: "Restart Job (test)"
+    permissions:
+      statuses: read
+      checks: write
+      actions: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Restart Job (test)"
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+        with:
+          script: |
+            const FAILURE_REGEX = /Process completed with exit code 1./
+            const PREEMPTION_REGEX = /(The runner has received a shutdown signal)|(The operation was canceled)/
+
+            function log(msg) {
+              core.notice(msg)
+            }
+
+            const wf_run = context.payload.workflow_run
+            log(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
+
+            log('Listing check runs for suite')
+            const check_suites = await github.rest.checks.listForSuite({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              check_suite_id: context.payload.workflow_run.check_suite_id,
+              per_page: 100 // FIXME: We don't have 100 check runs yet, but we should handle this better.
+            })
+
+            preemptions = [];
+            legitimate_failures = [];
+            for (check_run of check_suites.data.check_runs) {
+              log(`Checking check run: ${check_run.id}`);
+              if (check_run.status != 'completed') {
+                log('Check run was not completed. Skipping.');
+                continue;
+              }
+
+              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
+                log(`Check run had conclusion: ${check_run.conclusion}. Skipping.`);
+                continue;
+              }
+
+              annotations = await github.rest.checks.listAnnotations({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                check_run_id: check_run.id
+              })
+
+              preemption_annotation = annotations.data.find(function(annotation) {
+                return annotation.annotation_level == 'failure' &&
+                       annotation.message.match(PREEMPTION_REGEX) != null;
+              });
+              if (preemption_annotation != null) {
+                log(`Found preemption message: ${preemption_annotation.message}`);
+                preemptions.push(check_run);
+                break;
+              }
+
+              failure_annotation = annotations.data.find(function(annotation) {
+                return annotation.annotation_level == 'failure' &&
+                       annotation.message.match(FAILURE_REGEX) != null;
+              });
+              if (failure_annotation != null) {
+                log(`Found legitimate failure annotation: ${failure_annotation.message}`);
+                legitimate_failures.push(check_run);
+                break;
+              }
+            }
+
+            if (preemptions) {
+              log('Found some preempted jobs');
+              if (legitimate_failures) {
+                log('Also found some legitimate failures, so not restarting the workflow.');
+              } else {
+                log('Did not find any legitimate failures. Restarting workflow.');
+                await github.rest.actions.reRunWorkflowFailedJobs({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  run_id: context.payload.workflow_run.id
+                })
+              }
+            } else {
+              log('Did not find any preempted jobs. Not restarting the workflow.');
+            }
diff --git a/.gitignore b/.gitignore
index d558f17a7eec3..08457aba4869e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,6 +63,8 @@ autoconf/autom4te.cache
 # VS2017 and VSCode config files.
 .vscode
 .vs
+#zed config files
+.zed
 # pythonenv for github Codespaces
 pythonenv*
 # clangd index. (".clangd" is a config file now, thus trailing slash)
diff --git a/bolt/CODE_OWNERS.TXT b/bolt/Maintainers.txt
similarity index 100%
rename from bolt/CODE_OWNERS.TXT
rename to bolt/Maintainers.txt
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index c9b0e103ed514..115e59ca0697e 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -28,6 +28,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
@@ -276,11 +277,10 @@ class BinaryContext {
   void deregisterSectionName(const BinarySection &Section);
 
 public:
-  static Expected<std::unique_ptr<BinaryContext>>
-  createBinaryContext(Triple TheTriple, StringRef InputFileName,
-                      SubtargetFeatures *Features, bool IsPIC,
-                      std::unique_ptr<DWARFContext> DwCtx,
-                      JournalingStreams Logger);
+  static Expected<std::unique_ptr<BinaryContext>> createBinaryContext(
+      Triple TheTriple, std::shared_ptr<orc::SymbolStringPool> SSP,
+      StringRef InputFileName, SubtargetFeatures *Features, bool IsPIC,
+      std::unique_ptr<DWARFContext> DwCtx, JournalingStreams Logger);
 
   /// Superset of compiler units that will contain overwritten code that needs
   /// new debug info. In a few cases, functions may end up not being
@@ -372,6 +372,7 @@ class BinaryContext {
   bool hasSymbolsWithFileName() const { return HasSymbolsWithFileName; }
   void setHasSymbolsWithFileName(bool Value) { HasSymbolsWithFileName = Value; }
 
+  std::shared_ptr<orc::SymbolStringPool> getSymbolStringPool() { return SSP; }
   /// Return true if relocations against symbol with a given name
   /// must be created.
   bool forceSymbolRelocations(StringRef SymbolName) const;
@@ -631,6 +632,8 @@ class BinaryContext {
 
   std::unique_ptr<Triple> TheTriple;
 
+  std::shared_ptr<orc::SymbolStringPool> SSP;
+
   const Target *TheTarget;
 
   std::string TripleName;
@@ -807,8 +810,10 @@ class BinaryContext {
 
   BinaryContext(std::unique_ptr<MCContext> Ctx,
                 std::unique_ptr<DWARFContext> DwCtx,
-                std::unique_ptr<Triple> TheTriple, const Target *TheTarget,
-                std::string TripleName, std::unique_ptr<MCCodeEmitter> MCE,
+                std::unique_ptr<Triple> TheTriple,
+                std::shared_ptr<orc::SymbolStringPool> SSP,
+                const Target *TheTarget, std::string TripleName,
+                std::unique_ptr<MCCodeEmitter> MCE,
                 std::unique_ptr<MCObjectFileInfo> MOFI,
                 std::unique_ptr<const MCAsmInfo> AsmInfo,
                 std::unique_ptr<const MCInstrInfo> MII,
diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index 2880bfd03be78..320623cfa15af 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -170,6 +170,9 @@ class DataAggregator : public DataReader {
   std::string BuildIDBinaryName;
 
   /// Memory map info for a single file as recorded in perf.data
+  /// When a binary has multiple text segments, the Size is computed as the
+  /// difference of the last address of these segments from the BaseAddress.
+  /// The base addresses of all text segments must be the same.
   struct MMapInfo {
     uint64_t BaseAddress{0}; /// Base address of the mapped binary.
     uint64_t MMapAddress{0}; /// Address of the executable segment.
@@ -493,6 +496,11 @@ class DataAggregator : public DataReader {
   /// and return a file name matching a given \p FileBuildID.
   std::optional<StringRef> getFileNameForBuildID(StringRef FileBuildID);
 
+  /// Get a constant reference to the parsed binary mmap entries.
+  const std::unordered_map<uint64_t, MMapInfo> &getBinaryMMapInfo() {
+    return BinaryMMapInfo;
+  }
+
   friend class YAMLProfileWriter;
 };
 } // namespace bolt
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index a808ece12da19..ac96b836ed579 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -123,6 +123,7 @@ void BinaryContext::logBOLTErrorsAndQuitOnFatal(Error E) {
 BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
                              std::unique_ptr<DWARFContext> DwCtx,
                              std::unique_ptr<Triple> TheTriple,
+                             std::shared_ptr<orc::SymbolStringPool> SSP,
                              const Target *TheTarget, std::string TripleName,
                              std::unique_ptr<MCCodeEmitter> MCE,
                              std::unique_ptr<MCObjectFileInfo> MOFI,
@@ -136,12 +137,12 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
                              std::unique_ptr<MCDisassembler> DisAsm,
                              JournalingStreams Logger)
     : Ctx(std::move(Ctx)), DwCtx(std::move(DwCtx)),
-      TheTriple(std::move(TheTriple)), TheTarget(TheTarget),
-      TripleName(TripleName), MCE(std::move(MCE)), MOFI(std::move(MOFI)),
-      AsmInfo(std::move(AsmInfo)), MII(std::move(MII)), STI(std::move(STI)),
-      InstPrinter(std::move(InstPrinter)), MIA(std::move(MIA)),
-      MIB(std::move(MIB)), MRI(std::move(MRI)), DisAsm(std::move(DisAsm)),
-      Logger(Logger), InitialDynoStats(isAArch64()) {
+      TheTriple(std::move(TheTriple)), SSP(std::move(SSP)),
+      TheTarget(TheTarget), TripleName(TripleName), MCE(std::move(MCE)),
+      MOFI(std::move(MOFI)), AsmInfo(std::move(AsmInfo)), MII(std::move(MII)),
+      STI(std::move(STI)), InstPrinter(std::move(InstPrinter)),
+      MIA(std::move(MIA)), MIB(std::move(MIB)), MRI(std::move(MRI)),
+      DisAsm(std::move(DisAsm)), Logger(Logger), InitialDynoStats(isAArch64()) {
   RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
   PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
 }
@@ -159,8 +160,9 @@ BinaryContext::~BinaryContext() {
 /// Create BinaryContext for a given architecture \p ArchName and
 /// triple \p TripleName.
 Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
-    Triple TheTriple, StringRef InputFileName, SubtargetFeatures *Features,
-    bool IsPIC, std::unique_ptr<DWARFContext> DwCtx, JournalingStreams Logger) {
+    Triple TheTriple, std::shared_ptr<orc::SymbolStringPool> SSP,
+    StringRef InputFileName, SubtargetFeatures *Features, bool IsPIC,
+    std::unique_ptr<DWARFContext> DwCtx, JournalingStreams Logger) {
   StringRef ArchName = "";
   std::string FeaturesStr = "";
   switch (TheTriple.getArch()) {
@@ -283,8 +285,8 @@ Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
 
   auto BC = std::make_unique<BinaryContext>(
       std::move(Ctx), std::move(DwCtx), std::make_unique<Triple>(TheTriple),
-      TheTarget, std::string(TripleName), std::move(MCE), std::move(MOFI),
-      std::move(AsmInfo), std::move(MII), std::move(STI),
+      std::move(SSP), TheTarget, std::string(TripleName), std::move(MCE),
+      std::move(MOFI), std::move(AsmInfo), std::move(MII), std::move(STI),
       std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI),
       std::move(DisAsm), Logger);
 
diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp
index 640b29ec36d5c..280c7c505eeda 100644
--- a/bolt/lib/Core/DebugNames.cpp
+++ b/bolt/lib/Core/DebugNames.cpp
@@ -161,6 +161,7 @@ bool static canProcess(const DWARFUnit &Unit, const DIE &Die,
   case dwarf::DW_TAG_structure_type:
   case dwarf::DW_TAG_typedef:
   case dwarf::DW_TAG_unspecified_type:
+  case dwarf::DW_TAG_union_type:
     if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_name))
       return true;
     return false;
diff --git a/bolt/lib/Passes/VeneerElimination.cpp b/bolt/lib/Passes/VeneerElimination.cpp
index b386b2756a2b8..99d0ffeca8cc2 100644
--- a/bolt/lib/Passes/VeneerElimination.cpp
+++ b/bolt/lib/Passes/VeneerElimination.cpp
@@ -46,16 +46,17 @@ Error VeneerElimination::runOnFunctions(BinaryContext &BC) {
     if (BF.isIgnored())
       continue;
 
+    MCInst &FirstInstruction = *(BF.begin()->begin());
     const MCSymbol *VeneerTargetSymbol = 0;
     uint64_t TargetAddress;
-    if (BC.MIB->matchAbsLongVeneer(BF, TargetAddress)) {
+    if (BC.MIB->isTailCall(FirstInstruction)) {
+      VeneerTargetSymbol = BC.MIB->getTargetSymbol(FirstInstruction);
+    } else if (BC.MIB->matchAbsLongVeneer(BF, TargetAddress)) {
       if (BinaryFunction *TargetBF =
               BC.getBinaryFunctionAtAddress(TargetAddress))
         VeneerTargetSymbol = TargetBF->getSymbol();
-    } else {
-      MCInst &FirstInstruction = *(BF.begin()->begin());
-      if (BC.MIB->hasAnnotation(FirstInstruction, "AArch64Veneer"))
-        VeneerTargetSymbol = BC.MIB->getTargetSymbol(FirstInstruction, 1);
+    } else if (BC.MIB->hasAnnotation(FirstInstruction, "AArch64Veneer")) {
+      VeneerTargetSymbol = BC.MIB->getTargetSymbol(FirstInstruction, 1);
     }
 
     if (!VeneerTargetSymbol)
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 697cac9fbcaa0..2b02086e3e0c9 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -95,6 +95,12 @@ cl::opt<bool> ReadPreAggregated(
     "pa", cl::desc("skip perf and read data from a pre-aggregated file format"),
     cl::cat(AggregatorCategory));
 
+cl::opt<std::string>
+    ReadPerfEvents("perf-script-events",
+                   cl::desc("skip perf event collection by supplying a "
+                            "perf-script output in a textual format"),
+                   cl::ReallyHidden, cl::init(""), cl::cat(AggregatorCategory));
+
 static cl::opt<bool>
 TimeAggregator("time-aggr",
   cl::desc("time BOLT aggregator"),
@@ -167,8 +173,9 @@ void DataAggregator::findPerfExecutable() {
 void DataAggregator::start() {
   outs() << "PERF2BOLT: Starting data aggregation job for " << Filename << "\n";
 
-  // Don't launch perf for pre-aggregated files
-  if (opts::ReadPreAggregated)
+  // Don't launch perf for pre-aggregated files or when perf input is specified
+  // by the user.
+  if (opts::ReadPreAggregated || !opts::ReadPerfEvents.empty())
     return;
 
   findPerfExecutable();
@@ -464,6 +471,13 @@ void DataAggregator::filterBinaryMMapInfo() {
 
 int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
                                    PerfProcessErrorCallbackTy Callback) {
+  if (!opts::ReadPerfEvents.empty()) {
+    outs() << "PERF2BOLT: using pre-processed perf events for '" << Name
+           << "' (perf-script-events)\n";
+    ParsingBuf = opts::ReadPerfEvents;
+    return 0;
+  }
+
   std::string Error;
   outs() << "PERF2BOLT: waiting for perf " << Name
          << " collection to finish...\n";
@@ -2056,15 +2070,6 @@ std::error_code DataAggregator::parseMMapEvents() {
     if (FileMMapInfo.first == "(deleted)")
       continue;
 
-    // Consider only the first mapping of the file for any given PID
-    auto Range = GlobalMMapInfo.equal_range(FileMMapInfo.first);
-    bool PIDExists = llvm::any_of(make_range(Range), [&](const auto &MI) {
-      return MI.second.PID == FileMMapInfo.second.PID;
-    });
-
-    if (PIDExists)
-      continue;
-
     GlobalMMapInfo.insert(FileMMapInfo);
   }
 
@@ -2116,12 +2121,22 @@ std::error_code DataAggregator::parseMMapEvents() {
                << " using file offset 0x" << Twine::utohexstr(MMapInfo.Offset)
                << ". Ignoring profile data for this mapping\n";
         continue;
-      } else {
-        MMapInfo.BaseAddress = *BaseAddress;
       }
+      MMapInfo.BaseAddress = *BaseAddress;
     }
 
-    BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo));
+    // Try to add MMapInfo to the map and update its size. Large binaries may
+    // span to multiple text segments, so the mapping is inserted only on the
+    // first occurrence.
+    if (!BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo)).second)
+      assert(MMapInfo.BaseAddress == BinaryMMapInfo[MMapInfo.PID].BaseAddress &&
+             "Base address on multiple segment mappings should match");
+
+    // Update mapping size.
+    const uint64_t EndAddress = MMapInfo.MMapAddress + MMapInfo.Size;
+    const uint64_t Size = EndAddress - BinaryMMapInfo[MMapInfo.PID].BaseAddress;
+    if (Size > BinaryMMapInfo[MMapInfo.PID].Size)
+      BinaryMMapInfo[MMapInfo.PID].Size = Size;
   }
 
   if (BinaryMMapInfo.empty()) {
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 1b5ba8b49d363..308881081321a 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -1691,7 +1691,8 @@ namespace {
 std::unique_ptr<BinaryContext>
 createDwarfOnlyBC(const object::ObjectFile &File) {
   return cantFail(BinaryContext::createBinaryContext(
-      File.makeTriple(), File.getFileName(), nullptr, false,
+      File.makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+      File.getFileName(), nullptr, false,
       DWARFContext::create(File, DWARFContext::ProcessDebugRelocations::Ignore,
                            nullptr, "", WithColor::defaultErrorHandler,
                            WithColor::defaultWarningHandler),
diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp
index be8f9dd03467e..ba483ae4711df 100644
--- a/bolt/lib/Rewrite/JITLinkLinker.cpp
+++ b/bolt/lib/Rewrite/JITLinkLinker.cpp
@@ -122,7 +122,7 @@ struct JITLinkLinker::Context : jitlink::JITLinkContext {
     jitlink::AsyncLookupResult AllResults;
 
     for (const auto &Symbol : Symbols) {
-      std::string SymName = Symbol.first.str();
+      std::string SymName = (*Symbol.first).str();
       LLVM_DEBUG(dbgs() << "BOLT: looking for " << SymName << "\n");
 
       if (auto Address = Linker.lookupSymbol(SymName)) {
@@ -167,7 +167,9 @@ struct JITLinkLinker::Context : jitlink::JITLinkContext {
   Error notifyResolved(jitlink::LinkGraph &G) override {
     for (auto *Symbol : G.defined_symbols()) {
       SymbolInfo Info{Symbol->getAddress().getValue(), Symbol->getSize()};
-      Linker.Symtab.insert({Symbol->getName().str(), Info});
+      auto Name =
+          Symbol->hasName() ? (*Symbol->getName()).str() : std::string();
+      Linker.Symtab.insert({std::move(Name), Info});
     }
 
     return Error::success();
@@ -189,7 +191,7 @@ JITLinkLinker::~JITLinkLinker() { cantFail(MM->deallocate(std::move(Allocs))); }
 
 void JITLinkLinker::loadObject(MemoryBufferRef Obj,
                                SectionsMapper MapSections) {
-  auto LG = jitlink::createLinkGraphFromObject(Obj);
+  auto LG = jitlink::createLinkGraphFromObject(Obj, BC.getSymbolStringPool());
   if (auto E = LG.takeError()) {
     errs() << "BOLT-ERROR: JITLink failed: " << E << '\n';
     exit(1);
diff --git a/bolt/lib/Rewrite/MachORewriteInstance.cpp b/bolt/lib/Rewrite/MachORewriteInstance.cpp
index c328232de61a3..2f05b03290ba3 100644
--- a/bolt/lib/Rewrite/MachORewriteInstance.cpp
+++ b/bolt/lib/Rewrite/MachORewriteInstance.cpp
@@ -74,7 +74,8 @@ MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile,
   ErrorAsOutParameter EAO(&Err);
   Relocation::Arch = InputFile->makeTriple().getArch();
   auto BCOrErr = BinaryContext::createBinaryContext(
-      InputFile->makeTriple(), InputFile->getFileName(), nullptr,
+      InputFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+      InputFile->getFileName(), nullptr,
       /* IsPIC */ true, DWARFContext::create(*InputFile),
       {llvm::outs(), llvm::errs()});
   if (Error E = BCOrErr.takeError()) {
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 7059a3dd23109..76e1f0156f828 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -356,7 +356,8 @@ RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,
 
   Relocation::Arch = TheTriple.getArch();
   auto BCOrErr = BinaryContext::createBinaryContext(
-      TheTriple, File->getFileName(), Features.get(), IsPIC,
+      TheTriple, std::make_shared<orc::SymbolStringPool>(), File->getFileName(),
+      Features.get(), IsPIC,
       DWARFContext::create(*File, DWARFContext::ProcessDebugRelocations::Ignore,
                            nullptr, opts::DWPPathName,
                            WithColor::defaultErrorHandler,
@@ -2927,6 +2928,23 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");
 }
 
+static BinaryFunction *getInitFunctionIfStaticBinary(BinaryContext &BC) {
+  // Workaround for https://github.com/llvm/llvm-project/issues/100096
+  // ("[BOLT] GOT array pointer incorrectly rewritten"). In aarch64
+  // static glibc binaries, the .init section's _init function pointer can
+  // alias with a data pointer for the end of an array. GOT rewriting
+  // currently can't detect this and updates the data pointer to the
+  // moved _init, causing a runtime crash. Skipping _init on the other
+  // hand should be harmless.
+  if (!BC.IsStaticExecutable)
+    return nullptr;
+  const BinaryData *BD = BC.getBinaryDataByName("_init");
+  if (!BD || BD->getSectionName() != ".init")
+    return nullptr;
+  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: skip _init in for GOT workaround.\n");
+  return BC.getBinaryFunctionAtAddress(BD->getAddress());
+}
+
 void RewriteInstance::selectFunctionsToProcess() {
   // Extend the list of functions to process or skip from a file.
   auto populateFunctionNames = [](cl::opt<std::string> &FunctionNamesFile,
@@ -3047,6 +3065,9 @@ void RewriteInstance::selectFunctionsToProcess() {
     return true;
   };
 
+  if (BinaryFunction *Init = getInitFunctionIfStaticBinary(*BC))
+    Init->setIgnored();
+
   for (auto &BFI : BC->getBinaryFunctions()) {
     BinaryFunction &Function = BFI.second;
 
diff --git a/bolt/test/AArch64/check-init-not-moved.s b/bolt/test/AArch64/check-init-not-moved.s
new file mode 100644
index 0000000000000..ad4b80d2e60e2
--- /dev/null
+++ b/bolt/test/AArch64/check-init-not-moved.s
@@ -0,0 +1,43 @@
+# Regression test for https://github.com/llvm/llvm-project/issues/100096
+# static glibc binaries crash on startup because _init is moved and
+# shares its address with an array end pointer. The GOT rewriting can't
+# tell the two pointers apart and incorrectly updates the _array_end
+# address. Test checks that _init is not moved.
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -static -Wl,--section-start=.data=0x1000 -Wl,--section-start=.init=0x1004
+# RUN: llvm-bolt %t.exe -o %t.bolt
+# RUN: llvm-nm %t.exe | FileCheck --check-prefix=CHECK-ORIGINAL %s
+# RUN: llvm-nm %t.bolt | FileCheck --check-prefix=CHECK-BOLTED %s
+
+.section .data
+.globl _array_end
+_array_start:
+    .word 0x0
+
+_array_end:
+.section .init,"ax",@progbits
+.globl _init
+
+# Check that bolt doesn't move _init.
+#
+# CHECK-ORIGINAL: 0000000000001004 T _init
+# CHECK-BOLTED:   0000000000001004 T _init
+_init:
+    ret
+
+.section .text,"ax",@progbits
+.globl _start
+
+# Check that bolt is moving some other functions.
+#
+# CHECK-ORIGINAL:   0000000000001008 T _start
+# CHECK-BOLTED-NOT: 0000000000001008 T _start
+_start:
+    bl _init
+    adrp x0, #:got:_array_end
+    ldr x0, [x0, #:gotpage_lo15:_array_end]
+    adrp x0, #:got:_init
+    ldr x0, [x0, #:gotpage_lo15:_init]
+    ret
+
diff --git a/bolt/test/AArch64/veneer-lld-abs.s b/bolt/test/AArch64/veneer-lld-abs.s
index d10ff46e2cb01..7e6fe2d127060 100644
--- a/bolt/test/AArch64/veneer-lld-abs.s
+++ b/bolt/test/AArch64/veneer-lld-abs.s
@@ -1,5 +1,5 @@
-## Check that llvm-bolt correctly recognizes long absolute thunks generated
-## by LLD.
+## Check that llvm-bolt correctly recognizes veneers/thunks for absolute code
+## generated by LLD.
 
 # RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags -fno-PIC -no-pie %t.o -o %t.exe -nostdlib \
@@ -12,40 +12,63 @@
 
 .text
 .balign 4
-.global foo
-.type foo, %function
-foo:
-  adrp x1, foo
+.global far_function
+.type far_function, %function
+far_function:
   ret
-.size foo, .-foo
+.size far_function, .-far_function
+
+.global near_function
+.type near_function, %function
+near_function:
+  ret
+.size near_function, .-near_function
+
+## Force relocations against .text.
+.reloc 0, R_AARCH64_NONE
 
 .section ".mytext", "ax"
 .balign 4
 
-.global __AArch64AbsLongThunk_foo
-.type __AArch64AbsLongThunk_foo, %function
-__AArch64AbsLongThunk_foo:
+## This version of a thunk is always generated by LLD for function calls
+## spanning more than 256MB.
+.global __AArch64AbsLongThunk_far_function
+.type __AArch64AbsLongThunk_far_function, %function
+__AArch64AbsLongThunk_far_function:
   ldr x16, .L1
   br x16
-# CHECK-INPUT-LABEL: <__AArch64AbsLongThunk_foo>:
+# CHECK-INPUT-LABEL: <__AArch64AbsLongThunk_far_function>:
 # CHECK-INPUT-NEXT:    ldr
 # CHECK-INPUT-NEXT:    br
 .L1:
-  .quad foo
-.size __AArch64AbsLongThunk_foo, .-__AArch64AbsLongThunk_foo
+  .quad far_function
+.size __AArch64AbsLongThunk_far_function, .-__AArch64AbsLongThunk_far_function
+
+## If a callee is closer than 256MB away, LLD may generate a thunk with a direct
+## jump to the callee. Note, that the name might still include "AbsLong".
+.global __AArch64AbsLongThunk_near_function
+.type __AArch64AbsLongThunk_near_function, %function
+__AArch64AbsLongThunk_near_function:
+  b near_function
+# CHECK-INPUT-LABEL: <__AArch64AbsLongThunk_near_function>:
+# CHECK-INPUT-NEXT:    b {{.*}} <near_function>
+.size __AArch64AbsLongThunk_near_function, .-__AArch64AbsLongThunk_near_function
 
-## Check that the thunk was removed from .text and _start() calls foo()
+## Check that thunks were removed from .text, and _start calls functions
 ## directly.
 
-# CHECK-OUTPUT-NOT: __AArch64AbsLongThunk_foo
+# CHECK-OUTPUT-NOT: __AArch64AbsLongThunk_{{.*}}
 
 .global _start
 .type _start, %function
 _start:
 # CHECK-INPUT-LABEL:  <_start>:
 # CHECK-OUTPUT-LABEL: <_start>:
-  bl __AArch64AbsLongThunk_foo
-# CHECK-INPUT-NEXT:     bl {{.*}} <__AArch64AbsLongThunk_foo>
-# CHECK-OUTPUT-NEXT:    bl {{.*}} <foo>
+  bl __AArch64AbsLongThunk_far_function
+  bl __AArch64AbsLongThunk_near_function
+# CHECK-INPUT-NEXT:     bl {{.*}} <__AArch64AbsLongThunk_far_function>
+# CHECK-INPUT-NEXT:     bl {{.*}} <__AArch64AbsLongThunk_near_function>
+# CHECK-OUTPUT-NEXT:    bl {{.*}} <far_function>
+# CHECK-OUTPUT-NEXT:    bl {{.*}} <near_function>
   ret
 .size _start, .-_start
diff --git a/bolt/test/X86/dwarf5-debug-names-union.test b/bolt/test/X86/dwarf5-debug-names-union.test
new file mode 100644
index 0000000000000..c6da1db684ddc
--- /dev/null
+++ b/bolt/test/X86/dwarf5-debug-names-union.test
@@ -0,0 +1,496 @@
+# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %s   -o %tmain.o
+# RUN: %clang %cflags -gdwarf-5 %tmain.o -o %tmain.exe
+# RUN: llvm-bolt %tmain.exe -o %tmain.exe.bolt --update-debug-sections
+# RUN: llvm-dwarfdump --debug-names %tmain.exe.bolt > %tlog.txt
+# RUN: cat %tlog.txt | FileCheck -check-prefix=BOLT %s
+
+## This test checks that bolt correctly generates entry for DW_TAG_union_type for .debug_name section.
+
+# BOLT:       Abbreviations [
+# BOLT-NEXT:   Abbreviation [[ABBREV1:0x[0-9a-f]*]] {
+# BOLT-NEXT:     Tag: DW_TAG_subprogram
+# BOLT-NEXT:     DW_IDX_die_offset: DW_FORM_ref4
+# BOLT-NEXT:     DW_IDX_parent: DW_FORM_flag_present
+# BOLT-NEXT:   }
+# BOLT-NEXT:   Abbreviation [[ABBREV2:0x[0-9a-f]*]] {
+# BOLT-NEXT:     Tag: DW_TAG_base_type
+# BOLT-NEXT:     DW_IDX_die_offset: DW_FORM_ref4
+# BOLT-NEXT:     DW_IDX_parent: DW_FORM_flag_present
+# BOLT-NEXT:   }
+# BOLT-NEXT:   Abbreviation [[ABBREV3:0x[0-9a-f]*]] {
+# BOLT-NEXT:     Tag: DW_TAG_union_type
+# BOLT-NEXT:     DW_IDX_die_offset: DW_FORM_ref4
+# BOLT-NEXT:     DW_IDX_parent: DW_FORM_ref4
+# BOLT-NEXT:   }
+# BOLT-NEXT:   Abbreviation [[ABBREV4:0x[0-9a-f]*]] {
+# BOLT-NEXT:     Tag: DW_TAG_structure_type
+# BOLT-NEXT:     DW_IDX_die_offset: DW_FORM_ref4
+# BOLT-NEXT:     DW_IDX_parent: DW_FORM_ref4
+# BOLT-NEXT:   }
+# BOLT-NEXT: ]
+# BOLT-NEXT: Bucket 0 [
+# BOLT-NEXT:   EMPTY
+# BOLT-NEXT: ]
+# BOLT-NEXT: Bucket 1 [
+# BOLT-NEXT:   Name 1 {
+# BOLT-NEXT:     Hash: 0x7C9A7F6A
+# BOLT-NEXT:     String: {{.+}} "main"
+# BOLT-NEXT:     Entry @ [[ENTRY:0x[0-9a-f]*]]  {
+# BOLT-NEXT:       Abbrev: [[ABBREV1]]
+# BOLT-NEXT:       Tag: DW_TAG_subprogram
+# BOLT-NEXT:       DW_IDX_die_offset: 0x00000024
+# BOLT-NEXT:       DW_IDX_parent: <parent not indexed>
+# BOLT-NEXT:     }
+# BOLT-NEXT:   }
+# BOLT-NEXT: ]
+# BOLT-NEXT: Bucket 2 [
+# BOLT-NEXT:   EMPTY
+# BOLT-NEXT: ]
+# BOLT-NEXT: Bucket 3 [
+# BOLT-NEXT:   Name 2 {
+# BOLT-NEXT:     Hash: 0xB888030
+# BOLT-NEXT:     String: {{.+}} "int"
+# BOLT-NEXT:     Entry @ {{.+}} {
+# BOLT-NEXT:       Abbrev: [[ABBREV2]]
+# BOLT-NEXT:       Tag: DW_TAG_base_type
+# BOLT-NEXT:       DW_IDX_die_offset: 0x00000083
+# BOLT-NEXT:       DW_IDX_parent: <parent not indexed>
+# BOLT-NEXT:     }
+# BOLT-NEXT:   }
+# BOLT-NEXT:   Name 3 {
+# BOLT-NEXT:     Hash: 0xED0F01B4
+# BOLT-NEXT:     String: {{.+}} "MyUnion"
+# BOLT-NEXT:     Entry @ {{.+}} {
+# BOLT-NEXT:       Abbrev: [[ABBREV3]]
+# BOLT-NEXT:       Tag: DW_TAG_union_type
+# BOLT-NEXT:       DW_IDX_die_offset: 0x00000049
+# BOLT-NEXT:       DW_IDX_parent: Entry @ [[ENTRY]]
+# BOLT-NEXT:     }
+# BOLT-NEXT:   }
+# BOLT-NEXT: ]
+# BOLT-NEXT: Bucket 4 [
+# BOLT-NEXT:   Name 4 {
+# BOLT-NEXT:     Hash: 0x8AB681F0
+# BOLT-NEXT:     String: {{.+}} "MyStruct"
+# BOLT-NEXT:     Entry @ [[ENTRY2:0x[0-9a-f]*]] {
+# BOLT-NEXT:       Abbrev: [[ABBREV4]]
+# BOLT-NEXT:       Tag: DW_TAG_structure_type
+# BOLT-NEXT:       DW_IDX_die_offset: 0x00000062
+# BOLT-NEXT:       DW_IDX_parent: Entry @ [[ENTRY]]
+# BOLT-NEXT:     }
+# BOLT-NEXT:   }
+# BOLT-NEXT:   Name 5 {
+# BOLT-NEXT:     Hash: 0x8EEF3866
+# BOLT-NEXT:     String: {{.+}} "MyUnion2"
+# BOLT-NEXT:     Entry @ {{.+}} {
+# BOLT-NEXT:       Abbrev: [[ABBREV3]]
+# BOLT-NEXT:       Tag: DW_TAG_union_type
+# BOLT-NEXT:       DW_IDX_die_offset: 0x00000071
+# BOLT-NEXT:       DW_IDX_parent: Entry @ [[ENTRY2]]
+
+
+## int main() {
+##   union MyUnion {
+##     int a;
+##     int b;
+##   };
+##   struct MyStruct {
+##     union MyUnion2 {
+##       int a;
+##     };
+##     MyUnion2 myUnion2;
+##   };
+##   MyUnion myEnum;
+##   myEnum.a = 5;
+##   MyStruct myStruct;
+##   return myEnum.a + myStruct.myUnion2.a;
+## }
+
+	.text
+	.file	"main.cpp"
+	.globl	main                            # -- Begin function main
+	.p2align	4, 0x90
+	.type	main,@function
+main:                                   # @main
+.Lfunc_begin0:
+	.file	0 "union" "main.cpp" md5 0xb75b2512f2daa57bbcfe0c29f56d95f4
+	.loc	0 1 0                           # main.cpp:1:0
+	retq
+.Lfunc_end0:
+	.size	main, .-main
+                                        # -- End function
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                               # Abbreviation Code
+	.byte	17                              # DW_TAG_compile_unit
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	37                              # DW_AT_producer
+	.byte	37                              # DW_FORM_strx1
+	.byte	19                              # DW_AT_language
+	.byte	5                               # DW_FORM_data2
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	114                             # DW_AT_str_offsets_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	16                              # DW_AT_stmt_list
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	27                              # DW_AT_comp_dir
+	.byte	37                              # DW_FORM_strx1
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	115                             # DW_AT_addr_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	2                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	3                               # Abbreviation Code
+	.byte	52                              # DW_TAG_variable
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	4                               # Abbreviation Code
+	.byte	23                              # DW_TAG_union_type
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	54                              # DW_AT_calling_convention
+	.byte	11                              # DW_FORM_data1
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	5                               # Abbreviation Code
+	.byte	13                              # DW_TAG_member
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	56                              # DW_AT_data_member_location
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	6                               # Abbreviation Code
+	.byte	19                              # DW_TAG_structure_type
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	54                              # DW_AT_calling_convention
+	.byte	11                              # DW_FORM_data1
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	7                               # Abbreviation Code
+	.byte	36                              # DW_TAG_base_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	62                              # DW_AT_encoding
+	.byte	11                              # DW_FORM_data1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	0                               # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	5                               # DWARF version number
+	.byte	1                               # DWARF Unit Type
+	.byte	8                               # Address Size (in bytes)
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	1                               # Abbrev [1] 0xc:0x7b DW_TAG_compile_unit
+	.byte	0                               # DW_AT_producer
+	.short	33                              # DW_AT_language
+	.byte	1                               # DW_AT_name
+	.long	.Lstr_offsets_base0             # DW_AT_str_offsets_base
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.byte	2                               # DW_AT_comp_dir
+	.byte	0                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.long	.Laddr_table_base0              # DW_AT_addr_base
+	.byte	2                               # Abbrev [2] 0x23:0x5f DW_TAG_subprogram
+	.byte	0                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	86
+	.byte	3                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	1                               # DW_AT_decl_line
+	.long	130                             # DW_AT_type
+                                        # DW_AT_external
+	.byte	3                               # Abbrev [3] 0x32:0xb DW_TAG_variable
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	120
+	.byte	5                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	12                              # DW_AT_decl_line
+	.long	72                              # DW_AT_type
+	.byte	3                               # Abbrev [3] 0x3d:0xb DW_TAG_variable
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	116
+	.byte	9                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	14                              # DW_AT_decl_line
+	.long	97                              # DW_AT_type
+	.byte	4                               # Abbrev [4] 0x48:0x19 DW_TAG_union_type
+	.byte	5                               # DW_AT_calling_convention
+	.byte	8                               # DW_AT_name
+	.byte	4                               # DW_AT_byte_size
+	.byte	0                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.byte	5                               # Abbrev [5] 0x4e:0x9 DW_TAG_member
+	.byte	6                               # DW_AT_name
+	.long	130                             # DW_AT_type
+	.byte	0                               # DW_AT_decl_file
+	.byte	3                               # DW_AT_decl_line
+	.byte	0                               # DW_AT_data_member_location
+	.byte	5                               # Abbrev [5] 0x57:0x9 DW_TAG_member
+	.byte	7                               # DW_AT_name
+	.long	130                             # DW_AT_type
+	.byte	0                               # DW_AT_decl_file
+	.byte	4                               # DW_AT_decl_line
+	.byte	0                               # DW_AT_data_member_location
+	.byte	0                               # End Of Children Mark
+	.byte	6                               # Abbrev [6] 0x61:0x20 DW_TAG_structure_type
+	.byte	5                               # DW_AT_calling_convention
+	.byte	12                              # DW_AT_name
+	.byte	4                               # DW_AT_byte_size
+	.byte	0                               # DW_AT_decl_file
+	.byte	6                               # DW_AT_decl_line
+	.byte	5                               # Abbrev [5] 0x67:0x9 DW_TAG_member
+	.byte	10                              # DW_AT_name
+	.long	112                             # DW_AT_type
+	.byte	0                               # DW_AT_decl_file
+	.byte	10                              # DW_AT_decl_line
+	.byte	0                               # DW_AT_data_member_location
+	.byte	4                               # Abbrev [4] 0x70:0x10 DW_TAG_union_type
+	.byte	5                               # DW_AT_calling_convention
+	.byte	11                              # DW_AT_name
+	.byte	4                               # DW_AT_byte_size
+	.byte	0                               # DW_AT_decl_file
+	.byte	7                               # DW_AT_decl_line
+	.byte	5                               # Abbrev [5] 0x76:0x9 DW_TAG_member
+	.byte	6                               # DW_AT_name
+	.long	130                             # DW_AT_type
+	.byte	0                               # DW_AT_decl_file
+	.byte	8                               # DW_AT_decl_line
+	.byte	0                               # DW_AT_data_member_location
+	.byte	0                               # End Of Children Mark
+	.byte	0                               # End Of Children Mark
+	.byte	0                               # End Of Children Mark
+	.byte	7                               # Abbrev [7] 0x82:0x4 DW_TAG_base_type
+	.byte	4                               # DW_AT_name
+	.byte	5                               # DW_AT_encoding
+	.byte	4                               # DW_AT_byte_size
+	.byte	0                               # End Of Children Mark
+.Ldebug_info_end0:
+	.section	.debug_str_offsets,"",@progbits
+	.long	56                              # Length of String Offsets Set
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"clang version 20.0.0git"       # string offset=0
+.Linfo_string1:
+	.asciz	"main.cpp"                      # string offset=24
+.Linfo_string2:
+	.asciz	"union" # string offset=33
+.Linfo_string3:
+	.asciz	"main"                          # string offset=77
+.Linfo_string4:
+	.asciz	"int"                           # string offset=82
+.Linfo_string5:
+	.asciz	"myEnum"                        # string offset=86
+.Linfo_string6:
+	.asciz	"MyUnion"                       # string offset=93
+.Linfo_string7:
+	.asciz	"a"                             # string offset=101
+.Linfo_string8:
+	.asciz	"b"                             # string offset=103
+.Linfo_string9:
+	.asciz	"myStruct"                      # string offset=105
+.Linfo_string10:
+	.asciz	"MyStruct"                      # string offset=114
+.Linfo_string11:
+	.asciz	"myUnion2"                      # string offset=123
+.Linfo_string12:
+	.asciz	"MyUnion2"                      # string offset=132
+	.section	.debug_str_offsets,"",@progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.long	.Linfo_string3
+	.long	.Linfo_string4
+	.long	.Linfo_string5
+	.long	.Linfo_string7
+	.long	.Linfo_string8
+	.long	.Linfo_string6
+	.long	.Linfo_string9
+	.long	.Linfo_string11
+	.long	.Linfo_string12
+	.long	.Linfo_string10
+	.section	.debug_addr,"",@progbits
+	.long	.Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution
+.Ldebug_addr_start0:
+	.short	5                               # DWARF version number
+	.byte	8                               # Address size
+	.byte	0                               # Segment selector size
+.Laddr_table_base0:
+	.quad	.Lfunc_begin0
+.Ldebug_addr_end0:
+	.section	.debug_names,"",@progbits
+	.long	.Lnames_end0-.Lnames_start0     # Header: unit length
+.Lnames_start0:
+	.short	5                               # Header: version
+	.short	0                               # Header: padding
+	.long	1                               # Header: compilation unit count
+	.long	0                               # Header: local type unit count
+	.long	0                               # Header: foreign type unit count
+	.long	5                               # Header: bucket count
+	.long	5                               # Header: name count
+	.long	.Lnames_abbrev_end0-.Lnames_abbrev_start0 # Header: abbreviation table size
+	.long	8                               # Header: augmentation string size
+	.ascii	"LLVM0700"                      # Header: augmentation string
+	.long	.Lcu_begin0                     # Compilation unit 0
+	.long	0                               # Bucket 0
+	.long	1                               # Bucket 1
+	.long	0                               # Bucket 2
+	.long	2                               # Bucket 3
+	.long	4                               # Bucket 4
+	.long	2090499946                      # Hash in Bucket 1
+	.long	193495088                       # Hash in Bucket 3
+	.long	-317783628                      # Hash in Bucket 3
+	.long	-1967750672                     # Hash in Bucket 4
+	.long	-1896925082                     # Hash in Bucket 4
+	.long	.Linfo_string3                  # String in Bucket 1: main
+	.long	.Linfo_string4                  # String in Bucket 3: int
+	.long	.Linfo_string6                  # String in Bucket 3: MyUnion
+	.long	.Linfo_string10                 # String in Bucket 4: MyStruct
+	.long	.Linfo_string12                 # String in Bucket 4: MyUnion2
+	.long	.Lnames0-.Lnames_entries0       # Offset in Bucket 1
+	.long	.Lnames1-.Lnames_entries0       # Offset in Bucket 3
+	.long	.Lnames2-.Lnames_entries0       # Offset in Bucket 3
+	.long	.Lnames3-.Lnames_entries0       # Offset in Bucket 4
+	.long	.Lnames4-.Lnames_entries0       # Offset in Bucket 4
+.Lnames_abbrev_start0:
+	.byte	1                               # Abbrev code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	2                               # Abbrev code
+	.byte	36                              # DW_TAG_base_type
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	3                               # Abbrev code
+	.byte	23                              # DW_TAG_union_type
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	4                               # Abbrev code
+	.byte	19                              # DW_TAG_structure_type
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev list
+.Lnames_abbrev_end0:
+.Lnames_entries0:
+.Lnames0:
+.L3:
+	.byte	1                               # Abbreviation code
+	.long	35                              # DW_IDX_die_offset
+	.byte	0                               # DW_IDX_parent
+                                        # End of list: main
+.Lnames1:
+.L1:
+	.byte	2                               # Abbreviation code
+	.long	130                             # DW_IDX_die_offset
+	.byte	0                               # DW_IDX_parent
+                                        # End of list: int
+.Lnames2:
+.L4:
+	.byte	3                               # Abbreviation code
+	.long	72                              # DW_IDX_die_offset
+	.long	.L3-.Lnames_entries0            # DW_IDX_parent
+	.byte	0                               # End of list: MyUnion
+.Lnames3:
+.L2:
+	.byte	4                               # Abbreviation code
+	.long	97                              # DW_IDX_die_offset
+	.long	.L3-.Lnames_entries0            # DW_IDX_parent
+	.byte	0                               # End of list: MyStruct
+.Lnames4:
+.L0:
+	.byte	3                               # Abbreviation code
+	.long	112                             # DW_IDX_die_offset
+	.long	.L2-.Lnames_entries0            # DW_IDX_parent
+	.byte	0                               # End of list: MyUnion2
+	.p2align	2, 0x0
+.Lnames_end0:
+	.ident	"clang version 20.0.0git"
+	.section	".note.GNU-stack","",@progbits
+	.addrsig
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/bolt/unittests/Core/BinaryContext.cpp b/bolt/unittests/Core/BinaryContext.cpp
index 05b898d34af56..9819a8c2b777b 100644
--- a/bolt/unittests/Core/BinaryContext.cpp
+++ b/bolt/unittests/Core/BinaryContext.cpp
@@ -48,7 +48,8 @@ struct BinaryContextTester : public testing::TestWithParam<Triple::ArchType> {
   void initializeBOLT() {
     Relocation::Arch = ObjFile->makeTriple().getArch();
     BC = cantFail(BinaryContext::createBinaryContext(
-        ObjFile->makeTriple(), ObjFile->getFileName(), nullptr, true,
+        ObjFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+        ObjFile->getFileName(), nullptr, true,
         DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()}));
     ASSERT_FALSE(!BC);
   }
@@ -216,4 +217,4 @@ TEST_P(BinaryContextTester, BaseAddressSegmentsSmallerThanAlignment) {
       BC->getBaseAddressForMapping(0xaaaaaaab1000, 0x1000);
   ASSERT_TRUE(BaseAddress.has_value());
   ASSERT_EQ(*BaseAddress, 0xaaaaaaaa0000ULL);
-}
\ No newline at end of file
+}
diff --git a/bolt/unittests/Core/CMakeLists.txt b/bolt/unittests/Core/CMakeLists.txt
index bad7108dad0b7..208cf6ced7358 100644
--- a/bolt/unittests/Core/CMakeLists.txt
+++ b/bolt/unittests/Core/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_LINK_COMPONENTS
 add_bolt_unittest(CoreTests
   BinaryContext.cpp
   MCPlusBuilder.cpp
+  MemoryMaps.cpp
   DynoStats.cpp
 
   DISABLE_LLVM_LINK_LLVM_DYLIB
@@ -17,6 +18,8 @@ target_link_libraries(CoreTests
   PRIVATE
   LLVMBOLTCore
   LLVMBOLTRewrite
+  LLVMBOLTProfile
+  LLVMTestingSupport
   )
 
 foreach (tgt ${BOLT_TARGETS_TO_BUILD})
diff --git a/bolt/unittests/Core/MCPlusBuilder.cpp b/bolt/unittests/Core/MCPlusBuilder.cpp
index c66c2d0c0fb16..5488cae366284 100644
--- a/bolt/unittests/Core/MCPlusBuilder.cpp
+++ b/bolt/unittests/Core/MCPlusBuilder.cpp
@@ -58,7 +58,8 @@ struct MCPlusBuilderTester : public testing::TestWithParam<Triple::ArchType> {
   void initializeBolt() {
     Relocation::Arch = ObjFile->makeTriple().getArch();
     BC = cantFail(BinaryContext::createBinaryContext(
-        ObjFile->makeTriple(), ObjFile->getFileName(), nullptr, true,
+        ObjFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+        ObjFile->getFileName(), nullptr, true,
         DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()}));
     ASSERT_FALSE(!BC);
     BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(
@@ -90,14 +91,15 @@ INSTANTIATE_TEST_SUITE_P(AArch64, MCPlusBuilderTester,
                          ::testing::Values(Triple::aarch64));
 
 TEST_P(MCPlusBuilderTester, AliasX0) {
-  uint64_t AliasesX0[] = {AArch64::W0, AArch64::X0, AArch64::W0_W1,
+  uint64_t AliasesX0[] = {AArch64::W0,    AArch64::W0_HI,
+                          AArch64::X0,    AArch64::W0_W1,
                           AArch64::X0_X1, AArch64::X0_X1_X2_X3_X4_X5_X6_X7};
   size_t AliasesX0Count = sizeof(AliasesX0) / sizeof(*AliasesX0);
   testRegAliases(Triple::aarch64, AArch64::X0, AliasesX0, AliasesX0Count);
 }
 
 TEST_P(MCPlusBuilderTester, AliasSmallerX0) {
-  uint64_t AliasesX0[] = {AArch64::W0, AArch64::X0};
+  uint64_t AliasesX0[] = {AArch64::W0, AArch64::W0_HI, AArch64::X0};
   size_t AliasesX0Count = sizeof(AliasesX0) / sizeof(*AliasesX0);
   testRegAliases(Triple::aarch64, AArch64::X0, AliasesX0, AliasesX0Count, true);
 }
diff --git a/bolt/unittests/Core/MemoryMaps.cpp b/bolt/unittests/Core/MemoryMaps.cpp
new file mode 100644
index 0000000000000..06073d0a82e14
--- /dev/null
+++ b/bolt/unittests/Core/MemoryMaps.cpp
@@ -0,0 +1,143 @@
+//===- bolt/unittest/Core/MemoryMaps.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Core/BinaryContext.h"
+#include "bolt/Profile/DataAggregator.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace llvm::ELF;
+using namespace bolt;
+
+namespace opts {
+extern cl::opt<std::string> ReadPerfEvents;
+} // namespace opts
+
+namespace {
+
+/// Perform checks on memory map events normally captured in perf. Tests use
+/// the 'opts::ReadPerfEvents' flag to emulate these events, passing a custom
+/// 'perf script' output to DataAggregator.
+struct MemoryMapsTester : public testing::TestWithParam<Triple::ArchType> {
+  void SetUp() override {
+    initalizeLLVM();
+    prepareElf();
+    initializeBOLT();
+  }
+
+protected:
+  void initalizeLLVM() {
+    llvm::InitializeAllTargetInfos();
+    llvm::InitializeAllTargetMCs();
+    llvm::InitializeAllAsmParsers();
+    llvm::InitializeAllDisassemblers();
+    llvm::InitializeAllTargets();
+    llvm::InitializeAllAsmPrinters();
+  }
+
+  void prepareElf() {
+    memcpy(ElfBuf, "\177ELF", 4);
+    ELF64LE::Ehdr *EHdr = reinterpret_cast<typename ELF64LE::Ehdr *>(ElfBuf);
+    EHdr->e_ident[llvm::ELF::EI_CLASS] = llvm::ELF::ELFCLASS64;
+    EHdr->e_ident[llvm::ELF::EI_DATA] = llvm::ELF::ELFDATA2LSB;
+    EHdr->e_machine = GetParam() == Triple::aarch64 ? EM_AARCH64 : EM_X86_64;
+    MemoryBufferRef Source(StringRef(ElfBuf, sizeof(ElfBuf)), "ELF");
+    ObjFile = cantFail(ObjectFile::createObjectFile(Source));
+  }
+
+  void initializeBOLT() {
+    Relocation::Arch = ObjFile->makeTriple().getArch();
+    BC = cantFail(BinaryContext::createBinaryContext(
+        ObjFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+        ObjFile->getFileName(), nullptr, true,
+        DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()}));
+    ASSERT_FALSE(!BC);
+  }
+
+  char ElfBuf[sizeof(typename ELF64LE::Ehdr)] = {};
+  std::unique_ptr<ObjectFile> ObjFile;
+  std::unique_ptr<BinaryContext> BC;
+};
+} // namespace
+
+#ifdef X86_AVAILABLE
+
+INSTANTIATE_TEST_SUITE_P(X86, MemoryMapsTester,
+                         ::testing::Values(Triple::x86_64));
+
+#endif
+
+#ifdef AARCH64_AVAILABLE
+
+INSTANTIATE_TEST_SUITE_P(AArch64, MemoryMapsTester,
+                         ::testing::Values(Triple::aarch64));
+
+#endif
+
+/// Check that the correct mmap size is computed when we have multiple text
+/// segment mappings.
+TEST_P(MemoryMapsTester, ParseMultipleSegments) {
+  const int Pid = 1234;
+  StringRef Filename = "BINARY";
+  opts::ReadPerfEvents = formatv(
+      "name       0 [000]     0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
+      "[0xabc0000000(0x1000000) @ 0x11c0000 103:01 1573523 0]: r-xp {1}\n"
+      "name       0 [000]     0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
+      "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
+      Pid, Filename);
+
+  BC->SegmentMapInfo[0x11da000] =
+      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
+  BC->SegmentMapInfo[0x31d0000] =
+      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true};
+
+  DataAggregator DA("");
+  BC->setFilename(Filename);
+  Error Err = DA.preprocessProfile(*BC);
+
+  // Ignore errors from perf2bolt when parsing memory events later on.
+  ASSERT_THAT_ERROR(std::move(Err), Succeeded());
+
+  auto &BinaryMMapInfo = DA.getBinaryMMapInfo();
+  auto El = BinaryMMapInfo.find(Pid);
+  // Check that memory mapping is present and has the expected size.
+  ASSERT_NE(El, BinaryMMapInfo.end());
+  ASSERT_EQ(El->second.Size, static_cast<uint64_t>(0xb1d0000));
+}
+
+/// Check that DataAggregator aborts when pre-processing an input binary
+/// with multiple text segments that have different base addresses.
+TEST_P(MemoryMapsTester, MultipleSegmentsMismatchedBaseAddress) {
+  const int Pid = 1234;
+  StringRef Filename = "BINARY";
+  opts::ReadPerfEvents = formatv(
+      "name       0 [000]     0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
+      "[0xabc0000000(0x1000000) @ 0x11c0000 103:01 1573523 0]: r-xp {1}\n"
+      "name       0 [000]     0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
+      "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
+      Pid, Filename);
+
+  BC->SegmentMapInfo[0x11da000] =
+      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
+  // Using '0x31d0fff' FileOffset which triggers a different base address
+  // for this second text segment.
+  BC->SegmentMapInfo[0x31d0000] =
+      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true};
+
+  DataAggregator DA("");
+  BC->setFilename(Filename);
+  ASSERT_DEBUG_DEATH(
+      { Error Err = DA.preprocessProfile(*BC); },
+      "Base address on multiple segment mappings should match");
+}
diff --git a/clang-tools-extra/CODE_OWNERS.TXT b/clang-tools-extra/CODE_OWNERS.TXT
deleted file mode 100644
index 2831ec7e25f59..0000000000000
--- a/clang-tools-extra/CODE_OWNERS.TXT
+++ /dev/null
@@ -1,30 +0,0 @@
-This file is a list of the people responsible for ensuring that patches for a
-particular tool are reviewed, either by themself or by someone else. They are
-also the gatekeepers for their part of Clang, with the final word on what goes
-in or not.
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts.  The fields are: name (N), email (E), Phabricator
-handle (H), and description (D).
-
-N: Aaron Ballman
-E: aaron@aaronballman.com
-H: aaron.ballman
-D: clang-query
-
-N: Julie Hockett
-E: juliehockett@google.com
-D: clang-doc
-
-N: Nathan James
-E: n.james93@hotmail.co.uk
-H: njames93
-D: clang-tidy
-
-N: Manuel Klimek
-E: klimek@google.com
-D: all parts of clang-tools-extra not covered by someone else
-
-N: Sam McCall
-E: sammccall@google.com
-D: clangd
diff --git a/clang-tools-extra/Maintainers.txt b/clang-tools-extra/Maintainers.txt
new file mode 100644
index 0000000000000..689f2408820e9
--- /dev/null
+++ b/clang-tools-extra/Maintainers.txt
@@ -0,0 +1,75 @@
+=============================
+Clang Tools Extra Maintainers
+=============================
+
+This file is a list of the maintainers
+(https://llvm.org/docs/DeveloperPolicy.html#maintainers) for clang-tools-extra.
+
+
+Active Maintainers
+==================
+The following people are the active maintainers for the project. Please reach
+out to them for code reviews, questions about their area of expertise, or other
+assistance.
+
+Lead Maintainer
+---------------
+| Aaron Ballman
+| aaron@aaronballman.com (email), aaron.ballman (Phabricator), AaronBallman (GitHub), AaronBallman (Discourse), aaronballman (Discord), AaronBallman (IRC)
+
+
+clang-tidy
+----------
+| Congcong Cai
+| congcongcai0907@163.com (email), HerrCai0907 (GitHub), HerrCai0907 (Discourse)
+
+| Julian Schmidt
+| git.julian.schmidt@gmail.com (email), 5chmidti (GitHub), 5chmidti (Discourse), 5chmidti (Discord)
+
+| Piotr Zegar
+| me@piotrzegar.pl (email), PiotrZSL (GitHub), PiotrZSL (Discourse), PiotrZSL (Discord)
+
+
+clang-query
+-----------
+| Aaron Ballman
+| aaron@aaronballman.com (email), aaron.ballman (Phabricator), AaronBallman (GitHub), AaronBallman (Discourse), aaronballman (Discord), AaronBallman (IRC)
+
+
+clang-doc
+---------
+| Paul Kirth
+| paulkirth@google.com (email), ilovepi (GitHub), ilovepi (Discourse)
+
+| Peter Chou
+| peterchou411@gmail.com (email), PeterChou1 (GitHub), PeterChou1 (Discourse), .peterchou (Discord)
+
+
+clangd
+------
+| Nathan Ridge
+| zeratul976@hotmail.com (email), HighCommander4 (GitHub), HighCommander4 (Discourse), nridge (Discord)
+
+| Chris Bieneman
+| chris.bieneman@gmail.com (email), llvm-beanz (GitHub), beanz (Discord), beanz (Discourse)
+
+| Kadir Çetinkaya
+| kadircet@google.com (email), kadircet (GitHub) kadircet (Discourse), kadircet (Discord)
+
+
+Inactive Maintainers
+====================
+The following people have graciously spent time performing maintainership
+responsibilities but are no longer active in that role. Thank you for all your
+help with the success of the project!
+
+Emeritus Lead Maintainers
+-------------------------
+| Manuel Klimek (klimek@google.com (email), r4nt (GitHub))
+
+
+Inactive component maintainers
+------------------------------
+| Nathan James (n.james93@hotmail.co.uk) -- clang-tidy
+| Julie Hockett (juliehockett@google.com) -- clang-doc
+| Sam McCall (sammccall@google.com (email), sam-mccall (GitHub, Discourse, Discord)) -- clangd
diff --git a/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp b/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
index 346437e20322e..dc3a3b6211b7e 100644
--- a/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
+++ b/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
@@ -116,6 +116,28 @@ findMembersUsedInInitExpr(const CXXCtorInitializer *Initializer,
   return Results;
 }
 
+/// Returns the full source range for the field declaration up to (not
+/// including) the trailing semicolumn, including potential macro invocations,
+/// e.g. `int a GUARDED_BY(mu);`.
+static SourceRange getFullFieldSourceRange(const FieldDecl &Field,
+                                           const ASTContext &Context) {
+  SourceRange Range = Field.getSourceRange();
+  SourceLocation End = Range.getEnd();
+  const SourceManager &SM = Context.getSourceManager();
+  const LangOptions &LangOpts = Context.getLangOpts();
+  while (true) {
+    std::optional<Token> CurrentToken = Lexer::findNextToken(End, SM, LangOpts);
+
+    if (!CurrentToken || CurrentToken->is(tok::semi))
+      break;
+
+    if (CurrentToken->is(tok::eof))
+      return Range; // Something is wrong, return the original range.
+    End = CurrentToken->getLastLoc();
+  }
+  return SourceRange(Range.getBegin(), End);
+}
+
 /// Reorders fields in the definition of a struct/class.
 ///
 /// At the moment reordering of fields with
@@ -145,9 +167,10 @@ static bool reorderFieldsInDefinition(
     const auto FieldIndex = Field->getFieldIndex();
     if (FieldIndex == NewFieldsOrder[FieldIndex])
       continue;
-    addReplacement(Field->getSourceRange(),
-                   Fields[NewFieldsOrder[FieldIndex]]->getSourceRange(),
-                   Context, Replacements);
+    addReplacement(
+        getFullFieldSourceRange(*Field, Context),
+        getFullFieldSourceRange(*Fields[NewFieldsOrder[FieldIndex]], Context),
+        Context, Replacements);
   }
   return true;
 }
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index 97e16a12febd0..ff42f96a0477b 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -81,6 +81,9 @@ class ClangTidyContext {
 
   ~ClangTidyContext();
 
+  ClangTidyContext(const ClangTidyContext &) = delete;
+  ClangTidyContext &operator=(const ClangTidyContext &) = delete;
+
   /// Report any errors detected using this method.
   ///
   /// This is still under heavy development and will likely change towards using
diff --git a/clang-tools-extra/clang-tidy/NoLintDirectiveHandler.h b/clang-tools-extra/clang-tidy/NoLintDirectiveHandler.h
index e862195abaabb..f66285672d04a 100644
--- a/clang-tools-extra/clang-tidy/NoLintDirectiveHandler.h
+++ b/clang-tools-extra/clang-tidy/NoLintDirectiveHandler.h
@@ -31,6 +31,8 @@ class NoLintDirectiveHandler {
 public:
   NoLintDirectiveHandler();
   ~NoLintDirectiveHandler();
+  NoLintDirectiveHandler(const NoLintDirectiveHandler &) = delete;
+  NoLintDirectiveHandler &operator=(const NoLintDirectiveHandler &) = delete;
 
   bool shouldSuppress(DiagnosticsEngine::Level DiagLevel,
                       const Diagnostic &Diag, llvm::StringRef DiagName,
diff --git a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
index 356acf968db92..a7e25141b3fe2 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
@@ -102,6 +102,212 @@ void BranchCloneCheck::registerMatchers(MatchFinder *Finder) {
       this);
   Finder->addMatcher(switchStmt().bind("switch"), this);
   Finder->addMatcher(conditionalOperator().bind("condOp"), this);
+  Finder->addMatcher(
+      ifStmt((hasThen(hasDescendant(ifStmt())))).bind("ifWithDescendantIf"),
+      this);
+}
+
+/// Determines whether two statement trees are identical regarding
+/// operators and symbols.
+///
+/// Exceptions: expressions containing macros or functions with possible side
+/// effects are never considered identical.
+/// Limitations: (t + u) and (u + t) are not considered identical.
+/// t*(u + t) and t*u + t*t are not considered identical.
+///
+static bool isIdenticalStmt(const ASTContext &Ctx, const Stmt *Stmt1,
+                            const Stmt *Stmt2, bool IgnoreSideEffects) {
+
+  if (!Stmt1 || !Stmt2)
+    return !Stmt1 && !Stmt2;
+
+  // If Stmt1 & Stmt2 are of different class then they are not
+  // identical statements.
+  if (Stmt1->getStmtClass() != Stmt2->getStmtClass())
+    return false;
+
+  const auto *Expr1 = dyn_cast<Expr>(Stmt1);
+  const auto *Expr2 = dyn_cast<Expr>(Stmt2);
+
+  if (Expr1 && Expr2) {
+    // If Stmt1 has side effects then don't warn even if expressions
+    // are identical.
+    if (!IgnoreSideEffects && Expr1->HasSideEffects(Ctx) &&
+        Expr2->HasSideEffects(Ctx))
+      return false;
+    // If either expression comes from a macro then don't warn even if
+    // the expressions are identical.
+    if ((Expr1->getExprLoc().isMacroID()) || (Expr2->getExprLoc().isMacroID()))
+      return false;
+
+    // If all children of two expressions are identical, return true.
+    Expr::const_child_iterator I1 = Expr1->child_begin();
+    Expr::const_child_iterator I2 = Expr2->child_begin();
+    while (I1 != Expr1->child_end() && I2 != Expr2->child_end()) {
+      if (!isIdenticalStmt(Ctx, *I1, *I2, IgnoreSideEffects))
+        return false;
+      ++I1;
+      ++I2;
+    }
+    // If there are different number of children in the statements, return
+    // false.
+    if (I1 != Expr1->child_end())
+      return false;
+    if (I2 != Expr2->child_end())
+      return false;
+  }
+
+  switch (Stmt1->getStmtClass()) {
+  default:
+    return false;
+  case Stmt::CallExprClass:
+  case Stmt::ArraySubscriptExprClass:
+  case Stmt::ArraySectionExprClass:
+  case Stmt::OMPArrayShapingExprClass:
+  case Stmt::OMPIteratorExprClass:
+  case Stmt::ImplicitCastExprClass:
+  case Stmt::ParenExprClass:
+  case Stmt::BreakStmtClass:
+  case Stmt::ContinueStmtClass:
+  case Stmt::NullStmtClass:
+    return true;
+  case Stmt::CStyleCastExprClass: {
+    const auto *CastExpr1 = cast<CStyleCastExpr>(Stmt1);
+    const auto *CastExpr2 = cast<CStyleCastExpr>(Stmt2);
+
+    return CastExpr1->getTypeAsWritten() == CastExpr2->getTypeAsWritten();
+  }
+  case Stmt::ReturnStmtClass: {
+    const auto *ReturnStmt1 = cast<ReturnStmt>(Stmt1);
+    const auto *ReturnStmt2 = cast<ReturnStmt>(Stmt2);
+
+    return isIdenticalStmt(Ctx, ReturnStmt1->getRetValue(),
+                           ReturnStmt2->getRetValue(), IgnoreSideEffects);
+  }
+  case Stmt::ForStmtClass: {
+    const auto *ForStmt1 = cast<ForStmt>(Stmt1);
+    const auto *ForStmt2 = cast<ForStmt>(Stmt2);
+
+    if (!isIdenticalStmt(Ctx, ForStmt1->getInit(), ForStmt2->getInit(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, ForStmt1->getCond(), ForStmt2->getCond(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, ForStmt1->getInc(), ForStmt2->getInc(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, ForStmt1->getBody(), ForStmt2->getBody(),
+                         IgnoreSideEffects))
+      return false;
+    return true;
+  }
+  case Stmt::DoStmtClass: {
+    const auto *DStmt1 = cast<DoStmt>(Stmt1);
+    const auto *DStmt2 = cast<DoStmt>(Stmt2);
+
+    if (!isIdenticalStmt(Ctx, DStmt1->getCond(), DStmt2->getCond(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, DStmt1->getBody(), DStmt2->getBody(),
+                         IgnoreSideEffects))
+      return false;
+    return true;
+  }
+  case Stmt::WhileStmtClass: {
+    const auto *WStmt1 = cast<WhileStmt>(Stmt1);
+    const auto *WStmt2 = cast<WhileStmt>(Stmt2);
+
+    if (!isIdenticalStmt(Ctx, WStmt1->getCond(), WStmt2->getCond(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, WStmt1->getBody(), WStmt2->getBody(),
+                         IgnoreSideEffects))
+      return false;
+    return true;
+  }
+  case Stmt::IfStmtClass: {
+    const auto *IStmt1 = cast<IfStmt>(Stmt1);
+    const auto *IStmt2 = cast<IfStmt>(Stmt2);
+
+    if (!isIdenticalStmt(Ctx, IStmt1->getCond(), IStmt2->getCond(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, IStmt1->getThen(), IStmt2->getThen(),
+                         IgnoreSideEffects))
+      return false;
+    if (!isIdenticalStmt(Ctx, IStmt1->getElse(), IStmt2->getElse(),
+                         IgnoreSideEffects))
+      return false;
+    return true;
+  }
+  case Stmt::CompoundStmtClass: {
+    const auto *CompStmt1 = cast<CompoundStmt>(Stmt1);
+    const auto *CompStmt2 = cast<CompoundStmt>(Stmt2);
+
+    if (CompStmt1->size() != CompStmt2->size())
+      return false;
+
+    if (!llvm::all_of(llvm::zip(CompStmt1->body(), CompStmt2->body()),
+                      [&Ctx, IgnoreSideEffects](
+                          std::tuple<const Stmt *, const Stmt *> stmtPair) {
+                        const Stmt *stmt0 = std::get<0>(stmtPair);
+                        const Stmt *stmt1 = std::get<1>(stmtPair);
+                        return isIdenticalStmt(Ctx, stmt0, stmt1,
+                                               IgnoreSideEffects);
+                      })) {
+      return false;
+    }
+
+    return true;
+  }
+  case Stmt::CompoundAssignOperatorClass:
+  case Stmt::BinaryOperatorClass: {
+    const auto *BinOp1 = cast<BinaryOperator>(Stmt1);
+    const auto *BinOp2 = cast<BinaryOperator>(Stmt2);
+    return BinOp1->getOpcode() == BinOp2->getOpcode();
+  }
+  case Stmt::CharacterLiteralClass: {
+    const auto *CharLit1 = cast<CharacterLiteral>(Stmt1);
+    const auto *CharLit2 = cast<CharacterLiteral>(Stmt2);
+    return CharLit1->getValue() == CharLit2->getValue();
+  }
+  case Stmt::DeclRefExprClass: {
+    const auto *DeclRef1 = cast<DeclRefExpr>(Stmt1);
+    const auto *DeclRef2 = cast<DeclRefExpr>(Stmt2);
+    return DeclRef1->getDecl() == DeclRef2->getDecl();
+  }
+  case Stmt::IntegerLiteralClass: {
+    const auto *IntLit1 = cast<IntegerLiteral>(Stmt1);
+    const auto *IntLit2 = cast<IntegerLiteral>(Stmt2);
+
+    llvm::APInt I1 = IntLit1->getValue();
+    llvm::APInt I2 = IntLit2->getValue();
+    if (I1.getBitWidth() != I2.getBitWidth())
+      return false;
+    return I1 == I2;
+  }
+  case Stmt::FloatingLiteralClass: {
+    const auto *FloatLit1 = cast<FloatingLiteral>(Stmt1);
+    const auto *FloatLit2 = cast<FloatingLiteral>(Stmt2);
+    return FloatLit1->getValue().bitwiseIsEqual(FloatLit2->getValue());
+  }
+  case Stmt::StringLiteralClass: {
+    const auto *StringLit1 = cast<StringLiteral>(Stmt1);
+    const auto *StringLit2 = cast<StringLiteral>(Stmt2);
+    return StringLit1->getBytes() == StringLit2->getBytes();
+  }
+  case Stmt::MemberExprClass: {
+    const auto *MemberStmt1 = cast<MemberExpr>(Stmt1);
+    const auto *MemberStmt2 = cast<MemberExpr>(Stmt2);
+    return MemberStmt1->getMemberDecl() == MemberStmt2->getMemberDecl();
+  }
+  case Stmt::UnaryOperatorClass: {
+    const auto *UnaryOp1 = cast<UnaryOperator>(Stmt1);
+    const auto *UnaryOp2 = cast<UnaryOperator>(Stmt2);
+    return UnaryOp1->getOpcode() == UnaryOp2->getOpcode();
+  }
+  }
 }
 
 void BranchCloneCheck::check(const MatchFinder::MatchResult &Result) {
@@ -269,6 +475,21 @@ void BranchCloneCheck::check(const MatchFinder::MatchResult &Result) {
     return;
   }
 
+  if (const auto *IS = Result.Nodes.getNodeAs<IfStmt>("ifWithDescendantIf")) {
+    const Stmt *Then = IS->getThen();
+    auto CS = dyn_cast<CompoundStmt>(Then);
+    if (CS && (!CS->body_empty())) {
+      const auto *InnerIf = dyn_cast<IfStmt>(*CS->body_begin());
+      if (InnerIf && isIdenticalStmt(Context, IS->getCond(), InnerIf->getCond(),
+                                     /*IgnoreSideEffects=*/false)) {
+        diag(IS->getBeginLoc(), "if with identical inner if statement");
+        diag(InnerIf->getBeginLoc(), "inner if starts here",
+             DiagnosticIDs::Note);
+      }
+    }
+    return;
+  }
+
   llvm_unreachable("No if statement and no switch statement.");
 }
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index b0a2318acc059..13adad7c3dadb 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -16,6 +16,7 @@ add_clang_library(clangTidyBugproneModule STATIC
   ChainedComparisonCheck.cpp
   ComparePointerToMemberVirtualFunctionCheck.cpp
   CopyConstructorInitCheck.cpp
+  CrtpConstructorAccessibilityCheck.cpp
   DanglingHandleCheck.cpp
   DynamicStaticInitializersCheck.cpp
   EasilySwappableParametersCheck.cpp
@@ -26,11 +27,8 @@ add_clang_library(clangTidyBugproneModule STATIC
   ForwardingReferenceOverloadCheck.cpp
   ImplicitWideningOfMultiplicationResultCheck.cpp
   InaccurateEraseCheck.cpp
-  IncorrectEnableIfCheck.cpp
-  ReturnConstRefFromParameterCheck.cpp
-  SuspiciousStringviewDataUsageCheck.cpp
-  SwitchMissingDefaultCaseCheck.cpp
   IncDecInConditionsCheck.cpp
+  IncorrectEnableIfCheck.cpp
   IncorrectRoundingsCheck.cpp
   InfiniteLoopCheck.cpp
   IntegerDivisionCheck.cpp
@@ -45,8 +43,8 @@ add_clang_library(clangTidyBugproneModule STATIC
   MultipleNewInOneExpressionCheck.cpp
   MultipleStatementMacroCheck.cpp
   NoEscapeCheck.cpp
-  NondeterministicPointerIterationOrderCheck.cpp
   NonZeroEnumToBoolConversionCheck.cpp
+  NondeterministicPointerIterationOrderCheck.cpp
   NotNullTerminatedResultCheck.cpp
   OptionalValueConversionCheck.cpp
   ParentVirtualCallCheck.cpp
@@ -54,6 +52,7 @@ add_clang_library(clangTidyBugproneModule STATIC
   PosixReturnCheck.cpp
   RedundantBranchConditionCheck.cpp
   ReservedIdentifierCheck.cpp
+  ReturnConstRefFromParameterCheck.cpp
   SharedPtrArrayMismatchCheck.cpp
   SignalHandlerCheck.cpp
   SignedCharMisuseCheck.cpp
@@ -74,7 +73,9 @@ add_clang_library(clangTidyBugproneModule STATIC
   SuspiciousReallocUsageCheck.cpp
   SuspiciousSemicolonCheck.cpp
   SuspiciousStringCompareCheck.cpp
+  SuspiciousStringviewDataUsageCheck.cpp
   SwappedArgumentsCheck.cpp
+  SwitchMissingDefaultCaseCheck.cpp
   TaggedUnionMemberCountCheck.cpp
   TerminatingContinueCheck.cpp
   ThrowKeywordMissingCheck.cpp
@@ -85,7 +86,6 @@ add_clang_library(clangTidyBugproneModule STATIC
   UnhandledExceptionAtNewCheck.cpp
   UnhandledSelfAssignmentCheck.cpp
   UniquePtrArrayMismatchCheck.cpp
-  CrtpConstructorAccessibilityCheck.cpp
   UnsafeFunctionsCheck.cpp
   UnusedLocalNonTrivialVariableCheck.cpp
   UnusedRaiiCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
index 2b2d80ea9346b..b7f0c08b2a7d4 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -13,13 +13,15 @@
 #include "clang/Analysis/Analyses/ExprMutationAnalyzer.h"
 #include "clang/Analysis/CallGraph.h"
 #include "llvm/ADT/SCCIterator.h"
-#include "llvm/ADT/SmallVector.h"
 
 using namespace clang::ast_matchers;
+using clang::ast_matchers::internal::Matcher;
 using clang::tidy::utils::hasPtrOrReferenceInFunc;
 
 namespace clang {
-namespace ast_matchers {
+namespace tidy::bugprone {
+
+namespace {
 /// matches a Decl if it has a  "no return" attribute of any kind
 AST_MATCHER(Decl, declHasNoReturnAttr) {
   return Node.hasAttr<NoReturnAttr>() || Node.hasAttr<CXX11NoReturnAttr>() ||
@@ -30,23 +32,21 @@ AST_MATCHER(Decl, declHasNoReturnAttr) {
 AST_MATCHER(FunctionType, typeHasNoReturnAttr) {
   return Node.getNoReturnAttr();
 }
-} // namespace ast_matchers
-namespace tidy::bugprone {
+} // namespace
 
-static internal::Matcher<Stmt>
-loopEndingStmt(internal::Matcher<Stmt> Internal) {
-  internal::Matcher<QualType> isNoReturnFunType =
+static Matcher<Stmt> loopEndingStmt(Matcher<Stmt> Internal) {
+  Matcher<QualType> IsNoReturnFunType =
       ignoringParens(functionType(typeHasNoReturnAttr()));
-  internal::Matcher<Decl> isNoReturnDecl =
-      anyOf(declHasNoReturnAttr(), functionDecl(hasType(isNoReturnFunType)),
-            varDecl(hasType(blockPointerType(pointee(isNoReturnFunType)))));
+  Matcher<Decl> IsNoReturnDecl =
+      anyOf(declHasNoReturnAttr(), functionDecl(hasType(IsNoReturnFunType)),
+            varDecl(hasType(blockPointerType(pointee(IsNoReturnFunType)))));
 
   return stmt(anyOf(
       mapAnyOf(breakStmt, returnStmt, gotoStmt, cxxThrowExpr).with(Internal),
       callExpr(Internal,
                callee(mapAnyOf(functionDecl, /* block callee */ varDecl)
-                          .with(isNoReturnDecl))),
-      objcMessageExpr(Internal, callee(isNoReturnDecl))));
+                          .with(IsNoReturnDecl))),
+      objcMessageExpr(Internal, callee(IsNoReturnDecl))));
 }
 
 /// Return whether `Var` was changed in `LoopStmt`.
diff --git a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
index 600eab3755276..f2ff27d85fb00 100644
--- a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
@@ -43,18 +43,20 @@ OptionalValueConversionCheck::getCheckTraversalKind() const {
 }
 
 void OptionalValueConversionCheck::registerMatchers(MatchFinder *Finder) {
-  auto ConstructTypeMatcher =
-      qualType(hasCleanType(qualType().bind("optional-type")));
+  auto BindOptionalType = qualType(
+      hasCleanType(qualType(hasDeclaration(namedDecl(
+                                matchers::matchesAnyListedName(OptionalTypes))))
+                       .bind("optional-type")));
 
-  auto CallTypeMatcher =
+  auto EqualsBoundOptionalType =
       qualType(hasCleanType(equalsBoundNode("optional-type")));
 
   auto OptionalDereferenceMatcher = callExpr(
       anyOf(
           cxxOperatorCallExpr(hasOverloadedOperatorName("*"),
-                              hasUnaryOperand(hasType(CallTypeMatcher)))
+                              hasUnaryOperand(hasType(EqualsBoundOptionalType)))
               .bind("op-call"),
-          cxxMemberCallExpr(thisPointerType(CallTypeMatcher),
+          cxxMemberCallExpr(thisPointerType(EqualsBoundOptionalType),
                             callee(cxxMethodDecl(anyOf(
                                 hasOverloadedOperatorName("*"),
                                 matchers::matchesAnyListedName(ValueMethods)))))
@@ -66,10 +68,7 @@ void OptionalValueConversionCheck::registerMatchers(MatchFinder *Finder) {
                hasArgument(0, ignoringImpCasts(OptionalDereferenceMatcher)));
   Finder->addMatcher(
       cxxConstructExpr(
-          argumentCountIs(1U),
-          hasDeclaration(cxxConstructorDecl(
-              ofClass(matchers::matchesAnyListedName(OptionalTypes)))),
-          hasType(ConstructTypeMatcher),
+          argumentCountIs(1U), hasType(BindOptionalType),
           hasArgument(0U, ignoringImpCasts(anyOf(OptionalDereferenceMatcher,
                                                  StdMoveCallMatcher))),
           unless(anyOf(hasAncestor(typeLoc()),
diff --git a/clang-tools-extra/clang-tidy/bugprone/ReturnConstRefFromParameterCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ReturnConstRefFromParameterCheck.cpp
index 7cc4fe519d3a6..295955a971d7e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ReturnConstRefFromParameterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ReturnConstRefFromParameterCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ReturnConstRefFromParameterCheck.h"
+#include "clang/AST/Attrs.inc"
 #include "clang/AST/Expr.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
@@ -15,25 +16,35 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
+namespace {
+
+AST_MATCHER(ParmVarDecl, hasLifetimeBoundAttr) {
+  return Node.hasAttr<LifetimeBoundAttr>();
+}
+
+} // namespace
+
 void ReturnConstRefFromParameterCheck::registerMatchers(MatchFinder *Finder) {
   const auto DRef = ignoringParens(
       declRefExpr(
           to(parmVarDecl(hasType(hasCanonicalType(
                              qualType(lValueReferenceType(pointee(
                                           qualType(isConstQualified()))))
-                                 .bind("type"))))
+                                 .bind("type"))),
+                         hasDeclContext(functionDecl(
+                             equalsBoundNode("func"),
+                             hasReturnTypeLoc(loc(qualType(
+                                 hasCanonicalType(equalsBoundNode("type"))))))),
+                         unless(hasLifetimeBoundAttr()))
                  .bind("param")))
           .bind("dref"));
-  const auto Func =
-      functionDecl(hasReturnTypeLoc(loc(
-                       qualType(hasCanonicalType(equalsBoundNode("type"))))))
-          .bind("func");
 
-  Finder->addMatcher(returnStmt(hasReturnValue(DRef), hasAncestor(Func)), this);
   Finder->addMatcher(
-      returnStmt(hasReturnValue(ignoringParens(conditionalOperator(
-          eachOf(hasTrueExpression(DRef), hasFalseExpression(DRef)),
-          hasAncestor(Func))))),
+      returnStmt(
+          hasAncestor(functionDecl().bind("func")),
+          hasReturnValue(anyOf(
+              DRef, ignoringParens(conditionalOperator(eachOf(
+                        hasTrueExpression(DRef), hasFalseExpression(DRef))))))),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index 6bb9a349d69b1..fc35bc22c52e0 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -855,9 +855,6 @@ static bool areExprsMacroAndNonMacro(const Expr *&LhsExpr,
 } // namespace
 
 void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
-  const auto AnyLiteralExpr = ignoringParenImpCasts(
-      anyOf(cxxBoolLiteral(), characterLiteral(), integerLiteral()));
-
   const auto BannedIntegerLiteral =
       integerLiteral(expandedByMacro(KnownBannedMacroNames));
   const auto IsInUnevaluatedContext = expr(anyOf(
@@ -866,19 +863,16 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
   // Binary with equivalent operands, like (X != 2 && X != 2).
   Finder->addMatcher(
       traverse(TK_AsIs,
-               binaryOperator(
-                   anyOf(isComparisonOperator(),
-                         hasAnyOperatorName("-", "/", "%", "|", "&", "^", "&&",
-                                            "||", "=")),
-                   operandsAreEquivalent(),
-                   // Filter noisy false positives.
-                   unless(isInTemplateInstantiation()),
-                   unless(binaryOperatorIsInMacro()),
-                   unless(hasType(realFloatingPointType())),
-                   unless(hasEitherOperand(hasType(realFloatingPointType()))),
-                   unless(hasLHS(AnyLiteralExpr)),
-                   unless(hasDescendant(BannedIntegerLiteral)),
-                   unless(IsInUnevaluatedContext))
+               binaryOperator(anyOf(isComparisonOperator(),
+                                    hasAnyOperatorName("-", "/", "%", "|", "&",
+                                                       "^", "&&", "||", "=")),
+                              operandsAreEquivalent(),
+                              // Filter noisy false positives.
+                              unless(isInTemplateInstantiation()),
+                              unless(binaryOperatorIsInMacro()),
+                              unless(hasAncestor(arraySubscriptExpr())),
+                              unless(hasDescendant(BannedIntegerLiteral)),
+                              unless(IsInUnevaluatedContext))
                    .bind("binary")),
       this);
 
@@ -1238,6 +1232,50 @@ void RedundantExpressionCheck::check(const MatchFinder::MatchResult &Result) {
   if (const auto *BinOp = Result.Nodes.getNodeAs<BinaryOperator>("binary")) {
     // If the expression's constants are macros, check whether they are
     // intentional.
+
+    //
+    // Special case for floating-point representation.
+    //
+    // If expressions on both sides of comparison operator are of type float,
+    // then for some comparison operators no warning shall be
+    // reported even if the expressions are identical from a symbolic point of
+    // view. Comparison between expressions, declared variables and literals
+    // are treated differently.
+    //
+    // != and == between float literals that have the same value should NOT
+    // warn. < > between float literals that have the same value SHOULD warn.
+    //
+    // != and == between the same float declaration should NOT warn.
+    // < > between the same float declaration SHOULD warn.
+    //
+    // != and == between eq. expressions that evaluates into float
+    //           should NOT warn.
+    // < >       between eq. expressions that evaluates into float
+    //           should NOT warn.
+    //
+    const Expr *LHS = BinOp->getLHS()->IgnoreParenImpCasts();
+    const Expr *RHS = BinOp->getRHS()->IgnoreParenImpCasts();
+    const BinaryOperator::Opcode Op = BinOp->getOpcode();
+    const bool OpEqualEQorNE = ((Op == BO_EQ) || (Op == BO_NE));
+
+    const auto *DeclRef1 = dyn_cast<DeclRefExpr>(LHS);
+    const auto *DeclRef2 = dyn_cast<DeclRefExpr>(RHS);
+    const auto *FloatLit1 = dyn_cast<FloatingLiteral>(LHS);
+    const auto *FloatLit2 = dyn_cast<FloatingLiteral>(RHS);
+
+    if (DeclRef1 && DeclRef2 &&
+        DeclRef1->getType()->hasFloatingRepresentation() &&
+        DeclRef2->getType()->hasFloatingRepresentation() &&
+        (DeclRef1->getDecl() == DeclRef2->getDecl()) && OpEqualEQorNE) {
+      return;
+    }
+
+    if (FloatLit1 && FloatLit2 &&
+        FloatLit1->getValue().bitwiseIsEqual(FloatLit2->getValue()) &&
+        OpEqualEQorNE) {
+      return;
+    }
+
     if (areSidesBinaryConstExpressions(BinOp, Result.Context)) {
       const Expr *LhsConst = nullptr, *RhsConst = nullptr;
       BinaryOperatorKind MainOpcode{}, SideOpcode{};
diff --git a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
index 71eb2d94cd4f2..1e0f398a4a560 100644
--- a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
@@ -15,7 +15,9 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Lex/Token.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 
 using namespace clang::ast_matchers;
 
@@ -78,6 +80,22 @@ AST_POLYMORPHIC_MATCHER(isExternStorageClass,
   return Node.getStorageClass() == SC_Extern;
 }
 
+AST_MATCHER(FunctionDecl, isAllocationOrDeallocationOverloadedFunction) {
+  // [basic.stc.dynamic.allocation]
+  // An allocation function that is not a class member function shall belong to
+  // the global scope and not have a name with internal linkage.
+  // [basic.stc.dynamic.deallocation]
+  // A deallocation function that is not a class member function shall belong to
+  // the global scope and not have a name with internal linkage.
+  static const llvm::DenseSet<OverloadedOperatorKind> OverloadedOperators{
+      OverloadedOperatorKind::OO_New,
+      OverloadedOperatorKind::OO_Array_New,
+      OverloadedOperatorKind::OO_Delete,
+      OverloadedOperatorKind::OO_Array_Delete,
+  };
+  return OverloadedOperators.contains(Node.getOverloadedOperator());
+}
+
 } // namespace
 
 UseInternalLinkageCheck::UseInternalLinkageCheck(StringRef Name,
@@ -100,10 +118,16 @@ void UseInternalLinkageCheck::registerMatchers(MatchFinder *Finder) {
                 isExternStorageClass(), isExternC(),
                 // 3. template
                 isExplicitTemplateSpecialization(),
-                // 4. friend
-                hasAncestor(friendDecl()))));
+                hasAncestor(decl(anyOf(
+                    // 4. friend
+                    friendDecl(),
+                    // 5. module export decl
+                    exportDecl()))))));
   Finder->addMatcher(
-      functionDecl(Common, hasBody(), unless(cxxMethodDecl()), unless(isMain()))
+      functionDecl(Common, hasBody(),
+                   unless(anyOf(cxxMethodDecl(),
+                                isAllocationOrDeallocationOverloadedFunction(),
+                                isMain())))
           .bind("fn"),
       this);
   Finder->addMatcher(varDecl(Common, hasGlobalStorage()).bind("var"), this);
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
index 234f6790a7ed9..ab72c6796bb4c 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
@@ -20,12 +20,16 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::modernize {
 
 static bool isNegativeComparison(const Expr *ComparisonExpr) {
-  if (const auto *BO = llvm::dyn_cast<BinaryOperator>(ComparisonExpr))
-    return BO->getOpcode() == BO_NE;
+  if (const auto *Op = llvm::dyn_cast<BinaryOperator>(ComparisonExpr))
+    return Op->getOpcode() == BO_NE;
 
   if (const auto *Op = llvm::dyn_cast<CXXOperatorCallExpr>(ComparisonExpr))
     return Op->getOperator() == OO_ExclaimEqual;
 
+  if (const auto *Op =
+          llvm::dyn_cast<CXXRewrittenBinaryOperator>(ComparisonExpr))
+    return Op->getOperator() == BO_NE;
+
   return false;
 }
 
@@ -185,7 +189,7 @@ void UseStartsEndsWithCheck::registerMatchers(MatchFinder *Finder) {
 
   // Case 6: X.substr(0, LEN(Y)) [!=]= Y -> starts_with.
   Finder->addMatcher(
-      cxxOperatorCallExpr(
+      binaryOperation(
           hasAnyOperatorName("==", "!="),
           hasOperands(
               expr().bind("needle"),
diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.h b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.h
index 617dadce76bd3..acd8a6bfc50f5 100644
--- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.h
@@ -18,12 +18,10 @@ namespace clang::tidy::readability {
 /// a call to `empty()`.
 ///
 /// The emptiness of a container should be checked using the `empty()` method
-/// instead of the `size()` method. It is not guaranteed that `size()` is a
-/// constant-time function, and it is generally more efficient and also shows
-/// clearer intent to use `empty()`. Furthermore some containers may implement
-/// the `empty()` method but not implement the `size()` method. Using `empty()`
-/// whenever possible makes it easier to switch to another container in the
-/// future.
+/// instead of the `size()` method. It shows clearer intent to use `empty()`.
+/// Furthermore some containers may implement the `empty()` method but not
+/// implement the `size()` method. Using `empty()` whenever possible makes it
+/// easier to switch to another container in the future.
 class ContainerSizeEmptyCheck : public ClangTidyCheck {
 public:
   ContainerSizeEmptyCheck(StringRef Name, ClangTidyContext *Context);
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 05dd313d0a0d3..1e981825c7c15 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -1415,6 +1415,12 @@ void ClangdLSPServer::onInlayHint(const InlayHintsParams &Params,
                      std::move(Reply));
 }
 
+void ClangdLSPServer::onCallHierarchyOutgoingCalls(
+    const CallHierarchyOutgoingCallsParams &Params,
+    Callback<std::vector<CallHierarchyOutgoingCall>> Reply) {
+  Server->outgoingCalls(Params.item, std::move(Reply));
+}
+
 void ClangdLSPServer::applyConfiguration(
     const ConfigurationSettings &Settings) {
   // Per-file update to the compilation database.
@@ -1693,6 +1699,8 @@ void ClangdLSPServer::bindMethods(LSPBinder &Bind,
   Bind.method("typeHierarchy/subtypes", this, &ClangdLSPServer::onSubTypes);
   Bind.method("textDocument/prepareCallHierarchy", this, &ClangdLSPServer::onPrepareCallHierarchy);
   Bind.method("callHierarchy/incomingCalls", this, &ClangdLSPServer::onCallHierarchyIncomingCalls);
+  if (Opts.EnableOutgoingCalls)
+    Bind.method("callHierarchy/outgoingCalls", this, &ClangdLSPServer::onCallHierarchyOutgoingCalls);
   Bind.method("textDocument/selectionRange", this, &ClangdLSPServer::onSelectionRange);
   Bind.method("textDocument/documentLink", this, &ClangdLSPServer::onDocumentLink);
   Bind.method("textDocument/semanticTokens/full", this, &ClangdLSPServer::onSemanticTokens);
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index 0b8e4720f5323..f43734ec1ede3 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -73,6 +73,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   /// The destructor blocks on any outstanding background tasks.
   ~ClangdLSPServer();
 
+  ClangdLSPServer(const ClangdLSPServer &other) = delete;
+  ClangdLSPServer &operator=(const ClangdLSPServer &other) = delete;
+
   /// Run LSP server loop, communicating with the Transport provided in the
   /// constructor. This method must not be executed more than once.
   ///
@@ -156,6 +159,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   void onCallHierarchyIncomingCalls(
       const CallHierarchyIncomingCallsParams &,
       Callback<std::vector<CallHierarchyIncomingCall>>);
+  void onCallHierarchyOutgoingCalls(
+      const CallHierarchyOutgoingCallsParams &,
+      Callback<std::vector<CallHierarchyOutgoingCall>>);
   void onClangdInlayHints(const InlayHintsParams &,
                           Callback<llvm::json::Value>);
   void onInlayHint(const InlayHintsParams &, Callback<std::vector<InlayHint>>);
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 9b38be04e7ddd..52be15d3da936 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -215,7 +215,9 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, const Options &Opts,
                            Callbacks *Callbacks)
     : FeatureModules(Opts.FeatureModules), CDB(CDB), TFS(TFS),
-      DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
+      DynamicIdx(Opts.BuildDynamicSymbolIndex
+                     ? new FileIndex(Opts.EnableOutgoingCalls)
+                     : nullptr),
       ModulesManager(Opts.ModulesManager),
       ClangTidyProvider(Opts.ClangTidyProvider),
       UseDirtyHeaders(Opts.UseDirtyHeaders),
@@ -256,6 +258,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
         Callbacks->onBackgroundIndexProgress(S);
     };
     BGOpts.ContextProvider = Opts.ContextProvider;
+    BGOpts.SupportContainedRefs = Opts.EnableOutgoingCalls;
     BackgroundIdx = std::make_unique<BackgroundIndex>(
         TFS, CDB,
         BackgroundIndexStorage::createDiskBackedStorageFactory(
@@ -912,6 +915,15 @@ void ClangdServer::inlayHints(PathRef File, std::optional<Range> RestrictRange,
   WorkScheduler->runWithAST("InlayHints", File, std::move(Action), Transient);
 }
 
+void ClangdServer::outgoingCalls(
+    const CallHierarchyItem &Item,
+    Callback<std::vector<CallHierarchyOutgoingCall>> CB) {
+  WorkScheduler->run("Outgoing Calls", "",
+                     [CB = std::move(CB), Item, this]() mutable {
+                       CB(clangd::outgoingCalls(Item, Index));
+                     });
+}
+
 void ClangdServer::onFileEvent(const DidChangeWatchedFilesParams &Params) {
   // FIXME: Do nothing for now. This will be used for indexing and potentially
   // invalidating other caches.
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index a653cdb56b751..e030bf04122d5 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -110,6 +110,11 @@ class ClangdServer {
     /// Cached preambles are potentially large. If false, store them on disk.
     bool StorePreamblesInMemory = true;
 
+    /// Call hierarchy's outgoing calls feature requires additional index
+    /// serving structures which increase memory usage. If false, these are
+    /// not created and the feature is not enabled.
+    bool EnableOutgoingCalls = true;
+
     /// This throttler controls which preambles may be built at a given time.
     clangd::PreambleThrottler *PreambleThrottler = nullptr;
 
@@ -292,6 +297,10 @@ class ClangdServer {
   void incomingCalls(const CallHierarchyItem &Item,
                      Callback<std::vector<CallHierarchyIncomingCall>>);
 
+  /// Resolve outgoing calls for a given call hierarchy item.
+  void outgoingCalls(const CallHierarchyItem &Item,
+                     Callback<std::vector<CallHierarchyOutgoingCall>>);
+
   /// Resolve inlay hints for a given document.
   void inlayHints(PathRef File, std::optional<Range> RestrictRange,
                   Callback<std::vector<InlayHint>>);
diff --git a/clang-tools-extra/clangd/HeuristicResolver.cpp b/clang-tools-extra/clangd/HeuristicResolver.cpp
index 26d54200eeffd..9eb892e8e4a8e 100644
--- a/clang-tools-extra/clangd/HeuristicResolver.cpp
+++ b/clang-tools-extra/clangd/HeuristicResolver.cpp
@@ -118,6 +118,16 @@ const Type *resolveDeclsToType(const std::vector<const NamedDecl *> &Decls,
   return nullptr;
 }
 
+TemplateName getReferencedTemplateName(const Type *T) {
+  if (const auto *TST = T->getAs<TemplateSpecializationType>()) {
+    return TST->getTemplateName();
+  }
+  if (const auto *DTST = T->getAs<DeducedTemplateSpecializationType>()) {
+    return DTST->getTemplateName();
+  }
+  return TemplateName();
+}
+
 // Helper function for HeuristicResolver::resolveDependentMember()
 // which takes a possibly-dependent type `T` and heuristically
 // resolves it to a CXXRecordDecl in which we can try name lookup.
@@ -142,12 +152,12 @@ CXXRecordDecl *HeuristicResolverImpl::resolveTypeToRecordDecl(const Type *T) {
   if (!T)
     return nullptr;
 
-  const auto *TST = T->getAs<TemplateSpecializationType>();
-  if (!TST)
+  TemplateName TN = getReferencedTemplateName(T);
+  if (TN.isNull())
     return nullptr;
 
-  const ClassTemplateDecl *TD = dyn_cast_or_null<ClassTemplateDecl>(
-      TST->getTemplateName().getAsTemplateDecl());
+  const ClassTemplateDecl *TD =
+      dyn_cast_or_null<ClassTemplateDecl>(TN.getAsTemplateDecl());
   if (!TD)
     return nullptr;
 
diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp
index 29508901f85bb..bee31fe51555e 100644
--- a/clang-tools-extra/clangd/ModulesBuilder.cpp
+++ b/clang-tools-extra/clangd/ModulesBuilder.cpp
@@ -199,7 +199,7 @@ bool IsModuleFileUpToDate(PathRef ModuleFilePath,
 
   SourceManager SourceMgr(*Diags, FileMgr);
 
-  HeaderSearch HeaderInfo(HSOpts, SourceMgr, *Diags, LangOpts,
+  HeaderSearch HeaderInfo(std::move(HSOpts), SourceMgr, *Diags, LangOpts,
                           /*Target=*/nullptr);
 
   TrivialModuleLoader ModuleLoader;
diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h
index 63e564bd68a78..8d9d1e6456926 100644
--- a/clang-tools-extra/clangd/ParsedAST.h
+++ b/clang-tools-extra/clangd/ParsedAST.h
@@ -59,6 +59,9 @@ class ParsedAST {
 
   ~ParsedAST();
 
+  ParsedAST(const ParsedAST &Other) = delete;
+  ParsedAST &operator=(const ParsedAST &Other) = delete;
+
   /// Note that the returned ast will not contain decls from the preamble that
   /// were not deserialized during parsing. Clients should expect only decls
   /// from the main file to be in the AST.
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index 71548b59cc308..035e5e63d8fbb 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -411,6 +411,9 @@ class PreambleThrottlerRequest {
     if (Throttler)
       Throttler->release(ID);
   }
+  PreambleThrottlerRequest(const PreambleThrottlerRequest &) = delete;
+  PreambleThrottlerRequest &
+  operator=(const PreambleThrottlerRequest &) = delete;
 
 private:
   PreambleThrottler::RequestID ID;
@@ -621,7 +624,8 @@ class ASTWorker {
          AsyncTaskRunner *Tasks, Semaphore &Barrier,
          const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
   ~ASTWorker();
-
+  ASTWorker(const ASTWorker &other) = delete;
+  ASTWorker &operator=(const ASTWorker &other) = delete;
   void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
   void
   runWithAST(llvm::StringRef Name,
diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h
index fb936d46bbcf7..d0da20310a8b2 100644
--- a/clang-tools-extra/clangd/TUScheduler.h
+++ b/clang-tools-extra/clangd/TUScheduler.h
@@ -242,6 +242,9 @@ class TUScheduler {
               std::unique_ptr<ParsingCallbacks> ASTCallbacks = nullptr);
   ~TUScheduler();
 
+  TUScheduler(const TUScheduler &other) = delete;
+  TUScheduler &operator=(const TUScheduler &other) = delete;
+
   struct FileStats {
     std::size_t UsedBytesAST = 0;
     std::size_t UsedBytesPreamble = 0;
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 61fa66180376c..8d5ab2e491a40 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -121,31 +121,17 @@ void logIfOverflow(const SymbolLocation &Loc) {
 
 // Convert a SymbolLocation to LSP's Location.
 // TUPath is used to resolve the path of URI.
-// FIXME: figure out a good home for it, and share the implementation with
-// FindSymbols.
 std::optional<Location> toLSPLocation(const SymbolLocation &Loc,
                                       llvm::StringRef TUPath) {
   if (!Loc)
     return std::nullopt;
-  auto Uri = URI::parse(Loc.FileURI);
-  if (!Uri) {
-    elog("Could not parse URI {0}: {1}", Loc.FileURI, Uri.takeError());
+  auto LSPLoc = indexToLSPLocation(Loc, TUPath);
+  if (!LSPLoc) {
+    elog("{0}", LSPLoc.takeError());
     return std::nullopt;
   }
-  auto U = URIForFile::fromURI(*Uri, TUPath);
-  if (!U) {
-    elog("Could not resolve URI {0}: {1}", Loc.FileURI, U.takeError());
-    return std::nullopt;
-  }
-
-  Location LSPLoc;
-  LSPLoc.uri = std::move(*U);
-  LSPLoc.range.start.line = Loc.Start.line();
-  LSPLoc.range.start.character = Loc.Start.column();
-  LSPLoc.range.end.line = Loc.End.line();
-  LSPLoc.range.end.character = Loc.End.column();
   logIfOverflow(Loc);
-  return LSPLoc;
+  return *LSPLoc;
 }
 
 SymbolLocation toIndexLocation(const Location &Loc, std::string &URIStorage) {
@@ -1702,6 +1688,7 @@ declToHierarchyItem(const NamedDecl &ND, llvm::StringRef TUPath) {
 
   HierarchyItem HI;
   HI.name = printName(Ctx, ND);
+  // FIXME: Populate HI.detail the way we do in symbolToHierarchyItem?
   HI.kind = SK;
   HI.range = Range{sourceLocToPosition(SM, DeclRange->getBegin()),
                    sourceLocToPosition(SM, DeclRange->getEnd())};
@@ -1753,6 +1740,7 @@ static std::optional<HierarchyItem> symbolToHierarchyItem(const Symbol &S,
   }
   HierarchyItem HI;
   HI.name = std::string(S.Name);
+  HI.detail = (S.Scope + S.Name).str();
   HI.kind = indexSymbolKindToSymbolKind(S.SymInfo.Kind);
   HI.selectionRange = Loc->range;
   // FIXME: Populate 'range' correctly
@@ -2319,6 +2307,65 @@ incomingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index) {
   return Results;
 }
 
+std::vector<CallHierarchyOutgoingCall>
+outgoingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index) {
+  std::vector<CallHierarchyOutgoingCall> Results;
+  if (!Index || Item.data.empty())
+    return Results;
+  auto ID = SymbolID::fromStr(Item.data);
+  if (!ID) {
+    elog("outgoingCalls failed to find symbol: {0}", ID.takeError());
+    return Results;
+  }
+  // In this function, we find outgoing calls based on the index only.
+  ContainedRefsRequest Request;
+  Request.ID = *ID;
+  // Initially store the ranges in a map keyed by SymbolID of the callee.
+  // This allows us to group different calls to the same function
+  // into the same CallHierarchyOutgoingCall.
+  llvm::DenseMap<SymbolID, std::vector<Range>> CallsOut;
+  // We can populate the ranges based on a refs request only. As we do so, we
+  // also accumulate the callee IDs into a lookup request.
+  LookupRequest CallsOutLookup;
+  Index->containedRefs(Request, [&](const auto &R) {
+    auto Loc = indexToLSPLocation(R.Location, Item.uri.file());
+    if (!Loc) {
+      elog("outgoingCalls failed to convert location: {0}", Loc.takeError());
+      return;
+    }
+    auto It = CallsOut.try_emplace(R.Symbol, std::vector<Range>{}).first;
+    It->second.push_back(Loc->range);
+
+    CallsOutLookup.IDs.insert(R.Symbol);
+  });
+  // Perform the lookup request and combine its results with CallsOut to
+  // get complete CallHierarchyOutgoingCall objects.
+  Index->lookup(CallsOutLookup, [&](const Symbol &Callee) {
+    // The containedRefs request should only return symbols which are
+    // function-like, i.e. symbols for which references to them can be "calls".
+    using SK = index::SymbolKind;
+    auto Kind = Callee.SymInfo.Kind;
+    assert(Kind == SK::Function || Kind == SK::InstanceMethod ||
+           Kind == SK::ClassMethod || Kind == SK::StaticMethod ||
+           Kind == SK::Constructor || Kind == SK::Destructor ||
+           Kind == SK::ConversionFunction);
+    (void)Kind;
+    (void)SK::Function;
+
+    auto It = CallsOut.find(Callee.ID);
+    assert(It != CallsOut.end());
+    if (auto CHI = symbolToCallHierarchyItem(Callee, Item.uri.file()))
+      Results.push_back(
+          CallHierarchyOutgoingCall{std::move(*CHI), std::move(It->second)});
+  });
+  // Sort results by name of the callee.
+  llvm::sort(Results, [](const CallHierarchyOutgoingCall &A,
+                         const CallHierarchyOutgoingCall &B) {
+    return A.to.name < B.to.name;
+  });
+  return Results;
+}
+
 llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
                                                  const FunctionDecl *FD) {
   if (!FD->hasBody())
diff --git a/clang-tools-extra/clangd/XRefs.h b/clang-tools-extra/clangd/XRefs.h
index df91dd15303c1..247e52314c3f9 100644
--- a/clang-tools-extra/clangd/XRefs.h
+++ b/clang-tools-extra/clangd/XRefs.h
@@ -150,6 +150,9 @@ prepareCallHierarchy(ParsedAST &AST, Position Pos, PathRef TUPath);
 std::vector<CallHierarchyIncomingCall>
 incomingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index);
 
+std::vector<CallHierarchyOutgoingCall>
+outgoingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index);
+
 /// Returns all decls that are referenced in the \p FD except local symbols.
 llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
                                                  const FunctionDecl *FD);
diff --git a/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp b/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
index bd32041bbe1fc..354e75d6e1559 100644
--- a/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
+++ b/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
@@ -24,12 +24,12 @@ namespace {
 
 std::unique_ptr<SymbolIndex> buildMem() {
   return loadIndex(IndexFilename, clang::clangd::SymbolOrigin::Static,
-                   /*UseDex=*/false);
+                   /*UseDex=*/false, /*SupportContainedRefs=*/true);
 }
 
 std::unique_ptr<SymbolIndex> buildDex() {
   return loadIndex(IndexFilename, clang::clangd::SymbolOrigin::Static,
-                   /*UseDex=*/true);
+                   /*UseDex=*/true, /*SupportContainedRefs=*/true);
 }
 
 // Reads JSON array of serialized FuzzyFindRequest's from user-provided file.
diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp
index 5cde4937fee78..496d1455def4b 100644
--- a/clang-tools-extra/clangd/index/Background.cpp
+++ b/clang-tools-extra/clangd/index/Background.cpp
@@ -96,7 +96,7 @@ BackgroundIndex::BackgroundIndex(
     : SwapIndex(std::make_unique<MemIndex>()), TFS(TFS), CDB(CDB),
       IndexingPriority(Opts.IndexingPriority),
       ContextProvider(std::move(Opts.ContextProvider)),
-      IndexedSymbols(IndexContents::All),
+      IndexedSymbols(IndexContents::All, Opts.SupportContainedRefs),
       Rebuilder(this, &IndexedSymbols, Opts.ThreadPoolSize),
       IndexStorageFactory(std::move(IndexStorageFactory)),
       Queue(std::move(Opts.OnProgress)),
diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h
index 0d719ffdb957e..448e911201575 100644
--- a/clang-tools-extra/clangd/index/Background.h
+++ b/clang-tools-extra/clangd/index/Background.h
@@ -145,6 +145,9 @@ class BackgroundIndex : public SwapIndex {
     // file. Called with the empty string for other tasks.
     // (When called, the context from BackgroundIndex construction is active).
     std::function<Context(PathRef)> ContextProvider = nullptr;
+    // Whether the index needs to support the containedRefs() operation.
+    // May use extra memory.
+    bool SupportContainedRefs = true;
   };
 
   /// Creates a new background index and starts its threads.
diff --git a/clang-tools-extra/clangd/index/BackgroundRebuild.cpp b/clang-tools-extra/clangd/index/BackgroundRebuild.cpp
index 79383be012f83..4dc2d3b1d059b 100644
--- a/clang-tools-extra/clangd/index/BackgroundRebuild.cpp
+++ b/clang-tools-extra/clangd/index/BackgroundRebuild.cpp
@@ -1,4 +1,4 @@
-//===-- BackgroundRebuild.cpp - when to rebuild thei background index -----===//
+//===-- BackgroundRebuild.cpp - when to rebuild the background index ------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp
index eb9562d2b6bf8..aa573e312a756 100644
--- a/clang-tools-extra/clangd/index/FileIndex.cpp
+++ b/clang-tools-extra/clangd/index/FileIndex.cpp
@@ -239,8 +239,8 @@ SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
                       /*CollectMainFileRefs=*/false);
 }
 
-FileSymbols::FileSymbols(IndexContents IdxContents)
-    : IdxContents(IdxContents) {}
+FileSymbols::FileSymbols(IndexContents IdxContents, bool SupportContainedRefs)
+    : IdxContents(IdxContents), SupportContainedRefs(SupportContainedRefs) {}
 
 void FileSymbols::update(llvm::StringRef Key,
                          std::unique_ptr<SymbolSlab> Symbols,
@@ -395,7 +395,7 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
         std::move(AllRelations), std::move(Files), IdxContents,
         std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
                         std::move(RefsStorage), std::move(SymsStorage)),
-        StorageSize);
+        StorageSize, SupportContainedRefs);
   }
   llvm_unreachable("Unknown clangd::IndexType");
 }
@@ -419,11 +419,12 @@ void FileSymbols::profile(MemoryTree &MT) const {
   }
 }
 
-FileIndex::FileIndex()
+FileIndex::FileIndex(bool SupportContainedRefs)
     : MergedIndex(&MainFileIndex, &PreambleIndex),
-      PreambleSymbols(IndexContents::Symbols | IndexContents::Relations),
+      PreambleSymbols(IndexContents::Symbols | IndexContents::Relations,
+                      SupportContainedRefs),
       PreambleIndex(std::make_unique<MemIndex>()),
-      MainFileSymbols(IndexContents::All),
+      MainFileSymbols(IndexContents::All, SupportContainedRefs),
       MainFileIndex(std::make_unique<MemIndex>()) {}
 
 void FileIndex::updatePreamble(IndexFileIn IF) {
diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h
index 44f33e8fbcd51..8e88dc9712996 100644
--- a/clang-tools-extra/clangd/index/FileIndex.h
+++ b/clang-tools-extra/clangd/index/FileIndex.h
@@ -69,7 +69,7 @@ enum class DuplicateHandling {
 /// locking when we swap or obtain references to snapshots.
 class FileSymbols {
 public:
-  FileSymbols(IndexContents IdxContents);
+  FileSymbols(IndexContents IdxContents, bool SupportContainedRefs);
   /// Updates all slabs associated with the \p Key.
   /// If either is nullptr, corresponding data for \p Key will be removed.
   /// If CountReferences is true, \p Refs will be used for counting references
@@ -91,6 +91,7 @@ class FileSymbols {
 
 private:
   IndexContents IdxContents;
+  bool SupportContainedRefs;
 
   struct RefSlabAndCountReferences {
     std::shared_ptr<RefSlab> Slab;
@@ -108,7 +109,7 @@ class FileSymbols {
 /// FIXME: Expose an interface to remove files that are closed.
 class FileIndex : public MergedIndex {
 public:
-  FileIndex();
+  FileIndex(bool SupportContainedRefs);
 
   /// Update preamble symbols of file \p Path with all declarations in \p AST
   /// and macros in \p PP.
diff --git a/clang-tools-extra/clangd/index/Index.cpp b/clang-tools-extra/clangd/index/Index.cpp
index 7a0c23287db22..86dc6ed763344 100644
--- a/clang-tools-extra/clangd/index/Index.cpp
+++ b/clang-tools-extra/clangd/index/Index.cpp
@@ -66,6 +66,11 @@ bool SwapIndex::refs(const RefsRequest &R,
                      llvm::function_ref<void(const Ref &)> CB) const {
   return snapshot()->refs(R, CB);
 }
+bool SwapIndex::containedRefs(
+    const ContainedRefsRequest &R,
+    llvm::function_ref<void(const ContainedRefsResult &)> CB) const {
+  return snapshot()->containedRefs(R, CB);
+}
 void SwapIndex::relations(
     const RelationsRequest &R,
     llvm::function_ref<void(const SymbolID &, const Symbol &)> CB) const {
diff --git a/clang-tools-extra/clangd/index/Index.h b/clang-tools-extra/clangd/index/Index.h
index 047ce08e93e3a..a193b1a191216 100644
--- a/clang-tools-extra/clangd/index/Index.h
+++ b/clang-tools-extra/clangd/index/Index.h
@@ -77,6 +77,19 @@ struct RefsRequest {
   bool WantContainer = false;
 };
 
+struct ContainedRefsRequest {
+  /// Note that RefKind::Call just restricts the matched SymbolKind to
+  /// functions, not the form of the reference (e.g. address-of-function,
+  /// which can indicate an indirect call, should still be caught).
+  static const RefKind SupportedRefKinds = RefKind::Call;
+
+  SymbolID ID;
+  /// If set, limit the number of refers returned from the index. The index may
+  /// choose to return less than this, e.g. it tries to avoid returning stale
+  /// results.
+  std::optional<uint32_t> Limit;
+};
+
 struct RelationsRequest {
   llvm::DenseSet<SymbolID> Subjects;
   RelationKind Predicate;
@@ -84,6 +97,14 @@ struct RelationsRequest {
   std::optional<uint32_t> Limit;
 };
 
+struct ContainedRefsResult {
+  /// The source location where the symbol is named.
+  SymbolLocation Location;
+  RefKind Kind = RefKind::Unknown;
+  /// The ID of the symbol which is referred to
+  SymbolID Symbol;
+};
+
 /// Describes what data is covered by an index.
 ///
 /// Indexes may contain symbols but not references from a file, etc.
@@ -141,6 +162,17 @@ class SymbolIndex {
   virtual bool refs(const RefsRequest &Req,
                     llvm::function_ref<void(const Ref &)> Callback) const = 0;
 
+  /// Find all symbols that are referenced by a symbol and apply
+  /// \p Callback on each result.
+  ///
+  /// Results should be returned in arbitrary order.
+  /// The returned result must be deep-copied if it's used outside Callback.
+  ///
+  /// Returns true if there will be more results (limited by Req.Limit);
+  virtual bool containedRefs(
+      const ContainedRefsRequest &Req,
+      llvm::function_ref<void(const ContainedRefsResult &)> Callback) const = 0;
+
   /// Finds all relations (S, P, O) stored in the index such that S is among
   /// Req.Subjects and P is Req.Predicate, and invokes \p Callback for (S, O) in
   /// each.
@@ -175,6 +207,9 @@ class SwapIndex : public SymbolIndex {
               llvm::function_ref<void(const Symbol &)>) const override;
   bool refs(const RefsRequest &,
             llvm::function_ref<void(const Ref &)>) const override;
+  bool containedRefs(
+      const ContainedRefsRequest &,
+      llvm::function_ref<void(const ContainedRefsResult &)>) const override;
   void relations(const RelationsRequest &,
                  llvm::function_ref<void(const SymbolID &, const Symbol &)>)
       const override;
diff --git a/clang-tools-extra/clangd/index/MemIndex.cpp b/clang-tools-extra/clangd/index/MemIndex.cpp
index 2665d46b97d83..9c9d3942bdee6 100644
--- a/clang-tools-extra/clangd/index/MemIndex.cpp
+++ b/clang-tools-extra/clangd/index/MemIndex.cpp
@@ -9,6 +9,7 @@
 #include "MemIndex.h"
 #include "FuzzyMatch.h"
 #include "Quality.h"
+#include "index/Index.h"
 #include "support/Trace.h"
 
 namespace clang {
@@ -85,6 +86,25 @@ bool MemIndex::refs(const RefsRequest &Req,
   return false; // We reported all refs.
 }
 
+bool MemIndex::containedRefs(
+    const ContainedRefsRequest &Req,
+    llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
+  trace::Span Tracer("MemIndex refersTo");
+  uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
+  for (const auto &Pair : Refs) {
+    for (const auto &R : Pair.second) {
+      if (!static_cast<int>(ContainedRefsRequest::SupportedRefKinds & R.Kind) ||
+          Req.ID != R.Container)
+        continue;
+      if (Remaining == 0)
+        return true; // More refs were available.
+      --Remaining;
+      Callback({R.Location, R.Kind, Pair.first});
+    }
+  }
+  return false; // We reported all refs.
+}
+
 void MemIndex::relations(
     const RelationsRequest &Req,
     llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
diff --git a/clang-tools-extra/clangd/index/MemIndex.h b/clang-tools-extra/clangd/index/MemIndex.h
index fba2c1a7120a2..fb1052b0c7ca8 100644
--- a/clang-tools-extra/clangd/index/MemIndex.h
+++ b/clang-tools-extra/clangd/index/MemIndex.h
@@ -72,6 +72,10 @@ class MemIndex : public SymbolIndex {
   bool refs(const RefsRequest &Req,
             llvm::function_ref<void(const Ref &)> Callback) const override;
 
+  bool containedRefs(const ContainedRefsRequest &Req,
+                     llvm::function_ref<void(const ContainedRefsResult &)>
+                         Callback) const override;
+
   void relations(const RelationsRequest &Req,
                  llvm::function_ref<void(const SymbolID &, const Symbol &)>
                      Callback) const override;
@@ -93,7 +97,7 @@ class MemIndex : public SymbolIndex {
   // Set of files which were used during this index build.
   llvm::StringSet<> Files;
   // Contents of the index (symbols, references, etc.)
-  IndexContents IdxContents;
+  IndexContents IdxContents = IndexContents::None;
   std::shared_ptr<void> KeepAlive; // poor man's move-only std::any
   // Size of memory retained by KeepAlive.
   size_t BackingDataSize = 0;
diff --git a/clang-tools-extra/clangd/index/Merge.cpp b/clang-tools-extra/clangd/index/Merge.cpp
index 8221d4b1f4440..aecca38a885b6 100644
--- a/clang-tools-extra/clangd/index/Merge.cpp
+++ b/clang-tools-extra/clangd/index/Merge.cpp
@@ -155,6 +155,40 @@ bool MergedIndex::refs(const RefsRequest &Req,
   return More || StaticHadMore;
 }
 
+bool MergedIndex::containedRefs(
+    const ContainedRefsRequest &Req,
+    llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
+  trace::Span Tracer("MergedIndex refersTo");
+  bool More = false;
+  uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
+  // We don't want duplicated refs from the static/dynamic indexes,
+  // and we can't reliably deduplicate them because offsets may differ slightly.
+  // We consider the dynamic index authoritative and report all its refs,
+  // and only report static index refs from other files.
+  More |= Dynamic->containedRefs(Req, [&](const auto &O) {
+    Callback(O);
+    assert(Remaining != 0);
+    --Remaining;
+  });
+  if (Remaining == 0 && More)
+    return More;
+  auto DynamicContainsFile = Dynamic->indexedFiles();
+  // We return less than Req.Limit if static index returns more refs for dirty
+  // files.
+  bool StaticHadMore = Static->containedRefs(Req, [&](const auto &O) {
+    if ((DynamicContainsFile(O.Location.FileURI) & IndexContents::References) !=
+        IndexContents::None)
+      return; // ignore refs that have been seen from dynamic index.
+    if (Remaining == 0) {
+      More = true;
+      return;
+    }
+    --Remaining;
+    Callback(O);
+  });
+  return More || StaticHadMore;
+}
+
 llvm::unique_function<IndexContents(llvm::StringRef) const>
 MergedIndex::indexedFiles() const {
   return [DynamicContainsFile{Dynamic->indexedFiles()},
diff --git a/clang-tools-extra/clangd/index/Merge.h b/clang-tools-extra/clangd/index/Merge.h
index b8a562b0df5d9..7441be6e57e85 100644
--- a/clang-tools-extra/clangd/index/Merge.h
+++ b/clang-tools-extra/clangd/index/Merge.h
@@ -38,6 +38,9 @@ class MergedIndex : public SymbolIndex {
               llvm::function_ref<void(const Symbol &)>) const override;
   bool refs(const RefsRequest &,
             llvm::function_ref<void(const Ref &)>) const override;
+  bool containedRefs(
+      const ContainedRefsRequest &,
+      llvm::function_ref<void(const ContainedRefsResult &)>) const override;
   void relations(const RelationsRequest &,
                  llvm::function_ref<void(const SymbolID &, const Symbol &)>)
       const override;
diff --git a/clang-tools-extra/clangd/index/ProjectAware.cpp b/clang-tools-extra/clangd/index/ProjectAware.cpp
index 2c6f8273b35d0..9836f0130362a 100644
--- a/clang-tools-extra/clangd/index/ProjectAware.cpp
+++ b/clang-tools-extra/clangd/index/ProjectAware.cpp
@@ -35,6 +35,10 @@ class ProjectAwareIndex : public SymbolIndex {
   /// Query all indexes while prioritizing the associated one (if any).
   bool refs(const RefsRequest &Req,
             llvm::function_ref<void(const Ref &)> Callback) const override;
+  /// Query all indexes while prioritizing the associated one (if any).
+  bool containedRefs(const ContainedRefsRequest &Req,
+                     llvm::function_ref<void(const ContainedRefsResult &)>
+                         Callback) const override;
 
   /// Queries only the associates index when Req.RestrictForCodeCompletion is
   /// set, otherwise queries all.
@@ -94,6 +98,15 @@ bool ProjectAwareIndex::refs(
   return false;
 }
 
+bool ProjectAwareIndex::containedRefs(
+    const ContainedRefsRequest &Req,
+    llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
+  trace::Span Tracer("ProjectAwareIndex::refersTo");
+  if (auto *Idx = getIndex())
+    return Idx->containedRefs(Req, Callback);
+  return false;
+}
+
 bool ProjectAwareIndex::fuzzyFind(
     const FuzzyFindRequest &Req,
     llvm::function_ref<void(const Symbol &)> Callback) const {
diff --git a/clang-tools-extra/clangd/index/Ref.h b/clang-tools-extra/clangd/index/Ref.h
index 6e383e2ade3d2..870f77f56e6cb 100644
--- a/clang-tools-extra/clangd/index/Ref.h
+++ b/clang-tools-extra/clangd/index/Ref.h
@@ -63,6 +63,9 @@ enum class RefKind : uint8_t {
   //   ^ this references Foo, but does not explicitly spell out its name
   // };
   Spelled = 1 << 3,
+  // A reference which is a call. Used as a filter for which references
+  // to store in data structures used for computing outgoing calls.
+  Call = 1 << 4,
   All = Declaration | Definition | Reference | Spelled,
 };
 
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 72a4e8b007668..f03839599612c 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -457,7 +457,7 @@ readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
 // The current versioning scheme is simple - non-current versions are rejected.
 // If you make a breaking change, bump this version number to invalidate stored
 // data. Later we may want to support some backward compatibility.
-constexpr static uint32_t Version = 19;
+constexpr static uint32_t Version = 20;
 
 llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
                                      SymbolOrigin Origin) {
@@ -704,7 +704,8 @@ llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data,
 }
 
 std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
-                                       SymbolOrigin Origin, bool UseDex) {
+                                       SymbolOrigin Origin, bool UseDex,
+                                       bool SupportContainedRefs) {
   trace::Span OverallTracer("LoadIndex");
   auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
   if (!Buffer) {
@@ -735,10 +736,11 @@ std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
   size_t NumRelations = Relations.size();
 
   trace::Span Tracer("BuildIndex");
-  auto Index = UseDex ? dex::Dex::build(std::move(Symbols), std::move(Refs),
-                                        std::move(Relations))
-                      : MemIndex::build(std::move(Symbols), std::move(Refs),
-                                        std::move(Relations));
+  auto Index = UseDex
+                   ? dex::Dex::build(std::move(Symbols), std::move(Refs),
+                                     std::move(Relations), SupportContainedRefs)
+                   : MemIndex::build(std::move(Symbols), std::move(Refs),
+                                     std::move(Relations));
   vlog("Loaded {0} from {1} with estimated memory usage {2} bytes\n"
        "  - number of symbols: {3}\n"
        "  - number of refs: {4}\n"
diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h
index b6890d63d2c38..bf8e036afcb6c 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -83,7 +83,8 @@ std::string toYAML(const Ref &);
 // Build an in-memory static index from an index file.
 // The size should be relatively small, so data can be managed in memory.
 std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename,
-                                       SymbolOrigin Origin, bool UseDex = true);
+                                       SymbolOrigin Origin, bool UseDex,
+                                       bool SupportContainedRefs);
 
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index 91ae9d3003a97..81125dbb1aeaf 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -18,6 +18,7 @@
 #include "clang-include-cleaner/Record.h"
 #include "clang-include-cleaner/Types.h"
 #include "index/CanonicalIncludes.h"
+#include "index/Ref.h"
 #include "index/Relation.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
@@ -660,7 +661,7 @@ bool SymbolCollector::handleDeclOccurrence(
     auto FileLoc = SM.getFileLoc(Loc);
     auto FID = SM.getFileID(FileLoc);
     if (Opts.RefsInHeaders || FID == SM.getMainFileID()) {
-      addRef(ID, SymbolRef{FileLoc, FID, Roles,
+      addRef(ID, SymbolRef{FileLoc, FID, Roles, index::getSymbolInfo(ND).Kind,
                            getRefContainer(ASTNode.Parent, Opts),
                            isSpelled(FileLoc, *ND)});
     }
@@ -774,8 +775,10 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
     // FIXME: Populate container information for macro references.
     // FIXME: All MacroRefs are marked as Spelled now, but this should be
     // checked.
-    addRef(ID, SymbolRef{Loc, SM.getFileID(Loc), Roles, /*Container=*/nullptr,
-                         /*Spelled=*/true});
+    addRef(ID,
+           SymbolRef{Loc, SM.getFileID(Loc), Roles, index::SymbolKind::Macro,
+                     /*Container=*/nullptr,
+                     /*Spelled=*/true});
   }
 
   // Collect symbols.
@@ -1166,6 +1169,14 @@ bool SymbolCollector::shouldIndexFile(FileID FID) {
   return I.first->second;
 }
 
+static bool refIsCall(index::SymbolKind Kind) {
+  using SK = index::SymbolKind;
+  return Kind == SK::Function || Kind == SK::InstanceMethod ||
+         Kind == SK::ClassMethod || Kind == SK::StaticMethod ||
+         Kind == SK::Constructor || Kind == SK::Destructor ||
+         Kind == SK::ConversionFunction;
+}
+
 void SymbolCollector::addRef(SymbolID ID, const SymbolRef &SR) {
   const auto &SM = ASTCtx->getSourceManager();
   // FIXME: use the result to filter out references.
@@ -1177,6 +1188,9 @@ void SymbolCollector::addRef(SymbolID ID, const SymbolRef &SR) {
     R.Location.End = Range.second;
     R.Location.FileURI = HeaderFileURIs->toURI(*FE).c_str();
     R.Kind = toRefKind(SR.Roles, SR.Spelled);
+    if (refIsCall(SR.Kind)) {
+      R.Kind |= RefKind::Call;
+    }
     R.Container = getSymbolIDCached(SR.Container);
     Refs.insert(ID, R);
   }
diff --git a/clang-tools-extra/clangd/index/SymbolCollector.h b/clang-tools-extra/clangd/index/SymbolCollector.h
index 6ff7a0145ff87..e9eb27fd0f664 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.h
+++ b/clang-tools-extra/clangd/index/SymbolCollector.h
@@ -209,6 +209,7 @@ class SymbolCollector : public index::IndexDataConsumer {
     SourceLocation Loc;
     FileID FID;
     index::SymbolRoleSet Roles;
+    index::SymbolKind Kind;
     const Decl *Container;
     bool Spelled;
   };
diff --git a/clang-tools-extra/clangd/index/dex/Dex.cpp b/clang-tools-extra/clangd/index/dex/Dex.cpp
index b7d3063e19b49..5643ba0c5e4ce 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.cpp
+++ b/clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -33,13 +33,14 @@ namespace clangd {
 namespace dex {
 
 std::unique_ptr<SymbolIndex> Dex::build(SymbolSlab Symbols, RefSlab Refs,
-                                        RelationSlab Rels) {
+                                        RelationSlab Rels,
+                                        bool SupportContainedRefs) {
   auto Size = Symbols.bytes() + Refs.bytes();
   // There is no need to include "Rels" in Data because the relations are self-
   // contained, without references into a backing store.
   auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
   return std::make_unique<Dex>(Data.first, Data.second, Rels, std::move(Data),
-                                Size);
+                               Size, SupportContainedRefs);
 }
 
 namespace {
@@ -120,7 +121,7 @@ class IndexBuilder {
 
 } // namespace
 
-void Dex::buildIndex() {
+void Dex::buildIndex(bool SupportContainedRefs) {
   this->Corpus = dex::Corpus(Symbols.size());
   std::vector<std::pair<float, const Symbol *>> ScoredSymbols(Symbols.size());
 
@@ -147,6 +148,20 @@ void Dex::buildIndex() {
   for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
     Builder.add(*Symbols[SymbolRank], SymbolRank);
   InvertedIndex = std::move(Builder).build();
+
+  // If the containedRefs() operation is supported, build the RevRefs
+  // data structure used to implement it.
+  if (!SupportContainedRefs)
+    return;
+  for (const auto &[ID, RefList] : Refs)
+    for (const auto &R : RefList)
+      if ((R.Kind & ContainedRefsRequest::SupportedRefKinds) !=
+          RefKind::Unknown)
+        RevRefs.emplace_back(R, ID);
+  // Sort by container ID so we can use binary search for lookup.
+  llvm::sort(RevRefs, [](const RevRef &A, const RevRef &B) {
+    return A.ref().Container < B.ref().Container;
+  });
 }
 
 std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
@@ -314,6 +329,36 @@ bool Dex::refs(const RefsRequest &Req,
   return false; // We reported all refs.
 }
 
+llvm::iterator_range<std::vector<Dex::RevRef>::const_iterator>
+Dex::lookupRevRefs(const SymbolID &Container) const {
+  // equal_range() requires an element of the same type as the elements of the
+  // range, so construct a dummy RevRef with the container of interest.
+  Ref QueryRef;
+  QueryRef.Container = Container;
+  RevRef Query(QueryRef, SymbolID{});
+
+  auto ItPair = std::equal_range(RevRefs.cbegin(), RevRefs.cend(), Query,
+                                 [](const RevRef &A, const RevRef &B) {
+                                   return A.ref().Container < B.ref().Container;
+                                 });
+  return {ItPair.first, ItPair.second};
+}
+
+bool Dex::containedRefs(
+    const ContainedRefsRequest &Req,
+    llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
+  trace::Span Tracer("Dex reversed refs");
+  uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
+  for (const auto &Rev : lookupRevRefs(Req.ID)) {
+    // RevRefs are already filtered to ContainedRefsRequest::SupportedRefKinds
+    if (Remaining == 0)
+      return true; // More refs were available.
+    --Remaining;
+    Callback(Rev.containedRefsResult());
+  }
+  return false; // We reported all refs.
+}
+
 void Dex::relations(
     const RelationsRequest &Req,
     llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
@@ -350,6 +395,7 @@ size_t Dex::estimateMemoryUsage() const {
   for (const auto &TokenToPostingList : InvertedIndex)
     Bytes += TokenToPostingList.second.bytes();
   Bytes += Refs.getMemorySize();
+  Bytes += RevRefs.size() * sizeof(RevRef);
   Bytes += Relations.getMemorySize();
   return Bytes + BackingDataSize;
 }
diff --git a/clang-tools-extra/clangd/index/dex/Dex.h b/clang-tools-extra/clangd/index/dex/Dex.h
index 69e161d51135b..502f597d81ef0 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.h
+++ b/clang-tools-extra/clangd/index/dex/Dex.h
@@ -36,7 +36,8 @@ class Dex : public SymbolIndex {
 public:
   // All data must outlive this index.
   template <typename SymbolRange, typename RefsRange, typename RelationsRange>
-  Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations)
+  Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
+      bool SupportContainedRefs)
       : Corpus(0) {
     for (auto &&Sym : Symbols)
       this->Symbols.push_back(&Sym);
@@ -46,15 +47,15 @@ class Dex : public SymbolIndex {
       this->Relations[std::make_pair(Rel.Subject,
                                      static_cast<uint8_t>(Rel.Predicate))]
           .push_back(Rel.Object);
-    buildIndex();
+    buildIndex(SupportContainedRefs);
   }
   // Symbols and Refs are owned by BackingData, Index takes ownership.
   template <typename SymbolRange, typename RefsRange, typename RelationsRange,
             typename Payload>
   Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
-      Payload &&BackingData, size_t BackingDataSize)
+      Payload &&BackingData, size_t BackingDataSize, bool SupportContainedRefs)
       : Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),
-            std::forward<RelationsRange>(Relations)) {
+            std::forward<RelationsRange>(Relations), SupportContainedRefs) {
     KeepAlive = std::shared_ptr<void>(
         std::make_shared<Payload>(std::move(BackingData)), nullptr);
     this->BackingDataSize = BackingDataSize;
@@ -64,16 +65,18 @@ class Dex : public SymbolIndex {
             typename FileRange, typename Payload>
   Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
       FileRange &&Files, IndexContents IdxContents, Payload &&BackingData,
-      size_t BackingDataSize)
+      size_t BackingDataSize, bool SupportContainedRefs)
       : Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),
             std::forward<RelationsRange>(Relations),
-            std::forward<Payload>(BackingData), BackingDataSize) {
+            std::forward<Payload>(BackingData), BackingDataSize,
+            SupportContainedRefs) {
     this->Files = std::forward<FileRange>(Files);
     this->IdxContents = IdxContents;
   }
 
   /// Builds an index from slabs. The index takes ownership of the slab.
-  static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab);
+  static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab,
+                                            bool SupportContainedRefs);
 
   bool
   fuzzyFind(const FuzzyFindRequest &Req,
@@ -85,6 +88,10 @@ class Dex : public SymbolIndex {
   bool refs(const RefsRequest &Req,
             llvm::function_ref<void(const Ref &)> Callback) const override;
 
+  bool containedRefs(const ContainedRefsRequest &Req,
+                     llvm::function_ref<void(const ContainedRefsResult &)>
+                         Callback) const override;
+
   void relations(const RelationsRequest &Req,
                  llvm::function_ref<void(const SymbolID &, const Symbol &)>
                      Callback) const override;
@@ -95,7 +102,22 @@ class Dex : public SymbolIndex {
   size_t estimateMemoryUsage() const override;
 
 private:
-  void buildIndex();
+  class RevRef {
+    const Ref *Reference;
+    SymbolID Target;
+
+  public:
+    RevRef(const Ref &Reference, SymbolID Target)
+        : Reference(&Reference), Target(Target) {}
+    const Ref &ref() const { return *Reference; }
+    ContainedRefsResult containedRefsResult() const {
+      return {ref().Location, ref().Kind, Target};
+    }
+  };
+
+  void buildIndex(bool EnableOutgoingCalls);
+  llvm::iterator_range<std::vector<RevRef>::const_iterator>
+  lookupRevRefs(const SymbolID &Container) const;
   std::unique_ptr<Iterator> iterator(const Token &Tok) const;
   std::unique_ptr<Iterator>
   createFileProximityIterator(llvm::ArrayRef<std::string> ProximityPaths) const;
@@ -116,6 +138,7 @@ class Dex : public SymbolIndex {
   llvm::DenseMap<Token, PostingList> InvertedIndex;
   dex::Corpus Corpus;
   llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> Refs;
+  std::vector<RevRef> RevRefs; // sorted by container ID
   static_assert(sizeof(RelationKind) == sizeof(uint8_t),
                 "RelationKind should be of same size as a uint8_t");
   llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>> Relations;
@@ -123,7 +146,9 @@ class Dex : public SymbolIndex {
   // Set of files which were used during this index build.
   llvm::StringSet<> Files;
   // Contents of the index (symbols, references, etc.)
-  IndexContents IdxContents;
+  // This is only populated if `Files` is, which applies to some but not all
+  // consumers of this class.
+  IndexContents IdxContents = IndexContents::None;
   // Size of memory retained by KeepAlive.
   size_t BackingDataSize = 0;
 };
diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index cea59ae409914..f185808ae1544 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -375,7 +375,8 @@ std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
   return Index.starts_with("remote:")
              ? remote::getClient(Index.drop_front(strlen("remote:")),
                                  ProjectRoot)
-             : loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true);
+             : loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true,
+                         /*SupportContainedRefs=*/true);
 }
 
 bool runCommand(std::string Request, const SymbolIndex &Index) {
diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index 391da3916259c..79b827126b4ef 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -146,6 +146,13 @@ class IndexClient : public clangd::SymbolIndex {
     return streamRPC(Request, &remote::v1::SymbolIndex::Stub::Refs, Callback);
   }
 
+  bool containedRefs(const clangd::ContainedRefsRequest &Request,
+                     llvm::function_ref<void(const ContainedRefsResult &)>
+                         Callback) const override {
+    return streamRPC(Request, &remote::v1::SymbolIndex::Stub::ContainedRefs,
+                     Callback);
+  }
+
   void
   relations(const clangd::RelationsRequest &Request,
             llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>
diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto
index 3072299d8f345..3e39724e32086 100644
--- a/clang-tools-extra/clangd/index/remote/Index.proto
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -131,3 +131,21 @@ message Relation {
   optional string subject_id = 1;
   optional Symbol object = 2;
 }
+
+message ContainedRefsRequest {
+  optional string id = 1;
+  optional uint32 limit = 2;
+}
+
+message ContainedRefsReply {
+  oneof kind {
+    ContainedRef stream_result = 1;
+    FinalResult final_result = 2;
+  }
+}
+
+message ContainedRef {
+  optional SymbolLocation location = 1;
+  optional uint32 kind = 2;
+  optional string symbol = 3;
+}
diff --git a/clang-tools-extra/clangd/index/remote/Service.proto b/clang-tools-extra/clangd/index/remote/Service.proto
index 7c7efa530200d..43023321cb9e1 100644
--- a/clang-tools-extra/clangd/index/remote/Service.proto
+++ b/clang-tools-extra/clangd/index/remote/Service.proto
@@ -21,5 +21,7 @@ service SymbolIndex {
 
   rpc Refs(RefsRequest) returns (stream RefsReply) {}
 
+  rpc ContainedRefs(ContainedRefsRequest) returns (stream ContainedRefsReply) {}
+
   rpc Relations(RelationsRequest) returns (stream RelationsReply) {}
 }
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index 7e31ada18a657..d8d3b64a5ac18 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -126,6 +126,20 @@ Marshaller::fromProtobuf(const RefsRequest *Message) {
   return Req;
 }
 
+llvm::Expected<clangd::ContainedRefsRequest>
+Marshaller::fromProtobuf(const ContainedRefsRequest *Message) {
+  clangd::ContainedRefsRequest Req;
+  if (!Message->has_id())
+    return error("ContainedRefsRequest requires an id.");
+  auto ID = SymbolID::fromStr(Message->id());
+  if (!ID)
+    return ID.takeError();
+  Req.ID = *ID;
+  if (Message->has_limit())
+    Req.Limit = Message->limit();
+  return Req;
+}
+
 llvm::Expected<clangd::RelationsRequest>
 Marshaller::fromProtobuf(const RelationsRequest *Message) {
   clangd::RelationsRequest Req;
@@ -192,6 +206,27 @@ llvm::Expected<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
   return Result;
 }
 
+llvm::Expected<clangd::ContainedRefsResult>
+Marshaller::fromProtobuf(const ContainedRef &Message) {
+  clangd::ContainedRefsResult Result;
+  if (!Message.has_location())
+    return error("ContainedRef must have a location.");
+  if (!Message.has_kind())
+    return error("ContainedRef must have a kind.");
+  if (!Message.has_symbol())
+    return error("ContainedRef must have a symbol.");
+  auto Location = fromProtobuf(Message.location());
+  if (!Location)
+    return Location.takeError();
+  Result.Location = *Location;
+  Result.Kind = static_cast<RefKind>(Message.kind());
+  auto Symbol = SymbolID::fromStr(Message.symbol());
+  if (!Symbol)
+    return Symbol.takeError();
+  Result.Symbol = *Symbol;
+  return Result;
+}
+
 llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
 Marshaller::fromProtobuf(const Relation &Message) {
   auto SubjectID = SymbolID::fromStr(Message.subject_id());
@@ -244,6 +279,15 @@ RefsRequest Marshaller::toProtobuf(const clangd::RefsRequest &From) {
   return RPCRequest;
 }
 
+ContainedRefsRequest
+Marshaller::toProtobuf(const clangd::ContainedRefsRequest &From) {
+  ContainedRefsRequest RPCRequest;
+  RPCRequest.set_id(From.ID.str());
+  if (From.Limit)
+    RPCRequest.set_limit(*From.Limit);
+  return RPCRequest;
+}
+
 RelationsRequest Marshaller::toProtobuf(const clangd::RelationsRequest &From) {
   RelationsRequest RPCRequest;
   for (const auto &ID : From.Subjects)
@@ -299,6 +343,18 @@ llvm::Expected<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
   return Result;
 }
 
+llvm::Expected<ContainedRef>
+Marshaller::toProtobuf(const clangd::ContainedRefsResult &From) {
+  ContainedRef Result;
+  auto Location = toProtobuf(From.Location);
+  if (!Location)
+    return Location.takeError();
+  *Result.mutable_location() = *Location;
+  Result.set_kind(static_cast<uint32_t>(From.Kind));
+  *Result.mutable_symbol() = From.Symbol.str();
+  return Result;
+}
+
 llvm::Expected<Relation> Marshaller::toProtobuf(const clangd::SymbolID &Subject,
                                                 const clangd::Symbol &Object) {
   Relation Result;
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index e827b4c155a20..5bee9205aef58 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -40,6 +40,8 @@ class Marshaller {
 
   llvm::Expected<clangd::Symbol> fromProtobuf(const Symbol &Message);
   llvm::Expected<clangd::Ref> fromProtobuf(const Ref &Message);
+  llvm::Expected<clangd::ContainedRefsResult>
+  fromProtobuf(const ContainedRef &Message);
   llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
   fromProtobuf(const Relation &Message);
 
@@ -48,6 +50,8 @@ class Marshaller {
   llvm::Expected<clangd::FuzzyFindRequest>
   fromProtobuf(const FuzzyFindRequest *Message);
   llvm::Expected<clangd::RefsRequest> fromProtobuf(const RefsRequest *Message);
+  llvm::Expected<clangd::ContainedRefsRequest>
+  fromProtobuf(const ContainedRefsRequest *Message);
   llvm::Expected<clangd::RelationsRequest>
   fromProtobuf(const RelationsRequest *Message);
 
@@ -58,10 +62,13 @@ class Marshaller {
   LookupRequest toProtobuf(const clangd::LookupRequest &From);
   FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
   RefsRequest toProtobuf(const clangd::RefsRequest &From);
+  ContainedRefsRequest toProtobuf(const clangd::ContainedRefsRequest &From);
   RelationsRequest toProtobuf(const clangd::RelationsRequest &From);
 
   llvm::Expected<Symbol> toProtobuf(const clangd::Symbol &From);
   llvm::Expected<Ref> toProtobuf(const clangd::Ref &From);
+  llvm::Expected<ContainedRef>
+  toProtobuf(const clangd::ContainedRefsResult &From);
   llvm::Expected<Relation> toProtobuf(const clangd::SymbolID &Subject,
                                       const clangd::Symbol &Object);
 
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 52fca53260a16..890b6c27ed928 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -258,6 +258,53 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
     return grpc::Status::OK;
   }
 
+  grpc::Status
+  ContainedRefs(grpc::ServerContext *Context,
+                const ContainedRefsRequest *Request,
+                grpc::ServerWriter<ContainedRefsReply> *Reply) override {
+    auto StartTime = stopwatch::now();
+    WithContextValue WithRequestContext(CurrentRequest, Context);
+    logRequest(*Request);
+    trace::Span Tracer("ContainedRefsRequest");
+    auto Req = ProtobufMarshaller->fromProtobuf(Request);
+    if (!Req) {
+      elog("Can not parse ContainedRefsRequest from protobuf: {0}",
+           Req.takeError());
+      return grpc::Status::CANCELLED;
+    }
+    if (!Req->Limit || *Req->Limit > LimitResults) {
+      log("[public] Limiting result size for ContainedRefs request from {0} to "
+          "{1}.",
+          Req->Limit, LimitResults);
+      Req->Limit = LimitResults;
+    }
+    unsigned Sent = 0;
+    unsigned FailedToSend = 0;
+    bool HasMore =
+        Index.containedRefs(*Req, [&](const clangd::ContainedRefsResult &Item) {
+          auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
+          if (!SerializedItem) {
+            elog("Unable to convert ContainedRefsResult to protobuf: {0}",
+                 SerializedItem.takeError());
+            ++FailedToSend;
+            return;
+          }
+          ContainedRefsReply NextMessage;
+          *NextMessage.mutable_stream_result() = *SerializedItem;
+          logResponse(NextMessage);
+          Reply->Write(NextMessage);
+          ++Sent;
+        });
+    ContainedRefsReply LastMessage;
+    LastMessage.mutable_final_result()->set_has_more(HasMore);
+    logResponse(LastMessage);
+    Reply->Write(LastMessage);
+    SPAN_ATTACH(Tracer, "Sent", Sent);
+    SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    logRequestSummary("v1/ContainedRefs", Sent, StartTime);
+    return grpc::Status::OK;
+  }
+
   grpc::Status Relations(grpc::ServerContext *Context,
                          const RelationsRequest *Request,
                          grpc::ServerWriter<RelationsReply> *Reply) override {
@@ -396,7 +443,8 @@ void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath,
        LastStatus.getLastModificationTime(), Status->getLastModificationTime());
   LastStatus = *Status;
   std::unique_ptr<clang::clangd::SymbolIndex> NewIndex =
-      loadIndex(IndexPath, SymbolOrigin::Static);
+      loadIndex(IndexPath, SymbolOrigin::Static, /*UseDex=*/true,
+                /*SupportContainedRefs=*/true);
   if (!NewIndex) {
     elog("Failed to load new index. Old index will be served.");
     return;
@@ -532,8 +580,9 @@ int main(int argc, char *argv[]) {
     return Status.getError().value();
   }
 
-  auto SymIndex =
-      clang::clangd::loadIndex(IndexPath, clang::clangd::SymbolOrigin::Static);
+  auto SymIndex = clang::clangd::loadIndex(
+      IndexPath, clang::clangd::SymbolOrigin::Static, /*UseDex=*/true,
+      /*SupportContainedRefs=*/true);
   if (!SymIndex) {
     llvm::errs() << "Failed to open the index.\n";
     return -1;
diff --git a/clang-tools-extra/clangd/support/DirectiveTree.cpp b/clang-tools-extra/clangd/support/DirectiveTree.cpp
index d25da111681af..7ea08add7a107 100644
--- a/clang-tools-extra/clangd/support/DirectiveTree.cpp
+++ b/clang-tools-extra/clangd/support/DirectiveTree.cpp
@@ -328,6 +328,9 @@ class Preprocessor {
   Preprocessor(const TokenStream &In, TokenStream &Out) : In(In), Out(Out) {}
   ~Preprocessor() { Out.finalize(); }
 
+  Preprocessor(const Preprocessor &other) = delete;
+  Preprocessor &operator=(const Preprocessor &other) = delete;
+
   void walk(const DirectiveTree &T) {
     for (const auto &C : T.Chunks)
       std::visit(*this, C);
diff --git a/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx b/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx
index 0c04df86ae1c6..6368e7145b1e4 100644
Binary files a/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx and b/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx differ
diff --git a/clang-tools-extra/clangd/test/type-hierarchy-ext.test b/clang-tools-extra/clangd/test/type-hierarchy-ext.test
index ddb9a014be0c7..8d1a5dc31da0f 100644
--- a/clang-tools-extra/clangd/test/type-hierarchy-ext.test
+++ b/clang-tools-extra/clangd/test/type-hierarchy-ext.test
@@ -12,6 +12,7 @@
 # CHECK-NEXT:        "data": {
 # CHECK-NEXT:           "symbolID": "A6576FE083F2949A"
 # CHECK-NEXT:        },
+# CHECK-NEXT:        "detail": "Child3",
 # CHECK-NEXT:        "kind": 23,
 # CHECK-NEXT:        "name": "Child3",
 # CHECK-NEXT:        "range": {
@@ -153,6 +154,7 @@
 # CHECK-NEXT:        "data": {
 # CHECK-NEXT:          "symbolID": "5705B382DFC77CBC"
 # CHECK-NEXT:        },
+# CHECK-NEXT:        "detail": "Child4",
 # CHECK-NEXT:        "kind": 23,
 # CHECK-NEXT:        "name": "Child4",
 # CHECK-NEXT:        "range": {
diff --git a/clang-tools-extra/clangd/test/type-hierarchy.test b/clang-tools-extra/clangd/test/type-hierarchy.test
index 69751000a7c6c..a5f13ab13d0b3 100644
--- a/clang-tools-extra/clangd/test/type-hierarchy.test
+++ b/clang-tools-extra/clangd/test/type-hierarchy.test
@@ -62,6 +62,7 @@
 # CHECK-NEXT:         ],
 # CHECK-NEXT:         "symbolID": "ECDC0C46D75120F4"
 # CHECK-NEXT:       },
+# CHECK-NEXT:       "detail": "Child1",
 # CHECK-NEXT:       "kind": 23,
 # CHECK-NEXT:       "name": "Child1",
 # CHECK-NEXT:       "range": {
@@ -112,6 +113,7 @@
 # CHECK-NEXT:        ],
 # CHECK-NEXT:        "symbolID": "A6576FE083F2949A"
 # CHECK-NEXT:       },
+# CHECK-NEXT:       "detail": "Child3",
 # CHECK-NEXT:       "kind": 23,
 # CHECK-NEXT:       "name": "Child3",
 # CHECK-NEXT:       "range": {
diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp
index bc2eaa77a66ee..df8d075e80596 100644
--- a/clang-tools-extra/clangd/tool/Check.cpp
+++ b/clang-tools-extra/clangd/tool/Check.cpp
@@ -163,7 +163,7 @@ class Checker {
   unsigned ErrCount = 0;
 
   Checker(llvm::StringRef File, const ClangdLSPServer::Options &Opts)
-      : File(File), Opts(Opts) {}
+      : File(File), Opts(Opts), Index(/*SupportContainedRefs=*/true) {}
 
   // Read compilation database and choose a compile command for the file.
   bool buildCommand(const ThreadsafeFS &TFS) {
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index cc061e2d93231..80a0653f8f740 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -604,7 +604,7 @@ const char TestScheme::TestDir[] = "/clangd-test";
 
 std::unique_ptr<SymbolIndex>
 loadExternalIndex(const Config::ExternalIndexSpec &External,
-                  AsyncTaskRunner *Tasks) {
+                  AsyncTaskRunner *Tasks, bool SupportContainedRefs) {
   static const trace::Metric RemoteIndexUsed("used_remote_index",
                                              trace::Metric::Value, "address");
   switch (External.Kind) {
@@ -620,8 +620,9 @@ loadExternalIndex(const Config::ExternalIndexSpec &External,
         External.Location);
     auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
     auto IndexLoadTask = [File = External.Location,
-                          PlaceHolder = NewIndex.get()] {
-      if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true))
+                          PlaceHolder = NewIndex.get(), SupportContainedRefs] {
+      if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true,
+                               SupportContainedRefs))
         PlaceHolder->reset(std::move(Idx));
     };
     if (Tasks) {
@@ -909,7 +910,12 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
   Opts.BackgroundIndexPriority = BackgroundIndexPriority;
   Opts.ReferencesLimit = ReferencesLimit;
   Opts.Rename.LimitFiles = RenameFileLimit;
-  auto PAI = createProjectAwareIndex(loadExternalIndex, Sync);
+  auto PAI = createProjectAwareIndex(
+      [SupportContainedRefs = Opts.EnableOutgoingCalls](
+          const Config::ExternalIndexSpec &External, AsyncTaskRunner *Tasks) {
+        return loadExternalIndex(External, Tasks, SupportContainedRefs);
+      },
+      Sync);
   Opts.StaticIndex = PAI.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.MemoryCleanup = getMemoryCleanupFunction();
diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
index e51942462fbdf..ada14c9939318 100644
--- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -685,7 +685,8 @@ TEST_F(BackgroundIndexTest, Reindex) {
 class BackgroundIndexRebuilderTest : public testing::Test {
 protected:
   BackgroundIndexRebuilderTest()
-      : Source(IndexContents::All), Target(std::make_unique<MemIndex>()),
+      : Source(IndexContents::All, /*SupportContainedRefs=*/true),
+        Target(std::make_unique<MemIndex>()),
         Rebuilder(&Target, &Source, /*Threads=*/10) {
     // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
     TestSymbol.ID = SymbolID("foo");
diff --git a/clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp b/clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp
index 8821d3aad9c78..316b94305c9ae 100644
--- a/clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp
@@ -44,17 +44,27 @@ using ::testing::UnorderedElementsAre;
 
 // Helpers for matching call hierarchy data structures.
 MATCHER_P(withName, N, "") { return arg.name == N; }
+MATCHER_P(withDetail, N, "") { return arg.detail == N; }
 MATCHER_P(withSelectionRange, R, "") { return arg.selectionRange == R; }
 
 template <class ItemMatcher>
 ::testing::Matcher<CallHierarchyIncomingCall> from(ItemMatcher M) {
   return Field(&CallHierarchyIncomingCall::from, M);
 }
+template <class ItemMatcher>
+::testing::Matcher<CallHierarchyOutgoingCall> to(ItemMatcher M) {
+  return Field(&CallHierarchyOutgoingCall::to, M);
+}
 template <class... RangeMatchers>
-::testing::Matcher<CallHierarchyIncomingCall> fromRanges(RangeMatchers... M) {
+::testing::Matcher<CallHierarchyIncomingCall> iFromRanges(RangeMatchers... M) {
   return Field(&CallHierarchyIncomingCall::fromRanges,
                UnorderedElementsAre(M...));
 }
+template <class... RangeMatchers>
+::testing::Matcher<CallHierarchyOutgoingCall> oFromRanges(RangeMatchers... M) {
+  return Field(&CallHierarchyOutgoingCall::fromRanges,
+               UnorderedElementsAre(M...));
+}
 
 TEST(CallHierarchy, IncomingOneFileCpp) {
   Annotations Source(R"cpp(
@@ -79,21 +89,24 @@ TEST(CallHierarchy, IncomingOneFileCpp) {
       prepareCallHierarchy(AST, Source.point(), testPath(TU.Filename));
   ASSERT_THAT(Items, ElementsAre(withName("callee")));
   auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
-  ASSERT_THAT(IncomingLevel1,
-              ElementsAre(AllOf(from(withName("caller1")),
-                                fromRanges(Source.range("Callee")))));
+  ASSERT_THAT(
+      IncomingLevel1,
+      ElementsAre(AllOf(from(AllOf(withName("caller1"), withDetail("caller1"))),
+                        iFromRanges(Source.range("Callee")))));
   auto IncomingLevel2 = incomingCalls(IncomingLevel1[0].from, Index.get());
-  ASSERT_THAT(IncomingLevel2,
-              ElementsAre(AllOf(from(withName("caller2")),
-                                fromRanges(Source.range("Caller1A"),
-                                           Source.range("Caller1B"))),
-                          AllOf(from(withName("caller3")),
-                                fromRanges(Source.range("Caller1C")))));
+  ASSERT_THAT(
+      IncomingLevel2,
+      ElementsAre(AllOf(from(AllOf(withName("caller2"), withDetail("caller2"))),
+                        iFromRanges(Source.range("Caller1A"),
+                                    Source.range("Caller1B"))),
+                  AllOf(from(AllOf(withName("caller3"), withDetail("caller3"))),
+                        iFromRanges(Source.range("Caller1C")))));
 
   auto IncomingLevel3 = incomingCalls(IncomingLevel2[0].from, Index.get());
-  ASSERT_THAT(IncomingLevel3,
-              ElementsAre(AllOf(from(withName("caller3")),
-                                fromRanges(Source.range("Caller2")))));
+  ASSERT_THAT(
+      IncomingLevel3,
+      ElementsAre(AllOf(from(AllOf(withName("caller3"), withDetail("caller3"))),
+                        iFromRanges(Source.range("Caller2")))));
 
   auto IncomingLevel4 = incomingCalls(IncomingLevel3[0].from, Index.get());
   EXPECT_THAT(IncomingLevel4, IsEmpty());
@@ -125,20 +138,24 @@ TEST(CallHierarchy, IncomingOneFileObjC) {
   ASSERT_THAT(Items, ElementsAre(withName("callee")));
   auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
   ASSERT_THAT(IncomingLevel1,
-              ElementsAre(AllOf(from(withName("caller1")),
-                                fromRanges(Source.range("Callee")))));
+              ElementsAre(AllOf(from(AllOf(withName("caller1"),
+                                           withDetail("MyClass::caller1"))),
+                                iFromRanges(Source.range("Callee")))));
   auto IncomingLevel2 = incomingCalls(IncomingLevel1[0].from, Index.get());
   ASSERT_THAT(IncomingLevel2,
-              ElementsAre(AllOf(from(withName("caller2")),
-                                fromRanges(Source.range("Caller1A"),
-                                           Source.range("Caller1B"))),
-                          AllOf(from(withName("caller3")),
-                                fromRanges(Source.range("Caller1C")))));
+              ElementsAre(AllOf(from(AllOf(withName("caller2"),
+                                           withDetail("MyClass::caller2"))),
+                                iFromRanges(Source.range("Caller1A"),
+                                            Source.range("Caller1B"))),
+                          AllOf(from(AllOf(withName("caller3"),
+                                           withDetail("MyClass::caller3"))),
+                                iFromRanges(Source.range("Caller1C")))));
 
   auto IncomingLevel3 = incomingCalls(IncomingLevel2[0].from, Index.get());
   ASSERT_THAT(IncomingLevel3,
-              ElementsAre(AllOf(from(withName("caller3")),
-                                fromRanges(Source.range("Caller2")))));
+              ElementsAre(AllOf(from(AllOf(withName("caller3"),
+                                           withDetail("MyClass::caller3"))),
+                                iFromRanges(Source.range("Caller2")))));
 
   auto IncomingLevel4 = incomingCalls(IncomingLevel3[0].from, Index.get());
   EXPECT_THAT(IncomingLevel4, IsEmpty());
@@ -167,14 +184,16 @@ TEST(CallHierarchy, MainFileOnlyRef) {
       prepareCallHierarchy(AST, Source.point(), testPath(TU.Filename));
   ASSERT_THAT(Items, ElementsAre(withName("callee")));
   auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
-  ASSERT_THAT(IncomingLevel1,
-              ElementsAre(AllOf(from(withName("caller1")),
-                                fromRanges(Source.range("Callee")))));
+  ASSERT_THAT(
+      IncomingLevel1,
+      ElementsAre(AllOf(from(AllOf(withName("caller1"), withDetail("caller1"))),
+                        iFromRanges(Source.range("Callee")))));
 
   auto IncomingLevel2 = incomingCalls(IncomingLevel1[0].from, Index.get());
-  EXPECT_THAT(IncomingLevel2,
-              ElementsAre(AllOf(from(withName("caller2")),
-                                fromRanges(Source.range("Caller1")))));
+  EXPECT_THAT(
+      IncomingLevel2,
+      ElementsAre(AllOf(from(AllOf(withName("caller2"), withDetail("caller2"))),
+                        iFromRanges(Source.range("Caller1")))));
 }
 
 TEST(CallHierarchy, IncomingQualified) {
@@ -200,14 +219,72 @@ TEST(CallHierarchy, IncomingQualified) {
       prepareCallHierarchy(AST, Source.point(), testPath(TU.Filename));
   ASSERT_THAT(Items, ElementsAre(withName("Waldo::find")));
   auto Incoming = incomingCalls(Items[0], Index.get());
-  EXPECT_THAT(Incoming,
-              ElementsAre(AllOf(from(withName("caller1")),
-                                fromRanges(Source.range("Caller1"))),
-                          AllOf(from(withName("caller2")),
-                                fromRanges(Source.range("Caller2")))));
+  EXPECT_THAT(
+      Incoming,
+      ElementsAre(
+          AllOf(from(AllOf(withName("caller1"), withDetail("ns::caller1"))),
+                iFromRanges(Source.range("Caller1"))),
+          AllOf(from(AllOf(withName("caller2"), withDetail("ns::caller2"))),
+                iFromRanges(Source.range("Caller2")))));
 }
 
-TEST(CallHierarchy, IncomingMultiFileCpp) {
+TEST(CallHierarchy, OutgoingOneFile) {
+  // Test outgoing call on the main file, with namespaces and methods
+  Annotations Source(R"cpp(
+    void callee(int);
+    namespace ns {
+      struct Foo {
+        void caller1();
+      };
+      void Foo::caller1() {
+        $Callee[[callee]](42);
+      }
+    }
+    namespace {
+      void caller2(ns::Foo& F) {
+        F.$Caller1A[[caller1]]();
+        F.$Caller1B[[caller1]]();
+      }
+    }
+    void call^er3(ns::Foo& F) {
+      F.$Caller1C[[caller1]]();
+      $Caller2[[caller2]](F);
+    }
+  )cpp");
+  TestTU TU = TestTU::withCode(Source.code());
+  auto AST = TU.build();
+  auto Index = TU.index();
+
+  std::vector<CallHierarchyItem> Items =
+      prepareCallHierarchy(AST, Source.point(), testPath(TU.Filename));
+  ASSERT_THAT(Items, ElementsAre(withName("caller3")));
+  auto OugoingLevel1 = outgoingCalls(Items[0], Index.get());
+  ASSERT_THAT(
+      OugoingLevel1,
+      ElementsAre(
+          AllOf(to(AllOf(withName("caller1"), withDetail("ns::Foo::caller1"))),
+                oFromRanges(Source.range("Caller1C"))),
+          AllOf(to(AllOf(withName("caller2"), withDetail("caller2"))),
+                oFromRanges(Source.range("Caller2")))));
+
+  auto OutgoingLevel2 = outgoingCalls(OugoingLevel1[1].to, Index.get());
+  ASSERT_THAT(
+      OutgoingLevel2,
+      ElementsAre(AllOf(
+          to(AllOf(withName("caller1"), withDetail("ns::Foo::caller1"))),
+          oFromRanges(Source.range("Caller1A"), Source.range("Caller1B")))));
+
+  auto OutgoingLevel3 = outgoingCalls(OutgoingLevel2[0].to, Index.get());
+  ASSERT_THAT(
+      OutgoingLevel3,
+      ElementsAre(AllOf(to(AllOf(withName("callee"), withDetail("callee"))),
+                        oFromRanges(Source.range("Callee")))));
+
+  auto OutgoingLevel4 = outgoingCalls(OutgoingLevel3[0].to, Index.get());
+  EXPECT_THAT(OutgoingLevel4, IsEmpty());
+}
+
+TEST(CallHierarchy, MultiFileCpp) {
   // The test uses a .hh suffix for header files to get clang
   // to parse them in C++ mode. .h files are parsed in C mode
   // by default, which causes problems because e.g. symbol
@@ -221,32 +298,47 @@ TEST(CallHierarchy, IncomingMultiFileCpp) {
     void calle^e(int) {}
   )cpp");
   Annotations Caller1H(R"cpp(
-    void caller1();
+    namespace nsa {
+      void caller1();
+    }
   )cpp");
   Annotations Caller1C(R"cpp(
     #include "callee.hh"
     #include "caller1.hh"
-    void caller1() {
-      [[calle^e]](42);
+    namespace nsa {
+      void caller1() {
+        [[calle^e]](42);
+      }
     }
   )cpp");
   Annotations Caller2H(R"cpp(
-    void caller2();
+    namespace nsb {
+      void caller2();
+    }
   )cpp");
   Annotations Caller2C(R"cpp(
     #include "caller1.hh"
     #include "caller2.hh"
-    void caller2() {
-      $A[[caller1]]();
-      $B[[caller1]]();
+    namespace nsb {
+      void caller2() {
+        nsa::$A[[caller1]]();
+        nsa::$B[[caller1]]();
+      }
+    }
+  )cpp");
+  Annotations Caller3H(R"cpp(
+    namespace nsa {
+      void call^er3();
     }
   )cpp");
   Annotations Caller3C(R"cpp(
     #include "caller1.hh"
     #include "caller2.hh"
-    void caller3() {
-      $Caller1[[caller1]]();
-      $Caller2[[caller2]]();
+    namespace nsa {
+      void call^er3() {
+        $Caller1[[caller1]]();
+        nsb::$Caller2[[caller2]]();
+      }
     }
   )cpp");
 
@@ -254,6 +346,7 @@ TEST(CallHierarchy, IncomingMultiFileCpp) {
   Workspace.addSource("callee.hh", CalleeH.code());
   Workspace.addSource("caller1.hh", Caller1H.code());
   Workspace.addSource("caller2.hh", Caller2H.code());
+  Workspace.addSource("caller3.hh", Caller3H.code());
   Workspace.addMainFile("callee.cc", CalleeC.code());
   Workspace.addMainFile("caller1.cc", Caller1C.code());
   Workspace.addMainFile("caller2.cc", Caller2C.code());
@@ -261,46 +354,84 @@ TEST(CallHierarchy, IncomingMultiFileCpp) {
 
   auto Index = Workspace.index();
 
-  auto CheckCallHierarchy = [&](ParsedAST &AST, Position Pos, PathRef TUPath) {
+  auto CheckIncomingCalls = [&](ParsedAST &AST, Position Pos, PathRef TUPath) {
     std::vector<CallHierarchyItem> Items =
         prepareCallHierarchy(AST, Pos, TUPath);
     ASSERT_THAT(Items, ElementsAre(withName("callee")));
     auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
     ASSERT_THAT(IncomingLevel1,
-                ElementsAre(AllOf(from(withName("caller1")),
-                                  fromRanges(Caller1C.range()))));
+                ElementsAre(AllOf(from(AllOf(withName("caller1"),
+                                             withDetail("nsa::caller1"))),
+                                  iFromRanges(Caller1C.range()))));
 
     auto IncomingLevel2 = incomingCalls(IncomingLevel1[0].from, Index.get());
     ASSERT_THAT(
         IncomingLevel2,
-        ElementsAre(AllOf(from(withName("caller2")),
-                          fromRanges(Caller2C.range("A"), Caller2C.range("B"))),
-                    AllOf(from(withName("caller3")),
-                          fromRanges(Caller3C.range("Caller1")))));
+        ElementsAre(
+            AllOf(from(AllOf(withName("caller2"), withDetail("nsb::caller2"))),
+                  iFromRanges(Caller2C.range("A"), Caller2C.range("B"))),
+            AllOf(from(AllOf(withName("caller3"), withDetail("nsa::caller3"))),
+                  iFromRanges(Caller3C.range("Caller1")))));
 
     auto IncomingLevel3 = incomingCalls(IncomingLevel2[0].from, Index.get());
     ASSERT_THAT(IncomingLevel3,
-                ElementsAre(AllOf(from(withName("caller3")),
-                                  fromRanges(Caller3C.range("Caller2")))));
+                ElementsAre(AllOf(from(AllOf(withName("caller3"),
+                                             withDetail("nsa::caller3"))),
+                                  iFromRanges(Caller3C.range("Caller2")))));
 
     auto IncomingLevel4 = incomingCalls(IncomingLevel3[0].from, Index.get());
     EXPECT_THAT(IncomingLevel4, IsEmpty());
   };
 
+  auto CheckOutgoingCalls = [&](ParsedAST &AST, Position Pos, PathRef TUPath) {
+    std::vector<CallHierarchyItem> Items =
+        prepareCallHierarchy(AST, Pos, TUPath);
+    ASSERT_THAT(Items, ElementsAre(withName("caller3")));
+    auto OutgoingLevel1 = outgoingCalls(Items[0], Index.get());
+    ASSERT_THAT(
+        OutgoingLevel1,
+        ElementsAre(
+            AllOf(to(AllOf(withName("caller1"), withDetail("nsa::caller1"))),
+                  oFromRanges(Caller3C.range("Caller1"))),
+            AllOf(to(AllOf(withName("caller2"), withDetail("nsb::caller2"))),
+                  oFromRanges(Caller3C.range("Caller2")))));
+
+    auto OutgoingLevel2 = outgoingCalls(OutgoingLevel1[1].to, Index.get());
+    ASSERT_THAT(OutgoingLevel2,
+                ElementsAre(AllOf(
+                    to(AllOf(withName("caller1"), withDetail("nsa::caller1"))),
+                    oFromRanges(Caller2C.range("A"), Caller2C.range("B")))));
+
+    auto OutgoingLevel3 = outgoingCalls(OutgoingLevel2[0].to, Index.get());
+    ASSERT_THAT(
+        OutgoingLevel3,
+        ElementsAre(AllOf(to(AllOf(withName("callee"), withDetail("callee"))),
+                          oFromRanges(Caller1C.range()))));
+
+    auto OutgoingLevel4 = outgoingCalls(OutgoingLevel3[0].to, Index.get());
+    EXPECT_THAT(OutgoingLevel4, IsEmpty());
+  };
+
   // Check that invoking from a call site works.
   auto AST = Workspace.openFile("caller1.cc");
   ASSERT_TRUE(bool(AST));
-  CheckCallHierarchy(*AST, Caller1C.point(), testPath("caller1.cc"));
+  CheckIncomingCalls(*AST, Caller1C.point(), testPath("caller1.cc"));
 
   // Check that invoking from the declaration site works.
   AST = Workspace.openFile("callee.hh");
   ASSERT_TRUE(bool(AST));
-  CheckCallHierarchy(*AST, CalleeH.point(), testPath("callee.hh"));
+  CheckIncomingCalls(*AST, CalleeH.point(), testPath("callee.hh"));
+  AST = Workspace.openFile("caller3.hh");
+  ASSERT_TRUE(bool(AST));
+  CheckOutgoingCalls(*AST, Caller3H.point(), testPath("caller3.hh"));
 
   // Check that invoking from the definition site works.
   AST = Workspace.openFile("callee.cc");
   ASSERT_TRUE(bool(AST));
-  CheckCallHierarchy(*AST, CalleeC.point(), testPath("callee.cc"));
+  CheckIncomingCalls(*AST, CalleeC.point(), testPath("callee.cc"));
+  AST = Workspace.openFile("caller3.cc");
+  ASSERT_TRUE(bool(AST));
+  CheckOutgoingCalls(*AST, Caller3C.point(), testPath("caller3.cc"));
 }
 
 TEST(CallHierarchy, IncomingMultiFileObjC) {
@@ -377,20 +508,20 @@ TEST(CallHierarchy, IncomingMultiFileObjC) {
     auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
     ASSERT_THAT(IncomingLevel1,
                 ElementsAre(AllOf(from(withName("caller1")),
-                                  fromRanges(Caller1C.range()))));
+                                  iFromRanges(Caller1C.range()))));
 
     auto IncomingLevel2 = incomingCalls(IncomingLevel1[0].from, Index.get());
-    ASSERT_THAT(
-        IncomingLevel2,
-        ElementsAre(AllOf(from(withName("caller2")),
-                          fromRanges(Caller2C.range("A"), Caller2C.range("B"))),
-                    AllOf(from(withName("caller3")),
-                          fromRanges(Caller3C.range("Caller1")))));
+    ASSERT_THAT(IncomingLevel2,
+                ElementsAre(AllOf(from(withName("caller2")),
+                                  iFromRanges(Caller2C.range("A"),
+                                              Caller2C.range("B"))),
+                            AllOf(from(withName("caller3")),
+                                  iFromRanges(Caller3C.range("Caller1")))));
 
     auto IncomingLevel3 = incomingCalls(IncomingLevel2[0].from, Index.get());
     ASSERT_THAT(IncomingLevel3,
                 ElementsAre(AllOf(from(withName("caller3")),
-                                  fromRanges(Caller3C.range("Caller2")))));
+                                  iFromRanges(Caller3C.range("Caller2")))));
 
     auto IncomingLevel4 = incomingCalls(IncomingLevel3[0].from, Index.get());
     EXPECT_THAT(IncomingLevel4, IsEmpty());
@@ -438,12 +569,12 @@ TEST(CallHierarchy, CallInLocalVarDecl) {
   ASSERT_THAT(Items, ElementsAre(withName("callee")));
 
   auto Incoming = incomingCalls(Items[0], Index.get());
-  ASSERT_THAT(
-      Incoming,
-      ElementsAre(
-          AllOf(from(withName("caller1")), fromRanges(Source.range("call1"))),
-          AllOf(from(withName("caller2")), fromRanges(Source.range("call2"))),
-          AllOf(from(withName("caller3")), fromRanges(Source.range("call3")))));
+  ASSERT_THAT(Incoming, ElementsAre(AllOf(from(withName("caller1")),
+                                          iFromRanges(Source.range("call1"))),
+                                    AllOf(from(withName("caller2")),
+                                          iFromRanges(Source.range("call2"))),
+                                    AllOf(from(withName("caller3")),
+                                          iFromRanges(Source.range("call3")))));
 }
 
 TEST(CallHierarchy, HierarchyOnField) {
@@ -467,7 +598,7 @@ TEST(CallHierarchy, HierarchyOnField) {
   auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
   ASSERT_THAT(IncomingLevel1,
               ElementsAre(AllOf(from(withName("caller")),
-                                fromRanges(Source.range("Callee")))));
+                                iFromRanges(Source.range("Callee")))));
 }
 
 TEST(CallHierarchy, HierarchyOnVar) {
@@ -488,7 +619,7 @@ TEST(CallHierarchy, HierarchyOnVar) {
   auto IncomingLevel1 = incomingCalls(Items[0], Index.get());
   ASSERT_THAT(IncomingLevel1,
               ElementsAre(AllOf(from(withName("caller")),
-                                fromRanges(Source.range("Callee")))));
+                                iFromRanges(Source.range("Callee")))));
 }
 
 TEST(CallHierarchy, CallInDifferentFileThanCaller) {
@@ -517,7 +648,7 @@ TEST(CallHierarchy, CallInDifferentFileThanCaller) {
   // header. The protocol does not allow us to represent such calls, so we drop
   // them. (The call hierarchy item itself is kept.)
   EXPECT_THAT(Incoming,
-              ElementsAre(AllOf(from(withName("caller")), fromRanges())));
+              ElementsAre(AllOf(from(withName("caller")), iFromRanges())));
 }
 
 } // namespace
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index a89f499736226..3acacf496e77f 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -1703,6 +1703,12 @@ class IndexRequestCollector : public SymbolIndex {
     return false;
   }
 
+  bool containedRefs(
+      const ContainedRefsRequest &,
+      llvm::function_ref<void(const ContainedRefsResult &)>) const override {
+    return false;
+  }
+
   void relations(const RelationsRequest &,
                  llvm::function_ref<void(const SymbolID &, const Symbol &)>)
       const override {}
diff --git a/clang-tools-extra/clangd/unittests/DexTests.cpp b/clang-tools-extra/clangd/unittests/DexTests.cpp
index cafbfd324840c..ca8b81b5cb3c0 100644
--- a/clang-tools-extra/clangd/unittests/DexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DexTests.cpp
@@ -476,7 +476,7 @@ TEST(DexSearchTokens, SymbolPath) {
 
 TEST(Dex, Lookup) {
   auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc"));
   EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}),
               UnorderedElementsAre("ns::abc", "ns::xyz"));
@@ -489,7 +489,7 @@ TEST(Dex, FuzzyFind) {
   auto Index =
       Dex::build(generateSymbols({"ns::ABC", "ns::BCD", "::ABC",
                                   "ns::nested::ABC", "other::ABC", "other::A"}),
-                 RefSlab(), RelationSlab());
+                 RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "ABC";
   Req.Scopes = {"ns::"};
@@ -511,7 +511,8 @@ TEST(Dex, FuzzyFind) {
 }
 
 TEST(DexTest, DexLimitedNumMatches) {
-  auto I = Dex::build(generateNumSymbols(0, 100), RefSlab(), RelationSlab());
+  auto I =
+      Dex::build(generateNumSymbols(0, 100), RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "5";
   Req.AnyScope = true;
@@ -526,7 +527,7 @@ TEST(DexTest, DexLimitedNumMatches) {
 TEST(DexTest, FuzzyMatch) {
   auto I = Dex::build(
       generateSymbols({"LaughingOutLoud", "LionPopulation", "LittleOldLady"}),
-      RefSlab(), RelationSlab());
+      RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "lol";
   Req.AnyScope = true;
@@ -537,7 +538,7 @@ TEST(DexTest, FuzzyMatch) {
 
 TEST(DexTest, ShortQuery) {
   auto I = Dex::build(generateSymbols({"_OneTwoFourSix"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.AnyScope = true;
   bool Incomplete;
@@ -580,7 +581,7 @@ TEST(DexTest, ShortQuery) {
 
 TEST(DexTest, MatchQualifiedNamesWithoutSpecificScope) {
   auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.AnyScope = true;
   Req.Query = "y";
@@ -589,7 +590,7 @@ TEST(DexTest, MatchQualifiedNamesWithoutSpecificScope) {
 
 TEST(DexTest, MatchQualifiedNamesWithGlobalScope) {
   auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "y";
   Req.Scopes = {""};
@@ -599,7 +600,7 @@ TEST(DexTest, MatchQualifiedNamesWithGlobalScope) {
 TEST(DexTest, MatchQualifiedNamesWithOneScope) {
   auto I =
       Dex::build(generateSymbols({"a::y1", "a::y2", "a::x", "b::y2", "y3"}),
-                 RefSlab(), RelationSlab());
+                 RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "y";
   Req.Scopes = {"a::"};
@@ -609,7 +610,7 @@ TEST(DexTest, MatchQualifiedNamesWithOneScope) {
 TEST(DexTest, MatchQualifiedNamesWithMultipleScopes) {
   auto I =
       Dex::build(generateSymbols({"a::y1", "a::y2", "a::x", "b::y3", "y3"}),
-                 RefSlab(), RelationSlab());
+                 RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "y";
   Req.Scopes = {"a::", "b::"};
@@ -618,7 +619,7 @@ TEST(DexTest, MatchQualifiedNamesWithMultipleScopes) {
 
 TEST(DexTest, NoMatchNestedScopes) {
   auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "y";
   Req.Scopes = {"a::"};
@@ -627,7 +628,7 @@ TEST(DexTest, NoMatchNestedScopes) {
 
 TEST(DexTest, WildcardScope) {
   auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2", "c::y3"}),
-                      RefSlab(), RelationSlab());
+                      RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.AnyScope = true;
   Req.Query = "y";
@@ -638,7 +639,7 @@ TEST(DexTest, WildcardScope) {
 
 TEST(DexTest, IgnoreCases) {
   auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Query = "AB";
   Req.Scopes = {"ns::"};
@@ -648,7 +649,7 @@ TEST(DexTest, IgnoreCases) {
 TEST(DexTest, UnknownPostingList) {
   // Regression test: we used to ignore unknown scopes and accept any symbol.
   auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.Scopes = {"ns2::"};
   EXPECT_THAT(match(*I, Req), UnorderedElementsAre());
@@ -656,7 +657,7 @@ TEST(DexTest, UnknownPostingList) {
 
 TEST(DexTest, Lookup) {
   auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(),
-                      RelationSlab());
+                      RelationSlab(), true);
   EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc"));
   EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}),
               UnorderedElementsAre("ns::abc", "ns::xyz"));
@@ -671,7 +672,7 @@ TEST(DexTest, SymbolIndexOptionsFilter) {
   CodeCompletionSymbol.Flags = Symbol::SymbolFlag::IndexedForCodeCompletion;
   NonCodeCompletionSymbol.Flags = Symbol::SymbolFlag::None;
   std::vector<Symbol> Symbols{CodeCompletionSymbol, NonCodeCompletionSymbol};
-  Dex I(Symbols, RefSlab(), RelationSlab());
+  Dex I(Symbols, RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.AnyScope = true;
   Req.RestrictForCodeCompletion = false;
@@ -687,7 +688,7 @@ TEST(DexTest, ProximityPathsBoosting) {
   CloseSymbol.CanonicalDeclaration.FileURI = "unittest:///a/b/c/d/e/f/file.h";
 
   std::vector<Symbol> Symbols{CloseSymbol, RootSymbol};
-  Dex I(Symbols, RefSlab(), RelationSlab());
+  Dex I(Symbols, RefSlab(), RelationSlab(), true);
 
   FuzzyFindRequest Req;
   Req.AnyScope = true;
@@ -726,7 +727,7 @@ TEST(DexTests, Refs) {
   Req.Filter = RefKind::Declaration | RefKind::Definition;
 
   std::vector<std::string> Files;
-  EXPECT_FALSE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab())
+  EXPECT_FALSE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab(), true)
                    .refs(Req, [&](const Ref &R) {
                      Files.push_back(R.Location.FileURI);
                    }));
@@ -734,7 +735,7 @@ TEST(DexTests, Refs) {
 
   Req.Limit = 1;
   Files.clear();
-  EXPECT_TRUE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab())
+  EXPECT_TRUE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab(), true)
                   .refs(Req, [&](const Ref &R) {
                     Files.push_back(R.Location.FileURI);
                   }));
@@ -751,7 +752,7 @@ TEST(DexTests, Relations) {
   std::vector<Relation> Relations{{Parent.ID, RelationKind::BaseOf, Child1.ID},
                                   {Parent.ID, RelationKind::BaseOf, Child2.ID}};
 
-  Dex I{Symbols, RefSlab(), Relations};
+  Dex I{Symbols, RefSlab(), Relations, true};
 
   std::vector<SymbolID> Results;
   RelationsRequest Req;
@@ -770,7 +771,7 @@ TEST(DexIndex, IndexedFiles) {
   auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
   llvm::StringSet<> Files = {"unittest:///foo.cc", "unittest:///bar.cc"};
   Dex I(std::move(Data.first), std::move(Data.second), RelationSlab(),
-        std::move(Files), IndexContents::All, std::move(Data), Size);
+        std::move(Files), IndexContents::All, std::move(Data), Size, true);
   auto ContainsFile = I.indexedFiles();
   EXPECT_EQ(ContainsFile("unittest:///foo.cc"), IndexContents::All);
   EXPECT_EQ(ContainsFile("unittest:///bar.cc"), IndexContents::All);
@@ -784,7 +785,7 @@ TEST(DexTest, PreferredTypesBoosting) {
   Sym2.Type = "T2";
 
   std::vector<Symbol> Symbols{Sym1, Sym2};
-  Dex I(Symbols, RefSlab(), RelationSlab());
+  Dex I(Symbols, RefSlab(), RelationSlab(), true);
 
   FuzzyFindRequest Req;
   Req.AnyScope = true;
@@ -820,7 +821,8 @@ TEST(DexTest, TemplateSpecialization) {
       index::SymbolProperty::TemplatePartialSpecialization);
   B.insert(S);
 
-  auto I = dex::Dex::build(std::move(B).build(), RefSlab(), RelationSlab());
+  auto I =
+      dex::Dex::build(std::move(B).build(), RefSlab(), RelationSlab(), true);
   FuzzyFindRequest Req;
   Req.AnyScope = true;
 
diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
index 9f713564b2c01..a92142fbcd7c4 100644
--- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
@@ -104,7 +104,7 @@ std::unique_ptr<RelationSlab> relSlab(llvm::ArrayRef<const Relation> Rels) {
 }
 
 TEST(FileSymbolsTest, UpdateAndGet) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty());
 
   FS.update("f1", numSlab(1, 3), refSlab(SymbolID("1"), "f1.cc"), nullptr,
@@ -116,7 +116,7 @@ TEST(FileSymbolsTest, UpdateAndGet) {
 }
 
 TEST(FileSymbolsTest, Overlap) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   FS.update("f1", numSlab(1, 3), nullptr, nullptr, false);
   FS.update("f2", numSlab(3, 5), nullptr, nullptr, false);
   for (auto Type : {IndexType::Light, IndexType::Heavy})
@@ -126,7 +126,7 @@ TEST(FileSymbolsTest, Overlap) {
 }
 
 TEST(FileSymbolsTest, MergeOverlap) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   auto OneSymboSlab = [](Symbol Sym) {
     SymbolSlab::Builder S;
     S.insert(Sym);
@@ -147,7 +147,7 @@ TEST(FileSymbolsTest, MergeOverlap) {
 }
 
 TEST(FileSymbolsTest, SnapshotAliveAfterRemove) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
 
   SymbolID ID("1");
   FS.update("f1", numSlab(1, 3), refSlab(ID, "f1.cc"), nullptr, false);
@@ -180,14 +180,14 @@ void update(FileIndex &M, llvm::StringRef Basename, llvm::StringRef Code) {
 }
 
 TEST(FileIndexTest, CustomizedURIScheme) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f", "class string {};");
 
   EXPECT_THAT(runFuzzyFind(M, ""), ElementsAre(declURI("unittest:///f.h")));
 }
 
 TEST(FileIndexTest, IndexAST) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f1", "namespace ns { void f() {} class X {}; }");
 
   FuzzyFindRequest Req;
@@ -198,7 +198,7 @@ TEST(FileIndexTest, IndexAST) {
 }
 
 TEST(FileIndexTest, NoLocal) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f1", "namespace ns { void f() { int local = 0; } class X {}; }");
 
   EXPECT_THAT(
@@ -207,7 +207,7 @@ TEST(FileIndexTest, NoLocal) {
 }
 
 TEST(FileIndexTest, IndexMultiASTAndDeduplicate) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f1", "namespace ns { void f() {} class X {}; }");
   update(M, "f2", "namespace ns { void ff() {} class X {}; }");
 
@@ -219,7 +219,7 @@ TEST(FileIndexTest, IndexMultiASTAndDeduplicate) {
 }
 
 TEST(FileIndexTest, ClassMembers) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f1", "class X { static int m1; int m2; static void f(); };");
 
   EXPECT_THAT(runFuzzyFind(M, ""),
@@ -228,7 +228,7 @@ TEST(FileIndexTest, ClassMembers) {
 }
 
 TEST(FileIndexTest, IncludeCollected) {
-  FileIndex M;
+  FileIndex M(true);
   update(
       M, "f",
       "// IWYU pragma: private, include <the/good/header.h>\nclass string {};");
@@ -240,7 +240,7 @@ TEST(FileIndexTest, IncludeCollected) {
 }
 
 TEST(FileIndexTest, IWYUPragmaExport) {
-  FileIndex M;
+  FileIndex M(true);
 
   TestTU File;
   File.Code = R"cpp(#pragma once
@@ -286,7 +286,7 @@ template <class Ty, class Arg>
 vector<Ty> make_vector(Arg A) {}
 )cpp";
 
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f", Source);
 
   auto Symbols = runFuzzyFind(M, "");
@@ -334,7 +334,7 @@ TEST(FileIndexTest, RebuildWithPreamble) {
   IgnoreDiagnostics IgnoreDiags;
   auto CI = buildCompilerInvocation(PI, IgnoreDiags);
 
-  FileIndex Index;
+  FileIndex Index(true);
   bool IndexUpdated = false;
   buildPreamble(
       FooCpp, *CI, PI,
@@ -374,7 +374,7 @@ TEST(FileIndexTest, Refs) {
   RefsRequest Request;
   Request.IDs = {Foo.ID};
 
-  FileIndex Index;
+  FileIndex Index(true);
   // Add test.cc
   TestTU Test;
   Test.HeaderCode = HeaderCode;
@@ -409,7 +409,7 @@ TEST(FileIndexTest, MacroRefs) {
   }
   )cpp");
 
-  FileIndex Index;
+  FileIndex Index(true);
   // Add test.cc
   TestTU Test;
   Test.HeaderCode = std::string(HeaderCode.code());
@@ -432,7 +432,7 @@ TEST(FileIndexTest, MacroRefs) {
 }
 
 TEST(FileIndexTest, CollectMacros) {
-  FileIndex M;
+  FileIndex M(true);
   update(M, "f", "#define CLANGD 1");
   EXPECT_THAT(runFuzzyFind(M, ""), Contains(qName("CLANGD")));
 }
@@ -443,7 +443,7 @@ TEST(FileIndexTest, Relations) {
   TU.HeaderFilename = "f.h";
   TU.HeaderCode = "class A {}; class B : public A {};";
   auto AST = TU.build();
-  FileIndex Index;
+  FileIndex Index(true);
   Index.updatePreamble(testPath(TU.Filename), /*Version=*/"null",
                        AST.getASTContext(), AST.getPreprocessor(),
                        AST.getPragmaIncludes());
@@ -493,7 +493,7 @@ TEST(FileIndexTest, ReferencesInMainFileWithPreamble) {
   )cpp");
   TU.Code = std::string(Main.code());
   auto AST = TU.build();
-  FileIndex Index;
+  FileIndex Index(true);
   Index.updateMain(testPath(TU.Filename), AST);
 
   // Expect to see references in main file, references in headers are excluded
@@ -510,7 +510,7 @@ TEST(FileIndexTest, MergeMainFileSymbols) {
   Cpp.HeaderFilename = "foo.h";
   Cpp.HeaderCode = CommonHeader;
 
-  FileIndex Index;
+  FileIndex Index(true);
   auto HeaderAST = Header.build();
   auto CppAST = Cpp.build();
   Index.updateMain(testPath("foo.h"), HeaderAST);
@@ -524,7 +524,7 @@ TEST(FileIndexTest, MergeMainFileSymbols) {
 }
 
 TEST(FileSymbolsTest, CountReferencesNoRefSlabs) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   FS.update("f1", numSlab(1, 3), nullptr, nullptr, true);
   FS.update("f2", numSlab(1, 3), nullptr, nullptr, false);
   EXPECT_THAT(
@@ -536,7 +536,7 @@ TEST(FileSymbolsTest, CountReferencesNoRefSlabs) {
 }
 
 TEST(FileSymbolsTest, CountReferencesWithRefSlabs) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   FS.update("f1cpp", numSlab(1, 3), refSlab(SymbolID("1"), "f1.cpp"), nullptr,
             true);
   FS.update("f1h", numSlab(1, 3), refSlab(SymbolID("1"), "f1.h"), nullptr,
@@ -558,7 +558,7 @@ TEST(FileSymbolsTest, CountReferencesWithRefSlabs) {
 }
 
 TEST(FileIndexTest, StalePreambleSymbolsDeleted) {
-  FileIndex M;
+  FileIndex M(true);
   TestTU File;
   File.HeaderFilename = "a.h";
 
@@ -581,7 +581,7 @@ TEST(FileIndexTest, StalePreambleSymbolsDeleted) {
 
 // Verifies that concurrent calls to updateMain don't "lose" any updates.
 TEST(FileIndexTest, Threadsafety) {
-  FileIndex M;
+  FileIndex M(true);
   Notification Go;
 
   constexpr int Count = 10;
@@ -714,7 +714,7 @@ TEST(FileShardedIndexTest, Sharding) {
 }
 
 TEST(FileIndexTest, Profile) {
-  FileIndex FI;
+  FileIndex FI(true);
 
   auto FileName = testPath("foo.cpp");
   auto AST = TestTU::withHeaderCode("int a;").build();
@@ -738,7 +738,7 @@ TEST(FileIndexTest, Profile) {
 }
 
 TEST(FileSymbolsTest, Profile) {
-  FileSymbols FS(IndexContents::All);
+  FileSymbols FS(IndexContents::All, true);
   FS.update("f1", numSlab(1, 2), nullptr, nullptr, false);
   FS.update("f2", nullptr, refSlab(SymbolID("1"), "f1"), nullptr, false);
   FS.update("f3", nullptr, nullptr,
@@ -758,7 +758,7 @@ TEST(FileSymbolsTest, Profile) {
 }
 
 TEST(FileIndexTest, MacrosFromMainFile) {
-  FileIndex Idx;
+  FileIndex Idx(true);
   TestTU TU;
   TU.Code = "#pragma once\n#define FOO";
   TU.Filename = "foo.h";
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 3220a5a6a9825..fc54f89f4941e 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -842,6 +842,8 @@ TEST_F(TargetDeclTest, OverloadExpr) {
 }
 
 TEST_F(TargetDeclTest, DependentExprs) {
+  Flags.push_back("--std=c++20");
+
   // Heuristic resolution of method of dependent field
   Code = R"cpp(
         struct A { void foo() {} };
@@ -962,6 +964,21 @@ TEST_F(TargetDeclTest, DependentExprs) {
         };
   )cpp";
   EXPECT_DECLS("MemberExpr", "void find()");
+
+  // Base expression is the type of a non-type template parameter
+  // which is deduced using CTAD.
+  Code = R"cpp(
+        template <int N>
+        struct Waldo {
+          const int found = N;
+        };
+
+        template <Waldo W>
+        int test() {
+          return W.[[found]];
+        }
+  )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr", "const int found = N");
 }
 
 TEST_F(TargetDeclTest, DependentTypes) {
diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp
index 658b4e200004e..a66680d39c87d 100644
--- a/clang-tools-extra/clangd/unittests/IndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp
@@ -292,7 +292,7 @@ TEST(MergeIndexTest, Lookup) {
 }
 
 TEST(MergeIndexTest, LookupRemovedDefinition) {
-  FileIndex DynamicIndex, StaticIndex;
+  FileIndex DynamicIndex(true), StaticIndex(true);
   MergedIndex Merge(&DynamicIndex, &StaticIndex);
 
   const char *HeaderCode = "class Foo;";
@@ -349,7 +349,7 @@ TEST(MergeIndexTest, FuzzyFind) {
 }
 
 TEST(MergeIndexTest, FuzzyFindRemovedSymbol) {
-  FileIndex DynamicIndex, StaticIndex;
+  FileIndex DynamicIndex(true), StaticIndex(true);
   MergedIndex Merge(&DynamicIndex, &StaticIndex);
 
   const char *HeaderCode = "class Foo;";
@@ -446,8 +446,8 @@ TEST(MergeTest, PreferSymbolLocationInCodegenFile) {
 }
 
 TEST(MergeIndexTest, Refs) {
-  FileIndex Dyn;
-  FileIndex StaticIndex;
+  FileIndex Dyn(true);
+  FileIndex StaticIndex(true);
   MergedIndex Merge(&Dyn, &StaticIndex);
 
   const char *HeaderCode = "class Foo;";
diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp
index 7d9252110b27d..142ed171d1a1c 100644
--- a/clang-tools-extra/clangd/unittests/RenameTests.cpp
+++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp
@@ -1548,7 +1548,7 @@ TEST(CrossFileRenameTests, DirtyBuffer) {
   std::string BarPath = testPath("bar.cc");
   // Build the index, the index has "Foo" references from foo.cc and "Bar"
   // references from bar.cc.
-  FileSymbols FSymbols(IndexContents::All);
+  FileSymbols FSymbols(IndexContents::All, true);
   FSymbols.update(FooPath, nullptr, buildRefSlab(FooCode, "Foo", FooPath),
                   nullptr, false);
   FSymbols.update(BarPath, nullptr, buildRefSlab(BarCode, "Bar", BarPath),
@@ -1601,6 +1601,12 @@ TEST(CrossFileRenameTests, DirtyBuffer) {
       return true; // has more references
     }
 
+    bool containedRefs(const ContainedRefsRequest &Req,
+                       llvm::function_ref<void(const ContainedRefsResult &)>
+                           Callback) const override {
+      return false;
+    }
+
     bool fuzzyFind(
         const FuzzyFindRequest &Req,
         llvm::function_ref<void(const Symbol &)> Callback) const override {
@@ -1652,6 +1658,12 @@ TEST(CrossFileRenameTests, DeduplicateRefsFromIndex) {
       return false;
     }
 
+    bool containedRefs(const ContainedRefsRequest &Req,
+                       llvm::function_ref<void(const ContainedRefsResult &)>
+                           Callback) const override {
+      return false;
+    }
+
     bool fuzzyFind(const FuzzyFindRequest &,
                    llvm::function_ref<void(const Symbol &)>) const override {
       return false;
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index 30b9b1902aa9c..1ec51d862d0a6 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -1092,6 +1092,13 @@ sizeof...($TemplateParameter[[Elements]]);
             $Field_dependentName[[waldo]];
           }
         };
+    )cpp",
+      // Pointer-to-member with nested-name-specifiers
+      R"cpp(
+      struct $Class_def[[Outer]] {
+        struct $Class_def[[Inner]] {};
+      };
+      using $Typedef_decl[[Alias]] = void ($Class[[Outer]]::$Class[[Inner]]:: *)();
     )cpp"};
   for (const auto &TestCase : TestCases)
     // Mask off scope modifiers to keep the tests manageable.
diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp
index 1f02c04125b1e..3f8990c86f714 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.cpp
+++ b/clang-tools-extra/clangd/unittests/TestTU.cpp
@@ -174,7 +174,7 @@ RefSlab TestTU::headerRefs() const {
 
 std::unique_ptr<SymbolIndex> TestTU::index() const {
   auto AST = build();
-  auto Idx = std::make_unique<FileIndex>();
+  auto Idx = std::make_unique<FileIndex>(/*SupportContainedRefs=*/true);
   Idx->updatePreamble(testPath(Filename), /*Version=*/"null",
                       AST.getASTContext(), AST.getPreprocessor(),
                       AST.getPragmaIncludes());
diff --git a/clang-tools-extra/clangd/unittests/TestWorkspace.cpp b/clang-tools-extra/clangd/unittests/TestWorkspace.cpp
index 2130e7a4c6dd4..e9a50f1e8b63a 100644
--- a/clang-tools-extra/clangd/unittests/TestWorkspace.cpp
+++ b/clang-tools-extra/clangd/unittests/TestWorkspace.cpp
@@ -17,7 +17,7 @@ namespace clang {
 namespace clangd {
 
 std::unique_ptr<SymbolIndex> TestWorkspace::index() {
-  auto Index = std::make_unique<FileIndex>();
+  auto Index = std::make_unique<FileIndex>(/*SupportContainedRefs=*/true);
   for (const auto &Input : Inputs) {
     if (!Input.second.IsMainFile)
       continue;
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index d393c72974d44..7d824d659ad2c 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -1019,6 +1019,15 @@ TEST(LocateSymbol, All) {
           void *Value;
           void *getPointer() const { return Info::get^Pointer(Value); }
         };
+      )cpp",
+      R"cpp(// Deducing this
+        struct S {
+          int bar(this S&);
+        };
+        void foo() {
+          S [[waldo]];
+          int x = wa^ldo.bar();
+        }
     )cpp"};
   for (const char *Test : Tests) {
     Annotations T(Test);
@@ -1035,6 +1044,7 @@ TEST(LocateSymbol, All) {
     TU.Code = std::string(T.code());
 
     TU.ExtraArgs.push_back("-xobjective-c++");
+    TU.ExtraArgs.push_back("-std=c++23");
 
     auto AST = TU.build();
     auto Results = locateSymbolAt(AST, T.point());
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index f050391110385..b2b66dca6ccf8 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -156,6 +156,10 @@ Changes in existing checks
   <clang-tidy/checks/altera/id-dependent-backward-branch>` check by fixing
   crashes from invalid code.
 
+- Improved :doc:`bugprone-branch-clone
+  <clang-tidy/checks/bugprone/branch-clone>` check to improve detection of
+  branch clones by now detecting duplicate inner and outer if statements.
+
 - Improved :doc:`bugprone-casting-through-void
   <clang-tidy/checks/bugprone/casting-through-void>` check to suggest replacing
   the offending code with ``reinterpret_cast``, to more clearly express intent.
@@ -179,7 +183,9 @@ Changes in existing checks
 - Improved :doc:`bugprone-return-const-ref-from-parameter
   <clang-tidy/checks/bugprone/return-const-ref-from-parameter>` check to
   diagnose potential dangling references when returning a ``const &`` parameter
-  by using the conditional operator ``cond ? var1 : var2``.
+  by using the conditional operator ``cond ? var1 : var2`` and fixing false
+  positives for functions which contain lambda and ignore parameters
+  with ``[[clang::lifetimebound]]`` attribute.
   
 - Improved :doc:`bugprone-sizeof-expression
   <clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
@@ -230,6 +236,11 @@ Changes in existing checks
   <clang-tidy/checks/misc/definitions-in-headers>` check by rewording the
   diagnostic note that suggests adding ``inline``.
 
+- Improved :doc:`misc-redundant-expression
+  <clang-tidy/checks/misc/redundant-expression>` check by extending the
+  checker to detect floating point and integer literals in redundant
+  expressions.
+
 - Improved :doc:`misc-unconventional-assign-operator
   <clang-tidy/checks/misc/unconventional-assign-operator>` check to avoid
   false positive for C++23 deducing this.
@@ -237,7 +248,9 @@ Changes in existing checks
 - Improved :doc:`misc-use-internal-linkage
   <clang-tidy/checks/misc/use-internal-linkage>` check to insert ``static``
   keyword before type qualifiers such as ``const`` and ``volatile`` and fix
-  false positives for function declaration without body.
+  false positives for function declaration without body and fix false positives
+  for C++20 export declarations and fix false positives for global scoped
+  overloaded ``operator new`` and ``operator delete``.
 
 - Improved :doc:`modernize-avoid-c-arrays
   <clang-tidy/checks/modernize/avoid-c-arrays>` check to suggest using 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/branch-clone.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/branch-clone.rst
index 0ca34c2bc2320..a91645f32d96f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/branch-clone.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/branch-clone.rst
@@ -32,9 +32,28 @@ If this is the intended behavior, then there is no reason to use a conditional
 statement; otherwise the issue can be solved by fixing the branch that is
 handled incorrectly.
 
-The check also detects repeated branches in longer ``if/else if/else`` chains
+The check detects repeated branches in longer ``if/else if/else`` chains
 where it would be even harder to notice the problem.
 
+The check also detects repeated inner and outer ``if`` statements that may
+be a result of a copy-paste error. This check cannot currently detect
+identical inner and outer ``if`` statements if code is between the ``if``
+conditions. An example is as follows.
+
+.. code-block:: c++
+
+    void test_warn_inner_if_1(int x) {
+      if (x == 1) {    // warns, if with identical inner if
+        if (x == 1)    // inner if is here
+          ;
+      if (x == 1) {    // does not warn, cannot detect
+        int y = x;
+        if (x == 1)
+          ;
+      }
+    }
+
+
 In ``switch`` statements the check only reports repeated branches when they are
 consecutive, because it is relatively common that the ``case:`` labels have
 some natural ordering and rearranging them would decrease the readability of
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
index 2349e51477b7d..ba47399914de3 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
@@ -12,15 +12,6 @@ after the call. When the function returns such a parameter also as constant refe
 then the returned reference can be used after the object it refers to has been
 destroyed.
 
-This issue can be resolved by declaring an overload of the problematic function
-where the ``const &`` parameter is instead declared as ``&&``. The developer has
-to ensure that the implementation of that function does not produce a
-use-after-free, the exact error that this check is warning against.
-Marking such an ``&&`` overload as ``deleted``, will silence the warning as 
-well. In the case of different ``const &`` parameters being returned depending
-on the control flow of the function, an overload where all problematic
-``const &`` parameters have been declared as ``&&`` will resolve the issue.
-
 Example
 -------
 
@@ -38,3 +29,23 @@ Example
 
   const S& s = fn(S{1});
   s.v; // use after free
+
+
+This issue can be resolved by declaring an overload of the problematic function
+where the ``const &`` parameter is instead declared as ``&&``. The developer has
+to ensure that the implementation of that function does not produce a
+use-after-free, the exact error that this check is warning against.
+Marking such an ``&&`` overload as ``deleted``, will silence the warning as 
+well. In the case of different ``const &`` parameters being returned depending
+on the control flow of the function, an overload where all problematic
+``const &`` parameters have been declared as ``&&`` will resolve the issue.
+
+This issue can also be resolved by adding ``[[clang::lifetimebound]]``. Clang
+enable ``-Wdangling`` warning by default which can detect mis-uses of the
+annotated function. See `lifetimebound attribute <https://clang.llvm.org/docs/AttributeReference.html#id11>`_
+for details.
+
+.. code-block:: c++
+
+  const int &f(const int &a [[clang::lifetimebound]]) { return a; } // no warning
+  const int &v = f(1); // warning: temporary bound to local reference 'v' will be destroyed at the end of the full-expression [-Wdangling]
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
index 83c29bd75f5b9..44f0772d0bedf 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/redundant-expression.rst
@@ -23,3 +23,22 @@ Examples:
   (p->x == p->x)              // always true
   (p->x < p->x)               // always false
   (speed - speed + 1 == 12)   // speed - speed is always zero
+  int b = a | 4 | a           // identical expr on both sides
+  ((x=1) | (x=1))             // expression is identical
+
+Floats are handled except in the case that NaNs are checked like so:
+
+.. code-block:: c++
+
+  int TestFloat(float F) {
+    if (F == F)               // Identical float values used
+      return 1;
+    return 0;
+  }
+
+  int TestFloat(float F) {
+    // Testing NaN.
+    if (F != F && F == F)     // does not warn
+      return 1;
+    return 0;
+  }
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst
index b8bbcc6270610..508b0cac09a91 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/use-internal-linkage.rst
@@ -29,6 +29,11 @@ Example:
   void fn3(); // without function body in all declaration, maybe external linkage
   void fn3();
 
+  // export declarations
+  export void fn4() {}
+  export namespace t { void fn5() {} }
+  export int v2;
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/container-size-empty.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/container-size-empty.rst
index b5abd340b9b2b..6a007f69767ab 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/container-size-empty.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/container-size-empty.rst
@@ -8,12 +8,10 @@ Checks whether a call to the ``size()``/``length()`` method can be replaced
 with a call to ``empty()``.
 
 The emptiness of a container should be checked using the ``empty()`` method
-instead of the ``size()``/``length()`` method. It is not guaranteed that
-``size()``/``length()`` is a constant-time function, and it is generally more
-efficient and also shows clearer intent to use ``empty()``. Furthermore some
-containers may implement the ``empty()`` method but not implement the ``size()``
-or ``length()`` method. Using ``empty()`` whenever possible makes it easier to
-switch to another container in the future.
+instead of the ``size()``/``length()`` method. It shows clearer intent to use
+``empty()``. Furthermore some containers may implement the ``empty()`` method
+but not implement the ``size()`` or ``length()`` method. Using ``empty()``
+whenever possible makes it easier to switch to another container in the future.
 
 The check issues warning if a container has ``empty()`` and ``size()`` or
 ``length()`` methods matching following signatures:
diff --git a/clang-tools-extra/modularize/ModuleAssistant.cpp b/clang-tools-extra/modularize/ModuleAssistant.cpp
index 5c11ffdb8589d..c7259d70bd58f 100644
--- a/clang-tools-extra/modularize/ModuleAssistant.cpp
+++ b/clang-tools-extra/modularize/ModuleAssistant.cpp
@@ -46,6 +46,8 @@ class Module {
 public:
   Module(llvm::StringRef Name, bool Problem);
   ~Module();
+  Module(const Module &other) = delete;
+  Module &operator=(const Module &other) = delete;
   bool output(llvm::raw_fd_ostream &OS, int Indent);
   Module *findSubModule(llvm::StringRef SubName);
 
diff --git a/clang-tools-extra/test/clang-reorder-fields/FieldAnnotationsInMacros.cpp b/clang-tools-extra/test/clang-reorder-fields/FieldAnnotationsInMacros.cpp
new file mode 100644
index 0000000000000..aedec9556aa55
--- /dev/null
+++ b/clang-tools-extra/test/clang-reorder-fields/FieldAnnotationsInMacros.cpp
@@ -0,0 +1,9 @@
+// RUN: clang-reorder-fields -record-name Foo -fields-order y,x %s -- | FileCheck %s
+
+#define GUARDED_BY(x) __attribute__((guarded_by(x)))
+
+class Foo {
+  int x GUARDED_BY(x); // CHECK: {{^  int y;}}
+  int y;               // CHECK-NEXT: {{^  int x GUARDED_BY\(x\);}}
+};
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
index 51f6817416906..7e579e5319752 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
@@ -136,10 +136,6 @@ bool operator==(const std::string&, const std::string&);
 bool operator==(const std::string&, const char*);
 bool operator==(const char*, const std::string&);
 
-bool operator!=(const std::string&, const std::string&);
-bool operator!=(const std::string&, const char*);
-bool operator!=(const char*, const std::string&);
-
 bool operator==(const std::wstring&, const std::wstring&);
 bool operator==(const std::wstring&, const wchar_t*);
 bool operator==(const wchar_t*, const std::wstring&);
@@ -148,9 +144,15 @@ bool operator==(const std::string_view&, const std::string_view&);
 bool operator==(const std::string_view&, const char*);
 bool operator==(const char*, const std::string_view&);
 
+#if __cplusplus < 202002L
+bool operator!=(const std::string&, const std::string&);
+bool operator!=(const std::string&, const char*);
+bool operator!=(const char*, const std::string&);
+
 bool operator!=(const std::string_view&, const std::string_view&);
 bool operator!=(const std::string_view&, const char*);
 bool operator!=(const char*, const std::string_view&);
+#endif
 
 size_t strlen(const char* str);
 }
diff --git a/clang/test/Analysis/identical-expressions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/alpha-core-identicalexpr.cpp
similarity index 57%
rename from clang/test/Analysis/identical-expressions.cpp
rename to clang-tools-extra/test/clang-tidy/checkers/bugprone/alpha-core-identicalexpr.cpp
index 8bb82372b534a..8eff3ebc948de 100644
--- a/clang/test/Analysis/identical-expressions.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/alpha-core-identicalexpr.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core.IdenticalExpr -w -verify %s
+// RUN: clang-tidy %s -checks="-*,misc-redundant-expression" -- 2>&1 | FileCheck %s --check-prefix=CHECK-MESSAGES-IDENTEXPR
+// RUN: clang-tidy %s -checks="-*,bugprone-branch-clone" -- 2>&1 | FileCheck %s --check-prefix=CHECK-MESSAGES-BUGPRONEBRANCH
 
 /* Only one expected warning per function allowed at the very end. */
 
@@ -65,7 +66,8 @@ int checkNotEqualFloatDeclCompare6(void) {
 
 int checkNotEqualCastFloatDeclCompare11(void) {
   float f = 7.1F;
-  return ((int)f != (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f != (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkNotEqualCastFloatDeclCompare12(void) {
   float f = 7.1F;
@@ -130,7 +132,8 @@ int checkNotEqualNestedBinaryOpFloatCompare3(void) {
 /* '!=' with int*/
 
 int checkNotEqualIntLiteralCompare1(void) {
-  return (5 != 5); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (5 != 5);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkNotEqualIntLiteralCompare2(void) {
@@ -155,7 +158,8 @@ int checkNotEqualIntDeclCompare4(void) {
 
 int checkNotEqualCastIntDeclCompare11(void) {
   int f = 7;
-  return ((int)f != (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f != (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkNotEqualCastIntDeclCompare12(void) {
   int f = 7;
@@ -166,7 +170,8 @@ int checkNotEqualBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 4;
-  res = (f + 4 != f + 4);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (f + 4 != f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkNotEqualBinaryOpIntCompare2(void) {
@@ -181,7 +186,8 @@ int checkNotEqualBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 4;
-  res = ((int)f + 4 != (int)f + 4);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = ((int)f + 4 != (int)f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:21: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkNotEqualBinaryOpIntCompare4(void) {
@@ -196,7 +202,8 @@ int checkNotEqualBinaryOpIntCompare5(void) {
   int res;
   int t= 1;
   int u= 2;
-  res = (u + t != u + t);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (u + t != u + t);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -205,7 +212,8 @@ int checkNotEqualNestedBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (3 - u)*t) != ((int)f + (3 - u)*t));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (3 - u)*t) != ((int)f + (3 - u)*t));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:31: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -223,7 +231,8 @@ int checkNotEqualNestedBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (u - 3)*t) != ((int)f + (3 - u)*(t + 1 != t + 1)));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (u - 3)*t) != ((int)f + (3 - u)*(t + 1 != t + 1)));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:59: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -252,7 +261,8 @@ int checkNotEqualIntPointerDeclCompare1(void) {
 int checkNotEqualCastIntPointerDeclCompare11(void) {
   int k = 7;
   int* f = &k;
-  return ((int*)f != (int*)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int*)f != (int*)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:19: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkNotEqualCastIntPointerDeclCompare12(void) {
   int k = 7;
@@ -263,7 +273,8 @@ int checkNotEqualBinaryOpIntPointerCompare1(void) {
   int k = 7;
   int res;
   int* f= &k;
-  res = (f + 4 != f + 4);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (f + 4 != f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkNotEqualBinaryOpIntPointerCompare2(void) {
@@ -278,7 +289,8 @@ int checkNotEqualBinaryOpIntPointerCompare3(void) {
   int k = 7;
   int res;
   int* f= &k;
-  res = ((int*)f + 4 != (int*)f + 4);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = ((int*)f + 4 != (int*)f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:22: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkNotEqualBinaryOpIntPointerCompare4(void) {
@@ -295,7 +307,8 @@ int checkNotEqualNestedBinaryOpIntPointerCompare1(void) {
   int t= 1;
   int* u= &k+2;
   int* f= &k+3;
-  res = ((f + (3)*t) != (f + (3)*t));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = ((f + (3)*t) != (f + (3)*t));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:22: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -358,7 +371,8 @@ int checkEqualIntPointerDeclCompare(void) {
 int checkEqualIntPointerDeclCompare0(void) {
   int k = 3;
   int* f = &k;
-  return (f+1 == f+1); // expected-warning {{comparison of identical expressions always evaluates to true}}
+  return (f+1 == f+1);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:15: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 /* EQ with float*/
@@ -410,7 +424,8 @@ int checkEqualFloatDeclCompare6(void) {
 
 int checkEqualCastFloatDeclCompare11(void) {
   float f = 7.1F;
-  return ((int)f == (int)f); // expected-warning {{comparison of identical expressions always evaluates to true}}
+  return ((int)f == (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkEqualCastFloatDeclCompare12(void) {
   float f = 7.1F;
@@ -474,7 +489,8 @@ int checkEqualNestedBinaryOpFloatCompare3(void) {
 /* Equal with int*/
 
 int checkEqualIntLiteralCompare1(void) {
-  return (5 == 5); // expected-warning {{comparison of identical expressions always evaluates to true}}
+  return (5 == 5);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkEqualIntLiteralCompare2(void) {
@@ -489,7 +505,8 @@ int checkEqualIntDeclCompare1(void) {
 
 int checkEqualCastIntDeclCompare11(void) {
   int f = 7;
-  return ((int)f == (int)f); // expected-warning {{comparison of identical expressions always evaluates to true}}
+  return ((int)f == (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkEqualCastIntDeclCompare12(void) {
   int f = 7;
@@ -511,7 +528,8 @@ int checkEqualBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 4;
-  res = (f + 4 == f + 4);  // expected-warning {{comparison of identical expressions always evaluates to true}}
+  res = (f + 4 == f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkEqualBinaryOpIntCompare2(void) {
@@ -526,7 +544,8 @@ int checkEqualBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 4;
-  res = ((int)f + 4 == (int)f + 4);  // expected-warning {{comparison of identical expressions always evaluates to true}}
+  res = ((int)f + 4 == (int)f + 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:21: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 
 }
@@ -542,7 +561,8 @@ int checkEqualBinaryOpIntCompare5(void) {
   int res;
   int t= 1;
   int u= 2;
-  res = (u + t == u + t);  // expected-warning {{comparison of identical expressions always evaluates to true}}
+  res = (u + t == u + t);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -551,7 +571,8 @@ int checkEqualNestedBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (3 - u)*t) == ((int)f + (3 - u)*t));  // expected-warning {{comparison of identical expressions always evaluates to true}}
+  res = (((int)f + (3 - u)*t) == ((int)f + (3 - u)*t));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:31: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -569,7 +590,8 @@ int checkEqualNestedBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (u - 3)*t) == ((int)f + (3 - u)*(t + 1 == t + 1)));  // expected-warning {{comparison of identical expressions always evaluates to true}}
+  res = (((int)f + (u - 3)*t) == ((int)f + (3 - u)*(t + 1 == t + 1)));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:59: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -615,7 +637,8 @@ int checkEqualSameFunctionDifferentParam() {
 /*  LT with float */
 
 int checkLessThanFloatLiteralCompare1(void) {
-  return (5.14F < 5.14F); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (5.14F < 5.14F);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkLessThanFloatLiteralCompare2(void) {
@@ -630,7 +653,8 @@ int checkLessThanFloatDeclCompare1(void) {
 
 int checkLessThanFloatDeclCompare12(void) {
   float f = 7.1F;
-  return (f < f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (f < f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkLessThanFloatDeclCompare3(void) {
@@ -658,7 +682,8 @@ int checkLessThanFloatDeclCompare6(void) {
 
 int checkLessThanCastFloatDeclCompare11(void) {
   float f = 7.1F;
-  return ((int)f < (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f < (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkLessThanCastFloatDeclCompare12(void) {
   float f = 7.1F;
@@ -721,7 +746,8 @@ int checkLessThanNestedBinaryOpFloatCompare3(void) {
 
 
 int checkLessThanIntLiteralCompare1(void) {
-  return (5 < 5); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (5 < 5);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkLessThanIntLiteralCompare2(void) {
@@ -758,7 +784,8 @@ int checkLessThanIntDeclCompare6(void) {
 
 int checkLessThanCastIntDeclCompare11(void) {
   int f = 7;
-  return ((int)f < (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f < (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkLessThanCastIntDeclCompare12(void) {
   int f = 7;
@@ -767,7 +794,8 @@ int checkLessThanCastIntDeclCompare12(void) {
 int checkLessThanBinaryOpIntCompare1(void) {
   int res;
   int f= 3;
-  res = (f + 3 < f + 3);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (f + 3 < f + 3);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkLessThanBinaryOpIntCompare2(void) {
@@ -778,7 +806,8 @@ int checkLessThanBinaryOpIntCompare2(void) {
 int checkLessThanBinaryOpIntCompare3(void) {
   int res;
   int f= 3;
-  res = ((int)f + 3 < (int)f + 3);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = ((int)f + 3 < (int)f + 3);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:21: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkLessThanBinaryOpIntCompare4(void) {
@@ -793,7 +822,8 @@ int checkLessThanNestedBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (3 - u)*t) < ((int)f + (3 - u)*t));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (3 - u)*t) < ((int)f + (3 - u)*t));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:31: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -811,7 +841,8 @@ int checkLessThanNestedBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (u - 3)*t) < ((int)f + (3 - u)*(t + u < t + u)));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (u - 3)*t) < ((int)f + (3 - u)*(t + u < t + u)));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:58: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -825,7 +856,8 @@ int checkLessThanNestedBinaryOpIntCompare3(void) {
 /* GT with float */
 
 int checkGreaterThanFloatLiteralCompare1(void) {
-  return (5.14F > 5.14F); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (5.14F > 5.14F);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkGreaterThanFloatLiteralCompare2(void) {
@@ -841,7 +873,8 @@ int checkGreaterThanFloatDeclCompare1(void) {
 
 int checkGreaterThanFloatDeclCompare12(void) {
   float f = 7.1F;
-  return (f > f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (f > f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 
@@ -869,7 +902,8 @@ int checkGreaterThanFloatDeclCompare6(void) {
 
 int checkGreaterThanCastFloatDeclCompare11(void) {
   float f = 7.1F;
-  return ((int)f > (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f > (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkGreaterThanCastFloatDeclCompare12(void) {
   float f = 7.1F;
@@ -932,7 +966,8 @@ int checkGreaterThanNestedBinaryOpFloatCompare3(void) {
 
 
 int checkGreaterThanIntLiteralCompare1(void) {
-  return (5 > 5); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return (5 > 5);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 int checkGreaterThanIntLiteralCompare2(void) {
@@ -958,7 +993,8 @@ int checkGreaterThanIntDeclCompare4(void) {
 
 int checkGreaterThanCastIntDeclCompare11(void) {
   int f = 7;
-  return ((int)f > (int)f); // expected-warning {{comparison of identical expressions always evaluates to false}}
+  return ((int)f > (int)f);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 int checkGreaterThanCastIntDeclCompare12(void) {
   int f = 7;
@@ -967,7 +1003,8 @@ int checkGreaterThanCastIntDeclCompare12(void) {
 int checkGreaterThanBinaryOpIntCompare1(void) {
   int res;
   int f= 3;
-  res = (f + 3 > f + 3);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (f + 3 > f + 3);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:16: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkGreaterThanBinaryOpIntCompare2(void) {
@@ -978,7 +1015,8 @@ int checkGreaterThanBinaryOpIntCompare2(void) {
 int checkGreaterThanBinaryOpIntCompare3(void) {
   int res;
   int f= 3;
-  res = ((int)f + 3 > (int)f + 3);  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = ((int)f + 3 > (int)f + 3);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:21: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 int checkGreaterThanBinaryOpIntCompare4(void) {
@@ -993,7 +1031,8 @@ int checkGreaterThanNestedBinaryOpIntCompare1(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (3 - u)*t) > ((int)f + (3 - u)*t));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (3 - u)*t) > ((int)f + (3 - u)*t));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:31: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -1011,7 +1050,8 @@ int checkGreaterThanNestedBinaryOpIntCompare3(void) {
   int t= 1;
   int u= 2;
   int f= 3;
-  res = (((int)f + (u - 3)*t) > ((int)f + (3 - u)*(t + u > t + u)));  // expected-warning {{comparison of identical expressions always evaluates to false}}
+  res = (((int)f + (u - 3)*t) > ((int)f + (3 - u)*(t + u > t + u)));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:58: warning: both sides of operator are equivalent [misc-redundant-expression]
   return (0);
 }
 
@@ -1024,45 +1064,61 @@ int checkGreaterThanNestedBinaryOpIntCompare3(void) {
 
 unsigned test_unsigned(unsigned a) {
   unsigned b = 1;
-  a = a > 5 ? b : b; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? b : b;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
   return a;
 }
 
 void test_signed() {
   int a = 0;
-  a = a > 5 ? a : a; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a : a;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_bool(bool a) {
-  a = a > 0 ? a : a; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 0 ? a : a;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_float() {
   float a = 0;
   float b = 0;
-  a = a > 5 ? a : a; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a : a;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 const char *test_string() {
   float a = 0;
-  return a > 5 ? "abc" : "abc"; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  return a > 5 ? "abc" : "abc";
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:24: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:16: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_unsigned_expr() {
   unsigned a = 0;
   unsigned b = 0;
-  a = a > 5 ? a+b : a+b; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+b : a+b;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:19: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_signed_expr() {
   int a = 0;
   int b = 1;
-  a = a > 5 ? a+b : a+b; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+b : a+b;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:19: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_bool_expr(bool a) {
   bool b = 0;
-  a = a > 0 ? a&&b : a&&b; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 0 ? a&&b : a&&b;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:20: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_unsigned_expr_negative() {
@@ -1085,13 +1141,17 @@ void test_bool_expr_negative(bool a) {
 void test_float_expr_positive() {
   float a = 0;
   float b = 0;
-  a = a > 5 ? a+b : a+b; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+b : a+b;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:19: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_expr_positive_func() {
   unsigned a = 0;
   unsigned b = 1;
-  a = a > 5 ? a+func() : a+func(); // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+func() : a+func();
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:24: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_expr_negative_func() {
@@ -1103,7 +1163,9 @@ void test_expr_negative_func() {
 void test_expr_positive_funcParam() {
   unsigned a = 0;
   unsigned b = 1;
-  a = a > 5 ? a+funcParam(b) : a+funcParam(b); // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+funcParam(b) : a+funcParam(b);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:30: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_expr_negative_funcParam() {
@@ -1115,7 +1177,8 @@ void test_expr_negative_funcParam() {
 void test_expr_positive_inc() {
   unsigned a = 0;
   unsigned b = 1;
-  a = a > 5 ? a++ : a++; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a++ : a++;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_expr_negative_inc() {
@@ -1127,7 +1190,8 @@ void test_expr_negative_inc() {
 void test_expr_positive_assign() {
   unsigned a = 0;
   unsigned b = 1;
-  a = a > 5 ? a=1 : a=1;  // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a=1 : a=1;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_expr_negative_assign() {
@@ -1140,7 +1204,9 @@ void test_signed_nested_expr() {
   int a = 0;
   int b = 1;
   int c = 3;
-  a = a > 5 ? a+b+(c+a)*(a + b*(c+a)) : a+b+(c+a)*(a + b*(c+a)); // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? a+b+(c+a)*(a + b*(c+a)) : a+b+(c+a)*(a + b*(c+a));
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:39: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:13: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_signed_nested_expr_negative() {
@@ -1161,23 +1227,29 @@ void test_signed_nested_cond_expr() {
   int a = 0;
   int b = 1;
   int c = 3;
-  a = a > 5 ? (b > 5 ? 1 : 4) : (b > 5 ? 4 : 4); // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  a = a > 5 ? (b > 5 ? 1 : 4) : (b > 5 ? 4 : 4);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:44: warning: 'true' and 'false' expressions are equivalent [misc-redundant-expression]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:40: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 
 void test_identical_branches1(bool b) {
   int i = 0;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     ++i;
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     ++i;
   }
 }
 
 void test_identical_branches2(bool b) {
   int i = 0;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     ++i;
   } else
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     ++i;
 }
 
@@ -1192,33 +1264,41 @@ void test_identical_branches3(bool b) {
 
 void test_identical_branches4(bool b) {
   int i = 0;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
   }
 }
 
 void test_identical_branches_break(bool b) {
   while (true) {
-    if (b) // expected-warning {{true and false branches are identical}}
+    if (b)
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: if with identical then and else branches [bugprone-branch-clone]
       break;
     else
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
       break;
   }
 }
 
 void test_identical_branches_continue(bool b) {
   while (true) {
-    if (b) // expected-warning {{true and false branches are identical}}
+    if (b)
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: if with identical then and else branches [bugprone-branch-clone]
       continue;
     else
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
       continue;
   }
 }
 
 void test_identical_branches_func(bool b) {
-  if (b) // expected-warning {{true and false branches are identical}}
+  if (b)
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     func();
   else
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: note: else branch starts here
     func();
 }
 
@@ -1239,27 +1319,33 @@ void test_identical_branches_cast1(bool b) {
 
 void test_identical_branches_cast2(bool b) {
   long v = -7;
-  if (b) // expected-warning {{true and false branches are identical}}
+  if (b)
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     v = (signed int) v;
   else
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: note: else branch starts here
     v = (signed int) v;
 }
 
 int test_identical_branches_return_int(bool b) {
   int i = 0;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     i++;
     return i;
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     i++;
     return i;
   }
 }
 
 int test_identical_branches_return_func(bool b) {
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     return func();
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     return func();
   }
 }
@@ -1267,10 +1353,12 @@ int test_identical_branches_return_func(bool b) {
 void test_identical_branches_for(bool b) {
   int i;
   int j;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     for (i = 0, j = 0; i < 10; i++)
       j += 4;
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     for (i = 0, j = 0; i < 10; i++)
       j += 4;
   }
@@ -1278,10 +1366,12 @@ void test_identical_branches_for(bool b) {
 
 void test_identical_branches_while(bool b) {
   int i = 10;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     while (func())
       i--;
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     while (func())
       i--;
   }
@@ -1300,11 +1390,13 @@ void test_identical_branches_while_2(bool b) {
 
 void test_identical_branches_do_while(bool b) {
   int i = 10;
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     do {
       i--;
     } while (func());
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     do {
       i--;
     } while (func());
@@ -1312,27 +1404,32 @@ void test_identical_branches_do_while(bool b) {
 }
 
 void test_identical_branches_if(bool b, int i) {
-  if (b) { // expected-warning {{true and false branches are identical}}
+  if (b) {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical then and else branches [bugprone-branch-clone]
     if (i < 5)
       i += 10;
   } else {
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: else branch starts here
     if (i < 5)
       i += 10;
   }
 }
 
 void test_identical_bitwise1() {
-  int a = 5 | 5; // expected-warning {{identical expressions on both sides of bitwise operator}}
+  int a = 5 | 5;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 void test_identical_bitwise2() {
   int a = 5;
-  int b = a | a; // expected-warning {{identical expressions on both sides of bitwise operator}}
+  int b = a | a;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:13: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 void test_identical_bitwise3() {
   int a = 5;
-  int b = (a | a); // expected-warning {{identical expressions on both sides of bitwise operator}}
+  int b = (a | a);
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:14: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 void test_identical_bitwise4() {
@@ -1348,21 +1445,25 @@ void test_identical_bitwise5() {
 
 void test_identical_bitwise6() {
   int a = 5;
-  int b = a | 4 | a; // expected-warning {{identical expressions on both sides of bitwise operator}}
+  int b = a | 4 | a;
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:17: warning: operator has equivalent nested operands [misc-redundant-expression]
 }
 
 void test_identical_bitwise7() {
   int a = 5;
-  int b = func() | func(); // no-warning
+  int b = func() | func();
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:18: warning: both sides of operator are equivalent [misc-redundant-expression]
 }
 
 void test_identical_logical1(int a) {
-  if (a == 4 && a == 4) // expected-warning {{identical expressions on both sides of logical operator}}
+  if (a == 4 && a == 4)
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:14: warning: both sides of operator are equivalent [misc-redundant-expression]
     ;
 }
 
 void test_identical_logical2(int a) {
-  if (a == 4 || a == 5 || a == 4) // expected-warning {{identical expressions on both sides of logical operator}}
+  if (a == 4 || a == 5 || a == 4)
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:24: warning: operator has equivalent nested operands [misc-redundant-expression]
     ;
 }
 
@@ -1384,7 +1485,8 @@ void test_identical_logical5(int x, int y) {
 }
 
 void test_identical_logical6(int x, int y) {
-  if (x == 4 && y == 5 || x == 4 && y == 5) // expected-warning {{identical expressions on both sides of logical operator}}
+  if (x == 4 && y == 5 || x == 4 && y == 5)
+// CHECK-MESSAGES-IDENTEXPR: :[[@LINE-1]]:24: warning: both sides of operator are equivalent [misc-redundant-expression]
     ;
 }
 
@@ -1410,51 +1512,73 @@ void test_identical_logical9(int x, int y) {
 void test_warn_chained_if_stmts_1(int x) {
   if (x == 1)
     ;
-  else if (x == 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
+  else if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
 }
 
 void test_warn_chained_if_stmts_2(int x) {
   if (x == 1)
     ;
-  else if (x == 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
+  else if (x == 1)
     ;
-  else if (x == 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
+  else if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
 }
 
 void test_warn_chained_if_stmts_3(int x) {
   if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (x == 2)
     ;
-  else if (x == 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
+  else if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
 }
 
 void test_warn_chained_if_stmts_4(int x) {
   if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (func())
     ;
-  else if (x == 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
+  else if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
 }
 
 void test_warn_chained_if_stmts_5(int x) {
   if (x & 1)
     ;
-  else if (x & 1) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
+  else if (x & 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
 }
 
 void test_warn_chained_if_stmts_6(int x) {
   if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (x == 2)
     ;
-  else if (x == 2) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
+  else if (x == 2)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
   else if (x == 3)
     ;
 }
@@ -1462,58 +1586,83 @@ void test_warn_chained_if_stmts_6(int x) {
 void test_warn_chained_if_stmts_7(int x) {
   if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (x == 2)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
   else if (x == 3)
     ;
-  else if (x == 2) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
+  else if (x == 2)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 3 starts here
   else if (x == 5)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 4 starts here
 }
 
 void test_warn_chained_if_stmts_8(int x) {
   if (x == 1)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (x == 2)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
   else if (x == 3)
     ;
-  else if (x == 2) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
+  else if (x == 2)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 3 starts here
   else if (x == 5)
     ;
-  else if (x == 3) // expected-warning {{expression is identical to previous condition}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 4 starts here
+  else if (x == 3)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 5 starts here
   else if (x == 7)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 6 starts here
 }
 
 void test_nowarn_chained_if_stmts_1(int x) {
   if (func())
     ;
-  else if (func()) // no-warning
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
+  else if (func())
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
 }
 
 void test_nowarn_chained_if_stmts_2(int x) {
   if (func())
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
   else if (x == 1)
     ;
-  else if (func()) // no-warning
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
+  else if (func())
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 2 starts here
 }
 
 void test_nowarn_chained_if_stmts_3(int x) {
   if (x++)
     ;
-  else if (x++) // no-warning
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: warning: repeated branch body in conditional chain [bugprone-branch-clone]
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-2]]:6: note: end of the original
+  else if (x++)
     ;
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: clone 1 starts here
 }
 
 void test_warn_wchar() {
-  const wchar_t * a = 0 ? L"Warning" : L"Warning"; // expected-warning {{identical expressions on both sides of ':' in conditional expression}}
+  const wchar_t * a = 0 ? L"Warning" : L"Warning";
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:25: warning: conditional operator with identical true and false expressions [bugprone-branch-clone]
 }
 void test_nowarn_wchar() {
   const wchar_t * a = 0 ? L"No" : L"Warning";
@@ -1525,7 +1674,7 @@ void test_nowarn_long() {
   if (0) {
     b -= a;
     c = 0;
-  } else { // no-warning
+  } else {
     b -= a;
     c = 0LL;
   }
@@ -1535,7 +1684,9 @@ void test_nowarn_long() {
 
 void test_warn_inner_if_1(int x) {
   if (x == 1) {
-    if (x == 1) // expected-warning {{conditions of the inner and outer statements are identical}}
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:3: warning: if with identical inner if statement [bugprone-branch-clone]
+    if (x == 1)
+// CHECK-MESSAGES-BUGPRONEBRANCH: :[[@LINE-1]]:5: note: inner if starts here
       ;
   }
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/return-const-ref-from-parameter.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/return-const-ref-from-parameter.cpp
index d83d997a455d5..a3297ca0f8084 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/return-const-ref-from-parameter.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/return-const-ref-from-parameter.cpp
@@ -76,6 +76,9 @@ struct C {
 // CHECK-MESSAGES: :[[@LINE-1]]:38: warning: returning a constant reference parameter
 };
 
+const auto Lf1 = [](const T& t) -> const T& { return t; };
+// CHECK-MESSAGES: :[[@LINE-1]]:54: warning: returning a constant reference parameter
+
 } // namespace invalid
 
 namespace false_negative_because_dependent_and_not_instantiated {
@@ -151,6 +154,14 @@ void instantiate(const int &param, const float &paramf, int &mut_param, float &m
         itf6(mut_paramf);
 }
 
+template<class T>
+void f(const T& t) {
+    const auto get = [&t] -> const T& { return t; };
+    return T{};
+}
+
+const auto Lf1 = [](T& t) -> const T& { return t; };
+
 } // namespace valid
 
 namespace overload {
@@ -186,3 +197,32 @@ int const &overload_params_difference3(int p1, int const &a, int p2) { return a;
 int const &overload_params_difference3(int p1, long &&a, int p2);
 
 } // namespace overload
+
+namespace gh117696 {
+namespace use_lifetime_bound_attr {
+int const &f(int const &a [[clang::lifetimebound]]) { return a; }
+} // namespace use_lifetime_bound_attr
+} // namespace gh117696
+
+
+namespace lambda {
+using T = const int &;
+using K = const float &;
+T inner_valid_lambda(T a) {
+  [&]() -> T { return a; };
+  return a;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: returning a constant reference parameter
+}
+T inner_invalid_lambda(T a) {
+  [&](T a) -> T { return a; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: returning a constant reference parameter
+  return a;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: returning a constant reference parameter
+}
+T inner_invalid_lambda2(T a) {
+  [&](K a) -> K { return a; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: returning a constant reference parameter
+  return a;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: returning a constant reference parameter
+}
+} // namespace lambda
diff --git a/clang-tools-extra/test/clang-tidy/checkers/fuchsia/default-arguments-calls.cpp b/clang-tools-extra/test/clang-tidy/checkers/fuchsia/default-arguments-calls.cpp
index ed9079092f6ac..50b6d4c5676c3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/fuchsia/default-arguments-calls.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/fuchsia/default-arguments-calls.cpp
@@ -26,7 +26,7 @@ void S::x(int i = 12) {}
 int main() {
   S s;
   s.x();
-  // CHECK-NOTES: [[@LINE-1]]:3: warning: calling a function that uses a default argument is disallowed [fuchsia-default-arguments-calls]
+  // CHECK-NOTES: [[@LINE-1]]:5: warning: calling a function that uses a default argument is disallowed [fuchsia-default-arguments-calls]
   // CHECK-NOTES: [[@LINE-6]]:11: note: default parameter was declared here
   // CHECK-NEXT: void S::x(int i = 12) {}
   x();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
index 1b271630e0d19..7396d2dce76c4 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/redundant-expression.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s misc-redundant-expression %t -- -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy %s misc-redundant-expression %t -- -- -fno-delayed-template-parsing -Wno-array-compare-cxx26
 
 typedef __INT64_TYPE__ I64;
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp
index bf0d2c2513e56..68951fcf0aaac 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-func.cpp
@@ -85,3 +85,13 @@ void func_with_body() {}
 void func_without_body();
 void func_without_body();
 }
+
+// gh117489 start
+namespace std {
+using size_t = decltype(sizeof(int));
+}
+void * operator new(std::size_t) { return nullptr; }
+void * operator new[](std::size_t) { return nullptr; }
+void operator delete(void*) noexcept {}
+void operator delete[](void*) noexcept {}
+// gh117489 end
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-module.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-module.cpp
new file mode 100644
index 0000000000000..9b0d8cde429b6
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-module.cpp
@@ -0,0 +1,20 @@
+// RUN: %check_clang_tidy -std=c++20 %s misc-use-internal-linkage %t -- -- -I%S/Inputs/use-internal-linkage
+
+module;
+
+export module test;
+
+export void single_export_fn() {}
+export int single_export_var;
+
+export {
+  void group_export_fn1() {}
+  void group_export_fn2() {}
+  int group_export_var1;
+  int group_export_var2;
+}
+
+export namespace aa {
+void namespace_export_fn() {}
+int namespace_export_var;
+} // namespace aa
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
index 18a65f7f1cf26..8699ca03ba331 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
@@ -320,3 +320,13 @@ void test_substr() {
 
     str.substr(0, strlen("hello123")) == "hello";
 }
+
+void test_operator_rewriting(std::string str, std::string prefix) {
+  str.substr(0, prefix.size()) == prefix;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use starts_with instead of substr
+  // CHECK-FIXES: str.starts_with(prefix);
+
+  str.substr(0, prefix.size()) != prefix;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use starts_with instead of substr
+  // CHECK-FIXES: !str.starts_with(prefix);
+}
diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 7396211715a80..13242fa42684e 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -320,8 +320,8 @@ OpenMP conformance
 
 OpenCL conformance
 ~~~~~~~~~~~~~~~~~~
-| Anastasia Stulova
-| anastasia\@compiler-experts.com (email), Anastasia (Phabricator), AnastasiaStulova (GitHub)
+| Sven van Haastregt
+| sven.vanhaastregt@arm.com (email), svenvh (GitHub)
 
 
 OpenACC
@@ -365,6 +365,7 @@ Emeritus Lead Maintainers
 
 Inactive component maintainers
 ------------------------------
+| Anastasia Stulova (stulovaa\@gmail.com) -- OpenCL, C++ for OpenCL
 | Chandler Carruth (chandlerc\@gmail.com, chandlerc\@google.com) -- CMake, library layering
 | Devin Coughlin (dcoughlin\@apple.com) -- Clang static analyzer
 | Manuel Klimek (klimek\@google.com (email), klimek (Phabricator), r4nt (GitHub)) -- Tooling, AST matchers
diff --git a/clang/cmake/caches/CrossWinToARMLinux.cmake b/clang/cmake/caches/CrossWinToARMLinux.cmake
index 853217c6db61a..c47c4ac3bb73e 100644
--- a/clang/cmake/caches/CrossWinToARMLinux.cmake
+++ b/clang/cmake/caches/CrossWinToARMLinux.cmake
@@ -96,7 +96,11 @@ endif()
 if (NOT DEFINED TOOLCHAIN_SHARED_LIBS)
   set(TOOLCHAIN_SHARED_LIBS OFF)
 endif()
- 
+# Enable usage of the static libunwind and libc++abi libraries.
+if (NOT DEFINED TOOLCHAIN_USE_STATIC_LIBS)
+  set(TOOLCHAIN_USE_STATIC_LIBS ON)
+endif()
+
 if (NOT DEFINED LLVM_TARGETS_TO_BUILD)
   if ("${TOOLCHAIN_TARGET_TRIPLE}" MATCHES "^(armv|arm32)+")
     set(LLVM_TARGETS_TO_BUILD "ARM" CACHE STRING "")
@@ -206,7 +210,7 @@ set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_USE_COMPILER_RT
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBUNWIND_ENABLE_SHARED                   ${TOOLCHAIN_SHARED_LIBS} CACHE BOOL "")
 
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_USE_LLVM_UNWINDER               ON CACHE BOOL "")
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_STATIC_UNWINDER          ON CACHE BOOL "")
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_STATIC_UNWINDER          ${TOOLCHAIN_USE_STATIC_LIBS} CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_USE_COMPILER_RT                 ON CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS   OFF CACHE BOOL "")
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXXABI_ENABLE_SHARED                   ${TOOLCHAIN_SHARED_LIBS} CACHE BOOL "")
@@ -217,7 +221,7 @@ set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ABI_VERSION
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_CXX_ABI                            "libcxxabi" CACHE STRING "")    #!!!
 set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS      ON CACHE BOOL "")
 # Merge libc++ and libc++abi libraries into the single libc++ library file.
-set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_STATIC_ABI_LIBRARY          ON CACHE BOOL "")
+set(RUNTIMES_${TOOLCHAIN_TARGET_TRIPLE}_LIBCXX_ENABLE_STATIC_ABI_LIBRARY          ${TOOLCHAIN_USE_STATIC_LIBS} CACHE BOOL "")
 # Forcely disable the libc++ benchmarks on Windows build hosts
 # (current benchmark test configuration does not support the cross builds there).
 if (WIN32)
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 304e7833699a7..8e161f868d4af 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -6,7 +6,7 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "")
 
 set(PACKAGE_VENDOR Fuchsia CACHE STRING "")
 
-set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;libc;lld;llvm;polly")
+set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;lld;llvm;polly")
 set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
 
 set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
@@ -25,8 +25,6 @@ set(LLVM_ENABLE_ZLIB ON CACHE BOOL "")
 set(LLVM_FORCE_BUILD_RUNTIME ON CACHE BOOL "")
 set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
 set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
-set(LLVM_LIBC_FULL_BUILD ON CACHE BOOL "")
-set(LIBC_HDRGEN_ONLY ON CACHE BOOL "")
 set(LLVM_STATIC_LINK_CXX_STDLIB ON CACHE BOOL "")
 set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "")
 set(LLDB_ENABLE_CURSES OFF CACHE BOOL "")
@@ -302,19 +300,22 @@ if(FUCHSIA_SDK)
   set(LLVM_RUNTIME_MULTILIB_hwasan+noexcept_TARGETS "aarch64-unknown-fuchsia;riscv64-unknown-fuchsia" CACHE STRING "")
 endif()
 
-foreach(target armv6m-none-eabi;armv7m-none-eabi;armv8m.main-none-eabi;armv8.1m.main-none-eabi)
+foreach(target armv6m-none-eabi;armv7m-none-eabi;armv8m.main-none-eabi;armv8.1m.main-none-eabi;aarch64-none-elf)
   list(APPEND BUILTIN_TARGETS "${target}")
   set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "")
   set(BUILTINS_${target}_CMAKE_SYSTEM_PROCESSOR arm CACHE STRING "")
   set(BUILTINS_${target}_CMAKE_SYSROOT "" CACHE STRING "")
   set(BUILTINS_${target}_CMAKE_BUILD_TYPE MinSizeRel CACHE STRING "")
   foreach(lang C;CXX;ASM)
-    set(BUILTINS_${target}_CMAKE_${lang}_local_flags "--target=${target} -mthumb")
+    set(BUILTINS_${target}_CMAKE_${lang}_local_flags "--target=${target}")
+    if(NOT ${target} STREQUAL "aarch64-none-elf")
+      set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mthumb")
+    endif()
     if(${target} STREQUAL "armv8m.main-none-eabi")
-      set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=softfp -march=armv8m.main+fp+dsp -mcpu=cortex-m33" CACHE STRING "")
+      set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=softfp -march=armv8m.main+fp+dsp -mcpu=cortex-m33")
     endif()
     if(${target} STREQUAL "armv8.1m.main-none-eabi")
-      set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55" CACHE STRING "")
+      set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55")
     endif()
     set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "${BUILTINS_${target}_CMAKE_${lang}_local_flags}" CACHE STRING "")
   endforeach()
@@ -332,12 +333,15 @@ foreach(target armv6m-none-eabi;armv7m-none-eabi;armv8m.main-none-eabi;armv8.1m.
   foreach(lang C;CXX;ASM)
     # TODO: The preprocessor defines workaround various issues in libc and libc++ integration.
     # These should be addressed and removed over time.
-    set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "--target=${target} -mthumb -Wno-atomic-alignment \"-Dvfprintf(stream, format, vlist)=vprintf(format, vlist)\" \"-Dfprintf(stream, format, ...)=printf(format)\" \"-Dgettimeofday(tv, tz)\" -D_LIBCPP_PRINT=1")
+    set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "--target=${target} -Wno-atomic-alignment \"-Dvfprintf(stream, format, vlist)=vprintf(format, vlist)\" \"-Dfprintf(stream, format, ...)=printf(format)\" \"-Dgettimeofday(tv, tz)\" -D_LIBCPP_PRINT=1")
+    if(NOT ${target} STREQUAL "aarch64-none-elf")
+      set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mthumb")
+    endif()
     if(${target} STREQUAL "armv8m.main-none-eabi")
-      set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=softfp -march=armv8m.main+fp+dsp -mcpu=cortex-m33" CACHE STRING "")
+      set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=softfp -march=armv8m.main+fp+dsp -mcpu=cortex-m33")
     endif()
     if(${target} STREQUAL "armv8.1m.main-none-eabi")
-      set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55" CACHE STRING "")
+      set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55")
     endif()
     set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "${RUNTIMES_${target}_CMAKE_${lang}_local_flags}" CACHE STRING "")
   endforeach()
diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake
index 2d2dcb9ae6798..07637cd0ed08f 100644
--- a/clang/cmake/caches/Fuchsia.cmake
+++ b/clang/cmake/caches/Fuchsia.cmake
@@ -6,7 +6,7 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "")
 
 set(PACKAGE_VENDOR Fuchsia CACHE STRING "")
 
-set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;libc;lld;llvm;polly")
+set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;lld;llvm;polly")
 
 set(LLVM_ENABLE_DIA_SDK OFF CACHE BOOL "")
 set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
@@ -17,7 +17,6 @@ set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
 set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
 set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
 set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
-set(LIBC_HDRGEN_ONLY ON CACHE BOOL "")
 set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "")
 set(LLDB_ENABLE_CURSES OFF CACHE BOOL "")
 set(LLDB_ENABLE_LIBEDIT OFF CACHE BOOL "")
diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst
index e17d741b0a00e..c8f1d7f5a7758 100644
--- a/clang/docs/ClangFormat.rst
+++ b/clang/docs/ClangFormat.rst
@@ -49,7 +49,7 @@ to format C/C++/Java/JavaScript/JSON/Objective-C/Protobuf/C# code.
                                      supported:
                                        CSharp: .cs
                                        Java: .java
-                                       JavaScript: .mjs .js .ts
+                                       JavaScript: .js .mjs .cjs .ts
                                        Json: .json
                                        Objective-C: .m .mm
                                        Proto: .proto .protodevel
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index dc34094b5053a..4be448171699c 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -3759,9 +3759,9 @@ the configuration (without a prefix: ``Auto``).
   lists.
 
   Important differences:
-  - No spaces inside the braced list.
-  - No line break before the closing brace.
-  - Indentation with the continuation indent, not with the block indent.
+  * No spaces inside the braced list.
+  * No line break before the closing brace.
+  * Indentation with the continuation indent, not with the block indent.
 
   Fundamentally, C++11 braced lists are formatted exactly like function
   calls would be formatted in their place. If the braced list follows a name
@@ -4104,10 +4104,10 @@ the configuration (without a prefix: ``Auto``).
   When guessing whether a #include is the "main" include (to assign
   category 0, see above), use this regex of allowed suffixes to the header
   stem. A partial match is done, so that:
-  - "" means "arbitrary suffix"
-  - "$" means "no suffix"
+  * ``""`` means "arbitrary suffix"
+  * ``"$"`` means "no suffix"
 
-  For example, if configured to "(_test)?$", then a header a.h would be seen
+  For example, if configured to ``"(_test)?$"``, then a header a.h would be seen
   as the "main" include in both a.cc and a_test.cc.
 
 .. _IncludeIsMainSourceRegex:
@@ -5313,21 +5313,21 @@ the configuration (without a prefix: ``Auto``).
 
 **QualifierOrder** (``List of Strings``) :versionbadge:`clang-format 14` :ref:`¶ <QualifierOrder>`
   The order in which the qualifiers appear.
-  Order is an array that can contain any of the following:
+  The order is an array that can contain any of the following:
 
-    * const
-    * inline
-    * static
-    * friend
-    * constexpr
-    * volatile
-    * restrict
-    * type
+    * ``const``
+    * ``inline``
+    * ``static``
+    * ``friend``
+    * ``constexpr``
+    * ``volatile``
+    * ``restrict``
+    * ``type``
 
 
   .. note::
 
-   It **must** contain ``type``.
+   It must contain ``type``.
 
   Items to the left of ``type`` will be placed to the left of the type and
   aligned in the order supplied. Items to the right of ``type`` will be
@@ -6645,12 +6645,11 @@ the configuration (without a prefix: ``Auto``).
 .. _StatementMacros:
 
 **StatementMacros** (``List of Strings``) :versionbadge:`clang-format 8` :ref:`¶ <StatementMacros>`
-  A vector of macros that should be interpreted as complete
-  statements.
+  A vector of macros that should be interpreted as complete statements.
 
-  Typical macros are expressions, and require a semi-colon to be
-  added; sometimes this is not the case, and this allows to make
-  clang-format aware of such cases.
+  Typical macros are expressions and require a semicolon to be added.
+  Sometimes this is not the case, and this allows to make clang-format aware
+  of such cases.
 
   For example: Q_UNUSED
 
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index c82e3b017b9d1..475a6372ca696 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -648,7 +648,8 @@ elementwise to the input.
 Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±infinity
 
 The integer elementwise intrinsics, including ``__builtin_elementwise_popcount``,
-can be called in a ``constexpr`` context.
+``__builtin_elementwise_bitreverse``, ``__builtin_elementwise_add_sat``,
+``__builtin_elementwise_sub_sat`` can be called in a ``constexpr`` context.
 
 ============================================== ====================================================================== =========================================
          Name                                   Operation                                                             Supported element types
@@ -1989,7 +1990,7 @@ Enumerations with a fixed underlying type
 -----------------------------------------
 
 Clang provides support for C++11 enumerations with a fixed underlying type
-within Objective-C.  For example, one can write an enumeration type as:
+within Objective-C and C `prior to C23 <https://open-std.org/JTC1/SC22/WG14/www/docs/n3030.htm>`_.  For example, one can write an enumeration type as:
 
 .. code-block:: c++
 
@@ -2001,6 +2002,14 @@ value, is ``unsigned char``.
 Use ``__has_feature(objc_fixed_enum)`` to determine whether support for fixed
 underlying types is available in Objective-C.
 
+Use ``__has_extension(c_fixed_enum)`` to determine whether support for fixed
+underlying types is available in C prior to C23. This will also report ``true`` in C23
+and later modes as the functionality is available even if it's not an extension in
+those modes.
+
+Use ``__has_feature(c_fixed_enum)`` to determine whether support for fixed
+underlying types is available in C23 and later.
+
 Interoperability with C++11 lambdas
 -----------------------------------
 
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index c6307954d7f1b..f18e9cf134169 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -804,6 +804,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('exportDecl0')"><a name="exportDecl0Anchor">exportDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExportDecl.html">ExportDecl</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="exportDecl0"><pre>Matches any export declaration.
+
+Example matches following declarations.
+  export void foo();
+  export { void foo(); }
+  export namespace { void foo(); }
+  export int v;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('fieldDecl0')"><a name="fieldDecl0Anchor">fieldDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FieldDecl.html">FieldDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="fieldDecl0"><pre>Matches field declarations.
 
diff --git a/clang/docs/RealtimeSanitizer.rst b/clang/docs/RealtimeSanitizer.rst
index 233a91f668416..f5d29af2bef3c 100644
--- a/clang/docs/RealtimeSanitizer.rst
+++ b/clang/docs/RealtimeSanitizer.rst
@@ -187,16 +187,19 @@ A **partial** list of flags RealtimeSanitizer respects:
    * - ``abort_on_error``
      - OS dependent
      - boolean
-     - If true, the tool calls abort() instead of _exit() after printing the error report. On some OSes (OSX, for exmple) this is beneficial because a better stack trace is emitted on crash.
+     - If true, the tool calls ``abort()`` instead of ``_exit()`` after printing the error report. On some OSes (MacOS, for exmple) this is beneficial because a better stack trace is emitted on crash.
    * - ``symbolize``
      - ``true``
      - boolean
      - If set, use the symbolizer to turn virtual addresses to file/line locations. If false, can greatly speed up the error reporting.
    * - ``suppressions``
-     - ""
+     - ``""``
      - path
-     - If set to a valid suppressions file, will suppress issue reporting. See details in "Disabling", below.
-
+     - If set to a valid suppressions file, will suppress issue reporting. See details in `Disabling and Suppressing`_.
+   * - ``verify_interceptors``
+     - ``true``
+     - boolean
+     - If true, verifies interceptors are working at initialization. The program will abort with error ``==ERROR: Interceptors are not working. This may be because RealtimeSanitizer is loaded too late (e.g. via dlopen)`` if an issue is detected.
 
 Some issues with flags can be debugged using the ``verbosity=$NUM`` flag:
 
@@ -244,6 +247,7 @@ To register a callback which will be invoked before a RTSan kills the process:
     ...
   }
 
+.. _disabling-and-suppressing:
 
 Disabling and suppressing
 -------------------------
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6c40e48e2f49b..a541d399d1e74 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -52,6 +52,12 @@ code bases.
   `migrate to Vulkan <https://developer.android.com/guide/topics/renderscript/migrate>`_
   or other options.
 
+- Clang now emits distinct type-based alias analysis tags for incompatible
+  pointers by default, enabling more powerful alias analysis when accessing
+  pointer types. This change may silently change code behavior for code
+  containing strict-aliasing violations. The new default behavior can be
+  disabled using ``-fno-pointer-tbaa``.
+
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
 
@@ -158,6 +164,35 @@ C++ Specific Potentially Breaking Changes
 
   Previously, this code was erroneously accepted.
 
+- Clang will now consider the implicitly deleted destructor of a union or
+  a non-union class without virtual base class to be ``constexpr`` in C++20
+  mode (Clang 19 only did so in C++23 mode but the standard specification for
+  this changed in C++20). (#GH85550)
+
+  .. code-block:: c++
+
+    struct NonLiteral {
+      NonLiteral() {}
+      ~NonLiteral() {}
+    };
+
+    template <class T>
+    struct Opt {
+      union {
+        char c;
+        T data;
+      };
+      bool engaged = false;
+
+      constexpr Opt() {}
+      constexpr ~Opt() {
+        if (engaged)
+          data.~T();
+      }
+    };
+
+    // Previously only accepted in C++23 and later, now also accepted in C++20.
+    consteval void foo() { Opt<NonLiteral>{}; }
 
 ABI Changes in This Version
 ---------------------------
@@ -227,6 +262,8 @@ C++2c Feature Support
 - Added the ``__builtin_is_within_lifetime`` builtin, which supports
   `P2641R4 Checking if a union alternative is active <https://wg21.link/p2641r4>`_
 
+- Implemented `P3176R1 The Oxford variadic comma <https://wg21.link/P3176R1>`_
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 - Removed the restriction to literal types in constexpr functions in C++23 mode.
@@ -279,6 +316,9 @@ Resolutions to C++ Defect Reports
   by default.
   (`CWG2521: User-defined literals and reserved identifiers <https://cplusplus.github.io/CWG/issues/2521.html>`_).
 
+- Fix name lookup for a dependent base class that is the current instantiation.
+  (`CWG591: When a dependent base class is the current instantiation <https://cplusplus.github.io/CWG/issues/591.html>`_).
+
 C Language Changes
 ------------------
 
@@ -347,7 +387,7 @@ Non-comprehensive list of changes in this release
 
 - The new builtin ``__builtin_counted_by_ref`` was added. In contexts where the
   programmer needs access to the ``counted_by`` attribute's field, but it's not
-  available --- e.g. in macros. For instace, it can be used to automatically
+  available --- e.g. in macros. For instance, it can be used to automatically
   set the counter during allocation in the Linux kernel:
 
   .. code-block:: c
@@ -368,11 +408,11 @@ Non-comprehensive list of changes in this release
   The flexible array member (FAM) can now be accessed immediately without causing
   issues with the sanitizer because the counter is automatically set.
 
-- ``__builtin_reduce_add`` function can now be used in constant expressions.
-- ``__builtin_reduce_mul`` function can now be used in constant expressions.
-- ``__builtin_reduce_and`` function can now be used in constant expressions.
-- ``__builtin_reduce_or`` and ``__builtin_reduce_xor`` functions can now be used in constant expressions.
-- ``__builtin_elementwise_popcount`` function can now be used in constant expressions.
+- The following builtins can now be used in constant expressions: ``__builtin_reduce_add``,
+  ``__builtin_reduce_mul``, ``__builtin_reduce_and``, ``__builtin_reduce_or``,
+  ``__builtin_reduce_xor``, ``__builtin_elementwise_popcount``,
+  ``__builtin_elementwise_bitreverse``, ``__builtin_elementwise_add_sat``,
+  ``__builtin_elementwise_sub_sat``.
 
 New Compiler Flags
 ------------------
@@ -384,6 +424,12 @@ New Compiler Flags
   existing ``-fno-c++-static-destructors`` flag) skips all static
   destructors registration.
 
+- The ``-Warray-compare`` warning has been added to warn about array comparison
+  on versions older than C++20.
+
+- The ``-Warray-compare-cxx26`` warning has been added to warn about array comparison
+  starting from C++26, this warning is enabled as an error by default.
+
 Deprecated Compiler Flags
 -------------------------
 
@@ -407,10 +453,14 @@ Modified Compiler Flags
   to utilize these vector libraries. The behavior for all other vector function
   libraries remains unchanged.
 
-- The ``-Wnontrivial-memaccess`` warning has been updated to also warn about
-  passing non-trivially-copyable destrination parameter to ``memcpy``,
+- The ``-Wnontrivial-memcall`` warning has been added to warn about
+  passing non-trivially-copyable destination parameter to ``memcpy``,
   ``memset`` and similar functions for which it is a documented undefined
-  behavior.
+  behavior. It is implied by ``-Wnontrivial-memaccess``
+
+- Added ``-fmodules-reduced-bmi`` flag corresponding to
+  ``-fexperimental-modules-reduced-bmi`` flag. The ``-fmodules-reduced-bmi`` flag
+  is intended to be enabled by default in the future.
 
 Removed Compiler Flags
 -------------------------
@@ -444,6 +494,11 @@ Attribute Changes in Clang
 - The ``hybrid_patchable`` attribute is now supported on ARM64EC targets. It can be used to specify
   that a function requires an additional x86-64 thunk, which may be patched at runtime.
 
+- The attribute ``[[clang::no_specializations]]`` has been added to warn
+  users that a specific template shouldn't be specialized. This is useful for
+  e.g. standard library type traits, where adding a specialization results in
+  undefined behaviour.
+
 - ``[[clang::lifetimebound]]`` is now explicitly disallowed on explicit object member functions
   where they were previously silently ignored.
 
@@ -476,6 +531,9 @@ Attribute Changes in Clang
 
 - The ``target_version`` attribute is now only supported for AArch64 and RISC-V architectures.
 
+- Clang now permits the usage of the placement new operator in ``[[msvc::constexpr]]``
+  context outside of the std namespace. (#GH74924)
+
 Improvements to Clang's diagnostics
 -----------------------------------
 
@@ -590,6 +648,22 @@ Improvements to Clang's diagnostics
 
 - Fixed a false negative ``-Wunused-private-field`` diagnostic when a defaulted comparison operator is defined out of class (#GH116961).
 
+- Clang now diagnoses dangling references for C++20's parenthesized aggregate initialization (#101957).
+
+- Fixed a bug where Clang would not emit ``-Wunused-private-field`` warnings when an unrelated class 
+  defined a defaulted comparison operator (#GH116270).
+
+  .. code-block:: c++
+
+    class A {
+    private:
+      int a; // warning: private field 'a' is not used, no diagnostic previously
+    };
+
+    class C {
+      bool operator==(const C&) = default;
+    };
+
 Improvements to Clang's time-trace
 ----------------------------------
 
@@ -612,6 +686,9 @@ Bug Fixes in This Version
 - Fixed a crash when GNU statement expression contains invalid statement (#GH113468).
 - Fixed a failed assertion when using ``__attribute__((noderef))`` on an
   ``_Atomic``-qualified type (#GH116124).
+- No longer return ``false`` for ``noexcept`` expressions involving a
+  ``delete`` which resolves to a destroying delete but the type of the object
+  being deleted has a potentially throwing destructor (#GH118660).
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -625,6 +702,8 @@ Bug Fixes to Compiler Builtins
 
 - Fix ``__has_builtin`` incorrectly returning ``false`` for some C++ type traits. (#GH111477)
 
+- Fix ``__builtin_source_location`` incorrectly returning wrong column for method chains. (#GH119129)
+
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -636,7 +715,7 @@ Bug Fixes to C++ Support
 - Fixed a failed assertion when checking invalid delete operator declaration. (#GH96191)
 - Fix a crash when checking destructor reference with an invalid initializer. (#GH97230)
 - Clang now correctly parses potentially declarative nested-name-specifiers in pointer-to-member declarators.
-- Fix a crash when checking the initialzier of an object that was initialized
+- Fix a crash when checking the initializer of an object that was initialized
   with a string literal. (#GH82167)
 - Fix a crash when matching template template parameters with templates which have
   parameters of different class type. (#GH101394)
@@ -680,7 +759,7 @@ Bug Fixes to C++ Support
 - Fix a crash when using ``source_location`` in the trailing return type of a lambda expression. (#GH67134)
 - A follow-up fix was added for (#GH61460), as the previous fix was not entirely correct. (#GH86361), (#GH112352)
 - Fixed a crash in the typo correction of an invalid CTAD guide. (#GH107887)
-- Fixed a crash when clang tries to subtitute parameter pack while retaining the parameter
+- Fixed a crash when clang tries to substitute parameter pack while retaining the parameter
   pack. (#GH63819), (#GH107560)
 - Fix a crash when a static assert declaration has an invalid close location. (#GH108687)
 - Avoided a redundant friend declaration instantiation under a certain ``consteval`` context. (#GH107175)
@@ -719,9 +798,18 @@ Bug Fixes to C++ Support
 - Name independent data members were not correctly initialized from default member initializers. (#GH114069)
 - Fixed expression transformation for ``[[assume(...)]]``, allowing using pack indexing expressions within the
   assumption if they also occur inside of a dependent lambda. (#GH114787)
+- Lambdas now capture function types without considering top-level const qualifiers. (#GH84961)
 - Clang now uses valid deduced type locations when diagnosing functions with trailing return type
   missing placeholder return type. (#GH78694)
 - Fixed a bug where bounds of partially expanded pack indexing expressions were checked too early. (#GH116105)
+- Fixed an assertion failure caused by using ``consteval`` in condition in consumed analyses. (#GH117385)
+- Fix a crash caused by incorrect argument position in merging deduced template arguments. (#GH113659)
+- Fixed a parser crash when using pack indexing as a nested name specifier. (#GH119072) 
+- Fixed a null pointer dereference issue when heuristically computing ``sizeof...(pack)`` expressions. (#GH81436)
+- Fixed an assertion failure caused by mangled names with invalid identifiers. (#GH112205)
+- Fixed an incorrect lambda scope of generic lambdas that caused Clang to crash when computing potential lambda
+  captures at the end of a full expression. (#GH115931)
+- Clang no longer rejects deleting a pointer of incomplete enumeration type. (#GH99278)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -733,6 +821,7 @@ Bug Fixes to AST Handling
   and ``relatedalso`` comment commands.
 - Clang now uses the location of the begin of the member expression for ``CallExpr``
   involving deduced ``this``. (#GH116928)
+- Fixed printout of AST that uses pack indexing expression. (#GH116486)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -887,9 +976,15 @@ and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
 and [Non-trapping float-to-int Conversions] language features, which are
 [widely implemented in engines].
 
+A new Lime1 target CPU is added, -mcpu=lime1. This CPU follows the definition of
+the Lime1 CPU [here], and enables -mmultivalue, -mmutable-globals,
+-mcall-indirect-overlong, -msign-ext, -mbulk-memory-opt, -mnontrapping-fptoint,
+and -mextended-const.
+
 [Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
 [Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
 [widely implemented in engines]: https://webassembly.org/features/
+[here]: https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1
 
 AVR Support
 ^^^^^^^^^^^
@@ -923,6 +1018,14 @@ AST Matchers
 - Ensure ``hasName`` matches template specializations across inline namespaces,
   making `matchesNodeFullSlow` and `matchesNodeFullFast` consistent.
 
+- Improved the performance of the ``getExpansionLocOfMacro`` by tracking already processed macros during recursion.
+
+- Add ``exportDecl`` matcher to match export declaration.
+
+- Ensure ``hasType`` and ``hasDeclaration`` match Objective-C interface declarations.
+
+- Ensure ``pointee`` matches Objective-C pointer types.
+
 clang-format
 ------------
 
@@ -974,8 +1077,12 @@ Improvements
 Moved checkers
 ^^^^^^^^^^^^^^
 
+- The checker ``alpha.core.IdenticalExpr`` was deleted because it was
+  duplicated in the clang-tidy checkers ``misc-redundant-expression`` and
+  ``bugprone-branch-clone``.
+
 - The checker ``alpha.security.MallocOverflow`` was deleted because it was
-  badly implemented and its agressive logic produced too many false positives.
+  badly implemented and its aggressive logic produced too many false positives.
   To detect too large arguments passed to malloc, consider using the checker
   ``alpha.taint.TaintedAlloc``.
 
@@ -985,6 +1092,16 @@ Moved checkers
   original checkers were implemented only using AST matching and make more
   sense as a single clang-tidy check.
 
+- The checker ``alpha.unix.Chroot`` was modernized, improved and moved to
+  ``unix.Chroot``. Testing was done on open source projects that use chroot(),
+  and false issues addressed in the improvements based on real use cases. Open
+  source projects used for testing include nsjail, lxroot, dive and ruri.
+  This checker conforms to SEI Cert C recommendation `POS05-C. Limit access to
+  files by creating a jail
+  <https://wiki.sei.cmu.edu/confluence/display/c/POS05-C.+Limit+access+to+files+by+creating+a+jail>`_.
+  Fixes (#GH34697).
+  (#GH117791) [Documentation](https://clang.llvm.org/docs/analyzer/checkers.html#unix-chroot-c).
+
 .. _release-notes-sanitizers:
 
 Sanitizers
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index 8e22adad15106..93edce0cf90b7 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -602,16 +602,16 @@ unnecessary dependencies for the BMI. To mitigate the problem, Clang has a
 compiler option to reduce the information contained in the BMI. These two
 formats are known as Full BMI and Reduced BMI, respectively.
 
-Users can use the ``-fexperimental-modules-reduced-bmi`` option to produce a
+Users can use the ``-fmodules-reduced-bmi`` option to produce a
 Reduced BMI.
 
 For the one-phase compilation model (CMake implements this model), with
-``-fexperimental-modules-reduced-bmi``, the generated BMI will be a Reduced
+``-fmodules-reduced-bmi``, the generated BMI will be a Reduced
 BMI automatically. (The output path of the BMI is specified by
 ``-fmodule-output=`` as usual with the one-phase compilation model).
 
 It is also possible to produce a Reduced BMI with the two-phase compilation
-model. When ``-fexperimental-modules-reduced-bmi``, ``--precompile``, and
+model. When ``-fmodules-reduced-bmi``, ``--precompile``, and
 ``-fmodule-output=`` are specified, the generated BMI specified by ``-o`` will
 be a full BMI and the BMI specified by ``-fmodule-output=`` will be a Reduced
 BMI. The dependency graph in this case would look like:
@@ -625,7 +625,7 @@ BMI. The dependency graph in this case would look like:
                                                -> ...
                                                -> consumer_n.cpp
 
-Clang does not emit diagnostics when ``-fexperimental-modules-reduced-bmi`` is
+Clang does not emit diagnostics when ``-fmodules-reduced-bmi`` is
 used with a non-module unit. This design permits users of the one-phase
 compilation model to try using reduced BMIs without needing to modify the build
 system. The two-phase compilation module requires build system support.
@@ -691,11 +691,10 @@ ensure it is reachable, e.g. ``using N::g;``.
 Support for Reduced BMIs is still experimental, but it may become the default
 in the future. The expected roadmap for Reduced BMIs as of Clang 19.x is:
 
-1. ``-fexperimental-modules-reduced-bmi`` is opt-in for 1~2 releases. The period depends
+1. ``-fexperimental-modules-reduced-bmi`` was introduced in v19.x
+2. For v20.x, ``-fmodules-reduced-bmi`` is introduced as an equivalent non-experimental
+   option. It is expected to stay opt-in for 1~2 releases, though the period depends
    on user feedback and may be extended.
-2. Announce that Reduced BMIs are no longer experimental and introduce
-   ``-fmodules-reduced-bmi`` as a new option, and recommend use of the new
-   option. This transition is expected to take 1~2 additional releases as well.
 3. Finally, ``-fmodules-reduced-bmi`` will be the default. When that time
    comes, the term BMI will refer to the Reduced BMI and the Full BMI will only
    be meaningful to build systems which elect to support two-phase compilation.
@@ -814,8 +813,8 @@ With reduced BMI, non-cascading changes can be more powerful. For example,
 
 .. code-block:: console
 
-  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fexperimental-modules-reduced-bmi -o A.o
-  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fexperimental-modules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
+  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fmodules-reduced-bmi -o A.o
+  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fmodules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
   $ md5sum B.pcm
   6c2bd452ca32ab418bf35cd141b060b9  B.pcm
 
@@ -831,8 +830,8 @@ and recompile the example:
 
 .. code-block:: console
 
-  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fexperimental-modules-reduced-bmi -o A.o
-  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fexperimental-modules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
+  $ clang++ -std=c++20 A.cppm -c -fmodule-output=A.pcm  -fmodules-reduced-bmi -o A.o
+  $ clang++ -std=c++20 B.cppm -c -fmodule-output=B.pcm  -fmodules-reduced-bmi -o B.o -fmodule-file=A=A.pcm
   $ md5sum B.pcm
   6c2bd452ca32ab418bf35cd141b060b9  B.pcm
 
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 2ef603d64557f..8af9f5be644a0 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -1059,6 +1059,17 @@ In this way, the user may only need to specify a root configuration file with
     -L <CFGDIR>/lib
     -T <CFGDIR>/ldscripts/link.ld
 
+Usually, config file options are placed before command-line options, regardless
+of the actual operation to be performed. The exception is being made for the
+options prefixed with the ``$`` character. These will be used only when linker
+is being invoked, and added after all of the command-line specified linker
+inputs. Here is some example of ``$``-prefixed options:
+
+::
+
+    $-Wl,-Bstatic $-lm
+    $-Wl,-Bshared
+
 Language and Target-Independent Features
 ========================================
 
@@ -2660,7 +2671,7 @@ usual build cycle when using sample profilers for optimization:
 
      > clang-cl /O2 -gdwarf -gline-tables-only ^
        /clang:-fdebug-info-for-profiling /clang:-funique-internal-linkage-names ^
-       /fprofile-sample-use=code.prof code.cc /Fe:code /fuse-ld=lld /link /debug:dwarf
+       -fprofile-sample-use=code.prof code.cc /Fe:code -fuse-ld=lld /link /debug:dwarf
 
    [OPTIONAL] Sampling-based profiles can have inaccuracies or missing block/
    edge counters. The profile inference algorithm (profi) can be used to infer
@@ -2679,7 +2690,7 @@ usual build cycle when using sample profilers for optimization:
 
      > clang-cl /clang:-fsample-profile-use-profi /O2 -gdwarf -gline-tables-only ^
        /clang:-fdebug-info-for-profiling /clang:-funique-internal-linkage-names ^
-       /fprofile-sample-use=code.prof code.cc /Fe:code /fuse-ld=lld /link /debug:dwarf
+       -fprofile-sample-use=code.prof code.cc /Fe:code -fuse-ld=lld /link /debug:dwarf
 
 Sample Profile Formats
 """"""""""""""""""""""
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index f34b25cd04bd1..29d5e1f92a69c 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -1750,6 +1750,37 @@ Critical section handling functions modeled by this checker:
    }
  }
 
+.. _unix-Chroot:
+
+unix.Chroot (C)
+"""""""""""""""
+Check improper use of chroot described by SEI Cert C recommendation `POS05-C.
+Limit access to files by creating a jail
+<https://wiki.sei.cmu.edu/confluence/display/c/POS05-C.+Limit+access+to+files+by+creating+a+jail>`_.
+The checker finds usage patterns where ``chdir("/")`` is not called immediately
+after a call to ``chroot(path)``.
+
+.. code-block:: c
+
+ void f();
+
+ void test_bad() {
+   chroot("/usr/local");
+   f(); // warn: no call of chdir("/") immediately after chroot
+ }
+
+  void test_bad_path() {
+    chroot("/usr/local");
+    chdir("/usr"); // warn: no call of chdir("/") immediately after chroot
+    f();
+  }
+
+ void test_good() {
+   chroot("/usr/local");
+   chdir("/"); // no warning
+   f();
+ }
+
 .. _unix-Errno:
 
 unix.Errno (C)
@@ -1899,6 +1930,29 @@ Check the size argument passed into C string functions for common erroneous patt
      // warn: potential buffer overflow
  }
 
+.. _unix-cstring-NotNullTerminated:
+
+unix.cstring.NotNullTerminated (C)
+""""""""""""""""""""""""""""""""""
+Check for arguments which are not null-terminated strings;
+applies to the ``strlen``, ``strcpy``, ``strcat``, ``strcmp`` family of functions.
+
+Only very fundamental cases are detected where the passed memory block is
+absolutely different from a null-terminated string. This checker does not
+find if a memory buffer is passed where the terminating zero character
+is missing.
+
+.. code-block:: c
+
+ void test1() {
+   int l = strlen((char *)&test1); // warn
+ }
+
+ void test2() {
+ label:
+   int l = strlen((char *)&&label); // warn
+ }
+
 .. _unix-cstring-NullArg:
 
 unix.cstring.NullArg (C)
@@ -2788,36 +2842,6 @@ Check for assignment of a fixed address to a pointer.
    p = (int *) 0x10000; // warn
  }
 
-.. _alpha-core-IdenticalExpr:
-
-alpha.core.IdenticalExpr (C, C++)
-"""""""""""""""""""""""""""""""""
-Warn about unintended use of identical expressions in operators.
-
-.. code-block:: cpp
-
- // C
- void test() {
-   int a = 5;
-   int b = a | 4 | a; // warn: identical expr on both sides
- }
-
- // C++
- bool f(void);
-
- void test(bool b) {
-   int i = 10;
-   if (f()) { // warn: true and false branches are identical
-     do {
-       i--;
-     } while (f());
-   } else {
-     do {
-       i--;
-     } while (f());
-   }
- }
-
 .. _alpha-core-PointerArithm:
 
 alpha.core.PointerArithm (C)
@@ -3275,21 +3299,6 @@ SEI CERT checkers which tries to find errors based on their `C coding rules <htt
 alpha.unix
 ^^^^^^^^^^
 
-.. _alpha-unix-Chroot:
-
-alpha.unix.Chroot (C)
-"""""""""""""""""""""
-Check improper use of chroot.
-
-.. code-block:: c
-
- void f();
-
- void test() {
-   chroot("/usr/local");
-   f(); // warn: no call of chdir("/") immediately after chroot
- }
-
 .. _alpha-unix-PthreadLock:
 
 alpha.unix.PthreadLock (C)
@@ -3367,29 +3376,6 @@ Checks for overlap in two buffer arguments. Applies to:  ``memcpy, mempcpy, wmem
    memcpy(a + 2, a + 1, 8); // warn
  }
 
-.. _alpha-unix-cstring-NotNullTerminated:
-
-alpha.unix.cstring.NotNullTerminated (C)
-""""""""""""""""""""""""""""""""""""""""
-Check for arguments which are not null-terminated strings;
-applies to the ``strlen``, ``strcpy``, ``strcat``, ``strcmp`` family of functions.
-
-Only very fundamental cases are detected where the passed memory block is
-absolutely different from a null-terminated string. This checker does not
-find if a memory buffer is passed where the terminating zero character
-is missing.
-
-.. code-block:: c
-
- void test1() {
-   int l = strlen((char *)&test); // warn
- }
-
- void test2() {
- label:
-   int l = strlen((char *)&&label); // warn
- }
-
 .. _alpha-unix-cstring-OutOfBounds:
 
 alpha.unix.cstring.OutOfBounds (C)
@@ -3452,6 +3438,31 @@ alpha.WebKit
 
 .. _alpha-webkit-NoUncheckedPtrMemberChecker:
 
+alpha.webkit.MemoryUnsafeCastChecker
+""""""""""""""""""""""""""""""""""""""
+Check for all casts from a base type to its derived type as these might be memory-unsafe.
+
+Example:
+
+.. code-block:: cpp
+
+    class Base { };
+    class Derived : public Base { };
+
+    void f(Base* base) {
+        Derived* derived = static_cast<Derived*>(base); // ERROR
+    }
+
+For all cast operations (C-style casts, static_cast, reinterpret_cast, dynamic_cast), if the source type a `Base*` and the destination type is `Derived*`, where `Derived` inherits from `Base`, the static analyzer should signal an error.
+
+This applies to:
+
+- C structs, C++ structs and classes, and Objective-C classes and protocols.
+- Pointers and references.
+- Inside template instantiations and macro expansions that are visible to the compiler.
+
+For types like this, instead of using built in casts, the programmer will use helper functions that internally perform the appropriate type check and disable static analysis.
+
 alpha.webkit.NoUncheckedPtrMemberChecker
 """"""""""""""""""""""""""""""""""""""""
 Raw pointers and references to an object which supports CheckedPtr or CheckedRef can't be used as class members. Only CheckedPtr, CheckedRef, RefPtr, or Ref are allowed.
diff --git a/clang/docs/tools/dump_ast_matchers.py b/clang/docs/tools/dump_ast_matchers.py
index 705ff0d4d4098..b6f00657ec914 100755
--- a/clang/docs/tools/dump_ast_matchers.py
+++ b/clang/docs/tools/dump_ast_matchers.py
@@ -5,6 +5,7 @@
 
 import collections
 import re
+import os
 
 try:
     from urllib.request import urlopen
@@ -18,7 +19,11 @@
     CLASS_INDEX_PAGE = None
     print("Unable to get %s: %s" % (CLASS_INDEX_PAGE_URL, e))
 
-MATCHERS_FILE = "../../include/clang/ASTMatchers/ASTMatchers.h"
+CURRENT_DIR = os.path.dirname(__file__)
+MATCHERS_FILE = os.path.join(
+    CURRENT_DIR, "../../include/clang/ASTMatchers/ASTMatchers.h"
+)
+HTML_FILE = os.path.join(CURRENT_DIR, "../LibASTMatchersReference.html")
 
 # Each matcher is documented in one row of the form:
 #   result | name | argA
@@ -590,7 +595,7 @@ def sort_table(matcher_type, matcher_map):
 narrowing_matcher_table = sort_table("NARROWING", narrowing_matchers)
 traversal_matcher_table = sort_table("TRAVERSAL", traversal_matchers)
 
-reference = open("../LibASTMatchersReference.html").read()
+reference = open(HTML_FILE).read()
 reference = re.sub(
     r"<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->",
     node_matcher_table,
diff --git a/clang/docs/tools/dump_format_help.py b/clang/docs/tools/dump_format_help.py
index 1a9afde1b9db9..baf90048ee135 100755
--- a/clang/docs/tools/dump_format_help.py
+++ b/clang/docs/tools/dump_format_help.py
@@ -2,6 +2,7 @@
 # A tool to parse the output of `clang-format --help` and update the
 # documentation in ../ClangFormat.rst automatically.
 
+import argparse
 import os
 import re
 import subprocess
@@ -26,7 +27,7 @@ def indent(text, columns, indent_first_line=True):
 
 
 def get_help_output():
-    args = ["clang-format", "--help"]
+    args = [binary, "--help"]
     cmd = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     out, _ = cmd.communicate()
     out = out.decode(sys.stdout.encoding)
@@ -54,13 +55,22 @@ def validate(text, columns):
             print("warning: line too long:\n", line, file=sys.stderr)
 
 
+p = argparse.ArgumentParser()
+p.add_argument("-d", "--directory", help="directory of clang-format")
+p.add_argument("-o", "--output", help="path of output file")
+opts = p.parse_args()
+
+binary = "clang-format"
+if opts.directory:
+    binary = opts.directory + "/" + binary
+
 help_text = get_help_text()
 validate(help_text, 100)
 
-with open(DOC_FILE) as f:
+with open(DOC_FILE, encoding="utf-8") as f:
     contents = f.read()
 
 contents = substitute(contents, "FORMAT_HELP", help_text)
 
-with open(DOC_FILE, "wb") as output:
+with open(opts.output if opts.output else DOC_FILE, "wb") as output:
     output.write(contents.encode())
diff --git a/clang/docs/tools/dump_format_style.py b/clang/docs/tools/dump_format_style.py
index af0e658fcdc55..f00f3ee8b20e8 100755
--- a/clang/docs/tools/dump_format_style.py
+++ b/clang/docs/tools/dump_format_style.py
@@ -3,6 +3,7 @@
 # documentation in ../ClangFormatStyleOptions.rst automatically.
 # Run from the directory in which this file is located to update the docs.
 
+import argparse
 import inspect
 import os
 import re
@@ -20,7 +21,7 @@
 PLURALS_FILE = os.path.join(os.path.dirname(__file__), "plurals.txt")
 
 plurals: Set[str] = set()
-with open(PLURALS_FILE, "a+") as f:
+with open(PLURALS_FILE) as f:
     f.seek(0)
     plurals = set(f.read().splitlines())
 
@@ -474,6 +475,10 @@ class State:
         return options
 
 
+p = argparse.ArgumentParser()
+p.add_argument("-o", "--output", help="path of output file")
+args = p.parse_args()
+
 with open(FORMAT_STYLE_FILE) as f:
     opts = OptionsReader(f).read_options()
 with open(INCLUDE_STYLE_FILE) as f:
@@ -487,5 +492,5 @@ class State:
 
 contents = substitute(contents, "FORMAT_STYLE_OPTIONS", options_text)
 
-with open(DOC_FILE, "wb") as output:
+with open(args.output if args.output else DOC_FILE, "wb") as output:
     output.write(contents.encode())
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index ff374ad3ada06..9c01978fd49ef 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -542,6 +542,9 @@ class FunctionInfo : public CommonEntityInfo {
   /// The result type of this function, as a C type.
   std::string ResultType;
 
+  /// Ownership convention for return value
+  std::string SwiftReturnOwnership;
+
   /// The function parameters.
   std::vector<ParamInfo> Params;
 
@@ -622,7 +625,8 @@ inline bool operator==(const FunctionInfo &LHS, const FunctionInfo &RHS) {
          LHS.NumAdjustedNullable == RHS.NumAdjustedNullable &&
          LHS.NullabilityPayload == RHS.NullabilityPayload &&
          LHS.ResultType == RHS.ResultType && LHS.Params == RHS.Params &&
-         LHS.RawRetainCountConvention == RHS.RawRetainCountConvention;
+         LHS.RawRetainCountConvention == RHS.RawRetainCountConvention &&
+         LHS.SwiftReturnOwnership == RHS.SwiftReturnOwnership;
 }
 
 inline bool operator!=(const FunctionInfo &LHS, const FunctionInfo &RHS) {
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index e4a7957bfa775..cfb093fb39954 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -245,7 +245,11 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mutable llvm::FoldingSet<ObjCObjectPointerType> ObjCObjectPointerTypes;
   mutable llvm::FoldingSet<DependentUnaryTransformType>
     DependentUnaryTransformTypes;
-  mutable llvm::ContextualFoldingSet<AutoType, ASTContext&> AutoTypes;
+  // An AutoType can have a dependency on another AutoType via its template
+  // arguments. Since both dependent and dependency are on the same set,
+  // we can end up in an infinite recursion when looking for a node if we used
+  // a `FoldingSet`, since both could end up in the same bucket.
+  mutable llvm::DenseMap<llvm::FoldingSetNodeID, AutoType *> AutoTypes;
   mutable llvm::FoldingSet<DeducedTemplateSpecializationType>
     DeducedTemplateSpecializationTypes;
   mutable llvm::FoldingSet<AtomicType> AtomicTypes;
diff --git a/clang/include/clang/AST/AttrIterator.h b/clang/include/clang/AST/AttrIterator.h
index 66571e1cf0b8e..7e2bb0381d4c8 100644
--- a/clang/include/clang/AST/AttrIterator.h
+++ b/clang/include/clang/AST/AttrIterator.h
@@ -14,11 +14,13 @@
 #define LLVM_CLANG_AST_ATTRITERATOR_H
 
 #include "clang/Basic/LLVM.h"
+#include "llvm/ADT/ADL.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
 #include <cassert>
 #include <cstddef>
 #include <iterator>
+#include <type_traits>
 
 namespace clang {
 
@@ -113,13 +115,13 @@ inline bool hasSpecificAttr(const Container& container) {
           specific_attr_end<SpecificAttr>(container);
 }
 template <typename SpecificAttr, typename Container>
-inline SpecificAttr *getSpecificAttr(const Container& container) {
-  specific_attr_iterator<SpecificAttr, Container> i =
-      specific_attr_begin<SpecificAttr>(container);
-  if (i != specific_attr_end<SpecificAttr>(container))
-    return *i;
-  else
-    return nullptr;
+inline auto *getSpecificAttr(const Container &container) {
+  using ValueTy = llvm::detail::ValueOfRange<Container>;
+  using ValuePointeeTy = std::remove_pointer_t<ValueTy>;
+  using IterTy = std::conditional_t<std::is_const_v<ValuePointeeTy>,
+                                    const SpecificAttr, SpecificAttr>;
+  auto It = specific_attr_begin<IterTy>(container);
+  return It != specific_attr_end<IterTy>(container) ? *It : nullptr;
 }
 
 } // namespace clang
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index 2693cc0e95b4b..e389b5cd6df5b 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -890,6 +890,13 @@ class CXXRecordDecl : public RecordDecl {
             needsOverloadResolutionForDestructor()) &&
            "destructor should not be deleted");
     data().DefaultedDestructorIsDeleted = true;
+    // C++23 [dcl.constexpr]p3.2:
+    //   if the function is a constructor or destructor, its class does not have
+    //   any virtual base classes.
+    // C++20 [dcl.constexpr]p5:
+    //   The definition of a constexpr destructor whose function-body is
+    //   not = delete shall additionally satisfy...
+    data().DefaultedDestructorIsConstexpr = data().NumVBases == 0;
   }
 
   /// Determine whether this class should get an implicit move
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index e4bf54c3d77b7..dd92d40b80423 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -735,6 +735,7 @@ class RedeclarableTemplateDecl : public TemplateDecl,
   }
 
   void anchor() override;
+
 protected:
   template <typename EntryType> struct SpecEntryTraits {
     using DeclType = EntryType;
@@ -775,13 +776,22 @@ class RedeclarableTemplateDecl : public TemplateDecl,
     return SpecIterator<EntryType>(isEnd ? Specs.end() : Specs.begin());
   }
 
-  void loadLazySpecializationsImpl() const;
+  void loadLazySpecializationsImpl(bool OnlyPartial = false) const;
+
+  bool loadLazySpecializationsImpl(llvm::ArrayRef<TemplateArgument> Args,
+                                   TemplateParameterList *TPL = nullptr) const;
 
   template <class EntryType, typename ...ProfileArguments>
   typename SpecEntryTraits<EntryType>::DeclType*
   findSpecializationImpl(llvm::FoldingSetVector<EntryType> &Specs,
                          void *&InsertPos, ProfileArguments &&...ProfileArgs);
 
+  template <class EntryType, typename... ProfileArguments>
+  typename SpecEntryTraits<EntryType>::DeclType *
+  findSpecializationLocally(llvm::FoldingSetVector<EntryType> &Specs,
+                            void *&InsertPos,
+                            ProfileArguments &&...ProfileArgs);
+
   template <class Derived, class EntryType>
   void addSpecializationImpl(llvm::FoldingSetVector<EntryType> &Specs,
                              EntryType *Entry, void *InsertPos);
@@ -796,13 +806,6 @@ class RedeclarableTemplateDecl : public TemplateDecl,
     /// was explicitly specialized.
     llvm::PointerIntPair<RedeclarableTemplateDecl *, 1, bool>
         InstantiatedFromMember;
-
-    /// If non-null, points to an array of specializations (including
-    /// partial specializations) known only by their external declaration IDs.
-    ///
-    /// The first value in the array is the number of specializations/partial
-    /// specializations that follow.
-    GlobalDeclID *LazySpecializations = nullptr;
   };
 
   /// Pointer to the common data shared by all declarations of this
@@ -2283,7 +2286,7 @@ class ClassTemplateDecl : public RedeclarableTemplateDecl {
   friend class TemplateDeclInstantiator;
 
   /// Load any lazily-loaded specializations from the external source.
-  void LoadLazySpecializations() const;
+  void LoadLazySpecializations(bool OnlyPartial = false) const;
 
   /// Get the underlying class declarations of the template.
   CXXRecordDecl *getTemplatedDecl() const {
@@ -3033,7 +3036,7 @@ class VarTemplateDecl : public RedeclarableTemplateDecl {
   friend class ASTDeclWriter;
 
   /// Load any lazily-loaded specializations from the external source.
-  void LoadLazySpecializations() const;
+  void LoadLazySpecializations(bool OnlyPartial = false) const;
 
   /// Get the underlying variable declarations of the template.
   VarDecl *getTemplatedDecl() const {
diff --git a/clang/include/clang/AST/ExternalASTSource.h b/clang/include/clang/AST/ExternalASTSource.h
index 582ed7c65f58c..9f968ba05b446 100644
--- a/clang/include/clang/AST/ExternalASTSource.h
+++ b/clang/include/clang/AST/ExternalASTSource.h
@@ -152,6 +152,21 @@ class ExternalASTSource : public RefCountedBase<ExternalASTSource> {
   virtual bool
   FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name);
 
+  /// Load all the external specializations for the Decl \param D if \param
+  /// OnlyPartial is false. Otherwise, load all the external **partial**
+  /// specializations for the \param D.
+  ///
+  /// Return true if any new specializations get loaded. Return false otherwise.
+  virtual bool LoadExternalSpecializations(const Decl *D, bool OnlyPartial);
+
+  /// Load all the specializations for the Decl \param D with the same template
+  /// args specified by \param TemplateArgs.
+  ///
+  /// Return true if any new specializations get loaded. Return false otherwise.
+  virtual bool
+  LoadExternalSpecializations(const Decl *D,
+                              ArrayRef<TemplateArgument> TemplateArgs);
+
   /// Ensures that the table of all visible declarations inside this
   /// context is up to date.
   ///
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 5ad4c336b6c53..2588c3f645c02 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -483,6 +483,14 @@ class OpenACCGangClause final
     return {getGangKind(I), getExprs()[I]};
   }
 
+  bool hasExprOfKind(OpenACCGangKind GK) const {
+    for (unsigned I = 0; I < getNumExprs(); ++I) {
+      if (getGangKind(I) == GK)
+        return true;
+    }
+    return false;
+  }
+
   static OpenACCGangClause *
   Create(const ASTContext &Ctx, SourceLocation BeginLoc,
          SourceLocation LParenLoc, ArrayRef<OpenACCGangKind> GangKinds,
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 8068372154225..e306325a698bf 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -3764,6 +3764,8 @@ class ArrayParameterType : public ConstantArrayType {
   static bool classof(const Type *T) {
     return T->getTypeClass() == ArrayParameter;
   }
+
+  QualType getConstantArrayType(const ASTContext &Ctx) const;
 };
 
 /// Represents a C array with an unspecified size.  For example 'int A[]' has
@@ -6561,7 +6563,7 @@ class DeducedType : public Type {
 
 /// Represents a C++11 auto or C++14 decltype(auto) type, possibly constrained
 /// by a type-constraint.
-class AutoType : public DeducedType, public llvm::FoldingSetNode {
+class AutoType : public DeducedType {
   friend class ASTContext; // ASTContext creates these
 
   ConceptDecl *TypeConstraintConcept;
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 4bcaa953a61af..897aa25dc95cc 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -1236,6 +1236,17 @@ AST_MATCHER_P(TemplateArgument, equalsIntegralValue,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
        ObjCAutoreleasePoolStmt> autoreleasePoolStmt;
 
+/// Matches any export declaration.
+///
+/// Example matches following declarations.
+/// \code
+///   export void foo();
+///   export { void foo(); }
+///   export namespace { void foo(); }
+///   export int v;
+/// \endcode
+extern const internal::VariadicDynCastAllOfMatcher<Decl, ExportDecl> exportDecl;
+
 /// Matches any value declaration.
 ///
 /// Example matches A, B, C and F
@@ -4033,7 +4044,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     hasType,
     AST_POLYMORPHIC_SUPPORTED_TYPES(Expr, FriendDecl, ValueDecl,
-                                    CXXBaseSpecifier),
+                                    CXXBaseSpecifier, ObjCInterfaceDecl),
     internal::Matcher<Decl>, InnerMatcher, 1) {
   QualType QT = internal::getUnderlyingType(Node);
   if (!QT.isNull())
@@ -7434,7 +7445,8 @@ extern const AstTypeMatcher<RValueReferenceType> rValueReferenceType;
 AST_TYPELOC_TRAVERSE_MATCHER_DECL(
     pointee, getPointee,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BlockPointerType, MemberPointerType,
-                                    PointerType, ReferenceType));
+                                    PointerType, ReferenceType,
+                                    ObjCObjectPointerType));
 
 /// Matches typedef types.
 ///
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index ab8b146453e76..04804d5def046 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -161,6 +161,9 @@ inline QualType getUnderlyingType(const FriendDecl &Node) {
 inline QualType getUnderlyingType(const CXXBaseSpecifier &Node) {
   return Node.getType();
 }
+inline QualType getUnderlyingType(const ObjCInterfaceDecl &Node) {
+  return Node.getTypeForDecl()->getPointeeType();
+}
 
 /// Unifies obtaining a `TypeSourceInfo` from different node types.
 template <typename T,
@@ -1113,6 +1116,11 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
     return matchesDecl(Node.getDecl(), Finder, Builder);
   }
 
+  bool matchesSpecialized(const ObjCInterfaceDecl &Node, ASTMatchFinder *Finder,
+                          BoundNodesTreeBuilder *Builder) const {
+    return matchesDecl(Node.getCanonicalDecl(), Finder, Builder);
+  }
+
   /// Extracts the operator new of the new call and returns whether the
   /// inner matcher matches on it.
   bool matchesSpecialized(const CXXNewExpr &Node,
@@ -1213,7 +1221,7 @@ using HasDeclarationSupportedTypes =
              ElaboratedType, InjectedClassNameType, LabelStmt, AddrLabelExpr,
              MemberExpr, QualType, RecordType, TagType,
              TemplateSpecializationType, TemplateTypeParmType, TypedefType,
-             UnresolvedUsingType, ObjCIvarRefExpr>;
+             UnresolvedUsingType, ObjCIvarRefExpr, ObjCInterfaceDecl>;
 
 /// A Matcher that allows binding the node it matches to an id.
 ///
diff --git a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
index 6294c810626a7..8c7ee86d15c06 100644
--- a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
+++ b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
@@ -146,6 +146,10 @@ struct ReferencedDecls {
   /// Free functions and member functions which are referenced (but not
   /// necessarily called).
   llvm::DenseSet<const FunctionDecl *> Functions;
+  /// When analyzing a lambda's call operator, the set of all parameters (from
+  /// the surrounding function) that the lambda captures. Captured local
+  /// variables are already included in `Locals` above.
+  llvm::DenseSet<const ParmVarDecl *> LambdaCapturedParams;
 };
 
 /// Returns declarations that are declared in or referenced from `FD`.
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index fe8175fb249b4..0995d966eaa60 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -55,6 +55,9 @@ class Documentation {
   // When set, specifies that the attribute is deprecated and can optionally
   // specify a replacement attribute.
   DocDeprecated Deprecated;
+
+  // When set, specifies a label that can be used to reference the documentation.
+  string Label = "";
 }
 
 // Specifies that the attribute is explicitly omitted from the documentation,
@@ -103,6 +106,9 @@ def NonParmVar : SubsetSubject<Var,
 def NonLocalVar : SubsetSubject<Var,
                                 [{!S->hasLocalStorage()}],
                                 "variables with non-local storage">;
+def VarTmpl : SubsetSubject<Var, [{S->getDescribedVarTemplate()}],
+                            "variable templates">;
+
 def NonBitField : SubsetSubject<Field,
                                 [{!S->isBitField()}],
                                 "non-bit-field non-static data members">;
@@ -4284,33 +4290,28 @@ def Target : InheritableAttr {
   let Subjects = SubjectList<[Function], ErrorDiag>;
   let Documentation = [TargetDocs];
   let AdditionalMembers = [{
-    StringRef getArchitecture() const {
+    std::optional<StringRef> getX86Architecture() const {
       StringRef Features = getFeaturesStr();
-      if (Features == "default") return {};
-
-      SmallVector<StringRef, 1> AttrFeatures;
-      Features.split(AttrFeatures, ",");
-
-      for (auto &Feature : AttrFeatures) {
+      SmallVector<StringRef, 4> AttrFeatures;
+      Features.split(AttrFeatures, ',');
+      for (StringRef Feature : AttrFeatures) {
         Feature = Feature.trim();
         if (Feature.starts_with("arch="))
           return Feature.drop_front(sizeof("arch=") - 1);
       }
-      return "";
+      return std::nullopt;
     }
 
     // Gets the list of features as simple string-refs with no +/- or 'no-'.
     // Only adds the items to 'Out' that are additions.
-    void getAddedFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
+    void getX86AddedFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
+      if (isDefaultVersion())
+        return;
       StringRef Features = getFeaturesStr();
-      if (Features == "default") return;
-
-      SmallVector<StringRef, 1> AttrFeatures;
-      Features.split(AttrFeatures, ",");
-
+      SmallVector<StringRef, 4> AttrFeatures;
+      Features.split(AttrFeatures, ',');
       for (auto &Feature : AttrFeatures) {
         Feature = Feature.trim();
-
         if (!Feature.starts_with("no-") && !Feature.starts_with("arch=") &&
             !Feature.starts_with("fpmath=") && !Feature.starts_with("tune="))
           Out.push_back(Feature);
@@ -4328,17 +4329,17 @@ def TargetVersion : InheritableAttr, TargetSpecificAttr<TargetArch<!listconcat(T
   let Documentation = [TargetVersionDocs];
   let AdditionalMembers = [{
     StringRef getName() const { return getNamesStr().trim(); }
-    bool isDefaultVersion() const {
-      return getName() == "default";
-    }
-    void getFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
-      if (isDefaultVersion()) return;
-      StringRef Features = getName();
 
-      SmallVector<StringRef, 8> AttrFeatures;
-      Features.split(AttrFeatures, "+");
+    bool isDefaultVersion() const { return getName() == "default"; }
 
-      for (auto &Feature : AttrFeatures) {
+    void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
+                     char Delim = '+') const {
+      if (isDefaultVersion())
+        return;
+      StringRef Features = getName();
+      SmallVector<StringRef, 4> AttrFeatures;
+      Features.split(AttrFeatures, Delim);
+      for (StringRef Feature : AttrFeatures) {
         Feature = Feature.trim();
         Out.push_back(Feature);
       }
@@ -4355,20 +4356,40 @@ def TargetClones : InheritableAttr {
     StringRef getFeatureStr(unsigned Index) const {
       return *(featuresStrs_begin() + Index);
     }
+
     bool isDefaultVersion(unsigned Index) const {
       return getFeatureStr(Index) == "default";
     }
+
     void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
-                     unsigned Index) const {
-      if (isDefaultVersion(Index)) return;
+                     unsigned Index, char Delim = '+') const {
+      if (isDefaultVersion(Index))
+        return;
       StringRef Features = getFeatureStr(Index);
-      SmallVector<StringRef, 8> AttrFeatures;
-      Features.split(AttrFeatures, "+");
-      for (auto &Feature : AttrFeatures) {
+      SmallVector<StringRef, 4> AttrFeatures;
+      Features.split(AttrFeatures, Delim);
+      for (StringRef Feature : AttrFeatures) {
         Feature = Feature.trim();
         Out.push_back(Feature);
       }
     }
+
+    std::optional<StringRef> getX86Architecture(unsigned Index) const {
+      StringRef Feature = getFeatureStr(Index);
+      if (Feature.starts_with("arch="))
+        return Feature.drop_front(sizeof("arch=") - 1);
+      return std::nullopt;
+    }
+
+    void getX86Feature(llvm::SmallVectorImpl<StringRef> &Out,
+                       unsigned Index) const {
+      if (isDefaultVersion(Index))
+        return;
+      if (getX86Architecture(Index))
+        return;
+      Out.push_back(getFeatureStr(Index));
+    }
+
     // Given an index into the 'featuresStrs' sequence, compute a unique
     // ID to be used with function name mangling for the associated variant.
     // This mapping is necessary due to a requirement that the mangling ID
@@ -4452,6 +4473,15 @@ def DiagnoseIf : InheritableAttr {
   let Documentation = [DiagnoseIfDocs];
 }
 
+def NoSpecializations : InheritableAttr {
+  let Spellings = [Clang<"no_specializations", /*AllowInC*/0>];
+  let Args = [StringArgument<"Message", 1>];
+  let Subjects = SubjectList<[ClassTmpl, FunctionTmpl, VarTmpl]>;
+  let Documentation = [NoSpecializationsDocs];
+  let MeaningfulToClassTemplateDefinition = 1;
+  let TemplateDependent = 1;
+}
+
 def ArcWeakrefUnavailable : InheritableAttr {
   let Spellings = [Clang<"objc_arc_weak_reference_unavailable">];
   let Subjects = SubjectList<[ObjCInterface], ErrorDiag>;
@@ -5645,6 +5675,13 @@ def HLSLNumThreads: InheritableAttr {
   let Documentation = [NumThreadsDocs];
 }
 
+def HLSLSV_GroupThreadID: HLSLAnnotationAttr {
+  let Spellings = [HLSLAnnotation<"SV_GroupThreadID">];
+  let Subjects = SubjectList<[ParmVar, Field]>;
+  let LangOpts = [HLSL];
+  let Documentation = [HLSLSV_GroupThreadIDDocs];
+}
+
 def HLSLSV_GroupID: HLSLAnnotationAttr {
   let Spellings = [HLSLAnnotation<"SV_GroupID">];
   let Subjects = SubjectList<[ParmVar, Field]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index c59c6371ab225..2861dbcf73156 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1208,6 +1208,15 @@ Query for this feature with ``__has_attribute(diagnose_if)``.
   }];
 }
 
+def NoSpecializationsDocs : Documentation {
+  let Category = DocCatDecl;
+  let Content = [{
+``[[clang::no_specializations]]`` can be applied to function, class, or variable
+templates which should not be explicitly specialized by users. This is primarily
+used to diagnose user specializations of standard library type traits.
+  }];
+}
+
 def PassObjectSizeDocs : Documentation {
   let Category = DocCatVariable; // Technically it's a parameter doc, but eh.
   let Heading = "pass_object_size, pass_dynamic_object_size";
@@ -4982,9 +4991,8 @@ def NoSanitizeAddressDocs : Documentation {
   // This function has multiple distinct spellings, and so it requires a custom
   // heading to be specified. The most common spelling is sufficient.
   let Heading = "no_sanitize_address, no_address_safety_analysis";
+  let Label = "langext-address_sanitizer";
   let Content = [{
-.. _langext-address_sanitizer:
-
 Use ``__attribute__((no_sanitize_address))`` on a function or a global
 variable declaration to specify that address safety instrumentation
 (e.g. AddressSanitizer) should not be applied.
@@ -4994,9 +5002,8 @@ variable declaration to specify that address safety instrumentation
 def NoSanitizeThreadDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "no_sanitize_thread";
+  let Label = "langext-thread_sanitizer";
   let Content = [{
-.. _langext-thread_sanitizer:
-
 Use ``__attribute__((no_sanitize_thread))`` on a function declaration to
 specify that checks for data races on plain (non-atomic) memory accesses should
 not be inserted by ThreadSanitizer. The function is still instrumented by the
@@ -5007,9 +5014,8 @@ tool to avoid false positives and provide meaningful stack traces.
 def NoSanitizeMemoryDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "no_sanitize_memory";
+  let Label = "langext-memory_sanitizer";
   let Content = [{
-.. _langext-memory_sanitizer:
-
 Use ``__attribute__((no_sanitize_memory))`` on a function declaration to
 specify that checks for uninitialized memory should not be inserted
 (e.g. by MemorySanitizer). The function may still be instrumented by the tool
@@ -5020,9 +5026,8 @@ to avoid false positives in other places.
 def CFICanonicalJumpTableDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "cfi_canonical_jump_table";
+  let Label = "langext-cfi_canonical_jump_table";
   let Content = [{
-.. _langext-cfi_canonical_jump_table:
-
 Use ``__attribute__((cfi_canonical_jump_table))`` on a function declaration to
 make the function's CFI jump table canonical. See :ref:`the CFI documentation
 <cfi-canonical-jump-tables>` for more details.
@@ -5607,6 +5612,8 @@ The capturing entity ``X`` can be one of the following:
       std::set<std::string_view> s;
     };
 
+  Note: When applied to a constructor parameter, `[[clang::lifetime_capture_by(this)]]` is just an alias of `[[clang::lifetimebound]]`.
+
 - `global`, `unknown`.
 
   .. code-block:: c++
@@ -9657,6 +9664,17 @@ randomized.
   }];
 }
 
+def HLSLSV_GroupThreadIDDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``SV_GroupThreadID`` semantic, when applied to an input parameter, specifies which
+individual thread within a thread group is executing in. This attribute is
+only supported in compute shaders.
+
+The full documentation is available here: https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/sv-groupthreadid
+  }];
+}
+
 def HLSLSV_GroupIDDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.h b/clang/include/clang/Basic/Builtins.h
index 714e584ef0d01..f91b2d551ecf2 100644
--- a/clang/include/clang/Basic/Builtins.h
+++ b/clang/include/clang/Basic/Builtins.h
@@ -56,6 +56,7 @@ struct HeaderDesc {
 #undef HEADER
   } ID;
 
+  constexpr HeaderDesc() : ID() {}
   constexpr HeaderDesc(HeaderID ID) : ID(ID) {}
 
   const char *getName() const;
@@ -69,14 +70,140 @@ enum ID {
   FirstTSBuiltin
 };
 
+// The info used to represent each builtin.
 struct Info {
-  llvm::StringLiteral Name;
-  const char *Type, *Attributes;
-  const char *Features;
+  // Rather than store pointers to the string literals describing these four
+  // aspects of builtins, we store offsets into a common string table.
+  struct StrOffsets {
+    int Name;
+    int Type;
+    int Attributes;
+    int Features;
+  } Offsets;
+
   HeaderDesc Header;
   LanguageID Langs;
 };
 
+// The storage for `N` builtins. This contains a single pointer to the string
+// table used for these builtins and an array of metadata for each builtin.
+template <size_t N> struct Storage {
+  const char *StringTable;
+
+  std::array<Info, N> Infos;
+
+  // A constexpr function to construct the storage for a a given string table in
+  // the first argument and an array in the second argument. This is *only*
+  // expected to be used at compile time, we should mark it `consteval` when
+  // available.
+  //
+  // The `Infos` array is particularly special. This function expects an array
+  // of `Info` structs, where the string offsets of each entry refer to the
+  // *sizes* of those strings rather than their offsets, and for the target
+  // string to be in the provided string table at an offset the sum of all
+  // previous string sizes. This function walks the `Infos` array computing the
+  // running sum and replacing the sizes with the actual offsets in the string
+  // table that should be used. This arrangement is designed to make it easy to
+  // expand `.def` and `.inc` files with X-macros to construct both the string
+  // table and the `Info` structs in the arguments to this function.
+  static constexpr Storage<N> Make(const char *Strings,
+                                   std::array<Info, N> Infos) {
+    // Translate lengths to offsets.
+    int Offset = 0;
+    for (auto &I : Infos) {
+      Info::StrOffsets NewOffsets = {};
+      NewOffsets.Name = Offset;
+      Offset += I.Offsets.Name;
+      NewOffsets.Type = Offset;
+      Offset += I.Offsets.Type;
+      NewOffsets.Attributes = Offset;
+      Offset += I.Offsets.Attributes;
+      NewOffsets.Features = Offset;
+      Offset += I.Offsets.Features;
+      I.Offsets = NewOffsets;
+    }
+    return {Strings, Infos};
+  }
+};
+
+// A detail macro used below to emit a string literal that, after string literal
+// concatenation, ends up triggering the `-Woverlength-strings` warning. While
+// the warning is useful in general to catch accidentally excessive strings,
+// here we are creating them intentionally.
+//
+// This relies on a subtle aspect of `_Pragma`: that the *diagnostic* ones don't
+// turn into actual tokens that would disrupt string literal concatenation.
+#ifdef __clang__
+#define CLANG_BUILTIN_DETAIL_STR_TABLE(S)                                      \
+  _Pragma("clang diagnostic push")                                             \
+      _Pragma("clang diagnostic ignored \"-Woverlength-strings\"")             \
+          S _Pragma("clang diagnostic pop")
+#else
+#define CLANG_BUILTIN_DETAIL_STR_TABLE(S) S
+#endif
+
+// A macro that can be used with `Builtins.def` and similar files as an X-macro
+// to add the string arguments to a builtin string table. This is typically the
+// target for the `BUILTIN`, `LANGBUILTIN`, or `LIBBUILTIN` macros in those
+// files.
+#define CLANG_BUILTIN_STR_TABLE(ID, TYPE, ATTRS)                               \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" /*FEATURE*/ "\0")
+
+// A macro that can be used with target builtin `.def` and `.inc` files as an
+// X-macro to add the string arguments to a builtin string table. this is
+// typically the target for the `TARGET_BUILTIN` macro.
+#define CLANG_TARGET_BUILTIN_STR_TABLE(ID, TYPE, ATTRS, FEATURE)               \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" FEATURE "\0")
+
+// A macro that can be used with target builtin `.def` and `.inc` files as an
+// X-macro to add the string arguments to a builtin string table. this is
+// typically the target for the `TARGET_HEADER_BUILTIN` macro. We can't delegate
+// to `TARGET_BUILTIN` because the `FEATURE` string changes position.
+#define CLANG_TARGET_HEADER_BUILTIN_STR_TABLE(ID, TYPE, ATTRS, HEADER, LANGS,  \
+                                              FEATURE)                         \
+  CLANG_BUILTIN_DETAIL_STR_TABLE(#ID "\0" TYPE "\0" ATTRS "\0" FEATURE "\0")
+
+// A detail macro used internally to compute the desired string table
+// `StrOffsets` struct for arguments to `Storage::Make`.
+#define CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS)                      \
+  Builtin::Info::StrOffsets {                                                  \
+    sizeof(#ID), sizeof(TYPE), sizeof(ATTRS), sizeof("")                       \
+  }
+
+// A detail macro used internally to compute the desired string table
+// `StrOffsets` struct for arguments to `Storage::Make`.
+#define CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE)      \
+  Builtin::Info::StrOffsets {                                                  \
+    sizeof(#ID), sizeof(TYPE), sizeof(ATTRS), sizeof(FEATURE)                  \
+  }
+
+// A set of macros that can be used with builtin `.def' files as an X-macro to
+// create an `Info` struct for a particular builtin. It both computes the
+// `StrOffsets` value for the string table (the lengths here, translated to
+// offsets by the Storage::Make function), and the other metadata for each
+// builtin.
+//
+// There is a corresponding macro for each of `BUILTIN`, `LANGBUILTIN`,
+// `LIBBUILTIN`, `TARGET_BUILTIN`, and `TARGET_HEADER_BUILTIN`.
+#define CLANG_BUILTIN_ENTRY(ID, TYPE, ATTRS)                                   \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define CLANG_LANGBUILTIN_ENTRY(ID, TYPE, ATTRS, LANG)                         \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::NO_HEADER, LANG},
+#define CLANG_LIBBUILTIN_ENTRY(ID, TYPE, ATTRS, HEADER, LANG)                  \
+  Builtin::Info{CLANG_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS),             \
+                HeaderDesc::HEADER, LANG},
+#define CLANG_TARGET_BUILTIN_ENTRY(ID, TYPE, ATTRS, FEATURE)                   \
+  Builtin::Info{                                                               \
+      CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE),       \
+      HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define CLANG_TARGET_HEADER_BUILTIN_ENTRY(ID, TYPE, ATTRS, HEADER, LANG,       \
+                                          FEATURE)                             \
+  Builtin::Info{                                                               \
+      CLANG_TARGET_BUILTIN_DETAIL_STR_OFFSETS(ID, TYPE, ATTRS, FEATURE),       \
+      HeaderDesc::HEADER, LANG},
+
 /// Holds information about both target-independent and
 /// target-specific builtins, allowing easy queries by clients.
 ///
@@ -84,8 +211,11 @@ struct Info {
 /// AuxTSRecords. Their IDs are shifted up by TSRecords.size() and need to
 /// be translated back with getAuxBuiltinID() before use.
 class Context {
-  llvm::ArrayRef<Info> TSRecords;
-  llvm::ArrayRef<Info> AuxTSRecords;
+  const char *TSStrTable = nullptr;
+  const char *AuxTSStrTable = nullptr;
+
+  llvm::ArrayRef<Info> TSInfos;
+  llvm::ArrayRef<Info> AuxTSInfos;
 
 public:
   Context() = default;
@@ -101,12 +231,13 @@ class Context {
 
   /// Return the identifier name for the specified builtin,
   /// e.g. "__builtin_abs".
-  llvm::StringRef getName(unsigned ID) const { return getRecord(ID).Name; }
+  llvm::StringRef getName(unsigned ID) const;
 
   /// Get the type descriptor string for the specified builtin.
-  const char *getTypeString(unsigned ID) const {
-    return getRecord(ID).Type;
-  }
+  const char *getTypeString(unsigned ID) const;
+
+  /// Get the attributes descriptor string for the specified builtin.
+  const char *getAttributesString(unsigned ID) const;
 
   /// Return true if this function is a target-specific builtin.
   bool isTSBuiltin(unsigned ID) const {
@@ -115,40 +246,40 @@ class Context {
 
   /// Return true if this function has no side effects.
   bool isPure(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'U') != nullptr;
+    return strchr(getAttributesString(ID), 'U') != nullptr;
   }
 
   /// Return true if this function has no side effects and doesn't
   /// read memory.
   bool isConst(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'c') != nullptr;
+    return strchr(getAttributesString(ID), 'c') != nullptr;
   }
 
   /// Return true if we know this builtin never throws an exception.
   bool isNoThrow(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'n') != nullptr;
+    return strchr(getAttributesString(ID), 'n') != nullptr;
   }
 
   /// Return true if we know this builtin never returns.
   bool isNoReturn(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'r') != nullptr;
+    return strchr(getAttributesString(ID), 'r') != nullptr;
   }
 
   /// Return true if we know this builtin can return twice.
   bool isReturnsTwice(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'j') != nullptr;
+    return strchr(getAttributesString(ID), 'j') != nullptr;
   }
 
   /// Returns true if this builtin does not perform the side-effects
   /// of its arguments.
   bool isUnevaluated(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'u') != nullptr;
+    return strchr(getAttributesString(ID), 'u') != nullptr;
   }
 
   /// Return true if this is a builtin for a libc/libm function,
   /// with a "__builtin_" prefix (e.g. __builtin_abs).
   bool isLibFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'F') != nullptr;
+    return strchr(getAttributesString(ID), 'F') != nullptr;
   }
 
   /// Determines whether this builtin is a predefined libc/libm
@@ -159,21 +290,21 @@ class Context {
   /// they do not, but they are recognized as builtins once we see
   /// a declaration.
   bool isPredefinedLibFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'f') != nullptr;
+    return strchr(getAttributesString(ID), 'f') != nullptr;
   }
 
   /// Returns true if this builtin requires appropriate header in other
   /// compilers. In Clang it will work even without including it, but we can emit
   /// a warning about missing header.
   bool isHeaderDependentFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'h') != nullptr;
+    return strchr(getAttributesString(ID), 'h') != nullptr;
   }
 
   /// Determines whether this builtin is a predefined compiler-rt/libgcc
   /// function, such as "__clear_cache", where we know the signature a
   /// priori.
   bool isPredefinedRuntimeFunction(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'i') != nullptr;
+    return strchr(getAttributesString(ID), 'i') != nullptr;
   }
 
   /// Determines whether this builtin is a C++ standard library function
@@ -181,7 +312,7 @@ class Context {
   /// specialization, where the signature is determined by the standard library
   /// declaration.
   bool isInStdNamespace(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'z') != nullptr;
+    return strchr(getAttributesString(ID), 'z') != nullptr;
   }
 
   /// Determines whether this builtin can have its address taken with no
@@ -195,33 +326,33 @@ class Context {
 
   /// Determines whether this builtin has custom typechecking.
   bool hasCustomTypechecking(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 't') != nullptr;
+    return strchr(getAttributesString(ID), 't') != nullptr;
   }
 
   /// Determines whether a declaration of this builtin should be recognized
   /// even if the type doesn't match the specified signature.
   bool allowTypeMismatch(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'T') != nullptr ||
+    return strchr(getAttributesString(ID), 'T') != nullptr ||
            hasCustomTypechecking(ID);
   }
 
   /// Determines whether this builtin has a result or any arguments which
   /// are pointer types.
   bool hasPtrArgsOrResult(unsigned ID) const {
-    return strchr(getRecord(ID).Type, '*') != nullptr;
+    return strchr(getTypeString(ID), '*') != nullptr;
   }
 
   /// Return true if this builtin has a result or any arguments which are
   /// reference types.
   bool hasReferenceArgsOrResult(unsigned ID) const {
-    return strchr(getRecord(ID).Type, '&') != nullptr ||
-           strchr(getRecord(ID).Type, 'A') != nullptr;
+    return strchr(getTypeString(ID), '&') != nullptr ||
+           strchr(getTypeString(ID), 'A') != nullptr;
   }
 
   /// If this is a library function that comes from a specific
   /// header, retrieve that header name.
   const char *getHeaderName(unsigned ID) const {
-    return getRecord(ID).Header.getName();
+    return getInfo(ID).Header.getName();
   }
 
   /// Determine whether this builtin is like printf in its
@@ -246,27 +377,25 @@ class Context {
   /// Such functions can be const when the MathErrno lang option and FP
   /// exceptions are disabled.
   bool isConstWithoutErrnoAndExceptions(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'e') != nullptr;
+    return strchr(getAttributesString(ID), 'e') != nullptr;
   }
 
   bool isConstWithoutExceptions(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'g') != nullptr;
+    return strchr(getAttributesString(ID), 'g') != nullptr;
   }
 
-  const char *getRequiredFeatures(unsigned ID) const {
-    return getRecord(ID).Features;
-  }
+  const char *getRequiredFeatures(unsigned ID) const;
 
   unsigned getRequiredVectorWidth(unsigned ID) const;
 
   /// Return true if builtin ID belongs to AuxTarget.
   bool isAuxBuiltinID(unsigned ID) const {
-    return ID >= (Builtin::FirstTSBuiltin + TSRecords.size());
+    return ID >= (Builtin::FirstTSBuiltin + TSInfos.size());
   }
 
   /// Return real builtin ID (i.e. ID it would have during compilation
   /// for AuxTarget).
-  unsigned getAuxBuiltinID(unsigned ID) const { return ID - TSRecords.size(); }
+  unsigned getAuxBuiltinID(unsigned ID) const { return ID - TSInfos.size(); }
 
   /// Returns true if this is a libc/libm function without the '__builtin_'
   /// prefix.
@@ -278,16 +407,20 @@ class Context {
 
   /// Return true if this function can be constant evaluated by Clang frontend.
   bool isConstantEvaluated(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'E') != nullptr;
+    return strchr(getAttributesString(ID), 'E') != nullptr;
   }
 
   /// Returns true if this is an immediate (consteval) function
   bool isImmediate(unsigned ID) const {
-    return strchr(getRecord(ID).Attributes, 'G') != nullptr;
+    return strchr(getAttributesString(ID), 'G') != nullptr;
   }
 
 private:
-  const Info &getRecord(unsigned ID) const;
+  std::pair<const char *, const Info &> getStrTableAndInfo(unsigned ID) const;
+
+  const Info &getInfo(unsigned ID) const {
+    return getStrTableAndInfo(ID).second;
+  }
 
   /// Helper function for isPrintfLike and isScanfLike.
   bool isLike(unsigned ID, unsigned &FormatIdx, bool &HasVAListArg,
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index b48102ff25b3a..b67464f78b4c7 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1270,7 +1270,7 @@ def ElementwiseATan2 : Builtin {
 
 def ElementwiseBitreverse : Builtin {
   let Spellings = ["__builtin_elementwise_bitreverse"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -1450,13 +1450,13 @@ def ElementwiseFma : Builtin {
 
 def ElementwiseAddSat : Builtin {
   let Spellings = ["__builtin_elementwise_add_sat"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
 def ElementwiseSubSat : Builtin {
   let Spellings = ["__builtin_elementwise_sub_sat"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -4757,6 +4757,12 @@ def SYCLIsNDRangeKernel : LangBuiltin<"SYCL_LANG"> {
 }
 
 // HLSL
+def HLSLResourceGetPointer : LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_resource_getpointer"];
+  let Attributes = [NoThrow];
+  let Prototype = "void(...)";
+}
+
 def HLSLAll : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_all"];
   let Attributes = [NoThrow, Const];
@@ -4943,6 +4949,12 @@ def HLSLClip: LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLGroupMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_group_memory_barrier_with_group_sync"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void()";
+}
+
 // Builtins for XRay.
 def XRayCustomEvent : Builtin {
   let Spellings = ["__xray_customevent"];
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 473b1d4698f04..7933686ffe84e 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel,   "NiNiD*Ni",    "nh", INTRIN_H,
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf,  "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf,  "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq,  "ccD*cc",         "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf,   "ccD*cc",         "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel,   "NiNiD*NiNi",     "nh",
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf,  "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128,    "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -281,6 +286,16 @@ TARGET_HEADER_BUILTIN(__readx18word,  "UsUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES,
 TARGET_HEADER_BUILTIN(__readx18dword, "UNiUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(__readx18qword, "ULLiUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
+TARGET_HEADER_BUILTIN(__addx18byte,  "vUNiUc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__addx18word,  "vUNiUs", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__addx18dword, "vUNiUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__addx18qword, "vUNiULLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+
+TARGET_HEADER_BUILTIN(__incx18byte,  "vUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__incx18word,  "vUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__incx18dword, "vUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__incx18qword, "vUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+
 TARGET_HEADER_BUILTIN(_CopyDoubleFromInt64, "dSLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_CopyFloatFromInt32, "fSi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_CopyInt32FromFloat, "Sif", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index aa3731461d258..576ca70d27319 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -472,6 +472,8 @@ TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr4_b64_v2i32, "V2iV2i*3", "nc", "gfx950
 TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr6_b96_v3i32, "V3iV3i*3", "nc", "gfx950-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr8_b64_v2i32, "V2iV2i*3", "nc", "gfx950-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4i16, "V4sV4s*3", "nc", "gfx950-insts")
+TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4f16, "V4hV4h*3", "nc", "gfx950-insts")
+TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4bf16, "V4yV4y*3", "nc", "gfx950-insts")
 
 TARGET_BUILTIN(__builtin_amdgcn_ashr_pk_i8_i32, "UsUiUiUi", "nc", "ashr-pk-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ashr_pk_u8_i32, "UsUiUiUi", "nc", "ashr-pk-insts")
@@ -619,8 +621,8 @@ TARGET_BUILTIN(__builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_f32, "V6UiV32fUif", "nc
 TARGET_BUILTIN(__builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_bf16, "V6UiV32yUif", "nc", "f16bf16-to-fp6bf6-cvt-scale-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f16, "V6UiV32hUif", "nc", "f16bf16-to-fp6bf6-cvt-scale-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f32, "V6UiV32fUif", "nc", "f16bf16-to-fp6bf6-cvt-scale-insts")
-TARGET_BUILTIN(__builtin_amdgcn_bitop3_b32, "iiiiIUc", "nc", "bitop3-insts")
-TARGET_BUILTIN(__builtin_amdgcn_bitop3_b16, "ssssIUc", "nc", "bitop3-insts")
+TARGET_BUILTIN(__builtin_amdgcn_bitop3_b32, "iiiiIUi", "nc", "bitop3-insts")
+TARGET_BUILTIN(__builtin_amdgcn_bitop3_b16, "ssssIUi", "nc", "bitop3-insts")
 
 TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_bf16_f32, "V2yV2yfUiIb", "nc", "f32-to-f16bf16-cvt-sr-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_f16_f32, "V2hV2hfUiIb", "nc", "f32-to-f16bf16-cvt-sr-insts")
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 9ee918cb21475..5a7064a98045e 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel,   "NiNiD*Ni",    "nh", INTRIN_H,
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf,  "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf,  "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq,  "ccD*cc",         "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf,   "ccD*cc",         "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel,   "NiNiD*NiNi",     "nh",
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf,  "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_InterlockedOr8_acq,  "ccD*c",       "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedOr8_nf,   "ccD*c",       "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 161df386f00f0..bb7d54bbb793e 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -1138,5 +1138,6 @@ UNALIASED_CUSTOM_BUILTIN(mma_pmxvbf16ger2nn, "vW512*VVi15i15i3", true,
 // FIXME: Obviously incomplete.
 
 #undef BUILTIN
+#undef TARGET_BUILTIN
 #undef CUSTOM_BUILTIN
 #undef UNALIASED_CUSTOM_BUILTIN
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index a9715351726e2..1b90b19cfa769 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -243,7 +243,7 @@ ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Defa
 
 CODEGENOPT(RelaxAll          , 1, 0) ///< Relax all machine code instructions.
 CODEGENOPT(RelaxedAliasing   , 1, 0) ///< Set when -fno-strict-aliasing is enabled.
-CODEGENOPT(PointerTBAA, 1, 0)        ///< Whether or not to use distinct TBAA tags for pointers.
+CODEGENOPT(PointerTBAA       , 1, 1) ///< Whether or not to use distinct TBAA tags for pointers.
 CODEGENOPT(StructPathTBAA    , 1, 0) ///< Whether or not to use struct-path TBAA.
 CODEGENOPT(NewStructPathTBAA , 1, 0) ///< Whether or not to use enhanced struct-path TBAA.
 CODEGENOPT(SaveTempLabels    , 1, 0) ///< Save temporary labels.
diff --git a/clang/include/clang/Basic/Diagnostic.h b/clang/include/clang/Basic/Diagnostic.h
index ef46ac5229ecb..1b8b7153efa65 100644
--- a/clang/include/clang/Basic/Diagnostic.h
+++ b/clang/include/clang/Basic/Diagnostic.h
@@ -562,7 +562,8 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   ArgToStringFnTy ArgToStringFn;
 
   /// Whether the diagnostic should be suppressed in FilePath.
-  llvm::unique_function<bool(diag::kind, StringRef /*FilePath*/) const>
+  llvm::unique_function<bool(diag::kind, SourceLocation /*DiagLoc*/,
+                             const SourceManager &) const>
       DiagSuppressionMapping;
 
 public:
@@ -978,7 +979,7 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// These take presumed locations into account, and can still be overriden by
   /// clang-diagnostics pragmas.
   void setDiagSuppressionMapping(llvm::MemoryBuffer &Input);
-  bool isSuppressedViaMapping(diag::kind DiagId, StringRef FilePath) const;
+  bool isSuppressedViaMapping(diag::kind DiagId, SourceLocation DiagLoc) const;
 
   /// Issue the message to the client.
   ///
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index 6a658cf14356f..f630698757c5f 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -782,8 +782,8 @@ def err_module_odr_violation_field : Error<
   "%select{"
   "field %4|"
   "field %4 with type %5|"
-  "%select{non-|}5bitfield %4|"
-  "bitfield %4 with one width expression|"
+  "%select{non-|}5bit-field %4|"
+  "bit-field %4 with one width expression|"
   "%select{non-|}5mutable field %4|"
   "field %4 with %select{no|an}5 initializer|"
   "field %4 with an initializer"
@@ -793,8 +793,8 @@ def note_module_odr_violation_field : Note<
   "%select{"
   "field %3|"
   "field %3 with type %4|"
-  "%select{non-|}4bitfield %3|"
-  "bitfield %3 with different width expression|"
+  "%select{non-|}4bit-field %3|"
+  "bit-field %3 with different width expression|"
   "%select{non-|}4mutable field %3|"
   "field %3 with %select{no|an}4 initializer|"
   "field %3 with a different initializer"
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 30831c1971cc5..e5fa9465f638a 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -213,6 +213,7 @@ def DeprecatedWritableStr : DiagGroup<"deprecated-writable-strings",
                                       [CXX11CompatDeprecatedWritableStr]>;
 def DeprecatedPragma : DiagGroup<"deprecated-pragma">;
 def DeprecatedType : DiagGroup<"deprecated-type">;
+def DeprecatedMissingCommaVariadicParam : DiagGroup<"deprecated-missing-comma-variadic-parameter">;
 // FIXME: Why is DeprecatedImplementations not in this group?
 def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedArrayCompare,
@@ -237,7 +238,8 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedType,
                                           DeprecatedVolatile,
                                           DeprecatedWritableStr,
-                                          DeprecatedRedundantConstexprStaticDef
+                                          DeprecatedRedundantConstexprStaticDef,
+                                          DeprecatedMissingCommaVariadicParam
                                           ]>,
                  DiagCategory<"Deprecations">;
 
@@ -294,11 +296,13 @@ def : DiagGroup<"c++1z-compat-mangling", [CXX17CompatMangling]>;
 // Name of this warning in GCC.
 def NoexceptType : DiagGroup<"noexcept-type", [CXX17CompatMangling]>;
 
+def VariadicMacroArgumentsOmitted : DiagGroup<"variadic-macro-arguments-omitted">;
+
 // Warnings for C code which is not compatible with previous C standards.
 def CPre11Compat : DiagGroup<"pre-c11-compat">;
 def CPre11CompatPedantic : DiagGroup<"pre-c11-compat-pedantic",
                                      [CPre11Compat]>;
-def CPre23Compat : DiagGroup<"pre-c23-compat">;
+def CPre23Compat : DiagGroup<"pre-c23-compat", [VariadicMacroArgumentsOmitted]>;
 def CPre23CompatPedantic : DiagGroup<"pre-c23-compat-pedantic",
                                      [CPre23Compat]>;
 def : DiagGroup<"pre-c2x-compat", [CPre23Compat]>;
@@ -687,7 +691,8 @@ def SizeofArrayDecay : DiagGroup<"sizeof-array-decay">;
 def SizeofPointerMemaccess : DiagGroup<"sizeof-pointer-memaccess">;
 def MemsetTransposedArgs : DiagGroup<"memset-transposed-args">;
 def DynamicClassMemaccess : DiagGroup<"dynamic-class-memaccess">;
-def NonTrivialMemaccess : DiagGroup<"nontrivial-memaccess">;
+def NonTrivialMemcall : DiagGroup<"nontrivial-memcall">;
+def NonTrivialMemaccess : DiagGroup<"nontrivial-memaccess", [NonTrivialMemcall]>;
 def SuspiciousBzero : DiagGroup<"suspicious-bzero">;
 def SuspiciousMemaccess : DiagGroup<"suspicious-memaccess",
   [SizeofPointerMemaccess, DynamicClassMemaccess,
@@ -913,7 +918,7 @@ def VolatileRegisterVar : DiagGroup<"volatile-register-var">;
 def Visibility : DiagGroup<"visibility">;
 def ZeroLengthArray : DiagGroup<"zero-length-array">;
 def GNUZeroLineDirective : DiagGroup<"gnu-zero-line-directive">;
-def GNUZeroVariadicMacroArguments : DiagGroup<"gnu-zero-variadic-macro-arguments">;
+def GNUZeroVariadicMacroArguments : DiagGroup<"gnu-zero-variadic-macro-arguments", [VariadicMacroArgumentsOmitted]>;
 def MisleadingIndentation : DiagGroup<"misleading-indentation">;
 def PtrAuthNullPointers : DiagGroup<"ptrauth-null-pointers">;
 
@@ -1206,7 +1211,7 @@ def CXX17 : DiagGroup<"c++17-extensions", [CXX17Attrs]>;
 
 // A warning group for warnings about using C++20 features as extensions in
 // earlier C++ versions.
-def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>;
+def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs, VariadicMacroArgumentsOmitted]>;
 
 // A warning group for warnings about using C++23 features as extensions in
 // earlier C++ versions.
@@ -1233,7 +1238,7 @@ def C11 : DiagGroup<"c11-extensions">;
 def C99 : DiagGroup<"c99-extensions", [C99Designator]>;
 
 // A warning group for warnings about using C23 features as extensions.
-def C23 : DiagGroup<"c23-extensions">;
+def C23 : DiagGroup<"c23-extensions", [VariadicMacroArgumentsOmitted]>;
 
 def : DiagGroup<"c2x-extensions", [C23]>;
 
@@ -1614,4 +1619,3 @@ def ExplicitSpecializationStorageClass : DiagGroup<"explicit-specialization-stor
 
 // A warning for options that enable a feature that is not yet complete
 def ExperimentalOption : DiagGroup<"experimental-option">;
-
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 889370221f32f..959376b084721 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -486,14 +486,14 @@ def ext_embedded_directive : Extension<
   InGroup<DiagGroup<"embedded-directive">>;
 def ext_c_missing_varargs_arg : Extension<
   "passing no argument for the '...' parameter of a variadic macro is "
-  "a C23 extension">, InGroup<C23>;
+  "a C23 extension">, InGroup<VariadicMacroArgumentsOmitted>;
 def ext_cxx_missing_varargs_arg : Extension<
   "passing no argument for the '...' parameter of a variadic macro is "
-  "a C++20 extension">, InGroup<CXX20>;
+  "a C++20 extension">, InGroup<VariadicMacroArgumentsOmitted>;
 def warn_c17_compat_missing_varargs_arg : Warning<
   "passing no argument for the '...' parameter of a variadic macro is "
   "incompatible with C standards before C23">,
-  InGroup<CPre23Compat>, DefaultIgnore;
+  InGroup<VariadicMacroArgumentsOmitted>, DefaultIgnore;
 def warn_cxx17_compat_missing_varargs_arg : Warning<
   "passing no argument for the '...' parameter of a variadic macro is "
   "incompatible with C++ standards before C++20">,
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 77bf08453dea5..9fa8d5901bd0a 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -413,6 +413,9 @@ def err_function_scope_depth_exceeded : Error<
   "function scope depth exceeded maximum of %0">, DefaultFatal;
 def err_missing_comma_before_ellipsis : Error<
   "C requires a comma prior to the ellipsis in a variadic function type">;
+def warn_deprecated_missing_comma_before_ellipsis : Warning<
+  "declaration of a variadic function without a comma before '...' is deprecated">,
+  InGroup<DeprecatedMissingCommaVariadicParam>;
 def err_unexpected_typedef_ident : Error<
   "unexpected type name %0: expected identifier">;
 def warn_cxx98_compat_decltype : Warning<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 56226fa5bf4c7..90047b1701598 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -869,7 +869,7 @@ def warn_cstruct_memaccess : Warning<
 def warn_cxxstruct_memaccess : Warning<
   "first argument in call to "
   "%0 is a pointer to non-trivially copyable type %1">,
-  InGroup<NonTrivialMemaccess>;
+  InGroup<NonTrivialMemcall>;
 def note_nontrivial_field : Note<
   "field is non-trivial to %select{copy|default-initialize}0">;
 def err_non_trivial_c_union_in_invalid_context : Error<
@@ -5533,6 +5533,10 @@ def note_dependent_function_template_spec_discard_reason : Note<
   "candidate ignored: %select{not a function template|"
   "not a member of the enclosing %select{class template|"
   "namespace; did you mean to explicitly qualify the specialization?}1}0">;
+def warn_invalid_specialization : Warning<
+  "%0 cannot be specialized%select{|: %2}1">,
+  DefaultError, InGroup<DiagGroup<"invalid-specialization">>;
+def note_marked_here : Note<"marked %0 here">;
 
 // C++ class template specializations and out-of-line definitions
 def err_template_spec_needs_header : Error<
@@ -6358,7 +6362,7 @@ def err_typecheck_invalid_restrict_invalid_pointee : Error<
 def ext_typecheck_zero_array_size : Extension<
   "zero size arrays are an extension">, InGroup<ZeroLengthArray>;
 def err_typecheck_zero_array_size : Error<
-  "zero-length arrays are not permitted in %select{C++|SYCL device code}0">;
+  "zero-length arrays are not permitted in %select{C++|SYCL device code|HIP device code}0">;
 def err_array_size_non_int : Error<"size of array has non-integer type %0">;
 def err_init_element_not_constant : Error<
   "initializer element is not a compile-time constant">;
@@ -6505,7 +6509,7 @@ def warn_signed_bitfield_enum_conversion : Warning<
   "enumerators of %1">,
   InGroup<BitFieldEnumConversion>, DefaultIgnore;
 def note_change_bitfield_sign : Note<
-  "consider making the bitfield type %select{unsigned|signed}0">;
+  "consider making the bit-field type %select{unsigned|signed}0">;
 
 def warn_missing_braces : Warning<
   "suggest braces around initialization of subobject">,
@@ -6758,14 +6762,15 @@ def warn_counted_by_attr_elt_type_unknown_size :
 // __builtin_counted_by_ref diagnostics:
 def err_builtin_counted_by_ref_must_be_flex_array_member : Error<
   "'__builtin_counted_by_ref' argument must reference a flexible array member">;
+def err_builtin_counted_by_ref_has_side_effects : Error<
+  "'__builtin_counted_by_ref' argument cannot have side-effects">;
+
 def err_builtin_counted_by_ref_cannot_leak_reference : Error<
-  "value returned by '__builtin_counted_by_ref' cannot be assigned to a "
-  "variable, have its address taken, or passed into or returned from a function">;
-def err_builtin_counted_by_ref_invalid_lhs_use : Error<
+  "value returned by '__builtin_counted_by_ref' cannot be %select{assigned to a "
+  "variable|passed into a function|returned from a function}0">;
+def err_builtin_counted_by_ref_invalid_use : Error<
   "value returned by '__builtin_counted_by_ref' cannot be used in "
   "%select{an array subscript|a binary}0 expression">;
-def err_builtin_counted_by_ref_has_side_effects : Error<
-  "'__builtin_counted_by_ref' argument cannot have side-effects">;
 
 let CategoryName = "ARC Semantic Issue" in {
 
@@ -10370,6 +10375,16 @@ def warn_depr_array_comparison : Warning<
   "to compare array addresses, use unary '+' to decay operands to pointers">,
   InGroup<DeprecatedArrayCompare>;
 
+def warn_array_comparison : Warning<
+  "comparison between two arrays compare their addresses and will be deprecated in c++20; "
+  "to compare array addresses, use unary '+' to decay operands to pointers">,
+  InGroup<DiagGroup<"array-compare">>;
+
+def warn_array_comparison_cxx26 : Warning<
+  "comparison between two arrays is ill-formed in C++26; "
+  "to compare array addresses, use unary '+' to decay operands to pointers">,
+  InGroup<DiagGroup<"array-compare-cxx26">>, DefaultError;
+
 def warn_stringcompare : Warning<
   "result of comparison against %select{a string literal|@encode}0 is "
   "unspecified (use an explicit string comparison function instead)">,
@@ -13000,9 +13015,9 @@ def err_acc_clause_cannot_combine
     : Error<"OpenACC clause '%0' may not appear on the same construct as a "
             "'%1' clause on a '%2' construct">;
 def err_acc_reduction_num_gangs_conflict
-    : Error<
-          "OpenACC 'reduction' clause may not appear on a 'parallel' construct "
-          "with a 'num_gangs' clause with more than 1 argument, have %0">;
+    : Error<"OpenACC '%1' clause %select{|with more than 1 argument }0may not "
+            "appear on a '%2' construct "
+            "with a '%3' clause%select{ with more than 1 argument|}0">;
 def err_acc_reduction_type
     : Error<"OpenACC 'reduction' variable must be of scalar type, sub-array, or a "
             "composite of scalar types;%select{| sub-array base}1 type is %0">;
@@ -13027,48 +13042,50 @@ def err_acc_size_expr_value
           "OpenACC 'tile' clause size expression must be %select{an asterisk "
           "or a constant expression|positive integer value, evaluated to %1}0">;
 def err_acc_invalid_in_loop
-    : Error<"%select{OpenACC '%2' construct|while loop|do loop}0 cannot appear "
-            "in intervening code of a 'loop' with a '%1' clause">;
+    : Error<"%select{OpenACC '%3' construct|while loop|do loop}0 cannot appear "
+            "in intervening code of a '%1' with a '%2' clause">;
 def note_acc_active_clause_here
     : Note<"active '%0' clause defined here">;
 def err_acc_clause_multiple_loops
-    : Error<"more than one for-loop in a loop associated with OpenACC 'loop' "
-            "construct with a '%select{collapse|tile}0' clause">;
+    : Error<"more than one for-loop in a loop associated with OpenACC '%0' "
+            "construct with a '%1' clause">;
 def err_acc_insufficient_loops
     : Error<"'%0' clause specifies a loop count greater than the number "
             "of available loops">;
 def err_acc_intervening_code
     : Error<"inner loops must be tightly nested inside a '%0' clause on "
-            "a 'loop' construct">;
+            "a '%1' construct">;
 def err_acc_gang_multiple_elt
     : Error<"OpenACC 'gang' clause may have at most one %select{unnamed or "
             "'num'|'dim'|'static'}0 argument">;
 def err_acc_int_arg_invalid
-    : Error<"'%1' argument on '%0' clause is not permitted on a%select{n "
-            "orphaned|||}2 'loop' construct %select{|associated with a "
-            "'parallel' compute construct|associated with a 'kernels' compute "
-            "construct|associated with a 'serial' compute construct}2">;
+    : Error<"'%0' argument on '%1' clause is not permitted on a%select{|n "
+            "orphaned}2 '%3' construct%select{| associated with a '%5' compute "
+            "construct}4">;
 def err_acc_gang_dim_value
     : Error<"argument to 'gang' clause dimension must be %select{a constant "
             "expression|1, 2, or 3: evaluated to %1}0">;
 def err_acc_num_arg_conflict
-    : Error<"'num' argument to '%0' clause not allowed on a 'loop' construct "
-            "associated with a 'kernels' construct that has a "
-            "'%select{num_gangs|num_workers|vector_length}1' "
-            "clause">;
+    : Error<"'%0' argument to '%1' clause not allowed on a '%2' "
+            "construct%select{| associated with a '%4' construct}3 that has a "
+            "'%5' clause">;
+def err_acc_num_arg_conflict_reverse
+    : Error<"'%0' clause not allowed on a 'kernels loop' construct that "
+            "has a '%1' clause with a%select{n| 'num'}2 argument">;
 def err_acc_clause_in_clause_region
     : Error<"loop with a '%0' clause may not exist in the region of a '%1' "
-            "clause%select{| on a 'kernels' compute construct}2">;
+            "clause%select{| on a '%3' construct}2">;
 def err_acc_gang_reduction_conflict
     : Error<"%select{OpenACC 'gang' clause with a 'dim' value greater than "
             "1|OpenACC 'reduction' clause}0 cannot "
-            "appear on the same 'loop' construct as a %select{'reduction' "
+            "appear on the same '%1' construct as a %select{'reduction' "
             "clause|'gang' clause with a 'dim' value greater than 1}0">;
 def err_acc_gang_reduction_numgangs_conflict
-    : Error<"OpenACC '%0' clause cannot appear on the same 'loop' construct "
-            "as a '%1' clause inside a compute construct with a "
+    : Error<"OpenACC '%0' clause cannot appear on the same '%2' construct as a "
+            "'%1' clause %select{inside a compute construct with a|and a}3 "
             "'num_gangs' clause with more than one argument">;
-def err_reduction_op_mismatch
+
+    def err_reduction_op_mismatch
     : Error<"OpenACC 'reduction' variable must have the same operator in all "
             "nested constructs (%0 vs %1)">;
 def err_acc_loop_variable_type
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 9088c867d53ce..15c59c6bcdf29 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -163,6 +163,8 @@ FEATURE(c_atomic, LangOpts.C11)
 FEATURE(c_generic_selections, LangOpts.C11)
 FEATURE(c_static_assert, LangOpts.C11)
 FEATURE(c_thread_local, LangOpts.C11 &&PP.getTargetInfo().isTLSSupported())
+// C23 features
+FEATURE(c_fixed_enum, LangOpts.C23)
 // C++11 features
 FEATURE(cxx_access_control_sfinae, LangOpts.CPlusPlus11)
 FEATURE(cxx_alias_templates, LangOpts.CPlusPlus11)
@@ -269,6 +271,7 @@ EXTENSION(c_static_assert, true)
 EXTENSION(c_thread_local, PP.getTargetInfo().isTLSSupported())
 // C23 features supported by other languages as extensions
 EXTENSION(c_attributes, true)
+EXTENSION(c_fixed_enum, true)
 // C++11 features supported by other languages as extensions.
 EXTENSION(cxx_atomic, LangOpts.CPlusPlus)
 EXTENSION(cxx_default_function_template_args, LangOpts.CPlusPlus)
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 1499101624b9c..8132089f9736b 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -178,6 +178,7 @@ LANGOPT(PointerAuthInitFini, 1, 0, "sign function pointers in init/fini arrays")
 LANGOPT(PointerAuthInitFiniAddressDiscrimination, 1, 0,
         "incorporate address discrimination in authenticated function pointers in init/fini arrays")
 LANGOPT(PointerAuthELFGOT, 1, 0, "authenticate pointers from GOT")
+LANGOPT(AArch64JumpTableHardening, 1, 0, "use hardened lowering for jump-table dispatch")
 
 LANGOPT(DoubleSquareBracketAttributes, 1, 0, "'[[]]' attributes extension for all language standard modes")
 LANGOPT(ExperimentalLateParseAttributes, 1, 0, "experimental late parsing of attributes")
diff --git a/clang/include/clang/Basic/PointerAuthOptions.h b/clang/include/clang/Basic/PointerAuthOptions.h
index 3deb666b37513..a3a3e50bcde5d 100644
--- a/clang/include/clang/Basic/PointerAuthOptions.h
+++ b/clang/include/clang/Basic/PointerAuthOptions.h
@@ -172,6 +172,9 @@ struct PointerAuthOptions {
   /// Do indirect goto label addresses need to be authenticated?
   bool IndirectGotos = false;
 
+  /// Use hardened lowering for jump-table dispatch?
+  bool AArch64JumpTableHardening = false;
+
   /// The ABI for C function pointers.
   PointerAuthSchema FunctionPointers;
 
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index 89ebf5758a5b5..a14fd2c4b224d 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -336,6 +336,7 @@ namespace clang {
     bool isTupleSet() const { return Flags & IsTupleSet; }
     bool isReadZA() const { return Flags & IsReadZA; }
     bool isWriteZA() const { return Flags & IsWriteZA; }
+    bool setsFPMR() const { return Flags & SetsFPMR; }
     bool isReductionQV() const { return Flags & IsReductionQV; }
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 9cd23d123f2ba..52a1ac9781395 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/Builtins.h"
 #include "clang/Basic/CFProtectionOptions.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/LLVM.h"
@@ -1009,11 +1010,11 @@ class TargetInfo : public TransferrableTargetInfo,
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const = 0;
 
-
-  /// Return information about target-specific builtins for
-  /// the current primary target, and info about which builtins are non-portable
-  /// across the current set of primary and secondary targets.
-  virtual ArrayRef<Builtin::Info> getTargetBuiltins() const = 0;
+  /// Return information about target-specific builtins for the current primary
+  /// target, and info about which builtins are non-portable across the current
+  /// set of primary and secondary targets.
+  virtual std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const = 0;
 
   /// Returns target-specific min and max values VScale_Range.
   virtual std::optional<std::pair<unsigned, unsigned>>
@@ -1525,14 +1526,10 @@ class TargetInfo : public TransferrableTargetInfo,
 
   // Return the target-specific priority for features/cpus/vendors so
   // that they can be properly sorted for checking.
-  virtual unsigned multiVersionSortPriority(StringRef Name) const {
+  virtual unsigned getFMVPriority(ArrayRef<StringRef> Features) const {
     return 0;
   }
 
-  // Return the target-specific cost for feature
-  // that taken into account in priority sorting.
-  virtual unsigned multiVersionFeatureCost() const { return 0; }
-
   // Validate the contents of the __builtin_cpu_is(const char*)
   // argument.
   virtual bool validateCpuIs(StringRef Name) const { return false; }
diff --git a/clang/include/clang/Basic/arm_mve.td b/clang/include/clang/Basic/arm_mve.td
index 93abbc47c54dd..6dd8c52ddfd77 100644
--- a/clang/include/clang/Basic/arm_mve.td
+++ b/clang/include/clang/Basic/arm_mve.td
@@ -1270,13 +1270,13 @@ defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
 def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
 def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
 
-multiclass vadcsbc {
+multiclass vadcsbc<dag initial_carry_in> {
   def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
       (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
            (xval $pair, 0))>;
   def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
-      (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair,
+      (seq (IRInt<NAME, [Vector]> $a, $b, initial_carry_in):$pair,
            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
            (xval $pair, 0))>;
   def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
@@ -1288,13 +1288,13 @@ multiclass vadcsbc {
   def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
                                     Ptr<uint>:$carry, Predicate:$pred),
       (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
-               0, $pred):$pair,
+               initial_carry_in, $pred):$pair,
            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
            (xval $pair, 0))>;
 }
 let params = T.Int32 in {
-  defm vadc: vadcsbc;
-  defm vsbc: vadcsbc;
+  defm vadc: vadcsbc<(u32 0)>;
+  defm vsbc: vadcsbc<(shl 1, 29)>;
 }
 
 let params = T.Int in {
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 0f689e82bdb74..71b2c7cdd04f9 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -824,4 +824,14 @@ let SMETargetGuard = "sme-lutv2" in {
   def SVLUTI4_ZT_X4 : SInst<"svluti4_zt_{d}_x4", "4i2.u", "cUc", MergeNone, "aarch64_sme_luti4_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>]>;
 }
 
+let SMETargetGuard = "sme-f8f32" in {
+  def SVMOPA_FP8_ZA32 : Inst<"svmopa_za32[_mf8]_m_fpm", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za32",
+                             [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<0, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme-f8f16" in {
+  def SVMOPA_FP8_ZA16 : Inst<"svmopa_za16[_mf8]_m_fpm", "viPPdd>", "m", MergeNone, "aarch64_sme_fp8_fmopa_za16",
+                             [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<0, ImmCheck0_1>]>;
+}
+
 } // let SVETargetGuard = InvalidMode
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index c6b7cd637b9ec..7b8ecf29a9de6 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -762,14 +762,14 @@ def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone,
 ////////////////////////////////////////////////////////////////////////////////
 // While comparisons
 
-def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Counting bit
@@ -1365,10 +1365,10 @@ def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNon
 def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 }
 
 let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2"  in {
@@ -2326,7 +2326,7 @@ let SVETargetGuard = "sve2p1,bf16", SMETargetGuard = "sme2p1,bf16" in {
 // Multi-vector convert to/from floating-point.
 //
 let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
-  def SVCVT_F16_X2  : SInst<"svcvt_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>;
+  def SVCVT_F16_X2  : SInst<"svcvt_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>;
   def SVCVT_BF16_X2 : SInst<"svcvt_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvt_x2", [IsOverloadNone, IsStreaming],[]>;
 
   def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f",  MergeNone, "aarch64_sve_ucvtf_x2",  [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
@@ -2348,7 +2348,7 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in {
 // Multi-vector floating-point convert from single-precision to interleaved half-precision/BFloat16
 //
 let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
-  def SVCVTN_F16_X2  : SInst<"svcvtn_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>;
+  def SVCVTN_F16_X2  : SInst<"svcvtn_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>;
   def SVCVTN_BF16_X2 : SInst<"svcvtn_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvtn_x2", [IsOverloadNone, IsStreaming],[]>;
 }
 
@@ -2422,14 +2422,20 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
   def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x4", [IsStreaming], []>;
 }
 
-//
-// Multi-vector scaling
-//
-let  SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,fp8" in {
+let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,fp8" in {
+  // Multi-vector scaling
   def FSCALE_SINGLE_X2 : Inst<"svscale[_single_{d}_x2]", "22x", "fhd", MergeNone, "aarch64_sme_fp8_scale_single_x2", [IsStreaming],[]>;
   def FSCALE_SINGLE_X4 : Inst<"svscale[_single_{d}_x4]", "44x", "fhd", MergeNone, "aarch64_sme_fp8_scale_single_x4", [IsStreaming],[]>;
   def FSCALE_X2 : Inst<"svscale[_{d}_x2]", "222.x", "fhd", MergeNone, "aarch64_sme_fp8_scale_x2", [IsStreaming],[]>;
   def FSCALE_X4 : Inst<"svscale[_{d}_x4]", "444.x", "fhd", MergeNone, "aarch64_sme_fp8_scale_x4", [IsStreaming],[]>;
+
+  // Convert from FP8 to half-precision/BFloat16 multi-vector
+  def SVF1CVT_X2 : Inst<"svcvt1_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1_x2", [IsStreaming, SetsFPMR], []>;
+  def SVF2CVT_X2 : Inst<"svcvt2_{d}[_mf8]_x2_fpm", "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2_x2", [IsStreaming, SetsFPMR], []>;
+
+  // Convert from FP8 to deinterleaved half-precision/BFloat16 multi-vector
+  def SVF1CVTL_X2 : Inst<"svcvtl1_{d}[_mf8]_x2_fpm",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl1_x2",  [IsStreaming, SetsFPMR], []>;
+  def SVF2CVTL_X2 : Inst<"svcvtl2_{d}[_mf8]_x2_fpm",  "2~>", "bh", MergeNone, "aarch64_sve_fp8_cvtl2_x2",  [IsStreaming, SetsFPMR], []>;
 }
 
 let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in {
@@ -2445,3 +2451,15 @@ let SVETargetGuard = "sve2,faminmax", SMETargetGuard = "sme2,faminmax" in {
   defm SVAMIN : SInstZPZZ<"svamin", "hfd", "aarch64_sve_famin", "aarch64_sve_famin_u">;
   defm SVAMAX : SInstZPZZ<"svamax", "hfd", "aarch64_sve_famax", "aarch64_sve_famax_u">;
 }
+
+let SVETargetGuard = "sve2,fp8", SMETargetGuard = "sme2,fp8" in {
+  // SVE FP8 widening conversions
+
+  // 8-bit floating-point convert to BFloat16/Float16
+  def SVF1CVT : SInst<"svcvt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt1", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVF2CVT : SInst<"svcvt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvt2", [VerifyRuntimeMode, SetsFPMR]>;
+
+  // 8-bit floating-point convert to BFloat16/Float16 (top)
+  def SVF1CVTLT : SInst<"svcvtlt1_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt1", [VerifyRuntimeMode, SetsFPMR]>;
+  def SVF2CVTLT : SInst<"svcvtlt2_{d}[_mf8]_fpm", "d~>", "bh", MergeNone, "aarch64_sve_fp8_cvtlt2", [VerifyRuntimeMode, SetsFPMR]>;
+}
diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td
index 50911fb63e818..de10be7bdce0d 100644
--- a/clang/include/clang/Basic/arm_sve_sme_incl.td
+++ b/clang/include/clang/Basic/arm_sve_sme_incl.td
@@ -94,6 +94,7 @@ include "arm_immcheck_incl.td"
 // l: int64_t
 // m: uint32_t
 // n: uint64_t
+// >: fpm_t
 
 // [: svuint8_t
 // t: svint32_t
@@ -103,6 +104,7 @@ include "arm_immcheck_incl.td"
 // M: svfloat32_t
 // N: svfloat64_t
 // $: svbfloat16_t
+// ~: svmfloat8_t
 
 // J: Prefetch type (sv_prfop)
 
@@ -235,6 +237,7 @@ def IsInOutZA                       : FlagType<0x200000000000>;
 def IsInZT0                         : FlagType<0x400000000000>;
 def IsOutZT0                        : FlagType<0x800000000000>;
 def IsInOutZT0                      : FlagType<0x1000000000000>;
+def SetsFPMR                        : FlagType<0x2000000000000>;
 
 defvar InvalidMode = "";
 
diff --git a/clang/include/clang/CIR/CIRGenerator.h b/clang/include/clang/CIR/CIRGenerator.h
index c8ca7e4bfa728..414eba80b88b8 100644
--- a/clang/include/clang/CIR/CIRGenerator.h
+++ b/clang/include/clang/CIR/CIRGenerator.h
@@ -37,14 +37,14 @@ namespace cir {
 class CIRGenerator : public clang::ASTConsumer {
   virtual void anchor();
   clang::DiagnosticsEngine &diags;
-  clang::ASTContext *astCtx;
+  clang::ASTContext *astContext;
   // Only used for debug info.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs;
 
   const clang::CodeGenOptions &codeGenOpts;
 
 protected:
-  std::unique_ptr<mlir::MLIRContext> mlirCtx;
+  std::unique_ptr<mlir::MLIRContext> mlirContext;
   std::unique_ptr<clang::CIRGen::CIRGenModule> cgm;
 
 public:
@@ -52,7 +52,7 @@ class CIRGenerator : public clang::ASTConsumer {
                llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
                const clang::CodeGenOptions &cgo);
   ~CIRGenerator() override;
-  void Initialize(clang::ASTContext &astCtx) override;
+  void Initialize(clang::ASTContext &astContext) override;
   bool HandleTopLevelDecl(clang::DeclGroupRef group) override;
   mlir::ModuleOp getModule() const;
 };
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
new file mode 100644
index 0000000000000..75ae74e926fbc
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_CIR_DIALECT_BUILDER_CIRBASEBUILDER_H
+#define LLVM_CLANG_CIR_DIALECT_BUILDER_CIRBASEBUILDER_H
+
+#include "mlir/IR/Builders.h"
+
+namespace cir {
+
+class CIRBaseBuilderTy : public mlir::OpBuilder {
+
+public:
+  CIRBaseBuilderTy(mlir::MLIRContext &mlirContext)
+      : mlir::OpBuilder(&mlirContext) {}
+};
+
+} // namespace cir
+
+#endif
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 4462eb6fc00ba..0d6c65ecf4102 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_CIR_DIALECT_IR_CIROPS
 
 include "clang/CIR/Dialect/IR/CIRDialect.td"
+include "clang/CIR/Dialect/IR/CIRTypes.td"
 
 include "mlir/IR/BuiltinAttributeInterfaces.td"
 include "mlir/IR/EnumAttr.td"
@@ -74,6 +75,35 @@ class LLVMLoweringInfo {
 class CIR_Op<string mnemonic, list<Trait> traits = []> :
     Op<CIR_Dialect, mnemonic, traits>, LLVMLoweringInfo;
 
+//===----------------------------------------------------------------------===//
+// GlobalOp
+//===----------------------------------------------------------------------===//
+
+// TODO(CIR): For starters, cir.global has only name and type.  The other
+// properties of a global variable will be added over time as more of ClangIR
+// is upstreamed.
+
+def GlobalOp : CIR_Op<"global"> {
+  let summary = "Declare or define a global variable";
+  let description = [{
+    The `cir.global` operation declares or defines a named global variable.
+
+    The backing memory for the variable is allocated statically and is
+    described by the type of the variable.
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name, TypeAttr:$sym_type);
+
+  let assemblyFormat = [{ $sym_name `:` $sym_type attr-dict }];
+
+  let skipDefaultBuilders = 1;
+
+  let builders = [OpBuilder<(ins "llvm::StringRef":$sym_name,
+                                 "mlir::Type":$sym_type)>];
+
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // FuncOp
 //===----------------------------------------------------------------------===//
@@ -85,14 +115,15 @@ class CIR_Op<string mnemonic, list<Trait> traits = []> :
 def FuncOp : CIR_Op<"func"> {
   let summary = "Declare or define a function";
   let description = [{
-    ... lots of text to be added later ...
+    The `cir.func` operation defines a function, similar to the `mlir::FuncOp`
+    built-in.
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name);
 
   let skipDefaultBuilders = 1;
 
-  let builders = [OpBuilder<(ins "llvm::StringRef":$name)>];
+  let builders = [OpBuilder<(ins "llvm::StringRef":$sym_name)>];
 
   let hasCustomAssemblyFormat = 1;
   let hasVerifier = 1;
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.h b/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
new file mode 100644
index 0000000000000..2bc7d77b2bc8a
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the types in the CIR dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_CIR_IR_CIRTYPES_H_
+#define MLIR_DIALECT_CIR_IR_CIRTYPES_H_
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Types.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
+
+//===----------------------------------------------------------------------===//
+// CIR Dialect Tablegen'd Types
+//===----------------------------------------------------------------------===//
+
+#define GET_TYPEDEF_CLASSES
+#include "clang/CIR/Dialect/IR/CIROpsTypes.h.inc"
+
+#endif // MLIR_DIALECT_CIR_IR_CIRTYPES_H_
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
new file mode 100644
index 0000000000000..ce0b6ba1d68c5
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -0,0 +1,132 @@
+//===- CIRTypes.td - CIR dialect types ---------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the CIR dialect types.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CIR_DIALECT_CIR_TYPES
+#define MLIR_CIR_DIALECT_CIR_TYPES
+
+include "clang/CIR/Dialect/IR/CIRDialect.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
+include "mlir/IR/AttrTypeBase.td"
+
+//===----------------------------------------------------------------------===//
+// CIR Types
+//===----------------------------------------------------------------------===//
+
+class CIR_Type<string name, string typeMnemonic, list<Trait> traits = [],
+               string baseCppClass = "::mlir::Type">
+    : TypeDef<CIR_Dialect, name, traits, baseCppClass> {
+  let mnemonic = typeMnemonic;
+}
+
+//===----------------------------------------------------------------------===//
+// IntType
+//===----------------------------------------------------------------------===//
+
+def CIR_IntType : CIR_Type<"Int", "int",
+    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
+  let summary = "Integer type with arbitrary precision up to a fixed limit";
+  let description = [{
+    CIR type that represents integer types with arbitrary precision, including
+    standard integral types such as `int` and `long`, extended integral types
+    such as `__int128`, and arbitrary width types such as `_BitInt(n)`.
+
+    Those integer types that are directly available in C/C++ standard are called
+    primitive integer types. Said types are: `signed char`, `short`, `int`,
+    `long`, `long long`, and their unsigned variations.
+  }];
+  let parameters = (ins "unsigned":$width, "bool":$isSigned);
+  let hasCustomAssemblyFormat = 1;
+  let extraClassDeclaration = [{
+    /// Return true if this is a signed integer type.
+    bool isSigned() const { return getIsSigned(); }
+    /// Return true if this is an unsigned integer type.
+    bool isUnsigned() const { return !getIsSigned(); }
+    /// Return type alias.
+    std::string getAlias() const {
+      return (isSigned() ? 's' : 'u') + std::to_string(getWidth()) + 'i';
+    }
+    /// Return true if this is a primitive integer type (i.e. signed or unsigned
+    /// integer types whose bit width is 8, 16, 32, or 64).
+    bool isPrimitive() const {
+      return isValidPrimitiveIntBitwidth(getWidth());
+    }
+    bool isSignedPrimitive() const {
+      return isPrimitive() && isSigned();
+    }
+
+    /// Returns a minimum bitwidth of cir::IntType
+    static unsigned minBitwidth() { return 1; }
+    /// Returns a maximum bitwidth of cir::IntType
+    static unsigned maxBitwidth() { return 128; }
+
+    /// Returns true if cir::IntType that represents a primitive integer type
+    /// can be constructed from the provided bitwidth.
+    static bool isValidPrimitiveIntBitwidth(unsigned width) {
+      return width == 8 || width == 16 || width == 32 || width == 64;
+    }
+  }];
+  let genVerifyDecl = 1;
+}
+
+// Constraints
+
+// Unsigned integer type of a specific width.
+class UInt<int width>
+  : Type<And<[
+	CPred<"::mlir::isa<::cir::IntType>($_self)">,
+	CPred<"::mlir::cast<::cir::IntType>($_self).isUnsigned()">,
+	CPred<"::mlir::cast<::cir::IntType>($_self).getWidth() == " # width>
+        ]>, width # "-bit unsigned integer", "::cir::IntType">,
+    BuildableType<
+      "cir::IntType::get($_builder.getContext(), "
+      # width # ", /*isSigned=*/false)"> {
+  int bitwidth = width;
+}
+
+def UInt1  : UInt<1>;
+def UInt8  : UInt<8>;
+def UInt16 : UInt<16>;
+def UInt32 : UInt<32>;
+def UInt64 : UInt<64>;
+
+// Signed integer type of a specific width.
+class SInt<int width>
+  : Type<And<[
+	CPred<"::mlir::isa<::cir::IntType>($_self)">,
+	CPred<"::mlir::cast<::cir::IntType>($_self).isSigned()">,
+	CPred<"::mlir::cast<::cir::IntType>($_self).getWidth() == " # width>
+        ]>, width # "-bit signed integer", "::cir::IntType">,
+    BuildableType<
+      "cir::IntType::get($_builder.getContext(), "
+      # width # ", /*isSigned=*/true)"> {
+  int bitwidth = width;
+}
+
+def SInt1  : SInt<1>;
+def SInt8  : SInt<8>;
+def SInt16 : SInt<16>;
+def SInt32 : SInt<32>;
+def SInt64 : SInt<64>;
+
+def PrimitiveUInt
+    : AnyTypeOf<[UInt8, UInt16, UInt32, UInt64], "primitive unsigned int",
+                "::cir::IntType">;
+
+def PrimitiveSInt
+    : AnyTypeOf<[SInt8, SInt16, SInt32, SInt64], "primitive signed int",
+                "::cir::IntType">;
+
+def PrimitiveInt
+    : AnyTypeOf<[UInt8, UInt16, UInt32, UInt64, SInt8, SInt16, SInt32, SInt64],
+                "primitive int", "::cir::IntType">;
+
+#endif // MLIR_CIR_DIALECT_CIR_TYPES
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 47d2f9b11bc0e..8ce7fca59445b 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -299,8 +299,11 @@ class Driver {
   /// Object that stores strings read from configuration file.
   llvm::StringSaver Saver;
 
-  /// Arguments originated from configuration file.
-  std::unique_ptr<llvm::opt::InputArgList> CfgOptions;
+  /// Arguments originated from configuration file (head part).
+  std::unique_ptr<llvm::opt::InputArgList> CfgOptionsHead;
+
+  /// Arguments originated from configuration file (tail part).
+  std::unique_ptr<llvm::opt::InputArgList> CfgOptionsTail;
 
   /// Arguments originated from command line.
   std::unique_ptr<llvm::opt::InputArgList> CLOptions;
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 1c72374133177..2f0af3739d63b 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -246,7 +246,8 @@ def m_riscv_Features_Group : OptionGroup<"<riscv features group>">,
 def m_ve_Features_Group : OptionGroup<"<ve features group>">,
                           Group<m_Group>, DocName<"VE">;
 def m_loongarch_Features_Group : OptionGroup<"<loongarch features group>">,
-                                 Group<m_Group>, DocName<"LoongArch">;
+                                 Group<m_Group>, DocName<"LoongArch">,
+                                 Visibility<[ClangOption, CLOption, FlangOption]>;
 
 def m_libc_Group : OptionGroup<"<m libc group>">, Group<m_mips_Features_Group>,
                    Flags<[HelpHidden]>;
@@ -1808,7 +1809,7 @@ def fno_profile_sample_use : Flag<["-"], "fno-profile-sample-use">, Group<f_Grou
     Visibility<[ClangOption, CLOption]>;
 def fprofile_sample_use_EQ : Joined<["-"], "fprofile-sample-use=">,
     Group<f_Group>,
-    Visibility<[ClangOption, CC1Option]>,
+    Visibility<[ClangOption, CLOption, CC1Option]>,
     HelpText<"Enable sample-based profile guided optimizations">,
     MarshallingInfoString<CodeGenOpts<"SampleProfileFile">>;
 def fprofile_sample_accurate : Flag<["-"], "fprofile-sample-accurate">,
@@ -1858,12 +1859,12 @@ defm debug_info_for_profiling : BoolFOption<"debug-info-for-profiling",
   PosFlag<SetTrue, [], [ClangOption, CC1Option],
           "Emit extra debug info to make sample profile more accurate">,
   NegFlag<SetFalse>>;
-def fprofile_generate_cold_function_coverage : Flag<["-"], "fprofile-generate-cold-function-coverage">, 
+def fprofile_generate_cold_function_coverage : Flag<["-"], "fprofile-generate-cold-function-coverage">,
     Group<f_Group>, Visibility<[ClangOption, CLOption]>,
     HelpText<"Generate instrumented code to collect coverage info for cold functions into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">;
-def fprofile_generate_cold_function_coverage_EQ : Joined<["-"], "fprofile-generate-cold-function-coverage=">, 
+def fprofile_generate_cold_function_coverage_EQ : Joined<["-"], "fprofile-generate-cold-function-coverage=">,
     Group<f_Group>, Visibility<[ClangOption, CLOption]>, MetaVarName<"<directory>">,
-    HelpText<"Generate instrumented code to collect coverage info for cold functions into <directory>/default.profraw (overridden by LLVM_PROFILE_FILE env var)">; 
+    HelpText<"Generate instrumented code to collect coverage info for cold functions into <directory>/default.profraw (overridden by LLVM_PROFILE_FILE env var)">;
 def fprofile_instr_generate : Flag<["-"], "fprofile-instr-generate">,
     Group<f_Group>, Visibility<[ClangOption, CLOption]>,
     HelpText<"Generate instrumented code to collect execution counts into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">;
@@ -3274,11 +3275,14 @@ defm skip_odr_check_in_gmf : BoolOption<"f", "skip-odr-check-in-gmf",
           "Perform ODR checks for decls in the global module fragment.">>,
   Group<f_Group>;
 
-def modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">,
+def modules_reduced_bmi : Flag<["-"], "fmodules-reduced-bmi">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Generate the reduced BMI">,
   MarshallingInfoFlag<FrontendOpts<"GenReducedBMI">>;
 
+def experimental_modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">, 
+  Group<f_Group>, Visibility<[ClangOption, CC1Option]>, Alias<modules_reduced_bmi>;
+
 def fmodules_embed_all_files : Joined<["-"], "fmodules-embed-all-files">,
   Visibility<[ClangOption, CC1Option, CLOption]>,
   HelpText<"Embed the contents of all files read by this compilation into "
@@ -4446,6 +4450,7 @@ defm ptrauth_init_fini : OptInCC1FFlag<"ptrauth-init-fini", "Enable signing of f
 defm ptrauth_init_fini_address_discrimination : OptInCC1FFlag<"ptrauth-init-fini-address-discrimination",
   "Enable address discrimination of function pointers in init/fini arrays">;
 defm ptrauth_elf_got : OptInCC1FFlag<"ptrauth-elf-got", "Enable authentication of pointers from GOT (ELF only)">;
+defm aarch64_jump_table_hardening: OptInCC1FFlag<"aarch64-jump-table-hardening", "Use hardened lowering for jump-table dispatch">;
 }
 
 def fenable_matrix : Flag<["-"], "fenable-matrix">, Group<f_Group>,
@@ -5193,6 +5198,10 @@ def matomics : Flag<["-"], "matomics">, Group<m_wasm_Features_Group>;
 def mno_atomics : Flag<["-"], "mno-atomics">, Group<m_wasm_Features_Group>;
 def mbulk_memory : Flag<["-"], "mbulk-memory">, Group<m_wasm_Features_Group>;
 def mno_bulk_memory : Flag<["-"], "mno-bulk-memory">, Group<m_wasm_Features_Group>;
+def mbulk_memory_opt : Flag<["-"], "mbulk-memory-opt">, Group<m_wasm_Features_Group>;
+def mno_bulk_memory_opt : Flag<["-"], "mno-bulk-memory-opt">, Group<m_wasm_Features_Group>;
+def mcall_indirect_overlong : Flag<["-"], "mcall-indirect-overlong">, Group<m_wasm_Features_Group>;
+def mno_call_indirect_overlong : Flag<["-"], "mno-call-indirect-overlong">, Group<m_wasm_Features_Group>;
 def mexception_handing : Flag<["-"], "mexception-handling">, Group<m_wasm_Features_Group>;
 def mno_exception_handing : Flag<["-"], "mno-exception-handling">, Group<m_wasm_Features_Group>;
 def mextended_const : Flag<["-"], "mextended-const">, Group<m_wasm_Features_Group>;
@@ -5514,6 +5523,10 @@ def mlam_bh : Flag<["-"], "mlam-bh">, Group<m_loongarch_Features_Group>,
   HelpText<"Enable amswap[_db].{b/h} and amadd[_db].{b/h}">;
 def mno_lam_bh : Flag<["-"], "mno-lam-bh">, Group<m_loongarch_Features_Group>,
   HelpText<"Disable amswap[_db].{b/h} and amadd[_db].{b/h}">;
+def mlamcas : Flag<["-"], "mlamcas">, Group<m_loongarch_Features_Group>,
+  HelpText<"Enable amcas[_db].{b/h/w/d}">;
+def mno_lamcas : Flag<["-"], "mno-lamcas">, Group<m_loongarch_Features_Group>,
+  HelpText<"Disable amcas[_db].{b/h/w/d}">;
 def mld_seq_sa : Flag<["-"], "mld-seq-sa">, Group<m_loongarch_Features_Group>,
   HelpText<"Do not generate load-load barrier instructions (dbar 0x700)">;
 def mno_ld_seq_sa : Flag<["-"], "mno-ld-seq-sa">, Group<m_loongarch_Features_Group>,
@@ -7211,10 +7224,6 @@ def flang_deprecated_no_hlfir : Flag<["-"], "flang-deprecated-no-hlfir">,
   Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
   HelpText<"Do not use HLFIR lowering (deprecated)">;
 
-def flang_experimental_integer_overflow : Flag<["-"], "flang-experimental-integer-overflow">,
-  Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
-  HelpText<"Add nsw flag to internal operations such as do-variable increment (experimental)">;
-
 //===----------------------------------------------------------------------===//
 // FLangOption + Visibility<[FlangOption, CLOption, DXCOption] + NoXarchOption
 //===----------------------------------------------------------------------===//
@@ -7763,9 +7772,12 @@ def fuse_register_sized_bitfield_access: Flag<["-"], "fuse-register-sized-bitfie
 def relaxed_aliasing : Flag<["-"], "relaxed-aliasing">,
   HelpText<"Turn off Type Based Alias Analysis">,
   MarshallingInfoFlag<CodeGenOpts<"RelaxedAliasing">>;
-def pointer_tbaa: Flag<["-"], "pointer-tbaa">,
-  HelpText<"Turn on Type Based Alias Analysis for pointer accesses">,
-  MarshallingInfoFlag<CodeGenOpts<"PointerTBAA">>;
+defm pointer_tbaa: BoolOption<"", "pointer-tbaa", CodeGenOpts<"PointerTBAA">, 
+DefaultTrue,
+  PosFlag<SetTrue, [], [ClangOption], "Enable">,
+  NegFlag<SetFalse, [], [ClangOption], "Disable">,
+  BothFlags<[], [ClangOption], " that single precision floating-point divide and sqrt used in ">>
+  ;
 def no_struct_path_tbaa : Flag<["-"], "no-struct-path-tbaa">,
   HelpText<"Turn off struct-path aware Type Based Alias Analysis">,
   MarshallingInfoNegativeFlag<CodeGenOpts<"StructPathTBAA">>;
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 056fad2cc0ff8..6383934afa2c4 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -2506,9 +2506,9 @@ struct FormatStyle {
   /// lists.
   ///
   /// Important differences:
-  /// - No spaces inside the braced list.
-  /// - No line break before the closing brace.
-  /// - Indentation with the continuation indent, not with the block indent.
+  /// * No spaces inside the braced list.
+  /// * No line break before the closing brace.
+  /// * Indentation with the continuation indent, not with the block indent.
   ///
   /// Fundamentally, C++11 braced lists are formatted exactly like function
   /// calls would be formatted in their place. If the braced list follows a name
@@ -3742,19 +3742,19 @@ struct FormatStyle {
   QualifierAlignmentStyle QualifierAlignment;
 
   /// The order in which the qualifiers appear.
-  /// Order is an array that can contain any of the following:
+  /// The order is an array that can contain any of the following:
   ///
-  ///   * const
-  ///   * inline
-  ///   * static
-  ///   * friend
-  ///   * constexpr
-  ///   * volatile
-  ///   * restrict
-  ///   * type
+  ///   * ``const``
+  ///   * ``inline``
+  ///   * ``static``
+  ///   * ``friend``
+  ///   * ``constexpr``
+  ///   * ``volatile``
+  ///   * ``restrict``
+  ///   * ``type``
   ///
   /// \note
-  ///  It **must** contain ``type``.
+  ///  It must contain ``type``.
   /// \endnote
   ///
   /// Items to the left of ``type`` will be placed to the left of the type and
@@ -5449,10 +5449,10 @@ formatReplacements(StringRef Code, const tooling::Replacements &Replaces,
 /// cleaning up the code after that on success; otherwise, return an llvm::Error
 /// carrying llvm::StringError.
 /// This also supports inserting/deleting C++ #include directives:
-/// - If a replacement has offset UINT_MAX, length 0, and a replacement text
+/// * If a replacement has offset UINT_MAX, length 0, and a replacement text
 ///   that is an #include directive, this will insert the #include into the
 ///   correct block in the \p Code.
-/// - If a replacement has offset UINT_MAX, length 1, and a replacement text
+/// * If a replacement has offset UINT_MAX, length 1, and a replacement text
 ///   that is the name of the header to be removed, the header will be removed
 ///   from \p Code if it exists.
 /// The include manipulation is done via ``tooling::HeaderInclude``, see its
@@ -5558,13 +5558,12 @@ extern const char *DefaultFallbackStyle;
 ///
 /// ``StyleName`` can take several forms:
 /// * "{<key>: <value>, ...}" - Set specic style parameters.
-/// * "<style name>" - One of the style names supported by
-/// getPredefinedStyle().
+/// * "<style name>" - One of the style names supported by getPredefinedStyle().
 /// * "file" - Load style configuration from a file called ``.clang-format``
-/// located in one of the parent directories of ``FileName`` or the current
-/// directory if ``FileName`` is empty.
+///   located in one of the parent directories of ``FileName`` or the current
+///   directory if ``FileName`` is empty.
 /// * "file:<format_file_path>" to explicitly specify the configuration file to
-/// use.
+///   use.
 ///
 /// \param[in] StyleName Style name to interpret according to the description
 /// above.
diff --git a/clang/include/clang/Sema/MultiplexExternalSemaSource.h b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
index 3d1906d869926..0c92c52854c9e 100644
--- a/clang/include/clang/Sema/MultiplexExternalSemaSource.h
+++ b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
@@ -97,6 +97,12 @@ class MultiplexExternalSemaSource : public ExternalSemaSource {
   bool FindExternalVisibleDeclsByName(const DeclContext *DC,
                                       DeclarationName Name) override;
 
+  bool LoadExternalSpecializations(const Decl *D, bool OnlyPartial) override;
+
+  bool
+  LoadExternalSpecializations(const Decl *D,
+                              ArrayRef<TemplateArgument> TemplateArgs) override;
+
   /// Ensures that the table of all visible declarations inside this
   /// context is up to date.
   void completeVisibleDeclsMap(const DeclContext *DC) override;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 7058d98bd07f0..9c686bf5eb29d 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2531,6 +2531,17 @@ class Sema final : public SemaBase {
 
   bool BuiltinNonDeterministicValue(CallExpr *TheCall);
 
+  enum BuiltinCountedByRefKind {
+    AssignmentKind,
+    InitializerKind,
+    FunctionArgKind,
+    ReturnArgKind,
+    ArraySubscriptKind,
+    BinaryExprKind,
+  };
+
+  bool CheckInvalidBuiltinCountedByRef(const Expr *E,
+                                       BuiltinCountedByRefKind K);
   bool BuiltinCountedByRef(CallExpr *TheCall);
 
   // Matrix builtin handling.
@@ -5324,7 +5335,7 @@ class Sema final : public SemaBase {
   /// is complete.
   void ActOnFinishCXXInClassMemberInitializer(Decl *VarDecl,
                                               SourceLocation EqualLoc,
-                                              Expr *Init);
+                                              ExprResult Init);
 
   /// Handle a C++ member initializer using parentheses syntax.
   MemInitResult
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index ee685d95c9615..f4cd11f423a84 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -119,6 +119,7 @@ class SemaHLSL : public SemaBase {
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
   void handleWaveSizeAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL);
+  void handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL);
   void handlePackOffsetAttr(Decl *D, const ParsedAttr &AL);
   void handleShaderAttr(Decl *D, const ParsedAttr &AL);
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index e8f022362ea16..170a6f4885c96 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -87,6 +87,10 @@ class SemaOpenACC : public SemaBase {
     /// which allows us to diagnose if the value of 'N' is too large for the
     /// current number of 'for' loops.
     bool CollapseDepthSatisfied = true;
+
+    /// Records the kind of the directive that this clause is attached to, which
+    /// allows us to use it in diagnostics.
+    OpenACCDirectiveKind DirectiveKind = OpenACCDirectiveKind::Invalid;
   } CollapseInfo;
 
   /// The 'tile' clause requires a bit of additional checking as well, so like
@@ -103,6 +107,10 @@ class SemaOpenACC : public SemaBase {
     /// which allows us to diagnose if the number of arguments is too large for
     /// the current number of 'for' loops.
     bool TileDepthSatisfied = true;
+
+    /// Records the kind of the directive that this clause is attached to, which
+    /// allows us to use it in diagnostics.
+    OpenACCDirectiveKind DirectiveKind = OpenACCDirectiveKind::Invalid;
   } TileInfo;
 
   /// A list of the active reduction clauses, which allows us to check that all
@@ -156,9 +164,14 @@ class SemaOpenACC : public SemaBase {
   }
 
   /// If there is a current 'active' loop construct with a 'gang' clause on a
-  /// 'kernel' construct, this will have the source location for it. This
-  /// permits us to implement the restriction of no further 'gang' clauses.
-  SourceLocation LoopGangClauseOnKernelLoc;
+  /// 'kernel' construct, this will have the source location for it, and the
+  /// 'kernel kind'. This permits us to implement the restriction of no further
+  /// 'gang' clauses.
+  struct LoopGangOnKernelTy {
+    SourceLocation Loc;
+    OpenACCDirectiveKind DirKind = OpenACCDirectiveKind::Invalid;
+  } LoopGangClauseOnKernel;
+
   /// If there is a current 'active' loop construct with a 'worker' clause on it
   /// (on any sort of construct), this has the source location for it.  This
   /// permits us to implement the restriction of no further 'gang' or 'worker'
@@ -697,12 +710,15 @@ class SemaOpenACC : public SemaBase {
   ExprResult CheckTileSizeExpr(Expr *SizeExpr);
 
   // Check a single expression on a gang clause.
-  ExprResult CheckGangExpr(OpenACCGangKind GK, Expr *E);
+  ExprResult CheckGangExpr(ArrayRef<const OpenACCClause *> ExistingClauses,
+                           OpenACCDirectiveKind DK, OpenACCGangKind GK,
+                           Expr *E);
 
   // Does the checking for a 'gang' clause that needs to be done in dependent
   // and not dependent cases.
   OpenACCClause *
-  CheckGangClause(ArrayRef<const OpenACCClause *> ExistingClauses,
+  CheckGangClause(OpenACCDirectiveKind DirKind,
+                  ArrayRef<const OpenACCClause *> ExistingClauses,
                   SourceLocation BeginLoc, SourceLocation LParenLoc,
                   ArrayRef<OpenACCGangKind> GangKinds,
                   ArrayRef<Expr *> IntExprs, SourceLocation EndLoc);
@@ -763,7 +779,7 @@ class SemaOpenACC : public SemaBase {
     SemaOpenACC &SemaRef;
     ComputeConstructInfo OldActiveComputeConstructInfo;
     OpenACCDirectiveKind DirKind;
-    SourceLocation OldLoopGangClauseOnKernelLoc;
+    LoopGangOnKernelTy OldLoopGangClauseOnKernel;
     SourceLocation OldLoopWorkerClauseLoc;
     SourceLocation OldLoopVectorClauseLoc;
     LoopWithoutSeqCheckingInfo OldLoopWithoutSeqInfo;
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index c9fba0b809b7e..14b0964402372 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -733,6 +733,13 @@ enum ASTRecordTypes {
   /// Record code for Sema's vector of functions/blocks with effects to
   /// be verified.
   DECLS_WITH_EFFECTS_TO_VERIFY = 72,
+
+  /// Record code for updated specialization
+  UPDATE_SPECIALIZATION = 73,
+
+  CXX_ADDED_TEMPLATE_SPECIALIZATION = 74,
+
+  CXX_ADDED_TEMPLATE_PARTIAL_SPECIALIZATION = 75,
 };
 
 /// Record types used within a source manager block.
@@ -1507,6 +1514,12 @@ enum DeclCode {
   /// An ImplicitConceptSpecializationDecl record.
   DECL_IMPLICIT_CONCEPT_SPECIALIZATION,
 
+  // A decls specilization record.
+  DECL_SPECIALIZATIONS,
+
+  // A decls specilization record.
+  DECL_PARTIAL_SPECIALIZATIONS,
+
   DECL_LAST = DECL_IMPLICIT_CONCEPT_SPECIALIZATION
 };
 
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index f739fe688c110..f91052be5e129 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -354,6 +354,9 @@ class ASTIdentifierLookupTrait;
 /// The on-disk hash table(s) used for DeclContext name lookup.
 struct DeclContextLookupTable;
 
+/// The on-disk hash table(s) used for specialization decls.
+struct LazySpecializationInfoLookupTable;
+
 } // namespace reader
 
 } // namespace serialization
@@ -632,20 +635,40 @@ class ASTReader
   llvm::DenseMap<const DeclContext *,
                  serialization::reader::DeclContextLookupTable> Lookups;
 
+  using SpecLookupTableTy =
+      llvm::DenseMap<const Decl *,
+                     serialization::reader::LazySpecializationInfoLookupTable>;
+  /// Map from decls to specialized decls.
+  SpecLookupTableTy SpecializationsLookups;
+  /// Split partial specialization from specialization to speed up lookups.
+  SpecLookupTableTy PartialSpecializationsLookups;
+
+  bool LoadExternalSpecializationsImpl(SpecLookupTableTy &SpecLookups,
+                                       const Decl *D);
+  bool LoadExternalSpecializationsImpl(SpecLookupTableTy &SpecLookups,
+                                       const Decl *D,
+                                       ArrayRef<TemplateArgument> TemplateArgs);
+
   // Updates for visible decls can occur for other contexts than just the
   // TU, and when we read those update records, the actual context may not
   // be available yet, so have this pending map using the ID as a key. It
-  // will be realized when the context is actually loaded.
-  struct PendingVisibleUpdate {
+  // will be realized when the data is actually loaded.
+  struct UpdateData {
     ModuleFile *Mod;
     const unsigned char *Data;
   };
-  using DeclContextVisibleUpdates = SmallVector<PendingVisibleUpdate, 1>;
+  using DeclContextVisibleUpdates = SmallVector<UpdateData, 1>;
 
   /// Updates to the visible declarations of declaration contexts that
   /// haven't been loaded yet.
   llvm::DenseMap<GlobalDeclID, DeclContextVisibleUpdates> PendingVisibleUpdates;
 
+  using SpecializationsUpdate = SmallVector<UpdateData, 1>;
+  using SpecializationsUpdateMap =
+      llvm::DenseMap<GlobalDeclID, SpecializationsUpdate>;
+  SpecializationsUpdateMap PendingSpecializationsUpdates;
+  SpecializationsUpdateMap PendingPartialSpecializationsUpdates;
+
   /// The set of C++ or Objective-C classes that have forward
   /// declarations that have not yet been linked to their definitions.
   llvm::SmallPtrSet<Decl *, 4> PendingDefinitions;
@@ -678,6 +701,11 @@ class ASTReader
                                      llvm::BitstreamCursor &Cursor,
                                      uint64_t Offset, GlobalDeclID ID);
 
+  bool ReadSpecializations(ModuleFile &M, llvm::BitstreamCursor &Cursor,
+                           uint64_t Offset, Decl *D, bool IsPartial);
+  void AddSpecializations(const Decl *D, const unsigned char *Data,
+                          ModuleFile &M, bool IsPartial);
+
   /// A vector containing identifiers that have already been
   /// loaded.
   ///
@@ -1419,6 +1447,14 @@ class ASTReader
   const serialization::reader::DeclContextLookupTable *
   getLoadedLookupTables(DeclContext *Primary) const;
 
+  /// Get the loaded specializations lookup tables for \p D,
+  /// if any.
+  serialization::reader::LazySpecializationInfoLookupTable *
+  getLoadedSpecializationsLookupTables(const Decl *D, bool IsPartial);
+
+  /// If we have any unloaded specialization for \p D
+  bool haveUnloadedSpecializations(const Decl *D) const;
+
 private:
   struct ImportedModule {
     ModuleFile *Mod;
@@ -2076,6 +2112,12 @@ class ASTReader
                                       unsigned BlockID,
                                       uint64_t *StartOfBlockOffset = nullptr);
 
+  bool LoadExternalSpecializations(const Decl *D, bool OnlyPartial) override;
+
+  bool
+  LoadExternalSpecializations(const Decl *D,
+                              ArrayRef<TemplateArgument> TemplateArgs) override;
+
   /// Finds all the visible declarations with a given name.
   /// The current implementation of this method just loads the entire
   /// lookup table as unmaterialized references.
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index e418fdea44a0a..d98d23decbdc0 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -423,6 +423,13 @@ class ASTWriter : public ASTDeserializationListener,
   /// Only meaningful for reduced BMI.
   DeclUpdateMap DeclUpdatesFromGMF;
 
+  /// Mapping from decl templates and its new specialization in the
+  /// current TU.
+  using SpecializationUpdateMap =
+      llvm::MapVector<const NamedDecl *, SmallVector<const Decl *>>;
+  SpecializationUpdateMap SpecializationsUpdates;
+  SpecializationUpdateMap PartialSpecializationsUpdates;
+
   using FirstLatestDeclMap = llvm::DenseMap<Decl *, Decl *>;
 
   /// Map of first declarations from a chained PCH that point to the
@@ -575,6 +582,12 @@ class ASTWriter : public ASTDeserializationListener,
 
   bool isLookupResultExternal(StoredDeclsList &Result, DeclContext *DC);
 
+  void GenerateSpecializationInfoLookupTable(
+      const NamedDecl *D, llvm::SmallVectorImpl<const Decl *> &Specializations,
+      llvm::SmallVectorImpl<char> &LookupTable, bool IsPartial);
+  uint64_t WriteSpecializationInfoLookupTable(
+      const NamedDecl *D, llvm::SmallVectorImpl<const Decl *> &Specializations,
+      bool IsPartial);
   void GenerateNameLookupTable(ASTContext &Context, const DeclContext *DC,
                                llvm::SmallVectorImpl<char> &LookupTable);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context,
@@ -590,6 +603,7 @@ class ASTWriter : public ASTDeserializationListener,
   void WriteDeclAndTypes(ASTContext &Context);
   void PrepareWritingSpecialDecls(Sema &SemaRef);
   void WriteSpecialDeclRecords(Sema &SemaRef);
+  void WriteSpecializationsUpdates(bool IsPartial);
   void WriteDeclUpdatesBlocks(ASTContext &Context,
                               RecordDataImpl &OffsetsRecord);
   void WriteDeclContextVisibleUpdate(ASTContext &Context,
@@ -619,6 +633,9 @@ class ASTWriter : public ASTDeserializationListener,
   unsigned DeclEnumAbbrev = 0;
   unsigned DeclObjCIvarAbbrev = 0;
   unsigned DeclCXXMethodAbbrev = 0;
+  unsigned DeclSpecializationsAbbrev = 0;
+  unsigned DeclPartialSpecializationsAbbrev = 0;
+
   unsigned DeclDependentNonTemplateCXXMethodAbbrev = 0;
   unsigned DeclTemplateCXXMethodAbbrev = 0;
   unsigned DeclMemberSpecializedCXXMethodAbbrev = 0;
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index b03e707d63874..b34e940682fc5 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -275,10 +275,6 @@ def ConversionChecker : Checker<"Conversion">,
   HelpText<"Loss of sign/precision in implicit conversions">,
   Documentation<HasDocumentation>;
 
-def IdenticalExprChecker : Checker<"IdenticalExpr">,
-  HelpText<"Warn about unintended use of identical expressions in operators">,
-  Documentation<HasDocumentation>;
-
 def FixedAddressChecker : Checker<"FixedAddr">,
   HelpText<"Check for assignment of a fixed address to a pointer">,
   Documentation<HasDocumentation>;
@@ -457,6 +453,12 @@ def CStringModeling : Checker<"CStringModeling">,
   Documentation<NotDocumented>,
   Hidden;
 
+def CStringNotNullTerm : Checker<"NotNullTerminated">,
+  HelpText<"Check for arguments passed to C string functions which are not "
+           "null-terminated strings">,
+  Dependencies<[CStringModeling]>,
+  Documentation<HasDocumentation>;
+
 def CStringNullArg : Checker<"NullArg">,
   HelpText<"Check for null pointers being passed as arguments to C string "
            "functions">,
@@ -483,11 +485,6 @@ def CStringBufferOverlap : Checker<"BufferOverlap">,
   Dependencies<[CStringModeling]>,
   Documentation<HasDocumentation>;
 
-def CStringNotNullTerm : Checker<"NotNullTerminated">,
-  HelpText<"Check for arguments which are not null-terminating strings">,
-  Dependencies<[CStringModeling]>,
-  Documentation<HasDocumentation>;
-
 def CStringUninitializedRead : Checker<"UninitializedRead">,
   HelpText<"Checks if the string manipulation function would read uninitialized bytes">,
   Dependencies<[CStringModeling]>,
@@ -598,14 +595,14 @@ def VforkChecker : Checker<"Vfork">,
   HelpText<"Check for proper usage of vfork">,
   Documentation<HasDocumentation>;
 
-} // end "unix"
-
-let ParentPackage = UnixAlpha in {
-
 def ChrootChecker : Checker<"Chroot">,
   HelpText<"Check improper use of chroot">,
   Documentation<HasDocumentation>;
 
+} // end "unix"
+
+let ParentPackage = UnixAlpha in {
+
 def PthreadLockChecker : Checker<"PthreadLock">,
   HelpText<"Simple lock -> unlock checker">,
   Dependencies<[PthreadLockBase]>,
@@ -1752,6 +1749,10 @@ def UncountedLambdaCapturesChecker : Checker<"UncountedLambdaCapturesChecker">,
 
 let ParentPackage = WebKitAlpha in {
 
+def MemoryUnsafeCastChecker : Checker<"MemoryUnsafeCastChecker">,
+  HelpText<"Check for memory unsafe casts from base type to derived type.">,
+  Documentation<HasDocumentation>;
+
 def NoUncheckedPtrMemberChecker : Checker<"NoUncheckedPtrMemberChecker">,
   HelpText<"Check for no unchecked member variables.">,
   Documentation<HasDocumentation>;
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
index 0825ecbced3f5..a6d05a3ac67b4 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
@@ -429,47 +429,27 @@ class BranchNodeBuilder: public NodeBuilder {
   const CFGBlock *DstT;
   const CFGBlock *DstF;
 
-  bool InFeasibleTrue;
-  bool InFeasibleFalse;
-
   void anchor() override;
 
 public:
   BranchNodeBuilder(ExplodedNode *SrcNode, ExplodedNodeSet &DstSet,
-                    const NodeBuilderContext &C,
-                    const CFGBlock *dstT, const CFGBlock *dstF)
-      : NodeBuilder(SrcNode, DstSet, C), DstT(dstT), DstF(dstF),
-        InFeasibleTrue(!DstT), InFeasibleFalse(!DstF) {
+                    const NodeBuilderContext &C, const CFGBlock *DT,
+                    const CFGBlock *DF)
+      : NodeBuilder(SrcNode, DstSet, C), DstT(DT), DstF(DF) {
     // The branch node builder does not generate autotransitions.
     // If there are no successors it means that both branches are infeasible.
     takeNodes(SrcNode);
   }
 
   BranchNodeBuilder(const ExplodedNodeSet &SrcSet, ExplodedNodeSet &DstSet,
-                    const NodeBuilderContext &C,
-                    const CFGBlock *dstT, const CFGBlock *dstF)
-      : NodeBuilder(SrcSet, DstSet, C), DstT(dstT), DstF(dstF),
-        InFeasibleTrue(!DstT), InFeasibleFalse(!DstF) {
+                    const NodeBuilderContext &C, const CFGBlock *DT,
+                    const CFGBlock *DF)
+      : NodeBuilder(SrcSet, DstSet, C), DstT(DT), DstF(DF) {
     takeNodes(SrcSet);
   }
 
   ExplodedNode *generateNode(ProgramStateRef State, bool branch,
                              ExplodedNode *Pred);
-
-  const CFGBlock *getTargetBlock(bool branch) const {
-    return branch ? DstT : DstF;
-  }
-
-  void markInfeasible(bool branch) {
-    if (branch)
-      InFeasibleTrue = true;
-    else
-      InFeasibleFalse = true;
-  }
-
-  bool isFeasible(bool branch) {
-    return branch ? !InFeasibleTrue : !InFeasibleFalse;
-  }
 };
 
 class IndirectGotoNodeBuilder {
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
index eef7a54f03bf1..29f534eba2a26 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
@@ -487,6 +487,7 @@ class ProgramState : public llvm::FoldingSetNode {
   friend void ProgramStateRetain(const ProgramState *state);
   friend void ProgramStateRelease(const ProgramState *state);
 
+  SVal desugarReference(SVal Val) const;
   SVal wrapSymbolicRegion(SVal Base) const;
 };
 
diff --git a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
index d167b7e45467f..fba90d8c51a66 100644
--- a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
+++ b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
@@ -123,10 +123,10 @@ struct IncludeStyle {
   /// When guessing whether a #include is the "main" include (to assign
   /// category 0, see above), use this regex of allowed suffixes to the header
   /// stem. A partial match is done, so that:
-  /// - "" means "arbitrary suffix"
-  /// - "$" means "no suffix"
+  /// * ``""`` means "arbitrary suffix"
+  /// * ``"$"`` means "no suffix"
   ///
-  /// For example, if configured to "(_test)?$", then a header a.h would be seen
+  /// For example, if configured to ``"(_test)?$"``, then a header a.h would be seen
   /// as the "main" include in both a.cc and a_test.cc.
   /// \version 3.9
   std::string IncludeIsMainRegex;
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index a03cef36294db..939235179c363 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -24,7 +24,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// API notes file minor version number.
 ///
 /// When the format changes IN ANY WAY, this number should be incremented.
-const uint16_t VERSION_MINOR = 33; // SwiftEscapable
+const uint16_t VERSION_MINOR = 34; // SwiftReturnOwnership
 
 const uint8_t kSwiftConforms = 1;
 const uint8_t kSwiftDoesNotConform = 2;
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index 45a344c13f470..fa06dffdd14b0 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -373,6 +373,13 @@ void ReadFunctionInfo(const uint8_t *&Data, FunctionInfo &Info) {
       endian::readNext<uint16_t, llvm::endianness::little>(Data);
   Info.ResultType = std::string(Data, Data + ResultTypeLen);
   Data += ResultTypeLen;
+
+  unsigned SwiftReturnOwnershipLength =
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
+  Info.SwiftReturnOwnership = std::string(reinterpret_cast<const char *>(Data),
+                                          reinterpret_cast<const char *>(Data) +
+                                              SwiftReturnOwnershipLength);
+  Data += SwiftReturnOwnershipLength;
 }
 
 /// Used to deserialize the on-disk Objective-C method table.
diff --git a/clang/lib/APINotes/APINotesTypes.cpp b/clang/lib/APINotes/APINotesTypes.cpp
index d06277fa36727..f726faa832bcc 100644
--- a/clang/lib/APINotes/APINotesTypes.cpp
+++ b/clang/lib/APINotes/APINotesTypes.cpp
@@ -77,6 +77,8 @@ LLVM_DUMP_METHOD void FunctionInfo::dump(llvm::raw_ostream &OS) const {
      << "RawRetainCountConvention: " << RawRetainCountConvention << ' ';
   if (!ResultType.empty())
     OS << "Result Type: " << ResultType << ' ';
+  if (!SwiftReturnOwnership.empty())
+    OS << "SwiftReturnOwnership: " << SwiftReturnOwnership << ' ';
   if (!Params.empty())
     OS << '\n';
   for (auto &PI : Params)
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index 480e1190358d4..1aae07bbdd30e 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -1093,6 +1093,7 @@ unsigned getFunctionInfoSize(const FunctionInfo &FI) {
   for (const auto &P : FI.Params)
     size += getParamInfoSize(P);
   size += sizeof(uint16_t) + FI.ResultType.size();
+  size += sizeof(uint16_t) + FI.SwiftReturnOwnership.size();
   return size;
 }
 
@@ -1118,6 +1119,9 @@ void emitFunctionInfo(raw_ostream &OS, const FunctionInfo &FI) {
 
   writer.write<uint16_t>(FI.ResultType.size());
   writer.write(ArrayRef<char>{FI.ResultType.data(), FI.ResultType.size()});
+  writer.write<uint16_t>(FI.SwiftReturnOwnership.size());
+  writer.write(ArrayRef<char>{FI.SwiftReturnOwnership.data(),
+                              FI.SwiftReturnOwnership.size()});
 }
 
 /// Used to serialize the on-disk global function table.
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 0668dda910f2a..414a59a4f12d0 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -162,6 +162,7 @@ struct Method {
   bool DesignatedInit = false;
   bool Required = false;
   StringRef ResultType;
+  StringRef SwiftReturnOwnership;
 };
 
 typedef std::vector<Method> MethodsSeq;
@@ -196,6 +197,8 @@ template <> struct MappingTraits<Method> {
     IO.mapOptional("DesignatedInit", M.DesignatedInit, false);
     IO.mapOptional("Required", M.Required, false);
     IO.mapOptional("ResultType", M.ResultType, StringRef(""));
+    IO.mapOptional("SwiftReturnOwnership", M.SwiftReturnOwnership,
+                   StringRef(""));
   }
 };
 } // namespace yaml
@@ -291,6 +294,7 @@ struct Function {
   StringRef SwiftName;
   StringRef Type;
   StringRef ResultType;
+  StringRef SwiftReturnOwnership;
 };
 
 typedef std::vector<Function> FunctionsSeq;
@@ -313,6 +317,8 @@ template <> struct MappingTraits<Function> {
     IO.mapOptional("SwiftPrivate", F.SwiftPrivate);
     IO.mapOptional("SwiftName", F.SwiftName, StringRef(""));
     IO.mapOptional("ResultType", F.ResultType, StringRef(""));
+    IO.mapOptional("SwiftReturnOwnership", F.SwiftReturnOwnership,
+                   StringRef(""));
   }
 };
 } // namespace yaml
@@ -825,6 +831,7 @@ class YAMLConverter {
       emitError("'FactoryAsInit' is no longer valid; use 'SwiftName' instead");
 
     MI.ResultType = std::string(M.ResultType);
+    MI.SwiftReturnOwnership = std::string(M.SwiftReturnOwnership);
 
     // Translate parameter information.
     convertParams(M.Params, MI, MI.Self);
@@ -950,6 +957,7 @@ class YAMLConverter {
     convertNullability(Function.Nullability, Function.NullabilityOfRet, FI,
                        Function.Name);
     FI.ResultType = std::string(Function.ResultType);
+    FI.SwiftReturnOwnership = std::string(Function.SwiftReturnOwnership);
     FI.setRetainCountConvention(Function.RetainCountConvention);
   }
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 4a8ef1402d145..68ecdf471f32f 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -112,6 +112,27 @@ enum FloatingRank {
   Ibm128Rank
 };
 
+template <> struct llvm::DenseMapInfo<llvm::FoldingSetNodeID> {
+  static FoldingSetNodeID getEmptyKey() { return FoldingSetNodeID{}; }
+
+  static FoldingSetNodeID getTombstoneKey() {
+    FoldingSetNodeID id;
+    for (size_t i = 0; i < sizeof(id) / sizeof(unsigned); ++i) {
+      id.AddInteger(std::numeric_limits<unsigned>::max());
+    }
+    return id;
+  }
+
+  static unsigned getHashValue(const FoldingSetNodeID &Val) {
+    return Val.ComputeHash();
+  }
+
+  static bool isEqual(const FoldingSetNodeID &LHS,
+                      const FoldingSetNodeID &RHS) {
+    return LHS == RHS;
+  }
+};
+
 /// \returns The locations that are relevant when searching for Doc comments
 /// related to \p D.
 static SmallVector<SourceLocation, 2>
@@ -899,7 +920,7 @@ ASTContext::ASTContext(LangOptions &LOpts, SourceManager &SM,
       FunctionProtoTypes(this_(), FunctionProtoTypesLog2InitSize),
       DependentTypeOfExprTypes(this_()), DependentDecltypeTypes(this_()),
       TemplateSpecializationTypes(this_()),
-      DependentTemplateSpecializationTypes(this_()), AutoTypes(this_()),
+      DependentTemplateSpecializationTypes(this_()),
       DependentBitIntTypes(this_()), SubstTemplateTemplateParmPacks(this_()),
       DeducedTemplates(this_()), ArrayParameterTypes(this_()),
       CanonTemplateTemplateParms(this_()), SourceMgr(SM), LangOpts(LOpts),
@@ -6306,12 +6327,14 @@ QualType ASTContext::getAutoTypeInternal(
     return getAutoDeductType();
 
   // Look in the folding set for an existing type.
-  void *InsertPos = nullptr;
   llvm::FoldingSetNodeID ID;
-  AutoType::Profile(ID, *this, DeducedType, Keyword, IsDependent,
-                    TypeConstraintConcept, TypeConstraintArgs);
-  if (AutoType *AT = AutoTypes.FindNodeOrInsertPos(ID, InsertPos))
-    return QualType(AT, 0);
+  bool IsDeducedDependent =
+      !DeducedType.isNull() && DeducedType->isDependentType();
+  AutoType::Profile(ID, *this, DeducedType, Keyword,
+                    IsDependent || IsDeducedDependent, TypeConstraintConcept,
+                    TypeConstraintArgs);
+  if (auto const AT_iter = AutoTypes.find(ID); AT_iter != AutoTypes.end())
+    return QualType(AT_iter->getSecond(), 0);
 
   QualType Canon;
   if (!IsCanon) {
@@ -6326,10 +6349,6 @@ QualType ASTContext::getAutoTypeInternal(
         Canon =
             getAutoTypeInternal(QualType(), Keyword, IsDependent, IsPack,
                                 CanonicalConcept, CanonicalConceptArgs, true);
-        // Find the insert position again.
-        [[maybe_unused]] auto *Nothing =
-            AutoTypes.FindNodeOrInsertPos(ID, InsertPos);
-        assert(!Nothing && "canonical type broken");
       }
     }
   }
@@ -6343,8 +6362,13 @@ QualType ASTContext::getAutoTypeInternal(
                    : TypeDependence::None) |
           (IsPack ? TypeDependence::UnexpandedPack : TypeDependence::None),
       Canon, TypeConstraintConcept, TypeConstraintArgs);
+#ifndef NDEBUG
+  llvm::FoldingSetNodeID InsertedID;
+  AT->Profile(InsertedID, *this);
+  assert(InsertedID == ID && "ID does not match");
+#endif
   Types.push_back(AT);
-  AutoTypes.InsertNode(AT, InsertPos);
+  AutoTypes.try_emplace(ID, AT);
   return QualType(AT, 0);
 }
 
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 4f677b60e60da..7b873ee9833b3 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -1899,11 +1899,17 @@ class TemplateDiff {
 
     E = E->IgnoreImpCasts();
 
-    if (isa<IntegerLiteral>(E)) return false;
+    auto CheckIntegerLiteral = [](Expr *E) {
+      if (auto *TemplateExpr = dyn_cast<SubstNonTypeTemplateParmExpr>(E))
+        E = TemplateExpr->getReplacement();
+      return isa<IntegerLiteral>(E);
+    };
+
+    if (CheckIntegerLiteral(E)) return false;
 
     if (UnaryOperator *UO = dyn_cast<UnaryOperator>(E))
       if (UO->getOpcode() == UO_Minus)
-        if (isa<IntegerLiteral>(UO->getSubExpr()))
+        if (CheckIntegerLiteral(UO->getSubExpr()))
           return false;
 
     if (isa<CXXBoolLiteralExpr>(E))
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index d371a5a01a0b9..cff8e87e5d510 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -6099,8 +6099,7 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
                                               Decl::IDNS_TagFriend))
         continue;
 
-      Decl *Found = FoundDecl;
-      auto *FoundTemplate = dyn_cast<ClassTemplateDecl>(Found);
+      auto *FoundTemplate = dyn_cast<ClassTemplateDecl>(FoundDecl);
       if (FoundTemplate) {
         if (!hasSameVisibilityContextAndLinkage(FoundTemplate, D))
           continue;
@@ -6124,6 +6123,19 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
           // see ASTTests test ImportExistingFriendClassTemplateDef.
           continue;
         }
+        // When importing a friend, it is possible that multiple declarations
+        // with same name can co-exist in specific cases (if a template contains
+        // a friend template and has a specialization). For this case the
+        // declarations should match, except that the "template depth" is
+        // different. No linking of previous declaration is needed in this case.
+        // FIXME: This condition may need refinement.
+        if (D->getFriendObjectKind() != Decl::FOK_None &&
+            FoundTemplate->getFriendObjectKind() != Decl::FOK_None &&
+            D->getFriendObjectKind() != FoundTemplate->getFriendObjectKind() &&
+            IsStructuralMatch(D, FoundTemplate, /*Complain=*/false,
+                              /*IgnoreTemplateParmDepth=*/true))
+          continue;
+
         ConflictingDecls.push_back(FoundDecl);
       }
     }
diff --git a/clang/lib/AST/ByteCode/BitcastBuffer.cpp b/clang/lib/AST/ByteCode/BitcastBuffer.cpp
new file mode 100644
index 0000000000000..fbd500fd8f5f4
--- /dev/null
+++ b/clang/lib/AST/ByteCode/BitcastBuffer.cpp
@@ -0,0 +1,170 @@
+//===-------------------- Bitcastbuffer.cpp ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "BitcastBuffer.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace clang;
+using namespace clang::interp;
+
+/// Returns the value of the bit in the given sequence of bytes.
+static inline bool bitof(const std::byte *B, Bits BitIndex) {
+  return (B[BitIndex.roundToBytes()] &
+          (std::byte{1} << BitIndex.getOffsetInByte())) != std::byte{0};
+}
+
+void BitcastBuffer::pushData(const std::byte *In, Bits BitOffset, Bits BitWidth,
+                             Endian TargetEndianness) {
+  for (unsigned It = 0; It != BitWidth.getQuantity(); ++It) {
+    bool BitValue = bitof(In, Bits(It));
+    if (!BitValue)
+      continue;
+
+    Bits DstBit;
+    if (TargetEndianness == Endian::Little)
+      DstBit = BitOffset + Bits(It);
+    else
+      DstBit = size() - BitOffset - BitWidth + Bits(It);
+
+    size_t DstByte = DstBit.roundToBytes();
+    Data[DstByte] |= std::byte{1} << DstBit.getOffsetInByte();
+  }
+}
+
+std::unique_ptr<std::byte[]>
+BitcastBuffer::copyBits(Bits BitOffset, Bits BitWidth, Bits FullBitWidth,
+                        Endian TargetEndianness) const {
+  assert(BitWidth.getQuantity() <= FullBitWidth.getQuantity());
+  assert(FullBitWidth.isFullByte());
+  auto Out = std::make_unique<std::byte[]>(FullBitWidth.roundToBytes());
+
+  for (unsigned It = 0; It != BitWidth.getQuantity(); ++It) {
+    Bits BitIndex;
+    if (TargetEndianness == Endian::Little)
+      BitIndex = BitOffset + Bits(It);
+    else
+      BitIndex = size() - BitWidth - BitOffset + Bits(It);
+
+    bool BitValue = bitof(Data.get(), BitIndex);
+    if (!BitValue)
+      continue;
+
+    Bits DstBit = Bits(It);
+    size_t DstByte = DstBit.roundToBytes();
+    Out[DstByte] |= std::byte{1} << DstBit.getOffsetInByte();
+  }
+
+  return Out;
+}
+
+bool BitcastBuffer::allInitialized() const {
+  return rangeInitialized(Bits::zero(), FinalBitSize);
+}
+
+void BitcastBuffer::markInitialized(Bits Offset, Bits Length) {
+  if (Length.isZero())
+    return;
+
+  BitRange Element(Offset, Offset + Length - Bits(1));
+  if (InitializedBits.empty()) {
+    InitializedBits.push_back(Element);
+    return;
+  }
+
+  assert(InitializedBits.size() >= 1);
+  // Common case of just appending.
+  Bits End = InitializedBits.back().End;
+  if (End <= Offset) {
+    // Merge this range with the last one.
+    // In the best-case scenario, this means we only ever have
+    // one single bit range covering all bits.
+    if (End == (Offset - Bits(1))) {
+      InitializedBits.back().End = Element.End;
+      return;
+    }
+
+    // Otherwise, we can simply append.
+    InitializedBits.push_back(Element);
+  } else {
+    // Insert sorted.
+    auto It = std::upper_bound(InitializedBits.begin(), InitializedBits.end(),
+                               Element);
+    InitializedBits.insert(It, Element);
+  }
+
+#ifndef NDEBUG
+  // Ensure ranges are sorted and non-overlapping.
+  assert(llvm::is_sorted(InitializedBits));
+  for (unsigned I = 1; I != InitializedBits.size(); ++I) {
+    [[maybe_unused]] auto Prev = InitializedBits[I - 1];
+    [[maybe_unused]] auto Cur = InitializedBits[I];
+    assert(Prev.End.N < Cur.Start.N);
+  }
+#endif
+}
+
+bool BitcastBuffer::rangeInitialized(Bits Offset, Bits Length) const {
+  if (Length.isZero())
+    return true;
+
+  BitRange Range(Offset, Offset + Length - Bits(1));
+  Bits Sum;
+  bool FoundStart = false;
+  for (BitRange BR : InitializedBits) {
+    if (FoundStart) {
+      if (BR.contains(Range.End)) {
+        Sum += (Range.End - BR.Start + Bits(1));
+        break;
+      }
+
+      // Else, BR is completely inside Range.
+      Sum += BR.size();
+    }
+    if (BR.contains(Range.Start)) {
+      Sum += (BR.End - Range.Start + Bits(1));
+      FoundStart = true;
+    }
+  }
+
+  // Note that Sum can be larger than Range, e.g. when Range is fully
+  // contained in one range.
+  return Sum >= Range.size();
+}
+
+#if 0
+  template<typename T>
+  static std::string hex(T t) {
+    std::stringstream stream;
+    stream << std::hex << (int)t;
+    return std::string(stream.str());
+  }
+
+
+  void BitcastBuffer::dump(bool AsHex = true) const {
+    llvm::errs() << "LSB\n  ";
+    unsigned LineLength = 0;
+    for (unsigned I = 0; I != (FinalBitSize / 8); ++I) {
+      std::byte B = Data[I];
+      if (AsHex) {
+        std::stringstream stream;
+        stream << std::hex << (int)B;
+        llvm::errs() << stream.str();
+        LineLength += stream.str().size() + 1;
+      } else {
+        llvm::errs() << std::bitset<8>((int)B).to_string();
+        LineLength += 8 + 1;
+        // llvm::errs() << (int)B;
+      }
+      llvm::errs() << ' ';
+    }
+    llvm::errs() << '\n';
+
+    for (unsigned I = 0; I != LineLength; ++I)
+      llvm::errs() << ' ';
+    llvm::errs() << "MSB\n";
+  }
+#endif
diff --git a/clang/lib/AST/ByteCode/BitcastBuffer.h b/clang/lib/AST/ByteCode/BitcastBuffer.h
new file mode 100644
index 0000000000000..b1b6b9e5173a7
--- /dev/null
+++ b/clang/lib/AST/ByteCode/BitcastBuffer.h
@@ -0,0 +1,111 @@
+//===--------------------- BitcastBuffer.h ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_AST_INTERP_BITCAST_BUFFER_H
+#define LLVM_CLANG_AST_INTERP_BITCAST_BUFFER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include <cassert>
+#include <cstddef>
+#include <memory>
+
+namespace clang {
+namespace interp {
+
+enum class Endian { Little, Big };
+
+/// A quantity in bits.
+struct Bits {
+  size_t N = 0;
+  Bits() = default;
+  static Bits zero() { return Bits(0); }
+  explicit Bits(size_t Quantity) : N(Quantity) {}
+  size_t getQuantity() const { return N; }
+  size_t roundToBytes() const { return N / 8; }
+  size_t getOffsetInByte() const { return N % 8; }
+  bool isFullByte() const { return N % 8 == 0; }
+  bool nonZero() const { return N != 0; }
+  bool isZero() const { return N == 0; }
+
+  Bits operator-(Bits Other) const { return Bits(N - Other.N); }
+  Bits operator+(Bits Other) const { return Bits(N + Other.N); }
+  Bits operator+=(size_t O) {
+    N += O;
+    return *this;
+  }
+  Bits operator+=(Bits O) {
+    N += O.N;
+    return *this;
+  }
+
+  bool operator>=(Bits Other) const { return N >= Other.N; }
+  bool operator<=(Bits Other) const { return N <= Other.N; }
+  bool operator==(Bits Other) const { return N == Other.N; }
+  bool operator!=(Bits Other) const { return N != Other.N; }
+};
+
+/// A quantity in bytes.
+struct Bytes {
+  size_t N;
+  explicit Bytes(size_t Quantity) : N(Quantity) {}
+  size_t getQuantity() const { return N; }
+  Bits toBits() const { return Bits(N * 8); }
+};
+
+/// A bit range. Both Start and End are inclusive.
+struct BitRange {
+  Bits Start;
+  Bits End;
+
+  BitRange(Bits Start, Bits End) : Start(Start), End(End) {}
+  Bits size() const { return End - Start + Bits(1); }
+  bool operator<(BitRange Other) const { return Start.N < Other.Start.N; }
+
+  bool contains(Bits B) { return Start <= B && End >= B; }
+};
+
+/// Track what bits have been initialized to known values and which ones
+/// have indeterminate value.
+struct BitcastBuffer {
+  Bits FinalBitSize;
+  std::unique_ptr<std::byte[]> Data;
+  llvm::SmallVector<BitRange> InitializedBits;
+
+  BitcastBuffer(Bits FinalBitSize) : FinalBitSize(FinalBitSize) {
+    assert(FinalBitSize.isFullByte());
+    unsigned ByteSize = FinalBitSize.roundToBytes();
+    Data = std::make_unique<std::byte[]>(ByteSize);
+  }
+
+  /// Returns the buffer size in bits.
+  Bits size() const { return FinalBitSize; }
+
+  /// Returns \c true if all bits in the buffer have been initialized.
+  bool allInitialized() const;
+  /// Marks the bits in the given range as initialized.
+  /// FIXME: Can we do this automatically in pushData()?
+  void markInitialized(Bits Start, Bits Length);
+  bool rangeInitialized(Bits Offset, Bits Length) const;
+
+  /// Push \p BitWidth bits at \p BitOffset from \p In into the buffer.
+  /// \p TargetEndianness is the endianness of the target we're compiling for.
+  /// \p In must hold at least \p BitWidth many bits.
+  void pushData(const std::byte *In, Bits BitOffset, Bits BitWidth,
+                Endian TargetEndianness);
+
+  /// Copy \p BitWidth bits at offset \p BitOffset from the buffer.
+  /// \p TargetEndianness is the endianness of the target we're compiling for.
+  ///
+  /// The returned output holds exactly (\p FullBitWidth / 8) bytes.
+  std::unique_ptr<std::byte[]> copyBits(Bits BitOffset, Bits BitWidth,
+                                        Bits FullBitWidth,
+                                        Endian TargetEndianness) const;
+};
+
+} // namespace interp
+} // namespace clang
+#endif
diff --git a/clang/lib/AST/ByteCode/Boolean.h b/clang/lib/AST/ByteCode/Boolean.h
index 78d75e75c7531..8380e85865ac5 100644
--- a/clang/lib/AST/ByteCode/Boolean.h
+++ b/clang/lib/AST/ByteCode/Boolean.h
@@ -82,9 +82,7 @@ class Boolean final {
   Boolean truncate(unsigned TruncBits) const { return *this; }
 
   static Boolean bitcastFromMemory(const std::byte *Buff, unsigned BitWidth) {
-    // Boolean width is currently always 8 for all supported targets. If this
-    // changes we need to get the bool width from the target info.
-    assert(BitWidth == 8);
+    // Just load the first byte.
     bool Val = static_cast<bool>(*Buff);
     return Boolean(Val);
   }
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index f4cc284dfb6ab..7f6295e126dcf 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -355,7 +355,9 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
                                   std::nullopt, true, false,
                                   /*IsMutable=*/false, nullptr);
     }
-    return this->emitNull(classifyPrim(CE->getType()), Desc, CE);
+
+    uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(CE->getType());
+    return this->emitNull(classifyPrim(CE->getType()), Val, Desc, CE);
   }
 
   case CK_PointerToIntegral: {
@@ -448,8 +450,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
     QualType SubExprTy = SubExpr->getType();
     std::optional<PrimType> FromT = classify(SubExprTy);
-    // Casts from integer to vectors in C.
-    if (FromT && CE->getType()->isVectorType())
+    // Casts from integer/vector to vector.
+    if (CE->getType()->isVectorType())
       return this->emitBuiltinBitCast(CE);
 
     std::optional<PrimType> ToT = classify(CE->getType());
@@ -1000,7 +1002,10 @@ bool Compiler<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
     if (!visitAsPointer(RHS, *RT) || !visitAsPointer(LHS, *LT))
       return false;
 
-    return this->emitSubPtr(classifyPrim(E->getType()), E);
+    PrimType IntT = classifyPrim(E->getType());
+    if (!this->emitSubPtr(IntT, E))
+      return false;
+    return DiscardResult ? this->emitPop(IntT, E) : true;
   }
 
   PrimType OffsetType;
@@ -3814,7 +3819,7 @@ template <class Emitter> bool Compiler<Emitter>::visitBool(const Expr *E) {
 
   // Convert pointers to bool.
   if (T == PT_Ptr || T == PT_FnPtr) {
-    if (!this->emitNull(*T, nullptr, E))
+    if (!this->emitNull(*T, 0, nullptr, E))
       return false;
     return this->emitNE(*T, E);
   }
@@ -3854,11 +3859,12 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
   case PT_IntAPS:
     return this->emitZeroIntAPS(Ctx.getBitWidth(QT), E);
   case PT_Ptr:
-    return this->emitNullPtr(nullptr, E);
+    return this->emitNullPtr(Ctx.getASTContext().getTargetNullPointerValue(QT),
+                             nullptr, E);
   case PT_FnPtr:
-    return this->emitNullFnPtr(nullptr, E);
+    return this->emitNullFnPtr(0, nullptr, E);
   case PT_MemberPtr:
-    return this->emitNullMemberPtr(nullptr, E);
+    return this->emitNullMemberPtr(0, nullptr, E);
   case PT_Float:
     return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
   case PT_FixedPoint: {
@@ -4418,7 +4424,7 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
 
   if (Val.isLValue()) {
     if (Val.isNullPointer())
-      return this->emitNull(ValType, nullptr, E);
+      return this->emitNull(ValType, 0, nullptr, E);
     APValue::LValueBase Base = Val.getLValueBase();
     if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
       return this->visit(BaseExpr);
@@ -4428,7 +4434,7 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
   } else if (Val.isMemberPointer()) {
     if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl())
       return this->emitGetMemberPtr(MemberDecl, E);
-    return this->emitNullMemberPtr(nullptr, E);
+    return this->emitNullMemberPtr(0, nullptr, E);
   }
 
   return false;
@@ -4780,7 +4786,8 @@ bool Compiler<Emitter>::VisitCXXNullPtrLiteralExpr(
   if (DiscardResult)
     return true;
 
-  return this->emitNullPtr(nullptr, E);
+  uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(E->getType());
+  return this->emitNullPtr(Val, nullptr, E);
 }
 
 template <class Emitter>
@@ -5330,7 +5337,7 @@ bool Compiler<Emitter>::emitLambdaStaticInvokerBody(const CXXMethodDecl *MD) {
   // one here, and we don't need one either because the lambda cannot have
   // any captures, as verified above. Emit a null pointer. This is then
   // special-cased when interpreting to not emit any misleading diagnostics.
-  if (!this->emitNullPtr(nullptr, MD))
+  if (!this->emitNullPtr(0, nullptr, MD))
     return false;
 
   // Forward all arguments from the static invoker to the lambda call operator.
@@ -5911,6 +5918,9 @@ bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
     return this->discard(SubExpr);
 
   auto UnaryOp = E->getOpcode();
+  if (UnaryOp == UO_Extension)
+    return this->delegate(SubExpr);
+
   if (UnaryOp != UO_Plus && UnaryOp != UO_Minus && UnaryOp != UO_LNot &&
       UnaryOp != UO_Not && UnaryOp != UO_AddrOf)
     return this->emitInvalid(E);
@@ -6473,19 +6483,14 @@ bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
   QualType ToType = E->getType();
   std::optional<PrimType> ToT = classify(ToType);
 
+  // Bitcasting TO nullptr_t is always fine.
   if (ToType->isNullPtrType()) {
     if (!this->discard(SubExpr))
       return false;
 
-    return this->emitNullPtr(nullptr, E);
+    return this->emitNullPtr(0, nullptr, E);
   }
 
-  if (FromType->isNullPtrType() && ToT) {
-    if (!this->discard(SubExpr))
-      return false;
-
-    return visitZeroInitializer(*ToT, ToType, E);
-  }
   assert(!ToType->isReferenceType());
 
   // Prepare storage for the result in case we discard.
@@ -6502,7 +6507,7 @@ bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
   // we later assume it to be one (i.e. a PT_Ptr). However,
   // we call this function for other utility methods where
   // a bitcast might be useful, so convert it to a PT_Ptr in that case.
-  if (SubExpr->isGLValue()) {
+  if (SubExpr->isGLValue() || FromType->isVectorType()) {
     if (!this->visit(SubExpr))
       return false;
   } else if (std::optional<PrimType> FromT = classify(SubExpr)) {
@@ -6514,6 +6519,8 @@ bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
       return false;
     if (!this->emitGetPtrLocal(TempOffset, E))
       return false;
+  } else {
+    return false;
   }
 
   if (!ToT || ToT == PT_Ptr) {
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index 7088cf02901c6..b52892cf69bfb 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -34,8 +34,7 @@ bool Context::isPotentialConstantExpr(State &Parent, const FunctionDecl *FD) {
   if (!Func)
     return false;
 
-  APValue DummyResult;
-  if (!Run(Parent, Func, DummyResult))
+  if (!Run(Parent, Func))
     return false;
 
   return Func->isConstexpr();
@@ -213,13 +212,13 @@ const llvm::fltSemantics &Context::getFloatSemantics(QualType T) const {
   return Ctx.getFloatTypeSemantics(T);
 }
 
-bool Context::Run(State &Parent, const Function *Func, APValue &Result) {
+bool Context::Run(State &Parent, const Function *Func) {
 
   {
     InterpState State(Parent, *P, Stk, *this);
     State.Current = new InterpFrame(State, Func, /*Caller=*/nullptr, CodePtr(),
                                     Func->getArgSize());
-    if (Interpret(State, Result)) {
+    if (Interpret(State)) {
       assert(Stk.empty());
       return true;
     }
diff --git a/clang/lib/AST/ByteCode/Context.h b/clang/lib/AST/ByteCode/Context.h
index e0d4bafdebaf2..ed539def99efd 100644
--- a/clang/lib/AST/ByteCode/Context.h
+++ b/clang/lib/AST/ByteCode/Context.h
@@ -114,7 +114,7 @@ class Context final {
 
 private:
   /// Runs a function.
-  bool Run(State &Parent, const Function *Func, APValue &Result);
+  bool Run(State &Parent, const Function *Func);
 
   /// Current compilation context.
   ASTContext &Ctx;
diff --git a/clang/lib/AST/ByteCode/Integral.h b/clang/lib/AST/ByteCode/Integral.h
index ca3674263aef4..26585799e5ead 100644
--- a/clang/lib/AST/ByteCode/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -123,7 +123,9 @@ template <unsigned Bits, bool Signed> class Integral final {
   APSInt toAPSInt() const {
     return APSInt(APInt(Bits, static_cast<uint64_t>(V), Signed), !Signed);
   }
-  APSInt toAPSInt(unsigned BitWidth) const { return APSInt(toAPInt(BitWidth)); }
+  APSInt toAPSInt(unsigned BitWidth) const {
+    return APSInt(toAPInt(BitWidth), !Signed);
+  }
   APInt toAPInt(unsigned BitWidth) const {
     if constexpr (Signed)
       return APInt(Bits, static_cast<uint64_t>(V), Signed)
@@ -181,6 +183,7 @@ template <unsigned Bits, bool Signed> class Integral final {
   }
 
   Integral truncate(unsigned TruncBits) const {
+    assert(TruncBits >= 1);
     if (TruncBits >= Bits)
       return *this;
     const ReprT BitMask = (ReprT(1) << ReprT(TruncBits)) - 1;
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 329f1584be34b..7c7752080746e 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -27,7 +27,7 @@
 using namespace clang;
 using namespace clang::interp;
 
-static bool RetValue(InterpState &S, CodePtr &Pt, APValue &Result) {
+static bool RetValue(InterpState &S, CodePtr &Pt) {
   llvm::report_fatal_error("Interpreter cannot return values");
 }
 
@@ -1205,11 +1205,10 @@ bool CallVar(InterpState &S, CodePtr OpPC, const Function *Func,
   InterpFrame *FrameBefore = S.Current;
   S.Current = NewFrame.get();
 
-  APValue CallResult;
   // Note that we cannot assert(CallResult.hasValue()) here since
   // Ret() above only sets the APValue if the curent frame doesn't
   // have a caller set.
-  if (Interpret(S, CallResult)) {
+  if (Interpret(S)) {
     NewFrame.release(); // Frame was delete'd already.
     assert(S.Current == FrameBefore);
     return true;
@@ -1270,11 +1269,10 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
   S.Current = NewFrame.get();
 
   InterpStateCCOverride CCOverride(S, Func->getDecl()->isImmediateFunction());
-  APValue CallResult;
   // Note that we cannot assert(CallResult.hasValue()) here since
   // Ret() above only sets the APValue if the curent frame doesn't
   // have a caller set.
-  if (Interpret(S, CallResult)) {
+  if (Interpret(S)) {
     NewFrame.release(); // Frame was delete'd already.
     assert(S.Current == FrameBefore);
     return true;
@@ -1362,7 +1360,10 @@ bool CallVirt(InterpState &S, CodePtr OpPC, const Function *Func,
 
 bool CallBI(InterpState &S, CodePtr OpPC, const Function *Func,
             const CallExpr *CE, uint32_t BuiltinID) {
-  if (S.checkingPotentialConstantExpression())
+  // A little arbitrary, but the current interpreter allows evaluation
+  // of builtin functions in this mode, with some exceptions.
+  if (BuiltinID == Builtin::BI__builtin_operator_new &&
+      S.checkingPotentialConstantExpression())
     return false;
   auto NewFrame = std::make_unique<InterpFrame>(S, Func, OpPC);
 
@@ -1598,7 +1599,7 @@ bool CheckBitCast(InterpState &S, CodePtr OpPC, bool HasIndeterminateBits,
 #if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
 #pragma optimize("", off)
 #endif
-bool Interpret(InterpState &S, APValue &Result) {
+bool Interpret(InterpState &S) {
   // The current stack frame when we started Interpret().
   // This is being used by the ops to determine wheter
   // to return from this function and thus terminate
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 47dcfca79f735..cdf05e36304ac 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_AST_INTERP_INTERP_H
 
 #include "../ExprConstShared.h"
+#include "BitcastBuffer.h"
 #include "Boolean.h"
 #include "DynamicAllocator.h"
 #include "FixedPoint.h"
@@ -41,13 +42,6 @@ namespace interp {
 using APSInt = llvm::APSInt;
 using FixedPointSemantics = llvm::FixedPointSemantics;
 
-/// Convert a value to an APValue.
-template <typename T>
-bool ReturnValue(const InterpState &S, const T &V, APValue &R) {
-  R = V.toAPValue(S.getASTContext());
-  return true;
-}
-
 /// Checks if the variable has externally defined storage.
 bool CheckExtern(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
 
@@ -299,7 +293,7 @@ bool CheckFloatResult(InterpState &S, CodePtr OpPC, const Floating &Result,
 bool CheckDeclRef(InterpState &S, CodePtr OpPC, const DeclRefExpr *DR);
 
 /// Interpreter entry point.
-bool Interpret(InterpState &S, APValue &Result);
+bool Interpret(InterpState &S);
 
 /// Interpret a builtin function.
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
@@ -321,7 +315,7 @@ void cleanupAfterFunctionCall(InterpState &S, CodePtr OpPC,
                               const Function *Func);
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
-bool Ret(InterpState &S, CodePtr &PC, APValue &Result) {
+bool Ret(InterpState &S, CodePtr &PC) {
   const T &Ret = S.Stk.pop<T>();
 
   // Make sure returned pointers are live. We might be trying to return a
@@ -349,13 +343,13 @@ bool Ret(InterpState &S, CodePtr &PC, APValue &Result) {
   } else {
     delete S.Current;
     S.Current = nullptr;
-    if (!ReturnValue<T>(S, Ret, Result))
-      return false;
+    // The topmost frame should come from an EvalEmitter,
+    // which has its own implementation of the Ret<> instruction.
   }
   return true;
 }
 
-inline bool RetVoid(InterpState &S, CodePtr &PC, APValue &Result) {
+inline bool RetVoid(InterpState &S, CodePtr &PC) {
   assert(S.Current->getFrameOffset() == S.Stk.size() && "Invalid frame");
 
   if (!S.checkingPotentialConstantExpression() || S.Current->Caller)
@@ -2439,9 +2433,11 @@ static inline bool ZeroIntAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
-inline bool Null(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
-  // Note: Desc can be null.
-  S.Stk.push<T>(0, Desc);
+inline bool Null(InterpState &S, CodePtr OpPC, uint64_t Value,
+                 const Descriptor *Desc) {
+  // FIXME(perf): This is a somewhat often-used function and the value of a
+  // null pointer is almost always 0.
+  S.Stk.push<T>(Value, Desc);
   return true;
 }
 
@@ -3055,7 +3051,16 @@ inline bool BitCast(InterpState &S, CodePtr OpPC, bool TargetIsUCharOrByte,
   llvm::SmallVector<std::byte> Buff(BuffSize);
   bool HasIndeterminateBits = false;
 
-  if (!DoBitCast(S, OpPC, FromPtr, Buff.data(), BuffSize, HasIndeterminateBits))
+  Bits FullBitWidth(ResultBitWidth);
+  Bits BitWidth = FullBitWidth;
+
+  if constexpr (std::is_same_v<T, Floating>) {
+    assert(Sem);
+    BitWidth = Bits(llvm::APFloatBase::getSizeInBits(*Sem));
+  }
+
+  if (!DoBitCast(S, OpPC, FromPtr, Buff.data(), BitWidth, FullBitWidth,
+                 HasIndeterminateBits))
     return false;
 
   if (!CheckBitCast(S, OpPC, HasIndeterminateBits, TargetIsUCharOrByte))
@@ -3063,16 +3068,7 @@ inline bool BitCast(InterpState &S, CodePtr OpPC, bool TargetIsUCharOrByte,
 
   if constexpr (std::is_same_v<T, Floating>) {
     assert(Sem);
-    ptrdiff_t Offset = 0;
-
-    if (llvm::sys::IsBigEndianHost) {
-      unsigned NumBits = llvm::APFloatBase::getSizeInBits(*Sem);
-      assert(NumBits % 8 == 0);
-      assert(NumBits <= ResultBitWidth);
-      Offset = (ResultBitWidth - NumBits) / 8;
-    }
-
-    S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data() + Offset, *Sem));
+    S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
   } else {
     assert(!Sem);
     S.Stk.push<T>(T::bitcastFromMemory(Buff.data(), ResultBitWidth));
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index b450d8263c30b..a0de193ec32a2 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -89,13 +89,14 @@ static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
   std::optional<PrimType> T = S.getContext().classify(QT);
   assert(T);
 
+  unsigned BitWidth = S.getASTContext().getTypeSize(QT);
   if (QT->isSignedIntegerOrEnumerationType()) {
     int64_t V = Val.getSExtValue();
-    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
+    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
   } else {
     assert(QT->isUnsignedIntegerOrEnumerationType());
     uint64_t V = Val.getZExtValue();
-    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
+    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
   }
 }
 
@@ -103,6 +104,8 @@ template <typename T>
 static void pushInteger(InterpState &S, T Val, QualType QT) {
   if constexpr (std::is_same_v<T, APInt>)
     pushInteger(S, APSInt(Val, !std::is_signed_v<T>), QT);
+  else if constexpr (std::is_same_v<T, APSInt>)
+    pushInteger(S, Val, QT);
   else
     pushInteger(S,
                 APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
@@ -116,14 +119,14 @@ static void assignInteger(Pointer &Dest, PrimType ValueT, const APSInt &Value) {
       ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
 }
 
-static bool retPrimValue(InterpState &S, CodePtr OpPC, APValue &Result,
+static bool retPrimValue(InterpState &S, CodePtr OpPC,
                          std::optional<PrimType> &T) {
   if (!T)
-    return RetVoid(S, OpPC, Result);
+    return RetVoid(S, OpPC);
 
 #define RET_CASE(X)                                                            \
   case X:                                                                      \
-    return Ret<X>(S, OpPC, Result);
+    return Ret<X>(S, OpPC);
   switch (*T) {
     RET_CASE(PT_Ptr);
     RET_CASE(PT_FnPtr);
@@ -137,12 +140,25 @@ static bool retPrimValue(InterpState &S, CodePtr OpPC, APValue &Result,
     RET_CASE(PT_Uint32);
     RET_CASE(PT_Sint64);
     RET_CASE(PT_Uint64);
+    RET_CASE(PT_IntAP);
+    RET_CASE(PT_IntAPS);
   default:
     llvm_unreachable("Unsupported return type for builtin function");
   }
 #undef RET_CASE
 }
 
+static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
+                                        unsigned ID) {
+  auto Loc = S.Current->getSource(OpPC);
+  if (S.getLangOpts().CPlusPlus11)
+    S.CCEDiag(Loc, diag::note_constexpr_invalid_function)
+        << /*isConstexpr=*/0 << /*isConstructor=*/0
+        << ("'" + S.getASTContext().BuiltinInfo.getName(ID) + "'").str();
+  else
+    S.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
+}
+
 static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
                                                   const InterpFrame *Frame,
                                                   const CallExpr *Call) {
@@ -176,10 +192,24 @@ static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
 
 static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
-                                   const CallExpr *Call) {
+                                   const Function *Func, const CallExpr *Call) {
+  unsigned ID = Func->getBuiltinID();
   const Pointer &A = getParam<Pointer>(Frame, 0);
   const Pointer &B = getParam<Pointer>(Frame, 1);
 
+  if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp)
+    diagnoseNonConstexprBuiltin(S, OpPC, ID);
+
+  uint64_t Limit = ~static_cast<uint64_t>(0);
+  if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp)
+    Limit = peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)))
+                .getZExtValue();
+
+  if (Limit == 0) {
+    pushInteger(S, 0, Call->getType());
+    return true;
+  }
+
   if (!CheckLive(S, OpPC, A, AK_Read) || !CheckLive(S, OpPC, B, AK_Read))
     return false;
 
@@ -192,7 +222,11 @@ static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
   unsigned IndexA = A.getIndex();
   unsigned IndexB = B.getIndex();
   int32_t Result = 0;
-  for (;; ++IndexA, ++IndexB) {
+  uint64_t Steps = 0;
+  for (;; ++IndexA, ++IndexB, ++Steps) {
+
+    if (Steps >= Limit)
+      break;
     const Pointer &PA = A.atIndex(IndexA);
     const Pointer &PB = B.atIndex(IndexB);
     if (!CheckRange(S, OpPC, PA, AK_Read) ||
@@ -219,9 +253,13 @@ static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
 
 static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
-                                   const CallExpr *Call) {
+                                   const Function *Func, const CallExpr *Call) {
+  unsigned ID = Func->getBuiltinID();
   const Pointer &StrPtr = getParam<Pointer>(Frame, 0);
 
+  if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
+    diagnoseNonConstexprBuiltin(S, OpPC, ID);
+
   if (!CheckArray(S, OpPC, StrPtr))
     return false;
 
@@ -232,6 +270,12 @@ static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
     return false;
 
   assert(StrPtr.getFieldDesc()->isPrimitiveArray());
+  unsigned ElemSize = StrPtr.getFieldDesc()->getElemSize();
+
+  if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
+    [[maybe_unused]] const ASTContext &AC = S.getASTContext();
+    assert(ElemSize == AC.getTypeSizeInChars(AC.getWCharType()).getQuantity());
+  }
 
   size_t Len = 0;
   for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
@@ -240,7 +284,20 @@ static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
     if (!CheckRange(S, OpPC, ElemPtr, AK_Read))
       return false;
 
-    uint8_t Val = ElemPtr.deref<uint8_t>();
+    uint32_t Val;
+    switch (ElemSize) {
+    case 1:
+      Val = ElemPtr.deref<uint8_t>();
+      break;
+    case 2:
+      Val = ElemPtr.deref<uint16_t>();
+      break;
+    case 4:
+      Val = ElemPtr.deref<uint32_t>();
+      break;
+    default:
+      llvm_unreachable("Unsupported char size");
+    }
     if (Val == 0)
       break;
   }
@@ -1684,10 +1741,154 @@ static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
+                                          const InterpFrame *Frame,
+                                          const Function *Func,
+                                          const CallExpr *Call) {
+  const Pointer &Arg = S.Stk.peek<Pointer>();
+  assert(Arg.getFieldDesc()->isPrimitiveArray());
+
+  unsigned ID = Func->getBuiltinID();
+  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
+  assert(Call->getType() == ElemType);
+  PrimType ElemT = *S.getContext().classify(ElemType);
+  unsigned NumElems = Arg.getNumElems();
+
+  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+    T Result = Arg.atIndex(0).deref<T>();
+    unsigned BitWidth = Result.bitWidth();
+    for (unsigned I = 1; I != NumElems; ++I) {
+      T Elem = Arg.atIndex(I).deref<T>();
+      T PrevResult = Result;
+
+      if (ID == Builtin::BI__builtin_reduce_add) {
+        if (T::add(Result, Elem, BitWidth, &Result)) {
+          unsigned OverflowBits = BitWidth + 1;
+          (void)handleOverflow(S, OpPC,
+                               (PrevResult.toAPSInt(OverflowBits) +
+                                Elem.toAPSInt(OverflowBits)));
+          return false;
+        }
+      } else if (ID == Builtin::BI__builtin_reduce_mul) {
+        if (T::mul(Result, Elem, BitWidth, &Result)) {
+          unsigned OverflowBits = BitWidth * 2;
+          (void)handleOverflow(S, OpPC,
+                               (PrevResult.toAPSInt(OverflowBits) *
+                                Elem.toAPSInt(OverflowBits)));
+          return false;
+        }
+
+      } else if (ID == Builtin::BI__builtin_reduce_and) {
+        (void)T::bitAnd(Result, Elem, BitWidth, &Result);
+      } else if (ID == Builtin::BI__builtin_reduce_or) {
+        (void)T::bitOr(Result, Elem, BitWidth, &Result);
+      } else if (ID == Builtin::BI__builtin_reduce_xor) {
+        (void)T::bitXor(Result, Elem, BitWidth, &Result);
+      } else {
+        llvm_unreachable("Unhandled vector reduce builtin");
+      }
+    }
+    pushInteger(S, Result.toAPSInt(), Call->getType());
+  });
+
+  return true;
+}
+
+/// Can be called with an integer or vector as the first and only parameter.
+static bool interp__builtin_elementwise_popcount(InterpState &S, CodePtr OpPC,
+                                                 const InterpFrame *Frame,
+                                                 const Function *Func,
+                                                 const CallExpr *Call) {
+  assert(Call->getNumArgs() == 1);
+  if (Call->getArg(0)->getType()->isIntegerType()) {
+    PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
+    APSInt Val = peekToAPSInt(S.Stk, ArgT);
+    pushInteger(S, Val.popcount(), Call->getType());
+    return true;
+  }
+  // Otherwise, the argument must be a vector.
+  assert(Call->getArg(0)->getType()->isVectorType());
+  const Pointer &Arg = S.Stk.peek<Pointer>();
+  assert(Arg.getFieldDesc()->isPrimitiveArray());
+  const Pointer &Dst = S.Stk.peek<Pointer>(primSize(PT_Ptr) * 2);
+  assert(Dst.getFieldDesc()->isPrimitiveArray());
+  assert(Arg.getFieldDesc()->getNumElems() ==
+         Dst.getFieldDesc()->getNumElems());
+
+  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
+  PrimType ElemT = *S.getContext().classify(ElemType);
+  unsigned NumElems = Arg.getNumElems();
+
+  // FIXME: Reading from uninitialized vector elements?
+  for (unsigned I = 0; I != NumElems; ++I) {
+    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+      Dst.atIndex(I).deref<T>() =
+          T::from(Arg.atIndex(I).deref<T>().toAPSInt().popcount());
+      Dst.atIndex(I).initialize();
+    });
+  }
+
+  return true;
+}
+static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
+                                   const InterpFrame *Frame,
+                                   const Function *Func, const CallExpr *Call) {
+  assert(Call->getNumArgs() == 3);
+  unsigned ID = Func->getBuiltinID();
+  Pointer DestPtr = getParam<Pointer>(Frame, 0);
+  const Pointer &SrcPtr = getParam<Pointer>(Frame, 1);
+  const APSInt &Size =
+      peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)));
+  assert(!Size.isSigned() && "memcpy and friends take an unsigned size");
+
+  if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
+    diagnoseNonConstexprBuiltin(S, OpPC, ID);
+
+  bool Move = (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove);
+
+  // If the size is zero, we treat this as always being a valid no-op.
+  if (Size.isZero()) {
+    S.Stk.push<Pointer>(DestPtr);
+    return true;
+  }
+
+  if (SrcPtr.isZero() || DestPtr.isZero()) {
+    Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
+    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
+        << /*IsMove=*/Move << /*IsWchar=*/false << !SrcPtr.isZero()
+        << DiagPtr.toDiagnosticString(S.getASTContext());
+    return false;
+  }
+
+  QualType ElemType;
+  if (SrcPtr.getFieldDesc()->isArray())
+    ElemType = SrcPtr.getFieldDesc()->getElemQualType();
+  else
+    ElemType = SrcPtr.getType();
+
+  unsigned ElemSize =
+      S.getASTContext().getTypeSizeInChars(ElemType).getQuantity();
+  if (Size.urem(ElemSize) != 0) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memcpy_unsupported)
+        << Move << /*IsWchar=*/false << 0 << ElemType << Size << ElemSize;
+    return false;
+  }
+
+  // As a last resort, reject dummy pointers.
+  if (DestPtr.isDummy() || SrcPtr.isDummy())
+    return false;
+
+  if (!DoBitCastPtr(S, OpPC, SrcPtr, DestPtr, Size.getZExtValue()))
+    return false;
+
+  S.Stk.push<Pointer>(DestPtr);
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call, uint32_t BuiltinID) {
   const InterpFrame *Frame = S.Current;
-  APValue Dummy;
 
   std::optional<PrimType> ReturnT = S.getContext().classify(Call);
 
@@ -1700,11 +1901,17 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
   case Builtin::BI__assume:
     break;
   case Builtin::BI__builtin_strcmp:
-    if (!interp__builtin_strcmp(S, OpPC, Frame, Call))
+  case Builtin::BIstrcmp:
+  case Builtin::BI__builtin_strncmp:
+  case Builtin::BIstrncmp:
+    if (!interp__builtin_strcmp(S, OpPC, Frame, F, Call))
       return false;
     break;
   case Builtin::BI__builtin_strlen:
-    if (!interp__builtin_strlen(S, OpPC, Frame, Call))
+  case Builtin::BIstrlen:
+  case Builtin::BI__builtin_wcslen:
+  case Builtin::BIwcslen:
+    if (!interp__builtin_strlen(S, OpPC, Frame, F, Call))
       return false;
     break;
   case Builtin::BI__builtin_nan:
@@ -2130,6 +2337,28 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BI__builtin_reduce_add:
+  case Builtin::BI__builtin_reduce_mul:
+  case Builtin::BI__builtin_reduce_and:
+  case Builtin::BI__builtin_reduce_or:
+  case Builtin::BI__builtin_reduce_xor:
+    if (!interp__builtin_vector_reduce(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
+  case Builtin::BI__builtin_elementwise_popcount:
+    if (!interp__builtin_elementwise_popcount(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
+  case Builtin::BI__builtin_memcpy:
+  case Builtin::BImemcpy:
+  case Builtin::BI__builtin_memmove:
+  case Builtin::BImemmove:
+    if (!interp__builtin_memcpy(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
@@ -2138,7 +2367,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
     return false;
   }
 
-  return retPrimValue(S, OpPC, Dummy, ReturnT);
+  return retPrimValue(S, OpPC, ReturnT);
 }
 
 bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index b1230f92ddf1d..7ba0faff25253 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "InterpBuiltinBitCast.h"
+#include "BitcastBuffer.h"
 #include "Boolean.h"
 #include "Context.h"
 #include "Floating.h"
@@ -21,9 +22,19 @@
 using namespace clang;
 using namespace clang::interp;
 
+/// Implement __builtin_bit_cast and related operations.
+/// Since our internal representation for data is more complex than
+/// something we can simply memcpy or memcmp, we first bitcast all the data
+/// into a buffer, which we then later use to copy the data into the target.
+
+// TODO:
+//  - Try to minimize heap allocations.
+//  - Optimize the common case of only pushing and pulling full
+//    bytes to/from the buffer.
+
 /// Used to iterate over pointer fields.
 using DataFunc = llvm::function_ref<bool(const Pointer &P, PrimType Ty,
-                                         size_t BitOffset, bool PackedBools)>;
+                                         Bits BitOffset, bool PackedBools)>;
 
 #define BITCAST_TYPE_SWITCH(Expr, B)                                           \
   do {                                                                         \
@@ -61,156 +72,78 @@ using DataFunc = llvm::function_ref<bool(const Pointer &P, PrimType Ty,
     }                                                                          \
   } while (0)
 
-static bool bitof(std::byte B, unsigned BitIndex) {
-  return (B & (std::byte{1} << BitIndex)) != std::byte{0};
-}
-
 static void swapBytes(std::byte *M, size_t N) {
   for (size_t I = 0; I != (N / 2); ++I)
     std::swap(M[I], M[N - 1 - I]);
 }
 
-/// Track what bits have been initialized to known values and which ones
-/// have indeterminate value.
-/// All offsets are in bits.
-struct BitcastBuffer {
-  size_t SizeInBits = 0;
-  llvm::SmallVector<std::byte> Data;
-
-  BitcastBuffer() = default;
-
-  size_t size() const { return SizeInBits; }
-
-  const std::byte *data() const { return Data.data(); }
-
-  std::byte *getBytes(unsigned BitOffset) const {
-    assert(BitOffset % 8 == 0);
-    assert(BitOffset < SizeInBits);
-    return const_cast<std::byte *>(data() + (BitOffset / 8));
-  }
-
-  bool allInitialized() const {
-    // FIXME: Implement.
-    return true;
-  }
-
-  bool atByteBoundary() const { return (Data.size() * 8) == SizeInBits; }
-
-  void pushBit(bool Value) {
-    if (atByteBoundary())
-      Data.push_back(std::byte{0});
-
-    if (Value)
-      Data.back() |= (std::byte{1} << (SizeInBits % 8));
-    ++SizeInBits;
-  }
-
-  void pushData(const std::byte *data, size_t BitWidth, bool BigEndianTarget) {
-    bool OnlyFullBytes = BitWidth % 8 == 0;
-    unsigned NBytes = BitWidth / 8;
-
-    size_t BitsHandled = 0;
-    // Read all full bytes first
-    for (size_t I = 0; I != NBytes; ++I) {
-      std::byte B =
-          BigEndianTarget ? data[NBytes - OnlyFullBytes - I] : data[I];
-      for (unsigned X = 0; X != 8; ++X) {
-        pushBit(bitof(B, X));
-        ++BitsHandled;
-      }
-    }
-
-    if (BitsHandled == BitWidth)
-      return;
-
-    // Rest of the bits.
-    assert((BitWidth - BitsHandled) < 8);
-    std::byte B = BigEndianTarget ? data[0] : data[NBytes];
-    for (size_t I = 0, E = (BitWidth - BitsHandled); I != E; ++I) {
-      pushBit(bitof(B, I));
-      ++BitsHandled;
-    }
-
-    assert(BitsHandled == BitWidth);
-  }
-};
-
-/// We use this to recursively iterate over all fields and elemends of a pointer
+/// We use this to recursively iterate over all fields and elements of a pointer
 /// and extract relevant data for a bitcast.
-static bool enumerateData(const Pointer &P, const Context &Ctx, size_t Offset,
-                          DataFunc F) {
+static bool enumerateData(const Pointer &P, const Context &Ctx, Bits Offset,
+                          Bits BitsToRead, DataFunc F) {
   const Descriptor *FieldDesc = P.getFieldDesc();
   assert(FieldDesc);
 
   // Primitives.
   if (FieldDesc->isPrimitive())
-    return F(P, FieldDesc->getPrimType(), Offset, false);
+    return F(P, FieldDesc->getPrimType(), Offset, /*PackedBools=*/false);
 
   // Primitive arrays.
   if (FieldDesc->isPrimitiveArray()) {
-    bool BigEndianTarget = Ctx.getASTContext().getTargetInfo().isBigEndian();
     QualType ElemType = FieldDesc->getElemQualType();
     size_t ElemSizeInBits = Ctx.getASTContext().getTypeSize(ElemType);
     PrimType ElemT = *Ctx.classify(ElemType);
     // Special case, since the bools here are packed.
     bool PackedBools = FieldDesc->getType()->isExtVectorBoolType();
+    unsigned NumElems = FieldDesc->getNumElems();
     bool Ok = true;
-    for (unsigned I = 0; I != FieldDesc->getNumElems(); ++I) {
-      unsigned Index = BigEndianTarget ? (FieldDesc->getNumElems() - 1 - I) : I;
-      Ok = Ok && F(P.atIndex(Index), ElemT, Offset, PackedBools);
-      Offset += ElemSizeInBits;
+    for (unsigned I = P.getIndex(); I != NumElems; ++I) {
+      Ok = Ok && F(P.atIndex(I), ElemT, Offset, PackedBools);
+      Offset += PackedBools ? 1 : ElemSizeInBits;
+      if (Offset >= BitsToRead)
+        break;
     }
     return Ok;
   }
 
   // Composite arrays.
   if (FieldDesc->isCompositeArray()) {
-    bool BigEndianTarget = Ctx.getASTContext().getTargetInfo().isBigEndian();
     QualType ElemType = FieldDesc->getElemQualType();
     size_t ElemSizeInBits = Ctx.getASTContext().getTypeSize(ElemType);
     for (unsigned I = 0; I != FieldDesc->getNumElems(); ++I) {
-      unsigned Index = BigEndianTarget ? (FieldDesc->getNumElems() - 1 - I) : I;
-      enumerateData(P.atIndex(Index).narrow(), Ctx, Offset, F);
+      enumerateData(P.atIndex(I).narrow(), Ctx, Offset, BitsToRead, F);
       Offset += ElemSizeInBits;
+      if (Offset >= BitsToRead)
+        break;
     }
     return true;
   }
 
   // Records.
   if (FieldDesc->isRecord()) {
-    bool BigEndianTarget = Ctx.getASTContext().getTargetInfo().isBigEndian();
     const Record *R = FieldDesc->ElemRecord;
     const ASTRecordLayout &Layout =
         Ctx.getASTContext().getASTRecordLayout(R->getDecl());
     bool Ok = true;
 
-    auto enumerateFields = [&]() -> void {
-      for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
-        const Record::Field *Fi =
-            R->getField(BigEndianTarget ? (N - 1 - I) : I);
-        Pointer Elem = P.atField(Fi->Offset);
-        size_t BitOffset =
-            Offset + Layout.getFieldOffset(Fi->Decl->getFieldIndex());
-        Ok = Ok && enumerateData(Elem, Ctx, BitOffset, F);
-      }
-    };
-    auto enumerateBases = [&]() -> void {
-      for (unsigned I = 0, N = R->getNumBases(); I != N; ++I) {
-        const Record::Base *B = R->getBase(BigEndianTarget ? (N - 1 - I) : I);
-        Pointer Elem = P.atField(B->Offset);
-        CharUnits ByteOffset =
-            Layout.getBaseClassOffset(cast<CXXRecordDecl>(B->Decl));
-        size_t BitOffset = Offset + Ctx.getASTContext().toBits(ByteOffset);
-        Ok = Ok && enumerateData(Elem, Ctx, BitOffset, F);
-      }
-    };
-
-    if (BigEndianTarget) {
-      enumerateFields();
-      enumerateBases();
-    } else {
-      enumerateBases();
-      enumerateFields();
+    for (const Record::Field &Fi : R->fields()) {
+      if (Fi.isUnnamedBitField())
+        continue;
+      Pointer Elem = P.atField(Fi.Offset);
+      Bits BitOffset =
+          Offset + Bits(Layout.getFieldOffset(Fi.Decl->getFieldIndex()));
+      Ok = Ok && enumerateData(Elem, Ctx, BitOffset, BitsToRead, F);
+    }
+    for (const Record::Base &B : R->bases()) {
+      Pointer Elem = P.atField(B.Offset);
+      CharUnits ByteOffset =
+          Layout.getBaseClassOffset(cast<CXXRecordDecl>(B.Decl));
+      Bits BitOffset = Offset + Bits(Ctx.getASTContext().toBits(ByteOffset));
+      Ok = Ok && enumerateData(Elem, Ctx, BitOffset, BitsToRead, F);
+      // FIXME: We should only (need to) do this when bitcasting OUT of the
+      // buffer, not when copying data into it.
+      if (Ok)
+        Elem.initialize();
     }
 
     return Ok;
@@ -220,8 +153,8 @@ static bool enumerateData(const Pointer &P, const Context &Ctx, size_t Offset,
 }
 
 static bool enumeratePointerFields(const Pointer &P, const Context &Ctx,
-                                   DataFunc F) {
-  return enumerateData(P, Ctx, 0, F);
+                                   Bits BitsToRead, DataFunc F) {
+  return enumerateData(P, Ctx, Bits::zero(), BitsToRead, F);
 }
 
 //  This function is constexpr if and only if To, From, and the types of
@@ -289,98 +222,148 @@ static bool CheckBitcastType(InterpState &S, CodePtr OpPC, QualType T,
                         IsToType))
     return false;
 
+  if (const auto *VT = T->getAs<VectorType>()) {
+    const ASTContext &ASTCtx = S.getASTContext();
+    QualType EltTy = VT->getElementType();
+    unsigned NElts = VT->getNumElements();
+    unsigned EltSize =
+        VT->isExtVectorBoolType() ? 1 : ASTCtx.getTypeSize(EltTy);
+
+    if ((NElts * EltSize) % ASTCtx.getCharWidth() != 0) {
+      // The vector's size in bits is not a multiple of the target's byte size,
+      // so its layout is unspecified. For now, we'll simply treat these cases
+      // as unsupported (this should only be possible with OpenCL bool vectors
+      // whose element count isn't a multiple of the byte size).
+      const Expr *E = S.Current->getExpr(OpPC);
+      S.FFDiag(E, diag::note_constexpr_bit_cast_invalid_vector)
+          << QualType(VT, 0) << EltSize << NElts << ASTCtx.getCharWidth();
+      return false;
+    }
+
+    if (EltTy->isRealFloatingType() &&
+        &ASTCtx.getFloatTypeSemantics(EltTy) == &APFloat::x87DoubleExtended()) {
+      // The layout for x86_fp80 vectors seems to be handled very inconsistently
+      // by both clang and LLVM, so for now we won't allow bit_casts involving
+      // it in a constexpr context.
+      const Expr *E = S.Current->getExpr(OpPC);
+      S.FFDiag(E, diag::note_constexpr_bit_cast_unsupported_type) << EltTy;
+      return false;
+    }
+  }
+
   return true;
 }
 
 static bool readPointerToBuffer(const Context &Ctx, const Pointer &FromPtr,
                                 BitcastBuffer &Buffer, bool ReturnOnUninit) {
   const ASTContext &ASTCtx = Ctx.getASTContext();
-  bool SwapData = (ASTCtx.getTargetInfo().isLittleEndian() !=
-                   llvm::sys::IsLittleEndianHost);
-  bool BigEndianTarget = ASTCtx.getTargetInfo().isBigEndian();
+  Endian TargetEndianness =
+      ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big;
 
   return enumeratePointerFields(
-      FromPtr, Ctx,
-      [&](const Pointer &P, PrimType T, size_t BitOffset,
+      FromPtr, Ctx, Buffer.size(),
+      [&](const Pointer &P, PrimType T, Bits BitOffset,
           bool PackedBools) -> bool {
-        if (!P.isInitialized()) {
-          assert(false && "Implement uninitialized value tracking");
-          return ReturnOnUninit;
+        CharUnits ObjectReprChars = ASTCtx.getTypeSizeInChars(P.getType());
+        Bits BitWidth = Bits(ASTCtx.toBits(ObjectReprChars));
+        Bits FullBitWidth = BitWidth;
+
+        if (const FieldDecl *FD = P.getField(); FD && FD->isBitField()) {
+          BitWidth = Bits(std::min(FD->getBitWidthValue(ASTCtx),
+                                   (unsigned)FullBitWidth.getQuantity()));
+        } else if (T == PT_Bool && PackedBools)
+          BitWidth = Bits(1);
+
+        if (BitWidth.isZero())
+          return true;
+
+        // Bits will be left uninitialized and diagnosed when reading.
+        if (!P.isInitialized())
+          return true;
+
+        if (T == PT_Ptr) {
+          assert(P.getType()->isNullPtrType());
+          // Clang treats nullptr_t has having NO bits in its value
+          // representation. So, we accept it here and leave its bits
+          // uninitialized.
+          return true;
         }
 
         assert(P.isInitialized());
-        // nullptr_t is a PT_Ptr for us, but it's still not std::is_pointer_v.
-        if (T == PT_Ptr)
-          assert(false && "Implement casting to pointer types");
-
-        CharUnits ObjectReprChars = ASTCtx.getTypeSizeInChars(P.getType());
-        unsigned BitWidth = ASTCtx.toBits(ObjectReprChars);
-        llvm::SmallVector<std::byte> Buff(ObjectReprChars.getQuantity());
+        auto Buff =
+            std::make_unique<std::byte[]>(ObjectReprChars.getQuantity());
         // Work around floating point types that contain unused padding bytes.
         // This is really just `long double` on x86, which is the only
         // fundamental type with padding bytes.
         if (T == PT_Float) {
           const Floating &F = P.deref<Floating>();
-          unsigned NumBits =
-              llvm::APFloatBase::getSizeInBits(F.getAPFloat().getSemantics());
-          assert(NumBits % 8 == 0);
-          assert(NumBits <= (ObjectReprChars.getQuantity() * 8));
-          F.bitcastToMemory(Buff.data());
+          Bits NumBits = Bits(
+              llvm::APFloatBase::getSizeInBits(F.getAPFloat().getSemantics()));
+          assert(NumBits.isFullByte());
+          assert(NumBits.getQuantity() <= FullBitWidth.getQuantity());
+          F.bitcastToMemory(Buff.get());
           // Now, only (maybe) swap the actual size of the float, excluding the
           // padding bits.
-          if (SwapData)
-            swapBytes(Buff.data(), NumBits / 8);
+          if (llvm::sys::IsBigEndianHost)
+            swapBytes(Buff.get(), NumBits.roundToBytes());
 
+          Buffer.markInitialized(BitOffset, NumBits);
         } else {
-          if (const FieldDecl *FD = P.getField(); FD && FD->isBitField())
-            BitWidth = FD->getBitWidthValue(ASTCtx);
-          else if (T == PT_Bool && PackedBools)
-            BitWidth = 1;
-
-          BITCAST_TYPE_SWITCH(T, {
-            T Val = P.deref<T>();
-            Val.bitcastToMemory(Buff.data());
-          });
-          if (SwapData)
-            swapBytes(Buff.data(), ObjectReprChars.getQuantity());
-        }
+          BITCAST_TYPE_SWITCH(T, { P.deref<T>().bitcastToMemory(Buff.get()); });
 
-        if (BitWidth != (Buff.size() * 8) && BigEndianTarget) {
-          Buffer.pushData(Buff.data() + (Buff.size() - 1 - (BitWidth / 8)),
-                          BitWidth, BigEndianTarget);
-        } else {
-          Buffer.pushData(Buff.data(), BitWidth, BigEndianTarget);
+          if (llvm::sys::IsBigEndianHost)
+            swapBytes(Buff.get(), FullBitWidth.roundToBytes());
+          Buffer.markInitialized(BitOffset, BitWidth);
         }
+
+        Buffer.pushData(Buff.get(), BitOffset, BitWidth, TargetEndianness);
         return true;
       });
 }
 
 bool clang::interp::DoBitCast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                              std::byte *Buff, size_t BuffSize,
+                              std::byte *Buff, Bits BitWidth, Bits FullBitWidth,
                               bool &HasIndeterminateBits) {
   assert(Ptr.isLive());
   assert(Ptr.isBlockPointer());
   assert(Buff);
+  assert(BitWidth <= FullBitWidth);
+  assert(FullBitWidth.isFullByte());
+  assert(BitWidth.isFullByte());
 
-  BitcastBuffer Buffer;
+  BitcastBuffer Buffer(FullBitWidth);
+  size_t BuffSize = FullBitWidth.roundToBytes();
   if (!CheckBitcastType(S, OpPC, Ptr.getType(), /*IsToType=*/false))
     return false;
 
   bool Success = readPointerToBuffer(S.getContext(), Ptr, Buffer,
                                      /*ReturnOnUninit=*/false);
-  assert(Buffer.size() == BuffSize * 8);
+  HasIndeterminateBits = !Buffer.rangeInitialized(Bits::zero(), BitWidth);
 
-  HasIndeterminateBits = !Buffer.allInitialized();
-  std::memcpy(Buff, Buffer.data(), BuffSize);
+  const ASTContext &ASTCtx = S.getASTContext();
+  Endian TargetEndianness =
+      ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big;
+  auto B =
+      Buffer.copyBits(Bits::zero(), BitWidth, FullBitWidth, TargetEndianness);
+
+  std::memcpy(Buff, B.get(), BuffSize);
 
   if (llvm::sys::IsBigEndianHost)
-    swapBytes(Buff, BuffSize);
+    swapBytes(Buff, BitWidth.roundToBytes());
 
   return Success;
 }
-
 bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
                                  const Pointer &FromPtr, Pointer &ToPtr) {
+  const ASTContext &ASTCtx = S.getASTContext();
+  CharUnits ObjectReprChars = ASTCtx.getTypeSizeInChars(ToPtr.getType());
+
+  return DoBitCastPtr(S, OpPC, FromPtr, ToPtr, ObjectReprChars.getQuantity());
+}
+
+bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
+                                 const Pointer &FromPtr, Pointer &ToPtr,
+                                 size_t Size) {
   assert(FromPtr.isLive());
   assert(FromPtr.isBlockPointer());
   assert(ToPtr.isBlockPointer());
@@ -393,43 +376,76 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
   if (!CheckBitcastType(S, OpPC, FromType, /*IsToType=*/false))
     return false;
 
-  BitcastBuffer Buffer;
+  const ASTContext &ASTCtx = S.getASTContext();
+  BitcastBuffer Buffer(Bytes(Size).toBits());
   readPointerToBuffer(S.getContext(), FromPtr, Buffer,
                       /*ReturnOnUninit=*/false);
 
   // Now read the values out of the buffer again and into ToPtr.
-  const ASTContext &ASTCtx = S.getASTContext();
-  size_t BitOffset = 0;
+  Endian TargetEndianness =
+      ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big;
   bool Success = enumeratePointerFields(
-      ToPtr, S.getContext(),
-      [&](const Pointer &P, PrimType T, size_t _, bool PackedBools) -> bool {
+      ToPtr, S.getContext(), Buffer.size(),
+      [&](const Pointer &P, PrimType T, Bits BitOffset,
+          bool PackedBools) -> bool {
+        QualType PtrType = P.getType();
+        CharUnits ObjectReprChars = ASTCtx.getTypeSizeInChars(PtrType);
+        Bits FullBitWidth = Bits(ASTCtx.toBits(ObjectReprChars));
         if (T == PT_Float) {
-          CharUnits ObjectReprChars = ASTCtx.getTypeSizeInChars(P.getType());
-          const auto &Semantics = ASTCtx.getFloatTypeSemantics(P.getType());
-          unsigned NumBits = llvm::APFloatBase::getSizeInBits(Semantics);
-          assert(NumBits % 8 == 0);
-          assert(NumBits <= ASTCtx.toBits(ObjectReprChars));
-          std::byte *M = Buffer.getBytes(BitOffset);
+          const auto &Semantics = ASTCtx.getFloatTypeSemantics(PtrType);
+          Bits NumBits = Bits(llvm::APFloatBase::getSizeInBits(Semantics));
+          assert(NumBits.isFullByte());
+          assert(NumBits.getQuantity() <= FullBitWidth.getQuantity());
+          auto M = Buffer.copyBits(BitOffset, NumBits, FullBitWidth,
+                                   TargetEndianness);
 
           if (llvm::sys::IsBigEndianHost)
-            swapBytes(M, NumBits / 8);
+            swapBytes(M.get(), NumBits.roundToBytes());
 
-          P.deref<Floating>() = Floating::bitcastFromMemory(M, Semantics);
+          P.deref<Floating>() = Floating::bitcastFromMemory(M.get(), Semantics);
           P.initialize();
-          BitOffset += ASTCtx.toBits(ObjectReprChars);
           return true;
         }
 
-        BITCAST_TYPE_SWITCH_FIXED_SIZE(T, {
-          std::byte *M = Buffer.getBytes(BitOffset);
+        Bits BitWidth;
+        if (const FieldDecl *FD = P.getField(); FD && FD->isBitField())
+          BitWidth = Bits(std::min(FD->getBitWidthValue(ASTCtx),
+                                   (unsigned)FullBitWidth.getQuantity()));
+        else if (T == PT_Bool && PackedBools)
+          BitWidth = Bits(1);
+        else
+          BitWidth = FullBitWidth;
+
+        // If any of the bits are uninitialized, we need to abort unless the
+        // target type is std::byte or unsigned char.
+        bool Initialized = Buffer.rangeInitialized(BitOffset, BitWidth);
+        if (!Initialized) {
+          if (!PtrType->isStdByteType() &&
+              !PtrType->isSpecificBuiltinType(BuiltinType::UChar) &&
+              !PtrType->isSpecificBuiltinType(BuiltinType::Char_U)) {
+            const Expr *E = S.Current->getExpr(OpPC);
+            S.FFDiag(E, diag::note_constexpr_bit_cast_indet_dest)
+                << PtrType << S.getLangOpts().CharIsSigned
+                << E->getSourceRange();
+
+            return false;
+          }
+          return true;
+        }
 
-          if (llvm::sys::IsBigEndianHost)
-            swapBytes(M, T::bitWidth() / 8);
+        auto Memory = Buffer.copyBits(BitOffset, BitWidth, FullBitWidth,
+                                      TargetEndianness);
+        if (llvm::sys::IsBigEndianHost)
+          swapBytes(Memory.get(), FullBitWidth.roundToBytes());
 
-          P.deref<T>() = T::bitcastFromMemory(M, T::bitWidth());
-          P.initialize();
-          BitOffset += T::bitWidth();
+        BITCAST_TYPE_SWITCH_FIXED_SIZE(T, {
+          if (BitWidth.nonZero())
+            P.deref<T>() = T::bitcastFromMemory(Memory.get(), T::bitWidth())
+                               .truncate(BitWidth.getQuantity());
+          else
+            P.deref<T>() = T::zero();
         });
+        P.initialize();
         return true;
       });
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.h b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.h
index 494c0880a9c45..92e6ffc79fc4f 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.h
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_AST_INTERP_BUILITN_BIT_CAST_H
 #define LLVM_CLANG_AST_INTERP_BUILITN_BIT_CAST_H
 
+#include "BitcastBuffer.h"
 #include <cstddef>
 
 namespace clang {
@@ -18,9 +19,12 @@ class InterpState;
 class CodePtr;
 
 bool DoBitCast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-               std::byte *Buff, size_t BuffSize, bool &HasIndeterminateBits);
+               std::byte *Buff, Bits BitWidth, Bits FullBitWidth,
+               bool &HasIndeterminateBits);
 bool DoBitCastPtr(InterpState &S, CodePtr OpPC, const Pointer &FromPtr,
                   Pointer &ToPtr);
+bool DoBitCastPtr(InterpState &S, CodePtr OpPC, const Pointer &FromPtr,
+                  Pointer &ToPtr, size_t Size);
 
 } // namespace interp
 } // namespace clang
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 26d5f70b44396..0638f8249805f 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -281,7 +281,7 @@ def ZeroIntAPS : Opcode {
 // [] -> [Pointer]
 def Null : Opcode {
   let Types = [PtrTypeClass];
-  let Args = [ArgDesc];
+  let Args = [ArgUint64, ArgDesc];
   let HasGroup = 1;
 }
 
diff --git a/clang/lib/AST/ByteCode/Record.h b/clang/lib/AST/ByteCode/Record.h
index 7a5c482e4efcc..686b5dbc431d9 100644
--- a/clang/lib/AST/ByteCode/Record.h
+++ b/clang/lib/AST/ByteCode/Record.h
@@ -30,6 +30,7 @@ class Record final {
     unsigned Offset;
     const Descriptor *Desc;
     bool isBitField() const { return Decl->isBitField(); }
+    bool isUnnamedBitField() const { return Decl->isUnnamedBitField(); }
   };
 
   /// Describes a base class.
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index 52c6a45de9a26..cb13c5225b713 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -65,6 +65,7 @@ add_clang_library(clangAST
   ExternalASTSource.cpp
   FormatString.cpp
   InheritViz.cpp
+  ByteCode/BitcastBuffer.cpp
   ByteCode/ByteCodeEmitter.cpp
   ByteCode/Compiler.cpp
   ByteCode/Context.cpp
diff --git a/clang/lib/AST/CXXInheritance.cpp b/clang/lib/AST/CXXInheritance.cpp
index aefc06e9197cf..ee5775837d535 100644
--- a/clang/lib/AST/CXXInheritance.cpp
+++ b/clang/lib/AST/CXXInheritance.cpp
@@ -134,7 +134,7 @@ bool CXXRecordDecl::forallBases(ForallBasesCallback BaseMatches) const {
         return false;
 
       CXXRecordDecl *Base =
-            cast_or_null<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+          cast_if_present<CXXRecordDecl>(Ty->getDecl()->getDefinition());
       if (!Base ||
           (Base->isDependentContext() &&
            !Base->isCurrentInstantiation(Record))) {
@@ -169,13 +169,21 @@ bool CXXBasePaths::lookupInBases(ASTContext &Context,
     QualType BaseType =
         Context.getCanonicalType(BaseSpec.getType()).getUnqualifiedType();
 
+    bool isCurrentInstantiation = isa<InjectedClassNameType>(BaseType);
+    if (!isCurrentInstantiation) {
+      if (auto *BaseRecord = cast_if_present<CXXRecordDecl>(
+              BaseSpec.getType()->getAsRecordDecl()))
+        isCurrentInstantiation = BaseRecord->isDependentContext() &&
+                                 BaseRecord->isCurrentInstantiation(Record);
+    }
     // C++ [temp.dep]p3:
     //   In the definition of a class template or a member of a class template,
     //   if a base class of the class template depends on a template-parameter,
     //   the base class scope is not examined during unqualified name lookup
     //   either at the point of definition of the class template or member or
     //   during an instantiation of the class tem- plate or member.
-    if (!LookupInDependent && BaseType->isDependentType())
+    if (!LookupInDependent &&
+        (BaseType->isDependentType() && !isCurrentInstantiation))
       continue;
 
     // Determine whether we need to visit this base class at all,
@@ -243,9 +251,8 @@ bool CXXBasePaths::lookupInBases(ASTContext &Context,
         return FoundPath;
       }
     } else if (VisitBase) {
-      CXXRecordDecl *BaseRecord;
+      CXXRecordDecl *BaseRecord = nullptr;
       if (LookupInDependent) {
-        BaseRecord = nullptr;
         const TemplateSpecializationType *TST =
             BaseSpec.getType()->getAs<TemplateSpecializationType>();
         if (!TST) {
@@ -264,8 +271,7 @@ bool CXXBasePaths::lookupInBases(ASTContext &Context,
             BaseRecord = nullptr;
         }
       } else {
-        BaseRecord = cast<CXXRecordDecl>(
-            BaseSpec.getType()->castAs<RecordType>()->getDecl());
+        BaseRecord = cast<CXXRecordDecl>(BaseSpec.getType()->getAsRecordDecl());
       }
       if (BaseRecord &&
           lookupInBases(Context, BaseRecord, BaseMatches, LookupInDependent)) {
@@ -362,8 +368,8 @@ bool CXXRecordDecl::FindBaseClass(const CXXBaseSpecifier *Specifier,
                                   const CXXRecordDecl *BaseRecord) {
   assert(BaseRecord->getCanonicalDecl() == BaseRecord &&
          "User data for FindBaseClass is not canonical!");
-  return Specifier->getType()->castAs<RecordType>()->getDecl()
-            ->getCanonicalDecl() == BaseRecord;
+  return cast<CXXRecordDecl>(Specifier->getType()->getAsRecordDecl())
+             ->getCanonicalDecl() == BaseRecord;
 }
 
 bool CXXRecordDecl::FindVirtualBaseClass(const CXXBaseSpecifier *Specifier,
@@ -372,8 +378,8 @@ bool CXXRecordDecl::FindVirtualBaseClass(const CXXBaseSpecifier *Specifier,
   assert(BaseRecord->getCanonicalDecl() == BaseRecord &&
          "User data for FindBaseClass is not canonical!");
   return Specifier->isVirtual() &&
-         Specifier->getType()->castAs<RecordType>()->getDecl()
-            ->getCanonicalDecl() == BaseRecord;
+         cast<CXXRecordDecl>(Specifier->getType()->getAsRecordDecl())
+                 ->getCanonicalDecl() == BaseRecord;
 }
 
 static bool isOrdinaryMember(const NamedDecl *ND) {
@@ -686,7 +692,7 @@ AddIndirectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
            "Cannot get indirect primary bases for class with dependent bases.");
 
     const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(I.getType()->getAsRecordDecl());
 
     // Only bases with virtual bases participate in computing the
     // indirect primary virtual base classes.
@@ -708,7 +714,7 @@ CXXRecordDecl::getIndirectPrimaryBases(CXXIndirectPrimaryBaseSet& Bases) const {
            "Cannot get indirect primary bases for class with dependent bases.");
 
     const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(I.getType()->getAsRecordDecl());
 
     // Only bases with virtual bases participate in computing the
     // indirect primary virtual base classes.
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index f0250fc9fd55e..ec9a5b480aa29 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -8,8 +8,8 @@
 
 #include "clang/AST/CommentLexer.h"
 #include "clang/AST/CommentCommandTraits.h"
-#include "clang/AST/CommentDiagnostic.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/DiagnosticComment.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ConvertUTF.h"
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 61508fe886efc..12ed8e3f1b79a 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -8,9 +8,9 @@
 
 #include "clang/AST/CommentParser.h"
 #include "clang/AST/CommentCommandTraits.h"
-#include "clang/AST/CommentDiagnostic.h"
 #include "clang/AST/CommentSema.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/DiagnosticComment.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/Support/ErrorHandling.h"
 
diff --git a/clang/lib/AST/CommentSema.cpp b/clang/lib/AST/CommentSema.cpp
index 69eda00643a8f..bd2206bb8a3bc 100644
--- a/clang/lib/AST/CommentSema.cpp
+++ b/clang/lib/AST/CommentSema.cpp
@@ -9,9 +9,9 @@
 #include "clang/AST/CommentSema.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/CommentCommandTraits.h"
-#include "clang/AST/CommentDiagnostic.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/Basic/DiagnosticComment.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Preprocessor.h"
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 25560faae8672..af73c658d6a0c 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -546,9 +546,9 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) {
     data().NeedOverloadResolutionForDestructor = true;
   }
 
-  // C++2a [dcl.constexpr]p4:
-  //   The definition of a constexpr destructor [shall] satisfy the
-  //   following requirement:
+  // C++20 [dcl.constexpr]p5:
+  //  The definition of a constexpr destructor whose function-body is not
+  //  = delete shall additionally satisfy the following requirement:
   //   -- for every subobject of class type or (possibly multi-dimensional)
   //      array thereof, that class type shall have a constexpr destructor
   if (!Subobj->hasConstexprDestructor())
@@ -1213,8 +1213,13 @@ void CXXRecordDecl::addedMember(Decl *D) {
             data().DefaultedCopyAssignmentIsDeleted = true;
           if (FieldRec->hasNonTrivialMoveAssignment())
             data().DefaultedMoveAssignmentIsDeleted = true;
-          if (FieldRec->hasNonTrivialDestructor())
+          if (FieldRec->hasNonTrivialDestructor()) {
             data().DefaultedDestructorIsDeleted = true;
+            // C++20 [dcl.constexpr]p5:
+            //   The definition of a constexpr destructor whose function-body is
+            //   not = delete shall additionally satisfy...
+            data().DefaultedDestructorIsConstexpr = true;
+          }
         }
 
         // For an anonymous union member, our overload resolution will perform
@@ -2597,8 +2602,6 @@ bool CXXMethodDecl::isMoveAssignmentOperator() const {
 
 void CXXMethodDecl::addOverriddenMethod(const CXXMethodDecl *MD) {
   assert(MD->isCanonicalDecl() && "Method is not canonical!");
-  assert(!MD->getParent()->isDependentContext() &&
-         "Can't add an overridden method to a class template!");
   assert(MD->isVirtual() && "Method is not virtual!");
 
   getASTContext().addOverriddenMethod(this, MD);
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 1da3f26bf23cd..40ee3753c2422 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -16,7 +16,9 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExternalASTSource.h"
+#include "clang/AST/ODRHash.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
@@ -348,26 +350,39 @@ RedeclarableTemplateDecl::CommonBase *RedeclarableTemplateDecl::getCommonPtr() c
   return Common;
 }
 
-void RedeclarableTemplateDecl::loadLazySpecializationsImpl() const {
-  // Grab the most recent declaration to ensure we've loaded any lazy
-  // redeclarations of this template.
-  CommonBase *CommonBasePtr = getMostRecentDecl()->getCommonPtr();
-  if (CommonBasePtr->LazySpecializations) {
-    ASTContext &Context = getASTContext();
-    GlobalDeclID *Specs = CommonBasePtr->LazySpecializations;
-    CommonBasePtr->LazySpecializations = nullptr;
-    unsigned SpecSize = (*Specs++).getRawValue();
-    for (unsigned I = 0; I != SpecSize; ++I)
-      (void)Context.getExternalSource()->GetExternalDecl(Specs[I]);
-  }
+void RedeclarableTemplateDecl::loadLazySpecializationsImpl(
+    bool OnlyPartial /*=false*/) const {
+  auto *ExternalSource = getASTContext().getExternalSource();
+  if (!ExternalSource)
+    return;
+
+  ExternalSource->LoadExternalSpecializations(this->getCanonicalDecl(),
+                                              OnlyPartial);
+  return;
 }
 
-template<class EntryType, typename... ProfileArguments>
+bool RedeclarableTemplateDecl::loadLazySpecializationsImpl(
+    ArrayRef<TemplateArgument> Args, TemplateParameterList *TPL) const {
+  auto *ExternalSource = getASTContext().getExternalSource();
+  if (!ExternalSource)
+    return false;
+
+  // If TPL is not null, it implies that we're loading specializations for
+  // partial templates. We need to load all specializations in such cases.
+  if (TPL)
+    return ExternalSource->LoadExternalSpecializations(this->getCanonicalDecl(),
+                                                       /*OnlyPartial=*/false);
+
+  return ExternalSource->LoadExternalSpecializations(this->getCanonicalDecl(),
+                                                     Args);
+}
+
+template <class EntryType, typename... ProfileArguments>
 typename RedeclarableTemplateDecl::SpecEntryTraits<EntryType>::DeclType *
-RedeclarableTemplateDecl::findSpecializationImpl(
+RedeclarableTemplateDecl::findSpecializationLocally(
     llvm::FoldingSetVector<EntryType> &Specs, void *&InsertPos,
-    ProfileArguments&&... ProfileArgs) {
-  using SETraits = SpecEntryTraits<EntryType>;
+    ProfileArguments &&...ProfileArgs) {
+  using SETraits = RedeclarableTemplateDecl::SpecEntryTraits<EntryType>;
 
   llvm::FoldingSetNodeID ID;
   EntryType::Profile(ID, std::forward<ProfileArguments>(ProfileArgs)...,
@@ -376,6 +391,24 @@ RedeclarableTemplateDecl::findSpecializationImpl(
   return Entry ? SETraits::getDecl(Entry)->getMostRecentDecl() : nullptr;
 }
 
+template <class EntryType, typename... ProfileArguments>
+typename RedeclarableTemplateDecl::SpecEntryTraits<EntryType>::DeclType *
+RedeclarableTemplateDecl::findSpecializationImpl(
+    llvm::FoldingSetVector<EntryType> &Specs, void *&InsertPos,
+    ProfileArguments &&...ProfileArgs) {
+
+  if (auto *Found = findSpecializationLocally(
+          Specs, InsertPos, std::forward<ProfileArguments>(ProfileArgs)...))
+    return Found;
+
+  if (!loadLazySpecializationsImpl(
+          std::forward<ProfileArguments>(ProfileArgs)...))
+    return nullptr;
+
+  return findSpecializationLocally(
+      Specs, InsertPos, std::forward<ProfileArguments>(ProfileArgs)...);
+}
+
 template<class Derived, class EntryType>
 void RedeclarableTemplateDecl::addSpecializationImpl(
     llvm::FoldingSetVector<EntryType> &Specializations, EntryType *Entry,
@@ -384,10 +417,14 @@ void RedeclarableTemplateDecl::addSpecializationImpl(
 
   if (InsertPos) {
 #ifndef NDEBUG
+    auto Args = SETraits::getTemplateArgs(Entry);
+    // Due to hash collisions, it can happen that we load another template
+    // specialization with the same hash. This is fine, as long as the next
+    // call to findSpecializationImpl does not find a matching Decl for the
+    // template arguments.
+    loadLazySpecializationsImpl(Args);
     void *CorrectInsertPos;
-    assert(!findSpecializationImpl(Specializations,
-                                   CorrectInsertPos,
-                                   SETraits::getTemplateArgs(Entry)) &&
+    assert(!findSpecializationImpl(Specializations, CorrectInsertPos, Args) &&
            InsertPos == CorrectInsertPos &&
            "given incorrect InsertPos for specialization");
 #endif
@@ -445,12 +482,14 @@ FunctionTemplateDecl::getSpecializations() const {
 FunctionDecl *
 FunctionTemplateDecl::findSpecialization(ArrayRef<TemplateArgument> Args,
                                          void *&InsertPos) {
-  return findSpecializationImpl(getSpecializations(), InsertPos, Args);
+  auto *Common = getCommonPtr();
+  return findSpecializationImpl(Common->Specializations, InsertPos, Args);
 }
 
 void FunctionTemplateDecl::addSpecialization(
       FunctionTemplateSpecializationInfo *Info, void *InsertPos) {
-  addSpecializationImpl<FunctionTemplateDecl>(getSpecializations(), Info,
+  auto *Common = getCommonPtr();
+  addSpecializationImpl<FunctionTemplateDecl>(Common->Specializations, Info,
                                               InsertPos);
 }
 
@@ -510,8 +549,9 @@ ClassTemplateDecl *ClassTemplateDecl::CreateDeserialized(ASTContext &C,
                                        DeclarationName(), nullptr, nullptr);
 }
 
-void ClassTemplateDecl::LoadLazySpecializations() const {
-  loadLazySpecializationsImpl();
+void ClassTemplateDecl::LoadLazySpecializations(
+    bool OnlyPartial /*=false*/) const {
+  loadLazySpecializationsImpl(OnlyPartial);
 }
 
 llvm::FoldingSetVector<ClassTemplateSpecializationDecl> &
@@ -522,7 +562,7 @@ ClassTemplateDecl::getSpecializations() const {
 
 llvm::FoldingSetVector<ClassTemplatePartialSpecializationDecl> &
 ClassTemplateDecl::getPartialSpecializations() const {
-  LoadLazySpecializations();
+  LoadLazySpecializations(/*PartialOnly = */ true);
   return getCommonPtr()->PartialSpecializations;
 }
 
@@ -536,12 +576,15 @@ ClassTemplateDecl::newCommon(ASTContext &C) const {
 ClassTemplateSpecializationDecl *
 ClassTemplateDecl::findSpecialization(ArrayRef<TemplateArgument> Args,
                                       void *&InsertPos) {
-  return findSpecializationImpl(getSpecializations(), InsertPos, Args);
+  auto *Common = getCommonPtr();
+  return findSpecializationImpl(Common->Specializations, InsertPos, Args);
 }
 
 void ClassTemplateDecl::AddSpecialization(ClassTemplateSpecializationDecl *D,
                                           void *InsertPos) {
-  addSpecializationImpl<ClassTemplateDecl>(getSpecializations(), D, InsertPos);
+  auto *Common = getCommonPtr();
+  addSpecializationImpl<ClassTemplateDecl>(Common->Specializations, D,
+                                           InsertPos);
 }
 
 ClassTemplatePartialSpecializationDecl *
@@ -1259,8 +1302,9 @@ VarTemplateDecl *VarTemplateDecl::CreateDeserialized(ASTContext &C,
                                      DeclarationName(), nullptr, nullptr);
 }
 
-void VarTemplateDecl::LoadLazySpecializations() const {
-  loadLazySpecializationsImpl();
+void VarTemplateDecl::LoadLazySpecializations(
+    bool OnlyPartial /*=false*/) const {
+  loadLazySpecializationsImpl(OnlyPartial);
 }
 
 llvm::FoldingSetVector<VarTemplateSpecializationDecl> &
@@ -1271,7 +1315,7 @@ VarTemplateDecl::getSpecializations() const {
 
 llvm::FoldingSetVector<VarTemplatePartialSpecializationDecl> &
 VarTemplateDecl::getPartialSpecializations() const {
-  LoadLazySpecializations();
+  LoadLazySpecializations(/*PartialOnly = */ true);
   return getCommonPtr()->PartialSpecializations;
 }
 
@@ -1285,12 +1329,14 @@ VarTemplateDecl::newCommon(ASTContext &C) const {
 VarTemplateSpecializationDecl *
 VarTemplateDecl::findSpecialization(ArrayRef<TemplateArgument> Args,
                                     void *&InsertPos) {
-  return findSpecializationImpl(getSpecializations(), InsertPos, Args);
+  auto *Common = getCommonPtr();
+  return findSpecializationImpl(Common->Specializations, InsertPos, Args);
 }
 
 void VarTemplateDecl::AddSpecialization(VarTemplateSpecializationDecl *D,
                                         void *InsertPos) {
-  addSpecializationImpl<VarTemplateDecl>(getSpecializations(), D, InsertPos);
+  auto *Common = getCommonPtr();
+  addSpecializationImpl<VarTemplateDecl>(Common->Specializations, D, InsertPos);
 }
 
 VarTemplatePartialSpecializationDecl *
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index a74640e04ce77..6c206e1f7a231 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1694,11 +1694,19 @@ SourceLocation CallExpr::getBeginLoc() const {
   if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(this))
     return OCE->getBeginLoc();
 
+  if (const auto *Method =
+          dyn_cast_if_present<const CXXMethodDecl>(getCalleeDecl());
+      Method && Method->isExplicitObjectMemberFunction()) {
+    assert(getNumArgs() > 0 && getArg(0));
+    return getArg(0)->getBeginLoc();
+  }
+
   SourceLocation begin = getCallee()->getBeginLoc();
   if (begin.isInvalid() && getNumArgs() > 0 && getArg(0))
     begin = getArg(0)->getBeginLoc();
   return begin;
 }
+
 SourceLocation CallExpr::getEndLoc() const {
   if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(this))
     return OCE->getEndLoc();
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index ef22891868b19..66c7728e33248 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -7352,31 +7352,6 @@ class APValueToBufferConverter {
     const VectorType *VTy = Ty->castAs<VectorType>();
     QualType EltTy = VTy->getElementType();
     unsigned NElts = VTy->getNumElements();
-    unsigned EltSize =
-        VTy->isExtVectorBoolType() ? 1 : Info.Ctx.getTypeSize(EltTy);
-
-    if ((NElts * EltSize) % Info.Ctx.getCharWidth() != 0) {
-      // The vector's size in bits is not a multiple of the target's byte size,
-      // so its layout is unspecified. For now, we'll simply treat these cases
-      // as unsupported (this should only be possible with OpenCL bool vectors
-      // whose element count isn't a multiple of the byte size).
-      Info.FFDiag(BCE->getBeginLoc(),
-                  diag::note_constexpr_bit_cast_invalid_vector)
-          << Ty.getCanonicalType() << EltSize << NElts
-          << Info.Ctx.getCharWidth();
-      return false;
-    }
-
-    if (EltTy->isRealFloatingType() && &Info.Ctx.getFloatTypeSemantics(EltTy) ==
-                                           &APFloat::x87DoubleExtended()) {
-      // The layout for x86_fp80 vectors seems to be handled very inconsistently
-      // by both clang and LLVM, so for now we won't allow bit_casts involving
-      // it in a constexpr context.
-      Info.FFDiag(BCE->getBeginLoc(),
-                  diag::note_constexpr_bit_cast_unsupported_type)
-          << EltTy;
-      return false;
-    }
 
     if (VTy->isExtVectorBoolType()) {
       // Special handling for OpenCL bool vectors:
@@ -7643,28 +7618,6 @@ class BufferToAPValueConverter {
     unsigned EltSize =
         VTy->isExtVectorBoolType() ? 1 : Info.Ctx.getTypeSize(EltTy);
 
-    if ((NElts * EltSize) % Info.Ctx.getCharWidth() != 0) {
-      // The vector's size in bits is not a multiple of the target's byte size,
-      // so its layout is unspecified. For now, we'll simply treat these cases
-      // as unsupported (this should only be possible with OpenCL bool vectors
-      // whose element count isn't a multiple of the byte size).
-      Info.FFDiag(BCE->getBeginLoc(),
-                  diag::note_constexpr_bit_cast_invalid_vector)
-          << QualType(VTy, 0) << EltSize << NElts << Info.Ctx.getCharWidth();
-      return std::nullopt;
-    }
-
-    if (EltTy->isRealFloatingType() && &Info.Ctx.getFloatTypeSemantics(EltTy) ==
-                                           &APFloat::x87DoubleExtended()) {
-      // The layout for x86_fp80 vectors seems to be handled very inconsistently
-      // by both clang and LLVM, so for now we won't allow bit_casts involving
-      // it in a constexpr context.
-      Info.FFDiag(BCE->getBeginLoc(),
-                  diag::note_constexpr_bit_cast_unsupported_type)
-          << EltTy;
-      return std::nullopt;
-    }
-
     SmallVector<APValue, 4> Elts;
     Elts.reserve(NElts);
     if (VTy->isExtVectorBoolType()) {
@@ -7793,6 +7746,32 @@ static bool checkBitCastConstexprEligibilityType(SourceLocation Loc,
                                             Info, Ctx, CheckingDest))
     return false;
 
+  if (const auto *VTy = Ty->getAs<VectorType>()) {
+    QualType EltTy = VTy->getElementType();
+    unsigned NElts = VTy->getNumElements();
+    unsigned EltSize = VTy->isExtVectorBoolType() ? 1 : Ctx.getTypeSize(EltTy);
+
+    if ((NElts * EltSize) % Ctx.getCharWidth() != 0) {
+      // The vector's size in bits is not a multiple of the target's byte size,
+      // so its layout is unspecified. For now, we'll simply treat these cases
+      // as unsupported (this should only be possible with OpenCL bool vectors
+      // whose element count isn't a multiple of the byte size).
+      Info->FFDiag(Loc, diag::note_constexpr_bit_cast_invalid_vector)
+          << QualType(VTy, 0) << EltSize << NElts << Ctx.getCharWidth();
+      return false;
+    }
+
+    if (EltTy->isRealFloatingType() &&
+        &Ctx.getFloatTypeSemantics(EltTy) == &APFloat::x87DoubleExtended()) {
+      // The layout for x86_fp80 vectors seems to be handled very inconsistently
+      // by both clang and LLVM, so for now we won't allow bit_casts involving
+      // it in a constexpr context.
+      Info->FFDiag(Loc, diag::note_constexpr_bit_cast_unsupported_type)
+          << EltTy;
+      return false;
+    }
+  }
+
   return true;
 }
 
@@ -10190,7 +10169,9 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
       return false;
     IsNothrow = true;
   } else if (OperatorNew->isReservedGlobalPlacementOperator()) {
-    if (Info.CurrentCall->isStdFunction() || Info.getLangOpts().CPlusPlus26) {
+    if (Info.CurrentCall->isStdFunction() || Info.getLangOpts().CPlusPlus26 ||
+        (Info.CurrentCall->CanEvalMSConstexpr &&
+         OperatorNew->hasAttr<MSConstexprAttr>())) {
       if (!EvaluatePointer(E->getPlacementArg(0), Result, Info))
         return false;
       if (Result.Designator.Invalid)
@@ -11328,7 +11309,8 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
   switch (E->getBuiltinCallee()) {
   default:
     return false;
-  case Builtin::BI__builtin_elementwise_popcount: {
+  case Builtin::BI__builtin_elementwise_popcount:
+  case Builtin::BI__builtin_elementwise_bitreverse: {
     APValue Source;
     if (!EvaluateAsRValue(Info, E->getArg(0), Source))
       return false;
@@ -11340,9 +11322,49 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
 
     for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
       APSInt Elt = Source.getVectorElt(EltNum).getInt();
-      ResultElements.push_back(
-          APValue(APSInt(APInt(Info.Ctx.getIntWidth(DestEltTy), Elt.popcount()),
-                         DestEltTy->isUnsignedIntegerOrEnumerationType())));
+      switch (E->getBuiltinCallee()) {
+      case Builtin::BI__builtin_elementwise_popcount:
+        ResultElements.push_back(APValue(
+            APSInt(APInt(Info.Ctx.getIntWidth(DestEltTy), Elt.popcount()),
+                   DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      case Builtin::BI__builtin_elementwise_bitreverse:
+        ResultElements.push_back(
+            APValue(APSInt(Elt.reverseBits(),
+                           DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      }
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+  case Builtin::BI__builtin_elementwise_add_sat:
+  case Builtin::BI__builtin_elementwise_sub_sat: {
+    APValue SourceLHS, SourceRHS;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS) ||
+        !EvaluateAsRValue(Info, E->getArg(1), SourceRHS))
+      return false;
+
+    QualType DestEltTy = E->getType()->castAs<VectorType>()->getElementType();
+    unsigned SourceLen = SourceLHS.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      APSInt LHS = SourceLHS.getVectorElt(EltNum).getInt();
+      APSInt RHS = SourceRHS.getVectorElt(EltNum).getInt();
+      switch (E->getBuiltinCallee()) {
+      case Builtin::BI__builtin_elementwise_add_sat:
+        ResultElements.push_back(APValue(
+            APSInt(LHS.isSigned() ? LHS.sadd_sat(RHS) : RHS.uadd_sat(RHS),
+                   DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      case Builtin::BI__builtin_elementwise_sub_sat:
+        ResultElements.push_back(APValue(
+            APSInt(LHS.isSigned() ? LHS.ssub_sat(RHS) : RHS.usub_sat(RHS),
+                   DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      }
     }
 
     return Success(APValue(ResultElements.data(), ResultElements.size()), E);
@@ -12902,7 +12924,8 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   case Builtin::BI__builtin_bitreverse8:
   case Builtin::BI__builtin_bitreverse16:
   case Builtin::BI__builtin_bitreverse32:
-  case Builtin::BI__builtin_bitreverse64: {
+  case Builtin::BI__builtin_bitreverse64:
+  case Builtin::BI__builtin_elementwise_bitreverse: {
     APSInt Val;
     if (!EvaluateInteger(E->getArg(0), Val, Info))
       return false;
@@ -13262,6 +13285,25 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     return Success(Val.rotr(Amt.urem(Val.getBitWidth())), E);
   }
 
+  case Builtin::BI__builtin_elementwise_add_sat: {
+    APSInt LHS, RHS;
+    if (!EvaluateInteger(E->getArg(0), LHS, Info) ||
+        !EvaluateInteger(E->getArg(1), RHS, Info))
+      return false;
+
+    APInt Result = LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
+    return Success(APSInt(Result, !LHS.isSigned()), E);
+  }
+  case Builtin::BI__builtin_elementwise_sub_sat: {
+    APSInt LHS, RHS;
+    if (!EvaluateInteger(E->getArg(0), LHS, Info) ||
+        !EvaluateInteger(E->getArg(1), RHS, Info))
+      return false;
+
+    APInt Result = LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
+    return Success(APSInt(Result, !LHS.isSigned()), E);
+  }
+
   case Builtin::BIstrlen:
   case Builtin::BIwcslen:
     // A call to strlen is not a constant expression.
@@ -16555,7 +16597,7 @@ static bool FastEvaluateAsRValue(const Expr *Exp, Expr::EvalResult &Result,
                                  const ASTContext &Ctx, bool &IsConst) {
   // Fast-path evaluations of integer literals, since we sometimes see files
   // containing vast quantities of these.
-  if (const IntegerLiteral *L = dyn_cast<IntegerLiteral>(Exp)) {
+  if (const auto *L = dyn_cast<IntegerLiteral>(Exp)) {
     Result.Val = APValue(APSInt(L->getValue(),
                                 L->getType()->isUnsignedIntegerType()));
     IsConst = true;
@@ -16568,6 +16610,18 @@ static bool FastEvaluateAsRValue(const Expr *Exp, Expr::EvalResult &Result,
     return true;
   }
 
+  if (const auto *FL = dyn_cast<FloatingLiteral>(Exp)) {
+    Result.Val = APValue(FL->getValue());
+    IsConst = true;
+    return true;
+  }
+
+  if (const auto *L = dyn_cast<CharacterLiteral>(Exp)) {
+    Result.Val = APValue(Ctx.MakeIntValue(L->getValue(), L->getType()));
+    IsConst = true;
+    return true;
+  }
+
   if (const auto *CE = dyn_cast<ConstantExpr>(Exp)) {
     if (CE->hasAPValueResult()) {
       APValue APV = CE->getAPValueResult();
diff --git a/clang/lib/AST/ExternalASTSource.cpp b/clang/lib/AST/ExternalASTSource.cpp
index 7a14cc7d50ed0..543846c0093af 100644
--- a/clang/lib/AST/ExternalASTSource.cpp
+++ b/clang/lib/AST/ExternalASTSource.cpp
@@ -96,6 +96,15 @@ ExternalASTSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
   return false;
 }
 
+bool ExternalASTSource::LoadExternalSpecializations(const Decl *D, bool) {
+  return false;
+}
+
+bool ExternalASTSource::LoadExternalSpecializations(
+    const Decl *D, ArrayRef<TemplateArgument>) {
+  return false;
+}
+
 void ExternalASTSource::completeVisibleDeclsMap(const DeclContext *DC) {}
 
 void ExternalASTSource::FindExternalLexicalDecls(
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index c91e71fb4fc47..f016bcad273c5 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -574,6 +574,7 @@ class CXXNameMangler {
   static StringRef getCallingConvQualifierName(CallingConv CC);
   void mangleExtParameterInfo(FunctionProtoType::ExtParameterInfo info);
   void mangleExtFunctionInfo(const FunctionType *T);
+  void mangleSMEAttrs(unsigned SMEAttrs);
   void mangleBareFunctionType(const FunctionProtoType *T, bool MangleReturnType,
                               const FunctionDecl *FD = nullptr);
   void mangleNeonVectorType(const VectorType *T);
@@ -3540,6 +3541,69 @@ void CXXNameMangler::mangleExtFunctionInfo(const FunctionType *T) {
   // FIXME: noreturn
 }
 
+enum class AAPCSBitmaskSME : unsigned {
+  ArmStreamingBit = 1 << 0,
+  ArmStreamingCompatibleBit = 1 << 1,
+  ArmAgnosticSMEZAStateBit = 1 << 2,
+  ZA_Shift = 3,
+  ZT0_Shift = 6,
+  NoState = 0b000,
+  ArmIn = 0b001,
+  ArmOut = 0b010,
+  ArmInOut = 0b011,
+  ArmPreserves = 0b100,
+  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ArmPreserves << ZT0_Shift)
+};
+
+static AAPCSBitmaskSME encodeAAPCSZAState(unsigned SMEAttrs) {
+  switch (SMEAttrs) {
+  case FunctionType::ARM_None:
+    return AAPCSBitmaskSME::NoState;
+  case FunctionType::ARM_In:
+    return AAPCSBitmaskSME::ArmIn;
+  case FunctionType::ARM_Out:
+    return AAPCSBitmaskSME::ArmOut;
+  case FunctionType::ARM_InOut:
+    return AAPCSBitmaskSME::ArmInOut;
+  case FunctionType::ARM_Preserves:
+    return AAPCSBitmaskSME::ArmPreserves;
+  default:
+    llvm_unreachable("Unrecognised SME attribute");
+  }
+}
+
+// The mangling scheme for function types which have SME attributes is
+// implemented as a "pseudo" template:
+//
+//   '__SME_ATTRS<<normal_function_type>, <sme_state>>'
+//
+// Combining the function type with a bitmask representing the streaming and ZA
+// properties of the function's interface.
+//
+// Mangling of SME keywords is described in more detail in the AArch64 ACLE:
+// https://github.com/ARM-software/acle/blob/main/main/acle.md#c-mangling-of-sme-keywords
+//
+void CXXNameMangler::mangleSMEAttrs(unsigned SMEAttrs) {
+  if (!SMEAttrs)
+    return;
+
+  AAPCSBitmaskSME Bitmask = AAPCSBitmaskSME(0);
+  if (SMEAttrs & FunctionType::SME_PStateSMEnabledMask)
+    Bitmask |= AAPCSBitmaskSME::ArmStreamingBit;
+  else if (SMEAttrs & FunctionType::SME_PStateSMCompatibleMask)
+    Bitmask |= AAPCSBitmaskSME::ArmStreamingCompatibleBit;
+
+  // TODO: Must represent __arm_agnostic("sme_za_state")
+
+  Bitmask |= encodeAAPCSZAState(FunctionType::getArmZAState(SMEAttrs))
+             << AAPCSBitmaskSME::ZA_Shift;
+
+  Bitmask |= encodeAAPCSZAState(FunctionType::getArmZT0State(SMEAttrs))
+             << AAPCSBitmaskSME::ZT0_Shift;
+
+  Out << "Lj" << static_cast<unsigned>(Bitmask) << "EE";
+}
+
 void
 CXXNameMangler::mangleExtParameterInfo(FunctionProtoType::ExtParameterInfo PI) {
   // Vendor-specific qualifiers are emitted in reverse alphabetical order.
@@ -3577,6 +3641,11 @@ CXXNameMangler::mangleExtParameterInfo(FunctionProtoType::ExtParameterInfo PI) {
 // <function-type> ::= [<CV-qualifiers>] F [Y]
 //                      <bare-function-type> [<ref-qualifier>] E
 void CXXNameMangler::mangleType(const FunctionProtoType *T) {
+  unsigned SMEAttrs = T->getAArch64SMEAttributes();
+
+  if (SMEAttrs)
+    Out << "11__SME_ATTRSI";
+
   mangleExtFunctionInfo(T);
 
   // Mangle CV-qualifiers, if present.  These are 'this' qualifiers,
@@ -3611,6 +3680,8 @@ void CXXNameMangler::mangleType(const FunctionProtoType *T) {
   mangleRefQualifier(T->getRefQualifier());
 
   Out << 'E';
+
+  mangleSMEAttrs(SMEAttrs);
 }
 
 void CXXNameMangler::mangleType(const FunctionNoProtoType *T) {
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 61d63a9e9b1d7..399665074e89d 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -1014,6 +1014,7 @@ void MicrosoftCXXNameMangler::mangleFloat(llvm::APFloat Number) {
   case APFloat::S_x87DoubleExtended: Out << 'X'; break;
   case APFloat::S_IEEEquad: Out << 'Y'; break;
   case APFloat::S_PPCDoubleDouble: Out << 'Z'; break;
+  case APFloat::S_PPCDoubleDoubleLegacy:
   case APFloat::S_Float8E5M2:
   case APFloat::S_Float8E4M3:
   case APFloat::S_Float8E4M3FN:
@@ -3405,7 +3406,15 @@ void MicrosoftCXXNameMangler::mangleType(const MemberPointerType *T,
 
 void MicrosoftCXXNameMangler::mangleType(const TemplateTypeParmType *T,
                                          Qualifiers, SourceRange Range) {
-  Error(Range.getBegin(), "template type parameter type") << Range;
+  Out << '?';
+
+  llvm::SmallString<64> Name;
+  Name += "<TTPT_";
+  Name += llvm::utostr(T->getDepth());
+  Name += "_";
+  Name += llvm::utostr(T->getIndex());
+  Name += ">";
+  mangleSourceName(Name);
 }
 
 void MicrosoftCXXNameMangler::mangleType(const SubstTemplateTypeParmPackType *T,
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index 645ca6f0e7b71..7c5c287e6c15b 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -818,15 +818,20 @@ void ODRHash::AddDecl(const Decl *D) {
 
   AddDeclarationName(ND->getDeclName());
 
-  const auto *Specialization =
-            dyn_cast<ClassTemplateSpecializationDecl>(D);
-  AddBoolean(Specialization);
-  if (Specialization) {
-    const TemplateArgumentList &List = Specialization->getTemplateArgs();
-    ID.AddInteger(List.size());
-    for (const TemplateArgument &TA : List.asArray())
-      AddTemplateArgument(TA);
-  }
+  // If this was a specialization we should take into account its template
+  // arguments. This helps to reduce collisions coming when visiting template
+  // specialization types (eg. when processing type template arguments).
+  ArrayRef<TemplateArgument> Args;
+  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D))
+    Args = CTSD->getTemplateArgs().asArray();
+  else if (auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(D))
+    Args = VTSD->getTemplateArgs().asArray();
+  else if (auto *FD = dyn_cast<FunctionDecl>(D))
+    if (FD->getTemplateSpecializationArgs())
+      Args = FD->getTemplateSpecializationArgs()->asArray();
+
+  for (auto &TA : Args)
+    AddTemplateArgument(TA);
 }
 
 namespace {
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 86adf1458b471..a5184c208cf37 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -818,7 +818,7 @@ class ItaniumRecordLayoutBuilder {
   void setSize(CharUnits NewSize) { Size = Context.toBits(NewSize); }
   void setSize(uint64_t NewSize) { Size = NewSize; }
 
-  CharUnits getAligment() const { return Alignment; }
+  CharUnits getAlignment() const { return Alignment; }
 
   CharUnits getDataSize() const {
     assert(DataSize % Context.getCharWidth() == 0);
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index e3e31a70b6a08..b602a50e30108 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -2548,7 +2548,10 @@ void StmtPrinter::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
 }
 
 void StmtPrinter::VisitPackIndexingExpr(PackIndexingExpr *E) {
-  OS << E->getPackIdExpression() << "...[" << E->getIndexExpr() << "]";
+  PrintExpr(E->getPackIdExpression());
+  OS << "...[";
+  PrintExpr(E->getIndexExpr());
+  OS << "]";
 }
 
 void StmtPrinter::VisitSubstNonTypeTemplateParmPackExpr(
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index b8820e511ecc7..c2c38495751de 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -267,6 +267,12 @@ void ConstantArrayType::Profile(llvm::FoldingSetNodeID &ID,
     SizeExpr->Profile(ID, Context, true);
 }
 
+QualType ArrayParameterType::getConstantArrayType(const ASTContext &Ctx) const {
+  return Ctx.getConstantArrayType(getElementType(), getSize(), getSizeExpr(),
+                                  getSizeModifier(),
+                                  getIndexTypeQualifiers().getAsOpaqueValue());
+}
+
 DependentSizedArrayType::DependentSizedArrayType(QualType et, QualType can,
                                                  Expr *e, ArraySizeModifier sm,
                                                  unsigned tq,
@@ -4729,7 +4735,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
   case Type::Pipe:
     return computeTypeLinkageInfo(cast<PipeType>(T)->getElementType());
   case Type::HLSLAttributedResource:
-    llvm_unreachable("not yet implemented");
+    return computeTypeLinkageInfo(cast<HLSLAttributedResourceType>(T)
+                                      ->getContainedType()
+                                      ->getCanonicalTypeInternal());
   }
 
   llvm_unreachable("unhandled type class");
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 46dd44e6f2b24..bf9dc5f2373f9 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -21,6 +21,7 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -697,20 +698,27 @@ static bool isTokenAtLoc(const SourceManager &SM, const LangOptions &LangOpts,
   return !Invalid && Text == TokenText;
 }
 
-std::optional<SourceLocation>
-getExpansionLocOfMacro(StringRef MacroName, SourceLocation Loc,
-                       const ASTContext &Context) {
+static std::optional<SourceLocation> getExpansionLocOfMacroRecursive(
+    StringRef MacroName, SourceLocation Loc, const ASTContext &Context,
+    llvm::DenseSet<SourceLocation> &CheckedLocations) {
   auto &SM = Context.getSourceManager();
   const LangOptions &LangOpts = Context.getLangOpts();
   while (Loc.isMacroID()) {
+    if (CheckedLocations.count(Loc))
+      return std::nullopt;
+    CheckedLocations.insert(Loc);
     SrcMgr::ExpansionInfo Expansion =
         SM.getSLocEntry(SM.getFileID(Loc)).getExpansion();
-    if (Expansion.isMacroArgExpansion())
+    if (Expansion.isMacroArgExpansion()) {
       // Check macro argument for an expansion of the given macro. For example,
       // `F(G(3))`, where `MacroName` is `G`.
-      if (std::optional<SourceLocation> ArgLoc = getExpansionLocOfMacro(
-              MacroName, Expansion.getSpellingLoc(), Context))
+      if (std::optional<SourceLocation> ArgLoc =
+              getExpansionLocOfMacroRecursive(MacroName,
+                                              Expansion.getSpellingLoc(),
+                                              Context, CheckedLocations)) {
         return ArgLoc;
+      }
+    }
     Loc = Expansion.getExpansionLocStart();
     if (isTokenAtLoc(SM, LangOpts, MacroName, Loc))
       return Loc;
@@ -718,6 +726,14 @@ getExpansionLocOfMacro(StringRef MacroName, SourceLocation Loc,
   return std::nullopt;
 }
 
+std::optional<SourceLocation>
+getExpansionLocOfMacro(StringRef MacroName, SourceLocation Loc,
+                       const ASTContext &Context) {
+  llvm::DenseSet<SourceLocation> CheckedLocations;
+  return getExpansionLocOfMacroRecursive(MacroName, Loc, Context,
+                                         CheckedLocations);
+}
+
 std::shared_ptr<llvm::Regex> createAndVerifyRegex(StringRef Regex,
                                                   llvm::Regex::RegexFlags Flags,
                                                   StringRef MatcherID) {
@@ -799,6 +815,7 @@ const internal::VariadicDynCastAllOfMatcher<TypeLoc, ElaboratedTypeLoc>
 
 const internal::VariadicDynCastAllOfMatcher<Stmt, UnaryExprOrTypeTraitExpr>
     unaryExprOrTypeTraitExpr;
+const internal::VariadicDynCastAllOfMatcher<Decl, ExportDecl> exportDecl;
 const internal::VariadicDynCastAllOfMatcher<Decl, ValueDecl> valueDecl;
 const internal::VariadicDynCastAllOfMatcher<Decl, CXXConstructorDecl>
     cxxConstructorDecl;
@@ -1097,7 +1114,8 @@ AST_TYPELOC_TRAVERSE_MATCHER_DEF(hasValueType,
 AST_TYPELOC_TRAVERSE_MATCHER_DEF(
     pointee,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BlockPointerType, MemberPointerType,
-                                    PointerType, ReferenceType));
+                                    PointerType, ReferenceType,
+                                    ObjCObjectPointerType));
 
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPExecutableDirective>
     ompExecutableDirective;
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 8d36ad5c80b5d..837633fb2f060 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -243,6 +243,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(equalsBoundNode);
   REGISTER_MATCHER(equalsIntegralValue);
   REGISTER_MATCHER(explicitCastExpr);
+  REGISTER_MATCHER(exportDecl);
   REGISTER_MATCHER(expr);
   REGISTER_MATCHER(exprWithCleanups);
   REGISTER_MATCHER(fieldDecl);
diff --git a/clang/lib/Analysis/Consumed.cpp b/clang/lib/Analysis/Consumed.cpp
index 63c5943242944..3eb7e5abff714 100644
--- a/clang/lib/Analysis/Consumed.cpp
+++ b/clang/lib/Analysis/Consumed.cpp
@@ -1229,6 +1229,9 @@ bool ConsumedAnalyzer::splitState(const CFGBlock *CurrBlock,
 
   if (const auto *IfNode =
           dyn_cast_or_null<IfStmt>(CurrBlock->getTerminator().getStmt())) {
+    if (IfNode->isConsteval())
+      return false;
+
     const Expr *Cond = IfNode->getCond();
 
     PInfo = Visitor.getInfo(Cond);
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index be0e8aa5743dd..53b838e9ede4d 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -55,25 +55,13 @@ static bool canExprResolveTo(const Expr *Source, const Expr *Target) {
   // This is matched by `IgnoreDerivedToBase(canResolveToExpr(InnerMatcher))`
   // below.
   const auto ConditionalOperatorM = [Target](const Expr *E) {
-    if (const auto *OP = dyn_cast<ConditionalOperator>(E)) {
-      if (const auto *TE = OP->getTrueExpr()->IgnoreParens())
-        if (canExprResolveTo(TE, Target))
-          return true;
-      if (const auto *FE = OP->getFalseExpr()->IgnoreParens())
-        if (canExprResolveTo(FE, Target))
-          return true;
-    }
-    return false;
-  };
-
-  const auto ElvisOperator = [Target](const Expr *E) {
-    if (const auto *OP = dyn_cast<BinaryConditionalOperator>(E)) {
-      if (const auto *TE = OP->getTrueExpr()->IgnoreParens())
-        if (canExprResolveTo(TE, Target))
-          return true;
-      if (const auto *FE = OP->getFalseExpr()->IgnoreParens())
-        if (canExprResolveTo(FE, Target))
-          return true;
+    if (const auto *CO = dyn_cast<AbstractConditionalOperator>(E)) {
+      const auto *TE = CO->getTrueExpr()->IgnoreParens();
+      if (TE && canExprResolveTo(TE, Target))
+        return true;
+      const auto *FE = CO->getFalseExpr()->IgnoreParens();
+      if (FE && canExprResolveTo(FE, Target))
+        return true;
     }
     return false;
   };
@@ -81,8 +69,7 @@ static bool canExprResolveTo(const Expr *Source, const Expr *Target) {
   const Expr *SourceExprP = Source->IgnoreParens();
   return IgnoreDerivedToBase(SourceExprP,
                              [&](const Expr *E) {
-                               return E == Target || ConditionalOperatorM(E) ||
-                                      ElvisOperator(E);
+                               return E == Target || ConditionalOperatorM(E);
                              }) ||
          EvalCommaExpr(SourceExprP, [&](const Expr *E) {
            return IgnoreDerivedToBase(
diff --git a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
index 9e7821bfc1e89..9a46e79cf4a67 100644
--- a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
+++ b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/FlowSensitive/ASTOps.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/ComputeDependence.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
@@ -282,6 +283,28 @@ ReferencedDecls getReferencedDecls(const FunctionDecl &FD) {
   if (const auto *CtorDecl = dyn_cast<CXXConstructorDecl>(&FD))
     Visitor.traverseConstructorInits(CtorDecl);
 
+  // If analyzing a lambda call operator, collect all captures of parameters (of
+  // the surrounding function). This collects them even if they are not
+  // referenced in the body of the lambda call operator. Non-parameter local
+  // variables that are captured are already collected into
+  // `ReferencedDecls.Locals` when traversing the call operator body, but we
+  // collect parameters here to avoid needing to check at each referencing node
+  // whether the parameter is a lambda capture from a surrounding function or is
+  // a parameter of the current function. If it becomes necessary to limit this
+  // set to the parameters actually referenced in the body, alternative
+  // optimizations can be implemented to minimize duplicative work.
+  if (const auto *Method = dyn_cast<CXXMethodDecl>(&FD);
+      Method && isLambdaCallOperator(Method)) {
+    for (const auto &Capture : Method->getParent()->captures()) {
+      if (Capture.capturesVariable()) {
+        if (const auto *Param =
+                dyn_cast<ParmVarDecl>(Capture.getCapturedVar())) {
+          Result.LambdaCapturedParams.insert(Param);
+        }
+      }
+    }
+  }
+
   return Result;
 }
 
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 321097e16a45f..40f529e52b44a 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -172,6 +172,12 @@ class MatchDescendantVisitor : public DynamicRecursiveASTVisitor {
     return DynamicRecursiveASTVisitor::TraverseCXXTypeidExpr(Node);
   }
 
+  bool TraverseCXXDefaultInitExpr(CXXDefaultInitExpr *Node) override {
+    if (!TraverseStmt(Node->getExpr()))
+      return false;
+    return DynamicRecursiveASTVisitor::TraverseCXXDefaultInitExpr(Node);
+  }
+
   bool TraverseStmt(Stmt *Node) override {
     if (!Node)
       return true;
@@ -1987,14 +1993,18 @@ class DerefSimplePtrArithFixableGadget : public FixableGadget {
 };
 
 /// Scan the function and return a list of gadgets found with provided kits.
-static std::tuple<FixableGadgetList, WarningGadgetList, DeclUseTracker>
-findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
-            bool EmitSuggestions) {
+static void findGadgets(const Stmt *S, ASTContext &Ctx,
+                        const UnsafeBufferUsageHandler &Handler,
+                        bool EmitSuggestions, FixableGadgetList &FixableGadgets,
+                        WarningGadgetList &WarningGadgets,
+                        DeclUseTracker &Tracker) {
 
   struct GadgetFinderCallback : MatchFinder::MatchCallback {
-    FixableGadgetList FixableGadgets;
-    WarningGadgetList WarningGadgets;
-    DeclUseTracker Tracker;
+    GadgetFinderCallback(FixableGadgetList &FixableGadgets,
+                         WarningGadgetList &WarningGadgets,
+                         DeclUseTracker &Tracker)
+        : FixableGadgets(FixableGadgets), WarningGadgets(WarningGadgets),
+          Tracker(Tracker) {}
 
     void run(const MatchFinder::MatchResult &Result) override {
       // In debug mode, assert that we've found exactly one gadget.
@@ -2035,10 +2045,14 @@ findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
       assert(numFound >= 1 && "Gadgets not found in match result!");
       assert(numFound <= 1 && "Conflicting bind tags in gadgets!");
     }
+
+    FixableGadgetList &FixableGadgets;
+    WarningGadgetList &WarningGadgets;
+    DeclUseTracker &Tracker;
   };
 
   MatchFinder M;
-  GadgetFinderCallback CB;
+  GadgetFinderCallback CB{FixableGadgets, WarningGadgets, Tracker};
 
   // clang-format off
   M.addMatcher(
@@ -2083,9 +2097,7 @@ findGadgets(const Decl *D, const UnsafeBufferUsageHandler &Handler,
     // clang-format on
   }
 
-  M.match(*D->getBody(), D->getASTContext());
-  return {std::move(CB.FixableGadgets), std::move(CB.WarningGadgets),
-          std::move(CB.Tracker)};
+  M.match(*S, Ctx);
 }
 
 // Compares AST nodes by source locations.
@@ -3630,39 +3642,9 @@ class VariableGroupsManagerImpl : public VariableGroupsManager {
   }
 };
 
-void clang::checkUnsafeBufferUsage(const Decl *D,
-                                   UnsafeBufferUsageHandler &Handler,
-                                   bool EmitSuggestions) {
-#ifndef NDEBUG
-  Handler.clearDebugNotes();
-#endif
-
-  assert(D && D->getBody());
-  // We do not want to visit a Lambda expression defined inside a method
-  // independently. Instead, it should be visited along with the outer method.
-  // FIXME: do we want to do the same thing for `BlockDecl`s?
-  if (const auto *fd = dyn_cast<CXXMethodDecl>(D)) {
-    if (fd->getParent()->isLambda() && fd->getParent()->isLocalClass())
-      return;
-  }
-
-  // Do not emit fixit suggestions for functions declared in an
-  // extern "C" block.
-  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
-    for (FunctionDecl *FReDecl : FD->redecls()) {
-      if (FReDecl->isExternC()) {
-        EmitSuggestions = false;
-        break;
-      }
-    }
-  }
-
-  WarningGadgetSets UnsafeOps;
-  FixableGadgetSets FixablesForAllVars;
-
-  auto [FixableGadgets, WarningGadgets, Tracker] =
-      findGadgets(D, Handler, EmitSuggestions);
-
+void applyGadgets(const Decl *D, FixableGadgetList FixableGadgets,
+                  WarningGadgetList WarningGadgets, DeclUseTracker Tracker,
+                  UnsafeBufferUsageHandler &Handler, bool EmitSuggestions) {
   if (!EmitSuggestions) {
     // Our job is very easy without suggestions. Just warn about
     // every problematic operation and consider it done. No need to deal
@@ -3706,8 +3688,10 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
   if (WarningGadgets.empty())
     return;
 
-  UnsafeOps = groupWarningGadgetsByVar(std::move(WarningGadgets));
-  FixablesForAllVars = groupFixablesByVar(std::move(FixableGadgets));
+  WarningGadgetSets UnsafeOps =
+      groupWarningGadgetsByVar(std::move(WarningGadgets));
+  FixableGadgetSets FixablesForAllVars =
+      groupFixablesByVar(std::move(FixableGadgets));
 
   std::map<const VarDecl *, FixItList> FixItsForVariableGroup;
 
@@ -3928,3 +3912,56 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
     }
   }
 }
+
+void clang::checkUnsafeBufferUsage(const Decl *D,
+                                   UnsafeBufferUsageHandler &Handler,
+                                   bool EmitSuggestions) {
+#ifndef NDEBUG
+  Handler.clearDebugNotes();
+#endif
+
+  assert(D);
+
+  SmallVector<Stmt *> Stmts;
+
+  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
+    // We do not want to visit a Lambda expression defined inside a method
+    // independently. Instead, it should be visited along with the outer method.
+    // FIXME: do we want to do the same thing for `BlockDecl`s?
+    if (const auto *MD = dyn_cast<CXXMethodDecl>(D)) {
+      if (MD->getParent()->isLambda() && MD->getParent()->isLocalClass())
+        return;
+    }
+
+    for (FunctionDecl *FReDecl : FD->redecls()) {
+      if (FReDecl->isExternC()) {
+        // Do not emit fixit suggestions for functions declared in an
+        // extern "C" block.
+        EmitSuggestions = false;
+        break;
+      }
+    }
+
+    Stmts.push_back(FD->getBody());
+
+    if (const auto *ID = dyn_cast<CXXConstructorDecl>(D)) {
+      for (const CXXCtorInitializer *CI : ID->inits()) {
+        Stmts.push_back(CI->getInit());
+      }
+    }
+  } else if (isa<BlockDecl>(D) || isa<ObjCMethodDecl>(D)) {
+    Stmts.push_back(D->getBody());
+  }
+
+  assert(!Stmts.empty());
+
+  FixableGadgetList FixableGadgets;
+  WarningGadgetList WarningGadgets;
+  DeclUseTracker Tracker;
+  for (Stmt *S : Stmts) {
+    findGadgets(S, D->getASTContext(), Handler, EmitSuggestions, FixableGadgets,
+                WarningGadgets, Tracker);
+  }
+  applyGadgets(D, std::move(FixableGadgets), std::move(WarningGadgets),
+               std::move(Tracker), Handler, EmitSuggestions);
+}
diff --git a/clang/lib/Basic/Builtins.cpp b/clang/lib/Basic/Builtins.cpp
index 1f5cbbfebcdca..289843742e863 100644
--- a/clang/lib/Basic/Builtins.cpp
+++ b/clang/lib/Basic/Builtins.cpp
@@ -29,54 +29,93 @@ const char *HeaderDesc::getName() const {
   llvm_unreachable("Unknown HeaderDesc::HeaderID enum");
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-    {"not a builtin function", nullptr, nullptr, nullptr, HeaderDesc::NO_HEADER,
-     ALL_LANGUAGES},
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANGS)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANGS},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER, LANGS)                             \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, LANGS},
+static constexpr auto BuiltinStorage =
+    Builtin::Storage<Builtin::FirstTSBuiltin>::Make(
+        CLANG_BUILTIN_STR_TABLE("not a builtin function", "", "")
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
 #include "clang/Basic/Builtins.inc"
-};
+            ,
+        {CLANG_BUILTIN_ENTRY("not a builtin function", "", "")
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LANGBUILTIN CLANG_LANGBUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#include "clang/Basic/Builtins.inc"
+        });
 
-const Builtin::Info &Builtin::Context::getRecord(unsigned ID) const {
+std::pair<const char *, const Builtin::Info &>
+Builtin::Context::getStrTableAndInfo(unsigned ID) const {
   if (ID < Builtin::FirstTSBuiltin)
-    return BuiltinInfo[ID];
-  assert(((ID - Builtin::FirstTSBuiltin) <
-          (TSRecords.size() + AuxTSRecords.size())) &&
-         "Invalid builtin ID!");
+    return {BuiltinStorage.StringTable, BuiltinStorage.Infos[ID]};
+  assert(
+      ((ID - Builtin::FirstTSBuiltin) < (TSInfos.size() + AuxTSInfos.size())) &&
+      "Invalid builtin ID!");
   if (isAuxBuiltinID(ID))
-    return AuxTSRecords[getAuxBuiltinID(ID) - Builtin::FirstTSBuiltin];
-  return TSRecords[ID - Builtin::FirstTSBuiltin];
+    return {AuxTSStrTable,
+            AuxTSInfos[getAuxBuiltinID(ID) - Builtin::FirstTSBuiltin]};
+  return {TSStrTable, TSInfos[ID - Builtin::FirstTSBuiltin]};
+}
+
+static llvm::StringRef getStrFromTable(const char *StrTable, int Offset) {
+  return &StrTable[Offset];
+}
+
+/// Return the identifier name for the specified builtin,
+/// e.g. "__builtin_abs".
+llvm::StringRef Builtin::Context::getName(unsigned ID) const {
+  const auto &[StrTable, I] = getStrTableAndInfo(ID);
+  return getStrFromTable(StrTable, I.Offsets.Name);
+}
+
+const char *Builtin::Context::getTypeString(unsigned ID) const {
+  const auto &[StrTable, I] = getStrTableAndInfo(ID);
+  return getStrFromTable(StrTable, I.Offsets.Type).data();
+}
+
+const char *Builtin::Context::getAttributesString(unsigned ID) const {
+  const auto &[StrTable, I] = getStrTableAndInfo(ID);
+  return getStrFromTable(StrTable, I.Offsets.Attributes).data();
+}
+
+const char *Builtin::Context::getRequiredFeatures(unsigned ID) const {
+  const auto &[StrTable, I] = getStrTableAndInfo(ID);
+  return getStrFromTable(StrTable, I.Offsets.Features).data();
 }
 
 void Builtin::Context::InitializeTarget(const TargetInfo &Target,
                                         const TargetInfo *AuxTarget) {
-  assert(TSRecords.empty() && "Already initialized target?");
-  TSRecords = Target.getTargetBuiltins();
-  if (AuxTarget)
-    AuxTSRecords = AuxTarget->getTargetBuiltins();
+  assert(TSStrTable == nullptr && "Already initialized target?");
+  assert(TSInfos.empty() && "Already initialized target?");
+  std::tie(TSStrTable, TSInfos) = Target.getTargetBuiltinStorage();
+  if (AuxTarget) {
+    std::tie(AuxTSStrTable, AuxTSInfos) = AuxTarget->getTargetBuiltinStorage();
+  }
 }
 
 bool Builtin::Context::isBuiltinFunc(llvm::StringRef FuncName) {
   bool InStdNamespace = FuncName.consume_front("std-");
+  const char *StrTable = BuiltinStorage.StringTable;
   for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin;
        ++i) {
-    if (FuncName == BuiltinInfo[i].Name &&
-        (bool)strchr(BuiltinInfo[i].Attributes, 'z') == InStdNamespace)
-      return strchr(BuiltinInfo[i].Attributes, 'f') != nullptr;
+    const auto &I = BuiltinStorage.Infos[i];
+    if (FuncName == getStrFromTable(StrTable, I.Offsets.Name) &&
+        (bool)strchr(getStrFromTable(StrTable, I.Offsets.Attributes).data(),
+                     'z') == InStdNamespace)
+      return strchr(getStrFromTable(StrTable, I.Offsets.Attributes).data(),
+                    'f') != nullptr;
   }
 
   return false;
 }
 
 /// Is this builtin supported according to the given language options?
-static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
+static bool builtinIsSupported(const char *StrTable,
+                               const Builtin::Info &BuiltinInfo,
                                const LangOptions &LangOpts) {
+  auto AttributesStr =
+      getStrFromTable(StrTable, BuiltinInfo.Offsets.Attributes);
+
   /* Builtins Unsupported */
-  if (LangOpts.NoBuiltin && strchr(BuiltinInfo.Attributes, 'f') != nullptr)
+  if (LangOpts.NoBuiltin && strchr(AttributesStr.data(), 'f') != nullptr)
     return false;
   /* CorBuiltins Unsupported */
   if (!LangOpts.Coroutines && (BuiltinInfo.Langs & COR_LANG))
@@ -126,7 +165,7 @@ static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
   if (!LangOpts.CPlusPlus && BuiltinInfo.Langs == CXX_LANG)
     return false;
   /* consteval Unsupported */
-  if (!LangOpts.CPlusPlus20 && strchr(BuiltinInfo.Attributes, 'G') != nullptr)
+  if (!LangOpts.CPlusPlus20 && strchr(AttributesStr.data(), 'G') != nullptr)
     return false;
   return true;
 }
@@ -137,20 +176,23 @@ static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
 void Builtin::Context::initializeBuiltins(IdentifierTable &Table,
                                           const LangOptions& LangOpts) {
   // Step #1: mark all target-independent builtins with their ID's.
-  for (unsigned i = Builtin::NotBuiltin+1; i != Builtin::FirstTSBuiltin; ++i)
-    if (builtinIsSupported(BuiltinInfo[i], LangOpts)) {
-      Table.get(BuiltinInfo[i].Name).setBuiltinID(i);
+  for (const auto &&[Index, I] :
+       llvm::enumerate(llvm::ArrayRef(BuiltinStorage.Infos).drop_front()))
+    if (builtinIsSupported(BuiltinStorage.StringTable, I, LangOpts)) {
+      Table.get(getStrFromTable(BuiltinStorage.StringTable, I.Offsets.Name))
+          .setBuiltinID(Index + 1);
     }
 
   // Step #2: Register target-specific builtins.
-  for (unsigned i = 0, e = TSRecords.size(); i != e; ++i)
-    if (builtinIsSupported(TSRecords[i], LangOpts))
-      Table.get(TSRecords[i].Name).setBuiltinID(i + Builtin::FirstTSBuiltin);
+  for (const auto &&[Index, I] : llvm::enumerate(TSInfos))
+    if (builtinIsSupported(TSStrTable, I, LangOpts))
+      Table.get(getStrFromTable(TSStrTable, I.Offsets.Name))
+          .setBuiltinID(Index + Builtin::FirstTSBuiltin);
 
   // Step #3: Register target-specific builtins for AuxTarget.
-  for (unsigned i = 0, e = AuxTSRecords.size(); i != e; ++i)
-    Table.get(AuxTSRecords[i].Name)
-        .setBuiltinID(i + Builtin::FirstTSBuiltin + TSRecords.size());
+  for (const auto &&[Index, I] : llvm::enumerate(AuxTSInfos))
+    Table.get(getStrFromTable(AuxTSStrTable, I.Offsets.Name))
+        .setBuiltinID(Index + Builtin::FirstTSBuiltin + TSInfos.size());
 
   // Step #4: Unregister any builtins specified by -fno-builtin-foo.
   for (llvm::StringRef Name : LangOpts.NoBuiltinFuncs) {
@@ -167,7 +209,7 @@ void Builtin::Context::initializeBuiltins(IdentifierTable &Table,
 }
 
 unsigned Builtin::Context::getRequiredVectorWidth(unsigned ID) const {
-  const char *WidthPos = ::strchr(getRecord(ID).Attributes, 'V');
+  const char *WidthPos = ::strchr(getAttributesString(ID), 'V');
   if (!WidthPos)
     return 0;
 
@@ -190,7 +232,7 @@ bool Builtin::Context::isLike(unsigned ID, unsigned &FormatIdx,
   assert(::toupper(Fmt[0]) == Fmt[1] &&
          "Format string is not in the form \"xX\"");
 
-  const char *Like = ::strpbrk(getRecord(ID).Attributes, Fmt);
+  const char *Like = ::strpbrk(getAttributesString(ID), Fmt);
   if (!Like)
     return false;
 
@@ -217,7 +259,7 @@ bool Builtin::Context::isScanfLike(unsigned ID, unsigned &FormatIdx,
 
 bool Builtin::Context::performsCallback(unsigned ID,
                                         SmallVectorImpl<int> &Encoding) const {
-  const char *CalleePos = ::strchr(getRecord(ID).Attributes, 'C');
+  const char *CalleePos = ::strchr(getAttributesString(ID), 'C');
   if (!CalleePos)
     return false;
 
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index 2d0e358116362..ae71758bc81e0 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -500,7 +500,8 @@ class WarningsSpecialCaseList : public llvm::SpecialCaseList {
   // the last section take precedence in such cases.
   void processSections(DiagnosticsEngine &Diags);
 
-  bool isDiagSuppressed(diag::kind DiagId, StringRef FilePath) const;
+  bool isDiagSuppressed(diag::kind DiagId, SourceLocation DiagLoc,
+                        const SourceManager &SM) const;
 
 private:
   // Find the longest glob pattern that matches FilePath amongst
@@ -573,13 +574,14 @@ void DiagnosticsEngine::setDiagSuppressionMapping(llvm::MemoryBuffer &Input) {
   WarningSuppressionList->processSections(*this);
   DiagSuppressionMapping =
       [WarningSuppressionList(std::move(WarningSuppressionList))](
-          diag::kind DiagId, StringRef Path) {
-        return WarningSuppressionList->isDiagSuppressed(DiagId, Path);
+          diag::kind DiagId, SourceLocation DiagLoc, const SourceManager &SM) {
+        return WarningSuppressionList->isDiagSuppressed(DiagId, DiagLoc, SM);
       };
 }
 
 bool WarningsSpecialCaseList::isDiagSuppressed(diag::kind DiagId,
-                                               StringRef FilePath) const {
+                                               SourceLocation DiagLoc,
+                                               const SourceManager &SM) const {
   const Section *DiagSection = DiagToSection.lookup(DiagId);
   if (!DiagSection)
     return false;
@@ -589,7 +591,13 @@ bool WarningsSpecialCaseList::isDiagSuppressed(diag::kind DiagId,
     return false;
   const llvm::StringMap<llvm::SpecialCaseList::Matcher> &CategoriesToMatchers =
       SrcEntriesIt->getValue();
-  return globsMatches(CategoriesToMatchers, FilePath);
+  // We also use presumed locations here to improve reproducibility for
+  // preprocessed inputs.
+  if (PresumedLoc PLoc = SM.getPresumedLoc(DiagLoc); PLoc.isValid())
+    return globsMatches(
+        CategoriesToMatchers,
+        llvm::sys::path::remove_leading_dotslash(PLoc.getFilename()));
+  return false;
 }
 
 bool WarningsSpecialCaseList::globsMatches(
@@ -614,8 +622,10 @@ bool WarningsSpecialCaseList::globsMatches(
 }
 
 bool DiagnosticsEngine::isSuppressedViaMapping(diag::kind DiagId,
-                                               StringRef FilePath) const {
-  return DiagSuppressionMapping && DiagSuppressionMapping(DiagId, FilePath);
+                                               SourceLocation DiagLoc) const {
+  if (!hasSourceManager() || !DiagSuppressionMapping)
+    return false;
+  return DiagSuppressionMapping(DiagId, DiagLoc, getSourceManager());
 }
 
 void DiagnosticsEngine::Report(const StoredDiagnostic &storedDiag) {
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index 44922aa7872db..d77f28c80b2eb 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -601,15 +601,8 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
     return diag::Severity::Ignored;
 
   // Clang-diagnostics pragmas always take precedence over suppression mapping.
-  if (!Mapping.isPragma() && Diag.DiagSuppressionMapping) {
-    // We also use presumed locations here to improve reproducibility for
-    // preprocessed inputs.
-    if (PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-        PLoc.isValid() && Diag.isSuppressedViaMapping(
-                              DiagID, llvm::sys::path::remove_leading_dotslash(
-                                          PLoc.getFilename())))
-      return diag::Severity::Ignored;
-  }
+  if (!Mapping.isPragma() && Diag.isSuppressedViaMapping(DiagID, Loc))
+    return diag::Severity::Ignored;
 
   return Result;
 }
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index eb8a3ada03448..858f83d090d7a 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -26,35 +26,39 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#include "clang/Basic/BuiltinsNEON.def"
+static constexpr int NumBuiltins =
+    clang::AArch64::LastTSBuiltin - Builtin::FirstTSBuiltin;
 
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsNEON.def"
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsSVE.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsSME.def"
-
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANG)                                     \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANG},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsAArch64.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsNEON.def"
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsSVE.def"
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsSME.def"
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LANGBUILTIN CLANG_LANGBUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsAArch64.def"
+      });
 
 void AArch64TargetInfo::setArchFeatures() {
   if (*ArchInfo == llvm::AArch64::ARMV8R) {
@@ -232,14 +236,23 @@ bool AArch64TargetInfo::validateTarget(DiagnosticsEngine &Diags) const {
 
 bool AArch64TargetInfo::validateGlobalRegisterVariable(
     StringRef RegName, unsigned RegSize, bool &HasSizeMismatch) const {
-  if ((RegName == "sp") || RegName.starts_with("x")) {
+  if (RegName == "sp") {
     HasSizeMismatch = RegSize != 64;
     return true;
-  } else if (RegName.starts_with("w")) {
-    HasSizeMismatch = RegSize != 32;
-    return true;
   }
-  return false;
+  if (RegName.starts_with("w"))
+    HasSizeMismatch = RegSize != 32;
+  else if (RegName.starts_with("x"))
+    HasSizeMismatch = RegSize != 64;
+  else
+    return false;
+  StringRef RegNum = RegName.drop_front();
+  // Check if the register is reserved. See also
+  // AArch64TargetLowering::getRegisterByName().
+  return RegNum == "0" ||
+         (RegNum == "18" &&
+          llvm::AArch64::isX18ReservedByDefault(getTriple())) ||
+         getTargetOpts().FeatureMap.lookup(("reserve-x" + RegNum).str());
 }
 
 bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef,
@@ -447,6 +460,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__ARM_FP16_FORMAT_IEEE", "1");
   Builder.defineMacro("__ARM_FP16_ARGS", "1");
 
+  // Clang supports arm_neon_sve_bridge.h
+  Builder.defineMacro("__ARM_NEON_SVE_BRIDGE", "1");
+
   if (Opts.UnsafeFPMath)
     Builder.defineMacro("__ARM_FP_FAST", "1");
 
@@ -464,9 +480,6 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (FPU & SveMode)
     Builder.defineMacro("__ARM_FEATURE_SVE", "1");
 
-  if ((FPU & NeonMode) && (FPU & SveMode))
-    Builder.defineMacro("__ARM_NEON_SVE_BRIDGE", "1");
-
   if (HasSVE2)
     Builder.defineMacro("__ARM_FEATURE_SVE2", "1");
 
@@ -688,9 +701,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-ArrayRef<Builtin::Info> AArch64TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::AArch64::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+AArch64TargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 std::optional<std::pair<unsigned, unsigned>>
@@ -705,18 +718,8 @@ AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
   return std::nullopt;
 }
 
-unsigned AArch64TargetInfo::multiVersionSortPriority(StringRef Name) const {
-  if (Name == "default")
-    return 0;
-  if (auto Ext = llvm::AArch64::parseFMVExtension(Name))
-    return Ext->Priority;
-  return 0;
-}
-
-unsigned AArch64TargetInfo::multiVersionFeatureCost() const {
-  // Take the maximum priority as per feature cost, so more features win.
-  constexpr unsigned MaxFMVPriority = 1000;
-  return MaxFMVPriority;
+unsigned AArch64TargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
+  return llvm::AArch64::getFMVPriority(Features);
 }
 
 bool AArch64TargetInfo::doesFeatureAffectCodeGen(StringRef Name) const {
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 4c25bdb5bb16d..a3ce04e249e30 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -137,8 +137,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
   bool setCPU(const std::string &Name) override;
 
-  unsigned multiVersionSortPriority(StringRef Name) const override;
-  unsigned multiVersionFeatureCost() const override;
+  unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
 
   bool useFP16ConversionIntrinsics() const override {
     return false;
@@ -181,7 +180,8 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   std::optional<std::pair<unsigned, unsigned>>
   getVScaleRange(const LangOptions &LangOpts) const override;
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 99f8f2944e279..21281e7201b02 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -88,13 +88,18 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
 } // namespace targets
 } // namespace clang
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::AMDGPU::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsAMDGPU.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsAMDGPU.def"
+      });
 
 const char *const AMDGPUTargetInfo::GCCRegNames[] = {
   "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
@@ -266,9 +271,9 @@ void AMDGPUTargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
                      !isAMDGCN(getTriple()));
 }
 
-ArrayRef<Builtin::Info> AMDGPUTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::AMDGPU::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+AMDGPUTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 void AMDGPUTargetInfo::getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index a36f11b5c72bc..10fd855e927f2 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -257,7 +257,8 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
                  StringRef CPU,
                  const std::vector<std::string> &FeatureVec) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool useFP16ConversionIntrinsics() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/ARC.h b/clang/lib/Basic/Targets/ARC.h
index 7f3d0aa15ab81..d84daf2e527ca 100644
--- a/clang/lib/Basic/Targets/ARC.h
+++ b/clang/lib/Basic/Targets/ARC.h
@@ -40,7 +40,10 @@ class LLVM_LIBRARY_VISIBILITY ARCTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 370444057b429..fb10bc8249e98 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -1071,31 +1071,34 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::ARM::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsNEON.def"
 
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LANGBUILTIN(ID, TYPE, ATTRS, LANG)                                     \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, LANG},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsARM.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsNEON.def"
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LANGBUILTIN CLANG_LANGBUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsARM.def"
+      });
 
-ArrayRef<Builtin::Info> ARMTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::ARM::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+ARMTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 bool ARMTargetInfo::isCLZForZeroUndef() const { return false; }
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index 55ecb99d82d8f..71f44f7248f68 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -196,7 +196,8 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool isCLZForZeroUndef() const override;
   BuiltinVaListKind getBuiltinVaListKind() const override;
diff --git a/clang/lib/Basic/Targets/AVR.h b/clang/lib/Basic/Targets/AVR.h
index df1f8d171efba..f19dee3a88231 100644
--- a/clang/lib/Basic/Targets/AVR.h
+++ b/clang/lib/Basic/Targets/AVR.h
@@ -63,7 +63,10 @@ class LLVM_LIBRARY_VISIBILITY AVRTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp
index f4684765b7ffb..b62e792590185 100644
--- a/clang/lib/Basic/Targets/BPF.cpp
+++ b/clang/lib/Basic/Targets/BPF.cpp
@@ -19,11 +19,16 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::BPF::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsBPF.inc"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsBPF.inc"
-};
+      });
 
 void BPFTargetInfo::getTargetDefines(const LangOptions &Opts,
                                      MacroBuilder &Builder) const {
@@ -81,9 +86,9 @@ void BPFTargetInfo::fillValidCPUList(SmallVectorImpl<StringRef> &Values) const {
   Values.append(std::begin(ValidCPUNames), std::end(ValidCPUNames));
 }
 
-ArrayRef<Builtin::Info> BPFTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::BPF::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+BPFTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 bool BPFTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h
index 27a4b5f314970..8f5c0e31aa87c 100644
--- a/clang/lib/Basic/Targets/BPF.h
+++ b/clang/lib/Basic/Targets/BPF.h
@@ -58,7 +58,8 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/CSKY.cpp b/clang/lib/Basic/Targets/CSKY.cpp
index c8bf8b9234d24..e698508a2370c 100644
--- a/clang/lib/Basic/Targets/CSKY.cpp
+++ b/clang/lib/Basic/Targets/CSKY.cpp
@@ -139,10 +139,6 @@ bool CSKYTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   return true;
 }
 
-ArrayRef<Builtin::Info> CSKYTargetInfo::getTargetBuiltins() const {
-  return ArrayRef<Builtin::Info>();
-}
-
 ArrayRef<const char *> CSKYTargetInfo::getGCCRegNames() const {
   static const char *const GCCRegNames[] = {
       // Integer registers
diff --git a/clang/lib/Basic/Targets/CSKY.h b/clang/lib/Basic/Targets/CSKY.h
index 94d4eeb9a1fff..59c83340b8c31 100644
--- a/clang/lib/Basic/Targets/CSKY.h
+++ b/clang/lib/Basic/Targets/CSKY.h
@@ -73,7 +73,10 @@ class LLVM_LIBRARY_VISIBILITY CSKYTargetInfo : public TargetInfo {
 
   unsigned getMinGlobalAlign(uint64_t, bool HasNonWeakDef) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/DirectX.h b/clang/lib/Basic/Targets/DirectX.h
index ab22d1281a4df..3e6e84b0d9e6f 100644
--- a/clang/lib/Basic/Targets/DirectX.h
+++ b/clang/lib/Basic/Targets/DirectX.h
@@ -72,7 +72,10 @@ class LLVM_LIBRARY_VISIBILITY DirectXTargetInfo : public TargetInfo {
     return Feature == "directx";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp
index 0282ac812c306..70059d68c562b 100644
--- a/clang/lib/Basic/Targets/Hexagon.cpp
+++ b/clang/lib/Basic/Targets/Hexagon.cpp
@@ -198,15 +198,19 @@ ArrayRef<TargetInfo::GCCRegAlias> HexagonTargetInfo::getGCCRegAliases() const {
   return llvm::ArrayRef(GCCRegAliases);
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::Hexagon::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsHexagon.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsHexagon.def"
+      });
 
 bool HexagonTargetInfo::hasFeature(StringRef Feature) const {
   std::string VS = "hvxv" + HVXVersion;
@@ -264,7 +268,7 @@ void HexagonTargetInfo::fillValidCPUList(
     Values.push_back(Suffix.Name);
 }
 
-ArrayRef<Builtin::Info> HexagonTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::Hexagon::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+HexagonTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/Hexagon.h b/clang/lib/Basic/Targets/Hexagon.h
index 7f053ab7e4888..2230735b8971b 100644
--- a/clang/lib/Basic/Targets/Hexagon.h
+++ b/clang/lib/Basic/Targets/Hexagon.h
@@ -66,7 +66,8 @@ class LLVM_LIBRARY_VISIBILITY HexagonTargetInfo : public TargetInfo {
     BoolWidth = BoolAlign = 8;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &Info) const override {
diff --git a/clang/lib/Basic/Targets/Lanai.h b/clang/lib/Basic/Targets/Lanai.h
index f7e439c7c9e1c..f37b0fdb08731 100644
--- a/clang/lib/Basic/Targets/Lanai.h
+++ b/clang/lib/Basic/Targets/Lanai.h
@@ -78,7 +78,10 @@ class LLVM_LIBRARY_VISIBILITY LanaiTargetInfo : public TargetInfo {
     return TargetInfo::VoidPtrBuiltinVaList;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override {
diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp
index 9ed695358b1ca..7c714000af42f 100644
--- a/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/clang/lib/Basic/Targets/LoongArch.cpp
@@ -205,8 +205,8 @@ void LoongArchTargetInfo::getTargetDefines(const LangOptions &Opts,
       // TODO: As more features of the V1.1 ISA are supported, a unified "v1.1"
       // arch feature set will be used to include all sub-features belonging to
       // the V1.1 ISA version.
-      if (HasFeatureFrecipe && HasFeatureLAM_BH && HasFeatureLD_SEQ_SA &&
-          HasFeatureDiv32)
+      if (HasFeatureFrecipe && HasFeatureLAM_BH && HasFeatureLAMCAS &&
+          HasFeatureLD_SEQ_SA && HasFeatureDiv32)
         Builder.defineMacro("__loongarch_arch",
                             Twine('"') + "la64v1.1" + Twine('"'));
       else
@@ -240,6 +240,9 @@ void LoongArchTargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasFeatureLAM_BH)
     Builder.defineMacro("__loongarch_lam_bh", Twine(1));
 
+  if (HasFeatureLAMCAS)
+    Builder.defineMacro("__loongarch_lamcas", Twine(1));
+
   if (HasFeatureLD_SEQ_SA)
     Builder.defineMacro("__loongarch_ld_seq_sa", Twine(1));
 
@@ -267,13 +270,18 @@ void LoongArchTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8");
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::LoongArch::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsLoongArch.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsLoongArch.def"
-};
+      });
 
 bool LoongArchTargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
@@ -300,9 +308,9 @@ bool LoongArchTargetInfo::hasFeature(StringRef Feature) const {
       .Default(false);
 }
 
-ArrayRef<Builtin::Info> LoongArchTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::LoongArch::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+LoongArchTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 bool LoongArchTargetInfo::handleTargetFeatures(
@@ -324,6 +332,8 @@ bool LoongArchTargetInfo::handleTargetFeatures(
       HasFeatureFrecipe = true;
     else if (Feature == "+lam-bh")
       HasFeatureLAM_BH = true;
+    else if (Feature == "+lamcas")
+      HasFeatureLAMCAS = true;
     else if (Feature == "+ld-seq-sa")
       HasFeatureLD_SEQ_SA = true;
     else if (Feature == "+div32")
diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h
index 3002a0bbc4491..1bb82667cb512 100644
--- a/clang/lib/Basic/Targets/LoongArch.h
+++ b/clang/lib/Basic/Targets/LoongArch.h
@@ -31,6 +31,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
   bool HasFeatureLASX;
   bool HasFeatureFrecipe;
   bool HasFeatureLAM_BH;
+  bool HasFeatureLAMCAS;
   bool HasFeatureLD_SEQ_SA;
   bool HasFeatureDiv32;
 
@@ -43,6 +44,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
     HasFeatureLASX = false;
     HasFeatureFrecipe = false;
     HasFeatureLAM_BH = false;
+    HasFeatureLAMCAS = false;
     HasFeatureLD_SEQ_SA = false;
     HasFeatureDiv32 = false;
     LongDoubleWidth = 128;
@@ -68,7 +70,8 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index b5b29fd867563..5670c6309e717 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -115,9 +115,10 @@ void M68kTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__HAVE_68881__");
 }
 
-ArrayRef<Builtin::Info> M68kTargetInfo::getTargetBuiltins() const {
+std::pair<const char *, ArrayRef<Builtin::Info>>
+M68kTargetInfo::getTargetBuiltinStorage() const {
   // FIXME: Implement.
-  return {};
+  return {nullptr, {}};
 }
 
 bool M68kTargetInfo::hasFeature(StringRef Feature) const {
diff --git a/clang/lib/Basic/Targets/M68k.h b/clang/lib/Basic/Targets/M68k.h
index b732add77e034..c68b2ab132ea0 100644
--- a/clang/lib/Basic/Targets/M68k.h
+++ b/clang/lib/Basic/Targets/M68k.h
@@ -44,7 +44,8 @@ class LLVM_LIBRARY_VISIBILITY M68kTargetInfo : public TargetInfo {
 
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
   bool hasFeature(StringRef Feature) const override;
   ArrayRef<const char *> getGCCRegNames() const override;
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override;
diff --git a/clang/lib/Basic/Targets/MSP430.h b/clang/lib/Basic/Targets/MSP430.h
index 2266ada25c1dd..462bcdfaa9b21 100644
--- a/clang/lib/Basic/Targets/MSP430.h
+++ b/clang/lib/Basic/Targets/MSP430.h
@@ -50,9 +50,10 @@ class LLVM_LIBRARY_VISIBILITY MSP430TargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
     // FIXME: Implement.
-    return {};
+    return {nullptr, {}};
   }
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
diff --git a/clang/lib/Basic/Targets/Mips.cpp b/clang/lib/Basic/Targets/Mips.cpp
index 174bc9d2ab996..b4f137b2c1950 100644
--- a/clang/lib/Basic/Targets/Mips.cpp
+++ b/clang/lib/Basic/Targets/Mips.cpp
@@ -20,13 +20,17 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::Mips::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsMips.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsMips.def"
-};
+      });
 
 bool MipsTargetInfo::processorSupportsGPR64() const {
   return llvm::StringSwitch<bool>(CPU)
@@ -223,9 +227,9 @@ bool MipsTargetInfo::hasFeature(StringRef Feature) const {
       .Default(false);
 }
 
-ArrayRef<Builtin::Info> MipsTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::Mips::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+MipsTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 unsigned MipsTargetInfo::getUnwindWordWidth() const {
diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h
index 8acaf56523b21..2749b5d43a45d 100644
--- a/clang/lib/Basic/Targets/Mips.h
+++ b/clang/lib/Basic/Targets/Mips.h
@@ -197,7 +197,8 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool hasFeature(StringRef Feature) const override;
 
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 693152e1741c4..57671c89d50f3 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -20,15 +20,19 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::NVPTX::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsNVPTX.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsNVPTX.def"
-};
+      });
 
 const char *const NVPTXTargetInfo::GCCRegNames[] = {"r0"};
 
@@ -307,7 +311,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-ArrayRef<Builtin::Info> NVPTXTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::NVPTX::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+NVPTXTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index 0a427438bea66..2307d940d3233 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -74,7 +74,8 @@ class LLVM_LIBRARY_VISIBILITY NVPTXTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool useFP16ConversionIntrinsics() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 75f53e96ce28f..c0351f26e9bee 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -473,7 +473,7 @@ class LLVM_LIBRARY_VISIBILITY PS3PPUTargetInfo : public OSTargetInfo<Target> {
     this->IntMaxType = TargetInfo::SignedLongLong;
     this->Int64Type = TargetInfo::SignedLongLong;
     this->SizeType = TargetInfo::UnsignedInt;
-    this->resetDataLayout("E-m:e-p:32:32-Fi64-i64:64-n32:64");
+    this->resetDataLayout("E-m:e-p:32:32-Fi64-i64:64-i128:128-n32:64");
   }
 };
 
diff --git a/clang/lib/Basic/Targets/PNaCl.h b/clang/lib/Basic/Targets/PNaCl.h
index 7e0e10aa362d8..20c0892fbf8c8 100644
--- a/clang/lib/Basic/Targets/PNaCl.h
+++ b/clang/lib/Basic/Targets/PNaCl.h
@@ -52,7 +52,10 @@ class LLVM_LIBRARY_VISIBILITY PNaClTargetInfo : public TargetInfo {
     return Feature == "pnacl";
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::PNaClABIBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 1448069173b5f..6aa36839afa81 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -19,15 +19,19 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::PPC::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsPPC.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
+#include "clang/Basic/BuiltinsPPC.def"
+      });
 
 /// handleTargetFeatures - Perform initialization based on the user
 /// configured set of features.
@@ -927,9 +931,9 @@ void PPCTargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
     MaxAtomicInlineWidth = 128;
 }
 
-ArrayRef<Builtin::Info> PPCTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::PPC::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+PPCTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 bool PPCTargetInfo::validateCpuSupports(StringRef FeatureStr) const {
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index b0833d30550af..b58375f6f3793 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -187,7 +187,8 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
 
   StringRef getABI() const override { return ABI; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool isCLZForZeroUndef() const override { return false; }
 
@@ -462,12 +463,12 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
 
     if (Triple.isOSAIX()) {
       // TODO: Set appropriate ABI for AIX platform.
-      DataLayout = "E-m:a-Fi64-i64:64-n32:64";
+      DataLayout = "E-m:a-Fi64-i64:64-i128:128-n32:64";
       LongDoubleWidth = 64;
       LongDoubleAlign = DoubleAlign = 32;
       LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     } else if ((Triple.getArch() == llvm::Triple::ppc64le)) {
-      DataLayout = "e-m:e-Fn32-i64:64-n32:64";
+      DataLayout = "e-m:e-Fn32-i64:64-i128:128-n32:64";
       ABI = "elfv2";
     } else {
       DataLayout = "E-m:e";
@@ -478,7 +479,7 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
         ABI = "elfv1";
         DataLayout += "-Fi64";
       }
-      DataLayout += "-i64:64-n32:64";
+      DataLayout += "-i64:64-i128:128-n32:64";
     }
 
     if (Triple.isOSFreeBSD() || Triple.isOSOpenBSD() || Triple.isMusl()) {
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 0f8607c22d447..0add1dfdb2f57 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -240,22 +240,28 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 }
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::RISCV::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsRISCVVector.def"
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsRISCV.inc"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsRISCVVector.def"
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsRISCV.inc"
-};
+      });
 
-ArrayRef<Builtin::Info> RISCVTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::RISCV::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+RISCVTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 bool RISCVTargetInfo::initFeatureMap(
@@ -423,6 +429,24 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
   Features.split(AttrFeatures, ";");
   bool FoundArch = false;
 
+  auto handleArchExtension = [](StringRef AttrString,
+                                std::vector<std::string> &Features) {
+    SmallVector<StringRef, 1> Exts;
+    AttrString.split(Exts, ",");
+    for (auto Ext : Exts) {
+      if (Ext.empty())
+        continue;
+
+      StringRef ExtName = Ext.substr(1);
+      std::string TargetFeature =
+          llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName);
+      if (!TargetFeature.empty())
+        Features.push_back(Ext.front() + TargetFeature);
+      else
+        Features.push_back(Ext.str());
+    }
+  };
+
   for (auto &Feature : AttrFeatures) {
     Feature = Feature.trim();
     StringRef AttrString = Feature.split("=").second.trim();
@@ -436,20 +460,7 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
 
       if (AttrString.starts_with("+")) {
         // EXTENSION like arch=+v,+zbb
-        SmallVector<StringRef, 1> Exts;
-        AttrString.split(Exts, ",");
-        for (auto Ext : Exts) {
-          if (Ext.empty())
-            continue;
-
-          StringRef ExtName = Ext.substr(1);
-          std::string TargetFeature =
-              llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName);
-          if (!TargetFeature.empty())
-            Ret.Features.push_back(Ext.front() + TargetFeature);
-          else
-            Ret.Features.push_back(Ext.str());
-        }
+        handleArchExtension(AttrString, Ret.Features);
       } else {
         // full-arch-string like arch=rv64gcv
         handleFullArchString(AttrString, Ret.Features);
@@ -475,11 +486,35 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
       Ret.Tune = AttrString;
     } else if (Feature.starts_with("priority")) {
       // Skip because it only use for FMV.
+    } else if (Feature.starts_with("+")) {
+      // Handle target_version/target_clones attribute strings
+      // that are already delimited by ','
+      handleArchExtension(Feature, Ret.Features);
     }
   }
   return Ret;
 }
 
+unsigned RISCVTargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
+  // Priority is explicitly specified on RISC-V unlike on other targets, where
+  // it is derived by all the features of a specific version. Therefore if a
+  // feature contains the priority string, then return it immediately.
+  for (StringRef Feature : Features) {
+    auto [LHS, RHS] = Feature.rsplit(';');
+    if (LHS.consume_front("priority="))
+      Feature = LHS;
+    else if (RHS.consume_front("priority="))
+      Feature = RHS;
+    else
+      continue;
+    unsigned Priority;
+    if (!Feature.getAsInteger(0, Priority))
+      return Priority;
+  }
+  // Default Priority is zero.
+  return 0;
+}
+
 TargetInfo::CallingConvCheckResult
 RISCVTargetInfo::checkCallingConvention(CallingConv CC) const {
   switch (CC) {
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 3544ea64cb5e7..a168952e6c6f5 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -62,7 +62,8 @@ class RISCVTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
@@ -122,6 +123,7 @@ class RISCVTargetInfo : public TargetInfo {
   void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values) const override;
   bool supportsTargetAttributeTune() const override { return true; }
   ParsedTargetAttr parseTargetAttr(StringRef Str) const override;
+  unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
 
   std::pair<unsigned, unsigned> hardwareInterferenceSizes() const override {
     return std::make_pair(32, 32);
diff --git a/clang/lib/Basic/Targets/SPIR.cpp b/clang/lib/Basic/Targets/SPIR.cpp
index 040303983594f..b0e1fa35cca69 100644
--- a/clang/lib/Basic/Targets/SPIR.cpp
+++ b/clang/lib/Basic/Targets/SPIR.cpp
@@ -81,8 +81,9 @@ SPIRV64AMDGCNTargetInfo::convertConstraint(const char *&Constraint) const {
   return AMDGPUTI.convertConstraint(Constraint);
 }
 
-ArrayRef<Builtin::Info> SPIRV64AMDGCNTargetInfo::getTargetBuiltins() const {
-  return AMDGPUTI.getTargetBuiltins();
+std::pair<const char *, ArrayRef<Builtin::Info>>
+SPIRV64AMDGCNTargetInfo::getTargetBuiltinStorage() const {
+  return AMDGPUTI.getTargetBuiltinStorage();
 }
 
 void SPIRV64AMDGCNTargetInfo::getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 2b26261bec365..cc6a3100f2ae0 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -165,7 +165,10 @@ class LLVM_LIBRARY_VISIBILITY BaseSPIRTargetInfo : public TargetInfo {
   // memcpy as per section 3 of the SPIR spec.
   bool useFP16ConversionIntrinsics() const override { return false; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
@@ -609,7 +612,8 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
 
   std::string convertConstraint(const char *&Constraint) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
diff --git a/clang/lib/Basic/Targets/Sparc.h b/clang/lib/Basic/Targets/Sparc.h
index 9c529a5bc5e7f..480073fae35c5 100644
--- a/clang/lib/Basic/Targets/Sparc.h
+++ b/clang/lib/Basic/Targets/Sparc.h
@@ -48,9 +48,10 @@ class LLVM_LIBRARY_VISIBILITY SparcTargetInfo : public TargetInfo {
 
   bool hasFeature(StringRef Feature) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override {
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
     // FIXME: Implement!
-    return {};
+    return {nullptr, {}};
   }
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/SystemZ.cpp b/clang/lib/Basic/Targets/SystemZ.cpp
index 06f08db2eadd4..ffc55a80db9d9 100644
--- a/clang/lib/Basic/Targets/SystemZ.cpp
+++ b/clang/lib/Basic/Targets/SystemZ.cpp
@@ -20,13 +20,18 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::SystemZ::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsSystemZ.def"
-};
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsSystemZ.def"
+      });
 
 const char *const SystemZTargetInfo::GCCRegNames[] = {
     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
@@ -170,7 +175,7 @@ void SystemZTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__VEC__", "10304");
 }
 
-ArrayRef<Builtin::Info> SystemZTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::SystemZ::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+SystemZTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index ef9a07033a6e4..04e00bac64476 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -99,7 +99,8 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   ArrayRef<const char *> getGCCRegNames() const override;
 
@@ -247,6 +248,8 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
     return RegNo < 4 ? 6 + RegNo : -1;
   }
 
+  bool hasSjLjLowering() const override { return true; }
+
   std::pair<unsigned, unsigned> hardwareInterferenceSizes() const override {
     return std::make_pair(256, 256);
   }
diff --git a/clang/lib/Basic/Targets/TCE.h b/clang/lib/Basic/Targets/TCE.h
index d6280b02f07b2..52a2b13fe2805 100644
--- a/clang/lib/Basic/Targets/TCE.h
+++ b/clang/lib/Basic/Targets/TCE.h
@@ -95,7 +95,10 @@ class LLVM_LIBRARY_VISIBILITY TCETargetInfo : public TargetInfo {
 
   bool hasFeature(StringRef Feature) const override { return Feature == "tce"; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override { return {}; }
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override {
+    return {nullptr, {}};
+  }
 
   std::string_view getClobbers() const override { return ""; }
 
diff --git a/clang/lib/Basic/Targets/VE.cpp b/clang/lib/Basic/Targets/VE.cpp
index 67cae8faf6052..f54ddc00f6f68 100644
--- a/clang/lib/Basic/Targets/VE.cpp
+++ b/clang/lib/Basic/Targets/VE.cpp
@@ -18,11 +18,16 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::VE::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsVE.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
 #include "clang/Basic/BuiltinsVE.def"
-};
+      });
 
 void VETargetInfo::getTargetDefines(const LangOptions &Opts,
                                     MacroBuilder &Builder) const {
@@ -39,7 +44,7 @@ void VETargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8");
 }
 
-ArrayRef<Builtin::Info> VETargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::VE::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+VETargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/VE.h b/clang/lib/Basic/Targets/VE.h
index 7e8fdf6096ef2..d2b0cd6de4d51 100644
--- a/clang/lib/Basic/Targets/VE.h
+++ b/clang/lib/Basic/Targets/VE.h
@@ -55,7 +55,8 @@ class LLVM_LIBRARY_VISIBILITY VETargetInfo : public TargetInfo {
 
   bool hasSjLjLowering() const override { return true; }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/Basic/Targets/WebAssembly.cpp b/clang/lib/Basic/Targets/WebAssembly.cpp
index 0b380bdf835ff..14f1ac20747f5 100644
--- a/clang/lib/Basic/Targets/WebAssembly.cpp
+++ b/clang/lib/Basic/Targets/WebAssembly.cpp
@@ -20,18 +20,22 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    clang::WebAssembly::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsWebAssembly.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsWebAssembly.def"
-};
+      });
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
-    {"mvp"}, {"bleeding-edge"}, {"generic"}};
+    {"mvp"}, {"bleeding-edge"}, {"generic"}, {"lime1"}};
 
 StringRef WebAssemblyTargetInfo::getABI() const { return ABI; }
 
@@ -47,6 +51,8 @@ bool WebAssemblyTargetInfo::hasFeature(StringRef Feature) const {
   return llvm::StringSwitch<bool>(Feature)
       .Case("atomics", HasAtomics)
       .Case("bulk-memory", HasBulkMemory)
+      .Case("bulk-memory-opt", HasBulkMemoryOpt)
+      .Case("call-indirect-overlong", HasCallIndirectOverlong)
       .Case("exception-handling", HasExceptionHandling)
       .Case("extended-const", HasExtendedConst)
       .Case("fp16", HasFP16)
@@ -79,6 +85,8 @@ void WebAssemblyTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__wasm_atomics__");
   if (HasBulkMemory)
     Builder.defineMacro("__wasm_bulk_memory__");
+  if (HasBulkMemoryOpt)
+    Builder.defineMacro("__wasm_bulk_memory_opt__");
   if (HasExceptionHandling)
     Builder.defineMacro("__wasm_exception_handling__");
   if (HasExtendedConst)
@@ -155,12 +163,25 @@ bool WebAssemblyTargetInfo::initFeatureMap(
     const std::vector<std::string> &FeaturesVec) const {
   auto addGenericFeatures = [&]() {
     Features["bulk-memory"] = true;
+    Features["bulk-memory-opt"] = true;
+    Features["call-indirect-overlong"] = true;
     Features["multivalue"] = true;
     Features["mutable-globals"] = true;
     Features["nontrapping-fptoint"] = true;
     Features["reference-types"] = true;
     Features["sign-ext"] = true;
   };
+  auto addLime1Features = [&]() {
+    // Lime1:
+    // <https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1>
+    Features["bulk-memory-opt"] = true;
+    Features["call-indirect-overlong"] = true;
+    Features["extended-const"] = true;
+    Features["multivalue"] = true;
+    Features["mutable-globals"] = true;
+    Features["nontrapping-fptoint"] = true;
+    Features["sign-ext"] = true;
+  };
   auto addBleedingEdgeFeatures = [&]() {
     addGenericFeatures();
     Features["atomics"] = true;
@@ -174,6 +195,8 @@ bool WebAssemblyTargetInfo::initFeatureMap(
   };
   if (CPU == "generic") {
     addGenericFeatures();
+  } else if (CPU == "lime1") {
+    addLime1Features();
   } else if (CPU == "bleeding-edge") {
     addBleedingEdgeFeatures();
   }
@@ -200,6 +223,22 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
       HasBulkMemory = false;
       continue;
     }
+    if (Feature == "+bulk-memory-opt") {
+      HasBulkMemoryOpt = true;
+      continue;
+    }
+    if (Feature == "-bulk-memory-opt") {
+      HasBulkMemoryOpt = false;
+      continue;
+    }
+    if (Feature == "+call-indirect-overlong") {
+      HasCallIndirectOverlong = true;
+      continue;
+    }
+    if (Feature == "-call-indirect-overlong") {
+      HasCallIndirectOverlong = false;
+      continue;
+    }
     if (Feature == "+exception-handling") {
       HasExceptionHandling = true;
       continue;
@@ -310,12 +349,24 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
         << Feature << "-target-feature";
     return false;
   }
+
+  // bulk-memory-opt is a subset of bulk-memory.
+  if (HasBulkMemory) {
+    HasBulkMemoryOpt = true;
+  }
+
+  // The reference-types feature included the change to `call_indirect`
+  // encodings to support overlong immediates.
+  if (HasReferenceTypes) {
+    HasCallIndirectOverlong = true;
+  }
+
   return true;
 }
 
-ArrayRef<Builtin::Info> WebAssemblyTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo, clang::WebAssembly::LastTSBuiltin -
-                                         Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+WebAssemblyTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
 
 void WebAssemblyTargetInfo::adjust(DiagnosticsEngine &Diags,
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index 6c2fe8049ff47..6ce2bb00e5f2a 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -22,26 +22,26 @@ namespace clang {
 namespace targets {
 
 static const unsigned WebAssemblyAddrSpaceMap[] = {
-    0, // Default
-    0, // opencl_global
-    0, // opencl_local
-    0, // opencl_constant
-    0, // opencl_private
-    0, // opencl_generic
-    0, // opencl_global_device
-    0, // opencl_global_host
-    0, // cuda_device
-    0, // cuda_constant
-    0, // cuda_shared
-    0, // sycl_global
-    0, // sycl_global_device
-    0, // sycl_global_host
-    0, // sycl_local
-    0, // sycl_private
-    0, // ptr32_sptr
-    0, // ptr32_uptr
-    0, // ptr64
-    0, // hlsl_groupshared
+    0,  // Default
+    0,  // opencl_global
+    0,  // opencl_local
+    0,  // opencl_constant
+    0,  // opencl_private
+    0,  // opencl_generic
+    0,  // opencl_global_device
+    0,  // opencl_global_host
+    0,  // cuda_device
+    0,  // cuda_constant
+    0,  // cuda_shared
+    0,  // sycl_global
+    0,  // sycl_global_device
+    0,  // sycl_global_host
+    0,  // sycl_local
+    0,  // sycl_private
+    0,  // ptr32_sptr
+    0,  // ptr32_uptr
+    0,  // ptr64
+    0,  // hlsl_groupshared
     20, // wasm_funcref
 };
 
@@ -55,6 +55,8 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
 
   bool HasAtomics = false;
   bool HasBulkMemory = false;
+  bool HasBulkMemoryOpt = false;
+  bool HasCallIndirectOverlong = false;
   bool HasExceptionHandling = false;
   bool HasExtendedConst = false;
   bool HasFP16 = false;
@@ -118,7 +120,8 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
 
   bool setCPU(const std::string &Name) final { return isValidCPUName(Name); }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const final;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const final;
 
   BuiltinVaListKind getBuiltinVaListKind() const final {
     return VoidPtrBuiltinVaList;
@@ -180,11 +183,12 @@ class LLVM_LIBRARY_VISIBILITY WebAssembly32TargetInfo
                                    const TargetOptions &Opts)
       : WebAssemblyTargetInfo(T, Opts) {
     if (T.isOSEmscripten())
-      resetDataLayout("e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-f128:64-n32:64-"
-                      "S128-ni:1:10:20");
-    else
       resetDataLayout(
-          "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20");
+          "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-"
+          "S128-ni:1:10:20");
+    else
+      resetDataLayout("e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-"
+                      "S128-ni:1:10:20");
   }
 
 protected:
@@ -204,11 +208,12 @@ class LLVM_LIBRARY_VISIBILITY WebAssembly64TargetInfo
     PtrDiffType = SignedLong;
     IntPtrType = SignedLong;
     if (T.isOSEmscripten())
-      resetDataLayout("e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-f128:64-n32:64-"
-                      "S128-ni:1:10:20");
-    else
       resetDataLayout(
-          "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20");
+          "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-"
+          "S128-ni:1:10:20");
+    else
+      resetDataLayout("e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-"
+                      "S128-ni:1:10:20");
   }
 
 protected:
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 5993257e27d5a..0b071573aa1e0 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -23,31 +23,45 @@
 namespace clang {
 namespace targets {
 
-static constexpr Builtin::Info BuiltinInfoX86[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+// The x86-32 builtins are a subset and prefix of the x86-64 builtins.
+static constexpr int NumX86Builtins =
+    X86::LastX86CommonBuiltin - Builtin::FirstTSBuiltin + 1;
+static constexpr int NumX86_64Builtins =
+    X86::LastTSBuiltin - Builtin::FirstTSBuiltin;
+static_assert(NumX86Builtins < NumX86_64Builtins);
+
+static constexpr auto BuiltinStorage =
+    Builtin::Storage<NumX86_64Builtins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsX86.def"
 
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsX86.inc"
 
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE)                               \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE)         \
-  {#ID, TYPE, ATTRS, FEATURE, HeaderDesc::HEADER, LANGS},
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_STR_TABLE
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_STR_TABLE
 #include "clang/Basic/BuiltinsX86_64.def"
-};
+        , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsX86.def"
+
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsX86.inc"
+
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define TARGET_BUILTIN CLANG_TARGET_BUILTIN_ENTRY
+#define TARGET_HEADER_BUILTIN CLANG_TARGET_HEADER_BUILTIN_ENTRY
+#include "clang/Basic/BuiltinsX86_64.def"
+          });
 
 static const char *const GCCRegNames[] = {
     "ax",    "dx",    "cx",    "bx",    "si",      "di",    "bp",    "sp",
@@ -1162,6 +1176,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("pconfig", true)
       .Case("pku", true)
       .Case("popcnt", true)
+      .Case("prefer-256-bit", true)
       .Case("prefetchi", true)
       .Case("prfchw", true)
       .Case("ptwrite", true)
@@ -1364,19 +1379,26 @@ static llvm::X86::ProcessorFeatures getFeature(StringRef Name) {
   // correct, so it asserts if the value is out of range.
 }
 
-unsigned X86TargetInfo::multiVersionSortPriority(StringRef Name) const {
-  // Valid CPUs have a 'key feature' that compares just better than its key
-  // feature.
-  using namespace llvm::X86;
-  CPUKind Kind = parseArchX86(Name);
-  if (Kind != CK_None) {
-    ProcessorFeatures KeyFeature = getKeyFeature(Kind);
-    return (getFeaturePriority(KeyFeature) << 1) + 1;
-  }
-
-  // Now we know we have a feature, so get its priority and shift it a few so
-  // that we have sufficient room for the CPUs (above).
-  return getFeaturePriority(getFeature(Name)) << 1;
+unsigned X86TargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
+  auto getPriority = [](StringRef Feature) -> unsigned {
+    // Valid CPUs have a 'key feature' that compares just better than its key
+    // feature.
+    using namespace llvm::X86;
+    CPUKind Kind = parseArchX86(Feature);
+    if (Kind != CK_None) {
+      ProcessorFeatures KeyFeature = getKeyFeature(Kind);
+      return (getFeaturePriority(KeyFeature) << 1) + 1;
+    }
+    // Now we know we have a feature, so get its priority and shift it a few so
+    // that we have sufficient room for the CPUs (above).
+    return getFeaturePriority(getFeature(Feature)) << 1;
+  };
+
+  unsigned Priority = 0;
+  for (StringRef Feature : Features)
+    if (!Feature.empty())
+      Priority = std::max(Priority, getPriority(Feature));
+  return Priority;
 }
 
 bool X86TargetInfo::validateCPUSpecificCPUDispatch(StringRef Name) const {
@@ -1856,12 +1878,14 @@ ArrayRef<TargetInfo::AddlRegName> X86TargetInfo::getGCCAddlRegNames() const {
   return llvm::ArrayRef(AddlRegNames);
 }
 
-ArrayRef<Builtin::Info> X86_32TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfoX86, clang::X86::LastX86CommonBuiltin -
-                                            Builtin::FirstTSBuiltin + 1);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+X86_32TargetInfo::getTargetBuiltinStorage() const {
+  // Only use the relevant prefix of the infos, the string table base is common.
+  return {BuiltinStorage.StringTable,
+          llvm::ArrayRef(BuiltinStorage.Infos).take_front(NumX86Builtins)};
 }
 
-ArrayRef<Builtin::Info> X86_64TargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfoX86,
-                        X86::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+X86_64TargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 06a7eed8177cb..385d9e87a4e37 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -384,7 +384,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
     return CPU != llvm::X86::CK_None;
   }
 
-  unsigned multiVersionSortPriority(StringRef Name) const override;
+  unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
 
   bool setFPMath(StringRef Name) override;
 
@@ -508,7 +508,8 @@ class LLVM_LIBRARY_VISIBILITY X86_32TargetInfo : public X86TargetInfo {
       MaxAtomicInlineWidth = 64;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool hasBitIntType() const override { return true; }
   size_t getMaxBitIntWidth() const override {
@@ -812,7 +813,8 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
       MaxAtomicInlineWidth = 128;
   }
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   bool hasBitIntType() const override { return true; }
   size_t getMaxBitIntWidth() const override {
diff --git a/clang/lib/Basic/Targets/XCore.cpp b/clang/lib/Basic/Targets/XCore.cpp
index fd377bbfb90e1..403d421d0c067 100644
--- a/clang/lib/Basic/Targets/XCore.cpp
+++ b/clang/lib/Basic/Targets/XCore.cpp
@@ -18,13 +18,17 @@
 using namespace clang;
 using namespace clang::targets;
 
-static constexpr Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS)                                               \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::NO_HEADER, ALL_LANGUAGES},
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER)                                    \
-  {#ID, TYPE, ATTRS, nullptr, HeaderDesc::HEADER, ALL_LANGUAGES},
+static constexpr int NumBuiltins =
+    XCore::LastTSBuiltin - Builtin::FirstTSBuiltin;
+
+static constexpr auto BuiltinStorage = Builtin::Storage<NumBuiltins>::Make(
+#define BUILTIN CLANG_BUILTIN_STR_TABLE
+#include "clang/Basic/BuiltinsXCore.def"
+    , {
+#define BUILTIN CLANG_BUILTIN_ENTRY
+#define LIBBUILTIN CLANG_LIBBUILTIN_ENTRY
 #include "clang/Basic/BuiltinsXCore.def"
-};
+      });
 
 void XCoreTargetInfo::getTargetDefines(const LangOptions &Opts,
                                        MacroBuilder &Builder) const {
@@ -32,7 +36,7 @@ void XCoreTargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__XS1B__");
 }
 
-ArrayRef<Builtin::Info> XCoreTargetInfo::getTargetBuiltins() const {
-  return llvm::ArrayRef(BuiltinInfo,
-                        clang::XCore::LastTSBuiltin - Builtin::FirstTSBuiltin);
+std::pair<const char *, ArrayRef<Builtin::Info>>
+XCoreTargetInfo::getTargetBuiltinStorage() const {
+  return {BuiltinStorage.StringTable, BuiltinStorage.Infos};
 }
diff --git a/clang/lib/Basic/Targets/XCore.h b/clang/lib/Basic/Targets/XCore.h
index 84fd59d1a71e4..1082990d74b2b 100644
--- a/clang/lib/Basic/Targets/XCore.h
+++ b/clang/lib/Basic/Targets/XCore.h
@@ -43,7 +43,8 @@ class LLVM_LIBRARY_VISIBILITY XCoreTargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+  std::pair<const char *, ArrayRef<Builtin::Info>>
+  getTargetBuiltinStorage() const override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
     return TargetInfo::VoidPtrBuiltinVaList;
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
new file mode 100644
index 0000000000000..92115778518d4
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
+#define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
+
+#include "CIRGenTypeCache.h"
+
+#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
+
+namespace clang::CIRGen {
+
+class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
+  const CIRGenTypeCache &typeCache;
+
+public:
+  CIRGenBuilderTy(mlir::MLIRContext &mlirContext, const CIRGenTypeCache &tc)
+      : CIRBaseBuilderTy(mlirContext), typeCache(tc) {}
+};
+
+} // namespace clang::CIRGen
+
+#endif
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 5963d43bb9672..0db24c3b41d18 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -25,17 +25,31 @@
 using namespace clang;
 using namespace clang::CIRGen;
 
-CIRGenModule::CIRGenModule(mlir::MLIRContext &context,
-                           clang::ASTContext &astctx,
+CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
+                           clang::ASTContext &astContext,
                            const clang::CodeGenOptions &cgo,
                            DiagnosticsEngine &diags)
-    : builder(&context), astCtx(astctx), langOpts(astctx.getLangOpts()),
-      theModule{mlir::ModuleOp::create(mlir::UnknownLoc::get(&context))},
-      diags(diags), target(astCtx.getTargetInfo()) {}
+    : builder(mlirContext, *this), astContext(astContext),
+      langOpts(astContext.getLangOpts()),
+      theModule{mlir::ModuleOp::create(mlir::UnknownLoc::get(&mlirContext))},
+      diags(diags), target(astContext.getTargetInfo()), genTypes(*this) {
+
+  // Initialize cached types
+  SInt8Ty = cir::IntType::get(&getMLIRContext(), 8, /*isSigned=*/true);
+  SInt16Ty = cir::IntType::get(&getMLIRContext(), 16, /*isSigned=*/true);
+  SInt32Ty = cir::IntType::get(&getMLIRContext(), 32, /*isSigned=*/true);
+  SInt64Ty = cir::IntType::get(&getMLIRContext(), 64, /*isSigned=*/true);
+  SInt128Ty = cir::IntType::get(&getMLIRContext(), 128, /*isSigned=*/true);
+  UInt8Ty = cir::IntType::get(&getMLIRContext(), 8, /*isSigned=*/false);
+  UInt16Ty = cir::IntType::get(&getMLIRContext(), 16, /*isSigned=*/false);
+  UInt32Ty = cir::IntType::get(&getMLIRContext(), 32, /*isSigned=*/false);
+  UInt64Ty = cir::IntType::get(&getMLIRContext(), 64, /*isSigned=*/false);
+  UInt128Ty = cir::IntType::get(&getMLIRContext(), 128, /*isSigned=*/false);
+}
 
 mlir::Location CIRGenModule::getLoc(SourceLocation cLoc) {
   assert(cLoc.isValid() && "expected valid source location");
-  const SourceManager &sm = astCtx.getSourceManager();
+  const SourceManager &sm = astContext.getSourceManager();
   PresumedLoc pLoc = sm.getPresumedLoc(cLoc);
   StringRef filename = pLoc.getFilename();
   return mlir::FileLineColLoc::get(builder.getStringAttr(filename),
@@ -67,7 +81,8 @@ void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
       return;
     }
   } else {
-    errorNYI(global->getSourceRange(), "global variable declaration");
+    assert(cast<VarDecl>(global)->isFileVarDecl() &&
+           "Cannot emit local var decl as global");
   }
 
   // TODO(CIR): Defer emitting some global definitions until later
@@ -77,9 +92,27 @@ void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
 void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
                                                 mlir::Operation *op) {
   auto const *funcDecl = cast<FunctionDecl>(gd.getDecl());
-  auto funcOp = builder.create<cir::FuncOp>(
-      getLoc(funcDecl->getSourceRange()), funcDecl->getIdentifier()->getName());
-  theModule.push_back(funcOp);
+  if (clang::IdentifierInfo *identifier = funcDecl->getIdentifier()) {
+    auto funcOp = builder.create<cir::FuncOp>(
+        getLoc(funcDecl->getSourceRange()), identifier->getName());
+    theModule.push_back(funcOp);
+  } else {
+    errorNYI(funcDecl->getSourceRange().getBegin(),
+             "function definition with a non-identifier for a name");
+  }
+}
+
+void CIRGenModule::emitGlobalVarDefinition(const clang::VarDecl *vd,
+                                           bool isTentative) {
+  mlir::Type type = getTypes().convertType(vd->getType());
+  if (clang::IdentifierInfo *identifier = vd->getIdentifier()) {
+    auto varOp = builder.create<cir::GlobalOp>(getLoc(vd->getSourceRange()),
+                                               identifier->getName(), type);
+    theModule.push_back(varOp);
+  } else {
+    errorNYI(vd->getSourceRange().getBegin(),
+             "variable definition with a non-identifier for a name");
+  }
 }
 
 void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
@@ -103,6 +136,9 @@ void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
     return;
   }
 
+  if (const auto *vd = dyn_cast<VarDecl>(decl))
+    return emitGlobalVarDefinition(vd, !vd->hasDefinition());
+
   llvm_unreachable("Invalid argument to CIRGenModule::emitGlobalDefinition");
 }
 
@@ -126,13 +162,13 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
       emitGlobal(fd);
     break;
   }
-  }
-}
 
-DiagnosticBuilder CIRGenModule::errorNYI(llvm::StringRef feature) {
-  unsigned diagID = diags.getCustomDiagID(
-      DiagnosticsEngine::Error, "ClangIR code gen Not Yet Implemented: %0");
-  return diags.Report(diagID) << feature;
+  case Decl::Var: {
+    auto *vd = cast<VarDecl>(decl);
+    emitGlobal(vd);
+    break;
+  }
+  }
 }
 
 DiagnosticBuilder CIRGenModule::errorNYI(SourceLocation loc,
@@ -142,21 +178,7 @@ DiagnosticBuilder CIRGenModule::errorNYI(SourceLocation loc,
   return diags.Report(loc, diagID) << feature;
 }
 
-DiagnosticBuilder CIRGenModule::errorNYI(SourceLocation loc,
-                                         llvm::StringRef feature,
-                                         llvm::StringRef name) {
-  unsigned diagID = diags.getCustomDiagID(
-      DiagnosticsEngine::Error, "ClangIR code gen Not Yet Implemented: %0: %1");
-  return diags.Report(loc, diagID) << feature << name;
-}
-
 DiagnosticBuilder CIRGenModule::errorNYI(SourceRange loc,
                                          llvm::StringRef feature) {
   return errorNYI(loc.getBegin(), feature) << loc;
 }
-
-DiagnosticBuilder CIRGenModule::errorNYI(SourceRange loc,
-                                         llvm::StringRef feature,
-                                         llvm::StringRef name) {
-  return errorNYI(loc.getBegin(), feature, name) << loc;
-}
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index aaded92e6a633..1c7ed63773900 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -13,24 +13,24 @@
 #ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENMODULE_H
 #define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENMODULE_H
 
+#include "CIRGenBuilder.h"
 #include "CIRGenTypeCache.h"
+#include "CIRGenTypes.h"
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/MLIRContext.h"
+#include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
 class ASTContext;
 class CodeGenOptions;
 class Decl;
-class DiagnosticBuilder;
-class DiagnosticsEngine;
 class GlobalDecl;
 class LangOptions;
-class SourceLocation;
-class SourceRange;
 class TargetInfo;
+class VarDecl;
 
 namespace CIRGen {
 
@@ -41,19 +41,17 @@ class CIRGenModule : public CIRGenTypeCache {
   CIRGenModule &operator=(CIRGenModule &) = delete;
 
 public:
-  CIRGenModule(mlir::MLIRContext &context, clang::ASTContext &astctx,
+  CIRGenModule(mlir::MLIRContext &mlirContext, clang::ASTContext &astContext,
                const clang::CodeGenOptions &cgo,
                clang::DiagnosticsEngine &diags);
 
   ~CIRGenModule() = default;
 
 private:
-  // TODO(CIR) 'builder' will change to CIRGenBuilderTy once that type is
-  // defined
-  mlir::OpBuilder builder;
+  CIRGenBuilderTy builder;
 
   /// Hold Clang AST information.
-  clang::ASTContext &astCtx;
+  clang::ASTContext &astContext;
 
   const clang::LangOptions &langOpts;
 
@@ -64,8 +62,14 @@ class CIRGenModule : public CIRGenTypeCache {
 
   const clang::TargetInfo &target;
 
+  CIRGenTypes genTypes;
+
 public:
   mlir::ModuleOp getModule() const { return theModule; }
+  CIRGenBuilderTy &getBuilder() { return builder; }
+  clang::ASTContext &getASTContext() const { return astContext; }
+  CIRGenTypes &getTypes() { return genTypes; }
+  mlir::MLIRContext &getMLIRContext() { return *builder.getContext(); }
 
   /// Helpers to convert the presumed location of Clang's SourceLocation to an
   /// MLIR Location.
@@ -81,13 +85,28 @@ class CIRGenModule : public CIRGenTypeCache {
   void emitGlobalDefinition(clang::GlobalDecl gd,
                             mlir::Operation *op = nullptr);
   void emitGlobalFunctionDefinition(clang::GlobalDecl gd, mlir::Operation *op);
+  void emitGlobalVarDefinition(const clang::VarDecl *vd,
+                               bool isTentative = false);
 
   /// Helpers to emit "not yet implemented" error diagnostics
-  DiagnosticBuilder errorNYI(llvm::StringRef);
   DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef);
-  DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef, llvm::StringRef);
+
+  template <typename T>
+  DiagnosticBuilder errorNYI(SourceLocation loc, llvm::StringRef feature,
+                             const T &name) {
+    unsigned diagID =
+        diags.getCustomDiagID(DiagnosticsEngine::Error,
+                              "ClangIR code gen Not Yet Implemented: %0: %1");
+    return diags.Report(loc, diagID) << feature << name;
+  }
+
   DiagnosticBuilder errorNYI(SourceRange, llvm::StringRef);
-  DiagnosticBuilder errorNYI(SourceRange, llvm::StringRef, llvm::StringRef);
+
+  template <typename T>
+  DiagnosticBuilder errorNYI(SourceRange loc, llvm::StringRef feature,
+                             const T &name) {
+    return errorNYI(loc.getBegin(), feature, name) << loc;
+  }
 };
 } // namespace CIRGen
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
index fde9a355f5241..a357663c33e0f 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 #define LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+
 namespace clang::CIRGen {
 
 /// This structure provides a set of types that are commonly used
@@ -20,6 +22,20 @@ namespace clang::CIRGen {
 /// constructor and then copied around into new CIRGenFunction's.
 struct CIRGenTypeCache {
   CIRGenTypeCache() = default;
+
+  // ClangIR signed integral types of common sizes
+  cir::IntType SInt8Ty;
+  cir::IntType SInt16Ty;
+  cir::IntType SInt32Ty;
+  cir::IntType SInt64Ty;
+  cir::IntType SInt128Ty;
+
+  // ClangIR unsigned integral type of common sizes
+  cir::IntType UInt8Ty;
+  cir::IntType UInt16Ty;
+  cir::IntType UInt32Ty;
+  cir::IntType UInt64Ty;
+  cir::IntType UInt128Ty;
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
new file mode 100644
index 0000000000000..181af1898baff
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -0,0 +1,94 @@
+#include "CIRGenTypes.h"
+
+#include "CIRGenModule.h"
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Type.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+CIRGenTypes::CIRGenTypes(CIRGenModule &genModule)
+    : cgm(genModule), astContext(genModule.getASTContext()),
+      builder(cgm.getBuilder()) {}
+
+CIRGenTypes::~CIRGenTypes() {}
+
+mlir::MLIRContext &CIRGenTypes::getMLIRContext() const {
+  return *builder.getContext();
+}
+
+mlir::Type CIRGenTypes::convertType(QualType type) {
+  type = astContext.getCanonicalType(type);
+  const Type *ty = type.getTypePtr();
+
+  // Has the type already been processed?
+  TypeCacheTy::iterator tci = typeCache.find(ty);
+  if (tci != typeCache.end())
+    return tci->second;
+
+  // For types that haven't been implemented yet or are otherwise unsupported,
+  // report an error and return 'int'.
+
+  mlir::Type resultType = nullptr;
+  switch (ty->getTypeClass()) {
+  case Type::Builtin: {
+    switch (cast<BuiltinType>(ty)->getKind()) {
+    // Signed integral types.
+    case BuiltinType::Char_S:
+    case BuiltinType::Int:
+    case BuiltinType::Int128:
+    case BuiltinType::Long:
+    case BuiltinType::LongLong:
+    case BuiltinType::SChar:
+    case BuiltinType::Short:
+    case BuiltinType::WChar_S:
+      resultType =
+          cir::IntType::get(&getMLIRContext(), astContext.getTypeSize(ty),
+                            /*isSigned=*/true);
+      break;
+    // Unsigned integral types.
+    case BuiltinType::Char8:
+    case BuiltinType::Char16:
+    case BuiltinType::Char32:
+    case BuiltinType::Char_U:
+    case BuiltinType::UChar:
+    case BuiltinType::UInt:
+    case BuiltinType::UInt128:
+    case BuiltinType::ULong:
+    case BuiltinType::ULongLong:
+    case BuiltinType::UShort:
+    case BuiltinType::WChar_U:
+      resultType =
+          cir::IntType::get(&getMLIRContext(), astContext.getTypeSize(ty),
+                            /*isSigned=*/false);
+      break;
+    default:
+      cgm.errorNYI(SourceLocation(), "processing of built-in type", type);
+      resultType = cgm.SInt32Ty;
+      break;
+    }
+    break;
+  }
+  case Type::BitInt: {
+    const auto *bitIntTy = cast<BitIntType>(type);
+    if (bitIntTy->getNumBits() > cir::IntType::maxBitwidth()) {
+      cgm.errorNYI(SourceLocation(), "large _BitInt type", type);
+      resultType = cgm.SInt32Ty;
+    } else {
+      resultType = cir::IntType::get(&getMLIRContext(), bitIntTy->getNumBits(),
+                                     bitIntTy->isSigned());
+    }
+    break;
+  }
+  default:
+    cgm.errorNYI(SourceLocation(), "processing of type", type);
+    resultType = cgm.SInt32Ty;
+    break;
+  }
+
+  assert(resultType && "Type conversion not yet implemented");
+
+  typeCache[ty] = resultType;
+  return resultType;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
new file mode 100644
index 0000000000000..563d7759831fa
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -0,0 +1,59 @@
+//===--- CIRGenTypes.h - Type translation for CIR CodeGen -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the code that handles AST -> CIR type lowering.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H
+#define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H
+
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+
+#include "llvm/ADT/SmallPtrSet.h"
+
+namespace clang {
+class ASTContext;
+class QualType;
+class Type;
+} // namespace clang
+
+namespace mlir {
+class Type;
+}
+
+namespace clang::CIRGen {
+
+class CIRGenBuilderTy;
+class CIRGenModule;
+
+/// This class organizes the cross-module state that is used while lowering
+/// AST types to CIR types.
+class CIRGenTypes {
+  CIRGenModule &cgm;
+  clang::ASTContext &astContext;
+  CIRGenBuilderTy &builder;
+
+public:
+  CIRGenTypes(CIRGenModule &cgm);
+  ~CIRGenTypes();
+
+  /// This map of clang::Type to mlir::Type (which includes CIR type) is a
+  /// cache of types that have already been processed.
+  using TypeCacheTy = llvm::DenseMap<const clang::Type *, mlir::Type>;
+  TypeCacheTy typeCache;
+
+  mlir::MLIRContext &getMLIRContext() const;
+
+  /// Convert a Clang type into a mlir::Type.
+  mlir::Type convertType(clang::QualType type);
+};
+
+} // namespace clang::CIRGen
+
+#endif
diff --git a/clang/lib/CIR/CodeGen/CIRGenerator.cpp b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
index 8f3370c0041af..91070eda7d45a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenerator.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
@@ -29,15 +29,15 @@ CIRGenerator::CIRGenerator(clang::DiagnosticsEngine &diags,
     : diags(diags), fs(std::move(vfs)), codeGenOpts{cgo} {}
 CIRGenerator::~CIRGenerator() = default;
 
-void CIRGenerator::Initialize(ASTContext &astCtx) {
+void CIRGenerator::Initialize(ASTContext &astContext) {
   using namespace llvm;
 
-  this->astCtx = &astCtx;
+  this->astContext = &astContext;
 
-  mlirCtx = std::make_unique<mlir::MLIRContext>();
-  mlirCtx->loadDialect<cir::CIRDialect>();
-  cgm = std::make_unique<clang::CIRGen::CIRGenModule>(*mlirCtx.get(), astCtx,
-                                                      codeGenOpts, diags);
+  mlirContext = std::make_unique<mlir::MLIRContext>();
+  mlirContext->loadDialect<cir::CIRDialect>();
+  cgm = std::make_unique<clang::CIRGen::CIRGenModule>(
+      *mlirContext.get(), astContext, codeGenOpts, diags);
 }
 
 mlir::ModuleOp CIRGenerator::getModule() const { return cgm->getModule(); }
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 17a3aabfbd7f0..9ada31c11de95 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -9,6 +9,7 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 add_clang_library(clangCIR
   CIRGenerator.cpp
   CIRGenModule.cpp
+  CIRGenTypes.cpp
 
   DEPENDS
   MLIRCIR
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index f666e5ab4b999..dbdca1f840166 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -32,6 +32,22 @@ void cir::CIRDialect::initialize() {
       >();
 }
 
+//===----------------------------------------------------------------------===//
+// GlobalOp
+//===----------------------------------------------------------------------===//
+
+// TODO(CIR): The properties of global variables that require verification
+// haven't been implemented yet.
+mlir::LogicalResult cir::GlobalOp::verify() { return success(); }
+
+void cir::GlobalOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                          llvm::StringRef sym_name, mlir::Type sym_type) {
+  odsState.addAttribute(getSymNameAttrName(odsState.name),
+                        odsBuilder.getStringAttr(sym_name));
+  odsState.addAttribute(getSymTypeAttrName(odsState.name),
+                        mlir::TypeAttr::get(sym_type));
+}
+
 //===----------------------------------------------------------------------===//
 // FuncOp
 //===----------------------------------------------------------------------===//
@@ -56,6 +72,8 @@ void cir::FuncOp::print(OpAsmPrinter &p) {
   p.printSymbolName(getSymName());
 }
 
+// TODO(CIR): The properties of functions that require verification haven't
+// been implemented yet.
 mlir::LogicalResult cir::FuncOp::verify() { return success(); }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index 4eeb70f06f5f7..de38337057d3d 100644
--- a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -10,7 +10,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+
+#include "mlir/IR/DialectImplementation.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+//===----------------------------------------------------------------------===//
+// Get autogenerated stuff
+//===----------------------------------------------------------------------===//
+
+#define GET_TYPEDEF_CLASSES
+#include "clang/CIR/Dialect/IR/CIROpsTypes.cpp.inc"
 
 using namespace mlir;
 using namespace cir;
@@ -20,12 +31,106 @@ using namespace cir;
 //===----------------------------------------------------------------------===//
 
 Type CIRDialect::parseType(DialectAsmParser &parser) const {
-  // No types yet to parse
-  return Type{};
+  llvm::SMLoc typeLoc = parser.getCurrentLocation();
+  llvm::StringRef mnemonic;
+  Type genType;
+
+  // Try to parse as a tablegen'd type.
+  OptionalParseResult parseResult =
+      generatedTypeParser(parser, &mnemonic, genType);
+  if (parseResult.has_value())
+    return genType;
+
+  // TODO(CIR) Attempt to parse as a raw C++ type.
+  parser.emitError(typeLoc) << "unknown CIR type: " << mnemonic;
+  return Type();
 }
 
 void CIRDialect::printType(Type type, DialectAsmPrinter &os) const {
-  // No types yet to print
+  // Try to print as a tablegen'd type.
+  if (generatedTypePrinter(type, os).succeeded())
+    return;
+
+  // TODO(CIR) Attempt to print as a raw C++ type.
+  llvm::report_fatal_error("printer is missing a handler for this type");
+}
+
+//===----------------------------------------------------------------------===//
+// IntType Definitions
+//===----------------------------------------------------------------------===//
+
+Type IntType::parse(mlir::AsmParser &parser) {
+  mlir::MLIRContext *context = parser.getBuilder().getContext();
+  llvm::SMLoc loc = parser.getCurrentLocation();
+  bool isSigned;
+  unsigned width;
+
+  if (parser.parseLess())
+    return {};
+
+  // Fetch integer sign.
+  llvm::StringRef sign;
+  if (parser.parseKeyword(&sign))
+    return {};
+  if (sign == "s")
+    isSigned = true;
+  else if (sign == "u")
+    isSigned = false;
+  else {
+    parser.emitError(loc, "expected 's' or 'u'");
+    return {};
+  }
+
+  if (parser.parseComma())
+    return {};
+
+  // Fetch integer size.
+  if (parser.parseInteger(width))
+    return {};
+  if (width < IntType::minBitwidth() || width > IntType::maxBitwidth()) {
+    parser.emitError(loc, "expected integer width to be from ")
+        << IntType::minBitwidth() << " up to " << IntType::maxBitwidth();
+    return {};
+  }
+
+  if (parser.parseGreater())
+    return {};
+
+  return IntType::get(context, width, isSigned);
+}
+
+void IntType::print(mlir::AsmPrinter &printer) const {
+  char sign = isSigned() ? 's' : 'u';
+  printer << '<' << sign << ", " << getWidth() << '>';
+}
+
+llvm::TypeSize
+IntType::getTypeSizeInBits(const mlir::DataLayout &dataLayout,
+                           mlir::DataLayoutEntryListRef params) const {
+  return llvm::TypeSize::getFixed(getWidth());
+}
+
+uint64_t IntType::getABIAlignment(const mlir::DataLayout &dataLayout,
+                                  mlir::DataLayoutEntryListRef params) const {
+  return (uint64_t)(getWidth() / 8);
+}
+
+uint64_t
+IntType::getPreferredAlignment(const ::mlir::DataLayout &dataLayout,
+                               ::mlir::DataLayoutEntryListRef params) const {
+  return (uint64_t)(getWidth() / 8);
+}
+
+mlir::LogicalResult
+IntType::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
+                unsigned width, bool isSigned) {
+  if (width < IntType::minBitwidth() || width > IntType::maxBitwidth()) {
+    emitError() << "IntType only supports widths from "
+                << IntType::minBitwidth() << " up to "
+                << IntType::maxBitwidth();
+    return mlir::failure();
+  }
+  return mlir::success();
 }
 
 //===----------------------------------------------------------------------===//
@@ -33,5 +138,12 @@ void CIRDialect::printType(Type type, DialectAsmPrinter &os) const {
 //===----------------------------------------------------------------------===//
 
 void CIRDialect::registerTypes() {
-  // No types yet to register
+  // Register tablegen'd types.
+  addTypes<
+#define GET_TYPEDEF_LIST
+#include "clang/CIR/Dialect/IR/CIROpsTypes.cpp.inc"
+      >();
+
+  // Register raw C++ types.
+  // TODO(CIR) addTypes<StructType>();
 }
diff --git a/clang/lib/CIR/Dialect/IR/CMakeLists.txt b/clang/lib/CIR/Dialect/IR/CMakeLists.txt
index 75cee3f871130..7ddc4ce501907 100644
--- a/clang/lib/CIR/Dialect/IR/CMakeLists.txt
+++ b/clang/lib/CIR/Dialect/IR/CMakeLists.txt
@@ -8,4 +8,8 @@ add_clang_library(MLIRCIR
 
   LINK_LIBS PUBLIC
   MLIRIR
+  MLIRDLTIDialect
+  MLIRDataLayoutInterfaces
+  MLIRFuncDialect
+  clangAST
   )
diff --git a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
index 5a31e20708193..21b6bc56ed050 100644
--- a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
+++ b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
@@ -66,9 +66,6 @@ class CIRGenConsumer : public clang::ASTConsumer {
         MlirModule->print(*OutputStream, Flags);
       }
       break;
-    default:
-      llvm_unreachable("NYI: CIRGenAction other than EmitCIR");
-      break;
     }
   }
 };
diff --git a/clang/lib/CodeGen/ABIInfo.cpp b/clang/lib/CodeGen/ABIInfo.cpp
index edd7146dc1ac7..8e76cf15b642c 100644
--- a/clang/lib/CodeGen/ABIInfo.cpp
+++ b/clang/lib/CodeGen/ABIInfo.cpp
@@ -218,8 +218,8 @@ void ABIInfo::appendAttributeMangling(StringRef AttrStr,
     // only have "+" prefixes here.
     assert(LHS.starts_with("+") && RHS.starts_with("+") &&
            "Features should always have a prefix.");
-    return TI.multiVersionSortPriority(LHS.substr(1)) >
-           TI.multiVersionSortPriority(RHS.substr(1));
+    return TI.getFMVPriority({LHS.substr(1)}) >
+           TI.getFMVPriority({RHS.substr(1)});
   });
 
   bool IsFirst = true;
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 2d021664001d4..dd9bab1a161ce 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -831,13 +831,12 @@ static void addSanitizers(const Triple &TargetTriple,
   }
 
   if (LowerAllowCheckPass::IsRequested()) {
-    // We can optimize after inliner, and PGO profile matching. The hook below
-    // is called at the end `buildFunctionSimplificationPipeline`, which called
-    // from `buildInlinerPipeline`, which called after profile matching.
-    PB.registerScalarOptimizerLateEPCallback(
-        [](FunctionPassManager &FPM, OptimizationLevel Level) {
-          FPM.addPass(LowerAllowCheckPass());
-        });
+    // We want to call it after inline, which is about OptimizerEarlyEPCallback.
+    PB.registerOptimizerEarlyEPCallback([](ModulePassManager &MPM,
+                                           OptimizationLevel Level,
+                                           ThinOrFullLTOPhase Phase) {
+      MPM.addPass(createModuleToFunctionPassAdaptor(LowerAllowCheckPass()));
+    });
   }
 }
 
@@ -1100,12 +1099,14 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
             FPM.addPass(BoundsCheckingPass());
           });
 
-    if (LangOpts.Sanitize.has(SanitizerKind::Realtime))
+    if (LangOpts.Sanitize.has(SanitizerKind::Realtime)) {
       PB.registerScalarOptimizerLateEPCallback(
           [](FunctionPassManager &FPM, OptimizationLevel Level) {
             RealtimeSanitizerOptions Opts;
             FPM.addPass(RealtimeSanitizerPass(Opts));
           });
+      MPM.addPass(ModuleRealtimeSanitizerPass());
+    }
 
     // Don't add sanitizers if we are here from ThinLTO PostLink. That already
     // done on PreLink stage.
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index a6908f48096fb..b7a0ff0db5491 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -245,6 +245,19 @@ static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
   return CGF.Builder.CreateBitCast(BitVec, ResultType);
 }
 
+/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
+/// return it as an i8 pointer.
+Value *readX18AsPtr(CodeGenFunction &CGF) {
+  LLVMContext &Context = CGF.CGM.getLLVMContext();
+  llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
+  llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
+  llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
+  llvm::Function *F =
+      CGF.CGM.getIntrinsic(llvm::Intrinsic::read_register, {CGF.Int64Ty});
+  llvm::Value *X18 = CGF.Builder.CreateCall(F, Metadata);
+  return CGF.Builder.CreateIntToPtr(X18, CGF.Int8PtrTy);
+}
+
 /// getBuiltinLibFunction - Given a builtin id for a function like
 /// "__builtin_fabsf", return a Function* for "fabsf".
 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
@@ -510,8 +523,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
 
   Address DestAddr = CheckAtomicAlignment(CGF, E);
 
-  auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
   auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
+  auto *RTy = Exchange->getType();
+
+  auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
+
+  if (RTy->isPointerTy()) {
+    Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
+    Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
+  }
 
   // For Release ordering, the failure ordering should be Monotonic.
   auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@@ -522,10 +542,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
   // blocks the few atomics optimizations that LLVM has. If we want to optimize
   // _Interlocked* operations in the future, we will have to remove the volatile
   // marker.
-  auto *Result = CGF.Builder.CreateAtomicCmpXchg(
+  auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
       DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
-  Result->setVolatile(true);
-  return CGF.Builder.CreateExtractValue(Result, 0);
+  CmpXchg->setVolatile(true);
+
+  auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
+  if (RTy->isPointerTy()) {
+    Result = CGF.Builder.CreateIntToPtr(Result, RTy);
+  }
+
+  return Result;
 }
 
 // 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@@ -1685,6 +1711,7 @@ enum class CodeGenFunction::MSVCIntrin {
   _BitScanForward,
   _BitScanReverse,
   _InterlockedAnd,
+  _InterlockedCompareExchange,
   _InterlockedDecrement,
   _InterlockedExchange,
   _InterlockedExchangeAdd,
@@ -1770,26 +1797,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
   case clang::ARM::BI_InterlockedExchange16_acq:
   case clang::ARM::BI_InterlockedExchange_acq:
   case clang::ARM::BI_InterlockedExchange64_acq:
+  case clang::ARM::BI_InterlockedExchangePointer_acq:
     return MSVCIntrin::_InterlockedExchange_acq;
   case clang::ARM::BI_InterlockedExchange8_rel:
   case clang::ARM::BI_InterlockedExchange16_rel:
   case clang::ARM::BI_InterlockedExchange_rel:
   case clang::ARM::BI_InterlockedExchange64_rel:
+  case clang::ARM::BI_InterlockedExchangePointer_rel:
     return MSVCIntrin::_InterlockedExchange_rel;
   case clang::ARM::BI_InterlockedExchange8_nf:
   case clang::ARM::BI_InterlockedExchange16_nf:
   case clang::ARM::BI_InterlockedExchange_nf:
   case clang::ARM::BI_InterlockedExchange64_nf:
+  case clang::ARM::BI_InterlockedExchangePointer_nf:
     return MSVCIntrin::_InterlockedExchange_nf;
   case clang::ARM::BI_InterlockedCompareExchange8_acq:
   case clang::ARM::BI_InterlockedCompareExchange16_acq:
   case clang::ARM::BI_InterlockedCompareExchange_acq:
   case clang::ARM::BI_InterlockedCompareExchange64_acq:
+  case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
     return MSVCIntrin::_InterlockedCompareExchange_acq;
   case clang::ARM::BI_InterlockedCompareExchange8_rel:
   case clang::ARM::BI_InterlockedCompareExchange16_rel:
   case clang::ARM::BI_InterlockedCompareExchange_rel:
   case clang::ARM::BI_InterlockedCompareExchange64_rel:
+  case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
     return MSVCIntrin::_InterlockedCompareExchange_rel;
   case clang::ARM::BI_InterlockedCompareExchange8_nf:
   case clang::ARM::BI_InterlockedCompareExchange16_nf:
@@ -1916,26 +1948,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
   case clang::AArch64::BI_InterlockedExchange16_acq:
   case clang::AArch64::BI_InterlockedExchange_acq:
   case clang::AArch64::BI_InterlockedExchange64_acq:
+  case clang::AArch64::BI_InterlockedExchangePointer_acq:
     return MSVCIntrin::_InterlockedExchange_acq;
   case clang::AArch64::BI_InterlockedExchange8_rel:
   case clang::AArch64::BI_InterlockedExchange16_rel:
   case clang::AArch64::BI_InterlockedExchange_rel:
   case clang::AArch64::BI_InterlockedExchange64_rel:
+  case clang::AArch64::BI_InterlockedExchangePointer_rel:
     return MSVCIntrin::_InterlockedExchange_rel;
   case clang::AArch64::BI_InterlockedExchange8_nf:
   case clang::AArch64::BI_InterlockedExchange16_nf:
   case clang::AArch64::BI_InterlockedExchange_nf:
   case clang::AArch64::BI_InterlockedExchange64_nf:
+  case clang::AArch64::BI_InterlockedExchangePointer_nf:
     return MSVCIntrin::_InterlockedExchange_nf;
   case clang::AArch64::BI_InterlockedCompareExchange8_acq:
   case clang::AArch64::BI_InterlockedCompareExchange16_acq:
   case clang::AArch64::BI_InterlockedCompareExchange_acq:
   case clang::AArch64::BI_InterlockedCompareExchange64_acq:
+  case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
     return MSVCIntrin::_InterlockedCompareExchange_acq;
   case clang::AArch64::BI_InterlockedCompareExchange8_rel:
   case clang::AArch64::BI_InterlockedCompareExchange16_rel:
   case clang::AArch64::BI_InterlockedCompareExchange_rel:
   case clang::AArch64::BI_InterlockedCompareExchange64_rel:
+  case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
     return MSVCIntrin::_InterlockedCompareExchange_rel;
   case clang::AArch64::BI_InterlockedCompareExchange8_nf:
   case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@@ -2138,6 +2175,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
   case MSVCIntrin::_InterlockedExchange_nf:
     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
                                  AtomicOrdering::Monotonic);
+  case MSVCIntrin::_InterlockedCompareExchange:
+    return EmitAtomicCmpXchgForMSIntrin(*this, E);
   case MSVCIntrin::_InterlockedCompareExchange_acq:
     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
   case MSVCIntrin::_InterlockedCompareExchange_rel:
@@ -4957,6 +4996,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     // Buffer is a void**.
     Address Buf = EmitPointerWithAlignment(E->getArg(0));
 
+    if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
+      // On this target, the back end fills in the context buffer completely.
+      // It doesn't really matter if the frontend stores to the buffer before
+      // calling setjmp, the back-end is going to overwrite them anyway.
+      Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
+      return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this)));
+    }
+
     // Store the frame pointer to the setjmp buffer.
     Value *FrameAddr = Builder.CreateCall(
         CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
@@ -5843,32 +5890,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(
         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
   case Builtin::BI_InterlockedCompareExchangePointer:
-  case Builtin::BI_InterlockedCompareExchangePointer_nf: {
-    llvm::Type *RTy;
-    llvm::IntegerType *IntType = IntegerType::get(
-        getLLVMContext(), getContext().getTypeSize(E->getType()));
-
-    Address DestAddr = CheckAtomicAlignment(*this, E);
-
-    llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
-    RTy = Exchange->getType();
-    Exchange = Builder.CreatePtrToInt(Exchange, IntType);
-
-    llvm::Value *Comparand =
-      Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
-
-    auto Ordering =
-      BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
-      AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
-
-    auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
-                                              Ordering, Ordering);
-    Result->setVolatile(true);
-
-    return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
-                                                                         0),
-                                              RTy));
-  }
+    return RValue::get(
+        EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
+  case Builtin::BI_InterlockedCompareExchangePointer_nf:
+    return RValue::get(
+        EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
   case Builtin::BI_InterlockedCompareExchange8:
   case Builtin::BI_InterlockedCompareExchange16:
   case Builtin::BI_InterlockedCompareExchange:
@@ -10342,6 +10368,8 @@ CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
   case SVETypeFlags::EltTyInt64:
     return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
 
+  case SVETypeFlags::EltTyMFloat8:
+    return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
   case SVETypeFlags::EltTyFloat16:
     return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
   case SVETypeFlags::EltTyBFloat16:
@@ -10790,7 +10818,7 @@ Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
   NewOps.push_back(Ops[2]);
 
   llvm::Value *BasePtr = Ops[3];
-
+  llvm::Value *RealSlice = Ops[1];
   // If the intrinsic contains the vnum parameter, multiply it with the vector
   // size in bytes.
   if (Ops.size() == 5) {
@@ -10802,10 +10830,13 @@ Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
         Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl");
     // The type of the ptr parameter is void *, so use Int8Ty here.
     BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
+    RealSlice = Builder.CreateZExt(RealSlice, Int64Ty);
+    RealSlice = Builder.CreateAdd(RealSlice, Ops[4]);
+    RealSlice = Builder.CreateTrunc(RealSlice, Int32Ty);
   }
   NewOps.push_back(BasePtr);
   NewOps.push_back(Ops[0]);
-  NewOps.push_back(Ops[1]);
+  NewOps.push_back(RealSlice);
   Function *F = CGM.getIntrinsic(IntID);
   return Builder.CreateCall(F, NewOps);
 }
@@ -11021,6 +11052,10 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   else if (TypeFlags.isUndef())
     return UndefValue::get(Ty);
   else if (Builtin->LLVMIntrinsic != 0) {
+    // Emit set FPMR for intrinsics that require it
+    if (TypeFlags.setsFPMR())
+      Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
+                         Ops.pop_back_val());
     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
       InsertExplicitZeroOperand(Builder, Ty, Ops);
 
@@ -11392,6 +11427,10 @@ Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
            BuiltinID == SME::BI__builtin_sme_svstr_za)
     return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
 
+  // Emit set FPMR for intrinsics that require it
+  if (TypeFlags.setsFPMR())
+    Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
+                       Ops.pop_back_val());
   // Handle builtins which require their multi-vector operands to be swapped
   swapCommutativeSMEOperands(BuiltinID, Ops);
 
@@ -11997,21 +12036,18 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
       BuiltinID == AArch64::BI__writex18word ||
       BuiltinID == AArch64::BI__writex18dword ||
       BuiltinID == AArch64::BI__writex18qword) {
+    // Process the args first
+    Value *OffsetArg = EmitScalarExpr(E->getArg(0));
+    Value *DataArg = EmitScalarExpr(E->getArg(1));
+
     // Read x18 as i8*
-    LLVMContext &Context = CGM.getLLVMContext();
-    llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
-    llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
-    llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
-    llvm::Function *F =
-        CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
-    llvm::Value *X18 = Builder.CreateCall(F, Metadata);
-    X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
+    llvm::Value *X18 = readX18AsPtr(*this);
 
     // Store val at x18 + offset
-    Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
+    Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
     Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
-    Value *Val = EmitScalarExpr(E->getArg(1));
-    StoreInst *Store = Builder.CreateAlignedStore(Val, Ptr, CharUnits::One());
+    StoreInst *Store =
+        Builder.CreateAlignedStore(DataArg, Ptr, CharUnits::One());
     return Store;
   }
 
@@ -12019,25 +12055,74 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
       BuiltinID == AArch64::BI__readx18word ||
       BuiltinID == AArch64::BI__readx18dword ||
       BuiltinID == AArch64::BI__readx18qword) {
-    llvm::Type *IntTy = ConvertType(E->getType());
+    // Process the args first
+    Value *OffsetArg = EmitScalarExpr(E->getArg(0));
 
     // Read x18 as i8*
-    LLVMContext &Context = CGM.getLLVMContext();
-    llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
-    llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
-    llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
-    llvm::Function *F =
-        CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
-    llvm::Value *X18 = Builder.CreateCall(F, Metadata);
-    X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
+    llvm::Value *X18 = readX18AsPtr(*this);
 
     // Load x18 + offset
-    Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
+    Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
     Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
+    llvm::Type *IntTy = ConvertType(E->getType());
     LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
     return Load;
   }
 
+  if (BuiltinID == AArch64::BI__addx18byte ||
+      BuiltinID == AArch64::BI__addx18word ||
+      BuiltinID == AArch64::BI__addx18dword ||
+      BuiltinID == AArch64::BI__addx18qword ||
+      BuiltinID == AArch64::BI__incx18byte ||
+      BuiltinID == AArch64::BI__incx18word ||
+      BuiltinID == AArch64::BI__incx18dword ||
+      BuiltinID == AArch64::BI__incx18qword) {
+    llvm::Type *IntTy;
+    bool isIncrement;
+    switch (BuiltinID) {
+    case AArch64::BI__incx18byte:
+      IntTy = Int8Ty;
+      isIncrement = true;
+      break;
+    case AArch64::BI__incx18word:
+      IntTy = Int16Ty;
+      isIncrement = true;
+      break;
+    case AArch64::BI__incx18dword:
+      IntTy = Int32Ty;
+      isIncrement = true;
+      break;
+    case AArch64::BI__incx18qword:
+      IntTy = Int64Ty;
+      isIncrement = true;
+      break;
+    default:
+      IntTy = ConvertType(E->getArg(1)->getType());
+      isIncrement = false;
+      break;
+    }
+    // Process the args first
+    Value *OffsetArg = EmitScalarExpr(E->getArg(0));
+    Value *ValToAdd =
+        isIncrement ? ConstantInt::get(IntTy, 1) : EmitScalarExpr(E->getArg(1));
+
+    // Read x18 as i8*
+    llvm::Value *X18 = readX18AsPtr(*this);
+
+    // Load x18 + offset
+    Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
+    Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
+    LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
+
+    // Add values
+    Value *AddResult = Builder.CreateAdd(Load, ValToAdd);
+
+    // Store val at x18 + offset
+    StoreInst *Store =
+        Builder.CreateAlignedStore(AddResult, Ptr, CharUnits::One());
+    return Store;
+  }
+
   if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
       BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
       BuiltinID == AArch64::BI_CopyInt32FromFloat ||
@@ -19224,6 +19309,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return nullptr;
 
   switch (BuiltinID) {
+  case Builtin::BI__builtin_hlsl_resource_getpointer: {
+    Value *HandleOp = EmitScalarExpr(E->getArg(0));
+    Value *IndexOp = EmitScalarExpr(E->getArg(1));
+
+    // TODO: Map to an hlsl_device address space.
+    llvm::Type *RetTy = llvm::PointerType::getUnqual(getLLVMContext());
+
+    return Builder.CreateIntrinsic(RetTy, Intrinsic::dx_resource_getpointer,
+                                   ArrayRef<Value *>{HandleOp, IndexOp});
+  }
   case Builtin::BI__builtin_hlsl_all: {
     Value *Op0 = EmitScalarExpr(E->getArg(0));
     return Builder.CreateIntrinsic(
@@ -19610,6 +19705,12 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
     assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
            "clip operands types mismatch");
     return handleHlslClip(E, this);
+  case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
+    Intrinsic::ID ID =
+        CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
+    return EmitRuntimeCall(
+        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+  }
   }
   return nullptr;
 }
@@ -19890,6 +19991,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
+  case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: {
     Intrinsic::ID IID;
     switch (BuiltinID) {
@@ -19915,6 +20018,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
       IID = Intrinsic::amdgcn_ds_read_tr6_b96;
       break;
     case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
+    case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
+    case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
       IID = Intrinsic::amdgcn_ds_read_tr16_b64;
       break;
     }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e366add9d8202..19958ae8da990 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1323,10 +1323,10 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
       }
       if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
         auto *Load = CGF.Builder.CreateLoad(Src);
-        auto *UndefVec = llvm::UndefValue::get(ScalableDstTy);
+        auto *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);
         auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
         llvm::Value *Result = CGF.Builder.CreateInsertVector(
-            ScalableDstTy, UndefVec, Load, Zero, "cast.scalable");
+            ScalableDstTy, PoisonVec, Load, Zero, "cast.scalable");
         if (ScalableDstTy != Ty)
           Result = CGF.Builder.CreateBitCast(Result, Ty);
         return Result;
@@ -4476,7 +4476,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
       llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
                                                       "icr.to-use");
       phiToUse->addIncoming(valueToUse, copyBB);
-      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
+      phiToUse->addIncoming(llvm::PoisonValue::get(valueToUse->getType()),
                             originBB);
       valueToUse = phiToUse;
     }
@@ -4626,7 +4626,7 @@ void CodeGenFunction::EmitCallArgs(
       ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
                       MD->param_type_end());
     } else {
-      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
+      const auto *FPT = cast<const FunctionProtoType *>(Prototype.P);
       IsVariadic = FPT->isVariadic();
       ExplicitCC = FPT->getExtInfo().getCC();
       ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
@@ -4822,15 +4822,17 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     return emitWritebackArg(*this, args, CRE);
   }
 
-  assert(type->isReferenceType() == E->isGLValue() &&
-         "reference binding to unmaterialized r-value!");
-
   // Add writeback for HLSLOutParamExpr.
+  // Needs to be before the assert below because HLSLOutArgExpr is an LValue
+  // and is not a reference.
   if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(E)) {
     EmitHLSLOutArgExpr(OE, args, type);
     return;
   }
 
+  assert(type->isReferenceType() == E->isGLValue() &&
+         "reference binding to unmaterialized r-value!");
+
   if (E->isGLValue()) {
     assert(E->getObjectKind() == OK_Ordinary);
     return args.add(EmitReferenceBindingToExpr(E), type);
@@ -5418,6 +5420,14 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           IRCallArgs[FirstIRArg] = Val;
           break;
         }
+      } else if (I->getType()->isArrayParameterType()) {
+        // Don't produce a temporary for ArrayParameterType arguments.
+        // ArrayParameterType arguments are only created from
+        // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
+        // of which create temporaries already. This allows us to just use the
+        // scalar for the decayed array pointer as the argument directly.
+        IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
+        break;
       }
 
       // For non-aggregate args and aggregate args meeting conditions above
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 5f1a07670ce2d..b17eabec060e0 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1173,7 +1173,7 @@ llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
   Indices.push_back(Builder.getInt32(0));
   return Builder.CreateInBoundsGEP(
       ConvertType(QualType(RD->getTypeForDecl(), 0)), Res,
-      RecIndicesTy(llvm::reverse(Indices)), "..counted_by.gep");
+      RecIndicesTy(llvm::reverse(Indices)), "counted_by.gep");
 }
 
 /// This method is typically called in contexts where we can't generate
@@ -1188,7 +1188,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
     const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
   if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
     return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), GEP,
-                                     getIntAlign(), "..counted_by.load");
+                                     getIntAlign(), "counted_by.load");
   return nullptr;
 }
 
@@ -3590,6 +3590,12 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
                                llvm::AttributeList::FunctionIndex, B),
       /*Local=*/true);
   llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
+  bool NoMerge =
+      ClSanitizeDebugDeoptimization ||
+      !CGF.CGM.getCodeGenOpts().OptimizationLevel ||
+      (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
+  if (NoMerge)
+    HandlerCall->addFnAttr(llvm::Attribute::NoMerge);
   if (!MayReturn) {
     HandlerCall->setDoesNotReturn();
     CGF.Builder.CreateUnreachable();
@@ -5874,9 +5880,12 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
 // This function implements trivial copy assignment for HLSL's
 // assignable constant arrays.
 LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
-  LValue TrivialAssignmentRHS = EmitLValue(E->getRHS());
+  // Don't emit an LValue for the RHS because it might not be an LValue
   LValue LHS = EmitLValue(E->getLHS());
-  EmitAggregateAssign(LHS, TrivialAssignmentRHS, E->getLHS()->getType());
+  // In C the RHS of an assignment operator is an RValue.
+  // EmitAggregateAssign takes anan LValue for the RHS. Instead we can call
+  // EmitInitializationToLValue to emit an RValue into an LValue.
+  EmitInitializationToLValue(E->getRHS(), LHS);
   return LHS;
 }
 
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index dd38e5198cf98..4e2392f93e74b 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2122,7 +2122,8 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
       Expr *InitVector = E->getInit(0);
 
       // Initialize from another scalable vector of the same type.
-      if (InitVector->getType() == E->getType())
+      if (InitVector->getType().getCanonicalType() ==
+          E->getType().getCanonicalType())
         return Visit(InitVector);
     }
 
@@ -2395,10 +2396,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
               ScalableDstTy->getElementCount().getKnownMinValue() / 8);
         }
         if (FixedSrcTy->getElementType() == ScalableDstTy->getElementType()) {
-          llvm::Value *UndefVec = llvm::UndefValue::get(ScalableDstTy);
+          llvm::Value *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);
           llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
           llvm::Value *Result = Builder.CreateInsertVector(
-              ScalableDstTy, UndefVec, Src, Zero, "cast.scalable");
+              ScalableDstTy, PoisonVec, Src, Zero, "cast.scalable");
           if (Result->getType() != DstTy)
             Result = Builder.CreateBitCast(Result, DstTy);
           return Result;
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 2c293523fca8c..fb15b1993e74a 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -389,6 +389,11 @@ llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B,
         CGM.getIntrinsic(getThreadIdIntrinsic());
     return buildVectorInput(B, ThreadIDIntrinsic, Ty);
   }
+  if (D.hasAttr<HLSLSV_GroupThreadIDAttr>()) {
+    llvm::Function *GroupThreadIDIntrinsic =
+        CGM.getIntrinsic(getGroupThreadIdIntrinsic());
+    return buildVectorInput(B, GroupThreadIDIntrinsic, Ty);
+  }
   if (D.hasAttr<HLSLSV_GroupIDAttr>()) {
     llvm::Function *GroupIDIntrinsic = CGM.getIntrinsic(Intrinsic::dx_group_id);
     return buildVectorInput(B, GroupIDIntrinsic, Ty);
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 854214d6bc067..f9efb1bc99641 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -86,6 +86,7 @@ class CGHLSLRuntime {
   GENERATE_HLSL_INTRINSIC_FUNCTION(Step, step)
   GENERATE_HLSL_INTRINSIC_FUNCTION(Radians, radians)
   GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupThreadId, thread_id_in_group)
   GENERATE_HLSL_INTRINSIC_FUNCTION(FDot, fdot)
   GENERATE_HLSL_INTRINSIC_FUNCTION(SDot, sdot)
   GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
@@ -103,6 +104,8 @@ class CGHLSLRuntime {
 
   GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding, handle_fromBinding)
   GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, bufferUpdateCounter)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
+                                   group_memory_barrier_with_group_sync)
 
   //===----------------------------------------------------------------------===//
   // End of reserved area for HLSL intrinsic getters.
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index df115f287a942..0b339d0b4ea0e 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -1139,7 +1139,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
   // Identify loop attribute 'code_align' from Attrs.
   // For attribute code_align:
   // n - 'llvm.loop.align i32 n' metadata will be emitted.
-  if (const auto *CodeAlign = getSpecificAttr<const CodeAlignAttr>(Attrs)) {
+  if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(Attrs)) {
     const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment());
     llvm::APSInt ArgVal = CE->getResultAsAPSInt();
     setCodeAlign(ArgVal.getSExtValue());
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index d0ee1be43ba13..c3264cd1bd8c5 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -4666,7 +4666,7 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
                                CGF.getContext().VoidPtrTy);
   }
   enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
-  llvm::Value *TaskArgs[] = {
+  llvm::SmallVector<llvm::Value *, 12> TaskArgs{
       UpLoc,
       ThreadID,
       Result.NewTask,
@@ -4683,12 +4683,18 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
       Data.Schedule.getPointer()
           ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
                                       /*isSigned=*/false)
-          : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
-      Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-                             Result.TaskDupFn, CGF.VoidPtrTy)
-                       : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
+          : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
+  if (Data.HasModifier)
+    TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
+
+  TaskArgs.push_back(Result.TaskDupFn
+                         ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+                               Result.TaskDupFn, CGF.VoidPtrTy)
+                         : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
   CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                          CGM.getModule(), OMPRTL___kmpc_taskloop),
+                          CGM.getModule(), Data.HasModifier
+                                               ? OMPRTL___kmpc_taskloop_5
+                                               : OMPRTL___kmpc_taskloop),
                       TaskArgs);
 }
 
@@ -7766,7 +7772,7 @@ class MappableExprsHandler {
       if (const auto *Base = Data.dyn_cast<const CXXRecordDecl *>())
         getPlainLayout(Base, Layout, /*AsBase=*/true);
       else
-        Layout.push_back(Data.get<const FieldDecl *>());
+        Layout.push_back(cast<const FieldDecl *>(Data));
     }
   }
 
@@ -8329,9 +8335,9 @@ class MappableExprsHandler {
       MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
       const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
           llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
-    assert(CurDir.is<const OMPExecutableDirective *>() &&
+    assert(isa<const OMPExecutableDirective *>(CurDir) &&
            "Expect a executable directive");
-    const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
+    const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);
     generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
                               SkipVarSet);
   }
@@ -8341,9 +8347,9 @@ class MappableExprsHandler {
   /// in \a CombinedInfo).
   void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
                                 llvm::OpenMPIRBuilder &OMPBuilder) const {
-    assert(CurDir.is<const OMPDeclareMapperDecl *>() &&
+    assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
            "Expect a declare mapper directive");
-    const auto *CurMapperDir = CurDir.get<const OMPDeclareMapperDecl *>();
+    const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(CurDir);
     generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
                               OMPBuilder);
   }
@@ -8515,9 +8521,9 @@ class MappableExprsHandler {
         DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom, Unknown,
                                         /*IsImpicit = */ true, nullptr,
                                         nullptr);
-    assert(CurDir.is<const OMPExecutableDirective *>() &&
+    assert(isa<const OMPExecutableDirective *>(CurDir) &&
            "Expect a executable directive");
-    const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
+    const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);
     bool HasMapBasePtr = false;
     bool HasMapArraySec = false;
     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 5e7715743afb5..56d502d92806e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -122,6 +122,7 @@ struct OMPTaskDataTy final {
   bool IsReductionWithTaskMod = false;
   bool IsWorksharingReduction = false;
   bool HasNowaitClause = false;
+  bool HasModifier = false;
 };
 
 /// Class intended to support codegen of all kind of the reduction clauses.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index cab5161504578..d159bb29f4b16 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -7831,10 +7831,14 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
     // grainsize clause
     Data.Schedule.setInt(/*IntVal=*/false);
     Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
+    Data.HasModifier =
+        (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
   } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
     // num_tasks clause
     Data.Schedule.setInt(/*IntVal=*/true);
     Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
+    Data.HasModifier =
+        (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
   }
 
   auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 40a6b1b8b3395..f597f484c1a64 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1103,6 +1103,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     Fn->addFnAttr("ptrauth-auth-traps");
   if (CodeGenOpts.PointerAuth.IndirectGotos)
     Fn->addFnAttr("ptrauth-indirect-gotos");
+  if (CodeGenOpts.PointerAuth.AArch64JumpTableHardening)
+    Fn->addFnAttr("aarch64-jump-table-hardening");
 
   // Apply xray attributes to the function (as a string, for now)
   bool AlwaysXRayAttr = false;
@@ -3219,23 +3221,17 @@ void CodeGenFunction::EmitKCFIOperandBundle(
     Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
 }
 
-llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
-    const MultiVersionResolverOption &RO) {
-  llvm::SmallVector<StringRef, 8> CondFeatures;
-  for (const StringRef &Feature : RO.Conditions.Features)
-    CondFeatures.push_back(Feature);
-  if (!CondFeatures.empty()) {
-    return EmitAArch64CpuSupports(CondFeatures);
-  }
-  return nullptr;
+llvm::Value *
+CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
+  return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(RO.Features);
 }
 
-llvm::Value *CodeGenFunction::FormX86ResolverCondition(
-    const MultiVersionResolverOption &RO) {
+llvm::Value *
+CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
   llvm::Value *Condition = nullptr;
 
-  if (!RO.Conditions.Architecture.empty()) {
-    StringRef Arch = RO.Conditions.Architecture;
+  if (RO.Architecture) {
+    StringRef Arch = *RO.Architecture;
     // If arch= specifies an x86-64 micro-architecture level, test the feature
     // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
     if (Arch.starts_with("x86-64"))
@@ -3244,8 +3240,8 @@ llvm::Value *CodeGenFunction::FormX86ResolverCondition(
       Condition = EmitX86CpuIs(Arch);
   }
 
-  if (!RO.Conditions.Features.empty()) {
-    llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
+  if (!RO.Features.empty()) {
+    llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Features);
     Condition =
         Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
   }
@@ -3275,7 +3271,7 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
 }
 
 void CodeGenFunction::EmitMultiVersionResolver(
-    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
+    llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
 
   llvm::Triple::ArchType ArchType =
       getContext().getTargetInfo().getTriple().getArch();
@@ -3298,26 +3294,8 @@ void CodeGenFunction::EmitMultiVersionResolver(
   }
 }
 
-static unsigned getPriorityFromAttrString(StringRef AttrStr) {
-  SmallVector<StringRef, 8> Attrs;
-
-  AttrStr.split(Attrs, ';');
-
-  // Default Priority is zero.
-  unsigned Priority = 0;
-  for (auto Attr : Attrs) {
-    if (Attr.consume_front("priority=")) {
-      unsigned Result;
-      if (!Attr.getAsInteger(0, Result))
-        Priority = Result;
-    }
-  }
-
-  return Priority;
-}
-
 void CodeGenFunction::EmitRISCVMultiVersionResolver(
-    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
+    llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
 
   if (getContext().getTargetInfo().getTriple().getOS() !=
       llvm::Triple::OSType::Linux) {
@@ -3333,20 +3311,10 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
   bool HasDefault = false;
   unsigned DefaultIndex = 0;
 
-  SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> CurrOptions(
-      Options);
-
-  llvm::stable_sort(
-      CurrOptions, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
-                      const CodeGenFunction::MultiVersionResolverOption &RHS) {
-        return getPriorityFromAttrString(LHS.Conditions.Features[0]) >
-               getPriorityFromAttrString(RHS.Conditions.Features[0]);
-      });
-
   // Check the each candidate function.
-  for (unsigned Index = 0; Index < CurrOptions.size(); Index++) {
+  for (unsigned Index = 0; Index < Options.size(); Index++) {
 
-    if (CurrOptions[Index].Conditions.Features[0].starts_with("default")) {
+    if (Options[Index].Features.empty()) {
       HasDefault = true;
       DefaultIndex = Index;
       continue;
@@ -3354,15 +3322,6 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
 
     Builder.SetInsertPoint(CurBlock);
 
-    std::vector<std::string> TargetAttrFeats =
-        getContext()
-            .getTargetInfo()
-            .parseTargetAttr(CurrOptions[Index].Conditions.Features[0])
-            .Features;
-
-    if (TargetAttrFeats.empty())
-      continue;
-
     // FeaturesCondition: The bitmask of the required extension has been
     // enabled by the runtime object.
     // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
@@ -3385,20 +3344,32 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
     // Without checking the length first, we may access an incorrect memory
     // address when using different versions.
     llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
+    llvm::SmallVector<std::string, 8> TargetAttrFeats;
 
-    for (auto &Feat : TargetAttrFeats) {
-      StringRef CurrFeat = Feat;
-      if (CurrFeat.starts_with('+'))
-        CurrTargetAttrFeats.push_back(CurrFeat.substr(1));
+    for (StringRef Feat : Options[Index].Features) {
+      std::vector<std::string> FeatStr =
+          getContext().getTargetInfo().parseTargetAttr(Feat).Features;
+
+      assert(FeatStr.size() == 1 && "Feature string not delimited");
+
+      std::string &CurrFeat = FeatStr.front();
+      if (CurrFeat[0] == '+')
+        TargetAttrFeats.push_back(CurrFeat.substr(1));
     }
 
+    if (TargetAttrFeats.empty())
+      continue;
+
+    for (std::string &Feat : TargetAttrFeats)
+      CurrTargetAttrFeats.push_back(Feat);
+
     Builder.SetInsertPoint(CurBlock);
     llvm::Value *FeatsCondition = EmitRISCVCpuSupports(CurrTargetAttrFeats);
 
     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
     CGBuilderTy RetBuilder(*this, RetBlock);
-    CreateMultiVersionResolverReturn(
-        CGM, Resolver, RetBuilder, CurrOptions[Index].Function, SupportsIFunc);
+    CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder,
+                                     Options[Index].Function, SupportsIFunc);
     llvm::BasicBlock *ElseBlock = createBasicBlock("resolver_else", Resolver);
 
     Builder.SetInsertPoint(CurBlock);
@@ -3410,9 +3381,8 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
   // Finally, emit the default one.
   if (HasDefault) {
     Builder.SetInsertPoint(CurBlock);
-    CreateMultiVersionResolverReturn(CGM, Resolver, Builder,
-                                     CurrOptions[DefaultIndex].Function,
-                                     SupportsIFunc);
+    CreateMultiVersionResolverReturn(
+        CGM, Resolver, Builder, Options[DefaultIndex].Function, SupportsIFunc);
     return;
   }
 
@@ -3426,17 +3396,16 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
 }
 
 void CodeGenFunction::EmitAArch64MultiVersionResolver(
-    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
+    llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
   assert(!Options.empty() && "No multiversion resolver options found");
-  assert(Options.back().Conditions.Features.size() == 0 &&
-         "Default case must be last");
+  assert(Options.back().Features.size() == 0 && "Default case must be last");
   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
   assert(SupportsIFunc &&
          "Multiversion resolver requires target IFUNC support");
   bool AArch64CpuInitialized = false;
   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
 
-  for (const MultiVersionResolverOption &RO : Options) {
+  for (const FMVResolverOption &RO : Options) {
     Builder.SetInsertPoint(CurBlock);
     llvm::Value *Condition = FormAArch64ResolverCondition(RO);
 
@@ -3472,7 +3441,7 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
 }
 
 void CodeGenFunction::EmitX86MultiVersionResolver(
-    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
+    llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
 
   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
 
@@ -3481,7 +3450,7 @@ void CodeGenFunction::EmitX86MultiVersionResolver(
   Builder.SetInsertPoint(CurBlock);
   EmitX86CpuInit();
 
-  for (const MultiVersionResolverOption &RO : Options) {
+  for (const FMVResolverOption &RO : Options) {
     Builder.SetInsertPoint(CurBlock);
     llvm::Value *Condition = FormX86ResolverCondition(RO);
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6c47644d52b94..f7235d52022e4 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5392,35 +5392,27 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
 
-  struct MultiVersionResolverOption {
+  struct FMVResolverOption {
     llvm::Function *Function;
-    struct Conds {
-      StringRef Architecture;
-      llvm::SmallVector<StringRef, 8> Features;
+    llvm::SmallVector<StringRef, 8> Features;
+    std::optional<StringRef> Architecture;
 
-      Conds(StringRef Arch, ArrayRef<StringRef> Feats)
-          : Architecture(Arch), Features(Feats) {}
-    } Conditions;
-
-    MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
-                               ArrayRef<StringRef> Feats)
-        : Function(F), Conditions(Arch, Feats) {}
+    FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
+                      std::optional<StringRef> Arch = std::nullopt)
+        : Function(F), Features(Feats), Architecture(Arch) {}
   };
 
   // Emits the body of a multiversion function's resolver. Assumes that the
   // options are already sorted in the proper order, with the 'default' option
   // last (if it exists).
   void EmitMultiVersionResolver(llvm::Function *Resolver,
-                                ArrayRef<MultiVersionResolverOption> Options);
-  void
-  EmitX86MultiVersionResolver(llvm::Function *Resolver,
-                              ArrayRef<MultiVersionResolverOption> Options);
-  void
-  EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
-                                  ArrayRef<MultiVersionResolverOption> Options);
-  void
-  EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
-                                ArrayRef<MultiVersionResolverOption> Options);
+                                ArrayRef<FMVResolverOption> Options);
+  void EmitX86MultiVersionResolver(llvm::Function *Resolver,
+                                   ArrayRef<FMVResolverOption> Options);
+  void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
+                                       ArrayRef<FMVResolverOption> Options);
+  void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
+                                     ArrayRef<FMVResolverOption> Options);
 
 private:
   QualType getVarArgType(const Expr *Arg);
@@ -5439,10 +5431,9 @@ class CodeGenFunction : public CodeGenTypeCache {
   llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
   llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
   llvm::Value *EmitX86CpuInit();
-  llvm::Value *FormX86ResolverCondition(const MultiVersionResolverOption &RO);
+  llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
   llvm::Value *EmitAArch64CpuInit();
-  llvm::Value *
-  FormAArch64ResolverCondition(const MultiVersionResolverOption &RO);
+  llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
   llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
   llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
 };
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 4d18be3ebb55b..6b2f005ff2d6f 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -633,7 +633,8 @@ static bool checkAliasedGlobal(
     // mangled name.
     for (const auto &[Decl, Name] : MangledDeclNames) {
       if (const auto *ND = dyn_cast<NamedDecl>(Decl.getDecl())) {
-        if (ND->getName() == GV->getName()) {
+        IdentifierInfo *II = ND->getIdentifier();
+        if (II && II->getName() == GV->getName()) {
           Diags.Report(Location, diag::note_alias_mangled_name_alternative)
               << Name
               << FixItHint::CreateReplacement(
@@ -4675,23 +4676,12 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
 static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
                                                       llvm::Function *NewFn);
 
-static unsigned
-TargetMVPriority(const TargetInfo &TI,
-                 const CodeGenFunction::MultiVersionResolverOption &RO) {
-  unsigned Priority = 0;
-  unsigned NumFeatures = 0;
-  for (StringRef Feat : RO.Conditions.Features) {
-    Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
-    NumFeatures++;
-  }
-
-  if (!RO.Conditions.Architecture.empty())
-    Priority = std::max(
-        Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
-
-  Priority += TI.multiVersionFeatureCost() * NumFeatures;
-
-  return Priority;
+static unsigned getFMVPriority(const TargetInfo &TI,
+                               const CodeGenFunction::FMVResolverOption &RO) {
+  llvm::SmallVector<StringRef, 8> Features{RO.Features};
+  if (RO.Architecture)
+    Features.push_back(*RO.Architecture);
+  return TI.getFMVPriority(Features);
 }
 
 // Multiversion functions should be at most 'WeakODRLinkage' so that a different
@@ -4737,7 +4727,7 @@ void CodeGenModule::emitMultiVersionFunctions() {
     // target_version("default")) or target_clones() is present and defined
     // in this TU. For other architectures it is always emitted.
     bool ShouldEmitResolver = !getTarget().getTriple().isAArch64();
-    SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
+    SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
 
     getContext().forEachMultiversionedFunctionVersion(
         FD, [&](const FunctionDecl *CurFD) {
@@ -4745,20 +4735,17 @@ void CodeGenModule::emitMultiVersionFunctions() {
           bool IsDefined = CurFD->doesThisDeclarationHaveABody();
 
           if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
-            TA->getAddedFeatures(Feats);
+            assert(getTarget().getTriple().isX86() && "Unsupported target");
+            TA->getX86AddedFeatures(Feats);
             llvm::Function *Func = createFunction(CurFD);
-            Options.emplace_back(Func, TA->getArchitecture(), Feats);
+            Options.emplace_back(Func, Feats, TA->getX86Architecture());
           } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
             if (TVA->isDefaultVersion() && IsDefined)
               ShouldEmitResolver = true;
             llvm::Function *Func = createFunction(CurFD);
-            if (getTarget().getTriple().isRISCV()) {
-              Feats.push_back(TVA->getName());
-            } else {
-              assert(getTarget().getTriple().isAArch64());
-              TVA->getFeatures(Feats);
-            }
-            Options.emplace_back(Func, /*Architecture*/ "", Feats);
+            char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
+            TVA->getFeatures(Feats, Delim);
+            Options.emplace_back(Func, Feats);
           } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
             if (IsDefined)
               ShouldEmitResolver = true;
@@ -4767,21 +4754,15 @@ void CodeGenModule::emitMultiVersionFunctions() {
                 continue;
 
               llvm::Function *Func = createFunction(CurFD, I);
-              StringRef Architecture;
               Feats.clear();
-              if (getTarget().getTriple().isAArch64())
-                TC->getFeatures(Feats, I);
-              else if (getTarget().getTriple().isRISCV()) {
-                StringRef Version = TC->getFeatureStr(I);
-                Feats.push_back(Version);
+              if (getTarget().getTriple().isX86()) {
+                TC->getX86Feature(Feats, I);
+                Options.emplace_back(Func, Feats, TC->getX86Architecture(I));
               } else {
-                StringRef Version = TC->getFeatureStr(I);
-                if (Version.starts_with("arch="))
-                  Architecture = Version.drop_front(sizeof("arch=") - 1);
-                else if (Version != "default")
-                  Feats.push_back(Version);
+                char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
+                TC->getFeatures(Feats, I, Delim);
+                Options.emplace_back(Func, Feats);
               }
-              Options.emplace_back(Func, Architecture, Feats);
             }
           } else
             llvm_unreachable("unexpected MultiVersionKind");
@@ -4818,9 +4799,9 @@ void CodeGenModule::emitMultiVersionFunctions() {
 
     const TargetInfo &TI = getTarget();
     llvm::stable_sort(
-        Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
-                       const CodeGenFunction::MultiVersionResolverOption &RHS) {
-          return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
+        Options, [&TI](const CodeGenFunction::FMVResolverOption &LHS,
+                       const CodeGenFunction::FMVResolverOption &RHS) {
+          return getFMVPriority(TI, LHS) > getFMVPriority(TI, RHS);
         });
     CodeGenFunction CGF(*this);
     CGF.EmitMultiVersionResolver(ResolverFunc, Options);
@@ -4879,7 +4860,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
     ResolverFunc->setComdat(
         getModule().getOrInsertComdat(ResolverFunc->getName()));
 
-  SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
+  SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
   const TargetInfo &Target = getTarget();
   unsigned Index = 0;
   for (const IdentifierInfo *II : DD->cpus()) {
@@ -4913,25 +4894,23 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
     llvm::erase_if(Features, [&Target](StringRef Feat) {
       return !Target.validateCpuSupports(Feat);
     });
-    Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
+    Options.emplace_back(cast<llvm::Function>(Func), Features);
     ++Index;
   }
 
-  llvm::stable_sort(
-      Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
-                  const CodeGenFunction::MultiVersionResolverOption &RHS) {
-        return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) >
-               llvm::X86::getCpuSupportsMask(RHS.Conditions.Features);
-      });
+  llvm::stable_sort(Options, [](const CodeGenFunction::FMVResolverOption &LHS,
+                                const CodeGenFunction::FMVResolverOption &RHS) {
+    return llvm::X86::getCpuSupportsMask(LHS.Features) >
+           llvm::X86::getCpuSupportsMask(RHS.Features);
+  });
 
   // If the list contains multiple 'default' versions, such as when it contains
   // 'pentium' and 'generic', don't emit the call to the generic one (since we
   // always run on at least a 'pentium'). We do this by deleting the 'least
   // advanced' (read, lowest mangling letter).
-  while (Options.size() > 1 &&
-         llvm::all_of(llvm::X86::getCpuSupportsMask(
-                          (Options.end() - 2)->Conditions.Features),
-                      [](auto X) { return X == 0; })) {
+  while (Options.size() > 1 && llvm::all_of(llvm::X86::getCpuSupportsMask(
+                                                (Options.end() - 2)->Features),
+                                            [](auto X) { return X == 0; })) {
     StringRef LHSName = (Options.end() - 2)->Function->getName();
     StringRef RHSName = (Options.end() - 1)->Function->getName();
     if (LHSName.compare(RHSName) < 0)
diff --git a/clang/lib/CodeGen/ConstantInitBuilder.cpp b/clang/lib/CodeGen/ConstantInitBuilder.cpp
index 549d5dd66b123..ddbf3ef743370 100644
--- a/clang/lib/CodeGen/ConstantInitBuilder.cpp
+++ b/clang/lib/CodeGen/ConstantInitBuilder.cpp
@@ -20,10 +20,10 @@ using namespace CodeGen;
 
 llvm::Type *ConstantInitFuture::getType() const {
   assert(Data && "dereferencing null future");
-  if (Data.is<llvm::Constant*>()) {
-    return Data.get<llvm::Constant*>()->getType();
+  if (const auto *C = dyn_cast<llvm::Constant *>(Data)) {
+    return C->getType();
   } else {
-    return Data.get<ConstantInitBuilderBase*>()->Buffer[0]->getType();
+    return cast<ConstantInitBuilderBase *>(Data)->Buffer[0]->getType();
   }
 }
 
@@ -37,10 +37,10 @@ void ConstantInitFuture::abandon() {
 
 void ConstantInitFuture::installInGlobal(llvm::GlobalVariable *GV) {
   assert(Data && "installing null future");
-  if (Data.is<llvm::Constant*>()) {
-    GV->setInitializer(Data.get<llvm::Constant*>());
+  if (auto *C = dyn_cast<llvm::Constant *>(Data)) {
+    GV->setInitializer(C);
   } else {
-    auto &builder = *Data.get<ConstantInitBuilderBase*>();
+    auto &builder = *cast<ConstantInitBuilderBase *>(Data);
     assert(builder.Buffer.size() == 1);
     builder.setGlobalInitializer(GV, builder.Buffer[0]);
     builder.Buffer.clear();
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 144a53236f061..c32cb7fb2811a 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1465,14 +1465,12 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
   //
 }
 
-static void appendOneArg(InputArgList &Args, const Arg *Opt,
-                         const Arg *BaseArg) {
+static void appendOneArg(InputArgList &Args, const Arg *Opt) {
   // The args for config files or /clang: flags belong to different InputArgList
   // objects than Args. This copies an Arg from one of those other InputArgLists
   // to the ownership of Args.
   unsigned Index = Args.MakeIndex(Opt->getSpelling());
-  Arg *Copy = new llvm::opt::Arg(Opt->getOption(), Args.getArgString(Index),
-                                 Index, BaseArg);
+  Arg *Copy = new Arg(Opt->getOption(), Args.getArgString(Index), Index);
   Copy->getValues() = Opt->getValues();
   if (Opt->isClaimed())
     Copy->claim();
@@ -1508,38 +1506,59 @@ bool Driver::readConfigFile(StringRef FileName,
   }
 
   // Try reading the given file.
-  SmallVector<const char *, 32> NewCfgArgs;
-  if (llvm::Error Err = ExpCtx.readConfigFile(FileName, NewCfgArgs)) {
+  SmallVector<const char *, 32> NewCfgFileArgs;
+  if (llvm::Error Err = ExpCtx.readConfigFile(FileName, NewCfgFileArgs)) {
     Diag(diag::err_drv_cannot_read_config_file)
         << FileName << toString(std::move(Err));
     return true;
   }
 
+  // Populate head and tail lists. The tail list is used only when linking.
+  SmallVector<const char *, 32> NewCfgHeadArgs, NewCfgTailArgs;
+  for (const char *Opt : NewCfgFileArgs) {
+    // An $-prefixed option should go to the tail list.
+    if (Opt[0] == '$' && Opt[1])
+      NewCfgTailArgs.push_back(Opt + 1);
+    else
+      NewCfgHeadArgs.push_back(Opt);
+  }
+
   // Read options from config file.
   llvm::SmallString<128> CfgFileName(FileName);
   llvm::sys::path::native(CfgFileName);
-  bool ContainErrors;
-  std::unique_ptr<InputArgList> NewOptions = std::make_unique<InputArgList>(
-      ParseArgStrings(NewCfgArgs, /*UseDriverMode=*/true, ContainErrors));
+  bool ContainErrors = false;
+  auto NewHeadOptions = std::make_unique<InputArgList>(
+      ParseArgStrings(NewCfgHeadArgs, /*UseDriverMode=*/true, ContainErrors));
+  if (ContainErrors)
+    return true;
+  auto NewTailOptions = std::make_unique<InputArgList>(
+      ParseArgStrings(NewCfgTailArgs, /*UseDriverMode=*/true, ContainErrors));
   if (ContainErrors)
     return true;
 
   // Claim all arguments that come from a configuration file so that the driver
   // does not warn on any that is unused.
-  for (Arg *A : *NewOptions)
+  for (Arg *A : *NewHeadOptions)
+    A->claim();
+  for (Arg *A : *NewTailOptions)
     A->claim();
 
-  if (!CfgOptions)
-    CfgOptions = std::move(NewOptions);
+  if (!CfgOptionsHead)
+    CfgOptionsHead = std::move(NewHeadOptions);
   else {
     // If this is a subsequent config file, append options to the previous one.
-    for (auto *Opt : *NewOptions) {
-      const Arg *BaseArg = &Opt->getBaseArg();
-      if (BaseArg == Opt)
-        BaseArg = nullptr;
-      appendOneArg(*CfgOptions, Opt, BaseArg);
-    }
+    for (auto *Opt : *NewHeadOptions)
+      appendOneArg(*CfgOptionsHead, Opt);
+  }
+
+  if (!CfgOptionsTail)
+    CfgOptionsTail = std::move(NewTailOptions);
+  else {
+    // If this is a subsequent config file, append options to the previous one.
+    for (auto *Opt : *NewTailOptions)
+      appendOneArg(*CfgOptionsTail, Opt);
   }
+
   ConfigFiles.push_back(std::string(CfgFileName));
   return false;
 }
@@ -1716,21 +1735,17 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   // Try parsing configuration file.
   if (!ContainsError)
     ContainsError = loadConfigFiles();
-  bool HasConfigFile = !ContainsError && (CfgOptions.get() != nullptr);
+  bool HasConfigFileHead = !ContainsError && CfgOptionsHead;
+  bool HasConfigFileTail = !ContainsError && CfgOptionsTail;
 
   // All arguments, from both config file and command line.
-  InputArgList Args = std::move(HasConfigFile ? std::move(*CfgOptions)
-                                              : std::move(*CLOptions));
+  InputArgList Args =
+      HasConfigFileHead ? std::move(*CfgOptionsHead) : std::move(*CLOptions);
 
-  if (HasConfigFile)
-    for (auto *Opt : *CLOptions) {
-      if (Opt->getOption().matches(options::OPT_config))
-        continue;
-      const Arg *BaseArg = &Opt->getBaseArg();
-      if (BaseArg == Opt)
-        BaseArg = nullptr;
-      appendOneArg(Args, Opt, BaseArg);
-    }
+  if (HasConfigFileHead)
+    for (auto *Opt : *CLOptions)
+      if (!Opt->getOption().matches(options::OPT_config))
+        appendOneArg(Args, Opt);
 
   // In CL mode, look for any pass-through arguments
   if (IsCLMode() && !ContainsError) {
@@ -1748,9 +1763,8 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
                           ContainsError));
 
       if (!ContainsError)
-        for (auto *Opt : *CLModePassThroughOptions) {
-          appendOneArg(Args, Opt, nullptr);
-        }
+        for (auto *Opt : *CLModePassThroughOptions)
+          appendOneArg(Args, Opt);
     }
   }
 
@@ -2027,6 +2041,15 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   // Construct the list of inputs.
   InputList Inputs;
   BuildInputs(C->getDefaultToolChain(), *TranslatedArgs, Inputs);
+  if (HasConfigFileTail && Inputs.size()) {
+    Arg *FinalPhaseArg;
+    if (getFinalPhase(*TranslatedArgs, &FinalPhaseArg) == phases::Link) {
+      DerivedArgList TranslatedLinkerIns(*CfgOptionsTail);
+      for (Arg *A : *CfgOptionsTail)
+        TranslatedLinkerIns.append(A);
+      BuildInputs(C->getDefaultToolChain(), TranslatedLinkerIns, Inputs);
+    }
+  }
 
   // Determine if there are any offload static libraries.
   if (checkForOffloadStaticLib(*C, *TranslatedArgs))
@@ -7314,17 +7337,18 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     YcArg = YuArg = nullptr;
   }
 
-  unsigned LastPLSize = 0;
+  bool LinkOnly = phases::Link == FinalPhase && Inputs.size() > 0;
   for (auto &I : Inputs) {
     types::ID InputType = I.first;
     const Arg *InputArg = I.second;
 
     auto PL = types::getCompilationPhases(InputType);
-    LastPLSize = PL.size();
+
+    phases::ID InitialPhase = PL[0];
+    LinkOnly = LinkOnly && phases::Link == InitialPhase && PL.size() == 1;
 
     // If the first step comes after the final phase we are doing as part of
     // this compilation, warn the user about it.
-    phases::ID InitialPhase = PL[0];
     if (InitialPhase > FinalPhase) {
       if (InputArg->isClaimed())
         continue;
@@ -7379,10 +7403,8 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     }
   }
 
-  // If we are linking, claim any options which are obviously only used for
-  // compilation.
-  // FIXME: Understand why the last Phase List length is used here.
-  if (FinalPhase == phases::Link && LastPLSize == 1) {
+  // Claim any options which are obviously only used for compilation.
+  if (LinkOnly) {
     Args.ClaimAllArgs(options::OPT_CompileOnly_Group);
     Args.ClaimAllArgs(options::OPT_cl_compile_Group);
   }
@@ -7948,7 +7970,13 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
           ToolChain::getOpenMPTriple(Arg->getValue(0)) == TC->getTriple()) {
         Arg->claim();
         unsigned Index = Args.getBaseArgs().MakeIndex(Arg->getValue(1));
+        unsigned Prev = Index;
         ExtractedArg = getOpts().ParseOneArg(Args, Index);
+        if (!ExtractedArg || Index > Prev + 1) {
+          TC->getDriver().Diag(diag::err_drv_invalid_Xopenmp_target_with_args)
+              << Arg->getAsString(Args);
+          continue;
+        }
         Arg = ExtractedArg.get();
       }
     }
@@ -9533,15 +9561,10 @@ InputInfoList Driver::BuildJobsForActionNoCache(
     }
   } else {
     if (UnbundlingResults.empty())
-      T->ConstructJob(
-          C, *JA, Result, InputInfos,
-          C.getArgsForToolChain(TC, BoundArch, JA->getOffloadingDeviceKind()),
-          LinkingOutput);
+      T->ConstructJob(C, *JA, Result, InputInfos, Args, LinkingOutput);
     else
-      T->ConstructJobMultipleOutputs(
-          C, *JA, UnbundlingResults, InputInfos,
-          C.getArgsForToolChain(TC, BoundArch, JA->getOffloadingDeviceKind()),
-          LinkingOutput);
+      T->ConstructJobMultipleOutputs(C, *JA, UnbundlingResults, InputInfos,
+                                     Args, LinkingOutput);
   }
   return {Result};
 }
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index e6ee2317a160c..b8181ce6dc012 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -52,6 +52,26 @@ bool arm::isARMAProfile(const llvm::Triple &Triple) {
   return llvm::ARM::parseArchProfile(Arch) == llvm::ARM::ProfileKind::A;
 }
 
+/// Is the triple {arm,armeb,thumb,thumbeb}-none-none-{eabi,eabihf} ?
+bool arm::isARMEABIBareMetal(const llvm::Triple &Triple) {
+  auto arch = Triple.getArch();
+  if (arch != llvm::Triple::arm && arch != llvm::Triple::thumb &&
+      arch != llvm::Triple::armeb && arch != llvm::Triple::thumbeb)
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  if (Triple.getEnvironment() != llvm::Triple::EABI &&
+      Triple.getEnvironment() != llvm::Triple::EABIHF)
+    return false;
+
+  return true;
+}
+
 // Get Arch/CPU from args.
 void arm::getARMArchCPUFromArgs(const ArgList &Args, llvm::StringRef &Arch,
                                 llvm::StringRef &CPU, bool FromAs) {
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.h b/clang/lib/Driver/ToolChains/Arch/ARM.h
index fa62ac89e3a13..a23a8793a89e2 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.h
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.h
@@ -75,6 +75,7 @@ int getARMSubArchVersionNumber(const llvm::Triple &Triple);
 bool isARMMProfile(const llvm::Triple &Triple);
 bool isARMAProfile(const llvm::Triple &Triple);
 bool isARMBigEndian(const llvm::Triple &Triple, const llvm::opt::ArgList &Args);
+bool isARMEABIBareMetal(const llvm::Triple &Triple);
 
 } // end namespace arm
 } // end namespace tools
diff --git a/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp b/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
index 5be57e866d85e..bbd9397aa2378 100644
--- a/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
@@ -275,6 +275,15 @@ void loongarch::getLoongArchTargetFeatures(const Driver &D,
       Features.push_back("-lam-bh");
   }
 
+  // Select lamcas feature determined by -m[no-]lamcas.
+  if (const Arg *A =
+          Args.getLastArg(options::OPT_mlamcas, options::OPT_mno_lamcas)) {
+    if (A->getOption().matches(options::OPT_mlamcas))
+      Features.push_back("+lamcas");
+    else
+      Features.push_back("-lamcas");
+  }
+
   // Select ld-seq-sa feature determined by -m[no-]ld-seq-sa.
   if (const Arg *A = Args.getLastArg(options::OPT_mld_seq_sa,
                                      options::OPT_mno_ld_seq_sa)) {
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index f9a73f60973e4..eecaaa9a42930 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -128,27 +128,6 @@ BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
   }
 }
 
-/// Is the triple {arm,armeb,thumb,thumbeb}-none-none-{eabi,eabihf} ?
-static bool isARMBareMetal(const llvm::Triple &Triple) {
-  if (Triple.getArch() != llvm::Triple::arm &&
-      Triple.getArch() != llvm::Triple::thumb &&
-      Triple.getArch() != llvm::Triple::armeb &&
-      Triple.getArch() != llvm::Triple::thumbeb)
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
-
-  if (Triple.getEnvironment() != llvm::Triple::EABI &&
-      Triple.getEnvironment() != llvm::Triple::EABIHF)
-    return false;
-
-  return true;
-}
-
 /// Is the triple {aarch64.aarch64_be}-none-elf?
 static bool isAArch64BareMetal(const llvm::Triple &Triple) {
   if (Triple.getArch() != llvm::Triple::aarch64 &&
@@ -267,7 +246,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
 }
 
 bool BareMetal::handlesTarget(const llvm::Triple &Triple) {
-  return isARMBareMetal(Triple) || isAArch64BareMetal(Triple) ||
+  return arm::isARMEABIBareMetal(Triple) || isAArch64BareMetal(Triple) ||
          isRISCVBareMetal(Triple) || isPPCBareMetal(Triple);
 }
 
@@ -414,38 +393,6 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   }
 }
 
-void BareMetal::AddCXXStdlibLibArgs(const ArgList &Args,
-                                    ArgStringList &CmdArgs) const {
-  switch (GetCXXStdlibType(Args)) {
-  case ToolChain::CST_Libcxx:
-    CmdArgs.push_back("-lc++");
-    if (Args.hasArg(options::OPT_fexperimental_library))
-      CmdArgs.push_back("-lc++experimental");
-    CmdArgs.push_back("-lc++abi");
-    break;
-  case ToolChain::CST_Libstdcxx:
-    CmdArgs.push_back("-lstdc++");
-    CmdArgs.push_back("-lsupc++");
-    break;
-  }
-  CmdArgs.push_back("-lunwind");
-}
-
-void BareMetal::AddLinkRuntimeLib(const ArgList &Args,
-                                  ArgStringList &CmdArgs) const {
-  ToolChain::RuntimeLibType RLT = GetRuntimeLibType(Args);
-  switch (RLT) {
-  case ToolChain::RLT_CompilerRT: {
-    CmdArgs.push_back(getCompilerRTArgString(Args, "builtins"));
-    return;
-  }
-  case ToolChain::RLT_Libgcc:
-    CmdArgs.push_back("-lgcc");
-    return;
-  }
-  llvm_unreachable("Unhandled RuntimeLibType.");
-}
-
 void baremetal::StaticLibTool::ConstructJob(Compilation &C, const JobAction &JA,
                                             const InputInfo &Output,
                                             const InputInfoList &Inputs,
@@ -532,14 +479,21 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   for (const auto &LibPath : TC.getLibraryPaths())
     CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-L", LibPath)));
 
-  if (TC.ShouldLinkCXXStdlib(Args))
+  if (TC.ShouldLinkCXXStdlib(Args)) {
+    bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
+                               !Args.hasArg(options::OPT_static);
+    if (OnlyLibstdcxxStatic)
+      CmdArgs.push_back("-Bstatic");
     TC.AddCXXStdlibLibArgs(Args, CmdArgs);
+    if (OnlyLibstdcxxStatic)
+      CmdArgs.push_back("-Bdynamic");
+    CmdArgs.push_back("-lm");
+  }
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
-    CmdArgs.push_back("-lc");
-    CmdArgs.push_back("-lm");
+    AddRunTimeLibs(TC, D, CmdArgs, Args);
 
-    TC.AddLinkRuntimeLib(Args, CmdArgs);
+    CmdArgs.push_back("-lc");
   }
 
   if (D.isUsingLTO()) {
@@ -561,7 +515,7 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // The R_ARM_TARGET2 relocation must be treated as R_ARM_REL32 on arm*-*-elf
   // and arm*-*-eabi (the default is R_ARM_GOT_PREL, used on arm*-*-linux and
   // arm*-*-*bsd).
-  if (isARMBareMetal(TC.getTriple()))
+  if (arm::isARMEABIBareMetal(TC.getTriple()))
     CmdArgs.push_back("--target2=rel");
 
   CmdArgs.push_back("-o");
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index b385c8cf76aab..483b5efab5e6e 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -67,10 +67,6 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
   void AddClangCXXStdlibIncludeArgs(
       const llvm::opt::ArgList &DriverArgs,
       llvm::opt::ArgStringList &CC1Args) const override;
-  void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
-                           llvm::opt::ArgStringList &CmdArgs) const override;
-  void AddLinkRuntimeLib(const llvm::opt::ArgList &Args,
-                         llvm::opt::ArgStringList &CmdArgs) const;
   std::string computeSysRoot() const override;
   SanitizerMask getSupportedSanitizers() const override;
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 933cdbff2ad00..df98170dc528a 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1459,6 +1459,7 @@ static bool isSignedCharDefault(const llvm::Triple &Triple) {
   case llvm::Triple::riscv64:
   case llvm::Triple::systemz:
   case llvm::Triple::xcore:
+  case llvm::Triple::xtensa:
     return false;
   }
 }
@@ -1986,6 +1987,8 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   Args.addOptInFlag(CmdArgs,
                     options::OPT_fptrauth_init_fini_address_discrimination,
                     options::OPT_fno_ptrauth_init_fini_address_discrimination);
+  Args.addOptInFlag(CmdArgs, options::OPT_faarch64_jump_table_hardening,
+                    options::OPT_fno_aarch64_jump_table_hardening);
 }
 
 void Clang::AddLoongArchTargetArgs(const ArgList &Args,
@@ -3899,7 +3902,8 @@ static void RenderSCPOptions(const ToolChain &TC, const ArgList &Args,
     return;
 
   if (!EffectiveTriple.isX86() && !EffectiveTriple.isSystemZ() &&
-      !EffectiveTriple.isPPC64() && !EffectiveTriple.isAArch64())
+      !EffectiveTriple.isPPC64() && !EffectiveTriple.isAArch64() &&
+      !EffectiveTriple.isRISCV())
     return;
 
   Args.addOptInFlag(CmdArgs, options::OPT_fstack_clash_protection,
@@ -4382,7 +4386,7 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
   if (Args.hasArg(options::OPT_modules_reduced_bmi) &&
       (Input.getType() == driver::types::TY_CXXModule ||
        Input.getType() == driver::types::TY_PP_CXXModule)) {
-    CmdArgs.push_back("-fexperimental-modules-reduced-bmi");
+    CmdArgs.push_back("-fmodules-reduced-bmi");
 
     if (Args.hasArg(options::OPT_fmodule_output_EQ))
       Args.AddLastArg(CmdArgs, options::OPT_fmodule_output_EQ);
@@ -4392,7 +4396,7 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
           getCXX20NamedModuleOutputPath(Args, Input.getBaseInput())));
   }
 
-  // Noop if we see '-fexperimental-modules-reduced-bmi' with other translation
+  // Noop if we see '-fmodules-reduced-bmi' with other translation
   // units than module units. This is more user friendly to allow end uers to
   // enable this feature without asking for help from build systems.
   Args.ClaimAllArgs(options::OPT_modules_reduced_bmi);
@@ -4815,15 +4819,15 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
   Args.ClaimAllArgs(options::OPT_g_flags_Group);
 
   // Column info is included by default for everything except SCE and
-  // CodeView. Clang doesn't track end columns, just starting columns, which,
-  // in theory, is fine for CodeView (and PDB).  In practice, however, the
-  // Microsoft debuggers don't handle missing end columns well, and the AIX
-  // debugger DBX also doesn't handle the columns well, so it's better not to
-  // include any column info.
+  // CodeView if not use sampling PGO. Clang doesn't track end columns, just
+  // starting columns, which, in theory, is fine for CodeView (and PDB).  In
+  // practice, however, the Microsoft debuggers don't handle missing end columns
+  // well, and the AIX debugger DBX also doesn't handle the columns well, so
+  // it's better not to include any column info.
   if (const Arg *A = Args.getLastArg(options::OPT_gcolumn_info))
     (void)checkDebugInfoOption(A, Args, D, TC);
   if (!Args.hasFlag(options::OPT_gcolumn_info, options::OPT_gno_column_info,
-                    !EmitCodeView &&
+                    !(EmitCodeView && !getLastProfileSampleUseArg(Args)) &&
                         (DebuggerTuning != llvm::DebuggerKind::SCE &&
                          DebuggerTuning != llvm::DebuggerKind::DBX)))
     CmdArgs.push_back("-gno-column-info");
@@ -6673,9 +6677,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasFlag(options::OPT_fstrict_aliasing, StrictAliasingAliasOption,
                     options::OPT_fno_strict_aliasing, !IsWindowsMSVC))
     CmdArgs.push_back("-relaxed-aliasing");
-  if (Args.hasFlag(options::OPT_fpointer_tbaa, options::OPT_fno_pointer_tbaa,
+  if (Args.hasFlag(options::OPT_fno_pointer_tbaa, options::OPT_fpointer_tbaa,
                    false))
-    CmdArgs.push_back("-pointer-tbaa");
+    CmdArgs.push_back("-no-pointer-tbaa");
   if (!Args.hasFlag(options::OPT_fstruct_path_tbaa,
                     options::OPT_fno_struct_path_tbaa, true))
     CmdArgs.push_back("-no-struct-path-tbaa");
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index bb34324a7a1c6..8ee95d60c2c32 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -19,6 +19,7 @@
 #include "Arch/SystemZ.h"
 #include "Arch/VE.h"
 #include "Arch/X86.h"
+#include "BareMetal.h"
 #include "HIPAMD.h"
 #include "Hexagon.h"
 #include "MSP430.h"
@@ -151,6 +152,9 @@ static bool useFramePointerForTargetByDefault(const llvm::opt::ArgList &Args,
     }
   }
 
+  if (arm::isARMEABIBareMetal(Triple))
+    return false;
+
   return true;
 }
 
@@ -1412,7 +1416,7 @@ void tools::linkSanitizerRuntimeDeps(const ToolChain &TC,
   // libresolv.a, even if exists, is an empty archive to satisfy POSIX -lresolv
   // requirement.
   if (TC.getTriple().isOSLinux() && !TC.getTriple().isAndroid() &&
-      !TC.getTriple().isMusl())
+      !TC.getTriple().isMusl() && TC.getSanitizerArgs(Args).needsMsanRt())
     CmdArgs.push_back("-lresolv");
 }
 
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index a57e1524a0b85..c98fdbd157bac 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -148,7 +148,6 @@ void Flang::addCodegenOptions(const ArgList &Args,
 
   Args.addAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
                             options::OPT_flang_deprecated_no_hlfir,
-                            options::OPT_flang_experimental_integer_overflow,
                             options::OPT_fno_ppc_native_vec_elem_order,
                             options::OPT_fppc_native_vec_elem_order});
 }
@@ -203,6 +202,26 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
   }
 }
 
+void Flang::AddLoongArch64TargetArgs(const ArgList &Args,
+                                     ArgStringList &CmdArgs) const {
+  const Driver &D = getToolChain().getDriver();
+  // Currently, flang only support `-mabi=lp64d` in LoongArch64.
+  if (const Arg *A = Args.getLastArg(options::OPT_mabi_EQ)) {
+    StringRef V = A->getValue();
+    if (V != "lp64d") {
+      D.Diag(diag::err_drv_argument_not_allowed_with) << "-mabi" << V;
+    }
+  }
+
+  if (const Arg *A = Args.getLastArg(options::OPT_mannotate_tablejump,
+                                     options::OPT_mno_annotate_tablejump)) {
+    if (A->getOption().matches(options::OPT_mannotate_tablejump)) {
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back("-loongarch-annotate-tablejump");
+    }
+  }
+}
+
 void Flang::AddPPCTargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
   const Driver &D = getToolChain().getDriver();
@@ -416,6 +435,7 @@ void Flang::addTargetOptions(const ArgList &Args,
     break;
   case llvm::Triple::loongarch64:
     getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
+    AddLoongArch64TargetArgs(Args, CmdArgs);
     break;
   }
 
@@ -764,6 +784,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
   addFortranDialectOptions(Args, CmdArgs);
 
+  // 'flang -E' always produces output that is suitable for use as fixed form
+  // Fortran. However it is only valid free form source if the original is also
+  // free form.
+  if (InputType == types::TY_PP_Fortran &&
+      !Args.getLastArg(options::OPT_ffixed_form, options::OPT_ffree_form))
+    CmdArgs.push_back("-ffixed-form");
+
   handleColorDiagnosticsArgs(D, Args, CmdArgs);
 
   // LTO mode is parsed by the Clang driver library.
diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h
index 4d7d0b8cd9ea5..7c24a623af393 100644
--- a/clang/lib/Driver/ToolChains/Flang.h
+++ b/clang/lib/Driver/ToolChains/Flang.h
@@ -70,6 +70,13 @@ class LLVM_LIBRARY_VISIBILITY Flang : public Tool {
   void AddAMDGPUTargetArgs(const llvm::opt::ArgList &Args,
                            llvm::opt::ArgStringList &CmdArgs) const;
 
+  /// Add specific options for LoongArch64 target.
+  ///
+  /// \param [in] Args The list of input driver arguments
+  /// \param [out] CmdArgs The list of output command arguments
+  void AddLoongArch64TargetArgs(const llvm::opt::ArgList &Args,
+                                llvm::opt::ArgStringList &CmdArgs) const;
+
   /// Add specific options for RISC-V target.
   ///
   /// \param [in] Args The list of input driver arguments
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 72accbff4a3bf..c2badc80a7b45 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -156,6 +156,12 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   addLinkerCompressDebugSectionsOption(ToolChain, Args, CmdArgs);
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
+  // Sample these options first so they are claimed even under -nostdlib et al.
+  bool NoLibc = Args.hasArg(options::OPT_nolibc);
+  bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
+                             !Args.hasArg(options::OPT_static);
+  bool Pthreads = Args.hasArg(options::OPT_pthread, options::OPT_pthreads);
+  bool SplitStack = Args.hasArg(options::OPT_fsplit_stack);
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs,
                    options::OPT_r)) {
     if (Args.hasArg(options::OPT_static))
@@ -163,8 +169,6 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
     if (D.CCCIsCXX()) {
       if (ToolChain.ShouldLinkCXXStdlib(Args)) {
-        bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
-                                   !Args.hasArg(options::OPT_static);
         CmdArgs.push_back("--push-state");
         CmdArgs.push_back("--as-needed");
         if (OnlyLibstdcxxStatic)
@@ -188,14 +192,13 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
     AddRunTimeLibs(ToolChain, D, CmdArgs, Args);
 
-    if (Args.hasArg(options::OPT_pthread) ||
-        Args.hasArg(options::OPT_pthreads))
+    if (Pthreads)
       CmdArgs.push_back("-lpthread");
 
-    if (Args.hasArg(options::OPT_fsplit_stack))
+    if (SplitStack)
       CmdArgs.push_back("--wrap=pthread_create");
 
-    if (!Args.hasArg(options::OPT_nolibc))
+    if (!NoLibc)
       CmdArgs.push_back("-lc");
   }
 
@@ -229,7 +232,7 @@ void fuchsia::StaticLibTool::ConstructJob(Compilation &C, const JobAction &JA,
 
   for (const auto &II : Inputs) {
     if (II.isFilename()) {
-       CmdArgs.push_back(II.getFilename());
+      CmdArgs.push_back(II.getFilename());
     }
   }
 
@@ -343,16 +346,14 @@ std::string Fuchsia::ComputeEffectiveClangTriple(const ArgList &Args,
   return Triple.str();
 }
 
-Tool *Fuchsia::buildLinker() const {
-  return new tools::fuchsia::Linker(*this);
-}
+Tool *Fuchsia::buildLinker() const { return new tools::fuchsia::Linker(*this); }
 
 Tool *Fuchsia::buildStaticLibTool() const {
   return new tools::fuchsia::StaticLibTool(*this);
 }
 
-ToolChain::RuntimeLibType Fuchsia::GetRuntimeLibType(
-    const ArgList &Args) const {
+ToolChain::RuntimeLibType
+Fuchsia::GetRuntimeLibType(const ArgList &Args) const {
   if (Arg *A = Args.getLastArg(clang::driver::options::OPT_rtlib_EQ)) {
     StringRef Value = A->getValue();
     if (Value != "compiler-rt")
@@ -363,13 +364,12 @@ ToolChain::RuntimeLibType Fuchsia::GetRuntimeLibType(
   return ToolChain::RLT_CompilerRT;
 }
 
-ToolChain::CXXStdlibType
-Fuchsia::GetCXXStdlibType(const ArgList &Args) const {
+ToolChain::CXXStdlibType Fuchsia::GetCXXStdlibType(const ArgList &Args) const {
   if (Arg *A = Args.getLastArg(options::OPT_stdlib_EQ)) {
     StringRef Value = A->getValue();
     if (Value != "libc++")
       getDriver().Diag(diag::err_drv_invalid_stdlib_name)
-        << A->getAsString(Args);
+          << A->getAsString(Args);
   }
 
   return ToolChain::CST_Libcxx;
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 383dc8387e75e..76cedf312d68a 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -294,9 +294,10 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
   bool IncStartFiles = !Args.hasArg(options::OPT_nostartfiles);
   bool IncDefLibs = !Args.hasArg(options::OPT_nodefaultlibs);
   bool UseG0 = false;
-  const char *Exec = Args.MakeArgString(HTC.GetLinkerPath());
-  bool UseLLD = (llvm::sys::path::filename(Exec).equals_insensitive("ld.lld") ||
-                 llvm::sys::path::stem(Exec).equals_insensitive("ld.lld"));
+  bool UseLLD = false;
+  const char *Exec = Args.MakeArgString(HTC.GetLinkerPath(&UseLLD));
+  UseLLD = UseLLD || llvm::sys::path::filename(Exec).ends_with("ld.lld") ||
+           llvm::sys::path::stem(Exec).ends_with("ld.lld");
   bool UseShared = IsShared && !IsStatic;
   StringRef CpuVer = toolchains::HexagonToolChain::GetTargetCPUVersion(Args);
 
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index dd9a584c5fbf4..a4bd9c802a744 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -261,6 +261,24 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
       ExtraOpts.push_back("-z");
       ExtraOpts.push_back("max-page-size=16384");
     }
+    if (Triple.isAndroidVersionLT(29)) {
+      // https://github.com/android/ndk/issues/1196
+      // The unwinder used by the crash handler on versions of Android prior to
+      // API 29 did not correctly handle binaries built with rosegment, which is
+      // enabled by default for LLD. Android only supports LLD, so it's not an
+      // issue that this flag is not accepted by other linkers.
+      ExtraOpts.push_back("--no-rosegment");
+    }
+    if (!Triple.isAndroidVersionLT(28)) {
+      // Android supports relr packing starting with API 28 and had its own
+      // flavor (--pack-dyn-relocs=android) starting in API 23.
+      // TODO: It's possible to use both with --pack-dyn-relocs=android+relr,
+      // but we need to gather some data on the impact of that form before we
+      // can know if it's a good default.
+      // On the other hand, relr should always be an improvement.
+      ExtraOpts.push_back("--use-android-relr-tags");
+      ExtraOpts.push_back("--pack-dyn-relocs=relr");
+    }
   }
 
   if (GCCInstallation.getParentLibPath().contains("opt/rh/"))
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index de5c38ebc3abb..f8c213334a2b4 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -51,6 +51,8 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
     case llvm::Triple::mips64:
     case llvm::Triple::mips64el:
     case llvm::Triple::systemz:
+    case llvm::Triple::riscv32:
+    case llvm::Triple::riscv64:
       break;
     default:
       D.Diag(diag::err_drv_unsupported_opt_for_target)
diff --git a/clang/lib/Format/CMakeLists.txt b/clang/lib/Format/CMakeLists.txt
index ff987cc608fb8..9f4939824fdb8 100644
--- a/clang/lib/Format/CMakeLists.txt
+++ b/clang/lib/Format/CMakeLists.txt
@@ -41,15 +41,46 @@ file(GLOB_RECURSE files
 
 set(check_format_depends)
 set(i 0)
-foreach (file IN LISTS files)
-  add_custom_command(OUTPUT clang-format-check-format${i}
+foreach(file IN LISTS files)
+  add_custom_command(OUTPUT check_format_depend_${i}
     COMMAND clang-format ${file} | diff -u ${file} -
     VERBATIM
-    COMMENT "Checking format of ${file}..."
+    COMMENT "Checking format of ${file}"
     )
-  list(APPEND check_format_depends clang-format-check-format${i})
-
+  list(APPEND check_format_depends check_format_depend_${i})
   math(EXPR i ${i}+1)
-endforeach ()
-
+endforeach()
 add_custom_target(clang-format-check-format DEPENDS ${check_format_depends})
+
+set(docs_tools_dir ${CLANG_SOURCE_DIR}/docs/tools)
+
+set(format_style_depend ${CMAKE_CURRENT_BINARY_DIR}/format_style_depend)
+set(dump_style dump_format_style.py)
+set(style_options_rst ${CLANG_SOURCE_DIR}/docs/ClangFormatStyleOptions.rst)
+add_custom_command(OUTPUT ${format_style_depend}
+  COMMAND ${Python3_EXECUTABLE} ${dump_style} && touch ${format_style_depend}
+  WORKING_DIRECTORY ${docs_tools_dir}
+  VERBATIM
+  COMMENT "Updating ${style_options_rst}"
+  DEPENDS ${CLANG_SOURCE_DIR}/include/clang/Format/Format.h
+          ${CLANG_SOURCE_DIR}/include/clang/Tooling/Inclusions/IncludeStyle.h
+          ${style_options_rst}
+          ${docs_tools_dir}/plurals.txt
+          ${docs_tools_dir}/${dump_style}
+  )
+add_custom_target(clang-format-style DEPENDS ${format_style_depend})
+
+set(format_help_depend ${CMAKE_CURRENT_BINARY_DIR}/format_help_depend)
+set(dump_help dump_format_help.py)
+set(clang_format_rst ${CLANG_SOURCE_DIR}/docs/ClangFormat.rst)
+add_custom_command(OUTPUT ${format_help_depend}
+  COMMAND ${Python3_EXECUTABLE} ${dump_help} -d ${CMAKE_BINARY_DIR}/bin &&
+          touch ${format_help_depend}
+  WORKING_DIRECTORY ${docs_tools_dir}
+  VERBATIM
+  COMMENT "Updating ${clang_format_rst}"
+  DEPENDS clang-format
+          ${clang_format_rst}
+          ${docs_tools_dir}/${dump_help}
+  )
+add_custom_target(clang-format-help DEPENDS ${format_help_depend})
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index ee52972ce66f4..dcaac4b0d42cc 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3950,6 +3950,7 @@ static FormatStyle::LanguageKind getLanguageByFileName(StringRef FileName) {
     return FormatStyle::LK_Java;
   if (FileName.ends_with_insensitive(".js") ||
       FileName.ends_with_insensitive(".mjs") ||
+      FileName.ends_with_insensitive(".cjs") ||
       FileName.ends_with_insensitive(".ts")) {
     return FormatStyle::LK_JavaScript; // (module) JavaScript or TypeScript.
   }
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index bc5239209f3aa..05c86db55641f 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -3901,6 +3901,11 @@ bool TokenAnnotator::mustBreakForReturnType(const AnnotatedLine &Line) const {
 }
 
 void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
+  if (Line.Computed)
+    return;
+
+  Line.Computed = true;
+
   for (AnnotatedLine *ChildLine : Line.Children)
     calculateFormattingInformation(*ChildLine);
 
@@ -6105,6 +6110,35 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
       return false;
   }
 
+  // We can break before an r_brace if there was a break after the matching
+  // l_brace, which is tracked by BreakBeforeClosingBrace, or if we are in a
+  // block-indented initialization list.
+  if (Right.is(tok::r_brace)) {
+    return Right.MatchingParen && (Right.MatchingParen->is(BK_Block) ||
+                                   (Right.isBlockIndentedInitRBrace(Style)));
+  }
+
+  // We only break before r_paren if we're in a block indented context.
+  if (Right.is(tok::r_paren)) {
+    if (Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent ||
+        !Right.MatchingParen) {
+      return false;
+    }
+    auto Next = Right.Next;
+    if (Next && Next->is(tok::r_paren))
+      Next = Next->Next;
+    if (Next && Next->is(tok::l_paren))
+      return false;
+    const FormatToken *Previous = Right.MatchingParen->Previous;
+    return !(Previous && (Previous->is(tok::kw_for) || Previous->isIf()));
+  }
+
+  if (Left.isOneOf(tok::r_paren, TT_TrailingAnnotation) &&
+      Right.is(TT_TrailingAnnotation) &&
+      Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent) {
+    return false;
+  }
+
   if (Left.is(tok::at))
     return false;
   if (Left.Tok.getObjCKeywordID() == tok::objc_interface)
@@ -6260,34 +6294,6 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
     return false;
   }
 
-  // We only break before r_brace if there was a corresponding break before
-  // the l_brace, which is tracked by BreakBeforeClosingBrace.
-  if (Right.is(tok::r_brace)) {
-    return Right.MatchingParen && (Right.MatchingParen->is(BK_Block) ||
-                                   (Right.isBlockIndentedInitRBrace(Style)));
-  }
-
-  // We only break before r_paren if we're in a block indented context.
-  if (Right.is(tok::r_paren)) {
-    if (Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent ||
-        !Right.MatchingParen) {
-      return false;
-    }
-    auto Next = Right.Next;
-    if (Next && Next->is(tok::r_paren))
-      Next = Next->Next;
-    if (Next && Next->is(tok::l_paren))
-      return false;
-    const FormatToken *Previous = Right.MatchingParen->Previous;
-    return !(Previous && (Previous->is(tok::kw_for) || Previous->isIf()));
-  }
-
-  if (Left.isOneOf(tok::r_paren, TT_TrailingAnnotation) &&
-      Right.is(TT_TrailingAnnotation) &&
-      Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent) {
-    return false;
-  }
-
   // Allow breaking after a trailing annotation, e.g. after a method
   // declaration.
   if (Left.is(TT_TrailingAnnotation)) {
diff --git a/clang/lib/Format/TokenAnnotator.h b/clang/lib/Format/TokenAnnotator.h
index 5a02030e5ba7f..9117ca3f9fb7b 100644
--- a/clang/lib/Format/TokenAnnotator.h
+++ b/clang/lib/Format/TokenAnnotator.h
@@ -182,6 +182,9 @@ class AnnotatedLine {
   /// \c True if this line contains a macro call for which an expansion exists.
   bool ContainsMacroCall = false;
 
+  /// \c True if calculateFormattingInformation() has been called on this line.
+  bool Computed = false;
+
   /// \c True if this line should be formatted, i.e. intersects directly or
   /// indirectly with one of the input ranges.
   bool Affected;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index c182aaf0876d1..de7e261b21d30 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -512,7 +512,7 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
           break;
         do {
           NextTok = Tokens->getNextToken();
-        } while (NextTok->NewlinesBefore == 0 && NextTok->isNot(tok::eof));
+        } while (!NextTok->HasUnescapedNewline && NextTok->isNot(tok::eof));
 
         while (NextTok->is(tok::comment))
           NextTok = Tokens->getNextToken();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 327e83b4ca37b..f1c0fcc767bcf 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1518,6 +1518,7 @@ void CompilerInvocation::setDefaultPointerAuthOptions(
   Opts.ReturnAddresses = LangOpts.PointerAuthReturns;
   Opts.AuthTraps = LangOpts.PointerAuthAuthTraps;
   Opts.IndirectGotos = LangOpts.PointerAuthIndirectGotos;
+  Opts.AArch64JumpTableHardening = LangOpts.AArch64JumpTableHardening;
 }
 
 static void parsePointerAuthOptions(PointerAuthOptions &Opts,
@@ -1525,7 +1526,8 @@ static void parsePointerAuthOptions(PointerAuthOptions &Opts,
                                     const llvm::Triple &Triple,
                                     DiagnosticsEngine &Diags) {
   if (!LangOpts.PointerAuthCalls && !LangOpts.PointerAuthReturns &&
-      !LangOpts.PointerAuthAuthTraps && !LangOpts.PointerAuthIndirectGotos)
+      !LangOpts.PointerAuthAuthTraps && !LangOpts.PointerAuthIndirectGotos &&
+      !LangOpts.AArch64JumpTableHardening)
     return;
 
   CompilerInvocation::setDefaultPointerAuthOptions(Opts, LangOpts, Triple);
@@ -3470,6 +3472,8 @@ static void GeneratePointerAuthArgs(const LangOptions &Opts,
     GenerateArg(Consumer, OPT_fptrauth_init_fini_address_discrimination);
   if (Opts.PointerAuthELFGOT)
     GenerateArg(Consumer, OPT_fptrauth_elf_got);
+  if (Opts.AArch64JumpTableHardening)
+    GenerateArg(Consumer, OPT_faarch64_jump_table_hardening);
 }
 
 static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args,
@@ -3491,6 +3495,8 @@ static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args,
   Opts.PointerAuthInitFiniAddressDiscrimination =
       Args.hasArg(OPT_fptrauth_init_fini_address_discrimination);
   Opts.PointerAuthELFGOT = Args.hasArg(OPT_fptrauth_elf_got);
+  Opts.AArch64JumpTableHardening =
+      Args.hasArg(OPT_faarch64_jump_table_hardening);
 }
 
 /// Check if input file kind and language standard are compatible.
diff --git a/clang/lib/Headers/avx10_2copyintrin.h b/clang/lib/Headers/avx10_2copyintrin.h
index 7fc31190781d9..76b8f8ced540d 100644
--- a/clang/lib/Headers/avx10_2copyintrin.h
+++ b/clang/lib/Headers/avx10_2copyintrin.h
@@ -19,11 +19,43 @@
   __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"),    \
                  __min_vector_width__(128)))
 
+/// Constructs a 128-bit integer vector, setting the lower 32 bits to the
+///    lower 32 bits of the parameter \a __A; the upper bits are zeoroed.
+///
+/// \code{.operation}
+/// result[31:0] := __A[31:0]
+/// result[MAX:32] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> VMOVD </c> instruction.
+///
+/// \param __A
+///    A 128-bit integer vector.
+/// \returns A 128-bit integer vector. The lower 32 bits are copied from the
+///    parameter \a __A; the upper bits are zeroed.
 static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi32(__m128i __A) {
   return (__m128i)__builtin_shufflevector(
       (__v4si)__A, (__v4si)_mm_setzero_si128(), 0, 4, 4, 4);
 }
 
+/// Constructs a 128-bit integer vector, setting the lower 16 bits to the
+///    lower 16 bits of the parameter \a __A; the upper bits are zeoroed.
+///
+/// \code{.operation}
+/// result[15:0] := __A[15:0]
+/// result[MAX:16] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> VMOVW </c> instruction.
+///
+/// \param __A
+///    A 128-bit integer vector.
+/// \returns A 128-bit integer vector. The lower 16 bits are copied from the
+///    parameter \a __A; the upper bits are zeroed.
 static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi16(__m128i __A) {
   return (__m128i)__builtin_shufflevector(
       (__v8hi)__A, (__v8hi)_mm_setzero_si128(), 0, 8, 8, 8, 8, 8, 8, 8);
diff --git a/clang/lib/Headers/avx512vpopcntdqintrin.h b/clang/lib/Headers/avx512vpopcntdqintrin.h
index 7a77cbc2421b3..e24c2c5e1bcd9 100644
--- a/clang/lib/Headers/avx512vpopcntdqintrin.h
+++ b/clang/lib/Headers/avx512vpopcntdqintrin.h
@@ -21,7 +21,14 @@
                  __target__("avx512vpopcntdq,evex512"),                        \
                  __min_vector_width__(512)))
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi64(__m512i __A) {
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_popcnt_epi64(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v8du)__A);
 }
 
@@ -36,7 +43,8 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
   return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi32(__m512i __A) {
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_popcnt_epi32(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v16su)__A);
 }
 
diff --git a/clang/lib/Headers/avx512vpopcntdqvlintrin.h b/clang/lib/Headers/avx512vpopcntdqvlintrin.h
index b049a57a4e2a5..b6c819b0cb85e 100644
--- a/clang/lib/Headers/avx512vpopcntdqvlintrin.h
+++ b/clang/lib/Headers/avx512vpopcntdqvlintrin.h
@@ -25,7 +25,15 @@
                  __target__("avx512vpopcntdq,avx512vl,no-evex512"),            \
                  __min_vector_width__(256)))
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_popcnt_epi64(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v2du)__A);
 }
@@ -41,7 +49,7 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
   return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_popcnt_epi32(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v4su)__A);
 }
@@ -57,7 +65,7 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
   return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_popcnt_epi64(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v4du)__A);
 }
@@ -73,7 +81,7 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
   return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_popcnt_epi32(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v8su)__A);
 }
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index a3e0b5c65a6f5..1126e13600f8a 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -2481,5 +2481,17 @@ float3 radians(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_radians)
 float4 radians(float4);
 
+//===----------------------------------------------------------------------===//
+// GroupMemoryBarrierWithGroupSync builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn void GroupMemoryBarrierWithGroupSync(void)
+/// \brief Blocks execution of all threads in a group until all group shared
+/// accesses have been completed and all threads in the group have reached this
+/// call.
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_group_memory_barrier_with_group_sync)
+void GroupMemoryBarrierWithGroupSync(void);
+
 } // namespace hlsl
 #endif //_HLSL_HLSL_INTRINSICS_H_
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index 6308c865ca913..e8a01d1888026 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -330,33 +330,33 @@ static __inline__ void __DEFAULT_FN_ATTRS __halt(void) {
   __asm__ volatile("hlt");
 }
 
-static inline unsigned char __inbyte(unsigned short port) {
+static __inline__ unsigned char __inbyte(unsigned short port) {
   unsigned char ret;
   __asm__ __volatile__("inb %w1, %b0" : "=a"(ret) : "Nd"(port));
   return ret;
 }
 
-static inline unsigned short __inword(unsigned short port) {
+static __inline__ unsigned short __inword(unsigned short port) {
   unsigned short ret;
   __asm__ __volatile__("inw %w1, %w0" : "=a"(ret) : "Nd"(port));
   return ret;
 }
 
-static inline unsigned long __indword(unsigned short port) {
+static __inline__ unsigned long __indword(unsigned short port) {
   unsigned long ret;
   __asm__ __volatile__("inl %w1, %k0" : "=a"(ret) : "Nd"(port));
   return ret;
 }
 
-static inline void __outbyte(unsigned short port, unsigned char data) {
+static __inline__ void __outbyte(unsigned short port, unsigned char data) {
   __asm__ __volatile__("outb %b0, %w1" : : "a"(data), "Nd"(port));
 }
 
-static inline void __outword(unsigned short port, unsigned short data) {
+static __inline__ void __outword(unsigned short port, unsigned short data) {
   __asm__ __volatile__("outw %w0, %w1" : : "a"(data), "Nd"(port));
 }
 
-static inline void __outdword(unsigned short port, unsigned long data) {
+static __inline__ void __outdword(unsigned short port, unsigned long data) {
   __asm__ __volatile__("outl %k0, %w1" : : "a"(data), "Nd"(port));
 }
 #endif
@@ -396,6 +396,16 @@ unsigned short __readx18word(unsigned long offset);
 unsigned long __readx18dword(unsigned long offset);
 unsigned __int64 __readx18qword(unsigned long offset);
 
+void __addx18byte(unsigned long offset, unsigned char data);
+void __addx18word(unsigned long offset, unsigned short data);
+void __addx18dword(unsigned long offset, unsigned long data);
+void __addx18qword(unsigned long offset, unsigned __int64 data);
+
+void __incx18byte(unsigned long offset);
+void __incx18word(unsigned long offset);
+void __incx18dword(unsigned long offset);
+void __incx18qword(unsigned long offset);
+
 double _CopyDoubleFromInt64(__int64);
 float _CopyFloatFromInt32(__int32);
 __int32 _CopyInt32FromFloat(float);
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 6b01f3808652a..2bca9fc877e9c 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
 __int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
 __int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
 __int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
+void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
 
 /*----------------------------------------------------------------------------*\
 |* Interlocked Compare Exchange
@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
 __int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
                                           __int64 _Exchange,
                                           __int64 _Comparand);
+void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
+                                             void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
+                                            void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
+                                             void *_Exchange, void *_Comparand);
 #endif
 
 #ifdef __cplusplus
diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt
index df7ea82e0dada..bf70cdfbee01e 100644
--- a/clang/lib/Interpreter/CMakeLists.txt
+++ b/clang/lib/Interpreter/CMakeLists.txt
@@ -16,6 +16,7 @@ set(LLVM_LINK_COMPONENTS
 if (EMSCRIPTEN AND "lld" IN_LIST LLVM_ENABLE_PROJECTS)
   set(WASM_SRC Wasm.cpp)
   set(WASM_LINK lldWasm)
+  set(COMMON_LINK lldCommon)
 endif()
 
 add_clang_library(clangInterpreter
@@ -47,6 +48,7 @@ add_clang_library(clangInterpreter
   clangSema
   clangSerialization
   ${WASM_LINK}
+  ${COMMON_LINK}
   )
 
 if ((MINGW OR CYGWIN) AND BUILD_SHARED_LIBS)
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index 5dc67f6375098..6cd60a9bcf4f3 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -200,7 +200,7 @@ IncrementalCompilerBuilder::CreateCpp() {
 #ifdef __EMSCRIPTEN__
   Argv.push_back("-target");
   Argv.push_back("wasm32-unknown-emscripten");
-  Argv.push_back("-shared");
+  Argv.push_back("-fvisibility=default");
 #endif
   Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end());
 
@@ -719,7 +719,9 @@ llvm::Error Interpreter::LoadDynamicLibrary(const char *name) {
 
   if (auto DLSG = llvm::orc::DynamicLibrarySearchGenerator::Load(
           name, DL.getGlobalPrefix()))
-    EE->getMainJITDylib().addGenerator(std::move(*DLSG));
+    // FIXME: Eventually we should put each library in its own JITDylib and
+    //        turn off process symbols by default.
+    EE->getProcessSymbolsJITDylib()->addGenerator(std::move(*DLSG));
   else
     return DLSG.takeError();
 
diff --git a/clang/lib/Interpreter/Wasm.cpp b/clang/lib/Interpreter/Wasm.cpp
index 79efbaa03982d..aa10b160ccf84 100644
--- a/clang/lib/Interpreter/Wasm.cpp
+++ b/clang/lib/Interpreter/Wasm.cpp
@@ -23,6 +23,31 @@
 #include <string>
 
 namespace lld {
+enum Flavor {
+  Invalid,
+  Gnu,     // -flavor gnu
+  MinGW,   // -flavor gnu MinGW
+  WinLink, // -flavor link
+  Darwin,  // -flavor darwin
+  Wasm,    // -flavor wasm
+};
+
+using Driver = bool (*)(llvm::ArrayRef<const char *>, llvm::raw_ostream &,
+                        llvm::raw_ostream &, bool, bool);
+
+struct DriverDef {
+  Flavor f;
+  Driver d;
+};
+
+struct Result {
+  int retCode;
+  bool canRunAgain;
+};
+
+Result lldMain(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
+               llvm::raw_ostream &stderrOS, llvm::ArrayRef<DriverDef> drivers);
+
 namespace wasm {
 bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
           llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
@@ -51,13 +76,14 @@ llvm::Error WasmIncrementalExecutor::addModule(PartialTranslationUnit &PTU) {
   llvm::TargetMachine *TargetMachine = Target->createTargetMachine(
       PTU.TheModule->getTargetTriple(), "", "", TO, llvm::Reloc::Model::PIC_);
   PTU.TheModule->setDataLayout(TargetMachine->createDataLayout());
-  std::string OutputFileName = PTU.TheModule->getName().str() + ".wasm";
+  std::string ObjectFileName = PTU.TheModule->getName().str() + ".o";
+  std::string BinaryFileName = PTU.TheModule->getName().str() + ".wasm";
 
   std::error_code Error;
-  llvm::raw_fd_ostream OutputFile(llvm::StringRef(OutputFileName), Error);
+  llvm::raw_fd_ostream ObjectFileOutput(llvm::StringRef(ObjectFileName), Error);
 
   llvm::legacy::PassManager PM;
-  if (TargetMachine->addPassesToEmitFile(PM, OutputFile, nullptr,
+  if (TargetMachine->addPassesToEmitFile(PM, ObjectFileOutput, nullptr,
                                          llvm::CodeGenFileType::ObjectFile)) {
     return llvm::make_error<llvm::StringError>(
         "Wasm backend cannot produce object.", llvm::inconvertibleErrorCode());
@@ -69,27 +95,30 @@ llvm::Error WasmIncrementalExecutor::addModule(PartialTranslationUnit &PTU) {
                                                llvm::inconvertibleErrorCode());
   }
 
-  OutputFile.close();
+  ObjectFileOutput.close();
 
   std::vector<const char *> LinkerArgs = {"wasm-ld",
                                           "-shared",
                                           "--import-memory",
-                                          "--no-entry",
-                                          "--export-all",
                                           "--experimental-pic",
                                           "--stack-first",
                                           "--allow-undefined",
-                                          OutputFileName.c_str(),
+                                          ObjectFileName.c_str(),
                                           "-o",
-                                          OutputFileName.c_str()};
-  int Result =
-      lld::wasm::link(LinkerArgs, llvm::outs(), llvm::errs(), false, false);
-  if (!Result)
+                                          BinaryFileName.c_str()};
+
+  const lld::DriverDef WasmDriver = {lld::Flavor::Wasm, &lld::wasm::link};
+  std::vector<lld::DriverDef> WasmDriverArgs;
+  WasmDriverArgs.push_back(WasmDriver);
+  lld::Result Result =
+      lld::lldMain(LinkerArgs, llvm::outs(), llvm::errs(), WasmDriverArgs);
+
+  if (Result.retCode)
     return llvm::make_error<llvm::StringError>(
         "Failed to link incremental module", llvm::inconvertibleErrorCode());
 
   void *LoadedLibModule =
-      dlopen(OutputFileName.c_str(), RTLD_NOW | RTLD_GLOBAL);
+      dlopen(BinaryFileName.c_str(), RTLD_NOW | RTLD_GLOBAL);
   if (LoadedLibModule == nullptr) {
     llvm::errs() << dlerror() << '\n';
     return llvm::make_error<llvm::StringError>(
@@ -109,7 +138,7 @@ llvm::Error WasmIncrementalExecutor::runCtors() const {
   return llvm::Error::success();
 }
 
-llvm::Error WasmIncrementalExecutor::cleanUp() const {
+llvm::Error WasmIncrementalExecutor::cleanUp() {
   // Can't call cleanUp through IncrementalExecutor as it
   // tries to deinitialize JIT which hasn't been initialized
   return llvm::Error::success();
@@ -117,4 +146,4 @@ llvm::Error WasmIncrementalExecutor::cleanUp() const {
 
 WasmIncrementalExecutor::~WasmIncrementalExecutor() = default;
 
-} // namespace clang
+} // namespace clang
\ No newline at end of file
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index e58c8bc72ae5b..72364500a48f9 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -527,7 +527,7 @@ bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
 
   const char *StrData = Buffer.data()+LocInfo.second;
 
-  if (!IgnoreWhiteSpace && isWhitespace(StrData[0]))
+  if (!IgnoreWhiteSpace && isWhitespace(SkipEscapedNewLines(StrData)[0]))
     return true;
 
   // Create a lexer starting at the beginning of this token.
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 2ec66fe95708e..6c01af55ef3c4 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -722,8 +722,7 @@ void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
   ExprResult Init = ParseCXXMemberInitializer(MI.Field, /*IsFunction=*/false,
                                               EqualLoc);
 
-  Actions.ActOnFinishCXXInClassMemberInitializer(MI.Field, EqualLoc,
-                                                 Init.get());
+  Actions.ActOnFinishCXXInClassMemberInitializer(MI.Field, EqualLoc, Init);
 
   // The next token should be our artificial terminating EOF token.
   if (Tok.isNot(tok::eof)) {
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8ae0678a4be7b..98c499bd5dd35 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -8120,6 +8120,14 @@ void Parser::ParseParameterDeclarationClause(
     }
 
     if (TryConsumeToken(tok::ellipsis, EllipsisLoc)) {
+      if (getLangOpts().CPlusPlus26) {
+        // C++26 [dcl.dcl.fct]p3:
+        //   A parameter-declaration-clause of the form
+        //   parameter-list '...' is deprecated.
+        Diag(EllipsisLoc, diag::warn_deprecated_missing_comma_before_ellipsis)
+            << FixItHint::CreateInsertion(EllipsisLoc, ", ");
+      }
+
       if (!getLangOpts().CPlusPlus) {
         // We have ellipsis without a preceding ',', which is ill-formed
         // in C. Complain and provide the fix.
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index fae46cdd19afa..7c86a844e1ba9 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1199,7 +1199,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         // If the token is not annotated, then it might be an expression pack
         // indexing
         if (!TryAnnotateTypeOrScopeToken() &&
-            Tok.is(tok::annot_pack_indexing_type))
+            Tok.isOneOf(tok::annot_pack_indexing_type, tok::annot_cxxscope))
           return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
                                      isVectorLiteral, NotPrimaryExpression);
       }
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index 4de342b63ed80..443bf2b9ec626 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -280,6 +280,7 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
   case ParsedAttr::UnknownAttribute:
     Diag(Loc, diag::err_unknown_hlsl_semantic) << II;
     return;
+  case ParsedAttr::AT_HLSLSV_GroupThreadID:
   case ParsedAttr::AT_HLSLSV_GroupID:
   case ParsedAttr::AT_HLSLSV_GroupIndex:
   case ParsedAttr::AT_HLSLSV_DispatchThreadID:
diff --git a/clang/lib/Sema/CheckExprLifetime.cpp b/clang/lib/Sema/CheckExprLifetime.cpp
index 6cdd4dc629e50..868081292bc32 100644
--- a/clang/lib/Sema/CheckExprLifetime.cpp
+++ b/clang/lib/Sema/CheckExprLifetime.cpp
@@ -9,6 +9,7 @@
 #include "CheckExprLifetime.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/Type.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Sema.h"
@@ -203,6 +204,7 @@ struct IndirectLocalPathEntry {
     GslPointerInit,
     GslPointerAssignment,
     DefaultArg,
+    ParenAggInit,
   } Kind;
   Expr *E;
   union {
@@ -253,9 +255,35 @@ static void visitLocalsRetainedByReferenceBinding(IndirectLocalPath &Path,
                                                   LocalVisitor Visit);
 
 template <typename T> static bool isRecordWithAttr(QualType Type) {
-  if (auto *RD = Type->getAsCXXRecordDecl())
-    return RD->hasAttr<T>();
-  return false;
+  auto *RD = Type->getAsCXXRecordDecl();
+  if (!RD)
+    return false;
+  // Generally, if a primary template class declaration is annotated with an
+  // attribute, all its specializations generated from template instantiations
+  // should inherit the attribute.
+  //
+  // However, since lifetime analysis occurs during parsing, we may encounter
+  // cases where a full definition of the specialization is not required. In
+  // such cases, the specialization declaration remains incomplete and lacks the
+  // attribute. Therefore, we fall back to checking the primary template class.
+  //
+  // Note: it is possible for a specialization declaration to have an attribute
+  // even if the primary template does not.
+  //
+  // FIXME: What if the primary template and explicit specialization
+  // declarations have conflicting attributes? We should consider diagnosing
+  // this scenario.
+  bool Result = RD->hasAttr<T>();
+
+  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
+    Result |= CTSD->getSpecializedTemplate()->getTemplatedDecl()->hasAttr<T>();
+
+  return Result;
+}
+
+bool isPointerLikeType(QualType QT) {
+  return isRecordWithAttr<PointerAttr>(QT) || QT->isPointerType() ||
+         QT->isNullPtrType();
 }
 
 // Decl::isInStdNamespace will return false for iterators in some STL
@@ -276,11 +304,6 @@ static bool isInStlNamespace(const Decl *D) {
   return DC->isStdNamespace();
 }
 
-static bool isPointerLikeType(QualType Type) {
-  return isRecordWithAttr<PointerAttr>(Type) || Type->isPointerType() ||
-         Type->isNullPtrType();
-}
-
 // Returns true if the given Record decl is a form of `GSLOwner<Pointer>`
 // type, e.g. std::vector<string_view>, std::optional<string_view>.
 static bool isContainerOfPointer(const RecordDecl *Container) {
@@ -623,6 +646,26 @@ static void visitFunctionCallArguments(IndirectLocalPath &Path, Expr *Call,
     }
     if (CheckCoroCall || Callee->getParamDecl(I)->hasAttr<LifetimeBoundAttr>())
       VisitLifetimeBoundArg(Callee->getParamDecl(I), Arg);
+    else if (const auto *CaptureAttr =
+                 Callee->getParamDecl(I)->getAttr<LifetimeCaptureByAttr>();
+             CaptureAttr && isa<CXXConstructorDecl>(Callee) &&
+             llvm::any_of(CaptureAttr->params(), [](int ArgIdx) {
+               return ArgIdx == LifetimeCaptureByAttr::THIS;
+             }))
+      // `lifetime_capture_by(this)` in a class constructor has the same
+      // semantics as `lifetimebound`:
+      //
+      // struct Foo {
+      //   const int& a;
+      //   // Equivalent to Foo(const int& t [[clang::lifetimebound]])
+      //   Foo(const int& t [[clang::lifetime_capture_by(this)]]) : a(t) {}
+      // };
+      //
+      // In the implementation, `lifetime_capture_by` is treated as an alias for
+      // `lifetimebound` and shares the same code path. This implies the emitted
+      // diagnostics will be emitted under `-Wdangling`, not
+      // `-Wdangling-capture`.
+      VisitLifetimeBoundArg(Callee->getParamDecl(I), Arg);
     else if (EnableGSLAnalysis && I == 0) {
       // Perform GSL analysis for the first argument
       if (shouldTrackFirstArgument(Callee)) {
@@ -961,6 +1004,17 @@ static void visitLocalsRetainedByInitializer(IndirectLocalPath &Path,
   if (isa<CallExpr>(Init) || isa<CXXConstructExpr>(Init))
     return visitFunctionCallArguments(Path, Init, Visit);
 
+  if (auto *CPE = dyn_cast<CXXParenListInitExpr>(Init)) {
+    RevertToOldSizeRAII RAII(Path);
+    Path.push_back({IndirectLocalPathEntry::ParenAggInit, CPE});
+    for (auto *I : CPE->getInitExprs()) {
+      if (I->isGLValue())
+        visitLocalsRetainedByReferenceBinding(Path, I, RK_ReferenceBinding,
+                                              Visit);
+      else
+        visitLocalsRetainedByInitializer(Path, I, Visit, true);
+    }
+  }
   switch (Init->getStmtClass()) {
   case Stmt::UnaryOperatorClass: {
     auto *UO = cast<UnaryOperator>(Init);
@@ -1057,6 +1111,7 @@ static SourceRange nextPathEntryRange(const IndirectLocalPath &Path, unsigned I,
     case IndirectLocalPathEntry::GslReferenceInit:
     case IndirectLocalPathEntry::GslPointerInit:
     case IndirectLocalPathEntry::GslPointerAssignment:
+    case IndirectLocalPathEntry::ParenAggInit:
       // These exist primarily to mark the path as not permitting or
       // supporting lifetime extension.
       break;
@@ -1368,6 +1423,7 @@ checkExprLifetimeImpl(Sema &SemaRef, const InitializedEntity *InitEntity,
       switch (Elem.Kind) {
       case IndirectLocalPathEntry::AddressOf:
       case IndirectLocalPathEntry::LValToRVal:
+      case IndirectLocalPathEntry::ParenAggInit:
         // These exist primarily to mark the path as not permitting or
         // supporting lifetime extension.
         break;
diff --git a/clang/lib/Sema/CheckExprLifetime.h b/clang/lib/Sema/CheckExprLifetime.h
index b10c84363527a..95c249c610977 100644
--- a/clang/lib/Sema/CheckExprLifetime.h
+++ b/clang/lib/Sema/CheckExprLifetime.h
@@ -18,6 +18,10 @@
 
 namespace clang::sema {
 
+// Tells whether the type is annotated with [[gsl::Pointer]] or is a pointer
+// type.
+bool isPointerLikeType(QualType QT);
+
 /// Describes an entity that is being assigned.
 struct AssignedEntity {
   // The left-hand side expression of the assignment.
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index 951375cc84ba8..79fc2751b7381 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -35,14 +35,17 @@ namespace {
 
 struct TemplateParameterListBuilder;
 
-struct BuiltinTypeDeclBuilder {
-  Sema &SemaRef;
-  CXXRecordDecl *Record = nullptr;
+class BuiltinTypeDeclBuilder {
   ClassTemplateDecl *Template = nullptr;
   ClassTemplateDecl *PrevTemplate = nullptr;
   NamespaceDecl *HLSLNamespace = nullptr;
   llvm::StringMap<FieldDecl *> Fields;
 
+public:
+  Sema &SemaRef;
+  CXXRecordDecl *Record = nullptr;
+  friend struct TemplateParameterListBuilder;
+
   BuiltinTypeDeclBuilder(Sema &SemaRef, CXXRecordDecl *R)
       : SemaRef(SemaRef), Record(R) {
     Record->startDefinition();
@@ -51,7 +54,7 @@ struct BuiltinTypeDeclBuilder {
 
   BuiltinTypeDeclBuilder(Sema &SemaRef, NamespaceDecl *Namespace,
                          StringRef Name)
-      : SemaRef(SemaRef), HLSLNamespace(Namespace) {
+      : HLSLNamespace(Namespace), SemaRef(SemaRef) {
     ASTContext &AST = SemaRef.getASTContext();
     IdentifierInfo &II = AST.Idents.get(Name, tok::TokenKind::identifier);
 
@@ -91,6 +94,18 @@ struct BuiltinTypeDeclBuilder {
       HLSLNamespace->addDecl(Record);
   }
 
+  CXXRecordDecl *finalizeForwardDeclaration() {
+    // Force the QualType to be generated for the record declaration. In most
+    // cases this will happen naturally when something uses the type the
+    // QualType gets lazily created. Unfortunately, with our injected types if a
+    // type isn't used in a translation unit the QualType may not get
+    // automatically generated before a PCH is generated. To resolve this we
+    // just force that the QualType is generated after we create a forward
+    // declaration.
+    (void)Record->getASTContext().getRecordType(Record);
+    return Record;
+  }
+
   BuiltinTypeDeclBuilder &
   addMemberVariable(StringRef Name, QualType Type, llvm::ArrayRef<Attr *> Attrs,
                     AccessSpecifier Access = AccessSpecifier::AS_private) {
@@ -123,12 +138,8 @@ struct BuiltinTypeDeclBuilder {
     assert(!Record->isCompleteDefinition() && "record is already complete");
 
     ASTContext &Ctx = SemaRef.getASTContext();
-    TypeSourceInfo *ElementTypeInfo = nullptr;
-
-    QualType ElemTy = Ctx.Char8Ty;
-    if (Template)
-      ElemTy = getFirstTemplateTypeParam();
-    ElementTypeInfo = Ctx.getTrivialTypeSourceInfo(ElemTy, SourceLocation());
+    TypeSourceInfo *ElementTypeInfo =
+        Ctx.getTrivialTypeSourceInfo(getHandleElementType(), SourceLocation());
 
     // add handle member with resource type attributes
     QualType AttributedResTy = QualType();
@@ -171,80 +182,12 @@ struct BuiltinTypeDeclBuilder {
   }
 
   BuiltinTypeDeclBuilder &addArraySubscriptOperators() {
-    addArraySubscriptOperator(true);
-    addArraySubscriptOperator(false);
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &addArraySubscriptOperator(bool IsConst) {
-    assert(!Record->isCompleteDefinition() && "record is already complete");
-
     ASTContext &AST = Record->getASTContext();
-    QualType ElemTy = AST.Char8Ty;
-    if (Template)
-      ElemTy = getFirstTemplateTypeParam();
-    QualType ReturnTy = ElemTy;
-
-    FunctionProtoType::ExtProtoInfo ExtInfo;
-
-    // Subscript operators return references to elements, const makes the
-    // reference and method const so that the underlying data is not mutable.
-    if (IsConst) {
-      ExtInfo.TypeQuals.addConst();
-      ReturnTy.addConst();
-    }
-    ReturnTy = AST.getLValueReferenceType(ReturnTy);
-
-    QualType MethodTy =
-        AST.getFunctionType(ReturnTy, {AST.UnsignedIntTy}, ExtInfo);
-    auto *TSInfo = AST.getTrivialTypeSourceInfo(MethodTy, SourceLocation());
-    auto *MethodDecl = CXXMethodDecl::Create(
-        AST, Record, SourceLocation(),
-        DeclarationNameInfo(
-            AST.DeclarationNames.getCXXOperatorName(OO_Subscript),
-            SourceLocation()),
-        MethodTy, TSInfo, SC_None, false, false, ConstexprSpecKind::Unspecified,
-        SourceLocation());
-
-    IdentifierInfo &II = AST.Idents.get("Idx", tok::TokenKind::identifier);
-    auto *IdxParam = ParmVarDecl::Create(
-        AST, MethodDecl->getDeclContext(), SourceLocation(), SourceLocation(),
-        &II, AST.UnsignedIntTy,
-        AST.getTrivialTypeSourceInfo(AST.UnsignedIntTy, SourceLocation()),
-        SC_None, nullptr);
-    MethodDecl->setParams({IdxParam});
-
-    // Also add the parameter to the function prototype.
-    auto FnProtoLoc = TSInfo->getTypeLoc().getAs<FunctionProtoTypeLoc>();
-    FnProtoLoc.setParam(0, IdxParam);
-
-    // FIXME: Placeholder to make sure we return the correct type - create
-    // field of element_type and return reference to it. This field will go
-    // away once indexing into resources is properly implemented in
-    // llvm/llvm-project#95956.
-    if (Fields.count("e") == 0) {
-      addMemberVariable("e", ElemTy, {});
-    }
-    FieldDecl *ElemFieldDecl = Fields["e"];
-
-    auto *This =
-        CXXThisExpr::Create(AST, SourceLocation(),
-                            MethodDecl->getFunctionObjectParameterType(), true);
-    Expr *ElemField = MemberExpr::CreateImplicit(
-        AST, This, false, ElemFieldDecl, ElemFieldDecl->getType(), VK_LValue,
-        OK_Ordinary);
-    auto *Return =
-        ReturnStmt::Create(AST, SourceLocation(), ElemField, nullptr);
-
-    MethodDecl->setBody(CompoundStmt::Create(AST, {Return}, FPOptionsOverride(),
-                                             SourceLocation(),
-                                             SourceLocation()));
-    MethodDecl->setLexicalDeclContext(Record);
-    MethodDecl->setAccess(AccessSpecifier::AS_public);
-    MethodDecl->addAttr(AlwaysInlineAttr::CreateImplicit(
-        AST, SourceRange(), AlwaysInlineAttr::CXX11_clang_always_inline));
-    Record->addDecl(MethodDecl);
+    DeclarationName Subscript =
+        AST.DeclarationNames.getCXXOperatorName(OO_Subscript);
 
+    addHandleAccessFunction(Subscript, /*IsConst=*/true, /*IsRef=*/true);
+    addHandleAccessFunction(Subscript, /*IsConst=*/false, /*IsRef=*/true);
     return *this;
   }
 
@@ -265,6 +208,13 @@ struct BuiltinTypeDeclBuilder {
     return QualType();
   }
 
+  QualType getHandleElementType() {
+    if (Template)
+      return getFirstTemplateTypeParam();
+    // TODO: Should we default to VoidTy? Using `i8` is arguably ambiguous.
+    return SemaRef.getASTContext().Char8Ty;
+  }
+
   BuiltinTypeDeclBuilder &startDefinition() {
     assert(!Record->isCompleteDefinition() && "record is already complete");
     Record->startDefinition();
@@ -294,6 +244,10 @@ struct BuiltinTypeDeclBuilder {
   // Builtin types methods
   BuiltinTypeDeclBuilder &addIncrementCounterMethod();
   BuiltinTypeDeclBuilder &addDecrementCounterMethod();
+  BuiltinTypeDeclBuilder &addHandleAccessFunction(DeclarationName &Name,
+                                                  bool IsConst, bool IsRef);
+  BuiltinTypeDeclBuilder &addAppendMethod();
+  BuiltinTypeDeclBuilder &addConsumeMethod();
 };
 
 struct TemplateParameterListBuilder {
@@ -371,9 +325,9 @@ struct TemplateParameterListBuilder {
         Context,                          // AST context
         Builder.Record->getDeclContext(), // DeclContext
         SourceLocation(), SourceLocation(),
-        /*depth=*/0,                // Depth in the template parameter list
-        /*position=*/0,             // Position in the template parameter list
-        /*id=*/nullptr,             // Identifier for 'T'
+        /*D=*/0,                    // Depth in the template parameter list
+        /*P=*/0,                    // Position in the template parameter list
+        /*Id=*/nullptr,             // Identifier for 'T'
         /*Typename=*/true,          // Indicates this is a 'typename' or 'class'
         /*ParameterPack=*/false,    // Not a parameter pack
         /*HasTypeConstraint=*/false // Has no type constraint
@@ -453,7 +407,7 @@ struct TemplateParameterListBuilder {
 // Builder for methods of builtin types. Allows adding methods to builtin types
 // using the builder pattern like this:
 //
-//   BuiltinTypeMethodBuilder(Sema, RecordBuilder, "MethodName", ReturnType)
+//   BuiltinTypeMethodBuilder(RecordBuilder, "MethodName", ReturnType)
 //       .addParam("param_name", Type, InOutModifier)
 //       .callBuiltin("builtin_name", BuiltinParams...)
 //       .finalizeMethod();
@@ -486,18 +440,31 @@ struct BuiltinTypeMethodBuilder {
   DeclarationNameInfo NameInfo;
   QualType ReturnTy;
   CXXMethodDecl *Method;
+  bool IsConst;
   llvm::SmallVector<MethodParam> Params;
   llvm::SmallVector<Stmt *> StmtsList;
 
   // Argument placeholders, inspired by std::placeholder. These are the indices
-  // of arguments to forward to `callBuiltin`, and additionally `Handle` which
-  // refers to the resource handle.
-  enum class PlaceHolder { _0, _1, _2, _3, Handle = 127 };
+  // of arguments to forward to `callBuiltin` and other method builder methods.
+  // Additional special values are:
+  //   Handle   - refers to the resource handle.
+  //   LastStmt - refers to the last statement in the method body; referencing
+  //              LastStmt will remove the statement from the method body since
+  //              it will be linked from the new expression being constructed.
+  enum class PlaceHolder { _0, _1, _2, _3, Handle = 128, LastStmt };
 
   Expr *convertPlaceholder(PlaceHolder PH) {
     if (PH == PlaceHolder::Handle)
       return getResourceHandleExpr();
 
+    if (PH == PlaceHolder::LastStmt) {
+      assert(!StmtsList.empty() && "no statements in the list");
+      Stmt *LastStmt = StmtsList.pop_back_val();
+      assert(isa<ValueStmt>(LastStmt) &&
+             "last statement does not have a value");
+      return cast<ValueStmt>(LastStmt)->getExprStmt();
+    }
+
     ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
     ParmVarDecl *ParamDecl = Method->getParamDecl(static_cast<unsigned>(PH));
     return DeclRefExpr::Create(
@@ -508,11 +475,16 @@ struct BuiltinTypeMethodBuilder {
   Expr *convertPlaceholder(Expr *E) { return E; }
 
 public:
-  BuiltinTypeMethodBuilder(Sema &S, BuiltinTypeDeclBuilder &DB, StringRef Name,
-                           QualType ReturnTy)
-      : DeclBuilder(DB), ReturnTy(ReturnTy), Method(nullptr) {
+  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, DeclarationName &Name,
+                           QualType ReturnTy, bool IsConst = false)
+      : DeclBuilder(DB), NameInfo(DeclarationNameInfo(Name, SourceLocation())),
+        ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {}
+
+  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, StringRef Name,
+                           QualType ReturnTy, bool IsConst = false)
+      : DeclBuilder(DB), ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {
     const IdentifierInfo &II =
-        S.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
+        DB.SemaRef.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
     NameInfo = DeclarationNameInfo(DeclarationName(&II), SourceLocation());
   }
 
@@ -535,8 +507,12 @@ struct BuiltinTypeMethodBuilder {
     SmallVector<QualType> ParamTypes;
     for (MethodParam &MP : Params)
       ParamTypes.emplace_back(MP.Ty);
-    QualType MethodTy = AST.getFunctionType(ReturnTy, ParamTypes,
-                                            FunctionProtoType::ExtProtoInfo());
+
+    FunctionProtoType::ExtProtoInfo ExtInfo;
+    if (IsConst)
+      ExtInfo.TypeQuals.addConst();
+
+    QualType MethodTy = AST.getFunctionType(ReturnTy, ParamTypes, ExtInfo);
 
     // create method decl
     auto *TSInfo = AST.getTrivialTypeSourceInfo(MethodTy, SourceLocation());
@@ -586,7 +562,8 @@ struct BuiltinTypeMethodBuilder {
   }
 
   template <typename... Ts>
-  BuiltinTypeMethodBuilder &callBuiltin(StringRef BuiltinName, Ts... ArgSpecs) {
+  BuiltinTypeMethodBuilder &callBuiltin(StringRef BuiltinName,
+                                        QualType ReturnType, Ts... ArgSpecs) {
     std::array<Expr *, sizeof...(ArgSpecs)> Args{
         convertPlaceholder(std::forward<Ts>(ArgSpecs))...};
 
@@ -599,15 +576,40 @@ struct BuiltinTypeMethodBuilder {
     FunctionDecl *FD = lookupBuiltinFunction(DeclBuilder.SemaRef, BuiltinName);
     DeclRefExpr *DRE = DeclRefExpr::Create(
         AST, NestedNameSpecifierLoc(), SourceLocation(), FD, false,
-        FD->getNameInfo(), FD->getType(), VK_PRValue);
+        FD->getNameInfo(), AST.BuiltinFnTy, VK_PRValue);
 
-    Expr *Call =
-        CallExpr::Create(AST, DRE, Args, FD->getReturnType(), VK_PRValue,
-                         SourceLocation(), FPOptionsOverride());
+    if (ReturnType.isNull())
+      ReturnType = FD->getReturnType();
+
+    Expr *Call = CallExpr::Create(AST, DRE, Args, ReturnType, VK_PRValue,
+                                  SourceLocation(), FPOptionsOverride());
     StmtsList.push_back(Call);
     return *this;
   }
 
+  template <typename TLHS, typename TRHS>
+  BuiltinTypeMethodBuilder &assign(TLHS LHS, TRHS RHS) {
+    Expr *LHSExpr = convertPlaceholder(LHS);
+    Expr *RHSExpr = convertPlaceholder(RHS);
+    Stmt *AssignStmt = BinaryOperator::Create(
+        DeclBuilder.SemaRef.getASTContext(), LHSExpr, RHSExpr, BO_Assign,
+        LHSExpr->getType(), ExprValueKind::VK_PRValue,
+        ExprObjectKind::OK_Ordinary, SourceLocation(), FPOptionsOverride());
+    StmtsList.push_back(AssignStmt);
+    return *this;
+  }
+
+  template <typename T> BuiltinTypeMethodBuilder &dereference(T Ptr) {
+    Expr *PtrExpr = convertPlaceholder(Ptr);
+    Expr *Deref =
+        UnaryOperator::Create(DeclBuilder.SemaRef.getASTContext(), PtrExpr,
+                              UO_Deref, PtrExpr->getType()->getPointeeType(),
+                              VK_PRValue, OK_Ordinary, SourceLocation(),
+                              /*CanOverflow=*/false, FPOptionsOverride());
+    StmtsList.push_back(Deref);
+    return *this;
+  }
+
   BuiltinTypeDeclBuilder &finalizeMethod() {
     assert(!DeclBuilder.Record->isCompleteDefinition() &&
            "record is already complete");
@@ -621,11 +623,8 @@ struct BuiltinTypeMethodBuilder {
              "nothing to return from non-void method");
       if (ReturnTy != AST.VoidTy) {
         if (Expr *LastExpr = dyn_cast<Expr>(StmtsList.back())) {
-          assert(AST.hasSameUnqualifiedType(
-                     isa<CallExpr>(LastExpr)
-                         ? cast<CallExpr>(LastExpr)->getCallReturnType(AST)
-                         : LastExpr->getType(),
-                     ReturnTy) &&
+          assert(AST.hasSameUnqualifiedType(LastExpr->getType(),
+                                            ReturnTy.getNonReferenceType()) &&
                  "Return type of the last statement must match the return type "
                  "of the method");
           if (!isa<ReturnStmt>(LastExpr)) {
@@ -672,19 +671,71 @@ BuiltinTypeDeclBuilder::addSimpleTemplateParams(ArrayRef<StringRef> Names,
 
 BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addIncrementCounterMethod() {
   using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  return BuiltinTypeMethodBuilder(SemaRef, *this, "IncrementCounter",
+  return BuiltinTypeMethodBuilder(*this, "IncrementCounter",
                                   SemaRef.getASTContext().UnsignedIntTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", PH::Handle,
-                   getConstantIntExpr(1))
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
+                   PH::Handle, getConstantIntExpr(1))
       .finalizeMethod();
 }
 
 BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addDecrementCounterMethod() {
   using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  return BuiltinTypeMethodBuilder(SemaRef, *this, "DecrementCounter",
+  return BuiltinTypeMethodBuilder(*this, "DecrementCounter",
                                   SemaRef.getASTContext().UnsignedIntTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", PH::Handle,
-                   getConstantIntExpr(-1))
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
+                   PH::Handle, getConstantIntExpr(-1))
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addHandleAccessFunction(DeclarationName &Name,
+                                                bool IsConst, bool IsRef) {
+  assert(!Record->isCompleteDefinition() && "record is already complete");
+  ASTContext &AST = SemaRef.getASTContext();
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+
+  QualType ElemTy = getHandleElementType();
+  // TODO: Map to an hlsl_device address space.
+  QualType ElemPtrTy = AST.getPointerType(ElemTy);
+  QualType ReturnTy = ElemTy;
+  if (IsConst)
+    ReturnTy.addConst();
+  if (IsRef)
+    ReturnTy = AST.getLValueReferenceType(ReturnTy);
+
+  return BuiltinTypeMethodBuilder(*this, Name, ReturnTy, IsConst)
+      .addParam("Index", AST.UnsignedIntTy)
+      .callBuiltin("__builtin_hlsl_resource_getpointer", ElemPtrTy, PH::Handle,
+                   PH::_0)
+      .dereference(PH::LastStmt)
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  ASTContext &AST = SemaRef.getASTContext();
+  QualType ElemTy = getHandleElementType();
+  return BuiltinTypeMethodBuilder(*this, "Append", AST.VoidTy)
+      .addParam("value", ElemTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
+                   PH::Handle, getConstantIntExpr(1))
+      .callBuiltin("__builtin_hlsl_resource_getpointer",
+                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
+      .dereference(PH::LastStmt)
+      .assign(PH::LastStmt, PH::_0)
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addConsumeMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  ASTContext &AST = SemaRef.getASTContext();
+  QualType ElemTy = getHandleElementType();
+  return BuiltinTypeMethodBuilder(*this, "Consume", ElemTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
+                   PH::Handle, getConstantIntExpr(-1))
+      .callBuiltin("__builtin_hlsl_resource_getpointer",
+                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
+      .dereference(PH::LastStmt)
       .finalizeMethod();
 }
 
@@ -800,7 +851,7 @@ static Expr *constructTypedBufferConstraintExpr(Sema &S, SourceLocation NameLoc,
                                                 TemplateTypeParmDecl *T) {
   ASTContext &Context = S.getASTContext();
 
-  // Obtain the QualType for 'unsigned long'
+  // Obtain the QualType for 'bool'
   QualType BoolTy = Context.BoolTy;
 
   // Create a QualType that points to this TemplateTypeParmDecl
@@ -826,9 +877,9 @@ static ConceptDecl *constructTypedBufferConceptDecl(Sema &S,
   IdentifierInfo &ElementTypeII = Context.Idents.get("element_type");
   TemplateTypeParmDecl *T = TemplateTypeParmDecl::Create(
       Context, NSD->getDeclContext(), DeclLoc, DeclLoc,
-      /*depth=*/0,
-      /*position=*/0,
-      /*id=*/&ElementTypeII,
+      /*D=*/0,
+      /*P=*/0,
+      /*Id=*/&ElementTypeII,
       /*Typename=*/true,
       /*ParameterPack=*/false);
 
@@ -863,7 +914,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
       constructTypedBufferConceptDecl(*SemaPtr, HLSLNamespace);
   Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWBuffer")
              .addSimpleTemplateParams({"element_type"}, TypedBufferConcept)
-             .Record;
+             .finalizeForwardDeclaration();
 
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
@@ -876,7 +927,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
   Decl =
       BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RasterizerOrderedBuffer")
           .addSimpleTemplateParams({"element_type"})
-          .Record;
+          .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
                     ResourceKind::TypedBuffer, /*IsROV=*/true,
@@ -887,7 +938,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
 
   Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "StructuredBuffer")
              .addSimpleTemplateParams({"element_type"})
-             .Record;
+             .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::SRV, ResourceKind::RawBuffer,
                     /*IsROV=*/false, /*RawBuffer=*/true)
@@ -897,7 +948,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
 
   Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWStructuredBuffer")
              .addSimpleTemplateParams({"element_type"})
-             .Record;
+             .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
                     /*IsROV=*/false, /*RawBuffer=*/true)
@@ -910,27 +961,29 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
   Decl =
       BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "AppendStructuredBuffer")
           .addSimpleTemplateParams({"element_type"})
-          .Record;
+          .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
                     /*IsROV=*/false, /*RawBuffer=*/true)
+        .addAppendMethod()
         .completeDefinition();
   });
 
   Decl =
       BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "ConsumeStructuredBuffer")
           .addSimpleTemplateParams({"element_type"})
-          .Record;
+          .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
                     /*IsROV=*/false, /*RawBuffer=*/true)
+        .addConsumeMethod()
         .completeDefinition();
   });
 
   Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace,
                                 "RasterizerOrderedStructuredBuffer")
              .addSimpleTemplateParams({"element_type"})
-             .Record;
+             .finalizeForwardDeclaration();
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
                     /*IsROV=*/true, /*RawBuffer=*/true)
@@ -939,6 +992,32 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
         .addDecrementCounterMethod()
         .completeDefinition();
   });
+
+  Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "ByteAddressBuffer")
+             .finalizeForwardDeclaration();
+  onCompletion(Decl, [this](CXXRecordDecl *Decl) {
+    setupBufferType(Decl, *SemaPtr, ResourceClass::SRV, ResourceKind::RawBuffer,
+                    /*IsROV=*/false,
+                    /*RawBuffer=*/true)
+        .completeDefinition();
+  });
+  Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWByteAddressBuffer")
+             .finalizeForwardDeclaration();
+  onCompletion(Decl, [this](CXXRecordDecl *Decl) {
+    setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
+                    /*IsROV=*/false,
+                    /*RawBuffer=*/true)
+        .completeDefinition();
+  });
+  Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace,
+                                "RasterizerOrderedByteAddressBuffer")
+             .finalizeForwardDeclaration();
+  onCompletion(Decl, [this](CXXRecordDecl *Decl) {
+    setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, ResourceKind::RawBuffer,
+                    /*IsROV=*/true,
+                    /*RawBuffer=*/true)
+        .completeDefinition();
+  });
 }
 
 void HLSLExternalSemaSource::onCompletion(CXXRecordDecl *Record,
diff --git a/clang/lib/Sema/MultiplexExternalSemaSource.cpp b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
index cd44483b5cbe0..54944267b4868 100644
--- a/clang/lib/Sema/MultiplexExternalSemaSource.cpp
+++ b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -115,6 +115,23 @@ FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name) {
   return AnyDeclsFound;
 }
 
+bool MultiplexExternalSemaSource::LoadExternalSpecializations(
+    const Decl *D, bool OnlyPartial) {
+  bool Loaded = false;
+  for (size_t i = 0; i < Sources.size(); ++i)
+    Loaded |= Sources[i]->LoadExternalSpecializations(D, OnlyPartial);
+  return Loaded;
+}
+
+bool MultiplexExternalSemaSource::LoadExternalSpecializations(
+    const Decl *D, ArrayRef<TemplateArgument> TemplateArgs) {
+  bool AnyNewSpecsLoaded = false;
+  for (size_t i = 0; i < Sources.size(); ++i)
+    AnyNewSpecsLoaded |=
+        Sources[i]->LoadExternalSpecializations(D, TemplateArgs);
+  return AnyNewSpecsLoaded;
+}
+
 void MultiplexExternalSemaSource::completeVisibleDeclsMap(const DeclContext *DC){
   for(size_t i = 0; i < Sources.size(); ++i)
     Sources[i]->completeVisibleDeclsMap(DC);
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 744b8aa8cc724..93f0b6dbb2502 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -750,6 +750,15 @@ ExprResult Sema::ImpCastExprToType(Expr *E, QualType Ty,
   QualType ExprTy = Context.getCanonicalType(E->getType());
   QualType TypeTy = Context.getCanonicalType(Ty);
 
+  // This cast is used in place of a regular LValue to RValue cast for
+  // HLSL Array Parameter Types. It needs to be emitted even if
+  // ExprTy == TypeTy, except if E is an HLSLOutArgExpr
+  // Emitting a cast in that case will prevent HLSLOutArgExpr from
+  // being handled properly in EmitCallArg
+  if (Kind == CK_HLSLArrayRValue && !isa<HLSLOutArgExpr>(E))
+    return ImplicitCastExpr::Create(Context, Ty, Kind, E, BasePath, VK,
+                                    CurFPFeatureOverrides());
+
   if (ExprTy == TypeTy)
     return E;
 
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 028bf82f3e804..4f79775bc5e91 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -482,7 +482,7 @@ static void ProcessAPINotes(Sema &S, FunctionOrMethod AnyFunc,
   Decl *D = FD;
   ObjCMethodDecl *MD = nullptr;
   if (!D) {
-    MD = AnyFunc.get<ObjCMethodDecl *>();
+    MD = cast<ObjCMethodDecl *>(AnyFunc);
     D = MD;
   }
 
@@ -511,6 +511,11 @@ static void ProcessAPINotes(Sema &S, FunctionOrMethod AnyFunc,
       AnyTypeChanged = true;
   }
 
+  // returns_(un)retained
+  if (!Info.SwiftReturnOwnership.empty())
+    D->addAttr(SwiftAttrAttr::Create(S.Context,
+                                     "returns_" + Info.SwiftReturnOwnership));
+
   // Result type override.
   QualType OverriddenResultType;
   if (Metadata.IsActive && !Info.ResultType.empty() &&
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 716d8ed1fae4f..479f47962a7c3 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -269,42 +269,49 @@ void Sema::inferLifetimeBoundAttribute(FunctionDecl *FD) {
   }
 }
 
-static bool isPointerLikeType(QualType QT) {
-  QT = QT.getNonReferenceType();
-  if (QT->isPointerType())
-    return true;
-  auto *RD = QT->getAsCXXRecordDecl();
-  if (!RD)
-    return false;
-  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
-    RD = CTSD->getSpecializedTemplate()->getTemplatedDecl();
-  return RD->hasAttr<PointerAttr>();
-}
-
 void Sema::inferLifetimeCaptureByAttribute(FunctionDecl *FD) {
-  if (!FD)
+  auto *MD = dyn_cast_if_present<CXXMethodDecl>(FD);
+  if (!MD || !MD->getParent()->isInStdNamespace())
     return;
-  auto *MD = dyn_cast<CXXMethodDecl>(FD);
-  if (!MD || !MD->getIdentifier() || !MD->getParent()->isInStdNamespace())
+  auto Annotate = [this](const FunctionDecl *MD) {
+    // Do not infer if any parameter is explicitly annotated.
+    for (ParmVarDecl *PVD : MD->parameters())
+      if (PVD->hasAttr<LifetimeCaptureByAttr>())
+        return;
+    for (ParmVarDecl *PVD : MD->parameters()) {
+      // Methods in standard containers that capture values typically accept
+      // reference-type parameters, e.g., `void push_back(const T& value)`.
+      // We only apply the lifetime_capture_by attribute to parameters of
+      // pointer-like reference types (`const T&`, `T&&`).
+      if (PVD->getType()->isReferenceType() &&
+          sema::isPointerLikeType(PVD->getType().getNonReferenceType())) {
+        int CaptureByThis[] = {LifetimeCaptureByAttr::THIS};
+        PVD->addAttr(
+            LifetimeCaptureByAttr::CreateImplicit(Context, CaptureByThis, 1));
+      }
+    }
+  };
+
+  if (!MD->getIdentifier()) {
+    static const llvm::StringSet<> MapLikeContainer{
+        "map",
+        "multimap",
+        "unordered_map",
+        "unordered_multimap",
+    };
+    // Infer for the map's operator []:
+    //    std::map<string_view, ...> m;
+    //    m[ReturnString(..)] = ...; // !dangling references in m.
+    if (MD->getOverloadedOperator() == OO_Subscript &&
+        MapLikeContainer.contains(MD->getParent()->getName()))
+      Annotate(MD);
     return;
-  // FIXME: Infer for operator[] for map-like containers. For example:
-  //    std::map<string_view, ...> m;
-  //    m[ReturnString(..)] = ...;
+  }
   static const llvm::StringSet<> CapturingMethods{"insert", "push",
                                                   "push_front", "push_back"};
   if (!CapturingMethods.contains(MD->getName()))
     return;
-  // Do not infer if any parameter is explicitly annotated.
-  for (ParmVarDecl *PVD : MD->parameters())
-    if (PVD->hasAttr<LifetimeCaptureByAttr>())
-      return;
-  for (ParmVarDecl *PVD : MD->parameters()) {
-    if (isPointerLikeType(PVD->getType())) {
-      int CaptureByThis[] = {LifetimeCaptureByAttr::THIS};
-      PVD->addAttr(
-          LifetimeCaptureByAttr::CreateImplicit(Context, CaptureByThis, 1));
-    }
-  }
+  Annotate(MD);
 }
 
 void Sema::inferNullableClassAttribute(CXXRecordDecl *CRD) {
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e9497d37834b1..8253771ffe341 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -3020,6 +3020,11 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     }
     break;
   }
+  case Builtin::BI__builtin_counted_by_ref:
+    if (BuiltinCountedByRef(TheCall))
+      return ExprError();
+    break;
+
   case Builtin::BI__builtin_sycl_is_kernel:
   case Builtin::BI__builtin_sycl_is_single_task_kernel:
   case Builtin::BI__builtin_sycl_is_nd_range_kernel: {
@@ -3053,10 +3058,6 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
 
     break;
   }
-  case Builtin::BI__builtin_counted_by_ref:
-    if (BuiltinCountedByRef(TheCall))
-      return ExprError();
-    break;
   }
 
   if (getLangOpts().HLSL && HLSL().CheckBuiltinFunctionCall(BuiltinID, TheCall))
@@ -3648,8 +3649,14 @@ void Sema::checkLifetimeCaptureBy(FunctionDecl *FD, bool IsMemberFunction,
                                  unsigned ArgIdx) {
     if (!Attr)
       return;
+
     Expr *Captured = const_cast<Expr *>(GetArgAt(ArgIdx));
     for (int CapturingParamIdx : Attr->params()) {
+      // lifetime_capture_by(this) case is handled in the lifetimebound expr
+      // initialization codepath.
+      if (CapturingParamIdx == LifetimeCaptureByAttr::THIS &&
+          isa<CXXConstructorDecl>(FD))
+        continue;
       Expr *Capturing = const_cast<Expr *>(GetArgAt(CapturingParamIdx));
       CapturingEntity CE{Capturing};
       // Ensure that 'Captured' outlives the 'Capturing' entity.
@@ -6074,6 +6081,45 @@ bool Sema::BuiltinCountedByRef(CallExpr *TheCall) {
   return false;
 }
 
+/// The result of __builtin_counted_by_ref cannot be assigned to a variable.
+/// It allows leaking and modification of bounds safety information.
+bool Sema::CheckInvalidBuiltinCountedByRef(const Expr *E,
+                                           BuiltinCountedByRefKind K) {
+  const CallExpr *CE =
+      E ? dyn_cast<CallExpr>(E->IgnoreParenImpCasts()) : nullptr;
+  if (!CE || CE->getBuiltinCallee() != Builtin::BI__builtin_counted_by_ref)
+    return false;
+
+  switch (K) {
+  case AssignmentKind:
+  case InitializerKind:
+    Diag(E->getExprLoc(),
+         diag::err_builtin_counted_by_ref_cannot_leak_reference)
+        << 0 << E->getSourceRange();
+    break;
+  case FunctionArgKind:
+    Diag(E->getExprLoc(),
+         diag::err_builtin_counted_by_ref_cannot_leak_reference)
+        << 1 << E->getSourceRange();
+    break;
+  case ReturnArgKind:
+    Diag(E->getExprLoc(),
+         diag::err_builtin_counted_by_ref_cannot_leak_reference)
+        << 2 << E->getSourceRange();
+    break;
+  case ArraySubscriptKind:
+    Diag(E->getExprLoc(), diag::err_builtin_counted_by_ref_invalid_use)
+        << 0 << E->getSourceRange();
+    break;
+  case BinaryExprKind:
+    Diag(E->getExprLoc(), diag::err_builtin_counted_by_ref_invalid_use)
+        << 1 << E->getSourceRange();
+    break;
+  }
+
+  return true;
+}
+
 namespace {
 
 class UncoveredArgHandler {
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 60ea1383b2a6e..1f398bb004fa3 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -131,8 +131,8 @@ class ResultBuilder {
       }
 
       // Add the new element to the end of the vector.
-      DeclOrVector.get<DeclIndexPairVector *>()->push_back(
-          DeclIndexPair(ND, Index));
+      cast<DeclIndexPairVector *>(DeclOrVector)
+          ->push_back(DeclIndexPair(ND, Index));
     }
 
     ~ShadowMapEntry() {
@@ -659,13 +659,13 @@ class ResultBuilder::ShadowMapEntry::iterator {
       : DeclOrIterator(Iterator), SingleDeclIndex(0) {}
 
   iterator &operator++() {
-    if (DeclOrIterator.is<const NamedDecl *>()) {
+    if (isa<const NamedDecl *>(DeclOrIterator)) {
       DeclOrIterator = (NamedDecl *)nullptr;
       SingleDeclIndex = 0;
       return *this;
     }
 
-    const DeclIndexPair *I = DeclOrIterator.get<const DeclIndexPair *>();
+    const DeclIndexPair *I = cast<const DeclIndexPair *>(DeclOrIterator);
     ++I;
     DeclOrIterator = I;
     return *this;
@@ -681,7 +681,7 @@ class ResultBuilder::ShadowMapEntry::iterator {
     if (const NamedDecl *ND = DeclOrIterator.dyn_cast<const NamedDecl *>())
       return reference(ND, SingleDeclIndex);
 
-    return *DeclOrIterator.get<const DeclIndexPair *>();
+    return *cast<const DeclIndexPair *>(DeclOrIterator);
   }
 
   pointer operator->() const { return pointer(**this); }
@@ -705,15 +705,15 @@ ResultBuilder::ShadowMapEntry::begin() const {
   if (const NamedDecl *ND = DeclOrVector.dyn_cast<const NamedDecl *>())
     return iterator(ND, SingleDeclIndex);
 
-  return iterator(DeclOrVector.get<DeclIndexPairVector *>()->begin());
+  return iterator(cast<DeclIndexPairVector *>(DeclOrVector)->begin());
 }
 
 ResultBuilder::ShadowMapEntry::iterator
 ResultBuilder::ShadowMapEntry::end() const {
-  if (DeclOrVector.is<const NamedDecl *>() || DeclOrVector.isNull())
+  if (isa<const NamedDecl *>(DeclOrVector) || DeclOrVector.isNull())
     return iterator();
 
-  return iterator(DeclOrVector.get<DeclIndexPairVector *>()->end());
+  return iterator(cast<DeclIndexPairVector *>(DeclOrVector)->end());
 }
 
 /// Compute the qualification required to get from the current context
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 1bdf3a02b2924..ff1df7b71b1a4 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -1384,8 +1384,7 @@ static void diagnoseUnsatisfiedConstraintExpr(
     return;
   }
 
-  diagnoseWellFormedUnsatisfiedConstraintExpr(S,
-      Record.template get<Expr *>(), First);
+  diagnoseWellFormedUnsatisfiedConstraintExpr(S, cast<Expr *>(Record), First);
 }
 
 void
@@ -1557,12 +1556,12 @@ NormalizedConstraint::NormalizedConstraint(ASTContext &C,
 
 NormalizedConstraint &NormalizedConstraint::getLHS() const {
   assert(isCompound() && "getLHS called on a non-compound constraint.");
-  return Constraint.get<CompoundConstraint>().getPointer()->LHS;
+  return cast<CompoundConstraint>(Constraint).getPointer()->LHS;
 }
 
 NormalizedConstraint &NormalizedConstraint::getRHS() const {
   assert(isCompound() && "getRHS called on a non-compound constraint.");
-  return Constraint.get<CompoundConstraint>().getPointer()->RHS;
+  return cast<CompoundConstraint>(Constraint).getPointer()->RHS;
 }
 
 std::optional<NormalizedConstraint>
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 322919a09e5eb..efed132da0546 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -16,7 +16,6 @@
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/CharUnits.h"
-#include "clang/AST/CommentDiagnostic.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
@@ -30,6 +29,7 @@
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/DiagnosticComment.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -8867,6 +8867,19 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
     }
   }
 
+  // zero sized static arrays are not allowed in HIP device functions
+  if (getLangOpts().HIP && LangOpts.CUDAIsDevice) {
+    if (FunctionDecl *FD = getCurFunctionDecl();
+        FD &&
+        (FD->hasAttr<CUDADeviceAttr>() || FD->hasAttr<CUDAGlobalAttr>())) {
+      if (const ConstantArrayType *ArrayT =
+              getASTContext().getAsConstantArrayType(T);
+          ArrayT && ArrayT->isZeroSize()) {
+        Diag(NewVD->getLocation(), diag::err_typecheck_zero_array_size) << 2;
+      }
+    }
+  }
+
   bool isVM = T->isVariablyModifiedType();
   if (isVM || NewVD->hasAttr<CleanupAttr>() ||
       NewVD->hasAttr<BlocksAttr>())
@@ -14819,6 +14832,8 @@ void Sema::FinalizeDeclaration(Decl *ThisDecl) {
     }
   }
 
+  CheckInvalidBuiltinCountedByRef(VD->getInit(), InitializerKind);
+
   checkAttributesAfterMerging(*this, *VD);
 
   if (VD->isStaticLocal())
@@ -15624,10 +15639,25 @@ LambdaScopeInfo *Sema::RebuildLambdaScopeInfo(CXXMethodDecl *CallOperator) {
   LSI->CallOperator = CallOperator;
   LSI->Lambda = LambdaClass;
   LSI->ReturnType = CallOperator->getReturnType();
-  // This function in calls in situation where the context of the call operator
-  // is not entered, so we set AfterParameterList to false, so that
+  // When this function is called in situation where the context of the call
+  // operator is not entered, we set AfterParameterList to false, so that
   // `tryCaptureVariable` finds explicit captures in the appropriate context.
-  LSI->AfterParameterList = false;
+  // There is also at least a situation as in FinishTemplateArgumentDeduction(),
+  // where we would set the CurContext to the lambda operator before
+  // substituting into it. In this case the flag needs to be true such that
+  // tryCaptureVariable can correctly handle potential captures thereof.
+  LSI->AfterParameterList = CurContext == CallOperator;
+
+  // GLTemplateParameterList is necessary for getCurGenericLambda() which is
+  // used at the point of dealing with potential captures.
+  //
+  // We don't use LambdaClass->isGenericLambda() because this value doesn't
+  // flip for instantiated generic lambdas, where no FunctionTemplateDecls are
+  // associated. (Technically, we could recover that list from their
+  // instantiation patterns, but for now, the GLTemplateParameterList seems
+  // unnecessary in these cases.)
+  if (FunctionTemplateDecl *FTD = CallOperator->getDescribedFunctionTemplate())
+    LSI->GLTemplateParameterList = FTD->getTemplateParameters();
   const LambdaCaptureDefault LCD = LambdaClass->getLambdaCaptureDefault();
 
   if (LCD == LCD_None)
@@ -17405,7 +17435,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
         if (TypeSourceInfo *TI = EnumUnderlying.dyn_cast<TypeSourceInfo *>())
           ED->setIntegerTypeSourceInfo(TI);
         else
-          ED->setIntegerType(QualType(EnumUnderlying.get<const Type *>(), 0));
+          ED->setIntegerType(QualType(cast<const Type *>(EnumUnderlying), 0));
         QualType EnumTy = ED->getIntegerType();
         ED->setPromotionType(Context.isPromotableIntegerType(EnumTy)
                                  ? Context.getPromotedIntegerType(EnumTy)
@@ -18038,7 +18068,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
       if (TypeSourceInfo *TI = EnumUnderlying.dyn_cast<TypeSourceInfo*>())
         ED->setIntegerTypeSourceInfo(TI);
       else
-        ED->setIntegerType(QualType(EnumUnderlying.get<const Type *>(), 0));
+        ED->setIntegerType(QualType(cast<const Type *>(EnumUnderlying), 0));
       QualType EnumTy = ED->getIntegerType();
       ED->setPromotionType(Context.isPromotableIntegerType(EnumTy)
                                ? Context.getPromotedIntegerType(EnumTy)
@@ -20062,7 +20092,7 @@ static void CheckForDuplicateEnumValues(Sema &S, ArrayRef<Decl *> Elements,
       continue;
     }
 
-    ECDVector *Vec = Entry.get<ECDVector*>();
+    ECDVector *Vec = cast<ECDVector *>(Entry);
     // Make sure constants are not added more than once.
     if (*Vec->begin() == ECD)
       continue;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 534d8e4e33a64..216ca3fb0a5f8 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -822,9 +822,9 @@ static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D,
   auto DiagnoseType = [&](unsigned Index, AttributeArgumentNType T) {
     SourceLocation Loc = [&]() {
       auto Union = AL.getArg(Index - 1);
-      if (Union.is<Expr *>())
-        return Union.get<Expr *>()->getBeginLoc();
-      return Union.get<IdentifierLoc *>()->Loc;
+      if (auto *E = dyn_cast<Expr *>(Union))
+        return E->getBeginLoc();
+      return cast<IdentifierLoc *>(Union)->Loc;
     }();
 
     S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T;
@@ -1258,6 +1258,14 @@ static void handlePreferredName(Sema &S, Decl *D, const ParsedAttr &AL) {
         << TT->getDecl();
 }
 
+static void handleNoSpecializations(Sema &S, Decl *D, const ParsedAttr &AL) {
+  StringRef Message;
+  if (AL.getNumArgs() != 0)
+    S.checkStringLiteralArgumentAttr(AL, 0, Message);
+  D->getDescribedTemplate()->addAttr(
+      NoSpecializationsAttr::Create(S.Context, Message, AL));
+}
+
 bool Sema::isValidPointerAttrType(QualType T, bool RefOkay) {
   if (T->isDependentType())
     return true;
@@ -7056,6 +7064,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_PreferredName:
     handlePreferredName(S, D, AL);
     break;
+  case ParsedAttr::AT_NoSpecializations:
+    handleNoSpecializations(S, D, AL);
+    break;
   case ParsedAttr::AT_Section:
     handleSectionAttr(S, D, AL);
     break;
@@ -7246,6 +7257,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLWaveSize:
     S.HLSL().handleWaveSizeAttr(D, AL);
     break;
+  case ParsedAttr::AT_HLSLSV_GroupThreadID:
+    S.HLSL().handleSV_GroupThreadIDAttr(D, AL);
+    break;
   case ParsedAttr::AT_HLSLSV_GroupID:
     S.HLSL().handleSV_GroupIDAttr(D, AL);
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index df561fd143cac..a3d59164cc1c9 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -4102,24 +4102,28 @@ ExprResult Sema::ConvertMemberDefaultInitExpression(FieldDecl *FD,
 
 void Sema::ActOnFinishCXXInClassMemberInitializer(Decl *D,
                                                   SourceLocation InitLoc,
-                                                  Expr *InitExpr) {
+                                                  ExprResult InitExpr) {
   // Pop the notional constructor scope we created earlier.
   PopFunctionScopeInfo(nullptr, D);
 
-  FieldDecl *FD = dyn_cast<FieldDecl>(D);
-  assert((isa<MSPropertyDecl>(D) || FD->getInClassInitStyle() != ICIS_NoInit) &&
-         "must set init style when field is created");
-
-  if (!InitExpr) {
+  // Microsoft C++'s property declaration cannot have a default member
+  // initializer.
+  if (isa<MSPropertyDecl>(D)) {
     D->setInvalidDecl();
-    if (FD)
-      FD->removeInClassInitializer();
     return;
   }
 
-  if (DiagnoseUnexpandedParameterPack(InitExpr, UPPC_Initializer)) {
+  FieldDecl *FD = dyn_cast<FieldDecl>(D);
+  assert((FD && FD->getInClassInitStyle() != ICIS_NoInit) &&
+         "must set init style when field is created");
+
+  if (!InitExpr.isUsable() ||
+      DiagnoseUnexpandedParameterPack(InitExpr.get(), UPPC_Initializer)) {
     FD->setInvalidDecl();
-    FD->removeInClassInitializer();
+    ExprResult RecoveryInit =
+        CreateRecoveryExpr(InitLoc, InitLoc, {}, FD->getType());
+    if (RecoveryInit.isUsable())
+      FD->setInClassInitializer(RecoveryInit.get());
     return;
   }
 
@@ -7566,8 +7570,15 @@ void Sema::CheckExplicitlyDefaultedFunction(Scope *S, FunctionDecl *FD) {
     return;
   }
 
-  if (DefKind.isComparison())
-    UnusedPrivateFields.clear();
+  if (DefKind.isComparison()) {
+    auto PT = FD->getParamDecl(0)->getType();
+    if (const CXXRecordDecl *RD =
+            PT.getNonReferenceType()->getAsCXXRecordDecl()) {
+      for (FieldDecl *Field : RD->fields()) {
+        UnusedPrivateFields.remove(Field);
+      }
+    }
+  }
 
   if (DefKind.isSpecialMember()
           ? CheckExplicitlyDefaultedSpecialMember(cast<CXXMethodDecl>(FD),
@@ -9249,7 +9260,7 @@ struct SpecialMemberVisitor {
     if (auto *B = Subobj.dyn_cast<CXXBaseSpecifier*>())
       return B->getBaseTypeLoc();
     else
-      return Subobj.get<FieldDecl*>()->getLocation();
+      return cast<FieldDecl *>(Subobj)->getLocation();
   }
 
   enum BasesToVisit {
@@ -9400,7 +9411,7 @@ bool SpecialMemberDeletionInfo::shouldDeleteForSubobjectCall(
           << /*IsField*/ true << Field << DiagKind << IsDtorCallInCtor
           << /*IsObjCPtr*/ false;
     } else {
-      CXXBaseSpecifier *Base = Subobj.get<CXXBaseSpecifier*>();
+      CXXBaseSpecifier *Base = cast<CXXBaseSpecifier *>(Subobj);
       S.Diag(Base->getBeginLoc(),
              diag::note_deleted_special_member_class_subobject)
           << llvm::to_underlying(getEffectiveCSM()) << MD->getParent()
@@ -11497,8 +11508,8 @@ bool Sema::CheckDeductionGuideDeclarator(Declarator &D, QualType &R,
         // This could still instantiate to the right type, unless we know it
         // names the wrong class template.
         auto *TD = SpecifiedName.getAsTemplateDecl();
-        MightInstantiateToSpecialization = !(TD && isa<ClassTemplateDecl>(TD) &&
-                                             !TemplateMatches);
+        MightInstantiateToSpecialization =
+            !(TD && isa<ClassTemplateDecl>(TD) && !TemplateMatches);
       }
     } else if (!RetTy.hasQualifiers() && RetTy->isDependentType()) {
       MightInstantiateToSpecialization = true;
@@ -17518,7 +17529,7 @@ DeclResult Sema::ActOnTemplatedFriendTag(
     if (getDepthAndIndex(U).first >= FriendDeclDepth) {
       auto *ND = U.first.dyn_cast<NamedDecl *>();
       if (!ND)
-        ND = U.first.get<const TemplateTypeParmType *>()->getDecl();
+        ND = cast<const TemplateTypeParmType *>(U.first)->getDecl();
       Diag(U.second, diag::friend_template_decl_malformed_pack_expansion)
           << ND->getDeclName() << SourceRange(SS.getBeginLoc(), EllipsisLoc);
       return true;
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index ad1a02cf098b1..6e6174ba17c55 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1587,7 +1587,7 @@ void SemaObjC::actOnObjCTypeArgsOrProtocolQualifiers(
     if (auto *actualTypeDecl = typeDecl.dyn_cast<TypeDecl *>())
       type = Context.getTypeDeclType(actualTypeDecl);
     else
-      type = Context.getObjCInterfaceType(typeDecl.get<ObjCInterfaceDecl *>());
+      type = Context.getObjCInterfaceType(cast<ObjCInterfaceDecl *>(typeDecl));
     TypeSourceInfo *parsedTSInfo = Context.getTrivialTypeSourceInfo(type, loc);
     ParsedType parsedType = SemaRef.CreateParsedType(type, parsedTSInfo);
     DS.SetTypeSpecType(DeclSpec::TST_typename, loc, prevSpec, diagID,
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 36fffc0872a68..6cc817ab37fa5 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1205,15 +1205,16 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
     if (DTy.isNull() || DTy->isDependentType()) {
       CT = CT_Dependent;
     } else {
-      CT = canCalleeThrow(*this, DE, DE->getOperatorDelete());
-      if (const RecordType *RT = DTy->getAs<RecordType>()) {
-        const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-        const CXXDestructorDecl *DD = RD->getDestructor();
-        if (DD)
-          CT = mergeCanThrow(CT, canCalleeThrow(*this, DE, DD));
+      const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
+      CT = canCalleeThrow(*this, DE, OperatorDelete);
+      if (!OperatorDelete->isDestroyingOperatorDelete()) {
+        if (const auto *RD = DTy->getAsCXXRecordDecl()) {
+          if (const CXXDestructorDecl *DD = RD->getDestructor())
+            CT = mergeCanThrow(CT, canCalleeThrow(*this, DE, DD));
+        }
+        if (CT == CT_Can)
+          return CT;
       }
-      if (CT == CT_Can)
-        return CT;
     }
     return mergeCanThrow(CT, canSubStmtsThrow(*this, DE));
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 21f3c5c5ab6e1..84102007b2810 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -1052,6 +1052,9 @@ Sema::VarArgKind Sema::isValidVarArgType(const QualType &Ty) {
   if (Ty->isObjCObjectType())
     return VAK_Invalid;
 
+  if (getLangOpts().HLSL && Ty->getAs<HLSLAttributedResourceType>())
+    return VAK_Valid;
+
   if (getLangOpts().MSVCCompat)
     return VAK_MSVCUndefined;
 
@@ -4969,6 +4972,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
     return ExprError();
   }
 
+  CheckInvalidBuiltinCountedByRef(base, ArraySubscriptKind);
+
   // Handle any non-overload placeholder types in the base and index
   // expressions.  We can't handle overloads here because the other
   // operand might be an overloadable type, in which case the overload
@@ -5654,10 +5659,6 @@ static FieldDecl *FindFieldDeclInstantiationPattern(const ASTContext &Ctx,
 ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   assert(Field->hasInClassInitializer());
 
-  // If we might have already tried and failed to instantiate, don't try again.
-  if (Field->isInvalidDecl())
-    return ExprError();
-
   CXXThisScopeRAII This(*this, Field->getParent(), Qualifiers());
 
   auto *ParentRD = cast<CXXRecordDecl>(Field->getParent());
@@ -6015,7 +6016,7 @@ Sema::ConvertArgumentsForCall(CallExpr *Call, Expr *Fn,
   SmallVector<Expr *, 8> AllArgs;
   VariadicCallType CallType = getVariadicCallType(FDecl, Proto, Fn);
 
-  Invalid = GatherArgumentsForCall(Call->getBeginLoc(), FDecl, Proto, 0, Args,
+  Invalid = GatherArgumentsForCall(Call->getExprLoc(), FDecl, Proto, 0, Args,
                                    AllArgs, CallType);
   if (Invalid)
     return true;
@@ -6566,6 +6567,12 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
   if (CheckArgsForPlaceholders(ArgExprs))
     return ExprError();
 
+  // The result of __builtin_counted_by_ref cannot be used as a function
+  // argument. It allows leaking and modification of bounds safety information.
+  for (const Expr *Arg : ArgExprs)
+    if (CheckInvalidBuiltinCountedByRef(Arg, FunctionArgKind))
+      return ExprError();
+
   if (getLangOpts().CPlusPlus) {
     // If this is a pseudo-destructor expression, build the call immediately.
     if (isa<CXXPseudoDestructorExpr>(Fn)) {
@@ -9276,38 +9283,6 @@ Sema::CheckAssignmentConstraints(QualType LHSType, ExprResult &RHS,
   LHSType = Context.getCanonicalType(LHSType).getUnqualifiedType();
   RHSType = Context.getCanonicalType(RHSType).getUnqualifiedType();
 
-  // __builtin_counted_by_ref cannot be assigned to a variable, used in
-  // function call, or in a return.
-  auto FindBuiltinCountedByRefExpr = [&](Expr *E) -> CallExpr * {
-    struct BuiltinCountedByRefVisitor : DynamicRecursiveASTVisitor {
-      CallExpr *TheCall = nullptr;
-      bool VisitCallExpr(CallExpr *CE) override {
-        if (CE->getBuiltinCallee() == Builtin::BI__builtin_counted_by_ref) {
-          TheCall = CE;
-          return false;
-        }
-        return true;
-      }
-      bool
-      VisitUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *UE) override {
-        // A UnaryExprOrTypeTraitExpr---e.g. sizeof, __alignof, etc.---isn't
-        // the same as a CallExpr, so if we find a __builtin_counted_by_ref()
-        // call in one, ignore it.
-        return false;
-      }
-    } V;
-    V.TraverseStmt(E);
-    return V.TheCall;
-  };
-  static llvm::SmallPtrSet<CallExpr *, 4> Diagnosed;
-  if (auto *CE = FindBuiltinCountedByRefExpr(RHS.get());
-      CE && !Diagnosed.count(CE)) {
-    Diagnosed.insert(CE);
-    Diag(CE->getExprLoc(),
-         diag::err_builtin_counted_by_ref_cannot_leak_reference)
-        << CE->getSourceRange();
-  }
-
   // Common case: no conversion required.
   if (LHSType == RHSType) {
     Kind = CK_NoOp;
@@ -11931,14 +11906,23 @@ static void diagnoseTautologicalComparison(Sema &S, SourceLocation Loc,
     AlwaysEqual, // std::strong_ordering::equal from operator<=>
   };
 
+  // C++1a [array.comp]:
+  //   Equality and relational comparisons ([expr.eq], [expr.rel]) between two
+  //   operands of array type.
   // C++2a [depr.array.comp]:
   //   Equality and relational comparisons ([expr.eq], [expr.rel]) between two
   //   operands of array type are deprecated.
-  if (S.getLangOpts().CPlusPlus20 && LHSStripped->getType()->isArrayType() &&
+  if (S.getLangOpts().CPlusPlus && LHSStripped->getType()->isArrayType() &&
       RHSStripped->getType()->isArrayType()) {
-    S.Diag(Loc, diag::warn_depr_array_comparison)
-        << LHS->getSourceRange() << RHS->getSourceRange()
-        << LHSStripped->getType() << RHSStripped->getType();
+    auto IsDeprArrayComparionIgnored =
+        S.getDiagnostics().isIgnored(diag::warn_depr_array_comparison, Loc);
+    auto DiagID = S.getLangOpts().CPlusPlus26
+                      ? diag::warn_array_comparison_cxx26
+                  : !S.getLangOpts().CPlusPlus20 || IsDeprArrayComparionIgnored
+                      ? diag::warn_array_comparison
+                      : diag::warn_depr_array_comparison;
+    S.Diag(Loc, DiagID) << LHS->getSourceRange() << RHS->getSourceRange()
+                        << LHSStripped->getType() << RHSStripped->getType();
     // Carry on to produce the tautological comparison warning, if this
     // expression is potentially-evaluated, we can resolve the array to a
     // non-weak declaration, and so on.
@@ -13858,42 +13842,6 @@ QualType Sema::CheckAssignmentOperands(Expr *LHSExpr, ExprResult &RHS,
     ConvTy = CheckAssignmentConstraints(Loc, LHSType, RHSType);
   }
 
-  // __builtin_counted_by_ref can't be used in a binary expression or array
-  // subscript on the LHS.
-  int DiagOption = -1;
-  auto FindInvalidUseOfBoundsSafetyCounter = [&](Expr *E) -> CallExpr * {
-    struct BuiltinCountedByRefVisitor : DynamicRecursiveASTVisitor {
-      CallExpr *CE = nullptr;
-      bool InvalidUse = false;
-      int Option = -1;
-
-      bool VisitCallExpr(CallExpr *E) override {
-        if (E->getBuiltinCallee() == Builtin::BI__builtin_counted_by_ref) {
-          CE = E;
-          return false;
-        }
-        return true;
-      }
-
-      bool VisitArraySubscriptExpr(ArraySubscriptExpr *E) override {
-        InvalidUse = true;
-        Option = 0; // report 'array expression' in diagnostic.
-        return true;
-      }
-      bool VisitBinaryOperator(BinaryOperator *E) override {
-        InvalidUse = true;
-        Option = 1; // report 'binary expression' in diagnostic.
-        return true;
-      }
-    } V;
-    V.TraverseStmt(E);
-    DiagOption = V.Option;
-    return V.InvalidUse ? V.CE : nullptr;
-  };
-  if (auto *CE = FindInvalidUseOfBoundsSafetyCounter(LHSExpr))
-    Diag(CE->getExprLoc(), diag::err_builtin_counted_by_ref_invalid_lhs_use)
-        << DiagOption << CE->getSourceRange();
-
   if (DiagnoseAssignmentResult(ConvTy, Loc, LHSType, RHSType, RHS.get(),
                                AssignmentAction::Assigning))
     return QualType();
@@ -15354,6 +15302,12 @@ ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
   if (Kind == tok::TokenKind::slash)
     DetectPrecisionLossInComplexDivision(*this, TokLoc, LHSExpr);
 
+  BuiltinCountedByRefKind K =
+      BinaryOperator::isAssignmentOp(Opc) ? AssignmentKind : BinaryExprKind;
+
+  CheckInvalidBuiltinCountedByRef(LHSExpr, K);
+  CheckInvalidBuiltinCountedByRef(RHSExpr, K);
+
   return BuildBinOp(S, TokLoc, Opc, LHSExpr, RHSExpr);
 }
 
@@ -18559,7 +18513,11 @@ static bool isVariableAlreadyCapturedInScopeInfo(CapturingScopeInfo *CSI,
     // are mutable in the sense that user can change their value - they are
     // private instances of the captured declarations.
     const Capture &Cap = CSI->getCapture(Var);
-    if (Cap.isCopyCapture() &&
+    // C++ [expr.prim.lambda]p10:
+    //   The type of such a data member is [...] an lvalue reference to the
+    //   referenced function type if the entity is a reference to a function.
+    //   [...]
+    if (Cap.isCopyCapture() && !DeclRefType->isFunctionType() &&
         !(isa<LambdaScopeInfo>(CSI) &&
           !cast<LambdaScopeInfo>(CSI)->lambdaCaptureShouldBeConst()) &&
         !(isa<CapturedRegionScopeInfo>(CSI) &&
@@ -18869,7 +18827,12 @@ static bool captureInLambda(LambdaScopeInfo *LSI, ValueDecl *Var,
     //   parameter-declaration-clause is not followed by mutable.
     DeclRefType = CaptureType.getNonReferenceType();
     bool Const = LSI->lambdaCaptureShouldBeConst();
-    if (Const && !CaptureType->isReferenceType())
+    // C++ [expr.prim.lambda]p10:
+    //   The type of such a data member is [...] an lvalue reference to the
+    //   referenced function type if the entity is a reference to a function.
+    //   [...]
+    if (Const && !CaptureType->isReferenceType() &&
+        !DeclRefType->isFunctionType())
       DeclRefType.addConst();
   }
 
@@ -19407,7 +19370,7 @@ static ExprResult rebuildPotentialResultsAsNonOdrUsed(Sema &S, Expr *E,
       if (VD->getType()->isReferenceType())
         return true;
       if (auto *RD = VD->getType()->getAsCXXRecordDecl())
-        if (RD->hasMutableFields())
+        if (RD->hasDefinition() && RD->hasMutableFields())
           return true;
       if (!VD->isUsableInConstantExpressions(S.Context))
         return true;
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index ba02bb2b2de35..ac1018cb2c326 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -3756,7 +3756,8 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     } else if (!Pointee->isDependentType()) {
       // FIXME: This can result in errors if the definition was imported from a
       // module but is hidden.
-      if (!RequireCompleteType(StartLoc, Pointee,
+      if (Pointee->isEnumeralType() ||
+          !RequireCompleteType(StartLoc, Pointee,
                                LangOpts.CPlusPlus26
                                    ? diag::err_delete_incomplete
                                    : diag::warn_delete_incomplete,
@@ -4440,10 +4441,21 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     break;
 
   case ICK_HLSL_Array_RValue:
-    FromType = Context.getArrayParameterType(FromType);
-    From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                             /*BasePath=*/nullptr, CCK)
-               .get();
+    if (ToType->isArrayParameterType()) {
+      FromType = Context.getArrayParameterType(FromType);
+      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                               /*BasePath=*/nullptr, CCK)
+                 .get();
+    } else { // FromType must be ArrayParameterType
+      assert(FromType->isArrayParameterType() &&
+             "FromType must be ArrayParameterType in ICK_HLSL_Array_RValue \
+              if it is not ToType");
+      const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
+      FromType = APT->getConstantArrayType(Context);
+      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                               /*BasePath=*/nullptr, CCK)
+                 .get();
+    }
     break;
 
   case ICK_Function_To_Pointer:
diff --git a/clang/lib/Sema/SemaFunctionEffects.cpp b/clang/lib/Sema/SemaFunctionEffects.cpp
index c5c1e3fb41a2f..0d56a74b066e8 100644
--- a/clang/lib/Sema/SemaFunctionEffects.cpp
+++ b/clang/lib/Sema/SemaFunctionEffects.cpp
@@ -516,7 +516,7 @@ class Analyzer {
     CompleteFunctionAnalysis *completedAnalysisForDecl(const Decl *D) const {
       if (FuncAnalysisPtr AP = lookup(D);
           isa_and_nonnull<CompleteFunctionAnalysis *>(AP))
-        return AP.get<CompleteFunctionAnalysis *>();
+        return cast<CompleteFunctionAnalysis *>(AP);
       return nullptr;
     }
 
@@ -528,12 +528,10 @@ class Analyzer {
         OS << item.first << " " << CI.getNameForDiagnostic(SemaRef) << " : ";
         if (AP.isNull()) {
           OS << "null\n";
-        } else if (isa<CompleteFunctionAnalysis *>(AP)) {
-          auto *CFA = AP.get<CompleteFunctionAnalysis *>();
+        } else if (auto *CFA = dyn_cast<CompleteFunctionAnalysis *>(AP)) {
           OS << CFA << " ";
           CFA->dump(OS);
-        } else if (isa<PendingFunctionAnalysis *>(AP)) {
-          auto *PFA = AP.get<PendingFunctionAnalysis *>();
+        } else if (auto *PFA = dyn_cast<PendingFunctionAnalysis *>(AP)) {
           OS << PFA << " ";
           PFA->dump(SemaRef, OS);
         } else
@@ -1376,10 +1374,10 @@ class Analyzer {
 Analyzer::AnalysisMap::~AnalysisMap() {
   for (const auto &Item : *this) {
     FuncAnalysisPtr AP = Item.second;
-    if (isa<PendingFunctionAnalysis *>(AP))
-      delete AP.get<PendingFunctionAnalysis *>();
+    if (auto *PFA = dyn_cast<PendingFunctionAnalysis *>(AP))
+      delete PFA;
     else
-      delete AP.get<CompleteFunctionAnalysis *>();
+      delete cast<CompleteFunctionAnalysis *>(AP);
   }
 }
 
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 8b2f24a8e4be0..600c800029fd0 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -434,6 +434,7 @@ void SemaHLSL::CheckSemanticAnnotation(
   switch (AnnotationAttr->getKind()) {
   case attr::HLSLSV_DispatchThreadID:
   case attr::HLSLSV_GroupIndex:
+  case attr::HLSLSV_GroupThreadID:
   case attr::HLSLSV_GroupID:
     if (ST == llvm::Triple::Compute)
       return;
@@ -787,6 +788,15 @@ void SemaHLSL::handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL) {
                  HLSLSV_DispatchThreadIDAttr(getASTContext(), AL));
 }
 
+void SemaHLSL::handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL) {
+  auto *VD = cast<ValueDecl>(D);
+  if (!diagnoseInputIDType(VD->getType(), AL))
+    return;
+
+  D->addAttr(::new (getASTContext())
+                 HLSLSV_GroupThreadIDAttr(getASTContext(), AL));
+}
+
 void SemaHLSL::handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL) {
   auto *VD = cast<ValueDecl>(D);
   if (!diagnoseInputIDType(VD->getType(), AL))
@@ -1908,7 +1918,7 @@ static bool CheckResourceHandle(
   const HLSLAttributedResourceType *ResTy =
       ArgType.getTypePtr()->getAs<HLSLAttributedResourceType>();
   if (!ResTy) {
-    S->Diag(TheCall->getArg(0)->getBeginLoc(),
+    S->Diag(TheCall->getArg(ArgIndex)->getBeginLoc(),
             diag::err_typecheck_expect_hlsl_resource)
         << ArgType;
     return true;
@@ -1926,6 +1936,22 @@ static bool CheckResourceHandle(
 // returning an ExprError
 bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   switch (BuiltinID) {
+  case Builtin::BI__builtin_hlsl_resource_getpointer: {
+    if (SemaRef.checkArgCount(TheCall, 2) ||
+        CheckResourceHandle(&SemaRef, TheCall, 0) ||
+        CheckArgTypeMatches(&SemaRef, TheCall->getArg(1),
+                            SemaRef.getASTContext().UnsignedIntTy))
+      return true;
+
+    auto *ResourceTy =
+        TheCall->getArg(0)->getType()->castAs<HLSLAttributedResourceType>();
+    QualType ContainedTy = ResourceTy->getContainedType();
+    // TODO: Map to an hlsl_device address space.
+    TheCall->setType(getASTContext().getPointerType(ContainedTy));
+    TheCall->setValueKind(VK_LValue);
+
+    break;
+  }
   case Builtin::BI__builtin_hlsl_all:
   case Builtin::BI__builtin_hlsl_any: {
     if (SemaRef.checkArgCount(TheCall, 1))
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 9f47deeffc15d..5954c9d37d6a3 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -745,6 +745,7 @@ void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field,
     if (Field->hasInClassInitializer()) {
       if (VerifyOnly)
         return;
+
       ExprResult DIE;
       {
         // Enter a default initializer rebuild context, then we can support
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index d146edeabab74..62c3e778ab178 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -598,9 +598,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitTileClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // TODO OpenACC: Remove this when we get combined construct impl for this.
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-    return isNotImplemented();
 
   // Duplicates here are not really sensible.  We could possible permit
   // multiples if they all had the same value, but there isn't really a good
@@ -697,14 +694,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitSelfClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  // TODO OpenACC:  Remove this check when we have combined constructs for this
-  // clause.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-    return isNotImplemented();
-
   // There is no prose in the standard that says duplicates aren't allowed,
   // but this diagnostic is present in other compilers, as well as makes
   // sense.
@@ -730,10 +719,35 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
         << /*NoArgs=*/1 << Clause.getDirectiveKind() << MaxArgs
         << Clause.getIntExprs().size();
 
+  // OpenACC 3.3 Section 2.9.11: A reduction clause may not appear on a loop
+  // directive that has a gang clause and is within a compute construct that has
+  // a num_gangs clause with more than one explicit argument.
+  if (Clause.getIntExprs().size() > 1 &&
+      isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+    auto *GangClauseItr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
+    auto *ReductionClauseItr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
+
+    if (GangClauseItr != ExistingClauses.end() &&
+        ReductionClauseItr != ExistingClauses.end()) {
+      SemaRef.Diag(Clause.getBeginLoc(),
+                   diag::err_acc_gang_reduction_numgangs_conflict)
+          << OpenACCClauseKind::Reduction << OpenACCClauseKind::Gang
+          << Clause.getDirectiveKind() << /*is on combined directive=*/1;
+      SemaRef.Diag((*ReductionClauseItr)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      SemaRef.Diag((*GangClauseItr)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
+  }
+
   // OpenACC 3.3 Section 2.5.4:
   // A reduction clause may not appear on a parallel construct with a
   // num_gangs clause that has more than one argument.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel &&
+  if ((Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel ||
+       Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop) &&
       Clause.getIntExprs().size() > 1) {
     auto *Parallel =
         llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
@@ -741,12 +755,33 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
     if (Parallel != ExistingClauses.end()) {
       SemaRef.Diag(Clause.getBeginLoc(),
                    diag::err_acc_reduction_num_gangs_conflict)
-          << Clause.getIntExprs().size();
+          << /*>1 arg in first loc=*/1 << Clause.getClauseKind()
+          << Clause.getDirectiveKind() << OpenACCClauseKind::Reduction;
       SemaRef.Diag((*Parallel)->getBeginLoc(),
                    diag::note_acc_previous_clause_here);
       return nullptr;
     }
   }
+
+  // OpenACC 3.3 Section 2.9.2:
+  // An argument with no keyword or with the 'num' keyword is allowed only when
+  // the 'num_gangs' does not appear on the 'kernel' construct.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::KernelsLoop) {
+    auto GangClauses = llvm::make_filter_range(
+        ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
+
+    for (auto *GC : GangClauses) {
+      if (cast<OpenACCGangClause>(GC)->hasExprOfKind(OpenACCGangKind::Num)) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_num_arg_conflict_reverse)
+            << OpenACCClauseKind::NumGangs << OpenACCClauseKind::Gang
+            << /*Num argument*/ 1;
+        SemaRef.Diag(GC->getBeginLoc(), diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
+    }
+  }
+
   return OpenACCNumGangsClause::Create(
       Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs(),
       Clause.getEndLoc());
@@ -754,19 +789,31 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  // TODO: OpenACC: Remove when we get combined constructs.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-    return isNotImplemented();
-
   // There is no prose in the standard that says duplicates aren't allowed,
   // but this diagnostic is present in other compilers, as well as makes
   // sense.
   if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
     return nullptr;
 
+  // OpenACC 3.3 Section 2.9.2:
+  // An argument is allowed only when the 'num_workers' does not appear on the
+  // kernels construct.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::KernelsLoop) {
+    auto WorkerClauses = llvm::make_filter_range(
+        ExistingClauses, llvm::IsaPred<OpenACCWorkerClause>);
+
+    for (auto *WC : WorkerClauses) {
+      if (cast<OpenACCWorkerClause>(WC)->hasIntExpr()) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_num_arg_conflict_reverse)
+            << OpenACCClauseKind::NumWorkers << OpenACCClauseKind::Worker
+            << /*num argument*/ 0;
+        SemaRef.Diag(WC->getBeginLoc(), diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
+    }
+  }
+
   assert(Clause.getIntExprs().size() == 1 &&
          "Invalid number of expressions for NumWorkers");
   return OpenACCNumWorkersClause::Create(
@@ -776,19 +823,31 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorLengthClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  // TODO: OpenACC: Remove when we get combined constructs.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-    return isNotImplemented();
-
   // There is no prose in the standard that says duplicates aren't allowed,
   // but this diagnostic is present in other compilers, as well as makes
   // sense.
   if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
     return nullptr;
 
+  // OpenACC 3.3 Section 2.9.4:
+  // An argument is allowed only when the 'vector_length' does not appear on the
+  // 'kernels' construct.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::KernelsLoop) {
+    auto VectorClauses = llvm::make_filter_range(
+        ExistingClauses, llvm::IsaPred<OpenACCVectorClause>);
+
+    for (auto *VC : VectorClauses) {
+      if (cast<OpenACCVectorClause>(VC)->hasIntExpr()) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_num_arg_conflict_reverse)
+            << OpenACCClauseKind::VectorLength << OpenACCClauseKind::Vector
+            << /*num argument*/ 0;
+        SemaRef.Diag(VC->getBeginLoc(), diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
+    }
+  }
+
   assert(Clause.getIntExprs().size() == 1 &&
          "Invalid number of expressions for NumWorkers");
   return OpenACCVectorLengthClause::Create(
@@ -843,10 +902,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitFirstPrivateClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitNoCreateClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
   // ActOnVar ensured that everything is a valid variable reference, so there
   // really isn't anything to do here. GCC does some duplicate-finding, though
@@ -876,10 +936,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitPresentClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
   // ActOnVar ensured that everything is a valid variable reference, so there
   // really isn't anything to do here. GCC does some duplicate-finding, though
@@ -892,10 +953,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyInClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
   // ActOnVar ensured that everything is a valid variable reference, so there
   // really isn't anything to do here. GCC does some duplicate-finding, though
@@ -908,10 +970,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyInClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyOutClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
   // ActOnVar ensured that everything is a valid variable reference, so there
   // really isn't anything to do here. GCC does some duplicate-finding, though
@@ -924,10 +987,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyOutClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCreateClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
   // ActOnVar ensured that everything is a valid variable reference, so there
   // really isn't anything to do here. GCC does some duplicate-finding, though
@@ -940,10 +1004,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCreateClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitAttachClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   // ActOnVar ensured that everything is a valid variable reference, but we
@@ -961,10 +1026,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitAttachClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDevicePtrClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   // ActOnVar ensured that everything is a valid variable reference, but we
@@ -983,10 +1049,11 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDevicePtrClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+  // Restrictions only properly implemented on 'compute'/'combined' constructs,
+  // and 'compute'/'combined' constructs are the only construct that can do
+  // anything with this yet, so skip/treat as unimplemented in this case.
+  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   return OpenACCWaitClause::Create(
@@ -1049,90 +1116,237 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitIndependentClause(
                                           Clause.getEndLoc());
 }
 
+ExprResult CheckGangStaticExpr(SemaOpenACC &S, Expr *E) {
+  if (isa<OpenACCAsteriskSizeExpr>(E))
+    return E;
+  return S.ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
+                        E->getBeginLoc(), E);
+}
+
+bool IsOrphanLoop(OpenACCDirectiveKind DK, OpenACCDirectiveKind AssocKind) {
+  return DK == OpenACCDirectiveKind::Loop &&
+         AssocKind == OpenACCDirectiveKind::Invalid;
+}
+
+bool HasAssocKind(OpenACCDirectiveKind DK, OpenACCDirectiveKind AssocKind) {
+  return DK == OpenACCDirectiveKind::Loop &&
+         AssocKind != OpenACCDirectiveKind::Invalid;
+}
+
+ExprResult DiagIntArgInvalid(SemaOpenACC &S, Expr *E, OpenACCGangKind GK,
+                             OpenACCClauseKind CK, OpenACCDirectiveKind DK,
+                             OpenACCDirectiveKind AssocKind) {
+  S.Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
+      << GK << CK << IsOrphanLoop(DK, AssocKind) << DK
+      << HasAssocKind(DK, AssocKind) << AssocKind;
+  return ExprError();
+}
+ExprResult DiagIntArgInvalid(SemaOpenACC &S, Expr *E, StringRef TagKind,
+                             OpenACCClauseKind CK, OpenACCDirectiveKind DK,
+                             OpenACCDirectiveKind AssocKind) {
+  S.Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
+      << TagKind << CK << IsOrphanLoop(DK, AssocKind) << DK
+      << HasAssocKind(DK, AssocKind) << AssocKind;
+  return ExprError();
+}
+
+ExprResult CheckGangParallelExpr(SemaOpenACC &S, OpenACCDirectiveKind DK,
+                                 OpenACCDirectiveKind AssocKind,
+                                 OpenACCGangKind GK, Expr *E) {
+  switch (GK) {
+  case OpenACCGangKind::Static:
+    return CheckGangStaticExpr(S, E);
+  case OpenACCGangKind::Num:
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
+    // construct, or an orphaned loop construct, the gang clause behaves as
+    // follows. ... The num argument is not allowed.
+    return DiagIntArgInvalid(S, E, GK, OpenACCClauseKind::Gang, DK, AssocKind);
+  case OpenACCGangKind::Dim: {
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
+    // construct, or an orphaned loop construct, the gang clause behaves as
+    // follows. ... The dim argument must be a constant positive integer value
+    // 1, 2, or 3.
+    if (!E)
+      return ExprError();
+    ExprResult Res =
+        S.ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
+                       E->getBeginLoc(), E);
+
+    if (!Res.isUsable())
+      return Res;
+
+    if (Res.get()->isInstantiationDependent())
+      return Res;
+
+    std::optional<llvm::APSInt> ICE =
+        Res.get()->getIntegerConstantExpr(S.getASTContext());
+
+    if (!ICE || *ICE <= 0 || ICE > 3) {
+      S.Diag(Res.get()->getBeginLoc(), diag::err_acc_gang_dim_value)
+          << ICE.has_value() << ICE.value_or(llvm::APSInt{}).getExtValue();
+      return ExprError();
+    }
+
+    return ExprResult{
+        ConstantExpr::Create(S.getASTContext(), Res.get(), APValue{*ICE})};
+  }
+  }
+  llvm_unreachable("Unknown gang kind in gang parallel check");
+}
+
+ExprResult CheckGangKernelsExpr(SemaOpenACC &S,
+                                ArrayRef<const OpenACCClause *> ExistingClauses,
+                                OpenACCDirectiveKind DK,
+                                OpenACCDirectiveKind AssocKind,
+                                OpenACCGangKind GK, Expr *E) {
+  switch (GK) {
+  // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+  // construct, the gang clause behaves as follows. ... The dim argument is
+  // not allowed.
+  case OpenACCGangKind::Dim:
+    return DiagIntArgInvalid(S, E, GK, OpenACCClauseKind::Gang, DK, AssocKind);
+  case OpenACCGangKind::Num: {
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... An argument with no
+    // keyword or with num keyword is only allowed when num_gangs does not
+    // appear on the kernels construct. ... The region of a loop with the gang
+    // clause may not contain another loop with a gang clause unless within a
+    // nested compute region.
+
+    // If this is a 'combined' construct, search the list of existing clauses.
+    // Else we need to search the containing 'kernel'.
+    auto Collection = isOpenACCCombinedDirectiveKind(DK)
+                          ? ExistingClauses
+                          : S.getActiveComputeConstructInfo().Clauses;
+
+    const auto *Itr =
+        llvm::find_if(Collection, llvm::IsaPred<OpenACCNumGangsClause>);
+
+    if (Itr != Collection.end()) {
+      S.Diag(E->getBeginLoc(), diag::err_acc_num_arg_conflict)
+          << "num" << OpenACCClauseKind::Gang << DK
+          << HasAssocKind(DK, AssocKind) << AssocKind
+          << OpenACCClauseKind::NumGangs;
+
+      S.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+      return ExprError();
+    }
+    return ExprResult{E};
+  }
+  case OpenACCGangKind::Static:
+    return CheckGangStaticExpr(S, E);
+    return ExprError();
+  }
+  llvm_unreachable("Unknown gang kind in gang kernels check");
+}
+
+ExprResult CheckGangSerialExpr(SemaOpenACC &S, OpenACCDirectiveKind DK,
+                               OpenACCDirectiveKind AssocKind,
+                               OpenACCGangKind GK, Expr *E) {
+  switch (GK) {
+  // 'dim' and 'num' don't really make sense on serial, and GCC rejects them
+  // too, so we disallow them too.
+  case OpenACCGangKind::Dim:
+  case OpenACCGangKind::Num:
+    return DiagIntArgInvalid(S, E, GK, OpenACCClauseKind::Gang, DK, AssocKind);
+  case OpenACCGangKind::Static:
+    return CheckGangStaticExpr(S, E);
+  }
+  llvm_unreachable("Unknown gang kind in gang serial check");
+}
+
 OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
   if (DiagIfSeqClause(Clause))
     return nullptr;
-  // Restrictions only properly implemented on 'loop' constructs, and it is
-  // the only construct that can do anything with this, so skip/treat as
-  // unimplemented for the combined constructs.
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+
+  // Restrictions only properly implemented on 'loop'/'combined' constructs, and
+  // it is the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the routine constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   Expr *IntExpr =
       Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr;
   if (IntExpr) {
-    switch (SemaRef.getActiveComputeConstructInfo().Kind) {
-    case OpenACCDirectiveKind::Invalid:
-    case OpenACCDirectiveKind::Parallel:
-      // No restriction on when 'parallel' can contain an argument.
-      break;
-    case OpenACCDirectiveKind::Serial:
-      // GCC disallows this, and there is no real good reason for us to permit
-      // it, so disallow until we come up with a use case that makes sense.
-      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Vector << "num" << /*serial=*/3;
-      IntExpr = nullptr;
-      break;
-    case OpenACCDirectiveKind::Kernels: {
-      const auto *Itr =
-          llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
-                        llvm::IsaPred<OpenACCVectorLengthClause>);
-      if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
-        SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
-            << OpenACCClauseKind::Vector << /*vector_length=*/2;
-        SemaRef.Diag((*Itr)->getBeginLoc(),
-                     diag::note_acc_previous_clause_here);
-
+    if (!isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+      switch (SemaRef.getActiveComputeConstructInfo().Kind) {
+      case OpenACCDirectiveKind::Invalid:
+      case OpenACCDirectiveKind::Parallel:
+        // No restriction on when 'parallel' can contain an argument.
+        break;
+      case OpenACCDirectiveKind::Serial:
+        // GCC disallows this, and there is no real good reason for us to permit
+        // it, so disallow until we come up with a use case that makes sense.
+        DiagIntArgInvalid(SemaRef, IntExpr, "length", OpenACCClauseKind::Vector,
+                          Clause.getDirectiveKind(),
+                          SemaRef.getActiveComputeConstructInfo().Kind);
         IntExpr = nullptr;
+        break;
+      case OpenACCDirectiveKind::Kernels: {
+        const auto *Itr =
+            llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
+                          llvm::IsaPred<OpenACCVectorLengthClause>);
+        if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
+          SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
+              << "length" << OpenACCClauseKind::Vector
+              << Clause.getDirectiveKind()
+              << HasAssocKind(Clause.getDirectiveKind(),
+                              SemaRef.getActiveComputeConstructInfo().Kind)
+              << SemaRef.getActiveComputeConstructInfo().Kind
+              << OpenACCClauseKind::VectorLength;
+          SemaRef.Diag((*Itr)->getBeginLoc(),
+                       diag::note_acc_previous_clause_here);
+
+          IntExpr = nullptr;
+        }
+        break;
+      }
+      default:
+        llvm_unreachable("Non compute construct in active compute construct");
+      }
+    } else {
+      if (Clause.getDirectiveKind() == OpenACCDirectiveKind::SerialLoop) {
+        DiagIntArgInvalid(SemaRef, IntExpr, "length", OpenACCClauseKind::Vector,
+                          Clause.getDirectiveKind(),
+                          SemaRef.getActiveComputeConstructInfo().Kind);
+        IntExpr = nullptr;
+      } else if (Clause.getDirectiveKind() ==
+                 OpenACCDirectiveKind::KernelsLoop) {
+        const auto *Itr = llvm::find_if(
+            ExistingClauses, llvm::IsaPred<OpenACCVectorLengthClause>);
+        if (Itr != ExistingClauses.end()) {
+          SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
+              << "length" << OpenACCClauseKind::Vector
+              << Clause.getDirectiveKind()
+              << HasAssocKind(Clause.getDirectiveKind(),
+                              SemaRef.getActiveComputeConstructInfo().Kind)
+              << SemaRef.getActiveComputeConstructInfo().Kind
+              << OpenACCClauseKind::VectorLength;
+          SemaRef.Diag((*Itr)->getBeginLoc(),
+                       diag::note_acc_previous_clause_here);
+
+          IntExpr = nullptr;
+        }
       }
-      break;
-    }
-    default:
-      llvm_unreachable("Non compute construct in active compute construct");
     }
   }
 
-  // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
-  // construct, the gang clause behaves as follows. ... The region of a loop
-  // with a gang clause may not contain another loop with a gang clause unless
-  // within a nested compute region.
-  if (SemaRef.LoopGangClauseOnKernelLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Vector << OpenACCClauseKind::Gang
-        << /*skip kernels construct info*/ 0;
-    SemaRef.Diag(SemaRef.LoopGangClauseOnKernelLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
-
-  // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
-  // contain a loop with a gang or worker clause unless within a nested compute
-  // region.
-  if (SemaRef.LoopWorkerClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Vector << OpenACCClauseKind::Worker
-        << /*skip kernels construct info*/ 0;
-    SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
-  // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
-  // contain a loop with a gang, worker, or vector clause unless within a nested
-  // compute region.
-  if (SemaRef.LoopVectorClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Vector << OpenACCClauseKind::Vector
-        << /*skip kernels construct info*/ 0;
-    SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
+  if (!isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+    // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
+    // contain a loop with a gang, worker, or vector clause unless within a
+    // nested compute region.
+    if (SemaRef.LoopVectorClauseLoc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Vector << OpenACCClauseKind::Vector
+          << /*skip kernels construct info*/ 0;
+      SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
   }
 
   return OpenACCVectorClause::Create(Ctx, Clause.getBeginLoc(),
@@ -1145,77 +1359,107 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitWorkerClause(
   if (DiagIfSeqClause(Clause))
     return nullptr;
 
-  // Restrictions only properly implemented on 'loop' constructs, and it is
-  // the only construct that can do anything with this, so skip/treat as
-  // unimplemented for the combined constructs.
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+  // Restrictions only properly implemented on 'loop'/'combined' constructs, and
+  // it is the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the routine constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   Expr *IntExpr =
       Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr;
 
   if (IntExpr) {
-    switch (SemaRef.getActiveComputeConstructInfo().Kind) {
-    case OpenACCDirectiveKind::Invalid:
-      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Worker << "num" << /*orphan=*/0;
-      IntExpr = nullptr;
-      break;
-    case OpenACCDirectiveKind::Parallel:
-      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Worker << "num" << /*parallel=*/1;
-      IntExpr = nullptr;
-      break;
-    case OpenACCDirectiveKind::Serial:
-      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Worker << "num" << /*serial=*/3;
-      IntExpr = nullptr;
-      break;
-    case OpenACCDirectiveKind::Kernels: {
-      const auto *Itr =
-          llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
-                        llvm::IsaPred<OpenACCNumWorkersClause>);
-      if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
-        SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
-            << OpenACCClauseKind::Worker << /*num_workers=*/1;
-        SemaRef.Diag((*Itr)->getBeginLoc(),
-                     diag::note_acc_previous_clause_here);
-
+    if (!isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+      switch (SemaRef.getActiveComputeConstructInfo().Kind) {
+      case OpenACCDirectiveKind::Invalid:
+      case OpenACCDirectiveKind::ParallelLoop:
+      case OpenACCDirectiveKind::SerialLoop:
+      case OpenACCDirectiveKind::Parallel:
+      case OpenACCDirectiveKind::Serial:
+        DiagIntArgInvalid(SemaRef, IntExpr, OpenACCGangKind::Num,
+                          OpenACCClauseKind::Worker, Clause.getDirectiveKind(),
+                          SemaRef.getActiveComputeConstructInfo().Kind);
         IntExpr = nullptr;
+        break;
+      case OpenACCDirectiveKind::KernelsLoop:
+      case OpenACCDirectiveKind::Kernels: {
+        const auto *Itr =
+            llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
+                          llvm::IsaPred<OpenACCNumWorkersClause>);
+        if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
+          SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
+              << "num" << OpenACCClauseKind::Worker << Clause.getDirectiveKind()
+              << HasAssocKind(Clause.getDirectiveKind(),
+                              SemaRef.getActiveComputeConstructInfo().Kind)
+              << SemaRef.getActiveComputeConstructInfo().Kind
+              << OpenACCClauseKind::NumWorkers;
+          SemaRef.Diag((*Itr)->getBeginLoc(),
+                       diag::note_acc_previous_clause_here);
+
+          IntExpr = nullptr;
+        }
+        break;
+      }
+      default:
+        llvm_unreachable("Non compute construct in active compute construct");
+      }
+    } else {
+      if (Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop ||
+          Clause.getDirectiveKind() == OpenACCDirectiveKind::SerialLoop) {
+        DiagIntArgInvalid(SemaRef, IntExpr, OpenACCGangKind::Num,
+                          OpenACCClauseKind::Worker, Clause.getDirectiveKind(),
+                          SemaRef.getActiveComputeConstructInfo().Kind);
+        IntExpr = nullptr;
+      } else {
+        assert(Clause.getDirectiveKind() == OpenACCDirectiveKind::KernelsLoop &&
+               "Unknown combined directive kind?");
+        const auto *Itr = llvm::find_if(ExistingClauses,
+                                        llvm::IsaPred<OpenACCNumWorkersClause>);
+        if (Itr != ExistingClauses.end()) {
+          SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
+              << "num" << OpenACCClauseKind::Worker << Clause.getDirectiveKind()
+              << HasAssocKind(Clause.getDirectiveKind(),
+                              SemaRef.getActiveComputeConstructInfo().Kind)
+              << SemaRef.getActiveComputeConstructInfo().Kind
+              << OpenACCClauseKind::NumWorkers;
+          SemaRef.Diag((*Itr)->getBeginLoc(),
+                       diag::note_acc_previous_clause_here);
+
+          IntExpr = nullptr;
+        }
       }
-      break;
-    }
-    default:
-      llvm_unreachable("Non compute construct in active compute construct");
     }
   }
 
-  // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
-  // contain a loop with a gang or worker clause unless within a nested compute
-  // region.
-  if (SemaRef.LoopWorkerClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Worker << OpenACCClauseKind::Worker
-        << /*skip kernels construct info*/ 0;
-    SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
+  if (!isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+    // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
+    // contain a loop with a gang or worker clause unless within a nested
+    // compute region.
+    if (SemaRef.LoopWorkerClauseLoc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Worker << OpenACCClauseKind::Worker
+          << /*skip kernels construct info*/ 0;
+      SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
 
-  // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
-  // contain a loop with a gang, worker, or vector clause unless within a nested
-  // compute region.
-  if (SemaRef.LoopVectorClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Worker << OpenACCClauseKind::Vector
-        << /*skip kernels construct info*/ 0;
-    SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
+    // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
+    // contain a loop with a gang, worker, or vector clause unless within a
+    // nested compute region.
+    if (SemaRef.LoopVectorClauseLoc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Worker << OpenACCClauseKind::Vector
+          << /*skip kernels construct info*/ 0;
+      SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
   }
 
   return OpenACCWorkerClause::Create(Ctx, Clause.getBeginLoc(),
@@ -1231,36 +1475,43 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitGangClause(
   // Restrictions only properly implemented on 'loop' constructs, and it is
   // the only construct that can do anything with this, so skip/treat as
   // unimplemented for the combined constructs.
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop &&
+      !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
     return isNotImplemented();
 
   // OpenACC 3.3 Section 2.9.11: A reduction clause may not appear on a loop
   // directive that has a gang clause and is within a compute construct that has
   // a num_gangs clause with more than one explicit argument.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Loop &&
-      SemaRef.getActiveComputeConstructInfo().Kind !=
-          OpenACCDirectiveKind::Invalid) {
+  if ((Clause.getDirectiveKind() == OpenACCDirectiveKind::Loop &&
+       SemaRef.getActiveComputeConstructInfo().Kind !=
+           OpenACCDirectiveKind::Invalid) ||
+      isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
     // num_gangs clause on the active compute construct.
-    auto *NumGangsClauseItr =
-        llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
-                      llvm::IsaPred<OpenACCNumGangsClause>);
-
-    auto *ReductionClauseItr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
-
-    if (ReductionClauseItr != ExistingClauses.end() &&
-        NumGangsClauseItr !=
-            SemaRef.getActiveComputeConstructInfo().Clauses.end() &&
+    auto ActiveComputeConstructContainer =
+        isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())
+            ? ExistingClauses
+            : SemaRef.getActiveComputeConstructInfo().Clauses;
+    auto *NumGangsClauseItr = llvm::find_if(
+        ActiveComputeConstructContainer, llvm::IsaPred<OpenACCNumGangsClause>);
+
+    if (NumGangsClauseItr != ActiveComputeConstructContainer.end() &&
         cast<OpenACCNumGangsClause>(*NumGangsClauseItr)->getIntExprs().size() >
             1) {
-      SemaRef.Diag(Clause.getBeginLoc(),
-                   diag::err_acc_gang_reduction_numgangs_conflict)
-          << OpenACCClauseKind::Gang << OpenACCClauseKind::Reduction;
-      SemaRef.Diag((*ReductionClauseItr)->getBeginLoc(),
-                   diag::note_acc_previous_clause_here);
-      SemaRef.Diag((*NumGangsClauseItr)->getBeginLoc(),
-                   diag::note_acc_previous_clause_here);
-      return nullptr;
+      auto *ReductionClauseItr =
+          llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCReductionClause>);
+
+      if (ReductionClauseItr != ExistingClauses.end()) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_gang_reduction_numgangs_conflict)
+            << OpenACCClauseKind::Gang << OpenACCClauseKind::Reduction
+            << Clause.getDirectiveKind()
+            << isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind());
+        SemaRef.Diag((*ReductionClauseItr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+        SemaRef.Diag((*NumGangsClauseItr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
     }
   }
 
@@ -1273,31 +1524,13 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitGangClause(
 
   for (unsigned I = 0; I < Clause.getIntExprs().size(); ++I) {
     OpenACCGangKind GK = Clause.getGangKinds()[I];
-    ExprResult ER = SemaRef.CheckGangExpr(GK, Clause.getIntExprs()[I]);
+    ExprResult ER =
+        SemaRef.CheckGangExpr(ExistingClauses, Clause.getDirectiveKind(), GK,
+                              Clause.getIntExprs()[I]);
 
     if (!ER.isUsable())
       continue;
 
-    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
-    // construct, the gang clause behaves as follows. ... An argument with no
-    // keyword or with num keyword is only allowed when num_gangs does not
-    // appear on the kernels construct.
-    if (SemaRef.getActiveComputeConstructInfo().Kind ==
-            OpenACCDirectiveKind::Kernels &&
-        GK == OpenACCGangKind::Num) {
-      const auto *Itr =
-          llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
-                        llvm::IsaPred<OpenACCNumGangsClause>);
-
-      if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
-        SemaRef.Diag(ER.get()->getBeginLoc(), diag::err_acc_num_arg_conflict)
-            << OpenACCClauseKind::Gang << /*num_gangs=*/0;
-        SemaRef.Diag((*Itr)->getBeginLoc(),
-                     diag::note_acc_previous_clause_here);
-        continue;
-      }
-    }
-
     // OpenACC 3.3 2.9: 'gang-arg-list' may have at most one num, one dim, and
     // one static argument.
     if (ExistingElemLoc[static_cast<unsigned>(GK)].isValid()) {
@@ -1313,52 +1546,55 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitGangClause(
     IntExprs.push_back(ER.get());
   }
 
-  // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
-  // construct, the gang clause behaves as follows. ... The region of a loop
-  // with a gang clause may not contain another loop with a gang clause unless
-  // within a nested compute region.
-  if (SemaRef.LoopGangClauseOnKernelLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Gang << OpenACCClauseKind::Gang
-        << /*kernels construct info*/ 1;
-    SemaRef.Diag(SemaRef.LoopGangClauseOnKernelLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
+  if (!isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... The region of a loop
+    // with a gang clause may not contain another loop with a gang clause unless
+    // within a nested compute region.
+    if (SemaRef.LoopGangClauseOnKernel.Loc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Gang << OpenACCClauseKind::Gang
+          << /*kernels construct info*/ 1
+          << SemaRef.LoopGangClauseOnKernel.DirKind;
+      SemaRef.Diag(SemaRef.LoopGangClauseOnKernel.Loc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
 
-  // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
-  // contain a loop with a gang or worker clause unless within a nested compute
-  // region.
-  if (SemaRef.LoopWorkerClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Gang << OpenACCClauseKind::Worker
-        << /*kernels construct info*/ 1;
-    SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
+    // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
+    // contain a loop with a gang or worker clause unless within a nested
+    // compute region.
+    if (SemaRef.LoopWorkerClauseLoc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Gang << OpenACCClauseKind::Worker
+          << /*!kernels construct info*/ 0;
+      SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
 
-  // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
-  // contain a loop with a gang, worker, or vector clause unless within a nested
-  // compute region.
-  if (SemaRef.LoopVectorClauseLoc.isValid()) {
-    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
-    // one of these until we get to the end of the construct.
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
-        << OpenACCClauseKind::Gang << OpenACCClauseKind::Vector
-        << /*kernels construct info*/ 1;
-    SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
-                 diag::note_acc_previous_clause_here);
-    return nullptr;
+    // OpenACC 3.3 2.9.4: The region of a loop with a 'vector' clause may not
+    // contain a loop with a gang, worker, or vector clause unless within a
+    // nested compute region.
+    if (SemaRef.LoopVectorClauseLoc.isValid()) {
+      // This handles the 'inner loop' diagnostic, but we cannot set that we're
+      // on one of these until we get to the end of the construct.
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+          << OpenACCClauseKind::Gang << OpenACCClauseKind::Vector
+          << /*!kernels construct info*/ 0;
+      SemaRef.Diag(SemaRef.LoopVectorClauseLoc,
+                   diag::note_acc_previous_clause_here);
+      return nullptr;
+    }
   }
 
-  return SemaRef.CheckGangClause(ExistingClauses, Clause.getBeginLoc(),
-                                 Clause.getLParenLoc(), GangKinds, IntExprs,
-                                 Clause.getEndLoc());
+  return SemaRef.CheckGangClause(Clause.getDirectiveKind(), ExistingClauses,
+                                 Clause.getBeginLoc(), Clause.getLParenLoc(),
+                                 GangKinds, IntExprs, Clause.getEndLoc());
 }
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause(
@@ -1402,41 +1638,39 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on 'compute' constructs, and
-  // 'compute' constructs are the only construct that can do anything with
-  // this yet, so skip/treat as unimplemented in this case.
-  // TODO: OpenACC: Remove check once we get combined constructs for this clause.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
-      Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-    return isNotImplemented();
-
   // OpenACC 3.3 Section 2.9.11: A reduction clause may not appear on a loop
   // directive that has a gang clause and is within a compute construct that has
   // a num_gangs clause with more than one explicit argument.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Loop &&
-      SemaRef.getActiveComputeConstructInfo().Kind !=
-          OpenACCDirectiveKind::Invalid) {
+  if ((Clause.getDirectiveKind() == OpenACCDirectiveKind::Loop &&
+       SemaRef.getActiveComputeConstructInfo().Kind !=
+           OpenACCDirectiveKind::Invalid) ||
+      isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
     // num_gangs clause on the active compute construct.
-    auto *NumGangsClauseItr =
-        llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
-                      llvm::IsaPred<OpenACCNumGangsClause>);
-
-    auto *GangClauseItr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
-
-    if (GangClauseItr != ExistingClauses.end() &&
-        NumGangsClauseItr !=
-            SemaRef.getActiveComputeConstructInfo().Clauses.end() &&
+    auto ActiveComputeConstructContainer =
+        isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())
+            ? ExistingClauses
+            : SemaRef.getActiveComputeConstructInfo().Clauses;
+    auto *NumGangsClauseItr = llvm::find_if(
+        ActiveComputeConstructContainer, llvm::IsaPred<OpenACCNumGangsClause>);
+
+    if (NumGangsClauseItr != ActiveComputeConstructContainer.end() &&
         cast<OpenACCNumGangsClause>(*NumGangsClauseItr)->getIntExprs().size() >
             1) {
-      SemaRef.Diag(Clause.getBeginLoc(),
-                   diag::err_acc_gang_reduction_numgangs_conflict)
-          << OpenACCClauseKind::Reduction << OpenACCClauseKind::Gang;
-      SemaRef.Diag((*GangClauseItr)->getBeginLoc(),
-                   diag::note_acc_previous_clause_here);
-      SemaRef.Diag((*NumGangsClauseItr)->getBeginLoc(),
-                   diag::note_acc_previous_clause_here);
-      return nullptr;
+      auto *GangClauseItr =
+          llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
+
+      if (GangClauseItr != ExistingClauses.end()) {
+        SemaRef.Diag(Clause.getBeginLoc(),
+                     diag::err_acc_gang_reduction_numgangs_conflict)
+            << OpenACCClauseKind::Reduction << OpenACCClauseKind::Gang
+            << Clause.getDirectiveKind()
+            << isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind());
+        SemaRef.Diag((*GangClauseItr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+        SemaRef.Diag((*NumGangsClauseItr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+        return nullptr;
+      }
     }
   }
 
@@ -1460,7 +1694,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
   // OpenACC 3.3 Section 2.5.4:
   // A reduction clause may not appear on a parallel construct with a
   // num_gangs clause that has more than one argument.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel) {
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel ||
+      Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop) {
     auto NumGangsClauses = llvm::make_filter_range(
         ExistingClauses, llvm::IsaPred<OpenACCNumGangsClause>);
 
@@ -1471,7 +1706,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
       if (NumExprs > 1) {
         SemaRef.Diag(Clause.getBeginLoc(),
                      diag::err_acc_reduction_num_gangs_conflict)
-            << NumExprs;
+            << /*>1 arg in first loc=*/0 << Clause.getClauseKind()
+            << Clause.getDirectiveKind() << OpenACCClauseKind::NumGangs;
         SemaRef.Diag(NGC->getBeginLoc(), diag::note_acc_previous_clause_here);
         return nullptr;
       }
@@ -1496,10 +1732,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCollapseClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // TODO: Remove this check once we implement this for combined constructs.
-  if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) &&
-      Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
-    return isNotImplemented();
   // Duplicates here are not really sensible.  We could possible permit
   // multiples if they all had the same value, but there isn't really a good
   // reason to do so. Also, this simplifies the suppression of duplicates, in
@@ -1537,7 +1769,7 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
     ArrayRef<const OpenACCClause *> UnInstClauses,
     ArrayRef<OpenACCClause *> Clauses)
     : SemaRef(S), OldActiveComputeConstructInfo(S.ActiveComputeConstructInfo),
-      DirKind(DK), OldLoopGangClauseOnKernelLoc(S.LoopGangClauseOnKernelLoc),
+      DirKind(DK), OldLoopGangClauseOnKernel(S.LoopGangClauseOnKernel),
       OldLoopWorkerClauseLoc(S.LoopWorkerClauseLoc),
       OldLoopVectorClauseLoc(S.LoopVectorClauseLoc),
       OldLoopWithoutSeqInfo(S.LoopWithoutSeqInfo),
@@ -1558,7 +1790,7 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
     // within a nested compute region.
     //
     // Implement the 'unless within a nested compute region' part.
-    SemaRef.LoopGangClauseOnKernelLoc = {};
+    SemaRef.LoopGangClauseOnKernel = {};
     SemaRef.LoopWorkerClauseLoc = {};
     SemaRef.LoopVectorClauseLoc = {};
     SemaRef.LoopWithoutSeqInfo = {};
@@ -1572,8 +1804,7 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
     SetCollapseInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
     SetTileInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
 
-    // TODO: OpenACC: We need to set these 3, CollapseInfo, and TileInfo
-    SemaRef.LoopGangClauseOnKernelLoc = {};
+    SemaRef.LoopGangClauseOnKernel = {};
     SemaRef.LoopWorkerClauseLoc = {};
     SemaRef.LoopVectorClauseLoc = {};
 
@@ -1596,7 +1827,7 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
       // This handles the 'outer loop' part of this.
       auto *Itr = llvm::find_if(Clauses, llvm::IsaPred<OpenACCGangClause>);
       if (Itr != Clauses.end())
-        SemaRef.LoopGangClauseOnKernelLoc = (*Itr)->getBeginLoc();
+        SemaRef.LoopGangClauseOnKernel = {(*Itr)->getBeginLoc(), DirKind};
     }
 
     if (UnInstClauses.empty()) {
@@ -1634,7 +1865,8 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
       // This handles the 'outer loop' part of this.
       auto *Itr = llvm::find_if(Clauses, llvm::IsaPred<OpenACCGangClause>);
       if (Itr != Clauses.end())
-        SemaRef.LoopGangClauseOnKernelLoc = (*Itr)->getBeginLoc();
+        SemaRef.LoopGangClauseOnKernel = {(*Itr)->getBeginLoc(),
+                                          OpenACCDirectiveKind::Kernels};
     }
 
     if (UnInstClauses.empty()) {
@@ -1693,6 +1925,7 @@ void SemaOpenACC::AssociatedStmtRAII::SetCollapseInfoBeforeAssociatedStmt(
   SemaRef.CollapseInfo.CollapseDepthSatisfied = false;
   SemaRef.CollapseInfo.CurCollapseCount =
       cast<ConstantExpr>(LoopCount)->getResultAsAPSInt();
+  SemaRef.CollapseInfo.DirectiveKind = DirKind;
 }
 
 void SemaOpenACC::AssociatedStmtRAII::SetTileInfoBeforeAssociatedStmt(
@@ -1713,11 +1946,12 @@ void SemaOpenACC::AssociatedStmtRAII::SetTileInfoBeforeAssociatedStmt(
   SemaRef.TileInfo.ActiveTile = TileClause;
   SemaRef.TileInfo.TileDepthSatisfied = false;
   SemaRef.TileInfo.CurTileCount = TileClause->getSizeExprs().size();
+  SemaRef.TileInfo.DirectiveKind = DirKind;
 }
 
 SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
   SemaRef.ActiveComputeConstructInfo = OldActiveComputeConstructInfo;
-  SemaRef.LoopGangClauseOnKernelLoc = OldLoopGangClauseOnKernelLoc;
+  SemaRef.LoopGangClauseOnKernel = OldLoopGangClauseOnKernel;
   SemaRef.LoopWorkerClauseLoc = OldLoopWorkerClauseLoc;
   SemaRef.LoopVectorClauseLoc = OldLoopVectorClauseLoc;
   SemaRef.LoopWithoutSeqInfo = OldLoopWithoutSeqInfo;
@@ -2374,111 +2608,53 @@ ExprResult SemaOpenACC::CheckCollapseLoopCount(Expr *LoopCount) {
       ConstantExpr::Create(getASTContext(), LoopCount, APValue{*ICE})};
 }
 
-namespace {
-ExprResult CheckGangStaticExpr(SemaOpenACC &S, Expr *E) {
-  if (isa<OpenACCAsteriskSizeExpr>(E))
-    return E;
-  return S.ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
-                        E->getBeginLoc(), E);
-}
-} // namespace
-
-ExprResult SemaOpenACC::CheckGangExpr(OpenACCGangKind GK, Expr *E) {
-  // Gang Expr legality depends on the associated compute construct.
-  switch (ActiveComputeConstructInfo.Kind) {
-  case OpenACCDirectiveKind::Invalid:
-  case OpenACCDirectiveKind::Parallel: {
-    switch (GK) {
-      // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
-      // construct, or an orphaned loop construct, the gang clause behaves as
-      // follows. ... The dim argument must be a constant positive integer value
-      // 1, 2, or 3.
-    case OpenACCGangKind::Dim: {
-      if (!E)
-        return ExprError();
-      ExprResult Res =
-          ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
-                       E->getBeginLoc(), E);
-
-      if (!Res.isUsable())
-        return Res;
-
-      if (Res.get()->isInstantiationDependent())
-        return Res;
-
-      std::optional<llvm::APSInt> ICE =
-          Res.get()->getIntegerConstantExpr(getASTContext());
-
-      if (!ICE || *ICE <= 0 || ICE > 3) {
-        Diag(Res.get()->getBeginLoc(), diag::err_acc_gang_dim_value)
-            << ICE.has_value() << ICE.value_or(llvm::APSInt{}).getExtValue();
-        return ExprError();
-      }
-
-      return ExprResult{
-          ConstantExpr::Create(getASTContext(), Res.get(), APValue{*ICE})};
-    }
-      // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
-      // construct, or an orphaned loop construct, the gang clause behaves as
-      // follows. ... The num argument is not allowed.
-    case OpenACCGangKind::Num:
-      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Gang << GK
-          << (/*orphan/parallel=*/ActiveComputeConstructInfo.Kind ==
-                      OpenACCDirectiveKind::Parallel
-                  ? 1
-                  : 0);
-      return ExprError();
-    case OpenACCGangKind::Static:
-      return CheckGangStaticExpr(*this, E);
-    }
-  } break;
-  case OpenACCDirectiveKind::Kernels: {
-    switch (GK) {
-    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
-    // construct, the gang clause behaves as follows. ... The dim argument is
-    // not allowed.
-    case OpenACCGangKind::Dim:
-      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Gang << GK << /*kernels=*/2;
-      return ExprError();
-    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
-    // construct, the gang clause behaves as follows. ... An argument with no
-    // keyword or with num keyword is only allowed when num_gangs does not
-    // appear on the kernels construct. ... The region of a loop with the gang
-    // clause may not contain another loop with a gang clause unless within a
-    // nested compute region.
-    case OpenACCGangKind::Num:
-      // This isn't allowed if there is a 'num_gangs' on the kernel construct,
-      // and makes loop-with-gang-clause ill-formed inside of this 'loop', but
-      // nothing can be enforced here.
-      return ExprResult{E};
-    case OpenACCGangKind::Static:
-      return CheckGangStaticExpr(*this, E);
-    }
-  } break;
-  case OpenACCDirectiveKind::Serial: {
-    switch (GK) {
-    // 'dim' and 'num' don't really make sense on serial, and GCC rejects them
-    // too, so we disallow them too.
-    case OpenACCGangKind::Dim:
-    case OpenACCGangKind::Num:
-      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
-          << OpenACCClauseKind::Gang << GK << /*Kernels=*/3;
-      return ExprError();
-    case OpenACCGangKind::Static:
-      return CheckGangStaticExpr(*this, E);
+ExprResult
+SemaOpenACC::CheckGangExpr(ArrayRef<const OpenACCClause *> ExistingClauses,
+                           OpenACCDirectiveKind DK, OpenACCGangKind GK,
+                           Expr *E) {
+  // There are two cases for the enforcement here: the 'current' directive is a
+  // 'loop', where we need to check the active compute construct kind, or the
+  // current directive is a 'combined' construct, where we have to check the
+  // current one.
+  switch (DK) {
+  case OpenACCDirectiveKind::ParallelLoop:
+    return CheckGangParallelExpr(*this, DK, ActiveComputeConstructInfo.Kind, GK,
+                                 E);
+  case OpenACCDirectiveKind::SerialLoop:
+    return CheckGangSerialExpr(*this, DK, ActiveComputeConstructInfo.Kind, GK,
+                               E);
+  case OpenACCDirectiveKind::KernelsLoop:
+    return CheckGangKernelsExpr(*this, ExistingClauses, DK,
+                                ActiveComputeConstructInfo.Kind, GK, E);
+  case OpenACCDirectiveKind::Loop:
+    switch (ActiveComputeConstructInfo.Kind) {
+    case OpenACCDirectiveKind::Invalid:
+    case OpenACCDirectiveKind::Parallel:
+    case OpenACCDirectiveKind::ParallelLoop:
+      return CheckGangParallelExpr(*this, DK, ActiveComputeConstructInfo.Kind,
+                                   GK, E);
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::Serial:
+      return CheckGangSerialExpr(*this, DK, ActiveComputeConstructInfo.Kind, GK,
+                                 E);
+    case OpenACCDirectiveKind::KernelsLoop:
+    case OpenACCDirectiveKind::Kernels:
+      return CheckGangKernelsExpr(*this, ExistingClauses, DK,
+                                  ActiveComputeConstructInfo.Kind, GK, E);
+    default:
+      llvm_unreachable("Non compute construct in active compute construct?");
     }
-  } break;
   default:
-    llvm_unreachable("Non compute construct in active compute construct?");
+    // TODO: OpenACC: when we implement this on 'routine', we'll have to
+    // implement its checking here.
+    llvm_unreachable("Invalid directive kind for a Gang clause");
   }
-
   llvm_unreachable("Compute construct directive not handled?");
 }
 
 OpenACCClause *
-SemaOpenACC::CheckGangClause(ArrayRef<const OpenACCClause *> ExistingClauses,
+SemaOpenACC::CheckGangClause(OpenACCDirectiveKind DirKind,
+                             ArrayRef<const OpenACCClause *> ExistingClauses,
                              SourceLocation BeginLoc, SourceLocation LParenLoc,
                              ArrayRef<OpenACCGangKind> GangKinds,
                              ArrayRef<Expr *> IntExprs, SourceLocation EndLoc) {
@@ -2503,7 +2679,7 @@ SemaOpenACC::CheckGangClause(ArrayRef<const OpenACCClause *> ExistingClauses,
       if (const auto *DimVal = dyn_cast<ConstantExpr>(DimExpr);
           DimVal && DimVal->getResultAsAPSInt() > 1) {
         Diag(DimVal->getBeginLoc(), diag::err_acc_gang_reduction_conflict)
-            << /*gang/reduction=*/0;
+            << /*gang/reduction=*/0 << DirKind;
         Diag((*ReductionItr)->getBeginLoc(),
              diag::note_acc_previous_clause_here);
         return nullptr;
@@ -2520,30 +2696,29 @@ OpenACCClause *SemaOpenACC::CheckReductionClause(
     OpenACCDirectiveKind DirectiveKind, SourceLocation BeginLoc,
     SourceLocation LParenLoc, OpenACCReductionOperator ReductionOp,
     ArrayRef<Expr *> Vars, SourceLocation EndLoc) {
-  if (DirectiveKind == OpenACCDirectiveKind::Loop) {
+  if (DirectiveKind == OpenACCDirectiveKind::Loop ||
+      isOpenACCCombinedDirectiveKind(DirectiveKind)) {
     // OpenACC 3.3 2.9.11: A reduction clause may not appear on a loop directive
     // that has a gang clause with a dim: argument whose value is greater
     // than 1.
-    const auto *GangItr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
+    const auto GangClauses = llvm::make_filter_range(
+        ExistingClauses, llvm::IsaPred<OpenACCGangClause>);
 
-    while (GangItr != ExistingClauses.end()) {
-      auto *GangClause = cast<OpenACCGangClause>(*GangItr);
+    for (auto *GC : GangClauses) {
+      const auto *GangClause = cast<OpenACCGangClause>(GC);
       for (unsigned I = 0; I < GangClause->getNumExprs(); ++I) {
         std::pair<OpenACCGangKind, const Expr *> EPair = GangClause->getExpr(I);
-        // We know there is only 1 on this gang, so move onto the next gang.
         if (EPair.first != OpenACCGangKind::Dim)
-          break;
+          continue;
 
         if (const auto *DimVal = dyn_cast<ConstantExpr>(EPair.second);
             DimVal && DimVal->getResultAsAPSInt() > 1) {
           Diag(BeginLoc, diag::err_acc_gang_reduction_conflict)
-              << /*reduction/gang=*/1;
-          Diag((*GangItr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+              << /*reduction/gang=*/1 << DirectiveKind;
+          Diag(GangClause->getBeginLoc(), diag::note_acc_previous_clause_here);
           return nullptr;
         }
       }
-      ++GangItr;
     }
   }
 
@@ -2589,7 +2764,8 @@ void SemaOpenACC::ActOnWhileStmt(SourceLocation WhileLoc) {
 
   if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
     Diag(WhileLoc, diag::err_acc_invalid_in_loop)
-        << /*while loop*/ 1 << OpenACCClauseKind::Collapse;
+        << /*while loop*/ 1 << CollapseInfo.DirectiveKind
+        << OpenACCClauseKind::Collapse;
     assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
     Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
          diag::note_acc_active_clause_here)
@@ -2602,7 +2778,8 @@ void SemaOpenACC::ActOnWhileStmt(SourceLocation WhileLoc) {
 
   if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
     Diag(WhileLoc, diag::err_acc_invalid_in_loop)
-        << /*while loop*/ 1 << OpenACCClauseKind::Tile;
+        << /*while loop*/ 1 << TileInfo.DirectiveKind
+        << OpenACCClauseKind::Tile;
     assert(TileInfo.ActiveTile && "tile count without object?");
     Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
         << OpenACCClauseKind::Tile;
@@ -2622,7 +2799,8 @@ void SemaOpenACC::ActOnDoStmt(SourceLocation DoLoc) {
 
   if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
     Diag(DoLoc, diag::err_acc_invalid_in_loop)
-        << /*do loop*/ 2 << OpenACCClauseKind::Collapse;
+        << /*do loop*/ 2 << CollapseInfo.DirectiveKind
+        << OpenACCClauseKind::Collapse;
     assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
     Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
          diag::note_acc_active_clause_here)
@@ -2635,7 +2813,7 @@ void SemaOpenACC::ActOnDoStmt(SourceLocation DoLoc) {
 
   if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
     Diag(DoLoc, diag::err_acc_invalid_in_loop)
-        << /*do loop*/ 2 << OpenACCClauseKind::Tile;
+        << /*do loop*/ 2 << TileInfo.DirectiveKind << OpenACCClauseKind::Tile;
     assert(TileInfo.ActiveTile && "tile count without object?");
     Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
         << OpenACCClauseKind::Tile;
@@ -2662,7 +2840,8 @@ void SemaOpenACC::ForStmtBeginHelper(SourceLocation ForLoc,
     // This checks for more than 1 loop at the current level, the
     // 'depth'-satisifed checking manages the 'not zero' case.
     if (LoopInfo.CurLevelHasLoopAlready) {
-      Diag(ForLoc, diag::err_acc_clause_multiple_loops) << /*Collapse*/ 0;
+      Diag(ForLoc, diag::err_acc_clause_multiple_loops)
+          << CollapseInfo.DirectiveKind << OpenACCClauseKind::Collapse;
       assert(CollapseInfo.ActiveCollapse && "No collapse object?");
       Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
            diag::note_acc_active_clause_here)
@@ -2681,7 +2860,8 @@ void SemaOpenACC::ForStmtBeginHelper(SourceLocation ForLoc,
     C.check();
 
     if (LoopInfo.CurLevelHasLoopAlready) {
-      Diag(ForLoc, diag::err_acc_clause_multiple_loops) << /*Tile*/ 1;
+      Diag(ForLoc, diag::err_acc_clause_multiple_loops)
+          << TileInfo.DirectiveKind << OpenACCClauseKind::Tile;
       assert(TileInfo.ActiveTile && "No tile object?");
       Diag(TileInfo.ActiveTile->getBeginLoc(),
            diag::note_acc_active_clause_here)
@@ -3184,7 +3364,7 @@ void SemaOpenACC::ActOnForStmtEnd(SourceLocation ForLoc, StmtResult Body) {
 
     if (OtherStmtLoc.isValid() && IsActiveCollapse) {
       Diag(OtherStmtLoc, diag::err_acc_intervening_code)
-          << OpenACCClauseKind::Collapse;
+          << OpenACCClauseKind::Collapse << CollapseInfo.DirectiveKind;
       Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
            diag::note_acc_active_clause_here)
           << OpenACCClauseKind::Collapse;
@@ -3192,7 +3372,7 @@ void SemaOpenACC::ActOnForStmtEnd(SourceLocation ForLoc, StmtResult Body) {
 
     if (OtherStmtLoc.isValid() && IsActiveTile) {
       Diag(OtherStmtLoc, diag::err_acc_intervening_code)
-          << OpenACCClauseKind::Tile;
+          << OpenACCClauseKind::Tile << TileInfo.DirectiveKind;
       Diag(TileInfo.ActiveTile->getBeginLoc(),
            diag::note_acc_active_clause_here)
           << OpenACCClauseKind::Tile;
@@ -3212,7 +3392,8 @@ bool SemaOpenACC::ActOnStartStmtDirective(OpenACCDirectiveKind K,
   // ALL constructs are ill-formed if there is an active 'collapse'
   if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
     Diag(StartLoc, diag::err_acc_invalid_in_loop)
-        << /*OpenACC Construct*/ 0 << OpenACCClauseKind::Collapse << K;
+        << /*OpenACC Construct*/ 0 << CollapseInfo.DirectiveKind
+        << OpenACCClauseKind::Collapse << K;
     assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
     Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
          diag::note_acc_active_clause_here)
@@ -3220,7 +3401,8 @@ bool SemaOpenACC::ActOnStartStmtDirective(OpenACCDirectiveKind K,
   }
   if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
     Diag(StartLoc, diag::err_acc_invalid_in_loop)
-        << /*OpenACC Construct*/ 0 << OpenACCClauseKind::Tile << K;
+        << /*OpenACC Construct*/ 0 << TileInfo.DirectiveKind
+        << OpenACCClauseKind::Tile << K;
     assert(TileInfo.ActiveTile && "Tile count without object?");
     Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
         << OpenACCClauseKind::Tile;
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 4815248de921b..d3d3df4c2ab86 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -1611,10 +1611,10 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopMostTaskgroupReductionData(
       continue;
     const ReductionData &ReductionData = I->ReductionMap.lookup(D);
     if (!ReductionData.ReductionOp ||
-        ReductionData.ReductionOp.is<const Expr *>())
+        isa<const Expr *>(ReductionData.ReductionOp))
       return DSAVarData();
     SR = ReductionData.ReductionRange;
-    BOK = ReductionData.ReductionOp.get<ReductionData::BOKPtrType>();
+    BOK = cast<ReductionData::BOKPtrType>(ReductionData.ReductionOp);
     assert(I->TaskgroupReductionRef && "taskgroup reduction reference "
                                        "expression for the descriptor is not "
                                        "set.");
@@ -1638,10 +1638,10 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopMostTaskgroupReductionData(
       continue;
     const ReductionData &ReductionData = I->ReductionMap.lookup(D);
     if (!ReductionData.ReductionOp ||
-        !ReductionData.ReductionOp.is<const Expr *>())
+        !isa<const Expr *>(ReductionData.ReductionOp))
       return DSAVarData();
     SR = ReductionData.ReductionRange;
-    ReductionRef = ReductionData.ReductionOp.get<const Expr *>();
+    ReductionRef = cast<const Expr *>(ReductionData.ReductionOp);
     assert(I->TaskgroupReductionRef && "taskgroup reduction reference "
                                        "expression for the descriptor is not "
                                        "set.");
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 13f78143d2441..94542de216baa 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -2237,33 +2237,24 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
       return false;
     }
   }
-  // Lvalue-to-rvalue conversion (C++11 4.1):
-  //   A glvalue (3.10) of a non-function, non-array type T can
-  //   be converted to a prvalue.
-  bool argIsLValue = From->isGLValue();
-  if (argIsLValue && !FromType->canDecayToPointerType() &&
-      S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) {
-    SCS.First = ICK_Lvalue_To_Rvalue;
-
-    // C11 6.3.2.1p2:
-    //   ... if the lvalue has atomic type, the value has the non-atomic version
-    //   of the type of the lvalue ...
-    if (const AtomicType *Atomic = FromType->getAs<AtomicType>())
-      FromType = Atomic->getValueType();
 
-    // If T is a non-class type, the type of the rvalue is the
-    // cv-unqualified version of T. Otherwise, the type of the rvalue
-    // is T (C++ 4.1p1). C++ can't get here with class types; in C, we
-    // just strip the qualifiers because they don't matter.
-    FromType = FromType.getUnqualifiedType();
-  } else if (S.getLangOpts().HLSL && FromType->isConstantArrayType() &&
-             ToType->isConstantArrayType()) {
+  bool argIsLValue = From->isGLValue();
+  // To handle conversion from ArrayParameterType to ConstantArrayType
+  // this block must be above the one below because Array parameters
+  // do not decay and when handling HLSLOutArgExprs and
+  // the From expression is an LValue.
+  if (S.getLangOpts().HLSL && FromType->isConstantArrayType() &&
+      ToType->isConstantArrayType()) {
     // HLSL constant array parameters do not decay, so if the argument is a
     // constant array and the parameter is an ArrayParameterType we have special
     // handling here.
     if (ToType->isArrayParameterType()) {
       FromType = S.Context.getArrayParameterType(FromType);
       SCS.First = ICK_HLSL_Array_RValue;
+    } else if (FromType->isArrayParameterType()) {
+      const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
+      FromType = APT->getConstantArrayType(S.Context);
+      SCS.First = ICK_HLSL_Array_RValue;
     } else {
       SCS.First = ICK_Identity;
     }
@@ -2274,6 +2265,25 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
 
     SCS.setAllToTypes(ToType);
     return true;
+  } else if (argIsLValue && !FromType->canDecayToPointerType() &&
+             S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) {
+    // Lvalue-to-rvalue conversion (C++11 4.1):
+    //   A glvalue (3.10) of a non-function, non-array type T can
+    //   be converted to a prvalue.
+
+    SCS.First = ICK_Lvalue_To_Rvalue;
+
+    // C11 6.3.2.1p2:
+    //   ... if the lvalue has atomic type, the value has the non-atomic version
+    //   of the type of the lvalue ...
+    if (const AtomicType *Atomic = FromType->getAs<AtomicType>())
+      FromType = Atomic->getValueType();
+
+    // If T is a non-class type, the type of the rvalue is the
+    // cv-unqualified version of T. Otherwise, the type of the rvalue
+    // is T (C++ 4.1p1). C++ can't get here with class types; in C, we
+    // just strip the qualifiers because they don't matter.
+    FromType = FromType.getUnqualifiedType();
   } else if (FromType->isArrayType()) {
     // Array-to-pointer conversion (C++ 4.2)
     SCS.First = ICK_Array_To_Pointer;
@@ -15592,7 +15602,7 @@ ExprResult Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     // Build the actual expression node.
     ExprResult FnExpr =
         CreateFunctionRefExpr(*this, Method, FoundDecl, MemExpr,
-                              HadMultipleCandidates, MemExpr->getBeginLoc());
+                              HadMultipleCandidates, MemExpr->getExprLoc());
     if (FnExpr.isInvalid())
       return ExprError();
 
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index fd08af738201e..b57480299576e 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -3779,6 +3779,8 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
         << FSI->getFirstCoroutineStmtKeyword();
   }
 
+  CheckInvalidBuiltinCountedByRef(RetVal.get(), ReturnArgKind);
+
   StmtResult R =
       BuildReturnStmt(ReturnLoc, RetVal.get(), /*AllowRecovery=*/true);
   if (R.isInvalid() || ExprEvalContexts.back().isDiscardedStatementContext())
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index f6ca06fefb491..5e7a3c8484c88 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -4157,6 +4157,13 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
              << IsPartialSpecialization;
   }
 
+  if (const auto *DSA = VarTemplate->getAttr<NoSpecializationsAttr>()) {
+    auto Message = DSA->getMessage();
+    Diag(TemplateNameLoc, diag::warn_invalid_specialization)
+        << VarTemplate << !Message.empty() << Message;
+    Diag(DSA->getLoc(), diag::note_marked_here) << DSA;
+  }
+
   // Check for unexpanded parameter packs in any of the template arguments.
   for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
     if (DiagnoseUnexpandedParameterPack(TemplateArgs[I],
@@ -8291,6 +8298,13 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     return true;
   }
 
+  if (const auto *DSA = ClassTemplate->getAttr<NoSpecializationsAttr>()) {
+    auto Message = DSA->getMessage();
+    Diag(TemplateNameLoc, diag::warn_invalid_specialization)
+        << ClassTemplate << !Message.empty() << Message;
+    Diag(DSA->getLoc(), diag::note_marked_here) << DSA;
+  }
+
   if (S->isTemplateParamScope())
     EnterTemplatedContext(S, ClassTemplate->getTemplatedDecl());
 
@@ -9175,6 +9189,14 @@ bool Sema::CheckFunctionTemplateSpecialization(
   // Ignore access information;  it doesn't figure into redeclaration checking.
   FunctionDecl *Specialization = cast<FunctionDecl>(*Result);
 
+  if (const auto *PT = Specialization->getPrimaryTemplate();
+      const auto *DSA = PT->getAttr<NoSpecializationsAttr>()) {
+    auto Message = DSA->getMessage();
+    Diag(FD->getLocation(), diag::warn_invalid_specialization)
+        << PT << !Message.empty() << Message;
+    Diag(DSA->getLoc(), diag::note_marked_here) << DSA;
+  }
+
   // C++23 [except.spec]p13:
   //   An exception specification is considered to be needed when:
   //   - [...]
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 62a0c30d99502..fad20b37a7d9a 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -376,7 +376,7 @@ checkDeducedTemplateArguments(ASTContext &Context,
     for (TemplateArgument::pack_iterator
              XA = X.pack_begin(),
              XAEnd = X.pack_end(), YA = Y.pack_begin(), YAEnd = Y.pack_end();
-         XA != XAEnd; ++XA, ++YA) {
+         XA != XAEnd; ++XA) {
       if (YA != YAEnd) {
         TemplateArgument Merged = checkDeducedTemplateArguments(
             Context, DeducedTemplateArgument(*XA, X.wasDeducedFromArrayBound()),
@@ -384,6 +384,7 @@ checkDeducedTemplateArguments(ASTContext &Context,
         if (Merged.isNull() && !(XA->isNull() && YA->isNull()))
           return DeducedTemplateArgument();
         NewPack.push_back(Merged);
+        ++YA;
       } else {
         NewPack.push_back(*XA);
       }
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 14f0d0c5e25ff..599eee1d10d70 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -198,7 +198,7 @@ HandleVarTemplateSpec(const VarTemplateSpecializationDecl *VarTemplSpec,
     if (Partial->isMemberSpecialization())
       return Response::Done();
   } else {
-    VarTemplateDecl *Tmpl = Specialized.get<VarTemplateDecl *>();
+    VarTemplateDecl *Tmpl = cast<VarTemplateDecl *>(Specialized);
     if (!SkipForSpecialization)
       Result.addOuterTemplateArguments(
           Tmpl, VarTemplSpec->getTemplateInstantiationArgs().asArray(),
@@ -2555,7 +2555,7 @@ TemplateInstantiator::TransformFunctionParmPackRefExpr(DeclRefExpr *E,
 
     TransformedDecl = (*Pack)[getSema().ArgumentPackSubstitutionIndex];
   } else {
-    TransformedDecl = Found->get<Decl*>();
+    TransformedDecl = cast<Decl *>(*Found);
   }
 
   // We have either an unexpanded pack or a specific expansion.
@@ -2924,7 +2924,7 @@ TemplateInstantiator::TransformExprRequirement(concepts::ExprRequirement *Req) {
     return RebuildExprRequirement(E, Req->isSimple(), Req->getNoexceptLoc(),
                                   std::move(*TransRetReq));
   return RebuildExprRequirement(
-      TransExpr.get<concepts::Requirement::SubstitutionDiagnostic *>(),
+      cast<concepts::Requirement::SubstitutionDiagnostic *>(TransExpr),
       Req->isSimple(), Req->getNoexceptLoc(), std::move(*TransRetReq));
 }
 
@@ -4150,7 +4150,7 @@ getPatternForClassTemplateSpecialization(
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *>
       Specialized = ClassTemplateSpec->getSpecializedTemplateOrPartial();
-  if (!Specialized.is<ClassTemplatePartialSpecializationDecl *>()) {
+  if (!isa<ClassTemplatePartialSpecializationDecl *>(Specialized)) {
     // Find best matching specialization.
     ClassTemplateDecl *Template = ClassTemplateSpec->getSpecializedTemplate();
 
@@ -4761,14 +4761,14 @@ void LocalInstantiationScope::InstantiatedLocal(const Decl *D, Decl *Inst) {
   } else if (DeclArgumentPack *Pack = Stored.dyn_cast<DeclArgumentPack *>()) {
     Pack->push_back(cast<VarDecl>(Inst));
   } else {
-    assert(Stored.get<Decl *>() == Inst && "Already instantiated this local");
+    assert(cast<Decl *>(Stored) == Inst && "Already instantiated this local");
   }
 }
 
 void LocalInstantiationScope::InstantiatedLocalPackArg(const Decl *D,
                                                        VarDecl *Inst) {
   D = getCanonicalParmVarDecl(D);
-  DeclArgumentPack *Pack = LocalDecls[D].get<DeclArgumentPack *>();
+  DeclArgumentPack *Pack = cast<DeclArgumentPack *>(LocalDecls[D]);
   Pack->push_back(Inst);
 }
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 2b151337e5c14..bd08e76050238 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4240,8 +4240,8 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
   auto *PrevDeclInScope = D->getPrevDeclInScope();
   if (PrevDeclInScope && !PrevDeclInScope->isInvalidDecl()) {
     PrevDeclInScope = cast<OMPDeclareReductionDecl>(
-        SemaRef.CurrentInstantiationScope->findInstantiationOf(PrevDeclInScope)
-            ->get<Decl *>());
+        cast<Decl *>(*SemaRef.CurrentInstantiationScope->findInstantiationOf(
+            PrevDeclInScope)));
   }
   auto DRD = SemaRef.OpenMP().ActOnOpenMPDeclareReductionDirectiveStart(
       /*S=*/nullptr, Owner, D->getDeclName(), ReductionTypes, D->getAccess(),
@@ -4327,8 +4327,8 @@ TemplateDeclInstantiator::VisitOMPDeclareMapperDecl(OMPDeclareMapperDecl *D) {
   auto *PrevDeclInScope = D->getPrevDeclInScope();
   if (PrevDeclInScope && !PrevDeclInScope->isInvalidDecl()) {
     PrevDeclInScope = cast<OMPDeclareMapperDecl>(
-        SemaRef.CurrentInstantiationScope->findInstantiationOf(PrevDeclInScope)
-            ->get<Decl *>());
+        cast<Decl *>(*SemaRef.CurrentInstantiationScope->findInstantiationOf(
+            PrevDeclInScope)));
   }
   bool IsCorrect = true;
   SmallVector<OMPClause *, 6> Clauses;
@@ -6718,7 +6718,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
         assert(PackIdx != -1 &&
                "found declaration pack but not pack expanding");
         typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
-        return cast<NamedDecl>((*Found->get<DeclArgumentPack *>())[PackIdx]);
+        return cast<NamedDecl>((*cast<DeclArgumentPack *>(*Found))[PackIdx]);
       }
     }
 
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index 86d15f6324f4f..88a21240e1c80 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -371,7 +371,7 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
           auto *TTPD = dyn_cast<TemplateTypeParmDecl>(LocalPack);
           return TTPD && TTPD->getTypeForDecl() == TTPT;
         }
-        return declaresSameEntity(Pack.first.get<NamedDecl *>(), LocalPack);
+        return declaresSameEntity(cast<NamedDecl *>(Pack.first), LocalPack);
       };
       if (llvm::any_of(CSI->LocalPacks, DeclaresThisPack))
         ParamPackReferences.push_back(Pack);
@@ -423,7 +423,7 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
           = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>())
       Name = TTP->getIdentifier();
     else
-      Name = Unexpanded[I].first.get<NamedDecl *>()->getIdentifier();
+      Name = cast<NamedDecl *>(Unexpanded[I].first)->getIdentifier();
 
     if (Name && NamesKnown.insert(Name).second)
       Names.push_back(Name);
@@ -770,7 +770,7 @@ bool Sema::CheckParameterPacksForExpansion(
       Index = TTP->getIndex();
       Name = TTP->getIdentifier();
     } else {
-      NamedDecl *ND = ParmPack.first.get<NamedDecl *>();
+      NamedDecl *ND = cast<NamedDecl *>(ParmPack.first);
       if (isa<VarDecl>(ND))
         IsVarDeclPack = true;
       else
@@ -787,10 +787,10 @@ bool Sema::CheckParameterPacksForExpansion(
 
       llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation =
           CurrentInstantiationScope->findInstantiationOf(
-              ParmPack.first.get<NamedDecl *>());
-      if (Instantiation->is<DeclArgumentPack *>()) {
+              cast<NamedDecl *>(ParmPack.first));
+      if (isa<DeclArgumentPack *>(*Instantiation)) {
         // We could expand this function parameter pack.
-        NewPackSize = Instantiation->get<DeclArgumentPack *>()->size();
+        NewPackSize = cast<DeclArgumentPack *>(*Instantiation)->size();
       } else {
         // We can't expand this function parameter pack, so we can't expand
         // the pack expansion.
@@ -895,20 +895,20 @@ std::optional<unsigned> Sema::getNumArgumentsInExpansionFromUnexpanded(
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
     } else {
-      NamedDecl *ND = Unexpanded[I].first.get<NamedDecl *>();
+      NamedDecl *ND = cast<NamedDecl *>(Unexpanded[I].first);
       if (isa<VarDecl>(ND)) {
         // Function parameter pack or init-capture pack.
         typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
 
         llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation =
             CurrentInstantiationScope->findInstantiationOf(
-                Unexpanded[I].first.get<NamedDecl *>());
-        if (Instantiation->is<Decl *>())
+                cast<NamedDecl *>(Unexpanded[I].first));
+        if (isa<Decl *>(*Instantiation))
           // The pattern refers to an unexpanded pack. We're not ready to expand
           // this pack yet.
           return std::nullopt;
 
-        unsigned Size = Instantiation->get<DeclArgumentPack *>()->size();
+        unsigned Size = cast<DeclArgumentPack *>(*Instantiation)->size();
         assert((!Result || *Result == Size) && "inconsistent pack sizes");
         Result = Size;
         continue;
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 44e35ee904676..309f12d93ffd4 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -5373,15 +5373,23 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
 
         case NestedNameSpecifier::TypeSpec:
         case NestedNameSpecifier::TypeSpecWithTemplate:
-          ClsType = QualType(NNS->getAsType(), 0);
+          const Type *NNSType = NNS->getAsType();
+          ClsType = QualType(NNSType, 0);
           // Note: if the NNS has a prefix and ClsType is a nondependent
-          // TemplateSpecializationType, then the NNS prefix is NOT included
-          // in ClsType; hence we wrap ClsType into an ElaboratedType.
-          // NOTE: in particular, no wrap occurs if ClsType already is an
-          // Elaborated, DependentName, or DependentTemplateSpecialization.
-          if (isa<TemplateSpecializationType>(NNS->getAsType()))
+          // TemplateSpecializationType or a RecordType, then the NNS prefix is
+          // NOT included in ClsType; hence we wrap ClsType into an
+          // ElaboratedType. NOTE: in particular, no wrap occurs if ClsType
+          // already is an Elaborated, DependentName, or
+          // DependentTemplateSpecialization.
+          if (isa<DependentTemplateSpecializationType>(NNSType)) {
+            // FIXME: Rebuild DependentTemplateSpecializationType, adding the
+            // Prefix.
+          } else if (isa<TemplateSpecializationType, RecordType>(NNSType)) {
+            // Either the dependent case (TemplateSpecializationType), or the
+            // non-dependent one (RecordType).
             ClsType = Context.getElaboratedType(ElaboratedTypeKeyword::None,
                                                 NNSPrefix, ClsType);
+          }
           break;
         }
       } else {
@@ -5707,6 +5715,9 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
   assert(!T.isNull() && "T must not be null at the end of this function");
   if (!AreDeclaratorChunksValid)
     return Context.getTrivialTypeSourceInfo(T);
+
+  if (state.didParseHLSLParamMod() && !T->isConstantArrayType())
+    T = S.HLSL().getInoutParameterType(T);
   return GetTypeSourceInfoForDeclarator(state, T, TInfo);
 }
 
@@ -8721,7 +8732,6 @@ static void HandleHLSLParamModifierAttr(TypeProcessingState &State,
     return;
   if (Attr.getSemanticSpelling() == HLSLParamModifierAttr::Keyword_inout ||
       Attr.getSemanticSpelling() == HLSLParamModifierAttr::Keyword_out) {
-    CurType = S.HLSL().getInoutParameterType(CurType);
     State.setParsedHLSLParamMod(true);
   }
 }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index b2cb3e516cc62..968668b7b2595 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -12055,7 +12055,9 @@ void OpenACCClauseTransform<Derived>::VisitGangClause(
     if (!ER.isUsable())
       continue;
 
-    ER = Self.getSema().OpenACC().CheckGangExpr(C.getExpr(I).first, ER.get());
+    ER = Self.getSema().OpenACC().CheckGangExpr(ExistingClauses,
+                                                ParsedClause.getDirectiveKind(),
+                                                C.getExpr(I).first, ER.get());
     if (!ER.isUsable())
       continue;
     TransformedGangKinds.push_back(C.getExpr(I).first);
@@ -12063,7 +12065,8 @@ void OpenACCClauseTransform<Derived>::VisitGangClause(
   }
 
   NewClause = Self.getSema().OpenACC().CheckGangClause(
-      ExistingClauses, ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
+      ParsedClause.getDirectiveKind(), ExistingClauses,
+      ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
       TransformedGangKinds, TransformedIntExprs, ParsedClause.getEndLoc());
 }
 } // namespace
@@ -14564,7 +14567,7 @@ TreeTransform<Derived>::TransformExprRequirement(concepts::ExprRequirement *Req)
                                                Req->getNoexceptLoc(),
                                                std::move(*TransRetReq));
   return getDerived().RebuildExprRequirement(
-      TransExpr.get<concepts::Requirement::SubstitutionDiagnostic *>(),
+      cast<concepts::Requirement::SubstitutionDiagnostic *>(TransExpr),
       Req->isSimple(), Req->getNoexceptLoc(), std::move(*TransRetReq));
 }
 
diff --git a/clang/lib/Serialization/ASTCommon.h b/clang/lib/Serialization/ASTCommon.h
index 2a765eafe0895..7c9ec884ea049 100644
--- a/clang/lib/Serialization/ASTCommon.h
+++ b/clang/lib/Serialization/ASTCommon.h
@@ -24,7 +24,6 @@ namespace serialization {
 
 enum DeclUpdateKind {
   UPD_CXX_ADDED_IMPLICIT_MEMBER,
-  UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION,
   UPD_CXX_ADDED_ANONYMOUS_NAMESPACE,
   UPD_CXX_ADDED_FUNCTION_DEFINITION,
   UPD_CXX_ADDED_VAR_DEFINITION,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 316b7875a9775..4c524c0e72b6a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12,6 +12,7 @@
 
 #include "ASTCommon.h"
 #include "ASTReaderInternals.h"
+#include "TemplateArgumentHasher.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTMutationListener.h"
@@ -1295,6 +1296,42 @@ void ASTDeclContextNameLookupTrait::ReadDataInto(internal_key_type,
   }
 }
 
+ModuleFile *
+LazySpecializationInfoLookupTrait::ReadFileRef(const unsigned char *&d) {
+  using namespace llvm::support;
+
+  uint32_t ModuleFileID =
+      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+  return Reader.getLocalModuleFile(F, ModuleFileID);
+}
+
+LazySpecializationInfoLookupTrait::internal_key_type
+LazySpecializationInfoLookupTrait::ReadKey(const unsigned char *d, unsigned) {
+  using namespace llvm::support;
+  return endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+}
+
+std::pair<unsigned, unsigned>
+LazySpecializationInfoLookupTrait::ReadKeyDataLength(const unsigned char *&d) {
+  return readULEBKeyDataLength(d);
+}
+
+void LazySpecializationInfoLookupTrait::ReadDataInto(internal_key_type,
+                                                     const unsigned char *d,
+                                                     unsigned DataLen,
+                                                     data_type_builder &Val) {
+  using namespace llvm::support;
+
+  for (unsigned NumDecls =
+           DataLen / sizeof(serialization::reader::LazySpecializationInfo);
+       NumDecls; --NumDecls) {
+    LocalDeclID LocalID = LocalDeclID::get(
+        Reader, F,
+        endian::readNext<DeclID, llvm::endianness::little, unaligned>(d));
+    Val.insert(Reader.getGlobalDeclID(F, LocalID));
+  }
+}
+
 bool ASTReader::ReadLexicalDeclContextStorage(ModuleFile &M,
                                               BitstreamCursor &Cursor,
                                               uint64_t Offset,
@@ -1379,7 +1416,52 @@ bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
   // We can't safely determine the primary context yet, so delay attaching the
   // lookup table until we're done with recursive deserialization.
   auto *Data = (const unsigned char*)Blob.data();
-  PendingVisibleUpdates[ID].push_back(PendingVisibleUpdate{&M, Data});
+  PendingVisibleUpdates[ID].push_back(UpdateData{&M, Data});
+  return false;
+}
+
+void ASTReader::AddSpecializations(const Decl *D, const unsigned char *Data,
+                                   ModuleFile &M, bool IsPartial) {
+  D = D->getCanonicalDecl();
+  auto &SpecLookups =
+      IsPartial ? PartialSpecializationsLookups : SpecializationsLookups;
+  SpecLookups[D].Table.add(&M, Data,
+                           reader::LazySpecializationInfoLookupTrait(*this, M));
+}
+
+bool ASTReader::ReadSpecializations(ModuleFile &M, BitstreamCursor &Cursor,
+                                    uint64_t Offset, Decl *D, bool IsPartial) {
+  assert(Offset != 0);
+
+  SavedStreamPosition SavedPosition(Cursor);
+  if (llvm::Error Err = Cursor.JumpToBit(Offset)) {
+    Error(std::move(Err));
+    return true;
+  }
+
+  RecordData Record;
+  StringRef Blob;
+  Expected<unsigned> MaybeCode = Cursor.ReadCode();
+  if (!MaybeCode) {
+    Error(MaybeCode.takeError());
+    return true;
+  }
+  unsigned Code = MaybeCode.get();
+
+  Expected<unsigned> MaybeRecCode = Cursor.readRecord(Code, Record, &Blob);
+  if (!MaybeRecCode) {
+    Error(MaybeRecCode.takeError());
+    return true;
+  }
+  unsigned RecCode = MaybeRecCode.get();
+  if (RecCode != DECL_SPECIALIZATIONS &&
+      RecCode != DECL_PARTIAL_SPECIALIZATIONS) {
+    Error("Expected decl specs block");
+    return true;
+  }
+
+  auto *Data = (const unsigned char *)Blob.data();
+  AddSpecializations(D, Data, M, IsPartial);
   return false;
 }
 
@@ -3458,7 +3540,33 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       unsigned Idx = 0;
       GlobalDeclID ID = ReadDeclID(F, Record, Idx);
       auto *Data = (const unsigned char*)Blob.data();
-      PendingVisibleUpdates[ID].push_back(PendingVisibleUpdate{&F, Data});
+      PendingVisibleUpdates[ID].push_back(UpdateData{&F, Data});
+      // If we've already loaded the decl, perform the updates when we finish
+      // loading this block.
+      if (Decl *D = GetExistingDecl(ID))
+        PendingUpdateRecords.push_back(
+            PendingUpdateRecord(ID, D, /*JustLoaded=*/false));
+      break;
+    }
+
+    case CXX_ADDED_TEMPLATE_SPECIALIZATION: {
+      unsigned Idx = 0;
+      GlobalDeclID ID = ReadDeclID(F, Record, Idx);
+      auto *Data = (const unsigned char *)Blob.data();
+      PendingSpecializationsUpdates[ID].push_back(UpdateData{&F, Data});
+      // If we've already loaded the decl, perform the updates when we finish
+      // loading this block.
+      if (Decl *D = GetExistingDecl(ID))
+        PendingUpdateRecords.push_back(
+            PendingUpdateRecord(ID, D, /*JustLoaded=*/false));
+      break;
+    }
+
+    case CXX_ADDED_TEMPLATE_PARTIAL_SPECIALIZATION: {
+      unsigned Idx = 0;
+      GlobalDeclID ID = ReadDeclID(F, Record, Idx);
+      auto *Data = (const unsigned char *)Blob.data();
+      PendingPartialSpecializationsUpdates[ID].push_back(UpdateData{&F, Data});
       // If we've already loaded the decl, perform the updates when we finish
       // loading this block.
       if (Decl *D = GetExistingDecl(ID))
@@ -7661,13 +7769,28 @@ void ASTReader::CompleteRedeclChain(const Decl *D) {
     }
   }
 
-  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D))
-    CTSD->getSpecializedTemplate()->LoadLazySpecializations();
-  if (auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(D))
-    VTSD->getSpecializedTemplate()->LoadLazySpecializations();
-  if (auto *FD = dyn_cast<FunctionDecl>(D)) {
-    if (auto *Template = FD->getPrimaryTemplate())
-      Template->LoadLazySpecializations();
+  RedeclarableTemplateDecl *Template = nullptr;
+  ArrayRef<TemplateArgument> Args;
+  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    Template = CTSD->getSpecializedTemplate();
+    Args = CTSD->getTemplateArgs().asArray();
+  } else if (auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(D)) {
+    Template = VTSD->getSpecializedTemplate();
+    Args = VTSD->getTemplateArgs().asArray();
+  } else if (auto *FD = dyn_cast<FunctionDecl>(D)) {
+    if (auto *Tmplt = FD->getPrimaryTemplate()) {
+      Template = Tmplt;
+      Args = FD->getTemplateSpecializationArgs()->asArray();
+    }
+  }
+
+  if (Template) {
+    // For partitial specialization, load all the specializations for safety.
+    if (isa<ClassTemplatePartialSpecializationDecl,
+            VarTemplatePartialSpecializationDecl>(D))
+      Template->loadLazySpecializationsImpl();
+    else
+      Template->loadLazySpecializationsImpl(Args);
   }
 }
 
@@ -8049,6 +8172,86 @@ Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
   return ReadStmtFromStream(*Loc.F);
 }
 
+bool ASTReader::LoadExternalSpecializationsImpl(SpecLookupTableTy &SpecLookups,
+                                                const Decl *D) {
+  assert(D);
+
+  auto It = SpecLookups.find(D);
+  if (It == SpecLookups.end())
+    return false;
+
+  // Get Decl may violate the iterator from SpecializationsLookups so we store
+  // the DeclIDs in ahead.
+  llvm::SmallVector<serialization::reader::LazySpecializationInfo, 8> Infos =
+      It->second.Table.findAll();
+
+  // Since we've loaded all the specializations, we can erase it from
+  // the lookup table.
+  SpecLookups.erase(It);
+
+  bool NewSpecsFound = false;
+  Deserializing LookupResults(this);
+  for (auto &Info : Infos) {
+    if (GetExistingDecl(Info))
+      continue;
+    NewSpecsFound = true;
+    GetDecl(Info);
+  }
+
+  return NewSpecsFound;
+}
+
+bool ASTReader::LoadExternalSpecializations(const Decl *D, bool OnlyPartial) {
+  assert(D);
+
+  bool NewSpecsFound =
+      LoadExternalSpecializationsImpl(PartialSpecializationsLookups, D);
+  if (OnlyPartial)
+    return NewSpecsFound;
+
+  NewSpecsFound |= LoadExternalSpecializationsImpl(SpecializationsLookups, D);
+  return NewSpecsFound;
+}
+
+bool ASTReader::LoadExternalSpecializationsImpl(
+    SpecLookupTableTy &SpecLookups, const Decl *D,
+    ArrayRef<TemplateArgument> TemplateArgs) {
+  assert(D);
+
+  auto It = SpecLookups.find(D);
+  if (It == SpecLookups.end())
+    return false;
+
+  Deserializing LookupResults(this);
+  auto HashValue = StableHashForTemplateArguments(TemplateArgs);
+
+  // Get Decl may violate the iterator from SpecLookups
+  llvm::SmallVector<serialization::reader::LazySpecializationInfo, 8> Infos =
+      It->second.Table.find(HashValue);
+
+  bool NewSpecsFound = false;
+  for (auto &Info : Infos) {
+    if (GetExistingDecl(Info))
+      continue;
+    NewSpecsFound = true;
+    GetDecl(Info);
+  }
+
+  return NewSpecsFound;
+}
+
+bool ASTReader::LoadExternalSpecializations(
+    const Decl *D, ArrayRef<TemplateArgument> TemplateArgs) {
+  assert(D);
+
+  bool NewDeclsFound = LoadExternalSpecializationsImpl(
+      PartialSpecializationsLookups, D, TemplateArgs);
+  NewDeclsFound |=
+      LoadExternalSpecializationsImpl(SpecializationsLookups, D, TemplateArgs);
+
+  return NewDeclsFound;
+}
+
 void ASTReader::FindExternalLexicalDecls(
     const DeclContext *DC, llvm::function_ref<bool(Decl::Kind)> IsKindWeWant,
     SmallVectorImpl<Decl *> &Decls) {
@@ -8228,6 +8431,22 @@ ASTReader::getLoadedLookupTables(DeclContext *Primary) const {
   return I == Lookups.end() ? nullptr : &I->second;
 }
 
+serialization::reader::LazySpecializationInfoLookupTable *
+ASTReader::getLoadedSpecializationsLookupTables(const Decl *D, bool IsPartial) {
+  assert(D->isCanonicalDecl());
+  auto &LookupTable =
+      IsPartial ? PartialSpecializationsLookups : SpecializationsLookups;
+  auto I = LookupTable.find(D);
+  return I == LookupTable.end() ? nullptr : &I->second;
+}
+
+bool ASTReader::haveUnloadedSpecializations(const Decl *D) const {
+  assert(D->isCanonicalDecl());
+  return (PartialSpecializationsLookups.find(D) !=
+          PartialSpecializationsLookups.end()) ||
+         (SpecializationsLookups.find(D) != SpecializationsLookups.end());
+}
+
 /// Under non-PCH compilation the consumer receives the objc methods
 /// before receiving the implementation, and codegen depends on this.
 /// We simulate this by deserializing and passing to consumer the methods of the
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 6ece3ba7af9f4..be77df81b1ddf 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -187,11 +187,6 @@ class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
 
   std::string readString() { return Record.readString(); }
 
-  void readDeclIDList(SmallVectorImpl<GlobalDeclID> &IDs) {
-    for (unsigned I = 0, Size = Record.readInt(); I != Size; ++I)
-      IDs.push_back(readDeclID());
-  }
-
   Decl *readDecl() { return Record.readDecl(); }
 
   template <typename T> T *readDeclAs() { return Record.readDeclAs<T>(); }
@@ -284,30 +279,6 @@ class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
       : Reader(Reader), MergeImpl(Reader), Record(Record), Loc(Loc),
         ThisDeclID(thisDeclID), ThisDeclLoc(ThisDeclLoc) {}
 
-  template <typename T>
-  static void AddLazySpecializations(T *D, SmallVectorImpl<GlobalDeclID> &IDs) {
-    if (IDs.empty())
-      return;
-
-    // FIXME: We should avoid this pattern of getting the ASTContext.
-    ASTContext &C = D->getASTContext();
-
-    auto *&LazySpecializations = D->getCommonPtr()->LazySpecializations;
-
-    if (auto &Old = LazySpecializations) {
-      IDs.insert(IDs.end(), Old + 1, Old + 1 + Old[0].getRawValue());
-      llvm::sort(IDs);
-      IDs.erase(std::unique(IDs.begin(), IDs.end()), IDs.end());
-    }
-
-    auto *Result = new (C) GlobalDeclID[1 + IDs.size()];
-    *Result = GlobalDeclID(IDs.size());
-
-    std::copy(IDs.begin(), IDs.end(), Result + 1);
-
-    LazySpecializations = Result;
-  }
-
   template <typename DeclT>
   static Decl *getMostRecentDeclImpl(Redeclarable<DeclT> *D);
   static Decl *getMostRecentDeclImpl(...);
@@ -332,10 +303,13 @@ class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
   static void markIncompleteDeclChainImpl(Redeclarable<DeclT> *D);
   static void markIncompleteDeclChainImpl(...);
 
+  void ReadSpecializations(ModuleFile &M, Decl *D,
+                           llvm::BitstreamCursor &DeclsCursor, bool IsPartial);
+
   void ReadFunctionDefinition(FunctionDecl *FD);
   void Visit(Decl *D);
 
-  void UpdateDecl(Decl *D, SmallVectorImpl<GlobalDeclID> &);
+  void UpdateDecl(Decl *D);
 
   static void setNextObjCCategory(ObjCCategoryDecl *Cat,
                                   ObjCCategoryDecl *Next) {
@@ -2418,6 +2392,16 @@ void ASTDeclReader::VisitImplicitConceptSpecializationDecl(
 void ASTDeclReader::VisitRequiresExprBodyDecl(RequiresExprBodyDecl *D) {
 }
 
+void ASTDeclReader::ReadSpecializations(ModuleFile &M, Decl *D,
+                                        llvm::BitstreamCursor &DeclsCursor,
+                                        bool IsPartial) {
+  uint64_t Offset = ReadLocalOffset();
+  bool Failed =
+      Reader.ReadSpecializations(M, DeclsCursor, Offset, D, IsPartial);
+  (void)Failed;
+  assert(!Failed);
+}
+
 RedeclarableResult
 ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   RedeclarableResult Redecl = VisitRedeclarable(D);
@@ -2456,9 +2440,8 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   if (ThisDeclID == Redecl.getFirstID()) {
     // This ClassTemplateDecl owns a CommonPtr; read it to keep track of all of
     // the specializations.
-    SmallVector<GlobalDeclID, 32> SpecIDs;
-    readDeclIDList(SpecIDs);
-    ASTDeclReader::AddLazySpecializations(D, SpecIDs);
+    ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/false);
+    ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/true);
   }
 
   if (D->getTemplatedDecl()->TemplateOrInstantiation) {
@@ -2484,9 +2467,8 @@ void ASTDeclReader::VisitVarTemplateDecl(VarTemplateDecl *D) {
   if (ThisDeclID == Redecl.getFirstID()) {
     // This VarTemplateDecl owns a CommonPtr; read it to keep track of all of
     // the specializations.
-    SmallVector<GlobalDeclID, 32> SpecIDs;
-    readDeclIDList(SpecIDs);
-    ASTDeclReader::AddLazySpecializations(D, SpecIDs);
+    ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/false);
+    ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/true);
   }
 }
 
@@ -2585,9 +2567,7 @@ void ASTDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
 
   if (ThisDeclID == Redecl.getFirstID()) {
     // This FunctionTemplateDecl owns a CommonPtr; read it.
-    SmallVector<GlobalDeclID, 32> SpecIDs;
-    readDeclIDList(SpecIDs);
-    ASTDeclReader::AddLazySpecializations(D, SpecIDs);
+    ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/false);
   }
 }
 
@@ -3877,6 +3857,8 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
   switch ((DeclCode)MaybeDeclCode.get()) {
   case DECL_CONTEXT_LEXICAL:
   case DECL_CONTEXT_VISIBLE:
+  case DECL_SPECIALIZATIONS:
+  case DECL_PARTIAL_SPECIALIZATIONS:
     llvm_unreachable("Record cannot be de-serialized with readDeclRecord");
   case DECL_TYPEDEF:
     D = TypedefDecl::CreateDeserialized(Context, ID);
@@ -4286,8 +4268,6 @@ void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
   ProcessingUpdatesRAIIObj ProcessingUpdates(*this);
   DeclUpdateOffsetsMap::iterator UpdI = DeclUpdateOffsets.find(ID);
 
-  SmallVector<GlobalDeclID, 8> PendingLazySpecializationIDs;
-
   if (UpdI != DeclUpdateOffsets.end()) {
     auto UpdateOffsets = std::move(UpdI->second);
     DeclUpdateOffsets.erase(UpdI);
@@ -4324,7 +4304,7 @@ void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
 
       ASTDeclReader Reader(*this, Record, RecordLocation(F, Offset), ID,
                            SourceLocation());
-      Reader.UpdateDecl(D, PendingLazySpecializationIDs);
+      Reader.UpdateDecl(D);
 
       // We might have made this declaration interesting. If so, remember that
       // we need to hand it off to the consumer.
@@ -4334,17 +4314,6 @@ void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
       }
     }
   }
-  // Add the lazy specializations to the template.
-  assert((PendingLazySpecializationIDs.empty() || isa<ClassTemplateDecl>(D) ||
-          isa<FunctionTemplateDecl, VarTemplateDecl>(D)) &&
-         "Must not have pending specializations");
-  if (auto *CTD = dyn_cast<ClassTemplateDecl>(D))
-    ASTDeclReader::AddLazySpecializations(CTD, PendingLazySpecializationIDs);
-  else if (auto *FTD = dyn_cast<FunctionTemplateDecl>(D))
-    ASTDeclReader::AddLazySpecializations(FTD, PendingLazySpecializationIDs);
-  else if (auto *VTD = dyn_cast<VarTemplateDecl>(D))
-    ASTDeclReader::AddLazySpecializations(VTD, PendingLazySpecializationIDs);
-  PendingLazySpecializationIDs.clear();
 
   // Load the pending visible updates for this decl context, if it has any.
   auto I = PendingVisibleUpdates.find(ID);
@@ -4369,6 +4338,26 @@ void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
       FunctionToLambdasMap.erase(IT);
     }
   }
+
+  // Load the pending specializations update for this decl, if it has any.
+  if (auto I = PendingSpecializationsUpdates.find(ID);
+      I != PendingSpecializationsUpdates.end()) {
+    auto SpecializationUpdates = std::move(I->second);
+    PendingSpecializationsUpdates.erase(I);
+
+    for (const auto &Update : SpecializationUpdates)
+      AddSpecializations(D, Update.Data, *Update.Mod, /*IsPartial=*/false);
+  }
+
+  // Load the pending specializations update for this decl, if it has any.
+  if (auto I = PendingPartialSpecializationsUpdates.find(ID);
+      I != PendingPartialSpecializationsUpdates.end()) {
+    auto SpecializationUpdates = std::move(I->second);
+    PendingPartialSpecializationsUpdates.erase(I);
+
+    for (const auto &Update : SpecializationUpdates)
+      AddSpecializations(D, Update.Data, *Update.Mod, /*IsPartial=*/true);
+  }
 }
 
 void ASTReader::loadPendingDeclChain(Decl *FirstLocal, uint64_t LocalOffset) {
@@ -4561,9 +4550,7 @@ static void forAllLaterRedecls(DeclT *D, Fn F) {
   }
 }
 
-void ASTDeclReader::UpdateDecl(
-    Decl *D,
-    llvm::SmallVectorImpl<GlobalDeclID> &PendingLazySpecializationIDs) {
+void ASTDeclReader::UpdateDecl(Decl *D) {
   while (Record.getIdx() < Record.size()) {
     switch ((DeclUpdateKind)Record.readInt()) {
     case UPD_CXX_ADDED_IMPLICIT_MEMBER: {
@@ -4574,11 +4561,6 @@ void ASTDeclReader::UpdateDecl(
       break;
     }
 
-    case UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION:
-      // It will be added to the template's lazy specialization set.
-      PendingLazySpecializationIDs.push_back(readDeclID());
-      break;
-
     case UPD_CXX_ADDED_ANONYMOUS_NAMESPACE: {
       auto *Anon = readDeclAs<NamespaceDecl>();
 
@@ -4616,7 +4598,7 @@ void ASTDeclReader::UpdateDecl(
                     .dyn_cast<FunctionTemplateSpecializationInfo *>())
           FTSInfo->setPointOfInstantiation(POI);
         else
-          FD->TemplateOrSpecialization.get<MemberSpecializationInfo *>()
+          cast<MemberSpecializationInfo *>(FD->TemplateOrSpecialization)
               ->setPointOfInstantiation(POI);
       }
       break;
@@ -4715,8 +4697,8 @@ void ASTDeclReader::UpdateDecl(
 
           // FIXME: If we already have a partial specialization set,
           // check that it matches.
-          if (!Spec->getSpecializedTemplateOrPartial()
-                   .is<ClassTemplatePartialSpecializationDecl *>())
+          if (!isa<ClassTemplatePartialSpecializationDecl *>(
+                  Spec->getSpecializedTemplateOrPartial()))
             Spec->setInstantiationOf(PartialSpec, TemplArgList);
         }
       }
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 4f7e6f4b2741b..be0d22d1f4094 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -119,6 +119,88 @@ struct DeclContextLookupTable {
   MultiOnDiskHashTable<ASTDeclContextNameLookupTrait> Table;
 };
 
+using LazySpecializationInfo = GlobalDeclID;
+
+/// Class that performs lookup to specialized decls.
+class LazySpecializationInfoLookupTrait {
+  ASTReader &Reader;
+  ModuleFile &F;
+
+public:
+  // Maximum number of lookup tables we allow before condensing the tables.
+  static const int MaxTables = 4;
+
+  /// The lookup result is a list of global declaration IDs.
+  using data_type = SmallVector<LazySpecializationInfo, 4>;
+
+  struct data_type_builder {
+    data_type &Data;
+    llvm::DenseSet<LazySpecializationInfo> Found;
+
+    data_type_builder(data_type &D) : Data(D) {}
+
+    void insert(LazySpecializationInfo Info) {
+      // Just use a linear scan unless we have more than a few IDs.
+      if (Found.empty() && !Data.empty()) {
+        if (Data.size() <= 4) {
+          for (auto I : Found)
+            if (I == Info)
+              return;
+          Data.push_back(Info);
+          return;
+        }
+
+        // Switch to tracking found IDs in the set.
+        Found.insert(Data.begin(), Data.end());
+      }
+
+      if (Found.insert(Info).second)
+        Data.push_back(Info);
+    }
+  };
+  using hash_value_type = unsigned;
+  using offset_type = unsigned;
+  using file_type = ModuleFile *;
+
+  using external_key_type = unsigned;
+  using internal_key_type = unsigned;
+
+  explicit LazySpecializationInfoLookupTrait(ASTReader &Reader, ModuleFile &F)
+      : Reader(Reader), F(F) {}
+
+  static bool EqualKey(const internal_key_type &a, const internal_key_type &b) {
+    return a == b;
+  }
+
+  static hash_value_type ComputeHash(const internal_key_type &Key) {
+    return Key;
+  }
+
+  static internal_key_type GetInternalKey(const external_key_type &Name) {
+    return Name;
+  }
+
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char *&d);
+
+  internal_key_type ReadKey(const unsigned char *d, unsigned);
+
+  void ReadDataInto(internal_key_type, const unsigned char *d, unsigned DataLen,
+                    data_type_builder &Val);
+
+  static void MergeDataInto(const data_type &From, data_type_builder &To) {
+    To.Data.reserve(To.Data.size() + From.size());
+    for (LazySpecializationInfo Info : From)
+      To.insert(Info);
+  }
+
+  file_type ReadFileRef(const unsigned char *&d);
+};
+
+struct LazySpecializationInfoLookupTable {
+  MultiOnDiskHashTable<LazySpecializationInfoLookupTrait> Table;
+};
+
 /// Base class for the trait describing the on-disk hash table for the
 /// identifiers in an AST file.
 ///
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 4e5d39c81e803..0a0f8115730cf 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -921,9 +921,9 @@ void ASTStmtReader::VisitRequiresExpr(RequiresExpr *E) {
                   std::move(*Req), Status, SubstitutedConstraintExpr);
         else
           R = new (Record.getContext()) concepts::ExprRequirement(
-                  E.get<concepts::Requirement::SubstitutionDiagnostic *>(),
-                  RK == concepts::Requirement::RK_Simple, NoexceptLoc,
-                  std::move(*Req));
+              cast<concepts::Requirement::SubstitutionDiagnostic *>(E),
+              RK == concepts::Requirement::RK_Simple, NoexceptLoc,
+              std::move(*Req));
       } break;
       case concepts::Requirement::RK_Nested: {
         ASTContext &C = Record.getContext();
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a52d59c61c4ce..820da588a75f5 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -13,6 +13,7 @@
 #include "ASTCommon.h"
 #include "ASTReaderInternals.h"
 #include "MultiOnDiskHashTable.h"
+#include "TemplateArgumentHasher.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTUnresolvedSet.h"
 #include "clang/AST/AbstractTypeWriter.h"
@@ -184,14 +185,30 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
   const SourceManager &SM = PP.getSourceManager();
   const ModuleMap &MM = HS.getModuleMap();
 
-  llvm::DenseSet<FileID> ModuleMaps;
-
-  llvm::DenseSet<const Module *> ProcessedModules;
-  auto CollectModuleMapsForHierarchy = [&](const Module *M) {
+  // Module maps used only by textual headers are special. Their FileID is
+  // non-affecting, but their FileEntry is (i.e. must be written as InputFile).
+  enum AffectedReason : bool {
+    AR_TextualHeader = 0,
+    AR_ImportOrTextualHeader = 1,
+  };
+  auto AssignMostImportant = [](AffectedReason &LHS, AffectedReason RHS) {
+    LHS = std::max(LHS, RHS);
+  };
+  llvm::DenseMap<FileID, AffectedReason> ModuleMaps;
+  llvm::DenseMap<const Module *, AffectedReason> ProcessedModules;
+  auto CollectModuleMapsForHierarchy = [&](const Module *M,
+                                           AffectedReason Reason) {
     M = M->getTopLevelModule();
 
-    if (!ProcessedModules.insert(M).second)
+    // We need to process the header either when it was not present or when we
+    // previously flagged module map as textual headers and now we found a
+    // proper import.
+    if (auto [It, Inserted] = ProcessedModules.insert({M, Reason});
+        !Inserted && Reason <= It->second) {
       return;
+    } else {
+      It->second = Reason;
+    }
 
     std::queue<const Module *> Q;
     Q.push(M);
@@ -202,12 +219,12 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
       // The containing module map is affecting, because it's being pointed
       // into by Module::DefinitionLoc.
       if (auto F = MM.getContainingModuleMapFileID(Mod); F.isValid())
-        ModuleMaps.insert(F);
+        AssignMostImportant(ModuleMaps[F], Reason);
       // For inferred modules, the module map that allowed inferring is not
       // related to the virtual containing module map file. It did affect the
       // compilation, though.
       if (auto UniqF = MM.getModuleMapFileIDForUniquing(Mod); UniqF.isValid())
-        ModuleMaps.insert(UniqF);
+        AssignMostImportant(ModuleMaps[UniqF], Reason);
 
       for (auto *SubM : Mod->submodules())
         Q.push(SubM);
@@ -216,7 +233,7 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
 
   // Handle all the affecting modules referenced from the root module.
 
-  CollectModuleMapsForHierarchy(RootModule);
+  CollectModuleMapsForHierarchy(RootModule, AR_ImportOrTextualHeader);
 
   std::queue<const Module *> Q;
   Q.push(RootModule);
@@ -225,9 +242,9 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
     Q.pop();
 
     for (const Module *ImportedModule : CurrentModule->Imports)
-      CollectModuleMapsForHierarchy(ImportedModule);
+      CollectModuleMapsForHierarchy(ImportedModule, AR_ImportOrTextualHeader);
     for (const Module *UndeclaredModule : CurrentModule->UndeclaredUses)
-      CollectModuleMapsForHierarchy(UndeclaredModule);
+      CollectModuleMapsForHierarchy(UndeclaredModule, AR_ImportOrTextualHeader);
 
     for (auto *M : CurrentModule->submodules())
       Q.push(M);
@@ -256,7 +273,7 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
 
     for (const auto &KH : HS.findResolvedModulesForHeader(*File))
       if (const Module *M = KH.getModule())
-        CollectModuleMapsForHierarchy(M);
+        CollectModuleMapsForHierarchy(M, AR_TextualHeader);
   }
 
   // FIXME: This algorithm is not correct for module map hierarchies where
@@ -278,13 +295,16 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
   // includes a module map defining a module that's not a submodule of X.
 
   llvm::DenseSet<const FileEntry *> ModuleFileEntries;
-  for (FileID MM : ModuleMaps) {
-    if (auto *FE = SM.getFileEntryForID(MM))
+  llvm::DenseSet<FileID> ModuleFileIDs;
+  for (auto [FID, Reason] : ModuleMaps) {
+    if (Reason == AR_ImportOrTextualHeader)
+      ModuleFileIDs.insert(FID);
+    if (auto *FE = SM.getFileEntryForID(FID))
       ModuleFileEntries.insert(FE);
   }
 
   AffectingModuleMaps R;
-  R.DefinitionFileIDs = std::move(ModuleMaps);
+  R.DefinitionFileIDs = std::move(ModuleFileIDs);
   R.DefinitionFiles = std::move(ModuleFileEntries);
   return std::move(R);
 }
@@ -4148,6 +4168,175 @@ class ASTDeclContextNameLookupTrait {
 
 } // namespace
 
+namespace {
+class LazySpecializationInfoLookupTrait {
+  ASTWriter &Writer;
+  llvm::SmallVector<serialization::reader::LazySpecializationInfo, 64> Specs;
+
+public:
+  using key_type = unsigned;
+  using key_type_ref = key_type;
+
+  /// A start and end index into Specs, representing a sequence of decls.
+  using data_type = std::pair<unsigned, unsigned>;
+  using data_type_ref = const data_type &;
+
+  using hash_value_type = unsigned;
+  using offset_type = unsigned;
+
+  explicit LazySpecializationInfoLookupTrait(ASTWriter &Writer)
+      : Writer(Writer) {}
+
+  template <typename Col, typename Col2>
+  data_type getData(Col &&C, Col2 &ExistingInfo) {
+    unsigned Start = Specs.size();
+    for (auto *D : C) {
+      NamedDecl *ND = getDeclForLocalLookup(Writer.getLangOpts(),
+                                            const_cast<NamedDecl *>(D));
+      Specs.push_back(GlobalDeclID(Writer.GetDeclRef(ND).getRawValue()));
+    }
+    for (const serialization::reader::LazySpecializationInfo &Info :
+         ExistingInfo)
+      Specs.push_back(Info);
+    return std::make_pair(Start, Specs.size());
+  }
+
+  data_type ImportData(
+      const reader::LazySpecializationInfoLookupTrait::data_type &FromReader) {
+    unsigned Start = Specs.size();
+    for (auto ID : FromReader)
+      Specs.push_back(ID);
+    return std::make_pair(Start, Specs.size());
+  }
+
+  static bool EqualKey(key_type_ref a, key_type_ref b) { return a == b; }
+
+  hash_value_type ComputeHash(key_type Name) { return Name; }
+
+  void EmitFileRef(raw_ostream &Out, ModuleFile *F) const {
+    assert(Writer.hasChain() &&
+           "have reference to loaded module file but no chain?");
+
+    using namespace llvm::support;
+    endian::write<uint32_t>(Out, Writer.getChain()->getModuleFileID(F),
+                            llvm::endianness::little);
+  }
+
+  std::pair<unsigned, unsigned> EmitKeyDataLength(raw_ostream &Out,
+                                                  key_type HashValue,
+                                                  data_type_ref Lookup) {
+    // 4 bytes for each slot.
+    unsigned KeyLen = 4;
+    unsigned DataLen = sizeof(serialization::reader::LazySpecializationInfo) *
+                       (Lookup.second - Lookup.first);
+
+    return emitULEBKeyDataLength(KeyLen, DataLen, Out);
+  }
+
+  void EmitKey(raw_ostream &Out, key_type HashValue, unsigned) {
+    using namespace llvm::support;
+
+    endian::Writer LE(Out, llvm::endianness::little);
+    LE.write<uint32_t>(HashValue);
+  }
+
+  void EmitData(raw_ostream &Out, key_type_ref, data_type Lookup,
+                unsigned DataLen) {
+    using namespace llvm::support;
+
+    endian::Writer LE(Out, llvm::endianness::little);
+    uint64_t Start = Out.tell();
+    (void)Start;
+    for (unsigned I = Lookup.first, N = Lookup.second; I != N; ++I) {
+      LE.write<DeclID>(Specs[I].getRawValue());
+    }
+    assert(Out.tell() - Start == DataLen && "Data length is wrong");
+  }
+};
+
+unsigned CalculateODRHashForSpecs(const Decl *Spec) {
+  ArrayRef<TemplateArgument> Args;
+  if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(Spec))
+    Args = CTSD->getTemplateArgs().asArray();
+  else if (auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(Spec))
+    Args = VTSD->getTemplateArgs().asArray();
+  else if (auto *FD = dyn_cast<FunctionDecl>(Spec))
+    Args = FD->getTemplateSpecializationArgs()->asArray();
+  else
+    llvm_unreachable("New Specialization Kind?");
+
+  return StableHashForTemplateArguments(Args);
+}
+} // namespace
+
+void ASTWriter::GenerateSpecializationInfoLookupTable(
+    const NamedDecl *D, llvm::SmallVectorImpl<const Decl *> &Specializations,
+    llvm::SmallVectorImpl<char> &LookupTable, bool IsPartial) {
+  assert(D->isFirstDecl());
+
+  // Create the on-disk hash table representation.
+  MultiOnDiskHashTableGenerator<reader::LazySpecializationInfoLookupTrait,
+                                LazySpecializationInfoLookupTrait>
+      Generator;
+  LazySpecializationInfoLookupTrait Trait(*this);
+
+  llvm::DenseMap<unsigned, llvm::SmallVector<const NamedDecl *, 4>>
+      SpecializationMaps;
+
+  for (auto *Specialization : Specializations) {
+    unsigned HashedValue = CalculateODRHashForSpecs(Specialization);
+
+    auto Iter = SpecializationMaps.find(HashedValue);
+    if (Iter == SpecializationMaps.end())
+      Iter = SpecializationMaps
+                 .try_emplace(HashedValue,
+                              llvm::SmallVector<const NamedDecl *, 4>())
+                 .first;
+
+    Iter->second.push_back(cast<NamedDecl>(Specialization));
+  }
+
+  auto *Lookups =
+      Chain ? Chain->getLoadedSpecializationsLookupTables(D, IsPartial)
+            : nullptr;
+
+  for (auto &[HashValue, Specs] : SpecializationMaps) {
+    SmallVector<serialization::reader::LazySpecializationInfo, 16>
+        ExisitingSpecs;
+    // We have to merge the lookup table manually here. We can't depend on the
+    // merge mechanism offered by
+    // clang::serialization::MultiOnDiskHashTableGenerator since that generator
+    // assumes the we'll get the same value with the same key.
+    // And also underlying llvm::OnDiskChainedHashTableGenerator assumes that we
+    // won't insert the values with the same key twice. So we have to merge the
+    // lookup table here manually.
+    if (Lookups)
+      ExisitingSpecs = Lookups->Table.find(HashValue);
+
+    Generator.insert(HashValue, Trait.getData(Specs, ExisitingSpecs), Trait);
+  }
+
+  Generator.emit(LookupTable, Trait, Lookups ? &Lookups->Table : nullptr);
+}
+
+uint64_t ASTWriter::WriteSpecializationInfoLookupTable(
+    const NamedDecl *D, llvm::SmallVectorImpl<const Decl *> &Specializations,
+    bool IsPartial) {
+
+  llvm::SmallString<4096> LookupTable;
+  GenerateSpecializationInfoLookupTable(D, Specializations, LookupTable,
+                                        IsPartial);
+
+  uint64_t Offset = Stream.GetCurrentBitNo();
+  RecordData::value_type Record[] = {static_cast<RecordData::value_type>(
+      IsPartial ? DECL_PARTIAL_SPECIALIZATIONS : DECL_SPECIALIZATIONS)};
+  Stream.EmitRecordWithBlob(IsPartial ? DeclPartialSpecializationsAbbrev
+                                      : DeclSpecializationsAbbrev,
+                            Record, LookupTable);
+
+  return Offset;
+}
+
 bool ASTWriter::isLookupResultExternal(StoredDeclsList &Result,
                                        DeclContext *DC) {
   return Result.hasExternalDecls() &&
@@ -4922,7 +5111,7 @@ ASTWriter::WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject,
 
   Sema *SemaPtr = Subject.dyn_cast<Sema *>();
   Preprocessor &PPRef =
-      SemaPtr ? SemaPtr->getPreprocessor() : *Subject.get<Preprocessor *>();
+      SemaPtr ? SemaPtr->getPreprocessor() : *cast<Preprocessor *>(Subject);
 
   ASTHasCompilerErrors = PPRef.getDiagnostics().hasUncompilableErrorOccurred();
 
@@ -5729,7 +5918,7 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
   // Keep writing types, declarations, and declaration update records
   // until we've emitted all of them.
   RecordData DeclUpdatesOffsetsRecord;
-  Stream.EnterSubblock(DECLTYPES_BLOCK_ID, /*bits for abbreviations*/5);
+  Stream.EnterSubblock(DECLTYPES_BLOCK_ID, /*bits for abbreviations*/ 6);
   DeclTypesBlockStartOffset = Stream.GetCurrentBitNo();
   WriteTypeAbbrevs();
   WriteDeclAbbrevs();
@@ -5803,6 +5992,16 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
                       FunctionToLambdaMapAbbrev);
   }
 
+  if (!SpecializationsUpdates.empty()) {
+    WriteSpecializationsUpdates(/*IsPartial=*/false);
+    SpecializationsUpdates.clear();
+  }
+
+  if (!PartialSpecializationsUpdates.empty()) {
+    WriteSpecializationsUpdates(/*IsPartial=*/true);
+    PartialSpecializationsUpdates.clear();
+  }
+
   const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
   // Create a lexical update block containing all of the declarations in the
   // translation unit that do not come from other AST files.
@@ -5846,6 +6045,33 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
     WriteDeclContextVisibleUpdate(Context, DC);
 }
 
+void ASTWriter::WriteSpecializationsUpdates(bool IsPartial) {
+  auto RecordType = IsPartial ? CXX_ADDED_TEMPLATE_PARTIAL_SPECIALIZATION
+                              : CXX_ADDED_TEMPLATE_SPECIALIZATION;
+
+  auto Abv = std::make_shared<llvm::BitCodeAbbrev>();
+  Abv->Add(llvm::BitCodeAbbrevOp(RecordType));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  auto UpdateSpecializationAbbrev = Stream.EmitAbbrev(std::move(Abv));
+
+  auto &SpecUpdates =
+      IsPartial ? PartialSpecializationsUpdates : SpecializationsUpdates;
+  for (auto &SpecializationUpdate : SpecUpdates) {
+    const NamedDecl *D = SpecializationUpdate.first;
+
+    llvm::SmallString<4096> LookupTable;
+    GenerateSpecializationInfoLookupTable(D, SpecializationUpdate.second,
+                                          LookupTable, IsPartial);
+
+    // Write the lookup table
+    RecordData::value_type Record[] = {
+        static_cast<RecordData::value_type>(RecordType),
+        getDeclID(D).getRawValue()};
+    Stream.EmitRecordWithBlob(UpdateSpecializationAbbrev, Record, LookupTable);
+  }
+}
+
 void ASTWriter::WriteDeclUpdatesBlocks(ASTContext &Context,
                                        RecordDataImpl &OffsetsRecord) {
   if (DeclUpdates.empty())
@@ -5875,12 +6101,10 @@ void ASTWriter::WriteDeclUpdatesBlocks(ASTContext &Context,
 
       switch (Kind) {
       case UPD_CXX_ADDED_IMPLICIT_MEMBER:
-      case UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION:
       case UPD_CXX_ADDED_ANONYMOUS_NAMESPACE:
         assert(Update.getDecl() && "no decl to add?");
         Record.AddDeclRef(Update.getDecl());
         break;
-
       case UPD_CXX_ADDED_FUNCTION_DEFINITION:
       case UPD_CXX_ADDED_VAR_DEFINITION:
         break;
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index ad357e30d5752..68e2d6b399390 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -177,11 +177,12 @@ namespace clang {
       Record.AddSourceLocation(typeParams->getRAngleLoc());
     }
 
-    /// Add to the record the first declaration from each module file that
-    /// provides a declaration of D. The intent is to provide a sufficient
-    /// set such that reloading this set will load all current redeclarations.
-    void AddFirstDeclFromEachModule(const Decl *D, bool IncludeLocal) {
-      llvm::MapVector<ModuleFile*, const Decl*> Firsts;
+    /// Collect the first declaration from each module file that provides a
+    /// declaration of D.
+    void CollectFirstDeclFromEachModule(
+        const Decl *D, bool IncludeLocal,
+        llvm::MapVector<ModuleFile *, const Decl *> &Firsts) {
+
       // FIXME: We can skip entries that we know are implied by others.
       for (const Decl *R = D->getMostRecentDecl(); R; R = R->getPreviousDecl()) {
         if (R->isFromASTFile())
@@ -189,10 +190,41 @@ namespace clang {
         else if (IncludeLocal)
           Firsts[nullptr] = R;
       }
+    }
+
+    /// Add to the record the first declaration from each module file that
+    /// provides a declaration of D. The intent is to provide a sufficient
+    /// set such that reloading this set will load all current redeclarations.
+    void AddFirstDeclFromEachModule(const Decl *D, bool IncludeLocal) {
+      llvm::MapVector<ModuleFile *, const Decl *> Firsts;
+      CollectFirstDeclFromEachModule(D, IncludeLocal, Firsts);
+
       for (const auto &F : Firsts)
         Record.AddDeclRef(F.second);
     }
 
+    /// Add to the record the first template specialization from each module
+    /// file that provides a declaration of D. We store the DeclId and an
+    /// ODRHash of the template arguments of D which should provide enough
+    /// information to load D only if the template instantiator needs it.
+    void AddFirstSpecializationDeclFromEachModule(
+        const Decl *D, llvm::SmallVectorImpl<const Decl *> &SpecsInMap,
+        llvm::SmallVectorImpl<const Decl *> &PartialSpecsInMap) {
+      assert((isa<ClassTemplateSpecializationDecl>(D) ||
+              isa<VarTemplateSpecializationDecl>(D) || isa<FunctionDecl>(D)) &&
+             "Must not be called with other decls");
+      llvm::MapVector<ModuleFile *, const Decl *> Firsts;
+      CollectFirstDeclFromEachModule(D, /*IncludeLocal*/ true, Firsts);
+
+      for (const auto &F : Firsts) {
+        if (isa<ClassTemplatePartialSpecializationDecl,
+                VarTemplatePartialSpecializationDecl>(F.second))
+          PartialSpecsInMap.push_back(F.second);
+        else
+          SpecsInMap.push_back(F.second);
+      }
+    }
+
     /// Get the specialization decl from an entry in the specialization list.
     template <typename EntryType>
     typename RedeclarableTemplateDecl::SpecEntryTraits<EntryType>::DeclType *
@@ -205,8 +237,9 @@ namespace clang {
     decltype(T::PartialSpecializations) &getPartialSpecializations(T *Common) {
       return Common->PartialSpecializations;
     }
-    ArrayRef<Decl> getPartialSpecializations(FunctionTemplateDecl::Common *) {
-      return {};
+    MutableArrayRef<FunctionTemplateSpecializationInfo>
+    getPartialSpecializations(FunctionTemplateDecl::Common *) {
+      return std::nullopt;
     }
 
     template<typename DeclTy>
@@ -217,37 +250,37 @@ namespace clang {
       // our chained AST reader, we can just write out the DeclIDs. Otherwise,
       // we need to resolve them to actual declarations.
       if (Writer.Chain != Record.getASTContext().getExternalSource() &&
-          Common->LazySpecializations) {
+          Writer.Chain && Writer.Chain->haveUnloadedSpecializations(D)) {
         D->LoadLazySpecializations();
-        assert(!Common->LazySpecializations);
+        assert(!Writer.Chain->haveUnloadedSpecializations(D));
       }
 
-      ArrayRef<GlobalDeclID> LazySpecializations;
-      if (auto *LS = Common->LazySpecializations)
-        LazySpecializations = llvm::ArrayRef(LS + 1, LS[0].getRawValue());
-
-      // Add a slot to the record for the number of specializations.
-      unsigned I = Record.size();
-      Record.push_back(0);
-
-      // AddFirstDeclFromEachModule might trigger deserialization, invalidating
-      // *Specializations iterators.
-      llvm::SmallVector<const Decl*, 16> Specs;
+      // AddFirstSpecializationDeclFromEachModule might trigger deserialization,
+      // invalidating *Specializations iterators.
+      llvm::SmallVector<const Decl *, 16> AllSpecs;
       for (auto &Entry : Common->Specializations)
-        Specs.push_back(getSpecializationDecl(Entry));
+        AllSpecs.push_back(getSpecializationDecl(Entry));
       for (auto &Entry : getPartialSpecializations(Common))
-        Specs.push_back(getSpecializationDecl(Entry));
+        AllSpecs.push_back(getSpecializationDecl(Entry));
 
-      for (auto *D : Specs) {
+      llvm::SmallVector<const Decl *, 16> Specs;
+      llvm::SmallVector<const Decl *, 16> PartialSpecs;
+      for (auto *D : AllSpecs) {
         assert(D->isCanonicalDecl() && "non-canonical decl in set");
-        AddFirstDeclFromEachModule(D, /*IncludeLocal*/true);
+        AddFirstSpecializationDeclFromEachModule(D, Specs, PartialSpecs);
+      }
+
+      Record.AddOffset(Writer.WriteSpecializationInfoLookupTable(
+          D, Specs, /*IsPartial=*/false));
+
+      // Function Template Decl doesn't have partial decls.
+      if (isa<FunctionTemplateDecl>(D)) {
+        assert(PartialSpecs.empty());
+        return;
       }
-      Record.append(
-          DeclIDIterator<GlobalDeclID, DeclID>(LazySpecializations.begin()),
-          DeclIDIterator<GlobalDeclID, DeclID>(LazySpecializations.end()));
 
-      // Update the size entry we added earlier.
-      Record[I] = Record.size() - I - 1;
+      Record.AddOffset(Writer.WriteSpecializationInfoLookupTable(
+          D, PartialSpecs, /*IsPartial=*/true));
     }
 
     /// Ensure that this template specialization is associated with the specified
@@ -268,8 +301,13 @@ namespace clang {
       if (Writer.getFirstLocalDecl(Specialization) != Specialization)
         return;
 
-      Writer.DeclUpdates[Template].push_back(ASTWriter::DeclUpdate(
-          UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION, Specialization));
+      if (isa<ClassTemplatePartialSpecializationDecl,
+              VarTemplatePartialSpecializationDecl>(Specialization))
+        Writer.PartialSpecializationsUpdates[cast<NamedDecl>(Template)]
+            .push_back(cast<NamedDecl>(Specialization));
+      else
+        Writer.SpecializationsUpdates[cast<NamedDecl>(Template)].push_back(
+            cast<NamedDecl>(Specialization));
     }
   };
 }
@@ -1656,7 +1694,7 @@ void ASTDeclWriter::VisitFriendDecl(FriendDecl *D) {
   // so as to simplify memory allocation during deserialization.
   Record.push_back(D->NumTPLists);
   VisitDecl(D);
-  bool hasFriendDecl = D->Friend.is<NamedDecl*>();
+  bool hasFriendDecl = isa<NamedDecl *>(D->Friend);
   Record.push_back(hasFriendDecl);
   if (hasFriendDecl)
     Record.AddDeclRef(D->getFriendDecl());
@@ -1757,7 +1795,7 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
   if (Decl *InstFromD = InstFrom.dyn_cast<ClassTemplateDecl *>()) {
     Record.AddDeclRef(InstFromD);
   } else {
-    Record.AddDeclRef(InstFrom.get<ClassTemplatePartialSpecializationDecl *>());
+    Record.AddDeclRef(cast<ClassTemplatePartialSpecializationDecl *>(InstFrom));
     Record.AddTemplateArgumentList(&D->getTemplateInstantiationArgs());
   }
 
@@ -1835,7 +1873,7 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
   if (Decl *InstFromD = InstFrom.dyn_cast<VarTemplateDecl *>()) {
     Record.AddDeclRef(InstFromD);
   } else {
-    Record.AddDeclRef(InstFrom.get<VarTemplatePartialSpecializationDecl *>());
+    Record.AddDeclRef(cast<VarTemplatePartialSpecializationDecl *>(InstFrom));
     Record.AddTemplateArgumentList(&D->getTemplateInstantiationArgs());
   }
 
@@ -2774,6 +2812,16 @@ void ASTWriter::WriteDeclAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(serialization::DECL_CONTEXT_VISIBLE));
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   DeclContextVisibleLookupAbbrev = Stream.EmitAbbrev(std::move(Abv));
+
+  Abv = std::make_shared<BitCodeAbbrev>();
+  Abv->Add(BitCodeAbbrevOp(serialization::DECL_SPECIALIZATIONS));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  DeclSpecializationsAbbrev = Stream.EmitAbbrev(std::move(Abv));
+
+  Abv = std::make_shared<BitCodeAbbrev>();
+  Abv->Add(BitCodeAbbrevOp(serialization::DECL_PARTIAL_SPECIALIZATIONS));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  DeclPartialSpecializationsAbbrev = Stream.EmitAbbrev(std::move(Abv));
 }
 
 /// isRequiredDecl - Check if this is a "required" Decl, which must be seen by
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 05f996b89dc1d..e054cfdf1c627 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -480,7 +480,7 @@ addConstraintSatisfaction(ASTRecordWriter &Record,
       if (E)
         Record.AddStmt(E);
       else {
-        auto *Diag = DetailRecord.get<std::pair<SourceLocation, StringRef> *>();
+        auto *Diag = cast<std::pair<SourceLocation, StringRef> *>(DetailRecord);
         Record.AddSourceLocation(Diag->first);
         Record.AddString(Diag->second);
       }
@@ -532,10 +532,11 @@ void ASTStmtWriter::VisitRequiresExpr(RequiresExpr *E) {
       Record.push_back(ExprReq->getKind());
       Record.push_back(ExprReq->Status);
       if (ExprReq->isExprSubstitutionFailure()) {
-        addSubstitutionDiagnostic(Record,
-         ExprReq->Value.get<concepts::Requirement::SubstitutionDiagnostic *>());
+        addSubstitutionDiagnostic(
+            Record, cast<concepts::Requirement::SubstitutionDiagnostic *>(
+                        ExprReq->Value));
       } else
-        Record.AddStmt(ExprReq->Value.get<Expr *>());
+        Record.AddStmt(cast<Expr *>(ExprReq->Value));
       if (ExprReq->getKind() == concepts::Requirement::RK_Compound) {
         Record.AddSourceLocation(ExprReq->NoexceptLoc);
         const auto &RetReq = ExprReq->getReturnTypeRequirement();
@@ -1177,7 +1178,7 @@ void ASTStmtWriter::VisitInitListExpr(InitListExpr *E) {
   Record.AddStmt(E->getSyntacticForm());
   Record.AddSourceLocation(E->getLBraceLoc());
   Record.AddSourceLocation(E->getRBraceLoc());
-  bool isArrayFiller = E->ArrayFillerOrUnionFieldInit.is<Expr*>();
+  bool isArrayFiller = isa<Expr *>(E->ArrayFillerOrUnionFieldInit);
   Record.push_back(isArrayFiller);
   if (isArrayFiller)
     Record.AddStmt(E->getArrayFiller());
diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt
index 99c47c15a2f47..b1fc0345047f2 100644
--- a/clang/lib/Serialization/CMakeLists.txt
+++ b/clang/lib/Serialization/CMakeLists.txt
@@ -23,6 +23,7 @@ add_clang_library(clangSerialization
   ModuleManager.cpp
   PCHContainerOperations.cpp
   ObjectFilePCHContainerReader.cpp
+  TemplateArgumentHasher.cpp
 
   ADDITIONAL_HEADERS
   ASTCommon.h
diff --git a/clang/lib/Serialization/TemplateArgumentHasher.cpp b/clang/lib/Serialization/TemplateArgumentHasher.cpp
new file mode 100644
index 0000000000000..598f098f526d0
--- /dev/null
+++ b/clang/lib/Serialization/TemplateArgumentHasher.cpp
@@ -0,0 +1,409 @@
+//===- TemplateArgumentHasher.cpp - Hash Template Arguments -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TemplateArgumentHasher.h"
+#include "clang/AST/APValue.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/DeclarationName.h"
+#include "clang/AST/TypeVisitor.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "llvm/ADT/FoldingSet.h"
+
+using namespace clang;
+
+namespace {
+
+class TemplateArgumentHasher {
+  // If we bail out during the process of calculating hash values for
+  // template arguments for any reason. We're allowed to do it since
+  // TemplateArgumentHasher are only required to give the same hash value
+  // for the same template arguments, but not required to give different
+  // hash value for different template arguments.
+  //
+  // So in the worst case, it is still a valid implementation to give all
+  // inputs the same BailedOutValue as output.
+  bool BailedOut = false;
+  static constexpr unsigned BailedOutValue = 0x12345678;
+
+  llvm::FoldingSetNodeID ID;
+
+public:
+  TemplateArgumentHasher() = default;
+
+  void AddTemplateArgument(TemplateArgument TA);
+
+  void AddInteger(unsigned V) { ID.AddInteger(V); }
+
+  unsigned getValue() {
+    if (BailedOut)
+      return BailedOutValue;
+
+    return ID.computeStableHash();
+  }
+
+  void setBailedOut() { BailedOut = true; }
+
+  void AddType(const Type *T);
+  void AddQualType(QualType T);
+  void AddDecl(const Decl *D);
+  void AddStructuralValue(const APValue &);
+  void AddTemplateName(TemplateName Name);
+  void AddDeclarationName(DeclarationName Name);
+  void AddIdentifierInfo(const IdentifierInfo *II);
+};
+
+void TemplateArgumentHasher::AddTemplateArgument(TemplateArgument TA) {
+  const auto Kind = TA.getKind();
+  AddInteger(Kind);
+
+  switch (Kind) {
+  case TemplateArgument::Null:
+    llvm_unreachable("Expected valid TemplateArgument");
+  case TemplateArgument::Type:
+    AddQualType(TA.getAsType());
+    break;
+  case TemplateArgument::Declaration:
+    AddDecl(TA.getAsDecl());
+    break;
+  case TemplateArgument::NullPtr:
+    ID.AddPointer(nullptr);
+    break;
+  case TemplateArgument::Integral: {
+    // There are integrals (e.g.: _BitInt(128)) that cannot be represented as
+    // any builtin integral type, so we use the hash of APSInt instead.
+    TA.getAsIntegral().Profile(ID);
+    break;
+  }
+  case TemplateArgument::StructuralValue:
+    AddQualType(TA.getStructuralValueType());
+    AddStructuralValue(TA.getAsStructuralValue());
+    break;
+  case TemplateArgument::Template:
+  case TemplateArgument::TemplateExpansion:
+    AddTemplateName(TA.getAsTemplateOrTemplatePattern());
+    break;
+  case TemplateArgument::Expression:
+    // If we meet expression in template argument, it implies
+    // that the template is still dependent. It is meaningless
+    // to get a stable hash for the template. Bail out simply.
+    BailedOut = true;
+    break;
+  case TemplateArgument::Pack:
+    AddInteger(TA.pack_size());
+    for (auto SubTA : TA.pack_elements()) {
+      AddTemplateArgument(SubTA);
+    }
+    break;
+  }
+}
+
+void TemplateArgumentHasher::AddStructuralValue(const APValue &Value) {
+  auto Kind = Value.getKind();
+  AddInteger(Kind);
+
+  // 'APValue::Profile' uses pointer values to make hash for LValue and
+  // MemberPointer, but they differ from one compiler invocation to another.
+  // It may be difficult to handle such cases. Bail out simply.
+
+  if (Kind == APValue::LValue || Kind == APValue::MemberPointer) {
+    BailedOut = true;
+    return;
+  }
+
+  Value.Profile(ID);
+}
+
+void TemplateArgumentHasher::AddTemplateName(TemplateName Name) {
+  switch (Name.getKind()) {
+  case TemplateName::Template:
+    AddDecl(Name.getAsTemplateDecl());
+    break;
+  case TemplateName::QualifiedTemplate: {
+    QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName();
+    AddTemplateName(QTN->getUnderlyingTemplate());
+    break;
+  }
+  case TemplateName::OverloadedTemplate:
+  case TemplateName::AssumedTemplate:
+  case TemplateName::DependentTemplate:
+  case TemplateName::SubstTemplateTemplateParm:
+  case TemplateName::SubstTemplateTemplateParmPack:
+    BailedOut = true;
+    break;
+  case TemplateName::UsingTemplate: {
+    UsingShadowDecl *USD = Name.getAsUsingShadowDecl();
+    if (USD)
+      AddDecl(USD->getTargetDecl());
+    else
+      BailedOut = true;
+    break;
+  }
+  case TemplateName::DeducedTemplate:
+    AddTemplateName(Name.getAsDeducedTemplateName()->getUnderlying());
+    break;
+  }
+}
+
+void TemplateArgumentHasher::AddIdentifierInfo(const IdentifierInfo *II) {
+  assert(II && "Expecting non-null pointer.");
+  ID.AddString(II->getName());
+}
+
+void TemplateArgumentHasher::AddDeclarationName(DeclarationName Name) {
+  if (Name.isEmpty())
+    return;
+
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+    AddIdentifierInfo(Name.getAsIdentifierInfo());
+    break;
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    BailedOut = true;
+    break;
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+    AddQualType(Name.getCXXNameType());
+    break;
+  case DeclarationName::CXXOperatorName:
+    AddInteger(Name.getCXXOverloadedOperator());
+    break;
+  case DeclarationName::CXXLiteralOperatorName:
+    AddIdentifierInfo(Name.getCXXLiteralIdentifier());
+    break;
+  case DeclarationName::CXXConversionFunctionName:
+    AddQualType(Name.getCXXNameType());
+    break;
+  case DeclarationName::CXXUsingDirective:
+    break;
+  case DeclarationName::CXXDeductionGuideName: {
+    if (auto *Template = Name.getCXXDeductionGuideTemplate())
+      AddDecl(Template);
+  }
+  }
+}
+
+void TemplateArgumentHasher::AddDecl(const Decl *D) {
+  const NamedDecl *ND = dyn_cast<NamedDecl>(D);
+  if (!ND) {
+    BailedOut = true;
+    return;
+  }
+
+  AddDeclarationName(ND->getDeclName());
+}
+
+void TemplateArgumentHasher::AddQualType(QualType T) {
+  if (T.isNull()) {
+    BailedOut = true;
+    return;
+  }
+  SplitQualType split = T.split();
+  AddInteger(split.Quals.getAsOpaqueValue());
+  AddType(split.Ty);
+}
+
+// Process a Type pointer.  Add* methods call back into TemplateArgumentHasher
+// while Visit* methods process the relevant parts of the Type.
+// Any unhandled type will make the hash computation bail out.
+class TypeVisitorHelper : public TypeVisitor<TypeVisitorHelper> {
+  typedef TypeVisitor<TypeVisitorHelper> Inherited;
+  llvm::FoldingSetNodeID &ID;
+  TemplateArgumentHasher &Hash;
+
+public:
+  TypeVisitorHelper(llvm::FoldingSetNodeID &ID, TemplateArgumentHasher &Hash)
+      : ID(ID), Hash(Hash) {}
+
+  void AddDecl(const Decl *D) {
+    if (D)
+      Hash.AddDecl(D);
+    else
+      Hash.AddInteger(0);
+  }
+
+  void AddQualType(QualType T) { Hash.AddQualType(T); }
+
+  void AddType(const Type *T) {
+    if (T)
+      Hash.AddType(T);
+    else
+      Hash.AddInteger(0);
+  }
+
+  void VisitQualifiers(Qualifiers Quals) {
+    Hash.AddInteger(Quals.getAsOpaqueValue());
+  }
+
+  void Visit(const Type *T) { Inherited::Visit(T); }
+
+  // Unhandled types. Bail out simply.
+  void VisitType(const Type *T) { Hash.setBailedOut(); }
+
+  void VisitAdjustedType(const AdjustedType *T) {
+    AddQualType(T->getOriginalType());
+  }
+
+  void VisitDecayedType(const DecayedType *T) {
+    // getDecayedType and getPointeeType are derived from getAdjustedType
+    // and don't need to be separately processed.
+    VisitAdjustedType(T);
+  }
+
+  void VisitArrayType(const ArrayType *T) {
+    AddQualType(T->getElementType());
+    Hash.AddInteger(llvm::to_underlying(T->getSizeModifier()));
+    VisitQualifiers(T->getIndexTypeQualifiers());
+  }
+  void VisitConstantArrayType(const ConstantArrayType *T) {
+    T->getSize().Profile(ID);
+    VisitArrayType(T);
+  }
+
+  void VisitAttributedType(const AttributedType *T) {
+    Hash.AddInteger(T->getAttrKind());
+    AddQualType(T->getModifiedType());
+  }
+
+  void VisitBuiltinType(const BuiltinType *T) { Hash.AddInteger(T->getKind()); }
+
+  void VisitComplexType(const ComplexType *T) {
+    AddQualType(T->getElementType());
+  }
+
+  void VisitDecltypeType(const DecltypeType *T) {
+    AddQualType(T->getUnderlyingType());
+  }
+
+  void VisitDeducedType(const DeducedType *T) {
+    AddQualType(T->getDeducedType());
+  }
+
+  void VisitAutoType(const AutoType *T) { VisitDeducedType(T); }
+
+  void VisitDeducedTemplateSpecializationType(
+      const DeducedTemplateSpecializationType *T) {
+    Hash.AddTemplateName(T->getTemplateName());
+    VisitDeducedType(T);
+  }
+
+  void VisitFunctionType(const FunctionType *T) {
+    AddQualType(T->getReturnType());
+    T->getExtInfo().Profile(ID);
+    Hash.AddInteger(T->isConst());
+    Hash.AddInteger(T->isVolatile());
+    Hash.AddInteger(T->isRestrict());
+  }
+
+  void VisitFunctionNoProtoType(const FunctionNoProtoType *T) {
+    VisitFunctionType(T);
+  }
+
+  void VisitFunctionProtoType(const FunctionProtoType *T) {
+    Hash.AddInteger(T->getNumParams());
+    for (auto ParamType : T->getParamTypes())
+      AddQualType(ParamType);
+
+    VisitFunctionType(T);
+  }
+
+  void VisitMemberPointerType(const MemberPointerType *T) {
+    AddQualType(T->getPointeeType());
+    AddType(T->getClass());
+  }
+
+  void VisitPackExpansionType(const PackExpansionType *T) {
+    AddQualType(T->getPattern());
+  }
+
+  void VisitParenType(const ParenType *T) { AddQualType(T->getInnerType()); }
+
+  void VisitPointerType(const PointerType *T) {
+    AddQualType(T->getPointeeType());
+  }
+
+  void VisitReferenceType(const ReferenceType *T) {
+    AddQualType(T->getPointeeTypeAsWritten());
+  }
+
+  void VisitLValueReferenceType(const LValueReferenceType *T) {
+    VisitReferenceType(T);
+  }
+
+  void VisitRValueReferenceType(const RValueReferenceType *T) {
+    VisitReferenceType(T);
+  }
+
+  void
+  VisitSubstTemplateTypeParmPackType(const SubstTemplateTypeParmPackType *T) {
+    AddDecl(T->getAssociatedDecl());
+    Hash.AddTemplateArgument(T->getArgumentPack());
+  }
+
+  void VisitSubstTemplateTypeParmType(const SubstTemplateTypeParmType *T) {
+    AddDecl(T->getAssociatedDecl());
+    AddQualType(T->getReplacementType());
+  }
+
+  void VisitTagType(const TagType *T) { AddDecl(T->getDecl()); }
+
+  void VisitRecordType(const RecordType *T) { VisitTagType(T); }
+  void VisitEnumType(const EnumType *T) { VisitTagType(T); }
+
+  void VisitTemplateSpecializationType(const TemplateSpecializationType *T) {
+    Hash.AddInteger(T->template_arguments().size());
+    for (const auto &TA : T->template_arguments()) {
+      Hash.AddTemplateArgument(TA);
+    }
+    Hash.AddTemplateName(T->getTemplateName());
+  }
+
+  void VisitTemplateTypeParmType(const TemplateTypeParmType *T) {
+    Hash.AddInteger(T->getDepth());
+    Hash.AddInteger(T->getIndex());
+    Hash.AddInteger(T->isParameterPack());
+  }
+
+  void VisitTypedefType(const TypedefType *T) { AddDecl(T->getDecl()); }
+
+  void VisitElaboratedType(const ElaboratedType *T) {
+    AddQualType(T->getNamedType());
+  }
+
+  void VisitUnaryTransformType(const UnaryTransformType *T) {
+    AddQualType(T->getUnderlyingType());
+    AddQualType(T->getBaseType());
+  }
+
+  void VisitVectorType(const VectorType *T) {
+    AddQualType(T->getElementType());
+    Hash.AddInteger(T->getNumElements());
+    Hash.AddInteger(llvm::to_underlying(T->getVectorKind()));
+  }
+
+  void VisitExtVectorType(const ExtVectorType *T) { VisitVectorType(T); }
+};
+
+void TemplateArgumentHasher::AddType(const Type *T) {
+  assert(T && "Expecting non-null pointer.");
+  TypeVisitorHelper(ID, *this).Visit(T);
+}
+
+} // namespace
+
+unsigned clang::serialization::StableHashForTemplateArguments(
+    llvm::ArrayRef<TemplateArgument> Args) {
+  TemplateArgumentHasher Hasher;
+  Hasher.AddInteger(Args.size());
+  for (TemplateArgument Arg : Args)
+    Hasher.AddTemplateArgument(Arg);
+  return Hasher.getValue();
+}
diff --git a/clang/lib/Serialization/TemplateArgumentHasher.h b/clang/lib/Serialization/TemplateArgumentHasher.h
new file mode 100644
index 0000000000000..f23f1318afbbf
--- /dev/null
+++ b/clang/lib/Serialization/TemplateArgumentHasher.h
@@ -0,0 +1,34 @@
+//===- TemplateArgumentHasher.h - Hash Template Arguments -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/TemplateBase.h"
+
+namespace clang {
+namespace serialization {
+
+/// Calculate a stable hash value for template arguments. We guarantee that
+/// the same template arguments must have the same hashed values. But we don't
+/// guarantee that the template arguments with the same hashed value are the
+/// same template arguments.
+///
+/// ODR hashing may not be the best mechanism to hash the template
+/// arguments. ODR hashing is (or perhaps, should be) about determining whether
+/// two things are spelled the same way and have the same meaning (as required
+/// by the C++ ODR), whereas what we want here is whether they have the same
+/// meaning regardless of spelling. Maybe we can get away with reusing ODR
+/// hashing anyway, on the basis that any canonical, non-dependent template
+/// argument should have the same (invented) spelling in every translation
+/// unit, but it is not sure that's true in all cases. There may still be cases
+/// where the canonical type includes some aspect of "whatever we saw first",
+/// in which case the ODR hash can differ across translation units for
+/// non-dependent, canonical template arguments that are spelled differently
+/// but have the same meaning. But it is not easy to raise examples.
+unsigned StableHashForTemplateArguments(llvm::ArrayRef<TemplateArgument> Args);
+
+} // namespace serialization
+} // namespace clang
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index f40318f46dea1..fcbe8b864b6e4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -50,7 +50,6 @@ add_clang_library(clangStaticAnalyzerCheckers
   GCDAntipatternChecker.cpp
   GenericTaintChecker.cpp
   GTestChecker.cpp
-  IdenticalExprChecker.cpp
   InnerPointerChecker.cpp
   InvalidatedIteratorChecker.cpp
   cert/InvalidPtrChecker.cpp
@@ -132,6 +131,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   VirtualCallChecker.cpp
   WebKit/RawPtrRefMemberChecker.cpp
   WebKit/ASTUtils.cpp
+  WebKit/MemoryUnsafeCastChecker.cpp
   WebKit/PtrTypesSemantics.cpp
   WebKit/RefCntblBaseVirtualDtorChecker.cpp
   WebKit/RawPtrRefCallArgsChecker.cpp
diff --git a/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
index 3a0a01c23de03..99fc0a953ef17 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
@@ -7,9 +7,13 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines chroot checker, which checks improper use of chroot.
+//  This is described by the SEI Cert C rule POS05-C.
+//  The checker is a warning not a hard failure since it only checks for a
+//  recommended rule.
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ASTContext.h"
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
@@ -25,94 +29,128 @@ using namespace clang;
 using namespace ento;
 
 namespace {
+enum ChrootKind { NO_CHROOT, ROOT_CHANGED, ROOT_CHANGE_FAILED, JAIL_ENTERED };
+} // namespace
 
-// enum value that represent the jail state
-enum Kind { NO_CHROOT, ROOT_CHANGED, JAIL_ENTERED };
-
-bool isRootChanged(intptr_t k) { return k == ROOT_CHANGED; }
-//bool isJailEntered(intptr_t k) { return k == JAIL_ENTERED; }
+// Track chroot state changes for success, failure, state change
+// and "jail"
+REGISTER_TRAIT_WITH_PROGRAMSTATE(ChrootState, ChrootKind)
+namespace {
 
 // This checker checks improper use of chroot.
-// The state transition:
+// The state transitions
+//
+//                          -> ROOT_CHANGE_FAILED
+//                          |
 // NO_CHROOT ---chroot(path)--> ROOT_CHANGED ---chdir(/) --> JAIL_ENTERED
 //                                  |                               |
 //         ROOT_CHANGED<--chdir(..)--      JAIL_ENTERED<--chdir(..)--
 //                                  |                               |
 //                      bug<--foo()--          JAIL_ENTERED<--foo()--
-class ChrootChecker : public Checker<eval::Call, check::PreCall> {
-  // This bug refers to possibly break out of a chroot() jail.
-  const BugType BT_BreakJail{this, "Break out of jail"};
-
-  const CallDescription Chroot{CDM::CLibrary, {"chroot"}, 1},
-      Chdir{CDM::CLibrary, {"chdir"}, 1};
-
+//
+class ChrootChecker final : public Checker<eval::Call, check::PreCall> {
 public:
-  ChrootChecker() {}
-
-  static void *getTag() {
-    static int x;
-    return &x;
-  }
-
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
 
 private:
-  void evalChroot(const CallEvent &Call, CheckerContext &C) const;
-  void evalChdir(const CallEvent &Call, CheckerContext &C) const;
-};
+  bool evalChroot(const CallEvent &Call, CheckerContext &C) const;
+  bool evalChdir(const CallEvent &Call, CheckerContext &C) const;
 
-} // end anonymous namespace
+  const BugType BreakJailBug{this, "Break out of jail"};
+  const CallDescription Chroot{CDM::CLibrary, {"chroot"}, 1};
+  const CallDescription Chdir{CDM::CLibrary, {"chdir"}, 1};
+};
 
 bool ChrootChecker::evalCall(const CallEvent &Call, CheckerContext &C) const {
-  if (Chroot.matches(Call)) {
-    evalChroot(Call, C);
-    return true;
-  }
-  if (Chdir.matches(Call)) {
-    evalChdir(Call, C);
-    return true;
-  }
+  if (Chroot.matches(Call))
+    return evalChroot(Call, C);
+
+  if (Chdir.matches(Call))
+    return evalChdir(Call, C);
 
   return false;
 }
 
-void ChrootChecker::evalChroot(const CallEvent &Call, CheckerContext &C) const {
-  ProgramStateRef state = C.getState();
-  ProgramStateManager &Mgr = state->getStateManager();
+bool ChrootChecker::evalChroot(const CallEvent &Call, CheckerContext &C) const {
+  BasicValueFactory &BVF = C.getSValBuilder().getBasicValueFactory();
+  const LocationContext *LCtx = C.getLocationContext();
+  ProgramStateRef State = C.getState();
+  const auto *CE = cast<CallExpr>(Call.getOriginExpr());
+
+  const QualType IntTy = C.getASTContext().IntTy;
+  SVal Zero = nonloc::ConcreteInt{BVF.getValue(0, IntTy)};
+  SVal Minus1 = nonloc::ConcreteInt{BVF.getValue(-1, IntTy)};
 
-  // Once encouter a chroot(), set the enum value ROOT_CHANGED directly in
-  // the GDM.
-  state = Mgr.addGDM(state, ChrootChecker::getTag(), (void*) ROOT_CHANGED);
-  C.addTransition(state);
+  ProgramStateRef ChrootFailed = State->BindExpr(CE, LCtx, Minus1);
+  C.addTransition(ChrootFailed->set<ChrootState>(ROOT_CHANGE_FAILED));
+
+  ProgramStateRef ChrootSucceeded = State->BindExpr(CE, LCtx, Zero);
+  C.addTransition(ChrootSucceeded->set<ChrootState>(ROOT_CHANGED));
+  return true;
 }
 
-void ChrootChecker::evalChdir(const CallEvent &Call, CheckerContext &C) const {
-  ProgramStateRef state = C.getState();
-  ProgramStateManager &Mgr = state->getStateManager();
+bool ChrootChecker::evalChdir(const CallEvent &Call, CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
 
-  // If there are no jail state in the GDM, just return.
-  const void *k = state->FindGDM(ChrootChecker::getTag());
-  if (!k)
-    return;
+  // If there are no jail state, just return.
+  if (State->get<ChrootState>() == NO_CHROOT)
+    return false;
 
   // After chdir("/"), enter the jail, set the enum value JAIL_ENTERED.
-  const Expr *ArgExpr = Call.getArgExpr(0);
-  SVal ArgVal = C.getSVal(ArgExpr);
+  SVal ArgVal = Call.getArgSVal(0);
 
   if (const MemRegion *R = ArgVal.getAsRegion()) {
     R = R->StripCasts();
-    if (const StringRegion* StrRegion= dyn_cast<StringRegion>(R)) {
-      const StringLiteral* Str = StrRegion->getStringLiteral();
-      if (Str->getString() == "/")
-        state = Mgr.addGDM(state, ChrootChecker::getTag(),
-                           (void*) JAIL_ENTERED);
+    if (const auto *StrRegion = dyn_cast<StringRegion>(R)) {
+      if (StrRegion->getStringLiteral()->getString() == "/") {
+        C.addTransition(State->set<ChrootState>(JAIL_ENTERED));
+        return true;
+      }
     }
   }
-
-  C.addTransition(state);
+  return false;
 }
 
+class ChrootInvocationVisitor final : public BugReporterVisitor {
+public:
+  explicit ChrootInvocationVisitor(const CallDescription &Chroot)
+      : Chroot{Chroot} {}
+
+  PathDiagnosticPieceRef VisitNode(const ExplodedNode *N,
+                                   BugReporterContext &BRC,
+                                   PathSensitiveBugReport &BR) override {
+    if (Satisfied)
+      return nullptr;
+
+    auto StmtP = N->getLocation().getAs<StmtPoint>();
+    if (!StmtP)
+      return nullptr;
+
+    const CallExpr *Call = StmtP->getStmtAs<CallExpr>();
+    if (!Call)
+      return nullptr;
+
+    if (!Chroot.matchesAsWritten(*Call))
+      return nullptr;
+
+    Satisfied = true;
+    PathDiagnosticLocation Pos(Call, BRC.getSourceManager(),
+                               N->getLocationContext());
+    return std::make_shared<PathDiagnosticEventPiece>(Pos, "chroot called here",
+                                                      /*addPosRange=*/true);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const override {
+    static bool Tag;
+    ID.AddPointer(&Tag);
+  }
+
+private:
+  const CallDescription &Chroot;
+  bool Satisfied = false;
+};
+
 // Check the jail state before any function call except chroot and chdir().
 void ChrootChecker::checkPreCall(const CallEvent &Call,
                                  CheckerContext &C) const {
@@ -120,22 +158,26 @@ void ChrootChecker::checkPreCall(const CallEvent &Call,
   if (matchesAny(Call, Chroot, Chdir))
     return;
 
-  // If jail state is ROOT_CHANGED, generate BugReport.
-  void *const* k = C.getState()->FindGDM(ChrootChecker::getTag());
-  if (k)
-    if (isRootChanged((intptr_t) *k))
-      if (ExplodedNode *N = C.generateNonFatalErrorNode()) {
-        constexpr llvm::StringLiteral Msg =
-            "No call of chdir(\"/\") immediately after chroot";
-        C.emitReport(
-            std::make_unique<PathSensitiveBugReport>(BT_BreakJail, Msg, N));
-      }
-}
+  // If jail state is not ROOT_CHANGED just return.
+  if (C.getState()->get<ChrootState>() != ROOT_CHANGED)
+    return;
 
-void ento::registerChrootChecker(CheckerManager &mgr) {
-  mgr.registerChecker<ChrootChecker>();
+  // Generate bug report.
+  ExplodedNode *Err =
+      C.generateNonFatalErrorNode(C.getState(), C.getPredecessor());
+  if (!Err)
+    return;
+
+  auto R = std::make_unique<PathSensitiveBugReport>(
+      BreakJailBug, R"(No call of chdir("/") immediately after chroot)", Err);
+  R->addVisitor<ChrootInvocationVisitor>(Chroot);
+  C.emitReport(std::move(R));
 }
 
-bool ento::shouldRegisterChrootChecker(const CheckerManager &mgr) {
-  return true;
+} // namespace
+
+void ento::registerChrootChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<ChrootChecker>();
 }
+
+bool ento::shouldRegisterChrootChecker(const CheckerManager &) { return true; }
diff --git a/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp
deleted file mode 100644
index 45642483dd457..0000000000000
--- a/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp
+++ /dev/null
@@ -1,519 +0,0 @@
-//== IdenticalExprChecker.cpp - Identical expression checker----------------==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This defines IdenticalExprChecker, a check that warns about
-/// unintended use of identical expressions.
-///
-/// It checks for use of identical expressions with comparison operators and
-/// inside conditional expressions.
-///
-//===----------------------------------------------------------------------===//
-
-#include "clang/AST/DynamicRecursiveASTVisitor.h"
-#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
-#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-
-using namespace clang;
-using namespace ento;
-
-static bool isIdenticalStmt(const ASTContext &Ctx, const Stmt *Stmt1,
-                            const Stmt *Stmt2, bool IgnoreSideEffects = false);
-//===----------------------------------------------------------------------===//
-// FindIdenticalExprVisitor - Identify nodes using identical expressions.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class FindIdenticalExprVisitor : public DynamicRecursiveASTVisitor {
-  BugReporter &BR;
-  const CheckerBase *Checker;
-  AnalysisDeclContext *AC;
-public:
-  explicit FindIdenticalExprVisitor(BugReporter &B,
-                                    const CheckerBase *Checker,
-                                    AnalysisDeclContext *A)
-      : BR(B), Checker(Checker), AC(A) {}
-  // FindIdenticalExprVisitor only visits nodes
-  // that are binary operators, if statements or
-  // conditional operators.
-  bool VisitBinaryOperator(BinaryOperator *B) override;
-  bool VisitIfStmt(IfStmt *I) override;
-  bool VisitConditionalOperator(ConditionalOperator *C) override;
-
-private:
-  void reportIdenticalExpr(const BinaryOperator *B, bool CheckBitwise,
-                           ArrayRef<SourceRange> Sr);
-  void checkBitwiseOrLogicalOp(const BinaryOperator *B, bool CheckBitwise);
-  void checkComparisonOp(const BinaryOperator *B);
-};
-} // end anonymous namespace
-
-void FindIdenticalExprVisitor::reportIdenticalExpr(const BinaryOperator *B,
-                                                   bool CheckBitwise,
-                                                   ArrayRef<SourceRange> Sr) {
-  StringRef Message;
-  if (CheckBitwise)
-    Message = "identical expressions on both sides of bitwise operator";
-  else
-    Message = "identical expressions on both sides of logical operator";
-
-  PathDiagnosticLocation ELoc =
-      PathDiagnosticLocation::createOperatorLoc(B, BR.getSourceManager());
-  BR.EmitBasicReport(AC->getDecl(), Checker,
-                     "Use of identical expressions",
-                     categories::LogicError,
-                     Message, ELoc, Sr);
-}
-
-void FindIdenticalExprVisitor::checkBitwiseOrLogicalOp(const BinaryOperator *B,
-                                                       bool CheckBitwise) {
-  SourceRange Sr[2];
-
-  const Expr *LHS = B->getLHS();
-  const Expr *RHS = B->getRHS();
-
-  // Split operators as long as we still have operators to split on. We will
-  // get called for every binary operator in an expression so there is no need
-  // to check every one against each other here, just the right most one with
-  // the others.
-  while (const BinaryOperator *B2 = dyn_cast<BinaryOperator>(LHS)) {
-    if (B->getOpcode() != B2->getOpcode())
-      break;
-    if (isIdenticalStmt(AC->getASTContext(), RHS, B2->getRHS())) {
-      Sr[0] = RHS->getSourceRange();
-      Sr[1] = B2->getRHS()->getSourceRange();
-      reportIdenticalExpr(B, CheckBitwise, Sr);
-    }
-    LHS = B2->getLHS();
-  }
-
-  if (isIdenticalStmt(AC->getASTContext(), RHS, LHS)) {
-    Sr[0] = RHS->getSourceRange();
-    Sr[1] = LHS->getSourceRange();
-    reportIdenticalExpr(B, CheckBitwise, Sr);
-  }
-}
-
-bool FindIdenticalExprVisitor::VisitIfStmt(IfStmt *I) {
-  const Stmt *Stmt1 = I->getThen();
-  const Stmt *Stmt2 = I->getElse();
-
-  // Check for identical inner condition:
-  //
-  // if (x<10) {
-  //   if (x<10) {
-  //   ..
-  if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Stmt1)) {
-    if (!CS->body_empty()) {
-      const IfStmt *InnerIf = dyn_cast<IfStmt>(*CS->body_begin());
-      if (InnerIf && isIdenticalStmt(AC->getASTContext(), I->getCond(), InnerIf->getCond(), /*IgnoreSideEffects=*/ false)) {
-        PathDiagnosticLocation ELoc(InnerIf->getCond(), BR.getSourceManager(), AC);
-        BR.EmitBasicReport(AC->getDecl(), Checker, "Identical conditions",
-          categories::LogicError,
-          "conditions of the inner and outer statements are identical",
-          ELoc);
-      }
-    }
-  }
-
-  // Check for identical conditions:
-  //
-  // if (b) {
-  //   foo1();
-  // } else if (b) {
-  //   foo2();
-  // }
-  if (Stmt1 && Stmt2) {
-    const Expr *Cond1 = I->getCond();
-    const Stmt *Else = Stmt2;
-    while (const IfStmt *I2 = dyn_cast_or_null<IfStmt>(Else)) {
-      const Expr *Cond2 = I2->getCond();
-      if (isIdenticalStmt(AC->getASTContext(), Cond1, Cond2, false)) {
-        SourceRange Sr = Cond1->getSourceRange();
-        PathDiagnosticLocation ELoc(Cond2, BR.getSourceManager(), AC);
-        BR.EmitBasicReport(AC->getDecl(), Checker, "Identical conditions",
-                           categories::LogicError,
-                           "expression is identical to previous condition",
-                           ELoc, Sr);
-      }
-      Else = I2->getElse();
-    }
-  }
-
-  if (!Stmt1 || !Stmt2)
-    return true;
-
-  // Special handling for code like:
-  //
-  // if (b) {
-  //   i = 1;
-  // } else
-  //   i = 1;
-  if (const CompoundStmt *CompStmt = dyn_cast<CompoundStmt>(Stmt1)) {
-    if (CompStmt->size() == 1)
-      Stmt1 = CompStmt->body_back();
-  }
-  if (const CompoundStmt *CompStmt = dyn_cast<CompoundStmt>(Stmt2)) {
-    if (CompStmt->size() == 1)
-      Stmt2 = CompStmt->body_back();
-  }
-
-  if (isIdenticalStmt(AC->getASTContext(), Stmt1, Stmt2, true)) {
-      PathDiagnosticLocation ELoc =
-          PathDiagnosticLocation::createBegin(I, BR.getSourceManager(), AC);
-      BR.EmitBasicReport(AC->getDecl(), Checker,
-                         "Identical branches",
-                         categories::LogicError,
-                         "true and false branches are identical", ELoc);
-  }
-  return true;
-}
-
-bool FindIdenticalExprVisitor::VisitBinaryOperator(BinaryOperator *B) {
-  BinaryOperator::Opcode Op = B->getOpcode();
-
-  if (BinaryOperator::isBitwiseOp(Op))
-    checkBitwiseOrLogicalOp(B, true);
-
-  if (BinaryOperator::isLogicalOp(Op))
-    checkBitwiseOrLogicalOp(B, false);
-
-  if (BinaryOperator::isComparisonOp(Op))
-    checkComparisonOp(B);
-
-  // We want to visit ALL nodes (subexpressions of binary comparison
-  // expressions too) that contains comparison operators.
-  // True is always returned to traverse ALL nodes.
-  return true;
-}
-
-void FindIdenticalExprVisitor::checkComparisonOp(const BinaryOperator *B) {
-  BinaryOperator::Opcode Op = B->getOpcode();
-
-  //
-  // Special case for floating-point representation.
-  //
-  // If expressions on both sides of comparison operator are of type float,
-  // then for some comparison operators no warning shall be
-  // reported even if the expressions are identical from a symbolic point of
-  // view. Comparison between expressions, declared variables and literals
-  // are treated differently.
-  //
-  // != and == between float literals that have the same value should NOT warn.
-  // < > between float literals that have the same value SHOULD warn.
-  //
-  // != and == between the same float declaration should NOT warn.
-  // < > between the same float declaration SHOULD warn.
-  //
-  // != and == between eq. expressions that evaluates into float
-  //           should NOT warn.
-  // < >       between eq. expressions that evaluates into float
-  //           should NOT warn.
-  //
-  const Expr *LHS = B->getLHS()->IgnoreParenImpCasts();
-  const Expr *RHS = B->getRHS()->IgnoreParenImpCasts();
-
-  const DeclRefExpr *DeclRef1 = dyn_cast<DeclRefExpr>(LHS);
-  const DeclRefExpr *DeclRef2 = dyn_cast<DeclRefExpr>(RHS);
-  const FloatingLiteral *FloatLit1 = dyn_cast<FloatingLiteral>(LHS);
-  const FloatingLiteral *FloatLit2 = dyn_cast<FloatingLiteral>(RHS);
-  if ((DeclRef1) && (DeclRef2)) {
-    if ((DeclRef1->getType()->hasFloatingRepresentation()) &&
-        (DeclRef2->getType()->hasFloatingRepresentation())) {
-      if (DeclRef1->getDecl() == DeclRef2->getDecl()) {
-        if ((Op == BO_EQ) || (Op == BO_NE)) {
-          return;
-        }
-      }
-    }
-  } else if ((FloatLit1) && (FloatLit2)) {
-    if (FloatLit1->getValue().bitwiseIsEqual(FloatLit2->getValue())) {
-      if ((Op == BO_EQ) || (Op == BO_NE)) {
-        return;
-      }
-    }
-  } else if (LHS->getType()->hasFloatingRepresentation()) {
-    // If any side of comparison operator still has floating-point
-    // representation, then it's an expression. Don't warn.
-    // Here only LHS is checked since RHS will be implicit casted to float.
-    return;
-  } else {
-    // No special case with floating-point representation, report as usual.
-  }
-
-  if (isIdenticalStmt(AC->getASTContext(), B->getLHS(), B->getRHS())) {
-    PathDiagnosticLocation ELoc =
-        PathDiagnosticLocation::createOperatorLoc(B, BR.getSourceManager());
-    StringRef Message;
-    if (Op == BO_Cmp)
-      Message = "comparison of identical expressions always evaluates to "
-                "'equal'";
-    else if (((Op == BO_EQ) || (Op == BO_LE) || (Op == BO_GE)))
-      Message = "comparison of identical expressions always evaluates to true";
-    else
-      Message = "comparison of identical expressions always evaluates to false";
-    BR.EmitBasicReport(AC->getDecl(), Checker,
-                       "Compare of identical expressions",
-                       categories::LogicError, Message, ELoc);
-  }
-}
-
-bool FindIdenticalExprVisitor::VisitConditionalOperator(
-    ConditionalOperator *C) {
-
-  // Check if expressions in conditional expression are identical
-  // from a symbolic point of view.
-
-  if (isIdenticalStmt(AC->getASTContext(), C->getTrueExpr(),
-                      C->getFalseExpr(), true)) {
-    PathDiagnosticLocation ELoc =
-        PathDiagnosticLocation::createConditionalColonLoc(
-            C, BR.getSourceManager());
-
-    SourceRange Sr[2];
-    Sr[0] = C->getTrueExpr()->getSourceRange();
-    Sr[1] = C->getFalseExpr()->getSourceRange();
-    BR.EmitBasicReport(
-        AC->getDecl(), Checker,
-        "Identical expressions in conditional expression",
-        categories::LogicError,
-        "identical expressions on both sides of ':' in conditional expression",
-        ELoc, Sr);
-  }
-  // We want to visit ALL nodes (expressions in conditional
-  // expressions too) that contains conditional operators,
-  // thus always return true to traverse ALL nodes.
-  return true;
-}
-
-/// Determines whether two statement trees are identical regarding
-/// operators and symbols.
-///
-/// Exceptions: expressions containing macros or functions with possible side
-/// effects are never considered identical.
-/// Limitations: (t + u) and (u + t) are not considered identical.
-/// t*(u + t) and t*u + t*t are not considered identical.
-///
-static bool isIdenticalStmt(const ASTContext &Ctx, const Stmt *Stmt1,
-                            const Stmt *Stmt2, bool IgnoreSideEffects) {
-
-  if (!Stmt1 || !Stmt2) {
-    return !Stmt1 && !Stmt2;
-  }
-
-  // If Stmt1 & Stmt2 are of different class then they are not
-  // identical statements.
-  if (Stmt1->getStmtClass() != Stmt2->getStmtClass())
-    return false;
-
-  const Expr *Expr1 = dyn_cast<Expr>(Stmt1);
-  const Expr *Expr2 = dyn_cast<Expr>(Stmt2);
-
-  if (Expr1 && Expr2) {
-    // If Stmt1 has side effects then don't warn even if expressions
-    // are identical.
-    if (!IgnoreSideEffects && Expr1->HasSideEffects(Ctx))
-      return false;
-    // If either expression comes from a macro then don't warn even if
-    // the expressions are identical.
-    if ((Expr1->getExprLoc().isMacroID()) || (Expr2->getExprLoc().isMacroID()))
-      return false;
-
-    // If all children of two expressions are identical, return true.
-    Expr::const_child_iterator I1 = Expr1->child_begin();
-    Expr::const_child_iterator I2 = Expr2->child_begin();
-    while (I1 != Expr1->child_end() && I2 != Expr2->child_end()) {
-      if (!*I1 || !*I2 || !isIdenticalStmt(Ctx, *I1, *I2, IgnoreSideEffects))
-        return false;
-      ++I1;
-      ++I2;
-    }
-    // If there are different number of children in the statements, return
-    // false.
-    if (I1 != Expr1->child_end())
-      return false;
-    if (I2 != Expr2->child_end())
-      return false;
-  }
-
-  switch (Stmt1->getStmtClass()) {
-  default:
-    return false;
-  case Stmt::CallExprClass:
-  case Stmt::ArraySubscriptExprClass:
-  case Stmt::ArraySectionExprClass:
-  case Stmt::OMPArrayShapingExprClass:
-  case Stmt::OMPIteratorExprClass:
-  case Stmt::ImplicitCastExprClass:
-  case Stmt::ParenExprClass:
-  case Stmt::BreakStmtClass:
-  case Stmt::ContinueStmtClass:
-  case Stmt::NullStmtClass:
-    return true;
-  case Stmt::CStyleCastExprClass: {
-    const CStyleCastExpr* CastExpr1 = cast<CStyleCastExpr>(Stmt1);
-    const CStyleCastExpr* CastExpr2 = cast<CStyleCastExpr>(Stmt2);
-
-    return CastExpr1->getTypeAsWritten() == CastExpr2->getTypeAsWritten();
-  }
-  case Stmt::ReturnStmtClass: {
-    const ReturnStmt *ReturnStmt1 = cast<ReturnStmt>(Stmt1);
-    const ReturnStmt *ReturnStmt2 = cast<ReturnStmt>(Stmt2);
-
-    return isIdenticalStmt(Ctx, ReturnStmt1->getRetValue(),
-                           ReturnStmt2->getRetValue(), IgnoreSideEffects);
-  }
-  case Stmt::ForStmtClass: {
-    const ForStmt *ForStmt1 = cast<ForStmt>(Stmt1);
-    const ForStmt *ForStmt2 = cast<ForStmt>(Stmt2);
-
-    if (!isIdenticalStmt(Ctx, ForStmt1->getInit(), ForStmt2->getInit(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, ForStmt1->getCond(), ForStmt2->getCond(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, ForStmt1->getInc(), ForStmt2->getInc(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, ForStmt1->getBody(), ForStmt2->getBody(),
-                         IgnoreSideEffects))
-      return false;
-    return true;
-  }
-  case Stmt::DoStmtClass: {
-    const DoStmt *DStmt1 = cast<DoStmt>(Stmt1);
-    const DoStmt *DStmt2 = cast<DoStmt>(Stmt2);
-
-    if (!isIdenticalStmt(Ctx, DStmt1->getCond(), DStmt2->getCond(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, DStmt1->getBody(), DStmt2->getBody(),
-                         IgnoreSideEffects))
-      return false;
-    return true;
-  }
-  case Stmt::WhileStmtClass: {
-    const WhileStmt *WStmt1 = cast<WhileStmt>(Stmt1);
-    const WhileStmt *WStmt2 = cast<WhileStmt>(Stmt2);
-
-    if (!isIdenticalStmt(Ctx, WStmt1->getCond(), WStmt2->getCond(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, WStmt1->getBody(), WStmt2->getBody(),
-                         IgnoreSideEffects))
-      return false;
-    return true;
-  }
-  case Stmt::IfStmtClass: {
-    const IfStmt *IStmt1 = cast<IfStmt>(Stmt1);
-    const IfStmt *IStmt2 = cast<IfStmt>(Stmt2);
-
-    if (!isIdenticalStmt(Ctx, IStmt1->getCond(), IStmt2->getCond(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, IStmt1->getThen(), IStmt2->getThen(),
-                         IgnoreSideEffects))
-      return false;
-    if (!isIdenticalStmt(Ctx, IStmt1->getElse(), IStmt2->getElse(),
-                         IgnoreSideEffects))
-      return false;
-    return true;
-  }
-  case Stmt::CompoundStmtClass: {
-    const CompoundStmt *CompStmt1 = cast<CompoundStmt>(Stmt1);
-    const CompoundStmt *CompStmt2 = cast<CompoundStmt>(Stmt2);
-
-    if (CompStmt1->size() != CompStmt2->size())
-      return false;
-
-    CompoundStmt::const_body_iterator I1 = CompStmt1->body_begin();
-    CompoundStmt::const_body_iterator I2 = CompStmt2->body_begin();
-    while (I1 != CompStmt1->body_end() && I2 != CompStmt2->body_end()) {
-      if (!isIdenticalStmt(Ctx, *I1, *I2, IgnoreSideEffects))
-        return false;
-      ++I1;
-      ++I2;
-    }
-
-    return true;
-  }
-  case Stmt::CompoundAssignOperatorClass:
-  case Stmt::BinaryOperatorClass: {
-    const BinaryOperator *BinOp1 = cast<BinaryOperator>(Stmt1);
-    const BinaryOperator *BinOp2 = cast<BinaryOperator>(Stmt2);
-    return BinOp1->getOpcode() == BinOp2->getOpcode();
-  }
-  case Stmt::CharacterLiteralClass: {
-    const CharacterLiteral *CharLit1 = cast<CharacterLiteral>(Stmt1);
-    const CharacterLiteral *CharLit2 = cast<CharacterLiteral>(Stmt2);
-    return CharLit1->getValue() == CharLit2->getValue();
-  }
-  case Stmt::DeclRefExprClass: {
-    const DeclRefExpr *DeclRef1 = cast<DeclRefExpr>(Stmt1);
-    const DeclRefExpr *DeclRef2 = cast<DeclRefExpr>(Stmt2);
-    return DeclRef1->getDecl() == DeclRef2->getDecl();
-  }
-  case Stmt::IntegerLiteralClass: {
-    const IntegerLiteral *IntLit1 = cast<IntegerLiteral>(Stmt1);
-    const IntegerLiteral *IntLit2 = cast<IntegerLiteral>(Stmt2);
-
-    llvm::APInt I1 = IntLit1->getValue();
-    llvm::APInt I2 = IntLit2->getValue();
-    if (I1.getBitWidth() != I2.getBitWidth())
-      return false;
-    return  I1 == I2;
-  }
-  case Stmt::FloatingLiteralClass: {
-    const FloatingLiteral *FloatLit1 = cast<FloatingLiteral>(Stmt1);
-    const FloatingLiteral *FloatLit2 = cast<FloatingLiteral>(Stmt2);
-    return FloatLit1->getValue().bitwiseIsEqual(FloatLit2->getValue());
-  }
-  case Stmt::StringLiteralClass: {
-    const StringLiteral *StringLit1 = cast<StringLiteral>(Stmt1);
-    const StringLiteral *StringLit2 = cast<StringLiteral>(Stmt2);
-    return StringLit1->getBytes() == StringLit2->getBytes();
-  }
-  case Stmt::MemberExprClass: {
-    const MemberExpr *MemberStmt1 = cast<MemberExpr>(Stmt1);
-    const MemberExpr *MemberStmt2 = cast<MemberExpr>(Stmt2);
-    return MemberStmt1->getMemberDecl() == MemberStmt2->getMemberDecl();
-  }
-  case Stmt::UnaryOperatorClass: {
-    const UnaryOperator *UnaryOp1 = cast<UnaryOperator>(Stmt1);
-    const UnaryOperator *UnaryOp2 = cast<UnaryOperator>(Stmt2);
-    return UnaryOp1->getOpcode() == UnaryOp2->getOpcode();
-  }
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// FindIdenticalExprChecker
-//===----------------------------------------------------------------------===//
-
-namespace {
-class FindIdenticalExprChecker : public Checker<check::ASTCodeBody> {
-public:
-  void checkASTCodeBody(const Decl *D, AnalysisManager &Mgr,
-                        BugReporter &BR) const {
-    FindIdenticalExprVisitor Visitor(BR, this, Mgr.getAnalysisDeclContext(D));
-    Visitor.TraverseDecl(const_cast<Decl *>(D));
-  }
-};
-} // end anonymous namespace
-
-void ento::registerIdenticalExprChecker(CheckerManager &Mgr) {
-  Mgr.registerChecker<FindIdenticalExprChecker>();
-}
-
-bool ento::shouldRegisterIdenticalExprChecker(const CheckerManager &mgr) {
-  return true;
-}
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
index 9e81a6bd19fc5..df23735e4668e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
@@ -211,9 +211,9 @@ class MallocSizeofChecker : public Checker<check::ASTCodeBody> {
           continue;
 
         const TypeSourceInfo *TSI = nullptr;
-        if (CallRec.CastedExprParent.is<const VarDecl *>()) {
-          TSI = CallRec.CastedExprParent.get<const VarDecl *>()
-                    ->getTypeSourceInfo();
+        if (const auto *VD =
+                dyn_cast<const VarDecl *>(CallRec.CastedExprParent)) {
+          TSI = VD->getTypeSourceInfo();
         } else {
           TSI = CallRec.ExplicitCastType;
         }
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/MemoryUnsafeCastChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/MemoryUnsafeCastChecker.cpp
new file mode 100644
index 0000000000000..eeaccf9b70524
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/MemoryUnsafeCastChecker.cpp
@@ -0,0 +1,188 @@
+//=======- MemoryUnsafeCastChecker.cpp -------------------------*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines MemoryUnsafeCast checker, which checks for casts from a
+// base type to a derived type.
+//===----------------------------------------------------------------------===//
+
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+
+using namespace clang;
+using namespace ento;
+using namespace ast_matchers;
+
+namespace {
+static constexpr const char *const BaseNode = "BaseNode";
+static constexpr const char *const DerivedNode = "DerivedNode";
+static constexpr const char *const FromCastNode = "FromCast";
+static constexpr const char *const ToCastNode = "ToCast";
+static constexpr const char *const WarnRecordDecl = "WarnRecordDecl";
+
+class MemoryUnsafeCastChecker : public Checker<check::ASTCodeBody> {
+  BugType BT{this, "Unsafe cast", "WebKit coding guidelines"};
+
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager &Mgr,
+                        BugReporter &BR) const;
+};
+} // end namespace
+
+static void emitDiagnostics(const BoundNodes &Nodes, BugReporter &BR,
+                            AnalysisDeclContext *ADC,
+                            const MemoryUnsafeCastChecker *Checker,
+                            const BugType &BT) {
+  const auto *CE = Nodes.getNodeAs<CastExpr>(WarnRecordDecl);
+  const NamedDecl *Base = Nodes.getNodeAs<NamedDecl>(BaseNode);
+  const NamedDecl *Derived = Nodes.getNodeAs<NamedDecl>(DerivedNode);
+  assert(CE && Base && Derived);
+
+  std::string Diagnostics;
+  llvm::raw_string_ostream OS(Diagnostics);
+  OS << "Unsafe cast from base type '" << Base->getNameAsString()
+     << "' to derived type '" << Derived->getNameAsString() << "'";
+  PathDiagnosticLocation BSLoc(CE->getSourceRange().getBegin(),
+                               BR.getSourceManager());
+  auto Report = std::make_unique<BasicBugReport>(BT, OS.str(), BSLoc);
+  Report->addRange(CE->getSourceRange());
+  BR.emitReport(std::move(Report));
+}
+
+static void emitDiagnosticsUnrelated(const BoundNodes &Nodes, BugReporter &BR,
+                                     AnalysisDeclContext *ADC,
+                                     const MemoryUnsafeCastChecker *Checker,
+                                     const BugType &BT) {
+  const auto *CE = Nodes.getNodeAs<CastExpr>(WarnRecordDecl);
+  const NamedDecl *FromCast = Nodes.getNodeAs<NamedDecl>(FromCastNode);
+  const NamedDecl *ToCast = Nodes.getNodeAs<NamedDecl>(ToCastNode);
+  assert(CE && FromCast && ToCast);
+
+  std::string Diagnostics;
+  llvm::raw_string_ostream OS(Diagnostics);
+  OS << "Unsafe cast from type '" << FromCast->getNameAsString()
+     << "' to an unrelated type '" << ToCast->getNameAsString() << "'";
+  PathDiagnosticLocation BSLoc(CE->getSourceRange().getBegin(),
+                               BR.getSourceManager());
+  auto Report = std::make_unique<BasicBugReport>(BT, OS.str(), BSLoc);
+  Report->addRange(CE->getSourceRange());
+  BR.emitReport(std::move(Report));
+}
+
+namespace clang {
+namespace ast_matchers {
+AST_MATCHER_P(StringLiteral, mentionsBoundType, std::string, BindingID) {
+  return Builder->removeBindings([this, &Node](const BoundNodesMap &Nodes) {
+    const auto &BN = Nodes.getNode(this->BindingID);
+    if (const auto *ND = BN.get<NamedDecl>()) {
+      return ND->getName() != Node.getString();
+    }
+    return true;
+  });
+}
+} // end namespace ast_matchers
+} // end namespace clang
+
+static decltype(auto) hasTypePointingTo(DeclarationMatcher DeclM) {
+  return hasType(pointerType(pointee(hasDeclaration(DeclM))));
+}
+
+void MemoryUnsafeCastChecker::checkASTCodeBody(const Decl *D,
+                                               AnalysisManager &AM,
+                                               BugReporter &BR) const {
+
+  AnalysisDeclContext *ADC = AM.getAnalysisDeclContext(D);
+
+  // Match downcasts from base type to derived type and warn
+  auto MatchExprPtr = allOf(
+      hasSourceExpression(hasTypePointingTo(cxxRecordDecl().bind(BaseNode))),
+      hasTypePointingTo(cxxRecordDecl(isDerivedFrom(equalsBoundNode(BaseNode)))
+                            .bind(DerivedNode)),
+      unless(anyOf(hasSourceExpression(cxxThisExpr()),
+                   hasTypePointingTo(templateTypeParmDecl()))));
+  auto MatchExprPtrObjC = allOf(
+      hasSourceExpression(ignoringImpCasts(hasType(objcObjectPointerType(
+          pointee(hasDeclaration(objcInterfaceDecl().bind(BaseNode))))))),
+      ignoringImpCasts(hasType(objcObjectPointerType(pointee(hasDeclaration(
+          objcInterfaceDecl(isDerivedFrom(equalsBoundNode(BaseNode)))
+              .bind(DerivedNode)))))));
+  auto MatchExprRefTypeDef =
+      allOf(hasSourceExpression(hasType(hasUnqualifiedDesugaredType(recordType(
+                hasDeclaration(decl(cxxRecordDecl().bind(BaseNode))))))),
+            hasType(hasUnqualifiedDesugaredType(recordType(hasDeclaration(
+                decl(cxxRecordDecl(isDerivedFrom(equalsBoundNode(BaseNode)))
+                         .bind(DerivedNode)))))),
+            unless(anyOf(hasSourceExpression(hasDescendant(cxxThisExpr())),
+                         hasType(templateTypeParmDecl()))));
+
+  auto ExplicitCast = explicitCastExpr(anyOf(MatchExprPtr, MatchExprRefTypeDef,
+                                             MatchExprPtrObjC))
+                          .bind(WarnRecordDecl);
+  auto Cast = stmt(ExplicitCast);
+
+  auto Matches =
+      match(stmt(forEachDescendant(Cast)), *D->getBody(), AM.getASTContext());
+  for (BoundNodes Match : Matches)
+    emitDiagnostics(Match, BR, ADC, this, BT);
+
+  // Match casts between unrelated types and warn
+  auto MatchExprPtrUnrelatedTypes = allOf(
+      hasSourceExpression(
+          hasTypePointingTo(cxxRecordDecl().bind(FromCastNode))),
+      hasTypePointingTo(cxxRecordDecl().bind(ToCastNode)),
+      unless(anyOf(hasTypePointingTo(cxxRecordDecl(
+                       isSameOrDerivedFrom(equalsBoundNode(FromCastNode)))),
+                   hasSourceExpression(hasTypePointingTo(cxxRecordDecl(
+                       isSameOrDerivedFrom(equalsBoundNode(ToCastNode))))))));
+  auto MatchExprPtrObjCUnrelatedTypes = allOf(
+      hasSourceExpression(ignoringImpCasts(hasType(objcObjectPointerType(
+          pointee(hasDeclaration(objcInterfaceDecl().bind(FromCastNode))))))),
+      ignoringImpCasts(hasType(objcObjectPointerType(
+          pointee(hasDeclaration(objcInterfaceDecl().bind(ToCastNode)))))),
+      unless(anyOf(
+          ignoringImpCasts(hasType(
+              objcObjectPointerType(pointee(hasDeclaration(objcInterfaceDecl(
+                  isSameOrDerivedFrom(equalsBoundNode(FromCastNode)))))))),
+          hasSourceExpression(ignoringImpCasts(hasType(
+              objcObjectPointerType(pointee(hasDeclaration(objcInterfaceDecl(
+                  isSameOrDerivedFrom(equalsBoundNode(ToCastNode))))))))))));
+  auto MatchExprRefTypeDefUnrelated = allOf(
+      hasSourceExpression(hasType(hasUnqualifiedDesugaredType(recordType(
+          hasDeclaration(decl(cxxRecordDecl().bind(FromCastNode))))))),
+      hasType(hasUnqualifiedDesugaredType(
+          recordType(hasDeclaration(decl(cxxRecordDecl().bind(ToCastNode)))))),
+      unless(anyOf(
+          hasType(hasUnqualifiedDesugaredType(
+              recordType(hasDeclaration(decl(cxxRecordDecl(
+                  isSameOrDerivedFrom(equalsBoundNode(FromCastNode)))))))),
+          hasSourceExpression(hasType(hasUnqualifiedDesugaredType(
+              recordType(hasDeclaration(decl(cxxRecordDecl(
+                  isSameOrDerivedFrom(equalsBoundNode(ToCastNode))))))))))));
+
+  auto ExplicitCastUnrelated =
+      explicitCastExpr(anyOf(MatchExprPtrUnrelatedTypes,
+                             MatchExprPtrObjCUnrelatedTypes,
+                             MatchExprRefTypeDefUnrelated))
+          .bind(WarnRecordDecl);
+  auto CastUnrelated = stmt(ExplicitCastUnrelated);
+  auto MatchesUnrelatedTypes = match(stmt(forEachDescendant(CastUnrelated)),
+                                     *D->getBody(), AM.getASTContext());
+  for (BoundNodes Match : MatchesUnrelatedTypes)
+    emitDiagnosticsUnrelated(Match, BR, ADC, this, BT);
+}
+
+void ento::registerMemoryUnsafeCastChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<MemoryUnsafeCastChecker>();
+}
+
+bool ento::shouldRegisterMemoryUnsafeCastChecker(const CheckerManager &mgr) {
+  return true;
+}
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
index e0433c5c2c1a0..bb580b06e2c53 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
@@ -263,7 +263,7 @@ class RawPtrRefLocalVarsChecker
       if (tryToFindPtrOrigin(
               Value, /*StopAtFirstRefCountedObj=*/false,
               [&](const clang::Expr *InitArgOrigin, bool IsSafe) {
-                if (!InitArgOrigin)
+                if (!InitArgOrigin || IsSafe)
                   return true;
 
                 if (isa<CXXThisExpr>(InitArgOrigin))
diff --git a/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp b/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
index b0563b6c070f1..827c04143e658 100644
--- a/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
@@ -196,13 +196,13 @@ const PointerToMemberData *BasicValueFactory::accumCXXBase(
   const NamedDecl *ND = nullptr;
   llvm::ImmutableList<const CXXBaseSpecifier *> BaseSpecList;
 
-  if (PTMDT.isNull() || PTMDT.is<const NamedDecl *>()) {
-    if (PTMDT.is<const NamedDecl *>())
-      ND = PTMDT.get<const NamedDecl *>();
+  if (PTMDT.isNull() || isa<const NamedDecl *>(PTMDT)) {
+    if (const auto *NDP = dyn_cast_if_present<const NamedDecl *>(PTMDT))
+      ND = NDP;
 
     BaseSpecList = CXXBaseListFactory.getEmptyList();
   } else {
-    const PointerToMemberData *PTMD = PTMDT.get<const PointerToMemberData *>();
+    const auto *PTMD = cast<const PointerToMemberData *>(PTMDT);
     ND = PTMD->getDeclaratorDecl();
 
     BaseSpecList = PTMD->getCXXBaseList();
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index b67e6cd86c3d6..2904eab0097dc 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -2416,6 +2416,28 @@ PathSensitiveBugReport::getRanges() const {
   return Ranges;
 }
 
+static bool exitingDestructor(const ExplodedNode *N) {
+  // Need to loop here, as some times the Error node is already outside of the
+  // destructor context, and the previous node is an edge that is also outside.
+  while (N && !N->getLocation().getAs<StmtPoint>()) {
+    N = N->getFirstPred();
+  }
+  return N && isa<CXXDestructorDecl>(N->getLocationContext()->getDecl());
+}
+
+static const Stmt *
+findReasonableStmtCloseToFunctionExit(const ExplodedNode *N) {
+  if (exitingDestructor(N)) {
+    // If we are exiting a destructor call, it is more useful to point to
+    // the next stmt which is usually the temporary declaration.
+    if (const Stmt *S = N->getNextStmtForDiagnostics())
+      return S;
+    // If next stmt is not found, it is likely the end of a top-level
+    // function analysis. find the last execution statement then.
+  }
+  return N->getPreviousStmtForDiagnostics();
+}
+
 PathDiagnosticLocation
 PathSensitiveBugReport::getLocation() const {
   assert(ErrorNode && "Cannot create a location with a null node.");
@@ -2433,15 +2455,7 @@ PathSensitiveBugReport::getLocation() const {
       if (const ReturnStmt *RS = FE->getStmt())
         return PathDiagnosticLocation::createBegin(RS, SM, LC);
 
-      // If we are exiting a destructor call, it is more useful to point to the
-      // next stmt which is usually the temporary declaration.
-      // For non-destructor and non-top-level calls, the next stmt will still
-      // refer to the last executed stmt of the body.
-      S = ErrorNode->getNextStmtForDiagnostics();
-      // If next stmt is not found, it is likely the end of a top-level function
-      // analysis. find the last execution statement then.
-      if (!S)
-        S = ErrorNode->getPreviousStmtForDiagnostics();
+      S = findReasonableStmtCloseToFunctionExit(ErrorNode);
     }
     if (!S)
       S = ErrorNode->getNextStmtForDiagnostics();
diff --git a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
index 8605fa149e4f5..67b7d30853d9d 100644
--- a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp
@@ -649,14 +649,15 @@ StmtNodeBuilder::~StmtNodeBuilder() {
 void BranchNodeBuilder::anchor() {}
 
 ExplodedNode *BranchNodeBuilder::generateNode(ProgramStateRef State,
-                                              bool branch,
+                                              bool Branch,
                                               ExplodedNode *NodePred) {
-  // If the branch has been marked infeasible we should not generate a node.
-  if (!isFeasible(branch))
+  const CFGBlock *Dst = Branch ? DstT : DstF;
+
+  if (!Dst)
     return nullptr;
 
-  ProgramPoint Loc = BlockEdge(C.getBlock(), branch ? DstT : DstF,
-                               NodePred->getLocationContext());
+  ProgramPoint Loc =
+      BlockEdge(C.getBlock(), Dst, NodePred->getLocationContext());
   ExplodedNode *Succ = generateNodeImpl(Loc, State, NodePred);
   return Succ;
 }
diff --git a/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp b/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
index 11cef00ada330..c4af02f21f494 100644
--- a/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
@@ -211,9 +211,9 @@ void ExplodedNode::NodeGroup::replaceNode(ExplodedNode *node) {
   assert(!getFlag());
 
   GroupStorage &Storage = reinterpret_cast<GroupStorage&>(P);
-  assert(Storage.is<ExplodedNode *>());
+  assert(isa<ExplodedNode *>(Storage));
   Storage = node;
-  assert(Storage.is<ExplodedNode *>());
+  assert(isa<ExplodedNode *>(Storage));
 }
 
 void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
@@ -222,7 +222,7 @@ void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
   GroupStorage &Storage = reinterpret_cast<GroupStorage&>(P);
   if (Storage.isNull()) {
     Storage = N;
-    assert(Storage.is<ExplodedNode *>());
+    assert(isa<ExplodedNode *>(Storage));
     return;
   }
 
@@ -230,7 +230,7 @@ void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
 
   if (!V) {
     // Switch from single-node to multi-node representation.
-    ExplodedNode *Old = Storage.get<ExplodedNode *>();
+    auto *Old = cast<ExplodedNode *>(Storage);
 
     BumpVectorContext &Ctx = G.getNodeAllocator();
     V = new (G.getAllocator()) ExplodedNodeVector(Ctx, 4);
@@ -238,7 +238,7 @@ void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
 
     Storage = V;
     assert(!getFlag());
-    assert(Storage.is<ExplodedNodeVector *>());
+    assert(isa<ExplodedNodeVector *>(Storage));
   }
 
   V->push_back(N, G.getNodeAllocator());
@@ -349,6 +349,8 @@ const Stmt *ExplodedNode::getStmtForDiagnostics() const {
 
 const Stmt *ExplodedNode::getNextStmtForDiagnostics() const {
   for (const ExplodedNode *N = getFirstSucc(); N; N = N->getFirstSucc()) {
+    if (N->getLocation().isPurgeKind())
+      continue;
     if (const Stmt *S = N->getStmtForDiagnostics()) {
       // Check if the statement is '?' or '&&'/'||'.  These are "merges",
       // not actual statement points.
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c5c120b388b46..4580d0775fd5c 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1206,15 +1206,7 @@ void ExprEngine::ProcessInitializer(const CFGInitializer CFGInit,
         while ((ASE = dyn_cast<ArraySubscriptExpr>(Init)))
           Init = ASE->getBase()->IgnoreImplicit();
 
-        SVal LValue = State->getSVal(Init, stackFrame);
-        if (!Field->getType()->isReferenceType()) {
-          if (std::optional<Loc> LValueLoc = LValue.getAs<Loc>()) {
-            InitVal = State->getSVal(*LValueLoc);
-          } else if (auto CV = LValue.getAs<nonloc::CompoundVal>()) {
-            // Initializer list for an array.
-            InitVal = *CV;
-          }
-        }
+        InitVal = State->getSVal(Init, stackFrame);
 
         // If we fail to get the value for some reason, use a symbolic value.
         if (InitVal.isUnknownOrUndef()) {
@@ -1655,10 +1647,8 @@ void ExprEngine::processCleanupTemporaryBranch(const CXXBindTemporaryExpr *BTE,
   ProgramStateRef State = Pred->getState();
   const LocationContext *LC = Pred->getLocationContext();
   if (getObjectUnderConstruction(State, BTE, LC)) {
-    TempDtorBuilder.markInfeasible(false);
     TempDtorBuilder.generateNode(State, true, Pred);
   } else {
-    TempDtorBuilder.markInfeasible(true);
     TempDtorBuilder.generateNode(State, false, Pred);
   }
 }
@@ -2783,7 +2773,6 @@ void ExprEngine::processBranch(const Stmt *Condition,
   // Check for NULL conditions; e.g. "for(;;)"
   if (!Condition) {
     BranchNodeBuilder NullCondBldr(Pred, Dst, BldCtx, DstT, DstF);
-    NullCondBldr.markInfeasible(false);
     NullCondBldr.generateNode(Pred->getState(), true, Pred);
     return;
   }
@@ -2803,40 +2792,25 @@ void ExprEngine::processBranch(const Stmt *Condition,
   if (CheckersOutSet.empty())
     return;
 
-  BranchNodeBuilder builder(CheckersOutSet, Dst, BldCtx, DstT, DstF);
+  BranchNodeBuilder Builder(CheckersOutSet, Dst, BldCtx, DstT, DstF);
   for (ExplodedNode *PredN : CheckersOutSet) {
     if (PredN->isSink())
       continue;
 
     ProgramStateRef PrevState = PredN->getState();
 
-    ProgramStateRef StTrue, StFalse;
+    ProgramStateRef StTrue = PrevState, StFalse = PrevState;
     if (const auto KnownCondValueAssumption = assumeCondition(Condition, PredN))
       std::tie(StTrue, StFalse) = *KnownCondValueAssumption;
-    else {
-      assert(!isa<ObjCForCollectionStmt>(Condition));
-      builder.generateNode(PrevState, true, PredN);
-      builder.generateNode(PrevState, false, PredN);
-      continue;
-    }
+
     if (StTrue && StFalse)
       assert(!isa<ObjCForCollectionStmt>(Condition));
 
-    // Process the true branch.
-    if (builder.isFeasible(true)) {
-      if (StTrue)
-        builder.generateNode(StTrue, true, PredN);
-      else
-        builder.markInfeasible(true);
-    }
+    if (StTrue)
+      Builder.generateNode(StTrue, true, PredN);
 
-    // Process the false branch.
-    if (builder.isFeasible(false)) {
-      if (StFalse)
-        builder.generateNode(StFalse, false, PredN);
-      else
-        builder.markInfeasible(false);
-    }
+    if (StFalse)
+      Builder.generateNode(StFalse, false, PredN);
   }
   currBldrCtx = nullptr;
 }
@@ -2858,14 +2832,13 @@ void ExprEngine::processStaticInitializer(const DeclStmt *DS,
   const auto *VD = cast<VarDecl>(DS->getSingleDecl());
   ProgramStateRef state = Pred->getState();
   bool initHasRun = state->contains<InitializedGlobalsSet>(VD);
-  BranchNodeBuilder builder(Pred, Dst, BuilderCtx, DstT, DstF);
+  BranchNodeBuilder Builder(Pred, Dst, BuilderCtx, DstT, DstF);
 
   if (!initHasRun) {
     state = state->add<InitializedGlobalsSet>(VD);
   }
 
-  builder.generateNode(state, initHasRun, Pred);
-  builder.markInfeasible(!initHasRun);
+  Builder.generateNode(state, initHasRun, Pred);
 
   currBldrCtx = nullptr;
 }
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
index 2ca24d0c5ab22..02facf786830d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -353,14 +353,19 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
   ExplodedNodeSet CleanedNodes;
   if (LastSt && Blk && AMgr.options.AnalysisPurgeOpt != PurgeNone) {
     static SimpleProgramPointTag retValBind("ExprEngine", "Bind Return Value");
-    PostStmt Loc(LastSt, calleeCtx, &retValBind);
+    auto Loc = isa<ReturnStmt>(LastSt)
+                   ? ProgramPoint{PostStmt(LastSt, calleeCtx, &retValBind)}
+                   : ProgramPoint{EpsilonPoint(calleeCtx, /*Data1=*/nullptr,
+                                               /*Data2=*/nullptr, &retValBind)};
+    const CFGBlock *PrePurgeBlock =
+        isa<ReturnStmt>(LastSt) ? Blk : &CEBNode->getCFG().getExit();
     bool isNew;
     ExplodedNode *BindedRetNode = G.getNode(Loc, state, false, &isNew);
     BindedRetNode->addPredecessor(CEBNode, G);
     if (!isNew)
       return;
 
-    NodeBuilderContext Ctx(getCoreEngine(), Blk, BindedRetNode);
+    NodeBuilderContext Ctx(getCoreEngine(), PrePurgeBlock, BindedRetNode);
     currBldrCtx = &Ctx;
     // Here, we call the Symbol Reaper with 0 statement and callee location
     // context, telling it to clean up everything in the callee's context
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index 02d1358a2001e..bbf2303b9f6ef 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -65,6 +65,11 @@ using namespace ento;
 // MemRegion Construction.
 //===----------------------------------------------------------------------===//
 
+[[maybe_unused]] static bool isAReferenceTypedValueRegion(const MemRegion *R) {
+  const auto *TyReg = llvm::dyn_cast<TypedValueRegion>(R);
+  return TyReg && TyReg->getValueType()->isReferenceType();
+}
+
 template <typename RegionTy, typename SuperTy, typename Arg1Ty>
 RegionTy* MemRegionManager::getSubRegion(const Arg1Ty arg1,
                                          const SuperTy *superRegion) {
@@ -76,6 +81,7 @@ RegionTy* MemRegionManager::getSubRegion(const Arg1Ty arg1,
   if (!R) {
     R = new (A) RegionTy(arg1, superRegion);
     Regions.InsertNode(R, InsertPos);
+    assert(!isAReferenceTypedValueRegion(superRegion));
   }
 
   return R;
@@ -92,6 +98,7 @@ RegionTy* MemRegionManager::getSubRegion(const Arg1Ty arg1, const Arg2Ty arg2,
   if (!R) {
     R = new (A) RegionTy(arg1, arg2, superRegion);
     Regions.InsertNode(R, InsertPos);
+    assert(!isAReferenceTypedValueRegion(superRegion));
   }
 
   return R;
@@ -110,6 +117,7 @@ RegionTy* MemRegionManager::getSubRegion(const Arg1Ty arg1, const Arg2Ty arg2,
   if (!R) {
     R = new (A) RegionTy(arg1, arg2, arg3, superRegion);
     Regions.InsertNode(R, InsertPos);
+    assert(!isAReferenceTypedValueRegion(superRegion));
   }
 
   return R;
@@ -1060,10 +1068,10 @@ const VarRegion *MemRegionManager::getVarRegion(const VarDecl *D,
     llvm::PointerUnion<const StackFrameContext *, const VarRegion *> V =
       getStackOrCaptureRegionForDeclContext(LC, DC, D);
 
-    if (V.is<const VarRegion*>())
-      return V.get<const VarRegion*>();
+    if (const auto *VR = dyn_cast_if_present<const VarRegion *>(V))
+      return VR;
 
-    const auto *STC = V.get<const StackFrameContext *>();
+    const auto *STC = cast<const StackFrameContext *>(V);
 
     if (!STC) {
       // FIXME: Assign a more sensible memory space to static locals
diff --git a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
index 0be2709f0907d..d4f56342d934c 100644
--- a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
@@ -205,6 +205,16 @@ ProgramStateRef ProgramState::killBinding(Loc LV) const {
   return makeWithStore(newStore);
 }
 
+/// We should never form a MemRegion that would wrap a TypedValueRegion of a
+/// reference type. What we actually wanted was to create a MemRegion refering
+/// to the pointee of that reference.
+SVal ProgramState::desugarReference(SVal Val) const {
+  const auto *TyReg = dyn_cast_or_null<TypedValueRegion>(Val.getAsRegion());
+  if (!TyReg || !TyReg->getValueType()->isReferenceType())
+    return Val;
+  return getSVal(TyReg);
+}
+
 /// SymbolicRegions are expected to be wrapped by an ElementRegion as a
 /// canonical representation. As a canonical representation, SymbolicRegions
 /// should be wrapped by ElementRegions before getting a FieldRegion.
@@ -445,12 +455,14 @@ void ProgramState::setStore(const StoreRef &newStore) {
 }
 
 SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const {
+  Base = desugarReference(Base);
   Base = wrapSymbolicRegion(Base);
   return getStateManager().StoreMgr->getLValueField(D, Base);
 }
 
 SVal ProgramState::getLValue(const IndirectFieldDecl *D, SVal Base) const {
   StoreManager &SM = *getStateManager().StoreMgr;
+  Base = desugarReference(Base);
   Base = wrapSymbolicRegion(Base);
 
   // FIXME: This should work with `SM.getLValueField(D->getAnonField(), Base)`,
diff --git a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index 46e294a1741cf..ad45ab5757a5a 100644
--- a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -608,6 +608,8 @@ class RegionStoreManager : public StoreManager {
     return getBinding(getRegionBindings(S), L, T);
   }
 
+  std::optional<SVal> getUniqueDefaultBinding(RegionBindingsConstRef B,
+                                              const TypedValueRegion *R) const;
   std::optional<SVal>
   getUniqueDefaultBinding(nonloc::LazyCompoundVal LCV) const;
 
@@ -2349,6 +2351,11 @@ SVal RegionStoreManager::getBindingForStruct(RegionBindingsConstRef B,
   // behavior doesn't depend on the struct layout.
   // This way even an empty struct can carry taint, no matter if creduce drops
   // the last field member or not.
+
+  // Try to avoid creating a LCV if it would anyways just refer to a single
+  // default binding.
+  if (std::optional<SVal> Val = getUniqueDefaultBinding(B, R))
+    return *Val;
   return createLazyBinding(B, R);
 }
 
@@ -2609,14 +2616,12 @@ RegionBindingsRef RegionStoreManager::bindVector(RegionBindingsConstRef B,
 }
 
 std::optional<SVal>
-RegionStoreManager::getUniqueDefaultBinding(nonloc::LazyCompoundVal LCV) const {
-  const MemRegion *BaseR = LCV.getRegion();
-
-  // We only handle base regions.
-  if (BaseR != BaseR->getBaseRegion())
+RegionStoreManager::getUniqueDefaultBinding(RegionBindingsConstRef B,
+                                            const TypedValueRegion *R) const {
+  if (R != R->getBaseRegion())
     return std::nullopt;
 
-  const auto *Cluster = getRegionBindings(LCV.getStore()).lookup(BaseR);
+  const auto *Cluster = B.lookup(R);
   if (!Cluster || !llvm::hasSingleElement(*Cluster))
     return std::nullopt;
 
@@ -2624,6 +2629,12 @@ RegionStoreManager::getUniqueDefaultBinding(nonloc::LazyCompoundVal LCV) const {
   return Key.isDirect() ? std::optional<SVal>{} : Value;
 }
 
+std::optional<SVal>
+RegionStoreManager::getUniqueDefaultBinding(nonloc::LazyCompoundVal LCV) const {
+  RegionBindingsConstRef B = getRegionBindings(LCV.getStore());
+  return getUniqueDefaultBinding(B, LCV.getRegion());
+}
+
 std::optional<RegionBindingsRef> RegionStoreManager::tryBindSmallStruct(
     RegionBindingsConstRef B, const TypedValueRegion *R, const RecordDecl *RD,
     nonloc::LazyCompoundVal LCV) {
diff --git a/clang/lib/StaticAnalyzer/Core/SVals.cpp b/clang/lib/StaticAnalyzer/Core/SVals.cpp
index 84e7e033404c0..d009552965eca 100644
--- a/clang/lib/StaticAnalyzer/Core/SVals.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SVals.cpp
@@ -205,10 +205,10 @@ const NamedDecl *nonloc::PointerToMember::getDecl() const {
     return nullptr;
 
   const NamedDecl *ND = nullptr;
-  if (PTMD.is<const NamedDecl *>())
-    ND = PTMD.get<const NamedDecl *>();
+  if (const auto *NDP = dyn_cast<const NamedDecl *>(PTMD))
+    ND = NDP;
   else
-    ND = PTMD.get<const PointerToMemberData *>()->getDeclaratorDecl();
+    ND = cast<const PointerToMemberData *>(PTMD)->getDeclaratorDecl();
 
   return ND;
 }
@@ -227,16 +227,16 @@ nonloc::CompoundVal::iterator nonloc::CompoundVal::end() const {
 
 nonloc::PointerToMember::iterator nonloc::PointerToMember::begin() const {
   const PTMDataType PTMD = getPTMData();
-  if (PTMD.is<const NamedDecl *>())
+  if (isa<const NamedDecl *>(PTMD))
     return {};
-  return PTMD.get<const PointerToMemberData *>()->begin();
+  return cast<const PointerToMemberData *>(PTMD)->begin();
 }
 
 nonloc::PointerToMember::iterator nonloc::PointerToMember::end() const {
   const PTMDataType PTMD = getPTMData();
-  if (PTMD.is<const NamedDecl *>())
+  if (isa<const NamedDecl *>(PTMD))
     return {};
-  return PTMD.get<const PointerToMemberData *>()->end();
+  return cast<const PointerToMemberData *>(PTMD)->end();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
index c5171e2f287d2..88e0da1382d6c 100644
--- a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
@@ -3,6 +3,12 @@ Name: SwiftImportAs
 Tags:
 - Name: ImmortalRefType
   SwiftImportAs: reference
+  Methods:
+    - Name: methodReturningFrt__
+    - Name: methodReturningFrt_returns_unretained
+      SwiftReturnOwnership: unretained
+    - Name: methodReturningFrt_returns_retained
+      SwiftReturnOwnership: retained
 - Name: RefCountedType
   SwiftImportAs: reference
   SwiftReleaseOp: RCRelease
@@ -17,3 +23,10 @@ Tags:
   SwiftEscapable: false
 - Name: EscapableType
   SwiftEscapable: true
+
+Functions:
+  - Name: functionReturningFrt__
+  - Name: functionReturningFrt_returns_unretained
+    SwiftReturnOwnership: unretained
+  - Name: functionReturningFrt_returns_retained
+    SwiftReturnOwnership: retained
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
index f205cd3c6e7b7..b6900fee8a979 100644
--- a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
@@ -1,4 +1,13 @@
-struct ImmortalRefType {};
+struct ImmortalRefType {
+    ImmortalRefType * methodReturningFrt__(void);
+    ImmortalRefType * methodReturningFrt_returns_unretained(void);
+    ImmortalRefType * methodReturningFrt_returns_retained(void);
+};
+
+ImmortalRefType * functionReturningFrt__(void);
+ImmortalRefType * functionReturningFrt_returns_unretained(void);
+ImmortalRefType * functionReturningFrt_returns_retained(void);
+
 
 struct RefCountedType { int value; };
 
diff --git a/clang/test/APINotes/swift-import-as.cpp b/clang/test/APINotes/swift-import-as.cpp
index d0e7e31fc1d72..3981ef1ed419a 100644
--- a/clang/test/APINotes/swift-import-as.cpp
+++ b/clang/test/APINotes/swift-import-as.cpp
@@ -6,6 +6,12 @@
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter CopyableType | FileCheck -check-prefix=CHECK-COPYABLE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter NonEscapableType | FileCheck -check-prefix=CHECK-NON-ESCAPABLE %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter EscapableType | FileCheck -check-prefix=CHECK-ESCAPABLE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter functionReturningFrt__ | FileCheck -check-prefix=CHECK-FUNCTION-RETURNING-FRT %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter functionReturningFrt_returns_unretained | FileCheck -check-prefix=CHECK-FUNCTION-RETURNING-FRT-UNRETAINED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter functionReturningFrt_returns_retained | FileCheck -check-prefix=CHECK-FUNCTION-RETURNING-FRT-RETAINED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt__ | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt_returns_unretained | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT-UNRETAINED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt_returns_retained | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT-RETAINED %s
 
 #include <SwiftImportAs.h>
 
@@ -36,3 +42,29 @@
 // CHECK-ESCAPABLE: Dumping EscapableType:
 // CHECK-ESCAPABLE-NEXT: CXXRecordDecl {{.+}} imported in SwiftImportAs {{.+}} struct EscapableType
 // CHECK-ESCAPABLE: SwiftAttrAttr {{.+}} "Escapable"
+
+// CHECK-FUNCTION-RETURNING-FRT: Dumping functionReturningFrt__:
+// CHECK-FUNCTION-RETURNING-FRT: FunctionDecl {{.+}} imported in SwiftImportAs functionReturningFrt__ 'ImmortalRefType *()'
+// CHECK-FUNCTION-RETURNING-FRT-NOT: `-SwiftAttrAttr {{.+}} "returns_unretained"
+// CHECK-FUNCTION-RETURNING-FRT-NOT: `-SwiftAttrAttr {{.+}} "returns_retained"
+
+// CHECK-FUNCTION-RETURNING-FRT-UNRETAINED: Dumping functionReturningFrt_returns_unretained:
+// CHECK-FUNCTION-RETURNING-FRT-UNRETAINED: FunctionDecl {{.+}} imported in SwiftImportAs functionReturningFrt_returns_unretained 'ImmortalRefType *()'
+// CHECK-FUNCTION-RETURNING-FRT-UNRETAINED: `-SwiftAttrAttr {{.+}} "returns_unretained"
+
+// CHECK-FUNCTION-RETURNING-FRT-RETAINED: Dumping functionReturningFrt_returns_retained:
+// CHECK-FUNCTION-RETURNING-FRT-RETAINED: FunctionDecl {{.+}} imported in SwiftImportAs functionReturningFrt_returns_retained 'ImmortalRefType *()'
+// CHECK-FUNCTION-RETURNING-FRT-RETAINED: `-SwiftAttrAttr {{.+}} "returns_retained"
+
+// CHECK-METHOD-RETURNING-FRT: Dumping ImmortalRefType::methodReturningFrt__:
+// CHECK-METHOD-RETURNING-FRT: CXXMethodDecl {{.+}} imported in SwiftImportAs methodReturningFrt__ 'ImmortalRefType *()'
+// CHECK-METHOD-RETURNING-FRT-NOT: `-SwiftAttrAttr {{.+}} "returns_unretained"
+// CHECK-METHOD-RETURNING-FRT-NOT: `-SwiftAttrAttr {{.+}} "returns_retained"
+
+// CHECK-METHOD-RETURNING-FRT-UNRETAINED: Dumping ImmortalRefType::methodReturningFrt_returns_unretained:
+// CHECK-METHOD-RETURNING-FRT-UNRETAINED: CXXMethodDecl {{.+}} imported in SwiftImportAs methodReturningFrt_returns_unretained 'ImmortalRefType *()'
+// CHECK-METHOD-RETURNING-FRT-UNRETAINED: `-SwiftAttrAttr {{.+}} "returns_unretained"
+
+// CHECK-METHOD-RETURNING-FRT-RETAINED: Dumping ImmortalRefType::methodReturningFrt_returns_retained:
+// CHECK-METHOD-RETURNING-FRT-RETAINED: CXXMethodDecl {{.+}} imported in SwiftImportAs methodReturningFrt_returns_retained 'ImmortalRefType *()'
+// CHECK-METHOD-RETURNING-FRT-RETAINED: `-SwiftAttrAttr {{.+}} "returns_retained"
diff --git a/clang/test/AST/ByteCode/altivec.c b/clang/test/AST/ByteCode/altivec.c
new file mode 100644
index 0000000000000..39caedc9949d2
--- /dev/null
+++ b/clang/test/AST/ByteCode/altivec.c
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple=powerpc64-unknown-linux-gnu -target-feature +altivec -target-feature +vsx -fsyntax-only -verify=expected,both %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu -target-feature +altivec -target-feature -vsx -fsyntax-only -verify=expected,both %s
+// RUN: %clang_cc1 -triple=powerpc-ibm-aix -target-feature +altivec -fsyntax-only -verify=expected,both %s
+// RUN: %clang_cc1 -triple=powerpc64-ibm-aix -target-feature +altivec -target-feature -vsx -fsyntax-only -verify=expected,both %s
+
+// RUN: %clang_cc1 -triple=powerpc64-unknown-linux-gnu -target-feature +altivec -target-feature +vsx -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu -target-feature +altivec -target-feature -vsx -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -triple=powerpc-ibm-aix -target-feature +altivec -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -triple=powerpc64-ibm-aix -target-feature +altivec -target-feature -vsx -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter %s
+
+// both-no-diagnostics.
+
+/// From test/Parser/altivec.c
+vector char v1 = (vector char)((vector int)(1, 2, 3, 4));
+vector char v2 = (vector char)((vector float)(1.0f, 2.0f, 3.0f, 4.0f));
+vector char v3 = (vector char)((vector int)('a', 'b', 'c', 'd'));
+vector int v4 = (vector int)(1, 2, 3, 4);
+vector float v5 = (vector float)(1.0f, 2.0f, 3.0f, 4.0f);
+vector char v6 = (vector char)((vector int)(1+2, -2, (int)(2.0 * 3), -(5-3)));
diff --git a/clang/test/AST/ByteCode/amdgpu-nullptr.cl b/clang/test/AST/ByteCode/amdgpu-nullptr.cl
new file mode 100644
index 0000000000000..d79f1cd4a237b
--- /dev/null
+++ b/clang/test/AST/ByteCode/amdgpu-nullptr.cl
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -no-enable-noundef-analysis %s -cl-std=CL2.0 -triple amdgcn -emit-llvm -o - | FileCheck %s
+
+// RUN: %clang_cc1 -no-enable-noundef-analysis %s -cl-std=CL2.0 -triple amdgcn -emit-llvm -fexperimental-new-constant-interpreter -o - | FileCheck %s
+
+
+// CHECK: @fold_priv ={{.*}} local_unnamed_addr addrspace(1) global ptr addrspace(5) addrspacecast (ptr addrspace(1) null to ptr addrspace(5)), align 4
+private short *fold_priv = (private short*)(generic int*)(global void*)0;
+
+// CHECK: @fold_priv_arith ={{.*}} local_unnamed_addr addrspace(1) global ptr addrspace(5) inttoptr (i32 9 to ptr addrspace(5)), align 4
+private char *fold_priv_arith = (private char*)0 + 10;
+
+
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast-bitfields.cpp b/clang/test/AST/ByteCode/builtin-bit-cast-bitfields.cpp
new file mode 100644
index 0000000000000..5aa4e256e4638
--- /dev/null
+++ b/clang/test/AST/ByteCode/builtin-bit-cast-bitfields.cpp
@@ -0,0 +1,475 @@
+// RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -triple x86_64-linux-gnu -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -triple armv8 -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -triple aarch64_be-linux-gnu -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -fexperimental-new-constant-interpreter -triple powerpc64le-unknown-unknown -mabi=ieeelongdouble %s
+// RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -fexperimental-new-constant-interpreter -triple powerpc64-unknown-unknown -mabi=ieeelongdouble %s
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define LITTLE_END 1
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#  define LITTLE_END 0
+#else
+#  error "huh?"
+#endif
+
+typedef decltype(nullptr) nullptr_t;
+typedef __INTPTR_TYPE__ intptr_t;
+typedef unsigned __INT16_TYPE__ uint16_t;
+typedef unsigned __INT32_TYPE__ uint32_t;
+typedef unsigned __INT64_TYPE__ uint64_t;
+
+static_assert(sizeof(int) == 4);
+static_assert(sizeof(long long) == 8);
+
+template <class To, class From>
+constexpr To bit_cast(const From &from) {
+  static_assert(sizeof(To) == sizeof(From));
+  return __builtin_bit_cast(To, from);
+}
+
+template <class Intermediate, class Init>
+constexpr bool check_round_trip(const Init &init) {
+  return bit_cast<Init>(bit_cast<Intermediate>(init)) == init;
+}
+
+template <class Intermediate, class Init>
+constexpr Init round_trip(const Init &init) {
+  return bit_cast<Init>(bit_cast<Intermediate>(init));
+}
+
+namespace std {
+enum byte : unsigned char {};
+} // namespace std
+
+template <int N, typename T = unsigned char, int Pad = 0>
+struct bits {
+  T : Pad;
+  T bits : N;
+
+  constexpr bool operator==(const T& rhs) const {
+    return bits == rhs;
+  }
+};
+
+template <int N, typename T, int P>
+constexpr bool operator==(const struct bits<N, T, P>& lhs, const struct bits<N, T, P>& rhs) {
+  return lhs.bits == rhs.bits;
+}
+
+template<int N>
+struct bytes {
+  using size_t = unsigned int;
+  unsigned char d[N];
+
+  constexpr unsigned char operator[](size_t index) {
+    if (index < N)
+      return d[index]; // expected-note {{read of uninitialized object}}
+    return -1;
+  }
+};
+
+namespace Sanity {
+  /// This is just one byte, and we extract 2 bits from it.
+  ///
+  /// 3 is 0000'0011.
+  /// For both LE and BE, the buffer will contain exactly that
+  /// byte, unaltered and not reordered in any way. It contains all 8 bits.
+  static_assert(__builtin_bit_cast(bits<2>, (unsigned char)3) == (LITTLE_END ? 3 : 0));
+
+  /// Similarly, we have one full byte of data, with the two most-significant
+  /// bits set:
+  /// 192 is 1100'0000
+  static_assert(__builtin_bit_cast(bits<2>, (unsigned char)192) == (LITTLE_END ? 0 : 3));
+
+
+  /// Here we are instead bitcasting two 1-bits into a destination of 8 bits.
+  /// On LE, we should pick the two least-significant bits. On BE, the opposite.
+  /// NOTE: Can't verify this with gcc.
+  constexpr auto B1 = bits<2>{3};
+  static_assert(__builtin_bit_cast(unsigned char, B1) == (LITTLE_END ? 3 : 192));
+
+  /// This should be 0000'0110.
+  /// On LE, this should result in 6.
+  /// On BE, 1100'0000 = 192.
+  constexpr auto B2 = bits<3>{6};
+  static_assert(__builtin_bit_cast(unsigned char, B2) == (LITTLE_END ? 6 : 192));
+
+  constexpr auto B3 = bits<4>{6};
+  static_assert(__builtin_bit_cast(unsigned char, B3) == (LITTLE_END ? 6 : 96));
+
+  struct B {
+    std::byte b0 : 4;
+    std::byte b1 : 4;
+  };
+
+  /// We can properly decompose one byte (8 bit) int two 4-bit bitfields.
+  constexpr struct { unsigned char b0; } T = {0xee};
+  constexpr B MB = __builtin_bit_cast(B, T);
+  static_assert(MB.b0 == 0xe);
+  static_assert(MB.b1 == 0xe);
+}
+
+namespace BitFields {
+  struct BitFields {
+    unsigned a : 2;
+    unsigned b : 30;
+  };
+
+  constexpr unsigned A = __builtin_bit_cast(unsigned, BitFields{3, 16});
+  static_assert(A == (LITTLE_END ? 67 : 3221225488));
+
+  struct S {
+    unsigned a : 2;
+    unsigned b : 28;
+    unsigned c : 2;
+  };
+
+  constexpr S s = __builtin_bit_cast(S, 0xFFFFFFFF);
+  static_assert(s.a == 3);
+  static_assert(s.b == 268435455);
+  static_assert(s.c == 3);
+
+  void bitfield_indeterminate() {
+    struct BF { unsigned char z : 2; };
+    enum byte : unsigned char {};
+
+    constexpr BF bf = {0x3};
+    static_assert(bit_cast<bits<2>>(bf).bits == bf.z);
+    static_assert(bit_cast<unsigned char>(bf));
+
+    static_assert(__builtin_bit_cast(byte, bf)); // expected-error {{not an integral constant expression}} \
+                                                 // expected-note {{indeterminate value can only initialize an object of type 'unsigned char' or 'std::byte'; 'byte' is invalid}}
+
+    struct M {
+      // expected-note@+1 {{subobject declared here}}
+      unsigned char mem[sizeof(BF)];
+    };
+    // expected-error@+2 {{initialized by a constant expression}}
+    // expected-note@+1 {{not initialized}}
+    constexpr M m = bit_cast<M>(bf);
+
+    constexpr auto f = []() constexpr {
+      // bits<24, unsigned int, LITTLE_END ? 0 : 8> B = {0xc0ffee};
+      constexpr struct { unsigned short b1; unsigned char b0;  } B = {0xc0ff, 0xee};
+      return bit_cast<bytes<4>>(B);
+    };
+
+    static_assert(f()[0] + f()[1] + f()[2] == 0xc0 + 0xff + 0xee);
+    {
+      // expected-error@+2 {{initialized by a constant expression}}
+      // expected-note@+1 {{in call to}}
+      constexpr auto _bad = f()[3];
+    }
+
+    struct B {
+      unsigned short s0 : 8;
+      unsigned short s1 : 8;
+      std::byte b0 : 4;
+      std::byte b1 : 4;
+      std::byte b2 : 4;
+    };
+    constexpr auto g = [f]() constexpr {
+      return bit_cast<B>(f());
+    };
+    static_assert(g().s0 + g().s1 + g().b0 + g().b1 == 0xc0 + 0xff + 0xe + 0xe);
+    {
+      // expected-error@+2 {{initialized by a constant expression}}
+      // expected-note@+1 {{read of uninitialized object is not allowed in a constant expression}}
+      constexpr auto _bad = g().b2;
+    }
+  }
+}
+
+namespace BoolVectors {
+  typedef bool bool32 __attribute__((ext_vector_type(32)));
+  constexpr auto v = bit_cast<bool32>(0xa1c0ffee);
+#if LITTLE_END
+  static_assert(!v[0]);
+  static_assert(v[1]);
+  static_assert(v[2]);
+  static_assert(v[3]);
+  static_assert(!v[4]);
+  static_assert(v[5]);
+  static_assert(v[6]);
+  static_assert(v[7]);
+
+  static_assert(v[8]);
+  static_assert(v[9]);
+  static_assert(v[10]);
+  static_assert(v[11]);
+  static_assert(v[12]);
+  static_assert(v[13]);
+  static_assert(v[14]);
+  static_assert(v[15]);
+
+  static_assert(!v[16]);
+  static_assert(!v[17]);
+  static_assert(!v[18]);
+  static_assert(!v[19]);
+  static_assert(!v[20]);
+  static_assert(!v[21]);
+  static_assert(v[22]);
+  static_assert(v[23]);
+
+  static_assert(v[24]);
+  static_assert(!v[25]);
+  static_assert(!v[26]);
+  static_assert(!v[27]);
+  static_assert(!v[28]);
+  static_assert(v[29]);
+  static_assert(!v[30]);
+  static_assert(v[31]);
+
+#else
+  static_assert(v[0]);
+  static_assert(!v[1]);
+  static_assert(v[2]);
+  static_assert(!v[3]);
+  static_assert(!v[4]);
+  static_assert(!v[5]);
+  static_assert(!v[6]);
+  static_assert(v[7]);
+
+  static_assert(v[8]);
+  static_assert(v[9]);
+  static_assert(!v[10]);
+  static_assert(!v[11]);
+  static_assert(!v[12]);
+  static_assert(!v[13]);
+  static_assert(!v[14]);
+  static_assert(!v[15]);
+
+  static_assert(v[16]);
+  static_assert(v[17]);
+  static_assert(v[18]);
+  static_assert(v[19]);
+  static_assert(v[20]);
+  static_assert(v[21]);
+  static_assert(v[22]);
+  static_assert(v[23]);
+
+  static_assert(v[24]);
+  static_assert(v[25]);
+  static_assert(v[26]);
+  static_assert(!v[27]);
+  static_assert(v[28]);
+  static_assert(v[29]);
+  static_assert(v[30]);
+  static_assert(!v[31]);
+#endif
+
+  struct pad {
+    unsigned short s;
+    unsigned char c;
+  };
+
+  constexpr auto p = bit_cast<pad>(v);
+  static_assert(p.s == (LITTLE_END ? 0xffee : 0xa1c0));
+  static_assert(p.c == (LITTLE_END ? 0xc0 : 0xff));
+}
+
+namespace TwoShorts {
+  struct B {
+    unsigned short s0 : 8;
+    unsigned short s1 : 8;
+  };
+  constexpr struct { unsigned short b1;} T = {0xc0ff};
+  constexpr B MB = __builtin_bit_cast(B, T);
+#if LITTLE_END
+    static_assert(MB.s0 == 0xff);
+    static_assert(MB.s1 == 0xc0);
+#else
+    static_assert(MB.s0 == 0xc0);
+    static_assert(MB.s1 == 0xff);
+
+#endif
+}
+
+typedef bool bool8 __attribute__((ext_vector_type(8)));
+typedef bool bool9 __attribute__((ext_vector_type(9)));
+typedef bool bool16 __attribute__((ext_vector_type(16)));
+typedef bool bool17 __attribute__((ext_vector_type(17)));
+typedef bool bool32 __attribute__((ext_vector_type(32)));
+typedef bool bool128 __attribute__((ext_vector_type(128)));
+
+static_assert(bit_cast<unsigned char>(bool8{1,0,1,0,1,0,1,0}) == (LITTLE_END ? 0x55 : 0xAA), "");
+constexpr bool8 b8 = __builtin_bit_cast(bool8, 0x55); // both-error {{'__builtin_bit_cast' source type 'int' does not match destination type 'bool8' (vector of 8 'bool' values) (4 vs 1 bytes)}}
+static_assert(check_round_trip<bool8>(static_cast<unsigned char>(0)), "");
+static_assert(check_round_trip<bool8>(static_cast<unsigned char>(1)), "");
+static_assert(check_round_trip<bool8>(static_cast<unsigned char>(0x55)), "");
+
+static_assert(bit_cast<unsigned short>(bool16{1,1,1,1,1,0,0,0, 1,1,1,1,0,1,0,0}) == (LITTLE_END ? 0x2F1F : 0xF8F4), "");
+
+static_assert(check_round_trip<bool16>(static_cast<short>(0xCAFE)), "");
+static_assert(check_round_trip<bool32>(static_cast<int>(0xCAFEBABE)), "");
+
+#ifdef __SIZEOF_INT128__
+static_assert(check_round_trip<bool128>(static_cast<__int128_t>(0xCAFEBABE0C05FEFEULL)), "");
+#endif
+
+static_assert(bit_cast<bits<8, uint16_t, 7>, uint16_t>(0xcafe) == (LITTLE_END ? 0x95 : 0x7f));
+static_assert(bit_cast<bits<4, uint16_t, 10>, uint16_t>(0xcafe) == (LITTLE_END ? 0x2 : 0xf));
+static_assert(bit_cast<bits<4, uint32_t, 19>, uint32_t>(0xa1cafe) == (LITTLE_END ? 0x4 : 0x5));
+
+struct S {
+  // little endian:
+  //    MSB .... .... LSB
+  //        |y|   |x|
+  //
+  // big endian
+  //    MSB .... .... LSB
+  //        |x|   |y|
+
+  unsigned char x : 4;
+  unsigned char y : 4;
+
+  constexpr bool operator==(S const &other) const {
+    return x == other.x && y == other.y;
+  }
+};
+
+constexpr S s{0xa, 0xb};
+static_assert(bit_cast<bits<8>>(s) == (LITTLE_END ? 0xba : 0xab));
+static_assert(bit_cast<bits<7>>(s) == (LITTLE_END
+                                            ? 0xba & 0x7f
+                                            : (0xab & 0xfe) >> 1));
+
+static_assert(round_trip<bits<8>>(s) == s);
+
+struct R {
+  unsigned int r : 31;
+  unsigned int : 0;
+  unsigned int : 32;
+  constexpr bool operator==(R const &other) const {
+    return r == other.r;
+  }
+ };
+using T = bits<31, signed long long>;
+constexpr R r{0x4ac0ffee};
+constexpr T t = bit_cast<T>(r);
+static_assert(t == ((0xFFFFFFFF8 << 28) | 0x4ac0ffee)); // sign extension
+
+static_assert(round_trip<T>(r) == r);
+static_assert(round_trip<R>(t) == t);
+
+
+/// The oversized bitfield is an error on Windows and not just a warning.
+#if !defined(_WIN32)
+struct U {
+  // expected-warning@+1 {{exceeds the width of its type}}
+  uint32_t trunc : 33;
+  uint32_t u : 31;
+  constexpr bool operator==(U const &other) const {
+    return trunc == other.trunc && u == other.u;
+  }
+};
+struct V {
+  uint64_t notrunc : 32;
+  uint64_t : 1;
+  uint64_t v : 31;
+  constexpr bool operator==(V const &other) const {
+    return notrunc == other.notrunc && v == other.v;
+  }
+};
+
+constexpr U u{static_cast<unsigned int>(~0), 0x4ac0ffee};
+constexpr V v = bit_cast<V>(u);
+static_assert(v.v == 0x4ac0ffee);
+
+static_assert(round_trip<V>(u) == u);
+static_assert(round_trip<U>(v) == v);
+
+constexpr auto w = bit_cast<bits<12, unsigned long, 33>>(u);
+static_assert(w == (LITTLE_END
+                    ? 0x4ac0ffee & 0xFFF
+                    : (0x4ac0ffee & (0xFFF << (31 - 12))) >> (31-12)
+                  ));
+#endif
+
+
+namespace NestedStructures {
+  struct J {
+    struct {
+      uint16_t  k : 12;
+    } K;
+    struct {
+      uint16_t  l : 4;
+    } L;
+  };
+
+  static_assert(sizeof(J) == 4);
+  constexpr J j = bit_cast<J>(0x8c0ffee5);
+
+  static_assert(j.K.k == (LITTLE_END ? 0xee5 : 0x8c0));
+  static_assert(j.L.l == 0xf /* yay symmetry */);
+  static_assert(bit_cast<bits<4, uint16_t, 16>>(j) == 0xf);
+  struct N {
+    bits<12, uint16_t> k;
+    uint16_t : 16;
+  };
+  static_assert(bit_cast<N>(j).k == j.K.k);
+
+  struct M {
+    bits<4, uint16_t, 0> m[2];
+    constexpr bool operator==(const M& rhs) const {
+      return m[0] == rhs.m[0] && m[1] == rhs.m[1];
+    };
+  };
+  #if LITTLE_END == 1
+  constexpr uint16_t want[2] = {0x5, 0xf};
+  #else
+  constexpr uint16_t want[2] = {0x8000, 0xf000};
+  #endif
+
+  static_assert(bit_cast<M>(j) == bit_cast<M>(want));
+}
+
+namespace Enums {
+  // ensure we're packed into the top 2 bits
+  constexpr int pad = LITTLE_END ? 6 : 0;
+  struct X
+  {
+    char : pad;
+    enum class direction: char { left, right, up, down } direction : 2;
+  };
+
+  constexpr X x = { X::direction::down };
+  static_assert(bit_cast<bits<2, signed char, pad>>(x) == -1);
+  static_assert(bit_cast<bits<2, unsigned char, pad>>(x) == 3);
+  static_assert(
+    bit_cast<X>((unsigned char)0x40).direction == X::direction::right);
+}
+
+namespace IndeterminateBits {
+  struct S {
+    unsigned a : 13;
+    unsigned   : 17;
+    unsigned b : 2;
+  };
+  constexpr unsigned A = __builtin_bit_cast(unsigned, S{12, 3}); // expected-error {{must be initialized by a constant expression}} \
+                                                                 // expected-note {{indeterminate value can only initialize an object of type 'unsigned char' or 'std::byte'; 'unsigned int' is invalid}}
+
+
+  /// GCC refuses to compile this as soon as we access the indeterminate bits
+  /// in the static_assert. MSVC accepts it.
+  struct S2 {
+    unsigned char a : 2;
+  };
+  constexpr unsigned char B = __builtin_bit_cast(unsigned char, S2{3});
+  static_assert(B == (LITTLE_END ? 3 : 192));
+
+
+
+  struct S3 {
+    unsigned a : 13;
+    unsigned   : 17;
+    unsigned b : 2;
+  };
+
+  struct D {
+    unsigned a;
+  };
+  constexpr D s = __builtin_bit_cast(D, S3{12, 3}); // expected-error {{must be initialized by a constant expression}} \
+                                                    // expected-note {{indeterminate value can only initialize an object of type 'unsigned char' or 'std::byte'; 'unsigned int' is invalid}}
+
+}
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
index 0929f7cb70b74..710612bef8fd0 100644
--- a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
+++ b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
@@ -6,7 +6,10 @@
 // RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -triple x86_64-apple-macosx10.14.0 %s -fno-signed-char -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -verify=expected,both -std=c++2a -fsyntax-only -triple aarch64_be-linux-gnu %s -fexperimental-new-constant-interpreter
 
+#if !__x86_64
 // both-no-diagnostics
+#endif
+
 
 typedef decltype(nullptr) nullptr_t;
 typedef __INTPTR_TYPE__ intptr_t;
@@ -35,6 +38,7 @@ constexpr Init round_trip(const Init &init) {
 
 namespace test_long_double {
 #if __x86_64
+/// FIXME: We could enable this, but since it aborts, it causes the usual mempory leak.
 #if 0
 constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // expected-error{{must be initialized by a constant expression}}\
                                                                                  // expected-note{{in call}}
@@ -49,7 +53,13 @@ static_assert(round_trip<bytes>(ld), "");
 
 static_assert(round_trip<long double>(10.0L));
 
-#if 0
+constexpr long double foo() {
+  bytes A = __builtin_bit_cast(bytes, ld);
+  long double ld = __builtin_bit_cast(long double, A);
+  return ld;
+}
+static_assert(foo() == ld);
+
 constexpr bool f(bool read_uninit) {
   bytes b = bit_cast<bytes>(ld);
   unsigned char ld_bytes[10] = {
@@ -61,16 +71,15 @@ constexpr bool f(bool read_uninit) {
     if (ld_bytes[i] != b.d[i])
       return false;
 
-  if (read_uninit && b.d[10]) // expected-note{{read of uninitialized object is not allowed in a constant expression}}
+  if (read_uninit && b.d[10]) // both-note{{read of uninitialized object is not allowed in a constant expression}}
     return false;
 
   return true;
 }
 
 static_assert(f(/*read_uninit=*/false), "");
-static_assert(f(/*read_uninit=*/true), ""); // expected-error{{static assertion expression is not an integral constant expression}} \
-                                            // expected-note{{in call to 'f(true)'}}
-#endif
+static_assert(f(/*read_uninit=*/true), ""); // both-error{{static assertion expression is not an integral constant expression}} \
+                                            // both-note{{in call to 'f(true)'}}
 constexpr bytes ld539 = {
   0x0, 0x0,  0x0,  0x0,
   0x0, 0x0,  0xc0, 0x86,
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast.cpp b/clang/test/AST/ByteCode/builtin-bit-cast.cpp
index 0fecde59cd57c..8a5bef635b8fd 100644
--- a/clang/test/AST/ByteCode/builtin-bit-cast.cpp
+++ b/clang/test/AST/ByteCode/builtin-bit-cast.cpp
@@ -130,11 +130,13 @@ namespace simple {
   static_assert(check_round_trip<unsigned>((int)0x0C05FEFE));
   static_assert(round_trip<float>((int)0x0C05FEFE));
 
+  static_assert(__builtin_bit_cast(intptr_t, nullptr) == 0); // both-error {{not an integral constant expression}} \
+                                                             // both-note {{indeterminate value can only initialize an object}}
 
-  /// This works in GCC and in the bytecode interpreter, but the current interpreter
-  /// diagnoses it.
-  static_assert(__builtin_bit_cast(intptr_t, nullptr) == 0); // ref-error {{not an integral constant expression}} \
-                                                             // ref-note {{indeterminate value can only initialize an object}}
+  constexpr int test_from_nullptr_pass = (__builtin_bit_cast(unsigned char[sizeof(nullptr)], nullptr), 0);
+  constexpr unsigned char NPData[sizeof(nullptr)] = {1,2,3,4};
+  constexpr nullptr_t NP = __builtin_bit_cast(nullptr_t, NPData);
+  static_assert(NP == nullptr);
 }
 
 namespace Fail {
@@ -186,72 +188,6 @@ namespace bitint {
                                 // ref-note {{initializer of 'IB' is not a constant expression}}
 }
 
-namespace BitFields {
-  struct BitFields {
-    unsigned a : 2;
-    unsigned b : 30;
-  };
-
-  constexpr unsigned A = __builtin_bit_cast(unsigned, BitFields{3, 16}); // ref-error {{must be initialized by a constant expression}} \
-                                                                         // ref-note {{not yet supported}} \
-                                                                         // ref-note {{declared here}}
-  static_assert(A == (LITTLE_END ? 67 : 3221225488)); // ref-error {{not an integral constant expression}} \
-                                                      // ref-note {{initializer of 'A'}}
-
-
-  void bitfield_indeterminate() {
-    struct BF { unsigned char z : 2; };
-    enum byte : unsigned char {};
-
-    constexpr BF bf = {0x3};
-    /// Requires bitcasts to composite types.
-    // static_assert(bit_cast<bits<2>>(bf).bits == bf.z);
-    // static_assert(bit_cast<unsigned char>(bf));
-
-#if 0
-    // static_assert(__builtin_bit_cast(byte, bf));
-
-    struct M {
-      // expected-note@+1 {{subobject declared here}}
-      unsigned char mem[sizeof(BF)];
-    };
-    // expected-error@+2 {{initialized by a constant expression}}
-    // expected-note@+1 {{not initialized}}
-    constexpr M m = bit_cast<M>(bf);
-
-    constexpr auto f = []() constexpr {
-      // bits<24, unsigned int, LITTLE_END ? 0 : 8> B = {0xc0ffee};
-      constexpr struct { unsigned short b1; unsigned char b0;  } B = {0xc0ff, 0xee};
-      return bit_cast<bytes<4>>(B);
-    };
-
-    static_assert(f()[0] + f()[1] + f()[2] == 0xc0 + 0xff + 0xee);
-    {
-      // expected-error@+2 {{initialized by a constant expression}}
-      // expected-note@+1 {{read of uninitialized object is not allowed in a constant expression}}
-      constexpr auto _bad = f()[3];
-    }
-
-    struct B {
-      unsigned short s0 : 8;
-      unsigned short s1 : 8;
-      std::byte b0 : 4;
-      std::byte b1 : 4;
-      std::byte b2 : 4;
-    };
-    constexpr auto g = [f]() constexpr {
-      return bit_cast<B>(f());
-    };
-    static_assert(g().s0 + g().s1 + g().b0 + g().b1 == 0xc0 + 0xff + 0xe + 0xe);
-    {
-      // expected-error@+2 {{initialized by a constant expression}}
-      // expected-note@+1 {{read of uninitialized object is not allowed in a constant expression}}
-      constexpr auto _bad = g().b2;
-    }
-#endif
-  }
-}
-
 namespace Classes {
   class A {
   public:
@@ -331,6 +267,21 @@ static_assert(check_round_trip<long long>(splice));
 #endif
 
 
+namespace Overread {
+  /// This used to crash becaus we were reading all elements of the
+  /// source array even though we should only be reading 1.
+  constexpr int a[] = {2,3, 4, 5};
+  constexpr int b = __builtin_bit_cast(int, *(a + 1));
+  static_assert(b == 3);
+
+  struct S {
+    int a;
+  };
+  constexpr S ss[] = {{1},{2}};
+  constexpr int c = __builtin_bit_cast(int, *(ss + 1));
+  static_assert(c == 2);
+}
+
 
 /// ---------------------------------------------------------------------------
 /// From here on, it's things copied from test/SemaCXX/constexpr-builtin-bit.cast.cpp
@@ -401,9 +352,8 @@ void test_record() {
   static_assert(t4 == tuple4{1, 2, 3, 4});
   static_assert(check_round_trip<tuple4>(b));
 
-  /// FIXME: We need to initialize the base pointers in the pointer we're bitcasting to.
-//  constexpr auto b2 = bit_cast<bases>(t4);
-//  static_assert(t4 == b2);
+  constexpr auto b2 = bit_cast<bases>(t4);
+  static_assert(t4 == b2);
 }
 
 void test_partially_initialized() {
@@ -440,7 +390,6 @@ void bad_types() {
   };
   static_assert(__builtin_bit_cast(int, X{0}) == 0); // both-error {{not an integral constant expression}} \
                                                      // both-note {{bit_cast from a union type is not allowed in a constant expression}}
-#if 1
 
   struct G {
     int g;
@@ -451,19 +400,17 @@ void bad_types() {
   // both-error@+2 {{constexpr variable 'x' must be initialized by a constant expression}}
   // both-note@+1 {{bit_cast to a union type is not allowed in a constant expression}}
   constexpr X x = __builtin_bit_cast(X, G{0});
-#endif
+
   struct has_pointer {
-    int *ptr; // both-note {{invalid type 'int *' is a member of 'has_pointer'}}
+    int *ptr; // both-note 2{{invalid type 'int *' is a member of 'has_pointer'}}
   };
 
   constexpr intptr_t ptr = __builtin_bit_cast(intptr_t, has_pointer{0}); // both-error {{constexpr variable 'ptr' must be initialized by a constant expression}} \
                                                                          // both-note {{bit_cast from a pointer type is not allowed in a constant expression}}
 
-#if 0
-  // expected-error@+2 {{constexpr variable 'hptr' must be initialized by a constant expression}}
-  // expected-note@+1 {{bit_cast to a pointer type is not allowed in a constant expression}}
-  constexpr has_pointer hptr =  __builtin_bit_cast(has_pointer, 0ul);
-#endif
+  // both-error@+2 {{constexpr variable 'hptr' must be initialized by a constant expression}}
+  // both-note@+1 {{bit_cast to a pointer type is not allowed in a constant expression}}
+  constexpr has_pointer hptr =  __builtin_bit_cast(has_pointer, (intptr_t)0);
 }
 
 void test_array_fill() {
@@ -510,27 +457,6 @@ static_assert(bit_cast<unsigned long long>(test_vector) == (LITTLE_END
 static_assert(check_round_trip<uint2>(0xCAFEBABE0C05FEFEULL), "");
 static_assert(check_round_trip<byte8>(0xCAFEBABE0C05FEFEULL), "");
 
-typedef bool bool8 __attribute__((ext_vector_type(8)));
-typedef bool bool9 __attribute__((ext_vector_type(9)));
-typedef bool bool16 __attribute__((ext_vector_type(16)));
-typedef bool bool17 __attribute__((ext_vector_type(17)));
-typedef bool bool32 __attribute__((ext_vector_type(32)));
-typedef bool bool128 __attribute__((ext_vector_type(128)));
-
-static_assert(bit_cast<unsigned char>(bool8{1,0,1,0,1,0,1,0}) == (LITTLE_END ? 0x55 : 0xAA), "");
-constexpr bool8 b8 = __builtin_bit_cast(bool8, 0x55); // both-error {{'__builtin_bit_cast' source type 'int' does not match destination type 'bool8' (vector of 8 'bool' values) (4 vs 1 bytes)}}
-#if 0
-static_assert(check_round_trip<bool8>(static_cast<unsigned char>(0)), "");
-static_assert(check_round_trip<bool8>(static_cast<unsigned char>(1)), "");
-static_assert(check_round_trip<bool8>(static_cast<unsigned char>(0x55)), "");
-
-static_assert(bit_cast<unsigned short>(bool16{1,1,1,1,1,0,0,0, 1,1,1,1,0,1,0,0}) == (LITTLE_END ? 0x2F1F : 0xF8F4), "");
-
-static_assert(check_round_trip<bool16>(static_cast<short>(0xCAFE)), "");
-static_assert(check_round_trip<bool32>(static_cast<int>(0xCAFEBABE)), "");
-static_assert(check_round_trip<bool128>(static_cast<__int128_t>(0xCAFEBABE0C05FEFEULL)), "");
-#endif
-
 #if 0
 // expected-error@+2 {{constexpr variable 'bad_bool9_to_short' must be initialized by a constant expression}}
 // expected-note@+1 {{bit_cast involving type 'bool __attribute__((ext_vector_type(9)))' (vector of 9 'bool' values) is not allowed in a constant expression; element size 1 * element count 9 is not a multiple of the byte size 8}}
@@ -559,3 +485,25 @@ namespace test_complex {
   constexpr double D = __builtin_bit_cast(double, test_float_complex);
   constexpr int M = __builtin_bit_cast(int, test_int_complex); // both-error {{size of '__builtin_bit_cast' source type 'const _Complex unsigned int' does not match destination type 'int' (8 vs 4 bytes)}}
 }
+
+
+namespace OversizedBitField {
+#if defined(_WIN32)
+  /// This is an error (not just a warning) on Windows and the field ends up with a size of 1 instead of 4.
+#else
+  typedef unsigned __INT16_TYPE__ uint16_t;
+  typedef unsigned __INT32_TYPE__ uint32_t;
+  struct S {
+    uint16_t a : 20; // both-warning {{exceeds the width of its type}}
+  };
+
+  static_assert(sizeof(S) == 4);
+  static_assert(__builtin_bit_cast(S, (uint32_t)32).a == (LITTLE_END ? 32 : 0)); // ref-error {{not an integral constant expression}} \
+                                                                                 // ref-note {{constexpr bit_cast involving bit-field is not yet supported}}
+#endif
+}
+
+typedef bool bool9 __attribute__((ext_vector_type(9)));
+// both-error@+2 {{constexpr variable 'bad_bool9_to_short' must be initialized by a constant expression}}
+// both-note@+1 {{bit_cast involving type 'bool __attribute__((ext_vector_type(9)))' (vector of 9 'bool' values) is not allowed in a constant expression; element size 1 * element count 9 is not a multiple of the byte size 8}}
+constexpr unsigned short bad_bool9_to_short = __builtin_bit_cast(unsigned short, bool9{1,1,0,1,0,1,0,1,0});
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index b5d334178f821..7dd08cb5fa1c3 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -7,6 +7,18 @@
 // RUN: %clang_cc1 -triple avr -std=c++20 -Wno-string-plus-int -fexperimental-new-constant-interpreter %s -verify=expected,both
 // RUN: %clang_cc1 -triple avr -std=c++20 -Wno-string-plus-int -verify=ref,both %s -Wno-constant-evaluated
 
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define LITTLE_END 1
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define LITTLE_END 0
+#else
+#error "huh?"
+#endif
+
+extern "C" {
+  typedef decltype(sizeof(int)) size_t;
+  extern size_t wcslen(const wchar_t *p);
+}
 
 namespace strcmp {
   constexpr char kFoobar[6] = {'f','o','o','b','a','r'};
@@ -39,6 +51,15 @@ namespace strcmp {
     return __builtin_strcmp(buffer, "mutable") == 0;
   }
   static_assert(char_memchr_mutable(), "");
+
+  static_assert(__builtin_strncmp("abaa", "abba", 5) == -1);
+  static_assert(__builtin_strncmp("abaa", "abba", 4) == -1);
+  static_assert(__builtin_strncmp("abaa", "abba", 3) == -1);
+  static_assert(__builtin_strncmp("abaa", "abba", 2) == 0);
+  static_assert(__builtin_strncmp("abaa", "abba", 1) == 0);
+  static_assert(__builtin_strncmp("abaa", "abba", 0) == 0);
+  static_assert(__builtin_strncmp(0, 0, 0) == 0);
+  static_assert(__builtin_strncmp("abab\0banana", "abab\0canada", 100) == 0);
 }
 
 /// Copied from constant-expression-cxx11.cpp
@@ -85,6 +106,14 @@ constexpr const char *a = "foo\0quux";
   constexpr char d[] = { 'f', 'o', 'o' }; // no nul terminator.
   constexpr int bad = __builtin_strlen(d); // both-error {{constant expression}} \
                                            // both-note {{one-past-the-end}}
+
+  constexpr int wn = __builtin_wcslen(L"hello");
+  static_assert(wn == 5);
+  constexpr int wm = wcslen(L"hello"); // both-error {{constant expression}} \
+                                       // both-note {{non-constexpr function 'wcslen' cannot be used in a constant expression}}
+
+  int arr[3]; // both-note {{here}}
+  int wk = arr[wcslen(L"hello")]; // both-warning {{array index 5}}
 }
 
 namespace nan {
@@ -990,3 +1019,192 @@ namespace BuiltinInImplicitCtor {
   } Foo;
   static_assert(Foo.a == 0, "");
 }
+
+typedef double vector4double __attribute__((__vector_size__(32)));
+typedef float vector4float __attribute__((__vector_size__(16)));
+typedef long long vector4long __attribute__((__vector_size__(32)));
+typedef int vector4int __attribute__((__vector_size__(16)));
+typedef unsigned long long vector4ulong __attribute__((__vector_size__(32)));
+typedef unsigned int vector4uint __attribute__((__vector_size__(16)));
+typedef short vector4short __attribute__((__vector_size__(8)));
+typedef char vector4char __attribute__((__vector_size__(4)));
+typedef double vector8double __attribute__((__vector_size__(64)));
+typedef float vector8float __attribute__((__vector_size__(32)));
+typedef long long vector8long __attribute__((__vector_size__(64)));
+typedef int vector8int __attribute__((__vector_size__(32)));
+typedef short vector8short __attribute__((__vector_size__(16)));
+typedef char vector8char __attribute__((__vector_size__(8)));
+
+namespace RecuceAdd {
+  static_assert(__builtin_reduce_add((vector4char){}) == 0);
+  static_assert(__builtin_reduce_add((vector4char){1, 2, 3, 4}) == 10);
+  static_assert(__builtin_reduce_add((vector4short){10, 20, 30, 40}) == 100);
+  static_assert(__builtin_reduce_add((vector4int){100, 200, 300, 400}) == 1000);
+  static_assert(__builtin_reduce_add((vector4long){1000, 2000, 3000, 4000}) == 10000);
+  constexpr int reduceAddInt1 = __builtin_reduce_add((vector4int){~(1 << (sizeof(int) * 8 - 1)), 0, 0, 1});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'int'}}
+  constexpr long long reduceAddLong1 = __builtin_reduce_add((vector4long){~(1LL << (sizeof(long long) * 8 - 1)), 0, 0, 1});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'long long'}}
+  constexpr int reduceAddInt2 = __builtin_reduce_add((vector4int){(1 << (sizeof(int) * 8 - 1)), 0, 0, -1});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'int'}}
+  constexpr long long reduceAddLong2 = __builtin_reduce_add((vector4long){(1LL << (sizeof(long long) * 8 - 1)), 0, 0, -1});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'long long'}}
+  static_assert(__builtin_reduce_add((vector4uint){~0U, 0, 0, 1}) == 0);
+  static_assert(__builtin_reduce_add((vector4ulong){~0ULL, 0, 0, 1}) == 0);
+
+
+#ifdef __SIZEOF_INT128__
+  typedef __int128 v4i128 __attribute__((__vector_size__(128 * 2)));
+  constexpr __int128 reduceAddInt3 = __builtin_reduce_add((v4i128){});
+  static_assert(reduceAddInt3 == 0);
+#endif
+}
+
+namespace ReduceMul {
+  static_assert(__builtin_reduce_mul((vector4char){}) == 0);
+  static_assert(__builtin_reduce_mul((vector4char){1, 2, 3, 4}) == 24);
+  static_assert(__builtin_reduce_mul((vector4short){1, 2, 30, 40}) == 2400);
+#ifndef __AVR__
+  static_assert(__builtin_reduce_mul((vector4int){10, 20, 300, 400}) == 24'000'000);
+#endif
+  static_assert(__builtin_reduce_mul((vector4long){1000L, 2000L, 3000L, 4000L}) == 24'000'000'000'000L);
+  constexpr int reduceMulInt1 = __builtin_reduce_mul((vector4int){~(1 << (sizeof(int) * 8 - 1)), 1, 1, 2});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'int'}}
+  constexpr long long reduceMulLong1 = __builtin_reduce_mul((vector4long){~(1LL << (sizeof(long long) * 8 - 1)), 1, 1, 2});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'long long'}}
+  constexpr int reduceMulInt2 = __builtin_reduce_mul((vector4int){(1 << (sizeof(int) * 8 - 1)), 1, 1, 2});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'int'}}
+  constexpr long long reduceMulLong2 = __builtin_reduce_mul((vector4long){(1LL << (sizeof(long long) * 8 - 1)), 1, 1, 2});
+  // both-error@-1 {{must be initialized by a constant expression}} \
+  // both-note@-1 {{outside the range of representable values of type 'long long'}}
+  static_assert(__builtin_reduce_mul((vector4uint){~0U, 1, 1, 2}) ==
+#ifdef __AVR__
+      0);
+#else
+      (~0U - 1));
+#endif
+  static_assert(__builtin_reduce_mul((vector4ulong){~0ULL, 1, 1, 2}) == ~0ULL - 1);
+}
+
+namespace ReduceAnd {
+  static_assert(__builtin_reduce_and((vector4char){}) == 0);
+  static_assert(__builtin_reduce_and((vector4char){(char)0x11, (char)0x22, (char)0x44, (char)0x88}) == 0);
+  static_assert(__builtin_reduce_and((vector4short){(short)0x1111, (short)0x2222, (short)0x4444, (short)0x8888}) == 0);
+  static_assert(__builtin_reduce_and((vector4int){(int)0x11111111, (int)0x22222222, (int)0x44444444, (int)0x88888888}) == 0);
+#if __INT_WIDTH__ == 32
+  static_assert(__builtin_reduce_and((vector4long){(long long)0x1111111111111111L, (long long)0x2222222222222222L, (long long)0x4444444444444444L, (long long)0x8888888888888888L}) == 0L);
+  static_assert(__builtin_reduce_and((vector4char){(char)-1, (char)~0x22, (char)~0x44, (char)~0x88}) == 0x11);
+  static_assert(__builtin_reduce_and((vector4short){(short)~0x1111, (short)-1, (short)~0x4444, (short)~0x8888}) == 0x2222);
+  static_assert(__builtin_reduce_and((vector4int){(int)~0x11111111, (int)~0x22222222, (int)-1, (int)~0x88888888}) == 0x44444444);
+  static_assert(__builtin_reduce_and((vector4long){(long long)~0x1111111111111111L, (long long)~0x2222222222222222L, (long long)~0x4444444444444444L, (long long)-1}) == 0x8888888888888888L);
+  static_assert(__builtin_reduce_and((vector4uint){0x11111111U, 0x22222222U, 0x44444444U, 0x88888888U}) == 0U);
+  static_assert(__builtin_reduce_and((vector4ulong){0x1111111111111111UL, 0x2222222222222222UL, 0x4444444444444444UL, 0x8888888888888888UL}) == 0L);
+#endif
+}
+
+namespace ReduceOr {
+  static_assert(__builtin_reduce_or((vector4char){}) == 0);
+  static_assert(__builtin_reduce_or((vector4char){(char)0x11, (char)0x22, (char)0x44, (char)0x88}) == (char)0xFF);
+  static_assert(__builtin_reduce_or((vector4short){(short)0x1111, (short)0x2222, (short)0x4444, (short)0x8888}) == (short)0xFFFF);
+  static_assert(__builtin_reduce_or((vector4int){(int)0x11111111, (int)0x22222222, (int)0x44444444, (int)0x88888888}) == (int)0xFFFFFFFF);
+#if __INT_WIDTH__ == 32
+  static_assert(__builtin_reduce_or((vector4long){(long long)0x1111111111111111L, (long long)0x2222222222222222L, (long long)0x4444444444444444L, (long long)0x8888888888888888L}) == (long long)0xFFFFFFFFFFFFFFFFL);
+  static_assert(__builtin_reduce_or((vector4char){(char)0, (char)0x22, (char)0x44, (char)0x88}) == ~0x11);
+  static_assert(__builtin_reduce_or((vector4short){(short)0x1111, (short)0, (short)0x4444, (short)0x8888}) == ~0x2222);
+  static_assert(__builtin_reduce_or((vector4int){(int)0x11111111, (int)0x22222222, (int)0, (int)0x88888888}) == ~0x44444444);
+  static_assert(__builtin_reduce_or((vector4long){(long long)0x1111111111111111L, (long long)0x2222222222222222L, (long long)0x4444444444444444L, (long long)0}) == ~0x8888888888888888L);
+  static_assert(__builtin_reduce_or((vector4uint){0x11111111U, 0x22222222U, 0x44444444U, 0x88888888U}) == 0xFFFFFFFFU);
+  static_assert(__builtin_reduce_or((vector4ulong){0x1111111111111111UL, 0x2222222222222222UL, 0x4444444444444444UL, 0x8888888888888888UL}) == 0xFFFFFFFFFFFFFFFFL);
+#endif
+}
+
+namespace ReduceXor {
+  static_assert(__builtin_reduce_xor((vector4char){}) == 0);
+  static_assert(__builtin_reduce_xor((vector4char){(char)0x11, (char)0x22, (char)0x44, (char)0x88}) == (char)0xFF);
+  static_assert(__builtin_reduce_xor((vector4short){(short)0x1111, (short)0x2222, (short)0x4444, (short)0x8888}) == (short)0xFFFF);
+#if __INT_WIDTH__ == 32
+  static_assert(__builtin_reduce_xor((vector4int){(int)0x11111111, (int)0x22222222, (int)0x44444444, (int)0x88888888}) == (int)0xFFFFFFFF);
+  static_assert(__builtin_reduce_xor((vector4long){(long long)0x1111111111111111L, (long long)0x2222222222222222L, (long long)0x4444444444444444L, (long long)0x8888888888888888L}) == (long long)0xFFFFFFFFFFFFFFFFL);
+  static_assert(__builtin_reduce_xor((vector4uint){0x11111111U, 0x22222222U, 0x44444444U, 0x88888888U}) == 0xFFFFFFFFU);
+  static_assert(__builtin_reduce_xor((vector4ulong){0x1111111111111111UL, 0x2222222222222222UL, 0x4444444444444444UL, 0x8888888888888888UL}) == 0xFFFFFFFFFFFFFFFFUL);
+#endif
+}
+
+namespace ElementwisePopcount {
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4int){1, 2, 3, 4})) == 5);
+#if __INT_WIDTH__ == 32
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4int){0, 0xF0F0, ~0, ~0xF0F0})) == 16 * sizeof(int));
+#endif
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4long){1L, 2L, 3L, 4L})) == 5L);
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4long){0L, 0xF0F0L, ~0L, ~0xF0F0L})) == 16 * sizeof(long long));
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4uint){1U, 2U, 3U, 4U})) == 5U);
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4uint){0U, 0xF0F0U, ~0U, ~0xF0F0U})) == 16 * sizeof(int));
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4ulong){1UL, 2UL, 3UL, 4UL})) == 5UL);
+  static_assert(__builtin_reduce_add(__builtin_elementwise_popcount((vector4ulong){0ULL, 0xF0F0ULL, ~0ULL, ~0xF0F0ULL})) == 16 * sizeof(unsigned long long));
+  static_assert(__builtin_elementwise_popcount(0) == 0);
+  static_assert(__builtin_elementwise_popcount(0xF0F0) == 8);
+  static_assert(__builtin_elementwise_popcount(~0) == 8 * sizeof(int));
+  static_assert(__builtin_elementwise_popcount(0U) == 0);
+  static_assert(__builtin_elementwise_popcount(0xF0F0U) == 8);
+  static_assert(__builtin_elementwise_popcount(~0U) == 8 * sizeof(int));
+  static_assert(__builtin_elementwise_popcount(0L) == 0);
+  static_assert(__builtin_elementwise_popcount(0xF0F0L) == 8);
+  static_assert(__builtin_elementwise_popcount(~0LL) == 8 * sizeof(long long));
+
+#if __INT_WIDTH__ == 32
+  static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_popcount((vector4char){1, 2, 3, 4})) == (LITTLE_END ? 0x01020101 : 0x01010201));
+#endif
+}
+
+namespace BuiltinMemcpy {
+  constexpr int simple() {
+    int a = 12;
+    int b = 0;
+    __builtin_memcpy(&b, &a, sizeof(a));
+    return b;
+  }
+  static_assert(simple() == 12);
+
+  constexpr bool arrayMemcpy() {
+    char src[] = "abc";
+    char dst[4] = {};
+    __builtin_memcpy(dst, src, 4);
+    return dst[0] == 'a' && dst[1] == 'b' && dst[2] == 'c' && dst[3] == '\0';
+  }
+  static_assert(arrayMemcpy());
+
+  extern struct Incomplete incomplete;
+  constexpr struct Incomplete *null_incomplete = 0;
+  static_assert(__builtin_memcpy(null_incomplete, null_incomplete, sizeof(wchar_t))); // both-error {{not an integral constant expression}} \
+                                                                                      // both-note {{source of 'memcpy' is nullptr}}
+
+  wchar_t global;
+  constexpr wchar_t *null = 0;
+  static_assert(__builtin_memcpy(&global, null, sizeof(wchar_t))); // both-error {{not an integral constant expression}} \
+                                                                   // both-note {{source of 'memcpy' is nullptr}}
+
+  constexpr int simpleMove() {
+    int a = 12;
+    int b = 0;
+    __builtin_memmove(&b, &a, sizeof(a));
+    return b;
+  }
+  static_assert(simpleMove() == 12);
+
+  constexpr int memcpyTypeRem() { // both-error {{never produces a constant expression}}
+    int a = 12;
+    int b = 0;
+    __builtin_memmove(&b, &a, 1); // both-note {{'memmove' not supported: size to copy (1) is not a multiple of size of element type 'int'}} \
+                                  // both-note {{not supported}}
+    return b;
+  }
+  static_assert(memcpyTypeRem() == 12); // both-error {{not an integral constant expression}} \
+                                        // both-note {{in call to}}
+}
diff --git a/clang/test/AST/ByteCode/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
index 1803fb8ab2e9a..6a62ac11cde79 100644
--- a/clang/test/AST/ByteCode/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -263,7 +263,7 @@ namespace AnonUnionDtor {
   template <class T>
   struct opt
   {
-    union { // all20-note {{is not literal}}
+    union {
       char c;
       T data;
     };
@@ -279,7 +279,7 @@ namespace AnonUnionDtor {
   };
 
   consteval void foo() {
-    opt<A> a; // all20-error {{variable of non-literal type}}
+    opt<A> a;
   }
 
   void bar() { foo(); }
diff --git a/clang/test/AST/ByteCode/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
index 13d6c4feb3500..662823c49cd4a 100644
--- a/clang/test/AST/ByteCode/literals.cpp
+++ b/clang/test/AST/ByteCode/literals.cpp
@@ -980,6 +980,8 @@ namespace DiscardExprs {
     __uuidof(number); // both-error {{cannot call operator __uuidof on a type with no GUID}}
 
     requires{false;};
+    constexpr int *p = nullptr;
+    p - p;
 
     return 0;
   }
diff --git a/clang/test/AST/ByteCode/records.cpp b/clang/test/AST/ByteCode/records.cpp
index 2eeaafc04c516..4601aface135e 100644
--- a/clang/test/AST/ByteCode/records.cpp
+++ b/clang/test/AST/ByteCode/records.cpp
@@ -1015,11 +1015,13 @@ namespace ParenInit {
   };
 
   /// Not constexpr!
-  O o1(0);
+  O o1(0); // both-warning {{temporary whose address is used as value of}}
+  // FIXME: the secondary warning message is bogus, would be nice to suppress it.
   constinit O o2(0); // both-error {{variable does not have a constant initializer}} \
                      // both-note {{required by 'constinit' specifier}} \
                      // both-note {{reference to temporary is not a constant expression}} \
-                     // both-note {{temporary created here}}
+                     // both-note {{temporary created here}} \
+                     // both-warning {{temporary whose address is used as value}}
 
 
   /// Initializing an array.
diff --git a/clang/test/AST/ByteCode/vectors.cpp b/clang/test/AST/ByteCode/vectors.cpp
index a0aace44f3c98..a04b678a623a1 100644
--- a/clang/test/AST/ByteCode/vectors.cpp
+++ b/clang/test/AST/ByteCode/vectors.cpp
@@ -37,6 +37,7 @@ static_assert(arr4[1][0] == 0, "");
 static_assert(arr4[1][0] == 0, "");
 static_assert(arr4[1][0] == 0, "");
 
+constexpr VI4 B = __extension__(A);
 
 /// From constant-expression-cxx11.cpp
 namespace Vector {
@@ -55,10 +56,9 @@ namespace Vector {
   static_assert(__builtin_vectorelements(v2) == (32 / sizeof(double)), "");
 }
 
-/// FIXME: We need to support BitCasts between vector types.
 namespace {
   typedef float __attribute__((vector_size(16))) VI42;
-  constexpr VI42 A2 = A; // expected-error {{must be initialized by a constant expression}}
+  constexpr VI42 A2 = A;
 }
 
 namespace BoolToSignedIntegralCast{
diff --git a/clang/test/AST/HLSL/ArrayOutArgExpr.hlsl b/clang/test/AST/HLSL/ArrayOutArgExpr.hlsl
new file mode 100644
index 0000000000000..10825bf0f93bc
--- /dev/null
+++ b/clang/test/AST/HLSL/ArrayOutArgExpr.hlsl
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump %s | FileCheck %s
+
+// CHECK-LABEL: increment
+void increment(inout int Arr[2]) {
+  for (int I = 0; I < 2; I++)
+    Arr[0] += 2;
+}
+
+// CHECK-LABEL: call
+// CHECK: CallExpr 0x{{.*}} {{.*}} 'void'
+// CHECK: ImplicitCastExpr 0x{{.*}} {{.*}} 'void (*)(inout int[2])' <FunctionToPointerDecay>
+// CHECK: DeclRefExpr 0x{{.*}} {{.*}} 'void (inout int[2])' lvalue Function 0x{{.*}} 'increment' 'void (inout int[2])'
+// CHECK: HLSLOutArgExpr 0x{{.*}} {{.*}} 'int[2]' lvalue inout
+// CHECK: OpaqueValueExpr [[A:0x.*]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B:0x.*]] {{.*}} 'int[2]' lvalue Var [[E:0x.*]] 'A' 'int[2]'
+// CHECK: OpaqueValueExpr [[C:0x.*]] {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr [[D:0x.*]] {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B]] {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+// CHECK: BinaryOperator 0x{{.*}} {{.*}} 'int[2]' lvalue '='
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr 0x{{.*}} {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+// CHECK: ImplicitCastExpr 0x{{.*}} {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[C]] {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr [[D]] {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B]] {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+export int call() {
+  int A[2] = { 0, 1 };
+  increment(A);
+  return A[0];
+}
+
+// CHECK-LABEL: fn2
+void fn2(out int Arr[2]) {
+  Arr[0] += 5;
+  Arr[1] += 6;
+}
+
+// CHECK-LABEL: call2
+// CHECK: CallExpr 0x{{.*}} {{.*}} 'void'
+// CHECK: ImplicitCastExpr 0x{{.*}} {{.*}} 'void (*)(out int[2])' <FunctionToPointerDecay>
+// CHECK: DeclRefExpr 0x{{.*}} {{.*}} 'void (out int[2])' lvalue Function 0x{{.*}} 'fn2' 'void (out int[2])'
+// CHECK: HLSLOutArgExpr 0x{{.*}} {{.*}} 'int[2]' lvalue out
+// CHECK: OpaqueValueExpr [[A:0x.*]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B:0x.*]] {{.*}} 'int[2]' lvalue Var [[E:0x.*]] 'A' 'int[2]'
+// CHECK: OpaqueValueExpr [[C:0x.*]] {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr [[D:0x.*]] {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B]] {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+// CHECK: BinaryOperator 0x{{.*}} {{.*}} 'int[2]' lvalue '='
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B]] {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+// CHECK: ImplicitCastExpr 0x{{.*}} {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[C]] {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr [[D]] {{.*}} 'int[2]' <HLSLArrayRValue>
+// CHECK: OpaqueValueExpr [[A]] {{.*}} 'int[2]' lvalue
+// CHECK: DeclRefExpr [[B]] {{.*}} 'int[2]' lvalue Var [[E]] 'A' 'int[2]'
+export int call2() {
+  int A[2] = { 0, 1 };
+  fn2(A);
+  return 1;
+}
diff --git a/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
new file mode 100644
index 0000000000000..909cce17e344c
--- /dev/null
+++ b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=ByteAddressBuffer %s | FileCheck -DRESOURCE=ByteAddressBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=ByteAddressBuffer %s | FileCheck -DRESOURCE=ByteAddressBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-SRV,CHECK-NOSUBSCRIPT %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=RWByteAddressBuffer %s | FileCheck -DRESOURCE=RWByteAddressBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=RWByteAddressBuffer %s | FileCheck -DRESOURCE=RWByteAddressBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN:  -DRESOURCE=RasterizerOrderedByteAddressBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedByteAddressBuffer \
+// RUN:  -check-prefix=EMPTY %s
+//
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN:   -DRESOURCE=RasterizerOrderedByteAddressBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedByteAddressBuffer \
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
+
+// EMPTY: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit <undeserialized declarations> class [[RESOURCE]]
+// EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+
+// There should be no more occurrences of RESOURCE
+// EMPTY-NOT: {{[^[:alnum:]]}}[[RESOURCE]]
+
+#ifndef EMPTY
+
+RESOURCE Buffer;
+
+#endif
+
+// CHECK: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced <undeserialized declarations> class [[RESOURCE]] definition
+
+
+// CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit __handle '__hlsl_resource_t
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(char8_t)]]
+// CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
+
+// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
+// CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
diff --git a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
index 0b74b7af3fd66..6cb4379ef5f55 100644
--- a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
@@ -20,7 +20,7 @@
 //
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-APPEND %s
 //
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
@@ -28,7 +28,7 @@
 //
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
-// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
+// RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-CONSUME %s
 //
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
@@ -75,19 +75,35 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit RawBuffer
 
 // CHECK-SUBSCRIPT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
-// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
 // CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
 // CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const [[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
 // CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
 
 // CHECK-SUBSCRIPT-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
-// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-SUBSCRIPT-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
 // CHECK-SUBSCRIPT-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-SUBSCRIPT-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-SUBSCRIPT-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-SUBSCRIPT-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-SUBSCRIPT-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::resource_class(
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SUBSCRIPT-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
 // CHECK-SUBSCRIPT-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
 // CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
 
 // CHECK-NOSUBSCRIPT-NOT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
@@ -97,7 +113,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int (...) noexcept' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
 // CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
@@ -110,7 +126,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-COUNTER-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-COUNTER-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
-// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int (...) noexcept' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-COUNTER-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
 // CHECK-COUNTER-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
@@ -119,6 +135,48 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
 // CHECK-COUNTER-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
 
+// CHECK-APPEND: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Append 'void (element_type)'
+// CHECK-APPEND-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> value 'element_type'
+// CHECK-APPEND-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-APPEND-NEXT: BinaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' '='
+// CHECK-APPEND-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-APPEND-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-APPEND-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' 1
+// CHECK-APPEND-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' ParmVar 0x{{[0-9A-Fa-f]+}} 'value' 'element_type'
+
+// CHECK-CONSUME: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Consume 'element_type ()'
+// CHECK-CONSUME-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-CONSUME-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
+// CHECK-CONSUME-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int'
+// CHECK-CONSUME-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
+// CHECK-CONSUME-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
+// CHECK-CONSUME-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: IntegerLiteral 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'int' -1
+
 // CHECK: ClassTemplateSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class [[RESOURCE]] definition
 
 // CHECK: TemplateArgument type 'float'
diff --git a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
index b672e40e63a63..6bab39de5a233 100644
--- a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
@@ -58,19 +58,33 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
 
 // CHECK: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'const element_type &(unsigned int) const'
-// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const [[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
 
 // CHECK-NEXT: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &(unsigned int)'
-// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Idx 'unsigned int'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> Index 'unsigned int'
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue .e 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: UnaryOperator 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' prefix '*' cannot overflow
+// CHECK-NEXT: CallExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type *'
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '<builtin fn type>' Function 0x{{[0-9A-Fa-f]+}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
+// CHECK-SAME: ' lvalue .__handle 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
 
 // CHECK: ClassTemplateSpecializationDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> class [[RESOURCE]] definition
diff --git a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
index 04cff07376885..1b385e0fc3331 100644
--- a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
+++ b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
@@ -9,7 +9,7 @@ int main() {
   S s;
   int x = s.f();
   // CHECK: CallExpr 0x{{[^ ]*}} <col:11, col:15> 'int
-  // CHECK-NEXT: |-ImplicitCastExpr 0x{{[^ ]*}} <col:11> 'int (*)(S &)' <FunctionToPointerDecay>
-  // CHECK-NEXT: | `-DeclRefExpr 0x{{[^ ]*}} <col:11> 'int (S &)' lvalue CXXMethod 0x{{[^ ]*}} 'f' 'int (S &)'
+  // CHECK-NEXT: |-ImplicitCastExpr 0x{{[^ ]*}} <col:13> 'int (*)(S &)' <FunctionToPointerDecay>
+  // CHECK-NEXT: | `-DeclRefExpr 0x{{[^ ]*}} <col:13> 'int (S &)' lvalue CXXMethod 0x{{[^ ]*}} 'f' 'int (S &)'
 }
 }
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index 4388462224026..b59fa3778192f 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -480,3 +480,37 @@ void RecoveryForStmtCond() {
   // CHECK-NEXT:    `-CompoundStmt {{.*}}
   for (int i = 0; i < invalid; ++i) {}
 }
+
+// Fix crash issue https://github.com/llvm/llvm-project/issues/112560.
+// Make sure clang compiles the following code without crashing:
+
+// CHECK:NamespaceDecl {{.*}} GH112560
+// CHECK-NEXT:  |-CXXRecordDecl {{.*}} referenced union U definition
+// CHECK-NEXT:  | |-DefinitionData {{.*}}
+// CHECK-NEXT:  | | |-DefaultConstructor {{.*}}
+// CHECK-NEXT:  | | |-CopyConstructor {{.*}}
+// CHECK-NEXT:  | | |-MoveConstructor {{.*}}
+// CHECK-NEXT:  | | |-CopyAssignment {{.*}}
+// CHECK-NEXT:  | | |-MoveAssignment {{.*}}
+// CHECK-NEXT:  | | `-Destructor {{.*}}
+// CHECK-NEXT:  | |-CXXRecordDecl {{.*}} implicit union U
+// CHECK-NEXT:  | `-FieldDecl {{.*}} invalid f 'int'
+// CHECK-NEXT:  |   `-RecoveryExpr {{.*}} 'int' contains-errors
+// DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
+namespace GH112560 {
+union U {
+  int f = ;
+};
+
+// CHECK: FunctionDecl {{.*}} foo 'void ()'
+// CHECK-NEXT:    `-CompoundStmt {{.*}}
+// CHECK-NEXT:      `-DeclStmt {{.*}}
+// CHECK-NEXT:        `-VarDecl {{.*}} g 'U':'GH112560::U' listinit
+// CHECK-NEXT:          `-InitListExpr {{.*}} 'U':'GH112560::U' contains-errors field Field {{.*}} 'f' 'int'
+// CHECK-NEXT:            `-CXXDefaultInitExpr {{.*}} 'int' contains-errors has rewritten init
+// CHECK-NEXT:              `-RecoveryExpr {{.*}} 'int' contains-errors
+// DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
+void foo() {
+  U g{};
+}
+} // namespace GH112560
diff --git a/clang/test/AST/ast-dump-types-json.cpp b/clang/test/AST/ast-dump-types-json.cpp
index c1bb9266fa869..fa1fb53df0e85 100644
--- a/clang/test/AST/ast-dump-types-json.cpp
+++ b/clang/test/AST/ast-dump-types-json.cpp
@@ -248,15 +248,24 @@ using ::TestUsingShadowDeclType;
 // CHECK-NEXT:    "inner": [
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
-// CHECK-NEXT:      "kind": "RecordType",
+// CHECK-NEXT:      "kind": "ElaboratedType",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "T"
 // CHECK-NEXT:      },
-// CHECK-NEXT:      "decl": {
-// CHECK-NEXT:       "id": "0x{{.*}}",
-// CHECK-NEXT:       "kind": "CXXRecordDecl",
-// CHECK-NEXT:       "name": "T"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "RecordType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "T"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "decl": {
+// CHECK-NEXT:         "id": "0x{{.*}}",
+// CHECK-NEXT:         "kind": "CXXRecordDecl",
+// CHECK-NEXT:         "name": "T"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
@@ -325,15 +334,24 @@ using ::TestUsingShadowDeclType;
 // CHECK-NEXT:    "inner": [
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
-// CHECK-NEXT:      "kind": "RecordType",
+// CHECK-NEXT:      "kind": "ElaboratedType",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "T"
 // CHECK-NEXT:      },
-// CHECK-NEXT:      "decl": {
-// CHECK-NEXT:       "id": "0x{{.*}}",
-// CHECK-NEXT:       "kind": "CXXRecordDecl",
-// CHECK-NEXT:       "name": "T"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "RecordType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "T"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "decl": {
+// CHECK-NEXT:         "id": "0x{{.*}}",
+// CHECK-NEXT:         "kind": "CXXRecordDecl",
+// CHECK-NEXT:         "name": "T"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
diff --git a/clang/test/AST/ast-print-openacc-combined-construct.cpp b/clang/test/AST/ast-print-openacc-combined-construct.cpp
index e04c39ac9bc5b..25fa29cbbe04e 100644
--- a/clang/test/AST/ast-print-openacc-combined-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-combined-construct.cpp
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-deprecated-clause-alias -ast-print %s -o - | FileCheck %s
 
+constexpr int get_value() { return 1; }
 void foo() {
   int *iPtr;
 // CHECK: #pragma acc parallel loop
@@ -131,4 +132,285 @@ void foo() {
 // CHECK: #pragma acc kernels loop present(i, array[1], array, array[1:2])
 #pragma acc kernels loop present(i, array[1], array, array[1:2])
   for(int i = 0;i<5;++i);
+
+  float *arrayPtr[5];
+
+  // CHECK: #pragma acc kernels loop deviceptr(iPtr, arrayPtr[0])
+#pragma acc kernels loop deviceptr(iPtr, arrayPtr[0])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop wait()
+#pragma acc parallel loop wait()
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop wait(*iPtr, i)
+#pragma acc parallel loop wait(*iPtr, i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop wait(queues: *iPtr, i)
+#pragma acc parallel loop wait(queues:*iPtr, i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop wait(devnum: i : *iPtr, i)
+#pragma acc parallel loop wait(devnum:i:*iPtr, i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop wait(devnum: i : queues: *iPtr, i)
+#pragma acc parallel loop wait(devnum:i:queues:*iPtr, i)
+  for(int i = 0;i<5;++i);
+
+  // CHECK: #pragma acc serial loop attach(iPtr, arrayPtr[0])
+#pragma acc serial loop attach(iPtr, arrayPtr[0])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop no_create(i, array[1], array, array[1:2])
+#pragma acc parallel loop no_create(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop no_create(i, array[1], array, array[1:2]) present(i, array[1], array, array[1:2])
+#pragma acc parallel loop no_create(i, array[1], array, array[1:2]) present(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop copy(i, array[1], array, array[1:2]) pcopy(i, array[1], array, array[1:2]) present_or_copy(i, array[1], array, array[1:2])
+#pragma acc parallel loop copy(i, array[1], array, array[1:2]) pcopy(i, array[1], array, array[1:2]) present_or_copy(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop copyin(i, array[1], array, array[1:2]) pcopyin(readonly: i, array[1], array, array[1:2]) present_or_copyin(i, array[1], array, array[1:2])
+#pragma acc parallel loop copyin(i, array[1], array, array[1:2]) pcopyin(readonly:i, array[1], array, array[1:2]) present_or_copyin(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(i, array[1], array, array[1:2])
+#pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop create(i, array[1], array, array[1:2]) pcreate(zero: i, array[1], array, array[1:2]) present_or_create(i, array[1], array, array[1:2])
+#pragma acc parallel loop create(i, array[1], array, array[1:2]) pcreate(zero: i, array[1], array, array[1:2]) present_or_create(i, array[1], array, array[1:2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop collapse(1)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop collapse(1)
+  for(int i = 0;i<5;++i);
+// CHECK: #pragma acc serial loop collapse(force:1)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop collapse(force:1)
+  for(int i = 0;i<5;++i);
+// CHECK: #pragma acc kernels loop collapse(2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop collapse(2)
+  for(int i = 0;i<5;++i)
+    for(int i = 0;i<5;++i);
+// CHECK: #pragma acc parallel loop collapse(force:2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop collapse(force:2)
+  for(int i = 0;i<5;++i)
+    for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc serial loop tile(1, 3, *, get_value())
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop tile(1, 3, *, get_value())
+  for(int i = 0;i<5;++i)
+    for(int i = 0;i<5;++i)
+      for(int i = 0;i<5;++i)
+        for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop num_gangs(i, (int)array[2])
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop num_gangs(i, (int)array[2])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop num_workers(i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop num_workers(i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop vector_length((int)array[1])
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop vector_length((int)array[1])
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc serial loop gang(static: i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop gang(static:i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(static: i) gang(dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(static:i) gang(dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(static: i, dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(static:i, dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(static: i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(static:i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(static: i) gang(dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(static:i) gang(dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop gang(static: i, dim: 2)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop gang(static:i, dim:2)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc kernels loop gang(num: i) gang(static: i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop gang(i) gang(static:i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc kernels loop gang(num: i) gang(static: i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop gang(num:i) gang(static:i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc serial loop gang(static: i)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop gang(static:i)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc serial loop gang(static: *)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop gang(static:*)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop worker
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop worker
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc parallel loop worker
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop worker
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc serial loop worker
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop worker
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc kernels loop worker(num: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop worker(5)
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc kernels loop worker(num: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop worker(num:5)
+  for(int i = 0;i<5;++i);
+
+  // CHECK: #pragma acc parallel loop vector
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop vector
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop vector(length: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop vector(5)
+  for(int i = 0;i<5;++i);
+
+// CHECK: #pragma acc parallel loop vector(length: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc parallel loop vector(length:5)
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc kernels loop vector
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop vector
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc kernels loop vector(length: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop vector(5)
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc kernels loop vector(length: 5)
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc kernels loop vector(length:5)
+  for(int i = 0;i<5;++i);
+
+// CHECK-NEXT: #pragma acc serial loop vector
+// CHECK-NEXT: for (int i = 0; i < 5; ++i)
+// CHECK-NEXT: ;
+#pragma acc serial loop vector
+  for(int i = 0;i<5;++i);
+
+//CHECK: #pragma acc parallel loop reduction(+: iPtr)
+#pragma acc parallel loop reduction(+: iPtr)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc serial loop reduction(*: i)
+#pragma acc serial loop reduction(*: i)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc kernels loop reduction(max: SomeB)
+#pragma acc kernels loop reduction(max: SomeB)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc parallel loop reduction(min: iPtr)
+#pragma acc parallel loop reduction(min: iPtr)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc serial loop reduction(&: i)
+#pragma acc serial loop reduction(&: i)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc kernels loop reduction(|: SomeB)
+#pragma acc kernels loop reduction(|: SomeB)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc parallel loop reduction(^: iPtr)
+#pragma acc parallel loop reduction(^: iPtr)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc serial loop reduction(&&: i)
+#pragma acc serial loop reduction(&&: i)
+  for(int i = 0;i<5;++i);
+//CHECK: #pragma acc kernels loop reduction(||: SomeB)
+#pragma acc kernels loop reduction(||: SomeB)
+  for(int i = 0;i<5;++i);
 }
diff --git a/clang/test/AST/ast-print-packindexingexpr.cpp b/clang/test/AST/ast-print-packindexingexpr.cpp
new file mode 100644
index 0000000000000..157abeb99436a
--- /dev/null
+++ b/clang/test/AST/ast-print-packindexingexpr.cpp
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -ast-print -std=c++2c %s | FileCheck %s
+
+template <class... T, unsigned N>
+auto foo(T ...params) {
+  return params...[N];
+}
+
+// CHECK: template <class ...T, unsigned int N> auto foo(T ...params) {
+// CHECK-NEXT: return params...[N];
diff --git a/clang/test/AST/attr-lifetime-capture-by.cpp b/clang/test/AST/attr-lifetime-capture-by.cpp
index c3afe267301ad..71b348dac764b 100644
--- a/clang/test/AST/attr-lifetime-capture-by.cpp
+++ b/clang/test/AST/attr-lifetime-capture-by.cpp
@@ -30,6 +30,12 @@ struct vector {
 
   void insert(iterator, T&&);
 };
+
+template <typename Key, typename Value>
+struct map {
+  Value& operator[](Key&& p);
+  Value& operator[](const Key& p);
+};
 } // namespace std
 
 // CHECK-NOT:   LifetimeCaptureByAttr
@@ -37,67 +43,53 @@ struct vector {
 struct [[gsl::Pointer()]] View {};
 std::vector<View> views;
 // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct vector definition implicit_instantiation
-// CHECK:       TemplateArgument type 'View'
-// CHECK-NOT:   LifetimeCaptureByAttr
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (const View &)'
-// CHECK:           ParmVarDecl {{.*}} 'const View &'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'const View &'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (View &&)'
-// CHECK:           ParmVarDecl {{.*}} 'View &&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'View &&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 // CHECK:       CXXMethodDecl {{.*}} insert 'void (iterator, View &&)'
-// CHECK:           ParmVarDecl {{.*}} 'iterator'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK:           ParmVarDecl {{.*}} 'View &&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'iterator'
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'View &&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 template <class T> struct [[gsl::Pointer()]] ViewTemplate {};
 std::vector<ViewTemplate<int>> templated_views;
-// CHECK:   ClassTemplateSpecializationDecl {{.*}} struct vector definition implicit_instantiation
-// CHECK:       TemplateArgument type 'ViewTemplate<int>'
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK:       ClassTemplateSpecializationDecl {{.*}} struct vector definition implicit_instantiation
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (const ViewTemplate<int> &)'
-// CHECK:           ParmVarDecl {{.*}} 'const ViewTemplate<int> &'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'const ViewTemplate<int> &'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 // CHECK-NOT:   LifetimeCaptureByAttr
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (ViewTemplate<int> &&)'
-// CHECK:           ParmVarDecl {{.*}} 'ViewTemplate<int> &&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'ViewTemplate<int> &&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 // CHECK:       CXXMethodDecl {{.*}} insert 'void (iterator, ViewTemplate<int> &&)'
-// CHECK:           ParmVarDecl {{.*}} 'iterator'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK:           ParmVarDecl {{.*}} 'ViewTemplate<int> &&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'iterator'
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'ViewTemplate<int> &&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 std::vector<int*> pointers;
 // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct vector definition implicit_instantiation
-// CHECK:       TemplateArgument type 'int *'
-// CHECK-NOT:   LifetimeCaptureByAttr
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (int *const &)'
-// CHECK:           ParmVarDecl {{.*}} 'int *const &'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'int *const &'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 // CHECK:       CXXMethodDecl {{.*}} push_back 'void (int *&&)'
-// CHECK:           ParmVarDecl {{.*}} 'int *&&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'int *&&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 // CHECK:       CXXMethodDecl {{.*}} insert 'void (iterator, int *&&)'
-// CHECK:           ParmVarDecl {{.*}} 'iterator'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK:           ParmVarDecl {{.*}} 'int *&&'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
-// CHECK-NOT:   LifetimeCaptureByAttr
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'iterator'
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'int *&&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
 
 std::vector<int> ints;
 // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct vector definition implicit_instantiation
@@ -110,6 +102,16 @@ std::vector<int> ints;
 // CHECK-NOT:   LifetimeCaptureByAttr
 
 // CHECK:       CXXMethodDecl {{.*}} insert 'void (iterator, int &&)'
-// CHECK:           ParmVarDecl {{.*}} 'iterator'
-// CHECK:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'iterator'
+// CHECK-NEXT:           ParmVarDecl {{.*}} 'int &&'
 // CHECK-NOT:   LifetimeCaptureByAttr
+
+std::map<View, int> map;
+// CHECK:   ClassTemplateSpecializationDecl {{.*}} struct map definition implicit_instantiation
+
+// CHECK:       CXXMethodDecl {{.*}} operator[] 'int &(View &&)' implicit_instantiation
+// CHECK-NEXT:           ParmVarDecl {{.*}} p 'View &&'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
+// CHECK:       CXXMethodDecl {{.*}} operator[] 'int &(const View &)' implicit_instantiation
+// CHECK-NEXT:           ParmVarDecl {{.*}} p 'const View &'
+// CHECK-NEXT:               LifetimeCaptureByAttr {{.*}} Implicit
diff --git a/clang/test/AST/ms-constexpr-new.cpp b/clang/test/AST/ms-constexpr-new.cpp
new file mode 100644
index 0000000000000..4b534cf020764
--- /dev/null
+++ b/clang/test/AST/ms-constexpr-new.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -ast-dump %s | FileCheck %s
+
+// CHECK: used operator new
+// CHECK: MSConstexprAttr 0x{{[0-9a-f]+}} <col:17, col:23>
+[[nodiscard]] [[msvc::constexpr]] inline void* __cdecl operator new(decltype(sizeof(void*)), void* p) noexcept { return p; }
+
+// CHECK: used constexpr construct_at
+// CHECK: AttributedStmt 0x{{[0-9a-f]+}} <col:46, col:88>
+// CHECK-NEXT: MSConstexprAttr 0x{{[0-9a-f]+}} <col:48, col:54>
+// CHECK-NEXT: ReturnStmt 0x{{[0-9a-f]+}} <col:66, col:88>
+constexpr int* construct_at(int* p, int v) { [[msvc::constexpr]] return ::new (p) int(v); }
+constexpr bool check_construct_at() { int x; return *construct_at(&x, 42) == 42; }
+static_assert(check_construct_at());
diff --git a/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.cpp b/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.cpp
new file mode 100644
index 0000000000000..62c945c7a2c24
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.cpp
@@ -0,0 +1,266 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.MemoryUnsafeCastChecker -verify %s
+
+class Base { };
+class Derived : public Base { };
+
+template<typename Target, typename Source>
+Target& downcast_ref(Source& source){
+  [[clang::suppress]]
+  return static_cast<Target&>(source);
+}
+
+template<typename Target, typename Source>
+Target* downcast_ptr(Source* source){
+  [[clang::suppress]]
+  return static_cast<Target*>(source);
+}
+
+void test_pointers(Base *base) {
+  Derived *derived_static = static_cast<Derived*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived *derived_reinterpret = reinterpret_cast<Derived*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived *derived_c = (Derived*)base;
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived *derived_d = downcast_ptr<Derived, Base>(base);  // no warning
+}
+
+void test_non_pointers(Derived derived) {
+  Base base_static = static_cast<Base>(derived);  // no warning
+}
+
+void test_refs(Base &base) {
+  Derived &derived_static = static_cast<Derived&>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_reinterpret = reinterpret_cast<Derived&>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_c = (Derived&)base;
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_d = downcast_ref<Derived, Base>(base);  // no warning
+}
+
+class BaseVirtual {
+  virtual void virtual_base_function();
+};
+
+class DerivedVirtual : public BaseVirtual {
+  void virtual_base_function() override { }
+};
+
+void test_dynamic_casts(BaseVirtual *base_ptr, BaseVirtual &base_ref) {
+  DerivedVirtual *derived_dynamic_ptr = dynamic_cast<DerivedVirtual*>(base_ptr);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseVirtual' to derived type 'DerivedVirtual'}}
+  DerivedVirtual &derived_dynamic_ref = dynamic_cast<DerivedVirtual&>(base_ref);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseVirtual' to derived type 'DerivedVirtual'}}
+}
+
+struct BaseStruct { };
+struct DerivedStruct : BaseStruct { };
+
+void test_struct_pointers(struct BaseStruct *base_struct) {
+  struct DerivedStruct *derived_static = static_cast<struct DerivedStruct*>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+  struct DerivedStruct *derived_reinterpret = reinterpret_cast<struct DerivedStruct*>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+  struct DerivedStruct *derived_c = (struct DerivedStruct*)base_struct;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+}
+
+typedef struct BaseStruct BStruct;
+typedef struct DerivedStruct DStruct;
+
+void test_struct_refs(BStruct &base_struct) {
+  DStruct &derived_static = static_cast<DStruct&>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+  DStruct &derived_reinterpret = reinterpret_cast<DStruct&>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+  DStruct &derived_c = (DStruct&)base_struct;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+}
+
+int counter = 0;
+void test_recursive(BStruct &base_struct) {
+  if (counter == 5)
+    return;
+  counter++;
+  DStruct &derived_static = static_cast<DStruct&>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStruct' to derived type 'DerivedStruct'}}
+}
+
+template<typename T>
+class BaseTemplate { };
+
+template<typename T>
+class DerivedTemplate : public BaseTemplate<T> { };
+
+void test_templates(BaseTemplate<int> *base, BaseTemplate<int> &base_ref) {
+  DerivedTemplate<int> *derived_static = static_cast<DerivedTemplate<int>*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+  DerivedTemplate<int> *derived_reinterpret = reinterpret_cast<DerivedTemplate<int>*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+  DerivedTemplate<int> *derived_c = (DerivedTemplate<int>*)base;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+  DerivedTemplate<int> &derived_static_ref = static_cast<DerivedTemplate<int>&>(base_ref);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+  DerivedTemplate<int> &derived_reinterpret_ref = reinterpret_cast<DerivedTemplate<int>&>(base_ref);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+  DerivedTemplate<int> &derived_c_ref = (DerivedTemplate<int>&)base_ref;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseTemplate' to derived type 'DerivedTemplate'}}
+}
+
+#define CAST_MACRO_STATIC(X,Y) (static_cast<Y>(X))
+#define CAST_MACRO_REINTERPRET(X,Y) (reinterpret_cast<Y>(X))
+#define CAST_MACRO_C(X,Y) ((Y)X)
+
+void test_macro_static(Base *base, Derived *derived, Base &base_ref) {
+  Derived *derived_static = CAST_MACRO_STATIC(base, Derived*);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_static_ref = CAST_MACRO_STATIC(base_ref, Derived&);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Base *base_static_same = CAST_MACRO_STATIC(base, Base*);  // no warning
+  Base *base_static_upcast = CAST_MACRO_STATIC(derived, Base*);  // no warning
+}
+
+void test_macro_reinterpret(Base *base, Derived *derived, Base &base_ref) {
+  Derived *derived_reinterpret = CAST_MACRO_REINTERPRET(base, Derived*);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_reinterpret_ref = CAST_MACRO_REINTERPRET(base_ref, Derived&);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Base *base_reinterpret_same = CAST_MACRO_REINTERPRET(base, Base*);  // no warning
+  Base *base_reinterpret_upcast = CAST_MACRO_REINTERPRET(derived, Base*);  // no warning
+}
+
+void test_macro_c(Base *base, Derived *derived, Base &base_ref) {
+  Derived *derived_c = CAST_MACRO_C(base, Derived*);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Derived &derived_c_ref = CAST_MACRO_C(base_ref, Derived&);
+  // expected-warning@-1{{Unsafe cast from base type 'Base' to derived type 'Derived'}}
+  Base *base_c_same = CAST_MACRO_C(base, Base*);  // no warning
+  Base *base_c_upcast = CAST_MACRO_C(derived, Base*);  // no warning
+}
+
+struct BaseStructCpp {
+  int t;
+  void increment() { t++; }
+};
+struct DerivedStructCpp : BaseStructCpp {
+  void increment_t() {increment();}
+};
+
+void test_struct_cpp_pointers(struct BaseStructCpp *base_struct) {
+  struct DerivedStructCpp *derived_static = static_cast<struct DerivedStructCpp*>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+  struct DerivedStructCpp *derived_reinterpret = reinterpret_cast<struct DerivedStructCpp*>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+  struct DerivedStructCpp *derived_c = (struct DerivedStructCpp*)base_struct;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+}
+
+typedef struct BaseStructCpp BStructCpp;
+typedef struct DerivedStructCpp DStructCpp;
+
+void test_struct_cpp_refs(BStructCpp &base_struct, DStructCpp &derived_struct) {
+  DStructCpp &derived_static = static_cast<DStructCpp&>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+  DStructCpp &derived_reinterpret = reinterpret_cast<DStructCpp&>(base_struct);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+  DStructCpp &derived_c = (DStructCpp&)base_struct;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseStructCpp' to derived type 'DerivedStructCpp'}}
+  BStructCpp &base = (BStructCpp&)derived_struct; // no warning
+  BStructCpp &base_static = static_cast<BStructCpp&>(derived_struct); // no warning
+  BStructCpp &base_reinterpret = reinterpret_cast<BStructCpp&>(derived_struct); // no warning
+}
+
+struct stack_st { };
+
+#define STACK_OF(type) struct stack_st_##type
+
+void test_stack(stack_st *base) {
+  STACK_OF(void) *derived = (STACK_OF(void)*)base;
+  // expected-warning@-1{{Unsafe cast from type 'stack_st' to an unrelated type 'stack_st_void'}}
+}
+
+class Parent { };
+class Child1 : public Parent { };
+class Child2 : public Parent { };
+
+void test_common_parent(Child1 *c1, Child2 *c2) {
+  Child2 *c2_cstyle = (Child2 *)c1;
+  // expected-warning@-1{{Unsafe cast from type 'Child1' to an unrelated type 'Child2'}}
+  Child2 *c2_reinterpret = reinterpret_cast<Child2 *>(c1);
+  // expected-warning@-1{{Unsafe cast from type 'Child1' to an unrelated type 'Child2'}}
+}
+
+class Type1 { };
+class Type2 { };
+
+void test_unrelated_ref(Type1 &t1, Type2 &t2) {
+  Type2 &t2_cstyle = (Type2 &)t1;
+  // expected-warning@-1{{Unsafe cast from type 'Type1' to an unrelated type 'Type2'}}
+  Type2 &t2_reinterpret = reinterpret_cast<Type2 &>(t1);
+  // expected-warning@-1{{Unsafe cast from type 'Type1' to an unrelated type 'Type2'}}
+  Type2 &t2_same = reinterpret_cast<Type2 &>(t2); // no warning
+}
+
+
+class VirtualClass1 {
+  virtual void virtual_base_function();
+};
+
+class VirtualClass2 {
+  void virtual_base_function();
+};
+
+void test_unrelated_virtual(VirtualClass1 &v1) {
+  VirtualClass2 &v2 = dynamic_cast<VirtualClass2 &>(v1);
+  // expected-warning@-1{{Unsafe cast from type 'VirtualClass1' to an unrelated type 'VirtualClass2'}}
+}
+
+struct StructA { };
+struct StructB { };
+
+typedef struct StructA StA;
+typedef struct StructB StB;
+
+void test_struct_unrelated_refs(StA &a, StB &b) {
+  StB &b_reinterpret = reinterpret_cast<StB&>(a);
+  // expected-warning@-1{{Unsafe cast from type 'StructA' to an unrelated type 'StructB'}}
+  StB &b_c = (StB&)a;
+  // expected-warning@-1{{Unsafe cast from type 'StructA' to an unrelated type 'StructB'}}
+  StA &a_local = (StA&)b;
+  // expected-warning@-1{{Unsafe cast from type 'StructB' to an unrelated type 'StructA'}}
+  StA &a_reinterpret = reinterpret_cast<StA&>(b);
+  // expected-warning@-1{{Unsafe cast from type 'StructB' to an unrelated type 'StructA'}}
+  StA &a_same = (StA&)a; // no warning
+}
+
+template<typename T>
+class DeferrableRefCounted {
+public:
+  void deref() const {
+    auto this_to_T =  static_cast<const T*>(this); // no warning
+  }
+};
+
+class SomeArrayClass : public DeferrableRefCounted<SomeArrayClass> { };
+
+void test_this_to_template(SomeArrayClass *ptr) {
+  ptr->deref();
+};
+
+template<typename WeakPtrFactoryType>
+class CanMakeWeakPtrBase {
+public:
+  void initializeWeakPtrFactory() const {
+    auto &this_to_T = static_cast<const WeakPtrFactoryType&>(*this);
+  }
+};
+
+template<typename T>
+using CanMakeWeakPtr = CanMakeWeakPtrBase<T>;
+
+class EventLoop : public CanMakeWeakPtr<EventLoop> { };
+
+void test_this_to_template_ref(EventLoop *ptr) {
+  ptr->initializeWeakPtrFactory();
+};
diff --git a/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.mm b/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.mm
new file mode 100644
index 0000000000000..f9046d7981784
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/memory-unsafe-cast.mm
@@ -0,0 +1,64 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.MemoryUnsafeCastChecker -verify %s
+
+@protocol NSObject
++alloc;
+-init;
+@end
+
+@interface NSObject <NSObject> {}
+@end
+
+@interface BaseClass : NSObject
+@end
+
+@interface DerivedClass : BaseClass
+-(void)testCasts:(BaseClass*)base;
+@end
+
+@implementation DerivedClass
+-(void)testCasts:(BaseClass*)base {
+  DerivedClass *derived = (DerivedClass*)base;
+  // expected-warning@-1{{Unsafe cast from base type 'BaseClass' to derived type 'DerivedClass'}}
+  DerivedClass *derived_static = static_cast<DerivedClass*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseClass' to derived type 'DerivedClass'}}
+  DerivedClass *derived_reinterpret = reinterpret_cast<DerivedClass*>(base);
+  // expected-warning@-1{{Unsafe cast from base type 'BaseClass' to derived type 'DerivedClass'}}
+  base = (BaseClass*)derived;  // no warning
+  base = (BaseClass*)base;  // no warning
+}
+@end
+
+template <typename T>
+class WrappedObject
+{
+public:
+  T get() const { return mMetalObject; }
+  T mMetalObject = nullptr;
+};
+
+@protocol MTLCommandEncoder
+@end
+@protocol MTLRenderCommandEncoder
+@end
+class CommandEncoder : public WrappedObject<id<MTLCommandEncoder>> { };
+
+class RenderCommandEncoder final : public CommandEncoder
+{
+private:
+    id<MTLRenderCommandEncoder> get()
+    {
+        return static_cast<id<MTLRenderCommandEncoder>>(CommandEncoder::get());
+    }
+};
+
+@interface Class1
+@end
+
+@interface Class2
+@end
+
+void testUnrelated(Class1 *c1) {
+  Class2 *c2 = (Class2*)c1;
+  // expected-warning@-1{{Unsafe cast from type 'Class1' to an unrelated type 'Class2'}}
+  Class1 *c1_same = reinterpret_cast<Class1*>(c1); // no warning
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index d7fb689557a6f..52854cd10f68c 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -456,3 +456,12 @@ int TreeNode::recursiveWeight() {
 }
 
 } // namespace local_var_in_recursive_function
+
+namespace local_var_for_singleton {
+  RefCountable *singleton();
+  RefCountable *otherSingleton();
+  void foo() {
+    RefCountable* bar = singleton();
+    RefCountable* baz = otherSingleton();
+  }
+}
\ No newline at end of file
diff --git a/clang/test/Analysis/analyzer-enabled-checkers.c b/clang/test/Analysis/analyzer-enabled-checkers.c
index e605c62a66ad0..160e35c77462d 100644
--- a/clang/test/Analysis/analyzer-enabled-checkers.c
+++ b/clang/test/Analysis/analyzer-enabled-checkers.c
@@ -43,6 +43,7 @@
 // CHECK-NEXT: security.insecureAPI.vfork
 // CHECK-NEXT: unix.API
 // CHECK-NEXT: unix.BlockInCriticalSection
+// CHECK-NEXT: unix.Chroot
 // CHECK-NEXT: unix.cstring.CStringModeling
 // CHECK-NEXT: unix.DynamicMemoryModeling
 // CHECK-NEXT: unix.Errno
@@ -53,6 +54,7 @@
 // CHECK-NEXT: unix.StdCLibraryFunctions
 // CHECK-NEXT: unix.Vfork
 // CHECK-NEXT: unix.cstring.BadSizeArg
+// CHECK-NEXT: unix.cstring.NotNullTerminated
 // CHECK-NEXT: unix.cstring.NullArg
 
 int main() {
diff --git a/clang/test/Analysis/bstring.cpp b/clang/test/Analysis/bstring.cpp
index 1b6397c3455eb..9c30bef15d407 100644
--- a/clang/test/Analysis/bstring.cpp
+++ b/clang/test/Analysis/bstring.cpp
@@ -2,7 +2,7 @@
 // RUN: %clang_analyze_cc1 -DUSE_BUILTINS -analyzer-checker=core,unix.cstring,unix.Malloc,alpha.unix.cstring,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
 // RUN: %clang_analyze_cc1 -DVARIANT -analyzer-checker=core,unix.cstring,alpha.unix.cstring,unix.Malloc,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
 // RUN: %clang_analyze_cc1 -DUSE_BUILTINS -DVARIANT -analyzer-checker=core,unix.cstring,alpha.unix.cstring,unix.Malloc,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -DSUPPRESS_OUT_OF_BOUND -analyzer-checker=core,unix.cstring,unix.Malloc,alpha.unix.cstring.BufferOverlap,alpha.unix.cstring.NotNullTerminated,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -DSUPPRESS_OUT_OF_BOUND -analyzer-checker=core,unix.cstring,unix.Malloc,alpha.unix.cstring.BufferOverlap,unix.cstring.NotNullTerminated,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
 
 #include "Inputs/system-header-simulator-cxx.h"
 #include "Inputs/system-header-simulator-for-malloc.h"
diff --git a/clang/test/Analysis/chroot.c b/clang/test/Analysis/chroot.c
index eb512c05f86f7..34f460b0f0154 100644
--- a/clang/test/Analysis/chroot.c
+++ b/clang/test/Analysis/chroot.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.unix.Chroot -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=unix.Chroot -analyzer-output=text -verify %s
 
 extern int chroot(const char* path);
 extern int chdir(const char* path);
@@ -7,8 +7,10 @@ void foo(void) {
 }
 
 void f1(void) {
-  chroot("/usr/local"); // root changed.
-  foo(); // expected-warning {{No call of chdir("/") immediately after chroot}}
+  chroot("/usr/local"); // expected-note {{chroot called here}}
+  foo();
+  // expected-warning@-1 {{No call of chdir("/") immediately after chroot}}
+  // expected-note@-2    {{No call of chdir("/") immediately after chroot}}
 }
 
 void f2(void) {
@@ -18,7 +20,49 @@ void f2(void) {
 }
 
 void f3(void) {
-  chroot("/usr/local"); // root changed.
+  chroot("/usr/local"); // expected-note {{chroot called here}}
   chdir("../"); // change working directory, still out of jail.
-  foo(); // expected-warning {{No call of chdir("/") immediately after chroot}}
+  foo();
+  // expected-warning@-1 {{No call of chdir("/") immediately after chroot}}
+  // expected-note@-2    {{No call of chdir("/") immediately after chroot}}
+}
+
+void f4(void) {
+  if (chroot("/usr/local") == 0) {
+      chdir("../"); // change working directory, still out of jail.
+  }
+}
+
+void f5(void) {
+  int v = chroot("/usr/local");
+  if (v == -1) {
+      foo();        // no warning, chroot failed
+      chdir("../"); // change working directory, still out of jail.
+  }
+}
+
+void f6(void) {
+  if (chroot("/usr/local") == -1) {
+      chdir("../"); // change working directory, still out of jail.
+  }
+}
+
+void f7(void) {
+  int v = chroot("/usr/local"); // expected-note {{chroot called here}}
+  if (v == -1) { // expected-note {{Taking false branch}}
+      foo();        // no warning, chroot failed
+      chdir("../"); // change working directory, still out of jail.
+  } else {
+      foo();
+      // expected-warning@-1 {{No call of chdir("/") immediately after chroot}}
+      // expected-note@-2    {{No call of chdir("/") immediately after chroot}}
+  }
+}
+
+void f8() {
+  chroot("/usr/local"); // expected-note {{chroot called here}}
+  chdir("/usr"); // This chdir was ineffective because it's not exactly `chdir("/")`.
+  foo();
+  // expected-warning@-1 {{No call of chdir("/") immediately after chroot}}
+  // expected-note@-2    {{No call of chdir("/") immediately after chroot}}
 }
diff --git a/clang/test/Analysis/copy-elision.cpp b/clang/test/Analysis/copy-elision.cpp
index 423c4519f5bfc..cd941854fc7f1 100644
--- a/clang/test/Analysis/copy-elision.cpp
+++ b/clang/test/Analysis/copy-elision.cpp
@@ -263,17 +263,17 @@ void testVariable() {
 
 struct TestCtorInitializer {
   ClassWithDestructor c;
-  TestCtorInitializer(AddressVector<ClassWithDestructor> &v)
-    : c(ClassWithDestructor(v)) {}
-  // no-elide-warning@-1 {{Address of stack memory associated with temporary \
-object of type 'ClassWithDestructor' is still referred \
-to by the caller variable 'v' upon returning to the caller}}
+  TestCtorInitializer(AddressVector<ClassWithDestructor> &refParam)
+    : c(ClassWithDestructor(refParam)) {}
 };
 
 void testCtorInitializer() {
   AddressVector<ClassWithDestructor> v;
   {
     TestCtorInitializer t(v);
+    // no-elide-warning@-1 {{Address of stack memory associated with temporary \
+object of type 'ClassWithDestructor' is still referred \
+to by the caller variable 'v' upon returning to the caller}}
     // Check if the last destructor is an automatic destructor.
     // A temporary destructor would have fired by now.
 #if ELIDE
diff --git a/clang/test/Analysis/ctor-trivial-copy.cpp b/clang/test/Analysis/ctor-trivial-copy.cpp
index 41d0d97161bba..45c8ca4c51776 100644
--- a/clang/test/Analysis/ctor-trivial-copy.cpp
+++ b/clang/test/Analysis/ctor-trivial-copy.cpp
@@ -3,8 +3,10 @@
 
 
 void clang_analyzer_printState();
-template<typename T> void clang_analyzer_dump_lref(T&);
-template<typename T> void clang_analyzer_dump_val(T);
+template <typename T> void clang_analyzer_dump_lref(T& param);
+template <typename T> void clang_analyzer_dump_val(T param);
+template <typename T> void clang_analyzer_denote(T param, const char *name);
+template <typename T> void clang_analyzer_express(T param);
 template <typename T> T conjure();
 template <typename... Ts> void nop(const Ts &... args) {}
 
@@ -40,16 +42,17 @@ void test_assign_return() {
 namespace trivial_struct_copy {
 
 void _01_empty_structs() {
-  clang_analyzer_dump_val(conjure<empty>()); // expected-warning {{lazyCompoundVal}}
+  clang_analyzer_dump_val(conjure<empty>()); // expected-warning {{conj_$}}
   empty Empty = conjure<empty>();
   empty Empty2 = Empty;
   empty Empty3 = Empty2;
-  // All of these should refer to the exact same LCV, because all of
+  // All of these should refer to the exact same symbol, because all of
   // these trivial copies refer to the original conjured value.
   // There were Unknown before:
-  clang_analyzer_dump_val(Empty);  // expected-warning {{lazyCompoundVal}}
-  clang_analyzer_dump_val(Empty2); // expected-warning {{lazyCompoundVal}}
-  clang_analyzer_dump_val(Empty3); // expected-warning {{lazyCompoundVal}}
+  clang_analyzer_denote(Empty, "$Empty");
+  clang_analyzer_express(Empty);  // expected-warning {{$Empty}}
+  clang_analyzer_express(Empty2); // expected-warning {{$Empty}}
+  clang_analyzer_express(Empty3); // expected-warning {{$Empty}}
 
   // We should have the same Conjured symbol for "Empty", "Empty2" and "Empty3".
   clang_analyzer_printState();
@@ -75,15 +78,16 @@ void _01_empty_structs() {
 }
 
 void _02_structs_with_members() {
-  clang_analyzer_dump_val(conjure<aggr>()); // expected-warning {{lazyCompoundVal}}
+  clang_analyzer_dump_val(conjure<aggr>()); // expected-warning {{conj_$}}
   aggr Aggr = conjure<aggr>();
   aggr Aggr2 = Aggr;
   aggr Aggr3 = Aggr2;
-  // All of these should refer to the exact same LCV, because all of
+  // All of these should refer to the exact same symbol, because all of
   // these trivial copies refer to the original conjured value.
-  clang_analyzer_dump_val(Aggr);  // expected-warning {{lazyCompoundVal}}
-  clang_analyzer_dump_val(Aggr2); // expected-warning {{lazyCompoundVal}}
-  clang_analyzer_dump_val(Aggr3); // expected-warning {{lazyCompoundVal}}
+  clang_analyzer_denote(Aggr, "$Aggr");
+  clang_analyzer_express(Aggr);  // expected-warning {{$Aggr}}
+  clang_analyzer_express(Aggr2); // expected-warning {{$Aggr}}
+  clang_analyzer_express(Aggr3); // expected-warning {{$Aggr}}
 
   // We should have the same Conjured symbol for "Aggr", "Aggr2" and "Aggr3".
   // We used to have Derived symbols for the individual fields that were
diff --git a/clang/test/Analysis/cxx-uninitialized-object-unguarded-access.cpp b/clang/test/Analysis/cxx-uninitialized-object-unguarded-access.cpp
index 53e72e7c5f4f7..611e1d8255976 100644
--- a/clang/test/Analysis/cxx-uninitialized-object-unguarded-access.cpp
+++ b/clang/test/Analysis/cxx-uninitialized-object-unguarded-access.cpp
@@ -163,8 +163,8 @@ class UnguardedFieldThroughMethodTest {
       Volume = 0;
       break;
     case A:
-      Area = 0; // expected-warning {{1 uninitialized field}}
-      break;
+      Area = 0;
+      break; // expected-warning {{1 uninitialized field}}
     }
   }
 
@@ -201,8 +201,8 @@ class UnguardedPublicFieldsTest {
       Volume = 0;
       break;
     case A:
-      Area = 0; // expected-warning {{1 uninitialized field}}
-      break;
+      Area = 0;
+      break; // expected-warning {{1 uninitialized field}}
     }
   }
 
diff --git a/clang/test/Analysis/explain-svals.cpp b/clang/test/Analysis/explain-svals.cpp
index 33fce10c4e2b2..d1615e6cc6c9a 100644
--- a/clang/test/Analysis/explain-svals.cpp
+++ b/clang/test/Analysis/explain-svals.cpp
@@ -99,7 +99,7 @@ class C {
 } // end of anonymous namespace
 
 void test_6() {
-  clang_analyzer_explain(conjure_S()); // expected-warning-re{{{{^lazily frozen compound value of 1st parameter of function 'clang_analyzer_explain\(\)'$}}}}
+  clang_analyzer_explain(conjure_S()); // expected-warning-re{{{{^symbol of type 'int' conjured at statement 'conjure_S\(\)'$}}}}
   clang_analyzer_explain(conjure_S().z); // expected-warning-re{{{{^value derived from \(symbol of type 'int' conjured at statement 'conjure_S\(\)'\) for field 'z' of temporary object constructed at statement 'conjure_S\(\)'$}}}}
 }
 
diff --git a/clang/test/Analysis/initializer.cpp b/clang/test/Analysis/initializer.cpp
index 16d7a348fdfb6..f50afff25d245 100644
--- a/clang/test/Analysis/initializer.cpp
+++ b/clang/test/Analysis/initializer.cpp
@@ -366,3 +366,29 @@ void testI() {
   clang_analyzer_eval(B::b == 2); // expected-warning{{TRUE}}
 }
 } // namespace dont_skip_vbase_initializers_in_most_derived_class
+
+namespace elementwise_copy_small_array_from_post_initializer_of_cctor {
+struct String {
+  String(const String &) {}
+};
+
+struct MatchComponent {
+  unsigned numbers[2];
+  String prerelease;
+  MatchComponent(MatchComponent const &) = default;
+};
+
+MatchComponent get();
+void consume(MatchComponent const &);
+
+MatchComponent parseMatchComponent() {
+  MatchComponent component = get();
+  component.numbers[0] = 10;
+  component.numbers[1] = 20;
+  return component; // We should have no stack addr escape warning here.
+}
+
+void top() {
+  consume(parseMatchComponent());
+}
+} // namespace elementwise_copy_small_array_from_post_initializer_of_cctor
diff --git a/clang/test/Analysis/iterator-modeling.cpp b/clang/test/Analysis/iterator-modeling.cpp
index f1538839d06c8..78882da4431fd 100644
--- a/clang/test/Analysis/iterator-modeling.cpp
+++ b/clang/test/Analysis/iterator-modeling.cpp
@@ -2035,6 +2035,7 @@ void print_state(std::vector<int> &V) {
   // CHECK:      "checker_messages": [
   // CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
   // CHECK-NEXT:     "Iterator Positions :",
+  // CHECK-NEXT:     "conj_$[[#]]{int, LC[[#]], S[[#]], #[[#]]} : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}",
   // CHECK-NEXT:     "i0 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
   // CHECK-NEXT:   ]}
 
@@ -2045,6 +2046,7 @@ void print_state(std::vector<int> &V) {
   // CHECK:      "checker_messages": [
   // CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
   // CHECK-NEXT:     "Iterator Positions :",
+  // CHECK-NEXT:     "conj_$[[#]]{int, LC[[#]], S[[#]], #[[#]]} : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}",
   // CHECK-NEXT:     "i1 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
   // CHECK-NEXT:   ]}
 
diff --git a/clang/test/Analysis/reference.cpp b/clang/test/Analysis/reference.cpp
index b893aec2a7d54..91ba3cd23249a 100644
--- a/clang/test/Analysis/reference.cpp
+++ b/clang/test/Analysis/reference.cpp
@@ -89,8 +89,8 @@ namespace PR13440 {
     S s = { a };
     S2 s2 = { a };
 
-    if (s.x != a) return;
-    if (s2.x != a) return;
+    if (s.x != a) return;  // expected-warning {{comparison between two arrays}}
+    if (s2.x != a) return; // expected-warning {{comparison between two arrays}}
 
     a[0] = 42;
     clang_analyzer_eval(s.x[0] == 42); // expected-warning{{TRUE}}
diff --git a/clang/test/Analysis/show-checker-list.c b/clang/test/Analysis/show-checker-list.c
index 3d354c338b9b3..99d7d4ac8dae0 100644
--- a/clang/test/Analysis/show-checker-list.c
+++ b/clang/test/Analysis/show-checker-list.c
@@ -24,10 +24,6 @@
 // RUN:   -analyzer-checker-help-developer \
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-STABLE-ALPHA-DEVELOPER
 
-// CHECK-STABLE-NOT:    alpha.unix.Chroot
-// CHECK-DEVELOPER-NOT: alpha.unix.Chroot
-// CHECK-ALPHA:         alpha.unix.Chroot
-
 // Note that alpha.cplusplus.IteratorModeling is not only an alpha, but also a
 // hidden checker. In this case, we'd only like to see it in the developer list.
 // CHECK-ALPHA-NOT: alpha.cplusplus.IteratorModeling
@@ -42,10 +38,6 @@
 // CHECK-ALPHA-NOT:  debug.ConfigDumper
 
 
-// CHECK-STABLE-ALPHA:         alpha.unix.Chroot
-// CHECK-DEVELOPER-ALPHA:      alpha.unix.Chroot
-// CHECK-STABLE-DEVELOPER-NOT: alpha.unix.Chroot
-
 // CHECK-STABLE-ALPHA:        core.DivideZero
 // CHECK-DEVELOPER-ALPHA-NOT: core.DivideZero
 // CHECK-STABLE-DEVELOPER:    core.DivideZero
@@ -55,6 +47,5 @@
 // CHECK-STABLE-DEVELOPER: debug.ConfigDumper
 
 
-// CHECK-STABLE-ALPHA-DEVELOPER: alpha.unix.Chroot
 // CHECK-STABLE-ALPHA-DEVELOPER: core.DivideZero
 // CHECK-STABLE-ALPHA-DEVELOPER: debug.ConfigDumper
diff --git a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
index 345a4e8f44efd..0910f030b0f07 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
@@ -51,6 +51,7 @@
 // CHECK-NEXT: security.insecureAPI.vfork
 // CHECK-NEXT: unix.API
 // CHECK-NEXT: unix.BlockInCriticalSection
+// CHECK-NEXT: unix.Chroot
 // CHECK-NEXT: unix.cstring.CStringModeling
 // CHECK-NEXT: unix.DynamicMemoryModeling
 // CHECK-NEXT: unix.Errno
@@ -61,6 +62,7 @@
 // CHECK-NEXT: unix.StdCLibraryFunctions
 // CHECK-NEXT: unix.Vfork
 // CHECK-NEXT: unix.cstring.BadSizeArg
+// CHECK-NEXT: unix.cstring.NotNullTerminated
 // CHECK-NEXT: unix.cstring.NullArg
 
 int main() {
diff --git a/clang/test/Analysis/stl-algorithm-modeling-aggressive-std-find-modeling.cpp b/clang/test/Analysis/stl-algorithm-modeling-aggressive-std-find-modeling.cpp
index 98301cf7274fc..191af95cd2b9c 100644
--- a/clang/test/Analysis/stl-algorithm-modeling-aggressive-std-find-modeling.cpp
+++ b/clang/test/Analysis/stl-algorithm-modeling-aggressive-std-find-modeling.cpp
@@ -4,6 +4,16 @@
 // RUN:  -analyzer-config alpha.cplusplus.STLAlgorithmModeling:AggressiveStdFindModeling=true\
 // RUN:  -verify
 
+// STLAlgorithmModeling and DebugIteratorModeling are probably bugged because
+// these tests started failing after we just directly copy the symbol
+// representing the value of a variable instead of creating a LazyCompoundVal
+// of that single conjured value.
+// In theory, it shouldn't matter if we eagerly copy the value that we would
+// "load" from the LCV once requested or just directly binding the backing symbol.
+// Yet, these tests fail, so there is likely messed up how/what the checker
+// metadata is associated with.
+// XFAIL: *
+
 #include "Inputs/system-header-simulator-cxx.h"
 
 void clang_analyzer_eval(bool);
diff --git a/clang/test/Analysis/stl-algorithm-modeling.cpp b/clang/test/Analysis/stl-algorithm-modeling.cpp
index 5549c24a8c220..f7029c79b0942 100644
--- a/clang/test/Analysis/stl-algorithm-modeling.cpp
+++ b/clang/test/Analysis/stl-algorithm-modeling.cpp
@@ -3,6 +3,16 @@
 // RUN:  -analyzer-config aggressive-binary-operation-simplification=true\
 // RUN:  -verify
 
+// STLAlgorithmModeling and DebugIteratorModeling are probably bugged because
+// these tests started failing after we just directly copy the symbol
+// representing the value of a variable instead of creating a LazyCompoundVal
+// of that single conjured value.
+// In theory, it shouldn't matter if we eagerly copy the value that we would
+// "load" from the LCV once requested or just directly binding the backing symbol.
+// Yet, these tests fail, so there is likely messed up how/what the checker
+// metadata is associated with.
+// XFAIL: *
+
 #include "Inputs/system-header-simulator-cxx.h"
 
 void clang_analyzer_eval(bool);
diff --git a/clang/test/Analysis/string.c b/clang/test/Analysis/string.c
index 2e0a49d083b0b..e017aff3b4a1d 100644
--- a/clang/test/Analysis/string.c
+++ b/clang/test/Analysis/string.c
@@ -38,7 +38,7 @@
 // RUN:   -analyzer-checker=unix.cstring \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=alpha.unix.cstring.BufferOverlap \
-// RUN:   -analyzer-checker=alpha.unix.cstring.NotNullTerminated \
+// RUN:   -analyzer-checker=unix.cstring.NotNullTerminated \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false
 
diff --git a/clang/test/Analysis/string.cpp b/clang/test/Analysis/string.cpp
index c09422d192236..5e4580aa6ce1a 100644
--- a/clang/test/Analysis/string.cpp
+++ b/clang/test/Analysis/string.cpp
@@ -53,3 +53,10 @@ struct TestNotNullTerm {
     strlen((char *)&x); // expected-warning{{Argument to string length function is not a null-terminated string}}
   }
 };
+
+void test_notcstring_tempobject() {
+  // FIXME: This warning from cstring.NotNullTerminated is a false positive.
+  // Handling of similar cases is not perfect in other cstring checkers.
+  // The fix would be a larger change in CStringChecker and affect multiple checkers.
+  strlen((char[]){'a', 0}); // expected-warning{{Argument to string length function is a C++ temp object of type char[2], which is not a null-terminated string}}
+}
diff --git a/clang/test/Analysis/string_notnullterm.c b/clang/test/Analysis/string_notnullterm.c
new file mode 100644
index 0000000000000..9c6a6713e615b
--- /dev/null
+++ b/clang/test/Analysis/string_notnullterm.c
@@ -0,0 +1,10 @@
+// RUN: %clang_analyze_cc1 -verify %s -Wno-null-dereference \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=unix.cstring.NotNullTerminated \
+// RUN:   -analyzer-checker=debug.ExprInspection
+
+char *strcpy(char *restrict s1, const char *restrict s2);
+
+void strcpy_fn(char *x) {
+  strcpy(x, (char*)&strcpy_fn); // expected-warning{{Argument to string copy function is the address of the function 'strcpy_fn', which is not a null-terminated string}}
+}
diff --git a/clang/test/Analysis/template-param-objects.cpp b/clang/test/Analysis/template-param-objects.cpp
index dde95fa62cb65..b065f8756d4d8 100644
--- a/clang/test/Analysis/template-param-objects.cpp
+++ b/clang/test/Analysis/template-param-objects.cpp
@@ -11,7 +11,7 @@ bool operator ==(Box lhs, Box rhs) {
   return lhs.value == rhs.value;
 }
 template <Box V> void dumps() {
-  clang_analyzer_dump(V);        // expected-warning {{lazyCompoundVal}}
+  clang_analyzer_dump(V);        // expected-warning {{Unknown}}
   clang_analyzer_dump(&V);       // expected-warning {{Unknown}}
   clang_analyzer_dump(V.value);  // expected-warning {{Unknown}} FIXME: It should be '6 S32b'.
   clang_analyzer_dump(&V.value); // expected-warning {{Unknown}}
diff --git a/clang/test/Analysis/void-call-exit-modelling.c b/clang/test/Analysis/void-call-exit-modelling.c
new file mode 100644
index 0000000000000..6fc8cb54f00bc
--- /dev/null
+++ b/clang/test/Analysis/void-call-exit-modelling.c
@@ -0,0 +1,26 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -analyzer-output text -verify %s
+
+typedef __typeof(sizeof(int)) size_t;
+void *malloc(size_t size);
+
+void inf_loop_break_callee() {
+  void* data = malloc(10); // expected-note{{Memory is allocated}}
+  while (1) { // expected-note{{Loop condition is true}}
+    (void)data;
+    break; // No note that we jump to the line above from this break
+  } // expected-note@-1{{Execution jumps to the end of the function}}
+} // expected-warning{{Potential leak of memory pointed to by 'data'}}
+// expected-note@-1  {{Potential leak of memory pointed to by 'data'}}
+
+void inf_loop_break_caller() {
+  inf_loop_break_callee(); // expected-note{{Calling 'inf_loop_break_callee'}}
+}
+
+void inf_loop_break_top() {
+  void* data = malloc(10); // expected-note{{Memory is allocated}}
+  while (1) { // expected-note{{Loop condition is true}}
+    (void)data;
+    break; // No note that we jump to the line above from this break
+  } // expected-note@-1{{Execution jumps to the end of the function}}
+} // expected-warning{{Potential leak of memory pointed to by 'data'}}
+// expected-note@-1  {{Potential leak of memory pointed to by 'data'}}
diff --git a/clang/test/CIR/global-var-simple.cpp b/clang/test/CIR/global-var-simple.cpp
new file mode 100644
index 0000000000000..5230ff53f87d7
--- /dev/null
+++ b/clang/test/CIR/global-var-simple.cpp
@@ -0,0 +1,59 @@
+// Global variables of intergal types
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o -  | FileCheck %s
+
+char c;
+// CHECK: cir.global @c : !cir.int<s, 8>
+
+signed char sc;
+// CHECK: cir.global @sc : !cir.int<s, 8>
+
+unsigned char uc;
+// CHECK: cir.global @uc : !cir.int<u, 8>
+
+short ss;
+// CHECK: cir.global @ss : !cir.int<s, 16>
+
+unsigned short us;
+// CHECK: cir.global @us : !cir.int<u, 16>
+
+int si;
+// CHECK: cir.global @si : !cir.int<s, 32>
+
+unsigned ui;
+// CHECK: cir.global @ui : !cir.int<u, 32>
+
+long sl;
+// CHECK: cir.global @sl : !cir.int<s, 64>
+
+unsigned long ul;
+// CHECK: cir.global @ul : !cir.int<u, 64>
+
+long long sll;
+// CHECK: cir.global @sll : !cir.int<s, 64>
+
+unsigned long long ull;
+// CHECK: cir.global @ull : !cir.int<u, 64>
+
+__int128 s128;
+// CHECK: cir.global @s128 : !cir.int<s, 128>
+
+unsigned __int128 u128;
+// CHECK: cir.global @u128 : !cir.int<u, 128>
+
+wchar_t wc;
+// CHECK: cir.global @wc : !cir.int<s, 32>
+
+char8_t c8;
+// CHECK: cir.global @c8 : !cir.int<u, 8>
+
+char16_t c16;
+// CHECK: cir.global @c16 : !cir.int<u, 16>
+
+char32_t c32;
+// CHECK: cir.global @c32 : !cir.int<u, 32>
+
+_BitInt(20) sb20;
+// CHECK: cir.global @sb20 : !cir.int<s, 20>
+
+unsigned _BitInt(48) ub48;
+// CHECK: cir.global @ub48 : !cir.int<u, 48>
diff --git a/clang/test/CXX/conv/conv.mem/p4.cpp b/clang/test/CXX/conv/conv.mem/p4.cpp
index d052b7927b32d..552c908a6610b 100644
--- a/clang/test/CXX/conv/conv.mem/p4.cpp
+++ b/clang/test/CXX/conv/conv.mem/p4.cpp
@@ -19,8 +19,8 @@ namespace test0 {
 namespace test1 {
   struct Derived : private Base {}; // expected-note 2 {{declared private here}}
   void test() {
-    int (Derived::*d) = data_ptr; // expected-error {{cannot cast private base class 'Base' to 'test1::Derived'}}
-    int (Derived::*m)() = method_ptr; // expected-error {{cannot cast private base class 'Base' to 'test1::Derived'}}
+    int (Derived::*d) = data_ptr; // expected-error {{cannot cast private base class 'Base' to 'Derived'}}
+    int (Derived::*m)() = method_ptr; // expected-error {{cannot cast private base class 'Base' to 'Derived'}}
   }
 };
 
@@ -30,8 +30,8 @@ namespace test2 {
   struct B : Base {};
   struct Derived : A, B {};
   void test() {
-    int (Derived::*d) = data_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'test2::Derived':}}
-    int (Derived::*m)() = method_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'test2::Derived':}}
+    int (Derived::*d) = data_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'Derived':}}
+    int (Derived::*m)() = method_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'Derived':}}
   }
 }
 
@@ -39,8 +39,8 @@ namespace test2 {
 namespace test3 {
   struct Derived : virtual Base {};
   void test() {
-    int (Derived::*d) = data_ptr;  // expected-error {{conversion from pointer to member of class 'Base' to pointer to member of class 'test3::Derived' via virtual base 'Base' is not allowed}}
-    int (Derived::*m)() = method_ptr; // expected-error {{conversion from pointer to member of class 'Base' to pointer to member of class 'test3::Derived' via virtual base 'Base' is not allowed}}
+    int (Derived::*d) = data_ptr;  // expected-error {{conversion from pointer to member of class 'Base' to pointer to member of class 'Derived' via virtual base 'Base' is not allowed}}
+    int (Derived::*m)() = method_ptr; // expected-error {{conversion from pointer to member of class 'Base' to pointer to member of class 'Derived' via virtual base 'Base' is not allowed}}
   }
 }
 
@@ -49,8 +49,8 @@ namespace test4 {
   struct A : Base {};
   struct Derived : Base, virtual A {}; // expected-warning  {{direct base 'Base' is inaccessible due to ambiguity:\n    struct test4::Derived -> Base\n    struct test4::Derived -> A -> Base}}
   void test() {
-    int (Derived::*d) = data_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'test4::Derived':}}
-    int (Derived::*m)() = method_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'test4::Derived':}}
+    int (Derived::*d) = data_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'Derived':}}
+    int (Derived::*m)() = method_ptr; // expected-error {{ambiguous conversion from pointer to member of base class 'Base' to pointer to member of derived class 'Derived':}}
   }
 }
 
diff --git a/clang/test/CXX/drs/cwg0xx.cpp b/clang/test/CXX/drs/cwg0xx.cpp
index fe3e0cfc1d421..993b6f2923859 100644
--- a/clang/test/CXX/drs/cwg0xx.cpp
+++ b/clang/test/CXX/drs/cwg0xx.cpp
@@ -858,7 +858,7 @@ namespace cwg54 { // cwg54: 2.8
   // expected-error@-1 {{cannot cast 'struct B' to its private base class 'A'}}
   //   expected-note@#cwg54-B {{declared private here}}
   int A::*smab = static_cast<int A::*>(&B::b);
-  // expected-error@-1 {{cannot cast 'cwg54::B' to its private base class 'cwg54::A'}}
+  // expected-error@-1 {{cannot cast 'cwg54::B' to its private base class 'A'}}
   //   expected-note@#cwg54-B {{declared private here}}
   B &sba = static_cast<B&>(a);
   // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
@@ -867,19 +867,19 @@ namespace cwg54 { // cwg54: 2.8
   // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
   //   expected-note@#cwg54-B {{declared private here}}
   int B::*smba = static_cast<int B::*>(&A::a);
-  // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
+  // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'B'}}
   //   expected-note@#cwg54-B {{declared private here}}
 
   V &svb = static_cast<V&>(b);
   V *spvb = static_cast<V*>(&b);
   int V::*smvb = static_cast<int V::*>(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'cwg54::V' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
   B &sbv = static_cast<B&>(v);
   // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *spbv = static_cast<B*>(&v);
   // expected-error@-1 {{cannot cast 'cwg54::V *' to 'B *' via virtual base 'cwg54::V'}}
   int B::*smbv = static_cast<int B::*>(&V::v);
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'cwg54::B' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'B' via virtual base 'cwg54::V' is not allowed}}
 
   A &cab = (A&)(b);
   A *cpab = (A*)(&b);
@@ -891,13 +891,13 @@ namespace cwg54 { // cwg54: 2.8
   V &cvb = (V&)(b);
   V *cpvb = (V*)(&b);
   int V::*cmvb = (int V::*)(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'cwg54::V' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
   B &cbv = (B&)(v);
   // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *cpbv = (B*)(&v);
   // expected-error@-1 {{cannot cast 'cwg54::V *' to 'B *' via virtual base 'cwg54::V'}}
   int B::*cmbv = (int B::*)(&V::v);
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'cwg54::B' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'B' via virtual base 'cwg54::V' is not allowed}}
 }
 
 namespace cwg55 { // cwg55: yes
diff --git a/clang/test/CXX/drs/cwg13xx.cpp b/clang/test/CXX/drs/cwg13xx.cpp
index 416de7c536b1a..980fcb4eb18ac 100644
--- a/clang/test/CXX/drs/cwg13xx.cpp
+++ b/clang/test/CXX/drs/cwg13xx.cpp
@@ -204,7 +204,7 @@ namespace cwg1330 { // cwg1330: 4 c++11
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
   void (A::*af2)() throw() = &A::f;
   // cxx98-14-error@-1 {{target exception specification is not superset of source}}
-  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (cwg1330::A::*)() throw()' with an rvalue of type 'void (cwg1330::A::*)() throw(T)': different exception specifications}}
+  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (A::*)() throw()' with an rvalue of type 'void (cwg1330::A::*)() throw(T)': different exception specifications}}
 
 #if __cplusplus >= 201103L
   static_assert(noexcept(A().g()), "");
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index 10c8d86ed16a0..26b0f975effa1 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -537,9 +537,9 @@ namespace cwg330 { // cwg330: 7
     void f(A *a) {
       (void) reinterpret_cast<B1*>(a);
       (void) reinterpret_cast<B2*>(a);
-      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B2 *' (aka 'int *const cwg330::swift_17882::X::***') because it casts away qualifiers, even though the source and destination types are unrelated}}
+      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B2 *' (aka 'int *const X::***') because it casts away qualifiers, even though the source and destination types are unrelated}}
       (void) reinterpret_cast<B3*>(a);
-      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B3 *' (aka 'int *cwg330::swift_17882::X::*volatile **') because it casts away qualifiers, even though the source and destination types are unrelated}}
+      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B3 *' (aka 'int *X::*volatile **') because it casts away qualifiers, even though the source and destination types are unrelated}}
       (void) reinterpret_cast<B4*>(a);
     }
   }
@@ -971,9 +971,9 @@ namespace cwg354 { // cwg354: yes c++11
   ptr<(int S::*)0> p3; // #cwg354-p3
   // cxx98-error@#cwg354-p3 {{non-type template argument does not refer to any declaration}}
   //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
-  // cxx11-14-error@#cwg354-p3 {{null non-type template argument of type 'int cwg354::S::*' does not match template parameter of type 'int *'}}
+  // cxx11-14-error@#cwg354-p3 {{null non-type template argument of type 'int S::*' does not match template parameter of type 'int *'}}
   //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
-  // since-cxx17-error@#cwg354-p3 {{value of type 'int cwg354::S::*' is not implicitly convertible to 'int *'}}
+  // since-cxx17-error@#cwg354-p3 {{value of type 'int S::*' is not implicitly convertible to 'int *'}}
 
   template<int*> int both(); // #cwg354-both-int-ptr
   template<int> int both(); // #cwg354-both-int
@@ -985,25 +985,25 @@ namespace cwg354 { // cwg354: yes c++11
 
   template<int S::*> struct ptr_mem {}; // #cwg354-ptr_mem
   ptr_mem<0> m0; // #cwg354-m0
-  // cxx98-error@#cwg354-m0 {{non-type template argument of type 'int' cannot be converted to a value of type 'int cwg354::S::*'}}
+  // cxx98-error@#cwg354-m0 {{non-type template argument of type 'int' cannot be converted to a value of type 'int S::*'}}
   //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#cwg354-m0 {{null non-type template argument must be cast to template parameter type 'int cwg354::S::*'}}
+  // cxx11-14-error@#cwg354-m0 {{null non-type template argument must be cast to template parameter type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#cwg354-m0 {{conversion from 'int' to 'int cwg354::S::*' is not allowed in a converted constant expression}}
+  // since-cxx17-error@#cwg354-m0 {{conversion from 'int' to 'int S::*' is not allowed in a converted constant expression}}
   ptr_mem<(int S::*)0> m1;
   // cxx98-error@-1 {{non-type template argument is not a pointer to member constant}}
   ptr_mem<(float S::*)0> m2; // #cwg354-m2
-  // cxx98-error@#cwg354-m2 {{non-type template argument of type 'float cwg354::S::*' cannot be converted to a value of type 'int cwg354::S::*'}}
+  // cxx98-error@#cwg354-m2 {{non-type template argument of type 'float S::*' cannot be converted to a value of type 'int S::*'}}
   //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#cwg354-m2 {{null non-type template argument of type 'float cwg354::S::*' does not match template parameter of type 'int cwg354::S::*'}}
+  // cxx11-14-error@#cwg354-m2 {{null non-type template argument of type 'float S::*' does not match template parameter of type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#cwg354-m2 {{value of type 'float cwg354::S::*' is not implicitly convertible to 'int cwg354::S::*'}}
+  // since-cxx17-error@#cwg354-m2 {{value of type 'float S::*' is not implicitly convertible to 'int S::*'}}
   ptr_mem<(int *)0> m3; // #cwg354-m3
-  // cxx98-error@#cwg354-m3 {{non-type template argument of type 'int *' cannot be converted to a value of type 'int cwg354::S::*'}}
+  // cxx98-error@#cwg354-m3 {{non-type template argument of type 'int *' cannot be converted to a value of type 'int S::*'}}
   //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#cwg354-m3 {{null non-type template argument of type 'int *' does not match template parameter of type 'int cwg354::S::*'}}
+  // cxx11-14-error@#cwg354-m3 {{null non-type template argument of type 'int *' does not match template parameter of type 'int S::*'}}
   //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#cwg354-m3 {{value of type 'int *' is not implicitly convertible to 'int cwg354::S::*'}}
+  // since-cxx17-error@#cwg354-m3 {{value of type 'int *' is not implicitly convertible to 'int S::*'}}
 }
 
 struct cwg355_S; // cwg355: yes
@@ -1701,7 +1701,7 @@ namespace cwg395 { // cwg395: 3.0
     }
   } null1;
   int (S::*p)() = null1;
-  // expected-error@-1 {{no viable conversion from 'struct null1_t' to 'int (cwg395::S::*)()'}}
+  // expected-error@-1 {{no viable conversion from 'struct null1_t' to 'int (S::*)()'}}
   //   expected-note@#cwg395-conv-func {{candidate template ignored: couldn't infer template argument 'T'}}
 
   template <typename T> using id = T;
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index 98ff7553d989b..825065840f118 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -1172,11 +1172,11 @@ namespace cwg480 { // cwg480: yes
 
   int A::*a = &A::n;
   int D::*b = a;
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::A' to pointer to member of class 'cwg480::D' via virtual base 'cwg480::B' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'A' to pointer to member of class 'D' via virtual base 'cwg480::B' is not allowed}}
 
   extern int D::*c;
   int A::*d = static_cast<int A::*>(c);
-  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::D' to pointer to member of class 'cwg480::A' via virtual base 'cwg480::B' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::D' to pointer to member of class 'A' via virtual base 'cwg480::B' is not allowed}}
 
   D *e;
   A *f = e;
diff --git a/clang/test/CXX/drs/cwg5xx.cpp b/clang/test/CXX/drs/cwg5xx.cpp
index ed0c7159dfc88..9219a4c857b12 100644
--- a/clang/test/CXX/drs/cwg5xx.cpp
+++ b/clang/test/CXX/drs/cwg5xx.cpp
@@ -184,7 +184,7 @@ namespace cwg522 { // cwg522: yes
     b2(am);
     b2a(am);
     // expected-error@-1 {{no matching function for call to 'b2a'}}
-    //   expected-note@#cwg522-b2a {{candidate template ignored: deduced type 'volatile int *cwg522::S::*const *' of 1st parameter does not match adjusted type 'int *cwg522::S::**' of argument}}
+    //   expected-note@#cwg522-b2a {{candidate template ignored: deduced type 'volatile int *S::*const *' of 1st parameter does not match adjusted type 'int *S::**' of argument}}
     b3(d);
     b3(cd);
   }
@@ -1178,17 +1178,70 @@ namespace cwg590 { // cwg590: yes
   template<typename T> typename A<T>::B::C A<T>::B::C::f(A<T>::B::C) {}
 }
 
-namespace cwg591 { // cwg591: no
+namespace cwg591 { // cwg591: 20
   template<typename T> struct A {
     typedef int M;
     struct B {
       typedef void M;
       struct C;
+      struct D;
     };
   };
 
+  template<typename T> struct G {
+    struct B {
+      typedef int M;
+      struct C {
+        typedef void M;
+        struct D;
+      };
+    };
+  };
+
+  template<typename T> struct H {
+    template<typename U> struct B {
+      typedef int M;
+      template<typename F> struct C {
+        typedef void M;
+        struct D;
+        struct P;
+      };
+    };
+  };
+
+  template <typename, bool> struct M {
+    class P;
+    int M;
+  };
+
   template<typename T> struct A<T>::B::C : A<T> {
-    // FIXME: Should find member of non-dependent base class A<T>.
+    M m;
+  };
+
+  template<typename T> struct G<T>::B::C::D : B {
+    M m;
+  };
+
+  template<typename T>
+  template<typename U>
+  template<typename F>
+  struct H<T>::B<U>::C<F>::D : B<U> {
+    M m;
+  };
+
+  template<typename T, bool B> class M<T,B>::P : M {
+    int foo() { (void) M; }
+  };
+
+  template<typename T> struct A<T>::B::D : A<T*> {
+    M m;
+    // expected-error@-1 {{field has incomplete type 'M' (aka 'void'}}
+  };
+
+  template<typename T>
+  template<typename U>
+  template<typename F>
+  struct H<T>::B<U>::C<F>::P : B<F> {
     M m;
     // expected-error@-1 {{field has incomplete type 'M' (aka 'void'}}
   };
diff --git a/clang/test/CXX/drs/cwg722.cpp b/clang/test/CXX/drs/cwg722.cpp
index e90d9af360d9d..6e7d4569c5538 100644
--- a/clang/test/CXX/drs/cwg722.cpp
+++ b/clang/test/CXX/drs/cwg722.cpp
@@ -14,7 +14,7 @@ namespace std {
   using nullptr_t = decltype(nullptr);
 }
 
-void f(std::nullptr_t...);
+void f(std::nullptr_t, ...);
 std::nullptr_t g();
 void h() {
   std::nullptr_t np;
diff --git a/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp b/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
index 170ca0a3f1c6b..e49f396143ad6 100644
--- a/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
+++ b/clang/test/CXX/expr/expr.unary/expr.unary.op/p4.cpp
@@ -6,7 +6,7 @@ namespace test0 {
     template<typename T> void g(T);
 
     void test() {
-      foo(&g<int>); // expected-error-re {{cannot form member pointer of type 'void (test0::A::*)(int){{( __attribute__\(\(thiscall\)\))?}}' without '&' and class name}}
+      foo(&g<int>); // expected-error-re {{cannot form member pointer of type 'void (A::*)(int){{( __attribute__\(\(thiscall\)\))?}}' without '&' and class name}}
     }
   };
 }
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
index 5341290a1446e..034ad49d0715c 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
@@ -223,7 +223,7 @@ namespace pointer_to_member_function {
   template<int (Y::*)(int)> struct X0 {}; // expected-note 0-1{{template parameter is declared here}}
   X0<&Y::f> x0a;
   X0<&Y::g> x0b;
-  X0<&Y::h> x0c; // expected-error-re{{type 'float (pointer_to_member_function::Y::*)(float){{( __attribute__\(\(thiscall\)\))?}}' {{.*}} convert{{.*}} 'int (pointer_to_member_function::Y::*)(int){{( __attribute__\(\(thiscall\)\))?}}'}}
+  X0<&Y::h> x0c; // expected-error-re{{type 'float (pointer_to_member_function::Y::*)(float){{( __attribute__\(\(thiscall\)\))?}}' {{.*}} convert{{.*}} 'int (Y::*)(int){{( __attribute__\(\(thiscall\)\))?}}'}}
 }
 
 //     -- For a non-type template-parameter of type pointer to data member,
@@ -238,9 +238,9 @@ namespace pointer_to_member_data {
   X0<&Y::y> x0a;
   X0<&Y::x> x0b;
 #if __cplusplus <= 201402L
-  // expected-error@-2 {{non-type template argument of type 'int pointer_to_member_data::X::*' cannot be converted to a value of type 'int pointer_to_member_data::Y::*'}}
+  // expected-error@-2 {{non-type template argument of type 'int pointer_to_member_data::X::*' cannot be converted to a value of type 'int Y::*'}}
 #else
-  // expected-error@-4 {{conversion from 'int pointer_to_member_data::X::*' to 'int pointer_to_member_data::Y::*' is not allowed in a converted constant expression}}
+  // expected-error@-4 {{conversion from 'int pointer_to_member_data::X::*' to 'int Y::*' is not allowed in a converted constant expression}}
 #endif
 
   // Test qualification conversions
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/sizeofpack.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/sizeofpack.cpp
index 87c22a0d7e944..50dedb1a158d3 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/sizeofpack.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/sizeofpack.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
 // expected-no-diagnostics
 
 namespace pr12262 {
@@ -201,3 +202,24 @@ void func()
 }
 
 }
+
+#if __cplusplus >= 202002L
+namespace GH81436 {
+
+template <class E> struct Bar;
+
+template <class E>
+Bar(E) -> Bar<E>;
+
+template <int> struct Foo {};
+
+// Bar<Ts> doesn't have to be of a complete type.
+template <class... Ts>
+auto func() requires requires(Bar<Ts> ...init_lists) {
+  sizeof...(init_lists) > 0;
+} {}
+
+void f() { func<int>(); }
+
+} // namespace GH81436
+#endif
diff --git a/clang/test/CodeGen/AArch64/fixed-register-global.c b/clang/test/CodeGen/AArch64/fixed-register-global.c
new file mode 100644
index 0000000000000..25c7aa254547b
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fixed-register-global.c
@@ -0,0 +1,23 @@
+/// Check that -ffixed register handled for globals.
+/// Regression test for #76426, #109778
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang -c --target=aarch64-none-gnu -ffixed-x15 %s -o /dev/null 2>&1 | count 0
+
+// RUN: not %clang -c --target=aarch64-none-gnu %s -o /dev/null 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR_INVREG
+// ERR_INVREG: error: register 'x15' unsuitable for global register variables on this target
+
+// RUN: not %clang -c --target=aarch64-none-gnu -ffixed-x15 -DTYPE=short %s -o /dev/null 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=ERR_SIZE
+// ERR_SIZE: error: size of register 'x15' does not match variable size
+
+#ifndef TYPE
+#define TYPE long
+#endif
+
+register TYPE x15 __asm__("x15");
+
+TYPE foo() {
+  return x15;
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_cvt.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_cvt.c
new file mode 100644
index 0000000000000..13609f034da33
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_cvt.c
@@ -0,0 +1,145 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: @test_cvt1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_cvt1_f16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+svfloat16x2_t test_cvt1_f16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvt1_f16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvt2_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_cvt2_f16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+svfloat16x2_t test_cvt2_f16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvt2_f16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvt1_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_cvt1_bf16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+svbfloat16x2_t test_cvt1_bf16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvt1_bf16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvt2_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_cvt2_bf16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+svbfloat16x2_t test_cvt2_bf16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvt2_bf16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvtl1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_cvtl1_f16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+svfloat16x2_t test_cvtl1_f16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvtl1_f16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvtl2_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_cvtl2_f16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxv8f16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]]
+//
+svfloat16x2_t test_cvtl2_f16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvtl2_f16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvtl1_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_cvtl1_bf16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+svbfloat16x2_t test_cvtl1_bf16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvtl1_bf16,_mf8,_x2_fpm)(zn, fpmr);
+}
+
+// CHECK-LABEL: @test_cvtl2_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z18test_cvtl2_bf16_x2u13__SVMfloat8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxv8bf16(<vscale x 16 x i8> [[ZN:%.*]])
+// CPP-CHECK-NEXT:    ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP0]]
+//
+svbfloat16x2_t test_cvtl2_bf16_x2(svmfloat8_t zn, fpm_t fpmr)  __arm_streaming {
+  return SVE_ACLE_FUNC(svcvtl2_bf16,_mf8,_x2_fpm)(zn, fpmr);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_fmopa.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_fmopa.c
new file mode 100644
index 0000000000000..95d6383ab30ef
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sme2_fp8_fmopa.c
@@ -0,0 +1,55 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme-f8f16 -target-feature +sme-f8f32 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sme.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+
+// CHECK-LABEL: define dso_local void @test_svmopa_za16_mf8_m(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fp8.fmopa.za16(i32 1, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: define dso_local void @_Z22test_svmopa_za16_mf8_mu10__SVBool_tS_u13__SVMfloat8_tS0_m(
+// CPP-CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT:  [[ENTRY:.*:]]
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.fp8.fmopa.za16(i32 1, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svmopa_za16_mf8_m(svbool_t pn, svbool_t pm, svmfloat8_t zn,
+                            svmfloat8_t zm, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svmopa_za16,_mf8,_m_fpm)(1, pn, pm, zn, zm, fpmr);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmopa_za32_mf8_m(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fp8.fmopa.za32(i32 3, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: define dso_local void @_Z22test_svmopa_za32_mf8_mu10__SVBool_tS_u13__SVMfloat8_tS0_m(
+// CPP-CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  [[ENTRY:.*:]]
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.fp8.fmopa.za32(i32 3, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svmopa_za32_mf8_m(svbool_t pn, svbool_t pm, svmfloat8_t zn,
+                            svmfloat8_t zm, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svmopa_za32,_mf8,_m_fpm)(3, pn, pm, zn, zm, fpmr);
+}
diff --git a/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sve2_fp8_cvt.c b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sve2_fp8_cvt.c
new file mode 100644
index 0000000000000..c026b8aa216f3
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/fp8-intrinsics/acle_sve2_fp8_cvt.c
@@ -0,0 +1,173 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1        -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sve2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CHECK-CXX
+
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS        -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -x c++ -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sve2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CHECK-CXX
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sve2 -target-feature +fp8 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -target-feature +fp8 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#ifdef __ARM_FEATURE_SME
+#include <arm_sme.h>
+#else
+#include <arm_sve.h>
+#endif
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+#ifdef __ARM_FEATURE_SME
+#define STREAMING __arm_streaming
+#else
+#define STREAMING
+#endif
+
+// CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @test_svcvt1_bf16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt1.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z20test_svcvt1_bf16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt1.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+svbfloat16_t test_svcvt1_bf16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvt1_bf16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @test_svcvt2_bf16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt2.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z20test_svcvt2_bf16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt2.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+svbfloat16_t test_svcvt2_bf16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvt2_bf16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @test_svcvtlt1_bf16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt1.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z22test_svcvtlt1_bf16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt1.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+svbfloat16_t test_svcvtlt1_bf16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvtlt1_bf16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @test_svcvtlt2_bf16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt2.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z22test_svcvtlt2_bf16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt2.nxv8bf16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+svbfloat16_t test_svcvtlt2_bf16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvtlt2_bf16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x half> @test_svcvt1_f16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt1.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z19test_svcvt1_f16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt1.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+svfloat16_t test_svcvt1_f16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvt1_f16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x half> @test_svcvt2_f16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt2.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z19test_svcvt2_f16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt2.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+svfloat16_t test_svcvt2_f16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvt2_f16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x half> @test_svcvtlt1_f16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt1.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z21test_svcvtlt1_f16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt1.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+svfloat16_t test_svcvtlt1_f16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvtlt1_f16,_mf8,_fpm)(zn, fpm);
+}
+
+// CHECK-LABEL: define dso_local <vscale x 8 x half> @test_svcvtlt2_f16_mf8(
+// CHECK-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt2.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z21test_svcvtlt2_f16_mf8u13__SVMfloat8_tm(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZN:%.*]], i64 noundef [[FPM:%.*]]) #[[ATTR0]] {
+// CHECK-CXX-NEXT:  [[ENTRY:.*:]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPM]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt2.nxv8f16(<vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP0]]
+//
+svfloat16_t test_svcvtlt2_f16_mf8(svmfloat8_t zn, fpm_t fpm) STREAMING {
+  return SVE_ACLE_FUNC(svcvtlt2_f16,_mf8,_fpm)(zn, fpm);
+}
diff --git a/clang/test/CodeGen/AArch64/ls64-inline-asm.c b/clang/test/CodeGen/AArch64/ls64-inline-asm.c
index a01393525bcd4..8aa0684dba14d 100644
--- a/clang/test/CodeGen/AArch64/ls64-inline-asm.c
+++ b/clang/test/CodeGen/AArch64/ls64-inline-asm.c
@@ -5,7 +5,7 @@ struct foo { unsigned long long x[8]; };
 
 // CHECK-LABEL: @load(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call i512 asm sideeffect "ld64b $0,[$1]", "=r,r,~{memory}"(ptr [[ADDR:%.*]]) #[[ATTR1:[0-9]+]], !srcloc !2
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i512 asm sideeffect "ld64b $0,[$1]", "=r,r,~{memory}"(ptr [[ADDR:%.*]]) #[[ATTR1:[0-9]+]], !srcloc [[META2:![0-9]+]]
 // CHECK-NEXT:    store i512 [[TMP0]], ptr [[OUTPUT:%.*]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -17,7 +17,7 @@ void load(struct foo *output, void *addr)
 // CHECK-LABEL: @store(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load i512, ptr [[INPUT:%.*]], align 8
-// CHECK-NEXT:    tail call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 [[TMP0]], ptr [[ADDR:%.*]]) #[[ATTR1]], !srcloc !3
+// CHECK-NEXT:    tail call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 [[TMP0]], ptr [[ADDR:%.*]]) #[[ATTR1]], !srcloc [[META3:![0-9]+]]
 // CHECK-NEXT:    ret void
 //
 void store(const struct foo *input, void *addr)
@@ -29,25 +29,25 @@ void store(const struct foo *input, void *addr)
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[IN:%.*]], align 4, !tbaa [[TBAA4:![0-9]+]]
 // CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP0]] to i64
-// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 4
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV2:%.*]] = sext i32 [[TMP1]] to i64
-// CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 16
+// CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 16
 // CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV5:%.*]] = sext i32 [[TMP2]] to i64
-// CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 64
+// CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 64
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV8:%.*]] = sext i32 [[TMP3]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 100
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 100
 // CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV11:%.*]] = sext i32 [[TMP4]] to i64
-// CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 144
+// CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 144
 // CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX13]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV14:%.*]] = sext i32 [[TMP5]] to i64
-// CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 196
+// CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 196
 // CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX16]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV17:%.*]] = sext i32 [[TMP6]] to i64
-// CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 256
+// CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds nuw i8, ptr [[IN]], i64 256
 // CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX19]], align 4, !tbaa [[TBAA4]]
 // CHECK-NEXT:    [[CONV20:%.*]] = sext i32 [[TMP7]] to i64
 // CHECK-NEXT:    [[S_SROA_10_0_INSERT_EXT:%.*]] = zext i64 [[CONV20]] to i512
@@ -72,7 +72,7 @@ void store(const struct foo *input, void *addr)
 // CHECK-NEXT:    [[S_SROA_0_0_INSERT_EXT:%.*]] = zext i64 [[CONV]] to i512
 // CHECK-NEXT:    [[S_SROA_0_0_INSERT_MASK:%.*]] = or disjoint i512 [[S_SROA_4_0_INSERT_MASK]], [[S_SROA_4_0_INSERT_SHIFT]]
 // CHECK-NEXT:    [[S_SROA_0_0_INSERT_INSERT:%.*]] = or i512 [[S_SROA_0_0_INSERT_MASK]], [[S_SROA_0_0_INSERT_EXT]]
-// CHECK-NEXT:    tail call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 [[S_SROA_0_0_INSERT_INSERT]], ptr [[ADDR:%.*]]) #[[ATTR1]], !srcloc !8
+// CHECK-NEXT:    tail call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 [[S_SROA_0_0_INSERT_INSERT]], ptr [[ADDR:%.*]]) #[[ATTR1]], !srcloc [[META8:![0-9]+]]
 // CHECK-NEXT:    ret void
 //
 void store2(int *in, void *addr)
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index 5c74447100aa8..a8c3dd9288a5b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -70,23 +70,23 @@ void test_argpass_simple(PST *p) {
 // CHECK-AAPCS:      define dso_local void @test_argpass_simple(ptr nocapture noundef readonly %p)
 // CHECK-AAPCS-NEXT: entry:
 // CHECK-AAPCS-NEXT: %0 = load <2 x i8>, ptr %p, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %0, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> %0, i64 0)
 // CHECK-AAPCS-NEXT: %1 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
 // CHECK-AAPCS-NEXT: %2 = getelementptr inbounds nuw i8, ptr %p, i64 16
 // CHECK-AAPCS-NEXT: %3 = load <2 x double>, ptr %2, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable1 = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> %3, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable1 = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> %3, i64 0)
 // CHECK-AAPCS-NEXT: %4 = getelementptr inbounds nuw i8, ptr %p, i64 32
 // CHECK-AAPCS-NEXT: %5 = load <4 x float>, ptr %4, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable2 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %5, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable2 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %5, i64 0)
 // CHECK-AAPCS-NEXT: %6 = getelementptr inbounds nuw i8, ptr %p, i64 48
 // CHECK-AAPCS-NEXT: %7 = load <4 x float>, ptr %6, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable3 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %7, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable3 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %7, i64 0)
 // CHECK-AAPCS-NEXT: %8 = getelementptr inbounds nuw i8, ptr %p, i64 64
 // CHECK-AAPCS-NEXT: %9 = load <16 x i8>, ptr %8, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> %9, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> %9, i64 0)
 // CHECK-AAPCS-NEXT: %10 = getelementptr inbounds nuw i8, ptr %p, i64 80
 // CHECK-AAPCS-NEXT: %11 = load <2 x i8>, ptr %10, align 16
-// CHECK-AAPCS-NEXT: %cast.scalable5 = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %11, i64 0)
+// CHECK-AAPCS-NEXT: %cast.scalable5 = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> %11, i64 0)
 // CHECK-AAPCS-NEXT: %12 = bitcast <vscale x 2 x i8> %cast.scalable5 to <vscale x 16 x i1>
 // CHECK-AAPCS-NEXT: tail call void @argpass_simple_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12)
 // CHECK-AAPCS-NEXT: ret void
@@ -417,7 +417,7 @@ void test_va_arg(int n, ...) {
 // CHECK-AAPCS-EMPTY:
 // CHECK-AAPCS-NEXT: vaarg.on_stack:                                   ; preds = %vaarg.maybe_reg, %entry
 // CHECK-AAPCS-NEXT:   %stack = load ptr, ptr %ap, align 8
-// CHECK-AAPCS-NEXT:   %new_stack = getelementptr inbounds i8, ptr %stack, i64 8
+// CHECK-AAPCS-NEXT:   %new_stack = getelementptr inbounds nuw i8, ptr %stack, i64 8
 // CHECK-AAPCS-NEXT:   store ptr %new_stack, ptr %ap, align 8
 // CHECK-AAPCS-NEXT:   br label %vaarg.end
 // CHECK-AAPCS-EMPTY:
@@ -428,12 +428,12 @@ void test_va_arg(int n, ...) {
 // CHECK-AAPCS-NEXT:   %vaarg.addr = load ptr, ptr %vaargs.addr, align 8
 
 // CHECK-AAPCS-NEXT:   %v.sroa.0.0.copyload = load <2 x i8>, ptr %vaarg.addr, align 16
-// CHECK-AAPCS-NEXT:   %v.sroa.43.0.vaarg.addr.sroa_idx = getelementptr inbounds i8, ptr %vaarg.addr, i64 48
+// CHECK-AAPCS-NEXT:   %v.sroa.43.0.vaarg.addr.sroa_idx = getelementptr inbounds nuw i8, ptr %vaarg.addr, i64 48
 // CHECK-AAPCS-NEXT:   %v.sroa.43.0.copyload = load <4 x float>, ptr %v.sroa.43.0.vaarg.addr.sroa_idx, align 16
 // CHECK-AAPCS-NEXT:   call void @llvm.va_end.p0(ptr nonnull %ap)
-// CHECK-AAPCS-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %v.sroa.0.0.copyload, i64 0)
+// CHECK-AAPCS-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> %v.sroa.0.0.copyload, i64 0)
 // CHECK-AAPCS-NEXT:   %3 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
-// CHECK-AAPCS-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %v.sroa.43.0.copyload, i64 0)
+// CHECK-AAPCS-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %v.sroa.43.0.copyload, i64 0)
 // CHECK-AAPCS-NEXT:   call void @use1(<vscale x 16 x i1> noundef %3, <vscale x 4 x float> noundef %cast.scalable2)
 // CHECK-AAPCS-NEXT:   call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %ap)
 // CHECK-AAPCS-NEXT:   ret void
@@ -445,16 +445,16 @@ void test_va_arg(int n, ...) {
 // CHECK-DARWIN-NEXT:   call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   call void @llvm.va_start.p0(ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   %argp.cur = load ptr, ptr %ap, align 8
-// CHECK-DARWIN-NEXT:   %argp.next = getelementptr inbounds i8, ptr %argp.cur, i64 8
+// CHECK-DARWIN-NEXT:   %argp.next = getelementptr inbounds nuw i8, ptr %argp.cur, i64 8
 // CHECK-DARWIN-NEXT:   store ptr %argp.next, ptr %ap, align 8
 // CHECK-DARWIN-NEXT:   %0 = load ptr, ptr %argp.cur, align 8
 // CHECK-DARWIN-NEXT:   %v.sroa.0.0.copyload = load <2 x i8>, ptr %0, align 16
-// CHECK-DARWIN-NEXT:   %v.sroa.43.0..sroa_idx = getelementptr inbounds i8, ptr %0, i64 48
+// CHECK-DARWIN-NEXT:   %v.sroa.43.0..sroa_idx = getelementptr inbounds nuw i8, ptr %0, i64 48
 // CHECK-DARWIN-NEXT:   %v.sroa.43.0.copyload = load <4 x float>, ptr %v.sroa.43.0..sroa_idx, align 16
 // CHECK-DARWIN-NEXT:   call void @llvm.va_end.p0(ptr nonnull %ap)
-// CHECK-DARWIN-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> %v.sroa.0.0.copyload, i64 0)
+// CHECK-DARWIN-NEXT:   %cast.scalable = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> %v.sroa.0.0.copyload, i64 0)
 // CHECK-DARWIN-NEXT:   %1 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
-// CHECK-DARWIN-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %v.sroa.43.0.copyload, i64 0)
+// CHECK-DARWIN-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %v.sroa.43.0.copyload, i64 0)
 // CHECK-DARWIN-NEXT:   call void @use1(<vscale x 16 x i1> noundef %1, <vscale x 4 x float> noundef %cast.scalable2)
 // CHECK-DARWIN-NEXT:   call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   ret void
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
index fcbd17559dc70..fb86690f07f1d 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
@@ -12,9 +12,12 @@
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svld1_hor_vnum_za8ju10__SVBool_tPKvl(
@@ -23,9 +26,12 @@
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -40,9 +46,12 @@ void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za16ju10__SVBool_tPKvl(
@@ -52,9 +61,12 @@ void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -69,9 +81,12 @@ void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za32ju10__SVBool_tPKvl(
@@ -81,9 +96,12 @@ void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -98,9 +116,12 @@ void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za64ju10__SVBool_tPKvl(
@@ -110,9 +131,12 @@ void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -127,8 +151,10 @@ void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_hor_vnum_za128ju10__SVBool_tPKvl(
@@ -138,8 +164,10 @@ void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -153,9 +181,12 @@ void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z22test_svld1_ver_hor_za8ju10__SVBool_tPKvl(
@@ -164,9 +195,12 @@ void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -181,9 +215,12 @@ void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, i
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za16ju10__SVBool_tPKvl(
@@ -193,9 +230,12 @@ void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, i
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -210,9 +250,12 @@ void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za32ju10__SVBool_tPKvl(
@@ -222,9 +265,12 @@ void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -239,9 +285,12 @@ void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za64ju10__SVBool_tPKvl(
@@ -251,9 +300,12 @@ void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -268,8 +320,10 @@ void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_ver_vnum_za128ju10__SVBool_tPKvl(
@@ -279,8 +333,10 @@ void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
index b2c609b7c2242..dafc3d61a05f1 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
@@ -12,9 +12,12 @@
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_hor_vnum_za8ju10__SVBool_tPvl(
@@ -23,9 +26,12 @@
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -40,9 +46,12 @@ void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za16ju10__SVBool_tPvl(
@@ -52,9 +61,12 @@ void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -69,9 +81,12 @@ void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za32ju10__SVBool_tPvl(
@@ -81,9 +96,12 @@ void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -98,9 +116,12 @@ void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za64ju10__SVBool_tPvl(
@@ -110,9 +131,12 @@ void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -127,8 +151,10 @@ void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_hor_vnum_za128ju10__SVBool_tPvl(
@@ -138,8 +164,10 @@ void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -153,9 +181,12 @@ void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int6
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_ver_vnum_za8ju10__SVBool_tPvl(
@@ -164,9 +195,12 @@ void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int6
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -181,9 +215,12 @@ void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za16ju10__SVBool_tPvl(
@@ -193,9 +230,12 @@ void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -210,9 +250,12 @@ void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za32ju10__SVBool_tPvl(
@@ -222,9 +265,12 @@ void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -239,9 +285,12 @@ void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za64ju10__SVBool_tPvl(
@@ -251,9 +300,12 @@ void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -268,8 +320,10 @@ void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_ver_vnum_za128ju10__SVBool_tPvl(
@@ -279,8 +333,10 @@ void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
 // CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
 // CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index 54e90223a31de..28d69d52c9ae7 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -53,7 +53,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK128-LABEL: define{{.*}} <16 x i8> @f2(<16 x i8> noundef %x)
 // CHECK128-NEXT:  entry:
 // CHECK128-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-// CHECK128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> [[X:%.*]], i64 0)
+// CHECK128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> [[X:%.*]], i64 0)
 // CHECK128-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.asrd.nxv16i8(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i8> [[CASTSCALABLESVE]], i32 1)
 // CHECK128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[TMP1]], i64 0)
 // CHECK128-NEXT:    ret <16 x i8> [[CASTFIXEDSVE]]
@@ -63,7 +63,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECK-NEXT:   [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-// CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> undef, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
+// CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> poison, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
 // CHECK-NEXT:   [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.asrd.nxv16i8(<vscale x 16 x i1> [[TMP1]], <vscale x 16 x i8> [[CASTSCALABLESVE]], i32 1)
 // CHECK-NEXT:   [[CASTFIXEDSVE:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[TMP2]], i64 0)
 // CHECK-NEXT:   store <[[#div(VBITS,8)]] x i8> [[CASTFIXEDSVE]], ptr [[AGG_RESULT:%.*]], align 16, [[TBAA6]]
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index 30ea73b63bce1..e82069aab2486 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -52,7 +52,7 @@ void test02() {
 // CHECK-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS, 32)]] x i32> @llvm.vector.extract.v[[#div(VBITS, 32)]]i32.nxv4i32(<vscale x 4 x i32> [[X_COERCE:%.*]], i64 0)
 // CHECK-NEXT:   [[Y:%.*]] = tail call <[[#div(VBITS, 32)]] x i32> @llvm.vector.extract.v[[#div(VBITS, 32)]]i32.nxv4i32(<vscale x 4 x i32> [[X_COERCE1:%.*]], i64 0)
 // CHECK-NEXT:   [[ADD:%.*]] = add <[[#div(VBITS, 32)]] x i32> [[Y]], [[X]]
-// CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v[[#div(VBITS, 32)]]i32(<vscale x 4 x i32> undef, <[[#div(VBITS, 32)]] x i32> [[ADD]], i64 0)
+// CHECK-NEXT:   [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v[[#div(VBITS, 32)]]i32(<vscale x 4 x i32> poison, <[[#div(VBITS, 32)]] x i32> [[ADD]], i64 0)
 // CHECK-NEXT:   ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 typedef svint32_t vec __attribute__((arm_sve_vector_bits(N)));
 auto f(vec x, vec y) { return x + y; } // Returns a vec.
diff --git a/clang/test/CodeGen/AArch64/sve-vls-arith-ops.c b/clang/test/CodeGen/AArch64/sve-vls-arith-ops.c
index a4648e9d59dcb..571043bb29e20 100644
--- a/clang/test/CodeGen/AArch64/sve-vls-arith-ops.c
+++ b/clang/test/CodeGen/AArch64/sve-vls-arith-ops.c
@@ -32,7 +32,7 @@ typedef svbool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t add_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -44,7 +44,7 @@ fixed_int8_t add_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t add_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -56,7 +56,7 @@ fixed_int16_t add_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t add_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -68,7 +68,7 @@ fixed_int32_t add_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t add_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -80,7 +80,7 @@ fixed_int64_t add_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t add_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -92,7 +92,7 @@ fixed_uint8_t add_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t add_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -104,7 +104,7 @@ fixed_uint16_t add_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t add_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -116,7 +116,7 @@ fixed_uint32_t add_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t add_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -131,7 +131,7 @@ fixed_uint64_t add_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[ADD]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t add_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -143,7 +143,7 @@ fixed_float16_t add_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t add_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -155,7 +155,7 @@ fixed_float32_t add_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t add_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -167,7 +167,7 @@ fixed_float64_t add_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t add_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -179,7 +179,7 @@ fixed_int8_t add_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t add_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -191,7 +191,7 @@ fixed_int16_t add_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t add_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -203,7 +203,7 @@ fixed_int32_t add_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t add_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -215,7 +215,7 @@ fixed_int64_t add_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t add_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -227,7 +227,7 @@ fixed_uint8_t add_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t add_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -239,7 +239,7 @@ fixed_uint16_t add_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t add_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -251,7 +251,7 @@ fixed_uint32_t add_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t add_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -266,7 +266,7 @@ fixed_uint64_t add_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[A]] to <32 x float>
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <32 x float> [[CONV2]], [[CONV]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[ADD]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t add_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -278,7 +278,7 @@ fixed_float16_t add_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t add_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -290,7 +290,7 @@ fixed_float32_t add_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t add_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -303,7 +303,7 @@ fixed_float64_t add_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t add_scalar_i8(fixed_int8_t a, int8_t b) {
@@ -316,7 +316,7 @@ fixed_int8_t add_scalar_i8(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t add_scalar_i16(fixed_int16_t a, int16_t b) {
@@ -329,7 +329,7 @@ fixed_int16_t add_scalar_i16(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t add_scalar_i32(fixed_int32_t a, int32_t b) {
@@ -342,7 +342,7 @@ fixed_int32_t add_scalar_i32(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t add_scalar_i64(fixed_int64_t a, int64_t b) {
@@ -355,7 +355,7 @@ fixed_int64_t add_scalar_i64(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t add_scalar_u8(fixed_uint8_t a, uint8_t b) {
@@ -368,7 +368,7 @@ fixed_uint8_t add_scalar_u8(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t add_scalar_u16(fixed_uint16_t a, uint16_t b) {
@@ -381,7 +381,7 @@ fixed_uint16_t add_scalar_u16(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t add_scalar_u32(fixed_uint32_t a, uint32_t b) {
@@ -394,7 +394,7 @@ fixed_uint32_t add_scalar_u32(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t add_scalar_u64(fixed_uint64_t a, uint64_t b) {
@@ -407,7 +407,7 @@ fixed_uint64_t add_scalar_u64(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x half> poison, half [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x half> [[SPLAT_SPLATINSERT]], <32 x half> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <32 x half> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t add_scalar_f16(fixed_float16_t a, __fp16 b) {
@@ -420,7 +420,7 @@ fixed_float16_t add_scalar_f16(fixed_float16_t a, __fp16 b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x float> [[SPLAT_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <16 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t add_scalar_f32(fixed_float32_t a, float b) {
@@ -433,7 +433,7 @@ fixed_float32_t add_scalar_f32(fixed_float32_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x double> [[SPLAT_SPLATINSERT]], <8 x double> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t add_scalar_f64(fixed_float64_t a, double b) {
@@ -447,7 +447,7 @@ fixed_float64_t add_scalar_f64(fixed_float64_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t sub_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -459,7 +459,7 @@ fixed_int8_t sub_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t sub_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -471,7 +471,7 @@ fixed_int16_t sub_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t sub_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -483,7 +483,7 @@ fixed_int32_t sub_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t sub_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -495,7 +495,7 @@ fixed_int64_t sub_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t sub_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -507,7 +507,7 @@ fixed_uint8_t sub_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t sub_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -519,7 +519,7 @@ fixed_uint16_t sub_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t sub_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -531,7 +531,7 @@ fixed_uint32_t sub_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t sub_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -546,7 +546,7 @@ fixed_uint64_t sub_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[SUB]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t sub_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -558,7 +558,7 @@ fixed_float16_t sub_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t sub_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -570,7 +570,7 @@ fixed_float32_t sub_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t sub_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -582,7 +582,7 @@ fixed_float64_t sub_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t sub_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -594,7 +594,7 @@ fixed_int8_t sub_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t sub_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -606,7 +606,7 @@ fixed_int16_t sub_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t sub_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -618,7 +618,7 @@ fixed_int32_t sub_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t sub_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -630,7 +630,7 @@ fixed_int64_t sub_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t sub_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -642,7 +642,7 @@ fixed_uint8_t sub_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t sub_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -654,7 +654,7 @@ fixed_uint16_t sub_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t sub_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -666,7 +666,7 @@ fixed_uint32_t sub_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t sub_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -681,7 +681,7 @@ fixed_uint64_t sub_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[SUB]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t sub_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -693,7 +693,7 @@ fixed_float16_t sub_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t sub_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -705,7 +705,7 @@ fixed_float32_t sub_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t sub_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -718,7 +718,7 @@ fixed_float64_t sub_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t sub_scalar_i8(fixed_int8_t a, int8_t b) {
@@ -731,7 +731,7 @@ fixed_int8_t sub_scalar_i8(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t sub_scalar_i16(fixed_int16_t a, int16_t b) {
@@ -744,7 +744,7 @@ fixed_int16_t sub_scalar_i16(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t sub_scalar_i32(fixed_int32_t a, int32_t b) {
@@ -757,7 +757,7 @@ fixed_int32_t sub_scalar_i32(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t sub_scalar_i64(fixed_int64_t a, int64_t b) {
@@ -770,7 +770,7 @@ fixed_int64_t sub_scalar_i64(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t sub_scalar_u8(fixed_uint8_t a, uint8_t b) {
@@ -783,7 +783,7 @@ fixed_uint8_t sub_scalar_u8(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t sub_scalar_u16(fixed_uint16_t a, uint16_t b) {
@@ -796,7 +796,7 @@ fixed_uint16_t sub_scalar_u16(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t sub_scalar_u32(fixed_uint32_t a, uint32_t b) {
@@ -809,7 +809,7 @@ fixed_uint32_t sub_scalar_u32(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t sub_scalar_u64(fixed_uint64_t a, uint64_t b) {
@@ -822,7 +822,7 @@ fixed_uint64_t sub_scalar_u64(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x half> poison, half [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x half> [[SPLAT_SPLATINSERT]], <32 x half> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <32 x half> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t sub_scalar_f16(fixed_float16_t a, __fp16 b) {
@@ -835,7 +835,7 @@ fixed_float16_t sub_scalar_f16(fixed_float16_t a, __fp16 b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x float> [[SPLAT_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <16 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t sub_scalar_f32(fixed_float32_t a, float b) {
@@ -848,7 +848,7 @@ fixed_float32_t sub_scalar_f32(fixed_float32_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x double> [[SPLAT_SPLATINSERT]], <8 x double> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t sub_scalar_f64(fixed_float64_t a, double b) {
@@ -862,7 +862,7 @@ fixed_float64_t sub_scalar_f64(fixed_float64_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t mul_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -874,7 +874,7 @@ fixed_int8_t mul_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t mul_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -886,7 +886,7 @@ fixed_int16_t mul_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t mul_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -898,7 +898,7 @@ fixed_int32_t mul_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t mul_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -910,7 +910,7 @@ fixed_int64_t mul_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t mul_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -922,7 +922,7 @@ fixed_uint8_t mul_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t mul_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -934,7 +934,7 @@ fixed_uint16_t mul_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t mul_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -946,7 +946,7 @@ fixed_uint32_t mul_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t mul_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -961,7 +961,7 @@ fixed_uint64_t mul_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[MUL]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t mul_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -973,7 +973,7 @@ fixed_float16_t mul_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t mul_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -985,7 +985,7 @@ fixed_float32_t mul_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t mul_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -997,7 +997,7 @@ fixed_float64_t mul_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t mul_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -1009,7 +1009,7 @@ fixed_int8_t mul_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t mul_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -1021,7 +1021,7 @@ fixed_int16_t mul_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t mul_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -1033,7 +1033,7 @@ fixed_int32_t mul_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t mul_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -1045,7 +1045,7 @@ fixed_int64_t mul_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t mul_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -1057,7 +1057,7 @@ fixed_uint8_t mul_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t mul_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -1069,7 +1069,7 @@ fixed_uint16_t mul_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t mul_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -1081,7 +1081,7 @@ fixed_uint32_t mul_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t mul_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -1096,7 +1096,7 @@ fixed_uint64_t mul_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[MUL]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t mul_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -1108,7 +1108,7 @@ fixed_float16_t mul_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t mul_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -1120,7 +1120,7 @@ fixed_float32_t mul_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t mul_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -1133,7 +1133,7 @@ fixed_float64_t mul_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t mul_scalar_i8(fixed_int8_t a, int8_t b) {
@@ -1146,7 +1146,7 @@ fixed_int8_t mul_scalar_i8(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t mul_scalar_i16(fixed_int16_t a, int16_t b) {
@@ -1159,7 +1159,7 @@ fixed_int16_t mul_scalar_i16(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t mul_scalar_i32(fixed_int32_t a, int32_t b) {
@@ -1172,7 +1172,7 @@ fixed_int32_t mul_scalar_i32(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t mul_scalar_i64(fixed_int64_t a, int64_t b) {
@@ -1185,7 +1185,7 @@ fixed_int64_t mul_scalar_i64(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t mul_scalar_u8(fixed_uint8_t a, uint8_t b) {
@@ -1198,7 +1198,7 @@ fixed_uint8_t mul_scalar_u8(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t mul_scalar_u16(fixed_uint16_t a, uint16_t b) {
@@ -1211,7 +1211,7 @@ fixed_uint16_t mul_scalar_u16(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t mul_scalar_u32(fixed_uint32_t a, uint32_t b) {
@@ -1224,7 +1224,7 @@ fixed_uint32_t mul_scalar_u32(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t mul_scalar_u64(fixed_uint64_t a, uint64_t b) {
@@ -1237,7 +1237,7 @@ fixed_uint64_t mul_scalar_u64(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x half> poison, half [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x half> [[SPLAT_SPLATINSERT]], <32 x half> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <32 x half> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t mul_scalar_f16(fixed_float16_t a, __fp16 b) {
@@ -1250,7 +1250,7 @@ fixed_float16_t mul_scalar_f16(fixed_float16_t a, __fp16 b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x float> [[SPLAT_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <16 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t mul_scalar_f32(fixed_float32_t a, float b) {
@@ -1263,7 +1263,7 @@ fixed_float32_t mul_scalar_f32(fixed_float32_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x double> [[SPLAT_SPLATINSERT]], <8 x double> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t mul_scalar_f64(fixed_float64_t a, double b) {
@@ -1277,7 +1277,7 @@ fixed_float64_t mul_scalar_f64(fixed_float64_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t div_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -1289,7 +1289,7 @@ fixed_int8_t div_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t div_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -1301,7 +1301,7 @@ fixed_int16_t div_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t div_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -1313,7 +1313,7 @@ fixed_int32_t div_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t div_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -1325,7 +1325,7 @@ fixed_int64_t div_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t div_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -1337,7 +1337,7 @@ fixed_uint8_t div_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t div_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -1349,7 +1349,7 @@ fixed_uint16_t div_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t div_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -1361,7 +1361,7 @@ fixed_uint32_t div_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t div_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -1376,7 +1376,7 @@ fixed_uint64_t div_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[DIV]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t div_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -1388,7 +1388,7 @@ fixed_float16_t div_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t div_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -1400,7 +1400,7 @@ fixed_float32_t div_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t div_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -1412,7 +1412,7 @@ fixed_float64_t div_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t div_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -1424,7 +1424,7 @@ fixed_int8_t div_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t div_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -1436,7 +1436,7 @@ fixed_int16_t div_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t div_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -1448,7 +1448,7 @@ fixed_int32_t div_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t div_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -1460,7 +1460,7 @@ fixed_int64_t div_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t div_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -1472,7 +1472,7 @@ fixed_uint8_t div_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t div_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -1484,7 +1484,7 @@ fixed_uint16_t div_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t div_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -1496,7 +1496,7 @@ fixed_uint32_t div_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t div_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -1511,7 +1511,7 @@ fixed_uint64_t div_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CONV2:%.*]] = fpext <32 x half> [[B]] to <32 x float>
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[CONV3:%.*]] = fptrunc <32 x float> [[DIV]] to <32 x half>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t div_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -1523,7 +1523,7 @@ fixed_float16_t div_inplace_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <16 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t div_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -1535,7 +1535,7 @@ fixed_float32_t div_inplace_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t div_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -1548,7 +1548,7 @@ fixed_float64_t div_inplace_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t div_scalar_i8(fixed_int8_t a, int8_t b) {
@@ -1561,7 +1561,7 @@ fixed_int8_t div_scalar_i8(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t div_scalar_i16(fixed_int16_t a, int16_t b) {
@@ -1574,7 +1574,7 @@ fixed_int16_t div_scalar_i16(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t div_scalar_i32(fixed_int32_t a, int32_t b) {
@@ -1587,7 +1587,7 @@ fixed_int32_t div_scalar_i32(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t div_scalar_i64(fixed_int64_t a, int64_t b) {
@@ -1600,7 +1600,7 @@ fixed_int64_t div_scalar_i64(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t div_scalar_u8(fixed_uint8_t a, uint8_t b) {
@@ -1613,7 +1613,7 @@ fixed_uint8_t div_scalar_u8(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t div_scalar_u16(fixed_uint16_t a, uint16_t b) {
@@ -1626,7 +1626,7 @@ fixed_uint16_t div_scalar_u16(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t div_scalar_u32(fixed_uint32_t a, uint32_t b) {
@@ -1639,7 +1639,7 @@ fixed_uint32_t div_scalar_u32(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t div_scalar_u64(fixed_uint64_t a, uint64_t b) {
@@ -1652,7 +1652,7 @@ fixed_uint64_t div_scalar_u64(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x half> poison, half [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x half> [[SPLAT_SPLATINSERT]], <32 x half> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <32 x half> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> undef, <32 x half> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v32f16(<vscale x 8 x half> poison, <32 x half> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[CASTSCALABLESVE]]
 //
 fixed_float16_t div_scalar_f16(fixed_float16_t a, __fp16 b) {
@@ -1665,7 +1665,7 @@ fixed_float16_t div_scalar_f16(fixed_float16_t a, __fp16 b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x float> [[SPLAT_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <16 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> undef, <16 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[CASTSCALABLESVE]]
 //
 fixed_float32_t div_scalar_f32(fixed_float32_t a, float b) {
@@ -1678,7 +1678,7 @@ fixed_float32_t div_scalar_f32(fixed_float32_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x double> [[SPLAT_SPLATINSERT]], <8 x double> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
 //
 fixed_float64_t div_scalar_f64(fixed_float64_t a, double b) {
@@ -1692,7 +1692,7 @@ fixed_float64_t div_scalar_f64(fixed_float64_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rem_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -1704,7 +1704,7 @@ fixed_int8_t rem_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rem_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -1716,7 +1716,7 @@ fixed_int16_t rem_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rem_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -1728,7 +1728,7 @@ fixed_int32_t rem_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rem_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -1740,7 +1740,7 @@ fixed_int64_t rem_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rem_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -1752,7 +1752,7 @@ fixed_uint8_t rem_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rem_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -1764,7 +1764,7 @@ fixed_uint16_t rem_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rem_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -1776,7 +1776,7 @@ fixed_uint32_t rem_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rem_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -1788,7 +1788,7 @@ fixed_uint64_t rem_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rem_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -1800,7 +1800,7 @@ fixed_int8_t rem_inplace_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rem_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -1812,7 +1812,7 @@ fixed_int16_t rem_inplace_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rem_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -1824,7 +1824,7 @@ fixed_int32_t rem_inplace_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rem_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -1836,7 +1836,7 @@ fixed_int64_t rem_inplace_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rem_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -1848,7 +1848,7 @@ fixed_uint8_t rem_inplace_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rem_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -1860,7 +1860,7 @@ fixed_uint16_t rem_inplace_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rem_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -1872,7 +1872,7 @@ fixed_uint32_t rem_inplace_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rem_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -1885,7 +1885,7 @@ fixed_uint64_t rem_inplace_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rem_scalar_i8(fixed_int8_t a, int8_t b) {
@@ -1898,7 +1898,7 @@ fixed_int8_t rem_scalar_i8(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rem_scalar_i16(fixed_int16_t a, int16_t b) {
@@ -1911,7 +1911,7 @@ fixed_int16_t rem_scalar_i16(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rem_scalar_i32(fixed_int32_t a, int32_t b) {
@@ -1924,7 +1924,7 @@ fixed_int32_t rem_scalar_i32(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rem_scalar_i64(fixed_int64_t a, int64_t b) {
@@ -1937,7 +1937,7 @@ fixed_int64_t rem_scalar_i64(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <64 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rem_scalar_u8(fixed_uint8_t a, uint8_t b) {
@@ -1950,7 +1950,7 @@ fixed_uint8_t rem_scalar_u8(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rem_scalar_u16(fixed_uint16_t a, uint16_t b) {
@@ -1963,7 +1963,7 @@ fixed_uint16_t rem_scalar_u16(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rem_scalar_u32(fixed_uint32_t a, uint32_t b) {
@@ -1976,7 +1976,7 @@ fixed_uint32_t rem_scalar_u32(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rem_scalar_u64(fixed_uint64_t a, uint64_t b) {
diff --git a/clang/test/CodeGen/AArch64/sve-vls-bitwise-ops.c b/clang/test/CodeGen/AArch64/sve-vls-bitwise-ops.c
index 017c71aa41143..33c7a2d1bb630 100644
--- a/clang/test/CodeGen/AArch64/sve-vls-bitwise-ops.c
+++ b/clang/test/CodeGen/AArch64/sve-vls-bitwise-ops.c
@@ -34,7 +34,7 @@ typedef svbool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:    [[B_COERCE:%.*]] = bitcast <vscale x 16 x i1> [[TMP1:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <8 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[AND]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -47,7 +47,7 @@ fixed_bool_t and_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8_t and_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -59,7 +59,7 @@ fixed_int8_t and_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16_t and_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -71,7 +71,7 @@ fixed_int16_t and_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32_t and_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -83,7 +83,7 @@ fixed_int32_t and_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64_t and_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -95,7 +95,7 @@ fixed_int64_t and_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8_t and_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -107,7 +107,7 @@ fixed_uint8_t and_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16_t and_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -119,7 +119,7 @@ fixed_uint16_t and_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32_t and_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -131,7 +131,7 @@ fixed_uint32_t and_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64_t and_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -147,7 +147,7 @@ fixed_uint64_t and_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[B_COERCE:%.*]] = bitcast <vscale x 16 x i1> [[TMP1:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <8 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[OR]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -160,7 +160,7 @@ fixed_bool_t or_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8_t or_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -172,7 +172,7 @@ fixed_int8_t or_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16_t or_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -184,7 +184,7 @@ fixed_int16_t or_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32_t or_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -196,7 +196,7 @@ fixed_int32_t or_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64_t or_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -208,7 +208,7 @@ fixed_int64_t or_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8_t or_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -220,7 +220,7 @@ fixed_uint8_t or_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16_t or_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -232,7 +232,7 @@ fixed_uint16_t or_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32_t or_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -244,7 +244,7 @@ fixed_uint32_t or_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64_t or_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -260,7 +260,7 @@ fixed_uint64_t or_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[B_COERCE:%.*]] = bitcast <vscale x 16 x i1> [[TMP1:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[XOR]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -273,7 +273,7 @@ fixed_bool_t xor_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8_t xor_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -285,7 +285,7 @@ fixed_int8_t xor_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16_t xor_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -297,7 +297,7 @@ fixed_int16_t xor_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32_t xor_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -309,7 +309,7 @@ fixed_int32_t xor_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64_t xor_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -321,7 +321,7 @@ fixed_int64_t xor_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8_t xor_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -333,7 +333,7 @@ fixed_uint8_t xor_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16_t xor_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -345,7 +345,7 @@ fixed_uint16_t xor_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32_t xor_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -357,7 +357,7 @@ fixed_uint32_t xor_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64_t xor_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -371,7 +371,7 @@ fixed_uint64_t xor_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[A_COERCE:%.*]] = bitcast <vscale x 16 x i1> [[TMP0:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[A_COERCE]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i8> [[A]], splat (i8 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[NOT]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
@@ -383,7 +383,7 @@ fixed_bool_t neg_bool(fixed_bool_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <64 x i8> [[A]], splat (i8 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8_t neg_i8(fixed_int8_t a) {
@@ -394,7 +394,7 @@ fixed_int8_t neg_i8(fixed_int8_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i16> [[A]], splat (i16 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16_t neg_i16(fixed_int16_t a) {
@@ -405,7 +405,7 @@ fixed_int16_t neg_i16(fixed_int16_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i32> [[A]], splat (i32 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32_t neg_i32(fixed_int32_t a) {
@@ -416,7 +416,7 @@ fixed_int32_t neg_i32(fixed_int32_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i64> [[A]], splat (i64 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64_t neg_i64(fixed_int64_t a) {
@@ -427,7 +427,7 @@ fixed_int64_t neg_i64(fixed_int64_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <64 x i8> [[A]], splat (i8 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8_t neg_u8(fixed_uint8_t a) {
@@ -438,7 +438,7 @@ fixed_uint8_t neg_u8(fixed_uint8_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i16> [[A]], splat (i16 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16_t neg_u16(fixed_uint16_t a) {
@@ -449,7 +449,7 @@ fixed_uint16_t neg_u16(fixed_uint16_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i32> [[A]], splat (i32 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32_t neg_u32(fixed_uint32_t a) {
@@ -460,7 +460,7 @@ fixed_uint32_t neg_u32(fixed_uint32_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i64> [[A]], splat (i64 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64_t neg_u64(fixed_uint64_t a) {
diff --git a/clang/test/CodeGen/AArch64/sve-vls-compare-ops.c b/clang/test/CodeGen/AArch64/sve-vls-compare-ops.c
index fe81adfdb0826..3216ea859fbc8 100644
--- a/clang/test/CodeGen/AArch64/sve-vls-compare-ops.c
+++ b/clang/test/CodeGen/AArch64/sve-vls-compare-ops.c
@@ -35,7 +35,7 @@ typedef svbool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -49,7 +49,7 @@ fixed_bool_t eq_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t eq_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -62,7 +62,7 @@ fixed_int8_t eq_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t eq_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -75,7 +75,7 @@ fixed_int16_t eq_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t eq_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -88,7 +88,7 @@ fixed_int32_t eq_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t eq_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -101,7 +101,7 @@ fixed_int64_t eq_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t eq_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -114,7 +114,7 @@ fixed_int8_t eq_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t eq_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -127,7 +127,7 @@ fixed_int16_t eq_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t eq_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -140,7 +140,7 @@ fixed_int32_t eq_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t eq_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -156,7 +156,7 @@ fixed_int64_t eq_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t eq_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -169,7 +169,7 @@ fixed_int16_t eq_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t eq_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -182,7 +182,7 @@ fixed_int32_t eq_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t eq_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -199,7 +199,7 @@ fixed_int64_t eq_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -213,7 +213,7 @@ fixed_bool_t neq_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t neq_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -226,7 +226,7 @@ fixed_int8_t neq_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t neq_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -239,7 +239,7 @@ fixed_int16_t neq_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t neq_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -252,7 +252,7 @@ fixed_int32_t neq_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t neq_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -265,7 +265,7 @@ fixed_int64_t neq_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t neq_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -278,7 +278,7 @@ fixed_int8_t neq_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t neq_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -291,7 +291,7 @@ fixed_int16_t neq_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t neq_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -304,7 +304,7 @@ fixed_int32_t neq_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t neq_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -320,7 +320,7 @@ fixed_int64_t neq_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp une <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t neq_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -333,7 +333,7 @@ fixed_int16_t neq_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp une <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t neq_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -346,7 +346,7 @@ fixed_int32_t neq_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp une <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t neq_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -363,7 +363,7 @@ fixed_int64_t neq_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -377,7 +377,7 @@ fixed_bool_t lt_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t lt_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -390,7 +390,7 @@ fixed_int8_t lt_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lt_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -403,7 +403,7 @@ fixed_int16_t lt_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lt_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -416,7 +416,7 @@ fixed_int32_t lt_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lt_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -429,7 +429,7 @@ fixed_int64_t lt_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t lt_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -442,7 +442,7 @@ fixed_int8_t lt_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lt_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -455,7 +455,7 @@ fixed_int16_t lt_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lt_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -468,7 +468,7 @@ fixed_int32_t lt_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lt_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -484,7 +484,7 @@ fixed_int64_t lt_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lt_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -497,7 +497,7 @@ fixed_int16_t lt_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lt_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -510,7 +510,7 @@ fixed_int32_t lt_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lt_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -527,7 +527,7 @@ fixed_int64_t lt_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -541,7 +541,7 @@ fixed_bool_t leq_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t leq_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -554,7 +554,7 @@ fixed_int8_t leq_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t leq_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -567,7 +567,7 @@ fixed_int16_t leq_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t leq_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -580,7 +580,7 @@ fixed_int32_t leq_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t leq_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -593,7 +593,7 @@ fixed_int64_t leq_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t leq_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -606,7 +606,7 @@ fixed_int8_t leq_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t leq_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -619,7 +619,7 @@ fixed_int16_t leq_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t leq_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -632,7 +632,7 @@ fixed_int32_t leq_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t leq_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -648,7 +648,7 @@ fixed_int64_t leq_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t leq_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -661,7 +661,7 @@ fixed_int16_t leq_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t leq_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -674,7 +674,7 @@ fixed_int32_t leq_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t leq_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -691,7 +691,7 @@ fixed_int64_t leq_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -705,7 +705,7 @@ fixed_bool_t gt_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t gt_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -718,7 +718,7 @@ fixed_int8_t gt_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t gt_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -731,7 +731,7 @@ fixed_int16_t gt_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t gt_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -744,7 +744,7 @@ fixed_int32_t gt_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t gt_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -757,7 +757,7 @@ fixed_int64_t gt_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t gt_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -770,7 +770,7 @@ fixed_int8_t gt_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t gt_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -783,7 +783,7 @@ fixed_int16_t gt_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t gt_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -796,7 +796,7 @@ fixed_int32_t gt_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t gt_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -812,7 +812,7 @@ fixed_int64_t gt_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t gt_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -825,7 +825,7 @@ fixed_int16_t gt_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t gt_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -838,7 +838,7 @@ fixed_int32_t gt_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t gt_f64(fixed_float64_t a, fixed_float64_t b) {
@@ -855,7 +855,7 @@ fixed_int64_t gt_f64(fixed_float64_t a, fixed_float64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[B_COERCE]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <8 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
 //
@@ -869,7 +869,7 @@ fixed_bool_t geq_bool(fixed_bool_t a, fixed_bool_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t geq_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -882,7 +882,7 @@ fixed_int8_t geq_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t geq_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -895,7 +895,7 @@ fixed_int16_t geq_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t geq_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -908,7 +908,7 @@ fixed_int32_t geq_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t geq_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -921,7 +921,7 @@ fixed_int64_t geq_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <64 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <64 x i1> [[CMP]] to <64 x i8>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t geq_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -934,7 +934,7 @@ fixed_int8_t geq_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <32 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t geq_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -947,7 +947,7 @@ fixed_int16_t geq_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <16 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t geq_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -960,7 +960,7 @@ fixed_int32_t geq_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <8 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t geq_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -976,7 +976,7 @@ fixed_int64_t geq_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <32 x float> [[CONV]], [[CONV2]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i32>
 // CHECK-NEXT:    [[CONV3:%.*]] = trunc <32 x i32> [[SEXT]] to <32 x i16>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[CONV3]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[CONV3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t geq_f16(fixed_float16_t a, fixed_float16_t b) {
@@ -989,7 +989,7 @@ fixed_int16_t geq_f16(fixed_float16_t a, fixed_float16_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x float> @llvm.vector.extract.v16f32.nxv4f32(<vscale x 4 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <16 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i32>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t geq_f32(fixed_float32_t a, fixed_float32_t b) {
@@ -1002,7 +1002,7 @@ fixed_int32_t geq_f32(fixed_float32_t a, fixed_float32_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <8 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i64>
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t geq_f64(fixed_float64_t a, fixed_float64_t b) {
diff --git a/clang/test/CodeGen/AArch64/sve-vls-shift-ops.c b/clang/test/CodeGen/AArch64/sve-vls-shift-ops.c
index 05b8f3535df70..109cbb9b47a8e 100644
--- a/clang/test/CodeGen/AArch64/sve-vls-shift-ops.c
+++ b/clang/test/CodeGen/AArch64/sve-vls-shift-ops.c
@@ -30,7 +30,7 @@ typedef svbool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t lshift_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -42,7 +42,7 @@ fixed_int8_t lshift_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rshift_i8(fixed_int8_t a, fixed_int8_t b) {
@@ -54,7 +54,7 @@ fixed_int8_t rshift_i8(fixed_int8_t a, fixed_int8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t lshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -66,7 +66,7 @@ fixed_uint8_t lshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
@@ -78,7 +78,7 @@ fixed_uint8_t rshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lshift_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -90,7 +90,7 @@ fixed_int16_t lshift_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rshift_i16(fixed_int16_t a, fixed_int16_t b) {
@@ -102,7 +102,7 @@ fixed_int16_t rshift_i16(fixed_int16_t a, fixed_int16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t lshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -114,7 +114,7 @@ fixed_uint16_t lshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
@@ -126,7 +126,7 @@ fixed_uint16_t rshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lshift_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -138,7 +138,7 @@ fixed_int32_t lshift_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rshift_i32(fixed_int32_t a, fixed_int32_t b) {
@@ -150,7 +150,7 @@ fixed_int32_t rshift_i32(fixed_int32_t a, fixed_int32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t lshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -162,7 +162,7 @@ fixed_uint32_t lshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
@@ -174,7 +174,7 @@ fixed_uint32_t rshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lshift_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -186,7 +186,7 @@ fixed_int64_t lshift_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rshift_i64(fixed_int64_t a, fixed_int64_t b) {
@@ -198,7 +198,7 @@ fixed_int64_t rshift_i64(fixed_int64_t a, fixed_int64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t lshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -210,7 +210,7 @@ fixed_uint64_t lshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
@@ -225,7 +225,7 @@ fixed_uint64_t rshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t lshift_i8_rsplat(fixed_int8_t a, int8_t b) {
@@ -238,7 +238,7 @@ fixed_int8_t lshift_i8_rsplat(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t lshift_i8_lsplat(fixed_int8_t a, int8_t b) {
@@ -253,7 +253,7 @@ fixed_int8_t lshift_i8_lsplat(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rshift_i8_rsplat(fixed_int8_t a, int8_t b) {
@@ -266,7 +266,7 @@ fixed_int8_t rshift_i8_rsplat(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_int8_t rshift_i8_lsplat(fixed_int8_t a, int8_t b) {
@@ -281,7 +281,7 @@ fixed_int8_t rshift_i8_lsplat(fixed_int8_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t lshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
@@ -294,7 +294,7 @@ fixed_uint8_t lshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t lshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
@@ -309,7 +309,7 @@ fixed_uint8_t lshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
@@ -322,7 +322,7 @@ fixed_uint8_t rshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> poison, <64 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 //
 fixed_uint8_t rshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
@@ -337,7 +337,7 @@ fixed_uint8_t rshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lshift_i16_rsplat(fixed_int16_t a, int16_t b) {
@@ -350,7 +350,7 @@ fixed_int16_t lshift_i16_rsplat(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t lshift_i16_lsplat(fixed_int16_t a, int16_t b) {
@@ -365,7 +365,7 @@ fixed_int16_t lshift_i16_lsplat(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rshift_i16_rsplat(fixed_int16_t a, int16_t b) {
@@ -378,7 +378,7 @@ fixed_int16_t rshift_i16_rsplat(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_int16_t rshift_i16_lsplat(fixed_int16_t a, int16_t b) {
@@ -393,7 +393,7 @@ fixed_int16_t rshift_i16_lsplat(fixed_int16_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t lshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
@@ -406,7 +406,7 @@ fixed_uint16_t lshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t lshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
@@ -421,7 +421,7 @@ fixed_uint16_t lshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
@@ -434,7 +434,7 @@ fixed_uint16_t rshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> poison, <32 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 //
 fixed_uint16_t rshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
@@ -447,7 +447,7 @@ fixed_uint16_t rshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lshift_i32_rsplat(fixed_int32_t a, int32_t b) {
@@ -460,7 +460,7 @@ fixed_int32_t lshift_i32_rsplat(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t lshift_i32_lsplat(fixed_int32_t a, int32_t b) {
@@ -473,7 +473,7 @@ fixed_int32_t lshift_i32_lsplat(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rshift_i32_rsplat(fixed_int32_t a, int32_t b) {
@@ -486,7 +486,7 @@ fixed_int32_t rshift_i32_rsplat(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t rshift_i32_lsplat(fixed_int32_t a, int32_t b) {
@@ -499,7 +499,7 @@ fixed_int32_t rshift_i32_lsplat(fixed_int32_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t lshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
@@ -512,7 +512,7 @@ fixed_uint32_t lshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t lshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
@@ -525,7 +525,7 @@ fixed_uint32_t lshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
@@ -538,7 +538,7 @@ fixed_uint32_t rshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_uint32_t rshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
@@ -551,7 +551,7 @@ fixed_uint32_t rshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lshift_i64_rsplat(fixed_int64_t a, int64_t b) {
@@ -564,7 +564,7 @@ fixed_int64_t lshift_i64_rsplat(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t lshift_i64_lsplat(fixed_int64_t a, int64_t b) {
@@ -577,7 +577,7 @@ fixed_int64_t lshift_i64_lsplat(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rshift_i64_rsplat(fixed_int64_t a, int64_t b) {
@@ -590,7 +590,7 @@ fixed_int64_t rshift_i64_rsplat(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_int64_t rshift_i64_lsplat(fixed_int64_t a, int64_t b) {
@@ -603,7 +603,7 @@ fixed_int64_t rshift_i64_lsplat(fixed_int64_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t lshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
@@ -616,7 +616,7 @@ fixed_uint64_t lshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t lshift_u64_lsplat(fixed_uint64_t a, uint64_t b) {
@@ -629,7 +629,7 @@ fixed_uint64_t lshift_u64_lsplat(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
@@ -642,7 +642,7 @@ fixed_uint64_t rshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 fixed_uint64_t rshift_u64_lsplat(fixed_uint64_t a, uint64_t b) {
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c b/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
index 5422d993ff157..08ff936a0a797 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
@@ -25,13 +25,13 @@ void test1(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[TMP1]], 0
 // CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[RESP:%.*]], align 16
 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[TMP1]], 1
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[RESP]], i64 16
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[RESP]], i64 16
 // CHECK-NEXT:    store <16 x i8> [[TMP3]], ptr [[TMP4]], align 16
 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[TMP1]], 2
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[RESP]], i64 32
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[RESP]], i64 32
 // CHECK-NEXT:    store <16 x i8> [[TMP5]], ptr [[TMP6]], align 16
 // CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[TMP1]], 3
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[RESP]], i64 48
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[RESP]], i64 48
 // CHECK-NEXT:    store <16 x i8> [[TMP7]], ptr [[TMP8]], align 16
 // CHECK-NEXT:    ret void
 //
@@ -60,7 +60,7 @@ void test3(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[TMP1]], 0
 // CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[RESP:%.*]], align 16
 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[TMP1]], 1
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[RESP]], i64 16
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[RESP]], i64 16
 // CHECK-NEXT:    store <16 x i8> [[TMP3]], ptr [[TMP4]], align 16
 // CHECK-NEXT:    ret void
 //
@@ -1072,7 +1072,7 @@ void test76(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[TMP1]], 0
 // CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[RESP:%.*]], align 16
 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[TMP1]], 1
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[RESP]], i64 16
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[RESP]], i64 16
 // CHECK-NEXT:    store <16 x i8> [[TMP3]], ptr [[TMP4]], align 16
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c
index e684513bb1442..ecde52eb3d762 100644
--- a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c
+++ b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c
@@ -71,21 +71,21 @@ DEFINE_STRUCT(bool64)
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 8
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> poison, <1 x i64> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-128-LABEL: @read_int64m1(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v2i64(<vscale x 1 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v2i64(<vscale x 1 x i64> poison, <2 x i64> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_int64m1(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 vint64m1_t read_int64m1(struct struct_int64m1 *s) {
@@ -125,21 +125,21 @@ void write_int64m1(struct struct_int64m1 *s, vint64m1_t x) {
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 8
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x double>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v1f64(<vscale x 1 x double> undef, <1 x double> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v1f64(<vscale x 1 x double> poison, <1 x double> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 // CHECK-128-LABEL: @read_float64m1(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v2f64(<vscale x 1 x double> undef, <2 x double> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v2f64(<vscale x 1 x double> poison, <2 x double> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_float64m1(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 vfloat64m1_t read_float64m1(struct struct_float64m1 *s) {
@@ -179,7 +179,7 @@ void write_float64m1(struct struct_float64m1 *s, vfloat64m1_t x) {
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 8
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v8i8(<vscale x 8 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v8i8(<vscale x 8 x i8> poison, <8 x i8> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-64-NEXT:    ret <vscale x 64 x i1> [[TMP1]]
 //
@@ -187,7 +187,7 @@ void write_float64m1(struct struct_float64m1 *s, vfloat64m1_t x) {
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v16i8(<vscale x 8 x i8> undef, <16 x i8> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v16i8(<vscale x 8 x i8> poison, <16 x i8> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-128-NEXT:    ret <vscale x 64 x i1> [[TMP1]]
 //
@@ -195,7 +195,7 @@ void write_float64m1(struct struct_float64m1 *s, vfloat64m1_t x) {
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr [[Y]], align 8, !tbaa [[TBAA6]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-256-NEXT:    ret <vscale x 64 x i1> [[TMP1]]
 //
diff --git a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c
index 6fd3930981825..7992951346d54 100644
--- a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c
+++ b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c
@@ -120,7 +120,7 @@ vbool32_t to_vbool32_t(fixed_bool32_t type) {
 // CHECK-LABEL: @to_vint32m1_t__from_gnu_int32m1_t(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TYPE:%.*]] = load <8 x i32>, ptr [[TMP0:%.*]], align 32, !tbaa [[TBAA8]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TYPE]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 vint32m1_t to_vint32m1_t__from_gnu_int32m1_t(gnu_int32m1_t type) {
@@ -140,7 +140,7 @@ gnu_int32m1_t from_vint32m1_t__to_gnu_int32m1_t(vint32m1_t type) {
 // CHECK-LABEL: @to_fixed_int32m1_t__from_gnu_int32m1_t(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TYPE:%.*]] = load <8 x i32>, ptr [[TMP0:%.*]], align 32, !tbaa [[TBAA8]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TYPE]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t to_fixed_int32m1_t__from_gnu_int32m1_t(gnu_int32m1_t type) {
diff --git a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c
index a6b3f3732f281..d81855aea2e5e 100644
--- a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c
+++ b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c
@@ -57,19 +57,19 @@ fixed_bool32_t global_bool32;
 // CHECK-NEXT:    store <vscale x 64 x i8> [[VEC:%.*]], ptr [[VEC_ADDR]], align 1
 // CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 64 x i1>, ptr [[M_ADDR]], align 1
 // CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @global_bool1, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[TMP1]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 64 x i1> @llvm.riscv.vmand.nxv64i1.i64(<vscale x 64 x i1> [[TMP0]], <vscale x 64 x i1> [[TMP2]], i64 256)
 // CHECK-NEXT:    store <vscale x 64 x i1> [[TMP3]], ptr [[MASK]], align 1
 // CHECK-NEXT:    [[TMP4:%.*]] = load <vscale x 64 x i1>, ptr [[MASK]], align 1
 // CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 64 x i8>, ptr [[VEC_ADDR]], align 1
 // CHECK-NEXT:    [[TMP6:%.*]] = load <256 x i8>, ptr @global_vec_int8m8, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.v256i8(<vscale x 64 x i8> undef, <256 x i8> [[TMP6]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.v256i8(<vscale x 64 x i8> poison, <256 x i8> [[TMP6]], i64 0)
 // CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vadd.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> poison, <vscale x 64 x i8> [[TMP5]], <vscale x 64 x i8> [[CAST_SCALABLE1]], <vscale x 64 x i1> [[TMP4]], i64 256, i64 3)
 // CHECK-NEXT:    [[CAST_FIXED:%.*]] = call <256 x i8> @llvm.vector.extract.v256i8.nxv64i8(<vscale x 64 x i8> [[TMP7]], i64 0)
 // CHECK-NEXT:    store <256 x i8> [[CAST_FIXED]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = load <256 x i8>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE2:%.*]] = call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.v256i8(<vscale x 64 x i8> undef, <256 x i8> [[TMP8]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE2:%.*]] = call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.v256i8(<vscale x 64 x i8> poison, <256 x i8> [[TMP8]], i64 0)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[CAST_SCALABLE2]]
 //
 fixed_int8m8_t test_bool1(vbool1_t m, vint8m8_t vec) {
@@ -87,19 +87,19 @@ fixed_int8m8_t test_bool1(vbool1_t m, vint8m8_t vec) {
 // CHECK-NEXT:    store <vscale x 16 x i16> [[VEC:%.*]], ptr [[VEC_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 16 x i1>, ptr [[M_ADDR]], align 1
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @global_bool4, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP1]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i1> @llvm.riscv.vmand.nxv16i1.i64(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i1> [[TMP2]], i64 64)
 // CHECK-NEXT:    store <vscale x 16 x i1> [[TMP3]], ptr [[MASK]], align 1
 // CHECK-NEXT:    [[TMP4:%.*]] = load <vscale x 16 x i1>, ptr [[MASK]], align 1
 // CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 16 x i16>, ptr [[VEC_ADDR]], align 2
 // CHECK-NEXT:    [[TMP6:%.*]] = load <64 x i16>, ptr @global_vec_int16m4, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.v64i16(<vscale x 16 x i16> undef, <64 x i16> [[TMP6]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.v64i16(<vscale x 16 x i16> poison, <64 x i16> [[TMP6]], i64 0)
 // CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vadd.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> poison, <vscale x 16 x i16> [[TMP5]], <vscale x 16 x i16> [[CAST_SCALABLE1]], <vscale x 16 x i1> [[TMP4]], i64 64, i64 3)
 // CHECK-NEXT:    [[CAST_FIXED:%.*]] = call <64 x i16> @llvm.vector.extract.v64i16.nxv16i16(<vscale x 16 x i16> [[TMP7]], i64 0)
 // CHECK-NEXT:    store <64 x i16> [[CAST_FIXED]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = load <64 x i16>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE2:%.*]] = call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.v64i16(<vscale x 16 x i16> undef, <64 x i16> [[TMP8]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE2:%.*]] = call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.v64i16(<vscale x 16 x i16> poison, <64 x i16> [[TMP8]], i64 0)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[CAST_SCALABLE2]]
 //
 fixed_int16m4_t test_bool4(vbool4_t m, vint16m4_t vec) {
@@ -125,12 +125,12 @@ fixed_int16m4_t test_bool4(vbool4_t m, vint16m4_t vec) {
 // CHECK-NEXT:    [[TMP4:%.*]] = load <vscale x 2 x i1>, ptr [[MASK]], align 1
 // CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 2 x i32>, ptr [[VEC_ADDR]], align 4
 // CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @global_vec, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP6]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP6]], i64 0)
 // CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vadd.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[TMP5]], <vscale x 2 x i32> [[CAST_SCALABLE]], <vscale x 2 x i1> [[TMP4]], i64 8, i64 3)
 // CHECK-NEXT:    [[CAST_FIXED:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TMP7]], i64 0)
 // CHECK-NEXT:    store <8 x i32> [[CAST_FIXED]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP8]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP8]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE1]]
 //
 fixed_int32m1_t test_bool32(vbool32_t m, vint32m1_t vec) {
@@ -147,7 +147,7 @@ fixed_int32m1_t test_bool32(vbool32_t m, vint32m1_t vec) {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr [[TMP0]], align 8
 // CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t test_ptr_to_global() {
@@ -168,7 +168,7 @@ fixed_int32m1_t test_ptr_to_global() {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 8
 // CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t array_arg(fixed_int32m1_t arr[]) {
@@ -186,7 +186,7 @@ fixed_int32m1_t array_arg(fixed_int32m1_t arr[]) {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr [[TMP0]], align 8
 // CHECK-NEXT:    store <32 x i8> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[TMP2]], i64 0)
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-NEXT:    ret <vscale x 64 x i1> [[TMP3]]
 //
@@ -208,7 +208,7 @@ fixed_bool1_t address_of_array_idx_bool1() {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[TMP0]], align 8
 // CHECK-NEXT:    store <8 x i8> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP2]], i64 0)
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP3]]
 //
@@ -247,13 +247,13 @@ fixed_bool32_t address_of_array_idx_bool32() {
 // CHECK-NEXT:    [[VEC_ADDR:%.*]] = alloca <vscale x 2 x i32>, align 4
 // CHECK-NEXT:    store <vscale x 2 x i32> [[VEC:%.*]], ptr [[VEC_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr @global_vec, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP0]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP0]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 2 x i32>, ptr [[VEC_ADDR]], align 4
 // CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[CAST_SCALABLE]], <vscale x 2 x i32> [[TMP1]], i64 8)
 // CHECK-NEXT:    [[CAST_FIXED:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    store <8 x i32> [[CAST_FIXED]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[TMP3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE1]]
 //
 fixed_int32m1_t test_cast(vint32m1_t vec) {
@@ -269,7 +269,7 @@ fixed_int32m1_t test_cast(vint32m1_t vec) {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr [[TMP0]], align 8
 // CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m2_t test_ptr_to_global_m2() {
@@ -290,7 +290,7 @@ fixed_int32m2_t test_ptr_to_global_m2() {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 8
 // CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m2_t array_arg_m2(fixed_int32m2_t arr[]) {
@@ -303,13 +303,13 @@ fixed_int32m2_t array_arg_m2(fixed_int32m2_t arr[]) {
 // CHECK-NEXT:    [[VEC_ADDR:%.*]] = alloca <vscale x 4 x i32>, align 4
 // CHECK-NEXT:    store <vscale x 4 x i32> [[VEC:%.*]], ptr [[VEC_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr @global_vec_m2, align 8
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP0]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP0]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 4 x i32>, ptr [[VEC_ADDR]], align 4
 // CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> poison, <vscale x 4 x i32> [[CAST_SCALABLE]], <vscale x 4 x i32> [[TMP1]], i64 16)
 // CHECK-NEXT:    [[CAST_FIXED:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    store <16 x i32> [[CAST_FIXED]], ptr [[RETVAL]], align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 8
-// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP3]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE1]]
 //
 fixed_int32m2_t test_cast_m2(vint32m2_t vec) {
diff --git a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c
index 663e436b4dab6..4bd6311e05b03 100644
--- a/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c
+++ b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c
@@ -105,13 +105,13 @@ void write_global_bool32(vbool32_t v) { global_bool32 = v; }
 // CHECK-64-LABEL: @read_global_i64(
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr @global_i64, align 8, !tbaa [[TBAA6]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> poison, <1 x i64> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_global_i64(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr @global_i64, align 8, !tbaa [[TBAA6]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 vint64m1_t read_global_i64() { return global_i64; }
@@ -119,14 +119,14 @@ vint64m1_t read_global_i64() { return global_i64; }
 // CHECK-64-LABEL: @read_global_bool1(
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @global_bool1, align 8, !tbaa [[TBAA6]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v8i8(<vscale x 8 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v8i8(<vscale x 8 x i8> poison, <8 x i8> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-64-NEXT:    ret <vscale x 64 x i1> [[TMP1]]
 //
 // CHECK-256-LABEL: @read_global_bool1(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <32 x i8>, ptr @global_bool1, align 8, !tbaa [[TBAA6]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 8 x i8> [[CAST_SCALABLE]] to <vscale x 64 x i1>
 // CHECK-256-NEXT:    ret <vscale x 64 x i1> [[TMP1]]
 //
@@ -135,14 +135,14 @@ vbool1_t read_global_bool1() { return global_bool1; }
 // CHECK-64-LABEL: @read_global_bool4(
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr @global_bool4, align 2, !tbaa [[TBAA6]]
-// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> [[TMP0]], i64 0)
 // CHECK-64-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-64-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
 // CHECK-256-LABEL: @read_global_bool4(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @global_bool4, align 8, !tbaa [[TBAA6]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-256-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
diff --git a/clang/test/CodeGen/RISCV/riscv-inline-asm.c b/clang/test/CodeGen/RISCV/riscv-inline-asm.c
index de90e513ea1ff..9da306807ed0d 100644
--- a/clang/test/CodeGen/RISCV/riscv-inline-asm.c
+++ b/clang/test/CodeGen/RISCV/riscv-inline-asm.c
@@ -90,9 +90,9 @@ extern int var, arr[2][2];
 struct Pair { int a, b; } pair;
 
 // CHECK-LABEL: test_s(
-// CHECK:         call void asm sideeffect "// $0 $1 $2", "s,s,s"(ptr nonnull @var, ptr nonnull getelementptr inbounds (i8, ptr @arr, {{.*}}), ptr nonnull @test_s)
+// CHECK:         call void asm sideeffect "// $0 $1 $2", "s,s,s"(ptr nonnull @var, ptr nonnull getelementptr inbounds nuw (i8, ptr @arr, {{.*}}), ptr nonnull @test_s)
 // CHECK:         call void asm sideeffect "// $0", "s"(ptr nonnull getelementptr inbounds nuw (i8, ptr @pair, {{.*}}))
-// CHECK:         call void asm sideeffect "// $0 $1 $2", "S,S,S"(ptr nonnull @var, ptr nonnull getelementptr inbounds (i8, ptr @arr, {{.*}}), ptr nonnull @test_s)
+// CHECK:         call void asm sideeffect "// $0 $1 $2", "S,S,S"(ptr nonnull @var, ptr nonnull getelementptr inbounds nuw (i8, ptr @arr, {{.*}}), ptr nonnull @test_s)
 void test_s(void) {
   asm("// %0 %1 %2" :: "s"(&var), "s"(&arr[1][1]), "s"(test_s));
   asm("// %0" :: "s"(&pair.b));
diff --git a/clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c
index 76fcf38a0d98f..0f72541e431db 100644
--- a/clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c
+++ b/clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c
@@ -39,7 +39,7 @@ typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__ri
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t add_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -51,7 +51,7 @@ fixed_int8m1_t add_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t add_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -63,7 +63,7 @@ fixed_int16m1_t add_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t add_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -75,7 +75,7 @@ fixed_int32m1_t add_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t add_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -87,7 +87,7 @@ fixed_int64m1_t add_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t add_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -99,7 +99,7 @@ fixed_uint8m1_t add_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t add_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -111,7 +111,7 @@ fixed_uint16m1_t add_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t add_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -123,7 +123,7 @@ fixed_uint32m1_t add_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t add_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -135,7 +135,7 @@ fixed_uint64m1_t add_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t add_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -147,7 +147,7 @@ fixed_float32m1_t add_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t add_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -159,7 +159,7 @@ fixed_float64m1_t add_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t add_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -171,7 +171,7 @@ fixed_int8m1_t add_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t add_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -183,7 +183,7 @@ fixed_int16m1_t add_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t add_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -195,7 +195,7 @@ fixed_int32m1_t add_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t add_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -207,7 +207,7 @@ fixed_int64m1_t add_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t add_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -219,7 +219,7 @@ fixed_uint8m1_t add_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t add_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -231,7 +231,7 @@ fixed_uint16m1_t add_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t add_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -243,7 +243,7 @@ fixed_uint32m1_t add_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t add_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -255,7 +255,7 @@ fixed_uint64m1_t add_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t add_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -267,7 +267,7 @@ fixed_float32m1_t add_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t add_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -280,7 +280,7 @@ fixed_float64m1_t add_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t add_scalar_i8(fixed_int8m1_t a, int8_t b) {
@@ -293,7 +293,7 @@ fixed_int8m1_t add_scalar_i8(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t add_scalar_i16(fixed_int16m1_t a, int16_t b) {
@@ -306,7 +306,7 @@ fixed_int16m1_t add_scalar_i16(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t add_scalar_i32(fixed_int32m1_t a, int32_t b) {
@@ -319,7 +319,7 @@ fixed_int32m1_t add_scalar_i32(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t add_scalar_i64(fixed_int64m1_t a, int64_t b) {
@@ -332,7 +332,7 @@ fixed_int64m1_t add_scalar_i64(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t add_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
@@ -345,7 +345,7 @@ fixed_uint8m1_t add_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t add_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
@@ -358,7 +358,7 @@ fixed_uint16m1_t add_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t add_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
@@ -371,7 +371,7 @@ fixed_uint32m1_t add_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t add_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
@@ -384,7 +384,7 @@ fixed_uint64m1_t add_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t add_scalar_f32(fixed_float32m1_t a, float b) {
@@ -397,7 +397,7 @@ fixed_float32m1_t add_scalar_f32(fixed_float32m1_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[ADD]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t add_scalar_f64(fixed_float64m1_t a, double b) {
@@ -411,7 +411,7 @@ fixed_float64m1_t add_scalar_f64(fixed_float64m1_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t sub_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -423,7 +423,7 @@ fixed_int8m1_t sub_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t sub_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -435,7 +435,7 @@ fixed_int16m1_t sub_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t sub_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -447,7 +447,7 @@ fixed_int32m1_t sub_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t sub_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -459,7 +459,7 @@ fixed_int64m1_t sub_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t sub_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -471,7 +471,7 @@ fixed_uint8m1_t sub_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t sub_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -483,7 +483,7 @@ fixed_uint16m1_t sub_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t sub_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -495,7 +495,7 @@ fixed_uint32m1_t sub_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t sub_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -507,7 +507,7 @@ fixed_uint64m1_t sub_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t sub_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -519,7 +519,7 @@ fixed_float32m1_t sub_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t sub_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -531,7 +531,7 @@ fixed_float64m1_t sub_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t sub_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -543,7 +543,7 @@ fixed_int8m1_t sub_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t sub_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -555,7 +555,7 @@ fixed_int16m1_t sub_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t sub_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -567,7 +567,7 @@ fixed_int32m1_t sub_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t sub_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -579,7 +579,7 @@ fixed_int64m1_t sub_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t sub_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -591,7 +591,7 @@ fixed_uint8m1_t sub_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t sub_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -603,7 +603,7 @@ fixed_uint16m1_t sub_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t sub_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -615,7 +615,7 @@ fixed_uint32m1_t sub_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t sub_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -627,7 +627,7 @@ fixed_uint64m1_t sub_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t sub_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -639,7 +639,7 @@ fixed_float32m1_t sub_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t sub_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -652,7 +652,7 @@ fixed_float64m1_t sub_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t sub_scalar_i8(fixed_int8m1_t a, int8_t b) {
@@ -665,7 +665,7 @@ fixed_int8m1_t sub_scalar_i8(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t sub_scalar_i16(fixed_int16m1_t a, int16_t b) {
@@ -678,7 +678,7 @@ fixed_int16m1_t sub_scalar_i16(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t sub_scalar_i32(fixed_int32m1_t a, int32_t b) {
@@ -691,7 +691,7 @@ fixed_int32m1_t sub_scalar_i32(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t sub_scalar_i64(fixed_int64m1_t a, int64_t b) {
@@ -704,7 +704,7 @@ fixed_int64m1_t sub_scalar_i64(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t sub_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
@@ -717,7 +717,7 @@ fixed_uint8m1_t sub_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t sub_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
@@ -730,7 +730,7 @@ fixed_uint16m1_t sub_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t sub_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
@@ -743,7 +743,7 @@ fixed_uint32m1_t sub_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t sub_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
@@ -756,7 +756,7 @@ fixed_uint64m1_t sub_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t sub_scalar_f32(fixed_float32m1_t a, float b) {
@@ -769,7 +769,7 @@ fixed_float32m1_t sub_scalar_f32(fixed_float32m1_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[SUB]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t sub_scalar_f64(fixed_float64m1_t a, double b) {
@@ -783,7 +783,7 @@ fixed_float64m1_t sub_scalar_f64(fixed_float64m1_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t mul_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -795,7 +795,7 @@ fixed_int8m1_t mul_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t mul_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -807,7 +807,7 @@ fixed_int16m1_t mul_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t mul_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -819,7 +819,7 @@ fixed_int32m1_t mul_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t mul_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -831,7 +831,7 @@ fixed_int64m1_t mul_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t mul_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -843,7 +843,7 @@ fixed_uint8m1_t mul_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t mul_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -855,7 +855,7 @@ fixed_uint16m1_t mul_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t mul_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -867,7 +867,7 @@ fixed_uint32m1_t mul_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t mul_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -879,7 +879,7 @@ fixed_uint64m1_t mul_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t mul_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -891,7 +891,7 @@ fixed_float32m1_t mul_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t mul_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -903,7 +903,7 @@ fixed_float64m1_t mul_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t mul_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -915,7 +915,7 @@ fixed_int8m1_t mul_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t mul_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -927,7 +927,7 @@ fixed_int16m1_t mul_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t mul_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -939,7 +939,7 @@ fixed_int32m1_t mul_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t mul_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -951,7 +951,7 @@ fixed_int64m1_t mul_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t mul_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -963,7 +963,7 @@ fixed_uint8m1_t mul_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t mul_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -975,7 +975,7 @@ fixed_uint16m1_t mul_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t mul_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -987,7 +987,7 @@ fixed_uint32m1_t mul_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t mul_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -999,7 +999,7 @@ fixed_uint64m1_t mul_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t mul_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -1011,7 +1011,7 @@ fixed_float32m1_t mul_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t mul_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -1024,7 +1024,7 @@ fixed_float64m1_t mul_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t mul_scalar_i8(fixed_int8m1_t a, int8_t b) {
@@ -1037,7 +1037,7 @@ fixed_int8m1_t mul_scalar_i8(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t mul_scalar_i16(fixed_int16m1_t a, int16_t b) {
@@ -1050,7 +1050,7 @@ fixed_int16m1_t mul_scalar_i16(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t mul_scalar_i32(fixed_int32m1_t a, int32_t b) {
@@ -1063,7 +1063,7 @@ fixed_int32m1_t mul_scalar_i32(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t mul_scalar_i64(fixed_int64m1_t a, int64_t b) {
@@ -1076,7 +1076,7 @@ fixed_int64m1_t mul_scalar_i64(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t mul_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
@@ -1089,7 +1089,7 @@ fixed_uint8m1_t mul_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t mul_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
@@ -1102,7 +1102,7 @@ fixed_uint16m1_t mul_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t mul_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
@@ -1115,7 +1115,7 @@ fixed_uint32m1_t mul_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t mul_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
@@ -1128,7 +1128,7 @@ fixed_uint64m1_t mul_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t mul_scalar_f32(fixed_float32m1_t a, float b) {
@@ -1141,7 +1141,7 @@ fixed_float32m1_t mul_scalar_f32(fixed_float32m1_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[MUL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t mul_scalar_f64(fixed_float64m1_t a, double b) {
@@ -1155,7 +1155,7 @@ fixed_float64m1_t mul_scalar_f64(fixed_float64m1_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t div_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -1167,7 +1167,7 @@ fixed_int8m1_t div_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t div_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -1179,7 +1179,7 @@ fixed_int16m1_t div_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t div_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -1191,7 +1191,7 @@ fixed_int32m1_t div_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t div_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -1203,7 +1203,7 @@ fixed_int64m1_t div_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t div_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -1215,7 +1215,7 @@ fixed_uint8m1_t div_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t div_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -1227,7 +1227,7 @@ fixed_uint16m1_t div_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t div_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -1239,7 +1239,7 @@ fixed_uint32m1_t div_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t div_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -1251,7 +1251,7 @@ fixed_uint64m1_t div_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t div_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -1263,7 +1263,7 @@ fixed_float32m1_t div_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t div_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -1275,7 +1275,7 @@ fixed_float64m1_t div_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t div_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -1287,7 +1287,7 @@ fixed_int8m1_t div_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t div_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -1299,7 +1299,7 @@ fixed_int16m1_t div_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t div_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -1311,7 +1311,7 @@ fixed_int32m1_t div_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t div_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -1323,7 +1323,7 @@ fixed_int64m1_t div_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t div_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -1335,7 +1335,7 @@ fixed_uint8m1_t div_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t div_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -1347,7 +1347,7 @@ fixed_uint16m1_t div_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t div_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -1359,7 +1359,7 @@ fixed_uint32m1_t div_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t div_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -1371,7 +1371,7 @@ fixed_uint64m1_t div_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t div_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -1383,7 +1383,7 @@ fixed_float32m1_t div_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t div_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -1396,7 +1396,7 @@ fixed_float64m1_t div_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t div_scalar_i8(fixed_int8m1_t a, int8_t b) {
@@ -1409,7 +1409,7 @@ fixed_int8m1_t div_scalar_i8(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t div_scalar_i16(fixed_int16m1_t a, int16_t b) {
@@ -1422,7 +1422,7 @@ fixed_int16m1_t div_scalar_i16(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t div_scalar_i32(fixed_int32m1_t a, int32_t b) {
@@ -1435,7 +1435,7 @@ fixed_int32m1_t div_scalar_i32(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t div_scalar_i64(fixed_int64m1_t a, int64_t b) {
@@ -1448,7 +1448,7 @@ fixed_int64m1_t div_scalar_i64(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t div_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
@@ -1461,7 +1461,7 @@ fixed_uint8m1_t div_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t div_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
@@ -1474,7 +1474,7 @@ fixed_uint16m1_t div_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t div_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
@@ -1487,7 +1487,7 @@ fixed_uint32m1_t div_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t div_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
@@ -1500,7 +1500,7 @@ fixed_uint64m1_t div_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> poison, <8 x float> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x float> [[CAST_SCALABLE]]
 //
 fixed_float32m1_t div_scalar_f32(fixed_float32m1_t a, float b) {
@@ -1513,7 +1513,7 @@ fixed_float32m1_t div_scalar_f32(fixed_float32m1_t a, float b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> poison, <4 x double> [[DIV]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 fixed_float64m1_t div_scalar_f64(fixed_float64m1_t a, double b) {
@@ -1527,7 +1527,7 @@ fixed_float64m1_t div_scalar_f64(fixed_float64m1_t a, double b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rem_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -1539,7 +1539,7 @@ fixed_int8m1_t rem_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rem_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -1551,7 +1551,7 @@ fixed_int16m1_t rem_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rem_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -1563,7 +1563,7 @@ fixed_int32m1_t rem_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rem_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -1575,7 +1575,7 @@ fixed_int64m1_t rem_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rem_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -1587,7 +1587,7 @@ fixed_uint8m1_t rem_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rem_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -1599,7 +1599,7 @@ fixed_uint16m1_t rem_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rem_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -1611,7 +1611,7 @@ fixed_uint32m1_t rem_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rem_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -1623,7 +1623,7 @@ fixed_uint64m1_t rem_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rem_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -1635,7 +1635,7 @@ fixed_int8m1_t rem_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rem_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -1647,7 +1647,7 @@ fixed_int16m1_t rem_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rem_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -1659,7 +1659,7 @@ fixed_int32m1_t rem_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rem_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -1671,7 +1671,7 @@ fixed_int64m1_t rem_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rem_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -1683,7 +1683,7 @@ fixed_uint8m1_t rem_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rem_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -1695,7 +1695,7 @@ fixed_uint16m1_t rem_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rem_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -1707,7 +1707,7 @@ fixed_uint32m1_t rem_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rem_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -1720,7 +1720,7 @@ fixed_uint64m1_t rem_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rem_scalar_i8(fixed_int8m1_t a, int8_t b) {
@@ -1733,7 +1733,7 @@ fixed_int8m1_t rem_scalar_i8(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rem_scalar_i16(fixed_int16m1_t a, int16_t b) {
@@ -1746,7 +1746,7 @@ fixed_int16m1_t rem_scalar_i16(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rem_scalar_i32(fixed_int32m1_t a, int32_t b) {
@@ -1759,7 +1759,7 @@ fixed_int32m1_t rem_scalar_i32(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rem_scalar_i64(fixed_int64m1_t a, int64_t b) {
@@ -1772,7 +1772,7 @@ fixed_int64m1_t rem_scalar_i64(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rem_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
@@ -1785,7 +1785,7 @@ fixed_uint8m1_t rem_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rem_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
@@ -1798,7 +1798,7 @@ fixed_uint16m1_t rem_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rem_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
@@ -1811,7 +1811,7 @@ fixed_uint32m1_t rem_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[REM]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rem_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
diff --git a/clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c
index 0c81adcfe0b0b..8366347248a8f 100644
--- a/clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c
+++ b/clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c
@@ -39,7 +39,7 @@ typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__ri
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t and_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -51,7 +51,7 @@ fixed_int8m1_t and_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t and_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -63,7 +63,7 @@ fixed_int16m1_t and_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t and_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -75,7 +75,7 @@ fixed_int32m1_t and_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t and_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -87,7 +87,7 @@ fixed_int64m1_t and_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t and_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -99,7 +99,7 @@ fixed_uint8m1_t and_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t and_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -111,7 +111,7 @@ fixed_uint16m1_t and_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t and_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -123,7 +123,7 @@ fixed_uint32m1_t and_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[AND:%.*]] = and <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[AND]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t and_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -137,7 +137,7 @@ fixed_uint64m1_t and_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t or_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -149,7 +149,7 @@ fixed_int8m1_t or_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t or_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -161,7 +161,7 @@ fixed_int16m1_t or_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t or_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -173,7 +173,7 @@ fixed_int32m1_t or_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t or_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -185,7 +185,7 @@ fixed_int64m1_t or_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t or_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -197,7 +197,7 @@ fixed_uint8m1_t or_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t or_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -209,7 +209,7 @@ fixed_uint16m1_t or_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t or_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -221,7 +221,7 @@ fixed_uint32m1_t or_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[OR:%.*]] = or <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[OR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t or_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -235,7 +235,7 @@ fixed_uint64m1_t or_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t xor_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -247,7 +247,7 @@ fixed_int8m1_t xor_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t xor_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -259,7 +259,7 @@ fixed_int16m1_t xor_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t xor_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -271,7 +271,7 @@ fixed_int32m1_t xor_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t xor_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -283,7 +283,7 @@ fixed_int64m1_t xor_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t xor_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -295,7 +295,7 @@ fixed_uint8m1_t xor_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t xor_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -307,7 +307,7 @@ fixed_uint16m1_t xor_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t xor_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -319,7 +319,7 @@ fixed_uint32m1_t xor_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[XOR:%.*]] = xor <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[XOR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t xor_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -332,7 +332,7 @@ fixed_uint64m1_t xor_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i8> [[A]], splat (i8 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t not_i8(fixed_int8m1_t a) {
@@ -343,7 +343,7 @@ fixed_int8m1_t not_i8(fixed_int8m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i16> [[A]], splat (i16 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t not_i16(fixed_int16m1_t a) {
@@ -354,7 +354,7 @@ fixed_int16m1_t not_i16(fixed_int16m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i32> [[A]], splat (i32 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t not_i32(fixed_int32m1_t a) {
@@ -365,7 +365,7 @@ fixed_int32m1_t not_i32(fixed_int32m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <4 x i64> [[A]], splat (i64 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t not_i64(fixed_int64m1_t a) {
@@ -376,7 +376,7 @@ fixed_int64m1_t not_i64(fixed_int64m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i8> [[A]], splat (i8 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t not_u8(fixed_uint8m1_t a) {
@@ -387,7 +387,7 @@ fixed_uint8m1_t not_u8(fixed_uint8m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i16> [[A]], splat (i16 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t not_u16(fixed_uint16m1_t a) {
@@ -398,7 +398,7 @@ fixed_uint16m1_t not_u16(fixed_uint16m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i32> [[A]], splat (i32 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t not_u32(fixed_uint32m1_t a) {
@@ -409,7 +409,7 @@ fixed_uint32m1_t not_u32(fixed_uint32m1_t a) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[NOT:%.*]] = xor <4 x i64> [[A]], splat (i64 -1)
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[NOT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t not_u64(fixed_uint64m1_t a) {
diff --git a/clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c
index f22b4fbb5251f..dc0ba616bd1b3 100644
--- a/clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c
+++ b/clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c
@@ -40,7 +40,7 @@ typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__ri
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t eq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -53,7 +53,7 @@ fixed_int8m1_t eq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t eq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -66,7 +66,7 @@ fixed_int16m1_t eq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t eq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -79,7 +79,7 @@ fixed_int32m1_t eq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t eq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -92,7 +92,7 @@ fixed_int64m1_t eq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t eq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -105,7 +105,7 @@ fixed_int8m1_t eq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t eq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -118,7 +118,7 @@ fixed_int16m1_t eq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t eq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -131,7 +131,7 @@ fixed_int32m1_t eq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t eq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -144,7 +144,7 @@ fixed_int64m1_t eq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t eq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -157,7 +157,7 @@ fixed_int32m1_t eq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t eq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -172,7 +172,7 @@ fixed_int64m1_t eq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t neq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -185,7 +185,7 @@ fixed_int8m1_t neq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t neq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -198,7 +198,7 @@ fixed_int16m1_t neq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t neq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -211,7 +211,7 @@ fixed_int32m1_t neq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t neq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -224,7 +224,7 @@ fixed_int64m1_t neq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t neq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -237,7 +237,7 @@ fixed_int8m1_t neq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t neq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -250,7 +250,7 @@ fixed_int16m1_t neq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t neq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -263,7 +263,7 @@ fixed_int32m1_t neq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ne <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t neq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -276,7 +276,7 @@ fixed_int64m1_t neq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp une <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t neq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -289,7 +289,7 @@ fixed_int32m1_t neq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp une <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t neq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -304,7 +304,7 @@ fixed_int64m1_t neq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t lt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -317,7 +317,7 @@ fixed_int8m1_t lt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t lt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -330,7 +330,7 @@ fixed_int16m1_t lt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -343,7 +343,7 @@ fixed_int32m1_t lt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -356,7 +356,7 @@ fixed_int64m1_t lt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t lt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -369,7 +369,7 @@ fixed_int8m1_t lt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t lt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -382,7 +382,7 @@ fixed_int16m1_t lt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -395,7 +395,7 @@ fixed_int32m1_t lt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -408,7 +408,7 @@ fixed_int64m1_t lt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -421,7 +421,7 @@ fixed_int32m1_t lt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -436,7 +436,7 @@ fixed_int64m1_t lt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t leq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -449,7 +449,7 @@ fixed_int8m1_t leq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t leq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -462,7 +462,7 @@ fixed_int16m1_t leq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t leq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -475,7 +475,7 @@ fixed_int32m1_t leq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sle <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t leq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -488,7 +488,7 @@ fixed_int64m1_t leq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t leq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -501,7 +501,7 @@ fixed_int8m1_t leq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t leq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -514,7 +514,7 @@ fixed_int16m1_t leq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t leq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -527,7 +527,7 @@ fixed_int32m1_t leq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ule <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t leq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -540,7 +540,7 @@ fixed_int64m1_t leq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t leq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -553,7 +553,7 @@ fixed_int32m1_t leq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t leq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -568,7 +568,7 @@ fixed_int64m1_t leq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t gt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -581,7 +581,7 @@ fixed_int8m1_t gt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t gt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -594,7 +594,7 @@ fixed_int16m1_t gt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t gt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -607,7 +607,7 @@ fixed_int32m1_t gt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t gt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -620,7 +620,7 @@ fixed_int64m1_t gt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t gt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -633,7 +633,7 @@ fixed_int8m1_t gt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t gt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -646,7 +646,7 @@ fixed_int16m1_t gt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t gt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -659,7 +659,7 @@ fixed_int32m1_t gt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t gt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -672,7 +672,7 @@ fixed_int64m1_t gt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t gt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -685,7 +685,7 @@ fixed_int32m1_t gt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t gt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
@@ -700,7 +700,7 @@ fixed_int64m1_t gt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t geq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -713,7 +713,7 @@ fixed_int8m1_t geq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t geq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -726,7 +726,7 @@ fixed_int16m1_t geq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t geq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -739,7 +739,7 @@ fixed_int32m1_t geq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp sge <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t geq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -752,7 +752,7 @@ fixed_int64m1_t geq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <32 x i8> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t geq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -765,7 +765,7 @@ fixed_int8m1_t geq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <16 x i16> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t geq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -778,7 +778,7 @@ fixed_int16m1_t geq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <8 x i32> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t geq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -791,7 +791,7 @@ fixed_int32m1_t geq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = icmp uge <4 x i64> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t geq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -804,7 +804,7 @@ fixed_int64m1_t geq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <8 x float> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t geq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
@@ -817,7 +817,7 @@ fixed_int32m1_t geq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
 // CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <4 x double> [[A]], [[B]]
 // CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SEXT]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t geq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
diff --git a/clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c
index d88a5ba239e82..7c4196a879f61 100644
--- a/clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c
+++ b/clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c
@@ -37,7 +37,7 @@ typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__ri
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t lshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -49,7 +49,7 @@ fixed_int8m1_t lshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
@@ -61,7 +61,7 @@ fixed_int8m1_t rshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t lshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -73,7 +73,7 @@ fixed_uint8m1_t lshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
@@ -85,7 +85,7 @@ fixed_uint8m1_t rshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t lshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -97,7 +97,7 @@ fixed_int16m1_t lshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
@@ -109,7 +109,7 @@ fixed_int16m1_t rshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t lshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -121,7 +121,7 @@ fixed_uint16m1_t lshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
@@ -133,7 +133,7 @@ fixed_uint16m1_t rshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -145,7 +145,7 @@ fixed_int32m1_t lshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
@@ -157,7 +157,7 @@ fixed_int32m1_t rshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t lshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -169,7 +169,7 @@ fixed_uint32m1_t lshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
@@ -181,7 +181,7 @@ fixed_uint32m1_t rshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -193,7 +193,7 @@ fixed_int64m1_t lshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
@@ -205,7 +205,7 @@ fixed_int64m1_t rshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t lshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -217,7 +217,7 @@ fixed_uint64m1_t lshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[A]], [[B]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
@@ -232,7 +232,7 @@ fixed_uint64m1_t rshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t lshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
@@ -245,7 +245,7 @@ fixed_int8m1_t lshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t lshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
@@ -260,7 +260,7 @@ fixed_int8m1_t lshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
@@ -273,7 +273,7 @@ fixed_int8m1_t rshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_int8m1_t rshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
@@ -288,7 +288,7 @@ fixed_int8m1_t rshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t lshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
@@ -301,7 +301,7 @@ fixed_uint8m1_t lshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t lshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
@@ -316,7 +316,7 @@ fixed_uint8m1_t lshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
@@ -329,7 +329,7 @@ fixed_uint8m1_t rshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> poison, <32 x i8> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 8 x i8> [[CAST_SCALABLE]]
 //
 fixed_uint8m1_t rshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
@@ -344,7 +344,7 @@ fixed_uint8m1_t rshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t lshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
@@ -357,7 +357,7 @@ fixed_int16m1_t lshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t lshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
@@ -372,7 +372,7 @@ fixed_int16m1_t lshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
@@ -385,7 +385,7 @@ fixed_int16m1_t rshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_int16m1_t rshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
@@ -400,7 +400,7 @@ fixed_int16m1_t rshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t lshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
@@ -413,7 +413,7 @@ fixed_uint16m1_t lshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t lshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
@@ -428,7 +428,7 @@ fixed_uint16m1_t lshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[A]], [[SH_PROM]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
@@ -441,7 +441,7 @@ fixed_uint16m1_t rshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> poison, <16 x i16> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i16> [[CAST_SCALABLE]]
 //
 fixed_uint16m1_t rshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
@@ -454,7 +454,7 @@ fixed_uint16m1_t rshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
@@ -467,7 +467,7 @@ fixed_int32m1_t lshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t lshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
@@ -480,7 +480,7 @@ fixed_int32m1_t lshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
@@ -493,7 +493,7 @@ fixed_int32m1_t rshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32m1_t rshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
@@ -506,7 +506,7 @@ fixed_int32m1_t rshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t lshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
@@ -519,7 +519,7 @@ fixed_uint32m1_t lshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t lshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
@@ -532,7 +532,7 @@ fixed_uint32m1_t lshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
@@ -545,7 +545,7 @@ fixed_uint32m1_t rshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> poison, <8 x i32> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i32> [[CAST_SCALABLE]]
 //
 fixed_uint32m1_t rshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
@@ -558,7 +558,7 @@ fixed_uint32m1_t rshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
@@ -571,7 +571,7 @@ fixed_int64m1_t lshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t lshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
@@ -584,7 +584,7 @@ fixed_int64m1_t lshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
@@ -597,7 +597,7 @@ fixed_int64m1_t rshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_int64m1_t rshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
@@ -610,7 +610,7 @@ fixed_int64m1_t rshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t lshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
@@ -623,7 +623,7 @@ fixed_uint64m1_t lshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHL]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t lshift_u64_lsplat(fixed_uint64m1_t a, uint64_t b) {
@@ -636,7 +636,7 @@ fixed_uint64m1_t lshift_u64_lsplat(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[A]], [[SPLAT_SPLAT]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
@@ -649,7 +649,7 @@ fixed_uint64m1_t rshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[SPLAT_SPLAT]], [[A]]
-// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> poison, <4 x i64> [[SHR]], i64 0)
 // CHECK-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 fixed_uint64m1_t rshift_u64_lsplat(fixed_uint64m1_t a, uint64_t b) {
diff --git a/clang/test/CodeGen/SystemZ/builtin-setjmp-logjmp.c b/clang/test/CodeGen/SystemZ/builtin-setjmp-logjmp.c
new file mode 100644
index 0000000000000..898891fa182ea
--- /dev/null
+++ b/clang/test/CodeGen/SystemZ/builtin-setjmp-logjmp.c
@@ -0,0 +1,26 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple s390x-linux -emit-llvm -o - %s | FileCheck %s
+
+void *buf[20];
+
+// CHECK-LABEL: define dso_local void @foo(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.eh.sjlj.setjmp(ptr @buf)
+// CHECK-NEXT:    ret void
+//
+void foo()
+{
+  __builtin_setjmp (buf);
+}
+
+// CHECK-LABEL: define dso_local void @foo1(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    call void @llvm.eh.sjlj.longjmp(ptr @buf)
+// CHECK-NEXT:    unreachable
+//
+void foo1()
+{
+  __builtin_longjmp (buf, 1);
+}
diff --git a/clang/test/CodeGen/SystemZ/zos-mixed-ptr-sizes.c b/clang/test/CodeGen/SystemZ/zos-mixed-ptr-sizes.c
index 82bb7a52d05d1..33cf0ddb47236 100644
--- a/clang/test/CodeGen/SystemZ/zos-mixed-ptr-sizes.c
+++ b/clang/test/CodeGen/SystemZ/zos-mixed-ptr-sizes.c
@@ -48,21 +48,21 @@ void test_indexing(struct Foo *f) {
 
 void test_indexing_2(struct Foo *f) {
   // X64-LABEL: define void @test_indexing_2(ptr noundef initializes((16, 24)) %f)
-  // X64: getelementptr inbounds i8, ptr addrspace(1) {{%[0-9]}}, i32 16
-  // X64: getelementptr inbounds i8, ptr {{%[0-9]}}, i64 24
+  // X64: getelementptr inbounds nuw i8, ptr addrspace(1) {{%[0-9]}}, i32 16
+  // X64: getelementptr inbounds nuw i8, ptr {{%[0-9]}}, i64 24
   f->cp64 = ((char *** __ptr32 *)1028)[1][2][3];
   use_foo(f);
 }
 
 unsigned long* test_misc() {
   // X64-LABEL: define ptr @test_misc()
-  // X64: %arrayidx = getelementptr inbounds i8, ptr addrspace(1) %0, i32 88
+  // X64: %arrayidx = getelementptr inbounds nuw i8, ptr addrspace(1) %0, i32 88
   // X64-NEXT: %1 = load ptr, ptr addrspace(1) %arrayidx
-  // X64-NEXT: %arrayidx1 = getelementptr inbounds i8, ptr %1, i64 8
+  // X64-NEXT: %arrayidx1 = getelementptr inbounds nuw i8, ptr %1, i64 8
   // X64-NEXT: %2 = load ptr, ptr %arrayidx1
-  // X64-NEXT: %arrayidx2 = getelementptr inbounds i8, ptr %2, i64 904
+  // X64-NEXT: %arrayidx2 = getelementptr inbounds nuw i8, ptr %2, i64 904
   // X64-NEXT: %3 = load ptr, ptr %arrayidx2
-  // X64-NEXT: %arrayidx3 = getelementptr inbounds i8, ptr %3, i64 1192
+  // X64-NEXT: %arrayidx3 = getelementptr inbounds nuw i8, ptr %3, i64 1192
   unsigned long* x = (unsigned long*)((char***** __ptr32*)1208)[0][11][1][113][149];
   return x;
 }
@@ -71,9 +71,9 @@ char* __ptr32* __ptr32 test_misc_2() {
   // X64-LABEL: define ptr addrspace(1) @test_misc_2()
   // X64: br i1 %cmp, label %if.then, label %if.end
   // X64: %1 = load ptr addrspace(1), ptr inttoptr (i64 16 to ptr)
-  // X64-NEXT: %arrayidx = getelementptr inbounds i8, ptr addrspace(1) %1, i32 544
+  // X64-NEXT: %arrayidx = getelementptr inbounds nuw i8, ptr addrspace(1) %1, i32 544
   // X64-NEXT: %2 = load ptr addrspace(1), ptr addrspace(1) %arrayidx
-  // X64-NEXT: %arrayidx1 = getelementptr inbounds i8, ptr addrspace(1) %2, i32 24
+  // X64-NEXT: %arrayidx1 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i32 24
   // X64-NEXT: %3 = load ptr addrspace(1), ptr addrspace(1) %arrayidx1
   // X64-NEXT: store ptr addrspace(1) %3, ptr @test_misc_2.res
   // X64: ret ptr addrspace(1)
@@ -88,7 +88,7 @@ unsigned short test_misc_3() {
   // X64-LABEL: define zeroext i16 @test_misc_3()
   // X64: %0 = load ptr addrspace(1), ptr inttoptr (i64 548 to ptr)
   // X64-NEXT: %1 = addrspacecast ptr addrspace(1) %0 to ptr
-  // X64-NEXT: %arrayidx = getelementptr inbounds i8, ptr %1, i64 36
+  // X64-NEXT: %arrayidx = getelementptr inbounds nuw i8, ptr %1, i64 36
   // X64-NEXT: %2 = load i16, ptr %arrayidx, align 2
   // X64-NEXT: ret i16 %2
   unsigned short this_asid = ((unsigned short*)(*(char* __ptr32*)(0x224)))[18];
@@ -97,10 +97,10 @@ unsigned short test_misc_3() {
 
 int test_misc_4() {
   // X64-LABEL: define signext range(i32 0, 2) i32 @test_misc_4()
-  // X64: getelementptr inbounds i8, ptr addrspace(1) {{%[0-9]}}, i32 88
-  // X64: getelementptr inbounds i8, ptr {{%[0-9]}}, i64 8
-  // X64: getelementptr inbounds i8, ptr {{%[0-9]}}, i64 984
-  // X64: getelementptr inbounds i8, ptr %3, i64 80
+  // X64: getelementptr inbounds nuw i8, ptr addrspace(1) {{%[0-9]}}, i32 88
+  // X64: getelementptr inbounds nuw i8, ptr {{%[0-9]}}, i64 8
+  // X64: getelementptr inbounds nuw i8, ptr {{%[0-9]}}, i64 984
+  // X64: getelementptr inbounds nuw i8, ptr %3, i64 80
   // X64: icmp sgt i32 {{.*[0-9]}}, 67240703
   // X64: ret i32
   int a = (*(int*)(80 + ((char**** __ptr32*)1208)[0][11][1][123]) > 0x040202FF);
@@ -189,7 +189,7 @@ int test_function_ptr32_is_32bit() {
 int get_processor_count() {
   // X64-LABEL: define signext range(i32 -128, 128) i32 @get_processor_count()
   // X64: load ptr addrspace(1), ptr inttoptr (i64 16 to ptr)
-  // X64-NEXT: [[ARR_IDX1:%[a-z].*]] = getelementptr inbounds i8, ptr addrspace(1) %0, i32 660
+  // X64-NEXT: [[ARR_IDX1:%[a-z].*]] = getelementptr inbounds nuw i8, ptr addrspace(1) %0, i32 660
   // X64: load ptr addrspace(1), ptr addrspace(1) [[ARR_IDX1]]
   // X64: load i8, ptr addrspace(1) {{%[a-z].*}}
   // X64: sext i8 {{%[0-9]}} to i32
diff --git a/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c b/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
index 78e73947b92c4..ca8f5e482cc56 100644
--- a/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
@@ -1,35 +1,43 @@
 // RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m512i test_mm512_popcnt_epi64(__m512i __A) {
   // CHECK-LABEL: @test_mm512_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   return _mm512_popcnt_epi64(__A);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_popcnt_epi64((__m512i)(__v8di){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 31, 30, 1, 0, 24, 1, 25));
+
 __m512i test_mm512_mask_popcnt_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_popcnt_epi64(__W, __U, __A);
 }
+
 __m512i test_mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_popcnt_epi64(__U, __A);
 }
+
 __m512i test_mm512_popcnt_epi32(__m512i __A) {
   // CHECK-LABEL: @test_mm512_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
   return _mm512_popcnt_epi32(__A);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_popcnt_epi32((__m512i)(__v16si){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 31, 30, 1, 0, 24, 1, 25, 2, 2, 4, 2, 6, 2, 9, 2));
+
 __m512i test_mm512_mask_popcnt_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_popcnt_epi32(__W, __U, __A);
 }
+
 __m512i test_mm512_maskz_popcnt_epi32(__mmask16 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
diff --git a/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c b/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
index 8865c028038ae..5d18b68e46905 100644
--- a/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
@@ -1,35 +1,43 @@
 // RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m128i test_mm_popcnt_epi64(__m128i __A) {
   // CHECK-LABEL: @test_mm_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   return _mm_popcnt_epi64(__A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_popcnt_epi64((__m128i)(__v2di){+5, -3}), 2, 63));
+
 __m128i test_mm_mask_popcnt_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   // CHECK: select <2 x i1> %{{.+}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_popcnt_epi64(__W, __U, __A);
 }
+
 __m128i test_mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   // CHECK: select <2 x i1> %{{.+}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_popcnt_epi64(__U, __A);
 }
+
 __m128i test_mm_popcnt_epi32(__m128i __A) {
   // CHECK-LABEL: @test_mm_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
   return _mm_popcnt_epi32(__A);
 }
+TEST_CONSTEXPR(match_v4si(_mm_popcnt_epi32((__m128i)(__v4si){+5, -3, -10, +8}), 2, 31, 30, 1));
+
 __m128i test_mm_mask_popcnt_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
   // CHECK: select <4 x i1> %{{.+}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_popcnt_epi32(__W, __U, __A);
 }
+
 __m128i test_mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
@@ -42,29 +50,36 @@ __m256i test_mm256_popcnt_epi64(__m256i __A) {
   // CHECK: @llvm.ctpop.v4i64
   return _mm256_popcnt_epi64(__A);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_popcnt_epi64((__m256i)(__v4di){+5, -3, -10, +8}), 2, 63, 62, 1));
+
 __m256i test_mm256_mask_popcnt_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v4i64
   // CHECK: select <4 x i1> %{{.+}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_popcnt_epi64(__W, __U, __A);
 }
+
 __m256i test_mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v4i64
   // CHECK: select <4 x i1> %{{.+}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_popcnt_epi64(__U, __A);
 }
+
 __m256i test_mm256_popcnt_epi32(__m256i __A) {
   // CHECK-LABEL: @test_mm256_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
   return _mm256_popcnt_epi32(__A);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_popcnt_epi32((__m256i)(__v8si){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 31, 30, 1, 0, 24, 1, 25));
+
 __m256i test_mm256_mask_popcnt_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
   // CHECK: select <8 x i1> %{{.+}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_popcnt_epi32(__W, __U, __A);
 }
+
 __m256i test_mm256_maskz_popcnt_epi32(__mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
index c3307f749942b..22a87ce9623be 100644
--- a/clang/test/CodeGen/X86/builtin_test_helpers.h
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -73,6 +73,16 @@ constexpr bool match_m256i(__m256i _v, unsigned long long a, unsigned long long
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
 }
 
+constexpr bool match_v4di(__m256i _v, long long a, long long b, long long c, long long d) {
+  __v4di v = (__v4di)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+}
+
+constexpr bool match_v8si(__m256i _v, int a, int b, int c, int d, int e, int f, int g, int h) {
+  __v8si v = (__v8si)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
 constexpr bool match_m512(__m512 v, float a, float b, float c, float d, float e, float f, float g, float h, float i, float j, float k, float l, float m, float n, float o, float p) {
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
 }
@@ -86,6 +96,16 @@ constexpr bool match_m512i(__m512i _v, unsigned long long a, unsigned long long
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
 
+constexpr bool match_v8di(__m512i _v, long long a, long long b, long long c, long long d, long long e, long long f, long long g, long long h) {
+  __v8di v = (__v8di)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
+constexpr bool match_v16si(__m512i _v, int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) {
+  __v16si v = (__v16si)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
+}
+
 #define TEST_CONSTEXPR(...) static_assert(__VA_ARGS__)
 
 #else
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index a0e6dc219b36f..f87091427df71 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -266,14 +266,14 @@
 // BASIC_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // BASIC_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// BASIC_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// BASIC_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // BASIC_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // BASIC_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// BASIC_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// BASIC_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// BASIC_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // BASIC_FAST-NEXT:    [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // BASIC_FAST-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // BASIC_FAST-NEXT:    [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP0]], [[TMP1]]
@@ -285,8 +285,8 @@
 // BASIC_FAST-NEXT:    [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP7]]
 // BASIC_FAST-NEXT:    [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP2]], [[TMP5]]
 // BASIC_FAST-NEXT:    [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP8]], [[TMP5]]
-// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half
-// BASIC_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP9]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP10]] to half
 // BASIC_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // BASIC_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -307,22 +307,22 @@
 // FULL_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // FULL_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// FULL_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// FULL_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // FULL_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // FULL_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// FULL_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// FULL_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// FULL_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[EXT]], float noundef nofpclass(nan inf) [[EXT1]], float noundef nofpclass(nan inf) [[EXT2]], float noundef nofpclass(nan inf) [[EXT3]]) #[[ATTR1:[0-9]+]]
 // FULL_FAST-NEXT:    store <2 x float> [[CALL]], ptr [[COERCE]], align 4
 // FULL_FAST-NEXT:    [[COERCE_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4
 // FULL_FAST-NEXT:    [[COERCE_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4
-// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half
-// FULL_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[COERCE_IMAG]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[COERCE_REAL]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[COERCE_IMAG]] to half
 // FULL_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // FULL_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -342,14 +342,14 @@
 // IMPRVD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// IMPRVD_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// IMPRVD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // IMPRVD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// IMPRVD_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// IMPRVD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// IMPRVD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // IMPRVD_FAST-NEXT:    [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT2]])
 // IMPRVD_FAST-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT3]])
 // IMPRVD_FAST-NEXT:    [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP0]], [[TMP1]]
@@ -379,8 +379,8 @@
 // IMPRVD_FAST:       complex_div:
 // IMPRVD_FAST-NEXT:    [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
 // IMPRVD_FAST-NEXT:    [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[TMP21]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP20]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP21]] to half
 // IMPRVD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -400,14 +400,14 @@
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // PRMTD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // PRMTD_FAST-NEXT:    [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // PRMTD_FAST-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // PRMTD_FAST-NEXT:    [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP0]], [[TMP1]]
@@ -419,8 +419,8 @@
 // PRMTD_FAST-NEXT:    [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP7]]
 // PRMTD_FAST-NEXT:    [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP2]], [[TMP5]]
 // PRMTD_FAST-NEXT:    [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP8]], [[TMP5]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half
-// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP9]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP10]] to half
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -636,22 +636,22 @@ _Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) {
 // BASIC_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // BASIC_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// BASIC_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// BASIC_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // BASIC_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // BASIC_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// BASIC_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// BASIC_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// BASIC_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // BASIC_FAST-NEXT:    [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // BASIC_FAST-NEXT:    [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // BASIC_FAST-NEXT:    [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]]
 // BASIC_FAST-NEXT:    [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]]
 // BASIC_FAST-NEXT:    [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]]
 // BASIC_FAST-NEXT:    [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]]
-// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half
-// BASIC_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_R]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_I]] to half
 // BASIC_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // BASIC_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -672,14 +672,14 @@ _Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) {
 // FULL_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // FULL_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// FULL_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// FULL_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // FULL_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // FULL_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// FULL_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// FULL_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// FULL_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // FULL_FAST-NEXT:    [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // FULL_FAST-NEXT:    [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // FULL_FAST-NEXT:    [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]]
@@ -702,8 +702,8 @@ _Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) {
 // FULL_FAST:       complex_mul_cont:
 // FULL_FAST-NEXT:    [[REAL_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ]
 // FULL_FAST-NEXT:    [[IMAG_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ]
-// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half
-// FULL_FAST-NEXT:    [[UNPROMOTION5:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[REAL_MUL_PHI]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION5:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[IMAG_MUL_PHI]] to half
 // FULL_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // FULL_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -723,22 +723,22 @@ _Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) {
 // IMPRVD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// IMPRVD_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// IMPRVD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // IMPRVD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// IMPRVD_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// IMPRVD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// IMPRVD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // IMPRVD_FAST-NEXT:    [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // IMPRVD_FAST-NEXT:    [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // IMPRVD_FAST-NEXT:    [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]]
 // IMPRVD_FAST-NEXT:    [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]]
 // IMPRVD_FAST-NEXT:    [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]]
 // IMPRVD_FAST-NEXT:    [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]]
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_R]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_I]] to half
 // IMPRVD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -758,22 +758,22 @@ _Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[A_REAL]] to float
-// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext half [[A_IMAG]] to float
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // PRMTD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[B]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext half [[B_REAL]] to float
-// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext half [[B_IMAG]] to float
+// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_REAL]] to float
+// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[B_IMAG]] to float
 // PRMTD_FAST-NEXT:    [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]]
 // PRMTD_FAST-NEXT:    [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]]
 // PRMTD_FAST-NEXT:    [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]]
 // PRMTD_FAST-NEXT:    [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]]
 // PRMTD_FAST-NEXT:    [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]]
 // PRMTD_FAST-NEXT:    [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half
-// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_R]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[MUL_I]] to half
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -1158,8 +1158,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // BASIC_FAST-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // BASIC_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80
-// BASIC_FAST-NEXT:    [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80
+// BASIC_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_REAL]] to x86_fp80
+// BASIC_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_IMAG]] to x86_fp80
 // BASIC_FAST-NEXT:    [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV]]
 // BASIC_FAST-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV1]]
 // BASIC_FAST-NEXT:    [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]], [[TMP1]]
@@ -1171,16 +1171,16 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // BASIC_FAST-NEXT:    [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP7]]
 // BASIC_FAST-NEXT:    [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[TMP5]]
 // BASIC_FAST-NEXT:    [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP8]], [[TMP5]]
-// BASIC_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to half
-// BASIC_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to half
-// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext half [[CONV2]] to float
-// BASIC_FAST-NEXT:    [[EXT4:%.*]] = fpext half [[CONV3]] to float
+// BASIC_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]] to half
+// BASIC_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP10]] to half
+// BASIC_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV2]] to float
+// BASIC_FAST-NEXT:    [[EXT4:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV3]] to float
 // BASIC_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // BASIC_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// BASIC_FAST-NEXT:    [[EXT5:%.*]] = fpext half [[A_REAL]] to float
-// BASIC_FAST-NEXT:    [[EXT6:%.*]] = fpext half [[A_IMAG]] to float
+// BASIC_FAST-NEXT:    [[EXT5:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// BASIC_FAST-NEXT:    [[EXT6:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // BASIC_FAST-NEXT:    [[TMP11:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT5]]
 // BASIC_FAST-NEXT:    [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT6]]
 // BASIC_FAST-NEXT:    [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP11]], [[TMP12]]
@@ -1192,8 +1192,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // BASIC_FAST-NEXT:    [[TMP19:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP17]], [[TMP18]]
 // BASIC_FAST-NEXT:    [[TMP20:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP13]], [[TMP16]]
 // BASIC_FAST-NEXT:    [[TMP21:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP19]], [[TMP16]]
-// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half
-// BASIC_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc float [[TMP21]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP20]] to half
+// BASIC_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP21]] to half
 // BASIC_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // BASIC_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -1218,29 +1218,29 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // FULL_FAST-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // FULL_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80
-// FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80
+// FULL_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_REAL]] to x86_fp80
+// FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_IMAG]] to x86_fp80
 // FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR1]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
-// FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to half
-// FULL_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to half
-// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext half [[CONV2]] to float
-// FULL_FAST-NEXT:    [[EXT4:%.*]] = fpext half [[CONV3]] to float
+// FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]] to half
+// FULL_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP1]] to half
+// FULL_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV2]] to float
+// FULL_FAST-NEXT:    [[EXT4:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV3]] to float
 // FULL_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // FULL_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// FULL_FAST-NEXT:    [[EXT5:%.*]] = fpext half [[A_REAL]] to float
-// FULL_FAST-NEXT:    [[EXT6:%.*]] = fpext half [[A_IMAG]] to float
+// FULL_FAST-NEXT:    [[EXT5:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// FULL_FAST-NEXT:    [[EXT6:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // FULL_FAST-NEXT:    [[CALL7:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[EXT]], float noundef nofpclass(nan inf) [[EXT4]], float noundef nofpclass(nan inf) [[EXT5]], float noundef nofpclass(nan inf) [[EXT6]]) #[[ATTR1]]
 // FULL_FAST-NEXT:    store <2 x float> [[CALL7]], ptr [[COERCE]], align 4
 // FULL_FAST-NEXT:    [[COERCE_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4
 // FULL_FAST-NEXT:    [[COERCE_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4
-// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half
-// FULL_FAST-NEXT:    [[UNPROMOTION8:%.*]] = fptrunc float [[COERCE_IMAG]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[COERCE_REAL]] to half
+// FULL_FAST-NEXT:    [[UNPROMOTION8:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[COERCE_IMAG]] to half
 // FULL_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // FULL_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -1264,8 +1264,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // IMPRVD_FAST-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80
-// IMPRVD_FAST-NEXT:    [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80
+// IMPRVD_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_REAL]] to x86_fp80
+// IMPRVD_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_IMAG]] to x86_fp80
 // IMPRVD_FAST-NEXT:    [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]])
 // IMPRVD_FAST-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]])
 // IMPRVD_FAST-NEXT:    [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]]
@@ -1295,16 +1295,16 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // IMPRVD_FAST:       complex_div:
 // IMPRVD_FAST-NEXT:    [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
 // IMPRVD_FAST-NEXT:    [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
-// IMPRVD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half
-// IMPRVD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half
-// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[CONV2]] to float
-// IMPRVD_FAST-NEXT:    [[EXT4:%.*]] = fpext half [[CONV3]] to float
+// IMPRVD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP20]] to half
+// IMPRVD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP21]] to half
+// IMPRVD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV2]] to float
+// IMPRVD_FAST-NEXT:    [[EXT4:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV3]] to float
 // IMPRVD_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // IMPRVD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// IMPRVD_FAST-NEXT:    [[EXT5:%.*]] = fpext half [[A_REAL]] to float
-// IMPRVD_FAST-NEXT:    [[EXT6:%.*]] = fpext half [[A_IMAG]] to float
+// IMPRVD_FAST-NEXT:    [[EXT5:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// IMPRVD_FAST-NEXT:    [[EXT6:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // IMPRVD_FAST-NEXT:    [[TMP22:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT5]])
 // IMPRVD_FAST-NEXT:    [[TMP23:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT6]])
 // IMPRVD_FAST-NEXT:    [[ABS_CMP7:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP22]], [[TMP23]]
@@ -1334,8 +1334,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // IMPRVD_FAST:       complex_div10:
 // IMPRVD_FAST-NEXT:    [[TMP42:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP29]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP38]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ]
 // IMPRVD_FAST-NEXT:    [[TMP43:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP32]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP41]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ]
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP42]] to half
-// IMPRVD_FAST-NEXT:    [[UNPROMOTION11:%.*]] = fptrunc float [[TMP43]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP42]] to half
+// IMPRVD_FAST-NEXT:    [[UNPROMOTION11:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP43]] to half
 // IMPRVD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -1359,8 +1359,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD_FAST-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80
-// PRMTD_FAST-NEXT:    [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_REAL]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[C_IMAG]] to x86_fp80
 // PRMTD_FAST-NEXT:    [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]])
 // PRMTD_FAST-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]])
 // PRMTD_FAST-NEXT:    [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]]
@@ -1390,16 +1390,16 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD_FAST:       complex_div:
 // PRMTD_FAST-NEXT:    [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
 // PRMTD_FAST-NEXT:    [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
-// PRMTD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half
-// PRMTD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext half [[CONV2]] to float
-// PRMTD_FAST-NEXT:    [[EXT4:%.*]] = fpext half [[CONV3]] to float
+// PRMTD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP20]] to half
+// PRMTD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP21]] to half
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV2]] to float
+// PRMTD_FAST-NEXT:    [[EXT4:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[CONV3]] to float
 // PRMTD_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2
-// PRMTD_FAST-NEXT:    [[EXT5:%.*]] = fpext half [[A_REAL]] to float
-// PRMTD_FAST-NEXT:    [[EXT6:%.*]] = fpext half [[A_IMAG]] to float
+// PRMTD_FAST-NEXT:    [[EXT5:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_REAL]] to float
+// PRMTD_FAST-NEXT:    [[EXT6:%.*]] = fpext reassoc nnan ninf nsz arcp afn half [[A_IMAG]] to float
 // PRMTD_FAST-NEXT:    [[TMP22:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT5]]
 // PRMTD_FAST-NEXT:    [[TMP23:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT6]]
 // PRMTD_FAST-NEXT:    [[TMP24:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP22]], [[TMP23]]
@@ -1411,8 +1411,8 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // PRMTD_FAST-NEXT:    [[TMP30:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP28]], [[TMP29]]
 // PRMTD_FAST-NEXT:    [[TMP31:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP24]], [[TMP27]]
 // PRMTD_FAST-NEXT:    [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP30]], [[TMP27]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[TMP31]] to half
-// PRMTD_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc float [[TMP32]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP31]] to half
+// PRMTD_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn float [[TMP32]] to half
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
diff --git a/clang/test/CodeGen/alias.cpp b/clang/test/CodeGen/alias.cpp
index a468c31d369ed..43be6b99ce849 100644
--- a/clang/test/CodeGen/alias.cpp
+++ b/clang/test/CodeGen/alias.cpp
@@ -26,6 +26,9 @@ __attribute__((unused, alias("resolver"), deprecated("hahahaha, isn't C great?")
 void func();
 // expected-error@-2 {{alias must point to a defined variable or function}}
 // expected-note@-3 {{must refer to its mangled name}}
+
+void *operator new(unsigned long) __attribute__((alias("A"))); // expected-error {{alias must point to a defined variable or function}} \
+                                                               // expected-note {{the function or variable specified in an alias must refer to its mangled name}}
 #endif
 
 // CHECK: @_ZN4libc4log2Ed ={{.*}} alias double (double), ptr @log2
diff --git a/clang/test/CodeGen/allow-ubsan-check-inline.c b/clang/test/CodeGen/allow-ubsan-check-inline.c
new file mode 100644
index 0000000000000..1de24ab90dac0
--- /dev/null
+++ b/clang/test/CodeGen/allow-ubsan-check-inline.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -mllvm -ubsan-guard-checks -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check -fno-inline 2>&1 | FileCheck %s --check-prefixes=NOINL --implicit-check-not="remark:"
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-llvm -o - %s -fsanitize=signed-integer-overflow -mllvm -ubsan-guard-checks -O3 -mllvm -lower-allow-check-random-rate=1 -Rpass=lower-allow-check -Rpass-missed=lower-allow-check 2>&1 | FileCheck %s --check-prefixes=INLINE --implicit-check-not="remark:"
+
+int get();
+void set(int x);
+
+// We will only make decision in the `overflow` function.
+// NOINL-COUNT-1: remark: Allowed check:
+
+// We will make decision on every inline.
+// INLINE-COUNT-5: remark: Allowed check:
+
+static void overflow() {
+  set(get() + get());
+}
+
+void test() {
+  overflow();
+  overflow();
+  overflow();
+  overflow();
+  overflow();
+}
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/vadc.c b/clang/test/CodeGen/arm-mve-intrinsics/vadc.c
index 21087b83300c8..29e067d65c6c8 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/vadc.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/vadc.c
@@ -92,7 +92,7 @@ int32x4_t test_vadcq_m_s32(int32x4_t inactive, int32x4_t a, int32x4_t b, unsigne
 
 // CHECK-LABEL: @test_vsbciq_s32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 536870912)
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 1
 // CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 29
 // CHECK-NEXT:    [[TMP3:%.*]] = and i32 1, [[TMP2]]
@@ -110,7 +110,7 @@ int32x4_t test_vsbciq_s32(int32x4_t a, int32x4_t b, unsigned *carry_out) {
 
 // CHECK-LABEL: @test_vsbciq_u32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 536870912)
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 1
 // CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 29
 // CHECK-NEXT:    [[TMP3:%.*]] = and i32 1, [[TMP2]]
@@ -170,7 +170,7 @@ uint32x4_t test_vsbcq_u32(uint32x4_t a, uint32x4_t b, unsigned *carry) {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 536870912, <4 x i1> [[TMP1]])
 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 1
 // CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 29
 // CHECK-NEXT:    [[TMP5:%.*]] = and i32 1, [[TMP4]]
@@ -190,7 +190,7 @@ int32x4_t test_vsbciq_m_s32(int32x4_t inactive, int32x4_t a, int32x4_t b, unsign
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 536870912, <4 x i1> [[TMP1]])
 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 1
 // CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 29
 // CHECK-NEXT:    [[TMP5:%.*]] = and i32 1, [[TMP4]]
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 7953618d2f9d1..74c6f2f79e632 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -164,12 +164,12 @@ void check__writex18byte(unsigned LONG offset, unsigned char data) {
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i8 %data, ptr %[[DATA_ADDR]], align 1
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i8, ptr %[[DATA_ADDR]], align 1
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
-// CHECK-MSCOMPAT: %[[DATA:.*]] = load i8, ptr %[[DATA_ADDR]], align 1
 // CHECK-MSCOMPAT: store i8 %[[DATA]], ptr %[[PTR]], align 1
 
 #ifdef __LP64__
@@ -184,12 +184,12 @@ void check__writex18word(unsigned LONG offset, unsigned short data) {
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i16 %data, ptr %[[DATA_ADDR]], align 2
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i16, ptr %[[DATA_ADDR]], align 2
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
-// CHECK-MSCOMPAT: %[[DATA:.*]] = load i16, ptr %[[DATA_ADDR]], align 2
 // CHECK-MSCOMPAT: store i16 %[[DATA]], ptr %[[PTR]], align 1
 
 #ifdef __LP64__
@@ -204,12 +204,12 @@ void check__writex18dword(unsigned LONG offset, unsigned LONG data) {
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i32 %data, ptr %[[DATA_ADDR]], align 4
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i32, ptr %[[DATA_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
-// CHECK-MSCOMPAT: %[[DATA:.*]] = load i32, ptr %[[DATA_ADDR]], align 4
 // CHECK-MSCOMPAT: store i32 %[[DATA]], ptr %[[PTR]], align 1
 
 #ifdef __LP64__
@@ -224,12 +224,12 @@ void check__writex18qword(unsigned LONG offset, unsigned __int64 data) {
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i64 %data, ptr %[[DATA_ADDR]], align 8
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i64, ptr %[[DATA_ADDR]], align 8
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
-// CHECK-MSCOMPAT: %[[DATA:.*]] = load i64, ptr %[[DATA_ADDR]], align 8
 // CHECK-MSCOMPAT: store i64 %[[DATA]], ptr %[[PTR]], align 1
 
 unsigned char check__readx18byte(unsigned LONG offset) {
@@ -238,9 +238,9 @@ unsigned char check__readx18byte(unsigned LONG offset) {
 
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
 // CHECK-MSCOMPAT: %[[RETVAL:.*]] = load i8, ptr %[[PTR]], align 1
@@ -252,9 +252,9 @@ unsigned short check__readx18word(unsigned LONG offset) {
 
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
 // CHECK-MSCOMPAT: %[[RETVAL:.*]] = load i16, ptr %[[PTR]], align 1
@@ -266,9 +266,9 @@ unsigned LONG check__readx18dword(unsigned LONG offset) {
 
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
 // CHECK-MSCOMPAT: %[[RETVAL:.*]] = load i32, ptr %[[PTR]], align 1
@@ -280,14 +280,162 @@ unsigned __int64 check__readx18qword(unsigned LONG offset) {
 
 // CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
 // CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
 // CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
-// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
 // CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
 // CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
 // CHECK-MSCOMPAT: %[[RETVAL:.*]] = load i64, ptr %[[PTR]], align 1
 // CHECK-MSCOMPAT: ret i64 %[[RETVAL]]
 
+#ifdef __LP64__
+void check__addx18byte(unsigned char data, unsigned LONG offset) {
+#else
+void check__addx18byte(unsigned LONG offset, unsigned char data) {
+#endif
+  __addx18byte(offset, data);
+}
+
+// CHECK-MSCOMPAT: %[[DATA_ADDR:.*]] = alloca i8, align 1
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i8 %data, ptr %[[DATA_ADDR]], align 1
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i8, ptr %[[DATA_ADDR]], align 1
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i8, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i8 %[[ORIG_VAL]], %[[DATA]]
+// CHECK-MSCOMPAT: store i8 %[[SUM]], ptr %[[PTR]], align 1
+
+#ifdef __LP64__
+void check__addx18word(unsigned short data, unsigned LONG offset) {
+#else
+void check__addx18word(unsigned LONG offset, unsigned short data) {
+#endif
+  __addx18word(offset, data);
+}
+
+// CHECK-MSCOMPAT: %[[DATA_ADDR:.*]] = alloca i16, align 2
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i16 %data, ptr %[[DATA_ADDR]], align 2
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i16, ptr %[[DATA_ADDR]], align 2
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i16, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i16 %[[ORIG_VAL]], %[[DATA]]
+// CHECK-MSCOMPAT: store i16 %[[SUM]], ptr %[[PTR]], align 1
+
+#ifdef __LP64__
+void check__addx18dword(unsigned LONG data, unsigned LONG offset) {
+#else
+void check__addx18dword(unsigned LONG offset, unsigned LONG data) {
+#endif
+  __addx18dword(offset, data);
+}
+
+// CHECK-MSCOMPAT: %[[DATA_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i32 %data, ptr %[[DATA_ADDR]], align 4
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i32, ptr %[[DATA_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i32, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i32 %[[ORIG_VAL]], %[[DATA]]
+// CHECK-MSCOMPAT: store i32 %[[SUM]], ptr %[[PTR]], align 1
+
+#ifdef __LP64__
+void check__addx18qword(unsigned __int64 data, unsigned LONG offset) {
+#else
+void check__addx18qword(unsigned LONG offset, unsigned __int64 data) {
+#endif
+  __addx18qword(offset, data);
+}
+
+// CHECK-MSCOMPAT: %[[DATA_ADDR:.*]] = alloca i64, align 8
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i64 %data, ptr %[[DATA_ADDR]], align 8
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[DATA:.*]] = load i64, ptr %[[DATA_ADDR]], align 8
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i64, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i64 %[[ORIG_VAL]], %[[DATA]]
+// CHECK-MSCOMPAT: store i64 %[[SUM]], ptr %[[PTR]], align 1
+
+void check__incx18byte(unsigned LONG offset) {
+  __incx18byte(offset);
+}
+
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i8, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i8 %[[ORIG_VAL]], 1
+// CHECK-MSCOMPAT: store i8 %[[SUM]], ptr %[[PTR]], align 1
+
+void check__incx18word(unsigned LONG offset) {
+  __incx18word(offset);
+}
+
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i16, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i16 %[[ORIG_VAL]], 1
+// CHECK-MSCOMPAT: store i16 %[[SUM]], ptr %[[PTR]], align 1
+
+void check__incx18dword(unsigned LONG offset) {
+  __incx18dword(offset);
+}
+
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i32, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i32 %[[ORIG_VAL]], 1
+// CHECK-MSCOMPAT: store i32 %[[SUM]], ptr %[[PTR]], align 1
+
+void check__incx18qword(unsigned LONG offset) {
+  __incx18qword(offset);
+}
+
+// CHECK-MSCOMPAT: %[[OFFSET_ADDR:.*]] = alloca i32, align 4
+// CHECK-MSCOMPAT: store i32 %offset, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[OFFSET:.*]] = load i32, ptr %[[OFFSET_ADDR]], align 4
+// CHECK-MSCOMPAT: %[[X18:.*]] = call i64 @llvm.read_register.i64(metadata ![[MD2]])
+// CHECK-MSCOMPAT: %[[X18_AS_PTR:.*]] = inttoptr i64 %[[X18]] to ptr
+// CHECK-MSCOMPAT: %[[ZEXT_OFFSET:.*]] = zext i32 %[[OFFSET]] to i64
+// CHECK-MSCOMPAT: %[[PTR:.*]] = getelementptr i8, ptr %[[X18_AS_PTR]], i64 %[[ZEXT_OFFSET]]
+// CHECK-MSCOMPAT: %[[ORIG_VAL:.*]] = load i64, ptr %[[PTR]], align 1
+// CHECK-MSCOMPAT: %[[SUM:.*]] = add i64 %[[ORIG_VAL]], 1
+// CHECK-MSCOMPAT: store i64 %[[SUM]], ptr %[[PTR]], align 1
+
 double check__CopyDoubleFromInt64(__int64 arg1) {
   return _CopyDoubleFromInt64(arg1);
 }
diff --git a/clang/test/CodeGen/arm64_32-vaarg.c b/clang/test/CodeGen/arm64_32-vaarg.c
index 3f1f4443436da..72c23d4967d2d 100644
--- a/clang/test/CodeGen/arm64_32-vaarg.c
+++ b/clang/test/CodeGen/arm64_32-vaarg.c
@@ -10,7 +10,7 @@ typedef struct {
 int test_int(OneInt input, va_list *mylist) {
 // CHECK-LABEL: define{{.*}} i32 @test_int(i32 %input
 // CHECK: [[START:%.*]] = load ptr, ptr %mylist
-// CHECK: [[NEXT:%.*]] = getelementptr inbounds i8, ptr [[START]], i32 4
+// CHECK: [[NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[START]], i32 4
 // CHECK: store ptr [[NEXT]], ptr %mylist
 
 // CHECK: [[RES:%.*]] = load i32, ptr [[START]]
@@ -28,9 +28,9 @@ typedef struct {
 long long test_longlong(OneLongLong input, va_list *mylist) {
   // CHECK-LABEL: define{{.*}} i64 @test_longlong(i64 %input
   // CHECK: [[STARTPTR:%.*]] = load ptr, ptr %mylist
-  // CHECK: [[ALIGN_TMP:%.+]] = getelementptr inbounds i8, ptr [[STARTPTR]], i32 7
+  // CHECK: [[ALIGN_TMP:%.+]] = getelementptr inbounds nuw i8, ptr [[STARTPTR]], i32 7
   // CHECK: [[ALIGNED_ADDR:%.+]] = tail call align 8 ptr @llvm.ptrmask.p0.i32(ptr nonnull [[ALIGN_TMP]], i32 -8)
-  // CHECK: [[NEXT:%.*]] = getelementptr inbounds i8, ptr [[ALIGNED_ADDR]], i32 8
+  // CHECK: [[NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[ALIGNED_ADDR]], i32 8
   // CHECK: store ptr [[NEXT]], ptr %mylist
 
   // CHECK: [[RES:%.*]] = load i64, ptr [[ALIGNED_ADDR]]
@@ -49,7 +49,7 @@ float test_hfa(va_list *mylist) {
 // CHECK-LABEL: define{{.*}} float @test_hfa
 // CHECK: [[START:%.*]] = load ptr, ptr %mylist
 
-// CHECK: [[NEXT:%.*]] = getelementptr inbounds i8, ptr [[START]], i32 16
+// CHECK: [[NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[START]], i32 16
 // CHECK: store ptr [[NEXT]], ptr %mylist
 
 // CHECK: [[RES:%.*]] = load float, ptr [[START]]
@@ -76,7 +76,7 @@ typedef struct {
 long long test_bigstruct(BigStruct input, va_list *mylist) {
 // CHECK-LABEL: define{{.*}} i64 @test_bigstruct(ptr
 // CHECK: [[START:%.*]] = load ptr, ptr %mylist
-// CHECK: [[NEXT:%.*]] = getelementptr inbounds i8, ptr [[START]], i32 4
+// CHECK: [[NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[START]], i32 4
 // CHECK: store ptr [[NEXT]], ptr %mylist
 
 // CHECK: [[ADDR:%.*]] = load ptr, ptr [[START]]
@@ -97,7 +97,7 @@ short test_threeshorts(ThreeShorts input, va_list *mylist) {
 // CHECK-LABEL: define{{.*}} signext i16 @test_threeshorts([2 x i32] %input
 
 // CHECK: [[START:%.*]] = load ptr, ptr %mylist
-// CHECK: [[NEXT:%.*]] = getelementptr inbounds i8, ptr [[START]], i32 8
+// CHECK: [[NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[START]], i32 8
 // CHECK: store ptr [[NEXT]], ptr %mylist
 
 // CHECK: [[RES:%.*]] = load i16, ptr [[START]]
diff --git a/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c b/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
index a7c87e32a58eb..c5a410193bfb7 100644
--- a/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
+++ b/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
@@ -32,21 +32,21 @@ DEFINE_STRUCT(bool)
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_int64(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-512-LABEL: @read_int64(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 64
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 svint64_t read_int64(struct struct_int64 *s) {
@@ -86,21 +86,21 @@ void write_int64(struct struct_int64 *s, svint64_t x) {
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_float64(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
 // CHECK-512-LABEL: @read_float64(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 64
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> poison, <8 x double> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
 svfloat64_t read_float64(struct struct_float64 *s) {
@@ -140,21 +140,21 @@ void write_float64(struct struct_float64 *s, svfloat64_t x) {
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 16
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <8 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> poison, <8 x bfloat> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_bfloat16(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 32
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <16 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> poison, <16 x bfloat> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
 // CHECK-512-LABEL: @read_bfloat16(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 64
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <32 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> undef, <32 x bfloat> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> poison, <32 x bfloat> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
 svbfloat16_t read_bfloat16(struct struct_bfloat16 *s) {
@@ -194,7 +194,7 @@ void write_bfloat16(struct struct_bfloat16 *s, svbfloat16_t x) {
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 2
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-128-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
@@ -202,7 +202,7 @@ void write_bfloat16(struct struct_bfloat16 *s, svbfloat16_t x) {
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 4
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v4i8(<vscale x 2 x i8> undef, <4 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v4i8(<vscale x 2 x i8> poison, <4 x i8> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-256-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
@@ -210,7 +210,7 @@ void write_bfloat16(struct struct_bfloat16 *s, svbfloat16_t x) {
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 8
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-512-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
diff --git a/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c b/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
index 8be1123a2305a..e1e2220f94d6d 100644
--- a/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
+++ b/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
@@ -75,7 +75,7 @@ svint64_t lax_cast(fixed_int32_t type) {
 // CHECK-LABEL: @to_svint32_t__from_gnu_int32_t(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA6]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 svint32_t to_svint32_t__from_gnu_int32_t(gnu_int32_t type) {
@@ -95,7 +95,7 @@ gnu_int32_t from_svint32_t__to_gnu_int32_t(svint32_t type) {
 // CHECK-LABEL: @to_fixed_int32_t__from_gnu_int32_t(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA6]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t to_fixed_int32_t__from_gnu_int32_t(gnu_int32_t type) {
diff --git a/clang/test/CodeGen/attr-arm-sve-vector-bits-codegen.c b/clang/test/CodeGen/attr-arm-sve-vector-bits-codegen.c
index 5db75547abda1..06fbb0027d7c1 100644
--- a/clang/test/CodeGen/attr-arm-sve-vector-bits-codegen.c
+++ b/clang/test/CodeGen/attr-arm-sve-vector-bits-codegen.c
@@ -24,23 +24,23 @@ fixed_int32_t global_vec;
 // CHECK-NEXT:    store <vscale x 4 x i32> [[VEC:%.*]], ptr [[VEC_ADDR]], align 16
 // CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 16 x i1>, ptr [[PRED_ADDR]], align 2
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @global_pred, align 2
-// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP1]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTFIXEDSVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr @global_pred, align 2
-// CHECK-NEXT:    [[CASTFIXEDSVE2:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[CASTFIXEDSVE2:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP3]], i64 0)
 // CHECK-NEXT:    [[TMP4:%.*]] = bitcast <vscale x 2 x i8> [[CASTFIXEDSVE2]] to <vscale x 16 x i1>
 // CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.and.z.nxv16i1(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP4]])
 // CHECK-NEXT:    store <vscale x 16 x i1> [[TMP5]], ptr [[PG]], align 2
 // CHECK-NEXT:    [[TMP6:%.*]] = load <vscale x 16 x i1>, ptr [[PG]], align 2
 // CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i32>, ptr @global_vec, align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP7]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP7]], i64 0)
 // CHECK-NEXT:    [[TMP8:%.*]] = load <vscale x 4 x i32>, ptr [[VEC_ADDR]], align 16
 // CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP6]])
 // CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.add.nxv4i32(<vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[CASTSCALABLESVE]], <vscale x 4 x i32> [[TMP8]])
 // CHECK-NEXT:    [[CASTFIXEDSVE3:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TMP10]], i64 0)
 // CHECK-NEXT:    store <16 x i32> [[CASTFIXEDSVE3]], ptr [[RETVAL]], align 16
 // CHECK-NEXT:    [[TMP11:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP11]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP11]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE4]]
 //
 fixed_int32_t foo(svbool_t pred, svint32_t vec) {
@@ -57,7 +57,7 @@ fixed_int32_t foo(svbool_t pred, svint32_t vec) {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr [[TMP0]], align 16
 // CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr [[RETVAL]], align 16
 // CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t test_ptr_to_global() {
@@ -78,7 +78,7 @@ fixed_int32_t test_ptr_to_global() {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 16
 // CHECK-NEXT:    store <16 x i32> [[TMP1]], ptr [[RETVAL]], align 16
 // CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP2]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 //
 fixed_int32_t array_arg(fixed_int32_t arr[]) {
@@ -96,7 +96,7 @@ fixed_int32_t array_arg(fixed_int32_t arr[]) {
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[TMP0]], align 2
 // CHECK-NEXT:    store <8 x i8> [[TMP1]], ptr [[RETVAL]], align 2
 // CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr [[RETVAL]], align 2
-// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP2]], i64 0)
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <vscale x 2 x i8> [[CASTFIXEDSVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP3]]
 //
@@ -121,25 +121,25 @@ fixed_bool_t address_of_array_idx() {
 // CHECK-NEXT:    store <8 x i8> <i8 2, i8 5, i8 4, i8 6, i8 0, i8 0, i8 0, i8 0>, ptr [[YY]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 16 x i1>, ptr [[PRED_ADDR]], align 2
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @global_pred, align 2
-// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP1]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[CASTFIXEDSVE]] to <vscale x 16 x i1>
 // CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr [[XX]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr [[YY]], align 8
 // CHECK-NEXT:    [[ADD:%.*]] = add <8 x i8> [[TMP3]], [[TMP4]]
-// CHECK-NEXT:    [[CASTFIXEDSVE2:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    [[CASTFIXEDSVE2:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[ADD]], i64 0)
 // CHECK-NEXT:    [[TMP5:%.*]] = bitcast <vscale x 2 x i8> [[CASTFIXEDSVE2]] to <vscale x 16 x i1>
 // CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.and.z.nxv16i1(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP5]])
 // CHECK-NEXT:    store <vscale x 16 x i1> [[TMP6]], ptr [[PG]], align 2
 // CHECK-NEXT:    [[TMP7:%.*]] = load <vscale x 16 x i1>, ptr [[PG]], align 2
 // CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr @global_vec, align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP8]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP8]], i64 0)
 // CHECK-NEXT:    [[TMP9:%.*]] = load <vscale x 4 x i32>, ptr [[VEC_ADDR]], align 16
 // CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP7]])
 // CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.add.nxv4i32(<vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[CASTSCALABLESVE]], <vscale x 4 x i32> [[TMP9]])
 // CHECK-NEXT:    [[CASTFIXEDSVE3:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TMP11]], i64 0)
 // CHECK-NEXT:    store <16 x i32> [[CASTFIXEDSVE3]], ptr [[RETVAL]], align 16
 // CHECK-NEXT:    [[TMP12:%.*]] = load <16 x i32>, ptr [[RETVAL]], align 16
-// CHECK-NEXT:    [[CASTSCALABLESVE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP12]], i64 0)
+// CHECK-NEXT:    [[CASTSCALABLESVE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP12]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE4]]
 //
 fixed_int32_t test_cast(svbool_t pred, svint32_t vec) {
diff --git a/clang/test/CodeGen/attr-arm-sve-vector-bits-globals.c b/clang/test/CodeGen/attr-arm-sve-vector-bits-globals.c
index 7858a7bf0026e..011518c60f52f 100644
--- a/clang/test/CodeGen/attr-arm-sve-vector-bits-globals.c
+++ b/clang/test/CodeGen/attr-arm-sve-vector-bits-globals.c
@@ -71,13 +71,13 @@ void write_global_bool(svbool_t v) { global_bool = v; }
 // CHECK-128-LABEL: @read_global_i64(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @global_i64, align 16, !tbaa [[TBAA6]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 // CHECK-512-LABEL: @read_global_i64(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr @global_i64, align 16, !tbaa [[TBAA6]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> poison, <8 x i64> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 //
 svint64_t read_global_i64() { return global_i64; }
@@ -85,13 +85,13 @@ svint64_t read_global_i64() { return global_i64; }
 // CHECK-128-LABEL: @read_global_bf16(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <8 x bfloat>, ptr @global_bf16, align 16, !tbaa [[TBAA6]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> poison, <8 x bfloat> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    ret <vscale x 8 x bfloat> [[CASTSCALABLESVE]]
 //
 // CHECK-512-LABEL: @read_global_bf16(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <32 x bfloat>, ptr @global_bf16, align 16, !tbaa [[TBAA6]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> undef, <32 x bfloat> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> poison, <32 x bfloat> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    ret <vscale x 8 x bfloat> [[CASTSCALABLESVE]]
 //
 svbfloat16_t read_global_bf16() { return global_bf16; }
@@ -99,14 +99,14 @@ svbfloat16_t read_global_bf16() { return global_bf16; }
 // CHECK-128-LABEL: @read_global_bool(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr @global_bool, align 2, !tbaa [[TBAA6]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-128-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
 // CHECK-512-LABEL: @read_global_bool(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @global_bool, align 2, !tbaa [[TBAA6]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> poison, <8 x i8> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
 // CHECK-512-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
diff --git a/clang/test/CodeGen/attr-counted-by-pr110385.c b/clang/test/CodeGen/attr-counted-by-pr110385.c
index e120dcc583578..c2ff032334fe2 100644
--- a/clang/test/CodeGen/attr-counted-by-pr110385.c
+++ b/clang/test/CodeGen/attr-counted-by-pr110385.c
@@ -31,7 +31,7 @@ void init(void * __attribute__((pass_dynamic_object_size(0))));
 // CHECK-NEXT:    [[GROWABLE:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[GROWABLE]], align 8, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
-// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 // CHECK-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[TMP1]], 1
@@ -48,7 +48,7 @@ void test1(struct bucket *foo) {
 // CHECK-SAME: ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 16
-// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 12
+// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 12
 // CHECK-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 1
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index f70e552bca26a..be4c7f07e9215 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -60,17 +60,17 @@ struct anon_struct {
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9:[0-9]+]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[VAL]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -108,31 +108,29 @@ void test1(struct annotated *p, int index, int val) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test2(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i32 [[DOT_COUNTED_BY_LOAD]], 0
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP2]], 2
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test2(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i32 [[DOT_COUNTED_BY_LOAD]], 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP0]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP0]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[INDEX]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
@@ -161,7 +159,7 @@ void test2(struct annotated *p, size_t index) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
 // SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
@@ -172,7 +170,7 @@ void test2(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
@@ -197,13 +195,13 @@ size_t test2_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test3(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -268,7 +266,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[FAM_IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
@@ -276,7 +274,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont4:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD]], 2
@@ -284,53 +282,50 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 244
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 252
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV1:%.*]] = select i1 [[TMP2]], i32 [[TMP5]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD6:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD6]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[IDXPROM12]], [[TMP6]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP7]], label [[CONT19:%.*]], label [[HANDLER_OUT_OF_BOUNDS15:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[IDXPROM12]], [[TMP0]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP6]], label [[CONT19:%.*]], label [[HANDLER_OUT_OF_BOUNDS15:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds15:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM12]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM12]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont19:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD6]], 3
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD6]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], 240
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV8:%.*]] = select i1 [[TMP8]], i32 [[TMP11]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM12]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = add i32 [[TMP3]], 240
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 252
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV8:%.*]] = select i1 [[TMP7]], i32 [[TMP9]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM12]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV8]], ptr [[ARRAYIDX17]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD21:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[INDEX]], 2
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM28:%.*]] = sext i32 [[ADD27]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD21]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[IDXPROM28]], [[TMP12]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP13]], label [[CONT35:%.*]], label [[HANDLER_OUT_OF_BOUNDS31:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD21]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[IDXPROM28]], [[TMP10]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP11]], label [[CONT35:%.*]], label [[HANDLER_OUT_OF_BOUNDS31:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds31:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 [[IDXPROM28]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 [[IDXPROM28]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont35:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM28]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD21]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP16:%.*]] = sext i32 [[FAM_IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP17:%.*]] = sub nsw i64 [[TMP15]], [[TMP16]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP18:%.*]] = icmp sgt i64 [[TMP17]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP19:%.*]] = and i1 [[TMP14]], [[TMP18]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP17]] to i32
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP20:%.*]] = shl i32 [[DOTTR]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV23:%.*]] = select i1 [[TMP19]], i32 [[TMP21]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM28]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD21]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = sext i32 [[FAM_IDX]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = sub nsw i64 [[TMP13]], [[TMP14]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP16:%.*]] = icmp sgt i64 [[TMP15]], -1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP17:%.*]] = and i1 [[TMP12]], [[TMP16]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP15]] to i32
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP18:%.*]] = shl i32 [[DOTTR]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 252
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV23:%.*]] = select i1 [[TMP17]], i32 [[TMP19]], i32 0
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV23]], ptr [[ARRAYIDX33]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[FAM_IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], 244
@@ -412,7 +407,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -17179869180, 17179869181) i64 @test4_bdos(
 // SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
@@ -427,7 +422,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -17179869180, 17179869181) i64 @test4_bdos(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
@@ -457,12 +452,12 @@ size_t test4_bdos(struct annotated *p, int index) {
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i64, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[DOTCOUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -528,33 +523,31 @@ size_t test5_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test6(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i64, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[DOT_COUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[TMP1]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i64 [[DOT_COUNTED_BY_LOAD]], 0
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD_TR:%.*]] = trunc i64 [[DOT_COUNTED_BY_LOAD]] to i32
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD_TR]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.smax.i64(i64 [[DOT_COUNTED_BY_LOAD]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP2]] to i32
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[DOTTR]], 2
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test6(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i64, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i64 [[DOT_COUNTED_BY_LOAD]], 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD_TR:%.*]] = trunc i64 [[DOT_COUNTED_BY_LOAD]] to i32
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD_TR]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP0]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[DOT_COUNTED_BY_LOAD]], i64 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP0]] to i32
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[DOTTR]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[TMP1]], i64 0, i64 [[IDXPROM]]
@@ -586,21 +579,19 @@ void test6(struct anon_struct *p, int index) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -3) i64 @test6_bdos(
 // SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i64, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl nuw i64 [[DOT_COUNTED_BY_LOAD]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i64 [[DOT_COUNTED_BY_LOAD]], 0
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], i64 0, i64 [[TMP0]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[DOT_COUNTED_BY_LOAD]], i64 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -3) i64 @test6_bdos(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i64, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = shl nuw i64 [[DOT_COUNTED_BY_LOAD]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i64 [[DOT_COUNTED_BY_LOAD]], 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], i64 0, i64 [[TMP0]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.smax.i64(i64 [[DOT_COUNTED_BY_LOAD]], i64 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test6_bdos(
@@ -620,18 +611,18 @@ size_t test6_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test7(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i8 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -693,27 +684,27 @@ size_t test7_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test8(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i8, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[DOT_COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
-// SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 9
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i8 [[DOT_COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test8(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i8, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 9
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[INTS]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i8 [[DOT_COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
@@ -744,7 +735,7 @@ void test8(struct union_of_fams *p, int index) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
 // SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i8, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[DOT_COUNTED_BY_LOAD]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
@@ -752,7 +743,7 @@ void test8(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i8, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i8 [[DOT_COUNTED_BY_LOAD]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
@@ -774,18 +765,18 @@ size_t test8_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test9(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB14:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB14:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -847,18 +838,18 @@ size_t test9_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test10(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
-// SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i32 [[NARROW]] to i8
 // SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA8]]
@@ -867,11 +858,11 @@ size_t test9_bdos(struct union_of_fams *p) {
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test10(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = trunc i32 [[NARROW]] to i8
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i8], ptr [[BYTES]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i8 [[CONV]], ptr [[ARRAYIDX]], align 1, !tbaa [[TBAA6]]
@@ -902,7 +893,7 @@ void test10(struct union_of_fams *p, int index) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
 // SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[NARROW]] to i64
@@ -911,7 +902,7 @@ void test10(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[NARROW:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[NARROW]] to i64
@@ -935,17 +926,17 @@ size_t test10_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 4, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -1023,26 +1014,26 @@ int test12_a, test12_b;
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR10:[0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT9:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[TMP2]], ptr @test12_b, align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr @test12_foo, align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 0) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1052,7 +1043,7 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR11:[0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[TMP0]], ptr @test12_b, align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4), align 4, !tbaa [[TBAA2]]
@@ -1065,26 +1056,26 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR6:[0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR8:[0-9]+]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR7:[0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont:
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    store i32 [[TMP2]], ptr @test12_b, align 4, !tbaa [[TBAA2]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr @test12_foo, align 4
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1094,7 +1085,7 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 [[TMP0]], ptr @test12_b, align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4), align 4, !tbaa [[TBAA2]]
@@ -1126,18 +1117,18 @@ struct test13_bar {
 // SANITIZE-WITH-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 [[INDEX]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
-// SANITIZE-WITH-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
+// SANITIZE-WITH-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA15:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 0
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test13(
@@ -1146,25 +1137,25 @@ struct test13_bar {
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA8:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA12:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 0
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test13(
 // SANITIZE-WITHOUT-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont5:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA15:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 0
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test13(
@@ -1173,7 +1164,7 @@ struct test13_bar {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA8:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[REVMAP]], i64 0, i64 [[INDEX]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11:![0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store ptr null, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA12:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 0
 //
 int test13(long index) {
@@ -1190,14 +1181,13 @@ struct test14_foo {
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[TRAP:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       trap:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @llvm.trap() #[[ATTR9]]
-// SANITIZE-WITH-ATTR-NEXT:    unreachable
+// SANITIZE-WITH-ATTR:       cont3:
+// SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test14(
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
@@ -1216,14 +1206,13 @@ struct test14_foo {
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
-// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[TRAP:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR:       trap:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @llvm.trap() #[[ATTR8]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable
+// SANITIZE-WITHOUT-ATTR:       cont3:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test14(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
@@ -1246,14 +1235,13 @@ int test14(int idx) {
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[TRAP:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB27:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB27:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       trap:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @llvm.trap() #[[ATTR9]]
-// SANITIZE-WITH-ATTR-NEXT:    unreachable
+// SANITIZE-WITH-ATTR:       cont1:
+// SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15(
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
@@ -1267,14 +1255,13 @@ int test14(int idx) {
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
-// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[TRAP:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR:       trap:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @llvm.trap() #[[ATTR8]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    unreachable
+// SANITIZE-WITHOUT-ATTR:       cont1:
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
@@ -1426,36 +1413,36 @@ struct tests_foo {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test24(
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 40
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 40
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 10) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont4:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 84
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test24(
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 84
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test24(
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 84
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test24(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR6:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 84
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP0]]
 //
@@ -1467,39 +1454,39 @@ int test24(int c, struct tests_foo *var) {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test25(
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 10) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 44
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test25(
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 44
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test25(
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 44
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test25(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr nocapture noundef readonly [[VAR:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 44
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 [[TMP1]]
 //
@@ -1517,18 +1504,18 @@ struct test26_foo {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test26(
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[S:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4
+// SANITIZE-WITH-ATTR-NEXT:    [[S:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[S]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB30:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB30:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP2]]
 //
@@ -1589,18 +1576,18 @@ struct test27_foo {
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[ENTRIES]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x ptr], ptr [[ENTRIES]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[J]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [[STRUCT_TEST27_BAR:%.*]], ptr [[TMP2]], i64 [[IDXPROM4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret ptr [[ARRAYIDX5]]
@@ -1611,7 +1598,7 @@ struct test27_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[ENTRIES]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA16:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[J]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [[STRUCT_TEST27_BAR:%.*]], ptr [[TMP0]], i64 [[IDXPROM1]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret ptr [[ARRAYIDX2]]
@@ -1622,7 +1609,7 @@ struct test27_foo {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[ENTRIES]], i64 0, i64 [[IDXPROM]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM3:%.*]] = sext i32 [[J]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCT_TEST27_BAR:%.*]], ptr [[TMP0]], i64 [[IDXPROM3]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret ptr [[ARRAYIDX4]]
@@ -1633,7 +1620,7 @@ struct test27_foo {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x ptr], ptr [[ENTRIES]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA16:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[J]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [[STRUCT_TEST27_BAR:%.*]], ptr [[TMP0]], i64 [[IDXPROM1]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret ptr [[ARRAYIDX2]]
@@ -1651,30 +1638,30 @@ struct test28_foo {
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test28(
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA21]]
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT17:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont17:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP5]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test28(
 // NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA11]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA18:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA18]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA18]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
@@ -1684,9 +1671,9 @@ struct test28_foo {
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test28(
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA21]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
@@ -1696,9 +1683,9 @@ struct test28_foo {
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test28(
 // NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef readonly [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA11]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA18:![0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA18]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !tbaa [[TBAA18]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARR]], i64 0, i64 [[IDXPROM]]
@@ -1723,26 +1710,25 @@ struct annotated_struct_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB36:[0-9]+]], i64 [[TMP1]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB36:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA23:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM15:%.*]] = sext i32 [[IDX2]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM15]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT20:%.*]], label [[HANDLER_OUT_OF_BOUNDS16:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds16:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM15]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM15]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont20:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM15]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i32 [[DOT_COUNTED_BY_LOAD]], 0
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP5]]
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM15]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP5]], 2
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX18]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
@@ -1751,12 +1737,11 @@ struct annotated_struct_array {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX1]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA20:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTINV:%.*]] = icmp slt i32 [[DOT_COUNTED_BY_LOAD]], 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = select i1 [[DOTINV]], i32 0, i32 [[TMP1]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.smax.i32(i32 [[DOT_COUNTED_BY_LOAD]], i32 0)
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = shl i32 [[TMP1]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[IDX2]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM4]]
@@ -1770,11 +1755,11 @@ struct annotated_struct_array {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT21:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont21:
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA14]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA23:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM18:%.*]] = sext i32 [[IDX2]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM18]]
@@ -1786,7 +1771,7 @@ struct annotated_struct_array {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX1]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x ptr], ptr [[ANN]], i64 0, i64 [[IDXPROM]]
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA11]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA20:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[IDX2]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM5]]
@@ -1812,7 +1797,7 @@ struct test30_struct {
 // SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR4]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[TMP0]]) #[[ATTR9]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
@@ -1828,7 +1813,7 @@ struct test30_struct {
 // SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR7]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index 2033a8b4c335f..c92aad633082f 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -1,80 +1,115 @@
 // RUN: %clang_cc1 -triple i686-linux-gnu -target-cpu i686 -tune-cpu i686 -emit-llvm %s -o - | FileCheck %s
 
-int baz(int a) { return 4; }
+// CHECK: define {{.*}}@f_default({{.*}} [[f_default:#[0-9]+]]
+// CHECK: define {{.*}}@f_avx_sse4_2_ivybridge({{.*}} [[f_avx_sse4_2_ivybridge:#[0-9]+]]
+// CHECK: define {{.*}}@f_fpmath_387({{.*}} [[f_default]]
+// CHECK: define {{.*}}@f_no_sse2({{.*}} [[f_no_sse2:#[0-9]+]]
+// CHECK: define {{.*}}@f_sse4({{.*}} [[f_sse4:#[0-9]+]]
+// CHECK: define {{.*}}@f_no_sse4({{.*}} [[f_no_sse4:#[0-9]+]]
+// CHECK: define {{.*}}@f_default2({{.*}} [[f_default]]
+// CHECK: define {{.*}}@f_avx_sse4_2_ivybridge_2({{.*}} [[f_avx_sse4_2_ivybridge]]
+// CHECK: define {{.*}}@f_no_aes_ivybridge({{.*}} [[f_no_aes_ivybridge:#[0-9]+]]
+// CHECK: define {{.*}}@f_no_mmx({{.*}} [[f_no_mmx:#[0-9]+]]
+// CHECK: define {{.*}}@f_lakemont_mmx({{.*}} [[f_lakemont_mmx:#[0-9]+]]
+// CHECK: define {{.*}}@f_use_before_def({{.*}} [[f_lakemont_mmx]]
+// CHECK: define {{.*}}@f_tune_sandybridge({{.*}} [[f_tune_sandybridge:#[0-9]+]]
+// CHECK: define {{.*}}@f_x86_64_v2({{.*}} [[f_x86_64_v2:#[0-9]+]]
+// CHECK: define {{.*}}@f_x86_64_v3({{.*}} [[f_x86_64_v3:#[0-9]+]]
+// CHECK: define {{.*}}@f_x86_64_v4({{.*}} [[f_x86_64_v4:#[0-9]+]]
+// CHECK: define {{.*}}@f_avx10_1_256{{.*}} [[f_avx10_1_256:#[0-9]+]]
+// CHECK: define {{.*}}@f_avx10_1_512{{.*}} [[f_avx10_1_512:#[0-9]+]]
+// CHECK: define {{.*}}@f_prefer_256_bit({{.*}} [[f_prefer_256_bit:#[0-9]+]]
+// CHECK: define {{.*}}@f_no_prefer_256_bit({{.*}} [[f_no_prefer_256_bit:#[0-9]+]]
+
+// CHECK: [[f_default]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="i686"
+void f_default(void) {}
+
+// CHECK: [[f_avx_sse4_2_ivybridge]] = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
+__attribute__((target("avx,sse4.2,arch=ivybridge")))
+void f_avx_sse4_2_ivybridge(void) {}
+
+// We're currently ignoring the fpmath attribute. So checked above that
+// attributes are identical to f_default.
+__attribute__((target("fpmath=387")))
+void f_fpmath_387(void) {}
 
-int __attribute__((target("avx,sse4.2,arch=ivybridge"))) foo(int a) { return 4; }
-
-int __attribute__((target("fpmath=387"))) koala(int a) { return 4; }
-
-int __attribute__((target("no-sse2"))) echidna(int a) { return 4; }
-
-int __attribute__((target("sse4"))) panda(int a) { return 4; }
-int __attribute__((target("no-sse4"))) narwhal(int a) { return 4; }
+// CHECK-NOT: tune-cpu
+// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+__attribute__((target("no-sse2")))
+void f_no_sse2(void) {}
+
+// CHECK: [[f_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+crc32,+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "tune-cpu"="i686"
+__attribute__((target("sse4")))
+void f_sse4(void) {}
+
+// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+__attribute__((target("no-sse4")))
+void f_no_sse4(void) {}
+
+// checked above that attributes are identical to f_default
+void f_default2(void) {
+  f_avx_sse4_2_ivybridge();
+  return f_default();
+}
 
-int bar(int a) { return baz(a) + foo(a); }
+// Checked above to have same attributes as f_avx_sse4_2_ivybridge
+__attribute__((target("avx,      sse4.2,      arch=   ivybridge")))
+void f_avx_sse4_2_ivybridge_2(void) {}
 
-int __attribute__((target("avx,      sse4.2,      arch=   ivybridge"))) qux(int a) { return 4; }
-int __attribute__((target("no-aes, arch=ivybridge"))) qax(int a) { return 4; }
+// CHECK: [[f_no_aes_ivybridge]] = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-amx-avx512,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-vaes"
+__attribute__((target("no-aes, arch=ivybridge")))
+void f_no_aes_ivybridge(void) {}
 
-int __attribute__((target("no-mmx"))) qq(int a) { return 40; }
+// CHECK-NOT: tune-cpu
+// CHECK: [[f_no_mmx]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-mmx"
+__attribute__((target("no-mmx")))
+void f_no_mmx(void) {}
 
-int __attribute__((target("arch=lakemont,mmx"))) lake(int a) { return 4; }
+// CHECK: [[f_lakemont_mmx]] = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
+// Adding the attribute to a definition does update it in IR.
+__attribute__((target("arch=lakemont,mmx")))
+void f_lakemont_mmx(void) {}
 
-int use_before_def(void);
-int useage(void){
-  return use_before_def();
+void f_use_before_def(void);
+void usage(void){
+  f_use_before_def();
 }
 
-// Adding the attribute to a definition does update it in IR.
-int __attribute__((target("arch=lakemont,mmx"))) use_before_def(void) {
-  return 5;
-}
+// Checked above to have same attributes as f_lakemont_mmx
+__attribute__((target("arch=lakemont,mmx")))
+void f_use_before_def(void) {}
 
-int __attribute__((target("tune=sandybridge"))) walrus(int a) { return 4; }
-
-void __attribute__((target("arch=x86-64-v2"))) x86_64_v2(void) {}
-void __attribute__((target("arch=x86-64-v3"))) x86_64_v3(void) {}
-void __attribute__((target("arch=x86-64-v4"))) x86_64_v4(void) {}
-
-void __attribute__((target("avx10.1-256"))) avx10_1_256(void) {}
-void __attribute__((target("avx10.1-512"))) avx10_1_512(void) {}
-
-// Check that we emit the additional subtarget and cpu features for foo and not for baz or bar.
-// CHECK: baz{{.*}} #0
-// CHECK: foo{{.*}} #1
-// We're currently ignoring the fpmath attribute so koala should be identical to baz and bar.
-// CHECK: koala{{.*}} #0
-// CHECK: echidna{{.*}} #2
-// CHECK: panda{{.*}} #3
-// CHECK: narwhal{{.*}} #4
-// CHECK: bar{{.*}} #0
-// CHECK: qux{{.*}} #1
-// CHECK: qax{{.*}} #5
-// CHECK: qq{{.*}} #6
-// CHECK: lake{{.*}} #7
-// CHECK: use_before_def{{.*}} #7
-// CHECK: walrus{{.*}} #8
-// CHECK: avx10_1_256{{.*}} #12
-// CHECK: avx10_1_512{{.*}} #13
-// CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="i686"
-// CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
-// CHECK-NOT: tune-cpu
-// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
-// CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+crc32,+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "tune-cpu"="i686"
-// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
-// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-amx-avx512,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-vaes"
-// CHECK-NOT: tune-cpu
-// CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-mmx"
-// CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
-// CHECK-NOT: tune-cpu
-// CHECK: #8 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="sandybridge"
+// CHECK: [[f_tune_sandybridge]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="sandybridge"
+__attribute__((target("tune=sandybridge")))
+void f_tune_sandybridge(void) {}
 
-// CHECK: "target-cpu"="x86-64-v2"
+// CHECK: [[f_x86_64_v2]] ={{.*}}"target-cpu"="x86-64-v2"
 // CHECK-SAME: "target-features"="+cmov,+crc32,+cx16,+cx8,+fxsr,+mmx,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87"
-// CHECK: "target-cpu"="x86-64-v3"
+__attribute__((target("arch=x86-64-v2")))
+void f_x86_64_v2(void) {}
+
+// CHECK: [[f_x86_64_v3]] = {{.*}}"target-cpu"="x86-64-v3"
 // CHECK-SAME: "target-features"="+avx,+avx2,+bmi,+bmi2,+cmov,+crc32,+cx16,+cx8,+f16c,+fma,+fxsr,+lzcnt,+mmx,+movbe,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave"
-// CHECK: "target-cpu"="x86-64-v4"
+__attribute__((target("arch=x86-64-v3")))
+void f_x86_64_v3(void) {}
+
+// CHECK: [[f_x86_64_v4]] = {{.*}}"target-cpu"="x86-64-v4"
 // CHECK-SAME: "target-features"="+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fxsr,+lzcnt,+mmx,+movbe,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave"
+__attribute__((target("arch=x86-64-v4")))
+void f_x86_64_v4(void) {}
+
+// CHECK: [[f_avx10_1_256]] = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave,-amx-avx512,-avx10.1-512,-avx10.2-512,-evex512"
+__attribute__((target("avx10.1-256")))
+void f_avx10_1_256(void) {}
+
+// CHECK: [[f_avx10_1_512]] = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx10.1-512,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave"
+__attribute__((target("avx10.1-512")))
+void f_avx10_1_512(void) {}
+
+// CHECK: [[f_prefer_256_bit]] = {{.*}}"target-features"="{{.*}}+prefer-256-bit
+__attribute__((target("prefer-256-bit")))
+void f_prefer_256_bit(void) {}
 
-// CHECK: #12 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave,-amx-avx512,-avx10.1-512,-avx10.2-512,-evex512"
-// CHECK: #13 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx10.1-512,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave"
+// CHECK: [[f_no_prefer_256_bit]] = {{.*}}"target-features"="{{.*}}-prefer-256-bit
+__attribute__((target("no-prefer-256-bit")))
+void f_no_prefer_256_bit(void) {}
\ No newline at end of file
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index f1f34432ca0ea..7f6b5f26eb930 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -112,7 +112,7 @@ void test_builtin_elementwise_add_sat(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i32 @llvm.sadd.sat.i32(i32 [[IAS1]], i32 [[B]])
   int_as_one = __builtin_elementwise_add_sat(int_as_one, b);
 
-  // CHECK: call i32 @llvm.sadd.sat.i32(i32 1, i32 97)
+  // CHECK: store i64 98, ptr %i1.addr, align 8
   i1 = __builtin_elementwise_add_sat(1, 'a');
 }
 
@@ -165,7 +165,7 @@ void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i32 @llvm.ssub.sat.i32(i32 [[IAS1]], i32 [[B]])
   int_as_one = __builtin_elementwise_sub_sat(int_as_one, b);
 
-  // CHECK: call i32 @llvm.ssub.sat.i32(i32 1, i32 97)
+  // CHECK: store i64 -96, ptr %i1.addr, align 8
   i1 = __builtin_elementwise_sub_sat(1, 'a');
 }
 
@@ -443,7 +443,7 @@ void test_builtin_elementwise_bitreverse(si8 vi1, si8 vi2,
   // CHECK-NEXT: call i32 @llvm.bitreverse.i32(i32 [[IA1]])
   b = __builtin_elementwise_bitreverse(int_as_one);
 
-  // CHECK:   call i32 @llvm.bitreverse.i32(i32 -10)
+  // CHECK:      store i32 1879048191, ptr @b, align 4
   b = __builtin_elementwise_bitreverse(-10);
 
   // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr, align 2
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index b2259031d7563..88300041061aa 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -485,14 +485,14 @@
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext float [[A_REAL]] to double
-// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext float [[A_IMAG]] to double
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[A_REAL]] to double
+// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[A_IMAG]] to double
 // PRMTD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4
 // PRMTD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4
-// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext float [[B_REAL]] to double
-// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext float [[B_IMAG]] to double
+// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[B_REAL]] to double
+// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[B_IMAG]] to double
 // PRMTD_FAST-NEXT:    [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT2]]
 // PRMTD_FAST-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT1]], [[EXT3]]
 // PRMTD_FAST-NEXT:    [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP0]], [[TMP1]]
@@ -504,8 +504,8 @@
 // PRMTD_FAST-NEXT:    [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP6]], [[TMP7]]
 // PRMTD_FAST-NEXT:    [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP2]], [[TMP5]]
 // PRMTD_FAST-NEXT:    [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP8]], [[TMP5]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc double [[TMP9]] to float
-// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc double [[TMP10]] to float
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn double [[TMP9]] to float
+// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn double [[TMP10]] to float
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4
@@ -1530,14 +1530,14 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext double [[A_REAL]] to x86_fp80
-// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext double [[A_IMAG]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn double [[A_REAL]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[EXT1:%.*]] = fpext reassoc nnan ninf nsz arcp afn double [[A_IMAG]] to x86_fp80
 // PRMTD_FAST-NEXT:    [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
 // PRMTD_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8
-// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext double [[B_REAL]] to x86_fp80
-// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext double [[B_IMAG]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[EXT2:%.*]] = fpext reassoc nnan ninf nsz arcp afn double [[B_REAL]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[EXT3:%.*]] = fpext reassoc nnan ninf nsz arcp afn double [[B_IMAG]] to x86_fp80
 // PRMTD_FAST-NEXT:    [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT]], [[EXT2]]
 // PRMTD_FAST-NEXT:    [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT1]], [[EXT3]]
 // PRMTD_FAST-NEXT:    [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP4]], [[TMP5]]
@@ -1549,8 +1549,8 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // PRMTD_FAST-NEXT:    [[TMP12:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP10]], [[TMP11]]
 // PRMTD_FAST-NEXT:    [[TMP13:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP9]]
 // PRMTD_FAST-NEXT:    [[TMP14:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP12]], [[TMP9]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc x86_fp80 [[TMP13]] to double
-// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc x86_fp80 [[TMP14]] to double
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP13]] to double
+// PRMTD_FAST-NEXT:    [[UNPROMOTION4:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]] to double
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store double [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 8
@@ -3697,8 +3697,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // BASIC_FAST-NEXT:    [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4
 // BASIC_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[C]], i32 0, i32 1
 // BASIC_FAST-NEXT:    [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4
-// BASIC_FAST-NEXT:    [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80
-// BASIC_FAST-NEXT:    [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80
+// BASIC_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_REAL]] to x86_fp80
+// BASIC_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_IMAG]] to x86_fp80
 // BASIC_FAST-NEXT:    [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV]]
 // BASIC_FAST-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV1]]
 // BASIC_FAST-NEXT:    [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]], [[TMP1]]
@@ -3710,8 +3710,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // BASIC_FAST-NEXT:    [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP7]]
 // BASIC_FAST-NEXT:    [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[TMP5]]
 // BASIC_FAST-NEXT:    [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP8]], [[TMP5]]
-// BASIC_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to float
-// BASIC_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to float
+// BASIC_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]] to float
+// BASIC_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP10]] to float
 // BASIC_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 0
 // BASIC_FAST-NEXT:    [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4
 // BASIC_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 1
@@ -3751,13 +3751,13 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // FULL_FAST-NEXT:    [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4
 // FULL_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[C]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4
-// FULL_FAST-NEXT:    [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80
-// FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80
+// FULL_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_REAL]] to x86_fp80
+// FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_IMAG]] to x86_fp80
 // FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
-// FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to float
-// FULL_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to float
+// FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]] to float
+// FULL_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP1]] to float
 // FULL_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 0
 // FULL_FAST-NEXT:    [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4
 // FULL_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 1
@@ -3791,8 +3791,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // IMPRVD_FAST-NEXT:    [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4
 // IMPRVD_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[C]], i32 0, i32 1
 // IMPRVD_FAST-NEXT:    [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4
-// IMPRVD_FAST-NEXT:    [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80
-// IMPRVD_FAST-NEXT:    [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80
+// IMPRVD_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_REAL]] to x86_fp80
+// IMPRVD_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_IMAG]] to x86_fp80
 // IMPRVD_FAST-NEXT:    [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]])
 // IMPRVD_FAST-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]])
 // IMPRVD_FAST-NEXT:    [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]]
@@ -3822,8 +3822,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // IMPRVD_FAST:       complex_div:
 // IMPRVD_FAST-NEXT:    [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
 // IMPRVD_FAST-NEXT:    [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
-// IMPRVD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float
-// IMPRVD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float
+// IMPRVD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP20]] to float
+// IMPRVD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP21]] to float
 // IMPRVD_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 0
 // IMPRVD_FAST-NEXT:    [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4
 // IMPRVD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 1
@@ -3880,8 +3880,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4
 // PRMTD_FAST-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[C]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4
-// PRMTD_FAST-NEXT:    [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80
-// PRMTD_FAST-NEXT:    [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[CONV:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_REAL]] to x86_fp80
+// PRMTD_FAST-NEXT:    [[CONV1:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[C_IMAG]] to x86_fp80
 // PRMTD_FAST-NEXT:    [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]])
 // PRMTD_FAST-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]])
 // PRMTD_FAST-NEXT:    [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]]
@@ -3911,16 +3911,16 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST:       complex_div:
 // PRMTD_FAST-NEXT:    [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
 // PRMTD_FAST-NEXT:    [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ]
-// PRMTD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float
-// PRMTD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float
-// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext float [[CONV2]] to double
-// PRMTD_FAST-NEXT:    [[EXT4:%.*]] = fpext float [[CONV3]] to double
+// PRMTD_FAST-NEXT:    [[CONV2:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP20]] to float
+// PRMTD_FAST-NEXT:    [[CONV3:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP21]] to float
+// PRMTD_FAST-NEXT:    [[EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[CONV2]] to double
+// PRMTD_FAST-NEXT:    [[EXT4:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[CONV3]] to double
 // PRMTD_FAST-NEXT:    [[A_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4
 // PRMTD_FAST-NEXT:    [[A_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4
-// PRMTD_FAST-NEXT:    [[EXT5:%.*]] = fpext float [[A_REAL]] to double
-// PRMTD_FAST-NEXT:    [[EXT6:%.*]] = fpext float [[A_IMAG]] to double
+// PRMTD_FAST-NEXT:    [[EXT5:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[A_REAL]] to double
+// PRMTD_FAST-NEXT:    [[EXT6:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[A_IMAG]] to double
 // PRMTD_FAST-NEXT:    [[TMP22:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT5]]
 // PRMTD_FAST-NEXT:    [[TMP23:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT4]], [[EXT6]]
 // PRMTD_FAST-NEXT:    [[TMP24:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP22]], [[TMP23]]
@@ -3932,8 +3932,8 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // PRMTD_FAST-NEXT:    [[TMP30:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP28]], [[TMP29]]
 // PRMTD_FAST-NEXT:    [[TMP31:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP24]], [[TMP27]]
 // PRMTD_FAST-NEXT:    [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP30]], [[TMP27]]
-// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc double [[TMP31]] to float
-// PRMTD_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc double [[TMP32]] to float
+// PRMTD_FAST-NEXT:    [[UNPROMOTION:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn double [[TMP31]] to float
+// PRMTD_FAST-NEXT:    [[UNPROMOTION7:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn double [[TMP32]] to float
 // PRMTD_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RETVAL]], i32 0, i32 0
 // PRMTD_FAST-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RETVAL]], i32 0, i32 1
 // PRMTD_FAST-NEXT:    store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4
diff --git a/clang/test/CodeGen/ignore-overflow-pattern.c b/clang/test/CodeGen/ignore-overflow-pattern.c
index c4a9d07b07aaa..4e5ed82d7967e 100644
--- a/clang/test/CodeGen/ignore-overflow-pattern.c
+++ b/clang/test/CodeGen/ignore-overflow-pattern.c
@@ -21,7 +21,10 @@
 // unsigned negation, for example:
 // unsigned long A = -1UL;
 
+// Skip over parts of the IR containing this file's name.
+// CHECK: source_filename = {{.*}}
 
+// Ensure we don't see anything about handling overflow before the tests below.
 // CHECK-NOT: handle{{.*}}overflow
 
 extern unsigned a, b, c;
diff --git a/clang/test/CodeGen/math-libcalls-tbaa.c b/clang/test/CodeGen/math-libcalls-tbaa.c
index 9c86eea67d14d..17a93ed2aed2d 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa.c
@@ -16,7 +16,7 @@ float crealf(float _Complex);
 // CHECK-LABEL: define dso_local float @test_expf(
 // CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 40
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 40
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[CALL:%.*]] = tail call float @expf(float noundef [[TMP0]]) #[[ATTR9:[0-9]+]], !tbaa [[TBAA6:![0-9]+]]
 // CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP0]], [[CALL]]
@@ -31,7 +31,7 @@ float test_expf (float num[]) {
 // CHECK-LABEL: define dso_local float @test_builtin_expf(
 // CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 40
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 40
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[CALL:%.*]] = tail call float @expf(float noundef [[TMP0]]) #[[ATTR9]], !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP0]], [[CALL]]
@@ -48,7 +48,7 @@ float test_builtin_expf (float num[]) {
 // CHECK-LABEL: define dso_local double @test_fabs(
 // CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 80
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 80
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call double @llvm.fabs.f64(double [[TMP0]])
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP0]], [[TMP1]]
@@ -63,7 +63,7 @@ double test_fabs (double num[]) {
 // CHECK-LABEL: define dso_local double @test_remainder(
 // CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]], double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 80
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 80
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[CALL:%.*]] = tail call double @remainder(double noundef [[TMP0]], double noundef [[A]]) #[[ATTR9]], !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP0]], [[CALL]]
@@ -83,7 +83,7 @@ double test_remainder (double num[], double a) {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[E]]) #[[ATTR9]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 16
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 16
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[CALL:%.*]] = call double @frexp(double noundef [[TMP0]], ptr noundef nonnull [[E]]) #[[ATTR9]]
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP0]], [[CALL]]
@@ -107,7 +107,7 @@ double test_frexp (double num[]) {
 // CHECK-NEXT:    [[COS:%.*]] = alloca float, align 4
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[SIN]]) #[[ATTR9]]
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[COS]]) #[[ATTR9]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    call void @sincos(float noundef [[TMP0]], ptr noundef nonnull [[SIN]], ptr noundef nonnull [[COS]]) #[[ATTR9]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[SIN]], align 4, !tbaa [[TBAA2]]
@@ -130,7 +130,7 @@ float test_sincos (float num[]) {
 // CHECK-LABEL: define dso_local float @test_cacoshf(
 // CHECK-SAME: ptr nocapture noundef readonly [[NUM:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[NUM]], i64 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x float] poison, float [[TMP0]], 0
 // CHECK-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x float] [[DOTFCA_0_INSERT]], float 0.000000e+00, 1
diff --git a/clang/test/CodeGen/matrix-type-operators-fast-math.c b/clang/test/CodeGen/matrix-type-operators-fast-math.c
index 724b282830286..a5cb2ffdabcc5 100644
--- a/clang/test/CodeGen/matrix-type-operators-fast-math.c
+++ b/clang/test/CodeGen/matrix-type-operators-fast-math.c
@@ -41,7 +41,7 @@ void add_matrix_scalar_double_float(dx5x5_t a, float vf) {
   // CHECK-LABEL: define{{.*}} void @add_matrix_scalar_double_float(<25 x double> noundef nofpclass(nan inf) %a, float noundef nofpclass(nan inf) %vf)
   // CHECK:       [[MATRIX:%.*]] = load <25 x double>, ptr {{.*}}, align 8
   // CHECK-NEXT:  [[SCALAR:%.*]] = load float, ptr %vf.addr, align 4
-  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext float [[SCALAR]] to double
+  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[SCALAR]] to double
   // CHECK-NEXT:  [[SCALAR_EMBED:%.*]] = insertelement <25 x double> poison, double [[SCALAR_EXT]], i64 0
   // CHECK-NEXT:  [[SCALAR_EMBED1:%.*]] = shufflevector <25 x double> [[SCALAR_EMBED]], <25 x double> poison, <25 x i32> zeroinitializer
   // CHECK-NEXT:  [[RES:%.*]] = fadd reassoc nnan ninf nsz arcp afn <25 x double> [[MATRIX]], [[SCALAR_EMBED1]]
@@ -53,7 +53,7 @@ void add_matrix_scalar_double_float(dx5x5_t a, float vf) {
 void add_compound_matrix_scalar_double_float(dx5x5_t a, float vf) {
   // CHECK-LABEL: define{{.*}} void @add_compound_matrix_scalar_double_float(<25 x double> noundef nofpclass(nan inf) %a, float noundef nofpclass(nan inf) %vf)
   // CHECK:  [[SCALAR:%.*]] = load float, ptr %vf.addr, align 4
-  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext float [[SCALAR]] to double
+  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[SCALAR]] to double
   // CHECK-NEXT:  [[MATRIX:%.*]] = load <25 x double>, ptr {{.*}}, align 8
   // CHECK-NEXT:  [[SCALAR_EMBED:%.*]] = insertelement <25 x double> poison, double [[SCALAR_EXT]], i64 0
   // CHECK-NEXT:  [[SCALAR_EMBED1:%.*]] = shufflevector <25 x double> [[SCALAR_EMBED]], <25 x double> poison, <25 x i32> zeroinitializer
@@ -66,7 +66,7 @@ void add_compound_matrix_scalar_double_float(dx5x5_t a, float vf) {
 void subtract_compound_matrix_scalar_double_float(dx5x5_t a, float vf) {
   // CHECK-LABEL: define{{.*}} void @subtract_compound_matrix_scalar_double_float(<25 x double> noundef nofpclass(nan inf) %a, float noundef nofpclass(nan inf) %vf)
   // CHECK:  [[SCALAR:%.*]] = load float, ptr %vf.addr, align 4
-  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext float [[SCALAR]] to double
+  // CHECK-NEXT:  [[SCALAR_EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[SCALAR]] to double
   // CHECK-NEXT:  [[MATRIX:%.*]] = load <25 x double>, ptr {{.*}}, align 8
   // CHECK-NEXT:  [[SCALAR_EMBED:%.*]] = insertelement <25 x double> poison, double [[SCALAR_EXT]], i64 0
   // CHECK-NEXT:  [[SCALAR_EMBED1:%.*]] = shufflevector <25 x double> [[SCALAR_EMBED]], <25 x double> poison, <25 x i32> zeroinitializer
@@ -104,7 +104,7 @@ void multiply_compound_matrix_matrix_double(dx5x5_t b, dx5x5_t c) {
 // CHECK-LABEL: @multiply_double_matrix_scalar_float(
 // CHECK:         [[A:%.*]] = load <25 x double>, ptr {{.*}}, align 8
 // CHECK-NEXT:    [[S:%.*]] = load float, ptr %s.addr, align 4
-// CHECK-NEXT:    [[S_EXT:%.*]] = fpext float [[S]] to double
+// CHECK-NEXT:    [[S_EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[S]] to double
 // CHECK-NEXT:    [[VECINSERT:%.*]] = insertelement <25 x double> poison, double [[S_EXT]], i64 0
 // CHECK-NEXT:    [[VECSPLAT:%.*]] = shufflevector <25 x double> [[VECINSERT]], <25 x double> poison, <25 x i32> zeroinitializer
 // CHECK-NEXT:    [[RES:%.*]] = fmul reassoc nnan ninf nsz arcp afn <25 x double> [[A]], [[VECSPLAT]]
@@ -117,7 +117,7 @@ void multiply_double_matrix_scalar_float(dx5x5_t a, float s) {
 
 // CHECK-LABEL: @multiply_compound_double_matrix_scalar_float
 // CHECK:         [[S:%.*]] = load float, ptr %s.addr, align 4
-// CHECK-NEXT:    [[S_EXT:%.*]] = fpext float [[S]] to double
+// CHECK-NEXT:    [[S_EXT:%.*]] = fpext reassoc nnan ninf nsz arcp afn float [[S]] to double
 // CHECK-NEXT:    [[A:%.*]] = load <25 x double>, ptr {{.*}}, align 8
 // CHECK-NEXT:    [[VECINSERT:%.*]] = insertelement <25 x double> poison, double [[S_EXT]], i64 0
 // CHECK-NEXT:    [[VECSPLAT:%.*]] = shufflevector <25 x double> [[VECINSERT]], <25 x double> poison, <25 x i32> zeroinitializer
@@ -132,7 +132,7 @@ void multiply_compound_double_matrix_scalar_float(dx5x5_t a, float s) {
 // CHECK-LABEL: @divide_float_matrix_scalar_double(
 // CHECK:         [[MAT:%.*]] = load <6 x float>, ptr [[MAT_ADDR:%.*]], align 4
 // CHECK-NEXT:    [[S:%.*]] = load double, ptr %s.addr, align 8
-// CHECK-NEXT:    [[S_TRUNC:%.*]] = fptrunc double [[S]] to float
+// CHECK-NEXT:    [[S_TRUNC:%.*]] = fptrunc reassoc nnan ninf nsz arcp afn double [[S]] to float
 // CHECK-NEXT:    [[VECINSERT:%.*]] = insertelement <6 x float> poison, float [[S_TRUNC]], i64 0
 // CHECK-NEXT:    [[VECSPLAT:%.*]] = shufflevector <6 x float> [[VECINSERT]], <6 x float> poison, <6 x i32> zeroinitializer
 // CHECK-NEXT:    [[RES:%.*]] = fdiv reassoc nnan ninf nsz arcp afn <6 x float> [[MAT]], [[VECSPLAT]]
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 459a708d9b2e0..bb1b95fa7521a 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
 // CHECK:   ret ptr %[[RESULT]]
 // CHECK: }
 
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
+  return _InterlockedExchangePointer_acq(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64:   %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
+// CHECK-ARM-ARM64:   %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
+// CHECK-ARM-ARM64:   %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64:   ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
+  return _InterlockedExchangePointer_nf(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64:   %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
+// CHECK-ARM-ARM64:   %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64:   ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
+  return _InterlockedExchangePointer_rel(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64:   %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
+// CHECK-ARM-ARM64:   %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64:   ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+#endif
+
 void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
                                              void *Exchange, void *Comparand) {
   return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
 // CHECK:   ret ptr %[[RESULT:[0-9]+]]
 // CHECK: }
 
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
+                                             void *Exchange, void *Comparand) {
+  return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64:   %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
+// CHECK-ARM-ARM64:   %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64:   %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64:   ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+
+
+void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
+                                             void *Exchange, void *Comparand) {
+  return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64:   %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64:   %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
+// CHECK-ARM-ARM64:   %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64:   %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64:   ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+#endif
+
 char test_InterlockedExchange8(char volatile *value, char mask) {
   return _InterlockedExchange8(value, mask);
 }
diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c
index d9b34c8e383f9..75aa0318421de 100644
--- a/clang/test/CodeGen/nofpclass.c
+++ b/clang/test/CodeGen/nofpclass.c
@@ -679,14 +679,14 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // CFINITEONLY-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // CFINITEONLY-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// CFINITEONLY-NEXT:    [[EXT:%.*]] = fpext half [[C_REAL]] to float
-// CFINITEONLY-NEXT:    [[EXT1:%.*]] = fpext half [[C_IMAG]] to float
+// CFINITEONLY-NEXT:    [[EXT:%.*]] = fpext nnan ninf half [[C_REAL]] to float
+// CFINITEONLY-NEXT:    [[EXT1:%.*]] = fpext nnan ninf half [[C_IMAG]] to float
 // CFINITEONLY-NEXT:    [[C_REALP2:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[C_REAL3:%.*]] = load half, ptr [[C_REALP2]], align 2
 // CFINITEONLY-NEXT:    [[C_IMAGP4:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[C_IMAG5:%.*]] = load half, ptr [[C_IMAGP4]], align 2
-// CFINITEONLY-NEXT:    [[EXT6:%.*]] = fpext half [[C_REAL3]] to float
-// CFINITEONLY-NEXT:    [[EXT7:%.*]] = fpext half [[C_IMAG5]] to float
+// CFINITEONLY-NEXT:    [[EXT6:%.*]] = fpext nnan ninf half [[C_REAL3]] to float
+// CFINITEONLY-NEXT:    [[EXT7:%.*]] = fpext nnan ninf half [[C_IMAG5]] to float
 // CFINITEONLY-NEXT:    [[MUL_AC:%.*]] = fmul nnan ninf float [[EXT]], [[EXT6]]
 // CFINITEONLY-NEXT:    [[MUL_BD:%.*]] = fmul nnan ninf float [[EXT1]], [[EXT7]]
 // CFINITEONLY-NEXT:    [[MUL_AD:%.*]] = fmul nnan ninf float [[EXT]], [[EXT7]]
@@ -709,8 +709,8 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // CFINITEONLY:       complex_mul_cont:
 // CFINITEONLY-NEXT:    [[REAL_MUL_PHI:%.*]] = phi nnan ninf float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ]
 // CFINITEONLY-NEXT:    [[IMAG_MUL_PHI:%.*]] = phi nnan ninf float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ]
-// CFINITEONLY-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half
-// CFINITEONLY-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half
+// CFINITEONLY-NEXT:    [[UNPROMOTION:%.*]] = fptrunc nnan ninf float [[REAL_MUL_PHI]] to half
+// CFINITEONLY-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc nnan ninf float [[IMAG_MUL_PHI]] to half
 // CFINITEONLY-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // CFINITEONLY-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -723,16 +723,16 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // CLFINITEONLY-SAME: (<2 x half> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[C_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[C_COERCE]], i64 0
-// CLFINITEONLY-NEXT:    [[EXT:%.*]] = fpext half [[C_SROA_0_0_VEC_EXTRACT]] to float
+// CLFINITEONLY-NEXT:    [[EXT:%.*]] = fpext nnan ninf half [[C_SROA_0_0_VEC_EXTRACT]] to float
 // CLFINITEONLY-NEXT:    [[C_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[C_COERCE]], i64 1
-// CLFINITEONLY-NEXT:    [[EXT1:%.*]] = fpext half [[C_SROA_0_2_VEC_EXTRACT]] to float
+// CLFINITEONLY-NEXT:    [[EXT1:%.*]] = fpext nnan ninf half [[C_SROA_0_2_VEC_EXTRACT]] to float
 // CLFINITEONLY-NEXT:    [[MUL_AD:%.*]] = fmul nnan ninf float [[EXT]], [[EXT1]]
 // CLFINITEONLY-NEXT:    [[MUL_I:%.*]] = fadd nnan ninf float [[MUL_AD]], [[MUL_AD]]
 // CLFINITEONLY-NEXT:    [[MUL_AC:%.*]] = fmul nnan ninf float [[EXT]], [[EXT]]
 // CLFINITEONLY-NEXT:    [[MUL_BD:%.*]] = fmul nnan ninf float [[EXT1]], [[EXT1]]
 // CLFINITEONLY-NEXT:    [[MUL_R:%.*]] = fsub nnan ninf float [[MUL_AC]], [[MUL_BD]]
-// CLFINITEONLY-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half
-// CLFINITEONLY-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc float [[MUL_I]] to half
+// CLFINITEONLY-NEXT:    [[UNPROMOTION:%.*]] = fptrunc nnan ninf float [[MUL_R]] to half
+// CLFINITEONLY-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc nnan ninf float [[MUL_I]] to half
 // CLFINITEONLY-NEXT:    [[RETVAL_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[UNPROMOTION]], i64 0
 // CLFINITEONLY-NEXT:    [[RETVAL_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[RETVAL_SROA_0_0_VEC_INSERT]], half [[UNPROMOTION9]], i64 1
 // CLFINITEONLY-NEXT:    ret <2 x half> [[RETVAL_SROA_0_2_VEC_INSERT]]
@@ -749,14 +749,14 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // NONANS-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // NONANS-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // NONANS-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// NONANS-NEXT:    [[EXT:%.*]] = fpext half [[C_REAL]] to float
-// NONANS-NEXT:    [[EXT1:%.*]] = fpext half [[C_IMAG]] to float
+// NONANS-NEXT:    [[EXT:%.*]] = fpext nnan half [[C_REAL]] to float
+// NONANS-NEXT:    [[EXT1:%.*]] = fpext nnan half [[C_IMAG]] to float
 // NONANS-NEXT:    [[C_REALP2:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 0
 // NONANS-NEXT:    [[C_REAL3:%.*]] = load half, ptr [[C_REALP2]], align 2
 // NONANS-NEXT:    [[C_IMAGP4:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // NONANS-NEXT:    [[C_IMAG5:%.*]] = load half, ptr [[C_IMAGP4]], align 2
-// NONANS-NEXT:    [[EXT6:%.*]] = fpext half [[C_REAL3]] to float
-// NONANS-NEXT:    [[EXT7:%.*]] = fpext half [[C_IMAG5]] to float
+// NONANS-NEXT:    [[EXT6:%.*]] = fpext nnan half [[C_REAL3]] to float
+// NONANS-NEXT:    [[EXT7:%.*]] = fpext nnan half [[C_IMAG5]] to float
 // NONANS-NEXT:    [[MUL_AC:%.*]] = fmul nnan float [[EXT]], [[EXT6]]
 // NONANS-NEXT:    [[MUL_BD:%.*]] = fmul nnan float [[EXT1]], [[EXT7]]
 // NONANS-NEXT:    [[MUL_AD:%.*]] = fmul nnan float [[EXT]], [[EXT7]]
@@ -779,8 +779,8 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // NONANS:       complex_mul_cont:
 // NONANS-NEXT:    [[REAL_MUL_PHI:%.*]] = phi nnan float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ]
 // NONANS-NEXT:    [[IMAG_MUL_PHI:%.*]] = phi nnan float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ]
-// NONANS-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half
-// NONANS-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half
+// NONANS-NEXT:    [[UNPROMOTION:%.*]] = fptrunc nnan float [[REAL_MUL_PHI]] to half
+// NONANS-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc nnan float [[IMAG_MUL_PHI]] to half
 // NONANS-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // NONANS-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // NONANS-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -800,14 +800,14 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // NOINFS-NEXT:    [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2
 // NOINFS-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // NOINFS-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
-// NOINFS-NEXT:    [[EXT:%.*]] = fpext half [[C_REAL]] to float
-// NOINFS-NEXT:    [[EXT1:%.*]] = fpext half [[C_IMAG]] to float
+// NOINFS-NEXT:    [[EXT:%.*]] = fpext ninf half [[C_REAL]] to float
+// NOINFS-NEXT:    [[EXT1:%.*]] = fpext ninf half [[C_IMAG]] to float
 // NOINFS-NEXT:    [[C_REALP2:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 0
 // NOINFS-NEXT:    [[C_REAL3:%.*]] = load half, ptr [[C_REALP2]], align 2
 // NOINFS-NEXT:    [[C_IMAGP4:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[C]], i32 0, i32 1
 // NOINFS-NEXT:    [[C_IMAG5:%.*]] = load half, ptr [[C_IMAGP4]], align 2
-// NOINFS-NEXT:    [[EXT6:%.*]] = fpext half [[C_REAL3]] to float
-// NOINFS-NEXT:    [[EXT7:%.*]] = fpext half [[C_IMAG5]] to float
+// NOINFS-NEXT:    [[EXT6:%.*]] = fpext ninf half [[C_REAL3]] to float
+// NOINFS-NEXT:    [[EXT7:%.*]] = fpext ninf half [[C_IMAG5]] to float
 // NOINFS-NEXT:    [[MUL_AC:%.*]] = fmul ninf float [[EXT]], [[EXT6]]
 // NOINFS-NEXT:    [[MUL_BD:%.*]] = fmul ninf float [[EXT1]], [[EXT7]]
 // NOINFS-NEXT:    [[MUL_AD:%.*]] = fmul ninf float [[EXT]], [[EXT7]]
@@ -830,8 +830,8 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) {
 // NOINFS:       complex_mul_cont:
 // NOINFS-NEXT:    [[REAL_MUL_PHI:%.*]] = phi ninf float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ]
 // NOINFS-NEXT:    [[IMAG_MUL_PHI:%.*]] = phi ninf float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ]
-// NOINFS-NEXT:    [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half
-// NOINFS-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half
+// NOINFS-NEXT:    [[UNPROMOTION:%.*]] = fptrunc ninf float [[REAL_MUL_PHI]] to half
+// NOINFS-NEXT:    [[UNPROMOTION9:%.*]] = fptrunc ninf float [[IMAG_MUL_PHI]] to half
 // NOINFS-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 0
 // NOINFS-NEXT:    [[RETVAL_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[RETVAL]], i32 0, i32 1
 // NOINFS-NEXT:    store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2
@@ -879,7 +879,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // CFINITEONLY-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext float [[TMP3]] to double
+// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP3]] to double
 // CFINITEONLY-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // CFINITEONLY-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // CFINITEONLY-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
@@ -923,7 +923,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // CLFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nocapture noundef readonly byval({ half, half }) align 8 [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext float [[F32]] to double
+// CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
 // CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
 // CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
 // CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
@@ -973,7 +973,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NONANS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
 // NONANS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // NONANS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[CONV:%.*]] = fpext float [[TMP3]] to double
+// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP3]] to double
 // NONANS-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // NONANS-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // NONANS-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
@@ -1048,7 +1048,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NOINFS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
 // NOINFS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // NOINFS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[CONV:%.*]] = fpext float [[TMP3]] to double
+// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP3]] to double
 // NOINFS-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // NOINFS-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // NOINFS-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
@@ -1132,7 +1132,7 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // CFINITEONLY-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
 // CFINITEONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext float [[TMP4]] to double
+// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP4]] to double
 // CFINITEONLY-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // CFINITEONLY-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // CFINITEONLY-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
@@ -1176,7 +1176,7 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // CLFINITEONLY-SAME: (ptr nocapture noundef readonly [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nocapture noundef readonly byval({ half, half }) align 8 [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext float [[F32]] to double
+// CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
 // CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
 // CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
 // CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
@@ -1229,7 +1229,7 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NONANS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
 // NONANS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // NONANS-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[CONV:%.*]] = fpext float [[TMP4]] to double
+// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP4]] to double
 // NONANS-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // NONANS-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // NONANS-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
@@ -1307,7 +1307,7 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NOINFS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
 // NOINFS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
 // NOINFS-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[CONV:%.*]] = fpext float [[TMP4]] to double
+// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP4]] to double
 // NOINFS-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
 // NOINFS-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
 // NOINFS-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
diff --git a/clang/test/CodeGen/ptrauth-function-attributes.c b/clang/test/CodeGen/ptrauth-function-attributes.c
index e7081f00b4f68..caf613f0c66ab 100644
--- a/clang/test/CodeGen/ptrauth-function-attributes.c
+++ b/clang/test/CodeGen/ptrauth-function-attributes.c
@@ -14,6 +14,9 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios   -fptrauth-indirect-gotos -emit-llvm %s -o - | FileCheck %s --check-prefixes=ALL,GOTOS
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-indirect-gotos -emit-llvm %s -o - | FileCheck %s --check-prefixes=ALL,GOTOS
 
+// RUN: %clang_cc1 -triple arm64e-apple-ios  -faarch64-jump-table-hardening -emit-llvm %s -o - | FileCheck %s --check-prefixes=ALL,JMPTBL
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -faarch64-jump-table-hardening -emit-llvm %s -o - | FileCheck %s --check-prefixes=ALL,JMPTBL
+
 // ALL: define {{(dso_local )?}}void @test() #0
 void test() {
 }
@@ -26,4 +29,6 @@ void test() {
 
 // GOTOS: attributes #0 = {{{.*}} "ptrauth-indirect-gotos" {{.*}}}
 
+// JMPTBL: attributes #0 = {{{.*}} "aarch64-jump-table-hardening" {{.*}}}
+
 // OFF-NOT: attributes {{.*}} "ptrauth-
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 931d333c00768..4c80547f77c31 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -88,7 +88,7 @@
 
 // RUN: %clang_cc1 -triple powerpc64-lv2 -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=PS3
-// PS3: target datalayout = "E-m:e-p:32:32-Fi64-i64:64-n32:64"
+// PS3: target datalayout = "E-m:e-p:32:32-Fi64-i64:64-i128:128-n32:64"
 
 // RUN: %clang_cc1 -triple i686-nacl -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=I686-NACL
@@ -108,11 +108,11 @@
 
 // RUN: %clang_cc1 -triple wasm32-unknown-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=WEBASSEMBLY32
-// WEBASSEMBLY32: target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+// WEBASSEMBLY32: target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20"
 
 // RUN: %clang_cc1 -triple wasm64-unknown-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=WEBASSEMBLY64
-// WEBASSEMBLY64: target datalayout = "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+// WEBASSEMBLY64: target datalayout = "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20"
 
 // RUN: %clang_cc1 -triple lanai-unknown-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=LANAI
@@ -128,35 +128,35 @@
 
 // RUN: %clang_cc1 -triple powerpc64-freebsd -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=PPC64-FREEBSD
-// PPC64-FREEBSD: target datalayout = "E-m:e-Fn32-i64:64-n32:64"
+// PPC64-FREEBSD: target datalayout = "E-m:e-Fn32-i64:64-i128:128-n32:64"
 
 // RUN: %clang_cc1 -triple powerpc64le-freebsd -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=PPC64LE-FREEBSD
-// PPC64LE-FREEBSD: target datalayout = "e-m:e-Fn32-i64:64-n32:64"
+// PPC64LE-FREEBSD: target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64"
 
 // RUN: %clang_cc1 -triple powerpc64-linux -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=PPC64-LINUX
-// PPC64-LINUX: target datalayout = "E-m:e-Fi64-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64-LINUX: target datalayout = "E-m:e-Fi64-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple powerpc64-linux -o - -emit-llvm -target-cpu future %s | \
 // RUN: FileCheck %s -check-prefix=PPC64-FUTURE
-// PPC64-FUTURE: target datalayout = "E-m:e-Fi64-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64-FUTURE: target datalayout = "E-m:e-Fi64-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple powerpc64-linux -o - -emit-llvm -target-cpu pwr10 %s | \
 // RUN: FileCheck %s -check-prefix=PPC64-P10
-// PPC64-P10: target datalayout = "E-m:e-Fi64-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64-P10: target datalayout = "E-m:e-Fi64-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple powerpc64le-linux -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=PPC64LE-LINUX
-// PPC64LE-LINUX: target datalayout = "e-m:e-Fn32-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64LE-LINUX: target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple powerpc64le-linux -o - -emit-llvm -target-cpu future %s | \
 // RUN: FileCheck %s -check-prefix=PPC64LE-FUTURE
-// PPC64LE-FUTURE: target datalayout = "e-m:e-Fn32-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64LE-FUTURE: target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple powerpc64le-linux -o - -emit-llvm -target-cpu pwr10 %s | \
 // RUN: FileCheck %s -check-prefix=PPC64LE-P10
-// PPC64LE-P10: target datalayout = "e-m:e-Fn32-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+// PPC64LE-P10: target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
 
 // RUN: %clang_cc1 -triple nvptx-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=NVPTX
diff --git a/clang/test/CodeGen/tbaa-pointers.c b/clang/test/CodeGen/tbaa-pointers.c
index 068459f4dce11..6aee2ff3717ac 100644
--- a/clang/test/CodeGen/tbaa-pointers.c
+++ b/clang/test/CodeGen/tbaa-pointers.c
@@ -1,18 +1,18 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes -no-pointer-tbaa %s -emit-llvm -o - | FileCheck --check-prefixes=COMMON,DISABLE %s
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck --check-prefixes=COMMON,DEFAULT %s
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes -pointer-tbaa %s -emit-llvm -o - | FileCheck --check-prefixes=COMMON,ENABLED %s
+// RUN: %clang --target=x86_64-apple-darwin -O1 -fno-pointer-tbaa %s -emit-llvm -S -mllvm -disable-llvm-optzns -o - | FileCheck --check-prefixes=COMMON,DISABLE %s
 // RUN: %clang --target=x86_64-apple-darwin -O1 %s -emit-llvm -S -mllvm -disable-llvm-optzns -o - | FileCheck --check-prefixes=COMMON,DEFAULT %s
-// RUN: %clang --target=x86_64-apple-darwin -O1 -fpointer-tbaa %s -emit-llvm -S -mllvm -disable-llvm-optzns -o - | FileCheck --check-prefixes=COMMON,ENABLED %s
 
 void p2unsigned(unsigned **ptr) {
   // COMMON-LABEL: define void @p2unsigned(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:        [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0:!.+]]
-  // ENABLED-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
-  // ENABLED-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0:!.+]]
-  // DEFAULT-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR:!.+]]
-  // DEFAULT-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0:!.+]]
+  // DEFAULT-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // DEFAULT-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0:!.+]]
+  // DISABLE-NEXT:  store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR:!.+]]
+  // DISABLE-NEXT:  [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:  store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:   ret void
   //
   *ptr = 0;
@@ -22,12 +22,12 @@ void p2unsigned_volatile(unsigned *volatile *ptr) {
   // COMMON-LABEL: define void @p2unsigned_volatile(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
-  // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
-  // ENABLED-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2INT_0]]
+  // DEFAULT-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[P1INT_0]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   store volatile ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   *ptr = 0;
@@ -37,14 +37,14 @@ void p3int(int ***ptr) {
   // COMMON-LABEL: define void @p3int(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0:!.+]]
-  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0]]
-  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P2INT_0]]
-  // ENABLED-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[P1INT_0]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0:!.+]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P3INT_0]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P2INT_0]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[P1INT_0]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   store ptr null, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   **ptr = 0;
@@ -54,16 +54,16 @@ void p4char(char ****ptr) {
   // COMMON-LABEL: define void @p4char(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0:!.+]]
-  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0:!.+]]
-  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0:!.+]]
-  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0:!.+]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0:!.+]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0:!.+]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0:!.+]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0:!.+]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   ***ptr = 0;
@@ -73,16 +73,16 @@ void p4char_const1(const char ****ptr) {
   // COMMON-LABEL: define void @p4char_const1(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
-  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:   ret void
   //
   ***ptr = 0;
@@ -92,16 +92,16 @@ void p4char_const2(const char **const **ptr) {
   // COMMON-LABEL: define void @p4char_const2(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
-  // ENABLED-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
-  // ENABLED-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P4CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[P3CHAR_0]]
+  // DEFAULT-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[P2CHAR_0]]
+  // DEFAULT-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[P1CHAR_0]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_1:%.+]] = load ptr, ptr [[BASE_0]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE_2:%.+]] = load ptr, ptr [[BASE_1]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   store ptr null, ptr [[BASE_2]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   ***ptr = 0;
@@ -116,12 +116,12 @@ void p2struct(struct S1 **ptr) {
   // COMMON-LABEL: define void @p2struct(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_TAG:!.+]]
-  // DEFAULT-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // ENABLED-NEXT:    [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_TAG]]
-  // DEFAULT-NEXT:    [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // ENABLED-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_TAG:!.+]]
-  // DEFAULT-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_TAG:!.+]]
+  // DISABLE-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:    [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P2S1_TAG]]
+  // DISABLE-NEXT:    [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_TAG:!.+]]
+  // DISABLE-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   *ptr = 0;
@@ -133,8 +133,8 @@ void p2struct_const(struct S1 const **ptr) {
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
   // COMMON-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR:!.+]]
   // COMMON-NEXT:    [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // ENABLED-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_TAG]]
-  // DEFAULT-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[P1S1_TAG]]
+  // DISABLE-NEXT:    store ptr null, ptr [[BASE]], align 8, !tbaa [[ANYPTR]]
   // COMMON-NEXT:    ret void
   //
   *ptr = 0;
@@ -148,14 +148,14 @@ void p2struct2(struct S2 *ptr) {
   // COMMON-LABEL: define void @p2struct2(
   // COMMON-SAME:    ptr noundef [[PTR:%.+]])
   // COMMON:         [[PTR_ADDR:%.+]] = alloca ptr, align 8
-  // ENABLED-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1S2_TAG:!.+]]
-  // ENABLED-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P1S2_TAG]]
-  // ENABLED-NEXT:   [[S:%.+]] = getelementptr inbounds nuw %struct.S2, ptr [[BASE]], i32 0, i32 0
-  // ENABLED-NEXT:   store ptr null, ptr [[S]], align 8, !tbaa [[S2_S_TAG:!.+]]
-  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DEFAULT-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1S2_TAG:!.+]]
+  // DEFAULT-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P1S2_TAG]]
   // DEFAULT-NEXT:   [[S:%.+]] = getelementptr inbounds nuw %struct.S2, ptr [[BASE]], i32 0, i32 0
   // DEFAULT-NEXT:   store ptr null, ptr [[S]], align 8, !tbaa [[S2_S_TAG:!.+]]
+  // DISABLE-NEXT:   store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[BASE:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+  // DISABLE-NEXT:   [[S:%.+]] = getelementptr inbounds nuw %struct.S2, ptr [[BASE]], i32 0, i32 0
+  // DISABLE-NEXT:   store ptr null, ptr [[S]], align 8, !tbaa [[S2_S_TAG:!.+]]
   // COMMON-NEXT:    ret void
     ptr->s = 0;
 }
@@ -168,20 +168,20 @@ void vla1(int n, int ptr[][n], int idx) {
 // COMMON-NEXT:  [[PTR_ADDR:%.+]] = alloca ptr, align 8
 // COMMON-NEXT:  [[IDX_ADDR:%.+]] = alloca i32, align 4
 // COMMON-NEXT: store i32 [[N]], ptr [[N_ADDR]], align 4, !tbaa [[INT_TY:!.+]]
-// ENABLED-NEXT: store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1INT0:!.+]]
-// DEFAULT-NEXT: store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+// DEFAULT-NEXT: store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1INT0:!.+]]
+// DISABLE-NEXT: store ptr [[PTR]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
 // COMMON-NEXT: store i32 [[IDX]], ptr [[IDX_ADDR]], align 4, !tbaa [[INT_TY]]
 // COMMON-NEXT: [[L:%.+]] = load i32, ptr [[N_ADDR]], align 4, !tbaa [[INT_TY]]
 // COMMON-NEXT: [[L_EXT:%.+]] = zext i32 [[L]] to i64
-// ENABLED-NEXT: [[L_PTR:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P1INT0]]
-// DEFAULT-NEXT: [[L_PTR:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+// DEFAULT-NEXT: [[L_PTR:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[P1INT0]]
+// DISABLE-NEXT: [[L_PTR:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
 // COMMON-NEXT: [[L_IDX:%.+]] = load i32, ptr [[IDX_ADDR]], align 4, !tbaa [[INT_TY]]
 // COMMON-NEXT: [[IDX_EXT:%.+]] = sext i32 [[L_IDX]] to i64
 // COMMON-NEXT: [[MUL:%.+]] = mul nsw i64 [[IDX_EXT]], [[L_EXT]]
 // COMMON-NEXT: [[GEP1:%.+]] = getelementptr inbounds i32, ptr [[L_PTR]], i64 [[MUL]]
 // COMMON-NEXT: [[GEP2:%.+]] = getelementptr inbounds i32, ptr [[GEP1]], i64 0
 // COMMON-NEXT: store i32 0, ptr [[GEP2]], align 4, !tbaa [[INT_TAG:!.+]]
-// ENABLED-NEXT: ret void
+// DEFAULT-NEXT: ret void
 
     ptr[idx][0] = 0;
 }
@@ -194,10 +194,10 @@ void unamed_struct_typedef(TypedefS *ptr) {
 // COMMON-LABEL: define void @unamed_struct_typedef(
 // COMMON-SAME: ptr noundef [[PTRA:%.+]])
 // COMMON:   [[PTR_ADDR:%.+]]  = alloca ptr, align 8
-// DEFAULT-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
-// DEFAULT-NEXT:  [[L0:%.+]] = load ptr, ptr  [[PTR_ADDR]], align 8, !tbaa  [[ANYPTR]]
-// ENABLED-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1TYPEDEF:!.+]]
-// ENABLED-NEXT:  [[L0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa  [[P1TYPEDEF]]
+// DISABLE-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[ANYPTR]]
+// DISABLE-NEXT:  [[L0:%.+]] = load ptr, ptr  [[PTR_ADDR]], align 8, !tbaa  [[ANYPTR]]
+// DEFAULT-NEXT:  store ptr [[PTRA]], ptr [[PTR_ADDR]], align 8, !tbaa [[P1TYPEDEF:!.+]]
+// DEFAULT-NEXT:  [[L0:%.+]] = load ptr, ptr [[PTR_ADDR]], align 8, !tbaa  [[P1TYPEDEF]]
 // COMMON-NEXT:   [[GEP:%.+]]  = getelementptr inbounds nuw %struct.TypedefS, ptr [[L0]], i32 0, i32 0
 // COMMON-NEXT:   store i32 0, ptr [[GEP]], align 4
 // COMMON-NEXT:   ret void
@@ -205,35 +205,35 @@ void unamed_struct_typedef(TypedefS *ptr) {
   ptr->i1 = 0;
 }
 
-// ENABLED: [[P2INT_0]] = !{[[P2INT:!.+]], [[P2INT]], i64 0}
-// ENABLED: [[P2INT]] = !{!"p2 int", [[ANY_POINTER:!.+]], i64 0}
-// DEFAULT: [[ANYPTR]] = !{[[ANY_POINTER:!.+]], [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P2INT_0]] = !{[[P2INT:!.+]], [[P2INT]], i64 0}
+// DEFAULT: [[P2INT]] = !{!"p2 int", [[ANY_POINTER:!.+]], i64 0}
+// DISABLE: [[ANYPTR]] = !{[[ANY_POINTER:!.+]], [[ANY_POINTER]], i64 0}
 // COMMON: [[ANY_POINTER]] = !{!"any pointer", [[CHAR:!.+]], i64 0}
 // COMMON: [[CHAR]] = !{!"omnipotent char", [[TBAA_ROOT:!.+]], i64 0}
 // COMMON: [[TBAA_ROOT]] = !{!"Simple C/C++ TBAA"}
-// ENABLED: [[P1INT_0]] = !{[[P1INT:!.+]], [[P1INT]], i64 0}
-// ENABLED: [[P1INT]] = !{!"p1 int", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P3INT_0]] = !{[[P3INT:!.+]], [[P3INT]], i64 0}
-// ENABLED: [[P3INT]] = !{!"p3 int", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P4CHAR_0]] = !{[[P4CHAR:!.+]], [[P4CHAR]], i64 0}
-// ENABLED: [[P4CHAR]] = !{!"p4 omnipotent char", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P3CHAR_0]] = !{[[P3CHAR:!.+]], [[P3CHAR]], i64 0}
-// ENABLED: [[P3CHAR]] = !{!"p3 omnipotent char", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P2CHAR_0]] = !{[[P2CHAR:!.+]], [[P2CHAR]], i64 0}
-// ENABLED: [[P2CHAR]] = !{!"p2 omnipotent char", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P1CHAR_0]] = !{[[P1CHAR:!.+]], [[P1CHAR]], i64 0}
-// ENABLED: [[P1CHAR]] = !{!"p1 omnipotent char", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P2S1_TAG]] = !{[[P2S1:!.+]], [[P2S1]], i64 0}
-// ENABLED: [[P2S1]] = !{!"p2 _ZTS2S1", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P1S1_TAG:!.+]] = !{[[P1S1:!.+]], [[P1S1]], i64 0}
-// ENABLED: [[P1S1]] = !{!"p1 _ZTS2S1", [[ANY_POINTER]], i64 0}
-// ENABLED: [[P1S2_TAG]] = !{[[P1S2:!.+]], [[P1S2]], i64 0}
-// ENABLED: [[P1S2]] = !{!"p1 _ZTS2S2", [[ANY_POINTER]], i64 0}
-
-// ENABLED: [[S2_S_TAG]]  = !{[[S2_TY:!.+]], [[P1S1]], i64 0}
-// ENABLED: [[S2_TY]]  = !{!"S2", [[P1S1]], i64 0}
-// DEFAULT: [[S2_S_TAG]]  = !{[[S2_TY:!.+]], [[ANY_POINTER]], i64 0}
-// DEFAULT: [[S2_TY]]  = !{!"S2", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P1INT_0]] = !{[[P1INT:!.+]], [[P1INT]], i64 0}
+// DEFAULT: [[P1INT]] = !{!"p1 int", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P3INT_0]] = !{[[P3INT:!.+]], [[P3INT]], i64 0}
+// DEFAULT: [[P3INT]] = !{!"p3 int", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P4CHAR_0]] = !{[[P4CHAR:!.+]], [[P4CHAR]], i64 0}
+// DEFAULT: [[P4CHAR]] = !{!"p4 omnipotent char", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P3CHAR_0]] = !{[[P3CHAR:!.+]], [[P3CHAR]], i64 0}
+// DEFAULT: [[P3CHAR]] = !{!"p3 omnipotent char", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P2CHAR_0]] = !{[[P2CHAR:!.+]], [[P2CHAR]], i64 0}
+// DEFAULT: [[P2CHAR]] = !{!"p2 omnipotent char", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P1CHAR_0]] = !{[[P1CHAR:!.+]], [[P1CHAR]], i64 0}
+// DEFAULT: [[P1CHAR]] = !{!"p1 omnipotent char", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P2S1_TAG]] = !{[[P2S1:!.+]], [[P2S1]], i64 0}
+// DEFAULT: [[P2S1]] = !{!"p2 _ZTS2S1", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P1S1_TAG:!.+]] = !{[[P1S1:!.+]], [[P1S1]], i64 0}
+// DEFAULT: [[P1S1]] = !{!"p1 _ZTS2S1", [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P1S2_TAG]] = !{[[P1S2:!.+]], [[P1S2]], i64 0}
+// DEFAULT: [[P1S2]] = !{!"p1 _ZTS2S2", [[ANY_POINTER]], i64 0}
+
+// DEFAULT: [[S2_S_TAG]]  = !{[[S2_TY:!.+]], [[P1S1]], i64 0}
+// DEFAULT: [[S2_TY]]  = !{!"S2", [[P1S1]], i64 0}
+// DISABLE: [[S2_S_TAG]]  = !{[[S2_TY:!.+]], [[ANY_POINTER]], i64 0}
+// DISABLE: [[S2_TY]]  = !{!"S2", [[ANY_POINTER]], i64 0}
 // COMMON:  [[INT_TAG]] = !{[[INT_TY:!.+]], [[INT_TY]], i64 0}
 // COMMON:  [[INT_TY]] = !{!"int", [[CHAR]], i64 0}
-// ENABLED: [[P1TYPEDEF]] = !{[[ANY_POINTER]],  [[ANY_POINTER]], i64 0}
+// DEFAULT: [[P1TYPEDEF]] = !{[[ANY_POINTER]],  [[ANY_POINTER]], i64 0}
diff --git a/clang/test/CodeGen/tbaa-reference.cpp b/clang/test/CodeGen/tbaa-reference.cpp
index 8395badf35ded..1f62849bb6f42 100644
--- a/clang/test/CodeGen/tbaa-reference.cpp
+++ b/clang/test/CodeGen/tbaa-reference.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s  -check-prefixes=CHECK,OLD-PATH
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes -pointer-tbaa %s -emit-llvm -o - | FileCheck %s  -check-prefixes=CHECK,OLD-PATH-POINTER
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -emit-llvm -new-struct-path-tbaa -o - | FileCheck %s -check-prefixes=CHECK,NEW-PATH
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -pointer-tbaa -emit-llvm -new-struct-path-tbaa -o - | FileCheck %s -check-prefixes=CHECK,NEW-PATH-POINTER
+// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes -no-pointer-tbaa %s -emit-llvm -o - | FileCheck %s  -check-prefixes=CHECK,OLD-PATH
+// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s  -check-prefixes=CHECK,OLD-PATH-POINTER
+// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -no-pointer-tbaa -emit-llvm -new-struct-path-tbaa -o - | FileCheck %s -check-prefixes=CHECK,NEW-PATH
+// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s -emit-llvm -new-struct-path-tbaa -o - | FileCheck %s -check-prefixes=CHECK,NEW-PATH-POINTER
 //
 // Check that we generate correct TBAA information for reference accesses.
 
diff --git a/clang/test/CodeGen/ubsan-trap-merge.c b/clang/test/CodeGen/ubsan-trap-merge.c
index dccce0ce4c56b..412ec7b09744e 100644
--- a/clang/test/CodeGen/ubsan-trap-merge.c
+++ b/clang/test/CodeGen/ubsan-trap-merge.c
@@ -1,67 +1,168 @@
 // NOTE: Assertions have mostly been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
 // The most important assertion is the attributes at the end of the file, which
-// shows that -ubsan-unique-traps attaches 'nomerge' to each ubsantrap intrinsic.
+// shows whether -ubsan-unique-traps attaches 'nomerge' to each ubsan call.
 //
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=signed-integer-overflow -fsanitize-trap=signed-integer-overflow -O3 -mllvm -ubsan-unique-traps %s -o - \
-// RUN:     | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=signed-integer-overflow -O3 -mllvm -ubsan-unique-traps %s -o - -fsanitize-trap=signed-integer-overflow | FileCheck %s --check-prefix=TRAP
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=signed-integer-overflow -O3 -mllvm -ubsan-unique-traps %s -o -                                         | FileCheck %s --check-prefix=HANDLER
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=signed-integer-overflow -O3 -mllvm -ubsan-unique-traps %s -o - -fsanitize-minimal-runtime              | FileCheck %s --check-prefix=MINRT
 //
 // REQUIRES: x86-registered-target
 
-// CHECK-LABEL: define dso_local range(i32 -2147483523, -2147483648) i32 @f(
-// CHECK-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2:![0-9]+]]
-// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4:[0-9]+]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[CONT]]:
-// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// TRAP-LABEL: define dso_local range(i32 -2147483523, -2147483648) i32 @f(
+// TRAP-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// TRAP-NEXT:  [[ENTRY:.*:]]
+// TRAP-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2:![0-9]+]]
+// TRAP-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4:[0-9]+]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[CONT]]:
+// TRAP-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    ret i32 [[TMP2]]
+//
+// HANDLER-LABEL: define dso_local range(i32 -2147483523, -2147483648) i32 @f(
+// HANDLER-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// HANDLER-NEXT:  [[ENTRY:.*:]]
+// HANDLER-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2:![0-9]+]]
+// HANDLER-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW]]:
+// HANDLER-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[X]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP2]], i64 125) #[[ATTR4:[0-9]+]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[CONT]]:
+// HANDLER-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    ret i32 [[TMP3]]
+//
+// MINRT-LABEL: define dso_local range(i32 -2147483523, -2147483648) i32 @f(
+// MINRT-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// MINRT-NEXT:  [[ENTRY:.*:]]
+// MINRT-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2:![0-9]+]]
+// MINRT-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4:[0-9]+]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[CONT]]:
+// MINRT-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    ret i32 [[TMP2]]
 //
 int f(int x) {
   return x + 125;
 }
 
-// CHECK-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @g(
-// CHECK-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[CONT]]:
-// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// TRAP-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @g(
+// TRAP-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// TRAP-NEXT:  [[ENTRY:.*:]]
+// TRAP-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[CONT]]:
+// TRAP-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    ret i32 [[TMP2]]
+//
+// HANDLER-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @g(
+// HANDLER-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// HANDLER-NEXT:  [[ENTRY:.*:]]
+// HANDLER-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW]]:
+// HANDLER-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[X]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP2]], i64 127) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[CONT]]:
+// HANDLER-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    ret i32 [[TMP3]]
+//
+// MINRT-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @g(
+// MINRT-SAME: i32 noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// MINRT-NEXT:  [[ENTRY:.*:]]
+// MINRT-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[CONT]]:
+// MINRT-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    ret i32 [[TMP2]]
 //
 int g(int x) {
   return x + 127;
 }
 
-// CHECK-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @h(
-// CHECK-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[CONT]]:
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 129), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[TRAP1:.*]], label %[[CONT2:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP1]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[CONT2]]:
-// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    [[COND:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP5]], i32 [[TMP4]])
-// CHECK-NEXT:    ret i32 [[COND]]
+// TRAP-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @h(
+// TRAP-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// TRAP-NEXT:  [[ENTRY:.*:]]
+// TRAP-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP1]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[CONT]]:
+// TRAP-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 129), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP3]], label %[[TRAP1:.*]], label %[[CONT2:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP1]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[CONT2]]:
+// TRAP-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    [[COND:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP5]], i32 [[TMP4]])
+// TRAP-NEXT:    ret i32 [[COND]]
+//
+// HANDLER-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @h(
+// HANDLER-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// HANDLER-NEXT:  [[ENTRY:.*:]]
+// HANDLER-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW]]:
+// HANDLER-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[X]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP2]], i64 127) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[CONT]]:
+// HANDLER-NEXT:    [[TMP3:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 129), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP4]], label %[[HANDLER_ADD_OVERFLOW1:.*]], label %[[CONT2:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW1]]:
+// HANDLER-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[Y]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[TMP5]], i64 129) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[CONT2]]:
+// HANDLER-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP3]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    [[COND:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP7]], i32 [[TMP6]])
+// HANDLER-NEXT:    ret i32 [[COND]]
+//
+// MINRT-LABEL: define dso_local range(i32 -2147483521, -2147483648) i32 @h(
+// MINRT-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// MINRT-NEXT:  [[ENTRY:.*:]]
+// MINRT-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 127), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[CONT]]:
+// MINRT-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 129), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP3]], label %[[HANDLER_ADD_OVERFLOW1:.*]], label %[[CONT2:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW1]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[CONT2]]:
+// MINRT-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    [[COND:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP5]], i32 [[TMP4]])
+// MINRT-NEXT:    ret i32 [[COND]]
 //
 int h(int x, int y) {
   x += 127;
@@ -69,37 +170,100 @@ int h(int x, int y) {
   return x < y ? x : y;
 }
 
-// CHECK-LABEL: define dso_local noundef i32 @m(
-// CHECK-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP1]], label %[[TRAP_I:.*]], label %[[F_EXIT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP_I]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[F_EXIT]]:
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 127), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[TRAP_I2:.*]], label %[[G_EXIT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP_I2]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[G_EXIT]]:
-// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP6:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP4]], i32 [[TMP5]]), !nosanitize [[META2]]
-// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1, !nosanitize [[META2]]
-// CHECK-NEXT:    br i1 [[TMP7]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
-// CHECK:       [[TRAP]]:
-// CHECK-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
-// CHECK-NEXT:    unreachable, !nosanitize [[META2]]
-// CHECK:       [[CONT]]:
-// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP6]], 0, !nosanitize [[META2]]
-// CHECK-NEXT:    ret i32 [[TMP8]]
+// TRAP-LABEL: define dso_local noundef i32 @m(
+// TRAP-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// TRAP-NEXT:  [[ENTRY:.*:]]
+// TRAP-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP1]], label %[[TRAP_I:.*]], label %[[F_EXIT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP_I]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[F_EXIT]]:
+// TRAP-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 127), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP3]], label %[[TRAP_I2:.*]], label %[[G_EXIT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP_I2]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[G_EXIT]]:
+// TRAP-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP6:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP4]], i32 [[TMP5]]), !nosanitize [[META2]]
+// TRAP-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1, !nosanitize [[META2]]
+// TRAP-NEXT:    br i1 [[TMP7]], label %[[TRAP:.*]], label %[[CONT:.*]], !nosanitize [[META2]]
+// TRAP:       [[TRAP]]:
+// TRAP-NEXT:    tail call void @llvm.ubsantrap(i8 0) #[[ATTR4]], !nosanitize [[META2]]
+// TRAP-NEXT:    unreachable, !nosanitize [[META2]]
+// TRAP:       [[CONT]]:
+// TRAP-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP6]], 0, !nosanitize [[META2]]
+// TRAP-NEXT:    ret i32 [[TMP8]]
+//
+// HANDLER-LABEL: define dso_local noundef i32 @m(
+// HANDLER-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// HANDLER-NEXT:  [[ENTRY:.*:]]
+// HANDLER-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW_I:.*]], label %[[F_EXIT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW_I]]:
+// HANDLER-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[X]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB1]], i64 [[TMP2]], i64 125) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[F_EXIT]]:
+// HANDLER-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP4:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 127), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP5]], label %[[HANDLER_ADD_OVERFLOW_I2:.*]], label %[[G_EXIT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW_I2]]:
+// HANDLER-NEXT:    [[TMP6:%.*]] = zext nneg i32 [[Y]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB2]], i64 [[TMP6]], i64 127) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[G_EXIT]]:
+// HANDLER-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP8:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP3]], i32 [[TMP7]]), !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1, !nosanitize [[META2]]
+// HANDLER-NEXT:    br i1 [[TMP9]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// HANDLER:       [[HANDLER_ADD_OVERFLOW]]:
+// HANDLER-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP3]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64, !nosanitize [[META2]]
+// HANDLER-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR4]], !nosanitize [[META2]]
+// HANDLER-NEXT:    unreachable, !nosanitize [[META2]]
+// HANDLER:       [[CONT]]:
+// HANDLER-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0, !nosanitize [[META2]]
+// HANDLER-NEXT:    ret i32 [[TMP12]]
+//
+// MINRT-LABEL: define dso_local noundef i32 @m(
+// MINRT-SAME: i32 noundef [[X:%.*]], i32 noundef [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// MINRT-NEXT:  [[ENTRY:.*:]]
+// MINRT-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 125), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP1]], label %[[HANDLER_ADD_OVERFLOW_I:.*]], label %[[F_EXIT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW_I]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[F_EXIT]]:
+// MINRT-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[Y]], i32 127), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP3]], label %[[HANDLER_ADD_OVERFLOW_I2:.*]], label %[[G_EXIT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW_I2]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[G_EXIT]]:
+// MINRT-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP6:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP4]], i32 [[TMP5]]), !nosanitize [[META2]]
+// MINRT-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1, !nosanitize [[META2]]
+// MINRT-NEXT:    br i1 [[TMP7]], label %[[HANDLER_ADD_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// MINRT:       [[HANDLER_ADD_OVERFLOW]]:
+// MINRT-NEXT:    tail call void @__ubsan_handle_add_overflow_minimal_abort() #[[ATTR4]], !nosanitize [[META2]]
+// MINRT-NEXT:    unreachable, !nosanitize [[META2]]
+// MINRT:       [[CONT]]:
+// MINRT-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP6]], 0, !nosanitize [[META2]]
+// MINRT-NEXT:    ret i32 [[TMP8]]
 //
 int m(int x, int y) {
   return f(x) + g(y);
 }
-//.
-// CHECK: attributes #[[ATTR4]] = { nomerge noreturn nounwind }
+// TRAP: attributes #[[ATTR4]] = { nomerge noreturn nounwind }
+// HANDLER: attributes #[[ATTR4]] = { nomerge noreturn nounwind }
+// MINRT: attributes #[[ATTR4]] = { nomerge noreturn nounwind }
diff --git a/clang/test/CodeGen/union-tbaa1.c b/clang/test/CodeGen/union-tbaa1.c
index 5263b1714c8c6..7d44c9a3fbe6b 100644
--- a/clang/test/CodeGen/union-tbaa1.c
+++ b/clang/test/CodeGen/union-tbaa1.c
@@ -16,21 +16,21 @@ void bar(vect32 p[][2]);
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARR]], i32 [[TMP0]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[TMP1]], [[NUM]]
-// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP0]]
+// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP0]]
 // CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA6:![0-9]+]]
 // CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARR]], i32 [[TMP0]], i32 1
 // CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[MUL6:%.*]] = mul i32 [[TMP2]], [[NUM]]
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP0]], i32 1
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP0]], i32 1
 // CHECK-NEXT:    store i32 [[MUL6]], ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[MUL]], 16
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[VEC]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INDEX]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP4]], i32 1
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX14]], i32 2
+// CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds nuw [4 x [2 x %union.vect32]], ptr [[TMP]], i32 0, i32 [[TMP4]], i32 1
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX14]], i32 2
 // CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX15]], align 2, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[CONV16:%.*]] = zext i16 [[TMP5]] to i32
-// CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i8, ptr [[VEC]], i32 4
+// CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i32 4
 // CHECK-NEXT:    store i32 [[CONV16]], ptr [[ARRAYIDX17]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    call void @bar(ptr noundef nonnull [[TMP]]) #[[ATTR3]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[TMP]]) #[[ATTR3]]
diff --git a/clang/test/CodeGenCUDA/amdgpu-atomic-ops.cu b/clang/test/CodeGenCUDA/amdgpu-atomic-ops.cu
index efe75be8488b3..0e5fe8fa35cf1 100644
--- a/clang/test/CodeGenCUDA/amdgpu-atomic-ops.cu
+++ b/clang/test/CodeGenCUDA/amdgpu-atomic-ops.cu
@@ -157,7 +157,7 @@ __global__ void ffp3(long double *p) {
 
 __device__ double ffp4(double *p, float f) {
   // CHECK-LABEL: @_Z4ffp4Pdf
-  // CHECK: fpext float {{.*}} to double
+  // CHECK: fpext contract float {{.*}} to double
   // SAFEIR: atomicrmw fsub ptr {{.*}} monotonic, align 8{{$}}
   // UNSAFEIR: atomicrmw fsub ptr {{.*}} monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
 
diff --git a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
index 838bdda825728..fab87aac18310 100644
--- a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
+++ b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
@@ -331,7 +331,7 @@ struct S {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[COERCE_SROA_0_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[TMP0]], align 8, !amdgpu.noclobber [[META4:![0-9]+]]
 // OPT-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[COERCE_SROA_0_0_COPYLOAD]] to ptr addrspace(1)
-// OPT-NEXT:    [[COERCE_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP0]], i64 8
+// OPT-NEXT:    [[COERCE_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP0]], i64 8
 // OPT-NEXT:    [[COERCE_SROA_2_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[COERCE_SROA_2_0__SROA_IDX]], align 8, !amdgpu.noclobber [[META4]]
 // OPT-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[COERCE_SROA_2_0_COPYLOAD]] to ptr addrspace(1)
 // OPT-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[TMP1]], align 4, !amdgpu.noclobber [[META4]]
@@ -432,13 +432,13 @@ __global__ void kernel4(struct S s) {
 // CHECK-SPIRV-NEXT:    ret void
 //
 // OPT-LABEL: define dso_local amdgpu_kernel void @_Z7kernel5P1S(
-// OPT-SAME: ptr addrspace(1) nocapture noundef readonly [[S_COERCE:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// OPT-SAME: ptr addrspace(1) nocapture noundef readonly [[S_COERCE:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr addrspace(1) [[S_COERCE]], align 8
 // OPT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // OPT-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 // OPT-NEXT:    store i32 [[INC]], ptr [[TMP0]], align 4
-// OPT-NEXT:    [[Y:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[S_COERCE]], i64 8
+// OPT-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[S_COERCE]], i64 8
 // OPT-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(1) [[Y]], align 8
 // OPT-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4
 // OPT-NEXT:    [[ADD:%.*]] = fadd contract float [[TMP3]], 1.000000e+00
@@ -539,7 +539,7 @@ struct T {
 // OPT-NEXT:  [[ENTRY:.*:]]
 // OPT-NEXT:    [[COERCE_SROA_0_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[TMP0]], align 8, !amdgpu.noclobber [[META4]]
 // OPT-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[COERCE_SROA_0_0_COPYLOAD]] to ptr addrspace(1)
-// OPT-NEXT:    [[COERCE_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP0]], i64 8
+// OPT-NEXT:    [[COERCE_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP0]], i64 8
 // OPT-NEXT:    [[COERCE_SROA_2_0_COPYLOAD:%.*]] = load ptr, ptr addrspace(4) [[COERCE_SROA_2_0__SROA_IDX]], align 8, !amdgpu.noclobber [[META4]]
 // OPT-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[COERCE_SROA_2_0_COPYLOAD]] to ptr addrspace(1)
 // OPT-NEXT:    [[TMP3:%.*]] = load float, ptr addrspace(1) [[TMP1]], align 4, !amdgpu.noclobber [[META4]]
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sme-atts.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sme-atts.cpp
new file mode 100644
index 0000000000000..09db59ac621a2
--- /dev/null
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sme-atts.cpp
@@ -0,0 +1,74 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme2 %s -emit-llvm -o - | FileCheck %s
+
+typedef __attribute__((neon_vector_type(2))) int int32x2_t;
+
+//
+// Streaming-Mode Attributes
+//
+
+// CHECK: define dso_local void @_Z12fn_streamingP11__SME_ATTRSIFvvELj1EE
+void fn_streaming(void (*foo)() __arm_streaming) { foo(); }
+
+// CHECK: define dso_local void @_Z23fn_streaming_compatibleP11__SME_ATTRSIFivELj2EE(
+void fn_streaming_compatible(int (*foo)() __arm_streaming_compatible) { foo(); }
+
+//
+// ZA Attributes
+//
+
+// CHECK: define dso_local void @_Z15fn_za_preservedP11__SME_ATTRSIF11__Int32x2_tvELj32EE(
+__arm_new("za") void fn_za_preserved(int32x2_t (*foo)() __arm_preserves("za")) { foo(); }
+
+// CHECK: define dso_local void @_Z8fn_za_inP11__SME_ATTRSIFvu13__SVFloat64_tELj8EES_(
+__arm_new("za") void fn_za_in(void (*foo)(__SVFloat64_t) __arm_in("za"), __SVFloat64_t x) { foo(x); }
+
+// CHECK: define dso_local noundef i32 @_Z9fn_za_outP11__SME_ATTRSIFivELj16EE(
+__arm_new("za") int fn_za_out(int (*foo)() __arm_out("za")) { return foo(); }
+
+// CHECK: define dso_local void @_Z11fn_za_inoutP11__SME_ATTRSIFvvELj24EE(
+__arm_new("za") void fn_za_inout(void (*foo)() __arm_inout("za")) { foo(); }
+
+
+//
+// ZT0 Attributes
+//
+
+// CHECK: define dso_local void @_Z16fn_zt0_preservedP11__SME_ATTRSIFivELj256EE(
+__arm_new("zt0") void fn_zt0_preserved(int (*foo)() __arm_preserves("zt0")) { foo(); }
+
+// CHECK: define dso_local void @_Z9fn_zt0_inP11__SME_ATTRSIFivELj64EE(
+__arm_new("zt0") void fn_zt0_in(int (*foo)() __arm_in("zt0")) { foo(); }
+
+// CHECK: define dso_local void @_Z10fn_zt0_outP11__SME_ATTRSIFivELj128EE(
+__arm_new("zt0") void fn_zt0_out(int (*foo)() __arm_out("zt0")) { foo(); }
+
+// CHECK: define dso_local void @_Z12fn_zt0_inoutP11__SME_ATTRSIFivELj192EE(
+__arm_new("zt0") void fn_zt0_inout(int (*foo)() __arm_inout("zt0")) { foo(); }
+
+//
+// Streaming-mode, ZA & ZT0 Attributes
+//
+
+// CHECK: define dso_local void @_Z17fn_all_attr_typesP11__SME_ATTRSIFivELj282EE(
+__arm_new("za") __arm_new("zt0")
+void fn_all_attr_types(int (*foo)() __arm_streaming_compatible __arm_inout("za") __arm_preserves("zt0"))
+{ foo(); }
+
+//
+// No SME Attributes
+//
+
+// CHECK: define dso_local void @_Z12no_sme_attrsPFvvE(
+void no_sme_attrs(void (*foo)()) { foo(); }
+
+// CHECK: define dso_local void @_Z24locally_streaming_callerPFvvE(
+__arm_locally_streaming void locally_streaming_caller(void (*foo)()) { foo(); }
+
+// CHECK: define dso_local void @_Z16streaming_callerv(
+void streaming_caller() __arm_streaming {}
+
+// CHECK: define dso_local void @_Z16za_shared_callerv(
+void za_shared_caller() __arm_in("za") {}
+
+// CHECK: define dso_local void @_Z17zt0_shared_callerv(
+void zt0_shared_caller() __arm_out("zt0") {}
diff --git a/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp b/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
index f9068364d0dcb..bb71eb5636652 100644
--- a/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
+++ b/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
@@ -1212,3 +1212,20 @@ void test_copy_mf8x3(__clang_svmfloat8x3_t a) {
 void test_copy_mf8x4(__clang_svmfloat8x4_t a) {
   __clang_svmfloat8x4_t b{a};
 }
+
+/// Reduced from: https://github.com/llvm/llvm-project/issues/107609
+using vec_t = __SVInt8_t;
+
+// CHECK-LABEL: define dso_local void @_Z20test_copy_s8_typedefu10__SVInt8_t
+// CHECK-SAME: (<vscale x 16 x i8> [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <vscale x 16 x i8>, align 16
+// CHECK-NEXT:    [[VEC:%.*]] = alloca <vscale x 16 x i8>, align 16
+// CHECK-NEXT:    store <vscale x 16 x i8> [[A]], ptr [[A_ADDR]], align 16
+// CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 16 x i8>, ptr [[A_ADDR]], align 16
+// CHECK-NEXT:    store <vscale x 16 x i8> [[TMP0]], ptr [[VEC]], align 16
+// CHECK-NEXT:    ret void
+//
+void test_copy_s8_typedef(__SVInt8_t a) {
+  vec_t vec{a};
+}
diff --git a/clang/test/CodeGenCXX/auto-var-init.cpp b/clang/test/CodeGenCXX/auto-var-init.cpp
index 7c323c2d368c0..94386e44573b5 100644
--- a/clang/test/CodeGenCXX/auto-var-init.cpp
+++ b/clang/test/CodeGenCXX/auto-var-init.cpp
@@ -1296,7 +1296,7 @@ TEST_CUSTOM(semivolatile, semivolatile, { 0x44444444, 0x44444444 });
 // CHECK-NOT:   !annotation
 // CHECK-O0:  call void @{{.*}}used{{.*}}%custom)
 // PATTERN-O1:       store i32 1145324612, ptr %custom, align 4
-// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4
+// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds nuw i8, ptr %custom, i64 4
 // PATTERN-O1-NEXT:  store i32 1145324612, ptr %[[I]], align 4
 // ZERO-O1:          store i64 4919131752989213764, ptr %custom, align 8
 // CHECK-NOT:   !annotation
@@ -1338,7 +1338,7 @@ TEST_UNINIT(base, base);
 // PATTERN-O0: call void @llvm.memcpy{{.*}} @__const.test_base_uninit.uninit{{.+}}), !annotation [[AUTO_INIT]]
 // ZERO-LABEL: @test_base_uninit()
 // ZERO-O0: call void @llvm.memset{{.*}}, i8 0,{{.+}}), !annotation [[AUTO_INIT]]
-// ZERO-O1: store ptr getelementptr inbounds inrange(-16, 16) (i8, ptr @_ZTV4base, i64 16), {{.*}}, align 8
+// ZERO-O1: store ptr getelementptr inbounds nuw inrange(-16, 16) (i8, ptr @_ZTV4base, i64 16), {{.*}}, align 8
 // ZERO-O1-NOT: !annotation
 
 TEST_BRACES(base, base);
@@ -1359,7 +1359,7 @@ TEST_UNINIT(derived, derived);
 // ZERO-LABEL: @test_derived_uninit()
 // ZERO-O0: call void @llvm.memset{{.*}}, i8 0, {{.+}}), !annotation [[AUTO_INIT]]
 // ZERO-O1: store i64 0, {{.*}} align 8, !annotation [[AUTO_INIT]]
-// ZERO-O1: store ptr getelementptr inbounds inrange(-16, 16) (i8, ptr @_ZTV7derived, i64 16), {{.*}} align 8
+// ZERO-O1: store ptr getelementptr inbounds nuw inrange(-16, 16) (i8, ptr @_ZTV7derived, i64 16), {{.*}} align 8
 
 TEST_BRACES(derived, derived);
 // CHECK-LABEL: @test_derived_braces()
@@ -1494,11 +1494,11 @@ TEST_CUSTOM(unmatchedreverse, unmatchedreverse, { .c = 42  });
 // CHECK-NOT:   !annotation
 // CHECK-O0:    call void @{{.*}}used{{.*}}%custom)
 // PATTERN-O1:  store i8 42, ptr {{.*}}, align 4
-// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 1
+// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds nuw i8, ptr %custom, i64 1
 // PATTERN-O1-NEXT:  store i8 -86, ptr %[[I]], align {{.*}}
-// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 2
+// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds nuw i8, ptr %custom, i64 2
 // PATTERN-O1-NEXT:  store i8 -86, ptr %[[I]], align {{.*}}
-// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 3
+// PATTERN-O1-NEXT:  %[[I:[^ ]*]] = getelementptr inbounds nuw i8, ptr %custom, i64 3
 // PATTERN-O1-NEXT:  store i8 -86, ptr %[[I]], align {{.*}}
 // ZERO-O1:     store i32 42, ptr {{.*}}, align 4
 
diff --git a/clang/test/CodeGenCXX/catch-undef-behavior.cpp b/clang/test/CodeGenCXX/catch-undef-behavior.cpp
index a985183129911..419d1292551a0 100644
--- a/clang/test/CodeGenCXX/catch-undef-behavior.cpp
+++ b/clang/test/CodeGenCXX/catch-undef-behavior.cpp
@@ -734,6 +734,6 @@ void ThisAlign::this_align_lambda_2() {
   p();
 }
 
-// CHECK: attributes [[NR_NUW]] = { noreturn nounwind }
+// CHECK: attributes [[NR_NUW]] = { nomerge noreturn nounwind }
 
 // CHECK-FUNCSAN: ![[FUNCSAN]] = !{i32 -1056584962, i32 -1000226989}
diff --git a/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp b/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
index 4b046acf0d878..a5d0d5ad6e91f 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
@@ -18,7 +18,7 @@ T* test1(V* x) { return &dynamic_cast<T&>(*x); }
 T* test2(A* x) { return &dynamic_cast<T&>(*x); }
 // CHECK-LABEL: define dso_local noundef ptr @"?test2@@YAPAUT@@PAUA@@@Z"(ptr noundef %x)
 // CHECK:        [[VBTBL:%.*]] = load ptr, ptr %x, align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[VBOFFS]]
 // CHECK-NEXT:   [[CALL:%.*]] = tail call ptr @__RTDynamicCast(ptr nonnull [[ADJ]], i32 [[VBOFFS]], ptr nonnull @"??_R0?AUA@@@8", ptr nonnull @"??_R0?AUT@@@8", i32 1)
@@ -26,9 +26,9 @@ T* test2(A* x) { return &dynamic_cast<T&>(*x); }
 
 T* test3(B* x) { return &dynamic_cast<T&>(*x); }
 // CHECK-LABEL: define dso_local noundef ptr @"?test3@@YAPAUT@@PAUB@@@Z"(ptr noundef %x)
-// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds i8, ptr %x, i32 4
+// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds nuw i8, ptr %x, i32 4
 // CHECK-NEXT:   [[VBTBL:%.*]] = load ptr, ptr [[VBPTR:%.*]], align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[DELTA:%.*]] = add nsw i32 [[VBOFFS]], 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[DELTA]]
@@ -45,7 +45,7 @@ T* test5(A* x) { return dynamic_cast<T*>(x); }
 // CHECK:        [[CHECK:%.*]] = icmp eq ptr %x, null
 // CHECK-NEXT:   br i1 [[CHECK]]
 // CHECK:        [[VBTBL:%.*]] = load ptr, ptr %x, align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[VBOFFS]]
 // CHECK-NEXT:   [[CALL:%.*]] = tail call ptr @__RTDynamicCast(ptr nonnull [[ADJ]], i32 [[VBOFFS]], ptr {{.*}}@"??_R0?AUA@@@8", ptr {{.*}}@"??_R0?AUT@@@8", i32 0)
@@ -57,9 +57,9 @@ T* test6(B* x) { return dynamic_cast<T*>(x); }
 // CHECK-LABEL: define dso_local noundef ptr @"?test6@@YAPAUT@@PAUB@@@Z"(ptr noundef %x)
 // CHECK:        [[CHECK:%.*]] = icmp eq ptr %x, null
 // CHECK-NEXT:   br i1 [[CHECK]]
-// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds i8, ptr %x, i32 4
+// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds nuw i8, ptr %x, i32 4
 // CHECK-NEXT:   [[VBTBL:%.*]] = load ptr, ptr [[VBPTR]], align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[DELTA:%.*]] = add nsw i32 [[VBOFFS]], 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[DELTA]]
@@ -78,7 +78,7 @@ void* test8(A* x) { return dynamic_cast<void*>(x); }
 // CHECK:        [[CHECK:%.*]] = icmp eq ptr %x, null
 // CHECK-NEXT:   br i1 [[CHECK]]
 // CHECK:        [[VBTBL:%.*]] = load ptr, ptr %x, align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[VBOFFS]]
 // CHECK-NEXT:   [[RES:%.*]] = tail call ptr @__RTCastToVoid(ptr nonnull [[ADJ]])
@@ -90,9 +90,9 @@ void* test9(B* x) { return dynamic_cast<void*>(x); }
 // CHECK-LABEL: define dso_local noundef ptr @"?test9@@YAPAXPAUB@@@Z"(ptr noundef %x)
 // CHECK:        [[CHECK:%.*]] = icmp eq ptr %x, null
 // CHECK-NEXT:   br i1 [[CHECK]]
-// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds i8, ptr %x, i32 4
+// CHECK:        [[VBPTR:%.*]] = getelementptr inbounds nuw i8, ptr %x, i32 4
 // CHECK-NEXT:   [[VBTBL:%.*]] = load ptr, ptr [[VBPTR]], align 4
-// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
 // CHECK-NEXT:   [[BASE:%.*]] = getelementptr i8, ptr %x, i32 [[VBOFFS]]
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr i8, ptr [[BASE]], i32 4
diff --git a/clang/test/CodeGenCXX/microsoft-abi-typeid.cpp b/clang/test/CodeGenCXX/microsoft-abi-typeid.cpp
index 31d7d23fd6fc5..24a19cd856602 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-typeid.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-typeid.cpp
@@ -31,7 +31,7 @@ const std::type_info* test3_typeid() { return &typeid(*fn()); }
 // CHECK:        call ptr @__RTtypeid(ptr null)
 // CHECK-NEXT:   unreachable
 // CHECK:        [[VBTBL:%.*]] = load ptr, ptr [[CALL]], align 4
-// CHECK-NEXT:   [[VBSLOT:%.*]] = getelementptr inbounds i8, ptr [[VBTBL]], i32 4
+// CHECK-NEXT:   [[VBSLOT:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
 // CHECK-NEXT:   [[VBASE_OFFS:%.*]] = load i32, ptr [[VBSLOT]], align 4
 // CHECK-NEXT:   [[ADJ:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i32 [[VBASE_OFFS]]
 // CHECK-NEXT:   [[RT:%.*]] = tail call ptr @__RTtypeid(ptr nonnull [[ADJ]])
diff --git a/clang/test/CodeGenCXX/ms-uneval-context-crash.cpp b/clang/test/CodeGenCXX/ms-uneval-context-crash.cpp
new file mode 100644
index 0000000000000..b2f7e58381da8
--- /dev/null
+++ b/clang/test/CodeGenCXX/ms-uneval-context-crash.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -std=c++20 -fms-compatibility -fms-compatibility-version=19.33 -emit-llvm %s -o - -triple=x86_64-windows-msvc | FileCheck %s
+
+template <typename T>
+concept C = requires
+{
+    { T::test([](){}) };
+};
+
+template<typename T>
+struct Widget {};
+
+template <C T>
+struct Widget<T> {};
+
+struct Baz
+{
+    template<typename F>
+    static constexpr decltype(auto) test(F&&) {}
+};
+
+void test()
+{
+    Widget<Baz> w;
+}
+// CHECK: @"?test@@YAXXZ"
diff --git a/clang/test/CodeGenCXX/template-instantiation.cpp b/clang/test/CodeGenCXX/template-instantiation.cpp
index 15d409447c117..8453bcefc7ebb 100644
--- a/clang/test/CodeGenCXX/template-instantiation.cpp
+++ b/clang/test/CodeGenCXX/template-instantiation.cpp
@@ -17,6 +17,7 @@
 
 // CHECK2-NOT: _ZTVN5test31SIiEE
 // CHECK2-NOT: _ZTSN5test31SIiEE
+// CHECK2:  !{!"p1 _ZTSN5test31SIiEE",
 
 // CHECK-LABEL: define linkonce_odr void @_ZN5test21CIiEC1Ev(ptr {{[^,]*}} %this) unnamed_addr
 // CHECK-LABEL: define linkonce_odr void @_ZN5test21CIiE6foobarIdEEvT_(
diff --git a/clang/test/CodeGenHIP/printf_nonhostcall.cpp b/clang/test/CodeGenHIP/printf_nonhostcall.cpp
index 2c6d0ecac1e8a..1982eb864e2a4 100644
--- a/clang/test/CodeGenHIP/printf_nonhostcall.cpp
+++ b/clang/test/CodeGenHIP/printf_nonhostcall.cpp
@@ -263,7 +263,7 @@ __device__ _BitInt(128) Int128 = 45637;
 // CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[TMP1]] to i32
 // CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr addrspacecast (ptr addrspace(1) @n to ptr), align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @f1 to ptr), align 4
-// CHECK-NEXT:    [[CONV1:%.*]] = fpext float [[TMP3]] to double
+// CHECK-NEXT:    [[CONV1:%.*]] = fpext contract float [[TMP3]] to double
 // CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @f2 to ptr), align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = load half, ptr addrspacecast (ptr addrspace(1) @f3 to ptr), align 2
 // CHECK-NEXT:    [[TMP6:%.*]] = load bfloat, ptr addrspacecast (ptr addrspace(1) @f4 to ptr), align 2
@@ -324,7 +324,7 @@ __device__ _BitInt(128) Int128 = 45637;
 // CHECK_CONSTRAINED-NEXT:    [[CONV:%.*]] = zext i16 [[TMP1]] to i32
 // CHECK_CONSTRAINED-NEXT:    [[TMP2:%.*]] = load i64, ptr addrspacecast (ptr addrspace(1) @n to ptr), align 8
 // CHECK_CONSTRAINED-NEXT:    [[TMP3:%.*]] = load float, ptr addrspacecast (ptr addrspace(1) @f1 to ptr), align 4
-// CHECK_CONSTRAINED-NEXT:    [[CONV1:%.*]] = fpext float [[TMP3]] to double
+// CHECK_CONSTRAINED-NEXT:    [[CONV1:%.*]] = fpext contract float [[TMP3]] to double
 // CHECK_CONSTRAINED-NEXT:    [[TMP4:%.*]] = load double, ptr addrspacecast (ptr addrspace(1) @f2 to ptr), align 8
 // CHECK_CONSTRAINED-NEXT:    [[TMP5:%.*]] = load half, ptr addrspacecast (ptr addrspace(1) @f3 to ptr), align 2
 // CHECK_CONSTRAINED-NEXT:    [[TMP6:%.*]] = load bfloat, ptr addrspacecast (ptr addrspace(1) @f4 to ptr), align 2
diff --git a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
index a0dfe26e5d147..e2ff2de68ed99 100644
--- a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
+++ b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
@@ -100,18 +100,16 @@ void arr_assign6() {
 }
 
 // CHECK-LABEL: define void {{.*}}arr_assign7
-// CHECK: [[Arr3:%.*]] = alloca [2 x [2 x i32]], align 4
-// CHECK-NEXT: [[Arr4:%.*]] = alloca [2 x [2 x i32]], align 4
-// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK: [[Arr:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: [[Arr2:%.*]] = alloca [2 x [2 x i32]], align 4
 // CHECK-NOT: alloca
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 16, i1 false)
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr4]], ptr align 4 {{@.*}}, i32 16, i1 false)
-// CHECK-NEXT: store i32 6, ptr [[Tmp]], align 4
-// CHECK-NEXT: [[AIE:%.*]] = getelementptr inbounds i32, ptr [[Tmp]], i32 1
-// CHECK-NEXT: store i32 6, ptr [[AIE]], align 4
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 [[Arr4]], i32 16, i1 false)
-// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[Arr3]], i32 0, i32 0
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Idx]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr2]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 [[Arr2]], i32 16, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[Arr]], i32 0, i32 0
+// CHECK-NEXT: store i32 6, ptr [[Idx]], align 4
+// CHECK-NEXT: [[Idx2:%.*]] = getelementptr inbounds i32, ptr %arrayidx, i32 1
+// CHECK-NEXT: store i32 6, ptr [[Idx2]], align 4
 // CHECK-NEXT: ret void
 void arr_assign7() {
   int Arr[2][2] = {{0, 1}, {2, 3}};
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/ArrayOutputArguments.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/ArrayOutputArguments.hlsl
new file mode 100644
index 0000000000000..eb7d755bca61d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/ArrayOutputArguments.hlsl
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -disable-llvm-passes -emit-llvm -finclude-default-header -o - %s | FileCheck %s
+
+// CHECK-LABEL: increment
+void increment(inout int Arr[2]) {
+  for (int I = 0; I < 2; I++)
+    Arr[0] += 2;
+}
+
+// CHECK-LABEL: arrayCall
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: call void @{{.*}}increment{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i32 0, i32 0
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[Idx]], align 4
+// CHECK-NEXT: ret i32 [[B]]
+export int arrayCall() {
+  int A[2] = { 0, 1 };
+  increment(A);
+  return A[0];
+}
+
+// CHECK-LABEL: fn2
+void fn2(out int Arr[2]) {
+  Arr[0] += 5;
+  Arr[1] += 6;
+}
+
+// CHECK-LABEL: arrayCall2
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @{{.*}}fn2{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i32 0, i32 0
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[Idx]], align 4
+// CHECK-NEXT: ret i32 [[B]]
+export int arrayCall2() {
+  int A[2] = { 0, 1 };
+  fn2(A);
+  return A[0];
+}
+
+// CHECK-LABEL: nestedCall
+void nestedCall(inout int Arr[2], uint index) {
+  if (index < 2) {
+    Arr[index] += 2;
+    nestedCall(Arr, index+1);
+  }
+}
+
+// CHECK-LABEL: arrayCall3
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: call void @{{.*}}nestedCall{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]], i32 noundef 0) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i32 0, i32 1
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[Idx]], align 4
+// CHECK-NEXt: ret i32 [[B]]
+export int arrayCall3() {
+  int A[2] = { 0, 1 };
+  nestedCall(A, 0);
+  return A[1];
+}
+
+// CHECK-LABEL: outerCall
+// CHECK: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 %{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void {{.*}}increment{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 {{.*}}, ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: ret void
+void outerCall(inout int Arr[2]) {
+  increment(Arr);
+}
+
+// CHECK-LABEL: arrayCall4
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: call void @{{.*}}outerCall{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i32 0, i32 0
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[Idx]], align 4
+// CHECK-NEXT: ret i32 [[B]]
+export int arrayCall4() {
+  int A[2] = { 0, 1 };
+  outerCall(A);
+  return A[0];
+}
+
+// CHECK-LABEL: fn3
+void fn3(int Arr[2]) {}
+
+// CHECK-LABEL: outerCall2
+// CHECK: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void {{.*}}fn3{{.*}}(ptr noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: ret void
+void outerCall2(inout int Arr[2]) {
+  fn3(Arr);
+}
+
+// CHECK-LABEL: arrayCall5
+// CHECK: [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @{{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Tmp]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT: call void @{{.*}}outerCall2{{.*}}(ptr noalias noundef byval([2 x i32]) align 4 [[Tmp]]) #3
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i32 0, i32 0
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[Idx]], align 4
+// CHECK-NEXT: ret i32 [[B]]
+export int arrayCall5() {
+  int A[2] = { 0, 1 };
+  outerCall2(A);
+  return A[0];
+}
diff --git a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
index 1e8aae588fc33..85face8eaeb6c 100644
--- a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
@@ -18,7 +18,8 @@ struct MyStruct {
 // DXIL: %"class.hlsl::AppendStructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
-// DXIL: %"class.hlsl::AppendStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 1, 0)
+// DXIL: %"class.hlsl::AppendStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
+// DXIL: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
 
 AppendStructuredBuffer<int16_t> BufI16;
 AppendStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffers-constructors.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffers-constructors.hlsl
new file mode 100644
index 0000000000000..45e135427ba9c
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffers-constructors.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN-DISABLED: %clang_cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+
+// NOTE: SPIRV codegen for resource types is not yet implemented
+
+ByteAddressBuffer Buffer0: register(t0);
+RWByteAddressBuffer Buffer1: register(u1, space2);
+RasterizerOrderedByteAddressBuffer Buffer2: register(u3, space4);
+
+// CHECK: "class.hlsl::ByteAddressBuffer" = type { target("dx.RawBuffer", i8, 0, 0) }
+// CHECK: "class.hlsl::RWByteAddressBuffer" = type { target("dx.RawBuffer", i8, 1, 0) }
+// CHECK: "class.hlsl::RasterizerOrderedByteAddressBuffer" = type { target("dx.RawBuffer", i8, 1, 1) }
+
+// CHECK: @Buffer0 = global %"class.hlsl::ByteAddressBuffer" zeroinitializer, align 4
+// CHECK: @Buffer1 = global %"class.hlsl::RWByteAddressBuffer" zeroinitializer, align 4
+// CHECK: @Buffer2 = global %"class.hlsl::RasterizerOrderedByteAddressBuffer" zeroinitializer, align 4
+
+// CHECK: define internal void @_GLOBAL__sub_I_ByteAddressBuffers_constructors.hlsl()
+// CHECK: entry:
+// CHECK: call void @_init_resource_bindings()
+
+// CHECK: define internal void @_init_resource_bindings() {
+// CHECK-NEXT: entry:
+// CHECK-DXIL-NEXT: %Buffer0_h = call target("dx.RawBuffer", i8, 0, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false)
+// CHECK-DXIL-NEXT: store target("dx.RawBuffer", i8, 0, 0) %Buffer0_h, ptr @Buffer0, align 4
+// CHECK-DXIL-NEXT: %Buffer1_h = call target("dx.RawBuffer", i8, 1, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_1_0t(i32 2, i32 1, i32 1, i32 0, i1 false)
+// CHECK-DXIL-NEXT: store target("dx.RawBuffer", i8, 1, 0) %Buffer1_h, ptr @Buffer1, align 4
+// CHECK-DXIL-NEXT: %Buffer2_h = call target("dx.RawBuffer", i8, 1, 1) @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_1_1t(i32 4, i32 3, i32 1, i32 0, i1 false)
+// CHECK-DXIL-NEXT: store target("dx.RawBuffer", i8, 1, 1) %Buffer2_h, ptr @Buffer2, align 4
diff --git a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
index f8574c6460d4e..5ed9e9ad8160f 100644
--- a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
@@ -18,7 +18,8 @@ struct MyStruct {
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
-// DXIL: %"class.hlsl::ConsumeStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 1, 0)
+// DXIL: %"class.hlsl::ConsumeStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
+// DXIL: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
 
 ConsumeStructuredBuffer<int16_t> BufI16;
 ConsumeStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/GroupMemoryBarrierWithGroupSync.hlsl b/clang/test/CodeGenHLSL/builtins/GroupMemoryBarrierWithGroupSync.hlsl
new file mode 100644
index 0000000000000..9d95d54852c0b
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/GroupMemoryBarrierWithGroupSync.hlsl
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   -DTARGET=dx -DFNATTRS=noundef -check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   -DTARGET=spv -DFNATTRS="spir_func noundef" -check-prefixes=CHECK,CHECK-SPIRV
+
+// CHECK-DXIL: define void @
+// CHECK-SPIRV: define spir_func void @
+void test_GroupMemoryBarrierWithGroupSync() {
+// CHECK-DXIL: call void @llvm.[[TARGET]].group.memory.barrier.with.group.sync()
+// CHECK-SPIRV: call spir_func void @llvm.[[TARGET]].group.memory.barrier.with.group.sync()
+  GroupMemoryBarrierWithGroupSync();
+}
+
+// CHECK: declare void @llvm.[[TARGET]].group.memory.barrier.with.group.sync() #[[ATTRS:[0-9]+]]
+// CHECK-NOT: attributes #[[ATTRS]] = {{.+}}memory(none){{.+}}
+// CHECK: attributes #[[ATTRS]] = {
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
index 3949f7b943cfe..c2db56e2b2bdd 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
@@ -6,10 +6,10 @@
 
 RWBuffer<float> Buf : register(u5, space3);
 
-// CHECK: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", float, 1, 0, 0), float }
+// CHECK: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", float, 1, 0, 0) }
 // CHECK: @Buf = global %"class.hlsl::RWBuffer" zeroinitializer, align 4
 
-// CHECK: define linkonce_odr void @_ZN4hlsl8RWBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
+// CHECK: define linkonce_odr void @_ZN4hlsl8RWBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
 // CHECK-NEXT: entry:
 
 // CHECK: define internal void @_GLOBAL__sub_I_RWBuffer_constructor.hlsl()
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
index d737d76a9d8d9..11c77644a906d 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
@@ -1,37 +1,33 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
 // RUN: %clang_cc1 -triple spirv-pc-vulkan-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPIRV
 
-// NOTE: The type name number and whether the struct is packed or not will mostly
-// likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
-// and theinterim field of the contained type is removed.
+// DXIL: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", i16, 1, 0, 1) }
+// DXIL: %"class.hlsl::RWBuffer.0" = type { target("dx.TypedBuffer", i16, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.1" = type { target("dx.TypedBuffer", i32, 1, 0, 1) }
+// DXIL: %"class.hlsl::RWBuffer.2" = type { target("dx.TypedBuffer", i32, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.3" = type { target("dx.TypedBuffer", i64, 1, 0, 1) }
+// DXIL: %"class.hlsl::RWBuffer.4" = type { target("dx.TypedBuffer", i64, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.5" = type { target("dx.TypedBuffer", half, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.6" = type { target("dx.TypedBuffer", float, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.7" = type { target("dx.TypedBuffer", double, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.8" = type { target("dx.TypedBuffer", <4 x i16>, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.9" = type { target("dx.TypedBuffer", <3 x i32>, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.10" = type { target("dx.TypedBuffer", <2 x half>, 1, 0, 0) }
+// DXIL: %"class.hlsl::RWBuffer.11" = type { target("dx.TypedBuffer", <3 x float>, 1, 0, 0) }
 
-// DXIL: %"class.hlsl::RWBuffer" = type <{ target("dx.TypedBuffer", i16, 1, 0, 1)
-// DXIL: %"class.hlsl::RWBuffer.0" = type <{ target("dx.TypedBuffer", i16, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.2" = type { target("dx.TypedBuffer", i32, 1, 0, 1)
-// DXIL: %"class.hlsl::RWBuffer.3" = type { target("dx.TypedBuffer", i32, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.4" = type { target("dx.TypedBuffer", i64, 1, 0, 1)
-// DXIL: %"class.hlsl::RWBuffer.5" = type { target("dx.TypedBuffer", i64, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.6" = type <{ target("dx.TypedBuffer", half, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.8" = type { target("dx.TypedBuffer", float, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.9" = type { target("dx.TypedBuffer", double, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.10" = type { target("dx.TypedBuffer", <4 x i16>, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.11" = type { target("dx.TypedBuffer", <3 x i32>, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.12" = type { target("dx.TypedBuffer", <2 x half>, 1, 0, 0)
-// DXIL: %"class.hlsl::RWBuffer.13" = type { target("dx.TypedBuffer", <3 x float>, 1, 0, 0)
-
-// SPIRV: %"class.hlsl::RWBuffer" = type <{ target("spirv.Image", i16, 5, 2, 0, 0, 2, 0), i16, [6 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.0" = type <{ target("spirv.Image", i16, 5, 2, 0, 0, 2, 0), i16, [6 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.2" = type <{ target("spirv.Image", i32, 5, 2, 0, 0, 2, 0), i32, [4 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.4" = type <{ target("spirv.Image", i32, 5, 2, 0, 0, 2, 0), i32, [4 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.6" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0), i64 }
-// SPIRV: %"class.hlsl::RWBuffer.7" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0), i64 }
-// SPIRV: %"class.hlsl::RWBuffer.8" = type <{ target("spirv.Image", half, 5, 2, 0, 0, 2, 0), half, [6 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.10" = type <{ target("spirv.Image", float, 5, 2, 0, 0, 2, 0), float, [4 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.12" = type { target("spirv.Image", double, 5, 2, 0, 0, 2, 0), double }
-// SPIRV: %"class.hlsl::RWBuffer.13" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0), <4 x i16> }
-// SPIRV: %"class.hlsl::RWBuffer.14" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0), <3 x i32> }
-// SPIRV: %"class.hlsl::RWBuffer.15" = type <{ target("spirv.Image", half, 5, 2, 0, 0, 2, 0), <2 x half>, [4 x i8] }>
-// SPIRV: %"class.hlsl::RWBuffer.17" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0), <3 x float> }
+// SPIRV: %"class.hlsl::RWBuffer" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.0" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.1" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.2" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.3" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.4" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.5" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.6" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.7" = type { target("spirv.Image", double, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.8" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.9" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.10" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.11" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
 
 
 
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
index 2a350c1619bd6..8ce8417772530 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
@@ -5,13 +5,10 @@ RWBuffer<int> Out;
 
 [numthreads(1,1,1)]
 void main(unsigned GI : SV_GroupIndex) {
+  // CHECK: define void @main()
+  // CHECK: %[[INPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
+  // CHECK: %[[LOAD:.*]] = load i32, ptr %[[INPTR]]
+  // CHECK: %[[OUTPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
+  // CHECK: store i32 %[[LOAD]], ptr %[[OUTPTR]]
   Out[GI] = In[GI];
 }
-
-// Even at -O0 the subscript operators get inlined. The -O0 IR is a bit messy
-// and confusing to follow so the match here is pretty weak.
-
-// CHECK: define void @main()
-// Verify inlining leaves only calls to "llvm." intrinsics
-// CHECK-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
-// CHECK: ret void
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index 71b5b7a75fa43..f2f6956ce1541 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -1,22 +1,18 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
 
-// NOTE: The number in type name and whether the struct is packed or not will mostly
-// likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
-// and theinterim field of the contained type is removed.
-
-// CHECK: %"class.hlsl::RWStructuredBuffer" = type <{ target("dx.RawBuffer", i16, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.3" = type { target("dx.RawBuffer", i32, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.5" = type { target("dx.RawBuffer", i64, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 1, 0) 
-// CHECK: %"class.hlsl::RWStructuredBuffer.8" = type { target("dx.RawBuffer", float, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.9" = type { target("dx.RawBuffer", double, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 1, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.0" = type { target("dx.RawBuffer", i16, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.1" = type { target("dx.RawBuffer", i32, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.3" = type { target("dx.RawBuffer", i64, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.6" = type { target("dx.RawBuffer", float, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.7" = type { target("dx.RawBuffer", double, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.8" = type { target("dx.RawBuffer", <4 x i16>, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0) }
 
 RWStructuredBuffer<int16_t> BufI16;
 RWStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
index 1de3a4e3961af..68d626de689f2 100644
--- a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
@@ -1,28 +1,24 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
 
-// NOTE: The number in type name and whether the struct is packed or not will mostly
-// likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
-// and theinterim field of the contained type is removed.
-
 struct MyStruct {
   float4 a;
   int2 b;
 };
 
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer" = type <{ target("dx.RawBuffer", i16, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.3" = type { target("dx.RawBuffer", i32, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.5" = type { target("dx.RawBuffer", i64, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 1, 1) 
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.8" = type { target("dx.RawBuffer", float, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.9" = type { target("dx.RawBuffer", double, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 1, 1)
-// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.14" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 1, 1)
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.0" = type { target("dx.RawBuffer", i16, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.1" = type { target("dx.RawBuffer", i32, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.3" = type { target("dx.RawBuffer", i64, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 1)  }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.6" = type { target("dx.RawBuffer", float, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.7" = type { target("dx.RawBuffer", double, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.8" = type { target("dx.RawBuffer", <4 x i16>, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 1) }
+// DXIL: %"class.hlsl::RasterizerOrderedStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 1) }
+// DXIL: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
 
 RasterizerOrderedStructuredBuffer<int16_t> BufI16;
 RasterizerOrderedStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index 205e13b4de394..d322cdc0d0645 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -1,22 +1,18 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
 
-// NOTE: The number in type name and whether the struct is packed or not will mostly
-// likely change once subscript operators are properly implemented (llvm/llvm-project#95956) 
-// and theinterim field of the contained type is removed.
-
-// CHECK: %"class.hlsl::StructuredBuffer" = type <{ target("dx.RawBuffer", i16, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.0" = type <{ target("dx.RawBuffer", i16, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.2" = type { target("dx.RawBuffer", i32, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.3" = type { target("dx.RawBuffer", i32, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.4" = type { target("dx.RawBuffer", i64, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.5" = type { target("dx.RawBuffer", i64, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.6" = type <{ target("dx.RawBuffer", half, 0, 0) 
-// CHECK: %"class.hlsl::StructuredBuffer.8" = type { target("dx.RawBuffer", float, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.9" = type { target("dx.RawBuffer", double, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <4 x i16>, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x i32>, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", <2 x half>, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <3 x float>, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i16, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.0" = type { target("dx.RawBuffer", i16, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.1" = type { target("dx.RawBuffer", i32, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.2" = type { target("dx.RawBuffer", i32, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.3" = type { target("dx.RawBuffer", i64, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.4" = type { target("dx.RawBuffer", i64, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.5" = type { target("dx.RawBuffer", half, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.6" = type { target("dx.RawBuffer", float, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.7" = type { target("dx.RawBuffer", double, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.8" = type { target("dx.RawBuffer", <4 x i16>, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 0, 0) }
 
 StructuredBuffer<int16_t> BufI16;
 StructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
index 2e141b9279fa6..d84e92242ffb4 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-constructors.hlsl
@@ -9,11 +9,11 @@ AppendStructuredBuffer<float> Buf3 : register(u3);
 ConsumeStructuredBuffer<float> Buf4 : register(u4);
 RasterizerOrderedStructuredBuffer<float> Buf5 : register(u1, space2);
 
-// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", float, 0, 0)
-// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0)
-// CHECK: %"class.hlsl::AppendStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0)
-// CHECK: %"class.hlsl::ConsumeStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0)
-// CHECK: %"class.hlsl::RasterizerOrderedStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 1), float }
+// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", float, 0, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0) }
+// CHECK: %"class.hlsl::AppendStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0) }
+// CHECK: %"class.hlsl::ConsumeStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0) }
+// CHECK: %"class.hlsl::RasterizerOrderedStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 1) }
 
 // CHECK: @Buf = global %"class.hlsl::StructuredBuffer" zeroinitializer, align 4
 // CHECK: @Buf2 = global %"class.hlsl::RWStructuredBuffer" zeroinitializer, align 4
@@ -21,14 +21,14 @@ RasterizerOrderedStructuredBuffer<float> Buf5 : register(u1, space2);
 // CHECK: @Buf4 = global %"class.hlsl::ConsumeStructuredBuffer" zeroinitializer, align 4
 // CHECK: @Buf5 = global %"class.hlsl::RasterizerOrderedStructuredBuffer" zeroinitializer, align 4
 
-// CHECK: define linkonce_odr void @_ZN4hlsl16StructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
+// CHECK: define linkonce_odr void @_ZN4hlsl16StructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
 // CHECK-NEXT: entry:
-// CHECK: define linkonce_odr void @_ZN4hlsl18RWStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
+// CHECK: define linkonce_odr void @_ZN4hlsl18RWStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
 // CHECK-NEXT: entry:
 // CHECK: define linkonce_odr void @_ZN4hlsl22AppendStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
 // CHECK-NEXT: entry:
 // CHECK: define linkonce_odr void @_ZN4hlsl23ConsumeStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
-// CHECK: define linkonce_odr void @_ZN4hlsl33RasterizerOrderedStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(8) %this)
+// CHECK: define linkonce_odr void @_ZN4hlsl33RasterizerOrderedStructuredBufferIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) %this)
 // CHECK-NEXT: entry:
 
 // CHECK: define internal void @_GLOBAL__sub_I_StructuredBuffers_constructors.hlsl()
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-lib.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-lib.hlsl
index 128fff9b90a22..53abdc71bdd4b 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-lib.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-lib.hlsl
@@ -5,21 +5,45 @@
 
 RWStructuredBuffer<float> RWSB1 : register(u0);
 RWStructuredBuffer<float> RWSB2 : register(u1);
+AppendStructuredBuffer<float> ASB : register(u2);
+ConsumeStructuredBuffer<float> CSB : register(u3);
 
-// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0), float }
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0) }
 
-export void TestIncrementCounter() {
-    RWSB1.IncrementCounter();
+export int TestIncrementCounter() {
+    return RWSB1.IncrementCounter();
 }
 
-// CHECK: define void @_Z20TestIncrementCounterv()
-// CHECK-DXIL: call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i8 1)
+// CHECK: define noundef i32 @_Z20TestIncrementCounterv()
+// CHECK-DXIL: %[[INDEX:.*]] = call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i8 1)
+// CHECK-DXIL: ret i32 %[[INDEX]]
+export int TestDecrementCounter() {
+    return RWSB2.DecrementCounter();
+}
+
+// CHECK: define noundef i32 @_Z20TestDecrementCounterv()
+// CHECK-DXIL: %[[INDEX:.*]] = call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i8 -1)
+// CHECK-DXIL: ret i32 %[[INDEX]]
+
+export void TestAppend(float value) {
+    ASB.Append(value);
+}
+
+// CHECK: define void @_Z10TestAppendf(float noundef %value)
+// CHECK-DXIL: %[[VALUE:.*]] = load float, ptr %value.addr, align 4
+// CHECK-DXIL: %[[INDEX:.*]] = call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i8 1)
+// CHECK-DXIL: %[[RESPTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i32 %[[INDEX]])
+// CHECK-DXIL: store float %[[VALUE]], ptr %[[RESPTR]], align 4
 
-export void TestDecrementCounter() {
-    RWSB2.DecrementCounter();
+export float TestConsume() {
+    return CSB.Consume();
 }
 
-// CHECK: define void @_Z20TestDecrementCounterv()
-// CHECK-DXIL: call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %{{[0-9]+}}, i8 -1)
+// CHECK: define noundef float @_Z11TestConsumev()
+// CHECK-DXIL: %[[INDEX:.*]] = call i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %1, i8 -1)
+// CHECK-DXIL: %[[RESPTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0) %0, i32 %[[INDEX]])
+// CHECK-DXIL: %[[VALUE:.*]] = load float, ptr %[[RESPTR]], align 4
+// CHECK-DXIL: ret float %[[VALUE]]
 
 // CHECK: declare i32 @llvm.dx.bufferUpdateCounter.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0), i8)
+// CHECK: declare ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_f32_1_0t(target("dx.RawBuffer", float, 1, 0), i32)
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
index e895d30b54007..f6959b5cefb7c 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
@@ -6,7 +6,7 @@
 RWStructuredBuffer<float> RWSB1, RWSB2;
 RasterizerOrderedStructuredBuffer<float> ROSB1, ROSB2;
 
-// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0), float }
+// CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", float, 1, 0) }
 
 export void TestIncrementCounter() {
 // CHECK: define void @_Z20TestIncrementCounterv()
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
index 37ac9f32caf31..2af7c3ed3219f 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
@@ -6,14 +6,17 @@ RasterizerOrderedStructuredBuffer<int> Out2;
 
 [numthreads(1,1,1)]
 void main(unsigned GI : SV_GroupIndex) {
+  // CHECK: define void @main()
+
+  // CHECK: %[[INPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_0_0t(target("dx.RawBuffer", i32, 0, 0) %{{.*}}, i32 %{{.*}})
+  // CHECK: %[[LOAD:.*]] = load i32, ptr %[[INPTR]]
+  // CHECK: %[[OUT1PTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_1_0t(target("dx.RawBuffer", i32, 1, 0) %{{.*}}, i32 %{{.*}})
+  // CHECK: store i32 %[[LOAD]], ptr %[[OUT1PTR]]
   Out1[GI] = In[GI];
+
+  // CHECK: %[[INPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_0_0t(target("dx.RawBuffer", i32, 0, 0) %{{.*}}, i32 %{{.*}})
+  // CHECK: %[[LOAD:.*]] = load i32, ptr %[[INPTR]]
+  // CHECK: %[[OUT2PTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_1_1t(target("dx.RawBuffer", i32, 1, 1) %{{.*}}, i32 %{{.*}})
+  // CHECK: store i32 %[[LOAD]], ptr %[[OUT2PTR]]
   Out2[GI] = In[GI];
 }
-
-// Even at -O0 the subscript operators get inlined. The -O0 IR is a bit messy
-// and confusing to follow so the match here is pretty weak.
-
-// CHECK: define void @main()
-// Verify inlining leaves only calls to "llvm." intrinsics
-// CHECK-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
-// CHECK: ret void
diff --git a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
index c206874a4ca94..fda034ce7f203 100644
--- a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
@@ -3,7 +3,8 @@
 using handle_float_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]];
 
 // CHECK: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
-// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }, 0, 0)
+// CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", %struct.MyStruct, 0, 0)
+// CHECK: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
 
 // CHECK: define void @_Z2faU9_Res_u_CTfu17__hlsl_resource_t(target("dx.TypedBuffer", float, 1, 0, 0) %a)
 // CHECK: call void @_Z4foo1U9_Res_u_CTfu17__hlsl_resource_t(target("dx.TypedBuffer", float, 1, 0, 0) %0)
@@ -20,9 +21,9 @@ void fb(handle_float_t a) {
     handle_float_t b = a;
 }
 
-// CHECK: define void @_Z2fcN4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 16 %a)
-// CHECK: call void @_Z4foo2N4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 16 %agg.tmp)
-// CHECK: declare void @_Z4foo2N4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 16)
+// CHECK: define void @_Z2fcN4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 4 %a)
+// CHECK: call void @_Z4foo2N4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 4 %agg.tmp)
+// CHECK: declare void @_Z4foo2N4hlsl8RWBufferIDv4_fEE(ptr noundef byval(%"class.hlsl::RWBuffer") align 4)
 void foo2(RWBuffer<float4> buf);
 
 void fc(RWBuffer<float4> a) {
@@ -38,9 +39,9 @@ struct MyStruct {
   int2 i;
 };
 
-// CHECK: define void @_Z2feN4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 16 %a)
-// CHECK: call void @_Z4foo3N4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 16 %agg.tmp)
-// CHECK: declare void @_Z4foo3N4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 16)
+// CHECK: define void @_Z2feN4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 4 %a)
+// CHECK: call void @_Z4foo3N4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 4 %agg.tmp)
+// CHECK: declare void @_Z4foo3N4hlsl16StructuredBufferI8MyStructEE(ptr noundef byval(%"class.hlsl::StructuredBuffer") align 4)
 void foo3(StructuredBuffer<MyStruct> buf);
 
 void fe(StructuredBuffer<MyStruct> a) {
@@ -50,4 +51,3 @@ void fe(StructuredBuffer<MyStruct> a) {
 void ff(StructuredBuffer<MyStruct> a) {
   StructuredBuffer<MyStruct> b = a;
 }
-
diff --git a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
index 5efecc1489afc..ca01960678175 100644
--- a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
+++ b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
@@ -31,7 +31,7 @@ uint Find(Node SortedTree[MAX], uint key) {
 }
 
 // CHECK: Function Attrs:{{.*}}norecurse
-// CHECK: define noundef i1 @_Z8InitTreeA100_4NodeN4hlsl8RWBufferIDv4_jEEj(ptr noundef byval([100 x %struct.Node]) align 4 %tree, ptr noundef byval(%"class.hlsl::RWBuffer") align 16 %encodedTree, i32 noundef %maxDepth) [[ExtAttr:\#[0-9]+]]
+// CHECK: define noundef i1 @_Z8InitTreeA100_4NodeN4hlsl8RWBufferIDv4_jEEj(ptr noundef byval([100 x %struct.Node]) align 4 %tree, ptr noundef byval(%"class.hlsl::RWBuffer") align 4 %encodedTree, i32 noundef %maxDepth) [[ExtAttr:\#[0-9]+]]
 // CHECK: ret i1
 // Initialize tree with given buffer
 // Imagine the inout works
diff --git a/clang/test/CodeGenHLSL/resource-bindings.hlsl b/clang/test/CodeGenHLSL/resource-bindings.hlsl
index bfec90e1871f8..d4e6308210653 100644
--- a/clang/test/CodeGenHLSL/resource-bindings.hlsl
+++ b/clang/test/CodeGenHLSL/resource-bindings.hlsl
@@ -15,5 +15,5 @@ struct S {
   int i;
 };
 
-// CHECK: %T3S0_h = call target("dx.RawBuffer", %struct.S = type { <4 x float>, i32, [12 x i8] }, 0, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_s_struct.Ss_0_0t(i32 0, i32 3, i32 1, i32 0, i1 false)
+// CHECK: %T3S0_h = call target("dx.RawBuffer", %struct.S, 0, 0) @llvm.dx.handle.fromBinding.tdx.RawBuffer_s_struct.Ss_0_0t(i32 0, i32 3, i32 1, i32 0, i1 false)
 StructuredBuffer<S> T3S0 : register(t3);
diff --git a/clang/test/CodeGenHLSL/semantics/SV_GroupThreadID.hlsl b/clang/test/CodeGenHLSL/semantics/SV_GroupThreadID.hlsl
new file mode 100644
index 0000000000000..3d347b973f39c
--- /dev/null
+++ b/clang/test/CodeGenHLSL/semantics/SV_GroupThreadID.hlsl
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DXIL -DTARGET=dx
+// RUN: %clang_cc1 -triple spirv-linux-vulkan-library -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV -DTARGET=spv
+
+// Make sure SV_GroupThreadID translated into dx.thread.id.in.group for directx target and spv.thread.id.in.group for spirv target.
+
+// CHECK:       define void @foo()
+// CHECK:       %[[#ID:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 0)
+// CHECK-DXIL:       call void @{{.*}}foo{{.*}}(i32 %[[#ID]])
+// CHECK-SPIRV:      call spir_func void @{{.*}}foo{{.*}}(i32 %[[#ID]])
+[shader("compute")]
+[numthreads(8,8,1)]
+void foo(uint Idx : SV_GroupThreadID) {}
+
+// CHECK:       define void @bar()
+// CHECK:       %[[#ID_X:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 0)
+// CHECK:       %[[#ID_X_:]] = insertelement <2 x i32> poison, i32 %[[#ID_X]], i64 0
+// CHECK:       %[[#ID_Y:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 1)
+// CHECK:       %[[#ID_XY:]] = insertelement <2 x i32> %[[#ID_X_]], i32 %[[#ID_Y]], i64 1
+// CHECK-DXIL:  call void @{{.*}}bar{{.*}}(<2 x i32> %[[#ID_XY]])
+// CHECK-SPIRV:  call spir_func void @{{.*}}bar{{.*}}(<2 x i32> %[[#ID_XY]])
+[shader("compute")]
+[numthreads(8,8,1)]
+void bar(uint2 Idx : SV_GroupThreadID) {}
+
+// CHECK:  define void @test()
+// CHECK:  %[[#ID_X:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 0)
+// CHECK:  %[[#ID_X_:]] = insertelement <3 x i32> poison, i32 %[[#ID_X]], i64 0
+// CHECK:  %[[#ID_Y:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 1)
+// CHECK:  %[[#ID_XY:]] = insertelement <3 x i32> %[[#ID_X_]], i32 %[[#ID_Y]], i64 1
+// CHECK:  %[[#ID_Z:]] = call i32 @llvm.[[TARGET]].thread.id.in.group(i32 2)
+// CHECK:  %[[#ID_XYZ:]] = insertelement <3 x i32> %[[#ID_XY]], i32 %[[#ID_Z]], i64 2
+// CHECK-DXIL:   call void @{{.*}}test{{.*}}(<3 x i32> %[[#ID_XYZ]])
+// CHECK-SPIRV:  call spir_func void @{{.*}}test{{.*}}(<3 x i32> %[[#ID_XYZ]])
+[shader("compute")]
+[numthreads(8,8,1)]
+void test(uint3 Idx : SV_GroupThreadID) {}
diff --git a/clang/test/CodeGenObjC/arc-ternary-op.m b/clang/test/CodeGenObjC/arc-ternary-op.m
index d633851ac7617..4a3c00c9807a9 100644
--- a/clang/test/CodeGenObjC/arc-ternary-op.m
+++ b/clang/test/CodeGenObjC/arc-ternary-op.m
@@ -67,7 +67,7 @@ void test1(int cond) {
   // CHECK:      [[T0:%.*]] = load ptr, ptr [[ARG]]
   // CHECK-NEXT: store ptr [[T0]], ptr [[TEMP1]]
   // CHECK-NEXT: br label
-  // CHECK:      [[W:%.*]] = phi ptr [ [[T0]], {{%.*}} ], [ undef, {{%.*}} ]
+  // CHECK:      [[W:%.*]] = phi ptr [ [[T0]], {{%.*}} ], [ poison, {{%.*}} ]
   // CHECK-NEXT: call void @test1_sink(ptr noundef [[T1]])
   // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[ARG]], null
   // CHECK-NEXT: br i1 [[T0]],
diff --git a/clang/test/CodeGenObjCXX/arc.mm b/clang/test/CodeGenObjCXX/arc.mm
index 7a6de17e2baa0..7883378b33828 100644
--- a/clang/test/CodeGenObjCXX/arc.mm
+++ b/clang/test/CodeGenObjCXX/arc.mm
@@ -75,7 +75,7 @@ void test34(int cond) {
   // CHECK:      [[T0:%.*]] = load ptr, ptr [[ARG]]
   // CHECK-NEXT: store ptr [[T0]], ptr [[TEMP1]]
   // CHECK-NEXT: br label
-  // CHECK:      [[W0:%.*]] = phi ptr [ [[T0]], {{%.*}} ], [ undef, {{%.*}} ]
+  // CHECK:      [[W0:%.*]] = phi ptr [ [[T0]], {{%.*}} ], [ poison, {{%.*}} ]
   // CHECK:      call void @_Z11test34_sinkPU15__autoreleasingP11objc_object(ptr noundef [[T1]])
   // CHECK-NEXT: [[T0:%.*]] = icmp eq ptr [[ARG]], null
   // CHECK-NEXT: br i1 [[T0]],
diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
index 62b5661da9dbd..5599f4dd50f04 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
@@ -427,7 +427,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent norecurse nounwind
 // GFX900-LABEL: define {{[^@]+}}@test
-// GFX900-SAME: (ptr addrspace(1) noundef align 1 [[A:%.*]], i8 noundef [[B:%.*]], ptr addrspace(1) noundef align 8 [[C:%.*]], i64 noundef [[D:%.*]]) #[[ATTR2:[0-9]+]] !kernel_arg_addr_space [[META9:![0-9]+]] !kernel_arg_access_qual [[META10:![0-9]+]] !kernel_arg_type [[META11:![0-9]+]] !kernel_arg_base_type [[META11]] !kernel_arg_type_qual [[META12:![0-9]+]] {
+// GFX900-SAME: (ptr addrspace(1) noundef align 1 [[A:%.*]], i8 noundef [[B:%.*]], ptr addrspace(1) noundef align 8 [[C:%.*]], i64 noundef [[D:%.*]]) #[[ATTR2:[0-9]+]] !kernel_arg_addr_space [[META10:![0-9]+]] !kernel_arg_access_qual [[META11:![0-9]+]] !kernel_arg_type [[META12:![0-9]+]] !kernel_arg_base_type [[META12]] !kernel_arg_type_qual [[META13:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[A_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
 // GFX900-NEXT:    [[B_ADDR:%.*]] = alloca i8, align 1, addrspace(5)
@@ -463,17 +463,17 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK20_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[BLOCK20]] to ptr
 // GFX900-NEXT:    [[BLOCK21_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[BLOCK21]] to ptr
 // GFX900-NEXT:    [[TMP27_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VARTMP27]] to ptr
-// GFX900-NEXT:    store ptr addrspace(1) [[A]], ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    store i8 [[B]], ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA13:![0-9]+]]
+// GFX900-NEXT:    store ptr addrspace(1) [[A]], ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// GFX900-NEXT:    store i8 [[B]], ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16:![0-9]+]]
 // GFX900-NEXT:    store ptr addrspace(1) [[C]], ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store i64 [[D]], ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8:[0-9]+]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
-// GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14:![0-9]+]]
+// GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17:![0-9]+]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
-// GFX900-NEXT:    [[TMP0:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA16:![0-9]+]]
-// GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
-// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT18:![0-9]+]]
+// GFX900-NEXT:    [[TMP0:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19:![0-9]+]]
+// GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
+// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
 // GFX900-NEXT:    [[BLOCK_SIZE:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i32 25, ptr [[BLOCK_SIZE]], align 8
 // GFX900-NEXT:    [[BLOCK_ALIGN:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 1
@@ -481,15 +481,15 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_INVOKE:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 2
 // GFX900-NEXT:    store ptr @__test_block_invoke, ptr [[BLOCK_INVOKE]], align 8
 // GFX900-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    store ptr addrspace(1) [[TMP2]], ptr [[BLOCK_CAPTURED]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA14]]
+// GFX900-NEXT:    store ptr addrspace(1) [[TMP2]], ptr [[BLOCK_CAPTURED]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[BLOCK_CAPTURED1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 4
-// GFX900-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA13]]
-// GFX900-NEXT:    store i8 [[TMP3]], ptr [[BLOCK_CAPTURED1]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16]]
+// GFX900-NEXT:    store i8 [[TMP3]], ptr [[BLOCK_CAPTURED1]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[TMP4:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP0]], i32 [[TMP1]], ptr [[TMP_ASCAST]], ptr @__test_block_invoke_kernel, ptr [[BLOCK_ASCAST]])
-// GFX900-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP6:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
-// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP2_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT18]]
+// GFX900-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
+// GFX900-NEXT:    [[TMP6:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
+// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP2_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[BLOCK_SIZE4:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i32 41, ptr [[BLOCK_SIZE4]], align 8
 // GFX900-NEXT:    [[BLOCK_ALIGN5:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 1
@@ -497,11 +497,11 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_INVOKE6:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 2
 // GFX900-NEXT:    store ptr @__test_block_invoke_2, ptr [[BLOCK_INVOKE6]], align 8
 // GFX900-NEXT:    [[BLOCK_CAPTURED7:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP7:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    store ptr addrspace(1) [[TMP7]], ptr [[BLOCK_CAPTURED7]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP7:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA14]]
+// GFX900-NEXT:    store ptr addrspace(1) [[TMP7]], ptr [[BLOCK_CAPTURED7]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[BLOCK_CAPTURED8:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 6
-// GFX900-NEXT:    [[TMP8:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA13]]
-// GFX900-NEXT:    store i8 [[TMP8]], ptr [[BLOCK_CAPTURED8]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP8:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16]]
+// GFX900-NEXT:    store i8 [[TMP8]], ptr [[BLOCK_CAPTURED8]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURED9:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP9:%.*]] = load ptr addrspace(1), ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store ptr addrspace(1) [[TMP9]], ptr [[BLOCK_CAPTURED9]], align 8, !tbaa [[TBAA7]]
@@ -509,9 +509,9 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[TMP10:%.*]] = load i64, ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    store i64 [[TMP10]], ptr [[BLOCK_CAPTURED10]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    [[TMP11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP5]], i32 [[TMP6]], ptr [[TMP2_ASCAST]], ptr @__test_block_invoke_2_kernel, ptr [[BLOCK3_ASCAST]])
-// GFX900-NEXT:    [[TMP12:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP13:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
-// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP11_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT18]]
+// GFX900-NEXT:    [[TMP12:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
+// GFX900-NEXT:    [[TMP13:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
+// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP11_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[BLOCK_SIZE13:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i32 41, ptr [[BLOCK_SIZE13]], align 8
 // GFX900-NEXT:    [[BLOCK_ALIGN14:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 1
@@ -519,11 +519,11 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_INVOKE15:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 2
 // GFX900-NEXT:    store ptr @__test_block_invoke_3, ptr [[BLOCK_INVOKE15]], align 8
 // GFX900-NEXT:    [[BLOCK_CAPTURED16:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP14:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    store ptr addrspace(1) [[TMP14]], ptr [[BLOCK_CAPTURED16]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP14:%.*]] = load ptr addrspace(1), ptr [[A_ADDR_ASCAST]], align 8, !tbaa [[TBAA14]]
+// GFX900-NEXT:    store ptr addrspace(1) [[TMP14]], ptr [[BLOCK_CAPTURED16]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[BLOCK_CAPTURED17:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 6
-// GFX900-NEXT:    [[TMP15:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA13]]
-// GFX900-NEXT:    store i8 [[TMP15]], ptr [[BLOCK_CAPTURED17]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP15:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16]]
+// GFX900-NEXT:    store i8 [[TMP15]], ptr [[BLOCK_CAPTURED17]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURED18:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP16:%.*]] = load ptr addrspace(1), ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store ptr addrspace(1) [[TMP16]], ptr [[BLOCK_CAPTURED18]], align 8, !tbaa [[TBAA7]]
@@ -548,11 +548,11 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_CAPTURED26:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP21:%.*]] = load ptr addrspace(1), ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store ptr addrspace(1) [[TMP21]], ptr [[BLOCK_CAPTURED26]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    store ptr [[BLOCK21_ASCAST]], ptr [[BLOCK20_ASCAST]], align 8, !tbaa [[TBAA13]]
-// GFX900-NEXT:    [[TMP22:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP23:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
-// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP27_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT18]]
-// GFX900-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[BLOCK20_ASCAST]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    store ptr [[BLOCK21_ASCAST]], ptr [[BLOCK20_ASCAST]], align 8, !tbaa [[TBAA16]]
+// GFX900-NEXT:    [[TMP22:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
+// GFX900-NEXT:    [[TMP23:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
+// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP27_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
+// GFX900-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[BLOCK20_ASCAST]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr [[TMP27_ASCAST]], ptr @__test_block_invoke_4_kernel, ptr [[BLOCK21_ASCAST]])
 // GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR8]]
 // GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
@@ -569,17 +569,17 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBLOCK_DESCRIPTOR_ADDR]] to ptr
 // GFX900-NEXT:    store ptr [[DOTBLOCK_DESCRIPTOR]], ptr [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST]], align 8
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 4
-// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP1]], i64 0
-// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA13]]
+// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA16]]
 // GFX900-NEXT:    ret void
 //
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR6:[0-9]+]] !kernel_arg_addr_space [[META19:![0-9]+]] !kernel_arg_access_qual [[META20:![0-9]+]] !kernel_arg_type [[META21:![0-9]+]] !kernel_arg_base_type [[META21]] !kernel_arg_type_qual [[META22:![0-9]+]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR6:[0-9]+]] !kernel_arg_addr_space [[META22:![0-9]+]] !kernel_arg_access_qual [[META23:![0-9]+]] !kernel_arg_type [[META24:![0-9]+]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -596,11 +596,11 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBLOCK_DESCRIPTOR_ADDR]] to ptr
 // GFX900-NEXT:    store ptr [[DOTBLOCK_DESCRIPTOR]], ptr [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST]], align 8
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP1]], i64 0
-// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA13]]
+// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // GFX900-NEXT:    [[TMP2:%.*]] = load i64, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 4
@@ -612,7 +612,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_2_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META19]] !kernel_arg_access_qual [[META20]] !kernel_arg_type [[META21]] !kernel_arg_base_type [[META21]] !kernel_arg_type_qual [[META22]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -630,28 +630,28 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBLOCK_DESCRIPTOR_ADDR]] to ptr
 // GFX900-NEXT:    [[LP_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[LP_ADDR]] to ptr
 // GFX900-NEXT:    store ptr [[DOTBLOCK_DESCRIPTOR]], ptr [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST]], align 8
-// GFX900-NEXT:    store ptr addrspace(3) [[LP]], ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA7]]
+// GFX900-NEXT:    store ptr addrspace(3) [[LP]], ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA26:![0-9]+]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA13]]
+// GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 3
-// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA14]]
 // GFX900-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP1]], i64 0
-// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA13]]
+// GFX900-NEXT:    store i8 [[TMP0]], ptr addrspace(1) [[ARRAYIDX]], align 1, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
 // GFX900-NEXT:    [[TMP2:%.*]] = load i64, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP3:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[TMP3]], i64 0
 // GFX900-NEXT:    store i64 [[TMP2]], ptr addrspace(1) [[ARRAYIDX4]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    [[TMP4:%.*]] = load ptr addrspace(3), ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA7]]
+// GFX900-NEXT:    [[TMP4:%.*]] = load ptr addrspace(3), ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA26]]
 // GFX900-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TMP4]], i64 0
-// GFX900-NEXT:    store i32 1, ptr addrspace(3) [[ARRAYIDX5]], align 4, !tbaa [[TBAA14]]
+// GFX900-NEXT:    store i32 1, ptr addrspace(3) [[ARRAYIDX5]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    ret void
 //
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_3_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META23:![0-9]+]] !kernel_arg_access_qual [[META24:![0-9]+]] !kernel_arg_type [[META25:![0-9]+]] !kernel_arg_base_type [[META25]] !kernel_arg_type_qual [[META26:![0-9]+]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META28:![0-9]+]] !kernel_arg_access_qual [[META29:![0-9]+]] !kernel_arg_type [[META30:![0-9]+]] !kernel_arg_base_type [[META30]] !kernel_arg_type_qual [[META31:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP2:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP2]], align 8
@@ -677,7 +677,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_4_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META19]] !kernel_arg_access_qual [[META20]] !kernel_arg_type [[META21]] !kernel_arg_base_type [[META21]] !kernel_arg_type_qual [[META22]] {
+// GFX900-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, i64, ptr addrspace(1) }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -688,7 +688,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent norecurse nounwind
 // GFX900-LABEL: define {{[^@]+}}@test_target_features_kernel
-// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META27:![0-9]+]] !kernel_arg_access_qual [[META20]] !kernel_arg_type [[META28:![0-9]+]] !kernel_arg_base_type [[META28]] !kernel_arg_type_qual [[META22]] {
+// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META32:![0-9]+]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META33:![0-9]+]] !kernel_arg_base_type [[META33]] !kernel_arg_type_qual [[META25]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[I_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
 // GFX900-NEXT:    [[DEFAULT_QUEUE:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -700,15 +700,15 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[FLAGS_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[FLAGS]] to ptr
 // GFX900-NEXT:    [[NDRANGE_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[NDRANGE]] to ptr
 // GFX900-NEXT:    [[TMP_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP]] to ptr
-// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
+// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA34:![0-9]+]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
-// GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
+// GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
 // GFX900-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.s.memtime()
-// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP2:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA14]]
-// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT18]]
+// GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
+// GFX900-NEXT:    [[TMP2:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
+// GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr [[TMP_ASCAST]], ptr @__test_target_features_kernel_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
 // GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
 // GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
@@ -729,7 +729,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_target_features_kernel_block_invoke_kernel
-// GFX900-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META19]] !kernel_arg_access_qual [[META20]] !kernel_arg_type [[META21]] !kernel_arg_base_type [[META21]] !kernel_arg_type_qual [[META22]] {
+// GFX900-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca { i32, i32, ptr }, align 8, addrspace(5)
 // GFX900-NEXT:    store { i32, i32, ptr } [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -785,27 +785,34 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
 // GFX900: [[META6]] = !{!"Simple C/C++ TBAA"}
 // GFX900: [[TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0}
-// GFX900: [[META8]] = !{!"any pointer", [[META5]], i64 0}
-// GFX900: [[META9]] = !{i32 1, i32 0, i32 1, i32 0}
-// GFX900: [[META10]] = !{!"none", !"none", !"none", !"none"}
-// GFX900: [[META11]] = !{!"char*", !"char", !"long*", !"long"}
-// GFX900: [[META12]] = !{!"", !"", !"", !""}
-// GFX900: [[TBAA13]] = !{[[META5]], [[META5]], i64 0}
+// GFX900: [[META8]] = !{!"p1 long", [[META9:![0-9]+]], i64 0}
+// GFX900: [[META9]] = !{!"any pointer", [[META5]], i64 0}
+// GFX900: [[META10]] = !{i32 1, i32 0, i32 1, i32 0}
+// GFX900: [[META11]] = !{!"none", !"none", !"none", !"none"}
+// GFX900: [[META12]] = !{!"char*", !"char", !"long*", !"long"}
+// GFX900: [[META13]] = !{!"", !"", !"", !""}
 // GFX900: [[TBAA14]] = !{[[META15:![0-9]+]], [[META15]], i64 0}
-// GFX900: [[META15]] = !{!"int", [[META5]], i64 0}
-// GFX900: [[TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
-// GFX900: [[META17]] = !{!"queue_t", [[META5]], i64 0}
-// GFX900: [[TBAA_STRUCT18]] = !{i64 0, i64 4, [[TBAA14]]}
-// GFX900: [[META19]] = !{i32 0}
-// GFX900: [[META20]] = !{!"none"}
-// GFX900: [[META21]] = !{!"__block_literal"}
-// GFX900: [[META22]] = !{!""}
-// GFX900: [[META23]] = !{i32 0, i32 3}
-// GFX900: [[META24]] = !{!"none", !"none"}
-// GFX900: [[META25]] = !{!"__block_literal", !"void*"}
-// GFX900: [[META26]] = !{!"", !""}
-// GFX900: [[META27]] = !{i32 1}
-// GFX900: [[META28]] = !{!"int*"}
+// GFX900: [[META15]] = !{!"p1 omnipotent char", [[META9]], i64 0}
+// GFX900: [[TBAA16]] = !{[[META5]], [[META5]], i64 0}
+// GFX900: [[TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0}
+// GFX900: [[META18]] = !{!"int", [[META5]], i64 0}
+// GFX900: [[TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
+// GFX900: [[META20]] = !{!"queue_t", [[META5]], i64 0}
+// GFX900: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[TBAA17]]}
+// GFX900: [[META22]] = !{i32 0}
+// GFX900: [[META23]] = !{!"none"}
+// GFX900: [[META24]] = !{!"__block_literal"}
+// GFX900: [[META25]] = !{!""}
+// GFX900: [[TBAA26]] = !{[[META27:![0-9]+]], [[META27]], i64 0}
+// GFX900: [[META27]] = !{!"p1 void", [[META9]], i64 0}
+// GFX900: [[META28]] = !{i32 0, i32 3}
+// GFX900: [[META29]] = !{!"none", !"none"}
+// GFX900: [[META30]] = !{!"__block_literal", !"void*"}
+// GFX900: [[META31]] = !{!"", !""}
+// GFX900: [[META32]] = !{i32 1}
+// GFX900: [[META33]] = !{!"int*"}
+// GFX900: [[TBAA34]] = !{[[META35:![0-9]+]], [[META35]], i64 0}
+// GFX900: [[META35]] = !{!"p1 int", [[META9]], i64 0}
 //.
 //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 // CHECK: {{.*}}
diff --git a/clang/test/CodeGenOpenCL/amdgpu-nullptr.cl b/clang/test/CodeGenOpenCL/amdgpu-nullptr.cl
index 53cca49e87ef4..a0c106bca83c9 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-nullptr.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-nullptr.cl
@@ -513,9 +513,9 @@ typedef struct {
 
 // CHECK-LABEL: test_memset_private
 // CHECK: call void @llvm.memset.p5.i64(ptr addrspace(5) noundef align 8 {{.*}}, i8 0, i64 32, i1 false)
-// CHECK: [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(5) %ptr, i32 32
+// CHECK: [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) %ptr, i32 32
 // CHECK: store ptr addrspace(5) addrspacecast (ptr null to ptr addrspace(5)), ptr addrspace(5) [[GEP]]
-// CHECK: [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) {{.*}}, i32 36
+// CHECK: [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) {{.*}}, i32 36
 // CHECK: store i32 0, ptr addrspace(5) [[GEP1]], align 4
 void test_memset_private(private StructTy3 *ptr) {
   StructTy3 S3 = {0, 0, 0, 0, 0};
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
index 39fa46d5845f4..91e04430d4973 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950-read-tr.cl
@@ -4,6 +4,8 @@
 typedef int    v2i   __attribute__((ext_vector_type(2)));
 typedef int    v3i   __attribute__((ext_vector_type(3)));
 typedef short  v4s   __attribute__((ext_vector_type(4)));
+typedef half   v4h   __attribute__((ext_vector_type(4)));
+typedef __bf16 v4y   __attribute__((ext_vector_type(4)));
 
 // GFX950-LABEL: define dso_local <2 x i32> @test_amdgcn_ds_read_b64_tr_b4_v2i32(
 // GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
@@ -48,3 +50,24 @@ v4s test_amdgcn_ds_read_b64_tr_b16_v2i16(local v4s* inptr)
 {
   return __builtin_amdgcn_ds_read_tr16_b64_v4i16(inptr);
 }
+
+// GFX950-LABEL: define dso_local <4 x half> @test_amdgcn_ds_read_b64_tr_b16_v2f16(
+// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-NEXT:  entry:
+// GFX950-NEXT:    [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3) [[INPTR]])
+// GFX950-NEXT:    ret <4 x half> [[TMP0]]
+//
+v4h test_amdgcn_ds_read_b64_tr_b16_v2f16(local v4h* inptr)
+{
+  return __builtin_amdgcn_ds_read_tr16_b64_v4f16(inptr);
+}
+
+// GFX950-LABEL: define dso_local <4 x bfloat> @test_amdgcn_ds_read_b64_tr_b16_v2bf16(
+// GFX950-SAME: ptr addrspace(3) nocapture noundef readonly [[INPTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// GFX950-NEXT:  entry:
+// GFX950-NEXT:    [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16(ptr addrspace(3) [[INPTR]])
+// GFX950-NEXT:    ret <4 x bfloat> [[TMP0]]
+v4y test_amdgcn_ds_read_b64_tr_b16_v2bf16(local v4y* inptr)
+{
+  return __builtin_amdgcn_ds_read_tr16_b64_v4bf16(inptr);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950.cl
index f1259ef678f1e..8251d6c213e3d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx950.cl
@@ -1673,7 +1673,7 @@ void test_cvt_scalef32_sr_fp8_f32(global unsigned *out, float src, uint seed, fl
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(5) [[B_ADDR]], align 4
 // CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(5) [[C_ADDR]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.bitop3.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]], i8 1)
+// CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.bitop3.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]], i32 1)
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[OUT_ADDR]], align 8
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4
 // CHECK-NEXT:    ret void
@@ -1696,7 +1696,7 @@ void test_bitop3_b32(global uint* out, uint a, uint b, uint c)
 // CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr addrspace(5) [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr addrspace(5) [[B_ADDR]], align 2
 // CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(5) [[C_ADDR]], align 2
-// CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.amdgcn.bitop3.i16(i16 [[TMP0]], i16 [[TMP1]], i16 [[TMP2]], i8 1)
+// CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.amdgcn.bitop3.i16(i16 [[TMP0]], i16 [[TMP1]], i16 [[TMP2]], i32 1)
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[OUT_ADDR]], align 8
 // CHECK-NEXT:    store i16 [[TMP3]], ptr addrspace(1) [[TMP4]], align 2
 // CHECK-NEXT:    ret void
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index c22a43146a8c8..ded5f6b5ac4fd 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -620,11 +620,11 @@ void test_get_local_id(int d, global int *out)
 
 // CHECK-LABEL: @test_get_workgroup_size(
 // CHECK: {{.*}}call align 8 dereferenceable(256){{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-// CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 12
+// CHECK: getelementptr inbounds nuw i8, ptr addrspace(4) %{{.*}}, i64 12
 // CHECK: load i16, ptr addrspace(4) %{{.*}}, align 4, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef
-// CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 14
+// CHECK: getelementptr inbounds nuw i8, ptr addrspace(4) %{{.*}}, i64 14
 // CHECK: load i16, ptr addrspace(4) %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef
-// CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 16
+// CHECK: getelementptr inbounds nuw i8, ptr addrspace(4) %{{.*}}, i64 16
 // CHECK: load i16, ptr addrspace(4) %{{.*}}, align 8, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef
 void test_get_workgroup_size(int d, global int *out)
 {
@@ -638,7 +638,7 @@ void test_get_workgroup_size(int d, global int *out)
 
 // CHECK-LABEL: @test_get_grid_size(
 // CHECK: {{.*}}call align 4 dereferenceable(64){{.*}} ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-// CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 %{{.+}}
+// CHECK: getelementptr inbounds nuw i8, ptr addrspace(4) %{{.*}}, i64 %{{.+}}
 // CHECK: load i32, ptr addrspace(4) %{{.*}}, align 4, !range [[$GRID_RANGE:![0-9]+]], !invariant.load
 void test_get_grid_size(int d, global int *out)
 {
diff --git a/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp b/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
index 86f21ee556ce8..697e50e199a04 100644
--- a/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
+++ b/clang/test/CodeGenOpenCLCXX/array-type-infinite-loop.clcpp
@@ -2,11 +2,11 @@
 //RUN: %clang_cc1 %s -triple spir -emit-llvm -O1 -o - | FileCheck %s
 
 // CHECK-LABEL: define dso_local spir_kernel void @test(
-// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 8 [[IN:%.*]], ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META3:![0-9]+]] !kernel_arg_access_qual [[META4:![0-9]+]] !kernel_arg_type [[META5:![0-9]+]] !kernel_arg_base_type [[META5]] !kernel_arg_type_qual [[META6:![0-9]+]] {
+// CHECK-SAME: ptr addrspace(1) nocapture noundef readonly align 8 [[IN:%.*]], ptr addrspace(1) nocapture noundef writeonly align 8 initializes((0, 8)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META4:![0-9]+]] !kernel_arg_access_qual [[META5:![0-9]+]] !kernel_arg_type [[META6:![0-9]+]] !kernel_arg_base_type [[META6]] !kernel_arg_type_qual [[META7:![0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i32 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr addrspace(1) [[ARRAYIDX1]], align 8, !tbaa [[TBAA7:![0-9]+]]
-// CHECK-NEXT:    store i64 [[TMP0]], ptr addrspace(1) [[OUT]], align 8, !tbaa [[TBAA7]]
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[IN]], i32 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr addrspace(1) [[ARRAYIDX1]], align 8, !tbaa [[TBAA8:![0-9]+]]
+// CHECK-NEXT:    store i64 [[TMP0]], ptr addrspace(1) [[OUT]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    ret void
 //
 __kernel void test(__global long *In, __global long *Out) {
@@ -14,12 +14,12 @@ __kernel void test(__global long *In, __global long *Out) {
    *Out = m[1];
 }
 //.
-// CHECK: [[META3]] = !{i32 1, i32 1}
-// CHECK: [[META4]] = !{!"none", !"none"}
-// CHECK: [[META5]] = !{!"long*", !"long*"}
-// CHECK: [[META6]] = !{!"", !""}
-// CHECK: [[TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0}
-// CHECK: [[META8]] = !{!"long", [[META9:![0-9]+]], i64 0}
-// CHECK: [[META9]] = !{!"omnipotent char", [[META10:![0-9]+]], i64 0}
-// CHECK: [[META10]] = !{!"Simple C++ TBAA"}
+// CHECK: [[META4]] = !{i32 1, i32 1}
+// CHECK: [[META5]] = !{!"none", !"none"}
+// CHECK: [[META6]] = !{!"long*", !"long*"}
+// CHECK: [[META7]] = !{!"", !""}
+// CHECK: [[TBAA8]] = !{[[META9:![0-9]+]], [[META9]], i64 0}
+// CHECK: [[META9]] = !{!"long", [[META10:![0-9]+]], i64 0}
+// CHECK: [[META10]] = !{!"omnipotent char", [[META11:![0-9]+]], i64 0}
+// CHECK: [[META11]] = !{!"Simple C++ TBAA"}
 //.
diff --git a/clang/test/Driver/Inputs/config-l.cfg b/clang/test/Driver/Inputs/config-l.cfg
new file mode 100644
index 0000000000000..e76efafd3039a
--- /dev/null
+++ b/clang/test/Driver/Inputs/config-l.cfg
@@ -0,0 +1,3 @@
+-Wall $-lm
+-Wl,--as-needed $-Wl,-Bstatic
+$-lhappy $-Wl,-Bdynamic
diff --git a/clang/test/Driver/aarch64-fixed-register-global.c b/clang/test/Driver/aarch64-fixed-register-global.c
deleted file mode 100644
index 7b1fb118fcdf7..0000000000000
--- a/clang/test/Driver/aarch64-fixed-register-global.c
+++ /dev/null
@@ -1,12 +0,0 @@
-// Check that -ffixed register handled for globals.
-// Regression test for #76426
-// RUN: %clang --target=aarch64-none-gnu -ffixed-x15 -### %s 2>&1 | FileCheck %s
-// CHECK-NOT: fatal error: error in backend: Invalid register name "x15".
-register int i1 __asm__("x15");
-
-int foo() {
-  return i1;
-}
-int main() {
-  return foo();
-}
diff --git a/clang/test/Driver/aarch64-fujitsu-monaka.c b/clang/test/Driver/aarch64-fujitsu-monaka.c
new file mode 100644
index 0000000000000..df96b36bace68
--- /dev/null
+++ b/clang/test/Driver/aarch64-fujitsu-monaka.c
@@ -0,0 +1,13 @@
+// RUN: %clang --target=aarch64 -mcpu=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=fujitsu-monaka %s
+// RUN: %clang --target=aarch64 -mlittle-endian -mcpu=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=fujitsu-monaka %s
+// RUN: %clang --target=aarch64 -mtune=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=fujitsu-monaka-TUNE %s
+// RUN: %clang --target=aarch64 -mlittle-endian -mtune=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=fujitsu-monaka-TUNE %s
+// fujitsu-monaka: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "fujitsu-monaka"
+// fujitsu-monaka-TUNE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic"
+
+// RUN: %clang --target=arm64 -mcpu=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-fujitsu-monaka %s
+// RUN: %clang --target=arm64 -mlittle-endian -mcpu=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-fujitsu-monaka %s
+// RUN: %clang --target=arm64 -mtune=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-fujitsu-monaka-TUNE %s
+// RUN: %clang --target=arm64 -mlittle-endian -mtune=fujitsu-monaka -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-fujitsu-monaka-TUNE %s
+// ARM64-fujitsu-monaka: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "fujitsu-monaka"
+// ARM64-fujitsu-monaka-TUNE: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "generic"
diff --git a/clang/test/Driver/aarch64-ptrauth.c b/clang/test/Driver/aarch64-ptrauth.c
index 19eaa73b14d9e..d036189e61498 100644
--- a/clang/test/Driver/aarch64-ptrauth.c
+++ b/clang/test/Driver/aarch64-ptrauth.c
@@ -15,8 +15,9 @@
 // RUN:   -fno-ptrauth-indirect-gotos -fptrauth-indirect-gotos \
 // RUN:   -fno-ptrauth-init-fini -fptrauth-init-fini \
 // RUN:   -fno-ptrauth-init-fini-address-discrimination -fptrauth-init-fini-address-discrimination \
+// RUN:   -fno-aarch64-jump-table-hardening -faarch64-jump-table-hardening \
 // RUN:   %s 2>&1 | FileCheck %s --check-prefix=ALL
-// ALL: "-cc1"{{.*}} "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-type-info-vtable-pointer-discrimination" "-fptrauth-indirect-gotos" "-fptrauth-init-fini" "-fptrauth-init-fini-address-discrimination"
+// ALL: "-cc1"{{.*}} "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-type-info-vtable-pointer-discrimination" "-fptrauth-indirect-gotos" "-fptrauth-init-fini" "-fptrauth-init-fini-address-discrimination" "-faarch64-jump-table-hardening"
 
 // RUN: %clang -### -c --target=aarch64-linux -mabi=pauthtest %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI1
 // RUN: %clang -### -c --target=aarch64-linux-pauthtest %s 2>&1 | FileCheck %s --check-prefix=PAUTHABI1
@@ -38,7 +39,7 @@
 // RUN: not %clang -### -c --target=x86_64 -fptrauth-intrinsics -fptrauth-calls -fptrauth-returns -fptrauth-auth-traps \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination -fptrauth-vtable-pointer-type-discrimination \
 // RUN:   -fptrauth-type-info-vtable-pointer-discrimination -fptrauth-indirect-gotos -fptrauth-init-fini \
-// RUN:   -fptrauth-init-fini-address-discrimination %s 2>&1 | FileCheck %s --check-prefix=ERR1
+// RUN:   -fptrauth-init-fini-address-discrimination -faarch64-jump-table-hardening %s 2>&1 | FileCheck %s --check-prefix=ERR1
 // ERR1:      error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
 // ERR1-NEXT: error: unsupported option '-fptrauth-calls' for target '{{.*}}'
 // ERR1-NEXT: error: unsupported option '-fptrauth-returns' for target '{{.*}}'
@@ -49,6 +50,7 @@
 // ERR1-NEXT: error: unsupported option '-fptrauth-indirect-gotos' for target '{{.*}}'
 // ERR1-NEXT: error: unsupported option '-fptrauth-init-fini' for target '{{.*}}'
 // ERR1-NEXT: error: unsupported option '-fptrauth-init-fini-address-discrimination' for target '{{.*}}'
+// ERR1-NEXT: error: unsupported option '-faarch64-jump-table-hardening' for target '{{.*}}'
 
 //// Only support PAuth ABI for Linux as for now.
 // RUN: not %clang -o /dev/null -c --target=aarch64-unknown -mabi=pauthtest %s 2>&1 | FileCheck %s --check-prefix=ERR2
diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c
index 2f17a08140436..0ee8e05f378ef 100644
--- a/clang/test/Driver/arm-cortex-cpus-2.c
+++ b/clang/test/Driver/arm-cortex-cpus-2.c
@@ -544,6 +544,17 @@
 // CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+sha2"
 // CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+aes"
 
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a510 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A510 %s
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a510 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A510-MFPU %s
+// CHECK-CORTEX-A510: "-cc1"{{.*}} "-triple" "armv9a-{{.*}} "-target-cpu" "cortex-a510"
+// CHECK-CORTEX-A510-NOT: "-target-feature" "{{[+-]}}sm4"
+// CHECK-CORTEX-A510-NOT: "-target-feature" "{{[+-]}}sha3"
+// CHECK-CORTEX-A510: "-target-feature" "-aes"
+// CHECK-CORTEX-A510-SAME: {{$}}
+// CHECK-CORTEX-A510-MFPU: "-cc1"{{.*}} "-target-feature" "+fp-armv8"
+// CHECK-CORTEX-A510-MFPU: "-target-feature" "+sha2"
+// CHECK-CORTEX-A510-MFPU: "-target-feature" "+aes"
+
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710 %s
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710-MFPU %s
 // CHECK-CORTEX-A710: "-cc1"{{.*}} "-triple" "armv9a-{{.*}} "-target-cpu" "cortex-a710"
diff --git a/clang/test/Driver/baremetal-multilib.yaml b/clang/test/Driver/baremetal-multilib.yaml
index b6bfd0ed3a94c..853a4e9e36e43 100644
--- a/clang/test/Driver/baremetal-multilib.yaml
+++ b/clang/test/Driver/baremetal-multilib.yaml
@@ -10,7 +10,8 @@
 # CHECK-SAME: "-x" "c++" "{{.*}}baremetal-multilib.yaml"
 # CHECK-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 # CHECK-SAME: "-L[[SYSROOT]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/lib"
-# CHECK-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a"
+# CHECK-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+# CHECK-SAME: "-lc"
 # CHECK-SAME: "-o" "{{.*}}.tmp.out"
 
 # RUN: %clang --multi-lib-config=%s -no-canonical-prefixes -x c++ %s -### -o %t.out 2>&1 \
diff --git a/clang/test/Driver/baremetal-sysroot.cpp b/clang/test/Driver/baremetal-sysroot.cpp
index 18654be33b87c..5cbb7ac69a7ac 100644
--- a/clang/test/Driver/baremetal-sysroot.cpp
+++ b/clang/test/Driver/baremetal-sysroot.cpp
@@ -18,5 +18,6 @@
 // CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal-sysroot.cpp"
 // CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-V6M-C-SAME: "-L{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}lib"
-// CHECK-V6M-C-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-C-SAME: "-lc"
 // CHECK-V6M-C-SAME: "-o" "{{.*}}.o"
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index f09d7361e6c13..6d6255ef5970f 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -19,7 +19,9 @@
 // CHECK-V6M-C-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-C-SAME: "-T" "semihosted.lds" "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-V6M-C-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
-// CHECK-V6M-C-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "--target2=rel" "-o" "{{.*}}.tmp.out"
+// CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-C-SAME: "-lc"
+// CHECK-V6M-C-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
 
 // RUN: %clang %s -### --target=armv6m-none-eabi -nostdlibinc -nobuiltininc 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-LIBINC %s
@@ -40,7 +42,9 @@
 // CHECK-V6M-TREE-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
 // CHECK-V6M-TREE-SAME: "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi{{[/\\]+}}crt0.o"
 // CHECK-V6M-TREE-SAME: "-L[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi"
-// CHECK-V6M-TREE-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "--target2=rel" "-o" "{{.*}}.tmp.out"
+// CHECK-V6M-TREE-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-TREE-SAME: "-lc"
+// CHECK-V6M-TREE-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
 
 // RUN: %clang %s -### --target=armv7m-vendor-none-eabi -rtlib=compiler-rt 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm \
@@ -53,7 +57,8 @@
 // CHECK-ARMV7M-PER_TARGET: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-ARMV7M-PER-TARGET: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
 // CHECK-ARMV7M-PER-TARGET: "-L[[RESOURCE_DIR:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}armv7m-vendor-none-eabi
-// CHECK-ARMV7M-PER-TARGET: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-ARMV7M-PER-TARGET: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-ARMV7M-PER-TARGET: "-lc"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-DEFAULTCXX %s
@@ -61,8 +66,11 @@
 // CHECK-V6M-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
 // CHECK-V6M-DEFAULTCXX-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
-// CHECK-V6M-DEFAULTCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-V6M-DEFAULTCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "--target2=rel" "-o" "a.out"
+// CHECK-V6M-DEFAULTCXX-SAME: "-lc++"
+// CHECK-V6M-DEFAULTCXX-SAME: "-lm"
+// CHECK-V6M-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-DEFAULTCXX-SAME: "-lc"
+// CHECK-V6M-DEFAULTCXX-SAME: "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi -stdlib=libc++ 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-LIBCXX %s
@@ -71,8 +79,11 @@
 // CHECK-V6M-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
 // CHECK-V6M-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
-// CHECK-V6M-LIBCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-V6M-LIBCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "--target2=rel" "-o" "a.out"
+// CHECK-V6M-LIBCXX-SAME: "-lc++"
+// CHECK-V6M-LIBCXX-SAME: "-lm"
+// CHECK-V6M-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-LIBCXX-SAME: "-lc"
+// CHECK-V6M-LIBCXX-SAME: "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm \
@@ -83,8 +94,10 @@
 // CHECK-V6M-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}6.0.0"
 // CHECK-V6M-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
 // CHECK-V6M-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
-// CHECK-V6M-LIBSTDCXX-SAME: "-lstdc++" "-lsupc++" "-lunwind"
-// CHECK-V6M-LIBSTDCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "--target2=rel" "-o" "a.out"
+// CHECK-V6M-LIBSTDCXX-SAME: "-lstdc++" "-lm"
+// CHECK-V6M-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-LIBSTDCXX-SAME: "-lc"
+// CHECK-V6M-LIBSTDCXX-SAME: "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm \
@@ -106,8 +119,9 @@
 // CHECK-V6M-LIBCXX-USR-SAME: "-internal-isystem" "{{[^"]+}}baremetal_cxx_sysroot{{[/\\]+}}usr{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBCXX-USR: "{{[^"]*}}-Bstatic"
 // CHECK-V6M-LIBCXX-USR-SAME: "-L{{[^"]*}}{{[/\\]+}}baremetal_cxx_sysroot{{[/\\]+}}lib"
-// CHECK-V6M-LIBCXX-USR-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-V6M-LIBCXX-USR-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-LIBCXX-USR-SAME: "-lc++" "-lm"
+// CHECK-V6M-LIBCXX-USR-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-LIBCXX-USR-SAME: "-lc"
 
 // RUN: %clangxx --target=arm-none-eabi -v 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-THREAD-MODEL
@@ -125,7 +139,7 @@
 // RUN:   | FileCheck %s --check-prefix=CHECK-NOSTARTFILES
 // CHECK-NOSTARTFILES-NOT: "crt0.o"
 
-// RUN: %clang -### --target=arm-none-eabi -rtlib=libgcc -v %s 2>&1 \
+// RUN: %clang -### --target=arm-none-eabi -rtlib=libgcc --unwindlib=libgcc -v %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-RTLIB-GCC
 // CHECK-RTLIB-GCC: -lgcc
 
@@ -194,7 +208,9 @@
 // CHECK-RV64-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV64-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV64-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
-// CHECK-RV64-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "{{.*}}.tmp.out"
+// CHECK-RV64-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV64-SAME: "-lc"
+// CHECK-RV64-SAME: "-X" "-o" "{{.*}}.tmp.out"
 
 // RUN: %clangxx %s -### --target=riscv64-unknown-elf 2>&1 \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf \
@@ -202,8 +218,10 @@
 // CHECK-RV64-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV64-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV64-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV64-DEFAULTCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-RV64-DEFAULTCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV64-DEFAULTCXX-SAME: "-lc++" "-lm"
+// CHECK-RV64-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV64-DEFAULTCXX-SAME: "-lc"
+// CHECK-RV64-DEFAULTCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=riscv64-unknown-elf 2>&1 \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf \
@@ -214,8 +232,10 @@
 // CHECK-RV64-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV64-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV64-LIBCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-RV64-LIBCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV64-LIBCXX-SAME: "-lc++" "-lm"
+// CHECK-RV64-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV64-LIBCXX-SAME: "-lc"
+// CHECK-RV64-LIBCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clangxx %s -### 2>&1 --target=riscv64-unknown-elf \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf \
@@ -226,8 +246,10 @@
 // CHECK-RV64-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
 // CHECK-RV64-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV64-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV64-LIBSTDCXX-SAME: "-lstdc++" "-lsupc++" "-lunwind"
-// CHECK-RV64-LIBSTDCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV64-LIBSTDCXX-SAME: "-lstdc++" "-lm"
+// CHECK-RV64-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV64-LIBSTDCXX-SAME: "-lc"
+// CHECK-RV64-LIBSTDCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
 // RUN:     -L some/directory/user/asked/for \
@@ -242,7 +264,9 @@
 // CHECK-RV32-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV32-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV32-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
-// CHECK-RV32-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV32-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV32-SAME: "-lc"
+// CHECK-RV32-SAME: "-X" "-o" "a.out"
 
 // RUN: %clangxx %s -### 2>&1 --target=riscv32-unknown-elf \
 // RUN:     --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf \
@@ -250,8 +274,10 @@
 // CHECK-RV32-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV32-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV32-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV32-DEFAULTCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-RV32-DEFAULTCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV32-DEFAULTCXX-SAME: "-lc++" "-lm"
+// CHECK-RV32-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV32-DEFAULTCXX-SAME: "-lc"
+// CHECK-RV32-DEFAULTCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clangxx %s -### 2>&1 --target=riscv32-unknown-elf \
 // RUN:     --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf \
@@ -262,8 +288,9 @@
 // CHECK-RV32-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV32-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV32-LIBCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-RV32-LIBCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV32-LIBCXX-SAME: "-lc++" "-lm"
+// CHECK-RV32-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV32-LIBCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clangxx %s -### 2>&1 --target=riscv32-unknown-elf \
 // RUN:     --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf \
@@ -274,8 +301,9 @@
 // CHECK-RV32-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
 // CHECK-RV32-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-RV32-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
-// CHECK-RV32-LIBSTDCXX-SAME: "-lstdc++" "-lsupc++" "-lunwind"
-// CHECK-RV32-LIBSTDCXX-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-X" "-o" "a.out"
+// CHECK-RV32-LIBSTDCXX-SAME: "-lstdc++" "-lm"
+// CHECK-RV32-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-RV32-LIBSTDCXX-SAME: "-lc" "-X" "-o" "a.out"
 
 // RUN: %clang %s -### 2>&1 --target=riscv64-unknown-elf \
 // RUN:     -nostdlibinc -nobuiltininc \
@@ -397,7 +425,9 @@
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 // CHECK-PPCEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-PPCEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
-// CHECK-PPCEABI-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-o" "a.out"
+// CHECK-PPCEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-PPCEABI-SAME: "-lc"
+// CHECK-PPCEABI-SAME: "-o" "a.out"
 
 // RUN: %clang -no-canonical-prefixes %s -### --target=powerpc64-unknown-eabi 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-PPC64EABI %s
@@ -409,7 +439,9 @@
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 // CHECK-PPC64EABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-PPC64EABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
-// CHECK-PPC64EABI-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-o" "a.out"
+// CHECK-PPC64EABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-PPC64EABI-SAME: "-lc"
+// CHECK-PPC64EABI-SAME: "-o" "a.out"
 
 // RUN: %clang -no-canonical-prefixes %s -### --target=powerpcle-unknown-eabi 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-PPCLEEABI %s
@@ -421,7 +453,9 @@
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 // CHECK-PPCLEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-PPCLEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
-// CHECK-PPCLEEABI-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-o" "a.out"
+// CHECK-PPCLEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-PPCLEEABI-SAME: "-lc"
+// CHECK-PPCLEEABI-SAME: "-o" "a.out"
 
 // RUN: %clang -no-canonical-prefixes %s -### --target=powerpc64le-unknown-eabi 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-PPC64LEEABI %s
@@ -433,7 +467,9 @@
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 // CHECK-PPC64LEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
 // CHECK-PPC64LEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
-// CHECK-PPC64LEEABI-SAME: "-lc" "-lm" "{{[^"]*}}libclang_rt.builtins.a" "-o" "a.out"
+// CHECK-PPC64LEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-PPC64LEEABI-SAME: "-lc"
+// CHECK-PPC64LEEABI-SAME: "-o" "a.out"
 
 // Check that compiler-rt library without the arch filename suffix will
 // be used if present.
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index f534c1ddcec43..69e1e7ed471eb 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -717,6 +717,7 @@
 // RUN:     -fbracket-depth=123 \
 // RUN:     -fprofile-generate \
 // RUN:     -fprofile-generate=dir \
+// RUN:     -fprofile-sample-use=%S/Inputs/file.prof \
 // RUN:     -fno-profile-generate \
 // RUN:     -fno-profile-instr-generate \
 // RUN:     -fno-profile-instr-use \
diff --git a/clang/test/Driver/codeview-column-info.c b/clang/test/Driver/codeview-column-info.c
index 4cabefac06e64..0f1671a73acd5 100644
--- a/clang/test/Driver/codeview-column-info.c
+++ b/clang/test/Driver/codeview-column-info.c
@@ -14,5 +14,6 @@
 // CHECK: "-gno-column-info"
 
 // RUN: %clang_cl -### /Z7 -gcolumn-info -- %s 2>&1 | FileCheck --check-prefix=COLUMN %s
+// RUN: %clang_cl -### --target=x86_64-windows-msvc /Z7 -fprofile-sample-use=%S/Inputs/file.prof -- %s 2>&1 | FileCheck --check-prefix=COLUMN %s
 
 // COLUMN-NOT: "-gno-column-info"
diff --git a/clang/test/Driver/config-file.c b/clang/test/Driver/config-file.c
index 9df830ca4c753..e84d6dfca373a 100644
--- a/clang/test/Driver/config-file.c
+++ b/clang/test/Driver/config-file.c
@@ -82,3 +82,29 @@
 // CHECK-TWO-CONFIGS: -isysroot
 // CHECK-TWO-CONFIGS-SAME: /opt/data
 // CHECK-TWO-CONFIGS-SAME: -Wall
+
+//--- The linker input flags should be moved to the end of input list and appear only when linking.
+// RUN: %clang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg %s -lmylib -Wl,foo.a -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING
+// RUN: %clang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -fopenmp=libomp %s -lmylib -Wl,foo.a -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING-LIBOMP-GOES-AFTER
+// RUN: %clang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING
+// RUN: %clang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -fopenmp=libomp -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING-OPENMP
+// RUN: %clang --target=x86_64-pc-windows-msvc    --config %S/Inputs/config-l.cfg %s -lmylib -Wl,foo.lib -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING-MSVC
+// RUN: %clang --target=x86_64-pc-windows-msvc    --config %S/Inputs/config-l.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING-MSVC
+// CHECK-LINKING: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-LINKING: "-Wall"
+// CHECK-LINKING: "--as-needed" "{{.*}}-{{.*}}.o" "-lmylib" "foo.a" "-lm" "-Bstatic" "-lhappy" "-Bdynamic"
+// CHECK-LINKING-LIBOMP-GOES-AFTER: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-LINKING-LIBOMP-GOES-AFTER: "-Wall" {{.*}}"-fopenmp"
+// CHECK-LINKING-LIBOMP-GOES-AFTER: "--as-needed" "{{.*}}-{{.*}}.o" "-lmylib" "foo.a" "-lm" "-Bstatic" "-lhappy" "-Bdynamic" {{.*}}"-lomp"
+// CHECK-NOLINKING: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-NOLINKING: "-Wall"
+// CHECK-NOLINKING-NO: "-lm" "-Bstatic" "-lhappy" "-Bdynamic"
+// CHECK-NOLINKING-OPENMP: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-NOLINKING-OPENMP: "-Wall" {{.*}}"-fopenmp"
+// CHECK-NOLINKING-OPENMP-NO: "-lm" "-Bstatic" "-lhappy" "-Bdynamic" {{.*}}"-lomp"
+// CHECK-LINKING-MSVC: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-LINKING-MSVC: "-Wall"
+// CHECK-LINKING-MSVC: "--as-needed" "{{.*}}-{{.*}}.o" "mylib.lib" "foo.lib" "m.lib" "-Bstatic" "happy.lib" "-Bdynamic"
+// CHECK-NOLINKING-MSVC: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+// CHECK-NOLINKING-MSVC: "-Wall"
+// CHECK-NOLINKING-MSVC-NO: "m.lib" "-Bstatic" "happy.lib" "-Bdynamic"
diff --git a/clang/test/Driver/frame-pointer-elim.c b/clang/test/Driver/frame-pointer-elim.c
index cdedcc7ae4c89..f64ff6efc7261 100644
--- a/clang/test/Driver/frame-pointer-elim.c
+++ b/clang/test/Driver/frame-pointer-elim.c
@@ -162,5 +162,58 @@
 // RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
 // RUN: not %clang -### --target=riscv64-linux-android -mbig-endian -O1 -S %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+
+// On ARM backend bare metal targets, frame pointer is omitted
+// RUN: %clang -### --target=arm-arm-none-eabi -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=arm-arm-none-eabihf -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=arm-arm-none-eabi -S -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-arm-none-eabihf -S -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-arm-none-eabi -S -O1 %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=arm-arm-none-eabihf -S -O1 %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=arm-arm-none-eabi -S -O1 -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-arm-none-eabihf -S -O1 -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=armeb-arm-none-eabi -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=thumb-arm-none-eabi -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+// RUN: %clang -### --target=thumbeb-arm-none-eabi -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+
+// Check that for Apple bare metal targets, we're keeping frame pointers by default
+// RUN: %clang -### --target=thumbv6m-apple-none-macho -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=thumbv6m-apple-none-macho -S -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-apple-none-macho -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-apple-none-macho -S -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=thumbv6m-apple-none-macho -S -O1 %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=thumbv6m-apple-none-macho -S -O1 -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-apple-none-macho -S -O1 %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### --target=arm-apple-none-macho -S -O1 -fno-omit-frame-pointer %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+
+// AArch64 bare metal targets behave like hosted targets
+// RUN: %clang -### --target=aarch64-none-elf -S %s 2>&1 |  \
+// RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+// RUN: %clang -### --target=aarch64-none-elf -S -O1 %s 2>&1 |  \
+// RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+// RUN: %clang -### --target=aarch64-none-elf -S -fno-omit-frame-pointer %s 2>&1 |  \
+// RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+// RUN: %clang -### --target=aarch64-none-elf -S -O1 -fno-omit-frame-pointer %s 2>&1 |  \
+// RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+
 void f0() {}
 void f1() { f0(); }
diff --git a/clang/test/Driver/fuchsia.c b/clang/test/Driver/fuchsia.c
index 83dee16981690..cf92f85040901 100644
--- a/clang/test/Driver/fuchsia.c
+++ b/clang/test/Driver/fuchsia.c
@@ -297,3 +297,9 @@
 // RUN: %clang --target=riscv64-unknown-fuchsia -mno-relax -### %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=RISCV64-FLAGS %s
 // RISCV64-FLAGS: "-X" "--no-relax"
+
+// RUN: %clang -### %s --target=x86_64-unknown-fuchsia 2>&1 \
+// RUN:     -nostdlib -nolibc \
+// RUN:     | FileCheck %s -check-prefix=CHECK-NOSTDLIB-NOLIBC
+// CHECK-NOSTDLIB-NOLIBC-NOT: "warning:"
+// CHECK-NOSTDLIB-NOLIBC-NOT: "error:"
diff --git a/clang/test/Driver/linux-ld.c b/clang/test/Driver/linux-ld.c
index 28fb075a80dbb..4d641c8f1b46e 100644
--- a/clang/test/Driver/linux-ld.c
+++ b/clang/test/Driver/linux-ld.c
@@ -940,6 +940,36 @@
 // CHECK-ANDROID-HASH-STYLE-M: "{{.*}}ld{{(.exe)?}}"
 // CHECK-ANDROID-HASH-STYLE-M: "--hash-style=gnu"
 
+// Check that we pass --no-rosegment for pre-29 Android versions and do not for
+// 29+.
+// RUN: %clang %s -### -o %t.o 2>&1 \
+// RUN:     --target=armv7-linux-android28 \
+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-ROSEGMENT-28 %s
+// CHECK-ANDROID-ROSEGMENT-28: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ANDROID-ROSEGMENT-28: "--no-rosegment"
+//
+// RUN: %clang %s -### -o %t.o 2>&1 \
+// RUN:     --target=armv7-linux-android29 \
+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-ROSEGMENT-29 %s
+// CHECK-ANDROID-ROSEGMENT-29: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ANDROID-ROSEGMENT-29-NOT: "--no-rosegment"
+
+// Check that we pass --pack-dyn-relocs=relr for API 28+ and not before.
+// RUN: %clang %s -### -o %t.o 2>&1 \
+// RUN:     --target=armv7-linux-android27 \
+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-RELR-27 %s
+// CHECK-ANDROID-RELR-27: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ANDROID-RELR-27-NOT: "--pack-dyn-relocs=relr"
+// CHECK-ANDROID-RELR-27-NOT: "--pack-dyn-relocs=android+relr"
+//
+// RUN: %clang %s -### -o %t.o 2>&1 \
+// RUN:     --target=armv7-linux-android28 \
+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-RELR-28 %s
+// CHECK-ANDROID-RELR-28: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ANDROID-RELR-28: "--use-android-relr-tags"
+// CHECK-ANDROID-RELR-28: "--pack-dyn-relocs=relr"
+// CHECK-ANDROID-RELR-28-NOT: "--pack-dyn-relocs=android+relr"
+
 // RUN: %clang -### %s -no-pie 2>&1 --target=mips64-linux-gnuabin32 \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS64EL-GNUABIN32 %s
 // CHECK-MIPS64EL-GNUABIN32: "{{.*}}ld{{(.exe)?}}"
diff --git a/clang/test/Driver/loongarch-march.c b/clang/test/Driver/loongarch-march.c
index 981ae5c5c7dc1..cfcfa852efea5 100644
--- a/clang/test/Driver/loongarch-march.c
+++ b/clang/test/Driver/loongarch-march.c
@@ -39,21 +39,21 @@
 
 // CC1-LA64V1P1: "-target-cpu" "loongarch64"
 // CC1-LA64V1P1-NOT: "-target-feature"
-// CC1-LA64V1P1: "-target-feature" "+64bit" "-target-feature" "+d" "-target-feature" "+lsx" "-target-feature" "+ual" "-target-feature" "+frecipe" "-target-feature" "+lam-bh" "-target-feature" "+ld-seq-sa" "-target-feature" "+div32"
+// CC1-LA64V1P1: "-target-feature" "+64bit" "-target-feature" "+d" "-target-feature" "+lsx" "-target-feature" "+ual" "-target-feature" "+frecipe" "-target-feature" "+lam-bh" "-target-feature" "+lamcas" "-target-feature" "+ld-seq-sa" "-target-feature" "+div32"
 // CC1-LA64V1P1-NOT: "-target-feature"
 // CC1-LA64V1P1: "-target-abi" "lp64d"
 
 // CC1-LA664: "-target-cpu" "la664"
 // CC1-LA664-NOT: "-target-feature"
-// CC1-LA664: "-target-feature" "+64bit" "-target-feature" "+f" "-target-feature" "+d" "-target-feature" "+lsx" "-target-feature" "+lasx" "-target-feature" "+ual" "-target-feature" "+frecipe" "-target-feature" "+lam-bh" "-target-feature" "+ld-seq-sa" "-target-feature" "+div32"
+// CC1-LA664: "-target-feature" "+64bit" "-target-feature" "+f" "-target-feature" "+d" "-target-feature" "+lsx" "-target-feature" "+lasx" "-target-feature" "+ual" "-target-feature" "+frecipe" "-target-feature" "+lam-bh" "-target-feature" "+lamcas" "-target-feature" "+ld-seq-sa" "-target-feature" "+div32"
 // CC1-LA664-NOT: "-target-feature"
 // CC1-LA664: "-target-abi" "lp64d"
 
 // IR-LOONGARCH64: attributes #[[#]] ={{.*}}"target-cpu"="loongarch64" {{.*}}"target-features"="+64bit,+d,+f,+ual"
 // IR-LA464: attributes #[[#]] ={{.*}}"target-cpu"="la464" {{.*}}"target-features"="+64bit,+d,+f,+lasx,+lsx,+ual"
 // IR-LA64V1P0: attributes #[[#]] ={{.*}}"target-cpu"="loongarch64" {{.*}}"target-features"="+64bit,+d,+lsx,+ual"
-// IR-LA64V1P1: attributes #[[#]] ={{.*}}"target-cpu"="loongarch64" {{.*}}"target-features"="+64bit,+d,+div32,+frecipe,+lam-bh,+ld-seq-sa,+lsx,+ual"
-// IR-LA664: attributes #[[#]] ={{.*}}"target-cpu"="la664" {{.*}}"target-features"="+64bit,+d,+div32,+f,+frecipe,+lam-bh,+lasx,+ld-seq-sa,+lsx,+ual"
+// IR-LA64V1P1: attributes #[[#]] ={{.*}}"target-cpu"="loongarch64" {{.*}}"target-features"="+64bit,+d,+div32,+frecipe,+lam-bh,+lamcas,+ld-seq-sa,+lsx,+ual"
+// IR-LA664: attributes #[[#]] ={{.*}}"target-cpu"="la664" {{.*}}"target-features"="+64bit,+d,+div32,+f,+frecipe,+lam-bh,+lamcas,+lasx,+ld-seq-sa,+lsx,+ual"
 
 int foo(void) {
   return 3;
diff --git a/clang/test/Driver/loongarch-mlamcas.c b/clang/test/Driver/loongarch-mlamcas.c
new file mode 100644
index 0000000000000..2185a1a8115d6
--- /dev/null
+++ b/clang/test/Driver/loongarch-mlamcas.c
@@ -0,0 +1,30 @@
+/// Test -m[no]lamcas options.
+
+// RUN: %clang --target=loongarch64 -mlamcas -fsyntax-only %s -### 2>&1 | \
+// RUN:     FileCheck %s --check-prefix=CC1-LAMCAS
+// RUN: %clang --target=loongarch64 -mno-lamcas -fsyntax-only %s -### 2>&1 | \
+// RUN:     FileCheck %s --check-prefix=CC1-NO-LAMCAS
+// RUN: %clang --target=loongarch64 -mno-lamcas -mlamcas -fsyntax-only %s -### 2>&1 | \
+// RUN:     FileCheck %s --check-prefix=CC1-LAMCAS
+// RUN: %clang --target=loongarch64  -mlamcas -mno-lamcas -fsyntax-only %s -### 2>&1 | \
+// RUN:     FileCheck %s --check-prefix=CC1-NO-LAMCAS
+
+// RUN: %clang --target=loongarch64 -mlamcas -S -emit-llvm %s -o - | \
+// RUN: FileCheck %s --check-prefix=IR-LAMCAS
+// RUN: %clang --target=loongarch64 -mno-lamcas -S -emit-llvm %s -o - | \
+// RUN: FileCheck %s --check-prefix=IR-NO-LAMCAS
+// RUN: %clang --target=loongarch64 -mno-lamcas -mlamcas -S -emit-llvm %s -o - | \
+// RUN: FileCheck %s --check-prefix=IR-LAMCAS
+// RUN: %clang --target=loongarch64 -mlamcas -mno-lamcas -S -emit-llvm %s -o - | \
+// RUN: FileCheck %s --check-prefix=IR-NO-LAMCAS
+
+
+// CC1-LAMCAS: "-target-feature" "+lamcas"
+// CC1-NO-LAMCAS: "-target-feature" "-lamcas"
+
+// IR-LAMCAS: attributes #[[#]] ={{.*}}"target-features"="{{(.*,)?}}+lamcas{{(,.*)?}}"
+// IR-NO-LAMCAS: attributes #[[#]] ={{.*}}"target-features"="{{(.*,)?}}-lamcas{{(,.*)?}}"
+
+int foo(void) {
+  return 42;
+}
diff --git a/clang/test/Driver/module-fgen-reduced-bmi.cppm b/clang/test/Driver/module-fgen-reduced-bmi.cppm
index 1223189fb49b7..7329c12941d73 100644
--- a/clang/test/Driver/module-fgen-reduced-bmi.cppm
+++ b/clang/test/Driver/module-fgen-reduced-bmi.cppm
@@ -29,13 +29,41 @@
 //
 // RUN: %clang -std=c++20 Hello.cc -fexperimental-modules-reduced-bmi -Wall -Werror \
 // RUN:     -c -o Hello.o -### 2>&1 | FileCheck Hello.cc
+//
+// RUN: %clang -std=c++20 Hello.cppm -fmodule-output=Hello.pcm \
+// RUN:     -fmodules-reduced-bmi -c -o Hello.o -### 2>&1 | FileCheck Hello.cppm
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fmodules-reduced-bmi -c -o Hello.o -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-UNSPECIFIED
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fmodules-reduced-bmi -c -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-NO-O
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fmodules-reduced-bmi -c -o AnotherName.o -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-ANOTHER-NAME
+//
+// RUN: %clang -std=c++20 Hello.cppm --precompile -fmodules-reduced-bmi \
+// RUN:     -o Hello.full.pcm -### 2>&1 | FileCheck Hello.cppm \
+// RUN:     --check-prefix=CHECK-EMIT-MODULE-INTERFACE
+//
+// RUN: %clang -std=c++20 Hello.cc -fmodules-reduced-bmi -Wall -Werror \
+// RUN:     -c -o Hello.o -### 2>&1 | FileCheck Hello.cc
+//
+// RUN: %clang -std=c++20 Hello.cppm -fmodule-output=Hello.pcm -c -o Hello.o \
+// RUN:     -Wno-missing-reduced-bmi -### 2>&1 | FileCheck Hello.cppm -check-prefix=NO_WARN
+//
+// RUN: %clang -std=c++20 Hello.cppm --precompile -o Hello.pcm \
+// RUN:     -Wno-missing-reduced-bmi -### 2>&1 | FileCheck Hello.cppm -check-prefix=NO_WARN
 
 //--- Hello.cppm
 export module Hello;
 
 // Test that we won't generate the emit-module-interface as 2 phase compilation model.
 // CHECK-NOT: -emit-module-interface
-// CHECK: "-fexperimental-modules-reduced-bmi"
+// CHECK: "-fmodules-reduced-bmi"
 
 // CHECK-UNSPECIFIED: -fmodule-output=Hello.pcm
 
@@ -46,6 +74,8 @@ export module Hello;
 // flag.
 // CHECK-EMIT-MODULE-INTERFACE: -emit-module-interface
 
+// NO_WARN-NOT: warning
+
 //--- Hello.cc
 
-// CHECK-NOT: "-fexperimental-modules-reduced-bmi"
+// CHECK-NOT: "-fmodules-reduced-bmi"
diff --git a/clang/test/Driver/openmp-offload.c b/clang/test/Driver/openmp-offload.c
index cd1caf1156441..7fd33fa056b85 100644
--- a/clang/test/Driver/openmp-offload.c
+++ b/clang/test/Driver/openmp-offload.c
@@ -84,9 +84,15 @@
 
 /// Check -Xopenmp-target triggers error when an option requiring arguments is passed to it.
 // RUN:   not %clang -### -no-canonical-prefixes -fopenmp=libomp -fopenmp-targets=powerpc64le-ibm-linux-gnu -Xopenmp-target -Xopenmp-target -mcpu=pwr8 %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-NESTED-ERROR %s
+// RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-NESTED-ERROR_0 %s
 
-// CHK-FOPENMP-TARGET-NESTED-ERROR: clang{{.*}} error: invalid -Xopenmp-target argument: '-Xopenmp-target -Xopenmp-target', options requiring arguments are unsupported
+// CHK-FOPENMP-TARGET-NESTED-ERROR_0: error: invalid -Xopenmp-target argument: '-Xopenmp-target -Xopenmp-target', options requiring arguments are unsupported
+
+/// Check -Xopenmp-target= triggers error when an option requiring arguments is passed to it.
+// RUN:   not %clang -### -no-canonical-prefixes -fopenmp=libomp -fopenmp-targets=powerpc64le-ibm-linux-gnu -Xopenmp-target=powerpc64le-ibm-linux-gnu -Xopenmp-target -mcpu=pwr8 %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-NESTED-ERROR_1 %s
+
+// CHK-FOPENMP-TARGET-NESTED-ERROR_1: error: invalid -Xopenmp-target argument: '-Xopenmp-target=powerpc64le-ibm-linux-gnu -Xopenmp-target', options requiring arguments are unsupported
 
 /// ###########################################################################
 
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
new file mode 100644
index 0000000000000..3c74e3620df03
--- /dev/null
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
@@ -0,0 +1,82 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=fujitsu-monaka | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s
+
+// CHECK: Extensions enabled for the given AArch64 target
+// CHECK-EMPTY:
+// CHECK-NEXT:     Architecture Feature(s)                                Description
+// CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
+// CHECK-NEXT:     FEAT_AMUv1p1                                           Enable Armv8.6-A Activity Monitors Virtualization support
+// CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
+// CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
+// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
+// CHECK-NEXT:     FEAT_CLRBHB                                            Enable Clear BHB instruction
+// CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
+// CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
+// CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-NEXT:     FEAT_DPB2                                              Enable Armv8.5-A Cache Clean to Point of Deep Persistence
+// CHECK-NEXT:     FEAT_DotProd                                           Enable dot product support
+// CHECK-NEXT:     FEAT_ECV                                               Enable enhanced counter virtualization extension
+// CHECK-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// CHECK-NEXT:     FEAT_FAMINMAX                                          Enable FAMIN and FAMAX instructions
+// CHECK-NEXT:     FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
+// CHECK-NEXT:     FEAT_FGT                                               Enable fine grained virtualization traps extension
+// CHECK-NEXT:     FEAT_FHM                                               Enable FP16 FML instructions
+// CHECK-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-NEXT:     FEAT_FP8                                               Enable FP8 instructions
+// CHECK-NEXT:     FEAT_FP8DOT2                                           Enable FP8 2-way dot instructions
+// CHECK-NEXT:     FEAT_FP8DOT4                                           Enable FP8 4-way dot instructions
+// CHECK-NEXT:     FEAT_FP8FMA                                            Enable Armv9.5-A FP8 multiply-add instructions
+// CHECK-NEXT:     FEAT_FPAC                                              Enable Armv8.3-A Pointer Authentication Faulting enhancement
+// CHECK-NEXT:     FEAT_FRINTTS                                           Enable FRInt[32|64][Z|X] instructions that round a floating-point number to an integer (in FP format) forcing it to fit into a 32- or 64-bit int
+// CHECK-NEXT:     FEAT_FlagM                                             Enable Armv8.4-A Flag Manipulation instructions
+// CHECK-NEXT:     FEAT_FlagM2                                            Enable alternative NZCV format for floating point comparisons
+// CHECK-NEXT:     FEAT_HBC                                               Enable Armv8.8-A Hinted Conditional Branches Extension
+// CHECK-NEXT:     FEAT_HCX                                               Enable Armv8.7-A HCRX_EL2 system register
+// CHECK-NEXT:     FEAT_I8MM                                              Enable Matrix Multiply Int8 Extension
+// CHECK-NEXT:     FEAT_JSCVT                                             Enable Armv8.3-A JavaScript FP conversion instructions
+// CHECK-NEXT:     FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-NEXT:     FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-NEXT:     FEAT_LRCPC2                                            Enable Armv8.4-A RCPC instructions with Immediate Offsets
+// CHECK-NEXT:     FEAT_LS64, FEAT_LS64_V, FEAT_LS64_ACCDATA              Enable Armv8.7-A LD64B/ST64B Accelerator Extension
+// CHECK-NEXT:     FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-NEXT:     FEAT_LSE2                                              Enable Armv8.4-A Large System Extension 2 (LSE2) atomicity rules
+// CHECK-NEXT:     FEAT_LUT                                               Enable Lookup Table instructions
+// CHECK-NEXT:     FEAT_MEC                                               Enable Memory Encryption Contexts Extension
+// CHECK-NEXT:     FEAT_MOPS                                              Enable Armv8.8-A memcpy and memset acceleration instructions
+// CHECK-NEXT:     FEAT_MPAM                                              Enable Armv8.4-A Memory system Partitioning and Monitoring extension
+// CHECK-NEXT:     FEAT_NMI, FEAT_GICv3_NMI                               Enable Armv8.8-A Non-maskable Interrupts
+// CHECK-NEXT:     FEAT_NV, FEAT_NV2                                      Enable Armv8.4-A Nested Virtualization Enchancement
+// CHECK-NEXT:     FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-NEXT:     FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-NEXT:     FEAT_PAuth                                             Enable Armv8.3-A Pointer Authentication extension
+// CHECK-NEXT:     FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-NEXT:     FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-NEXT:     FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-NEXT:     FEAT_RME                                               Enable Realm Management Extension
+// CHECK-NEXT:     FEAT_RNG                                               Enable Random Number generation instructions
+// CHECK-NEXT:     FEAT_SB                                                Enable Armv8.5-A Speculation Barrier
+// CHECK-NEXT:     FEAT_SEL2                                              Enable Armv8.4-A Secure Exception Level 2 extension
+// CHECK-NEXT:     FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-NEXT:     FEAT_SHA3, FEAT_SHA512                                 Enable SHA512 and SHA3 support
+// CHECK-NEXT:     FEAT_SM4, FEAT_SM3                                     Enable SM3 and SM4 support
+// CHECK-NEXT:     FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
+// CHECK-NEXT:     FEAT_SPECRES2                                          Enable Speculation Restriction Instruction
+// CHECK-NEXT:     FEAT_SPEv1p2                                           Enable extra register in the Statistical Profiling Extension
+// CHECK-NEXT:     FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
+// CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
+// CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SHA3 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
+// CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+// CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
+// CHECK-NEXT:     FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-NEXT:     FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
+// CHECK-NEXT:     FEAT_WFxT                                              Enable Armv8.7-A WFET and WFIT instruction
+// CHECK-NEXT:     FEAT_XS                                                Enable Armv8.7-A limited-TLB-maintenance instruction
\ No newline at end of file
diff --git a/clang/test/Driver/print-supported-cpus-aarch64.c b/clang/test/Driver/print-supported-cpus-aarch64.c
new file mode 100644
index 0000000000000..3c1dcebf7c6c8
--- /dev/null
+++ b/clang/test/Driver/print-supported-cpus-aarch64.c
@@ -0,0 +1,27 @@
+// Test that --print-supported-cpus lists supported CPU models, including aliases.
+
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang --target=arm64-apple-macosx --print-supported-cpus 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK --implicit-check-not=apple-latest
+
+// CHECK: Target: arm64-apple-macosx
+
+// CHECK: apple-a11
+// CHECK: apple-a12
+// CHECK: apple-a13
+// CHECK: apple-a14
+// CHECK: apple-a15
+// CHECK: apple-a16
+// CHECK: apple-a17
+// CHECK: apple-a7
+// CHECK: apple-a8
+// CHECK: apple-a9
+// CHECK: apple-m1
+// CHECK: apple-m2
+// CHECK: apple-m3
+// CHECK: apple-m4
+// CHECK: apple-s4
+// CHECK: apple-s5
+
+// CHECK: Use -mcpu or -mtune to specify the target's processor.
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 151e4701a1da8..9df903115b57c 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -188,6 +188,9 @@
 // CHECK-NEXT:     smctr                1.0       'Smctr' (Control Transfer Records Machine Level)
 // CHECK-NEXT:     ssctr                1.0       'Ssctr' (Control Transfer Records Supervisor Level)
 // CHECK-NEXT:     svukte               0.3       'Svukte' (Address-Independent Latency of User-Mode Faults to Supervisor Addresses)
+// CHECK-NEXT:     xqcia                0.2       'Xqcia' (Qualcomm uC Arithmetic Extension)
+// CHECK-NEXT:     xqcicsr              0.2       'Xqcicsr' (Qualcomm uC CSR Extension)
+// CHECK-NEXT:     xqcisls              0.2       'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
 // CHECK-EMPTY:
 // CHECK-NEXT: Supported Profiles
 // CHECK-NEXT:     rva20s64
diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index c83066a334001..877a01c3de304 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -15,7 +15,7 @@
 // CHECK-ASAN-LINUX: "-lpthread"
 // CHECK-ASAN-LINUX: "-lrt"
 // CHECK-ASAN-LINUX: "-ldl"
-// CHECK-ASAN-LINUX: "-lresolv"
+// CHECK-ASAN-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=address -fno-sanitize-link-runtime -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -143,7 +143,7 @@
 // CHECK-ASAN-LINUX-CXX: "-lpthread"
 // CHECK-ASAN-LINUX-CXX: "-lrt"
 // CHECK-ASAN-LINUX-CXX: "-ldl"
-// CHECK-ASAN-LINUX-CXX: "-lresolv"
+// CHECK-ASAN-LINUX-CXX-NOT: "-lresolv"
 
 // RUN: %clang -### %s -o /dev/null -fsanitize=address \
 // RUN:     --target=i386-unknown-linux -fuse-ld=ld -stdlib=platform \
@@ -292,7 +292,7 @@
 // CHECK-TSAN-LINUX-CXX: "-lpthread"
 // CHECK-TSAN-LINUX-CXX: "-lrt"
 // CHECK-TSAN-LINUX-CXX: "-ldl"
-// CHECK-TSAN-LINUX-CXX: "-lresolv"
+// CHECK-TSAN-LINUX-CXX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=thread -fno-sanitize-link-runtime -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -365,7 +365,7 @@
 // CHECK-UBSAN-LINUX-NOT: libclang_rt.ubsan_standalone_cxx
 // CHECK-UBSAN-LINUX-NOT: "-lstdc++"
 // CHECK-UBSAN-LINUX: "-lpthread"
-// CHECK-UBSAN-LINUX: "-lresolv"
+// CHECK-UBSAN-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=undefined -fno-sanitize-link-runtime -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -438,7 +438,7 @@
 // CHECK-UBSAN-LINUX-CXX: "-lstdc++"
 // CHECK-UBSAN-LINUX-CXX-NOT: libclang_rt.asan
 // CHECK-UBSAN-LINUX-CXX: "-lpthread"
-// CHECK-UBSAN-LINUX-CXX: "-lresolv"
+// CHECK-UBSAN-LINUX-CXX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=undefined -fsanitize-minimal-runtime -### %s 2>&1 \
 // RUN:     --target=i386-unknown-linux -fuse-ld=ld \
@@ -448,7 +448,7 @@
 // CHECK-UBSAN-MINIMAL-LINUX: "{{.*}}ld{{(.exe)?}}"
 // CHECK-UBSAN-MINIMAL-LINUX: "--whole-archive" "{{.*}}libclang_rt.ubsan_minimal.a" "--no-whole-archive"
 // CHECK-UBSAN-MINIMAL-LINUX: "-lpthread"
-// CHECK-UBSAN-MINIMAL-LINUX: "-lresolv"
+// CHECK-UBSAN-MINIMAL-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=undefined -fsanitize-minimal-runtime -### %s 2>&1 \
 // RUN:     --target=x86_64-apple-darwin -fuse-ld=ld \
@@ -485,7 +485,7 @@
 // CHECK-ASAN-UBSAN-LINUX-NOT: libclang_rt.ubsan
 // CHECK-ASAN-UBSAN-LINUX-NOT: "-lstdc++"
 // CHECK-ASAN-UBSAN-LINUX: "-lpthread"
-// CHECK-ASAN-UBSAN-LINUX: "-lresolv"
+// CHECK-ASAN-UBSAN-LINUX-NOT: "-lresolv"
 
 // RUN: %clangxx -fsanitize=address,undefined -### %s 2>&1 \
 // RUN:     --target=i386-unknown-linux -fuse-ld=ld -stdlib=platform \
@@ -498,7 +498,7 @@
 // CHECK-ASAN-UBSAN-LINUX-CXX-NOT: libclang_rt.ubsan
 // CHECK-ASAN-UBSAN-LINUX-CXX: "-lstdc++"
 // CHECK-ASAN-UBSAN-LINUX-CXX: "-lpthread"
-// CHECK-ASAN-UBSAN-LINUX-CXX: "-lresolv"
+// CHECK-ASAN-UBSAN-LINUX-CXX-NOT: "-lresolv"
 
 // RUN: %clangxx -fsanitize=memory,undefined -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -541,7 +541,7 @@
 // CHECK-LSAN-LINUX: libclang_rt.lsan.a"
 // CHECK-LSAN-LINUX: "-lpthread"
 // CHECK-LSAN-LINUX: "-ldl"
-// CHECK-LSAN-LINUX: "-lresolv"
+// CHECK-LSAN-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=leak -fno-sanitize-link-runtime -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -564,7 +564,7 @@
 // CHECK-LSAN-COV-LINUX-NOT: libclang_rt.ubsan
 // CHECK-LSAN-COV-LINUX: "-lpthread"
 // CHECK-LSAN-COV-LINUX: "-ldl"
-// CHECK-LSAN-COV-LINUX: "-lresolv"
+// CHECK-LSAN-COV-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=leak,address -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -586,7 +586,7 @@
 // CHECK-ASAN-COV-LINUX-NOT: libclang_rt.ubsan
 // CHECK-ASAN-COV-LINUX-NOT: "-lstdc++"
 // CHECK-ASAN-COV-LINUX: "-lpthread"
-// CHECK-ASAN-COV-LINUX: "-lresolv"
+// CHECK-ASAN-COV-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=memory -fsanitize-coverage=func -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -610,7 +610,7 @@
 // CHECK-DFSAN-COV-LINUX-NOT: libclang_rt.ubsan
 // CHECK-DFSAN-COV-LINUX-NOT: "-lstdc++"
 // CHECK-DFSAN-COV-LINUX: "-lpthread"
-// CHECK-DFSAN-COV-LINUX: "-lresolv"
+// CHECK-DFSAN-COV-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=undefined -fsanitize-coverage=func -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -621,7 +621,7 @@
 // CHECK-UBSAN-COV-LINUX: "--whole-archive" "{{.*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 // CHECK-UBSAN-COV-LINUX-NOT: "-lstdc++"
 // CHECK-UBSAN-COV-LINUX: "-lpthread"
-// CHECK-UBSAN-COV-LINUX: "-lresolv"
+// CHECK-UBSAN-COV-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize-coverage=func -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -632,7 +632,7 @@
 // CHECK-COV-LINUX: "--whole-archive" "{{.*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 // CHECK-COV-LINUX-NOT: "-lstdc++"
 // CHECK-COV-LINUX: "-lpthread"
-// CHECK-COV-LINUX: "-lresolv"
+// CHECK-COV-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -fsanitize=numerical \
@@ -644,7 +644,8 @@
 // CHECK-NSAN-LINUX-NOT: "-lc"
 // CHECK-NSAN-LINUX-NOT: libclang_rt.ubsan
 // CHECK-NSAN-LINUX: libclang_rt.nsan.a"
-// CHECK-NSAN-LINUX: "-lpthread" "-lrt" "-lm" "-ldl" "-lresolv"
+// CHECK-NSAN-LINUX: "-lpthread" "-lrt" "-lm" "-ldl"
+// CHECK-NSAN-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -### %s 2>&1 --target=x86_64-unknown-linux -fuse-ld=ld -fsanitize=numerical -shared-libsan \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
@@ -757,7 +758,7 @@
 // CHECK-SAFESTACK-LINUX: libclang_rt.safestack.a"
 // CHECK-SAFESTACK-LINUX: "-lpthread"
 // CHECK-SAFESTACK-LINUX: "-ldl"
-// CHECK-SAFESTACK-LINUX: "-lresolv"
+// CHECK-SAFESTACK-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -fsanitize=shadow-call-stack -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
@@ -957,7 +958,7 @@
 // CHECK-SCUDO-LINUX-NOT: "-lstdc++"
 // CHECK-SCUDO-LINUX: "-lpthread"
 // CHECK-SCUDO-LINUX: "-ldl"
-// CHECK-SCUDO-LINUX: "-lresolv"
+// CHECK-SCUDO-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -### %s -o %t.so -shared 2>&1 \
 // RUN:     --target=i386-unknown-linux -fuse-ld=ld -fsanitize=scudo -shared-libsan \
@@ -1018,7 +1019,7 @@
 // CHECK-HWASAN-X86-64-LINUX: "-lpthread"
 // CHECK-HWASAN-X86-64-LINUX: "-lrt"
 // CHECK-HWASAN-X86-64-LINUX: "-ldl"
-// CHECK-HWASAN-X86-64-LINUX: "-lresolv"
+// CHECK-HWASAN-X86-64-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -fsanitize=hwaddress \
@@ -1067,7 +1068,7 @@
 // CHECK-HWASAN-AARCH64-LINUX: "-lpthread"
 // CHECK-HWASAN-AARCH64-LINUX: "-lrt"
 // CHECK-HWASAN-AARCH64-LINUX: "-ldl"
-// CHECK-HWASAN-AARCH64-LINUX: "-lresolv"
+// CHECK-HWASAN-AARCH64-LINUX-NOT: "-lresolv"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=aarch64-unknown-linux -fuse-ld=ld -fsanitize=hwaddress \
diff --git a/clang/test/Driver/unsupported-option.c b/clang/test/Driver/unsupported-option.c
index 6be531d9df7de..af836cf003374 100644
--- a/clang/test/Driver/unsupported-option.c
+++ b/clang/test/Driver/unsupported-option.c
@@ -17,3 +17,13 @@
 // RUN: not %clang -fprofile-sample-use=code.prof --target=powerpc-ibm-aix %s 2>&1 | \
 // RUN: FileCheck %s --check-prefix=AIX-PROFILE-SAMPLE
 // AIX-PROFILE-SAMPLE: error: unsupported option '-fprofile-sample-use=' for target
+
+// -mhtm is unsupported on x86_64. Test that using it in different command
+// line permutations generates an `unsupported option` error.
+// RUN: not %clang --target=x86_64 -### %s -mhtm 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=UNSUP_OPT
+// RUN: not %clang --target=x86_64 -### %s -mhtm -lc 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=UNSUP_OPT
+// RUN: not %clang --target=x86_64 -### -mhtm -lc %s 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=UNSUP_OPT
+// UNSUP_OPT: error: unsupported option
diff --git a/clang/test/Driver/xtensa-char.c b/clang/test/Driver/xtensa-char.c
new file mode 100644
index 0000000000000..13f8f67727e75
--- /dev/null
+++ b/clang/test/Driver/xtensa-char.c
@@ -0,0 +1,4 @@
+/// Check that char is unsigned by default.
+// RUN: %clang -### %s --target=xtensa -c 2>&1 | FileCheck %s
+// CHECK: "-cc1" "-triple" "xtensa"
+// CHECK-SAME: "-fno-signed-char"
diff --git a/clang/test/Format/docs_updated.test b/clang/test/Format/docs_updated.test
new file mode 100644
index 0000000000000..17066650a1267
--- /dev/null
+++ b/clang/test/Format/docs_updated.test
@@ -0,0 +1,5 @@
+// RUN: %python %S/../../docs/tools/dump_format_style.py -o %t.style
+// RUN: diff %t.style %S/../../docs/ClangFormatStyleOptions.rst
+
+// RUN: %python %S/../../docs/tools/dump_format_help.py -o %t.help
+// RUN: diff %t.help %S/../../docs/ClangFormat.rst
diff --git a/clang/test/Format/lit.local.cfg b/clang/test/Format/lit.local.cfg
index 3eb0f54ceaa6f..8acf02725d701 100644
--- a/clang/test/Format/lit.local.cfg
+++ b/clang/test/Format/lit.local.cfg
@@ -17,4 +17,5 @@ config.suffixes = [
     ".textpb",
     ".asciipb",
     ".td",
+    ".test"
 ]
diff --git a/clang/test/Headers/__clang_hip_math_ocml_rounded_ops.hip b/clang/test/Headers/__clang_hip_math_ocml_rounded_ops.hip
index 9e30c2c9c2ccb..9bccb0f571643 100644
--- a/clang/test/Headers/__clang_hip_math_ocml_rounded_ops.hip
+++ b/clang/test/Headers/__clang_hip_math_ocml_rounded_ops.hip
@@ -297,9 +297,9 @@ extern "C" __device__ double test___dmul_rz(double x, double y) {
 
 // CHECK-LABEL: @test___drcp_rd(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_div_rtn_f64(double noundef 1.000000e+00, double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___drcp_rd(float x) {
@@ -308,9 +308,9 @@ extern "C" __device__ float test___drcp_rd(float x) {
 
 // CHECK-LABEL: @test___drcp_rn(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_div_rte_f64(double noundef 1.000000e+00, double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___drcp_rn(float x) {
@@ -319,9 +319,9 @@ extern "C" __device__ float test___drcp_rn(float x) {
 
 // CHECK-LABEL: @test___drcp_ru(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_div_rtp_f64(double noundef 1.000000e+00, double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___drcp_ru(float x) {
@@ -330,9 +330,9 @@ extern "C" __device__ float test___drcp_ru(float x) {
 
 // CHECK-LABEL: @test___drcp_rz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_div_rtz_f64(double noundef 1.000000e+00, double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___drcp_rz(float x) {
@@ -341,9 +341,9 @@ extern "C" __device__ float test___drcp_rz(float x) {
 
 // CHECK-LABEL: @test___dsqrt_rd(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_sqrt_rtn_f64(double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___dsqrt_rd(float x) {
@@ -352,9 +352,9 @@ extern "C" __device__ float test___dsqrt_rd(float x) {
 
 // CHECK-LABEL: @test___dsqrt_rn(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_sqrt_rte_f64(double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___dsqrt_rn(float x) {
@@ -363,9 +363,9 @@ extern "C" __device__ float test___dsqrt_rn(float x) {
 
 // CHECK-LABEL: @test___dsqrt_ru(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_sqrt_rtp_f64(double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___dsqrt_ru(float x) {
@@ -374,9 +374,9 @@ extern "C" __device__ float test___dsqrt_ru(float x) {
 
 // CHECK-LABEL: @test___dsqrt_rz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[X:%.*]] to double
+// CHECK-NEXT:    [[CONV:%.*]] = fpext contract float [[X:%.*]] to double
 // CHECK-NEXT:    [[CALL_I:%.*]] = tail call contract noundef double @__ocml_sqrt_rtz_f64(double noundef [[CONV]]) #[[ATTR3]]
-// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL_I]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc contract double [[CALL_I]] to float
 // CHECK-NEXT:    ret float [[CONV1]]
 //
 extern "C" __device__ float test___dsqrt_rz(float x) {
diff --git a/clang/test/Index/print-type.cpp b/clang/test/Index/print-type.cpp
index b64469e921a8d..35bf511303332 100644
--- a/clang/test/Index/print-type.cpp
+++ b/clang/test/Index/print-type.cpp
@@ -163,7 +163,7 @@ inline namespace InlineNS {}
 // CHECK: DeclRefExpr=i:44:14 [type=int] [typekind=Int] [isPOD=1]
 // CHECK: StructDecl=Blob:46:8 (Definition) [type=Blob] [typekind=Record] [isPOD=1] [nbFields=2]
 // CHECK: FieldDecl=i:47:7 (Definition) [type=int] [typekind=Int] [isPOD=1]
-// CHECK: VarDecl=member_pointer:50:12 (Definition) [type=int Blob::*] [typekind=MemberPointer] [isPOD=1]
+// CHECK: VarDecl=member_pointer:50:12 (Definition) [type=int Blob::*] [typekind=MemberPointer] [canonicaltype=int Blob::*] [canonicaltypekind=MemberPointer] [isPOD=1]
 // CHECK: FunctionDecl=elaboratedNamespaceType:52:42 [type=NS::Type (const NS::Type)] [typekind=FunctionProto] [canonicaltype=NS::Type (NS::Type)] [canonicaltypekind=FunctionProto] [resulttype=NS::Type] [resulttypekind=Elaborated] [args= [const NS::Type] [Elaborated]] [isPOD=0]
 // CHECK: NamespaceRef=NS:52:11 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TypeRef=struct NS::Type:52:23 [type=NS::Type] [typekind=Record] [isPOD=1]
diff --git a/clang/test/Interpreter/crash.cpp b/clang/test/Interpreter/crash.cpp
new file mode 100644
index 0000000000000..1ab24b0febfa1
--- /dev/null
+++ b/clang/test/Interpreter/crash.cpp
@@ -0,0 +1,20 @@
+// REQUIRES: host-supports-jit, x86_64-linux
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+//
+// RUN: split-file %s %t
+//
+// RUN: %clang++ -std=c++20 -fPIC -c %t/vec.cpp -o %t/vec.o
+// RUN: %clang++ -shared %t/vec.o -o %t/vec.so
+//
+// RUN: cat %t/Test.cpp | LD_LIBRARY_PATH=%t:$LD_LIBRARY_PATH clang-repl
+
+//--- vec.cpp
+#include <vector>
+
+//--- Test.cpp
+%lib vec.so
+#include <vector>
+std::vector<int> v;
+%quit
diff --git a/clang/test/Layout/ms-x86-member-pointers.cpp b/clang/test/Layout/ms-x86-member-pointers.cpp
index a45359f23a9c4..3952226e7d787 100644
--- a/clang/test/Layout/ms-x86-member-pointers.cpp
+++ b/clang/test/Layout/ms-x86-member-pointers.cpp
@@ -17,56 +17,56 @@ struct UF { char a; int (U::*mp)(); };
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct SD
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    4 |   int struct S::* mp
+// CHECK-NEXT:    4 |   int S::* mp
 // CHECK-NEXT:      | [sizeof=8, align=4
 // CHECK-NEXT:      |  nvsize=8, nvalign=4]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct MD
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    4 |   int struct M::* mp
+// CHECK-NEXT:    4 |   int M::* mp
 // CHECK-NEXT:      | [sizeof=8, align=4
 // CHECK-NEXT:      |  nvsize=8, nvalign=4]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct VD
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    8 |   int struct V::* mp
+// CHECK-NEXT:    8 |   int V::* mp
 // CHECK-NEXT:      | [sizeof=16, align=8
 // CHECK-NEXT:      |  nvsize=16, nvalign=8]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct UD
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    8 |   int struct U::* mp
+// CHECK-NEXT:    8 |   int U::* mp
 // CHECK-NEXT:      | [sizeof=24, align=8
 // CHECK-NEXT:      |  nvsize=24, nvalign=8]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct SF
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    4 |   int (struct S::*)(void) __attribute__((thiscall)) mp
+// CHECK-NEXT:    4 |   int (S::*)(void) __attribute__((thiscall)) mp
 // CHECK-NEXT:      | [sizeof=8, align=4
 // CHECK-NEXT:      |  nvsize=8, nvalign=4]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct MF
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    8 |   int (struct M::*)(void) __attribute__((thiscall)) mp
+// CHECK-NEXT:    8 |   int (M::*)(void) __attribute__((thiscall)) mp
 // CHECK-NEXT:      | [sizeof=16, align=8
 // CHECK-NEXT:      |  nvsize=16, nvalign=8]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct VF
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    8 |   int (struct V::*)(void) __attribute__((thiscall)) mp
+// CHECK-NEXT:    8 |   int (V::*)(void) __attribute__((thiscall)) mp
 // CHECK-NEXT:      | [sizeof=24, align=8
 // CHECK-NEXT:      |  nvsize=24, nvalign=8]
 
 // CHECK: *** Dumping AST Record Layout
 // CHECK-NEXT:    0 | struct UF
 // CHECK-NEXT:    0 |   char a
-// CHECK-NEXT:    8 |   int (struct U::*)(void) __attribute__((thiscall)) mp
+// CHECK-NEXT:    8 |   int (U::*)(void) __attribute__((thiscall)) mp
 // CHECK-NEXT:      | [sizeof=24, align=8
 // CHECK-NEXT:      |  nvsize=24, nvalign=8]
 
diff --git a/clang/test/Lexer/gnu-flags.c b/clang/test/Lexer/gnu-flags.c
index 6c7bf9405ddf0..30cfcf710f346 100644
--- a/clang/test/Lexer/gnu-flags.c
+++ b/clang/test/Lexer/gnu-flags.c
@@ -16,6 +16,8 @@
 
 
 #if ALL || ZEROARGS
+// expected-warning@+9 {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}}
+// expected-note@+4 {{macro 'efoo' defined here}}
 // expected-warning@+3 {{token pasting of ',' and __VA_ARGS__ is a GNU extension}}
 #endif
 
diff --git a/clang/test/Misc/diag-template-diffing-cxx98.cpp b/clang/test/Misc/diag-template-diffing-cxx98.cpp
index 7b1a08c6b8691..888b9f8b0e5e3 100644
--- a/clang/test/Misc/diag-template-diffing-cxx98.cpp
+++ b/clang/test/Misc/diag-template-diffing-cxx98.cpp
@@ -47,3 +47,35 @@ namespace qualifiers {
 
   // CHECK: candidate template ignored: deduced conflicting types for parameter 'T' ('const vector<...>' vs. 'volatile vector<...>')
 }
+
+namespace integers {
+  template <int x>
+  class wrapper{};
+
+  template <int x>
+  class foo {
+   public:
+    wrapper<x> make();
+  };
+
+  wrapper<1> w1 = foo<2>().make();
+  // CHECK: no viable conversion from 'wrapper<2>' to 'wrapper<1>'
+
+  wrapper<1> w2 = foo<-3>().make();
+  // CHECK: no viable conversion from 'wrapper<-3>' to 'wrapper<1>'
+
+  template <int x>
+  wrapper<x> make();
+
+  wrapper<1> w3 = make<4>();
+  // CHECK: no viable conversion from 'wrapper<4>' to 'wrapper<1>'
+
+  template <int x>
+  wrapper<-x> makeNegative();
+
+  wrapper<1> w4 = makeNegative<5>();
+  // CHECK: no viable conversion from 'wrapper<-5>' to 'wrapper<1>'
+
+  wrapper<1> w5 = makeNegative<-6>();
+  // CHECK: no viable conversion from 'wrapper<6>' to 'wrapper<1>'
+}
diff --git a/clang/test/Misc/target-invalid-cpu-note/aarch64.c b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
index ab83f299ac599..e6ff09557fe07 100644
--- a/clang/test/Misc/target-invalid-cpu-note/aarch64.c
+++ b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
@@ -67,6 +67,7 @@
 // CHECK-SAME: {{^}}, exynos-m4
 // CHECK-SAME: {{^}}, exynos-m5
 // CHECK-SAME: {{^}}, falkor
+// CHECK-SAME: {{^}}, fujitsu-monaka
 // CHECK-SAME: {{^}}, generic
 // CHECK-SAME: {{^}}, grace
 // CHECK-SAME: {{^}}, kryo
diff --git a/clang/test/Misc/target-invalid-cpu-note/arm.c b/clang/test/Misc/target-invalid-cpu-note/arm.c
index 17280a9edd221..12acdabbb7756 100644
--- a/clang/test/Misc/target-invalid-cpu-note/arm.c
+++ b/clang/test/Misc/target-invalid-cpu-note/arm.c
@@ -84,6 +84,7 @@
 // CHECK-SAME: {{^}}, cortex-a78
 // CHECK-SAME: {{^}}, cortex-a78ae
 // CHECK-SAME: {{^}}, cortex-a78c
+// CHECK-SAME: {{^}}, cortex-a510
 // CHECK-SAME: {{^}}, cortex-a710
 // CHECK-SAME: {{^}}, cortex-x1
 // CHECK-SAME: {{^}}, cortex-x1c
diff --git a/clang/test/Modules/compare-record.c b/clang/test/Modules/compare-record.c
index 60998696969e1..ef4a3a5b0e90d 100644
--- a/clang/test/Modules/compare-record.c
+++ b/clang/test/Modules/compare-record.c
@@ -226,17 +226,17 @@ struct CompareBitfieldWidthExpression {
 #else
 struct CompareMatchingBitfields compareMatchingBitfields;
 struct CompareBitfieldPresence1 compareBitfieldPresence1;
-// expected-error@first.h:* {{'CompareBitfieldPresence1' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bitfield 'bitfieldPresence1'}}
-// expected-note@second.h:* {{but in 'Second' found non-bitfield 'bitfieldPresence1'}}
+// expected-error@first.h:* {{'CompareBitfieldPresence1' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bit-field 'bitfieldPresence1'}}
+// expected-note@second.h:* {{but in 'Second' found non-bit-field 'bitfieldPresence1'}}
 struct CompareBitfieldPresence2 compareBitfieldPresence2;
-// expected-error@first.h:* {{'CompareBitfieldPresence2' has different definitions in different modules; first difference is definition in module 'First.Hidden' found non-bitfield 'bitfieldPresence2'}}
-// expected-note@second.h:* {{but in 'Second' found bitfield 'bitfieldPresence2'}}
+// expected-error@first.h:* {{'CompareBitfieldPresence2' has different definitions in different modules; first difference is definition in module 'First.Hidden' found non-bit-field 'bitfieldPresence2'}}
+// expected-note@second.h:* {{but in 'Second' found bit-field 'bitfieldPresence2'}}
 struct CompareBitfieldWidth compareBitfieldWidth;
-// expected-error@first.h:* {{'CompareBitfieldWidth' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bitfield 'bitfieldWidth' with one width expression}}
-// expected-note@second.h:* {{but in 'Second' found bitfield 'bitfieldWidth' with different width expression}}
+// expected-error@first.h:* {{'CompareBitfieldWidth' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bit-field 'bitfieldWidth' with one width expression}}
+// expected-note@second.h:* {{but in 'Second' found bit-field 'bitfieldWidth' with different width expression}}
 struct CompareBitfieldWidthExpression compareBitfieldWidthExpression;
-// expected-error@first.h:* {{'CompareBitfieldWidthExpression' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bitfield 'bitfieldWidthExpression' with one width expression}}
-// expected-note@second.h:* {{but in 'Second' found bitfield 'bitfieldWidthExpression' with different width expressio}}
+// expected-error@first.h:* {{'CompareBitfieldWidthExpression' has different definitions in different modules; first difference is definition in module 'First.Hidden' found bit-field 'bitfieldWidthExpression' with one width expression}}
+// expected-note@second.h:* {{but in 'Second' found bit-field 'bitfieldWidthExpression' with different width expressio}}
 #endif
 
 #if defined(FIRST)
diff --git a/clang/test/Modules/cxx-templates.cpp b/clang/test/Modules/cxx-templates.cpp
index b7d5741e69af6..7b41a0b0bfb2c 100644
--- a/clang/test/Modules/cxx-templates.cpp
+++ b/clang/test/Modules/cxx-templates.cpp
@@ -251,7 +251,7 @@ namespace Std {
 
 // CHECK-DUMP:      ClassTemplateDecl {{.*}} <{{.*[/\\]}}cxx-templates-common.h:1:1, {{.*}}>  col:{{.*}} in cxx_templates_common SomeTemplate
 // CHECK-DUMP:        ClassTemplateSpecializationDecl {{.*}} prev {{.*}} SomeTemplate
-// CHECK-DUMP-NEXT:     TemplateArgument type 'char[2]'
+// CHECK-DUMP-NEXT:     TemplateArgument type 'char[{{1|2}}]'
 // CHECK-DUMP:        ClassTemplateSpecializationDecl {{.*}} SomeTemplate definition
 // CHECK-DUMP-NEXT:     DefinitionData
 // CHECK-DUMP-NEXT:       DefaultConstructor
@@ -260,9 +260,9 @@ namespace Std {
 // CHECK-DUMP-NEXT:       CopyAssignment
 // CHECK-DUMP-NEXT:       MoveAssignment
 // CHECK-DUMP-NEXT:       Destructor
-// CHECK-DUMP-NEXT:     TemplateArgument type 'char[2]'
+// CHECK-DUMP-NEXT:     TemplateArgument type 'char[{{1|2}}]'
 // CHECK-DUMP:        ClassTemplateSpecializationDecl {{.*}} prev {{.*}} SomeTemplate
-// CHECK-DUMP-NEXT:     TemplateArgument type 'char[1]'
+// CHECK-DUMP-NEXT:     TemplateArgument type 'char[{{1|2}}]'
 // CHECK-DUMP:        ClassTemplateSpecializationDecl {{.*}} SomeTemplate definition
 // CHECK-DUMP-NEXT:     DefinitionData
 // CHECK-DUMP-NEXT:       DefaultConstructor
@@ -271,4 +271,4 @@ namespace Std {
 // CHECK-DUMP-NEXT:       CopyAssignment
 // CHECK-DUMP-NEXT:       MoveAssignment
 // CHECK-DUMP-NEXT:       Destructor
-// CHECK-DUMP-NEXT:     TemplateArgument type 'char[1]'
+// CHECK-DUMP-NEXT:     TemplateArgument type 'char[{{1|2}}]'
diff --git a/clang/test/Modules/odr_hash.cpp b/clang/test/Modules/odr_hash.cpp
index fa8b2c81ab46e..4de0e50dbc0eb 100644
--- a/clang/test/Modules/odr_hash.cpp
+++ b/clang/test/Modules/odr_hash.cpp
@@ -264,8 +264,8 @@ struct S6 {
 };
 #else
 S6 s6;
-// expected-error@second.h:* {{'Field::S6' has different definitions in different modules; first difference is definition in module 'SecondModule' found bitfield 'x'}}
-// expected-note@first.h:* {{but in 'FirstModule' found non-bitfield 'x'}}
+// expected-error@second.h:* {{'Field::S6' has different definitions in different modules; first difference is definition in module 'SecondModule' found bit-field 'x'}}
+// expected-note@first.h:* {{but in 'FirstModule' found non-bit-field 'x'}}
 #endif
 
 #if defined(FIRST)
@@ -278,8 +278,8 @@ struct S7 {
 };
 #else
 S7 s7;
-// expected-error@second.h:* {{'Field::S7' has different definitions in different modules; first difference is definition in module 'SecondModule' found bitfield 'x' with one width expression}}
-// expected-note@first.h:* {{but in 'FirstModule' found bitfield 'x' with different width expression}}
+// expected-error@second.h:* {{'Field::S7' has different definitions in different modules; first difference is definition in module 'SecondModule' found bit-field 'x' with one width expression}}
+// expected-note@first.h:* {{but in 'FirstModule' found bit-field 'x' with different width expression}}
 #endif
 
 #if defined(FIRST)
@@ -292,8 +292,8 @@ struct S8 {
 };
 #else
 S8 s8;
-// expected-error@second.h:* {{'Field::S8' has different definitions in different modules; first difference is definition in module 'SecondModule' found bitfield 'x' with one width expression}}
-// expected-note@first.h:* {{but in 'FirstModule' found bitfield 'x' with different width expression}}
+// expected-error@second.h:* {{'Field::S8' has different definitions in different modules; first difference is definition in module 'SecondModule' found bit-field 'x' with one width expression}}
+// expected-note@first.h:* {{but in 'FirstModule' found bit-field 'x' with different width expression}}
 #endif
 
 #if defined(FIRST)
@@ -3084,8 +3084,8 @@ struct S5 {
 };
 #else
 S5 s5;
-// expected-error@second.h:* {{'PointersAndReferences::S5::x' from module 'SecondModule' is not present in definition of 'PointersAndReferences::S5' in module 'FirstModule'}}
-// expected-note@first.h:* {{declaration of 'x' does not match}}
+// expected-error@first.h:* {{'PointersAndReferences::S5::x' from module 'FirstModule' is not present in definition of 'PointersAndReferences::S5' in module 'SecondModule'}}
+// expected-note@second.h:* {{declaration of 'x' does not match}}
 #endif
 
 #if defined(FIRST)
diff --git a/clang/test/Modules/prune-non-affecting-module-map-repeated-textual.cpp b/clang/test/Modules/prune-non-affecting-module-map-repeated-textual.cpp
new file mode 100644
index 0000000000000..30914056c55fa
--- /dev/null
+++ b/clang/test/Modules/prune-non-affecting-module-map-repeated-textual.cpp
@@ -0,0 +1,104 @@
+// Same as prune-non-affecting-module-map-repeated.cpp, but check that textual-only
+// inclusions do not cause duplication of the module map files they are defined in.
+
+// RUN: rm -rf %t && mkdir %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-map-file=%t/base.map -fmodule-map-file=%t/mixed.map\
+// RUN:   -fmodule-map-file=%t/mod1.map \
+// RUN:   -fmodule-name=mod1 -emit-module %t/mod1.map -o %t/mod1.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-map-file=%t/mixed.map\
+// RUN:   -fmodule-name=mixed -emit-module %t/mixed.map -o %t/mixed.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-map-file=%t/base.map -fmodule-map-file=%t/mod1.map -fmodule-map-file=%t/mod2.map \
+// RUN:   -fmodule-file=%t/mod1.pcm -fmodule-file=%t/mixed.pcm \
+// RUN:   -fmodule-name=mod2 -emit-module %t/mod2.map -o %t/mod2.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-map-file=%t/base.map -fmodule-map-file=%t/mod2.map -fmodule-map-file=%t/mod3.map \
+// RUN:   -fmodule-file=%t/mod2.pcm -fmodule-file=%t/mixed.pcm \
+// RUN:   -fmodule-name=mod3 -emit-module %t/mod3.map -o %t/mod3.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN:   -fmodule-map-file=%t/base.map -fmodule-map-file=%t/mod3.map -fmodule-map-file=%t/mod4.map \
+// RUN:   -fmodule-file=%t/mod3.pcm \
+// RUN:   -fmodule-name=mod4 -emit-module %t/mod4.map -o %t/mod4.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules -fmodule-map-file=%t/base.map -fmodule-map-file=%t/mod4.map -fmodule-file=%t/mod4.pcm -fsyntax-only -verify %t/check_slocs.cc
+
+//--- base.map
+module base { textual header "vector.h" }
+//--- mixed.map
+module mixed { textual header "mixed_text.h" header "mixed_mod.h"}
+//--- mod1.map
+module mod1 { header "mod1.h" }
+//--- mod2.map
+module mod2 { header "mod2.h" }
+//--- mod3.map
+module mod3 { header "mod3.h" }
+//--- mod4.map
+module mod4 { header "mod4.h" }
+//--- check_slocs.cc
+#include "mod4.h"
+#include "vector.h"
+#pragma clang __debug sloc_usage // expected-remark {{source manager location address space usage}}
+// expected-note@* {{% of available space}}
+
+// base.map must only be present once, despite being used in each module.
+// Because its location in every module compile should be non-affecting.
+
+// expected-note@base.map:1 {{file entered 1 time}}
+
+// different modules use either only textual header from mixed.map or both textual and modular
+// headers. Either combination must lead to only 1 use at the end, because the module is ultimately
+// in the import chain and any textual uses should not change that.
+
+// expected-note@mixed.map:1 {{file entered 1 time}}
+
+// expected-note@* + {{file entered}}
+
+
+//--- vector.h
+#ifndef VECTOR_H
+#define VECTOR_H
+#endif
+
+//--- mixed_text.h
+#ifndef MIXED_TEXT_H
+#define MIXED_TEXT_H
+#endif
+//--- mixed_mod.h
+#ifndef MIXED_MOD_H
+#define MIXED_MOD_H
+#endif
+
+//--- mod1.h
+#ifndef MOD1
+#define MOD1
+#include "vector.h"
+#include "mixed_text.h"
+int mod1();
+#endif
+//--- mod2.h
+#ifndef MOD2
+#define MOD2
+#include "vector.h"
+#include "mod1.h"
+#include "mixed_mod.h"
+int mod2();
+#endif
+//--- mod3.h
+#ifndef MOD3
+#define MOD3
+#include "vector.h"
+#include "mod2.h"
+#include "mixed_text.h"
+#include "mixed_mod.h"
+int mod3();
+#endif
+//--- mod4.h
+#ifndef MOD4
+#define MOD4
+#include "vector.h"
+#include "mod3.h"
+int mod4();
+#endif
diff --git a/clang/test/Modules/recursive-instantiations.cppm b/clang/test/Modules/recursive-instantiations.cppm
new file mode 100644
index 0000000000000..d5854b0e647e3
--- /dev/null
+++ b/clang/test/Modules/recursive-instantiations.cppm
@@ -0,0 +1,40 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/type_traits.cppm -emit-module-interface -o %t/type_traits.pcm
+// RUN: %clang_cc1 -std=c++20 %t/test.cpp -fprebuilt-module-path=%t -verify
+
+//--- type_traits.cppm
+export module type_traits;
+
+export template <typename T>
+constexpr bool is_pod_v = __is_pod(T);
+
+//--- test.cpp
+// expected-no-diagnostics
+import type_traits;
+// Base is either void or wrapper<T>.
+template <class Base> struct wrapper : Base {};
+template <> struct wrapper<void> {};
+
+// wrap<0>::type<T> is wrapper<T>, wrap<1>::type<T> is wrapper<wrapper<T>>,
+// and so on.
+template <int N>
+struct wrap {
+  template <class Base>
+  using type = wrapper<typename wrap<N-1>::template type<Base>>;
+};
+
+template <>
+struct wrap<0> {
+  template <class Base>
+  using type = wrapper<Base>;
+};
+
+inline constexpr int kMaxRank = 40;
+template <int N, class Base = void>
+using rank = typename wrap<N>::template type<Base>;
+using rank_selector_t = rank<kMaxRank>;
+
+static_assert(is_pod_v<rank_selector_t>, "Must be POD");
diff --git a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
index cec9700f2fccd..232cfdad043f0 100644
--- a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
+++ b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
@@ -43,7 +43,7 @@ void test() {
 // CHECK1:       user_code.entry:
 // CHECK1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA14:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA15:![0-9]+]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4:[0-9]+]]
 // CHECK1-NEXT:    call void @__kmpc_target_deinit()
 // CHECK1-NEXT:    ret void
@@ -66,78 +66,78 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[REF_TMP2:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 8)
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA16:![0-9]+]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19:![0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP2]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP2]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP2]]) #[[ATTR11:[0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -161,9 +161,9 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA25:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA25]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
@@ -197,79 +197,79 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP15:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[REF_TMP16:%.*]] = alloca float, align 4
 // CHECK1-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], 1
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP6]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP6]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP6]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3:[0-9]+]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND:%.*]]
 // CHECK1:       omp.dispatch.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP19]], 1
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
 // CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -278,28 +278,28 @@ void test() {
 // CHECK1:       omp.dispatch.body:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP21]], 1
 // CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
 // CHECK1-NEXT:    br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to float
-// CHECK1-NEXT:    store float [[CONV]], ptr [[REF_TMP15]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT:    store float [[CONV]], ptr [[REF_TMP15]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to float
-// CHECK1-NEXT:    store float [[CONV17]], ptr [[REF_TMP16]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT:    store float [[CONV17]], ptr [[REF_TMP16]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP15]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP16]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
@@ -309,25 +309,25 @@ void test() {
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_INC:%.*]]
 // CHECK1:       omp.dispatch.inc:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND]]
 // CHECK1:       omp.dispatch.end:
 // CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
 // CHECK1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -358,21 +358,21 @@ void test() {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call float @_ZNKSt7complexIfE4realEv(ptr nonnull align 4 dereferenceable(8) [[TMP0]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA27:![0-9]+]]
 // CHECK1-NEXT:    [[ADD:%.*]] = fadd float [[TMP1]], [[CALL]]
-// CHECK1-NEXT:    store float [[ADD]], ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store float [[ADD]], ptr [[__RE_]], align 4, !tbaa [[TBAA27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[CALL2:%.*]] = call float @_ZNKSt7complexIfE4imagEv(ptr nonnull align 4 dereferenceable(8) [[TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA20:![0-9]+]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA29:![0-9]+]]
 // CHECK1-NEXT:    [[ADD3:%.*]] = fadd float [[TMP3]], [[CALL2]]
-// CHECK1-NEXT:    store float [[ADD3]], ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT:    store float [[ADD3]], ptr [[__IM_]], align 4, !tbaa [[TBAA29]]
 // CHECK1-NEXT:    ret ptr [[THIS1]]
 //
 //
@@ -507,17 +507,17 @@ void test() {
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA21:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA30:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
 // CHECK1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA32:![0-9]+]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA32]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA34:![0-9]+]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
@@ -535,7 +535,7 @@ void test() {
 // CHECK1:       user_code.entry:
 // CHECK1-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @__kmpc_target_deinit()
 // CHECK1-NEXT:    ret void
@@ -558,78 +558,78 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[REF_TMP2:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 16)
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA23:![0-9]+]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA36:![0-9]+]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP2]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP2]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA21]]
 // CHECK1-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -653,9 +653,9 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA38:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA40:![0-9]+]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA40]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
@@ -689,79 +689,79 @@ void test() {
 // CHECK1-NEXT:    [[REF_TMP15:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[REF_TMP16:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA38]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
 // CHECK1-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], 1
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP6]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP6]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP6]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND:%.*]]
 // CHECK1:       omp.dispatch.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
 // CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
 // CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP19]], 1
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
 // CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -770,28 +770,28 @@ void test() {
 // CHECK1:       omp.dispatch.body:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP21]], 1
 // CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
 // CHECK1-NEXT:    br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
 // CHECK1:       omp.inner.for.cond.cleanup:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to double
-// CHECK1-NEXT:    store double [[CONV]], ptr [[REF_TMP15]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store double [[CONV]], ptr [[REF_TMP15]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to double
-// CHECK1-NEXT:    store double [[CONV17]], ptr [[REF_TMP16]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store double [[CONV17]], ptr [[REF_TMP16]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP15]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP16]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]]) #[[ATTR11]]
 // CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
@@ -801,25 +801,25 @@ void test() {
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_INC:%.*]]
 // CHECK1:       omp.dispatch.inc:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    br label [[OMP_DISPATCH_COND]]
 // CHECK1:       omp.dispatch.end:
 // CHECK1-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
 // CHECK1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -850,21 +850,21 @@ void test() {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA38]]
+// CHECK1-NEXT:    store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call double @_ZNKSt7complexIdE4realEv(ptr nonnull align 8 dereferenceable(16) [[TMP0]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA25:![0-9]+]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA42:![0-9]+]]
 // CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP1]], [[CALL]]
-// CHECK1-NEXT:    store double [[ADD]], ptr [[__RE_]], align 8, !tbaa [[TBAA25]]
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store double [[ADD]], ptr [[__RE_]], align 8, !tbaa [[TBAA42]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    [[CALL2:%.*]] = call double @_ZNKSt7complexIdE4imagEv(ptr nonnull align 8 dereferenceable(16) [[TMP2]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA27:![0-9]+]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA44:![0-9]+]]
 // CHECK1-NEXT:    [[ADD3:%.*]] = fadd double [[TMP3]], [[CALL2]]
-// CHECK1-NEXT:    store double [[ADD3]], ptr [[__IM_]], align 8, !tbaa [[TBAA27]]
+// CHECK1-NEXT:    store double [[ADD3]], ptr [[__IM_]], align 8, !tbaa [[TBAA44]]
 // CHECK1-NEXT:    ret ptr [[THIS1]]
 //
 //
@@ -1012,17 +1012,17 @@ void test() {
 // CHECK1-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT:    store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA30]]
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTZERO_ADDR]], align 4
 // CHECK1-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA32]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA32]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA45:![0-9]+]]
 // CHECK1-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
@@ -1033,18 +1033,18 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA25]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA25]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4, !tbaa [[TBAA16]]
-// CHECK1-NEXT:    store float [[TMP1]], ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA25]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    store float [[TMP1]], ptr [[__RE_]], align 4, !tbaa [[TBAA27]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4, !tbaa [[TBAA16]]
-// CHECK1-NEXT:    store float [[TMP3]], ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA25]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4, !tbaa [[TBAA19]]
+// CHECK1-NEXT:    store float [[TMP3]], ptr [[__IM_]], align 4, !tbaa [[TBAA29]]
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1052,10 +1052,10 @@ void test() {
 // CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA27]]
 // CHECK1-NEXT:    ret float [[TMP0]]
 //
 //
@@ -1063,10 +1063,10 @@ void test() {
 // CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA23]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA29]]
 // CHECK1-NEXT:    ret float [[TMP0]]
 //
 //
@@ -1076,18 +1076,18 @@ void test() {
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__RE_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA38]]
+// CHECK1-NEXT:    store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA40]]
+// CHECK1-NEXT:    store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA40]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8, !tbaa [[TBAA23]]
-// CHECK1-NEXT:    store double [[TMP1]], ptr [[__RE_]], align 8, !tbaa [[TBAA25]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA40]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8, !tbaa [[TBAA36]]
+// CHECK1-NEXT:    store double [[TMP1]], ptr [[__RE_]], align 8, !tbaa [[TBAA42]]
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8, !tbaa [[TBAA23]]
-// CHECK1-NEXT:    store double [[TMP3]], ptr [[__IM_]], align 8, !tbaa [[TBAA27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA40]]
+// CHECK1-NEXT:    [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8, !tbaa [[TBAA36]]
+// CHECK1-NEXT:    store double [[TMP3]], ptr [[__IM_]], align 8, !tbaa [[TBAA44]]
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1095,10 +1095,10 @@ void test() {
 // CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__RE_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA25]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA42]]
 // CHECK1-NEXT:    ret double [[TMP0]]
 //
 //
@@ -1106,9 +1106,9 @@ void test() {
 // CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR5]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA38]]
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[__IM_:%.*]] = getelementptr inbounds nuw %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA27]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA44]]
 // CHECK1-NEXT:    ret double [[TMP0]]
 //
diff --git a/clang/test/OpenMP/target_parallel_ast_print.cpp b/clang/test/OpenMP/target_parallel_ast_print.cpp
index 7e27ac7b92ca4..3ee98bc525c1b 100644
--- a/clang/test/OpenMP/target_parallel_ast_print.cpp
+++ b/clang/test/OpenMP/target_parallel_ast_print.cpp
@@ -38,10 +38,6 @@ struct S {
 // CHECK:        static int TS;
 // CHECK-NEXT:   #pragma omp threadprivate(S<int>::TS)
 // CHECK-NEXT: }
-// CHECK:      template<> struct S<char> {
-// CHECK:        static char TS;
-// CHECK-NEXT:   #pragma omp threadprivate(S<char>::TS)
-// CHECK-NEXT: }
 
 template <typename T, int C>
 T tmain(T argc, T *argv) {
diff --git a/clang/test/OpenMP/target_teams_ast_print.cpp b/clang/test/OpenMP/target_teams_ast_print.cpp
index 8338f2a68f922..cc47ae92efac0 100644
--- a/clang/test/OpenMP/target_teams_ast_print.cpp
+++ b/clang/test/OpenMP/target_teams_ast_print.cpp
@@ -40,10 +40,6 @@ struct S {
 // CHECK:        static int TS;
 // CHECK-NEXT:   #pragma omp threadprivate(S<int>::TS)
 // CHECK-NEXT: }
-// CHECK:      template<> struct S<long> {
-// CHECK:        static long TS;
-// CHECK-NEXT:   #pragma omp threadprivate(S<long>::TS)
-// CHECK-NEXT: }
 
 template <typename T, int C>
 T tmain(T argc, T *argv) {
diff --git a/clang/test/OpenMP/task_ast_print.cpp b/clang/test/OpenMP/task_ast_print.cpp
index 2a6b8908a1e2d..30fb7ab75cc87 100644
--- a/clang/test/OpenMP/task_ast_print.cpp
+++ b/clang/test/OpenMP/task_ast_print.cpp
@@ -87,10 +87,6 @@ struct S {
 // CHECK:        static int TS;
 // CHECK-NEXT:   #pragma omp threadprivate(S<int>::TS)
 // CHECK-NEXT: }
-// CHECK:      template<> struct S<long> {
-// CHECK:        static long TS;
-// CHECK-NEXT:   #pragma omp threadprivate(S<long>::TS)
-// CHECK-NEXT: }
 
 template <typename T, int C>
 T tmain(T argc, T *argv) {
diff --git a/clang/test/OpenMP/taskloop_strictmodifier_codegen.cpp b/clang/test/OpenMP/taskloop_strictmodifier_codegen.cpp
new file mode 100644
index 0000000000000..82dd07a1a63bb
--- /dev/null
+++ b/clang/test/OpenMP/taskloop_strictmodifier_codegen.cpp
@@ -0,0 +1,165 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --prefix-filecheck-ir-name _ --version 5
+// RUN: %clang_cc1 -fopenmp -O1 -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+int main(int argc, char **argv) {
+#pragma omp task
+#pragma omp master taskloop priority(argc)
+  for (int i = 0; i < 10; ++i)
+    ;
+#pragma omp master taskloop nogroup grainsize(strict:argc)
+  for (int i = 0; i < 10; ++i)
+    ;
+  int i;
+#pragma omp master taskloop if(argc) shared(argc, argv) collapse(2) num_tasks(strict: 4)
+  for (i = 0; i < argc; ++i)
+  for (int j = argc; j < argv[argc][argc]; ++j)
+    ;
+#pragma omp master taskloop
+  for (int i = 0; i < 10; ++i) {
+#pragma omp cancel taskgroup
+#pragma omp cancellation point taskgroup
+  }
+}
+struct S {
+  int a;
+  S(int c) {
+#pragma omp master taskloop shared(c) num_tasks(strict:a)
+    for (a = 0; a < c; ++a)
+      ;
+  }
+} s(1);
+
+#endif
+
+// CHECK-LABEL: define noundef i32 @main(
+// CHECK-SAME: i32 noundef [[ARGC:%.*]], ptr noundef [[ARGV:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[ARGC_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[ARGV_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1:[0-9]+]])
+// CHECK-NEXT:    store i32 [[ARGC]], ptr [[ARGC_ADDR]], align 4, !tbaa [[TBAA3:![0-9]+]]
+// CHECK-NEXT:    store ptr [[ARGV]], ptr [[ARGV_ADDR]], align 8, !tbaa [[TBAA7:![0-9]+]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call ptr @__kmpc_omp_task_alloc(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i64 48, i64 1, ptr nonnull @.omp_task_entry..2)
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARGC_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP2]], align 8, !tbaa [[TBAA10:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @__kmpc_omp_task(ptr nonnull @[[GLOB1]], i32 [[TMP0]], ptr [[TMP1]])
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @__kmpc_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP5]], 0
+// CHECK-NEXT:    br i1 [[DOTNOT]], label %[[OMP_IF_END:.*]], label %[[OMP_IF_THEN:.*]]
+// CHECK:       [[OMP_IF_THEN]]:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARGC_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call ptr @__kmpc_omp_task_alloc(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i64 80, i64 1, ptr nonnull @.omp_task_entry..4)
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 40
+// CHECK-NEXT:    store i64 0, ptr [[TMP8]], align 8, !tbaa [[TBAA15:![0-9]+]]
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 48
+// CHECK-NEXT:    store i64 9, ptr [[TMP9]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 56
+// CHECK-NEXT:    store i64 1, ptr [[TMP10]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 72
+// CHECK-NEXT:    store i64 0, ptr [[TMP11]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP6]] to i64
+// CHECK-NEXT:    tail call void @__kmpc_taskloop_5(ptr nonnull @[[GLOB1]], i32 [[TMP0]], ptr [[TMP7]], i32 1, ptr nonnull [[TMP8]], ptr nonnull [[TMP9]], i64 1, i32 1, i32 1, i64 [[TMP12]], i32 1, ptr null) #[[ATTR1:[0-9]+]]
+// CHECK-NEXT:    tail call void @__kmpc_end_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    br label %[[OMP_IF_END]]
+// CHECK:       [[OMP_IF_END]]:
+// CHECK-NEXT:    [[TMP13:%.*]] = tail call i32 @__kmpc_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[DOTNOT22:%.*]] = icmp eq i32 [[TMP13]], 0
+// CHECK-NEXT:    br i1 [[DOTNOT22]], label %[[OMP_IF_END17:.*]], label %[[OMP_IF_THEN2:.*]]
+// CHECK:       [[OMP_IF_THEN2]]:
+// CHECK-NEXT:    tail call void @__kmpc_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARGC_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[ARGV_ADDR]], align 8, !tbaa [[TBAA7]]
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[TMP15]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX9]], align 1, !tbaa [[TBAA19:![0-9]+]]
+// CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP17]] to i32
+// CHECK-NEXT:    [[SUB12:%.*]] = sub i32 [[CONV]], [[TMP14]]
+// CHECK-NEXT:    [[CONV15:%.*]] = zext i32 [[SUB12]] to i64
+// CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV15]], [[IDXPROM]]
+// CHECK-NEXT:    [[SUB16:%.*]] = add nsw i64 [[MUL]], -1
+// CHECK-NEXT:    [[TMP18:%.*]] = tail call ptr @__kmpc_omp_task_alloc(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i64 80, i64 16, ptr nonnull @.omp_task_entry..6)
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[TMP18]], align 8, !tbaa [[TBAA20:![0-9]+]]
+// CHECK-NEXT:    store ptr [[ARGC_ADDR]], ptr [[TMP19]], align 8, !tbaa [[TBAA23:![0-9]+]]
+// CHECK-NEXT:    [[AGG_CAPTURED3_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP19]], i64 8
+// CHECK-NEXT:    store ptr [[ARGV_ADDR]], ptr [[AGG_CAPTURED3_SROA_2_0__SROA_IDX]], align 8, !tbaa [[TBAA25:![0-9]+]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARGC_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP20]], 0
+// CHECK-NEXT:    [[TMP21:%.*]] = sext i1 [[TOBOOL]] to i32
+// CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP18]], i64 40
+// CHECK-NEXT:    store i64 0, ptr [[TMP22]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP18]], i64 48
+// CHECK-NEXT:    store i64 [[SUB16]], ptr [[TMP23]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP18]], i64 56
+// CHECK-NEXT:    store i64 1, ptr [[TMP24]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP18]], i64 72
+// CHECK-NEXT:    store i64 0, ptr [[TMP25]], align 8
+// CHECK-NEXT:    call void @__kmpc_taskloop_5(ptr nonnull @[[GLOB1]], i32 [[TMP0]], ptr nonnull [[TMP18]], i32 [[TMP21]], ptr nonnull [[TMP22]], ptr nonnull [[TMP23]], i64 1, i32 1, i32 2, i64 4, i32 1, ptr null) #[[ATTR1]]
+// CHECK-NEXT:    call void @__kmpc_end_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_end_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    br label %[[OMP_IF_END17]]
+// CHECK:       [[OMP_IF_END17]]:
+// CHECK-NEXT:    [[TMP26:%.*]] = call i32 @__kmpc_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[DOTNOT23:%.*]] = icmp eq i32 [[TMP26]], 0
+// CHECK-NEXT:    br i1 [[DOTNOT23]], label %[[OMP_IF_END21:.*]], label %[[OMP_IF_THEN18:.*]]
+// CHECK:       [[OMP_IF_THEN18]]:
+// CHECK-NEXT:    call void @__kmpc_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP27:%.*]] = call ptr @__kmpc_omp_task_alloc(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i64 80, i64 1, ptr nonnull @.omp_task_entry..8)
+// CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 40
+// CHECK-NEXT:    store i64 0, ptr [[TMP28]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 48
+// CHECK-NEXT:    store i64 9, ptr [[TMP29]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 56
+// CHECK-NEXT:    store i64 1, ptr [[TMP30]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 72
+// CHECK-NEXT:    store i64 0, ptr [[TMP31]], align 8
+// CHECK-NEXT:    call void @__kmpc_taskloop(ptr nonnull @[[GLOB1]], i32 [[TMP0]], ptr [[TMP27]], i32 1, ptr nonnull [[TMP28]], ptr nonnull [[TMP29]], i64 1, i32 1, i32 0, i64 0, ptr null)
+// CHECK-NEXT:    call void @__kmpc_end_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_end_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    br label %[[OMP_IF_END21]]
+// CHECK:       [[OMP_IF_END21]]:
+// CHECK-NEXT:    ret i32 0
+//
+//
+//
+// CHECK-LABEL: define linkonce_odr void @_ZN1SC2Ei(
+// CHECK-SAME: ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]], i32 noundef [[C:%.*]]) unnamed_addr #[[ATTR6:[0-9]+]] align 2 {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[C_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]])
+// CHECK-NEXT:    store i32 [[C]], ptr [[C_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP1]], 0
+// CHECK-NEXT:    br i1 [[DOTNOT]], label %[[OMP_IF_END:.*]], label %[[OMP_IF_THEN:.*]]
+// CHECK:       [[OMP_IF_THEN]]:
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[THIS]], align 4, !tbaa [[TBAA35:![0-9]+]]
+// CHECK-NEXT:    tail call void @__kmpc_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[C_ADDR]], align 4, !tbaa [[TBAA3]]
+// CHECK-NEXT:    [[SUB4:%.*]] = add nsw i32 [[TMP3]], -1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call ptr @__kmpc_omp_task_alloc(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i64 80, i64 16, ptr nonnull @.omp_task_entry..10)
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8, !tbaa [[TBAA20]]
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[TMP5]], align 8, !tbaa [[TBAA37:![0-9]+]]
+// CHECK-NEXT:    [[AGG_CAPTURED_SROA_2_0__SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 8
+// CHECK-NEXT:    store ptr [[C_ADDR]], ptr [[AGG_CAPTURED_SROA_2_0__SROA_IDX]], align 8, !tbaa [[TBAA23]]
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 40
+// CHECK-NEXT:    store i64 0, ptr [[TMP6]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 48
+// CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[SUB4]] to i64
+// CHECK-NEXT:    store i64 [[CONV]], ptr [[TMP7]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 56
+// CHECK-NEXT:    store i64 1, ptr [[TMP8]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 72
+// CHECK-NEXT:    store i64 0, ptr [[TMP9]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP2]] to i64
+// CHECK-NEXT:    call void @__kmpc_taskloop_5(ptr nonnull @[[GLOB1]], i32 [[TMP0]], ptr nonnull [[TMP4]], i32 1, ptr nonnull [[TMP6]], ptr nonnull [[TMP7]], i64 1, i32 1, i32 2, i64 [[TMP10]], i32 1, ptr null) #[[ATTR1]]
+// CHECK-NEXT:    call void @__kmpc_end_taskgroup(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_end_master(ptr nonnull @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    br label %[[OMP_IF_END]]
+// CHECK:       [[OMP_IF_END]]:
+// CHECK-NEXT:    ret void
+
diff --git a/clang/test/OpenMP/teams_ast_print.cpp b/clang/test/OpenMP/teams_ast_print.cpp
index 0087f71ac9f74..597a9b2bdbdc5 100644
--- a/clang/test/OpenMP/teams_ast_print.cpp
+++ b/clang/test/OpenMP/teams_ast_print.cpp
@@ -27,10 +27,6 @@ struct S {
 // CHECK:        static int TS;
 // CHECK-NEXT:   #pragma omp threadprivate(S<int>::TS)
 // CHECK-NEXT: }
-// CHECK:      template<> struct S<long> {
-// CHECK:        static long TS;
-// CHECK-NEXT:   #pragma omp threadprivate(S<long>::TS)
-// CHECK-NEXT: }
 
 template <typename T, int C>
 T tmain(T argc, T *argv) {
diff --git a/clang/test/Parser/cxx1z-decomposition.cpp b/clang/test/Parser/cxx1z-decomposition.cpp
index 4b17f72effb0f..acf3f99069185 100644
--- a/clang/test/Parser/cxx1z-decomposition.cpp
+++ b/clang/test/Parser/cxx1z-decomposition.cpp
@@ -110,7 +110,7 @@ namespace BadSpecifiers {
     int [5] arr = {0}; // expected-error {{place the brackets after the name}}
 
     auto *[f] = s; // expected-error {{cannot be declared with type 'auto *'}} expected-error {{incompatible initializer}}
-    auto S::*[g] = s; // expected-error {{cannot be declared with type 'auto BadSpecifiers::S::*'}} expected-error {{incompatible initializer}}
+    auto S::*[g] = s; // expected-error {{cannot be declared with type 'auto S::*'}} expected-error {{incompatible initializer}}
 
     // ref-qualifiers are OK.
     auto &&[ok_1] = S();
diff --git a/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp b/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp
new file mode 100644
index 0000000000000..b8015b4815b2e
--- /dev/null
+++ b/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp
@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 -std=c++2c -fsyntax-only -fblocks -verify %s
+
+void a(...);
+
+void b(auto...);
+void c(auto, ...);
+
+void d(auto......); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}} \
+                    // expected-warning {{'...' in this location creates a C-style varargs function}} \
+                    // expected-note {{preceding '...' declares a function parameter pack}} \
+                    // expected-note {{insert ',' before '...' to silence this warning}}
+void e(auto..., ...);
+
+void f(auto x...); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+void g(auto x, ...);
+
+void h(auto... x...); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}} \
+                      // expected-warning {{'...' in this location creates a C-style varargs function}} \
+                      // expected-note {{preceding '...' declares a function parameter pack}} \
+                      // expected-note {{insert ',' before '...' to silence this warning}}
+void i(auto... x, ...);
+
+template<class ...Ts>
+void j(Ts... t...) {}; // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}} \
+                       // expected-warning {{'...' in this location creates a C-style varargs function}} \
+                       // expected-note {{preceding '...' declares a function parameter pack}} \
+                       // expected-note {{insert ',' before '...' to silence this warning}}
+template<class ...Ts>
+void k(Ts... t, ...) {}
+
+void l(int...); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+void m(int, ...);
+
+void n(int x...); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+void o(int x, ...);
+
+struct S {
+  void p(this S...) {} // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+};
+
+template<class ...Ts>
+void q(Ts......) {} // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}} \
+                    // expected-warning {{'...' in this location creates a C-style varargs function}} \
+                    // expected-note {{preceding '...' declares a function parameter pack}} \
+                    // expected-note {{insert ',' before '...' to silence this warning}}
+
+template<class T>
+void r(T...) {} // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+
+auto type_specifier = (void (*)(int...)) nullptr; // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+
+auto lambda = [](int...) {}; // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+
+auto block = ^(int...){}; // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
diff --git a/clang/test/Parser/cxx2c-pack-indexing.cpp b/clang/test/Parser/cxx2c-pack-indexing.cpp
index c279bdd7af8c4..99347a2f8f157 100644
--- a/clang/test/Parser/cxx2c-pack-indexing.cpp
+++ b/clang/test/Parser/cxx2c-pack-indexing.cpp
@@ -74,3 +74,12 @@ struct SS {
     }
 };
 }
+
+namespace GH119072 {
+
+template<typename... Ts>
+void foo() {
+  decltype(Ts...[0]::t) value;
+}
+
+}
diff --git a/clang/test/Parser/gh110231.cpp b/clang/test/Parser/gh110231.cpp
new file mode 100644
index 0000000000000..b1405517505ff
--- /dev/null
+++ b/clang/test/Parser/gh110231.cpp
@@ -0,0 +1,12 @@
+// RUN: seq 100 | xargs -Ifoo %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+// expected-no-diagnostics
+// This is a regression test for a non-deterministic stack-overflow.
+
+template < typename >
+concept C1 = true;
+
+template < typename , auto >
+concept C2 = true;
+
+template < C1 auto V, C2< V > auto>
+struct S;
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 037a3e186ee55..86265f630296c 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -39,6 +39,7 @@
 // CHECK-NOT: __ARM_FP_FAST 1
 // CHECK: __ARM_NEON 1
 // CHECK: __ARM_NEON_FP 0xE
+// CHECK: __ARM_NEON_SVE_BRIDGE 1
 // CHECK: __ARM_PCS_AAPCS64 1
 // CHECK-NOT: __ARM_PCS 1
 // CHECK-NOT: __ARM_PCS_VFP 1
diff --git a/clang/test/Preprocessor/init-aarch64.c b/clang/test/Preprocessor/init-aarch64.c
index 8ee6c6ba60af4..3d2f4b83abcb8 100644
--- a/clang/test/Preprocessor/init-aarch64.c
+++ b/clang/test/Preprocessor/init-aarch64.c
@@ -30,6 +30,7 @@
 // AARCH64-NEXT: #define __ARM_FP 0xE
 // AARCH64-NEXT: #define __ARM_FP16_ARGS 1
 // AARCH64-NEXT: #define __ARM_FP16_FORMAT_IEEE 1
+// AARCH64-NEXT: #define __ARM_NEON_SVE_BRIDGE 1
 // AARCH64-NEXT: #define __ARM_PCS_AAPCS64 1
 // AARCH64-NEXT: #define __ARM_SIZEOF_MINIMAL_ENUM 4
 // AARCH64-NEXT: #define __ARM_SIZEOF_WCHAR_T 4
diff --git a/clang/test/Preprocessor/init-loongarch.c b/clang/test/Preprocessor/init-loongarch.c
index 9045073cbb789..19458a2b14f40 100644
--- a/clang/test/Preprocessor/init-loongarch.c
+++ b/clang/test/Preprocessor/init-loongarch.c
@@ -798,7 +798,7 @@
 // LA64-FPU0-LP64S-NOT: #define __loongarch_single_float
 // LA64-FPU0-LP64S: #define __loongarch_soft_float 1
 
-/// Check __loongarch_arch{_tune/_frecipe/_lam_bh/_ld_seq_sa/_div32}.
+/// Check __loongarch_arch{_tune/_frecipe/_lam_bh/_lamcas/_ld_seq_sa/_div32}.
 
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - | \
 // RUN:   FileCheck --match-full-lines --check-prefix=ARCH-TUNE -DARCH=la64v1.0 -DTUNE=loongarch64 %s
@@ -823,11 +823,11 @@
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx | \
 // RUN:   FileCheck --match-full-lines --check-prefix=ARCH-TUNE -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 | \
-// RUN:   FileCheck --match-full-lines  --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=la64v1.1 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines  --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=la64v1.1 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -frecipe | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAM-BH,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -lsx | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=loongarch64 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=loongarch64 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +frecipe | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE -DARCH=loongarch64 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx -Xclang -target-feature -Xclang +frecipe | \
@@ -835,15 +835,23 @@
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +lam-bh | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAM-BH -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -lam-bh | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LD-SEQ-SA,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lam-bh | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAM-BH -DARCH=loongarch64 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx -Xclang -target-feature -Xclang +lam-bh | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAM-BH -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +lamcas | \
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAMCAS -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -lamcas | \
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lamcas | \
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAMCAS -DARCH=loongarch64 -DTUNE=loongarch64 %s
+// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx -Xclang -target-feature -Xclang +lamcas | \
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LAMCAS -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +ld-seq-sa | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LD-SEQ-SA -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -ld-seq-sa | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +ld-seq-sa | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,LD-SEQ-SA -DARCH=loongarch64 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx -Xclang -target-feature -Xclang +ld-seq-sa | \
@@ -851,26 +859,27 @@
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +div32 | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.1 -Xclang -target-feature -Xclang -div32| \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA -DARCH=la64v1.0 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,LD-SEQ-SA -DARCH=la64v1.0 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +div32 | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,DIV32 -DARCH=loongarch64 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -Xclang -target-feature -Xclang +lsx -Xclang -target-feature -Xclang +div32 | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,DIV32 -DARCH=la64v1.0 -DTUNE=loongarch64 %s
-// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +frecipe -Xclang -target-feature -Xclang +lam-bh  -Xclang -target-feature -Xclang +ld-seq-sa -Xclang -target-feature -Xclang +div32 | \
+// RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la64v1.0 -Xclang -target-feature -Xclang +frecipe -Xclang -target-feature -Xclang +lam-bh  -Xclang -target-feature -Xclang +lamcas -Xclang -target-feature -Xclang +ld-seq-sa -Xclang -target-feature -Xclang +div32 | \
 // RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE -DARCH=la64v1.1 -DTUNE=loongarch64 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la664 | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=la664 -DTUNE=la664 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=la664 -DTUNE=la664 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -mtune=la664 | \
 // RUN:   FileCheck --match-full-lines --check-prefix=ARCH-TUNE -DARCH=la64v1.0 -DTUNE=la664 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=loongarch64 -mtune=la664 | \
 // RUN:   FileCheck --match-full-lines --check-prefix=ARCH-TUNE -DARCH=loongarch64 -DTUNE=la664 %s
 // RUN: %clang --target=loongarch64 -x c -E -dM %s -o - -march=la664 -mtune=loongarch64 | \
-// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LD-SEQ-SA,DIV32 -DARCH=la664 -DTUNE=loongarch64 %s
+// RUN:   FileCheck --match-full-lines --check-prefixes=ARCH-TUNE,FRECIPE,LAM-BH,LAMCAS,LD-SEQ-SA,DIV32 -DARCH=la664 -DTUNE=loongarch64 %s
 
 // ARCH-TUNE: #define __loongarch_arch "[[ARCH]]"
 // DIV32: #define __loongarch_div32 1
 // FRECIPE: #define __loongarch_frecipe 1
 // LAM-BH: #define __loongarch_lam_bh 1
+// LAMCAS: #define __loongarch_lamcas 1
 // LD-SEQ-SA: #define __loongarch_ld_seq_sa 1
 // ARCH-TUNE: #define __loongarch_tune "[[TUNE]]"
 
diff --git a/clang/test/Preprocessor/macro_fn.c b/clang/test/Preprocessor/macro_fn.c
index 81d8363214078..2e72bd272084e 100644
--- a/clang/test/Preprocessor/macro_fn.c
+++ b/clang/test/Preprocessor/macro_fn.c
@@ -1,11 +1,17 @@
 /* RUN: %clang_cc1 %s -Eonly -std=c89 -pedantic -verify
 */
+// RUN: %clang_cc1 %s -Eonly -std=c89 -pedantic -Wno-gnu-zero-variadic-macro-arguments -verify -DOMIT_VARIADIC_MACRO_ARGS -DVARIADIC_MACRO_ARGS_REMOVE_COMMA
+// RUN: %clang_cc1 %s -Eonly -std=c89 -pedantic -Wno-variadic-macro-arguments-omitted -verify -DOMIT_VARIADIC_MACRO_ARGS
 /* PR3937 */
 #define zero() 0 /* expected-note 2 {{defined here}} */
 #define one(x) 0 /* expected-note 2 {{defined here}} */
 #define two(x, y) 0 /* expected-note 4 {{defined here}} */
 #define zero_dot(...) 0   /* expected-warning {{variadic macros are a C99 feature}} */
-#define one_dot(x, ...) 0 /* expected-warning {{variadic macros are a C99 feature}} expected-note 2{{macro 'one_dot' defined here}} */
+#define one_dot(x, ...) 0 /* expected-warning {{variadic macros are a C99 feature}} */
+
+#ifndef OMIT_VARIADIC_MACRO_ARGS
+/* expected-note@-3 2{{macro 'one_dot' defined here}} */
+#endif
 
 zero()
 zero(1);          /* expected-error {{too many arguments provided to function-like macro invocation}} */
@@ -37,16 +43,24 @@ e(x)
 e()
 
 zero_dot()
-one_dot(x)  /* empty ... argument: expected-warning {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}}  */
-one_dot()   /* empty first argument, elided ...: expected-warning {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} */
+one_dot(x)  /* empty ... argument */
+one_dot()   /* empty first argument, elided ... */
 
+#ifndef OMIT_VARIADIC_MACRO_ARGS
+/* expected-warning@-4 {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} */
+/* expected-warning@-4 {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} */
+#endif
 
 /* Crash with function-like macro test at end of directive. */
 #define E() (i == 0)
 #if E
 #endif
 
-
 #define NSAssert(condition, desc, ...) /* expected-warning {{variadic macros are a C99 feature}} */ \
-    SomeComplicatedStuff((desc), ##__VA_ARGS__) /* expected-warning {{token pasting of ',' and __VA_ARGS__ is a GNU extension}} */
+    SomeComplicatedStuff((desc), ##__VA_ARGS__)
+
+#ifndef VARIADIC_MACRO_ARGS_REMOVE_COMMA
+/* expected-warning@-3 {{token pasting of ',' and __VA_ARGS__ is a GNU extension}} */
+#endif
+
 NSAssert(somecond, somedesc)
diff --git a/clang/test/Sema/aarch64-fixed-global-register.c b/clang/test/Sema/aarch64-fixed-global-register.c
index 9b4a422d8c1b2..f66c3475dae38 100644
--- a/clang/test/Sema/aarch64-fixed-global-register.c
+++ b/clang/test/Sema/aarch64-fixed-global-register.c
@@ -1,4 +1,13 @@
-// RUN: %clang_cc1 -triple aarch64-unknown-none-gnu %s -verify -fsyntax-only
+// RUN: %clang_cc1 -triple aarch64-unknown-none-gnu %s -target-feature +reserve-x4 -target-feature +reserve-x15 -verify -verify=no_x18 -fsyntax-only
+// RUN: %clang_cc1 -triple aarch64-unknown-android %s -target-feature +reserve-x4 -target-feature +reserve-x15 -verify -fsyntax-only
 
+register int w0 __asm__ ("w0");
+register long x0 __asm__ ("x0");
 register char i1 __asm__ ("x15"); // expected-error {{size of register 'x15' does not match variable size}}
-register long long l2 __asm__ ("w14"); // expected-error {{size of register 'w14' does not match variable size}}
+register long long l2 __asm__ ("w15"); // expected-error {{size of register 'w15' does not match variable size}}
+register int w3 __asm__ ("w3"); // expected-error {{register 'w3' unsuitable for global register variables on this target}}
+register long x3 __asm__ ("x3"); // expected-error {{register 'x3' unsuitable for global register variables on this target}}
+register int w4 __asm__ ("w4");
+register long x4 __asm__ ("x4");
+register int w18 __asm__ ("w18"); // no_x18-error {{register 'w18' unsuitable for global register variables on this target}}
+register long x18 __asm__ ("x18"); // no_x18-error {{register 'x18' unsuitable for global register variables on this target}}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_cvt.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_cvt.c
new file mode 100644
index 0000000000000..af1ef46ea6972
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_cvt.c
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify -emit-llvm-only %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+
+void test_features_sme2_fp8(svmfloat8_t zn, fpm_t fpmr) __arm_streaming {
+    // expected-error@+1 {{'svcvtl1_f16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvtl1_f16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvtl2_f16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvtl2_f16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvtl1_bf16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvtl1_bf16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvtl2_bf16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvtl2_bf16_mf8_x2_fpm(zn, fpmr);
+
+    // expected-error@+1 {{'svcvt1_f16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvt1_f16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvt2_f16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvt2_f16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvt1_bf16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvt1_bf16_mf8_x2_fpm(zn, fpmr);
+    // expected-error@+1 {{'svcvt2_bf16_mf8_x2_fpm' needs target feature sme,sme2,fp8}}
+    svcvt2_bf16_mf8_x2_fpm(zn, fpmr);
+}
\ No newline at end of file
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_imm.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_imm.c
new file mode 100644
index 0000000000000..62cad9cfa4c8f
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_imm.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -fsyntax-only -verify  %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_svmopa(svbool_t pn, svbool_t pm, svmfloat8_t zn, svmfloat8_t zm,
+                 fpm_t fpmr) __arm_streaming __arm_inout("za") {
+    // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+    svmopa_za16_mf8_m_fpm(-1, pn, pm, zn, zm, fpmr);
+    // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+    svmopa_za16_mf8_m_fpm(2, pn, pm, zn, zm, fpmr);
+
+    // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+    svmopa_za32_mf8_m_fpm(-1, pn, pm, zn, zm, fpmr);
+    // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svmopa_za32_mf8_m_fpm(4, pn, pm, zn, zm, fpmr);
+}
diff --git a/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_mopa.c b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_mopa.c
new file mode 100644
index 0000000000000..86426abcd4329
--- /dev/null
+++ b/clang/test/Sema/aarch64-fp8-intrinsics/acle_sme2_fp8_mopa.c
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify -emit-llvm-only %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features(svbool_t pn, svbool_t pm, svmfloat8_t zn, svmfloat8_t zm,
+                   fpm_t fpmr) __arm_streaming __arm_inout("za") {
+    // expected-error@+1 {{'svmopa_za16_mf8_m_fpm' needs target feature sme,sme-f8f16}}
+    svmopa_za16_mf8_m_fpm(0, pn, pm, zn, zm, fpmr);
+    // expected-error@+1 {{'svmopa_za32_mf8_m_fpm' needs target feature sme,sme-f8f32}}
+    svmopa_za32_mf8_m_fpm(0, pn, pm, zn, zm, fpmr);
+}
diff --git a/clang/test/Sema/aarch64-sve-intrinsics/acle_neon_sve_bridge.cpp b/clang/test/Sema/aarch64-sve-intrinsics/acle_neon_sve_bridge.cpp
new file mode 100644
index 0000000000000..a3f3764939156
--- /dev/null
+++ b/clang/test/Sema/aarch64-sve-intrinsics/acle_neon_sve_bridge.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu  -verify -emit-llvm -o - %s
+// REQUIRES: aarch64-registered-target
+
+// Test that we can use __ARM_NEON_SVE_BRIDGE to guard to inclusion of arm_neon_sve_bridge.h,
+// and use the associated intrinsics via a target() attribute.
+
+// expected-no-diagnostics
+
+#ifdef __ARM_NEON_SVE_BRIDGE
+#include <arm_neon_sve_bridge.h>
+#endif
+
+uint32x4_t __attribute__((target("+sve"))) foo(svuint32_t a) {
+    return svget_neonq_u32(a);
+}
\ No newline at end of file
diff --git a/clang/test/Sema/aarch64-sve2-intrinsics/acle_sve2_fp8.c b/clang/test/Sema/aarch64-sve2-intrinsics/acle_sve2_fp8.c
new file mode 100644
index 0000000000000..2e94c2314f182
--- /dev/null
+++ b/clang/test/Sema/aarch64-sve2-intrinsics/acle_sve2_fp8.c
@@ -0,0 +1,24 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -verify -emit-llvm -o - %s
+
+#include <arm_sve.h>
+
+void test_features(svmfloat8_t zn, fpm_t fpm) {
+  svcvt1_bf16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvt1_bf16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvt2_bf16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvt2_bf16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvtlt1_bf16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvtlt1_bf16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvtlt2_bf16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvtlt2_bf16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvt1_f16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvt1_f16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvt2_f16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvt2_f16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvtlt1_f16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvtlt1_f16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+  svcvtlt2_f16_mf8_fpm(zn, fpm);
+  // expected-error@-1 {{'svcvtlt2_f16_mf8_fpm' needs target feature (sve,sve2,fp8)|(sme,sme2,fp8)}}
+}
diff --git a/clang/test/Sema/builtin-counted-by-ref.c b/clang/test/Sema/builtin-counted-by-ref.c
index 5a7ecefcb7897..a9f46a3223006 100644
--- a/clang/test/Sema/builtin-counted-by-ref.c
+++ b/clang/test/Sema/builtin-counted-by-ref.c
@@ -11,62 +11,63 @@ struct fam_struct {
   int array[] __attribute__((counted_by(count)));
 };
 
-void test1(struct fam_struct *ptr, int size, int idx) {
-  size_t size_of = sizeof(__builtin_counted_by_ref(ptr->array)); // ok
-
-  *__builtin_counted_by_ref(ptr->array) = size;             // ok
+void g(char *);
+void h(char);
 
-  {
-      size_t __ignored_assignment;
-      *_Generic(__builtin_counted_by_ref(ptr->array),
-               void *: &__ignored_assignment,
-               default: __builtin_counted_by_ref(ptr->array)) = 42; // ok
-  }
+void test1(struct fam_struct *ptr, int size, int idx) {
+  size_t size_of = sizeof(__builtin_counted_by_ref(ptr->array));    // ok
+  int align_of = __alignof(__builtin_counted_by_ref(ptr->array));   // ok
+  size_t __ignored_assignment;
+
+  *__builtin_counted_by_ref(ptr->array) = size;                     // ok
+  *_Generic(__builtin_counted_by_ref(ptr->array),
+           void *: &__ignored_assignment,
+           default: __builtin_counted_by_ref(ptr->array)) = 42;     // ok
+  h(*__builtin_counted_by_ref(ptr->array));                         // ok
 }
 
 void test2(struct fam_struct *ptr, int idx) {
-  __builtin_counted_by_ref();                               // expected-error {{too few arguments to function call, expected 1, have 0}}
-  __builtin_counted_by_ref(ptr->array, ptr->x, ptr->count); // expected-error {{too many arguments to function call, expected 1, have 3}}
+  __builtin_counted_by_ref();                                       // expected-error {{too few arguments to function call, expected 1, have 0}}
+  __builtin_counted_by_ref(ptr->array, ptr->x, ptr->count);         // expected-error {{too many arguments to function call, expected 1, have 3}}
 }
 
 void test3(struct fam_struct *ptr, int idx) {
-  __builtin_counted_by_ref(&ptr->array[0]);                 // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(&ptr->array[idx]);               // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(&ptr->array);                    // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(ptr->x);                         // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(&ptr->x);                        // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(global_array);                   // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(global_int);                     // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  __builtin_counted_by_ref(&global_int);                    // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(&ptr->array[0]);                         // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(&ptr->array[idx]);                       // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(&ptr->array);                            // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(ptr->x);                                 // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(&ptr->x);                                // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(global_array);                           // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(global_int);                             // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  __builtin_counted_by_ref(&global_int);                            // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
 }
 
 void test4(struct fam_struct *ptr, int idx) {
-  __builtin_counted_by_ref(ptr++->array);                   // expected-error {{'__builtin_counted_by_ref' argument cannot have side-effects}}
-  __builtin_counted_by_ref(&ptr->array[idx++]);             // expected-error {{'__builtin_counted_by_ref' argument cannot have side-effects}}
+  __builtin_counted_by_ref(ptr++->array);                           // expected-error {{'__builtin_counted_by_ref' argument cannot have side-effects}}
+  __builtin_counted_by_ref(&ptr->array[idx++]);                     // expected-error {{'__builtin_counted_by_ref' argument cannot have side-effects}}
 }
 
-void foo(char *);
-
 void *test5(struct fam_struct *ptr, int size, int idx) {
-  char *ref = __builtin_counted_by_ref(ptr->array);         // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
+  char *ref = __builtin_counted_by_ref(ptr->array);                 // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable}}
+  char *int_ptr;
+  char *p;
 
-  ref = __builtin_counted_by_ref(ptr->array);               // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
-  ref = (char *)(int *)(42 + &*__builtin_counted_by_ref(ptr->array)); // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
-  foo(__builtin_counted_by_ref(ptr->array));                // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
-  foo(ref = __builtin_counted_by_ref(ptr->array));          // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
+  ref = __builtin_counted_by_ref(ptr->array);                       // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable}}
+  g(__builtin_counted_by_ref(ptr->array));                          // expected-error {{value returned by '__builtin_counted_by_ref' cannot be passed into a function}}
+  g(ref = __builtin_counted_by_ref(ptr->array));                    // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable}}
 
-  if ((ref = __builtin_counted_by_ref(ptr->array)))         // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
+  if ((ref = __builtin_counted_by_ref(ptr->array)))                 // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable}}
     ;
 
-  for (char *p = __builtin_counted_by_ref(ptr->array); p && *p; ++p) // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
+  for (p = __builtin_counted_by_ref(ptr->array); p && *p; ++p)      // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable}}
     ;
 
-  return __builtin_counted_by_ref(ptr->array);              // expected-error {{value returned by '__builtin_counted_by_ref' cannot be assigned to a variable, have its address taken, or passed into or returned from a function}}
+  return __builtin_counted_by_ref(ptr->array);                      // expected-error {{value returned by '__builtin_counted_by_ref' cannot be returned from a function}}
 }
 
 void test6(struct fam_struct *ptr, int size, int idx) {
-  *(__builtin_counted_by_ref(ptr->array) + 4) = 37;         // expected-error {{value returned by '__builtin_counted_by_ref' cannot be used in a binary expression}}
-  __builtin_counted_by_ref(ptr->array)[3] = 37;             // expected-error {{value returned by '__builtin_counted_by_ref' cannot be used in an array subscript expression}}
+  *(__builtin_counted_by_ref(ptr->array) + 4) = 37;                 // expected-error {{value returned by '__builtin_counted_by_ref' cannot be used in a binary expression}}
+  __builtin_counted_by_ref(ptr->array)[3] = 37;                     // expected-error {{value returned by '__builtin_counted_by_ref' cannot be used in an array subscript expression}}
 }
 
 struct non_fam_struct {
@@ -77,10 +78,10 @@ struct non_fam_struct {
 };
 
 void *test7(struct non_fam_struct *ptr, int size) {
-  *__builtin_counted_by_ref(ptr->array) = size              // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  *__builtin_counted_by_ref(&ptr->array[0]) = size;         // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  *__builtin_counted_by_ref(ptr->pointer) = size;           // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
-  *__builtin_counted_by_ref(&ptr->pointer[0]) = size;       // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  *__builtin_counted_by_ref(ptr->array) = size                      // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  *__builtin_counted_by_ref(&ptr->array[0]) = size;                 // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  *__builtin_counted_by_ref(ptr->pointer) = size;                   // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
+  *__builtin_counted_by_ref(&ptr->pointer[0]) = size;               // expected-error {{'__builtin_counted_by_ref' argument must reference a flexible array member}}
 }
 
 struct char_count {
diff --git a/clang/test/Sema/constant_builtins_vector.cpp b/clang/test/Sema/constant_builtins_vector.cpp
index 772a682141ce4..b2f56e5a87ab1 100644
--- a/clang/test/Sema/constant_builtins_vector.cpp
+++ b/clang/test/Sema/constant_builtins_vector.cpp
@@ -817,3 +817,24 @@ static_assert(__builtin_elementwise_popcount(~0U) == 8 * sizeof(int));
 static_assert(__builtin_elementwise_popcount(0L) == 0);
 static_assert(__builtin_elementwise_popcount(0xF0F0L) == 8);
 static_assert(__builtin_elementwise_popcount(~0LL) == 8 * sizeof(long long));
+
+static_assert(__builtin_elementwise_bitreverse(0x12345678) == 0x1E6A2C48);
+static_assert(__builtin_elementwise_bitreverse(0x0123456789ABCDEFULL) == 0xF7B3D591E6A2C480);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_bitreverse((vector4char){1, 2, 4, 8})) == (LITTLE_END ? 0x10204080 : 0x80402010));
+static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_bitreverse((vector4short){1, 2, 4, 8})) == (LITTLE_END ? 0x1000200040008000 : 0x8000400020001000));
+
+static_assert(__builtin_elementwise_add_sat(1, 2) == 3);
+static_assert(__builtin_elementwise_add_sat(1U, 2U) == 3U);
+static_assert(__builtin_elementwise_add_sat(~(1 << 31), 42) == ~(1 << 31));
+static_assert(__builtin_elementwise_add_sat((1 << 31), -42) == (1 << 31));
+static_assert(__builtin_elementwise_add_sat(~0U, 1U) == ~0U);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_add_sat((vector4char){1, 2, 3, 4}, (vector4char){1, 2, 3, 4})) == (LITTLE_END ? 0x08060402 : 0x02040608));
+static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_add_sat((vector4short){(short)0x8000, (short)0x8001, (short)0x8002, (short)0x8003}, (vector4short){-7, -8, -9, -10}) == (LITTLE_END ? 0x8000800080008000 : 0x8000800080008000)));
+
+static_assert(__builtin_elementwise_sub_sat(1, 2) == -1);
+static_assert(__builtin_elementwise_sub_sat(2U, 1U) == 1U);
+static_assert(__builtin_elementwise_sub_sat(~(1 << 31), -42) == ~(1 << 31));
+static_assert(__builtin_elementwise_sub_sat((1 << 31), 42) == (1 << 31));
+static_assert(__builtin_elementwise_sub_sat(0U, 1U) == 0U);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_sub_sat((vector4char){5, 4, 3, 2}, (vector4char){1, 1, 1, 1})) == (LITTLE_END ? 0x01020304 : 0x04030201));
+static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_sub_sat((vector4short){(short)0x8000, (short)0x8001, (short)0x8002, (short)0x8003}, (vector4short){7, 8, 9, 10}) == (LITTLE_END ? 0x8000800080008000 : 0x8000800080008000)));
diff --git a/clang/test/Sema/enum.c b/clang/test/Sema/enum.c
index 4f6d04ba7f918..3db301dab0a45 100644
--- a/clang/test/Sema/enum.c
+++ b/clang/test/Sema/enum.c
@@ -121,6 +121,21 @@ int NegativeShortTest[NegativeShort == -1 ? 1 : -1];
 enum Color { Red, Green, Blue }; // expected-note{{previous use is here}}
 typedef struct Color NewColor; // expected-error {{use of 'Color' with tag type that does not match previous declaration}}
 
+// Enumerations with a fixed underlying type. 
+// https://github.com/llvm/llvm-project/issues/116880
+#if __STDC_VERSION__ >= 202311L
+  static_assert(__has_feature(c_fixed_enum));
+  static_assert(__has_extension(c_fixed_enum)); // Matches behavior for c_alignas, etc
+#else
+  _Static_assert(__has_extension(c_fixed_enum), "");
+  _Static_assert(!__has_feature(c_fixed_enum), "");
+#if __STDC_VERSION__ < 201112L
+  // expected-warning@-3 {{'_Static_assert' is a C11 extension}}
+  // expected-warning@-3 {{'_Static_assert' is a C11 extension}}
+#endif
+#endif
+typedef enum : unsigned char { Pink, Black, Cyan } Color; // pre-c23-warning {{enumeration types with a fixed underlying type are a C23 extension}}
+
 // PR28903
 // In C it is valid to define tags inside enums.
 struct PR28903 {
diff --git a/clang/test/Sema/warn-lifetime-analysis-capture-by.cpp b/clang/test/Sema/warn-lifetime-analysis-capture-by.cpp
index 4d562bac1e305..12b933e63edd7 100644
--- a/clang/test/Sema/warn-lifetime-analysis-capture-by.cpp
+++ b/clang/test/Sema/warn-lifetime-analysis-capture-by.cpp
@@ -143,6 +143,20 @@ void use() {
 }
 } // namespace this_is_captured
 
+namespace temporary_capturing_object {
+struct S {
+  void add(const int& x [[clang::lifetime_capture_by(this)]]);
+};
+
+void test() {
+  // We still give an warning even the capturing object is a temoprary.
+  // It is possible that the capturing object uses the captured object in its
+  // destructor.
+  S().add(1); // expected-warning {{object whose reference is captured}}
+  S{}.add(1); // expected-warning {{object whose reference is captured}}
+}
+} // namespace ignore_temporary_class_object
+
 // ****************************************************************************
 // Capture by Global and Unknown.
 // ****************************************************************************
@@ -411,3 +425,29 @@ void use() {
 }
 } // namespace with_span
 } // namespace inferred_capture_by
+
+namespace on_constructor {
+struct T {
+  T(const int& t [[clang::lifetime_capture_by(this)]]);
+};
+struct T2 {
+  T2(const int& t [[clang::lifetime_capture_by(x)]], int& x);
+};
+struct T3 {
+  T3(const T& t [[clang::lifetime_capture_by(this)]]);
+};
+
+int foo(const T& t);
+int bar(const T& t[[clang::lifetimebound]]);
+
+void test() {
+  auto x = foo(T(1)); // OK. no diagnosic
+  T(1); // OK. no diagnostic
+  T t(1); // expected-warning {{temporary whose address is used}}
+  auto y = bar(T(1)); // expected-warning {{temporary whose address is used}}
+  T3 t3(T(1)); // expected-warning {{temporary whose address is used}}
+    
+  int a;
+  T2(1, a); // expected-warning {{object whose reference is captured by}}
+}
+} // namespace on_constructor
diff --git a/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp b/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
index c18ecd86ad06f..fc876926ba2e6 100644
--- a/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
+++ b/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
@@ -670,3 +670,26 @@ void test13() {
 }
 
 } // namespace GH100526
+
+namespace std {
+template <typename T>
+class __set_iterator {};
+
+template<typename T>
+struct BB {
+  typedef  __set_iterator<T> iterator;
+};
+
+template <typename T>
+class set {
+public:
+  typedef typename BB<T>::iterator iterator;
+  iterator begin() const;
+};
+} // namespace std
+namespace GH118064{
+
+void test() {
+  auto y = std::set<int>{}.begin(); // expected-warning {{object backing the pointer}}
+}
+} // namespace GH118064
diff --git a/clang/test/Sema/warn-stringcompare.c b/clang/test/Sema/warn-stringcompare.c
index 77876ad6abd6e..44dc0a5de3a4e 100644
--- a/clang/test/Sema/warn-stringcompare.c
+++ b/clang/test/Sema/warn-stringcompare.c
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -x c -fsyntax-only -verify %s
-// RUN: %clang_cc1 -x c++ -fsyntax-only -verify %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify=expected,cxx %s
+// RUN: %clang_cc1 -x c++ -std=c++26 -fsyntax-only -verify=expected,cxx26 %s
 
 #define DELIM "/"
 #define DOT "."
@@ -15,15 +16,15 @@ void test(const char *d) {
   if (NULL == "/")
     return;
   if ("/" != DELIM) // expected-warning {{result of comparison against a string literal is unspecified (use an explicit string comparison function instead)}}
-    return;
+    return;         // cxx-warning@-1 {{comparison between two arrays}} cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
   if (DELIM == "/") // expected-warning {{result of comparison against a string literal is unspecified (use an explicit string comparison function instead)}}
-    return;
+    return;         // cxx-warning@-1 {{comparison between two arrays}} cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
   if (DELIM != NULL)
     return;
   if (NULL == DELIM)
     return;
   if (DOT != DELIM) // expected-warning {{result of comparison against a string literal is unspecified (use an explicit string comparison function instead)}}
-    return;
+    return;         // cxx-warning@-1 {{comparison between two arrays}} cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
   if (DELIM == DOT) // expected-warning {{result of comparison against a string literal is unspecified (use an explicit string comparison function instead)}}
-    return;
+    return;         // cxx-warning@-1 {{comparison between two arrays}} cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
 }
diff --git a/clang/test/SemaCXX/addr-of-overloaded-function.cpp b/clang/test/SemaCXX/addr-of-overloaded-function.cpp
index dd1c3462c8c1f..99f5ba613938d 100644
--- a/clang/test/SemaCXX/addr-of-overloaded-function.cpp
+++ b/clang/test/SemaCXX/addr-of-overloaded-function.cpp
@@ -221,20 +221,20 @@ namespace test1 {
 
   void QualifierTest() {
     void (Qualifiers::*X)();
-    X = &Qualifiers::C; // expected-error-re {{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const': different qualifiers (unqualified vs 'const')}}
-    X = &Qualifiers::V; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} volatile': different qualifiers (unqualified vs 'volatile')}}
-    X = &Qualifiers::R; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} __restrict': different qualifiers (unqualified vs '__restrict')}}
-    X = &Qualifiers::CV; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const volatile': different qualifiers (unqualified vs 'const volatile')}}
-    X = &Qualifiers::CR; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const __restrict': different qualifiers (unqualified vs 'const __restrict')}}
-    X = &Qualifiers::VR; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} volatile __restrict': different qualifiers (unqualified vs 'volatile __restrict')}}
-    X = &Qualifiers::CVR; // expected-error-re{{assigning to 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const volatile __restrict': different qualifiers (unqualified vs 'const volatile __restrict')}}
+    X = &Qualifiers::C; // expected-error-re {{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const': different qualifiers (unqualified vs 'const')}}
+    X = &Qualifiers::V; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} volatile': different qualifiers (unqualified vs 'volatile')}}
+    X = &Qualifiers::R; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} __restrict': different qualifiers (unqualified vs '__restrict')}}
+    X = &Qualifiers::CV; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const volatile': different qualifiers (unqualified vs 'const volatile')}}
+    X = &Qualifiers::CR; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const __restrict': different qualifiers (unqualified vs 'const __restrict')}}
+    X = &Qualifiers::VR; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} volatile __restrict': different qualifiers (unqualified vs 'volatile __restrict')}}
+    X = &Qualifiers::CVR; // expected-error-re{{assigning to 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' from incompatible type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}} const volatile __restrict': different qualifiers (unqualified vs 'const volatile __restrict')}}
   }
 
   struct Dummy {
     void N() {};
   };
 
-  void (Qualifiers::*X)() = &Dummy::N; // expected-error-re{{cannot initialize a variable of type 'void (test1::Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' with an rvalue of type 'void (test1::Dummy::*)(){{( __attribute__\(\(thiscall\)\))?}}': different classes ('test1::Qualifiers' vs 'test1::Dummy')}}
+  void (Qualifiers::*X)() = &Dummy::N; // expected-error-re{{cannot initialize a variable of type 'void (Qualifiers::*)(){{( __attribute__\(\(thiscall\)\))?}}' with an rvalue of type 'void (test1::Dummy::*)(){{( __attribute__\(\(thiscall\)\))?}}': different classes ('Qualifiers' vs 'test1::Dummy')}}
 }
 
 template <typename T> class PR16561 {
diff --git a/clang/test/SemaCXX/attr-no-specializations.cpp b/clang/test/SemaCXX/attr-no-specializations.cpp
new file mode 100644
index 0000000000000..64f6a76416605
--- /dev/null
+++ b/clang/test/SemaCXX/attr-no-specializations.cpp
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 %s -verify
+
+#if !__has_cpp_attribute(clang::no_specializations)
+#  error
+#endif
+
+struct [[clang::no_specializations]] S {}; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+
+template <class T, class U>
+struct [[clang::no_specializations]] is_same { // expected-note 2 {{marked 'no_specializations' here}}
+  static constexpr bool value = __is_same(T, U);
+};
+
+template <class T>
+using alias [[clang::no_specializations]] = T; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+
+template <>
+struct is_same<int, char> {}; // expected-error {{'is_same' cannot be specialized}}
+
+template <class>
+struct Template {};
+
+template <class T>
+struct is_same<Template<T>, Template <T>> {}; // expected-error {{'is_same' cannot be specialized}}
+
+bool test_instantiation1 = is_same<int, int>::value;
+
+template <class T, class U>
+[[clang::no_specializations]] inline constexpr bool is_same_v = __is_same(T, U); // expected-note 2 {{marked 'no_specializations' here}}
+
+template <>
+inline constexpr bool is_same_v<int, char> = false; // expected-error {{'is_same_v' cannot be specialized}}
+
+template <class T>
+inline constexpr bool is_same_v<Template <T>, Template <T>> = true; // expected-error {{'is_same_v' cannot be specialized}}
+
+bool test_instantiation2 = is_same_v<int, int>;
+
+template <class T>
+struct [[clang::no_specializations("specializing type traits results in undefined behaviour")]] is_trivial { // expected-note {{marked 'no_specializations' here}}
+  static constexpr bool value = __is_trivial(T);
+};
+
+template <>
+struct is_trivial<int> {}; // expected-error {{'is_trivial' cannot be specialized: specializing type traits results in undefined behaviour}}
+
+template <class T>
+[[clang::no_specializations("specializing type traits results in undefined behaviour")]] inline constexpr bool is_trivial_v = __is_trivial(T); // expected-note {{marked 'no_specializations' here}}
+
+template <>
+inline constexpr bool is_trivial_v<int> = false; // expected-error {{'is_trivial_v' cannot be specialized: specializing type traits results in undefined behaviour}}
+
+template <class T>
+struct Partial {};
+
+template <class T>
+struct [[clang::no_specializations]] Partial<Template <T>> {}; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+
+template <class T>
+[[clang::no_specializations]] void func(); // expected-note {{marked 'no_specializations' here}}
+
+template <> void func<int>(); // expected-error {{'func' cannot be specialized}}
diff --git a/clang/test/SemaCXX/calling-conv-compat.cpp b/clang/test/SemaCXX/calling-conv-compat.cpp
index 2af944defd4bc..5a51e34b1478c 100644
--- a/clang/test/SemaCXX/calling-conv-compat.cpp
+++ b/clang/test/SemaCXX/calling-conv-compat.cpp
@@ -183,31 +183,31 @@ typedef void (           C::*memb_c_default)();
 typedef void (__cdecl    C::*memb_c_cdecl)();
 typedef void (__thiscall C::*memb_c_thiscall)();
 
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_a_default' (aka 'void (NonVariadic::A::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_a_default' (aka 'void (A::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_a_default(memb_a_default ptr);
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_a_cdecl' (aka 'void (NonVariadic::A::*)() __attribute__((cdecl))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_a_cdecl' (aka 'void (NonVariadic::A::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_a_cdecl' (aka 'void (A::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_a_cdecl' (aka 'void (A::*)() __attribute__((cdecl))') for 1st argument}}
 void cb_memb_a_cdecl(memb_a_cdecl ptr);
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_a_thiscall' (aka 'void (NonVariadic::A::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_a_thiscall' (aka 'void (A::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_a_thiscall(memb_a_thiscall ptr);
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_b_default' (aka 'void (NonVariadic::B::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_b_default' (aka 'void (B::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_b_default(memb_b_default ptr);
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_b_cdecl' (aka 'void (NonVariadic::B::*)() __attribute__((cdecl))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_b_cdecl' (aka 'void (NonVariadic::B::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_b_cdecl' (aka 'void (B::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_b_cdecl' (aka 'void (B::*)() __attribute__((cdecl))') for 1st argument}}
 void cb_memb_b_cdecl(memb_b_cdecl ptr);
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_b_thiscall' (aka 'void (NonVariadic::B::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_b_thiscall' (aka 'void (B::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_b_thiscall(memb_b_thiscall ptr);
-// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_default' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_default' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_default' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_default' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_default' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_default' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_c_default(memb_c_default ptr);
-// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_cdecl' (aka 'void (NonVariadic::C::*)() __attribute__((cdecl))') for 1st argument}}
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_cdecl' (aka 'void (NonVariadic::C::*)() __attribute__((cdecl))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_cdecl' (aka 'void (NonVariadic::C::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_cdecl' (aka 'void (C::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_cdecl' (aka 'void (C::*)() __attribute__((cdecl))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_cdecl' (aka 'void (C::*)() __attribute__((cdecl))') for 1st argument}}
 void cb_memb_c_cdecl(memb_c_cdecl ptr);
-// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_thiscall' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_thiscall' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_thiscall' (aka 'void (NonVariadic::C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+3 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_thiscall' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((cdecl))' to 'memb_c_thiscall' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (NonVariadic::A::*)() __attribute__((thiscall))' to 'memb_c_thiscall' (aka 'void (C::*)() __attribute__((thiscall))') for 1st argument}}
 void cb_memb_c_thiscall(memb_c_thiscall ptr);
 
 void call_member() {
@@ -279,11 +279,11 @@ void cb_memb_a_default(memb_a_default ptr);
 void cb_memb_a_cdecl(memb_a_cdecl ptr);
 void cb_memb_b_default(memb_b_default ptr);
 void cb_memb_b_cdecl(memb_b_cdecl ptr);
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...)' to 'memb_c_default' (aka 'void (Variadic::C::*)(int, ...)') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...) __attribute__((cdecl))' to 'memb_c_default' (aka 'void (Variadic::C::*)(int, ...)') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...)' to 'memb_c_default' (aka 'void (C::*)(int, ...)') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...) __attribute__((cdecl))' to 'memb_c_default' (aka 'void (C::*)(int, ...)') for 1st argument}}
 void cb_memb_c_default(memb_c_default ptr);
-// expected-note@+2 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...)' to 'memb_c_cdecl' (aka 'void (Variadic::C::*)(int, ...) __attribute__((cdecl))') for 1st argument}}
-// expected-note@+1 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...) __attribute__((cdecl))' to 'memb_c_cdecl' (aka 'void (Variadic::C::*)(int, ...) __attribute__((cdecl))') for 1st argument}}
+// expected-note@+2 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...)' to 'memb_c_cdecl' (aka 'void (C::*)(int, ...) __attribute__((cdecl))') for 1st argument}}
+// expected-note@+1 {{candidate function not viable: no known conversion from 'void (Variadic::A::*)(int, ...) __attribute__((cdecl))' to 'memb_c_cdecl' (aka 'void (C::*)(int, ...) __attribute__((cdecl))') for 1st argument}}
 void cb_memb_c_cdecl(memb_c_cdecl ptr);
 
 void call_member() {
@@ -330,7 +330,7 @@ mptr_t __stdcall return_mptr_std(short) {
 void (A::*(*return_fptr_std_mptr(char))(short))(int) {
   return return_mptr_std;
 #if !_M_X64
-  // expected-error@-2 {{cannot initialize return object of type 'void (MultiChunkDecls::A::*(*)(short))(int) __attribute__((thiscall))' with an lvalue of type 'mptr_t (short) __attribute__((stdcall))'}}
+  // expected-error@-2 {{cannot initialize return object of type 'void (A::*(*)(short))(int) __attribute__((thiscall))' with an lvalue of type 'mptr_t (short) __attribute__((stdcall))'}}
 #endif
 }
 
diff --git a/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp b/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp
index 19e7d4976428a..e3228dddef5f6 100644
--- a/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp
+++ b/clang/test/SemaCXX/constexpr-return-non-void-cxx2b.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++23 -fsyntax-only -Wimplicit-fallthrough -verify %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -Wimplicit-fallthrough -Wconsumed -verify %s
 
 constexpr int f() { } // expected-warning {{non-void function does not return a value}}
 static_assert(__is_same(decltype([] constexpr -> int { }( )), int)); // expected-warning {{non-void lambda does not return a value}}
@@ -34,3 +34,10 @@ constinit bool l = j(); // expected-error {{variable does not have a constant in
                         // expected-note {{in call to 'j()'}}
 
 }
+
+namespace GH117385 {
+void f() {
+  if consteval {
+  }
+}
+}
diff --git a/clang/test/SemaCXX/cxx1y-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx1y-initializer-aggregates.cpp
index 03a6800898d18..8360b8fd7d8ee 100644
--- a/clang/test/SemaCXX/cxx1y-initializer-aggregates.cpp
+++ b/clang/test/SemaCXX/cxx1y-initializer-aggregates.cpp
@@ -115,3 +115,14 @@ namespace nested_union {
   // of Test3, or we should exclude f(Test3) as a candidate.
   static_assert(f({1}) == 2, ""); // expected-error {{call to 'f' is ambiguous}}
 }
+
+// Fix crash issue https://github.com/llvm/llvm-project/issues/112560.
+// Make sure clang compiles the following code without crashing:
+namespace GH112560 {
+union U {
+  int f = ; // expected-error {{expected expression}}
+};
+void foo() {
+  U g{};
+}
+} // namespace GH112560
diff --git a/clang/test/SemaCXX/cxx20-warn-dangling-paren-list-agg-init.cpp b/clang/test/SemaCXX/cxx20-warn-dangling-paren-list-agg-init.cpp
new file mode 100644
index 0000000000000..645a7caba714e
--- /dev/null
+++ b/clang/test/SemaCXX/cxx20-warn-dangling-paren-list-agg-init.cpp
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -verify -std=c++20 %s -fsyntax-only
+
+namespace std {
+template <class T> struct remove_reference { typedef T type; };
+template <class T> struct remove_reference<T&> { typedef T type; };
+template <class T> struct remove_reference<T&&> { typedef T type; };
+
+template <class T> typename remove_reference<T>::type &&move(T &&t);
+} // namespace std
+
+// dcl.init 16.6.2.2
+struct A {
+  int a;
+  int&& r;
+};
+
+int f();
+int n = 10;
+
+A a1{1, f()}; // OK, lifetime is extended for direct-list-initialization
+// well-formed, but dangling reference
+A a2(1, f()); // expected-warning {{temporary whose address is used as value}}
+// well-formed, but dangling reference
+A a4(1.0, 1); // expected-warning {{temporary whose address is used as value}}
+A a5(1.0, std::move(n));  // OK
+
+
+
+struct B {
+  const int& r;
+};
+B test(int local) {
+  return B(1); // expected-warning {{returning address}}
+  return B(local); // expected-warning {{address of stack memory}}
+}
+
+void f(B b);
+void test2(int local) {
+  // No diagnostic on the following cases where both the aggregate object and
+  // temporary end at the end of the full expression.
+  f(B(1));
+  f(B(local));
+}
+
+// Test nested struct.
+struct C {
+  B b;
+};
+
+struct D {
+  C c;
+};
+
+C c1(B(
+  1  // expected-warning {{temporary whose address is used as value}}
+)); 
+D d1(C(B(
+  1  // expected-warning {{temporary whose address is used as value}}
+)));
diff --git a/clang/test/SemaCXX/deprecated.cpp b/clang/test/SemaCXX/deprecated.cpp
index 667f4d7d3edb0..4282239af81b4 100644
--- a/clang/test/SemaCXX/deprecated.cpp
+++ b/clang/test/SemaCXX/deprecated.cpp
@@ -246,17 +246,19 @@ namespace ArithConv {
 
 namespace ArrayComp {
   int arr1[3], arr2[4];
-  bool b1 = arr1 == arr2; // expected-warning {{array comparison always evaluates to false}} cxx20-warning {{comparison between two arrays is deprecated}}
-  bool b2 = arr1 < arr2; // expected-warning {{array comparison always evaluates to a constant}} cxx20-warning {{comparison between two arrays is deprecated}}
+  bool b1 = arr1 == arr2; // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+                          // expected-warning@-1 {{array comparison always evaluates to false}} 
+  bool b2 = arr1 < arr2; // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+                         // expected-warning@-1 {{array comparison always evaluates to a constant}}
   __attribute__((weak)) int arr3[3];
-  bool b3 = arr1 == arr3; // cxx20-warning {{comparison between two arrays is deprecated}}
-  bool b4 = arr1 < arr3; // cxx20-warning {{comparison between two arrays is deprecated}}
+  bool b3 = arr1 == arr3; // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+  bool b4 = arr1 < arr3; // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
 #if __cplusplus > 201703L
   bool b5 = arr1 <=> arr2; // cxx20-error {{invalid operands}}
 #endif
 
   int (&f())[3];
-  bool b6 = arr1 == f(); // cxx20-warning {{comparison between two arrays is deprecated}}
+  bool b6 = arr1 == f(); // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
   bool b7 = arr1 == +f();
 }
 
diff --git a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
index a86f301898992..b7a3d77cfc2f4 100644
--- a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
+++ b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
@@ -298,6 +298,22 @@ void __trans_tmp_1() {
 
 }
 
+namespace GH115931 {
+
+struct Range {};
+
+template <Range>
+struct LengthPercentage {};
+
+void reflectSum() {
+  Range resultR;
+  [&] (auto) -> LengthPercentage<resultR> { 
+    return {};
+  }(0);
+}
+
+} // namespace GH115931
+
 namespace GH47400 {
 
 struct Foo {};
@@ -319,3 +335,17 @@ constexpr void foo() {
 }
 
 } // namespace GH47400
+
+namespace GH84961 {
+
+template <typename T> void g(const T &t) {}
+
+template <typename T> void f(const T &t) {
+  [t] { g(t); }();
+}
+
+void h() {
+  f(h);
+}
+
+} // namespace GH84961
diff --git a/clang/test/SemaCXX/literal-type.cpp b/clang/test/SemaCXX/literal-type.cpp
index 88535c169fe01..44b6ba62262fd 100644
--- a/clang/test/SemaCXX/literal-type.cpp
+++ b/clang/test/SemaCXX/literal-type.cpp
@@ -20,6 +20,7 @@ static_assert(__is_literal(VectorExt), "fail");
 //    [...]
 //    -- a class type that has all of the following properties:
 //        -- it has a trivial destructor
+//           [P0784R7 changed the condition to "constexpr destructor" in C++20]
 //        -- every constructor call and full-expression in the
 //           brace-or-equal-initializers for non-static data members (if an) is
 //           a constant expression,
@@ -108,3 +109,70 @@ void test() {
 
 }
 #endif
+
+#if __cplusplus >= 201103L
+namespace GH85550 {
+struct HasDefaultCtorAndNonConstexprDtor {
+  constexpr HasDefaultCtorAndNonConstexprDtor() = default;
+  ~HasDefaultCtorAndNonConstexprDtor() {}
+};
+
+union UnionWithNonLiteralMember {
+  HasDefaultCtorAndNonConstexprDtor x;
+  int y;
+
+  constexpr UnionWithNonLiteralMember() : x{} {}
+};
+#if __cplusplus >= 202002L
+static_assert(__is_literal(UnionWithNonLiteralMember), "fail");
+#else
+static_assert(!__is_literal(UnionWithNonLiteralMember), "fail");
+#endif
+
+union UnionWithNonLiteralMemberExplicitDtor1 {
+  HasDefaultCtorAndNonConstexprDtor x;
+  int y;
+  // expected-note@-2 {{destructor of 'UnionWithNonLiteralMemberExplicitDtor1' is implicitly deleted because variant field 'x' has a non-trivial destructor}}
+
+  constexpr UnionWithNonLiteralMemberExplicitDtor1() : x{} {}
+  ~UnionWithNonLiteralMemberExplicitDtor1() = default; // expected-warning {{explicitly defaulted destructor is implicitly deleted}}
+  // expected-note@-1 {{replace 'default' with 'delete'}}
+};
+#if __cplusplus >= 202002L
+static_assert(__is_literal(UnionWithNonLiteralMemberExplicitDtor1), "fail");
+#else
+static_assert(!__is_literal(UnionWithNonLiteralMemberExplicitDtor1), "fail");
+#endif
+
+union UnionWithNonLiteralMemberExplicitDtor2 {
+  HasDefaultCtorAndNonConstexprDtor x;
+  int y;
+
+  constexpr UnionWithNonLiteralMemberExplicitDtor2() : x{} {}
+  ~UnionWithNonLiteralMemberExplicitDtor2() = delete;
+};
+static_assert(!__is_literal(UnionWithNonLiteralMemberExplicitDtor2), "fail");
+
+#if __cplusplus >= 202002L
+union UnionWithNonLiteralMemberConstexprDtor1 {
+  HasDefaultCtorAndNonConstexprDtor x;
+  int y;
+  // expected-note@-2 {{destructor of 'UnionWithNonLiteralMemberConstexprDtor1' is implicitly deleted because variant field 'x' has a non-trivial destructor}}
+
+  constexpr UnionWithNonLiteralMemberConstexprDtor1() : x{} {}
+  constexpr ~UnionWithNonLiteralMemberConstexprDtor1() = default; // expected-warning {{explicitly defaulted destructor is implicitly deleted}}
+  // expected-note@-1 {{replace 'default' with 'delete'}}
+};
+static_assert(__is_literal(UnionWithNonLiteralMemberConstexprDtor1), "fail");
+
+union UnionWithNonLiteralMemberConstexprDtor2 {
+  HasDefaultCtorAndNonConstexprDtor x;
+  int y;
+
+  constexpr UnionWithNonLiteralMemberConstexprDtor2() : x{} {}
+  constexpr ~UnionWithNonLiteralMemberConstexprDtor2() = delete;
+};
+static_assert(__is_literal(UnionWithNonLiteralMemberConstexprDtor2), "fail");
+#endif
+}
+#endif
diff --git a/clang/test/SemaCXX/ms-constexpr-new.cpp b/clang/test/SemaCXX/ms-constexpr-new.cpp
index 30567740b2ecb..096014be803e7 100644
--- a/clang/test/SemaCXX/ms-constexpr-new.cpp
+++ b/clang/test/SemaCXX/ms-constexpr-new.cpp
@@ -12,5 +12,12 @@ namespace std {
   }
 }
 
-constexpr bool check_construct_at() { int x; return *std::construct_at(&x, 42) == 42; }
-static_assert(check_construct_at());
+constexpr bool check_std_construct_at() { int x; return *std::construct_at(&x, 42) == 42; }
+static_assert(check_std_construct_at());
+
+constexpr int* construct_at(int* p, int v) { [[msvc::constexpr]] return ::new (p) int(v); } // unsupported-error {{constexpr function never produces a constant expression}} \
+                                                                                            // unsupported-warning {{unknown attribute 'constexpr' ignored}} \
+                                                                                            // unsupported-note 2{{this placement new expression is not supported in constant expressions before C++2c}}
+constexpr bool check_construct_at() { int x; return *construct_at(&x, 42) == 42; }          // unsupported-note {{in call to 'construct_at(&x, 42)'}}
+static_assert(check_construct_at());                                                        // unsupported-error {{static assertion expression is not an integral constant expression}}\
+                                                                                            // unsupported-note {{in call to 'check_construct_at()'}}
diff --git a/clang/test/SemaCXX/new-delete.cpp b/clang/test/SemaCXX/new-delete.cpp
index 595bdc689d694..fb4810ad673ad 100644
--- a/clang/test/SemaCXX/new-delete.cpp
+++ b/clang/test/SemaCXX/new-delete.cpp
@@ -540,6 +540,22 @@ namespace PR10504 {
   void f(A *x) { delete x; } // expected-warning {{delete called on 'PR10504::A' that is abstract but has non-virtual destructor}}
 }
 
+#if __cplusplus >= 201103L
+enum GH99278_1 {
+    zero = decltype(delete static_cast<GH99278_1*>(nullptr), 0){}
+    // expected-warning@-1 {{expression with side effects has no effect in an unevaluated context}}
+};
+template <typename = void>
+struct GH99278_2 {
+  union b {};
+  struct c {
+    c() { delete d; }
+    b *d;
+  } f;
+};
+GH99278_2<void> e;
+#endif
+
 struct PlacementArg {};
 inline void *operator new[](size_t, const PlacementArg &) throw () {
   return 0;
diff --git a/clang/test/SemaCXX/noexcept-destroying-delete.cpp b/clang/test/SemaCXX/noexcept-destroying-delete.cpp
new file mode 100644
index 0000000000000..92ccbc1fb3f96
--- /dev/null
+++ b/clang/test/SemaCXX/noexcept-destroying-delete.cpp
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -fcxx-exceptions -Wno-unevaluated-expression -std=c++20 %s
+// expected-no-diagnostics
+
+namespace std {
+  struct destroying_delete_t {
+    explicit destroying_delete_t() = default;
+  };
+
+  inline constexpr destroying_delete_t destroying_delete{};
+}
+
+struct Explicit {
+    ~Explicit() noexcept(false) {}
+
+    void operator delete(Explicit*, std::destroying_delete_t) noexcept {
+    }
+};
+
+Explicit *qn = nullptr;
+// This assertion used to fail, see GH118660
+static_assert(noexcept(delete(qn)));
+
+struct ThrowingDestroyingDelete {
+    ~ThrowingDestroyingDelete() noexcept(false) {}
+
+    void operator delete(ThrowingDestroyingDelete*, std::destroying_delete_t) noexcept(false) {
+    }
+};
+
+ThrowingDestroyingDelete *pn = nullptr;
+// noexcept should return false here because the destroying delete itself is a
+// potentially throwing function.
+static_assert(!noexcept(delete(pn)));
diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp
index cc2a9d88dd4a6..61afba85e1dff 100644
--- a/clang/test/SemaCXX/paren-list-agg-init.cpp
+++ b/clang/test/SemaCXX/paren-list-agg-init.cpp
@@ -116,6 +116,7 @@ void foo(int n) { // expected-note {{declared here}}
   B b2(A(1), {}, 1);
   // beforecxx20-warning@-1 {{aggregate initialization of type 'A' from a parenthesized list of values is a C++20 extension}}
   // beforecxx20-warning@-2 {{aggregate initialization of type 'B' from a parenthesized list of values is a C++20 extension}}
+  // expected-warning@-3 {{temporary whose address is used as value of local variable 'b2' will be destroyed at the end of the full-expression}}
 
   C c(A(1), 1, 2, 3, 4);
   // expected-error@-1 {{array initializer must be an initializer list}}
@@ -262,9 +263,12 @@ struct O {
 
 O o1(0, 0, 0); // no-error
 // beforecxx20-warning@-1 {{aggregate initialization of type 'O' from a parenthesized list of values is a C++20 extension}}
+// expected-warning@-2 {{temporary whose address is used as value of local variable 'o1' will be destroyed at the end of the full-expression}}
+// expected-warning@-3 {{temporary whose address is used as value of local variable 'o1' will be destroyed at the end of the full-expression}}
 
 O o2(0, 0); // no-error
 // beforecxx20-warning@-1 {{aggregate initialization of type 'O' from a parenthesized list of values is a C++20 extension}}
+// expected-warning@-2 {{temporary whose address is used as value of local variable 'o2' will be destroyed at the end of the full-expression}}
 
 O o3(0);
 // expected-error@-1 {{reference member of type 'int &&' uninitialized}}
diff --git a/clang/test/SemaCXX/source_location.cpp b/clang/test/SemaCXX/source_location.cpp
index e08f872c2cd34..59e3c745773ab 100644
--- a/clang/test/SemaCXX/source_location.cpp
+++ b/clang/test/SemaCXX/source_location.cpp
@@ -985,3 +985,21 @@ int h = Var<int>;
 
 
 }
+
+namespace GH119129 {
+struct X{
+  constexpr int foo(std::source_location loc = std::source_location::current()) {
+    return loc.line();
+  }
+};
+static_assert(X{}.foo() == __LINE__);
+static_assert(X{}.
+                foo() == __LINE__);
+static_assert(X{}.
+
+
+                foo() == __LINE__);
+#line 10000
+static_assert(X{}.
+                foo() == 10001);
+}
diff --git a/clang/test/SemaCXX/static-cast.cpp b/clang/test/SemaCXX/static-cast.cpp
index 7cc4bea423d60..4b36dad64a06b 100644
--- a/clang/test/SemaCXX/static-cast.cpp
+++ b/clang/test/SemaCXX/static-cast.cpp
@@ -199,6 +199,6 @@ namespace PR6072 {
     (void)static_cast<void (A::*)()>(&B::f);
     (void)static_cast<void (B::*)()>(&B::f);
     (void)static_cast<void (C::*)()>(&B::f);
-    (void)static_cast<void (D::*)()>(&B::f); // expected-error-re{{address of overloaded function 'f' cannot be static_cast to type 'void (PR6072::D::*)(){{( __attribute__\(\(thiscall\)\))?}}'}}
+    (void)static_cast<void (D::*)()>(&B::f); // expected-error-re{{address of overloaded function 'f' cannot be static_cast to type 'void (D::*)(){{( __attribute__\(\(thiscall\)\))?}}'}}
   }
 }
diff --git a/clang/test/SemaCXX/warn-array-comparion.cpp b/clang/test/SemaCXX/warn-array-comparion.cpp
new file mode 100644
index 0000000000000..2bd4218f3a279
--- /dev/null
+++ b/clang/test/SemaCXX/warn-array-comparion.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s -verify=expected,not-cxx20
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -Wno-deprecated-array-compare -verify %s -verify=expected,not-cxx20
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -Wdeprecated -verify %s -verify=expected,cxx20
+// RUN: %clang_cc1 -std=c++26 -fsyntax-only -Wdeprecated -verify %s -verify=expected,cxx26
+
+typedef struct {
+  char str[16];
+  int id[16];
+} Object;
+
+bool object_equal(const Object &obj1, const Object &obj2) {
+  if (obj1.str != obj2.str) // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+    return false;           // cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
+  if (obj1.id != obj2.id) // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+    return false;         // cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
+  return true;
+}
+
+
+void foo(int (&array1)[2], int (&array2)[2]) {
+  if (array1 == array2) { } // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+                            // cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
+}
diff --git a/clang/test/SemaCXX/warn-memaccess.cpp b/clang/test/SemaCXX/warn-memaccess.cpp
index 070b44891a91a..2e60539b3e48e 100644
--- a/clang/test/SemaCXX/warn-memaccess.cpp
+++ b/clang/test/SemaCXX/warn-memaccess.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -Wnontrivial-memaccess %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -Wnontrivial-memcall %s
 
 extern "C" void *bzero(void *, unsigned);
 extern "C" void *memset(void *, int, unsigned);
diff --git a/clang/test/SemaCXX/warn-self-comparisons.cpp b/clang/test/SemaCXX/warn-self-comparisons.cpp
index 2e8d130bcd5a0..1ea8b971fcd56 100644
--- a/clang/test/SemaCXX/warn-self-comparisons.cpp
+++ b/clang/test/SemaCXX/warn-self-comparisons.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
-// expected-no-diagnostics
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s -verify=expected,not-cxx20
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -Wdeprecated -verify %s -verify=expected,cxx20
+// RUN: %clang_cc1 -std=c++26 -fsyntax-only -Wdeprecated -verify %s -verify=expected,cxx26
 
 void f(int (&array1)[2], int (&array2)[2]) {
-  if (array1 == array2) { } // no warning
+  if (array1 == array2) { } // not-cxx20-warning {{comparison between two arrays compare their addresses}} cxx20-warning {{comparison between two arrays is deprecated}}
+                            // cxx26-error@-1 {{comparison between two arrays is ill-formed in C++26}}
 }
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
index bfc34b55c1f66..724d444638b57 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
@@ -111,6 +111,37 @@ int testFoldExpression(Vs&&... v) {
     return (... + v);  // expected-warning{{function introduces unsafe buffer manipulation}}
 }
 
+struct HoldsUnsafeMembers {
+    HoldsUnsafeMembers()
+        : FromCtor(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{3}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    [[clang::unsafe_buffer_usage]]
+    HoldsUnsafeMembers(int i)
+        : FromCtor(i),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{i}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    HoldsUnsafeMembers(float f)
+        : HoldsUnsafeMembers(0) {}  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    UnsafeMembers FromCtor;
+    UnsafeMembers FromCtor2;
+    UnsafeMembers FromField{3};  // expected-warning 2{{function introduces unsafe buffer manipulation}}
+};
+
+struct SubclassUnsafeMembers : public UnsafeMembers {
+    SubclassUnsafeMembers()
+        : UnsafeMembers(3)  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    [[clang::unsafe_buffer_usage]]
+    SubclassUnsafeMembers(int i)
+        : UnsafeMembers(i)  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+};
+
 // https://github.com/llvm/llvm-project/issues/80482
 void testClassMembers() {
     UnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
@@ -122,4 +153,95 @@ void testClassMembers() {
     UnsafeMembers()();  // expected-warning{{function introduces unsafe buffer manipulation}}
 
     testFoldExpression(UnsafeMembers(), UnsafeMembers());
+
+    HoldsUnsafeMembers();
+    HoldsUnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    SubclassUnsafeMembers();
+    SubclassUnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
+}
+
+// Not an aggregate, so its constructor is not implicit code and will be
+// visited/checked for warnings.
+struct NotCalledHoldsUnsafeMembers {
+    NotCalledHoldsUnsafeMembers()
+        : FromCtor(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+          FromCtor2{3}  // expected-warning{{function introduces unsafe buffer manipulation}}
+    {}
+
+    UnsafeMembers FromCtor;
+    UnsafeMembers FromCtor2;
+    UnsafeMembers FromField{3};  // expected-warning{{function introduces unsafe buffer manipulation}}
+};
+
+// An aggregate, so its constructor is implicit code. Since it's not called, it
+// is never generated.
+struct AggregateUnused {
+    UnsafeMembers f1;
+    // While this field would trigger the warning during initialization, since
+    // it's unused, there's no code generated that does the initialization, so
+    // no warning.
+    UnsafeMembers f2{3};
+};
+
+struct AggregateExplicitlyInitializedSafe {
+    UnsafeMembers f1;
+    // The warning is not fired as the field is always explicltly initialized
+    // elsewhere. This initializer is never used.
+    UnsafeMembers f2{3};
+};
+
+void testAggregateExplicitlyInitializedSafe() {
+    AggregateExplicitlyInitializedSafe A{
+        .f2 = UnsafeMembers(),  // A safe constructor.
+    };
 }
+
+struct AggregateExplicitlyInitializedUnsafe {
+    UnsafeMembers f1;
+    // The warning is not fired as the field is always explicltly initialized
+    // elsewhere. This initializer is never used.
+    UnsafeMembers f2{3};
+};
+
+void testAggregateExplicitlyInitializedUnsafe() {
+    AggregateExplicitlyInitializedUnsafe A{
+        .f2 = UnsafeMembers(3),  // expected-warning{{function introduces unsafe buffer manipulation}}
+    };
+}
+
+struct AggregateViaAggregateInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaAggregateInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaAggregateInit() {
+    AggregateViaAggregateInit A{};
+};
+
+struct AggregateViaValueInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaValueInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaValueInit() {
+    auto A = AggregateViaValueInit();
+};
+
+struct AggregateViaDefaultInit {
+    UnsafeMembers f1;
+    // FIXME: A construction of this class does initialize the field through
+    // this initializer, so it should warn. Ideally it should also point to
+    // where the site of the construction is in testAggregateViaValueInit().
+    UnsafeMembers f2{3};
+};
+
+void testAggregateViaDefaultInit() {
+    AggregateViaDefaultInit A;
+};
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index 638c76c58c025..30b6d4ba9fb90 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -174,3 +174,23 @@ namespace test_flag {
 
   }
 } //namespace test_flag
+
+struct HoldsStdSpanAndInitializedInCtor {
+  char* Ptr;
+  unsigned Size;
+  std::span<char> Span{Ptr, Size};  // no-warning (this code is unreachable)
+
+  HoldsStdSpanAndInitializedInCtor(char* P, unsigned S)
+      : Span(P, S)  // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  {}
+};
+
+struct HoldsStdSpanAndNotInitializedInCtor {
+  char* Ptr;
+  unsigned Size;
+  std::span<char> Span{Ptr, Size}; // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+
+  HoldsStdSpanAndNotInitializedInCtor(char* P, unsigned S)
+      : Ptr(P), Size(S)
+  {}
+};
diff --git a/clang/test/SemaCXX/warn-unused-private-field.cpp b/clang/test/SemaCXX/warn-unused-private-field.cpp
index bf104b1a76a65..0cc6f687f1b35 100644
--- a/clang/test/SemaCXX/warn-unused-private-field.cpp
+++ b/clang/test/SemaCXX/warn-unused-private-field.cpp
@@ -40,6 +40,20 @@ class FriendEqDefaultCompareOutOfClass {
 
 bool operator==(const FriendEqDefaultCompareOutOfClass &, const FriendEqDefaultCompareOutOfClass &) = default;
 
+class HasUnusedField {
+  int unused_; // expected-warning{{private field 'unused_' is not used}}
+};
+
+class FriendEqDefaultCompare {
+  int used;
+  friend auto operator==(FriendEqDefaultCompare, FriendEqDefaultCompare) -> bool = default;
+};
+
+class UnrelatedFriendEqDefaultCompare {
+  friend auto operator==(UnrelatedFriendEqDefaultCompare, UnrelatedFriendEqDefaultCompare) -> bool = default;
+  int operator<=>(const UnrelatedFriendEqDefaultCompare &) const = default;
+};
+
 #endif
 
 class NotFullyDefined {
diff --git a/clang/test/SemaHIP/zero-sized-device-array.hip b/clang/test/SemaHIP/zero-sized-device-array.hip
new file mode 100644
index 0000000000000..8b25ec38fb01a
--- /dev/null
+++ b/clang/test/SemaHIP/zero-sized-device-array.hip
@@ -0,0 +1,38 @@
+// REQUIRES: amdgpu-registered-target
+// RUN:  %clang_cc1 -fsyntax-only -x hip -fcuda-is-device -verify -triple amdgcn %s
+#define __device__ __attribute__((device))
+#define __host__ __attribute__((host))
+#define __global__ __attribute__((global))
+#define __shared__ __attribute__((shared))
+
+typedef float ZEROARR[0];
+
+float global_array[0];
+
+__global__ void global_fun() {
+    extern __shared__ float externArray[];
+    ZEROARR TypeDef; // expected-error {{zero-length arrays are not permitted in HIP device code}}
+    float array[0];  // expected-error {{zero-length arrays are not permitted in HIP device code}}
+}
+
+// should not throw error for host side code.
+__host__ void host_fun() {
+    float array[0];
+}
+
+template <typename Ty, unsigned Size>
+__device__ void templated()
+{
+   Ty arr[Size];  // expected-error {{zero-length arrays are not permitted in HIP device code}}
+}
+
+__host__ __device__ void host_dev_fun()
+{
+    float array[0]; // expected-error {{zero-length arrays are not permitted in HIP device code}}
+}
+
+__device__ void device_fun()
+{
+    __shared__ float array[0]; // expected-error {{zero-length arrays are not permitted in HIP device code}}
+    templated<int,0>(); // expected-note {{in instantiation of function template specialization 'templated<int, 0U>' requested here}} 
+}
diff --git a/clang/test/SemaHLSL/ArrayTemporary.hlsl b/clang/test/SemaHLSL/ArrayTemporary.hlsl
index dff9aff7d9b29..0266a198e7ec9 100644
--- a/clang/test/SemaHLSL/ArrayTemporary.hlsl
+++ b/clang/test/SemaHLSL/ArrayTemporary.hlsl
@@ -75,17 +75,17 @@ void template_fn(T Val) {}
 // CHECK: CallExpr {{.*}} 'void'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[2])' <FunctionToPointerDecay>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[2])' lvalue Function {{.*}} 'template_fn' 'void (float[2])' (FunctionTemplate {{.*}} 'template_fn')
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[2]' <LValueToRValue>
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[2]' <HLSLArrayRValue>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'float[2]' lvalue ParmVar {{.*}} 'FA2' 'float[2]'
 // CHECK-NEXT: CallExpr {{.*}} 'void'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float[4])' <FunctionToPointerDecay>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float[4])' lvalue Function {{.*}} 'template_fn' 'void (float[4])' (FunctionTemplate {{.*}} 'template_fn')
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[4]' <LValueToRValue>
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float[4]' <HLSLArrayRValue>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'float[4]' lvalue ParmVar {{.*}} 'FA4' 'float[4]'
 // CHECK-NEXT: CallExpr {{.*}} 'void'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(int[3])' <FunctionToPointerDecay>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'void (int[3])' lvalue Function {{.*}} 'template_fn' 'void (int[3])' (FunctionTemplate {{.*}} 'template_fn')
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int[3]' <LValueToRValue>
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int[3]' <HLSLArrayRValue>
 // CHECK-NEXT: DeclRefExpr {{.*}} 'int[3]' lvalue ParmVar {{.*}} 'IA3' 'int[3]'
 
 void call(float FA2[2], float FA4[4], int IA3[3]) {
diff --git a/clang/test/SemaHLSL/BuiltIns/GroupMemoryBarrierWithGroupSync-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/GroupMemoryBarrierWithGroupSync-errors.hlsl
new file mode 100644
index 0000000000000..24067fbb275b7
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/GroupMemoryBarrierWithGroupSync-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+void test_too_many_arg() {
+  __builtin_hlsl_group_memory_barrier_with_group_sync(0);
+  // expected-error@-1 {{too many arguments to function call, expected 0, have 1}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl
new file mode 100644
index 0000000000000..0f2064fe7e45f
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+// RWBuffer<int>
+using handle_t = __hlsl_resource_t
+    [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]];
+
+void test_args(unsigned int x) {
+  // expected-error@+1 {{too few arguments to function call, expected 2, have 1}}
+  __builtin_hlsl_resource_getpointer(x);
+
+  // expected-error@+1 {{too many arguments to function call, expected 2, have 3}}
+  __builtin_hlsl_resource_getpointer(x, x, x);
+
+  // expected-error@+1 {{used type 'unsigned int' where __hlsl_resource_t is required}}
+  __builtin_hlsl_resource_getpointer(x, x);
+
+  handle_t res;
+
+  // expected-error@+1 {{passing 'const char *' to parameter of incompatible type 'unsigned int'}}
+  __builtin_hlsl_resource_getpointer(res, "1");
+
+  // no error
+  __builtin_hlsl_resource_getpointer(res, 0u);
+
+  // no error
+  __builtin_hlsl_resource_getpointer(res, x);
+}
diff --git a/clang/test/SemaHLSL/Language/ArrayOutputArgs-errors.hlsl b/clang/test/SemaHLSL/Language/ArrayOutputArgs-errors.hlsl
new file mode 100644
index 0000000000000..46bed0d5a7cbd
--- /dev/null
+++ b/clang/test/SemaHLSL/Language/ArrayOutputArgs-errors.hlsl
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -verify
+
+void increment(inout int Arr[2]) {
+  for (int I = 0; I < 2; I++)
+    Arr[0] += 2;
+}
+
+export int wrongSize() {
+  int A[3] = { 0, 1, 2 };
+  increment(A);
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: no known conversion from 'int[3]' to 'int[2]' for 1st argument}}
+  return A[0];
+}
+
+export int wrongSize2() {
+  int A[1] = { 0 };
+  increment(A);
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: no known conversion from 'int[1]' to 'int[2]' for 1st argument}}
+  return A[0];
+}
+
+export void tooFewArgs() {
+  increment();
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: requires single argument 'Arr', but no arguments were provided}}
+}
+
+export float wrongType() {
+  float A[2] = { 0, 1 };
+  increment(A);
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: no known conversion from 'float[2]' to 'int[2]' for 1st argument}}
+  return A[0];
+}
+
+export int wrongType2() {
+  increment(5);
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: no known conversion from 'int' to 'int[2]' for 1st argument}}
+  return 1;
+}
+
+export void tooManyArgs() {
+  int A[2] = { 0, 1 };
+  int B[2] = { 2, 3 };
+  increment(A, B);
+  // expected-error@-1 {{no matching function for call to 'increment'}}
+  // expected-note@*:* {{candidate function not viable: requires single argument 'Arr', but 2 arguments were provided}}
+}
diff --git a/clang/test/SemaHLSL/Semantics/entry_parameter.hlsl b/clang/test/SemaHLSL/Semantics/entry_parameter.hlsl
index 13c07038d2e4a..393d7300605c0 100644
--- a/clang/test/SemaHLSL/Semantics/entry_parameter.hlsl
+++ b/clang/test/SemaHLSL/Semantics/entry_parameter.hlsl
@@ -1,16 +1,14 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -hlsl-entry CSMain -x hlsl  -finclude-default-header  -ast-dump -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-mesh -hlsl-entry CSMain -x hlsl -finclude-default-header  -verify -o - %s
 
 [numthreads(8,8,1)]
-// expected-error@+3 {{attribute 'SV_GroupIndex' is unsupported in 'mesh' shaders, requires compute}}
-// expected-error@+2 {{attribute 'SV_DispatchThreadID' is unsupported in 'mesh' shaders, requires compute}}
-// expected-error@+1 {{attribute 'SV_GroupID' is unsupported in 'mesh' shaders, requires compute}}
-void CSMain(int GI : SV_GroupIndex, uint ID : SV_DispatchThreadID, uint GID : SV_GroupID) {
-// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain 'void (int, uint, uint)'
+void CSMain(int GI : SV_GroupIndex, uint ID : SV_DispatchThreadID, uint GID : SV_GroupID, uint GThreadID : SV_GroupThreadID) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain 'void (int, uint, uint, uint)'
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:17 GI 'int'
 // CHECK-NEXT: HLSLSV_GroupIndexAttr
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:42 ID 'uint'
 // CHECK-NEXT: HLSLSV_DispatchThreadIDAttr
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:73 GID 'uint'
 // CHECK-NEXT: HLSLSV_GroupIDAttr
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:96 GThreadID 'uint'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
 }
diff --git a/clang/test/SemaHLSL/Semantics/invalid_entry_parameter.hlsl b/clang/test/SemaHLSL/Semantics/invalid_entry_parameter.hlsl
index 4e1f88aa2294b..1bb4ee5182d62 100644
--- a/clang/test/SemaHLSL/Semantics/invalid_entry_parameter.hlsl
+++ b/clang/test/SemaHLSL/Semantics/invalid_entry_parameter.hlsl
@@ -49,3 +49,33 @@ struct ST2_GID {
     static uint GID : SV_GroupID;
     uint s_gid : SV_GroupID;
 };
+
+[numthreads(8,8,1)]
+// expected-error@+1 {{attribute 'SV_GroupThreadID' only applies to a field or parameter of type 'uint/uint2/uint3'}}
+void CSMain_GThreadID(float ID : SV_GroupThreadID) {
+}
+
+[numthreads(8,8,1)]
+// expected-error@+1 {{attribute 'SV_GroupThreadID' only applies to a field or parameter of type 'uint/uint2/uint3'}}
+void CSMain2_GThreadID(ST GID : SV_GroupThreadID) {
+
+}
+
+void foo_GThreadID() {
+// expected-warning@+1 {{'SV_GroupThreadID' attribute only applies to parameters and non-static data members}}
+  uint GThreadIS : SV_GroupThreadID;
+}
+
+struct ST2_GThreadID {
+// expected-warning@+1 {{'SV_GroupThreadID' attribute only applies to parameters and non-static data members}}
+    static uint GThreadID : SV_GroupThreadID;
+    uint s_gthreadid : SV_GroupThreadID;
+};
+
+
+[shader("vertex")]
+// expected-error@+4 {{attribute 'SV_GroupIndex' is unsupported in 'vertex' shaders, requires compute}}
+// expected-error@+3 {{attribute 'SV_DispatchThreadID' is unsupported in 'vertex' shaders, requires compute}}
+// expected-error@+2 {{attribute 'SV_GroupID' is unsupported in 'vertex' shaders, requires compute}}
+// expected-error@+1 {{attribute 'SV_GroupThreadID' is unsupported in 'vertex' shaders, requires compute}}
+void vs_main(int GI : SV_GroupIndex, uint ID : SV_DispatchThreadID, uint GID : SV_GroupID, uint GThreadID : SV_GroupThreadID) {}
diff --git a/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl b/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
index 10a5e5dabac87..6781f9241df24 100644
--- a/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
+++ b/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
@@ -49,3 +49,28 @@ void CSMain3_GID(uint3 : SV_GroupID) {
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:24 'uint3'
 // CHECK-NEXT: HLSLSV_GroupIDAttr
 }
+
+[numthreads(8,8,1)]
+void CSMain_GThreadID(uint ID : SV_GroupThreadID) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain_GThreadID 'void (uint)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:28 ID 'uint'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
+}
+[numthreads(8,8,1)]
+void CSMain1_GThreadID(uint2 ID : SV_GroupThreadID) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain1_GThreadID 'void (uint2)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:30 ID 'uint2'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
+}
+[numthreads(8,8,1)]
+void CSMain2_GThreadID(uint3 ID : SV_GroupThreadID) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain2_GThreadID 'void (uint3)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:30 ID 'uint3'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
+}
+[numthreads(8,8,1)]
+void CSMain3_GThreadID(uint3 : SV_GroupThreadID) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain3_GThreadID 'void (uint3)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:30 'uint3'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-attach-ast.cpp b/clang/test/SemaOpenACC/combined-construct-attach-ast.cpp
new file mode 100644
index 0000000000000..c0b8d1b2eab52
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-attach-ast.cpp
@@ -0,0 +1,66 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop attach(PointerParam) deviceptr(PointerParam)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: attach clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}} 'PointerParam' 'float *'
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}} 'PointerParam' 'float *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T>
+void TemplUses(T *PointerParam) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 T
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T *)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced PointerParam 'T *'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop attach(PointerParam) deviceptr(PointerParam)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: attach clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T *' lvalue ParmVar{{.*}} 'PointerParam' 'T *'
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T *' lvalue ParmVar{{.*}} 'PointerParam' 'T *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used PointerParam 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+//#pragma acc parallel loop attach(PointerParam) deviceptr(PointerParam)
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: attach clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'PointerParam' 'int *'
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'PointerParam' 'int *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+}
+
+void Inst() {
+  int i;
+  TemplUses(&i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-attach-clause.c b/clang/test/SemaOpenACC/combined-construct-attach-clause.c
new file mode 100644
index 0000000000000..49a64a48196ba
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-attach-clause.c
@@ -0,0 +1,65 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S {
+  int IntMem;
+  int *PtrMem;
+};
+
+void uses() {
+  int LocalInt;
+  int *LocalPtr;
+  int Array[5];
+  int *PtrArray[5];
+  struct S s;
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop attach(&LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc serial loop attach(LocalPtr)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int[5]'}}
+#pragma acc kernels loop attach(Array)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(Array[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(Array[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int *[5]'}}
+#pragma acc parallel loop attach(PtrArray)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(PtrArray[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(PtrArray[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'struct S'}}
+#pragma acc parallel loop attach(s)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(s.IntMem)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(s.PtrMem)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'attach' clause is not valid on 'loop' directive}}
+#pragma acc loop attach(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-attach-clause.cpp b/clang/test/SemaOpenACC/combined-construct-attach-clause.cpp
new file mode 100644
index 0000000000000..dfbd01f72eeee
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-attach-clause.cpp
@@ -0,0 +1,120 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S {
+  int IntMem;
+  int *PtrMem;
+  operator int*();
+};
+
+void uses() {
+  int LocalInt;
+  int *LocalPtr;
+  int Array[5];
+  int *PtrArray[5];
+  struct S s;
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(LocalPtr)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int[5]'}}
+#pragma acc parallel loop attach(Array)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(Array[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(Array[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int *[5]'}}
+#pragma acc parallel loop attach(PtrArray)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(PtrArray[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(PtrArray[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'struct S'}}
+#pragma acc parallel loop attach(s)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(s.IntMem)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(s.PtrMem)
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+template<typename T, typename TPtr, typename TStruct, auto &R1>
+void Templ() {
+  T SomeInt;
+  TPtr SomePtr;
+  T SomeIntArray[5];
+  TPtr SomeIntPtrArray[5];
+  TStruct SomeStruct;
+
+  // expected-error@+2{{expected pointer in 'attach' clause, type is 'int'}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+#pragma acc parallel loop attach(SomeInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(SomePtr)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int[5]'}}
+#pragma acc parallel loop attach(SomeIntArray)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(SomeIntArray[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(SomeIntArray[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int *[5]'}}
+#pragma acc parallel loop attach(SomeIntPtrArray)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(SomeIntPtrArray[0])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop attach(SomeIntPtrArray[0:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'S'}}
+#pragma acc parallel loop attach(SomeStruct)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(SomeStruct.IntMem)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop attach(SomeStruct.PtrMem)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'attach' clause, type is 'int'}}
+#pragma acc parallel loop attach(R1)
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+void inst() {
+  static constexpr int CEVar = 1;
+  Templ<int, int*, S, CEVar>(); // #INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
index 69f93a6c60515..16bdcc177bfde 100644
--- a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
@@ -1,9 +1,5 @@
-
 // RUN: %clang_cc1 %s -fopenacc -verify
 
-// TODO: OpenACC: A number of the 'not yet implemented' diagnostics interfere
-// with the diagnostics we want to make here, so as we implement these, we need
-// to replace the errors we should have.
 void uses() {
 #pragma acc parallel loop auto
   for(unsigned i = 0; i < 5; ++i);
@@ -47,10 +43,8 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc parallel loop auto if_present
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc parallel loop auto worker
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
 #pragma acc parallel loop auto vector
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'nohost' not yet implemented}}
@@ -62,26 +56,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto self
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop auto copy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop auto pcopy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop auto present_or_copy(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop auto use_device(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop auto attach(Var)
+#pragma acc parallel loop auto attach(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop auto delete(Var)
@@ -92,8 +78,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop auto device(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop auto deviceptr(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -107,75 +91,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop auto link(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop auto no_create(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto present(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto private(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop auto copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop auto pcopyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop auto present_or_copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop auto copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop auto pcopyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop auto present_or_copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop auto create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop auto pcreate(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop auto present_or_create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop auto reduction(+:Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop auto collapse(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop auto bind(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop auto vector_length(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop auto num_gangs(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop auto num_workers(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -190,15 +146,11 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto async
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop auto tile(1+2, 1)
   for(unsigned j = 0; j < 5; ++j)
     for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc parallel loop auto gang
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop auto wait
   for(unsigned i = 0; i < 5; ++i);
 
@@ -208,10 +160,8 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc parallel loop if_present auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc parallel loop worker auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
 #pragma acc parallel loop vector auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'nohost' not yet implemented}}
@@ -223,26 +173,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop self auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop copy(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop pcopy(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop present_or_copy(Var) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop use_device(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop attach(Var) auto
+#pragma acc parallel loop attach(VarPtr) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop delete(Var) auto
@@ -253,8 +195,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop device(VarPtr) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop deviceptr(VarPtr) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -268,75 +208,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop link(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop no_create(Var) auto
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop present(Var) auto
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop private(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop copyout(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop pcopyout(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop present_or_copyout(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop copyin(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop pcopyin(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop present_or_copyin(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop create(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop pcreate(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop present_or_create(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop reduction(+:Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop collapse(1) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop bind(Var) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop vector_length(1) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop num_gangs(1) auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop num_workers(1) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -351,15 +263,11 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop async auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop tile(1+2, 1) auto
   for(unsigned j = 0; j < 5; ++j)
     for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc parallel loop gang auto
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop wait auto
   for(unsigned i = 0; i < 5; ++i);
 
@@ -370,10 +278,8 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc parallel loop independent if_present
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc parallel loop independent worker
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
 #pragma acc parallel loop independent vector
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'nohost' not yet implemented}}
@@ -385,26 +291,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent self
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop independent copy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop independent pcopy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop independent present_or_copy(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop independent use_device(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop independent attach(Var)
+#pragma acc parallel loop independent attach(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop independent delete(Var)
@@ -415,8 +313,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop independent device(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop independent deviceptr(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -430,75 +326,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop independent link(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop independent no_create(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent present(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent private(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop independent copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop independent pcopyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop independent present_or_copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop independent copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop independent pcopyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop independent present_or_copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop independent create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop independent pcreate(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop independent present_or_create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop independent reduction(+:Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop independent collapse(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop independent bind(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop independent vector_length(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop independent num_gangs(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop independent num_workers(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -513,15 +381,11 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent async
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop independent tile(1+2, 1)
   for(unsigned j = 0; j < 5; ++j)
     for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc parallel loop independent gang
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop independent wait
   for(unsigned i = 0; i < 5; ++i);
 
@@ -531,10 +395,8 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc parallel loop if_present independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc parallel loop worker independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
 #pragma acc parallel loop vector independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'nohost' not yet implemented}}
@@ -546,26 +408,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop self independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop copy(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop pcopy(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop present_or_copy(Var) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop use_device(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop attach(Var) independent
+#pragma acc parallel loop attach(VarPtr) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop delete(Var) independent
@@ -576,8 +430,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop device(VarPtr) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop deviceptr(VarPtr) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -591,75 +443,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop link(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop no_create(Var) independent
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop present(Var) independent
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop private(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop copyout(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TDOOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop pcopyout(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop present_or_copyout(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TDOOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop copyin(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop pcopyin(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop present_or_copyin(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop create(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop pcreate(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop present_or_create(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop reduction(+:Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop collapse(1) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop bind(Var) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop vector_length(1) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop num_gangs(1) independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop num_workers(1) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -674,15 +498,11 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop async independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop tile(1+2, 1) independent
   for(unsigned j = 0; j < 5; ++j)
     for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc parallel loop gang independent
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop wait independent
   for(unsigned i = 0; i < 5; ++i);
 
@@ -714,26 +534,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq self
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop seq copy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop seq pcopy(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop seq present_or_copy(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop seq use_device(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop seq attach(Var)
+#pragma acc parallel loop seq attach(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop seq delete(Var)
@@ -744,8 +556,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop seq device(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop seq deviceptr(VarPtr)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -759,75 +569,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop seq link(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop seq no_create(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq present(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq private(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop seq copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop seq pcopyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop seq present_or_copyout(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop seq copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop seq pcopyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop seq present_or_copyin(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop seq create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop seq pcreate(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop seq present_or_create(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop seq reduction(+:Var)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop seq collapse(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop seq bind(Var)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop seq vector_length(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop seq num_gangs(1)
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop seq num_workers(1)
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -842,28 +624,22 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq async
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop seq tile(1+2, 1)
   for(;;)
     for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop seq wait
   for(unsigned i = 0; i < 5; ++i);
 
-  // TODOexpected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'parallel loop' construct}}
-  // TODOexpected-note@+1{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'parallel loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop gang seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'worker' clause on a 'parallel loop' construct}}
-  // TODOexpected-note@+1{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'worker' clause on a 'parallel loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop worker seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'vector' clause on a 'parallel loop' construct}}
-  // TODOexpected-note@+1{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'vector' clause on a 'parallel loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop vector seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'finalize' not yet implemented}}
@@ -881,26 +657,18 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop self seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copy' not yet implemented}}
 #pragma acc parallel loop copy(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'pcopy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop pcopy(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copy' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
 #pragma acc parallel loop present_or_copy(Var) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'use_device' not yet implemented}}
 #pragma acc parallel loop use_device(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'attach' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'attach' not yet implemented}}
-#pragma acc parallel loop attach(Var) seq
+#pragma acc parallel loop attach(VarPtr) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'delete' not yet implemented}}
 #pragma acc parallel loop delete(Var) seq
@@ -911,8 +679,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'device' not yet implemented}}
 #pragma acc parallel loop device(VarPtr) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'deviceptr' not yet implemented}}
 #pragma acc parallel loop deviceptr(VarPtr) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_resident' not yet implemented}}
@@ -926,75 +692,47 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'link' not yet implemented}}
 #pragma acc parallel loop link(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'no_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'no_create' not yet implemented}}
 #pragma acc parallel loop no_create(Var) seq
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop present(Var) seq
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop private(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyout' not yet implemented}}
 #pragma acc parallel loop copyout(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop pcopyout(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyout' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
 #pragma acc parallel loop present_or_copyout(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'copyin' not yet implemented}}
 #pragma acc parallel loop copyin(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'pcopyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop pcopyin(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_copyin' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
 #pragma acc parallel loop present_or_copyin(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'create' not yet implemented}}
 #pragma acc parallel loop create(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'pcreate' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'pcreate' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop pcreate(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+2{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
-  // expected-warning@+1{{OpenACC clause 'present_or_create' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
 #pragma acc parallel loop present_or_create(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented}}
 #pragma acc parallel loop reduction(+:Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented}}
 #pragma acc parallel loop collapse(1) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
 #pragma acc parallel loop bind(Var) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'vector_length' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'vector_length' not yet implemented}}
 #pragma acc parallel loop vector_length(1) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_gangs' not yet implemented}}
 #pragma acc parallel loop num_gangs(1) seq
   for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'num_workers' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented}}
 #pragma acc parallel loop num_workers(1) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-warning@+1{{OpenACC clause 'device_num' not yet implemented}}
@@ -1009,12 +747,9 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop async seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
 #pragma acc parallel loop tile(1+2, 1) seq
   for(;;)
     for(unsigned i = 0; i < 5; ++i);
-  // TODOexpected-error@+1{{OpenACC 'wait' clause is not valid on 'parallel loop' directive}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented}}
 #pragma acc parallel loop wait seq
   for(unsigned i = 0; i < 5; ++i);
 }
diff --git a/clang/test/SemaOpenACC/combined-construct-collapse-ast.cpp b/clang/test/SemaOpenACC/combined-construct-collapse-ast.cpp
new file mode 100644
index 0000000000000..89ba7cb24814f
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-collapse-ast.cpp
@@ -0,0 +1,219 @@
+
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+struct S {
+  constexpr S(){};
+  constexpr operator auto() {return 1;}
+};
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop collapse(1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: ConstantExpr{{.*}}'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: NullStmt
+
+#pragma acc serial loop collapse(force:S{})
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: ConstantExpr{{.*}}'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}}'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}}'S' 'void ()' list
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: NullStmt
+}
+
+template<typename T, unsigned Value>
+void TemplUses() {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop collapse(Value)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'unsigned int' NonTypeTemplateParm{{.*}} 'Value'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: NullStmt
+
+#pragma acc kernels loop collapse(force:T{} + S{})
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: BinaryOperator {{.*}}'+'
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}}'T' 'T' list
+  // CHECK-NEXT: InitListExpr
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}}'S' 'void ()' list
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: NullStmt
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'S'
+  // CHECK-NEXT: RecordType{{.*}} 'S'
+  // CHECK-NEXT: CXXRecord{{.*}} 'S'
+  // CHECK-NEXT: TemplateArgument integral '2U'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: ConstantExpr{{.*}}'unsigned int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral {{.*}} 'unsigned int' 2
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: collapse clause
+  // CHECK-NEXT: ConstantExpr{{.*}}'int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: BinaryOperator {{.*}}'+'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}}'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}}'S' 'void ()' list
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}}'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}}'S' 'void ()' list
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: NullStmt
+
+}
+
+void Inst() {
+  TemplUses<S, 2>();
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp b/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp
new file mode 100644
index 0000000000000..c7db9669a9879
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp
@@ -0,0 +1,390 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<unsigned Val>
+void depth_too_high_templ() {
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(Val)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
+constexpr int three() { return 3; }
+struct ConvertsThree{
+  constexpr ConvertsThree(){};
+
+  constexpr operator int(){ return 3; }
+};
+
+void depth_too_high() {
+  depth_too_high_templ<3>(); // expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc serial loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(three())
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(ConvertsThree{})
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
+
+template<typename T, unsigned Three>
+void not_single_loop_templ() {
+  T Arr[5];
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(3)
+  for(auto x : Arr) {
+    for(auto y : Arr){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'parallel loop' with a 'collapse' clause}}
+    }
+  }
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc serial loop collapse(Three)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'serial loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc kernels loop collapse(Three)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k) {
+        do{}while(true);
+      }
+    }
+  }
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(Three)
+  for(auto x : Arr) {
+    for(auto y: Arr) {
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'parallel loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc serial loop collapse(Three)
+  for(auto x : Arr) {
+    for(auto y: Arr) {
+      for(auto z: Arr) {
+        do{}while(true);
+      }
+    }
+  }
+}
+
+void not_single_loop() {
+  not_single_loop_templ<int, 3>(); // expected-note{{in instantiation of function template}}
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k);
+    }
+    while(true); // expected-error{{while loop cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k);
+    }
+    do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'parallel loop' with a 'collapse' clause}}
+  }
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc serial loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      while(true); // expected-error{{while loop cannot appear in intervening code of a 'serial loop' with a 'collapse' clause}}
+    }
+  }
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      do{}while(true);
+    }
+  }
+#pragma acc serial loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      while(true);
+    }
+  }
+
+  int Arr[5];
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(3)
+  for(auto x : Arr) {
+    for(auto y : Arr){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
+    }
+  }
+
+  // expected-note@+1 {{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k);
+    }
+    // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'parallel loop' construct with a 'collapse' clause}}
+      for(unsigned k = 0; k < 5;++k);
+  }
+
+  // expected-note@+1 {{active 'collapse' clause defined here}}
+#pragma acc serial loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k);
+    // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'serial loop' construct with a 'collapse' clause}}
+      for(unsigned k = 0; k < 5;++k);
+    }
+  }
+
+  for(unsigned k = 0; k < 5;++k);
+#pragma acc kernels loop collapse(3)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+      for(unsigned k = 0; k < 5;++k);
+    }
+  }
+}
+
+template<unsigned Two, unsigned Three>
+void no_other_directives() {
+#pragma acc parallel loop collapse(Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {// last loop associated with the top level.
+    // expected-error@+1{{'collapse' clause specifies a loop count greater than the number of available loops}}
+#pragma acc serial loop collapse(Three) // expected-note 2{{active 'collapse' clause defined here}}
+      for(unsigned k = 0; k < 6;++k) {
+        for(unsigned l = 0; l < 5; ++l) {
+    // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'serial loop' with a 'collapse' clause}}
+#pragma acc serial
+          ;
+        }
+      }
+    }
+  }
+#pragma acc kernels loop collapse(Two)// expected-note{{active 'collapse' clause defined here}}
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {// last loop associated with the top level.
+#pragma acc parallel loop collapse(Three)
+      for(unsigned k = 0; k < 6;++k) {
+        for(unsigned l = 0; l < 5; ++l) {
+          for(unsigned m = 0; m < 5; ++m);
+        }
+      }
+    }
+    // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
+#pragma acc serial
+          ;
+  }
+}
+
+void no_other_directives() {
+  no_other_directives<2,3>(); // expected-note{{in instantiation of function template specialization}}
+
+  // Ok, not inside the intervening list
+#pragma acc serial loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+#pragma acc data // expected-warning{{OpenACC construct 'data' not yet implemented}}
+    }
+  }
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    // expected-error@+1{{OpenACC 'data' construct cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
+#pragma acc data // expected-warning{{OpenACC construct 'data' not yet implemented}}
+    for(unsigned j = 0; j < 5; ++j) {
+    }
+  }
+}
+
+void call();
+
+template<unsigned Two>
+void intervening_without_force_templ() {
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'parallel loop' construct}}
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc serial loop collapse(Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'serial loop' construct}}
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc kernels loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'kernels loop' construct}}
+    call();
+  }
+
+#pragma acc parallel loop collapse(force:2)
+  for(unsigned i = 0; i < 5; ++i) {
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+#pragma acc parallel loop collapse(force:Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+
+#pragma acc parallel loop collapse(force:2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    call();
+  }
+
+#pragma acc parallel loop collapse(force:Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    call();
+  }
+
+#pragma acc parallel loop collapse(Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+    call();
+    }
+  }
+
+#pragma acc parallel loop collapse(Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    {
+      {
+        for(unsigned j = 0; j < 5; ++j) {
+          call();
+        }
+      }
+    }
+  }
+
+#pragma acc parallel loop collapse(force:Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+    call();
+    }
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(Two)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'parallel loop' construct}}
+    call();
+  }
+
+#pragma acc parallel loop collapse(2)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-2{{'parallel loop' construct is here}}
+  for(int i = 0;;++i)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-5{{'parallel loop' construct is here}}
+    for(int j = 0;;++j)
+      for(;;);
+}
+
+void intervening_without_force() {
+  intervening_without_force_templ<2>(); // expected-note{{in instantiation of function template specialization}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'parallel loop' construct}}
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'parallel loop' construct}}
+    call();
+  }
+
+  // The below two are fine, as they use the 'force' tag.
+#pragma acc parallel loop collapse(force:2)
+  for(unsigned i = 0; i < 5; ++i) {
+    call();
+    for(unsigned j = 0; j < 5; ++j);
+  }
+
+#pragma acc parallel loop collapse(force:2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j);
+    call();
+  }
+
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+    call();
+    }
+  }
+#pragma acc parallel loop collapse(2)
+  for(unsigned i = 0; i < 5; ++i) {
+    {
+      {
+        for(unsigned j = 0; j < 5; ++j) {
+          call();
+        }
+      }
+    }
+  }
+
+#pragma acc parallel loop collapse(force:2)
+  for(unsigned i = 0; i < 5; ++i) {
+    for(unsigned j = 0; j < 5; ++j) {
+    call();
+    }
+  }
+
+#pragma acc parallel loop collapse(2)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-2{{'parallel loop' construct is here}}
+  for(int i = 0;;++i)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-5{{'parallel loop' construct is here}}
+    for(int j = 0;;++j)
+      for(;;);
+}
+
diff --git a/clang/test/SemaOpenACC/combined-construct-copy-ast.cpp b/clang/test/SemaOpenACC/combined-construct-copy-ast.cpp
new file mode 100644
index 0000000000000..a696badb0847c
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copy-ast.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copy(GlobalArray) pcopy(PointerParam[Global]) present_or_copy(Global)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copy clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
+  // CHECK-NEXT: pcopy clause
+  // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: present_or_copy clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK:NullStmt
+}
+
+template<auto &NTTP, typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: NonTypeTemplateParmDecl {{.*}}referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copy(t) pcopy(NTTP, u) present_or_copy(u[0:t])
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copy clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: pcopy clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: present_or_copy clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK:NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int, int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument decl
+  // CHECK-NEXT: Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument type 'int *'
+  // CHECK-NEXT: PointerType{{.*}} 'int *'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel copy(t) pcopy(NTTP, u) present_or_copy(u[0:t])
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copy clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: pcopy clause
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: present_or_copy clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK:NullStmt
+}
+
+void Inst() {
+  static constexpr unsigned CEVar = 1;
+  int i;
+  TemplUses<CEVar>(i, &i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-copy-clause.c b/clang/test/SemaOpenACC/combined-construct-copy-clause.c
new file mode 100644
index 0000000000000..c588fd8d94987
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copy-clause.c
@@ -0,0 +1,72 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  void *PointerMember;
+} Complete;
+void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete CompositeParam) {
+  int LocalInt;
+  short *LocalPointer;
+  float LocalArray[5];
+  Complete LocalComposite;
+  // Check Appertainment:
+#pragma acc parallel loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'pcopy' is a deprecated clause name and is now an alias for 'copy'}}
+#pragma acc parallel loop pcopy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'present_or_copy' is a deprecated clause name and is now an alias for 'copy'}}
+#pragma acc parallel loop present_or_copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copy(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copy(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copy(LocalComposite.ScalarMember, LocalComposite.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copy(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copy(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'copy' clause is not valid on 'loop' directive}}
+#pragma acc loop copy(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'pcopy' clause is not valid on 'loop' directive}}
+#pragma acc loop pcopy(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'present_or_copy' clause is not valid on 'loop' directive}}
+#pragma acc loop present_or_copy(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copy-clause.cpp b/clang/test/SemaOpenACC/combined-construct-copy-clause.cpp
new file mode 100644
index 0000000000000..043a7ec4dc000
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copy-clause.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+enum SomeE{};
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  SomeE EnumMember;
+  char *PointerMember;
+} Complete;
+
+void uses(int IntParam, char *PointerParam, float ArrayParam[5], Complete CompositeParam, int &IntParamRef) {
+  int LocalInt;
+  char *LocalPointer;
+  float LocalArray[5];
+  // Check Appertainment:
+#pragma acc parallel loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copy(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copy(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copy(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+  Complete LocalComposite2;
+#pragma acc parallel loop copy(LocalComposite2.ScalarMember, LocalComposite2.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copy(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copy(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename T, unsigned I, typename V>
+void TemplUses(T t, T (&arrayT)[I], V TemplComp) {
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(+t)
+  for(int i = 0; i < 5; ++i);
+
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#TEMPL_USES_INST{{in instantiation of}}
+#pragma acc parallel loop copy(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copy(t, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copy(arrayT)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copy(TemplComp)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copy(TemplComp.PointerMember[5])
+  for(int i = 0; i < 5; ++i);
+ int *Pointer;
+#pragma acc parallel loop copy(Pointer[:I])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copy(Pointer[:t])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copy(Pointer[1:])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned I, auto &NTTP_REF>
+void NTTP() {
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#NTTP_INST{{in instantiation of}}
+#pragma acc parallel loop copy(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copy(NTTP_REF)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  static constexpr int NTTP_REFed = 1;
+  int i;
+  int Arr[5];
+  Complete C;
+  TemplUses(i, Arr, C); // #TEMPL_USES_INST
+  NTTP<5, NTTP_REFed>(); // #NTTP_INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copyin-ast.cpp b/clang/test/SemaOpenACC/combined-construct-copyin-ast.cpp
new file mode 100644
index 0000000000000..e7ac8238a7db6
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyin-ast.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copyin(GlobalArray) pcopyin(readonly:PointerParam[Global]) present_or_copyin(Global)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyin clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
+  // CHECK-NEXT: pcopyin clause : readonly
+  // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: present_or_copyin clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<auto &NTTP, typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: NonTypeTemplateParmDecl {{.*}}referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copyin(t) pcopyin(readonly: NTTP, u) present_or_copyin(u[0:t])
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyin clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: pcopyin clause : readonly
+  // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: present_or_copyin clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int, int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument decl
+  // CHECK-NEXT: Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument type 'int *'
+  // CHECK-NEXT: PointerType{{.*}} 'int *'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel copyin(t) pcopyin(readonly: NTTP, u) present_or_copyin(u[0:t])
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyin clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: pcopyin clause : readonly
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: present_or_copyin clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void Inst() {
+  static constexpr unsigned CEVar = 1;
+  int i;
+  TemplUses<CEVar>(i, &i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-copyin-clause.c b/clang/test/SemaOpenACC/combined-construct-copyin-clause.c
new file mode 100644
index 0000000000000..4cb635ddb2470
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyin-clause.c
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  void *PointerMember;
+} Complete;
+void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete CompositeParam) {
+  int LocalInt;
+  short *LocalPointer;
+  float LocalArray[5];
+  Complete LocalComposite;
+  // Check Appertainment:
+#pragma acc parallel loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'pcopyin' is a deprecated clause name and is now an alias for 'copyin'}}
+#pragma acc parallel loop pcopyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyin' is a deprecated clause name and is now an alias for 'copyin'}}
+#pragma acc parallel loop present_or_copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copyin(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyin(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyin(readonly:LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyin(LocalComposite.ScalarMember, LocalComposite.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyin(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copyin(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{invalid tag 'invalid' on 'copyin' clause}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(invalid:(float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'copyin' clause is not valid on 'loop' directive}}
+#pragma acc loop copyin(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'pcopyin' clause is not valid on 'loop' directive}}
+#pragma acc loop pcopyin(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'present_or_copyin' clause is not valid on 'loop' directive}}
+#pragma acc loop present_or_copyin(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copyin-clause.cpp b/clang/test/SemaOpenACC/combined-construct-copyin-clause.cpp
new file mode 100644
index 0000000000000..9f9c2aa641c43
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyin-clause.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+enum SomeE{};
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  SomeE EnumMember;
+  char *PointerMember;
+} Complete;
+
+void uses(int IntParam, char *PointerParam, float ArrayParam[5], Complete CompositeParam, int &IntParamRef) {
+  int LocalInt;
+  char *LocalPointer;
+  float LocalArray[5];
+  // Check Appertainment:
+#pragma acc parallel loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copyin(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copyin(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyin(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+  Complete LocalComposite2;
+#pragma acc parallel loop copyin(LocalComposite2.ScalarMember, LocalComposite2.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyin(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copyin(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename T, unsigned I, typename V>
+void TemplUses(T t, T (&arrayT)[I], V TemplComp) {
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(+t)
+  for(int i = 0; i < 5; ++i);
+
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#TEMPL_USES_INST{{in instantiation of}}
+#pragma acc parallel loop copyin(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyin(t, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyin(arrayT)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyin(TemplComp)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyin(TemplComp.PointerMember[5])
+  for(int i = 0; i < 5; ++i);
+ int *Pointer;
+#pragma acc parallel loop copyin(Pointer[:I])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyin(Pointer[:t])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyin(Pointer[1:])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned I, auto &NTTP_REF>
+void NTTP() {
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#NTTP_INST{{in instantiation of}}
+#pragma acc parallel loop copyin(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyin(NTTP_REF)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  static constexpr int NTTP_REFed = 1;
+  int i;
+  int Arr[5];
+  Complete C;
+  TemplUses(i, Arr, C); // #TEMPL_USES_INST
+  NTTP<5, NTTP_REFed>(); // #NTTP_INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp b/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp
new file mode 100644
index 0000000000000..02eda413dd53f
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copyout(GlobalArray) pcopyout(zero:PointerParam[Global]) present_or_copyout(Global)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyout clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
+  // CHECK-NEXT: pcopyout clause : zero
+  // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: present_or_copyout clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: For
+  // CHECK: NullStmt
+}
+
+template<auto &NTTP, typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: NonTypeTemplateParmDecl {{.*}}referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop copyout(t) pcopyout(zero: NTTP, u) present_or_copyout(u[0:t])
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyout clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: pcopyout clause : zero
+  // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: present_or_copyout clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int, int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument decl
+  // CHECK-NEXT: Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument type 'int *'
+  // CHECK-NEXT: PointerType{{.*}} 'int *'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel copyout(t) pcopyout(zero: NTTP, u) present_or_copyout(u[0:t])
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: copyout clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: pcopyout clause : zero
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: present_or_copyout clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void Inst() {
+  static constexpr unsigned CEVar = 1;
+  int i;
+  TemplUses<CEVar>(i, &i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-copyout-clause.c b/clang/test/SemaOpenACC/combined-construct-copyout-clause.c
new file mode 100644
index 0000000000000..c43f9592062fa
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyout-clause.c
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  void *PointerMember;
+} Complete;
+void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete CompositeParam) {
+  int LocalInt;
+  short *LocalPointer;
+  float LocalArray[5];
+  Complete LocalComposite;
+  // Check Appertainment:
+#pragma acc parallel loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'pcopyout' is a deprecated clause name and is now an alias for 'copyout'}}
+#pragma acc parallel loop pcopyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'present_or_copyout' is a deprecated clause name and is now an alias for 'copyout'}}
+#pragma acc parallel loop present_or_copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copyout(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyout(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyout(zero:LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyout(LocalComposite.ScalarMember, LocalComposite.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyout(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copyout(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{invalid tag 'invalid' on 'copyout' clause}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(invalid:(float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'copyout' clause is not valid on 'loop' directive}}
+#pragma acc loop copyout(LocalInt)
+  for(int i = 0; i < 6;++i);
+  // expected-error@+1{{OpenACC 'pcopyout' clause is not valid on 'loop' directive}}
+#pragma acc loop pcopyout(LocalInt)
+  for(int i = 0; i < 6;++i);
+  // expected-error@+1{{OpenACC 'present_or_copyout' clause is not valid on 'loop' directive}}
+#pragma acc loop present_or_copyout(LocalInt)
+  for(int i = 0; i < 6;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copyout-clause.cpp b/clang/test/SemaOpenACC/combined-construct-copyout-clause.cpp
new file mode 100644
index 0000000000000..cc91ea4d24986
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-copyout-clause.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+enum SomeE{};
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  SomeE EnumMember;
+  char *PointerMember;
+} Complete;
+
+void uses(int IntParam, char *PointerParam, float ArrayParam[5], Complete CompositeParam, int &IntParamRef) {
+  int LocalInt;
+  char *LocalPointer;
+  float LocalArray[5];
+  // Check Appertainment:
+#pragma acc parallel loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop copyout(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop copyout(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyout(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+  Complete LocalComposite2;
+#pragma acc parallel loop copyout(LocalComposite2.ScalarMember, LocalComposite2.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyout(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop copyout(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename T, unsigned I, typename V>
+void TemplUses(T t, T (&arrayT)[I], V TemplComp) {
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(+t)
+  for(int i = 0; i < 5; ++i);
+
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#TEMPL_USES_INST{{in instantiation of}}
+#pragma acc parallel loop copyout(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop copyout(t, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyout(arrayT)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyout(TemplComp)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyout(TemplComp.PointerMember[5])
+  for(int i = 0; i < 5; ++i);
+ int *Pointer;
+#pragma acc parallel loop copyout(Pointer[:I])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop copyout(Pointer[:t])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop copyout(Pointer[1:])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned I, auto &NTTP_REF>
+void NTTP() {
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#NTTP_INST{{in instantiation of}}
+#pragma acc parallel loop copyout(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop copyout(NTTP_REF)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  static constexpr int NTTP_REFed = 1;
+  int i;
+  int Arr[5];
+  Complete C;
+  TemplUses(i, Arr, C); // #TEMPL_USES_INST
+  NTTP<5, NTTP_REFed>(); // #NTTP_INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-create-ast.cpp b/clang/test/SemaOpenACC/combined-construct-create-ast.cpp
new file mode 100644
index 0000000000000..16268eda4e5d2
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-create-ast.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-deprecated-clause-alias -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop create(GlobalArray) pcreate(zero:PointerParam[Global]) present_or_create(Global)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
+  // CHECK-NEXT: pcreate clause : zero
+  // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: present_or_create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<auto &NTTP, typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: NonTypeTemplateParmDecl {{.*}}referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop create(t) pcreate(zero: NTTP, u) present_or_create(u[0:t])
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: pcreate clause : zero
+  // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: present_or_create clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int, int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument decl
+  // CHECK-NEXT: Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument type 'int *'
+  // CHECK-NEXT: PointerType{{.*}} 'int *'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel create(t) pcreate(zero: NTTP, u) present_or_create(u[0:t])
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: pcreate clause : zero
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: present_or_create clause
+  // CHECK-NEXT: ArraySectionExpr
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void Inst() {
+  static constexpr unsigned CEVar = 1;
+  int i;
+  TemplUses<CEVar>(i, &i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-create-clause.c b/clang/test/SemaOpenACC/combined-construct-create-clause.c
new file mode 100644
index 0000000000000..c17e6921a7da2
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-create-clause.c
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  void *PointerMember;
+} Complete;
+void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete CompositeParam) {
+  int LocalInt;
+  short *LocalPointer;
+  float LocalArray[5];
+  Complete LocalComposite;
+  // Check Appertainment:
+#pragma acc parallel loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'pcreate' is a deprecated clause name and is now an alias for 'create'}}
+#pragma acc parallel loop pcreate(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-warning@+1{{OpenACC clause name 'present_or_create' is a deprecated clause name and is now an alias for 'create'}}
+#pragma acc parallel loop present_or_create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop create(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop create(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop create(zero:LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop create(LocalComposite.ScalarMember, LocalComposite.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop create(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop create(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{invalid tag 'invalid' on 'create' clause}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(invalid:(float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'create' clause is not valid on 'loop' directive}}
+#pragma acc loop create(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'pcreate' clause is not valid on 'loop' directive}}
+#pragma acc loop pcreate(LocalInt)
+  for(int i = 5; i < 10;++i);
+  // expected-error@+1{{OpenACC 'present_or_create' clause is not valid on 'loop' directive}}
+#pragma acc loop present_or_create(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-create-clause.cpp b/clang/test/SemaOpenACC/combined-construct-create-clause.cpp
new file mode 100644
index 0000000000000..0c8f90eab3d08
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-create-clause.cpp
@@ -0,0 +1,113 @@
+
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+enum SomeE{};
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  SomeE EnumMember;
+  char *PointerMember;
+} Complete;
+
+void uses(int IntParam, char *PointerParam, float ArrayParam[5], Complete CompositeParam, int &IntParamRef) {
+  int LocalInt;
+  char *LocalPointer;
+  float LocalArray[5];
+  // Check Appertainment:
+#pragma acc parallel loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc serial loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop create(LocalInt)
+  for(int i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop create(LocalInt, LocalPointer, LocalArray)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop create(LocalArray[2:1])
+  for(int i = 0; i < 5; ++i);
+
+  Complete LocalComposite2;
+#pragma acc parallel loop create(LocalComposite2.ScalarMember, LocalComposite2.ScalarMember)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(1 + IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(+IntParam)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop create(PointerParam[2:])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop create(ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create((float*)ArrayParam[2:5])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create((float)ArrayParam[2])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename T, unsigned I, typename V>
+void TemplUses(T t, T (&arrayT)[I], V TemplComp) {
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(+t)
+  for(int i = 0; i < 5; ++i);
+
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#TEMPL_USES_INST{{in instantiation of}}
+#pragma acc parallel loop create(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop create(t, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop create(arrayT)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop create(TemplComp)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop create(TemplComp.PointerMember[5])
+  for(int i = 0; i < 5; ++i);
+ int *Pointer;
+#pragma acc parallel loop create(Pointer[:I])
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop create(Pointer[:t])
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop create(Pointer[1:])
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned I, auto &NTTP_REF>
+void NTTP() {
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#NTTP_INST{{in instantiation of}}
+#pragma acc parallel loop create(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop create(NTTP_REF)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  static constexpr int NTTP_REFed = 1;
+  int i;
+  int Arr[5];
+  Complete C;
+  TemplUses(i, Arr, C); // #TEMPL_USES_INST
+  NTTP<5, NTTP_REFed>(); // #NTTP_INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-default-clause.c b/clang/test/SemaOpenACC/combined-construct-default-clause.c
index c420c8ebeb31d..a9c90240cb122 100644
--- a/clang/test/SemaOpenACC/combined-construct-default-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-default-clause.c
@@ -11,13 +11,11 @@ void SingleOnly() {
 
   int i;
 
-  // expected-warning@+3{{OpenACC clause 'copy' not yet implemented}}
   // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'kernels loop' directive}}
   // expected-note@+1{{previous clause is here}}
   #pragma acc kernels loop self default(present) present(i) default(none) copy(i)
   for(int i = 5; i < 10;++i);
 
-  // expected-warning@+3{{OpenACC clause 'copy' not yet implemented}}
   // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'parallel loop' directive}}
   // expected-note@+1{{previous clause is here}}
   #pragma acc parallel loop self default(present) private(i) default(none) copy(i)
diff --git a/clang/test/SemaOpenACC/combined-construct-default-clause.cpp b/clang/test/SemaOpenACC/combined-construct-default-clause.cpp
index 70c9a7cb47813..60c4d21712e68 100644
--- a/clang/test/SemaOpenACC/combined-construct-default-clause.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-default-clause.cpp
@@ -12,19 +12,16 @@ void SingleOnly() {
   #pragma acc parallel loop default(present) async default(none)
   for (unsigned I = 0; I < 5; ++I);
 
-  // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}}
   // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'serial loop' directive}}
   // expected-note@+1{{previous clause is here}}
   #pragma acc serial loop async default(present) copy(i) default(none) self
   for (unsigned I = 0; I < 5; ++I);
 
-  // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}}
   // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'kernels loop' directive}}
   // expected-note@+1{{previous clause is here}}
   #pragma acc kernels loop async default(present) copy(i) default(none) self
   for (unsigned I = 0; I < 5; ++I);
 
-  // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}}
   // expected-error@+1{{expected '('}}
   #pragma acc parallel loop async default(none) copy(i) default self
   for (unsigned I = 0; I < 5; ++I);
diff --git a/clang/test/SemaOpenACC/combined-construct-device_type-clause.c b/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
index 9b7a3ec6b0ada..11bb342a2f638 100644
--- a/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
@@ -39,7 +39,6 @@ void uses() {
   // 'worker', 'vector', 'seq', 'independent', 'auto', and 'tile'  after
   // 'device_type'.
 
-  //expected-warning@+1{{OpenACC clause 'vector' not yet implemented, clause ignored}}
 #pragma acc parallel loop device_type(*) vector
   for(int i = 0; i < 5; ++i);
 
@@ -57,7 +56,6 @@ void uses() {
   for(int i = 0; i < 5; ++i);
 #pragma acc kernels loop device_type(*) auto
   for(int i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
 #pragma acc parallel loop device_type(*) worker
   for(int i = 0; i < 5; ++i);
   // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a 'serial loop' construct}}
@@ -183,7 +181,6 @@ void uses() {
   // expected-note@+1{{previous clause is here}}
 #pragma acc serial loop device_type(*) reduction(+:Var)
   for(int i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'collapse' not yet implemented, clause ignored}}
 #pragma acc serial loop device_type(*) collapse(1)
   for(int i = 0; i < 5; ++i);
   // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a 'parallel loop' construct}}
@@ -196,7 +193,6 @@ void uses() {
   // expected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'serial loop' directive}}
 #pragma acc serial loop device_type(*) num_gangs(1)
   for(int i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'num_workers' not yet implemented, clause ignored}}
 #pragma acc parallel loop device_type(*) num_workers(1)
   for(int i = 0; i < 5; ++i);
   // expected-error@+2{{OpenACC clause 'device_num' may not follow a 'device_type' clause in a 'serial loop' construct}}
@@ -210,15 +206,12 @@ void uses() {
 #pragma acc parallel loop device_type(*) async
   for(int i = 0; i < 5; ++i);
 
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
 #pragma acc serial loop device_type(*) tile(*, 1)
   for(int j = 0; j < 5; ++j)
     for(int i = 0; i < 5; ++i);
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
 #pragma acc serial loop dtype(*) gang
   for(int i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
 #pragma acc parallel loop device_type(*) wait
   for(int i = 0; i < 5; ++i);
 }
diff --git a/clang/test/SemaOpenACC/combined-construct-deviceptr-ast.cpp b/clang/test/SemaOpenACC/combined-construct-deviceptr-ast.cpp
new file mode 100644
index 0000000000000..5cfc958401690
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-deviceptr-ast.cpp
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop deviceptr(PointerParam)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}} 'PointerParam' 'float *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T>
+void TemplUses(T *t) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 T
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T *)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T *'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop deviceptr(t)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T *' lvalue ParmVar{{.*}} 't' 'T *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel loop deviceptr(t)
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: deviceptr clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 't' 'int *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+}
+
+void Inst() {
+  int i;
+  TemplUses(&i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.c b/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.c
new file mode 100644
index 0000000000000..29b787cf934af
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.c
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S {
+  int IntMem;
+  int *PtrMem;
+};
+
+void uses() {
+  int LocalInt;
+  int *LocalPtr;
+  int Array[5];
+  int *PtrArray[5];
+  struct S s;
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(LocalInt)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop deviceptr(&LocalInt)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc serial loop deviceptr(LocalPtr)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop deviceptr(LocalPtr)
+  for (int i = 0; i < 5; ++i);
+
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int[5]'}}
+#pragma acc kernels loop deviceptr(Array)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(Array[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(Array[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int *[5]'}}
+#pragma acc parallel loop deviceptr(PtrArray)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(PtrArray[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(PtrArray[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'struct S'}}
+#pragma acc parallel loop deviceptr(s)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(s.IntMem)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(s.PtrMem)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'deviceptr' clause is not valid on 'loop' directive}}
+#pragma acc loop deviceptr(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.cpp b/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.cpp
new file mode 100644
index 0000000000000..4baa4ce987405
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-deviceptr-clause.cpp
@@ -0,0 +1,120 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S {
+  int IntMem;
+  int *PtrMem;
+  operator int*();
+};
+
+void uses() {
+  int LocalInt;
+  int *LocalPtr;
+  int Array[5];
+  int *PtrArray[5];
+  struct S s;
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(LocalInt)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(LocalPtr)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int[5]'}}
+#pragma acc parallel loop deviceptr(Array)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(Array[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(Array[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int *[5]'}}
+#pragma acc parallel loop deviceptr(PtrArray)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(PtrArray[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(PtrArray[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'struct S'}}
+#pragma acc parallel loop deviceptr(s)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(s.IntMem)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(s.PtrMem)
+  for (int i = 0; i < 5; ++i);
+}
+
+template<typename T, typename TPtr, typename TStruct, auto &R1>
+void Templ() {
+  T SomeInt;
+  TPtr SomePtr;
+  T SomeIntArray[5];
+  TPtr SomeIntPtrArray[5];
+  TStruct SomeStruct;
+
+  // expected-error@+2{{expected pointer in 'deviceptr' clause, type is 'int'}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+#pragma acc parallel loop deviceptr(SomeInt)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(SomePtr)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int[5]'}}
+#pragma acc parallel loop deviceptr(SomeIntArray)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(SomeIntArray[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(SomeIntArray[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int *[5]'}}
+#pragma acc parallel loop deviceptr(SomeIntPtrArray)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(SomeIntPtrArray[0])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array is not allowed here}}
+  // expected-note@+1{{expected variable of pointer type}}
+#pragma acc parallel loop deviceptr(SomeIntPtrArray[0:1])
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'S'}}
+#pragma acc parallel loop deviceptr(SomeStruct)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(SomeStruct.IntMem)
+  for (int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop deviceptr(SomeStruct.PtrMem)
+  for (int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected pointer in 'deviceptr' clause, type is 'int'}}
+#pragma acc parallel loop deviceptr(R1)
+  for (int i = 0; i < 5; ++i);
+}
+
+void inst() {
+  static constexpr int CEVar = 1;
+  Templ<int, int*, S, CEVar>(); // #INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-gang-ast.cpp b/clang/test/SemaOpenACC/combined-construct-gang-ast.cpp
new file mode 100644
index 0000000000000..167f56289c58d
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-gang-ast.cpp
@@ -0,0 +1,215 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  int Val;
+
+#pragma acc parallel loop gang(dim:1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop gang(static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop gang(num:1) gang(static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: gang clause num
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop gang(dim:1, static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: ConstantExpr
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop gang(static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop gang(static:*)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop gang
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: gang clause num
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T, unsigned One>
+void TemplateUses(T Val) {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplateUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplateUses 'void (T)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced Val 'T'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop gang(dim:One)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop gang(static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+
+#pragma acc parallel loop gang(static:*)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop gang(dim:One) gang(static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop gang(dim:One, static:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop gang
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplateUses 'void (int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument integral '1U'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used Val 'int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: ConstantExpr
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void inst() {
+  TemplateUses<int, 1>(5);
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-gang-clause.cpp b/clang/test/SemaOpenACC/combined-construct-gang-clause.cpp
new file mode 100644
index 0000000000000..c5346c4cf5b90
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-gang-clause.cpp
@@ -0,0 +1,307 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S{};
+struct Converts{
+  operator int();
+};
+
+template<typename T, unsigned Zero, unsigned Two, unsigned Four>
+void ParallelTempl() {
+  T i;
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop gang(i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop gang(num:i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc parallel loop gang(dim:i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc parallel loop gang(dim:Zero)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:Two)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc parallel loop gang(dim:Four)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(static:i) gang(dim:Two)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(static:i, dim:Two)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:Two) gang(static:*)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:Two, static:*)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+4{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+3{{previous expression is here}}
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one 'dim' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc parallel loop gang(dim:Two, static:*, dim:1, static:i)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Parallel() {
+  ParallelTempl<int, 0, 2, 4>(); // expected-note{{in instantiation of function template}}
+  int i;
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop gang(i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop gang(num:i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc parallel loop gang(dim:i)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc parallel loop gang(dim:0)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:2)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc parallel loop gang(dim:4)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(static:i) gang(dim:2)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(static:i, dim:2)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:2) gang(static:*)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:2, static:*)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+4{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+3{{previous expression is here}}
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one 'dim' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc parallel loop gang(dim:2, static:*, dim:1, static:i)
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename SomeS, typename SomeC, typename Int>
+void StaticIsIntegralTempl() {
+  SomeS s;
+  // expected-error@+1{{OpenACC clause 'gang' requires expression of integer type ('S' invalid)}}
+#pragma acc parallel loop gang(dim:2) gang(static:s)
+  for(int i = 0; i < 5; ++i);
+
+  SomeC C;
+#pragma acc parallel loop gang(dim:2) gang(static:C)
+  for(int i = 0; i < 5; ++i);
+  Int i;
+#pragma acc parallel loop gang(dim:2) gang(static:i)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop gang(dim:2) gang(static:*)
+  for(int i = 0; i < 5; ++i);
+}
+
+void StaticIsIntegral() {
+  StaticIsIntegralTempl<S, Converts, int>();// expected-note{{in instantiation of function template}}
+
+  S s;
+  // expected-error@+1{{OpenACC clause 'gang' requires expression of integer type ('S' invalid)}}
+#pragma acc parallel loop gang(dim:2) gang(static:s)
+  for(int i = 0; i < 5; ++i);
+
+  Converts C;
+#pragma acc parallel loop gang(dim:2) gang(static:C)
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned I>
+void SerialTempl() {
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(num:I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'dim' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(dim:I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc serial loop gang(static:I)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Serial() {
+  SerialTempl<2>();
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(1)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(num:1)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'dim' argument on 'gang' clause is not permitted on a 'serial loop'}}
+#pragma acc serial loop gang(dim:1)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc serial loop gang(static:1)
+  for(int i = 0; i < 5; ++i);
+
+  int i;
+#pragma acc serial loop gang(static:i)
+  for(int i = 0; i < 5; ++i);
+}
+
+template<typename T>
+void KernelsTempl() {
+  T t;
+  // expected-error@+1{{'dim' argument on 'gang' clause is not permitted on a 'kernels loop'}}
+#pragma acc kernels loop gang(dim:t)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop gang(static:t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_gangs' clause not allowed on a 'kernels loop' construct that has a 'gang' clause with a 'num' argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop gang(t) num_gangs(t)
+  for(int i = 0; i < 5; ++i);
+
+  // OK, kernels loop should block this.
+#pragma acc kernels num_gangs(t)
+#pragma acc kernels loop gang(static:t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'gang' clause not allowed on a 'kernels loop' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(t) gang(t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_gangs' clause not allowed on a 'kernels loop' construct that has a 'gang' clause with a 'num' argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop gang(num:t) num_gangs(t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'gang' clause not allowed on a 'kernels loop' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(t) gang(num:t)
+  for(int i = 0; i < 5; ++i);
+}
+
+void Kernels() {
+  KernelsTempl<unsigned>();
+
+  // expected-error@+1{{'dim' argument on 'gang' clause is not permitted on a 'kernels loop'}}
+#pragma acc kernels loop gang(dim:1)
+  for(int i = 0; i < 5; ++i);
+
+  unsigned t;
+#pragma acc kernels loop gang(static:t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_gangs' clause not allowed on a 'kernels loop' construct that has a 'gang' clause with a 'num' argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop gang(t) num_gangs(t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'gang' clause not allowed on a 'kernels loop' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(t) gang(t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_gangs' clause not allowed on a 'kernels loop' construct that has a 'gang' clause with a 'num' argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop gang(num:t) num_gangs(t)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'gang' clause not allowed on a 'kernels loop' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(t) gang(num:t)
+  for(int i = 0; i < 5; ++i);
+
+  // OK, intervening compute/combined construct.
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i) {
+#pragma acc serial loop gang(static:1)
+    for(int i = 0; i < 5; ++i);
+  }
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i) {
+#pragma acc serial
+#pragma acc loop gang(static:1)
+    for(int i = 0; i < 5; ++i);
+  }
+#pragma acc kernels
+#pragma acc loop gang(num:1)
+  for(int i = 0; i < 5; ++i) {
+#pragma acc serial loop gang(static:1)
+    for(int i = 0; i < 5; ++i);
+  }
+
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i) {
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'gang' clause on a 'kernels loop' construct}}
+    // expected-note@-3{{previous clause is here}}
+#pragma acc loop gang(static:1)
+    for(int i = 0; i < 5; ++i);
+  }
+
+  // OK, on a different 'loop', not in the associated stmt.
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i);
+#pragma acc loop gang(static:1)
+  for(int i = 0; i < 5; ++i);
+
+  // OK, on a different 'loop', not in the associated stmt.
+#pragma acc kernels loop gang(num:1)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop gang(static:1)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc kernels loop gang(5, num:1)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc kernels loop gang(num:5, 1)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+3{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+2{{previous expression is here}}
+#pragma acc kernels
+#pragma acc kernels loop gang(num:5, num:1)
+  for(int i = 0; i < 5; ++i);
+}
+
+void MaxOneEntry() {
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc kernels loop gang(static: 1, static:1)
+    for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop gang gang(static:1)
+    for(int i = 0; i < 5; ++i);
+}
+
+
diff --git a/clang/test/SemaOpenACC/combined-construct-no_create-ast.cpp b/clang/test/SemaOpenACC/combined-construct-no_create-ast.cpp
new file mode 100644
index 0000000000000..193ec89fb48f1
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-no_create-ast.cpp
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int Global;
+short GlobalArray[5];
+void NormalUses(float *PointerParam) {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop no_create(GlobalArray, PointerParam[Global])
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: no_create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
+  // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float *' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<auto &NTTP, typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: NonTypeTemplateParmDecl {{.*}}referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+
+#pragma acc parallel loop no_create(t) present(NTTP, u)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: no_create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: present clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (int, int *)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument decl
+  // CHECK-NEXT: Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument type 'int *'
+  // CHECK-NEXT: PointerType{{.*}} 'int *'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'int *'
+  // CHECK-NEXT: CompoundStmt
+
+// #pragma acc parallel loop no_create(t) present(NTTP, u)
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: no_create clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
+  // CHECK-NEXT: present clause
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
+  // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void Inst() {
+  static constexpr unsigned CEVar = 1;
+  int i;
+  TemplUses<CEVar>(i, &i);
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-no_create-clause.c b/clang/test/SemaOpenACC/combined-construct-no_create-clause.c
new file mode 100644
index 0000000000000..b0caab2ab9efd
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-no_create-clause.c
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  void *PointerMember;
+} Complete;
+void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete CompositeParam) {
+  int LocalInt;
+  short *LocalPointer;
+  float LocalArray[5];
+  Complete LocalComposite;
+  // Check Appertainment:
+#pragma acc parallel loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc serial loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc kernels loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop no_create(LocalInt, LocalPointer, LocalArray)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc parallel loop no_create(LocalArray[2:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop no_create(LocalComposite.ScalarMember, LocalComposite.ScalarMember)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(1 + IntParam)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(+IntParam)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop no_create(PointerParam[2:])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop no_create(ArrayParam[2:5])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create((float*)ArrayParam[2:5])
+  for (unsigned i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create((float)ArrayParam[2])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'no_create' clause is not valid on 'loop' directive}}
+#pragma acc loop no_create(LocalInt)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-no_create-clause.cpp b/clang/test/SemaOpenACC/combined-construct-no_create-clause.cpp
new file mode 100644
index 0000000000000..f01c46e4877f7
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-no_create-clause.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+enum SomeE{};
+typedef struct IsComplete {
+  struct S { int A; } CompositeMember;
+  int ScalarMember;
+  float ArrayMember[5];
+  SomeE EnumMember;
+  char *PointerMember;
+} Complete;
+
+void uses(int IntParam, char *PointerParam, float ArrayParam[5], Complete CompositeParam, int &IntParamRef) {
+  int LocalInt;
+  char *LocalPointer;
+  float LocalArray[5];
+  // Check Appertainment:
+#pragma acc parallel loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc serial loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc kernels loop no_create(LocalInt)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // Valid cases:
+#pragma acc parallel loop no_create(LocalInt, LocalPointer, LocalArray)
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc parallel loop no_create(LocalArray[2:1])
+  for (unsigned i = 0; i < 5; ++i);
+
+  Complete LocalComposite2;
+#pragma acc parallel loop no_create(LocalComposite2.ScalarMember, LocalComposite2.ScalarMember)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(1 + IntParam)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(+IntParam)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop no_create(PointerParam[2:])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+#pragma acc parallel loop no_create(ArrayParam[2:5])
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC sub-array specified range [2:5] would be out of the range of the subscripted array size of 5}}
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create((float*)ArrayParam[2:5])
+  for (unsigned i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create((float)ArrayParam[2])
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+template<typename T, unsigned I, typename V>
+void TemplUses(T t, T (&arrayT)[I], V TemplComp) {
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(+t)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#TEMPL_USES_INST{{in instantiation of}}
+#pragma acc parallel loop no_create(I)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+#pragma acc parallel loop no_create(t, I)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop no_create(arrayT)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop no_create(TemplComp)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop no_create(TemplComp.PointerMember[5])
+  for (unsigned i = 0; i < 5; ++i);
+ int *Pointer;
+#pragma acc parallel loop no_create(Pointer[:I])
+  for (unsigned i = 0; i < 5; ++i);
+#pragma acc parallel loop no_create(Pointer[:t])
+  for (unsigned i = 0; i < 5; ++i);
+  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
+#pragma acc parallel loop no_create(Pointer[1:])
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+template<unsigned I, auto &NTTP_REF>
+void NTTP() {
+  // NTTP's are only valid if it is a reference to something.
+  // expected-error@+2{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
+  // expected-note@#NTTP_INST{{in instantiation of}}
+#pragma acc parallel loop no_create(I)
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop no_create(NTTP_REF)
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  static constexpr int NTTP_REFed = 1;
+  int i;
+  int Arr[5];
+  Complete C;
+  TemplUses(i, Arr, C); // #TEMPL_USES_INST
+  NTTP<5, NTTP_REFed>(); // #NTTP_INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-num_gangs-ast.cpp b/clang/test/SemaOpenACC/combined-construct-num_gangs-ast.cpp
new file mode 100644
index 0000000000000..6e75a00943364
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-num_gangs-ast.cpp
@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+int some_int();
+short some_short();
+long some_long();
+struct CorrectConvert {
+  operator int();
+} Convert;
+
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop num_gangs(some_int(), some_long(), some_short())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: CallExpr{{.*}}'short'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'short (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short ()' lvalue Function{{.*}} 'some_short' 'short ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_gangs(some_int())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels loop num_gangs(u)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_gangs(u, U::value)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (CorrectConvert, HasInt)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'CorrectConvert'
+  // CHECK-NEXT: RecordType{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: CXXRecord{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: TemplateArgument type 'HasInt'
+  // CHECK-NEXT: RecordType{{.*}} 'HasInt'
+  // CHECK-NEXT: CXXRecord{{.*}} 'HasInt'
+  // CHECK-NEXT: ParmVarDecl{{.*}} t 'CorrectConvert'
+  // CHECK-NEXT: ParmVarDecl{{.*}} u 'HasInt'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_gangs clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+
+  operator char();
+};
+
+void Inst() {
+  TemplUses<CorrectConvert, HasInt>({}, {});
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c b/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c
new file mode 100644
index 0000000000000..bd035bd4a51a2
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+short getS();
+float getF();
+void Test() {
+#pragma acc kernels loop num_gangs(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'serial loop' directive}}
+#pragma acc serial loop num_gangs(1)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop num_gangs(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC clause 'num_gangs' requires expression of integer type}}
+#pragma acc parallel loop num_gangs(getF())
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc kernels loop num_gangs()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel loop num_gangs()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'kernels loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'parallel loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{too many integer expression arguments provided to OpenACC 'num_gangs' clause: 'kernels loop' directive expects maximum of 1, 2 were provided}}
+#pragma acc kernels loop num_gangs(1, getS())
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{too many integer expression arguments provided to OpenACC 'num_gangs' clause: 'parallel loop' directive expects maximum of 3, 4 were provided}}
+#pragma acc parallel loop num_gangs(getS(), 1, getS(), 1)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp b/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp
new file mode 100644
index 0000000000000..8aa361c7b037c
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp
@@ -0,0 +1,230 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+int some_int();
+short some_short();
+long some_long();
+enum E{};
+E some_enum();
+struct CorrectConvert {
+  operator int();
+} Convert;
+
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel loop num_workers(some_int())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_workers(some_short())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CallExpr{{.*}}'short'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'short (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short ()' lvalue Function{{.*}} 'some_short' 'short ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_workers(some_long())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_workers(some_enum())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CallExpr{{.*}}'E'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'E (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'E ()' lvalue Function{{.*}} 'some_enum' 'E ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_workers(Convert)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator int
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'struct CorrectConvert':'CorrectConvert' lvalue Var
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop num_workers(t)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'T' lvalue ParmVar{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_workers(u)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_workers(U::value)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_workers(T{})
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'T' 'T' list
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_workers(U{})
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'U' 'U' list
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop num_workers(typename U::IntTy{})
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'typename U::IntTy' 'typename U::IntTy' list
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop num_workers(typename U::ShortTy{})
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'typename U::ShortTy' 'typename U::ShortTy' list
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (CorrectConvert, HasInt)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'CorrectConvert'
+  // CHECK-NEXT: RecordType{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: CXXRecord{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: TemplateArgument type 'HasInt'
+  // CHECK-NEXT: RecordType{{.*}} 'HasInt'
+  // CHECK-NEXT: CXXRecord{{.*}} 'HasInt'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used t 'CorrectConvert'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'HasInt'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator int
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'CorrectConvert' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator int
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'CorrectConvert' lvalue
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}} 'CorrectConvert' functional cast to struct CorrectConvert <NoOp>
+  // CHECK-NEXT: InitListExpr{{.*}}'CorrectConvert'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'HasInt' lvalue
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}} 'HasInt' functional cast to struct HasInt <NoOp>
+  // CHECK-NEXT: InitListExpr{{.*}}'HasInt'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}} 'typename HasInt::IntTy':'int' functional cast to typename struct HasInt::IntTy <NoOp>
+  // CHECK-NEXT: InitListExpr{{.*}}'typename HasInt::IntTy':'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: num_workers clause
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}} 'typename HasInt::ShortTy':'short' functional cast to typename struct HasInt::ShortTy <NoOp>
+  // CHECK-NEXT: InitListExpr{{.*}}'typename HasInt::ShortTy':'short'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+
+  operator char();
+};
+
+void Inst() {
+  TemplUses<CorrectConvert, HasInt>({}, {});
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c b/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c
new file mode 100644
index 0000000000000..a5891f071bb03
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+short getS();
+float getF();
+void Test() {
+#pragma acc kernels loop num_workers(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC 'num_workers' clause is not valid on 'serial loop' directive}}
+#pragma acc serial loop num_workers(1)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop num_workers(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC clause 'num_workers' requires expression of integer type}}
+#pragma acc parallel loop num_workers(getF())
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc kernels loop num_workers()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel loop num_workers()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{expected ')'}}
+  // expected-note@+1{{to match this '('}}
+#pragma acc kernels loop num_workers(1, 2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{expected ')'}}
+  // expected-note@+1{{to match this '('}}
+#pragma acc parallel loop num_workers(1, 2)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-reduction-ast.cpp b/clang/test/SemaOpenACC/combined-construct-reduction-ast.cpp
new file mode 100644
index 0000000000000..502b7a2613e40
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-reduction-ast.cpp
@@ -0,0 +1,129 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+void NormalFunc(int i, float f) {
+  // CHECK: FunctionDecl{{.*}}NormalFunc
+  // CHECK-NEXT: ParmVarDecl
+  // CHECK-NEXT: ParmVarDecl
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop reduction(+: i)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: reduction clause Operator: +
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue ParmVar{{.*}} 'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop reduction(*: f)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: reduction clause Operator: *
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'float' lvalue ParmVar{{.*}} 'f' 'float'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+
+#pragma acc kernels loop reduction(max: i)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: reduction clause Operator: max
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue ParmVar{{.*}} 'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  }
+
+template<typename T>
+void TemplFunc() {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplFunc
+  // CHECK-NEXT: TemplateTypeParmDecl
+
+  // Match the prototype:
+  // CHECK-NEXT: FunctionDecl{{.*}}TemplFunc
+  // CHECK-NEXT: CompoundStmt
+
+  T t;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} t 'T'
+
+#pragma acc parallel loop reduction(+: t)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: reduction clause Operator: +
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'T' lvalue Var{{.*}} 't' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop reduction(*: T::SomeFloat)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: reduction clause Operator: *
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  typename T::IntTy i;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'typename T::IntTy'
+
+#pragma acc kernels loop reduction(max: i)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: reduction clause Operator: max
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'typename T::IntTy' lvalue Var{{.*}} 'i' 'typename T::IntTy'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Match the instantiation:
+
+  // CHECK: FunctionDecl{{.*}}TemplFunc 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'InstTy'
+  // CHECK-NEXT: RecordType{{.*}} 'InstTy'
+  // CHECK-NEXT: CXXRecord{{.*}} 'InstTy'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} used t 'InstTy'
+  // CHECK-NEXT: CXXConstructExpr{{.*}} 'InstTy' 'void () noexcept'
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: reduction clause Operator: +
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'InstTy' lvalue Var{{.*}} 't' 'InstTy'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: reduction clause Operator: *
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'typename InstTy::IntTy':'int'
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: reduction clause Operator: max
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'typename InstTy::IntTy':'int' lvalue Var{{.*}} 'i' 'typename InstTy::IntTy':'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+}
+
+struct InstTy {
+  using IntTy = int;
+  static constexpr float SomeFloat = 5.0;
+};
+
+void Instantiate() {
+  TemplFunc<InstTy>();
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp b/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp
new file mode 100644
index 0000000000000..082f1ed67a86f
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp
@@ -0,0 +1,169 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct CompositeOfScalars {
+  int I;
+  float F;
+  short J;
+  char C;
+  double D;
+  _Complex float CF;
+  _Complex double CD;
+};
+
+struct CompositeHasComposite {
+  int I;
+  float F;
+  short J;
+  char C;
+  double D;
+  _Complex float CF;
+  _Complex double CD;
+  struct CompositeOfScalars COS; // #COS_FIELD
+};
+
+  // All of the type checking is done for compute and loop constructs, so only check the basics + the parts that are combined specific.
+void uses(unsigned Parm) {
+  struct CompositeOfScalars CoS;
+  struct CompositeHasComposite ChC;
+  int I;
+  float F;
+  int Array[5];
+
+  // legal on all 3 kinds of combined constructs
+#pragma acc parallel loop reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc serial loop reduction(&: CoS, I, F)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop reduction(min: CoS, Array[I], Array[0:I])
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
+  // expected-note@#COS_FIELD{{invalid field is here}}
+#pragma acc parallel loop reduction(&: ChC)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop reduction(+:Parm) num_gangs(I)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC 'num_gangs' clause with more than 1 argument may not appear on a 'parallel loop' construct with a 'reduction' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop reduction(+:Parm) num_gangs(I, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop num_gangs(I) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel loop' construct with a 'num_gangs' clause with more than 1 argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_gangs(I, I) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+
+  // Reduction cannot appear on a loop with a 'gang' of dim>1.
+#pragma acc parallel loop gang(dim:1) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC 'reduction' clause cannot appear on the same 'parallel loop' construct as a 'gang' clause with a 'dim' value greater than 1}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop gang(dim:2) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel loop reduction(+:Parm) gang(dim:1)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC 'gang' clause with a 'dim' value greater than 1 cannot appear on the same 'parallel loop' construct as a 'reduction' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop reduction(+:Parm) gang(dim:2)
+  for(int i = 0; i < 5; ++i);
+
+  // Reduction cannot appear on a loop with a gang and a num_gangs with >1
+  // explicit argument.
+#pragma acc kernels loop num_gangs(I) reduction(+:Parm) gang
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop num_gangs(I) gang reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop reduction(+:Parm) num_gangs(I) gang
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop reduction(+:Parm) gang num_gangs(I)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop gang num_gangs(I) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+#pragma acc kernels loop gang reduction(+:Parm) num_gangs(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel loop' construct with a 'num_gangs' clause with more than 1 argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_gangs(I, I) reduction(+:Parm) gang
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+3{{OpenACC 'reduction' clause cannot appear on the same 'parallel loop' construct as a 'gang' clause and a 'num_gangs' clause with more than one argument}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_gangs(I, I) gang reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC 'num_gangs' clause with more than 1 argument may not appear on a 'parallel loop' construct with a 'reduction' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop reduction(+:Parm) num_gangs(I, I) gang
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+3{{OpenACC 'reduction' clause cannot appear on the same 'parallel loop' construct as a 'gang' clause and a 'num_gangs' clause with more than one argument}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop reduction(+:Parm) gang num_gangs(I, I)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+3{{OpenACC 'reduction' clause cannot appear on the same 'parallel loop' construct as a 'gang' clause and a 'num_gangs' clause with more than one argument}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop gang num_gangs(I, I) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+  // expected-error@+3{{OpenACC 'reduction' clause cannot appear on the same 'parallel loop' construct as a 'gang' clause and a 'num_gangs' clause with more than one argument}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop gang reduction(+:Parm) num_gangs(I, I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel  loop num_gangs(I) reduction(+:Parm) gang
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel  loop num_gangs(I) gang reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel  loop reduction(+:Parm) num_gangs(I) gang
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel  loop reduction(+:Parm) gang num_gangs(I)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel  loop gang num_gangs(I) reduction(+:Parm)
+  for(int i = 0; i < 5; ++i);
+#pragma acc parallel  loop gang reduction(+:Parm) num_gangs(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc parallel loop reduction(+:I)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{OpenACC 'reduction' variable must have the same operator in all nested constructs (& vs +)}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc loop reduction(&:I)
+    for(int i = 0; i < 5; ++i);
+  }
+#pragma acc parallel loop reduction(+:I)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{OpenACC 'reduction' variable must have the same operator in all nested constructs (& vs +)}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc parallel reduction(&:I)
+    for(int i = 0; i < 5; ++i);
+  }
+
+#pragma acc parallel loop reduction(+:I)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{OpenACC 'reduction' variable must have the same operator in all nested constructs (& vs +)}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc parallel loop reduction(&:I)
+    for(int i = 0; i < 5; ++i);
+  }
+#pragma acc loop reduction(+:I)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{OpenACC 'reduction' variable must have the same operator in all nested constructs (& vs +)}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc parallel loop reduction(&:I)
+    for(int i = 0; i < 5; ++i);
+  }
+
+#pragma acc parallel reduction(+:I)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{OpenACC 'reduction' variable must have the same operator in all nested constructs (& vs +)}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc parallel loop reduction(&:I)
+    for(int i = 0; i < 5; ++i);
+  }
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-tile-ast.cpp b/clang/test/SemaOpenACC/combined-construct-tile-ast.cpp
new file mode 100644
index 0000000000000..55a334c276ec8
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-tile-ast.cpp
@@ -0,0 +1,327 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+struct S {
+  constexpr S(){};
+  constexpr operator auto() {return 1;}
+};
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop tile(S{}, 1, 2, *)
+  for(int i = 0;i < 5;++i)
+    for(int j = 0;j < 5;++j)
+      for(int k = 0;k < 5;++k)
+        for(int l = 0;l < 5;++l);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: tile clause
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} k 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} l 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: NullStmt
+}
+
+template<typename T, unsigned Value>
+void TemplUses() {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc serial loop tile(S{}, T{}, *, T{} + S{}, Value, Value + 3)
+  for(int i = 0;i < 5;++i)
+    for(int j = 0;j < 5;++j)
+      for(int k = 0;k < 5;++k)
+        for(int l = 0;l < 5;++l)
+          for(int m = 0;m < 5;++m)
+            for(int n = 0;n < 5;++n);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: tile clause
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'T' 'T' list
+  // CHECK-NEXT: InitListExpr{{.*}}'void'
+  //
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  //
+  // CHECK-NEXT: BinaryOperator{{.*}}'<dependent type>' '+'
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'T' 'T' list
+  // CHECK-NEXT: InitListExpr{{.*}}'void'
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}}'Value' 'unsigned int'
+  //
+  // CHECK-NEXT: BinaryOperator{{.*}}'unsigned int' '+'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}}'Value' 'unsigned int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'unsigned int' <IntegralCast>
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 3
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} k 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} l 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} m 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'m' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'m' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} n 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'n' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'n' 'int'
+  // CHECK-NEXT: NullStmt
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'S'
+  // CHECK-NEXT: RecordType{{.*}} 'S'
+  // CHECK-NEXT: CXXRecord{{.*}} 'S'
+  // CHECK-NEXT: TemplateArgument integral '2U'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: tile clause
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: BinaryOperator{{.*}}'int' '+'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'unsigned int' 2
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 5
+  // CHECK-NEXT: BinaryOperator{{.*}}'unsigned int' '+'
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'unsigned int' 2
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'unsigned int' <IntegralCast>
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 3
+
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} i 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'i' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} j 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'j' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} k 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'k' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} l 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'l' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} m 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'m' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'m' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} n 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator{{.*}}'<'
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'n' 'int'
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 5
+  // CHECK-NEXT: UnaryOperator{{.*}}++
+  // CHECK-NEXT: DeclRefExpr{{.*}}'n' 'int'
+  // CHECK-NEXT: NullStmt
+
+}
+
+void Inst() {
+  TemplUses<S, 2>();
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp b/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp
new file mode 100644
index 0000000000000..00c551e163666
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp
@@ -0,0 +1,396 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+constexpr int three() { return 3; }
+constexpr int one() { return 1; }
+constexpr int neg() { return -1; }
+constexpr int zero() { return 0; }
+
+struct NotConstexpr {
+  constexpr NotConstexpr(){};
+
+  operator int(){ return 1; }
+};
+struct ConvertsNegative {
+  constexpr ConvertsNegative(){};
+
+  constexpr operator int(){ return -1; }
+};
+struct ConvertsOne{
+  constexpr ConvertsOne(){};
+
+  constexpr operator int(){ return 1; }
+};
+
+struct ConvertsThree{
+  constexpr ConvertsThree(){};
+
+  constexpr operator int(){ return 3; }
+};
+
+template<typename T, int Val>
+void negative_zero_constexpr_templ() {
+  // expected-error@+1 2{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc serial loop tile(*, T{})
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc parallel loop tile(Val, *)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc kernels loop tile(zero(), *)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+}
+
+void negative_zero_constexpr() {
+  negative_zero_constexpr_templ<int, 1>(); // expected-note{{in instantiation of function template specialization}}
+  negative_zero_constexpr_templ<int, -1>(); // expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc serial loop tile(0, *)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc parallel loop tile(1, 0)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc kernels loop tile(1, -1)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc parallel loop tile(-1, 0)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc serial loop tile(zero(), 0)
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc kernels loop tile(1, neg())
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be an asterisk or a constant expression}}
+#pragma acc parallel loop tile(NotConstexpr{})
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc serial loop tile(1, ConvertsNegative{})
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop tile(*, ConvertsOne{})
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+}
+
+template<unsigned One>
+void only_for_loops_templ() {
+  // expected-note@+1{{'parallel loop' construct is here}}
+#pragma acc parallel loop tile(One)
+  // expected-error@+1{{OpenACC 'parallel loop' construct can only be applied to a 'for' loop}}
+  while(true);
+
+  // expected-note@+1{{'serial loop' construct is here}}
+#pragma acc serial loop tile(One)
+  // expected-error@+1{{OpenACC 'serial loop' construct can only be applied to a 'for' loop}}
+  do {} while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc kernels loop tile(One, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'kernels loop' with a 'tile' clause}}
+    while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc serial loop tile(One, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'serial loop' with a 'tile' clause}}
+    do{}while(true);
+}
+
+
+void only_for_loops() {
+  // expected-note@+1{{'parallel loop' construct is here}}
+#pragma acc parallel loop tile(1)
+  // expected-error@+1{{OpenACC 'parallel loop' construct can only be applied to a 'for' loop}}
+  while(true);
+
+  // expected-note@+1{{'serial loop' construct is here}}
+#pragma acc serial loop tile(1)
+  // expected-error@+1{{OpenACC 'serial loop' construct can only be applied to a 'for' loop}}
+  do {} while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc kernels loop tile(1, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'kernels loop' with a 'tile' clause}}
+    while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc parallel loop tile(1, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+    do{}while(true);
+}
+
+void only_one_on_loop() {
+  // expected-error@+2{{OpenACC 'tile' clause cannot appear more than once on a 'serial loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc serial loop tile(1) tile(1)
+  for(int i = 0; i < 5; ++i);
+}
+
+template<unsigned Val>
+void depth_too_high_templ() {
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc kernels loop tile (Val, *, Val) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc parallel loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+      while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc serial loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'serial loop' with a 'tile' clause}}
+      do{}while(true);
+
+  int Arr[Val+5];
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc kernels loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(auto x : Arr)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'kernels loop' with a 'tile' clause}}
+      while(true)
+        for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop tile (Val, *, Val)
+  for(int i = 0; i < 5; ++i)
+    for(auto x : Arr)
+      for(int j = 0; j < 5; ++j)
+        while(true);
+}
+
+void depth_too_high() {
+  depth_too_high_templ<3>();
+
+int Arr[5];
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc serial loop tile (1, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc parallel loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+      while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc parallel loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+      do{}while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc parallel loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i)
+    for(int j = 0; j < 5; ++j)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+      while(true)
+        for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop tile (1, *, 3)
+  for(int i = 0; i < 5; ++i)
+    for(auto x : Arr)
+      for(int j = 0; j < 5; ++j)
+        while(true);
+}
+
+template<unsigned Val>
+void not_single_loop_templ() {
+
+  int Arr[Val];
+
+#pragma acc parallel loop tile (Val, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    for (auto x : Arr)
+      for(int k = 0; k < 5; ++k);
+  // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'parallel loop' construct with a 'tile' clause}}
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+}
+
+void not_single_loop() {
+  not_single_loop_templ<3>(); // no diagnostic, was diagnosed in phase 1.
+
+  int Arr[5];
+
+#pragma acc parallel loop tile (1, *, 3)// expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    for (auto x : Arr)
+      for(int k = 0; k < 5; ++k);
+  // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'parallel loop' construct with a 'tile' clause}}
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+}
+
+template<unsigned Val>
+void no_other_directives_templ() {
+
+  int Arr[Val];
+
+#pragma acc parallel loop tile (Val, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    for (auto x : Arr) {
+  // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+#pragma acc serial
+      ;
+      for(int j = 0; j < 5; ++j);
+    }
+  }
+
+  // OK, in innermost
+#pragma acc parallel loop tile (Val, *, 3)
+  for(int i = 0; i < 5; ++i) {
+    for(int j = 0; j < 5; ++j) {
+      for (auto x : Arr) {
+#pragma acc serial
+      ;
+      }
+    }
+  }
+}
+
+void no_other_directives() {
+  no_other_directives_templ<3>();
+  int Arr[5];
+
+#pragma acc parallel loop tile (1, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    for (auto x : Arr) {
+  // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'parallel loop' with a 'tile' clause}}
+#pragma acc serial
+      ;
+      for(int j = 0; j < 5; ++j);
+    }
+  }
+
+  // OK, in innermost
+#pragma acc parallel loop tile (3, *, 3)
+  for(int i = 0; i < 5; ++i) {
+    for(int j = 0; j < 5; ++j) {
+      for (auto x : Arr) {
+#pragma acc serial
+      ;
+      }
+    }
+  }
+}
+
+void call();
+template<unsigned Val>
+void intervening_templ() {
+#pragma acc parallel loop tile(1, Val, *) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'parallel loop' construct}}
+    call();
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+
+#pragma acc parallel loop tile(1, Val, *) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'parallel loop' construct}}
+    unsigned I;
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+
+#pragma acc parallel loop tile(1, Val, *)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-2{{'parallel loop' construct is here}}
+  for(int i = 0;;++i) {
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-5{{'parallel loop' construct is here}}
+    for(int j = 0;;++j)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-8{{'parallel loop' construct is here}}
+      for(int k = 0;;++k)
+        call();
+  }
+}
+
+void intervening() {
+  intervening_templ<3>();
+
+#pragma acc parallel loop tile(1, 2, *) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'parallel loop' construct}}
+    call();
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+
+#pragma acc parallel loop tile(1, 2, *) // expected-note{{active 'tile' clause defined here}}
+  for(int i = 0; i < 5; ++i) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'parallel loop' construct}}
+    unsigned I;
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k);
+  }
+
+#pragma acc parallel loop tile(1, 2, *)
+  for(int i = 0; i < 5; ++i) {
+    for(int j = 0; j < 5; ++j)
+      for(int k = 0; k < 5; ++k)
+        call();
+  }
+
+#pragma acc parallel loop tile(1, 2, *)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-2{{'parallel loop' construct is here}}
+  for(int i = 0;;++i) {
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-5{{'parallel loop' construct is here}}
+    for(int j = 0;;++j)
+  // expected-error@+2{{OpenACC 'parallel loop' construct must have a terminating condition}}
+  // expected-note@-8{{'parallel loop' construct is here}}
+      for(int k = 0;;++k)
+        for(;;)
+        call();
+  }
+}
+
+void collapse_tile_depth() {
+  // expected-error@+4{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+3{{active 'collapse' clause defined here}}
+  // expected-error@+2{{'tile' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'tile' clause defined here}}
+#pragma acc parallel loop tile(1, 2, 3) collapse (3)
+  for(int i = 0; i < 5;++i) {
+    for(int j = 0; j < 5; ++j);
+  }
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-vector-ast.cpp b/clang/test/SemaOpenACC/combined-construct-vector-ast.cpp
new file mode 100644
index 0000000000000..7b103021ccd55
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-vector-ast.cpp
@@ -0,0 +1,176 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  int Val;
+
+#pragma acc parallel loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop vector(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop vector(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T, unsigned One>
+void TemplateUses(T Val) {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplateUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplateUses 'void (T)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced Val 'T'
+  // CHECK-NEXT: CompoundStmt
+
+
+#pragma acc parallel loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop vector(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop vector(length:One)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc serial loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop vector
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop vector(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop vector(length:One)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplateUses 'void (int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument integral '1U'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used Val 'int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: vector clause
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void inst() {
+  TemplateUses<int, 1>(5);
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-vector-clause.cpp b/clang/test/SemaOpenACC/combined-construct-vector-clause.cpp
new file mode 100644
index 0000000000000..f96a36c1e395b
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-vector-clause.cpp
@@ -0,0 +1,171 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<typename T, unsigned I>
+void TemplUses(T t) {
+
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc serial loop vector
+  for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+  // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector vector_length(t)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector_length(t) vector
+  for(int j = 0; j < 5; ++j);
+
+  // expected-error@+2{{'vector_length' clause not allowed on a 'kernels loop' construct that has a 'vector' clause with an argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector(I) vector_length(t)
+  for(int j = 0; j < 5; ++j);
+
+  // expected-error@+2{{'length' argument to 'vector' clause not allowed on a 'kernels loop' construct that has a 'vector_length' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector_length(t) vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j) {
+    // expected-error@+4{{loop with a 'vector' clause may not exist in the region of a 'vector' clause}}
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'vector' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'vector' clause}}
+    // expected-note@-5 3{{previous clause is here}}
+#pragma acc loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j) {
+#pragma acc serial loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc loop vector
+  for(int j = 0; j < 5; ++j) {
+#pragma acc serial loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc kernels vector_length(t)
+  for(int j = 0; j < 5; ++j) {
+    // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(I)
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc kernels vector_length(t)
+  for(int j = 0; j < 5; ++j) {
+#pragma acc parallel loop vector(I)
+    for(int j = 0; j < 5; ++j);
+  }
+}
+
+void uses() {
+  TemplUses<int, 5>(5);
+
+  unsigned I;
+  int t;
+
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc serial loop vector
+  for(int j = 0; j < 5; ++j);
+
+    // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+    // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector(length:I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector vector_length(t)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc kernels loop vector_length(t) vector
+  for(int j = 0; j < 5; ++j);
+
+  // expected-error@+2{{'vector_length' clause not allowed on a 'kernels loop' construct that has a 'vector' clause with an argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector(I) vector_length(t)
+  for(int j = 0; j < 5; ++j);
+
+    // expected-error@+2{{'length' argument to 'vector' clause not allowed on a 'kernels loop' construct that has a 'vector_length' clause}}
+    // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector_length(t) vector(I)
+  for(int j = 0; j < 5; ++j);
+
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j) {
+    // expected-error@+4{{loop with a 'vector' clause may not exist in the region of a 'vector' clause}}
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'vector' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'vector' clause}}
+    // expected-note@-5 3{{previous clause is here}}
+#pragma acc loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+#pragma acc parallel loop vector
+  for(int j = 0; j < 5; ++j) {
+#pragma acc serial loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc loop vector
+  for(int j = 0; j < 5; ++j) {
+#pragma acc serial loop vector worker, gang
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc kernels vector_length(t)
+  for(int j = 0; j < 5; ++j) {
+#pragma acc parallel loop vector(I)
+    for(int j = 0; j < 5; ++j);
+  }
+
+#pragma acc kernels vector_length(t)
+  for(int j = 0; j < 5; ++j) {
+    // expected-error@+1{{'length' argument on 'vector' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop vector(I)
+    for(int j = 0; j < 5; ++j);
+  }
+}
+
diff --git a/clang/test/SemaOpenACC/combined-construct-vector_length-ast.cpp b/clang/test/SemaOpenACC/combined-construct-vector_length-ast.cpp
new file mode 100644
index 0000000000000..6cfc9c6b8b2c2
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-vector_length-ast.cpp
@@ -0,0 +1,98 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+short some_short();
+
+struct CorrectConvert {
+  operator int();
+} Convert;
+
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels loop vector_length(some_short())
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: vector_length clause
+  // CHECK-NEXT: CallExpr{{.*}}'short'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'short (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'short ()' lvalue Function{{.*}} 'some_short' 'short ()'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+template<typename T, typename U>
+void TemplUses(T t, U u) {
+  // CHECK-NEXT: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 T
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (T, U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} t 'T'
+  // CHECK-NEXT: ParmVarDecl{{.*}} u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels loop vector_length(u)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: vector_length clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc parallel loop vector_length(U::value)
+  for (unsigned i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: vector_length clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void (CorrectConvert, HasInt)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'CorrectConvert'
+  // CHECK-NEXT: RecordType{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: CXXRecord{{.*}} 'CorrectConvert'
+  // CHECK-NEXT: TemplateArgument type 'HasInt'
+  // CHECK-NEXT: RecordType{{.*}} 'HasInt'
+  // CHECK-NEXT: CXXRecord{{.*}} 'HasInt'
+  // CHECK-NEXT: ParmVarDecl{{.*}} t 'CorrectConvert'
+  // CHECK-NEXT: ParmVarDecl{{.*}} u 'HasInt'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: vector_length clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: vector_length clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+
+  operator char();
+};
+
+void Inst() {
+  TemplUses<CorrectConvert, HasInt>({}, {});
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c b/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c
new file mode 100644
index 0000000000000..8b6dedd9b83ba
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+short getS();
+float getF();
+void Test() {
+#pragma acc kernels loop vector_length(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'serial loop' directive}}
+#pragma acc serial loop vector_length(1)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop vector_length(1)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{OpenACC clause 'vector_length' requires expression of integer type}}
+#pragma acc parallel loop vector_length(getF())
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc kernels loop vector_length()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel loop vector_length()
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{expected ')'}}
+  // expected-note@+1{{to match this '('}}
+#pragma acc kernels loop vector_length(1, 2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{expected ')'}}
+  // expected-note@+1{{to match this '('}}
+#pragma acc parallel loop vector_length(1, 2)
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-wait-ast.cpp b/clang/test/SemaOpenACC/combined-construct-wait-ast.cpp
new file mode 100644
index 0000000000000..0620901fd9fdb
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-wait-ast.cpp
@@ -0,0 +1,239 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+int some_int();
+long some_long();
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop wait
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+#pragma acc serial loop wait()
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+#pragma acc kernels loop wait(some_int(), some_long())
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: ForStmt
+#pragma acc parallel loop wait(queues:some_int(), some_long())
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: ForStmt
+#pragma acc serial loop wait(devnum: some_int() :some_int(), some_long())
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: ForStmt
+#pragma acc kernels loop wait(devnum: some_int() : queues :some_int(), some_long()) wait(devnum: some_int() : queues :some_int(), some_long())
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: ForStmt
+}
+
+template<typename U>
+void TemplUses(U u) {
+  // CHECK: FunctionTemplateDecl
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 0 U
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (U)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop wait
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+
+#pragma acc serial loop wait()
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+
+#pragma acc kernels loop wait(U::value, u)
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+
+#pragma acc parallel loop wait(queues: U::value, u)
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+
+#pragma acc serial loop wait(devnum:u:queues: U::value, u)
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+
+#pragma acc kernels loop wait(devnum:u: U::value, u)
+  for (int i = 0; i < 5; ++i) {}
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: ForStmt
+
+  // Check the instantiated versions of the above.
+  // CHECK: FunctionDecl{{.*}} used TemplUses 'void (HasInt)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'HasInt'
+  // CHECK-NEXT: RecordType{{.*}} 'HasInt'
+  // CHECK-NEXT: CXXRecord{{.*}} 'HasInt'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used u 'HasInt'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}parallel loop
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}serial loop
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+
+  // CHECK: OpenACCCombinedConstruct{{.*}}kernels loop
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+
+  operator char();
+};
+
+void Inst() {
+  TemplUses<HasInt>({});
+}
+#endif
diff --git a/clang/test/SemaOpenACC/combined-construct-wait-clause.c b/clang/test/SemaOpenACC/combined-construct-wait-clause.c
new file mode 100644
index 0000000000000..7551dd1840963
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-wait-clause.c
@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct NotConvertible{} NC;
+short getS();
+int getI();
+
+void uses() {
+  int arr[5];
+
+#pragma acc parallel loop wait
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc serial loop wait()
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc kernels loop wait(getS(), getI())
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop wait(devnum:getS(): getI())
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop wait(devnum:getS(): queues: getI()) wait(devnum:getI(): queues: getS(), getI(), 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel loop wait(devnum:NC : 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel loop wait(devnum:5 : NC)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('int[5]' invalid)}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('int[5]' invalid)}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel loop wait(devnum:arr : queues: arr, NC, 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
+#pragma acc loop wait
+  for(int i = 5; i < 10;++i);
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-wait-clause.cpp b/clang/test/SemaOpenACC/combined-construct-wait-clause.cpp
new file mode 100644
index 0000000000000..9ce04290cbe5a
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-wait-clause.cpp
@@ -0,0 +1,104 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct ExplicitConvertOnly {
+  explicit operator int() const; // #EXPL_CONV
+} Explicit;
+
+struct AmbiguousConvert{
+  operator int(); // #AMBIG_INT
+  operator short(); // #AMBIG_SHORT
+  operator float();
+} Ambiguous;
+
+void Test() {
+
+  // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(Ambiguous)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel loop wait(4, Explicit, 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(queues: Ambiguous, 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel loop wait(devnum: Explicit: 5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel loop wait(devnum: Explicit:queues:  5)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{use of undeclared identifier 'queues'}}
+#pragma acc parallel loop wait(devnum: queues:  5)
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+  static constexpr AmbiguousConvert ACValue;
+  static constexpr ExplicitConvertOnly EXValue;
+
+  operator char();
+};
+
+template<typename T>
+void TestInst() {
+
+#pragma acc parallel loop wait(T{})
+  for (unsigned i = 0; i < 5; ++i);
+
+#pragma acc parallel loop wait(devnum:typename T::ShortTy{}:queues:typename T::IntTy{})
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+4{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(devnum:T::value :queues:T::ACValue)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(devnum:T::EXValue :queues:T::ACValue)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(T::EXValue, T::ACValue)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel loop wait(queues: T::EXValue, T::ACValue)
+  for (unsigned i = 0; i < 5; ++i);
+
+  // expected-error@+1{{no member named 'Invalid' in 'HasInt'}}
+#pragma acc parallel loop wait(queues: T::Invalid, T::Invalid2)
+  for (unsigned i = 0; i < 5; ++i);
+}
+
+void Inst() {
+  TestInst<HasInt>(); // #INST
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-worker-ast.cpp b/clang/test/SemaOpenACC/combined-construct-worker-ast.cpp
new file mode 100644
index 0000000000000..6ac0fdb35709f
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-worker-ast.cpp
@@ -0,0 +1,135 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  int Val;
+
+#pragma acc parallel loop worker
+  for(int i = 0; i < 5; ++i);
+  // CHECK: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+#pragma acc serial loop worker
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop worker(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+#pragma acc kernels loop worker(num:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+template<typename T, unsigned One>
+void TemplateUses(T Val) {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplateUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplateUses 'void (T)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced Val 'T'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel loop worker
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+#pragma acc serial loop worker
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop worker(Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+#pragma acc kernels loop worker(num:Val)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+#pragma acc kernels loop worker(num:One)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: DeclRefExpr{{.*}}'One' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplateUses 'void (int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument integral '1U'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used Val 'int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} serial loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+
+  // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} kernels loop
+  // CHECK-NEXT: worker clause
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr
+  // CHECK-NEXT: NonTypeTemplateParmDecl
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK: NullStmt
+}
+
+void inst() {
+  TemplateUses<int, 1>(5);
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/combined-construct-worker-clause.cpp b/clang/test/SemaOpenACC/combined-construct-worker-clause.cpp
new file mode 100644
index 0000000000000..1351f9ea0c69c
--- /dev/null
+++ b/clang/test/SemaOpenACC/combined-construct-worker-clause.cpp
@@ -0,0 +1,150 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<unsigned I>
+void TemplUses() {
+
+#pragma acc parallel loop worker
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel loop worker()
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc serial loop worker
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc serial loop worker()
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'worker' clause not allowed on a 'kernels loop' construct that has a 'num_workers' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_workers(1) worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_workers' clause not allowed on a 'kernels loop' construct that has a 'worker' clause with an argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop worker(num:I) num_workers(1)
+  for(int i = 0; i < 5; ++i);
+}
+
+void NormalUses() {
+  TemplUses<4>();
+
+  int I;
+#pragma acc parallel loop worker
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel loop worker()
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'parallel loop' construct}}
+#pragma acc parallel loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc serial loop worker
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{expected expression}}
+#pragma acc serial loop worker()
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on a 'serial loop' construct}}
+#pragma acc serial loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop worker(I)
+  for(int i = 0; i < 5; ++i);
+
+#pragma acc kernels loop worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num' argument to 'worker' clause not allowed on a 'kernels loop' construct that has a 'num_workers' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_workers(1) worker(num:I)
+  for(int i = 0; i < 5; ++i);
+
+  // expected-error@+2{{'num_workers' clause not allowed on a 'kernels loop' construct that has a 'worker' clause with an argument}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop worker(num:I) num_workers(1)
+  for(int i = 0; i < 5; ++i);
+
+  // OK, kernels loop is a new compute construct
+#pragma acc kernels num_workers(1)
+  for(int i = 0; i < 5; ++i) {
+#pragma acc kernels loop worker(num:1)
+  for(int i = 0; i < 5; ++i);
+  }
+#pragma acc kernels loop num_workers(1)
+  for(int i = 0; i < 5; ++i) {
+#pragma acc kernels loop worker(num:1)
+  for(int i = 0; i < 5; ++i);
+  }
+
+#pragma acc kernels loop num_workers(1)
+  for(int i = 0; i < 5; ++i) {
+  // expected-error@+2{{'num' argument to 'worker' clause not allowed on a 'loop' construct associated with a 'kernels loop' construct that has a 'num_workers' clause}}
+  // expected-note@-3{{previous clause is here}}
+#pragma acc loop worker(num:1)
+  for(int i = 0; i < 5; ++i);
+  }
+
+#pragma acc parallel loop worker
+  for(int i = 0; i < 5; ++i) {
+    // expected-error@+4{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-3{{previous clause is here}}
+    // expected-error@+2{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-5{{previous clause is here}}
+#pragma acc loop gang, worker, vector
+  for(int i = 0; i < 5; ++i);
+  }
+#pragma acc kernels loop worker
+  for(int i = 0; i < 5; ++i) {
+    // expected-error@+4{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-3{{previous clause is here}}
+    // expected-error@+2{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-5{{previous clause is here}}
+#pragma acc loop gang, worker, vector
+  for(int i = 0; i < 5; ++i);
+  }
+#pragma acc serial loop worker
+  for(int i = 0; i < 5; ++i) {
+    // expected-error@+4{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-3{{previous clause is here}}
+    // expected-error@+2{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-5{{previous clause is here}}
+#pragma acc loop gang, worker, vector
+  for(int i = 0; i < 5; ++i);
+  }
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-default-clause.c b/clang/test/SemaOpenACC/compute-construct-default-clause.c
index e71fee47e8a29..70e29f3e8ac05 100644
--- a/clang/test/SemaOpenACC/compute-construct-default-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-default-clause.c
@@ -16,17 +16,16 @@ void SingleOnly() {
   #pragma acc kernels self default(present) present(i) default(none) copy(i)
   while(0);
 
-  // expected-warning@+3{{OpenACC clause 'copy' not yet implemented}}
-  // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'parallel loop' directive}}
+  // expected-error@+2{{OpenACC 'default' clause cannot appear more than once on a 'parallel' directive}}
   // expected-note@+1{{previous clause is here}}
-  #pragma acc parallel loop self default(present) private(i) default(none) copy(i)
+  #pragma acc parallel self default(present) private(i) default(none) copy(i)
   for(int i = 0; i < 5; ++i);
 
   // expected-error@+1{{expected '('}}
-  #pragma acc serial loop self default private(i) default(none) if(i)
+  #pragma acc serial self default private(i) default(none) if(i)
   for(int i = 0; i < 5; ++i);
 
-  #pragma acc kernels loop default(none)
+  #pragma acc kernels default(none)
   for(int i = 0; i < 5; ++i);
 
   // expected-warning@+2{{OpenACC construct 'data' not yet implemented}}
diff --git a/clang/test/SemaOpenACC/compute-construct-reduction-clause.c b/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
index 80310f0e7afc6..fcc4ca2655c20 100644
--- a/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
@@ -41,12 +41,12 @@ void uses(unsigned Parm) {
 #pragma acc parallel num_gangs(IVar) reduction(+:IVar)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'num_gangs' clause with more than 1 argument may not appear on a 'parallel' construct with a 'reduction' clause}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel reduction(+:Parm) num_gangs(Parm, IVar)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel num_gangs(Parm, IVar) reduction(+:Var)
   while (1);
diff --git a/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp b/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
index 532dbb2387165..7372f683e2eb3 100644
--- a/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
@@ -41,12 +41,12 @@ void uses(unsigned Parm) {
 #pragma acc parallel num_gangs(IVar) reduction(+:Var)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'num_gangs' clause with more than 1 argument may not appear on a 'parallel' construct with a 'reduction' clause}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel reduction(+:Parm) num_gangs(Parm, IVar)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel num_gangs(Parm, IVar) reduction(+:Var)
   while (1);
@@ -116,12 +116,12 @@ void TemplUses(T Parm, U CoS, V ChC) {
 #pragma acc parallel num_gangs(Var) reduction(+:Var)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'num_gangs' clause with more than 1 argument may not appear on a 'parallel' construct with a 'reduction' clause}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel reduction(+:Parm) num_gangs(Parm, Var)
   while (1);
 
-  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument, have 2}}
+  // expected-error@+2{{OpenACC 'reduction' clause may not appear on a 'parallel' construct with a 'num_gangs' clause with more than 1 argument}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel num_gangs(Parm, Var) reduction(+:Var)
   while (1);
diff --git a/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp b/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp
index a22e7e865a801..6b4c03bbc50c6 100644
--- a/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp
@@ -278,12 +278,19 @@ void Kernels() {
 #pragma acc kernels
 #pragma acc loop gang(num:1)
   for(int j = 0; j < 5; ++j) {
-    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'gang' clause on a 'kernels' compute construct}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'gang' clause on a 'kernels' construct}}
     // expected-note@-3{{previous clause is here}}
 #pragma acc loop gang(static:1)
     for(int i = 0; i < 5; ++i);
   }
 
+#pragma acc kernels
+#pragma acc loop gang(num:1)
+  for(int j = 0; j < 5; ++j) {
+#pragma acc loop worker(1) vector(1)
+    for(int i = 0; i < 5; ++i);
+  }
+
 #pragma acc kernels
 #pragma acc loop gang(num:1)
   for(int j = 0; j < 5; ++j) {
diff --git a/clang/test/SemaOpenACC/loop-construct-vector-clause.cpp b/clang/test/SemaOpenACC/loop-construct-vector-clause.cpp
index 3260e368fbb42..1fed82c510208 100644
--- a/clang/test/SemaOpenACC/loop-construct-vector-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-vector-clause.cpp
@@ -18,12 +18,12 @@ void TemplUses(Int I, NotInt NI, ConvertsToInt CTI) {
 #pragma acc loop vector(length: NI)
   for(int j = 0; j < 5; ++j);
 
-  // expected-error@+2{{'num' argument on 'vector' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+  // expected-error@+2{{'length' argument on 'vector' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
 #pragma acc serial
 #pragma acc loop vector(length: I)
   for(int j = 0; j < 5; ++j);
 
-  // expected-error@+3{{'num' argument to 'vector' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'vector_length' clause}}
+  // expected-error@+3{{'length' argument to 'vector' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'vector_length' clause}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc kernels vector_length(I)
 #pragma acc loop vector(length: CTI)
@@ -87,12 +87,12 @@ void uses() {
 #pragma acc loop vector(length: NI)
   for(int j = 0; j < 5; ++j);
 
-  // expected-error@+2{{'num' argument on 'vector' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+  // expected-error@+2{{'length' argument on 'vector' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
 #pragma acc serial
 #pragma acc loop vector(length: i)
   for(int j = 0; j < 5; ++j);
 
-  // expected-error@+3{{'num' argument to 'vector' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'vector_length' clause}}
+  // expected-error@+3{{'length' argument to 'vector' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'vector_length' clause}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc kernels vector_length(i)
 #pragma acc loop vector(length: i)
diff --git a/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp b/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp
index 8b95981c2b80f..35c63e467a505 100644
--- a/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp
@@ -138,6 +138,12 @@ void uses() {
   for(int k = 0; k < 5; ++k);
   }
 
+#pragma acc loop worker
+  for(int i= 0; i< 5; ++i) {
+#pragma acc loop vector
+  for(int k = 0; k < 5; ++k);
+  }
+
 #pragma acc loop worker
   for(int i= 0; i< 5; ++i) {
 #pragma acc parallel
diff --git a/clang/test/SemaTemplate/template-args-deduction-aggregates.cpp b/clang/test/SemaTemplate/template-args-deduction-aggregates.cpp
new file mode 100644
index 0000000000000..afe78d0873bb3
--- /dev/null
+++ b/clang/test/SemaTemplate/template-args-deduction-aggregates.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++20
+
+// expected-no-diagnostics
+
+namespace GH113659 {
+template <class... Args> struct S {};
+struct T {};
+struct U {};
+
+template <class... Args> struct B : S<Args...>, Args... {};
+B b{S<T, U>{}};
+}
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 5481bb6b87503..28610052b9b74 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -87,7 +87,7 @@ static cl::opt<std::string> AssumeFileName(
              "supported:\n"
              "  CSharp: .cs\n"
              "  Java: .java\n"
-             "  JavaScript: .mjs .js .ts\n"
+             "  JavaScript: .js .mjs .cjs .ts\n"
              "  Json: .json\n"
              "  Objective-C: .m .mm\n"
              "  Proto: .proto .protodevel\n"
diff --git a/clang/tools/clang-format/git-clang-format b/clang/tools/clang-format/git-clang-format
index bacbd8de24566..da271bbe6e3a0 100755
--- a/clang/tools/clang-format/git-clang-format
+++ b/clang/tools/clang-format/git-clang-format
@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 #
-#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===#
+# ===- git-clang-format - ClangFormat Git Integration -------*- python -*--=== #
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
-#===------------------------------------------------------------------------===#
+# ===----------------------------------------------------------------------=== #
 
 r"""
 clang-format git integration
@@ -32,10 +32,11 @@ import re
 import subprocess
 import sys
 
-usage = ('git clang-format [OPTIONS] [<commit>] [<commit>|--staged] '
-         '[--] [<file>...]')
+usage = (
+    "git clang-format [OPTIONS] [<commit>] [<commit>|--staged] [--] [<file>...]"
+)
 
-desc = '''
+desc = """
 If zero or one commits are given, run clang-format on all lines that differ
 between the working directory and <commit>, which defaults to HEAD.  Changes are
 only applied to the working directory, or in the stage/index.
@@ -58,637 +59,815 @@ The following git-config settings set the default of the corresponding option:
   clangFormat.commit
   clangFormat.extensions
   clangFormat.style
-'''
+"""
 
 # Name of the temporary index file in which save the output of clang-format.
 # This file is created within the .git directory.
-temp_index_basename = 'clang-format-index'
+temp_index_basename = "clang-format-index"
 
 
-Range = collections.namedtuple('Range', 'start, count')
+Range = collections.namedtuple("Range", "start, count")
 
 
 def main():
-  config = load_git_config()
-
-  # In order to keep '--' yet allow options after positionals, we need to
-  # check for '--' ourselves.  (Setting nargs='*' throws away the '--', while
-  # nargs=argparse.REMAINDER disallows options after positionals.)
-  argv = sys.argv[1:]
-  try:
-    idx = argv.index('--')
-  except ValueError:
-    dash_dash = []
-  else:
-    dash_dash = argv[idx:]
-    argv = argv[:idx]
-
-  default_extensions = ','.join([
-      # From clang/lib/Frontend/FrontendOptions.cpp, all lower case
-      'c', 'h',  # C
-      'm',  # ObjC
-      'mm',  # ObjC++
-      'cc', 'cp', 'cpp', 'c++', 'cxx', 'hh', 'hpp', 'hxx', 'inc',  # C++
-      'ccm', 'cppm', 'cxxm', 'c++m',  # C++ Modules
-      'cu', 'cuh',  # CUDA
-      # Other languages that clang-format supports
-      'proto', 'protodevel',  # Protocol Buffers
-      'java',  # Java
-      'js',  # JavaScript
-      'ts',  # TypeScript
-      'cs',  # C Sharp
-      'json',  # Json
-      'sv', 'svh', 'v', 'vh', # Verilog
-      ])
-
-  p = argparse.ArgumentParser(
-    usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter,
-    description=desc)
-  p.add_argument('--binary',
-                 default=config.get('clangformat.binary', 'clang-format'),
-                 help='path to clang-format'),
-  p.add_argument('--commit',
-                 default=config.get('clangformat.commit', 'HEAD'),
-                 help='default commit to use if none is specified'),
-  p.add_argument('--diff', action='store_true',
-                 help='print a diff instead of applying the changes')
-  p.add_argument('--diffstat', action='store_true',
-                 help='print a diffstat instead of applying the changes')
-  p.add_argument('--extensions',
-                 default=config.get('clangformat.extensions',
-                                    default_extensions),
-                 help=('comma-separated list of file extensions to format, '
-                       'excluding the period and case-insensitive')),
-  p.add_argument('-f', '--force', action='store_true',
-                 help='allow changes to unstaged files')
-  p.add_argument('-p', '--patch', action='store_true',
-                 help='select hunks interactively')
-  p.add_argument('-q', '--quiet', action='count', default=0,
-                 help='print less information')
-  p.add_argument('--staged', '--cached', action='store_true',
-                 help='format lines in the stage instead of the working dir')
-  p.add_argument('--style',
-                 default=config.get('clangformat.style', None),
-                 help='passed to clang-format'),
-  p.add_argument('-v', '--verbose', action='count', default=0,
-                 help='print extra information')
-  p.add_argument('--diff_from_common_commit', action='store_true',
-                 help=('diff from the last common commit for commits in '
-                      'separate branches rather than the exact point of the '
-                      'commits'))
-  # We gather all the remaining positional arguments into 'args' since we need
-  # to use some heuristics to determine whether or not <commit> was present.
-  # However, to print pretty messages, we make use of metavar and help.
-  p.add_argument('args', nargs='*', metavar='<commit>',
-                 help='revision from which to compute the diff')
-  p.add_argument('ignored', nargs='*', metavar='<file>...',
-                 help='if specified, only consider differences in these files')
-  opts = p.parse_args(argv)
-
-  opts.verbose -= opts.quiet
-  del opts.quiet
-
-  commits, files = interpret_args(opts.args, dash_dash, opts.commit)
-  if len(commits) > 2:
-    die('at most two commits allowed; %d given' % len(commits))
-  if len(commits) == 2:
-    if opts.staged:
-      die('--staged is not allowed when two commits are given')
-    if not opts.diff:
-      die('--diff is required when two commits are given')
-  elif opts.diff_from_common_commit:
-    die('--diff_from_common_commit is only allowed when two commits are given')
-
-  if os.path.dirname(opts.binary):
-    opts.binary = os.path.abspath(opts.binary)
-
-  changed_lines = compute_diff_and_extract_lines(commits,
-                                                 files,
-                                                 opts.staged,
-                                                 opts.diff_from_common_commit)
-  if opts.verbose >= 1:
-    ignored_files = set(changed_lines)
-  filter_by_extension(changed_lines, opts.extensions.lower().split(','))
-  # The computed diff outputs absolute paths, so we must cd before accessing
-  # those files.
-  cd_to_toplevel()
-  filter_symlinks(changed_lines)
-  filter_ignored_files(changed_lines, binary=opts.binary)
-  if opts.verbose >= 1:
-    ignored_files.difference_update(changed_lines)
-    if ignored_files:
-      print('Ignoring the following files (wrong extension, symlink, or '
-            'ignored by clang-format):')
-      for filename in ignored_files:
-        print('    %s' % filename)
-    if changed_lines:
-      print('Running clang-format on the following files:')
-      for filename in changed_lines:
-        print('    %s' % filename)
-
-  if not changed_lines:
-    if opts.verbose >= 0:
-      print('no modified files to format')
-    return 0
-
-  if len(commits) > 1:
-    old_tree = commits[1]
-    revision = old_tree
-  elif opts.staged:
-    old_tree = create_tree_from_index(changed_lines)
-    revision = ''
-  else:
-    old_tree = create_tree_from_workdir(changed_lines)
-    revision = None
-  new_tree = run_clang_format_and_save_to_tree(changed_lines,
-                                               revision,
-                                               binary=opts.binary,
-                                               style=opts.style)
-  if opts.verbose >= 1:
-    print('old tree: %s' % old_tree)
-    print('new tree: %s' % new_tree)
-
-  if old_tree == new_tree:
-    if opts.verbose >= 0:
-      print('clang-format did not modify any files')
-    return 0
-
-  if opts.diff:
-    return print_diff(old_tree, new_tree)
-  if opts.diffstat:
-    return print_diffstat(old_tree, new_tree)
-
-  changed_files = apply_changes(old_tree, new_tree, force=opts.force,
-                                patch_mode=opts.patch)
-  if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1:
-    print('changed files:')
-    for filename in changed_files:
-      print('    %s' % filename)
-
-  return 1
+    config = load_git_config()
+
+    # In order to keep '--' yet allow options after positionals, we need to
+    # check for '--' ourselves.  (Setting nargs='*' throws away the '--', while
+    # nargs=argparse.REMAINDER disallows options after positionals.)
+    argv = sys.argv[1:]
+    try:
+        idx = argv.index("--")
+    except ValueError:
+        dash_dash = []
+    else:
+        dash_dash = argv[idx:]
+        argv = argv[:idx]
+
+    default_extensions = ",".join(
+        [
+            # From clang/lib/Frontend/FrontendOptions.cpp, all lower case
+            "c",
+            "h",  # C
+            "m",  # ObjC
+            "mm",  # ObjC++
+            "cc",
+            "cp",
+            "cpp",
+            "c++",
+            "cxx",
+            "hh",
+            "hpp",
+            "hxx",
+            "inc",  # C++
+            "ccm",
+            "cppm",
+            "cxxm",
+            "c++m",  # C++ Modules
+            "cu",
+            "cuh",  # CUDA
+            # Other languages that clang-format supports
+            "proto",
+            "protodevel",  # Protocol Buffers
+            "java",  # Java
+            "js",
+            "mjs",
+            "cjs",  # JavaScript
+            "ts",  # TypeScript
+            "cs",  # C Sharp
+            "json",  # Json
+            "sv",
+            "svh",
+            "v",
+            "vh",  # Verilog
+            "td",  # TableGen
+            "txtpb",
+            "textpb",
+            "pb.txt",
+            "textproto",
+            "asciipb",  # TextProto
+        ]
+    )
+
+    p = argparse.ArgumentParser(
+        usage=usage,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=desc,
+    )
+    p.add_argument(
+        "--binary",
+        default=config.get("clangformat.binary", "clang-format"),
+        help="path to clang-format",
+    ),
+    p.add_argument(
+        "--commit",
+        default=config.get("clangformat.commit", "HEAD"),
+        help="default commit to use if none is specified",
+    ),
+    p.add_argument(
+        "--diff",
+        action="store_true",
+        help="print a diff instead of applying the changes",
+    )
+    p.add_argument(
+        "--diffstat",
+        action="store_true",
+        help="print a diffstat instead of applying the changes",
+    )
+    p.add_argument(
+        "--extensions",
+        default=config.get("clangformat.extensions", default_extensions),
+        help=(
+            "comma-separated list of file extensions to format, "
+            "excluding the period and case-insensitive"
+        ),
+    ),
+    p.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="allow changes to unstaged files",
+    )
+    p.add_argument(
+        "-p", "--patch", action="store_true", help="select hunks interactively"
+    )
+    p.add_argument(
+        "-q",
+        "--quiet",
+        action="count",
+        default=0,
+        help="print less information",
+    )
+    p.add_argument(
+        "--staged",
+        "--cached",
+        action="store_true",
+        help="format lines in the stage instead of the working dir",
+    )
+    p.add_argument(
+        "--style",
+        default=config.get("clangformat.style", None),
+        help="passed to clang-format",
+    ),
+    p.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="print extra information",
+    )
+    p.add_argument(
+        "--diff_from_common_commit",
+        action="store_true",
+        help=(
+            "diff from the last common commit for commits in "
+            "separate branches rather than the exact point of the "
+            "commits"
+        ),
+    )
+    # We gather all the remaining positional arguments into 'args' since we need
+    # to use some heuristics to determine whether or not <commit> was present.
+    # However, to print pretty messages, we make use of metavar and help.
+    p.add_argument(
+        "args",
+        nargs="*",
+        metavar="<commit>",
+        help="revision from which to compute the diff",
+    )
+    p.add_argument(
+        "ignored",
+        nargs="*",
+        metavar="<file>...",
+        help="if specified, only consider differences in these files",
+    )
+    opts = p.parse_args(argv)
+
+    opts.verbose -= opts.quiet
+    del opts.quiet
+
+    commits, files = interpret_args(opts.args, dash_dash, opts.commit)
+    if len(commits) > 2:
+        die("at most two commits allowed; %d given" % len(commits))
+    if len(commits) == 2:
+        if opts.staged:
+            die("--staged is not allowed when two commits are given")
+        if not opts.diff:
+            die("--diff is required when two commits are given")
+    elif opts.diff_from_common_commit:
+        die(
+            "--diff_from_common_commit is only allowed when two commits are "
+            "given"
+        )
+
+    if os.path.dirname(opts.binary):
+        opts.binary = os.path.abspath(opts.binary)
+
+    changed_lines = compute_diff_and_extract_lines(
+        commits, files, opts.staged, opts.diff_from_common_commit
+    )
+    if opts.verbose >= 1:
+        ignored_files = set(changed_lines)
+    filter_by_extension(changed_lines, opts.extensions.lower().split(","))
+    # The computed diff outputs absolute paths, so we must cd before accessing
+    # those files.
+    cd_to_toplevel()
+    filter_symlinks(changed_lines)
+    filter_ignored_files(changed_lines, binary=opts.binary)
+    if opts.verbose >= 1:
+        ignored_files.difference_update(changed_lines)
+        if ignored_files:
+            print(
+                "Ignoring the following files (wrong extension, symlink, or "
+                "ignored by clang-format):"
+            )
+            for filename in ignored_files:
+                print("    %s" % filename)
+        if changed_lines:
+            print("Running clang-format on the following files:")
+            for filename in changed_lines:
+                print("    %s" % filename)
+
+    if not changed_lines:
+        if opts.verbose >= 0:
+            print("no modified files to format")
+        return 0
+
+    if len(commits) > 1:
+        old_tree = commits[1]
+        revision = old_tree
+    elif opts.staged:
+        old_tree = create_tree_from_index(changed_lines)
+        revision = ""
+    else:
+        old_tree = create_tree_from_workdir(changed_lines)
+        revision = None
+    new_tree = run_clang_format_and_save_to_tree(
+        changed_lines, revision, binary=opts.binary, style=opts.style
+    )
+    if opts.verbose >= 1:
+        print("old tree: %s" % old_tree)
+        print("new tree: %s" % new_tree)
+
+    if old_tree == new_tree:
+        if opts.verbose >= 0:
+            print("clang-format did not modify any files")
+        return 0
+
+    if opts.diff:
+        return print_diff(old_tree, new_tree)
+    if opts.diffstat:
+        return print_diffstat(old_tree, new_tree)
+
+    changed_files = apply_changes(
+        old_tree, new_tree, force=opts.force, patch_mode=opts.patch
+    )
+    if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1:
+        print("changed files:")
+        for filename in changed_files:
+            print("    %s" % filename)
+
+    return 1
 
 
 def load_git_config(non_string_options=None):
-  """Return the git configuration as a dictionary.
-
-  All options are assumed to be strings unless in `non_string_options`, in which
-  is a dictionary mapping option name (in lower case) to either "--bool" or
-  "--int"."""
-  if non_string_options is None:
-    non_string_options = {}
-  out = {}
-  for entry in run('git', 'config', '--list', '--null').split('\0'):
-    if entry:
-      if '\n' in entry:
-        name, value = entry.split('\n', 1)
-      else:
-        # A setting with no '=' ('\n' with --null) is implicitly 'true'
-        name = entry
-        value = 'true'
-      if name in non_string_options:
-        value = run('git', 'config', non_string_options[name], name)
-      out[name] = value
-  return out
+    """Return the git configuration as a dictionary.
+
+    All options are assumed to be strings unless in `non_string_options`, in
+    which is a dictionary mapping option name (in lower case) to either "--bool"
+    or "--int"."""
+    if non_string_options is None:
+        non_string_options = {}
+    out = {}
+    for entry in run("git", "config", "--list", "--null").split("\0"):
+        if entry:
+            if "\n" in entry:
+                name, value = entry.split("\n", 1)
+            else:
+                # A setting with no '=' ('\n' with --null) is implicitly 'true'
+                name = entry
+                value = "true"
+            if name in non_string_options:
+                value = run("git", "config", non_string_options[name], name)
+            out[name] = value
+    return out
 
 
 def interpret_args(args, dash_dash, default_commit):
-  """Interpret `args` as "[commits] [--] [files]" and return (commits, files).
-
-  It is assumed that "--" and everything that follows has been removed from
-  args and placed in `dash_dash`.
-
-  If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its
-  left (if present) are taken as commits.  Otherwise, the arguments are checked
-  from left to right if they are commits or files.  If commits are not given,
-  a list with `default_commit` is used."""
-  if dash_dash:
-    if len(args) == 0:
-      commits = [default_commit]
-    else:
-      commits = args
-    for commit in commits:
-      object_type = get_object_type(commit)
-      if object_type not in ('commit', 'tag'):
-        if object_type is None:
-          die("'%s' is not a commit" % commit)
+    """Interpret `args` as "[commits] [--] [files]" and return (commits, files).
+
+    It is assumed that "--" and everything that follows has been removed from
+    args and placed in `dash_dash`.
+
+    If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its
+    left (if present) are taken as commits.  Otherwise, the arguments are
+    checked from left to right if they are commits or files.  If commits are not
+    given, a list with `default_commit` is used."""
+    if dash_dash:
+        if len(args) == 0:
+            commits = [default_commit]
         else:
-          die("'%s' is a %s, but a commit was expected" % (commit, object_type))
-    files = dash_dash[1:]
-  elif args:
-    commits = []
-    while args:
-      if not disambiguate_revision(args[0]):
-        break
-      commits.append(args.pop(0))
-    if not commits:
-      commits = [default_commit]
-    files = args
-  else:
-    commits = [default_commit]
-    files = []
-  return commits, files
+            commits = args
+        for commit in commits:
+            object_type = get_object_type(commit)
+            if object_type not in ("commit", "tag"):
+                if object_type is None:
+                    die("'%s' is not a commit" % commit)
+                else:
+                    die(
+                        "'%s' is a %s, but a commit was expected"
+                        % (commit, object_type)
+                    )
+        files = dash_dash[1:]
+    elif args:
+        commits = []
+        while args:
+            if not disambiguate_revision(args[0]):
+                break
+            commits.append(args.pop(0))
+        if not commits:
+            commits = [default_commit]
+        files = args
+    else:
+        commits = [default_commit]
+        files = []
+    return commits, files
 
 
 def disambiguate_revision(value):
-  """Returns True if `value` is a revision, False if it is a file, or dies."""
-  # If `value` is ambiguous (neither a commit nor a file), the following
-  # command will die with an appropriate error message.
-  run('git', 'rev-parse', value, verbose=False)
-  object_type = get_object_type(value)
-  if object_type is None:
-    return False
-  if object_type in ('commit', 'tag'):
-    return True
-  die('`%s` is a %s, but a commit or filename was expected' %
-      (value, object_type))
+    """Returns True if `value` is a revision, False if it is a file, or dies."""
+    # If `value` is ambiguous (neither a commit nor a file), the following
+    # command will die with an appropriate error message.
+    run("git", "rev-parse", value, verbose=False)
+    object_type = get_object_type(value)
+    if object_type is None:
+        return False
+    if object_type in ("commit", "tag"):
+        return True
+    die(
+        "`%s` is a %s, but a commit or filename was expected"
+        % (value, object_type)
+    )
 
 
 def get_object_type(value):
-  """Returns a string description of an object's type, or None if it is not
-  a valid git object."""
-  cmd = ['git', 'cat-file', '-t', value]
-  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = p.communicate()
-  if p.returncode != 0:
-    return None
-  return convert_string(stdout.strip())
+    """Returns a string description of an object's type, or None if it is not
+    a valid git object."""
+    cmd = ["git", "cat-file", "-t", value]
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    if p.returncode != 0:
+        return None
+    return convert_string(stdout.strip())
 
 
 def compute_diff_and_extract_lines(commits, files, staged, diff_common_commit):
-  """Calls compute_diff() followed by extract_lines()."""
-  diff_process = compute_diff(commits, files, staged, diff_common_commit)
-  changed_lines = extract_lines(diff_process.stdout)
-  diff_process.stdout.close()
-  diff_process.wait()
-  if diff_process.returncode != 0:
-    # Assume error was already printed to stderr.
-    sys.exit(2)
-  return changed_lines
+    """Calls compute_diff() followed by extract_lines()."""
+    diff_process = compute_diff(commits, files, staged, diff_common_commit)
+    changed_lines = extract_lines(diff_process.stdout)
+    diff_process.stdout.close()
+    diff_process.wait()
+    if diff_process.returncode != 0:
+        # Assume error was already printed to stderr.
+        sys.exit(2)
+    return changed_lines
 
 
 def compute_diff(commits, files, staged, diff_common_commit):
-  """Return a subprocess object producing the diff from `commits`.
-
-  The return value's `stdin` file object will produce a patch with the
-  differences between the working directory (or stage if --staged is used) and
-  the first commit if a single one was specified, or the difference between
-  both specified commits, filtered on `files` (if non-empty).
-  Zero context lines are used in the patch."""
-  git_tool = 'diff-index'
-  extra_args = []
-  if len(commits) == 2:
-    git_tool = 'diff-tree'
-    if diff_common_commit:
-      commits = [f'{commits[0]}...{commits[1]}']
-  elif staged:
-    extra_args += ['--cached']
-
-  cmd = ['git', git_tool, '-p', '-U0'] + extra_args + commits + ['--']
-  cmd.extend(files)
-  p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-  p.stdin.close()
-  return p
+    """Return a subprocess object producing the diff from `commits`.
+
+    The return value's `stdin` file object will produce a patch with the
+    differences between the working directory (or stage if --staged is used) and
+    the first commit if a single one was specified, or the difference between
+    both specified commits, filtered on `files` (if non-empty).
+    Zero context lines are used in the patch."""
+    git_tool = "diff-index"
+    extra_args = []
+    if len(commits) == 2:
+        git_tool = "diff-tree"
+        if diff_common_commit:
+            commits = [f"{commits[0]}...{commits[1]}"]
+    elif staged:
+        extra_args += ["--cached"]
+
+    cmd = ["git", git_tool, "-p", "-U0"] + extra_args + commits + ["--"]
+    cmd.extend(files)
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    p.stdin.close()
+    return p
 
 
 def extract_lines(patch_file):
-  """Extract the changed lines in `patch_file`.
-
-  The return value is a dictionary mapping filename to a list of (start_line,
-  line_count) pairs.
-
-  The input must have been produced with ``-U0``, meaning unidiff format with
-  zero lines of context.  The return value is a dict mapping filename to a
-  list of line `Range`s."""
-  matches = {}
-  for line in patch_file:
-    line = convert_string(line)
-    match = re.search(r'^\+\+\+\ [^/]+/(.*)', line)
-    if match:
-      filename = match.group(1).rstrip('\r\n\t')
-    match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line)
-    if match:
-      start_line = int(match.group(1))
-      line_count = 1
-      if match.group(3):
-        line_count = int(match.group(3))
-      if line_count == 0:
-        line_count = 1
-      if start_line == 0:
-        continue
-      matches.setdefault(filename, []).append(Range(start_line, line_count))
-  return matches
+    """Extract the changed lines in `patch_file`.
+
+    The return value is a dictionary mapping filename to a list of (start_line,
+    line_count) pairs.
+
+    The input must have been produced with ``-U0``, meaning unidiff format with
+    zero lines of context.  The return value is a dict mapping filename to a
+    list of line `Range`s."""
+    matches = {}
+    for line in patch_file:
+        line = convert_string(line)
+        match = re.search(r"^\+\+\+\ [^/]+/(.*)", line)
+        if match:
+            filename = match.group(1).rstrip("\r\n\t")
+        match = re.search(r"^@@ -[0-9,]+ \+(\d+)(,(\d+))?", line)
+        if match:
+            start_line = int(match.group(1))
+            line_count = 1
+            if match.group(3):
+                line_count = int(match.group(3))
+            if line_count == 0:
+                line_count = 1
+            if start_line == 0:
+                continue
+            matches.setdefault(filename, []).append(
+                Range(start_line, line_count)
+            )
+    return matches
 
 
 def filter_by_extension(dictionary, allowed_extensions):
-  """Delete every key in `dictionary` that doesn't have an allowed extension.
+    """Delete every key in `dictionary` that doesn't have an allowed extension.
 
-  `allowed_extensions` must be a collection of lowercase file extensions,
-  excluding the period."""
-  allowed_extensions = frozenset(allowed_extensions)
-  for filename in list(dictionary.keys()):
-    base_ext = filename.rsplit('.', 1)
-    if len(base_ext) == 1 and '' in allowed_extensions:
-        continue
-    if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions:
-      del dictionary[filename]
+    `allowed_extensions` must be a collection of lowercase file extensions,
+    excluding the period."""
+    allowed_extensions = frozenset(allowed_extensions)
+    for filename in list(dictionary.keys()):
+        base_ext = filename.rsplit(".", 1)
+        if len(base_ext) == 1 and "" in allowed_extensions:
+            continue
+        if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions:
+            del dictionary[filename]
 
 
 def filter_symlinks(dictionary):
-  """Delete every key in `dictionary` that is a symlink."""
-  for filename in list(dictionary.keys()):
-    if os.path.islink(filename):
-      del dictionary[filename]
+    """Delete every key in `dictionary` that is a symlink."""
+    for filename in list(dictionary.keys()):
+        if os.path.islink(filename):
+            del dictionary[filename]
 
 
 def filter_ignored_files(dictionary, binary):
-  """Delete every key in `dictionary` that is ignored by clang-format."""
-  ignored_files = run(binary, '-list-ignored', *dictionary.keys())
-  if not ignored_files:
-    return
-  ignored_files = ignored_files.split('\n')
-  for filename in ignored_files:
-    del dictionary[filename]
+    """Delete every key in `dictionary` that is ignored by clang-format."""
+    ignored_files = run(binary, "-list-ignored", *dictionary.keys())
+    if not ignored_files:
+        return
+    ignored_files = ignored_files.split("\n")
+    for filename in ignored_files:
+        del dictionary[filename]
 
 
 def cd_to_toplevel():
-  """Change to the top level of the git repository."""
-  toplevel = run('git', 'rev-parse', '--show-toplevel')
-  os.chdir(toplevel)
+    """Change to the top level of the git repository."""
+    toplevel = run("git", "rev-parse", "--show-toplevel")
+    os.chdir(toplevel)
 
 
 def create_tree_from_workdir(filenames):
-  """Create a new git tree with the given files from the working directory.
+    """Create a new git tree with the given files from the working directory.
 
-  Returns the object ID (SHA-1) of the created tree."""
-  return create_tree(filenames, '--stdin')
+    Returns the object ID (SHA-1) of the created tree."""
+    return create_tree(filenames, "--stdin")
 
 
 def create_tree_from_index(filenames):
-  # Copy the environment, because the files have to be read from the original
-  # index.
-  env = os.environ.copy()
-  def index_contents_generator():
-    for filename in filenames:
-      git_ls_files_cmd = ['git', 'ls-files', '--stage', '-z', '--', filename]
-      git_ls_files = subprocess.Popen(git_ls_files_cmd, env=env,
-                                      stdin=subprocess.PIPE,
-                                      stdout=subprocess.PIPE)
-      stdout = git_ls_files.communicate()[0]
-      yield convert_string(stdout.split(b'\0')[0])
-  return create_tree(index_contents_generator(), '--index-info')
-
-
-def run_clang_format_and_save_to_tree(changed_lines, revision=None,
-                                      binary='clang-format', style=None):
-  """Run clang-format on each file and save the result to a git tree.
-
-  Returns the object ID (SHA-1) of the created tree."""
-  # Copy the environment when formatting the files in the index, because the
-  # files have to be read from the original index.
-  env = os.environ.copy() if revision == '' else None
-  def iteritems(container):
-      try:
-          return container.iteritems() # Python 2
-      except AttributeError:
-          return container.items() # Python 3
-  def index_info_generator():
-    for filename, line_ranges in iteritems(changed_lines):
-      if revision is not None:
-        if len(revision) > 0:
-          git_metadata_cmd = ['git', 'ls-tree',
-                              '%s:%s' % (revision, os.path.dirname(filename)),
-                              os.path.basename(filename)]
-        else:
-          git_metadata_cmd = ['git', 'ls-files', '--stage', '--', filename]
-        git_metadata = subprocess.Popen(git_metadata_cmd, env=env,
-                                        stdin=subprocess.PIPE,
-                                        stdout=subprocess.PIPE)
-        stdout = git_metadata.communicate()[0]
-        mode = oct(int(stdout.split()[0], 8))
-      else:
-        mode = oct(os.stat(filename).st_mode)
-      # Adjust python3 octal format so that it matches what git expects
-      if mode.startswith('0o'):
-          mode = '0' + mode[2:]
-      blob_id = clang_format_to_blob(filename, line_ranges,
-                                     revision=revision,
-                                     binary=binary,
-                                     style=style,
-                                     env=env)
-      yield '%s %s\t%s' % (mode, blob_id, filename)
-  return create_tree(index_info_generator(), '--index-info')
+    # Copy the environment, because the files have to be read from the original
+    # index.
+    env = os.environ.copy()
+
+    def index_contents_generator():
+        for filename in filenames:
+            git_ls_files_cmd = [
+                "git",
+                "ls-files",
+                "--stage",
+                "-z",
+                "--",
+                filename,
+            ]
+            git_ls_files = subprocess.Popen(
+                git_ls_files_cmd,
+                env=env,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
+            stdout = git_ls_files.communicate()[0]
+            yield convert_string(stdout.split(b"\0")[0])
+
+    return create_tree(index_contents_generator(), "--index-info")
+
+
+def run_clang_format_and_save_to_tree(
+    changed_lines, revision=None, binary="clang-format", style=None
+):
+    """Run clang-format on each file and save the result to a git tree.
+
+    Returns the object ID (SHA-1) of the created tree."""
+    # Copy the environment when formatting the files in the index, because the
+    # files have to be read from the original index.
+    env = os.environ.copy() if revision == "" else None
+
+    def iteritems(container):
+        try:
+            return container.iteritems()  # Python 2
+        except AttributeError:
+            return container.items()  # Python 3
+
+    def index_info_generator():
+        for filename, line_ranges in iteritems(changed_lines):
+            if revision is not None:
+                if len(revision) > 0:
+                    git_metadata_cmd = [
+                        "git",
+                        "ls-tree",
+                        "%s:%s" % (revision, os.path.dirname(filename)),
+                        os.path.basename(filename),
+                    ]
+                else:
+                    git_metadata_cmd = [
+                        "git",
+                        "ls-files",
+                        "--stage",
+                        "--",
+                        filename,
+                    ]
+                git_metadata = subprocess.Popen(
+                    git_metadata_cmd,
+                    env=env,
+                    stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE,
+                )
+                stdout = git_metadata.communicate()[0]
+                mode = oct(int(stdout.split()[0], 8))
+            else:
+                mode = oct(os.stat(filename).st_mode)
+            # Adjust python3 octal format so that it matches what git expects
+            if mode.startswith("0o"):
+                mode = "0" + mode[2:]
+            blob_id = clang_format_to_blob(
+                filename,
+                line_ranges,
+                revision=revision,
+                binary=binary,
+                style=style,
+                env=env,
+            )
+            yield "%s %s\t%s" % (mode, blob_id, filename)
+
+    return create_tree(index_info_generator(), "--index-info")
 
 
 def create_tree(input_lines, mode):
-  """Create a tree object from the given input.
-
-  If mode is '--stdin', it must be a list of filenames.  If mode is
-  '--index-info' is must be a list of values suitable for "git update-index
-  --index-info", such as "<mode> <SP> <sha1> <TAB> <filename>".  Any other mode
-  is invalid."""
-  assert mode in ('--stdin', '--index-info')
-  cmd = ['git', 'update-index', '--add', '-z', mode]
-  with temporary_index_file():
-    p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
-    for line in input_lines:
-      p.stdin.write(to_bytes('%s\0' % line))
-    p.stdin.close()
-    if p.wait() != 0:
-      die('`%s` failed' % ' '.join(cmd))
-    tree_id = run('git', 'write-tree')
-    return tree_id
-
-
-def clang_format_to_blob(filename, line_ranges, revision=None,
-                         binary='clang-format', style=None, env=None):
-  """Run clang-format on the given file and save the result to a git blob.
-
-  Runs on the file in `revision` if not None, or on the file in the working
-  directory if `revision` is None. Revision can be set to an empty string to run
-  clang-format on the file in the index.
-
-  Returns the object ID (SHA-1) of the created blob."""
-  clang_format_cmd = [binary]
-  if style:
-    clang_format_cmd.extend(['--style='+style])
-  clang_format_cmd.extend([
-      '--lines=%s:%s' % (start_line, start_line+line_count-1)
-      for start_line, line_count in line_ranges])
-  if revision is not None:
-    clang_format_cmd.extend(['--assume-filename='+filename])
-    git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)]
-    git_show = subprocess.Popen(git_show_cmd, env=env, stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
-    git_show.stdin.close()
-    clang_format_stdin = git_show.stdout
-  else:
-    clang_format_cmd.extend([filename])
-    git_show = None
-    clang_format_stdin = subprocess.PIPE
-  try:
-    clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin,
-                                    stdout=subprocess.PIPE)
-    if clang_format_stdin == subprocess.PIPE:
-      clang_format_stdin = clang_format.stdin
-  except OSError as e:
-    if e.errno == errno.ENOENT:
-      die('cannot find executable "%s"' % binary)
+    """Create a tree object from the given input.
+
+    If mode is '--stdin', it must be a list of filenames.  If mode is
+    '--index-info' is must be a list of values suitable for "git update-index
+    --index-info", such as "<mode> <SP> <sha1> <TAB> <filename>".  Any other
+    mode is invalid."""
+    assert mode in ("--stdin", "--index-info")
+    cmd = ["git", "update-index", "--add", "-z", mode]
+    with temporary_index_file():
+        p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+        for line in input_lines:
+            p.stdin.write(to_bytes("%s\0" % line))
+        p.stdin.close()
+        if p.wait() != 0:
+            die("`%s` failed" % " ".join(cmd))
+        tree_id = run("git", "write-tree")
+        return tree_id
+
+
+def clang_format_to_blob(
+    filename,
+    line_ranges,
+    revision=None,
+    binary="clang-format",
+    style=None,
+    env=None,
+):
+    """Run clang-format on the given file and save the result to a git blob.
+
+    Runs on the file in `revision` if not None, or on the file in the working
+    directory if `revision` is None. Revision can be set to an empty string to
+    run clang-format on the file in the index.
+
+    Returns the object ID (SHA-1) of the created blob."""
+    clang_format_cmd = [binary]
+    if style:
+        clang_format_cmd.extend(["--style=" + style])
+    clang_format_cmd.extend(
+        [
+            "--lines=%s:%s" % (start_line, start_line + line_count - 1)
+            for start_line, line_count in line_ranges
+        ]
+    )
+    if revision is not None:
+        clang_format_cmd.extend(["--assume-filename=" + filename])
+        git_show_cmd = [
+            "git",
+            "cat-file",
+            "blob",
+            "%s:%s" % (revision, filename),
+        ]
+        git_show = subprocess.Popen(
+            git_show_cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
+        git_show.stdin.close()
+        clang_format_stdin = git_show.stdout
     else:
-      raise
-  clang_format_stdin.close()
-  hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin']
-  hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout,
-                                 stdout=subprocess.PIPE)
-  clang_format.stdout.close()
-  stdout = hash_object.communicate()[0]
-  if hash_object.returncode != 0:
-    die('`%s` failed' % ' '.join(hash_object_cmd))
-  if clang_format.wait() != 0:
-    die('`%s` failed' % ' '.join(clang_format_cmd))
-  if git_show and git_show.wait() != 0:
-    die('`%s` failed' % ' '.join(git_show_cmd))
-  return convert_string(stdout).rstrip('\r\n')
+        clang_format_cmd.extend([filename])
+        git_show = None
+        clang_format_stdin = subprocess.PIPE
+    try:
+        clang_format = subprocess.Popen(
+            clang_format_cmd, stdin=clang_format_stdin, stdout=subprocess.PIPE
+        )
+        if clang_format_stdin == subprocess.PIPE:
+            clang_format_stdin = clang_format.stdin
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            die('cannot find executable "%s"' % binary)
+        else:
+            raise
+    clang_format_stdin.close()
+    hash_object_cmd = [
+        "git",
+        "hash-object",
+        "-w",
+        "--path=" + filename,
+        "--stdin",
+    ]
+    hash_object = subprocess.Popen(
+        hash_object_cmd, stdin=clang_format.stdout, stdout=subprocess.PIPE
+    )
+    clang_format.stdout.close()
+    stdout = hash_object.communicate()[0]
+    if hash_object.returncode != 0:
+        die("`%s` failed" % " ".join(hash_object_cmd))
+    if clang_format.wait() != 0:
+        die("`%s` failed" % " ".join(clang_format_cmd))
+    if git_show and git_show.wait() != 0:
+        die("`%s` failed" % " ".join(git_show_cmd))
+    return convert_string(stdout).rstrip("\r\n")
 
 
 @contextlib.contextmanager
 def temporary_index_file(tree=None):
-  """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting
-  the file afterward."""
-  index_path = create_temporary_index(tree)
-  old_index_path = os.environ.get('GIT_INDEX_FILE')
-  os.environ['GIT_INDEX_FILE'] = index_path
-  try:
-    yield
-  finally:
-    if old_index_path is None:
-      del os.environ['GIT_INDEX_FILE']
-    else:
-      os.environ['GIT_INDEX_FILE'] = old_index_path
-    os.remove(index_path)
+    """Context manager for setting GIT_INDEX_FILE to a temporary file and
+    deleting the file afterward."""
+    index_path = create_temporary_index(tree)
+    old_index_path = os.environ.get("GIT_INDEX_FILE")
+    os.environ["GIT_INDEX_FILE"] = index_path
+    try:
+        yield
+    finally:
+        if old_index_path is None:
+            del os.environ["GIT_INDEX_FILE"]
+        else:
+            os.environ["GIT_INDEX_FILE"] = old_index_path
+        os.remove(index_path)
 
 
 def create_temporary_index(tree=None):
-  """Create a temporary index file and return the created file's path.
+    """Create a temporary index file and return the created file's path.
 
-  If `tree` is not None, use that as the tree to read in.  Otherwise, an
-  empty index is created."""
-  gitdir = run('git', 'rev-parse', '--git-dir')
-  path = os.path.join(gitdir, temp_index_basename)
-  if tree is None:
-    tree = '--empty'
-  run('git', 'read-tree', '--index-output='+path, tree)
-  return path
+    If `tree` is not None, use that as the tree to read in.  Otherwise, an
+    empty index is created."""
+    gitdir = run("git", "rev-parse", "--git-dir")
+    path = os.path.join(gitdir, temp_index_basename)
+    if tree is None:
+        tree = "--empty"
+    run("git", "read-tree", "--index-output=" + path, tree)
+    return path
 
 
 def print_diff(old_tree, new_tree):
-  """Print the diff between the two trees to stdout."""
-  # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output
-  # is expected to be viewed by the user, and only the former does nice things
-  # like color and pagination.
-  #
-  # We also only print modified files since `new_tree` only contains the files
-  # that were modified, so unmodified files would show as deleted without the
-  # filter.
-  return subprocess.run(['git', 'diff', '--diff-filter=M',
-                         '--exit-code', old_tree, new_tree]).returncode
+    """Print the diff between the two trees to stdout."""
+    # We use the porcelain 'diff' and not plumbing 'diff-tree' because the
+    # output is expected to be viewed by the user, and only the former does nice
+    # things like color and pagination.
+    #
+    # We also only print modified files since `new_tree` only contains the files
+    # that were modified, so unmodified files would show as deleted without the
+    # filter.
+    return subprocess.run(
+        ["git", "diff", "--diff-filter=M", "--exit-code", old_tree, new_tree]
+    ).returncode
+
 
 def print_diffstat(old_tree, new_tree):
-  """Print the diffstat between the two trees to stdout."""
-  # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output
-  # is expected to be viewed by the user, and only the former does nice things
-  # like color and pagination.
-  #
-  # We also only print modified files since `new_tree` only contains the files
-  # that were modified, so unmodified files would show as deleted without the
-  # filter.
-  return subprocess.run(['git', 'diff', '--diff-filter=M', '--exit-code',
-                         '--stat', old_tree, new_tree]).returncode
+    """Print the diffstat between the two trees to stdout."""
+    # We use the porcelain 'diff' and not plumbing 'diff-tree' because the
+    # output is expected to be viewed by the user, and only the former does nice
+    # things like color and pagination.
+    #
+    # We also only print modified files since `new_tree` only contains the files
+    # that were modified, so unmodified files would show as deleted without the
+    # filter.
+    return subprocess.run(
+        [
+            "git",
+            "diff",
+            "--diff-filter=M",
+            "--exit-code",
+            "--stat",
+            old_tree,
+            new_tree,
+        ]
+    ).returncode
+
 
 def apply_changes(old_tree, new_tree, force=False, patch_mode=False):
-  """Apply the changes in `new_tree` to the working directory.
-
-  Bails if there are local changes in those files and not `force`.  If
-  `patch_mode`, runs `git checkout --patch` to select hunks interactively."""
-  changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z',
-                      '--name-only', old_tree,
-                      new_tree).rstrip('\0').split('\0')
-  if not force:
-    unstaged_files = run('git', 'diff-files', '--name-status', *changed_files)
-    if unstaged_files:
-      print('The following files would be modified but '
-                'have unstaged changes:', file=sys.stderr)
-      print(unstaged_files, file=sys.stderr)
-      print('Please commit, stage, or stash them first.', file=sys.stderr)
-      sys.exit(2)
-  if patch_mode:
-    # In patch mode, we could just as well create an index from the new tree
-    # and checkout from that, but then the user will be presented with a
-    # message saying "Discard ... from worktree".  Instead, we use the old
-    # tree as the index and checkout from new_tree, which gives the slightly
-    # better message, "Apply ... to index and worktree".  This is not quite
-    # right, since it won't be applied to the user's index, but oh well.
-    with temporary_index_file(old_tree):
-      subprocess.run(['git', 'checkout', '--patch', new_tree], check=True)
-    index_tree = old_tree
-  else:
-    with temporary_index_file(new_tree):
-      run('git', 'checkout-index', '-f', '--', *changed_files)
-  return changed_files
+    """Apply the changes in `new_tree` to the working directory.
+
+    Bails if there are local changes in those files and not `force`.  If
+    `patch_mode`, runs `git checkout --patch` to select hunks interactively."""
+    changed_files = (
+        run(
+            "git",
+            "diff-tree",
+            "--diff-filter=M",
+            "-r",
+            "-z",
+            "--name-only",
+            old_tree,
+            new_tree,
+        )
+        .rstrip("\0")
+        .split("\0")
+    )
+    if not force:
+        unstaged_files = run(
+            "git", "diff-files", "--name-status", *changed_files
+        )
+        if unstaged_files:
+            print(
+                "The following files would be modified but have unstaged "
+                "changes:",
+                file=sys.stderr,
+            )
+            print(unstaged_files, file=sys.stderr)
+            print("Please commit, stage, or stash them first.", file=sys.stderr)
+            sys.exit(2)
+    if patch_mode:
+        # In patch mode, we could just as well create an index from the new tree
+        # and checkout from that, but then the user will be presented with a
+        # message saying "Discard ... from worktree".  Instead, we use the old
+        # tree as the index and checkout from new_tree, which gives the slightly
+        # better message, "Apply ... to index and worktree".  This is not quite
+        # right, since it won't be applied to the user's index, but oh well.
+        with temporary_index_file(old_tree):
+            subprocess.run(["git", "checkout", "--patch", new_tree], check=True)
+        index_tree = old_tree
+    else:
+        with temporary_index_file(new_tree):
+            run("git", "checkout-index", "-f", "--", *changed_files)
+    return changed_files
 
 
 def run(*args, **kwargs):
-  stdin = kwargs.pop('stdin', '')
-  verbose = kwargs.pop('verbose', True)
-  strip = kwargs.pop('strip', True)
-  for name in kwargs:
-    raise TypeError("run() got an unexpected keyword argument '%s'" % name)
-  p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                       stdin=subprocess.PIPE)
-  stdout, stderr = p.communicate(input=stdin)
-
-  stdout = convert_string(stdout)
-  stderr = convert_string(stderr)
-
-  if p.returncode == 0:
+    stdin = kwargs.pop("stdin", "")
+    verbose = kwargs.pop("verbose", True)
+    strip = kwargs.pop("strip", True)
+    for name in kwargs:
+        raise TypeError("run() got an unexpected keyword argument '%s'" % name)
+    p = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+    )
+    stdout, stderr = p.communicate(input=stdin)
+
+    stdout = convert_string(stdout)
+    stderr = convert_string(stderr)
+
+    if p.returncode == 0:
+        if stderr:
+            if verbose:
+                print(
+                    "`%s` printed to stderr:" % " ".join(args), file=sys.stderr
+                )
+            print(stderr.rstrip(), file=sys.stderr)
+        if strip:
+            stdout = stdout.rstrip("\r\n")
+        return stdout
+    if verbose:
+        print(
+            "`%s` returned %s" % (" ".join(args), p.returncode), file=sys.stderr
+        )
     if stderr:
-      if verbose:
-        print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr)
-      print(stderr.rstrip(), file=sys.stderr)
-    if strip:
-      stdout = stdout.rstrip('\r\n')
-    return stdout
-  if verbose:
-    print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr)
-  if stderr:
-    print(stderr.rstrip(), file=sys.stderr)
-  sys.exit(2)
+        print(stderr.rstrip(), file=sys.stderr)
+    sys.exit(2)
 
 
 def die(message):
-  print('error:', message, file=sys.stderr)
-  sys.exit(2)
+    print("error:", message, file=sys.stderr)
+    sys.exit(2)
 
 
 def to_bytes(str_input):
     # Encode to UTF-8 to get binary data.
     if isinstance(str_input, bytes):
         return str_input
-    return str_input.encode('utf-8')
+    return str_input.encode("utf-8")
 
 
 def to_string(bytes_input):
     if isinstance(bytes_input, str):
         return bytes_input
-    return bytes_input.encode('utf-8')
+    return bytes_input.encode("utf-8")
 
 
 def convert_string(bytes_input):
     try:
-        return to_string(bytes_input.decode('utf-8'))
-    except AttributeError: # 'str' object has no attribute 'decode'.
+        return to_string(bytes_input.decode("utf-8"))
+    except AttributeError:  # 'str' object has no attribute 'decode'.
         return str(bytes_input)
     except UnicodeError:
         return str(bytes_input)
 
-if __name__ == '__main__':
-  sys.exit(main())
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
index f423e2acb86c8..27be56bd2fb0f 100644
--- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -240,13 +240,14 @@ int main(int argc, const char **argv) {
 
   auto reportError = [argv](Error E) {
     logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0]));
-    exit(1);
+    return 1;
   };
 
   auto doWork = [&](std::function<llvm::Error()> Work) {
     if (llvm::Error Err = Work()) {
-      reportError(std::move(Err));
+      return reportError(std::move(Err));
     }
+    return 0;
   };
 
   auto warningOS = [argv]() -> raw_ostream & {
@@ -267,14 +268,14 @@ int main(int argc, const char **argv) {
   if (!Objcopy)
     Objcopy = sys::findProgramByName("llvm-objcopy");
   if (!Objcopy)
-    reportError(createStringError(Objcopy.getError(),
-                             "unable to find 'llvm-objcopy' in path"));
+    return reportError(createStringError(
+        Objcopy.getError(), "unable to find 'llvm-objcopy' in path"));
   else
     BundlerConfig.ObjcopyPath = *Objcopy;
 
   if (InputFileNames.getNumOccurrences() != 0 &&
       InputFileNamesDeprecatedOpt.getNumOccurrences() != 0) {
-    reportError(createStringError(
+    return reportError(createStringError(
         errc::invalid_argument,
         "-inputs and -input cannot be used together, use only -input instead"));
   }
@@ -290,9 +291,9 @@ int main(int argc, const char **argv) {
 
   if (OutputFileNames.getNumOccurrences() != 0 &&
       OutputFileNamesDeprecatedOpt.getNumOccurrences() != 0) {
-    reportError(createStringError(errc::invalid_argument,
-                                  "-outputs and -output cannot be used "
-                                  "together, use only -output instead"));
+    return reportError(createStringError(errc::invalid_argument,
+                                         "-outputs and -output cannot be used "
+                                         "together, use only -output instead"));
   }
 
   if (OutputFileNamesDeprecatedOpt.size()) {
@@ -306,49 +307,49 @@ int main(int argc, const char **argv) {
 
   if (ListBundleIDs) {
     if (Unbundle) {
-      reportError(
+      return reportError(
           createStringError(errc::invalid_argument,
                             "-unbundle and -list cannot be used together"));
     }
     if (InputFileNames.size() != 1) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "only one input file supported for -list"));
+      return reportError(createStringError(
+          errc::invalid_argument, "only one input file supported for -list"));
     }
     if (OutputFileNames.size()) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "-outputs option is invalid for -list"));
+      return reportError(createStringError(
+          errc::invalid_argument, "-outputs option is invalid for -list"));
     }
     if (TargetNames.size()) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "-targets option is invalid for -list"));
+      return reportError(createStringError(
+          errc::invalid_argument, "-targets option is invalid for -list"));
     }
 
-    doWork([&]() { return OffloadBundler::ListBundleIDsInFile(
-          InputFileNames.front(),
-          BundlerConfig); });
-    return 0;
+    return doWork([&]() {
+      return OffloadBundler::ListBundleIDsInFile(InputFileNames.front(),
+                                                 BundlerConfig);
+    });
   }
 
   if (BundlerConfig.CheckInputArchive) {
     if (!Unbundle) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "-check-input-archive cannot be used while "
-                                    "bundling"));
+      return reportError(createStringError(
+          errc::invalid_argument, "-check-input-archive cannot be used while "
+                                  "bundling"));
     }
     if (Unbundle && BundlerConfig.FilesType != "a") {
-      reportError(createStringError(errc::invalid_argument,
-                                    "-check-input-archive can only be used for "
-                                    "unbundling archives (-type=a)"));
+      return reportError(createStringError(
+          errc::invalid_argument, "-check-input-archive can only be used for "
+                                  "unbundling archives (-type=a)"));
     }
   }
 
   if (OutputFileNames.size() == 0 && !CheckSection) {
-    reportError(
+    return reportError(
         createStringError(errc::invalid_argument, "no output file specified!"));
   }
 
   if (TargetNames.getNumOccurrences() == 0) {
-    reportError(createStringError(
+    return reportError(createStringError(
         errc::invalid_argument,
         "for the --targets option: must be specified at least once!"));
   }
@@ -380,30 +381,30 @@ int main(int argc, const char **argv) {
   // -unbundle
   else if (Unbundle) {
     if (InputFileNames.size() != 1) {
-      reportError(createStringError(
+      return reportError(createStringError(
           errc::invalid_argument,
           "only one input file supported in unbundling mode"));
     }
     if (OutputFileNames.size() != TargetNames.size()) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "number of output files and targets should "
-                                    "match in unbundling mode"));
+      return reportError(createStringError(
+          errc::invalid_argument, "number of output files and targets should "
+                                  "match in unbundling mode"));
     }
   }
   // no explicit option: bundle
   else {
     if (BundlerConfig.FilesType == "a") {
-      reportError(createStringError(errc::invalid_argument,
-                                    "Archive files are only supported "
-                                    "for unbundling"));
+      return reportError(createStringError(errc::invalid_argument,
+                                           "Archive files are only supported "
+                                           "for unbundling"));
     }
     if (OutputFileNames.size() != 1) {
-      reportError(createStringError(
-          errc::invalid_argument,
-          "only one output file supported in bundling mode"));
+      return reportError(
+          createStringError(errc::invalid_argument,
+                            "only one output file supported in bundling mode"));
     }
     if (InputFileNames.size() != TargetNames.size()) {
-      reportError(createStringError(
+      return reportError(createStringError(
           errc::invalid_argument,
           "number of input files and targets should match in bundling mode"));
     }
@@ -425,8 +426,8 @@ int main(int argc, const char **argv) {
   std::map<std::string, std::set<StringRef>> TargetIDs;
   for (StringRef Target : TargetNames) {
     if (!ParsedTargets.insert(Target).second) {
-      reportError(createStringError(errc::invalid_argument,
-                                    "Duplicate targets are not allowed"));
+      return reportError(createStringError(
+          errc::invalid_argument, "Duplicate targets are not allowed"));
     }
 
     auto OffloadInfo = OffloadTargetInfo(Target, BundlerConfig);
@@ -441,7 +442,7 @@ int main(int argc, const char **argv) {
         Msg << ", unknown offloading kind '" << OffloadInfo.OffloadKind << "'";
       if (!TripleIsValid)
         Msg << ", unknown target triple '" << OffloadInfo.Triple.str() << "'";
-      reportError(createStringError(errc::invalid_argument, Msg.str()));
+      return reportError(createStringError(errc::invalid_argument, Msg.str()));
     }
 
     TargetIDs[OffloadInfo.OffloadKind.str() + "-" + OffloadInfo.Triple.str()]
@@ -465,7 +466,7 @@ int main(int argc, const char **argv) {
       Msg << "Cannot bundle inputs with conflicting targets: '"
           << TargetID.first + "-" + ConflictingTID->first << "' and '"
           << TargetID.first + "-" + ConflictingTID->second << "'";
-      reportError(createStringError(errc::invalid_argument, Msg.str()));
+      return reportError(createStringError(errc::invalid_argument, Msg.str()));
     }
   }
 
@@ -487,14 +488,14 @@ int main(int argc, const char **argv) {
   // treat missing host triple as error if we do unbundling.
   if ((Unbundle && HostTargetNum > 1) ||
       (!Unbundle && HostTargetNum != 1 && !BundlerConfig.AllowNoHost)) {
-    reportError(createStringError(errc::invalid_argument,
-                                  "expecting exactly one host target but got " +
-                                      Twine(HostTargetNum)));
+    return reportError(createStringError(
+        errc::invalid_argument,
+        "expecting exactly one host target but got " + Twine(HostTargetNum)));
   }
 
   OffloadBundler Bundler(BundlerConfig);
 
-  doWork([&]() {
+  return doWork([&]() {
     if (Unbundle) {
       if (BundlerConfig.FilesType == "a")
         return Bundler.UnbundleArchive();
@@ -503,5 +504,4 @@ int main(int argc, const char **argv) {
     } else
       return Bundler.BundleFiles();
   });
-  return 0;
 }
diff --git a/clang/tools/clang-repl/CMakeLists.txt b/clang/tools/clang-repl/CMakeLists.txt
index 7aebbe7a19436..f9a911b0ae8e2 100644
--- a/clang/tools/clang-repl/CMakeLists.txt
+++ b/clang/tools/clang-repl/CMakeLists.txt
@@ -66,7 +66,7 @@ clang_target_link_libraries(clang-repl PRIVATE
 # start to exceed this limit, e.g. when linking for arm-linux-gnueabihf with
 # gold. This flag tells the linker to build a PLT for the full address range.
 # Linkers without this flag are assumed to support proper PLTs by default.
-set(flag_long_plt "-Wl,--long-plt")
+set(flag_long_plt "LINKER:--long-plt")
 check_linker_flag(CXX ${flag_long_plt} HAVE_LINKER_FLAG_LONG_PLT)
 if(HAVE_LINKER_FLAG_LONG_PLT)
   target_link_options(clang-repl PRIVATE ${flag_long_plt})
diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp
index 08c54e6cafa90..7af8e4f25d99e 100644
--- a/clang/tools/clang-repl/ClangRepl.cpp
+++ b/clang/tools/clang-repl/ClangRepl.cpp
@@ -139,6 +139,8 @@ ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos,
 
 llvm::ExitOnError ExitOnErr;
 int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
   ExitOnErr.setBanner("clang-repl: ");
   llvm::cl::ParseCommandLineOptions(argc, argv);
 
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index bf7313f882e45..abf07d094e62c 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -10181,6 +10181,111 @@ TEST_P(ImportTemplateParmDeclDefaultValue,
       FromD, FromDInherited);
 }
 
+TEST_P(ASTImporterOptionSpecificTestBase,
+       ExistingUndeclaredImportDeclaredFriend) {
+  Decl *ToTU = getToTuDecl(
+      R"(
+        template <class A, A>
+        struct foo;
+
+        template <class A>
+        struct X {
+          template <class A1, A1>
+          friend struct foo;
+        };
+      )",
+      Lang_CXX11);
+  Decl *FromTU = getTuDecl(
+      R"(
+        template <class A, A>
+        struct foo;
+
+        template <class A>
+        struct X {
+          template <class A1, A1>
+          friend struct foo;
+        };
+
+        X<int> x;
+      )",
+      Lang_CXX11);
+
+  auto *ToFr1 = FirstDeclMatcher<FriendDecl>().match(ToTU, friendDecl());
+  auto *ToFrD1 = ToFr1->getFriendDecl();
+
+  auto *FromFr1 = FirstDeclMatcher<FriendDecl>().match(FromTU, friendDecl());
+  auto *FromFr2 = LastDeclMatcher<FriendDecl>().match(FromTU, friendDecl());
+
+  auto *FromFrD1 = FromFr1->getFriendDecl();
+  auto *FromFrD2 = FromFr2->getFriendDecl();
+
+  auto *Ctx1 = cast<Decl>(FromFrD1->getDeclContext());
+  auto *Ctx2 = cast<Decl>(FromFrD2->getDeclContext());
+
+  ASSERT_EQ(Ctx1, Ctx2);
+  ASSERT_EQ(ToFrD1->getTemplateDepth(), 1u);
+  ASSERT_EQ(FromFrD2->getTemplateDepth(), 0u);
+  ASSERT_EQ(ToFrD1->getFriendObjectKind(), Decl::FOK_Undeclared);
+  ASSERT_EQ(FromFrD2->getFriendObjectKind(), Decl::FOK_Declared);
+
+  auto *ToFr2Imp = Import(FromFr2, Lang_CXX11);
+
+  EXPECT_TRUE(ToFr2Imp);
+}
+
+TEST_P(ASTImporterOptionSpecificTestBase,
+       ExistingDeclaredImportUndeclaredFriend) {
+  Decl *ToTU = getToTuDecl(
+      R"(
+        template <class A, A>
+        struct foo;
+
+        template <class A>
+        struct X {
+          template <class A1, A1>
+          friend struct foo;
+        };
+
+        X<int> x;
+      )",
+      Lang_CXX11);
+  Decl *FromTU = getTuDecl(
+      R"(
+        template <class A, A>
+        struct foo;
+
+        template <class A>
+        struct X {
+          template <class A1, A1>
+          friend struct foo;
+        };
+      )",
+      Lang_CXX11);
+
+  auto *ToFr1 = FirstDeclMatcher<FriendDecl>().match(ToTU, friendDecl());
+  auto *ToFr2 = LastDeclMatcher<FriendDecl>().match(ToTU, friendDecl());
+
+  auto *ToFrD1 = ToFr1->getFriendDecl();
+  auto *ToFrD2 = ToFr2->getFriendDecl();
+
+  auto *FromFr1 = FirstDeclMatcher<FriendDecl>().match(FromTU, friendDecl());
+  auto *FromFrD1 = FromFr1->getFriendDecl();
+
+  auto *Ctx1 = cast<Decl>(ToFrD1->getDeclContext());
+  auto *Ctx2 = cast<Decl>(ToFrD2->getDeclContext());
+
+  ASSERT_EQ(Ctx1, Ctx2);
+  ASSERT_EQ(FromFrD1->getTemplateDepth(), 1u);
+  ASSERT_EQ(ToFrD2->getTemplateDepth(), 0u);
+  ASSERT_EQ(FromFrD1->getFriendObjectKind(), Decl::FOK_Undeclared);
+  ASSERT_EQ(ToFrD2->getFriendObjectKind(), Decl::FOK_Declared);
+
+  auto *ToFr1Imp = Import(FromFr1, Lang_CXX11);
+
+  EXPECT_TRUE(ToFr1Imp);
+  EXPECT_EQ(ToFr1Imp, ToFr1);
+}
+
 INSTANTIATE_TEST_SUITE_P(ParameterizedTests, ASTImporterLookupTableTest,
                          DefaultTestValuesForRunOptions);
 
diff --git a/clang/unittests/AST/ByteCode/BitcastBuffer.cpp b/clang/unittests/AST/ByteCode/BitcastBuffer.cpp
new file mode 100644
index 0000000000000..02c38a2201363
--- /dev/null
+++ b/clang/unittests/AST/ByteCode/BitcastBuffer.cpp
@@ -0,0 +1,87 @@
+#include "../../../lib/AST/ByteCode/BitcastBuffer.h"
+#include "clang/AST/ASTContext.h"
+#include "gtest/gtest.h"
+#include <bitset>
+#include <cassert>
+#include <cmath>
+#include <memory>
+#include <string>
+
+using namespace clang;
+using namespace clang::interp;
+
+TEST(BitcastBuffer, PushData) {
+  BitcastBuffer Buff1(Bytes(sizeof(int)).toBits());
+
+  const unsigned V = 0xCAFEBABE;
+  Bits VSize = Bytes(sizeof(V)).toBits();
+  std::byte Data[sizeof(V)];
+  std::memcpy(Data, &V, sizeof(V));
+
+  Endian HostEndianness =
+      llvm::sys::IsLittleEndianHost ? Endian::Little : Endian::Big;
+
+  Buff1.pushData(Data, Bits::zero(), VSize, HostEndianness);
+
+  // The buffer is in host-endianness.
+  if (llvm::sys::IsLittleEndianHost) {
+    ASSERT_EQ(Buff1.Data[0], std::byte{0xbe});
+    ASSERT_EQ(Buff1.Data[1], std::byte{0xba});
+    ASSERT_EQ(Buff1.Data[2], std::byte{0xfe});
+    ASSERT_EQ(Buff1.Data[3], std::byte{0xca});
+  } else {
+    ASSERT_EQ(Buff1.Data[0], std::byte{0xca});
+    ASSERT_EQ(Buff1.Data[1], std::byte{0xfe});
+    ASSERT_EQ(Buff1.Data[2], std::byte{0xba});
+    ASSERT_EQ(Buff1.Data[3], std::byte{0xbe});
+  }
+
+  {
+    unsigned V2;
+    auto D = Buff1.copyBits(Bits::zero(), VSize, VSize, Endian::Little);
+    std::memcpy(&V2, D.get(), sizeof(V));
+    ASSERT_EQ(V, V2);
+
+    D = Buff1.copyBits(Bits::zero(), VSize, VSize, Endian::Big);
+    std::memcpy(&V2, D.get(), sizeof(V));
+    ASSERT_EQ(V, V2);
+  }
+
+  BitcastBuffer Buff2(Bytes(sizeof(int)).toBits());
+  {
+    short s1 = 0xCAFE;
+    short s2 = 0xBABE;
+    std::byte sdata[2];
+
+    std::memcpy(sdata, &s1, sizeof(s1));
+    Buff2.pushData(sdata, Bits::zero(), Bits(sizeof(s1) * 8), HostEndianness);
+    std::memcpy(sdata, &s2, sizeof(s2));
+    Buff2.pushData(sdata, Bits(sizeof(s1) * 8), Bits(sizeof(s2) * 8),
+                   HostEndianness);
+  }
+
+  if (llvm::sys::IsLittleEndianHost) {
+    ASSERT_EQ(Buff2.Data[0], std::byte{0xfe});
+    ASSERT_EQ(Buff2.Data[1], std::byte{0xca});
+    ASSERT_EQ(Buff2.Data[2], std::byte{0xbe});
+    ASSERT_EQ(Buff2.Data[3], std::byte{0xba});
+  } else {
+    ASSERT_EQ(Buff2.Data[0], std::byte{0xba});
+    ASSERT_EQ(Buff2.Data[1], std::byte{0xbe});
+    ASSERT_EQ(Buff2.Data[2], std::byte{0xca});
+    ASSERT_EQ(Buff2.Data[3], std::byte{0xfe});
+  }
+
+  {
+    unsigned V;
+    auto D = Buff2.copyBits(Bits::zero(), Bits(sizeof(V) * 8),
+                            Bits(sizeof(V) * 8), Endian::Little);
+    std::memcpy(&V, D.get(), sizeof(V));
+    ASSERT_EQ(V, 0xBABECAFE);
+
+    D = Buff2.copyBits(Bits::zero(), Bits(sizeof(V) * 8), Bits(sizeof(V) * 8),
+                       Endian::Big);
+    std::memcpy(&V, D.get(), sizeof(V));
+    ASSERT_EQ(V, 0xBABECAFE);
+  }
+}
diff --git a/clang/unittests/AST/ByteCode/CMakeLists.txt b/clang/unittests/AST/ByteCode/CMakeLists.txt
index ea727cdd4412b..b862fb4834fbd 100644
--- a/clang/unittests/AST/ByteCode/CMakeLists.txt
+++ b/clang/unittests/AST/ByteCode/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_clang_unittest(InterpTests
+  BitcastBuffer.cpp
   Descriptor.cpp
   toAPValue.cpp
   )
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index ebf548eb25431..9bc287e07224a 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -189,6 +189,21 @@ TEST(ASTMatchersTestCUDA, HasAttrCUDA) {
                                   hasAttr(clang::attr::CUDAGlobal)));
 }
 
+TEST_P(ASTMatchersTest, ExportDecl) {
+  if (!GetParam().isCXX20OrLater()) {
+    return;
+  }
+  const std::string moduleHeader = "module;export module ast_matcher_test;";
+  EXPECT_TRUE(matches(moduleHeader + "export void foo();",
+                      exportDecl(has(functionDecl()))));
+  EXPECT_TRUE(matches(moduleHeader + "export { void foo(); int v; }",
+                      exportDecl(has(functionDecl()))));
+  EXPECT_TRUE(matches(moduleHeader + "export { void foo(); int v; }",
+                      exportDecl(has(varDecl()))));
+  EXPECT_TRUE(matches(moduleHeader + "export namespace aa { void foo(); }",
+                      exportDecl(has(namespaceDecl()))));
+}
+
 TEST_P(ASTMatchersTest, ValueDecl) {
   if (!GetParam().isCXX()) {
     // FIXME: Fix this test in non-C++ language modes.
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 1d18869a6b8af..75d6ca5ba17f8 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -283,6 +283,12 @@ TEST(HasDeclaration, HasDeclarationOfTypeAlias) {
           hasDeclaration(typeAliasTemplateDecl()))))))));
 }
 
+TEST(HasDeclaration, HasDeclarationOfObjCInterface) {
+  EXPECT_TRUE(matchesObjC("@interface BaseClass @end void f() {BaseClass* b;}",
+                          varDecl(hasType(objcObjectPointerType(
+                              pointee(hasDeclaration(objcInterfaceDecl())))))));
+}
+
 TEST(HasUnqualifiedDesugaredType, DesugarsUsing) {
   EXPECT_TRUE(
       matches("struct A {}; using B = A; B b;",
diff --git a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
index 834aa7f4c2ac2..e086ea3c892f1 100644
--- a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
@@ -8,15 +8,23 @@
 
 #include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "TestingSupport.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Frontend/ASTUnit.h"
+#include "clang/Tooling/Tooling.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <memory>
+#include <string>
 
 namespace {
 
 using namespace clang;
 using namespace dataflow;
 
+using ast_matchers::cxxMethodDecl;
 using ast_matchers::cxxRecordDecl;
 using ast_matchers::hasName;
 using ast_matchers::hasType;
@@ -107,4 +115,37 @@ TEST(ASTOpsTest, ReferencedDeclsLocalsNotParamsOrStatics) {
               UnorderedElementsAre(LocalDecl));
 }
 
+TEST(ASTOpsTest, LambdaCaptures) {
+  std::string Code = R"cc(
+    void func(int CapturedByRef, int CapturedByValue, int NotCaptured) {
+    int Local;
+      auto Lambda = [&CapturedByRef, CapturedByValue, &Local](int LambdaParam) {
+      };
+    }
+  )cc";
+  std::unique_ptr<ASTUnit> Unit =
+      tooling::buildASTFromCodeWithArgs(Code, {"-fsyntax-only", "-std=c++17"});
+  auto &ASTCtx = Unit->getASTContext();
+
+  ASSERT_EQ(ASTCtx.getDiagnostics().getClient()->getNumErrors(), 0U);
+
+  auto *LambdaCallOp = selectFirst<CXXMethodDecl>(
+      "l", match(cxxMethodDecl(hasName("operator()")).bind("l"), ASTCtx));
+  ASSERT_NE(LambdaCallOp, nullptr);
+  auto *Func = cast<FunctionDecl>(findValueDecl(ASTCtx, "func"));
+  ASSERT_NE(Func, nullptr);
+  auto *CapturedByRefDecl = Func->getParamDecl(0);
+  ASSERT_NE(CapturedByRefDecl, nullptr);
+  auto *CapturedByValueDecl = Func->getParamDecl(1);
+  ASSERT_NE(CapturedByValueDecl, nullptr);
+
+  EXPECT_THAT(getReferencedDecls(*Func).LambdaCapturedParams, IsEmpty());
+  ReferencedDecls ForLambda = getReferencedDecls(*LambdaCallOp);
+  EXPECT_THAT(ForLambda.LambdaCapturedParams,
+              UnorderedElementsAre(CapturedByRefDecl, CapturedByValueDecl));
+  // Captured locals must be seen in the body for them to appear in
+  // ReferencedDecls.
+  EXPECT_THAT(ForLambda.Locals, IsEmpty());
+}
+
 } // namespace
diff --git a/clang/unittests/Basic/DiagnosticTest.cpp b/clang/unittests/Basic/DiagnosticTest.cpp
index 36a77c7247655..e03d9a464df7f 100644
--- a/clang/unittests/Basic/DiagnosticTest.cpp
+++ b/clang/unittests/Basic/DiagnosticTest.cpp
@@ -12,6 +12,7 @@
 #include "clang/Basic/DiagnosticLex.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -20,6 +21,7 @@
 #include "llvm/Support/VirtualFileSystem.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <memory>
 #include <optional>
 #include <vector>
 
@@ -196,7 +198,17 @@ class SuppressionMappingTest : public testing::Test {
     return CaptureConsumer.StoredDiags;
   }
 
+  SourceLocation locForFile(llvm::StringRef FileName) {
+    auto Buf = MemoryBuffer::getMemBuffer("", FileName);
+    SourceManager &SM = Diags.getSourceManager();
+    FileID FooID = SM.createFileID(std::move(Buf));
+    return SM.getLocForStartOfFile(FooID);
+  }
+
 private:
+  FileManager FM{{}, FS};
+  SourceManager SM{Diags, FM};
+
   class CaptureDiagnosticConsumer : public DiagnosticConsumer {
   public:
     std::vector<StoredDiagnostic> StoredDiags;
@@ -255,9 +267,9 @@ TEST_F(SuppressionMappingTest, SuppressesGroup) {
   clang::ProcessWarningOptions(Diags, Diags.getDiagnosticOptions(), *FS);
   EXPECT_THAT(diags(), IsEmpty());
 
-  EXPECT_TRUE(
-      Diags.isSuppressedViaMapping(diag::warn_unused_function, "foo.cpp"));
-  EXPECT_FALSE(Diags.isSuppressedViaMapping(diag::warn_deprecated, "foo.cpp"));
+  SourceLocation FooLoc = locForFile("foo.cpp");
+  EXPECT_TRUE(Diags.isSuppressedViaMapping(diag::warn_unused_function, FooLoc));
+  EXPECT_FALSE(Diags.isSuppressedViaMapping(diag::warn_deprecated, FooLoc));
 }
 
 TEST_F(SuppressionMappingTest, EmitCategoryIsExcluded) {
@@ -271,10 +283,10 @@ TEST_F(SuppressionMappingTest, EmitCategoryIsExcluded) {
   clang::ProcessWarningOptions(Diags, Diags.getDiagnosticOptions(), *FS);
   EXPECT_THAT(diags(), IsEmpty());
 
-  EXPECT_TRUE(
-      Diags.isSuppressedViaMapping(diag::warn_unused_function, "bar.cpp"));
-  EXPECT_FALSE(
-      Diags.isSuppressedViaMapping(diag::warn_unused_function, "foo.cpp"));
+  EXPECT_TRUE(Diags.isSuppressedViaMapping(diag::warn_unused_function,
+                                           locForFile("bar.cpp")));
+  EXPECT_FALSE(Diags.isSuppressedViaMapping(diag::warn_unused_function,
+                                            locForFile("foo.cpp")));
 }
 
 TEST_F(SuppressionMappingTest, LongestMatchWins) {
@@ -289,12 +301,12 @@ TEST_F(SuppressionMappingTest, LongestMatchWins) {
   clang::ProcessWarningOptions(Diags, Diags.getDiagnosticOptions(), *FS);
   EXPECT_THAT(diags(), IsEmpty());
 
+  EXPECT_TRUE(Diags.isSuppressedViaMapping(
+      diag::warn_unused_function, locForFile("clang/lib/Basic/foo.h")));
+  EXPECT_FALSE(Diags.isSuppressedViaMapping(
+      diag::warn_unused_function, locForFile("clang/lib/Sema/bar.h")));
   EXPECT_TRUE(Diags.isSuppressedViaMapping(diag::warn_unused_function,
-                                           "clang/lib/Basic/foo.h"));
-  EXPECT_FALSE(Diags.isSuppressedViaMapping(diag::warn_unused_function,
-                                            "clang/lib/Sema/bar.h"));
-  EXPECT_TRUE(Diags.isSuppressedViaMapping(diag::warn_unused_function,
-                                           "clang/lib/Sema/foo.h"));
+                                           locForFile("clang/lib/Sema/foo.h")));
 }
 
 TEST_F(SuppressionMappingTest, IsIgnored) {
@@ -308,9 +320,7 @@ TEST_F(SuppressionMappingTest, IsIgnored) {
   clang::ProcessWarningOptions(Diags, Diags.getDiagnosticOptions(), *FS);
   ASSERT_THAT(diags(), IsEmpty());
 
-  FileManager FM({}, FS);
-  SourceManager SM(Diags, FM);
-
+  SourceManager &SM = Diags.getSourceManager();
   auto ClangID =
       SM.createFileID(llvm::MemoryBuffer::getMemBuffer("", "clang/foo.h"));
   auto NonClangID =
diff --git a/clang/unittests/CodeGen/TBAAMetadataTest.cpp b/clang/unittests/CodeGen/TBAAMetadataTest.cpp
index 92bf7e4730412..cad8783ea73fb 100644
--- a/clang/unittests/CodeGen/TBAAMetadataTest.cpp
+++ b/clang/unittests/CodeGen/TBAAMetadataTest.cpp
@@ -39,6 +39,11 @@ auto OmnipotentCharC = MMTuple(
   MConstInt(0, 64)
 );
 
+auto AnyPtr = MMTuple(
+  MMString("any pointer"),
+  OmnipotentCharC,
+  MConstInt(0, 64)
+);
 
 auto OmnipotentCharCXX = MMTuple(
   MMString("omnipotent char"),
@@ -112,19 +117,27 @@ TEST(TBAAMetadataTest, BasicTypes) {
   ASSERT_TRUE(I);
 
   I = matchNext(I,
-                MInstruction(Instruction::Store,
-                             MValType(PointerType::getUnqual(Compiler.Context)),
-                             MMTuple(MMTuple(MMString("any pointer"),
-                                             OmnipotentCharC, MConstInt(0)),
-                                     MSameAs(0), MConstInt(0))));
+      MInstruction(Instruction::Store,
+        MValType(PointerType::getUnqual(Compiler.Context)),
+        MMTuple(
+          MMTuple(
+            MMString("p1 void"),
+            AnyPtr,
+            MConstInt(0)),
+          MSameAs(0),
+          MConstInt(0))));
   ASSERT_TRUE(I);
 
   I = matchNext(I,
-                MInstruction(Instruction::Store,
-                             MValType(PointerType::getUnqual(Compiler.Context)),
-                             MMTuple(MMTuple(MMString("any pointer"),
-                                             OmnipotentCharC, MConstInt(0)),
-                                     MSameAs(0), MConstInt(0))));
+      MInstruction(Instruction::Store,
+        MValType(PointerType::getUnqual(Compiler.Context)),
+        MMTuple(
+          MMTuple(
+            MMString("p1 int"),
+            AnyPtr,
+            MConstInt(0)),
+          MSameAs(0),
+          MConstInt(0))));
   ASSERT_TRUE(I);
 }
 
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 250e51b542166..eeb857a914b1a 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -5724,6 +5724,24 @@ TEST_F(FormatTest, HashInMacroDefinition) {
                getLLVMStyleWithColumns(22));
 
   verifyFormat("#define A void # ## #", getLLVMStyleWithColumns(22));
+
+#if 0
+  // FIXME: The correct format is:
+  verifyFormat("{\n"
+               "  {\n"
+               "#define GEN_ID(_x) char *_x{#_x}\n"
+               "    GEN_ID(one);\n"
+               "  }\n"
+               "}");
+#endif
+  verifyFormat("{\n"
+               "  {\n"
+               "#define GEN_ID(_x) \\\n"
+               "  char *_x { #_x }\n"
+               "    GEN_ID(one);\n"
+               "  }\n"
+               "}",
+               getGoogleStyle());
 }
 
 TEST_F(FormatTest, RespectWhitespaceInMacroDefinitions) {
@@ -9441,6 +9459,15 @@ TEST_F(FormatTest, AlignsAfterOpenBracket) {
       "    aaaaaaaaaaaaaaaa\n"
       ");",
       Style);
+  verifyFormat("void foo(\n"
+               "    void (*foobarpntr)(\n"
+               "        aaaaaaaaaaaaaaaaaa *,\n"
+               "        bbbbbbbbbbbbbb *,\n"
+               "        cccccccccccccccccccc *,\n"
+               "        dddddddddddddddddd *\n"
+               "    )\n"
+               ");",
+               Style);
   verifyFormat("aaaaaaa<bbbbbbbb> const aaaaaaaaaa{\n"
                "    aaaaaaaaaaaaa(aaaaaaaaaaa, aaaaaaaaaaaaaaaa)\n"
                "};",
@@ -27386,6 +27413,13 @@ TEST_F(FormatTest, RemoveSemicolon) {
                Style);
 #endif
 
+  verifyFormat("auto sgf = [] {\n"
+               "  ogl = {\n"
+               "      a, b, c, d, e,\n"
+               "  };\n"
+               "};",
+               Style);
+
   Style.TypenameMacros.push_back("STRUCT");
   verifyFormat("STRUCT(T, B) { int i; };", Style);
 }
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 9db3187ac44e7..38658fcb0e999 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -3346,6 +3346,25 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
   EXPECT_BRACE_KIND(Tokens[13], BK_Block);
 
+  Tokens = annotate("{\n"
+                    "  {\n"
+                    "#define GEN_ID(_x) char *_x{#_x}\n"
+                    "    GEN_ID(one);\n"
+                    "  }\n"
+                    "}");
+  ASSERT_EQ(Tokens.size(), 23u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
+  EXPECT_BRACE_KIND(Tokens[0], BK_Block);
+  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
+  EXPECT_BRACE_KIND(Tokens[1], BK_Block);
+#if 0
+  // FIXME:
+  EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
+  EXPECT_BRACE_KIND(Tokens[14], BK_BracedInit);
+#endif
+  EXPECT_BRACE_KIND(Tokens[20], BK_Block);
+  EXPECT_BRACE_KIND(Tokens[21], BK_Block);
+
   Tokens = annotate("a = class extends goog.a {};",
                     getGoogleStyle(FormatStyle::LK_JavaScript));
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp
index 47aa2c131a304..aead7fb899d0a 100644
--- a/clang/unittests/Lex/LexerTest.cpp
+++ b/clang/unittests/Lex/LexerTest.cpp
@@ -652,6 +652,38 @@ TEST_F(LexerTest, RawAndNormalLexSameForLineComments) {
   EXPECT_TRUE(ToksView.empty());
 }
 
+TEST_F(LexerTest, GetRawTokenOnEscapedNewLineChecksWhitespace) {
+  const llvm::StringLiteral Source = R"cc(
+  #define ONE \
+  1
+
+  int i = ONE;
+  )cc";
+  std::vector<Token> Toks =
+      CheckLex(Source, {tok::kw_int, tok::identifier, tok::equal,
+                        tok::numeric_constant, tok::semi});
+
+  // Set up by getting the raw token for the `1` in the macro definition.
+  const Token &OneExpanded = Toks[3];
+  Token Tok;
+  ASSERT_FALSE(
+      Lexer::getRawToken(OneExpanded.getLocation(), Tok, SourceMgr, LangOpts));
+  // The `ONE`.
+  ASSERT_EQ(Tok.getKind(), tok::raw_identifier);
+  ASSERT_FALSE(
+      Lexer::getRawToken(SourceMgr.getSpellingLoc(OneExpanded.getLocation()),
+                         Tok, SourceMgr, LangOpts));
+  // The `1` in the macro definition.
+  ASSERT_EQ(Tok.getKind(), tok::numeric_constant);
+
+  // Go back 4 characters: two spaces, one newline, and the backslash.
+  SourceLocation EscapedNewLineLoc = Tok.getLocation().getLocWithOffset(-4);
+  // Expect true (=failure) because the whitespace immediately after the
+  // escaped newline is not ignored.
+  EXPECT_TRUE(Lexer::getRawToken(EscapedNewLineLoc, Tok, SourceMgr, LangOpts,
+                                 /*IgnoreWhiteSpace=*/false));
+}
+
 TEST(LexerPreambleTest, PreambleBounds) {
   std::vector<std::string> Cases = {
       R"cc([[
diff --git a/clang/unittests/Serialization/CMakeLists.txt b/clang/unittests/Serialization/CMakeLists.txt
index e7eebd0cb9823..e7005b5d511eb 100644
--- a/clang/unittests/Serialization/CMakeLists.txt
+++ b/clang/unittests/Serialization/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_unittest(SerializationTests
   ModuleCacheTest.cpp
   NoCommentsTest.cpp
   PreambleInNamedModulesTest.cpp
+  LoadSpecLazilyTest.cpp
   SourceLocationEncodingTest.cpp
   VarDeclConstantInitTest.cpp
   )
diff --git a/clang/unittests/Serialization/LoadSpecLazilyTest.cpp b/clang/unittests/Serialization/LoadSpecLazilyTest.cpp
new file mode 100644
index 0000000000000..0e452652a940d
--- /dev/null
+++ b/clang/unittests/Serialization/LoadSpecLazilyTest.cpp
@@ -0,0 +1,262 @@
+//== unittests/Serialization/LoadSpecLazily.cpp ----------------------========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Parse/ParseAST.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
+#include "clang/Tooling/Tooling.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::tooling;
+
+namespace {
+
+class LoadSpecLazilyTest : public ::testing::Test {
+  void SetUp() override {
+    ASSERT_FALSE(
+        sys::fs::createUniqueDirectory("load-spec-lazily-test", TestDir));
+  }
+
+  void TearDown() override { sys::fs::remove_directories(TestDir); }
+
+public:
+  SmallString<256> TestDir;
+
+  void addFile(StringRef Path, StringRef Contents) {
+    ASSERT_FALSE(sys::path::is_absolute(Path));
+
+    SmallString<256> AbsPath(TestDir);
+    sys::path::append(AbsPath, Path);
+
+    ASSERT_FALSE(
+        sys::fs::create_directories(llvm::sys::path::parent_path(AbsPath)));
+
+    std::error_code EC;
+    llvm::raw_fd_ostream OS(AbsPath, EC);
+    ASSERT_FALSE(EC);
+    OS << Contents;
+  }
+
+  std::string GenerateModuleInterface(StringRef ModuleName,
+                                      StringRef Contents) {
+    std::string FileName = llvm::Twine(ModuleName + ".cppm").str();
+    addFile(FileName, Contents);
+
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
+        llvm::vfs::createPhysicalFileSystem();
+    IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+        CompilerInstance::createDiagnostics(*VFS, new DiagnosticOptions());
+    CreateInvocationOptions CIOpts;
+    CIOpts.Diags = Diags;
+    CIOpts.VFS = VFS;
+
+    std::string CacheBMIPath =
+        llvm::Twine(TestDir + "/" + ModuleName + ".pcm").str();
+    std::string PrebuiltModulePath =
+        "-fprebuilt-module-path=" + TestDir.str().str();
+    const char *Args[] = {"clang++",
+                          "-std=c++20",
+                          "--precompile",
+                          PrebuiltModulePath.c_str(),
+                          "-working-directory",
+                          TestDir.c_str(),
+                          "-I",
+                          TestDir.c_str(),
+                          FileName.c_str(),
+                          "-o",
+                          CacheBMIPath.c_str()};
+    std::shared_ptr<CompilerInvocation> Invocation =
+        createInvocation(Args, CIOpts);
+    EXPECT_TRUE(Invocation);
+
+    CompilerInstance Instance;
+    Instance.setDiagnostics(Diags.get());
+    Instance.setInvocation(Invocation);
+    Instance.getFrontendOpts().OutputFile = CacheBMIPath;
+    GenerateModuleInterfaceAction Action;
+    EXPECT_TRUE(Instance.ExecuteAction(Action));
+    EXPECT_FALSE(Diags->hasErrorOccurred());
+
+    return CacheBMIPath;
+  }
+};
+
+enum class CheckingMode { Forbidden, Required };
+
+class DeclsReaderListener : public ASTDeserializationListener {
+  StringRef SpeficiedName;
+  CheckingMode Mode;
+
+  bool ReadedSpecifiedName = false;
+
+public:
+  void DeclRead(GlobalDeclID ID, const Decl *D) override {
+    auto *ND = dyn_cast<NamedDecl>(D);
+    if (!ND)
+      return;
+
+    ReadedSpecifiedName |= ND->getName().contains(SpeficiedName);
+    if (Mode == CheckingMode::Forbidden) {
+      EXPECT_FALSE(ReadedSpecifiedName);
+    }
+  }
+
+  DeclsReaderListener(StringRef SpeficiedName, CheckingMode Mode)
+      : SpeficiedName(SpeficiedName), Mode(Mode) {}
+
+  ~DeclsReaderListener() {
+    if (Mode == CheckingMode::Required) {
+      EXPECT_TRUE(ReadedSpecifiedName);
+    }
+  }
+};
+
+class LoadSpecLazilyConsumer : public ASTConsumer {
+  DeclsReaderListener Listener;
+
+public:
+  LoadSpecLazilyConsumer(StringRef SpecifiedName, CheckingMode Mode)
+      : Listener(SpecifiedName, Mode) {}
+
+  ASTDeserializationListener *GetASTDeserializationListener() override {
+    return &Listener;
+  }
+};
+
+class CheckLoadSpecLazilyAction : public ASTFrontendAction {
+  StringRef SpecifiedName;
+  CheckingMode Mode;
+
+public:
+  std::unique_ptr<ASTConsumer>
+  CreateASTConsumer(CompilerInstance &CI, StringRef /*Unused*/) override {
+    return std::make_unique<LoadSpecLazilyConsumer>(SpecifiedName, Mode);
+  }
+
+  CheckLoadSpecLazilyAction(StringRef SpecifiedName, CheckingMode Mode)
+      : SpecifiedName(SpecifiedName), Mode(Mode) {}
+};
+
+TEST_F(LoadSpecLazilyTest, BasicTest) {
+  GenerateModuleInterface("M", R"cpp(
+export module M;
+export template <class T>
+class A {};
+export class ShouldNotBeLoaded {};
+export class Temp {
+   A<ShouldNotBeLoaded> AS;
+};
+  )cpp");
+
+  const char *test_file_contents = R"cpp(
+import M;
+A<int> a;
+  )cpp";
+  std::string DepArg = "-fprebuilt-module-path=" + TestDir.str().str();
+  EXPECT_TRUE(
+      runToolOnCodeWithArgs(std::make_unique<CheckLoadSpecLazilyAction>(
+                                "ShouldNotBeLoaded", CheckingMode::Forbidden),
+                            test_file_contents,
+                            {
+                                "-std=c++20",
+                                DepArg.c_str(),
+                                "-I",
+                                TestDir.c_str(),
+                            },
+                            "test.cpp"));
+}
+
+TEST_F(LoadSpecLazilyTest, ChainedTest) {
+  GenerateModuleInterface("M", R"cpp(
+export module M;
+export template <class T>
+class A {};
+  )cpp");
+
+  GenerateModuleInterface("N", R"cpp(
+export module N;
+export import M;
+export class ShouldNotBeLoaded {};
+export class Temp {
+   A<ShouldNotBeLoaded> AS;
+};
+  )cpp");
+
+  const char *test_file_contents = R"cpp(
+import N;
+A<int> a;
+  )cpp";
+  std::string DepArg = "-fprebuilt-module-path=" + TestDir.str().str();
+  EXPECT_TRUE(
+      runToolOnCodeWithArgs(std::make_unique<CheckLoadSpecLazilyAction>(
+                                "ShouldNotBeLoaded", CheckingMode::Forbidden),
+                            test_file_contents,
+                            {
+                                "-std=c++20",
+                                DepArg.c_str(),
+                                "-I",
+                                TestDir.c_str(),
+                            },
+                            "test.cpp"));
+}
+
+/// Test that we won't crash due to we may invalidate the lazy specialization
+/// lookup table during the loading process.
+TEST_F(LoadSpecLazilyTest, ChainedTest2) {
+  GenerateModuleInterface("M", R"cpp(
+export module M;
+export template <class T>
+class A {};
+
+export class B {};
+
+export class C {
+  A<B> D;
+};
+  )cpp");
+
+  GenerateModuleInterface("N", R"cpp(
+export module N;
+export import M;
+export class MayBeLoaded {};
+
+export class Temp {
+   A<MayBeLoaded> AS;
+};
+
+export class ExportedClass {};
+
+export template<> class A<ExportedClass> {
+   A<MayBeLoaded> AS;
+   A<B>           AB;
+};
+  )cpp");
+
+  const char *test_file_contents = R"cpp(
+import N;
+Temp T;
+A<ExportedClass> a;
+  )cpp";
+  std::string DepArg = "-fprebuilt-module-path=" + TestDir.str().str();
+  EXPECT_TRUE(runToolOnCodeWithArgs(std::make_unique<CheckLoadSpecLazilyAction>(
+                                        "MayBeLoaded", CheckingMode::Required),
+                                    test_file_contents,
+                                    {
+                                        "-std=c++20",
+                                        DepArg.c_str(),
+                                        "-I",
+                                        TestDir.c_str(),
+                                    },
+                                    "test.cpp"));
+}
+
+} // namespace
diff --git a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
index 4ff5e8b65a686..a551f83ff3f9c 100644
--- a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
@@ -5659,8 +5659,6 @@ struct X {
 };
 [[void (X::*xp)();]]
 [[void (X::**xpp)(const int*);]]
-// FIXME: Generate the right syntax tree for this type,
-// i.e. create a syntax node for the outer member pointer
 [[void (X::Y::*xyp)(const int*, char);]]
 )cpp",
       {R"txt(
@@ -5714,9 +5712,9 @@ SimpleDeclaration
 | `-SimpleDeclarator ListElement
 |   |-ParenDeclarator
 |   | |-'(' OpenParen
-|   | |-'X'
-|   | |-'::'
 |   | |-MemberPointer
+|   | | |-'X'
+|   | | |-'::'
 |   | | |-'Y'
 |   | | |-'::'
 |   | | `-'*'
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index ebc1db374c775..22e6b9c31eb57 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -5182,6 +5182,9 @@ GetAttributeHeadingAndSpellings(const Record &Documentation,
 
 static void WriteDocumentation(const RecordKeeper &Records,
                                const DocumentationData &Doc, raw_ostream &OS) {
+  if (StringRef Label = Doc.Documentation->getValueAsString("Label");
+      !Label.empty())
+    OS << ".. _" << Label << ":\n\n";
   OS << Doc.Heading << "\n" << std::string(Doc.Heading.length(), '-') << "\n";
 
   // List what spelling syntaxes the attribute supports.
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index 6a4a64a081306..a9faba0d84403 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -1907,8 +1907,7 @@ void clang::EmitClangDiagDocs(const RecordKeeper &Records, raw_ostream &OS) {
   // Write out the diagnostic groups.
   for (const Record *G : DiagGroups) {
     bool IsRemarkGroup = isRemarkGroup(G, DiagsInGroup);
-    auto &GroupInfo =
-        DiagsInGroup[std::string(G->getValueAsString("GroupName"))];
+    auto &GroupInfo = DiagsInGroup[G->getValueAsString("GroupName")];
     bool IsSynonym = GroupInfo.DiagsInGroup.empty() &&
                      GroupInfo.SubGroups.size() == 1;
 
diff --git a/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
index d6ab307dd80bc..64534a50877ec 100644
--- a/clang/utils/TableGen/ClangOpcodesEmitter.cpp
+++ b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
@@ -146,8 +146,6 @@ void ClangOpcodesEmitter::EmitInterp(raw_ostream &OS, StringRef N,
                 OS << ", PC";
               else
                 OS << ", OpPC";
-              if (CanReturn)
-                OS << ", Result";
               for (size_t I = 0, N = Args.size(); I < N; ++I)
                 OS << ", V" << I;
               OS << "))\n";
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index e8883488f3235..2d9f5c3381018 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -50,20 +50,32 @@ using TypeSpec = std::string;
 
 namespace {
 class SVEType {
-  bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat, MFloat;
-  bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
-      Svcount;
+
+  enum TypeKind {
+    Invalid,
+    Void,
+    Float,
+    SInt,
+    UInt,
+    BFloat16,
+    MFloat8,
+    Svcount,
+    PrefetchOp,
+    PredicatePattern,
+    Predicate,
+    Fpm
+  };
+  TypeKind Kind;
+  bool Immediate, Constant, Pointer, DefaultType, IsScalable;
   unsigned Bitwidth, ElementBitwidth, NumVectors;
 
 public:
   SVEType() : SVEType("", 'v') {}
 
   SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
-      : Float(false), Signed(true), Immediate(false), Void(false),
-        Constant(false), Pointer(false), BFloat(false), MFloat(false),
-        DefaultType(false), IsScalable(true), Predicate(false),
-        PredicatePattern(false), PrefetchOp(false), Svcount(false),
-        Bitwidth(128), ElementBitwidth(~0U), NumVectors(NumVectors) {
+      : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
+        DefaultType(false), IsScalable(true), Bitwidth(128),
+        ElementBitwidth(~0U), NumVectors(NumVectors) {
     if (!TS.empty())
       applyTypespec(TS);
     applyModifier(CharMod);
@@ -74,39 +86,39 @@ class SVEType {
   }
 
   bool isPointer() const { return Pointer; }
-  bool isVoidPointer() const { return Pointer && Void; }
-  bool isSigned() const { return Signed; }
+  bool isConstant() const { return Constant; }
   bool isImmediate() const { return Immediate; }
   bool isScalar() const { return NumVectors == 0; }
   bool isVector() const { return NumVectors > 0; }
   bool isScalableVector() const { return isVector() && IsScalable; }
   bool isFixedLengthVector() const { return isVector() && !IsScalable; }
-  bool isChar() const { return ElementBitwidth == 8 && !MFloat; }
-  bool isVoid() const { return Void && !Pointer; }
+  bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
+  bool isVoid() const { return Kind == Void; }
   bool isDefault() const { return DefaultType; }
-  bool isFloat() const { return Float && !BFloat && !MFloat; }
-  bool isBFloat() const { return BFloat && !Float && !MFloat; }
-  bool isMFloat() const {
-    return MFloat && !BFloat && !Float;
-  }
-  bool isFloatingPoint() const { return Float || BFloat; }
-  bool isInteger() const {
-    return !isFloatingPoint() && !Predicate && !Svcount;
+  bool isFloat() const { return Kind == Float; }
+  bool isBFloat() const { return Kind == BFloat16; }
+  bool isMFloat() const { return Kind == MFloat8; }
+  bool isFloatingPoint() const {
+    return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
   }
+  bool isInteger() const { return Kind == SInt || Kind == UInt; }
+  bool isSignedInteger() const { return Kind == SInt; }
+  bool isUnsignedInteger() const { return Kind == UInt; }
   bool isScalarPredicate() const {
-    return !isFloatingPoint() && Predicate && NumVectors == 0;
+    return Kind == Predicate && NumVectors == 0;
   }
-  bool isPredicateVector() const { return Predicate; }
-  bool isPredicatePattern() const { return PredicatePattern; }
-  bool isPrefetchOp() const { return PrefetchOp; }
-  bool isSvcount() const { return Svcount; }
-  bool isConstant() const { return Constant; }
+  bool isPredicate() const { return Kind == Predicate; }
+  bool isPredicatePattern() const { return Kind == PredicatePattern; }
+  bool isPrefetchOp() const { return Kind == PrefetchOp; }
+  bool isSvcount() const { return Kind == Svcount; }
+  bool isFpm() const { return Kind == Fpm; }
+  bool isInvalid() const { return Kind == Invalid; }
   unsigned getElementSizeInBits() const { return ElementBitwidth; }
   unsigned getNumVectors() const { return NumVectors; }
 
   unsigned getNumElements() const {
     assert(ElementBitwidth != ~0U);
-    return Bitwidth / ElementBitwidth;
+    return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
   }
   unsigned getSizeInBits() const {
     return Bitwidth;
@@ -126,6 +138,9 @@ class SVEType {
 
   /// Applies a prototype modifier to the type.
   void applyModifier(char Mod);
+
+  /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
+  std::string builtinBaseType() const;
 };
 
 class SVEEmitter;
@@ -423,170 +438,201 @@ const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
 // Type implementation
 //===----------------------------------------------------------------------===//
 
-std::string SVEType::builtin_str() const {
-  std::string S;
-  if (isVoid())
+std::string SVEType::builtinBaseType() const {
+  switch (Kind) {
+  case TypeKind::Void:
     return "v";
-
-  if (isScalarPredicate())
-    return "b";
-
-  if (isSvcount())
+  case TypeKind::Svcount:
     return "Qa";
-
-  if (isVoidPointer())
-    S += "v";
-  else if (!isFloatingPoint())
+  case TypeKind::PrefetchOp:
+  case TypeKind::PredicatePattern:
+    return "i";
+  case TypeKind::Fpm:
+    return "Wi";
+  case TypeKind::Predicate:
+    return "b";
+  case TypeKind::BFloat16:
+    assert(ElementBitwidth == 16 && "Invalid BFloat16!");
+    return "y";
+  case TypeKind::MFloat8:
+    assert(ElementBitwidth == 8 && "Invalid MFloat8!");
+    return "c";
+  case TypeKind::Float:
     switch (ElementBitwidth) {
-    case 1: S += "b"; break;
-    case 8: S += "c"; break;
-    case 16: S += "s"; break;
-    case 32: S += "i"; break;
-    case 64: S += "Wi"; break;
-    case 128: S += "LLLi"; break;
-    default: llvm_unreachable("Unhandled case!");
+    case 16:
+      return "h";
+    case 32:
+      return "f";
+    case 64:
+      return "d";
+    default:
+      llvm_unreachable("Unhandled float width!");
     }
-  else if (isFloat())
+  case TypeKind::SInt:
+  case TypeKind::UInt:
     switch (ElementBitwidth) {
-    case 16: S += "h"; break;
-    case 32: S += "f"; break;
-    case 64: S += "d"; break;
-    default: llvm_unreachable("Unhandled case!");
+    case 1:
+      return "b";
+    case 8:
+      return "c";
+    case 16:
+      return "s";
+    case 32:
+      return "i";
+    case 64:
+      return "Wi";
+    case 128:
+      return "LLLi";
+    default:
+      llvm_unreachable("Unhandled bitwidth!");
     }
-  else if (isBFloat()) {
-    assert(ElementBitwidth == 16 && "Not a valid BFloat.");
-    S += "y";
-  } else if (isMFloat()) {
-    assert(ElementBitwidth == 8 && "Not a valid MFloat.");
-    S += "m";
+  case TypeKind::Invalid:
+    llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
   }
+  llvm_unreachable("Unhandled TypeKind!");
+}
 
-  if (!isFloatingPoint()) {
-    if ((isChar() || isPointer()) && !isVoidPointer()) {
-      // Make chars and typed pointers explicitly signed.
-      if (Signed)
-        S = "S" + S;
-      else if (!Signed)
-        S = "U" + S;
-    } else if (!isVoidPointer() && !Signed) {
-      S = "U" + S;
-    }
+std::string SVEType::builtin_str() const {
+  std::string Prefix;
+
+  if (isScalableVector())
+    Prefix = "q" + llvm::utostr(getNumElements() * NumVectors);
+  else if (isFixedLengthVector())
+    Prefix = "V" + llvm::utostr(getNumElements() * NumVectors);
+  else if (isImmediate()) {
+    assert(!isFloatingPoint() && "fp immediates are not supported");
+    Prefix = "I";
   }
 
-  // Constant indices are "int", but have the "constant expression" modifier.
-  if (isImmediate()) {
-    assert(!isFloat() && "fp immediates are not supported");
-    S = "I" + S;
-  }
+  // Make chars and integer pointers explicitly signed.
+  if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
+    Prefix += "S";
+  else if (isUnsignedInteger())
+    Prefix += "U";
 
-  if (isScalar()) {
-    if (Constant) S += "C";
-    if (Pointer) S += "*";
-    return S;
-  }
+  std::string BuiltinStr = Prefix + builtinBaseType();
+  if (isConstant())
+    BuiltinStr += "C";
+  if (isPointer())
+    BuiltinStr += "*";
 
-  if (isFixedLengthVector())
-    return "V" + utostr(getNumElements() * NumVectors) + S;
-  return "q" + utostr(getNumElements() * NumVectors) + S;
+  return BuiltinStr;
 }
 
 std::string SVEType::str() const {
-  if (isPredicatePattern())
-    return "enum svpattern";
+  std::string TypeStr;
 
-  if (isPrefetchOp())
+  switch (Kind) {
+  case TypeKind::PrefetchOp:
     return "enum svprfop";
-
-  std::string S;
-  if (Void)
-    S += "void";
-  else {
-    if (isScalableVector() || isSvcount())
-      S += "sv";
-    if (!Signed && !isFloatingPoint())
-      S += "u";
-
-    if (Float)
-      S += "float";
-    else if (isSvcount())
-      S += "count";
-    else if (isScalarPredicate() || isPredicateVector())
-      S += "bool";
-    else if (isBFloat())
-      S += "bfloat";
-    else if (isMFloat())
-      S += "mfloat";
-    else
-      S += "int";
-
-    if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
-      S += utostr(ElementBitwidth);
-    if (isFixedLengthVector())
-      S += "x" + utostr(getNumElements());
-    if (NumVectors > 1)
-      S += "x" + utostr(NumVectors);
-    if (!isScalarPredicate())
-      S += "_t";
+  case TypeKind::PredicatePattern:
+    return "enum svpattern";
+  case TypeKind::Fpm:
+    TypeStr += "fpm";
+    break;
+  case TypeKind::Void:
+    TypeStr += "void";
+    break;
+  case TypeKind::Float:
+    TypeStr += "float" + llvm::utostr(ElementBitwidth);
+    break;
+  case TypeKind::Svcount:
+    TypeStr += "svcount";
+    break;
+  case TypeKind::Predicate:
+    TypeStr += "bool";
+    break;
+  case TypeKind::BFloat16:
+    TypeStr += "bfloat16";
+    break;
+  case TypeKind::MFloat8:
+    TypeStr += "mfloat8";
+    break;
+  case TypeKind::SInt:
+    TypeStr += "int" + llvm::utostr(ElementBitwidth);
+    break;
+  case TypeKind::UInt:
+    TypeStr += "uint" + llvm::utostr(ElementBitwidth);
+    break;
+  case TypeKind::Invalid:
+    llvm_unreachable("Attempting to resolve type name from Invalid type!");
   }
 
-  if (Constant)
-    S += " const";
-  if (Pointer)
-    S += " *";
-
-  return S;
+  if (isFixedLengthVector())
+    TypeStr += "x" + llvm::utostr(getNumElements());
+  else if (isScalableVector())
+    TypeStr = "sv" + TypeStr;
+
+  if (NumVectors > 1)
+    TypeStr += "x" + llvm::utostr(NumVectors);
+  if (!isScalarPredicate() && !isVoid())
+    TypeStr += "_t";
+  if (isConstant())
+    TypeStr += " const";
+  if (isPointer())
+    TypeStr += " *";
+
+  return TypeStr;
 }
 
 void SVEType::applyTypespec(StringRef TS) {
   for (char I : TS) {
     switch (I) {
     case 'Q':
-      Svcount = true;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = Svcount;
       break;
     case 'P':
-      Predicate = true;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = Predicate;
       break;
     case 'U':
-      Signed = false;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = UInt;
       break;
     case 'c':
+      Kind = isInvalid() ? SInt : Kind;
       ElementBitwidth = 8;
       break;
     case 's':
+      Kind = isInvalid() ? SInt : Kind;
       ElementBitwidth = 16;
       break;
     case 'i':
+      Kind = isInvalid() ? SInt : Kind;
       ElementBitwidth = 32;
       break;
     case 'l':
+      Kind = isInvalid() ? SInt : Kind;
       ElementBitwidth = 64;
       break;
     case 'q':
+      Kind = isInvalid() ? SInt : Kind;
       ElementBitwidth = 128;
       break;
     case 'h':
-      Float = true;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = Float;
       ElementBitwidth = 16;
       break;
     case 'f':
-      Float = true;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = Float;
       ElementBitwidth = 32;
       break;
     case 'd':
-      Float = true;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = Float;
       ElementBitwidth = 64;
       break;
     case 'b':
-      BFloat = true;
-      Float = false;
-      MFloat = false;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = BFloat16;
       ElementBitwidth = 16;
       break;
     case 'm':
-      Signed = false;
-      MFloat = true;
-      Float = false;
-      BFloat = false;
+      assert(isInvalid() && "Unexpected use of typespec modifier");
+      Kind = MFloat8;
       ElementBitwidth = 8;
       break;
     default:
@@ -599,7 +645,8 @@ void SVEType::applyTypespec(StringRef TS) {
 void SVEType::applyModifier(char Mod) {
   switch (Mod) {
   case 'v':
-    Void = true;
+    Kind = Void;
+    NumVectors = 0;
     break;
   case 'd':
     DefaultType = true;
@@ -613,7 +660,7 @@ void SVEType::applyModifier(char Mod) {
     NumVectors = 0;
     break;
   case 'e':
-    Signed = false;
+    Kind = UInt;
     ElementBitwidth /= 2;
     break;
   case 'h':
@@ -623,20 +670,14 @@ void SVEType::applyModifier(char Mod) {
     ElementBitwidth /= 4;
     break;
   case 'b':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth /= 4;
     break;
   case 'o':
     ElementBitwidth *= 4;
     break;
   case 'P':
-    Signed = true;
-    Float = false;
-    BFloat = false;
-    Predicate = true;
-    Svcount = false;
+    Kind = Predicate;
     Bitwidth = 16;
     ElementBitwidth = 1;
     break;
@@ -659,105 +700,66 @@ void SVEType::applyModifier(char Mod) {
     NumVectors = 0;
     break;
   case '@':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth /= 4;
     NumVectors = 0;
     break;
   case 'K':
-    Signed = true;
-    Float = false;
-    BFloat = false;
+    Kind = SInt;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
   case 'L':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
   case 'u':
-    Predicate = false;
-    Svcount = false;
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     break;
   case 'x':
-    Predicate = false;
-    Svcount = false;
-    Signed = true;
-    Float = false;
-    BFloat = false;
+    Kind = SInt;
     break;
   case 'i':
-    Predicate = false;
-    Svcount = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
-    Signed = false;
     Immediate = true;
     break;
   case 'I':
-    Predicate = false;
-    Svcount = false;
-    Float = false;
-    BFloat = false;
+    Kind = PredicatePattern;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = true;
     Immediate = true;
-    PredicatePattern = true;
     break;
   case 'J':
-    Predicate = false;
-    Svcount = false;
-    Float = false;
-    BFloat = false;
+    Kind = PrefetchOp;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = true;
     Immediate = true;
-    PrefetchOp = true;
     break;
   case 'k':
-    Predicate = false;
-    Svcount = false;
-    Signed = true;
-    Float = false;
-    BFloat = false;
+    Kind = SInt;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
   case 'l':
-    Predicate = false;
-    Svcount = false;
-    Signed = true;
-    Float = false;
-    BFloat = false;
+    Kind = SInt;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
   case 'm':
-    Predicate = false;
-    Svcount = false;
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
+  case '>':
+    Kind = Fpm;
+    ElementBitwidth = Bitwidth = 64;
+    NumVectors = 0;
+    break;
   case 'n':
-    Predicate = false;
-    Svcount = false;
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
@@ -769,162 +771,144 @@ void SVEType::applyModifier(char Mod) {
     NumVectors = 0;
     break;
   case 'f':
-    Signed = false;
+    Kind = UInt;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
   case 'g':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = 64;
     break;
   case '[':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = 8;
     break;
   case 't':
-    Signed = true;
-    Float = false;
-    BFloat = false;
+    Kind = SInt;
     ElementBitwidth = 32;
     break;
   case 'z':
-    Signed = false;
-    Float = false;
-    BFloat = false;
+    Kind = UInt;
     ElementBitwidth = 32;
     break;
   case 'O':
-    Predicate = false;
-    Svcount = false;
-    Float = true;
+    Kind = Float;
     ElementBitwidth = 16;
     break;
   case 'M':
-    Predicate = false;
-    Svcount = false;
-    Float = true;
-    BFloat = false;
+    Kind = Float;
     ElementBitwidth = 32;
     break;
   case 'N':
-    Predicate = false;
-    Svcount = false;
-    Float = true;
+    Kind = Float;
     ElementBitwidth = 64;
     break;
   case 'Q':
+    Kind = Void;
     Constant = true;
     Pointer = true;
-    Void = true;
     NumVectors = 0;
     break;
   case 'S':
+    Kind = SInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 8;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'W':
+    Kind = UInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 8;
     NumVectors = 0;
-    Signed = false;
     break;
   case 'T':
+    Kind = SInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 16;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'X':
+    Kind = UInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 16;
     NumVectors = 0;
-    Signed = false;
     break;
   case 'Y':
+    Kind = UInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = false;
     break;
   case 'U':
+    Kind = SInt;
     Constant = true;
     Pointer = true;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = true;
     break;
   case '%':
+    Kind = Void;
     Pointer = true;
-    Void = true;
     NumVectors = 0;
     break;
   case 'A':
+    Kind = SInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 8;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'B':
+    Kind = SInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 16;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'C':
+    Kind = SInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'D':
+    Kind = SInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
-    Signed = true;
     break;
   case 'E':
+    Kind = UInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 8;
     NumVectors = 0;
-    Signed = false;
     break;
   case 'F':
+    Kind = UInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 16;
     NumVectors = 0;
-    Signed = false;
     break;
   case 'G':
+    Kind = UInt;
     Pointer = true;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
-    Signed = false;
     break;
   case '$':
-    Predicate = false;
-    Svcount = false;
-    Float = false;
-    BFloat = true;
+    Kind = BFloat16;
     ElementBitwidth = 16;
     break;
   case '}':
-    Predicate = false;
-    Signed = true;
-    Svcount = true;
+    Kind = Svcount;
     NumVectors = 0;
-    Float = false;
-    BFloat = false;
+    break;
+  case '~':
+    Kind = MFloat8;
+    ElementBitwidth = 8;
     break;
   case '.':
     llvm_unreachable(". is never a type in itself");
@@ -1044,11 +1028,14 @@ std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
 
     // Replace templated arg with the right suffix (e.g. u32)
     std::string TypeCode;
-    if (T.isInteger())
-      TypeCode = T.isSigned() ? 's' : 'u';
+
+    if (T.isSignedInteger())
+      TypeCode = 's';
+    else if (T.isUnsignedInteger())
+      TypeCode = 'u';
     else if (T.isSvcount())
       TypeCode = 'c';
-    else if (T.isPredicateVector())
+    else if (T.isPredicate())
       TypeCode = 'b';
     else if (T.isBFloat())
       TypeCode = "bf";
@@ -1152,7 +1139,7 @@ uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
     return encodeEltType("EltTyMFloat8");
   }
 
-  if (T.isPredicateVector() || T.isSvcount()) {
+  if (T.isPredicate() || T.isSvcount()) {
     switch (T.getElementSizeInBits()) {
     case 8:
       return encodeEltType("EltTyBool8");
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index 3ed42a187fd80..d76c6ede3fe5a 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -36,7 +36,7 @@ def clean(args):
             + "\tRemoves all files with extension from <path>."
         )
         return 1
-    for path in args[1:-1]:
+    for path in args[0:-1]:
         for filename in findFilesWithExtension(path, args[-1]):
             os.remove(filename)
     return 0
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 186f7cc0ace54..cdedbcbaa4072 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -3599,7 +3599,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/591.html">591</a></td>
     <td>CD4</td>
     <td>When a dependent base class is the current instantiation</td>
-    <td class="none" align="center">No</td>
+    <td class="unreleased" align="center">Clang 20</td>
   </tr>
   <tr id="592">
     <td><a href="https://cplusplus.github.io/CWG/issues/592.html">592</a></td>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index da01cf6ceab59..a8e79cd3475ab 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -239,7 +239,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Remove Deprecated Array Comparisons from C++26</td>
   <td><a href="https://wg21.link/P2865R6">P2865R6</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 20</td>
  </tr>
  <tr>
   <td>Structured Bindings can introduce a Pack</td>
@@ -249,7 +249,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>The Oxford variadic comma</td>
   <td><a href="https://wg21.link/P3176R1">P3176R1</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 20</td>
  </tr>
 </table>
 </details>
diff --git a/clang/www/hacking.html b/clang/www/hacking.html
index 78b67cd05c53c..a17e791eed477 100755
--- a/clang/www/hacking.html
+++ b/clang/www/hacking.html
@@ -89,9 +89,9 @@ <h2 id="debugging">Debugging</h2>
       qualifiers, and the <tt>getTypePtr()</tt> method to get the
       wrapped <tt>Type*</tt> which you can then dump.</li>
       <li>For <a href="https://lldb.llvm.org"> <tt>LLDB</tt></a> users there are
-      data formatters for clang data structures in
-      <a href="https://github.com/llvm/llvm-project/blob/main/clang/utils/ClangDataFormat.py">
-      <tt>clang/utils/ClangDataFormat.py</tt></a>.</li>
+      data formatters for LLVM data structures in
+      <a href="https://github.com/llvm/llvm-project/blob/main/llvm/utils/lldbDataFormatters.py">
+      <tt>llvm/utils/lldbDataFormatters.py</tt></a>.</li>
     </ul>
 
   <!--=====================================================================-->
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index 20054c6e85a40..80d5aaabfd8c3 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -768,7 +768,7 @@ if (CMAKE_LINKER MATCHES "link.exe$")
   # it, but CMake doesn't seem to have a way to set linker flags for
   # individual static libraries, so we enable the suppression flag for
   # the whole compiler-rt project.
-  set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /IGNORE:4221")
+  set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} ${CMAKE_CXX_LINKER_WRAPPER_FLAG}/IGNORE:4221")
 endif()
 
 add_subdirectory(include)
diff --git a/compiler-rt/CODE_OWNERS.TXT b/compiler-rt/CODE_OWNERS.TXT
deleted file mode 100644
index bd51a1073cc38..0000000000000
--- a/compiler-rt/CODE_OWNERS.TXT
+++ /dev/null
@@ -1,77 +0,0 @@
-This file is a list of the people responsible for ensuring that patches for a
-particular part of compiler-rt are reviewed, either by themself or by
-someone else. They are also the gatekeepers for their part of compiler-rt, with
-the final word on what goes in or not.
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts. The fields are: name (N), email (E), web-address
-(W), PGP key ID and fingerprint (P), description (D), and snail-mail address
-(S).
-
-N: Saleem Abdulrasool
-E: compnerd@compnerd.org
-D: builtins library
-
-N: Andrew Browne
-E: browneee@google.com
-D: DataFlowSanitizer
-
-N: Vitaly Buka
-E: vitalybuka@google.com
-D: Sanitizers
-
-N: Peter Collingbourne
-E: peter@pcc.me.uk
-D: CFI, SafeStack
-
-N: Lang Hames
-E: lhames@gmail.com
-D: ORC
-
-N: Petr Hosek
-E: phosek@google.com
-D: CRT, CMake build
-
-N: Teresa Johnson
-E: tejohnson@google.com
-D: MemProf
-
-N: Kostya Kortchinsky
-E: kostya.kortchinsky@gmail.com
-D: SCUDO
-
-N: Mitch Phillips
-E: mitchp@google.com
-D: GWP ASAN
-
-N: Alexander Potapenko
-E: glider@google.com
-D: Sanitizers
-
-N: Kostya Serebryany
-E: kcc@google.com
-D: AddressSanitizer, sanitizer_common, LeakSanitizer, LibFuzzer
-
-N: Richard Smith
-E: richard-llvm@metafoo.co.uk
-D: UndefinedBehaviorSanitizer
-
-N: Evgeniy Stepanov
-E: eugenis@google.com
-D: MemorySanitizer, Android port of sanitizers
-
-N: Dmitry Vyukov
-E: dvyukov@google.com
-D: ThreadSanitizer
-
-N: Bill Wendling
-E: isanbard@gmail.com
-D: Profile runtime library
-
-N: Christopher Apple, David Trevelyan
-E: cja-private@pm.me, realtime.sanitizer@gmail.com
-D: Realtime Sanitizer (RTSan)
-
-N: Alexander Shaposhnikov
-E: alexander.v.shaposhnikov@gmail.com
-D: Numerical Sanitizer (NSAN)
diff --git a/compiler-rt/Maintainers.md b/compiler-rt/Maintainers.md
new file mode 100644
index 0000000000000..5faf6741c4679
--- /dev/null
+++ b/compiler-rt/Maintainers.md
@@ -0,0 +1,110 @@
+# Compiler-rt maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+LLVM compiler-rt.
+
+## Current Maintainers
+
+The following people are the active maintainers for the project. Please reach
+out to them for code reviews, questions about their area of expertise, or other
+assistance.
+
+### Builtins Library
+
+Saleem Abdulrasool \
+compnerd@compnerd.org (email), [compnerd](https://github.com/compnerd) (GitHub)
+
+### CFI
+
+Peter Collingbourne \
+peter@pcc.me.uk (email), [pcc](https://github.com/pcc) (GitHub)
+
+### CMake build
+
+Petr Hosek \
+phosek@google.com (email), [petrhosek](https://github.com/petrhosek) (GitHub)
+
+### CRT
+
+Petr Hosek \
+phosek@google.com (email), [petrhosek](https://github.com/petrhosek) (GitHub)
+
+### GWP ASAN
+
+Christopher Ferris \
+cferris@google.com (email), [cferris1000](https://github.com/cferris1000) (GitHub)
+
+### MemProfiling
+
+Teresa Johnson \
+tejohnson@google.com (email), [teresajohnson](https://github.com/teresajohnson) (GitHub)
+
+### SafeStack
+
+Peter Collingbourne \
+peter@pcc.me.uk (email), [pcc](https://github.com/pcc) (GitHub)
+
+### Sanitizers
+
+#### Sanitizers not covered by someone else
+
+Vitaly Buka \
+vitalybuka@google.com (email), [vitalybuka](https://github.com/vitalybuka) (GitHub) \
+Alexander Potapenko \
+glider@google.com (email), [ramosian-glider](https://github.com/ramosian-glider) (GitHub)
+
+#### Data Flow Sanitizer
+
+Andrew Browne \
+browneee@google.com (email), [browneee](https://github.com/browneee) (GitHub)
+
+#### Numerical Sanitizer (NSAN)
+
+Alexander Shaposhnikov \
+alexander.v.shaposhnikov@gmail.com (email), [alexander-shaposhnikov](https://github.com/alexander-shaposhnikov) (GitHub)
+
+#### Realtime Sanitizer (RTSan)
+
+Christopher Apple \
+cja-private@pm.me (email), [cjappl](https://github.com/cjappl) (GitHub) \
+David Trevelyan \
+david.trevelyan@gmail.com (email), [davidtrevelyan](https://github.com/davidtrevelyan) (GitHub)
+
+#### Thread Sanitizer
+
+Dmitry Vyukov \
+dvyukov@google.com (email), [dvyukov](https://github.com/dvyukov) (GitHub)
+
+#### Undefined Behavior Sanitizer
+
+Richard Smith \
+richard-llvm@metafoo.co.uk (email), [zygoloid](https://github.com/zygoloid) (GitHub)
+
+### ORC
+
+Lang Hames \
+lhames@gmail.com (email), [lhames](https://github.com/lhames) (GitHub)
+
+### Profile runtime library
+
+Bill Wendling \
+isanbard@gmail.com (email), [isanbard](https://github.com/isanbard) (GitHub)
+
+### SCUDO
+
+Christopher Ferris \
+cferris@google.com (email), [cferris1000](https://github.com/cferris1000) (GitHub)
+
+## Inactive Maintainers
+
+The following people have graciously spent time performing maintainer
+responsibilities but are no longer active in that role. Thank you for all your
+help with the success of the project!
+
+### Inactive or former component maintainers
+
+Kostya Serebryany ([kcc](https://github.com/kcc)) -- Sanitizers \
+Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers \
+Kostya Kortchinsky ([cryptoad](https://github.com/cryptoad)) -- SCUDO \
+Mitch Phillips ([hctim](https://github.com/hctim)) -- GWP ASAN
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index b29ae179c2b4f..5a1e8db61023b 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -102,7 +102,7 @@ if(APPLE)
 set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM64})
 else()
 set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64}
-		powerpc64le ${HEXAGON} ${LOONGARCH64})
+               powerpc64le ${HEXAGON} ${LOONGARCH64} ${RISCV32} ${RISCV64})
 endif()
 set(ALL_XRAY_DSO_SUPPORTED_ARCH ${X86_64} ${ARM64})
 set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index f3c8fbe2c2fec..74a5d4edcd859 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -136,14 +136,13 @@ function(darwin_test_archs os valid_archs)
 
   # The simple program will build for x86_64h on the simulator because it is
   # compatible with x86_64 libraries (mostly), but since x86_64h isn't actually
-  # a valid or useful architecture for the iOS simulator we should drop it.
+  # a valid or useful architecture for the simulators. We should drop it.
   if(${os} MATCHES "^(iossim|tvossim|watchossim)$")
     list(REMOVE_ITEM archs "x86_64h")
-  endif()
-
-  if(${os} MATCHES "iossim")
-    message(STATUS "Disabling i386 slice for iossim")
-    list(REMOVE_ITEM archs "i386")
+    if ("i386" IN_LIST archs)
+      list(REMOVE_ITEM archs "i386")
+      message(STATUS "Disabling i386 slice for simulator")
+    endif()
   endif()
 
   if(${os} MATCHES "^ios$")
diff --git a/compiler-rt/cmake/base-config-ix.cmake b/compiler-rt/cmake/base-config-ix.cmake
index 286a622a4b520..9717c21d8977b 100644
--- a/compiler-rt/cmake/base-config-ix.cmake
+++ b/compiler-rt/cmake/base-config-ix.cmake
@@ -77,13 +77,15 @@ else()
   set(COMPILER_RT_TEST_CXX_COMPILER ${CMAKE_CXX_COMPILER} CACHE PATH "C++ Compiler to use for testing")
 endif()
 
-if("${COMPILER_RT_TEST_COMPILER}" MATCHES "clang[+]*$")
+get_filename_component(_test_compiler_name "${COMPILER_RT_TEST_COMPILER}" NAME)
+if("${COMPILER_RT_TEST_COMPILER}" STREQUAL "${CMAKE_C_COMPILER}")
+  set(COMPILER_RT_TEST_COMPILER_ID "${CMAKE_C_COMPILER_ID}")
+elseif("${_test_compiler_name}" MATCHES "clang.*")
   set(COMPILER_RT_TEST_COMPILER_ID Clang)
-elseif("${COMPILER_RT_TEST_COMPILER}" MATCHES "clang.*.exe$")
-  set(COMPILER_RT_TEST_COMPILER_ID Clang)
-elseif("${COMPILER_RT_TEST_COMPILER}" MATCHES "cl.exe$")
+elseif("${_test_compiler_name}" MATCHES "cl.exe$")
   set(COMPILER_RT_TEST_COMPILER_ID MSVC)
 else()
+  message(STATUS "Unknown compiler ${COMPILER_RT_TEST_COMPILER}, assuming GNU")
   set(COMPILER_RT_TEST_COMPILER_ID GNU)
 endif()
 
diff --git a/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake b/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake
index baf149306f91b..4f1d67dcb38e0 100644
--- a/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake
+++ b/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake
@@ -11,4 +11,5 @@ set(COMPILER_RT_BUILD_BUILTINS OFF CACHE BOOL "")
 set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "")
 set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "")
 set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_CTX_PROFILE OFF CACHE BOOL "")
 
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index c66b0465a0b54..39613da81ecb4 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -730,10 +730,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_COVMAP_VERSION 6
 
 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
- * version for other variants of profile. We set the 8th most significant bit 
+ * version for other variants of profile. We set the 8th most significant bit
  * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
  * generated profile, and 0 if this is a Clang FE generated profile.
  * 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 54th bit indicates whether to always instrument loop entry blocks.
+ * The 58th bit indicates whether to always instrument function entry blocks.
  * The 59th bit indicates whether to use debug info to correlate profiles.
  * The 60th bit indicates single byte coverage instrumentation.
  * The 61st bit indicates function entry instrumentation only.
@@ -742,6 +744,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
  */
 #define VARIANT_MASKS_ALL 0xffffffff00000000ULL
 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
+#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
diff --git a/compiler-rt/include/sanitizer/memprof_interface.h b/compiler-rt/include/sanitizer/memprof_interface.h
index 4660a7818c92b..6d9b2a2394f41 100644
--- a/compiler-rt/include/sanitizer/memprof_interface.h
+++ b/compiler-rt/include/sanitizer/memprof_interface.h
@@ -47,9 +47,9 @@ void SANITIZER_CDECL __memprof_print_accumulated_stats(void);
 
 /// User-provided default option settings.
 ///
-/// You can provide your own implementation of this function to return a string
-/// containing MemProf runtime options (for example,
-/// <c>verbosity=1:print_stats=1</c>).
+/// You can set these options via the -memprof-runtime-default-options LLVM flag
+/// or you can provide your own implementation of this function. See
+/// memprof_flags.h for more info.
 ///
 /// \returns Default options string.
 const char *SANITIZER_CDECL __memprof_default_options(void);
diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index fb3d74283a61e..5ec995ae159b7 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -141,7 +141,7 @@ append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
 # LLVM turns /OPT:ICF back on when LLVM_ENABLE_PDBs is set
 # we _REALLY_ need to turn it back off for ASAN, because the way
 # asan emulates weak functions from DLLs requires NOICF
-append_list_if(MSVC "/DEBUG;/OPT:NOICF" ASAN_DYNAMIC_LINK_FLAGS)
+append_list_if(MSVC "LINKER:/DEBUG;LINKER:/OPT:NOICF" ASAN_DYNAMIC_LINK_FLAGS)
 
 set(ASAN_DYNAMIC_LIBS
   ${COMPILER_RT_UNWINDER_LINK_LIBS}
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index 4129ee8076122..e6c95dfb82c32 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -85,7 +85,7 @@ int OnExit() {
 // ---------------------- Wrappers ---------------- {{{1
 using namespace __asan;
 
-DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
 
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
@@ -529,13 +529,13 @@ DEFINE_REAL(char*, index, const char *string, int c)
     return REAL(strcat)(to, from);
   }
 
-INTERCEPTOR(char*, strncat, char *to, const char *from, uptr size) {
+INTERCEPTOR(char*, strncat, char *to, const char *from, usize size) {
   void *ctx;
   ASAN_INTERCEPTOR_ENTER(ctx, strncat);
   AsanInitFromRtl();
   if (flags()->replace_str) {
     uptr from_length = MaybeRealStrnlen(from, size);
-    uptr copy_length = Min(size, from_length + 1);
+    uptr copy_length = Min<uptr>(size, from_length + 1);
     ASAN_READ_RANGE(ctx, from, copy_length);
     uptr to_length = internal_strlen(to);
     ASAN_READ_STRING_OF_LEN(ctx, to, to_length, to_length);
@@ -617,12 +617,12 @@ INTERCEPTOR(char*, __strdup, const char *s) {
 }
 #endif // ASAN_INTERCEPT___STRDUP
 
-INTERCEPTOR(char*, strncpy, char *to, const char *from, uptr size) {
+INTERCEPTOR(char*, strncpy, char *to, const char *from, usize size) {
   void *ctx;
   ASAN_INTERCEPTOR_ENTER(ctx, strncpy);
   AsanInitFromRtl();
   if (flags()->replace_str) {
-    uptr from_size = Min(size, MaybeRealStrnlen(from, size) + 1);
+    uptr from_size = Min<uptr>(size, MaybeRealStrnlen(from, size) + 1);
     CHECK_RANGES_OVERLAP("strncpy", to, from_size, from, from_size);
     ASAN_READ_RANGE(ctx, from, from_size);
     ASAN_WRITE_RANGE(ctx, to, size);
diff --git a/compiler-rt/lib/asan/asan_interceptors.h b/compiler-rt/lib/asan/asan_interceptors.h
index 826b45f5ada8c..3e2386eaf8092 100644
--- a/compiler-rt/lib/asan/asan_interceptors.h
+++ b/compiler-rt/lib/asan/asan_interceptors.h
@@ -124,11 +124,11 @@ void InitializePlatformInterceptors();
 # define ASAN_INTERCEPT_PTHREAD_ATFORK 0
 #endif
 
-DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
+DECLARE_REAL(int, memcmp, const void *a1, const void *a2, SIZE_T size)
 DECLARE_REAL(char*, strchr, const char *str, int c)
 DECLARE_REAL(SIZE_T, strlen, const char *s)
-DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size)
-DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
+DECLARE_REAL(char*, strncpy, char *to, const char *from, SIZE_T size)
+DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
 DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
 
 #  if !SANITIZER_APPLE
diff --git a/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
index eb44f8f2f729b..14727a5d665ed 100644
--- a/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
+++ b/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
@@ -18,8 +18,8 @@
 #include "asan_mapping.h"
 #include "interception/interception.h"
 
-DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size)
-DECLARE_REAL(void *, memset, void *block, int c, uptr size)
+DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size)
+DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size)
 
 namespace __asan {
 
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 73c1b5062ec9a..70dc7d860d8f6 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -574,7 +574,7 @@ set(aarch64_SOURCES
 
 if (COMPILER_RT_HAS_AARCH64_SME)
   if (NOT COMPILER_RT_DISABLE_AARCH64_FMV AND COMPILER_RT_HAS_FNO_BUILTIN_FLAG AND (COMPILER_RT_HAS_AUXV OR COMPILER_RT_BAREMETAL_BUILD))
-    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-libc-mem-routines.S aarch64/sme-abi-init.c aarch64/sme-abi-vg.c aarch64/sme-libc-routines.c)
+    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-libc-mem-routines.S aarch64/sme-abi-init.c aarch64/sme-libc-routines.c)
     message(STATUS "AArch64 SME ABI routines enabled")
     set_source_files_properties(aarch64/sme-libc-routines.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
   else()
diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c b/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c
deleted file mode 100644
index 4b9ee8c1d382d..0000000000000
--- a/compiler-rt/lib/builtins/aarch64/sme-abi-vg.c
+++ /dev/null
@@ -1,18 +0,0 @@
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include "../cpu_model/aarch64.h"
-
-struct FEATURES {
-  unsigned long long features;
-};
-
-extern struct FEATURES __aarch64_cpu_features;
-
-CONSTRUCTOR_ATTRIBUTE static void get_aarch64_cpu_features(void) {
-  if (__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
-    return;
-
-  __init_cpu_features();
-}
diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi.S b/compiler-rt/lib/builtins/aarch64/sme-abi.S
index 3e9bd2c23b2fc..623a95dd4dae5 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-abi.S
+++ b/compiler-rt/lib/builtins/aarch64/sme-abi.S
@@ -8,6 +8,8 @@
 
 #include "../assembly.h"
 
+.set FEAT_SVE_BIT, 30
+.set SVCR_PSTATE_SM_BIT, 0
 
 #if !defined(__APPLE__)
 #define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
@@ -188,39 +190,20 @@ DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_get_current_vg)
   .variant_pcs __arm_get_current_vg
   BTI_C
 
-  stp     x29, x30, [sp, #-16]!
-  .cfi_def_cfa_offset 16
-  mov     x29, sp
-  .cfi_def_cfa w29, 16
-  .cfi_offset w30, -8
-  .cfi_offset w29, -16
   adrp    x17, CPU_FEATS_SYMBOL
   ldr     w17, [x17, CPU_FEATS_SYMBOL_OFFSET]
-  tbnz    w17, #30, 0f
-  adrp    x16, TPIDR2_SYMBOL
-  ldrb    w16, [x16, TPIDR2_SYMBOL_OFFSET]
-  cbz     w16, 1f
+  tbnz    w17, #FEAT_SVE_BIT, 1f
+  adrp    x17, TPIDR2_SYMBOL
+  ldrb    w17, [x17, TPIDR2_SYMBOL_OFFSET]
+  cbz     x17, 2f
 0:
-  mov     x18, x1
-  bl      __arm_sme_state
-  mov     x1, x18
-  and     x17, x17, #0x40000000
-  bfxil   x17, x0, #0, #1
-  cbz     x17, 1f
+  mrs     x17, SVCR
+  tbz     x17, #SVCR_PSTATE_SM_BIT, 2f
+1:
   cntd    x0
-  .cfi_def_cfa wsp, 16
-  ldp     x29, x30, [sp], #16
-  .cfi_def_cfa_offset 0
-  .cfi_restore w30
-  .cfi_restore w29
   ret
-1:
+2:
   mov     x0, xzr
-  .cfi_def_cfa wsp, 16
-  ldp     x29, x30, [sp], #16
-  .cfi_def_cfa_offset 0
-  .cfi_restore w30
-  .cfi_restore w29
   ret
 END_COMPILERRT_OUTLINE_FUNCTION(__arm_get_current_vg)
 
diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c
index 3315dceb7f562..606571d527501 100644
--- a/compiler-rt/lib/builtins/cpu_model/x86.c
+++ b/compiler-rt/lib/builtins/cpu_model/x86.c
@@ -461,16 +461,31 @@ static const char *getIntelProcessorTypeAndSubtype(unsigned Family,
     // Alderlake:
     case 0x97:
     case 0x9a:
+      CPU = "alderlake";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Raptorlake:
     case 0xb7:
     case 0xba:
     case 0xbf:
+      CPU = "raptorlake";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Meteorlake:
     case 0xaa:
     case 0xac:
+      CPU = "meteorlake";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Gracemont:
     case 0xbe:
-      CPU = "alderlake";
+      CPU = "gracemont";
       *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_ALDERLAKE;
       break;
@@ -486,9 +501,14 @@ static const char *getIntelProcessorTypeAndSubtype(unsigned Family,
 
     // Arrowlake S:
     case 0xc6:
+      CPU = "arrowlake-s";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_ARROWLAKE_S;
+      break;
+
     // Lunarlake:
     case 0xbd:
-      CPU = "arrowlake-s";
+      CPU = "lunarlake";
       *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_ARROWLAKE_S;
       break;
@@ -510,6 +530,11 @@ static const char *getIntelProcessorTypeAndSubtype(unsigned Family,
 
     // Emerald Rapids:
     case 0xcf:
+      CPU = "emeraldrapids";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_SAPPHIRERAPIDS;
+      break;
+
     // Sapphire Rapids:
     case 0x8f:
       CPU = "sapphirerapids";
diff --git a/compiler-rt/lib/interception/interception.h b/compiler-rt/lib/interception/interception.h
index 0580d97edda68..3cb6b446638e0 100644
--- a/compiler-rt/lib/interception/interception.h
+++ b/compiler-rt/lib/interception/interception.h
@@ -37,7 +37,7 @@
 #endif
 
 #define SIZE_T __sanitizer::usize
-#define SSIZE_T __sanitizer::sptr
+#define SSIZE_T __sanitizer::ssize
 typedef __sanitizer::sptr    PTRDIFF_T;
 typedef __sanitizer::s64     INTMAX_T;
 typedef __sanitizer::u64     UINTMAX_T;
diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp
index 8767d8e79881c..b0fb263b62f15 100644
--- a/compiler-rt/lib/interception/interception_win.cpp
+++ b/compiler-rt/lib/interception/interception_win.cpp
@@ -418,7 +418,7 @@ static void *AllocateTrampolineRegion(uptr min_addr, uptr max_addr,
   ReportError(
       "interception_win: AllocateTrampolineRegion failed to find free memory; "
       "min_addr: %p, max_addr: %p, func_addr: %p, granularity: %zu\n",
-      (void *)min_addr, (void *)max_addr, granularity);
+      (void *)min_addr, (void *)max_addr, (void *)func_addr, granularity);
   return nullptr;
 #else
   return ::VirtualAlloc(nullptr,
@@ -634,8 +634,11 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xD284:  // 84 D2 : test dl,dl
       return 2;
 
+    case 0xC1F6:  // F6 C1 XX : test cl, XX
+      return 3;
+
     // Cannot overwrite control-instruction. Return 0 to indicate failure.
-    case 0x25FF:  // FF 25 XX XX XX XX : jmp [XXXXXXXX]
+    case 0x25FF:  // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
       return 0;
   }
 
@@ -723,8 +726,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
       return 7;
   }
 
-  switch (0x00FFFFFF & *(u32*)address) {
-    case 0x07c1f6:    // f6 c1 07 : test cl, 0x7
+  switch (0x00FFFFFF & *(u32 *)address) {
     case 0x10b70f:    // 0f b7 10 : movzx edx, WORD PTR [rax]
     case 0xc00b4d:    // 4d 0b c0 : or r8, r8
     case 0xc03345:    // 45 33 c0 : xor r8d, r8d
@@ -808,7 +810,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x798141:  // 41 81 79 XX YY YY YY YY : cmp DWORD PTR [r9+YY], XX XX XX XX
     case 0x7a8141:  // 41 81 7a XX YY YY YY YY : cmp DWORD PTR [r10+YY], XX XX XX XX
     case 0x7b8141:  // 41 81 7b XX YY YY YY YY : cmp DWORD PTR [r11+YY], XX XX XX XX
-    case 0x7c8141:  // 41 81 7c XX YY YY YY YY : cmp DWORD PTR [r12+YY], XX XX XX XX
     case 0x7d8141:  // 41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX
     case 0x7e8141:  // 41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX
     case 0x7f8141:  // 41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX
@@ -835,6 +836,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x2444c7:    // C7 44 24 XX YY YY YY YY
                       //   mov dword ptr [rsp + XX], YYYYYYYY
       return 8;
+
+    case 0x7c8141:  // 41 81 7c ZZ YY XX XX XX XX
+                    // cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX
+      return 9;
   }
 
   switch (*(u32*)(address)) {
@@ -872,7 +877,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x75FF:  // FF 75 XX : push dword ptr [ebp + XX]
       return 3;
     case 0xC1F7:  // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
-    case 0x25FF:  // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
       return 6;
     case 0x3D83:  // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
       return 7;
@@ -1248,7 +1252,7 @@ uptr InternalGetProcAddress(void *module, const char *func_name) {
         char function_name[256];
         size_t funtion_name_length = _strlen(func);
         if (funtion_name_length >= sizeof(function_name) - 1) {
-          ReportError("interception_win: func too long: '%s'\n", func);
+          ReportError("interception_win: func too long: '%s'\n", (char *)func);
           InterceptionFailed();
         }
 
diff --git a/compiler-rt/lib/interception/tests/interception_win_test.cpp b/compiler-rt/lib/interception/tests/interception_win_test.cpp
index cfa1b592f1e46..761c07d4288f7 100644
--- a/compiler-rt/lib/interception/tests/interception_win_test.cpp
+++ b/compiler-rt/lib/interception/tests/interception_win_test.cpp
@@ -801,11 +801,219 @@ const struct InstructionSizeData {
   size_t size;  // hold instruction size or 0 for failure,
                 // e.g. on control instructions
   u8 instr[16];
-  size_t rel_offset;
+  size_t rel_offset;  // filled just for instructions with two operands
+                      // and displacement length of four bytes.
   const char *comment;
 } data[] = {
-    /* sorted list */
-    {1, {0x50}, 0, "50 : push eax / rax"},
+    // clang-format off
+    // sorted list
+    { 0, {0x70, 0x71}, 0, "70 XX : jo XX (short conditional jump)"},
+    { 0, {0x71, 0x71}, 0, "71 XX : jno XX (short conditional jump)"},
+    { 0, {0x72, 0x71}, 0, "72 XX : jb XX (short conditional jump)"},
+    { 0, {0x73, 0x71}, 0, "73 XX : jae XX (short conditional jump)"},
+    { 0, {0x74, 0x71}, 0, "74 XX : je XX (short conditional jump)"},
+    { 0, {0x75, 0x71}, 0, "75 XX : jne XX (short conditional jump)"},
+    { 0, {0x76, 0x71}, 0, "76 XX : jbe XX (short conditional jump)"},
+    { 0, {0x77, 0x71}, 0, "77 XX : ja XX (short conditional jump)"},
+    { 0, {0x78, 0x71}, 0, "78 XX : js XX (short conditional jump)"},
+    { 0, {0x79, 0x71}, 0, "79 XX : jns XX (short conditional jump)"},
+    { 0, {0x7A, 0x71}, 0, "7A XX : jp XX (short conditional jump)"},
+    { 0, {0x7B, 0x71}, 0, "7B XX : jnp XX (short conditional jump)"},
+    { 0, {0x7C, 0x71}, 0, "7C XX : jl XX (short conditional jump)"},
+    { 0, {0x7D, 0x71}, 0, "7D XX : jge XX (short conditional jump)"},
+    { 0, {0x7E, 0x71}, 0, "7E XX : jle XX (short conditional jump)"},
+    { 0, {0x7F, 0x71}, 0, "7F XX : jg XX (short conditional jump)"},
+    { 0, {0xE8, 0x71, 0x72, 0x73, 0x74}, 0, "E8 XX XX XX XX : call <func>"},
+    { 0, {0xE9, 0x71, 0x72, 0x73, 0x74}, 0, "E9 XX XX XX XX : jmp <label>"},
+    { 0, {0xEB, 0x71}, 0, "EB XX : jmp XX (short jump)"},
+    { 0, {0xFF, 0x25, 0x72, 0x73, 0x74, 0x75}, 0, "FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]"},
+    { 1, {0x50}, 0, "50 : push eax / rax"},
+    { 1, {0x51}, 0, "51 : push ecx / rcx"},
+    { 1, {0x52}, 0, "52 : push edx / rdx"},
+    { 1, {0x53}, 0, "53 : push ebx / rbx"},
+    { 1, {0x54}, 0, "54 : push esp / rsp"},
+    { 1, {0x55}, 0, "55 : push ebp / rbp"},
+    { 1, {0x56}, 0, "56 : push esi / rsi"},
+    { 1, {0x57}, 0, "57 : push edi / rdi"},
+    { 1, {0x5D}, 0, "5D : pop ebp / rbp"},
+    { 1, {0x90}, 0, "90 : nop"},
+    { 1, {0xC3}, 0, "C3 : ret   (for small/empty function interception"},
+    { 1, {0xCC}, 0, "CC : int 3  i.e. registering weak functions)"},
+    { 2, {0x33, 0xC0}, 0, "33 C0 : xor eax, eax"},
+    { 2, {0x33, 0xC9}, 0, "33 C9 : xor ecx, ecx"},
+    { 2, {0x33, 0xD2}, 0, "33 D2 : xor edx, edx"},
+    { 2, {0x6A, 0x71}, 0, "6A XX : push XX"},
+    { 2, {0x84, 0xC9}, 0, "84 C9 : test cl,cl"},
+    { 2, {0x84, 0xD2}, 0, "84 D2 : test dl,dl"},
+    { 2, {0x84, 0xDB}, 0, "84 DB : test bl,bl"},
+    { 2, {0x89, 0xc8}, 0, "89 C8 : mov eax, ecx"},
+    { 2, {0x89, 0xE5}, 0, "89 E5 : mov ebp, esp"},
+    { 2, {0x8A, 0x01}, 0, "8A 01 : mov al, byte ptr [ecx]"},
+    { 2, {0x8B, 0xC1}, 0, "8B C1 : mov eax, ecx"},
+    { 2, {0x8B, 0xEC}, 0, "8B EC : mov ebp, esp"},
+    { 2, {0x8B, 0xFF}, 0, "8B FF : mov edi, edi"},
+    { 3, {0xc2, 0x71, 0x72}, 0, "C2 XX XX : ret XX (needed for registering weak functions)"},
+    { 5, {0x68, 0x71, 0x72, 0x73, 0x74}, 0, "68 XX XX XX XX : push imm32"},
+    { 5, {0xb8, 0x71, 0x72, 0x73, 0x74}, 0, "b8 XX XX XX XX : mov eax, XX XX XX XX"},
+    { 5, {0xB9, 0x71, 0x72, 0x73, 0x74}, 0, "b9 XX XX XX XX : mov ecx, XX XX XX XX"},
+#if SANITIZER_WINDOWS_x64
+    // sorted list
+    { 2, {0x40, 0x50}, 0, "40 50 : push rax"},
+    { 2, {0x40, 0x51}, 0, "40 51 : push rcx"},
+    { 2, {0x40, 0x52}, 0, "40 52 : push rdx"},
+    { 2, {0x40, 0x53}, 0, "40 53 : push rbx"},
+    { 2, {0x40, 0x54}, 0, "40 54 : push rsp"},
+    { 2, {0x40, 0x55}, 0, "40 55 : push rbp"},
+    { 2, {0x40, 0x56}, 0, "40 56 : push rsi"},
+    { 2, {0x40, 0x57}, 0, "40 57 : push rdi"},
+    { 2, {0x41, 0x54}, 0, "41 54 : push r12"},
+    { 2, {0x41, 0x55}, 0, "41 55 : push r13"},
+    { 2, {0x41, 0x56}, 0, "41 56 : push r14"},
+    { 2, {0x41, 0x57}, 0, "41 57 : push r15"},
+    { 2, {0x66, 0x90}, 0, "66 90 : Two-byte NOP"},
+    { 2, {0x84, 0xc0}, 0, "84 c0 : test al, al"},
+    { 2, {0x8a, 0x01}, 0, "8a 01 : mov al, byte ptr [rcx]"},
+    { 3, {0x0f, 0xb6, 0xc2}, 0, "0f b6 c2 : movzx eax, dl"},
+    { 3, {0x0f, 0xb6, 0xd2}, 0, "0f b6 d2 : movzx edx, dl"},
+    { 3, {0x0f, 0xb7, 0x10}, 0, "0f b7 10 : movzx edx, WORD PTR [rax]"},
+    { 3, {0x41, 0x8b, 0xc0}, 0, "41 8b c0 : mov eax, r8d"},
+    { 3, {0x41, 0x8b, 0xc1}, 0, "41 8b c1 : mov eax, r9d"},
+    { 3, {0x41, 0x8b, 0xc2}, 0, "41 8b c2 : mov eax, r10d"},
+    { 3, {0x41, 0x8b, 0xc3}, 0, "41 8b c3 : mov eax, r11d"},
+    { 3, {0x41, 0x8b, 0xc4}, 0, "41 8b c4 : mov eax, r12d"},
+    { 3, {0x45, 0x33, 0xc0}, 0, "45 33 c0 : xor r8d, r8d"},
+    { 3, {0x45, 0x33, 0xc9}, 0, "45 33 c9 : xor r9d, r9d"},
+    { 3, {0x45, 0x33, 0xdb}, 0, "45 33 db : xor r11d, r11d"},
+    { 3, {0x48, 0x2b, 0xca}, 0, "48 2b ca : sub rcx, rdx"},
+    { 3, {0x48, 0x2b, 0xd1}, 0, "48 2b d1 : sub rdx, rcx"},
+    { 3, {0x48, 0x3b, 0xca}, 0, "48 3b ca : cmp rcx, rdx"},
+    { 3, {0x48, 0x85, 0xc0}, 0, "48 85 c0 : test rax, rax"},
+    { 3, {0x48, 0x85, 0xc9}, 0, "48 85 c9 : test rcx, rcx"},
+    { 3, {0x48, 0x85, 0xd2}, 0, "48 85 d2 : test rdx, rdx"},
+    { 3, {0x48, 0x85, 0xdb}, 0, "48 85 db : test rbx, rbx"},
+    { 3, {0x48, 0x85, 0xe4}, 0, "48 85 e4 : test rsp, rsp"},
+    { 3, {0x48, 0x85, 0xed}, 0, "48 85 ed : test rbp, rbp"},
+    { 3, {0x48, 0x89, 0xe5}, 0, "48 89 e5 : mov rbp, rsp"},
+    { 3, {0x48, 0x8b, 0xc1}, 0, "48 8b c1 : mov rax, rcx"},
+    { 3, {0x48, 0x8b, 0xc4}, 0, "48 8b c4 : mov rax, rsp"},
+    { 3, {0x48, 0x8b, 0xd1}, 0, "48 8b d1 : mov rdx, rcx"},
+    { 3, {0x48, 0xf7, 0xd9}, 0, "48 f7 d9 : neg rcx"},
+    { 3, {0x48, 0xff, 0xc0}, 0, "48 ff c0 : inc rax"},
+    { 3, {0x48, 0xff, 0xc1}, 0, "48 ff c1 : inc rcx"},
+    { 3, {0x48, 0xff, 0xc2}, 0, "48 ff c2 : inc rdx"},
+    { 3, {0x48, 0xff, 0xc3}, 0, "48 ff c3 : inc rbx"},
+    { 3, {0x48, 0xff, 0xc6}, 0, "48 ff c6 : inc rsi"},
+    { 3, {0x48, 0xff, 0xc7}, 0, "48 ff c7 : inc rdi"},
+    { 3, {0x49, 0xff, 0xc0}, 0, "49 ff c0 : inc r8"},
+    { 3, {0x49, 0xff, 0xc1}, 0, "49 ff c1 : inc r9"},
+    { 3, {0x49, 0xff, 0xc2}, 0, "49 ff c2 : inc r10"},
+    { 3, {0x49, 0xff, 0xc3}, 0, "49 ff c3 : inc r11"},
+    { 3, {0x49, 0xff, 0xc4}, 0, "49 ff c4 : inc r12"},
+    { 3, {0x49, 0xff, 0xc5}, 0, "49 ff c5 : inc r13"},
+    { 3, {0x49, 0xff, 0xc6}, 0, "49 ff c6 : inc r14"},
+    { 3, {0x49, 0xff, 0xc7}, 0, "49 ff c7 : inc r15"},
+    { 3, {0x4c, 0x8b, 0xc1}, 0, "4c 8b c1 : mov r8, rcx"},
+    { 3, {0x4c, 0x8b, 0xc9}, 0, "4c 8b c9 : mov r9, rcx"},
+    { 3, {0x4c, 0x8b, 0xd1}, 0, "4c 8b d1 : mov r10, rcx"},
+    { 3, {0x4c, 0x8b, 0xd2}, 0, "4c 8b d2 : mov r10, rdx"},
+    { 3, {0x4c, 0x8b, 0xd9}, 0, "4c 8b d9 : mov r11, rcx"},
+    { 3, {0x4c, 0x8b, 0xdc}, 0, "4c 8b dc : mov r11, rsp"},
+    { 3, {0x4d, 0x0b, 0xc0}, 0, "4d 0b c0 : or r8, r8"},
+    { 3, {0x4d, 0x85, 0xc0}, 0, "4d 85 c0 : test r8, r8"},
+    { 3, {0x4d, 0x85, 0xc9}, 0, "4d 85 c9 : test r9, r9"},
+    { 3, {0x4d, 0x85, 0xd2}, 0, "4d 85 d2 : test r10, r10"},
+    { 3, {0x4d, 0x85, 0xdb}, 0, "4d 85 db : test r11, r11"},
+    { 3, {0x4d, 0x85, 0xe4}, 0, "4d 85 e4 : test r12, r12"},
+    { 3, {0x4d, 0x85, 0xed}, 0, "4d 85 ed : test r13, r13"},
+    { 3, {0x4d, 0x85, 0xf6}, 0, "4d 85 f6 : test r14, r14"},
+    { 3, {0x4d, 0x85, 0xff}, 0, "4d 85 ff : test r15, r15"},
+    { 3, {0xf6, 0xc1, 0x72}, 0, "f6 c1 XX : test cl, XX"},
+    { 4, {0x44, 0x0f, 0xb6, 0x1a}, 0, "44 0f b6 1a : movzx r11d, BYTE PTR [rdx]"},
+    { 4, {0x44, 0x8d, 0x42, 0x73}, 0, "44 8d 42 XX : lea r8d , [rdx + XX]"},
+    { 4, {0x48, 0x83, 0xec, 0x73}, 0, "48 83 ec XX : sub rsp, XX"},
+    { 4, {0x48, 0x89, 0x58, 0x73}, 0, "48 89 58 XX : mov QWORD PTR[rax + XX], rbx"},
+    { 4, {0x49, 0x83, 0xf8, 0x73}, 0, "49 83 f8 XX : cmp r8, XX"},
+    { 4, {0x80, 0x78, 0x72, 0x73}, 0, "80 78 YY XX : cmp BYTE PTR [rax+YY], XX"},
+    { 4, {0x80, 0x79, 0x72, 0x73}, 0, "80 79 YY XX : cmp BYTE ptr [rcx+YY], XX"},
+    { 4, {0x80, 0x7A, 0x72, 0x73}, 0, "80 7A YY XX : cmp BYTE PTR [rdx+YY], XX"},
+    { 4, {0x80, 0x7B, 0x72, 0x73}, 0, "80 7B YY XX : cmp BYTE PTR [rbx+YY], XX"},
+    { 4, {0x80, 0x7D, 0x72, 0x73}, 0, "80 7D YY XX : cmp BYTE PTR [rbp+YY], XX"},
+    { 4, {0x80, 0x7E, 0x72, 0x73}, 0, "80 7E YY XX : cmp BYTE PTR [rsi+YY], XX"},
+    { 4, {0x89, 0x54, 0x24, 0x73}, 0, "89 54 24 XX : mov DWORD PTR[rsp + XX], edx"},
+    { 5, {0x44, 0x89, 0x44, 0x24, 0x74}, 0, "44 89 44 24 XX : mov DWORD PTR [rsp + XX], r8d"},
+    { 5, {0x44, 0x89, 0x4c, 0x24, 0x74}, 0, "44 89 4c 24 XX : mov DWORD PTR [rsp + XX], r9d"},
+    { 5, {0x48, 0x89, 0x4C, 0x24, 0x74}, 0, "48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx"},
+    { 5, {0x48, 0x89, 0x54, 0x24, 0x74}, 0, "48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx"},
+    { 5, {0x48, 0x89, 0x5c, 0x24, 0x74}, 0, "48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx"},
+    { 5, {0x48, 0x89, 0x6c, 0x24, 0x74}, 0, "48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp"},
+    { 5, {0x48, 0x89, 0x74, 0x24, 0x74}, 0, "48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi"},
+    { 5, {0x48, 0x89, 0x7c, 0x24, 0x74}, 0, "48 89 7c 24 XX : mov QWORD PTR [rsp + XX], rdi"},
+    { 5, {0x48, 0x8b, 0x44, 0x24, 0x74}, 0, "48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]"},
+    { 5, {0x48, 0x8d, 0x6c, 0x24, 0x74}, 0, "48 8d 6c 24 XX : lea rbp, [rsp + XX]"},
+    { 5, {0x4c, 0x89, 0x44, 0x24, 0x74}, 0, "4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8"},
+    { 5, {0x4c, 0x89, 0x4c, 0x24, 0x74}, 0, "4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9"},
+    { 5, {0x83, 0x44, 0x72, 0x73, 0x74}, 0, "83 44 72 XX YY : add DWORD PTR [rdx+rsi*2+XX],YY"},
+    { 5, {0x83, 0x64, 0x24, 0x73, 0x74}, 0, "83 64 24 XX YY : and DWORD PTR [rsp+XX], YY"},
+    { 6, {0x48, 0x83, 0x64, 0x24, 0x74, 0x75}, 0, "48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY"},
+    { 6, {0x66, 0x81, 0x78, 0x73, 0x74, 0x75}, 0, "66 81 78 XX YY YY : cmp WORD PTR [rax+XX], YY YY"},
+    { 6, {0x66, 0x81, 0x79, 0x73, 0x74, 0x75}, 0, "66 81 79 XX YY YY : cmp WORD PTR [rcx+XX], YY YY"},
+    { 6, {0x66, 0x81, 0x7a, 0x73, 0x74, 0x75}, 0, "66 81 7a XX YY YY : cmp WORD PTR [rdx+XX], YY YY"},
+    { 6, {0x66, 0x81, 0x7b, 0x73, 0x74, 0x75}, 0, "66 81 7b XX YY YY : cmp WORD PTR [rbx+XX], YY YY"},
+    { 6, {0x66, 0x81, 0x7e, 0x73, 0x74, 0x75}, 0, "66 81 7e XX YY YY : cmp WORD PTR [rsi+XX], YY YY"},
+    { 6, {0x66, 0x81, 0x7f, 0x73, 0x74, 0x75}, 0, "66 81 7f XX YY YY : cmp WORD PTR [rdi+XX], YY YY"},
+    { 6, {0x8A, 0x05, 0x72, 0x73, 0x74, 0x75}, 2, "8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]"},
+    { 6, {0x8B, 0x05, 0x72, 0x73, 0x74, 0x75}, 2, "8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]"},
+    { 6, {0xF2, 0x0f, 0x11, 0x44, 0x24, 0x75}, 0, "f2 0f 11 44 24 XX : movsd QWORD PTR [rsp + XX], xmm0"},
+    { 6, {0xF2, 0x0f, 0x11, 0x4c, 0x24, 0x75}, 0, "f2 0f 11 4c 24 XX : movsd QWORD PTR [rsp + XX], xmm1"},
+    { 6, {0xF2, 0x0f, 0x11, 0x54, 0x24, 0x75}, 0, "f2 0f 11 54 24 XX : movsd QWORD PTR [rsp + XX], xmm2"},
+    { 6, {0xF2, 0x0f, 0x11, 0x5c, 0x24, 0x75}, 0, "f2 0f 11 5c 24 XX : movsd QWORD PTR [rsp + XX], xmm3"},
+    { 6, {0xF2, 0x0f, 0x11, 0x64, 0x24, 0x75}, 0, "f2 0f 11 64 24 XX : movsd QWORD PTR [rsp + XX], xmm4"},
+    { 7, {0x48, 0x81, 0xec, 0x73, 0x74, 0x75, 0x76}, 0, "48 81 EC XX XX XX XX : sub rsp, XXXXXXXX"},
+    { 7, {0x48, 0x89, 0x0d, 0x73, 0x74, 0x75, 0x76}, 3, "48 89 0d XX XX XX XX : mov QWORD PTR [rip + XXXXXXXX], rcx"},
+    { 7, {0x48, 0x89, 0x15, 0x73, 0x74, 0x75, 0x76}, 3, "48 89 15 XX XX XX XX : mov QWORD PTR [rip + XXXXXXXX], rdx"},
+    { 7, {0x48, 0x8b, 0x05, 0x73, 0x74, 0x75, 0x76}, 3, "48 8b 05 XX XX XX XX : mov rax, QWORD PTR [rip + XXXXXXXX]"},
+    { 7, {0x48, 0x8d, 0x05, 0x73, 0x74, 0x75, 0x76}, 3, "48 8d 05 XX XX XX XX : lea rax, QWORD PTR [rip + XXXXXXXX]"},
+    { 7, {0x48, 0xff, 0x25, 0x73, 0x74, 0x75, 0x76}, 3, "48 ff 25 XX XX XX XX : rex.W jmp QWORD PTR [rip + XXXXXXXX]"},
+    { 7, {0x4C, 0x8D, 0x15, 0x73, 0x74, 0x75, 0x76}, 3, "4c 8d 15 XX XX XX XX : lea r10, [rip + XX]"},
+    { 7, {0x81, 0x78, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 78 YY XX XX XX XX : cmp DWORD PTR [rax+YY], XX XX XX XX"},
+    { 7, {0x81, 0x79, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 79 YY XX XX XX XX : cmp dword ptr [rcx+YY], XX XX XX XX"},
+    { 7, {0x81, 0x7A, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 7A YY XX XX XX XX : cmp DWORD PTR [rdx+YY], XX XX XX XX"},
+    { 7, {0x81, 0x7B, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 7B YY XX XX XX XX : cmp DWORD PTR [rbx+YY], XX XX XX XX"},
+    { 7, {0x81, 0x7D, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 7D YY XX XX XX XX : cmp DWORD PTR [rbp+YY], XX XX XX XX"},
+    { 7, {0x81, 0x7E, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "81 7E YY XX XX XX XX : cmp DWORD PTR [rsi+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x78, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 78 XX YY YY YY YY : cmp DWORD PTR [r8+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x79, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 79 XX YY YY YY YY : cmp DWORD PTR [r9+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x7a, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7a XX YY YY YY YY : cmp DWORD PTR [r10+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x7b, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7b XX YY YY YY YY : cmp DWORD PTR [r11+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x7d, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX"},
+    { 8, {0x41, 0x81, 0x7f, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX"},
+    { 8, {0x81, 0x7c, 0x24, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "81 7c 24 YY XX XX XX XX : cmp DWORD PTR [rsp+YY], XX XX XX XX"},
+    { 8, {0xc7, 0x44, 0x24, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "C7 44 24 XX YY YY YY YY : mov dword ptr [rsp + XX], YYYYYYYY"},
+    { 9, {0x41, 0x81, 0x7c, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78}, 0, "41 81 7c ZZ YY XX XX XX XX : cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX"},
+    { 9, {0xA1, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78}, 0, "A1 XX XX XX XX XX XX XX XX : movabs eax, dword ptr ds:[XXXXXXXX]"},
+#else
+    // sorted list
+    { 3, {0x8B, 0x45, 0x72}, 0, "8B 45 XX : mov eax, dword ptr [ebp + XX]"},
+    { 3, {0x8B, 0x5D, 0x72}, 0, "8B 5D XX : mov ebx, dword ptr [ebp + XX]"},
+    { 3, {0x8B, 0x75, 0x72}, 0, "8B 75 XX : mov esi, dword ptr [ebp + XX]"},
+    { 3, {0x8B, 0x7D, 0x72}, 0, "8B 7D XX : mov edi, dword ptr [ebp + XX]"},
+    { 3, {0xFF, 0x75, 0x72}, 0, "FF 75 XX : push dword ptr [ebp + XX]"},
+    { 4, {0x83, 0x7D, 0x72, 0x73}, 0, "83 7D XX YY : cmp dword ptr [ebp + XX], YY"},
+    { 4, {0x8A, 0x44, 0x24, 0x73}, 0, "8A 44 24 XX : mov eal, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x44, 0x24, 0x73}, 0, "8B 44 24 XX : mov eax, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x4C, 0x24, 0x73}, 0, "8B 4C 24 XX : mov ecx, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x54, 0x24, 0x73}, 0, "8B 54 24 XX : mov edx, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x5C, 0x24, 0x73}, 0, "8B 5C 24 XX : mov ebx, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x6C, 0x24, 0x73}, 0, "8B 6C 24 XX : mov ebp, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x74, 0x24, 0x73}, 0, "8B 74 24 XX : mov esi, dword ptr [esp + XX]"},
+    { 4, {0x8B, 0x7C, 0x24, 0x73}, 0, "8B 7C 24 XX : mov edi, dword ptr [esp + XX]"},
+    { 5, {0x0F, 0xB6, 0x44, 0x24, 0x74}, 0, "0F B6 44 24 XX : movzx eax, byte ptr [esp + XX]"},
+    { 5, {0xA1, 0x71, 0x72, 0x73, 0x74}, 0, "A1 XX XX XX XX : mov eax, dword ptr ds:[XXXXXXXX]"},
+    { 6, {0xF7, 0xC1, 0x72, 0x73, 0x74, 0x75}, 0, "F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX"},
+    { 7, {0x83, 0x3D, 0x72, 0x73, 0x74, 0x75, 0x76}, 0, "83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX"},
+#endif
+    // clang-format on
 };
 
 std::string dumpInstruction(unsigned arrayIndex,
diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp
index 19b2b90106824..c3448c22532bb 100644
--- a/compiler-rt/lib/memprof/memprof_allocator.cpp
+++ b/compiler-rt/lib/memprof/memprof_allocator.cpp
@@ -301,7 +301,8 @@ struct Allocator {
 
   ~Allocator() {
     atomic_store_relaxed(&destructing, 1);
-    FinishAndWrite();
+    if (flags()->dump_at_exit)
+      FinishAndWrite();
   }
 
   static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value,
diff --git a/compiler-rt/lib/memprof/memprof_flags.cpp b/compiler-rt/lib/memprof/memprof_flags.cpp
index b107ff8fa0a7c..95fde9d9672d6 100644
--- a/compiler-rt/lib/memprof/memprof_flags.cpp
+++ b/compiler-rt/lib/memprof/memprof_flags.cpp
@@ -89,5 +89,5 @@ void InitializeFlags() {
 } // namespace __memprof
 
 SANITIZER_INTERFACE_WEAK_DEF(const char *, __memprof_default_options, void) {
-  return "";
+  return __memprof_default_options_str;
 }
diff --git a/compiler-rt/lib/memprof/memprof_flags.h b/compiler-rt/lib/memprof/memprof_flags.h
index 2f2b628653dc1..4dd395a6be94a 100644
--- a/compiler-rt/lib/memprof/memprof_flags.h
+++ b/compiler-rt/lib/memprof/memprof_flags.h
@@ -17,13 +17,15 @@
 #include "sanitizer_common/sanitizer_flag_parser.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 
-// MemProf flag values can be defined in four ways:
-// 1) initialized with default values at startup.
-// 2) overriden during compilation of MemProf runtime by providing
-//    compile definition MEMPROF_DEFAULT_OPTIONS.
-// 3) overriden from string returned by user-specified function
-//    __memprof_default_options().
-// 4) overriden from env variable MEMPROF_OPTIONS.
+// Default MemProf flags are defined in memprof_flags.inc and sancov_flags.inc.
+// These values can be overridded in a number of ways, each option overrides the
+// prior one:
+//  1) by setting MEMPROF_DEFAULT_OPTIONS during the compilation of the MemProf
+//     runtime
+//  2) by setting the LLVM flag -memprof-runtime-default-options during the
+//     compilation of your binary
+//  3) by overriding the user-specified function __memprof_default_options()
+//  4) by setting the environment variable MEMPROF_OPTIONS during runtime
 
 namespace __memprof {
 
diff --git a/compiler-rt/lib/memprof/memprof_flags.inc b/compiler-rt/lib/memprof/memprof_flags.inc
index 7c5dc091f7935..1d8e77752caa5 100644
--- a/compiler-rt/lib/memprof/memprof_flags.inc
+++ b/compiler-rt/lib/memprof/memprof_flags.inc
@@ -38,4 +38,7 @@ MEMPROF_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true,
 MEMPROF_FLAG(bool, print_text, false,
   "If set, prints the heap profile in text format. Else use the raw binary serialization format.")
 MEMPROF_FLAG(bool, print_terse, false,
-             "If set, prints memory profile in a terse format. Only applicable if print_text = true.")
\ No newline at end of file
+             "If set, prints memory profile in a terse format. Only applicable "
+             "if print_text = true.")
+MEMPROF_FLAG(bool, dump_at_exit, true,
+             "If set, dump profiles when the program terminates.")
diff --git a/compiler-rt/lib/memprof/memprof_interceptors.cpp b/compiler-rt/lib/memprof/memprof_interceptors.cpp
index 53ee4e953419b..f4d7fd46e6198 100644
--- a/compiler-rt/lib/memprof/memprof_interceptors.cpp
+++ b/compiler-rt/lib/memprof/memprof_interceptors.cpp
@@ -49,7 +49,7 @@ int OnExit() {
 // ---------------------- Wrappers ---------------- {{{1
 using namespace __memprof;
 
-DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
 
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                               \
@@ -185,12 +185,12 @@ INTERCEPTOR(char *, strcat, char *to, const char *from) {
   return REAL(strcat)(to, from);
 }
 
-INTERCEPTOR(char *, strncat, char *to, const char *from, uptr size) {
+INTERCEPTOR(char *, strncat, char *to, const char *from, usize size) {
   void *ctx;
   MEMPROF_INTERCEPTOR_ENTER(ctx, strncat);
   ENSURE_MEMPROF_INITED();
   uptr from_length = MaybeRealStrnlen(from, size);
-  uptr copy_length = Min(size, from_length + 1);
+  uptr copy_length = Min<uptr>(size, from_length + 1);
   MEMPROF_READ_RANGE(from, copy_length);
   uptr to_length = internal_strlen(to);
   MEMPROF_READ_STRING(to, to_length);
@@ -239,11 +239,11 @@ INTERCEPTOR(char *, __strdup, const char *s) {
   return reinterpret_cast<char *>(new_mem);
 }
 
-INTERCEPTOR(char *, strncpy, char *to, const char *from, uptr size) {
+INTERCEPTOR(char *, strncpy, char *to, const char *from, usize size) {
   void *ctx;
   MEMPROF_INTERCEPTOR_ENTER(ctx, strncpy);
   ENSURE_MEMPROF_INITED();
-  uptr from_size = Min(size, MaybeRealStrnlen(from, size) + 1);
+  uptr from_size = Min<uptr>(size, MaybeRealStrnlen(from, size) + 1);
   MEMPROF_READ_RANGE(from, from_size);
   MEMPROF_WRITE_RANGE(to, size);
   return REAL(strncpy)(to, from, size);
diff --git a/compiler-rt/lib/memprof/memprof_interceptors.h b/compiler-rt/lib/memprof/memprof_interceptors.h
index 20edef42a5150..53d685706b849 100644
--- a/compiler-rt/lib/memprof/memprof_interceptors.h
+++ b/compiler-rt/lib/memprof/memprof_interceptors.h
@@ -33,11 +33,11 @@ void InitializePlatformInterceptors();
 
 } // namespace __memprof
 
-DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
+DECLARE_REAL(int, memcmp, const void *a1, const void *a2, SIZE_T size)
 DECLARE_REAL(char *, strchr, const char *str, int c)
 DECLARE_REAL(SIZE_T, strlen, const char *s)
-DECLARE_REAL(char *, strncpy, char *to, const char *from, uptr size)
-DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
+DECLARE_REAL(char *, strncpy, char *to, const char *from, SIZE_T size)
+DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
 DECLARE_REAL(char *, strstr, const char *s1, const char *s2)
 
 #define MEMPROF_INTERCEPT_FUNC(name)                                           \
diff --git a/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h b/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h
index 0b87a6f3522af..99b6ccae126e1 100644
--- a/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h
+++ b/compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h
@@ -18,8 +18,8 @@
 #include "memprof_internal.h"
 #include "memprof_mapping.h"
 
-DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size)
-DECLARE_REAL(void *, memset, void *block, int c, uptr size)
+DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size)
+DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size)
 
 namespace __memprof {
 
diff --git a/compiler-rt/lib/memprof/memprof_interface_internal.h b/compiler-rt/lib/memprof/memprof_interface_internal.h
index 318bc41044056..7d3a937814a34 100644
--- a/compiler-rt/lib/memprof/memprof_interface_internal.h
+++ b/compiler-rt/lib/memprof/memprof_interface_internal.h
@@ -40,6 +40,9 @@ void __memprof_record_access_range(void const volatile *addr, uptr size);
 
 SANITIZER_INTERFACE_ATTRIBUTE void __memprof_print_accumulated_stats();
 
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE extern char
+    __memprof_default_options_str[1];
+
 SANITIZER_INTERFACE_ATTRIBUTE
 const char *__memprof_default_options();
 
diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp
index 2cc6c2df5a6fe..ef8884a7e56f4 100644
--- a/compiler-rt/lib/memprof/memprof_rtl.cpp
+++ b/compiler-rt/lib/memprof/memprof_rtl.cpp
@@ -27,6 +27,8 @@
 
 #include <time.h>
 
+SANITIZER_WEAK_ATTRIBUTE char __memprof_default_options_str[1];
+
 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
 
 // Allow the user to specify a profile output file via the binary.
diff --git a/compiler-rt/lib/memprof/weak_symbols.txt b/compiler-rt/lib/memprof/weak_symbols.txt
index 271813612ab60..bfece89e2e157 100644
--- a/compiler-rt/lib/memprof/weak_symbols.txt
+++ b/compiler-rt/lib/memprof/weak_symbols.txt
@@ -1 +1 @@
-___memprof_default_options __memprof_profile_filename
+___memprof_default_options_str ___memprof_default_options __memprof_profile_filename
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index f05c20618780b..f0b1dc905678b 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -59,8 +59,8 @@ using __sanitizer::atomic_uintptr_t;
 
 DECLARE_REAL(SIZE_T, strlen, const char *s)
 DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
-DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
-DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)
+DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)
 
 // True if this is a nested interceptor.
 static THREADLOCAL int in_interceptor_scope;
@@ -1358,6 +1358,79 @@ INTERCEPTOR(int, forkpty, int *aparent, char *name, const void *termp,
 #define MSAN_MAYBE_INTERCEPT_FORKPTY
 #endif
 
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+INTERCEPTOR(int, __b64_ntop, unsigned char const *src, SIZE_T srclength,
+            char *target, SIZE_T targsize) {
+  ENSURE_MSAN_INITED();
+  CHECK_UNPOISONED(src, srclength);
+  InterceptorScope interceptor_scope;
+  int res = REAL(__b64_ntop)(src, srclength, target, targsize);
+  if (res >= 0)
+    __msan_unpoison(target, res + 1);
+  return res;
+}
+INTERCEPTOR(int, __b64_pton, char const *src, char *target, SIZE_T targsize) {
+  ENSURE_MSAN_INITED();
+  CHECK_UNPOISONED(src, internal_strlen(src) + 1);
+  InterceptorScope interceptor_scope;
+  int res = REAL(__b64_pton)(src, target, targsize);
+  if (res >= 0)
+    __msan_unpoison(target, res);
+  return res;
+}
+#  define MSAN_MAYBE_INTERCEPT___B64_TO \
+    MSAN_INTERCEPT_FUNC(__b64_ntop);    \
+    COMMON_INTERCEPT_FUNCTION(__b64_pton);
+#else
+#  define MSAN_MAYBE_INTERCEPT___B64_TO
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#  if __GLIBC_PREREQ(2, 34)
+// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf
+#    define DN_COMP_INTERCEPTOR_NAME dn_comp
+#    define DN_EXPAND_INTERCEPTOR_NAME dn_expand
+#  else
+#    define DN_COMP_INTERCEPTOR_NAME __dn_comp
+#    define DN_EXPAND_INTERCEPTOR_NAME __dn_expand
+#  endif
+INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn,
+            unsigned char *comp_dn, int length, unsigned char **dnptrs,
+            unsigned char **lastdnptr) {
+  ENSURE_MSAN_INITED();
+  InterceptorScope interceptor_scope;
+  int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs,
+                                           lastdnptr);
+  if (res >= 0) {
+    __msan_unpoison(comp_dn, res);
+    if (dnptrs && lastdnptr) {
+      unsigned char **p = dnptrs;
+      for (; p != lastdnptr && *p; ++p);
+      if (p != lastdnptr)
+        ++p;
+      __msan_unpoison(dnptrs, (p - dnptrs) * sizeof(*p));
+    }
+  }
+  return res;
+}
+INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base,
+            unsigned char const *end, unsigned char const *src, char *dest,
+            int space) {
+  ENSURE_MSAN_INITED();
+  // TODO: add read check if __dn_comp intercept added
+  InterceptorScope interceptor_scope;
+  int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space);
+  if (res >= 0)
+    __msan_unpoison(dest, internal_strlen(dest) + 1);
+  return res;
+}
+#  define MSAN_MAYBE_INTERCEPT_DN_COMP_EXPAND      \
+    MSAN_INTERCEPT_FUNC(DN_COMP_INTERCEPTOR_NAME); \
+    MSAN_INTERCEPT_FUNC(DN_EXPAND_INTERCEPTOR_NAME);
+#else
+#  define MSAN_MAYBE_INTERCEPT_DN_COMP_EXPAND
+#endif
+
 struct MSanInterceptorContext {
   bool in_interceptor_scope;
 };
@@ -1916,6 +1989,9 @@ void InitializeInterceptors() {
   MSAN_MAYBE_INTERCEPT_OPENPTY;
   MSAN_MAYBE_INTERCEPT_FORKPTY;
 
+  MSAN_MAYBE_INTERCEPT___B64_TO;
+  MSAN_MAYBE_INTERCEPT_DN_COMP_EXPAND;
+
   inited = 1;
 }
 } // namespace __msan
diff --git a/compiler-rt/lib/msan/msan_poisoning.cpp b/compiler-rt/lib/msan/msan_poisoning.cpp
index 1889e980bfc0e..ac403681af406 100644
--- a/compiler-rt/lib/msan/msan_poisoning.cpp
+++ b/compiler-rt/lib/msan/msan_poisoning.cpp
@@ -17,9 +17,9 @@
 #include "msan_thread.h"
 #include "sanitizer_common/sanitizer_common.h"
 
-DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
-DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
-DECLARE_REAL(void *, memmove, void *dest, const void *src, uptr n)
+DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)
+DECLARE_REAL(void *, memmove, void *dest, const void *src, SIZE_T n)
 
 namespace __msan {
 
diff --git a/compiler-rt/lib/nsan/nsan_allocator.cpp b/compiler-rt/lib/nsan/nsan_allocator.cpp
index 19004ad7dc8db..128d37f52ba3c 100644
--- a/compiler-rt/lib/nsan/nsan_allocator.cpp
+++ b/compiler-rt/lib/nsan/nsan_allocator.cpp
@@ -25,8 +25,8 @@
 
 using namespace __nsan;
 
-DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
-DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)
+DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)
 
 namespace {
 struct Metadata {
diff --git a/compiler-rt/lib/nsan/nsan_interceptors.cpp b/compiler-rt/lib/nsan/nsan_interceptors.cpp
index 5f25eba856070..2549f492b736a 100644
--- a/compiler-rt/lib/nsan/nsan_interceptors.cpp
+++ b/compiler-rt/lib/nsan/nsan_interceptors.cpp
@@ -28,7 +28,7 @@ using namespace __sanitizer;
 
 template <typename T> T min(T a, T b) { return a < b ? a : b; }
 
-INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
+INTERCEPTOR(void *, memset, void *dst, int v, usize size) {
   // NOTE: This guard is needed because nsan's initialization code might call
   // memset.
   if (!nsan_initialized && REAL(memset) == nullptr)
@@ -39,13 +39,13 @@ INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
   return res;
 }
 
-INTERCEPTOR(wchar_t *, wmemset, wchar_t *dst, wchar_t v, uptr size) {
+INTERCEPTOR(wchar_t *, wmemset, wchar_t *dst, wchar_t v, usize size) {
   wchar_t *res = REAL(wmemset)(dst, v, size);
   __nsan_set_value_unknown((u8 *)dst, sizeof(wchar_t) * size);
   return res;
 }
 
-INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
+INTERCEPTOR(void *, memmove, void *dst, const void *src, usize size) {
   // NOTE: This guard is needed because nsan's initialization code might call
   // memmove.
   if (!nsan_initialized && REAL(memmove) == nullptr)
@@ -57,13 +57,13 @@ INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
   return res;
 }
 
-INTERCEPTOR(wchar_t *, wmemmove, wchar_t *dst, const wchar_t *src, uptr size) {
+INTERCEPTOR(wchar_t *, wmemmove, wchar_t *dst, const wchar_t *src, usize size) {
   wchar_t *res = REAL(wmemmove)(dst, src, size);
   __nsan_copy_values((u8 *)dst, (const u8 *)src, sizeof(wchar_t) * size);
   return res;
 }
 
-INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, usize size) {
   // NOTE: This guard is needed because nsan's initialization code might call
   // memcpy.
   if (!nsan_initialized && REAL(memcpy) == nullptr) {
@@ -78,7 +78,7 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
   return res;
 }
 
-INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dst, const wchar_t *src, uptr size) {
+INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dst, const wchar_t *src, usize size) {
   wchar_t *res = REAL(wmemcpy)(dst, src, size);
   __nsan_copy_values((u8 *)dst, (const u8 *)src, sizeof(wchar_t) * size);
   return res;
@@ -136,7 +136,7 @@ INTERCEPTOR(wchar_t *, wcsdup, const wchar_t *S) {
   return res;
 }
 
-INTERCEPTOR(char *, strndup, const char *S, uptr size) {
+INTERCEPTOR(char *, strndup, const char *S, usize size) {
   char *res = REAL(strndup)(S, size);
   if (res) {
     nsanCopyZeroTerminated(res, S, min(internal_strlen(S), size));
@@ -156,7 +156,7 @@ INTERCEPTOR(wchar_t *, wcscpy, wchar_t *dst, const wchar_t *src) {
   return res;
 }
 
-INTERCEPTOR(char *, strncpy, char *dst, const char *src, uptr size) {
+INTERCEPTOR(char *, strncpy, char *dst, const char *src, usize size) {
   char *res = REAL(strncpy)(dst, src, size);
   nsanCopyZeroTerminated(dst, src, min(size, internal_strlen(src)));
   return res;
@@ -176,7 +176,7 @@ INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) {
   return res;
 }
 
-INTERCEPTOR(char *, strncat, char *dst, const char *src, uptr size) {
+INTERCEPTOR(char *, strncat, char *dst, const char *src, usize size) {
   const auto DstLen = internal_strlen(dst);
   char *res = REAL(strncat)(dst, src, size);
   nsanCopyZeroTerminated(dst + DstLen, src, min(size, internal_strlen(src)));
@@ -195,11 +195,10 @@ INTERCEPTOR(wchar_t *, wcpcpy, wchar_t *dst, const wchar_t *src) {
   return res;
 }
 
-INTERCEPTOR(uptr, strxfrm, char *dst, const char *src, uptr size) {
+INTERCEPTOR(usize, strxfrm, char *dst, const char *src, usize size) {
   // This is overly conservative, but this function should very rarely be used.
   __nsan_set_value_unknown(reinterpret_cast<u8 *>(dst), internal_strlen(dst));
-  const uptr res = REAL(strxfrm)(dst, src, size);
-  return res;
+  return REAL(strxfrm)(dst, src, size);
 }
 
 extern "C" int pthread_attr_init(void *attr);
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 36f4349a240e3..c95700ab53876 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -3,10 +3,11 @@
 # ORC runtime library common implementation files.
 set(ORC_COMMON_SOURCES
   debug.cpp
+  dlfcn_wrapper.cpp
   extensible_rtti.cpp
   log_error_to_stderr.cpp
   run_program_wrapper.cpp
-  dlfcn_wrapper.cpp
+  resolve.cpp
   )
 
 # Common implementation headers will go here.
@@ -51,6 +52,7 @@ if (APPLE)
   set(ORC_ASM_SOURCES
     macho_tlv.x86-64.S
     macho_tlv.arm64.S
+    sysv_reenter.arm64.S
     )
 
   set(ORC_IMPL_HEADERS
@@ -116,6 +118,7 @@ else() # not Apple
       elfnix_tls.x86-64.S
       elfnix_tls.aarch64.S
       elfnix_tls.ppc64.S
+      sysv_reenter.arm64.S
       )
   endif()
 
diff --git a/compiler-rt/lib/orc/resolve.cpp b/compiler-rt/lib/orc/resolve.cpp
new file mode 100644
index 0000000000000..3d9b37d14d2a4
--- /dev/null
+++ b/compiler-rt/lib/orc/resolve.cpp
@@ -0,0 +1,49 @@
+//===- resolve.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a generic "resolver" function compatible with the
+// __orc_rt_reenter function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "executor_symbol_def.h"
+#include "jit_dispatch.h"
+#include "wrapper_function_utils.h"
+
+#include <stdio.h>
+
+#define DEBUG_TYPE "resolve"
+
+using namespace orc_rt;
+
+// Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_resolve_tag)
+
+// FIXME: Make this configurable via an alias.
+static void __orc_rt_resolve_fail(void *Caller, const char *ErrMsg) {
+  fprintf(stderr, "error resolving implementation for stub %p: %s\n", Caller,
+          ErrMsg);
+  abort();
+}
+
+extern "C" ORC_RT_HIDDEN void *__orc_rt_resolve(void *Caller) {
+  Expected<ExecutorSymbolDef> Result((ExecutorSymbolDef()));
+  if (auto Err = WrapperFunction<SPSExpected<SPSExecutorSymbolDef>(
+          SPSExecutorAddr)>::call(JITDispatch(&__orc_rt_resolve_tag), Result,
+                                  ExecutorAddr::fromPtr(Caller))) {
+    __orc_rt_resolve_fail(Caller, toString(std::move(Err)).c_str());
+    return nullptr; // Unreachable.
+  }
+
+  if (!Result) {
+    __orc_rt_resolve_fail(Caller, toString(Result.takeError()).c_str());
+    return nullptr; // Unreachable.
+  }
+
+  return Result->getAddress().toPtr<void *>();
+}
diff --git a/compiler-rt/lib/orc/sysv_reenter.arm64.S b/compiler-rt/lib/orc/sysv_reenter.arm64.S
new file mode 100644
index 0000000000000..bd74a33bc6344
--- /dev/null
+++ b/compiler-rt/lib/orc/sysv_reenter.arm64.S
@@ -0,0 +1,102 @@
+//===-- sysv_reenter.arm64.s ------------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is arm64-only
+#if defined(__arm64__) || defined(__aarch64__)
+
+        .text
+
+        // Saves GPRs, calls __orc_rt_sysv_resolve
+        .globl __orc_rt_sysv_reenter
+__orc_rt_sysv_reenter:
+        // Save register state, set up new stack frome.
+        stp  x27, x28, [sp, #-16]!
+        stp  x25, x26, [sp, #-16]!
+        stp  x23, x24, [sp, #-16]!
+        stp  x21, x22, [sp, #-16]!
+        stp  x19, x20, [sp, #-16]!
+        stp  x14, x15, [sp, #-16]!
+        stp  x12, x13, [sp, #-16]!
+        stp  x10, x11, [sp, #-16]!
+        stp   x8,  x9, [sp, #-16]!
+        stp   x6,  x7, [sp, #-16]!
+        stp   x4,  x5, [sp, #-16]!
+        stp   x2,  x3, [sp, #-16]!
+        stp   x0,  x1, [sp, #-16]!
+        stp  q30, q31, [sp, #-32]!
+        stp  q28, q29, [sp, #-32]!
+        stp  q26, q27, [sp, #-32]!
+        stp  q24, q25, [sp, #-32]!
+        stp  q22, q23, [sp, #-32]!
+        stp  q20, q21, [sp, #-32]!
+        stp  q18, q19, [sp, #-32]!
+        stp  q16, q17, [sp, #-32]!
+        stp  q14, q15, [sp, #-32]!
+        stp  q12, q13, [sp, #-32]!
+        stp  q10, q11, [sp, #-32]!
+        stp   q8,  q9, [sp, #-32]!
+        stp   q6,  q7, [sp, #-32]!
+        stp   q4,  q5, [sp, #-32]!
+        stp   q2,  q3, [sp, #-32]!
+        stp   q0,  q1, [sp, #-32]!
+
+        // Look up the return address and subtract 8 from it (on the
+        // assumption that it's a standard arm64 reentry trampoline) to get
+        // back the trampoline's address.
+        sub   x0, x30, #8
+
+        // Call __orc_rt_sysv_resolve to look up the implementation
+        // corresponding to the calling stub, then store this in x17 (which
+        // we'll return to below.
+#if !defined(__APPLE__)
+        bl    __orc_rt_resolve
+#else
+        bl    ___orc_rt_resolve
+#endif
+        mov   x17, x0
+
+        // Restore the register state.
+        ldp   q0,  q1, [sp], #32
+        ldp   q2,  q3, [sp], #32
+        ldp   q4,  q5, [sp], #32
+        ldp   q6,  q7, [sp], #32
+        ldp   q8,  q9, [sp], #32
+        ldp  q10, q11, [sp], #32
+        ldp  q12, q13, [sp], #32
+        ldp  q14, q15, [sp], #32
+        ldp  q16, q17, [sp], #32
+        ldp  q18, q19, [sp], #32
+        ldp  q20, q21, [sp], #32
+        ldp  q22, q23, [sp], #32
+        ldp  q24, q25, [sp], #32
+        ldp  q26, q27, [sp], #32
+        ldp  q28, q29, [sp], #32
+        ldp  q30, q31, [sp], #32
+        ldp   x0,  x1, [sp], #16
+        ldp   x2,  x3, [sp], #16
+        ldp   x4,  x5, [sp], #16
+        ldp   x6,  x7, [sp], #16
+        ldp   x8,  x9, [sp], #16
+        ldp  x10, x11, [sp], #16
+        ldp  x12, x13, [sp], #16
+        ldp  x14, x15, [sp], #16
+        ldp  x19, x20, [sp], #16
+        ldp  x21, x22, [sp], #16
+        ldp  x23, x24, [sp], #16
+        ldp  x25, x26, [sp], #16
+        ldp  x27, x28, [sp], #16
+        ldp  x29, x30, [sp], #16
+
+        // Return to the function implementation (rather than the stub).
+        ret  x17
+
+#endif // defined(__arm64__) || defined(__aarch64__)
diff --git a/compiler-rt/lib/orc/tests/unit/CMakeLists.txt b/compiler-rt/lib/orc/tests/unit/CMakeLists.txt
index aec689a407be6..16cd16c45683d 100644
--- a/compiler-rt/lib/orc/tests/unit/CMakeLists.txt
+++ b/compiler-rt/lib/orc/tests/unit/CMakeLists.txt
@@ -2,6 +2,7 @@ set(UNITTEST_SOURCES
   adt_test.cpp
   bitmask_enum_test.cpp
   c_api_test.cpp
+  common.cpp
   endian_test.cpp
   error_test.cpp
   executor_address_test.cpp
diff --git a/compiler-rt/lib/orc/tests/unit/common.cpp b/compiler-rt/lib/orc/tests/unit/common.cpp
new file mode 100644
index 0000000000000..9479cc8094308
--- /dev/null
+++ b/compiler-rt/lib/orc/tests/unit/common.cpp
@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+/// Defined so that tests can use code that logs errors.
+extern "C" void __orc_rt_log_error(const char *ErrMsg) {
+  fprintf(stderr, "orc runtime error: %s\n", ErrMsg);
+}
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 7cf1679811eb0..92721c4fd55ea 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -137,27 +137,23 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     return 1;
   }
 
-  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
-  char *SrcCountersStart, *DstCounter;
-  const char *SrcCountersEnd, *SrcCounter;
-  const char *SrcBitmapStart;
-  const char *SrcNameStart;
-  const char *SrcValueProfDataStart, *SrcValueProfData;
   uintptr_t CountersDelta = Header->CountersDelta;
   uintptr_t BitmapDelta = Header->BitmapDelta;
 
-  SrcDataStart =
+  __llvm_profile_data *SrcDataStart =
       (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
                               Header->BinaryIdsSize);
-  SrcDataEnd = SrcDataStart + Header->NumData;
-  SrcCountersStart = (char *)SrcDataEnd;
-  SrcCountersEnd = SrcCountersStart +
-                   Header->NumCounters * __llvm_profile_counter_entry_size();
-  SrcBitmapStart = SrcCountersEnd + __llvm_profile_get_num_padding_bytes(
-                                        SrcCountersEnd - SrcCountersStart);
-  SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
-  SrcValueProfDataStart =
+  __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData;
+  uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd;
+  uintptr_t SrcCountersEnd =
+      SrcCountersStart +
+      Header->NumCounters * __llvm_profile_counter_entry_size();
+  uintptr_t SrcBitmapStart =
+      SrcCountersEnd +
+      __llvm_profile_get_num_padding_bytes(SrcCountersEnd - SrcCountersStart);
+  uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
+  uintptr_t SrcValueProfDataStart =
       SrcNameStart + getDistanceFromCounterToValueProf(Header);
   if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
     return 1;
@@ -165,13 +161,13 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
   // Merge counters by iterating the entire counter section when data section is
   // empty due to correlation.
   if (Header->NumData == 0) {
-    for (SrcCounter = SrcCountersStart,
-        DstCounter = __llvm_profile_begin_counters();
+    for (uintptr_t SrcCounter = SrcCountersStart,
+                   DstCounter = (uintptr_t)__llvm_profile_begin_counters();
          SrcCounter < SrcCountersEnd;) {
       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
-        *DstCounter &= *SrcCounter;
+        *(char *)DstCounter &= *(const char *)SrcCounter;
       } else {
-        *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
+        *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter;
       }
       SrcCounter += __llvm_profile_counter_entry_size();
       DstCounter += __llvm_profile_counter_entry_size();
@@ -179,6 +175,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     return 0;
   }
 
+  __llvm_profile_data *SrcData, *DstData;
+  uintptr_t SrcValueProfData;
   for (SrcData = SrcDataStart,
       DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
       SrcValueProfData = SrcValueProfDataStart;
@@ -187,10 +185,10 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     // address of the data to the start address of the counter. On WIN64,
     // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
     // extend CounterPtr to get the original value.
-    char *DstCounters =
-        (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
-    char *DstBitmap =
-        (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
+    uintptr_t DstCounters =
+        (uintptr_t)DstData + signextIfWin64(DstData->CounterPtr);
+    uintptr_t DstBitmap =
+        (uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr);
     unsigned NVK = 0;
 
     // SrcData is a serialized representation of the memory image. We need to
@@ -200,7 +198,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     // CountersDelta computes the offset into the in-buffer counter section.
     //
     // On WIN64, CountersDelta is truncated as well, so no need for signext.
-    char *SrcCounters =
+    uintptr_t SrcCounters =
         SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
     // CountersDelta needs to be decreased as we advance to the next data
     // record.
@@ -214,13 +212,13 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     for (unsigned I = 0; I < NC; I++) {
       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
         // A value of zero signifies the function is covered.
-        DstCounters[I] &= SrcCounters[I];
+        ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I];
       } else {
-        ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
+        ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I];
       }
     }
 
-    const char *SrcBitmap =
+    uintptr_t SrcBitmap =
         SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
     // BitmapDelta also needs to be decreased as we advance to the next data
     // record.
@@ -232,7 +230,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
         return 1;
       // Merge Src and Dst Bitmap bytes by simply ORing them together.
       for (unsigned I = 0; I < NB; I++)
-        DstBitmap[I] |= SrcBitmap[I];
+        ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I];
     }
 
     /* Now merge value profile data. */
@@ -245,7 +243,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
     if (!NVK)
       continue;
 
-    if (SrcValueProfData >= ProfileData + ProfileSize)
+    if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize)
       return 1;
     VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
     SrcValueProfData =
diff --git a/compiler-rt/lib/rtsan/rtsan.cpp b/compiler-rt/lib/rtsan/rtsan.cpp
index 81cedb3b5114f..73340b34f6d4b 100644
--- a/compiler-rt/lib/rtsan/rtsan.cpp
+++ b/compiler-rt/lib/rtsan/rtsan.cpp
@@ -83,6 +83,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
 
   SanitizerToolName = "RealtimeSanitizer";
   InitializeFlags();
+
+  InitializePlatformEarly();
+
   InitializeInterceptors();
 
   InitializeSuppressions();
diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index 5debf13ab9815..f076c7eae9a22 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -47,6 +47,7 @@ void OSSpinLockLock(volatile OSSpinLock *__lock);
 #include <stdarg.h>
 #include <stdio.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
 
@@ -243,14 +244,29 @@ INTERCEPTOR(FILE *, fopen, const char *path, const char *mode) {
   return REAL(fopen)(path, mode);
 }
 
+INTERCEPTOR(FILE *, freopen, const char *path, const char *mode, FILE *stream) {
+  __rtsan_notify_intercepted_call("freopen");
+  return REAL(freopen)(path, mode, stream);
+}
+
+// Streams
+
 #if SANITIZER_INTERCEPT_FOPEN64
 INTERCEPTOR(FILE *, fopen64, const char *path, const char *mode) {
   __rtsan_notify_intercepted_call("fopen64");
   return REAL(fopen64)(path, mode);
 }
-#define RTSAN_MAYBE_INTERCEPT_FOPEN64 INTERCEPT_FUNCTION(fopen64)
+
+INTERCEPTOR(FILE *, freopen64, const char *path, const char *mode,
+            FILE *stream) {
+  __rtsan_notify_intercepted_call("freopen64");
+  return REAL(freopen64)(path, mode, stream);
+}
+#define RTSAN_MAYBE_INTERCEPT_FOPEN64 INTERCEPT_FUNCTION(fopen64);
+#define RTSAN_MAYBE_INTERCEPT_FREOPEN64 INTERCEPT_FUNCTION(freopen64);
 #else
 #define RTSAN_MAYBE_INTERCEPT_FOPEN64
+#define RTSAN_MAYBE_INTERCEPT_FREOPEN64
 #endif // SANITIZER_INTERCEPT_FOPEN64
 
 INTERCEPTOR(size_t, fread, void *ptr, size_t size, size_t nitems,
@@ -275,7 +291,11 @@ INTERCEPTOR(int, fputs, const char *s, FILE *stream) {
   return REAL(fputs)(s, stream);
 }
 
-// Streams
+INTERCEPTOR(FILE *, fdopen, int fd, const char *mode) {
+  __rtsan_notify_intercepted_call("fdopen");
+  return REAL(fdopen)(fd, mode);
+}
+
 INTERCEPTOR(int, puts, const char *s) {
   __rtsan_notify_intercepted_call("puts");
   return REAL(puts)(s);
@@ -358,6 +378,31 @@ INTERCEPTOR(int, dup2, int oldfd, int newfd) {
   return REAL(dup2)(oldfd, newfd);
 }
 
+INTERCEPTOR(int, chmod, const char *path, mode_t mode) {
+  __rtsan_notify_intercepted_call("chmod");
+  return REAL(chmod)(path, mode);
+}
+
+INTERCEPTOR(int, fchmod, int fd, mode_t mode) {
+  __rtsan_notify_intercepted_call("fchmod");
+  return REAL(fchmod)(fd, mode);
+}
+
+INTERCEPTOR(int, mkdir, const char *path, mode_t mode) {
+  __rtsan_notify_intercepted_call("mkdir");
+  return REAL(mkdir)(path, mode);
+}
+
+INTERCEPTOR(int, rmdir, const char *path) {
+  __rtsan_notify_intercepted_call("rmdir");
+  return REAL(rmdir)(path);
+}
+
+INTERCEPTOR(mode_t, umask, mode_t cmask) {
+  __rtsan_notify_intercepted_call("umask");
+  return REAL(umask)(cmask);
+}
+
 // Concurrency
 #if SANITIZER_APPLE
 #pragma clang diagnostic push
@@ -368,17 +413,32 @@ INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
   return REAL(OSSpinLockLock)(lock);
 }
 #pragma clang diagnostic pop
+#define RTSAN_MAYBE_INTERCEPT_OSSPINLOCKLOCK INTERCEPT_FUNCTION(OSSpinLockLock)
+#else
+#define RTSAN_MAYBE_INTERCEPT_OSSPINLOCKLOCK
+#endif // SANITIZER_APPLE
 
+#if SANITIZER_APPLE
 INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
   __rtsan_notify_intercepted_call("os_unfair_lock_lock");
   return REAL(os_unfair_lock_lock)(lock);
 }
-#elif SANITIZER_LINUX
+#define RTSAN_MAYBE_INTERCEPT_OS_UNFAIR_LOCK_LOCK                              \
+  INTERCEPT_FUNCTION(os_unfair_lock_lock)
+#else
+#define RTSAN_MAYBE_INTERCEPT_OS_UNFAIR_LOCK_LOCK
+#endif // SANITIZER_APPLE
+
+#if SANITIZER_LINUX
 INTERCEPTOR(int, pthread_spin_lock, pthread_spinlock_t *spinlock) {
   __rtsan_notify_intercepted_call("pthread_spin_lock");
   return REAL(pthread_spin_lock)(spinlock);
 }
-#endif
+#define RTSAN_MAYBE_INTERCEPT_PTHREAD_SPIN_LOCK                                \
+  INTERCEPT_FUNCTION(pthread_spin_lock)
+#else
+#define RTSAN_MAYBE_INTERCEPT_PTHREAD_SPIN_LOCK
+#endif // SANITIZER_LINUX
 
 INTERCEPTOR(int, pthread_create, pthread_t *thread, const pthread_attr_t *attr,
             void *(*start_routine)(void *), void *arg) {
@@ -539,6 +599,9 @@ INTERCEPTOR(void *, memalign, size_t alignment, size_t size) {
   __rtsan_notify_intercepted_call("memalign");
   return REAL(memalign)(alignment, size);
 }
+#define RTSAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define RTSAN_MAYBE_INTERCEPT_MEMALIGN
 #endif
 
 #if SANITIZER_INTERCEPT_PVALLOC
@@ -546,6 +609,9 @@ INTERCEPTOR(void *, pvalloc, size_t size) {
   __rtsan_notify_intercepted_call("pvalloc");
   return REAL(pvalloc)(size);
 }
+#define RTSAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
+#else
+#define RTSAN_MAYBE_INTERCEPT_PVALLOC
 #endif
 
 INTERCEPTOR(void *, mmap, void *addr, size_t length, int prot, int flags,
@@ -660,6 +726,17 @@ INTERCEPTOR(int, shutdown, int socket, int how) {
   return REAL(shutdown)(socket, how);
 }
 
+#if SANITIZER_INTERCEPT_ACCEPT4
+INTERCEPTOR(int, accept4, int socket, struct sockaddr *address,
+            socklen_t *address_len, int flags) {
+  __rtsan_notify_intercepted_call("accept4");
+  return REAL(accept4)(socket, address, address_len, flags);
+}
+#define RTSAN_MAYBE_INTERCEPT_ACCEPT4 INTERCEPT_FUNCTION(accept4)
+#else
+#define RTSAN_MAYBE_INTERCEPT_ACCEPT4
+#endif
+
 // I/O Multiplexing
 
 INTERCEPTOR(int, poll, struct pollfd *fds, nfds_t nfds, int timeout) {
@@ -768,6 +845,59 @@ INTERCEPTOR(int, mkfifo, const char *pathname, mode_t mode) {
   return REAL(mkfifo)(pathname, mode);
 }
 
+INTERCEPTOR(pid_t, fork, void) {
+  __rtsan_notify_intercepted_call("fork");
+  return REAL(fork)();
+}
+
+INTERCEPTOR(int, execve, const char *filename, char *const argv[],
+            char *const envp[]) {
+  __rtsan_notify_intercepted_call("execve");
+  return REAL(execve)(filename, argv, envp);
+}
+
+// TODO: the `wait` family of functions is an oddity. In testing, if you
+// intercept them, Darwin seemingly ignores them, and linux never returns from
+// the test. Revisit this in the future, but hopefully intercepting fork/exec is
+// enough to dissuade usage of wait by proxy.
+
+#if SANITIZER_APPLE
+#define INT_TYPE_SYSCALL int
+#else
+#define INT_TYPE_SYSCALL long
+#endif
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+INTERCEPTOR(INT_TYPE_SYSCALL, syscall, INT_TYPE_SYSCALL number, ...) {
+  __rtsan_notify_intercepted_call("syscall");
+
+  va_list args;
+  va_start(args, number);
+
+  // the goal is to pick something large enough to hold all syscall args
+  // see fcntl for more discussion and why we always pull all 6 args
+  using arg_type = unsigned long;
+  arg_type arg1 = va_arg(args, arg_type);
+  arg_type arg2 = va_arg(args, arg_type);
+  arg_type arg3 = va_arg(args, arg_type);
+  arg_type arg4 = va_arg(args, arg_type);
+  arg_type arg5 = va_arg(args, arg_type);
+  arg_type arg6 = va_arg(args, arg_type);
+
+  // these are various examples of things that COULD be passed
+  static_assert(sizeof(arg_type) >= sizeof(off_t));
+  static_assert(sizeof(arg_type) >= sizeof(struct flock *));
+  static_assert(sizeof(arg_type) >= sizeof(const char *));
+  static_assert(sizeof(arg_type) >= sizeof(int));
+  static_assert(sizeof(arg_type) >= sizeof(unsigned long));
+
+  va_end(args);
+
+  return REAL(syscall)(number, arg1, arg2, arg3, arg4, arg5, arg6);
+}
+#pragma clang diagnostic pop
+
 // Preinit
 void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(calloc);
@@ -783,12 +913,8 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(munmap);
   INTERCEPT_FUNCTION(shm_open);
   INTERCEPT_FUNCTION(shm_unlink);
-#if SANITIZER_INTERCEPT_MEMALIGN
-  INTERCEPT_FUNCTION(memalign);
-#endif
-#if SANITIZER_INTERCEPT_PVALLOC
-  INTERCEPT_FUNCTION(pvalloc);
-#endif
+  RTSAN_MAYBE_INTERCEPT_MEMALIGN;
+  RTSAN_MAYBE_INTERCEPT_PVALLOC;
 
   INTERCEPT_FUNCTION(open);
   RTSAN_MAYBE_INTERCEPT_OPEN64;
@@ -797,6 +923,7 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(close);
   INTERCEPT_FUNCTION(fopen);
   RTSAN_MAYBE_INTERCEPT_FOPEN64;
+  RTSAN_MAYBE_INTERCEPT_FREOPEN64;
   INTERCEPT_FUNCTION(fread);
   INTERCEPT_FUNCTION(read);
   INTERCEPT_FUNCTION(write);
@@ -814,18 +941,22 @@ void __rtsan::InitializeInterceptors() {
   RTSAN_MAYBE_INTERCEPT_CREAT64;
   INTERCEPT_FUNCTION(puts);
   INTERCEPT_FUNCTION(fputs);
+  INTERCEPT_FUNCTION(fdopen);
+  INTERCEPT_FUNCTION(freopen);
   INTERCEPT_FUNCTION(lseek);
   RTSAN_MAYBE_INTERCEPT_LSEEK64;
   INTERCEPT_FUNCTION(dup);
   INTERCEPT_FUNCTION(dup2);
+  INTERCEPT_FUNCTION(chmod);
+  INTERCEPT_FUNCTION(fchmod);
+  INTERCEPT_FUNCTION(mkdir);
+  INTERCEPT_FUNCTION(rmdir);
+  INTERCEPT_FUNCTION(umask);
   INTERCEPT_FUNCTION(ioctl);
 
-#if SANITIZER_APPLE
-  INTERCEPT_FUNCTION(OSSpinLockLock);
-  INTERCEPT_FUNCTION(os_unfair_lock_lock);
-#elif SANITIZER_LINUX
-  INTERCEPT_FUNCTION(pthread_spin_lock);
-#endif
+  RTSAN_MAYBE_INTERCEPT_OSSPINLOCKLOCK;
+  RTSAN_MAYBE_INTERCEPT_OS_UNFAIR_LOCK_LOCK;
+  RTSAN_MAYBE_INTERCEPT_PTHREAD_SPIN_LOCK;
 
   INTERCEPT_FUNCTION(pthread_create);
   INTERCEPT_FUNCTION(pthread_mutex_lock);
@@ -858,6 +989,7 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(sendto);
   INTERCEPT_FUNCTION(shutdown);
   INTERCEPT_FUNCTION(socket);
+  RTSAN_MAYBE_INTERCEPT_ACCEPT4;
 
   RTSAN_MAYBE_INTERCEPT_SELECT;
   INTERCEPT_FUNCTION(pselect);
@@ -873,6 +1005,11 @@ void __rtsan::InitializeInterceptors() {
 
   INTERCEPT_FUNCTION(pipe);
   INTERCEPT_FUNCTION(mkfifo);
+
+  INTERCEPT_FUNCTION(fork);
+  INTERCEPT_FUNCTION(execve);
+
+  INTERCEPT_FUNCTION(syscall);
 }
 
 #endif // SANITIZER_POSIX
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index 8551424717de6..b3fb32ee8dcd4 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -46,6 +46,8 @@
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
 #include <sys/types.h>
 #include <sys/uio.h>
 
@@ -389,6 +391,26 @@ TEST_F(RtsanOpenedFileTest, IoctlBehavesWithOutputArg) {
   EXPECT_THAT(arg, Ge(0));
 }
 
+TEST_F(RtsanOpenedFileTest, FdopenDiesWhenRealtime) {
+  auto Func = [&]() {
+    FILE *f = fdopen(GetOpenFd(), "w");
+    EXPECT_THAT(f, Ne(nullptr));
+  };
+
+  ExpectRealtimeDeath(Func, "fdopen");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, FreopenDiesWhenRealtime) {
+  auto Func = [&]() {
+    FILE *newfile = freopen(GetTemporaryFilePath(), "w", GetOpenFile());
+    EXPECT_THAT(newfile, Ne(nullptr));
+  };
+
+  ExpectRealtimeDeath(Func, MAYBE_APPEND_64("freopen"));
+  ExpectNonRealtimeSurvival(Func);
+}
+
 TEST(TestRtsanInterceptors, IoctlBehavesWithOutputPointer) {
   // These initial checks just see if we CAN run these tests.
   // If we can't (can't open a socket, or can't find an interface, just
@@ -442,6 +464,60 @@ TEST_F(RtsanOpenedFileTest, Dup2DiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+TEST_F(RtsanFileTest, ChmodDiesWhenRealtime) {
+  auto Func = [this]() { chmod(GetTemporaryFilePath(), 0777); };
+  ExpectRealtimeDeath(Func, "chmod");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanOpenedFileTest, FchmodDiesWhenRealtime) {
+  auto Func = [this]() { fchmod(GetOpenFd(), 0777); };
+  ExpectRealtimeDeath(Func, "fchmod");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, UmaskDiesWhenRealtime) {
+  auto Func = []() { umask(0); };
+  ExpectRealtimeDeath(Func, "umask");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+class RtsanDirectoryTest : public ::testing::Test {
+protected:
+  void SetUp() override {
+    const ::testing::TestInfo *const test_info =
+        ::testing::UnitTest::GetInstance()->current_test_info();
+    directory_path_ = std::string("/tmp/rtsan_temp_dir_") + test_info->name();
+    RemoveTemporaryDirectory();
+  }
+
+  const char *GetTemporaryDirectoryPath() const {
+    return directory_path_.c_str();
+  }
+
+  void TearDown() override { RemoveTemporaryDirectory(); }
+
+private:
+  void RemoveTemporaryDirectory() const {
+    std::remove(GetTemporaryDirectoryPath());
+  }
+  std::string directory_path_;
+};
+
+TEST_F(RtsanDirectoryTest, MkdirDiesWhenRealtime) {
+  auto Func = [this]() { mkdir(GetTemporaryDirectoryPath(), 0777); };
+  ExpectRealtimeDeath(Func, "mkdir");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST_F(RtsanDirectoryTest, RmdirDiesWhenRealtime) {
+  // We don't actually create this directory before we try to remove it
+  // Thats OK - we are just making sure the call gets intercepted
+  auto Func = [this]() { rmdir(GetTemporaryDirectoryPath()); };
+  ExpectRealtimeDeath(Func, "rmdir");
+  ExpectNonRealtimeSurvival(Func);
+}
+
 TEST_F(RtsanOpenedFileTest, FreadDiesWhenRealtime) {
   auto Func = [this]() {
     char c{};
@@ -804,6 +880,16 @@ TEST(TestRtsanInterceptors, AcceptingASocketDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+#if SANITIZER_INTERCEPT_ACCEPT4
+TEST(TestRtsanInterceptors, Accepting4ASocketDiesWhenRealtime) {
+  auto Func = []() {
+    EXPECT_LT(accept4(kNotASocketFd, nullptr, nullptr, 0), 0);
+  };
+  ExpectRealtimeDeath(Func, "accept4");
+  ExpectNonRealtimeSurvival(Func);
+}
+#endif
+
 TEST(TestRtsanInterceptors, ConnectingASocketDiesWhenRealtime) {
   auto Func = []() { EXPECT_NE(connect(kNotASocketFd, nullptr, 0), 0); };
   ExpectRealtimeDeath(Func, "connect");
@@ -1035,4 +1121,20 @@ TEST(TestRtsanInterceptors, PipeDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+TEST(TestRtsanInterceptors, SyscallDiesWhenRealtime) {
+  auto Func = []() { syscall(SYS_getpid); };
+  ExpectRealtimeDeath(Func, "syscall");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, GetPidReturnsSame) {
+  int pid = syscall(SYS_getpid);
+  EXPECT_THAT(pid, Ne(-1));
+
+  EXPECT_THAT(getpid(), Eq(pid));
+}
+#pragma clang diagnostic pop
+
 #endif // SANITIZER_POSIX
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
index 50c726e09f287..ca560fc21035c 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
@@ -19,7 +19,10 @@ extern "C" const char *__rtsan_default_options() {
   // and make sure we do not overwhelm the syslog while testing. Also, let's
   // turn symbolization off to speed up testing, especially when not running
   // with llvm-symbolizer but with atos.
-  return "symbolize=false:abort_on_error=0:log_to_syslog=0";
+  return "symbolize=false:"
+         "abort_on_error=0:"
+         "log_to_syslog=0:"
+         "verify_interceptors=0:"; // some of our tests don't need interceptors
 #else
   // Let's turn symbolization off to speed up testing (more than 3 times speedup
   // observed).
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index ba3693dbd11f6..133eec755bfa5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -347,7 +347,7 @@ extern const short *_tolower_tab_;
   uptr copy_length = internal_strnlen(s, size);                               \
   char *new_mem = (char *)WRAP(malloc)(copy_length + 1);                      \
   if (common_flags()->intercept_strndup) {                                    \
-    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1));       \
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min<uptr>(size, copy_length + 1)); \
   }                                                                           \
   if (new_mem) {                                                              \
     COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length);             \
@@ -445,7 +445,7 @@ INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) {
 #endif
 
 #if SANITIZER_INTERCEPT_STRNDUP
-INTERCEPTOR(char*, strndup, const char *s, uptr size) {
+INTERCEPTOR(char*, strndup, const char *s, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
 }
@@ -455,7 +455,7 @@ INTERCEPTOR(char*, strndup, const char *s, uptr size) {
 #endif // SANITIZER_INTERCEPT_STRNDUP
 
 #if SANITIZER_INTERCEPT___STRNDUP
-INTERCEPTOR(char*, __strndup, const char *s, uptr size) {
+INTERCEPTOR(char*, __strndup, const char *s, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
 }
@@ -511,10 +511,10 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) {
 }
 
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc,
-                              const char *s1, const char *s2, uptr n,
+                              const char *s1, const char *s2, usize n,
                               int result)
 
-INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) {
+INTERCEPTOR(int, strncmp, const char *s1, const char *s2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_strncmp(s1, s2, size);
   void *ctx;
@@ -576,7 +576,7 @@ INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) {
 }
 
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc,
-                              const char *s1, const char *s2, uptr size,
+                              const char *s1, const char *s2, usize size,
                               int result)
 
 INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T size) {
@@ -833,13 +833,13 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
 
 #if SANITIZER_INTERCEPT_MEMCMP
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
-                              const void *s1, const void *s2, uptr n,
+                              const void *s1, const void *s2, usize n,
                               int result)
 
 // Common code for `memcmp` and `bcmp`.
 int MemcmpInterceptorCommon(void *ctx,
-                            int (*real_fn)(const void *, const void *, uptr),
-                            const void *a1, const void *a2, uptr size) {
+                            int (*real_fn)(const void *, const void *, usize),
+                            const void *a1, const void *a2, usize size) {
   if (common_flags()->intercept_memcmp) {
     if (common_flags()->strict_memcmp) {
       // Check the entire regions even if the first bytes of the buffers are
@@ -871,7 +871,7 @@ int MemcmpInterceptorCommon(void *ctx,
   return result;
 }
 
-INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) {
+INTERCEPTOR(int, memcmp, const void *a1, const void *a2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_memcmp(a1, a2, size);
   void *ctx;
@@ -885,7 +885,7 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_BCMP
-INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) {
+INTERCEPTOR(int, bcmp, const void *a1, const void *a2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_memcmp(a1, a2, size);
   void *ctx;
@@ -1138,7 +1138,7 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
 #endif
 
 #if SANITIZER_INTERCEPT_FWRITE
-INTERCEPTOR(SIZE_T, fwrite, const void *p, uptr size, uptr nmemb, void *file) {
+INTERCEPTOR(SIZE_T, fwrite, const void *p, usize size, usize nmemb, void *file) {
   // libc file streams can call user-supplied functions, see fopencookie.
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file);
@@ -1313,7 +1313,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64));
   } else if (res != -1 && option == PR_GET_PDEATHSIG) {
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int));
-#  if !SANITIZER_ANDROID
+#  if SANITIZER_GLIBC
   } else if (res != -1 && option == PR_SET_SECCOMP &&
              arg2 == SECCOMP_MODE_FILTER) {
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg3), struct_sock_fprog_sz);
@@ -2538,82 +2538,6 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,
 #define INIT_GLOB64
 #endif  // SANITIZER_INTERCEPT_GLOB64
 
-#if SANITIZER_INTERCEPT___B64_TO
-INTERCEPTOR(int, __b64_ntop, unsigned char const *src, SIZE_T srclength,
-            char *target, SIZE_T targsize) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, __b64_ntop, src, srclength, target, targsize);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclength);
-  int res = REAL(__b64_ntop)(src, srclength, target, targsize);
-  if (res >= 0)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res + 1);
-  return res;
-}
-INTERCEPTOR(int, __b64_pton, char const *src, char *target, SIZE_T targsize) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, __b64_pton, src, target, targsize);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
-  int res = REAL(__b64_pton)(src, target, targsize);
-  if (res >= 0)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res);
-  return res;
-}
-#define INIT___B64_TO                      \
-    COMMON_INTERCEPT_FUNCTION(__b64_ntop); \
-    COMMON_INTERCEPT_FUNCTION(__b64_pton);
-#else  // SANITIZER_INTERCEPT___B64_TO
-#define INIT___B64_TO
-#endif  // SANITIZER_INTERCEPT___B64_TO
-
-#if SANITIZER_INTERCEPT_DN_COMP_EXPAND
-#  if __GLIBC_PREREQ(2, 34)
-// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf
-#    define DN_COMP_INTERCEPTOR_NAME dn_comp
-#    define DN_EXPAND_INTERCEPTOR_NAME dn_expand
-#  else
-#    define DN_COMP_INTERCEPTOR_NAME __dn_comp
-#    define DN_EXPAND_INTERCEPTOR_NAME __dn_expand
-#  endif
-INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn,
-            unsigned char *comp_dn, int length, unsigned char **dnptrs,
-            unsigned char **lastdnptr) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, DN_COMP_INTERCEPTOR_NAME, exp_dn, comp_dn,
-                           length, dnptrs, lastdnptr);
-  int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs,
-                                           lastdnptr);
-  if (res >= 0) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, comp_dn, res);
-    if (dnptrs && lastdnptr) {
-      unsigned char **p = dnptrs;
-      for (; p != lastdnptr && *p; ++p)
-        ;
-      if (p != lastdnptr)
-        ++p;
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dnptrs, (p - dnptrs) * sizeof(*p));
-    }
-  }
-  return res;
-}
-INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base,
-            unsigned char const *end, unsigned char const *src, char *dest,
-            int space) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, DN_EXPAND_INTERCEPTOR_NAME, base, end, src,
-                           dest, space);
-  // TODO: add read check if __dn_comp intercept added
-  int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space);
-  if (res >= 0)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, internal_strlen(dest) + 1);
-  return res;
-}
-#  define INIT_DN_COMP_EXPAND                            \
-    COMMON_INTERCEPT_FUNCTION(DN_COMP_INTERCEPTOR_NAME); \
-    COMMON_INTERCEPT_FUNCTION(DN_EXPAND_INTERCEPTOR_NAME);
-#else  // SANITIZER_INTERCEPT_DN_COMP_EXPAND
-#  define INIT_DN_COMP_EXPAND
-#endif  // SANITIZER_INTERCEPT_DN_COMP_EXPAND
-
 #if SANITIZER_INTERCEPT_POSIX_SPAWN
 
 template <class RealSpawnPtr>
@@ -6553,12 +6477,12 @@ static void MlockIsUnsupported() {
           SanitizerToolName);
 }
 
-INTERCEPTOR(int, mlock, const void *addr, uptr len) {
+INTERCEPTOR(int, mlock, const void *addr, usize len) {
   MlockIsUnsupported();
   return 0;
 }
 
-INTERCEPTOR(int, munlock, const void *addr, uptr len) {
+INTERCEPTOR(int, munlock, const void *addr, usize len) {
   MlockIsUnsupported();
   return 0;
 }
@@ -10346,8 +10270,6 @@ static void InitializeCommonInterceptors() {
   INIT_TIMESPEC_GET;
   INIT_GLOB;
   INIT_GLOB64;
-  INIT___B64_TO;
-  INIT_DN_COMP_EXPAND;
   INIT_POSIX_SPAWN;
   INIT_WAIT;
   INIT_WAIT4;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
index 52e489d02cda8..1565a494140f6 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
@@ -82,7 +82,7 @@
 #endif
 
 #if SANITIZER_INTERCEPT_MEMSET
-INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
+INTERCEPTOR(void *, memset, void *dst, int v, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
 }
@@ -93,7 +93,7 @@ INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_MEMMOVE
-INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
+INTERCEPTOR(void *, memmove, void *dst, const void *src, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
 }
@@ -104,7 +104,7 @@ INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_MEMCPY
-INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, usize size) {
   // On OS X, calling internal_memcpy here will cause memory corruptions,
   // because memcpy and memmove are actually aliases of the same
   // implementation.  We need to use internal_memmove here.
@@ -133,63 +133,63 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_AEABI_MEM
-INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
+INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
 
 // Note the argument order.
-INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
+INTERCEPTOR(void *, __aeabi_memset, void *block, usize size, int c) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
+INTERCEPTOR(void *, __aeabi_memset4, void *block, usize size, int c) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
+INTERCEPTOR(void *, __aeabi_memset8, void *block, usize size, int c) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
+INTERCEPTOR(void *, __aeabi_memclr, void *block, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
+INTERCEPTOR(void *, __aeabi_memclr4, void *block, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
 
-INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
+INTERCEPTOR(void *, __aeabi_memclr8, void *block, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
@@ -212,7 +212,7 @@ INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
 #endif  // SANITIZER_INTERCEPT_AEABI_MEM
 
 #if SANITIZER_INTERCEPT___BZERO
-INTERCEPTOR(void *, __bzero, void *block, uptr size) {
+INTERCEPTOR(void *, __bzero, void *block, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
@@ -222,7 +222,7 @@ INTERCEPTOR(void *, __bzero, void *block, uptr size) {
 #endif  // SANITIZER_INTERCEPT___BZERO
 
 #if SANITIZER_INTERCEPT_BZERO
-INTERCEPTOR(void *, bzero, void *block, uptr size) {
+INTERCEPTOR(void *, bzero, void *block, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
index 9208b12552ff5..fff60c96f632f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -203,6 +203,12 @@ typedef __SIZE_TYPE__ usize;
 typedef uptr usize;
 #endif
 
+#if defined(__s390__) && !defined(__s390x__)
+typedef long ssize;
+#else
+typedef sptr ssize;
+#endif
+
 typedef u64 tid_t;
 
 // ----------- ATTENTION -------------
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 525bc1038619d..62b1dc43dce13 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -57,7 +57,7 @@
 // that, it was never implemented. So just define it to zero.
 #    undef MAP_NORESERVE
 #    define MAP_NORESERVE 0
-extern const Elf_Auxinfo *__elf_aux_vector;
+extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak));
 extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
                                struct sigaction *oldact);
 #  endif
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 26d2e8d4ed768..c8a0afccb254e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -972,8 +972,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
 LowLevelAllocator allocator_for_env;
 
 static bool ShouldCheckInterceptors() {
-  // Restrict "interceptors working?" check to ASan and TSan.
-  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
+  // Restrict "interceptors working?" check
+  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer",
+                                   "RealtimeSanitizer"};
   size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
   for (size_t i = 0; i < count; i++) {
     if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 190cad7cf7c3f..b4940b66da971 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -266,8 +266,6 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_TIMESPEC_GET SI_LINUX
 #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
-#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_DN_COMP_EXPAND SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX
 #define SANITIZER_INTERCEPT_WAIT SI_POSIX
 #define SANITIZER_INTERCEPT_INET SI_POSIX
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 7d98f8e9a9d80..cacbb5b9959e0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -313,7 +313,7 @@ extern unsigned struct_statvfs_sz;
 
 struct __sanitizer_iovec {
   void *iov_base;
-  uptr iov_len;
+  usize iov_len;
 };
 
 #if !SANITIZER_ANDROID
@@ -523,6 +523,7 @@ struct __sanitizer_dirent64 {
   unsigned short d_reclen;
   // more fields that we don't care about
 };
+extern unsigned struct_sock_fprog_sz;
 #endif
 
 #if defined(__x86_64__) && !defined(_LP64)
@@ -1076,7 +1077,6 @@ extern unsigned struct_serial_struct_sz;
 extern unsigned struct_sockaddr_ax25_sz;
 extern unsigned struct_unimapdesc_sz;
 extern unsigned struct_unimapinit_sz;
-extern unsigned struct_sock_fprog_sz;
 #  endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
 
 extern const unsigned long __sanitizer_bufsiz;
diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index dc700cec9becb..3f9ae866a7553 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -98,6 +98,7 @@ set(SCUDO_HEADERS
   tsd_exclusive.h
   tsd_shared.h
   tsd.h
+  type_traits.h
   vector.h
   wrappers_c_checks.h
   wrappers_c.h
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
index 5477236ac1f39..ea12a5b1447f6 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
@@ -12,35 +12,7 @@
 #include "condition_variable.h"
 #include "internal_defs.h"
 #include "secondary.h"
-
-namespace {
-
-template <typename T> struct removeConst {
-  using type = T;
-};
-template <typename T> struct removeConst<const T> {
-  using type = T;
-};
-
-// This is only used for SFINAE when detecting if a type is defined.
-template <typename T> struct voidAdaptor {
-  using type = void;
-};
-
-// This is used for detecting the case that defines the flag with wrong type and
-// it'll be viewed as undefined optional flag.
-template <typename L, typename R> struct assertSameType {
-  template <typename, typename> struct isSame {
-    static constexpr bool value = false;
-  };
-  template <typename T> struct isSame<T, T> {
-    static constexpr bool value = true;
-  };
-  static_assert(isSame<L, R>::value, "Flag type mismatches");
-  using type = R;
-};
-
-} // namespace
+#include "type_traits.h"
 
 namespace scudo {
 
diff --git a/compiler-rt/lib/scudo/standalone/list.h b/compiler-rt/lib/scudo/standalone/list.h
index 6b952a610e305..e0b82788251b7 100644
--- a/compiler-rt/lib/scudo/standalone/list.h
+++ b/compiler-rt/lib/scudo/standalone/list.h
@@ -10,17 +10,7 @@
 #define SCUDO_LIST_H_
 
 #include "internal_defs.h"
-
-// TODO: Move the helpers to a header.
-namespace {
-template <typename T> struct isPointer {
-  static constexpr bool value = false;
-};
-
-template <typename T> struct isPointer<T *> {
-  static constexpr bool value = true;
-};
-} // namespace
+#include "type_traits.h"
 
 namespace scudo {
 
@@ -58,13 +48,15 @@ class LinkOp {
 
 template <class T> class LinkOp<T, /*LinkWithPtr=*/false> {
 public:
-  using LinkTy = decltype(T::Next);
+  using LinkTy = typename assertSameType<
+      typename removeConst<decltype(T::Next)>::type,
+      typename removeConst<decltype(T::EndOfListVal)>::type>::type;
 
   LinkOp() = default;
-  LinkOp(T *BaseT, uptr BaseSize) : Base(BaseT), Size(BaseSize) {}
+  LinkOp(T *BaseT, uptr BaseSize)
+      : Base(BaseT), Size(static_cast<LinkTy>(BaseSize)) {}
   void init(T *LinkBase, uptr BaseSize) {
     Base = LinkBase;
-    // TODO: Check if the `BaseSize` can fit in `Size`.
     Size = static_cast<LinkTy>(BaseSize);
   }
   T *getBase() const { return Base; }
@@ -79,11 +71,12 @@ template <class T> class LinkOp<T, /*LinkWithPtr=*/false> {
   }
   // Set `X->Next` to `Next`.
   void setNext(T *X, T *Next) const {
-    // TODO: Check if the offset fits in the size of `LinkTy`.
-    if (Next == nullptr)
+    if (Next == nullptr) {
       X->Next = getEndOfListVal();
-    else
+    } else {
+      assertElementInRange(Next);
       X->Next = static_cast<LinkTy>(Next - Base);
+    }
   }
 
   T *getPrev(T *X) const {
@@ -95,17 +88,22 @@ template <class T> class LinkOp<T, /*LinkWithPtr=*/false> {
   }
   // Set `X->Prev` to `Prev`.
   void setPrev(T *X, T *Prev) const {
-    DCHECK_LT(reinterpret_cast<uptr>(Prev),
-              reinterpret_cast<uptr>(Base + Size));
-    if (Prev == nullptr)
+    if (Prev == nullptr) {
       X->Prev = getEndOfListVal();
-    else
+    } else {
+      assertElementInRange(Prev);
       X->Prev = static_cast<LinkTy>(Prev - Base);
+    }
   }
 
-  // TODO: `LinkTy` should be the same as decltype(T::EndOfListVal).
   LinkTy getEndOfListVal() const { return T::EndOfListVal; }
 
+private:
+  void assertElementInRange(T *X) const {
+    DCHECK_GE(reinterpret_cast<uptr>(X), reinterpret_cast<uptr>(Base));
+    DCHECK_LE(static_cast<LinkTy>(X - Base), Size);
+  }
+
 protected:
   T *Base = nullptr;
   LinkTy Size = 0;
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 2fae29e5a2168..25b82356ca20f 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -71,7 +71,8 @@ namespace {
 
 struct CachedBlock {
   static constexpr u16 CacheIndexMax = UINT16_MAX;
-  static constexpr u16 InvalidEntry = CacheIndexMax;
+  static constexpr u16 EndOfListVal = CacheIndexMax;
+
   // We allow a certain amount of fragmentation and part of the fragmented bytes
   // will be released by `releaseAndZeroPagesToOS()`. This increases the chance
   // of cache hit rate and reduces the overhead to the RSS at the same time. See
@@ -206,17 +207,16 @@ class MapAllocatorCache {
                       &Fractional);
     const s32 Interval = atomic_load_relaxed(&ReleaseToOsIntervalMs);
     Str->append(
-        "Stats: MapAllocatorCache: EntriesCount: %d, "
+        "Stats: MapAllocatorCache: EntriesCount: %zu, "
         "MaxEntriesCount: %u, MaxEntrySize: %zu, ReleaseToOsIntervalMs = %d\n",
-        EntriesCount, atomic_load_relaxed(&MaxEntriesCount),
+        LRUEntries.size(), atomic_load_relaxed(&MaxEntriesCount),
         atomic_load_relaxed(&MaxEntrySize), Interval >= 0 ? Interval : -1);
     Str->append("Stats: CacheRetrievalStats: SuccessRate: %u/%u "
                 "(%zu.%02zu%%)\n",
                 SuccessfulRetrieves, CallsToRetrieve, Integral, Fractional);
     Str->append("Cache Entry Info (Most Recent -> Least Recent):\n");
 
-    for (u32 I = LRUHead; I != CachedBlock::InvalidEntry; I = Entries[I].Next) {
-      CachedBlock &Entry = Entries[I];
+    for (CachedBlock &Entry : LRUEntries) {
       Str->append("  StartBlockAddress: 0x%zx, EndBlockAddress: 0x%zx, "
                   "BlockSize: %zu %s\n",
                   Entry.CommitBase, Entry.CommitBase + Entry.CommitSize,
@@ -234,7 +234,7 @@ class MapAllocatorCache {
                 "Cache entry array is too large to be indexed.");
 
   void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS {
-    DCHECK_EQ(EntriesCount, 0U);
+    DCHECK_EQ(LRUEntries.size(), 0U);
     setOption(Option::MaxCacheEntriesCount,
               static_cast<sptr>(Config::getDefaultMaxEntriesCount()));
     setOption(Option::MaxCacheEntrySize,
@@ -244,17 +244,13 @@ class MapAllocatorCache {
       ReleaseToOsInterval = Config::getDefaultReleaseToOsIntervalMs();
     setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
 
-    // The cache is initially empty
-    LRUHead = CachedBlock::InvalidEntry;
-    LRUTail = CachedBlock::InvalidEntry;
-
-    // Available entries will be retrieved starting from the beginning of the
-    // Entries array
-    AvailableHead = 0;
-    for (u32 I = 0; I < Config::getEntriesArraySize() - 1; I++)
-      Entries[I].Next = static_cast<u16>(I + 1);
+    LRUEntries.clear();
+    LRUEntries.init(Entries, sizeof(Entries));
 
-    Entries[Config::getEntriesArraySize() - 1].Next = CachedBlock::InvalidEntry;
+    AvailEntries.clear();
+    AvailEntries.init(Entries, sizeof(Entries));
+    for (u32 I = 0; I < Config::getEntriesArraySize(); I++)
+      AvailEntries.push_back(&Entries[I]);
   }
 
   void store(const Options &Options, uptr CommitBase, uptr CommitSize,
@@ -326,11 +322,15 @@ class MapAllocatorCache {
         Entry = PrevEntry;
       }
 
-      // All excess entries are evicted from the cache
-      while (needToEvict()) {
+      // All excess entries are evicted from the cache. Note that when
+      // `MaxEntriesCount` is zero, cache storing shouldn't happen and it's
+      // guarded by the `DCHECK(canCache(CommitSize))` above. As a result, we
+      // won't try to pop `LRUEntries` when it's empty.
+      while (LRUEntries.size() >= atomic_load_relaxed(&MaxEntriesCount)) {
         // Save MemMaps of evicted entries to perform unmap outside of lock
-        EvictionMemMaps.push_back(Entries[LRUTail].MemMap);
-        remove(LRUTail);
+        CachedBlock *Entry = LRUEntries.back();
+        EvictionMemMaps.push_back(Entry->MemMap);
+        remove(Entry);
       }
 
       insert(Entry);
@@ -360,9 +360,9 @@ class MapAllocatorCache {
     {
       ScopedLock L(Mutex);
       CallsToRetrieve++;
-      if (EntriesCount == 0)
+      if (LRUEntries.size() == 0)
         return {};
-      u16 RetrievedIndex = CachedBlock::InvalidEntry;
+      CachedBlock *RetrievedEntry = nullptr;
       uptr MinDiff = UINTPTR_MAX;
 
       //  Since allocation sizes don't always match cached memory chunk sizes
@@ -382,10 +382,9 @@ class MapAllocatorCache {
       //  well as the header metadata. If EntryHeaderPos - CommitBase exceeds
       //  MaxAllowedFragmentedPages * PageSize, the cached memory chunk is
       //  not considered valid for retrieval.
-      for (u16 I = LRUHead; I != CachedBlock::InvalidEntry;
-           I = Entries[I].Next) {
-        const uptr CommitBase = Entries[I].CommitBase;
-        const uptr CommitSize = Entries[I].CommitSize;
+      for (CachedBlock &Entry : LRUEntries) {
+        const uptr CommitBase = Entry.CommitBase;
+        const uptr CommitSize = Entry.CommitSize;
         const uptr AllocPos =
             roundDown(CommitBase + CommitSize - Size, Alignment);
         const uptr HeaderPos = AllocPos - HeadersSize;
@@ -408,7 +407,7 @@ class MapAllocatorCache {
           continue;
 
         MinDiff = Diff;
-        RetrievedIndex = I;
+        RetrievedEntry = &Entry;
         EntryHeaderPos = HeaderPos;
 
         // Immediately use a cached block if its size is close enough to the
@@ -418,9 +417,10 @@ class MapAllocatorCache {
         if (Diff <= OptimalFitThesholdBytes)
           break;
       }
-      if (RetrievedIndex != CachedBlock::InvalidEntry) {
-        Entry = Entries[RetrievedIndex];
-        remove(RetrievedIndex);
+
+      if (RetrievedEntry != nullptr) {
+        Entry = *RetrievedEntry;
+        remove(RetrievedEntry);
         SuccessfulRetrieves++;
       }
     }
@@ -499,10 +499,8 @@ class MapAllocatorCache {
         Quarantine[I].invalidate();
       }
     }
-    for (u32 I = LRUHead; I != CachedBlock::InvalidEntry; I = Entries[I].Next) {
-      Entries[I].MemMap.setMemoryPermission(Entries[I].CommitBase,
-                                            Entries[I].CommitSize, 0);
-    }
+    for (CachedBlock &Entry : LRUEntries)
+      Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0);
     QuarantinePos = -1U;
   }
 
@@ -513,64 +511,19 @@ class MapAllocatorCache {
   void unmapTestOnly() { empty(); }
 
 private:
-  bool needToEvict() REQUIRES(Mutex) {
-    return (EntriesCount >= atomic_load_relaxed(&MaxEntriesCount));
-  }
-
   void insert(const CachedBlock &Entry) REQUIRES(Mutex) {
-    DCHECK_LT(EntriesCount, atomic_load_relaxed(&MaxEntriesCount));
-
-    // Cache should be populated with valid entries when not empty
-    DCHECK_NE(AvailableHead, CachedBlock::InvalidEntry);
-
-    u32 FreeIndex = AvailableHead;
-    AvailableHead = Entries[AvailableHead].Next;
-
-    if (EntriesCount == 0) {
-      LRUTail = static_cast<u16>(FreeIndex);
-    } else {
-      // Check list order
-      if (EntriesCount > 1)
-        DCHECK_GE(Entries[LRUHead].Time, Entries[Entries[LRUHead].Next].Time);
-      Entries[LRUHead].Prev = static_cast<u16>(FreeIndex);
-    }
+    CachedBlock *AvailEntry = AvailEntries.front();
+    AvailEntries.pop_front();
 
-    Entries[FreeIndex] = Entry;
-    Entries[FreeIndex].Next = LRUHead;
-    Entries[FreeIndex].Prev = CachedBlock::InvalidEntry;
-    LRUHead = static_cast<u16>(FreeIndex);
-    EntriesCount++;
-
-    // Availability stack should not have available entries when all entries
-    // are in use
-    if (EntriesCount == Config::getEntriesArraySize())
-      DCHECK_EQ(AvailableHead, CachedBlock::InvalidEntry);
+    *AvailEntry = Entry;
+    LRUEntries.push_front(AvailEntry);
   }
 
-  void remove(uptr I) REQUIRES(Mutex) {
-    DCHECK(Entries[I].isValid());
-
-    Entries[I].invalidate();
-
-    if (I == LRUHead)
-      LRUHead = Entries[I].Next;
-    else
-      Entries[Entries[I].Prev].Next = Entries[I].Next;
-
-    if (I == LRUTail)
-      LRUTail = Entries[I].Prev;
-    else
-      Entries[Entries[I].Next].Prev = Entries[I].Prev;
-
-    Entries[I].Next = AvailableHead;
-    AvailableHead = static_cast<u16>(I);
-    EntriesCount--;
-
-    // Cache should not have valid entries when not empty
-    if (EntriesCount == 0) {
-      DCHECK_EQ(LRUHead, CachedBlock::InvalidEntry);
-      DCHECK_EQ(LRUTail, CachedBlock::InvalidEntry);
-    }
+  void remove(CachedBlock *Entry) REQUIRES(Mutex) {
+    DCHECK(Entry->isValid());
+    LRUEntries.remove(Entry);
+    Entry->invalidate();
+    AvailEntries.push_front(Entry);
   }
 
   void empty() {
@@ -578,14 +531,10 @@ class MapAllocatorCache {
     uptr N = 0;
     {
       ScopedLock L(Mutex);
-      for (uptr I = 0; I < Config::getEntriesArraySize(); I++) {
-        if (!Entries[I].isValid())
-          continue;
-        MapInfo[N] = Entries[I].MemMap;
-        remove(I);
-        N++;
-      }
-      EntriesCount = 0;
+
+      for (CachedBlock &Entry : LRUEntries)
+        MapInfo[N++] = Entry.MemMap;
+      LRUEntries.clear();
     }
     for (uptr I = 0; I < N; I++) {
       MemMapT &MemMap = MapInfo[I];
@@ -607,7 +556,7 @@ class MapAllocatorCache {
 
   void releaseOlderThan(u64 Time) EXCLUDES(Mutex) {
     ScopedLock L(Mutex);
-    if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
+    if (!LRUEntries.size() || OldestTime == 0 || OldestTime > Time)
       return;
     OldestTime = 0;
     for (uptr I = 0; I < Config::getQuarantineSize(); I++)
@@ -617,7 +566,6 @@ class MapAllocatorCache {
   }
 
   HybridMutex Mutex;
-  u32 EntriesCount GUARDED_BY(Mutex) = 0;
   u32 QuarantinePos GUARDED_BY(Mutex) = 0;
   atomic_u32 MaxEntriesCount = {};
   atomic_uptr MaxEntrySize = {};
@@ -630,12 +578,10 @@ class MapAllocatorCache {
   NonZeroLengthArray<CachedBlock, Config::getQuarantineSize()>
       Quarantine GUARDED_BY(Mutex) = {};
 
-  // The LRUHead of the cache is the most recently used cache entry
-  u16 LRUHead GUARDED_BY(Mutex) = 0;
-  // The LRUTail of the cache is the least recently used cache entry
-  u16 LRUTail GUARDED_BY(Mutex) = 0;
-  // The AvailableHead is the top of the stack of available entries
-  u16 AvailableHead GUARDED_BY(Mutex) = 0;
+  // Cached blocks stored in LRU order
+  DoublyLinkedList<CachedBlock> LRUEntries GUARDED_BY(Mutex);
+  // The unused Entries
+  SinglyLinkedList<CachedBlock> AvailEntries GUARDED_BY(Mutex);
 };
 
 template <typename Config> class MapAllocator {
diff --git a/compiler-rt/lib/scudo/standalone/type_traits.h b/compiler-rt/lib/scudo/standalone/type_traits.h
new file mode 100644
index 0000000000000..16ed5a048f82b
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/type_traits.h
@@ -0,0 +1,47 @@
+//===-- type_traits.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TYPE_TRAITS_H_
+#define SCUDO_TYPE_TRAITS_H_
+
+namespace scudo {
+
+template <typename T> struct removeConst {
+  using type = T;
+};
+template <typename T> struct removeConst<const T> {
+  using type = T;
+};
+
+// This is only used for SFINAE when detecting if a type is defined.
+template <typename T> struct voidAdaptor {
+  using type = void;
+};
+
+template <typename L, typename R> struct assertSameType {
+  template <typename, typename> struct isSame {
+    static constexpr bool value = false;
+  };
+  template <typename T> struct isSame<T, T> {
+    static constexpr bool value = true;
+  };
+  static_assert(isSame<L, R>::value, "Type mismatches");
+  using type = R;
+};
+
+template <typename T> struct isPointer {
+  static constexpr bool value = false;
+};
+
+template <typename T> struct isPointer<T *> {
+  static constexpr bool value = true;
+};
+
+} // namespace scudo
+
+#endif // SCUDO_TYPE_TRAITS_H_
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 423d97e94d81a..f671c8167a3c4 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -97,7 +97,7 @@ extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
 extern "C" int pthread_setspecific(unsigned key, const void *v);
 DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
-DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize size)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 extern "C" int pthread_equal(void *t1, void *t2);
 extern "C" void *pthread_self();
@@ -768,7 +768,7 @@ TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
   return REAL(strcpy)(dst, src);
 }
 
-TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
+TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, usize n) {
   SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
   uptr srclen = internal_strnlen(src, n);
   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
diff --git a/compiler-rt/lib/xray/CMakeLists.txt b/compiler-rt/lib/xray/CMakeLists.txt
index 7e3f1a0aa616e..e7f01a2f4f164 100644
--- a/compiler-rt/lib/xray/CMakeLists.txt
+++ b/compiler-rt/lib/xray/CMakeLists.txt
@@ -96,6 +96,16 @@ set(hexagon_SOURCES
   xray_trampoline_hexagon.S
   )
 
+set(riscv32_SOURCES
+  xray_riscv.cpp
+  xray_trampoline_riscv32.S
+  )
+
+set(riscv64_SOURCES
+  xray_riscv.cpp
+  xray_trampoline_riscv64.S
+  )
+
 set(XRAY_SOURCE_ARCHS
   arm
   armhf
@@ -156,6 +166,8 @@ set(XRAY_ALL_SOURCE_FILES
   ${mips64_SOURCES}
   ${mips64el_SOURCES}
   ${powerpc64le_SOURCES}
+  ${riscv32_SOURCES}
+  ${riscv64_SOURCES}
   ${XRAY_IMPL_HEADERS}
   )
 list(REMOVE_DUPLICATES XRAY_ALL_SOURCE_FILES)
diff --git a/compiler-rt/lib/xray/weak_symbols.txt b/compiler-rt/lib/xray/weak_symbols.txt
index 963fff2d697ee..379749796af25 100644
--- a/compiler-rt/lib/xray/weak_symbols.txt
+++ b/compiler-rt/lib/xray/weak_symbols.txt
@@ -2,3 +2,4 @@ ___start_xray_fn_idx
 ___start_xray_instr_map
 ___stop_xray_fn_idx
 ___stop_xray_instr_map
+___xray_default_options
diff --git a/compiler-rt/lib/xray/xray_flags.cpp b/compiler-rt/lib/xray/xray_flags.cpp
index e4c6906dc4434..9d3ee3ea85528 100644
--- a/compiler-rt/lib/xray/xray_flags.cpp
+++ b/compiler-rt/lib/xray/xray_flags.cpp
@@ -67,6 +67,10 @@ void initializeFlags() XRAY_NEVER_INSTRUMENT {
   const char *XRayCompileFlags = useCompilerDefinedFlags();
   XRayParser.ParseString(XRayCompileFlags);
 
+  // Use options provided at build time of the instrumented program.
+  const char *XRayDefaultOptions = __xray_default_options();
+  XRayParser.ParseString(XRayDefaultOptions);
+
   // Override from environment variables.
   XRayParser.ParseStringFromEnv("XRAY_OPTIONS");
 
@@ -82,3 +86,7 @@ void initializeFlags() XRAY_NEVER_INSTRUMENT {
 }
 
 } // namespace __xray
+
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __xray_default_options, void) {
+  return "";
+}
diff --git a/compiler-rt/lib/xray/xray_flags.h b/compiler-rt/lib/xray/xray_flags.h
index cce6fe9d62f9f..e29d62e0a3b91 100644
--- a/compiler-rt/lib/xray/xray_flags.h
+++ b/compiler-rt/lib/xray/xray_flags.h
@@ -17,6 +17,13 @@
 #include "sanitizer_common/sanitizer_flag_parser.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 
+extern "C" {
+// Users can specify their default options upon building the instrumented
+// binaries by provide a definition of this function.
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__xray_default_options();
+}
+
 namespace __xray {
 
 struct Flags {
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index b6f0e6762f168..4ec492c266d80 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -57,6 +57,10 @@ static const int16_t cSledLength = 64;
 static const int16_t cSledLength = 8;
 #elif defined(__hexagon__)
 static const int16_t cSledLength = 20;
+#elif defined(__riscv) && (__riscv_xlen == 64)
+static const int16_t cSledLength = 68;
+#elif defined(__riscv) && (__riscv_xlen == 32)
+static const int16_t cSledLength = 52;
 #else
 #error "Unsupported CPU Architecture"
 #endif /* CPU architecture */
diff --git a/compiler-rt/lib/xray/xray_riscv.cpp b/compiler-rt/lib/xray/xray_riscv.cpp
new file mode 100644
index 0000000000000..e3a7cdb18b640
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_riscv.cpp
@@ -0,0 +1,266 @@
+//===-- xray_riscv.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// Implementation of RISC-V specific routines (32- and 64-bit).
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_common.h"
+#include "xray_defs.h"
+#include "xray_interface_internal.h"
+#include <atomic>
+
+namespace __xray {
+
+// The machine codes for some instructions used in runtime patching.
+enum PatchOpcodes : uint32_t {
+  PO_ADDI = 0x00000013, // addi rd, rs1, imm
+  PO_ADD = 0x00000033,  // add rd, rs1, rs2
+  PO_SW = 0x00002023,   // sw rs2, imm(rs1)
+  PO_SD = 0x00003023,   // sd rs2, imm(rs1)
+  PO_LUI = 0x00000037,  // lui rd, imm
+  PO_OR = 0x00006033,   // or rd, rs1, rs2
+  PO_SLLI = 0x00001013, // slli rd, rs1, shamt
+  PO_JALR = 0x00000067, // jalr rd, rs1
+  PO_LW = 0x00002003,   // lw rd, imm(rs1)
+  PO_LD = 0x00003003,   // ld rd, imm(rs1)
+  PO_J = 0x0000006f,    // jal imm
+  PO_NOP = PO_ADDI,     // addi x0, x0, 0
+};
+
+enum RegNum : uint32_t {
+  RN_X0 = 0,
+  RN_RA = 1,
+  RN_SP = 2,
+  RN_T1 = 6,
+  RN_A0 = 10,
+};
+
+static inline uint32_t encodeRTypeInstruction(uint32_t Opcode, uint32_t Rs1,
+                                              uint32_t Rs2, uint32_t Rd) {
+  return Rs2 << 20 | Rs1 << 15 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeITypeInstruction(uint32_t Opcode, uint32_t Rs1,
+                                              uint32_t Rd, uint32_t Imm) {
+  return Imm << 20 | Rs1 << 15 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeSTypeInstruction(uint32_t Opcode, uint32_t Rs1,
+                                              uint32_t Rs2, uint32_t Imm) {
+  uint32_t ImmMSB = (Imm & 0xfe0) << 20;
+  uint32_t ImmLSB = (Imm & 0x01f) << 7;
+  return ImmMSB | Rs2 << 20 | Rs1 << 15 | ImmLSB | Opcode;
+}
+
+static inline uint32_t encodeUTypeInstruction(uint32_t Opcode, uint32_t Rd,
+                                              uint32_t Imm) {
+  return Imm << 12 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeJTypeInstruction(uint32_t Opcode, uint32_t Rd,
+                                              uint32_t Imm) {
+  uint32_t ImmMSB = (Imm & 0x100000) << 11;
+  uint32_t ImmLSB = (Imm & 0x7fe) << 20;
+  uint32_t Imm11 = (Imm & 0x800) << 9;
+  uint32_t Imm1912 = (Imm & 0xff000);
+  return ImmMSB | ImmLSB | Imm11 | Imm1912 | Rd << 7 | Opcode;
+}
+
+static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }
+static uint32_t lo12(uint32_t val) { return val & 0xfff; }
+
+static inline bool patchSled(const bool Enable, const uint32_t FuncId,
+                             const XRaySledEntry &Sled,
+                             void (*TracingHook)()) XRAY_NEVER_INSTRUMENT {
+  // When |Enable| == true,
+  // We replace the following compile-time stub (sled):
+  //
+  // xray_sled_n:
+  //	J .tmpN
+  //	21 or 33 C.NOPs (42 or 66 bytes)
+  //	.tmpN
+  //
+  // With one of the following runtime patches:
+  //
+  // xray_sled_n (32-bit):
+  //    addi sp, sp, -16                                ;create stack frame
+  //    sw ra, 12(sp)                                   ;save return address
+  //    sw a0, 8(sp)                                    ;save register a0
+  //    lui ra, %hi(__xray_FunctionEntry/Exit)
+  //    addi ra, ra, %lo(__xray_FunctionEntry/Exit)
+  //    lui a0, %hi(function_id)
+  //    addi a0, a0, %lo(function_id)                   ;pass function id
+  //    jalr ra                                         ;call Tracing hook
+  //    lw a0, 8(sp)                                    ;restore register a0
+  //    lw ra, 12(sp)                                   ;restore return address
+  //    addi sp, sp, 16                                 ;delete stack frame
+  //
+  // xray_sled_n (64-bit):
+  //    addi sp, sp, -32                                ;create stack frame
+  //    sd ra, 24(sp)                                   ;save return address
+  //    sd a0, 16(sp)                                   ;save register a0
+  //    sd t1, 8(sp)                                    ;save register t1
+  //    lui t1, %highest(__xray_FunctionEntry/Exit)
+  //    addi t1, t1, %higher(__xray_FunctionEntry/Exit)
+  //    slli t1, t1, 32
+  //    lui ra, ra, %hi(__xray_FunctionEntry/Exit)
+  //    addi ra, ra, %lo(__xray_FunctionEntry/Exit)
+  //    add ra, t1, ra
+  //    lui a0, %hi(function_id)
+  //    addi a0, a0, %lo(function_id)                   ;pass function id
+  //    jalr ra                                         ;call Tracing hook
+  //    ld t1, 8(sp)                                    ;restore register t1
+  //    ld a0, 16(sp)                                   ;restore register a0
+  //    ld ra, 24(sp)                                   ;restore return address
+  //    addi sp, sp, 32                                 ;delete stack frame
+  //
+  // Replacement of the first 4-byte instruction should be the last and atomic
+  // operation, so that the user code which reaches the sled concurrently
+  // either jumps over the whole sled, or executes the whole sled when the
+  // latter is ready.
+  //
+  // When |Enable|==false, we set back the first instruction in the sled to be
+  //   J 44 bytes (rv32)
+  //   J 68 bytes (rv64)
+
+  uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
+  if (Enable) {
+#if __riscv_xlen == 64
+    // If the ISA is RV64, the Tracing Hook needs to be typecast to a 64 bit
+    // value.
+    uint32_t LoTracingHookAddr = lo12(reinterpret_cast<uint64_t>(TracingHook));
+    uint32_t HiTracingHookAddr = hi20(reinterpret_cast<uint64_t>(TracingHook));
+    uint32_t HigherTracingHookAddr =
+        lo12((reinterpret_cast<uint64_t>(TracingHook) + 0x80000000) >> 32);
+    uint32_t HighestTracingHookAddr =
+        hi20((reinterpret_cast<uint64_t>(TracingHook) + 0x80000000) >> 32);
+#elif __riscv_xlen == 32
+    // We typecast the Tracing Hook to a 32 bit value for RV32
+    uint32_t LoTracingHookAddr = lo12(reinterpret_cast<uint32_t>(TracingHook));
+    uint32_t HiTracingHookAddr = hi20((reinterpret_cast<uint32_t>(TracingHook));
+#endif
+    uint32_t LoFunctionID = lo12(FuncId);
+    uint32_t HiFunctionID = hi20(FuncId);
+
+    // The sled that is patched in for RISCV64 defined below. We need the entire
+    // sleds corresponding to both ISAs to be protected by defines because the
+    // first few instructions are all different, because we store doubles in
+    // case of RV64 and store words for RV32. Subsequently, we have LUI - and in
+    // case of RV64, we need extra instructions from this point on, so we see
+    // differences in addresses to which instructions are stored.
+    size_t Idx = 1U;
+    const uint32_t XLenBytes = __riscv_xlen / 8;
+#if __riscv_xlen == 64
+    const uint32_t LoadOp = PatchOpcodes::PO_LD;
+    const uint32_t StoreOp = PatchOpcodes::PO_SD;
+#elif __riscv_xlen == 32
+    const uint32_t LoadOp = PatchOpcodes::PO_LW;
+    const uint32_t StoreOp = PatchOpcodes::PO_SW;
+#endif
+
+    Address[Idx++] = encodeSTypeInstruction(StoreOp, RegNum::RN_SP,
+                                            RegNum::RN_RA, 3 * XLenBytes);
+    Address[Idx++] = encodeSTypeInstruction(StoreOp, RegNum::RN_SP,
+                                            RegNum::RN_A0, 2 * XLenBytes);
+
+#if __riscv_xlen == 64
+    Address[Idx++] = encodeSTypeInstruction(StoreOp, RegNum::RN_SP,
+                                            RegNum::RN_T1, XLenBytes);
+    Address[Idx++] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_T1,
+                                            HighestTracingHookAddr);
+    Address[Idx++] =
+        encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_T1,
+                               RegNum::RN_T1, HigherTracingHookAddr);
+    Address[Idx++] = encodeITypeInstruction(PatchOpcodes::PO_SLLI,
+                                            RegNum::RN_T1, RegNum::RN_T1, 32);
+#endif
+    Address[Idx++] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_RA,
+                                            HiTracingHookAddr);
+    Address[Idx++] = encodeITypeInstruction(
+        PatchOpcodes::PO_ADDI, RegNum::RN_RA, RegNum::RN_RA, LoTracingHookAddr);
+#if __riscv_xlen == 64
+    Address[Idx++] = encodeRTypeInstruction(PatchOpcodes::PO_ADD, RegNum::RN_RA,
+                                            RegNum::RN_T1, RegNum::RN_RA);
+#endif
+    Address[Idx++] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_A0,
+                                            HiFunctionID);
+    Address[Idx++] = encodeITypeInstruction(
+        PatchOpcodes::PO_ADDI, RegNum::RN_A0, RegNum::RN_A0, LoFunctionID);
+    Address[Idx++] = encodeITypeInstruction(PatchOpcodes::PO_JALR,
+                                            RegNum::RN_RA, RegNum::RN_RA, 0);
+
+#if __riscv_xlen == 64
+    Address[Idx++] =
+        encodeITypeInstruction(LoadOp, RegNum::RN_SP, RegNum::RN_T1, XLenBytes);
+#endif
+    Address[Idx++] = encodeITypeInstruction(LoadOp, RegNum::RN_SP,
+                                            RegNum::RN_A0, 2 * XLenBytes);
+    Address[Idx++] = encodeITypeInstruction(LoadOp, RegNum::RN_SP,
+                                            RegNum::RN_RA, 3 * XLenBytes);
+    Address[Idx++] = encodeITypeInstruction(
+        PatchOpcodes::PO_ADDI, RegNum::RN_SP, RegNum::RN_SP, 4 * XLenBytes);
+
+    uint32_t CreateStackSpace = encodeITypeInstruction(
+        PatchOpcodes::PO_ADDI, RegNum::RN_SP, RegNum::RN_SP, -4 * XLenBytes);
+
+    std::atomic_store_explicit(
+        reinterpret_cast<std::atomic<uint32_t> *>(Address), CreateStackSpace,
+        std::memory_order_release);
+  } else {
+    uint32_t CreateBranch = encodeJTypeInstruction(
+    // Jump distance is different in both ISAs due to difference in size of
+    // sleds
+#if __riscv_xlen == 64
+        PatchOpcodes::PO_J, RegNum::RN_X0,
+        68); // jump encodes an offset of 68
+#elif __riscv_xlen == 32
+        PatchOpcodes::PO_J, RegNum::RN_X0,
+        44); // jump encodes an offset of 44
+#endif
+    std::atomic_store_explicit(
+        reinterpret_cast<std::atomic<uint32_t> *>(Address), CreateBranch,
+        std::memory_order_release);
+  }
+  return true;
+}
+
+bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
+                        const XRaySledEntry &Sled,
+                        const XRayTrampolines &Trampolines,
+                        bool LogArgs) XRAY_NEVER_INSTRUMENT {
+  // We don't support logging argument at this moment, so we always
+  // use EntryTrampoline.
+  return patchSled(Enable, FuncId, Sled, Trampolines.EntryTrampoline);
+}
+
+bool patchFunctionExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
+}
+
+bool patchFunctionTailExit(
+    const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+    const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampolines.TailExitTrampoline);
+}
+
+bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
+                      const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  return false;
+}
+
+bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
+                     const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  return false;
+}
+} // namespace __xray
+
+extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {}
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv32.S b/compiler-rt/lib/xray/xray_trampoline_riscv32.S
new file mode 100644
index 0000000000000..05e3d61e5ef71
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv32.S
@@ -0,0 +1,89 @@
+//===-- xray_trampoline_riscv32.s ----------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the riscv32-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../sanitizer_common/sanitizer_asm.h"
+
+.macro SAVE_ARG_REGISTERS
+	// Push argument registers to stack
+	addi	sp, sp, -112
+        CFI_DEF_CFA_OFFSET(112)
+	sw	ra, 108(sp)
+	sw	a7, 104(sp)
+	sw	a6, 100(sp)
+	sw	a5, 96(sp)
+	sw	a4, 92(sp)
+	sw	a3, 88(sp)
+	sw	a2, 84(sp)
+	sw	a1, 80(sp)
+	sw	a0, 76(sp)
+	fsd	fa7, 64(sp)
+	fsd	fa6, 56(sp)
+	fsd	fa5, 48(sp)
+	fsd	fa4, 40(sp)
+	fsd	fa3, 32(sp)
+	fsd	fa2, 24(sp)
+	fsd	fa1, 16(sp)
+	fsd	fa0, 8(sp)
+.endm
+
+.macro RESTORE_ARG_REGISTERS
+	// Restore argument registers
+	fld	fa0, 8(sp)
+	fld	fa1, 16(sp)
+	fld	fa2, 24(sp)
+	fld	fa3, 32(sp)
+	fld	fa4, 40(sp)
+	fld	fa5, 48(sp)
+	fld	fa6, 56(sp)
+	fld	fa7, 64(sp)
+	lw	a0, 76(sp)
+	lw	a1, 80(sp)
+	lw	a2, 84(sp)
+	lw	a3, 88(sp)
+	lw	a4, 92(sp)
+	lw	a5, 96(sp)
+	lw	a6, 100(sp)
+	lw	a7, 104(sp)
+	lw	ra, 108(sp)
+	addi	sp, sp, 112
+        CFI_DEF_CFA_OFFSET(0)
+.endm
+
+.macro SAVE_RET_REGISTERS
+	// Push return registers to stack
+	addi	sp, sp, -32
+        CFI_DEF_CFA_OFFSET(32)
+	sw	ra, 28(sp)
+	sw	a1, 24(sp)
+	sw	a0, 20(sp)
+	fsd	fa1, 8(sp)
+	fsd	fa0, 0(sp)
+.endm
+
+.macro RESTORE_RET_REGISTERS
+	// Restore return registers
+	fld	fa0, 0(sp)
+	fld	fa1, 8(sp)
+	lw	a0, 20(sp)
+	lw	a1, 24(sp)
+	lw	ra, 28(sp)
+	addi	sp, sp, 32
+        CFI_DEF_CFA_OFFSET(0)
+.endm
+
+.macro LOAD_XLEN, rd, src
+        lw      \rd, \src
+.endm
+
+#include "xray_trampoline_riscv_common.S"
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv64.S b/compiler-rt/lib/xray/xray_trampoline_riscv64.S
new file mode 100644
index 0000000000000..692350eaaa38e
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv64.S
@@ -0,0 +1,89 @@
+//===-- xray_trampoline_riscv64.s ----------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the riscv64-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../sanitizer_common/sanitizer_asm.h"
+
+.macro SAVE_ARG_REGISTERS
+	// Push return registers to stack
+	addi	sp, sp, -144
+        CFI_DEF_CFA_OFFSET(144)
+        sd	ra, 136(sp)
+        sd	a7, 128(sp)
+        sd	a6, 120(sp)
+        sd	a5, 112(sp)
+        sd	a4, 104(sp)
+        sd	a3, 96(sp)
+        sd	a2, 88(sp)
+        sd	a1, 80(sp)
+        sd	a0, 72(sp)
+        fsd	fa7, 64(sp)
+        fsd	fa6, 56(sp)
+        fsd	fa5, 48(sp)
+        fsd	fa4, 40(sp)
+        fsd	fa3, 32(sp)
+        fsd	fa2, 24(sp)
+        fsd	fa1, 16(sp)
+        fsd	fa0, 8(sp)
+.endm
+
+.macro SAVE_RET_REGISTERS
+	// Push return registers to stack
+        addi    sp, sp, -48
+        CFI_DEF_CFA_OFFSET(48)
+        sd      ra, 40(sp)
+        sd      a1, 32(sp)
+        sd      a0, 24(sp)
+        fsd     fa1, 16(sp)
+        fsd     fa0, 8(sp)
+.endm
+
+.macro RESTORE_RET_REGISTERS
+	// Restore return registers
+        fld     fa0, 8(sp)
+        fld     fa1, 16(sp)
+        ld      a0, 24(sp)
+        ld      a1, 32(sp)
+        ld      ra, 40(sp)
+        addi    sp, sp, 48
+        CFI_DEF_CFA_OFFSET(0)
+.endm
+
+.macro RESTORE_ARG_REGISTERS
+        // Restore argument registers
+	fld	fa0, 8(sp)
+	fld	fa1, 16(sp)
+	fld	fa2, 24(sp)
+	fld	fa3, 32(sp)
+	fld	fa4, 40(sp)
+	fld	fa5, 48(sp)
+	fld	fa6, 56(sp)
+	fld	fa7, 64(sp)
+	ld	a0, 72(sp)
+	ld	a1, 80(sp)
+	ld	a2, 88(sp)
+	ld	a3, 96(sp)
+	ld	a4, 104(sp)
+	ld	a5, 112(sp)
+	ld	a6, 120(sp)
+	ld	a7, 128(sp)
+	ld	ra, 136(sp)
+	addi	sp, sp, 144
+        CFI_DEF_CFA_OFFSET(0)
+.endm
+
+.macro LOAD_XLEN, rd, src
+        ld      \rd, \src
+.endm
+
+#include "xray_trampoline_riscv_common.S"
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv_common.S b/compiler-rt/lib/xray/xray_trampoline_riscv_common.S
new file mode 100644
index 0000000000000..746d612e98204
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv_common.S
@@ -0,0 +1,96 @@
+//===-- xray_trampoline_riscv_common.s --------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the trampolines code shared between riscv32 and riscv64.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../builtins/assembly.h"
+
+	.text
+	.p2align 2
+	.global ASM_SYMBOL(__xray_FunctionEntry)
+	ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+ASM_SYMBOL(__xray_FunctionEntry):
+	CFI_STARTPROC
+        SAVE_ARG_REGISTERS
+
+	// Load the handler function pointer into a2
+	la	a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+        LOAD_XLEN   a2, 0(a2)
+
+	// Handler address will be null if it is not set
+	beq	a2, x0, 1f
+
+	// If we reach here, we are tracing an event
+	// a0 already contains function id
+	// a1 = 0 means we are tracing an entry event
+	li	a1, 0
+	jalr	a2
+
+1:
+        RESTORE_ARG_REGISTERS
+	jr	ra
+	ASM_SIZE(__xray_FunctionEntry)
+	CFI_ENDPROC
+
+	.text
+	.p2align 2
+	.global ASM_SYMBOL(__xray_FunctionExit)
+	ASM_TYPE_FUNCTION(__xray_FunctionExit)
+ASM_SYMBOL(__xray_FunctionExit):
+	CFI_STARTPROC
+	SAVE_RET_REGISTERS
+
+	// Load the handler function pointer into a2
+	la	a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+        LOAD_XLEN   a2, 0(a2)
+
+	// Handler address will be null if it is not set
+	beq	a2, x0, 1f
+
+	// If we reach here, we are tracing an event
+	// a0 already contains function id
+	// a1 = 1 means we are tracing an exit event
+	li	a1, 1
+	jalr	a2
+
+1:
+	RESTORE_RET_REGISTERS
+        jr	ra
+	ASM_SIZE(__xray_FunctionExit)
+	CFI_ENDPROC
+
+	.text
+	.p2align 2
+	.global ASM_SYMBOL(__xray_FunctionTailExit)
+	ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+ASM_SYMBOL(__xray_FunctionTailExit):
+	CFI_STARTPROC
+        SAVE_ARG_REGISTERS
+
+	// Load the handler function pointer into a2
+	la	a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+        LOAD_XLEN   a2, 0(a2)
+
+	// Handler address will be null if it is not set
+	beq	a2, x0, 1f
+
+	// If we reach here, we are tracing an event
+	// a0 already contains function id
+	// a1 = 2 means we are tracing a tail exit event
+	li	a1, 2
+	jalr	a2
+
+1:
+        RESTORE_ARG_REGISTERS
+	jr	ra
+	ASM_SIZE(__xray_FunctionTailExit)
+	CFI_ENDPROC
diff --git a/compiler-rt/lib/xray/xray_tsc.h b/compiler-rt/lib/xray/xray_tsc.h
index e1cafe1bf11d2..b62a686d6ce0f 100644
--- a/compiler-rt/lib/xray/xray_tsc.h
+++ b/compiler-rt/lib/xray/xray_tsc.h
@@ -43,7 +43,7 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
 #elif defined(__powerpc64__)
 #include "xray_powerpc64.inc"
 #elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) ||         \
-    defined(__hexagon__) || defined(__loongarch_lp64)
+    defined(__hexagon__) || defined(__loongarch_lp64) || defined(__riscv)
 // Emulated TSC.
 // There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
 //   not have a constant frequency like TSC on x86(_64), it may go faster
diff --git a/compiler-rt/test/asan/TestCases/Linux/global-overflow-bfd.cpp b/compiler-rt/test/asan/TestCases/Linux/global-overflow-bfd.cpp
index bf82a517df37e..67331606c3ba0 100644
--- a/compiler-rt/test/asan/TestCases/Linux/global-overflow-bfd.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/global-overflow-bfd.cpp
@@ -2,6 +2,9 @@
 // false negatives with the BFD linker.
 // RUN: %clangxx_asan -fuse-ld=bfd -Wl,-gc-sections -ffunction-sections -fdata-sections -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+// Android does not use bfd.
+// UNSUPPORTED: android
+
 #include <string.h>
 int main(int argc, char **argv) {
   static char XXX[10];
diff --git a/compiler-rt/test/builtins/Unit/atomic_test.c b/compiler-rt/test/builtins/Unit/atomic_test.c
index f8281f3649379..346c9b3be55a7 100644
--- a/compiler-rt/test/builtins/Unit/atomic_test.c
+++ b/compiler-rt/test/builtins/Unit/atomic_test.c
@@ -144,7 +144,7 @@ typedef uint64_t maxuint_t;
 
 #define LEN(array) (sizeof(array) / sizeof(array[0]))
 
-__attribute__((aligned(16))) static const char data[] = {
+__attribute__((aligned(16))) static char data[] = {
     0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
     0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
     0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
diff --git a/compiler-rt/test/builtins/Unit/extendhfxf2_test.c b/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
index bbe40a433c008..d7227a9f1d156 100644
--- a/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
+++ b/compiler-rt/test/builtins/Unit/extendhfxf2_test.c
@@ -7,59 +7,58 @@
 
 #include "fp_test.h"
 
-#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
-    defined(COMPILER_RT_HAS_FLOAT16)
+#if HAS_80_BIT_LONG_DOUBLE
 
 xf_float __extendhfxf2(TYPE_FP16 f);
 
-int test_extendhfxf2(TYPE_FP16 a, uint64_t expectedHi, uint64_t expectedLo) {
+int test_extendhfxf2(TYPE_FP16 a, uint16_t expectedHi, uint64_t expectedLo) {
   xf_float x = __extendhfxf2(a);
   int ret = compareResultF80(x, expectedHi, expectedLo);
   if (ret) {
     printf("error in test__extendhfxf2(%#.4x) = %.20Lf, "
            "expected %.20Lf\n",
-           toRep16(a), x, F80FromRep128(expectedHi, expectedLo));
+           toRep16(a), x, F80FromRep80(expectedHi, expectedLo));
   }
   return ret;
 }
 
 int main() {
   // Small positive value
-  if (test_extendhfxf2(fromRep16(0x2e66), UINT64_C(0x3ffb),
+  if (test_extendhfxf2(fromRep16(0x2e66), UINT16_C(0x3ffb),
                        UINT64_C(0xccc0000000000000)))
     return 1;
 
   // Small negative value
-  if (test_extendhfxf2(fromRep16(0xae66), UINT64_C(0xbffb),
+  if (test_extendhfxf2(fromRep16(0xae66), UINT16_C(0xbffb),
                        UINT64_C(0xccc0000000000000)))
     return 1;
 
   // Zero
-  if (test_extendhfxf2(fromRep16(0), UINT64_C(0x0), UINT64_C(0x0)))
+  if (test_extendhfxf2(fromRep16(0), UINT16_C(0x0), UINT64_C(0x0)))
     return 1;
 
   // Smallest positive non-zero value
-  if (test_extendhfxf2(fromRep16(0x0100), UINT64_C(0x3fef),
+  if (test_extendhfxf2(fromRep16(0x0100), UINT16_C(0x3fef),
                        UINT64_C(0x8000000000000000)))
     return 1;
 
   // Smallest negative non-zero value
-  if (test_extendhfxf2(fromRep16(0x8100), UINT64_C(0xbfef),
+  if (test_extendhfxf2(fromRep16(0x8100), UINT16_C(0xbfef),
                        UINT64_C(0x8000000000000000)))
     return 1;
 
   // Positive infinity
-  if (test_extendhfxf2(makeInf16(), UINT64_C(0x7fff),
+  if (test_extendhfxf2(makeInf16(), UINT16_C(0x7fff),
                        UINT64_C(0x8000000000000000)))
     return 1;
 
   // Negative infinity
-  if (test_extendhfxf2(makeNegativeInf16(), UINT64_C(0xffff),
+  if (test_extendhfxf2(makeNegativeInf16(), UINT16_C(0xffff),
                        UINT64_C(0x8000000000000000)))
     return 1;
 
   // NaN
-  if (test_extendhfxf2(makeQNaN16(), UINT64_C(0x7fff),
+  if (test_extendhfxf2(makeQNaN16(), UINT16_C(0x7fff),
                        UINT64_C(0xc000000000000000)))
     return 1;
 
diff --git a/compiler-rt/test/builtins/Unit/fp_test.h b/compiler-rt/test/builtins/Unit/fp_test.h
index b23e2ef7f3a16..3a8968a966044 100644
--- a/compiler-rt/test/builtins/Unit/fp_test.h
+++ b/compiler-rt/test/builtins/Unit/fp_test.h
@@ -1,7 +1,8 @@
-#include <stdlib.h>
+#include <assert.h>
 #include <limits.h>
-#include <string.h>
 #include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
 
 #include "int_types.h"
 
@@ -230,44 +231,49 @@ static inline double makeQNaN64(void)
     return fromRep64(0x7ff8000000000000UL);
 }
 
-#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__)
-static inline long double F80FromRep128(uint64_t hi, uint64_t lo) {
-    __uint128_t x = ((__uint128_t)hi << 64) + lo;
-    long double ret;
-    memcpy(&ret, &x, 16);
-    return ret;
+#if HAS_80_BIT_LONG_DOUBLE
+static inline xf_float F80FromRep80(uint16_t hi, uint64_t lo) {
+  uqwords bits;
+  bits.high.all = hi;
+  bits.low.all = lo;
+  xf_float ret;
+  static_assert(sizeof(xf_float) <= sizeof(uqwords), "wrong representation");
+  memcpy(&ret, &bits, sizeof(ret));
+  return ret;
 }
 
-static inline __uint128_t F80ToRep128(long double x) {
-    __uint128_t ret;
-    memcpy(&ret, &x, 16);
-    return ret;
+static inline uqwords F80ToRep80(xf_float x) {
+  uqwords ret;
+  memset(&ret, 0, sizeof(ret));
+  memcpy(&ret, &x, sizeof(x));
+  // Any bits beyond the first 16 in high are undefined.
+  ret.high.all = (uint16_t)ret.high.all;
+  return ret;
 }
 
-static inline int compareResultF80(long double result, uint64_t expectedHi,
+static inline int compareResultF80(xf_float result, uint16_t expectedHi,
                                    uint64_t expectedLo) {
-    __uint128_t rep = F80ToRep128(result);
-    // F80 occupies the lower 80 bits of __uint128_t.
-    uint64_t hi = (rep >> 64) & ((1UL << (80 - 64)) - 1);
-    uint64_t lo = rep;
-    return !(hi == expectedHi && lo == expectedLo);
+  uqwords rep = F80ToRep80(result);
+  // F80 high occupies the lower 16 bits of high.
+  assert((uint64_t)(uint16_t)rep.high.all == rep.high.all);
+  return !(rep.high.all == expectedHi && rep.low.all == expectedLo);
 }
 
-static inline long double makeQNaN80(void) {
-    return F80FromRep128(0x7fffUL, 0xc000000000000000UL);
+static inline xf_float makeQNaN80(void) {
+  return F80FromRep80(0x7fffu, 0xc000000000000000UL);
 }
 
-static inline long double makeNaN80(uint64_t rand) {
-    return F80FromRep128(0x7fffUL,
-                         0x8000000000000000 | (rand & 0x3fffffffffffffff));
+static inline xf_float makeNaN80(uint64_t rand) {
+  return F80FromRep80(0x7fffu,
+                      0x8000000000000000 | (rand & 0x3fffffffffffffff));
 }
 
-static inline long double makeInf80(void) {
-    return F80FromRep128(0x7fffUL, 0x8000000000000000UL);
+static inline xf_float makeInf80(void) {
+  return F80FromRep80(0x7fffu, 0x8000000000000000UL);
 }
 
-static inline long double makeNegativeInf80(void) {
-  return F80FromRep128(0xffffUL, 0x8000000000000000UL);
+static inline xf_float makeNegativeInf80(void) {
+  return F80FromRep80(0xffffu, 0x8000000000000000UL);
 }
 #endif
 
diff --git a/compiler-rt/test/memprof/TestCases/default_options.cpp b/compiler-rt/test/memprof/TestCases/default_options.cpp
index 1b6b61fc048b6..4042a7df588b0 100644
--- a/compiler-rt/test/memprof/TestCases/default_options.cpp
+++ b/compiler-rt/test/memprof/TestCases/default_options.cpp
@@ -1,5 +1,8 @@
 // RUN: %clangxx_memprof -O2 %s -o %t && %run %t 2>&1 | FileCheck %s
 
+// Check that overriding __memprof_default_options() takes precedence over the LLVM flag
+// RUN: %clangxx_memprof -O2 %s -o %t-flag -mllvm -memprof-runtime-default-options="verbosity=0 help=0" && %run %t-flag 2>&1 | FileCheck %s
+
 const char *kMemProfDefaultOptions = "verbosity=1 help=1";
 
 extern "C" const char *__memprof_default_options() {
diff --git a/compiler-rt/test/memprof/TestCases/dump_at_exit.cpp b/compiler-rt/test/memprof/TestCases/dump_at_exit.cpp
new file mode 100644
index 0000000000000..426849d1cea01
--- /dev/null
+++ b/compiler-rt/test/memprof/TestCases/dump_at_exit.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx_memprof %s -o %t
+
+// RUN: %env_memprof_opts=print_text=true:log_path=stdout:dump_at_exit=false %run %t | count 0
+// RUN: %env_memprof_opts=print_text=true:log_path=stdout:dump_at_exit=true %run %t | FileCheck %s
+
+#include <stdlib.h>
+#include <string.h>
+
+int main() {
+  char *x = (char *)malloc(10);
+  memset(x, 0, 10);
+  free(x);
+  return 0;
+}
+
+// CHECK: Recorded MIBs
diff --git a/compiler-rt/test/memprof/TestCases/set_options.cpp b/compiler-rt/test/memprof/TestCases/set_options.cpp
new file mode 100644
index 0000000000000..dddcfcf98c5c0
--- /dev/null
+++ b/compiler-rt/test/memprof/TestCases/set_options.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx_memprof %s -o %t-default
+// RUN: %run %t-default | FileCheck %s --check-prefix=DEFAULT
+
+// RUN: %clangxx_memprof %s -mllvm -memprof-runtime-default-options="print_text=true,log_path=stdout,atexit=false" -o %t
+// RUN: %run %t | FileCheck %s
+
+#include <sanitizer/memprof_interface.h>
+#include <stdio.h>
+
+int main() {
+  printf("Options: \"%s\"\n", __memprof_default_options());
+  return 0;
+}
+
+// DEFAULT: Options: ""
+// CHECK: Options: "print_text=true,log_path=stdout,atexit=false"
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/dn_expand.cpp b/compiler-rt/test/msan/Linux/dn_expand.cpp
similarity index 79%
rename from compiler-rt/test/sanitizer_common/TestCases/Linux/dn_expand.cpp
rename to compiler-rt/test/msan/Linux/dn_expand.cpp
index 8fdfa7a6cc1a4..a37da83df0022 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/dn_expand.cpp
+++ b/compiler-rt/test/msan/Linux/dn_expand.cpp
@@ -1,10 +1,10 @@
-// RUN: %clangxx %s -o %t && %run %t %p
+// RUN: %clangxx_msan %s -o %t && %run %t %p
 
 #include <assert.h>
 #include <resolv.h>
 #include <string.h>
 
-#include "sanitizer_common/sanitizer_specific.h"
+#include <sanitizer/msan_interface.h>
 
 void testWrite() {
   char unsigned input[] = {0xff, 0xc5, 0xf7, 0xff, 0x00, 0x00, 0xff, 0x0a, 0x00,
@@ -18,7 +18,7 @@ void testWrite() {
 
   assert(res == 12);
   assert(strcmp(output, "google\\.com") == 0);
-  check_mem_is_good(output, strlen(output) + 1);
+  __msan_check_mem_is_initialized(output, strlen(output) + 1);
 }
 
 void testWriteZeroLength() {
@@ -33,7 +33,7 @@ void testWriteZeroLength() {
 
   assert(res == 1);
   assert(strcmp(output, "") == 0);
-  check_mem_is_good(output, strlen(output) + 1);
+  __msan_check_mem_is_initialized(output, strlen(output) + 1);
 }
 
 void testComp() {
@@ -49,17 +49,17 @@ void testComp() {
   char unsigned *n1 = mb;
   int res = dn_comp("llvm.org", mb, me - mb, pb, pe);
   assert(res == 10);
-  check_mem_is_good(mb, res);
+  __msan_check_mem_is_initialized(mb, res);
   // pb is [msg, llvm.org, nullptr]
-  check_mem_is_good(pb, sizeof(*pb) * 3);
+  __msan_check_mem_is_initialized(pb, sizeof(*pb) * 3);
   mb += res;
 
   char unsigned *n2 = mb;
   res = dn_comp("lab.llvm.org", mb, me - mb, pb, pe);
   assert(res == 6);
-  check_mem_is_good(mb, res);
+  __msan_check_mem_is_initialized(mb, res);
   // pb is [msg, llvm.org, lab.llvm.org, nullptr]
-  check_mem_is_good(pb, sizeof(*pb) * 4);
+  __msan_check_mem_is_initialized(pb, sizeof(*pb) * 4);
   mb += res;
 
   {
@@ -69,7 +69,7 @@ void testComp() {
     fprintf(stderr, "%d\n", res);
     assert(res == 10);
     assert(strcmp(output, "llvm.org") == 0);
-    check_mem_is_good(output, strlen(output) + 1);
+    __msan_check_mem_is_initialized(output, strlen(output) + 1);
   }
 
   {
@@ -78,7 +78,7 @@ void testComp() {
 
     assert(res == 6);
     assert(strcmp(output, "lab.llvm.org") == 0);
-    check_mem_is_good(output, strlen(output) + 1);
+    __msan_check_mem_is_initialized(output, strlen(output) + 1);
   }
 }
 
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/Inputs/foo-ret-42.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/foo-ret-42.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/JITLink/Generic/Inputs/foo-ret-42.ll
rename to compiler-rt/test/orc/TestCases/Generic/Inputs/foo-ret-42.ll
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/Inputs/var-x-42.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/var-x-42.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/JITLink/Generic/Inputs/var-x-42.ll
rename to compiler-rt/test/orc/TestCases/Generic/Inputs/var-x-42.ll
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
similarity index 70%
rename from llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll
rename to compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
index 72325e198721b..e722b813c3fe0 100644
--- a/llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll
+++ b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
@@ -4,13 +4,14 @@
 ; referenced by main, should be linked (despite being passed with -lazy).
 ;
 ; RUN: rm -rf %t && mkdir -p %t
-; RUN: llc -filetype=obj -o %t/foo.o %S/Inputs/foo-ret-42.ll
-; RUN: llc -filetype=obj -o %t/x.o %S/Inputs/var-x-42.ll
-; RUN: llc -filetype=obj -o %t/main.o %s
-; RUN: llvm-jitlink -noexec -show-linked-files %t/main.o -lazy %t/foo.o \
+; RUN: %clang -c -o %t/foo.o %S/Inputs/foo-ret-42.ll
+; RUN: %clang -c -o %t/x.o %S/Inputs/var-x-42.ll
+; RUN: %clang -c -o %t/main.o %s
+; RUN: %llvm_jitlink -noexec -show-linked-files %t/main.o -lazy %t/foo.o \
 ; RUN:     -lazy %t/x.o | FileCheck %s
 ;
-; UNSUPPORTED: system-windows, target={{arm[^6][^4].*}}, target=powerpc64{{.*}}
+; UNSUPPORTED: system-windows
+; REQUIRES: target={{(arm|aarch)64.*}}
 ;
 ; CHECK: Linking {{.*}}main.o
 ; CHECK-DAG: Linking <indirect stubs graph #1>
diff --git a/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test b/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test
deleted file mode 100644
index 951688d7961d4..0000000000000
--- a/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test
+++ /dev/null
@@ -1,6 +0,0 @@
-// Test that the orc-remote-executor tool errors out as expected when called
-// with no arguments.
-//
-// RUN: not %orc_rt_executor 2>&1 | FileCheck %s
-
-// CHECK: usage: orc-rt-executor [help] [<mode>] <program arguments>...
diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py
index 6dfa94b11cc9d..7a6eb4e7de325 100644
--- a/compiler-rt/test/orc/lit.cfg.py
+++ b/compiler-rt/test/orc/lit.cfg.py
@@ -16,6 +16,11 @@
     host_arch_compatible = True
 if host_arch_compatible:
     config.available_features.add("host-arch-compatible")
+
+# If the target OS hasn't been set then assume host.
+if not config.target_os:
+    config.target_os = config.host_os
+
 config.test_target_is_host_executable = (
     config.target_os == config.host_os and host_arch_compatible
 )
diff --git a/compiler-rt/test/rtsan/Darwin/dlopen.cpp b/compiler-rt/test/rtsan/Darwin/dlopen.cpp
new file mode 100644
index 0000000000000..1aabe5cb6e580
--- /dev/null
+++ b/compiler-rt/test/rtsan/Darwin/dlopen.cpp
@@ -0,0 +1,44 @@
+// Checks that on OS X 10.11+ dlopen'ing a RTsanified library from a
+// non-instrumented program exits with a user-friendly message.
+
+// REQUIRES: osx-autointerception
+
+// XFAIL: ios
+
+// RUN: %clangxx -fsanitize=realtime %s -o %t.so -shared -DSHARED_LIB
+// RUN: %clangxx %s -o %t
+
+// RUN: RTSAN_DYLIB_PATH=`%clangxx -fsanitize=realtime %s -### 2>&1 \
+// RUN:   | grep "libclang_rt.rtsan_osx_dynamic.dylib" \
+// RUN:   | sed -e 's/.*"\(.*libclang_rt.rtsan_osx_dynamic.dylib\)".*/\1/'`
+
+// Launching a non-instrumented binary that dlopen's an instrumented library should fail.
+// RUN: not %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-FAIL
+// Launching a non-instrumented binary with an explicit DYLD_INSERT_LIBRARIES should work.
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s
+
+// Launching an instrumented binary with the DYLD_INSERT_LIBRARIES env variable has no error
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-INSTRUMENTED
+
+#include <dlfcn.h>
+#include <stdio.h>
+
+#if defined(SHARED_LIB)
+extern "C" void foo() { fprintf(stderr, "Hello world.\n"); }
+#else  // defined(SHARED_LIB)
+int main(int argc, char *argv[]) {
+  void *handle = dlopen(argv[1], RTLD_NOW);
+  void (*foo)() = (void (*)())dlsym(handle, "foo");
+  foo();
+}
+#endif // defined(SHARED_LIB)
+
+// CHECK: Hello world.
+// CHECK-NOT: ERROR: Interceptors are not working.
+
+// CHECK-FAIL-NOT: Hello world.
+// CHECK-FAIL: ERROR: Interceptors are not working.
+
+// CHECK-INSTRUMENTED-NOT: ERROR: Interceptors are not working
+// CHECK-INSTRUMENTED: Hello world.
diff --git a/compiler-rt/test/rtsan/fork_exec.cpp b/compiler-rt/test/rtsan/fork_exec.cpp
new file mode 100644
index 0000000000000..3b2d2e5ca2f5d
--- /dev/null
+++ b/compiler-rt/test/rtsan/fork_exec.cpp
@@ -0,0 +1,59 @@
+// RUN: %clangxx -fsanitize=realtime -DIS_NONBLOCKING=1 %s -o %t
+// RUN: %env_rtsan_opts="halt_on_error=true" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-HALT
+// RUN: %env_rtsan_opts="halt_on_error=false" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NOHALT
+
+// RUN: %clangxx -fsanitize=realtime -DIS_NONBLOCKING=0 %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-OK
+// RUN: %env_rtsan_opts="halt_on_error=false" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-OK
+
+// UNSUPPORTED: ios
+
+// Intent: Ensure fork/exec dies when realtime and survives otherwise
+//         This behavior is difficult to test in a gtest, because the process is
+//         wiped away with exec.
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#if IS_NONBLOCKING
+#  define MAYBE_NONBLOCKING [[clang::nonblocking]]
+#else
+#  define MAYBE_NONBLOCKING
+#endif
+
+int main() MAYBE_NONBLOCKING {
+  const pid_t pid = fork();
+
+  if (pid == 0) {
+    char *args[] = {"/bin/ls", nullptr};
+    execve(args[0], args, nullptr);
+    perror("execve failed");
+    return 1;
+  } else if (pid > 0) {
+    int status;
+    waitpid(pid, &status, 0);
+    usleep(1);
+  } else {
+    perror("fork failed");
+    return 1;
+  }
+
+  printf("fork/exec succeeded\n");
+  return 0;
+}
+
+// CHECK-NOHALT: Intercepted call to {{.*}} `fork` {{.*}}
+// CHECK-NOHALT: Intercepted call to {{.*}} `execve` {{.*}}
+
+// usleep checks that rtsan is still enabled in the parent process
+// See note in our interceptors file for why we don't look for `wait`
+// CHECK-NOHALT: Intercepted call to {{.*}} `usleep` {{.*}}
+
+// CHECK-NOHALT: fork/exec succeeded
+
+// CHECK-HALT: ==ERROR: RealtimeSanitizer: unsafe-library-call
+// CHECK-HALT-NEXT: Intercepted call to {{.*}} `fork` {{.*}}
+
+// CHECK-OK: fork/exec succeeded
diff --git a/compiler-rt/test/rtsan/syscall.cpp b/compiler-rt/test/rtsan/syscall.cpp
new file mode 100644
index 0000000000000..a23a934184a69
--- /dev/null
+++ b/compiler-rt/test/rtsan/syscall.cpp
@@ -0,0 +1,78 @@
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: %env_rtsan_opts="halt_on_error=false" %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-RTSAN,CHECK
+
+// RUN: %clangxx %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: ios
+
+// Intent: Ensure the `syscall` call behaves in the same way with/without the
+//         sanitizer disabled
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+const char *GetTemporaryFilePath() { return "/tmp/rtsan_syscall_test.txt"; }
+
+void custom_assert(bool condition, const char *message) {
+  if (!condition) {
+    fprintf(stderr, "ASSERTION FAILED: %s\n", message);
+    exit(1);
+  }
+}
+
+class ScopedFileCleanup {
+public:
+  [[nodiscard]] ScopedFileCleanup() = default;
+  ~ScopedFileCleanup() {
+    if (access(GetTemporaryFilePath(), F_OK) != -1)
+      unlink(GetTemporaryFilePath());
+  }
+};
+
+// Apple has deprecated `syscall`, ignore that error
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+int main() [[clang::nonblocking]] {
+  ScopedFileCleanup cleanup;
+
+  {
+    int fd = syscall(SYS_openat, AT_FDCWD, GetTemporaryFilePath(),
+                     O_CREAT | O_WRONLY, 0644);
+
+    custom_assert(fd != -1, "Failed to open file - write");
+
+    int written = syscall(SYS_write, fd, "Hello, world!", 13);
+    custom_assert(written == 13, "Failed to write to file");
+
+    custom_assert(syscall(SYS_close, fd) == 0, "Failed to close file - write");
+  }
+
+  {
+    int fd = syscall(SYS_openat, AT_FDCWD, GetTemporaryFilePath(), O_RDONLY);
+    custom_assert(fd != -1, "Failed to open file - read");
+
+    char buffer[13];
+    int read = syscall(SYS_read, fd, buffer, 13);
+    custom_assert(read == 13, "Failed to read from file");
+
+    custom_assert(memcmp(buffer, "Hello, world!", 13) == 0,
+                  "Read data does not match written data");
+
+    custom_assert(syscall(SYS_close, fd) == 0, "Failed to close file - read");
+  }
+
+  unlink(GetTemporaryFilePath());
+  printf("DONE\n");
+}
+#pragma clang diagnostic pop
+
+// CHECK-NOT: ASSERTION FAILED
+// CHECK-RTSAN-COUNT-6: Intercepted call to real-time unsafe function `syscall`
+
+// CHECK: DONE
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/b64.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/b64.cpp
deleted file mode 100644
index 96fd0138648b3..0000000000000
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/b64.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-// RUN: %clangxx %s -o %t && %run %t %p
-
-#include <assert.h>
-#include <resolv.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-
-int main(int iArgc, char *szArgv[]) {
-  // Check NTOP writing
-  const char *src = "base64 test data";
-  size_t src_len = strlen(src);
-  size_t dst_len = ((src_len + 2) / 3) * 4 + 1;
-  char dst[dst_len];
-  int res = b64_ntop(reinterpret_cast<const unsigned char *>(src), src_len, dst,
-                     dst_len);
-  assert(res >= 0);
-  assert(strcmp(dst, "YmFzZTY0IHRlc3QgZGF0YQ==") == 0);
-
-  // Check PTON writing
-  unsigned char target[dst_len];
-  res = b64_pton(dst, target, dst_len);
-  assert(res >= 0);
-  assert(strncmp(reinterpret_cast<const char *>(target), src, res) == 0);
-
-  // Check NTOP writing for zero length src
-  src = "";
-  src_len = strlen(src);
-  assert(((src_len + 2) / 3) * 4 + 1 < dst_len);
-  res = b64_ntop(reinterpret_cast<const unsigned char *>(src), src_len, dst,
-                 dst_len);
-  assert(res >= 0);
-  assert(strcmp(dst, "") == 0);
-
-  // Check PTON writing for zero length src
-  dst[0] = '\0';
-  res = b64_pton(dst, target, dst_len);
-  assert(res >= 0);
-  assert(strncmp(reinterpret_cast<const char *>(target), src, res) == 0);
-
-  return 0;
-}
diff --git a/compiler-rt/test/xray/TestCases/Posix/default-options.cpp b/compiler-rt/test/xray/TestCases/Posix/default-options.cpp
new file mode 100644
index 0000000000000..e00ff3ba0a5cb
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/default-options.cpp
@@ -0,0 +1,16 @@
+// RUN: rm -fr %t && mkdir %t && cd %t
+// RUN: %clang_xray %s -o a.out
+// RUN: %run %t/a.out 2>&1 | FileCheck %s
+
+// REQUIRES: built-in-llvm-tree
+
+extern "C" __attribute__((xray_never_instrument)) const char *
+__xray_default_options() {
+  return "patch_premain=true:verbosity=1:xray_mode=xray-basic";
+}
+
+__attribute__((xray_always_instrument)) void always() {}
+
+int main() { always(); }
+
+// CHECK: =={{[0-9].*}}==XRay: Log file in '{{.*}}'
diff --git a/flang/Maintainers.md b/flang/Maintainers.md
new file mode 100644
index 0000000000000..f4a7635389138
--- /dev/null
+++ b/flang/Maintainers.md
@@ -0,0 +1,95 @@
+# Flang maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+Flang
+
+## Active maintainers
+
+### Lead maintainers
+The following people are the active maintainers for the project. Please reach
+out to them for code reviews, questions about their area of expertise, or other
+assistance.
+
+#### All areas not covered by others
+Steve Scalpone \
+sscalpone@nvidia.com (email), sscalpone (GitHub), sscalpone (Discourse)
+
+####  Backend (Lowering, Fortran dialects, Codegen)
+Jean Perier \
+jperier@nvidia.com (email), jeanPerier (GitHub), jeanPerier (Discourse)
+
+### Component maintainers
+These maintainers are responsible for particular high-level components within
+Flang that are typically contained to one area of the compiler.
+
+#### Driver
+Andrzej Warzyński \
+andrzej.warzynski@arm.com (email), banach-space (GitHub), banach-space (Discourse)
+
+Tarun Prabhu \
+tarun@lanl.gov (email), tarunprabhu (GitHub), tarunprabhu (Discourse)
+
+#### Alias Analysis
+Renaud Kauffmann \
+rkauffmann@nvidia.com (email), Renaud-K (GitHub), Renaud-K (Discourse)
+
+#### PFT, Block, IEEE support
+Val Donaldson \
+vdonaldson@nvidia.com (email), vdonaldson (GitHub)
+
+#### Lowering, HLFIR, FIR, Codegen
+Jean Perier \
+jperier@nvidia.com (email), jeanPerier (GitHub), jeanPerier (Discourse)
+
+Slava Zakharin \
+szakharin@nvidia.com (email), vzakhari (GitHub), szakharin (Discourse)
+
+Tom Eccles \
+tom.eccles@arm.com (email), tblah (GitHub), tblah (Discourse)
+
+Valentin Clement \
+clementval@gmail.com (email), clementval (GitHub), clementval (Discourse)
+
+#### OpenMP
+##### CPU support
+Tom Eccles \
+tom.eccles@arm.com (email), tblah (GitHub), tblah (Discourse)
+
+##### OpenMP 6.0
+Krzysztof Parzyszek \
+Krzysztof.Parzyszek@amd.com (email), kparzysz (GitHub), kparzysz (Discourse)
+
+##### Target Offload
+Sergio Afonso \
+safonsof@amd.com (email), skatrak (GitHub), skatrak (Discourse) 
+
+#### OpenACC
+Valentin Clement \
+clementval@gmail.com (email), clementval (GitHub), clementval (Discourse)
+
+Razvan Lupusoru \
+razvan.lupusoru@gmail.com (email), razvanlupusoru (GitHub), razvan.lupusoru (Discourse)
+
+#### CUDA Fortran
+Valentin Clement \
+clementval@gmail.com (email), clementval (GitHub), clementval (Discourse)
+
+#### Debug
+Abid Qadeer \
+haqadeer@amd.com (email), abidh (GitHub), abidh (Discourse)
+
+## Inactive Maintainers
+### Lead Maintainers
+#### Backend : (Lowering, FIR, Codegen)
+Eric Schweitz \
+eschweitz@nvidia.com (email), schweitzpgi (GitHub), schweitz (Discourse)
+
+### Component Maintainers
+#### Semantics
+Tim Keith
+
+#### OpenMP
+Kiran Chandramohan \
+kiran.chandramohan@arm.com (email), kiranchandramohan (GitHub),  kiranchandramohan (Discourse)
+
diff --git a/flang/Maintainers.txt b/flang/Maintainers.txt
deleted file mode 100644
index a8e2fe07a9853..0000000000000
--- a/flang/Maintainers.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-This file is a list of the people responsible for ensuring that patches for a
-particular part of Flang are reviewed, either by themself or by someone else.
-They are also the gatekeepers for their part of Flang, with the final word on
-what goes in or not.
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts. The fields are: name (N), email (E), web-address
-(W), PGP key ID and fingerprint (P), description (D), snail-mail address
-(S) and (I) IRC handle. Each entry should contain at least the (N), (E) and
-(D) fields.
-
-N: Steve Scalpone
-E: sscalpone@nvidia.com
-D: Anything not covered by others
-
-N: Andrzej Warzynski
-E: andrzej.warzynski@arm.com
-D: Driver
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index f85a3eb39ed19..4b4b516d0fb69 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -391,6 +391,9 @@ end
   has the SAVE attribute and was initialized.
 * `PRINT namelistname` is accepted and interpreted as
   `WRITE(*,NML=namelistname)`, a near-universal extension.
+* A character length specifier in a component or entity declaration
+  is accepted before an array specification (`ch*3(2)`) as well
+  as afterwards.
 
 ### Extensions supported when enabled by options
 
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 94bca7a3972b6..d0b7999fbd067 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -705,6 +705,7 @@ MALLOC, FREE
 
 ### Library subroutine 
 ```
+CALL BACKTRACE()
 CALL FDATE(TIME)
 CALL GETLOG(USRNAME)
 CALL GETENV(NAME [, VALUE, LENGTH, STATUS, TRIM_NAME, ERRMSG ])
@@ -769,7 +770,7 @@ This phase currently supports all the intrinsic procedures listed above but the
 | Intrinsic subroutines |MVBITS (elemental), CPU_TIME, DATE_AND_TIME, EVENT_QUERY, EXECUTE_COMMAND_LINE, GET_COMMAND, GET_COMMAND_ARGUMENT, GET_ENVIRONMENT_VARIABLE, MOVE_ALLOC, RANDOM_INIT, RANDOM_NUMBER, RANDOM_SEED, SIGNAL, SLEEP, SYSTEM, SYSTEM_CLOCK |
 | Atomic intrinsic subroutines | ATOMIC_ADD |
 | Collective intrinsic subroutines | CO_REDUCE |
-| Library subroutines | FDATE, GETLOG, GETENV |
+| Library subroutines | BACKTRACE, FDATE, GETLOG, GETENV |
 
 
 ### Intrinsic Function Folding
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index 6ae92acf20608..41a6255207976 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -465,7 +465,7 @@ struct NodeVisitor {
   READ_FEATURE(OmpDeclareTargetWithClause)
   READ_FEATURE(OmpDeclareTargetWithList)
   READ_FEATURE(OmpDefaultClause)
-  READ_FEATURE(OmpDefaultClause::Type)
+  READ_FEATURE(OmpDefaultClause::DataSharingAttribute)
   READ_FEATURE(OmpDefaultmapClause)
   READ_FEATURE(OmpDefaultmapClause::ImplicitBehavior)
   READ_FEATURE(OmpVariableCategory::Value)
@@ -488,9 +488,12 @@ struct NodeVisitor {
   READ_FEATURE(OmpEndLoopDirective)
   READ_FEATURE(OmpEndSectionsDirective)
   READ_FEATURE(OmpGrainsizeClause)
-  READ_FEATURE(OmpGrainsizeClause::Prescriptiveness)
+  READ_FEATURE(OmpGrainsizeClause::Modifier)
+  READ_FEATURE(OmpPrescriptiveness)
+  READ_FEATURE(OmpPrescriptiveness::Value)
   READ_FEATURE(OmpIfClause)
-  READ_FEATURE(OmpIfClause::DirectiveNameModifier)
+  READ_FEATURE(OmpIfClause::Modifier)
+  READ_FEATURE(OmpDirectiveNameModifier)
   READ_FEATURE(OmpLinearClause)
   READ_FEATURE(OmpLinearClause::WithModifier)
   READ_FEATURE(OmpLinearClause::WithoutModifier)
@@ -500,7 +503,7 @@ struct NodeVisitor {
   READ_FEATURE(OmpMapClause)
   READ_FEATURE(OmpMapClause::Modifier)
   READ_FEATURE(OmpNumTasksClause)
-  READ_FEATURE(OmpNumTasksClause::Prescriptiveness)
+  READ_FEATURE(OmpNumTasksClause::Modifier)
   READ_FEATURE(OmpObject)
   READ_FEATURE(OmpObjectList)
   READ_FEATURE(OmpOrderClause)
@@ -508,7 +511,7 @@ struct NodeVisitor {
   READ_FEATURE(OmpOrderModifier)
   READ_FEATURE(OmpOrderModifier::Value)
   READ_FEATURE(OmpProcBindClause)
-  READ_FEATURE(OmpProcBindClause::Type)
+  READ_FEATURE(OmpProcBindClause::AffinityPolicy)
   READ_FEATURE(OmpReductionClause)
   READ_FEATURE(OmpInReductionClause)
   READ_FEATURE(OmpReductionCombiner)
@@ -516,17 +519,18 @@ struct NodeVisitor {
   READ_FEATURE(OmpReductionInitializerClause)
   READ_FEATURE(OmpReductionIdentifier)
   READ_FEATURE(OmpAllocateClause)
-  READ_FEATURE(OmpAllocateClause::AllocateModifier)
-  READ_FEATURE(OmpAllocateClause::AllocateModifier::Allocator)
-  READ_FEATURE(OmpAllocateClause::AllocateModifier::ComplexModifier)
-  READ_FEATURE(OmpAllocateClause::AllocateModifier::Align)
+  READ_FEATURE(OmpAllocateClause::Modifier)
+  READ_FEATURE(OmpAllocatorSimpleModifier)
+  READ_FEATURE(OmpAllocatorComplexModifier)
+  READ_FEATURE(OmpAlignModifier)
   READ_FEATURE(OmpScheduleClause)
   READ_FEATURE(OmpScheduleClause::Kind)
   READ_FEATURE(OmpScheduleClause::Modifier)
+  READ_FEATURE(OmpDeviceModifier)
   READ_FEATURE(OmpDeviceClause)
-  READ_FEATURE(OmpDeviceClause::DeviceModifier)
+  READ_FEATURE(OmpDeviceClause::Modifier)
   READ_FEATURE(OmpDeviceTypeClause)
-  READ_FEATURE(OmpDeviceTypeClause::Type)
+  READ_FEATURE(OmpDeviceTypeClause::DeviceTypeDescription)
   READ_FEATURE(OmpChunkModifier)
   READ_FEATURE(OmpChunkModifier::Value)
   READ_FEATURE(OmpOrderingModifier)
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index 5bd8c76199278..2dc480f0c901b 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -8,6 +8,7 @@
 
 #include "FlangOmpReportVisitor.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 namespace Fortran {
 namespace parser {
@@ -190,15 +191,17 @@ void OpenMPCounterVisitor::PostConstructsCommon() {
   delete curConstruct;
 }
 
-void OpenMPCounterVisitor::Post(const OmpProcBindClause::Type &c) {
+void OpenMPCounterVisitor::Post(const OmpProcBindClause::AffinityPolicy &c) {
   clauseDetails +=
       "type=" + std::string{OmpProcBindClause::EnumToString(c)} + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpDefaultClause::Type &c) {
+void OpenMPCounterVisitor::Post(
+    const OmpDefaultClause::DataSharingAttribute &c) {
   clauseDetails +=
       "type=" + std::string{OmpDefaultClause::EnumToString(c)} + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpDeviceTypeClause::Type &c) {
+void OpenMPCounterVisitor::Post(
+    const OmpDeviceTypeClause::DeviceTypeDescription &c) {
   clauseDetails +=
       "type=" + std::string{OmpDeviceTypeClause::EnumToString(c)} + ";";
 }
@@ -236,9 +239,9 @@ void OpenMPCounterVisitor::Post(const OmpScheduleClause::Kind &c) {
   clauseDetails +=
       "type=" + std::string{OmpScheduleClause::EnumToString(c)} + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpIfClause::DirectiveNameModifier &c) {
+void OpenMPCounterVisitor::Post(const OmpDirectiveNameModifier &c) {
   clauseDetails +=
-      "name_modifier=" + std::string{OmpIfClause::EnumToString(c)} + ";";
+      "name_modifier=" + llvm::omp::getOpenMPDirectiveName(c.v).str() + ";";
 }
 void OpenMPCounterVisitor::Post(const OmpCancelType::Type &c) {
   clauseDetails += "type=" + std::string{OmpCancelType::EnumToString(c)} + ";";
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
index 7e9ae94bef297..59bdac8594cb7 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
@@ -66,18 +66,18 @@ struct OpenMPCounterVisitor {
   void Post(const OpenMPConstruct &);
   void PostConstructsCommon();
 
-  void Post(const OmpProcBindClause::Type &c);
-  void Post(const OmpDefaultClause::Type &c);
+  void Post(const OmpProcBindClause::AffinityPolicy &c);
+  void Post(const OmpDefaultClause::DataSharingAttribute &c);
   void Post(const OmpDefaultmapClause::ImplicitBehavior &c);
   void Post(const OmpVariableCategory::Value &c);
-  void Post(const OmpDeviceTypeClause::Type &c);
+  void Post(const OmpDeviceTypeClause::DeviceTypeDescription &c);
   void Post(const OmpChunkModifier::Value &c);
   void Post(const OmpLinearModifier::Value &c);
   void Post(const OmpOrderingModifier::Value &c);
   void Post(const OmpTaskDependenceType::Value &c);
   void Post(const OmpMapType::Value &c);
   void Post(const OmpScheduleClause::Kind &c);
-  void Post(const OmpIfClause::DirectiveNameModifier &c);
+  void Post(const OmpDirectiveNameModifier &c);
   void Post(const OmpCancelType::Type &c);
   void Post(const OmpClause &c);
   void PostClauseCommon(const ClauseInfo &ci);
diff --git a/flang/include/flang/Common/Fortran-consts.h b/flang/include/flang/Common/Fortran-consts.h
new file mode 100644
index 0000000000000..eedcdae335c40
--- /dev/null
+++ b/flang/include/flang/Common/Fortran-consts.h
@@ -0,0 +1,44 @@
+//===-- include/flang/Common/Fortran-consts.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_COMMON_FORTRAN_CONSTS_H_
+#define FORTRAN_COMMON_FORTRAN_CONSTS_H_
+
+#include "flang/Common/enum-class.h"
+#include <cstdint>
+
+namespace Fortran::common {
+
+// Fortran has five kinds of intrinsic data types, plus the derived types.
+ENUM_CLASS(TypeCategory, Integer, Real, Complex, Character, Logical, Derived)
+ENUM_CLASS(VectorElementCategory, Integer, Unsigned, Real)
+
+ENUM_CLASS(IoStmtKind, None, Backspace, Close, Endfile, Flush, Inquire, Open,
+    Print, Read, Rewind, Wait, Write)
+
+// Defined I/O variants
+ENUM_CLASS(
+    DefinedIo, ReadFormatted, ReadUnformatted, WriteFormatted, WriteUnformatted)
+
+// Fortran arrays may have up to 15 dimensions (See Fortran 2018 section 5.4.6).
+static constexpr int maxRank{15};
+
+// Floating-point rounding modes; these are packed into a byte to save
+// room in the runtime's format processing context structure.  These
+// enumerators are defined with the corresponding values returned from
+// llvm.get.rounding.
+enum class RoundingMode : std::uint8_t {
+  ToZero, // ROUND=ZERO, RZ - truncation
+  TiesToEven, // ROUND=NEAREST, RN - default IEEE rounding
+  Up, // ROUND=UP, RU
+  Down, // ROUND=DOWN, RD
+  TiesAwayFromZero, // ROUND=COMPATIBLE, RC - ties round away from zero
+};
+
+} // namespace Fortran::common
+#endif /* FORTRAN_COMMON_FORTRAN_CONSTS_H_ */
diff --git a/flang/include/flang/Common/Fortran-features.h b/flang/include/flang/Common/Fortran-features.h
index c6ab846cce2fc..b04f6117ae965 100644
--- a/flang/include/flang/Common/Fortran-features.h
+++ b/flang/include/flang/Common/Fortran-features.h
@@ -53,7 +53,8 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     NonBindCInteroperability, CudaManaged, CudaUnified,
     PolymorphicActualAllocatableOrPointerToMonomorphicDummy, RelaxedPureDummy,
     UndefinableAsynchronousOrVolatileActual, AutomaticInMainProgram, PrintCptr,
-    SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank)
+    SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
+    IgnoreIrrelevantAttributes)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
@@ -72,7 +73,7 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
     PreviousScalarUse, RedeclaredInaccessibleComponent, ImplicitShared,
     IndexVarRedefinition, IncompatibleImplicitInterfaces, BadTypeForTarget,
     VectorSubscriptFinalization, UndefinedFunctionResult, UselessIomsg,
-    MismatchingDummyProcedure)
+    MismatchingDummyProcedure, SubscriptedEmptyArray)
 
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
diff --git a/flang/include/flang/Common/Fortran.h b/flang/include/flang/Common/Fortran.h
index cb109ad574cf6..72e4348a42a3f 100644
--- a/flang/include/flang/Common/Fortran.h
+++ b/flang/include/flang/Common/Fortran.h
@@ -14,6 +14,7 @@
 
 #include "enum-set.h"
 #include "idioms.h"
+#include "flang/Common/Fortran-consts.h"
 #include <cinttypes>
 #include <optional>
 #include <string>
@@ -21,10 +22,6 @@
 namespace Fortran::common {
 class LanguageFeatureControl;
 
-// Fortran has five kinds of intrinsic data types, plus the derived types.
-ENUM_CLASS(TypeCategory, Integer, Real, Complex, Character, Logical, Derived)
-ENUM_CLASS(VectorElementCategory, Integer, Unsigned, Real)
-
 constexpr bool IsNumericTypeCategory(TypeCategory category) {
   return category == TypeCategory::Integer || category == TypeCategory::Real ||
       category == TypeCategory::Complex;
@@ -47,9 +44,6 @@ const char *AsFortran(RelationalOperator);
 
 ENUM_CLASS(Intent, Default, In, Out, InOut)
 
-ENUM_CLASS(IoStmtKind, None, Backspace, Close, Endfile, Flush, Inquire, Open,
-    Print, Read, Rewind, Wait, Write)
-
 // Union of specifiers for all I/O statements.
 ENUM_CLASS(IoSpecKind, Access, Action, Advance, Asynchronous, Blank, Decimal,
     Delim, Direct, Encoding, End, Eor, Err, Exist, File, Fmt, Form, Formatted,
@@ -61,29 +55,11 @@ ENUM_CLASS(IoSpecKind, Access, Action, Advance, Asynchronous, Blank, Decimal,
     Dispose, // nonstandard
 )
 
-// Defined I/O variants
-ENUM_CLASS(
-    DefinedIo, ReadFormatted, ReadUnformatted, WriteFormatted, WriteUnformatted)
 const char *AsFortran(DefinedIo);
 
-// Floating-point rounding modes; these are packed into a byte to save
-// room in the runtime's format processing context structure.  These
-// enumerators are defined with the corresponding values returned from
-// llvm.get.rounding.
-enum class RoundingMode : std::uint8_t {
-  ToZero, // ROUND=ZERO, RZ - truncation
-  TiesToEven, // ROUND=NEAREST, RN - default IEEE rounding
-  Up, // ROUND=UP, RU
-  Down, // ROUND=DOWN, RD
-  TiesAwayFromZero, // ROUND=COMPATIBLE, RC - ties round away from zero
-};
-
 // Fortran label. Must be in [1..99999].
 using Label = std::uint64_t;
 
-// Fortran arrays may have up to 15 dimensions (See Fortran 2018 section 5.4.6).
-static constexpr int maxRank{15};
-
 // CUDA subprogram attribute combinations
 ENUM_CLASS(CUDASubprogramAttrs, Host, Device, HostDevice, Global, Grid_Global)
 
diff --git a/flang/include/flang/Common/OpenMP-utils.h b/flang/include/flang/Common/OpenMP-utils.h
new file mode 100644
index 0000000000000..e6a3f1bac1c60
--- /dev/null
+++ b/flang/include/flang/Common/OpenMP-utils.h
@@ -0,0 +1,68 @@
+//===-- include/flang/Common/OpenMP-utils.h --------------------*- C++ -*-====//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_COMMON_OPENMP_UTILS_H_
+#define FORTRAN_COMMON_OPENMP_UTILS_H_
+
+#include "flang/Semantics/symbol.h"
+
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Value.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+namespace Fortran::common::openmp {
+/// Structure holding the information needed to create and bind entry block
+/// arguments associated to a single clause.
+struct EntryBlockArgsEntry {
+  llvm::ArrayRef<const Fortran::semantics::Symbol *> syms;
+  llvm::ArrayRef<mlir::Value> vars;
+
+  bool isValid() const {
+    // This check allows specifying a smaller number of symbols than values
+    // because in some case cases a single symbol generates multiple block
+    // arguments.
+    return syms.size() <= vars.size();
+  }
+};
+
+/// Structure holding the information needed to create and bind entry block
+/// arguments associated to all clauses that can define them.
+struct EntryBlockArgs {
+  EntryBlockArgsEntry inReduction;
+  EntryBlockArgsEntry map;
+  EntryBlockArgsEntry priv;
+  EntryBlockArgsEntry reduction;
+  EntryBlockArgsEntry taskReduction;
+  EntryBlockArgsEntry useDeviceAddr;
+  EntryBlockArgsEntry useDevicePtr;
+
+  bool isValid() const {
+    return inReduction.isValid() && map.isValid() && priv.isValid() &&
+        reduction.isValid() && taskReduction.isValid() &&
+        useDeviceAddr.isValid() && useDevicePtr.isValid();
+  }
+
+  auto getSyms() const {
+    return llvm::concat<const Fortran::semantics::Symbol *const>(
+        inReduction.syms, map.syms, priv.syms, reduction.syms,
+        taskReduction.syms, useDeviceAddr.syms, useDevicePtr.syms);
+  }
+
+  auto getVars() const {
+    return llvm::concat<const mlir::Value>(inReduction.vars, map.vars,
+        priv.vars, reduction.vars, taskReduction.vars, useDeviceAddr.vars,
+        useDevicePtr.vars);
+  }
+};
+
+mlir::Block *genEntryBlock(
+    mlir::OpBuilder &builder, const EntryBlockArgs &args, mlir::Region &region);
+} // namespace Fortran::common::openmp
+
+#endif // FORTRAN_COMMON_OPENMP_UTILS_H_
diff --git a/flang/include/flang/Common/format.h b/flang/include/flang/Common/format.h
index 67d37bee32ab3..de6967139c5ca 100644
--- a/flang/include/flang/Common/format.h
+++ b/flang/include/flang/Common/format.h
@@ -10,7 +10,7 @@
 #define FORTRAN_COMMON_FORMAT_H_
 
 #include "enum-set.h"
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include <cstring>
 
 // Define a FormatValidator class template to validate a format expression
diff --git a/flang/include/flang/Common/target-rounding.h b/flang/include/flang/Common/target-rounding.h
new file mode 100644
index 0000000000000..c0c9f6c49b26a
--- /dev/null
+++ b/flang/include/flang/Common/target-rounding.h
@@ -0,0 +1,37 @@
+//===-- include/flang/Common/target-rounding.h ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_COMMON_TARGET_ROUNDING_H_
+#define FORTRAN_COMMON_TARGET_ROUNDING_H_
+
+#include "flang/Common/Fortran-consts.h"
+#include "flang/Common/enum-set.h"
+
+namespace Fortran::common {
+
+// Floating-point rounding control
+struct Rounding {
+  common::RoundingMode mode{common::RoundingMode::TiesToEven};
+  // When set, emulate status flag behavior peculiar to x86
+  // (viz., fail to set the Underflow flag when an inexact product of a
+  // multiplication is rounded up to a normal number from a subnormal
+  // in some rounding modes)
+#if __x86_64__ || __riscv || __loongarch__
+  bool x86CompatibleBehavior{true};
+#else
+  bool x86CompatibleBehavior{false};
+#endif
+};
+
+// These are ordered like the bits in a common fenv.h header file.
+ENUM_CLASS(RealFlag, InvalidArgument, Denorm, DivideByZero, Overflow, Underflow,
+    Inexact)
+using RealFlags = common::EnumSet<RealFlag, RealFlag_enumSize>;
+
+} // namespace Fortran::common
+#endif /* FORTRAN_COMMON_TARGET_ROUNDING_H_ */
diff --git a/flang/include/flang/Evaluate/common.h b/flang/include/flang/Evaluate/common.h
index d493e5fe04417..915e95169c7f8 100644
--- a/flang/include/flang/Evaluate/common.h
+++ b/flang/include/flang/Evaluate/common.h
@@ -16,6 +16,7 @@
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/restorer.h"
+#include "flang/Common/target-rounding.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/message.h"
 #include <cinttypes>
@@ -32,6 +33,8 @@ class IntrinsicProcTable;
 class TargetCharacteristics;
 
 using common::ConstantSubscript;
+using common::RealFlag;
+using common::RealFlags;
 using common::RelationalOperator;
 
 // Integers are always ordered; reals may not be.
@@ -128,11 +131,6 @@ static constexpr bool Satisfies(RelationalOperator op, Relation relation) {
   return false; // silence g++ warning
 }
 
-// These are ordered like the bits in a common fenv.h header file.
-ENUM_CLASS(RealFlag, InvalidArgument, Denorm, DivideByZero, Overflow, Underflow,
-    Inexact)
-using RealFlags = common::EnumSet<RealFlag, RealFlag_enumSize>;
-
 template <typename A> struct ValueWithRealFlags {
   A AccumulateFlags(RealFlags &f) {
     f |= flags;
diff --git a/flang/include/flang/Evaluate/target.h b/flang/include/flang/Evaluate/target.h
index b347c549e012d..9d86000b2f8aa 100644
--- a/flang/include/flang/Evaluate/target.h
+++ b/flang/include/flang/Evaluate/target.h
@@ -15,24 +15,13 @@
 #include "flang/Common/Fortran.h"
 #include "flang/Common/enum-class.h"
 #include "flang/Common/enum-set.h"
+#include "flang/Common/target-rounding.h"
 #include "flang/Evaluate/common.h"
 #include <cstdint>
 
 namespace Fortran::evaluate {
 
-// Floating-point rounding control
-struct Rounding {
-  common::RoundingMode mode{common::RoundingMode::TiesToEven};
-  // When set, emulate status flag behavior peculiar to x86
-  // (viz., fail to set the Underflow flag when an inexact product of a
-  // multiplication is rounded up to a normal number from a subnormal
-  // in some rounding modes)
-#if __x86_64__ || __riscv || __loongarch__
-  bool x86CompatibleBehavior{true};
-#else
-  bool x86CompatibleBehavior{false};
-#endif
-};
+using common::Rounding;
 
 ENUM_CLASS(IeeeFeature, Denormal, Divide, Flags, Halting, Inf, Io, NaN,
     Rounding, Sqrt, Standard, Subnormal, UnderflowControl)
@@ -52,6 +41,12 @@ class TargetCharacteristics {
   }
   void set_areSubnormalsFlushedToZero(bool yes = true);
 
+  // Check if a given real kind has flushing control.
+  bool hasSubnormalFlushingControl(int kind) const;
+  // Check if any or all real kinds have flushing control.
+  bool hasSubnormalFlushingControl(bool any = false) const;
+  void set_hasSubnormalFlushingControl(int kind, bool yes = true);
+
   Rounding roundingMode() const { return roundingMode_; }
   void set_roundingMode(Rounding);
 
@@ -111,13 +106,14 @@ class TargetCharacteristics {
   const IeeeFeatures &ieeeFeatures() const { return ieeeFeatures_; }
 
 private:
-  static constexpr int maxKind{32};
-  std::uint8_t byteSize_[common::TypeCategory_enumSize][maxKind]{};
-  std::uint8_t align_[common::TypeCategory_enumSize][maxKind]{};
+  static constexpr int maxKind{16};
+  std::uint8_t byteSize_[common::TypeCategory_enumSize][maxKind + 1]{};
+  std::uint8_t align_[common::TypeCategory_enumSize][maxKind + 1]{};
   bool isBigEndian_{false};
   bool isPPC_{false};
   bool isOSWindows_{false};
   bool areSubnormalsFlushedToZero_{false};
+  bool hasSubnormalFlushingControl_[maxKind + 1]{};
   Rounding roundingMode_{defaultRounding};
   std::size_t procedurePointerByteSize_{8};
   std::size_t procedurePointerAlignment_{8};
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 583fa6fb215a7..307ba6a918777 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -118,8 +118,11 @@ class AbstractConverter {
 
   /// For a given symbol which is host-associated, create a clone using
   /// parameters from the host-associated symbol.
+  /// The clone is default initialized if its type has any default
+  /// initialization unless `skipDefaultInit` is set.
   virtual bool
-  createHostAssociateVarClone(const Fortran::semantics::Symbol &sym) = 0;
+  createHostAssociateVarClone(const Fortran::semantics::Symbol &sym,
+                              bool skipDefaultInit) = 0;
 
   virtual void
   createHostAssociateVarCloneDealloc(const Fortran::semantics::Symbol &sym) = 0;
diff --git a/flang/include/flang/Lower/Allocatable.h b/flang/include/flang/Lower/Allocatable.h
index 1209b157ed1f4..0e89af94af40f 100644
--- a/flang/include/flang/Lower/Allocatable.h
+++ b/flang/include/flang/Lower/Allocatable.h
@@ -15,7 +15,7 @@
 
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Optimizer/Builder/MutableBox.h"
-#include "flang/Runtime/allocator-registry.h"
+#include "flang/Runtime/allocator-registry-consts.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace mlir {
diff --git a/flang/include/flang/Lower/ConvertCall.h b/flang/include/flang/Lower/ConvertCall.h
index bc082907e6176..f1cd4f938320b 100644
--- a/flang/include/flang/Lower/ConvertCall.h
+++ b/flang/include/flang/Lower/ConvertCall.h
@@ -24,6 +24,11 @@
 
 namespace Fortran::lower {
 
+/// Data structure packaging the SSA value(s) produced for the result of lowered
+/// function calls.
+using LoweredResult =
+    std::variant<fir::ExtendedValue, hlfir::EntityWithAttributes>;
+
 /// Given a call site for which the arguments were already lowered, generate
 /// the call and return the result. This function deals with explicit result
 /// allocation and lowering if needed. It also deals with passing the host
@@ -32,7 +37,7 @@ namespace Fortran::lower {
 /// It is only used for HLFIR.
 /// The returned boolean indicates if finalization has been emitted in
 /// \p stmtCtx for the result.
-std::pair<fir::ExtendedValue, bool> genCallOpAndResult(
+std::pair<LoweredResult, bool> genCallOpAndResult(
     mlir::Location loc, Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symMap, Fortran::lower::StatementContext &stmtCtx,
     Fortran::lower::CallerInterface &caller, mlir::FunctionType callSiteType,
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 231de533fbd30..0b22e54b648e9 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -38,10 +38,5 @@ ENUM_LOWERINGOPT(Underscoring, unsigned, 1, 1)
 /// (i.e. wraps around as two's complement). Off by default.
 ENUM_LOWERINGOPT(IntegerWrapAround, unsigned, 1, 0)
 
-/// If true, add nsw flags to loop variable increments.
-/// Off by default.
-/// TODO: integrate this option with the above
-ENUM_LOWERINGOPT(NSWOnLoopVarInc, unsigned, 1, 0)
-
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Lower/PFTBuilder.h b/flang/include/flang/Lower/PFTBuilder.h
index 9b9d9febc190a..42d6546b77553 100644
--- a/flang/include/flang/Lower/PFTBuilder.h
+++ b/flang/include/flang/Lower/PFTBuilder.h
@@ -723,6 +723,7 @@ struct FunctionLikeUnit : public ProgramUnit {
   bool hasIeeeAccess{false};
   bool mayModifyHaltingMode{false};
   bool mayModifyRoundingMode{false};
+  bool mayModifyUnderflowMode{false};
   /// Terminal basic block (if any)
   mlir::Block *finalBlock{};
   HostAssociations hostAssociations;
diff --git a/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h b/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
index e410831c0fc3e..8d17e4e476d10 100644
--- a/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
+++ b/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
@@ -198,6 +198,12 @@ struct AliasAnalysis {
   /// Return the modify-reference behavior of `op` on `location`.
   mlir::ModRefResult getModRef(mlir::Operation *op, mlir::Value location);
 
+  /// Return the modify-reference behavior of operations inside `region` on
+  /// `location`. Contrary to getModRef(operation, location), this will visit
+  /// nested regions recursively according to the HasRecursiveMemoryEffects
+  /// trait.
+  mlir::ModRefResult getModRef(mlir::Region &region, mlir::Value location);
+
   /// Return the memory source of a value.
   /// If getLastInstantiationPoint is true, the search for the source
   /// will stop at [hl]fir.declare if it represents a dummy
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index f073f494b3fb2..efbd9e4f50d43 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -33,6 +33,7 @@ class AssociateOp;
 class ElementalOp;
 class ElementalOpInterface;
 class ElementalAddrOp;
+class EvaluateInMemoryOp;
 class YieldElementOp;
 
 /// Is this a Fortran variable for which the defining op carrying the Fortran
@@ -398,6 +399,24 @@ mlir::Value inlineElementalOp(
     mlir::IRMapping &mapper,
     const std::function<bool(hlfir::ElementalOp)> &mustRecursivelyInline);
 
+/// Create a new temporary with the shape and parameters of the provided
+/// hlfir.eval_in_mem operation and clone the body of the hlfir.eval_in_mem
+/// operating on this new temporary.  returns the temporary and whether the
+/// temporary is heap or stack allocated.
+std::pair<hlfir::Entity, bool>
+computeEvaluateOpInNewTemp(mlir::Location, fir::FirOpBuilder &,
+                           hlfir::EvaluateInMemoryOp evalInMem,
+                           mlir::Value shape, mlir::ValueRange typeParams);
+
+// Clone the body of the hlfir.eval_in_mem operating on this the provided
+// storage.  The provided storage must be a contiguous "raw" memory reference
+// (not a fir.box) big enough to hold the value computed by hlfir.eval_in_mem.
+// No runtime check is inserted by this utility to enforce that. It is also
+// usually invalid to provide some storage that is already addressed directly
+// or indirectly inside the hlfir.eval_in_mem body.
+void computeEvaluateOpIn(mlir::Location, fir::FirOpBuilder &,
+                         hlfir::EvaluateInMemoryOp, mlir::Value storage);
+
 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>
 convertToValue(mlir::Location loc, fir::FirOpBuilder &builder,
                hlfir::Entity entity);
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 19c623cc1ec00..bc0020e614db2 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -16,7 +16,7 @@
 #include "flang/Optimizer/Builder/Runtime/Numeric.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Runtime/entry-names.h"
-#include "flang/Runtime/iostat.h"
+#include "flang/Runtime/iostat-consts.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Math/IR/Math.h"
@@ -166,11 +166,6 @@ struct IntrinsicLibrary {
   getRuntimeCallGenerator(llvm::StringRef name,
                           mlir::FunctionType soughtFuncType);
 
-  /// Helper to generate TODOs for module procedures that must be intercepted in
-  /// lowering and are not yet implemented.
-  template <const char *intrinsicName>
-  void genModuleProcTODO(llvm::ArrayRef<fir::ExtendedValue>);
-
   void genAbort(llvm::ArrayRef<fir::ExtendedValue>);
   /// Lowering for the ABS intrinsic. The ABS intrinsic expects one argument in
   /// the llvm::ArrayRef. The ABS intrinsic is lowered into MLIR/FIR operation
@@ -278,6 +273,7 @@ struct IntrinsicLibrary {
   template <bool isGet>
   void genIeeeGetOrSetStatus(llvm::ArrayRef<fir::ExtendedValue>);
   void genIeeeGetRoundingMode(llvm::ArrayRef<fir::ExtendedValue>);
+  void genIeeeGetUnderflowMode(llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genIeeeInt(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genIeeeIsFinite(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genIeeeIsNan(mlir::Type, llvm::ArrayRef<mlir::Value>);
@@ -295,6 +291,7 @@ struct IntrinsicLibrary {
   template <bool isFlag>
   void genIeeeSetFlagOrHaltingMode(llvm::ArrayRef<fir::ExtendedValue>);
   void genIeeeSetRoundingMode(llvm::ArrayRef<fir::ExtendedValue>);
+  void genIeeeSetUnderflowMode(llvm::ArrayRef<fir::ExtendedValue>);
   template <mlir::arith::CmpFPredicate pred>
   mlir::Value genIeeeSignalingCompare(mlir::Type resultType,
                                       llvm::ArrayRef<mlir::Value>);
diff --git a/flang/include/flang/Optimizer/Builder/MutableBox.h b/flang/include/flang/Optimizer/Builder/MutableBox.h
index fea7c7204837b..39657ddaf6e03 100644
--- a/flang/include/flang/Optimizer/Builder/MutableBox.h
+++ b/flang/include/flang/Optimizer/Builder/MutableBox.h
@@ -14,7 +14,7 @@
 #define FORTRAN_OPTIMIZER_BUILDER_MUTABLEBOX_H
 
 #include "flang/Optimizer/Builder/BoxValue.h"
-#include "flang/Runtime/allocator-registry.h"
+#include "flang/Runtime/allocator-registry-consts.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace mlir {
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Exceptions.h b/flang/include/flang/Optimizer/Builder/Runtime/Exceptions.h
index aa6e33c7440ad..f2f83b46f20fd 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Exceptions.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Exceptions.h
@@ -26,5 +26,9 @@ namespace fir::runtime {
 mlir::Value genMapExcept(fir::FirOpBuilder &builder, mlir::Location loc,
                          mlir::Value excepts);
 
+mlir::Value genGetUnderflowMode(fir::FirOpBuilder &builder, mlir::Location loc);
+void genSetUnderflowMode(fir::FirOpBuilder &builder, mlir::Location loc,
+                         mlir::Value bit);
+
 } // namespace fir::runtime
 #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_EXCEPTIONS_H
diff --git a/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h b/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
index ff0cf29e8073e..9cccf8db87270 100644
--- a/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
+++ b/flang/include/flang/Optimizer/CodeGen/DescriptorModel.h
@@ -23,7 +23,7 @@
 #define OPTIMIZER_DESCRIPTOR_MODEL_H
 
 #include "flang/ISO_Fortran_binding_wrapper.h"
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index c820b83834de6..35749dae5d7e9 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -195,6 +195,8 @@ template <typename SourceOp>
 class FIROpConversion : public ConvertFIRToLLVMPattern {
 public:
   using OpAdaptor = typename SourceOp::Adaptor;
+  using OneToNOpAdaptor = typename SourceOp::template GenericAdaptor<
+      mlir::ArrayRef<mlir::ValueRange>>;
 
   explicit FIROpConversion(const LLVMTypeConverter &typeConverter,
                            const fir::FIRToLLVMPassOptions &options,
@@ -209,6 +211,12 @@ class FIROpConversion : public ConvertFIRToLLVMPattern {
     rewrite(mlir::cast<SourceOp>(op),
             OpAdaptor(operands, mlir::cast<SourceOp>(op)), rewriter);
   }
+  void rewrite(mlir::Operation *op, mlir::ArrayRef<mlir::ValueRange> operands,
+               mlir::ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = llvm::cast<SourceOp>(op);
+    rewrite(llvm::cast<SourceOp>(op), OneToNOpAdaptor(operands, sourceOp),
+            rewriter);
+  }
   llvm::LogicalResult match(mlir::Operation *op) const final {
     return match(mlir::cast<SourceOp>(op));
   }
@@ -219,7 +227,14 @@ class FIROpConversion : public ConvertFIRToLLVMPattern {
                            OpAdaptor(operands, mlir::cast<SourceOp>(op)),
                            rewriter);
   }
-
+  llvm::LogicalResult
+  matchAndRewrite(mlir::Operation *op,
+                  mlir::ArrayRef<mlir::ValueRange> operands,
+                  mlir::ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = mlir::cast<SourceOp>(op);
+    return matchAndRewrite(sourceOp, OneToNOpAdaptor(operands, sourceOp),
+                           rewriter);
+  }
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
   /// overridden by the derived pattern class.
   virtual llvm::LogicalResult match(SourceOp op) const {
@@ -229,6 +244,12 @@ class FIROpConversion : public ConvertFIRToLLVMPattern {
                        mlir::ConversionPatternRewriter &rewriter) const {
     llvm_unreachable("must override rewrite or matchAndRewrite");
   }
+  virtual void rewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                       mlir::ConversionPatternRewriter &rewriter) const {
+    llvm::SmallVector<mlir::Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    rewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
   virtual llvm::LogicalResult
   matchAndRewrite(SourceOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const {
@@ -237,6 +258,13 @@ class FIROpConversion : public ConvertFIRToLLVMPattern {
     rewrite(op, adaptor, rewriter);
     return mlir::success();
   }
+  virtual llvm::LogicalResult
+  matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const {
+    llvm::SmallVector<mlir::Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
 
 private:
   using ConvertFIRToLLVMPattern::matchAndRewrite;
diff --git a/flang/include/flang/Optimizer/CodeGen/Target.h b/flang/include/flang/Optimizer/CodeGen/Target.h
index 3b38583511927..54bd8c9c457a1 100644
--- a/flang/include/flang/Optimizer/CodeGen/Target.h
+++ b/flang/include/flang/Optimizer/CodeGen/Target.h
@@ -133,15 +133,7 @@ class CodeGenSpecifics {
 
   /// Type representation of a `boxchar<n>` type argument when passed by value.
   /// An argument value may need to be passed as a (safe) reference argument.
-  ///
-  /// A function that returns a `boxchar<n>` type value must already have
-  /// converted that return value to a parameter decorated with the 'sret'
-  /// Attribute (https://llvm.org/docs/LangRef.html#parameter-attributes).
-  /// This requirement is in keeping with Fortran semantics, which require the
-  /// caller to allocate the space for the return CHARACTER value and pass
-  /// a pointer and the length of that space (a boxchar) to the called function.
-  virtual Marshalling boxcharArgumentType(mlir::Type eleTy,
-                                          bool sret = false) const = 0;
+  virtual Marshalling boxcharArgumentType(mlir::Type eleTy) const = 0;
 
   // Compute ABI rules for an integer argument of the given mlir::IntegerType
   // \p argTy. Note that this methods is supposed to be called for
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h b/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
index 3830237f96f3c..15296aa7e8c75 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
@@ -61,6 +61,10 @@ inline mlir::Type getFortranElementOrSequenceType(mlir::Type type) {
   return type;
 }
 
+/// Build the hlfir.expr type for the value held in a variable of type \p
+/// variableType.
+mlir::Type getExprType(mlir::Type variableType);
+
 /// Is this a fir.box or fir.class address type?
 inline bool isBoxAddressType(mlir::Type type) {
   type = fir::dyn_cast_ptrEleTy(type);
@@ -132,6 +136,9 @@ mlir::Value genExprShape(mlir::OpBuilder &builder, const mlir::Location &loc,
 /// This has to be cleaned up, when HLFIR is the default.
 bool mayHaveAllocatableComponent(mlir::Type ty);
 
+/// Scalar integer or a sequence of integers (via boxed array or expr).
+bool isFortranIntegerScalarOrArrayObject(mlir::Type type);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_HLFIR_HLFIRDIALECT_H
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
index d967a407a7588..404ab5f633bf7 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
@@ -155,6 +155,12 @@ def IsPolymorphicObjectPred
 def AnyPolymorphicObject : Type<IsPolymorphicObjectPred,
     "any polymorphic object">;
 
+def IsFortranIntegerScalarOrArrayPred
+    : CPred<"::hlfir::isFortranIntegerScalarOrArrayObject($_self)">;
+def AnyFortranIntegerScalarOrArrayObject
+    : Type<IsFortranIntegerScalarOrArrayPred,
+           "A scalar or array object containing integers">;
+
 def hlfir_CharExtremumPredicateAttr : I32EnumAttr<
     "CharExtremumPredicate", "",
     [
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 1ab8793f72652..f11162dc0d95e 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -699,6 +699,27 @@ def hlfir_MatmulTransposeOp : hlfir_Op<"matmul_transpose",
   let hasVerifier = 1;
 }
 
+def hlfir_CShiftOp
+    : hlfir_Op<
+          "cshift", [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+  let summary = "CSHIFT transformational intrinsic";
+  let description = [{
+    Circular shift of an array
+  }];
+
+  let arguments = (ins AnyFortranArrayObject:$array,
+      AnyFortranIntegerScalarOrArrayObject:$shift,
+      Optional<AnyIntegerType>:$dim);
+
+  let results = (outs hlfir_ExprType);
+
+  let assemblyFormat = [{
+    $array $shift (`dim` $dim^)? attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
 // An allocation effect is needed because the value produced by the associate
 // is "deallocated" by hlfir.end_associate (the end_associate must not be
 // removed, and there must be only one hlfir.end_associate).
@@ -1755,4 +1776,63 @@ def hlfir_CharExtremumOp : hlfir_Op<"char_extremum",
   let hasVerifier = 1;
 }
 
+def hlfir_EvaluateInMemoryOp : hlfir_Op<"eval_in_mem", [AttrSizedOperandSegments,
+    RecursiveMemoryEffects, RecursivelySpeculatable,
+    SingleBlockImplicitTerminator<"fir::FirEndOp">]> {
+  let summary = "Wrap an in-memory implementation that computes expression value";
+  let description = [{
+    Returns a Fortran expression value for which the computation is
+    implemented inside the region operating on the block argument which
+    is a raw memory reference corresponding to the expression type.
+
+    The shape and type parameters of the expressions are operands of the
+    operations.
+
+    The memory cannot escape the region, and it is not described how it is
+    allocated. This facilitates later elision of the temporary storage for the
+    expression evaluation if it can be evaluated in some other storage (like a
+    left-hand side variable).
+
+    Example:
+
+    A function returning an array can be represented as:
+    ```
+      %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+      %2 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+      ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>):
+        %3 = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+        fir.save_result %3 to %arg0(%1) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+      }
+    ```
+  }];
+
+  let arguments = (ins
+    Optional<fir_ShapeType>:$shape,
+    Variadic<AnyIntegerType>:$typeparams
+  );
+
+  let results = (outs hlfir_ExprType);
+  let regions = (region  SizedRegion<1>:$body);
+
+  let assemblyFormat = [{
+    (`shape` $shape^)? (`typeparams` $typeparams^)?
+    attr-dict `:` functional-type(operands, results)
+    $body}];
+
+  let skipDefaultBuilders = 1;
+  let builders = [
+    OpBuilder<(ins "mlir::Type":$result_type, "mlir::Value":$shape,
+      CArg<"mlir::ValueRange", "{}">:$typeparams)>
+  ];
+
+  let extraClassDeclaration = [{
+      // Return block argument representing the memory where the expression
+      // is evaluated.
+      mlir::Value getMemory() {return getBody().getArgument(0);}
+  }];
+
+  let hasVerifier = 1;
+}
+
+
 #endif // FORTRAN_DIALECT_HLFIR_OPS
diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 37977334c1e9e..3add0c560f88d 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -50,9 +50,29 @@ def FunctionFilteringPass : Pass<"omp-function-filtering"> {
   ];
 }
 
+
 // Needs to be scheduled on Module as we create functions in it
 def LowerWorkshare : Pass<"lower-workshare", "::mlir::ModuleOp"> {
   let summary = "Lower workshare construct";
 }
 
+def GenericLoopConversionPass
+    : Pass<"omp-generic-loop-conversion", "mlir::func::FuncOp"> {
+  let summary = "Converts OpenMP generic `omp.loop` to semantically "
+                "equivalent OpenMP ops";
+  let description = [{
+     Rewrites `omp.loop` ops to their semantically equivalent nest of ops. The
+     rewrite depends on the nesting/combination structure of the `loop` op
+     within its surrounding context as well as its `bind` clause value.
+
+     We assume for now that all `omp.loop` ops will occur inside `FuncOp`'s. This
+     will most likely remain the case in the future; even if, for example, we 
+     need a loop in copying data for a `firstprivate` variable, this loop will
+     be nested in a constructor, an overloaded operator, or a runtime function.
+  }];
+  let dependentDialects = [
+    "mlir::omp::OpenMPDialect"
+  ];
+}
+
 #endif //FORTRAN_OPTIMIZER_OPENMP_PASSES
diff --git a/flang/include/flang/Optimizer/Passes/Pipelines.h b/flang/include/flang/Optimizer/Passes/Pipelines.h
index 55fafc2e6b36f..ef5d44ded706c 100644
--- a/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -20,6 +20,7 @@
 #include "flang/Tools/CrossToolHelpers.h"
 #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
 #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
@@ -32,16 +33,16 @@ namespace fir {
 
 using PassConstructor = std::unique_ptr<mlir::Pass>();
 
-template <typename OP>
-void addNestedPassToOps(mlir::PassManager &pm, PassConstructor ctor) {
+template <typename F, typename OP>
+void addNestedPassToOps(mlir::PassManager &pm, F ctor) {
   pm.addNestedPass<OP>(ctor());
 }
 
-template <typename OP, typename... OPS,
+template <typename F, typename OP, typename... OPS,
           typename = std::enable_if_t<sizeof...(OPS) != 0>>
-void addNestedPassToOps(mlir::PassManager &pm, PassConstructor ctor) {
-  addNestedPassToOps<OP>(pm, ctor);
-  addNestedPassToOps<OPS...>(pm, ctor);
+void addNestedPassToOps(mlir::PassManager &pm, F ctor) {
+  addNestedPassToOps<F, OP>(pm, ctor);
+  addNestedPassToOps<F, OPS...>(pm, ctor);
 }
 
 /// Generic for adding a pass to the pass manager if it is not disabled.
@@ -59,11 +60,12 @@ void addNestedPassConditionally(mlir::PassManager &pm,
     pm.addNestedPass<OP>(ctor());
 }
 
-void addNestedPassToAllTopLevelOperations(mlir::PassManager &pm,
-                                          PassConstructor ctor);
+template <typename F>
+void addNestedPassToAllTopLevelOperations(mlir::PassManager &pm, F ctor);
 
+template <typename F>
 void addNestedPassToAllTopLevelOperationsConditionally(
-    mlir::PassManager &pm, llvm::cl::opt<bool> &disabled, PassConstructor ctor);
+    mlir::PassManager &pm, llvm::cl::opt<bool> &disabled, F ctor);
 
 /// Add MLIR Canonicalizer pass with region simplification disabled.
 /// FIR does not support the promotion of some SSA value to block arguments (or
diff --git a/flang/include/flang/Optimizer/Transforms/CUFCommon.h b/flang/include/flang/Optimizer/Transforms/CUFCommon.h
index f019d1893bda4..df1b709dc8608 100644
--- a/flang/include/flang/Optimizer/Transforms/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Transforms/CUFCommon.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_TRANSFORMS_CUFCOMMON_H_
 #define FORTRAN_OPTIMIZER_TRANSFORMS_CUFCOMMON_H_
 
+#include "flang/Optimizer/Dialect/FIROps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/IR/BuiltinOps.h"
 
@@ -21,6 +22,7 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isInCUDADeviceContext(mlir::Operation *op);
+bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 
 } // namespace cuf
 
diff --git a/flang/include/flang/Optimizer/Transforms/CUFOpConversion.h b/flang/include/flang/Optimizer/Transforms/CUFOpConversion.h
index f061323db1704..336cf46d82bab 100644
--- a/flang/include/flang/Optimizer/Transforms/CUFOpConversion.h
+++ b/flang/include/flang/Optimizer/Transforms/CUFOpConversion.h
@@ -23,11 +23,16 @@ class SymbolTable;
 
 namespace cuf {
 
+/// Patterns that convert CUF operations to runtime calls.
 void populateCUFToFIRConversionPatterns(const fir::LLVMTypeConverter &converter,
                                         mlir::DataLayout &dl,
                                         const mlir::SymbolTable &symtab,
                                         mlir::RewritePatternSet &patterns);
 
+/// Patterns that updates fir operations in presence of CUF.
+void populateFIRCUFConversionPatterns(const mlir::SymbolTable &symtab,
+                                      mlir::RewritePatternSet &patterns);
+
 } // namespace cuf
 
 #endif // FORTRAN_OPTIMIZER_TRANSFORMS_CUFOPCONVERSION_H_
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index e8f0a8444a31a..10e1c999d4533 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -66,7 +66,6 @@ namespace fir {
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
-std::unique_ptr<mlir::Pass> createCFGConversionPassWithNSW();
 std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
 std::unique_ptr<mlir::Pass>
@@ -83,7 +82,7 @@ createVScaleAttrPass(std::pair<unsigned, unsigned> vscaleAttr);
 
 void populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
                                    bool forceLoopToExecuteOnce = false,
-                                   bool setNSW = false);
+                                   bool setNSW = true);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 1db41fa7972bf..61f8b0835c958 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -153,7 +153,7 @@ def CFGConversion : Pass<"cfg-conversion"> {
            /*default=*/"false",
            "force the body of a loop to execute at least once">,
     Option<"setNSW", "set-nsw", "bool",
-           /*default=*/"false",
+           /*default=*/"true",
            "set nsw on loop variable increment">
   ];
 }
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 68f9406dc2830..c6f35a07d81ea 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -484,11 +484,16 @@ class ParseTreeDumper {
   NODE(parser, OmpIteratorSpecifier)
   NODE(parser, OmpIterator)
   NODE(parser, OmpAffinityClause)
+  NODE(OmpAffinityClause, Modifier)
+  NODE(parser, OmpAlignment)
   NODE(parser, OmpAlignedClause)
+  NODE(OmpAlignedClause, Modifier)
   NODE(parser, OmpAtomic)
   NODE(parser, OmpAtomicCapture)
   NODE(OmpAtomicCapture, Stmt1)
   NODE(OmpAtomicCapture, Stmt2)
+  NODE(parser, OmpAtomicCompare)
+  NODE(parser, OmpAtomicCompareIfStmt)
   NODE(parser, OmpAtomicRead)
   NODE(parser, OmpAtomicUpdate)
   NODE(parser, OmpAtomicWrite)
@@ -513,7 +518,7 @@ class ParseTreeDumper {
   NODE(parser, OmpDeclareTargetWithList)
   NODE(parser, OmpDeclareMapperSpecifier)
   NODE(parser, OmpDefaultClause)
-  NODE_ENUM(OmpDefaultClause, Type)
+  NODE_ENUM(OmpDefaultClause, DataSharingAttribute)
   NODE(parser, OmpVariableCategory)
   NODE_ENUM(OmpVariableCategory, Value)
   NODE(parser, OmpDefaultmapClause)
@@ -531,6 +536,7 @@ class ParseTreeDumper {
   NODE(OmpDoacross, Source)
   NODE(parser, OmpDependClause)
   NODE(OmpDependClause, TaskDep)
+  NODE(OmpDependClause::TaskDep, Modifier)
   NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
@@ -544,10 +550,13 @@ class ParseTreeDumper {
   NODE(OmpFromClause, Modifier)
   NODE(parser, OmpExpectation)
   NODE_ENUM(OmpExpectation, Value)
+  NODE(parser, OmpDirectiveNameModifier)
   NODE(parser, OmpIfClause)
-  NODE_ENUM(OmpIfClause, DirectiveNameModifier)
-  NODE_ENUM(OmpLastprivateClause, LastprivateModifier)
+  NODE(OmpIfClause, Modifier)
   NODE(parser, OmpLastprivateClause)
+  NODE(OmpLastprivateClause, Modifier)
+  NODE(parser, OmpLastprivateModifier)
+  NODE_ENUM(OmpLastprivateModifier, Value)
   NODE(parser, OmpLinearClause)
   NODE(OmpLinearClause, WithModifier)
   NODE(OmpLinearClause, WithoutModifier)
@@ -569,13 +578,15 @@ class ParseTreeDumper {
   NODE(parser, OmpOrderModifier)
   NODE_ENUM(OmpOrderModifier, Value)
   NODE(parser, OmpGrainsizeClause)
-  NODE_ENUM(OmpGrainsizeClause, Prescriptiveness)
+  NODE(OmpGrainsizeClause, Modifier)
+  NODE(parser, OmpPrescriptiveness)
+  NODE_ENUM(OmpPrescriptiveness, Value)
   NODE(parser, OmpNumTasksClause)
-  NODE_ENUM(OmpNumTasksClause, Prescriptiveness)
+  NODE(OmpNumTasksClause, Modifier)
   NODE(parser, OmpBindClause)
-  NODE_ENUM(OmpBindClause, Type)
+  NODE_ENUM(OmpBindClause, Binding)
   NODE(parser, OmpProcBindClause)
-  NODE_ENUM(OmpProcBindClause, Type)
+  NODE_ENUM(OmpProcBindClause, AffinityPolicy)
   NODE(parser, OmpReductionModifier)
   NODE_ENUM(OmpReductionModifier, Value)
   NODE(parser, OmpReductionClause)
@@ -586,17 +597,19 @@ class ParseTreeDumper {
   NODE(parser, OmpReductionInitializerClause)
   NODE(parser, OmpReductionIdentifier)
   NODE(parser, OmpAllocateClause)
-  NODE(OmpAllocateClause, AllocateModifier)
-  NODE(OmpAllocateClause::AllocateModifier, Allocator)
-  NODE(OmpAllocateClause::AllocateModifier, ComplexModifier)
-  NODE(OmpAllocateClause::AllocateModifier, Align)
+  NODE(OmpAllocateClause, Modifier)
+  NODE(parser, OmpAlignModifier)
+  NODE(parser, OmpAllocatorComplexModifier)
+  NODE(parser, OmpAllocatorSimpleModifier)
   NODE(parser, OmpScheduleClause)
   NODE(OmpScheduleClause, Modifier)
   NODE_ENUM(OmpScheduleClause, Kind)
   NODE(parser, OmpDeviceClause)
-  NODE_ENUM(OmpDeviceClause, DeviceModifier)
+  NODE(OmpDeviceClause, Modifier)
+  NODE(parser, OmpDeviceModifier)
+  NODE_ENUM(OmpDeviceModifier, Value)
   NODE(parser, OmpDeviceTypeClause)
-  NODE_ENUM(OmpDeviceTypeClause, Type)
+  NODE_ENUM(OmpDeviceTypeClause, DeviceTypeDescription)
   NODE(parser, OmpUpdateClause)
   NODE(parser, OmpChunkModifier)
   NODE_ENUM(OmpChunkModifier, Value)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 8d7119a56b7f8..8160b095f06dd 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -1024,10 +1024,18 @@ struct Initialization {
 
 // R739 component-decl ->
 //        component-name [( component-array-spec )]
-//        [lbracket coarray-spec rbracket] [* char-length]
-//        [component-initialization]
+//          [lbracket coarray-spec rbracket] [* char-length]
+//          [component-initialization] |
+//        component-name *char-length [( component-array-spec )]
+//          [lbracket coarray-spec rbracket] [component-initialization]
 struct ComponentDecl {
   TUPLE_CLASS_BOILERPLATE(ComponentDecl);
+  ComponentDecl(Name &&name, CharLength &&length,
+      std::optional<ComponentArraySpec> &&aSpec,
+      std::optional<CoarraySpec> &&coaSpec,
+      std::optional<Initialization> &&init)
+      : t{std::move(name), std::move(aSpec), std::move(coaSpec),
+            std::move(length), std::move(init)} {}
   std::tuple<Name, std::optional<ComponentArraySpec>,
       std::optional<CoarraySpec>, std::optional<CharLength>,
       std::optional<Initialization>>
@@ -1381,9 +1389,16 @@ struct AttrSpec {
 // R803 entity-decl ->
 //        object-name [( array-spec )] [lbracket coarray-spec rbracket]
 //          [* char-length] [initialization] |
-//        function-name [* char-length]
+//        function-name [* char-length] |
+// (ext.) object-name *char-length [( array-spec )]
+//          [lbracket coarray-spec rbracket] [initialization]
 struct EntityDecl {
   TUPLE_CLASS_BOILERPLATE(EntityDecl);
+  EntityDecl(ObjectName &&name, CharLength &&length,
+      std::optional<ArraySpec> &&aSpec, std::optional<CoarraySpec> &&coaSpec,
+      std::optional<Initialization> &&init)
+      : t{std::move(name), std::move(aSpec), std::move(coaSpec),
+            std::move(length), std::move(init)} {}
   std::tuple<ObjectName, std::optional<ArraySpec>, std::optional<CoarraySpec>,
       std::optional<CharLength>, std::optional<Initialization>>
       t;
@@ -3457,6 +3472,38 @@ inline namespace modifier {
 //   ENUM_CLASS(Value, Keyword1, Keyword2);
 // };
 
+// Ref: [4.5:72-81], [5.0:110-119], [5.1:134-143], [5.2:169-170]
+//
+// alignment ->
+//    scalar-integer-expression                     // since 4.5
+struct OmpAlignment {
+  WRAPPER_CLASS_BOILERPLATE(OmpAlignment, ScalarIntExpr);
+};
+
+// Ref: [5.1:184-185], [5.2:178-179]
+//
+// align-modifier ->
+//    ALIGN(alignment)                              // since 5.1
+struct OmpAlignModifier {
+  WRAPPER_CLASS_BOILERPLATE(OmpAlignModifier, ScalarIntExpr);
+};
+
+// Ref: [5.0:158-159], [5.1:184-185], [5.2:178-179]
+//
+// allocator-simple-modifier ->
+//    allocator                                     // since 5.0
+struct OmpAllocatorSimpleModifier {
+  WRAPPER_CLASS_BOILERPLATE(OmpAllocatorSimpleModifier, ScalarIntExpr);
+};
+
+// Ref: [5.1:184-185], [5.2:178-179]
+//
+// allocator-complex-modifier ->
+//    ALLOCATOR(allocator)                          // since 5.1
+struct OmpAllocatorComplexModifier {
+  WRAPPER_CLASS_BOILERPLATE(OmpAllocatorComplexModifier, ScalarIntExpr);
+};
+
 // Ref: [5.2:252-254]
 //
 // chunk-modifier ->
@@ -3502,6 +3549,32 @@ struct OmpDependenceType {
   WRAPPER_CLASS_BOILERPLATE(OmpDependenceType, Value);
 };
 
+// Ref: [5.0:170-176], [5.1:197-205], [5.2:276-277]
+//
+// device-modifier ->
+//    ANCESTOR | DEVICE_NUM                         // since 5.0
+struct OmpDeviceModifier {
+  ENUM_CLASS(Value, Ancestor, Device_Num)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeviceModifier, Value);
+};
+
+// Ref: [5.2:72-73,230-323], in 4.5-5.1 it's scattered over individual
+// directives that allow the IF clause.
+//
+// directive-name-modifier ->
+//    PARALLEL | TARGET | TARGET DATA |
+//    TARGET ENTER DATA | TARGET EXIT DATA |
+//    TARGET UPDATE | TASK | TASKLOOP |             // since 4.5
+//    CANCEL[*] | SIMD |                            // since 5.0
+//    TEAMS                                         // since 5.2
+//
+// [*] The IF clause is allowed on CANCEL in OpenMP 4.5, but only without
+// the directive-name-modifier. For the sake of uniformity CANCEL can be
+// considered a valid value in 4.5 as well.
+struct OmpDirectiveNameModifier {
+  WRAPPER_CLASS_BOILERPLATE(OmpDirectiveNameModifier, llvm::omp::Directive);
+};
+
 // Ref: [5.1:205-209], [5.2:166-168]
 //
 // motion-modifier ->
@@ -3525,6 +3598,15 @@ struct OmpIterator {
   WRAPPER_CLASS_BOILERPLATE(OmpIterator, std::list<OmpIteratorSpecifier>);
 };
 
+// Ref: [5.0:288-290], [5.1:321-322], [5.2:115-117]
+//
+// lastprivate-modifier ->
+//    CONDITIONAL                                   // since 5.0
+struct OmpLastprivateModifier {
+  ENUM_CLASS(Value, Conditional)
+  WRAPPER_CLASS_BOILERPLATE(OmpLastprivateModifier, Value);
+};
+
 // Ref: [4.5:207-210], [5.0:290-293], [5.1:323-325], [5.2:117-120]
 //
 // linear-modifier ->
@@ -3586,6 +3668,15 @@ struct OmpOrderModifier {
   WRAPPER_CLASS_BOILERPLATE(OmpOrderModifier, Value);
 };
 
+// Ref: [5.1:166-171], [5.2:269-270]
+//
+// prescriptiveness ->
+//    STRICT                                        // since 5.1
+struct OmpPrescriptiveness {
+  ENUM_CLASS(Value, Strict)
+  WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
+};
+
 // Ref: [4.5:201-207], [5.0:293-299], [5.1:325-331], [5.2:124]
 //
 // reduction-identifier ->
@@ -3632,55 +3723,89 @@ struct OmpVariableCategory {
 
 // --- Clauses
 
-// OMP 5.0 2.10.1 affinity([aff-modifier:] locator-list)
-//                aff-modifier: interator-modifier
+// Ref: [5.0:135-140], [5.1:161-166], [5.2:264-265]
+//
+// affinity-clause ->
+//    AFFINITY([aff-modifier:] locator-list)        // since 5.0
+// aff-modifier ->
+//    interator-modifier                            // since 5.0
 struct OmpAffinityClause {
   TUPLE_CLASS_BOILERPLATE(OmpAffinityClause);
-  std::tuple<std::optional<OmpIterator>, OmpObjectList> t;
+  MODIFIER_BOILERPLATE(OmpIterator);
+  std::tuple<MODIFIERS(), OmpObjectList> t;
 };
 
-// 2.8.1 aligned-clause -> ALIGNED (variable-name-list[ : scalar-constant])
+// Ref: [4.5:72-81], [5.0:110-119], [5.1:134-143], [5.2:169-170]
+//
+// aligned-clause ->
+//    ALIGNED(list [: alignment])                   // since 4.5
 struct OmpAlignedClause {
   TUPLE_CLASS_BOILERPLATE(OmpAlignedClause);
-  CharBlock source;
-  std::tuple<OmpObjectList, std::optional<ScalarIntConstantExpr>> t;
+  MODIFIER_BOILERPLATE(OmpAlignment);
+  std::tuple<OmpObjectList, MODIFIERS()> t;
 };
 
-// OMP 5.0 2.11.4 allocate-clause -> ALLOCATE ([allocator:] variable-name-list)
-// OMP 5.2 2.13.4 allocate-clause -> ALLOCATE ([allocate-modifier [,
-//  	                               allocate-modifier] :]
-//                                   variable-name-list)
-//                allocate-modifier -> allocator | align
+// Ref: [5.0:158-159], [5.1:184-185], [5.2:178-179]
+//
+// allocate-clause ->
+//    ALLOCATE(
+//        [allocator-simple-modifier:] list) |      // since 5.0
+//    ALLOCATE([modifier...:] list)                 // since 5.1
+// modifier ->
+//    allocator-simple-modifier |
+//    allocator-complex-modifier | align-modifier   // since 5.1
 struct OmpAllocateClause {
-  struct AllocateModifier {
-    WRAPPER_CLASS(Allocator, ScalarIntExpr);
-    WRAPPER_CLASS(Align, ScalarIntExpr);
-    struct ComplexModifier {
-      TUPLE_CLASS_BOILERPLATE(ComplexModifier);
-      std::tuple<Allocator, Align> t;
-    };
-    UNION_CLASS_BOILERPLATE(AllocateModifier);
-    std::variant<Allocator, ComplexModifier, Align> u;
-  };
+  MODIFIER_BOILERPLATE(OmpAlignModifier, OmpAllocatorSimpleModifier,
+      OmpAllocatorComplexModifier);
   TUPLE_CLASS_BOILERPLATE(OmpAllocateClause);
-  std::tuple<std::optional<AllocateModifier>, OmpObjectList> t;
+  std::tuple<MODIFIERS(), OmpObjectList> t;
 };
 
-// OMP 5.0 2.4 atomic-default-mem-order-clause ->
-//                 ATOMIC_DEFAULT_MEM_ORDER (SEQ_CST | ACQ_REL |
-//                                           RELAXED)
+// Ref: [5.0:60-63], [5.1:83-86], [5.2:210-213]
+//
+// atomic-default-mem-order-clause ->
+//    ATOMIC_DEFAULT_MEM_ORDER(memory-order)        // since 5.0
+// memory-order ->
+//    SEQ_CST | ACQ_REL | RELAXED |                 // since 5.0
+//    ACQUIRE | RELEASE                             // since 5.2
 struct OmpAtomicDefaultMemOrderClause {
-  WRAPPER_CLASS_BOILERPLATE(
-      OmpAtomicDefaultMemOrderClause, common::OmpAtomicDefaultMemOrderType);
+  using MemoryOrder = common::OmpAtomicDefaultMemOrderType;
+  WRAPPER_CLASS_BOILERPLATE(OmpAtomicDefaultMemOrderClause, MemoryOrder);
+};
+
+// Ref: [5.0:128-131], [5.1:151-154], [5.2:258-259]
+//
+// bind-clause ->
+//    BIND(binding)                                 // since 5.0
+// binding ->
+//    TEAMS | PARALLEL | THREAD                     // since 5.0
+struct OmpBindClause {
+  ENUM_CLASS(Binding, Parallel, Teams, Thread)
+  WRAPPER_CLASS_BOILERPLATE(OmpBindClause, Binding);
 };
 
-// 2.15.3.1 default-clause -> DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
+// Ref: [4.5:46-50], [5.0:74-78], [5.1:92-96], [5.2:109]
+//
+// default-clause ->
+//    DEFAULT(data-sharing-attribute)               // since 4.5
+// data-sharing-attribute ->
+//    SHARED | NONE |                               // since 4.5
+//    PRIVATE | FIRSTPRIVATE                        // since 5.0
 struct OmpDefaultClause {
-  ENUM_CLASS(Type, Private, Firstprivate, Shared, None)
-  WRAPPER_CLASS_BOILERPLATE(OmpDefaultClause, Type);
+  ENUM_CLASS(DataSharingAttribute, Private, Firstprivate, Shared, None)
+  WRAPPER_CLASS_BOILERPLATE(OmpDefaultClause, DataSharingAttribute);
 };
 
-// 2.15.5.2 defaultmap -> DEFAULTMAP (implicit-behavior[:variable-category])
+// Ref: [4.5:103-107], [5.0:324-325], [5.1:357-358], [5.2:161-162]
+//
+// defaultmap-clause ->
+//    DEFAULTMAP(implicit-behavior
+//        [: variable-category])                    // since 5.0
+// implicit-behavior ->
+//    TOFROM |                                      // since 4.5
+//    ALLOC | TO | FROM | FIRSTPRIVATE | NONE |
+//    DEFAULT |                                     // since 5.0
+//    PRESENT                                       // since 5.1
 struct OmpDefaultmapClause {
   TUPLE_CLASS_BOILERPLATE(OmpDefaultmapClause);
   ENUM_CLASS(
@@ -3689,23 +3814,35 @@ struct OmpDefaultmapClause {
   std::tuple<ImplicitBehavior, MODIFIERS()> t;
 };
 
-// 2.13.9 iteration-offset -> +/- non-negative-constant
+// Ref: [4.5:169-172], [5.0:255-259], [5.1:288-292], [5.2:91-93]
+//
+// iteration-offset ->
+//    +|- non-negative-constant                     // since 4.5
 struct OmpIterationOffset {
   TUPLE_CLASS_BOILERPLATE(OmpIterationOffset);
   std::tuple<DefinedOperator, ScalarIntConstantExpr> t;
 };
 
-// 2.13.9 iteration -> induction-variable [iteration-offset]
+// Ref: [4.5:169-172], [5.0:255-259], [5.1:288-292], [5.2:91-93]
+//
+// iteration ->
+//    induction-variable [iteration-offset]         // since 4.5
 struct OmpIteration {
   TUPLE_CLASS_BOILERPLATE(OmpIteration);
   std::tuple<Name, std::optional<OmpIterationOffset>> t;
 };
 
+// Ref: [4.5:169-172], [5.0:255-259], [5.1:288-292], [5.2:91-93]
+//
+// iteration-vector ->
+//    [iteration...]                                // since 4.5
 WRAPPER_CLASS(OmpIterationVector, std::list<OmpIteration>);
 
 // Extract this into a separate structure (instead of having it directly in
 // OmpDoacrossClause), so that the context in TYPE_CONTEXT_PARSER can be set
 // separately for OmpDependClause and OmpDoacrossClause.
+//
+// See: depend-clause, doacross-clause
 struct OmpDoacross {
   OmpDependenceType::Value GetDepType() const;
 
@@ -3715,54 +3852,74 @@ struct OmpDoacross {
   std::variant<Sink, Source> u;
 };
 
-// Ref: [4.5:169-170], [5.0:255-256], [5.1:288-289], [5.2:323-324]
+// Ref: [4.5:169-172], [5.0:255-259], [5.1:288-292], [5.2:323-326]
 //
 // depend-clause ->
-//    DEPEND(SOURCE) |                               // since 4.5, until 5.1
-//    DEPEND(SINK: iteration-vector) |               // since 4.5, until 5.1
+//    DEPEND(SOURCE) |                              // since 4.5, until 5.1
+//    DEPEND(SINK: iteration-vector) |              // since 4.5, until 5.1
 //    DEPEND([depend-modifier,]
-//           task-dependence-type: locator-list)     // since 4.5
+//           task-dependence-type: locator-list)    // since 4.5
 //
-// depend-modifier -> iterator-modifier              // since 5.0
+// depend-modifier -> iterator-modifier             // since 5.0
 struct OmpDependClause {
   UNION_CLASS_BOILERPLATE(OmpDependClause);
   struct TaskDep {
     OmpTaskDependenceType::Value GetTaskDepType() const;
     TUPLE_CLASS_BOILERPLATE(TaskDep);
-    std::tuple<std::optional<OmpIterator>, OmpTaskDependenceType, OmpObjectList>
-        t;
+    MODIFIER_BOILERPLATE(OmpIterator, OmpTaskDependenceType);
+    std::tuple<MODIFIERS(), OmpObjectList> t;
   };
   std::variant<TaskDep, OmpDoacross> u;
 };
 
+// Ref: [5.2:326-328]
+//
+// doacross-clause ->
+//    DOACROSS(dependence-type: iteration-vector)   // since 5.2
 WRAPPER_CLASS(OmpDoacrossClause, OmpDoacross);
 
 // Ref: [5.0:254-255], [5.1:287-288], [5.2:73]
 //
 // destroy-clause ->
-//    DESTROY |             // since 5.0, until 5.2
-//    DESTROY(variable)     // since 5.2
+//    DESTROY |                                     // since 5.0, until 5.1
+//    DESTROY(variable)                             // since 5.2
 WRAPPER_CLASS(OmpDestroyClause, OmpObject);
 
-// device([ device-modifier :] scalar-integer-expression)
+// Ref: [5.0:135-140], [5.1:161-166], [5.2:265-266]
+//
+// detach-clause ->
+//    DETACH(event-handle)                          // since 5.0
+struct OmpDetachClause {
+  WRAPPER_CLASS_BOILERPLATE(OmpDetachClause, OmpObject);
+};
+
+// Ref: [4.5:103-107], [5.0:170-176], [5.1:197-205], [5.2:276-277]
+//
+// device-clause ->
+//    DEVICE(scalar-integer-expression) |           // since 4.5
+//    DEVICE([device-modifier:]
+//        scalar-integer-expression)                // since 5.0
 struct OmpDeviceClause {
   TUPLE_CLASS_BOILERPLATE(OmpDeviceClause);
-  ENUM_CLASS(DeviceModifier, Ancestor, Device_Num)
-  std::tuple<std::optional<DeviceModifier>, ScalarIntExpr> t;
+  MODIFIER_BOILERPLATE(OmpDeviceModifier);
+  std::tuple<MODIFIERS(), ScalarIntExpr> t;
 };
 
-// device_type(any | host | nohost)
+// Ref: [5.0:180-185], [5.1:210-216], [5.2:275]
+//
+// device-type-clause ->
+//    DEVICE_TYPE(ANY | HOST | NOHOST)              // since 5.0
 struct OmpDeviceTypeClause {
-  ENUM_CLASS(Type, Any, Host, Nohost)
-  WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, Type);
+  ENUM_CLASS(DeviceTypeDescription, Any, Host, Nohost)
+  WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, DeviceTypeDescription);
 };
 
 // Ref: [4.5:107-109], [5.0:176-180], [5.1:205-210], [5.2:167-168]
 //
 // from-clause ->
 //    FROM(locator-list) |
-//    FROM(mapper-modifier: locator-list) |             // since 5.0
-//    FROM(motion-modifier[,] ...: locator-list)        // since 5.1
+//    FROM(mapper-modifier: locator-list) |         // since 5.0
+//    FROM(motion-modifier[,] ...: locator-list)    // since 5.1
 //  motion-modifier ->
 //    PRESENT | mapper-modifier | iterator-modifier
 struct OmpFromClause {
@@ -3771,24 +3928,27 @@ struct OmpFromClause {
   std::tuple<MODIFIERS(), OmpObjectList, bool> t;
 };
 
-// OMP 5.2 12.6.1 grainsize-clause -> grainsize ([prescriptiveness :] value)
+// Ref: [4.5:87-91], [5.0:140-146], [5.1:166-171], [5.2:269]
+//
+// grainsize-clause ->
+//    GRAINSIZE(grain-size) |                       // since 4.5
+//    GRAINSIZE([prescriptiveness:] grain-size)     // since 5.1
 struct OmpGrainsizeClause {
   TUPLE_CLASS_BOILERPLATE(OmpGrainsizeClause);
-  ENUM_CLASS(Prescriptiveness, Strict);
-  std::tuple<std::optional<Prescriptiveness>, ScalarIntExpr> t;
+  MODIFIER_BOILERPLATE(OmpPrescriptiveness);
+  std::tuple<MODIFIERS(), ScalarIntExpr> t;
 };
 
-// 2.12 if-clause -> IF ([ directive-name-modifier :] scalar-logical-expr)
+// Ref: [5.2:72-73], in 4.5-5.1 it's scattered over individual directives
+// that allow the IF clause.
+//
+// if-clause ->
+//    IF([directive-name-modifier:]
+//        scalar-logical-expression)                // since 4.5
 struct OmpIfClause {
   TUPLE_CLASS_BOILERPLATE(OmpIfClause);
-  ENUM_CLASS(DirectiveNameModifier, Parallel, Simd, Target, TargetData,
-      TargetEnterData, TargetExitData, TargetUpdate, Task, Taskloop, Teams)
-  std::tuple<std::optional<DirectiveNameModifier>, ScalarLogicalExpr> t;
-};
-
-// OpenMPv5.2 12.5.2 detach-clause -> DETACH (event-handle)
-struct OmpDetachClause {
-  WRAPPER_CLASS_BOILERPLATE(OmpDetachClause, OmpObject);
+  MODIFIER_BOILERPLATE(OmpDirectiveNameModifier);
+  std::tuple<MODIFIERS(), ScalarLogicalExpr> t;
 };
 
 // OMP 5.0 2.19.5.6 in_reduction-clause -> IN_REDUCTION (reduction-identifier:
@@ -3798,13 +3958,15 @@ struct OmpInReductionClause {
   std::tuple<OmpReductionIdentifier, OmpObjectList> t;
 };
 
-// OMP 5.0 2.19.4.5 lastprivate-clause ->
-//                    LASTPRIVATE ([lastprivate-modifier :] list)
-//                  lastprivate-modifier -> CONDITIONAL
+// Ref: [4.5:199-201], [5.0:288-290], [5.1:321-322], [5.2:115-117]
+//
+// lastprivate-clause ->
+//    LASTPRIVATE(list) |                           // since 4.5
+//    LASTPRIVATE([lastprivate-modifier:] list)     // since 5.0
 struct OmpLastprivateClause {
   TUPLE_CLASS_BOILERPLATE(OmpLastprivateClause);
-  ENUM_CLASS(LastprivateModifier, Conditional);
-  std::tuple<std::optional<LastprivateModifier>, OmpObjectList> t;
+  MODIFIER_BOILERPLATE(OmpLastprivateModifier);
+  std::tuple<MODIFIERS(), OmpObjectList> t;
 };
 
 // 2.15.3.7 linear-clause -> LINEAR (linear-list[ : linear-step])
@@ -3846,6 +4008,17 @@ struct OmpMapClause {
   std::tuple<MODIFIERS(), OmpObjectList, bool> t;
 };
 
+// Ref: [4.5:87-91], [5.0:140-146], [5.1:166-171], [5.2:270]
+//
+// num-tasks-clause ->
+//    NUM_TASKS(num-tasks) |                        // since 4.5
+//    NUM_TASKS([prescriptiveness:] num-tasks)      // since 5.1
+struct OmpNumTasksClause {
+  TUPLE_CLASS_BOILERPLATE(OmpNumTasksClause);
+  MODIFIER_BOILERPLATE(OmpPrescriptiveness);
+  std::tuple<MODIFIERS(), ScalarIntExpr> t;
+};
+
 // Ref: [5.0:101-109], [5.1:126-134], [5.2:233-234]
 //
 // order-clause ->
@@ -3858,10 +4031,16 @@ struct OmpOrderClause {
   std::tuple<MODIFIERS(), Ordering> t;
 };
 
-// 2.5 proc-bind-clause -> PROC_BIND (MASTER | CLOSE | SPREAD)
+// Ref: [4.5:46-50], [5.0:74-78], [5.1:92-96], [5.2:229-230]
+//
+// proc-bind-clause ->
+//    PROC_BIND(affinity-policy)                    // since 4.5
+// affinity-policy ->
+//    CLOSE | PRIMARY | SPREAD |                    // since 4.5
+//    MASTER                                        // since 4.5, until 5.2
 struct OmpProcBindClause {
-  ENUM_CLASS(Type, Close, Master, Spread, Primary)
-  WRAPPER_CLASS_BOILERPLATE(OmpProcBindClause, Type);
+  ENUM_CLASS(AffinityPolicy, Close, Master, Spread, Primary)
+  WRAPPER_CLASS_BOILERPLATE(OmpProcBindClause, AffinityPolicy);
 };
 
 // Ref: [4.5:201-207], [5.0:300-302], [5.1:332-334], [5.2:134-137]
@@ -3907,13 +4086,6 @@ struct OmpToClause {
   std::tuple<MODIFIERS(), OmpObjectList, bool> t;
 };
 
-// OMP 5.2 12.6.2 num_tasks-clause -> num_tasks ([prescriptiveness :] value)
-struct OmpNumTasksClause {
-  TUPLE_CLASS_BOILERPLATE(OmpNumTasksClause);
-  ENUM_CLASS(Prescriptiveness, Strict);
-  std::tuple<std::optional<Prescriptiveness>, ScalarIntExpr> t;
-};
-
 // Ref: [5.0:254-255], [5.1:287-288], [5.2:321-322]
 //
 // update-clause ->
@@ -3922,16 +4094,10 @@ struct OmpNumTasksClause {
 //    UPDATE(task-dependence-type)                  // since 5.2
 struct OmpUpdateClause {
   UNION_CLASS_BOILERPLATE(OmpUpdateClause);
+  // The dependence type is an argument here, not a modifier.
   std::variant<OmpDependenceType, OmpTaskDependenceType> u;
 };
 
-// OMP 5.2 11.7.1 bind-clause ->
-//                  BIND( PARALLEL | TEAMS | THREAD )
-struct OmpBindClause {
-  ENUM_CLASS(Type, Parallel, Teams, Thread)
-  WRAPPER_CLASS_BOILERPLATE(OmpBindClause, Type);
-};
-
 // OpenMP Clauses
 struct OmpClause {
   UNION_CLASS_BOILERPLATE(OmpClause);
@@ -4205,6 +4371,20 @@ struct OmpAtomicCapture {
       t;
 };
 
+struct OmpAtomicCompareIfStmt {
+  UNION_CLASS_BOILERPLATE(OmpAtomicCompareIfStmt);
+  std::variant<common::Indirection<IfStmt>, common::Indirection<IfConstruct>> u;
+};
+
+// ATOMIC COMPARE (OpenMP 5.1, OPenMP 5.2 spec: 15.8.4)
+struct OmpAtomicCompare {
+  TUPLE_CLASS_BOILERPLATE(OmpAtomicCompare);
+  CharBlock source;
+  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
+      OmpAtomicCompareIfStmt, std::optional<OmpEndAtomic>>
+      t;
+};
+
 // ATOMIC
 struct OmpAtomic {
   TUPLE_CLASS_BOILERPLATE(OmpAtomic);
@@ -4217,11 +4397,11 @@ struct OmpAtomic {
 // 2.17.7 atomic ->
 //        ATOMIC [atomic-clause-list] atomic-construct [atomic-clause-list] |
 //        ATOMIC [atomic-clause-list]
-//        atomic-construct -> READ | WRITE | UPDATE | CAPTURE
+//        atomic-construct -> READ | WRITE | UPDATE | CAPTURE | COMPARE
 struct OpenMPAtomicConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPAtomicConstruct);
   std::variant<OmpAtomicRead, OmpAtomicWrite, OmpAtomicCapture, OmpAtomicUpdate,
-      OmpAtomic>
+      OmpAtomicCompare, OmpAtomic>
       u;
 };
 
diff --git a/flang/include/flang/Runtime/CUDA/allocatable.h b/flang/include/flang/Runtime/CUDA/allocatable.h
index e2156281d1b2b..0a96f73b6be44 100644
--- a/flang/include/flang/Runtime/CUDA/allocatable.h
+++ b/flang/include/flang/Runtime/CUDA/allocatable.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_CUDA_ALLOCATABLE_H_
 #define FORTRAN_RUNTIME_CUDA_ALLOCATABLE_H_
 
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/entry-names.h"
 
 namespace Fortran::runtime::cuda {
diff --git a/flang/include/flang/Runtime/CUDA/allocator.h b/flang/include/flang/Runtime/CUDA/allocator.h
index 06bda81c6f75a..4fb4c94c5e9b0 100644
--- a/flang/include/flang/Runtime/CUDA/allocator.h
+++ b/flang/include/flang/Runtime/CUDA/allocator.h
@@ -9,7 +9,8 @@
 #ifndef FORTRAN_RUNTIME_CUDA_ALLOCATOR_H_
 #define FORTRAN_RUNTIME_CUDA_ALLOCATOR_H_
 
-#include "flang/Runtime/descriptor.h"
+#include "common.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/entry-names.h"
 
 namespace Fortran::runtime::cuda {
diff --git a/flang/include/flang/Runtime/CUDA/common.h b/flang/include/flang/Runtime/CUDA/common.h
index e9f61932230e9..474f8e6578b89 100644
--- a/flang/include/flang/Runtime/CUDA/common.h
+++ b/flang/include/flang/Runtime/CUDA/common.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_CUDA_COMMON_H_
 #define FORTRAN_RUNTIME_CUDA_COMMON_H_
 
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/entry-names.h"
 
 /// Type of memory for allocation/deallocation
diff --git a/flang/include/flang/Runtime/CUDA/descriptor.h b/flang/include/flang/Runtime/CUDA/descriptor.h
index 93791012fdcc7..4c6c2c4694fd4 100644
--- a/flang/include/flang/Runtime/CUDA/descriptor.h
+++ b/flang/include/flang/Runtime/CUDA/descriptor.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_CUDA_DESCRIPTOR_H_
 #define FORTRAN_RUNTIME_CUDA_DESCRIPTOR_H_
 
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
 
diff --git a/flang/include/flang/Runtime/CUDA/memory.h b/flang/include/flang/Runtime/CUDA/memory.h
index 2bb083b0dd75c..fbe4941260ff9 100644
--- a/flang/include/flang/Runtime/CUDA/memory.h
+++ b/flang/include/flang/Runtime/CUDA/memory.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_CUDA_MEMORY_H_
 #define FORTRAN_RUNTIME_CUDA_MEMORY_H_
 
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/entry-names.h"
 #include <cstddef>
 
diff --git a/flang/include/flang/Runtime/allocator-registry-consts.h b/flang/include/flang/Runtime/allocator-registry-consts.h
new file mode 100644
index 0000000000000..70735c2fc7a71
--- /dev/null
+++ b/flang/include/flang/Runtime/allocator-registry-consts.h
@@ -0,0 +1,20 @@
+//===-- include/flang/Runtime/allocator-registry-consts.h -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_ALLOCATOR_REGISTRY_CONSTS_H_
+#define FORTRAN_RUNTIME_ALLOCATOR_REGISTRY_CONSTS_H_
+
+static constexpr unsigned kDefaultAllocator = 0;
+
+// Allocator used for CUF
+static constexpr unsigned kPinnedAllocatorPos = 1;
+static constexpr unsigned kDeviceAllocatorPos = 2;
+static constexpr unsigned kManagedAllocatorPos = 3;
+static constexpr unsigned kUnifiedAllocatorPos = 4;
+
+#endif /* FORTRAN_RUNTIME_ALLOCATOR_REGISTRY_CONSTS_H_ */
diff --git a/flang/include/flang/Runtime/allocator-registry.h b/flang/include/flang/Runtime/allocator-registry.h
index 3ccee56dc3fc0..29302c5d825bc 100644
--- a/flang/include/flang/Runtime/allocator-registry.h
+++ b/flang/include/flang/Runtime/allocator-registry.h
@@ -10,17 +10,10 @@
 #define FORTRAN_RUNTIME_ALLOCATOR_REGISTRY_H_
 
 #include "flang/Common/api-attrs.h"
+#include "flang/Runtime/allocator-registry-consts.h"
 #include <cstdlib>
 #include <vector>
 
-static constexpr unsigned kDefaultAllocator = 0;
-
-// Allocator used for CUF
-static constexpr unsigned kPinnedAllocatorPos = 1;
-static constexpr unsigned kDeviceAllocatorPos = 2;
-static constexpr unsigned kManagedAllocatorPos = 3;
-static constexpr unsigned kUnifiedAllocatorPos = 4;
-
 #define MAX_ALLOCATOR 7 // 3 bits are reserved in the descriptor.
 
 namespace Fortran::runtime {
diff --git a/flang/include/flang/Runtime/array-constructor-consts.h b/flang/include/flang/Runtime/array-constructor-consts.h
new file mode 100644
index 0000000000000..ad3583eef29aa
--- /dev/null
+++ b/flang/include/flang/Runtime/array-constructor-consts.h
@@ -0,0 +1,97 @@
+//===-- include/flang/Runtime/array-constructor-consts.h --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_ARRAY_CONSTRUCTOR_CONSTS_H_
+#define FORTRAN_RUNTIME_ARRAY_CONSTRUCTOR_CONSTS_H_
+
+#include "flang/Runtime/descriptor-consts.h"
+#include "flang/Runtime/entry-names.h"
+#include <cstdint>
+
+namespace Fortran::runtime {
+struct ArrayConstructorVector;
+
+// Max sizeof(ArrayConstructorVector) and sizeof(ArrayConstructorVector) for any
+// target.
+// TODO: Use target-specific size/alignment instead of overapproximation.
+constexpr std::size_t MaxArrayConstructorVectorSizeInBytes = 2 * 40;
+constexpr std::size_t MaxArrayConstructorVectorAlignInBytes = 8;
+
+// This file defines an API to "push" an evaluated array constructor value
+// "from" into some storage "to" of an array constructor. It can be seen as a
+// form of std::vector::push_back() implementation for Fortran array
+// constructors. In the APIs and ArrayConstructorVector struct above:
+//
+// - "to" is a ranked-1 descriptor whose declared type is already set to the
+// array constructor derived type. It may be already allocated, even before the
+// first call to this API, or it may be unallocated. "to" extent is increased
+// every time a "from" is pushed past its current extent. At this end of the
+// API calls, its extent is the extent of the array constructor. If "to" is
+// unallocated and its extent is not null, it is assumed this is the final array
+// constructor extent value, and the first allocation already "reserves" storage
+// space accordingly to avoid reallocations.
+//  - "from" is a scalar or array descriptor for the evaluated array
+//  constructor value that must be copied into the storage of "to" at
+//  "nextValuePosition".
+//  - "useValueLengthParameters" must be set to true if the array constructor
+//  has length parameters and no type spec. If it is true and "to" is
+//  unallocated, "to" will take the length parameters of "from". If it is true
+//  and "to" is an allocated character array constructor, it will be checked
+//  that "from" length matches the one from "to". When it is false, the
+//  character length must already be set in "to" before the first call to this
+//  API and "from" character lengths are allowed to mismatch from "to".
+// - "nextValuePosition" is the zero based sequence position of "from" in the
+// array constructor. It is updated after this call by the number of "from"
+// elements. It should be set to zero by the caller of this API before the first
+// call.
+// - "actualAllocationSize" is the current allocation size of "to" storage. It
+// may be bigger than "to" extent for reallocation optimization purposes, but
+// should never be smaller, unless this is the first call and "to" is
+// unallocated. It is updated by the runtime after each successful allocation or
+// reallocation. It should be set to "to" extent if "to" is allocated before the
+// first call of this API, and can be left undefined otherwise.
+//
+// Note that this API can be used with "to" being a variable (that can be
+// discontiguous). This can be done when the variable is the left hand side of
+// an assignment from an array constructor as long as:
+//  - none of the ac-value overlaps with the variable,
+//  - this is an intrinsic assignment that is not a whole allocatable
+//  assignment, *and* for a type that has no components requiring user defined
+//  assignments,
+//  - the variable is properly finalized before using this API if its need to,
+//  - "useValueLengthParameters" should be set to false in this case, even if
+//  the array constructor has no type-spec, since the variable may have a
+//  different character length than the array constructor values.
+
+extern "C" {
+// API to initialize an ArrayConstructorVector before any values are pushed to
+// it. Inlined code is only expected to allocate the "ArrayConstructorVector"
+// class instance storage with sufficient size
+// (MaxArrayConstructorVectorSizeInBytes is expected to be large enough for all
+// supported targets). This avoids the need for the runtime to maintain a state,
+// or to use dynamic allocation for it.
+void RTDECL(InitArrayConstructorVector)(ArrayConstructorVector &vector,
+    Descriptor &to, bool useValueLengthParameters,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+
+// Generic API to push any kind of entity into the array constructor (any
+// Fortran type and any rank).
+void RTDECL(PushArrayConstructorValue)(
+    ArrayConstructorVector &vector, const Descriptor &from);
+
+// API to push scalar array constructor value of:
+//   - a numerical or logical type,
+//   - or a derived type that has no length parameters, and no allocatable
+//   component (that would require deep copies).
+// It requires no descriptor for the value that is passed via its base address.
+void RTDECL(PushArrayConstructorSimpleScalar)(
+    ArrayConstructorVector &vector, void *from);
+} // extern "C"
+} // namespace Fortran::runtime
+
+#endif /* FORTRAN_RUNTIME_ARRAY_CONSTRUCTOR_CONSTS_H_ */
diff --git a/flang/include/flang/Runtime/array-constructor.h b/flang/include/flang/Runtime/array-constructor.h
index 46fc0418c7991..2f6aaae17c650 100644
--- a/flang/include/flang/Runtime/array-constructor.h
+++ b/flang/include/flang/Runtime/array-constructor.h
@@ -12,6 +12,7 @@
 #ifndef FORTRAN_RUNTIME_ARRAYCONSTRUCTOR_H_
 #define FORTRAN_RUNTIME_ARRAYCONSTRUCTOR_H_
 
+#include "flang/Runtime/array-constructor-consts.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/entry-names.h"
 #include <cstdint>
@@ -43,76 +44,14 @@ struct ArrayConstructorVector {
   unsigned char useValueLengthParameters_ : 1;
 };
 
-// This file defines an API to "push" an evaluated array constructor value
-// "from" into some storage "to" of an array constructor. It can be seen as a
-// form of std::vector::push_back() implementation for Fortran array
-// constructors. In the APIs and ArrayConstructorVector struct above:
-//
-// - "to" is a ranked-1 descriptor whose declared type is already set to the
-// array constructor derived type. It may be already allocated, even before the
-// first call to this API, or it may be unallocated. "to" extent is increased
-// every time a "from" is pushed past its current extent. At this end of the
-// API calls, its extent is the extent of the array constructor. If "to" is
-// unallocated and its extent is not null, it is assumed this is the final array
-// constructor extent value, and the first allocation already "reserves" storage
-// space accordingly to avoid reallocations.
-//  - "from" is a scalar or array descriptor for the evaluated array
-//  constructor value that must be copied into the storage of "to" at
-//  "nextValuePosition".
-//  - "useValueLengthParameters" must be set to true if the array constructor
-//  has length parameters and no type spec. If it is true and "to" is
-//  unallocated, "to" will take the length parameters of "from". If it is true
-//  and "to" is an allocated character array constructor, it will be checked
-//  that "from" length matches the one from "to". When it is false, the
-//  character length must already be set in "to" before the first call to this
-//  API and "from" character lengths are allowed to mismatch from "to".
-// - "nextValuePosition" is the zero based sequence position of "from" in the
-// array constructor. It is updated after this call by the number of "from"
-// elements. It should be set to zero by the caller of this API before the first
-// call.
-// - "actualAllocationSize" is the current allocation size of "to" storage. It
-// may be bigger than "to" extent for reallocation optimization purposes, but
-// should never be smaller, unless this is the first call and "to" is
-// unallocated. It is updated by the runtime after each successful allocation or
-// reallocation. It should be set to "to" extent if "to" is allocated before the
-// first call of this API, and can be left undefined otherwise.
-//
-// Note that this API can be used with "to" being a variable (that can be
-// discontiguous). This can be done when the variable is the left hand side of
-// an assignment from an array constructor as long as:
-//  - none of the ac-value overlaps with the variable,
-//  - this is an intrinsic assignment that is not a whole allocatable
-//  assignment, *and* for a type that has no components requiring user defined
-//  assignments,
-//  - the variable is properly finalized before using this API if its need to,
-//  - "useValueLengthParameters" should be set to false in this case, even if
-//  the array constructor has no type-spec, since the variable may have a
-//  different character length than the array constructor values.
-
-extern "C" {
-// API to initialize an ArrayConstructorVector before any values are pushed to
-// it. Inlined code is only expected to allocate the "ArrayConstructorVector"
-// class instance storage with sufficient size (using
-// "2*sizeof(ArrayConstructorVector)" on the host should be safe regardless of
-// the target the runtime is compiled for). This avoids the need for the runtime
-// to maintain a state, or to use dynamic allocation for it. "vectorClassSize"
-// is used to validate that lowering allocated enough space for it.
-void RTDECL(InitArrayConstructorVector)(ArrayConstructorVector &vector,
-    Descriptor &to, bool useValueLengthParameters, int vectorClassSize,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-
-// Generic API to push any kind of entity into the array constructor (any
-// Fortran type and any rank).
-void RTDECL(PushArrayConstructorValue)(
-    ArrayConstructorVector &vector, const Descriptor &from);
+static_assert(sizeof(Fortran::runtime::ArrayConstructorVector) <=
+        MaxArrayConstructorVectorSizeInBytes,
+    "ABI requires sizeof(ArrayConstructorVector) to be smaller than "
+    "MaxArrayConstructorVectorSizeInBytes");
+static_assert(alignof(Fortran::runtime::ArrayConstructorVector) <=
+        MaxArrayConstructorVectorAlignInBytes,
+    "ABI requires alignof(ArrayConstructorVector) to be smaller than "
+    "MaxArrayConstructorVectorAlignInBytes");
 
-// API to push scalar array constructor value of:
-//   - a numerical or logical type,
-//   - or a derived type that has no length parameters, and no allocatable
-//   component (that would require deep copies).
-// It requires no descriptor for the value that is passed via its base address.
-void RTDECL(PushArrayConstructorSimpleScalar)(
-    ArrayConstructorVector &vector, void *from);
-} // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_ARRAYCONSTRUCTOR_H_
diff --git a/flang/include/flang/Runtime/cpp-type.h b/flang/include/flang/Runtime/cpp-type.h
index f6180b4a9ab60..b8c543d34e6fa 100644
--- a/flang/include/flang/Runtime/cpp-type.h
+++ b/flang/include/flang/Runtime/cpp-type.h
@@ -11,7 +11,7 @@
 #ifndef FORTRAN_RUNTIME_CPP_TYPE_H_
 #define FORTRAN_RUNTIME_CPP_TYPE_H_
 
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include "flang/Common/float128.h"
 #include "flang/Common/float80.h"
 #include "flang/Common/uint128.h"
diff --git a/flang/include/flang/Runtime/descriptor-consts.h b/flang/include/flang/Runtime/descriptor-consts.h
new file mode 100644
index 0000000000000..3b2537579d586
--- /dev/null
+++ b/flang/include/flang/Runtime/descriptor-consts.h
@@ -0,0 +1,74 @@
+//===-- include/flang/Runtime/descriptor-consts.h ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_DESCRIPTOR_CONSTS_H_
+#define FORTRAN_RUNTIME_DESCRIPTOR_CONSTS_H_
+
+#include "flang/Common/api-attrs.h"
+#include "flang/ISO_Fortran_binding_wrapper.h"
+#include <cstddef>
+#include <cstdint>
+
+// Value of the addendum presence flag.
+#define _CFI_ADDENDUM_FLAG 1
+// Number of bits needed to be shifted when manipulating the allocator index.
+#define _CFI_ALLOCATOR_IDX_SHIFT 1
+// Allocator index mask.
+#define _CFI_ALLOCATOR_IDX_MASK 0b00001110
+
+namespace Fortran::runtime::typeInfo {
+using TypeParameterValue = std::int64_t;
+class DerivedType;
+} // namespace Fortran::runtime::typeInfo
+
+namespace Fortran::runtime {
+class Descriptor;
+using SubscriptValue = ISO::CFI_index_t;
+
+/// Returns size in bytes of the descriptor (not the data)
+/// This must be at least as large as the largest descriptor of any target
+/// triple.
+static constexpr RT_API_ATTRS std::size_t MaxDescriptorSizeInBytes(
+    int rank, bool addendum = false, int lengthTypeParameters = 0) {
+  // Layout:
+  //
+  // fortran::runtime::Descriptor {
+  //   ISO::CFI_cdesc_t {
+  //     void *base_addr;           (pointer -> up to 8 bytes)
+  //     size_t elem_len;           (up to 8 bytes)
+  //     int version;               (up to 4 bytes)
+  //     CFI_rank_t rank;           (unsigned char -> 1 byte)
+  //     CFI_type_t type;           (signed char -> 1 byte)
+  //     CFI_attribute_t attribute; (unsigned char -> 1 byte)
+  //     unsigned char extra;       (1 byte)
+  //   }
+  // }
+  // fortran::runtime::Dimension[rank] {
+  //   ISO::CFI_dim_t {
+  //     CFI_index_t lower_bound; (ptrdiff_t -> up to 8 bytes)
+  //     CFI_index_t extent;      (ptrdiff_t -> up to 8 bytes)
+  //     CFI_index_t sm;          (ptrdiff_t -> up to 8 bytes)
+  //   }
+  // }
+  // fortran::runtime::DescriptorAddendum {
+  //   const typeInfo::DerivedType *derivedType_;        (pointer -> up to 8
+  //   bytes) typeInfo::TypeParameterValue len_[lenParameters]; (int64_t -> 8
+  //   bytes)
+  // }
+  std::size_t bytes{24u + rank * 24u};
+  if (addendum || lengthTypeParameters > 0) {
+    if (lengthTypeParameters < 1)
+      lengthTypeParameters = 1;
+    bytes += 8u + static_cast<std::size_t>(lengthTypeParameters) * 8u;
+  }
+  return bytes;
+}
+
+} // namespace Fortran::runtime
+
+#endif /* FORTRAN_RUNTIME_DESCRIPTOR_CONSTS_H_ */
diff --git a/flang/include/flang/Runtime/descriptor.h b/flang/include/flang/Runtime/descriptor.h
index 030d0c1031fba..dd36fba157ca9 100644
--- a/flang/include/flang/Runtime/descriptor.h
+++ b/flang/include/flang/Runtime/descriptor.h
@@ -19,6 +19,7 @@
 // but should never reference this internal header.
 
 #include "flang/ISO_Fortran_binding_wrapper.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Runtime/memory.h"
 #include "flang/Runtime/type-code.h"
 #include <algorithm>
@@ -28,14 +29,8 @@
 #include <cstdio>
 #include <cstring>
 
-namespace Fortran::runtime::typeInfo {
-using TypeParameterValue = std::int64_t;
-class DerivedType;
-} // namespace Fortran::runtime::typeInfo
-
 namespace Fortran::runtime {
 
-using SubscriptValue = ISO::CFI_index_t;
 class Terminator;
 
 RT_VAR_GROUP_BEGIN
@@ -420,13 +415,6 @@ class Descriptor {
 
   void Dump(FILE * = stdout) const;
 
-// Value of the addendum presence flag.
-#define _CFI_ADDENDUM_FLAG 1
-// Number of bits needed to be shifted when manipulating the allocator index.
-#define _CFI_ALLOCATOR_IDX_SHIFT 1
-// Allocator index mask.
-#define _CFI_ALLOCATOR_IDX_MASK 0b00001110
-
   RT_API_ATTRS inline bool HasAddendum() const {
     return raw_.extra & _CFI_ADDENDUM_FLAG;
   }
@@ -464,6 +452,8 @@ class alignas(Descriptor) StaticDescriptor {
   static constexpr bool hasAddendum{ADDENDUM || MAX_LEN_PARMS > 0};
   static constexpr std::size_t byteSize{
       Descriptor::SizeInBytes(maxRank, hasAddendum, maxLengthTypeParameters)};
+  static_assert(byteSize <=
+      MaxDescriptorSizeInBytes(maxRank, hasAddendum, maxLengthTypeParameters));
   RT_OFFLOAD_VAR_GROUP_END
 
   RT_API_ATTRS Descriptor &descriptor() {
diff --git a/flang/include/flang/Runtime/exceptions.h b/flang/include/flang/Runtime/exceptions.h
index 1ab22da103a50..bd6c439b150ab 100644
--- a/flang/include/flang/Runtime/exceptions.h
+++ b/flang/include/flang/Runtime/exceptions.h
@@ -24,6 +24,10 @@ extern "C" {
 // This mapping is done at runtime to support cross compilation.
 std::uint32_t RTNAME(MapException)(std::uint32_t excepts);
 
+// Get and set the ieee underflow mode if supported; otherwise nops.
+bool RTNAME(GetUnderflowMode)(void);
+void RTNAME(SetUnderflowMode)(bool flag);
+
 } // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_EXCEPTIONS_H_
diff --git a/flang/include/flang/Runtime/io-api-consts.h b/flang/include/flang/Runtime/io-api-consts.h
new file mode 100644
index 0000000000000..7ed8bf1489b3c
--- /dev/null
+++ b/flang/include/flang/Runtime/io-api-consts.h
@@ -0,0 +1,368 @@
+//===-- include/flang/Runtime/io-api-consts.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_IO_API_CONSTS_H_
+#define FORTRAN_RUNTIME_IO_API_CONSTS_H_
+
+#include "flang/Common/uint128.h"
+#include "flang/Runtime/entry-names.h"
+#include "flang/Runtime/iostat-consts.h"
+#include "flang/Runtime/magic-numbers.h"
+#include <cinttypes>
+#include <cstddef>
+
+namespace Fortran::runtime {
+class Descriptor;
+} // namespace Fortran::runtime
+
+namespace Fortran::runtime::io {
+
+struct NonTbpDefinedIoTable;
+class NamelistGroup;
+class IoStatementState;
+using Cookie = IoStatementState *;
+using ExternalUnit = int;
+using AsynchronousId = int;
+
+static constexpr ExternalUnit DefaultOutputUnit{FORTRAN_DEFAULT_OUTPUT_UNIT};
+static constexpr ExternalUnit DefaultInputUnit{FORTRAN_DEFAULT_INPUT_UNIT};
+
+// INQUIRE specifiers are encoded as simple base-26 packings of
+// the spellings of their keywords.
+using InquiryKeywordHash = std::uint64_t;
+constexpr InquiryKeywordHash HashInquiryKeyword(const char *p) {
+  InquiryKeywordHash hash{1};
+  while (char ch{*p++}) {
+    std::uint64_t letter{0};
+    if (ch >= 'a' && ch <= 'z') {
+      letter = ch - 'a';
+    } else {
+      letter = ch - 'A';
+    }
+    hash = 26 * hash + letter;
+  }
+  return hash;
+}
+
+extern "C" {
+
+#define IONAME(name) RTNAME(io##name)
+
+#ifndef IODECL
+#define IODECL(name) RT_API_ATTRS IONAME(name)
+#endif
+
+#ifndef IODEF
+#define IODEF(name) RT_API_ATTRS IONAME(name)
+#endif
+
+// These functions initiate data transfer statements (READ, WRITE, PRINT).
+// Example: PRINT *, 666 is implemented as the series of calls:
+//   Cookie cookie{BeginExternalListOutput(DefaultOutputUnit,
+//                                         __FILE__, __LINE__)};
+//   OutputInteger32(cookie, 666);
+//   EndIoStatement(cookie);
+// Formatted I/O with explicit formats can supply the format as a
+// const char * pointer with a length, or with a descriptor.
+
+// Internal I/O initiation
+// Internal I/O can loan the runtime library an optional block of memory
+// in which the library can maintain state across the calls that implement
+// the internal transfer; use of these blocks can reduce the need for dynamic
+// memory allocation &/or thread-local storage.  The block must be sufficiently
+// aligned to hold a pointer.
+constexpr std::size_t RecommendedInternalIoScratchAreaBytes(
+    int maxFormatParenthesesNestingDepth) {
+  return 32 + 8 * maxFormatParenthesesNestingDepth;
+}
+
+// For NAMELIST I/O, use the API for the appropriate form of list-directed
+// I/O initiation and configuration, then call OutputNamelist/InputNamelist
+// below.
+
+// Internal I/O to/from character arrays &/or non-default-kind character
+// requires a descriptor, which is copied.
+Cookie IODECL(BeginInternalArrayListOutput)(const Descriptor &,
+    void **scratchArea = nullptr, std::size_t scratchBytes = 0,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginInternalArrayListInput)(const Descriptor &,
+    void **scratchArea = nullptr, std::size_t scratchBytes = 0,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginInternalArrayFormattedOutput)(const Descriptor &,
+    const char *format, std::size_t formatLength,
+    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginInternalArrayFormattedInput)(const Descriptor &,
+    const char *format, std::size_t formatLength,
+    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+
+// Internal I/O to/from a default-kind character scalar can avoid a
+// descriptor.
+Cookie IODECL(BeginInternalListOutput)(char *internal,
+    std::size_t internalLength, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginInternalListInput)(const char *internal,
+    std::size_t internalLength, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginInternalFormattedOutput)(char *internal,
+    std::size_t internalLength, const char *format, std::size_t formatLength,
+    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginInternalFormattedInput)(const char *internal,
+    std::size_t internalLength, const char *format, std::size_t formatLength,
+    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
+    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+
+// External unit numbers must fit in default integers. When the integer
+// provided as UNIT is of a wider type than the default integer, it could
+// overflow when converted to a default integer.
+// CheckUnitNumberInRange should be called to verify that a unit number of a
+// wide integer type can fit in a default integer. Since it should be called
+// before the BeginXXX(unit, ...) call, it has its own error handling interface.
+// If handleError is false, and the unit number is out of range, the program
+// will be terminated. Otherwise, if unit is out of range, a nonzero Iostat
+// code is returned and ioMsg is set if it is not a nullptr.
+enum Iostat IODECL(CheckUnitNumberInRange64)(std::int64_t unit,
+    bool handleError, char *ioMsg = nullptr, std::size_t ioMsgLength = 0,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+enum Iostat IODECL(CheckUnitNumberInRange128)(common::int128_t unit,
+    bool handleError, char *ioMsg = nullptr, std::size_t ioMsgLength = 0,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+
+// External synchronous I/O initiation
+Cookie IODECL(BeginExternalListOutput)(ExternalUnit = DefaultOutputUnit,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginExternalListInput)(ExternalUnit = DefaultInputUnit,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginExternalFormattedOutput)(const char *format, std::size_t,
+    const Descriptor *formatDescriptor = nullptr,
+    ExternalUnit = DefaultOutputUnit, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginExternalFormattedInput)(const char *format, std::size_t,
+    const Descriptor *formatDescriptor = nullptr,
+    ExternalUnit = DefaultInputUnit, const char *sourceFile = nullptr,
+    int sourceLine = 0);
+Cookie IODECL(BeginUnformattedOutput)(ExternalUnit = DefaultOutputUnit,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginUnformattedInput)(ExternalUnit = DefaultInputUnit,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+
+// WAIT(ID=)
+Cookie IODECL(BeginWait)(ExternalUnit, AsynchronousId,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+// WAIT(no ID=)
+Cookie IODECL(BeginWaitAll)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+
+// Other I/O statements
+Cookie IODECL(BeginClose)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginFlush)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginBackspace)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginEndfile)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginRewind)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+
+// OPEN(UNIT=) and OPEN(NEWUNIT=) have distinct interfaces.
+Cookie IODECL(BeginOpenUnit)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginOpenNewUnit)(
+    const char *sourceFile = nullptr, int sourceLine = 0);
+
+// The variant forms of INQUIRE() statements have distinct interfaces.
+// BeginInquireIoLength() is basically a no-op output statement.
+Cookie IODECL(BeginInquireUnit)(
+    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginInquireFile)(const char *, std::size_t,
+    const char *sourceFile = nullptr, int sourceLine = 0);
+Cookie IODECL(BeginInquireIoLength)(
+    const char *sourceFile = nullptr, int sourceLine = 0);
+
+// If an I/O statement has any IOSTAT=, ERR=, END=, or EOR= specifiers,
+// call EnableHandlers() immediately after the Begin...() call.
+// An output or OPEN statement may not enable HasEnd or HasEor.
+// This call makes the runtime library defer those particular error/end
+// conditions to the EndIoStatement() call rather than terminating
+// the image.  E.g., for READ(*,*,END=666) A, B, (C(J),J=1,N)
+//   Cookie cookie{BeginExternalListInput(DefaultInputUnit,__FILE__,__LINE__)};
+//   EnableHandlers(cookie, false, false, true /*END=*/, false);
+//   if (InputReal64(cookie, &A)) {
+//     if (InputReal64(cookie, &B)) {
+//       for (int J{1}; J<=N; ++J) {
+//         if (!InputReal64(cookie, &C[J])) break;
+//       }
+//     }
+//   }
+//   if (EndIoStatement(cookie) == FORTRAN_RUTIME_IOSTAT_END) goto label666;
+void IODECL(EnableHandlers)(Cookie, bool hasIoStat = false, bool hasErr = false,
+    bool hasEnd = false, bool hasEor = false, bool hasIoMsg = false);
+
+// ASYNCHRONOUS='YES' or 'NO' on READ/WRITE/OPEN
+// Use GetAsynchronousId() to handle ID=.
+bool IODECL(SetAsynchronous)(Cookie, const char *, std::size_t);
+
+// Control list options.  These return false on a error that the
+// Begin...() call has specified will be handled by the caller.
+// The interfaces that pass a default-kind CHARACTER argument
+// are limited to passing specific case-insensitive keyword values.
+// ADVANCE=YES, NO
+bool IODECL(SetAdvance)(Cookie, const char *, std::size_t);
+// BLANK=NULL, ZERO
+bool IODECL(SetBlank)(Cookie, const char *, std::size_t);
+// DECIMAL=COMMA, POINT
+bool IODECL(SetDecimal)(Cookie, const char *, std::size_t);
+// DELIM=APOSTROPHE, QUOTE, NONE
+bool IODECL(SetDelim)(Cookie, const char *, std::size_t);
+// PAD=YES, NO
+bool IODECL(SetPad)(Cookie, const char *, std::size_t);
+bool IODECL(SetPos)(Cookie, std::int64_t);
+bool IODECL(SetRec)(Cookie, std::int64_t);
+// ROUND=UP, DOWN, ZERO, NEAREST, COMPATIBLE, PROCESSOR_DEFINED
+bool IODECL(SetRound)(Cookie, const char *, std::size_t);
+// SIGN=PLUS, SUPPRESS, PROCESSOR_DEFINED
+bool IODECL(SetSign)(Cookie, const char *, std::size_t);
+
+// Data item transfer for modes other than NAMELIST:
+// Any data object that can be passed as an actual argument without the
+// use of a temporary can be transferred by means of a descriptor;
+// vector-valued subscripts and coindexing will require elementwise
+// transfers &/or data copies.  Unformatted transfers to/from contiguous
+// blocks of local image memory can avoid the descriptor, and there
+// are specializations for the most common scalar types.
+//
+// These functions return false when the I/O statement has encountered an
+// error or end-of-file/record condition that the caller has indicated
+// should not cause termination of the image by the runtime library.
+// Once the statement has encountered an error, all following items will be
+// ignored and also return false; but compiled code should check for errors
+// and avoid the following items when they might crash.
+bool IODECL(OutputDescriptor)(Cookie, const Descriptor &);
+bool IODECL(InputDescriptor)(Cookie, const Descriptor &);
+// Formatted (including list directed) I/O data items
+bool IODECL(OutputInteger8)(Cookie, std::int8_t);
+bool IODECL(OutputInteger16)(Cookie, std::int16_t);
+bool IODECL(OutputInteger32)(Cookie, std::int32_t);
+bool IODECL(OutputInteger64)(Cookie, std::int64_t);
+bool IODECL(OutputInteger128)(Cookie, common::int128_t);
+bool IODECL(InputInteger)(Cookie, std::int64_t &, int kind = 8);
+bool IODECL(OutputReal32)(Cookie, float);
+bool IODECL(InputReal32)(Cookie, float &);
+bool IODECL(OutputReal64)(Cookie, double);
+bool IODECL(InputReal64)(Cookie, double &);
+bool IODECL(OutputComplex32)(Cookie, float, float);
+bool IODECL(InputComplex32)(Cookie, float[2]);
+bool IODECL(OutputComplex64)(Cookie, double, double);
+bool IODECL(InputComplex64)(Cookie, double[2]);
+bool IODECL(OutputCharacter)(Cookie, const char *, std::size_t, int kind = 1);
+bool IODECL(OutputAscii)(Cookie, const char *, std::size_t);
+bool IODECL(InputCharacter)(Cookie, char *, std::size_t, int kind = 1);
+bool IODECL(InputAscii)(Cookie, char *, std::size_t);
+bool IODECL(OutputLogical)(Cookie, bool);
+bool IODECL(InputLogical)(Cookie, bool &);
+
+// NAMELIST I/O must be the only data item in an (otherwise)
+// list-directed I/O statement.
+bool IODECL(OutputNamelist)(Cookie, const NamelistGroup &);
+bool IODECL(InputNamelist)(Cookie, const NamelistGroup &);
+
+// When an I/O list item has a derived type with a specific defined
+// I/O subroutine of the appropriate generic kind for the active
+// I/O data transfer statement (read/write, formatted/unformatted)
+// that pertains to the type or its components, and those subroutines
+// are dynamic or neither type-bound nor defined with interfaces
+// in the same scope as the derived type (or an IMPORT statement has
+// made such a generic interface inaccessible), these data item transfer
+// APIs enable the I/O runtime to make the right calls to defined I/O
+// subroutines.
+bool IODECL(OutputDerivedType)(
+    Cookie, const Descriptor &, const NonTbpDefinedIoTable *);
+bool IODECL(InputDerivedType)(
+    Cookie, const Descriptor &, const NonTbpDefinedIoTable *);
+
+// Additional specifier interfaces for the connection-list of
+// on OPEN statement (only).  SetBlank(), SetDecimal(),
+// SetDelim(), GetIoMsg(), SetPad(), SetRound(), SetSign(),
+// & SetAsynchronous() are also acceptable for OPEN.
+// ACCESS=SEQUENTIAL, DIRECT, STREAM
+bool IODECL(SetAccess)(Cookie, const char *, std::size_t);
+// ACTION=READ, WRITE, or READWRITE
+bool IODECL(SetAction)(Cookie, const char *, std::size_t);
+// CARRIAGECONTROL=LIST, FORTRAN, NONE
+bool IODECL(SetCarriagecontrol)(Cookie, const char *, std::size_t);
+// CONVERT=NATIVE, LITTLE_ENDIAN, BIG_ENDIAN, or SWAP
+bool IODECL(SetConvert)(Cookie, const char *, std::size_t);
+// ENCODING=UTF-8, DEFAULT
+bool IODECL(SetEncoding)(Cookie, const char *, std::size_t);
+// FORM=FORMATTED, UNFORMATTED
+bool IODECL(SetForm)(Cookie, const char *, std::size_t);
+// POSITION=ASIS, REWIND, APPEND
+bool IODECL(SetPosition)(Cookie, const char *, std::size_t);
+bool IODECL(SetRecl)(Cookie, std::size_t); // RECL=
+
+// STATUS can be set during an OPEN or CLOSE statement.
+// For OPEN: STATUS=OLD, NEW, SCRATCH, REPLACE, UNKNOWN
+// For CLOSE: STATUS=KEEP, DELETE
+bool IODECL(SetStatus)(Cookie, const char *, std::size_t);
+
+bool IODECL(SetFile)(Cookie, const char *, std::size_t chars);
+
+// Acquires the runtime-created unit number for OPEN(NEWUNIT=)
+bool IODECL(GetNewUnit)(Cookie, int &, int kind = 4);
+
+// READ(SIZE=), after all input items
+std::size_t IODECL(GetSize)(Cookie);
+
+// INQUIRE(IOLENGTH=), after all output items
+std::size_t IODECL(GetIoLength)(Cookie);
+
+// GetIoMsg() does not modify its argument unless an error or
+// end-of-record/file condition is present.
+void IODECL(GetIoMsg)(Cookie, char *, std::size_t); // IOMSG=
+
+// Defines ID= on READ/WRITE(ASYNCHRONOUS='YES')
+AsynchronousId IODECL(GetAsynchronousId)(Cookie);
+
+// INQUIRE() specifiers are mostly identified by their NUL-terminated
+// case-insensitive names.
+// ACCESS, ACTION, ASYNCHRONOUS, BLANK, CONVERT, DECIMAL, DELIM, DIRECT,
+// ENCODING, FORM, FORMATTED, NAME, PAD, POSITION, READ, READWRITE, ROUND,
+// SEQUENTIAL, SIGN, STREAM, UNFORMATTED, WRITE:
+bool IODECL(InquireCharacter)(Cookie, InquiryKeywordHash, char *, std::size_t);
+// EXIST, NAMED, OPENED, and PENDING (without ID):
+bool IODECL(InquireLogical)(Cookie, InquiryKeywordHash, bool &);
+// PENDING with ID
+bool IODECL(InquirePendingId)(Cookie, AsynchronousId, bool &);
+// NEXTREC, NUMBER, POS, RECL, SIZE
+bool IODECL(InquireInteger64)(
+    Cookie, InquiryKeywordHash, std::int64_t &, int kind = 8);
+
+// This function must be called to end an I/O statement, and its
+// cookie value may not be used afterwards unless it is recycled
+// by the runtime library to serve a later I/O statement.
+// The return value can be used to implement IOSTAT=, ERR=, END=, & EOR=;
+// store it into the IOSTAT= variable if there is one, and test
+// it to implement the various branches.  The error condition
+// returned is guaranteed to only be one of the problems that the
+// EnableHandlers() call has indicated should be handled in compiled code
+// rather than by terminating the image.
+enum Iostat IODECL(EndIoStatement)(Cookie);
+
+} // extern "C"
+} // namespace Fortran::runtime::io
+
+#endif /* FORTRAN_RUNTIME_IO_API_CONSTS_H_ */
diff --git a/flang/include/flang/Runtime/io-api.h b/flang/include/flang/Runtime/io-api.h
index 328afc715a3f1..b86c3cecb32c5 100644
--- a/flang/include/flang/Runtime/io-api.h
+++ b/flang/include/flang/Runtime/io-api.h
@@ -13,7 +13,8 @@
 
 #include "flang/Common/uint128.h"
 #include "flang/Runtime/entry-names.h"
-#include "flang/Runtime/iostat.h"
+#include "flang/Runtime/io-api-consts.h"
+#include "flang/Runtime/iostat-consts.h"
 #include "flang/Runtime/magic-numbers.h"
 #include <cinttypes>
 #include <cstddef>
@@ -31,342 +32,8 @@ using Cookie = IoStatementState *;
 using ExternalUnit = int;
 using AsynchronousId = int;
 
-static constexpr ExternalUnit DefaultOutputUnit{FORTRAN_DEFAULT_OUTPUT_UNIT};
-static constexpr ExternalUnit DefaultInputUnit{FORTRAN_DEFAULT_INPUT_UNIT};
-
-// INQUIRE specifiers are encoded as simple base-26 packings of
-// the spellings of their keywords.
-using InquiryKeywordHash = std::uint64_t;
-constexpr InquiryKeywordHash HashInquiryKeyword(const char *p) {
-  InquiryKeywordHash hash{1};
-  while (char ch{*p++}) {
-    std::uint64_t letter{0};
-    if (ch >= 'a' && ch <= 'z') {
-      letter = ch - 'a';
-    } else {
-      letter = ch - 'A';
-    }
-    hash = 26 * hash + letter;
-  }
-  return hash;
-}
-
 RT_API_ATTRS const char *InquiryKeywordHashDecode(
     char *buffer, std::size_t, InquiryKeywordHash);
 
-extern "C" {
-
-#define IONAME(name) RTNAME(io##name)
-
-#ifndef IODECL
-#define IODECL(name) RT_API_ATTRS IONAME(name)
-#endif
-
-#ifndef IODEF
-#define IODEF(name) RT_API_ATTRS IONAME(name)
-#endif
-
-// These functions initiate data transfer statements (READ, WRITE, PRINT).
-// Example: PRINT *, 666 is implemented as the series of calls:
-//   Cookie cookie{BeginExternalListOutput(DefaultOutputUnit,
-//                                         __FILE__, __LINE__)};
-//   OutputInteger32(cookie, 666);
-//   EndIoStatement(cookie);
-// Formatted I/O with explicit formats can supply the format as a
-// const char * pointer with a length, or with a descriptor.
-
-// Internal I/O initiation
-// Internal I/O can loan the runtime library an optional block of memory
-// in which the library can maintain state across the calls that implement
-// the internal transfer; use of these blocks can reduce the need for dynamic
-// memory allocation &/or thread-local storage.  The block must be sufficiently
-// aligned to hold a pointer.
-constexpr std::size_t RecommendedInternalIoScratchAreaBytes(
-    int maxFormatParenthesesNestingDepth) {
-  return 32 + 8 * maxFormatParenthesesNestingDepth;
-}
-
-// For NAMELIST I/O, use the API for the appropriate form of list-directed
-// I/O initiation and configuration, then call OutputNamelist/InputNamelist
-// below.
-
-// Internal I/O to/from character arrays &/or non-default-kind character
-// requires a descriptor, which is copied.
-Cookie IODECL(BeginInternalArrayListOutput)(const Descriptor &,
-    void **scratchArea = nullptr, std::size_t scratchBytes = 0,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginInternalArrayListInput)(const Descriptor &,
-    void **scratchArea = nullptr, std::size_t scratchBytes = 0,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginInternalArrayFormattedOutput)(const Descriptor &,
-    const char *format, std::size_t formatLength,
-    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginInternalArrayFormattedInput)(const Descriptor &,
-    const char *format, std::size_t formatLength,
-    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-
-// Internal I/O to/from a default-kind character scalar can avoid a
-// descriptor.
-Cookie IODECL(BeginInternalListOutput)(char *internal,
-    std::size_t internalLength, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginInternalListInput)(const char *internal,
-    std::size_t internalLength, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginInternalFormattedOutput)(char *internal,
-    std::size_t internalLength, const char *format, std::size_t formatLength,
-    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginInternalFormattedInput)(const char *internal,
-    std::size_t internalLength, const char *format, std::size_t formatLength,
-    const Descriptor *formatDescriptor = nullptr, void **scratchArea = nullptr,
-    std::size_t scratchBytes = 0, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-
-// External unit numbers must fit in default integers. When the integer
-// provided as UNIT is of a wider type than the default integer, it could
-// overflow when converted to a default integer.
-// CheckUnitNumberInRange should be called to verify that a unit number of a
-// wide integer type can fit in a default integer. Since it should be called
-// before the BeginXXX(unit, ...) call, it has its own error handling interface.
-// If handleError is false, and the unit number is out of range, the program
-// will be terminated. Otherwise, if unit is out of range, a nonzero Iostat
-// code is returned and ioMsg is set if it is not a nullptr.
-enum Iostat IODECL(CheckUnitNumberInRange64)(std::int64_t unit,
-    bool handleError, char *ioMsg = nullptr, std::size_t ioMsgLength = 0,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-enum Iostat IODECL(CheckUnitNumberInRange128)(common::int128_t unit,
-    bool handleError, char *ioMsg = nullptr, std::size_t ioMsgLength = 0,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-
-// External synchronous I/O initiation
-Cookie IODECL(BeginExternalListOutput)(ExternalUnit = DefaultOutputUnit,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginExternalListInput)(ExternalUnit = DefaultInputUnit,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginExternalFormattedOutput)(const char *format, std::size_t,
-    const Descriptor *formatDescriptor = nullptr,
-    ExternalUnit = DefaultOutputUnit, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginExternalFormattedInput)(const char *format, std::size_t,
-    const Descriptor *formatDescriptor = nullptr,
-    ExternalUnit = DefaultInputUnit, const char *sourceFile = nullptr,
-    int sourceLine = 0);
-Cookie IODECL(BeginUnformattedOutput)(ExternalUnit = DefaultOutputUnit,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginUnformattedInput)(ExternalUnit = DefaultInputUnit,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-
-// WAIT(ID=)
-Cookie IODECL(BeginWait)(ExternalUnit, AsynchronousId,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-// WAIT(no ID=)
-Cookie IODECL(BeginWaitAll)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-
-// Other I/O statements
-Cookie IODECL(BeginClose)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginFlush)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginBackspace)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginEndfile)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginRewind)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-
-// OPEN(UNIT=) and OPEN(NEWUNIT=) have distinct interfaces.
-Cookie IODECL(BeginOpenUnit)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginOpenNewUnit)(
-    const char *sourceFile = nullptr, int sourceLine = 0);
-
-// The variant forms of INQUIRE() statements have distinct interfaces.
-// BeginInquireIoLength() is basically a no-op output statement.
-Cookie IODECL(BeginInquireUnit)(
-    ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginInquireFile)(const char *, std::size_t,
-    const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IODECL(BeginInquireIoLength)(
-    const char *sourceFile = nullptr, int sourceLine = 0);
-
-// If an I/O statement has any IOSTAT=, ERR=, END=, or EOR= specifiers,
-// call EnableHandlers() immediately after the Begin...() call.
-// An output or OPEN statement may not enable HasEnd or HasEor.
-// This call makes the runtime library defer those particular error/end
-// conditions to the EndIoStatement() call rather than terminating
-// the image.  E.g., for READ(*,*,END=666) A, B, (C(J),J=1,N)
-//   Cookie cookie{BeginExternalListInput(DefaultInputUnit,__FILE__,__LINE__)};
-//   EnableHandlers(cookie, false, false, true /*END=*/, false);
-//   if (InputReal64(cookie, &A)) {
-//     if (InputReal64(cookie, &B)) {
-//       for (int J{1}; J<=N; ++J) {
-//         if (!InputReal64(cookie, &C[J])) break;
-//       }
-//     }
-//   }
-//   if (EndIoStatement(cookie) == FORTRAN_RUTIME_IOSTAT_END) goto label666;
-void IODECL(EnableHandlers)(Cookie, bool hasIoStat = false, bool hasErr = false,
-    bool hasEnd = false, bool hasEor = false, bool hasIoMsg = false);
-
-// ASYNCHRONOUS='YES' or 'NO' on READ/WRITE/OPEN
-// Use GetAsynchronousId() to handle ID=.
-bool IODECL(SetAsynchronous)(Cookie, const char *, std::size_t);
-
-// Control list options.  These return false on a error that the
-// Begin...() call has specified will be handled by the caller.
-// The interfaces that pass a default-kind CHARACTER argument
-// are limited to passing specific case-insensitive keyword values.
-// ADVANCE=YES, NO
-bool IODECL(SetAdvance)(Cookie, const char *, std::size_t);
-// BLANK=NULL, ZERO
-bool IODECL(SetBlank)(Cookie, const char *, std::size_t);
-// DECIMAL=COMMA, POINT
-bool IODECL(SetDecimal)(Cookie, const char *, std::size_t);
-// DELIM=APOSTROPHE, QUOTE, NONE
-bool IODECL(SetDelim)(Cookie, const char *, std::size_t);
-// PAD=YES, NO
-bool IODECL(SetPad)(Cookie, const char *, std::size_t);
-bool IODECL(SetPos)(Cookie, std::int64_t);
-bool IODECL(SetRec)(Cookie, std::int64_t);
-// ROUND=UP, DOWN, ZERO, NEAREST, COMPATIBLE, PROCESSOR_DEFINED
-bool IODECL(SetRound)(Cookie, const char *, std::size_t);
-// SIGN=PLUS, SUPPRESS, PROCESSOR_DEFINED
-bool IODECL(SetSign)(Cookie, const char *, std::size_t);
-
-// Data item transfer for modes other than NAMELIST:
-// Any data object that can be passed as an actual argument without the
-// use of a temporary can be transferred by means of a descriptor;
-// vector-valued subscripts and coindexing will require elementwise
-// transfers &/or data copies.  Unformatted transfers to/from contiguous
-// blocks of local image memory can avoid the descriptor, and there
-// are specializations for the most common scalar types.
-//
-// These functions return false when the I/O statement has encountered an
-// error or end-of-file/record condition that the caller has indicated
-// should not cause termination of the image by the runtime library.
-// Once the statement has encountered an error, all following items will be
-// ignored and also return false; but compiled code should check for errors
-// and avoid the following items when they might crash.
-bool IODECL(OutputDescriptor)(Cookie, const Descriptor &);
-bool IODECL(InputDescriptor)(Cookie, const Descriptor &);
-// Formatted (including list directed) I/O data items
-bool IODECL(OutputInteger8)(Cookie, std::int8_t);
-bool IODECL(OutputInteger16)(Cookie, std::int16_t);
-bool IODECL(OutputInteger32)(Cookie, std::int32_t);
-bool IODECL(OutputInteger64)(Cookie, std::int64_t);
-bool IODECL(OutputInteger128)(Cookie, common::int128_t);
-bool IODECL(InputInteger)(Cookie, std::int64_t &, int kind = 8);
-bool IODECL(OutputReal32)(Cookie, float);
-bool IODECL(InputReal32)(Cookie, float &);
-bool IODECL(OutputReal64)(Cookie, double);
-bool IODECL(InputReal64)(Cookie, double &);
-bool IODECL(OutputComplex32)(Cookie, float, float);
-bool IODECL(InputComplex32)(Cookie, float[2]);
-bool IODECL(OutputComplex64)(Cookie, double, double);
-bool IODECL(InputComplex64)(Cookie, double[2]);
-bool IODECL(OutputCharacter)(Cookie, const char *, std::size_t, int kind = 1);
-bool IODECL(OutputAscii)(Cookie, const char *, std::size_t);
-bool IODECL(InputCharacter)(Cookie, char *, std::size_t, int kind = 1);
-bool IODECL(InputAscii)(Cookie, char *, std::size_t);
-bool IODECL(OutputLogical)(Cookie, bool);
-bool IODECL(InputLogical)(Cookie, bool &);
-
-// NAMELIST I/O must be the only data item in an (otherwise)
-// list-directed I/O statement.
-bool IODECL(OutputNamelist)(Cookie, const NamelistGroup &);
-bool IODECL(InputNamelist)(Cookie, const NamelistGroup &);
-
-// When an I/O list item has a derived type with a specific defined
-// I/O subroutine of the appropriate generic kind for the active
-// I/O data transfer statement (read/write, formatted/unformatted)
-// that pertains to the type or its components, and those subroutines
-// are dynamic or neither type-bound nor defined with interfaces
-// in the same scope as the derived type (or an IMPORT statement has
-// made such a generic interface inaccessible), these data item transfer
-// APIs enable the I/O runtime to make the right calls to defined I/O
-// subroutines.
-bool IODECL(OutputDerivedType)(
-    Cookie, const Descriptor &, const NonTbpDefinedIoTable *);
-bool IODECL(InputDerivedType)(
-    Cookie, const Descriptor &, const NonTbpDefinedIoTable *);
-
-// Additional specifier interfaces for the connection-list of
-// on OPEN statement (only).  SetBlank(), SetDecimal(),
-// SetDelim(), GetIoMsg(), SetPad(), SetRound(), SetSign(),
-// & SetAsynchronous() are also acceptable for OPEN.
-// ACCESS=SEQUENTIAL, DIRECT, STREAM
-bool IODECL(SetAccess)(Cookie, const char *, std::size_t);
-// ACTION=READ, WRITE, or READWRITE
-bool IODECL(SetAction)(Cookie, const char *, std::size_t);
-// CARRIAGECONTROL=LIST, FORTRAN, NONE
-bool IODECL(SetCarriagecontrol)(Cookie, const char *, std::size_t);
-// CONVERT=NATIVE, LITTLE_ENDIAN, BIG_ENDIAN, or SWAP
-bool IODECL(SetConvert)(Cookie, const char *, std::size_t);
-// ENCODING=UTF-8, DEFAULT
-bool IODECL(SetEncoding)(Cookie, const char *, std::size_t);
-// FORM=FORMATTED, UNFORMATTED
-bool IODECL(SetForm)(Cookie, const char *, std::size_t);
-// POSITION=ASIS, REWIND, APPEND
-bool IODECL(SetPosition)(Cookie, const char *, std::size_t);
-bool IODECL(SetRecl)(Cookie, std::size_t); // RECL=
-
-// STATUS can be set during an OPEN or CLOSE statement.
-// For OPEN: STATUS=OLD, NEW, SCRATCH, REPLACE, UNKNOWN
-// For CLOSE: STATUS=KEEP, DELETE
-bool IODECL(SetStatus)(Cookie, const char *, std::size_t);
-
-bool IODECL(SetFile)(Cookie, const char *, std::size_t chars);
-
-// Acquires the runtime-created unit number for OPEN(NEWUNIT=)
-bool IODECL(GetNewUnit)(Cookie, int &, int kind = 4);
-
-// READ(SIZE=), after all input items
-std::size_t IODECL(GetSize)(Cookie);
-
-// INQUIRE(IOLENGTH=), after all output items
-std::size_t IODECL(GetIoLength)(Cookie);
-
-// GetIoMsg() does not modify its argument unless an error or
-// end-of-record/file condition is present.
-void IODECL(GetIoMsg)(Cookie, char *, std::size_t); // IOMSG=
-
-// Defines ID= on READ/WRITE(ASYNCHRONOUS='YES')
-AsynchronousId IODECL(GetAsynchronousId)(Cookie);
-
-// INQUIRE() specifiers are mostly identified by their NUL-terminated
-// case-insensitive names.
-// ACCESS, ACTION, ASYNCHRONOUS, BLANK, CONVERT, DECIMAL, DELIM, DIRECT,
-// ENCODING, FORM, FORMATTED, NAME, PAD, POSITION, READ, READWRITE, ROUND,
-// SEQUENTIAL, SIGN, STREAM, UNFORMATTED, WRITE:
-bool IODECL(InquireCharacter)(Cookie, InquiryKeywordHash, char *, std::size_t);
-// EXIST, NAMED, OPENED, and PENDING (without ID):
-bool IODECL(InquireLogical)(Cookie, InquiryKeywordHash, bool &);
-// PENDING with ID
-bool IODECL(InquirePendingId)(Cookie, AsynchronousId, bool &);
-// NEXTREC, NUMBER, POS, RECL, SIZE
-bool IODECL(InquireInteger64)(
-    Cookie, InquiryKeywordHash, std::int64_t &, int kind = 8);
-
-// This function must be called to end an I/O statement, and its
-// cookie value may not be used afterwards unless it is recycled
-// by the runtime library to serve a later I/O statement.
-// The return value can be used to implement IOSTAT=, ERR=, END=, & EOR=;
-// store it into the IOSTAT= variable if there is one, and test
-// it to implement the various branches.  The error condition
-// returned is guaranteed to only be one of the problems that the
-// EnableHandlers() call has indicated should be handled in compiled code
-// rather than by terminating the image.
-enum Iostat IODECL(EndIoStatement)(Cookie);
-
-} // extern "C"
 } // namespace Fortran::runtime::io
 #endif
diff --git a/flang/include/flang/Runtime/iostat-consts.h b/flang/include/flang/Runtime/iostat-consts.h
new file mode 100644
index 0000000000000..26bf75f59fa0d
--- /dev/null
+++ b/flang/include/flang/Runtime/iostat-consts.h
@@ -0,0 +1,93 @@
+//===-- include/flang/Runtime/iostat-consts.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_IOSTAT_CONSTS_H_
+#define FORTRAN_RUNTIME_IOSTAT_CONSTS_H_
+
+#include "flang/Common/api-attrs.h"
+#include "flang/Runtime/magic-numbers.h"
+
+namespace Fortran::runtime::io {
+
+// The value of IOSTAT= is zero when no error, end-of-record,
+// or end-of-file condition has arisen; errors are positive values.
+// (See 12.11.5 in Fortran 2018 for the complete requirements;
+// these constants must match the values of their corresponding
+// named constants in the predefined module ISO_FORTRAN_ENV, so
+// they're actually defined in another magic-numbers.h header file
+// so that they can be included both here and there.)
+enum Iostat {
+  IostatOk = 0, // no error, EOF, or EOR condition
+
+  // These error codes are required by Fortran (see 12.10.2.16-17) to be
+  // negative integer values
+  IostatEnd = FORTRAN_RUNTIME_IOSTAT_END, // end-of-file on input & no error
+  // End-of-record on non-advancing input, no EOF or error
+  IostatEor = FORTRAN_RUNTIME_IOSTAT_EOR,
+
+  // This value is also required to be negative (12.11.5 bullet 6).
+  // It signifies a FLUSH statement on an unflushable unit.
+  IostatUnflushable = FORTRAN_RUNTIME_IOSTAT_FLUSH,
+
+  // Other errors are positive.  We use "errno" values unchanged.
+  // This error is exported in ISO_Fortran_env.
+  IostatInquireInternalUnit = FORTRAN_RUNTIME_IOSTAT_INQUIRE_INTERNAL_UNIT,
+
+  // The remaining error codes are not exported.
+  IostatGenericError = 1001, // see IOMSG= for details
+  IostatRecordWriteOverrun,
+  IostatRecordReadOverrun,
+  IostatInternalWriteOverrun,
+  IostatErrorInFormat,
+  IostatErrorInKeyword,
+  IostatEndfileDirect,
+  IostatEndfileUnwritable,
+  IostatOpenBadRecl,
+  IostatOpenUnknownSize,
+  IostatOpenBadAppend,
+  IostatWriteToReadOnly,
+  IostatReadFromWriteOnly,
+  IostatBackspaceNonSequential,
+  IostatBackspaceAtFirstRecord,
+  IostatRewindNonSequential,
+  IostatWriteAfterEndfile,
+  IostatFormattedIoOnUnformattedUnit,
+  IostatUnformattedIoOnFormattedUnit,
+  IostatListIoOnDirectAccessUnit,
+  IostatUnformattedChildOnFormattedParent,
+  IostatFormattedChildOnUnformattedParent,
+  IostatChildInputFromOutputParent,
+  IostatChildOutputToInputParent,
+  IostatShortRead,
+  IostatMissingTerminator,
+  IostatBadUnformattedRecord,
+  IostatUTF8Decoding,
+  IostatUnitOverflow,
+  IostatBadRealInput,
+  IostatBadScaleFactor,
+  IostatBadAsynchronous,
+  IostatBadWaitUnit,
+  IostatBOZInputOverflow,
+  IostatIntegerInputOverflow,
+  IostatRealInputOverflow,
+  IostatOpenAlreadyConnected,
+  IostatCannotReposition,
+  IostatBadWaitId,
+  IostatTooManyAsyncOps,
+  IostatBadBackspaceUnit,
+  IostatBadUnitNumber,
+  IostatBadFlushUnit,
+  IostatBadOpOnChildUnit,
+  IostatBadNewUnit,
+  IostatBadListDirectedInputSeparator,
+  IostatNonExternalDefinedUnformattedIo,
+};
+
+} // namespace Fortran::runtime::io
+
+#endif // FORTRAN_RUNTIME_IOSTAT_CONSTS_H_
diff --git a/flang/include/flang/Runtime/iostat.h b/flang/include/flang/Runtime/iostat.h
index 6ce7c82b424eb..d8db68a3a1c2e 100644
--- a/flang/include/flang/Runtime/iostat.h
+++ b/flang/include/flang/Runtime/iostat.h
@@ -11,83 +11,11 @@
 
 #ifndef FORTRAN_RUNTIME_IOSTAT_H_
 #define FORTRAN_RUNTIME_IOSTAT_H_
-#include "flang/Common/api-attrs.h"
-#include "flang/Runtime/magic-numbers.h"
-namespace Fortran::runtime::io {
-
-// The value of IOSTAT= is zero when no error, end-of-record,
-// or end-of-file condition has arisen; errors are positive values.
-// (See 12.11.5 in Fortran 2018 for the complete requirements;
-// these constants must match the values of their corresponding
-// named constants in the predefined module ISO_FORTRAN_ENV, so
-// they're actually defined in another magic-numbers.h header file
-// so that they can be included both here and there.)
-enum Iostat {
-  IostatOk = 0, // no error, EOF, or EOR condition
-
-  // These error codes are required by Fortran (see 12.10.2.16-17) to be
-  // negative integer values
-  IostatEnd = FORTRAN_RUNTIME_IOSTAT_END, // end-of-file on input & no error
-  // End-of-record on non-advancing input, no EOF or error
-  IostatEor = FORTRAN_RUNTIME_IOSTAT_EOR,
 
-  // This value is also required to be negative (12.11.5 bullet 6).
-  // It signifies a FLUSH statement on an unflushable unit.
-  IostatUnflushable = FORTRAN_RUNTIME_IOSTAT_FLUSH,
-
-  // Other errors are positive.  We use "errno" values unchanged.
-  // This error is exported in ISO_Fortran_env.
-  IostatInquireInternalUnit = FORTRAN_RUNTIME_IOSTAT_INQUIRE_INTERNAL_UNIT,
+#include "flang/Common/api-attrs.h"
+#include "flang/Runtime/iostat-consts.h"
 
-  // The remaining error codes are not exported.
-  IostatGenericError = 1001, // see IOMSG= for details
-  IostatRecordWriteOverrun,
-  IostatRecordReadOverrun,
-  IostatInternalWriteOverrun,
-  IostatErrorInFormat,
-  IostatErrorInKeyword,
-  IostatEndfileDirect,
-  IostatEndfileUnwritable,
-  IostatOpenBadRecl,
-  IostatOpenUnknownSize,
-  IostatOpenBadAppend,
-  IostatWriteToReadOnly,
-  IostatReadFromWriteOnly,
-  IostatBackspaceNonSequential,
-  IostatBackspaceAtFirstRecord,
-  IostatRewindNonSequential,
-  IostatWriteAfterEndfile,
-  IostatFormattedIoOnUnformattedUnit,
-  IostatUnformattedIoOnFormattedUnit,
-  IostatListIoOnDirectAccessUnit,
-  IostatUnformattedChildOnFormattedParent,
-  IostatFormattedChildOnUnformattedParent,
-  IostatChildInputFromOutputParent,
-  IostatChildOutputToInputParent,
-  IostatShortRead,
-  IostatMissingTerminator,
-  IostatBadUnformattedRecord,
-  IostatUTF8Decoding,
-  IostatUnitOverflow,
-  IostatBadRealInput,
-  IostatBadScaleFactor,
-  IostatBadAsynchronous,
-  IostatBadWaitUnit,
-  IostatBOZInputOverflow,
-  IostatIntegerInputOverflow,
-  IostatRealInputOverflow,
-  IostatOpenAlreadyConnected,
-  IostatCannotReposition,
-  IostatBadWaitId,
-  IostatTooManyAsyncOps,
-  IostatBadBackspaceUnit,
-  IostatBadUnitNumber,
-  IostatBadFlushUnit,
-  IostatBadOpOnChildUnit,
-  IostatBadNewUnit,
-  IostatBadListDirectedInputSeparator,
-  IostatNonExternalDefinedUnformattedIo,
-};
+namespace Fortran::runtime::io {
 
 RT_API_ATTRS const char *IostatErrorString(int);
 
diff --git a/flang/include/flang/Runtime/stop.h b/flang/include/flang/Runtime/stop.h
index f7c4ffe7403e8..24ae2cbe01ec6 100644
--- a/flang/include/flang/Runtime/stop.h
+++ b/flang/include/flang/Runtime/stop.h
@@ -11,6 +11,7 @@
 
 #include "flang/Runtime/c-or-cpp.h"
 #include "flang/Runtime/entry-names.h"
+#include "flang/Runtime/extensions.h"
 #include <stdlib.h>
 
 FORTRAN_EXTERN_C_BEGIN
@@ -29,6 +30,7 @@ NORETURN void RTNAME(ProgramEndStatement)(NO_ARGUMENTS);
 // Extensions
 NORETURN void RTNAME(Exit)(int status DEFAULT_VALUE(EXIT_SUCCESS));
 NORETURN void RTNAME(Abort)(NO_ARGUMENTS);
+void FORTRAN_PROCEDURE_NAME(backtrace)(NO_ARGUMENTS);
 
 // Crash with an error message when the program dynamically violates a Fortran
 // constraint.
diff --git a/flang/include/flang/Runtime/type-code.h b/flang/include/flang/Runtime/type-code.h
index 8e7314e0af1ef..dd3a9f2690ee7 100644
--- a/flang/include/flang/Runtime/type-code.h
+++ b/flang/include/flang/Runtime/type-code.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_TYPE_CODE_H_
 #define FORTRAN_RUNTIME_TYPE_CODE_H_
 
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include "flang/Common/optional.h"
 #include "flang/ISO_Fortran_binding_wrapper.h"
 #include <utility>
diff --git a/flang/include/flang/Semantics/openmp-modifiers.h b/flang/include/flang/Semantics/openmp-modifiers.h
index 60f116e6f0033..4025ce112d9ca 100644
--- a/flang/include/flang/Semantics/openmp-modifiers.h
+++ b/flang/include/flang/Semantics/openmp-modifiers.h
@@ -67,16 +67,24 @@ template <typename SpecificTy> const OmpModifierDescriptor &OmpGetDescriptor();
 #define DECLARE_DESCRIPTOR(name) \
   template <> const OmpModifierDescriptor &OmpGetDescriptor<name>()
 
+DECLARE_DESCRIPTOR(parser::OmpAlignment);
+DECLARE_DESCRIPTOR(parser::OmpAlignModifier);
+DECLARE_DESCRIPTOR(parser::OmpAllocatorComplexModifier);
+DECLARE_DESCRIPTOR(parser::OmpAllocatorSimpleModifier);
 DECLARE_DESCRIPTOR(parser::OmpChunkModifier);
 DECLARE_DESCRIPTOR(parser::OmpDependenceType);
+DECLARE_DESCRIPTOR(parser::OmpDeviceModifier);
+DECLARE_DESCRIPTOR(parser::OmpDirectiveNameModifier);
 DECLARE_DESCRIPTOR(parser::OmpExpectation);
 DECLARE_DESCRIPTOR(parser::OmpIterator);
+DECLARE_DESCRIPTOR(parser::OmpLastprivateModifier);
 DECLARE_DESCRIPTOR(parser::OmpLinearModifier);
 DECLARE_DESCRIPTOR(parser::OmpMapper);
 DECLARE_DESCRIPTOR(parser::OmpMapType);
 DECLARE_DESCRIPTOR(parser::OmpMapTypeModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderModifier);
 DECLARE_DESCRIPTOR(parser::OmpOrderingModifier);
+DECLARE_DESCRIPTOR(parser::OmpPrescriptiveness);
 DECLARE_DESCRIPTOR(parser::OmpReductionIdentifier);
 DECLARE_DESCRIPTOR(parser::OmpReductionModifier);
 DECLARE_DESCRIPTOR(parser::OmpTaskDependenceType);
@@ -216,10 +224,26 @@ OmpGetRepeatableModifier(const std::optional<std::list<UnionTy>> &modifiers) {
       OmpSpecificModifierIterator(items, items->end()));
 }
 
+// Attempt to prevent creating a range based on an expiring modifier list.
 template <typename SpecificTy, typename UnionTy>
 llvm::iterator_range<OmpSpecificModifierIterator<SpecificTy>>
 OmpGetRepeatableModifier(std::optional<std::list<UnionTy>> &&) = delete;
 
+template <typename SpecificTy, typename UnionTy>
+Fortran::parser::CharBlock OmpGetModifierSource(
+    const std::optional<std::list<UnionTy>> &modifiers,
+    const SpecificTy *specific) {
+  if (!modifiers || !specific) {
+    return Fortran::parser::CharBlock{};
+  }
+  for (auto &m : *modifiers) {
+    if (std::get_if<SpecificTy>(&m.u) == specific) {
+      return m.source;
+    }
+  }
+  llvm_unreachable("`specific` must be a member of `modifiers`");
+}
+
 namespace detail {
 template <typename T> constexpr const T *make_nullptr() {
   return static_cast<const T *>(nullptr);
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index d936b739e5815..c0091e1c953b8 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -122,7 +122,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   bool NoSignedZerosFPMath =
       false; ///< Set no-signed-zeros-fp-math attribute for functions.
   bool UnsafeFPMath = false; ///< Set unsafe-fp-math attribute for functions.
-  bool NSWOnLoopVarInc = false; ///< Add nsw flag to loop variable increments.
+  bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
   bool EnableOpenMP = false; ///< Enable OpenMP lowering.
 };
 
diff --git a/flang/include/flang/Tools/TargetSetup.h b/flang/include/flang/Tools/TargetSetup.h
index f52b5ddaa8d49..1889140ddce75 100644
--- a/flang/include/flang/Tools/TargetSetup.h
+++ b/flang/include/flang/Tools/TargetSetup.h
@@ -29,6 +29,11 @@ namespace Fortran::tools {
     targetCharacteristics.DisableType(
         Fortran::common::TypeCategory::Real, /*kind=*/10);
   }
+  if (targetTriple.getArch() == llvm::Triple::ArchType::x86_64) {
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/3);
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/4);
+    targetCharacteristics.set_hasSubnormalFlushingControl(/*kind=*/8);
+  }
 
   // Figure out if we can support F128: see
   // flang/runtime/Float128Math/math-entries.h
diff --git a/flang/lib/Common/CMakeLists.txt b/flang/lib/Common/CMakeLists.txt
index be72391847f3d..de6bea396f3cb 100644
--- a/flang/lib/Common/CMakeLists.txt
+++ b/flang/lib/Common/CMakeLists.txt
@@ -40,9 +40,13 @@ add_flang_library(FortranCommon
   default-kinds.cpp
   idioms.cpp
   LangOptions.cpp
+  OpenMP-utils.cpp
   Version.cpp
   ${version_inc}
 
   LINK_COMPONENTS
   Support
+
+  LINK_LIBS
+  MLIRIR
 )
diff --git a/flang/lib/Common/Fortran-features.cpp b/flang/lib/Common/Fortran-features.cpp
index f47a4f17a6ba4..c86432874b8d8 100644
--- a/flang/lib/Common/Fortran-features.cpp
+++ b/flang/lib/Common/Fortran-features.cpp
@@ -43,6 +43,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::BadBranchTarget);
   warnLanguage_.set(LanguageFeature::HollerithPolymorphic);
   warnLanguage_.set(LanguageFeature::ListDirectedSize);
+  warnLanguage_.set(LanguageFeature::IgnoreIrrelevantAttributes);
   warnUsage_.set(UsageWarning::ShortArrayActual);
   warnUsage_.set(UsageWarning::FoldingException);
   warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);
diff --git a/flang/lib/Common/OpenMP-utils.cpp b/flang/lib/Common/OpenMP-utils.cpp
new file mode 100644
index 0000000000000..f5115f475d6a1
--- /dev/null
+++ b/flang/lib/Common/OpenMP-utils.cpp
@@ -0,0 +1,47 @@
+//===-- include/flang/Common/OpenMP-utils.cpp ------------------*- C++ -*-====//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Common/OpenMP-utils.h"
+
+#include "mlir/IR/OpDefinition.h"
+
+namespace Fortran::common::openmp {
+mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
+    mlir::Region &region) {
+  assert(args.isValid() && "invalid args");
+  assert(region.empty() && "non-empty region");
+
+  llvm::SmallVector<mlir::Type> types;
+  llvm::SmallVector<mlir::Location> locs;
+  unsigned numVars = args.inReduction.vars.size() + args.map.vars.size() +
+      args.priv.vars.size() + args.reduction.vars.size() +
+      args.taskReduction.vars.size() + args.useDeviceAddr.vars.size() +
+      args.useDevicePtr.vars.size();
+  types.reserve(numVars);
+  locs.reserve(numVars);
+
+  auto extractTypeLoc = [&types, &locs](llvm::ArrayRef<mlir::Value> vals) {
+    llvm::transform(vals, std::back_inserter(types),
+        [](mlir::Value v) { return v.getType(); });
+    llvm::transform(vals, std::back_inserter(locs),
+        [](mlir::Value v) { return v.getLoc(); });
+  };
+
+  // Populate block arguments in clause name alphabetical order to match
+  // expected order by the BlockArgOpenMPOpInterface.
+  extractTypeLoc(args.inReduction.vars);
+  extractTypeLoc(args.map.vars);
+  extractTypeLoc(args.priv.vars);
+  extractTypeLoc(args.reduction.vars);
+  extractTypeLoc(args.taskReduction.vars);
+  extractTypeLoc(args.useDeviceAddr.vars);
+  extractTypeLoc(args.useDevicePtr.vars);
+
+  return builder.createBlock(&region, {}, types, locs);
+}
+} // namespace Fortran::common::openmp
diff --git a/flang/lib/Evaluate/fold-logical.cpp b/flang/lib/Evaluate/fold-logical.cpp
index f5bbe7e429335..6f24f09e78dd9 100644
--- a/flang/lib/Evaluate/fold-logical.cpp
+++ b/flang/lib/Evaluate/fold-logical.cpp
@@ -890,8 +890,16 @@ Expr<Type<TypeCategory::Logical, KIND>> FoldIntrinsicFunction(
     return Expr<T>{context.targetCharacteristics().ieeeFeatures().test(
         IeeeFeature::Subnormal)};
   } else if (name == "__builtin_ieee_support_underflow_control") {
-    return Expr<T>{context.targetCharacteristics().ieeeFeatures().test(
-        IeeeFeature::UnderflowControl)};
+    // Setting kind=0 checks subnormal flushing control across all type kinds.
+    if (args[0]) {
+      return Expr<T>{
+          context.targetCharacteristics().hasSubnormalFlushingControl(
+              args[0]->GetType().value().kind())};
+    } else {
+      return Expr<T>{
+          context.targetCharacteristics().hasSubnormalFlushingControl(
+              /*any=*/false)};
+    }
   }
   return Expr<T>{std::move(funcRef)};
 }
diff --git a/flang/lib/Evaluate/target.cpp b/flang/lib/Evaluate/target.cpp
index 1e2cf6b0d298d..5a556b339c930 100644
--- a/flang/lib/Evaluate/target.cpp
+++ b/flang/lib/Evaluate/target.cpp
@@ -17,7 +17,7 @@ Rounding TargetCharacteristics::defaultRounding;
 
 TargetCharacteristics::TargetCharacteristics() {
   auto enableCategoryKinds{[this](TypeCategory category) {
-    for (int kind{0}; kind < maxKind; ++kind) {
+    for (int kind{1}; kind <= maxKind; ++kind) {
       if (CanSupportType(category, kind)) {
         auto byteSize{static_cast<std::size_t>(kind)};
         if (category == TypeCategory::Real ||
@@ -70,14 +70,14 @@ bool TargetCharacteristics::EnableType(common::TypeCategory category,
 
 void TargetCharacteristics::DisableType(
     common::TypeCategory category, std::int64_t kind) {
-  if (kind >= 0 && kind < maxKind) {
+  if (kind > 0 && kind <= maxKind) {
     align_[static_cast<int>(category)][kind] = 0;
   }
 }
 
 std::size_t TargetCharacteristics::GetByteSize(
     common::TypeCategory category, std::int64_t kind) const {
-  if (kind >= 0 && kind < maxKind) {
+  if (kind > 0 && kind <= maxKind) {
     return byteSize_[static_cast<int>(category)][kind];
   } else {
     return 0;
@@ -86,7 +86,7 @@ std::size_t TargetCharacteristics::GetByteSize(
 
 std::size_t TargetCharacteristics::GetAlignment(
     common::TypeCategory category, std::int64_t kind) const {
-  if (kind >= 0 && kind < maxKind) {
+  if (kind > 0 && kind <= maxKind) {
     return align_[static_cast<int>(category)][kind];
   } else {
     return 0;
@@ -108,6 +108,30 @@ void TargetCharacteristics::set_areSubnormalsFlushedToZero(bool yes) {
   areSubnormalsFlushedToZero_ = yes;
 }
 
+// Check if a given real kind has flushing control.
+bool TargetCharacteristics::hasSubnormalFlushingControl(int kind) const {
+  CHECK(kind > 0 && kind <= maxKind);
+  CHECK(CanSupportType(TypeCategory::Real, kind));
+  return hasSubnormalFlushingControl_[kind];
+}
+
+// Check if any or all real kinds have flushing control.
+bool TargetCharacteristics::hasSubnormalFlushingControl(bool any) const {
+  for (int kind{1}; kind <= maxKind; ++kind) {
+    if (CanSupportType(TypeCategory::Real, kind) &&
+        hasSubnormalFlushingControl_[kind] == any) {
+      return any;
+    }
+  }
+  return !any;
+}
+
+void TargetCharacteristics::set_hasSubnormalFlushingControl(
+    int kind, bool yes) {
+  CHECK(kind > 0 && kind <= maxKind);
+  hasSubnormalFlushingControl_[kind] = yes;
+}
+
 void TargetCharacteristics::set_roundingMode(Rounding rounding) {
   roundingMode_ = rounding;
 }
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 0b79c95eade0d..648b88e84051c 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1362,12 +1362,6 @@ bool CompilerInvocation::createFromArgs(
     invoc.loweringOpts.setNoPPCNativeVecElemOrder(true);
   }
 
-  // -flang-experimental-integer-overflow
-  if (args.hasArg(
-          clang::driver::options::OPT_flang_experimental_integer_overflow)) {
-    invoc.loweringOpts.setNSWOnLoopVarInc(true);
-  }
-
   // Preserve all the remark options requested, i.e. -Rpass, -Rpass-missed or
   // -Rpass-analysis. This will be used later when processing and outputting the
   // remarks generated by LLVM in ExecuteCompilerInvocation.cpp.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 8c21fe18e67b4..77631f70dfd19 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -836,8 +836,8 @@ void CodeGenAction::generateLLVMIR() {
           Fortran::common::LanguageFeature::OpenMP))
     config.EnableOpenMP = true;
 
-  if (ci.getInvocation().getLoweringOpts().getNSWOnLoopVarInc())
-    config.NSWOnLoopVarInc = true;
+  if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
+    config.NSWOnLoopVarInc = false;
 
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 17b58604da3bf..de2b941b688be 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -39,6 +39,7 @@
 #include "flang/Optimizer/Builder/Runtime/Character.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Runtime/EnvironmentDefaults.h"
+#include "flang/Optimizer/Builder/Runtime/Exceptions.h"
 #include "flang/Optimizer/Builder/Runtime/Main.h"
 #include "flang/Optimizer/Builder/Runtime/Ragged.h"
 #include "flang/Optimizer/Builder/Runtime/Stop.h"
@@ -55,7 +56,7 @@
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Parser/parse-tree.h"
-#include "flang/Runtime/iostat.h"
+#include "flang/Runtime/iostat-consts.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
@@ -710,8 +711,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     return bool(shallowLookupSymbol(sym));
   }
 
-  bool createHostAssociateVarClone(
-      const Fortran::semantics::Symbol &sym) override final {
+  bool createHostAssociateVarClone(const Fortran::semantics::Symbol &sym,
+                                   bool skipDefaultInit) override final {
     mlir::Location loc = genLocation(sym.name());
     mlir::Type symType = genType(sym);
     const auto *details = sym.detailsIf<Fortran::semantics::HostAssocDetails>();
@@ -768,13 +769,21 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // Initialise cloned allocatable
     hexv.match(
         [&](const fir::MutableBoxValue &box) -> void {
-          // Do not process pointers
+          const auto new_box = exv.getBoxOf<fir::MutableBoxValue>();
           if (Fortran::semantics::IsPointer(sym.GetUltimate())) {
+            // Establish the pointer descriptors. The rank and type code/size
+            // at least must be set properly for later inquiry of the pointer
+            // to work, and new pointers are always given disassociated status
+            // by flang for safety, even if this is not required by the
+            // language.
+            auto empty = fir::factory::createUnallocatedBox(
+                *builder, loc, new_box->getBoxTy(), box.nonDeferredLenParams(),
+                {});
+            builder->create<fir::StoreOp>(loc, empty, new_box->getAddr());
             return;
           }
-          // Allocate storage for a pointer/allocatble descriptor.
-          // No shape/lengths to be passed to the alloca.
-          const auto new_box = exv.getBoxOf<fir::MutableBoxValue>();
+          // Copy allocation status of Allocatables, creating new storage if
+          // needed.
 
           // allocate if allocated
           mlir::Value isAllocated =
@@ -822,7 +831,22 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           if_builder.end();
         },
         [&](const auto &) -> void {
-          // Do nothing
+          if (skipDefaultInit)
+            return;
+          // Initialize local/private derived types with default
+          // initialization (Fortran 2023 section 11.1.7.5 and OpenMP 5.2
+          // section 5.3). Pointer and allocatable components, when allowed,
+          // also need to be established so that flang runtime can later work
+          // with them.
+          if (const Fortran::semantics::DeclTypeSpec *declTypeSpec =
+                  sym.GetType())
+            if (const Fortran::semantics::DerivedTypeSpec *derivedTypeSpec =
+                    declTypeSpec->AsDerived())
+              if (derivedTypeSpec->HasDefaultInitialization(
+                      /*ignoreAllocatable=*/false, /*ignorePointer=*/false)) {
+                mlir::Value box = builder->createBox(loc, exv);
+                fir::runtime::genDerivedTypeInitialize(*builder, loc, box);
+              }
         });
 
     return bindIfNewSymbol(sym, exv);
@@ -1965,9 +1989,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     Fortran::semantics::SemanticsContext &semanticsContext =
         bridge.getSemanticsContext();
     for (const Fortran::semantics::Symbol *sym : info.localSymList)
-      createHostAssociateVarClone(*sym);
+      createHostAssociateVarClone(*sym, /*skipDefaultInit=*/false);
     for (const Fortran::semantics::Symbol *sym : info.localInitSymList) {
-      createHostAssociateVarClone(*sym);
+      createHostAssociateVarClone(*sym, /*skipDefaultInit=*/true);
       const auto *hostDetails =
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
       assert(hostDetails && "missing locality spec host symbol");
@@ -1985,6 +2009,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
       copySymbolBinding(hostDetails->symbol(), *sym);
     }
+    // Note that allocatable, types with ultimate components, and type
+    // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130),
+    // so no clean-up needs to be generated for these entities.
   }
 
   /// Generate FIR for a DO construct. There are six variants:
@@ -2137,6 +2164,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
     mlir::Operation *boundsAndStepIP = nullptr;
+    mlir::arith::IntegerOverflowFlags iofBackup{};
 
     for (IncrementLoopInfo &info : incrementLoopNestInfo) {
       mlir::Value lowerValue;
@@ -2153,11 +2181,18 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
         info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
                                                    info.isUnordered);
+        if (!getLoweringOptions().getIntegerWrapAround()) {
+          iofBackup = builder->getIntegerOverflowFlags();
+          builder->setIntegerOverflowFlags(
+              mlir::arith::IntegerOverflowFlags::nsw);
+        }
         lowerValue = genControlValue(info.lowerExpr, info);
         upperValue = genControlValue(info.upperExpr, info);
         bool isConst = true;
         stepValue = genControlValue(info.stepExpr, info,
                                     info.isStructured() ? nullptr : &isConst);
+        if (!getLoweringOptions().getIntegerWrapAround())
+          builder->setIntegerOverflowFlags(iofBackup);
         boundsAndStepIP = stepValue.getDefiningOp();
 
         // Use a temp variable for unstructured loops with non-const step.
@@ -2295,7 +2330,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
     mlir::arith::IntegerOverflowFlags flags{};
-    if (getLoweringOptions().getNSWOnLoopVarInc())
+    if (!getLoweringOptions().getIntegerWrapAround())
       flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
     auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
         builder->getContext(), flags);
@@ -3002,8 +3037,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           fir::getBase(genExprValue(*Fortran::semantics::GetExpr(bounds->upper),
                                     stmtCtx))));
       if (bounds->step)
-        steps.push_back(fir::getBase(
-            genExprValue(*Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
+        steps.push_back(builder->createConvert(
+            crtLoc, idxTy,
+            fir::getBase(genExprValue(
+                *Fortran::semantics::GetExpr(bounds->step), stmtCtx))));
       else // If `step` is not present, assume it is `1`.
         steps.push_back(builder->createIntegerConstant(loc, idxTy, 1));
 
@@ -5173,8 +5210,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       genOpenMPSymbolProperties(*this, var);
   }
 
-  /// Where applicable, save the exception state and halting and rounding
-  /// modes at function entry and restore them at function exits.
+  /// Where applicable, save the exception state and halting, rounding, and
+  /// underflow modes at function entry, and restore them at function exits.
   void manageFPEnvironment(Fortran::lower::pft::FunctionLikeUnit &funit) {
     mlir::Location loc = toLocation();
     mlir::Location endLoc =
@@ -5216,7 +5253,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       });
     }
     if (funit.mayModifyRoundingMode) {
-      // F18 Clause 17.4.5: In a procedure [...], the processor shall not
+      // F18 Clause 17.4p5: In a procedure [...], the processor shall not
       // change the rounding modes on entry, and on return shall ensure that
       // the rounding modes are the same as they were on entry.
       mlir::func::FuncOp getRounding =
@@ -5229,6 +5266,18 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         builder->create<fir::CallOp>(endLoc, setRounding, roundingMode);
       });
     }
+    if ((funit.mayModifyUnderflowMode) &&
+        (bridge.getTargetCharacteristics().hasSubnormalFlushingControl(
+            /*any=*/true))) {
+      // F18 Clause 17.5p2: In a procedure [...], the processor shall not
+      // change the underflow mode on entry, and on return shall ensure that
+      // the underflow mode is the same as it was on entry.
+      mlir::Value underflowMode =
+          fir::runtime::genGetUnderflowMode(*builder, loc);
+      bridge.fctCtx().attachCleanup([=]() {
+        fir::runtime::genSetUnderflowMode(*builder, loc, {underflowMode});
+      });
+    }
   }
 
   /// Start translation of a function.
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index e84e7afbe82e0..40cd106e63018 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -284,7 +284,8 @@ static void remapActualToDummyDescriptors(
   }
 }
 
-std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
+std::pair<Fortran::lower::LoweredResult, bool>
+Fortran::lower::genCallOpAndResult(
     mlir::Location loc, Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symMap, Fortran::lower::StatementContext &stmtCtx,
     Fortran::lower::CallerInterface &caller, mlir::FunctionType callSiteType,
@@ -326,6 +327,11 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
     }
   }
 
+  const bool isExprCall =
+      converter.getLoweringOptions().getLowerToHighLevelFIR() &&
+      callSiteType.getNumResults() == 1 &&
+      llvm::isa<fir::SequenceType>(callSiteType.getResult(0));
+
   mlir::IndexType idxTy = builder.getIndexType();
   auto lowerSpecExpr = [&](const auto &expr) -> mlir::Value {
     mlir::Value convertExpr = builder.createConvert(
@@ -333,6 +339,8 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
     return fir::factory::genMaxWithZero(builder, loc, convertExpr);
   };
   llvm::SmallVector<mlir::Value> resultLengths;
+  mlir::Value arrayResultShape;
+  hlfir::EvaluateInMemoryOp evaluateInMemory;
   auto allocatedResult = [&]() -> std::optional<fir::ExtendedValue> {
     llvm::SmallVector<mlir::Value> extents;
     llvm::SmallVector<mlir::Value> lengths;
@@ -366,6 +374,18 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
       resultLengths = lengths;
     }
 
+    if (!extents.empty())
+      arrayResultShape = builder.genShape(loc, extents);
+
+    if (isExprCall) {
+      mlir::Type exprType = hlfir::getExprType(type);
+      evaluateInMemory = builder.create<hlfir::EvaluateInMemoryOp>(
+          loc, exprType, arrayResultShape, resultLengths);
+      builder.setInsertionPointToStart(&evaluateInMemory.getBody().front());
+      return toExtendedValue(loc, evaluateInMemory.getMemory(), extents,
+                             lengths);
+    }
+
     if ((!extents.empty() || !lengths.empty()) && !isElemental) {
       // Note: in the elemental context, the alloca ownership inside the
       // elemental region is implicit, and later pass in lowering (stack
@@ -384,8 +404,7 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
   if (mustPopSymMap)
     symMap.popScope();
 
-  // Place allocated result or prepare the fir.save_result arguments.
-  mlir::Value arrayResultShape;
+  // Place allocated result
   if (allocatedResult) {
     if (std::optional<Fortran::lower::CallInterface<
             Fortran::lower::CallerInterface>::PassedEntity>
@@ -399,16 +418,6 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
       else
         fir::emitFatalError(
             loc, "only expect character scalar result to be passed by ref");
-    } else {
-      assert(caller.mustSaveResult());
-      arrayResultShape = allocatedResult->match(
-          [&](const fir::CharArrayBoxValue &) {
-            return builder.createShape(loc, *allocatedResult);
-          },
-          [&](const fir::ArrayBoxValue &) {
-            return builder.createShape(loc, *allocatedResult);
-          },
-          [&](const auto &) { return mlir::Value{}; });
     }
   }
 
@@ -642,6 +651,19 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
       callResult = call.getResult(0);
   }
 
+  std::optional<Fortran::evaluate::DynamicType> retTy =
+      caller.getCallDescription().proc().GetType();
+  // With HLFIR lowering, isElemental must be set to true
+  // if we are producing an elemental call. In this case,
+  // the elemental results must not be destroyed, instead,
+  // the resulting array result will be finalized/destroyed
+  // as needed by hlfir.destroy.
+  const bool mustFinalizeResult =
+      !isElemental && callSiteType.getNumResults() > 0 &&
+      !fir::isPointerType(callSiteType.getResult(0)) && retTy.has_value() &&
+      (retTy->category() == Fortran::common::TypeCategory::Derived ||
+       retTy->IsPolymorphic() || retTy->IsUnlimitedPolymorphic());
+
   if (caller.mustSaveResult()) {
     assert(allocatedResult.has_value());
     builder.create<fir::SaveResultOp>(loc, callResult,
@@ -649,6 +671,19 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
                                       arrayResultShape, resultLengths);
   }
 
+  if (evaluateInMemory) {
+    builder.setInsertionPointAfter(evaluateInMemory);
+    mlir::Value expr = evaluateInMemory.getResult();
+    fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
+    if (!isElemental)
+      stmtCtx.attachCleanup([bldr, loc, expr, mustFinalizeResult]() {
+        bldr->create<hlfir::DestroyOp>(loc, expr,
+                                       /*finalize=*/mustFinalizeResult);
+      });
+    return {LoweredResult{hlfir::EntityWithAttributes{expr}},
+            mustFinalizeResult};
+  }
+
   if (allocatedResult) {
     // The result must be optionally destroyed (if it is of a derived type
     // that may need finalization or deallocation of the components).
@@ -679,17 +714,7 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
     // derived-type.
     // For polymorphic and unlimited polymorphic enities call the runtime
     // in any cases.
-    std::optional<Fortran::evaluate::DynamicType> retTy =
-        caller.getCallDescription().proc().GetType();
-    // With HLFIR lowering, isElemental must be set to true
-    // if we are producing an elemental call. In this case,
-    // the elemental results must not be destroyed, instead,
-    // the resulting array result will be finalized/destroyed
-    // as needed by hlfir.destroy.
-    if (!isElemental && !fir::isPointerType(funcType.getResults()[0]) &&
-        retTy &&
-        (retTy->category() == Fortran::common::TypeCategory::Derived ||
-         retTy->IsPolymorphic() || retTy->IsUnlimitedPolymorphic())) {
+    if (mustFinalizeResult) {
       if (retTy->IsPolymorphic() || retTy->IsUnlimitedPolymorphic()) {
         auto *bldr = &converter.getFirOpBuilder();
         stmtCtx.attachCleanup([bldr, loc, allocatedResult]() {
@@ -715,12 +740,13 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
         }
       }
     }
-    return {*allocatedResult, resultIsFinalized};
+    return {LoweredResult{*allocatedResult}, resultIsFinalized};
   }
 
   // subroutine call
   if (!resultType)
-    return {fir::ExtendedValue{mlir::Value{}}, /*resultIsFinalized=*/false};
+    return {LoweredResult{fir::ExtendedValue{mlir::Value{}}},
+            /*resultIsFinalized=*/false};
 
   // For now, Fortran return values are implemented with a single MLIR
   // function return value.
@@ -734,10 +760,13 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
         mlir::dyn_cast<fir::CharacterType>(funcType.getResults()[0]);
     mlir::Value len = builder.createIntegerConstant(
         loc, builder.getCharacterLengthType(), charTy.getLen());
-    return {fir::CharBoxValue{callResult, len}, /*resultIsFinalized=*/false};
+    return {
+        LoweredResult{fir::ExtendedValue{fir::CharBoxValue{callResult, len}}},
+        /*resultIsFinalized=*/false};
   }
 
-  return {callResult, /*resultIsFinalized=*/false};
+  return {LoweredResult{fir::ExtendedValue{callResult}},
+          /*resultIsFinalized=*/false};
 }
 
 static hlfir::EntityWithAttributes genStmtFunctionRef(
@@ -1661,19 +1690,25 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals,
   // Prepare lowered arguments according to the interface
   // and map the lowered values to the dummy
   // arguments.
-  auto [result, resultIsFinalized] = Fortran::lower::genCallOpAndResult(
+  auto [loweredResult, resultIsFinalized] = Fortran::lower::genCallOpAndResult(
       loc, callContext.converter, callContext.symMap, callContext.stmtCtx,
       caller, callSiteType, callContext.resultType,
       callContext.isElementalProcWithArrayArgs());
-  // For procedure pointer function result, just return the call.
-  if (callContext.resultType &&
-      mlir::isa<fir::BoxProcType>(*callContext.resultType))
-    return hlfir::EntityWithAttributes(fir::getBase(result));
 
   /// Clean-up associations and copy-in.
   for (auto cleanUp : callCleanUps)
     cleanUp.genCleanUp(loc, builder);
 
+  if (auto *entity = std::get_if<hlfir::EntityWithAttributes>(&loweredResult))
+    return *entity;
+
+  auto &result = std::get<fir::ExtendedValue>(loweredResult);
+
+  // For procedure pointer function result, just return the call.
+  if (callContext.resultType &&
+      mlir::isa<fir::BoxProcType>(*callContext.resultType))
+    return hlfir::EntityWithAttributes(fir::getBase(result));
+
   if (!fir::getBase(result))
     return std::nullopt; // subroutine call.
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 46168b81dd3a0..7698fac89c223 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -2852,10 +2852,11 @@ class ScalarExprLowering {
       }
     }
 
-    ExtValue result =
+    auto loweredResult =
         Fortran::lower::genCallOpAndResult(loc, converter, symMap, stmtCtx,
                                            caller, callSiteType, resultType)
             .first;
+    auto &result = std::get<ExtValue>(loweredResult);
 
     // Sync pointers and allocatables that may have been modified during the
     // call.
@@ -4881,10 +4882,12 @@ class ArrayExprLowering {
             [&](const auto &) { return fir::getBase(exv); });
         caller.placeInput(argIface, arg);
       }
-      return Fortran::lower::genCallOpAndResult(loc, converter, symMap,
-                                                getElementCtx(), caller,
-                                                callSiteType, retTy)
-          .first;
+      Fortran::lower::LoweredResult res =
+          Fortran::lower::genCallOpAndResult(loc, converter, symMap,
+                                             getElementCtx(), caller,
+                                             callSiteType, retTy)
+              .first;
+      return std::get<ExtValue>(res);
     };
   }
 
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index e93fbc562f9b1..4ab319b016caf 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1696,18 +1696,17 @@ class HlfirBuilder {
     // required chains of hlfir.designate to address the parent components so
     // that the StructureConstructor can later be lowered by addressing these
     // parent components if needed. Note: the front-end orders the components in
-    // structure constructors. The code below relies on the component to appear
-    // in order.
+    // structure constructors.
     using ValueAndParent = std::tuple<const Fortran::lower::SomeExpr &,
                                       const Fortran::semantics::Symbol &,
                                       hlfir::EntityWithAttributes>;
     llvm::SmallVector<ValueAndParent> valuesAndParents;
-    Fortran::lower::ComponentReverseIterator compIterator(
-        ctor.result().derivedTypeSpec());
-    hlfir::EntityWithAttributes currentParent = varOp;
     for (const auto &value : llvm::reverse(ctor.values())) {
       const Fortran::semantics::Symbol &compSym = *value.first;
-      while (!compIterator.lookup(compSym.name())) {
+      hlfir::EntityWithAttributes currentParent = varOp;
+      for (Fortran::lower::ComponentReverseIterator compIterator(
+               ctor.result().derivedTypeSpec());
+           !compIterator.lookup(compSym.name());) {
         const auto &parentType = compIterator.advanceToParentType();
         llvm::StringRef parentName = toStringRef(parentType.name());
         auto baseRecTy = mlir::cast<fir::RecordType>(
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 197e526973b4d..ff122c21e37ad 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -39,7 +39,7 @@
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Support/Utils.h"
-#include "flang/Runtime/allocator-registry.h"
+#include "flang/Runtime/allocator-registry-consts.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 310b62697f710..9d3cd3a5c8fa1 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -159,6 +159,17 @@ class HlfirCharExtremumLowering : public HlfirTransformationalIntrinsic {
   hlfir::CharExtremumPredicate pred;
 };
 
+class HlfirCShiftLowering : public HlfirTransformationalIntrinsic {
+public:
+  using HlfirTransformationalIntrinsic::HlfirTransformationalIntrinsic;
+
+protected:
+  mlir::Value
+  lowerImpl(const Fortran::lower::PreparedActualArguments &loweredActuals,
+            const fir::IntrinsicArgumentLoweringRules *argLowering,
+            mlir::Type stmtResultType) override;
+};
+
 } // namespace
 
 mlir::Value HlfirTransformationalIntrinsic::loadBoxAddress(
@@ -270,11 +281,12 @@ HlfirTransformationalIntrinsic::computeResultType(mlir::Value argArray,
         hlfir::ExprType::Shape{array.getShape()};
     mlir::Type elementType = array.getEleTy();
     return hlfir::ExprType::get(builder.getContext(), resultShape, elementType,
-                                /*polymorphic=*/false);
+                                fir::isPolymorphicType(stmtResultType));
   } else if (auto resCharType =
                  mlir::dyn_cast<fir::CharacterType>(stmtResultType)) {
     normalisedResult = hlfir::ExprType::get(
-        builder.getContext(), hlfir::ExprType::Shape{}, resCharType, false);
+        builder.getContext(), hlfir::ExprType::Shape{}, resCharType,
+        /*polymorphic=*/false);
   }
   return normalisedResult;
 }
@@ -387,6 +399,26 @@ mlir::Value HlfirCharExtremumLowering::lowerImpl(
   return createOp<hlfir::CharExtremumOp>(pred, mlir::ValueRange{operands});
 }
 
+mlir::Value HlfirCShiftLowering::lowerImpl(
+    const Fortran::lower::PreparedActualArguments &loweredActuals,
+    const fir::IntrinsicArgumentLoweringRules *argLowering,
+    mlir::Type stmtResultType) {
+  auto operands = getOperandVector(loweredActuals, argLowering);
+  assert(operands.size() == 3);
+  mlir::Value dim = operands[2];
+  if (!dim) {
+    // If DIM is not present, drop the last element which is a null Value.
+    operands.truncate(2);
+  } else {
+    // If DIM is present, then dereference it if it is a ref.
+    dim = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{dim});
+    operands[2] = dim;
+  }
+
+  mlir::Type resultType = computeResultType(operands[0], stmtResultType);
+  return createOp<hlfir::CShiftOp>(resultType, operands);
+}
+
 std::optional<hlfir::EntityWithAttributes> Fortran::lower::lowerHlfirIntrinsic(
     fir::FirOpBuilder &builder, mlir::Location loc, const std::string &name,
     const Fortran::lower::PreparedActualArguments &loweredActuals,
@@ -432,6 +464,9 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::lowerHlfirIntrinsic(
   if (name == "maxloc")
     return HlfirMaxlocLowering{builder, loc}.lower(loweredActuals, argLowering,
                                                    stmtResultType);
+  if (name == "cshift")
+    return HlfirCShiftLowering{builder, loc}.lower(loweredActuals, argLowering,
+                                                   stmtResultType);
   if (mlir::isa<fir::CharacterType>(stmtResultType)) {
     if (name == "min")
       return HlfirCharExtremumLowering{builder, loc,
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 1894b0cfd1bec..89e0aa2228655 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -33,7 +33,7 @@
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Parser/parse-tree.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/tools.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
@@ -929,7 +929,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
   mlir::arith::IntegerOverflowFlags flags{};
-  if (converter.getLoweringOptions().getNSWOnLoopVarInc())
+  if (!converter.getLoweringOptions().getIntegerWrapAround())
     flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
   auto iofAttr =
       mlir::arith::IntegerOverflowFlagsAttr::get(builder.getContext(), flags);
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 878dccc4ecbc4..75dcf6ec3e110 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -139,6 +139,8 @@ createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
   op.setStructured(structured);
   op.setImplicit(implicit);
   op.setDataClause(dataClause);
+  op.setVarType(mlir::cast<mlir::acc::PointerLikeType>(baseAddr.getType())
+                    .getElementType());
   op->setAttr(Op::getOperandSegmentSizeAttr(),
               builder.getDenseI32ArrayAttr(operandSegments));
   if (!asyncDeviceTypes.empty())
@@ -266,8 +268,8 @@ static void createDeclareDeallocFuncWithArg(
   if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp> ||
                 std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
     builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(),
-                           entryOp.getVarPtr(), entryOp.getBounds(),
-                           entryOp.getAsyncOperands(),
+                           entryOp.getVarPtr(), entryOp.getVarType(),
+                           entryOp.getBounds(), entryOp.getAsyncOperands(),
                            entryOp.getAsyncOperandsDeviceTypeAttr(),
                            entryOp.getAsyncOnlyAttr(), entryOp.getDataClause(),
                            /*structured=*/false, /*implicit=*/false,
@@ -450,7 +452,7 @@ static void genDataExitOperations(fir::FirOpBuilder &builder,
                   std::is_same_v<ExitOp, mlir::acc::UpdateHostOp>)
       builder.create<ExitOp>(
           entryOp.getLoc(), entryOp.getAccPtr(), entryOp.getVarPtr(),
-          entryOp.getBounds(), entryOp.getAsyncOperands(),
+          entryOp.getVarType(), entryOp.getBounds(), entryOp.getAsyncOperands(),
           entryOp.getAsyncOperandsDeviceTypeAttr(), entryOp.getAsyncOnlyAttr(),
           entryOp.getDataClause(), structured, entryOp.getImplicit(),
           builder.getStringAttr(*entryOp.getName()));
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 6baa22a44eafb..48c559a78b9bc 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -81,12 +81,8 @@ genAllocateClause(lower::AbstractConverter &converter,
   // Check if allocate clause has allocator specified. If so, add it
   // to list of allocators, otherwise, add default allocator to
   // list of allocators.
-  using SimpleModifier = Allocate::AllocatorSimpleModifier;
   using ComplexModifier = Allocate::AllocatorComplexModifier;
-  if (auto &mod = std::get<std::optional<SimpleModifier>>(clause.t)) {
-    mlir::Value operand = fir::getBase(converter.genExprValue(*mod, stmtCtx));
-    allocatorOperands.append(objects.size(), operand);
-  } else if (auto &mod = std::get<std::optional<ComplexModifier>>(clause.t)) {
+  if (auto &mod = std::get<std::optional<ComplexModifier>>(clause.t)) {
     mlir::Value operand = fir::getBase(converter.genExprValue(mod->v, stmtCtx));
     allocatorOperands.append(objects.size(), operand);
   } else {
@@ -462,6 +458,16 @@ bool ClauseProcessor::processPriority(
   return false;
 }
 
+bool ClauseProcessor::processDetach(mlir::omp::DetachClauseOps &result) const {
+  if (auto *clause = findUniqueClause<omp::clause::Detach>()) {
+    semantics::Symbol *sym = clause->v.sym();
+    mlir::Value symVal = converter.getSymbolAddress(*sym);
+    result.eventHandle = symVal;
+    return true;
+  }
+  return false;
+}
+
 bool ClauseProcessor::processProcBind(
     mlir::omp::ProcBindClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::ProcBind>()) {
@@ -744,6 +750,7 @@ createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
   mlir::func::FuncOp funcOp =
       modBuilder.create<mlir::func::FuncOp>(loc, copyFuncName, funcType);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
+  fir::factory::setInternalLinkage(funcOp);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
                       {loc, loc});
   builder.setInsertionPointToStart(&funcOp.getRegion().back());
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 217d7c6917bd6..e0fe917c50e8f 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -90,6 +90,7 @@ class ClauseProcessor {
                           mlir::omp::ThreadLimitClauseOps &result) const;
   bool processUntied(mlir::omp::UntiedClauseOps &result) const;
 
+  bool processDetach(mlir::omp::DetachClauseOps &result) const;
   // 'Repeatable' clauses: They can appear multiple times in the clause list.
   bool processAligned(mlir::omp::AlignedClauseOps &result) const;
   bool processAllocate(mlir::omp::AllocateClauseOps &result) const;
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index bf20f42bdecaf..10c31963ec493 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -369,6 +369,15 @@ clause::DependenceType makeDepType(const parser::OmpTaskDependenceType &inp) {
   llvm_unreachable("Unexpected task dependence type");
 }
 
+clause::Prescriptiveness
+makePrescriptiveness(parser::OmpPrescriptiveness::Value v) {
+  switch (v) {
+  case parser::OmpPrescriptiveness::Value::Strict:
+    return clause::Prescriptiveness::Strict;
+  }
+  llvm_unreachable("Unexpected prescriptiveness");
+}
+
 // --------------------------------------------------------------------
 // Actual clauses. Each T (where tomp::T exists in ClauseT) has its "make".
 
@@ -384,11 +393,12 @@ Absent make(const parser::OmpClause::Absent &inp,
 Affinity make(const parser::OmpClause::Affinity &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpAffinityClause
-  auto &t0 = std::get<std::optional<parser::OmpIterator>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
 
   auto &&maybeIter =
-      maybeApply([&](auto &&s) { return makeIterator(s, semaCtx); }, t0);
+      m0 ? makeIterator(*m0, semaCtx) : std::optional<Iterator>{};
 
   return Affinity{{/*Iterator=*/std::move(maybeIter),
                    /*LocatorList=*/makeObjects(t1, semaCtx)}};
@@ -403,11 +413,12 @@ Align make(const parser::OmpClause::Align &inp,
 Aligned make(const parser::OmpClause::Aligned &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpAlignedClause
+  auto &mods = semantics::OmpGetModifiers(inp.v);
   auto &t0 = std::get<parser::OmpObjectList>(inp.v.t);
-  auto &t1 = std::get<std::optional<parser::ScalarIntConstantExpr>>(inp.v.t);
+  auto *m1 = semantics::OmpGetUniqueModifier<parser::OmpAlignment>(mods);
 
   return Aligned{{
-      /*Alignment=*/maybeApply(makeExprFn(semaCtx), t1),
+      /*Alignment=*/maybeApplyToV(makeExprFn(semaCtx), m1),
       /*List=*/makeObjects(t0, semaCtx),
   }};
 }
@@ -415,47 +426,29 @@ Aligned make(const parser::OmpClause::Aligned &inp,
 Allocate make(const parser::OmpClause::Allocate &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpAllocateClause
-  using wrapped = parser::OmpAllocateClause;
-  auto &t0 = std::get<std::optional<wrapped::AllocateModifier>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpAlignModifier>(mods);
+  auto *m1 =
+      semantics::OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(
+          mods);
+  auto *m2 =
+      semantics::OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(mods);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  if (!t0) {
-    return Allocate{{/*AllocatorSimpleModifier=*/std::nullopt,
-                     /*AllocatorComplexModifier=*/std::nullopt,
-                     /*AlignModifier=*/std::nullopt,
-                     /*List=*/makeObjects(t1, semaCtx)}};
-  }
+  auto makeAllocator = [&](auto *mod) -> std::optional<Allocator> {
+    if (mod)
+      return Allocator{makeExpr(mod->v, semaCtx)};
+    return std::nullopt;
+  };
 
-  using Tuple = decltype(Allocate::t);
+  auto makeAlign = [&](const parser::ScalarIntExpr &expr) {
+    return Align{makeExpr(expr, semaCtx)};
+  };
 
-  return Allocate{Fortran::common::visit(
-      common::visitors{
-          // simple-modifier
-          [&](const wrapped::AllocateModifier::Allocator &v) -> Tuple {
-            return {/*AllocatorSimpleModifier=*/makeExpr(v.v, semaCtx),
-                    /*AllocatorComplexModifier=*/std::nullopt,
-                    /*AlignModifier=*/std::nullopt,
-                    /*List=*/makeObjects(t1, semaCtx)};
-          },
-          // complex-modifier + align-modifier
-          [&](const wrapped::AllocateModifier::ComplexModifier &v) -> Tuple {
-            auto &s0 = std::get<wrapped::AllocateModifier::Allocator>(v.t);
-            auto &s1 = std::get<wrapped::AllocateModifier::Align>(v.t);
-            return {
-                /*AllocatorSimpleModifier=*/std::nullopt,
-                /*AllocatorComplexModifier=*/Allocator{makeExpr(s0.v, semaCtx)},
-                /*AlignModifier=*/Align{makeExpr(s1.v, semaCtx)},
-                /*List=*/makeObjects(t1, semaCtx)};
-          },
-          // align-modifier
-          [&](const wrapped::AllocateModifier::Align &v) -> Tuple {
-            return {/*AllocatorSimpleModifier=*/std::nullopt,
-                    /*AllocatorComplexModifier=*/std::nullopt,
-                    /*AlignModifier=*/Align{makeExpr(v.v, semaCtx)},
-                    /*List=*/makeObjects(t1, semaCtx)};
-          },
-      },
-      t0->u)};
+  auto maybeAllocator = m1 ? makeAllocator(m1) : makeAllocator(m2);
+  return Allocate{{/*AllocatorComplexModifier=*/std::move(maybeAllocator),
+                   /*AlignModifier=*/maybeApplyToV(makeAlign, m0),
+                   /*List=*/makeObjects(t1, semaCtx)}};
 }
 
 Allocator make(const parser::OmpClause::Allocator &inp,
@@ -496,7 +489,7 @@ Bind make(const parser::OmpClause::Bind &inp,
   using wrapped = parser::OmpBindClause;
 
   CLAUSET_ENUM_CONVERT( //
-      convert, wrapped::Type, Bind::Binding,
+      convert, wrapped::Binding, Bind::Binding,
       // clang-format off
       MS(Teams, Teams)
       MS(Parallel, Parallel)
@@ -541,7 +534,7 @@ Default make(const parser::OmpClause::Default &inp,
   using wrapped = parser::OmpDefaultClause;
 
   CLAUSET_ENUM_CONVERT( //
-      convert, wrapped::Type, Default::DataSharingAttribute,
+      convert, wrapped::DataSharingAttribute, Default::DataSharingAttribute,
       // clang-format off
       MS(Firstprivate, Firstprivate)
       MS(None,         None)
@@ -631,15 +624,18 @@ Depend make(const parser::OmpClause::Depend &inp,
   using Variant = decltype(Depend::u);
 
   auto visitTaskDep = [&](const wrapped::TaskDep &s) -> Variant {
-    auto &t0 = std::get<std::optional<parser::OmpIterator>>(s.t);
-    auto &t1 = std::get<parser::OmpTaskDependenceType>(s.t);
-    auto &t2 = std::get<parser::OmpObjectList>(s.t);
+    auto &mods = semantics::OmpGetModifiers(s);
+    auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpIterator>(mods);
+    auto *m1 =
+        semantics::OmpGetUniqueModifier<parser::OmpTaskDependenceType>(mods);
+    auto &t1 = std::get<parser::OmpObjectList>(s.t);
+    assert(m1 && "expecting task dependence type");
 
     auto &&maybeIter =
-        maybeApply([&](auto &&s) { return makeIterator(s, semaCtx); }, t0);
-    return Depend::TaskDep{{/*DependenceType=*/makeDepType(t1),
+        m0 ? makeIterator(*m0, semaCtx) : std::optional<Iterator>{};
+    return Depend::TaskDep{{/*DependenceType=*/makeDepType(*m1),
                             /*Iterator=*/std::move(maybeIter),
-                            /*LocatorList=*/makeObjects(t2, semaCtx)}};
+                            /*LocatorList=*/makeObjects(t1, semaCtx)}};
   };
 
   return Depend{common::visit( //
@@ -677,18 +673,18 @@ Detach make(const parser::OmpClause::Detach &inp,
 Device make(const parser::OmpClause::Device &inp,
             semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpDeviceClause
-  using wrapped = parser::OmpDeviceClause;
-
   CLAUSET_ENUM_CONVERT( //
-      convert, parser::OmpDeviceClause::DeviceModifier, Device::DeviceModifier,
+      convert, parser::OmpDeviceModifier::Value, Device::DeviceModifier,
       // clang-format off
       MS(Ancestor,   Ancestor)
       MS(Device_Num, DeviceNum)
       // clang-format on
   );
-  auto &t0 = std::get<std::optional<wrapped::DeviceModifier>>(inp.v.t);
+
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpDeviceModifier>(mods);
   auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
-  return Device{{/*DeviceModifier=*/maybeApply(convert, t0),
+  return Device{{/*DeviceModifier=*/maybeApplyToV(convert, m0),
                  /*DeviceDescription=*/makeExpr(t1, semaCtx)}};
 }
 
@@ -698,7 +694,8 @@ DeviceType make(const parser::OmpClause::DeviceType &inp,
   using wrapped = parser::OmpDeviceTypeClause;
 
   CLAUSET_ENUM_CONVERT( //
-      convert, wrapped::Type, DeviceType::DeviceTypeDescription,
+      convert, wrapped::DeviceTypeDescription,
+      DeviceType::DeviceTypeDescription,
       // clang-format off
       MS(Any,    Any)
       MS(Host,   Host)
@@ -800,19 +797,12 @@ From make(const parser::OmpClause::From &inp,
 Grainsize make(const parser::OmpClause::Grainsize &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpGrainsizeClause
-  using wrapped = parser::OmpGrainsizeClause;
-
-  CLAUSET_ENUM_CONVERT( //
-      convert, parser::OmpGrainsizeClause::Prescriptiveness,
-      Grainsize::Prescriptiveness,
-      // clang-format off
-      MS(Strict,   Strict)
-      // clang-format on
-  );
-  auto &t0 = std::get<std::optional<wrapped::Prescriptiveness>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpPrescriptiveness>(mods);
   auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
-  return Grainsize{{/*Prescriptiveness=*/maybeApply(convert, t0),
-                    /*Grainsize=*/makeExpr(t1, semaCtx)}};
+  return Grainsize{
+      {/*Prescriptiveness=*/maybeApplyToV(makePrescriptiveness, m0),
+       /*Grainsize=*/makeExpr(t1, semaCtx)}};
 }
 
 HasDeviceAddr make(const parser::OmpClause::HasDeviceAddr &inp,
@@ -835,27 +825,13 @@ Holds make(const parser::OmpClause::Holds &inp,
 If make(const parser::OmpClause::If &inp,
         semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpIfClause
-  using wrapped = parser::OmpIfClause;
-
-  CLAUSET_ENUM_CONVERT( //
-      convert, wrapped::DirectiveNameModifier, llvm::omp::Directive,
-      // clang-format off
-      MS(Parallel,         OMPD_parallel)
-      MS(Simd,             OMPD_simd)
-      MS(Target,           OMPD_target)
-      MS(TargetData,       OMPD_target_data)
-      MS(TargetEnterData,  OMPD_target_enter_data)
-      MS(TargetExitData,   OMPD_target_exit_data)
-      MS(TargetUpdate,     OMPD_target_update)
-      MS(Task,             OMPD_task)
-      MS(Taskloop,         OMPD_taskloop)
-      MS(Teams,            OMPD_teams)
-      // clang-format on
-  );
-  auto &t0 = std::get<std::optional<wrapped::DirectiveNameModifier>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 =
+      semantics::OmpGetUniqueModifier<parser::OmpDirectiveNameModifier>(mods);
   auto &t1 = std::get<parser::ScalarLogicalExpr>(inp.v.t);
-  return If{{/*DirectiveNameModifier=*/maybeApply(convert, t0),
-             /*IfExpression=*/makeExpr(t1, semaCtx)}};
+  return If{
+      {/*DirectiveNameModifier=*/maybeApplyToV([](auto &&s) { return s; }, m0),
+       /*IfExpression=*/makeExpr(t1, semaCtx)}};
 }
 
 // Inbranch: empty
@@ -899,20 +875,20 @@ IsDevicePtr make(const parser::OmpClause::IsDevicePtr &inp,
 Lastprivate make(const parser::OmpClause::Lastprivate &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpLastprivateClause
-  using wrapped = parser::OmpLastprivateClause;
-
   CLAUSET_ENUM_CONVERT( //
-      convert, parser::OmpLastprivateClause::LastprivateModifier,
+      convert, parser::OmpLastprivateModifier::Value,
       Lastprivate::LastprivateModifier,
       // clang-format off
       MS(Conditional, Conditional)
       // clang-format on
   );
 
-  auto &t0 = std::get<std::optional<wrapped::LastprivateModifier>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 =
+      semantics::OmpGetUniqueModifier<parser::OmpLastprivateModifier>(mods);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  return Lastprivate{{/*LastprivateModifier=*/maybeApply(convert, t0),
+  return Lastprivate{{/*LastprivateModifier=*/maybeApplyToV(convert, m0),
                       /*List=*/makeObjects(t1, semaCtx)}};
 }
 
@@ -1062,18 +1038,10 @@ Novariants make(const parser::OmpClause::Novariants &inp,
 NumTasks make(const parser::OmpClause::NumTasks &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpNumTasksClause
-  using wrapped = parser::OmpNumTasksClause;
-
-  CLAUSET_ENUM_CONVERT( //
-      convert, parser::OmpNumTasksClause::Prescriptiveness,
-      NumTasks::Prescriptiveness,
-      // clang-format off
-      MS(Strict,   Strict)
-      // clang-format on
-  );
-  auto &t0 = std::get<std::optional<wrapped::Prescriptiveness>>(inp.v.t);
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpPrescriptiveness>(mods);
   auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
-  return NumTasks{{/*Prescriptiveness=*/maybeApply(convert, t0),
+  return NumTasks{{/*Prescriptiveness=*/maybeApplyToV(makePrescriptiveness, m0),
                    /*NumTasks=*/makeExpr(t1, semaCtx)}};
 }
 
@@ -1160,7 +1128,7 @@ ProcBind make(const parser::OmpClause::ProcBind &inp,
   using wrapped = parser::OmpProcBindClause;
 
   CLAUSET_ENUM_CONVERT( //
-      convert, wrapped::Type, ProcBind::AffinityPolicy,
+      convert, wrapped::AffinityPolicy, ProcBind::AffinityPolicy,
       // clang-format off
       MS(Close,    Close)
       MS(Master,   Master)
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 5fac5c2271c3b..65282d243d87a 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -153,10 +153,11 @@ std::optional<ResultTy> maybeApply(FuncTy &&func,
   return func(*arg);
 }
 
-template <
+template <           //
     typename FuncTy, //
     typename ArgTy,  //
-    typename ResultTy = std::invoke_result_t<FuncTy, typename ArgTy::Value>>
+    typename ResultTy =
+        std::invoke_result_t<FuncTy, decltype(std::declval<ArgTy>().v)>>
 std::optional<ResultTy> maybeApplyToV(FuncTy &&func, const ArgTy *arg) {
   if (!arg)
     return std::nullopt;
@@ -175,6 +176,7 @@ using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
 using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
 using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
 using DependenceType = tomp::type::DependenceType;
+using Prescriptiveness = tomp::type::Prescriptiveness;
 
 // "Requires" clauses are handled early on, and the aggregated information
 // is stored in the Symbol details of modules, programs, and subprograms.
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 709ac402cc702..a1c0e8f417bcd 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -111,14 +111,11 @@ void DataSharingProcessor::insertDeallocs() {
 }
 
 void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
-  bool success = converter.createHostAssociateVarClone(*sym);
+  bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
+  bool success = converter.createHostAssociateVarClone(
+      *sym, /*skipDefaultInit=*/isFirstPrivate);
   (void)success;
   assert(success && "Privatization failed due to existing binding");
-
-  bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
-  if (!isFirstPrivate &&
-      Fortran::lower::hasDefaultInitialization(sym->GetUltimate()))
-    Fortran::lower::defaultInitializeAtRuntime(converter, *sym, *symTable);
 }
 
 void DataSharingProcessor::copyFirstPrivateSymbol(
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ea2d1eb66bfea..2ef4d184a6321 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -19,6 +19,7 @@
 #include "DirectivesCommon.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
+#include "flang/Common/OpenMP-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/ConvertExpr.h"
@@ -41,57 +42,12 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 
 using namespace Fortran::lower::omp;
+using namespace Fortran::common::openmp;
 
 //===----------------------------------------------------------------------===//
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
 
-namespace {
-/// Structure holding the information needed to create and bind entry block
-/// arguments associated to a single clause.
-struct EntryBlockArgsEntry {
-  llvm::ArrayRef<const semantics::Symbol *> syms;
-  llvm::ArrayRef<mlir::Value> vars;
-
-  bool isValid() const {
-    // This check allows specifying a smaller number of symbols than values
-    // because in some case cases a single symbol generates multiple block
-    // arguments.
-    return syms.size() <= vars.size();
-  }
-};
-
-/// Structure holding the information needed to create and bind entry block
-/// arguments associated to all clauses that can define them.
-struct EntryBlockArgs {
-  EntryBlockArgsEntry inReduction;
-  EntryBlockArgsEntry map;
-  EntryBlockArgsEntry priv;
-  EntryBlockArgsEntry reduction;
-  EntryBlockArgsEntry taskReduction;
-  EntryBlockArgsEntry useDeviceAddr;
-  EntryBlockArgsEntry useDevicePtr;
-
-  bool isValid() const {
-    return inReduction.isValid() && map.isValid() && priv.isValid() &&
-           reduction.isValid() && taskReduction.isValid() &&
-           useDeviceAddr.isValid() && useDevicePtr.isValid();
-  }
-
-  auto getSyms() const {
-    return llvm::concat<const semantics::Symbol *const>(
-        inReduction.syms, map.syms, priv.syms, reduction.syms,
-        taskReduction.syms, useDeviceAddr.syms, useDevicePtr.syms);
-  }
-
-  auto getVars() const {
-    return llvm::concat<const mlir::Value>(
-        inReduction.vars, map.vars, priv.vars, reduction.vars,
-        taskReduction.vars, useDeviceAddr.vars, useDevicePtr.vars);
-  }
-};
-} // namespace
-
 static void genOMPDispatch(lower::AbstractConverter &converter,
                            lower::SymMap &symTable,
                            semantics::SemanticsContext &semaCtx,
@@ -487,13 +443,15 @@ static void getDeclareTargetInfo(
   } else if (const auto *clauseList{
                  parser::Unwrap<parser::OmpClauseList>(spec.u)}) {
     List<Clause> clauses = makeClauses(*clauseList, semaCtx);
-    if (clauses.empty() &&
-        (!eval.getOwningProcedure()->isMainProgram() ||
-         eval.getOwningProcedure()->getMainProgramSymbol())) {
-      // Case: declare target, implicit capture of function
-      symbolAndClause.emplace_back(
-          mlir::omp::DeclareTargetCaptureClause::to,
-          eval.getOwningProcedure()->getSubprogramSymbol());
+    if (clauses.empty()) {
+      Fortran::lower::pft::FunctionLikeUnit *owningProc =
+          eval.getOwningProcedure();
+      if (owningProc && (!owningProc->isMainProgram() ||
+                         owningProc->getMainProgramSymbol())) {
+        // Case: declare target, implicit capture of function
+        symbolAndClause.emplace_back(mlir::omp::DeclareTargetCaptureClause::to,
+                                     owningProc->getSubprogramSymbol());
+      }
     }
 
     ClauseProcessor cp(converter, semaCtx, clauses);
@@ -623,50 +581,6 @@ static void genLoopVars(
   firOpBuilder.setInsertionPointAfter(storeOp);
 }
 
-/// Create an entry block for the given region, including the clause-defined
-/// arguments specified.
-///
-/// \param [in] converter - PFT to MLIR conversion interface.
-/// \param [in]      args - entry block arguments information for the given
-///                         operation.
-/// \param [in]    region - Empty region in which to create the entry block.
-static mlir::Block *genEntryBlock(lower::AbstractConverter &converter,
-                                  const EntryBlockArgs &args,
-                                  mlir::Region &region) {
-  assert(args.isValid() && "invalid args");
-  assert(region.empty() && "non-empty region");
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  llvm::SmallVector<mlir::Type> types;
-  llvm::SmallVector<mlir::Location> locs;
-  unsigned numVars = args.inReduction.vars.size() + args.map.vars.size() +
-                     args.priv.vars.size() + args.reduction.vars.size() +
-                     args.taskReduction.vars.size() +
-                     args.useDeviceAddr.vars.size() +
-                     args.useDevicePtr.vars.size();
-  types.reserve(numVars);
-  locs.reserve(numVars);
-
-  auto extractTypeLoc = [&types, &locs](llvm::ArrayRef<mlir::Value> vals) {
-    llvm::transform(vals, std::back_inserter(types),
-                    [](mlir::Value v) { return v.getType(); });
-    llvm::transform(vals, std::back_inserter(locs),
-                    [](mlir::Value v) { return v.getLoc(); });
-  };
-
-  // Populate block arguments in clause name alphabetical order to match
-  // expected order by the BlockArgOpenMPOpInterface.
-  extractTypeLoc(args.inReduction.vars);
-  extractTypeLoc(args.map.vars);
-  extractTypeLoc(args.priv.vars);
-  extractTypeLoc(args.reduction.vars);
-  extractTypeLoc(args.taskReduction.vars);
-  extractTypeLoc(args.useDeviceAddr.vars);
-  extractTypeLoc(args.useDevicePtr.vars);
-
-  return firOpBuilder.createBlock(&region, {}, types, locs);
-}
-
 static void
 markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
                   mlir::omp::DeclareTargetCaptureClause captureClause,
@@ -919,7 +833,7 @@ static void genBodyOfTargetDataOp(
     ConstructQueue::const_iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
-  genEntryBlock(converter, args, dataOp.getRegion());
+  genEntryBlock(firOpBuilder, args, dataOp.getRegion());
   bindEntryBlockArgs(converter, dataOp, args);
 
   // Insert dummy instruction to remember the insertion position. The
@@ -996,7 +910,7 @@ static void genBodyOfTargetOp(
   auto argIface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(*targetOp);
 
   mlir::Region &region = targetOp.getRegion();
-  mlir::Block *entryBlock = genEntryBlock(converter, args, region);
+  mlir::Block *entryBlock = genEntryBlock(firOpBuilder, args, region);
   bindEntryBlockArgs(converter, targetOp, args);
 
   // Check if cloning the bounds introduced any dependency on the outer region.
@@ -1122,7 +1036,7 @@ static OpTy genWrapperOp(lower::AbstractConverter &converter,
   auto op = firOpBuilder.create<OpTy>(loc, clauseOps);
 
   // Create entry block with arguments.
-  genEntryBlock(converter, args, op.getRegion());
+  genEntryBlock(firOpBuilder, args, op.getRegion());
 
   return op;
 }
@@ -1347,9 +1261,10 @@ static void genTaskClauses(lower::AbstractConverter &converter,
   cp.processMergeable(clauseOps);
   cp.processPriority(stmtCtx, clauseOps);
   cp.processUntied(clauseOps);
+  cp.processDetach(clauseOps);
   // TODO Support delayed privatization.
 
-  cp.processTODO<clause::Affinity, clause::Detach, clause::InReduction>(
+  cp.processTODO<clause::Affinity, clause::InReduction>(
       loc, llvm::omp::Directive::OMPD_task);
 }
 
@@ -1588,7 +1503,7 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
               const EntryBlockArgs &args, DataSharingProcessor *dsp,
               bool isComposite = false) {
   auto genRegionEntryCB = [&](mlir::Operation *op) {
-    genEntryBlock(converter, args, op->getRegion(0));
+    genEntryBlock(converter.getFirOpBuilder(), args, op->getRegion(0));
     bindEntryBlockArgs(
         converter, llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op), args);
     return llvm::to_vector(args.getSyms());
@@ -1661,12 +1576,12 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   args.reduction.syms = reductionSyms;
   args.reduction.vars = clauseOps.reductionVars;
 
-  genEntryBlock(converter, args, sectionsOp.getRegion());
+  genEntryBlock(builder, args, sectionsOp.getRegion());
   mlir::Operation *terminator =
       lower::genOpenMPTerminator(builder, sectionsOp, loc);
 
   auto genRegionEntryCB = [&](mlir::Operation *op) {
-    genEntryBlock(converter, args, op->getRegion(0));
+    genEntryBlock(builder, args, op->getRegion(0));
     bindEntryBlockArgs(
         converter, llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op), args);
     return llvm::to_vector(args.getSyms());
@@ -1989,7 +1904,7 @@ genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
   taskArgs.priv.vars = clauseOps.privateVars;
 
   auto genRegionEntryCB = [&](mlir::Operation *op) {
-    genEntryBlock(converter, taskArgs, op->getRegion(0));
+    genEntryBlock(converter.getFirOpBuilder(), taskArgs, op->getRegion(0));
     bindEntryBlockArgs(converter,
                        llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op),
                        taskArgs);
@@ -2128,11 +2043,10 @@ static void genStandaloneDo(lower::AbstractConverter &converter,
   genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                    wsloopClauseOps, wsloopReductionSyms);
 
-  // TODO: Support delayed privatization.
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
                            /*shouldCollectPreDeterminedSymbols=*/true,
-                           /*useDelayedPrivatization=*/false, &symTable);
-  dsp.processStep1();
+                           enableDelayedPrivatizationStaging, &symTable);
+  dsp.processStep1(&wsloopClauseOps);
 
   mlir::omp::LoopNestOperands loopNestClauseOps;
   llvm::SmallVector<const semantics::Symbol *> iv;
@@ -2140,7 +2054,8 @@ static void genStandaloneDo(lower::AbstractConverter &converter,
                      loopNestClauseOps, iv);
 
   EntryBlockArgs wsloopArgs;
-  // TODO: Add private syms and vars.
+  wsloopArgs.priv.syms = dsp.getDelayedPrivSymbols();
+  wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
   wsloopArgs.reduction.syms = wsloopReductionSyms;
   wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
   auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
@@ -2896,6 +2811,10 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                                           parser::OmpAtomicClauseList>(
                 converter, atomicCapture, loc);
           },
+          [&](const parser::OmpAtomicCompare &atomicCompare) {
+            mlir::Location loc = converter.genLocation(atomicCompare.source);
+            TODO(loc, "OpenMP atomic compare");
+          },
       },
       atomicConstruct.u);
 }
@@ -2951,7 +2870,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::UseDevicePtr>(clause.u) &&
         !std::holds_alternative<clause::InReduction>(clause.u) &&
         !std::holds_alternative<clause::Mergeable>(clause.u) &&
-        !std::holds_alternative<clause::TaskReduction>(clause.u)) {
+        !std::holds_alternative<clause::TaskReduction>(clause.u) &&
+        !std::holds_alternative<clause::Detach>(clause.u)) {
       std::string name =
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
       TODO(clauseLocation, name + " clause is not implemented yet");
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index 793e291a168ad..41bdff4dca471 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -161,11 +161,14 @@ class PFTBuilder {
       return;
     if (procName.starts_with("ieee_set_modes_") ||
         procName.starts_with("ieee_set_status_"))
-      proc->mayModifyHaltingMode = proc->mayModifyRoundingMode = true;
+      proc->mayModifyHaltingMode = proc->mayModifyRoundingMode =
+          proc->mayModifyUnderflowMode = true;
     else if (procName.starts_with("ieee_set_halting_mode_"))
       proc->mayModifyHaltingMode = true;
     else if (procName.starts_with("ieee_set_rounding_mode_"))
       proc->mayModifyRoundingMode = true;
+    else if (procName.starts_with("ieee_set_underflow_mode_"))
+      proc->mayModifyUnderflowMode = true;
   }
 
   /// Convert an IfStmt into an IfConstruct, retaining the IfStmt as the
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index 2b24791d6c7c5..0b0f83d024ce3 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -91,6 +91,13 @@ bool AliasAnalysis::Source::isDummyArgument() const {
   return false;
 }
 
+static bool isEvaluateInMemoryBlockArg(mlir::Value v) {
+  if (auto evalInMem = llvm::dyn_cast_or_null<hlfir::EvaluateInMemoryOp>(
+          v.getParentRegion()->getParentOp()))
+    return evalInMem.getMemory() == v;
+  return false;
+}
+
 bool AliasAnalysis::Source::isData() const { return origin.isData; }
 bool AliasAnalysis::Source::isBoxData() const {
   return mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(valueType)) &&
@@ -457,6 +464,33 @@ ModRefResult AliasAnalysis::getModRef(Operation *op, Value location) {
   return result;
 }
 
+ModRefResult AliasAnalysis::getModRef(mlir::Region &region,
+                                      mlir::Value location) {
+  ModRefResult result = ModRefResult::getNoModRef();
+  for (mlir::Operation &op : region.getOps()) {
+    if (op.hasTrait<mlir::OpTrait::HasRecursiveMemoryEffects>()) {
+      for (mlir::Region &subRegion : op.getRegions()) {
+        result = result.merge(getModRef(subRegion, location));
+        // Fast return is already mod and ref.
+        if (result.isModAndRef())
+          return result;
+      }
+      // In MLIR, RecursiveMemoryEffects can be combined with
+      // MemoryEffectOpInterface to describe extra effects on top of the
+      // effects of the nested operations.  However, the presence of
+      // RecursiveMemoryEffects and the absence of MemoryEffectOpInterface
+      // implies the operation has no other memory effects than the one of its
+      // nested operations.
+      if (!mlir::isa<mlir::MemoryEffectOpInterface>(op))
+        continue;
+    }
+    result = result.merge(getModRef(&op, location));
+    if (result.isModAndRef())
+      return result;
+  }
+  return result;
+}
+
 AliasAnalysis::Source::Attributes
 getAttrsFromVariable(fir::FortranVariableOpInterface var) {
   AliasAnalysis::Source::Attributes attrs;
@@ -698,7 +732,7 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
           breakFromLoop = true;
         });
   }
-  if (!defOp && type == SourceKind::Unknown)
+  if (!defOp && type == SourceKind::Unknown) {
     // Check if the memory source is coming through a dummy argument.
     if (isDummyArgument(v)) {
       type = SourceKind::Argument;
@@ -708,7 +742,12 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
 
       if (isPointerReference(ty))
         attributes.set(Attribute::Pointer);
+    } else if (isEvaluateInMemoryBlockArg(v)) {
+      // hlfir.eval_in_mem block operands is allocated by the operation.
+      type = SourceKind::Allocate;
+      ty = v.getType();
     }
+  }
 
   if (type == SourceKind::Global) {
     return {{global, instantiationPoint, followingData},
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 7425ccf7fc0e3..1bd950f2445ee 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -535,6 +535,8 @@ static mlir::Value tryRetrievingShapeOrShift(hlfir::Entity entity) {
   if (mlir::isa<hlfir::ExprType>(entity.getType())) {
     if (auto elemental = entity.getDefiningOp<hlfir::ElementalOp>())
       return elemental.getShape();
+    if (auto evalInMem = entity.getDefiningOp<hlfir::EvaluateInMemoryOp>())
+      return evalInMem.getShape();
     return mlir::Value{};
   }
   if (auto varIface = entity.getIfVariableInterface())
@@ -642,6 +644,11 @@ void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
       result.append(elemental.getTypeparams().begin(),
                     elemental.getTypeparams().end());
       return;
+    } else if (auto evalInMem =
+                   expr.getDefiningOp<hlfir::EvaluateInMemoryOp>()) {
+      result.append(evalInMem.getTypeparams().begin(),
+                    evalInMem.getTypeparams().end());
+      return;
     } else if (auto apply = expr.getDefiningOp<hlfir::ApplyOp>()) {
       result.append(apply.getTypeparams().begin(), apply.getTypeparams().end());
       return;
@@ -1313,3 +1320,43 @@ hlfir::genTypeAndKindConvert(mlir::Location loc, fir::FirOpBuilder &builder,
   };
   return {hlfir::Entity{convertedRhs}, cleanup};
 }
+
+std::pair<hlfir::Entity, bool> hlfir::computeEvaluateOpInNewTemp(
+    mlir::Location loc, fir::FirOpBuilder &builder,
+    hlfir::EvaluateInMemoryOp evalInMem, mlir::Value shape,
+    mlir::ValueRange typeParams) {
+  llvm::StringRef tmpName{".tmp.expr_result"};
+  llvm::SmallVector<mlir::Value> extents =
+      hlfir::getIndexExtents(loc, builder, shape);
+  mlir::Type baseType =
+      hlfir::getFortranElementOrSequenceType(evalInMem.getType());
+  bool heapAllocated = fir::hasDynamicSize(baseType);
+  // Note: temporaries are stack allocated here when possible (do not require
+  // stack save/restore) because flang has always stack allocated function
+  // results.
+  mlir::Value temp = heapAllocated
+                         ? builder.createHeapTemporary(loc, baseType, tmpName,
+                                                       extents, typeParams)
+                         : builder.createTemporary(loc, baseType, tmpName,
+                                                   extents, typeParams);
+  mlir::Value innerMemory = evalInMem.getMemory();
+  temp = builder.createConvert(loc, innerMemory.getType(), temp);
+  auto declareOp = builder.create<hlfir::DeclareOp>(
+      loc, temp, tmpName, shape, typeParams,
+      /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+  computeEvaluateOpIn(loc, builder, evalInMem, declareOp.getOriginalBase());
+  return {hlfir::Entity{declareOp.getBase()}, /*heapAllocated=*/heapAllocated};
+}
+
+void hlfir::computeEvaluateOpIn(mlir::Location loc, fir::FirOpBuilder &builder,
+                                hlfir::EvaluateInMemoryOp evalInMem,
+                                mlir::Value storage) {
+  mlir::Value innerMemory = evalInMem.getMemory();
+  mlir::Value storageCast =
+      builder.createConvert(loc, innerMemory.getType(), storage);
+  mlir::IRMapping mapper;
+  mapper.map(innerMemory, storageCast);
+  for (auto &op : evalInMem.getBody().front().without_terminator())
+    builder.clone(op, mapper);
+  return;
+}
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 5dfa53e047f42..547cebefd2df4 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -41,7 +41,7 @@
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/Utils.h"
 #include "flang/Runtime/entry-names.h"
-#include "flang/Runtime/iostat.h"
+#include "flang/Runtime/iostat-consts.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Math/IR/Math.h"
@@ -95,10 +95,6 @@ static bool isStaticallyPresent(const fir::ExtendedValue &exv) {
   return !isStaticallyAbsent(exv);
 }
 
-/// IEEE module procedure names not yet implemented for genModuleProcTODO.
-static constexpr char ieee_get_underflow_mode[] = "ieee_get_underflow_mode";
-static constexpr char ieee_set_underflow_mode[] = "ieee_set_underflow_mode";
-
 using I = IntrinsicLibrary;
 
 /// Flag to indicate that an intrinsic argument has to be handled as
@@ -328,7 +324,10 @@ static constexpr IntrinsicHandler handlers[]{
        {"radix", asValue, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"ieee_get_status", &I::genIeeeGetOrSetStatus</*isGet=*/true>},
-    {"ieee_get_underflow_mode", &I::genModuleProcTODO<ieee_get_underflow_mode>},
+    {"ieee_get_underflow_mode",
+     &I::genIeeeGetUnderflowMode,
+     {{{"gradual", asAddr}}},
+     /*isElemental=*/false},
     {"ieee_int", &I::genIeeeInt},
     {"ieee_is_finite", &I::genIeeeIsFinite},
     {"ieee_is_nan", &I::genIeeeIsNan},
@@ -375,7 +374,7 @@ static constexpr IntrinsicHandler handlers[]{
        {"radix", asValue, handleDynamicOptional}}},
      /*isElemental=*/false},
     {"ieee_set_status", &I::genIeeeGetOrSetStatus</*isGet=*/false>},
-    {"ieee_set_underflow_mode", &I::genModuleProcTODO<ieee_set_underflow_mode>},
+    {"ieee_set_underflow_mode", &I::genIeeeSetUnderflowMode},
     {"ieee_signaling_eq",
      &I::genIeeeSignalingCompare<mlir::arith::CmpFPredicate::OEQ>},
     {"ieee_signaling_ge",
@@ -2295,12 +2294,6 @@ mlir::Value IntrinsicLibrary::genConversion(mlir::Type resultType,
   return builder.convertWithSemantics(loc, resultType, args[0]);
 }
 
-template <const char *intrinsicName>
-void IntrinsicLibrary::genModuleProcTODO(
-    llvm::ArrayRef<fir::ExtendedValue> args) {
-  crashOnMissingIntrinsic(loc, intrinsicName);
-}
-
 // ABORT
 void IntrinsicLibrary::genAbort(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 0);
@@ -4471,6 +4464,14 @@ void IntrinsicLibrary::genIeeeGetOrSetStatus(
   genRuntimeCall(isGet ? "fegetenv" : "fesetenv", i32Ty, addr);
 }
 
+// IEEE_GET_UNDERFLOW_MODE
+void IntrinsicLibrary::genIeeeGetUnderflowMode(
+    llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert(args.size() == 1);
+  mlir::Value flag = fir::runtime::genGetUnderflowMode(builder, loc);
+  builder.createStoreWithConvert(loc, flag, fir::getBase(args[0]));
+}
+
 // IEEE_INT
 mlir::Value IntrinsicLibrary::genIeeeInt(mlir::Type resultType,
                                          llvm::ArrayRef<mlir::Value> args) {
@@ -5135,6 +5136,15 @@ void IntrinsicLibrary::genIeeeSetRoundingMode(
   builder.create<fir::CallOp>(loc, setRound, mode);
 }
 
+// IEEE_SET_UNDERFLOW_MODE
+void IntrinsicLibrary::genIeeeSetUnderflowMode(
+    llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert(args.size() == 1);
+  mlir::Value gradual = builder.create<fir::ConvertOp>(loc, builder.getI1Type(),
+                                                       getBase(args[0]));
+  fir::runtime::genSetUnderflowMode(builder, loc, {gradual});
+}
+
 // IEEE_SIGNALING_EQ, IEEE_SIGNALING_GE, IEEE_SIGNALING_GT,
 // IEEE_SIGNALING_LE, IEEE_SIGNALING_LT, IEEE_SIGNALING_NE
 template <mlir::arith::CmpFPredicate pred>
diff --git a/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp b/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
index c786bef5cb1c4..0d56cd2edc99b 100644
--- a/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/ArrayConstructor.cpp
@@ -9,7 +9,7 @@
 #include "flang/Optimizer/Builder/Runtime/ArrayConstructor.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
-#include "flang/Runtime/array-constructor.h"
+#include "flang/Runtime/array-constructor-consts.h"
 
 using namespace Fortran::runtime;
 
@@ -25,12 +25,13 @@ mlir::Value fir::runtime::genInitArrayConstructorVector(
     mlir::Location loc, fir::FirOpBuilder &builder, mlir::Value toBox,
     mlir::Value useValueLengthParameters) {
   // Allocate storage for the runtime cookie for the array constructor vector.
-  // Use the "host" size and alignment, but double them to be safe regardless of
-  // the target. The "cookieSize" argument is used to validate this wild
-  // assumption until runtime interfaces are improved.
+  // Use pessimistic values for size and alignment that are valid for all
+  // supported targets. Whether the actual ArrayConstructorVector object fits
+  // into the available MaxArrayConstructorVectorSizeInBytes is verified when
+  // building clang-rt.
   std::size_t arrayVectorStructBitSize =
-      2 * sizeof(Fortran::runtime::ArrayConstructorVector) * 8;
-  std::size_t alignLike = alignof(Fortran::runtime::ArrayConstructorVector) * 8;
+      MaxArrayConstructorVectorSizeInBytes * 8;
+  std::size_t alignLike = MaxArrayConstructorVectorAlignInBytes * 8;
   fir::SequenceType::Extent numElem =
       (arrayVectorStructBitSize + alignLike - 1) / alignLike;
   mlir::Type intType = builder.getIntegerType(alignLike);
@@ -43,14 +44,12 @@ mlir::Value fir::runtime::genInitArrayConstructorVector(
           loc, builder);
   mlir::FunctionType funcType = func.getFunctionType();
   cookie = builder.createConvert(loc, funcType.getInput(0), cookie);
-  mlir::Value cookieSize = builder.createIntegerConstant(
-      loc, funcType.getInput(3), numElem * alignLike / 8);
   mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
   mlir::Value sourceLine =
-      fir::factory::locationToLineNo(builder, loc, funcType.getInput(5));
+      fir::factory::locationToLineNo(builder, loc, funcType.getInput(4));
   auto args = fir::runtime::createArguments(builder, loc, funcType, cookie,
                                             toBox, useValueLengthParameters,
-                                            cookieSize, sourceFile, sourceLine);
+                                            sourceFile, sourceLine);
   builder.create<fir::CallOp>(loc, func, args);
   return cookie;
 }
diff --git a/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp b/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
index 8775b50437af2..85f38424eabdc 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Exceptions.cpp
@@ -20,3 +20,17 @@ mlir::Value fir::runtime::genMapExcept(fir::FirOpBuilder &builder,
       fir::runtime::getRuntimeFunc<mkRTKey(MapException)>(loc, builder)};
   return builder.create<fir::CallOp>(loc, func, excepts).getResult(0);
 }
+
+mlir::Value fir::runtime::genGetUnderflowMode(fir::FirOpBuilder &builder,
+                                              mlir::Location loc) {
+  mlir::func::FuncOp func{
+      fir::runtime::getRuntimeFunc<mkRTKey(GetUnderflowMode)>(loc, builder)};
+  return builder.create<fir::CallOp>(loc, func).getResult(0);
+}
+
+void fir::runtime::genSetUnderflowMode(fir::FirOpBuilder &builder,
+                                       mlir::Location loc, mlir::Value flag) {
+  mlir::func::FuncOp func{
+      fir::runtime::getRuntimeFunc<mkRTKey(SetUnderflowMode)>(loc, builder)};
+  builder.create<fir::CallOp>(loc, func, flag);
+}
diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index c536fd19fcc69..1bb91d252529f 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -103,6 +103,8 @@ class BoxprocTypeRewriter : public mlir::TypeConverter {
       return needsConversion(unwrapRefType(ty));
     if (auto t = mlir::dyn_cast<SequenceType>(ty))
       return needsConversion(unwrapSequenceType(ty));
+    if (auto t = mlir::dyn_cast<TypeDescType>(ty))
+      return needsConversion(t.getOfTy());
     return false;
   }
 
@@ -167,6 +169,9 @@ class BoxprocTypeRewriter : public mlir::TypeConverter {
       rec.finalize(ps, cs);
       return rec;
     });
+    addConversion([&](TypeDescType ty) {
+      return TypeDescType::get(convertType(ty.getOfTy()));
+    });
     addArgumentMaterialization(materializeProcedure);
     addSourceMaterialization(materializeProcedure);
     addTargetMaterialization(materializeProcedure);
@@ -220,7 +225,6 @@ class BoxedProcedurePass
       auto *context = &getContext();
       mlir::IRRewriter rewriter(context);
       BoxprocTypeRewriter typeConverter(mlir::UnknownLoc::get(context));
-      mlir::Dialect *firDialect = context->getLoadedDialect("fir");
       getModule().walk([&](mlir::Operation *op) {
         bool opIsValid = true;
         typeConverter.setLocation(op->getLoc());
@@ -366,13 +370,22 @@ class BoxedProcedurePass
                 index, toTy, index.getFieldId(), toOnTy, index.getTypeparams());
             opIsValid = false;
           }
-        } else if (op->getDialect() == firDialect) {
+        } else {
           rewriter.startOpModification(op);
+          // Convert the operands if needed
           for (auto i : llvm::enumerate(op->getResultTypes()))
             if (typeConverter.needsConversion(i.value())) {
               auto toTy = typeConverter.convertType(i.value());
               op->getResult(i.index()).setType(toTy);
             }
+
+          // Convert the type attributes if needed
+          for (const mlir::NamedAttribute &attr : op->getAttrDictionary())
+            if (auto tyAttr = llvm::dyn_cast<mlir::TypeAttr>(attr.getValue()))
+              if (typeConverter.needsConversion(tyAttr.getValue())) {
+                auto toTy = typeConverter.convertType(tyAttr.getValue());
+                op->setAttr(attr.getName(), mlir::TypeAttr::get(toTy));
+              }
           rewriter.finalizeOpModification(op);
         }
         // Ensure block arguments are updated if needed.
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 2eeb182735094..5345d64c330f0 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -23,8 +23,8 @@
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Support/TypeCode.h"
 #include "flang/Optimizer/Support/Utils.h"
-#include "flang/Runtime/allocator-registry.h"
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/allocator-registry-consts.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "mlir/Conversion/ArithCommon/AttrToLLVMConverter.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
@@ -1322,16 +1322,12 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
           insertField(rewriter, loc, descriptor, {kExtraPosInBox}, extraField);
     } else {
       // Compute the value of the extra field based on allocator_idx and
-      // addendum present using a Descriptor object.
-      Fortran::runtime::StaticDescriptor staticDescriptor;
-      Fortran::runtime::Descriptor &desc{staticDescriptor.descriptor()};
-      desc.raw().extra = 0;
-      desc.SetAllocIdx(allocatorIdx);
+      // addendum present.
+      unsigned extra = allocatorIdx << _CFI_ALLOCATOR_IDX_SHIFT;
       if (hasAddendum)
-        desc.SetHasAddendum();
-      descriptor =
-          insertField(rewriter, loc, descriptor, {kExtraPosInBox},
-                      this->genI32Constant(loc, rewriter, desc.raw().extra));
+        extra |= _CFI_ADDENDUM_FLAG;
+      descriptor = insertField(rewriter, loc, descriptor, {kExtraPosInBox},
+                               this->genI32Constant(loc, rewriter, extra));
     }
 
     if (hasAddendum) {
diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index 9ec1655d92f44..f7bffbf53c190 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -80,17 +80,17 @@ struct GenericTarget : public CodeGenSpecifics {
                                 mlir::TypeRange{ptrTy, idxTy});
   }
 
-  Marshalling boxcharArgumentType(mlir::Type eleTy, bool sret) const override {
+  Marshalling boxcharArgumentType(mlir::Type eleTy) const override {
     CodeGenSpecifics::Marshalling marshal;
     auto idxTy = mlir::IntegerType::get(eleTy.getContext(), S::defaultWidth);
     auto ptrTy = fir::ReferenceType::get(eleTy);
     marshal.emplace_back(ptrTy, AT{});
-    // Return value arguments are grouped as a pair. Others are passed in a
-    // split format with all pointers first (in the declared position) and all
-    // LEN arguments appended after all of the dummy arguments.
+    // Characters are passed in a split format with all pointers first (in the
+    // declared position) and all LEN arguments appended after all of the dummy
+    // arguments.
     // NB: Other conventions/ABIs can/should be supported via options.
     marshal.emplace_back(idxTy, AT{/*alignment=*/0, /*byval=*/false,
-                                   /*sret=*/sret, /*append=*/!sret});
+                                   /*sret=*/false, /*append=*/true});
     return marshal;
   }
 
@@ -1172,6 +1172,9 @@ struct TargetLoongArch64 : public GenericTarget<TargetLoongArch64> {
   using GenericTarget::GenericTarget;
 
   static constexpr int defaultWidth = 64;
+  static constexpr int GRLen = defaultWidth; /* eight bytes */
+  static constexpr int GRLenInChar = GRLen / 8;
+  static constexpr int FRLen = defaultWidth; /* eight bytes */
 
   CodeGenSpecifics::Marshalling
   complexArgumentType(mlir::Location loc, mlir::Type eleTy) const override {
@@ -1242,6 +1245,313 @@ struct TargetLoongArch64 : public GenericTarget<TargetLoongArch64> {
 
     return GenericTarget::integerArgumentType(loc, argTy);
   }
+
+  /// Flatten non-basic types, resulting in an array of types containing only
+  /// `IntegerType` and `FloatType`.
+  llvm::SmallVector<mlir::Type> flattenTypeList(mlir::Location loc,
+                                                const mlir::Type type) const {
+    llvm::SmallVector<mlir::Type> flatTypes;
+
+    llvm::TypeSwitch<mlir::Type>(type)
+        .template Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
+          if (intTy.getWidth() != 0)
+            flatTypes.push_back(intTy);
+        })
+        .template Case<mlir::FloatType>([&](mlir::FloatType floatTy) {
+          if (floatTy.getWidth() != 0)
+            flatTypes.push_back(floatTy);
+        })
+        .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
+          const auto *sem = &floatToSemantics(kindMap, cmplx.getElementType());
+          if (sem == &llvm::APFloat::IEEEsingle() ||
+              sem == &llvm::APFloat::IEEEdouble() ||
+              sem == &llvm::APFloat::IEEEquad())
+            std::fill_n(std::back_inserter(flatTypes), 2,
+                        cmplx.getElementType());
+          else
+            TODO(loc, "unsupported complex type(not IEEEsingle, IEEEdouble, "
+                      "IEEEquad) as a structure component for BIND(C), "
+                      "VALUE derived type argument and type return");
+        })
+        .template Case<fir::LogicalType>([&](fir::LogicalType logicalTy) {
+          const unsigned width =
+              kindMap.getLogicalBitsize(logicalTy.getFKind());
+          if (width != 0)
+            flatTypes.push_back(
+                mlir::IntegerType::get(type.getContext(), width));
+        })
+        .template Case<fir::CharacterType>([&](fir::CharacterType charTy) {
+          assert(kindMap.getCharacterBitsize(charTy.getFKind()) <= 8 &&
+                 "the bit size of characterType as an interoperable type must "
+                 "not exceed 8");
+          for (unsigned i = 0; i < charTy.getLen(); ++i)
+            flatTypes.push_back(mlir::IntegerType::get(type.getContext(), 8));
+        })
+        .template Case<fir::SequenceType>([&](fir::SequenceType seqTy) {
+          if (!seqTy.hasDynamicExtents()) {
+            const std::uint64_t numOfEle = seqTy.getConstantArraySize();
+            mlir::Type eleTy = seqTy.getEleTy();
+            if (!mlir::isa<mlir::IntegerType, mlir::FloatType>(eleTy)) {
+              llvm::SmallVector<mlir::Type> subTypeList =
+                  flattenTypeList(loc, eleTy);
+              if (subTypeList.size() != 0)
+                for (std::uint64_t i = 0; i < numOfEle; ++i)
+                  llvm::copy(subTypeList, std::back_inserter(flatTypes));
+            } else {
+              std::fill_n(std::back_inserter(flatTypes), numOfEle, eleTy);
+            }
+          } else
+            TODO(loc, "unsupported dynamic extent sequence type as a structure "
+                      "component for BIND(C), "
+                      "VALUE derived type argument and type return");
+        })
+        .template Case<fir::RecordType>([&](fir::RecordType recTy) {
+          for (auto &component : recTy.getTypeList()) {
+            mlir::Type eleTy = component.second;
+            llvm::SmallVector<mlir::Type> subTypeList =
+                flattenTypeList(loc, eleTy);
+            if (subTypeList.size() != 0)
+              llvm::copy(subTypeList, std::back_inserter(flatTypes));
+          }
+        })
+        .template Case<fir::VectorType>([&](fir::VectorType vecTy) {
+          auto sizeAndAlign = fir::getTypeSizeAndAlignmentOrCrash(
+              loc, vecTy, getDataLayout(), kindMap);
+          if (sizeAndAlign.first == 2 * GRLenInChar)
+            flatTypes.push_back(
+                mlir::IntegerType::get(type.getContext(), 2 * GRLen));
+          else
+            TODO(loc, "unsupported vector width(must be 128 bits)");
+        })
+        .Default([&](mlir::Type ty) {
+          if (fir::conformsWithPassByRef(ty))
+            flatTypes.push_back(
+                mlir::IntegerType::get(type.getContext(), GRLen));
+          else
+            TODO(loc, "unsupported component type for BIND(C), VALUE derived "
+                      "type argument and type return");
+        });
+
+    return flatTypes;
+  }
+
+  /// Determine if a struct is eligible to be passed in FARs (and GARs) (i.e.,
+  /// when flattened it contains a single fp value, fp+fp, or int+fp of
+  /// appropriate size).
+  bool detectFARsEligibleStruct(mlir::Location loc, fir::RecordType recTy,
+                                mlir::Type &field1Ty,
+                                mlir::Type &field2Ty) const {
+    field1Ty = field2Ty = nullptr;
+    llvm::SmallVector<mlir::Type> flatTypes = flattenTypeList(loc, recTy);
+    size_t flatSize = flatTypes.size();
+
+    // Cannot be eligible if the number of flattened types is equal to 0 or
+    // greater than 2.
+    if (flatSize == 0 || flatSize > 2)
+      return false;
+
+    bool isFirstAvaliableFloat = false;
+
+    assert((mlir::isa<mlir::IntegerType, mlir::FloatType>(flatTypes[0])) &&
+           "Type must be integerType or floatType after flattening");
+    if (auto floatTy = mlir::dyn_cast<mlir::FloatType>(flatTypes[0])) {
+      const unsigned Size = floatTy.getWidth();
+      // Can't be eligible if larger than the FP registers. Half precision isn't
+      // currently supported on LoongArch and the ABI hasn't been confirmed, so
+      // default to the integer ABI in that case.
+      if (Size > FRLen || Size < 32)
+        return false;
+      isFirstAvaliableFloat = true;
+      field1Ty = floatTy;
+    } else if (auto intTy = mlir::dyn_cast<mlir::IntegerType>(flatTypes[0])) {
+      if (intTy.getWidth() > GRLen)
+        return false;
+      field1Ty = intTy;
+    }
+
+    // flatTypes has two elements
+    if (flatSize == 2) {
+      assert((mlir::isa<mlir::IntegerType, mlir::FloatType>(flatTypes[1])) &&
+             "Type must be integerType or floatType after flattening");
+      if (auto floatTy = mlir::dyn_cast<mlir::FloatType>(flatTypes[1])) {
+        const unsigned Size = floatTy.getWidth();
+        if (Size > FRLen || Size < 32)
+          return false;
+        field2Ty = floatTy;
+        return true;
+      } else if (auto intTy = mlir::dyn_cast<mlir::IntegerType>(flatTypes[1])) {
+        // Can't be eligible if an integer type was already found (int+int pairs
+        // are not eligible).
+        if (!isFirstAvaliableFloat)
+          return false;
+        if (intTy.getWidth() > GRLen)
+          return false;
+        field2Ty = intTy;
+        return true;
+      }
+    }
+
+    // return isFirstAvaliableFloat if flatTypes only has one element
+    return isFirstAvaliableFloat;
+  }
+
+  bool checkTypeHasEnoughRegs(mlir::Location loc, int &GARsLeft, int &FARsLeft,
+                              const mlir::Type type) const {
+    if (!type)
+      return true;
+
+    llvm::TypeSwitch<mlir::Type>(type)
+        .template Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
+          const unsigned width = intTy.getWidth();
+          if (width > 128)
+            TODO(loc,
+                 "integerType with width exceeding 128 bits is unsupported");
+          if (width == 0)
+            return;
+          if (width <= GRLen)
+            --GARsLeft;
+          else if (width <= 2 * GRLen)
+            GARsLeft = GARsLeft - 2;
+        })
+        .template Case<mlir::FloatType>([&](mlir::FloatType floatTy) {
+          const unsigned width = floatTy.getWidth();
+          if (width > 128)
+            TODO(loc, "floatType with width exceeding 128 bits is unsupported");
+          if (width == 0)
+            return;
+          if (width == 32 || width == 64)
+            --FARsLeft;
+          else if (width <= GRLen)
+            --GARsLeft;
+          else if (width <= 2 * GRLen)
+            GARsLeft = GARsLeft - 2;
+        })
+        .Default([&](mlir::Type ty) {
+          if (fir::conformsWithPassByRef(ty))
+            --GARsLeft; // Pointers.
+          else
+            TODO(loc, "unsupported component type for BIND(C), VALUE derived "
+                      "type argument and type return");
+        });
+
+    return GARsLeft >= 0 && FARsLeft >= 0;
+  }
+
+  bool hasEnoughRegisters(mlir::Location loc, int GARsLeft, int FARsLeft,
+                          const Marshalling &previousArguments,
+                          const mlir::Type &field1Ty,
+                          const mlir::Type &field2Ty) const {
+    for (auto &typeAndAttr : previousArguments) {
+      const auto &attr = std::get<Attributes>(typeAndAttr);
+      if (attr.isByVal()) {
+        // Previous argument passed on the stack, and its address is passed in
+        // GAR.
+        --GARsLeft;
+        continue;
+      }
+
+      // Previous aggregate arguments were marshalled into simpler arguments.
+      const auto &type = std::get<mlir::Type>(typeAndAttr);
+      llvm::SmallVector<mlir::Type> flatTypes = flattenTypeList(loc, type);
+
+      for (auto &flatTy : flatTypes) {
+        if (!checkTypeHasEnoughRegs(loc, GARsLeft, FARsLeft, flatTy))
+          return false;
+      }
+    }
+
+    if (!checkTypeHasEnoughRegs(loc, GARsLeft, FARsLeft, field1Ty))
+      return false;
+    if (!checkTypeHasEnoughRegs(loc, GARsLeft, FARsLeft, field2Ty))
+      return false;
+    return true;
+  }
+
+  /// LoongArch64 subroutine calling sequence ABI in:
+  /// https://github.com/loongson/la-abi-specs/blob/release/lapcs.adoc#subroutine-calling-sequence
+  CodeGenSpecifics::Marshalling
+  classifyStruct(mlir::Location loc, fir::RecordType recTy, int GARsLeft,
+                 int FARsLeft, bool isResult,
+                 const Marshalling &previousArguments) const {
+    CodeGenSpecifics::Marshalling marshal;
+
+    auto [recSize, recAlign] = fir::getTypeSizeAndAlignmentOrCrash(
+        loc, recTy, getDataLayout(), kindMap);
+    mlir::MLIRContext *context = recTy.getContext();
+
+    if (recSize == 0) {
+      TODO(loc, "unsupported empty struct type for BIND(C), "
+                "VALUE derived type argument and type return");
+    }
+
+    if (recSize > 2 * GRLenInChar) {
+      marshal.emplace_back(
+          fir::ReferenceType::get(recTy),
+          AT{recAlign, /*byval=*/!isResult, /*sret=*/isResult});
+      return marshal;
+    }
+
+    // Pass by FARs(and GARs)
+    mlir::Type field1Ty = nullptr, field2Ty = nullptr;
+    if (detectFARsEligibleStruct(loc, recTy, field1Ty, field2Ty) &&
+        hasEnoughRegisters(loc, GARsLeft, FARsLeft, previousArguments, field1Ty,
+                           field2Ty)) {
+      if (!isResult) {
+        if (field1Ty)
+          marshal.emplace_back(field1Ty, AT{});
+        if (field2Ty)
+          marshal.emplace_back(field2Ty, AT{});
+      } else {
+        // field1Ty is always preferred over field2Ty for assignment, so there
+        // will never be a case where field1Ty == nullptr and field2Ty !=
+        // nullptr.
+        if (field1Ty && !field2Ty)
+          marshal.emplace_back(field1Ty, AT{});
+        else if (field1Ty && field2Ty)
+          marshal.emplace_back(
+              mlir::TupleType::get(context,
+                                   mlir::TypeRange{field1Ty, field2Ty}),
+              AT{/*alignment=*/0, /*byval=*/true});
+      }
+      return marshal;
+    }
+
+    if (recSize <= GRLenInChar) {
+      marshal.emplace_back(mlir::IntegerType::get(context, GRLen), AT{});
+      return marshal;
+    }
+
+    if (recAlign == 2 * GRLenInChar) {
+      marshal.emplace_back(mlir::IntegerType::get(context, 2 * GRLen), AT{});
+      return marshal;
+    }
+
+    // recSize > GRLenInChar && recSize <= 2 * GRLenInChar
+    marshal.emplace_back(
+        fir::SequenceType::get({2}, mlir::IntegerType::get(context, GRLen)),
+        AT{});
+    return marshal;
+  }
+
+  /// Marshal a derived type passed by value like a C struct.
+  CodeGenSpecifics::Marshalling
+  structArgumentType(mlir::Location loc, fir::RecordType recTy,
+                     const Marshalling &previousArguments) const override {
+    int GARsLeft = 8;
+    int FARsLeft = FRLen ? 8 : 0;
+
+    return classifyStruct(loc, recTy, GARsLeft, FARsLeft, /*isResult=*/false,
+                          previousArguments);
+  }
+
+  CodeGenSpecifics::Marshalling
+  structReturnType(mlir::Location loc, fir::RecordType recTy) const override {
+    // The rules for return and argument types are the same.
+    int GARsLeft = 2;
+    int FARsLeft = FRLen ? 2 : 0;
+    return classifyStruct(loc, recTy, GARsLeft, FARsLeft, /*isResult=*/true,
+                          {});
+  }
 };
 } // namespace
 
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index 04a3ea684642c..5a042b34a58c0 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -27,6 +27,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/DataLayout.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/STLExtras.h"
@@ -61,14 +62,21 @@ struct FixupTy {
   FixupTy(Codes code, std::size_t index,
           std::function<void(mlir::func::FuncOp)> &&finalizer)
       : code{code}, index{index}, finalizer{finalizer} {}
+  FixupTy(Codes code, std::size_t index,
+          std::function<void(mlir::gpu::GPUFuncOp)> &&finalizer)
+      : code{code}, index{index}, gpuFinalizer{finalizer} {}
   FixupTy(Codes code, std::size_t index, std::size_t second,
           std::function<void(mlir::func::FuncOp)> &&finalizer)
       : code{code}, index{index}, second{second}, finalizer{finalizer} {}
+  FixupTy(Codes code, std::size_t index, std::size_t second,
+          std::function<void(mlir::gpu::GPUFuncOp)> &&finalizer)
+      : code{code}, index{index}, second{second}, gpuFinalizer{finalizer} {}
 
   Codes code;
   std::size_t index;
   std::size_t second{};
   std::optional<std::function<void(mlir::func::FuncOp)>> finalizer{};
+  std::optional<std::function<void(mlir::gpu::GPUFuncOp)>> gpuFinalizer{};
 }; // namespace
 
 /// Target-specific rewriting of the FIR. This is a prerequisite pass to code
@@ -403,7 +411,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       unsigned index = e.index();
       llvm::TypeSwitch<mlir::Type>(ty)
           .template Case<fir::BoxCharType>([&](fir::BoxCharType boxTy) {
-            bool sret;
             if constexpr (std::is_same_v<std::decay_t<A>, fir::CallOp>) {
               if (noCharacterConversion) {
                 newInTyAndAttrs.push_back(
@@ -411,17 +418,14 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                 newOpers.push_back(oper);
                 return;
               }
-              sret = callOp.getCallee() &&
-                     functionArgIsSRet(
-                         index, getModule().lookupSymbol<mlir::func::FuncOp>(
-                                    *callOp.getCallee()));
             } else {
-              // TODO: dispatch case; how do we put arguments on a call?
-              // We cannot put both an sret and the dispatch object first.
-              sret = false;
-              TODO(loc, "dispatch + sret not supported yet");
+              // TODO: dispatch case; it used to be a to-do because of sret,
+              // but is not tested and maybe should be removed. This pass is
+              // anyway ran after lowering fir.dispatch in flang, so maybe that
+              // should just be a requirement of the pass.
+              TODO(loc, "ABI of fir.dispatch with character arguments");
             }
-            auto m = specifics->boxcharArgumentType(boxTy.getEleTy(), sret);
+            auto m = specifics->boxcharArgumentType(boxTy.getEleTy());
             auto unbox = rewriter->create<fir::UnboxCharOp>(
                 loc, std::get<mlir::Type>(m[0]), std::get<mlir::Type>(m[1]),
                 oper);
@@ -722,8 +726,16 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       if (targetFeaturesAttr)
         fn->setAttr("target_features", targetFeaturesAttr);
 
-      convertSignature(fn);
+      convertSignature<mlir::func::ReturnOp, mlir::func::FuncOp>(fn);
     }
+
+    for (auto gpuMod : mod.getOps<mlir::gpu::GPUModuleOp>()) {
+      for (auto fn : gpuMod.getOps<mlir::func::FuncOp>())
+        convertSignature<mlir::func::ReturnOp, mlir::func::FuncOp>(fn);
+      for (auto fn : gpuMod.getOps<mlir::gpu::GPUFuncOp>())
+        convertSignature<mlir::gpu::ReturnOp, mlir::gpu::GPUFuncOp>(fn);
+    }
+
     return mlir::success();
   }
 
@@ -768,17 +780,20 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
 
   /// Determine if the signature has host associations. The host association
   /// argument may need special target specific rewriting.
-  static bool hasHostAssociations(mlir::func::FuncOp func) {
+  template <typename OpTy>
+  static bool hasHostAssociations(OpTy func) {
     std::size_t end = func.getFunctionType().getInputs().size();
     for (std::size_t i = 0; i < end; ++i)
-      if (func.getArgAttrOfType<mlir::UnitAttr>(i, fir::getHostAssocAttrName()))
+      if (func.template getArgAttrOfType<mlir::UnitAttr>(
+              i, fir::getHostAssocAttrName()))
         return true;
     return false;
   }
 
   /// Rewrite the signatures and body of the `FuncOp`s in the module for
   /// the immediately subsequent target code gen.
-  void convertSignature(mlir::func::FuncOp func) {
+  template <typename ReturnOpTy, typename FuncOpTy>
+  void convertSignature(FuncOpTy func) {
     auto funcTy = mlir::cast<mlir::FunctionType>(func.getFunctionType());
     if (hasPortableSignature(funcTy, func) && !hasHostAssociations(func))
       return;
@@ -803,13 +818,13 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     // Convert return value(s)
     for (auto ty : funcTy.getResults())
       llvm::TypeSwitch<mlir::Type>(ty)
-          .Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
+          .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             if (noComplexConversion)
               newResTys.push_back(cmplx);
             else
               doComplexReturn(func, cmplx, newResTys, newInTyAndAttrs, fixups);
           })
-          .Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
+          .template Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
             auto m = specifics->integerArgumentType(func.getLoc(), intTy);
             assert(m.size() == 1);
             auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
@@ -823,7 +838,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                                                 rewriter->getUnitAttr()));
             newResTys.push_back(retTy);
           })
-          .Case<fir::RecordType>([&](fir::RecordType recTy) {
+          .template Case<fir::RecordType>([&](fir::RecordType recTy) {
             doStructReturn(func, recTy, newResTys, newInTyAndAttrs, fixups);
           })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
@@ -838,7 +853,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       auto ty = e.value();
       unsigned index = e.index();
       llvm::TypeSwitch<mlir::Type>(ty)
-          .Case<fir::BoxCharType>([&](fir::BoxCharType boxTy) {
+          .template Case<fir::BoxCharType>([&](fir::BoxCharType boxTy) {
             if (noCharacterConversion) {
               newInTyAndAttrs.push_back(
                   fir::CodeGenSpecifics::getTypeAndAttr(boxTy));
@@ -846,33 +861,25 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
               // Convert a CHARACTER argument type. This can involve separating
               // the pointer and the LEN into two arguments and moving the LEN
               // argument to the end of the arg list.
-              bool sret = functionArgIsSRet(index, func);
-              for (auto e : llvm::enumerate(specifics->boxcharArgumentType(
-                       boxTy.getEleTy(), sret))) {
-                auto &tup = e.value();
-                auto index = e.index();
+              for (auto &tup :
+                   specifics->boxcharArgumentType(boxTy.getEleTy())) {
                 auto attr = std::get<fir::CodeGenSpecifics::Attributes>(tup);
                 auto argTy = std::get<mlir::Type>(tup);
                 if (attr.isAppend()) {
                   trailingTys.push_back(argTy);
                 } else {
-                  if (sret) {
-                    fixups.emplace_back(FixupTy::Codes::CharPair,
-                                        newInTyAndAttrs.size(), index);
-                  } else {
-                    fixups.emplace_back(FixupTy::Codes::Trailing,
-                                        newInTyAndAttrs.size(),
-                                        trailingTys.size());
-                  }
+                  fixups.emplace_back(FixupTy::Codes::Trailing,
+                                      newInTyAndAttrs.size(),
+                                      trailingTys.size());
                   newInTyAndAttrs.push_back(tup);
                 }
               }
             }
           })
-          .Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
+          .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             doComplexArg(func, cmplx, newInTyAndAttrs, fixups);
           })
-          .Case<mlir::TupleType>([&](mlir::TupleType tuple) {
+          .template Case<mlir::TupleType>([&](mlir::TupleType tuple) {
             if (fir::isCharacterProcedureTuple(tuple)) {
               fixups.emplace_back(FixupTy::Codes::TrailingCharProc,
                                   newInTyAndAttrs.size(), trailingTys.size());
@@ -884,7 +891,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                   fir::CodeGenSpecifics::getTypeAndAttr(ty));
             }
           })
-          .Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
+          .template Case<mlir::IntegerType>([&](mlir::IntegerType intTy) {
             auto m = specifics->integerArgumentType(func.getLoc(), intTy);
             assert(m.size() == 1);
             auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
@@ -893,7 +900,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
             if (!extensionAttrName.empty() &&
                 isFuncWithCCallingConvention(func))
               fixups.emplace_back(FixupTy::Codes::ArgumentType, argNo,
-                                  [=](mlir::func::FuncOp func) {
+                                  [=](FuncOpTy func) {
                                     func.setArgAttr(
                                         argNo, extensionAttrName,
                                         mlir::UnitAttr::get(func.getContext()));
@@ -909,8 +916,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                 fir::CodeGenSpecifics::getTypeAndAttr(ty));
           });
 
-      if (func.getArgAttrOfType<mlir::UnitAttr>(index,
-                                                fir::getHostAssocAttrName())) {
+      if (func.template getArgAttrOfType<mlir::UnitAttr>(
+              index, fir::getHostAssocAttrName())) {
         extraAttrs.push_back(
             {newInTyAndAttrs.size() - 1,
              rewriter->getNamedAttr("llvm.nest", rewriter->getUnitAttr())});
@@ -985,27 +992,27 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
           auto newArg =
               func.front().insertArgument(fixup.index, fixupType, loc);
           offset++;
-          func.walk([&](mlir::func::ReturnOp ret) {
+          func.walk([&](ReturnOpTy ret) {
             rewriter->setInsertionPoint(ret);
             auto oldOper = ret.getOperand(0);
             auto oldOperTy = fir::ReferenceType::get(oldOper.getType());
             auto cast =
                 rewriter->create<fir::ConvertOp>(loc, oldOperTy, newArg);
             rewriter->create<fir::StoreOp>(loc, oldOper, cast);
-            rewriter->create<mlir::func::ReturnOp>(loc);
+            rewriter->create<ReturnOpTy>(loc);
             ret.erase();
           });
         } break;
         case FixupTy::Codes::ReturnType: {
           // The function is still returning a value, but its type has likely
           // changed to suit the target ABI convention.
-          func.walk([&](mlir::func::ReturnOp ret) {
+          func.walk([&](ReturnOpTy ret) {
             rewriter->setInsertionPoint(ret);
             auto oldOper = ret.getOperand(0);
             mlir::Value bitcast =
                 convertValueInMemory(loc, oldOper, newResTys[fixup.index],
                                      /*inputMayBeBigger=*/false);
-            rewriter->create<mlir::func::ReturnOp>(loc, bitcast);
+            rewriter->create<ReturnOpTy>(loc, bitcast);
             ret.erase();
           });
         } break;
@@ -1107,19 +1114,18 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       }
     }
 
-    for (auto &fixup : fixups)
-      if (fixup.finalizer)
-        (*fixup.finalizer)(func);
-  }
-
-  inline bool functionArgIsSRet(unsigned index, mlir::func::FuncOp func) {
-    if (auto attr = func.getArgAttrOfType<mlir::TypeAttr>(index, "llvm.sret"))
-      return true;
-    return false;
+    for (auto &fixup : fixups) {
+      if constexpr (std::is_same_v<FuncOpTy, mlir::func::FuncOp>)
+        if (fixup.finalizer)
+          (*fixup.finalizer)(func);
+      if constexpr (std::is_same_v<FuncOpTy, mlir::gpu::GPUFuncOp>)
+        if (fixup.gpuFinalizer)
+          (*fixup.gpuFinalizer)(func);
+    }
   }
 
-  template <typename Ty, typename FIXUPS>
-  void doReturn(mlir::func::FuncOp func, Ty &newResTys,
+  template <typename OpTy, typename Ty, typename FIXUPS>
+  void doReturn(OpTy func, Ty &newResTys,
                 fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
                 FIXUPS &fixups, fir::CodeGenSpecifics::Marshalling &m) {
     assert(m.size() == 1 &&
@@ -1131,7 +1137,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       unsigned argNo = newInTyAndAttrs.size();
       if (auto align = attr.getAlignment())
         fixups.emplace_back(
-            FixupTy::Codes::ReturnAsStore, argNo, [=](mlir::func::FuncOp func) {
+            FixupTy::Codes::ReturnAsStore, argNo, [=](OpTy func) {
               auto elemType = fir::dyn_cast_ptrOrBoxEleTy(
                   func.getFunctionType().getInput(argNo));
               func.setArgAttr(argNo, "llvm.sret",
@@ -1142,7 +1148,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
             });
       else
         fixups.emplace_back(FixupTy::Codes::ReturnAsStore, argNo,
-                            [=](mlir::func::FuncOp func) {
+                            [=](OpTy func) {
                               auto elemType = fir::dyn_cast_ptrOrBoxEleTy(
                                   func.getFunctionType().getInput(argNo));
                               func.setArgAttr(argNo, "llvm.sret",
@@ -1153,8 +1159,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     }
     if (auto align = attr.getAlignment())
       fixups.emplace_back(
-          FixupTy::Codes::ReturnType, newResTys.size(),
-          [=](mlir::func::FuncOp func) {
+          FixupTy::Codes::ReturnType, newResTys.size(), [=](OpTy func) {
             func.setArgAttr(
                 newResTys.size(), "llvm.align",
                 rewriter->getIntegerAttr(rewriter->getIntegerType(32), align));
@@ -1167,9 +1172,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
   /// Convert a complex return value. This can involve converting the return
   /// value to a "hidden" first argument or packing the complex into a wide
   /// GPR.
-  template <typename Ty, typename FIXUPS>
-  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
-                       Ty &newResTys,
+  template <typename OpTy, typename Ty, typename FIXUPS>
+  void doComplexReturn(OpTy func, mlir::ComplexType cmplx, Ty &newResTys,
                        fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
                        FIXUPS &fixups) {
     if (noComplexConversion) {
@@ -1181,9 +1185,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
   }
 
-  template <typename Ty, typename FIXUPS>
-  void doStructReturn(mlir::func::FuncOp func, fir::RecordType recTy,
-                      Ty &newResTys,
+  template <typename OpTy, typename Ty, typename FIXUPS>
+  void doStructReturn(OpTy func, fir::RecordType recTy, Ty &newResTys,
                       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
                       FIXUPS &fixups) {
     if (noStructConversion) {
@@ -1194,12 +1197,10 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
   }
 
-  template <typename FIXUPS>
-  void
-  createFuncOpArgFixups(mlir::func::FuncOp func,
-                        fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
-                        fir::CodeGenSpecifics::Marshalling &argsInTys,
-                        FIXUPS &fixups) {
+  template <typename OpTy, typename FIXUPS>
+  void createFuncOpArgFixups(
+      OpTy func, fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+      fir::CodeGenSpecifics::Marshalling &argsInTys, FIXUPS &fixups) {
     const auto fixupCode = argsInTys.size() > 1 ? FixupTy::Codes::Split
                                                 : FixupTy::Codes::ArgumentType;
     for (auto e : llvm::enumerate(argsInTys)) {
@@ -1210,7 +1211,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       if (attr.isByVal()) {
         if (auto align = attr.getAlignment())
           fixups.emplace_back(FixupTy::Codes::ArgumentAsLoad, argNo,
-                              [=](mlir::func::FuncOp func) {
+                              [=](OpTy func) {
                                 auto elemType = fir::dyn_cast_ptrOrBoxEleTy(
                                     func.getFunctionType().getInput(argNo));
                                 func.setArgAttr(argNo, "llvm.byval",
@@ -1222,8 +1223,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                               });
         else
           fixups.emplace_back(FixupTy::Codes::ArgumentAsLoad,
-                              newInTyAndAttrs.size(),
-                              [=](mlir::func::FuncOp func) {
+                              newInTyAndAttrs.size(), [=](OpTy func) {
                                 auto elemType = fir::dyn_cast_ptrOrBoxEleTy(
                                     func.getFunctionType().getInput(argNo));
                                 func.setArgAttr(argNo, "llvm.byval",
@@ -1232,7 +1232,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       } else {
         if (auto align = attr.getAlignment())
           fixups.emplace_back(
-              fixupCode, argNo, index, [=](mlir::func::FuncOp func) {
+              fixupCode, argNo, index, [=](OpTy func) {
                 func.setArgAttr(argNo, "llvm.align",
                                 rewriter->getIntegerAttr(
                                     rewriter->getIntegerType(32), align));
@@ -1247,8 +1247,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
   /// Convert a complex argument value. This can involve storing the value to
   /// a temporary memory location or factoring the value into two distinct
   /// arguments.
-  template <typename FIXUPS>
-  void doComplexArg(mlir::func::FuncOp func, mlir::ComplexType cmplx,
+  template <typename OpTy, typename FIXUPS>
+  void doComplexArg(OpTy func, mlir::ComplexType cmplx,
                     fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
                     FIXUPS &fixups) {
     if (noComplexConversion) {
@@ -1260,8 +1260,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     createFuncOpArgFixups(func, newInTyAndAttrs, cplxArgs, fixups);
   }
 
-  template <typename FIXUPS>
-  void doStructArg(mlir::func::FuncOp func, fir::RecordType recTy,
+  template <typename OpTy, typename FIXUPS>
+  void doStructArg(OpTy func, fir::RecordType recTy,
                    fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
                    FIXUPS &fixups) {
     if (noStructConversion) {
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
index 63eac46a99771..0c1424d11b515 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFToLLVMIRTranslation.cpp
@@ -66,8 +66,12 @@ LogicalResult registerKernel(cuf::RegisterKernelOp op,
       llvm::FunctionType::get(
           ptrTy, ArrayRef<llvm::Type *>({ptrTy, ptrTy, ptrTy}), false));
   llvm::Value *modulePtr = moduleTranslation.lookupValue(op.getModulePtr());
+  if (!modulePtr)
+    return op.emitError() << "Couldn't find the module ptr";
   llvm::Function *fctSym =
       moduleTranslation.lookupFunction(op.getKernelName().str());
+  if (!fctSym)
+    return op.emitError() << "Couldn't find kernel name symbol";
   builder.CreateCall(fct, {modulePtr, fctSym,
                            getOrCreateFunctionName(
                                module, builder, op.getKernelModuleName().str(),
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
index 0b61c0edce622..cb77aef74acd5 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIRDialect.cpp
@@ -215,3 +215,25 @@ bool hlfir::mayHaveAllocatableComponent(mlir::Type ty) {
   return fir::isPolymorphicType(ty) || fir::isUnlimitedPolymorphicType(ty) ||
          fir::isRecordWithAllocatableMember(hlfir::getFortranElementType(ty));
 }
+
+mlir::Type hlfir::getExprType(mlir::Type variableType) {
+  hlfir::ExprType::Shape typeShape;
+  bool isPolymorphic = fir::isPolymorphicType(variableType);
+  mlir::Type type = getFortranElementOrSequenceType(variableType);
+  if (auto seqType = mlir::dyn_cast<fir::SequenceType>(type)) {
+    assert(!seqType.hasUnknownShape() && "assumed-rank cannot be expressions");
+    typeShape.append(seqType.getShape().begin(), seqType.getShape().end());
+    type = seqType.getEleTy();
+  }
+  return hlfir::ExprType::get(variableType.getContext(), typeShape, type,
+                              isPolymorphic);
+}
+
+bool hlfir::isFortranIntegerScalarOrArrayObject(mlir::Type type) {
+  if (isBoxAddressType(type))
+    return false;
+
+  mlir::Type unwrappedType = fir::unwrapPassByRefType(fir::unwrapRefType(type));
+  mlir::Type elementType = getFortranElementType(unwrappedType);
+  return mlir::isa<mlir::IntegerType>(elementType);
+}
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index b593383ff2848..ad53527f43441 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -333,6 +333,25 @@ static void printDesignatorComplexPart(mlir::OpAsmPrinter &p,
       p << "real";
   }
 }
+template <typename Op>
+static llvm::LogicalResult verifyTypeparams(Op &op, mlir::Type elementType,
+                                            unsigned numLenParam) {
+  if (mlir::isa<fir::CharacterType>(elementType)) {
+    if (numLenParam != 1)
+      return op.emitOpError("must be provided one length parameter when the "
+                            "result is a character");
+  } else if (fir::isRecordWithTypeParameters(elementType)) {
+    if (numLenParam !=
+        mlir::cast<fir::RecordType>(elementType).getNumLenParams())
+      return op.emitOpError("must be provided the same number of length "
+                            "parameters as in the result derived type");
+  } else if (numLenParam != 0) {
+    return op.emitOpError(
+        "must not be provided length parameters if the result "
+        "type does not have length parameters");
+  }
+  return mlir::success();
+}
 
 llvm::LogicalResult hlfir::DesignateOp::verify() {
   mlir::Type memrefType = getMemref().getType();
@@ -462,20 +481,10 @@ llvm::LogicalResult hlfir::DesignateOp::verify() {
         return emitOpError("shape must be a fir.shape or fir.shapeshift with "
                            "the rank of the result");
     }
-    auto numLenParam = getTypeparams().size();
-    if (mlir::isa<fir::CharacterType>(outputElementType)) {
-      if (numLenParam != 1)
-        return emitOpError("must be provided one length parameter when the "
-                           "result is a character");
-    } else if (fir::isRecordWithTypeParameters(outputElementType)) {
-      if (numLenParam !=
-          mlir::cast<fir::RecordType>(outputElementType).getNumLenParams())
-        return emitOpError("must be provided the same number of length "
-                           "parameters as in the result derived type");
-    } else if (numLenParam != 0) {
-      return emitOpError("must not be provided length parameters if the result "
-                         "type does not have length parameters");
-    }
+    if (auto res =
+            verifyTypeparams(*this, outputElementType, getTypeparams().size());
+        failed(res))
+      return res;
   }
   return mlir::success();
 }
@@ -1332,6 +1341,108 @@ void hlfir::MatmulTransposeOp::getEffects(
   getIntrinsicEffects(getOperation(), effects);
 }
 
+//===----------------------------------------------------------------------===//
+// CShiftOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult hlfir::CShiftOp::verify() {
+  mlir::Value array = getArray();
+  fir::SequenceType arrayTy = mlir::cast<fir::SequenceType>(
+      hlfir::getFortranElementOrSequenceType(array.getType()));
+  llvm::ArrayRef<int64_t> inShape = arrayTy.getShape();
+  std::size_t arrayRank = inShape.size();
+  mlir::Type eleTy = arrayTy.getEleTy();
+  hlfir::ExprType resultTy = mlir::cast<hlfir::ExprType>(getResult().getType());
+  llvm::ArrayRef<int64_t> resultShape = resultTy.getShape();
+  std::size_t resultRank = resultShape.size();
+  mlir::Type resultEleTy = resultTy.getEleTy();
+  mlir::Value shift = getShift();
+  mlir::Type shiftTy = hlfir::getFortranElementOrSequenceType(shift.getType());
+
+  if (eleTy != resultEleTy) {
+    if (mlir::isa<fir::CharacterType>(eleTy) &&
+        mlir::isa<fir::CharacterType>(resultEleTy)) {
+      auto eleCharTy = mlir::cast<fir::CharacterType>(eleTy);
+      auto resultCharTy = mlir::cast<fir::CharacterType>(resultEleTy);
+      if (eleCharTy.getFKind() != resultCharTy.getFKind())
+        return emitOpError("kind mismatch between input and output arrays");
+      if (eleCharTy.getLen() != fir::CharacterType::unknownLen() &&
+          resultCharTy.getLen() != fir::CharacterType::unknownLen() &&
+          eleCharTy.getLen() != resultCharTy.getLen())
+        return emitOpError(
+            "character LEN mismatch between input and output arrays");
+    } else {
+      return emitOpError(
+          "input and output arrays should have the same element type");
+    }
+  }
+
+  if (arrayRank != resultRank)
+    return emitOpError("input and output arrays should have the same rank");
+
+  constexpr int64_t unknownExtent = fir::SequenceType::getUnknownExtent();
+  for (auto [inDim, resultDim] : llvm::zip(inShape, resultShape))
+    if (inDim != unknownExtent && resultDim != unknownExtent &&
+        inDim != resultDim)
+      return emitOpError(
+          "output array's shape conflicts with the input array's shape");
+
+  int64_t dimVal = -1;
+  if (!getDim())
+    dimVal = 1;
+  else if (auto dim = fir::getIntIfConstant(getDim()))
+    dimVal = *dim;
+
+  // The DIM argument may be statically invalid (e.g. exceed the
+  // input array rank) in dead code after constant propagation,
+  // so avoid some checks unless useStrictIntrinsicVerifier is true.
+  if (useStrictIntrinsicVerifier && dimVal != -1) {
+    if (dimVal < 1)
+      return emitOpError("DIM must be >= 1");
+    if (dimVal > static_cast<int64_t>(arrayRank))
+      return emitOpError("DIM must be <= input array's rank");
+  }
+
+  if (auto shiftSeqTy = mlir::dyn_cast<fir::SequenceType>(shiftTy)) {
+    // SHIFT is an array. Verify the rank and the shape (if DIM is constant).
+    llvm::ArrayRef<int64_t> shiftShape = shiftSeqTy.getShape();
+    std::size_t shiftRank = shiftShape.size();
+    if (shiftRank != arrayRank - 1)
+      return emitOpError(
+          "SHIFT's rank must be 1 less than the input array's rank");
+
+    if (useStrictIntrinsicVerifier && dimVal != -1) {
+      // SHIFT's shape must be [d(1), d(2), ..., d(DIM-1), d(DIM+1), ..., d(n)],
+      // where [d(1), d(2), ..., d(n)] is the shape of the ARRAY.
+      int64_t arrayDimIdx = 0;
+      int64_t shiftDimIdx = 0;
+      for (auto shiftDim : shiftShape) {
+        if (arrayDimIdx == dimVal - 1)
+          ++arrayDimIdx;
+
+        if (inShape[arrayDimIdx] != unknownExtent &&
+            shiftDim != unknownExtent && inShape[arrayDimIdx] != shiftDim)
+          return emitOpError("SHAPE(ARRAY)(" + llvm::Twine(arrayDimIdx + 1) +
+                             ") must be equal to SHAPE(SHIFT)(" +
+                             llvm::Twine(shiftDimIdx + 1) +
+                             "): " + llvm::Twine(inShape[arrayDimIdx]) +
+                             " != " + llvm::Twine(shiftDim));
+        ++arrayDimIdx;
+        ++shiftDimIdx;
+      }
+    }
+  }
+
+  return mlir::success();
+}
+
+void hlfir::CShiftOp::getEffects(
+    llvm::SmallVectorImpl<
+        mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
+        &effects) {
+  getIntrinsicEffects(getOperation(), effects);
+}
+
 //===----------------------------------------------------------------------===//
 // AssociateOp
 //===----------------------------------------------------------------------===//
@@ -1418,16 +1529,7 @@ llvm::LogicalResult hlfir::EndAssociateOp::verify() {
 void hlfir::AsExprOp::build(mlir::OpBuilder &builder,
                             mlir::OperationState &result, mlir::Value var,
                             mlir::Value mustFree) {
-  hlfir::ExprType::Shape typeShape;
-  bool isPolymorphic = fir::isPolymorphicType(var.getType());
-  mlir::Type type = getFortranElementOrSequenceType(var.getType());
-  if (auto seqType = mlir::dyn_cast<fir::SequenceType>(type)) {
-    typeShape.append(seqType.getShape().begin(), seqType.getShape().end());
-    type = seqType.getEleTy();
-  }
-
-  auto resultType = hlfir::ExprType::get(builder.getContext(), typeShape, type,
-                                         isPolymorphic);
+  mlir::Type resultType = hlfir::getExprType(var.getType());
   return build(builder, result, resultType, var, mustFree);
 }
 
@@ -1989,6 +2091,45 @@ hlfir::GetLengthOp::canonicalize(GetLengthOp getLength,
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// EvaluateInMemoryOp
+//===----------------------------------------------------------------------===//
+
+void hlfir::EvaluateInMemoryOp::build(mlir::OpBuilder &builder,
+                                      mlir::OperationState &odsState,
+                                      mlir::Type resultType, mlir::Value shape,
+                                      mlir::ValueRange typeparams) {
+  odsState.addTypes(resultType);
+  if (shape)
+    odsState.addOperands(shape);
+  odsState.addOperands(typeparams);
+  odsState.addAttribute(
+      getOperandSegmentSizeAttr(),
+      builder.getDenseI32ArrayAttr(
+          {shape ? 1 : 0, static_cast<int32_t>(typeparams.size())}));
+  mlir::Region *bodyRegion = odsState.addRegion();
+  bodyRegion->push_back(new mlir::Block{});
+  mlir::Type memType = fir::ReferenceType::get(
+      hlfir::getFortranElementOrSequenceType(resultType));
+  bodyRegion->front().addArgument(memType, odsState.location);
+  EvaluateInMemoryOp::ensureTerminator(*bodyRegion, builder, odsState.location);
+}
+
+llvm::LogicalResult hlfir::EvaluateInMemoryOp::verify() {
+  unsigned shapeRank = 0;
+  if (mlir::Value shape = getShape())
+    if (auto shapeTy = mlir::dyn_cast<fir::ShapeType>(shape.getType()))
+      shapeRank = shapeTy.getRank();
+  auto exprType = mlir::cast<hlfir::ExprType>(getResult().getType());
+  if (shapeRank != exprType.getRank())
+    return emitOpError("`shape` rank must match the result rank");
+  mlir::Type elementType = exprType.getElementType();
+  if (auto res = verifyTypeparams(*this, elementType, getTypeparams().size());
+      failed(res))
+    return res;
+  return mlir::success();
+}
+
 #include "flang/Optimizer/HLFIR/HLFIROpInterfaces.cpp.inc"
 #define GET_OP_CLASSES
 #include "flang/Optimizer/HLFIR/HLFIREnums.cpp.inc"
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 1848dbe2c7a2c..48e3db0eb39b6 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -806,8 +806,12 @@ struct ElementalOpConversion
     // elemental is a "view" over a variable (e.g parentheses or transpose).
     if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) {
       if (asExpr->hasOneUse() && !asExpr.isMove()) {
-        elementValue = hlfir::Entity{asExpr.getVar()};
-        rewriter.eraseOp(asExpr);
+        // Check that the asExpr is the final operation before the yield,
+        // otherwise, clean-ups could impact the memory being re-used.
+        if (asExpr->getNextNode() == yield.getOperation()) {
+          elementValue = hlfir::Entity{asExpr.getVar()};
+          rewriter.eraseOp(asExpr);
+        }
       }
     }
     rewriter.eraseOp(yield);
@@ -905,6 +909,26 @@ struct CharExtremumOpConversion
   }
 };
 
+struct EvaluateInMemoryOpConversion
+    : public mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp> {
+  using mlir::OpConversionPattern<
+      hlfir::EvaluateInMemoryOp>::OpConversionPattern;
+  explicit EvaluateInMemoryOpConversion(mlir::MLIRContext *ctx)
+      : mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp>{ctx} {}
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::EvaluateInMemoryOp evalInMemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Location loc = evalInMemOp->getLoc();
+    fir::FirOpBuilder builder(rewriter, evalInMemOp.getOperation());
+    auto [temp, isHeapAlloc] = hlfir::computeEvaluateOpInNewTemp(
+        loc, builder, evalInMemOp, adaptor.getShape(), adaptor.getTypeparams());
+    mlir::Value bufferizedExpr =
+        packageBufferizedExpr(loc, builder, temp, isHeapAlloc);
+    rewriter.replaceOp(evalInMemOp, bufferizedExpr);
+    return mlir::success();
+  }
+};
+
 class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
 public:
   void runOnOperation() override {
@@ -918,12 +942,13 @@ class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
     auto module = this->getOperation();
     auto *context = &getContext();
     mlir::RewritePatternSet patterns(context);
-    patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
-                    AssociateOpConversion, CharExtremumOpConversion,
-                    ConcatOpConversion, DestroyOpConversion,
-                    ElementalOpConversion, EndAssociateOpConversion,
-                    NoReassocOpConversion, SetLengthOpConversion,
-                    ShapeOfOpConversion, GetLengthOpConversion>(context);
+    patterns
+        .insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
+                AssociateOpConversion, CharExtremumOpConversion,
+                ConcatOpConversion, DestroyOpConversion, ElementalOpConversion,
+                EndAssociateOpConversion, EvaluateInMemoryOpConversion,
+                NoReassocOpConversion, SetLengthOpConversion,
+                ShapeOfOpConversion, GetLengthOpConversion>(context);
     mlir::ConversionTarget target(*context);
     // Note that YieldElementOp is not marked as an illegal operation.
     // It must be erased by its parent converter and there is no explicit
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index bfb4148d93bfd..36fae90c83fd6 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -463,6 +463,37 @@ struct MatmulTransposeOpConversion
   }
 };
 
+class CShiftOpConversion : public HlfirIntrinsicConversion<hlfir::CShiftOp> {
+  using HlfirIntrinsicConversion<hlfir::CShiftOp>::HlfirIntrinsicConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::CShiftOp cshift,
+                  mlir::PatternRewriter &rewriter) const override {
+    fir::FirOpBuilder builder{rewriter, cshift.getOperation()};
+    const mlir::Location &loc = cshift->getLoc();
+
+    llvm::SmallVector<IntrinsicArgument, 3> inArgs;
+    mlir::Value array = cshift.getArray();
+    inArgs.push_back({array, array.getType()});
+    mlir::Value shift = cshift.getShift();
+    inArgs.push_back({shift, shift.getType()});
+    inArgs.push_back({cshift.getDim(), builder.getI32Type()});
+
+    auto *argLowering = fir::getIntrinsicArgumentLowering("cshift");
+    llvm::SmallVector<fir::ExtendedValue, 3> args =
+        lowerArguments(cshift, inArgs, rewriter, argLowering);
+
+    mlir::Type scalarResultType =
+        hlfir::getFortranElementType(cshift.getType());
+
+    auto [resultExv, mustBeFreed] =
+        fir::genIntrinsicCall(builder, loc, "cshift", scalarResultType, args);
+
+    processReturnValue(cshift, resultExv, mustBeFreed, builder, rewriter);
+    return mlir::success();
+  }
+};
+
 class LowerHLFIRIntrinsics
     : public hlfir::impl::LowerHLFIRIntrinsicsBase<LowerHLFIRIntrinsics> {
 public:
@@ -475,7 +506,8 @@ class LowerHLFIRIntrinsics
                 AllOpConversion, AnyOpConversion, SumOpConversion,
                 ProductOpConversion, TransposeOpConversion, CountOpConversion,
                 DotProductOpConversion, MaxvalOpConversion, MinvalOpConversion,
-                MinlocOpConversion, MaxlocOpConversion>(context);
+                MinlocOpConversion, MaxlocOpConversion, CShiftOpConversion>(
+            context);
 
     // While conceptually this pass is performing dialect conversion, we use
     // pattern rewrites here instead of dialect conversion because this pass
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index a0160b233e3cd..ef6aabbceacb7 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -338,34 +338,20 @@ ElementalAssignBufferization::findMatch(hlfir::ElementalOp elemental) {
   if (!fir::isa_trivial(eleTy))
     return std::nullopt;
 
-  // the array must have the same shape as the elemental. CSE should have
-  // deduplicated the fir.shape operations where they are provably the same
-  // so we just have to check for the same ssa value
-  // TODO: add more ways of getting the shape of the array
-  mlir::Value arrayShape;
-  if (match.array.getDefiningOp())
-    arrayShape =
-        mlir::TypeSwitch<mlir::Operation *, mlir::Value>(
-            match.array.getDefiningOp())
-            .Case([](hlfir::DesignateOp designate) {
-              return designate.getShape();
-            })
-            .Case([](hlfir::DeclareOp declare) { return declare.getShape(); })
-            .Default([](mlir::Operation *) { return mlir::Value{}; });
-  if (!arrayShape) {
-    LLVM_DEBUG(llvm::dbgs() << "Can't get shape of " << match.array << " at "
-                            << elemental->getLoc() << "\n");
+  // The array must have the same shape as the elemental.
+  //
+  // f2018 10.2.1.2 (3) requires the lhs and rhs of an assignment to be
+  // conformable unless the lhs is an allocatable array. In HLFIR we can
+  // see this from the presence or absence of the realloc attribute on
+  // hlfir.assign. If it is not a realloc assignment, we can trust that
+  // the shapes do conform.
+  //
+  // TODO: the lhs's shape is dynamic, so it is hard to prove that
+  // there is no reallocation of the lhs due to the assignment.
+  // We can probably try generating multiple versions of the code
+  // with checking for the shape match, length parameters match, etc.
+  if (match.assign.getRealloc())
     return std::nullopt;
-  }
-  if (arrayShape != elemental.getShape()) {
-    // f2018 10.2.1.2 (3) requires the lhs and rhs of an assignment to be
-    // conformable unless the lhs is an allocatable array. In HLFIR we can
-    // see this from the presence or absence of the realloc attribute on
-    // hlfir.assign. If it is not a realloc assignment, we can trust that
-    // the shapes do conform
-    if (match.assign.getRealloc())
-      return std::nullopt;
-  }
 
   // the transformation wants to apply the elemental in a do-loop at the
   // hlfir.assign, check there are no effects which make this unsafe
@@ -1108,6 +1094,100 @@ class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
   }
 };
 
+class EvaluateIntoMemoryAssignBufferization
+    : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
+
+public:
+  using mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::EvaluateInMemoryOp,
+                  mlir::PatternRewriter &rewriter) const override;
+};
+
+static llvm::LogicalResult
+tryUsingAssignLhsDirectly(hlfir::EvaluateInMemoryOp evalInMem,
+                          mlir::PatternRewriter &rewriter) {
+  mlir::Location loc = evalInMem.getLoc();
+  hlfir::DestroyOp destroy;
+  hlfir::AssignOp assign;
+  for (auto user : llvm::enumerate(evalInMem->getUsers())) {
+    if (user.index() > 2)
+      return mlir::failure();
+    mlir::TypeSwitch<mlir::Operation *, void>(user.value())
+        .Case([&](hlfir::AssignOp op) { assign = op; })
+        .Case([&](hlfir::DestroyOp op) { destroy = op; });
+  }
+  if (!assign || !destroy || destroy.mustFinalizeExpr() ||
+      assign.isAllocatableAssignment())
+    return mlir::failure();
+
+  hlfir::Entity lhs{assign.getLhs()};
+  // EvaluateInMemoryOp memory is contiguous, so in general, it can only be
+  // replace by the LHS if the LHS is contiguous.
+  if (!lhs.isSimplyContiguous())
+    return mlir::failure();
+  // Character assignment may involves truncation/padding, so the LHS
+  // cannot be used to evaluate RHS in place without proving the LHS and
+  // RHS lengths are the same.
+  if (lhs.isCharacter())
+    return mlir::failure();
+  fir::AliasAnalysis aliasAnalysis;
+  // The region must not read or write the LHS.
+  // Note that getModRef is used instead of mlir::MemoryEffects because
+  // EvaluateInMemoryOp is typically expected to hold fir.calls and that
+  // Fortran calls cannot be modeled in a useful way with mlir::MemoryEffects:
+  // it is hard/impossible to list all the read/written SSA values in a call,
+  // but it is often possible to tell that an SSA value cannot be accessed,
+  // hence getModRef is needed here and below. Also note that getModRef uses
+  // mlir::MemoryEffects for operations that do not have special handling in
+  // getModRef.
+  if (aliasAnalysis.getModRef(evalInMem.getBody(), lhs).isModOrRef())
+    return mlir::failure();
+  // Any variables affected between the hlfir.evalInMem and assignment must not
+  // be read or written inside the region since it will be moved at the
+  // assignment insertion point.
+  auto effects = getEffectsBetween(evalInMem->getNextNode(), assign);
+  if (!effects) {
+    LLVM_DEBUG(
+        llvm::dbgs()
+        << "operation with unknown effects between eval_in_mem and assign\n");
+    return mlir::failure();
+  }
+  for (const mlir::MemoryEffects::EffectInstance &effect : *effects) {
+    mlir::Value affected = effect.getValue();
+    if (!affected ||
+        aliasAnalysis.getModRef(evalInMem.getBody(), affected).isModOrRef())
+      return mlir::failure();
+  }
+
+  rewriter.setInsertionPoint(assign);
+  fir::FirOpBuilder builder(rewriter, evalInMem.getOperation());
+  mlir::Value rawLhs = hlfir::genVariableRawAddress(loc, builder, lhs);
+  hlfir::computeEvaluateOpIn(loc, builder, evalInMem, rawLhs);
+  rewriter.eraseOp(assign);
+  rewriter.eraseOp(destroy);
+  rewriter.eraseOp(evalInMem);
+  return mlir::success();
+}
+
+llvm::LogicalResult EvaluateIntoMemoryAssignBufferization::matchAndRewrite(
+    hlfir::EvaluateInMemoryOp evalInMem,
+    mlir::PatternRewriter &rewriter) const {
+  if (mlir::succeeded(tryUsingAssignLhsDirectly(evalInMem, rewriter)))
+    return mlir::success();
+  // Rewrite to temp + as_expr here so that the assign + as_expr pattern can
+  // kick-in for simple types and at least implement the assignment inline
+  // instead of call Assign runtime.
+  fir::FirOpBuilder builder(rewriter, evalInMem.getOperation());
+  mlir::Location loc = evalInMem.getLoc();
+  auto [temp, isHeapAllocated] = hlfir::computeEvaluateOpInNewTemp(
+      loc, builder, evalInMem, evalInMem.getShape(), evalInMem.getTypeparams());
+  rewriter.replaceOpWithNewOp<hlfir::AsExprOp>(
+      evalInMem, temp, /*mustFree=*/builder.createBool(loc, isHeapAllocated));
+  return mlir::success();
+}
+
 class OptimizedBufferizationPass
     : public hlfir::impl::OptimizedBufferizationBase<
           OptimizedBufferizationPass> {
@@ -1130,6 +1210,7 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<VariableAssignBufferization>(context);
+    patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
     patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
     patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
     patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index 60b06437e6a98..b61f9767ccc2b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -10,6 +10,7 @@
 // into the calling function.
 //===----------------------------------------------------------------------===//
 
+#include "flang/Optimizer/Builder/Complex.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
@@ -28,6 +29,10 @@ namespace hlfir {
 #include "flang/Optimizer/HLFIR/Passes.h.inc"
 } // namespace hlfir
 
+static llvm::cl::opt<bool>
+    simplifySum("flang-simplify-hlfir-sum",
+                llvm::cl::desc("Expand hlfir.sum into an inline sequence"),
+                llvm::cl::init(false));
 namespace {
 
 class TransposeAsElementalConversion
@@ -90,6 +95,242 @@ class TransposeAsElementalConversion
   }
 };
 
+// Expand the SUM(DIM=CONSTANT) operation into .
+class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
+public:
+  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::SumOp sum,
+                  mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = sum.getLoc();
+    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
+    hlfir::ExprType expr = mlir::dyn_cast<hlfir::ExprType>(sum.getType());
+    assert(expr && "expected an expression type for the result of hlfir.sum");
+    mlir::Type elementType = expr.getElementType();
+    hlfir::Entity array = hlfir::Entity{sum.getArray()};
+    mlir::Value mask = sum.getMask();
+    mlir::Value dim = sum.getDim();
+    int64_t dimVal = fir::getIntIfConstant(dim).value_or(0);
+    mlir::Value resultShape, dimExtent;
+    std::tie(resultShape, dimExtent) =
+        genResultShape(loc, builder, array, dimVal);
+
+    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange inputIndices) -> hlfir::Entity {
+      // Loop over all indices in the DIM dimension, and reduce all values.
+      // We do not need to create the reduction loop always: if we can
+      // slice the input array given the inputIndices, then we can
+      // just apply a new SUM operation (total reduction) to the slice.
+      // For the time being, generate the explicit loop because the slicing
+      // requires generating an elemental operation for the input array
+      // (and the mask, if present).
+      // TODO: produce the slices and new SUM after adding a pattern
+      // for expanding total reduction SUM case.
+      mlir::Type indexType = builder.getIndexType();
+      auto one = builder.createIntegerConstant(loc, indexType, 1);
+      auto ub = builder.createConvert(loc, indexType, dimExtent);
+
+      // Initial value for the reduction.
+      mlir::Value initValue = genInitValue(loc, builder, elementType);
+
+      // The reduction loop may be unordered if FastMathFlags::reassoc
+      // transformations are allowed. The integer reduction is always
+      // unordered.
+      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
+                         static_cast<bool>(sum.getFastmath() &
+                                           mlir::arith::FastMathFlags::reassoc);
+
+      // If the mask is present and is a scalar, then we'd better load its value
+      // outside of the reduction loop making the loop unswitching easier.
+      // Maybe it is worth hoisting it from the elemental operation as well.
+      mlir::Value isPresentPred, maskValue;
+      if (mask) {
+        if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+          // MASK represented by a box might be dynamically optional,
+          // so we have to check for its presence before accessing it.
+          isPresentPred =
+              builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+        }
+
+        if (hlfir::Entity{mask}.isScalar())
+          maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
+      }
+
+      // NOTE: the outer elemental operation may be lowered into
+      // omp.workshare.loop_wrapper/omp.loop_nest later, so the reduction
+      // loop may appear disjoint from the workshare loop nest.
+      // Moreover, the inner loop is not strictly nested (due to the reduction
+      // starting value initialization), and the above omp dialect operations
+      // cannot produce results.
+      // It is unclear what we should do about it yet.
+      auto doLoop = builder.create<fir::DoLoopOp>(
+          loc, one, ub, one, isUnordered, /*finalCountValue=*/false,
+          mlir::ValueRange{initValue});
+
+      // Address the input array using the reduction loop's IV
+      // for the DIM dimension.
+      mlir::Value iv = doLoop.getInductionVar();
+      llvm::SmallVector<mlir::Value> indices{inputIndices};
+      indices.insert(indices.begin() + dimVal - 1, iv);
+
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      builder.setInsertionPointToStart(doLoop.getBody());
+      mlir::Value reductionValue = doLoop.getRegionIterArgs()[0];
+      fir::IfOp ifOp;
+      if (mask) {
+        // Make the reduction value update conditional on the value
+        // of the mask.
+        if (!maskValue) {
+          // If the mask is an array, use the elemental and the loop indices
+          // to address the proper mask element.
+          maskValue = genMaskValue(loc, builder, mask, isPresentPred, indices);
+        }
+        mlir::Value isUnmasked =
+            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
+        ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
+                                         /*withElseRegion=*/true);
+        // In the 'else' block return the current reduction value.
+        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+        builder.create<fir::ResultOp>(loc, reductionValue);
+
+        // In the 'then' block do the actual addition.
+        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      }
+
+      hlfir::Entity element = hlfir::getElementAt(loc, builder, array, indices);
+      hlfir::Entity elementValue =
+          hlfir::loadTrivialScalar(loc, builder, element);
+      // NOTE: we can use "Kahan summation" same way as the runtime
+      // (e.g. when fast-math is not allowed), but let's start with
+      // the simple version.
+      reductionValue = genScalarAdd(loc, builder, reductionValue, elementValue);
+      builder.create<fir::ResultOp>(loc, reductionValue);
+
+      if (ifOp) {
+        builder.setInsertionPointAfter(ifOp);
+        builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+      }
+
+      return hlfir::Entity{doLoop.getResult(0)};
+    };
+    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+        loc, builder, elementType, resultShape, {}, genKernel,
+        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
+        sum.getResult().getType());
+
+    // it wouldn't be safe to replace block arguments with a different
+    // hlfir.expr type. Types can differ due to differing amounts of shape
+    // information
+    assert(elementalOp.getResult().getType() == sum.getResult().getType());
+
+    rewriter.replaceOp(sum, elementalOp);
+    return mlir::success();
+  }
+
+private:
+  // Return fir.shape specifying the shape of the result
+  // of a SUM reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  static std::tuple<mlir::Value, mlir::Value>
+  genResultShape(mlir::Location loc, fir::FirOpBuilder &builder,
+                 hlfir::Entity array, int64_t dimVal) {
+    mlir::Value inShape = hlfir::genShape(loc, builder, array);
+    llvm::SmallVector<mlir::Value> inExtents =
+        hlfir::getExplicitExtentsFromShape(inShape, builder);
+    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+           "DIM must be present and a positive constant not exceeding "
+           "the array's rank");
+    if (inShape.getUses().empty())
+      inShape.getDefiningOp()->erase();
+
+    mlir::Value dimExtent = inExtents[dimVal - 1];
+    inExtents.erase(inExtents.begin() + dimVal - 1);
+    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+  }
+
+  // Generate the initial value for a SUM reduction with the given
+  // data type.
+  static mlir::Value genInitValue(mlir::Location loc,
+                                  fir::FirOpBuilder &builder,
+                                  mlir::Type elementType) {
+    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
+      const llvm::fltSemantics &sem = ty.getFloatSemantics();
+      return builder.createRealConstant(loc, elementType,
+                                        llvm::APFloat::getZero(sem));
+    } else if (auto ty = mlir::dyn_cast<mlir::ComplexType>(elementType)) {
+      mlir::Value initValue = genInitValue(loc, builder, ty.getElementType());
+      return fir::factory::Complex{builder, loc}.createComplex(ty, initValue,
+                                                               initValue);
+    } else if (mlir::isa<mlir::IntegerType>(elementType)) {
+      return builder.createIntegerConstant(loc, elementType, 0);
+    }
+
+    llvm_unreachable("unsupported SUM reduction type");
+  }
+
+  // Generate scalar addition of the two values (of the same data type).
+  static mlir::Value genScalarAdd(mlir::Location loc,
+                                  fir::FirOpBuilder &builder,
+                                  mlir::Value value1, mlir::Value value2) {
+    mlir::Type ty = value1.getType();
+    assert(ty == value2.getType() && "reduction values' types do not match");
+    if (mlir::isa<mlir::FloatType>(ty))
+      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::ComplexType>(ty))
+      return builder.create<fir::AddcOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::IntegerType>(ty))
+      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+    llvm_unreachable("unsupported SUM reduction type");
+  }
+
+  static mlir::Value genMaskValue(mlir::Location loc,
+                                  fir::FirOpBuilder &builder, mlir::Value mask,
+                                  mlir::Value isPresentPred,
+                                  mlir::ValueRange indices) {
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    fir::IfOp ifOp;
+    mlir::Type maskType =
+        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+    if (isPresentPred) {
+      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                       /*withElseRegion=*/true);
+
+      // Use 'true', if the mask is not present.
+      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+      mlir::Value trueValue = builder.createBool(loc, true);
+      trueValue = builder.createConvert(loc, maskType, trueValue);
+      builder.create<fir::ResultOp>(loc, trueValue);
+
+      // Load the mask value, if the mask is present.
+      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+    }
+
+    hlfir::Entity maskVar{mask};
+    if (maskVar.isScalar()) {
+      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+        // MASK may be a boxed scalar.
+        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
+      } else {
+        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+      }
+    } else {
+      // Load from the mask array.
+      assert(!indices.empty() && "no indices for addressing the mask array");
+      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
+      mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+    }
+
+    if (!isPresentPred)
+      return mask;
+
+    builder.create<fir::ResultOp>(loc, mask);
+    return ifOp.getResult(0);
+  }
+};
+
 class SimplifyHLFIRIntrinsics
     : public hlfir::impl::SimplifyHLFIRIntrinsicsBase<SimplifyHLFIRIntrinsics> {
 public:
@@ -97,6 +338,7 @@ class SimplifyHLFIRIntrinsics
     mlir::MLIRContext *context = &getContext();
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
+    patterns.insert<SumAsElementalConversion>(context);
     mlir::ConversionTarget target(*context);
     // don't transform transpose of polymorphic arrays (not currently supported
     // by hlfir.elemental)
@@ -105,6 +347,34 @@ class SimplifyHLFIRIntrinsics
           return mlir::cast<hlfir::ExprType>(transpose.getType())
               .isPolymorphic();
         });
+    // Handle only SUM(DIM=CONSTANT) case for now.
+    // It may be beneficial to expand the non-DIM case as well.
+    // E.g. when the input array is an elemental array expression,
+    // expanding the SUM into a total reduction loop nest
+    // would avoid creating a temporary for the elemental array expression.
+    target.addDynamicallyLegalOp<hlfir::SumOp>([](hlfir::SumOp sum) {
+      if (!simplifySum)
+        return true;
+      if (mlir::Value dim = sum.getDim()) {
+        if (auto dimVal = fir::getIntIfConstant(dim)) {
+          if (!fir::isa_trivial(sum.getType())) {
+            // Ignore the case SUM(a, DIM=X), where 'a' is a 1D array.
+            // It is only legal when X is 1, and it should probably be
+            // canonicalized into SUM(a).
+            fir::SequenceType arrayTy = mlir::cast<fir::SequenceType>(
+                hlfir::getFortranElementOrSequenceType(
+                    sum.getArray().getType()));
+            if (*dimVal > 0 && *dimVal <= arrayTy.getDimension()) {
+              // Ignore SUMs with illegal DIM values.
+              // They may appear in dead code,
+              // and they do not have to be converted.
+              return false;
+            }
+          }
+        }
+      }
+      return true;
+    });
     target.markUnknownOpDynamicallyLegal(
         [](mlir::Operation *) { return true; });
     if (mlir::failed(mlir::applyFullConversion(getOperation(), target,
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index b1e0dbf6e707e..51ecbe1a664f9 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -2,6 +2,7 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_flang_library(FlangOpenMPTransforms
   FunctionFiltering.cpp
+  GenericLoopConversion.cpp
   MapsForPrivatizedSymbols.cpp
   MapInfoFinalization.cpp
   MarkDeclareTarget.cpp
@@ -25,4 +26,5 @@ add_flang_library(FlangOpenMPTransforms
   HLFIRDialect
   MLIRIR
   MLIRPass
+  MLIRTransformUtils
 )
diff --git a/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
new file mode 100644
index 0000000000000..c3c1f3b2848b8
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/GenericLoopConversion.cpp
@@ -0,0 +1,181 @@
+//===- GenericLoopConversion.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Common/OpenMP-utils.h"
+
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/IRMapping.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+#include <memory>
+
+namespace flangomp {
+#define GEN_PASS_DEF_GENERICLOOPCONVERSIONPASS
+#include "flang/Optimizer/OpenMP/Passes.h.inc"
+} // namespace flangomp
+
+namespace {
+
+/// A conversion pattern to handle various combined forms of `omp.loop`. For how
+/// combined/composite directive are handled see:
+/// https://discourse.llvm.org/t/rfc-representing-combined-composite-constructs-in-the-openmp-dialect/76986.
+class GenericLoopConversionPattern
+    : public mlir::OpConversionPattern<mlir::omp::LoopOp> {
+public:
+  enum class GenericLoopCombinedInfo {
+    None,
+    TargetTeamsLoop,
+    TargetParallelLoop
+  };
+
+  using mlir::OpConversionPattern<mlir::omp::LoopOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::omp::LoopOp loopOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    assert(mlir::succeeded(checkLoopConversionSupportStatus(loopOp)));
+
+    rewriteToDistributeParallelDo(loopOp, rewriter);
+    rewriter.eraseOp(loopOp);
+    return mlir::success();
+  }
+
+  static mlir::LogicalResult
+  checkLoopConversionSupportStatus(mlir::omp::LoopOp loopOp) {
+    GenericLoopCombinedInfo combinedInfo = findGenericLoopCombineInfo(loopOp);
+
+    switch (combinedInfo) {
+    case GenericLoopCombinedInfo::None:
+      return loopOp.emitError(
+          "not yet implemented: Standalone `omp loop` directive");
+    case GenericLoopCombinedInfo::TargetParallelLoop:
+      return loopOp.emitError(
+          "not yet implemented: Combined `omp target parallel loop` directive");
+    case GenericLoopCombinedInfo::TargetTeamsLoop:
+      break;
+    }
+
+    auto todo = [&loopOp](mlir::StringRef clauseName) {
+      return loopOp.emitError()
+             << "not yet implemented: Unhandled clause " << clauseName << " in "
+             << loopOp->getName() << " operation";
+    };
+
+    if (loopOp.getBindKind())
+      return todo("bind");
+
+    if (loopOp.getOrder())
+      return todo("order");
+
+    if (!loopOp.getReductionVars().empty())
+      return todo("reduction");
+
+    // TODO For `target teams loop`, check similar constrains to what is checked
+    // by `TeamsLoopChecker` in SemaOpenMP.cpp.
+    return mlir::success();
+  }
+
+private:
+  static GenericLoopCombinedInfo
+  findGenericLoopCombineInfo(mlir::omp::LoopOp loopOp) {
+    mlir::Operation *parentOp = loopOp->getParentOp();
+    GenericLoopCombinedInfo result = GenericLoopCombinedInfo::None;
+
+    if (auto teamsOp = mlir::dyn_cast_if_present<mlir::omp::TeamsOp>(parentOp))
+      if (mlir::isa_and_present<mlir::omp::TargetOp>(teamsOp->getParentOp()))
+        result = GenericLoopCombinedInfo::TargetTeamsLoop;
+
+    if (auto parallelOp =
+            mlir::dyn_cast_if_present<mlir::omp::ParallelOp>(parentOp))
+      if (mlir::isa_and_present<mlir::omp::TargetOp>(parallelOp->getParentOp()))
+        result = GenericLoopCombinedInfo::TargetParallelLoop;
+
+    return result;
+  }
+
+  void rewriteToDistributeParallelDo(
+      mlir::omp::LoopOp loopOp,
+      mlir::ConversionPatternRewriter &rewriter) const {
+    mlir::omp::ParallelOperands parallelClauseOps;
+    parallelClauseOps.privateVars = loopOp.getPrivateVars();
+
+    auto privateSyms = loopOp.getPrivateSyms();
+    if (privateSyms)
+      parallelClauseOps.privateSyms.assign(privateSyms->begin(),
+                                           privateSyms->end());
+
+    Fortran::common::openmp::EntryBlockArgs parallelArgs;
+    parallelArgs.priv.vars = parallelClauseOps.privateVars;
+
+    auto parallelOp = rewriter.create<mlir::omp::ParallelOp>(loopOp.getLoc(),
+                                                             parallelClauseOps);
+    mlir::Block *parallelBlock =
+        genEntryBlock(rewriter, parallelArgs, parallelOp.getRegion());
+    parallelOp.setComposite(true);
+    rewriter.setInsertionPoint(
+        rewriter.create<mlir::omp::TerminatorOp>(loopOp.getLoc()));
+
+    mlir::omp::DistributeOperands distributeClauseOps;
+    auto distributeOp = rewriter.create<mlir::omp::DistributeOp>(
+        loopOp.getLoc(), distributeClauseOps);
+    distributeOp.setComposite(true);
+    rewriter.createBlock(&distributeOp.getRegion());
+
+    mlir::omp::WsloopOperands wsloopClauseOps;
+    auto wsloopOp =
+        rewriter.create<mlir::omp::WsloopOp>(loopOp.getLoc(), wsloopClauseOps);
+    wsloopOp.setComposite(true);
+    rewriter.createBlock(&wsloopOp.getRegion());
+
+    mlir::IRMapping mapper;
+    mlir::Block &loopBlock = *loopOp.getRegion().begin();
+
+    for (auto [loopOpArg, parallelOpArg] : llvm::zip_equal(
+             loopBlock.getArguments(), parallelBlock->getArguments()))
+      mapper.map(loopOpArg, parallelOpArg);
+
+    rewriter.clone(*loopOp.begin(), mapper);
+  }
+};
+
+class GenericLoopConversionPass
+    : public flangomp::impl::GenericLoopConversionPassBase<
+          GenericLoopConversionPass> {
+public:
+  GenericLoopConversionPass() = default;
+
+  void runOnOperation() override {
+    mlir::func::FuncOp func = getOperation();
+
+    if (func.isDeclaration())
+      return;
+
+    mlir::MLIRContext *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    patterns.insert<GenericLoopConversionPattern>(context);
+    mlir::ConversionTarget target(*context);
+
+    target.markUnknownOpDynamicallyLegal(
+        [](mlir::Operation *) { return true; });
+    target.addDynamicallyLegalOp<mlir::omp::LoopOp>(
+        [](mlir::omp::LoopOp loopOp) {
+          return mlir::failed(
+              GenericLoopConversionPattern::checkLoopConversionSupportStatus(
+                  loopOp));
+        });
+
+    if (mlir::failed(mlir::applyFullConversion(getOperation(), target,
+                                               std::move(patterns)))) {
+      mlir::emitError(func.getLoc(), "error in converting `omp.loop` op");
+      signalPassFailure();
+    }
+  }
+};
+} // namespace
diff --git a/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp b/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
index 6e130a96eb8dd..27a57f73d4e97 100644
--- a/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
+++ b/flang/lib/Optimizer/OpenMP/LowerWorkshare.cpp
@@ -153,6 +153,7 @@ static mlir::func::FuncOp createCopyFunc(mlir::Location loc, mlir::Type varType,
 
   if (auto decl = module.lookupSymbol<mlir::func::FuncOp>(copyFuncName))
     return decl;
+
   // create function
   mlir::OpBuilder::InsertionGuard guard(builder);
   mlir::OpBuilder modBuilder(module.getBodyRegion());
@@ -161,6 +162,7 @@ static mlir::func::FuncOp createCopyFunc(mlir::Location loc, mlir::Type varType,
   mlir::func::FuncOp funcOp =
       modBuilder.create<mlir::func::FuncOp>(loc, copyFuncName, funcType);
   funcOp.setVisibility(mlir::SymbolTable::Visibility::Private);
+  fir::factory::setInternalLinkage(funcOp);
   builder.createBlock(&funcOp.getRegion(), funcOp.getRegion().end(), argsTy,
                       {loc, loc});
   builder.setInsertionPointToStart(&funcOp.getRegion().back());
diff --git a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
index 289e648eed854..d2c814cc958dd 100644
--- a/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
@@ -1,5 +1,4 @@
-//===- MapsForPrivatizedSymbols.cpp
-//-----------------------------------------===//
+//===- MapsForPrivatizedSymbols.cpp ---------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -28,6 +27,7 @@
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -124,6 +124,8 @@ class MapsForPrivatizedSymbolsPass
       if (targetOp.getPrivateVars().empty())
         return;
       OperandRange privVars = targetOp.getPrivateVars();
+      llvm::SmallVector<int64_t> privVarMapIdx;
+
       std::optional<ArrayAttr> privSyms = targetOp.getPrivateSyms();
       SmallVector<omp::MapInfoOp, 4> mapInfoOps;
       for (auto [privVar, privSym] : llvm::zip_equal(privVars, *privSyms)) {
@@ -133,17 +135,25 @@ class MapsForPrivatizedSymbolsPass
             SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
                 targetOp, privatizerName);
         if (!privatizerNeedsMap(privatizer)) {
+          privVarMapIdx.push_back(-1);
           continue;
         }
+
+        privVarMapIdx.push_back(targetOp.getMapVars().size() +
+                                mapInfoOps.size());
+
         builder.setInsertionPoint(targetOp);
         Location loc = targetOp.getLoc();
         omp::MapInfoOp mapInfoOp = createMapInfo(loc, privVar, builder);
         mapInfoOps.push_back(mapInfoOp);
+
         LLVM_DEBUG(llvm::dbgs() << "MapsForPrivatizedSymbolsPass created ->\n");
         LLVM_DEBUG(mapInfoOp.dump());
       }
       if (!mapInfoOps.empty()) {
         mapInfoOpsForTarget.insert({targetOp.getOperation(), mapInfoOps});
+        targetOp.setPrivateMapsAttr(
+            mlir::DenseI64ArrayAttr::get(targetOp.getContext(), privVarMapIdx));
       }
     });
     if (!mapInfoOpsForTarget.empty()) {
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 31af3531641dd..72803aa3793ce 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -13,17 +13,24 @@
 
 namespace fir {
 
-void addNestedPassToAllTopLevelOperations(mlir::PassManager &pm,
-                                          PassConstructor ctor) {
-  addNestedPassToOps<mlir::func::FuncOp, mlir::omp::DeclareReductionOp,
+template <typename F>
+void addNestedPassToAllTopLevelOperations(mlir::PassManager &pm, F ctor) {
+  addNestedPassToOps<F, mlir::func::FuncOp, mlir::omp::DeclareReductionOp,
                      mlir::omp::PrivateClauseOp, fir::GlobalOp>(pm, ctor);
 }
 
+template <typename F>
+void addPassToGPUModuleOperations(mlir::PassManager &pm, F ctor) {
+  mlir::OpPassManager &nestPM = pm.nest<mlir::gpu::GPUModuleOp>();
+  nestPM.addNestedPass<mlir::func::FuncOp>(ctor());
+  nestPM.addNestedPass<mlir::gpu::GPUFuncOp>(ctor());
+}
+
+template <typename F>
 void addNestedPassToAllTopLevelOperationsConditionally(
-    mlir::PassManager &pm, llvm::cl::opt<bool> &disabled,
-    PassConstructor ctor) {
+    mlir::PassManager &pm, llvm::cl::opt<bool> &disabled, F ctor) {
   if (!disabled)
-    addNestedPassToAllTopLevelOperations(pm, ctor);
+    addNestedPassToAllTopLevelOperations<F>(pm, ctor);
 }
 
 void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
@@ -34,12 +41,11 @@ void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
 
 void addCfgConversionPass(mlir::PassManager &pm,
                           const MLIRToLLVMPassPipelineConfig &config) {
-  if (config.NSWOnLoopVarInc)
-    addNestedPassToAllTopLevelOperationsConditionally(
-        pm, disableCfgConversion, fir::createCFGConversionPassWithNSW);
-  else
-    addNestedPassToAllTopLevelOperationsConditionally(pm, disableCfgConversion,
-                                                      fir::createCFGConversion);
+  fir::CFGConversionOptions options;
+  if (!config.NSWOnLoopVarInc)
+    options.setNSW = false;
+  addNestedPassToAllTopLevelOperationsConditionally(
+      pm, disableCfgConversion, [&]() { return createCFGConversion(options); });
 }
 
 void addAVC(mlir::PassManager &pm, const llvm::OptimizationLevel &optLevel) {
@@ -160,7 +166,8 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
   pm.addPass(mlir::createCSEPass());
   fir::addAVC(pm, pc.OptLevel);
-  addNestedPassToAllTopLevelOperations(pm, fir::createCharacterConversion);
+  addNestedPassToAllTopLevelOperations<PassConstructor>(
+      pm, fir::createCharacterConversion);
   pm.addPass(mlir::createCanonicalizerPass(config));
   pm.addPass(fir::createSimplifyRegionLite());
   if (pc.OptLevel.isOptimizingForSpeed()) {
@@ -194,7 +201,8 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
   if (pc.AliasAnalysis && !disableFirAliasTags && !useOldAliasTags)
     pm.addPass(fir::createAddAliasTags());
 
-  addNestedPassToAllTopLevelOperations(pm, fir::createStackReclaim);
+  addNestedPassToAllTopLevelOperations<PassConstructor>(
+      pm, fir::createStackReclaim);
   // convert control flow to CFG form
   fir::addCfgConversionPass(pm, pc);
   pm.addPass(mlir::createConvertSCFToCFPass());
@@ -216,15 +224,16 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
                                   llvm::OptimizationLevel optLevel) {
   if (optLevel.isOptimizingForSpeed()) {
     addCanonicalizerPassWithoutRegionSimplification(pm);
-    addNestedPassToAllTopLevelOperations(pm,
-                                         hlfir::createSimplifyHLFIRIntrinsics);
+    addNestedPassToAllTopLevelOperations<PassConstructor>(
+        pm, hlfir::createSimplifyHLFIRIntrinsics);
   }
-  addNestedPassToAllTopLevelOperations(pm, hlfir::createInlineElementals);
+  addNestedPassToAllTopLevelOperations<PassConstructor>(
+      pm, hlfir::createInlineElementals);
   if (optLevel.isOptimizingForSpeed()) {
     addCanonicalizerPassWithoutRegionSimplification(pm);
     pm.addPass(mlir::createCSEPass());
-    addNestedPassToAllTopLevelOperations(pm,
-                                         hlfir::createOptimizedBufferization);
+    addNestedPassToAllTopLevelOperations<PassConstructor>(
+        pm, hlfir::createOptimizedBufferization);
   }
   pm.addPass(hlfir::createLowerHLFIROrderedAssignments());
   pm.addPass(hlfir::createLowerHLFIRIntrinsics());
@@ -247,6 +256,7 @@ void createOpenMPFIRPassPipeline(mlir::PassManager &pm, bool isTargetDevice) {
   pm.addPass(flangomp::createMapInfoFinalizationPass());
   pm.addPass(flangomp::createMapsForPrivatizedSymbolsPass());
   pm.addPass(flangomp::createMarkDeclareTargetPass());
+  pm.addPass(flangomp::createGenericLoopConversionPass());
   if (isTargetDevice)
     pm.addPass(flangomp::createFunctionFilteringPass());
 }
@@ -263,7 +273,10 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
                                          MLIRToLLVMPassPipelineConfig config,
                                          llvm::StringRef inputFilename) {
   fir::addBoxedProcedurePass(pm);
-  addNestedPassToAllTopLevelOperations(pm, fir::createAbstractResultOpt);
+  addNestedPassToAllTopLevelOperations<PassConstructor>(
+      pm, fir::createAbstractResultOpt);
+  addPassToGPUModuleOperations<PassConstructor>(pm,
+                                                fir::createAbstractResultOpt);
   fir::addCodeGenRewritePass(
       pm, (config.DebugInfo != llvm::codegenoptions::NoDebugInfo));
   fir::addExternalNameConversionPass(pm, config.Underscoring);
@@ -277,22 +290,17 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
   // Add function attributes
   mlir::LLVM::framePointerKind::FramePointerKind framePointerKind;
 
-  if (config.FramePointerKind != llvm::FramePointerKind::None ||
-      config.NoInfsFPMath || config.NoNaNsFPMath || config.ApproxFuncFPMath ||
-      config.NoSignedZerosFPMath || config.UnsafeFPMath) {
-    if (config.FramePointerKind == llvm::FramePointerKind::NonLeaf)
-      framePointerKind =
-          mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
-    else if (config.FramePointerKind == llvm::FramePointerKind::All)
-      framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
-    else
-      framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
-
-    pm.addPass(fir::createFunctionAttr(
-        {framePointerKind, config.NoInfsFPMath, config.NoNaNsFPMath,
-         config.ApproxFuncFPMath, config.NoSignedZerosFPMath,
-         config.UnsafeFPMath}));
-  }
+  if (config.FramePointerKind == llvm::FramePointerKind::NonLeaf)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
+  else if (config.FramePointerKind == llvm::FramePointerKind::All)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
+  else
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
+
+  pm.addPass(fir::createFunctionAttr(
+      {framePointerKind, config.NoInfsFPMath, config.NoNaNsFPMath,
+       config.ApproxFuncFPMath, config.NoSignedZerosFPMath,
+       config.UnsafeFPMath}));
 
   fir::addFIRToLLVMPass(pm, config);
 }
diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index e64280508755a..b0327cc10e9de 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -14,6 +14,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -233,6 +234,60 @@ class SaveResultOpConversion
   }
 };
 
+template <typename OpTy>
+static mlir::LogicalResult
+processReturnLikeOp(OpTy ret, mlir::Value newArg,
+                    mlir::PatternRewriter &rewriter) {
+  auto loc = ret.getLoc();
+  rewriter.setInsertionPoint(ret);
+  mlir::Value resultValue = ret.getOperand(0);
+  fir::LoadOp resultLoad;
+  mlir::Value resultStorage;
+  // Identify result local storage.
+  if (auto load = resultValue.getDefiningOp<fir::LoadOp>()) {
+    resultLoad = load;
+    resultStorage = load.getMemref();
+    // The result alloca may be behind a fir.declare, if any.
+    if (auto declare = resultStorage.getDefiningOp<fir::DeclareOp>())
+      resultStorage = declare.getMemref();
+  }
+  // Replace old local storage with new storage argument, unless
+  // the derived type is C_PTR/C_FUN_PTR, in which case the return
+  // type is updated to return void* (no new argument is passed).
+  if (fir::isa_builtin_cptr_type(resultValue.getType())) {
+    auto module = ret->template getParentOfType<mlir::ModuleOp>();
+    FirOpBuilder builder(rewriter, module);
+    mlir::Value cptr = resultValue;
+    if (resultLoad) {
+      // Replace whole derived type load by component load.
+      cptr = resultLoad.getMemref();
+      rewriter.setInsertionPoint(resultLoad);
+    }
+    mlir::Value newResultValue =
+        fir::factory::genCPtrOrCFunptrValue(builder, loc, cptr);
+    newResultValue = builder.createConvert(
+        loc, getVoidPtrType(ret.getContext()), newResultValue);
+    rewriter.setInsertionPoint(ret);
+    rewriter.replaceOpWithNewOp<OpTy>(ret, mlir::ValueRange{newResultValue});
+  } else if (resultStorage) {
+    resultStorage.replaceAllUsesWith(newArg);
+    rewriter.replaceOpWithNewOp<OpTy>(ret);
+  } else {
+    // The result storage may have been optimized out by a memory to
+    // register pass, this is possible for fir.box results, or fir.record
+    // with no length parameters. Simply store the result in the result
+    // storage. at the return point.
+    rewriter.create<fir::StoreOp>(loc, resultValue, newArg);
+    rewriter.replaceOpWithNewOp<OpTy>(ret);
+  }
+  // Delete result old local storage if unused.
+  if (resultStorage)
+    if (auto alloc = resultStorage.getDefiningOp<fir::AllocaOp>())
+      if (alloc->use_empty())
+        rewriter.eraseOp(alloc);
+  return mlir::success();
+}
+
 class ReturnOpConversion : public mlir::OpRewritePattern<mlir::func::ReturnOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
@@ -241,55 +296,23 @@ class ReturnOpConversion : public mlir::OpRewritePattern<mlir::func::ReturnOp> {
   llvm::LogicalResult
   matchAndRewrite(mlir::func::ReturnOp ret,
                   mlir::PatternRewriter &rewriter) const override {
-    auto loc = ret.getLoc();
-    rewriter.setInsertionPoint(ret);
-    mlir::Value resultValue = ret.getOperand(0);
-    fir::LoadOp resultLoad;
-    mlir::Value resultStorage;
-    // Identify result local storage.
-    if (auto load = resultValue.getDefiningOp<fir::LoadOp>()) {
-      resultLoad = load;
-      resultStorage = load.getMemref();
-      // The result alloca may be behind a fir.declare, if any.
-      if (auto declare = resultStorage.getDefiningOp<fir::DeclareOp>())
-        resultStorage = declare.getMemref();
-    }
-    // Replace old local storage with new storage argument, unless
-    // the derived type is C_PTR/C_FUN_PTR, in which case the return
-    // type is updated to return void* (no new argument is passed).
-    if (fir::isa_builtin_cptr_type(resultValue.getType())) {
-      auto module = ret->getParentOfType<mlir::ModuleOp>();
-      FirOpBuilder builder(rewriter, module);
-      mlir::Value cptr = resultValue;
-      if (resultLoad) {
-        // Replace whole derived type load by component load.
-        cptr = resultLoad.getMemref();
-        rewriter.setInsertionPoint(resultLoad);
-      }
-      mlir::Value newResultValue =
-          fir::factory::genCPtrOrCFunptrValue(builder, loc, cptr);
-      newResultValue = builder.createConvert(
-          loc, getVoidPtrType(ret.getContext()), newResultValue);
-      rewriter.setInsertionPoint(ret);
-      rewriter.replaceOpWithNewOp<mlir::func::ReturnOp>(
-          ret, mlir::ValueRange{newResultValue});
-    } else if (resultStorage) {
-      resultStorage.replaceAllUsesWith(newArg);
-      rewriter.replaceOpWithNewOp<mlir::func::ReturnOp>(ret);
-    } else {
-      // The result storage may have been optimized out by a memory to
-      // register pass, this is possible for fir.box results, or fir.record
-      // with no length parameters. Simply store the result in the result
-      // storage. at the return point.
-      rewriter.create<fir::StoreOp>(loc, resultValue, newArg);
-      rewriter.replaceOpWithNewOp<mlir::func::ReturnOp>(ret);
-    }
-    // Delete result old local storage if unused.
-    if (resultStorage)
-      if (auto alloc = resultStorage.getDefiningOp<fir::AllocaOp>())
-        if (alloc->use_empty())
-          rewriter.eraseOp(alloc);
-    return mlir::success();
+    return processReturnLikeOp(ret, newArg, rewriter);
+  }
+
+private:
+  mlir::Value newArg;
+};
+
+class GPUReturnOpConversion
+    : public mlir::OpRewritePattern<mlir::gpu::ReturnOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  GPUReturnOpConversion(mlir::MLIRContext *context, mlir::Value newArg)
+      : OpRewritePattern(context), newArg{newArg} {}
+  llvm::LogicalResult
+  matchAndRewrite(mlir::gpu::ReturnOp ret,
+                  mlir::PatternRewriter &rewriter) const override {
+    return processReturnLikeOp(ret, newArg, rewriter);
   }
 
 private:
@@ -331,9 +354,10 @@ class AbstractResultOpt
   using fir::impl::AbstractResultOptBase<
       AbstractResultOpt>::AbstractResultOptBase;
 
-  void runOnSpecificOperation(mlir::func::FuncOp func, bool shouldBoxResult,
-                              mlir::RewritePatternSet &patterns,
-                              mlir::ConversionTarget &target) {
+  template <typename OpTy>
+  void runOnFunctionLikeOperation(OpTy func, bool shouldBoxResult,
+                                  mlir::RewritePatternSet &patterns,
+                                  mlir::ConversionTarget &target) {
     auto loc = func.getLoc();
     auto *context = &getContext();
     // Convert function type itself if it has an abstract result.
@@ -371,6 +395,9 @@ class AbstractResultOpt
         patterns.insert<ReturnOpConversion>(context, newArg);
         target.addDynamicallyLegalOp<mlir::func::ReturnOp>(
             [](mlir::func::ReturnOp ret) { return ret.getOperands().empty(); });
+        patterns.insert<GPUReturnOpConversion>(context, newArg);
+        target.addDynamicallyLegalOp<mlir::gpu::ReturnOp>(
+            [](mlir::gpu::ReturnOp ret) { return ret.getOperands().empty(); });
         assert(func.getFunctionType() ==
                getNewFunctionType(funcTy, shouldBoxResult));
       } else {
@@ -384,6 +411,18 @@ class AbstractResultOpt
     }
   }
 
+  void runOnSpecificOperation(mlir::func::FuncOp func, bool shouldBoxResult,
+                              mlir::RewritePatternSet &patterns,
+                              mlir::ConversionTarget &target) {
+    runOnFunctionLikeOperation(func, shouldBoxResult, patterns, target);
+  }
+
+  void runOnSpecificOperation(mlir::gpu::GPUFuncOp func, bool shouldBoxResult,
+                              mlir::RewritePatternSet &patterns,
+                              mlir::ConversionTarget &target) {
+    runOnFunctionLikeOperation(func, shouldBoxResult, patterns, target);
+  }
+
   inline static bool containsFunctionTypeWithAbstractResult(mlir::Type type) {
     return mlir::TypeSwitch<mlir::Type, bool>(type)
         .Case([](fir::BoxProcType boxProc) {
@@ -446,9 +485,10 @@ class AbstractResultOpt
     const bool shouldBoxResult = this->passResultAsBox.getValue();
 
     mlir::TypeSwitch<mlir::Operation *, void>(op)
-        .Case<mlir::func::FuncOp, fir::GlobalOp>([&](auto op) {
-          runOnSpecificOperation(op, shouldBoxResult, patterns, target);
-        });
+        .Case<mlir::func::FuncOp, fir::GlobalOp, mlir::gpu::GPUFuncOp>(
+            [&](auto op) {
+              runOnSpecificOperation(op, shouldBoxResult, patterns, target);
+            });
 
     // Convert the calls and, if needed,  the ReturnOp in the function body.
     target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect,
diff --git a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
index dd204126be5db..9591f48c5d417 100644
--- a/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp
@@ -145,8 +145,6 @@ struct CUFAddConstructor
         default:
           break;
         }
-        if (!func)
-          continue;
       }
     }
     builder.create<mlir::LLVM::ReturnOp>(loc, mlir::ValueRange{});
diff --git a/flang/lib/Optimizer/Transforms/CUFCommon.cpp b/flang/lib/Optimizer/Transforms/CUFCommon.cpp
index 5b7631bbacb5f..bbe33217e8f45 100644
--- a/flang/lib/Optimizer/Transforms/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFCommon.cpp
@@ -43,3 +43,14 @@ bool cuf::isInCUDADeviceContext(mlir::Operation *op) {
   }
   return false;
 }
+
+bool cuf::isRegisteredDeviceGlobal(fir::GlobalOp op) {
+  if (op.getConstant())
+    return false;
+  auto attr = op.getDataAttr();
+  if (attr && (*attr == cuf::DataAttribute::Device ||
+               *attr == cuf::DataAttribute::Managed ||
+               *attr == cuf::DataAttribute::Constant))
+    return true;
+  return false;
+}
diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index 714b0b291be1e..72b894d1cec75 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/DenseSet.h"
 
 namespace fir {
 #define GEN_PASS_DEF_CUFDEVICEGLOBAL
@@ -27,36 +28,30 @@ namespace fir {
 namespace {
 
 static void processAddrOfOp(fir::AddrOfOp addrOfOp,
-                            mlir::SymbolTable &symbolTable, bool onlyConstant) {
+                            mlir::SymbolTable &symbolTable,
+                            llvm::DenseSet<fir::GlobalOp> &candidates) {
   if (auto globalOp = symbolTable.lookup<fir::GlobalOp>(
           addrOfOp.getSymbol().getRootReference().getValue())) {
-    bool isCandidate{(onlyConstant ? globalOp.getConstant() : true) &&
-                     !globalOp.getDataAttr()};
-    if (isCandidate)
-      globalOp.setDataAttrAttr(cuf::DataAttributeAttr::get(
-          addrOfOp.getContext(), globalOp.getConstant()
-                                     ? cuf::DataAttribute::Constant
-                                     : cuf::DataAttribute::Device));
+    // TO DO: limit candidates to non-scalars. Scalars appear to have been
+    // folded in already.
+    if (globalOp.getConstant()) {
+      candidates.insert(globalOp);
+    }
   }
 }
 
-static void prepareImplicitDeviceGlobals(mlir::func::FuncOp funcOp,
-                                         mlir::SymbolTable &symbolTable,
-                                         bool onlyConstant = true) {
+static void
+prepareImplicitDeviceGlobals(mlir::func::FuncOp funcOp,
+                             mlir::SymbolTable &symbolTable,
+                             llvm::DenseSet<fir::GlobalOp> &candidates) {
+
   auto cudaProcAttr{
       funcOp->getAttrOfType<cuf::ProcAttributeAttr>(cuf::getProcAttrName())};
-  if (!cudaProcAttr || cudaProcAttr.getValue() == cuf::ProcAttribute::Host) {
-    // Look for globlas in CUF KERNEL DO operations.
-    for (auto cufKernelOp : funcOp.getBody().getOps<cuf::KernelOp>()) {
-      cufKernelOp.walk([&](fir::AddrOfOp addrOfOp) {
-        processAddrOfOp(addrOfOp, symbolTable, onlyConstant);
-      });
-    }
-    return;
+  if (cudaProcAttr && cudaProcAttr.getValue() != cuf::ProcAttribute::Host) {
+    funcOp.walk([&](fir::AddrOfOp addrOfOp) {
+      processAddrOfOp(addrOfOp, symbolTable, candidates);
+    });
   }
-  funcOp.walk([&](fir::AddrOfOp addrOfOp) {
-    processAddrOfOp(addrOfOp, symbolTable, onlyConstant);
-  });
 }
 
 class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
@@ -67,11 +62,17 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
     if (!mod)
       return signalPassFailure();
 
+    llvm::DenseSet<fir::GlobalOp> candidates;
     mlir::SymbolTable symTable(mod);
     mod.walk([&](mlir::func::FuncOp funcOp) {
-      prepareImplicitDeviceGlobals(funcOp, symTable);
+      prepareImplicitDeviceGlobals(funcOp, symTable, candidates);
       return mlir::WalkResult::advance();
     });
+    mod.walk([&](cuf::KernelOp kernelOp) {
+      kernelOp.walk([&](fir::AddrOfOp addrOfOp) {
+        processAddrOfOp(addrOfOp, symTable, candidates);
+      });
+    });
 
     // Copying the device global variable into the gpu module
     mlir::SymbolTable parentSymTable(mod);
@@ -80,22 +81,15 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
       return signalPassFailure();
     mlir::SymbolTable gpuSymTable(gpuMod);
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
-      auto attr = globalOp.getDataAttrAttr();
-      if (!attr)
-        continue;
-      switch (attr.getValue()) {
-      case cuf::DataAttribute::Device:
-      case cuf::DataAttribute::Constant:
-      case cuf::DataAttribute::Managed: {
-        auto globalName{globalOp.getSymbol().getValue()};
-        if (gpuSymTable.lookup<fir::GlobalOp>(globalName)) {
-          break;
-        }
-        gpuSymTable.insert(globalOp->clone());
-      } break;
-      default:
+      if (cuf::isRegisteredDeviceGlobal(globalOp))
+        candidates.insert(globalOp);
+    }
+    for (auto globalOp : candidates) {
+      auto globalName{globalOp.getSymbol().getValue()};
+      if (gpuSymTable.lookup<fir::GlobalOp>(globalName)) {
         break;
       }
+      gpuSymTable.insert(globalOp->clone());
     }
   }
 };
diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index c64f35542a6e5..60aa401e1cc8c 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -42,6 +42,8 @@ static mlir::Value createKernelArgArray(mlir::Location loc,
   auto structTy = mlir::LLVM::LLVMStructType::getLiteral(ctx, structTypes);
   auto ptrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
   mlir::Type i32Ty = rewriter.getI32Type();
+  auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
+      loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 0));
   auto one = rewriter.create<mlir::LLVM::ConstantOp>(
       loc, i32Ty, rewriter.getIntegerAttr(i32Ty, 1));
   mlir::Value argStruct =
@@ -55,10 +57,11 @@ static mlir::Value createKernelArgArray(mlir::Location loc,
     auto indice = rewriter.create<mlir::LLVM::ConstantOp>(
         loc, i32Ty, rewriter.getIntegerAttr(i32Ty, i));
     mlir::Value structMember = rewriter.create<LLVM::GEPOp>(
-        loc, ptrTy, structTy, argStruct, mlir::ArrayRef<mlir::Value>({indice}));
+        loc, ptrTy, structTy, argStruct,
+        mlir::ArrayRef<mlir::Value>({zero, indice}));
     rewriter.create<LLVM::StoreOp>(loc, arg, structMember);
     mlir::Value arrayMember = rewriter.create<LLVM::GEPOp>(
-        loc, ptrTy, structTy, argArray, mlir::ArrayRef<mlir::Value>({indice}));
+        loc, ptrTy, ptrTy, argArray, mlir::ArrayRef<mlir::Value>({indice}));
     rewriter.create<LLVM::StoreOp>(loc, structMember, arrayMember);
   }
   return argArray;
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 337ea04755d1a..1df82e6accfed 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -89,62 +89,6 @@ static mlir::Value createConvertOp(mlir::PatternRewriter &rewriter,
   return val;
 }
 
-mlir::Value getDeviceAddress(mlir::PatternRewriter &rewriter,
-                             mlir::OpOperand &operand,
-                             const mlir::SymbolTable &symtab) {
-  mlir::Value v = operand.get();
-  auto declareOp = v.getDefiningOp<fir::DeclareOp>();
-  if (!declareOp)
-    return v;
-
-  auto addrOfOp = declareOp.getMemref().getDefiningOp<fir::AddrOfOp>();
-  if (!addrOfOp)
-    return v;
-
-  auto globalOp = symtab.lookup<fir::GlobalOp>(
-      addrOfOp.getSymbol().getRootReference().getValue());
-
-  if (!globalOp)
-    return v;
-
-  bool isDevGlobal{false};
-  auto attr = globalOp.getDataAttrAttr();
-  if (attr) {
-    switch (attr.getValue()) {
-    case cuf::DataAttribute::Device:
-    case cuf::DataAttribute::Managed:
-    case cuf::DataAttribute::Constant:
-      isDevGlobal = true;
-      break;
-    default:
-      break;
-    }
-  }
-  if (!isDevGlobal)
-    return v;
-  mlir::OpBuilder::InsertionGuard guard(rewriter);
-  rewriter.setInsertionPoint(operand.getOwner());
-  auto loc = declareOp.getLoc();
-  auto mod = declareOp->getParentOfType<mlir::ModuleOp>();
-  fir::FirOpBuilder builder(rewriter, mod);
-
-  mlir::func::FuncOp callee =
-      fir::runtime::getRuntimeFunc<mkRTKey(CUFGetDeviceAddress)>(loc, builder);
-  auto fTy = callee.getFunctionType();
-  auto toTy = fTy.getInput(0);
-  mlir::Value inputArg =
-      createConvertOp(rewriter, loc, toTy, declareOp.getResult());
-  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
-  mlir::Value sourceLine =
-      fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
-  llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
-      builder, loc, fTy, inputArg, sourceFile, sourceLine)};
-  auto call = rewriter.create<fir::CallOp>(loc, callee, args);
-  mlir::Value cast = createConvertOp(
-      rewriter, loc, declareOp.getMemref().getType(), call->getResult(0));
-  return cast;
-}
-
 template <typename OpTy>
 static mlir::LogicalResult convertOpToCall(OpTy op,
                                            mlir::PatternRewriter &rewriter,
@@ -422,6 +366,54 @@ struct CUFAllocOpConversion : public mlir::OpRewritePattern<cuf::AllocOp> {
   const fir::LLVMTypeConverter *typeConverter;
 };
 
+struct DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  DeclareOpConversion(mlir::MLIRContext *context,
+                      const mlir::SymbolTable &symtab)
+      : OpRewritePattern(context), symTab{symtab} {}
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::DeclareOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+    if (auto addrOfOp = op.getMemref().getDefiningOp<fir::AddrOfOp>()) {
+      if (auto global = symTab.lookup<fir::GlobalOp>(
+              addrOfOp.getSymbol().getRootReference().getValue())) {
+        if (cuf::isRegisteredDeviceGlobal(global)) {
+          rewriter.setInsertionPointAfter(addrOfOp);
+          auto mod = op->getParentOfType<mlir::ModuleOp>();
+          fir::FirOpBuilder builder(rewriter, mod);
+          mlir::Location loc = op.getLoc();
+          mlir::func::FuncOp callee =
+              fir::runtime::getRuntimeFunc<mkRTKey(CUFGetDeviceAddress)>(
+                  loc, builder);
+          auto fTy = callee.getFunctionType();
+          mlir::Type toTy = fTy.getInput(0);
+          mlir::Value inputArg =
+              createConvertOp(rewriter, loc, toTy, addrOfOp.getResult());
+          mlir::Value sourceFile =
+              fir::factory::locationToFilename(builder, loc);
+          mlir::Value sourceLine =
+              fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
+          llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+              builder, loc, fTy, inputArg, sourceFile, sourceLine)};
+          auto call = rewriter.create<fir::CallOp>(loc, callee, args);
+          mlir::Value cast = createConvertOp(
+              rewriter, loc, op.getMemref().getType(), call->getResult(0));
+          rewriter.startOpModification(op);
+          op.getMemrefMutable().assign(cast);
+          rewriter.finalizeOpModification(op);
+          return success();
+        }
+      }
+    }
+    return failure();
+  }
+
+private:
+  const mlir::SymbolTable &symTab;
+};
+
 struct CUFFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
   using OpRewritePattern::OpRewritePattern;
 
@@ -511,7 +503,7 @@ static mlir::Value emboxSrc(mlir::PatternRewriter &rewriter,
     builder.create<fir::StoreOp>(loc, src, alloc);
     addr = alloc;
   } else {
-    addr = getDeviceAddress(rewriter, op.getSrcMutable(), symtab);
+    addr = op.getSrc();
   }
   llvm::SmallVector<mlir::Value> lenParams;
   mlir::Type boxTy = fir::BoxType::get(srcTy);
@@ -531,7 +523,7 @@ static mlir::Value emboxDst(mlir::PatternRewriter &rewriter,
   mlir::Location loc = op.getLoc();
   fir::FirOpBuilder builder(rewriter, mod);
   mlir::Type dstTy = fir::unwrapRefType(op.getDst().getType());
-  mlir::Value dstAddr = getDeviceAddress(rewriter, op.getDstMutable(), symtab);
+  mlir::Value dstAddr = op.getDst();
   mlir::Type dstBoxTy = fir::BoxType::get(dstTy);
   llvm::SmallVector<mlir::Value> lenParams;
   mlir::Value dstBox =
@@ -652,8 +644,8 @@ struct CUFDataTransferOpConversion
       mlir::Value sourceLine =
           fir::factory::locationToLineNo(builder, loc, fTy.getInput(5));
 
-      mlir::Value dst = getDeviceAddress(rewriter, op.getDstMutable(), symtab);
-      mlir::Value src = getDeviceAddress(rewriter, op.getSrcMutable(), symtab);
+      mlir::Value dst = op.getDst();
+      mlir::Value src = op.getSrc();
       // Materialize the src if constant.
       if (matchPattern(src.getDefiningOp(), mlir::m_Constant())) {
         mlir::Value temp = builder.createTemporary(loc, srcTy);
@@ -823,6 +815,30 @@ class CUFOpConversion : public fir::impl::CUFOpConversionBase<CUFOpConversion> {
                       "error in CUF op conversion\n");
       signalPassFailure();
     }
+
+    target.addDynamicallyLegalOp<fir::DeclareOp>([&](fir::DeclareOp op) {
+      if (inDeviceContext(op))
+        return true;
+      if (auto addrOfOp = op.getMemref().getDefiningOp<fir::AddrOfOp>()) {
+        if (auto global = symtab.lookup<fir::GlobalOp>(
+                addrOfOp.getSymbol().getRootReference().getValue())) {
+          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(global.getType())))
+            return true;
+          if (cuf::isRegisteredDeviceGlobal(global))
+            return false;
+        }
+      }
+      return true;
+    });
+
+    patterns.clear();
+    cuf::populateFIRCUFConversionPatterns(symtab, patterns);
+    if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
+                                                  std::move(patterns)))) {
+      mlir::emitError(mlir::UnknownLoc::get(ctx),
+                      "error in CUF op conversion\n");
+      signalPassFailure();
+    }
   }
 };
 } // namespace
@@ -837,3 +853,8 @@ void cuf::populateCUFToFIRConversionPatterns(
                                                &dl, &converter);
   patterns.insert<CUFLaunchOpConversion>(patterns.getContext(), symtab);
 }
+
+void cuf::populateFIRCUFConversionPatterns(const mlir::SymbolTable &symtab,
+                                           mlir::RewritePatternSet &patterns) {
+  patterns.insert<DeclareOpConversion>(patterns.getContext(), symtab);
+}
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index 3b79d6d311b71..b09bbf6106dbb 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -332,8 +332,6 @@ class CfgConversion : public fir::impl::CFGConversionBase<CfgConversion> {
 public:
   using CFGConversionBase<CfgConversion>::CFGConversionBase;
 
-  CfgConversion(bool setNSW) { this->setNSW = setNSW; }
-
   void runOnOperation() override {
     auto *context = &this->getContext();
     mlir::RewritePatternSet patterns(context);
@@ -365,7 +363,3 @@ void fir::populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
   patterns.insert<CfgLoopConv, CfgIfConv, CfgIterWhileConv>(
       patterns.getContext(), forceLoopToExecuteOnce, setNSW);
 }
-
-std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithNSW() {
-  return std::make_unique<CfgConversion>(true);
-}
diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
index cfd90ff723793..4f6974ee52695 100644
--- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp
@@ -60,23 +60,30 @@ void ExternalNameConversionPass::runOnOperation() {
 
   llvm::DenseMap<mlir::StringAttr, mlir::FlatSymbolRefAttr> remappings;
 
+  auto processFctOrGlobal = [&](mlir::Operation &funcOrGlobal) {
+    auto symName = funcOrGlobal.getAttrOfType<mlir::StringAttr>(
+        mlir::SymbolTable::getSymbolAttrName());
+    auto deconstructedName = fir::NameUniquer::deconstruct(symName);
+    if (fir::NameUniquer::isExternalFacingUniquedName(deconstructedName)) {
+      auto newName = mangleExternalName(deconstructedName, appendUnderscoreOpt);
+      auto newAttr = mlir::StringAttr::get(context, newName);
+      mlir::SymbolTable::setSymbolName(&funcOrGlobal, newAttr);
+      auto newSymRef = mlir::FlatSymbolRefAttr::get(newAttr);
+      remappings.try_emplace(symName, newSymRef);
+      if (llvm::isa<mlir::func::FuncOp>(funcOrGlobal))
+        funcOrGlobal.setAttr(fir::getInternalFuncNameAttrName(), symName);
+    }
+  };
+
   auto renameFuncOrGlobalInModule = [&](mlir::Operation *module) {
-    for (auto &funcOrGlobal : module->getRegion(0).front()) {
-      if (llvm::isa<mlir::func::FuncOp>(funcOrGlobal) ||
-          llvm::isa<fir::GlobalOp>(funcOrGlobal)) {
-        auto symName = funcOrGlobal.getAttrOfType<mlir::StringAttr>(
-            mlir::SymbolTable::getSymbolAttrName());
-        auto deconstructedName = fir::NameUniquer::deconstruct(symName);
-        if (fir::NameUniquer::isExternalFacingUniquedName(deconstructedName)) {
-          auto newName =
-              mangleExternalName(deconstructedName, appendUnderscoreOpt);
-          auto newAttr = mlir::StringAttr::get(context, newName);
-          mlir::SymbolTable::setSymbolName(&funcOrGlobal, newAttr);
-          auto newSymRef = mlir::FlatSymbolRefAttr::get(newAttr);
-          remappings.try_emplace(symName, newSymRef);
-          if (llvm::isa<mlir::func::FuncOp>(funcOrGlobal))
-            funcOrGlobal.setAttr(fir::getInternalFuncNameAttrName(), symName);
-        }
+    for (auto &op : module->getRegion(0).front()) {
+      if (mlir::isa<mlir::func::FuncOp, fir::GlobalOp>(op)) {
+        processFctOrGlobal(op);
+      } else if (auto gpuMod = mlir::dyn_cast<mlir::gpu::GPUModuleOp>(op)) {
+        for (auto &gpuOp : gpuMod.getBodyRegion().front())
+          if (mlir::isa<mlir::func::FuncOp, fir::GlobalOp,
+                        mlir::gpu::GPUFuncOp>(gpuOp))
+            processFctOrGlobal(gpuOp);
       }
     }
   };
@@ -85,11 +92,6 @@ void ExternalNameConversionPass::runOnOperation() {
   // globals.
   renameFuncOrGlobalInModule(op);
 
-  // Do the same in GPU modules.
-  if (auto mod = mlir::dyn_cast_or_null<mlir::ModuleOp>(*op))
-    for (auto gpuMod : mod.getOps<mlir::gpu::GPUModuleOp>())
-      renameFuncOrGlobalInModule(gpuMod);
-
   if (remappings.empty())
     return;
 
@@ -97,11 +99,18 @@ void ExternalNameConversionPass::runOnOperation() {
   op.walk([&remappings](mlir::Operation *nestedOp) {
     llvm::SmallVector<std::pair<mlir::StringAttr, mlir::SymbolRefAttr>> updates;
     for (const mlir::NamedAttribute &attr : nestedOp->getAttrDictionary())
-      if (auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(attr.getValue()))
-        if (auto remap = remappings.find(symRef.getRootReference());
-            remap != remappings.end())
+      if (auto symRef = llvm::dyn_cast<mlir::SymbolRefAttr>(attr.getValue())) {
+        if (auto remap = remappings.find(symRef.getLeafReference());
+            remap != remappings.end()) {
+          mlir::SymbolRefAttr symAttr = mlir::FlatSymbolRefAttr(remap->second);
+          if (mlir::isa<mlir::gpu::LaunchFuncOp>(nestedOp))
+            symAttr = mlir::SymbolRefAttr::get(
+                symRef.getRootReference(),
+                {mlir::FlatSymbolRefAttr(remap->second)});
           updates.emplace_back(std::pair<mlir::StringAttr, mlir::SymbolRefAttr>{
-              attr.getName(), mlir::SymbolRefAttr(remap->second)});
+              attr.getName(), symAttr});
+        }
+      }
     for (auto update : updates)
       nestedOp->setAttr(update.first, update.second);
   });
diff --git a/flang/lib/Optimizer/Transforms/FunctionAttr.cpp b/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
index 69c41595974ef..c79843fac4ce2 100644
--- a/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
+++ b/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
@@ -10,6 +10,8 @@
 /// \file
 /// This is a generic pass for adding attributes to functions.
 //===----------------------------------------------------------------------===//
+#include "flang/Optimizer/Dialect/FIROpsSupport.h"
+#include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -45,6 +47,24 @@ void FunctionAttrPass::runOnOperation() {
 
   LLVM_DEBUG(llvm::dbgs() << "Func-name:" << func.getSymName() << "\n");
 
+  llvm::StringRef name = func.getSymName();
+  auto deconstructed = fir::NameUniquer::deconstruct(name);
+  bool isFromModule = !deconstructed.second.modules.empty();
+
+  if ((isFromModule || !func.isDeclaration()) &&
+      !fir::hasBindcAttr(func.getOperation())) {
+    llvm::StringRef nocapture = mlir::LLVM::LLVMDialect::getNoCaptureAttrName();
+    mlir::UnitAttr unitAttr = mlir::UnitAttr::get(func.getContext());
+
+    for (auto [index, argType] : llvm::enumerate(func.getArgumentTypes())) {
+      if (mlir::isa<fir::ReferenceType>(argType) &&
+          !func.getArgAttr(index, fir::getTargetAttrName()) &&
+          !func.getArgAttr(index, fir::getAsynchronousAttrName()) &&
+          !func.getArgAttr(index, fir::getVolatileAttrName()))
+        func.setArgAttr(index, nocapture, unitAttr);
+    }
+  }
+
   mlir::MLIRContext *context = &getContext();
   if (framePointerKind != mlir::LLVM::framePointerKind::FramePointerKind::None)
     func->setAttr("frame_pointer", mlir::LLVM::FramePointerKindAttr::get(
diff --git a/flang/lib/Parser/CMakeLists.txt b/flang/lib/Parser/CMakeLists.txt
index 600a2f67df443..d364671d7a322 100644
--- a/flang/lib/Parser/CMakeLists.txt
+++ b/flang/lib/Parser/CMakeLists.txt
@@ -30,6 +30,7 @@ add_flang_library(FortranParser
   LINK_COMPONENTS
   Support
   FrontendOpenACC
+  FrontendOpenMP
 
   DEPENDS
   omp_gen
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index 0bdc4c4e033c7..a3d2c36310807 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -460,7 +460,7 @@ TYPE_PARSER(construct<ComponentAttrSpec>(accessSpec) ||
     construct<ComponentAttrSpec>(allocatable) ||
     construct<ComponentAttrSpec>("CODIMENSION" >> coarraySpec) ||
     construct<ComponentAttrSpec>(contiguous) ||
-    construct<ComponentAttrSpec>("DIMENSION" >> Parser<ComponentArraySpec>{}) ||
+    construct<ComponentAttrSpec>("DIMENSION" >> componentArraySpec) ||
     construct<ComponentAttrSpec>(pointer) ||
     extension<LanguageFeature::CUDA>(
         construct<ComponentAttrSpec>(Parser<common::CUDADataAttr>{})) ||
@@ -471,17 +471,23 @@ TYPE_PARSER(construct<ComponentAttrSpec>(accessSpec) ||
 
 // R739 component-decl ->
 //        component-name [( component-array-spec )]
-//        [lbracket coarray-spec rbracket] [* char-length]
-//        [component-initialization]
+//          [lbracket coarray-spec rbracket] [* char-length]
+//          [component-initialization] |
+// (ext.) component-name *char-length [(component-array-spec)]
+//          [lbracket coarray-spec rbracket] [* char-length]
+//          [component-initialization]
 TYPE_CONTEXT_PARSER("component declaration"_en_US,
-    construct<ComponentDecl>(name, maybe(Parser<ComponentArraySpec>{}),
-        maybe(coarraySpec), maybe("*" >> charLength), maybe(initialization)))
+    construct<ComponentDecl>(name, "*" >> charLength, maybe(componentArraySpec),
+        maybe(coarraySpec), maybe(initialization)) ||
+        construct<ComponentDecl>(name, maybe(componentArraySpec),
+            maybe(coarraySpec), maybe("*" >> charLength),
+            maybe(initialization)))
 // The source field of the Name will be replaced with a distinct generated name.
 TYPE_CONTEXT_PARSER("%FILL item"_en_US,
     extension<LanguageFeature::DECStructures>(
         "nonstandard usage: %FILL"_port_en_US,
         construct<FillDecl>(space >> sourced("%FILL" >> construct<Name>()),
-            maybe(Parser<ComponentArraySpec>{}), maybe("*" >> charLength))))
+            maybe(componentArraySpec), maybe("*" >> charLength))))
 TYPE_PARSER(construct<ComponentOrFill>(Parser<ComponentDecl>{}) ||
     construct<ComponentOrFill>(Parser<FillDecl>{}))
 
@@ -658,9 +664,13 @@ TYPE_PARSER(recovery("END ENUM"_tok, constructEndStmtErrorRecovery) >>
 
 // R801 type-declaration-stmt ->
 //        declaration-type-spec [[, attr-spec]... ::] entity-decl-list
-constexpr auto entityDeclWithoutEqInit{construct<EntityDecl>(name,
-    maybe(arraySpec), maybe(coarraySpec), maybe("*" >> charLength),
-    !"="_tok >> maybe(initialization))}; // old-style REAL A/0/ still works
+constexpr auto entityDeclWithoutEqInit{
+    construct<EntityDecl>(name, "*" >> charLength, maybe(arraySpec),
+        maybe(coarraySpec), !"="_tok >> maybe(initialization)) ||
+    construct<EntityDecl>(name, maybe(arraySpec), maybe(coarraySpec),
+        maybe("*" >> charLength),
+        !"="_tok >>
+            maybe(initialization) /* old-style REAL A/0/ still works */)};
 TYPE_PARSER(
     construct<TypeDeclarationStmt>(declarationTypeSpec,
         defaulted("," >> nonemptyList(Parser<AttrSpec>{})) / "::",
@@ -720,9 +730,13 @@ constexpr auto objectName{name};
 // R803 entity-decl ->
 //        object-name [( array-spec )] [lbracket coarray-spec rbracket]
 //          [* char-length] [initialization] |
-//        function-name [* char-length]
-TYPE_PARSER(construct<EntityDecl>(objectName, maybe(arraySpec),
-    maybe(coarraySpec), maybe("*" >> charLength), maybe(initialization)))
+//        function-name [* char-length] |
+// (ext.) object-name *char-length [(array-spec)]
+//          [lbracket coarray-spec rbracket] [initialization]
+TYPE_PARSER(construct<EntityDecl>(objectName, "*" >> charLength,
+                maybe(arraySpec), maybe(coarraySpec), maybe(initialization)) ||
+    construct<EntityDecl>(objectName, maybe(arraySpec), maybe(coarraySpec),
+        maybe("*" >> charLength), maybe(initialization)))
 
 // R806 null-init -> function-reference   ... which must resolve to NULL()
 TYPE_PARSER(lookAhead(name / "( )") >> construct<NullInit>(expr))
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 2040a3e7ed5ae..86d475c1a1542 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -16,6 +16,10 @@
 #include "token-parsers.h"
 #include "type-parser-implementation.h"
 #include "flang/Parser/parse-tree.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 // OpenMP Directives and Clauses
 namespace Fortran::parser {
@@ -23,6 +27,47 @@ namespace Fortran::parser {
 constexpr auto startOmpLine = skipStuffBeforeStatement >> "!$OMP "_sptok;
 constexpr auto endOmpLine = space >> endOfLine;
 
+/// Parse OpenMP directive name (this includes compound directives).
+struct OmpDirectiveNameParser {
+  using resultType = llvm::omp::Directive;
+  using Token = TokenStringMatch<false, false>;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    for (const NameWithId &nid : directives()) {
+      if (attempt(Token(nid.first.data())).Parse(state)) {
+        return nid.second;
+      }
+    }
+    return std::nullopt;
+  }
+
+private:
+  using NameWithId = std::pair<std::string, llvm::omp::Directive>;
+
+  llvm::iterator_range<const NameWithId *> directives() const;
+  void initTokens(NameWithId *) const;
+};
+
+llvm::iterator_range<const OmpDirectiveNameParser::NameWithId *>
+OmpDirectiveNameParser::directives() const {
+  static NameWithId table[llvm::omp::Directive_enumSize];
+  [[maybe_unused]] static bool init = (initTokens(table), true);
+  return llvm::make_range(std::cbegin(table), std::cend(table));
+}
+
+void OmpDirectiveNameParser::initTokens(NameWithId *table) const {
+  for (size_t i{0}, e{llvm::omp::Directive_enumSize}; i != e; ++i) {
+    auto id{static_cast<llvm::omp::Directive>(i)};
+    llvm::StringRef name{llvm::omp::getOpenMPDirectiveName(id)};
+    table[i] = std::make_pair(name.str(), id);
+  }
+  // Sort the table with respect to the directive name length in a descending
+  // order. This is to make sure that longer names are tried first, before
+  // any potential prefix (e.g. "target update" before "target").
+  std::sort(table, table + llvm::omp::Directive_enumSize,
+      [](auto &a, auto &b) { return a.first.size() > b.first.size(); });
+}
+
 template <typename Clause, typename Separator> struct ModifierList {
   constexpr ModifierList(Separator sep) : sep_(sep) {}
   constexpr ModifierList(const ModifierList &) = default;
@@ -91,6 +136,16 @@ static TypeDeclarationStmt makeIterSpecDecl(std::list<ObjectName> &&names) {
 
 // --- Parsers for clause modifiers -----------------------------------
 
+TYPE_PARSER(construct<OmpAlignment>(scalarIntExpr))
+
+TYPE_PARSER(construct<OmpAlignModifier>( //
+    "ALIGN" >> parenthesized(scalarIntExpr)))
+
+TYPE_PARSER(construct<OmpAllocatorComplexModifier>(
+    "ALLOCATOR" >> parenthesized(scalarIntExpr)))
+
+TYPE_PARSER(construct<OmpAllocatorSimpleModifier>(scalarIntExpr))
+
 TYPE_PARSER(construct<OmpChunkModifier>( //
     "SIMD" >> pure(OmpChunkModifier::Value::Simd)))
 
@@ -98,6 +153,10 @@ TYPE_PARSER(construct<OmpDependenceType>(
     "SINK" >> pure(OmpDependenceType::Value::Sink) ||
     "SOURCE" >> pure(OmpDependenceType::Value::Source)))
 
+TYPE_PARSER(construct<OmpDeviceModifier>(
+    "ANCESTOR" >> pure(OmpDeviceModifier::Value::Ancestor) ||
+    "DEVICE_NUM" >> pure(OmpDeviceModifier::Value::Device_Num)))
+
 TYPE_PARSER(construct<OmpExpectation>( //
     "PRESENT" >> pure(OmpExpectation::Value::Present)))
 
@@ -122,6 +181,9 @@ TYPE_PARSER(construct<OmpIterator>( //
     "ITERATOR" >>
     parenthesized(nonemptyList(sourced(Parser<OmpIteratorSpecifier>{})))))
 
+TYPE_PARSER(construct<OmpLastprivateModifier>(
+    "CONDITIONAL" >> pure(OmpLastprivateModifier::Value::Conditional)))
+
 // 2.15.3.7 LINEAR (linear-list: linear-step)
 //          linear-list -> list | modifier(list)
 //          linear-modifier -> REF | VAL | UVAL
@@ -162,6 +224,9 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
     "NONMONOTONIC" >> pure(OmpOrderingModifier::Value::Nonmonotonic) ||
     "SIMD" >> pure(OmpOrderingModifier::Value::Simd)))
 
+TYPE_PARSER(construct<OmpPrescriptiveness>(
+    "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
+
 TYPE_PARSER(construct<OmpReductionModifier>(
     "INSCAN" >> pure(OmpReductionModifier::Value::Inscan) ||
     "TASK" >> pure(OmpReductionModifier::Value::Task) ||
@@ -183,11 +248,43 @@ TYPE_PARSER(construct<OmpVariableCategory>(
     "SCALAR" >> pure(OmpVariableCategory::Value::Scalar)))
 
 // This could be auto-generated.
+TYPE_PARSER(
+    sourced(construct<OmpAffinityClause::Modifier>(Parser<OmpIterator>{})))
+
+TYPE_PARSER(
+    sourced(construct<OmpAlignedClause::Modifier>(Parser<OmpAlignment>{})))
+
+TYPE_PARSER(sourced(construct<OmpAllocateClause::Modifier>(sourced(
+    construct<OmpAllocateClause::Modifier>(Parser<OmpAlignModifier>{}) ||
+    construct<OmpAllocateClause::Modifier>(
+        Parser<OmpAllocatorComplexModifier>{}) ||
+    construct<OmpAllocateClause::Modifier>(
+        Parser<OmpAllocatorSimpleModifier>{})))))
+
+TYPE_PARSER(sourced(
+    construct<OmpDefaultmapClause::Modifier>(Parser<OmpVariableCategory>{})))
+
+TYPE_PARSER(sourced(construct<OmpDependClause::TaskDep::Modifier>(sourced(
+    construct<OmpDependClause::TaskDep::Modifier>(Parser<OmpIterator>{}) ||
+    construct<OmpDependClause::TaskDep::Modifier>(
+        Parser<OmpTaskDependenceType>{})))))
+
+TYPE_PARSER(
+    sourced(construct<OmpDeviceClause::Modifier>(Parser<OmpDeviceModifier>{})))
+
 TYPE_PARSER(sourced(construct<OmpFromClause::Modifier>(
     sourced(construct<OmpFromClause::Modifier>(Parser<OmpExpectation>{}) ||
         construct<OmpFromClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpFromClause::Modifier>(Parser<OmpIterator>{})))))
 
+TYPE_PARSER(sourced(
+    construct<OmpGrainsizeClause::Modifier>(Parser<OmpPrescriptiveness>{})))
+
+TYPE_PARSER(sourced(construct<OmpIfClause::Modifier>(OmpDirectiveNameParser{})))
+
+TYPE_PARSER(sourced(construct<OmpLastprivateClause::Modifier>(
+    Parser<OmpLastprivateModifier>{})))
+
 TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
     sourced(construct<OmpMapClause::Modifier>(Parser<OmpMapTypeModifier>{}) ||
         construct<OmpMapClause::Modifier>(Parser<OmpMapper>{}) ||
@@ -197,6 +294,9 @@ TYPE_PARSER(sourced(construct<OmpMapClause::Modifier>(
 TYPE_PARSER(
     sourced(construct<OmpOrderClause::Modifier>(Parser<OmpOrderModifier>{})))
 
+TYPE_PARSER(sourced(
+    construct<OmpNumTasksClause::Modifier>(Parser<OmpPrescriptiveness>{})))
+
 TYPE_PARSER(sourced(construct<OmpReductionClause::Modifier>(sourced(
     construct<OmpReductionClause::Modifier>(Parser<OmpReductionModifier>{}) ||
     construct<OmpReductionClause::Modifier>(
@@ -211,9 +311,6 @@ TYPE_PARSER(sourced(construct<OmpToClause::Modifier>(
         construct<OmpToClause::Modifier>(Parser<OmpMapper>{}) ||
         construct<OmpToClause::Modifier>(Parser<OmpIterator>{})))))
 
-TYPE_PARSER(sourced(
-    construct<OmpDefaultmapClause::Modifier>(Parser<OmpVariableCategory>{})))
-
 // --- Parsers for clauses --------------------------------------------
 
 /// `MOBClause` is a clause that has a
@@ -236,21 +333,23 @@ static inline MOBClause makeMobClause(
 // [5.0] 2.10.1 affinity([aff-modifier:] locator-list)
 //              aff-modifier: interator-modifier
 TYPE_PARSER(construct<OmpAffinityClause>(
-    maybe(Parser<OmpIterator>{} / ":"), Parser<OmpObjectList>{}))
+    maybe(nonemptyList(Parser<OmpAffinityClause::Modifier>{}) / ":"),
+    Parser<OmpObjectList>{}))
 
 // 2.15.3.1 DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
 TYPE_PARSER(construct<OmpDefaultClause>(
-    "PRIVATE" >> pure(OmpDefaultClause::Type::Private) ||
-    "FIRSTPRIVATE" >> pure(OmpDefaultClause::Type::Firstprivate) ||
-    "SHARED" >> pure(OmpDefaultClause::Type::Shared) ||
-    "NONE" >> pure(OmpDefaultClause::Type::None)))
+    "PRIVATE" >> pure(OmpDefaultClause::DataSharingAttribute::Private) ||
+    "FIRSTPRIVATE" >>
+        pure(OmpDefaultClause::DataSharingAttribute::Firstprivate) ||
+    "SHARED" >> pure(OmpDefaultClause::DataSharingAttribute::Shared) ||
+    "NONE" >> pure(OmpDefaultClause::DataSharingAttribute::None)))
 
 // 2.5 PROC_BIND (MASTER | CLOSE | PRIMARY | SPREAD)
 TYPE_PARSER(construct<OmpProcBindClause>(
-    "CLOSE" >> pure(OmpProcBindClause::Type::Close) ||
-    "MASTER" >> pure(OmpProcBindClause::Type::Master) ||
-    "PRIMARY" >> pure(OmpProcBindClause::Type::Primary) ||
-    "SPREAD" >> pure(OmpProcBindClause::Type::Spread)))
+    "CLOSE" >> pure(OmpProcBindClause::AffinityPolicy::Close) ||
+    "MASTER" >> pure(OmpProcBindClause::AffinityPolicy::Master) ||
+    "PRIMARY" >> pure(OmpProcBindClause::AffinityPolicy::Primary) ||
+    "SPREAD" >> pure(OmpProcBindClause::AffinityPolicy::Spread)))
 
 TYPE_PARSER(construct<OmpMapClause>(
     applyFunction<OmpMapClause>(makeMobClause<true>,
@@ -288,36 +387,18 @@ TYPE_PARSER(construct<OmpScheduleClause>(
 
 // device([ device-modifier :] scalar-integer-expression)
 TYPE_PARSER(construct<OmpDeviceClause>(
-    maybe(
-        ("ANCESTOR" >> pure(OmpDeviceClause::DeviceModifier::Ancestor) ||
-            "DEVICE_NUM" >> pure(OmpDeviceClause::DeviceModifier::Device_Num)) /
-        ":"),
+    maybe(nonemptyList(Parser<OmpDeviceClause::Modifier>{}) / ":"),
     scalarIntExpr))
 
 // device_type(any | host | nohost)
 TYPE_PARSER(construct<OmpDeviceTypeClause>(
-    "ANY" >> pure(OmpDeviceTypeClause::Type::Any) ||
-    "HOST" >> pure(OmpDeviceTypeClause::Type::Host) ||
-    "NOHOST" >> pure(OmpDeviceTypeClause::Type::Nohost)))
+    "ANY" >> pure(OmpDeviceTypeClause::DeviceTypeDescription::Any) ||
+    "HOST" >> pure(OmpDeviceTypeClause::DeviceTypeDescription::Host) ||
+    "NOHOST" >> pure(OmpDeviceTypeClause::DeviceTypeDescription::Nohost)))
 
 // 2.12 IF (directive-name-modifier: scalar-logical-expr)
 TYPE_PARSER(construct<OmpIfClause>(
-    maybe(
-        ("PARALLEL" >> pure(OmpIfClause::DirectiveNameModifier::Parallel) ||
-            "SIMD" >> pure(OmpIfClause::DirectiveNameModifier::Simd) ||
-            "TARGET ENTER DATA" >>
-                pure(OmpIfClause::DirectiveNameModifier::TargetEnterData) ||
-            "TARGET EXIT DATA" >>
-                pure(OmpIfClause::DirectiveNameModifier::TargetExitData) ||
-            "TARGET DATA" >>
-                pure(OmpIfClause::DirectiveNameModifier::TargetData) ||
-            "TARGET UPDATE" >>
-                pure(OmpIfClause::DirectiveNameModifier::TargetUpdate) ||
-            "TARGET" >> pure(OmpIfClause::DirectiveNameModifier::Target) ||
-            "TASK"_id >> pure(OmpIfClause::DirectiveNameModifier::Task) ||
-            "TASKLOOP" >> pure(OmpIfClause::DirectiveNameModifier::Taskloop) ||
-            "TEAMS" >> pure(OmpIfClause::DirectiveNameModifier::Teams)) /
-        ":"),
+    maybe(nonemptyList(Parser<OmpIfClause::Modifier>{}) / ":"),
     scalarLogicalExpr))
 
 TYPE_PARSER(construct<OmpReductionClause>(
@@ -334,29 +415,7 @@ TYPE_PARSER(construct<OmpInReductionClause>(
 //                                   variable-name-list)
 //                allocate-modifier -> allocator | align
 TYPE_PARSER(construct<OmpAllocateClause>(
-    maybe(
-        first(
-            construct<OmpAllocateClause::AllocateModifier>("ALLOCATOR" >>
-                construct<OmpAllocateClause::AllocateModifier::ComplexModifier>(
-                    parenthesized(construct<
-                        OmpAllocateClause::AllocateModifier::Allocator>(
-                        scalarIntExpr)) /
-                        ",",
-                    "ALIGN" >> parenthesized(construct<
-                                   OmpAllocateClause::AllocateModifier::Align>(
-                                   scalarIntExpr)))),
-            construct<OmpAllocateClause::AllocateModifier>("ALLOCATOR" >>
-                parenthesized(
-                    construct<OmpAllocateClause::AllocateModifier::Allocator>(
-                        scalarIntExpr))),
-            construct<OmpAllocateClause::AllocateModifier>("ALIGN" >>
-                parenthesized(
-                    construct<OmpAllocateClause::AllocateModifier::Align>(
-                        scalarIntExpr))),
-            construct<OmpAllocateClause::AllocateModifier>(
-                construct<OmpAllocateClause::AllocateModifier::Allocator>(
-                    scalarIntExpr))) /
-        ":"),
+    maybe(nonemptyList(Parser<OmpAllocateClause::Modifier>{}) / ":"),
     Parser<OmpObjectList>{}))
 
 // iteration-offset -> +/- non-negative-constant-expr
@@ -375,10 +434,16 @@ TYPE_PARSER(construct<OmpDoacross>(
 
 TYPE_CONTEXT_PARSER("Omp Depend clause"_en_US,
     construct<OmpDependClause>(
+        // Try to parse OmpDoacross first, because TaskDep will succeed on
+        // "sink: xxx", interpreting it to not have any modifiers, and "sink"
+        // being an OmpObject. Parsing of the TaskDep variant will stop right
+        // after the "sink", leaving the ": xxx" unvisited.
+        construct<OmpDependClause>(Parser<OmpDoacross>{}) ||
+        // Parse TaskDep after Doacross.
         construct<OmpDependClause>(construct<OmpDependClause::TaskDep>(
-            maybe(Parser<OmpIterator>{} / ","_tok),
-            Parser<OmpTaskDependenceType>{} / ":", Parser<OmpObjectList>{})) ||
-        construct<OmpDependClause>(Parser<OmpDoacross>{})))
+            maybe(nonemptyList(Parser<OmpDependClause::TaskDep::Modifier>{}) /
+                ": "),
+            Parser<OmpObjectList>{}))))
 
 TYPE_CONTEXT_PARSER("Omp Doacross clause"_en_US,
     construct<OmpDoacrossClause>(Parser<OmpDoacross>{}))
@@ -407,8 +472,8 @@ TYPE_CONTEXT_PARSER("Omp LINEAR clause"_en_US,
 TYPE_PARSER(construct<OmpDetachClause>(Parser<OmpObject>{}))
 
 // 2.8.1 ALIGNED (list: alignment)
-TYPE_PARSER(construct<OmpAlignedClause>(
-    Parser<OmpObjectList>{}, maybe(":" >> scalarIntConstantExpr)))
+TYPE_PARSER(construct<OmpAlignedClause>(Parser<OmpObjectList>{},
+    maybe(":" >> nonemptyList(Parser<OmpAlignedClause::Modifier>{}))))
 
 TYPE_PARSER(construct<OmpUpdateClause>(
     construct<OmpUpdateClause>(Parser<OmpDependenceType>{}) ||
@@ -420,12 +485,12 @@ TYPE_PARSER(construct<OmpOrderClause>(
 
 // OMP 5.2 12.6.1 grainsize([ prescriptiveness :] scalar-integer-expression)
 TYPE_PARSER(construct<OmpGrainsizeClause>(
-    maybe("STRICT" >> pure(OmpGrainsizeClause::Prescriptiveness::Strict) / ":"),
+    maybe(nonemptyList(Parser<OmpGrainsizeClause::Modifier>{}) / ":"),
     scalarIntExpr))
 
 // OMP 5.2 12.6.2 num_tasks([ prescriptiveness :] scalar-integer-expression)
 TYPE_PARSER(construct<OmpNumTasksClause>(
-    maybe("STRICT" >> pure(OmpNumTasksClause::Prescriptiveness::Strict) / ":"),
+    maybe(nonemptyList(Parser<OmpNumTasksClause::Modifier>{}) / ":"),
     scalarIntExpr))
 
 TYPE_PARSER(
@@ -433,15 +498,14 @@ TYPE_PARSER(
 
 // OMP 5.0 2.19.4.5 LASTPRIVATE ([lastprivate-modifier :] list)
 TYPE_PARSER(construct<OmpLastprivateClause>(
-    maybe("CONDITIONAL" >>
-        pure(OmpLastprivateClause::LastprivateModifier::Conditional) / ":"),
+    maybe(nonemptyList(Parser<OmpLastprivateClause::Modifier>{}) / ":"),
     Parser<OmpObjectList>{}))
 
 // OMP 5.2 11.7.1 BIND ( PARALLEL | TEAMS | THREAD )
 TYPE_PARSER(construct<OmpBindClause>(
-    "PARALLEL" >> pure(OmpBindClause::Type::Parallel) ||
-    "TEAMS" >> pure(OmpBindClause::Type::Teams) ||
-    "THREAD" >> pure(OmpBindClause::Type::Thread)))
+    "PARALLEL" >> pure(OmpBindClause::Binding::Parallel) ||
+    "TEAMS" >> pure(OmpBindClause::Binding::Teams) ||
+    "THREAD" >> pure(OmpBindClause::Binding::Thread)))
 
 TYPE_PARSER(
     "ACQUIRE" >> construct<OmpClause>(construct<OmpClause::Acquire>()) ||
@@ -832,6 +896,17 @@ TYPE_PARSER("ATOMIC" >>
         Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
         statement(assignmentStmt), Parser<OmpEndAtomic>{} / endOmpLine)))
 
+TYPE_PARSER(construct<OmpAtomicCompareIfStmt>(indirect(Parser<IfStmt>{})) ||
+    construct<OmpAtomicCompareIfStmt>(indirect(Parser<IfConstruct>{})))
+
+// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] COMPARE [MEMORY-ORDER-CLAUSE-LIST]
+TYPE_PARSER("ATOMIC" >>
+    sourced(construct<OmpAtomicCompare>(
+        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("COMPARE"_tok),
+        Parser<OmpAtomicClauseList>{} / endOmpLine,
+        Parser<OmpAtomicCompareIfStmt>{},
+        maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
+
 // OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] UPDATE [MEMORY-ORDER-CLAUSE-LIST]
 TYPE_PARSER("ATOMIC" >>
     sourced(construct<OmpAtomicUpdate>(
@@ -854,6 +929,7 @@ TYPE_PARSER("ATOMIC" >>
 // Atomic Construct
 TYPE_PARSER(construct<OpenMPAtomicConstruct>(Parser<OmpAtomicRead>{}) ||
     construct<OpenMPAtomicConstruct>(Parser<OmpAtomicCapture>{}) ||
+    construct<OpenMPAtomicConstruct>(Parser<OmpAtomicCompare>{}) ||
     construct<OpenMPAtomicConstruct>(Parser<OmpAtomicWrite>{}) ||
     construct<OpenMPAtomicConstruct>(Parser<OmpAtomicUpdate>{}) ||
     construct<OpenMPAtomicConstruct>(Parser<OmpAtomic>{}))
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 24b2902f286f4..a414f226058e3 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -267,7 +267,18 @@ OmpDependenceType::Value OmpDoacross::GetDepType() const {
 }
 
 OmpTaskDependenceType::Value OmpDependClause::TaskDep::GetTaskDepType() const {
-  return std::get<parser::OmpTaskDependenceType>(t).v;
+  using Modifier = OmpDependClause::TaskDep::Modifier;
+  auto &modifiers{std::get<std::optional<std::list<Modifier>>>(t)};
+  if (modifiers) {
+    for (auto &m : *modifiers) {
+      if (auto *dep{std::get_if<OmpTaskDependenceType>(&m.u)}) {
+        return dep->v;
+      }
+    }
+    llvm_unreachable("expecting OmpTaskDependenceType in TaskDep");
+  } else {
+    llvm_unreachable("expecting modifiers on OmpDependClause::TaskDep");
+  }
 }
 
 } // namespace Fortran::parser
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 34e660f8d2664..3cd32d7e6c92e 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -234,7 +234,7 @@ void Prescanner::Statement() {
             directiveSentinel_ = newLineClass.sentinel;
             disableSourceContinuation_ = false;
           } else {
-            disableSourceContinuation_ =
+            disableSourceContinuation_ = !replaced->empty() &&
                 newLineClass.kind != LineClassification::Kind::Source;
           }
         }
diff --git a/flang/lib/Parser/type-parsers.h b/flang/lib/Parser/type-parsers.h
index f800398a22de9..d7e0cd06c3f44 100644
--- a/flang/lib/Parser/type-parsers.h
+++ b/flang/lib/Parser/type-parsers.h
@@ -72,6 +72,7 @@ constexpr Parser<LanguageBindingSpec> languageBindingSpec; // R808, R1528
 constexpr Parser<EntityDecl> entityDecl; // R803
 constexpr Parser<CoarraySpec> coarraySpec; // R809
 constexpr Parser<ArraySpec> arraySpec; // R815
+constexpr Parser<ComponentArraySpec> componentArraySpec;
 constexpr Parser<ExplicitShapeSpec> explicitShapeSpec; // R816
 constexpr Parser<DeferredShapeSpecList> deferredShapeSpecList; // R820
 constexpr Parser<AssumedImpliedSpec> assumedImpliedSpec; // R821
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index e1708157b63dc..4782cc1f2d7d7 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2074,6 +2074,9 @@ class UnparseVisitor {
                   },
         x.u);
   }
+  void Unparse(const OmpDirectiveNameModifier &x) {
+    Word(llvm::omp::getOpenMPDirectiveName(x.v));
+  }
   void Unparse(const OmpIteratorSpecifier &x) {
     Walk(std::get<TypeDeclarationStmt>(x.t));
     Put(" = ");
@@ -2090,9 +2093,8 @@ class UnparseVisitor {
     Put(")");
   }
   void Unparse(const OmpLastprivateClause &x) {
-    Walk(
-        std::get<std::optional<OmpLastprivateClause::LastprivateModifier>>(x.t),
-        ":");
+    using Modifier = OmpLastprivateClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<OmpObjectList>(x.t));
   }
   void Unparse(const OmpMapClause &x) {
@@ -2107,17 +2109,19 @@ class UnparseVisitor {
     Walk(",", std::get<std::optional<ScalarIntExpr>>(x.t));
   }
   void Unparse(const OmpDeviceClause &x) {
-    Walk(std::get<std::optional<OmpDeviceClause::DeviceModifier>>(x.t), ":");
+    using Modifier = OmpDeviceClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<ScalarIntExpr>(x.t));
   }
   void Unparse(const OmpAffinityClause &x) {
-    Walk(std::get<std::optional<OmpIterator>>(x.t), ":");
+    using Modifier = OmpAffinityClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<OmpObjectList>(x.t));
   }
   void Unparse(const OmpAlignedClause &x) {
+    using Modifier = OmpAlignedClause::Modifier;
     Walk(std::get<OmpObjectList>(x.t));
-    Put(",");
-    Walk(std::get<std::optional<ScalarIntConstantExpr>>(x.t));
+    Walk(": ", std::get<std::optional<std::list<Modifier>>>(x.t));
   }
   void Unparse(const OmpFromClause &x) {
     using Modifier = OmpFromClause::Modifier;
@@ -2125,7 +2129,8 @@ class UnparseVisitor {
     Walk(std::get<OmpObjectList>(x.t));
   }
   void Unparse(const OmpIfClause &x) {
-    Walk(std::get<std::optional<OmpIfClause::DirectiveNameModifier>>(x.t), ":");
+    using Modifier = OmpIfClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<ScalarLogicalExpr>(x.t));
   }
   void Unparse(const OmpLinearClause::WithoutModifier &x) {
@@ -2148,48 +2153,34 @@ class UnparseVisitor {
     Walk(std::get<OmpObjectList>(x.t));
   }
   void Unparse(const OmpAllocateClause &x) {
-    Walk(
-        std::get<std::optional<OmpAllocateClause::AllocateModifier>>(x.t), ":");
+    using Modifier = OmpAllocateClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<OmpObjectList>(x.t));
   }
-  void Unparse(const OmpAllocateClause::AllocateModifier &x) {
-    common::visit(
-        common::visitors{
-            [&](const OmpAllocateClause::AllocateModifier::Allocator &y) {
-              Walk(y);
-            },
-            [&](const OmpAllocateClause::AllocateModifier::ComplexModifier &y) {
-              Word("ALLOCATOR(");
-              Walk(std::get<OmpAllocateClause::AllocateModifier::Allocator>(
-                  y.t));
-              Put(")");
-              Put(",");
-              Walk(std::get<OmpAllocateClause::AllocateModifier::Align>(y.t));
-            },
-            [&](const OmpAllocateClause::AllocateModifier::Align &y) {
-              Walk(y);
-            },
-        },
-        x.u);
-  }
-  void Unparse(const OmpAllocateClause::AllocateModifier::Align &x) {
+  void Unparse(const OmpAlignModifier &x) {
     Word("ALIGN(");
     Walk(x.v);
     Put(")");
   }
+  void Unparse(const OmpAllocatorSimpleModifier &x) { Walk(x.v); }
+  void Unparse(const OmpAllocatorComplexModifier &x) {
+    Word("ALLOCATOR(");
+    Walk(x.v);
+    Put(")");
+  }
   void Unparse(const OmpOrderClause &x) {
     using Modifier = OmpOrderClause::Modifier;
     Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ":");
     Walk(std::get<OmpOrderClause::Ordering>(x.t));
   }
   void Unparse(const OmpGrainsizeClause &x) {
-    Walk(std::get<std::optional<OmpGrainsizeClause::Prescriptiveness>>(x.t),
-        ":");
+    using Modifier = OmpGrainsizeClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<ScalarIntExpr>(x.t));
   }
   void Unparse(const OmpNumTasksClause &x) {
-    Walk(
-        std::get<std::optional<OmpNumTasksClause::Prescriptiveness>>(x.t), ":");
+    using Modifier = OmpNumTasksClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<ScalarIntExpr>(x.t));
   }
   void Unparse(const OmpDoacross::Sink &x) {
@@ -2198,8 +2189,8 @@ class UnparseVisitor {
   }
   void Unparse(const OmpDoacross::Source &) { Word("SOURCE"); }
   void Unparse(const OmpDependClause::TaskDep &x) {
-    Walk(std::get<OmpTaskDependenceType>(x.t));
-    Put(":");
+    using Modifier = OmpDependClause::TaskDep::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
     Walk(std::get<OmpObjectList>(x.t));
   }
   void Unparse(const OmpDefaultmapClause &x) {
@@ -2451,6 +2442,16 @@ class UnparseVisitor {
     Word("!$OMP END ATOMIC\n");
     EndOpenMP();
   }
+  void Unparse(const OmpAtomicCompare &x) {
+    BeginOpenMP();
+    Word("!$OMP ATOMIC");
+    Walk(std::get<0>(x.t));
+    Word(" COMPARE");
+    Walk(std::get<2>(x.t));
+    Put("\n");
+    EndOpenMP();
+    Walk(std::get<OmpAtomicCompareIfStmt>(x.t));
+  }
   void Unparse(const OmpAtomicRead &x) {
     BeginOpenMP();
     Word("!$OMP ATOMIC");
@@ -2834,29 +2835,26 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(InquireSpec::LogVar, Kind)
   WALK_NESTED_ENUM(ProcedureStmt, Kind) // R1506
   WALK_NESTED_ENUM(UseStmt, ModuleNature) // R1410
-  WALK_NESTED_ENUM(OmpBindClause, Type) // OMP bind
-  WALK_NESTED_ENUM(OmpProcBindClause, Type) // OMP PROC_BIND
-  WALK_NESTED_ENUM(OmpDefaultClause, Type) // OMP DEFAULT
-  WALK_NESTED_ENUM(OmpDefaultmapClause, ImplicitBehavior) // OMP DEFAULTMAP
+  WALK_NESTED_ENUM(OmpBindClause, Binding) // OMP bind
+  WALK_NESTED_ENUM(OmpProcBindClause, AffinityPolicy) // OMP proc_bind
+  WALK_NESTED_ENUM(OmpDefaultClause, DataSharingAttribute) // OMP default
+  WALK_NESTED_ENUM(OmpDefaultmapClause, ImplicitBehavior) // OMP defaultmap
   WALK_NESTED_ENUM(OmpVariableCategory, Value) // OMP variable-category
-  WALK_NESTED_ENUM(
-      OmpLastprivateClause, LastprivateModifier) // OMP lastprivate-modifier
+  WALK_NESTED_ENUM(OmpLastprivateModifier, Value) // OMP lastprivate-modifier
   WALK_NESTED_ENUM(OmpChunkModifier, Value) // OMP chunk-modifier
   WALK_NESTED_ENUM(OmpLinearModifier, Value) // OMP linear-modifier
   WALK_NESTED_ENUM(OmpOrderingModifier, Value) // OMP ordering-modifier
   WALK_NESTED_ENUM(OmpTaskDependenceType, Value) // OMP task-dependence-type
   WALK_NESTED_ENUM(OmpScheduleClause, Kind) // OMP schedule-kind
-  WALK_NESTED_ENUM(OmpDeviceClause, DeviceModifier) // OMP device modifier
-  WALK_NESTED_ENUM(OmpDeviceTypeClause, Type) // OMP DEVICE_TYPE
+  WALK_NESTED_ENUM(OmpDeviceModifier, Value) // OMP device modifier
+  WALK_NESTED_ENUM(
+      OmpDeviceTypeClause, DeviceTypeDescription) // OMP device_type
   WALK_NESTED_ENUM(OmpReductionModifier, Value) // OMP reduction-modifier
   WALK_NESTED_ENUM(OmpExpectation, Value) // OMP motion-expectation
-  WALK_NESTED_ENUM(OmpIfClause, DirectiveNameModifier) // OMP directive-modifier
   WALK_NESTED_ENUM(OmpCancelType, Type) // OMP cancel-type
   WALK_NESTED_ENUM(OmpOrderClause, Ordering) // OMP ordering
   WALK_NESTED_ENUM(OmpOrderModifier, Value) // OMP order-modifier
-  WALK_NESTED_ENUM(
-      OmpGrainsizeClause, Prescriptiveness) // OMP grainsize-modifier
-  WALK_NESTED_ENUM(OmpNumTasksClause, Prescriptiveness) // OMP numtasks-modifier
+  WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
   WALK_NESTED_ENUM(OmpMapType, Value) // OMP map-type
   WALK_NESTED_ENUM(OmpMapTypeModifier, Value) // OMP map-type-modifier
 #undef WALK_NESTED_ENUM
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index c9656d031b2e1..99c6e16c4260f 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -320,8 +320,14 @@ void CheckHelper::Check(const Symbol &symbol) {
   if (symbol.attrs().HasAny({Attr::INTENT_IN, Attr::INTENT_INOUT,
           Attr::INTENT_OUT, Attr::OPTIONAL, Attr::VALUE}) &&
       !IsDummy(symbol)) {
-    messages_.Say(
-        "Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute"_err_en_US);
+    if (context_.IsEnabled(
+            common::LanguageFeature::IgnoreIrrelevantAttributes)) {
+      context_.Warn(common::LanguageFeature::IgnoreIrrelevantAttributes,
+          "Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute"_warn_en_US);
+    } else {
+      messages_.Say(
+          "Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute"_err_en_US);
+    }
   } else if (symbol.attrs().test(Attr::VALUE)) {
     CheckValue(symbol, derived);
   }
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 3733ebfaf9492..3c9c5a02a338a 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -51,6 +51,30 @@ std::string TryVersion(unsigned version) {
   return "try -fopenmp-version=" + std::to_string(version);
 }
 
+static const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+static const parser::DataRef *GetDataRefFromObj(
+    const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+static const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -222,6 +246,10 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
+bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
 bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
   return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
 }
@@ -302,22 +330,26 @@ struct ContiguousHelper {
 };
 } // namespace
 
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
 std::optional<bool> OmpStructureChecker::IsContiguous(
     const parser::OmpObject &object) {
-  return common::visit(common::visitors{
-                           [&](const parser::Name &x) {
-                             // Any member of a common block must be contiguous.
-                             return std::optional<bool>{true};
-                           },
-                           [&](const parser::Designator &x) {
-                             evaluate::ExpressionAnalyzer ea{context_};
-                             if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-                               return ContiguousHelper{context_}.Visit(
-                                   *maybeExpr);
-                             }
-                             return std::optional<bool>{};
-                           },
-                       },
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{context_};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{context_}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
       object.u);
 }
 
@@ -379,6 +411,19 @@ void OmpStructureChecker::CheckMultListItems() {
     CheckMultipleOccurrence(
         listVars, nontempNameList, clause->source, "NONTEMPORAL");
   }
+
+  // Linear clause
+  for (auto [_, clause] : FindClauses(llvm::omp::Clause::OMPC_linear)) {
+    const auto &linearClause{std::get<parser::OmpClause::Linear>(clause->u)};
+    std::list<parser::Name> nameList;
+    common::visit(
+        [&](const auto &u) {
+          std::copy(
+              u.names.begin(), u.names.end(), std::back_inserter(nameList));
+        },
+        linearClause.v.u);
+    CheckMultipleOccurrence(listVars, nameList, clause->source, "LINEAR");
+  }
 }
 
 bool OmpStructureChecker::HasInvalidWorksharingNesting(
@@ -428,7 +473,7 @@ void OmpStructureChecker::HasInvalidLoopBinding(
     for (const auto &clause : clauseList.v) {
       if (const auto *bindClause{
               std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Type::Teams) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
           context_.Say(beginDir.source, msg);
         }
       }
@@ -1481,34 +1526,26 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Allocator &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Allocate &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_allocate);
-  if (const auto &modifier{
-          std::get<std::optional<parser::OmpAllocateClause::AllocateModifier>>(
-              x.v.t)}) {
-    common::visit(
-        common::visitors{
-            [&](const parser::OmpAllocateClause::AllocateModifier::Allocator
-                    &y) {
-              RequiresPositiveParameter(llvm::omp::Clause::OMPC_allocate, y.v);
-              isPredefinedAllocator = GetIntValue(y.v).has_value();
-            },
-            [&](const parser::OmpAllocateClause::AllocateModifier::
-                    ComplexModifier &y) {
-              const auto &alloc = std::get<
-                  parser::OmpAllocateClause::AllocateModifier::Allocator>(y.t);
-              const auto &align =
-                  std::get<parser::OmpAllocateClause::AllocateModifier::Align>(
-                      y.t);
-              RequiresPositiveParameter(
-                  llvm::omp::Clause::OMPC_allocate, alloc.v);
-              RequiresPositiveParameter(
-                  llvm::omp::Clause::OMPC_allocate, align.v);
-              isPredefinedAllocator = GetIntValue(alloc.v).has_value();
-            },
-            [&](const parser::OmpAllocateClause::AllocateModifier::Align &y) {
-              RequiresPositiveParameter(llvm::omp::Clause::OMPC_allocate, y.v);
-            },
-        },
-        modifier->u);
+  if (OmpVerifyModifiers(
+          x.v, llvm::omp::OMPC_allocate, GetContext().clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    if (auto *align{
+            OmpGetUniqueModifier<parser::OmpAlignModifier>(modifiers)}) {
+      if (const auto &v{GetIntValue(align->v)}; !v || *v <= 0) {
+        context_.Say(OmpGetModifierSource(modifiers, align),
+            "The alignment value should be a constant positive integer"_err_en_US);
+      }
+    }
+    // The simple and complex modifiers have the same structure. They only
+    // differ in their syntax.
+    if (auto *alloc{OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(
+            modifiers)}) {
+      isPredefinedAllocator = GetIntValue(alloc->v).has_value();
+    }
+    if (auto *alloc{OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(
+            modifiers)}) {
+      isPredefinedAllocator = GetIntValue(alloc->v).has_value();
+    }
   }
 }
 
@@ -1652,7 +1689,7 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareTargetConstruct &x) {
               [&](const parser::OmpClause::DeviceType &deviceTypeClause) {
                 deviceTypeClauseFound = true;
                 if (deviceTypeClause.v.v !=
-                    parser::OmpDeviceTypeClause::Type::Host) {
+                    parser::OmpDeviceTypeClause::DeviceTypeDescription::Host) {
                   // Function / subroutine explicitly marked as runnable by the
                   // target device.
                   deviceConstructFound_ = true;
@@ -2421,6 +2458,23 @@ void OmpStructureChecker::CheckAtomicUpdateStmt(
   ErrIfAllocatableVariable(var);
 }
 
+void OmpStructureChecker::CheckAtomicCompareConstruct(
+    const parser::OmpAtomicCompare &atomicCompareConstruct) {
+
+  // TODO: Check that the if-stmt is `if (var == expr) var = new`
+  //       [with or without then/end-do]
+
+  unsigned version{context_.langOptions().OpenMPVersion};
+  if (version < 51) {
+    context_.Say(atomicCompareConstruct.source,
+        "%s construct not allowed in %s, %s"_err_en_US,
+        atomicCompareConstruct.source, ThisVersion(version), TryVersion(51));
+  }
+
+  // TODO: More work needed here. Some of the Update restrictions need to
+  // be added, but Update isn't the same either.
+}
+
 // TODO: Allow cond-update-stmt once compare clause is supported.
 void OmpStructureChecker::CheckAtomicCaptureConstruct(
     const parser::OmpAtomicCapture &atomicCaptureConstruct) {
@@ -2566,6 +2620,16 @@ void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
                 &std::get<0>(atomicCapture.t), &std::get<2>(atomicCapture.t));
             CheckAtomicCaptureConstruct(atomicCapture);
           },
+          [&](const parser::OmpAtomicCompare &atomicCompare) {
+            const auto &dir{std::get<parser::Verbatim>(atomicCompare.t)};
+            PushContextAndClauseSets(
+                dir.source, llvm::omp::Directive::OMPD_atomic);
+            CheckAtomicMemoryOrderClause(
+                &std::get<0>(atomicCompare.t), &std::get<2>(atomicCompare.t));
+            CheckHintClause<const parser::OmpAtomicClauseList>(
+                &std::get<0>(atomicCompare.t), &std::get<2>(atomicCompare.t));
+            CheckAtomicCompareConstruct(atomicCompare);
+          },
       },
       x.u);
 }
@@ -2663,12 +2727,12 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
         }
       }
     }
-
-    // Sema checks related to presence of multiple list items within the same
-    // clause
-    CheckMultListItems();
   } // SIMD
 
+  // Semantic checks related to presence of multiple list items within the same
+  // clause
+  CheckMultListItems();
+
   // 2.7.3 Single Construct Restriction
   if (GetContext().directive == llvm::omp::Directive::OMPD_end_single) {
     CheckNotAllowedIfClause(
@@ -2742,9 +2806,9 @@ void OmpStructureChecker::Enter(const parser::OmpClause &x) {
   if (const parser::OmpObjectList *objList{GetOmpObjectList(x)}) {
     SymbolSourceMap symbols;
     GetSymbolsInObjectList(*objList, symbols);
-    for (const auto &[sym, source] : symbols) {
-      if (!IsVariableListItem(*sym)) {
-        deferredNonVariables_.insert({sym, source});
+    for (const auto &[symbol, source] : symbols) {
+      if (!IsVariableListItem(*symbol)) {
+        deferredNonVariables_.insert({symbol, source});
       }
     }
   }
@@ -2860,6 +2924,8 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Destroy &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_reduction);
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+
   if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_reduction,
           GetContext().clauseSource, context_)) {
     if (CheckReductionOperators(x)) {
@@ -2872,6 +2938,13 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
       CheckReductionModifier(*maybeModifier);
     }
   }
+  CheckReductionObjects(objects, llvm::omp::Clause::OMPC_reduction);
+
+  // If this is a worksharing construct then ensure the reduction variable
+  // is not private in the parallel region that it binds to.
+  if (llvm::omp::nestedReduceWorkshareAllowedSet.test(GetContext().directive)) {
+    CheckSharedBindingInOuterContext(objects);
+  }
 }
 
 bool OmpStructureChecker::CheckReductionOperators(
@@ -2940,6 +3013,69 @@ bool OmpStructureChecker::CheckIntrinsicOperator(
   return false;
 }
 
+/// Check restrictions on objects that are common to all reduction clauses.
+void OmpStructureChecker::CheckReductionObjects(
+    const parser::OmpObjectList &objects, llvm::omp::Clause clauseId) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  SymbolSourceMap symbols;
+  GetSymbolsInObjectList(objects, symbols);
+
+  // Array sections must be a contiguous storage, have non-zero length.
+  for (const parser::OmpObject &object : objects.v) {
+    CheckIfContiguous(object);
+  }
+  CheckReductionArraySection(objects);
+  // An object must be definable.
+  CheckDefinableObjects(symbols, clauseId);
+  // Procedure pointers are not allowed.
+  CheckProcedurePointer(symbols, clauseId);
+  // Pointers must not have INTENT(IN).
+  CheckIntentInPointer(symbols, clauseId);
+
+  // Disallow common blocks.
+  // Iterate on objects because `GetSymbolsInObjectList` expands common block
+  // names into the lists of their members.
+  for (const parser::OmpObject &object : objects.v) {
+    auto *symbol{GetObjectSymbol(object)};
+    assert(symbol && "Expecting a symbol for object");
+    if (IsCommonBlock(*symbol)) {
+      auto source{GetObjectSource(object)};
+      context_.Say(source ? *source : GetContext().clauseSource,
+          "Common block names are not allowed in %s clause"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
+    }
+  }
+
+  if (version >= 50) {
+    // Object cannot be a part of another object (except array elements)
+    CheckStructureComponent(objects, clauseId);
+    // If object is an array section or element, the base expression must be
+    // a language identifier.
+    for (const parser::OmpObject &object : objects.v) {
+      if (auto *elem{GetArrayElementFromObj(object)}) {
+        const parser::DataRef &base = elem->base;
+        if (!std::holds_alternative<parser::Name>(base.u)) {
+          auto source{GetObjectSource(object)};
+          context_.Say(source ? *source : GetContext().clauseSource,
+              "The base expression of an array element or section in %s clause must be an identifier"_err_en_US,
+              parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
+        }
+      }
+    }
+    // Type parameter inquiries are not allowed.
+    for (const parser::OmpObject &object : objects.v) {
+      if (auto *dataRef{GetDataRefFromObj(object)}) {
+        if (IsDataRefTypeParamInquiry(dataRef)) {
+          auto source{GetObjectSource(object)};
+          context_.Say(source ? *source : GetContext().clauseSource,
+              "Type parameter inquiry is not permitted in %s clause"_err_en_US,
+              parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
+        }
+      }
+    }
+  }
+}
+
 static bool IsReductionAllowedForType(
     const parser::OmpClause::Reduction &x, const DeclTypeSpec &type) {
   auto &modifiers{OmpGetModifiers(x.v)};
@@ -3029,26 +3165,18 @@ static bool IsReductionAllowedForType(
 void OmpStructureChecker::CheckReductionTypeList(
     const parser::OmpClause::Reduction &x) {
   const auto &ompObjectList{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckIntentInPointerAndDefinable(
-      ompObjectList, llvm::omp::Clause::OMPC_reduction);
-  CheckReductionArraySection(ompObjectList);
-  // If this is a worksharing construct then ensure the reduction variable
-  // is not private in the parallel region that it binds to.
-  if (llvm::omp::nestedReduceWorkshareAllowedSet.test(GetContext().directive)) {
-    CheckSharedBindingInOuterContext(ompObjectList);
-  }
-
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(ompObjectList, symbols);
+
   for (auto &[symbol, source] : symbols) {
-    if (IsProcedurePointer(*symbol)) {
-      context_.Say(source,
-          "A procedure pointer '%s' must not appear in a REDUCTION clause."_err_en_US,
-          symbol->name());
-    } else if (!IsReductionAllowedForType(x, DEREF(symbol->GetType()))) {
-      context_.Say(source,
-          "The type of '%s' is incompatible with the reduction operator."_err_en_US,
-          symbol->name());
+    if (auto *type{symbol->GetType()}) {
+      if (!IsReductionAllowedForType(x, *type)) {
+        context_.Say(source,
+            "The type of '%s' is incompatible with the reduction operator."_err_en_US,
+            symbol->name());
+      }
+    } else {
+      assert(IsProcedurePointer(*symbol) && "Unexpected symbol properties");
     }
   }
 }
@@ -3104,43 +3232,14 @@ void OmpStructureChecker::CheckReductionModifier(
   }
 }
 
-void OmpStructureChecker::CheckIntentInPointerAndDefinable(
-    const parser::OmpObjectList &objectList, const llvm::omp::Clause clause) {
-  for (const auto &ompObject : objectList.v) {
-    if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
-      if (const auto *symbol{name->symbol}) {
-        if (IsPointer(symbol->GetUltimate()) &&
-            IsIntentIn(symbol->GetUltimate())) {
-          context_.Say(GetContext().clauseSource,
-              "Pointer '%s' with the INTENT(IN) attribute may not appear "
-              "in a %s clause"_err_en_US,
-              symbol->name(),
-              parser::ToUpperCaseLetters(getClauseName(clause).str()));
-        } else if (auto msg{WhyNotDefinable(name->source,
-                       context_.FindScope(name->source), DefinabilityFlags{},
-                       *symbol)}) {
-          context_
-              .Say(GetContext().clauseSource,
-                  "Variable '%s' on the %s clause is not definable"_err_en_US,
-                  symbol->name(),
-                  parser::ToUpperCaseLetters(getClauseName(clause).str()))
-              .Attach(std::move(msg->set_severity(parser::Severity::Because)));
-        }
-      }
-    }
-  }
-}
-
 void OmpStructureChecker::CheckReductionArraySection(
     const parser::OmpObjectList &ompObjectList) {
   for (const auto &ompObject : ompObjectList.v) {
     if (const auto *dataRef{parser::Unwrap<parser::DataRef>(ompObject)}) {
       if (const auto *arrayElement{
               parser::Unwrap<parser::ArrayElement>(ompObject)}) {
-        if (arrayElement) {
-          CheckArraySection(*arrayElement, GetLastName(*dataRef),
-              llvm::omp::Clause::OMPC_reduction);
-        }
+        CheckArraySection(*arrayElement, GetLastName(*dataRef),
+            llvm::omp::Clause::OMPC_reduction);
       }
     }
   }
@@ -3209,9 +3308,11 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Shared &x) {
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "SHARED");
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
+  SymbolSourceMap symbols;
+  GetSymbolsInObjectList(x.v, symbols);
   CheckAllowedClause(llvm::omp::Clause::OMPC_private);
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "PRIVATE");
-  CheckIntentInPointer(x.v, llvm::omp::Clause::OMPC_private);
+  CheckIntentInPointer(symbols, llvm::omp::Clause::OMPC_private);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Nowait &x) {
@@ -3380,10 +3481,15 @@ void OmpStructureChecker::Leave(const parser::OmpAtomic &) {
 // generalized restrictions.
 void OmpStructureChecker::Enter(const parser::OmpClause::Aligned &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_aligned);
-
-  if (const auto &expr{
-          std::get<std::optional<parser::ScalarIntConstantExpr>>(x.v.t)}) {
-    RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_aligned, *expr);
+  if (OmpVerifyModifiers(
+          x.v, llvm::omp::OMPC_aligned, GetContext().clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    if (auto *align{OmpGetUniqueModifier<parser::OmpAlignment>(modifiers)}) {
+      if (const auto &v{GetIntValue(align->v)}; !v || *v <= 0) {
+        context_.Say(OmpGetModifierSource(modifiers, align),
+            "The alignment value should be a constant positive integer"_err_en_US);
+      }
+    }
   }
   // 2.8.1 TODO: list-item attribute check
 }
@@ -3431,36 +3537,82 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Defaultmap &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_if);
-  using dirNameModifier = parser::OmpIfClause::DirectiveNameModifier;
-  // TODO Check that, when multiple 'if' clauses are applied to a combined
-  // construct, at most one of them applies to each directive.
-  static std::unordered_map<dirNameModifier, OmpDirectiveSet>
-      dirNameModifierMap{{dirNameModifier::Parallel, llvm::omp::allParallelSet},
-          {dirNameModifier::Simd, llvm::omp::allSimdSet},
-          {dirNameModifier::Target, llvm::omp::allTargetSet},
-          {dirNameModifier::TargetData,
-              {llvm::omp::Directive::OMPD_target_data}},
-          {dirNameModifier::TargetEnterData,
-              {llvm::omp::Directive::OMPD_target_enter_data}},
-          {dirNameModifier::TargetExitData,
-              {llvm::omp::Directive::OMPD_target_exit_data}},
-          {dirNameModifier::TargetUpdate,
-              {llvm::omp::Directive::OMPD_target_update}},
-          {dirNameModifier::Task, {llvm::omp::Directive::OMPD_task}},
-          {dirNameModifier::Taskloop, llvm::omp::allTaskloopSet},
-          {dirNameModifier::Teams, llvm::omp::allTeamsSet}};
-  if (const auto &directiveName{
-          std::get<std::optional<dirNameModifier>>(x.v.t)}) {
-    auto search{dirNameModifierMap.find(*directiveName)};
-    if (search == dirNameModifierMap.end() ||
-        !search->second.test(GetContext().directive)) {
-      context_
-          .Say(GetContext().clauseSource,
-              "Unmatched directive name modifier %s on the IF clause"_err_en_US,
-              parser::ToUpperCaseLetters(
-                  parser::OmpIfClause::EnumToString(*directiveName)))
-          .Attach(
-              GetContext().directiveSource, "Cannot apply to directive"_en_US);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+
+  auto isConstituent{[](llvm::omp::Directive dir, llvm::omp::Directive part) {
+    using namespace llvm::omp;
+    llvm::ArrayRef<Directive> dirLeafs{getLeafConstructsOrSelf(dir)};
+    llvm::ArrayRef<Directive> partLeafs{getLeafConstructsOrSelf(part)};
+    // Maybe it's sufficient to check if every leaf of `part` is also a leaf
+    // of `dir`, but to be safe check if `partLeafs` is a sub-sequence of
+    // `dirLeafs`.
+    size_t dirSize{dirLeafs.size()}, partSize{partLeafs.size()};
+    // Find the first leaf from `part` in `dir`.
+    if (auto first = llvm::find(dirLeafs, partLeafs.front());
+        first != dirLeafs.end()) {
+      // A leaf can only appear once in a compound directive, so if `part`
+      // is a subsequence of `dir`, it must start here.
+      size_t firstPos{
+          static_cast<size_t>(std::distance(dirLeafs.begin(), first))};
+      llvm::ArrayRef<Directive> subSeq{
+          first, std::min<size_t>(dirSize - firstPos, partSize)};
+      return subSeq == partLeafs;
+    }
+    return false;
+  }};
+
+  if (OmpVerifyModifiers(
+          x.v, llvm::omp::OMPC_if, GetContext().clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    if (auto *dnm{OmpGetUniqueModifier<parser::OmpDirectiveNameModifier>(
+            modifiers)}) {
+      llvm::omp::Directive sub{dnm->v};
+      std::string subName{parser::ToUpperCaseLetters(
+          llvm::omp::getOpenMPDirectiveName(sub).str())};
+      std::string dirName{parser::ToUpperCaseLetters(
+          llvm::omp::getOpenMPDirectiveName(dir).str())};
+
+      parser::CharBlock modifierSource{OmpGetModifierSource(modifiers, dnm)};
+      auto desc{OmpGetDescriptor<parser::OmpDirectiveNameModifier>()};
+      std::string modName{desc.name.str()};
+
+      if (!isConstituent(dir, sub)) {
+        context_
+            .Say(modifierSource,
+                "%s is not a constituent of the %s directive"_err_en_US,
+                subName, dirName)
+            .Attach(GetContext().directiveSource,
+                "Cannot apply to directive"_en_US);
+      } else {
+        static llvm::omp::Directive valid45[]{
+            llvm::omp::OMPD_cancel, //
+            llvm::omp::OMPD_parallel, //
+            /* OMP 5.0+ also allows OMPD_simd */
+            llvm::omp::OMPD_target, //
+            llvm::omp::OMPD_target_data, //
+            llvm::omp::OMPD_target_enter_data, //
+            llvm::omp::OMPD_target_exit_data, //
+            llvm::omp::OMPD_target_update, //
+            llvm::omp::OMPD_task, //
+            llvm::omp::OMPD_taskloop, //
+            /* OMP 5.2+ also allows OMPD_teams */
+        };
+        if (version < 50 && sub == llvm::omp::OMPD_simd) {
+          context_.Say(modifierSource,
+              "%s is not allowed as '%s' in %s, %s"_warn_en_US, subName,
+              modName, ThisVersion(version), TryVersion(50));
+        } else if (version < 52 && sub == llvm::omp::OMPD_teams) {
+          context_.Say(modifierSource,
+              "%s is not allowed as '%s' in %s, %s"_warn_en_US, subName,
+              modName, ThisVersion(version), TryVersion(52));
+        } else if (!llvm::is_contained(valid45, sub) &&
+            sub != llvm::omp::OMPD_simd && sub != llvm::omp::OMPD_teams) {
+          context_.Say(modifierSource,
+              "%s is not allowed as '%s' in %s"_err_en_US, subName, modName,
+              ThisVersion(version));
+        }
+      }
     }
   }
 }
@@ -3468,16 +3620,95 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
 void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
 
+  parser::CharBlock source{GetContext().clauseSource};
   // 2.7 Loop Construct Restriction
   if ((llvm::omp::allDoSet | llvm::omp::allSimdSet)
           .test(GetContext().directive)) {
     if (std::holds_alternative<parser::OmpLinearClause::WithModifier>(x.v.u)) {
-      context_.Say(GetContext().clauseSource,
+      context_.Say(source,
           "A modifier may not be specified in a LINEAR clause "
           "on the %s directive"_err_en_US,
           ContextDirectiveAsFortran());
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
     }
   }
+
+  auto checkForValidLinearClause_01 = [&](const parser::Name &name,
+                                          bool is_ref) {
+    std::string listItemName{name.ToString()};
+    if (!is_ref && !name.symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      context_.Say(source,
+          "The list item `%s` specified with other than linear-modifier `REF` must be of type INTEGER"_err_en_US,
+          listItemName);
+    }
+    if (GetContext().directive == llvm::omp::Directive::OMPD_declare_simd &&
+        !IsDummy(*name.symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          listItemName);
+    }
+    if (IsPointer(*name.symbol) ||
+        name.symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          listItemName);
+    }
+    if (FindCommonBlockContaining(*name.symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          listItemName);
+    }
+  };
+
+  auto checkForValidLinearClause_02 = [&](const parser::Name &name,
+                                          const parser::OmpLinearModifier::Value
+                                              &modifierValue) {
+    std::string listItemName{name.ToString()};
+    checkForValidLinearClause_01(
+        name, (modifierValue == parser::OmpLinearModifier::Value::Ref));
+    if (modifierValue != parser::OmpLinearModifier::Value::Val &&
+        IsDummy(*name.symbol) && IsValue(*name.symbol)) {
+      context_.Say(source,
+          "The list item `%s` specified with the linear-modifier `REF` or `UVAL` must be a dummy argument without `VALUE` attribute"_err_en_US,
+          listItemName);
+    }
+    if (modifierValue == parser::OmpLinearModifier::Value::Ref &&
+        !(IsAllocatable(*name.symbol) || IsAssumedShape(*name.symbol) ||
+            IsPolymorphic(*name.symbol))) {
+      context_.Say(source,
+          "The list item `%s` specified with the linear-modifier `REF` must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+          listItemName);
+    }
+  };
+
+  // OpenMP 5.2: Linear clause Restrictions
+  common::visit(
+      common::visitors{
+          [&](const parser::OmpLinearClause::WithoutModifier &withoutModifier) {
+            for (const auto &name : withoutModifier.names) {
+              if (name.symbol) {
+                checkForValidLinearClause_01(name, false);
+              }
+            }
+          },
+          [&](const parser::OmpLinearClause::WithModifier &withModifier) {
+            for (const auto &name : withModifier.names) {
+              if (name.symbol) {
+                checkForValidLinearClause_02(name, withModifier.modifier.v);
+              }
+            }
+          },
+      },
+      x.v.u);
 }
 
 void OmpStructureChecker::CheckAllowedMapTypes(
@@ -3605,19 +3836,25 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Schedule &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Device &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_device);
-  const parser::OmpDeviceClause &deviceClause = x.v;
-  const auto &device{std::get<1>(deviceClause.t)};
+  const parser::OmpDeviceClause &deviceClause{x.v};
+  const auto &device{std::get<parser::ScalarIntExpr>(deviceClause.t)};
   RequiresPositiveParameter(
       llvm::omp::Clause::OMPC_device, device, "device expression");
-  std::optional<parser::OmpDeviceClause::DeviceModifier> modifier =
-      std::get<0>(deviceClause.t);
-  if (modifier &&
-      *modifier == parser::OmpDeviceClause::DeviceModifier::Ancestor) {
-    if (GetContext().directive != llvm::omp::OMPD_target) {
-      context_.Say(GetContext().clauseSource,
-          "The ANCESTOR device-modifier must not appear on the DEVICE clause on"
-          " any directive other than the TARGET construct. Found on %s construct."_err_en_US,
-          parser::ToUpperCaseLetters(getDirectiveName(GetContext().directive)));
+  llvm::omp::Directive dir{GetContext().directive};
+
+  if (OmpVerifyModifiers(deviceClause, llvm::omp::OMPC_device,
+          GetContext().clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(deviceClause)};
+
+    if (auto *deviceMod{
+            OmpGetUniqueModifier<parser::OmpDeviceModifier>(modifiers)}) {
+      using Value = parser::OmpDeviceModifier::Value;
+      if (dir != llvm::omp::OMPD_target && deviceMod->v == Value::Ancestor) {
+        auto name{OmpGetDescriptor<parser::OmpDeviceModifier>().name};
+        context_.Say(OmpGetModifierSource(modifiers, deviceMod),
+            "The ANCESTOR %s must not appear on the DEVICE clause on any directive other than the TARGET construct. Found on %s construct."_err_en_US,
+            name.str(), parser::ToUpperCaseLetters(getDirectiveName(dir)));
+      }
     }
   }
 }
@@ -3695,18 +3932,13 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
         }
       }
     }
-    if (std::get<std::optional<parser::OmpIterator>>(taskDep->t)) {
-      unsigned allowedInVersion{50};
-      if (version < allowedInVersion) {
-        context_.Say(GetContext().clauseSource,
-            "Iterator modifiers are not supported in %s, %s"_warn_en_US,
-            ThisVersion(version), TryVersion(allowedInVersion));
-      } else {
+    if (OmpVerifyModifiers(*taskDep, llvm::omp::OMPC_depend,
+            GetContext().clauseSource, context_)) {
+      auto &modifiers{OmpGetModifiers(*taskDep)};
+      if (OmpGetUniqueModifier<parser::OmpIterator>(modifiers)) {
         if (dir == llvm::omp::OMPD_depobj) {
           context_.Say(GetContext().clauseSource,
-              "An iterator-modifier may specify multiple locators, "
-              "a DEPEND clause on a DEPOBJ construct must only specify "
-              "one locator"_warn_en_US);
+              "An iterator-modifier may specify multiple locators, a DEPEND clause on a DEPOBJ construct must only specify one locator"_warn_en_US);
         }
       }
     }
@@ -3803,9 +4035,7 @@ void OmpStructureChecker::CheckDoacross(const parser::OmpDoacross &doa) {
 void OmpStructureChecker::CheckCopyingPolymorphicAllocatable(
     SymbolSourceMap &symbols, const llvm::omp::Clause clause) {
   if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-    for (auto it{symbols.begin()}; it != symbols.end(); ++it) {
-      const auto *symbol{it->first};
-      const auto source{it->second};
+    for (auto &[symbol, source] : symbols) {
       if (IsPolymorphicAllocatable(*symbol)) {
         context_.Warn(common::UsageWarning::Portability, source,
             "If a polymorphic variable with allocatable attribute '%s' is in %s clause, the behavior is unspecified"_port_en_US,
@@ -3818,11 +4048,11 @@ void OmpStructureChecker::CheckCopyingPolymorphicAllocatable(
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_copyprivate);
-  CheckIntentInPointer(x.v, llvm::omp::Clause::OMPC_copyprivate);
-  SymbolSourceMap currSymbols;
-  GetSymbolsInObjectList(x.v, currSymbols);
+  SymbolSourceMap symbols;
+  GetSymbolsInObjectList(x.v, symbols);
+  CheckIntentInPointer(symbols, llvm::omp::Clause::OMPC_copyprivate);
   CheckCopyingPolymorphicAllocatable(
-      currSymbols, llvm::omp::Clause::OMPC_copyprivate);
+      symbols, llvm::omp::Clause::OMPC_copyprivate);
   if (GetContext().directive == llvm::omp::Directive::OMPD_single) {
     context_.Say(GetContext().clauseSource,
         "%s clause is not allowed on the OMP %s directive,"
@@ -3859,20 +4089,8 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
   CheckPrivateSymbolsInOuterCxt(
       currSymbols, dirClauseTriple, llvm::omp::Clause::OMPC_lastprivate);
 
-  using LastprivateModifier = parser::OmpLastprivateClause::LastprivateModifier;
-  const auto &maybeMod{std::get<std::optional<LastprivateModifier>>(x.v.t)};
-  if (maybeMod) {
-    unsigned version{context_.langOptions().OpenMPVersion};
-    unsigned allowedInVersion = 50;
-    if (version < allowedInVersion) {
-      std::string thisVersion{
-          std::to_string(version / 10) + "." + std::to_string(version % 10)};
-      context_.Say(GetContext().clauseSource,
-          "LASTPRIVATE clause with CONDITIONAL modifier is not "
-          "allowed in %s, %s"_err_en_US,
-          ThisVersion(version), TryVersion(allowedInVersion));
-    }
-  }
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_lastprivate, GetContext().clauseSource, context_);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyin &x) {
@@ -3884,29 +4102,28 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Copyin &x) {
       currSymbols, llvm::omp::Clause::OMPC_copyin);
 }
 
-void OmpStructureChecker::CheckStructureElement(
-    const parser::OmpObjectList &ompObjectList,
-    const llvm::omp::Clause clause) {
-  for (const auto &ompObject : ompObjectList.v) {
+void OmpStructureChecker::CheckStructureComponent(
+    const parser::OmpObjectList &objects, llvm::omp::Clause clauseId) {
+  auto CheckComponent{[&](const parser::Designator &designator) {
+    if (auto *dataRef{std::get_if<parser::DataRef>(&designator.u)}) {
+      if (!IsDataRefTypeParamInquiry(dataRef)) {
+        if (auto *comp{parser::Unwrap<parser::StructureComponent>(*dataRef)}) {
+          context_.Say(comp->component.source,
+              "A variable that is part of another variable cannot appear on the %s clause"_err_en_US,
+              parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
+        }
+      }
+    }
+  }};
+
+  for (const auto &object : objects.v) {
     common::visit(
         common::visitors{
-            [&](const parser::Designator &designator) {
-              if (std::get_if<parser::DataRef>(&designator.u)) {
-                if (parser::Unwrap<parser::StructureComponent>(ompObject)) {
-                  context_.Say(GetContext().clauseSource,
-                      "A variable that is part of another variable "
-                      "(structure element) cannot appear on the %s "
-                      "%s clause"_err_en_US,
-                      ContextDirectiveAsFortran(),
-                      parser::ToUpperCaseLetters(getClauseName(clause).str()));
-                }
-              }
-            },
+            CheckComponent,
             [&](const parser::Name &name) {},
         },
-        ompObject.u);
+        object.u);
   }
-  return;
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Update &x) {
@@ -3948,7 +4165,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Update &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
-  CheckStructureElement(x.v, llvm::omp::Clause::OMPC_use_device_ptr);
+  CheckStructureComponent(x.v, llvm::omp::Clause::OMPC_use_device_ptr);
   CheckAllowedClause(llvm::omp::Clause::OMPC_use_device_ptr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
@@ -3977,7 +4194,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDeviceAddr &x) {
-  CheckStructureElement(x.v, llvm::omp::Clause::OMPC_use_device_addr);
+  CheckStructureComponent(x.v, llvm::omp::Clause::OMPC_use_device_addr);
   CheckAllowedClause(llvm::omp::Clause::OMPC_use_device_addr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
@@ -4058,10 +4275,10 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Enter &x) {
   const parser::OmpObjectList &objList{x.v};
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(objList, symbols);
-  for (const auto &[sym, source] : symbols) {
-    if (!IsExtendedListItem(*sym)) {
-      context_.SayWithDecl(*sym, source,
-          "'%s' must be a variable or a procedure"_err_en_US, sym->name());
+  for (const auto &[symbol, source] : symbols) {
+    if (!IsExtendedListItem(*symbol)) {
+      context_.SayWithDecl(*symbol, source,
+          "'%s' must be a variable or a procedure"_err_en_US, symbol->name());
     }
   }
 }
@@ -4083,10 +4300,10 @@ void OmpStructureChecker::Enter(const parser::OmpClause::From &x) {
   const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(objList, symbols);
-  for (const auto &[sym, source] : symbols) {
-    if (!IsVariableListItem(*sym)) {
+  for (const auto &[symbol, source] : symbols) {
+    if (!IsVariableListItem(*symbol)) {
       context_.SayWithDecl(
-          *sym, source, "'%s' must be a variable"_err_en_US, sym->name());
+          *symbol, source, "'%s' must be a variable"_err_en_US, symbol->name());
     }
   }
 
@@ -4128,10 +4345,10 @@ void OmpStructureChecker::Enter(const parser::OmpClause::To &x) {
   const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(objList, symbols);
-  for (const auto &[sym, source] : symbols) {
-    if (!IsVariableListItem(*sym)) {
+  for (const auto &[symbol, source] : symbols) {
+    if (!IsVariableListItem(*symbol)) {
       context_.SayWithDecl(
-          *sym, source, "'%s' must be a variable"_err_en_US, sym->name());
+          *symbol, source, "'%s' must be a variable"_err_en_US, symbol->name());
     }
   }
 
@@ -4153,6 +4370,26 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive);
 }
 
+const Symbol *OmpStructureChecker::GetObjectSymbol(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return &name->symbol->GetUltimate();
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return &GetLastName(*desg).symbol->GetUltimate();
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -4206,15 +4443,6 @@ void OmpStructureChecker::CheckArraySection(
                       "DEPEND "
                       "clause"_err_en_US);
                 }
-                const auto stride{GetIntValue(strideExpr)};
-                if ((stride && stride != 1)) {
-                  context_.Say(GetContext().clauseSource,
-                      "A list item that appears in a REDUCTION clause"
-                      " should have a contiguous storage array "
-                      "section."_err_en_US,
-                      ContextDirectiveAsFortran());
-                  break;
-                }
               }
             }
           }
@@ -4225,16 +4453,23 @@ void OmpStructureChecker::CheckArraySection(
 }
 
 void OmpStructureChecker::CheckIntentInPointer(
-    const parser::OmpObjectList &objectList, const llvm::omp::Clause clause) {
-  SymbolSourceMap symbols;
-  GetSymbolsInObjectList(objectList, symbols);
-  for (auto it{symbols.begin()}; it != symbols.end(); ++it) {
-    const auto *symbol{it->first};
-    const auto source{it->second};
+    SymbolSourceMap &symbols, llvm::omp::Clause clauseId) {
+  for (auto &[symbol, source] : symbols) {
     if (IsPointer(*symbol) && IsIntentIn(*symbol)) {
       context_.Say(source,
-          "Pointer '%s' with the INTENT(IN) attribute may not appear "
-          "in a %s clause"_err_en_US,
+          "Pointer '%s' with the INTENT(IN) attribute may not appear in a %s clause"_err_en_US,
+          symbol->name(),
+          parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckProcedurePointer(
+    SymbolSourceMap &symbols, llvm::omp::Clause clause) {
+  for (const auto &[symbol, source] : symbols) {
+    if (IsProcedurePointer(*symbol)) {
+      context_.Say(source,
+          "Procedure pointer '%s' may not appear in a %s clause"_err_en_US,
           symbol->name(),
           parser::ToUpperCaseLetters(getClauseName(clause).str()));
     }
@@ -4261,9 +4496,7 @@ void OmpStructureChecker::GetSymbolsInObjectList(
 
 void OmpStructureChecker::CheckDefinableObjects(
     SymbolSourceMap &symbols, const llvm::omp::Clause clause) {
-  for (auto it{symbols.begin()}; it != symbols.end(); ++it) {
-    const auto *symbol{it->first};
-    const auto source{it->second};
+  for (auto &[symbol, source] : symbols) {
     if (auto msg{WhyNotDefinable(source, context_.FindScope(source),
             DefinabilityFlags{}, *symbol)}) {
       context_
@@ -4295,9 +4528,7 @@ void OmpStructureChecker::CheckPrivateSymbolsInOuterCxt(
       }
 
       // Check if the symbols in current context are private in outer context
-      for (auto iter{currSymbols.begin()}; iter != currSymbols.end(); ++iter) {
-        const auto *symbol{iter->first};
-        const auto source{iter->second};
+      for (auto &[symbol, source] : currSymbols) {
         if (enclosingSymbols.find(symbol) != enclosingSymbols.end()) {
           context_.Say(source,
               "%s variable '%s' is PRIVATE in outer context"_err_en_US,
@@ -4413,7 +4644,7 @@ void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
-        "Reference to %s must be a contiguous object"_err_en_US,
+        "Reference to '%s' must be a contiguous object"_err_en_US,
         name->ToString());
   }
 }
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 4ce52bebd5a73..89af46d9171ad 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -146,13 +146,14 @@ class OmpStructureChecker
   bool CheckAllowedClause(llvmOmpClause clause);
   bool IsVariableListItem(const Symbol &sym);
   bool IsExtendedListItem(const Symbol &sym);
+  bool IsCommonBlock(const Symbol &sym);
   std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
       const std::string &clauseName);
   void CheckMultListItems();
-  void CheckStructureElement(const parser::OmpObjectList &ompObjectList,
-      const llvm::omp::Clause clause);
+  void CheckStructureComponent(
+      const parser::OmpObjectList &objects, llvm::omp::Clause clauseId);
   bool HasInvalidWorksharingNesting(
       const parser::CharBlock &, const OmpDirectiveSet &);
   bool IsCloselyNestedRegion(const OmpDirectiveSet &set);
@@ -171,6 +172,9 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
+  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+  std::optional<parser::CharBlock> GetObjectSource(
+      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
@@ -182,8 +186,8 @@ class OmpStructureChecker
       const parser::OmpObjectList &objList);
   void CheckSymbolNames(
       const parser::CharBlock &source, const parser::OmpObjectList &objList);
-  void CheckIntentInPointer(
-      const parser::OmpObjectList &, const llvm::omp::Clause);
+  void CheckIntentInPointer(SymbolSourceMap &, const llvm::omp::Clause);
+  void CheckProcedurePointer(SymbolSourceMap &, const llvm::omp::Clause);
   void GetSymbolsInObjectList(const parser::OmpObjectList &, SymbolSourceMap &);
   void CheckDefinableObjects(SymbolSourceMap &, const llvm::omp::Clause);
   void CheckCopyingPolymorphicAllocatable(
@@ -209,6 +213,7 @@ class OmpStructureChecker
   void CheckAtomicCaptureStmt(const parser::AssignmentStmt &);
   void CheckAtomicWriteStmt(const parser::AssignmentStmt &);
   void CheckAtomicCaptureConstruct(const parser::OmpAtomicCapture &);
+  void CheckAtomicCompareConstruct(const parser::OmpAtomicCompare &);
   void CheckAtomicConstructStructure(const parser::OpenMPAtomicConstruct &);
   void CheckDistLinear(const parser::OpenMPLoopConstruct &x);
   void CheckSIMDNest(const parser::OpenMPConstruct &x);
@@ -219,6 +224,8 @@ class OmpStructureChecker
   void CheckCancellationNest(
       const parser::CharBlock &source, const parser::OmpCancelType::Type &type);
   std::int64_t GetOrdCollapseLevel(const parser::OpenMPLoopConstruct &x);
+  void CheckReductionObjects(
+      const parser::OmpObjectList &objects, llvm::omp::Clause clauseId);
   bool CheckReductionOperators(const parser::OmpClause::Reduction &);
   bool CheckIntrinsicOperator(
       const parser::DefinedOperator::IntrinsicOperator &);
@@ -231,8 +238,6 @@ class OmpStructureChecker
   void ChecksOnOrderedAsStandalone();
   void CheckOrderedDependClause(std::optional<std::int64_t> orderedValue);
   void CheckReductionArraySection(const parser::OmpObjectList &ompObjectList);
-  void CheckIntentInPointerAndDefinable(
-      const parser::OmpObjectList &, const llvm::omp::Clause);
   void CheckArraySection(const parser::ArrayElement &arrayElement,
       const parser::Name &name, const llvm::omp::Clause clause);
   void CheckSharedBindingInOuterContext(
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index 028633813a91b..94640fa30baa5 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -11,7 +11,7 @@
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/type.h"
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
@@ -339,8 +339,12 @@ auto ComputeOffsetsHelper::GetSizeAndAlignment(
     const auto *derived{evaluate::GetDerivedTypeSpec(dyType)};
     int lenParams{derived ? CountLenParameters(*derived) : 0};
     bool needAddendum{derived || (dyType && dyType->IsUnlimitedPolymorphic())};
-    std::size_t size{runtime::Descriptor::SizeInBytes(
+
+    // FIXME: Get descriptor size from targetCharacteristics instead
+    // overapproximation
+    std::size_t size{runtime::MaxDescriptorSizeInBytes(
         symbol.Rank(), needAddendum, lenParams)};
+
     return {size, targetCharacteristics.descriptorAlignment()};
   }
   if (IsProcedurePointer(symbol)) {
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index ead9982112678..b9be586f4d772 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -392,7 +392,8 @@ void ExpressionAnalyzer::CheckSubscripts(ArrayRef &ref) {
     auto dimUB{ToInt64(ub[dim])};
     if (dimUB && dimLB && *dimUB < *dimLB) {
       AttachDeclaration(
-          Say("Empty array dimension %d cannot be subscripted as an element or non-empty array section"_err_en_US,
+          Warn(common::UsageWarning::SubscriptedEmptyArray,
+              "Empty array dimension %d should not be subscripted as an element or non-empty array section"_err_en_US,
               dim + 1),
           arraySymbol);
       break;
diff --git a/flang/lib/Semantics/openmp-modifiers.cpp b/flang/lib/Semantics/openmp-modifiers.cpp
index 1fd2358aa594e..f8f81e6c6ffa1 100644
--- a/flang/lib/Semantics/openmp-modifiers.cpp
+++ b/flang/lib/Semantics/openmp-modifiers.cpp
@@ -74,6 +74,72 @@ unsigned OmpModifierDescriptor::since(llvm::omp::Clause id) const {
 // Note: The intent for these functions is to have them be automatically-
 // generated in the future.
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlignment>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"alignment",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique, OmpProperty::Ultimate, OmpProperty::Post}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_aligned}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpAlignModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"align-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_allocate}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &
+OmpGetDescriptor<parser::OmpAllocatorComplexModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"allocator-complex-modifier",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_allocate}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &
+OmpGetDescriptor<parser::OmpAllocatorSimpleModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"allocator-simple-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Exclusive, OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_allocate}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpChunkModifier>() {
   static const OmpModifierDescriptor desc{
@@ -108,6 +174,39 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDependenceType>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpDeviceModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"device-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_device}},
+      },
+  };
+  return desc;
+}
+
+template <>
+const OmpModifierDescriptor &
+OmpGetDescriptor<parser::OmpDirectiveNameModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"directive-name-modifier",
+      /*props=*/
+      {
+          {45, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {45, {Clause::OMPC_if}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpExpectation>() {
   static const OmpModifierDescriptor desc{
@@ -143,6 +242,23 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpIterator>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &
+OmpGetDescriptor<parser::OmpLastprivateModifier>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"lastprivate-modifier",
+      /*props=*/
+      {
+          {50, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {50, {Clause::OMPC_lastprivate}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpLinearModifier>() {
   static const OmpModifierDescriptor desc{
@@ -239,6 +355,22 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpOrderingModifier>() {
   return desc;
 }
 
+template <>
+const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpPrescriptiveness>() {
+  static const OmpModifierDescriptor desc{
+      /*name=*/"prescriptiveness",
+      /*props=*/
+      {
+          {51, {OmpProperty::Unique}},
+      },
+      /*clauses=*/
+      {
+          {51, {Clause::OMPC_grainsize, Clause::OMPC_num_tasks}},
+      },
+  };
+  return desc;
+}
+
 template <>
 const OmpModifierDescriptor &
 OmpGetDescriptor<parser::OmpReductionIdentifier>() {
@@ -281,11 +413,12 @@ const OmpModifierDescriptor &OmpGetDescriptor<parser::OmpTaskDependenceType>() {
       /*name=*/"task-dependence-type",
       /*props=*/
       {
-          {52, {OmpProperty::Required, OmpProperty::Ultimate}},
+          {45, {OmpProperty::Required, OmpProperty::Ultimate}},
       },
       /*clauses=*/
       {
-          {52, {Clause::OMPC_depend, Clause::OMPC_update}},
+          {45, {Clause::OMPC_depend}},
+          {51, {Clause::OMPC_depend, Clause::OMPC_update}},
       },
   };
   return desc;
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 0c3708b3fd29b..80a086acebba2 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1895,22 +1895,29 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   }
 
   const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-    // go through all the nested do-loops and resolve index variables
-    const parser::Name *iv{GetLoopIndex(*loop)};
-    if (iv) {
-      if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-        symbol->set(Symbol::Flag::OmpPreDetermined);
-        iv->symbol = symbol; // adjust the symbol within region
-        AddToContextObjectWithDSA(*symbol, ivDSA);
-      }
+  if (outer.has_value()) {
+    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
+      // go through all the nested do-loops and resolve index variables
+      const parser::Name *iv{GetLoopIndex(*loop)};
+      if (iv) {
+        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+          symbol->set(Symbol::Flag::OmpPreDetermined);
+          iv->symbol = symbol; // adjust the symbol within region
+          AddToContextObjectWithDSA(*symbol, ivDSA);
+        }
 
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+      }
     }
+    CheckAssocLoopLevel(level, GetAssociatedClause());
+  } else {
+    context_.Say(GetContext().directiveSource,
+        "A DO loop must follow the %s directive"_err_en_US,
+        parser::ToUpperCaseLetters(
+            llvm::omp::getOpenMPDirectiveName(GetContext().directive).str()));
   }
-  CheckAssocLoopLevel(level, GetAssociatedClause());
 }
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
@@ -2023,16 +2030,16 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
 void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
   if (!dirContext_.empty()) {
     switch (x.v) {
-    case parser::OmpDefaultClause::Type::Private:
+    case parser::OmpDefaultClause::DataSharingAttribute::Private:
       SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
       break;
-    case parser::OmpDefaultClause::Type::Firstprivate:
+    case parser::OmpDefaultClause::DataSharingAttribute::Firstprivate:
       SetContextDefaultDSA(Symbol::Flag::OmpFirstPrivate);
       break;
-    case parser::OmpDefaultClause::Type::Shared:
+    case parser::OmpDefaultClause::DataSharingAttribute::Shared:
       SetContextDefaultDSA(Symbol::Flag::OmpShared);
       break;
-    case parser::OmpDefaultClause::Type::None:
+    case parser::OmpDefaultClause::DataSharingAttribute::None:
       SetContextDefaultDSA(Symbol::Flag::OmpNone);
       break;
     }
@@ -2097,17 +2104,16 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
           std::get<parser::OmpObjectList>(alloc->v.t),
           std::get<parser::Statement<parser::AllocateStmt>>(x.t).statement);
 
-      const auto &allocMod{
-          std::get<std::optional<parser::OmpAllocateClause::AllocateModifier>>(
-              alloc->v.t)};
+      auto &modifiers{OmpGetModifiers(alloc->v)};
+      bool hasAllocator{
+          OmpGetUniqueModifier<parser::OmpAllocatorSimpleModifier>(modifiers) ||
+          OmpGetUniqueModifier<parser::OmpAllocatorComplexModifier>(modifiers)};
+
       // TODO: As with allocate directive, exclude the case when a requires
       //       directive with the dynamic_allocators clause is present in
       //       the same compilation unit (OMP5.0 2.11.3).
       if (IsNestedInDirective(llvm::omp::Directive::OMPD_target) &&
-          (!allocMod.has_value() ||
-              std::holds_alternative<
-                  parser::OmpAllocateClause::AllocateModifier::Align>(
-                  allocMod->u))) {
+          !hasAllocator) {
         context_.Say(x.source,
             "ALLOCATORS directives that appear in a TARGET region "
             "must specify an allocator"_err_en_US);
@@ -2119,7 +2125,7 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
 
 static bool IsPrivatizable(const Symbol *sym) {
   auto *misc{sym->detailsIf<MiscDetails>()};
-  return !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
+  return IsVariableName(*sym) && !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
       !semantics::IsAssumedSizeArray(
           *sym) && /* OpenMP 5.2, 5.1.1: Assumed-size arrays are shared*/
       !sym->owner().IsDerivedType() &&
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index cdd2de541c673..fbfaae9a88064 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -59,6 +59,11 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     )
 endif()
 
+# function checks
+find_package(Backtrace)
+set(HAVE_BACKTRACE ${Backtrace_FOUND})
+set(BACKTRACE_HEADER ${Backtrace_HEADER})
+
 include(CheckCXXSymbolExists)
 include(CheckCXXSourceCompiles)
 check_cxx_symbol_exists(strerror_r string.h HAVE_STRERROR_R)
diff --git a/flang/runtime/CUDA/descriptor.cpp b/flang/runtime/CUDA/descriptor.cpp
index 7ce1429cd94d4..58bc0dbed6bab 100644
--- a/flang/runtime/CUDA/descriptor.cpp
+++ b/flang/runtime/CUDA/descriptor.cpp
@@ -10,6 +10,7 @@
 #include "../terminator.h"
 #include "flang/Runtime/CUDA/allocator.h"
 #include "flang/Runtime/CUDA/common.h"
+#include "flang/Runtime/descriptor.h"
 
 #include "cuda_runtime.h"
 
diff --git a/flang/runtime/array-constructor.cpp b/flang/runtime/array-constructor.cpp
index 72e08feff7fd1..c6953167f5fb2 100644
--- a/flang/runtime/array-constructor.cpp
+++ b/flang/runtime/array-constructor.cpp
@@ -92,13 +92,10 @@ extern "C" {
 RT_EXT_API_GROUP_BEGIN
 
 void RTDEF(InitArrayConstructorVector)(ArrayConstructorVector &vector,
-    Descriptor &to, bool useValueLengthParameters, int vectorClassSize,
-    const char *sourceFile, int sourceLine) {
+    Descriptor &to, bool useValueLengthParameters, const char *sourceFile,
+    int sourceLine) {
   Terminator terminator{vector.sourceFile, vector.sourceLine};
-  RUNTIME_CHECK(terminator,
-      to.rank() == 1 &&
-          sizeof(ArrayConstructorVector) <=
-              static_cast<std::size_t>(vectorClassSize));
+  RUNTIME_CHECK(terminator, to.rank() == 1);
   SubscriptValue actualAllocationSize{
       to.IsAllocated() ? static_cast<SubscriptValue>(to.Elements()) : 0};
   (void)new (&vector) ArrayConstructorVector{to, /*nextValuePosition=*/0,
diff --git a/flang/runtime/config.h.cmake b/flang/runtime/config.h.cmake
index 0a1d1394b9bc4..a2271be77b8c6 100644
--- a/flang/runtime/config.h.cmake
+++ b/flang/runtime/config.h.cmake
@@ -8,4 +8,9 @@
    don't. */
 #cmakedefine01 HAVE_DECL_STRERROR_S
 
+/* Define to 1 if you have the `backtrace' function. */
+#cmakedefine HAVE_BACKTRACE ${HAVE_BACKTRACE}
+
+#define BACKTRACE_HEADER <${BACKTRACE_HEADER}>
+
 #endif
diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index 2cee35e23f31a..b5725a90ba82a 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -976,14 +976,10 @@ static RT_API_ATTRS bool EditListDirectedCharacterInput(
     return false;
   }
   // Undelimited list-directed character input: stop at a value separator
-  // or the end of the current record.  Subtlety: the "remaining" count
-  // here is a dummy that's used to avoid the interpretation of separators
-  // in NextInField.
-  Fortran::common::optional<int> remaining{length > 0 ? maxUTF8Bytes : 0};
-  while (Fortran::common::optional<char32_t> next{
-      io.NextInField(remaining, edit)}) {
+  // or the end of the current record.
+  while (auto ch{io.GetCurrentChar(byteCount)}) {
     bool isSep{false};
-    switch (*next) {
+    switch (*ch) {
     case ' ':
     case '\t':
     case '/':
@@ -1003,11 +999,17 @@ static RT_API_ATTRS bool EditListDirectedCharacterInput(
       break;
     }
     if (isSep) {
-      remaining = 0;
-    } else {
-      *x++ = *next;
-      remaining = --length > 0 ? maxUTF8Bytes : 0;
+      break;
+    }
+    if (length > 0) {
+      *x++ = *ch;
+      --length;
+    } else if (edit.IsNamelist()) {
+      // GNU compatibility
+      break;
     }
+    io.HandleRelativePosition(byteCount);
+    io.GotChar(byteCount);
   }
   Fortran::runtime::fill_n(x, length, ' ');
   return true;
diff --git a/flang/runtime/environment-default-list.h b/flang/runtime/environment-default-list.h
old mode 100755
new mode 100644
diff --git a/flang/runtime/exceptions.cpp b/flang/runtime/exceptions.cpp
index 8239c556bcea9..993c996c9ce75 100644
--- a/flang/runtime/exceptions.cpp
+++ b/flang/runtime/exceptions.cpp
@@ -11,6 +11,9 @@
 #include "flang/Runtime/exceptions.h"
 #include "terminator.h"
 #include <cfenv>
+#if __x86_64__
+#include <xmmintrin.h>
+#endif
 
 // When not supported, these macro are undefined in cfenv.h,
 // set them to zero in that case.
@@ -78,5 +81,22 @@ uint32_t RTNAME(MapException)(uint32_t excepts) {
 // on some systems, e.g. Solaris, so omit object size comparison for now.
 // TODO: consider femode_t object size comparison once its more mature.
 
+bool RTNAME(GetUnderflowMode)(void) {
+#if __x86_64__
+  // The MXCSR Flush to Zero flag is the negation of the ieee_get_underflow_mode
+  // GRADUAL argument. It affects real computations of kinds 3, 4, and 8.
+  return _MM_GET_FLUSH_ZERO_MODE() == _MM_FLUSH_ZERO_OFF;
+#else
+  return false;
+#endif
+}
+void RTNAME(SetUnderflowMode)(bool flag) {
+#if __x86_64__
+  // The MXCSR Flush to Zero flag is the negation of the ieee_set_underflow_mode
+  // GRADUAL argument. It affects real computations of kinds 3, 4, and 8.
+  _MM_SET_FLUSH_ZERO_MODE(flag ? _MM_FLUSH_ZERO_OFF : _MM_FLUSH_ZERO_ON);
+#endif
+}
+
 } // extern "C"
 } // namespace Fortran::runtime
diff --git a/flang/runtime/format.h b/flang/runtime/format.h
index 5329f2482d3e4..815bf70685e64 100644
--- a/flang/runtime/format.h
+++ b/flang/runtime/format.h
@@ -13,7 +13,7 @@
 
 #include "environment.h"
 #include "io-error.h"
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include "flang/Common/optional.h"
 #include "flang/Decimal/decimal.h"
 #include "flang/Runtime/freestanding-tools.h"
diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp
index f28700ee01581..f8f3877efb20e 100644
--- a/flang/runtime/internal-unit.cpp
+++ b/flang/runtime/internal-unit.cpp
@@ -36,6 +36,8 @@ RT_API_ATTRS InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
   Descriptor &d{descriptor()};
   RUNTIME_CHECK(
       terminator, that.SizeInBytes() <= d.SizeInBytes(maxRank, true, 0));
+  RUNTIME_CHECK(terminator,
+      that.SizeInBytes() <= MaxDescriptorSizeInBytes(maxRank, true, 0));
   new (&d) Descriptor{that};
   d.Check();
   internalIoCharKind = thatType->second;
diff --git a/flang/runtime/non-tbp-dio.h b/flang/runtime/non-tbp-dio.h
index 05038a264ed99..8429d790fea57 100644
--- a/flang/runtime/non-tbp-dio.h
+++ b/flang/runtime/non-tbp-dio.h
@@ -22,7 +22,8 @@
 #ifndef FORTRAN_RUNTIME_NON_TBP_DIO_H_
 #define FORTRAN_RUNTIME_NON_TBP_DIO_H_
 
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
+#include "flang/Common/api-attrs.h"
 #include <cstddef>
 
 namespace Fortran::runtime::typeInfo {
diff --git a/flang/runtime/stop.cpp b/flang/runtime/stop.cpp
index cfb36b4084020..f8457e10566a2 100644
--- a/flang/runtime/stop.cpp
+++ b/flang/runtime/stop.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Runtime/stop.h"
+#include "config.h"
 #include "environment.h"
 #include "file.h"
 #include "io-error.h"
@@ -16,6 +17,10 @@
 #include <cstdio>
 #include <cstdlib>
 
+#ifdef HAVE_BACKTRACE
+#include BACKTRACE_HEADER
+#endif
+
 extern "C" {
 
 static void DescribeIEEESignaledExceptions() {
@@ -152,11 +157,37 @@ void RTNAME(PauseStatementText)(const char *code, std::size_t length) {
   std::exit(status);
 }
 
+static void PrintBacktrace() {
+#ifdef HAVE_BACKTRACE
+  // TODO: Need to parse DWARF information to print function line numbers
+  constexpr int MAX_CALL_STACK{999};
+  void *buffer[MAX_CALL_STACK];
+  int nptrs{backtrace(buffer, MAX_CALL_STACK)};
+
+  if (char **symbols{backtrace_symbols(buffer, nptrs)}) {
+    for (int i = 0; i < nptrs; i++) {
+      Fortran::runtime::Terminator{}.PrintCrashArgs("#%d %s\n", i, symbols[i]);
+    }
+    free(symbols);
+  }
+
+#else
+
+  // TODO: Need to implement the version for other platforms.
+  Fortran::runtime::Terminator{}.PrintCrashArgs("backtrace is not supported.");
+
+#endif
+}
+
 [[noreturn]] void RTNAME(Abort)() {
-  // TODO: Add backtrace call, unless with `-fno-backtrace`.
+#ifdef HAVE_BACKTRACE
+  PrintBacktrace();
+#endif
   std::abort();
 }
 
+void FORTRAN_PROCEDURE_NAME(backtrace)() { PrintBacktrace(); }
+
 [[noreturn]] void RTNAME(ReportFatalUserError)(
     const char *message, const char *source, int line) {
   Fortran::runtime::Terminator{source, line}.Crash(message);
diff --git a/flang/runtime/type-info.h b/flang/runtime/type-info.h
index c3f3595e32ef2..32403b1db5169 100644
--- a/flang/runtime/type-info.h
+++ b/flang/runtime/type-info.h
@@ -13,7 +13,7 @@
 // flang/module/__fortran_type_info.f90.
 
 #include "terminator.h"
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include "flang/Common/bit-population-count.h"
 #include "flang/Common/optional.h"
 #include "flang/Runtime/descriptor.h"
diff --git a/flang/test/Driver/Inputs/config-l.cfg b/flang/test/Driver/Inputs/config-l.cfg
new file mode 100644
index 0000000000000..8cc3abf166adf
--- /dev/null
+++ b/flang/test/Driver/Inputs/config-l.cfg
@@ -0,0 +1,3 @@
+-ffast-math $-lm
+-Wl,--as-needed $-Wl,-Bstatic
+$-lhappy $-Wl,-Bdynamic
diff --git a/flang/test/Driver/config-file.f90 b/flang/test/Driver/config-file.f90
index 70316dd971f36..6991fda9bd483 100644
--- a/flang/test/Driver/config-file.f90
+++ b/flang/test/Driver/config-file.f90
@@ -61,3 +61,29 @@
 ! CHECK-TWO-CONFIGS-NEXT: Configuration file: {{.*}}Inputs{{.}}config2{{.}}config-4.cfg
 ! CHECK-TWO-CONFIGS: -ffp-contract=fast
 ! CHECK-TWO-CONFIGS: -O3
+
+!--- The linker input flags should be moved to the end of input list and appear only when linking.
+! RUN: %flang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg %s -lmylib -Wl,foo.a -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING
+! RUN: %flang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -fopenmp=libomp %s -lmylib -Wl,foo.a -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING-LIBOMP-GOES-AFTER
+! RUN: %flang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING
+! RUN: %flang --target=aarch64-unknown-linux-gnu --config %S/Inputs/config-l.cfg -fopenmp=libomp -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING-OPENMP
+! RUN: %flang --target=x86_64-pc-windows-msvc    --config %S/Inputs/config-l.cfg %s -lmylib -Wl,foo.lib -### 2>&1 | FileCheck %s -check-prefix CHECK-LINKING-MSVC
+! RUN: %flang --target=x86_64-pc-windows-msvc    --config %S/Inputs/config-l.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NOLINKING-MSVC
+! CHECK-LINKING: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-LINKING: "-ffast-math"
+! CHECK-LINKING: "--as-needed" "{{.*}}-{{.*}}.o" "-lmylib" "foo.a" "-lm" "-Bstatic" "-lhappy" "-Bdynamic"
+! CHECK-LINKING-LIBOMP-GOES-AFTER: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-LINKING-LIBOMP-GOES-AFTER: "-ffast-math" {{.*}}"-fopenmp"
+! CHECK-LINKING-LIBOMP-GOES-AFTER: "--as-needed" "{{.*}}-{{.*}}.o" "-lmylib" "foo.a" "-lm" "-Bstatic" "-lhappy" "-Bdynamic" {{.*}}"-lomp"
+! CHECK-NOLINKING: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-NOLINKING: "-ffast-math"
+! CHECK-NOLINKING-NO: "-lm" "-Bstatic" "-lhappy" "-Bdynamic"
+! CHECK-NOLINKING-OPENMP: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-NOLINKING-OPENMP: "-ffast-math" {{.*}}"-fopenmp"
+! CHECK-NOLINKING-OPENMP-NO: "-lm" "-Bstatic" "-lhappy" "-Bdynamic" {{.}}"-lomp"
+! CHECK-LINKING-MSVC: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-LINKING-MSVC: "-ffast-math"
+! CHECK-LINKING-MSVC: "--as-needed" "{{.*}}-{{.*}}.o" "mylib.lib" "foo.lib" "m.lib" "-Bstatic" "happy.lib" "-Bdynamic"
+! CHECK-NOLINKING-MSVC: Configuration file: {{.*}}Inputs{{.}}config-l.cfg
+! CHECK-NOLINKING-MSVC: "-ffast-math"
+! CHECK-NOLINKING-MSVC-NO: "m.lib" "-Bstatic" "happy.lib" "-Bdynamic"
diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index ff2d660952146..55a74ccf40467 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -20,7 +20,6 @@
 ! RUN:     -fversion-loops-for-stride \
 ! RUN:     -flang-experimental-hlfir \
 ! RUN:     -flang-deprecated-no-hlfir \
-! RUN:     -flang-experimental-integer-overflow \
 ! RUN:     -fno-ppc-native-vector-element-order \
 ! RUN:     -fppc-native-vector-element-order \
 ! RUN:     -mllvm -print-before-all \
@@ -52,7 +51,6 @@
 ! CHECK: "-fversion-loops-for-stride"
 ! CHECK: "-flang-experimental-hlfir"
 ! CHECK: "-flang-deprecated-no-hlfir"
-! CHECK: "-flang-experimental-integer-overflow"
 ! CHECK: "-fno-ppc-native-vector-element-order"
 ! CHECK: "-fppc-native-vector-element-order"
 ! CHECK: "-Rpass"
diff --git a/flang/test/Driver/mabi-loongarch.f90 b/flang/test/Driver/mabi-loongarch.f90
new file mode 100644
index 0000000000000..a3a0283bcbb21
--- /dev/null
+++ b/flang/test/Driver/mabi-loongarch.f90
@@ -0,0 +1,10 @@
+! RUN: not %flang -c --target=loongarch64-unknown-linux -mabi=lp64s %s -### 2>&1 | FileCheck --check-prefix=INVALID1 %s
+! RUN: not %flang -c --target=loongarch64-unknown-linux -mabi=lp64f %s -### 2>&1 | FileCheck --check-prefix=INVALID2 %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mabi=lp64d %s -### 2>&1 | FileCheck --check-prefix=ABI %s
+! RUN: %flang -c --target=loongarch64-unknown-linux %s -### 2>&1 | FileCheck --check-prefix=ABI %s
+
+! INVALID1: error: invalid argument '-mabi' not allowed with 'lp64s'
+! INVALID2: error: invalid argument '-mabi' not allowed with 'lp64f'
+
+! ABI: "-target-feature" "+d"
+
diff --git a/flang/test/Driver/mlir-debug-pass-pipeline.f90 b/flang/test/Driver/mlir-debug-pass-pipeline.f90
index ab5ddedf5fc18..edc6f59b0ad7c 100644
--- a/flang/test/Driver/mlir-debug-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-debug-pass-pipeline.f90
@@ -99,11 +99,17 @@
 ! ALL-NEXT:   (S) 0 num-dce'd - Number of operations DCE'd
 ! ALL-NEXT: BoxedProcedurePass
 
-! ALL-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'omp.declare_reduction', 'omp.private']
+! ALL-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'gpu.module', 'omp.declare_reduction', 'omp.private']
 ! ALL-NEXT:   'fir.global' Pipeline
 ! ALL-NEXT:     AbstractResultOpt
 ! ALL-NEXT:   'func.func' Pipeline
 ! ALL-NEXT:     AbstractResultOpt
+! ALL-NEXT:   'gpu.module' Pipeline
+! ALL-NEXT:   Pipeline Collection : ['func.func', 'gpu.func'] 
+! ALL-NEXT:   'func.func' Pipeline 
+! ALL-NEXT:   AbstractResultOpt
+! ALL-NEXT:   'gpu.func' Pipeline 
+! ALL-NEXT:   AbstractResultOpt
 ! ALL-NEXT:   'omp.declare_reduction' Pipeline
 ! ALL-NEXT:     AbstractResultOpt
 ! ALL-NEXT:   'omp.private' Pipeline
diff --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index 7d57135c5fee3..b30affe691b84 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -108,11 +108,17 @@
 ! ALL-NEXT:   (S) 0 num-dce'd - Number of operations DCE'd
 ! ALL-NEXT: BoxedProcedurePass
 
-! ALL-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'omp.declare_reduction', 'omp.private']
+! ALL-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'gpu.module', 'omp.declare_reduction', 'omp.private']
 ! ALL-NEXT:   'fir.global' Pipeline
 ! ALL-NEXT:    AbstractResultOpt
 ! ALL-NEXT:  'func.func' Pipeline
 ! ALL-NEXT:    AbstractResultOpt
+! ALL-NEXT:  'gpu.module' Pipeline
+! ALL-NEXT:   Pipeline Collection : ['func.func', 'gpu.func'] 
+! ALL-NEXT:   'func.func' Pipeline 
+! ALL-NEXT:   AbstractResultOpt
+! ALL-NEXT:   'gpu.func' Pipeline 
+! ALL-NEXT:   AbstractResultOpt
 ! ALL-NEXT:  'omp.declare_reduction' Pipeline
 ! ALL-NEXT:    AbstractResultOpt
 ! ALL-NEXT:  'omp.private' Pipeline
@@ -123,5 +129,7 @@
 ! ALL-NEXT: ExternalNameConversion
 ! ALL-NEXT: TargetRewrite
 ! ALL-NEXT: CompilerGeneratedNamesConversion
+! ALL-NEXT:  'func.func' Pipeline
+! ALL-NEXT:   FunctionAttr
 ! ALL-NEXT: FIRToLLVMLowering
 ! ALL-NOT: LLVMIRLoweringPass
diff --git a/flang/test/Driver/options-loongarch.f90 b/flang/test/Driver/options-loongarch.f90
new file mode 100644
index 0000000000000..717bf6385b06d
--- /dev/null
+++ b/flang/test/Driver/options-loongarch.f90
@@ -0,0 +1,43 @@
+!! This test tests options from clang, which are also supported by flang in LoongArch.
+
+! RUN: %flang -c --target=loongarch64-unknown-linux -mlsx %s -### 2>&1 | FileCheck --check-prefixes=LSX,NOLASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-lsx %s -### 2>&1 | FileCheck --check-prefixes=NOLSX,NOLASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mlasx %s -### 2>&1 | FileCheck --check-prefixes=LSX,LASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-lasx %s -### 2>&1 | FileCheck --check-prefixes=LSX,NOLASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -msimd=none %s -### 2>&1 | FileCheck --check-prefixes=NOLSX,NOLASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -msimd=lsx %s -### 2>&1 | FileCheck --check-prefixes=LSX,NOLASX %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -msimd=lasx %s -### 2>&1 | FileCheck --check-prefixes=LSX,LASX %s
+! RUN: not %flang -c --target=loongarch64-unknown-linux -msimd=supper %s -### 2>&1 | FileCheck --check-prefix=MSIMD-INVALID %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mfrecipe %s -### 2>&1 | FileCheck --check-prefix=FRECIPE %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-frecipe %s -### 2>&1 | FileCheck --check-prefix=NOFRECIPE %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mlam-bh %s -### 2>&1 | FileCheck --check-prefix=LAMBH %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-lam-bh %s -### 2>&1 | FileCheck --check-prefix=NOLAMBH %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mlamcas %s -### 2>&1 | FileCheck --check-prefix=LAMCAS %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-lamcas %s -### 2>&1 | FileCheck --check-prefix=NOLAMCAS %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mld-seq-sa %s -### 2>&1 | FileCheck --check-prefix=LD-SEQ-SA %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-ld-seq-sa %s -### 2>&1 | FileCheck --check-prefix=NOLD-SEQ-SA %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mdiv32 %s -### 2>&1 | FileCheck --check-prefix=DIV32 %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-div32 %s -### 2>&1 | FileCheck --check-prefix=NODIV32 %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mannotate-tablejump %s -### 2>&1 | FileCheck --check-prefix=ANOTATE %s
+! RUN: %flang -c --target=loongarch64-unknown-linux -mno-annotate-tablejump %s -### 2>&1 | FileCheck --check-prefix=NOANOTATE %s
+
+! MSIMD-INVALID: error: invalid argument 'supper' to -msimd=; must be one of: none, lsx, lasx
+! FRECIPE: "-target-feature" "+frecipe"
+! NOFRECIPE-NOT: "-target-feature" "+frecipe"
+! LAMBH: "-target-feature" "+lam-bh"
+! NOLAMBH-NOT: "-target-feature" "+lam-bh"
+! LAMCAS: "-target-feature" "+lamcas"
+! NOLAMCAS-NOT: "-target-feature" "+lamcas"
+! LD-SEQ-SA: "-target-feature" "+ld-seq-sa"
+! NOLD-SEQ-SA-NOT: "-target-feature" "+ld-seq-sa"
+! DIV32: "-target-feature" "+div32"
+! NODIV32-NOT: "-target-feature" "+div32"
+! ANOTATE: "-mllvm" "-loongarch-annotate-tablejump"
+! NOANOTATE-NOT: "-loongarch-annotate-tablejump"
+
+! NOLSX-NOT: "-target-feature" "+lsx"
+! NOLASX-NOT: "-target-feature" "+lasx"
+! LSX-DAG: "-target-feature" "+lsx"
+! LASX-DAG: "-target-feature" "+lasx"
+! NOLSX-NOT: "-target-feature" "+lsx"
+! NOLASX-NOT: "-target-feature" "+lasx"
diff --git a/flang/test/Driver/pp-fixed-form.f90 b/flang/test/Driver/pp-fixed-form.f90
new file mode 100644
index 0000000000000..4695da78763ae
--- /dev/null
+++ b/flang/test/Driver/pp-fixed-form.f90
@@ -0,0 +1,19 @@
+!RUN: %flang -save-temps -### %S/Inputs/free-form-test.f90  2>&1 | FileCheck %s --check-prefix=FREE
+FREE:       "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/free-form-test.f90"
+FREE-NEXT:  "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95" "free-form-test.i"
+
+!RUN: %flang -save-temps -### %S/Inputs/fixed-form-test.f  2>&1 | FileCheck %s --check-prefix=FIXED
+FIXED:      "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/fixed-form-test.f"
+FIXED-NEXT: "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95" "fixed-form-test.i"
+
+!RUN: %flang -save-temps -### -ffree-form %S/Inputs/free-form-test.f90  2>&1 | FileCheck %s --check-prefix=FREE-FLAG
+FREE-FLAG:           "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/free-form-test.f90"
+FREE-FLAG-NEXT:      "-fc1" {{.*}} "-emit-llvm-bc" "-ffree-form"
+FREE-FLAG-NOT:       "-ffixed-form"
+FREE-FLAG-SAME:      "-x" "f95" "free-form-test.i"
+
+!RUN: %flang -save-temps -### -ffixed-form %S/Inputs/fixed-form-test.f  2>&1 | FileCheck %s --check-prefix=FIXED-FLAG
+FIXED-FLAG:          "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/fixed-form-test.f"
+FIXED-FLAG-NEXT:     "-fc1" {{.*}} "-emit-llvm-bc" "-ffixed-form"
+FIXED-FLAG-NOT:      "-ffixed-form"
+FIXED-FLAG-SAME:     "-x" "f95" "fixed-form-test.i"
diff --git a/flang/test/Evaluate/fold-ieee.f90 b/flang/test/Evaluate/fold-ieee.f90
index 536db6481e670..e70d558af1668 100644
--- a/flang/test/Evaluate/fold-ieee.f90
+++ b/flang/test/Evaluate/fold-ieee.f90
@@ -58,7 +58,7 @@ module m
   logical, parameter :: test_sn_all = ieee_support_subnormal()
   logical, parameter :: test_sn_4 = ieee_support_subnormal(1.)
   logical, parameter :: test_sn_8 = ieee_support_subnormal(1.d0)
-  logical, parameter :: test_uc_all = ieee_support_underflow_control()
-  logical, parameter :: test_uc_4 = ieee_support_underflow_control(1.)
-  logical, parameter :: test_uc_8 = ieee_support_underflow_control(1.d0)
+! logical, parameter :: test_uc_all = .not. ieee_support_underflow_control()
+! logical, parameter :: test_uc_4 = ieee_support_underflow_control(1.)
+! logical, parameter :: test_uc_8 = ieee_support_underflow_control(1.d0)
 end
diff --git a/flang/test/Evaluate/folding18.f90 b/flang/test/Evaluate/folding18.f90
index 9110689cf5d63..52aeb6a3532d0 100644
--- a/flang/test/Evaluate/folding18.f90
+++ b/flang/test/Evaluate/folding18.f90
@@ -65,11 +65,12 @@ module m
     .and. ieee_support_subnormal(1.0_8) &
     .and. ieee_support_subnormal(1.0_10) &
     .and. ieee_support_subnormal(1.0_16)
-  logical, parameter :: test_ieee_support_underflow_control = ieee_support_underflow_control() &
-    .and. ieee_support_underflow_control(1.0_2) &
-    .and. ieee_support_underflow_control(1.0_3) &
-    .and. ieee_support_underflow_control(1.0_4) &
-    .and. ieee_support_underflow_control(1.0_8) &
-    .and. ieee_support_underflow_control(1.0_10) &
-    .and. ieee_support_underflow_control(1.0_16)
+! varies by architecture
+! logical, parameter :: test_ieee_support_underflow_control = .not. ieee_support_underflow_control() &
+!   .and. .not. ieee_support_underflow_control(1.0_2) &
+!   .and. ieee_support_underflow_control(1.0_3) &
+!   .and. ieee_support_underflow_control(1.0_4) &
+!   .and. ieee_support_underflow_control(1.0_8) &
+!   .and. .not. ieee_support_underflow_control(1.0_10) &
+!   .and. .not. ieee_support_underflow_control(1.0_16)
 end module
diff --git a/flang/test/Examples/omp-declarative-directive.f90 b/flang/test/Examples/omp-declarative-directive.f90
index 632ebcec17885..4a9ad9142bb18 100644
--- a/flang/test/Examples/omp-declarative-directive.f90
+++ b/flang/test/Examples/omp-declarative-directive.f90
@@ -7,7 +7,7 @@
 ! 2.8.2 declare-simd
 
 subroutine declare_simd_1(a, b)
-  real(8), intent(inout) :: a, b
+  real(8), intent(inout), allocatable :: a, b
   !$omp declare simd(declare_simd_1) aligned(a)
   a = 3.14 + b
 end subroutine declare_simd_1
diff --git a/flang/test/Fir/CUDA/cuda-abstract-result.mlir b/flang/test/Fir/CUDA/cuda-abstract-result.mlir
new file mode 100644
index 0000000000000..8c59487ca5cd5
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-abstract-result.mlir
@@ -0,0 +1,37 @@
+// RUN: fir-opt -pass-pipeline='builtin.module(gpu.module(gpu.func(abstract-result)))' %s | FileCheck %s
+
+gpu.module @test {
+ gpu.func @_QMinterval_mPtest1(%arg0: !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, %arg1: !fir.ref<f32>) -> !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}> {
+    %c1_i32 = arith.constant 1 : i32
+    %18 = fir.dummy_scope : !fir.dscope
+    %19 = fir.declare %arg0 dummy_scope %18 {uniq_name = "_QMinterval_mFtest1Ea"} : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, !fir.dscope) -> !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>
+    %20 = fir.declare %arg1 dummy_scope %18 {uniq_name = "_QMinterval_mFtest1Eb"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+    %21 = fir.alloca !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}> {bindc_name = "c", uniq_name = "_QMinterval_mFtest1Ec"}
+    %22 = fir.declare %21 {uniq_name = "_QMinterval_mFtest1Ec"} : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>) -> !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>
+    %23 = fir.alloca i32 {bindc_name = "warpsize", uniq_name = "_QMcudadeviceECwarpsize"}
+    %24 = fir.declare %23 {uniq_name = "_QMcudadeviceECwarpsize"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %25 = fir.field_index inf, !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>
+    %26 = fir.coordinate_of %19, %25 : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, !fir.field) -> !fir.ref<f32>
+    %27 = fir.load %20 : !fir.ref<f32>
+    %28 = arith.negf %27 fastmath<contract> : f32
+    %29 = fir.load %26 : !fir.ref<f32>
+    %30 = fir.call @__fadd_rd(%29, %28) proc_attrs<bind_c> fastmath<contract> : (f32, f32) -> f32
+    %31 = fir.field_index inf, !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>
+    %32 = fir.coordinate_of %22, %31 : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, !fir.field) -> !fir.ref<f32>
+    fir.store %30 to %32 : !fir.ref<f32>
+    %33 = fir.field_index sup, !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>
+    %34 = fir.coordinate_of %19, %33 : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, !fir.field) -> !fir.ref<f32>
+    %35 = fir.load %20 : !fir.ref<f32>
+    %36 = arith.negf %35 fastmath<contract> : f32
+    %37 = fir.load %34 : !fir.ref<f32>
+    %38 = fir.call @__fadd_ru(%37, %36) proc_attrs<bind_c> fastmath<contract> : (f32, f32) -> f32
+    %39 = fir.field_index sup, !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>
+    %40 = fir.coordinate_of %22, %39 : (!fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, !fir.field) -> !fir.ref<f32>
+    fir.store %38 to %40 : !fir.ref<f32>
+    %41 = fir.load %22 : !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>
+    gpu.return %41 : !fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>
+  }
+}
+
+// CHECK: gpu.func @_QMinterval_mPtest1(%arg0: !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, %arg1: !fir.ref<!fir.type<_QMinterval_mTinterval{inf:f32,sup:f32}>>, %arg2: !fir.ref<f32>) {
+// CHECK: gpu.return{{$}}
diff --git a/flang/test/Fir/CUDA/cuda-constructor-2.f90 b/flang/test/Fir/CUDA/cuda-constructor-2.f90
index 901497e2cde55..eb118ccee311c 100644
--- a/flang/test/Fir/CUDA/cuda-constructor-2.f90
+++ b/flang/test/Fir/CUDA/cuda-constructor-2.f90
@@ -30,3 +30,33 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr, dense<
 // CHECK-DAG: %[[BOXREF:.*]] = fir.convert %[[BOX]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<i8>
 // CHECK-DAG: fir.call @_FortranACUFRegisterVariable(%[[MODULE:.*]], %[[BOXREF]], %{{.*}}, %{{.*}}) 
 //
+
+// ----- 
+
+// Checking that constant global variables are not registered
+
+// CHECK: @_FortranACUFRegisterAllocator
+// CHECK-NOT: fir.call @_FortranACUFRegisterVariable
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git 3372303188df0f7f8ac26e7ab610cf8b0f716d42)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  fir.global @_QMiso_c_bindingECc_int constant : i32
+  
+
+  fir.type_info @_QM__fortran_builtinsT__builtin_c_ptr noinit nodestroy nofinal : !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+  gpu.module @cuda_device_mod {
+    fir.global @_QMiso_c_bindingECc_int {data_attr = #cuf.cuda<constant>} constant : i32
+    gpu.func @_QMdevmodPdevsub(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> i32 {
+      %c0 = arith.constant 0 : index
+      %c4_i32 = arith.constant 4 : i32
+      %0 = fir.alloca i32 {bindc_name = "devsub", uniq_name = "_QMdevmodFdevsubEdevsub"}
+      %1 = fir.alloca i32 {bindc_name = "__builtin_warpsize", uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"}
+      %2 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+      %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+      %4 = fir.convert %3#1 : (index) -> i32
+      %5 = arith.muli %4, %c4_i32 : i32
+      fir.store %5 to %0 : !fir.ref<i32>
+      %6 = fir.load %0 : !fir.ref<i32>
+      gpu.return %6 : i32
+    }
+  }
+}
\ No newline at end of file
diff --git a/flang/test/Fir/CUDA/cuda-data-transfer.fir b/flang/test/Fir/CUDA/cuda-data-transfer.fir
index b371d39777728..7203c33e7eb11 100644
--- a/flang/test/Fir/CUDA/cuda-data-transfer.fir
+++ b/flang/test/Fir/CUDA/cuda-data-transfer.fir
@@ -199,12 +199,12 @@ func.func @_QPsub8() attributes {fir.bindc_name = "t"} {
 // CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<5xi32> 
 // CHECK: %[[LOCAL:.*]] = fir.declare %[[ALLOCA]]
 // CHECK: %[[GBL:.*]] = fir.address_of(@_QMmtestsEn) : !fir.ref<!fir.array<5xi32>>
-// CHECK: %[[DECL:.*]] = fir.declare %[[GBL]]
-// CHECK: %[[HOST:.*]] = fir.convert %[[DECL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
-// CHECK: %[[SRC:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[HOST]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
-// CHECK: %[[SRC_CONV:.*]] = fir.convert %[[SRC]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<5xi32>>
+// CHECK: %[[GBL_CONV:.*]] = fir.convert %[[GBL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[GBL_CONV]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR_CONV:.*]] = fir.convert %[[ADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<5xi32>>
+// CHECK: %[[DECL:.*]] = fir.declare %[[ADDR_CONV]]
 // CHECK: %[[DST:.*]] = fir.convert %[[LOCAL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
-// CHECK: %[[SRC:.*]] = fir.convert %[[SRC_CONV]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[SRC:.*]] = fir.convert %[[DECL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
 // CHECK: fir.call @_FortranACUFDataTransferPtrPtr(%[[DST]], %[[SRC]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
 
 
@@ -223,11 +223,11 @@ func.func @_QPsub9() {
 // CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<5xi32> 
 // CHECK: %[[LOCAL:.*]] = fir.declare %[[ALLOCA]]
 // CHECK: %[[GBL:.*]] = fir.address_of(@_QMmtestsEn) : !fir.ref<!fir.array<5xi32>>
-// CHECK: %[[DECL:.*]] = fir.declare %[[GBL]]
-// CHECK: %[[HOST:.*]] = fir.convert %[[DECL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
-// CHECK: %[[DST:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[HOST]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
-// CHECK: %[[DST_CONV:.*]] = fir.convert %[[DST]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<5xi32>>
-// CHECK: %[[DST:.*]] = fir.convert %[[DST_CONV]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[GBL_CONV:.*]] = fir.convert %[[GBL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[GBL_CONV]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR_CONV:.*]] = fir.convert %[[ADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<5xi32>>
+// CHECK: %[[DECL:.*]] = fir.declare %[[ADDR_CONV]]
+// CHECK: %[[DST:.*]] = fir.convert %[[DECL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
 // CHECK: %[[SRC:.*]] = fir.convert %[[LOCAL]] : (!fir.ref<!fir.array<5xi32>>) -> !fir.llvm_ptr<i8>
 // CHECK: fir.call @_FortranACUFDataTransferPtrPtr(%[[DST]], %[[SRC]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
 
@@ -380,9 +380,12 @@ func.func @_QPdevice_addr_conv() {
 }
 
 // CHECK-LABEL: func.func @_QPdevice_addr_conv()
-// CHECK: %[[DEV_ADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
-// CHECK: %[[DEV_ADDR_CONV:.*]] = fir.convert %[[DEV_ADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<4xf32>>
-// CHECK: fir.embox %[[DEV_ADDR_CONV]](%{{.*}}) : (!fir.ref<!fir.array<4xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK: %[[GBL:.*]] = fir.address_of(@_QMmod1Ea_dev) : !fir.ref<!fir.array<4xf32>>
+// CHECK: %[[GBL_CONV:.*]] = fir.convert %[[GBL]] : (!fir.ref<!fir.array<4xf32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[GBL_CONV]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADDR_CONV:.*]] = fir.convert %[[ADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<4xf32>>
+// CHECK: %[[DECL:.*]] = fir.declare %[[ADDR_CONV]](%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QMmod1Ea_dev"} : (!fir.ref<!fir.array<4xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<4xf32>>
+// CHECK: fir.embox %[[DECL]](%{{.*}}) : (!fir.ref<!fir.array<4xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
 // CHECK: fir.call @_FortranACUFDataTransferCstDesc
 
 func.func @_QQchar_transfer() attributes {fir.bindc_name = "char_transfer"} {
diff --git a/flang/test/Fir/CUDA/cuda-external-mangling.mlir b/flang/test/Fir/CUDA/cuda-external-mangling.mlir
new file mode 100644
index 0000000000000..cd028a201e6fa
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-external-mangling.mlir
@@ -0,0 +1,25 @@
+// RUN: fir-opt --split-input-file --external-name-interop %s | FileCheck %s
+
+module @mod attributes {gpu.container_module} {
+
+gpu.module @cuda_device_mod {
+  gpu.func @_QPfoo() kernel {
+    fir.call @_QPthreadfence() fastmath<contract> : () -> ()
+    gpu.return
+  }
+  func.func private @_QPthreadfence() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
+}
+
+func.func @test() -> () {
+  %0 = llvm.mlir.constant(0 : i64) : i64
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  gpu.launch_func  @cuda_device_mod::@_QPfoo blocks in (%0, %0, %0) threads in (%0, %0, %0) : i64 dynamic_shared_memory_size %1
+  return
+}
+
+// CHECK-LABEL: gpu.func @foo_()
+// CHECK: fir.call @threadfence_()
+// CHECK: func.func private @threadfence_()
+// CHECK: gpu.launch_func  @cuda_device_mod::@foo_ 
+
+}
diff --git a/flang/test/Fir/CUDA/cuda-extranal-mangling.mlir b/flang/test/Fir/CUDA/cuda-extranal-mangling.mlir
deleted file mode 100644
index 551a89a7018c2..0000000000000
--- a/flang/test/Fir/CUDA/cuda-extranal-mangling.mlir
+++ /dev/null
@@ -1,13 +0,0 @@
-// RUN: fir-opt --split-input-file --external-name-interop %s | FileCheck %s
-
-gpu.module @cuda_device_mod {
-  gpu.func @_QPfoo() {
-    fir.call @_QPthreadfence() fastmath<contract> : () -> ()
-    gpu.return
-  }
-  func.func private @_QPthreadfence() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
-}
-
-// CHECK-LABEL: gpu.func @_QPfoo
-// CHECK: fir.call @threadfence_()
-// CHECK: func.func private @threadfence_()
diff --git a/flang/test/Fir/CUDA/cuda-global-addr.mlir b/flang/test/Fir/CUDA/cuda-global-addr.mlir
new file mode 100644
index 0000000000000..94ee74736f650
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-global-addr.mlir
@@ -0,0 +1,66 @@
+// RUN: fir-opt --split-input-file --cuf-convert %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+fir.global @_QMmod1Eadev {data_attr = #cuf.cuda<device>} : !fir.array<10xi32> {
+  %0 = fir.zero_bits !fir.array<10xi32>
+  fir.has_value %0 : !fir.array<10xi32>
+}
+func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+  %c14_i32 = arith.constant 14 : i32
+  %c6_i32 = arith.constant 6 : i32
+  %c4 = arith.constant 4 : index
+  %c1_i32 = arith.constant 1 : i32
+  %c0_i32 = arith.constant 0 : i32
+  %c10 = arith.constant 10 : index
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %3 = fir.address_of(@_QMmod1Eadev) : !fir.ref<!fir.array<10xi32>>
+  %4 = fir.declare %3(%1) {data_attr = #cuf.cuda<device>, uniq_name = "_QMmod1Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+  %5 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+  %6 = fir.declare %5 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  fir.store %c0_i32 to %6 : !fir.ref<i32>
+  %7 = fir.array_coor %4(%1) %c4 : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+  cuf.data_transfer %c1_i32 to %7 {transfer_kind = #cuf.cuda_transfer<host_device>} : i32, !fir.ref<i32>
+  return
+}
+
+}
+
+// CHECK-LABEL: func.func @_QQmain()
+// CHECK: %[[ADDR:.*]] = fir.address_of(@_QMmod1Eadev) : !fir.ref<!fir.array<10xi32>>
+// CHECK: %[[ADDRPTR:.*]] = fir.convert %[[ADDR]] : (!fir.ref<!fir.array<10xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[DEVICE_ADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[ADDRPTR]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
+// CHECK: %[[DEVICE_ADDR_CONV:.*]] = fir.convert %[[DEVICE_ADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.array<10xi32>>
+// CHECK: %[[DECL:.*]] = fir.declare %[[DEVICE_ADDR_CONV]](%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QMmod1Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+// CHECK: %[[ARRAY_COOR:.*]] = fir.array_coor %[[DECL]](%{{.*}}) %c4{{.*}} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+// CHECK: %[[ARRAY_COOR_PTR:.*]] = fir.convert %[[ARRAY_COOR]] : (!fir.ref<i32>) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFDataTransferPtrPtr(%[[ARRAY_COOR_PTR]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
+
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+
+  fir.global @_QMdevmodEdarray {data_attr = #cuf.cuda<device>} : !fir.box<!fir.heap<!fir.array<?xf32>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) {allocator_idx = 2 : i32} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?xf32>>>
+  }
+  func.func @_QQmain() attributes {fir.bindc_name = "arraysize"} {
+    %0 = fir.address_of(@_QMiso_c_bindingECc_int) : !fir.ref<i32>
+    %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QMiso_c_bindingECc_int"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %2 = fir.address_of(@_QMdevmodEdarray) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %3 = fir.declare %2 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMdevmodEdarray"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %4 = fir.alloca i32 {bindc_name = "exp", uniq_name = "_QFEexp"}
+    %5 = fir.declare %4 {uniq_name = "_QFEexp"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %6 = fir.alloca i32 {bindc_name = "hsize", uniq_name = "_QFEhsize"}
+    %7 = fir.declare %6 {uniq_name = "_QFEhsize"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    return
+  }
+  fir.global @_QMiso_c_bindingECc_int constant : i32
+}
+
+// We cannot call _FortranACUFGetDeviceAddress on a constant global. 
+// There is no symbol for it and the call would result into an unresolved reference.
+// CHECK-NOT: fir.call {{.*}}GetDeviceAddress
+
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
index 7fede7c6c17b7..3db2336c90a7d 100644
--- a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -99,9 +99,16 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : ve
 }
 
 // CHECK-LABEL: _QMmod1Phost_sub
-
+// CHECK: %[[STRUCT:.*]] = llvm.alloca %{{.*}} x !llvm.struct<(ptr)> : (i32) -> !llvm.ptr
+// CHECK: %[[PARAMS:.*]] = llvm.alloca %{{.*}} x !llvm.ptr : (i32) -> !llvm.ptr
+// CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK: %[[STRUCT_PTR:.*]] = llvm.getelementptr %[[STRUCT]][%{{.*}}, {{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<(ptr)>
+// CHECK: llvm.store %{{.*}}, %[[STRUCT_PTR]] : !llvm.ptr, !llvm.ptr
+// CHECK: %[[PARAM_PTR:.*]] = llvm.getelementptr %[[PARAMS]][%[[ZERO]]] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.ptr
+// CHECK: llvm.store %[[STRUCT_PTR]], %[[PARAM_PTR]] : !llvm.ptr, !llvm.ptr
 // CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1 : !llvm.ptr
-// CHECK: llvm.call @_FortranACUFLaunchKernel(%[[KERNEL_PTR]], {{.*}})
+// CHECK: %[[NULL:.*]] = llvm.mlir.zero : !llvm.ptr
+// CHECK: llvm.call @_FortranACUFLaunchKernel(%[[KERNEL_PTR]], {{.*}}, %[[PARAMS]], %[[NULL]])
 
 // -----
 
diff --git a/flang/test/Fir/CUDA/cuda-implicit-device-global.f90 b/flang/test/Fir/CUDA/cuda-implicit-device-global.f90
index 772e2696171a6..ec5ed06824e22 100644
--- a/flang/test/Fir/CUDA/cuda-implicit-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-implicit-device-global.f90
@@ -23,7 +23,7 @@ // Test that global used in device function are flagged with the correct
 // CHECK: %[[GLOBAL:.*]] = fir.address_of(@_QQcl[[SYMBOL:.*]]) : !fir.ref<!fir.char<1,32>>
 // CHECK: %[[CONV:.*]] = fir.convert %[[GLOBAL]] : (!fir.ref<!fir.char<1,32>>) -> !fir.ref<i8>
 // CHECK: fir.call @_FortranAioBeginExternalListOutput(%{{.*}}, %[[CONV]], %{{.*}}) fastmath<contract> : (i32, !fir.ref<i8>, i32) -> !fir.ref<i8>
-// CHECK: fir.global linkonce @_QQcl[[SYMBOL]] {data_attr = #cuf.cuda<constant>} constant : !fir.char<1,32>
+// CHECK: fir.global linkonce @_QQcl[[SYMBOL]] constant : !fir.char<1,32>
 
 // CHECK-LABEL: gpu.module @cuda_device_mod
 // CHECK: fir.global linkonce @_QQclX6995815537abaf90e86ce166af128f3a
@@ -99,10 +99,11 @@ // Test that global used in device function are flagged with the correct
   fir.has_value %0 : !fir.char<1,11>
 }
 
-// CHECK: fir.global linkonce @_QQclX5465737420504153534544 {data_attr = #cuf.cuda<constant>} constant : !fir.char<1,11>
+// Checking that a constant fir.global that is only used in host code is not copied over to the device
+// CHECK: fir.global linkonce @_QQclX5465737420504153534544 constant : !fir.char<1,11>
 
 // CHECK-LABEL: gpu.module @cuda_device_mod
-// CHECK: fir.global linkonce @_QQclX5465737420504153534544 {data_attr = #cuf.cuda<constant>} constant
+// CHECK-NOT: fir.global linkonce @_QQclX5465737420504153534544
 
 // -----
 
@@ -140,7 +141,53 @@ // Test that global used in device function are flagged with the correct
 }
 func.func private @_FortranAioEndIoStatement(!fir.ref<i8>) -> i32 attributes {fir.io, fir.runtime}
 
-// CHECK: fir.global linkonce @_QQclX5465737420504153534544 {data_attr = #cuf.cuda<constant>} constant : !fir.char<1,11>
+// Checking that a constant fir.global that is used in device code is copied over to the device
+// CHECK: fir.global linkonce @_QQclX5465737420504153534544 constant : !fir.char<1,11>
 
 // CHECK-LABEL: gpu.module @cuda_device_mod 
-// CHECK: fir.global linkonce @_QQclX5465737420504153534544 {data_attr = #cuf.cuda<constant>} constant
+// CHECK: fir.global linkonce @_QQclX5465737420504153534544 constant
+
+// -----
+
+func.func @_QQmain() attributes {fir.bindc_name = "cufkernel_global"} {
+  %c10 = arith.constant 10 : index
+  %c5_i32 = arith.constant 5 : i32
+  %c6_i32 = arith.constant 6 : i32
+  %c1 = arith.constant 1 : index
+  %c1_i32 = arith.constant 1 : i32
+  %c10_i32 = arith.constant 10 : i32
+  %0 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+  %1:2 = hlfir.declare %0 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  cuf.kernel<<<%c10_i32, %c1_i32>>> (%arg0 : index) = (%c1 : index) to (%c10 : index)  step (%c1 : index) {
+    %2 = fir.convert %arg0 : (index) -> i32
+    fir.store %2 to %1#1 : !fir.ref<i32>
+    %3 = fir.load %1#0 : !fir.ref<i32>
+    %4 = arith.cmpi eq, %3, %c1_i32 : i32
+    cf.cond_br %4, ^bb1, ^bb2
+  ^bb1:  // pred: ^bb0
+    %5 = fir.address_of(@_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5) : !fir.ref<!fir.char<1,50>>
+    %6 = fir.convert %5 : (!fir.ref<!fir.char<1,50>>) -> !fir.ref<i8>
+    %7 = fir.call @_FortranAioBeginExternalListOutput(%c6_i32, %6, %c5_i32) fastmath<contract> : (i32, !fir.ref<i8>, i32) -> !fir.ref<i8>
+    %8 = fir.load %1#0 : !fir.ref<i32>
+    %9 = fir.call @_FortranAioOutputInteger32(%7, %8) fastmath<contract> : (!fir.ref<i8>, i32) -> i1
+    %10 = fir.call @_FortranAioEndIoStatement(%7) fastmath<contract> : (!fir.ref<i8>) -> i32
+    cf.br ^bb2
+  ^bb2:  // 2 preds: ^bb0, ^bb1
+    "fir.end"() : () -> ()
+  }
+  return
+}
+func.func private @_FortranAioBeginExternalListOutput(i32, !fir.ref<i8>, i32) -> !fir.ref<i8> attributes {fir.io, fir.runtime}
+fir.global linkonce @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 constant : !fir.char<1,50> {
+  %0 = fir.string_lit "/local/home/vclement/llvm-project/build/dummy.cuf\00"(50) : !fir.char<1,50>
+  fir.has_value %0 : !fir.char<1,50>
+}
+func.func private @_FortranAioOutputInteger32(!fir.ref<i8>, i32) -> i1 attributes {fir.io, fir.runtime}
+func.func private @_FortranAioEndIoStatement(!fir.ref<i8>) -> i32 attributes {fir.io, fir.runtime}
+func.func private @_FortranAProgramStart(i32, !llvm.ptr, !llvm.ptr, !llvm.ptr)
+func.func private @_FortranAProgramEndStatement()
+
+// CHECK-LABEL: func.func @_QQmain()
+// CHECK: fir.global linkonce @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 constant : !fir.char<1,50>
+// CHECK: gpu.module @cuda_device_mod
+// CHECK: fir.global linkonce @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 constant : !fir.char<1,50>
diff --git a/flang/test/Fir/CUDA/cuda-target-rewrite.mlir b/flang/test/Fir/CUDA/cuda-target-rewrite.mlir
new file mode 100644
index 0000000000000..d88b6776795a0
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-target-rewrite.mlir
@@ -0,0 +1,29 @@
+// REQUIRES: x86-registered-target
+// RUN: fir-opt --split-input-file --target-rewrite="target=x86_64-unknown-linux-gnu" %s | FileCheck %s
+
+gpu.module @testmod {
+  gpu.func @_QPvcpowdk(%arg0: !fir.ref<complex<f64>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "a"}) attributes {cuf.proc_attr = #cuf.cuda_proc<global>} {
+    %0 = fir.alloca i64
+    %1 = fir.load %0 : !fir.ref<i64>
+    %2 = fir.load %arg0 : !fir.ref<complex<f64>>
+    %3 = fir.call @_FortranAzpowk(%2, %1) fastmath<contract> : (complex<f64>, i64) -> complex<f64>
+    gpu.return
+  }
+  func.func private @_FortranAzpowk(complex<f64>, i64) -> complex<f64> attributes {fir.bindc_name = "_FortranAzpowk", fir.runtime}
+}
+
+// CHECK-LABEL: gpu.func @_QPvcpowdk
+// CHECK: %{{.*}} = fir.call @_FortranAzpowk(%{{.*}}, %{{.*}}, %{{.*}}) : (f64, f64, i64) -> tuple<f64, f64>
+// CHECK: func.func private @_FortranAzpowk(f64, f64, i64) -> tuple<f64, f64> attributes {fir.bindc_name = "_FortranAzpowk", fir.runtime}
+
+// -----
+
+gpu.module @testmod {
+  gpu.func @_QPtest(%arg0: complex<f64>) -> (complex<f64>) {
+    gpu.return %arg0 : complex<f64>
+  }
+}
+
+// CHECK-LABEL: gpu.func @_QPtest
+// CHECK-SAME: (%arg0: f64, %arg1: f64) -> tuple<f64, f64> {
+// CHECK: gpu.return %{{.*}} : tuple<f64, f64>
diff --git a/flang/test/Fir/arrayset.fir b/flang/test/Fir/arrayset.fir
index a9630c815fc42..d201e34ce72af 100644
--- a/flang/test/Fir/arrayset.fir
+++ b/flang/test/Fir/arrayset.fir
@@ -1,7 +1,7 @@
 // RUN: tco %s | FileCheck %s
 // RUN: %flang_fc1 -emit-llvm %s -o - | FileCheck %s
 
-// CHECK-LABEL: define void @x(ptr %0)
+// CHECK-LABEL: define void @x(ptr nocapture %0)
 func.func @x(%arr : !fir.ref<!fir.array<10xf32>>) {
   %1 = arith.constant 0 : index
   %2 = arith.constant 9 : index
diff --git a/flang/test/Fir/arrexp.fir b/flang/test/Fir/arrexp.fir
index 5d265a5e3a08d..3363ead15d8a8 100644
--- a/flang/test/Fir/arrexp.fir
+++ b/flang/test/Fir/arrexp.fir
@@ -1,7 +1,7 @@
 // RUN: tco %s | FileCheck %s
 
 // CHECK-LINE: define void @f1
-// CHECK: (ptr %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f1(%a : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -23,7 +23,7 @@ func.func @f1(%a : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : i
 }
 
 // CHECK-LINE: define void @f2
-// CHECK: (ptr %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f2(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -47,7 +47,7 @@ func.func @f2(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf
 }
 
 // CHECK-LINE: define void @f3
-// CHECK: (ptr %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f3(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -72,7 +72,7 @@ func.func @f3(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf
 }
 
 // CHECK-LINE: define void @f4
-// CHECK: (ptr %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
+// CHECK: (ptr nocapture %[[A:[^,]*]], {{.*}}, float %[[F:.*]])
 func.func @f4(%a : !fir.ref<!fir.array<?x?xf32>>, %b : !fir.ref<!fir.array<?x?xf32>>, %n : index, %m : index, %o : index, %p : index, %f : f32) {
   %c1 = arith.constant 1 : index
   %s = fir.shape_shift %o, %n, %p, %m : (index, index, index, index) -> !fir.shapeshift<2>
@@ -165,7 +165,7 @@ func.func @f6(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: f32) {
 // Non contiguous array with lower bounds (x = y(100), with y(4:))
 // Test array_coor offset computation.
 // CHECK-LABEL:  define void @f7(
-// CHECK: ptr %[[X:[^,]*]], ptr %[[Y:.*]])
+// CHECK: ptr nocapture %[[X:[^,]*]], ptr %[[Y:.*]])
 func.func @f7(%arg0: !fir.ref<f32>, %arg1: !fir.box<!fir.array<?xf32>>) {
   %c4 = arith.constant 4 : index
   %c100 = arith.constant 100 : index
@@ -181,7 +181,7 @@ func.func @f7(%arg0: !fir.ref<f32>, %arg1: !fir.box<!fir.array<?xf32>>) {
 
 // Test A(:, :)%x reference codegen with A constant shape.
 // CHECK-LABEL:  define void @f8(
-// CHECK-SAME: ptr %[[A:.*]], i32 %[[I:.*]])
+// CHECK-SAME: ptr nocapture %[[A:.*]], i32 %[[I:.*]])
 func.func @f8(%a : !fir.ref<!fir.array<2x2x!fir.type<t{i:i32}>>>, %i : i32) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -198,7 +198,7 @@ func.func @f8(%a : !fir.ref<!fir.array<2x2x!fir.type<t{i:i32}>>>, %i : i32) {
 
 // Test casts in in array_coor offset computation when type parameters are not i64
 // CHECK-LABEL: define ptr @f9(
-// CHECK-SAME: i32 %[[I:.*]], i64 %{{.*}}, i64 %{{.*}}, ptr %[[C:.*]])
+// CHECK-SAME: i32 %[[I:.*]], i64 %{{.*}}, i64 %{{.*}}, ptr nocapture %[[C:.*]])
 func.func @f9(%i: i32, %e : i64, %j: i64, %c: !fir.ref<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.char<1,?>> {
   %s = fir.shape %e, %e : (i64, i64) -> !fir.shape<2>
   // CHECK: %[[CAST:.*]] = sext i32 %[[I]] to i64
diff --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index 4b18acb7c2b43..d2788008c3893 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -106,11 +106,17 @@ func.func @_QQmain() {
 // PASSES-NEXT:   (S) 0 num-dce'd - Number of operations DCE'd
 // PASSES-NEXT: BoxedProcedurePass
 
-// PASSES-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'omp.declare_reduction', 'omp.private']
+// PASSES-NEXT: Pipeline Collection : ['fir.global', 'func.func', 'gpu.module', 'omp.declare_reduction', 'omp.private']
 // PASSES-NEXT:  'fir.global' Pipeline
 // PASSES-NEXT:    AbstractResultOpt
 // PASSES-NEXT:  'func.func' Pipeline
 // PASSES-NEXT:    AbstractResultOpt
+// PASSES-NEXT:  'gpu.module' Pipeline
+// PASSES-NEXT:   Pipeline Collection : ['func.func', 'gpu.func'] 
+// PASSES-NEXT:   'func.func' Pipeline 
+// PASSES-NEXT:   AbstractResultOpt
+// PASSES-NEXT:   'gpu.func' Pipeline 
+// PASSES-NEXT:   AbstractResultOpt
 // PASSES-NEXT:  'omp.declare_reduction' Pipeline
 // PASSES-NEXT:    AbstractResultOpt
 // PASSES-NEXT:  'omp.private' Pipeline
@@ -120,6 +126,8 @@ func.func @_QQmain() {
 // PASSES-NEXT:   (S) 0 num-dce'd - Number of operations eliminated
 // PASSES-NEXT: TargetRewrite
 // PASSES-NEXT: CompilerGeneratedNamesConversion
+// PASSES-NEXT: 'func.func' Pipeline
+// PASSES-NEXT:  FunctionAttr
 // PASSES-NEXT: FIRToLLVMLowering
 // PASSES-NEXT: ReconcileUnrealizedCasts
 // PASSES-NEXT: LLVMIRLoweringPass
diff --git a/flang/test/Fir/box-offset-codegen.fir b/flang/test/Fir/box-offset-codegen.fir
index 389ceebcc065c..5be3ad1f3eeed 100644
--- a/flang/test/Fir/box-offset-codegen.fir
+++ b/flang/test/Fir/box-offset-codegen.fir
@@ -7,7 +7,7 @@ func.func @scalar_addr(%scalar : !fir.ref<!fir.box<!fir.type<t>>>) -> !fir.llvm_
   return %addr : !fir.llvm_ptr<!fir.ref<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @scalar_addr(
-// CHECK-SAME: ptr %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 0
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -16,7 +16,7 @@ func.func @scalar_tdesc(%scalar : !fir.ref<!fir.box<!fir.type<t>>>) -> !fir.llvm
   return %tdesc : !fir.llvm_ptr<!fir.tdesc<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @scalar_tdesc(
-// CHECK-SAME: ptr %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 7
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -25,7 +25,7 @@ func.func @array_addr(%array : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.ty
   return %addr : !fir.llvm_ptr<!fir.ptr<!fir.array<?x!fir.type<t>>>>
 }
 // CHECK-LABEL: define ptr @array_addr(
-// CHECK-SAME: ptr %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 0
 // CHECK:    ret ptr %[[VAL_0]]
 
@@ -34,6 +34,6 @@ func.func @array_tdesc(%array : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.t
   return %tdesc : !fir.llvm_ptr<!fir.tdesc<!fir.type<t>>>
 }
 // CHECK-LABEL: define ptr @array_tdesc(
-// CHECK-SAME: ptr %[[BOX:.*]]){{.*}}{
+// CHECK-SAME: ptr nocapture %[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 8
 // CHECK:    ret ptr %[[VAL_0]]
diff --git a/flang/test/Fir/box.fir b/flang/test/Fir/box.fir
index fd9fa1f2b3aab..841e10d3debbc 100644
--- a/flang/test/Fir/box.fir
+++ b/flang/test/Fir/box.fir
@@ -24,7 +24,7 @@ func.func private @g(%b : !fir.box<f32>)
 func.func private @ga(%b : !fir.box<!fir.array<?xf32>>)
 
 // CHECK-LABEL: define void @f
-// CHECK: (ptr %[[ARG:.*]])
+// CHECK: (ptr nocapture %[[ARG:.*]])
 func.func @f(%a : !fir.ref<f32>) {
   // CHECK: %[[DESC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }
   // CHECK: %[[INS0:.*]] = insertvalue {{.*}} { ptr undef, i64 ptrtoint (ptr getelementptr (float, ptr null, i32 1) to i64), i32 20240719, i8 0, i8 27, i8 0, i8 0 }, ptr %[[ARG]], 0
@@ -38,7 +38,7 @@ func.func @f(%a : !fir.ref<f32>) {
 }
 
 // CHECK-LABEL: define void @fa
-// CHECK: (ptr %[[ARG:.*]])
+// CHECK: (ptr nocapture %[[ARG:.*]])
 func.func @fa(%a : !fir.ref<!fir.array<100xf32>>) {
   %c = fir.convert %a : (!fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<?xf32>>
   %c1 = arith.constant 1 : index
@@ -54,7 +54,7 @@ func.func @fa(%a : !fir.ref<!fir.array<100xf32>>) {
 
 // Boxing of a scalar character of dynamic length
 // CHECK-LABEL: define void @b1(
-// CHECK-SAME: ptr %[[res:.*]], ptr %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.char<1,?>> {
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }
   // CHECK: %[[size:.*]] = mul i64 ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64), %[[arg1]]
@@ -69,8 +69,8 @@ func.func @b1(%arg0 : !fir.ref<!fir.char<1,?>>, %arg1 : index) -> !fir.box<!fir.
 
 // Boxing of a dynamic array of character with static length (5)
 // CHECK-LABEL: define void @b2(
-// CHECK-SAME: ptr %[[res]],
-// CHECK-SAME: ptr %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr nocapture %[[res]],
+// CHECK-SAME: ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) -> !fir.box<!fir.array<?x!fir.char<1,5>>> {
   %1 = fir.shape %arg1 : (index) -> !fir.shape<1>
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
@@ -85,7 +85,7 @@ func.func @b2(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,5>>>, %arg1 : index) ->
 
 // Boxing of a dynamic array of character of dynamic length
 // CHECK-LABEL: define void @b3(
-// CHECK-SAME: ptr %[[res:.*]], ptr %[[arg0:.*]], i64 %[[arg1:.*]], i64 %[[arg2:.*]])
+// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]], i64 %[[arg2:.*]])
 func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %arg2 : index) -> !fir.box<!fir.array<?x!fir.char<1,?>>> {
   %1 = fir.shape %arg2 : (index) -> !fir.shape<1>
   // CHECK: %[[alloca:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
@@ -103,7 +103,7 @@ func.func @b3(%arg0 : !fir.ref<!fir.array<?x!fir.char<1,?>>>, %arg1 : index, %ar
 
 // Boxing of a static array of character of dynamic length
 // CHECK-LABEL: define void @b4(
-// CHECK-SAME: ptr %[[res:.*]], ptr %[[arg0:.*]], i64 %[[arg1:.*]])
+// CHECK-SAME: ptr nocapture %[[res:.*]], ptr nocapture %[[arg0:.*]], i64 %[[arg1:.*]])
 func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) -> !fir.box<!fir.array<7x!fir.char<1,?>>> {
   %c_7 = arith.constant 7 : index
   %1 = fir.shape %c_7 : (index) -> !fir.shape<1>
@@ -122,7 +122,7 @@ func.func @b4(%arg0 : !fir.ref<!fir.array<7x!fir.char<1,?>>>, %arg1 : index) ->
 
 // Storing a fir.box into a fir.ref<fir.box> (modifying descriptors).
 // CHECK-LABEL: define void @b5(
-// CHECK-SAME: ptr %[[arg0:.*]], ptr %[[arg1:.*]])
+// CHECK-SAME: ptr nocapture %[[arg0:.*]], ptr %[[arg1:.*]])
 func.func @b5(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1 : !fir.box<!fir.heap<!fir.array<?x?xf32>>>) {
   fir.store %arg1 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
   // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %0, ptr %1, i32 72, i1 false)
@@ -132,7 +132,7 @@ func.func @b5(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, %arg1
 func.func private @callee6(!fir.box<none>) -> i32
 
 // CHECK-LABEL: define i32 @box6(
-// CHECK-SAME: ptr %[[ARG0:.*]], i64 %[[ARG1:.*]], i64 %[[ARG2:.*]])
+// CHECK-SAME: ptr nocapture %[[ARG0:.*]], i64 %[[ARG1:.*]], i64 %[[ARG2:.*]])
 func.func @box6(%0 : !fir.ref<!fir.array<?x?x?x?xf32>>, %1 : index, %2 : index) -> i32 {
   %c100 = arith.constant 100 : index
   %c50 = arith.constant 50 : index
diff --git a/flang/test/Fir/boxproc-2.fir b/flang/test/Fir/boxproc-2.fir
index 84132f89afebb..963dfa03663cb 100644
--- a/flang/test/Fir/boxproc-2.fir
+++ b/flang/test/Fir/boxproc-2.fir
@@ -13,6 +13,9 @@ func.func private @test3(!fir.boxproc<() -> (!fir.type<a{x:i32, y:f64}>)>) -> no
 //CHECK-LABEL:  func.func private @test5(((i32) -> f32) -> ())
 func.func private @test5(!fir.boxproc<(!fir.boxproc<(i32) -> (f32)>) -> ()>)
 
+//CHECK-LABEL:  func.func private @test6(!fir.tdesc<!fir.type<_QMcartesian_2d_objectsTcartesian_2d_objectUnboxProc{base_pde_object:!fir.type<_QMbase_pde_objectsTbase_pde_objectUnboxProc{process_p:(!fir.class<!fir.type<_QMbase_pde_objectsTbase_pde_objectUnboxProc>>) -> !fir.class<!fir.heap<!fir.type<_QMbase_pde_objectsTbase_pde_objectUnboxProc>>>,source_p:(!fir.class<!fir.type<_QMbase_pde_objectsTbase_pde_objectUnboxProc>>, !fir.ref<f32>) -> !fir.class<!fir.heap<!fir.type<_QMbase_pde_objectsTbase_pde_objectUnboxProc>>>}>,c:!fir.box<!fir.heap<!fir.array<?x?xf32>>>,dx:f32,dy:f32}>>) -> !fir.ref<none>
+func.func private @test6(!fir.tdesc<!fir.type<_QMcartesian_2d_objectsTcartesian_2d_object{base_pde_object:!fir.type<_QMbase_pde_objectsTbase_pde_object{process_p:!fir.boxproc<(!fir.class<!fir.type<_QMbase_pde_objectsTbase_pde_object>>) -> !fir.class<!fir.heap<!fir.type<_QMbase_pde_objectsTbase_pde_object>>>>,source_p:!fir.boxproc<(!fir.class<!fir.type<_QMbase_pde_objectsTbase_pde_object>>, !fir.ref<f32>) -> !fir.class<!fir.heap<!fir.type<_QMbase_pde_objectsTbase_pde_object>>>>}>,c:!fir.box<!fir.heap<!fir.array<?x?xf32>>>,dx:f32,dy:f32}>>) -> !fir.ref<none>
+
 // CHECK-LABEL:   func.func @proc_pointer_component(
 // CHECK-SAME:                                      %[[VAL_0:.*]]: (!fir.ref<i32>) -> f32,
 // CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32>) {
diff --git a/flang/test/Fir/boxproc-openmp.fir b/flang/test/Fir/boxproc-openmp.fir
new file mode 100644
index 0000000000000..8b714539b5e85
--- /dev/null
+++ b/flang/test/Fir/boxproc-openmp.fir
@@ -0,0 +1,87 @@
+// RUN: fir-opt --boxed-procedure %s | FileCheck %s
+// Test the boxed procedure pass with OpenMP declaration operations.
+// Check minimally, only arguments, yields and the private types.
+
+// Test a private declaration with one region (alloc)
+//CHECK: omp.private {type = private}  @_QFsub1Et1_private_ref_rec__QFsub1Tt : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>> alloc {
+omp.private {type = private} @_QFsub1Et1_private_ref_rec__QFsub1Tt : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>> alloc {
+//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>):
+^bb0(%arg0: !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>):
+  %c1_i32 = arith.constant 1 : i32
+  %0 = fir.alloca !fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}> {bindc_name = "t1", pinned, uniq_name = "_QFsub1Et1"}
+  %1 = fir.declare %0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
+  %2 = fir.embox %1 : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
+  %3 = fir.address_of(@_QQclXea6256ba131ddd9c2210e68030a0edd3) : !fir.ref<!fir.char<1,49>>
+  %4 = fir.convert %2 : (!fir.box<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<none>
+  %5 = fir.convert %3 : (!fir.ref<!fir.char<1,49>>) -> !fir.ref<i8>
+  %6 = fir.call @_FortranAInitialize(%4, %5, %c1_i32) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
+//CHECK: omp.yield(%{{.*}} : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>)
+  omp.yield(%1 : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>)
+}
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}> {bindc_name = "t1", uniq_name = "_QFsub1Et1"}
+  %1 = fir.declare %0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
+//CHECK: omp.parallel private(@_QFsub1Et1_private_ref_rec__QFsub1Tt %{{.*}} -> %{{.*}} : !fir.ref<!fir.type<_QFsub1TtUnboxProc{p1:() -> ()}>>) {
+  omp.parallel private(@_QFsub1Et1_private_ref_rec__QFsub1Tt %1 -> %arg0 : !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) {
+    %2 = fir.declare %arg0 {uniq_name = "_QFsub1Et1"} : (!fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.ref<!fir.type<_QFsub1Tt{p1:!fir.boxproc<() -> ()>}>>
+    omp.terminator
+  }
+  return
+}
+
+
+// Test a private declaration with all regions (alloc, copy, dealloc)
+//CHECK: omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt : 
+//CHECK-SAME: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>> alloc {
+omp.private {type = firstprivate} @_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>> alloc {
+//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>):
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>):
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>> {bindc_name = "t1", pinned, uniq_name = "_QFsub2Et1"}
+  %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
+//CHECK: omp.yield(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>)
+  omp.yield(%1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>)
+} copy {
+//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>,
+//CHECK-SAME: %{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>):
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>, %arg1: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>):
+  %c5_i32 = arith.constant 5 : i32
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
+  %1 = fir.box_addr %0 : (!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>) -> !fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>
+  %2 = fir.embox %1 : (!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>
+  %3 = fir.address_of(@_QQclXea) : !fir.ref<!fir.char<1,8>>
+  %4 = fir.convert %arg1 : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<none>>
+  %5 = fir.convert %2 : (!fir.box<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>) -> !fir.box<none>
+  %6 = fir.convert %3 : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
+  %7 = fir.call @_FortranAAssign(%4, %5, %6, %c5_i32) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+//CHECK: omp.yield(%{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>)
+  omp.yield(%arg1 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>)
+} dealloc {
+//CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>):
+^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>):
+  %c5_i32 = arith.constant 5 : i32
+  %false = arith.constant false
+  %0 = fir.absent !fir.box<none>
+  %1 = fir.address_of(@_QQclXea) : !fir.ref<!fir.char<1,8>>
+  %2 = fir.convert %arg0 : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<none>>
+  %3 = fir.convert %1 : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
+  %4 = fir.call @_FortranAAllocatableDeallocate(%2, %false, %0, %3, %c5_i32) fastmath<contract> : (!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+  omp.yield
+}
+func.func @_QPsub2() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>> {bindc_name = "t1", uniq_name = "_QFsub2Et1"}
+  %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
+//CHECK: omp.parallel private(@_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt %{{.*}} -> %{{.*}} :
+//CHECK-SAME: !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2TtUnboxProc{p1:() -> ()}>>>>) {
+  omp.parallel private(@_QFsub2Et1_firstprivate_ref_box_heap_rec__QFsub2Tt %1 -> %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) {
+    %2 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Et1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QFsub2Tt{p1:!fir.boxproc<() -> ()>}>>>>
+    omp.terminator
+  }
+  return
+}
+func.func private @_FortranAInitialize(!fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
+fir.global linkonce @_QQclXea constant : !fir.char<1,8> {
+  %0 = fir.string_lit "pp.f90\00"(8) : !fir.char<1,8>
+  fir.has_value %0 : !fir.char<1,8>
+}
+func.func private @_FortranAAllocatableDeallocate(!fir.ref<!fir.box<none>>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32 attributes {fir.runtime}
+func.func private @_FortranAAssign(!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
diff --git a/flang/test/Fir/boxproc.fir b/flang/test/Fir/boxproc.fir
index 9e4ea0bc21077..27d8953236e72 100644
--- a/flang/test/Fir/boxproc.fir
+++ b/flang/test/Fir/boxproc.fir
@@ -12,7 +12,7 @@
 // CHECK:         call void @_QPtest_proc_dummy_other(ptr %[[VAL_6]])
 
 // CHECK-LABEL: define void @_QFtest_proc_dummyPtest_proc_dummy_a(ptr
-// CHECK-SAME:              %[[VAL_0:.*]], ptr nest %[[VAL_1:.*]])
+// CHECK-SAME:              nocapture %[[VAL_0:.*]], ptr nest nocapture %[[VAL_1:.*]])
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy_other(ptr
 // CHECK-SAME:              %[[VAL_0:.*]])
@@ -86,7 +86,7 @@ func.func @_QPtest_proc_dummy_other(%arg0: !fir.boxproc<() -> ()>) {
 // CHECK:         call void @llvm.stackrestore.p0(ptr %[[VAL_27]])
 
 // CHECK-LABEL: define { ptr, i64 } @_QFtest_proc_dummy_charPgen_message(ptr
-// CHECK-SAME:                %[[VAL_0:.*]], i64 %[[VAL_1:.*]], ptr nest %[[VAL_2:.*]])
+// CHECK-SAME:                nocapture %[[VAL_0:.*]], i64 %[[VAL_1:.*]], ptr nest nocapture %[[VAL_2:.*]])
 // CHECK:         %[[VAL_3:.*]] = getelementptr { { ptr, i64 } }, ptr %[[VAL_2]], i32 0, i32 0
 // CHECK:         %[[VAL_4:.*]] = load { ptr, i64 }, ptr %[[VAL_3]], align 8
 // CHECK:         %[[VAL_5:.*]] = extractvalue { ptr, i64 } %[[VAL_4]], 0
diff --git a/flang/test/Fir/commute.fir b/flang/test/Fir/commute.fir
index be25fd61953de..75338161d2d99 100644
--- a/flang/test/Fir/commute.fir
+++ b/flang/test/Fir/commute.fir
@@ -11,7 +11,7 @@ func.func @f1(%a : i32, %b : i32) -> i32 {
   return %3 : i32
 }
 
-// CHECK-LABEL: define i32 @f2(ptr %0)
+// CHECK-LABEL: define i32 @f2(ptr nocapture %0)
 func.func @f2(%a : !fir.ref<i32>) -> i32 {
   %1 = fir.load %a : !fir.ref<i32>
   // CHECK: %[[r2:.*]] = load
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 6e4ac824fbd9c..8e4e1fe824d9f 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -358,10 +358,10 @@ func.func @_QPopenmp_target_data_region() {
       %9 = arith.subi %8, %c1_i64 : i64
       %10 = fir.coordinate_of %0, %9 : (!fir.ref<!fir.array<1024xi32>>, i64) -> !fir.ref<i32>
       fir.store %6 to %10 : !fir.ref<i32>
-      %11 = arith.addi %arg0, %c1 : index
+      %11 = arith.addi %arg0, %c1 overflow<nsw> : index
       %12 = fir.convert %c1 : (index) -> i32
       %13 = fir.load %1 : !fir.ref<i32>
-      %14 = arith.addi %13, %12 : i32
+      %14 = arith.addi %13, %12 overflow<nsw> : i32
       fir.result %11, %14 : index, i32
     }
     fir.store %5#1 to %1 : !fir.ref<i32>
@@ -404,11 +404,11 @@ func.func @_QPopenmp_target_data_region() {
 // CHECK:             %[[VAL_21:.*]] = llvm.sub %[[VAL_19]], %[[VAL_20]]  : i64
 // CHECK:             %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_1]][0, %[[VAL_21]]] : (!llvm.ptr, i64) -> !llvm.ptr
 // CHECK:             llvm.store %[[VAL_17]], %[[VAL_22]] : i32, !llvm.ptr
-// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
+// CHECK:             %[[VAL_23:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
 // CHECK:             %[[VAL_24:.*]] = llvm.trunc %[[VAL_8]] : i64 to i32
 // CHECK:             %[[VAL_25:.*]] = llvm.load %[[VAL_3]] : !llvm.ptr -> i32
-// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]]  : i32
-// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]]  : i64
+// CHECK:             %[[VAL_26:.*]] = llvm.add %[[VAL_25]], %[[VAL_24]] overflow<nsw> : i32
+// CHECK:             %[[VAL_27:.*]] = llvm.add %[[VAL_12]], %[[VAL_8]] overflow<nsw> : i64
 // CHECK:             %[[VAL_28:.*]] = llvm.mlir.constant(1 : index) : i64
 // CHECK:             %[[VAL_29:.*]] = llvm.sub %[[VAL_14]], %[[VAL_28]]  : i64
 // CHECK:             llvm.br ^bb1(%[[VAL_27]], %[[VAL_26]], %[[VAL_29]] : i64, i32, i64)
diff --git a/flang/test/Fir/coordinateof.fir b/flang/test/Fir/coordinateof.fir
index acb9fd57c0955..f96f5b48bb355 100644
--- a/flang/test/Fir/coordinateof.fir
+++ b/flang/test/Fir/coordinateof.fir
@@ -62,7 +62,7 @@ func.func @foo5(%box : !fir.box<!fir.ptr<!fir.array<?xi32>>>, %i : index) -> i32
 }
 
 // CHECK-LABEL: @foo6
-// CHECK-SAME: (ptr %[[box:.*]], i64 %{{.*}}, ptr %{{.*}}) 
+// CHECK-SAME: (ptr %[[box:.*]], i64 %{{.*}}, ptr nocapture %{{.*}}) 
 func.func @foo6(%box : !fir.box<!fir.ptr<!fir.array<?x!fir.char<1>>>>, %i : i64 , %res : !fir.ref<!fir.char<1>>) {
   // CHECK: %[[addr_gep:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[box]], i32 0, i32 0
   // CHECK: %[[addr:.*]] = load ptr, ptr %[[addr_gep]]
diff --git a/flang/test/Fir/embox.fir b/flang/test/Fir/embox.fir
index cd13c5cfaaa9a..b21e1349b0552 100644
--- a/flang/test/Fir/embox.fir
+++ b/flang/test/Fir/embox.fir
@@ -64,7 +64,7 @@ func.func @_QPtest_dt_slice() {
 func.func private @takesRank2CharBox(!fir.box<!fir.array<?x?x!fir.char<1,?>>>)
 
 // CHECK-LABEL: define void @emboxSubstring(
-// CHECK-SAME: ptr %[[arg0:.*]])
+// CHECK-SAME: ptr nocapture %[[arg0:.*]])
 func.func @emboxSubstring(%arg0: !fir.ref<!fir.array<2x3x!fir.char<1,4>>>) {
   %c2 = arith.constant 2 : index
   %c3 = arith.constant 3 : index
@@ -85,7 +85,7 @@ func.func @emboxSubstring(%arg0: !fir.ref<!fir.array<2x3x!fir.char<1,4>>>) {
 
 func.func private @do_something(!fir.box<!fir.array<?xf32>>) -> ()
 // CHECK: define void @fir_dev_issue_1416
-// CHECK-SAME: ptr %[[base_addr:.*]], i64 %[[low:.*]], i64 %[[up:.*]], i64 %[[at:.*]])
+// CHECK-SAME: ptr nocapture %[[base_addr:.*]], i64 %[[low:.*]], i64 %[[up:.*]], i64 %[[at:.*]])
 func.func @fir_dev_issue_1416(%arg0: !fir.ref<!fir.array<40x?xf32>>, %low: index, %up: index, %at : index) {
     // Test fir.embox with a constant interior array shape.
     %c1 = arith.constant 1 : index
diff --git a/flang/test/Fir/field-index.fir b/flang/test/Fir/field-index.fir
index 8c9391f3901ec..08653752d71d3 100644
--- a/flang/test/Fir/field-index.fir
+++ b/flang/test/Fir/field-index.fir
@@ -7,7 +7,7 @@
 // CHECK-DAG: %[[c:.*]] = type { float, %[[b]] }
 
 // CHECK-LABEL: @simple_field
-// CHECK-SAME: (ptr %[[arg0:.*]])
+// CHECK-SAME: (ptr nocapture %[[arg0:.*]])
 func.func @simple_field(%arg0: !fir.ref<!fir.type<a{x:f32,i:i32}>>) -> i32 {
   %1 = fir.field_index i, !fir.type<a{x:f32,i:i32}>
   // CHECK: %[[GEP:.*]] = getelementptr %a, ptr %[[arg0]], i32 0, i32 1
@@ -18,7 +18,7 @@ func.func @simple_field(%arg0: !fir.ref<!fir.type<a{x:f32,i:i32}>>) -> i32 {
 }
 
 // CHECK-LABEL: @derived_field
-// CHECK-SAME: (ptr %[[arg0:.*]])
+// CHECK-SAME: (ptr nocapture %[[arg0:.*]])
 func.func @derived_field(%arg0: !fir.ref<!fir.type<c{x:f32,some_b:!fir.type<b{x:f32,i:i32}>}>>) -> i32 {
   %1 = fir.field_index some_b, !fir.type<c{x:f32,some_b:!fir.type<b{x:f32,i:i32}>}>
   %2 = fir.field_index i, !fir.type<b{x:f32,i:i32}>
diff --git a/flang/test/Fir/ignore-missing-type-descriptor.fir b/flang/test/Fir/ignore-missing-type-descriptor.fir
index 228d9a1c7af66..009e110a22b30 100644
--- a/flang/test/Fir/ignore-missing-type-descriptor.fir
+++ b/flang/test/Fir/ignore-missing-type-descriptor.fir
@@ -15,7 +15,7 @@ func.func @test_embox(%addr: !fir.ref<!some_freestyle_type>) {
   return
 }
 // CHECK-LABEL: define void @test_embox(
-// CHECK-SAME: ptr %[[ADDR:.*]])
+// CHECK-SAME: ptr nocapture %[[ADDR:.*]])
 // CHECK: insertvalue { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK-SAME: { ptr undef, i64 ptrtoint (ptr getelementptr (%some_not_mangled_type, ptr null, i32 1) to i64),
 // CHECK-SAME: i32 20240719, i8 0, i8 42, i8 0, i8 1, ptr null, [1 x i64] zeroinitializer },
diff --git a/flang/test/Fir/loop01.fir b/flang/test/Fir/loop01.fir
index c1cbb522c378c..30d10b9bbdb97 100644
--- a/flang/test/Fir/loop01.fir
+++ b/flang/test/Fir/loop01.fir
@@ -1,5 +1,7 @@
 // RUN: fir-opt --split-input-file --cfg-conversion %s | FileCheck %s
-// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=true" %s | FileCheck %s --check-prefix=NSW
+// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=false" %s | FileCheck %s --check-prefix=NO-NSW
+
+// NO-NSW-NOT: overflow<nsw>
 
 func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<index>) {
   fir.do_loop %iv = %lb to %ub step %step unordered {
@@ -35,7 +37,7 @@ func.func private @f2() -> i1
 // CHECK:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
 // CHECK:       br ^bb5
 // CHECK:     ^bb5:
-// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] : index
+// CHECK:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:       %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
@@ -44,34 +46,6 @@ func.func private @f2() -> i1
 // CHECK:     }
 // CHECK:     func private @f2() -> i1
 
-// NSW:     func @x(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: i1, %[[VAL_4:.*]]: !fir.ref<index>) {
-// NSW:       %[[VAL_5:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:       %[[VAL_6:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:       %[[VAL_7:.*]] = arith.divsi %[[VAL_6]], %[[VAL_2]] : index
-// NSW:       br ^bb1(%[[VAL_0]], %[[VAL_7]] : index, index)
-// NSW:     ^bb1(%[[VAL_8:.*]]: index, %[[VAL_9:.*]]: index):
-// NSW:       %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:       %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:       cond_br %[[VAL_11]], ^bb2, ^bb6
-// NSW:     ^bb2:
-// NSW:       cond_br %[[VAL_3]], ^bb3, ^bb4
-// NSW:     ^bb3:
-// NSW:       fir.store %[[VAL_8]] to %[[VAL_4]] : !fir.ref<index>
-// NSW:       br ^bb5
-// NSW:     ^bb4:
-// NSW:       %[[VAL_12:.*]] = arith.constant 0 : index
-// NSW:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
-// NSW:       br ^bb5
-// NSW:     ^bb5:
-// NSW:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
-// NSW:       %[[VAL_14:.*]] = arith.constant 1 : index
-// NSW:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
-// NSW:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
-// NSW:     ^bb6:
-// NSW:       return
-// NSW:     }
-// NSW:     func private @f2() -> i1
-
 // -----
 
 func.func @x2(%lo : index, %up : index, %ok : i1) {
@@ -101,36 +75,13 @@ func.func private @f3(i16)
 // CHECK:     cond_br %[[VAL_14]], ^bb2, ^bb3
 // CHECK:   ^bb2:
 // CHECK:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
-// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] : index
+// CHECK:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
 // CHECK:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
 // CHECK:   ^bb3:
 // CHECK:     return
 // CHECK:   }
 // CHECK:   func private @f3(i16)
 
-// NSW:   func @x2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: i1) {
-// NSW:     %[[VAL_3:.*]] = arith.constant 1 : index
-// NSW:     br ^bb1(%[[VAL_0]], %[[VAL_2]] : index, i1)
-// NSW:   ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
-// NSW:     %[[VAL_6:.*]] = arith.constant 0 : index
-// NSW:     %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_3]] : index
-// NSW:     %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
-// NSW:     %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_3]], %[[VAL_6]] : index
-// NSW:     %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
-// NSW:     %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
-// NSW:     %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:     %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:     %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
-// NSW:     cond_br %[[VAL_14]], ^bb2, ^bb3
-// NSW:   ^bb2:
-// NSW:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
-// NSW:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
-// NSW:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
-// NSW:   ^bb3:
-// NSW:     return
-// NSW:   }
-// NSW:   func private @f3(i16)
-
 // -----
 
 // do_loop with an extra loop-carried value
@@ -159,7 +110,7 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           cond_br %[[VAL_11]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
 // CHECK:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
@@ -167,29 +118,6 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           return %[[VAL_8]] : i1
 // CHECK:         }
 
-// NSW-LABEL:   func @x3(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i1, index)
-// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i1, %[[VAL_9:.*]]: index):
-// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_14:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
-// NSW:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_8]] : i1
-// NSW:         }
-
 // -----
 
 // iterate_while with an extra loop-carried value
@@ -227,7 +155,7 @@ func.func private @f4(i32) -> i1
 // CHECK:           cond_br %[[VAL_16]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
 // CHECK:         ^bb3:
 // CHECK:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
@@ -235,34 +163,6 @@ func.func private @f4(i32) -> i1
 // CHECK:         }
 // CHECK:         func private @f4(i32) -> i1
 
-// NSW-LABEL:   func @y3(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           %[[VAL_4:.*]] = fir.call @f2() : () -> i1
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_4]] : index, i1, i1)
-// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i1):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
-// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
-// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
-// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
-// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
-// NSW:         ^bb3:
-// NSW:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
-// NSW:           return %[[VAL_19]] : i1
-// NSW:         }
-// NSW:         func private @f4(i32) -> i1
-
 // -----
 
 // do_loop that returns the final value of the induction
@@ -291,7 +191,7 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
 // CHECK:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
-// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_13:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
 // CHECK:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
@@ -299,29 +199,6 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_6]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @x4(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_2]] : index
-// NSW:           %[[VAL_5:.*]] = arith.divsi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_5]] : index, index)
-// NSW:         ^bb1(%[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_7]], %[[VAL_8]] : index
-// NSW:           cond_br %[[VAL_9]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
-// NSW:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
-// NSW:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_13:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
-// NSW:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_6]] : index
-// NSW:         }
-
 // -----
 
 // iterate_while that returns the final value of both inductions
@@ -356,38 +233,12 @@ func.func @y4(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
 // CHECK:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
-// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
 // CHECK:         ^bb3:
 // CHECK:           return %[[VAL_4]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @y4(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant true
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]] : index, i1)
-// NSW:         ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
-// NSW:           %[[VAL_6:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_2]] : index
-// NSW:           %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_6]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
-// NSW:           %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
-// NSW:           %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
-// NSW:           cond_br %[[VAL_14]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// NSW:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
-// NSW:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
-// NSW:         ^bb3:
-// NSW:           return %[[VAL_4]] : index
-// NSW:         }
-
 // -----
 
 // do_loop that returns the final induction value
@@ -420,7 +271,7 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
@@ -429,31 +280,6 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_7]] : index
 // CHECK:         }
 
-// NSW-LABEL:   func @x5(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
-// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
-// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
-// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i16, index)
-// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i16, %[[VAL_9:.*]]: index):
-// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
-// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
-// NSW:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           %[[VAL_15:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
-// NSW:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
-// NSW:         ^bb3:
-// NSW:           fir.call @f3(%[[VAL_8]]) : (i16) -> ()
-// NSW:           return %[[VAL_7]] : index
-// NSW:         }
-
 // -----
 
 // iterate_while that returns the both induction values
@@ -496,7 +322,7 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:         ^bb2:
 // CHECK:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
 // CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
-// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
 // CHECK:         ^bb3:
 // CHECK:           cond_br %[[VAL_6]], ^bb4, ^bb5
@@ -508,37 +334,3 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
 // CHECK:           return %[[VAL_5]] : index
 // CHECK:         }
-
-// NSW-LABEL:   func @y5(
-// NSW-SAME:             %[[VAL_0:.*]]: index,
-// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
-// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
-// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
-// NSW:           %[[VAL_4:.*]] = arith.constant true
-// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_4]], %[[VAL_3]] : index, i1, i16)
-// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i16):
-// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
-// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
-// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
-// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
-// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
-// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
-// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
-// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
-// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
-// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
-// NSW:         ^bb2:
-// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
-// NSW:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
-// NSW:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
-// NSW:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
-// NSW:         ^bb3:
-// NSW:           cond_br %[[VAL_6]], ^bb4, ^bb5
-// NSW:         ^bb4:
-// NSW:           %[[VAL_20:.*]] = arith.constant 0 : i32
-// NSW:           %[[VAL_21:.*]] = fir.call @f4(%[[VAL_20]]) : (i32) -> i1
-// NSW:           br ^bb5
-// NSW:         ^bb5:
-// NSW:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
-// NSW:           return %[[VAL_5]] : index
-// NSW:         }
diff --git a/flang/test/Fir/loop02.fir b/flang/test/Fir/loop02.fir
index 50948e0e7aa6b..fb209a9dfeb42 100644
--- a/flang/test/Fir/loop02.fir
+++ b/flang/test/Fir/loop02.fir
@@ -31,7 +31,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // CHECK:           cond_br %[[VAL_13]], ^bb2, ^bb3
 // CHECK:         ^bb2:
 // CHECK:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]], %[[VAL_2]] overflow<nsw> : index
 // CHECK:           %[[VAL_15:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_16:.*]] = arith.subi %[[VAL_11]], %[[VAL_15]] : index
 // CHECK:           br ^bb1(%[[VAL_14]], %[[VAL_16]] : index, index)
@@ -54,7 +54,7 @@ func.func private @y(%addr : !fir.ref<index>)
 // NOOPT:           cond_br %[[VAL_9]], ^bb2, ^bb3
 // NOOPT:         ^bb2:
 // NOOPT:           fir.call @y(%[[VAL_0]]) : (!fir.ref<index>) -> ()
-// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] : index
+// NOOPT:           %[[VAL_10:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
 // NOOPT:           %[[VAL_11:.*]] = arith.constant 1 : index
 // NOOPT:           %[[VAL_12:.*]] = arith.subi %[[VAL_7]], %[[VAL_11]] : index
 // NOOPT:           br ^bb1(%[[VAL_10]], %[[VAL_12]] : index, index)
diff --git a/flang/test/Fir/polymorphic.fir b/flang/test/Fir/polymorphic.fir
index 78e5b8dcf84c7..74b29ed6ca729 100644
--- a/flang/test/Fir/polymorphic.fir
+++ b/flang/test/Fir/polymorphic.fir
@@ -175,7 +175,7 @@ func.func @_QMmod2Pinitp(%arg0: !fir.ref<!fir.class<!fir.ptr<none>>> {fir.bindc_
 func.func private @_FortranAPointerAssociate(!fir.ref<!fir.box<none>>, !fir.box<none>) -> none attributes {fir.runtime}
 
 // CHECK-LABEL: define void @_QMmod2Pinitp(
-// CHECK-SAME: ptr %[[ARG0:.*]]){{.*}}{
+// CHECK-SAME: ptr nocapture %[[ARG0:.*]]){{.*}}{
 // CHECK: %[[ALLOCA_CLASS_NONE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }
 // CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[ALLOCA_CLASS_NONE]], ptr %[[ARG0]], i32 40, i1 false)
 // CHECK: %{{.*}} = call {} @_FortranAPointerAssociate(ptr @_QMmod2Ep, ptr %[[ALLOCA_CLASS_NONE]])
diff --git a/flang/test/Fir/struct-passing-loongarch64-byreg.fir b/flang/test/Fir/struct-passing-loongarch64-byreg.fir
new file mode 100644
index 0000000000000..585884504aacf
--- /dev/null
+++ b/flang/test/Fir/struct-passing-loongarch64-byreg.fir
@@ -0,0 +1,232 @@
+/// Test LoongArch64 ABI rewrite of struct passed by value (BIND(C), VALUE derived types).
+/// This test test cases where the struct can be passed in registers.
+/// Test cases can be roughly divided into two categories:
+///  - struct with a single intrinsic component;
+///  - sturct with more than one field;
+/// Since the argument marshalling logic is largely the same within each category,
+/// only the first example in each category checks the entire invocation process,
+/// while the other examples only check the signatures.
+
+// REQUIRES: loongarch-registered-target
+// RUN: fir-opt --split-input-file --target-rewrite="target=loongarch64-unknown-linux-gnu" %s | FileCheck %s
+
+
+/// *********************** Struct with a single intrinsic component *********************** ///
+
+!ty_i16   = !fir.type<ti16{i:i16}>
+!ty_i32   = !fir.type<ti32{i:i32}>
+!ty_i64   = !fir.type<ti64{i:i64}>
+!ty_i128  = !fir.type<ti128{i:i128}>
+!ty_f16   = !fir.type<tf16{i:f16}>
+!ty_f32   = !fir.type<tf32{i:f32}>
+!ty_f64   = !fir.type<tf64{i:f64}>
+!ty_f128  = !fir.type<tf128{i:f128}>
+!ty_bf16  = !fir.type<tbf16{i:bf16}>
+!ty_char1 = !fir.type<tchar1{i:!fir.char<1>}>
+!ty_char2 = !fir.type<tchar2{i:!fir.char<1,2>}>
+!ty_log1  = !fir.type<tlog1{i:!fir.logical<1>}>
+!ty_log2  = !fir.type<tlog2{i:!fir.logical<2>}>
+!ty_log4  = !fir.type<tlog4{i:!fir.logical<4>}>
+!ty_log8  = !fir.type<tlog8{i:!fir.logical<8>}>
+!ty_log16 = !fir.type<tlog16{i:!fir.logical<16>}>
+!ty_cmplx_f32  = !fir.type<tcmplx_f32{i:complex<f32>}>
+!ty_cmplx_f64  = !fir.type<tcmplx_f64{i:complex<f64>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
+// CHECK-LABEL: func.func private @test_func_i16(i64)
+func.func private @test_func_i16(%arg0: !ty_i16)
+// CHECK-LABEL: func.func @test_call_i16(
+// CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.type<ti16{i:i16}>>) {
+func.func @test_call_i16(%arg0: !fir.ref<!ty_i16>) {
+  // CHECK: %[[IN:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.type<ti16{i:i16}>>
+  // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
+  // CHECK: %[[ARR:.*]] = fir.alloca i64
+  // CHECK: %[[CVT:.*]] = fir.convert %[[ARR]] : (!fir.ref<i64>) -> !fir.ref<!fir.type<ti16{i:i16}>>
+  // CHECK: fir.store %[[IN]] to %[[CVT]] : !fir.ref<!fir.type<ti16{i:i16}>>
+  // CHECK: %[[LD:.*]] = fir.load %[[ARR]] : !fir.ref<i64>
+  %in = fir.load %arg0 : !fir.ref<!ty_i16>
+  // CHECK: fir.call @test_func_i16(%[[LD]]) : (i64) -> ()
+  // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
+  fir.call @test_func_i16(%in) : (!ty_i16) -> ()
+  // CHECK: return
+  return
+}
+
+// CHECK-LABEL: func.func private @test_func_i32(i64)
+func.func private @test_func_i32(%arg0: !ty_i32)
+
+// CHECK-LABEL: func.func private @test_func_i64(i64)
+func.func private @test_func_i64(%arg0: !ty_i64)
+
+// CHECK-LABEL: func.func private @test_func_i128(i128)
+func.func private @test_func_i128(%arg0: !ty_i128)
+
+// CHECK-LABEL: func.func private @test_func_f16(i64)
+func.func private @test_func_f16(%arg0: !ty_f16)
+
+// CHECK-LABEL: func.func private @test_func_f32(f32)
+func.func private @test_func_f32(%arg0: !ty_f32)
+
+// CHECK-LABEL: func.func private @test_func_f64(f64)
+func.func private @test_func_f64(%arg0: !ty_f64)
+
+// CHECK-LABEL: func.func private @test_func_f128(i128)
+func.func private @test_func_f128(%arg0: !ty_f128)
+
+// CHECK-LABEL: func.func private @test_func_bf16(i64)
+func.func private @test_func_bf16(%arg0: !ty_bf16)
+
+// CHECK-LABEL: func.func private @test_func_char1(i64)
+func.func private @test_func_char1(%arg0: !ty_char1)
+
+// CHECK-LABEL: func.func private @test_func_char2(i64)
+func.func private @test_func_char2(%arg0: !ty_char2)
+
+// CHECK-LABEL: func.func private @test_func_log1(i64)
+func.func private @test_func_log1(%arg0: !ty_log1)
+
+// CHECK-LABEL: func.func private @test_func_log2(i64)
+func.func private @test_func_log2(%arg0: !ty_log2)
+
+// CHECK-LABEL: func.func private @test_func_log4(i64)
+func.func private @test_func_log4(%arg0: !ty_log4)
+
+// CHECK-LABEL: func.func private @test_func_log8(i64)
+func.func private @test_func_log8(%arg0: !ty_log8)
+
+// CHECK-LABEL: func.func private @test_func_log16(i128)
+func.func private @test_func_log16(%arg0: !ty_log16)
+
+// CHECK-LABEL: func.func private @test_func_cmplx_f32(f32, f32)
+func.func private @test_func_cmplx_f32(%arg0: !ty_cmplx_f32)
+
+// CHECK-LABEL: func.func private @test_func_cmplx_f64(f64, f64)
+func.func private @test_func_cmplx_f64(%arg0: !ty_cmplx_f64)
+}
+
+
+/// *************************** Struct with more than one field **************************** ///
+
+// -----
+
+!ty_i32_f32 = !fir.type<ti32_f32{i:i32,j:f32}>
+!ty_i32_f64 = !fir.type<ti32_f64{i:i32,j:f64}>
+!ty_i64_f32 = !fir.type<ti64_f32{i:i64,j:f32}>
+!ty_i64_f64 = !fir.type<ti64_f64{i:i64,j:f64}>
+!ty_f64_i64 = !fir.type<tf64_i64{i:f64,j:i64}>
+!ty_f16_f16 = !fir.type<tf16_f16{i:f16,j:f16}>
+!ty_f32_f32 = !fir.type<tf32_f32{i:f32,j:f32}>
+!ty_f64_f64 = !fir.type<tf64_f64{i:f64,j:f64}>
+!ty_f32_i32_i32 = !fir.type<tf32_i32_i32{i:f32,j:i32,k:i32}>
+!ty_f32_f32_i32 = !fir.type<tf32_f32_i32{i:f32,j:f32,k:i32}>
+!ty_f32_f32_f32 = !fir.type<tf32_f32_f32{i:f32,j:f32,k:f32}>
+
+!ty_i8_a8  = !fir.type<ti8_a8{i:!fir.array<8xi8>}>
+!ty_i8_a16 = !fir.type<ti8_a16{i:!fir.array<16xi8>}>
+!ty_f32_a2 = !fir.type<tf32_a2{i:!fir.array<2xf32>}>
+!ty_f64_a2 = !fir.type<tf64_a2{i:!fir.array<2xf64>}>
+!ty_nested_i32_f32 = !fir.type<t11{i:!ty_i32_f32}>
+!ty_nested_i8_a8_i32 = !fir.type<t12{i:!ty_i8_a8, j:i32}>
+!ty_char1_a8 = !fir.type<t_char_a8{i:!fir.array<8x!fir.char<1>>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
+// CHECK-LABEL: func.func private @test_func_i32_f32(i32, f32)
+func.func private @test_func_i32_f32(%arg0: !ty_i32_f32)
+// CHECK-LABEL: func.func @test_call_i32_f32(
+// CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>) {
+func.func @test_call_i32_f32(%arg0: !fir.ref<!ty_i32_f32>) {
+  // CHECK: %[[IN:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
+  // CHECK: %[[ARR:.*]] = fir.alloca tuple<i32, f32>
+  // CHECK: %[[CVT:.*]] = fir.convert %[[ARR]] : (!fir.ref<tuple<i32, f32>>) -> !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  // CHECK: fir.store %[[IN]] to %[[CVT]] : !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  // CHECK: %[[LD:.*]] = fir.load %[[ARR]] : !fir.ref<tuple<i32, f32>>
+  // CHECK: %[[VAL_0:.*]] = fir.extract_value %[[LD]], [0 : i32] : (tuple<i32, f32>) -> i32
+  // CHECK: %[[VAL_1:.*]] = fir.extract_value %[[LD]], [1 : i32] : (tuple<i32, f32>) -> f32
+  %in = fir.load %arg0 : !fir.ref<!ty_i32_f32>
+  // CHECK: fir.call @test_func_i32_f32(%[[VAL_0]], %[[VAL_1]]) : (i32, f32) -> ()
+  // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
+  fir.call @test_func_i32_f32(%in) : (!ty_i32_f32) -> ()
+  // CHECK: return
+  return
+}
+
+// CHECK-LABEL: func.func private @test_func_i32_f64(i32, f64)
+func.func private @test_func_i32_f64(%arg0: !ty_i32_f64)
+
+// CHECK-LABEL: func.func private @test_func_i64_f32(i64, f32)
+func.func private @test_func_i64_f32(%arg0: !ty_i64_f32)
+
+// CHECK-LABEL: func.func private @test_func_i64_f64(i64, f64)
+func.func private @test_func_i64_f64(%arg0: !ty_i64_f64)
+
+// CHECK-LABEL: func.func private @test_func_f64_i64(f64, i64)
+func.func private @test_func_f64_i64(%arg0: !ty_f64_i64)
+
+// CHECK-LABEL: func.func private @test_func_f16_f16(i64)
+func.func private @test_func_f16_f16(%arg0: !ty_f16_f16)
+
+// CHECK-LABEL: func.func private @test_func_f32_f32(f32, f32)
+func.func private @test_func_f32_f32(%arg0: !ty_f32_f32)
+
+// CHECK-LABEL: func.func private @test_func_f64_f64(f64, f64)
+func.func private @test_func_f64_f64(%arg0: !ty_f64_f64)
+
+// CHECK-LABEL: func.func private @test_func_f32_i32_i32(!fir.array<2xi64>)
+func.func private @test_func_f32_i32_i32(%arg0: !ty_f32_i32_i32)
+
+// CHECK-LABEL: func.func private @test_func_f32_f32_i32(!fir.array<2xi64>)
+func.func private @test_func_f32_f32_i32(%arg0: !ty_f32_f32_i32)
+
+// CHECK-LABEL: func.func private @test_func_f32_f32_f32(!fir.array<2xi64>)
+func.func private @test_func_f32_f32_f32(%arg0: !ty_f32_f32_f32)
+
+// CHECK-LABEL: func.func private @test_func_i8_a8(i64)
+func.func private @test_func_i8_a8(%arg0: !ty_i8_a8)
+
+// CHECK-LABEL: func.func private @test_func_i8_a16(!fir.array<2xi64>)
+func.func private @test_func_i8_a16(%arg0: !ty_i8_a16)
+
+// CHECK-LABEL: func.func private @test_func_f32_a2(f32, f32)
+func.func private @test_func_f32_a2(%arg0: !ty_f32_a2)
+
+// CHECK-LABEL: func.func private @test_func_f64_a2(f64, f64)
+func.func private @test_func_f64_a2(%arg0: !ty_f64_a2)
+
+// CHECK-LABEL: func.func private @test_func_nested_i32_f32(i32, f32)
+func.func private @test_func_nested_i32_f32(%arg0: !ty_nested_i32_f32)
+
+// CHECK-LABEL: func.func private @test_func_nested_i8_a8_i32(!fir.array<2xi64>)
+func.func private @test_func_nested_i8_a8_i32(%arg0: !ty_nested_i8_a8_i32)
+
+
+// CHECK: func.func private @not_enough_int_reg_1(i32, i32, i32, i32, i32, i32, i32, i32, i64)
+func.func private @not_enough_int_reg_1(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32,
+                                        %arg5: i32, %arg6: i32, %arg7: i32, %arg8: !ty_i32_f32)
+
+// CHECK: func.func private @not_enough_int_reg_1b(!fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>, i64)
+func.func private @not_enough_int_reg_1b(%arg0: !fir.ref<i32>, %arg1: !fir.ref<i32>, %arg2: !fir.ref<i32>, %arg3: !fir.ref<i32>, %arg4: !fir.ref<i32>,
+                                         %arg5: !fir.ref<i32>, %arg6: !fir.ref<i32>, %arg7: !fir.ref<i32>, %arg8: !ty_i32_f32)
+
+// CHECK: func.func private @not_enough_int_reg_2(i32, i32, i32, i32, i32, i32, i32, i32, !fir.array<2xi64>)
+func.func private @not_enough_int_reg_2(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32,
+                                        %arg5: i32, %arg6: i32, %arg7: i32, %arg8: !ty_i64_f64)
+
+// CHECK: func.func private @not_enough_fp_reg_1(f32, f32, f32, f32, f32, f32, f32, f32, i64)
+func.func private @not_enough_fp_reg_1(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32,
+                                        %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !ty_i32_f32)
+
+// CHECK: func.func private @not_enough_fp_reg_1b(!fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, !fir.ref<f32>, i64)
+func.func private @not_enough_fp_reg_1b(%arg0: !fir.ref<f32>, %arg1: !fir.ref<f32>, %arg2: !fir.ref<f32>, %arg3: !fir.ref<f32>, %arg4: !fir.ref<f32>,
+                                        %arg5: !fir.ref<f32>, %arg6: !fir.ref<f32>, %arg7: !fir.ref<f32>, %arg8: !ty_i32_f32)
+
+// CHECK: func.func private @not_enough_fp_reg_2(f32, f32, f32, f32, f32, f32, f32, f32, !fir.array<2xi64>)
+func.func private @not_enough_fp_reg_2(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32,
+                                        %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !ty_i64_f64)
+
+// CHECK: func.func private @char_not_enough_int_reg(i32, i32, i32, i32, i32, i32, i32, i32, i64)
+func.func private @char_not_enough_int_reg(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32,
+                                        %arg5: i32, %arg6: i32, %arg7: i32, %arg8: !ty_char1_a8)
+}
diff --git a/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir b/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir
new file mode 100644
index 0000000000000..5041a39e69798
--- /dev/null
+++ b/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir
@@ -0,0 +1,80 @@
+/// Test LoongArch64 ABI rewrite of struct passed and returned by value (BIND(C), VALUE derived types).
+/// This test test cases where the struct must be passed or returned on the stack.
+
+// REQUIRES: loongarch-registered-target
+// RUN: tco --target=loongarch64-unknown-linux-gnu %s | FileCheck %s
+
+!ty_int_toobig = !fir.type<int_toobig{i:!fir.array<5xi32>}>
+!ty_int_toobig_align16 = !fir.type<int_toobig_align16{i:i128,j:i8}>
+!ty_fp_toobig = !fir.type<fp_toobig{i:!fir.array<5xf64>}>
+!ty_fp_toobig_align16 = !fir.type<fp_toobig_align16{i:f128,j:f32}>
+
+!ty_i32_f32 = !fir.type<i32_f32{i:i32,j:f32}>
+!ty_nested_toobig = !fir.type<nested_toobig{i:!fir.array<3x!ty_i32_f32>}>
+!ty_badly_aligned = !fir.type<badly_aligned{i:f32,j:f64,k:f32}>
+!ty_logical_toobig = !fir.type<logical_toobig{i:!fir.array<17x!fir.logical<1>>}>
+!ty_cmplx_toobig = !fir.type<cmplx_toobig{i:!fir.array<4xcomplex<f32>>}>
+!ty_char_toobig = !fir.type<char_toobig{i:!fir.array<17x!fir.char<1>>}>
+!ty_cmplx_f128 = !fir.type<cmplx_f128{i:complex<f128>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
+// CHECK: declare void @takes_int_toobig(ptr byval(%int_toobig) align 4)
+func.func private @takes_int_toobig(%arg0: !ty_int_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_int_toobig(ptr sret(%int_toobig) align 4)
+func.func private @return_int_toobig() -> !ty_int_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_int_toobig_align16(ptr byval(%int_toobig_align16) align 16)
+func.func private @takes_int_toobig_align16(%arg0: !ty_int_toobig_align16) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_int_toobig_align16(ptr sret(%int_toobig_align16) align 16)
+func.func private @return_int_toobig_align16() -> !ty_int_toobig_align16 attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_fp_toobig(ptr byval(%fp_toobig) align 8)
+func.func private @takes_fp_toobig(%arg0: !ty_fp_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_fp_toobig(ptr sret(%fp_toobig) align 8)
+func.func private @return_fp_toobig() -> !ty_fp_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_fp_toobig_align16(ptr byval(%fp_toobig_align16) align 16)
+func.func private @takes_fp_toobig_align16(%arg0: !ty_fp_toobig_align16) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_fp_toobig_align16(ptr sret(%fp_toobig_align16) align 16)
+func.func private @return_fp_toobig_align16() -> !ty_fp_toobig_align16 attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_nested_toobig(ptr byval(%nested_toobig) align 4)
+func.func private @takes_nested_toobig(%arg0: !ty_nested_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_nested_toobig(ptr sret(%nested_toobig) align 4)
+func.func private @return_nested_toobig() -> !ty_nested_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_badly_aligned(ptr byval(%badly_aligned) align 8)
+func.func private @takes_badly_aligned(%arg0: !ty_badly_aligned) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_badly_aligned(ptr sret(%badly_aligned) align 8)
+func.func private @return_badly_aligned() -> !ty_badly_aligned attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_logical_toobig(ptr byval(%logical_toobig) align 1)
+func.func private @takes_logical_toobig(%arg0: !ty_logical_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_logical_toobig(ptr sret(%logical_toobig) align 1)
+func.func private @return_logical_toobig() -> !ty_logical_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_cmplx_toobig(ptr byval(%cmplx_toobig) align 4)
+func.func private @takes_cmplx_toobig(%arg0: !ty_cmplx_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_cmplx_toobig(ptr sret(%cmplx_toobig) align 4)
+func.func private @return_cmplx_toobig() -> !ty_cmplx_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_char_toobig(ptr byval(%char_toobig) align 1)
+func.func private @takes_char_toobig(%arg0: !ty_char_toobig) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+// CHECK: declare void @return_char_toobig(ptr sret(%char_toobig) align 1)
+func.func private @return_char_toobig() -> !ty_char_toobig attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare { i32, float } @takes_and_return(float, float, float, float, float, float, float, float, ptr byval(%cmplx_f128) align 16)
+func.func private @takes_and_return(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32,
+                                    %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !ty_cmplx_f128) -> !ty_i32_f32 attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_and_return2(ptr sret(%cmplx_f128) align 16, i32, i32, i32, i32, i32, i32, i32, i32, i64)
+func.func private @takes_and_return2(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32,
+                                     %arg5: i32, %arg6: i32, %arg7: i32, %arg8: !ty_i32_f32) -> !ty_cmplx_f128 attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+// CHECK: declare void @takes_multi_byval_arguments(ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, ptr byval(%cmplx_f128) align 16, i32, float, i64)
+func.func private @takes_multi_byval_arguments(%arg0: !ty_cmplx_f128, %arg1: !ty_cmplx_f128, %arg2: !ty_cmplx_f128, %arg3: !ty_cmplx_f128,
+                                               %arg4: !ty_cmplx_f128, %arg5: !ty_cmplx_f128, %arg6: !ty_cmplx_f128,
+                                               %arg7: !ty_i32_f32, %arg8: !ty_i32_f32) attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+}
+
diff --git a/flang/test/Fir/struct-passing-x86-64-byval.fir b/flang/test/Fir/struct-passing-x86-64-byval.fir
index b6869014dbdde..44256c0c0a048 100644
--- a/flang/test/Fir/struct-passing-x86-64-byval.fir
+++ b/flang/test/Fir/struct-passing-x86-64-byval.fir
@@ -81,27 +81,27 @@ func.func @not_enough_int_reg_3(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32,
 }
 }
 
-// CHECK: define void @takes_toobig(ptr byval(%toobig) align 8 %{{.*}}) {
-// CHECK: define void @takes_toobig_align16(ptr byval(%toobig_align16) align 16 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_1b(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_2(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_2_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @ftakes_toobig(ptr byval(%ftoobig) align 8 %{{.*}}) {
-// CHECK: define void @ftakes_toobig_align16(ptr byval(%ftoobig_align16) align 16 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1b(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_1c(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, ptr byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%fits_in_2_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_contains_x87(ptr byval(%contains_x87) align 16 %{{.*}}) {
-// CHECK: define void @test_contains_complex_x87(ptr byval(%contains_complex_x87) align 16 %{{.*}}) {
-// CHECK: define void @test_nested_toobig(ptr byval(%nested_toobig) align 8 %{{.*}}) {
-// CHECK: define void @test_badly_aligned(ptr byval(%badly_aligned) align 8 %{{.*}}) {
-// CHECK: define void @test_logical_toobig(ptr byval(%logical_too_big) align 8 %{{.*}}) {
-// CHECK: define void @l_not_enough_int_reg(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%l_fits_in_2_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_complex_toobig(ptr byval(%complex_too_big) align 8 %{{.*}}) {
-// CHECK: define void @cplx_not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%cplx_fits_in_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @test_char_to_big(ptr byval(%char_too_big) align 8 %{{.*}}) {
-// CHECK: define void @char_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%char_fits_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @mix_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%mix_in_1_int_reg) align 8 %{{.*}}) {
-// CHECK: define void @mix_not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr byval(%mix_in_1_int_reg_1_sse_reg) align 8 %{{.*}}) {
-// CHECK: define void @not_enough_int_reg_3(ptr sret({ fp128, fp128 }) align 16 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr byval(%fits_in_1_int_reg) align 8 %{{.*}})
+// CHECK: define void @takes_toobig(ptr nocapture byval(%toobig) align 8 %{{.*}}) {
+// CHECK: define void @takes_toobig_align16(ptr nocapture byval(%toobig_align16) align 16 %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_1b(ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_2(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_2_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @ftakes_toobig(ptr nocapture byval(%ftoobig) align 8 %{{.*}}) {
+// CHECK: define void @ftakes_toobig_align16(ptr nocapture byval(%ftoobig_align16) align 16 %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1b(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_1c(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, ptr nocapture byval(%fits_in_1_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%fits_in_2_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @test_contains_x87(ptr nocapture byval(%contains_x87) align 16 %{{.*}}) {
+// CHECK: define void @test_contains_complex_x87(ptr nocapture byval(%contains_complex_x87) align 16 %{{.*}}) {
+// CHECK: define void @test_nested_toobig(ptr nocapture byval(%nested_toobig) align 8 %{{.*}}) {
+// CHECK: define void @test_badly_aligned(ptr nocapture byval(%badly_aligned) align 8 %{{.*}}) {
+// CHECK: define void @test_logical_toobig(ptr nocapture byval(%logical_too_big) align 8 %{{.*}}) {
+// CHECK: define void @l_not_enough_int_reg(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%l_fits_in_2_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @test_complex_toobig(ptr nocapture byval(%complex_too_big) align 8 %{{.*}}) {
+// CHECK: define void @cplx_not_enough_sse_reg_1(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%cplx_fits_in_1_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @test_char_to_big(ptr nocapture byval(%char_too_big) align 8 %{{.*}}) {
+// CHECK: define void @char_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%char_fits_in_1_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @mix_not_enough_int_reg_1(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%mix_in_1_int_reg) align 8 %{{.*}}) {
+// CHECK: define void @mix_not_enough_sse_reg_2(float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, ptr nocapture byval(%mix_in_1_int_reg_1_sse_reg) align 8 %{{.*}}) {
+// CHECK: define void @not_enough_int_reg_3(ptr nocapture sret({ fp128, fp128 }) align 16 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, ptr nocapture byval(%fits_in_1_int_reg) align 8 %{{.*}})
diff --git a/flang/test/Fir/struct-return-loongarch64-byreg.fir b/flang/test/Fir/struct-return-loongarch64-byreg.fir
new file mode 100644
index 0000000000000..eb3d4d50d8866
--- /dev/null
+++ b/flang/test/Fir/struct-return-loongarch64-byreg.fir
@@ -0,0 +1,200 @@
+/// Test LoongArch64 ABI rewrite of struct returned by value (BIND(C), VALUE derived types).
+/// This test test cases where the struct can be returned in registers.
+/// Test cases can be roughly divided into two categories:
+///  - struct with a single intrinsic component;
+///  - sturct with more than one field;
+/// Since the argument marshalling logic is largely the same within each category,
+/// only the first example in each category checks the entire invocation process,
+/// while the other examples only check the signatures.
+
+// REQUIRES: loongarch-registered-target
+// RUN: fir-opt --split-input-file --target-rewrite="target=loongarch64-unknown-linux-gnu" %s | FileCheck %s
+
+
+/// *********************** Struct with a single intrinsic component *********************** ///
+
+!ty_i16   = !fir.type<ti16{i:i16}>
+!ty_i32   = !fir.type<ti32{i:i32}>
+!ty_i64   = !fir.type<ti64{i:i64}>
+!ty_i128  = !fir.type<ti128{i:i128}>
+!ty_f16   = !fir.type<tf16{i:f16}>
+!ty_f32   = !fir.type<tf32{i:f32}>
+!ty_f64   = !fir.type<tf64{i:f64}>
+!ty_f128  = !fir.type<tf128{i:f128}>
+!ty_bf16  = !fir.type<tbf16{i:bf16}>
+!ty_char1 = !fir.type<tchar1{i:!fir.char<1>}>
+!ty_char2 = !fir.type<tchar2{i:!fir.char<1,2>}>
+!ty_log1  = !fir.type<tlog1{i:!fir.logical<1>}>
+!ty_log2  = !fir.type<tlog2{i:!fir.logical<2>}>
+!ty_log4  = !fir.type<tlog4{i:!fir.logical<4>}>
+!ty_log8  = !fir.type<tlog8{i:!fir.logical<8>}>
+!ty_log16 = !fir.type<tlog16{i:!fir.logical<16>}>
+!ty_cmplx_f32  = !fir.type<tcmplx_f32{i:complex<f32>}>
+!ty_cmplx_f64  = !fir.type<tcmplx_f64{i:complex<f64>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
+// CHECK-LABEL: func.func private @test_func_i16() -> i64
+func.func private @test_func_i16() -> !ty_i16
+// CHECK-LABEL: func.func @test_call_i16(
+// CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.type<ti16{i:i16}>>) {
+func.func @test_call_i16(%arg0: !fir.ref<!ty_i16>) {
+  // CHECK: %[[OUT:.*]] = fir.call @test_func_i16() : () -> i64
+  // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
+  // CHECK: %[[ARR:.*]] = fir.alloca i64
+  // CHECK: fir.store %[[OUT]] to %[[ARR]] : !fir.ref<i64>
+  // CHECK: %[[CVT:.*]] = fir.convert %[[ARR]] : (!fir.ref<i64>) -> !fir.ref<!fir.type<ti16{i:i16}>>
+  // CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<ti16{i:i16}>>
+  // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
+  %out = fir.call @test_func_i16() : () -> !ty_i16
+  // CHECK: fir.store %[[LD]] to %[[ARG0]] : !fir.ref<!fir.type<ti16{i:i16}>>
+  fir.store %out to %arg0 : !fir.ref<!ty_i16>
+  // CHECK: return
+  return
+}
+
+// CHECK-LABEL: func.func private @test_func_i32() -> i64
+func.func private @test_func_i32() -> !ty_i32
+
+// CHECK-LABEL: func.func private @test_func_i64() -> i64
+func.func private @test_func_i64() -> !ty_i64
+
+// CHECK-LABEL: func.func private @test_func_i128() -> i128
+func.func private @test_func_i128() -> !ty_i128
+
+// CHECK-LABEL: func.func private @test_func_f16() -> i64
+func.func private @test_func_f16() -> !ty_f16
+
+// CHECK-LABEL: func.func private @test_func_f32() -> f32
+func.func private @test_func_f32() -> !ty_f32
+
+// CHECK-LABEL: func.func private @test_func_f64() -> f64
+func.func private @test_func_f64() -> !ty_f64
+
+// CHECK-LABEL: func.func private @test_func_f128() -> i128
+func.func private @test_func_f128() -> !ty_f128
+
+// CHECK-LABEL: func.func private @test_func_bf16() -> i64
+func.func private @test_func_bf16() -> !ty_bf16
+
+// CHECK-LABEL: func.func private @test_func_char1() -> i64
+func.func private @test_func_char1() -> !ty_char1
+
+// CHECK-LABEL: func.func private @test_func_char2() -> i64
+func.func private @test_func_char2() -> !ty_char2
+
+// CHECK-LABEL: func.func private @test_func_log1() -> i64
+func.func private @test_func_log1() -> !ty_log1
+
+// CHECK-LABEL: func.func private @test_func_log2() -> i64
+func.func private @test_func_log2() -> !ty_log2
+
+// CHECK-LABEL: func.func private @test_func_log4() -> i64
+func.func private @test_func_log4() -> !ty_log4
+
+// CHECK-LABEL: func.func private @test_func_log8() -> i64
+func.func private @test_func_log8() -> !ty_log8
+
+// CHECK-LABEL: func.func private @test_func_log16() -> i128
+func.func private @test_func_log16() -> !ty_log16
+
+// CHECK-LABEL: func.func private @test_func_cmplx_f32() -> tuple<f32, f32>
+func.func private @test_func_cmplx_f32() -> !ty_cmplx_f32
+
+// CHECK-LABEL: func.func private @test_func_cmplx_f64() -> tuple<f64, f64>
+func.func private @test_func_cmplx_f64() -> !ty_cmplx_f64
+}
+
+
+/// *************************** Struct with more than one field **************************** ///
+
+// -----
+
+!ty_i32_f32 = !fir.type<ti32_f32{i:i32,j:f32}>
+!ty_i32_f64 = !fir.type<ti32_f64{i:i32,j:f64}>
+!ty_i64_f32 = !fir.type<ti64_f32{i:i64,j:f32}>
+!ty_i64_f64 = !fir.type<ti64_f64{i:i64,j:f64}>
+!ty_f64_i64 = !fir.type<tf64_i64{i:f64,j:i64}>
+!ty_f16_f16 = !fir.type<tf16_f16{i:f16,j:f16}>
+!ty_f32_f32 = !fir.type<tf32_f32{i:f32,j:f32}>
+!ty_f64_f64 = !fir.type<tf64_f64{i:f64,j:f64}>
+!ty_f32_i32_i32 = !fir.type<tf32_i32_i32{i:f32,j:i32,k:i32}>
+!ty_f32_f32_i32 = !fir.type<tf32_f32_i32{i:f32,j:f32,k:i32}>
+!ty_f32_f32_f32 = !fir.type<tf32_f32_f32{i:f32,j:f32,k:f32}>
+
+!ty_i8_a8  = !fir.type<ti8_a8{i:!fir.array<8xi8>}>
+!ty_i8_a16 = !fir.type<ti8_a16{i:!fir.array<16xi8>}>
+!ty_f32_a2 = !fir.type<tf32_a2{i:!fir.array<2xf32>}>
+!ty_f64_a2 = !fir.type<tf64_a2{i:!fir.array<2xf64>}>
+!ty_nested_i32_f32 = !fir.type<t11{i:!ty_i32_f32}>
+!ty_nested_i8_a8_i32 = !fir.type<t12{i:!ty_i8_a8, j:i32}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
+// CHECK-LABEL: func.func private @test_func_i32_f32() -> tuple<i32, f32>
+func.func private @test_func_i32_f32() -> !ty_i32_f32
+// CHECK-LABEL: func.func @test_call_i32_f32(
+// CHECK-SAME:    %[[ARG0:.*]]: !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>) {
+func.func @test_call_i32_f32(%arg0: !fir.ref<!ty_i32_f32>) {
+  // CHECK: %[[OUT:.*]] = fir.call @test_func_i32_f32() : () -> tuple<i32, f32>
+  // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
+  // CHECK: %[[ARR:.*]] = fir.alloca tuple<i32, f32>
+  // CHECK: fir.store %[[OUT]] to %[[ARR]] : !fir.ref<tuple<i32, f32>>
+  // CHECK: %[[CVT:.*]] = fir.convert %[[ARR]] : (!fir.ref<tuple<i32, f32>>) -> !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  // CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
+  %out = fir.call @test_func_i32_f32() : () -> !ty_i32_f32
+  // CHECK: fir.store %[[LD]] to %[[ARG0]] : !fir.ref<!fir.type<ti32_f32{i:i32,j:f32}>>
+  fir.store %out to %arg0 : !fir.ref<!ty_i32_f32>
+  // CHECK: return
+  return
+}
+
+// CHECK-LABEL: func.func private @test_func_i32_f64() -> tuple<i32, f64>
+func.func private @test_func_i32_f64() -> !ty_i32_f64
+
+// CHECK-LABEL: func.func private @test_func_i64_f32() -> tuple<i64, f32>
+func.func private @test_func_i64_f32() -> !ty_i64_f32
+
+// CHECK-LABEL: func.func private @test_func_i64_f64() -> tuple<i64, f64>
+func.func private @test_func_i64_f64() -> !ty_i64_f64
+
+// CHECK-LABEL: func.func private @test_func_f64_i64() -> tuple<f64, i64>
+func.func private @test_func_f64_i64() -> !ty_f64_i64
+
+// CHECK-LABEL: func.func private @test_func_f16_f16() -> i64
+func.func private @test_func_f16_f16() -> !ty_f16_f16
+
+// CHECK-LABEL: func.func private @test_func_f32_f32() -> tuple<f32, f32>
+func.func private @test_func_f32_f32() -> !ty_f32_f32
+
+// CHECK-LABEL: func.func private @test_func_f64_f64() -> tuple<f64, f64>
+func.func private @test_func_f64_f64() -> !ty_f64_f64
+
+// CHECK-LABEL: func.func private @test_func_f32_i32_i32() -> !fir.array<2xi64>
+func.func private @test_func_f32_i32_i32() -> !ty_f32_i32_i32
+
+// CHECK-LABEL: func.func private @test_func_f32_f32_i32() -> !fir.array<2xi64>
+func.func private @test_func_f32_f32_i32() -> !ty_f32_f32_i32
+
+// CHECK-LABEL: func.func private @test_func_f32_f32_f32() -> !fir.array<2xi64>
+func.func private @test_func_f32_f32_f32() -> !ty_f32_f32_f32
+
+// CHECK-LABEL: func.func private @test_func_i8_a8() -> i64
+func.func private @test_func_i8_a8() -> !ty_i8_a8
+
+// CHECK-LABEL: func.func private @test_func_i8_a16() -> !fir.array<2xi64>
+func.func private @test_func_i8_a16() -> !ty_i8_a16
+
+// CHECK-LABEL: func.func private @test_func_f32_a2() -> tuple<f32, f32>
+func.func private @test_func_f32_a2() -> !ty_f32_a2
+
+// CHECK-LABEL: func.func private @test_func_f64_a2() -> tuple<f64, f64>
+func.func private @test_func_f64_a2() -> !ty_f64_a2
+
+// CHECK-LABEL: func.func private @test_func_nested_i32_f32() -> tuple<i32, f32>
+func.func private @test_func_nested_i32_f32() -> !ty_nested_i32_f32
+
+// CHECK-LABEL: func.func private @test_func_nested_i8_a8_i32() -> !fir.array<2xi64>
+func.func private @test_func_nested_i8_a8_i32() -> !ty_nested_i8_a8_i32
+}
diff --git a/flang/test/Fir/target-rewrite-boxchar.fir b/flang/test/Fir/target-rewrite-boxchar.fir
index b87cb35b46eb6..681536cde6a05 100644
--- a/flang/test/Fir/target-rewrite-boxchar.fir
+++ b/flang/test/Fir/target-rewrite-boxchar.fir
@@ -27,43 +27,13 @@ func.func @boxcharparams(%arg0 : !fir.boxchar<1>, %arg1 : !fir.boxchar<1>) -> i6
   return %3 : i64
 }
 
-// Test that we rewrite the signatures and bodies of functions that return a
-// boxchar.
-// INT32-LABEL: @boxcharsret
-// INT32-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<!fir.char<1,?>> {llvm.sret = !fir.char<1,?>}, [[ARG1:%[0-9A-Za-z]+]]: i32, [[ARG2:%[0-9A-Za-z]+]]: !fir.ref<!fir.char<1,?>>, [[ARG3:%[0-9A-Za-z]+]]: i32)
-// INT64-LABEL: @boxcharsret
-// INT64-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<!fir.char<1,?>> {llvm.sret = !fir.char<1,?>}, [[ARG1:%[0-9A-Za-z]+]]: i64, [[ARG2:%[0-9A-Za-z]+]]: !fir.ref<!fir.char<1,?>>, [[ARG3:%[0-9A-Za-z]+]]: i64)
-func.func @boxcharsret(%arg0 : !fir.boxchar<1> {llvm.sret = !fir.char<1,?>}, %arg1 : !fir.boxchar<1>) {
-  // INT32-DAG: [[B0:%[0-9]+]] = fir.emboxchar [[ARG0]], [[ARG1]] : (!fir.ref<!fir.char<1,?>>, i32) -> !fir.boxchar<1>
-  // INT32-DAG: [[B1:%[0-9]+]] = fir.emboxchar [[ARG2]], [[ARG3]] : (!fir.ref<!fir.char<1,?>>, i32) -> !fir.boxchar<1>
-  // INT32-DAG: fir.unboxchar [[B0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  // INT32-DAG: fir.unboxchar [[B1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  // INT64-DAG: [[B0:%[0-9]+]] = fir.emboxchar [[ARG0]], [[ARG1]] : (!fir.ref<!fir.char<1,?>>, i64) -> !fir.boxchar<1>
-  // INT64-DAG: [[B1:%[0-9]+]] = fir.emboxchar [[ARG2]], [[ARG3]] : (!fir.ref<!fir.char<1,?>>, i64) -> !fir.boxchar<1>
-  // INT64-DAG: fir.unboxchar [[B0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  // INT64-DAG: fir.unboxchar [[B1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  %1:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %3 = fir.convert %1#1 : (i64) -> index
-  %last = arith.subi %3, %c1 : index
-  fir.do_loop %i = %c0 to %last step %c1 {
-    %in_pos = fir.coordinate_of %2#0, %i : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-    %out_pos = fir.coordinate_of %1#0, %i : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-    %ch = fir.load %in_pos : !fir.ref<!fir.char<1>>
-    fir.store %ch to %out_pos : !fir.ref<!fir.char<1>>
-  }
-  return
-}
-
-// Test that we rewrite the signatures of functions with a sret parameter and
+// Test that we rewrite the signatures of functions with several parameters.
 // several other parameters.
 // INT32-LABEL: @boxcharmultiple
-// INT32-SAME: ({{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>> {llvm.sret = !fir.char<1,?>}, {{%[0-9A-Za-z]+}}: i32, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: i32, {{%[0-9A-Za-z]+}}: i32)
+// INT32-SAME: ({{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: i32, {{%[0-9A-Za-z]+}}: i32, {{%[0-9A-Za-z]+}}: i32)
 // INT64-LABEL: @boxcharmultiple
-// INT64-SAME: ({{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>> {llvm.sret = !fir.char<1,?>}, {{%[0-9A-Za-z]+}}: i64, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: i64, {{%[0-9A-Za-z]+}}: i64)
-func.func @boxcharmultiple(%arg0 : !fir.boxchar<1> {llvm.sret = !fir.char<1,?>}, %arg1 : !fir.boxchar<1>, %arg2 : !fir.boxchar<1>) {
+// INT64-SAME: ({{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: !fir.ref<!fir.char<1,?>>, {{%[0-9A-Za-z]+}}: i64, {{%[0-9A-Za-z]+}}: i64, {{%[0-9A-Za-z]+}}: i64)
+func.func @boxcharmultiple(%arg0 : !fir.boxchar<1>, %arg1 : !fir.boxchar<1>, %arg2 : !fir.boxchar<1>) {
   return
 }
 
diff --git a/flang/test/Fir/target-rewrite-complex-10-x86.fir b/flang/test/Fir/target-rewrite-complex-10-x86.fir
index 404f5f9f0615b..651d1e78fdc93 100644
--- a/flang/test/Fir/target-rewrite-complex-10-x86.fir
+++ b/flang/test/Fir/target-rewrite-complex-10-x86.fir
@@ -29,4 +29,4 @@ func.func @takecomplex10(%z: complex<f80>) {
 // AMD64:    %[[VAL_3:.*]] = fir.alloca complex<f80>
 // AMD64:    fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<complex<f80>>
 
-// AMD64_LLVM: define void @takecomplex10(ptr byval({ x86_fp80, x86_fp80 }) align 16 %0)
+// AMD64_LLVM: define void @takecomplex10(ptr nocapture byval({ x86_fp80, x86_fp80 }) align 16 %0)
diff --git a/flang/test/Fir/target.fir b/flang/test/Fir/target.fir
index 788deebcb5e84..d101db47c8051 100644
--- a/flang/test/Fir/target.fir
+++ b/flang/test/Fir/target.fir
@@ -26,7 +26,7 @@ func.func @gen4() -> complex<f32> {
   return %6 : complex<f32>
 }
 
-// I32-LABEL: define void @gen8(ptr sret({ double, double }) align 4 %
+// I32-LABEL: define void @gen8(ptr nocapture sret({ double, double }) align 4 %
 // X64-LABEL: define { double, double } @gen8()
 // AARCH64-LABEL: define { double, double } @gen8()
 // PPC-LABEL: define { double, double } @gen8()
@@ -93,9 +93,9 @@ func.func @call8() {
   return
 }
 
-// I32-LABEL: define i64 @char1lensum(ptr %0, ptr %1, i32 %2, i32 %3)
-// X64-LABEL: define i64 @char1lensum(ptr %0, ptr %1, i64 %2, i64 %3)
-// PPC-LABEL: define i64 @char1lensum(ptr %0, ptr %1, i64 %2, i64 %3)
+// I32-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i32 %2, i32 %3)
+// X64-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i64 %2, i64 %3)
+// PPC-LABEL: define i64 @char1lensum(ptr nocapture %0, ptr nocapture %1, i64 %2, i64 %3)
 func.func @char1lensum(%arg0 : !fir.boxchar<1>, %arg1 : !fir.boxchar<1>) -> i64 {
   // X64-DAG: %[[p0:.*]] = insertvalue { ptr, i64 } undef, ptr %1, 0
   // X64-DAG: = insertvalue { ptr, i64 } %[[p0]], i64 %3, 1
@@ -110,32 +110,3 @@ func.func @char1lensum(%arg0 : !fir.boxchar<1>, %arg1 : !fir.boxchar<1>) -> i64
   // X64: ret i64 %[[add]]
   return %3 : i64
 }
-
-// I32-LABEL: define void @char1copy(ptr sret(i8) %0, i32 %1, ptr %2, i32 %3)
-// I64-LABEL: define void @char1copy(ptr sret(i8) %0, i64 %1, ptr %2, i64 %3)
-// PPC-LABEL: define void @char1copy(ptr sret(i8) %0, i64 %1, ptr %2, i64 %3)
-func.func @char1copy(%arg0 : !fir.boxchar<1> {llvm.sret = !fir.char<1, ?>}, %arg1 : !fir.boxchar<1>) {
-  // I32-DAG: %[[p0:.*]] = insertvalue { ptr, i32 } undef, ptr %2, 0
-  // I32-DAG: = insertvalue { ptr, i32 } %[[p0]], i32 %3, 1
-  // I32-DAG: %[[p1:.*]] = insertvalue { ptr, i32 } undef, ptr %0, 0
-  // I32-DAG: = insertvalue { ptr, i32 } %[[p1]], i32 %1, 1
-  // X64-DAG: %[[p0:.*]] = insertvalue { ptr, i64 } undef, ptr %2, 0
-  // X64-DAG: = insertvalue { ptr, i64 } %[[p0]], i64 %3, 1
-  // X64-DAG: %[[p1:.*]] = insertvalue { ptr, i64 } undef, ptr %0, 0
-  // X64-DAG: = insertvalue { ptr, i64 } %[[p1]], i64 %1, 1
-  %1:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.array<?x!fir.char<1>>>, i64)
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %3 = fir.convert %1#1 : (i64) -> index
-  %last = arith.subi %3, %c1 : index
-  fir.do_loop %i = %c0 to %last step %c1 {
-    %in_pos = fir.coordinate_of %2#0, %i : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-    %out_pos = fir.coordinate_of %1#0, %i : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-    %ch = fir.load %in_pos : !fir.ref<!fir.char<1>>
-    fir.store %ch to %out_pos : !fir.ref<!fir.char<1>>
-  }
-  // I32: ret void
-  // X64: ret void
-  return
-}
diff --git a/flang/test/HLFIR/cshift-lowering.fir b/flang/test/HLFIR/cshift-lowering.fir
new file mode 100644
index 0000000000000..386b81c4dbff6
--- /dev/null
+++ b/flang/test/HLFIR/cshift-lowering.fir
@@ -0,0 +1,217 @@
+// Test hlfir.cshift operation lowering to fir runtime call
+// RUN: fir-opt %s -lower-hlfir-intrinsics | FileCheck %s
+
+// 1d boxed vector shift by scalar
+func.func @cshift1(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "sh"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = hlfir.cshift %0#0 %1#0 : (!fir.box<!fir.array<?xi32>>, !fir.ref<i32>) -> !hlfir.expr<?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @cshift1(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "sh"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_8:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.embox %[[VAL_8]](%[[VAL_9]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_7]]#1 : !fir.ref<i32>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_6]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_11]] : (i32) -> i64
+// CHECK:           %[[VAL_17:.*]] = fir.call @_FortranACshiftVector(%[[VAL_13]], %[[VAL_14]], %[[VAL_15]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, !fir.ref<i8>, i32) -> none
+
+// 2d boxed array shift by scalar
+func.func @cshift2(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: i32 {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %2 = hlfir.cshift %0#0 %arg1 : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @cshift2(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: i32 {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_11]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_7]] : (!fir.ref<i32>) -> !fir.box<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_12]] : (!fir.box<i32>) -> !fir.box<none>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           %[[VAL_19:.*]] = fir.call @_FortranACshift(%[[VAL_14]], %[[VAL_15]], %[[VAL_16]], %[[VAL_17]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+// 2d boxed array shift by boxed array
+func.func @cshift3(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "sh"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = hlfir.cshift %0#0 %1#0 : (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @cshift3(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "sh"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_11]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_7]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           %[[VAL_18:.*]] = fir.call @_FortranACshift(%[[VAL_13]], %[[VAL_14]], %[[VAL_15]], %[[VAL_16]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+// 2d boxed array shift by array expr
+func.func @cshift4(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %2 = hlfir.cshift %0#0 %arg1 : (!fir.box<!fir.array<?x?xi32>>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @cshift4(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_8]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_11:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_11]](%[[VAL_12]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_13]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_9]]#1(%[[VAL_14]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_7]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_15]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           %[[VAL_22:.*]] = fir.call @_FortranACshift(%[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %[[VAL_20]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+// 2d array expr shift by array expr
+func.func @cshift5(%arg0: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+  %2 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+  hlfir.destroy %2 : !hlfir.expr<?x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @cshift5(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]]:3 = hlfir.associate %[[VAL_0]](%[[VAL_7]]) {adapt.valuebyref} : (!hlfir.expr<?x?xi32>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>, i1)
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_11:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_11]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_11]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_9]], %[[VAL_10]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_8]]#1(%[[VAL_14]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_13]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_20:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_19]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_15]] : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           %[[VAL_27:.*]] = fir.call @_FortranACshift(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[VAL_25]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+// 2d array expr shift by array expr with explicit dim
+func.func @cshift6(%arg0: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}, %dim : i16) {
+  %2 = hlfir.cshift %arg0 %arg1 dim %dim : (!hlfir.expr<?x?xi32>, !hlfir.expr<?xi32>, i16) -> !hlfir.expr<?x?xi32>
+  hlfir.destroy %2 : !hlfir.expr<?x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @cshift6(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"},
+// CHECK-SAME:                       %[[VAL_2:.*]]: i16) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_0]](%[[VAL_8]]) {adapt.valuebyref} : (!hlfir.expr<?x?xi32>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>, i1)
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_12]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_14:.*]] = hlfir.get_extent %[[VAL_12]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_2]] : (i16) -> i32
+// CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_10]], %[[VAL_11]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_17:.*]] = fir.embox %[[VAL_9]]#1(%[[VAL_16]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_6]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_20:.*]] = fir.embox %[[VAL_18]](%[[VAL_19]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_20]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_21:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_22:.*]] = fir.embox %[[VAL_13]]#1(%[[VAL_21]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_17]] : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_22]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_28:.*]] = fir.call @_FortranACshift(%[[VAL_24]], %[[VAL_25]], %[[VAL_26]], %[[VAL_15]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+// shift of polymorphic array
+func.func @cshift7(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, %arg1: !fir.ref<f32>) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "a"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+  %2:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "sh"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %c2_i32 = arith.constant 2 : i32
+  %3 = fir.load %1#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  %4 = hlfir.cshift %3 %c2_i32 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  hlfir.assign %4 to %1#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  hlfir.destroy %4 : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  return
+}
+// CHECK-LABEL:   func.func @cshift7(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>,
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.ref<f32>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_7:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "a"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_8]] {uniq_name = "sh"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]](%[[VAL_13]]) source_box %[[VAL_11]] : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_6]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_11]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+// CHECK:           %[[VAL_21:.*]] = fir.call @_FortranACshiftVector(%[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, !fir.ref<i8>, i32) -> none
diff --git a/flang/test/HLFIR/element-codegen-issue-118922.fir b/flang/test/HLFIR/element-codegen-issue-118922.fir
new file mode 100644
index 0000000000000..8fc0aa92a6b35
--- /dev/null
+++ b/flang/test/HLFIR/element-codegen-issue-118922.fir
@@ -0,0 +1,48 @@
+// Test issue 113843 and 118922 fix: do not elide hlfir.elemental final as_expr
+// copy if this is not the last operation.
+// RUN: fir-opt %s --bufferize-hlfir | FileCheck %s
+
+func.func @_QMmPbug(%val: !fir.char<1>, %var: !fir.ref<!fir.array<10x!fir.char<1>>>) {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %expr = hlfir.elemental %1 typeparams %c1 unordered : (!fir.shape<1>, index) -> !hlfir.expr<10x!fir.char<1>> {
+  ^bb0(%arg2: index):
+    %alloc = fir.allocmem !fir.char<1>
+    fir.store %val to %alloc : !fir.heap<!fir.char<1>>
+    %addr = fir.convert %alloc: (!fir.heap<!fir.char<1>>) -> !fir.ref<!fir.char<1>>
+    %9 = hlfir.as_expr %addr : (!fir.ref<!fir.char<1>>) -> !hlfir.expr<!fir.char<1>>
+    fir.freemem  %alloc : !fir.heap<!fir.char<1>>
+    hlfir.yield_element %9 : !hlfir.expr<!fir.char<1>>
+  }
+  hlfir.assign %expr to %var : !hlfir.expr<10x!fir.char<1>>, !fir.ref<!fir.array<10x!fir.char<1>>>
+  hlfir.destroy %expr : !hlfir.expr<10x!fir.char<1>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QMmPbug(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.char<1>,
+// CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.array<10x!fir.char<1>>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.char<1> {bindc_name = ".tmp"}
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = fir.allocmem !fir.array<10x!fir.char<1>> {bindc_name = ".tmp.array", uniq_name = ""}
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) typeparams %[[VAL_3]] {uniq_name = ".tmp.array"} : (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.shape<1>, index) -> (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.heap<!fir.array<10x!fir.char<1>>>)
+// CHECK:           %[[VAL_8:.*]] = arith.constant true
+// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_10:.*]] = %[[VAL_9]] to %[[VAL_4]] step %[[VAL_9]] unordered {
+// CHECK:             %[[VAL_11:.*]] = fir.allocmem !fir.char<1>
+// CHECK:             fir.store %[[VAL_0]] to %[[VAL_11]] : !fir.heap<!fir.char<1>>
+// CHECK:             %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.heap<!fir.char<1>>) -> !fir.ref<!fir.char<1>>
+// CHECK:             %[[VAL_13:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_14:.*]] = arith.constant false
+// CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[VAL_13]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+// CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]]#0 temporary_lhs : !fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>
+// CHECK:             %[[VAL_16:.*]] = fir.undefined tuple<!fir.ref<!fir.char<1>>, i1>
+// CHECK:             %[[VAL_17:.*]] = fir.insert_value %[[VAL_16]], %[[VAL_14]], [1 : index] : (tuple<!fir.ref<!fir.char<1>>, i1>, i1) -> tuple<!fir.ref<!fir.char<1>>, i1>
+// CHECK:             %[[VAL_18:.*]] = fir.insert_value %[[VAL_17]], %[[VAL_15]]#0, [0 : index] : (tuple<!fir.ref<!fir.char<1>>, i1>, !fir.ref<!fir.char<1>>) -> tuple<!fir.ref<!fir.char<1>>, i1>
+// CHECK:             fir.freemem %[[VAL_11]] : !fir.heap<!fir.char<1>>
+// CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_10]])  typeparams %[[VAL_3]] : (!fir.heap<!fir.array<10x!fir.char<1>>>, index, index) -> !fir.ref<!fir.char<1>>
+// CHECK:             hlfir.assign %[[VAL_15]]#0 to %[[VAL_19]] temporary_lhs : !fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>
+// CHECK:           }
diff --git a/flang/test/HLFIR/eval_in_mem-codegen.fir b/flang/test/HLFIR/eval_in_mem-codegen.fir
new file mode 100644
index 0000000000000..26a989832ca92
--- /dev/null
+++ b/flang/test/HLFIR/eval_in_mem-codegen.fir
@@ -0,0 +1,107 @@
+// Test hlfir.eval_in_mem default code generation.
+
+// RUN: fir-opt %s --bufferize-hlfir -o - | FileCheck %s
+
+func.func @_QPtest() {
+  %c10 = arith.constant 10 : index
+  %0 = fir.address_of(@_QFtestEx) : !fir.ref<!fir.array<10xf32>>
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>):
+    %3 = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+    fir.save_result %3 to %arg0(%1) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+  }
+  hlfir.assign %2 to %0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+  hlfir.destroy %2 : !hlfir.expr<10xf32>
+  return
+}
+fir.global internal @_QFtestEx : !fir.array<10xf32>
+func.func private @_QParray_func() -> !fir.array<10xf32>
+
+
+func.func @_QPtest_char() {
+  %c10 = arith.constant 10 : index
+  %c5 = arith.constant 5 : index
+  %0 = fir.address_of(@_QFtest_charEx) : !fir.ref<!fir.array<10x!fir.char<1,5>>>
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2 = hlfir.eval_in_mem shape %1 typeparams %c5 : (!fir.shape<1>, index) -> !hlfir.expr<10x!fir.char<1,5>> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10x!fir.char<1,5>>>):
+    %3 = fir.call @_QPchar_array_func() fastmath<contract> : () -> !fir.array<10x!fir.char<1,5>>
+    fir.save_result %3 to %arg0(%1) typeparams %c5 : !fir.array<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index
+  }
+  hlfir.assign %2 to %0 : !hlfir.expr<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>
+  hlfir.destroy %2 : !hlfir.expr<10x!fir.char<1,5>>
+  return
+}
+
+fir.global internal @_QFtest_charEx : !fir.array<10x!fir.char<1,5>>
+func.func private @_QPchar_array_func() -> !fir.array<10x!fir.char<1,5>>
+
+func.func @test_dynamic(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: index) {
+   %0 = fir.shape %arg1 : (index) -> !fir.shape<1>
+   %1 = hlfir.eval_in_mem shape %0 : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+   ^bb0(%arg2: !fir.ref<!fir.array<?xf32>>):
+     %2 = fir.call @_QPdyn_array_func(%arg1) : (index) -> !fir.array<?xf32>
+     fir.save_result %2 to %arg2(%0) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+   }
+   hlfir.assign %1 to %arg0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+   hlfir.destroy %1 : !hlfir.expr<?xf32>
+   return
+}
+func.func private @_QPdyn_array_func(index) -> !fir.array<?xf32>
+
+// CHECK-LABEL:   func.func @_QPtest() {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.array<10xf32> {bindc_name = ".tmp.expr_result"}
+// CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFtestEx) : !fir.ref<!fir.array<10xf32>>
+// CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = ".tmp.expr_result"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+// CHECK:           %[[VAL_5:.*]] = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+// CHECK:           fir.save_result %[[VAL_5]] to %[[VAL_4]]#1(%[[VAL_3]]) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = arith.constant false
+// CHECK:           %[[VAL_7:.*]] = fir.undefined tuple<!fir.ref<!fir.array<10xf32>>, i1>
+// CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_6]], [1 : index] : (tuple<!fir.ref<!fir.array<10xf32>>, i1>, i1) -> tuple<!fir.ref<!fir.array<10xf32>>, i1>
+// CHECK:           %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_4]]#0, [0 : index] : (tuple<!fir.ref<!fir.array<10xf32>>, i1>, !fir.ref<!fir.array<10xf32>>) -> tuple<!fir.ref<!fir.array<10xf32>>, i1>
+// CHECK:           hlfir.assign %[[VAL_4]]#0 to %[[VAL_2]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         fir.global internal @_QFtestEx : !fir.array<10xf32>
+// CHECK:         func.func private @_QParray_func() -> !fir.array<10xf32>
+
+// CHECK-LABEL:   func.func @_QPtest_char() {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.array<10x!fir.char<1,5>> {bindc_name = ".tmp.expr_result"}
+// CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 5 : index
+// CHECK:           %[[VAL_3:.*]] = fir.address_of(@_QFtest_charEx) : !fir.ref<!fir.array<10x!fir.char<1,5>>>
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) typeparams %[[VAL_2]] {uniq_name = ".tmp.expr_result"} : (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>)
+// CHECK:           %[[VAL_6:.*]] = fir.call @_QPchar_array_func() fastmath<contract> : () -> !fir.array<10x!fir.char<1,5>>
+// CHECK:           fir.save_result %[[VAL_6]] to %[[VAL_5]]#1(%[[VAL_4]]) typeparams %[[VAL_2]] : !fir.array<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index
+// CHECK:           %[[VAL_7:.*]] = arith.constant false
+// CHECK:           %[[VAL_8:.*]] = fir.undefined tuple<!fir.ref<!fir.array<10x!fir.char<1,5>>>, i1>
+// CHECK:           %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_7]], [1 : index] : (tuple<!fir.ref<!fir.array<10x!fir.char<1,5>>>, i1>, i1) -> tuple<!fir.ref<!fir.array<10x!fir.char<1,5>>>, i1>
+// CHECK:           %[[VAL_10:.*]] = fir.insert_value %[[VAL_9]], %[[VAL_5]]#0, [0 : index] : (tuple<!fir.ref<!fir.array<10x!fir.char<1,5>>>, i1>, !fir.ref<!fir.array<10x!fir.char<1,5>>>) -> tuple<!fir.ref<!fir.array<10x!fir.char<1,5>>>, i1>
+// CHECK:           hlfir.assign %[[VAL_5]]#0 to %[[VAL_3]] : !fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         fir.global internal @_QFtest_charEx : !fir.array<10x!fir.char<1,5>>
+// CHECK:         func.func private @_QPchar_array_func() -> !fir.array<10x!fir.char<1,5>>
+
+// CHECK-LABEL:   func.func @test_dynamic(
+// CHECK-SAME:                            %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                            %[[VAL_1:.*]]: index) {
+// CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_3:.*]] = fir.allocmem !fir.array<?xf32>, %[[VAL_1]] {bindc_name = ".tmp.expr_result", uniq_name = ""}
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.heap<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_2]]) {uniq_name = ".tmp.expr_result"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_6:.*]] = fir.call @_QPdyn_array_func(%[[VAL_1]]) : (index) -> !fir.array<?xf32>
+// CHECK:           fir.save_result %[[VAL_6]] to %[[VAL_5]]#1(%[[VAL_2]]) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = arith.constant true
+// CHECK:           %[[VAL_8:.*]] = fir.undefined tuple<!fir.box<!fir.array<?xf32>>, i1>
+// CHECK:           %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_7]], [1 : index] : (tuple<!fir.box<!fir.array<?xf32>>, i1>, i1) -> tuple<!fir.box<!fir.array<?xf32>>, i1>
+// CHECK:           %[[VAL_10:.*]] = fir.insert_value %[[VAL_9]], %[[VAL_5]]#0, [0 : index] : (tuple<!fir.box<!fir.array<?xf32>>, i1>, !fir.box<!fir.array<?xf32>>) -> tuple<!fir.box<!fir.array<?xf32>>, i1>
+// CHECK:           hlfir.assign %[[VAL_5]]#0 to %[[VAL_0]] : !fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>
+// CHECK:           %[[VAL_11:.*]] = fir.box_addr %[[VAL_5]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:           fir.freemem %[[VAL_11]] : !fir.heap<!fir.array<?xf32>>
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/HLFIR/eval_in_mem.fir b/flang/test/HLFIR/eval_in_mem.fir
new file mode 100644
index 0000000000000..34e48ed5be545
--- /dev/null
+++ b/flang/test/HLFIR/eval_in_mem.fir
@@ -0,0 +1,99 @@
+// Test hlfir.eval_in_mem operation parse, verify (no errors), and unparse.
+
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @_QPtest() {
+  %c10 = arith.constant 10 : index
+  %0 = fir.address_of(@_QFtestEx) : !fir.ref<!fir.array<10xf32>>
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>):
+    %3 = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+    fir.save_result %3 to %arg0(%1) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+  }
+  hlfir.assign %2 to %0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+  hlfir.destroy %2 : !hlfir.expr<10xf32>
+  return
+}
+fir.global internal @_QFtestEx : !fir.array<10xf32>
+func.func private @_QParray_func() -> !fir.array<10xf32>
+
+
+func.func @_QPtest_char() {
+  %c10 = arith.constant 10 : index
+  %c5 = arith.constant 5 : index
+  %0 = fir.address_of(@_QFtest_charEx) : !fir.ref<!fir.array<10x!fir.char<1,5>>>
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2 = hlfir.eval_in_mem shape %1 typeparams %c5 : (!fir.shape<1>, index) -> !hlfir.expr<10x!fir.char<1,5>> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10x!fir.char<1,5>>>):
+    %3 = fir.call @_QPchar_array_func() fastmath<contract> : () -> !fir.array<10x!fir.char<1,5>>
+    fir.save_result %3 to %arg0(%1) typeparams %c5 : !fir.array<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index
+  }
+  hlfir.assign %2 to %0 : !hlfir.expr<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>
+  hlfir.destroy %2 : !hlfir.expr<10x!fir.char<1,5>>
+  return
+}
+
+fir.global internal @_QFtest_charEx : !fir.array<10x!fir.char<1,5>>
+func.func private @_QPchar_array_func() -> !fir.array<10x!fir.char<1,5>>
+
+func.func @test_dynamic(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: index) {
+   %0 = fir.shape %arg1 : (index) -> !fir.shape<1>
+   %1 = hlfir.eval_in_mem shape %0 : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+   ^bb0(%arg2: !fir.ref<!fir.array<?xf32>>):
+     %2 = fir.call @_QPdyn_array_func(%arg1) : (index) -> !fir.array<?xf32>
+     fir.save_result %2 to %arg2(%0) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+   }
+   hlfir.assign %1 to %arg0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+   hlfir.destroy %1 : !hlfir.expr<?xf32>
+   return
+}
+func.func private @_QPdyn_array_func(index) -> !fir.array<?xf32>
+
+// CHECK-LABEL:   func.func @_QPtest() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_1:.*]] = fir.address_of(@_QFtestEx) : !fir.ref<!fir.array<10xf32>>
+// CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_3:.*]] = hlfir.eval_in_mem shape %[[VAL_2]] : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+// CHECK:           ^bb0(%[[VAL_4:.*]]: !fir.ref<!fir.array<10xf32>>):
+// CHECK:             %[[VAL_5:.*]] = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+// CHECK:             fir.save_result %[[VAL_5]] to %[[VAL_4]](%[[VAL_2]]) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_3]] to %[[VAL_1]] : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+// CHECK:           hlfir.destroy %[[VAL_3]] : !hlfir.expr<10xf32>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         fir.global internal @_QFtestEx : !fir.array<10xf32>
+// CHECK:         func.func private @_QParray_func() -> !fir.array<10xf32>
+
+// CHECK-LABEL:   func.func @_QPtest_char() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 5 : index
+// CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFtest_charEx) : !fir.ref<!fir.array<10x!fir.char<1,5>>>
+// CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_4:.*]] = hlfir.eval_in_mem shape %[[VAL_3]] typeparams %[[VAL_1]] : (!fir.shape<1>, index) -> !hlfir.expr<10x!fir.char<1,5>> {
+// CHECK:           ^bb0(%[[VAL_5:.*]]: !fir.ref<!fir.array<10x!fir.char<1,5>>>):
+// CHECK:             %[[VAL_6:.*]] = fir.call @_QPchar_array_func() fastmath<contract> : () -> !fir.array<10x!fir.char<1,5>>
+// CHECK:             fir.save_result %[[VAL_6]] to %[[VAL_5]](%[[VAL_3]]) typeparams %[[VAL_1]] : !fir.array<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>, !fir.shape<1>, index
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_2]] : !hlfir.expr<10x!fir.char<1,5>>, !fir.ref<!fir.array<10x!fir.char<1,5>>>
+// CHECK:           hlfir.destroy %[[VAL_4]] : !hlfir.expr<10x!fir.char<1,5>>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         fir.global internal @_QFtest_charEx : !fir.array<10x!fir.char<1,5>>
+// CHECK:         func.func private @_QPchar_array_func() -> !fir.array<10x!fir.char<1,5>>
+
+// CHECK-LABEL:   func.func @test_dynamic(
+// CHECK-SAME:                            %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                            %[[VAL_1:.*]]: index) {
+// CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_3:.*]] = hlfir.eval_in_mem shape %[[VAL_2]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_4:.*]]: !fir.ref<!fir.array<?xf32>>):
+// CHECK:             %[[VAL_5:.*]] = fir.call @_QPdyn_array_func(%[[VAL_1]]) : (index) -> !fir.array<?xf32>
+// CHECK:             fir.save_result %[[VAL_5]] to %[[VAL_4]](%[[VAL_2]]) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_3]] to %[[VAL_0]] : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_3]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         func.func private @_QPdyn_array_func(index) -> !fir.array<?xf32>
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index c390dddcf3f38..b35bec4b2a899 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -1314,3 +1314,112 @@ func.func @end_associate_with_alloc_comp(%var: !hlfir.expr<?x!fir.type<_QMtypesT
   hlfir.end_associate %4#1, %4#2 : !fir.ref<!fir.array<?x!fir.type<_QMtypesTt{x:!fir.box<!fir.heap<f32>>}>>>, i1
   return
 }
+
+// -----
+
+func.func @bad_eval_in_mem_1() {
+  %c10 = arith.constant 10 : index
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+// expected-error@+1 {{'hlfir.eval_in_mem' op result #0 must be The type of an array, character, or derived type Fortran expression, but got '!fir.array<10xf32>'}}
+  %2 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !fir.array<10xf32> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>):
+  }
+  return
+}
+
+// -----
+
+func.func @bad_eval_in_mem_2() {
+  %c10 = arith.constant 10 : index
+  %1 = fir.shape %c10, %c10 : (index, index) -> !fir.shape<2>
+  // expected-error@+1 {{'hlfir.eval_in_mem' op `shape` rank must match the result rank}}
+  %2 = hlfir.eval_in_mem shape %1 : (!fir.shape<2>) -> !hlfir.expr<10xf32> {
+  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>):
+  }
+  return
+}
+
+// -----
+
+func.func @bad_eval_in_mem_3() {
+  // expected-error@+1 {{'hlfir.eval_in_mem' op must be provided one length parameter when the result is a character}}
+  %1 = hlfir.eval_in_mem  : () -> !hlfir.expr<!fir.char<1,?>> {
+  ^bb0(%arg0: !fir.ref<!fir.char<1,?>>):
+  }
+  return
+}
+
+// -----
+
+func.func @bad_cshift1(%arg0: !hlfir.expr<?x?xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.cshift' op input and output arrays should have the same element type}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, i32) -> !hlfir.expr<?x?xf32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift2(%arg0: !hlfir.expr<?x?xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.cshift' op input and output arrays should have the same rank}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, i32) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift3(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.cshift' op output array's shape conflicts with the input array's shape}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<2x2xi32>, i32) -> !hlfir.expr<2x3xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift4(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+1 {{'hlfir.cshift' op DIM must be >= 1}}
+  %0 = hlfir.cshift %arg0 %arg1 dim %c0 : (!hlfir.expr<2x2xi32>, i32, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift5(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  %c10 = arith.constant 10 : index
+  // expected-error@+1 {{'hlfir.cshift' op DIM must be <= input array's rank}}
+  %0 = hlfir.cshift %arg0 %arg1 dim %c10 : (!hlfir.expr<2x2xi32>, i32, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift6(%arg0: !hlfir.expr<2x2xi32>, %arg1: !hlfir.expr<2x2xi32>) {
+  // expected-error@+1 {{'hlfir.cshift' op SHIFT's rank must be 1 less than the input array's rank}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<2x2xi32>, !hlfir.expr<2x2xi32>) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift7(%arg0: !hlfir.expr<?x2xi32>, %arg1: !hlfir.expr<3xi32>) {
+  %c1 = arith.constant 1 : index
+  // expected-error@+1 {{'hlfir.cshift' op SHAPE(ARRAY)(2) must be equal to SHAPE(SHIFT)(1): 2 != 3}}
+  %0 = hlfir.cshift %arg0 %arg1 dim %c1 : (!hlfir.expr<?x2xi32>, !hlfir.expr<3xi32>, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_cshift8(%arg0: !hlfir.expr<?x!fir.char<1,?>>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.cshift' op kind mismatch between input and output arrays}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,?>>, i32) -> !hlfir.expr<?x!fir.char<2,?>>
+  return
+}
+
+// -----
+
+func.func @bad_cshift9(%arg0: !hlfir.expr<?x!fir.char<1,1>>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.cshift' op character LEN mismatch between input and output arrays}}
+  %0 = hlfir.cshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,1>>, i32) -> !hlfir.expr<?x!fir.char<1,2>>
+  return
+}
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
index 45993c5eee0c9..5fa482a7b904e 100644
--- a/flang/test/HLFIR/minloc-elemental.fir
+++ b/flang/test/HLFIR/minloc-elemental.fir
@@ -188,67 +188,65 @@ func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_n
   hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
   return
 }
-// The minloc has other uses, not an assign that gets optimized out.
-// CHECK-LABEL: _QPtest_kind2_convert
-// CHECK-SAME:     (%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:   %false = arith.constant false
-// CHECK-NEXT:   %true = arith.constant true
-// CHECK-NEXT:   %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:   %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:   %c0 = arith.constant 0 : index
-// CHECK-NEXT:   %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:   %c1 = arith.constant 1 : index
-// CHECK-NEXT:   %[[V0:.*]] = fir.alloca i16
-// CHECK-NEXT:   %[[V1:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK-NEXT:   %[[V2:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:   %[[V3:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:   %[[V4:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:   %[[V5:.*]] = fir.load %[[V4]]#0 : !fir.ref<i32>
-// CHECK-NEXT:   %[[V6:.*]] = hlfir.designate %[[V1]] (%c1)  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:   fir.store %c0_i16 to %[[V6]] : !fir.ref<i16>
-// CHECK-NEXT:   fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:   %[[V7:.*]]:3 = fir.box_dims %[[V2]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:   %[[V8:.*]] = arith.subi %[[V7]]#1, %c1 : index
-// CHECK-NEXT:   %[[V9:.*]] = fir.do_loop %arg3 = %c0 to %[[V8]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:     %[[V15:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V16:.*]] = hlfir.designate %[[V2]]#0 (%[[V15]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:     %[[V17:.*]] = fir.load %[[V16]] : !fir.ref<i32>
-// CHECK-NEXT:     %[[V18:.*]] = arith.cmpi sge, %[[V17]], %[[V5]] : i32
-// CHECK-NEXT:     %[[V19:.*]] = fir.if %[[V18]] -> (i32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:       %[[V23:.*]] = hlfir.designate %[[V2]]#0 (%[[V15]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V24:.*]] = fir.load %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V25:.*]] = arith.cmpi slt, %[[V24]], %arg4 : i32
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[V25]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V26:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:         fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:         %[[V27:.*]] = hlfir.designate %[[V1]] (%c1)  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:         %[[V28:.*]] = fir.convert %[[V15]] : (index) -> i16
-// CHECK-NEXT:         fir.store %[[V28]] to %[[V27]] : !fir.ref<i16>
-// CHECK-NEXT:         fir.result %[[V24]] : i32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : i32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V26]] : i32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V19]] : i32
-// CHECK-NEXT:   }
-// CHECK-NEXT:   %[[V12:.*]] = hlfir.as_expr %[[V1]] move %false : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK-NEXT:   %[[V13:.*]] = fir.shape %c1 : (index) -> !fir.shape<1>
-// CHECK-NEXT:   %[[V14:.*]] = hlfir.elemental %[[V13]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-// CHECK-NEXT:   ^bb0(%arg3: index):
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.apply %[[V12]], %arg3 : (!hlfir.expr<1xi16>, index) -> i16
-// CHECK-NEXT:     %[[V16:.*]] = fir.convert %[[V15]] : (i16) -> i32
-// CHECK-NEXT:     hlfir.yield_element %[[V16]] : i32
-// CHECK-NEXT:   }
-// CHECK-NEXT:   hlfir.assign %[[V14]] to %[[V3]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:   hlfir.destroy %[[V14]] : !hlfir.expr<?xi32>
-// CHECK-NEXT:   return
-
+// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
+// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
+// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
+// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
+// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
+// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
+// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
+// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
+// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
+// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
+// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
+// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
+// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
+// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
+// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
+// CHECK:                 fir.result %[[VAL_29]] : i32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_21]] : i32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_34]] : i32
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_21]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_26]] : i32
+// CHECK:           }
+// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
+// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
+// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
 
 
 func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
diff --git a/flang/test/HLFIR/opt-bufferization-eval_in_mem.fir b/flang/test/HLFIR/opt-bufferization-eval_in_mem.fir
new file mode 100644
index 0000000000000..984c0bcbaddcc
--- /dev/null
+++ b/flang/test/HLFIR/opt-bufferization-eval_in_mem.fir
@@ -0,0 +1,67 @@
+// RUN: fir-opt --opt-bufferization %s | FileCheck %s
+
+// Fortran F2023 15.5.2.14 point 4. ensures that _QPfoo cannot access _QFtestEx
+// and the temporary storage for the result can be avoided.
+func.func @_QPtest(%arg0: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "x"}) {
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {uniq_name = "_QFtestEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+  %3 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+  ^bb0(%arg1: !fir.ref<!fir.array<10xf32>>):
+    %4 = fir.call @_QPfoo() fastmath<contract> : () -> !fir.array<10xf32>
+    fir.save_result %4 to %arg1(%1) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+  }
+  hlfir.assign %3 to %2#0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+  hlfir.destroy %3 : !hlfir.expr<10xf32>
+  return
+}
+func.func private @_QPfoo() -> !fir.array<10xf32>
+
+// CHECK-LABEL: func.func @_QPtest(
+// CHECK-SAME:                     %[[VAL_0:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "x"}) {
+// CHECK:         %[[VAL_1:.*]] = arith.constant 10 : index
+// CHECK:         %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:         %[[VAL_3:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:         %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %[[VAL_2]] {uniq_name = "_QFtestEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+// CHECK:         %[[VAL_5:.*]] = fir.call @_QPfoo() fastmath<contract> : () -> !fir.array<10xf32>
+// CHECK:         fir.save_result %[[VAL_5]] to %[[VAL_4]]#1(%[[VAL_3]]) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+// CHECK:         return
+// CHECK:       }
+
+
+// Temporary storage cannot be avoided in this case since
+// _QFnegative_test_is_targetEx has the TARGET attribute.
+func.func @_QPnegative_test_is_target(%arg0: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "x", fir.target}) {
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFnegative_test_is_targetEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+  %3 = hlfir.eval_in_mem shape %1 : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+  ^bb0(%arg1: !fir.ref<!fir.array<10xf32>>):
+    %4 = fir.call @_QPfoo() fastmath<contract> : () -> !fir.array<10xf32>
+    fir.save_result %4 to %arg1(%1) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+  }
+  hlfir.assign %3 to %2#0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+  hlfir.destroy %3 : !hlfir.expr<10xf32>
+  return
+}
+// CHECK-LABEL: func.func @_QPnegative_test_is_target(
+// CHECK-SAME:                                        %[[VAL_0:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "x", fir.target}) {
+// CHECK:         %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:         %[[VAL_2:.*]] = arith.constant false
+// CHECK:         %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:         %[[VAL_4:.*]] = fir.alloca !fir.array<10xf32>
+// CHECK:         %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]]{{.*}}
+// CHECK:         %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_4]]{{.*}}
+// CHECK:         %[[VAL_9:.*]] = fir.call @_QPfoo() fastmath<contract> : () -> !fir.array<10xf32>
+// CHECK:         fir.save_result %[[VAL_9]] to %[[VAL_8]]#1{{.*}}
+// CHECK:         %[[VAL_10:.*]] = hlfir.as_expr %[[VAL_8]]#0 move %[[VAL_2]] : (!fir.ref<!fir.array<10xf32>>, i1) -> !hlfir.expr<10xf32>
+// CHECK:         fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_3]] step %[[VAL_1]] unordered {
+// CHECK:           %[[VAL_12:.*]] = hlfir.apply %[[VAL_10]], %[[VAL_11]] : (!hlfir.expr<10xf32>, index) -> f32
+// CHECK:           %[[VAL_13:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_11]])  : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_13]] : f32, !fir.ref<f32>
+// CHECK:         }
+// CHECK:         hlfir.destroy %[[VAL_10]] : !hlfir.expr<10xf32>
+// CHECK:         return
+// CHECK:       }
diff --git a/flang/test/HLFIR/opt-bufferization-non-realloc-assignment.fir b/flang/test/HLFIR/opt-bufferization-non-realloc-assignment.fir
new file mode 100644
index 0000000000000..cc65dec01cc3c
--- /dev/null
+++ b/flang/test/HLFIR/opt-bufferization-non-realloc-assignment.fir
@@ -0,0 +1,50 @@
+// RUN: fir-opt --opt-bufferization %s | FileCheck %s
+
+// Verify that the shape match is not required for optimizing
+// elemental assignment, when lhs not an allocatable.
+// The shapes of lhs and rhs must conform in a legal program.
+//
+// Example:
+// subroutine test(a,b)
+//   integer :: a(:), b(:)
+//   a = b + 1
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+  %c0 = arith.constant 0 : index
+  %c1_i32 = arith.constant 1 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtestEb"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %3:3 = fir.box_dims %2#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %4 = fir.shape %3#1 : (index) -> !fir.shape<1>
+  %5 = hlfir.elemental %4 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg2: index):
+    %6 = hlfir.designate %2#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %7 = fir.load %6 : !fir.ref<i32>
+    %8 = arith.addi %7, %c1_i32 : i32
+    hlfir.yield_element %8 : i32
+  }
+  hlfir.assign %5 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %5 : !hlfir.expr<?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPtest(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_5]] {uniq_name = "_QFtestEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_5]] {uniq_name = "_QFtestEb"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_7]]#0, %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_8]]#1 step %[[VAL_2]] unordered {
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_9]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+// CHECK:             %[[VAL_12:.*]] = arith.addi %[[VAL_11]], %[[VAL_4]] : i32
+// CHECK:             %[[VAL_13:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_9]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_13]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/HLFIR/order_assignments/where-scheduling.f90 b/flang/test/HLFIR/order_assignments/where-scheduling.f90
index 3010476d4a188..6feaba0d3389a 100644
--- a/flang/test/HLFIR/order_assignments/where-scheduling.f90
+++ b/flang/test/HLFIR/order_assignments/where-scheduling.f90
@@ -134,7 +134,7 @@ end function f
 !CHECK-NEXT: run 1 save    : where/mask
 !CHECK-NEXT: run 2 evaluate: where/region_assign1
 !CHECK-LABEL: ------------ scheduling where in _QPonly_once ------------
-!CHECK-NEXT: unknown effect: %{{[0-9]+}} = llvm.intr.stacksave : !llvm.ptr
+!CHECK-NEXT: unknown effect: %11 = fir.call @_QPcall_me_only_once() fastmath<contract> : () -> !fir.array<10x!fir.logical<4>>
 !CHECK-NEXT: saving eval because write effect prevents re-evaluation
 !CHECK-NEXT: run 1 save  (w): where/mask
 !CHECK-NEXT: run 2 evaluate: where/region_assign1
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-sum.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-sum.fir
new file mode 100644
index 0000000000000..54a592a66670f
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-sum.fir
@@ -0,0 +1,431 @@
+// RUN: fir-opt --simplify-hlfir-intrinsics -flang-simplify-hlfir-sum %s | FileCheck %s
+
+// box with known extents
+func.func @sum_box_known_extents(%arg0: !fir.box<!fir.array<2x3xi32>>) {
+  %cst = arith.constant 2 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!fir.box<!fir.array<2x3xi32>>, i32) -> !hlfir.expr<2xi32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_box_known_extents(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<2x3xi32>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_5:.*]] = hlfir.elemental %[[VAL_4]] unordered : (!fir.shape<1>) -> !hlfir.expr<2xi32> {
+// CHECK:           ^bb0(%[[VAL_6:.*]]: index):
+// CHECK:             %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_7]] to %[[VAL_3]] step %[[VAL_7]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i32) {
+// CHECK:               %[[VAL_12:.*]] = arith.constant 0 : index
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_12]] : (!fir.box<!fir.array<2x3xi32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_14]] : (!fir.box<!fir.array<2x3xi32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_6]], %[[VAL_17]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_16]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_10]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_18]], %[[VAL_20]])  : (!fir.box<!fir.array<2x3xi32>>, index, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<i32>
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : i32
+// CHECK:               fir.result %[[VAL_23]] : i32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_9]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// expr with known extents
+func.func @sum_expr_known_extents(%arg0: !hlfir.expr<2x3xi32>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!hlfir.expr<2x3xi32>, i32) -> !hlfir.expr<3xi32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_expr_known_extents(
+// CHECK-SAME:                                      %[[VAL_0:.*]]: !hlfir.expr<2x3xi32>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_5:.*]] = hlfir.elemental %[[VAL_4]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xi32> {
+// CHECK:           ^bb0(%[[VAL_6:.*]]: index):
+// CHECK:             %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_7]] to %[[VAL_2]] step %[[VAL_7]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i32) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_6]] : (!hlfir.expr<2x3xi32>, index, index) -> i32
+// CHECK:               %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_13]] : i32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_9]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// box with unknown extent
+func.func @sum_box_unknown_extent1(%arg0: !fir.box<!fir.array<?x3xcomplex<f64>>>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!fir.box<!fir.array<?x3xcomplex<f64>>>, i32) -> !hlfir.expr<3xcomplex<f64>>
+  return
+}
+// CHECK-LABEL:   func.func @sum_box_unknown_extent1(
+// CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x3xcomplex<f64>>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xcomplex<f64>> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f64
+// CHECK:             %[[VAL_10:.*]] = fir.undefined complex<f64>
+// CHECK:             %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_9]], [0 : index] : (complex<f64>, f64) -> complex<f64>
+// CHECK:             %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (complex<f64>, f64) -> complex<f64>
+// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_8]] to %[[VAL_3]]#1 step %[[VAL_8]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (complex<f64>) {
+// CHECK:               %[[VAL_16:.*]] = arith.constant 0 : index
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_16]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_18]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_20:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_20]] : index
+// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_7]], %[[VAL_23]] : index
+// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index, index) -> !fir.ref<complex<f64>>
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<complex<f64>>
+// CHECK:               %[[VAL_27:.*]] = fir.addc %[[VAL_15]], %[[VAL_26]] : complex<f64>
+// CHECK:               fir.result %[[VAL_27]] : complex<f64>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_13]] : complex<f64>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @sum_box_unknown_extent2(%arg0: !fir.box<!fir.array<?x3xcomplex<f64>>>) {
+  %cst = arith.constant 2 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!fir.box<!fir.array<?x3xcomplex<f64>>>, i32) -> !hlfir.expr<?xcomplex<f64>>
+  return
+}
+// CHECK-LABEL:   func.func @sum_box_unknown_extent2(
+// CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x3xcomplex<f64>>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_3]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xcomplex<f64>> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f64
+// CHECK:             %[[VAL_10:.*]] = fir.undefined complex<f64>
+// CHECK:             %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_9]], [0 : index] : (complex<f64>, f64) -> complex<f64>
+// CHECK:             %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (complex<f64>, f64) -> complex<f64>
+// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_8]] to %[[VAL_4]] step %[[VAL_8]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (complex<f64>) {
+// CHECK:               %[[VAL_16:.*]] = arith.constant 0 : index
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_16]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_18]] : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_20:.*]] = arith.constant 1 : index
+// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_7]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_20]] : index
+// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_14]], %[[VAL_23]] : index
+// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x3xcomplex<f64>>>, index, index) -> !fir.ref<complex<f64>>
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<complex<f64>>
+// CHECK:               %[[VAL_27:.*]] = fir.addc %[[VAL_15]], %[[VAL_26]] : complex<f64>
+// CHECK:               fir.result %[[VAL_27]] : complex<f64>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_13]] : complex<f64>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// expr with unknown extent
+func.func @sum_expr_unknown_extent1(%arg0: !hlfir.expr<?x3xf32>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!hlfir.expr<?x3xf32>, i32) -> !hlfir.expr<3xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_expr_unknown_extent1(
+// CHECK-SAME:                                         %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_3:.*]] = hlfir.get_extent %[[VAL_2]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_8]] to %[[VAL_3]] step %[[VAL_8]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_7]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:               %[[VAL_14:.*]] = arith.addf %[[VAL_12]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_14]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @sum_expr_unknown_extent2(%arg0: !hlfir.expr<?x3xf32>) {
+  %cst = arith.constant 2 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!hlfir.expr<?x3xf32>, i32) -> !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_expr_unknown_extent2(
+// CHECK-SAME:                                         %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_3:.*]] = hlfir.get_extent %[[VAL_2]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]] = hlfir.elemental %[[VAL_5]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:             %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_8]] to %[[VAL_4]] step %[[VAL_8]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_7]], %[[VAL_11]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:               %[[VAL_14:.*]] = arith.addf %[[VAL_12]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_14]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// scalar mask
+func.func @sum_scalar_mask(%arg0: !hlfir.expr<?x3xf32>, %mask: !fir.ref<!fir.logical<1>>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst mask %mask : (!hlfir.expr<?x3xf32>, i32, !fir.ref<!fir.logical<1>>) -> !hlfir.expr<3xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_scalar_mask(
+// CHECK-SAME:                               %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>,
+// CHECK-SAME:                               %[[VAL_1:.*]]: !fir.ref<!fir.logical<1>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.logical<1>>
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_9]] to %[[VAL_4]] step %[[VAL_9]] iter_args(%[[VAL_14:.*]] = %[[VAL_10]]) -> (f32) {
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_11]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f32) {
+// CHECK:                 %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_13]], %[[VAL_8]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_18:.*]] = arith.addf %[[VAL_14]], %[[VAL_17]] : f32
+// CHECK:                 fir.result %[[VAL_18]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// scalar boxed mask
+func.func @sum_scalar_boxed_mask(%arg0: !hlfir.expr<?x3xf32>, %mask: !fir.box<!fir.logical<1>>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst mask %mask : (!hlfir.expr<?x3xf32>, i32, !fir.box<!fir.logical<1>>) -> !hlfir.expr<3xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_scalar_boxed_mask(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>,
+// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.box<!fir.logical<1>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.logical<1>>) -> i1
+// CHECK:             %[[VAL_12:.*]] = fir.if %[[VAL_11]] -> (!fir.logical<1>) {
+// CHECK:               %[[VAL_13:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.logical<1>>) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               fir.result %[[VAL_14]] : !fir.logical<1>
+// CHECK:             } else {
+// CHECK:               %[[VAL_15:.*]] = arith.constant true
+// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i1) -> !fir.logical<1>
+// CHECK:               fir.result %[[VAL_16]] : !fir.logical<1>
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.do_loop %[[VAL_18:.*]] = %[[VAL_9]] to %[[VAL_4]] step %[[VAL_9]] iter_args(%[[VAL_19:.*]] = %[[VAL_10]]) -> (f32) {
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_21:.*]] = fir.if %[[VAL_20]] -> (f32) {
+// CHECK:                 %[[VAL_22:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_18]], %[[VAL_8]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_23:.*]] = arith.addf %[[VAL_19]], %[[VAL_22]] : f32
+// CHECK:                 fir.result %[[VAL_23]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_21]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_17]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// array mask
+func.func @sum_array_mask(%arg0: !hlfir.expr<?x3xf32>, %mask: !fir.box<!fir.array<?x3x!fir.logical<1>>>) {
+  %cst = arith.constant 2 : i32
+  %res = hlfir.sum %arg0 dim %cst mask %mask : (!hlfir.expr<?x3xf32>, i32, !fir.box<!fir.array<?x3x!fir.logical<1>>>) -> !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_array_mask(
+// CHECK-SAME:                              %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>,
+// CHECK-SAME:                              %[[VAL_1:.*]]: !fir.box<!fir.array<?x3x!fir.logical<1>>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x3x!fir.logical<1>>>) -> i1
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_9]] to %[[VAL_5]] step %[[VAL_9]] iter_args(%[[VAL_14:.*]] = %[[VAL_10]]) -> (f32) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_11]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]] = arith.constant 0 : index
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_16]] : (!fir.box<!fir.array<?x3x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.constant 1 : index
+// CHECK:                 %[[VAL_19:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_18]] : (!fir.box<!fir.array<?x3x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_20:.*]] = arith.constant 1 : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = arith.subi %[[VAL_19]]#0, %[[VAL_20]] : index
+// CHECK:                 %[[VAL_24:.*]] = arith.addi %[[VAL_13]], %[[VAL_23]] : index
+// CHECK:                 %[[VAL_25:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x3x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_26]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_27:.*]] = arith.constant true
+// CHECK:                 %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_28]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_30:.*]] = fir.if %[[VAL_29]] -> (f32) {
+// CHECK:                 %[[VAL_31:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_8]], %[[VAL_13]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_32:.*]] = arith.addf %[[VAL_14]], %[[VAL_31]] : f32
+// CHECK:                 fir.result %[[VAL_32]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_30]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// array expr mask
+func.func @sum_array_expr_mask(%arg0: !hlfir.expr<?x3xf32>, %mask: !hlfir.expr<?x3x!fir.logical<1>>) {
+  %cst = arith.constant 2 : i32
+  %res = hlfir.sum %arg0 dim %cst mask %mask : (!hlfir.expr<?x3xf32>, i32, !hlfir.expr<?x3x!fir.logical<1>>) -> !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_array_expr_mask(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: !hlfir.expr<?x3xf32>,
+// CHECK-SAME:                                   %[[VAL_1:.*]]: !hlfir.expr<?x3x!fir.logical<1>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x3xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+// CHECK:             %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_9]] to %[[VAL_5]] step %[[VAL_9]] iter_args(%[[VAL_13:.*]] = %[[VAL_10]]) -> (f32) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_8]], %[[VAL_12]] : (!hlfir.expr<?x3x!fir.logical<1>>, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f32) {
+// CHECK:                 %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_8]], %[[VAL_12]] : (!hlfir.expr<?x3xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_18:.*]] = arith.addf %[[VAL_13]], %[[VAL_17]] : f32
+// CHECK:                 fir.result %[[VAL_18]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_13]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_11]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// unordered floating point reduction
+func.func @sum_unordered_reduction(%arg0: !hlfir.expr<2x3xf32>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<2x3xf32>, i32) -> !hlfir.expr<3xf32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_unordered_reduction(
+// CHECK-SAME:                                       %[[VAL_0:.*]]: !hlfir.expr<2x3xf32>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_5:.*]] = hlfir.elemental %[[VAL_4]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+// CHECK:           ^bb0(%[[VAL_6:.*]]: index):
+// CHECK:             %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_7]] to %[[VAL_2]] step %[[VAL_7]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (f32) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_6]] : (!hlfir.expr<2x3xf32>, index, index) -> f32
+// CHECK:               %[[VAL_13:.*]] = arith.addf %[[VAL_11]], %[[VAL_12]] fastmath<reassoc> : f32
+// CHECK:               fir.result %[[VAL_13]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_9]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// negative: total reduction
+func.func @sum_total_reduction(%arg0: !fir.box<!fir.array<3xi32>>) {
+  %cst = arith.constant 1 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!fir.box<!fir.array<3xi32>>, i32) -> i32
+  return
+}
+// CHECK-LABEL:   func.func @sum_total_reduction(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: !fir.box<!fir.array<3xi32>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = hlfir.sum %[[VAL_0]] dim %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>, i32) -> i32
+// CHECK:           return
+// CHECK:         }
+
+// negative: non-const dim
+func.func @sum_non_const_dim(%arg0: !fir.box<!fir.array<3xi32>>, %dim: i32) {
+  %res = hlfir.sum %arg0 dim %dim : (!fir.box<!fir.array<3xi32>>, i32) -> i32
+  return
+}
+// CHECK-LABEL:   func.func @sum_non_const_dim(
+// CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.box<!fir.array<3xi32>>,
+// CHECK-SAME:                                 %[[VAL_1:.*]]: i32) {
+// CHECK:           %[[VAL_2:.*]] = hlfir.sum %[[VAL_0]] dim %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>, i32) -> i32
+// CHECK:           return
+// CHECK:         }
+
+// negative: invalid dim==0
+func.func @sum_invalid_dim0(%arg0: !hlfir.expr<2x3xi32>) {
+  %cst = arith.constant 0 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!hlfir.expr<2x3xi32>, i32) -> !hlfir.expr<3xi32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_invalid_dim0(
+// CHECK:           hlfir.sum %{{.*}} dim %{{.*}} : (!hlfir.expr<2x3xi32>, i32) -> !hlfir.expr<3xi32>
+
+// negative: invalid dim>rank
+func.func @sum_invalid_dim_big(%arg0: !hlfir.expr<2x3xi32>) {
+  %cst = arith.constant 3 : i32
+  %res = hlfir.sum %arg0 dim %cst : (!hlfir.expr<2x3xi32>, i32) -> !hlfir.expr<3xi32>
+  return
+}
+// CHECK-LABEL:   func.func @sum_invalid_dim_big(
+// CHECK:           hlfir.sum %{{.*}} dim %{{.*}} : (!hlfir.expr<2x3xi32>, i32) -> !hlfir.expr<3xi32>
diff --git a/flang/test/Integration/OpenMP/copyprivate.f90 b/flang/test/Integration/OpenMP/copyprivate.f90
index d32319a18c28b..d38fd20020f34 100644
--- a/flang/test/Integration/OpenMP/copyprivate.f90
+++ b/flang/test/Integration/OpenMP/copyprivate.f90
@@ -8,25 +8,25 @@
 
 !RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s
 
-!CHECK-DAG: define void @_copy_box_Uxi32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_10xi32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_i64(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_box_Uxi64(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_f32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_2x3xf32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_z32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_10xz32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_l32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_5xl32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_c8x8(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_10xc8x8(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_c16x5(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_rec__QFtest_typesTdt(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_box_heap_Uxi32(ptr %{{.*}}, ptr %{{.*}})
-!CHECK-DAG: define void @_copy_box_ptr_Uxc8x9(ptr %{{.*}}, ptr %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_Uxi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_10xi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_i64(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_Uxi64(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_f32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_2x3xf32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_z32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_10xz32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_l32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_5xl32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_c8x8(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_10xc8x8(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_c16x5(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_rec__QFtest_typesTdt(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_heap_Uxi32(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
+!CHECK-DAG: define internal void @_copy_box_ptr_Uxc8x9(ptr nocapture %{{.*}}, ptr nocapture %{{.*}})
 
-!CHECK-LABEL: define void @_copy_i32(
-!CHECK-SAME:                         ptr %[[DST:.*]], ptr %[[SRC:.*]]){{.*}} {
+!CHECK-LABEL: define internal void @_copy_i32(
+!CHECK-SAME:                         ptr nocapture %[[DST:.*]], ptr nocapture %[[SRC:.*]]){{.*}} {
 !CHECK-NEXT:    %[[SRC_VAL:.*]] = load i32, ptr %[[SRC]]
 !CHECK-NEXT:    store i32 %[[SRC_VAL]], ptr %[[DST]]
 !CHECK-NEXT:    ret void
diff --git a/flang/test/Integration/OpenMP/workshare-axpy.f90 b/flang/test/Integration/OpenMP/workshare-axpy.f90
index 0c4524f855290..12246e54d3432 100644
--- a/flang/test/Integration/OpenMP/workshare-axpy.f90
+++ b/flang/test/Integration/OpenMP/workshare-axpy.f90
@@ -13,7 +13,7 @@ subroutine sb1(a, x, y, z)
   integer :: a
   integer :: x(:)
   integer :: y(:)
-  integer :: z(:)
+  integer, allocatable :: z(:)
   !$omp parallel workshare
   z = a * x + y
   !$omp end parallel workshare
@@ -43,7 +43,7 @@ subroutine sb1(a, x, y, z)
 
 ! FIR:  func.func @_QPsb1
 ! FIR:    omp.parallel {
-! FIR:      omp.single copyprivate(%9 -> @_workshare_copy_i32 : !fir.ref<i32>, %10 -> @_workshare_copy_heap_Uxi32 : !fir.ref<!fir.heap<!fir.array<?xi32>>>) {
+! FIR:      omp.single copyprivate(%{{[a-z0-9]+}} -> @_workshare_copy_i32 : !fir.ref<i32>, %{{[a-z0-9]+}} -> @_workshare_copy_heap_Uxi32 : !fir.ref<!fir.heap<!fir.array<?xi32>>>) {
 ! FIR:        fir.allocmem
 ! FIR:      omp.wsloop {
 ! FIR:        omp.loop_nest
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index 525474a378ed6..5a675cbe1786d 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -27,7 +27,7 @@
 ! BOTH-LABEL: }
 
 ! BOTH-LABEL: define {{.*}}i64 @_QFPfn1
-! BOTH-SAME: (ptr %[[ARG1:.*]], ptr %[[ARG2:.*]], ptr %[[ARG3:.*]])
+! BOTH-SAME: (ptr nocapture %[[ARG1:.*]], ptr nocapture %[[ARG2:.*]], ptr nocapture %[[ARG3:.*]])
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
@@ -40,7 +40,7 @@
 ! BOTH-LABEL: }
 
 ! BOTH-LABEL: define {{.*}}i32 @_QFPfn2
-! BOTH-SAME: (ptr %[[FN2ARG1:.*]], ptr %[[FN2ARG2:.*]], ptr %[[FN2ARG3:.*]])
+! BOTH-SAME: (ptr nocapture %[[FN2ARG1:.*]], ptr nocapture %[[FN2ARG2:.*]], ptr nocapture %[[FN2ARG3:.*]])
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
 ! INTRINSICS-DAG: call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
diff --git a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
index aac569b6eb35b..09aefc055e6fa 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
@@ -98,3 +98,18 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub2
 ! CHECK: cuf.kernel
+
+subroutine sub3()
+  integer, device :: a(10), b(10)
+  integer :: lb = 1
+  integer :: n = 10
+  integer :: s = 1
+
+  !$cuf kernel do <<< *, * >>>
+  do i = lb, n, s
+    a(i) = a(i) * b(i)
+  end do
+end
+
+! CHECK-LABEL: func.func @_QPsub3
+! CHECK: cuf.kernel
diff --git a/flang/test/Lower/HLFIR/array-ctor-as-runtime-temp.f90 b/flang/test/Lower/HLFIR/array-ctor-as-runtime-temp.f90
index e1e65fc48baba..727eff7613e48 100644
--- a/flang/test/Lower/HLFIR/array-ctor-as-runtime-temp.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-as-runtime-temp.f90
@@ -17,12 +17,11 @@ subroutine test_loops()
 ! CHECK:           fir.store %[[VAL_6]] to %[[VAL_2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_7:.*]] = arith.constant false
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
-! CHECK:           %[[VAL_9:.*]] = arith.constant 80 : i32
 ! CHECK:           %[[VAL_10:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_11:.*]] = arith.constant 7 : i32
 ! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
-! CHECK:           %[[VAL_14:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_8]], %[[VAL_12]], %[[VAL_7]], %[[VAL_9]], %[[VAL_13]], %[[VAL_11]]) fastmath<contract> : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:           %[[VAL_14:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_8]], %[[VAL_12]], %[[VAL_7]], %[[VAL_13]], %[[VAL_11]]) fastmath<contract> : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i64
 ! CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i64) -> index
 ! CHECK:           %[[VAL_17:.*]] = fir.call @_QMarrayctorPibar() fastmath<contract> : () -> i32
@@ -86,7 +85,7 @@ subroutine test_arrays(a)
 ! CHECK:  %[[VAL_26:.*]] = arith.constant false
 ! CHECK:  %[[VAL_27:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_31:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
-! CHECK:  %[[VAL_33:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_27]], %[[VAL_31]], %[[VAL_26]], %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:  %[[VAL_33:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_27]], %[[VAL_31]], %[[VAL_26]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:  %[[VAL_34:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
 ! CHECK:  %[[VAL_35:.*]] = fir.call @_FortranAPushArrayConstructorValue(%[[VAL_27]], %[[VAL_34]]) {{.*}}: (!fir.llvm_ptr<i8>, !fir.box<none>) -> none
 ! CHECK:  %[[VAL_36:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
@@ -107,7 +106,7 @@ subroutine test_arrays_unpredictable_size()
 ! CHECK:  %[[VAL_9:.*]] = arith.constant false
 ! CHECK:  %[[VAL_10:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_14:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
-! CHECK:  %[[VAL_16:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_10]], %[[VAL_14]], %[[VAL_9]], %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:  %[[VAL_16:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_10]], %[[VAL_14]], %[[VAL_9]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:  fir.call @_QMarrayctorPrank1() {{.*}}: () -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:  %[[VAL_21:.*]] = fir.call @_FortranAPushArrayConstructorValue(%[[VAL_10]], %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.box<none>) -> none
 ! CHECK:  fir.call @_QMarrayctorPrank3() {{.*}}: () -> !fir.box<!fir.heap<!fir.array<?x?x?xi32>>>
diff --git a/flang/test/Lower/HLFIR/array-ctor-character.f90 b/flang/test/Lower/HLFIR/array-ctor-character.f90
index 881085b370ffe..7cbad5218f588 100644
--- a/flang/test/Lower/HLFIR/array-ctor-character.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-character.f90
@@ -52,7 +52,7 @@ subroutine test_dynamic_length()
 ! CHECK:  %[[VAL_15:.*]] = arith.constant true
 ! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_20:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<2x!fir.char<1,?>>>>>) -> !fir.ref<!fir.box<none>>
-! CHECK:  %[[VAL_22:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_16]], %[[VAL_20]], %[[VAL_15]], %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:  %[[VAL_22:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_16]], %[[VAL_20]], %[[VAL_15]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:  fir.call @_QMchararrayctorPchar_pointer(
 ! CHECK:  fir.call @_FortranAPushArrayConstructorValue(%[[VAL_16]], %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.box<none>) -> none
 ! CHECK:  fir.call @_QMchararrayctorPchar_pointer(
diff --git a/flang/test/Lower/HLFIR/array-ctor-derived.f90 b/flang/test/Lower/HLFIR/array-ctor-derived.f90
index 111225462a4bb..22f7fbd72cb59 100644
--- a/flang/test/Lower/HLFIR/array-ctor-derived.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-derived.f90
@@ -28,7 +28,7 @@ subroutine test_simple(s1, s2)
 ! CHECK:  %[[VAL_11:.*]] = arith.constant false
 ! CHECK:  %[[VAL_12:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<2x!fir.type<_QMtypesTsimple{i:i32,j:i32}>>>>>) -> !fir.ref<!fir.box<none>>
-! CHECK:  %[[VAL_18:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_12]], %[[VAL_16]], %[[VAL_11]], %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:  %[[VAL_18:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_12]], %[[VAL_16]], %[[VAL_11]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:  %[[VAL_19:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.ref<!fir.type<_QMtypesTsimple{i:i32,j:i32}>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_20:.*]] = fir.call @_FortranAPushArrayConstructorSimpleScalar(%[[VAL_12]], %[[VAL_19]]) {{.*}}: (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>) -> none
 ! CHECK:  %[[VAL_21:.*]] = fir.convert %[[VAL_5]]#1 : (!fir.ref<!fir.type<_QMtypesTsimple{i:i32,j:i32}>>) -> !fir.llvm_ptr<i8>
@@ -56,7 +56,7 @@ subroutine test_with_polymorphic(s1, s2)
 ! CHECK:  %[[VAL_11:.*]] = arith.constant false
 ! CHECK:  %[[VAL_12:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<2x!fir.type<_QMtypesTsimple{i:i32,j:i32}>>>>>) -> !fir.ref<!fir.box<none>>
-! CHECK:  %[[VAL_18:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_12]], %[[VAL_16]], %[[VAL_11]], %{{.*}}, %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:  %[[VAL_18:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_12]], %[[VAL_16]], %[[VAL_11]], %{{.*}}, %{{.*}}) {{.*}}: (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:  %[[VAL_19A:.*]] = fir.box_addr %[[VAL_4]]#1 : (!fir.class<!fir.type<_QMtypesTsimple{i:i32,j:i32}>>) -> !fir.ref<!fir.type<_QMtypesTsimple{i:i32,j:i32}>>
 ! CHECK:  %[[VAL_19:.*]] = fir.convert %[[VAL_19A]] : (!fir.ref<!fir.type<_QMtypesTsimple{i:i32,j:i32}>>) -> !fir.llvm_ptr<i8>
 ! CHECK:  %[[VAL_20:.*]] = fir.call @_FortranAPushArrayConstructorSimpleScalar(%[[VAL_12]], %[[VAL_19]]) {{.*}}: (!fir.llvm_ptr<i8>, !fir.llvm_ptr<i8>) -> none
diff --git a/flang/test/Lower/HLFIR/calls-array-results.f90 b/flang/test/Lower/HLFIR/calls-array-results.f90
new file mode 100644
index 0000000000000..d91844cc2e6f8
--- /dev/null
+++ b/flang/test/Lower/HLFIR/calls-array-results.f90
@@ -0,0 +1,131 @@
+! RUN: bbc -emit-hlfir -o - %s -I nowhere | FileCheck %s
+
+subroutine simple_test()
+  implicit none
+  interface
+    function array_func()
+      real :: array_func(10)
+    end function
+  end interface
+  real :: x(10)
+  x = array_func()
+end subroutine
+
+subroutine arg_test(n)
+  implicit none
+  interface
+    function array_func_2(n)
+      integer(8) :: n
+      real :: array_func_2(n)
+    end function
+  end interface
+  integer(8) :: n
+  real :: x(n)
+  x = array_func_2(n)
+end subroutine
+
+module type_defs
+  interface
+    function array_func()
+      real :: array_func(10)
+    end function
+  end interface
+  type t
+    contains
+    procedure, nopass :: array_func => array_func
+  end type
+end module
+
+subroutine dispatch_test(x, a)
+  use type_defs, only : t
+  implicit none
+  real :: x(10)
+  class(t) :: a
+  x = a%array_func()
+end subroutine
+
+! CHECK-LABEL:   func.func @_QPsimple_test() {
+! CHECK:           %[[VAL_0:.*]] = arith.constant 10 : index
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<10xf32> {bindc_name = "x", uniq_name = "_QFsimple_testEx"}
+! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_2]]) {uniq_name = "_QFsimple_testEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 10 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_6:.*]] = arith.subi %[[VAL_4]], %[[VAL_5]] : i64
+! CHECK:           %[[VAL_7:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_8:.*]] = arith.addi %[[VAL_6]], %[[VAL_7]] : i64
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i64) -> index
+! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+! CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_9]], %[[VAL_10]] : index
+! CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_14:.*]] = hlfir.eval_in_mem shape %[[VAL_13]] : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+! CHECK:           ^bb0(%[[VAL_15:.*]]: !fir.ref<!fir.array<10xf32>>):
+! CHECK:             %[[VAL_16:.*]] = fir.call @_QParray_func() fastmath<contract> : () -> !fir.array<10xf32>
+! CHECK:             fir.save_result %[[VAL_16]] to %[[VAL_15]](%[[VAL_13]]) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+! CHECK:           }
+! CHECK:           hlfir.assign %[[VAL_14]] to %[[VAL_3]]#0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+! CHECK:           hlfir.destroy %[[VAL_14]] : !hlfir.expr<10xf32>
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QParg_test(
+! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "n"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFarg_testEn"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i64>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i64) -> index
+! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_6:.*]] = arith.cmpi sgt, %[[VAL_4]], %[[VAL_5]] : index
+! CHECK:           %[[VAL_7:.*]] = arith.select %[[VAL_6]], %[[VAL_4]], %[[VAL_5]] : index
+! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<?xf32>, %[[VAL_7]] {bindc_name = "x", uniq_name = "_QFarg_testEx"}
+! CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_9]]) {uniq_name = "_QFarg_testEx"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]]#1 {uniq_name = "_QFarg_testFarray_func_2En"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_12:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i64>
+! CHECK:           %[[VAL_13:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_14:.*]] = arith.subi %[[VAL_12]], %[[VAL_13]] : i64
+! CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_16:.*]] = arith.addi %[[VAL_14]], %[[VAL_15]] : i64
+! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i64) -> index
+! CHECK:           %[[VAL_18:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_17]], %[[VAL_18]] : index
+! CHECK:           %[[VAL_20:.*]] = arith.select %[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index
+! CHECK:           %[[VAL_21:.*]] = fir.shape %[[VAL_20]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_22:.*]] = hlfir.eval_in_mem shape %[[VAL_21]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:           ^bb0(%[[VAL_23:.*]]: !fir.ref<!fir.array<?xf32>>):
+! CHECK:             %[[VAL_24:.*]] = fir.call @_QParray_func_2(%[[VAL_2]]#1) fastmath<contract> : (!fir.ref<i64>) -> !fir.array<?xf32>
+! CHECK:             fir.save_result %[[VAL_24]] to %[[VAL_23]](%[[VAL_21]]) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+! CHECK:           }
+! CHECK:           hlfir.assign %[[VAL_22]] to %[[VAL_10]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+! CHECK:           hlfir.destroy %[[VAL_22]] : !hlfir.expr<?xf32>
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPdispatch_test(
+! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "x"},
+! CHECK-SAME:                                %[[VAL_1:.*]]: !fir.class<!fir.type<_QMtype_defsTt>> {fir.bindc_name = "a"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFdispatch_testEa"} : (!fir.class<!fir.type<_QMtype_defsTt>>, !fir.dscope) -> (!fir.class<!fir.type<_QMtype_defsTt>>, !fir.class<!fir.type<_QMtype_defsTt>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 10 : index
+! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) dummy_scope %[[VAL_2]] {uniq_name = "_QFdispatch_testEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK:           %[[VAL_7:.*]] = arith.constant 10 : i64
+! CHECK:           %[[VAL_8:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_9:.*]] = arith.subi %[[VAL_7]], %[[VAL_8]] : i64
+! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_10]] : i64
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i64) -> index
+! CHECK:           %[[VAL_13:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_13]] : index
+! CHECK:           %[[VAL_15:.*]] = arith.select %[[VAL_14]], %[[VAL_12]], %[[VAL_13]] : index
+! CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_15]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_17:.*]] = hlfir.eval_in_mem shape %[[VAL_16]] : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+! CHECK:           ^bb0(%[[VAL_18:.*]]: !fir.ref<!fir.array<10xf32>>):
+! CHECK:             %[[VAL_19:.*]] = fir.dispatch "array_func"(%[[VAL_3]]#1 : !fir.class<!fir.type<_QMtype_defsTt>>) -> !fir.array<10xf32>
+! CHECK:             fir.save_result %[[VAL_19]] to %[[VAL_18]](%[[VAL_16]]) : !fir.array<10xf32>, !fir.ref<!fir.array<10xf32>>, !fir.shape<1>
+! CHECK:           }
+! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_6]]#0 : !hlfir.expr<10xf32>, !fir.ref<!fir.array<10xf32>>
+! CHECK:           hlfir.destroy %[[VAL_17]] : !hlfir.expr<10xf32>
+! CHECK:           return
+! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/cshift.f90 b/flang/test/Lower/HLFIR/cshift.f90
new file mode 100644
index 0000000000000..c3743068da4d7
--- /dev/null
+++ b/flang/test/Lower/HLFIR/cshift.f90
@@ -0,0 +1,218 @@
+! Test lowering of CSHIFT intrinsic to HLFIR
+! RUN: bbc -emit-hlfir -o - -I nowhere %s 2>&1 | FileCheck %s
+
+module types
+  type t
+  end type t
+end module types
+
+! 1d shift by scalar
+subroutine cshift1(a, s)
+  integer :: a(:), s
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift1(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 1d shift by scalar with dim
+subroutine cshift2(a, s)
+  integer :: a(:), s
+  a = CSHIFT(a, 2, 1)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift2(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] dim %[[VAL_6]] : (!fir.box<!fir.array<?xi32>>, i32, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_3]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by scalar
+subroutine cshift3(a, s)
+  integer :: a(:,:), s
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift3(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by scalar with dim
+subroutine cshift4(a, s)
+  integer :: a(:,:), s
+  a = CSHIFT(a, 2, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift4(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] dim %[[VAL_6]] : (!fir.box<!fir.array<?x?xi32>>, i32, i32) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by array
+subroutine cshift5(a, s)
+  integer :: a(:,:), s(:)
+  a = CSHIFT(a, s)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift5(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_4]]#0 : (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_5]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by array expr
+subroutine cshift6(a, s)
+  integer :: a(:,:), s(:)
+  a = CSHIFT(a, s + 1)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift6(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_6]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]]#1 : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32>
+! CHECK:           %[[VAL_14:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_9]] : (!fir.box<!fir.array<?x?xi32>>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_14]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_14]] : !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.destroy %[[VAL_9]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 1d character(10,2) shift by scalar
+subroutine cshift7(a, s)
+  character(10,2) :: a(:)
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift7(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<2,10>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_6:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.cshift %[[VAL_4]]#0 %[[VAL_6]] : (!fir.box<!fir.array<?x!fir.char<2,10>>>, i32) -> !hlfir.expr<?x!fir.char<2,10>>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_4]]#0 : !hlfir.expr<?x!fir.char<2,10>>, !fir.box<!fir.array<?x!fir.char<2,10>>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x!fir.char<2,10>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d character(*) shift by scalar
+subroutine cshift8(a, s)
+  character(*) :: a(:)
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift8(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32) -> !hlfir.expr<?x!fir.char<1,?>>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x!fir.char<1,?>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d type(t) shift by scalar
+subroutine cshift9(a, s)
+  use types
+  type(t) :: a(:)
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift9(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.cshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x!fir.type<_QMtypesTt>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d class(t) shift by scalar
+subroutine cshift10(a, s)
+  use types
+  class(t), allocatable :: a(:)
+  a = CSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift10(
+! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+! CHECK:           %[[VAL_7:.*]] = hlfir.cshift %[[VAL_6]] %[[VAL_5]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_3]]#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+! CHECK:           return
+! CHECK:         }
+
+! 1d shift by scalar with variable dim
+subroutine cshift11(a, s, d)
+  integer :: a(:), s, d
+  a = CSHIFT(a, 2, d)
+end subroutine
+! CHECK-LABEL:   func.func @_QPcshift11(
+! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"},
+! CHECK-SAME:                           %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "d"}) {
+! CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFcshift11Ea"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFcshift11Ed"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFcshift11Es"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_9:.*]] = hlfir.cshift %[[VAL_4]]#0 %[[VAL_7]] dim %[[VAL_8]] : (!fir.box<!fir.array<?xi32>>, i32, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_9]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/goto-do-body.f90 b/flang/test/Lower/HLFIR/goto-do-body.f90
index 383b839e591e3..9e2c07f8fa292 100644
--- a/flang/test/Lower/HLFIR/goto-do-body.f90
+++ b/flang/test/Lower/HLFIR/goto-do-body.f90
@@ -40,7 +40,7 @@ subroutine sub1()
 ! CHECK:    %[[TMP5:.*]] = arith.subi %[[TMP4]], %[[C1]] : i32
 ! CHECK:    fir.store %[[TMP5]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP6:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
-! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] : i32
+! CHECK:    %[[TMP7:.*]] = arith.addi %[[TMP6]], %[[C1]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP7]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -83,8 +83,8 @@ subroutine sub2()
 
   do i = 1, 2, 3 * j - 8
 ! CHECK:    %[[TMP2:.*]] = fir.load %[[J]]#0 : !fir.ref<i32>
-! CHECK:    %[[TMP3:.*]] = arith.muli %[[TMP2]], %[[C3]] : i32
-! CHECK:    %[[STEP:.*]] = arith.subi %[[TMP3]], %[[C8]] : i32
+! CHECK:    %[[TMP3:.*]] = arith.muli %[[TMP2]], %[[C3]] overflow<nsw> : i32
+! CHECK:    %[[STEP:.*]] = arith.subi %[[TMP3]], %[[C8]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[STEP]] to %[[STEP_VAR:.*]] : !fir.ref<i32>
 ! CHECK:    %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[C_7]] : i32
 ! CHECK:    %[[TMP5:.*]] = arith.divsi %[[TMP4]], %[[STEP]] : i32
@@ -104,7 +104,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP9]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP10:.*]] = fir.load %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] : i32
+! CHECK:    %[[TMP11:.*]] = arith.addi %[[TMP10]], %[[STEP_VAL]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP11]] to %[[I]]#1 : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90 b/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
index 3f97a9f848d43..26d19c308feae 100644
--- a/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
+++ b/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
@@ -17,12 +17,9 @@ subroutine test1(x)
   call callee(cshift(x, 1))
 end subroutine test1
 ! CHECK-LABEL:   func.func @_QPtest1(
-! CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
-! CHECK:           %[[VAL_22:.*]] = arith.constant true
-! CHECK:           %[[VAL_23:.*]] = hlfir.as_expr %[[VAL_21]]#0 move %[[VAL_22]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
-! CHECK:           %[[VAL_24:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_21]]#0, %[[VAL_24]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-! CHECK:           %[[VAL_26:.*]] = fir.shape %[[VAL_25]]#1 : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_23:.*]] = hlfir.cshift %{{.*}} %[[VAL_4]] : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+! CHECK:           %[[VAL_26:.*]] = hlfir.shape_of %[[VAL_23]] : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>) -> !fir.shape<1>
 ! CHECK:           %[[VAL_27:.*]]:3 = hlfir.associate %[[VAL_23]](%[[VAL_26]]) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1)
 ! CHECK:           %[[VAL_28:.*]] = fir.rebox %[[VAL_27]]#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
 ! CHECK:           %[[VAL_29:.*]]:2 = hlfir.copy_in %[[VAL_28]] to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i1)
diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90
index 41d08c14f5fa9..ed9ee5d0ac363 100644
--- a/flang/test/Lower/HLFIR/structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/structure-constructor.f90
@@ -273,12 +273,11 @@ end subroutine test6
 ! CHECK:           fir.store %[[VAL_49]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.heap<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>>>
 ! CHECK:           %[[VAL_50:.*]] = arith.constant false
 ! CHECK:           %[[VAL_51:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.array<10xi64>>) -> !fir.llvm_ptr<i8>
-! CHECK:           %[[VAL_52:.*]] = arith.constant 80 : i32
 ! CHECK:           %[[VAL_53:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_54:.*]] = arith.constant {{[0-9]*}} : i32
 ! CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>>>) -> !fir.ref<!fir.box<none>>
 ! CHECK:           %[[VAL_56:.*]] = fir.convert %[[VAL_53]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
-! CHECK:           %[[VAL_57:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_51]], %[[VAL_55]], %[[VAL_50]], %[[VAL_52]], %[[VAL_56]], %[[VAL_54]]) fastmath<contract> : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, i32, !fir.ref<i8>, i32) -> none
+! CHECK:           %[[VAL_57:.*]] = fir.call @_FortranAInitArrayConstructorVector(%[[VAL_51]], %[[VAL_55]], %[[VAL_50]], %[[VAL_56]], %[[VAL_54]]) fastmath<contract> : (!fir.llvm_ptr<i8>, !fir.ref<!fir.box<none>>, i1, !fir.ref<i8>, i32) -> none
 ! CHECK:           %[[VAL_58:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>)
 ! CHECK:           %[[VAL_59:.*]] = fir.embox %[[VAL_58]]#0 : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> !fir.box<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>
 ! CHECK:           %[[VAL_60:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
diff --git a/flang/test/Lower/HLFIR/where-nonelemental.f90 b/flang/test/Lower/HLFIR/where-nonelemental.f90
index 643f417c47674..7be5831890012 100644
--- a/flang/test/Lower/HLFIR/where-nonelemental.f90
+++ b/flang/test/Lower/HLFIR/where-nonelemental.f90
@@ -26,11 +26,12 @@ real elemental function elem_func(x)
 ! CHECK-LABEL:   func.func @_QPtest_where(
 ! CHECK:           hlfir.where {
 ! CHECK-NOT: hlfir.exactly_once
-! CHECK:             %[[VAL_17:.*]] = llvm.intr.stacksave : !llvm.ptr
-! CHECK:             %[[VAL_19:.*]] = fir.call @_QPlogical_func1() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
-! CHECK:             hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
-! CHECK:               llvm.intr.stackrestore %[[VAL_17]] : !llvm.ptr
-! CHECK:             }
+! CHECK:               %[[VAL_19:.*]] = hlfir.eval_in_mem {{.*}} {
+! CHECK:                 fir.call @_QPlogical_func1() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:               }
+! CHECK:               hlfir.yield %[[VAL_19]] : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                 hlfir.destroy %[[VAL_19]]
+! CHECK:               }
 ! CHECK:           } do {
 ! CHECK:             hlfir.region_assign {
 ! CHECK:               %[[VAL_24:.*]] = hlfir.exactly_once : f32 {
@@ -70,10 +71,11 @@ real elemental function elem_func(x)
 ! CHECK:             }
 ! CHECK:             hlfir.elsewhere mask {
 ! CHECK:               %[[VAL_62:.*]] = hlfir.exactly_once : !hlfir.expr<100x!fir.logical<4>> {
-! CHECK:                 %[[VAL_72:.*]] = llvm.intr.stacksave : !llvm.ptr
-! CHECK:                 fir.call @_QPlogical_func2() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
-! CHECK:                 hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
-! CHECK:                   llvm.intr.stackrestore %[[VAL_72]] : !llvm.ptr
+! CHECK:                 %[[VAL_72:.*]] = hlfir.eval_in_mem {{.*}} {
+! CHECK:                  fir.call @_QPlogical_func2() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:                 }
+! CHECK:                 hlfir.yield %[[VAL_72]] : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                   hlfir.destroy %[[VAL_72]]
 ! CHECK:                 }
 ! CHECK:               }
 ! CHECK:               hlfir.yield %[[VAL_62]] : !hlfir.expr<100x!fir.logical<4>>
@@ -123,11 +125,12 @@ integer pure function pure_ifoo()
 ! CHECK:           }  (%[[VAL_10:.*]]: i32) {
 ! CHECK:             %[[VAL_11:.*]] = hlfir.forall_index "i" %[[VAL_10]] : (i32) -> !fir.ref<i32>
 ! CHECK:             hlfir.where {
-! CHECK:               %[[VAL_21:.*]] = llvm.intr.stacksave : !llvm.ptr
 ! CHECK-NOT: hlfir.exactly_once
-! CHECK:               %[[VAL_23:.*]] = fir.call @_QPpure_logical_func1() proc_attrs<pure> fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
-! CHECK:               hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
-! CHECK:                 llvm.intr.stackrestore %[[VAL_21]] : !llvm.ptr
+! CHECK:               %[[VAL_23:.*]] = hlfir.eval_in_mem {{.*}} {
+! CHECK:                  fir.call @_QPpure_logical_func1() proc_attrs<pure> fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:               }
+! CHECK:               hlfir.yield %[[VAL_23]] : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                 hlfir.destroy %[[VAL_23]]
 ! CHECK:               }
 ! CHECK:             } do {
 ! CHECK:               hlfir.region_assign {
@@ -172,10 +175,11 @@ integer pure function pure_ifoo()
 ! CHECK:               }
 ! CHECK:               hlfir.elsewhere mask {
 ! CHECK:                 %[[VAL_129:.*]] = hlfir.exactly_once : !hlfir.expr<100x!fir.logical<4>> {
-! CHECK:                   %[[VAL_139:.*]] = llvm.intr.stacksave : !llvm.ptr
-! CHECK:                   %[[VAL_141:.*]] = fir.call @_QPpure_logical_func2() proc_attrs<pure> fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
-! CHECK:                   hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
-! CHECK:                     llvm.intr.stackrestore %[[VAL_139]] : !llvm.ptr
+! CHECK:                   %[[VAL_139:.*]] = hlfir.eval_in_mem {{.*}} {
+! CHECK:                    fir.call @_QPpure_logical_func2() proc_attrs<pure> fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:                   }
+! CHECK:                   hlfir.yield %[[VAL_139]] : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                     hlfir.destroy %[[VAL_139]]
 ! CHECK:                   }
 ! CHECK:                 }
 ! CHECK:                 hlfir.yield %[[VAL_129]] : !hlfir.expr<100x!fir.logical<4>>
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index b0c76ff3845f8..f3f9bbe4a76a2 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -171,12 +171,12 @@ end subroutine target_allocatable
 ! CHECK_SAME        %[[CHAR_VAR_DESC_MAP]] -> %[[MAPPED_ARG3:.[^,]+]] :
 ! CHECK-SAME        !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.array<?xf32>>>, !fir.ref<!fir.boxchar<1>>)
 ! CHECK-SAME:     private(
-! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]],
+! CHECK-SAME:       @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:[^,]+]] [map_idx=1],
 ! CHECK-SAME:       @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:[^,]+]],
 ! CHECK-SAME:       @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:[^,]+]],
-! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:[^,]+]],
+! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:[^,]+]] [map_idx=2],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
-! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] :
+! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] [map_idx=3] :
 ! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.box<!fir.array<?xf32>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90
new file mode 100644
index 0000000000000..66fd120085c78
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/wsloop.f90
@@ -0,0 +1,42 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN:   | FileCheck %s
+
+subroutine wsloop_private
+    implicit none
+    integer :: x, i
+
+    !$omp parallel do firstprivate(x)
+    do i = 0, 10
+      x = x + i
+    end do
+end subroutine wsloop_private
+
+! CHECK: omp.private {type = private} @[[I_PRIVATIZER:.*i_private_ref_i32]]
+! CHECK: omp.private {type = firstprivate} @[[X_PRIVATIZER:.*x_firstprivate_ref_i32]]
+
+! CHECK: func.func @{{.*}}() {
+! CHECK:   %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}i"}
+! CHECK:   %[[X_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}x"}
+
+! CHECK:   omp.parallel {
+! CHECK:     omp.wsloop private(
+! CHECK-SAME:  @[[X_PRIVATIZER]] %[[X_DECL]]#0 -> %[[X_ARG:[^[:space:]]+]],
+! CHECK-SAME:  @[[I_PRIVATIZER]] %[[I_DECL]]#0 -> %[[I_ARG:.*]] : {{.*}}) {
+
+! CHECK:       omp.loop_nest (%[[IV:.*]]) : i32 = {{.*}} {
+! CHECK:         %[[X_PRIV_DECL:.*]]:2 = hlfir.declare %[[X_ARG]] {uniq_name = "{{.*}}x"}
+! CHECK:         %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_ARG]] {uniq_name = "{{.*}}i"}
+! CHECK:         fir.store %[[IV]] to %[[I_PRIV_DECL]]#1
+! CHECK:         %[[X_VAL:.*]] = fir.load %[[X_PRIV_DECL]]#0
+! CHECK:         %[[I_VAL:.*]] = fir.load %[[I_PRIV_DECL]]#0
+! CHECK:         %[[ADD_VAL:.*]] = arith.addi %[[X_VAL]], %[[I_VAL]]
+! CHECK:         hlfir.assign %[[ADD_VAL]] to %[[X_PRIV_DECL]]#0
+! CHECK:         omp.yield
+! CHECK:       }
+! CHECK:     }
+
+! CHECK:     omp.terminator
+! CHECK:   }
+! CHECK: }
diff --git a/flang/test/Lower/OpenMP/Todo/atomic-compare.f90 b/flang/test/Lower/OpenMP/Todo/atomic-compare.f90
new file mode 100644
index 0000000000000..88ec6fe910b9e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/atomic-compare.f90
@@ -0,0 +1,11 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: OpenMP atomic compare
+program p
+  integer :: x
+  logical :: r
+  !$omp atomic compare
+  if (x .eq. 0) then
+     x = 2
+  end if
+end program p
diff --git a/flang/test/Lower/OpenMP/copyprivate.f90 b/flang/test/Lower/OpenMP/copyprivate.f90
index 3f72343be666a..761e6190ed6ef 100644
--- a/flang/test/Lower/OpenMP/copyprivate.f90
+++ b/flang/test/Lower/OpenMP/copyprivate.f90
@@ -31,7 +31,7 @@
 !CHECK-DAG: func private @_copy_box_ptr_Uxc8x9(%{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,9>>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,9>>>>>)
 
 !CHECK-LABEL: func private @_copy_i32(
-!CHECK-SAME:                  %[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>) {
+!CHECK-SAME:                  %[[ARG0:.*]]: !fir.ref<i32>, %[[ARG1:.*]]: !fir.ref<i32>) attributes {llvm.linkage = #llvm.linkage<internal>} {
 !CHECK-NEXT:    %[[DST:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_copy_i32_dst"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-NEXT:    %[[SRC:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_copy_i32_src"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-NEXT:    %[[SRC_VAL:.*]] = fir.load %[[SRC]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/copyprivate2.f90 b/flang/test/Lower/OpenMP/copyprivate2.f90
index 848ebe39e45f4..3412ba2c63c4d 100644
--- a/flang/test/Lower/OpenMP/copyprivate2.f90
+++ b/flang/test/Lower/OpenMP/copyprivate2.f90
@@ -3,7 +3,8 @@
 
 !CHECK-LABEL: func private @_copy_box_ptr_i32(
 !CHECK-SAME:                  %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>>,
-!CHECK-SAME:                  %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>>) {
+!CHECK-SAME:                  %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>>)
+!CHECK-SAME:                  attributes {llvm.linkage = #llvm.linkage<internal>} {
 !CHECK-NEXT:    %[[DST:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<pointer>,
 !CHECK-SAME:      uniq_name = "_copy_box_ptr_i32_dst"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) ->
 !CHECK-SAME:      (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
@@ -17,7 +18,8 @@
 
 !CHECK-LABEL: func private @_copy_box_heap_Uxi32(
 !CHECK-SAME:        %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>,
-!CHECK-SAME:        %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+!CHECK-SAME:        %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+!CHECK-SAME:        attributes {llvm.linkage = #llvm.linkage<internal>} {
 !CHECK-NEXT:    %[[DST:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>,
 !CHECK-SAME:      uniq_name = "_copy_box_heap_Uxi32_dst"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) ->
 !CHECK-SAME:      (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
diff --git a/flang/test/Lower/OpenMP/declare-target-interface.f90 b/flang/test/Lower/OpenMP/declare-target-interface.f90
new file mode 100644
index 0000000000000..e92d1905c7ad0
--- /dev/null
+++ b/flang/test/Lower/OpenMP/declare-target-interface.f90
@@ -0,0 +1,11 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s
+
+!CHECK: module attributes
+module iface
+interface
+  subroutine a()
+  !$omp declare target
+  end subroutine
+end interface
+end module
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90 b/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90
new file mode 100644
index 0000000000000..0eeebe0afea54
--- /dev/null
+++ b/flang/test/Lower/OpenMP/delayed-privatization-default-init.f90
@@ -0,0 +1,47 @@
+! Test delayed privatization for derived types with default initialization.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 |\
+! RUN:   FileCheck %s
+
+subroutine delayed_privatization_default_init
+  implicit none
+  type t
+    integer :: i = 2
+  end type
+  integer :: i, res(4)
+  type(t) :: a
+  !$omp parallel private(a)
+    call do_something(a%i)
+  !$omp end parallel
+end subroutine
+
+subroutine delayed_privatization_default_init_firstprivate
+  implicit none
+  type t
+    integer :: i = 2
+  end type
+  integer :: i, res(4)
+  type(t) :: a
+  !$omp parallel firstprivate(a)
+    call do_something(a%i)
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL:   omp.private {type = firstprivate} @_QFdelayed_privatization_default_init_firstprivateEa_firstprivate_ref_rec__QFdelayed_privatization_default_init_firstprivateTt : !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>> alloc {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>):
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFdelayed_privatization_default_init_firstprivateEa"}
+! CHECK-NEXT:      %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFdelayed_privatization_default_init_firstprivateEa"} : (!fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>, !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>)
+! CHECK:           omp.yield(%[[VAL_9]]#0 : !fir.ref<!fir.type<_QFdelayed_privatization_default_init_firstprivateTt{i:i32}>>)
+! CHECK:   }
+
+! CHECK-LABEL:   omp.private {type = private} @_QFdelayed_privatization_default_initEa_private_ref_rec__QFdelayed_privatization_default_initTt : !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>> alloc {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>):
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QFdelayed_privatization_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFdelayed_privatization_default_initEa"}
+! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]] : (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.box<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> !fir.box<none>
+! CHECK:           %[[VAL_8:.*]] = fir.call @_FortranAInitialize(%[[VAL_6]],{{.*}}
+! CHECK-NEXT:      %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFdelayed_privatization_default_initEa"} : (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>, !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>)
+! CHECK:           omp.yield(%[[VAL_9]]#0 : !fir.ref<!fir.type<_QFdelayed_privatization_default_initTt{i:i32}>>)
+! CHECK:   }
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90 b/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
index 796e4720c8c95..c96b0b49fd530 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-pointer.f90
@@ -20,6 +20,9 @@ subroutine delayed_privatization_pointer
 ! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]):
 
 ! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "var1", pinned, uniq_name = "_QFdelayed_privatization_pointerEvar1"}
+! CHECK-NEXT:   %[[NULL:.*]] = fir.zero_bits !fir.ptr<i32>
+! CHECK-NEXT:   %[[INIT:.*]] = fir.embox %[[NULL]] : (!fir.ptr<i32>) -> !fir.box<!fir.ptr<i32>>
+! CHECK-NEXT:   fir.store %[[INIT]] to %[[PRIV_ALLOC]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
 ! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]])
 
diff --git a/flang/test/Lower/OpenMP/generic-loop-rewriting.f90 b/flang/test/Lower/OpenMP/generic-loop-rewriting.f90
new file mode 100644
index 0000000000000..fa26425356dd9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/generic-loop-rewriting.f90
@@ -0,0 +1,41 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+subroutine target_teams_loop
+    implicit none
+    integer :: x, i
+
+    !$omp target teams loop
+    do i = 0, 10
+      x = x + i
+    end do
+end subroutine target_teams_loop
+
+!CHECK-LABEL: func.func @_QPtarget_teams_loop
+!CHECK:         omp.target map_entries(
+!CHECK-SAME:      %{{.*}} -> %[[I_ARG:[^[:space:]]+]],
+!CHECK-SAME:      %{{.*}} -> %[[X_ARG:[^[:space:]]+]] : {{.*}}) {
+
+!CHECK:           %[[I_DECL:.*]]:2 = hlfir.declare %[[I_ARG]]
+!CHECK:           %[[X_DECL:.*]]:2 = hlfir.declare %[[X_ARG]]
+
+!CHECK:           omp.teams {
+
+!CHECK:             %[[LB:.*]] = arith.constant 0 : i32
+!CHECK:             %[[UB:.*]] = arith.constant 10 : i32
+!CHECK:             %[[STEP:.*]] = arith.constant 1 : i32
+
+!CHECK:             omp.parallel private(@{{.*}} %[[I_DECL]]#0 
+!CHECK-SAME:          -> %[[I_PRIV_ARG:[^[:space:]]+]] : !fir.ref<i32>) {
+!CHECK:               omp.distribute {
+!CHECK:                 omp.wsloop {
+
+!CHECK:                   omp.loop_nest (%{{.*}}) : i32 = 
+!CHECK-SAME:                (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+!CHECK:                     %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ARG]]
+!CHECK:                     fir.store %{{.*}} to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
+!CHECK:                   }
+!CHECK:                 }
+!CHECK:               }
+!CHECK:             }
+!CHECK:           }
+!CHECK:         }
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index 5e76e8ff1663b..99323e69113bc 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -82,10 +82,10 @@
 ! CHECK:                 %[[VAL_15:.*]] = fir.load %[[PRIV_J_DECL]]#0 : !fir.ref<i32>
 ! CHECK:                 %[[VAL_16:.*]] = arith.addi %[[LOAD]], %[[VAL_15]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[PRIV_X_DECL]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] : index
+! CHECK:                 %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] overflow<nsw> : index
 ! CHECK:                 %[[STEPCAST:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
 ! CHECK:                 %[[IVLOAD:.*]] = fir.load %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
-! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]]
+! CHECK:                 %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> :
 ! CHECK:                 fir.result %[[VAL_17]], %[[IVINC]] : index, i32
 ! CHECK:               }
 ! CHECK:               fir.store %[[VAL_12]]#1 to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/private-derived-type.f90 b/flang/test/Lower/OpenMP/private-derived-type.f90
index af9a5b72e7175..9d680cd5d6114 100644
--- a/flang/test/Lower/OpenMP/private-derived-type.f90
+++ b/flang/test/Lower/OpenMP/private-derived-type.f90
@@ -28,14 +28,14 @@ end subroutine s4
 ! CHECK:           %[[VAL_15:.*]] = fir.call @_FortranAInitialize(%[[VAL_13]], %[[VAL_14]], %[[VAL_12]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_23:.*]] = fir.alloca !fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}> {bindc_name = "v", pinned, uniq_name = "_QFs4Ev"}
-! CHECK:             %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFs4Ev"} : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>, !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>)
-! CHECK:             %[[VAL_25:.*]] = fir.embox %[[VAL_24]]#1 : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:             %[[VAL_25:.*]] = fir.embox %[[VAL_23]] : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
 ! CHECK:             %[[VAL_26:.*]] = fir.address_of
-! CHECK:             %[[VAL_27:.*]] = arith.constant 4 : i32
+! CHECK:             %[[VAL_27:.*]] = arith.constant 8 : i32
 ! CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_25]] : (!fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
 ! CHECK:             %[[VAL_29:.*]] = fir.convert %[[VAL_26]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
 !                    Check we do call FortranAInitialize on the derived type
 ! CHECK:             %[[VAL_30:.*]] = fir.call @_FortranAInitialize(%[[VAL_28]], %[[VAL_29]], %[[VAL_27]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
+! CHECK:             %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFs4Ev"} : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>, !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>)
 ! CHECK:             omp.wsloop {
 ! CHECK:           }
 ! CHECK:           %[[VAL_39:.*]] = fir.embox %[[VAL_9]]#1 : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
diff --git a/flang/test/Lower/OpenMP/Todo/task_detach.f90 b/flang/test/Lower/OpenMP/task_detach.f90
similarity index 57%
rename from flang/test/Lower/OpenMP/Todo/task_detach.f90
rename to flang/test/Lower/OpenMP/task_detach.f90
index cb6943073d4b3..f943abe6c591a 100644
--- a/flang/test/Lower/OpenMP/Todo/task_detach.f90
+++ b/flang/test/Lower/OpenMP/task_detach.f90
@@ -1,12 +1,14 @@
 ! REQUIRES: openmp_runtime
-! RUN: %not_todo_cmd bbc -emit-fir %openmp_flags -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir %openmp_flags -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags -fopenmp -fopenmp-version=50 -o - %s | FileCheck %s
 
 !===============================================================================
 ! `detach` clause
 !===============================================================================
 
-! CHECK: not yet implemented: DETACH clause is not implemented yet
+!CHECK: omp.task detach(%[[EVENT_HANDLE:.*]] : !fir.ref<i64>) {
+!CHECK: fir.call @_QPfoo() fastmath<contract> : () -> ()
+!CHECK: omp.terminator
+!CHECK: }
 subroutine omp_task_detach()
   use omp_lib
   integer (kind=omp_event_handle_kind) :: event
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index a49eba69ff38c..ce45d09d77a22 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -206,10 +206,10 @@ program reduce15
 ! CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (i32) -> i64
 ! CHECK:             %[[VAL_49:.*]] = hlfir.designate %[[VAL_46]] (%[[VAL_48]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, i64) -> !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_45]] to %[[VAL_49]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] : index
+! CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] overflow<nsw> : index
 ! CHECK:             %[[VAL_51:.*]] = fir.convert %[[VAL_40]] : (index) -> i32
 ! CHECK:             %[[VAL_52:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<i32>
-! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] : i32
+! CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_51]] overflow<nsw> : i32
 ! CHECK:             fir.result %[[VAL_50]], %[[VAL_53]] : index, i32
 ! CHECK:           }
 ! CHECK:           fir.store %[[VAL_54:.*]]#1 to %[[VAL_3]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index 8d235c10fa1d6..cc77ce754d97e 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -150,10 +150,10 @@ subroutine wsloop_variable_sub
 !CHECK:                 %[[VAL_42:.*]] = arith.addi %[[VAL_40]], %[[VAL_41]] : i64
 !CHECK:                 %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (i64) -> f32
 !CHECK:                 hlfir.assign %[[VAL_43]] to %[[VAL_21]]#0 : f32, !fir.ref<f32>
-!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] : index
+!CHECK:                 %[[VAL_44:.*]] = arith.addi %[[VAL_37]], %[[VAL_34]] overflow<nsw> : index
 !CHECK:                 %[[VAL_45:.*]] = fir.convert %[[VAL_34]] : (index) -> i64
 !CHECK:                 %[[VAL_46:.*]] = fir.load %[[VAL_17]]#1 : !fir.ref<i64>
-!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] : i64
+!CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_45]] overflow<nsw> : i64
 !CHECK:                 fir.result %[[VAL_44]], %[[VAL_47]] : index, i64
 !CHECK:               }
 !CHECK:               fir.store %[[VAL_48:.*]]#1 to %[[VAL_17]]#1 : !fir.ref<i64>
diff --git a/flang/test/Lower/array-character.f90 b/flang/test/Lower/array-character.f90
index c93ef4be30823..53adc5c02958c 100644
--- a/flang/test/Lower/array-character.f90
+++ b/flang/test/Lower/array-character.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | fir-opt --canonicalize --cse | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | fir-opt --canonicalize --cse | FileCheck %s
 
 ! CHECK-LABEL: func @_QPissue(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}, %[[VAL_1:.*]]: !fir.boxchar<1>{{.*}}) {
diff --git a/flang/test/Lower/array-derived-assignments.f90 b/flang/test/Lower/array-derived-assignments.f90
index 71e61f651302a..f4e51271d5936 100644
--- a/flang/test/Lower/array-derived-assignments.f90
+++ b/flang/test/Lower/array-derived-assignments.f90
@@ -1,5 +1,5 @@
 ! Test derived type assignment lowering inside array expression
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module array_derived_assign
   type simple_copy
diff --git a/flang/test/Lower/array-derived.f90 b/flang/test/Lower/array-derived.f90
index b5eb7621c90f1..a0c55f5d88255 100644
--- a/flang/test/Lower/array-derived.f90
+++ b/flang/test/Lower/array-derived.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module cs
   type r
diff --git a/flang/test/Lower/array-elemental-calls-char-byval.f90 b/flang/test/Lower/array-elemental-calls-char-byval.f90
index c321614e7fc5b..682191fc78956 100644
--- a/flang/test/Lower/array-elemental-calls-char-byval.f90
+++ b/flang/test/Lower/array-elemental-calls-char-byval.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! with the VALUE attribute.
-! RUN: bbc -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
 
 
 module char_elem_byval
diff --git a/flang/test/Lower/array-elemental-calls-char.f90 b/flang/test/Lower/array-elemental-calls-char.f90
index 603cc677805fc..00e2f8e8f9c12 100644
--- a/flang/test/Lower/array-elemental-calls-char.f90
+++ b/flang/test/Lower/array-elemental-calls-char.f90
@@ -1,6 +1,6 @@
 ! Test lowering of elemental calls with character argument
 ! without the VALUE attribute.
-! RUN: bbc -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - %s | FileCheck %s
 
 module char_elem
 
diff --git a/flang/test/Lower/array-expression-assumed-size.f90 b/flang/test/Lower/array-expression-assumed-size.f90
index ae35da951538b..2fbf315aff114 100644
--- a/flang/test/Lower/array-expression-assumed-size.f90
+++ b/flang/test/Lower/array-expression-assumed-size.f90
@@ -1,5 +1,5 @@
 ! RUN: bbc --emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false %s -o - | FileCheck --check-prefix=PostOpt %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck --check-prefix=PostOpt %s
 
 
 subroutine assumed_size_test(a)
diff --git a/flang/test/Lower/array-expression-slice-1.f90 b/flang/test/Lower/array-expression-slice-1.f90
index 1524509024329..b597814bc0d9f 100644
--- a/flang/test/Lower/array-expression-slice-1.f90
+++ b/flang/test/Lower/array-expression-slice-1.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false -o - --outline-intrinsics %s | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv -o - --outline-intrinsics %s | FileCheck %s
 
 ! CHECK-LABEL: func @_QQmain() attributes {fir.bindc_name = "p"} {
 ! CHECK-DAG:         %[[VAL_0:.*]] = arith.constant 10 : index
diff --git a/flang/test/Lower/array-substring.f90 b/flang/test/Lower/array-substring.f90
index 2e283997e3e00..02101039120e9 100644
--- a/flang/test/Lower/array-substring.f90
+++ b/flang/test/Lower/array-substring.f90
@@ -1,5 +1,7 @@
 ! RUN: bbc -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
+
+! NO-NSW-NOT: overflow<nsw>
 
 ! CHECK-LABEL: func @_QPtest(
 ! CHECK-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
@@ -32,8 +34,9 @@
 ! CHECK:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
 ! CHECK:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
+! CHECK:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
 ! CHECK:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
-! CHECK:         br ^bb1(%[[VAL_15]], %[[VAL_28]] : index, index)
+! CHECK:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
 ! CHECK:       ^bb3:
 ! CHECK:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
 ! CHECK:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
@@ -46,42 +49,3 @@ function test(C)
 
   test = C(1:1)(1:8) == (/'ABCDabcd'/) 
 end function test
-
-! NSW-LABEL: func @_QPtest(
-! NSW-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
-! NSW-DAG:         %[[VAL_1:.*]] = arith.constant 1 : index
-! NSW-DAG:         %[[VAL_2:.*]] = arith.constant 0 : index
-! NSW-DAG:         %[[VAL_3:.*]] = arith.constant 0 : i32
-! NSW-DAG:         %[[VAL_4:.*]] = arith.constant 8 : index
-! NSW:         %[[VAL_6:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! NSW:         %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<1x!fir.char<1,12>>>
-! NSW:         %[[VAL_8:.*]] = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFtestEtest"}
-! NSW:         %[[VAL_9:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! NSW:         %[[VAL_10:.*]] = fir.slice %[[VAL_1]], %[[VAL_1]], %[[VAL_1]] : (index, index, index) -> !fir.slice<1>
-! NSW:         %[[VAL_11:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.array<1x!fir.char<1,8>>>
-! NSW:         br ^bb1(%[[VAL_2]], %[[VAL_1]] : index, index)
-! NSW:       ^bb1(%[[VAL_12:.*]]: index, %[[VAL_13:.*]]: index):
-! NSW:         %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %[[VAL_2]] : index
-! NSW:         cond_br %[[VAL_14]], ^bb2, ^bb3
-! NSW:       ^bb2:
-! NSW:         %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : index
-! NSW:         %[[VAL_16:.*]] = fir.array_coor %[[VAL_7]](%[[VAL_9]]) {{\[}}%[[VAL_10]]] %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,12>>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<!fir.char<1,12>>
-! NSW:         %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<!fir.array<12x!fir.char<1>>>
-! NSW:         %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_17]], %[[VAL_2]] : (!fir.ref<!fir.array<12x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! NSW:         %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-! NSW:         %[[VAL_20:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,8>>>, !fir.shape<1>, index) -> !fir.ref<!fir.char<1,8>>
-! NSW:         %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! NSW:         %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
-! NSW:         %[[VAL_23:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-! NSW:         %[[VAL_24:.*]] = fir.call @_FortranACharacterCompareScalar1(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_23]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i64) -> i32
-! NSW:         %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_24]], %[[VAL_3]] : i32
-! NSW:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
-! NSW:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
-! NSW:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
-! NSW:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
-! NSW:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
-! NSW:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
-! NSW:       ^bb3:
-! NSW:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
-! NSW:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
-! NSW:       }
diff --git a/flang/test/Lower/array-temp.f90 b/flang/test/Lower/array-temp.f90
index 10c5ee91d44bd..718aef84a4e85 100644
--- a/flang/test/Lower/array-temp.f90
+++ b/flang/test/Lower/array-temp.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPss1()
 subroutine ss1
diff --git a/flang/test/Lower/components.f90 b/flang/test/Lower/components.f90
index e1582a8a31e0d..28e836c5d1045 100644
--- a/flang/test/Lower/components.f90
+++ b/flang/test/Lower/components.f90
@@ -1,4 +1,4 @@
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 module components_test
   type t1
diff --git a/flang/test/Lower/do_concurrent_local_default_init.f90 b/flang/test/Lower/do_concurrent_local_default_init.f90
new file mode 100644
index 0000000000000..1766e0a104ff6
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_local_default_init.f90
@@ -0,0 +1,52 @@
+! Test default initialization of DO CONCURRENT LOCAL() entities.
+! RUN: bbc -emit-hlfir -I nowhere -o - %s | FileCheck %s
+
+subroutine test_ptr(p)
+  interface
+    pure subroutine takes_ptr(p)
+      character(*), intent(in), pointer :: p(:)
+    end subroutine
+  end interface
+  character(*), pointer :: p(:)
+  integer :: i
+  do concurrent (i=1:10) local(p)
+    call takes_ptr(p)
+  end do
+end subroutine
+
+subroutine test_default_init()
+  type t
+    integer :: i = 2
+  end type
+  integer :: i, res(4)
+  type(t) :: a
+  do concurrent (i=1:4) local(a)
+    res(i) = a%i
+  end do
+  call something(res)
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_ptr(
+! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>> {fir.bindc_name = "p"}) {
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
+! CHECK:           %[[VAL_7:.*]] = fir.box_elesize %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>) -> index
+! CHECK:           fir.do_loop
+! CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "p", pinned, uniq_name = "_QFtest_ptrEp"}
+! CHECK:             %[[VAL_17:.*]] = fir.zero_bits !fir.ptr<!fir.array<?x!fir.char<1,?>>>
+! CHECK:             %[[VAL_18:.*]] = arith.constant 0 : index
+! CHECK:             %[[VAL_19:.*]] = fir.shape %[[VAL_18]] : (index) -> !fir.shape<1>
+! CHECK:             %[[VAL_20:.*]] = fir.embox %[[VAL_17]](%[[VAL_19]]) typeparams %[[VAL_7]] : (!fir.ptr<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>
+! CHECK:             fir.store %[[VAL_20]] to %[[VAL_16]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
+! CHECK:             %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_16]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptrEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>)
+! CHECK:             fir.call @_QPtakes_ptr(%[[VAL_21]]#0) proc_attrs<pure> fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> ()
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPtest_default_init(
+! CHECK:           fir.do_loop
+! CHECK:             %[[VAL_26:.*]] = fir.alloca !fir.type<_QFtest_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFtest_default_initEa"}
+! CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_26]] : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>
+! CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<none>
+! CHECK:             %[[VAL_32:.*]] = fir.call @_FortranAInitialize(%[[VAL_30]], {{.*}}
+! CHECK:             %[[VAL_33:.*]]:2 = hlfir.declare %[[VAL_26]] {uniq_name = "_QFtest_default_initEa"} : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>, !fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>)
+! CHECK:           }
diff --git a/flang/test/Lower/do_loop.f90 b/flang/test/Lower/do_loop.f90
index a46e6c947391b..5d8343b8d68a4 100644
--- a/flang/test/Lower/do_loop.f90
+++ b/flang/test/Lower/do_loop.f90
@@ -1,17 +1,17 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Simple tests for structured ordered loops with loop-control.
 ! Tests the structure of the loop, storage to index variable and return and 
 ! storage of the final value of the index variable.
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! Test a simple loop with the final value of the index variable read outside the loop
 ! CHECK-LABEL: simple_loop
-! NSW-LABEL:   simple_loop
 subroutine simple_loop
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
   integer :: i
 
   ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
@@ -21,18 +21,14 @@ subroutine simple_loop
   ! CHECK: %[[C1:.*]] = arith.constant 1 : index
   ! CHECK: %[[LB:.*]] = fir.convert %[[C1_CVT]] : (index) -> i32
   ! CHECK: %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C1_CVT]] to %[[C5_CVT]] step %[[C1]]
   ! CHECK-SAME: iter_args(%[[IV:.*]] = %[[LB]]) -> (index, i32) {
   do i=1,5
   ! CHECK:   fir.store %[[IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] : index
-  ! NSW:     %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1:.*]] overflow<nsw> : index
+  ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] overflow<nsw> : index
   ! CHECK:   %[[STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
   ! CHECK:   %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:     %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] : i32
-  ! NSW:     %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -44,14 +40,11 @@ subroutine simple_loop
 
 ! Test a 2-nested loop with a body composed of a reduction. Values are read from a 2d array.
 ! CHECK-LABEL: nested_loop
-! NSW-LABEL:   nested_loop
 subroutine nested_loop
   ! CHECK: %[[ARR_REF:.*]] = fir.alloca !fir.array<5x5xi32> {bindc_name = "arr", uniq_name = "_QFnested_loopEarr"}
   ! CHECK: %[[ASUM_REF:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFnested_loopEasum"}
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
   ! CHECK: %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
-  ! NSW:   %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
   integer :: asum, arr(5,5)
   integer :: i, j
   asum = 0
@@ -62,7 +55,6 @@ subroutine nested_loop
   ! CHECK: %[[ST_I:.*]] = arith.constant 1 : index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_I_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_I_CVT]] to %[[E_I_CVT]] step %[[ST_I]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=1,5
@@ -74,7 +66,6 @@ subroutine nested_loop
     ! CHECK: %[[ST_J:.*]] = arith.constant 1 : index
     ! CHECK: %[[J_LB:.*]] = fir.convert %[[S_J_CVT]] : (index) -> i32
     ! CHECK: %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
-    ! NSW:   %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
     ! CHECK-SAME: %[[S_J_CVT]] to %[[E_J_CVT]] step %[[ST_J]]
     ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
     do j=1,5
@@ -93,24 +84,18 @@ subroutine nested_loop
       ! CHECK: %[[ASUM_NEW:.*]] = arith.addi %[[ASUM]], %[[ARR_VAL]] : i32
       ! CHECK: fir.store %[[ASUM_NEW]] to %[[ASUM_REF]] : !fir.ref<i32>
       asum = asum + arr(i,j)
-      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] : index
-      ! NSW:   %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J:.*]] overflow<nsw> : index
+      ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] overflow<nsw> : index
       ! CHECK: %[[J_STEPCAST:.*]] = fir.convert %[[ST_J]] : (index) -> i32
       ! CHECK: %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-      ! NSW:   %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
-      ! NSW:   %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST:.*]] overflow<nsw> : i32
+      ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
       ! CHECK: fir.result %[[LJ_NEXT]], %[[J_IVINC]] : index, i32
     ! CHECK: }
     end do
     ! CHECK: fir.store %[[J_RES]]#1 to %[[J_REF]] : !fir.ref<i32>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] : index
-    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I:.*]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_I]] : (index) -> i32
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -119,11 +104,9 @@ subroutine nested_loop
 
 ! Test a downcounting loop
 ! CHECK-LABEL: down_counting_loop
-! NSW-LABEL:   down_counting_loop
 subroutine down_counting_loop()
   integer :: i
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
 
   ! CHECK: %[[C5:.*]] = arith.constant 5 : i32
   ! CHECK: %[[C5_CVT:.*]] = fir.convert %[[C5]] : (i32) -> index
@@ -133,18 +116,14 @@ subroutine down_counting_loop()
   ! CHECK: %[[CMINUS1_STEP_CVT:.*]] = fir.convert %[[CMINUS1]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[C5_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C5_CVT]] to %[[C1_CVT]] step %[[CMINUS1_STEP_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=5,1,-1
   ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] : index
-  ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT:.*]] overflow<nsw> : index
+  ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[CMINUS1_STEP_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
   ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -153,7 +132,6 @@ subroutine down_counting_loop()
 
 ! Test a general loop with a variable step
 ! CHECK-LABEL: loop_with_variable_step
-! NSW-LABEL:   loop_with_variable_step
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st"}) {
 subroutine loop_with_variable_step(s,e,st)
   integer :: s, e, st
@@ -166,18 +144,14 @@ subroutine loop_with_variable_step(s,e,st)
   ! CHECK: %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=s,e,st
   ! CHECK:  fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
-  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-  ! NSW:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+  ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+  ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -186,13 +160,11 @@ subroutine loop_with_variable_step(s,e,st)
 
 ! Test usage of pointer variables as index, start, end and step variables
 ! CHECK-LABEL: loop_with_pointer_variables
-! NSW-LABEL:   loop_with_pointer_variables
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s", fir.target}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e", fir.target}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st", fir.target}) {
 subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[E_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEeptr.addr"}
 ! CHECK:  %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", fir.target, uniq_name = "_QFloop_with_pointer_variablesEi"}
 ! CHECK:  %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
-! NSW:    %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
 ! CHECK:  %[[S_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEsptr.addr"}
 ! CHECK:  %[[ST_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEstptr.addr"}
   integer, target :: i
@@ -213,7 +185,6 @@ subroutine loop_with_pointer_variables(s,e,st)
   stptr => st
 
 ! CHECK:  %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
-! NSW:    %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S_PTR:.*]] = fir.load %[[S_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S:.*]] = fir.load %[[S_PTR]] : !fir.ptr<i32>
 ! CHECK:  %[[S_CVT:.*]] = fir.convert %[[S]] : (i32) -> index
@@ -225,18 +196,14 @@ subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
 ! CHECK:  %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
 ! CHECK:  %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-! NSW:    %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
 ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
 ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do iptr=sptr,eptr,stptr
 ! CHECK:    fir.store %[[I_IV]] to %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-! NSW:      %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
 ! CHECK:    %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
 ! CHECK:    %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
-! NSW:      %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
-! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
-! NSW:      %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
+! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   end do
 ! CHECK:  }
@@ -245,11 +212,9 @@ subroutine loop_with_pointer_variables(s,e,st)
 
 ! Test usage of non-default integer kind for loop control and loop index variable
 ! CHECK-LABEL: loop_with_non_default_integer
-! NSW-LABEL:   loop_with_non_default_integer
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "st"}) {
 subroutine loop_with_non_default_integer(s,e,st)
   ! CHECK: %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
-  ! NSW:   %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
   integer(kind=8):: i
   ! CHECK: %[[S:.*]] = fir.load %[[S_REF]] : !fir.ref<i64>
   ! CHECK: %[[S_CVT:.*]] = fir.convert %[[S]] : (i64) -> index
@@ -261,18 +226,14 @@ subroutine loop_with_non_default_integer(s,e,st)
 
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i64
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
-  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i64) {
   do i=s,e,st
     ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
-    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
+    ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i64
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
-    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
-    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i64
-    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i64
+    ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i64
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i64
   end do
   ! CHECK: }
diff --git a/flang/test/Lower/do_loop_unstructured.f90 b/flang/test/Lower/do_loop_unstructured.f90
index e1a669e09c9a8..d8890b2d0926e 100644
--- a/flang/test/Lower/do_loop_unstructured.f90
+++ b/flang/test/Lower/do_loop_unstructured.f90
@@ -1,9 +1,11 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Tests for unstructured loops.
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! Test a simple unstructured loop. Test for the existence of,
 ! -> The initialization of the trip-count and loop-variable
 ! -> The branch to the body or the exit inside the header
@@ -39,42 +41,12 @@ subroutine simple_unstructured()
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
-! NSW-LABEL: simple_unstructured
-! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructuredEi"}
-! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
-! NSW:   %[[STEP_ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP_ONE]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP_ONE]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
-
 ! Test an unstructured loop with a step. Mostly similar to the previous one.
 ! Only difference is a non-unit step.
 subroutine simple_unstructured_with_step()
@@ -108,42 +80,12 @@ subroutine simple_unstructured_with_step()
 ! CHECK:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   %[[STEP_2:.*]] = arith.constant 2 : i32
-! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
-! NSW-LABEL: simple_unstructured_with_step
-! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructured_with_stepEi"}
-! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
-! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
-! NSW:   %[[STEP:.*]] = arith.constant 2 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   %[[STEP_2:.*]] = arith.constant 2 : i32
-! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
-
 ! Test a three nested unstructured loop. Three nesting is the basic case where
 ! we have loops that are neither innermost or outermost.
 subroutine nested_unstructured()
@@ -215,7 +157,7 @@ subroutine nested_unstructured()
 ! CHECK:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_K]]
 ! CHECK: ^[[EXIT_K]]:
@@ -225,7 +167,7 @@ subroutine nested_unstructured()
 ! CHECK:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_J]]
 ! CHECK: ^[[EXIT_J]]:
@@ -235,96 +177,12 @@ subroutine nested_unstructured()
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER_I]]
 ! CHECK: ^[[EXIT_I]]:
 ! CHECK:   return
 
-! NSW-LABEL: nested_unstructured
-! NSW:   %[[TRIP_VAR_K_REF:.*]] = fir.alloca i32
-! NSW:   %[[TRIP_VAR_J_REF:.*]] = fir.alloca i32
-! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_unstructuredEi"}
-! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_unstructuredEj"}
-! NSW:   %[[LOOP_VAR_K_REF:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFnested_unstructuredEk"}
-! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
-! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_I:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_I]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_I:.*]]
-! NSW: ^[[HEADER_I]]:
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_1:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_I:.*]] = arith.cmpi sgt, %[[TRIP_VAR_I]], %[[ZERO_1]] : i32
-! NSW:   cf.cond_br %[[COND_I]], ^[[BODY_I:.*]], ^[[EXIT_I:.*]]
-! NSW: ^[[BODY_I]]:
-! NSW:   %[[J_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[J_END:.*]] = arith.constant 200 : i32
-! NSW:   %[[J_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP3:.*]] = arith.subi %[[J_END]], %[[J_START]] : i32
-! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[J_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_J:.*]] = arith.divsi %[[TMP4]], %[[J_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_J]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[J_START]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_J:.*]]
-! NSW: ^[[HEADER_J]]:
-! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_J:.*]] = arith.cmpi sgt, %[[TRIP_VAR_J]], %[[ZERO_2]] : i32
-! NSW:   cf.cond_br %[[COND_J]], ^[[BODY_J:.*]], ^[[EXIT_J:.*]]
-! NSW: ^[[BODY_J]]:
-! NSW:   %[[K_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[K_END:.*]] = arith.constant 300 : i32
-! NSW:   %[[K_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP3:.*]] = arith.subi %[[K_END]], %[[K_START]] : i32
-! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[K_STEP]] : i32
-! NSW:   %[[TRIP_COUNT_K:.*]] = arith.divsi %[[TMP4]], %[[K_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT_K]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[K_START]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_K:.*]]
-! NSW: ^[[HEADER_K]]:
-! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND_K:.*]] = arith.cmpi sgt, %[[TRIP_VAR_K]], %[[ZERO_2]] : i32
-! NSW:   cf.cond_br %[[COND_K]], ^[[BODY_K:.*]], ^[[EXIT_K:.*]]
-! NSW: ^[[BODY_K]]:
-! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_K_NEXT:.*]] = arith.subi %[[TRIP_VAR_K]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_K]]
-! NSW: ^[[EXIT_K]]:
-! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_J_NEXT:.*]] = arith.subi %[[TRIP_VAR_J]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_J]]
-! NSW: ^[[EXIT_J]]:
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[ONE_1]] : i32
-! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER_I]]
-! NSW: ^[[EXIT_I]]:
-! NSW:   return
-
 ! Test the existence of a structured loop inside an unstructured loop.
 ! Only minimal checks are inserted for the structured loop.
 subroutine nested_structured_in_unstructured()
@@ -359,9 +217,9 @@ subroutine nested_structured_in_unstructured()
 ! CHECK-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
 ! CHECK:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] : index
+! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
 ! CHECK:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} : i32
+! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
 ! CHECK:   }
 ! CHECK:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[C1_3:.*]] = arith.constant 1 : i32
@@ -369,47 +227,8 @@ subroutine nested_structured_in_unstructured()
 ! CHECK:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] : i32
+! CHECK:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
 ! CHECK:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
-
-! NSW-LABEL: nested_structured_in_unstructured
-! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
-! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_structured_in_unstructuredEi"}
-! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_structured_in_unstructuredEj"}
-! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
-! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
-! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
-! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
-! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
-! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
-! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER:.*]]
-! NSW: ^[[HEADER]]:
-! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
-! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
-! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
-! NSW: ^[[BODY]]:
-! NSW:   %{{.*}} = fir.do_loop %[[J_INDEX:[^ ]*]] =
-! NSW-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
-! NSW-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
-! NSW:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
-! NSW:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
-! NSW:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
-! NSW:   }
-! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[C1_3:.*]] = arith.constant 1 : i32
-! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[C1_3]] : i32
-! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
-! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
-! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
-! NSW:   cf.br ^[[HEADER]]
-! NSW: ^[[EXIT]]:
-! NSW:   return
diff --git a/flang/test/Lower/explicit-interface-results-2.f90 b/flang/test/Lower/explicit-interface-results-2.f90
index 95aee84f4a644..2336053c32a54 100644
--- a/flang/test/Lower/explicit-interface-results-2.f90
+++ b/flang/test/Lower/explicit-interface-results-2.f90
@@ -252,12 +252,10 @@ subroutine test_call_to_used_interface(dummy_proc)
   call takes_array(dummy_proc())
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_2:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = ".result"}
-! CHECK:  %[[VAL_3:.*]] = llvm.intr.stacksave : !llvm.ptr
 ! CHECK:  %[[VAL_4:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
 ! CHECK:  %[[VAL_5:.*]] = fir.box_addr %[[VAL_0]] : (!fir.boxproc<() -> ()>) -> (() -> !fir.array<100xf32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.call %[[VAL_5]]() {{.*}}: () -> !fir.array<100xf32>
 ! CHECK:  fir.save_result %[[VAL_6]] to %[[VAL_2]](%[[VAL_4]]) : !fir.array<100xf32>, !fir.ref<!fir.array<100xf32>>, !fir.shape<1>
 ! CHECK:  %[[VAL_7:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<?xf32>>
 ! CHECK:  fir.call @_QPtakes_array(%[[VAL_7]]) {{.*}}: (!fir.ref<!fir.array<?xf32>>) -> ()
-! CHECK:  llvm.intr.stackrestore %[[VAL_3]] : !llvm.ptr
 end subroutine
diff --git a/flang/test/Lower/explicit-interface-results.f90 b/flang/test/Lower/explicit-interface-results.f90
index 623e875b5f9c9..612d57be36448 100644
--- a/flang/test/Lower/explicit-interface-results.f90
+++ b/flang/test/Lower/explicit-interface-results.f90
@@ -195,8 +195,8 @@ subroutine dyn_array(m, n)
   ! CHECK-DAG: %[[ncast2:.*]] = fir.convert %[[nadd]] : (i64) -> index
   ! CHECK-DAG: %[[ncmpi:.*]] = arith.cmpi sgt, %[[ncast2]], %{{.*}} : index
   ! CHECK-DAG: %[[nselect:.*]] = arith.select %[[ncmpi]], %[[ncast2]], %{{.*}} : index
-  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?xf32>, %[[mselect]], %[[nselect]]
   ! CHECK: %[[shape:.*]] = fir.shape %[[mselect]], %[[nselect]] : (index, index) -> !fir.shape<2>
+  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?xf32>, %[[mselect]], %[[nselect]]
   ! CHECK: %[[res:.*]] = fir.call @_QMcalleePreturn_dyn_array(%[[m]], %[[n]]) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> !fir.array<?x?xf32>
   ! CHECK: fir.save_result %[[res]] to %[[tmp]](%[[shape]]) : !fir.array<?x?xf32>, !fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>
   print *, return_dyn_array(m, n)
@@ -211,8 +211,8 @@ subroutine dyn_char_cst_array(l)
   ! CHECK: %[[lcast2:.*]] = fir.convert %[[lcast]] : (i64) -> index
   ! CHECK: %[[cmpi:.*]] = arith.cmpi sgt, %[[lcast2]], %{{.*}} : index
   ! CHECK: %[[select:.*]] = arith.select %[[cmpi]], %[[lcast2]], %{{.*}} : index
-  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<20x30x!fir.char<1,?>>(%[[select]] : index)
   ! CHECK: %[[shape:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<20x30x!fir.char<1,?>>(%[[select]] : index)
   ! CHECK: %[[res:.*]] = fir.call @_QMcalleePreturn_dyn_char_cst_array(%[[l]]) {{.*}}: (!fir.ref<i32>) -> !fir.array<20x30x!fir.char<1,?>>
   ! CHECK: fir.save_result %[[res]] to %[[tmp]](%[[shape]]) typeparams %[[select]] : !fir.array<20x30x!fir.char<1,?>>, !fir.ref<!fir.array<20x30x!fir.char<1,?>>>, !fir.shape<2>, index
   print *, return_dyn_char_cst_array(l)
@@ -236,8 +236,8 @@ subroutine cst_char_dyn_array(m, n)
   ! CHECK-DAG: %[[ncast2:.*]] = fir.convert %[[nadd]] : (i64) -> index
   ! CHECK-DAG: %[[ncmpi:.*]] = arith.cmpi sgt, %[[ncast2]], %{{.*}} : index
   ! CHECK-DAG: %[[nselect:.*]] = arith.select %[[ncmpi]], %[[ncast2]], %{{.*}} : index
-  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,10>>, %[[mselect]], %[[nselect]]
   ! CHECK: %[[shape:.*]] = fir.shape %[[mselect]], %[[nselect]] : (index, index) -> !fir.shape<2>
+  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,10>>, %[[mselect]], %[[nselect]]
   ! CHECK: %[[res:.*]] = fir.call @_QMcalleePreturn_cst_char_dyn_array(%[[m]], %[[n]]) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> !fir.array<?x?x!fir.char<1,10>>
   ! CHECK: fir.save_result %[[res]] to %[[tmp]](%[[shape]]) typeparams {{.*}} : !fir.array<?x?x!fir.char<1,10>>, !fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index
   print *, return_cst_char_dyn_array(m, n)
@@ -267,8 +267,8 @@ subroutine dyn_char_dyn_array(l, m, n)
   ! CHECK-DAG: %[[lcast2:.*]] = fir.convert %[[lcast]] : (i64) -> index
   ! CHECK-DAG: %[[lcmpi:.*]] = arith.cmpi sgt, %[[lcast2]], %{{.*}} : index
   ! CHECK-DAG: %[[lselect:.*]] = arith.select %[[lcmpi]], %[[lcast2]], %{{.*}} : index
-  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,?>>(%[[lselect]] : index), %[[mselect]], %[[nselect]]
   ! CHECK: %[[shape:.*]] = fir.shape %[[mselect]], %[[nselect]] : (index, index) -> !fir.shape<2>
+  ! CHECK: %[[tmp:.*]] = fir.alloca !fir.array<?x?x!fir.char<1,?>>(%[[lselect]] : index), %[[mselect]], %[[nselect]]
   ! CHECK: %[[res:.*]] = fir.call @_QMcalleePreturn_dyn_char_dyn_array(%[[l]], %[[m]], %[[n]]) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>) -> !fir.array<?x?x!fir.char<1,?>>
   ! CHECK: fir.save_result %[[res]] to %[[tmp]](%[[shape]]) typeparams {{.*}} : !fir.array<?x?x!fir.char<1,?>>, !fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index
   integer :: l, m, n
diff --git a/flang/test/Lower/forall/array-constructor.f90 b/flang/test/Lower/forall/array-constructor.f90
index 4c8c756ea689c..6b6b46fdd4688 100644
--- a/flang/test/Lower/forall/array-constructor.f90
+++ b/flang/test/Lower/forall/array-constructor.f90
@@ -232,8 +232,8 @@ end subroutine ac2
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[CMPI:.*]] = arith.cmpi sgt, %[[VAL_80]], %[[C0]] : index
 ! CHECK:           %[[SELECT:.*]] = arith.select %[[CMPI]], %[[VAL_80]], %[[C0]] : index
-! CHECK:           %[[VAL_81:.*]] = llvm.intr.stacksave : !llvm.ptr
 ! CHECK:           %[[VAL_82:.*]] = fir.shape %[[SELECT]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_81:.*]] = llvm.intr.stacksave : !llvm.ptr
 ! CHECK:           %[[VAL_83:.*]] = fir.convert %[[VAL_74]] : (!fir.box<!fir.array<1xi32>>) -> !fir.box<!fir.array<?xi32>>
 ! CHECK:           %[[VAL_84:.*]] = fir.call @_QFac2Pfunc(%[[VAL_83]]) {{.*}}: (!fir.box<!fir.array<?xi32>>) -> !fir.array<3xi32>
 ! CHECK:           fir.save_result %[[VAL_84]] to %[[VAL_2]](%[[VAL_82]]) : !fir.array<3xi32>, !fir.ref<!fir.array<3xi32>>, !fir.shape<1>
diff --git a/flang/test/Lower/goto-do-body.f90 b/flang/test/Lower/goto-do-body.f90
index 910e55f1839fd..880417c888104 100644
--- a/flang/test/Lower/goto-do-body.f90
+++ b/flang/test/Lower/goto-do-body.f90
@@ -48,7 +48,7 @@ subroutine sub1()
 ! CHECK:    fir.store %[[TMP8]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP9:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[C1_4:.*]] = arith.constant 1 : i32
-! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] : i32
+! CHECK:    %[[TMP10:.*]] = arith.addi %[[TMP9]], %[[C1_4]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP10]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
@@ -90,9 +90,9 @@ subroutine sub2()
 ! CHECK:    %[[C2_2:.*]] = arith.constant 2 : i32
 ! CHECK:    %[[C3_2:.*]] = arith.constant 3 : i32
 ! CHECK:    %[[TMP2:.*]] = fir.load %[[J]] : !fir.ref<i32>
-! CHECK:    %[[TMP3:.*]] = arith.muli %[[C3_2]], %[[TMP2]] : i32
+! CHECK:    %[[TMP3:.*]] = arith.muli %[[C3_2]], %[[TMP2]] overflow<nsw> : i32
 ! CHECK:    %[[C8_1:.*]] = arith.constant 8 : i32
-! CHECK:    %[[STEP:.*]] = arith.subi %[[TMP3]], %[[C8_1]] : i32
+! CHECK:    %[[STEP:.*]] = arith.subi %[[TMP3]], %[[C8_1]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[STEP]] to %[[STEP_VAR:.*]] : !fir.ref<i32>
 ! CHECK:    %[[TMP4:.*]] = arith.subi %[[C2_2]], %[[C1_1]] : i32
 ! CHECK:    %[[TMP5:.*]] = arith.addi %[[TMP4]], %[[STEP]] : i32
@@ -115,7 +115,7 @@ subroutine sub2()
 ! CHECK:    fir.store %[[TMP10]] to %[[TRIP]] : !fir.ref<i32>
 ! CHECK:    %[[TMP11:.*]] = fir.load %[[I]] : !fir.ref<i32>
 ! CHECK:    %[[STEP_VAL:.*]] = fir.load %[[STEP_VAR]] : !fir.ref<i32>
-! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] : i32
+! CHECK:    %[[TMP12:.*]] = arith.addi %[[TMP11]], %[[STEP_VAL]] overflow<nsw> : i32
 ! CHECK:    fir.store %[[TMP12]] to %[[I]] : !fir.ref<i32>
 ! CHECK:    cf.br ^[[HEADER]]
   end do
diff --git a/flang/test/Lower/host-associated.f90 b/flang/test/Lower/host-associated.f90
index 9b4269df7bfcb..33acdff1bb74c 100644
--- a/flang/test/Lower/host-associated.f90
+++ b/flang/test/Lower/host-associated.f90
@@ -1,5 +1,5 @@
 ! Test internal procedure host association lowering.
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -fwrapv %s -o - | FileCheck %s
 
 ! -----------------------------------------------------------------------------
 !     Test non character intrinsic scalars
diff --git a/flang/test/Lower/infinite_loop.f90 b/flang/test/Lower/infinite_loop.f90
index 6942dda8d7a23..de0bee779c5b6 100644
--- a/flang/test/Lower/infinite_loop.f90
+++ b/flang/test/Lower/infinite_loop.f90
@@ -1,9 +1,11 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fwrapv -o - %s | FileCheck %s --check-prefix=NO-NSW
 
 ! Tests for infinite loop.
 
+! NO-NSW-NOT: overflow<nsw>
+
 subroutine empty_infinite()
   do
   end do
@@ -96,10 +98,10 @@ subroutine structured_loop_in_infinite(i)
 ! CHECK-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
 ! CHECK:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] : index
+! CHECK:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
 ! CHECK:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
 ! CHECK:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
+! CHECK:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
 ! CHECK:  }
 ! CHECK:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
@@ -107,39 +109,6 @@ subroutine structured_loop_in_infinite(i)
 ! CHECK: ^[[RETURN]]:
 ! CHECK:   return
 
-! NSW-LABEL: structured_loop_in_infinite
-! NSW-SAME: %[[I_REF:.*]]: !fir.ref<i32>
-! NSW:  %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFstructured_loop_in_infiniteEj"}
-! NSW:  cf.br ^[[BODY1:.*]]
-! NSW: ^[[BODY1]]:
-! NSW:  %[[I:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-! NSW:  %[[C100:.*]] = arith.constant 100 : i32
-! NSW:  %[[COND:.*]] = arith.cmpi sgt, %[[I]], %[[C100]] : i32
-! NSW:  cf.cond_br %[[COND]], ^[[EXIT:.*]], ^[[BODY2:.*]]
-! NSW: ^[[EXIT]]:
-! NSW:  cf.br ^[[RETURN:.*]]
-! NSW: ^[[BODY2:.*]]:
-! NSW:  %[[C1:.*]] = arith.constant 1 : i32
-! NSW:  %[[C1_INDEX:.*]] = fir.convert %[[C1]] : (i32) -> index
-! NSW:  %[[C10:.*]] = arith.constant 10 : i32
-! NSW:  %[[C10_INDEX:.*]] = fir.convert %[[C10]] : (i32) -> index
-! NSW:  %[[C1_1:.*]] = arith.constant 1 : index
-! NSW:  %[[J_LB:.*]] = fir.convert %[[C1_INDEX]] : (index) -> i32
-! NSW:  %[[J_FINAL:.*]]:2 = fir.do_loop %[[J:[^ ]*]] =
-! NSW-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
-! NSW-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
-! NSW:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
-! NSW:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
-! NSW:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
-! NSW:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
-! NSW:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
-! NSW:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
-! NSW:  }
-! NSW:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
-! NSW:  cf.br ^[[BODY1]]
-! NSW: ^[[RETURN]]:
-! NSW:   return
-
 subroutine empty_infinite_in_while(i)
   integer :: i
   do while (i .gt. 50)
diff --git a/flang/test/Lower/io-implied-do-fixes.f90 b/flang/test/Lower/io-implied-do-fixes.f90
index a6c115fa80ded..cd4fd43e05194 100644
--- a/flang/test/Lower/io-implied-do-fixes.f90
+++ b/flang/test/Lower/io-implied-do-fixes.f90
@@ -1,30 +1,20 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
+! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
 ! UNSUPPORTED: system-windows
 
+! NO-NSW-NOT: overflow<nsw>
+
 ! CHECK-LABEL: func @_QPido1
 ! CHECK: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
 ! CHECK: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
-! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
-
-! NSW-LABEL: func @_QPido1
-! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
-! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
-! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
-! NSW:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
-! NSW:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW:   fir.result %[[J_VAL_NEXT]] : index
-! NSW: }
-! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
 subroutine ido1
   integer, pointer :: iptr
   integer, target :: itgt
@@ -38,23 +28,11 @@ subroutine ido1
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK: fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
-
-! NSW-LABEL: func @_QPido2
-! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido2Eiptr.addr"}
-! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
-! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
-! NSW: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! NSW: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW: fir.result %[[J_VAL_NEXT]] : index
-! NSW: }
-! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
-! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
 subroutine ido2
   integer, allocatable :: iptr
   allocate(iptr)
@@ -69,27 +47,12 @@ subroutine ido2
 ! CHECK:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
 ! CHECK:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
 ! CHECK:    }
-! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
 ! CHECK:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
 ! CHECK:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
 ! CHECK:  }
 ! CHECK:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
 ! CHECK:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
-
-! NSW-LABEL: func @_QPido3
-! NSW:  %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido3Ej.addr"}
-! NSW:  %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
-! NSW:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and (%[[OK:.*]] = {{.*}}) -> (index, i1) {
-! NSW:    %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
-! NSW:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
-! NSW:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
-! NSW:    }
-! NSW:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
-! NSW:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
-! NSW:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
-! NSW:  }
-! NSW:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
-! NSW:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
 subroutine ido3
   integer, allocatable :: j
   allocate(j)
diff --git a/flang/test/Lower/loops2.f90 b/flang/test/Lower/loops2.f90
index 0a587234a991b..60a6bf6c53119 100644
--- a/flang/test/Lower/loops2.f90
+++ b/flang/test/Lower/loops2.f90
@@ -107,7 +107,7 @@ subroutine test_pointer_unstructured_loop()
 ! CHECK:       ^bb4:
 ! CHECK:         %[[VAL_20:.*]] = fir.load %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         %[[VAL_21:.*]] = arith.constant 1 : i32
-! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] : i32
+! CHECK:         %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_21]] overflow<nsw> : i32
 ! CHECK:         fir.store %[[VAL_22]] to %[[VAL_3]] : !fir.ptr<i32>
 ! CHECK:         br ^bb1
 ! CHECK:       ^bb5:
diff --git a/flang/test/Lower/mixed_loops.f90 b/flang/test/Lower/mixed_loops.f90
index 1aa0225129bed..991fd7aa82bb9 100644
--- a/flang/test/Lower/mixed_loops.f90
+++ b/flang/test/Lower/mixed_loops.f90
@@ -53,7 +53,7 @@ subroutine while_inside_do_loop
   ! CHECK: fir.store %[[TDEC]] to %[[T_REF]]
   ! CHECK: %[[I3:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[C1_2:.*]] = arith.constant 1 : i32
-  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] : i32
+  ! CHECK: %[[IINC:.*]] = arith.addi %[[I3]], %[[C1_2]] overflow<nsw> : i32
   ! CHECK: fir.store %[[IINC]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK: br ^[[HDR1]]
   end do
@@ -100,10 +100,10 @@ subroutine do_inside_while_loop
         ! CHECK-DAG: %[[C2:.*]] = arith.constant 2 : i32
         ! CHECK: %[[JINC:.*]] = arith.muli %[[C2]], %[[J2]] : i32
         ! CHECK: fir.store %[[JINC]] to %[[J_REF]] : !fir.ref<i32>
-        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] : index
+        ! CHECK: %[[IINC:.*]] = arith.addi %[[IDX]], %[[C1]] overflow<nsw> : index
         ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
         ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
-        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+        ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] overflow<nsw> : i32
         ! CHECK: fir.result %[[IINC]], %[[I_IVINC]] : index, i32
       do i=8,13
         j=j*2
diff --git a/flang/test/Lower/nsw.f90 b/flang/test/Lower/nsw.f90
index 84435b7133042..4ee9e5da829e6 100644
--- a/flang/test/Lower/nsw.f90
+++ b/flang/test/Lower/nsw.f90
@@ -59,3 +59,98 @@ subroutine bitwise_comparison(a, b)
 ! CHECK-LABEL:   func.func @_QPbitwise_comparison(
 ! CHECK-NOT: overflow<nsw>
 ! CHECK:           return
+
+subroutine loop_params(a,lb,ub,st)
+  integer :: i, lb, ub, st
+  integer :: a(lb:ub)
+  do i = lb+1, ub-1, st*2
+    a(i) = i
+  end do
+end subroutine
+! CHECK-LABEL:   func.func @_QPloop_params(
+! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_9:.*]] = fir.declare %{{.*}}lb"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_10:.*]] = fir.declare %{{.*}}ub"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:           %[[VAL_13:.*]] = fir.declare %{{.*}}st"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CHECK:           %[[VAL_25:.*]] = arith.addi %[[VAL_14]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i32) -> index
+! CHECK:           %[[VAL_27:.*]] = arith.subi %[[VAL_16]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (i32) -> index
+! CHECK:           %[[VAL_29:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+! CHECK:           %[[VAL_30:.*]] = arith.muli %[[VAL_29]], %[[VAL_4]] overflow<nsw> : i32
+! CHECK:           %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i32) -> index
+! CHECK:           %[[VAL_32:.*]] = fir.convert %[[VAL_26]] : (index) -> i32
+! CHECK:           %[[VAL_33:.*]]:2 = fir.do_loop %[[VAL_34:.*]] = %[[VAL_26]] to %[[VAL_28]] step %[[VAL_31]] iter_args(%[[VAL_35:.*]] = %[[VAL_32]]) -> (index, i32) {
+
+subroutine loop_params2(a,lb,ub,st)
+  integer :: i, lb, ub, st
+  integer :: a(lb:ub)
+  real :: ii
+  do ii = lb+1, ub-1, st*2
+    i = ii
+    a(i) = i
+  end do
+end subroutine
+! CHECK-LABEL:   func.func @_QPloop_params2(
+! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_8:.*]] = fir.alloca index
+! CHECK:           %[[VAL_9:.*]] = fir.alloca f32
+! CHECK:           %[[VAL_11:.*]] = fir.declare %{{.*}}lb"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}ub"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_14:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:           %[[VAL_16:.*]] = fir.declare %{{.*}}ii"} : (!fir.ref<f32>) -> !fir.ref<f32>
+! CHECK:           %[[VAL_17:.*]] = fir.declare %{{.*}}st"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_18:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_12]] : !fir.ref<i32>
+! CHECK:           %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (i32) -> f32
+! CHECK:           %[[VAL_31:.*]] = arith.subi %[[VAL_20]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> f32
+! CHECK:           %[[VAL_33:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK:           %[[VAL_34:.*]] = arith.muli %[[VAL_33]], %[[VAL_4]] overflow<nsw> : i32
+! CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_34]] : (i32) -> f32
+! CHECK:           fir.store %[[VAL_35]] to %[[VAL_9]] : !fir.ref<f32>
+! CHECK:           %[[VAL_36:.*]] = arith.subf %[[VAL_32]], %[[VAL_30]] fastmath<contract> : f32
+! CHECK:           %[[VAL_37:.*]] = arith.addf %[[VAL_36]], %[[VAL_35]] fastmath<contract> : f32
+! CHECK:           %[[VAL_38:.*]] = arith.divf %[[VAL_37]], %[[VAL_35]] fastmath<contract> : f32
+! CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (f32) -> index
+! CHECK:           fir.store %[[VAL_39]] to %[[VAL_8]] : !fir.ref<index>
+! CHECK:           fir.store %[[VAL_30]] to %[[VAL_16]] : !fir.ref<f32>
+! CHECK:           cf.br ^bb1
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_40:.*]] = fir.load %[[VAL_8]] : !fir.ref<index>
+! CHECK:           %[[VAL_41:.*]] = arith.cmpi sgt, %[[VAL_40]], %[[VAL_6]] : index
+! CHECK:           cf.cond_br %[[VAL_41]], ^bb2, ^bb3
+! CHECK:         ^bb2:
+
+subroutine loop_params3(a,lb,ub,st)
+  integer :: i, lb, ub, st
+  integer :: a(lb:ub)
+  do concurrent (i=lb+1:ub-1:st*2)
+    a(i) = i
+  end do
+end subroutine
+! CHECK-LABEL:   func.func @_QPloop_params3(
+! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_9:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:           %[[VAL_11:.*]] = fir.declare %{{.*}}lb"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}ub"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_14:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:           %[[VAL_15:.*]] = fir.declare %{{.*}}st"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK:           %[[VAL_18:.*]] = fir.load %[[VAL_12]] : !fir.ref<i32>
+! CHECK:           %[[VAL_27:.*]] = arith.addi %[[VAL_16]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (i32) -> index
+! CHECK:           %[[VAL_29:.*]] = arith.subi %[[VAL_18]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (i32) -> index
+! CHECK:           %[[VAL_31:.*]] = fir.load %[[VAL_15]] : !fir.ref<i32>
+! CHECK:           %[[VAL_32:.*]] = arith.muli %[[VAL_31]], %[[VAL_4]] overflow<nsw> : i32
+! CHECK:           %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> index
+! CHECK:           fir.do_loop %[[VAL_34:.*]] = %[[VAL_28]] to %[[VAL_30]] step %[[VAL_33]] unordered {
diff --git a/flang/test/Lower/vector-subscript-io.f90 b/flang/test/Lower/vector-subscript-io.f90
index 129e3ee1206c0..372130fd09907 100644
--- a/flang/test/Lower/vector-subscript-io.f90
+++ b/flang/test/Lower/vector-subscript-io.f90
@@ -30,7 +30,7 @@ subroutine simple(x, y)
 ! CHECK:   %[[VAL_19:.*]] = fir.array_coor %[[VAL_20]](%[[VAL_10]]) {{\[}}%[[VAL_11]]] %[[VAL_18]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_22:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_9]], %[[VAL_21]], %[[VAL_3]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] : index
+! CHECK:   %[[VAL_23:.*]] = arith.addi %[[VAL_12]], %[[VAL_6]] overflow<nsw> : index
 ! CHECK:   %[[VAL_24:.*]] = arith.subi %[[VAL_13]], %[[VAL_6]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_23]], %[[VAL_24]] : index, index)
 ! CHECK: ^bb3:
@@ -79,7 +79,7 @@ integer function get_substcript()
 ! CHECK:   %[[VAL_49:.*]] = fir.convert %[[VAL_48]] : (i32) -> index
 ! CHECK:   %[[VAL_50:.*]] = fir.array_coor %[[VAL_51]] {{\[}}%[[VAL_42]]] %[[VAL_46]], %[[VAL_49]] : (!fir.box<!fir.array<?x?xf32>>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_52:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_34]], %[[VAL_50]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] : index
+! CHECK:   %[[VAL_53:.*]] = arith.addi %[[VAL_43]], %[[VAL_30]] overflow<nsw> : index
 ! CHECK:   %[[VAL_54:.*]] = arith.subi %[[VAL_44]], %[[VAL_30]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_53]], %[[VAL_54]] : index, index)
 ! CHECK: ^bb3:
@@ -122,7 +122,7 @@ subroutine with_assumed_shapes(x, y)
 ! CHECK:   %[[VAL_77:.*]] = fir.array_coor %[[VAL_78]] {{\[}}%[[VAL_70]]] %[[VAL_76]] : (!fir.box<!fir.array<?xi32>>, !fir.slice<1>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_79:.*]] = fir.convert %[[VAL_77]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_80:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_67]], %[[VAL_79]], %[[VAL_62]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] : index
+! CHECK:   %[[VAL_81:.*]] = arith.addi %[[VAL_71]], %[[VAL_64]] overflow<nsw> : index
 ! CHECK:   %[[VAL_82:.*]] = arith.subi %[[VAL_72]], %[[VAL_64]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_81]], %[[VAL_82]] : index, index)
 ! CHECK: ^bb3:
@@ -162,7 +162,7 @@ subroutine lower_bounds(x, y)
 ! CHECK:   %[[VAL_107:.*]] = fir.array_coor %[[VAL_108]](%[[VAL_97]]) {{\[}}%[[VAL_99]]] %[[VAL_91]], %[[VAL_106]] : (!fir.ref<!fir.array<4x6xi32>>, !fir.shapeshift<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_109:.*]] = fir.convert %[[VAL_107]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_110:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_96]], %[[VAL_109]], %[[VAL_90]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] : index
+! CHECK:   %[[VAL_111:.*]] = arith.addi %[[VAL_100]], %[[VAL_93]] overflow<nsw> : index
 ! CHECK:   %[[VAL_112:.*]] = arith.subi %[[VAL_101]], %[[VAL_93]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_111]], %[[VAL_112]] : index, index)
 ! CHECK: ^bb3:
@@ -202,11 +202,11 @@ subroutine two_vectors(x, y1, y2)
 ! CHECK:   %[[VAL_138:.*]] = fir.convert %[[VAL_137]] : (i32) -> index
 ! CHECK:   %[[VAL_139:.*]] = fir.array_coor %[[VAL_140]](%[[VAL_123]]) {{\[}}%[[VAL_124]]] %[[VAL_134]], %[[VAL_138]] : (!fir.ref<!fir.array<4x4xf32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_141:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_122]], %[[VAL_139]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] : index
+! CHECK:   %[[VAL_142:.*]] = arith.addi %[[VAL_128]], %[[VAL_119]] overflow<nsw> : index
 ! CHECK:   %[[VAL_143:.*]] = arith.subi %[[VAL_129]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_142]], %[[VAL_143]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] : index
+! CHECK:   %[[VAL_144:.*]] = arith.addi %[[VAL_125]], %[[VAL_119]] overflow<nsw> : index
 ! CHECK:   %[[VAL_145:.*]] = arith.subi %[[VAL_126]], %[[VAL_119]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_144]], %[[VAL_145]] : index, index)
 ! CHECK: ^bb5:
@@ -245,11 +245,11 @@ subroutine triplets_and_vector(x, y)
 ! CHECK:   %[[VAL_169:.*]] = fir.array_coor %[[VAL_170]](%[[VAL_157]]) {{\[}}%[[VAL_158]]] %[[VAL_162]], %[[VAL_168]] : (!fir.ref<!fir.array<4x4xcomplex<f32>>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<complex<f32>>
 ! CHECK:   %[[VAL_171:.*]] = fir.convert %[[VAL_169]] : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_172:.*]] = fir.call @_FortranAioInputComplex32(%[[VAL_156]], %[[VAL_171]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] : index
+! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] overflow<nsw> : index
 ! CHECK:   %[[VAL_174:.*]] = arith.subi %[[VAL_163]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_173]], %[[VAL_174]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] : index
+! CHECK:   %[[VAL_175:.*]] = arith.addi %[[VAL_159]], %[[VAL_153]] overflow<nsw> : index
 ! CHECK:   %[[VAL_176:.*]] = arith.subi %[[VAL_160]], %[[VAL_153]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_175]], %[[VAL_176]] : index, index)
 ! CHECK: ^bb5:
@@ -287,7 +287,7 @@ subroutine simple_char(x, y)
 ! CHECK:   %[[VAL_200:.*]] = fir.convert %[[VAL_199]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_201:.*]] = fir.convert %[[VAL_184]]#1 : (index) -> i64
 ! CHECK:   %[[VAL_202:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_189]], %[[VAL_200]], %[[VAL_201]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] : index
+! CHECK:   %[[VAL_203:.*]] = arith.addi %[[VAL_192]], %[[VAL_183]] overflow<nsw> : index
 ! CHECK:   %[[VAL_204:.*]] = arith.subi %[[VAL_193]], %[[VAL_183]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_203]], %[[VAL_204]] : index, index)
 ! CHECK: ^bb3:
@@ -333,7 +333,7 @@ subroutine substring(x, y, i, j)
 ! CHECK:   %[[VAL_238:.*]] = fir.convert %[[VAL_233]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:   %[[VAL_239:.*]] = fir.convert %[[VAL_237]] : (index) -> i64
 ! CHECK:   %[[VAL_240:.*]] = fir.call @_FortranAioInputAscii(%[[VAL_213]], %[[VAL_238]], %[[VAL_239]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64) -> i1
-! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] : index
+! CHECK:   %[[VAL_241:.*]] = arith.addi %[[VAL_221]], %[[VAL_210]] overflow<nsw> : index
 ! CHECK:   %[[VAL_242:.*]] = arith.subi %[[VAL_222]], %[[VAL_210]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_241]], %[[VAL_242]] : index, index)
 ! CHECK: ^bb3:
@@ -366,7 +366,7 @@ subroutine complex_part(z, y)
 ! CHECK:   %[[VAL_260:.*]] = fir.convert %[[VAL_259]] : (i32) -> index
 ! CHECK:   %[[VAL_261:.*]] = fir.array_coor %[[VAL_262]] {{\[}}%[[VAL_254]]] %[[VAL_260]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_263:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_251]], %[[VAL_261]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] : index
+! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] overflow<nsw> : index
 ! CHECK:   %[[VAL_265:.*]] = arith.subi %[[VAL_256]], %[[VAL_248]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_264]], %[[VAL_265]] : index, index)
 ! CHECK: ^bb3:
@@ -414,7 +414,7 @@ subroutine simple_derived(x, y)
 ! CHECK:   %[[VAL_288:.*]] = fir.embox %[[VAL_286]] : (!fir.ref<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>
 ! CHECK:   %[[VAL_289:.*]] = fir.convert %[[VAL_288]] : (!fir.box<!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>) -> !fir.box<none>
 ! CHECK:   %[[VAL_290:.*]] = fir.call @_FortranAioInputDerivedType(%[[VAL_276]], %[[VAL_289]], {{.*}}) {{.*}}: (!fir.ref<i8>, !fir.box<none>, !fir.ref<none>) -> i1
-! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] : index
+! CHECK:   %[[VAL_291:.*]] = arith.addi %[[VAL_279]], %[[VAL_273]] overflow<nsw> : index
 ! CHECK:   %[[VAL_292:.*]] = arith.subi %[[VAL_280]], %[[VAL_273]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_291]], %[[VAL_292]] : index, index)
 ! CHECK: ^bb3:
@@ -463,11 +463,11 @@ subroutine with_path(b, i)
 ! CHECK:   %[[VAL_325:.*]] = fir.array_coor %[[VAL_326:.*]](%[[VAL_313]]) {{\[}}%[[VAL_315]]] %[[VAL_301]], %[[VAL_324]], %[[VAL_316]] : (!fir.box<!fir.array<?x?x?x!fir.type<_QMderived_typesTt2{a:!fir.array<5x5x!fir.type<_QMderived_typesTt{i:i32,c:!fir.char<1,2>}>>}>>>, !fir.shift<3>, !fir.slice<3>, index, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_327:.*]] = fir.convert %[[VAL_325]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_328:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_308]], %[[VAL_327]], %[[VAL_302]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] : index
+! CHECK:   %[[VAL_329:.*]] = arith.addi %[[VAL_319]], %[[VAL_305]] overflow<nsw> : index
 ! CHECK:   %[[VAL_330:.*]] = arith.subi %[[VAL_320]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb2(%[[VAL_329]], %[[VAL_330]] : index, index)
 ! CHECK: ^bb4:
-! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] : index
+! CHECK:   %[[VAL_331:.*]] = arith.addi %[[VAL_316]], %[[VAL_305]] overflow<nsw> : index
 ! CHECK:   %[[VAL_332:.*]] = arith.subi %[[VAL_317]], %[[VAL_305]] : index
 ! CHECK:   cf.br ^bb1(%[[VAL_331]], %[[VAL_332]] : index, index)
 ! CHECK: ^bb5:
@@ -505,7 +505,7 @@ subroutine simple_iostat(x, y, j, stat)
 ! CHECK:   %[[VAL_355:.*]] = fir.convert %[[VAL_354]] : (i32) -> index
 ! CHECK:   %[[VAL_356:.*]] = fir.array_coor %[[VAL_357]] {{\[}}%[[VAL_347]]] %[[VAL_355]] : (!fir.box<!fir.array<?xf32>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_358:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_343]], %[[VAL_356]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
-! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] : index
+! CHECK:   %[[VAL_359:.*]] = arith.addi %[[VAL_349]], %[[VAL_338]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb1(%[[VAL_359]], %[[VAL_358]] : index, i1)
 ! CHECK: ^bb3:
 ! CHECK:   cf.cond_br %[[VAL_350]], ^bb4, ^bb5
@@ -568,10 +568,10 @@ subroutine iostat_in_io_loop(k, j, stat)
 ! CHECK:   %[[VAL_399:.*]] = fir.array_coor %[[VAL_400]](%[[VAL_387]]) {{\[}}%[[VAL_389]]] %[[VAL_394]], %[[VAL_398]] : (!fir.ref<!fir.array<3x5xi32>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_401:.*]] = fir.convert %[[VAL_399]] : (!fir.ref<i32>) -> !fir.ref<i64>
 ! CHECK:   %[[VAL_402:.*]] = fir.call @_FortranAioInputInteger(%[[VAL_378]], %[[VAL_401]], %[[VAL_374]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
-! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] : index
+! CHECK:   %[[VAL_403:.*]] = arith.addi %[[VAL_390]], %[[VAL_371]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb4(%[[VAL_403]], %[[VAL_402]] : index, i1)
 ! CHECK: ^bb6(%[[VAL_404:.*]]: i1):
-! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] : index
+! CHECK:   %[[VAL_405:.*]] = arith.addi %[[VAL_380]], %[[VAL_371]] overflow<nsw> : index
 ! CHECK:   cf.br ^bb1(%[[VAL_405]], %[[VAL_404]] : index, i1)
 ! CHECK: ^bb7:
 ! CHECK:   %[[VAL_406:.*]] = fir.convert %[[VAL_380]] : (index) -> i32
diff --git a/flang/test/Parser/OpenMP/affinity-clause.f90 b/flang/test/Parser/OpenMP/affinity-clause.f90
index 5e9e0a2194bab..642af6aeb7e49 100644
--- a/flang/test/Parser/OpenMP/affinity-clause.f90
+++ b/flang/test/Parser/OpenMP/affinity-clause.f90
@@ -55,7 +55,7 @@ subroutine f02(x)
 
 !UNPARSE: SUBROUTINE f02 (x)
 !UNPARSE:  INTEGER x(10_4)
-!UNPARSE: !$OMP TASK  AFFINITY(ITERATOR(INTEGER i = 1_4:3_4):x(i))
+!UNPARSE: !$OMP TASK  AFFINITY(ITERATOR(INTEGER i = 1_4:3_4): x(i))
 !UNPARSE:   x=x+1_4
 !UNPARSE: !$OMP END TASK
 !UNPARSE: END SUBROUTINE
@@ -63,7 +63,7 @@ subroutine f02(x)
 !PARSE-TREE: OmpBeginBlockDirective
 !PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = task
 !PARSE-TREE: | OmpClauseList -> OmpClause -> Affinity -> OmpAffinityClause
-!PARSE-TREE: | | OmpIterator -> OmpIteratorSpecifier
+!PARSE-TREE: | | Modifier -> OmpIterator -> OmpIteratorSpecifier
 !PARSE-TREE: | | | TypeDeclarationStmt
 !PARSE-TREE: | | | | DeclarationTypeSpec -> IntrinsicTypeSpec -> IntegerTypeSpec ->
 !PARSE-TREE: | | | | EntityDecl
diff --git a/flang/test/Parser/OpenMP/allocators-unparse.f90 b/flang/test/Parser/OpenMP/allocators-unparse.f90
index 062a48b02635f..5cd0230471fc4 100644
--- a/flang/test/Parser/OpenMP/allocators-unparse.f90
+++ b/flang/test/Parser/OpenMP/allocators-unparse.f90
@@ -18,18 +18,18 @@ subroutine allocate()
 end subroutine allocate
 
 !CHECK: INTEGER, ALLOCATABLE :: arr1(:), arr2(:,:)
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc:arr1)
+!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(omp_default_mem_alloc: arr1)
 !CHECK-NEXT: ALLOCATE(arr1(5))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc),ALIGN(32):arr1) ALLOC&
-!CHECK-NEXT:!$OMP&ATE(omp_default_mem_alloc:arr2)
+!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALLOCATOR(omp_default_mem_alloc), ALIGN(32): arr1) ALL&
+!CHECK-NEXT:!$OMP&OCATE(omp_default_mem_alloc: arr2)
 !CHECK-NEXT: ALLOCATE(arr1(10), arr2(3,2))
-!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32):arr2)
+!CHECK-NEXT:!$OMP ALLOCATE ALLOCATE(ALIGN(32): arr2)
 !CHECK-NEXT: ALLOCATE(arr2(5,3))
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
 !PARSE-TREE-NEXT: Verbatim
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: AllocateModifier -> Allocator -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name =
+!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
 !PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
 !PARSE-TREE-NEXT: AllocateStmt
 !PARSE-TREE-NEXT: Allocation
@@ -38,12 +38,11 @@ end subroutine allocate
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
 !PARSE-TREE-NEXT: Verbatim
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: AllocateModifier -> ComplexModifier
-!PARSE-TREE-NEXT: Allocator -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name =
-!PARSE-TREE-NEXT: Align -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: Modifier -> OmpAllocatorComplexModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
+!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
 !PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr1'
 !PARSE-TREE-NEXT: OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: AllocateModifier -> Allocator -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name =
+!PARSE-TREE-NEXT: Modifier -> OmpAllocatorSimpleModifier -> Scalar -> Integer -> Expr -> Designator -> DataRef -> Name = 'omp_default_mem_alloc'
 !PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
 !PARSE-TREE-NEXT: AllocateStmt
 !PARSE-TREE-NEXT: Allocation
@@ -56,7 +55,7 @@ end subroutine allocate
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAllocatorsConstruct
 !PARSE-TREE-NEXT: Verbatim
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Allocate -> OmpAllocateClause
-!PARSE-TREE-NEXT: AllocateModifier -> Align -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
+!PARSE-TREE-NEXT: Modifier -> OmpAlignModifier -> Scalar -> Integer -> Expr -> LiteralConstant -> IntLiteralConstant = '32'
 !PARSE-TREE-NEXT: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'arr2'
 !PARSE-TREE-NEXT: AllocateStmt
 !PARSE-TREE-NEXT: Allocation
diff --git a/flang/test/Parser/OpenMP/atomic-compare.f90 b/flang/test/Parser/OpenMP/atomic-compare.f90
new file mode 100644
index 0000000000000..5cd02698ff482
--- /dev/null
+++ b/flang/test/Parser/OpenMP/atomic-compare.f90
@@ -0,0 +1,16 @@
+! RUN: not %flang_fc1  -fopenmp-version=51 -fopenmp %s 2>&1 | FileCheck %s
+! OpenMP version for documentation purposes only - it isn't used until Sema.
+! This is testing for Parser errors that bail out before Sema. 
+program main
+   implicit none
+   integer :: i, j = 10
+   logical :: r
+
+  !CHECK: error: expected OpenMP construct
+  !$omp atomic compare write
+  r =  i .eq. j + 1
+
+  !CHECK: error: expected end of line
+  !$omp atomic compare num_threads(4)
+  r = i .eq. j
+end program main
diff --git a/flang/test/Parser/OpenMP/atomic-unparse.f90 b/flang/test/Parser/OpenMP/atomic-unparse.f90
index f9d8ec5d5c681..64fa79fb1d1a2 100644
--- a/flang/test/Parser/OpenMP/atomic-unparse.f90
+++ b/flang/test/Parser/OpenMP/atomic-unparse.f90
@@ -3,6 +3,7 @@
 program main
    implicit none
    integer :: i, j = 10
+   integer :: k
 !READ
 !$omp atomic read
    i = j
@@ -121,6 +122,49 @@ program main
    i = j
 !$omp end atomic
 
+!COMPARE
+!$omp atomic compare
+   if (k == i) k = j
+!$omp atomic seq_cst compare
+   if (k == j) then
+      k = i
+   end if
+!$omp atomic compare seq_cst
+   if (k .eq. j) then
+      k = i
+   end if
+!$omp atomic release compare
+   if (i .eq. j) k = i
+!$omp atomic compare release
+   if (i .eq. j) then
+      i = k
+   end if
+!$omp atomic acq_rel compare
+   if (k .eq. j) then
+      j = i
+   end if
+!$omp atomic compare acq_rel
+   if (i .eq. j) then
+      i = k
+   end if
+!$omp atomic acquire compare
+   if (i .eq. j + 1) then
+      i = j
+   end if
+   
+!$omp atomic compare acquire
+   if (i .eq. j) then
+      i = k
+   end if
+!$omp atomic relaxed compare
+   if (i .eq. j) then
+      i = k
+   end if
+!$omp atomic compare relaxed
+   if (i .eq. k) then
+      i = j
+   end if
+
 !ATOMIC
 !$omp atomic
    i = j
@@ -205,6 +249,20 @@ end program main
 !CHECK: !$OMP ATOMIC CAPTURE RELAXED
 !CHECK: !$OMP END ATOMIC
 
+!COMPARE
+
+!CHECK: !$OMP ATOMIC COMPARE
+!CHECK: !$OMP ATOMIC SEQ_CST COMPARE
+!CHECK: !$OMP ATOMIC COMPARE SEQ_CST
+!CHECK: !$OMP ATOMIC RELEASE COMPARE
+!CHECK: !$OMP ATOMIC COMPARE RELEASE
+!CHECK: !$OMP ATOMIC ACQ_REL COMPARE
+!CHECK: !$OMP ATOMIC COMPARE ACQ_REL
+!CHECK: !$OMP ATOMIC ACQUIRE COMPARE
+!CHECK: !$OMP ATOMIC COMPARE ACQUIRE
+!CHECK: !$OMP ATOMIC RELAXED COMPARE
+!CHECK: !$OMP ATOMIC COMPARE RELAXED
+
 !ATOMIC
 !CHECK: !$OMP ATOMIC
 !CHECK: !$OMP ATOMIC SEQ_CST
diff --git a/flang/test/Parser/OpenMP/bind-clause.f90 b/flang/test/Parser/OpenMP/bind-clause.f90
index 332e5fe26ed8a..5f1e6b47f1c8d 100644
--- a/flang/test/Parser/OpenMP/bind-clause.f90
+++ b/flang/test/Parser/OpenMP/bind-clause.f90
@@ -20,6 +20,6 @@ subroutine f00
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
 !PARSE-TREE: | OmpBeginLoopDirective
 !PARSE-TREE: | | OmpLoopDirective -> llvm::omp::Directive = loop
-!PARSE-TREE: | | OmpClauseList -> OmpClause -> Bind -> OmpBindClause -> Type = Parallel
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Bind -> OmpBindClause -> Binding = Parallel
 !PARSE-TREE: | DoConstruct
 
diff --git a/flang/test/Parser/OpenMP/declare_target-device_type.f90 b/flang/test/Parser/OpenMP/declare_target-device_type.f90
index 40eb1c2fa4cae..b6903614a628e 100644
--- a/flang/test/Parser/OpenMP/declare_target-device_type.f90
+++ b/flang/test/Parser/OpenMP/declare_target-device_type.f90
@@ -31,7 +31,7 @@ subroutine openmp_declare_target
     end do
 
 !PARSE-TREE: OpenMPDeclarativeConstruct -> OpenMPDeclareTargetConstruct
-!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> Type = Host
-!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> Type = Nohost
-!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> Type = Any
+!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> DeviceTypeDescription = Host
+!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> DeviceTypeDescription = Nohost
+!PARSE-TREE: OmpDeclareTargetSpecifier -> OmpDeclareTargetWithClause -> OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> DeviceTypeDescription = Any
 END subroutine openmp_declare_target
diff --git a/flang/test/Parser/OpenMP/depobj-construct.f90 b/flang/test/Parser/OpenMP/depobj-construct.f90
index 51726a5adf99e..f186c82a2ccc3 100644
--- a/flang/test/Parser/OpenMP/depobj-construct.f90
+++ b/flang/test/Parser/OpenMP/depobj-construct.f90
@@ -8,14 +8,14 @@ subroutine f00
 
 !UNPARSE: SUBROUTINE f00
 !UNPARSE:  INTEGER x, y
-!UNPARSE: !$OMP DEPOBJ(x) DEPEND(IN:y)
+!UNPARSE: !$OMP DEPOBJ(x) DEPEND(IN: y)
 !UNPARSE: END SUBROUTINE
 
 !PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPDepobjConstruct
 !PARSE-TREE: | Verbatim
 !PARSE-TREE: | OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: | OmpClause -> Depend -> OmpDependClause -> TaskDep
-!PARSE-TREE: | | OmpTaskDependenceType -> Value = In
+!PARSE-TREE: | | Modifier -> OmpTaskDependenceType -> Value = In
 !PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'y'
 
 subroutine f01
diff --git a/flang/test/Parser/OpenMP/if-clause-unparse.f90 b/flang/test/Parser/OpenMP/if-clause-unparse.f90
index 20f7b36e743e1..ce058caa54a93 100644
--- a/flang/test/Parser/OpenMP/if-clause-unparse.f90
+++ b/flang/test/Parser/OpenMP/if-clause-unparse.f90
@@ -10,50 +10,50 @@ program if_unparse
   !$omp target update if(cond)
 
   ! CHECK: !$OMP TARGET UPDATE
-  ! CHECK-SAME: IF(TARGETUPDATE:cond)
+  ! CHECK-SAME: IF(TARGET UPDATE: cond)
   !$omp target update if(target update: cond)
   
   ! CHECK: !$OMP TARGET UPDATE
-  ! CHECK-SAME: IF(TARGETUPDATE:cond)
+  ! CHECK-SAME: IF(TARGET UPDATE: cond)
   !$omp target update if(targetupdate: cond)
 
   ! CHECK: !$OMP TARGET ENTER DATA
-  ! CHECK-SAME: IF(TARGETENTERDATA:cond)
+  ! CHECK-SAME: IF(TARGET ENTER DATA: cond)
   !$omp target enter data map(to: i) if(target enter data: cond)
 
   ! CHECK: !$OMP TARGET EXIT DATA
-  ! CHECK-SAME: IF(TARGETEXITDATA:cond)
+  ! CHECK-SAME: IF(TARGET EXIT DATA: cond)
   !$omp target exit data map(from: i) if(target exit data: cond)
 
   ! CHECK: !$OMP TARGET DATA
-  ! CHECK-SAME: IF(TARGETDATA:cond)
+  ! CHECK-SAME: IF(TARGET DATA: cond)
   !$omp target data map(tofrom: i) if(target data: cond)
   !$omp end target data
 
   ! CHECK: !$OMP TARGET
-  ! CHECK-SAME: IF(TARGET:cond)
+  ! CHECK-SAME: IF(TARGET: cond)
   !$omp target if(target: cond)
   !$omp end target
 
   ! CHECK: !$OMP TEAMS
-  ! CHECK-SAME: IF(TEAMS:cond)
+  ! CHECK-SAME: IF(TEAMS: cond)
   !$omp teams if(teams: cond)
   !$omp end teams
 
   ! CHECK: !$OMP PARALLEL DO SIMD
-  ! CHECK-SAME: IF(PARALLEL:i<10) IF(SIMD:.FALSE.)
+  ! CHECK-SAME: IF(PARALLEL: i<10) IF(SIMD: .FALSE.)
   !$omp parallel do simd if(parallel: i < 10) if(simd: .false.)
   do i = 1, 10
   end do
   !$omp end parallel do simd
 
   ! CHECK: !$OMP TASK
-  ! CHECK-SAME: IF(TASK:cond)
+  ! CHECK-SAME: IF(TASK: cond)
   !$omp task if(task: cond)
   !$omp end task
 
   ! CHECK: !$OMP TASKLOOP
-  ! CHECK-SAME: IF(TASKLOOP:cond)
+  ! CHECK-SAME: IF(TASKLOOP: cond)
   !$omp taskloop if(taskloop: cond)
   do i = 1, 10
   end do
diff --git a/flang/test/Parser/OpenMP/if-clause.f90 b/flang/test/Parser/OpenMP/if-clause.f90
index 6d69e16e7cc73..b3e3913f8bd1c 100644
--- a/flang/test/Parser/OpenMP/if-clause.f90
+++ b/flang/test/Parser/OpenMP/if-clause.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck %s
 
 program openmp_parse_if
   logical :: cond
@@ -11,34 +11,34 @@ program openmp_parse_if
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target update
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = TargetUpdate
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target update
   !$omp target update if(target update: cond) to(i)
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target enter data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = TargetEnterData
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target enter data
   !$omp target enter data map(to: i) if(target enter data: cond)
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target exit data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = TargetExitData
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target exit data
   !$omp target exit data map(from: i) if(target exit data: cond)
 
   ! CHECK: OmpBlockDirective -> llvm::omp::Directive = target data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = TargetData
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target data
   !$omp target data map(tofrom: i) if(target data: cond)
   !$omp end target data
 
   ! CHECK: OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do simd
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Target
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Teams
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = teams
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Parallel
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = parallel
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Simd
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = simd
   !$omp target teams distribute parallel do simd if(target: cond) &
   !$omp&    if(teams: cond) if(parallel: cond) if(simd: cond)
   do i = 1, 10
@@ -47,13 +47,13 @@ program openmp_parse_if
 
   ! CHECK: OmpBlockDirective -> llvm::omp::Directive = task
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Task
+  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = task
   !$omp task if(task: cond)
   !$omp end task
 
   ! CHECK: OmpLoopDirective -> llvm::omp::Directive = taskloop
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier = Taskloop
+  ! CHECK-NEXT: DirectiveNameModifier -> llvm::omp::Directive = taskloop
   !$omp taskloop if(taskloop: cond)
   do i = 1, 10
   end do
diff --git a/flang/test/Parser/OpenMP/lastprivate-clause.f90 b/flang/test/Parser/OpenMP/lastprivate-clause.f90
index 382f02c75e7f1..ac25174f3cc42 100644
--- a/flang/test/Parser/OpenMP/lastprivate-clause.f90
+++ b/flang/test/Parser/OpenMP/lastprivate-clause.f90
@@ -39,7 +39,7 @@ subroutine foo2()
 !UNPARSE: SUBROUTINE foo2
 !UNPARSE:  INTEGER x, i
 !UNPARSE:   x=1_4
-!UNPARSE: !$OMP PARALLEL DO  LASTPRIVATE(CONDITIONAL:x)
+!UNPARSE: !$OMP PARALLEL DO  LASTPRIVATE(CONDITIONAL: x)
 !UNPARSE:  DO i=1_4,100_4
 !UNPARSE:    x=x+1_4
 !UNPARSE:  END DO
@@ -49,6 +49,6 @@ subroutine foo2()
 !PARSE-TREE:   Name = 'foo2'
 !PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
 !PARSE-TREE: OmpClauseList -> OmpClause -> Lastprivate -> OmpLastprivateClause
-!PARSE-TREE:   LastprivateModifier = Conditional
+!PARSE-TREE:   Modifier -> OmpLastprivateModifier -> Value = Conditional
 !PARSE-TREE:   OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: EndSubroutineStmt
diff --git a/flang/test/Parser/OpenMP/proc-bind.f90 b/flang/test/Parser/OpenMP/proc-bind.f90
index 08bcf69e5e765..3115b3771f27d 100644
--- a/flang/test/Parser/OpenMP/proc-bind.f90
+++ b/flang/test/Parser/OpenMP/proc-bind.f90
@@ -6,7 +6,7 @@
 ! PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
 ! PARSE-TREE:  OmpBeginBlockDirective
 ! PARSE-TREE:   OmpBlockDirective -> llvm::omp::Directive = parallel
-! PARSE-TREE:   OmpClauseList -> OmpClause -> ProcBind -> OmpProcBindClause -> Type = Primary
+! PARSE-TREE:   OmpClauseList -> OmpClause -> ProcBind -> OmpProcBindClause -> AffinityPolicy = Primary
 subroutine sb1
   !$omp parallel proc_bind(primary)
   print *, "Hello"
diff --git a/flang/test/Parser/OpenMP/target-loop-unparse.f90 b/flang/test/Parser/OpenMP/target-loop-unparse.f90
index b204707049652..ee0013f613403 100644
--- a/flang/test/Parser/OpenMP/target-loop-unparse.f90
+++ b/flang/test/Parser/OpenMP/target-loop-unparse.f90
@@ -19,7 +19,7 @@ subroutine test_loop
 
   !PARSE-TREE: OmpBeginLoopDirective
   !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = loop
-  !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Bind -> OmpBindClause -> Type = Thread
+  !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Bind -> OmpBindClause -> Binding = Thread
   !CHECK: !$omp loop
   !$omp loop bind(thread)
   do i=1,10
diff --git a/flang/test/Parser/OpenMP/target_device_parse.f90 b/flang/test/Parser/OpenMP/target_device_parse.f90
index 30464c5b08ab8..7f5bee3793b2e 100644
--- a/flang/test/Parser/OpenMP/target_device_parse.f90
+++ b/flang/test/Parser/OpenMP/target_device_parse.f90
@@ -96,7 +96,7 @@ PROGRAM main
 !------------------------------------------------------
 ! Check Device Ancestor clause with a constant argument
 !------------------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(ANCESTOR:1)
+!CHECK: !$OMP TARGET DEVICE(ANCESTOR: 1)
 !$OMP TARGET DEVICE(ANCESTOR: 1)
   M = M + 1
 !CHECK: !$OMP END TARGET
@@ -105,7 +105,7 @@ PROGRAM main
 !PARSE-TREE: OmpBeginBlockDirective
 !PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = target
 !PARSE-TREE: OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!PARSE-TREE: DeviceModifier = Ancestor
+!PARSE-TREE: OmpDeviceModifier -> Value = Ancestor
 !PARSE-TREE: Scalar -> Integer -> Expr = '1_4'
 !PARSE-TREE: LiteralConstant -> IntLiteralConstant = '1'
 !PARSE-TREE: OmpEndBlockDirective
@@ -116,7 +116,7 @@ PROGRAM main
 !--------------------------------------------------------
 ! Check Device Devive-Num clause with a constant argument
 !--------------------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM:2)
+!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM: 2)
 !$OMP TARGET DEVICE(DEVICE_NUM: 2)
   M = M + 1
 !CHECK: !$OMP END TARGET
@@ -125,7 +125,7 @@ PROGRAM main
 !PARSE-TREE: OmpBeginBlockDirective
 !PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = target
 !PARSE-TREE: OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!PARSE-TREE: DeviceModifier = Device_Num
+!PARSE-TREE: OmpDeviceModifier -> Value = Device_Num
 !PARSE-TREE: Scalar -> Integer -> Expr = '2_4'
 !PARSE-TREE: LiteralConstant -> IntLiteralConstant = '2'
 !PARSE-TREE: OmpEndBlockDirective
@@ -136,7 +136,7 @@ PROGRAM main
 !-------------------------------------------------------------------
 ! Check Device Ancestor clause with a variable expression argument
 !-------------------------------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(ANCESTOR:X+Y)
+!CHECK: !$OMP TARGET DEVICE(ANCESTOR: X+Y)
 !$OMP TARGET DEVICE(ANCESTOR: X + Y)
   M = M + 1
 !CHECK: !$OMP END TARGET
@@ -145,7 +145,7 @@ PROGRAM main
 !PARSE-TREE: OmpBeginBlockDirective
 !PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = target
 !PARSE-TREE: OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!PARSE-TREE: DeviceModifier = Ancestor
+!PARSE-TREE: OmpDeviceModifier -> Value = Ancestor
 !PARSE-TREE: Scalar -> Integer -> Expr = 'x+y'
 !PARSE-TREE: Add
 !PARSE-TREE: Expr = 'x'
@@ -160,7 +160,7 @@ PROGRAM main
 !-------------------------------------------------------------------
 ! Check Device Devive-Num clause with a variable expression argument
 !-------------------------------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM:X-Y)
+!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM: X-Y)
 !$OMP TARGET DEVICE(DEVICE_NUM: X - Y)
   M = M + 1
 !CHECK: !$OMP END TARGET
@@ -169,7 +169,7 @@ PROGRAM main
 !PARSE-TREE: OmpBeginBlockDirective
 !PARSE-TREE: OmpBlockDirective -> llvm::omp::Directive = target
 !PARSE-TREE: OmpClauseList -> OmpClause -> Device -> OmpDeviceClause
-!PARSE-TREE: DeviceModifier = Device_Num
+!PARSE-TREE: OmpDeviceModifier -> Value = Device_Num
 !PARSE-TREE: Scalar -> Integer -> Expr = 'x-y'
 !PARSE-TREE: Subtract
 !PARSE-TREE: Expr = 'x'
diff --git a/flang/test/Parser/OpenMP/target_device_unparse.f90 b/flang/test/Parser/OpenMP/target_device_unparse.f90
index 605c2baf1b06b..403b64887e747 100644
--- a/flang/test/Parser/OpenMP/target_device_unparse.f90
+++ b/flang/test/Parser/OpenMP/target_device_unparse.f90
@@ -45,7 +45,7 @@ PROGRAM main
 !--------------------------------------------
 ! Ancestor followed by constant argument
 !--------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(ANCESTOR:0)
+!CHECK: !$OMP TARGET DEVICE(ANCESTOR: 0)
 !CHECK: !$OMP END TARGET
 !$OMP TARGET DEVICE(ANCESTOR: 0)
   M = M + 1
@@ -54,7 +54,7 @@ PROGRAM main
 !--------------------------------------------
 ! Device_Num followed by constant argument
 !--------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM:1)
+!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM: 1)
 !CHECK: !$OMP END TARGET
 !$OMP TARGET DEVICE(DEVICE_NUM: 1)
   M = M + 1
@@ -63,7 +63,7 @@ PROGRAM main
 !--------------------------------------------
 ! Ancestor followed by variable expression argument
 !--------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(ANCESTOR:X+Y)
+!CHECK: !$OMP TARGET DEVICE(ANCESTOR: X+Y)
 !CHECK: !$OMP END TARGET
 !$OMP TARGET DEVICE(ANCESTOR: X + Y)
   M = M + 1
@@ -72,7 +72,7 @@ PROGRAM main
 !--------------------------------------------
 ! Device_Num followed by variable expression argument
 !--------------------------------------------
-!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM:X-Y)
+!CHECK: !$OMP TARGET DEVICE(DEVICE_NUM: X-Y)
 !CHECK: !$OMP END TARGET
 !$OMP TARGET DEVICE(DEVICE_NUM: X - Y)
   M = M + 1
diff --git a/flang/test/Parser/OpenMP/taskloop.f90 b/flang/test/Parser/OpenMP/taskloop.f90
index a9c361046bd5f..f053aa7f0cff3 100644
--- a/flang/test/Parser/OpenMP/taskloop.f90
+++ b/flang/test/Parser/OpenMP/taskloop.f90
@@ -4,11 +4,11 @@
 subroutine parallel_work
   integer :: i
 
-!CHECK: !$OMP TASKLOOP  GRAINSIZE(STRICT:500_4)
+!CHECK: !$OMP TASKLOOP  GRAINSIZE(STRICT: 500_4)
 !PARSE-TREE: OmpBeginLoopDirective
 !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> Grainsize -> OmpGrainsizeClause
-!PARSE-TREE-NEXT: Prescriptiveness = Strict
+!PARSE-TREE-NEXT: Modifier -> OmpPrescriptiveness -> Value = Strict
 !PARSE-TREE-NEXT: Scalar -> Integer -> Expr = '500_4'
   !$omp taskloop grainsize(strict: 500)
   do i=1,10000
@@ -27,11 +27,11 @@ subroutine parallel_work
   end do
   !$omp end taskloop
 
-!CHECK: !$OMP TASKLOOP  NUM_TASKS(STRICT:500_4)
+!CHECK: !$OMP TASKLOOP  NUM_TASKS(STRICT: 500_4)
 !PARSE-TREE: OmpBeginLoopDirective
 !PARSE-TREE-NEXT: OmpLoopDirective -> llvm::omp::Directive = taskloop
 !PARSE-TREE-NEXT: OmpClauseList -> OmpClause -> NumTasks -> OmpNumTasksClause
-!PARSE-TREE-NEXT: Prescriptiveness = Strict
+!PARSE-TREE-NEXT: Modifier -> OmpPrescriptiveness -> Value = Strict
 !PARSE-TREE-NEXT: Scalar -> Integer -> Expr = '500_4'
   !$omp taskloop num_tasks(strict: 500)
   do i=1,10000
diff --git a/flang/test/Parser/decl-char-length.f90 b/flang/test/Parser/decl-char-length.f90
new file mode 100644
index 0000000000000..c6b39560cb62d
--- /dev/null
+++ b/flang/test/Parser/decl-char-length.f90
@@ -0,0 +1,17 @@
+! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+! Test parsing of alternative order of char-length in an
+! entity-decl or component-decl.
+program p
+  type t
+    !CHECK: CHARACTER c1(2_4)*3/"abc", "def"/
+    character c1*3(2)/'abc','def'/
+  end type
+  integer, parameter :: n=3
+  !CHECK: CHARACTER v1(2_4)*(3_4)/"ghi", "jkl"/
+  character v1*(n)(2)/'ghi','jkl'/
+  !CHECK: CHARACTER :: v2(1_4)*2 = "mn"
+  character::v2*2(1)='mn'
+end
+
+
+
diff --git a/flang/test/Preprocessing/bug117297.F90 b/flang/test/Preprocessing/bug117297.F90
new file mode 100644
index 0000000000000..88b54000f0463
--- /dev/null
+++ b/flang/test/Preprocessing/bug117297.F90
@@ -0,0 +1,7 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+!CHECK: CALL myfunc( 'hello ' // 'world' // 'again')
+#define NOCOMMENT
+NOCOMMENT CALL myfunc( 'hello ' // &
+NOCOMMENT 'world' // &
+NOCOMMENT 'again' )
+end
diff --git a/flang/test/Semantics/OpenMP/allocate-clause01.f90 b/flang/test/Semantics/OpenMP/allocate-clause01.f90
index 2b9a72e928eba..a014b548a8771 100644
--- a/flang/test/Semantics/OpenMP/allocate-clause01.f90
+++ b/flang/test/Semantics/OpenMP/allocate-clause01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=51
 ! OpenMP Version 5.2
 ! The allocate clause's allocator modifier must be of type allocator_handle
 ! and the align modifier must be constant, positive integer expression
@@ -10,15 +10,11 @@ subroutine allocate()
 
     integer, allocatable :: a, b, c
 
-    !ERROR: The parameter of the ALLOCATE clause must be a positive integer expression
-    !$omp allocators allocate(-1: a)
-        allocate(a)
-
-    !ERROR: The parameter of the ALLOCATE clause must be a positive integer expression
+    !ERROR: The alignment value should be a constant positive integer
     !$omp allocators allocate(allocator(-2), align(-3): b)
         allocate(b)
 
-    !ERROR: The parameter of the ALLOCATE clause must be a positive integer expression
+    !ERROR: The alignment value should be a constant positive integer
     !$omp allocators allocate(align(-4): c)
         allocate(c)
 end subroutine
diff --git a/flang/test/Semantics/OpenMP/allocators01.f90 b/flang/test/Semantics/OpenMP/allocators01.f90
index c75c522ecae10..ff92fa3b23463 100644
--- a/flang/test/Semantics/OpenMP/allocators01.f90
+++ b/flang/test/Semantics/OpenMP/allocators01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! OpenMP Version 5.2
 ! 6.7 allocators construct
 ! A list item that appears in an allocate clause must appear as
diff --git a/flang/test/Semantics/OpenMP/allocators04.f90 b/flang/test/Semantics/OpenMP/allocators04.f90
index 1d2e96443a9da..212e48fbd1b26 100644
--- a/flang/test/Semantics/OpenMP/allocators04.f90
+++ b/flang/test/Semantics/OpenMP/allocators04.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate Directive
 ! If list items within the ALLOCATE directive have the SAVE attribute, are a common block name, or are declared in the scope of a
diff --git a/flang/test/Semantics/OpenMP/allocators05.f90 b/flang/test/Semantics/OpenMP/allocators05.f90
index d0e11ca5874d7..0e8366a2461e6 100644
--- a/flang/test/Semantics/OpenMP/allocators05.f90
+++ b/flang/test/Semantics/OpenMP/allocators05.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate directive
 ! allocate directives that appear in a target region must specify an
diff --git a/flang/test/Semantics/OpenMP/allocators06.f90 b/flang/test/Semantics/OpenMP/allocators06.f90
index a975204c11339..8e63512369e38 100644
--- a/flang/test/Semantics/OpenMP/allocators06.f90
+++ b/flang/test/Semantics/OpenMP/allocators06.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate directive
 ! The allocate directive must appear in the same scope as the declarations of
diff --git a/flang/test/Semantics/OpenMP/atomic-compare.f90 b/flang/test/Semantics/OpenMP/atomic-compare.f90
new file mode 100644
index 0000000000000..85644ad909107
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/atomic-compare.f90
@@ -0,0 +1,79 @@
+! REQUIRES: openmp_runtime
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=51
+  use omp_lib
+  implicit none
+  ! Check atomic compare. This combines elements from multiple other "atomic*.f90", as
+  ! to avoid having several files with just a few lines in them. atomic compare needs
+  ! higher openmp version than the others, so need a separate file.
+  
+
+  real a, b, c
+  a = 1.0
+  b = 2.0
+  c = 3.0
+  !$omp parallel num_threads(4)
+  ! First a few things that should compile without error.
+  !$omp atomic seq_cst, compare
+  if (b .eq. a) then
+     b = c
+  end if
+
+  !$omp atomic seq_cst compare
+  if (a .eq. b) a = c
+  !$omp end atomic
+
+  !$omp atomic compare acquire hint(OMP_LOCK_HINT_CONTENDED)
+  if (b .eq. a) b = c
+
+  !$omp atomic release hint(OMP_LOCK_HINT_UNCONTENDED) compare
+  if (b .eq. a) b = c
+
+  !$omp atomic compare seq_cst
+  if (b .eq. c) b = a
+
+  !$omp atomic hint(1) acq_rel compare
+  if (b .eq. a) b = c
+  !$omp end atomic
+
+  ! Check for error conditions:
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !$omp atomic seq_cst seq_cst compare
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !$omp atomic compare seq_cst seq_cst
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !$omp atomic seq_cst compare seq_cst
+  if (b .eq. c) b = a
+
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !$omp atomic acquire acquire compare
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !$omp atomic compare acquire acquire
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !$omp atomic acquire compare acquire
+  if (b .eq. c) b = a
+
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !$omp atomic relaxed relaxed compare
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !$omp atomic compare relaxed relaxed
+  if (b .eq. c) b = a
+  !ERROR: More than one memory order clause not allowed on OpenMP Atomic construct
+  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !$omp atomic relaxed compare relaxed
+  if (b .eq. c) b = a
+
+  !$omp end parallel
+end
diff --git a/flang/test/Semantics/OpenMP/atomic.f90 b/flang/test/Semantics/OpenMP/atomic.f90
index 44f06b7460bf1..0e100871ea9b4 100644
--- a/flang/test/Semantics/OpenMP/atomic.f90
+++ b/flang/test/Semantics/OpenMP/atomic.f90
@@ -35,6 +35,7 @@
   a = a + 1
   !ERROR: expected 'UPDATE'
   !ERROR: expected 'WRITE'
+  !ERROR: expected 'COMPARE'
   !ERROR: expected 'CAPTURE'
   !ERROR: expected 'READ'
   !$omp atomic num_threads(4)
@@ -49,6 +50,7 @@
 
   !ERROR: expected 'UPDATE'
   !ERROR: expected 'WRITE'
+  !ERROR: expected 'COMPARE'
   !ERROR: expected 'CAPTURE'
   !ERROR: expected 'READ'
   !$omp atomic num_threads write
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 8dd6d10200cd3..66e11e4b540f0 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -241,7 +241,7 @@
   enddo
   !$omp end parallel do simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL DO directive
   !$omp parallel do if(target:a>1.)
   do i = 1, N
   enddo
@@ -376,7 +376,7 @@
      a = 3.14
   enddo
 
-  !ERROR: The parameter of the ALIGNED clause must be a constant positive integer expression
+  !ERROR: The alignment value should be a constant positive integer
   !$omp simd aligned(cpt:-2)
   do i = 1, N
      a = 3.14
@@ -532,7 +532,7 @@
   a = 1.
   !$omp end task
 
-  !ERROR: Unmatched directive name modifier TASKLOOP on the IF clause
+  !ERROR: TASKLOOP is not a constituent of the TASK directive
   !$omp task private(a) if(taskloop:a.eq.1)
   a = 1.
   !$omp end task
@@ -572,7 +572,7 @@
 
   allocate(allc)
   !ERROR: The parameter of the SIMDLEN clause must be a constant positive integer expression
-  !ERROR: The parameter of the ALIGNED clause must be a constant positive integer expression
+  !ERROR: The alignment value should be a constant positive integer
   !$omp taskloop simd simdlen(-1) aligned(allc:-2)
   do i = 1, N
      allc = 3.14
diff --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
new file mode 100644
index 0000000000000..dd972e6e52949
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -0,0 +1,17 @@
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
+! Test that we do not make a private copy of the critical name
+
+!CHECK:  MainProgram scope: mn
+!CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
+!CHECK-NEXT:    OtherConstruct scope:
+!CHECK-NEXT:      j (OmpPrivate): HostAssoc
+!CHECK-NEXT:      k2 (OmpCriticalLock): Unknown
+program mn
+  integer :: j
+  j=2
+  !$omp parallel default(private)
+    !$omp critical(k2)
+    j=200
+    !$omp end critical(k2)
+  !$omp end parallel
+end 
diff --git a/flang/test/Semantics/OpenMP/declarative-directive01.f90 b/flang/test/Semantics/OpenMP/declarative-directive01.f90
index 8d6762b87adb9..17dc50b70e542 100644
--- a/flang/test/Semantics/OpenMP/declarative-directive01.f90
+++ b/flang/test/Semantics/OpenMP/declarative-directive01.f90
@@ -23,6 +23,7 @@ end subroutine requires_2
 
 subroutine declare_simd_1(a, b)
   real(8), intent(inout) :: a, b
+  !ERROR: 'a' in ALIGNED clause must be of type C_PTR, POINTER or ALLOCATABLE
   !$omp declare simd(declare_simd_1) aligned(a)
   a = 3.14 + b
 end subroutine declare_simd_1
diff --git a/flang/test/Semantics/OpenMP/depend05.f90 b/flang/test/Semantics/OpenMP/depend05.f90
index 53fd82bd08a9e..3ca091ef3187d 100644
--- a/flang/test/Semantics/OpenMP/depend05.f90
+++ b/flang/test/Semantics/OpenMP/depend05.f90
@@ -2,7 +2,7 @@
 
 subroutine f00(x)
   integer :: x(10)
-!WARNING: Iterator modifiers are not supported in OpenMP v4.5, try -fopenmp-version=50
+!WARNING: 'iterator' modifier is not supported in OpenMP v4.5, try -fopenmp-version=50
   !$omp task depend(iterator(i = 1:10), in: x(i))
   x = 0
   !$omp end task
diff --git a/flang/test/Semantics/OpenMP/do21.f90 b/flang/test/Semantics/OpenMP/do21.f90
new file mode 100644
index 0000000000000..2f5815c10c11a
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/do21.f90
@@ -0,0 +1,27 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! Check for existence of loop following a DO directive
+
+subroutine do1
+  !ERROR: A DO loop must follow the DO directive
+  !$omp do
+end subroutine
+
+subroutine do2
+  !ERROR: A DO loop must follow the PARALLEL DO directive
+  !$omp parallel do
+end subroutine
+
+subroutine do3
+  !ERROR: A DO loop must follow the SIMD directive
+  !$omp simd
+end subroutine
+
+subroutine do4
+  !ERROR: A DO loop must follow the DO SIMD directive
+  !$omp do simd
+end subroutine
+
+subroutine do5
+  !ERROR: A DO loop must follow the LOOP directive
+  !$omp loop
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/from-clause-v45.f90 b/flang/test/Semantics/OpenMP/from-clause-v45.f90
index 98dff295c879d..201128fc3031c 100644
--- a/flang/test/Semantics/OpenMP/from-clause-v45.f90
+++ b/flang/test/Semantics/OpenMP/from-clause-v45.f90
@@ -2,7 +2,7 @@
 
 subroutine f00(x)
   integer :: x(10)
-!ERROR: Reference to x must be a contiguous object
+!ERROR: Reference to 'x' must be a contiguous object
   !$omp target update from(x(1:10:2))
 end
 
diff --git a/flang/test/Semantics/OpenMP/if-clause.f90 b/flang/test/Semantics/OpenMP/if-clause.f90
index 7aeb617e53563..23be4a751c892 100644
--- a/flang/test/Semantics/OpenMP/if-clause.f90
+++ b/flang/test/Semantics/OpenMP/if-clause.f90
@@ -18,7 +18,7 @@ program main
   end do
   !$omp end distribute parallel do
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the DISTRIBUTE PARALLEL DO directive
   !$omp distribute parallel do if(target: .true.)
   do i = 1, 10
   end do
@@ -45,7 +45,7 @@ program main
   end do
   !$omp end distribute parallel do simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the DISTRIBUTE PARALLEL DO SIMD directive
   !$omp distribute parallel do simd if(target: .true.)
   do i = 1, 10
   end do
@@ -66,7 +66,7 @@ program main
   end do
   !$omp end distribute simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the DISTRIBUTE SIMD directive
   !$omp distribute simd if(target: .true.)
   do i = 1, 10
   end do
@@ -92,7 +92,7 @@ program main
   end do
   !$omp end do simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the DO SIMD directive
   !$omp do simd if(target: .true.)
   do i = 1, 10
   end do
@@ -113,7 +113,7 @@ program main
   !$omp parallel if(parallel: .true.)
   !$omp end parallel
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL directive
   !$omp parallel if(target: .true.)
   !$omp end parallel
 
@@ -134,7 +134,7 @@ program main
   end do
   !$omp end parallel do
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL DO directive
   !$omp parallel do if(target: .true.)
   do i = 1, 10
   end do
@@ -159,7 +159,7 @@ program main
   end do
   !$omp end parallel do simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL DO SIMD directive
   !$omp parallel do simd if(target: .true.)
   do i = 1, 10
   end do
@@ -174,7 +174,7 @@ program main
   !$omp parallel sections if(parallel: .true.)
   !$omp end parallel sections
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL SECTIONS directive
   !$omp parallel sections if(target: .true.)
   !$omp end parallel sections
 
@@ -191,7 +191,7 @@ program main
   !$omp parallel workshare if(parallel: .true.)
   !$omp end parallel workshare
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the PARALLEL WORKSHARE directive
   !$omp parallel workshare if(target: .true.)
   !$omp end parallel workshare
 
@@ -212,7 +212,7 @@ program main
   end do
   !$omp end simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the SIMD directive
   !$omp simd if(target: .true.)
   do i = 1, 10
   end do
@@ -233,7 +233,7 @@ program main
   !$omp target if(target: .true.)
   !$omp end target
 
-  !ERROR: Unmatched directive name modifier PARALLEL on the IF clause
+  !ERROR: PARALLEL is not a constituent of the TARGET directive
   !$omp target if(parallel: .true.)
   !$omp end target
 
@@ -250,7 +250,7 @@ program main
   !$omp target data map(tofrom: i) if(target data: .true.)
   !$omp end target data
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TARGET DATA directive
   !$omp target data map(tofrom: i) if(target: .true.)
   !$omp end target data
 
@@ -265,7 +265,7 @@ program main
 
   !$omp target enter data map(to: i) if(target enter data: .true.)
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TARGET ENTER DATA directive
   !$omp target enter data map(to: i) if(target: .true.)
 
   !ERROR: At most one IF clause can appear on the TARGET ENTER DATA directive
@@ -278,7 +278,7 @@ program main
 
   !$omp target exit data map(from: i) if(target exit data: .true.)
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TARGET EXIT DATA directive
   !$omp target exit data map(from: i) if(target: .true.)
   
   !ERROR: At most one IF clause can appear on the TARGET EXIT DATA directive
@@ -293,7 +293,7 @@ program main
   !$omp target parallel if(target: .true.) if(parallel: .false.)
   !$omp end target parallel
 
-  !ERROR: Unmatched directive name modifier SIMD on the IF clause
+  !ERROR: SIMD is not a constituent of the TARGET PARALLEL directive
   !$omp target parallel if(simd: .true.)
   !$omp end target parallel
 
@@ -310,7 +310,7 @@ program main
   end do
   !$omp end target parallel do
 
-  !ERROR: Unmatched directive name modifier SIMD on the IF clause
+  !ERROR: SIMD is not a constituent of the TARGET PARALLEL DO directive
   !$omp target parallel do if(simd: .true.)
   do i = 1, 10
   end do
@@ -330,7 +330,7 @@ program main
   end do
   !$omp end target parallel do simd
 
-  !ERROR: Unmatched directive name modifier TEAMS on the IF clause
+  !ERROR: TEAMS is not a constituent of the TARGET PARALLEL DO SIMD directive
   !$omp target parallel do simd if(teams: .true.)
   do i = 1, 10
   end do
@@ -349,7 +349,7 @@ program main
   end do
   !$omp end target simd
 
-  !ERROR: Unmatched directive name modifier PARALLEL on the IF clause
+  !ERROR: PARALLEL is not a constituent of the TARGET SIMD directive
   !$omp target simd if(parallel: .true.)
   do i = 1, 10
   end do
@@ -364,7 +364,7 @@ program main
   !$omp target teams if(target: .true.) if(teams: .false.)
   !$omp end target teams
 
-  !ERROR: Unmatched directive name modifier PARALLEL on the IF clause
+  !ERROR: PARALLEL is not a constituent of the TARGET TEAMS directive
   !$omp target teams if(parallel: .true.)
   !$omp end target teams
 
@@ -381,7 +381,7 @@ program main
   end do
   !$omp end target teams distribute
 
-  !ERROR: Unmatched directive name modifier PARALLEL on the IF clause
+  !ERROR: PARALLEL is not a constituent of the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute if(parallel: .true.)
   do i = 1, 10
   end do
@@ -401,7 +401,7 @@ program main
   end do
   !$omp end target teams distribute parallel do
 
-  !ERROR: Unmatched directive name modifier SIMD on the IF clause
+  !ERROR: SIMD is not a constituent of the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do if(simd: .true.)
   do i = 1, 10
   end do
@@ -422,7 +422,7 @@ program main
   end do
   !$omp end target teams distribute parallel do simd
 
-  !ERROR: Unmatched directive name modifier TASK on the IF clause
+  !ERROR: TASK is not a constituent of the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd if(task: .true.)
   do i = 1, 10
   end do
@@ -442,7 +442,7 @@ program main
   end do
   !$omp end target teams distribute simd
 
-  !ERROR: Unmatched directive name modifier PARALLEL on the IF clause
+  !ERROR: PARALLEL is not a constituent of the TARGET TEAMS DISTRIBUTE SIMD directive
   !$omp target teams distribute simd if(parallel: .true.)
   do i = 1, 10
   end do
@@ -455,7 +455,7 @@ program main
   
   !$omp target update to(i) if(target update: .true.)
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TARGET UPDATE directive
   !$omp target update to(i) if(target: .true.)
 
   !ERROR: At most one IF clause can appear on the TARGET UPDATE directive
@@ -470,7 +470,7 @@ program main
   !$omp task if(task: .true.)
   !$omp end task
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TASK directive
   !$omp task if(target: .true.)
   !$omp end task
 
@@ -491,7 +491,7 @@ program main
   end do
   !$omp end taskloop
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TASKLOOP directive
   !$omp taskloop if(target: .true.)
   do i = 1, 10
   end do
@@ -516,7 +516,7 @@ program main
   end do
   !$omp end taskloop simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TASKLOOP SIMD directive
   !$omp taskloop simd if(target: .true.)
   do i = 1, 10
   end do
@@ -531,7 +531,7 @@ program main
   !$omp teams if(teams: .true.)
   !$omp end teams
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TEAMS directive
   !$omp teams if(target: .true.)
   !$omp end teams
 
@@ -552,7 +552,7 @@ program main
   end do
   !$omp end teams distribute
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TEAMS DISTRIBUTE directive
   !$omp teams distribute if(target: .true.)
   do i = 1, 10
   end do
@@ -577,7 +577,7 @@ program main
   end do
   !$omp end teams distribute parallel do
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp teams distribute parallel do if(target: .true.)
   do i = 1, 10
   end do
@@ -597,7 +597,7 @@ program main
   end do
   !$omp end teams distribute parallel do simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp teams distribute parallel do simd if(target: .true.)
   do i = 1, 10
   end do
@@ -616,7 +616,7 @@ program main
   end do
   !$omp end teams distribute simd
 
-  !ERROR: Unmatched directive name modifier TARGET on the IF clause
+  !ERROR: TARGET is not a constituent of the TEAMS DISTRIBUTE SIMD directive
   !$omp teams distribute simd if(target: .true.)
   do i = 1, 10
   end do
diff --git a/flang/test/Semantics/OpenMP/linear-clause01.f90 b/flang/test/Semantics/OpenMP/linear-clause01.f90
new file mode 100644
index 0000000000000..654aa07f5bd40
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/linear-clause01.f90
@@ -0,0 +1,45 @@
+! REQUIRES: openmp_runtime
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! OpenMP Version 5.2
+! Various checks for the linear clause
+! 5.4.6 `linear` Clause
+
+! Case 1
+subroutine linear_clause_01(arg)
+    integer, intent(in) :: arg(:)
+    !ERROR: A modifier may not be specified in a LINEAR clause on the DO directive
+    !$omp do linear(uval(arg))
+    do i = 1, 5
+        print *, arg(i)
+    end do
+end subroutine linear_clause_01
+
+! Case 2
+subroutine linear_clause_02(arg_01, arg_02)
+    !ERROR: The list item `arg_01` specified with other than linear-modifier `REF` must be of type INTEGER
+    !$omp declare simd linear(val(arg_01))
+    real, intent(in) :: arg_01(:)
+
+    !ERROR: The list item `arg_02` specified with the linear-modifier `REF` or `UVAL` must be a dummy argument without `VALUE` attribute
+    !$omp declare simd linear(uval(arg_02))
+    integer, value, intent(in) :: arg_02
+
+    !ERROR: The list item `var` must be a dummy argument
+    !ERROR: The list item `var` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute
+    !$omp declare simd linear(uval(var))
+    integer, pointer :: var
+end subroutine linear_clause_02
+
+! Case 3
+subroutine linear_clause_03(arg)
+    integer, intent(in) :: arg
+    !ERROR: The list item `arg` specified with the linear-modifier `REF` must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute
+    !ERROR: List item 'arg' present at multiple LINEAR clauses
+    !$omp declare simd linear(ref(arg)) linear(arg)
+
+    integer :: i
+    common /cc/ i
+    !ERROR: The list item `i` must be a dummy argument
+    !ERROR: 'i' is a common block name and must not appear in an LINEAR clause
+    !$omp declare simd linear(i)
+end subroutine linear_clause_03
diff --git a/flang/test/Semantics/OpenMP/reduction04.f90 b/flang/test/Semantics/OpenMP/reduction04.f90
index 319ed9f245abe..e86b67b6efa96 100644
--- a/flang/test/Semantics/OpenMP/reduction04.f90
+++ b/flang/test/Semantics/OpenMP/reduction04.f90
@@ -14,8 +14,7 @@ program omp_Reduction
   end do
   !$omp end parallel do
 
-  !ERROR: Variable 'c' on the REDUCTION clause is not definable
-  !BECAUSE: 'c' is not a variable
+  !ERROR: Common block names are not allowed in REDUCTION clause
   !$omp parallel do reduction(*:/c/)
   do i = 1, 10
     l = k + 1
diff --git a/flang/test/Semantics/OpenMP/reduction06.f90 b/flang/test/Semantics/OpenMP/reduction06.f90
index 58290c61cae86..66aa9fc62c1bf 100644
--- a/flang/test/Semantics/OpenMP/reduction06.f90
+++ b/flang/test/Semantics/OpenMP/reduction06.f90
@@ -8,21 +8,21 @@ program omp_reduction
   integer :: k = 10
   integer :: a(10), b(10,10,10)
 
-  !ERROR: A list item that appears in a REDUCTION clause should have a contiguous storage array section.
+  !ERROR: Reference to 'a' must be a contiguous object
   !$omp parallel do reduction(+:a(1:10:3))
   do i = 1, 10
     k = k + 1
   end do
   !$omp end parallel do
 
-  !ERROR: A list item that appears in a REDUCTION clause should have a contiguous storage array section.
+  !ERROR: Reference to 'b' must be a contiguous object
   !$omp parallel do reduction(+:b(1:10:3,1:8:1,1:5:1))
   do i = 1, 10
     k = k + 1
   end do
   !$omp end parallel do
 
-  !ERROR: A list item that appears in a REDUCTION clause should have a contiguous storage array section.
+  !ERROR: Reference to 'b' must be a contiguous object
   !$omp parallel do reduction(+:b(1:10:1,1:8:2,1:5:1))
   do i = 1, 10
     k = k + 1
diff --git a/flang/test/Semantics/OpenMP/reduction12.f90 b/flang/test/Semantics/OpenMP/reduction12.f90
index f896ca4aa60b6..f7e3baf046562 100644
--- a/flang/test/Semantics/OpenMP/reduction12.f90
+++ b/flang/test/Semantics/OpenMP/reduction12.f90
@@ -6,7 +6,7 @@
   procedure(foo), pointer :: ptr
   integer :: i
   ptr => foo
-!ERROR: A procedure pointer 'ptr' must not appear in a REDUCTION clause.
+!ERROR: Procedure pointer 'ptr' may not appear in a REDUCTION clause
 !$omp do reduction (+ : ptr)
   do i = 1, 10
   end do
diff --git a/flang/test/Semantics/OpenMP/reduction15.f90 b/flang/test/Semantics/OpenMP/reduction15.f90
new file mode 100644
index 0000000000000..1d4de6ff702bb
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/reduction15.f90
@@ -0,0 +1,32 @@
+!RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=50
+
+!Ref: [5.0:297:28-29]
+! If a list item is an array section or an array element, its base expression
+! must be a base language identifier.
+
+module m
+  type t
+    integer :: a(10)
+  end type
+
+contains
+
+  subroutine f00
+    type(t) :: x
+  !ERROR: The base expression of an array element or section in REDUCTION clause must be an identifier
+  !$omp do reduction (+ : x%a(2))
+    do i = 1, 10
+    end do
+  !$omp end do
+  end subroutine
+
+  subroutine f01
+    type(t) :: x
+  !ERROR: The base expression of an array element or section in REDUCTION clause must be an identifier
+  !$omp do reduction (+ : x%a(1:10))
+    do i = 1, 10
+    end do
+  !$omp end do
+  end subroutine
+end
+
diff --git a/flang/test/Semantics/OpenMP/reduction16.f90 b/flang/test/Semantics/OpenMP/reduction16.f90
new file mode 100644
index 0000000000000..6bb218e306128
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/reduction16.f90
@@ -0,0 +1,14 @@
+!RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=50
+
+!Ref: [5.0:298:19]
+! A type parameter inquiry cannot appear in a reduction clause.
+
+subroutine f00
+  integer :: x
+!ERROR: Type parameter inquiry is not permitted in REDUCTION clause
+!$omp do reduction (+ : x%kind)
+  do i = 1, 10
+  end do
+!$omp end do
+end subroutine
+
diff --git a/flang/test/Semantics/OpenMP/resolve06.f90 b/flang/test/Semantics/OpenMP/resolve06.f90
index 358b1b1cc2826..cf544dfb73cad 100644
--- a/flang/test/Semantics/OpenMP/resolve06.f90
+++ b/flang/test/Semantics/OpenMP/resolve06.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 use omp_lib
 !2.11.4 Allocate Clause
 !For any list item that is specified in the allocate
diff --git a/flang/test/Semantics/OpenMP/to-clause-v45.f90 b/flang/test/Semantics/OpenMP/to-clause-v45.f90
index e4d8967ca14df..9797c06cdeddf 100644
--- a/flang/test/Semantics/OpenMP/to-clause-v45.f90
+++ b/flang/test/Semantics/OpenMP/to-clause-v45.f90
@@ -2,7 +2,7 @@
 
 subroutine f00(x)
   integer :: x(10)
-!ERROR: Reference to x must be a contiguous object
+!ERROR: Reference to 'x' must be a contiguous object
   !$omp target update to(x(1:10:2))
 end
 
diff --git a/flang/test/Semantics/OpenMP/use_device_addr1.f90 b/flang/test/Semantics/OpenMP/use_device_addr1.f90
index e6a3e6e5b2a2d..350d236e00598 100644
--- a/flang/test/Semantics/OpenMP/use_device_addr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_addr1.f90
@@ -15,7 +15,7 @@ subroutine omp_target_data
    type(my_type) :: my_var
    a = 1
 
-   !ERROR: A variable that is part of another variable (structure element) cannot appear on the TARGET DATA USE_DEVICE_ADDR clause
+   !ERROR: A variable that is part of another variable cannot appear on the USE_DEVICE_ADDR clause
    !$omp target data map(tofrom: a) use_device_addr(my_var%my_b)
       my_var%my_b = a
    !$omp end target data
diff --git a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
index 41dbadc59ce7c..5a5437e618450 100644
--- a/flang/test/Semantics/OpenMP/use_device_ptr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
@@ -20,7 +20,7 @@ subroutine omp_target_data
    type(my_type) :: my_var
    a = 1
 
-   !ERROR: A variable that is part of another variable (structure element) cannot appear on the TARGET DATA USE_DEVICE_PTR clause
+   !ERROR: A variable that is part of another variable cannot appear on the USE_DEVICE_PTR clause
    !$omp target data map(tofrom: a, arrayB) use_device_ptr(my_var%my_cptr)
       allocate(arrayB)
       call c_f_pointer(my_var%my_cptr, arrayB)
diff --git a/flang/test/Semantics/call14.f90 b/flang/test/Semantics/call14.f90
index e586d4eebd253..fba11d35790b2 100644
--- a/flang/test/Semantics/call14.f90
+++ b/flang/test/Semantics/call14.f90
@@ -9,7 +9,7 @@ module m
   !ERROR: VALUE attribute may apply only to a dummy data object
   subroutine C863(notData,assumedSize,coarray,coarrayComponent,assumedRank,assumedLen)
     external :: notData
-    !ERROR: Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute
+    !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
     real, value :: notADummy
     value :: notData
     !ERROR: VALUE attribute may not apply to an assumed-size array
diff --git a/flang/test/Semantics/expr-errors06.f90 b/flang/test/Semantics/expr-errors06.f90
index bdcb92cbe5e4d..07cda25e5cb0c 100644
--- a/flang/test/Semantics/expr-errors06.f90
+++ b/flang/test/Semantics/expr-errors06.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic -Werror
 ! Check out-of-range subscripts
 subroutine subr(da)
   real a(10), da(2,1), empty(1:0,1)
@@ -43,6 +43,6 @@ subroutine subr(da)
   print *, empty(1:0,1) ! ok
   print *, empty(:,1) ! ok
   print *, empty(i:j,k) ! ok
-  !ERROR: Empty array dimension 1 cannot be subscripted as an element or non-empty array section
+  !WARNING: Empty array dimension 1 should not be subscripted as an element or non-empty array section
   print *, empty(i,1)
 end
diff --git a/flang/test/Semantics/resolve58.f90 b/flang/test/Semantics/resolve58.f90
index 2e42eb157f5b5..7686de4898404 100644
--- a/flang/test/Semantics/resolve58.f90
+++ b/flang/test/Semantics/resolve58.f90
@@ -69,12 +69,12 @@ subroutine s6()
 
   !ERROR: Implied-shape array 'local1' must be a named constant or a dummy argument
   real, dimension (*) :: local1
-  !ERROR: Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
   real, intent(in) :: local2
-  !ERROR: Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
   procedure(), intent(in) :: p1
-  !ERROR: Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
   real, optional :: local3
-  !ERROR: Only a dummy argument may have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
   procedure(), optional :: p2
 end subroutine
diff --git a/flang/test/Semantics/test_symbols.py b/flang/test/Semantics/test_symbols.py
index 24dc5004a4229..612ae0df94e23 100755
--- a/flang/test/Semantics/test_symbols.py
+++ b/flang/test/Semantics/test_symbols.py
@@ -29,7 +29,7 @@
 # Strips out blank lines and all comments except for "!DEF:", "!REF:", "!$acc" and "!$omp"
 with open(src, "r") as text_in:
     for line in text_in:
-        text = re.sub(r"!(?![DR]EF:|\$omp|\$acc).*", "", line)
+        text = re.sub(r"!(?![DR]EF:|\$omp|\$acc).*", "", line, flags=re.I)
         text = re.sub(r"^\s*$", "", text)
         diff1 += text
 
diff --git a/flang/test/Transforms/OpenMP/lower-workshare-alloca.mlir b/flang/test/Transforms/OpenMP/lower-workshare-alloca.mlir
index 12b0558d06ed5..82daa142042a3 100644
--- a/flang/test/Transforms/OpenMP/lower-workshare-alloca.mlir
+++ b/flang/test/Transforms/OpenMP/lower-workshare-alloca.mlir
@@ -22,7 +22,8 @@ func.func @wsfunc() {
 
 // CHECK-LABEL:   func.func private @_workshare_copy_i32(
 // CHECK-SAME:                                           %[[VAL_0:.*]]: !fir.ref<i32>,
-// CHECK-SAME:                                           %[[VAL_1:.*]]: !fir.ref<i32>) {
+// CHECK-SAME:                                           %[[VAL_1:.*]]: !fir.ref<i32>)
+// CHECK-SAME:                   attributes {llvm.linkage = #llvm.linkage<internal>} {
 // CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
 // CHECK:           fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<i32>
 // CHECK:           return
diff --git a/flang/test/Transforms/constant-argument-globalisation.fir b/flang/test/Transforms/constant-argument-globalisation.fir
index 553c3c6c24845..5656de0f601e7 100644
--- a/flang/test/Transforms/constant-argument-globalisation.fir
+++ b/flang/test/Transforms/constant-argument-globalisation.fir
@@ -49,8 +49,8 @@ module {
 // DISABLE-LABEL: ; ModuleID =
 // DISABLE-NOT: @_extruded
 // DISABLE:  define void @sub1(
-// DISABLE-SAME: ptr [[ARG0:%.*]],
-// DISABLE-SAME: ptr [[ARG1:%.*]])
+// DISABLE-SAME: ptr nocapture [[ARG0:%.*]],
+// DISABLE-SAME: ptr nocapture [[ARG1:%.*]])
 // DISABLE-SAME: {
 // DISABLE: [[CONST_R0:%.*]] = alloca double
 // DISABLE: [[CONST_R1:%.*]] = alloca double
diff --git a/flang/test/Transforms/function-attrs.fir b/flang/test/Transforms/function-attrs.fir
new file mode 100644
index 0000000000000..5f871a1a7b6c5
--- /dev/null
+++ b/flang/test/Transforms/function-attrs.fir
@@ -0,0 +1,45 @@
+// RUN: fir-opt --function-attr %s | FileCheck %s
+
+// If a function has a body and is not bind(c), and if the dummy argument doesn't have the target,
+// asynchronous, volatile, or pointer attribute, then add llvm.nocapture to the dummy argument.
+
+func.func @_QParg_nocapture(%arg0: !fir.ref<i32> {fir.bindc_name = "tar", fir.target}, %arg1: !fir.ref<i32> {fir.asynchronous, fir.bindc_name = "asynch"}, %arg2: !fir.ref<i32> {fir.bindc_name = "vol", fir.volatile}, %arg3: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "ptr"}, %arg4: !fir.ref<i32> {fir.bindc_name = "nocap"}) {
+    %0 = fir.dummy_scope : !fir.dscope
+    %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFarg_nocaptureEtar"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    %2 = fir.declare %arg1 dummy_scope %0 {fortran_attrs = #fir.var_attrs<asynchronous>, uniq_name = "_QFarg_nocaptureEasynch"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    %3 = fir.declare %arg2 dummy_scope %0 {uniq_name = "_QFarg_nocaptureEvol"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    %4 = fir.declare %arg3 dummy_scope %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFarg_nocaptureEptr"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> !fir.ref<!fir.box<!fir.ptr<i32>>>
+    %5 = fir.declare %arg4 dummy_scope %0 {uniq_name = "_QFarg_nocaptureEnocap"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    return
+}
+// CHECK-LABEL: func.func @_QParg_nocapture(
+// CHECK-SAME:                                 %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "tar", fir.target},
+// CHECK-SAME:                                 %[[ARG1:.*]]: !fir.ref<i32> {fir.asynchronous, fir.bindc_name = "asynch"},
+// CHECK-SAME:                                 %[[ARG2:.*]]: !fir.ref<i32> {fir.bindc_name = "vol", fir.volatile},
+// CHECK-SAME:                                 %[[ARG3:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "ptr", llvm.nocapture},
+// CHECK-SAME:                                 %[[ARG4:.*]]: !fir.ref<i32> {fir.bindc_name = "nocap", llvm.nocapture}) {
+// CHECK:    return
+// CHECK-NEXT: }
+
+func.func @arg_nocapture_bindc(%arg0: !fir.ref<i32> {fir.bindc_name = "tar", fir.target}, %arg1: !fir.ref<i32> {fir.bindc_name = "nocap"}) attributes {fir.bindc_name = "arg_nocapture_bindc", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
+    %0 = fir.dummy_scope : !fir.dscope
+    %1 = fir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFarg_nocapture_bindcEtar"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    %2 = fir.declare %arg1 dummy_scope %0 {uniq_name = "_QFarg_nocapture_bindcEnocap"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    return
+}
+// CHECK-LABEL: func.func @arg_nocapture_bindc(
+// CHECK-NOT:  llvm.nocapture
+
+
+// If a function declaration is from a module and is not bind(c), and if the dummy argument doesn't have
+// the target, asynchronous, volatile, or pointer attribute, then add llvm.nocapture to the dummy argument.
+
+func.func private @_QMarg_modPcheck_args(!fir.ref<i32> {fir.target}, !fir.ref<i32> {fir.asynchronous}, !fir.ref<i32> {fir.volatile}, !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<i32>, !fir.boxchar<1>, !fir.ref<complex<f32>>)
+// CHECK-LABEL: func.func private @_QMarg_modPcheck_args(
+// CHECK-SAME:                                 !fir.ref<i32> {fir.target},
+// CHECK-SAME:                                 !fir.ref<i32> {fir.asynchronous},
+// CHECK-SAME:                                 !fir.ref<i32> {fir.volatile},
+// CHECK-SAME:                                 !fir.ref<!fir.box<!fir.ptr<i32>>> {llvm.nocapture},
+// CHECK-SAME:                                 !fir.ref<i32> {llvm.nocapture},
+// CHECK-SAME:                                 !fir.boxchar<1>,
+// CHECK-SAME:                                 !fir.ref<complex<f32>> {llvm.nocapture})
diff --git a/flang/test/Transforms/generic-loop-rewriting-todo.mlir b/flang/test/Transforms/generic-loop-rewriting-todo.mlir
new file mode 100644
index 0000000000000..9ea6bf001b668
--- /dev/null
+++ b/flang/test/Transforms/generic-loop-rewriting-todo.mlir
@@ -0,0 +1,84 @@
+// RUN: fir-opt --omp-generic-loop-conversion -verify-diagnostics %s
+
+func.func @_QPtarget_loop() {
+  %c0 = arith.constant 0 : i32
+  %c10 = arith.constant 10 : i32
+  %c1 = arith.constant 1 : i32
+  // expected-error@below {{not yet implemented: Standalone `omp loop` directive}}
+  omp.loop {
+    omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+      omp.yield
+    }
+  }
+  return
+}
+
+func.func @_QPtarget_parallel_loop() {
+  omp.target {
+    omp.parallel {
+      %c0 = arith.constant 0 : i32
+      %c10 = arith.constant 10 : i32
+      %c1 = arith.constant 1 : i32
+      // expected-error@below {{not yet implemented: Combined `omp target parallel loop` directive}}
+      omp.loop {
+        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+func.func @_QPtarget_loop_bind() {
+  omp.target {
+    omp.teams {
+      %c0 = arith.constant 0 : i32
+      %c10 = arith.constant 10 : i32
+      %c1 = arith.constant 1 : i32
+      // expected-error@below {{not yet implemented: Unhandled clause bind in omp.loop operation}}
+      omp.loop bind(thread) {
+        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+  ^bb0(%arg0: i32):
+    %c0_i32 = arith.constant 0 : i32
+    omp.yield(%c0_i32 : i32)
+  } combiner {
+  ^bb0(%arg0: i32, %arg1: i32):
+    %0 = arith.addi %arg0, %arg1 : i32
+    omp.yield(%0 : i32)
+  }
+
+func.func @_QPtarget_loop_order() {
+
+  omp.target {
+    omp.teams {
+      %c0 = arith.constant 0 : i32
+      %c10 = arith.constant 10 : i32
+      %c1 = arith.constant 1 : i32
+      %sum = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_orderEi"}
+
+      // expected-error@below {{not yet implemented: Unhandled clause reduction in omp.loop operation}}
+      omp.loop reduction(@add_reduction_i32 %sum -> %arg2 : !fir.ref<i32>) {
+        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
diff --git a/flang/test/Transforms/generic-loop-rewriting.mlir b/flang/test/Transforms/generic-loop-rewriting.mlir
new file mode 100644
index 0000000000000..a18ea9853602a
--- /dev/null
+++ b/flang/test/Transforms/generic-loop-rewriting.mlir
@@ -0,0 +1,57 @@
+// RUN: fir-opt --omp-generic-loop-conversion %s | FileCheck %s
+
+omp.private {type = private} @_QFtarget_teams_loopEi_private_ref_i32 : !fir.ref<i32> alloc {
+^bb0(%arg0: !fir.ref<i32>):
+  omp.yield(%arg0 : !fir.ref<i32>)
+}
+
+func.func @_QPtarget_teams_loop() {
+  %i = fir.alloca i32
+  %i_map = omp.map.info var_ptr(%i : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = "i"}
+  omp.target map_entries(%i_map -> %arg0 : !fir.ref<i32>) {
+    omp.teams {
+      %c0 = arith.constant 0 : i32
+      %c10 = arith.constant 10 : i32
+      %c1 = arith.constant 1 : i32
+      omp.loop private(@_QFtarget_teams_loopEi_private_ref_i32 %arg0 -> %arg2 : !fir.ref<i32>) {
+        omp.loop_nest (%arg3) : i32 = (%c0) to (%c10) inclusive step (%c1) {
+          fir.store %arg3 to %arg2 : !fir.ref<i32>
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtarget_teams_loop
+// CHECK:         omp.target map_entries(
+// CHECK-SAME:      %{{.*}} -> %[[I_ARG:[^[:space:]]+]] : {{.*}}) {
+// 
+// CHECK:           omp.teams {
+// 
+// TODO we probably need to move the `loop_nest` bounds ops from the `teams`
+// region to the `parallel` region to avoid making these values `shared`. We can
+// find the backward slices of these bounds that are within the `teams` region
+// and move these slices to the `parallel` op.
+
+// CHECK:             %[[LB:.*]] = arith.constant 0 : i32
+// CHECK:             %[[UB:.*]] = arith.constant 10 : i32
+// CHECK:             %[[STEP:.*]] = arith.constant 1 : i32
+// 
+// CHECK:             omp.parallel private(@{{.*}} %[[I_ARG]]
+// CHECK-SAME:          -> %[[I_PRIV_ARG:[^[:space:]]+]] : !fir.ref<i32>) {
+// CHECK:               omp.distribute {
+// CHECK:                 omp.wsloop {
+// 
+// CHECK:                   omp.loop_nest (%{{.*}}) : i32 = 
+// CHECK-SAME:                (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
+// CHECK:                     fir.store %{{.*}} to %[[I_PRIV_ARG]] : !fir.ref<i32>
+// CHECK:                   }
+// CHECK:                 }
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:         }
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 1c24979bbcdaf..02ecd6a75ef7a 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -234,12 +234,6 @@ static llvm::cl::opt<bool> integerWrapAround(
     llvm::cl::desc("Treat signed integer overflow as two's complement"),
     llvm::cl::init(false));
 
-// TODO: integrate this option with the above
-static llvm::cl::opt<bool>
-    setNSW("integer-overflow",
-           llvm::cl::desc("add nsw flag to internal operations"),
-           llvm::cl::init(false));
-
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
@@ -381,7 +375,6 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setNoPPCNativeVecElemOrder(enableNoPPCNativeVecElemOrder);
   loweringOptions.setLowerToHighLevelFIR(useHLFIR || emitHLFIR);
   loweringOptions.setIntegerWrapAround(integerWrapAround);
-  loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;
@@ -470,7 +463,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
     MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
     if (enableOpenMP)
       config.EnableOpenMP = true;
-    config.NSWOnLoopVarInc = setNSW;
+    config.NSWOnLoopVarInc = !integerWrapAround;
     fir::registerDefaultInlinerPass(config);
     fir::createDefaultFIROptimizerPassPipeline(pm, config);
   }
diff --git a/flang/unittests/Evaluate/fp-testing.cpp b/flang/unittests/Evaluate/fp-testing.cpp
index 94d8d5086d000..1a1d7425d5824 100644
--- a/flang/unittests/Evaluate/fp-testing.cpp
+++ b/flang/unittests/Evaluate/fp-testing.cpp
@@ -7,8 +7,8 @@
 #include <xmmintrin.h>
 #endif
 
+using Fortran::common::RealFlag;
 using Fortran::common::RoundingMode;
-using Fortran::evaluate::RealFlag;
 
 ScopedHostFloatingPointEnvironment::ScopedHostFloatingPointEnvironment(
 #if __x86_64__
diff --git a/flang/unittests/Evaluate/fp-testing.h b/flang/unittests/Evaluate/fp-testing.h
index 22dfa2d7d80c6..9091963a99b32 100644
--- a/flang/unittests/Evaluate/fp-testing.h
+++ b/flang/unittests/Evaluate/fp-testing.h
@@ -1,12 +1,12 @@
 #ifndef FORTRAN_TEST_EVALUATE_FP_TESTING_H_
 #define FORTRAN_TEST_EVALUATE_FP_TESTING_H_
 
-#include "flang/Evaluate/target.h"
+#include "flang/Common/target-rounding.h"
 #include <fenv.h>
 
+using Fortran::common::RealFlags;
+using Fortran::common::Rounding;
 using Fortran::common::RoundingMode;
-using Fortran::evaluate::RealFlags;
-using Fortran::evaluate::Rounding;
 
 class ScopedHostFloatingPointEnvironment {
 public:
diff --git a/flang/unittests/Optimizer/Builder/Runtime/AllocatableTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/AllocatableTest.cpp
index 1db43cacc90f0..f618e72d7b7f3 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/AllocatableTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/AllocatableTest.cpp
@@ -9,7 +9,7 @@
 #include "flang/Optimizer/Builder/Runtime/Allocatable.h"
 #include "RuntimeCallTestBase.h"
 #include "gtest/gtest.h"
-#include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/descriptor-consts.h"
 
 using namespace Fortran::runtime;
 
diff --git a/flang/unittests/Runtime/ArrayConstructor.cpp b/flang/unittests/Runtime/ArrayConstructor.cpp
index 9d78da7962361..62e3b780a27e7 100644
--- a/flang/unittests/Runtime/ArrayConstructor.cpp
+++ b/flang/unittests/Runtime/ArrayConstructor.cpp
@@ -43,8 +43,7 @@ TEST(ArrayConstructor, Basic) {
   result.GetDimension(0).SetBounds(1, 0);
 
   RTNAME(InitArrayConstructorVector)
-  (*acVector, result, /*useValueLengthParameters=*/false,
-      /*vectorClassSize=*/sizeof(ArrayConstructorVector));
+  (*acVector, result, /*useValueLengthParameters=*/false);
   for (std::int32_t i{0}; i <= 99; ++i) {
     RTNAME(PushArrayConstructorSimpleScalar)(*acVector, &i);
     RTNAME(PushArrayConstructorValue)(*acVector, *x);
@@ -71,8 +70,7 @@ TEST(ArrayConstructor, Basic) {
   // and is allocated when the first value is pushed.
   result.GetDimension(0).SetBounds(1, 1234);
   RTNAME(InitArrayConstructorVector)
-  (*acVector, result, /*useValueLengthParameters=*/false,
-      /*vectorClassSize=*/sizeof(ArrayConstructorVector));
+  (*acVector, result, /*useValueLengthParameters=*/false);
   EXPECT_EQ(0, acVector->actualAllocationSize);
   std::int32_t i{42};
   RTNAME(PushArrayConstructorSimpleScalar)(*acVector, &i);
@@ -109,8 +107,7 @@ TEST(ArrayConstructor, Character) {
   static constexpr std::size_t expectedElements{10 * (1 + 4 + 2 * 3)};
   result.GetDimension(0).SetBounds(1, 0);
   RTNAME(InitArrayConstructorVector)
-  (*acVector, result, /*useValueLengthParameters=*/true,
-      /*vectorClassSize=*/sizeof(ArrayConstructorVector));
+  (*acVector, result, /*useValueLengthParameters=*/true);
   for (std::int32_t i{1}; i <= 10; ++i) {
     RTNAME(PushArrayConstructorValue)(*acVector, *c);
     RTNAME(PushArrayConstructorValue)(*acVector, *x);
@@ -151,8 +148,7 @@ TEST(ArrayConstructor, CharacterRuntimeCheck) {
 
   result.GetDimension(0).SetBounds(1, 0);
   RTNAME(InitArrayConstructorVector)
-  (*acVector, result, /*useValueLengthParameters=*/true,
-      /*vectorClassSize=*/sizeof(ArrayConstructorVector));
+  (*acVector, result, /*useValueLengthParameters=*/true);
   RTNAME(PushArrayConstructorValue)(*acVector, *c2);
   ASSERT_DEATH(RTNAME(PushArrayConstructorValue)(*acVector, *c3),
       "Array constructor: mismatched character lengths");
diff --git a/flang/unittests/Runtime/CommandTest.cpp b/flang/unittests/Runtime/CommandTest.cpp
index 05287d80e14f5..ecb325330f1ad 100644
--- a/flang/unittests/Runtime/CommandTest.cpp
+++ b/flang/unittests/Runtime/CommandTest.cpp
@@ -352,9 +352,6 @@ TEST_F(ZeroArguments, ECLGeneralErrorCommandErrorSync) {
 #if defined(_WIN32)
   CheckDescriptorEqInt<std::int64_t>(cmdStat.get(), 6);
   CheckDescriptorEqStr(cmdMsg.get(), "Invalid command lineXXXXXXXXX");
-#elif defined(_AIX)
-  CheckDescriptorEqInt<std::int64_t>(cmdStat.get(), 6);
-  CheckDescriptorEqStr(cmdMsg.get(), "Invalid command lineXXXXXXXXX");
 #else
   CheckDescriptorEqInt<std::int64_t>(cmdStat.get(), 3);
   CheckDescriptorEqStr(cmdMsg.get(), "Command line execution failed");
diff --git a/flang/unittests/Runtime/Complex.cpp b/flang/unittests/Runtime/Complex.cpp
index 46f3ad2f2712b..d714da24dc4e5 100644
--- a/flang/unittests/Runtime/Complex.cpp
+++ b/flang/unittests/Runtime/Complex.cpp
@@ -13,7 +13,7 @@
 #pragma clang diagnostic ignored "-Wc99-extensions"
 #endif
 
-#include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-consts.h"
 #include "flang/Runtime/cpp-type.h"
 #include "flang/Runtime/entry-names.h"
 
diff --git a/flang/unittests/Runtime/ExternalIOTest.cpp b/flang/unittests/Runtime/ExternalIOTest.cpp
index 13327964e12a4..b9407b5e7a591 100644
--- a/flang/unittests/Runtime/ExternalIOTest.cpp
+++ b/flang/unittests/Runtime/ExternalIOTest.cpp
@@ -13,7 +13,7 @@
 #include "CrashHandlerFixture.h"
 #include "gtest/gtest.h"
 #include "flang/Runtime/descriptor.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include "flang/Runtime/main.h"
 #include "flang/Runtime/stop.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/flang/unittests/Runtime/ListInputTest.cpp b/flang/unittests/Runtime/ListInputTest.cpp
index a4eba5283add6..38c758b7ef966 100644
--- a/flang/unittests/Runtime/ListInputTest.cpp
+++ b/flang/unittests/Runtime/ListInputTest.cpp
@@ -9,7 +9,7 @@
 #include "CrashHandlerFixture.h"
 #include "../../runtime/io-error.h"
 #include "flang/Runtime/descriptor.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 
 using namespace Fortran::runtime;
 using namespace Fortran::runtime::io;
diff --git a/flang/unittests/Runtime/LogicalFormatTest.cpp b/flang/unittests/Runtime/LogicalFormatTest.cpp
index a2c19d1e1ca94..c4fbfc81f06a4 100644
--- a/flang/unittests/Runtime/LogicalFormatTest.cpp
+++ b/flang/unittests/Runtime/LogicalFormatTest.cpp
@@ -8,7 +8,7 @@
 
 #include "CrashHandlerFixture.h"
 #include "flang/Runtime/descriptor.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include <algorithm>
 #include <array>
 #include <cstring>
diff --git a/flang/unittests/Runtime/Namelist.cpp b/flang/unittests/Runtime/Namelist.cpp
index 9037fa15a97cb..0a28f3590b86e 100644
--- a/flang/unittests/Runtime/Namelist.cpp
+++ b/flang/unittests/Runtime/Namelist.cpp
@@ -10,7 +10,7 @@
 #include "CrashHandlerFixture.h"
 #include "tools.h"
 #include "flang/Runtime/descriptor.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include <algorithm>
 #include <cinttypes>
 #include <complex>
diff --git a/flang/unittests/Runtime/NumericalFormatTest.cpp b/flang/unittests/Runtime/NumericalFormatTest.cpp
index f005515320350..274498b8e8695 100644
--- a/flang/unittests/Runtime/NumericalFormatTest.cpp
+++ b/flang/unittests/Runtime/NumericalFormatTest.cpp
@@ -8,7 +8,7 @@
 
 #include "CrashHandlerFixture.h"
 #include "flang/Runtime/descriptor.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include <algorithm>
 #include <array>
 #include <cstring>
diff --git a/flang/unittests/Runtime/RuntimeCrashTest.cpp b/flang/unittests/Runtime/RuntimeCrashTest.cpp
index a649051fdca0c..72a0b290cf864 100644
--- a/flang/unittests/Runtime/RuntimeCrashTest.cpp
+++ b/flang/unittests/Runtime/RuntimeCrashTest.cpp
@@ -13,7 +13,7 @@
 #include "CrashHandlerFixture.h"
 #include "tools.h"
 #include "../../runtime/terminator.h"
-#include "flang/Runtime/io-api.h"
+#include "flang/Runtime/io-api-consts.h"
 #include "flang/Runtime/transformational.h"
 #include <gtest/gtest.h>
 
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index eee5b63bab513..6d48489284fae 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -7,6 +7,7 @@ if(NOT DEFINED LLVM_COMMON_CMAKE_UTILS)
 endif()
 include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
   NO_POLICY_SCOPE)
+include(CheckCXXCompilerFlag)
 
 if (LIBC_CMAKE_VERBOSE_LOGGING)
   get_directory_property(LIBC_OLD_PREPROCESSOR_DEFS COMPILE_DEFINITIONS)
@@ -50,52 +51,14 @@ set(LIBC_NAMESPACE ${default_namespace}
   CACHE STRING "The namespace to use to enclose internal implementations. Must start with '__llvm_libc'."
 )
 
-
-add_subdirectory(newhdrgen)
-
-
-if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD)
-  if(NOT LIBC_HDRGEN_EXE)
-    # We need to set up hdrgen first since other targets depend on it.
-    add_subdirectory(utils/LibcTableGenUtil)
-    add_subdirectory(utils/HdrGen)
-    # Calling add_tablegen sets variables like LIBC_TABLEGEN_EXE in
-    # PARENT_SCOPE which get lost until saved in the cache.
-    set(LIBC_TABLEGEN_EXE "${LIBC_TABLEGEN_EXE}" CACHE INTERNAL "")
-    set(LIBC_TABLEGEN_TARGET "${LIBC_TABLEGEN_TARGET}" CACHE INTERNAL "")
-  else()
-    message(STATUS "Will use ${LIBC_HDRGEN_EXE} for libc header generation.")
-  endif()
-endif()
 # We will build the GPU utilities if we are not doing a runtimes build.
 option(LIBC_BUILD_GPU_LOADER "Always build the GPU loader utilities" OFF)
-if(LIBC_BUILD_GPU_LOADER OR (LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD))
+if(LIBC_BUILD_GPU_LOADER OR ((NOT LLVM_RUNTIMES_BUILD) AND LLVM_LIBC_GPU_BUILD))
   add_subdirectory(utils/gpu)
-endif()
-
-option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" ON)
-
-set(NEED_LIBC_HDRGEN FALSE)
-if(NOT LLVM_RUNTIMES_BUILD)
-  if("libc" IN_LIST LLVM_ENABLE_RUNTIMES)
-    set(NEED_LIBC_HDRGEN TRUE)
-  else()
-    foreach(_name ${LLVM_RUNTIME_TARGETS})
-      if("libc" IN_LIST RUNTIMES_${_name}_LLVM_ENABLE_RUNTIMES)
-        set(NEED_LIBC_HDRGEN TRUE)
-        break()
-      endif()
-    endforeach()
-  endif()
-endif()
-option(LIBC_HDRGEN_ONLY "Only build the 'libc-hdrgen' executable" OFF)
-if(LIBC_HDRGEN_ONLY OR NEED_LIBC_HDRGEN)
-  # When libc is build as part of the runtimes/bootstrap build's CMake run, we
-  # only need to build the host tools to build the libc. So, we just do enough
-  # to build libc-hdrgen and return.
   return()
 endif()
-unset(NEED_LIBC_HDRGEN)
+
+add_subdirectory(hdrgen)
 
 option(LIBC_CMAKE_VERBOSE_LOGGING
   "Log details warnings and notifications during CMake configuration." OFF)
diff --git a/libc/benchmarks/CMakeLists.txt b/libc/benchmarks/CMakeLists.txt
index 52e3f942d16ea..60f522d7d8c65 100644
--- a/libc/benchmarks/CMakeLists.txt
+++ b/libc/benchmarks/CMakeLists.txt
@@ -204,13 +204,11 @@ target_link_libraries(libc.benchmarks.memory_functions.opt_host
   PRIVATE
   libc-memory-benchmark
   libc.src.string.memcmp_opt_host.__internal__
-  libc.src.string.bcmp_opt_host.__internal__
   libc.src.string.memcpy_opt_host.__internal__
-  libc.src.string.memset_opt_host.__internal__
-  libc.src.string.bzero_opt_host.__internal__
   libc.src.string.memmove_opt_host.__internal__
+  libc.src.string.memset_opt_host.__internal__
+  libc.src.strings.bcmp_opt_host.__internal__
+  libc.src.strings.bzero_opt_host.__internal__
   benchmark_main
 )
 llvm_update_compile_flags(libc.benchmarks.memory_functions.opt_host)
-
-add_subdirectory(automemcpy)
diff --git a/libc/benchmarks/automemcpy/CMakeLists.txt b/libc/benchmarks/automemcpy/CMakeLists.txt
deleted file mode 100644
index ef9b4218c8d61..0000000000000
--- a/libc/benchmarks/automemcpy/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-if(NOT LIBC_BUILD_AUTOMEMCPY)
-  return ()
-endif()
-
-if(NOT LLVM_WITH_Z3)
-  MESSAGE(FATAL_ERROR "Building llvm-libc automemcpy requires Z3")
-endif()
-
-set(LIBC_AUTOMEMCPY_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
-
-add_subdirectory(lib)
-add_subdirectory(unittests)
diff --git a/libc/benchmarks/automemcpy/README.md b/libc/benchmarks/automemcpy/README.md
deleted file mode 100644
index 8583368993ef0..0000000000000
--- a/libc/benchmarks/automemcpy/README.md
+++ /dev/null
@@ -1,111 +0,0 @@
-This folder contains an implementation of [automemcpy: A framework for automatic generation of fundamental memory operations](https://research.google/pubs/pub50338/).
-
-It uses the [Z3 theorem prover](https://github.com/Z3Prover/z3) to enumerate a subset of valid memory function implementations. These implementations are then materialized as C++ code and can be [benchmarked](../) against various [size distributions](../distributions). This process helps the design of efficient implementations for a particular environnement (size distribution, processor or custom compilation options).
-
-This is not enabled by default, as it is mostly useful when working on tuning the library implementation. To build it, use `LIBC_BUILD_AUTOMEMCPY=ON` (see below).
-
-## Prerequisites
-
-You may need to install `Z3` from source if it's not available on your system.
-Here we show instructions to install it into `<Z3_INSTALL_DIR>`.
-You may need to `sudo` to `make install`.
-
-```shell
-mkdir -p ~/git
-cd ~/git
-git clone https://github.com/Z3Prover/z3.git
-python scripts/mk_make.py --prefix=<Z3_INSTALL_DIR>
-cd build
-make -j
-make install
-```
-
-## Configuration
-
-```shell
-mkdir -p <BUILD_DIR>
-cd <LLVM_PROJECT_DIR>/llvm
-cmake -DCMAKE_C_COMPILER=/usr/bin/clang \
- -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \
- -DLLVM_ENABLE_PROJECTS="libc" \
- -DLLVM_ENABLE_Z3_SOLVER=ON \
- -DLLVM_Z3_INSTALL_DIR=<Z3_INSTALL_DIR> \
- -DLIBC_BUILD_AUTOMEMCPY=ON \
- -DCMAKE_BUILD_TYPE=Release \
- -B<BUILD_DIR>
-```
-
-## Targets and compilation
-
-There are three main CMake targets
- 1. `automemcpy_implementations`
-    - runs `Z3` and materializes valid memory functions as C++ code, a message will display its ondisk location.
-    - the source code is then compiled using the native host optimizations (i.e. `-march=native` or `-mcpu=native` depending on the architecture).
- 2. `automemcpy`
-    - the binary that benchmarks the autogenerated implementations.
- 3. `automemcpy_result_analyzer`
-    - the binary that analyses the benchmark results.
-
-You may only compile the binaries as they both pull the autogenerated code as a dependency.
-
-```shell
-make -C <BUILD_DIR> -j automemcpy automemcpy_result_analyzer
-```
-
-## Running the benchmarks
-
-Make sure to save the results of the benchmark as a json file.
-
-```shell
-<BUILD_DIR>/bin/automemcpy --benchmark_out_format=json --benchmark_out=<RESULTS_DIR>/results.json
-```
-
-### Additional useful options
-
-
- - `--benchmark_min_time=.2`
-
-     By default, each function is benchmarked for at least one second, here we lower it to 200ms.
-
- - `--benchmark_filter="BM_Memset|BM_Bzero"`
-
-     By default, all functions are benchmarked, here we restrict them to `memset` and `bzero`.
-
-Other options might be useful, use `--help` for more information.
-
-## Analyzing the benchmarks
-
-Analysis is performed by running `automemcpy_result_analyzer` on one or more json result files.
-
-```shell
-<BUILD_DIR>/bin/automemcpy_result_analyzer <RESULTS_DIR>/results.json
-```
-
-What it does:
-  1. Gathers all throughput values for each function / distribution pair and picks the median one.\
-  This allows picking a representative value over many runs of the benchmark. Please make sure all the runs happen under similar circumstances.
-
-  2. For each distribution, look at the span of throughputs for functions of the same type (e.g. For distribution `A`, memcpy throughput spans from 2GiB/s to 5GiB/s).
-
-  3. For each distribution, give a normalized score to each function (e.g. For distribution `A`, function `M` scores 0.65).\
-  This score is then turned into a grade `EXCELLENT`, `VERY_GOOD`, `GOOD`, `PASSABLE`, `INADEQUATE`, `MEDIOCRE`, `BAD` - so that each distribution categorizes how function perform according to them.
-
-  4. A [Majority Judgement](https://en.wikipedia.org/wiki/Majority_judgment) process is then used to categorize each function. This enables finer analysis of how distributions agree on which function is better. In the following example, `Function_1` and `Function_2` are rated `EXCELLENT` but looking at the grade's distribution might help decide which is best.
-
-|            | EXCELLENT | VERY_GOOD | GOOD | PASSABLE | INADEQUATE | MEDIOCRE | BAD |
-|------------|:---------:|:---------:|:----:|:--------:|:----------:|:--------:|:---:|
-| Function_1 |     7     |     1     |   2  |          |            |          |     |
-| Function_2 |     6     |     4     |      |          |            |          |     |
-
-The tool outputs the histogram of grades for each function. In case of tie, other dimensions might help decide (e.g. code size, performance on other microarchitectures).
-
-```
-EXCELLENT  |█▁▂    |  Function_0
-EXCELLENT  |█▅     |  Function_1
-VERY_GOOD  |▂█▁ ▁  |  Function_2
-GOOD       | ▁█▄   |  Function_3
-PASSABLE   | ▂▆▄█  |  Function_4
-INADEQUATE |  ▃▃█▁ |  Function_5
-MEDIOCRE   |    █▆▁|  Function_6
-BAD        |    ▁▁█|  Function_7
-```
diff --git a/libc/benchmarks/automemcpy/include/automemcpy/CodeGen.h b/libc/benchmarks/automemcpy/include/automemcpy/CodeGen.h
deleted file mode 100644
index 389e8249f9399..0000000000000
--- a/libc/benchmarks/automemcpy/include/automemcpy/CodeGen.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===-- C++ code generation from NamedFunctionDescriptors -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LIBC_BENCHMARKS_AUTOMEMCPY_CODEGEN_H
-#define LIBC_BENCHMARKS_AUTOMEMCPY_CODEGEN_H
-
-#include "automemcpy/FunctionDescriptor.h"
-#include <llvm/ADT/ArrayRef.h>
-#include <llvm/Support/raw_ostream.h>
-#include <vector>
-
-namespace llvm {
-namespace automemcpy {
-
-// This function serializes the array of FunctionDescriptors as a C++ file.
-void Serialize(raw_ostream &Stream, ArrayRef<FunctionDescriptor> FD);
-
-} // namespace automemcpy
-} // namespace llvm
-
-#endif // LIBC_BENCHMARKS_AUTOMEMCPY_CODEGEN_H
diff --git a/libc/benchmarks/automemcpy/include/automemcpy/FunctionDescriptor.h b/libc/benchmarks/automemcpy/include/automemcpy/FunctionDescriptor.h
deleted file mode 100644
index 65477d9d72a0e..0000000000000
--- a/libc/benchmarks/automemcpy/include/automemcpy/FunctionDescriptor.h
+++ /dev/null
@@ -1,159 +0,0 @@
-//===-- Pod structs to describe a memory function----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_COMMON_H
-#define LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_COMMON_H
-
-#include <climits>
-#include <cstddef>
-#include <llvm/ADT/ArrayRef.h>
-#include <llvm/ADT/Hashing.h>
-#include <llvm/ADT/StringRef.h>
-#include <optional>
-#include <tuple>
-
-namespace llvm {
-namespace automemcpy {
-
-// Boilerplate code to be able to sort and hash types.
-#define COMPARABLE_AND_HASHABLE(T, ...)                                        \
-  inline auto asTuple() const { return std::tie(__VA_ARGS__); }                \
-  bool operator==(const T &O) const { return asTuple() == O.asTuple(); }       \
-  bool operator<(const T &O) const { return asTuple() < O.asTuple(); }         \
-  struct Hasher {                                                              \
-    std::size_t operator()(const T &K) const {                                 \
-      return llvm::hash_value(K.asTuple());                                    \
-    }                                                                          \
-  };
-
-// Represents the maximum value for the size parameter of a memory function.
-// This is an `int` so we can use it as an expression in Z3.
-// It also allows for a more readable and compact representation when storing
-// the SizeSpan in the autogenerated C++ file.
-static constexpr int kMaxSize = INT_MAX;
-
-// This mimics the `Arg` type in libc/src/string/memory_utils/elements.h without
-// having to depend on it.
-enum class AlignArg { _1, _2, ARRAY_SIZE };
-
-// Describes a range of sizes.
-// We use the begin/end representation instead of first/last to allow for empty
-// range (i.e. Begin == End)
-struct SizeSpan {
-  size_t Begin = 0;
-  size_t End = 0;
-
-  COMPARABLE_AND_HASHABLE(SizeSpan, Begin, End)
-};
-
-// Describes a contiguous region.
-// In such a region all sizes are handled individually.
-// e.g. with Span = {0, 2};
-// if(size == 0) return Handle<0>();
-// if(size == 1) return Handle<1>();
-struct Contiguous {
-  SizeSpan Span;
-
-  COMPARABLE_AND_HASHABLE(Contiguous, Span)
-};
-
-// This struct represents a range of sizes over which to use an overlapping
-// strategy. An overlapping strategy of size N handles all sizes from N to 2xN.
-// The span may represent several contiguous overlaps.
-// e.g. with Span = {16, 128};
-// if(size >= 16 and size < 32) return Handle<Overlap<16>>();
-// if(size >= 32 and size < 64) return Handle<Overlap<32>>();
-// if(size >= 64 and size < 128) return Handle<Overlap<64>>();
-struct Overlap {
-  SizeSpan Span;
-
-  COMPARABLE_AND_HASHABLE(Overlap, Span)
-};
-
-// Describes a region using a loop handling BlockSize bytes at a time. The
-// remaining bytes of the loop are handled with an overlapping operation.
-struct Loop {
-  SizeSpan Span;
-  size_t BlockSize = 0;
-
-  COMPARABLE_AND_HASHABLE(Loop, Span, BlockSize)
-};
-
-// Same as `Loop` but starts by aligning a buffer on `Alignment` bytes.
-// A first operation handling 'Alignment` bytes is performed followed by a
-// sequence of Loop.BlockSize bytes operation. The Loop starts processing from
-// the next aligned byte in the chosen buffer. The remaining bytes of the loop
-// are handled with an overlapping operation.
-struct AlignedLoop {
-  Loop Loop;
-  size_t Alignment = 0;            // Size of the alignment.
-  AlignArg AlignTo = AlignArg::_1; // Which buffer to align.
-
-  COMPARABLE_AND_HASHABLE(AlignedLoop, Loop, Alignment, AlignTo)
-};
-
-// Some processors offer special instruction to handle the memory function
-// completely, we refer to such instructions as accelerators.
-struct Accelerator {
-  SizeSpan Span;
-
-  COMPARABLE_AND_HASHABLE(Accelerator, Span)
-};
-
-// The memory functions are assembled out of primitives that can be implemented
-// with regular scalar operations (SCALAR), with the help of vector or bitcount
-// instructions (NATIVE) or by deferring it to the compiler (BUILTIN).
-enum class ElementTypeClass {
-  SCALAR,
-  NATIVE,
-  BUILTIN,
-};
-
-// A simple enum to categorize which function is being implemented.
-enum class FunctionType {
-  MEMCPY,
-  MEMCMP,
-  BCMP,
-  MEMSET,
-  BZERO,
-};
-
-// This struct describes the skeleton of the implementation, it does not go into
-// every detail but is enough to uniquely identify the implementation.
-struct FunctionDescriptor {
-  FunctionType Type;
-  std::optional<Contiguous> Contiguous;
-  std::optional<Overlap> Overlap;
-  std::optional<Loop> Loop;
-  std::optional<AlignedLoop> AlignedLoop;
-  std::optional<Accelerator> Accelerator;
-  ElementTypeClass ElementClass;
-
-  COMPARABLE_AND_HASHABLE(FunctionDescriptor, Type, Contiguous, Overlap, Loop,
-                          AlignedLoop, Accelerator, ElementClass)
-
-  inline size_t id() const { return llvm::hash_value(asTuple()); }
-};
-
-// Same as above but with the function name.
-struct NamedFunctionDescriptor {
-  StringRef Name;
-  FunctionDescriptor Desc;
-};
-
-template <typename T> llvm::hash_code hash_value(const ArrayRef<T> &V) {
-  return llvm::hash_combine_range(V.begin(), V.end());
-}
-template <typename T> llvm::hash_code hash_value(const T &O) {
-  return llvm::hash_value(O.asTuple());
-}
-
-} // namespace automemcpy
-} // namespace llvm
-
-#endif /* LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_COMMON_H */
diff --git a/libc/benchmarks/automemcpy/include/automemcpy/RandomFunctionGenerator.h b/libc/benchmarks/automemcpy/include/automemcpy/RandomFunctionGenerator.h
deleted file mode 100644
index 28756e8f86c0e..0000000000000
--- a/libc/benchmarks/automemcpy/include/automemcpy/RandomFunctionGenerator.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//===-- Generate random but valid function descriptors  ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_RANDOM_FUNCTION_GENERATOR_H
-#define LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_RANDOM_FUNCTION_GENERATOR_H
-
-#include "automemcpy/FunctionDescriptor.h"
-#include <cstddef>
-#include <cstdint>
-#include <llvm/ADT/ArrayRef.h>
-#include <llvm/ADT/StringRef.h>
-#include <optional>
-#include <vector>
-#include <z3++.h>
-
-namespace llvm {
-namespace automemcpy {
-
-// Holds the state for the constraint solver.
-// It implements a single method that returns the next valid description.
-struct RandomFunctionGenerator {
-  RandomFunctionGenerator();
-
-  // Get the next valid FunctionDescriptor or std::nullopt.
-  std::optional<FunctionDescriptor> next();
-
-private:
-  // Returns an expression where `Variable` is forced to be one of the `Values`.
-  z3::expr inSetConstraint(z3::expr &Variable, ArrayRef<int> Values) const;
-  // Add constaints to `Begin` and `End` so that they are:
-  // - between 0 and kMaxSize (inclusive)
-  // - ordered (begin<=End)
-  // - amongst a set of predefined values.
-  void addBoundsAndAnchors(z3::expr &Begin, z3::expr &End);
-  // Add constraints to make sure that the loop block size is amongst a set of
-  // predefined values. Also makes sure that the loop that the loop is iterated
-  // at least `LoopMinIter` times.
-  void addLoopConstraints(const z3::expr &LoopBegin, const z3::expr &LoopEnd,
-                          z3::expr &LoopBlockSize, int LoopMinIter);
-
-  z3::context Context;
-  z3::solver Solver;
-
-  z3::expr Type;
-  z3::expr ContiguousBegin, ContiguousEnd;
-  z3::expr OverlapBegin, OverlapEnd;
-  z3::expr LoopBegin, LoopEnd, LoopBlockSize;
-  z3::expr AlignedLoopBegin, AlignedLoopEnd, AlignedLoopBlockSize,
-      AlignedAlignment, AlignedArg;
-  z3::expr AcceleratorBegin, AcceleratorEnd;
-  z3::expr ElementClass;
-};
-
-} // namespace automemcpy
-} // namespace llvm
-
-#endif /* LLVM_LIBC_BENCHMARKS_AUTOMEMCPY_RANDOM_FUNCTION_GENERATOR_H */
diff --git a/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h b/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
deleted file mode 100644
index d4bf272582767..0000000000000
--- a/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
+++ /dev/null
@@ -1,109 +0,0 @@
-//===-- Analyze benchmark JSON files ----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
-#define LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
-
-#include "automemcpy/FunctionDescriptor.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringMap.h"
-#include <array>
-#include <vector>
-
-namespace llvm {
-namespace automemcpy {
-
-// A Grade as in the Majority Judgment voting system.
-struct Grade {
-  enum GradeEnum {
-    EXCELLENT,
-    VERY_GOOD,
-    GOOD,
-    PASSABLE,
-    INADEQUATE,
-    MEDIOCRE,
-    BAD,
-    ARRAY_SIZE,
-  };
-
-  // Returns a human readable string of the enum.
-  static StringRef getString(const GradeEnum &GE);
-
-  // Turns 'Score' into a GradeEnum.
-  static GradeEnum judge(double Score);
-};
-
-// A 'GradeEnum' indexed array with counts for each grade.
-using GradeHistogram = std::array<size_t, Grade::ARRAY_SIZE>;
-
-// Identifies a Function by its name and type. Used as a key in a map.
-struct FunctionId {
-  StringRef Name;
-  FunctionType Type;
-  COMPARABLE_AND_HASHABLE(FunctionId, Type, Name)
-};
-
-struct PerDistributionData {
-  std::vector<double> BytesPerSecondSamples;
-  double BytesPerSecondMedian;   // Median of samples for this distribution.
-  double BytesPerSecondMean;     // Mean of samples for this distribution.
-  double BytesPerSecondVariance; // Variance of samples for this distribution.
-  double Score;                  // Normalized score for this distribution.
-  Grade::GradeEnum Grade;        // Grade for this distribution.
-};
-
-struct FunctionData {
-  FunctionId Id;
-  StringMap<PerDistributionData> PerDistributionData;
-  double ScoresGeoMean;           // Geomean of scores for each distribution.
-  GradeHistogram GradeHisto = {}; // GradeEnum indexed array
-  Grade::GradeEnum FinalGrade = Grade::BAD; // Overall grade for this function
-};
-
-// Identifies a Distribution by its name. Used as a key in a map.
-struct DistributionId {
-  StringRef Name;
-  COMPARABLE_AND_HASHABLE(DistributionId, Name)
-};
-
-// Identifies a Sample by its distribution and function. Used as a key in a map.
-struct SampleId {
-  FunctionId Function;
-  DistributionId Distribution;
-  COMPARABLE_AND_HASHABLE(SampleId, Function.Type, Function.Name,
-                          Distribution.Name)
-};
-
-// The type of Samples as reported by the Google Benchmark's JSON result file.
-// We are only interested in the "iteration" samples, the "aggregate" ones
-// represent derived metrics such as 'mean' or 'median'.
-enum class SampleType { UNKNOWN, ITERATION, AGGREGATE };
-
-// A SampleId with an associated measured throughput.
-struct Sample {
-  SampleId Id;
-  SampleType Type = SampleType::UNKNOWN;
-  double BytesPerSecond = 0;
-};
-
-// This function collects Samples that belong to the same distribution and
-// function and retains the median one. It then stores each of them into a
-// 'FunctionData' and returns them as a vector.
-std::vector<FunctionData> getThroughputs(ArrayRef<Sample> Samples);
-
-// Normalize the function's throughput per distribution.
-void fillScores(MutableArrayRef<FunctionData> Functions);
-
-// Convert scores into Grades, stores an histogram of Grade for each functions
-// and cast a median grade for the function.
-void castVotes(MutableArrayRef<FunctionData> Functions);
-
-} // namespace automemcpy
-} // namespace llvm
-
-#endif // LIBC_BENCHMARKS_AUTOMEMCPY_RESULTANALYZER_H
diff --git a/libc/benchmarks/automemcpy/lib/CMakeLists.txt b/libc/benchmarks/automemcpy/lib/CMakeLists.txt
deleted file mode 100644
index e66b9045b6074..0000000000000
--- a/libc/benchmarks/automemcpy/lib/CMakeLists.txt
+++ /dev/null
@@ -1,37 +0,0 @@
-add_library(automemcpy_codegen CodeGen.cpp)
-target_link_libraries(automemcpy_codegen PUBLIC LLVMSupport)
-target_include_directories(automemcpy_codegen PUBLIC ${LIBC_AUTOMEMCPY_INCLUDE_DIR})
-llvm_update_compile_flags(automemcpy_codegen)
-
-add_executable(automemcpy_codegen_main CodeGenMain.cpp RandomFunctionGenerator.cpp)
-target_link_libraries(automemcpy_codegen_main PUBLIC automemcpy_codegen ${Z3_LIBRARIES})
-llvm_update_compile_flags(automemcpy_codegen_main)
-
-set(Implementations "${CMAKE_CURRENT_BINARY_DIR}/Implementations.cpp")
-add_custom_command(
-    OUTPUT ${Implementations}
-    COMMAND "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/automemcpy_codegen_main" > "${Implementations}"
-    COMMAND echo "automemcpy implementations generated in ${Implementations}"
-    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-    DEPENDS automemcpy_codegen_main
-)
-
-add_library(automemcpy_implementations "${Implementations}")
-target_link_libraries(automemcpy_implementations PUBLIC LLVMSupport libc-memory-benchmark)
-target_include_directories(automemcpy_implementations PRIVATE 
-                           ${LIBC_SOURCE_DIR} ${LIBC_AUTOMEMCPY_INCLUDE_DIR})
-target_compile_options(automemcpy_implementations PRIVATE ${LIBC_COMPILE_OPTIONS_NATIVE} "SHELL:-mllvm -combiner-global-alias-analysis" -fno-builtin)
-llvm_update_compile_flags(automemcpy_implementations)
-
-add_executable(automemcpy EXCLUDE_FROM_ALL ${LIBC_SOURCE_DIR}/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp)
-target_link_libraries(automemcpy PRIVATE libc-memory-benchmark benchmark_main automemcpy_implementations)
-llvm_update_compile_flags(automemcpy)
-
-add_library(automemcpy_result_analyzer_lib EXCLUDE_FROM_ALL ResultAnalyzer.cpp)
-target_link_libraries(automemcpy_result_analyzer_lib PUBLIC LLVMSupport)
-target_include_directories(automemcpy_result_analyzer_lib PUBLIC ${LIBC_AUTOMEMCPY_INCLUDE_DIR})
-llvm_update_compile_flags(automemcpy_result_analyzer_lib)
-
-add_executable(automemcpy_result_analyzer EXCLUDE_FROM_ALL ResultAnalyzerMain.cpp)
-target_link_libraries(automemcpy_result_analyzer PRIVATE automemcpy_result_analyzer_lib automemcpy_implementations)
-llvm_update_compile_flags(automemcpy_result_analyzer)
diff --git a/libc/benchmarks/automemcpy/lib/CodeGen.cpp b/libc/benchmarks/automemcpy/lib/CodeGen.cpp
deleted file mode 100644
index d1336eaf31f34..0000000000000
--- a/libc/benchmarks/automemcpy/lib/CodeGen.cpp
+++ /dev/null
@@ -1,644 +0,0 @@
-//===-- C++ code generation from NamedFunctionDescriptors -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// This code is responsible for generating the "Implementation.cpp" file.
-// The file is composed like this:
-//
-// 1. Includes
-// 2. Using statements to help readability.
-// 3. Source code for all the mem function implementations.
-// 4. The function to retrieve all the function descriptors with their name.
-//      llvm::ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors();
-// 5. The functions for the benchmarking infrastructure:
-//      llvm::ArrayRef<MemcpyConfiguration> getMemcpyConfigurations();
-//      llvm::ArrayRef<MemcmpOrBcmpConfiguration> getMemcmpConfigurations();
-//      llvm::ArrayRef<MemcmpOrBcmpConfiguration> getBcmpConfigurations();
-//      llvm::ArrayRef<MemsetConfiguration> getMemsetConfigurations();
-//      llvm::ArrayRef<BzeroConfiguration> getBzeroConfigurations();
-//
-//
-// Sections 3, 4 and 5 are handled by the following namespaces:
-// - codegen::functions
-// - codegen::descriptors
-// - codegen::configurations
-//
-// The programming style is functionnal. In each of these namespace, the
-// original `NamedFunctionDescriptor` object is turned into a different type. We
-// make use of overloaded stream operators to format the resulting type into
-// either a function, a descriptor or a configuration. The entry point of each
-// namespace is the Serialize function.
-//
-// Note the code here is better understood by starting from the `Serialize`
-// function at the end of the file.
-
-#include "automemcpy/CodeGen.h"
-#include "src/__support/macros/config.h"
-#include <cassert>
-#include <llvm/ADT/STLExtras.h>
-#include <llvm/ADT/StringSet.h>
-#include <llvm/Support/FormatVariadic.h>
-#include <llvm/Support/raw_ostream.h>
-#include <optional>
-#include <set>
-
-namespace llvm {
-namespace automemcpy {
-namespace codegen {
-
-// The indentation string.
-static constexpr StringRef kIndent = "  ";
-
-// The codegen namespace handles the serialization of a NamedFunctionDescriptor
-// into source code for the function, the descriptor and the configuration.
-
-namespace functions {
-
-// This namespace turns a NamedFunctionDescriptor into an actual implementation.
-// -----------------------------------------------------------------------------
-// e.g.
-// static void memcpy_0xB20D4702493C397E(char *__restrict dst,
-//                                       const char *__restrict src,
-//                                       size_t size) {
-//   using namespace LIBC_NAMESPACE::x86;
-//   if(size == 0) return;
-//   if(size == 1) return copy<_1>(dst, src);
-//   if(size < 4) return copy<HeadTail<_2>>(dst, src, size);
-//   if(size < 8) return copy<HeadTail<_4>>(dst, src, size);
-//   if(size < 16) return copy<HeadTail<_8>>(dst, src, size);
-//   if(size < 32) return copy<HeadTail<_16>>(dst, src, size);
-//   return copy<Accelerator>(dst, src, size);
-// }
-
-// The `Serialize` method turns a `NamedFunctionDescriptor` into a
-// `FunctionImplementation` which holds all the information needed to produce
-// the C++ source code.
-
-// An Element with its size (e.g. `_16` in the example above).
-struct ElementType {
-  size_t Size;
-};
-// The case `if(size == 0)` is encoded as a the Zero type.
-struct Zero {
-  StringRef DefaultReturnValue;
-};
-// An individual size `if(size == X)` is encoded as an Individual type.
-struct Individual {
-  size_t IfEq;
-  ElementType Element;
-};
-// An overlap strategy is encoded as an Overlap type.
-struct Overlap {
-  size_t IfLt;
-  ElementType Element;
-};
-// A loop strategy is encoded as a Loop type.
-struct Loop {
-  size_t IfLt;
-  ElementType Element;
-};
-// An aligned loop strategy is encoded as an AlignedLoop type.
-struct AlignedLoop {
-  size_t IfLt;
-  ElementType Element;
-  ElementType Alignment;
-  StringRef AlignTo;
-};
-// The accelerator strategy.
-struct Accelerator {
-  size_t IfLt;
-};
-// The Context stores data about the function type.
-struct Context {
-  StringRef FunctionReturnType; // e.g. void* or int
-  StringRef FunctionArgs;
-  StringRef ElementOp; // copy, three_way_compare, splat_set, ...
-  StringRef FixedSizeArgs;
-  StringRef RuntimeSizeArgs;
-  StringRef DefaultReturnValue;
-};
-// A detailed representation of the function implementation mapped from the
-// NamedFunctionDescriptor.
-struct FunctionImplementation {
-  Context Ctx;
-  StringRef Name;
-  std::vector<Individual> Individuals;
-  std::vector<Overlap> Overlaps;
-  std::optional<Loop> Loop;
-  std::optional<AlignedLoop> AlignedLoop;
-  std::optional<Accelerator> Accelerator;
-  ElementTypeClass ElementClass;
-};
-
-// Returns the Context for each FunctionType.
-static Context getCtx(FunctionType FT) {
-  switch (FT) {
-  case FunctionType::MEMCPY:
-    return {"void",
-            "(char *__restrict dst, const char *__restrict src, size_t size)",
-            "copy",
-            "(dst, src)",
-            "(dst, src, size)",
-            ""};
-  case FunctionType::MEMCMP:
-    return {"int",
-            "(const char * lhs, const char * rhs, size_t size)",
-            "three_way_compare",
-            "(lhs, rhs)",
-            "(lhs, rhs, size)",
-            "0"};
-  case FunctionType::MEMSET:
-    return {"void",
-            "(char * dst, int value, size_t size)",
-            "splat_set",
-            "(dst, value)",
-            "(dst, value, size)",
-            ""};
-  case FunctionType::BZERO:
-    return {"void",           "(char * dst, size_t size)",
-            "splat_set",      "(dst, 0)",
-            "(dst, 0, size)", ""};
-  default:
-    report_fatal_error("Not yet implemented");
-  }
-}
-
-static StringRef getAligntoString(const AlignArg &AlignTo) {
-  switch (AlignTo) {
-  case AlignArg::_1:
-    return "Arg::P1";
-  case AlignArg::_2:
-    return "Arg::P2";
-  case AlignArg::ARRAY_SIZE:
-    report_fatal_error("logic error");
-  }
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream, const ElementType &E) {
-  return Stream << '_' << E.Size;
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Individual &O) {
-  return Stream << O.Element;
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Overlap &O) {
-  return Stream << "HeadTail<" << O.Element << '>';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Loop &O) {
-  return Stream << "Loop<" << O.Element << '>';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const AlignedLoop &O) {
-  return Stream << "Align<" << O.Alignment << ',' << O.AlignTo << ">::Then<"
-                << Loop{O.IfLt, O.Element} << ">";
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Accelerator &O) {
-  return Stream << "Accelerator";
-}
-
-template <typename T> struct IfEq {
-  StringRef Op;
-  StringRef Args;
-  const T &Element;
-};
-
-template <typename T> struct IfLt {
-  StringRef Op;
-  StringRef Args;
-  const T &Element;
-};
-
-static raw_ostream &operator<<(raw_ostream &Stream, const Zero &O) {
-  Stream << kIndent << "if(size == 0) return";
-  if (!O.DefaultReturnValue.empty())
-    Stream << ' ' << O.DefaultReturnValue;
-  return Stream << ";\n";
-}
-
-template <typename T>
-static raw_ostream &operator<<(raw_ostream &Stream, const IfEq<T> &O) {
-  return Stream << kIndent << "if(size == " << O.Element.IfEq << ") return "
-                << O.Op << '<' << O.Element << '>' << O.Args << ";\n";
-}
-
-template <typename T>
-static raw_ostream &operator<<(raw_ostream &Stream, const IfLt<T> &O) {
-  Stream << kIndent;
-  if (O.Element.IfLt != kMaxSize)
-    Stream << "if(size < " << O.Element.IfLt << ") ";
-  return Stream << "return " << O.Op << '<' << O.Element << '>' << O.Args
-                << ";\n";
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const ElementTypeClass &Class) {
-  switch (Class) {
-  case ElementTypeClass::SCALAR:
-    return Stream << "scalar";
-  case ElementTypeClass::BUILTIN:
-    return Stream << "builtin";
-  case ElementTypeClass::NATIVE:
-    // FIXME: the framework should provide a `native` namespace that redirect to
-    // x86, arm or other architectures.
-    return Stream << "x86";
-  }
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const FunctionImplementation &FI) {
-  const auto &Ctx = FI.Ctx;
-  Stream << "static " << Ctx.FunctionReturnType << ' ' << FI.Name
-         << Ctx.FunctionArgs << " {\n";
-  Stream << kIndent << "using namespace LIBC_NAMESPACE::" << FI.ElementClass
-         << ";\n";
-  for (const auto &I : FI.Individuals)
-    if (I.Element.Size == 0)
-      Stream << Zero{Ctx.DefaultReturnValue};
-    else
-      Stream << IfEq<Individual>{Ctx.ElementOp, Ctx.FixedSizeArgs, I};
-  for (const auto &O : FI.Overlaps)
-    Stream << IfLt<Overlap>{Ctx.ElementOp, Ctx.RuntimeSizeArgs, O};
-  if (const auto &C = FI.Loop)
-    Stream << IfLt<Loop>{Ctx.ElementOp, Ctx.RuntimeSizeArgs, *C};
-  if (const auto &C = FI.AlignedLoop)
-    Stream << IfLt<AlignedLoop>{Ctx.ElementOp, Ctx.RuntimeSizeArgs, *C};
-  if (const auto &C = FI.Accelerator)
-    Stream << IfLt<Accelerator>{Ctx.ElementOp, Ctx.RuntimeSizeArgs, *C};
-  return Stream << "}\n";
-}
-
-// Turns a `NamedFunctionDescriptor` into a `FunctionImplementation` unfolding
-// the contiguous and overlap region into several statements. The zero case is
-// also mapped to its own type.
-static FunctionImplementation
-getImplementation(const NamedFunctionDescriptor &NamedFD) {
-  const FunctionDescriptor &FD = NamedFD.Desc;
-  FunctionImplementation Impl;
-  Impl.Ctx = getCtx(FD.Type);
-  Impl.Name = NamedFD.Name;
-  Impl.ElementClass = FD.ElementClass;
-  if (auto C = FD.Contiguous)
-    for (size_t I = C->Span.Begin; I < C->Span.End; ++I)
-      Impl.Individuals.push_back(Individual{I, ElementType{I}});
-  if (auto C = FD.Overlap)
-    for (size_t I = C->Span.Begin; I < C->Span.End; I *= 2)
-      Impl.Overlaps.push_back(Overlap{2 * I, ElementType{I}});
-  if (const auto &L = FD.Loop)
-    Impl.Loop = Loop{L->Span.End, ElementType{L->BlockSize}};
-  if (const auto &AL = FD.AlignedLoop)
-    Impl.AlignedLoop =
-        AlignedLoop{AL->Loop.Span.End, ElementType{AL->Loop.BlockSize},
-                    ElementType{AL->Alignment}, getAligntoString(AL->AlignTo)};
-  if (const auto &A = FD.Accelerator)
-    Impl.Accelerator = Accelerator{A->Span.End};
-  return Impl;
-}
-
-static void Serialize(raw_ostream &Stream,
-                      ArrayRef<NamedFunctionDescriptor> Descriptors) {
-
-  for (const auto &FD : Descriptors)
-    Stream << getImplementation(FD);
-}
-
-} // namespace functions
-
-namespace descriptors {
-
-// This namespace generates the getFunctionDescriptors function:
-// -------------------------------------------------------------
-// e.g.
-// ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors() {
-//   static constexpr NamedFunctionDescriptor kDescriptors[] = {
-//     {"memcpy_0xE00E29EE73994E2B",{FunctionType::MEMCPY,std::nullopt,std::nullopt,std::nullopt,std::nullopt,Accelerator{{0,kMaxSize}},ElementTypeClass::NATIVE}},
-//     {"memcpy_0x8661D80472487AB5",{FunctionType::MEMCPY,Contiguous{{0,1}},std::nullopt,std::nullopt,std::nullopt,Accelerator{{1,kMaxSize}},ElementTypeClass::NATIVE}},
-//     ...
-//   };
-//   return ArrayRef(kDescriptors);
-// }
-
-static raw_ostream &operator<<(raw_ostream &Stream, const SizeSpan &SS) {
-  Stream << "{" << SS.Begin << ',';
-  if (SS.End == kMaxSize)
-    Stream << "kMaxSize";
-  else
-    Stream << SS.End;
-  return Stream << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Contiguous &O) {
-  return Stream << "Contiguous{" << O.Span << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Overlap &O) {
-  return Stream << "Overlap{" << O.Span << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Loop &O) {
-  return Stream << "Loop{" << O.Span << ',' << O.BlockSize << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const AlignArg &O) {
-  switch (O) {
-  case AlignArg::_1:
-    return Stream << "AlignArg::_1";
-  case AlignArg::_2:
-    return Stream << "AlignArg::_2";
-  case AlignArg::ARRAY_SIZE:
-    report_fatal_error("logic error");
-  }
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const AlignedLoop &O) {
-  return Stream << "AlignedLoop{" << O.Loop << ',' << O.Alignment << ','
-                << O.AlignTo << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const Accelerator &O) {
-  return Stream << "Accelerator{" << O.Span << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const ElementTypeClass &O) {
-  switch (O) {
-  case ElementTypeClass::SCALAR:
-    return Stream << "ElementTypeClass::SCALAR";
-  case ElementTypeClass::BUILTIN:
-    return Stream << "ElementTypeClass::BUILTIN";
-  case ElementTypeClass::NATIVE:
-    return Stream << "ElementTypeClass::NATIVE";
-  }
-}
-static raw_ostream &operator<<(raw_ostream &Stream, const FunctionType &T) {
-  switch (T) {
-  case FunctionType::MEMCPY:
-    return Stream << "FunctionType::MEMCPY";
-  case FunctionType::MEMCMP:
-    return Stream << "FunctionType::MEMCMP";
-  case FunctionType::BCMP:
-    return Stream << "FunctionType::BCMP";
-  case FunctionType::MEMSET:
-    return Stream << "FunctionType::MEMSET";
-  case FunctionType::BZERO:
-    return Stream << "FunctionType::BZERO";
-  }
-}
-template <typename T>
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const std::optional<T> &MaybeT) {
-  if (MaybeT)
-    return Stream << *MaybeT;
-  return Stream << "std::nullopt";
-}
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const FunctionDescriptor &FD) {
-  return Stream << '{' << FD.Type << ',' << FD.Contiguous << ',' << FD.Overlap
-                << ',' << FD.Loop << ',' << FD.AlignedLoop << ','
-                << FD.Accelerator << ',' << FD.ElementClass << '}';
-}
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const NamedFunctionDescriptor &NFD) {
-  return Stream << '{' << '"' << NFD.Name << '"' << ',' << NFD.Desc << '}';
-}
-template <typename T>
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const std::vector<T> &VectorT) {
-  Stream << '{';
-  bool First = true;
-  for (const auto &Obj : VectorT) {
-    if (!First)
-      Stream << ',';
-    Stream << Obj;
-    First = false;
-  }
-  return Stream << '}';
-}
-
-static void Serialize(raw_ostream &Stream,
-                      ArrayRef<NamedFunctionDescriptor> Descriptors) {
-  Stream << R"(ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors() {
-  static constexpr NamedFunctionDescriptor kDescriptors[] = {
-)";
-  for (size_t I = 0, E = Descriptors.size(); I < E; ++I) {
-    Stream << kIndent << kIndent << Descriptors[I] << ",\n";
-  }
-  Stream << R"(  };
-  return ArrayRef(kDescriptors);
-}
-)";
-}
-
-} // namespace descriptors
-
-namespace configurations {
-
-// This namespace generates the getXXXConfigurations functions:
-// ------------------------------------------------------------
-// e.g.
-// llvm::ArrayRef<MemcpyConfiguration> getMemcpyConfigurations() {
-//   using namespace LIBC_NAMESPACE;
-//   static constexpr MemcpyConfiguration kConfigurations[] = {
-//     {Wrap<memcpy_0xE00E29EE73994E2B>, "memcpy_0xE00E29EE73994E2B"},
-//     {Wrap<memcpy_0x8661D80472487AB5>, "memcpy_0x8661D80472487AB5"},
-//     ...
-//   };
-//   return llvm::ArrayRef(kConfigurations);
-// }
-
-// The `Wrap` template function is provided in the `Main` function below.
-// It is used to adapt the gnerated code to the prototype of the C function.
-// For instance, the generated code for a `memcpy` takes `char*` pointers and
-// returns nothing but the original C `memcpy` function take and returns `void*`
-// pointers.
-
-struct FunctionName {
-  FunctionType ForType;
-};
-
-struct ReturnType {
-  FunctionType ForType;
-};
-
-struct Configuration {
-  FunctionName Name;
-  ReturnType Type;
-  std::vector<const NamedFunctionDescriptor *> Descriptors;
-};
-
-static raw_ostream &operator<<(raw_ostream &Stream, const FunctionName &FN) {
-  switch (FN.ForType) {
-  case FunctionType::MEMCPY:
-    return Stream << "getMemcpyConfigurations";
-  case FunctionType::MEMCMP:
-    return Stream << "getMemcmpConfigurations";
-  case FunctionType::BCMP:
-    return Stream << "getBcmpConfigurations";
-  case FunctionType::MEMSET:
-    return Stream << "getMemsetConfigurations";
-  case FunctionType::BZERO:
-    return Stream << "getBzeroConfigurations";
-  }
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream, const ReturnType &RT) {
-  switch (RT.ForType) {
-  case FunctionType::MEMCPY:
-    return Stream << "MemcpyConfiguration";
-  case FunctionType::MEMCMP:
-  case FunctionType::BCMP:
-    return Stream << "MemcmpOrBcmpConfiguration";
-  case FunctionType::MEMSET:
-    return Stream << "MemsetConfiguration";
-  case FunctionType::BZERO:
-    return Stream << "BzeroConfiguration";
-  }
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream,
-                               const NamedFunctionDescriptor *FD) {
-  return Stream << formatv("{Wrap<{0}>, \"{0}\"}", FD->Name);
-}
-
-static raw_ostream &
-operator<<(raw_ostream &Stream,
-           const std::vector<const NamedFunctionDescriptor *> &Descriptors) {
-  for (size_t I = 0, E = Descriptors.size(); I < E; ++I)
-    Stream << kIndent << kIndent << Descriptors[I] << ",\n";
-  return Stream;
-}
-
-static raw_ostream &operator<<(raw_ostream &Stream, const Configuration &C) {
-  Stream << "llvm::ArrayRef<" << C.Type << "> " << C.Name << "() {\n";
-  if (C.Descriptors.empty())
-    Stream << kIndent << "return {};\n";
-  else {
-    Stream << kIndent << "using namespace LIBC_NAMESPACE;\n";
-    Stream << kIndent << "static constexpr " << C.Type
-           << " kConfigurations[] = {\n";
-    Stream << C.Descriptors;
-    Stream << kIndent << "};\n";
-    Stream << kIndent << "return llvm::ArrayRef(kConfigurations);\n";
-  }
-  Stream << "}\n";
-  return Stream;
-}
-
-static void Serialize(raw_ostream &Stream, FunctionType FT,
-                      ArrayRef<NamedFunctionDescriptor> Descriptors) {
-  Configuration Conf;
-  Conf.Name = {FT};
-  Conf.Type = {FT};
-  for (const auto &FD : Descriptors)
-    if (FD.Desc.Type == FT)
-      Conf.Descriptors.push_back(&FD);
-  Stream << Conf;
-}
-
-} // namespace configurations
-static void Serialize(raw_ostream &Stream,
-                      ArrayRef<NamedFunctionDescriptor> Descriptors) {
-  Stream << "// This file is auto-generated by libc/benchmarks/automemcpy.\n";
-  Stream << "// Functions : " << Descriptors.size() << "\n";
-  Stream << "\n";
-  Stream << "#include \"LibcFunctionPrototypes.h\"\n";
-  Stream << "#include \"automemcpy/FunctionDescriptor.h\"\n";
-  Stream << "#include \"src/string/memory_utils/elements.h\"\n";
-  Stream << "\n";
-  Stream << "using llvm::libc_benchmarks::BzeroConfiguration;\n";
-  Stream << "using llvm::libc_benchmarks::MemcmpOrBcmpConfiguration;\n";
-  Stream << "using llvm::libc_benchmarks::MemcpyConfiguration;\n";
-  Stream << "using llvm::libc_benchmarks::MemmoveConfiguration;\n";
-  Stream << "using llvm::libc_benchmarks::MemsetConfiguration;\n";
-  Stream << "\n";
-  Stream << "namespace LIBC_NAMESPACE_DECL {\n";
-  Stream << "\n";
-  codegen::functions::Serialize(Stream, Descriptors);
-  Stream << "\n";
-  Stream << "} // namespace LIBC_NAMESPACE_DECL\n";
-  Stream << "\n";
-  Stream << "namespace llvm {\n";
-  Stream << "namespace automemcpy {\n";
-  Stream << "\n";
-  codegen::descriptors::Serialize(Stream, Descriptors);
-  Stream << "\n";
-  Stream << "} // namespace automemcpy\n";
-  Stream << "} // namespace llvm\n";
-  Stream << "\n";
-  Stream << R"(
-using MemcpyStub = void (*)(char *__restrict, const char *__restrict, size_t);
-template <MemcpyStub Foo>
-void *Wrap(void *__restrict dst, const void *__restrict src, size_t size) {
-  Foo(reinterpret_cast<char *__restrict>(dst),
-      reinterpret_cast<const char *__restrict>(src), size);
-  return dst;
-}
-)";
-  codegen::configurations::Serialize(Stream, FunctionType::MEMCPY, Descriptors);
-  Stream << R"(
-using MemcmpStub = int (*)(const char *, const char *, size_t);
-template <MemcmpStub Foo>
-int Wrap(const void *lhs, const void *rhs, size_t size) {
-  return Foo(reinterpret_cast<const char *>(lhs),
-             reinterpret_cast<const char *>(rhs), size);
-}
-)";
-  codegen::configurations::Serialize(Stream, FunctionType::MEMCMP, Descriptors);
-  codegen::configurations::Serialize(Stream, FunctionType::BCMP, Descriptors);
-  Stream << R"(
-using MemsetStub = void (*)(char *, int, size_t);
-template <MemsetStub Foo> void *Wrap(void *dst, int value, size_t size) {
-  Foo(reinterpret_cast<char *>(dst), value, size);
-  return dst;
-}
-)";
-  codegen::configurations::Serialize(Stream, FunctionType::MEMSET, Descriptors);
-  Stream << R"(
-using BzeroStub = void (*)(char *, size_t);
-template <BzeroStub Foo> void Wrap(void *dst, size_t size) {
-  Foo(reinterpret_cast<char *>(dst), size);
-}
-)";
-  codegen::configurations::Serialize(Stream, FunctionType::BZERO, Descriptors);
-  Stream << R"(
-llvm::ArrayRef<MemmoveConfiguration> getMemmoveConfigurations() {
-  return {};
-}
-)";
-  Stream << "// Functions : " << Descriptors.size() << "\n";
-}
-
-} // namespace codegen
-
-// Stores `VolatileStr` into a cache and returns a StringRef of the cached
-// version.
-StringRef getInternalizedString(std::string VolatileStr) {
-  static llvm::StringSet StringCache;
-  return StringCache.insert(std::move(VolatileStr)).first->getKey();
-}
-
-static StringRef getString(FunctionType FT) {
-  switch (FT) {
-  case FunctionType::MEMCPY:
-    return "memcpy";
-  case FunctionType::MEMCMP:
-    return "memcmp";
-  case FunctionType::BCMP:
-    return "bcmp";
-  case FunctionType::MEMSET:
-    return "memset";
-  case FunctionType::BZERO:
-    return "bzero";
-  }
-}
-
-void Serialize(raw_ostream &Stream, ArrayRef<FunctionDescriptor> Descriptors) {
-  std::vector<NamedFunctionDescriptor> FunctionDescriptors;
-  FunctionDescriptors.reserve(Descriptors.size());
-  for (auto &FD : Descriptors) {
-    FunctionDescriptors.emplace_back();
-    FunctionDescriptors.back().Name = getInternalizedString(
-        formatv("{0}_{1:X16}", getString(FD.Type), FD.id()));
-    FunctionDescriptors.back().Desc = std::move(FD);
-  }
-  // Sort functions so they are easier to spot in the generated C++ file.
-  std::sort(FunctionDescriptors.begin(), FunctionDescriptors.end(),
-            [](const NamedFunctionDescriptor &A,
-               const NamedFunctionDescriptor &B) { return A.Desc < B.Desc; });
-  codegen::Serialize(Stream, FunctionDescriptors);
-}
-
-} // namespace automemcpy
-} // namespace llvm
diff --git a/libc/benchmarks/automemcpy/lib/CodeGenMain.cpp b/libc/benchmarks/automemcpy/lib/CodeGenMain.cpp
deleted file mode 100644
index 3f4e6fc0423a1..0000000000000
--- a/libc/benchmarks/automemcpy/lib/CodeGenMain.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-#include "automemcpy/CodeGen.h"
-#include "automemcpy/RandomFunctionGenerator.h"
-#include <optional>
-#include <unordered_set>
-
-namespace llvm {
-namespace automemcpy {
-
-std::vector<FunctionDescriptor> generateFunctionDescriptors() {
-  std::unordered_set<FunctionDescriptor, FunctionDescriptor::Hasher> Seen;
-  std::vector<FunctionDescriptor> FunctionDescriptors;
-  RandomFunctionGenerator P;
-  while (std::optional<FunctionDescriptor> MaybeFD = P.next()) {
-    FunctionDescriptor FD = *MaybeFD;
-    if (Seen.count(FD)) // FIXME: Z3 sometimes returns twice the same object.
-      continue;
-    Seen.insert(FD);
-    FunctionDescriptors.push_back(std::move(FD));
-  }
-  return FunctionDescriptors;
-}
-
-} // namespace automemcpy
-} // namespace llvm
-
-int main(int, char **) {
-  llvm::automemcpy::Serialize(llvm::outs(),
-                              llvm::automemcpy::generateFunctionDescriptors());
-}
diff --git a/libc/benchmarks/automemcpy/lib/RandomFunctionGenerator.cpp b/libc/benchmarks/automemcpy/lib/RandomFunctionGenerator.cpp
deleted file mode 100644
index f438e2a405bd0..0000000000000
--- a/libc/benchmarks/automemcpy/lib/RandomFunctionGenerator.cpp
+++ /dev/null
@@ -1,280 +0,0 @@
-//===-- Generate random but valid function descriptors  -------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "automemcpy/RandomFunctionGenerator.h"
-
-#include <llvm/ADT/StringRef.h>
-#include <llvm/Support/raw_ostream.h>
-
-#include <optional>
-#include <set>
-
-namespace llvm {
-namespace automemcpy {
-
-// Exploration parameters
-// ----------------------
-// Here we define a set of values that will contraint the exploration and
-// limit combinatorial explosion.
-
-// We limit the number of cases for individual sizes to sizes up to 4.
-// More individual sizes don't bring much over the overlapping strategy.
-static constexpr int kMaxIndividualSize = 4;
-
-// We limit Overlapping Strategy to sizes up to 256.
-// An overlap of 256B means accessing 128B at once which is usually not
-// feasible by current CPUs. We rely on the compiler to generate multiple
-// loads/stores if needed but higher sizes are unlikely to benefit from hardware
-// acceleration.
-static constexpr int kMaxOverlapSize = 256;
-
-// For the loop strategies, we make sure that they iterate at least a certain
-// number of times to amortize the cost of looping.
-static constexpr int kLoopMinIter = 3;
-static constexpr int kAlignedLoopMinIter = 2;
-
-// We restrict the size of the block of data to handle in a loop.
-// Generally speaking block size <= 16 perform poorly.
-static constexpr int kLoopBlockSize[] = {16, 32, 64};
-
-// We restrict alignment to the following values.
-static constexpr int kLoopAlignments[] = {16, 32, 64};
-
-// We make sure that the region bounds are one of the following values.
-static constexpr int kAnchors[] = {0,  1,  2,   4,   8,   16,   32,      48,
-                                   64, 96, 128, 256, 512, 1024, kMaxSize};
-
-// We also allow disabling loops, aligned loops and accelerators.
-static constexpr bool kDisableLoop = false;
-static constexpr bool kDisableAlignedLoop = false;
-static constexpr bool kDisableAccelerator = false;
-
-// For memcpy, we can also explore whether aligning on source or destination has
-// an effect.
-static constexpr bool kExploreAlignmentArg = true;
-
-// The function we generate code for.
-// BCMP is specifically disabled for now.
-static constexpr int kFunctionTypes[] = {
-    (int)FunctionType::MEMCPY,
-    (int)FunctionType::MEMCMP,
-    //  (int)FunctionType::BCMP,
-    (int)FunctionType::MEMSET,
-    (int)FunctionType::BZERO,
-};
-
-// The actual implementation of each function can be handled via primitive types
-// (SCALAR), vector types where available (NATIVE) or by the compiler (BUILTIN).
-// We want to move toward delegating the code generation entirely to the
-// compiler but for now we have to make use of -per microarchitecture- custom
-// implementations. Scalar being more portable but also less performant, we
-// remove it as well.
-static constexpr int kElementClasses[] = {
-    // (int)ElementTypeClass::SCALAR,
-    (int)ElementTypeClass::NATIVE,
-    // (int)ElementTypeClass::BUILTIN
-};
-
-RandomFunctionGenerator::RandomFunctionGenerator()
-    : Solver(Context), Type(Context.int_const("Type")),
-      ContiguousBegin(Context.int_const("ContiguousBegin")),
-      ContiguousEnd(Context.int_const("ContiguousEnd")),
-      OverlapBegin(Context.int_const("OverlapBegin")),
-      OverlapEnd(Context.int_const("OverlapEnd")),
-      LoopBegin(Context.int_const("LoopBegin")),
-      LoopEnd(Context.int_const("LoopEnd")),
-      LoopBlockSize(Context.int_const("LoopBlockSize")),
-      AlignedLoopBegin(Context.int_const("AlignedLoopBegin")),
-      AlignedLoopEnd(Context.int_const("AlignedLoopEnd")),
-      AlignedLoopBlockSize(Context.int_const("AlignedLoopBlockSize")),
-      AlignedAlignment(Context.int_const("AlignedAlignment")),
-      AlignedArg(Context.int_const("AlignedArg")),
-      AcceleratorBegin(Context.int_const("AcceleratorBegin")),
-      AcceleratorEnd(Context.int_const("AcceleratorEnd")),
-      ElementClass(Context.int_const("ElementClass")) {
-  // All possible functions.
-  Solver.add(inSetConstraint(Type, kFunctionTypes));
-
-  // Add constraints for region bounds.
-  addBoundsAndAnchors(ContiguousBegin, ContiguousEnd);
-  addBoundsAndAnchors(OverlapBegin, OverlapEnd);
-  addBoundsAndAnchors(LoopBegin, LoopEnd);
-  addBoundsAndAnchors(AlignedLoopBegin, AlignedLoopEnd);
-  addBoundsAndAnchors(AcceleratorBegin, AcceleratorEnd);
-  // We always consider strategies in this order, and we
-  // always end with the `Accelerator` strategy, as it's typically more
-  // efficient for large sizes.
-  // Contiguous <= Overlap <= Loop <= AlignedLoop <= Accelerator
-  Solver.add(ContiguousEnd == OverlapBegin);
-  Solver.add(OverlapEnd == LoopBegin);
-  Solver.add(LoopEnd == AlignedLoopBegin);
-  Solver.add(AlignedLoopEnd == AcceleratorBegin);
-  // Fix endpoints: The minimum size that we want to copy is 0, and we always
-  // start with the `Contiguous` strategy. The max size is `kMaxSize`.
-  Solver.add(ContiguousBegin == 0);
-  Solver.add(AcceleratorEnd == kMaxSize);
-  // Contiguous
-  Solver.add(ContiguousEnd <= kMaxIndividualSize + 1);
-  // Overlap
-  Solver.add(OverlapEnd <= kMaxOverlapSize + 1);
-  // Overlap only ever makes sense when accessing multiple bytes at a time.
-  // i.e. Overlap<1> is useless.
-  Solver.add(OverlapBegin == OverlapEnd || OverlapBegin >= 2);
-  // Loop
-  addLoopConstraints(LoopBegin, LoopEnd, LoopBlockSize, kLoopMinIter);
-  // Aligned Loop
-  addLoopConstraints(AlignedLoopBegin, AlignedLoopEnd, AlignedLoopBlockSize,
-                     kAlignedLoopMinIter);
-  Solver.add(inSetConstraint(AlignedAlignment, kLoopAlignments));
-  Solver.add(AlignedLoopBegin == AlignedLoopEnd || AlignedLoopBegin >= 64);
-  Solver.add(AlignedLoopBlockSize >= AlignedAlignment);
-  Solver.add(AlignedLoopBlockSize >= LoopBlockSize);
-  z3::expr IsMemcpy = Type == (int)FunctionType::MEMCPY;
-  z3::expr ExploreAlignment = IsMemcpy && kExploreAlignmentArg;
-  Solver.add(
-      (ExploreAlignment &&
-       inSetConstraint(AlignedArg, {(int)AlignArg::_1, (int)AlignArg::_2})) ||
-      (!ExploreAlignment && AlignedArg == (int)AlignArg::_1));
-  // Accelerator
-  Solver.add(IsMemcpy ||
-             (AcceleratorBegin ==
-              AcceleratorEnd)); // Only Memcpy has accelerator for now.
-  // Element classes
-  Solver.add(inSetConstraint(ElementClass, kElementClasses));
-
-  if (kDisableLoop)
-    Solver.add(LoopBegin == LoopEnd);
-  if (kDisableAlignedLoop)
-    Solver.add(AlignedLoopBegin == AlignedLoopEnd);
-  if (kDisableAccelerator)
-    Solver.add(AcceleratorBegin == AcceleratorEnd);
-}
-
-// Creates SizeSpan from Begin/End values.
-// Returns std::nullopt if Begin==End.
-static std::optional<SizeSpan> AsSizeSpan(size_t Begin, size_t End) {
-  if (Begin == End)
-    return std::nullopt;
-  SizeSpan SS;
-  SS.Begin = Begin;
-  SS.End = End;
-  return SS;
-}
-
-// Generic method to create a `Region` struct with a Span or std::nullopt if
-// span is empty.
-template <typename Region>
-static std::optional<Region> As(size_t Begin, size_t End) {
-  if (auto Span = AsSizeSpan(Begin, End)) {
-    Region Output;
-    Output.Span = *Span;
-    return Output;
-  }
-  return std::nullopt;
-}
-
-// Returns a Loop struct or std::nullopt if span is empty.
-static std::optional<Loop> AsLoop(size_t Begin, size_t End, size_t BlockSize) {
-  if (auto Span = AsSizeSpan(Begin, End)) {
-    Loop Output;
-    Output.Span = *Span;
-    Output.BlockSize = BlockSize;
-    return Output;
-  }
-  return std::nullopt;
-}
-
-// Returns an AlignedLoop struct or std::nullopt if span is empty.
-static std::optional<AlignedLoop> AsAlignedLoop(size_t Begin, size_t End,
-                                                size_t BlockSize,
-                                                size_t Alignment,
-                                                AlignArg AlignTo) {
-  if (auto Loop = AsLoop(Begin, End, BlockSize)) {
-    AlignedLoop Output;
-    Output.Loop = *Loop;
-    Output.Alignment = Alignment;
-    Output.AlignTo = AlignTo;
-    return Output;
-  }
-  return std::nullopt;
-}
-
-std::optional<FunctionDescriptor> RandomFunctionGenerator::next() {
-  if (Solver.check() != z3::sat)
-    return {};
-
-  z3::model m = Solver.get_model();
-
-  // Helper method to get the current numerical value of a z3::expr.
-  const auto E = [&m](z3::expr &V) -> int {
-    return m.eval(V).get_numeral_int();
-  };
-
-  // Fill is the function descriptor to return.
-  FunctionDescriptor R;
-  R.Type = FunctionType(E(Type));
-  R.Contiguous = As<Contiguous>(E(ContiguousBegin), E(ContiguousEnd));
-  R.Overlap = As<Overlap>(E(OverlapBegin), E(OverlapEnd));
-  R.Loop = AsLoop(E(LoopBegin), E(LoopEnd), E(LoopBlockSize));
-  R.AlignedLoop = AsAlignedLoop(E(AlignedLoopBegin), E(AlignedLoopEnd),
-                                E(AlignedLoopBlockSize), E(AlignedAlignment),
-                                AlignArg(E(AlignedArg)));
-  R.Accelerator = As<Accelerator>(E(AcceleratorBegin), E(AcceleratorEnd));
-  R.ElementClass = ElementTypeClass(E(ElementClass));
-
-  // Express current state as a set of constraints.
-  z3::expr CurrentLayout =
-      (Type == E(Type)) && (ContiguousBegin == E(ContiguousBegin)) &&
-      (ContiguousEnd == E(ContiguousEnd)) &&
-      (OverlapBegin == E(OverlapBegin)) && (OverlapEnd == E(OverlapEnd)) &&
-      (LoopBegin == E(LoopBegin)) && (LoopEnd == E(LoopEnd)) &&
-      (LoopBlockSize == E(LoopBlockSize)) &&
-      (AlignedLoopBegin == E(AlignedLoopBegin)) &&
-      (AlignedLoopEnd == E(AlignedLoopEnd)) &&
-      (AlignedLoopBlockSize == E(AlignedLoopBlockSize)) &&
-      (AlignedAlignment == E(AlignedAlignment)) &&
-      (AlignedArg == E(AlignedArg)) &&
-      (AcceleratorBegin == E(AcceleratorBegin)) &&
-      (AcceleratorEnd == E(AcceleratorEnd)) &&
-      (ElementClass == E(ElementClass));
-
-  // Ask solver to never show this configuration ever again.
-  Solver.add(!CurrentLayout);
-  return R;
-}
-
-// Make sure `Variable` is one of the provided values.
-z3::expr RandomFunctionGenerator::inSetConstraint(z3::expr &Variable,
-                                                  ArrayRef<int> Values) const {
-  z3::expr_vector Args(Variable.ctx());
-  for (int Value : Values)
-    Args.push_back(Variable == Value);
-  return z3::mk_or(Args);
-}
-
-void RandomFunctionGenerator::addBoundsAndAnchors(z3::expr &Begin,
-                                                  z3::expr &End) {
-  // Begin and End are picked amongst a set of predefined values.
-  Solver.add(inSetConstraint(Begin, kAnchors));
-  Solver.add(inSetConstraint(End, kAnchors));
-  Solver.add(Begin >= 0);
-  Solver.add(Begin <= End);
-  Solver.add(End <= kMaxSize);
-}
-
-void RandomFunctionGenerator::addLoopConstraints(const z3::expr &LoopBegin,
-                                                 const z3::expr &LoopEnd,
-                                                 z3::expr &LoopBlockSize,
-                                                 int LoopMinIter) {
-  Solver.add(inSetConstraint(LoopBlockSize, kLoopBlockSize));
-  Solver.add(LoopBegin == LoopEnd ||
-             (LoopBegin > (LoopMinIter * LoopBlockSize)));
-}
-
-} // namespace automemcpy
-} // namespace llvm
diff --git a/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp b/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
deleted file mode 100644
index b134f6c83a0df..0000000000000
--- a/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-//===-- Analyze benchmark JSON files --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// This code analyzes the json file produced by the `automemcpy` binary.
-//
-// As a remainder, `automemcpy` will benchmark each autogenerated memory
-// functions against one of the predefined distributions available in the
-// `libc/benchmarks/distributions` folder.
-//
-// It works as follows:
-// - Reads one or more json files.
-// - If there are several runs for the same function and distribution, picks the
-//   median throughput (aka `BytesPerSecond`).
-// - Aggregates the throughput per distributions and scores them from worst (0)
-//   to best (1).
-// - Each distribution categorizes each function into one of the following
-//   categories: EXCELLENT, VERY_GOOD, GOOD, PASSABLE, INADEQUATE, MEDIOCRE,
-//   BAD.
-// - A process similar to the Majority Judgment voting system is used to `elect`
-//   the best function. The histogram of grades is returned so we can
-//   distinguish between functions with the same final grade. In the following
-//   example both functions grade EXCELLENT but we may prefer the second one.
-//
-//   |            | EXCELLENT | VERY_GOOD | GOOD | PASSABLE | ...
-//   |------------|-----------|-----------|------|----------| ...
-//   | Function_1 |     7     |     1     |   2  |          | ...
-//   | Function_2 |     6     |     4     |      |          | ...
-
-#include "automemcpy/ResultAnalyzer.h"
-#include "llvm/ADT/StringRef.h"
-#include <numeric>
-#include <unordered_map>
-
-namespace llvm {
-
-namespace automemcpy {
-
-StringRef Grade::getString(const GradeEnum &GE) {
-  switch (GE) {
-  case EXCELLENT:
-    return "EXCELLENT";
-  case VERY_GOOD:
-    return "VERY_GOOD";
-  case GOOD:
-    return "GOOD";
-  case PASSABLE:
-    return "PASSABLE";
-  case INADEQUATE:
-    return "INADEQUATE";
-  case MEDIOCRE:
-    return "MEDIOCRE";
-  case BAD:
-    return "BAD";
-  case ARRAY_SIZE:
-    report_fatal_error("logic error");
-  }
-}
-
-Grade::GradeEnum Grade::judge(double Score) {
-  if (Score >= 6. / 7)
-    return EXCELLENT;
-  if (Score >= 5. / 7)
-    return VERY_GOOD;
-  if (Score >= 4. / 7)
-    return GOOD;
-  if (Score >= 3. / 7)
-    return PASSABLE;
-  if (Score >= 2. / 7)
-    return INADEQUATE;
-  if (Score >= 1. / 7)
-    return MEDIOCRE;
-  return BAD;
-}
-
-static double computeUnbiasedSampleVariance(const std::vector<double> &Samples,
-                                            const double SampleMean) {
-  assert(!Samples.empty());
-  if (Samples.size() == 1)
-    return 0;
-  double DiffSquaresSum = 0;
-  for (const double S : Samples) {
-    const double Diff = S - SampleMean;
-    DiffSquaresSum += Diff * Diff;
-  }
-  return DiffSquaresSum / (Samples.size() - 1);
-}
-
-static void processPerDistributionData(PerDistributionData &Data) {
-  auto &Samples = Data.BytesPerSecondSamples;
-  assert(!Samples.empty());
-  // Sample Mean
-  const double Sum = std::accumulate(Samples.begin(), Samples.end(), 0.0);
-  Data.BytesPerSecondMean = Sum / Samples.size();
-  // Unbiased Sample Variance
-  Data.BytesPerSecondVariance =
-      computeUnbiasedSampleVariance(Samples, Data.BytesPerSecondMean);
-  // Median
-  const size_t HalfSize = Samples.size() / 2;
-  std::nth_element(Samples.begin(), Samples.begin() + HalfSize, Samples.end());
-  Data.BytesPerSecondMedian = Samples[HalfSize];
-}
-
-std::vector<FunctionData> getThroughputs(ArrayRef<Sample> Samples) {
-  std::unordered_map<FunctionId, FunctionData, FunctionId::Hasher> Functions;
-  for (const auto &S : Samples) {
-    if (S.Type != SampleType::ITERATION)
-      break;
-    auto &Function = Functions[S.Id.Function];
-    auto &Data = Function.PerDistributionData[S.Id.Distribution.Name];
-    Data.BytesPerSecondSamples.push_back(S.BytesPerSecond);
-  }
-
-  std::vector<FunctionData> Output;
-  for (auto &[FunctionId, Function] : Functions) {
-    Function.Id = FunctionId;
-    for (auto &Pair : Function.PerDistributionData)
-      processPerDistributionData(Pair.second);
-    Output.push_back(std::move(Function));
-  }
-  return Output;
-}
-
-void fillScores(MutableArrayRef<FunctionData> Functions) {
-  // A key to bucket throughput per function type and distribution.
-  struct Key {
-    FunctionType Type;
-    StringRef Distribution;
-
-    COMPARABLE_AND_HASHABLE(Key, Type, Distribution)
-  };
-
-  // Tracks minimum and maximum values.
-  struct MinMax {
-    double Min = std::numeric_limits<double>::max();
-    double Max = std::numeric_limits<double>::min();
-    void update(double Value) {
-      if (Value < Min)
-        Min = Value;
-      if (Value > Max)
-        Max = Value;
-    }
-    double normalize(double Value) const { return (Value - Min) / (Max - Min); }
-  };
-
-  std::unordered_map<Key, MinMax, Key::Hasher> ThroughputMinMax;
-  for (const auto &Function : Functions) {
-    const FunctionType Type = Function.Id.Type;
-    for (const auto &Pair : Function.PerDistributionData) {
-      const auto &Distribution = Pair.getKey();
-      const double Throughput = Pair.getValue().BytesPerSecondMedian;
-      const Key K{Type, Distribution};
-      ThroughputMinMax[K].update(Throughput);
-    }
-  }
-
-  for (auto &Function : Functions) {
-    const FunctionType Type = Function.Id.Type;
-    for (const auto &Pair : Function.PerDistributionData) {
-      const auto &Distribution = Pair.getKey();
-      const double Throughput = Pair.getValue().BytesPerSecondMedian;
-      const Key K{Type, Distribution};
-      Function.PerDistributionData[Distribution].Score =
-          ThroughputMinMax[K].normalize(Throughput);
-    }
-  }
-}
-
-void castVotes(MutableArrayRef<FunctionData> Functions) {
-  for (FunctionData &Function : Functions) {
-    Function.ScoresGeoMean = 1.0;
-    for (const auto &Pair : Function.PerDistributionData) {
-      const StringRef Distribution = Pair.getKey();
-      const double Score = Pair.getValue().Score;
-      Function.ScoresGeoMean *= Score;
-      const auto G = Grade::judge(Score);
-      ++(Function.GradeHisto[G]);
-      Function.PerDistributionData[Distribution].Grade = G;
-    }
-  }
-
-  for (FunctionData &Function : Functions) {
-    const auto &GradeHisto = Function.GradeHisto;
-    const size_t Votes =
-        std::accumulate(GradeHisto.begin(), GradeHisto.end(), 0U);
-    const size_t MedianVote = Votes / 2;
-    size_t CountedVotes = 0;
-    Grade::GradeEnum MedianGrade = Grade::BAD;
-    for (size_t I = 0; I < GradeHisto.size(); ++I) {
-      CountedVotes += GradeHisto[I];
-      if (CountedVotes > MedianVote) {
-        MedianGrade = Grade::GradeEnum(I);
-        break;
-      }
-    }
-    Function.FinalGrade = MedianGrade;
-  }
-}
-
-} // namespace automemcpy
-} // namespace llvm
diff --git a/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp b/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
deleted file mode 100644
index 00eef73a3f38a..0000000000000
--- a/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
+++ /dev/null
@@ -1,175 +0,0 @@
-//===-- Application to analyze benchmark JSON files -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "automemcpy/ResultAnalyzer.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/JSON.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-namespace llvm {
-
-// User can specify one or more json filenames to process on the command line.
-static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
-                                            cl::desc("<input json files>"));
-
-// User can filter the distributions to be taken into account.
-static cl::list<std::string>
-    KeepOnlyDistributions("keep-only-distributions",
-                          cl::desc("<comma separated list of distribution "
-                                   "names, keeps all if unspecified>"));
-
-namespace automemcpy {
-
-// This is defined in the autogenerated 'Implementations.cpp' file.
-extern ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors();
-
-// Iterates over all functions and fills a map of function name to function
-// descriptor pointers.
-static StringMap<const FunctionDescriptor *> createFunctionDescriptorMap() {
-  StringMap<const FunctionDescriptor *> Descriptors;
-  for (const NamedFunctionDescriptor &FD : getFunctionDescriptors())
-    Descriptors.insert_or_assign(FD.Name, &FD.Desc);
-  return Descriptors;
-}
-
-// Retrieves the function descriptor for a particular function name.
-static const FunctionDescriptor &getFunctionDescriptor(StringRef FunctionName) {
-  static StringMap<const FunctionDescriptor *> Descriptors =
-      createFunctionDescriptorMap();
-  const auto *FD = Descriptors.lookup(FunctionName);
-  if (!FD)
-    report_fatal_error(
-        Twine("No FunctionDescriptor for ").concat(FunctionName));
-  return *FD;
-}
-
-// Functions and distributions names are stored quite a few times so it's more
-// efficient to internalize these strings and refer to them through 'StringRef'.
-static StringRef getInternalizedString(StringRef VolatileStr) {
-  static llvm::StringSet StringCache;
-  return StringCache.insert(VolatileStr).first->getKey();
-}
-
-// Helper function for the LLVM JSON API.
-bool fromJSON(const json::Value &V, Sample &Out, json::Path P) {
-  std::string Label;
-  std::string RunType;
-  json::ObjectMapper O(V, P);
-  if (O && O.map("bytes_per_second", Out.BytesPerSecond) &&
-      O.map("run_type", RunType) && O.map("label", Label)) {
-    const auto LabelPair = StringRef(Label).split(',');
-    Out.Id.Function.Name = getInternalizedString(LabelPair.first);
-    Out.Id.Function.Type = getFunctionDescriptor(LabelPair.first).Type;
-    Out.Id.Distribution.Name = getInternalizedString(LabelPair.second);
-    Out.Type = StringSwitch<SampleType>(RunType)
-                   .Case("aggregate", SampleType::AGGREGATE)
-                   .Case("iteration", SampleType::ITERATION);
-    return true;
-  }
-  return false;
-}
-
-// An object to represent the content of the JSON file.
-// This is easier to parse/serialize JSON when the structures of the json file
-// maps the structure of the object.
-struct JsonFile {
-  std::vector<Sample> Samples;
-};
-
-// Helper function for the LLVM JSON API.
-bool fromJSON(const json::Value &V, JsonFile &JF, json::Path P) {
-  json::ObjectMapper O(V, P);
-  return O && O.map("benchmarks", JF.Samples);
-}
-
-// Global object to ease error reporting, it consumes errors and crash the
-// application with a meaningful message.
-static ExitOnError ExitOnErr;
-
-// Main JSON parsing method. Reads the content of the file pointed to by
-// 'Filename' and returns a JsonFile object.
-JsonFile parseJsonResultFile(StringRef Filename) {
-  auto Buf = ExitOnErr(errorOrToExpected(
-      MemoryBuffer::getFile(Filename, /*bool IsText=*/true,
-                            /*RequiresNullTerminator=*/false)));
-  auto JsonValue = ExitOnErr(json::parse(Buf->getBuffer()));
-  json::Path::Root Root;
-  JsonFile JF;
-  if (!fromJSON(JsonValue, JF, Root))
-    ExitOnErr(Root.getError());
-  return JF;
-}
-
-// Serializes the 'GradeHisto' to the provided 'Stream'.
-static void Serialize(raw_ostream &Stream, const GradeHistogram &GH) {
-  static constexpr std::array<StringRef, 9> kCharacters = {
-      " ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
-
-  const size_t Max = *std::max_element(GH.begin(), GH.end());
-  for (size_t I = 0; I < GH.size(); ++I) {
-    size_t Index = (float(GH[I]) / Max) * (kCharacters.size() - 1);
-    Stream << kCharacters.at(Index);
-  }
-}
-
-int Main(int argc, char **argv) {
-  ExitOnErr.setBanner("Automemcpy Json Results Analyzer stopped with error: ");
-  cl::ParseCommandLineOptions(argc, argv, "Automemcpy Json Results Analyzer\n");
-
-  // Reads all samples stored in the input JSON files.
-  std::vector<Sample> Samples;
-  for (const auto &Filename : InputFilenames) {
-    auto Result = parseJsonResultFile(Filename);
-    llvm::append_range(Samples, Result.Samples);
-  }
-
-  if (!KeepOnlyDistributions.empty()) {
-    llvm::StringSet ValidDistributions;
-    ValidDistributions.insert(KeepOnlyDistributions.begin(),
-                              KeepOnlyDistributions.end());
-    llvm::erase_if(Samples, [&ValidDistributions](const Sample &S) {
-      return !ValidDistributions.contains(S.Id.Distribution.Name);
-    });
-  }
-
-  // Extracts median of throughputs.
-  std::vector<FunctionData> Functions = getThroughputs(Samples);
-  fillScores(Functions);
-  castVotes(Functions);
-
-  // Present data by function type, Grade and Geomean of scores.
-  std::sort(Functions.begin(), Functions.end(),
-            [](const FunctionData &A, const FunctionData &B) {
-              const auto Less = [](const FunctionData &FD) {
-                return std::make_tuple(FD.Id.Type, FD.FinalGrade,
-                                       -FD.ScoresGeoMean);
-              };
-              return Less(A) < Less(B);
-            });
-
-  // Print result.
-  for (const FunctionData &Function : Functions) {
-    outs() << formatv("{0,-10}", Grade::getString(Function.FinalGrade));
-    outs() << " |";
-    Serialize(outs(), Function.GradeHisto);
-    outs() << "| ";
-    outs().resetColor();
-    outs() << formatv("{0,+25}", Function.Id.Name);
-    outs() << "\n";
-  }
-
-  return EXIT_SUCCESS;
-}
-
-} // namespace automemcpy
-} // namespace llvm
-
-int main(int argc, char **argv) { return llvm::automemcpy::Main(argc, argv); }
diff --git a/libc/benchmarks/automemcpy/unittests/CMakeLists.txt b/libc/benchmarks/automemcpy/unittests/CMakeLists.txt
deleted file mode 100644
index 35caaac1519ba..0000000000000
--- a/libc/benchmarks/automemcpy/unittests/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-add_libc_benchmark_unittest(libc-automemcpy-codegen-test
-    SRCS CodeGenTest.cpp
-    DEPENDS automemcpy_codegen
-)
-
-add_libc_benchmark_unittest(libc-automemcpy-result-analyzer-test
-    SRCS ResultAnalyzerTest.cpp
-    DEPENDS automemcpy_result_analyzer_lib
-)
diff --git a/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp b/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
deleted file mode 100644
index a7fc8570a73b0..0000000000000
--- a/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
+++ /dev/null
@@ -1,226 +0,0 @@
-//===-- Automemcpy CodeGen Test -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "automemcpy/CodeGen.h"
-#include "automemcpy/RandomFunctionGenerator.h"
-#include "src/__support/macros/config.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include <optional>
-
-using testing::AllOf;
-using testing::AnyOf;
-using testing::ElementsAre;
-using testing::Ge;
-using testing::Gt;
-using testing::Le;
-using testing::Lt;
-
-namespace llvm {
-namespace automemcpy {
-namespace {
-
-TEST(Automemcpy, Codegen) {
-  static constexpr FunctionDescriptor kDescriptors[] = {
-      {FunctionType::MEMCPY, std::nullopt, std::nullopt, std::nullopt, std::nullopt,
-       Accelerator{{0, kMaxSize}}, ElementTypeClass::NATIVE},
-      {FunctionType::MEMCPY, Contiguous{{0, 4}}, Overlap{{4, 256}},
-       Loop{{256, kMaxSize}, 64}, std::nullopt, std::nullopt,
-       ElementTypeClass::NATIVE},
-      {FunctionType::MEMCMP, Contiguous{{0, 2}}, Overlap{{2, 64}}, std::nullopt,
-       AlignedLoop{Loop{{64, kMaxSize}, 16}, 16, AlignArg::_1}, std::nullopt,
-       ElementTypeClass::NATIVE},
-      {FunctionType::MEMSET, Contiguous{{0, 2}}, Overlap{{2, 256}}, std::nullopt,
-       AlignedLoop{Loop{{256, kMaxSize}, 32}, 16, AlignArg::_1}, std::nullopt,
-       ElementTypeClass::NATIVE},
-      {FunctionType::MEMSET, Contiguous{{0, 2}}, Overlap{{2, 256}}, std::nullopt,
-       AlignedLoop{Loop{{256, kMaxSize}, 32}, 32, AlignArg::_1}, std::nullopt,
-       ElementTypeClass::NATIVE},
-      {FunctionType::BZERO, Contiguous{{0, 4}}, Overlap{{4, 128}}, std::nullopt,
-       AlignedLoop{Loop{{128, kMaxSize}, 32}, 32, AlignArg::_1}, std::nullopt,
-       ElementTypeClass::NATIVE},
-  };
-
-  std::string Output;
-  raw_string_ostream OutputStream(Output);
-  Serialize(OutputStream, kDescriptors);
-
-  EXPECT_STREQ(Output.c_str(),
-               R"(// This file is auto-generated by libc/benchmarks/automemcpy.
-// Functions : 6
-
-#include "LibcFunctionPrototypes.h"
-#include "automemcpy/FunctionDescriptor.h"
-#include "src/string/memory_utils/elements.h"
-
-using llvm::libc_benchmarks::BzeroConfiguration;
-using llvm::libc_benchmarks::MemcmpOrBcmpConfiguration;
-using llvm::libc_benchmarks::MemcpyConfiguration;
-using llvm::libc_benchmarks::MemmoveConfiguration;
-using llvm::libc_benchmarks::MemsetConfiguration;
-
-namespace LIBC_NAMESPACE_DECL {
-
-static void memcpy_0xE00E29EE73994E2B(char *__restrict dst, const char *__restrict src, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  return copy<Accelerator>(dst, src, size);
-}
-static void memcpy_0x7381B60C7BE75EF9(char *__restrict dst, const char *__restrict src, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  if(size == 0) return;
-  if(size == 1) return copy<_1>(dst, src);
-  if(size == 2) return copy<_2>(dst, src);
-  if(size == 3) return copy<_3>(dst, src);
-  if(size < 8) return copy<HeadTail<_4>>(dst, src, size);
-  if(size < 16) return copy<HeadTail<_8>>(dst, src, size);
-  if(size < 32) return copy<HeadTail<_16>>(dst, src, size);
-  if(size < 64) return copy<HeadTail<_32>>(dst, src, size);
-  if(size < 128) return copy<HeadTail<_64>>(dst, src, size);
-  if(size < 256) return copy<HeadTail<_128>>(dst, src, size);
-  return copy<Loop<_64>>(dst, src, size);
-}
-static int memcmp_0x348D7BA6DB0EE033(const char * lhs, const char * rhs, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  if(size == 0) return 0;
-  if(size == 1) return three_way_compare<_1>(lhs, rhs);
-  if(size < 4) return three_way_compare<HeadTail<_2>>(lhs, rhs, size);
-  if(size < 8) return three_way_compare<HeadTail<_4>>(lhs, rhs, size);
-  if(size < 16) return three_way_compare<HeadTail<_8>>(lhs, rhs, size);
-  if(size < 32) return three_way_compare<HeadTail<_16>>(lhs, rhs, size);
-  if(size < 64) return three_way_compare<HeadTail<_32>>(lhs, rhs, size);
-  return three_way_compare<Align<_16,Arg::Lhs>::Then<Loop<_16>>>(lhs, rhs, size);
-}
-static void memset_0x71E761699B999863(char * dst, int value, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  if(size == 0) return;
-  if(size == 1) return splat_set<_1>(dst, value);
-  if(size < 4) return splat_set<HeadTail<_2>>(dst, value, size);
-  if(size < 8) return splat_set<HeadTail<_4>>(dst, value, size);
-  if(size < 16) return splat_set<HeadTail<_8>>(dst, value, size);
-  if(size < 32) return splat_set<HeadTail<_16>>(dst, value, size);
-  if(size < 64) return splat_set<HeadTail<_32>>(dst, value, size);
-  if(size < 128) return splat_set<HeadTail<_64>>(dst, value, size);
-  if(size < 256) return splat_set<HeadTail<_128>>(dst, value, size);
-  return splat_set<Align<_16,Arg::Dst>::Then<Loop<_32>>>(dst, value, size);
-}
-static void memset_0x3DF0F44E2ED6A50F(char * dst, int value, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  if(size == 0) return;
-  if(size == 1) return splat_set<_1>(dst, value);
-  if(size < 4) return splat_set<HeadTail<_2>>(dst, value, size);
-  if(size < 8) return splat_set<HeadTail<_4>>(dst, value, size);
-  if(size < 16) return splat_set<HeadTail<_8>>(dst, value, size);
-  if(size < 32) return splat_set<HeadTail<_16>>(dst, value, size);
-  if(size < 64) return splat_set<HeadTail<_32>>(dst, value, size);
-  if(size < 128) return splat_set<HeadTail<_64>>(dst, value, size);
-  if(size < 256) return splat_set<HeadTail<_128>>(dst, value, size);
-  return splat_set<Align<_32,Arg::Dst>::Then<Loop<_32>>>(dst, value, size);
-}
-static void bzero_0x475977492C218AD4(char * dst, size_t size) {
-  using namespace LIBC_NAMESPACE::x86;
-  if(size == 0) return;
-  if(size == 1) return splat_set<_1>(dst, 0);
-  if(size == 2) return splat_set<_2>(dst, 0);
-  if(size == 3) return splat_set<_3>(dst, 0);
-  if(size < 8) return splat_set<HeadTail<_4>>(dst, 0, size);
-  if(size < 16) return splat_set<HeadTail<_8>>(dst, 0, size);
-  if(size < 32) return splat_set<HeadTail<_16>>(dst, 0, size);
-  if(size < 64) return splat_set<HeadTail<_32>>(dst, 0, size);
-  if(size < 128) return splat_set<HeadTail<_64>>(dst, 0, size);
-  return splat_set<Align<_32,Arg::Dst>::Then<Loop<_32>>>(dst, 0, size);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
-
-namespace llvm {
-namespace automemcpy {
-
-ArrayRef<NamedFunctionDescriptor> getFunctionDescriptors() {
-  static constexpr NamedFunctionDescriptor kDescriptors[] = {
-    {"memcpy_0xE00E29EE73994E2B",{FunctionType::MEMCPY,std::nullopt,std::nullopt,std::nullopt,std::nullopt,Accelerator{{0,kMaxSize}},ElementTypeClass::NATIVE}},
-    {"memcpy_0x7381B60C7BE75EF9",{FunctionType::MEMCPY,Contiguous{{0,4}},Overlap{{4,256}},Loop{{256,kMaxSize},64},std::nullopt,std::nullopt,ElementTypeClass::NATIVE}},
-    {"memcmp_0x348D7BA6DB0EE033",{FunctionType::MEMCMP,Contiguous{{0,2}},Overlap{{2,64}},std::nullopt,AlignedLoop{Loop{{64,kMaxSize},16},16,AlignArg::_1},std::nullopt,ElementTypeClass::NATIVE}},
-    {"memset_0x71E761699B999863",{FunctionType::MEMSET,Contiguous{{0,2}},Overlap{{2,256}},std::nullopt,AlignedLoop{Loop{{256,kMaxSize},32},16,AlignArg::_1},std::nullopt,ElementTypeClass::NATIVE}},
-    {"memset_0x3DF0F44E2ED6A50F",{FunctionType::MEMSET,Contiguous{{0,2}},Overlap{{2,256}},std::nullopt,AlignedLoop{Loop{{256,kMaxSize},32},32,AlignArg::_1},std::nullopt,ElementTypeClass::NATIVE}},
-    {"bzero_0x475977492C218AD4",{FunctionType::BZERO,Contiguous{{0,4}},Overlap{{4,128}},std::nullopt,AlignedLoop{Loop{{128,kMaxSize},32},32,AlignArg::_1},std::nullopt,ElementTypeClass::NATIVE}},
-  };
-  return ArrayRef(kDescriptors);
-}
-
-} // namespace automemcpy
-} // namespace llvm
-
-
-using MemcpyStub = void (*)(char *__restrict, const char *__restrict, size_t);
-template <MemcpyStub Foo>
-void *Wrap(void *__restrict dst, const void *__restrict src, size_t size) {
-  Foo(reinterpret_cast<char *__restrict>(dst),
-      reinterpret_cast<const char *__restrict>(src), size);
-  return dst;
-}
-llvm::ArrayRef<MemcpyConfiguration> getMemcpyConfigurations() {
-  using namespace LIBC_NAMESPACE;
-  static constexpr MemcpyConfiguration kConfigurations[] = {
-    {Wrap<memcpy_0xE00E29EE73994E2B>, "memcpy_0xE00E29EE73994E2B"},
-    {Wrap<memcpy_0x7381B60C7BE75EF9>, "memcpy_0x7381B60C7BE75EF9"},
-  };
-  return llvm::ArrayRef(kConfigurations);
-}
-
-using MemcmpStub = int (*)(const char *, const char *, size_t);
-template <MemcmpStub Foo>
-int Wrap(const void *lhs, const void *rhs, size_t size) {
-  return Foo(reinterpret_cast<const char *>(lhs),
-             reinterpret_cast<const char *>(rhs), size);
-}
-llvm::ArrayRef<MemcmpOrBcmpConfiguration> getMemcmpConfigurations() {
-  using namespace LIBC_NAMESPACE;
-  static constexpr MemcmpOrBcmpConfiguration kConfigurations[] = {
-    {Wrap<memcmp_0x348D7BA6DB0EE033>, "memcmp_0x348D7BA6DB0EE033"},
-  };
-  return llvm::ArrayRef(kConfigurations);
-}
-llvm::ArrayRef<MemcmpOrBcmpConfiguration> getBcmpConfigurations() {
-  return {};
-}
-
-using MemsetStub = void (*)(char *, int, size_t);
-template <MemsetStub Foo> void *Wrap(void *dst, int value, size_t size) {
-  Foo(reinterpret_cast<char *>(dst), value, size);
-  return dst;
-}
-llvm::ArrayRef<MemsetConfiguration> getMemsetConfigurations() {
-  using namespace LIBC_NAMESPACE;
-  static constexpr MemsetConfiguration kConfigurations[] = {
-    {Wrap<memset_0x71E761699B999863>, "memset_0x71E761699B999863"},
-    {Wrap<memset_0x3DF0F44E2ED6A50F>, "memset_0x3DF0F44E2ED6A50F"},
-  };
-  return llvm::ArrayRef(kConfigurations);
-}
-
-using BzeroStub = void (*)(char *, size_t);
-template <BzeroStub Foo> void Wrap(void *dst, size_t size) {
-  Foo(reinterpret_cast<char *>(dst), size);
-}
-llvm::ArrayRef<BzeroConfiguration> getBzeroConfigurations() {
-  using namespace LIBC_NAMESPACE;
-  static constexpr BzeroConfiguration kConfigurations[] = {
-    {Wrap<bzero_0x475977492C218AD4>, "bzero_0x475977492C218AD4"},
-  };
-  return llvm::ArrayRef(kConfigurations);
-}
-
-llvm::ArrayRef<MemmoveConfiguration> getMemmoveConfigurations() {
-  return {};
-}
-// Functions : 6
-)");
-}
-} // namespace
-} // namespace automemcpy
-} // namespace llvm
diff --git a/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp b/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
deleted file mode 100644
index 7b67f70eb89cd..0000000000000
--- a/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
+++ /dev/null
@@ -1,191 +0,0 @@
-//===--  Automemcpy Json Results Analyzer Test ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "automemcpy/ResultAnalyzer.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-using testing::DoubleNear;
-using testing::ElementsAre;
-using testing::Pair;
-using testing::SizeIs;
-
-namespace llvm {
-namespace automemcpy {
-namespace {
-
-TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsOneSample) {
-  static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
-  static constexpr DistributionId DistA = {{"A"}};
-  static constexpr SampleId Id = {Foo1, DistA};
-  static constexpr Sample kSamples[] = {
-      Sample{Id, SampleType::ITERATION, 4},
-      Sample{Id, SampleType::AGGREGATE, -1}, // Aggegates gets discarded
-  };
-
-  const std::vector<FunctionData> Data = getThroughputs(kSamples);
-  EXPECT_THAT(Data, SizeIs(1));
-  EXPECT_THAT(Data[0].Id, Foo1);
-  EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
-  // A single value is provided.
-  const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
-  EXPECT_THAT(DistributionData.BytesPerSecondMedian, 4);
-  EXPECT_THAT(DistributionData.BytesPerSecondMean, 4);
-  EXPECT_THAT(DistributionData.BytesPerSecondVariance, 0);
-}
-
-TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsManySamplesSameBucket) {
-  static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
-  static constexpr DistributionId DistA = {{"A"}};
-  static constexpr SampleId Id = {Foo1, DistA};
-  static constexpr Sample kSamples[] = {Sample{Id, SampleType::ITERATION, 4},
-                                        Sample{Id, SampleType::ITERATION, 5},
-                                        Sample{Id, SampleType::ITERATION, 5}};
-
-  const std::vector<FunctionData> Data = getThroughputs(kSamples);
-  EXPECT_THAT(Data, SizeIs(1));
-  EXPECT_THAT(Data[0].Id, Foo1);
-  EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
-  // When multiple values are provided we pick the median one (here median of 4,
-  // 5, 5).
-  const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
-  EXPECT_THAT(DistributionData.BytesPerSecondMedian, 5);
-  EXPECT_THAT(DistributionData.BytesPerSecondMean, DoubleNear(4.6, 0.1));
-  EXPECT_THAT(DistributionData.BytesPerSecondVariance, DoubleNear(0.33, 0.01));
-}
-
-TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsServeralFunctionAndDist) {
-  static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
-  static constexpr DistributionId DistA = {{"A"}};
-  static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
-  static constexpr DistributionId DistB = {{"B"}};
-  static constexpr Sample kSamples[] = {
-      Sample{{Foo1, DistA}, SampleType::ITERATION, 1},
-      Sample{{Foo1, DistB}, SampleType::ITERATION, 2},
-      Sample{{Foo2, DistA}, SampleType::ITERATION, 3},
-      Sample{{Foo2, DistB}, SampleType::ITERATION, 4}};
-  // Data is aggregated per function.
-  const std::vector<FunctionData> Data = getThroughputs(kSamples);
-  EXPECT_THAT(Data, SizeIs(2)); // 2 functions Foo1 and Foo2.
-  // Each function has data for both distributions DistA and DistB.
-  EXPECT_THAT(Data[0].PerDistributionData, SizeIs(2));
-  EXPECT_THAT(Data[1].PerDistributionData, SizeIs(2));
-}
-
-TEST(AutomemcpyJsonResultsAnalyzer, getScore) {
-  static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
-  static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
-  static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
-  static constexpr DistributionId Dist = {{"A"}};
-  static constexpr Sample kSamples[] = {
-      Sample{{Foo1, Dist}, SampleType::ITERATION, 1},
-      Sample{{Foo2, Dist}, SampleType::ITERATION, 2},
-      Sample{{Foo3, Dist}, SampleType::ITERATION, 3}};
-
-  // Data is aggregated per function.
-  std::vector<FunctionData> Data = getThroughputs(kSamples);
-
-  // Sort Data by function name so we can test them.
-  std::sort(
-      Data.begin(), Data.end(),
-      [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
-
-  EXPECT_THAT(Data[0].Id, Foo1);
-  EXPECT_THAT(Data[0].PerDistributionData.lookup("A").BytesPerSecondMedian, 1);
-  EXPECT_THAT(Data[1].Id, Foo2);
-  EXPECT_THAT(Data[1].PerDistributionData.lookup("A").BytesPerSecondMedian, 2);
-  EXPECT_THAT(Data[2].Id, Foo3);
-  EXPECT_THAT(Data[2].PerDistributionData.lookup("A").BytesPerSecondMedian, 3);
-
-  // Normalizes throughput per distribution.
-  fillScores(Data);
-  EXPECT_THAT(Data[0].PerDistributionData.lookup("A").Score, 0);
-  EXPECT_THAT(Data[1].PerDistributionData.lookup("A").Score, 0.5);
-  EXPECT_THAT(Data[2].PerDistributionData.lookup("A").Score, 1);
-}
-
-TEST(AutomemcpyJsonResultsAnalyzer, castVotes) {
-  static constexpr double kAbsErr = 0.01;
-
-  static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
-  static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
-  static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
-  static constexpr DistributionId DistA = {{"A"}};
-  static constexpr DistributionId DistB = {{"B"}};
-  static constexpr Sample kSamples[] = {
-      Sample{{Foo1, DistA}, SampleType::ITERATION, 0},
-      Sample{{Foo1, DistB}, SampleType::ITERATION, 30},
-      Sample{{Foo2, DistA}, SampleType::ITERATION, 1},
-      Sample{{Foo2, DistB}, SampleType::ITERATION, 100},
-      Sample{{Foo3, DistA}, SampleType::ITERATION, 7},
-      Sample{{Foo3, DistB}, SampleType::ITERATION, 100},
-  };
-
-  // DistA Thoughput ranges from 0 to 7.
-  // DistB Thoughput ranges from 30 to 100.
-
-  // Data is aggregated per function.
-  std::vector<FunctionData> Data = getThroughputs(kSamples);
-
-  // Sort Data by function name so we can test them.
-  std::sort(
-      Data.begin(), Data.end(),
-      [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
-
-  // Normalizes throughput per distribution.
-  fillScores(Data);
-
-  // Cast votes
-  castVotes(Data);
-
-  EXPECT_THAT(Data[0].Id, Foo1);
-  EXPECT_THAT(Data[1].Id, Foo2);
-  EXPECT_THAT(Data[2].Id, Foo3);
-
-  const auto GetDistData = [&Data](size_t Index, StringRef Name) {
-    return Data[Index].PerDistributionData.lookup(Name);
-  };
-
-  // Distribution A
-  // Throughput is 0, 1 and 7, so normalized scores are 0, 1/7 and 1.
-  EXPECT_THAT(GetDistData(0, "A").Score, DoubleNear(0, kAbsErr));
-  EXPECT_THAT(GetDistData(1, "A").Score, DoubleNear(1. / 7, kAbsErr));
-  EXPECT_THAT(GetDistData(2, "A").Score, DoubleNear(1, kAbsErr));
-  // which are turned into grades BAD,  MEDIOCRE and EXCELLENT.
-  EXPECT_THAT(GetDistData(0, "A").Grade, Grade::BAD);
-  EXPECT_THAT(GetDistData(1, "A").Grade, Grade::MEDIOCRE);
-  EXPECT_THAT(GetDistData(2, "A").Grade, Grade::EXCELLENT);
-
-  // Distribution B
-  // Throughput is 30, 100 and 100, so normalized scores are 0, 1 and 1.
-  EXPECT_THAT(GetDistData(0, "B").Score, DoubleNear(0, kAbsErr));
-  EXPECT_THAT(GetDistData(1, "B").Score, DoubleNear(1, kAbsErr));
-  EXPECT_THAT(GetDistData(2, "B").Score, DoubleNear(1, kAbsErr));
-  // which are turned into grades BAD, EXCELLENT and EXCELLENT.
-  EXPECT_THAT(GetDistData(0, "B").Grade, Grade::BAD);
-  EXPECT_THAT(GetDistData(1, "B").Grade, Grade::EXCELLENT);
-  EXPECT_THAT(GetDistData(2, "B").Grade, Grade::EXCELLENT);
-
-  // Now looking from the functions point of view.
-  EXPECT_THAT(Data[0].ScoresGeoMean, DoubleNear(0, kAbsErr));
-  EXPECT_THAT(Data[1].ScoresGeoMean, DoubleNear(1. * (1. / 7), kAbsErr));
-  EXPECT_THAT(Data[2].ScoresGeoMean, DoubleNear(1, kAbsErr));
-
-  // Note the array is indexed by GradeEnum values (EXCELLENT=0 / BAD = 6)
-  EXPECT_THAT(Data[0].GradeHisto, ElementsAre(0, 0, 0, 0, 0, 0, 2));
-  EXPECT_THAT(Data[1].GradeHisto, ElementsAre(1, 0, 0, 0, 0, 1, 0));
-  EXPECT_THAT(Data[2].GradeHisto, ElementsAre(2, 0, 0, 0, 0, 0, 0));
-
-  EXPECT_THAT(Data[0].FinalGrade, Grade::BAD);
-  EXPECT_THAT(Data[1].FinalGrade, Grade::MEDIOCRE);
-  EXPECT_THAT(Data[2].FinalGrade, Grade::EXCELLENT);
-}
-
-} // namespace
-} // namespace automemcpy
-} // namespace llvm
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 65851f1c86571..8dcee1ec42246 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -1,3 +1,15 @@
+if(NOT DEFINED LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
+  if(CMAKE_COMPILER_IS_GNUCXX)
+    set(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE ON)
+  elseif( MSVC )
+    set(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE OFF)
+  elseif( "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
+    set(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE ON)
+  elseif( "${CMAKE_CXX_COMPILER_ID}" MATCHES "Intel" )
+    set(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE ON)
+  endif()
+endif()
+
 function(_get_compile_options_from_flags output_var)
   set(compile_options "")
 
@@ -8,7 +20,7 @@ function(_get_compile_options_from_flags output_var)
   check_flag(ADD_EXPLICIT_SIMD_OPT_FLAG ${EXPLICIT_SIMD_OPT_FLAG} ${ARGN})
   check_flag(ADD_MISC_MATH_BASIC_OPS_OPT_FLAG ${MISC_MATH_BASIC_OPS_OPT_FLAG} ${ARGN})
 
-  if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
+  if(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
     if(ADD_FMA_FLAG)
       if(LIBC_TARGET_ARCHITECTURE_IS_X86_64)
         list(APPEND compile_options "-mavx2")
@@ -96,7 +108,7 @@ function(_get_common_compile_options output_var flags)
 
   set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${compile_flags} ${config_flags})
 
-  if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
+  if(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fpie")
 
     if(LLVM_LIBC_FULL_BUILD)
@@ -210,7 +222,7 @@ function(_get_common_test_compile_options output_var c_test flags)
       ${LIBC_TEST_COMPILE_OPTIONS_DEFAULT}
       ${compile_flags})
 
-  if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
+  if(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fpie")
 
     if(LLVM_LIBC_FULL_BUILD)
diff --git a/libc/cmake/modules/LLVMLibCHeaderRules.cmake b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
index 76c4e1f2d3244..31a88f0ef93be 100644
--- a/libc/cmake/modules/LLVMLibCHeaderRules.cmake
+++ b/libc/cmake/modules/LLVMLibCHeaderRules.cmake
@@ -71,9 +71,9 @@ function(add_header target_name)
   )
 endfunction(add_header)
 
-function(add_gen_header2 target_name)
+function(add_gen_header target_name)
   cmake_parse_arguments(
-    "ADD_GEN_HDR2"
+    "ADD_GEN_HDR"
     "PUBLIC" # No optional arguments
     "YAML_FILE;DEF_FILE;GEN_HDR" # Single value arguments
     "DEPENDS"     # Multi value arguments
@@ -84,25 +84,25 @@ function(add_gen_header2 target_name)
     add_library(${fq_target_name} INTERFACE)
     return()
   endif()
-  if(NOT ADD_GEN_HDR2_DEF_FILE)
-    message(FATAL_ERROR "`add_gen_hdr2` rule requires DEF_FILE to be specified.")
+  if(NOT ADD_GEN_HDR_DEF_FILE)
+    message(FATAL_ERROR "`add_gen_hdr` rule requires DEF_FILE to be specified.")
   endif()
-  if(NOT ADD_GEN_HDR2_GEN_HDR)
-    message(FATAL_ERROR "`add_gen_hdr2` rule requires GEN_HDR to be specified.")
+  if(NOT ADD_GEN_HDR_GEN_HDR)
+    message(FATAL_ERROR "`add_gen_hdr` rule requires GEN_HDR to be specified.")
   endif()
-  if(NOT ADD_GEN_HDR2_YAML_FILE)
-    message(FATAL_ERROR "`add_gen_hdr2` rule requires YAML_FILE to be specified.")
+  if(NOT ADD_GEN_HDR_YAML_FILE)
+    message(FATAL_ERROR "`add_gen_hdr` rule requires YAML_FILE to be specified.")
   endif()
 
-  set(absolute_path ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR2_GEN_HDR})
+  set(absolute_path ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR_GEN_HDR})
   file(RELATIVE_PATH relative_path ${LIBC_INCLUDE_SOURCE_DIR} ${absolute_path})
   set(out_file ${LIBC_INCLUDE_DIR}/${relative_path})
-  set(yaml_file ${CMAKE_SOURCE_DIR}/${ADD_GEN_HDR2_YAML_FILE})
-  set(def_file ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR2_DEF_FILE})
+  set(yaml_file ${CMAKE_SOURCE_DIR}/${ADD_GEN_HDR_YAML_FILE})
+  set(def_file ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR_DEF_FILE})
 
   set(fq_data_files "")
-  if(ADD_GEN_HDR2_DATA_FILES)
-    foreach(data_file IN LISTS ADD_GEN_HDR2_DATA_FILES)
+  if(ADD_GEN_HDR_DATA_FILES)
+    foreach(data_file IN LISTS ADD_GEN_HDR_DATA_FILES)
       list(APPEND fq_data_files "${CMAKE_CURRENT_SOURCE_DIR}/${data_file}")
     endforeach(data_file)
   endif()
@@ -112,13 +112,13 @@ function(add_gen_header2 target_name)
 
   add_custom_command(
     OUTPUT ${out_file}
-    COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/newhdrgen/yaml_to_classes.py
+    COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/hdrgen/yaml_to_classes.py
             ${yaml_file}
             --h_def_file ${def_file}
             ${entry_points}
             --output_dir ${out_file}
     DEPENDS ${yaml_file} ${def_file} ${fq_data_files}
-    COMMENT "Generating header ${ADD_GEN_HDR2_GEN_HDR} from ${yaml_file} and ${def_file}"
+    COMMENT "Generating header ${ADD_GEN_HDR_GEN_HDR} from ${yaml_file} and ${def_file}"
   )
   if(LIBC_TARGET_OS_IS_GPU)
     file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/llvm-libc-decls)
@@ -126,7 +126,7 @@ function(add_gen_header2 target_name)
     set(decl_out_file ${LIBC_INCLUDE_DIR}/llvm-libc-decls/${relative_path})
     add_custom_command(
       OUTPUT ${decl_out_file}
-      COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/newhdrgen/yaml_to_classes.py
+      COMMAND ${Python3_EXECUTABLE} ${LIBC_SOURCE_DIR}/hdrgen/yaml_to_classes.py
               ${yaml_file}
               --export-decls
               ${entry_points}
@@ -136,132 +136,6 @@ function(add_gen_header2 target_name)
     )
   endif()
 
-  if(ADD_GEN_HDR2_DEPENDS)
-    get_fq_deps_list(fq_deps_list ${ADD_GEN_HDR2_DEPENDS})
-    # Dependencies of a add_header target can only be another add_gen_header target
-    # or an add_header target.
-    foreach(dep IN LISTS fq_deps_list)
-      get_target_property(header_file ${dep} HEADER_FILE_PATH)
-      if(NOT header_file)
-        message(FATAL_ERROR "Invalid dependency '${dep}' for '${fq_target_name}'.")
-      endif()
-    endforeach()
-  endif()
-  set(generated_hdr_target ${fq_target_name}.__generated_hdr__)
-  add_custom_target(
-    ${generated_hdr_target}
-    DEPENDS ${out_file} ${fq_deps_list} ${decl_out_file}
-  )
-
-  add_header_library(
-    ${target_name}
-    HDRS
-      ${out_file}
-  )
-
-  add_dependencies(${fq_target_name} ${generated_hdr_target})
-
-  set_target_properties(
-    ${fq_target_name}
-    PROPERTIES
-      HEADER_FILE_PATH ${out_file}
-      DECLS_FILE_PATH "${decl_out_file}"
-      DEPS "${fq_deps_list}"
-  )
-
-
-endfunction(add_gen_header2)
-
-# Usage:
-#     add_gen_header(
-#       <target name>
-#       DEF_FILE <.h.def file>
-#       GEN_HDR <generated header file name>
-#       PARAMS <list of name=value pairs>
-#       DATA_FILES <list input data files>
-#     )
-function(add_gen_header target_name)
-  cmake_parse_arguments(
-    "ADD_GEN_HDR"
-    "PUBLIC" # No optional arguments
-    "DEF_FILE;GEN_HDR" # Single value arguments
-    "PARAMS;DATA_FILES;DEPENDS"     # Multi value arguments
-    ${ARGN}
-  )
-  get_fq_target_name(${target_name} fq_target_name)
-  if(NOT LLVM_LIBC_FULL_BUILD)
-    # We don't want to use generated headers if we are doing a non-full-build.
-    add_library(${fq_target_name} INTERFACE)
-    return()
-  endif()
-  if(NOT ADD_GEN_HDR_DEF_FILE)
-    message(FATAL_ERROR "`add_gen_hdr` rule requires DEF_FILE to be specified.")
-  endif()
-  if(NOT ADD_GEN_HDR_GEN_HDR)
-    message(FATAL_ERROR "`add_gen_hdr` rule requires GEN_HDR to be specified.")
-  endif()
-
-  set(absolute_path ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR_GEN_HDR})
-  file(RELATIVE_PATH relative_path ${LIBC_INCLUDE_SOURCE_DIR} ${absolute_path})
-  set(out_file ${LIBC_INCLUDE_DIR}/${relative_path})
-  set(in_file ${CMAKE_CURRENT_SOURCE_DIR}/${ADD_GEN_HDR_DEF_FILE})
-
-  set(fq_data_files "")
-  if(ADD_GEN_HDR_DATA_FILES)
-    foreach(data_file IN LISTS ADD_GEN_HDR_DATA_FILES)
-      list(APPEND fq_data_files "${CMAKE_CURRENT_SOURCE_DIR}/${data_file}")
-    endforeach(data_file)
-  endif()
-
-  set(replacement_params "")
-  if(ADD_GEN_HDR_PARAMS)
-    list(APPEND replacement_params "--args" ${ADD_GEN_HDR_PARAMS})
-  endif()
-
-  set(gen_hdr_script "${LIBC_BUILD_SCRIPTS_DIR}/gen_hdr.py")
-
-  file(GLOB td_includes ${LIBC_SOURCE_DIR}/spec/*.td)
-
-  set(ENTRYPOINT_NAME_LIST_ARG ${TARGET_ENTRYPOINT_NAME_LIST})
-  list(TRANSFORM ENTRYPOINT_NAME_LIST_ARG PREPEND "--e=")
-
-  if(LIBC_HDRGEN_EXE)
-    set(hdrgen_exe ${LIBC_HDRGEN_EXE})
-  else()
-    set(hdrgen_exe ${LIBC_TABLEGEN_EXE})
-    set(hdrgen_deps "${LIBC_TABLEGEN_EXE};${LIBC_TABLEGEN_TARGET}")
-  endif()
-  add_custom_command(
-    OUTPUT ${out_file}
-    COMMAND ${hdrgen_exe} -o ${out_file} --header ${ADD_GEN_HDR_GEN_HDR}
-            --def ${in_file} ${replacement_params} -I ${LIBC_SOURCE_DIR}
-           ${ENTRYPOINT_NAME_LIST_ARG}
-           ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td
-
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    DEPENDS ${in_file} ${fq_data_files} ${td_includes}
-            ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td
-            ${hdrgen_deps}
-  )
-
-  if(LIBC_TARGET_OS_IS_GPU)
-    file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/llvm-libc-decls)
-    file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/llvm-libc-decls/gpu)
-    set(decl_out_file ${LIBC_INCLUDE_DIR}/llvm-libc-decls/${relative_path})
-    add_custom_command(
-      OUTPUT ${decl_out_file}
-      COMMAND ${hdrgen_exe} -o ${decl_out_file}
-              --header ${ADD_GEN_HDR_GEN_HDR} --def ${in_file} --export-decls
-              ${replacement_params} -I ${LIBC_SOURCE_DIR} ${ENTRYPOINT_NAME_LIST_ARG}
-              ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td
-
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      DEPENDS ${in_file} ${fq_data_files} ${td_includes}
-              ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td
-              ${hdrgen_deps}
-    )
-  endif()
-
   if(ADD_GEN_HDR_DEPENDS)
     get_fq_deps_list(fq_deps_list ${ADD_GEN_HDR_DEPENDS})
     # Dependencies of a add_header target can only be another add_gen_header target
@@ -285,9 +159,6 @@ function(add_gen_header target_name)
       ${out_file}
   )
 
-  # We add the dependencies separately and not list under add_header_library's
-  # DEPENDS option above. This is because, deps of add_header_library are
-  # used with target_link_libraries.
   add_dependencies(${fq_target_name} ${generated_hdr_target})
 
   set_target_properties(
@@ -297,4 +168,6 @@ function(add_gen_header target_name)
       DECLS_FILE_PATH "${decl_out_file}"
       DEPS "${fq_deps_list}"
   )
+
+
 endfunction(add_gen_header)
diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 68b5ed1ed51c0..142778d9ea1cc 100644
--- a/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -452,3 +452,41 @@ function(add_redirector_object target_name)
     BEFORE PRIVATE -fPIC ${LIBC_COMPILE_OPTIONS_DEFAULT}
   )
 endfunction(add_redirector_object)
+
+# Helper to define a function with multiple implementations
+# - Computes flags to satisfy required/rejected features and arch,
+# - Declares an entry point,
+# - Attach the REQUIRE_CPU_FEATURES property to the target,
+# - Add the fully qualified target to `${name}_implementations` global property for tests.
+function(add_implementation name impl_name)
+  cmake_parse_arguments(
+    "ADD_IMPL"
+    "" # Optional arguments
+    "" # Single value arguments
+    "REQUIRE;SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;MLLVM_COMPILE_OPTIONS" # Multi value arguments
+    ${ARGN})
+
+  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+    # Note that '-mllvm' needs to be prefixed with 'SHELL:' to prevent CMake flag deduplication.
+    foreach(opt IN LISTS ADD_IMPL_MLLVM_COMPILE_OPTIONS)
+      list(APPEND ADD_IMPL_COMPILE_OPTIONS "SHELL:-mllvm ${opt}")
+    endforeach()
+  endif()
+
+  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
+    # Prevent warning when passing x86 SIMD types as template arguments.
+    # e.g. "warning: ignoring attributes on template argument ‘__m128i’ [-Wignored-attributes]"
+    list(APPEND ADD_IMPL_COMPILE_OPTIONS "-Wno-ignored-attributes")
+  endif()
+
+  add_entrypoint_object(${impl_name}
+    NAME ${name}
+    SRCS ${ADD_IMPL_SRCS}
+    HDRS ${ADD_IMPL_HDRS}
+    DEPENDS ${ADD_IMPL_DEPENDS}
+    COMPILE_OPTIONS ${libc_opt_high_flag} ${ADD_IMPL_COMPILE_OPTIONS}
+  )
+  get_fq_target_name(${impl_name} fq_target_name)
+  set_target_properties(${fq_target_name} PROPERTIES REQUIRE_CPU_FEATURES "${ADD_IMPL_REQUIRE}")
+  set_property(GLOBAL APPEND PROPERTY "${name}_implementations" "${fq_target_name}")
+endfunction()
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index c3a0f371cd620..84f3125d557ea 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -416,12 +416,12 @@ function(add_integration_test test_name)
       libc.test.IntegrationTest.test
       # We always add the memory functions objects. This is because the
       # compiler's codegen can emit calls to the C memory functions.
-      libc.src.string.bcmp
-      libc.src.string.bzero
       libc.src.string.memcmp
       libc.src.string.memcpy
       libc.src.string.memmove
       libc.src.string.memset
+      libc.src.strings.bcmp
+      libc.src.strings.bzero
   )
 
   if(libc.src.compiler.__stack_chk_fail IN_LIST TARGET_LLVMLIBC_ENTRYPOINTS)
@@ -476,14 +476,14 @@ function(add_integration_test test_name)
 
   if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_link_options(${fq_build_target_name} PRIVATE
-      ${LIBC_COMPILE_OPTIONS_DEFAULT} -Wno-multi-gpu
-      -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto
+      ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
+      -Wno-multi-gpu -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto
       "-Wl,-mllvm,-amdgpu-lower-global-ctor-dtor=0" -nostdlib -static
       "-Wl,-mllvm,-amdhsa-code-object-version=${LIBC_GPU_CODE_OBJECT_VERSION}")
   elseif(LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
     target_link_options(${fq_build_target_name} PRIVATE
-      ${LIBC_COMPILE_OPTIONS_DEFAULT} -Wno-multi-gpu
-      "-Wl,--suppress-stack-size-warning"
+      ${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
+      "-Wl,--suppress-stack-size-warning" -Wno-multi-gpu
       "-Wl,-mllvm,-nvptx-lower-global-ctor-dtor=1"
       "-Wl,-mllvm,-nvptx-emit-init-fini-kernel"
       -march=${LIBC_GPU_TARGET_ARCHITECTURE} -nostdlib -static
@@ -583,13 +583,13 @@ function(add_libc_hermetic test_name)
       libc.startup.${LIBC_TARGET_OS}.crt1
       # We always add the memory functions objects. This is because the
       # compiler's codegen can emit calls to the C memory functions.
-      libc.src.string.bcmp
-      libc.src.string.bzero
+      libc.src.__support.StringUtil.error_to_string
       libc.src.string.memcmp
       libc.src.string.memcpy
       libc.src.string.memmove
       libc.src.string.memset
-      libc.src.__support.StringUtil.error_to_string
+      libc.src.strings.bcmp
+      libc.src.strings.bzero
   )
 
   if(libc.src.compiler.__stack_chk_fail IN_LIST TARGET_LLVMLIBC_ENTRYPOINTS)
@@ -766,3 +766,33 @@ function(add_libc_test test_name)
     endif()
   endif()
 endfunction(add_libc_test)
+
+# Tests all implementations that can run on the target CPU.
+function(add_libc_multi_impl_test name suite)
+  get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
+  foreach(fq_config_name IN LISTS fq_implementations)
+    get_target_property(required_cpu_features ${fq_config_name} REQUIRE_CPU_FEATURES)
+    cpu_supports(can_run "${required_cpu_features}")
+    if(can_run)
+      string(FIND ${fq_config_name} "." last_dot_loc REVERSE)
+      math(EXPR name_loc "${last_dot_loc} + 1")
+      string(SUBSTRING ${fq_config_name} ${name_loc} -1 target_name)
+      add_libc_test(
+        ${target_name}_test
+        SUITE
+          ${suite}
+        COMPILE_OPTIONS
+          ${LIBC_COMPILE_OPTIONS_NATIVE}
+        LINK_LIBRARIES
+          LibcMemoryHelpers
+        ${ARGN}
+        DEPENDS
+          ${fq_config_name}
+          libc.src.__support.macros.sanitizer
+      )
+      get_fq_target_name(${fq_config_name}_test fq_target_name)
+    else()
+      message(STATUS "Skipping test for '${fq_config_name}' insufficient host cpu features '${required_cpu_features}'")
+    endif()
+  endforeach()
+endfunction()
diff --git a/libc/config/baremetal/aarch64/entrypoints.txt b/libc/config/baremetal/aarch64/entrypoints.txt
new file mode 100644
index 0000000000000..71b49d9894291
--- /dev/null
+++ b/libc/config/baremetal/aarch64/entrypoints.txt
@@ -0,0 +1,479 @@
+set(TARGET_LIBC_ENTRYPOINTS
+    # assert.h entrypoints
+    libc.src.assert.__assert_fail
+
+    # ctype.h entrypoints
+    libc.src.ctype.isalnum
+    libc.src.ctype.isalpha
+    libc.src.ctype.isascii
+    libc.src.ctype.isblank
+    libc.src.ctype.iscntrl
+    libc.src.ctype.isdigit
+    libc.src.ctype.isgraph
+    libc.src.ctype.islower
+    libc.src.ctype.isprint
+    libc.src.ctype.ispunct
+    libc.src.ctype.isspace
+    libc.src.ctype.isupper
+    libc.src.ctype.isxdigit
+    libc.src.ctype.toascii
+    libc.src.ctype.tolower
+    libc.src.ctype.toupper
+
+    # compiler entrypoints (no corresponding header)
+    libc.src.compiler.__stack_chk_fail
+
+    # errno.h entrypoints
+    libc.src.errno.errno
+
+    # setjmp.h entrypoints
+    libc.src.setjmp.longjmp
+    libc.src.setjmp.setjmp
+
+    # string.h entrypoints
+    libc.src.string.memccpy
+    libc.src.string.memchr
+    libc.src.string.memcmp
+    libc.src.string.memcpy
+    libc.src.string.memmem
+    libc.src.string.memmove
+    libc.src.string.mempcpy
+    libc.src.string.memrchr
+    libc.src.string.memset
+    libc.src.string.memset_explicit
+    libc.src.string.stpcpy
+    libc.src.string.stpncpy
+    libc.src.string.strcasestr
+    libc.src.string.strcat
+    libc.src.string.strchr
+    libc.src.string.strchrnul
+    libc.src.string.strcmp
+    libc.src.string.strcoll
+    libc.src.string.strcpy
+    libc.src.string.strcspn
+    libc.src.string.strerror
+    libc.src.string.strerror_r
+    libc.src.string.strlcat
+    libc.src.string.strlcpy
+    libc.src.string.strlen
+    libc.src.string.strncat
+    libc.src.string.strncmp
+    libc.src.string.strncpy
+    libc.src.string.strnlen
+    libc.src.string.strpbrk
+    libc.src.string.strrchr
+    libc.src.string.strsep
+    libc.src.string.strspn
+    libc.src.string.strstr
+    libc.src.string.strtok
+    libc.src.string.strtok_r
+    libc.src.string.strxfrm
+
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
+    # inttypes.h entrypoints
+    libc.src.inttypes.imaxabs
+    libc.src.inttypes.imaxdiv
+    libc.src.inttypes.strtoimax
+    libc.src.inttypes.strtoumax
+
+    # stdio.h entrypoints
+    libc.src.stdio.getchar
+    libc.src.stdio.printf
+    libc.src.stdio.putchar
+    libc.src.stdio.puts
+    libc.src.stdio.remove
+    libc.src.stdio.snprintf
+    libc.src.stdio.sprintf
+    libc.src.stdio.asprintf
+    libc.src.stdio.vprintf
+    libc.src.stdio.vsnprintf
+    libc.src.stdio.vsprintf
+    libc.src.stdio.vasprintf
+
+    # stdbit.h entrypoints
+    libc.src.stdbit.stdc_bit_ceil_uc
+    libc.src.stdbit.stdc_bit_ceil_ui
+    libc.src.stdbit.stdc_bit_ceil_ul
+    libc.src.stdbit.stdc_bit_ceil_ull
+    libc.src.stdbit.stdc_bit_ceil_us
+    libc.src.stdbit.stdc_bit_floor_uc
+    libc.src.stdbit.stdc_bit_floor_ui
+    libc.src.stdbit.stdc_bit_floor_ul
+    libc.src.stdbit.stdc_bit_floor_ull
+    libc.src.stdbit.stdc_bit_floor_us
+    libc.src.stdbit.stdc_bit_width_uc
+    libc.src.stdbit.stdc_bit_width_ui
+    libc.src.stdbit.stdc_bit_width_ul
+    libc.src.stdbit.stdc_bit_width_ull
+    libc.src.stdbit.stdc_bit_width_us
+    libc.src.stdbit.stdc_count_ones_uc
+    libc.src.stdbit.stdc_count_ones_ui
+    libc.src.stdbit.stdc_count_ones_ul
+    libc.src.stdbit.stdc_count_ones_ull
+    libc.src.stdbit.stdc_count_ones_us
+    libc.src.stdbit.stdc_count_zeros_uc
+    libc.src.stdbit.stdc_count_zeros_ui
+    libc.src.stdbit.stdc_count_zeros_ul
+    libc.src.stdbit.stdc_count_zeros_ull
+    libc.src.stdbit.stdc_count_zeros_us
+    libc.src.stdbit.stdc_first_leading_one_uc
+    libc.src.stdbit.stdc_first_leading_one_ui
+    libc.src.stdbit.stdc_first_leading_one_ul
+    libc.src.stdbit.stdc_first_leading_one_ull
+    libc.src.stdbit.stdc_first_leading_one_us
+    libc.src.stdbit.stdc_first_leading_zero_uc
+    libc.src.stdbit.stdc_first_leading_zero_ui
+    libc.src.stdbit.stdc_first_leading_zero_ul
+    libc.src.stdbit.stdc_first_leading_zero_ull
+    libc.src.stdbit.stdc_first_leading_zero_us
+    libc.src.stdbit.stdc_first_trailing_one_uc
+    libc.src.stdbit.stdc_first_trailing_one_ui
+    libc.src.stdbit.stdc_first_trailing_one_ul
+    libc.src.stdbit.stdc_first_trailing_one_ull
+    libc.src.stdbit.stdc_first_trailing_one_us
+    libc.src.stdbit.stdc_first_trailing_zero_uc
+    libc.src.stdbit.stdc_first_trailing_zero_ui
+    libc.src.stdbit.stdc_first_trailing_zero_ul
+    libc.src.stdbit.stdc_first_trailing_zero_ull
+    libc.src.stdbit.stdc_first_trailing_zero_us
+    libc.src.stdbit.stdc_has_single_bit_uc
+    libc.src.stdbit.stdc_has_single_bit_ui
+    libc.src.stdbit.stdc_has_single_bit_ul
+    libc.src.stdbit.stdc_has_single_bit_ull
+    libc.src.stdbit.stdc_has_single_bit_us
+    libc.src.stdbit.stdc_leading_ones_uc
+    libc.src.stdbit.stdc_leading_ones_ui
+    libc.src.stdbit.stdc_leading_ones_ul
+    libc.src.stdbit.stdc_leading_ones_ull
+    libc.src.stdbit.stdc_leading_ones_us
+    libc.src.stdbit.stdc_leading_zeros_uc
+    libc.src.stdbit.stdc_leading_zeros_ui
+    libc.src.stdbit.stdc_leading_zeros_ul
+    libc.src.stdbit.stdc_leading_zeros_ull
+    libc.src.stdbit.stdc_leading_zeros_us
+    libc.src.stdbit.stdc_trailing_ones_uc
+    libc.src.stdbit.stdc_trailing_ones_ui
+    libc.src.stdbit.stdc_trailing_ones_ul
+    libc.src.stdbit.stdc_trailing_ones_ull
+    libc.src.stdbit.stdc_trailing_ones_us
+    libc.src.stdbit.stdc_trailing_zeros_uc
+    libc.src.stdbit.stdc_trailing_zeros_ui
+    libc.src.stdbit.stdc_trailing_zeros_ul
+    libc.src.stdbit.stdc_trailing_zeros_ull
+    libc.src.stdbit.stdc_trailing_zeros_us
+
+    # stdlib.h entrypoints
+    libc.src.stdlib._Exit
+    libc.src.stdlib.abort
+    libc.src.stdlib.abs
+    libc.src.stdlib.aligned_alloc
+    libc.src.stdlib.atof
+    libc.src.stdlib.atoi
+    libc.src.stdlib.atol
+    libc.src.stdlib.atoll
+    libc.src.stdlib.bsearch
+    libc.src.stdlib.calloc
+    libc.src.stdlib.div
+    libc.src.stdlib.exit
+    libc.src.stdlib.free
+    libc.src.stdlib.freelist_malloc
+    libc.src.stdlib.labs
+    libc.src.stdlib.ldiv
+    libc.src.stdlib.llabs
+    libc.src.stdlib.lldiv
+    libc.src.stdlib.malloc
+    libc.src.stdlib.qsort
+    libc.src.stdlib.rand
+    libc.src.stdlib.realloc
+    libc.src.stdlib.srand
+    libc.src.stdlib.strtod
+    libc.src.stdlib.strtof
+    libc.src.stdlib.strtol
+    libc.src.stdlib.strtold
+    libc.src.stdlib.strtoll
+    libc.src.stdlib.strtoul
+    libc.src.stdlib.strtoull
+
+    # time.h entrypoints
+    libc.src.time.asctime
+    libc.src.time.asctime_r
+    libc.src.time.ctime
+    libc.src.time.ctime_r
+    libc.src.time.difftime
+    libc.src.time.gmtime
+    libc.src.time.gmtime_r
+    libc.src.time.mktime
+    libc.src.time.timespec_get
+
+    # internal entrypoints
+    libc.startup.baremetal.init
+    libc.startup.baremetal.fini
+)
+
+set(TARGET_LIBM_ENTRYPOINTS
+    # fenv.h entrypoints
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.fedisableexcept
+    libc.src.fenv.feenableexcept
+    libc.src.fenv.fegetenv
+    libc.src.fenv.fegetexcept
+    libc.src.fenv.fegetexceptflag
+    libc.src.fenv.fegetround
+    libc.src.fenv.feholdexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fesetenv
+    libc.src.fenv.fesetexcept
+    libc.src.fenv.fesetexceptflag
+    libc.src.fenv.fesetround
+    libc.src.fenv.fetestexcept
+    libc.src.fenv.fetestexceptflag
+    libc.src.fenv.feupdateenv
+
+    # math.h entrypoints
+    libc.src.math.acosf
+    libc.src.math.acoshf
+    libc.src.math.asinf
+    libc.src.math.asinhf
+    libc.src.math.atan2
+    libc.src.math.atan2f
+    libc.src.math.atanf
+    libc.src.math.atanhf
+    libc.src.math.canonicalize
+    libc.src.math.canonicalizef
+    libc.src.math.canonicalizel
+    libc.src.math.cbrt
+    libc.src.math.cbrtf
+    libc.src.math.ceil
+    libc.src.math.ceilf
+    libc.src.math.ceill
+    libc.src.math.copysign
+    libc.src.math.copysignf
+    libc.src.math.copysignl
+    libc.src.math.cos
+    libc.src.math.cosf
+    libc.src.math.coshf
+    libc.src.math.erff
+    libc.src.math.exp
+    libc.src.math.exp10
+    libc.src.math.exp10f
+    libc.src.math.exp2
+    libc.src.math.exp2f
+    libc.src.math.exp2m1f
+    libc.src.math.expf
+    libc.src.math.expm1
+    libc.src.math.expm1f
+    libc.src.math.fabs
+    libc.src.math.fabsf
+    libc.src.math.fabsl
+    libc.src.math.fdim
+    libc.src.math.fdimf
+    libc.src.math.fdiml
+    libc.src.math.floor
+    libc.src.math.floorf
+    libc.src.math.floorl
+    libc.src.math.fma
+    libc.src.math.fmaf
+    libc.src.math.fmax
+    libc.src.math.fmaxf
+    libc.src.math.fmaximum
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaxl
+    libc.src.math.fmin
+    libc.src.math.fminf
+    libc.src.math.fminimum
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminl
+    libc.src.math.fmod
+    libc.src.math.fmodf
+    libc.src.math.fmodl
+    libc.src.math.fmul
+    libc.src.math.frexp
+    libc.src.math.frexpf
+    libc.src.math.frexpl
+    libc.src.math.fromfp
+    libc.src.math.fromfpf
+    libc.src.math.fromfpl
+    libc.src.math.fromfpx
+    libc.src.math.fromfpxf
+    libc.src.math.fromfpxl
+    libc.src.math.hypot
+    libc.src.math.hypotf
+    libc.src.math.ilogb
+    libc.src.math.ilogbf
+    libc.src.math.ilogbl
+    libc.src.math.isnan
+    libc.src.math.isnanf
+    libc.src.math.isnanl
+    libc.src.math.ldexp
+    libc.src.math.ldexpf
+    libc.src.math.ldexpl
+    libc.src.math.llogb
+    libc.src.math.llogbf
+    libc.src.math.llogbl
+    libc.src.math.llrint
+    libc.src.math.llrintf
+    libc.src.math.llrintl
+    libc.src.math.llround
+    libc.src.math.llroundf
+    libc.src.math.llroundl
+    libc.src.math.log
+    libc.src.math.log10
+    libc.src.math.log10f
+    libc.src.math.log1p
+    libc.src.math.log1pf
+    libc.src.math.log2
+    libc.src.math.log2f
+    libc.src.math.logb
+    libc.src.math.logbf
+    libc.src.math.logbl
+    libc.src.math.logf
+    libc.src.math.lrint
+    libc.src.math.lrintf
+    libc.src.math.lrintl
+    libc.src.math.lround
+    libc.src.math.lroundf
+    libc.src.math.lroundl
+    libc.src.math.modf
+    libc.src.math.modff
+    libc.src.math.modfl
+    libc.src.math.nan
+    libc.src.math.nanf
+    libc.src.math.nanl
+    libc.src.math.nearbyint
+    libc.src.math.nearbyintf
+    libc.src.math.nearbyintl
+    libc.src.math.nextafter
+    libc.src.math.nextafterf
+    libc.src.math.nextafterl
+    libc.src.math.nextdown
+    libc.src.math.nextdownf
+    libc.src.math.nextdownl
+    libc.src.math.nexttoward
+    libc.src.math.nexttowardf
+    libc.src.math.nexttowardl
+    libc.src.math.nextup
+    libc.src.math.nextupf
+    libc.src.math.nextupl
+    libc.src.math.pow
+    libc.src.math.powf
+    libc.src.math.remainder
+    libc.src.math.remainderf
+    libc.src.math.remainderl
+    libc.src.math.remquo
+    libc.src.math.remquof
+    libc.src.math.remquol
+    libc.src.math.rint
+    libc.src.math.rintf
+    libc.src.math.rintl
+    libc.src.math.round
+    libc.src.math.roundeven
+    libc.src.math.roundevenf
+    libc.src.math.roundevenl
+    libc.src.math.roundf
+    libc.src.math.roundl
+    libc.src.math.scalbln
+    libc.src.math.scalblnf
+    libc.src.math.scalblnl
+    libc.src.math.scalbn
+    libc.src.math.scalbnf
+    libc.src.math.scalbnl
+    libc.src.math.sin
+    libc.src.math.sincos
+    libc.src.math.sincosf
+    libc.src.math.sinf
+    libc.src.math.sinhf
+    libc.src.math.sqrt
+    libc.src.math.sqrtf
+    libc.src.math.sqrtl
+    libc.src.math.tan
+    libc.src.math.tanf
+    libc.src.math.tanhf
+    libc.src.math.trunc
+    libc.src.math.truncf
+    libc.src.math.truncl
+    libc.src.math.ufromfp
+    libc.src.math.ufromfpf
+    libc.src.math.ufromfpl
+    libc.src.math.ufromfpx
+    libc.src.math.ufromfpxf
+    libc.src.math.ufromfpxl
+)
+
+if(LIBC_COMPILER_HAS_FIXED_POINT)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # stdfix.h _Fract and _Accum entrypoints
+    libc.src.stdfix.abshk
+    libc.src.stdfix.abshr
+    libc.src.stdfix.absk
+    libc.src.stdfix.abslk
+    libc.src.stdfix.abslr
+    libc.src.stdfix.absr
+    libc.src.stdfix.exphk
+    libc.src.stdfix.expk
+    libc.src.stdfix.roundhk
+    libc.src.stdfix.roundhr
+    libc.src.stdfix.roundk
+    libc.src.stdfix.roundlk
+    libc.src.stdfix.roundlr
+    libc.src.stdfix.roundr
+    libc.src.stdfix.rounduhk
+    libc.src.stdfix.rounduhr
+    libc.src.stdfix.rounduk
+    libc.src.stdfix.roundulk
+    libc.src.stdfix.roundulr
+    libc.src.stdfix.roundur
+    libc.src.stdfix.sqrtuhk
+    libc.src.stdfix.sqrtuhr
+    libc.src.stdfix.sqrtuk
+    libc.src.stdfix.sqrtur
+    # libc.src.stdfix.sqrtulk
+    libc.src.stdfix.sqrtulr
+    libc.src.stdfix.uhksqrtus
+    libc.src.stdfix.uksqrtui
+    libc.src.stdfix.hrbits
+    libc.src.stdfix.uhrbits
+    libc.src.stdfix.rbits
+    libc.src.stdfix.urbits
+    libc.src.stdfix.lrbits
+    libc.src.stdfix.ulrbits
+    libc.src.stdfix.hkbits
+    libc.src.stdfix.uhkbits
+    libc.src.stdfix.kbits
+    libc.src.stdfix.ukbits
+    libc.src.stdfix.lkbits
+    libc.src.stdfix.ulkbits
+  )
+endif()
+
+set(TARGET_LLVMLIBC_ENTRYPOINTS
+  ${TARGET_LIBC_ENTRYPOINTS}
+  ${TARGET_LIBM_ENTRYPOINTS}
+)
diff --git a/libc/config/baremetal/aarch64/headers.txt b/libc/config/baremetal/aarch64/headers.txt
new file mode 100644
index 0000000000000..6cc8a80455d3a
--- /dev/null
+++ b/libc/config/baremetal/aarch64/headers.txt
@@ -0,0 +1,20 @@
+set(TARGET_PUBLIC_HEADERS
+    libc.include.assert
+    libc.include.ctype
+    libc.include.errno
+    libc.include.features
+    libc.include.fenv
+    libc.include.float
+    libc.include.inttypes
+    libc.include.math
+    libc.include.setjmp
+    libc.include.stdfix
+    libc.include.stdint
+    libc.include.stdio
+    libc.include.stdlib
+    libc.include.string
+    libc.include.strings
+    libc.include.sys_queue
+    libc.include.time
+    libc.include.uchar
+)
diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td
deleted file mode 100644
index 7421d86fabeb0..0000000000000
--- a/libc/config/baremetal/api.td
+++ /dev/null
@@ -1,59 +0,0 @@
-include "config/public_api.td"
-
-include "spec/stdc.td"
-include "spec/stdc_ext.td"
-include "spec/bsd_ext.td"
-include "spec/llvm_libc_stdfix_ext.td"
-
-
-def CTypeAPI : PublicAPI<"ctype.h"> {
-}
-
-def FEnvAPI : PublicAPI<"fenv.h"> {
-  let Types = ["fenv_t", "fexcept_t"];
-}
-
-def IntTypesAPI : PublicAPI<"inttypes.h"> {
-  let Types = ["imaxdiv_t"];
-}
-
-def MathAPI : PublicAPI<"math.h"> {
-  let Types = ["double_t", "float_t"];
-}
-
-def StdIOAPI : PublicAPI<"stdio.h"> {
-  let Types = ["size_t"];
-}
-
-def StdlibAPI : PublicAPI<"stdlib.h"> {
-  let Types = [
-    "div_t",
-    "ldiv_t",
-    "lldiv_t",
-    "size_t",
-    "__bsearchcompare_t",
-    "__qsortcompare_t",
-  ];
-}
-
-def StringAPI : PublicAPI<"string.h"> {
-  let Types = ["size_t"];
-}
-
-def TimeAPI : PublicAPI<"time.h"> {
-  let Types = [
-    "clock_t",
-    "time_t",
-    "struct tm",
-    "struct timespec",
-  ];
-}
-
-def UCharAPI : PublicAPI<"uchar.h"> {
-  let Types = [
-    "mbstate_t",
-    "char8_t",
-    "char16_t",
-    "char32_t",
-  ];
-}
diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt
index b85ae1119345d..71b49d9894291 100644
--- a/libc/config/baremetal/arm/entrypoints.txt
+++ b/libc/config/baremetal/arm/entrypoints.txt
@@ -31,10 +31,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.setjmp.setjmp
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -45,10 +41,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memrchr
     libc.src.string.memset
     libc.src.string.memset_explicit
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -62,7 +56,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -76,6 +69,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok_r
     libc.src.string.strxfrm
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -209,6 +211,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.time.gmtime
     libc.src.time.gmtime_r
     libc.src.time.mktime
+    libc.src.time.timespec_get
 
     # internal entrypoints
     libc.startup.baremetal.init
diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt
index 199a030ee6371..e84d139d09dd8 100644
--- a/libc/config/baremetal/riscv/entrypoints.txt
+++ b/libc/config/baremetal/riscv/entrypoints.txt
@@ -27,10 +27,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.errno.errno
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -41,10 +37,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memrchr
     libc.src.string.memset
     libc.src.string.memset_explicit
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -58,7 +52,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -72,6 +65,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok_r
     libc.src.string.strxfrm
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -205,6 +207,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.time.gmtime
     libc.src.time.gmtime_r
     libc.src.time.mktime
+    libc.src.time.timespec_get
 
     # internal entrypoints
     libc.startup.baremetal.init
diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt
index 2d5dbeff48574..7972d285f963a 100644
--- a/libc/config/darwin/arm/entrypoints.txt
+++ b/libc/config/darwin/arm/entrypoints.txt
@@ -21,9 +21,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.errno.errno
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -35,7 +32,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memset
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -46,7 +42,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -62,6 +57,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strdup
     libc.src.string.strndup
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt
index 64eeed18f3819..19230cde47198 100644
--- a/libc/config/darwin/x86_64/entrypoints.txt
+++ b/libc/config/darwin/x86_64/entrypoints.txt
@@ -18,11 +18,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.toupper
 
     # search.h entrypoints
-    libc.src.search.lfind 
+    libc.src.search.lfind
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bzero
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -58,6 +56,10 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strdup
     libc.src.string.strndup
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bzero
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
diff --git a/libc/config/gpu/api.td b/libc/config/gpu/api.td
deleted file mode 100644
index 995ff31c4ac9e..0000000000000
--- a/libc/config/gpu/api.td
+++ /dev/null
@@ -1,50 +0,0 @@
-include "config/public_api.td"
-
-include "spec/stdc.td"
-include "spec/posix.td"
-include "spec/gpu_ext.td"
-include "spec/gnu_ext.td"
-include "spec/stdc_ext.td"
-include "spec/llvm_libc_ext.td"
-
-
-def StringAPI : PublicAPI<"string.h"> {
-  let Types = ["size_t"];
-}
-
-def StdlibAPI : PublicAPI<"stdlib.h"> {
-  let Types = [
-    "div_t",
-    "ldiv_t",
-    "lldiv_t",
-    "size_t",
-    "__bsearchcompare_t",
-    "__qsortcompare_t",
-    "__qsortrcompare_t",
-    "__atexithandler_t",
-  ];
-}
-
-def FenvAPI: PublicAPI<"fenv.h"> {
-  let Types = ["fenv_t"];
-}
-
-def StdIOAPI : PublicAPI<"stdio.h"> {
-  let Types = [
-    "FILE",
-    "off_t",
-    "size_t",
-  ];
-}
-
-def IntTypesAPI : PublicAPI<"inttypes.h"> {
-  let Types = ["imaxdiv_t"];
-}
-
-def TimeAPI : PublicAPI<"time.h"> {
-  let Types = [
-    "clock_t",
-    "time_t",
-    "struct timespec",
-  ];
-}
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index 38e9f2e685cae..e3ebd3ca47256 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -35,10 +35,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.toupper_l
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -48,10 +44,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.mempcpy
     libc.src.string.memrchr
     libc.src.string.memset
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -66,7 +60,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -82,6 +75,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strxfrm
     libc.src.string.strxfrm_l
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # stdbit.h entrypoints
     libc.src.stdbit.stdc_bit_ceil_uc
     libc.src.stdbit.stdc_bit_ceil_ui
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 74ca3742977a5..c2179938b40d1 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -45,10 +45,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sched.sched_yield
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -59,10 +55,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memrchr
     libc.src.string.memset
     libc.src.string.memset_explicit
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -77,7 +71,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -93,6 +86,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok_r
     libc.src.string.strxfrm
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -365,6 +367,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.complex.cimag
     libc.src.complex.cimagf
     libc.src.complex.cimagl
+    libc.src.complex.conj
+    libc.src.complex.conjf
+    libc.src.complex.conjl
 
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -616,6 +621,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     # complex.h C23 _Complex _Float16 entrypoints
     # libc.src.complex.crealf16
     # libc.src.complex.cimagf16
+    # libc.src.complex.conjf16
     
     # math.h C23 _Float16 entrypoints
     libc.src.math.canonicalizef16
@@ -721,6 +727,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     # complex.h C23 _Complex _Float128 entrypoints
     libc.src.complex.crealf128
     libc.src.complex.cimagf128
+    libc.src.complex.conjf128
 
     # math.h C23 _Float128 entrypoints
     libc.src.math.canonicalizef128
@@ -804,11 +811,11 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.dirent.opendir
     libc.src.dirent.readdir
 
-    # network.h entrypoints
-    libc.src.network.htonl
-    libc.src.network.htons
-    libc.src.network.ntohl
-    libc.src.network.ntohs
+    # arpa/inet.h entrypoints
+    libc.src.arpa.inet.htonl
+    libc.src.arpa.inet.htons
+    libc.src.arpa.inet.ntohl
+    libc.src.arpa.inet.ntohs
 
     # pthread.h entrypoints
     libc.src.pthread.pthread_atfork
@@ -999,6 +1006,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.time.mktime
     libc.src.time.nanosleep
     libc.src.time.time
+    libc.src.time.timespec_get
 
     # unistd.h entrypoints
     libc.src.unistd.__llvm_libc_syscall
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
deleted file mode 100644
index a00e0f61b90df..0000000000000
--- a/libc/config/linux/api.td
+++ /dev/null
@@ -1,276 +0,0 @@
-include "config/public_api.td"
-
-include "spec/stdc.td"
-include "spec/posix.td"
-include "spec/linux.td"
-include "spec/gnu_ext.td"
-include "spec/bsd_ext.td"
-include "spec/stdc_ext.td"
-include "spec/llvm_libc_ext.td"
-include "spec/llvm_libc_stdfix_ext.td"
-
-def CTypeAPI : PublicAPI<"ctype.h"> {
-}
-
-def FCntlAPI : PublicAPI<"fcntl.h"> {
-  let Types = [
-    "mode_t",
-    "off_t",
-  ];
-}
-
-def IntTypesAPI : PublicAPI<"inttypes.h"> {
-  let Types = ["imaxdiv_t"];
-}
-
-def MathAPI : PublicAPI<"math.h"> {
-  let Types = ["double_t", "float_t", "float128"];
-}
-
-def FenvAPI: PublicAPI<"fenv.h"> {
-  let Types = ["fenv_t", "fexcept_t"];
-}
-
-def StringAPI : PublicAPI<"string.h"> {
-  let Types = ["size_t"];
-}
-
-def StdIOAPI : PublicAPI<"stdio.h"> {
-  let Macros = [
-    SimpleMacroDef<"stderr", "stderr">,
-    SimpleMacroDef<"stdin", "stdin">,
-    SimpleMacroDef<"stdout", "stdout">,
-  ];
-  let Types = [
-    "FILE",
-    "cookie_io_functions_t",
-    "off_t",
-    "size_t",
-  ];
-}
-
-def StdlibAPI : PublicAPI<"stdlib.h"> {
-  let Types = [
-    "div_t",
-    "ldiv_t",
-    "lldiv_t",
-    "size_t",
-    "__bsearchcompare_t",
-    "__qsortcompare_t",
-    "__qsortrcompare_t",
-    "__atexithandler_t",
-  ];
-}
-
-def TimeAPI : PublicAPI<"time.h"> {
-  let Types = [
-    "clock_t",
-    "time_t",
-    "struct tm",
-    "struct timespec",
-    "struct timeval",
-    "clockid_t",
-  ];
-}
-
-def SchedAPI : PublicAPI<"sched.h"> {
-  let Types = [
-    "pid_t",
-    "size_t",
-    "cpu_set_t",
-    "struct sched_param",
-    // Needed according to posix standard
-    "time_t",
-    "struct timespec",
-  ];
-}
-
-def SysMManAPI : PublicAPI<"sys/mman.h"> {
-  let Types = ["off_t", "size_t", "mode_t"];
-}
-
-def SignalAPI : PublicAPI<"signal.h"> {
-  let Types = [
-    "sig_atomic_t",
-    "sigset_t",
-    "struct sigaction",
-    "union sigval",
-    "siginfo_t",
-    "stack_t",
-    "pid_t",
-  ];
-}
-
-def ThreadsAPI : PublicAPI<"threads.h"> {
-  let Macros = [
-    SimpleMacroDef<"ONCE_FLAG_INIT", "{0}">,
-  ];
-
-  let Types = [
-    "__call_once_func_t",
-    "once_flag",
-    "cnd_t",
-    "mtx_t",
-    "thrd_t",
-    "thrd_start_t",
-    "tss_t",
-    "tss_dtor_t",
-  ];
-
-  let Enumerations = [
-    "mtx_plain",
-    "mtx_recursive",
-    "mtx_timed",
-    "thrd_timedout",
-    "thrd_success",
-    "thrd_busy",
-    "thrd_error",
-    "thrd_nomem",
-  ];
-}
-
-def PThreadAPI : PublicAPI<"pthread.h"> {
-  let Types = [
-      "__atfork_callback_t",
-      "__pthread_once_func_t",
-      "__pthread_start_t",
-      "__pthread_tss_dtor_t",
-      "pthread_attr_t",
-      "pthread_condattr_t",
-      "pthread_key_t",
-      "pthread_mutex_t",
-      "pthread_mutexattr_t",
-      "pthread_once_t",
-      "pthread_rwlockattr_t",
-      "pthread_rwlock_t",
-      "pthread_spinlock_t",
-      "pthread_t",
-  ];
-}
-
-def DirentAPI : PublicAPI<"dirent.h"> {
-  let Types = [
-    "ino_t",
-    "DIR",
-    "struct dirent",
-  ];
-}
-
-def UniStdAPI : PublicAPI<"unistd.h"> {
-  let Types = ["__exec_argv_t", "__exec_envp_t", "off_t", "pid_t", "size_t",
-               "ssize_t", "uid_t", "__getoptargv_t"];
-}
-
-def WCharAPI : PublicAPI<"wchar.h"> {
-  let Types = [
-    "wchar_t",
-    "wint_t",
-    "size_t",
-  ];
-}
-
-def UCharAPI : PublicAPI<"uchar.h"> {
-  let Types = [
-    "mbstate_t",
-    "char8_t",
-    "char16_t",
-    "char32_t",
-  ];
-}
-
-def SysRandomAPI : PublicAPI<"sys/random.h"> {
-  let Types = ["size_t", "ssize_t"];
-}
-
-def SysSelectAPI : PublicAPI<"sys/select.h"> {
-  let Types = ["fd_set", "sigset_t", "suseconds_t", "time_t", "struct timespec",
-               "struct timeval"];
-}
-
-def SysSocketAPI : PublicAPI<"sys/socket.h"> {
-  let Types = [
-    "sa_family_t",
-    "socklen_t",
-    "struct sockaddr",
-    "struct sockaddr_un",
-    "struct msghdr",
-    "struct iovec",
-    "size_t",
-    "ssize_t",
-  ];
-}
-
-def SysResourceAPI : PublicAPI<"sys/resource.h"> {
-  let Types = ["rlim_t", "struct rlimit"];
-}
-
-def SysStatAPI : PublicAPI<"sys/stat.h"> {
-  let Types = ["mode_t", "dev_t", "ino_t", "nlink_t", "uid_t", "gid_t", "off_t",
-               "struct timespec", "struct timeval", "blksize_t", "blkcnt_t",
-               "struct stat"];
-}
-
-def SysWaitAPI : PublicAPI<"sys/wait.h"> {
-  let Types = ["pid_t", "struct rusage", "siginfo_t"];
-}
-
-def SysSendfileAPI : PublicAPI<"sys/sendfile.h"> {
-  let Types = ["off_t", "size_t", "ssize_t"];
-}
-
-def SysTypesAPI : PublicAPI<"sys/types.h"> {
-  let Types = [
-    "blkcnt_t",
-    "blksize_t",
-    "clockid_t",
-    "dev_t",
-    "gid_t",
-    "ino_t",
-    "mode_t",
-    "nlink_t",
-    "off_t",
-    "pid_t",
-    "pthread_attr_t",
-    "pthread_condattr_t",
-    "pthread_key_t",
-    "pthread_mutex_t",
-    "pthread_mutexattr_t",
-    "pthread_once_t",
-    "pthread_rwlockattr_t",
-    "pthread_rwlock_t",
-    "pthread_t",
-    "size_t",
-    "ssize_t",
-    "suseconds_t",
-    "time_t",
-    "uid_t"
-  ];
-}
-
-def SysUtsNameAPI : PublicAPI<"sys/utsname.h"> {
-  let Types = ["struct utsname"];
-}
-
-def SysEpollAPI : PublicAPI<"sys/epoll.h"> {
-  let Types = ["struct epoll_event", "struct epoll_data", "sigset_t", "struct timespec"];
-}
-
-def SpawnAPI : PublicAPI<"spawn.h"> {
-  let Types = ["mode_t", "pid_t", "posix_spawnattr_t", "posix_spawn_file_actions_t"];
-}
-
-def TermiosAPI : PublicAPI<"termios.h"> {
-  let Types = ["cc_t", "pid_t", "speed_t", "struct termios", "tcflag_t"];
-}
-
-def SetJmpAPI : PublicAPI<"setjmp.h"> {
-  let Types = ["jmp_buf"];
-}
-
-def SearchAPI : PublicAPI<"search.h"> {
-  let Types = ["ACTION", "ENTRY", "struct hsearch_data", "__lsearchcompare_t"];
-}
-
-def SysStatvfsAPI : PublicAPI<"sys/statvfs.h"> {
-  let Types = ["struct statvfs"];
-}
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 31d81de06fb6b..f5e9827727396 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -21,10 +21,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.errno.errno
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -34,10 +30,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.mempcpy
     libc.src.string.memrchr
     libc.src.string.memset
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -48,7 +42,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -61,6 +54,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok
     libc.src.string.strtok_r
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -207,6 +209,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.complex.cimag
     libc.src.complex.cimagf
     libc.src.complex.cimagl
+    libc.src.complex.conj
+    libc.src.complex.conjf
+    libc.src.complex.conjl
     
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 5419462d4f5b3..e2df6aca38bf9 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -45,10 +45,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sched.sched_yield
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -59,10 +55,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memrchr
     libc.src.string.memset
     libc.src.string.memset_explicit
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -77,7 +71,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -93,6 +86,14 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok_r
     libc.src.string.strxfrm
 
+    # strings.h entrypoints
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.strcasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -363,6 +364,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.complex.cimag
     libc.src.complex.cimagf
     libc.src.complex.cimagl
+    libc.src.complex.conj
+    libc.src.complex.conjf
+    libc.src.complex.conjl
 
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -618,6 +622,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     # complex.h C23 _Complex _Float128 entrypoints
     libc.src.complex.crealf128
     libc.src.complex.cimagf128
+    libc.src.complex.conjf128
     
     # math.h C23 _Float128 entrypoints
     libc.src.math.canonicalizef128
@@ -747,11 +752,11 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.dirent.opendir
     libc.src.dirent.readdir
 
-    # network.h entrypoints
-    libc.src.network.htonl
-    libc.src.network.htons
-    libc.src.network.ntohl
-    libc.src.network.ntohs
+    # arpa/inet.h entrypoints
+    libc.src.arpa.inet.htonl
+    libc.src.arpa.inet.htons
+    libc.src.arpa.inet.ntohl
+    libc.src.arpa.inet.ntohs
 
     # pthread.h entrypoints
     libc.src.pthread.pthread_atfork
@@ -937,6 +942,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.time.mktime
     libc.src.time.nanosleep
     libc.src.time.time
+    libc.src.time.timespec_get
 
     # unistd.h entrypoints
     libc.src.unistd.__llvm_libc_syscall
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 957e28bd66cc4..68d0501d3d358 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -45,10 +45,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sched.sched_yield
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
-    libc.src.string.index
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -59,10 +55,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memrchr
     libc.src.string.memset
     libc.src.string.memset_explicit
-    libc.src.string.rindex
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -77,7 +71,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -93,6 +86,15 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strtok_r
     libc.src.string.strxfrm
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.index
+    libc.src.strings.rindex
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -365,6 +367,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.complex.cimag
     libc.src.complex.cimagf
     libc.src.complex.cimagl
+    libc.src.complex.conj
+    libc.src.complex.conjf
+    libc.src.complex.conjl
     
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -621,6 +626,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     # complex.h C23 _Complex _Float16 entrypoints
     libc.src.complex.crealf16
     libc.src.complex.cimagf16
+    libc.src.complex.conjf16
 
     # math.h C23 _Float16 entrypoints
     libc.src.math.canonicalizef16
@@ -699,6 +705,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.scalbnf16
     libc.src.math.setpayloadf16
     libc.src.math.setpayloadsigf16
+    libc.src.math.sinf16
     libc.src.math.sinhf16
     libc.src.math.sinpif16
     libc.src.math.sqrtf16
@@ -729,6 +736,7 @@ if(LIBC_TYPES_HAS_FLOAT128)
     # complex.h C23 _Complex _Float128 entrypoints
     # libc.src.complex.crealf128
     # libc.src.complex.cimagf128
+    # libc.src.complex.conjf128
     
     # math.h C23 _Float128 entrypoints
     libc.src.math.canonicalizef128
@@ -887,11 +895,11 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.dirent.opendir
     libc.src.dirent.readdir
 
-    # network.h entrypoints
-    libc.src.network.htonl
-    libc.src.network.htons
-    libc.src.network.ntohl
-    libc.src.network.ntohs
+    # arpa/inet.h entrypoints
+    libc.src.arpa.inet.htonl
+    libc.src.arpa.inet.htons
+    libc.src.arpa.inet.ntohl
+    libc.src.arpa.inet.ntohs
 
     # pthread.h entrypoints
     libc.src.pthread.pthread_atfork
@@ -1082,6 +1090,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.time.mktime
     libc.src.time.nanosleep
     libc.src.time.time
+    libc.src.time.timespec_get
 
     # locale.h entrypoints
     libc.src.locale.localeconv
diff --git a/libc/config/public_api.td b/libc/config/public_api.td
deleted file mode 100644
index 1b34506c643c3..0000000000000
--- a/libc/config/public_api.td
+++ /dev/null
@@ -1,26 +0,0 @@
-include "spec/spec.td"
-
-class MacroDef<string name> {
-  string Name = name;
-  string Defn = "";
-}
-
-class SimpleMacroDef<string name, string value> : MacroDef<name> {
-  let Defn = !strconcat("#define ", name, " ", value);
-}
-
-class MacroDefineIfNot<string name, string value> : MacroDef<name> {
-  let Defn = !strconcat("#ifndef ", name, "\n",
-                        "#define " , name, " ", value, "\n",
-                        "#endif // ", name);
-}
-
-class PublicAPI<string name> {
-  string HeaderName = name;
-  list<MacroDef> Macros = [];
-  list<string> Types = [];
-  list<string> Enumerations = [];
-  list<string> Structs = [];
-  list<string> Functions = [];
-  list<string> Objects = [];
-}
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index 8f0b50bcc83ea..4ecc3ada9c768 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -18,9 +18,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.toupper
 
     # string.h entrypoints
-    libc.src.string.bcmp
-    libc.src.string.bcopy
-    libc.src.string.bzero
     libc.src.string.memccpy
     libc.src.string.memchr
     libc.src.string.memcmp
@@ -32,7 +29,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.memset
     libc.src.string.stpcpy
     libc.src.string.stpncpy
-    libc.src.string.strcasecmp
     libc.src.string.strcasestr
     libc.src.string.strcat
     libc.src.string.strchr
@@ -43,7 +39,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlcat
     libc.src.string.strlcpy
     libc.src.string.strlen
-    libc.src.string.strncasecmp
     libc.src.string.strncat
     libc.src.string.strncmp
     libc.src.string.strncpy
@@ -59,6 +54,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strdup
     libc.src.string.strndup
 
+    # strings.h entrypoints
+    libc.src.strings.bcmp
+    libc.src.strings.bcopy
+    libc.src.strings.bzero
+    libc.src.strings.strcasecmp
+    libc.src.strings.strncasecmp
+
     # inttypes.h entrypoints
     libc.src.inttypes.imaxabs
     libc.src.inttypes.imaxdiv
@@ -95,6 +97,10 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # errno.h entrypoints
     libc.src.errno.errno
+
+    # time.h entrypoints
+    libc.src.time.time
+    libc.src.time.clock_getres
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
diff --git a/libc/config/windows/headers.txt b/libc/config/windows/headers.txt
new file mode 100644
index 0000000000000..bccc04f7697e5
--- /dev/null
+++ b/libc/config/windows/headers.txt
@@ -0,0 +1,9 @@
+set(TARGET_PUBLIC_HEADERS
+    libc.include.ctype
+    libc.include.string
+    libc.include.inttypes
+    libc.include.stdlib
+    libc.include.errno
+    libc.include.fenv
+    libc.include.math
+)
diff --git a/libc/docs/dev/header_generation.rst b/libc/docs/dev/header_generation.rst
index ec4206217ca77..2c586cc87b699 100644
--- a/libc/docs/dev/header_generation.rst
+++ b/libc/docs/dev/header_generation.rst
@@ -3,21 +3,15 @@
 Generating Public and Internal headers
 ======================================
 
-This is a new implementation of the previous libc header generator. The old
-header generator (libc-hdrgen aka "Headergen") was based on TableGen, which
-created an awkward dependency on the rest of LLVM for our build system. By
-creating a new standalone Headergen we can eliminate these dependencies for
-easier cross compatibility.
-
-There are 3 main components of the new Headergen. The first component are the
-YAML files that contain all the function header information and are separated by
-header specification and standard. The second component are the classes that are
-created for each component of the function header: macros, enumerations, types,
-function, arguments, and objects. The third component is the Python script that
-uses the class representation to deserialize YAML files into its specific
-components and then reserializes the components into the function header. The
-Python script also combines the generated header content with header definitions
-and extra macro and type inclusions from the .h.def file.
+There are 3 main components of the Headergen. The first component are the YAML
+files that contain all the function header information and are separated by
+header specification and standard. The second component are the classes that
+are created for each component of the function header: macros, enumerations,
+types, function, arguments, and objects. The third component is the Python
+script that uses the class representation to deserialize YAML files into its
+specific components and then reserializes the components into the function
+header. The Python script also combines the generated header content with
+header definitions and extra macro and type inclusions from the .h.def file.
 
 
 Instructions
@@ -30,7 +24,7 @@ Required Versions:
 1. Keep full-build mode on when building, otherwise headers will not be
    generated.
 2. Once the build is complete, enter in the command line within the build
-   directory ``ninja check-newhdrgen`` to ensure that the integration tests are
+   directory ``ninja check-hdrgen`` to ensure that the integration tests are
    passing.
 3. Then enter in the command line ``ninja libc`` to generate headers. Headers
    will be in ``build/projects/libc/include`` or ``build/libc/include`` in a
@@ -38,9 +32,6 @@ Required Versions:
    ``build/projects/libc/include/sys``.
 
 
-New Headergen is turned on by default, but if you want to use old Headergen,
-you can include this statement when building: ``-DLIBC_USE_NEW_HEADER_GEN=OFF``
-
 To add a function to the YAML files, you can either manually enter it in the
 YAML file corresponding to the header it belongs to or add it through the
 command line.
@@ -53,41 +44,41 @@ To add through the command line:
 
    .. code-block:: none
 
-     python3 libc/newhdrgen/yaml_to_classes.py
-     libc/newhdrgen/yaml/[yaml_file.yaml] --add_function "<return_type>" <function_name> "<function_arg1, function_arg2>" <standard> <guard> <attribute>
+     python3 libc/hdrgen/yaml_to_classes.py
+     libc/hdrgen/yaml/[yaml_file.yaml] --add_function "<return_type>" <function_name> "<function_arg1, function_arg2>" <standard> <guard> <attribute>
 
    Example:
 
    .. code-block:: none
 
-      python3 libc/newhdrgen/yaml_to_classes.py
-      libc/newhdrgen/yaml/ctype.yaml --add_function "char" example_function
+      python3 libc/hdrgen/yaml_to_classes.py
+      libc/hdrgen/yaml/ctype.yaml --add_function "char" example_function
       "int, void, const void" stdc example_float example_attribute
 
    Keep in mind only the return_type and arguments have quotes around them. If
    you do not have any guards or attributes you may enter "null" for both.
 
 3. Check the YAML file that the added function is present. You will also get a
-   generated header file with the new addition in the newhdrgen directory to
+   generated header file with the new addition in the hdrgen directory to
    examine.
 
-If you want to sort the functions alphabetically you can check out libc/newhdrgen/yaml_functions_sorted.py.
+If you want to sort the functions alphabetically you can check out libc/hdrgen/yaml_functions_sorted.py.
 
 
 Testing
 -------
 
-New Headergen has an integration test that you may run once you have configured
-your CMake within the build directory. In the command line, enter the following:
-``ninja check-newhdrgen``. The integration test is one test that ensures the
-process of YAML to classes to generate headers works properly. If there are any
-new additions on formatting headers, make sure the test is updated with the
-specific addition.
+Headergen has an integration test that you may run once you have configured
+your CMake within the build directory. In the command line, enter the
+following: ``ninja check-hdrgen``. The integration test is one test that
+ensures the process of YAML to classes to generate headers works properly. If
+there are any new additions on formatting headers, make sure the test is
+updated with the specific addition.
 
-Integration Test can be found in: ``libc/newhdrgen/tests/test_integration.py``
+Integration Test can be found in: ``libc/hdrgen/tests/test_integration.py``
 
 File to modify if adding something to formatting:
-``libc/newhdrgen/tests/expected_output/test_header.h``
+``libc/hdrgen/tests/expected_output/test_header.h``
 
 
 Common Errors
@@ -98,7 +89,7 @@ Common Errors
 
    .. code-block:: none
 
-      "/llvm-project/libc/newhdrgen/yaml_to_classes.py", line 67, in yaml_to_classes function_data["return_type"]
+      "/llvm-project/libc/hdrgen/yaml_to_classes.py", line 67, in yaml_to_classes function_data["return_type"]
 
    If you receive this error or any error pertaining to
    ``function_data[function_specific_component]`` while building the headers
@@ -126,7 +117,7 @@ Common Errors
    missing. Ensure the correct style and required files are present:
 
    | ``[header_name]``
-   | ``[../libc/newhdrgen/yaml/[yaml_file.yaml]``
+   | ``[../libc/hdrgen/yaml/[yaml_file.yaml]``
    | ``[header_name.h.def]``
    | ``[header_name.h]``
    | ``DEPENDS``
@@ -156,13 +147,13 @@ Common Errors
 
    .. code-block:: none
 
-     File "/llvm-project/libc/newhdrgen/header.py", line 60, in __str__ for
+     File "/llvm-project/libc/hdrgen/header.py", line 60, in __str__ for
      function in self.functions: AttributeError: 'HeaderFile' object has no
      attribute 'functions'
 
    When running ``ninja libc`` in the build directory to generate headers you
    may receive the error above. Essentially this means that in
-   ``libc/newhdrgen/header.py`` there is a missing attribute named functions.
+   ``libc/hdrgen/header.py`` there is a missing attribute named functions.
    Make sure all function components are defined within this file and there are
    no missing functions to add these components.
 
@@ -193,12 +184,12 @@ Common Errors
    Sometimes the integration test will fail but that
    still means the process is working unless the comparison between the output
    and expected_output is not showing. If that is the case make sure in
-   ``libc/newhdrgen/tests/test_integration.py`` there are no missing arguments
+   ``libc/hdrgen/tests/test_integration.py`` there are no missing arguments
    that run through the script.
 
    If the integration tests are failing due to mismatching of lines or small
    errors in spacing that is nothing to worry about. If this is happening while
    you are making a new change to the formatting of the headers, then
    ensure the expected output file
-   ``libc/newhdrgen/tests/expected_output/test_header.h`` has the changes you
+   ``libc/hdrgen/tests/expected_output/test_header.h`` has the changes you
    are applying.
diff --git a/libc/docs/dev/source_tree_layout.rst b/libc/docs/dev/source_tree_layout.rst
index 8b423a1712cc8..bd9d6ca453e08 100644
--- a/libc/docs/dev/source_tree_layout.rst
+++ b/libc/docs/dev/source_tree_layout.rst
@@ -15,9 +15,9 @@ directories::
         - examples
         - fuzzing
         - hdr
+        - hdrgen
         - include
         - lib
-        - newhdrgen
         - src
         - startup
         - test
@@ -29,8 +29,7 @@ The ``benchmarks`` directory
 ----------------------------
 
 The ``benchmarks`` directory contains LLVM-libc's benchmarking utilities. These
-are mostly used for the memory functions. This also includes the automemcpy
-subdirectory for automatic generation of optimized memory functions.
+are mostly used for the memory functions.
 
 The ``config`` directory
 ------------------------
@@ -89,13 +88,13 @@ The ``lib`` directory
 This directory contains a ``CMakeLists.txt`` file listing the targets for the
 public libraries ``libc.a``, ``libm.a`` etc.
 
-The ``newhdrgen`` directory
+The ``hdrgen`` directory
 ---------------------------
 
 This directory contains the sources and specifications for the types, macros
 and entrypoint functions. These definitions are organized in the ``yaml``
 subdirectory and match the organization of the ``*.h.def`` files. This folder
-also contains the python sources for new headergen, which is what generates the
+also contains the python sources for headergen, which is what generates the
 headers.
 
 The ``src`` directory
diff --git a/libc/docs/gpu/building.rst b/libc/docs/gpu/building.rst
index 37dccdab6dc34..88643575ae4d9 100644
--- a/libc/docs/gpu/building.rst
+++ b/libc/docs/gpu/building.rst
@@ -63,10 +63,6 @@ targeting the default host environment as well.
 Runtimes cross build
 --------------------
 
-.. note::
-  These instructions need to be updated for new headergen. They may be
-  inaccurate.
-
 For users wanting more direct control over the build process, the build steps
 can be done manually instead. This build closely follows the instructions in the
 :ref:`main documentation<full_cross_build>` but is specialized for the GPU
@@ -82,20 +78,17 @@ compiler. These tools must all be up-to-date with the libc source.
   $> HOST_CXX_COMPILER=<C++ compiler for the host> # For example "clang++"
   $> cmake ../llvm                            \
      -G Ninja                                 \
-     -DLLVM_ENABLE_PROJECTS="clang;libc"      \
+     -DLLVM_ENABLE_PROJECTS="clang"           \
      -DCMAKE_C_COMPILER=$HOST_C_COMPILER      \
      -DCMAKE_CXX_COMPILER=$HOST_CXX_COMPILER  \
      -DLLVM_LIBC_FULL_BUILD=ON                \
-     -DLIBC_HDRGEN_ONLY=ON    \ # Only build the 'libc-hdrgen' tool
      -DCMAKE_BUILD_TYPE=Release # Release suggested to make "clang" fast
   $> ninja # Build the 'clang' compiler
-  $> ninja libc-hdrgen # Build the 'libc-hdrgen' tool
 
-Once this has finished the build directory should contain the ``clang`` compiler
-and the ``libc-hdrgen`` executable. We will use the ``clang`` compiler to build
-the GPU code and the ``libc-hdrgen`` tool to create the necessary headers. We
-use these tools to bootstrap the build out of the runtimes directory targeting a
-GPU architecture.
+Once this has finished the build directory should contain the ``clang``
+compiler executable. We will use the ``clang`` compiler to build the GPU code.
+We use these tools to bootstrap the build out of the runtimes directory
+targeting a GPU architecture.
 
 .. code-block:: sh
 
@@ -105,7 +98,6 @@ GPU architecture.
   $> TARGET_TRIPLE=<amdgcn-amd-amdhsa or nvptx64-nvidia-cuda>
   $> TARGET_C_COMPILER=</path/to/clang>
   $> TARGET_CXX_COMPILER=</path/to/clang++>
-  $> HDRGEN=</path/to/libc-hdrgen>
   $> cmake ../runtimes \ # Point to the runtimes build
      -G Ninja                                  \
      -DLLVM_ENABLE_RUNTIMES=libc               \
@@ -113,7 +105,6 @@ GPU architecture.
      -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \
      -DLLVM_LIBC_FULL_BUILD=ON                 \
      -DLLVM_RUNTIMES_TARGET=$TARGET_TRIPLE     \
-     -DLIBC_HDRGEN_EXE=$HDRGEN                 \
      -DCMAKE_BUILD_TYPE=Release
   $> ninja install
 
diff --git a/libc/docs/header_gen_scheme.svg b/libc/docs/header_gen_scheme.svg
deleted file mode 100644
index 2b2318b1a045a..0000000000000
--- a/libc/docs/header_gen_scheme.svg
+++ /dev/null
@@ -1 +0,0 @@
-<svg version="1.1" viewBox="0.0 0.0 962.0 926.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l962.0 0l0 926.0l-962.0 0l0 -926.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l962.0 0l0 926.0l-962.0 0z" fill-rule="evenodd"/><path fill="#cfe2f3" d="m16.17323 19.209974l140.53543 0l0 161.76378l-140.53543 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.17323 19.209974l140.53543 0l0 161.76378l-140.53543 0z" fill-rule="evenodd"/><path fill="#000000" d="m31.92323 40.626236l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.998184 0l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm11.183868 2.359375l0 0.375q0.328125 -0.3125 0.59375 -0.4375q0.265625 -0.125 0.609375 -0.125q0.28125 0 0.5625 0.140625q0.296875 0.140625 0.578125 0.4375q0.34375 -0.296875 0.6875 -0.421875q0.359375 -0.140625 0.734375 -0.140625q0.734375 0 1.203125 0.390625q0.609375 0.515625 0.609375 1.359375l0 3.0q0.421875 0 0.625 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.265625 0l0 -4.203125q0 -0.3125 -0.109375 -0.421875q-0.109375 -0.125 -0.34375 -0.125q-0.21875 0 -0.40625 0.109375q-0.25 0.171875 -0.59375 0.625l0 2.6875q0.421875 0 0.625 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.265625 0l0 -4.203125q0 -0.3125 -0.125 -0.421875q-0.109375 -0.125 -0.328125 -0.125q-0.234375 0 -0.46875 0.15625q-0.21875 0.140625 -0.546875 0.578125l0 2.6875q0.421875 0 0.625 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.625 -0.1875l0 -3.25q-0.421875 0 -0.625 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.265625 0zm11.263809 5.90625l0 -0.3125q-0.5 0.265625 -1.109375 0.390625q-0.609375 0.140625 -1.09375 0.140625q-1.078125 0 -1.75 -0.5625q-0.671875 -0.578125 -0.671875 -1.265625q0 -0.84375 0.859375 -1.5625q0.859375 -0.71875 2.359375 -0.71875q0.609375 0 1.40625 0.140625l0 -0.328125q0 -0.296875 -0.265625 -0.484375q-0.25 -0.1875 -0.96875 -0.1875q-0.59375 0 -1.546875 0.234375q-0.34375 0.078125 -0.546875 0.078125q-0.265625 0 -0.453125 -0.1875q-0.171875 -0.1875 -0.171875 -0.484375q0 -0.171875 0.0625 -0.296875q0.0625 -0.125 0.171875 -0.203125q0.125 -0.078125 0.5 -0.171875q0.484375 -0.140625 1.0 -0.21875q0.515625 -0.078125 0.9375 -0.078125q1.234375 0 1.921875 0.53125q0.6875 0.53125 0.6875 1.46875l0 2.75l0.234375 0q0.4843712 0 0.6874962 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6874962 0.1875l-1.5625 0zm0 -2.390625q-0.796875 -0.15625 -1.484375 -0.15625q-0.8125 0 -1.390625 0.40625q-0.375 0.25 -0.375 0.5q0 0.203125 0.1875 0.3125q0.3125 0.21875 0.890625 0.21875q0.46875 0 1.078125 -0.1875q0.609375 -0.1875 1.09375 -0.515625l0 -0.578125zm6.4669304 -2.1875l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.34375 -0.15625 0.53125 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.265625 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.46875 0zm6.951309 -3.859375l0 2.84375q0.421875 -0.25 0.84375 -0.359375q0.421875 -0.125 0.859375 -0.125q0.6875 0 1.21875 0.234375q0.546875 0.234375 0.90625 0.734375q0.359375 0.5 0.359375 1.265625l0 2.515625q0.5 0 0.65625 0.09375q0.3125 0.203125 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.171875 -0.109375 0.671875 -0.09375l0 -2.421875q0 -0.515625 -0.234375 -0.71875q-0.3125 -0.28125 -0.90625 -0.28125q-0.4375 0 -0.78125 0.171875q-0.34375 0.171875 -0.921875 0.75l0 2.5q0.515625 0 0.65625 0.09375q0.328125 0.203125 0.328125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.109375 0.65625 -0.09375l0 -5.78125l-0.21875 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm10.388809 7.765625q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm5.607559 -7.765625l0 2.84375q0.421875 -0.25 0.84375 -0.359375q0.421875 -0.125 0.859375 -0.125q0.6875 0 1.21875 0.234375q0.546875 0.234375 0.90625 0.734375q0.359375 0.5 0.359375 1.265625l0 2.515625q0.5 0 0.65625 0.09375q0.3125 0.203125 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.171875 -0.109375 0.671875 -0.09375l0 -2.421875q0 -0.515625 -0.234375 -0.71875q-0.3125 -0.28125 -0.90625 -0.28125q-0.4375 0 -0.78125 0.171875q-0.34375 0.171875 -0.921875 0.75l0 2.5q0.515625 0 0.65625 0.09375q0.328125 0.203125 0.328125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.109375 0.65625 -0.09375l0 -5.78125l-0.21875 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm10.388809 7.765625q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm10.045059 -7.765625l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875zm9.607559 0.5l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm6.107559 -0.96875l0 3.25l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.640625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.640625 0l0 -0.515625q0 -0.859375 0.65625 -1.4375q0.65625 -0.578125 1.890625 -0.578125q0.5625 0 1.28125 0.109375q0.7187576 0.09375 0.9218826 0.28125q0.21875 0.171875 0.21875 0.453125q0 0.3125 -0.1875 0.515625q-0.1875 0.1875 -0.453125 0.1875q-0.125 0 -0.35938263 -0.046875q-0.828125 -0.171875 -1.46875 -0.171875q-0.671875 0 -0.921875 0.203125q-0.25 0.203125 -0.25 0.484375l0 0.515625l2.0625 0q0.484375 0 0.6875 0.1875q0.20313263 0.1875 0.20313263 0.484375q0 0.28125 -0.20313263 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.0625 0z" fill-rule="nonzero"/><path fill="#000000" d="m27.11073 80.42311q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.453125 0.1875q-0.28125 0 -0.46875 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.46875 -0.1875q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125z" fill-rule="nonzero"/><path fill="#000000" d="m29.876354 106.438736q0 0.6875 -0.484375 1.171875q-0.46875 0.484375 -1.140625 0.484375q-0.671875 0 -1.15625 -0.484375q-0.46875 -0.484375 -0.46875 -1.171875q0 -0.6875 0.46875 -1.15625q0.484375 -0.484375 1.15625 -0.484375q0.6875 0 1.15625 0.484375q0.46875 0.46875 0.46875 1.15625zm-0.375 0q0 -0.515625 -0.375 -0.890625q-0.359375 -0.375 -0.875 -0.375q-0.5 0 -0.875 0.375q-0.359375 0.375 -0.359375 0.890625q0 0.53125 0.359375 0.90625q0.375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.90625zm2.109375 1.84375l-4.9375 1.609375q-0.09375 0.03125 -0.140625 0.03125q-0.0625 0 -0.125 -0.046875q-0.046875 -0.0625 -0.046875 -0.140625q0 -0.0625 0.046875 -0.125q0.03125 -0.046875 0.140625 -0.078125l4.9375 -1.609375q0.09375 -0.03125 0.140625 -0.03125q0.0625 0 0.109375 0.0625q0.0625 0.0625 0.0625 0.125q0 0.078125 -0.046875 0.140625q-0.03125 0.03125 -0.140625 0.0625zm-0.078125 3.125q0 0.671875 -0.484375 1.15625q-0.484375 0.484375 -1.15625 0.484375q-0.65625 0 -1.140625 -0.484375q-0.484375 -0.484375 -0.484375 -1.15625q0 -0.6875 0.46875 -1.171875q0.484375 -0.484375 1.15625 -0.484375q0.671875 0 1.15625 0.484375q0.484375 0.484375 0.484375 1.171875zm-0.390625 0q0 -0.546875 -0.375 -0.90625q-0.359375 -0.359375 -0.875 -0.359375q-0.515625 0 -0.875 0.375q-0.359375 0.359375 -0.359375 0.890625q0 0.515625 0.359375 0.890625q0.359375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.890625zm6.5356445 -4.96875q0 0.6875 -0.484375 1.171875q-0.46875 0.484375 -1.140625 0.484375q-0.671875 0 -1.15625 -0.484375q-0.46875 -0.484375 -0.46875 -1.171875q0 -0.6875 0.46875 -1.15625q0.484375 -0.484375 1.15625 -0.484375q0.6875 0 1.15625 0.484375q0.46875 0.46875 0.46875 1.15625zm-0.375 0q0 -0.515625 -0.375 -0.890625q-0.359375 -0.375 -0.875 -0.375q-0.5 0 -0.875 0.375q-0.359375 0.375 -0.359375 0.890625q0 0.53125 0.359375 0.90625q0.375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.90625zm2.109375 1.84375l-4.9375 1.609375q-0.09375 0.03125 -0.140625 0.03125q-0.0625 0 -0.125 -0.046875q-0.046875 -0.0625 -0.046875 -0.140625q0 -0.0625 0.046875 -0.125q0.03125 -0.046875 0.140625 -0.078125l4.9375 -1.609375q0.09375 -0.03125 0.140625 -0.03125q0.0625 0 0.109375 0.0625q0.0625 0.0625 0.0625 0.125q0 0.078125 -0.046875 0.140625q-0.03125 0.03125 -0.140625 0.0625zm-0.078125 3.125q0 0.671875 -0.484375 1.15625q-0.484375 0.484375 -1.15625 0.484375q-0.65625 0 -1.140625 -0.484375q-0.484375 -0.484375 -0.484375 -1.15625q0 -0.6875 0.46875 -1.171875q0.484375 -0.484375 1.15625 -0.484375q0.671875 0 1.15625 0.484375q0.484375 0.484375 0.484375 1.171875zm-0.390625 0q0 -0.546875 -0.375 -0.90625q-0.359375 -0.359375 -0.875 -0.359375q-0.515625 0 -0.875 0.375q-0.359375 0.359375 -0.359375 0.890625q0 0.515625 0.359375 0.890625q0.359375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.890625zm3.7231445 -4.015625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm7.0043945 2.90625l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm3.9731445 -7.96875l0 3.53125q0.96875 -1.25 2.328125 -1.25q1.171875 0 2.0 0.84375q0.828125 0.84375 0.828125 2.078125q0 1.25 -0.84375 2.109375q-0.828125 0.859375 -1.984375 0.859375q-1.390625 0 -2.328125 -1.25l0 1.046875l-1.25 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.71875 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 5.234375q0 -1.015625 -0.6875 -1.703125q-0.6875 -0.703125 -1.625 -0.703125q-0.921875 0 -1.625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.703125 0.703125 1.625 0.703125q0.9375 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm5.4575157 -5.234375l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm7.8012695 -0.265625l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm9.129395 8.515625l-7.09375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l7.09375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.125 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125zm5.7856445 -3.5625l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm4.2543945 -2.84375l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm5.4575195 -5.328125l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm7.4575195 2.34375q0 -0.734375 0.1875 -1.546875q0.1875 -0.828125 0.71875 -1.890625q0.546875 -1.078125 0.796875 -1.296875q0.078125 -0.078125 0.171875 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.0625 -0.046875 0.140625q-0.6875 1.28125 -1.0 2.328125q-0.296875 1.03125 -0.296875 2.078125q0 1.046875 0.296875 2.09375q0.3125 1.03125 1.0 2.3125q0.046875 0.078125 0.046875 0.140625q0 0.09375 -0.078125 0.171875q-0.078125 0.09375 -0.1875 0.09375q-0.09375 0 -0.171875 -0.078125q-0.234375 -0.21875 -0.765625 -1.265625q-0.53125 -1.046875 -0.734375 -1.84375q-0.203125 -0.8125 -0.203125 -1.625zm8.020027 0q0 0.734375 -0.203125 1.546875q-0.1875 0.8125 -0.734375 1.890625q-0.53125 1.078125 -0.78125 1.296875q-0.078125 0.078125 -0.15625 0.078125q-0.125 0 -0.203125 -0.09375q-0.078125 -0.078125 -0.078125 -0.171875q0 -0.0625 0.046875 -0.140625q0.703125 -1.28125 1.0 -2.3125q0.296875 -1.046875 0.296875 -2.09375q0 -1.046875 -0.296875 -2.078125q-0.296875 -1.046875 -1.0 -2.328125q-0.046875 -0.078125 -0.046875 -0.140625q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.203125 -0.078125q0.078125 0 0.15625 0.078125q0.234375 0.203125 0.765625 1.25q0.546875 1.046875 0.75 1.859375q0.203125 0.8125 0.203125 1.625z" fill-rule="nonzero"/><path fill="#000000" d="m27.11073 144.42311q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.453125 0.1875q-0.28125 0 -0.46875 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.46875 -0.1875q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m222.58792 9.887139l290.4252 0l0 403.68503l-290.4252 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m222.58792 9.887139l290.4252 0l0 403.68503l-290.4252 0z" fill-rule="evenodd"/><path fill="#000000" d="m238.33792 56.26403l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.998184 0l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.326309 2.59375q0.234375 -0.21875 0.5 -0.21875q0.28125 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 0.859375q0 0.484375 -0.1875 0.6875q-0.1875 0.203125 -0.484375 0.203125q-0.28125 0 -0.46875 -0.15625q-0.125 -0.125 -0.203125 -0.484375q-0.078125 -0.359375 -0.390625 -0.53125q-0.515625 -0.3125 -1.34375 -0.3125q-0.9375 0 -1.515625 0.5625q-0.5625 0.546875 -0.5625 1.390625q0 0.78125 0.546875 1.234375q0.546875 0.453125 1.8125 0.453125q0.828125 0 1.359375 -0.171875q0.3125 -0.109375 0.59375 -0.359375q0.28125 -0.25 0.5 -0.25q0.28125 0 0.46875 0.203125q0.203125 0.203125 0.203125 0.484375q0 0.4375 -0.59375 0.828125q-0.90625 0.59375 -2.640625 0.59375q-1.546875 0 -2.421875 -0.640625q-1.171875 -0.859375 -1.171875 -2.375q0 -1.421875 0.953125 -2.34375q0.953125 -0.9375 2.484375 -0.9375q0.546875 0 1.015625 0.109375q0.484375 0.09375 0.890625 0.296875zm9.388824 2.828125q0 0.765625 -0.4375 1.484375q-0.421875 0.71875 -1.265625 1.15625q-0.84375 0.421875 -1.75 0.421875q-0.921875 0 -1.75 -0.421875q-0.828125 -0.421875 -1.265625 -1.140625q-0.4375 -0.71875 -0.4375 -1.515625q0 -0.796875 0.4375 -1.5625q0.4375 -0.78125 1.265625 -1.21875q0.84375 -0.4375 1.75 -0.4375q0.90625 0 1.75 0.453125q0.84375 0.4375 1.265625 1.203125q0.4375 0.765625 0.4375 1.578125zm-1.34375 0q0 -0.640625 -0.453125 -1.171875q-0.640625 -0.734375 -1.65625 -0.734375q-0.90625 0 -1.515625 0.578125q-0.609375 0.578125 -0.609375 1.34375q0 0.625 0.609375 1.171875q0.625 0.53125 1.515625 0.53125q0.890625 0 1.5 -0.53125q0.609375 -0.546875 0.609375 -1.1875zm4.529419 -3.0625l0 0.453125q0.359375 -0.3125 0.796875 -0.46875q0.4375 -0.15625 0.953125 -0.15625q1.1875 0 1.875 0.734375q0.546875 0.578125 0.546875 1.53125l0 2.484375q0.421875 0 0.625 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.1875 -0.46875q0.203125 -0.1875 0.640625 -0.1875l0 -2.53125q0 -0.421875 -0.234375 -0.625q-0.3125 -0.265625 -0.921875 -0.265625q-0.46875 0 -0.8125 0.1875q-0.34375 0.171875 -0.875 0.75l0 2.484375q0.5 0 0.65625 0.09375q0.3125 0.1875 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.09375 0.671875 -0.09375l0 -3.25q-0.421875 0 -0.625 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.28125 0zm9.326324 1.328125l0 3.25l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.640625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.640625 0l0 -0.515625q0 -0.859375 0.65625 -1.4375q0.65625 -0.578125 1.890625 -0.578125q0.5625 0 1.28125 0.109375q0.71875 0.09375 0.921875 0.28125q0.21875 0.171875 0.21875 0.453125q0 0.3125 -0.1875 0.515625q-0.1875 0.1875 -0.453125 0.1875q-0.125 0 -0.359375 -0.046875q-0.828125 -0.171875 -1.46875 -0.171875q-0.671875 0 -0.921875 0.203125q-0.25 0.203125 -0.25 0.484375l0 0.515625l2.0625 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.0625 0zm10.763794 -3.6875l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm5.9200745 -0.171875l0 7.109375l1.59375 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.53125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.59375 0l0 -5.78125l-1.0625 0q-0.484375 0 -0.703125 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.21875 -0.1875 0.703125 -0.1875l2.40625 0zm7.810669 0l0 1.40625l-1.59375 0l0 -1.40625l1.59375 0zm0.171875 2.53125l0 4.578125l1.609375 0q0.46875 0 0.671875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.671875 0.1875l-4.546875 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.21875 -0.1875 0.6875 -0.1875l1.609375 0l0 -3.25l-1.078125 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l2.40625 0zm5.9981995 0l0 0.453125q0.359375 -0.3125 0.796875 -0.46875q0.4375 -0.15625 0.953125 -0.15625q1.1875 0 1.875 0.734375q0.546875 0.578125 0.546875 1.53125l0 2.484375q0.421875 0 0.625 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.1875 -0.46875q0.203125 -0.1875 0.640625 -0.1875l0 -2.53125q0 -0.421875 -0.234375 -0.625q-0.3125 -0.265625 -0.921875 -0.265625q-0.46875 0 -0.8125 0.1875q-0.34375 0.171875 -0.875 0.75l0 2.484375q0.5 0 0.65625 0.09375q0.3125 0.1875 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.09375 0.671875 -0.09375l0 -3.25q-0.421875 0 -0.625 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.28125 0zm12.170044 0l0 4.578125q0.421875 0 0.625 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.265625 0l0 -0.3125q-0.578125 0.265625 -1.09375 0.390625q-0.515625 0.140625 -0.984375 0.140625q-0.65625 0 -1.140625 -0.28125q-0.484375 -0.28125 -0.765625 -0.765625q-0.203125 -0.34375 -0.203125 -0.875l0 -2.875l-0.21875 0q-0.484375 0 -0.703125 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.21875 -0.1875 0.703125 -0.1875l1.5625 0l0 3.96875q0 0.421875 0.203125 0.625q0.203125 0.1875 0.625 0.1875q0.40625 0 0.859375 -0.15625q0.46875 -0.15625 1.15625 -0.578125l0 -2.71875l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.8125 0zm6.1700745 2.828125l2.015625 1.75q0.375 0.015625 0.546875 0.203125q0.1875 0.1875 0.1875 0.46875q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.234375 0.140625 -0.40625q0.15625 -0.1875 0.40625 -0.25l-1.0 -0.859375l-1.015625 0.859375q0.3125 0.078125 0.4375 0.234375q0.140625 0.15625 0.140625 0.421875q0 0.296875 -0.203125 0.484375q-0.203125 0.1875 -0.6875 0.1875l-1.5 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.265625 0.171875 -0.453125q0.1875 -0.1875 0.5625 -0.203125l1.96875 -1.75l-1.75 -1.5q-0.34375 -0.015625 -0.53125 -0.203125q-0.171875 -0.1875 -0.171875 -0.46875q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.25 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.46875q0 0.375 -0.359375 0.625l0.8125 0.671875l0.78125 -0.6875q-0.359375 -0.25 -0.359375 -0.578125q0 -0.3125 0.203125 -0.5q0.21875 -0.1875 0.703125 -0.1875l1.234375 0q0.46875 0 0.671875 0.1875q0.21875 0.1875 0.21875 0.484375q0 0.265625 -0.1875 0.453125q-0.171875 0.1875 -0.53125 0.203125l-1.75 1.5zm9.748169 -5.1875l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm3.8731995 -0.171875l0 2.84375q0.421875 -0.25 0.84375 -0.359375q0.421875 -0.125 0.859375 -0.125q0.6875 0 1.21875 0.234375q0.546875 0.234375 0.90625 0.734375q0.359375 0.5 0.359375 1.265625l0 2.515625q0.5 0 0.65625 0.09375q0.3125 0.203125 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.171875 -0.109375 0.671875 -0.09375l0 -2.421875q0 -0.515625 -0.234375 -0.71875q-0.3125 -0.28125 -0.90625 -0.28125q-0.4375 0 -0.78125 0.171875q-0.34375 0.171875 -0.921875 0.75l0 2.5q0.515625 0 0.65625 0.09375q0.328125 0.203125 0.328125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.109375 0.65625 -0.09375l0 -5.78125l-0.21875 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm12.716919 6.15625l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm7.3263245 3.609375l0 -0.3125q-0.5 0.265625 -1.109375 0.390625q-0.609375 0.140625 -1.09375 0.140625q-1.078125 0 -1.75 -0.5625q-0.671875 -0.578125 -0.671875 -1.265625q0 -0.84375 0.859375 -1.5625q0.859375 -0.71875 2.359375 -0.71875q0.609375 0 1.40625 0.140625l0 -0.328125q0 -0.296875 -0.265625 -0.484375q-0.25 -0.1875 -0.96875 -0.1875q-0.59375 0 -1.546875 0.234375q-0.34375 0.078125 -0.546875 0.078125q-0.265625 0 -0.453125 -0.1875q-0.171875 -0.1875 -0.171875 -0.484375q0 -0.171875 0.0625 -0.296875q0.0625 -0.125 0.171875 -0.203125q0.125 -0.078125 0.5 -0.171875q0.484375 -0.140625 1.0 -0.21875q0.515625 -0.078125 0.9375 -0.078125q1.234375 0 1.921875 0.53125q0.6875 0.53125 0.6875 1.46875l0 2.75l0.234375 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.5625 0zm0 -2.390625q-0.796875 -0.15625 -1.484375 -0.15625q-0.8125 0 -1.390625 0.40625q-0.375 0.25 -0.375 0.5q0 0.203125 0.1875 0.3125q0.3125 0.21875 0.890625 0.21875q0.46875 0 1.078125 -0.1875q0.609375 -0.1875 1.09375 -0.515625l0 -0.578125zm9.857544 -6.046875l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875zm9.607574 0.5l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm5.841919 -2.296875l0 0.828125q0.84375 -0.609375 1.328125 -0.796875q0.484375 -0.203125 0.90625 -0.203125q0.65625 0 1.265625 0.46875q0.421875 0.328125 0.421875 0.671875q0 0.28125 -0.203125 0.484375q-0.203125 0.203125 -0.484375 0.203125q-0.25 0 -0.53125 -0.25q-0.265625 -0.25 -0.46875 -0.25q-0.28125 0 -0.84375 0.359375q-0.546875 0.34375 -1.390625 1.046875l0 2.015625l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.8125 0zm11.966949 9.90625l-7.328125 0q-0.46875 0 -0.671875 -0.1875q-0.21875 -0.171875 -0.21875 -0.46875q0 -0.296875 0.21875 -0.484375q0.203125 -0.1875 0.671875 -0.1875l7.328125 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875zm6.388794 -9.671875q0.234375 -0.21875 0.5 -0.21875q0.28125 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 0.859375q0 0.484375 -0.1875 0.6875q-0.1875 0.203125 -0.484375 0.203125q-0.28125 0 -0.46875 -0.15625q-0.125 -0.125 -0.203125 -0.484375q-0.078125 -0.359375 -0.390625 -0.53125q-0.515625 -0.3125 -1.34375 -0.3125q-0.9375 0 -1.515625 0.5625q-0.5625 0.546875 -0.5625 1.390625q0 0.78125 0.546875 1.234375q0.546875 0.453125 1.8125 0.453125q0.828125 0 1.359375 -0.171875q0.3125 -0.109375 0.59375 -0.359375q0.28125 -0.25 0.5 -0.25q0.28125 0 0.46875 0.203125q0.203125 0.203125 0.203125 0.484375q0 0.4375 -0.59375 0.828125q-0.90625 0.59375 -2.640625 0.59375q-1.546875 0 -2.421875 -0.640625q-1.171875 -0.859375 -1.171875 -2.375q0 -1.421875 0.953125 -2.34375q0.953125 -0.9375 2.484375 -0.9375q0.546875 0 1.015625 0.109375q0.484375 0.09375 0.890625 0.296875zm9.388824 2.828125q0 0.765625 -0.4375 1.484375q-0.421875 0.71875 -1.265625 1.15625q-0.84375 0.421875 -1.75 0.421875q-0.921875 0 -1.75 -0.421875q-0.828125 -0.421875 -1.265625 -1.140625q-0.4375 -0.71875 -0.4375 -1.515625q0 -0.796875 0.4375 -1.5625q0.4375 -0.78125 1.265625 -1.21875q0.84375 -0.4375 1.75 -0.4375q0.90625 0 1.75 0.453125q0.84375 0.4375 1.265625 1.203125q0.4375 0.765625 0.4375 1.578125zm-1.34375 0q0 -0.640625 -0.453125 -1.171875q-0.640625 -0.734375 -1.65625 -0.734375q-0.90625 0 -1.515625 0.578125q-0.609375 0.578125 -0.609375 1.34375q0 0.625 0.609375 1.171875q0.625 0.53125 1.515625 0.53125q0.890625 0 1.5 -0.53125q0.609375 -0.546875 0.609375 -1.1875zm4.529419 -3.0625l0 0.453125q0.359375 -0.3125 0.796875 -0.46875q0.4375 -0.15625 0.953125 -0.15625q1.1875 0 1.875 0.734375q0.546875 0.578125 0.546875 1.53125l0 2.484375q0.421875 0 0.625 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.1875 -0.46875q0.203125 -0.1875 0.640625 -0.1875l0 -2.53125q0 -0.421875 -0.234375 -0.625q-0.3125 -0.265625 -0.921875 -0.265625q-0.46875 0 -0.8125 0.1875q-0.34375 0.171875 -0.875 0.75l0 2.484375q0.5 0 0.65625 0.09375q0.3125 0.1875 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.09375 0.671875 -0.09375l0 -3.25q-0.421875 0 -0.625 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.28125 0zm9.326324 1.328125l0 3.25l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.640625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.640625 0l0 -0.515625q0 -0.859375 0.65625 -1.4375q0.65625 -0.578125 1.890625 -0.578125q0.5625 0 1.28125 0.109375q0.71875 0.09375 0.921875 0.28125q0.21875 0.171875 0.21875 0.453125q0 0.3125 -0.1875 0.515625q-0.1875 0.1875 -0.453125 0.1875q-0.125 0 -0.359375 -0.046875q-0.828125 -0.171875 -1.46875 -0.171875q-0.671875 0 -0.921875 0.203125q-0.25 0.203125 -0.25 0.484375l0 0.515625l2.0625 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.0625 0zm9.029419 3.90625q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm6.6544495 -3.90625l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.34375 -0.15625 0.53125 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.265625 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.46875 0zm11.388794 -3.859375l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875z" fill-rule="nonzero"/><path fill="#000000" d="m235.7598 88.29528l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm5.7856445 0l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm5.8481445 -3.015625l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm9.348145 7.96875l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm8.598145 -7.96875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm10.493164 -4.8125l1.375 0l1.09375 2.734375q0.046875 0.15625 0.046875 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.171875 0 -0.328125 -0.25l-1.859375 -3.03125zm2.734375 0l1.390625 0l1.078125 2.734375q0.0625 0.15625 0.0625 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.1875 0 -0.34375 -0.25l-1.859375 -3.03125zm9.58252 2.828125q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.0356445 -0.359375l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm7.9731445 -3.40625l-4.296875 9.203125q-0.109375 0.234375 -0.265625 0.234375q-0.109375 0 -0.203125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.0625 0.078125 -0.203125l4.28125 -9.1875q0.0625 -0.15625 0.125 -0.1875q0.0625 -0.046875 0.15625 -0.046875q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.171875q0 0.0625 -0.0625 0.203125zm7.1606445 3.21875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.504395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm7.9575195 -2.21875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm5.4731445 3.421875l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm6.8168945 -3.96875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.504395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.3793945 -5.234375l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375z" fill-rule="nonzero"/><path fill="#000000" d="m235.7598 105.63903l0 6.359375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 -6.359375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l4.1875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0zm5.8012695 1.390625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm5.8481445 -3.015625l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm9.348145 7.96875l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm8.598145 -7.96875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm10.493164 -4.8125l1.375 0l1.09375 2.734375q0.046875 0.15625 0.046875 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.171875 0 -0.328125 -0.25l-1.859375 -3.03125zm2.734375 0l1.390625 0l1.078125 2.734375q0.0625 0.15625 0.0625 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.1875 0 -0.34375 -0.25l-1.859375 -3.03125zm9.58252 2.828125q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.0356445 -0.359375l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm7.9731445 -3.40625l-4.296875 9.203125q-0.109375 0.234375 -0.265625 0.234375q-0.109375 0 -0.203125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.0625 0.078125 -0.203125l4.28125 -9.1875q0.0625 -0.15625 0.125 -0.1875q0.0625 -0.046875 0.15625 -0.046875q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.171875q0 0.0625 -0.0625 0.203125zm3.3950195 2.859375l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.160645 0.171875q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm7.1137695 -2.40625q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm6.3168945 -3.09375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm7.8793945 2.625l2.4375 2.34375q0.25 0 0.3125 0.03125q0.0625 0.015625 0.109375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.890625 0l-2.078125 -1.984375l-2.0625 1.984375l0.890625 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.296875 -0.03125l2.453125 -2.34375l-2.171875 -2.078125q-0.234375 0 -0.296875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.046875 -0.0625 -0.046875 -0.15625q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.640625 0l1.796875 1.734375l1.8125 -1.734375l-0.640625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.03125 -0.296875 0.03125l-2.171875 2.078125zm7.3325195 1.34375l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm6.8168945 -3.96875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.504395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.3793945 -5.234375l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375z" fill-rule="nonzero"/><path fill="#000000" d="m238.10355 136.56091l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm5.0981445 -1.796875l0 4.421875l2.34375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.171875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.421875l-1.15625 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.15625 0l0 -0.796875q0 -0.671875 0.53125 -1.171875q0.546875 -0.5 1.453125 -0.5q0.75 0 1.609375 0.140625q0.328125 0.046875 0.390625 0.125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.0625 -0.203125 0.0625q-0.0625 0 -0.171875 -0.015625q-0.96875 -0.140625 -1.625 -0.140625q-0.703125 0 -1.078125 0.34375q-0.375 0.34375 -0.375 0.78125l0 0.796875l2.5 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.5 0zm16.305664 2.71875l-0.609375 0l-2.046875 -4.65625l-0.109375 0l0 6.359375l0.984375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.953125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.4375 0l0 -6.359375l-0.328125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.3125 0l2.015625 4.59375l1.984375 -4.59375l1.3125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.34375 0l0 6.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.96875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -6.359375l-0.109375 0l-2.015625 4.65625zm8.98877 2.234375l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm5.1762695 -2.84375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm7.1293945 -2.46875l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm17.25879 2.46875l0.21875 0q0.359375 0 0.609375 0.265625q0.25 0.25 0.25 0.59375q0 0.375 -0.25 0.625q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.265625 -0.25 -0.609375q0 -0.375 0.25 -0.625q0.265625 -0.25 0.625 -0.25zm0 3.96875l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm14.555664 -1.515625l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm10.77002 3.578125l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm3.9731445 -7.96875l0 3.53125q0.96875 -1.25 2.328125 -1.25q1.171875 0 2.0 0.84375q0.828125 0.84375 0.828125 2.078125q0 1.25 -0.84375 2.109375q-0.828125 0.859375 -1.984375 0.859375q-1.390625 0 -2.328125 -1.25l0 1.046875l-1.25 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.71875 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 5.234375q0 -1.015625 -0.6875 -1.703125q-0.6875 -0.703125 -1.625 -0.703125q-0.921875 0 -1.625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.703125 0.703125 1.625 0.703125q0.9375 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm5.4575195 -5.234375l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm7.8012695 -0.265625l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm7.3325195 2.46875l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.0981445 -0.03125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm9.223145 -3.3125l0 6.359375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 -6.359375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l4.1875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0zm4.2543945 3.1875l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm9.723145 -4.265625l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm3.9887695 2.46875l0 0.546875q0.6875 -0.734375 1.375 -0.734375q0.40625 0 0.71875 0.21875q0.3125 0.21875 0.515625 0.65625q0.359375 -0.4375 0.71875 -0.65625q0.375 -0.21875 0.734375 -0.21875q0.578125 0 0.921875 0.375q0.453125 0.484375 0.453125 1.046875l0 3.734375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.984375 0l0 -4.203125q0 -0.40625 -0.25 -0.671875q-0.25 -0.265625 -0.578125 -0.265625q-0.28125 0 -0.609375 0.21875q-0.328125 0.21875 -0.734375 0.859375l0 3.53125l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 -4.171875q0 -0.421875 -0.25 -0.6875q-0.25 -0.28125 -0.5625 -0.28125q-0.28125 0 -0.5625 0.171875q-0.390625 0.265625 -0.8125 0.90625l0 3.53125l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.453125 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.77002 5.5l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm5.1762695 -2.84375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm7.1293945 -2.46875l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm8.20752 0l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm9.691895 4.265625l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm12.024414 0.546875q0.3125 0.234375 0.4375 0.453125q0.140625 0.28125 0.140625 0.671875l0 2.28125q0 0.359375 0.203125 0.59375q0.21875 0.234375 0.53125 0.265625q0.1875 0.015625 0.25 0.078125q0.09375 0.078125 0.09375 0.1875q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.234375 0.078125q-0.5 0 -0.84375 -0.3125q-0.453125 -0.4375 -0.453125 -1.0625l0 -2.296875q0 -0.34375 -0.203125 -0.578125q-0.203125 -0.234375 -0.53125 -0.265625q-0.1875 -0.015625 -0.25 -0.078125q-0.0625 -0.078125 -0.0625 -0.203125q0 -0.125 0.0625 -0.1875q0.0625 -0.078125 0.25 -0.09375q0.328125 -0.03125 0.53125 -0.265625q0.203125 -0.234375 0.203125 -0.59375l0 -2.28125q0 -0.59375 0.375 -0.984375q0.390625 -0.40625 0.921875 -0.40625q0.15625 0 0.234375 0.078125q0.09375 0.078125 0.09375 0.1875q0 0.125 -0.09375 0.1875q-0.046875 0.0625 -0.25 0.09375q-0.328125 0.03125 -0.53125 0.265625q-0.203125 0.234375 -0.203125 0.578125l0 2.28125q0 0.40625 -0.125 0.65625q-0.125 0.234375 -0.453125 0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m251.36234 152.56091l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm10.410645 5.34375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.3637695 -2.34375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm15.399414 1.90625l0 3.0625l1.828125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-3.078125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.09375 -0.203125q0.0625 -0.0625 0.25 -0.0625l5.875 0l0 1.828125q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -1.296875l-4.09375 0l0 2.765625l1.921875 0l0 -0.59375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.734375q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -0.609375l-1.921875 0zm10.910645 3.59375l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm4.2543945 -5.5l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.160645 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.48877 2.765625q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.285645 0.359375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm16.75879 0.734375l-5.8125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125zm0 1.90625l-5.8125 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.125 -0.09375 0.203125q-0.078125 0.0625 -0.265625 0.0625zm12.961914 -4.921875l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m263.93362 168.56091l1.375 0l1.09375 2.734375q0.046875 0.15625 0.046875 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.171875 0 -0.328125 -0.25l-1.859375 -3.03125zm2.734375 0l1.390625 0l1.078125 2.734375q0.0625 0.15625 0.0625 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.1875 0 -0.34375 -0.25l-1.859375 -3.03125zm9.36377 7.96875l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm8.48877 -2.296875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm8.55127 2.21875q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm7.1137695 -2.40625q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.6918945 -2.828125l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm5.3637695 6.046875l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m266.58987 190.99841l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25z" fill-rule="nonzero"/><path fill="#000000" d="m250.84671 209.63904l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0zm7.9262695 -6.609375l0.234375 0q0.359375 0 0.609375 0.265625q0.25 0.25 0.25 0.59375q0 0.375 -0.25 0.625q-0.25 0.25 -0.609375 0.25l-0.234375 0q-0.359375 0 -0.609375 -0.25q-0.25 -0.265625 -0.25 -0.609375q0 -0.375 0.25 -0.625q0.265625 -0.25 0.609375 -0.25zm-0.921875 3.578125l1.671875 0l-1.796875 2.953125q-0.125 0.21875 -0.21875 0.265625q-0.078125 0.0625 -0.171875 0.0625q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.09375 -0.234375q0 -0.078125 0.03125 -0.1875l0.84375 -2.765625z" fill-rule="nonzero"/><path fill="#000000" d="m235.80667 221.37341q-0.328125 -0.25 -0.4375 -0.46875q-0.140625 -0.265625 -0.140625 -0.671875l0 -2.28125q0 -0.359375 -0.21875 -0.59375q-0.203125 -0.234375 -0.515625 -0.25q-0.203125 -0.03125 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.234375 -0.078125q0.515625 0 0.84375 0.328125q0.453125 0.421875 0.453125 1.0625l0 2.28125q0 0.359375 0.203125 0.59375q0.21875 0.234375 0.546875 0.265625q0.171875 0.015625 0.234375 0.09375q0.078125 0.0625 0.078125 0.1875q0 0.125 -0.078125 0.203125q-0.0625 0.0625 -0.234375 0.078125q-0.328125 0.03125 -0.546875 0.265625q-0.203125 0.234375 -0.203125 0.578125l0 2.296875q0 0.59375 -0.390625 0.984375q-0.375 0.40625 -0.90625 0.40625q-0.15625 0 -0.234375 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.0625 -0.0625 0.265625 -0.078125q0.3125 -0.03125 0.515625 -0.265625q0.21875 -0.234375 0.21875 -0.59375l0 -2.28125q0 -0.390625 0.125 -0.640625q0.125 -0.25 0.453125 -0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m238.10355 248.56091l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm5.0981445 -1.796875l0 4.421875l2.34375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.171875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.421875l-1.15625 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.15625 0l0 -0.796875q0 -0.671875 0.53125 -1.171875q0.546875 -0.5 1.453125 -0.5q0.75 0 1.609375 0.140625q0.328125 0.046875 0.390625 0.125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.0625 -0.203125 0.0625q-0.0625 0 -0.171875 -0.015625q-0.96875 -0.140625 -1.625 -0.140625q-0.703125 0 -1.078125 0.34375q-0.375 0.34375 -0.375 0.78125l0 0.796875l2.5 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.5 0zm17.868164 -1.9375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm4.8012695 1.390625l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm5.7856445 0l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.316895 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm13.711914 -2.59375l0.21875 0q0.359375 0 0.609375 0.265625q0.25 0.25 0.25 0.59375q0 0.375 -0.25 0.625q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.265625 -0.25 -0.609375q0 -0.375 0.25 -0.625q0.265625 -0.25 0.625 -0.25zm0 3.96875l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm14.555664 -1.515625l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm10.77002 3.578125l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm3.9731445 -7.96875l0 3.53125q0.96875 -1.25 2.328125 -1.25q1.171875 0 2.0 0.84375q0.828125 0.84375 0.828125 2.078125q0 1.25 -0.84375 2.109375q-0.828125 0.859375 -1.984375 0.859375q-1.390625 0 -2.328125 -1.25l0 1.046875l-1.25 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.71875 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 5.234375q0 -1.015625 -0.6875 -1.703125q-0.6875 -0.703125 -1.625 -0.703125q-0.921875 0 -1.625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.703125 0.703125 1.625 0.703125q0.9375 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm5.4575195 -5.234375l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm7.8012695 -0.265625l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm7.3325195 2.46875l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.0981445 -0.03125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm9.223145 -3.3125l0 6.359375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 -6.359375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l4.1875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0zm4.2543945 3.1875l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm9.723145 -4.265625l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm8.17627 2.828125q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm5.7856445 0l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.316895 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm4.6606445 -5.0625l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm9.691895 4.265625l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm12.024414 0.546875q0.3125 0.234375 0.4375 0.453125q0.140625 0.28125 0.140625 0.671875l0 2.28125q0 0.359375 0.203125 0.59375q0.21875 0.234375 0.53125 0.265625q0.1875 0.015625 0.25 0.078125q0.09375 0.078125 0.09375 0.1875q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.234375 0.078125q-0.5 0 -0.84375 -0.3125q-0.453125 -0.4375 -0.453125 -1.0625l0 -2.296875q0 -0.34375 -0.203125 -0.578125q-0.203125 -0.234375 -0.53125 -0.265625q-0.1875 -0.015625 -0.25 -0.078125q-0.0625 -0.078125 -0.0625 -0.203125q0 -0.125 0.0625 -0.1875q0.0625 -0.078125 0.25 -0.09375q0.328125 -0.03125 0.53125 -0.265625q0.203125 -0.234375 0.203125 -0.59375l0 -2.28125q0 -0.59375 0.375 -0.984375q0.390625 -0.40625 0.921875 -0.40625q0.15625 0 0.234375 0.078125q0.09375 0.078125 0.09375 0.1875q0 0.125 -0.09375 0.1875q-0.046875 0.0625 -0.25 0.09375q-0.328125 0.03125 -0.53125 0.265625q-0.203125 0.234375 -0.203125 0.578125l0 2.28125q0 0.40625 -0.125 0.65625q-0.125 0.234375 -0.453125 0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m251.36234 264.5609l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm10.410645 5.34375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.3637695 -2.34375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm15.399414 1.90625l0 3.0625l1.828125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-3.078125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.09375 -0.203125q0.0625 -0.0625 0.25 -0.0625l5.875 0l0 1.828125q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -1.296875l-4.09375 0l0 2.765625l1.921875 0l0 -0.59375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.734375q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -0.609375l-1.921875 0zm10.910645 3.59375l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm4.2543945 -5.5l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.160645 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.48877 2.765625q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.285645 0.359375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm16.75879 0.734375l-5.8125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125zm0 1.90625l-5.8125 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.125 -0.09375 0.203125q-0.078125 0.0625 -0.265625 0.0625zm12.961914 -4.921875l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m263.93362 280.5609l1.375 0l1.09375 2.734375q0.046875 0.15625 0.046875 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.171875 0 -0.328125 -0.25l-1.859375 -3.03125zm2.734375 0l1.390625 0l1.078125 2.734375q0.0625 0.15625 0.0625 0.21875q0 0.140625 -0.09375 0.234375q-0.09375 0.09375 -0.234375 0.09375q-0.1875 0 -0.34375 -0.25l-1.859375 -3.03125zm5.3950195 2.46875l0 0.546875q0.6875 -0.734375 1.375 -0.734375q0.40625 0 0.71875 0.21875q0.3125 0.21875 0.515625 0.65625q0.359375 -0.4375 0.71875 -0.65625q0.375 -0.21875 0.734375 -0.21875q0.578125 0 0.921875 0.375q0.453125 0.484375 0.453125 1.046875l0 3.734375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.984375 0l0 -4.203125q0 -0.40625 -0.25 -0.671875q-0.25 -0.265625 -0.578125 -0.265625q-0.28125 0 -0.609375 0.21875q-0.328125 0.21875 -0.734375 0.859375l0 3.53125l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 -4.171875q0 -0.421875 -0.25 -0.6875q-0.25 -0.28125 -0.5625 -0.28125q-0.28125 0 -0.5625 0.171875q-0.390625 0.265625 -0.8125 0.90625l0 3.53125l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.453125 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm13.11377 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm3.0200195 -2.34375l0 0.546875q0.6875 -0.734375 1.375 -0.734375q0.40625 0 0.71875 0.21875q0.3125 0.21875 0.515625 0.65625q0.359375 -0.4375 0.71875 -0.65625q0.375 -0.21875 0.734375 -0.21875q0.578125 0 0.921875 0.375q0.453125 0.484375 0.453125 1.046875l0 3.734375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.984375 0l0 -4.203125q0 -0.40625 -0.25 -0.671875q-0.25 -0.265625 -0.578125 -0.265625q-0.28125 0 -0.609375 0.21875q-0.328125 0.21875 -0.734375 0.859375l0 3.53125l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 -4.171875q0 -0.421875 -0.25 -0.6875q-0.25 -0.28125 -0.5625 -0.28125q-0.28125 0 -0.5625 0.171875q-0.390625 0.265625 -0.8125 0.90625l0 3.53125l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.453125 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm12.45752 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm3.5668945 -0.546875l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm5.2231445 2.90625l-2.484375 -4.953125l-0.15625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.09375 0.03125 -0.15625q0.046875 -0.0625 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l2.171875 4.375l2.140625 -4.375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.0625 0.15625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.015625 -0.359375 0.015625l-3.375 6.875l0.84375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.078125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l1.671875 0l0.9375 -1.921875zm6.4106445 -7.96875l1.375 0l-1.859375 3.03125q-0.15625 0.25 -0.34375 0.25q-0.140625 0 -0.234375 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.0625 -0.234375l1.09375 -2.734375zm2.734375 0l1.375 0l-1.84375 3.03125q-0.171875 0.25 -0.34375 0.25q-0.140625 0 -0.25 -0.09375q-0.09375 -0.09375 -0.09375 -0.21875q0 -0.078125 0.078125 -0.234375l1.078125 -2.734375zm5.3637695 6.046875l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m266.58987 302.9984l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25z" fill-rule="nonzero"/><path fill="#000000" d="m250.84671 321.63904l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0zm7.9262695 -6.609375l0.234375 0q0.359375 0 0.609375 0.265625q0.25 0.25 0.25 0.59375q0 0.375 -0.25 0.625q-0.25 0.25 -0.609375 0.25l-0.234375 0q-0.359375 0 -0.609375 -0.25q-0.25 -0.265625 -0.25 -0.609375q0 -0.375 0.25 -0.625q0.265625 -0.25 0.609375 -0.25zm-0.921875 3.578125l1.671875 0l-1.796875 2.953125q-0.125 0.21875 -0.21875 0.265625q-0.078125 0.0625 -0.171875 0.0625q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.09375 -0.234375q0 -0.078125 0.03125 -0.1875l0.84375 -2.765625z" fill-rule="nonzero"/><path fill="#000000" d="m235.80667 333.3734q-0.328125 -0.25 -0.4375 -0.46875q-0.140625 -0.265625 -0.140625 -0.671875l0 -2.28125q0 -0.359375 -0.21875 -0.59375q-0.203125 -0.234375 -0.515625 -0.25q-0.203125 -0.03125 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.234375 -0.078125q0.515625 0 0.84375 0.328125q0.453125 0.421875 0.453125 1.0625l0 2.28125q0 0.359375 0.203125 0.59375q0.21875 0.234375 0.546875 0.265625q0.171875 0.015625 0.234375 0.09375q0.078125 0.0625 0.078125 0.1875q0 0.125 -0.078125 0.203125q-0.0625 0.0625 -0.234375 0.078125q-0.328125 0.03125 -0.546875 0.265625q-0.203125 0.234375 -0.203125 0.578125l0 2.296875q0 0.59375 -0.390625 0.984375q-0.375 0.40625 -0.90625 0.40625q-0.15625 0 -0.234375 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.0625 -0.0625 0.265625 -0.078125q0.3125 -0.03125 0.515625 -0.265625q0.21875 -0.234375 0.21875 -0.59375l0 -2.28125q0 -0.390625 0.125 -0.640625q0.125 -0.25 0.453125 -0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m235.3848 366.9984l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m584.1916 15.228347l359.81104 0l0 463.3386l-359.81104 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m584.1916 15.228347l359.81104 0l0 463.3386l-359.81104 0z" fill-rule="evenodd"/><path fill="#000000" d="m599.9416 19.432013l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.998169 0l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm14.683899 3.921875q-0.328125 -0.203125 -0.6875 -0.296875q-0.34375 -0.109375 -0.734375 -0.109375q-0.78125 0 -1.234375 0.25q-0.203125 0.109375 -0.203125 0.25q0 0.140625 0.265625 0.28125q0.21875 0.109375 0.9375 0.203125q1.328125 0.1875 1.84375 0.375q0.6875 0.234375 1.046875 0.71875q0.375 0.46875 0.375 1.0q0 0.703125 -0.625 1.1875q-0.90625 0.703125 -2.34375 0.703125q-0.578125 0 -1.078125 -0.109375q-0.484375 -0.09375 -0.890625 -0.296875q-0.109375 0.09375 -0.21875 0.140625q-0.109375 0.046875 -0.21875 0.046875q-0.3125 0 -0.5 -0.203125q-0.1875 -0.21875 -0.1875 -0.6875l0 -0.453125q0 -0.484375 0.1875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.234375 0 0.390625 0.140625q0.171875 0.125 0.265625 0.453125q0.296875 0.25 0.71875 0.390625q0.4375 0.125 1.0 0.125q0.921875 0 1.421875 -0.28125q0.25 -0.140625 0.25 -0.296875q0 -0.265625 -0.34375 -0.4375q-0.34375 -0.15625 -1.421875 -0.28125q-1.609375 -0.171875 -2.15625 -0.65625q-0.53125 -0.46875 -0.53125 -1.171875q0 -0.703125 0.59375 -1.1875q0.828125 -0.640625 2.15625 -0.640625q0.453125 0 0.875 0.09375q0.4375 0.078125 0.828125 0.25q0.125 -0.078125 0.234375 -0.125q0.109375 -0.046875 0.1875 -0.046875q0.28125 0 0.453125 0.203125q0.1875 0.203125 0.1875 0.6875l0 0.328125q0 0.4375 -0.09375 0.59375q-0.21875 0.296875 -0.578125 0.296875q-0.234375 0 -0.421875 -0.140625q-0.171875 -0.15625 -0.234375 -0.40625zm4.904419 3.609375l0 2.1875l0.796875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.296875 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.359375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.171875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.21875 -0.1875 0.6875 -0.1875l0.234375 0l0 -6.03125l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0l0 0.453125q0.46875 -0.3125 0.96875 -0.46875q0.5 -0.15625 1.03125 -0.15625q1.359375 0 2.328125 0.921875q0.96875 0.921875 0.96875 2.125q0 1.3125 -1.140625 2.171875q-0.953125 0.71875 -2.140625 0.71875q-0.515625 0 -1.015625 -0.140625q-0.5 -0.15625 -1.0 -0.453125zm3.953125 -2.296875q0 -0.28125 -0.21875 -0.703125q-0.21875 -0.4375 -0.6875 -0.71875q-0.453125 -0.296875 -1.0625 -0.296875q-1.0 0 -1.59375 0.75q-0.390625 0.515625 -0.390625 0.984375q0 0.53125 0.5625 1.046875q0.578125 0.5 1.421875 0.5q0.84375 0 1.40625 -0.5q0.5625 -0.5 0.5625 -1.0625zm9.045044 0.75l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm8.201355 -2.0625q0.234375 -0.21875 0.5 -0.21875q0.28125 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 0.859375q0 0.484375 -0.1875 0.6875q-0.1875 0.203125 -0.484375 0.203125q-0.28125 0 -0.46875 -0.15625q-0.125 -0.125 -0.203125 -0.484375q-0.078125 -0.359375 -0.390625 -0.53125q-0.515625 -0.3125 -1.34375 -0.3125q-0.9375 0 -1.515625 0.5625q-0.5625 0.546875 -0.5625 1.390625q0 0.78125 0.546875 1.234375q0.546875 0.453125 1.8125 0.453125q0.828125 0 1.359375 -0.171875q0.3125 -0.109375 0.59375 -0.359375q0.28125 -0.25 0.5 -0.25q0.28125 0 0.46875 0.203125q0.203125 0.203125 0.203125 0.484375q0 0.4375 -0.59375 0.828125q-0.90625 0.59375 -2.640625 0.59375q-1.546875 0 -2.421875 -0.640625q-1.171875 -0.859375 -1.171875 -2.375q0 -1.421875 0.953125 -2.34375q0.953125 -0.9375 2.484375 -0.9375q0.546875 0 1.015625 0.109375q0.484375 0.09375 0.890625 0.296875zm8.670044 -2.59375l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm6.685669 3.921875q-0.328125 -0.203125 -0.6875 -0.296875q-0.34375 -0.109375 -0.734375 -0.109375q-0.78125 0 -1.234375 0.25q-0.203125 0.109375 -0.203125 0.25q0 0.140625 0.265625 0.28125q0.21875 0.109375 0.9375 0.203125q1.328125 0.1875 1.84375 0.375q0.6875 0.234375 1.046875 0.71875q0.375 0.46875 0.375 1.0q0 0.703125 -0.625 1.1875q-0.90625 0.703125 -2.34375 0.703125q-0.578125 0 -1.078125 -0.109375q-0.484375 -0.09375 -0.890625 -0.296875q-0.109375 0.09375 -0.21875 0.140625q-0.109375 0.046875 -0.21875 0.046875q-0.3125 0 -0.5 -0.203125q-0.1875 -0.21875 -0.1875 -0.6875l0 -0.453125q0 -0.484375 0.1875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.234375 0 0.390625 0.140625q0.171875 0.125 0.265625 0.453125q0.296875 0.25 0.71875 0.390625q0.4375 0.125 1.0 0.125q0.921875 0 1.421875 -0.28125q0.25 -0.140625 0.25 -0.296875q0 -0.265625 -0.34375 -0.4375q-0.34375 -0.15625 -1.421875 -0.28125q-1.609375 -0.171875 -2.15625 -0.65625q-0.53125 -0.46875 -0.53125 -1.171875q0 -0.703125 0.59375 -1.1875q0.828125 -0.640625 2.15625 -0.640625q0.453125 0 0.875 0.09375q0.4375 0.078125 0.828125 0.25q0.125 -0.078125 0.234375 -0.125q0.109375 -0.046875 0.1875 -0.046875q0.28125 0 0.453125 0.203125q0.1875 0.203125 0.1875 0.6875l0 0.328125q0 0.4375 -0.09375 0.59375q-0.21875 0.296875 -0.578125 0.296875q-0.234375 0 -0.421875 -0.140625q-0.171875 -0.15625 -0.234375 -0.40625zm6.232544 -0.234375l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.34375 -0.15625 0.53125 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.265625 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.46875 0zm11.388855 -3.859375l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875zm8.341919 -2.890625q0.234375 -0.21875 0.5 -0.21875q0.28125 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 0.859375q0 0.484375 -0.1875 0.6875q-0.1875 0.203125 -0.484375 0.203125q-0.28125 0 -0.46875 -0.15625q-0.125 -0.125 -0.203125 -0.484375q-0.078125 -0.359375 -0.390625 -0.53125q-0.515625 -0.3125 -1.34375 -0.3125q-0.9375 0 -1.515625 0.5625q-0.5625 0.546875 -0.5625 1.390625q0 0.78125 0.546875 1.234375q0.546875 0.453125 1.8125 0.453125q0.828125 0 1.359375 -0.171875q0.3125 -0.109375 0.59375 -0.359375q0.28125 -0.25 0.5 -0.25q0.28125 0 0.46875 0.203125q0.203125 0.203125 0.203125 0.484375q0 0.4375 -0.59375 0.828125q-0.90625 0.59375 -2.640625 0.59375q-1.546875 0 -2.421875 -0.640625q-1.171875 -0.859375 -1.171875 -2.375q0 -1.421875 0.953125 -2.34375q0.953125 -0.9375 2.484375 -0.9375q0.546875 0 1.015625 0.109375q0.484375 0.09375 0.890625 0.296875zm6.935669 5.0q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm6.654419 -3.90625l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.34375 -0.15625 0.53125 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.265625 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.46875 0zm11.388855 -3.859375l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875z" fill-rule="nonzero"/><path fill="#000000" d="m598.8947 56.10389l-3.53125 0l0 3.0625l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.859375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.609375 0l0 -6.359375l-0.328125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.578125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l0 2.765625l3.53125 0l0 -2.765625l-0.703125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.578125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.328125 0l0 6.359375l0.59375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.84375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -3.0625zm8.879395 0.96875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm6.9887695 3.15625l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm8.879395 -5.3125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.8168945 -2.34375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.348145 -1.390625l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm15.290039 -1.9375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm4.8012695 1.390625l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm5.7856445 0l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.316895 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm16.72754 -1.5l-5.8125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125zm0 1.90625l-5.8125 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l5.8125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.125 -0.09375 0.203125q-0.078125 0.0625 -0.265625 0.0625zm14.493164 -1.09375l-3.53125 0l0 3.0625l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.859375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.609375 0l0 -6.359375l-0.328125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.578125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l0 2.765625l3.53125 0l0 -2.765625l-0.703125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.578125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.328125 0l0 6.359375l0.59375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.84375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -3.0625zm8.879395 0.96875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm6.9887695 3.15625l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm8.879395 -5.3125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm8.61377 0.109375l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.8168945 -2.34375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.348145 -1.390625l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875z" fill-rule="nonzero"/><path fill="#000000" d="m610.4816 67.869514l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm9.05127 2.6875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm5.7856445 0l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.316895 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm5.9106445 1.375l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm6.1450195 -6.4375l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm7.3481445 0.140625l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm6.2387695 5.90625l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.966 84.275764l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m628.64667 105.46327l-0.609375 0l-2.046875 -4.65625l-0.109375 0l0 6.359375l0.984375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.953125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.4375 0l0 -6.359375l-0.328125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.3125 0l2.015625 4.59375l1.984375 -4.59375l1.3125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.34375 0l0 6.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.96875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -6.359375l-0.109375 0l-2.015625 4.65625zm8.98877 2.234375l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm8.48877 -2.296875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.9418945 -0.546875l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.410645 2.765625q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm2.0825195 -0.96875l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm8.86377 -4.125l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm9.910645 7.828125l-0.6875 0l-3.9375 -6.640625l0 6.109375l0.984375 0q0.203125 0 0.28125 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.953125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.4375 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.40625 0l3.9375 6.65625l0 -6.125l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.96875 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.453125 0l0 6.890625zm7.7700195 -6.890625l0 4.453125q0 1.125 -0.75 1.890625q-0.75 0.75 -1.8125 0.75q-0.703125 0 -1.265625 -0.296875q-0.546875 -0.3125 -0.9375 -0.9375q-0.390625 -0.640625 -0.390625 -1.40625l0 -4.453125l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.0625 0.265625 -0.0625l1.96875 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 4.453125q0 0.875 0.609375 1.5q0.609375 0.609375 1.421875 0.609375q0.53125 0 0.96875 -0.234375q0.4375 -0.25 0.765625 -0.75q0.328125 -0.5 0.328125 -1.125l0 -4.453125l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.96875 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.453125 0zm4.3012695 0l0 6.359375l3.53125 0l0 -2.125q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 2.65625l-5.890625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.265625 0l0 -6.359375l-1.265625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l3.078125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.265625 0zm7.8012695 0l0 6.359375l3.53125 0l0 -2.125q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 2.65625l-5.890625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.265625 0l0 -6.359375l-1.265625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l3.078125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.265625 0zm6.5043945 -0.9375l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm10.566895 4.125l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.4504 124.80702l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0zm6.9887695 -3.03125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.966 132.27577l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m630.20917 148.80702l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm6.1606445 -1.34375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.848145 0l0 0.484375l-4.0 4.484375l3.609375 0l0 -0.734375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l-0.015625 1.265625l-4.875 0l0 -0.46875l3.984375 -4.484375l-3.3125 0l0 0.71875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.265625l4.609375 0zm8.348145 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm5.7231445 -3.734375l0 6.359375l1.375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.3125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.390625 0l0 -6.359375l-2.1875 0l0 1.84375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -2.375l5.96875 0l0 2.375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -1.84375l-2.171875 0zm7.8012695 0l0 6.359375l1.375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.3125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.390625 0l0 -6.359375l-2.1875 0l0 1.84375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -2.375l5.96875 0l0 2.375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -1.84375l-2.171875 0zm7.5668945 6.890625l-2.484375 -4.953125l-0.15625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.09375 0.03125 -0.15625q0.046875 -0.0625 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l2.171875 4.375l2.140625 -4.375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.0625 0.15625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.015625 -0.359375 0.015625l-3.375 6.875l0.84375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.078125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l1.671875 0l0.9375 -1.921875zm5.7543945 -5.5l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.3950195 1.234375l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.4504 172.80702l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0zm6.9887695 -3.03125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.966 180.27577l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m627.00604 200.1039l0 3.0625l1.828125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-3.078125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.09375 -0.203125q0.0625 -0.0625 0.25 -0.0625l5.875 0l0 1.828125q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -1.296875l-4.09375 0l0 2.765625l1.921875 0l0 -0.59375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.734375q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -0.609375l-1.921875 0zm10.910645 3.59375l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm4.2543945 -5.5l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.160645 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.48877 2.765625q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.441895 -1.390625l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875z" fill-rule="nonzero"/><path fill="#000000" d="m641.6867 211.86952l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm9.05127 2.6875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.660645 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm3.5668945 -0.546875l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm5.2231445 2.90625l-2.484375 -4.953125l-0.15625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.09375 0.03125 -0.15625q0.046875 -0.0625 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l2.171875 4.375l2.140625 -4.375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.0625 0.15625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.015625 -0.359375 0.015625l-3.375 6.875l0.84375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.078125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l1.671875 0l0.9375 -1.921875zm5.5512695 -7.828125l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm6.2387695 5.90625l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m642.1711 232.38515l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.3637695 -2.34375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.285645 5.5l-0.734375 0l-2.6875 -6.890625l-0.421875 0q-0.140625 0 -0.203125 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.03125 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.046875 0.03125 -0.203125 0.03125l-1.015625 0l2.46875 6.359375l0.046875 0l2.4375 -6.359375l-1.015625 0q-0.140625 0 -0.21875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.03125 0.0625 -0.09375 0.09375q-0.0625 0.03125 -0.21875 0.03125l-0.40625 0l-2.640625 6.890625zm8.941895 0l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm6.5356445 -5.3125l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm9.441895 1.078125l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm13.42627 -3.0l0 -0.375q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.484375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.234375q-0.03125 -0.546875 -0.578125 -0.953125q-0.734375 -0.5625 -1.671875 -0.5625q-0.625 0 -1.171875 0.265625q-0.40625 0.203125 -0.640625 0.484375q-0.421875 0.5 -0.671875 1.09375q-0.171875 0.453125 -0.171875 1.015625l0 0.921875q0 1.203125 0.859375 2.078125q0.859375 0.875 2.0 0.875q0.6875 0 1.234375 -0.296875q0.546875 -0.296875 1.046875 -0.90625q0.109375 -0.125 0.25 -0.125q0.109375 0 0.171875 0.078125q0.078125 0.0625 0.078125 0.171875q0 0.140625 -0.28125 0.4375q-0.515625 0.59375 -1.171875 0.890625q-0.640625 0.28125 -1.3125 0.28125q-0.578125 0 -1.171875 -0.21875q-0.453125 -0.171875 -0.75 -0.390625q-0.296875 -0.234375 -0.734375 -0.78125q-0.4375 -0.546875 -0.59375 -1.0q-0.15625 -0.453125 -0.15625 -1.015625l0 -1.078125q0 -0.796875 0.421875 -1.609375q0.421875 -0.828125 1.140625 -1.265625q0.734375 -0.4375 1.59375 -0.4375q1.328125 0 2.28125 0.890625zm3.6918945 -1.265625l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm11.098145 7.96875l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm5.6293945 -2.84375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm7.2856445 2.453125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m636.3698 244.27577l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0zm9.285645 4.9375l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.6137695 -2.484375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.223145 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm7.9262695 -3.984375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm13.42627 -3.0l0 -0.375q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.484375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.234375q-0.03125 -0.546875 -0.578125 -0.953125q-0.734375 -0.5625 -1.671875 -0.5625q-0.625 0 -1.171875 0.265625q-0.40625 0.203125 -0.640625 0.484375q-0.421875 0.5 -0.671875 1.09375q-0.171875 0.453125 -0.171875 1.015625l0 0.921875q0 1.203125 0.859375 2.078125q0.859375 0.875 2.0 0.875q0.6875 0 1.234375 -0.296875q0.546875 -0.296875 1.046875 -0.90625q0.109375 -0.125 0.25 -0.125q0.109375 0 0.171875 0.078125q0.078125 0.0625 0.078125 0.171875q0 0.140625 -0.28125 0.4375q-0.515625 0.59375 -1.171875 0.890625q-0.640625 0.28125 -1.3125 0.28125q-0.578125 0 -1.171875 -0.21875q-0.453125 -0.171875 -0.75 -0.390625q-0.296875 -0.234375 -0.734375 -0.78125q-0.4375 -0.546875 -0.59375 -1.0q-0.15625 -0.453125 -0.15625 -1.015625l0 -1.078125q0 -0.796875 0.421875 -1.609375q0.421875 -0.828125 1.140625 -1.265625q0.734375 -0.4375 1.59375 -0.4375q1.328125 0 2.28125 0.890625zm3.6918945 -1.265625l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm11.098145 7.96875l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm5.6293945 -2.84375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm6.7075195 2.1875l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.2075195 -1.984375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.77002 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm8.066895 -4.8125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.5825195 -0.3125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m645.65546 265.21326l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.6137695 -2.484375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.223145 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm7.9262695 -3.984375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm13.42627 -3.0l0 -0.375q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.484375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.234375q-0.03125 -0.546875 -0.578125 -0.953125q-0.734375 -0.5625 -1.671875 -0.5625q-0.625 0 -1.171875 0.265625q-0.40625 0.203125 -0.640625 0.484375q-0.421875 0.5 -0.671875 1.09375q-0.171875 0.453125 -0.171875 1.015625l0 0.921875q0 1.203125 0.859375 2.078125q0.859375 0.875 2.0 0.875q0.6875 0 1.234375 -0.296875q0.546875 -0.296875 1.046875 -0.90625q0.109375 -0.125 0.25 -0.125q0.109375 0 0.171875 0.078125q0.078125 0.0625 0.078125 0.171875q0 0.140625 -0.28125 0.4375q-0.515625 0.59375 -1.171875 0.890625q-0.640625 0.28125 -1.3125 0.28125q-0.578125 0 -1.171875 -0.21875q-0.453125 -0.171875 -0.75 -0.390625q-0.296875 -0.234375 -0.734375 -0.78125q-0.4375 -0.546875 -0.59375 -1.0q-0.15625 -0.453125 -0.15625 -1.015625l0 -1.078125q0 -0.796875 0.421875 -1.609375q0.421875 -0.828125 1.140625 -1.265625q0.734375 -0.4375 1.59375 -0.4375q1.328125 0 2.28125 0.890625zm8.42627 3.96875q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.285645 0.359375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.23877 -1.203125l0 -0.375q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.484375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.234375q-0.03125 -0.546875 -0.578125 -0.953125q-0.734375 -0.5625 -1.671875 -0.5625q-0.625 0 -1.171875 0.265625q-0.40625 0.203125 -0.640625 0.484375q-0.421875 0.5 -0.671875 1.09375q-0.171875 0.453125 -0.171875 1.015625l0 0.921875q0 1.203125 0.859375 2.078125q0.859375 0.875 2.0 0.875q0.6875 0 1.234375 -0.296875q0.546875 -0.296875 1.046875 -0.90625q0.109375 -0.125 0.25 -0.125q0.109375 0 0.171875 0.078125q0.078125 0.0625 0.078125 0.171875q0 0.140625 -0.28125 0.4375q-0.515625 0.59375 -1.171875 0.890625q-0.640625 0.28125 -1.3125 0.28125q-0.578125 0 -1.171875 -0.21875q-0.453125 -0.171875 -0.75 -0.390625q-0.296875 -0.234375 -0.734375 -0.78125q-0.4375 -0.546875 -0.59375 -1.0q-0.15625 -0.453125 -0.15625 -1.015625l0 -1.078125q0 -0.796875 0.421875 -1.609375q0.421875 -0.828125 1.140625 -1.265625q0.734375 -0.4375 1.59375 -0.4375q1.328125 0 2.28125 0.890625zm3.6918945 -1.265625l0 3.28125q0.515625 -0.546875 0.984375 -0.765625q0.46875 -0.234375 1.046875 -0.234375q0.609375 0 1.046875 0.21875q0.4375 0.21875 0.71875 0.671875q0.296875 0.453125 0.296875 0.953125l0 3.3125l0.609375 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.296875 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.296875 -0.078125l0.59375 0l0 -3.265625q0 -0.578125 -0.421875 -0.96875q-0.40625 -0.390625 -1.15625 -0.390625q-0.578125 0 -1.0 0.28125q-0.296875 0.203125 -0.984375 0.96875l0 3.375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.25 0zm11.098145 7.96875l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm5.6293945 -2.84375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm6.7075195 2.1875l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.2075195 -1.984375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.77002 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm8.066895 -4.8125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.5825195 -0.3125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm4.2856445 4.8125l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m631.56854 279.9945l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m627.00604 296.10388l0 3.0625l1.828125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-3.078125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.09375 -0.203125q0.0625 -0.0625 0.25 -0.0625l5.875 0l0 1.828125q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -1.296875l-4.09375 0l0 2.765625l1.921875 0l0 -0.59375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.734375q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.09375 -0.0625 -0.28125l0 -0.609375l-1.921875 0zm10.910645 3.59375l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm4.2543945 -5.5l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.754395 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.160645 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.48877 2.765625q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.441895 -1.390625l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875z" fill-rule="nonzero"/><path fill="#000000" d="m641.6867 307.8695l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm4.8637695 2.328125l0 0.546875q0.6875 -0.734375 1.375 -0.734375q0.40625 0 0.71875 0.21875q0.3125 0.21875 0.515625 0.65625q0.359375 -0.4375 0.71875 -0.65625q0.375 -0.21875 0.734375 -0.21875q0.578125 0 0.921875 0.375q0.453125 0.484375 0.453125 1.046875l0 3.734375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.984375 0l0 -4.203125q0 -0.40625 -0.25 -0.671875q-0.25 -0.265625 -0.578125 -0.265625q-0.28125 0 -0.609375 0.21875q-0.328125 0.21875 -0.734375 0.859375l0 3.53125l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 -4.171875q0 -0.421875 -0.25 -0.6875q-0.25 -0.28125 -0.5625 -0.28125q-0.28125 0 -0.5625 0.171875q-0.390625 0.265625 -0.8125 0.90625l0 3.53125l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.453125 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm13.11377 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm3.0200195 -2.34375l0 0.546875q0.6875 -0.734375 1.375 -0.734375q0.40625 0 0.71875 0.21875q0.3125 0.21875 0.515625 0.65625q0.359375 -0.4375 0.71875 -0.65625q0.375 -0.21875 0.734375 -0.21875q0.578125 0 0.921875 0.375q0.453125 0.484375 0.453125 1.046875l0 3.734375l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.984375 0l0 -4.203125q0 -0.40625 -0.25 -0.671875q-0.25 -0.265625 -0.578125 -0.265625q-0.28125 0 -0.609375 0.21875q-0.328125 0.21875 -0.734375 0.859375l0 3.53125l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0l0 -4.171875q0 -0.421875 -0.25 -0.6875q-0.25 -0.28125 -0.5625 -0.28125q-0.28125 0 -0.5625 0.171875q-0.390625 0.265625 -0.8125 0.90625l0 3.53125l0.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.453125 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm12.45752 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm3.5668945 -0.546875l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm5.2231445 2.90625l-2.484375 -4.953125l-0.15625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.09375 0.03125 -0.15625q0.046875 -0.0625 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l2.171875 4.375l2.140625 -4.375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.0625 0.15625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.015625 -0.359375 0.015625l-3.375 6.875l0.84375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.078125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l1.671875 0l0.9375 -1.921875zm5.5512695 -7.828125l1.6875 0l-0.4375 3.078125q-0.046875 0.25 -0.15625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm2.71875 0l1.6875 0l-0.4375 3.078125q-0.03125 0.25 -0.140625 0.34375q-0.09375 0.09375 -0.25 0.09375q-0.171875 0 -0.28125 -0.09375q-0.09375 -0.09375 -0.125 -0.34375l-0.453125 -3.078125zm6.2387695 5.90625l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m642.1711 328.38513l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm4.3637695 -2.34375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.285645 5.5l-0.734375 0l-2.6875 -6.890625l-0.421875 0q-0.140625 0 -0.203125 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.03125 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.046875 0.03125 -0.203125 0.03125l-1.015625 0l2.46875 6.359375l0.046875 0l2.4375 -6.359375l-1.015625 0q-0.140625 0 -0.21875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.03125 0.0625 -0.09375 0.09375q-0.0625 0.03125 -0.21875 0.03125l-0.40625 0l-2.640625 6.890625zm8.941895 0l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm6.5356445 -5.3125l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm9.441895 1.078125l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm11.473145 3.703125l-0.734375 0l-2.6875 -6.890625l-0.421875 0q-0.140625 0 -0.203125 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.03125 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.046875 0.03125 -0.203125 0.03125l-1.015625 0l2.46875 6.359375l0.046875 0l2.4375 -6.359375l-1.015625 0q-0.140625 0 -0.21875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.03125 0.0625 -0.09375 0.09375q-0.0625 0.03125 -0.21875 0.03125l-0.40625 0l-2.640625 6.890625zm10.379395 -2.734375q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm5.6293945 -5.5l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.129395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.5825195 -0.3125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m644.1711 340.27576l0 8.53125l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.125 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.828125 0l0 -9.609375l1.828125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.296875 0zm9.285645 4.9375l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.6137695 -2.484375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.223145 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm7.9262695 -3.984375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm11.473145 3.703125l-0.734375 0l-2.6875 -6.890625l-0.421875 0q-0.140625 0 -0.203125 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.03125 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.046875 0.03125 -0.203125 0.03125l-1.015625 0l2.46875 6.359375l0.046875 0l2.4375 -6.359375l-1.015625 0q-0.140625 0 -0.21875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.03125 0.0625 -0.09375 0.09375q-0.0625 0.03125 -0.21875 0.03125l-0.40625 0l-2.640625 6.890625zm10.379395 -2.734375q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm5.6293945 -5.5l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.129395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.0043945 -0.578125l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.2075195 -1.984375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.77002 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm8.066895 -4.8125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.5825195 -0.3125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m653.4567 361.21326l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.6137695 -2.484375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.223145 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm7.9262695 -3.984375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm13.42627 -3.0l0 -0.375q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.484375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.234375q-0.03125 -0.546875 -0.578125 -0.953125q-0.734375 -0.5625 -1.671875 -0.5625q-0.625 0 -1.171875 0.265625q-0.40625 0.203125 -0.640625 0.484375q-0.421875 0.5 -0.671875 1.09375q-0.171875 0.453125 -0.171875 1.015625l0 0.921875q0 1.203125 0.859375 2.078125q0.859375 0.875 2.0 0.875q0.6875 0 1.234375 -0.296875q0.546875 -0.296875 1.046875 -0.90625q0.109375 -0.125 0.25 -0.125q0.109375 0 0.171875 0.078125q0.078125 0.0625 0.078125 0.171875q0 0.140625 -0.28125 0.4375q-0.515625 0.59375 -1.171875 0.890625q-0.640625 0.28125 -1.3125 0.28125q-0.578125 0 -1.171875 -0.21875q-0.453125 -0.171875 -0.75 -0.390625q-0.296875 -0.234375 -0.734375 -0.78125q-0.4375 -0.546875 -0.59375 -1.0q-0.15625 -0.453125 -0.15625 -1.015625l0 -1.078125q0 -0.796875 0.421875 -1.609375q0.421875 -0.828125 1.140625 -1.265625q0.734375 -0.4375 1.59375 -0.4375q1.328125 0 2.28125 0.890625zm8.42627 3.96875q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm3.6293945 -2.765625l0 0.8125q0.5625 -0.5625 1.015625 -0.78125q0.453125 -0.21875 1.015625 -0.21875q0.609375 0 1.109375 0.265625q0.359375 0.1875 0.640625 0.625q0.296875 0.4375 0.296875 0.90625l0 3.359375l0.453125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.4375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.453125 0l0 -3.265625q0 -0.578125 -0.421875 -0.953125q-0.40625 -0.390625 -1.09375 -0.390625q-0.53125 0 -0.921875 0.21875q-0.390625 0.203125 -1.109375 1.046875l0 3.34375l0.609375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.734375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.59375 0l0 -4.421875l-0.453125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.984375 0zm11.285645 0.359375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm9.285645 5.5l-0.734375 0l-2.6875 -6.890625l-0.421875 0q-0.140625 0 -0.203125 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.03125 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.046875 0.0625 -0.109375 0.09375q-0.046875 0.03125 -0.203125 0.03125l-1.015625 0l2.46875 6.359375l0.046875 0l2.4375 -6.359375l-1.015625 0q-0.140625 0 -0.21875 -0.03125q-0.0625 -0.03125 -0.109375 -0.09375q-0.03125 -0.0625 -0.03125 -0.140625q0 -0.078125 0.046875 -0.140625q0.046875 -0.078125 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.984375 0q0.15625 0 0.21875 0.03125q0.0625 0.015625 0.09375 0.09375q0.046875 0.0625 0.046875 0.140625q0 0.078125 -0.046875 0.140625q-0.03125 0.0625 -0.09375 0.09375q-0.0625 0.03125 -0.21875 0.03125l-0.40625 0l-2.640625 6.890625zm10.379395 -2.734375q0 1.21875 -0.875 2.078125q-0.859375 0.859375 -2.09375 0.859375q-1.25 0 -2.125 -0.859375q-0.859375 -0.875 -0.859375 -2.078125q0 -1.21875 0.859375 -2.078125q0.875 -0.875 2.125 -0.875q1.234375 0 2.09375 0.859375q0.875 0.859375 0.875 2.09375zm-0.53125 0q0 -1.0 -0.71875 -1.703125q-0.71875 -0.703125 -1.734375 -0.703125q-1.015625 0 -1.734375 0.703125q-0.703125 0.703125 -0.703125 1.703125q0 0.984375 0.703125 1.703125q0.71875 0.703125 1.734375 0.703125q1.015625 0 1.734375 -0.703125q0.71875 -0.703125 0.71875 -1.703125zm5.6293945 -5.5l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm10.129395 -2.46875l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.0043945 -0.578125l0 2.78125l0.984375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.234375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.703125 0l0 -6.359375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l3.21875 0q1.0 0 1.671875 0.640625q0.6875 0.625 0.6875 1.40625q0 0.578125 -0.421875 1.078125q-0.421875 0.5 -1.40625 0.84375q0.5625 0.390625 0.96875 0.890625q0.40625 0.5 1.296875 2.03125l0.40625 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0q-0.984375 -1.765625 -1.53125 -2.375q-0.546875 -0.609375 -1.25 -0.9375l-1.6875 0zm0 -0.53125l1.453125 0q0.6875 0 1.25 -0.25q0.578125 -0.265625 0.828125 -0.609375q0.265625 -0.34375 0.265625 -0.6875q0 -0.53125 -0.546875 -1.015625q-0.53125 -0.484375 -1.296875 -0.484375l-1.953125 0l0 3.046875zm12.410645 1.21875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.2075195 -1.984375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 0.921875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.09375 0 -0.171875 -0.0625q-0.078125 -0.078125 -0.09375 -0.25q-0.03125 -0.375 -0.390625 -0.625q-0.515625 -0.34375 -1.375 -0.34375q-0.890625 0 -1.390625 0.359375q-0.375 0.265625 -0.375 0.609375q0 0.375 0.453125 0.625q0.296875 0.1875 1.140625 0.28125q1.125 0.109375 1.546875 0.265625q0.625 0.21875 0.921875 0.625q0.296875 0.390625 0.296875 0.84375q0 0.6875 -0.65625 1.21875q-0.640625 0.515625 -1.90625 0.515625q-1.265625 0 -2.078125 -0.640625q0 0.21875 -0.03125 0.28125q-0.015625 0.0625 -0.09375 0.109375q-0.0625 0.046875 -0.140625 0.046875q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.09375q0 -0.1875 0.0625 -0.265625q0.078125 -0.09375 0.203125 -0.09375q0.109375 0 0.1875 0.078125q0.09375 0.078125 0.09375 0.21875q0 0.296875 0.140625 0.484375q0.21875 0.3125 0.703125 0.515625q0.484375 0.1875 1.1875 0.1875q1.046875 0 1.546875 -0.375q0.515625 -0.390625 0.515625 -0.828125q0 -0.5 -0.515625 -0.796875q-0.515625 -0.296875 -1.515625 -0.390625q-0.984375 -0.109375 -1.421875 -0.265625q-0.4375 -0.171875 -0.671875 -0.5q-0.234375 -0.328125 -0.234375 -0.71875q0 -0.671875 0.671875 -1.078125q0.671875 -0.40625 1.59375 -0.40625q1.109375 0 1.796875 0.546875zm4.9575195 -0.359375l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm8.70752 -2.734375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm4.4887695 -0.546875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm11.77002 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm8.066895 -4.8125l0 7.4375l0.703125 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-1.25 0l0 -1.0625q-0.921875 1.265625 -2.34375 1.265625q-0.734375 0 -1.40625 -0.375q-0.65625 -0.390625 -1.046875 -1.09375q-0.375 -0.71875 -0.375 -1.46875q0 -0.765625 0.375 -1.46875q0.390625 -0.703125 1.046875 -1.09375q0.671875 -0.390625 1.40625 -0.390625q1.390625 0 2.34375 1.265625l0 -3.0l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.25 0zm-0.546875 5.234375q0 -1.015625 -0.6875 -1.703125q-0.671875 -0.703125 -1.625 -0.703125q-0.953125 0 -1.640625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.6875 0.703125 1.640625 0.703125q0.953125 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm4.5825195 -0.3125l0 2.515625l1.828125 0q0.1875 0 0.265625 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.09375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l0.71875 0l0 -6.359375l-0.71875 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l3.109375 0q1.09375 0 1.78125 0.65625q0.703125 0.640625 0.703125 1.5q0 0.515625 -0.234375 0.9375q-0.21875 0.40625 -0.53125 0.640625q-0.484375 0.359375 -0.984375 0.515625q-0.375 0.125 -0.921875 0.125l-1.65625 0zm0 -0.53125l1.6875 0q0.59375 0 1.109375 -0.265625q0.515625 -0.265625 0.75 -0.640625q0.25 -0.390625 0.25 -0.78125q0 -0.625 -0.546875 -1.125q-0.53125 -0.5 -1.359375 -0.5l-1.890625 0l0 3.3125zm7.8637695 -1.921875l2.890625 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.890625 0l0 3.546875q0 0.453125 0.359375 0.765625q0.375 0.3125 1.09375 0.3125q0.546875 0 1.171875 -0.15625q0.625 -0.15625 0.96875 -0.359375q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.09375 -0.078125 0.1875q-0.203125 0.203125 -1.0 0.453125q-0.78125 0.234375 -1.5 0.234375q-0.921875 0 -1.484375 -0.4375q-0.546875 -0.4375 -0.546875 -1.171875l0 -3.546875l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.984375 0l0 -1.578125q0 -0.1875 0.078125 -0.265625q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.203125 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.578125zm8.254395 0l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm11.70752 1.796875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m653.4567 377.21326l-3.5 0l-0.71875 1.953125l1.015625 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l0.40625 0l2.359375 -6.359375l-1.578125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.6875 0l2.59375 6.890625l0.421875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.984375 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.015625 0l-0.734375 -1.953125zm-0.203125 -0.53125l-1.46875 -3.875l-0.203125 0l-1.421875 3.875l3.09375 0zm5.6137695 -2.484375l0 1.34375q1.03125 -0.9375 1.546875 -1.203125q0.53125 -0.265625 0.96875 -0.265625q0.46875 0 0.875 0.328125q0.421875 0.3125 0.421875 0.46875q0 0.125 -0.09375 0.203125q-0.078125 0.078125 -0.1875 0.078125q-0.0625 0 -0.109375 -0.015625q-0.046875 -0.03125 -0.171875 -0.140625q-0.21875 -0.234375 -0.390625 -0.3125q-0.15625 -0.078125 -0.3125 -0.078125q-0.359375 0 -0.859375 0.28125q-0.484375 0.28125 -1.6875 1.359375l0 2.921875l2.359375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.1875 0q-0.1875 0 -0.28125 -0.0625q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.171875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0l0 -4.4375l-0.984375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.515625 0zm10.223145 0.984375l0 -0.984375l1.25 0q0.203125 0 0.28125 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.703125 0l0 5.328125q0 0.53125 -0.234375 0.953125q-0.15625 0.265625 -0.515625 0.5625q-0.34375 0.3125 -0.640625 0.4375q-0.296875 0.125 -0.78125 0.125l-1.515625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.265625 -0.078125l1.53125 0.015625q0.46875 0 0.84375 -0.25q0.390625 -0.234375 0.625 -0.703125q0.140625 -0.265625 0.140625 -0.671875l0 -1.609375q-0.84375 1.171875 -2.203125 1.171875q-1.09375 0 -1.890625 -0.8125q-0.796875 -0.8125 -0.796875 -1.953125q0 -1.15625 0.796875 -1.96875q0.796875 -0.8125 1.890625 -0.8125q1.359375 0 2.203125 1.171875zm0 1.609375q0 -0.953125 -0.640625 -1.59375q-0.640625 -0.640625 -1.53125 -0.640625q-0.90625 0 -1.546875 0.65625q-0.640625 0.640625 -0.640625 1.578125q0 0.9375 0.640625 1.59375q0.640625 0.640625 1.546875 0.640625q0.890625 0 1.53125 -0.640625q0.640625 -0.65625 0.640625 -1.59375zm7.9262695 -3.984375l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm3.8793945 1.390625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm7.6762695 -1.796875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm2.3012695 1.25l5.765625 -3.203125q0.140625 -0.078125 0.203125 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.15625 -0.21875 0.28125l-4.9375 2.734375l4.9375 2.75q0.21875 0.109375 0.21875 0.28125q0 0.09375 -0.078125 0.1875q-0.078125 0.078125 -0.1875 0.078125q-0.078125 0 -0.21875 -0.078125l-5.75 -3.21875zm12.98877 -3.1875l0 -0.1875q0 -0.1875 0.0625 -0.265625q0.078125 -0.078125 0.203125 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.265625l0 1.359375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.125 0 -0.203125 -0.078125q-0.0625 -0.078125 -0.0625 -0.25q-0.03125 -0.53125 -0.5625 -0.953125q-0.515625 -0.421875 -1.34375 -0.421875q-0.890625 0 -1.40625 0.453125q-0.515625 0.453125 -0.515625 1.0625q0 0.328125 0.140625 0.59375q0.140625 0.25 0.375 0.421875q0.25 0.15625 0.546875 0.25q0.3125 0.09375 0.96875 0.203125q1.078125 0.1875 1.5 0.359375q0.546875 0.25 0.828125 0.6875q0.28125 0.4375 0.28125 1.03125q0 0.921875 -0.734375 1.5625q-0.71875 0.640625 -1.96875 0.640625q-1.375 0 -2.21875 -0.859375l0 0.296875q0 0.1875 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.46875q0 -0.203125 0.078125 -0.28125q0.078125 -0.078125 0.1875 -0.078125q0.125 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.25q0.03125 0.578125 0.625 1.0625q0.59375 0.46875 1.59375 0.46875q1.015625 0 1.59375 -0.5q0.578125 -0.5 0.578125 -1.1875q0 -0.421875 -0.21875 -0.734375q-0.21875 -0.328125 -0.65625 -0.53125q-0.3125 -0.125 -1.28125 -0.296875q-1.328125 -0.21875 -1.90625 -0.671875q-0.578125 -0.453125 -0.578125 -1.296875q0 -0.84375 0.65625 -1.4375q0.671875 -0.609375 1.78125 -0.609375q1.109375 0 1.921875 0.703125zm6.1606445 -1.34375l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.848145 0l0 0.484375l-4.0 4.484375l3.609375 0l0 -0.734375q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l-0.015625 1.265625l-4.875 0l0 -0.46875l3.984375 -4.484375l-3.3125 0l0 0.71875q0 0.1875 -0.078125 0.28125q-0.078125 0.078125 -0.203125 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.28125l0 -1.265625l4.609375 0zm8.348145 2.875l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm5.7231445 -3.734375l0 6.359375l1.375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.3125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.390625 0l0 -6.359375l-2.1875 0l0 1.84375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -2.375l5.96875 0l0 2.375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -1.84375l-2.171875 0zm7.8012695 0l0 6.359375l1.375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.3125 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.390625 0l0 -6.359375l-2.1875 0l0 1.84375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -2.375l5.96875 0l0 2.375q0 0.203125 -0.078125 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.078125 -0.078125 -0.28125l0 -1.84375l-2.171875 0zm7.5668945 6.890625l-2.484375 -4.953125l-0.15625 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.09375 0.03125 -0.15625q0.046875 -0.0625 0.109375 -0.09375q0.0625 -0.03125 0.203125 -0.03125l1.46875 0q0.1875 0 0.265625 0.078125q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-0.71875 0l2.171875 4.375l2.140625 -4.375l-0.703125 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l1.453125 0q0.1875 0 0.265625 0.078125q0.09375 0.078125 0.09375 0.203125q0 0.078125 -0.0625 0.15625q-0.046875 0.0625 -0.109375 0.09375q-0.0625 0.015625 -0.359375 0.015625l-3.375 6.875l0.84375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125l-3.078125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l1.671875 0l0.9375 -1.921875zm5.7543945 -5.5l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm8.066895 0.28125l-5.421875 0q0.125 1.03125 0.859375 1.671875q0.734375 0.625 1.796875 0.625q0.59375 0 1.25 -0.1875q0.65625 -0.203125 1.0625 -0.53125q0.125 -0.09375 0.21875 -0.09375q0.09375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.109375 -0.09375 0.21875q-0.3125 0.3125 -1.09375 0.59375q-0.765625 0.265625 -1.59375 0.265625q-1.375 0 -2.296875 -0.890625q-0.90625 -0.90625 -0.90625 -2.171875q0 -1.171875 0.859375 -2.0q0.859375 -0.828125 2.125 -0.828125q1.3125 0 2.15625 0.859375q0.84375 0.84375 0.828125 2.203125zm-0.53125 -0.53125q-0.15625 -0.890625 -0.84375 -1.4375q-0.671875 -0.546875 -1.609375 -0.546875q-0.921875 0 -1.59375 0.546875q-0.671875 0.53125 -0.84375 1.4375l4.890625 0zm8.723145 -0.546875l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm4.2856445 4.8125l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m631.56854 391.9945l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm3.4731445 1.78125l1.734375 0l-1.890625 3.515625q-0.171875 0.3125 -0.40625 0.3125q-0.15625 0 -0.28125 -0.109375q-0.109375 -0.109375 -0.109375 -0.265625q0 -0.078125 0.03125 -0.140625l0.921875 -3.3125z" fill-rule="nonzero"/><path fill="#000000" d="m612.4504 412.807l0 -8.53125l-1.296875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.078125 0.28125 -0.078125l1.828125 0l0 9.609375l-1.828125 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.296875 0z" fill-rule="nonzero"/><path fill="#000000" d="m600.36346 423.9945l-5.75 3.21875q-0.140625 0.078125 -0.21875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.078125 -0.09375 -0.078125 -0.1875q0 -0.171875 0.21875 -0.28125l4.9375 -2.75l-4.9375 -2.734375q-0.203125 -0.125 -0.203125 -0.28125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.1875 -0.078125q0.0625 0 0.203125 0.078125l5.75 3.203125zm4.4106445 -1.796875l0.234375 0q0.359375 0 0.609375 0.265625q0.25 0.25 0.25 0.59375q0 0.375 -0.25 0.625q-0.25 0.25 -0.609375 0.25l-0.234375 0q-0.359375 0 -0.609375 -0.25q-0.25 -0.265625 -0.25 -0.609375q0 -0.375 0.25 -0.625q0.265625 -0.25 0.609375 -0.25zm-0.921875 3.578125l1.671875 0l-1.796875 2.953125q-0.125 0.21875 -0.21875 0.265625q-0.078125 0.0625 -0.171875 0.0625q-0.15625 0 -0.265625 -0.09375q-0.09375 -0.09375 -0.09375 -0.234375q0 -0.078125 0.03125 -0.1875l0.84375 -2.765625z" fill-rule="nonzero"/><path fill="#000000" d="m596.98846 458.16638l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m584.1916 501.57217l359.81104 0l0 106.99216l-359.81104 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m584.1916 501.57217l359.81104 0l0 106.99216l-359.81104 0z" fill-rule="evenodd"/><path fill="#000000" d="m599.9416 535.6026l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.998169 0l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm14.683899 3.921875q-0.328125 -0.203125 -0.6875 -0.296875q-0.34375 -0.109375 -0.734375 -0.109375q-0.78125 0 -1.234375 0.25q-0.203125 0.109375 -0.203125 0.25q0 0.140625 0.265625 0.28125q0.21875 0.109375 0.9375 0.203125q1.328125 0.1875 1.84375 0.375q0.6875 0.234375 1.046875 0.71875q0.375 0.46875 0.375 1.0q0 0.703125 -0.625 1.1875q-0.90625 0.703125 -2.34375 0.703125q-0.578125 0 -1.078125 -0.109375q-0.484375 -0.09375 -0.890625 -0.296875q-0.109375 0.09375 -0.21875 0.140625q-0.109375 0.046875 -0.21875 0.046875q-0.3125 0 -0.5 -0.203125q-0.1875 -0.21875 -0.1875 -0.6875l0 -0.453125q0 -0.484375 0.1875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.234375 0 0.390625 0.140625q0.171875 0.125 0.265625 0.453125q0.296875 0.25 0.71875 0.390625q0.4375 0.125 1.0 0.125q0.921875 0 1.421875 -0.28125q0.25 -0.140625 0.25 -0.296875q0 -0.265625 -0.34375 -0.4375q-0.34375 -0.15625 -1.421875 -0.28125q-1.609375 -0.171875 -2.15625 -0.65625q-0.53125 -0.46875 -0.53125 -1.171875q0 -0.703125 0.59375 -1.1875q0.828125 -0.640625 2.15625 -0.640625q0.453125 0 0.875 0.09375q0.4375 0.078125 0.828125 0.25q0.125 -0.078125 0.234375 -0.125q0.109375 -0.046875 0.1875 -0.046875q0.28125 0 0.453125 0.203125q0.1875 0.203125 0.1875 0.6875l0 0.328125q0 0.4375 -0.09375 0.59375q-0.21875 0.296875 -0.578125 0.296875q-0.234375 0 -0.421875 -0.140625q-0.171875 -0.15625 -0.234375 -0.40625zm4.904419 3.609375l0 2.1875l0.796875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.296875 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.359375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.171875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.21875 -0.1875 0.6875 -0.1875l0.234375 0l0 -6.03125l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0l0 0.453125q0.46875 -0.3125 0.96875 -0.46875q0.5 -0.15625 1.03125 -0.15625q1.359375 0 2.328125 0.921875q0.96875 0.921875 0.96875 2.125q0 1.3125 -1.140625 2.171875q-0.953125 0.71875 -2.140625 0.71875q-0.515625 0 -1.015625 -0.140625q-0.5 -0.15625 -1.0 -0.453125zm3.953125 -2.296875q0 -0.28125 -0.21875 -0.703125q-0.21875 -0.4375 -0.6875 -0.71875q-0.453125 -0.296875 -1.0625 -0.296875q-1.0 0 -1.59375 0.75q-0.390625 0.515625 -0.390625 0.984375q0 0.53125 0.5625 1.046875q0.578125 0.5 1.421875 0.5q0.84375 0 1.40625 -0.5q0.5625 -0.5 0.5625 -1.0625zm9.045044 0.75l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm8.201355 -2.0625q0.234375 -0.21875 0.5 -0.21875q0.28125 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 0.859375q0 0.484375 -0.1875 0.6875q-0.1875 0.203125 -0.484375 0.203125q-0.28125 0 -0.46875 -0.15625q-0.125 -0.125 -0.203125 -0.484375q-0.078125 -0.359375 -0.390625 -0.53125q-0.515625 -0.3125 -1.34375 -0.3125q-0.9375 0 -1.515625 0.5625q-0.5625 0.546875 -0.5625 1.390625q0 0.78125 0.546875 1.234375q0.546875 0.453125 1.8125 0.453125q0.828125 0 1.359375 -0.171875q0.3125 -0.109375 0.59375 -0.359375q0.28125 -0.25 0.5 -0.25q0.28125 0 0.46875 0.203125q0.203125 0.203125 0.203125 0.484375q0 0.4375 -0.59375 0.828125q-0.90625 0.59375 -2.640625 0.59375q-1.546875 0 -2.421875 -0.640625q-1.171875 -0.859375 -1.171875 -2.375q0 -1.421875 0.953125 -2.34375q0.953125 -0.9375 2.484375 -0.9375q0.546875 0 1.015625 0.109375q0.484375 0.09375 0.890625 0.296875zm8.670044 -2.59375l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm4.482544 5.5625l0 1.375l1.34375 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.890625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.203125 0l0 -5.21875l-0.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l3.0625 0q1.359375 0 2.140625 0.75q0.796875 0.734375 0.796875 1.796875q0 0.578125 -0.265625 1.09375q-0.203125 0.390625 -0.671875 0.78125q-0.46875 0.375 -0.96875 0.578125q-0.484375 0.1875 -1.296875 0.1875l-1.25 0zm0 -1.34375l1.234375 0q0.875 0 1.375 -0.390625q0.515625 -0.390625 0.515625 -0.9375q0 -0.46875 -0.421875 -0.8125q-0.40625 -0.359375 -1.171875 -0.359375l-1.53125 0l0 2.5zm12.513794 0.109375q0 1.765625 -1.140625 2.953125q-1.125 1.171875 -2.609375 1.171875q-1.328125 0 -2.40625 -0.96875q-1.328125 -1.21875 -1.328125 -3.15625q0 -1.953125 1.328125 -3.171875q1.078125 -0.984375 2.40625 -0.984375q1.484375 0 2.609375 1.1875q1.140625 1.171875 1.140625 2.96875zm-1.34375 0q0 -1.21875 -0.734375 -2.015625q-0.734375 -0.796875 -1.671875 -0.796875q-0.921875 0 -1.671875 0.796875q-0.734375 0.796875 -0.734375 2.015625q0 1.203125 0.734375 2.0q0.75 0.796875 1.671875 0.796875q0.9375 0 1.671875 -0.796875q0.734375 -0.796875 0.734375 -2.0zm3.638855 3.59375q-0.171875 0.21875 -0.28125 0.28125q-0.109375 0.046875 -0.265625 0.046875q-0.296875 0 -0.484375 -0.203125q-0.1875 -0.203125 -0.1875 -0.671875l0 -0.90625q0 -0.484375 0.1875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.234375 0 0.390625 0.125q0.171875 0.125 0.25 0.421875q0.078125 0.28125 0.171875 0.390625q0.1875 0.1875 0.671875 0.40625q0.484375 0.203125 1.0625 0.203125q0.890625 0 1.453125 -0.421875q0.375 -0.25 0.375 -0.625q0 -0.234375 -0.1875 -0.453125q-0.171875 -0.21875 -0.5625 -0.359375q-0.265625 -0.109375 -1.171875 -0.28125q-1.09375 -0.1875 -1.65625 -0.46875q-0.546875 -0.296875 -0.875 -0.8125q-0.328125 -0.53125 -0.328125 -1.125q0 -0.96875 0.796875 -1.6875q0.796875 -0.71875 2.078125 -0.71875q0.515625 0 0.953125 0.125q0.453125 0.109375 0.8125 0.34375q0.25 -0.25 0.515625 -0.25q0.296875 0 0.46875 0.203125q0.1875 0.203125 0.1875 0.671875l0 1.0q0 0.484375 -0.1875 0.6875q-0.171875 0.203125 -0.46875 0.203125q-0.25 0 -0.4375 -0.140625q-0.140625 -0.109375 -0.21875 -0.4375q-0.0625 -0.34375 -0.171875 -0.484375q-0.1875 -0.25 -0.5625 -0.40625q-0.375 -0.171875 -0.875 -0.171875q-0.71875 0 -1.140625 0.34375q-0.421875 0.328125 -0.421875 0.6875q0 0.25 0.171875 0.484375q0.171875 0.21875 0.5 0.359375q0.21875 0.078125 1.25 0.28125q1.03125 0.203125 1.578125 0.453125q0.546875 0.234375 0.90625 0.75q0.375 0.515625 0.375 1.21875q0 1.0 -0.703125 1.59375q-0.921875 0.765625 -2.359375 0.765625q-0.546875 0 -1.078125 -0.140625q-0.515625 -0.125 -1.015625 -0.390625zm10.623169 -6.203125l0 5.21875l1.34375 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.34375 0l0 -5.21875l-1.34375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l4.03125 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.296875 -0.203125 0.484375q-0.203125 0.1875 -0.6875 0.1875l-1.34375 0zm8.201294 2.53125l2.21875 2.6875q0.40625 0 0.59375 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.5 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.1875 -0.46875q0.1875 -0.1875 0.578125 -0.1875l-1.34375 -1.65625l-1.359375 1.65625q0.375 0 0.5625 0.203125q0.1875 0.1875 0.1875 0.46875q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.5 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.59375 -0.1875l2.21875 -2.703125l-2.109375 -2.515625q-0.390625 -0.015625 -0.59375 -0.203125q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.234375 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.21875 -0.125 0.40625q-0.09375 0.109375 -0.390625 0.265625l1.25 1.484375l1.21875 -1.484375q-0.296875 -0.15625 -0.375 -0.265625q-0.140625 -0.1875 -0.140625 -0.40625q0 -0.296875 0.203125 -0.484375q0.21875 -0.1875 0.6875 -0.1875l1.234375 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.578125 0.203125l-2.078125 2.53125zm8.138794 3.34375q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm6.65448 -3.90625l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.34375 -0.15625 0.53125 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.265625 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.46875 0zm11.388794 -3.859375l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.546875 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm-1.328125 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875z" fill-rule="nonzero"/><path fill="#000000" d="m596.98846 574.337l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25zm7.8012695 0l0.21875 0q0.359375 0 0.609375 0.25q0.25 0.25 0.25 0.609375q0 0.359375 -0.25 0.609375q-0.25 0.25 -0.609375 0.25l-0.21875 0q-0.359375 0 -0.625 -0.25q-0.25 -0.25 -0.25 -0.609375q0 -0.359375 0.25 -0.609375q0.265625 -0.25 0.625 -0.25z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m410.0862 91.29473c15.769165 3.0465012 68.47415 28.027802 94.61499 18.279c26.140839 -9.748802 49.212494 -62.554802 62.23001 -76.771805c13.017517 -14.216999 13.229187 -7.1084995 15.875 -8.530199" fill-rule="evenodd"/><path stroke="#6aa84f" stroke-width="4.0" stroke-linejoin="round" stroke-linecap="butt" d="m410.0862 91.29473c15.7691345 3.0464935 68.47415 28.027794 94.61499 18.278992c13.070404 -4.8743896 25.373505 -20.5131 36.036224 -36.71032c5.33136 -8.098618 10.2526245 -16.336853 14.654663 -23.439007c2.2009888 -3.5510788 4.272217 -6.8181305 6.199951 -9.641697c0.24090576 -0.35294724 0.4796753 -0.69896317 0.7161255 -1.0377388l0.6710205 -0.949646" fill-rule="evenodd"/><path fill="#6aa84f" stroke="#6aa84f" stroke-width="4.0" stroke-linecap="butt" d="m566.702 43.253487l11.273315 -15.686729l-18.719055 4.7703934z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m414.75577 111.0c20.873352 10.806366 101.86185 4.285286 125.24002 64.838196c23.378113 60.552917 7.7231445 243.88852 15.028809 298.4793c7.305664 54.59076 24.004333 24.22116 28.805176 29.0654" fill-rule="evenodd"/><path stroke="#6aa84f" stroke-width="4.0" stroke-linejoin="round" stroke-linecap="butt" d="m414.75577 111.0c20.873322 10.806366 101.86191 4.2852783 125.24002 64.838196c23.378113 60.552917 7.7231445 243.88852 15.028748 298.4793c0.91326904 6.8238525 1.9732056 12.320221 3.1383667 16.723755c0.58258057 2.2017822 1.1914673 4.1303406 1.8214111 5.815094c0.07873535 0.21057129 0.15783691 0.4173584 0.23724365 0.6203613l0.21588135 0.5397339" fill-rule="evenodd"/><path fill="#6aa84f" stroke="#6aa84f" stroke-width="4.0" stroke-linecap="butt" d="m558.9601 504.45612l19.170105 -2.3807373l-16.215454 -10.498566z" fill-rule="evenodd"/><path fill="#6aa84f" d="m135.75601 114.20727l86.70888 52.629555l3.0393677 -5.0078735l3.936798 16.094849l-16.094269 3.9366455l3.0393677 -5.0078735l-86.70889 -52.629555l8.047134 -1.9683151z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m135.75601 114.20727l86.70888 52.629555l3.0393677 -5.0078735l3.936798 16.094849l-16.094269 3.9366455l3.0393677 -5.0078735l-86.70889 -52.629555l8.047134 -1.9683151z" fill-rule="evenodd"/><path fill="#cfe2f3" d="m16.181103 251.20998l140.53543 0l0 161.76378l-140.53543 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.181103 251.20998l140.53543 0l0 161.76378l-140.53543 0z" fill-rule="evenodd"/><path fill="#000000" d="m31.931103 272.62625l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm7.998184 0l-4.28125 9.21875q-0.15625 0.34375 -0.28125 0.453125q-0.171875 0.125 -0.421875 0.125q-0.265625 0 -0.46875 -0.203125q-0.203125 -0.1875 -0.203125 -0.4375q0 -0.171875 0.15625 -0.5l4.296875 -9.21875q0.15625 -0.34375 0.28125 -0.453125q0.171875 -0.125 0.40625 -0.125q0.28125 0 0.46875 0.1875q0.203125 0.1875 0.203125 0.4375q0 0.171875 -0.15625 0.515625zm14.683868 3.921875q-0.328125 -0.203125 -0.6875 -0.296875q-0.34375 -0.109375 -0.734375 -0.109375q-0.78125 0 -1.234375 0.25q-0.203125 0.109375 -0.203125 0.25q0 0.140625 0.265625 0.28125q0.21875 0.109375 0.9375 0.203125q1.328125 0.1875 1.84375 0.375q0.6875 0.234375 1.046875 0.71875q0.375 0.46875 0.375 1.0q0 0.703125 -0.625 1.1875q-0.90625 0.703125 -2.34375 0.703125q-0.578125 0 -1.078125 -0.109375q-0.484375 -0.09375 -0.890625 -0.296875q-0.109375 0.09375 -0.21875 0.140625q-0.109375 0.046875 -0.21875 0.046875q-0.3125 0 -0.5 -0.203125q-0.1875 -0.21875 -0.1875 -0.6875l0 -0.453125q0 -0.484375 0.1875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.234375 0 0.390625 0.140625q0.171875 0.125 0.265625 0.453125q0.296875 0.25 0.71875 0.390625q0.4375 0.125 1.0 0.125q0.921875 0 1.421875 -0.28125q0.25 -0.140625 0.25 -0.296875q0 -0.265625 -0.34375 -0.4375q-0.34375 -0.15625 -1.421875 -0.28125q-1.609375 -0.171875 -2.15625 -0.65625q-0.53125 -0.46875 -0.53125 -1.171875q0 -0.703125 0.59375 -1.1875q0.828125 -0.640625 2.15625 -0.640625q0.453125 0 0.875 0.09375q0.4375 0.078125 0.828125 0.25q0.125 -0.078125 0.234375 -0.125q0.109375 -0.046875 0.1875 -0.046875q0.28125 0 0.453125 0.203125q0.1875 0.203125 0.1875 0.6875l0 0.328125q0 0.4375 -0.09375 0.59375q-0.21875 0.296875 -0.578125 0.296875q-0.234375 0 -0.421875 -0.140625q-0.171875 -0.15625 -0.234375 -0.40625zm6.232559 -0.234375l0 2.671875q0 0.4375 0.171875 0.578125q0.28125 0.203125 0.984375 0.203125q1.015625 0 1.875 -0.4375q0.3437462 -0.15625 0.5312462 -0.15625q0.25 0 0.4375 0.203125q0.203125 0.1875 0.203125 0.46875q0 0.25 -0.203125 0.4375q-0.328125 0.3125 -1.2656212 0.578125q-0.9375 0.25 -1.578125 0.25q-1.25 0 -1.875 -0.53125q-0.609375 -0.546875 -0.609375 -1.328125l0 -2.9375l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.484375 0l0 -1.203125q0 -0.484375 0.171875 -0.6875q0.1875 -0.203125 0.484375 -0.203125q0.296875 0 0.484375 0.203125q0.1875 0.203125 0.1875 0.6875l0 1.203125l2.46875 0q0.484375 0 0.6874962 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.20312119 0.1875 -0.6874962 0.1875l-2.46875 0zm8.045055 -1.328125l0 0.828125q0.84375 -0.609375 1.328125 -0.796875q0.484375 -0.203125 0.90625 -0.203125q0.65625 0 1.265625 0.46875q0.421875 0.328125 0.421875 0.671875q0 0.28125 -0.203125 0.484375q-0.203125 0.203125 -0.484375 0.203125q-0.25 0 -0.53125 -0.25q-0.265625 -0.25 -0.46875 -0.25q-0.28125 0 -0.84375 0.359375q-0.546875 0.34375 -1.390625 1.046875l0 2.015625l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.484375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.8125 0zm8.763809 -2.53125l0 1.40625l-1.59375 0l0 -1.40625l1.59375 0zm0.171875 2.53125l0 4.578125l1.609375 0q0.46875 0 0.671875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.671875 0.1875l-4.546875 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.21875 -0.1875 0.6875 -0.1875l1.609375 0l0 -3.25l-1.078125 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l2.40625 0zm5.998184 0l0 0.453125q0.359375 -0.3125 0.796875 -0.46875q0.4375 -0.15625 0.953125 -0.15625q1.1875 0 1.875 0.734375q0.546875 0.578125 0.546875 1.53125l0 2.484375q0.421875 0 0.625 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.203125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.1875 -0.46875q0.203125 -0.1875 0.640625 -0.1875l0 -2.53125q0 -0.421875 -0.234375 -0.625q-0.3125 -0.265625 -0.921875 -0.265625q-0.46875 0 -0.8125 0.1875q-0.34375 0.171875 -0.875 0.75l0 2.484375q0.5 0 0.65625 0.09375q0.3125 0.1875 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.09375 0.671875 -0.09375l0 -3.25q-0.421875 0 -0.625 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.28125 0zm10.810684 0.390625l0 -0.390625l1.5625 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-0.234375 0l0 4.859375q0 0.703125 -0.296875 1.21875q-0.296875 0.53125 -0.90625 0.90625q-0.609375 0.375 -1.375 0.375l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.171875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.46875 0q0.625 0 0.953125 -0.34375q0.34375 -0.328125 0.34375 -0.828125l0 -0.65625q-0.4375 0.296875 -0.90625 0.4375q-0.453125 0.140625 -0.9375 0.140625q-1.359375 0 -2.28125 -0.90625q-0.921875 -0.90625 -0.921875 -2.25q0 -1.328125 0.921875 -2.234375q0.921875 -0.921875 2.28125 -0.921875q0.5 0 0.953125 0.15625q0.46875 0.140625 0.890625 0.4375zm-0.015625 2.5625q0 -0.734375 -0.546875 -1.265625q-0.53125 -0.546875 -1.296875 -0.546875q-0.765625 0 -1.3125 0.546875q-0.53125 0.53125 -0.53125 1.265625q0 0.734375 0.53125 1.28125q0.546875 0.53125 1.3125 0.53125q0.765625 0 1.296875 -0.53125q0.546875 -0.546875 0.546875 -1.28125zm7.560684 2.28125q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm5.607559 -7.765625l0 2.84375q0.421875 -0.25 0.84375 -0.359375q0.421875 -0.125 0.859375 -0.125q0.6875 0 1.21875 0.234375q0.546875 0.234375 0.90625 0.734375q0.359375 0.5 0.359375 1.265625l0 2.515625q0.5 0 0.65625 0.09375q0.3125 0.203125 0.3125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.46875 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.171875 -0.109375 0.671875 -0.09375l0 -2.421875q0 -0.515625 -0.234375 -0.71875q-0.3125 -0.28125 -0.90625 -0.28125q-0.4375 0 -0.78125 0.171875q-0.34375 0.171875 -0.921875 0.75l0 2.5q0.515625 0 0.65625 0.09375q0.328125 0.203125 0.328125 0.578125q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.515625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.359375 0.3125 -0.5625q0.15625 -0.109375 0.65625 -0.09375l0 -5.78125l-0.21875 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l1.5625 0zm10.388809 7.765625q0 0.375 -0.265625 0.640625q-0.265625 0.25 -0.734375 0.25q-0.46875 0 -0.734375 -0.25q-0.265625 -0.265625 -0.265625 -0.640625q0 -0.375 0.265625 -0.625q0.265625 -0.265625 0.734375 -0.265625q0.46875 0 0.734375 0.265625q0.265625 0.25 0.265625 0.625zm10.045067 -7.765625l0 7.109375l0.21875 0q0.484375 0 0.6875 0.1875q0.21875 0.171875 0.21875 0.484375q0 0.28125 -0.21875 0.46875q-0.203125 0.1875 -0.6875 0.1875l-1.5468826 0l0 -0.328125q-0.46875 0.265625 -0.96875 0.40625q-0.484375 0.140625 -1.03125 0.140625q-1.5 0 -2.421875 -0.859375q-0.90625 -0.875 -0.90625 -2.171875q0 -1.359375 0.953125 -2.3125q0.96875 -0.953125 2.359375 -0.953125q0.515625 0 1.015625 0.15625q0.5 0.15625 1.0 0.46875l0 -1.65625l-0.234375 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.46875q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.6875 -0.1875l1.5625076 0zm-1.3281326 5.65625q0 -0.828125 -0.59375 -1.390625q-0.578125 -0.578125 -1.40625 -0.578125q-0.84375 0 -1.421875 0.578125q-0.578125 0.5625 -0.578125 1.375q0 0.734375 0.515625 1.203125q0.53125 0.46875 1.484375 0.46875q0.9375 0 1.46875 -0.46875q0.53125 -0.46875 0.53125 -1.1875zm9.607567 0.5l-5.46875 0q0.203125 0.53125 0.734375 0.84375q0.53125 0.3125 1.4375 0.3125q0.75 0 1.96875 -0.3125q0.515625 -0.125 0.703125 -0.125q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.46875q0 0.265625 -0.203125 0.4375q-0.265625 0.25 -1.28125 0.46875q-1.0 0.21875 -1.921875 0.21875q-1.609375 0 -2.578125 -0.90625q-0.953125 -0.90625 -0.953125 -2.234375q0 -1.40625 1.03125 -2.28125q1.046875 -0.875 2.40625 -0.875q0.8125 0 1.484375 0.28125q0.6875 0.28125 1.015625 0.609375q0.46875 0.484375 0.78125 1.203125q0.203125 0.484375 0.203125 1.140625l0 0.5625zm-1.46875 -1.328125q-0.3125 -0.578125 -0.8125 -0.859375q-0.484375 -0.28125 -1.171875 -0.28125q-0.671875 0 -1.171875 0.28125q-0.5 0.28125 -0.8125 0.859375l3.96875 0zm6.107559 -0.96875l0 3.25l1.90625 0q0.484375 0 0.6875 0.1875q0.203125 0.171875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-4.03125 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.296875 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.796875 0l0 -3.25l-0.640625 0q-0.484375 0 -0.6875 -0.1875q-0.203125 -0.1875 -0.203125 -0.484375q0 -0.28125 0.203125 -0.46875q0.203125 -0.1875 0.6875 -0.1875l0.640625 0l0 -0.515625q0 -0.859375 0.65625 -1.4375q0.65625 -0.578125 1.890625 -0.578125q0.5625 0 1.28125 0.109375q0.71875 0.09375 0.921875 0.28125q0.21875 0.171875 0.21875 0.453125q0 0.3125 -0.1875 0.515625q-0.1875 0.1875 -0.453125 0.1875q-0.125 0 -0.359375 -0.046875q-0.828125 -0.171875 -1.46875 -0.171875q-0.671875 0 -0.921875 0.203125q-0.25 0.203125 -0.25 0.484375l0 0.515625l2.0625 0q0.484375 0 0.6875 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.28125 -0.203125 0.46875q-0.203125 0.1875 -0.6875 0.1875l-2.0625 0z" fill-rule="nonzero"/><path fill="#000000" d="m27.118603 312.42313q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.453125 0.1875q-0.28125 0 -0.46875 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.46875 -0.1875q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125z" fill-rule="nonzero"/><path fill="#000000" d="m29.884228 338.43872q0 0.6875 -0.484375 1.171875q-0.46875 0.484375 -1.140625 0.484375q-0.671875 0 -1.15625 -0.484375q-0.46875 -0.484375 -0.46875 -1.171875q0 -0.6875 0.46875 -1.15625q0.484375 -0.484375 1.15625 -0.484375q0.6875 0 1.15625 0.484375q0.46875 0.46875 0.46875 1.15625zm-0.375 0q0 -0.515625 -0.375 -0.890625q-0.359375 -0.375 -0.875 -0.375q-0.5 0 -0.875 0.375q-0.359375 0.375 -0.359375 0.890625q0 0.53125 0.359375 0.90625q0.375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.90625zm2.109375 1.84375l-4.9375 1.609375q-0.09375 0.03125 -0.140625 0.03125q-0.0625 0 -0.125 -0.046875q-0.046875 -0.0625 -0.046875 -0.140625q0 -0.0625 0.046875 -0.125q0.03125 -0.046875 0.140625 -0.078125l4.9375 -1.609375q0.09375 -0.03125 0.140625 -0.03125q0.0625 0 0.109375 0.0625q0.0625 0.0625 0.0625 0.125q0 0.078125 -0.046875 0.140625q-0.03125 0.03125 -0.140625 0.0625zm-0.078125 3.125q0 0.671875 -0.484375 1.15625q-0.484375 0.484375 -1.15625 0.484375q-0.65625 0 -1.140625 -0.484375q-0.484375 -0.484375 -0.484375 -1.15625q0 -0.6875 0.46875 -1.171875q0.484375 -0.484375 1.15625 -0.484375q0.671875 0 1.15625 0.484375q0.484375 0.484375 0.484375 1.171875zm-0.390625 0q0 -0.546875 -0.375 -0.90625q-0.359375 -0.359375 -0.875 -0.359375q-0.515625 0 -0.875 0.375q-0.359375 0.359375 -0.359375 0.890625q0 0.515625 0.359375 0.890625q0.359375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.890625zm6.5356445 -4.96875q0 0.6875 -0.484375 1.171875q-0.46875 0.484375 -1.140625 0.484375q-0.671875 0 -1.15625 -0.484375q-0.46875 -0.484375 -0.46875 -1.171875q0 -0.6875 0.46875 -1.15625q0.484375 -0.484375 1.15625 -0.484375q0.6875 0 1.15625 0.484375q0.46875 0.46875 0.46875 1.15625zm-0.375 0q0 -0.515625 -0.375 -0.890625q-0.359375 -0.375 -0.875 -0.375q-0.5 0 -0.875 0.375q-0.359375 0.375 -0.359375 0.890625q0 0.53125 0.359375 0.90625q0.375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.90625zm2.109375 1.84375l-4.9375 1.609375q-0.09375 0.03125 -0.140625 0.03125q-0.0625 0 -0.125 -0.046875q-0.046875 -0.0625 -0.046875 -0.140625q0 -0.0625 0.046875 -0.125q0.03125 -0.046875 0.140625 -0.078125l4.9375 -1.609375q0.09375 -0.03125 0.140625 -0.03125q0.0625 0 0.109375 0.0625q0.0625 0.0625 0.0625 0.125q0 0.078125 -0.046875 0.140625q-0.03125 0.03125 -0.140625 0.0625zm-0.078125 3.125q0 0.671875 -0.484375 1.15625q-0.484375 0.484375 -1.15625 0.484375q-0.65625 0 -1.140625 -0.484375q-0.484375 -0.484375 -0.484375 -1.15625q0 -0.6875 0.46875 -1.171875q0.484375 -0.484375 1.15625 -0.484375q0.671875 0 1.15625 0.484375q0.484375 0.484375 0.484375 1.171875zm-0.390625 0q0 -0.546875 -0.375 -0.90625q-0.359375 -0.359375 -0.875 -0.359375q-0.515625 0 -0.875 0.375q-0.359375 0.359375 -0.359375 0.890625q0 0.515625 0.359375 0.890625q0.359375 0.359375 0.875 0.359375q0.515625 0 0.875 -0.359375q0.375 -0.375 0.375 -0.890625zm3.7231445 -4.015625l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm7.0043945 2.90625l0 -0.78125q-1.078125 0.984375 -2.359375 0.984375q-0.78125 0 -1.1875 -0.421875q-0.515625 -0.5625 -0.515625 -1.296875l0 -3.4375l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0l0 3.984375q0 0.515625 0.328125 0.859375q0.328125 0.328125 0.828125 0.328125q1.296875 0 2.375 -1.1875l0 -3.4375l-0.984375 0q-0.1875 0 -0.265625 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.265625 -0.078125l1.53125 0l0 4.96875l0.4375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-0.984375 0zm3.9731445 -7.96875l0 3.53125q0.96875 -1.25 2.328125 -1.25q1.171875 0 2.0 0.84375q0.828125 0.84375 0.828125 2.078125q0 1.25 -0.84375 2.109375q-0.828125 0.859375 -1.984375 0.859375q-1.390625 0 -2.328125 -1.25l0 1.046875l-1.25 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l0.71875 0l0 -6.890625l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 5.234375q0 -1.015625 -0.6875 -1.703125q-0.6875 -0.703125 -1.625 -0.703125q-0.921875 0 -1.625 0.703125q-0.6875 0.6875 -0.6875 1.703125q0 1.0 0.6875 1.703125q0.703125 0.703125 1.625 0.703125q0.9375 0 1.625 -0.703125q0.6875 -0.703125 0.6875 -1.703125zm5.4575157 -5.234375l0 7.4375l2.09375 0q0.203125 0 0.28125 0.078125q0.078125 0.0625 0.078125 0.1875q0 0.109375 -0.078125 0.1875q-0.078125 0.078125 -0.28125 0.078125l-4.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.09375 0l0 -6.890625l-1.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l2.078125 0zm7.8012695 -0.265625l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm9.73877 0.546875l0 -0.1875q0 -0.1875 0.078125 -0.265625q0.078125 -0.09375 0.1875 -0.09375q0.125 0 0.203125 0.09375q0.078125 0.078125 0.078125 0.265625l0 1.21875q-0.015625 0.1875 -0.09375 0.28125q-0.0625 0.078125 -0.1875 0.078125q-0.109375 0 -0.1875 -0.078125q-0.0625 -0.078125 -0.078125 -0.25q-0.03125 -0.4375 -0.59375 -0.84375q-0.546875 -0.40625 -1.5 -0.40625q-1.1875 0 -1.796875 0.75q-0.609375 0.734375 -0.609375 1.6875q0 1.03125 0.671875 1.703125q0.6875 0.671875 1.765625 0.671875q0.609375 0 1.25 -0.21875q0.65625 -0.234375 1.171875 -0.734375q0.140625 -0.140625 0.234375 -0.140625q0.109375 0 0.171875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.265625 -0.640625 0.6875q-1.03125 0.671875 -2.296875 0.671875q-1.28125 0 -2.125 -0.8125q-0.828125 -0.828125 -0.828125 -2.09375q0 -1.28125 0.84375 -2.125q0.859375 -0.859375 2.140625 -0.859375q1.234375 0 2.0625 0.734375zm9.129395 8.515625l-7.09375 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.0625 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l7.09375 0q0.1875 0 0.265625 0.0625q0.078125 0.078125 0.078125 0.203125q0 0.125 -0.078125 0.1875q-0.078125 0.078125 -0.265625 0.078125zm5.7856445 -3.5625l0 -0.765625q-1.171875 0.96875 -2.484375 0.96875q-0.953125 0 -1.5 -0.484375q-0.53125 -0.484375 -0.53125 -1.1875q0 -0.765625 0.703125 -1.34375q0.71875 -0.578125 2.078125 -0.578125q0.359375 0 0.796875 0.046875q0.4375 0.046875 0.9375 0.140625l0 -0.859375q0 -0.4375 -0.40625 -0.75q-0.40625 -0.328125 -1.21875 -0.328125q-0.625 0 -1.75 0.359375q-0.203125 0.0625 -0.265625 0.0625q-0.09375 0 -0.171875 -0.078125q-0.0625 -0.078125 -0.0625 -0.1875q0 -0.109375 0.0625 -0.171875q0.078125 -0.09375 0.71875 -0.265625q0.984375 -0.265625 1.484375 -0.265625q1.015625 0 1.578125 0.5q0.5625 0.5 0.5625 1.125l0 3.53125l0.71875 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-1.25 0zm0 -2.65625q-0.375 -0.109375 -0.796875 -0.15625q-0.421875 -0.046875 -0.875 -0.046875q-1.171875 0 -1.828125 0.5q-0.484375 0.375 -0.484375 0.890625q0 0.484375 0.375 0.8125q0.375 0.328125 1.09375 0.328125q0.703125 0 1.296875 -0.265625q0.59375 -0.28125 1.21875 -0.890625l0 -1.171875zm4.2543945 -2.84375l0 0.96875q0.484375 -0.578125 1.015625 -0.859375q0.546875 -0.296875 1.296875 -0.296875q0.78125 0 1.453125 0.375q0.671875 0.359375 1.03125 1.015625q0.359375 0.65625 0.359375 1.390625q0 1.140625 -0.828125 1.953125q-0.8125 0.8125 -2.015625 0.8125q-1.421875 0 -2.3125 -1.15625l0 3.21875l1.296875 0q0.1875 0 0.265625 0.0625q0.09375 0.078125 0.09375 0.203125q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-2.546875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.09375 -0.0625 0.28125 -0.0625l0.71875 0l0 -6.875l-0.71875 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.203125q0 -0.109375 0.078125 -0.1875q0.09375 -0.078125 0.28125 -0.078125l1.25 0zm4.625 2.59375q0 -0.921875 -0.671875 -1.578125q-0.671875 -0.65625 -1.640625 -0.65625q-0.96875 0 -1.640625 0.65625q-0.671875 0.65625 -0.671875 1.578125q0 0.90625 0.671875 1.578125q0.671875 0.65625 1.640625 0.65625q0.96875 0 1.640625 -0.65625q0.671875 -0.65625 0.671875 -1.578125zm5.4575195 -5.328125l0 1.375l-0.796875 0l0 -1.375l0.796875 0zm0.015625 2.734375l0 4.96875l2.09375 0q0.1875 0 0.265625 0.078125q0.09375 0.0625 0.09375 0.1875q0 0.109375 -0.09375 0.1875q-0.078125 0.078125 -0.265625 0.078125l-4.71875 0q-0.203125 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.1875q0.078125 -0.078125 0.28125 -0.078125l2.09375 0l0 -4.421875l-1.5625 0q-0.1875 0 -0.28125 -0.078125q-0.078125 -0.078125 -0.078125 -0.1875q0 -0.125 0.078125 -0.203125q0.078125 -0.078125 0.28125 -0.078125l2.09375 0zm7.4575195 2.34375q0 -0.734375 0.1875 -1.546875q0.1875 -0.828125 0.71875 -1.890625q0.546875 -1.078125 0.796875 -1.296875q0.078125 -0.078125 0.171875 -0.078125q0.109375 0 0.1875 0.078125q0.078125 0.078125 0.078125 0.1875q0 0.0625 -0.046875 0.140625q-0.6875 1.28125 -1.0 2.328125q-0.296875 1.03125 -0.296875 2.078125q0 1.046875 0.296875 2.09375q0.3125 1.03125 1.0 2.3125q0.046875 0.078125 0.046875 0.140625q0 0.09375 -0.078125 0.171875q-0.078125 0.09375 -0.1875 0.09375q-0.09375 0 -0.171875 -0.078125q-0.234375 -0.21875 -0.765625 -1.265625q-0.53125 -1.046875 -0.734375 -1.84375q-0.203125 -0.8125 -0.203125 -1.625zm8.020027 0q0 0.734375 -0.203125 1.546875q-0.1875 0.8125 -0.734375 1.890625q-0.53125 1.078125 -0.78125 1.296875q-0.078125 0.078125 -0.15625 0.078125q-0.125 0 -0.203125 -0.09375q-0.078125 -0.078125 -0.078125 -0.171875q0 -0.0625 0.046875 -0.140625q0.703125 -1.28125 1.0 -2.3125q0.296875 -1.046875 0.296875 -2.09375q0 -1.046875 -0.296875 -2.078125q-0.296875 -1.046875 -1.0 -2.328125q-0.046875 -0.078125 -0.046875 -0.140625q0 -0.109375 0.078125 -0.1875q0.078125 -0.078125 0.203125 -0.078125q0.078125 0 0.15625 0.078125q0.234375 0.203125 0.765625 1.25q0.546875 1.046875 0.75 1.859375q0.203125 0.8125 0.203125 1.625z" fill-rule="nonzero"/><path fill="#000000" d="m27.118603 376.4231q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.453125 0.1875q-0.28125 0 -0.46875 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.46875 -0.1875q0.265625 0 0.453125 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125zm2.609375 0q0 0.28125 -0.1875 0.46875q-0.1875 0.1875 -0.46875 0.1875q-0.265625 0 -0.453125 -0.1875q-0.1875 -0.1875 -0.1875 -0.46875q0 -0.265625 0.1875 -0.453125q0.1875 -0.1875 0.453125 -0.1875q0.28125 0 0.46875 0.1875q0.1875 0.1875 0.1875 0.453125z" fill-rule="nonzero"/><path fill="#6aa84f" d="m133.29906 336.70108l85.46431 -41.795807l-2.2992096 -4.700775l14.000244 4.8036804l-4.803406 13.999481l-2.2992096 -4.7008057l-85.46431 41.795807l2.401703 -6.999756z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m133.29906 336.70108l85.46431 -41.795807l-2.2992096 -4.700775l14.000244 4.8036804l-4.803406 13.999481l-2.2992096 -4.7008057l-85.46431 41.795807l2.401703 -6.999756z" fill-rule="evenodd"/><path fill="#e6b8af" d="m197.99963 550.50757l124.0 0l0 51.496094l-124.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m197.99963 550.50757l124.0 0l0 51.496094l-124.0 0z" fill-rule="evenodd"/><path fill="#000000" d="m208.49963 572.1756l0 -13.359375l1.765625 0l0 5.484375l6.9375 0l0 -5.484375l1.765625 0l0 13.359375l-1.765625 0l0 -6.296875l-6.9375 0l0 6.296875l-1.765625 0zm19.832321 -3.109375l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm15.453842 4.578125q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm10.469467 4.859375l0 -1.21875q-0.90625 1.4375 -2.703125 1.4375q-1.15625 0 -2.125 -0.640625q-0.96875 -0.640625 -1.5 -1.78125q-0.53125 -1.140625 -0.53125 -2.625q0 -1.453125 0.484375 -2.625q0.484375 -1.1875 1.4375 -1.8125q0.96875 -0.625 2.171875 -0.625q0.875 0 1.546875 0.375q0.6875 0.359375 1.109375 0.953125l0 -4.796875l1.640625 0l0 13.359375l-1.53125 0zm-5.171875 -4.828125q0 1.859375 0.78125 2.78125q0.78125 0.921875 1.84375 0.921875q1.078125 0 1.828125 -0.875q0.75 -0.890625 0.75 -2.6875q0 -1.984375 -0.765625 -2.90625q-0.765625 -0.9375 -1.890625 -0.9375q-1.078125 0 -1.8125 0.890625q-0.734375 0.890625 -0.734375 2.8125zm15.906967 1.71875l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.125702 5.765625l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0z" fill-rule="nonzero"/><path fill="#000000" d="m214.68713 588.9412l0 -1.578125l5.65625 0l0 4.953125q-1.296875 1.046875 -2.6875 1.578125q-1.375 0.515625 -2.84375 0.515625q-1.96875 0 -3.578125 -0.84375q-1.609375 -0.84375 -2.421875 -2.4375q-0.8125 -1.59375 -0.8125 -3.5625q0 -1.953125 0.8125 -3.640625q0.8125 -1.6875 2.34375 -2.5q1.53125 -0.828125 3.515625 -0.828125q1.453125 0 2.625 0.46875q1.171875 0.46875 1.828125 1.3125q0.671875 0.828125 1.015625 2.171875l-1.59375 0.4375q-0.296875 -1.015625 -0.75 -1.59375q-0.4375 -0.59375 -1.265625 -0.9375q-0.828125 -0.34375 -1.84375 -0.34375q-1.203125 0 -2.09375 0.375q-0.890625 0.359375 -1.4375 0.96875q-0.53125 0.59375 -0.828125 1.3125q-0.515625 1.234375 -0.515625 2.6875q0 1.78125 0.609375 2.984375q0.625 1.203125 1.796875 1.796875q1.171875 0.578125 2.5 0.578125q1.140625 0 2.234375 -0.4375q1.09375 -0.453125 1.65625 -0.953125l0 -2.484375l-3.921875 0zm14.683304 2.125l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.141342 5.765625l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm17.000717 -3.109375l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.125717 5.765625l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm12.540802 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm7.781952 3.390625l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm0.9958191 -3.375q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm9.281952 4.84375l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m75.44882 198.69029l21.984245 0l0 26.078735l-21.984245 0z" fill-rule="evenodd"/><path stroke="#ffffff" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m75.44882 198.69029l21.984245 0l0 26.078735l-21.984245 0z" fill-rule="evenodd"/><path fill="#000000" d="m86.13632 196.64966l0 -1.875l1.875 0l0 1.875l-1.875 0z" fill-rule="nonzero"/><path fill="#000000" d="m86.13632 218.64966l0 -1.875l1.875 0l0 1.875l-1.875 0z" fill-rule="nonzero"/><path fill="#000000" d="m86.13632 240.64966l0 -1.875l1.875 0l0 1.875l-1.875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m255.81665 602.0026l-85.63779 73.00787" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m255.81667 602.0026l-81.07185 69.115295" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m173.67323 669.86096l-2.3818817 4.201111l4.5250397 -1.6871948z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m255.81665 602.0026l90.99213 74.99213" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m255.81667 602.0026l86.36195 71.17615" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m341.1281 674.45337l4.5525208 1.6116333l-2.4515076 -4.1608887z" fill-rule="evenodd"/><path fill="#fff2cc" d="m118.0 675.99945l82.96063 0l0 102.17297l-13.827057 13.827026l-69.133575 0z" fill-rule="evenodd"/><path fill="#cbc1a3" d="m187.13358 791.99945l2.7654114 -11.0616455l11.0616455 -2.7653809z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m187.13358 791.99945l2.7654114 -11.0616455l11.0616455 -2.7653809l-13.827057 13.827026l-69.133575 0l0 -116.0l82.96063 0l0 102.17297" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m187.13358 791.99945l2.7654114 -11.0616455l11.0616455 -2.7653809l-13.827057 13.827026l-69.133575 0l0 -116.0l82.96063 0l0 102.17297" fill-rule="evenodd"/><path fill="#000000" d="m132.39062 704.71655l-3.40625 7.375q-0.140625 0.265625 -0.234375 0.359375q-0.140625 0.109375 -0.328125 0.109375q-0.21875 0 -0.390625 -0.15625q-0.15625 -0.15625 -0.15625 -0.359375q0 -0.140625 0.125 -0.40625l3.4375 -7.375q0.125 -0.265625 0.21875 -0.359375q0.140625 -0.109375 0.328125 -0.109375q0.21875 0 0.375 0.15625q0.15625 0.15625 0.15625 0.359375q0 0.140625 -0.125 0.40625zm6.3947906 0l-3.40625 7.375q-0.140625 0.265625 -0.234375 0.359375q-0.140625 0.109375 -0.328125 0.109375q-0.21875 0 -0.390625 -0.15625q-0.15625 -0.15625 -0.15625 -0.359375q0 -0.140625 0.125 -0.40625l3.4375 -7.375q0.125 -0.265625 0.21875 -0.359375q0.140625 -0.109375 0.328125 -0.109375q0.21875 0 0.375 0.15625q0.15625 0.15625 0.15625 0.359375q0 0.140625 -0.125 0.40625zm8.945831 1.890625l0 0.296875q0.265625 -0.25 0.46875 -0.34375q0.21875 -0.109375 0.5 -0.109375q0.21875 0 0.453125 0.125q0.234375 0.109375 0.453125 0.34375q0.265625 -0.234375 0.546875 -0.34375q0.296875 -0.109375 0.59375 -0.109375q0.59375 0 0.953125 0.3125q0.5 0.421875 0.5 1.09375l0 2.390625q0.328125 0 0.484375 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.015625 0l0 -3.359375q0 -0.25 -0.09375 -0.34375q-0.078125 -0.09375 -0.265625 -0.09375q-0.171875 0 -0.328125 0.09375q-0.1875 0.125 -0.46875 0.484375l0 2.15625q0.34375 0 0.5 0.15625q0.15625 0.140625 0.15625 0.375q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.015625 0l0 -3.359375q0 -0.25 -0.09375 -0.34375q-0.078125 -0.09375 -0.265625 -0.09375q-0.1875 0 -0.375 0.125q-0.171875 0.109375 -0.4375 0.453125l0 2.15625q0.34375 0 0.5 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-0.96875 0q-0.375 0 -0.53125 -0.140625q-0.171875 -0.15625 -0.171875 -0.390625q0 -0.234375 0.15625 -0.375q0.15625 -0.15625 0.5 -0.15625l0 -2.59375q-0.34375 0 -0.5 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.171875 -0.375q0.15625 -0.15625 0.53125 -0.15625l1.015625 0zm9.004166 4.71875l0 -0.25q-0.40625 0.21875 -0.890625 0.3125q-0.484375 0.109375 -0.875 0.109375q-0.859375 0 -1.390625 -0.453125q-0.53125 -0.453125 -0.53125 -1.0q0 -0.671875 0.671875 -1.25q0.6875 -0.578125 1.890625 -0.578125q0.484375 0 1.125 0.109375l0 -0.25q0 -0.25 -0.203125 -0.390625q-0.203125 -0.15625 -0.78125 -0.15625q-0.484375 0 -1.234375 0.1875q-0.28125 0.0625 -0.4375 0.0625q-0.203125 0 -0.359375 -0.140625q-0.140625 -0.15625 -0.140625 -0.40625q0 -0.125 0.046875 -0.21875q0.046875 -0.109375 0.140625 -0.171875q0.09375 -0.0625 0.390625 -0.140625q0.40625 -0.109375 0.8125 -0.171875q0.40625 -0.0625 0.734375 -0.0625q1.0 0 1.546875 0.4375q0.546875 0.421875 0.546875 1.15625l0 2.203125l0.1875 0q0.390625 0 0.546875 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.25 0zm0 -1.90625q-0.640625 -0.125 -1.1875 -0.125q-0.640625 0 -1.109375 0.3125q-0.296875 0.203125 -0.296875 0.40625q0 0.15625 0.140625 0.25q0.265625 0.171875 0.71875 0.171875q0.375 0 0.859375 -0.140625q0.5 -0.15625 0.875 -0.421875l0 -0.453125zm5.1760406 -1.75l0 2.140625q0 0.34375 0.140625 0.453125q0.21875 0.171875 0.78125 0.171875q0.8125 0 1.5 -0.359375q0.265625 -0.125 0.421875 -0.125q0.203125 0 0.359375 0.15625q0.15625 0.15625 0.15625 0.375q0 0.203125 -0.171875 0.359375q-0.25 0.25 -1.0 0.453125q-0.75 0.203125 -1.265625 0.203125q-1.0 0 -1.5 -0.421875q-0.484375 -0.4375 -0.484375 -1.0625l0 -2.34375l-0.390625 0q-0.390625 0 -0.5625 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.5625 -0.15625l0.390625 0l0 -0.96875q0 -0.390625 0.140625 -0.546875q0.15625 -0.171875 0.390625 -0.171875q0.234375 0 0.375 0.171875q0.15625 0.15625 0.15625 0.546875l0 0.96875l1.96875 0q0.390625 0 0.546875 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.96875 0zm5.5510406 -3.09375l0 2.28125q0.34375 -0.203125 0.671875 -0.296875q0.34375 -0.09375 0.6875 -0.09375q0.5625 0 0.984375 0.1875q0.4375 0.1875 0.71875 0.59375q0.28125 0.390625 0.28125 1.015625l0 2.0q0.40625 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -1.9375q0 -0.40625 -0.1875 -0.578125q-0.25 -0.21875 -0.734375 -0.21875q-0.34375 0 -0.625 0.140625q-0.265625 0.140625 -0.734375 0.59375l0 2.0q0.421875 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -4.625l-0.1875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.546875 -0.15625l1.25 0zm8.316666 6.21875q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625zm4.4729156 -6.21875l0 2.28125q0.34375 -0.203125 0.671875 -0.296875q0.34375 -0.09375 0.6875 -0.09375q0.5625 0 0.984375 0.1875q0.4375 0.1875 0.71875 0.59375q0.28125 0.390625 0.28125 1.015625l0 2.0q0.40625 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -1.9375q0 -0.40625 -0.1875 -0.578125q-0.25 -0.21875 -0.734375 -0.21875q-0.34375 0 -0.625 0.140625q-0.265625 0.140625 -0.734375 0.59375l0 2.0q0.421875 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -4.625l-0.1875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.546875 -0.15625l1.25 0z" fill-rule="nonzero"/><path fill="#000000" d="m131.01562 723.7947q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#000000" d="m131.01562 736.7947q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#000000" d="m131.01562 749.7947q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#fff2cc" d="m312.0141 675.99945l90.99213 0l0 100.83435l-15.16568 15.165649l-75.82645 0z" fill-rule="evenodd"/><path fill="#cbc1a3" d="m387.84055 791.99945l3.033142 -12.132507l12.132538 -3.033142z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m387.84055 791.99945l3.033142 -12.132507l12.132538 -3.033142l-15.16568 15.165649l-75.82645 0l0 -116.0l90.99213 0l0 100.83435" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m387.84055 791.99945l3.033142 -12.132507l12.132538 -3.033142l-15.16568 15.165649l-75.82645 0l0 -116.0l90.99213 0l0 100.83435" fill-rule="evenodd"/><path fill="#000000" d="m326.40472 704.04724l-3.40625 7.375q-0.140625 0.265625 -0.234375 0.359375q-0.140625 0.109375 -0.328125 0.109375q-0.21875 0 -0.390625 -0.15625q-0.15625 -0.15625 -0.15625 -0.359375q0 -0.140625 0.125 -0.40625l3.4375 -7.375q0.125 -0.265625 0.21875 -0.359375q0.140625 -0.109375 0.328125 -0.109375q0.21875 0 0.375 0.15625q0.15625 0.15625 0.15625 0.359375q0 0.140625 -0.125 0.40625zm6.3947754 0l-3.40625 7.375q-0.140625 0.265625 -0.234375 0.359375q-0.140625 0.109375 -0.328125 0.109375q-0.21875 0 -0.390625 -0.15625q-0.15625 -0.15625 -0.15625 -0.359375q0 -0.140625 0.125 -0.40625l3.4375 -7.375q0.125 -0.265625 0.21875 -0.359375q0.140625 -0.109375 0.328125 -0.109375q0.21875 0 0.375 0.15625q0.15625 0.15625 0.15625 0.359375q0 0.140625 -0.125 0.40625zm11.742706 3.140625q-0.25 -0.171875 -0.546875 -0.25q-0.28125 -0.078125 -0.59375 -0.078125q-0.609375 0 -0.984375 0.203125q-0.15625 0.09375 -0.15625 0.1875q0 0.125 0.21875 0.234375q0.171875 0.09375 0.734375 0.171875q1.0625 0.140625 1.484375 0.28125q0.546875 0.1875 0.84375 0.578125q0.296875 0.375 0.296875 0.796875q0 0.578125 -0.5 0.953125q-0.734375 0.5625 -1.890625 0.5625q-0.453125 0 -0.84375 -0.078125q-0.390625 -0.078125 -0.71875 -0.234375q-0.078125 0.0625 -0.171875 0.109375q-0.09375 0.03125 -0.1875 0.03125q-0.25 0 -0.40625 -0.15625q-0.140625 -0.171875 -0.140625 -0.5625l0 -0.359375q0 -0.390625 0.140625 -0.546875q0.15625 -0.171875 0.40625 -0.171875q0.1875 0 0.3125 0.109375q0.140625 0.109375 0.203125 0.375q0.25 0.203125 0.59375 0.3125q0.34375 0.109375 0.78125 0.109375q0.734375 0 1.140625 -0.234375q0.203125 -0.109375 0.203125 -0.234375q0 -0.21875 -0.28125 -0.34375q-0.28125 -0.140625 -1.140625 -0.234375q-1.28125 -0.140625 -1.71875 -0.515625q-0.421875 -0.375 -0.421875 -0.9375q0 -0.578125 0.484375 -0.953125q0.65625 -0.515625 1.703125 -0.515625q0.375 0 0.71875 0.078125q0.34375 0.0625 0.65625 0.203125q0.09375 -0.078125 0.171875 -0.109375q0.09375 -0.03125 0.171875 -0.03125q0.21875 0 0.359375 0.171875q0.140625 0.15625 0.140625 0.53125l0 0.265625q0 0.34375 -0.078125 0.46875q-0.171875 0.25 -0.453125 0.25q-0.1875 0 -0.34375 -0.125q-0.140625 -0.125 -0.1875 -0.3125zm4.988556 -0.1875l0 2.140625q0 0.34375 0.140625 0.453125q0.21875 0.171875 0.78125 0.171875q0.8125 0 1.5 -0.359375q0.265625 -0.125 0.421875 -0.125q0.203125 0 0.359375 0.15625q0.15625 0.15625 0.15625 0.375q0 0.203125 -0.171875 0.359375q-0.25 0.25 -1.0 0.453125q-0.75 0.203125 -1.265625 0.203125q-1.0 0 -1.5 -0.421875q-0.484375 -0.4375 -0.484375 -1.0625l0 -2.34375l-0.390625 0q-0.390625 0 -0.5625 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.5625 -0.15625l0.390625 0l0 -0.96875q0 -0.390625 0.140625 -0.546875q0.15625 -0.171875 0.390625 -0.171875q0.234375 0 0.375 0.171875q0.15625 0.15625 0.15625 0.546875l0 0.96875l1.96875 0q0.390625 0 0.546875 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.96875 0zm6.4260254 -1.0625l0 0.65625q0.671875 -0.484375 1.0625 -0.640625q0.390625 -0.15625 0.734375 -0.15625q0.515625 0 1.0 0.375q0.34375 0.265625 0.34375 0.53125q0 0.234375 -0.171875 0.390625q-0.15625 0.15625 -0.375 0.15625q-0.203125 0 -0.421875 -0.1875q-0.21875 -0.203125 -0.390625 -0.203125q-0.21875 0 -0.671875 0.28125q-0.4375 0.28125 -1.109375 0.84375l0 1.609375l1.53125 0q0.375 0 0.53125 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.53125 0.140625l-3.21875 0q-0.390625 0 -0.5625 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.5625 -0.15625l0.625 0l0 -2.59375l-0.375 0q-0.390625 0 -0.5625 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.5625 -0.15625l1.4375 0zm7.004181 -2.03125l0 1.125l-1.265625 0l0 -1.125l1.265625 0zm0.140625 2.03125l0 3.65625l1.28125 0q0.390625 0 0.546875 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-3.625 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.546875 -0.15625l1.28125 0l0 -2.59375l-0.859375 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.546875 -0.15625l1.921875 0zm4.8010254 0l0 0.359375q0.28125 -0.25 0.625 -0.375q0.359375 -0.125 0.765625 -0.125q0.953125 0 1.5 0.578125q0.4375 0.46875 0.4375 1.234375l0 1.984375q0.34375 0 0.5 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-0.96875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.15625 -0.15625 0.5 -0.15625l0 -2.015625q0 -0.34375 -0.1875 -0.515625q-0.25 -0.203125 -0.734375 -0.203125q-0.375 0 -0.65625 0.140625q-0.265625 0.140625 -0.6875 0.609375l0 1.984375q0.40625 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.203125 0q-0.390625 0 -0.5625 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.078125 0.53125 -0.078125l0 -2.59375q-0.328125 0 -0.5 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.5625 -0.15625l1.015625 0zm8.644806 0.3125l0 -0.3125l1.25 0q0.375 0 0.53125 0.15625q0.171875 0.140625 0.171875 0.375q0 0.234375 -0.171875 0.390625q-0.15625 0.140625 -0.53125 0.140625l-0.1875 0l0 3.875q0 0.5625 -0.234375 0.984375q-0.234375 0.421875 -0.734375 0.71875q-0.484375 0.3125 -1.09375 0.3125l-1.203125 0q-0.390625 0 -0.5625 -0.15625q-0.15625 -0.140625 -0.15625 -0.375q0 -0.25 0.15625 -0.40625q0.171875 -0.140625 0.5625 -0.140625l1.171875 0q0.484375 0 0.75 -0.265625q0.28125 -0.265625 0.28125 -0.671875l0 -0.515625q-0.359375 0.234375 -0.734375 0.34375q-0.359375 0.109375 -0.75 0.109375q-1.078125 0 -1.8125 -0.71875q-0.734375 -0.734375 -0.734375 -1.796875q0 -1.0625 0.734375 -1.78125q0.734375 -0.734375 1.8125 -0.734375q0.40625 0 0.765625 0.125q0.375 0.109375 0.71875 0.34375zm-0.015625 2.046875q0 -0.59375 -0.4375 -1.015625q-0.421875 -0.4375 -1.03125 -0.4375q-0.609375 0 -1.046875 0.4375q-0.4375 0.421875 -0.4375 1.015625q0 0.59375 0.4375 1.03125q0.4375 0.421875 1.046875 0.421875q0.609375 0 1.03125 -0.421875q0.4375 -0.4375 0.4375 -1.03125zm6.0510254 1.828125q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625zm4.472931 -6.21875l0 2.28125q0.34375 -0.203125 0.671875 -0.296875q0.34375 -0.09375 0.6875 -0.09375q0.5625 0 0.984375 0.1875q0.4375 0.1875 0.71875 0.59375q0.28125 0.390625 0.28125 1.015625l0 2.0q0.40625 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -1.9375q0 -0.40625 -0.1875 -0.578125q-0.25 -0.21875 -0.734375 -0.21875q-0.34375 0 -0.625 0.140625q-0.265625 0.140625 -0.734375 0.59375l0 2.0q0.421875 0 0.53125 0.078125q0.25 0.15625 0.25 0.453125q0 0.234375 -0.15625 0.390625q-0.15625 0.140625 -0.546875 0.140625l-1.21875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.296875 0.25 -0.453125q0.125 -0.09375 0.53125 -0.078125l0 -4.625l-0.1875 0q-0.375 0 -0.546875 -0.140625q-0.15625 -0.15625 -0.15625 -0.390625q0 -0.234375 0.15625 -0.375q0.171875 -0.15625 0.546875 -0.15625l1.25 0z" fill-rule="nonzero"/><path fill="#000000" d="m325.02972 723.12537q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#000000" d="m325.02972 736.12537q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#000000" d="m325.02972 749.12537q0 0.296875 -0.21875 0.5q-0.21875 0.203125 -0.59375 0.203125q-0.375 0 -0.59375 -0.203125q-0.203125 -0.203125 -0.203125 -0.5q0 -0.3125 0.203125 -0.515625q0.21875 -0.203125 0.59375 -0.203125q0.375 0 0.59375 0.203125q0.21875 0.203125 0.21875 0.515625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m239.43922 747.1704l0 -21.984253l17.448807 0l0 21.984253z" fill-rule="evenodd"/><path stroke="#ffffff" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m239.43922 747.1704l0 -21.984253l17.448807 0l0 21.984253z" fill-rule="evenodd"/><path fill="#000000" d="m233.08363 736.4829l-1.875 0l0 -1.875l1.875 0l0 1.875z" fill-rule="nonzero"/><path fill="#000000" d="m255.08363 736.4829l-1.875 0l0 -1.875l1.875 0l0 1.875z" fill-rule="nonzero"/><path fill="#000000" d="m277.08365 736.4829l-1.875 0l0 -1.875l1.875 0l0 1.875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m239.43922 667.1699l0 -21.984253l17.448807 0l0 21.984253z" fill-rule="evenodd"/><path stroke="#ffffff" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m239.43922 667.1699l0 -21.984253l17.448807 0l0 21.984253z" fill-rule="evenodd"/><path fill="#000000" d="m221.88364 655.2637l-3.203125 0l0 -3.203125l3.203125 0l0 3.203125z" fill-rule="nonzero"/><path fill="#000000" d="m259.88364 655.2637l-3.203125 0l0 -3.203125l3.203125 0l0 3.203125z" fill-rule="nonzero"/><path fill="#000000" d="m297.88364 655.2637l-3.203125 0l0 -3.203125l3.203125 0l0 3.203125z" fill-rule="nonzero"/><path fill="#e6b8af" d="m513.0 700.0l132.0 0l0 51.496033l-132.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m513.0 700.0l132.0 0l0 51.496033l-132.0 0z" fill-rule="evenodd"/><path fill="#000000" d="m523.4375 721.668l0 -13.359375l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.703125q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0625 0.359375 -2.578125q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.203125zm10.894836 -3.265625q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm15.610046 1.296875l1.609375 0.21875q-0.265625 1.65625 -1.359375 2.609375q-1.078125 0.9375 -2.671875 0.9375q-1.984375 0 -3.1875 -1.296875q-1.203125 -1.296875 -1.203125 -3.71875q0 -1.578125 0.515625 -2.75q0.515625 -1.171875 1.578125 -1.75q1.0625 -0.59375 2.3125 -0.59375q1.578125 0 2.578125 0.796875q1.0 0.796875 1.28125 2.265625l-1.59375 0.234375q-0.234375 -0.96875 -0.8125 -1.453125q-0.578125 -0.5 -1.390625 -0.5q-1.234375 0 -2.015625 0.890625q-0.78125 0.890625 -0.78125 2.8125q0 1.953125 0.75 2.84375q0.75 0.875 1.953125 0.875q0.96875 0 1.609375 -0.59375q0.65625 -0.59375 0.828125 -1.828125z" fill-rule="nonzero"/><path fill="#000000" d="m529.6875 738.43365l0 -1.578125l5.65625 0l0 4.953125q-1.296875 1.046875 -2.6875 1.578125q-1.375 0.515625 -2.84375 0.515625q-1.96875 0 -3.578125 -0.84375q-1.609375 -0.84375 -2.421875 -2.4375q-0.8125 -1.59375 -0.8125 -3.5625q0 -1.953125 0.8125 -3.640625q0.8125 -1.6875 2.34375 -2.5q1.53125 -0.828125 3.515625 -0.828125q1.453125 0 2.625 0.46875q1.171875 0.46875 1.828125 1.3125q0.671875 0.828125 1.015625 2.171875l-1.59375 0.4375q-0.296875 -1.015625 -0.75 -1.59375q-0.4375 -0.59375 -1.265625 -0.9375q-0.828125 -0.34375 -1.84375 -0.34375q-1.203125 0 -2.09375 0.375q-0.890625 0.359375 -1.4375 0.96875q-0.53125 0.59375 -0.828125 1.3125q-0.515625 1.234375 -0.515625 2.6875q0 1.78125 0.609375 2.984375q0.625 1.203125 1.796875 1.796875q1.171875 0.578125 2.5 0.578125q1.140625 0 2.234375 -0.4375q1.09375 -0.453125 1.65625 -0.953125l0 -2.484375l-3.921875 0zm14.683289 2.125l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.141357 5.765625l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm17.000732 -3.109375l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.125671 5.765625l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm12.5408325 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm7.7819824 3.390625l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm0.9957886 -3.375q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm9.281982 4.84375l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0z" fill-rule="nonzero"/><path fill="#e6b8af" d="m697.0 748.0l132.0 0l0 51.496033l-132.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m697.0 748.0l132.0 0l0 51.496033l-132.0 0z" fill-rule="evenodd"/><path fill="#000000" d="m709.7656 769.668l-3.53125 -13.359375l1.8125 0l2.03125 8.765625q0.328125 1.375 0.5625 2.71875q0.5 -2.140625 0.59375 -2.46875l2.546875 -9.015625l2.125 0l1.921875 6.765625q0.71875 2.515625 1.03125 4.71875q0.265625 -1.265625 0.671875 -2.890625l2.09375 -8.59375l1.78125 0l-3.671875 13.359375l-1.703125 0l-2.8125 -10.171875q-0.359375 -1.28125 -0.421875 -1.5625q-0.203125 0.90625 -0.390625 1.5625l-2.828125 10.171875l-1.8125 0zm14.724548 0l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm12.5408325 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm4.2037964 8.5625l0 -13.375l1.484375 0l0 1.25q0.53125 -0.734375 1.1875 -1.09375q0.671875 -0.375 1.625 -0.375q1.234375 0 2.171875 0.640625q0.953125 0.625 1.4375 1.796875q0.484375 1.15625 0.484375 2.546875q0 1.484375 -0.53125 2.671875q-0.53125 1.1875 -1.546875 1.828125q-1.015625 0.625 -2.140625 0.625q-0.8125 0 -1.46875 -0.34375q-0.65625 -0.34375 -1.0625 -0.875l0 4.703125l-1.640625 0zm1.484375 -8.484375q0 1.859375 0.75 2.765625q0.765625 0.890625 1.828125 0.890625q1.09375 0 1.875 -0.921875q0.78125 -0.9375 0.78125 -2.875q0 -1.84375 -0.765625 -2.765625q-0.75 -0.921875 -1.8125 -0.921875q-1.046875 0 -1.859375 0.984375q-0.796875 0.96875 -0.796875 2.84375zm8.891357 8.484375l0 -13.375l1.484375 0l0 1.25q0.53125 -0.734375 1.1875 -1.09375q0.671875 -0.375 1.625 -0.375q1.234375 0 2.171875 0.640625q0.953125 0.625 1.4375 1.796875q0.484375 1.15625 0.484375 2.546875q0 1.484375 -0.53125 2.671875q-0.53125 1.1875 -1.546875 1.828125q-1.015625 0.625 -2.140625 0.625q-0.8125 0 -1.46875 -0.34375q-0.65625 -0.34375 -1.0625 -0.875l0 4.703125l-1.640625 0zm1.484375 -8.484375q0 1.859375 0.75 2.765625q0.765625 0.890625 1.828125 0.890625q1.09375 0 1.875 -0.921875q0.78125 -0.9375 0.78125 -2.875q0 -1.84375 -0.765625 -2.765625q-0.75 -0.921875 -1.8125 -0.921875q-1.046875 0 -1.859375 0.984375q-0.796875 0.96875 -0.796875 2.84375zm15.516357 1.671875l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.125671 5.765625l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0z" fill-rule="nonzero"/><path fill="#000000" d="m713.6875 786.43365l0 -1.578125l5.65625 0l0 4.953125q-1.296875 1.046875 -2.6875 1.578125q-1.375 0.515625 -2.84375 0.515625q-1.96875 0 -3.578125 -0.84375q-1.609375 -0.84375 -2.421875 -2.4375q-0.8125 -1.59375 -0.8125 -3.5625q0 -1.953125 0.8125 -3.640625q0.8125 -1.6875 2.34375 -2.5q1.53125 -0.828125 3.515625 -0.828125q1.453125 0 2.625 0.46875q1.171875 0.46875 1.828125 1.3125q0.671875 0.828125 1.015625 2.171875l-1.59375 0.4375q-0.296875 -1.015625 -0.75 -1.59375q-0.4375 -0.59375 -1.265625 -0.9375q-0.828125 -0.34375 -1.84375 -0.34375q-1.203125 0 -2.09375 0.375q-0.890625 0.359375 -1.4375 0.96875q-0.53125 0.59375 -0.828125 1.3125q-0.515625 1.234375 -0.515625 2.6875q0 1.78125 0.609375 2.984375q0.625 1.203125 1.796875 1.796875q1.171875 0.578125 2.5 0.578125q1.140625 0 2.234375 -0.4375q1.09375 -0.453125 1.65625 -0.953125l0 -2.484375l-3.921875 0zm14.683289 2.125l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.141357 5.765625l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm17.000732 -3.109375l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.125671 5.765625l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm12.5408325 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm7.7819824 3.390625l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm0.9957886 -3.375q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm9.281982 4.84375l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m579.0 751.49603l1.0078735 58.48822" fill-rule="evenodd"/><path stroke="#1155cc" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m579.0 751.49603l0.8010864 46.49005" fill-rule="evenodd"/><path fill="#1155cc" stroke="#1155cc" stroke-width="2.0" stroke-linecap="butt" d="m576.49817 798.04297l3.4593506 9.017883l3.1465454 -9.131714z" fill-rule="evenodd"/><path fill="#d9ead3" d="m542.71063 884.9764c2.588562 0 4.6870117 -2.0984497 4.6870117 -4.6870117l-4.6870117 0c1.2942505 0 2.3435059 -1.0491943 2.3435059 -2.3435059c0 -1.2942505 -1.0492554 -2.3435059 -2.3435059 -2.3435059l4.6870117 0l0 -60.93109l0 0c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117l64.212585 0c2.588562 0 4.6870117 2.0984497 4.6870117 4.6870117c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117l-4.6870117 0l0 60.93109c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117zm14.061035 -70.305115l0 0c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117c-1.2943115 0 -2.3435059 -1.0492554 -2.3435059 -2.3435059c0 -1.2943115 1.0491943 -2.3435059 2.3435059 -2.3435059z" fill-rule="evenodd"/><path fill="#adbba8" d="m556.77167 814.67126l0 0c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117c-1.2943115 0 -2.3435059 -1.0492554 -2.3435059 -2.3435059c0 -1.2943115 1.0491943 -2.3435059 2.3435059 -2.3435059zm-9.374023 65.6181c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117c-2.588562 0 -4.6870117 -2.0984497 -4.6870117 -4.6870117c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117l0 0c1.2942505 0 2.3435059 1.0492554 2.3435059 2.3435059c0 1.2943115 -1.0492554 2.3435059 -2.3435059 2.3435059z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m547.39764 875.60236l0 -60.93109l0 0c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117l64.212585 0c2.588562 0 4.6870117 2.0984497 4.6870117 4.6870117c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117l-4.6870117 0l0 60.93109c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117l-64.212585 0c-2.588562 0 -4.6870117 -2.0984497 -4.6870117 -4.6870117c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117zm4.6870117 -65.6181l0 0c2.588562 0 4.6870117 2.0984497 4.6870117 4.6870117c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117c-1.2943115 0 -2.3435059 -1.0492554 -2.3435059 -2.3435059c0 -1.2943115 1.0491943 -2.3435059 2.3435059 -2.3435059l4.6870117 0m54.838562 4.6870117l-59.525574 0m-9.374023 56.24408l0 0c1.2942505 0 2.3435059 1.0492554 2.3435059 2.3435059c0 1.2943115 -1.0492554 2.3435059 -2.3435059 2.3435059l4.6870117 0m-4.6870117 4.6870117c2.588562 0 4.6870117 -2.0984497 4.6870117 -4.6870117l0 -4.6870117" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m547.39764 875.60236l0 -60.93109l0 0c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117l64.212585 0c2.588562 0 4.6870117 2.0984497 4.6870117 4.6870117c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117l-4.6870117 0l0 60.93109c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117l-64.212585 0c-2.588562 0 -4.6870117 -2.0984497 -4.6870117 -4.6870117c0 -2.588562 2.0984497 -4.6870117 4.6870117 -4.6870117zm4.6870117 -65.6181l0 0c2.588562 0 4.6870117 2.0984497 4.6870117 4.6870117c0 2.588562 -2.0984497 4.6870117 -4.6870117 4.6870117c-1.2943115 0 -2.3435059 -1.0492554 -2.3435059 -2.3435059c0 -1.2943115 1.0491943 -2.3435059 2.3435059 -2.3435059l4.6870117 0m54.838562 4.6870117l-59.525574 0m-9.374023 56.24408l0 0c1.2942505 0 2.3435059 1.0492554 2.3435059 2.3435059c0 1.2943115 -1.0492554 2.3435059 -2.3435059 2.3435059l4.6870117 0m-4.6870117 4.6870117c2.588562 0 4.6870117 -2.0984497 4.6870117 -4.6870117l0 -4.6870117" fill-rule="evenodd"/><path fill="#000000" d="m557.83514 856.74384l0 -13.359375l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.703125q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0625 0.359375 -2.578125q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.203125zm10.894836 -3.265625q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm15.610046 1.296875l1.609375 0.21875q-0.265625 1.65625 -1.359375 2.609375q-1.078125 0.9375 -2.671875 0.9375q-1.984375 0 -3.1875 -1.296875q-1.203125 -1.296875 -1.203125 -3.71875q0 -1.578125 0.515625 -2.75q0.515625 -1.171875 1.578125 -1.75q1.0625 -0.59375 2.3125 -0.59375q1.578125 0 2.578125 0.796875q1.0 0.796875 1.28125 2.265625l-1.59375 0.234375q-0.234375 -0.96875 -0.8125 -1.453125q-0.578125 -0.5 -1.390625 -0.5q-1.234375 0 -2.015625 0.890625q-0.78125 0.890625 -0.78125 2.8125q0 1.953125 0.75 2.84375q0.75 0.875 1.953125 0.875q0.96875 0 1.609375 -0.59375q0.65625 -0.59375 0.828125 -1.828125zm2.359375 0.65625l1.625 -0.25q0.125 0.96875 0.75 1.5q0.625 0.515625 1.75 0.515625q1.125 0 1.671875 -0.453125q0.546875 -0.46875 0.546875 -1.09375q0 -0.546875 -0.484375 -0.875q-0.328125 -0.21875 -1.671875 -0.546875q-1.8125 -0.46875 -2.515625 -0.796875q-0.6875 -0.328125 -1.046875 -0.90625q-0.359375 -0.59375 -0.359375 -1.3125q0 -0.640625 0.296875 -1.1875q0.296875 -0.5625 0.8125 -0.921875q0.375 -0.28125 1.03125 -0.46875q0.671875 -0.203125 1.421875 -0.203125q1.140625 0 2.0 0.328125q0.859375 0.328125 1.265625 0.890625q0.421875 0.5625 0.578125 1.5l-1.609375 0.21875q-0.109375 -0.75 -0.640625 -1.171875q-0.515625 -0.421875 -1.46875 -0.421875q-1.140625 0 -1.625 0.375q-0.46875 0.375 -0.46875 0.875q0 0.3125 0.1875 0.578125q0.203125 0.265625 0.640625 0.4375q0.234375 0.09375 1.4375 0.421875q1.75 0.453125 2.4375 0.75q0.6875 0.296875 1.078125 0.859375q0.390625 0.5625 0.390625 1.40625q0 0.828125 -0.484375 1.546875q-0.46875 0.71875 -1.375 1.125q-0.90625 0.390625 -2.046875 0.390625q-1.875 0 -2.875 -0.78125q-0.984375 -0.78125 -1.25 -2.328125z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m763.0 633.0l157.98425 0l0 34.173218l-157.98425 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m763.0 633.0l157.98425 0l0 34.173218l-157.98425 0z" fill-rule="evenodd"/><path fill="#000000" d="m779.8594 653.8972l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm9.141357 5.765625l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm13.953857 -1.46875l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm1.5895386 1.46875l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm6.1501465 3.71875l-0.1875 -1.53125q0.546875 0.140625 0.9375 0.140625q0.546875 0 0.875 -0.1875q0.328125 -0.171875 0.546875 -0.5q0.15625 -0.25 0.5 -1.21875q0.046875 -0.140625 0.140625 -0.40625l-3.671875 -9.6875l1.765625 0l2.015625 5.59375q0.390625 1.078125 0.703125 2.25q0.28125 -1.125 0.671875 -2.203125l2.078125 -5.640625l1.640625 0l-3.6875 9.828125q-0.59375 1.609375 -0.921875 2.203125q-0.4375 0.8125 -1.0 1.1875q-0.5625 0.375 -1.34375 0.375q-0.484375 0 -1.0625 -0.203125zm9.40625 -0.015625l0 -13.375l1.484375 0l0 1.25q0.53125 -0.734375 1.1875 -1.09375q0.671875 -0.375 1.625 -0.375q1.234375 0 2.171875 0.640625q0.953125 0.625 1.4375 1.796875q0.484375 1.15625 0.484375 2.546875q0 1.484375 -0.53125 2.671875q-0.53125 1.1875 -1.546875 1.828125q-1.015625 0.625 -2.140625 0.625q-0.8125 0 -1.46875 -0.34375q-0.65625 -0.34375 -1.0625 -0.875l0 4.703125l-1.640625 0zm1.484375 -8.484375q0 1.859375 0.75 2.765625q0.765625 0.890625 1.828125 0.890625q1.09375 0 1.875 -0.921875q0.78125 -0.9375 0.78125 -2.875q0 -1.84375 -0.765625 -2.765625q-0.75 -0.921875 -1.8125 -0.921875q-1.046875 0 -1.859375 0.984375q-0.796875 0.96875 -0.796875 2.84375zm8.281982 -0.0625q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm9.297607 -6.625l0 -1.890625l1.640625 0l0 1.890625l-1.640625 0zm0 11.46875l0 -9.671875l1.640625 0l0 9.671875l-1.640625 0zm4.1448364 0l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm13.953796 -1.46875l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm10.366638 0l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625zm7.9176636 0.28125q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm4.1882324 4.859375l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm5.9157715 0.796875l1.59375 0.234375q0.109375 0.75 0.5625 1.078125q0.609375 0.453125 1.671875 0.453125q1.140625 0 1.75 -0.453125q0.625 -0.453125 0.84375 -1.265625q0.125 -0.5 0.109375 -2.109375q-1.0625 1.265625 -2.671875 1.265625q-2.0 0 -3.09375 -1.4375q-1.09375 -1.4375 -1.09375 -3.453125q0 -1.390625 0.5 -2.5625q0.515625 -1.171875 1.453125 -1.796875q0.953125 -0.640625 2.25 -0.640625q1.703125 0 2.8125 1.375l0 -1.15625l1.515625 0l0 8.359375q0 2.265625 -0.46875 3.203125q-0.453125 0.9375 -1.453125 1.484375q-0.984375 0.546875 -2.453125 0.546875q-1.71875 0 -2.796875 -0.78125q-1.0625 -0.765625 -1.03125 -2.34375zm1.359375 -5.8125q0 1.90625 0.75 2.78125q0.765625 0.875 1.90625 0.875q1.125 0 1.890625 -0.859375q0.765625 -0.875 0.765625 -2.734375q0 -1.78125 -0.796875 -2.671875q-0.78125 -0.90625 -1.890625 -0.90625q-1.09375 0 -1.859375 0.890625q-0.765625 0.875 -0.765625 2.625zm15.953857 1.90625l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm12.719482 4.296875l0.234375 1.453125q-0.6875 0.140625 -1.234375 0.140625q-0.890625 0 -1.390625 -0.28125q-0.484375 -0.28125 -0.6875 -0.734375q-0.203125 -0.46875 -0.203125 -1.9375l0 -5.578125l-1.203125 0l0 -1.265625l1.203125 0l0 -2.390625l1.625 -0.984375l0 3.375l1.65625 0l0 1.265625l-1.65625 0l0 5.671875q0 0.6875 0.078125 0.890625q0.09375 0.203125 0.28125 0.328125q0.203125 0.109375 0.578125 0.109375q0.265625 0 0.71875 -0.0625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m841.9921 667.1732l-78.99213 80.81891" fill-rule="evenodd"/><path stroke="#cc0000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m841.9921 667.1732l-70.60443 72.23724" fill-rule="evenodd"/><path fill="#cc0000" stroke="#cc0000" stroke-width="2.0" stroke-linecap="butt" d="m769.02527 737.1014l-3.9815674 8.799805l8.706482 -4.1817017z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m763.0 799.49603l0 28.503967" fill-rule="evenodd"/><path stroke="#cc0000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m763.0 799.49603l0 16.503967" fill-rule="evenodd"/><path fill="#cc0000" stroke="#cc0000" stroke-width="2.0" stroke-linecap="butt" d="m759.69653 816.0l3.3034668 9.076172l3.3034668 -9.076172z" fill-rule="evenodd"/><path fill="#fff2cc" d="m721.5197 828.0l82.96057 0l0 65.616516l-13.123596 13.123657l-69.836975 0z" fill-rule="evenodd"/><path fill="#cbc1a3" d="m791.3567 906.7402l2.6247559 -10.498901l10.49884 -2.6247559z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m791.3567 906.7402l2.6247559 -10.498901l10.49884 -2.6247559l-13.123596 13.123657l-69.836975 0l0 -78.74017l82.96057 0l0 65.616516" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m791.3567 906.7402l2.6247559 -10.498901l10.49884 -2.6247559l-13.123596 13.123657l-69.836975 0l0 -78.74017l82.96057 0l0 65.616516" fill-rule="evenodd"/><path fill="#000000" d="m731.5666 857.6083l0 -9.546875l6.90625 0l0 1.125l-5.640625 0l0 2.921875l5.28125 0l0 1.125l-5.28125 0l0 3.25l5.859375 0l0 1.125l-7.125 0zm8.717834 0l0 -6.90625l1.0625 0l0 0.984375q0.75 -1.140625 2.1875 -1.140625q0.625 0 1.15625 0.21875q0.53125 0.21875 0.78125 0.59375q0.265625 0.359375 0.375 0.859375q0.0625 0.328125 0.0625 1.140625l0 4.25l-1.171875 0l0 -4.203125q0 -0.71875 -0.140625 -1.0625q-0.140625 -0.359375 -0.484375 -0.5625q-0.34375 -0.21875 -0.8125 -0.21875q-0.75 0 -1.296875 0.46875q-0.546875 0.46875 -0.546875 1.796875l0 3.78125l-1.171875 0zm9.974976 -1.046875l0.171875 1.03125q-0.5 0.109375 -0.890625 0.109375q-0.640625 0 -1.0 -0.203125q-0.34375 -0.203125 -0.484375 -0.53125q-0.140625 -0.328125 -0.140625 -1.390625l0 -3.96875l-0.859375 0l0 -0.90625l0.859375 0l0 -1.71875l1.171875 -0.703125l0 2.421875l1.171875 0l0 0.90625l-1.171875 0l0 4.046875q0 0.5 0.046875 0.640625q0.0625 0.140625 0.203125 0.234375q0.140625 0.078125 0.40625 0.078125q0.203125 0 0.515625 -0.046875zm1.1248779 1.046875l0 -6.90625l1.0625 0l0 1.046875q0.40625 -0.734375 0.734375 -0.96875q0.34375 -0.234375 0.765625 -0.234375q0.59375 0 1.203125 0.375l-0.40625 1.078125q-0.4375 -0.25 -0.859375 -0.25q-0.390625 0 -0.703125 0.234375q-0.296875 0.234375 -0.421875 0.640625q-0.203125 0.625 -0.203125 1.359375l0 3.625l-1.171875 0zm4.4071045 2.65625l-0.125 -1.09375q0.375 0.109375 0.65625 0.109375q0.390625 0 0.625 -0.140625q0.234375 -0.125 0.390625 -0.359375q0.109375 -0.171875 0.359375 -0.875q0.03125 -0.09375 0.109375 -0.28125l-2.625 -6.921875l1.265625 0l1.4375 4.0q0.28125 0.765625 0.5 1.59375q0.203125 -0.796875 0.46875 -1.578125l1.484375 -4.015625l1.171875 0l-2.625 7.015625q-0.421875 1.140625 -0.65625 1.578125q-0.3125 0.578125 -0.71875 0.84375q-0.40625 0.28125 -0.96875 0.28125q-0.328125 0 -0.75 -0.15625zm6.7109375 0l0 -9.5625l1.078125 0l0 0.890625q0.375 -0.53125 0.84375 -0.78125q0.484375 -0.265625 1.15625 -0.265625q0.875 0 1.546875 0.453125q0.6875 0.453125 1.03125 1.28125q0.34375 0.828125 0.34375 1.828125q0 1.046875 -0.375 1.90625q-0.375 0.84375 -1.109375 1.296875q-0.71875 0.453125 -1.53125 0.453125q-0.578125 0 -1.046875 -0.25q-0.46875 -0.25 -0.765625 -0.625l0 3.375l-1.171875 0zm1.0625 -6.078125q0 1.34375 0.53125 1.984375q0.546875 0.625 1.3125 0.625q0.78125 0 1.34375 -0.65625q0.5625 -0.65625 0.5625 -2.046875q0 -1.3125 -0.546875 -1.96875q-0.546875 -0.671875 -1.296875 -0.671875q-0.75 0 -1.328125 0.703125q-0.578125 0.703125 -0.578125 2.03125zm5.9124756 -0.03125q0 -1.921875 1.078125 -2.84375q0.890625 -0.765625 2.171875 -0.765625q1.421875 0 2.328125 0.9375q0.90625 0.921875 0.90625 2.578125q0 1.328125 -0.40625 2.09375q-0.390625 0.765625 -1.15625 1.1875q-0.765625 0.421875 -1.671875 0.421875q-1.453125 0 -2.359375 -0.921875q-0.890625 -0.9375 -0.890625 -2.6875zm1.203125 0q0 1.328125 0.578125 1.984375q0.59375 0.65625 1.46875 0.65625q0.875 0 1.453125 -0.65625q0.578125 -0.671875 0.578125 -2.03125q0 -1.28125 -0.59375 -1.9375q-0.578125 -0.65625 -1.4375 -0.65625q-0.875 0 -1.46875 0.65625q-0.578125 0.65625 -0.578125 1.984375zm6.6624756 -4.734375l0 -1.359375l1.171875 0l0 1.359375l-1.171875 0zm0 8.1875l0 -6.90625l1.171875 0l0 6.90625l-1.171875 0zm2.9454956 0l0 -6.90625l1.0625 0l0 0.984375q0.75 -1.140625 2.1875 -1.140625q0.625 0 1.15625 0.21875q0.53125 0.21875 0.78125 0.59375q0.265625 0.359375 0.375 0.859375q0.0625 0.328125 0.0625 1.140625l0 4.25l-1.171875 0l0 -4.203125q0 -0.71875 -0.140625 -1.0625q-0.140625 -0.359375 -0.484375 -0.5625q-0.34375 -0.21875 -0.8125 -0.21875q-0.75 0 -1.296875 0.46875q-0.546875 0.46875 -0.546875 1.796875l0 3.78125l-1.171875 0zm9.974976 -1.046875l0.171875 1.03125q-0.5 0.109375 -0.890625 0.109375q-0.640625 0 -1.0 -0.203125q-0.34375 -0.203125 -0.484375 -0.53125q-0.140625 -0.328125 -0.140625 -1.390625l0 -3.96875l-0.859375 0l0 -0.90625l0.859375 0l0 -1.71875l1.171875 -0.703125l0 2.421875l1.171875 0l0 0.90625l-1.171875 0l0 4.046875q0 0.5 0.046875 0.640625q0.0625 0.140625 0.203125 0.234375q0.140625 0.078125 0.40625 0.078125q0.203125 0 0.515625 -0.046875z" fill-rule="nonzero"/><path fill="#000000" d="m733.2072 873.6083l-2.53125 -9.546875l1.296875 0l1.453125 6.265625q0.234375 0.984375 0.40625 1.953125q0.359375 -1.53125 0.421875 -1.765625l1.828125 -6.453125l1.515625 0l1.375 4.828125q0.515625 1.796875 0.734375 3.390625q0.1875 -0.90625 0.484375 -2.078125l1.484375 -6.140625l1.28125 0l-2.625 9.546875l-1.21875 0l-2.0 -7.265625q-0.265625 -0.921875 -0.3125 -1.125q-0.140625 0.65625 -0.28125 1.125l-2.015625 7.265625l-1.296875 0zm10.510681 0l0 -6.90625l1.0625 0l0 1.046875q0.40625 -0.734375 0.734375 -0.96875q0.34375 -0.234375 0.765625 -0.234375q0.59375 0 1.203125 0.375l-0.40625 1.078125q-0.4375 -0.25 -0.859375 -0.25q-0.390625 0 -0.703125 0.234375q-0.296875 0.234375 -0.421875 0.640625q-0.203125 0.625 -0.203125 1.359375l0 3.625l-1.171875 0zm8.9696045 -0.859375q-0.65625 0.5625 -1.265625 0.796875q-0.59375 0.21875 -1.28125 0.21875q-1.140625 0 -1.75 -0.546875q-0.609375 -0.5625 -0.609375 -1.4375q0 -0.5 0.21875 -0.921875q0.234375 -0.421875 0.609375 -0.671875q0.375 -0.25 0.84375 -0.390625q0.34375 -0.078125 1.046875 -0.171875q1.421875 -0.171875 2.09375 -0.40625q0 -0.234375 0 -0.296875q0 -0.71875 -0.328125 -1.015625q-0.453125 -0.390625 -1.34375 -0.390625q-0.8125 0 -1.21875 0.296875q-0.390625 0.28125 -0.578125 1.015625l-1.140625 -0.15625q0.15625 -0.734375 0.515625 -1.1875q0.359375 -0.453125 1.03125 -0.6875q0.671875 -0.25 1.5625 -0.25q0.890625 0 1.4375 0.203125q0.5625 0.203125 0.8125 0.53125q0.265625 0.3125 0.375 0.796875q0.046875 0.296875 0.046875 1.078125l0 1.5625q0 1.625 0.078125 2.0625q0.078125 0.4375 0.296875 0.828125l-1.21875 0q-0.1875 -0.359375 -0.234375 -0.859375zm-0.09375 -2.609375q-0.640625 0.265625 -1.921875 0.4375q-0.71875 0.109375 -1.015625 0.25q-0.296875 0.125 -0.46875 0.375q-0.15625 0.25 -0.15625 0.546875q0 0.46875 0.34375 0.78125q0.359375 0.3125 1.046875 0.3125q0.671875 0 1.203125 -0.296875q0.53125 -0.296875 0.78125 -0.8125q0.1875 -0.390625 0.1875 -1.171875l0 -0.421875zm2.9906006 6.125l0 -9.5625l1.078125 0l0 0.890625q0.375 -0.53125 0.84375 -0.78125q0.484375 -0.265625 1.15625 -0.265625q0.875 0 1.546875 0.453125q0.6875 0.453125 1.03125 1.28125q0.34375 0.828125 0.34375 1.828125q0 1.046875 -0.375 1.90625q-0.375 0.84375 -1.109375 1.296875q-0.71875 0.453125 -1.53125 0.453125q-0.578125 0 -1.046875 -0.25q-0.46875 -0.25 -0.765625 -0.625l0 3.375l-1.171875 0zm1.0625 -6.078125q0 1.34375 0.53125 1.984375q0.546875 0.625 1.3125 0.625q0.78125 0 1.34375 -0.65625q0.5625 -0.65625 0.5625 -2.046875q0 -1.3125 -0.546875 -1.96875q-0.546875 -0.671875 -1.296875 -0.671875q-0.75 0 -1.328125 0.703125q-0.578125 0.703125 -0.578125 2.03125zm6.3499756 6.078125l0 -9.5625l1.078125 0l0 0.890625q0.375 -0.53125 0.84375 -0.78125q0.484375 -0.265625 1.15625 -0.265625q0.875 0 1.546875 0.453125q0.6875 0.453125 1.03125 1.28125q0.34375 0.828125 0.34375 1.828125q0 1.046875 -0.375 1.90625q-0.375 0.84375 -1.109375 1.296875q-0.71875 0.453125 -1.53125 0.453125q-0.578125 0 -1.046875 -0.25q-0.46875 -0.25 -0.765625 -0.625l0 3.375l-1.171875 0zm1.0625 -6.078125q0 1.34375 0.53125 1.984375q0.546875 0.625 1.3125 0.625q0.78125 0 1.34375 -0.65625q0.5625 -0.65625 0.5625 -2.046875q0 -1.3125 -0.546875 -1.96875q-0.546875 -0.671875 -1.296875 -0.671875q-0.75 0 -1.328125 0.703125q-0.578125 0.703125 -0.578125 2.03125zm11.084351 1.203125l1.203125 0.140625q-0.28125 1.0625 -1.0625 1.65625q-0.765625 0.578125 -1.96875 0.578125q-1.515625 0 -2.40625 -0.9375q-0.890625 -0.9375 -0.890625 -2.609375q0 -1.75 0.890625 -2.703125q0.90625 -0.96875 2.34375 -0.96875q1.390625 0 2.265625 0.9375q0.875 0.9375 0.875 2.65625q0 0.109375 0 0.3125l-5.15625 0q0.0625 1.140625 0.640625 1.75q0.578125 0.59375 1.4375 0.59375q0.65625 0 1.109375 -0.328125q0.453125 -0.34375 0.71875 -1.078125zm-3.84375 -1.90625l3.859375 0q-0.078125 -0.859375 -0.4375 -1.296875q-0.5625 -0.6875 -1.453125 -0.6875q-0.8125 0 -1.359375 0.546875q-0.546875 0.53125 -0.609375 1.4375zm6.5062256 4.125l0 -6.90625l1.0625 0l0 1.046875q0.40625 -0.734375 0.734375 -0.96875q0.34375 -0.234375 0.765625 -0.234375q0.59375 0 1.203125 0.375l-0.40625 1.078125q-0.4375 -0.25 -0.859375 -0.25q-0.390625 0 -0.703125 0.234375q-0.296875 0.234375 -0.421875 0.640625q-0.203125 0.625 -0.203125 1.359375l0 3.625l-1.171875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m86.0 414.0c3.1928024 15.0 2.8244019 71.0 19.1568 90.0c16.332397 19.0 63.364807 16.0 78.8376 24.0c15.472794 8.0 11.666 20.0 13.999207 24.0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m86.0 414.0c3.1928024 15.0 2.8244019 71.0 19.156807 90.0c16.332397 19.0 63.3648 16.0 78.83759 24.0c7.7364044 4.0 10.652893 9.0 11.926941 13.5c0.31851196 1.125 0.5343628 2.21875 0.69721985 3.2578125c0.040725708 0.25976562 0.078125 0.5161133 0.11299133 0.76867676l0.07752991 0.59155273" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m195.18985 546.44415l2.5151367 4.1227417l0.7233124 -4.7749023z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m158.0 106.0c2.8660736 1.6666641 12.787064 -51.833332 17.196396 10.0c4.4093323 61.83333 -1.4330292 297.66666 9.259598 361.0c10.692642 63.333313 48.061737 6.6666565 54.89621 19.0c6.8344574 12.3333435 -11.574509 45.833313 -13.889404 55.0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m158.0 106.0c2.8660736 1.6666641 12.787064 -51.833332 17.196396 10.0c4.4093323 61.83333 -1.4330292 297.66666 9.259598 361.0c10.692642 63.333313 48.061737 6.6666565 54.89621 19.0c3.4172363 6.1666565 0.5235901 17.625 -3.5137024 28.6875c-2.0186462 5.53125 -4.3232117 10.963562 -6.2677765 15.5859375c-0.48616028 1.1556396 -0.9497986 2.260559 -1.3808594 3.303833c-0.21551514 0.5216675 -0.4229126 1.027832 -0.6208801 1.5171509l-0.10069275 0.2506714" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m225.91158 544.793l0.039871216 4.8291626l3.0735931 -3.7249756z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m367.0 414.0c-11.1666565 16.333344 -53.166656 75.33334 -67.0 98.0c-13.8333435 22.666687 -13.3333435 31.666687 -16.0 38.0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m367.0 414.0c-11.1666565 16.333344 -53.166656 75.33334 -67.0 98.0c-6.9166565 11.333313 -10.25 19.25 -12.1875 25.125c-0.484375 1.46875 -0.88153076 2.8098755 -1.2255859 4.044922c-0.17202759 0.6174927 -0.33081055 1.2084961 -0.48059082 1.7756348l-0.34466553 1.3189087" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m284.18274 543.7795l0.24649048 4.823059l2.911377 -3.8531494z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m585.418 264.4777c-13.140381 37.342743 -33.932007 180.76526 -78.84201 224.0564c-44.910004 43.291107 -155.85434 25.445953 -190.61801 35.6904c-34.76367 10.244446 -14.970001 21.480347 -17.96399 25.776428" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m585.418 264.4777c-13.14032 37.342712 -33.932007 180.76526 -78.84204 224.0564c-44.909973 43.291107 -155.85434 25.445953 -190.61798 35.6904c-17.381836 5.1221924 -21.124329 10.49231 -20.89563 15.118835c0.05718994 1.1566162 0.36254883 2.2667847 0.7650757 3.3150024c0.20126343 0.5241089 0.42678833 1.0327148 0.65774536 1.5238647l0.00491333 0.010253906" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m294.8911 544.60645l2.7364807 3.979248l0.46151733 -4.807251z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m585.0 558.00116c-36.5 -4.5 -172.33334 -25.833374 -219.0 -27.0c-46.666656 -1.166687 -50.833344 16.666626 -61.0 20.0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m585.0 558.00116c-36.5 -4.5 -172.33334 -25.833374 -219.0 -27.0c-23.333344 -0.583374 -36.041656 3.583313 -44.1875 8.3125c-4.0729065 2.364563 -7.0052185 4.869751 -9.5546875 6.9921875c-0.6373596 0.5305786 -1.250824 1.0372314 -1.8521729 1.5118408l-0.21755981 0.16949463" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m309.35837 546.55896l-3.0942993 3.7078247l4.753723 -0.8513794z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m585.0 267.9996c-4.5 2.8333435 -18.833313 -55.0 -27.0 17.0c-8.166687 72.0 -18.333313 345.83334 -22.0 414.99997" fill-rule="evenodd"/><path stroke="#1155cc" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m585.0 267.9996c-4.5 2.8333435 -18.833313 -55.0 -27.0 17.0c-4.083313 36.0 -8.666687 122.45831 -12.6875 208.5625c-2.010437 43.052094 -3.880188 86.015594 -5.4765625 122.53909c-0.79815674 18.261658 -1.5280151 34.91339 -2.1728516 49.161133c-0.3223877 7.1238403 -0.6236572 13.6466675 -0.90148926 19.4693l-0.15814209 3.283142" fill-rule="evenodd"/><path fill="#1155cc" stroke="#1155cc" stroke-width="2.0" stroke-linecap="butt" d="m533.30414 687.84863l2.8428955 9.230835l3.7556763 -8.89856z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m585.0 559.0c-5.333313 15.333313 -25.833313 68.83331 -32.0 92.0c-6.166687 23.166687 -4.166687 39.166687 -5.0 47.0" fill-rule="evenodd"/><path stroke="#1155cc" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m585.0 559.0c-5.333313 15.333313 -25.833313 68.83331 -32.0 92.0c-3.083313 11.583313 -4.125 21.375 -4.5 29.25c-0.09375 1.96875 -0.14581299 3.817688 -0.17773438 5.544922l-0.0033569336 0.20928955" fill-rule="evenodd"/><path fill="#1155cc" stroke="#1155cc" stroke-width="2.0" stroke-linecap="butt" d="m545.0166 685.91644l3.0610962 9.160767l3.543518 -8.985168z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m707.0 609.0c5.833313 23.166687 29.166687 115.83331 35.0 139.0" fill-rule="evenodd"/><path stroke="#cc0000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m707.0 609.0l32.069885 127.36328" fill-rule="evenodd"/><path fill="#cc0000" stroke="#cc0000" stroke-width="2.0" stroke-linecap="butt" d="m735.8664 737.16986l5.4196777 7.994812l0.9873047 -9.608093z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m584.00006 267.0c-9.333374 10.1666565 -46.5 0.5 -56.0 61.0c-9.5 60.5 -30.833344 241.5 -1.0 302.0c29.833313 60.5 145.66663 41.333313 180.0 61.0c34.333313 19.666687 21.666626 47.5 26.0 57.0" fill-rule="evenodd"/><path stroke="#cc0000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m584.00006 267.0c-9.333374 10.1666565 -46.5 0.5 -56.0 61.0c-9.5 60.5 -30.833344 241.5 -1.0 302.0c29.833313 60.5 145.66663 41.333313 179.99994 61.0c17.166687 9.833313 22.583313 21.708313 24.25 32.3125c0.416687 2.651062 0.598938 5.2227173 0.67193604 7.663086c0.036376953 1.2202148 0.04547119 2.4075928 0.04284668 3.555725l-0.0095825195 1.5142822" fill-rule="evenodd"/><path fill="#cc0000" stroke="#cc0000" stroke-width="2.0" stroke-linecap="butt" d="m728.6643 736.3332l4.0811157 8.754089l2.5007324 -9.329346z" fill-rule="evenodd"/></g></svg>
\ No newline at end of file
diff --git a/libc/docs/headers/assert.rst b/libc/docs/headers/assert.rst
new file mode 100644
index 0000000000000..06ea27966de1a
--- /dev/null
+++ b/libc/docs/headers/assert.rst
@@ -0,0 +1,27 @@
+.. include:: ../check.rst
+
+========
+assert.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - __STDC_VERSION_ASSERT_H__
+    - |check|
+    - 7.2.1
+    -
+  * - assert
+    -
+    - 7.2.1
+    -
+
diff --git a/libc/docs/complex.rst b/libc/docs/headers/complex.rst
similarity index 98%
rename from libc/docs/complex.rst
rename to libc/docs/headers/complex.rst
index 1e74f16b60e66..b6a340543fad1 100644
--- a/libc/docs/complex.rst
+++ b/libc/docs/headers/complex.rst
@@ -1,4 +1,4 @@
-.. include:: check.rst
+.. include:: ../check.rst
 
 =========
 complex.h
@@ -57,7 +57,7 @@ Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | cimag     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.3.9.2                | N/A                        |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| conj      |                  |                 |                        |                      |                        | 7.3.9.4                | N/A                        |
+| conj      | |check|          | |check|         | |check|                | |check|              | |check|                | 7.3.9.4                | N/A                        |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | cproj     |                  |                 |                        |                      |                        | 7.3.9.5                | N/A                        |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/docs/ctype.rst b/libc/docs/headers/ctype.rst
similarity index 93%
rename from libc/docs/ctype.rst
rename to libc/docs/headers/ctype.rst
index 828785c9b6708..e506830809f79 100644
--- a/libc/docs/ctype.rst
+++ b/libc/docs/headers/ctype.rst
@@ -1,4 +1,4 @@
-.. include:: check.rst
+.. include:: ../check.rst
 
 =======
 ctype.h
@@ -15,7 +15,7 @@ Functions
   * - Function
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - isalnum
     - |check|
     - 7.4.1.1
diff --git a/libc/docs/headers/errno.rst b/libc/docs/headers/errno.rst
new file mode 100644
index 0000000000000..f25aae4f23b2c
--- /dev/null
+++ b/libc/docs/headers/errno.rst
@@ -0,0 +1,35 @@
+.. include:: ../check.rst
+
+=======
+errno.h
+=======
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - EDOM
+    -
+    - 7.5
+    -
+  * - EILSEQ
+    -
+    - 7.5
+    -
+  * - ERANGE
+    -
+    - 7.5
+    -
+  * - errno
+    -
+    - 7.5
+    -
+
diff --git a/libc/docs/fenv.rst b/libc/docs/headers/fenv.rst
similarity index 95%
rename from libc/docs/fenv.rst
rename to libc/docs/headers/fenv.rst
index e7a5a3fb2c815..374b46ef57be0 100644
--- a/libc/docs/fenv.rst
+++ b/libc/docs/headers/fenv.rst
@@ -1,4 +1,4 @@
-.. include:: check.rst
+.. include:: ../check.rst
 
 ======
 fenv.h
@@ -12,10 +12,10 @@ Macros
   :align: center
   :header-rows: 1
 
-  * - Function
+  * - Macro
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - FE_ALL_EXCEPT
     - |check|
     - 7.6.12
@@ -104,7 +104,7 @@ Functions
   * - Function
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - fe_dec_getround
     -
     - 7.6.5.3
diff --git a/libc/docs/headers/float.rst b/libc/docs/headers/float.rst
new file mode 100644
index 0000000000000..b603867fcef9c
--- /dev/null
+++ b/libc/docs/headers/float.rst
@@ -0,0 +1,227 @@
+.. include:: ../check.rst
+
+=======
+float.h
+=======
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - DBL_DECIMAL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_EPSILON
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_HAS_SUBNORM
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_IS_IEC_60559
+    -
+    - 5.3.5.3.3
+    -
+  * - DBL_MANT_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MAX
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MAX_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MAX_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MIN_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_MIN_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DBL_NORM_MAX
+    -
+    - 5.3.5.3.3
+    -
+  * - DBL_SNAN
+    -
+    - 5.3.5.3.3
+    -
+  * - DBL_TRUE_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - DECIMAL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_DECIMAL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_EPSILON
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_EVAL_METHOD
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_HAS_SUBNORM
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_IS_IEC_60559
+    -
+    - 5.3.5.3.3
+    -
+  * - FLT_MANT_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MAX
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MAX_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MAX_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MIN_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_MIN_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_NORM_MAX
+    -
+    - 5.3.5.3.3
+    -
+  * - FLT_RADIX
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_ROUNDS
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - FLT_SNAN
+    -
+    - 5.3.5.3.3
+    -
+  * - FLT_TRUE_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - INFINITY
+    -
+    - 5.3.5.3.3
+    -
+  * - LDBL_DECIMAL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_EPSILON
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_HAS_SUBNORM
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_IS_IEC_60559
+    -
+    - 5.3.5.3.3
+    -
+  * - LDBL_MANT_DIG
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MAX
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MAX_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MAX_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MIN_10_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_MIN_EXP
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - LDBL_NORM_MAX
+    -
+    - 5.3.5.3.3
+    -
+  * - LDBL_SNAN
+    -
+    - 5.3.5.3.3
+    -
+  * - LDBL_TRUE_MIN
+    - |check|
+    - 5.3.5.3.3
+    -
+  * - NAN
+    -
+    - 5.3.5.3.3
+    -
+  * - __STDC_VERSION_FLOAT_H__
+    -
+    - 7.7
+    -
+
diff --git a/libc/docs/headers/index.rst b/libc/docs/headers/index.rst
new file mode 100644
index 0000000000000..9bd6396843e78
--- /dev/null
+++ b/libc/docs/headers/index.rst
@@ -0,0 +1,28 @@
+Implementation Status
+=====================
+
+.. toctree::
+   :maxdepth: 1
+
+   assert
+   complex
+   ctype
+   errno
+   fenv
+   float
+   inttypes
+   locale
+   math/index.rst
+   search
+   setjmp
+   signal
+   stdbit
+   stdio
+   stdlib
+   string
+   strings
+   threads
+   time
+   uchar
+   wchar
+   wctype
diff --git a/libc/docs/headers/inttypes.rst b/libc/docs/headers/inttypes.rst
new file mode 100644
index 0000000000000..f43c80f095c6a
--- /dev/null
+++ b/libc/docs/headers/inttypes.rst
@@ -0,0 +1,42 @@
+.. include:: ../check.rst
+
+==========
+inttypes.h
+==========
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - imaxabs
+    - |check|
+    - 7.8.2.1
+    -
+  * - imaxdiv
+    - |check|
+    - 7.8.2.2
+    -
+  * - strtoimax
+    - |check|
+    - 7.8.2.3
+    -
+  * - strtoumax
+    - |check|
+    - 7.8.2.3
+    -
+  * - wcstoimax
+    -
+    - 7.8.2.4
+    -
+  * - wcstoumax
+    -
+    - 7.8.2.4
+    -
diff --git a/libc/docs/headers/locale.rst b/libc/docs/headers/locale.rst
new file mode 100644
index 0000000000000..2d5525bd3f2f9
--- /dev/null
+++ b/libc/docs/headers/locale.rst
@@ -0,0 +1,63 @@
+.. include:: ../check.rst
+
+========
+locale.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - LC_ALL
+    - |check|
+    - 7.11
+    -
+  * - LC_COLLATE
+    - |check|
+    - 7.11
+    -
+  * - LC_CTYPE
+    - |check|
+    - 7.11
+    -
+  * - LC_MONETARY
+    - |check|
+    - 7.11
+    -
+  * - LC_NUMERIC
+    - |check|
+    - 7.11
+    -
+  * - LC_TIME
+    - |check|
+    - 7.11
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - localeconv
+    - |check|
+    - 7.11.2.1
+    -
+  * - setlocale
+    - |check|
+    - 7.11.1.1
+    -
diff --git a/libc/docs/math/index.rst b/libc/docs/headers/math/index.rst
similarity index 99%
rename from libc/docs/math/index.rst
rename to libc/docs/headers/math/index.rst
index 2b86f49a3619e..adc0afa080ec2 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/headers/math/index.rst
@@ -1,10 +1,10 @@
 .. _math:
 
-==============
-Math Functions
-==============
+======
+math.h
+======
 
-.. include:: ../check.rst
+.. include:: ../../check.rst
 
 .. raw:: html
 
@@ -336,7 +336,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | rsqrt     |                  |                 |                        |                      |                        | 7.12.7.9               | F.10.4.9                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| sin       | |check|          | |check|         |                        |                      |                        | 7.12.4.6               | F.10.1.6                   |
+| sin       | |check|          | |check|         |                        | |check|              |                        | 7.12.4.6               | F.10.1.6                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | sincos    | |check|          | |check|         |                        |                      |                        |                        |                            |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/docs/math/log.rst b/libc/docs/headers/math/log.rst
similarity index 100%
rename from libc/docs/math/log.rst
rename to libc/docs/headers/math/log.rst
diff --git a/libc/docs/math/stdfix.rst b/libc/docs/headers/math/stdfix.rst
similarity index 99%
rename from libc/docs/math/stdfix.rst
rename to libc/docs/headers/math/stdfix.rst
index d8dcb0cfa4c52..58052f000995c 100644
--- a/libc/docs/math/stdfix.rst
+++ b/libc/docs/headers/math/stdfix.rst
@@ -2,7 +2,7 @@
 StdFix Functions
 ================
 
-.. include:: ../check.rst
+.. include:: ../../check.rst
 
 Standards and Goals
 -------------------
diff --git a/libc/docs/libc_search.rst b/libc/docs/headers/search.rst
similarity index 95%
rename from libc/docs/libc_search.rst
rename to libc/docs/headers/search.rst
index 774622d1e66c3..51832e9bdc2ed 100644
--- a/libc/docs/libc_search.rst
+++ b/libc/docs/headers/search.rst
@@ -1,8 +1,8 @@
-=============
-Search Tables
-=============
+========
+search.h
+========
 
-.. include:: check.rst
+.. include:: ../check.rst
 
 ---------------
 Source Location
diff --git a/libc/docs/headers/setjmp.rst b/libc/docs/headers/setjmp.rst
new file mode 100644
index 0000000000000..b0091134f1a6b
--- /dev/null
+++ b/libc/docs/headers/setjmp.rst
@@ -0,0 +1,43 @@
+.. include:: ../check.rst
+
+========
+setjmp.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - __STDC_VERSION_SETJMP_H__
+    -
+    - 7.13.2
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - longjmp
+    - |check|
+    - 7.13.2.1
+    -
+  * - setjmp
+    - |check|
+    - 7.13.1.1
+    -
diff --git a/libc/docs/headers/signal.rst b/libc/docs/headers/signal.rst
new file mode 100644
index 0000000000000..b59ae09342357
--- /dev/null
+++ b/libc/docs/headers/signal.rst
@@ -0,0 +1,207 @@
+.. include:: ../check.rst
+
+========
+signal.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - SIGABRT
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGALRM
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGBUS
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGCHLD
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGCONT
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGFPE
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGHUP
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGILL
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGINT
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGKILL
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGPIPE
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGPOLL
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGPROF
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGQUIT
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGRTMAX
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGRTMIN
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGSEGV
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGSTOP
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGSYS
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGTERM
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGTRAP
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGTSTP
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGTTIN
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGTTOU
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGURG
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGUSR1
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGUSR2
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGVTALRM
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGXCPU
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIGXFSZ
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIG_DFL
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIG_ERR
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIG_HOLD
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+  * - SIG_IGN
+    - |check|
+    - 7.14.3
+    - https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - kill
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/kill.html
+  * - raise
+    - |check|
+    - 7.14.2.1
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/raise.html
+  * - sigaction
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaction.html
+  * - sigaddset
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaddset.html
+  * - sigaltstack
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaltstack.html
+  * - sigdelset
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigdelset.html
+  * - sigemptyset
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigemptyset.html
+  * - sigfillset
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigfillset.html
+  * - signal
+    - |check|
+    - 7.14.1.1
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/signal.html
+  * - sigprocmask
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigprocmask.html
diff --git a/libc/docs/stdbit.rst b/libc/docs/headers/stdbit.rst
similarity index 98%
rename from libc/docs/stdbit.rst
rename to libc/docs/headers/stdbit.rst
index 71f9bbfd1d000..0484d951e19c6 100644
--- a/libc/docs/stdbit.rst
+++ b/libc/docs/headers/stdbit.rst
@@ -1,4 +1,4 @@
-.. include:: check.rst
+.. include:: ../check.rst
 
 ========
 stdbit.h
@@ -12,10 +12,10 @@ Macros
   :align: center
   :header-rows: 1
 
-  * - Function
+  * - Macro
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - __STDC_ENDIAN_BIG__
     - |check|
     - 7.18.2.2
@@ -100,7 +100,7 @@ Functions
   * - Function
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - stdc_bit_ceil_uc
     - |check|
     - 7.18.16
diff --git a/libc/docs/stdio.rst b/libc/docs/headers/stdio.rst
similarity index 96%
rename from libc/docs/stdio.rst
rename to libc/docs/headers/stdio.rst
index d17821562c255..3cd355529d400 100644
--- a/libc/docs/stdio.rst
+++ b/libc/docs/headers/stdio.rst
@@ -1,8 +1,8 @@
-===============
-StdIO Functions
-===============
+=======
+stdio.h
+=======
 
-.. include:: check.rst
+.. include:: ../check.rst
 
 ---------------
 Source location
diff --git a/libc/docs/headers/stdlib.rst b/libc/docs/headers/stdlib.rst
new file mode 100644
index 0000000000000..139d9b4a92228
--- /dev/null
+++ b/libc/docs/headers/stdlib.rst
@@ -0,0 +1,255 @@
+.. include:: ../check.rst
+
+========
+stdlib.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - EXIT_FAILURE
+    - |check|
+    - 7.24
+    -
+  * - EXIT_SUCCESS
+    - |check|
+    - 7.24
+    -
+  * - MB_CUR_MAX
+    - |check|
+    - 7.24
+    -
+  * - RAND_MAX
+    - |check|
+    - 7.24
+    -
+  * - __STDC_VERSION_STDLIB_H__
+    -
+    - 7.24
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - _Exit
+    - |check|
+    - 7.24.4.5
+    -
+  * - abort
+    - |check|
+    - 7.24.4.1
+    -
+  * - abs
+    - |check|
+    - 7.24.6.1
+    -
+  * - aligned_alloc
+    - |check|
+    - 7.24.3.1
+    -
+  * - at_quick_exit
+    - |check|
+    - 7.24.4.3
+    -
+  * - atexit
+    - |check|
+    - 7.24.4.2
+    -
+  * - atof
+    - |check|
+    - 7.24.1.1
+    -
+  * - atoi
+    - |check|
+    - 7.24.1.2
+    -
+  * - atol
+    - |check|
+    - 7.24.1.2
+    -
+  * - atoll
+    - |check|
+    - 7.24.1.2
+    -
+  * - bsearch
+    - |check|
+    - 7.24.5.1
+    -
+  * - calloc
+    - |check|
+    - 7.24.3.2
+    -
+  * - div
+    - |check|
+    - 7.24.6.2
+    -
+  * - exit
+    - |check|
+    - 7.24.4.4
+    -
+  * - free
+    - |check|
+    - 7.24.3.3
+    -
+  * - free_aligned_sized
+    -
+    - 7.24.3.5
+    -
+  * - free_sized
+    -
+    - 7.24.3.4
+    -
+  * - getenv
+    - |check|
+    - 7.24.4.6
+    -
+  * - labs
+    - |check|
+    - 7.24.6.1
+    -
+  * - ldiv
+    - |check|
+    - 7.24.6.2
+    -
+  * - llabs
+    - |check|
+    - 7.24.6.1
+    -
+  * - lldiv
+    - |check|
+    - 7.24.6.2
+    -
+  * - malloc
+    - |check|
+    - 7.24.3.6
+    -
+  * - mblen
+    -
+    - 7.24.7.1
+    -
+  * - mbstowcs
+    -
+    - 7.24.8.1
+    -
+  * - mbtowc
+    -
+    - 7.24.7.2
+    -
+  * - memalignment
+    -
+    - 7.24.9.1
+    -
+  * - qsort
+    - |check|
+    - 7.24.5.2
+    -
+  * - quick_exit
+    - |check|
+    - 7.24.4.7
+    -
+  * - rand
+    - |check|
+    - 7.24.2.1
+    -
+  * - realloc
+    - |check|
+    - 7.24.3.7
+    -
+  * - srand
+    - |check|
+    - 7.24.2.2
+    -
+  * - strfromd
+    - |check|
+    - 7.24.1.3
+    -
+  * - strfromd128
+    -
+    - 7.24.1.4
+    -
+  * - strfromd32
+    -
+    - 7.24.1.4
+    -
+  * - strfromd64
+    -
+    - 7.24.1.4
+    -
+  * - strfromf
+    - |check|
+    - 7.24.1.3
+    -
+  * - strfroml
+    - |check|
+    - 7.24.1.3
+    -
+  * - strtod
+    - |check|
+    - 7.24.1.5
+    -
+  * - strtod128
+    -
+    - 7.24.1.6
+    -
+  * - strtod32
+    -
+    - 7.24.1.6
+    -
+  * - strtod64
+    -
+    - 7.24.1.6
+    -
+  * - strtof
+    - |check|
+    - 7.24.1.5
+    -
+  * - strtol
+    - |check|
+    - 7.24.1.7
+    -
+  * - strtold
+    - |check|
+    - 7.24.1.5
+    -
+  * - strtoll
+    - |check|
+    - 7.24.1.7
+    -
+  * - strtoul
+    - |check|
+    - 7.24.1.7
+    -
+  * - strtoull
+    - |check|
+    - 7.24.1.7
+    -
+  * - system
+    - |check|
+    - 7.24.4.8
+    -
+  * - wcstombs
+    -
+    - 7.24.8.2
+    -
+  * - wctomb
+    -
+    - 7.24.7.3
+    -
diff --git a/libc/docs/headers/string.rst b/libc/docs/headers/string.rst
new file mode 100644
index 0000000000000..55f779c830ea8
--- /dev/null
+++ b/libc/docs/headers/string.rst
@@ -0,0 +1,155 @@
+.. include:: ../check.rst
+
+========
+string.h
+========
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - __STDC_VERSION_STRING_H__
+    -
+    - 7.26.1
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - memccpy
+    - |check|
+    - 7.26.2.2
+    -
+  * - memchr
+    - |check|
+    - 7.26.5.2
+    -
+  * - memcmp
+    - |check|
+    - 7.26.4.1
+    -
+  * - memcpy
+    - |check|
+    - 7.26.2.1
+    -
+  * - memmove
+    - |check|
+    - 7.26.2.3
+    -
+  * - mempcpy
+    - |check|
+    - TODO: glibc extension
+    -
+  * - memset
+    - |check|
+    - 7.26.6.1
+    -
+  * - memset_explicit
+    - |check|
+    - 7.26.6.2
+    -
+  * - stpcpy
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/stpcpy.html
+  * - stpncpy
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/stpncpy.html
+  * - strcat
+    - |check|
+    - 7.26.3.1
+    -
+  * - strchr
+    - |check|
+    - 7.26.5.3
+    -
+  * - strcmp
+    - |check|
+    - 7.26.4.2
+    -
+  * - strcoll
+    - |check|
+    - 7.26.4.3
+    -
+  * - strcpy
+    - |check|
+    - 7.26.2.4
+    -
+  * - strcspn
+    - |check|
+    - 7.26.5.4
+    -
+  * - strdup
+    - |check|
+    - 7.26.2.6
+    -
+  * - strerror
+    - |check|
+    - 7.26.6.3
+    -
+  * - strlen
+    - |check|
+    - 7.26.6.4
+    -
+  * - strncat
+    - |check|
+    - 7.26.3.2
+    -
+  * - strncmp
+    - |check|
+    - 7.26.4.4
+    -
+  * - strncpy
+    - |check|
+    - 7.26.2.5
+    -
+  * - strndup
+    - |check|
+    - 7.26.2.7
+    -
+  * - strpbrk
+    - |check|
+    - 7.26.5.5
+    -
+  * - strrchr
+    - |check|
+    - 7.26.5.6
+    -
+  * - strspn
+    - |check|
+    - 7.26.5.7
+    -
+  * - strstr
+    - |check|
+    - 7.26.5.8
+    -
+  * - strtok
+    - |check|
+    - 7.26.5.9
+    -
+  * - strtok_r
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/strtok_r.html
+  * - strxfrm
+    - |check|
+    - 7.26.4.5
+    -
diff --git a/libc/docs/headers/strings.rst b/libc/docs/headers/strings.rst
new file mode 100644
index 0000000000000..b5935d2683d08
--- /dev/null
+++ b/libc/docs/headers/strings.rst
@@ -0,0 +1,66 @@
+.. include:: ../check.rst
+
+=========
+strings.h
+=========
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - bcmp
+    - |check|
+    -
+    - removed in POSIX.1-2008
+  * - bcopy
+    - |check|
+    -
+    - removed in POSIX.1-2008
+  * - bzero
+    - |check|
+    -
+    - removed in POSIX.1-2008
+  * - ffs
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffs.html
+  * - ffsl
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffsl.html
+  * - ffsll
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffsll.html
+  * - index
+    - |check|
+    -
+    - removed in POSIX.1-2008
+  * - rindex
+    - |check|
+    -
+    - removed in POSIX.1-2008
+  * - strcasecmp
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html
+  * - strcasecmp_l
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html
+  * - strncasecmp
+    - |check|
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html
+  * - strncasecmp_l
+    -
+    -
+    - https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html
diff --git a/libc/docs/threads.rst b/libc/docs/headers/threads.rst
similarity index 88%
rename from libc/docs/threads.rst
rename to libc/docs/headers/threads.rst
index 63cd6c40e1455..be313c6013b8d 100644
--- a/libc/docs/threads.rst
+++ b/libc/docs/headers/threads.rst
@@ -1,4 +1,4 @@
-.. include:: check.rst
+.. include:: ../check.rst
 
 =========
 threads.h
@@ -12,17 +12,21 @@ Macros
   :align: center
   :header-rows: 1
 
-  * - Function
+  * - Macro
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - ONCE_FLAG_INIT
     -
-    - 7.28.1.3
+    - 7.28.1
     -
   * - TSS_DTOR_ITERATIONS
     -
-    - 7.28.1.3
+    - 7.28.1
+    -
+  * - __STDC_NO_THREADS__
+    -
+    - 7.28.1
     -
 
 Functions
@@ -36,7 +40,7 @@ Functions
   * - Function
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section
+    - POSIX.1-2024 Standard Section
   * - call_once
     - |check|
     - 7.28.2.1
@@ -67,27 +71,27 @@ Functions
     -
   * - mtx_destroy
     - |check|
-    - 7.28.4.1
+    - 7.28.4.2
     -
   * - mtx_init
     - |check|
-    - 7.28.4.2
+    - 7.28.4.3
     -
   * - mtx_lock
     - |check|
-    - 7.28.4.3
+    - 7.28.4.4
     -
   * - mtx_timedlock
     -
-    - 7.28.4.4
+    - 7.28.4.5
     -
   * - mtx_trylock
     -
-    - 7.28.4.5
+    - 7.28.4.6
     -
   * - mtx_unlock
     - |check|
-    - 7.28.4.6
+    - 7.28.4.7
     -
   * - thrd_create
     - |check|
diff --git a/libc/docs/date_and_time.rst b/libc/docs/headers/time.rst
similarity index 99%
rename from libc/docs/date_and_time.rst
rename to libc/docs/headers/time.rst
index b745a3b416f80..de82d80a2bec4 100644
--- a/libc/docs/date_and_time.rst
+++ b/libc/docs/headers/time.rst
@@ -1,8 +1,8 @@
-=======================
-Date and Time Functions
-=======================
+======
+time.h
+======
 
-.. include:: check.rst
+.. include:: ../check.rst
 
 ---------------
 Source location
diff --git a/libc/docs/headers/uchar.rst b/libc/docs/headers/uchar.rst
new file mode 100644
index 0000000000000..4645109c8c378
--- /dev/null
+++ b/libc/docs/headers/uchar.rst
@@ -0,0 +1,59 @@
+.. include:: ../check.rst
+
+=======
+uchar.h
+=======
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - __STDC_VERSION_UCHAR_H__
+    -
+    - 7.30.1
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - c16rtomb
+    -
+    - 7.30.2.5
+    -
+  * - c32rtomb
+    -
+    - 7.30.2.7
+    -
+  * - c8rtomb
+    -
+    - 7.30.2.3
+    -
+  * - mbrtoc16
+    -
+    - 7.30.2.4
+    -
+  * - mbrtoc32
+    -
+    - 7.30.2.6
+    -
+  * - mbrtoc8
+    -
+    - 7.30.2.2
+    -
diff --git a/libc/docs/headers/wchar.rst b/libc/docs/headers/wchar.rst
new file mode 100644
index 0000000000000..ce2be3389a2ec
--- /dev/null
+++ b/libc/docs/headers/wchar.rst
@@ -0,0 +1,287 @@
+.. include:: ../check.rst
+
+=======
+wchar.h
+=======
+
+Macros
+======
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Macro
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - WEOF
+    - |check|
+    - 7.31.1
+    -
+  * - __STDC_VERSION_WCHAR_H__
+    -
+    - 7.31.1
+    -
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - btowc
+    - |check|
+    - 7.31.6.2.1
+    -
+  * - fgetwc
+    -
+    - 7.31.3.1
+    -
+  * - fgetws
+    -
+    - 7.31.3.2
+    -
+  * - fputwc
+    -
+    - 7.31.3.3
+    -
+  * - fputws
+    -
+    - 7.31.3.4
+    -
+  * - fwide
+    -
+    - 7.31.3.5
+    -
+  * - fwprintf
+    -
+    - 7.31.2.2
+    -
+  * - fwscanf
+    -
+    - 7.31.2.3
+    -
+  * - getwc
+    -
+    - 7.31.3.6
+    -
+  * - getwchar
+    -
+    - 7.31.3.7
+    -
+  * - mbrlen
+    -
+    - 7.31.6.4.2
+    -
+  * - mbrtowc
+    -
+    - 7.31.6.4.3
+    -
+  * - mbsinit
+    -
+    - 7.31.6.3.1
+    -
+  * - mbsrtowcs
+    -
+    - 7.31.6.5.2
+    -
+  * - putwc
+    -
+    - 7.31.3.8
+    -
+  * - putwchar
+    -
+    - 7.31.3.9
+    -
+  * - swprintf
+    -
+    - 7.31.2.4
+    -
+  * - swscanf
+    -
+    - 7.31.2.5
+    -
+  * - ungetwc
+    -
+    - 7.31.3.10
+    -
+  * - vfwprintf
+    -
+    - 7.31.2.6
+    -
+  * - vfwscanf
+    -
+    - 7.31.2.7
+    -
+  * - vswprintf
+    -
+    - 7.31.2.8
+    -
+  * - vswscanf
+    -
+    - 7.31.2.9
+    -
+  * - vwprintf
+    -
+    - 7.31.2.10
+    -
+  * - vwscanf
+    -
+    - 7.31.2.11
+    -
+  * - wcrtomb
+    -
+    - 7.31.6.4.4
+    -
+  * - wcscat
+    -
+    - 7.31.4.4.1
+    -
+  * - wcschr
+    -
+    - 7.31.4.6.2
+    -
+  * - wcscmp
+    -
+    - 7.31.4.5.2
+    -
+  * - wcscoll
+    -
+    - 7.31.4.5.3
+    -
+  * - wcscpy
+    -
+    - 7.31.4.3.1
+    -
+  * - wcscspn
+    -
+    - 7.31.4.6.3
+    -
+  * - wcsftime
+    -
+    - 7.31.5.1
+    -
+  * - wcslen
+    -
+    - 7.31.4.7.1
+    -
+  * - wcsncat
+    -
+    - 7.31.4.4.2
+    -
+  * - wcsncmp
+    -
+    - 7.31.4.5.4
+    -
+  * - wcsncpy
+    -
+    - 7.31.4.3.2
+    -
+  * - wcspbrk
+    -
+    - 7.31.4.6.4
+    -
+  * - wcsrchr
+    -
+    - 7.31.4.6.5
+    -
+  * - wcsrtombs
+    -
+    - 7.31.6.5.3
+    -
+  * - wcsspn
+    -
+    - 7.31.4.6.6
+    -
+  * - wcsstr
+    -
+    - 7.31.4.6.7
+    -
+  * - wcstod
+    -
+    - 7.31.4.2.2
+    -
+  * - wcstod128
+    -
+    - 7.31.4.2.3
+    -
+  * - wcstod32
+    -
+    - 7.31.4.2.3
+    -
+  * - wcstod64
+    -
+    - 7.31.4.2.3
+    -
+  * - wcstof
+    -
+    - 7.31.4.2.2
+    -
+  * - wcstok
+    -
+    - 7.31.4.6.8
+    -
+  * - wcstol
+    -
+    - 7.31.4.2.4
+    -
+  * - wcstold
+    -
+    - 7.31.4.2.2
+    -
+  * - wcstoll
+    -
+    - 7.31.4.2.4
+    -
+  * - wcstoul
+    -
+    - 7.31.4.2.4
+    -
+  * - wcstoull
+    -
+    - 7.31.4.2.4
+    -
+  * - wcsxfrm
+    -
+    - 7.31.4.5.5
+    -
+  * - wctob
+    - |check|
+    - 7.31.6.2.2
+    -
+  * - wmemchr
+    -
+    - 7.31.4.6.9
+    -
+  * - wmemcmp
+    -
+    - 7.31.4.5.6
+    -
+  * - wmemcpy
+    -
+    - 7.31.4.3.3
+    -
+  * - wmemmove
+    -
+    - 7.31.4.3.4
+    -
+  * - wmemset
+    -
+    - 7.31.4.7.2
+    -
+  * - wprintf
+    -
+    - 7.31.2.12
+    -
+  * - wscanf
+    -
+    - 7.31.2.13
+    -
diff --git a/libc/docs/headers/wctype.rst b/libc/docs/headers/wctype.rst
new file mode 100644
index 0000000000000..48096c3e25804
--- /dev/null
+++ b/libc/docs/headers/wctype.rst
@@ -0,0 +1,86 @@
+.. include:: ../check.rst
+
+========
+wctype.h
+========
+
+Functions
+=========
+
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
+
+  * - Function
+    - Implemented
+    - C23 Standard Section
+    - POSIX.1-2024 Standard Section
+  * - iswalnum
+    -
+    - 7.32.2.1.1
+    -
+  * - iswalpha
+    -
+    - 7.32.2.1.2
+    -
+  * - iswblank
+    -
+    - 7.32.2.1.4
+    -
+  * - iswctype
+    -
+    - 7.32.2.2.1
+    -
+  * - iswdigit
+    -
+    - 7.32.2.1.5
+    -
+  * - iswgraph
+    -
+    - 7.32.2.1.6
+    -
+  * - iswlower
+    -
+    - 7.32.2.1.7
+    -
+  * - iswprint
+    -
+    - 7.32.2.1.8
+    -
+  * - iswpunct
+    -
+    - 7.32.2.1.9
+    -
+  * - iswspace
+    -
+    - 7.32.2.1.10
+    -
+  * - iswupper
+    -
+    - 7.32.2.1.11
+    -
+  * - iswxdigit
+    -
+    - 7.32.2.1.12
+    -
+  * - towctrans
+    -
+    - 7.32.3.2.1
+    -
+  * - towlower
+    -
+    - 7.32.3.1.1
+    -
+  * - towupper
+    -
+    - 7.32.3.1.2
+    -
+  * - wctrans
+    -
+    - 7.32.3.2.2
+    -
+  * - wctype
+    -
+    - 7.32.2.2.2
+    -
diff --git a/libc/docs/index.rst b/libc/docs/index.rst
index 6f759aa215b62..a17717aff9f78 100644
--- a/libc/docs/index.rst
+++ b/libc/docs/index.rst
@@ -40,6 +40,16 @@ and MacOS have limited support (may be broken).  The Fuchsia platform is
 slowly replacing functions from its bundled libc with functions from this
 project.
 
+LLVM-libc does not guarantee backward compatibility with operating systems that have reached their EOL.
+Compatibility patches for obsolete operating systems will not be accepted.
+
+For Linux, we support kernel versions as listed on `kernel.org <https://kernel.org/>`_, including
+``longterm`` (not past EOL date), ``stable``, and ``mainline`` versions. We actively adopt new features
+from ``linux-next``.
+
+For Windows, we plan to support products within their lifecycle. Please refer to 
+`Search Product and Services Lifecycle Information <https://learn.microsoft.com/en-us/lifecycle/products/?products=windows>`_ for more information.
+
 ABI Compatibility
 =================
 
@@ -64,19 +74,8 @@ stages there is no ABI stability in any form.
    :caption: Status
 
    compiler_support
-   date_and_time
-   math/index.rst
-   strings
-   stdio
-   stdbit
-   fenv
-   libc_search
+   headers/index.rst
    c23
-   ctype
-   complex
-   signal
-   threads
-   setjmp
 
 .. toctree::
    :hidden:
diff --git a/libc/docs/setjmp.rst b/libc/docs/setjmp.rst
deleted file mode 100644
index d9188dfe1d5e4..0000000000000
--- a/libc/docs/setjmp.rst
+++ /dev/null
@@ -1,16 +0,0 @@
-.. include:: check.rst
-
-setjmp.h Functions
-==================
-
-.. list-table::
-  :widths: auto
-  :align: center
-  :header-rows: 1
-
-  * - Function
-    - Implemented
-    - Standard
-  * - longjmp
-    - |check|
-    - 7.13.2.1
diff --git a/libc/docs/signal.rst b/libc/docs/signal.rst
deleted file mode 100644
index d1a7cb6095602..0000000000000
--- a/libc/docs/signal.rst
+++ /dev/null
@@ -1,207 +0,0 @@
-.. include:: check.rst
-
-========
-signal.h
-========
-
-Macros
-======
-
-.. list-table::
-  :widths: auto
-  :align: center
-  :header-rows: 1
-
-  * - Function
-    - Implemented
-    - C23 Standard Section
-    - POSIX.1-2017 Standard Section
-  * - SIGABRT
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGALRM
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGBUS
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGCHLD
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGCONT
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGFPE
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGHUP
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGILL
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGINT
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGKILL
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGPIPE
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGPOLL
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGPROF
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGQUIT
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGRTMAX
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGRTMIN
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGSEGV
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGSTOP
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGSYS
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGTERM
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGTRAP
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGTSTP
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGTTIN
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGTTOU
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGURG
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGUSR1
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGUSR2
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGVTALRM
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGXCPU
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIGXFSZ
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIG_DFL
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIG_ERR
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIG_HOLD
-    -
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-  * - SIG_IGN
-    - |check|
-    - 7.14.3
-    - https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html
-
-Functions
-=========
-
-.. list-table::
-  :widths: auto
-  :align: center
-  :header-rows: 1
-
-  * - Function
-    - Implemented
-    - C23 Standard Section
-    - POSIX.1-2017 Standard Section
-  * - kill
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/kill.html
-  * - raise
-    - |check|
-    - 7.14.2.1
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/raise.html
-  * - sigaction
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaction.html
-  * - sigaddset
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaddset.html
-  * - sigaltstack
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaltstack.html
-  * - sigdelset
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigdelset.html
-  * - sigemptyset
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigemptyset.html
-  * - sigfillset
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigfillset.html
-  * - signal
-    - |check|
-    - 7.14.1.1
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/signal.html
-  * - sigprocmask
-    - |check|
-    -
-    - https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigprocmask.html
diff --git a/libc/docs/strings.rst b/libc/docs/strings.rst
deleted file mode 100644
index 427644c407ae8..0000000000000
--- a/libc/docs/strings.rst
+++ /dev/null
@@ -1,166 +0,0 @@
-================
-String Functions
-================
-
-.. include:: check.rst
-
----------------
-Source location
----------------
-
--   The main source for string functions is located at:
-    ``libc/src/string``.
-
--   The source for string conversion functions is located at:
-    ``libc/src/stdlib`` and
-    ``libc/src/__support``.
-
--   The tests are located at:
-    ``libc/test/src/string``,
-    ``libc/test/src/stdlib``, and
-    ``libc/test/src/__support``
-    respectively.
-
----------------------
-Implementation Status
----------------------
-
-Primary memory functions
-========================
-
-.. TODO(gchatelet): add details about the memory functions.
-
-
-=============  =========
-Function Name  Available
-=============  =========
-bzero          |check|
-bcmp           |check|
-bcopy          |check|
-memcpy         |check|
-memset         |check|
-memcmp         |check|
-memmove        |check|
-=============  =========
-
-
-Other Raw Memory Functions
-==========================
-
-=============  =========
-Function Name  Available
-=============  =========
-memchr         |check|
-memrchr        |check|
-memccpy        |check|
-mempcpy        |check|
-=============  =========
-
-String Memory Functions
-=======================
-
-=============  =========
-Function Name  Available
-=============  =========
-stpcpy         |check|
-stpncpy        |check|
-strcpy         |check|
-strncpy        |check|
-strcat         |check|
-strncat        |check|
-strdup         |check|
-strndup        |check|
-=============  =========
-
-String Examination Functions
-============================
-
-=============  =========
-Function Name  Available
-=============  =========
-strlen         |check|
-strnlen        |check|
-strcmp         |check|
-strncmp        |check|
-strchr         |check|
-strrchr        |check|
-strspn         |check|
-strcspn        |check|
-strpbrk        |check|
-strstr         |check|
-strtok         |check|
-strtok_r       |check|
-=============  =========
-
-String Conversion Functions
-============================
-
-These functions are not in strings.h, but are still primarily string
-functions, and are therefore tracked along with the rest of the string
-functions.
-
-The String to float functions were implemented using the Eisel-Lemire algorithm
-(read more about the algorithm here: `The Eisel-Lemire ParseNumberF64 Algorithm
-<https://nigeltao.github.io/blog/2020/eisel-lemire.html>`_). This improved
-the performance of string to float and double, and allowed it to complete this
-comprehensive test 15% faster than glibc: `Parse Number FXX Test Data
-<https://github.com/nigeltao/parse-number-fxx-test-data>`_. The test was done
-with LLVM-libc built on 2022-04-14 and Debian GLibc version 2.33-6. The targets
-``libc_str_to_float_comparison_test`` and
-``libc_system_str_to_float_comparison_test`` were built and run on the test data
-10 times each, skipping the first run since it was an outlier.
-
-
-=============  =========
-Function Name  Available
-=============  =========
-atof           |check|
-atoi           |check|
-atol           |check|
-atoll          |check|
-strtol         |check|
-strtoll        |check|
-strtoul        |check|
-strtoull       |check|
-strtof         |check|
-strtod         |check|
-strtold        |check|
-strtoimax      |check|
-strtoumax      |check|
-=============  =========
-
-String Error Functions
-======================
-
-=============  =========
-Function Name  Available
-=============  =========
-strerror       |check|
-strerror_r     |check|
-=============  =========
-
-Localized String Functions
-==========================
-
-These functions require locale.h, and will be finished when locale support is
-implemented in LLVM-libc.
-
-=============  =========
-Function Name  Available
-=============  =========
-strcoll        Partially
-strxfrm        Partially
-=============  =========
-
----------------------------
-\<name\>_s String Functions
----------------------------
-
-Many String functions have an equivalent _s version, which is intended to be
-more secure and safe than the previous standard. These functions add runtime
-error detection and overflow protection. While they can be seen as an
-improvement, adoption remains relatively low among users. In addition, they are
-being considered for removal, see
-`Field Experience With Annex K — Bounds Checking Interfaces
-<http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1967.htm>`_. For these reasons,
-there is no ongoing work to implement them.
diff --git a/libc/fuzzing/string/CMakeLists.txt b/libc/fuzzing/string/CMakeLists.txt
index 9dd4fceee3b59..efda80b59c951 100644
--- a/libc/fuzzing/string/CMakeLists.txt
+++ b/libc/fuzzing/string/CMakeLists.txt
@@ -38,5 +38,5 @@ add_libc_fuzzer(
   SRCS
     bcmp_fuzz.cpp
   DEPENDS
-    libc.src.string.bcmp
+    libc.src.strings.bcmp
 )
diff --git a/libc/fuzzing/string/bcmp_fuzz.cpp b/libc/fuzzing/string/bcmp_fuzz.cpp
index 65949dfdf0c56..710d115b0ae80 100644
--- a/libc/fuzzing/string/bcmp_fuzz.cpp
+++ b/libc/fuzzing/string/bcmp_fuzz.cpp
@@ -9,7 +9,7 @@
 /// Fuzzing test for llvm-libc bcmp implementation.
 ///
 //===----------------------------------------------------------------------===//
-#include "src/string/bcmp.h"
+#include "src/strings/bcmp.h"
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 93da271f5e040..5eb311f4bb229 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -135,10 +135,18 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.unistd_macros
 )
 
+if (WIN32)
+  set(windows_addtional_time_macros libc.include.llvm-libc-macros.windows.time_macros_ext)
+else()
+  set(windows_addtional_time_macros "")
+endif()
+
 add_proxy_header_library(
   time_macros
   HDRS
     time_macros.h
+  DEPENDS
+    ${windows_addtional_time_macros}
   FULL_BUILD_DEPENDS
     libc.include.time
     libc.include.llvm-libc-macros.time_macros
diff --git a/libc/hdr/fcntl_overlay.h b/libc/hdr/fcntl_overlay.h
index c1cc98b0ebb2c..17ae78b3d0eca 100644
--- a/libc/hdr/fcntl_overlay.h
+++ b/libc/hdr/fcntl_overlay.h
@@ -20,6 +20,11 @@
 // `__USE_FORTIFY_LEVEL`, which will be temporarily disabled
 // with `_FORTIFY_SOURCE`.
 
+#ifdef _FORTIFY_SOURCE
+#define LIBC_OLD_FORTIFY_SOURCE _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+
 #ifdef __USE_FORTIFY_LEVEL
 #define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL
 #undef __USE_FORTIFY_LEVEL
@@ -28,6 +33,11 @@
 
 #include <fcntl.h>
 
+#ifdef LIBC_OLD_FORTIFY_SOURCE
+#define _FORTIFY_SOURCE LIBC_OLD_FORTIFY_SOURCE
+#undef LIBC_OLD_FORTIFY_SOURCE
+#endif
+
 #ifdef LIBC_OLD_USE_FORTIFY_LEVEL
 #undef __USE_FORTIFY_LEVEL
 #define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL
diff --git a/libc/hdr/func/free.h b/libc/hdr/func/free.h
index b1190a777da32..316556b21e3b0 100644
--- a/libc/hdr/func/free.h
+++ b/libc/hdr/func/free.h
@@ -10,7 +10,8 @@
 #define LLVM_LIBC_HDR_FUNC_FREE_H
 
 #ifdef LIBC_FULL_BUILD
-extern "C" void free(void *);
+
+extern "C" void free(void *) noexcept;
 
 #else // Overlay mode
 
diff --git a/libc/hdr/func/malloc.h b/libc/hdr/func/malloc.h
index b395f41f2bce2..8281021f79969 100644
--- a/libc/hdr/func/malloc.h
+++ b/libc/hdr/func/malloc.h
@@ -10,8 +10,10 @@
 #define LLVM_LIBC_HDR_FUNC_MALLOC_H
 
 #ifdef LIBC_FULL_BUILD
+
 #include "hdr/types/size_t.h"
-extern "C" void *malloc(size_t);
+
+extern "C" void *malloc(size_t) noexcept;
 
 #else // Overlay mode
 
diff --git a/libc/hdr/func/realloc.h b/libc/hdr/func/realloc.h
index 0096045e8330b..ecb29541fe34a 100644
--- a/libc/hdr/func/realloc.h
+++ b/libc/hdr/func/realloc.h
@@ -10,8 +10,10 @@
 #define LLVM_LIBC_HDR_FUNC_REALLOC_H
 
 #ifdef LIBC_FULL_BUILD
+
 #include "hdr/types/size_t.h"
-extern "C" void *realloc(void *ptr, size_t new_size);
+
+extern "C" void *realloc(void *ptr, size_t new_size) noexcept;
 
 #else // Overlay mode
 
diff --git a/libc/hdr/time_macros.h b/libc/hdr/time_macros.h
index dc36fe66f7a80..4488a24848c35 100644
--- a/libc/hdr/time_macros.h
+++ b/libc/hdr/time_macros.h
@@ -19,4 +19,10 @@
 
 #endif // LLVM_LIBC_FULL_BUILD
 
+// TODO: For now, on windows, let's always include the extension header.
+// We will need to decide how to export this header.
+#ifdef _WIN32
+#include "include/llvm-libc-macros/windows/time-macros-ext.h"
+#endif // _WIN32
+
 #endif // LLVM_LIBC_HDR_TIME_MACROS_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index c4d14e6f89313..68a0e9603f975 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -51,6 +51,8 @@ add_proxy_header_library(
   struct_flock
   HDRS
     struct_flock.h
+  DEPENDS
+    libc.hdr.fcntl_overlay
   FULL_BUILD_DEPENDS
     libc.include.llvm-libc-types.struct_flock
 )
@@ -59,6 +61,8 @@ add_proxy_header_library(
   struct_flock64
   HDRS
     struct_flock64.h
+  DEPENDS
+    libc.hdr.fcntl_overlay
   FULL_BUILD_DEPENDS
     libc.include.llvm-libc-types.struct_flock64
 )
@@ -67,6 +71,8 @@ add_proxy_header_library(
   struct_f_owner_ex
   HDRS
     struct_f_owner_ex.h
+  DEPENDS
+    libc.hdr.fcntl_overlay
   FULL_BUILD_DEPENDS
     libc.include.llvm-libc-types.struct_f_owner_ex
 )
diff --git a/libc/hdr/types/clockid_t.h b/libc/hdr/types/clockid_t.h
index 333342072a2ff..729e580aba438 100644
--- a/libc/hdr/types/clockid_t.h
+++ b/libc/hdr/types/clockid_t.h
@@ -9,7 +9,8 @@
 #ifndef LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
 #define LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
 
-#ifdef LIBC_FULL_BUILD
+// TODO: we will need to decide how to export extension to windows.
+#if defined(LIBC_FULL_BUILD) || defined(_WIN32)
 
 #include "include/llvm-libc-types/clockid_t.h"
 
diff --git a/libc/hdr/types/struct_f_owner_ex.h b/libc/hdr/types/struct_f_owner_ex.h
index 49985115ae4bb..6e37cea6df84d 100644
--- a/libc/hdr/types/struct_f_owner_ex.h
+++ b/libc/hdr/types/struct_f_owner_ex.h
@@ -14,7 +14,7 @@
 
 #else
 
-#include <fcntl.h>
+#include "hdr/fcntl_overlay.h"
 
 #endif // LIBC_FULL_BUILD
 
diff --git a/libc/hdr/types/struct_flock.h b/libc/hdr/types/struct_flock.h
index a552b91c432b3..6a6c928e6fa6d 100644
--- a/libc/hdr/types/struct_flock.h
+++ b/libc/hdr/types/struct_flock.h
@@ -14,7 +14,7 @@
 
 #else
 
-#include <fcntl.h>
+#include "hdr/fcntl_overlay.h"
 
 #endif // LIBC_FULL_BUILD
 
diff --git a/libc/hdr/types/struct_flock64.h b/libc/hdr/types/struct_flock64.h
index 84fe67816c337..fcfda59479a29 100644
--- a/libc/hdr/types/struct_flock64.h
+++ b/libc/hdr/types/struct_flock64.h
@@ -14,7 +14,7 @@
 
 #else
 
-#include <fcntl.h>
+#include "hdr/fcntl_overlay.h"
 
 #endif // LIBC_FULL_BUILD
 
diff --git a/libc/hdrgen/CMakeLists.txt b/libc/hdrgen/CMakeLists.txt
new file mode 100644
index 0000000000000..8ebde4e3e4588
--- /dev/null
+++ b/libc/hdrgen/CMakeLists.txt
@@ -0,0 +1,17 @@
+if(LLVM_LIBC_FULL_BUILD)
+  enable_testing()
+
+  set(NEWHDGEN_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+  set(TEST_OUTPUT_DIR ${CMAKE_BINARY_DIR}/hdrgen/output)
+
+  add_test(
+    NAME hdrgen_integration_test
+    COMMAND python3 ${NEWHDGEN_TESTS_DIR}/test_integration.py --output_dir ${TEST_OUTPUT_DIR}
+  )
+
+  add_custom_target(check-hdrgen
+    COMMAND ${CMAKE_CTEST_COMMAND} -R hdrgen_integration_test --output-on-failure
+  )
+
+  message(STATUS "Integration test for hdrgen added.")
+endif()
diff --git a/libc/newhdrgen/class_implementation/classes/enumeration.py b/libc/hdrgen/class_implementation/classes/enumeration.py
similarity index 100%
rename from libc/newhdrgen/class_implementation/classes/enumeration.py
rename to libc/hdrgen/class_implementation/classes/enumeration.py
diff --git a/libc/newhdrgen/class_implementation/classes/function.py b/libc/hdrgen/class_implementation/classes/function.py
similarity index 100%
rename from libc/newhdrgen/class_implementation/classes/function.py
rename to libc/hdrgen/class_implementation/classes/function.py
diff --git a/libc/newhdrgen/class_implementation/classes/macro.py b/libc/hdrgen/class_implementation/classes/macro.py
similarity index 100%
rename from libc/newhdrgen/class_implementation/classes/macro.py
rename to libc/hdrgen/class_implementation/classes/macro.py
diff --git a/libc/newhdrgen/class_implementation/classes/object.py b/libc/hdrgen/class_implementation/classes/object.py
similarity index 100%
rename from libc/newhdrgen/class_implementation/classes/object.py
rename to libc/hdrgen/class_implementation/classes/object.py
diff --git a/libc/newhdrgen/class_implementation/classes/type.py b/libc/hdrgen/class_implementation/classes/type.py
similarity index 100%
rename from libc/newhdrgen/class_implementation/classes/type.py
rename to libc/hdrgen/class_implementation/classes/type.py
diff --git a/libc/newhdrgen/gpu_headers.py b/libc/hdrgen/gpu_headers.py
similarity index 100%
rename from libc/newhdrgen/gpu_headers.py
rename to libc/hdrgen/gpu_headers.py
diff --git a/libc/newhdrgen/header.py b/libc/hdrgen/header.py
similarity index 100%
rename from libc/newhdrgen/header.py
rename to libc/hdrgen/header.py
diff --git a/libc/newhdrgen/tests/expected_output/test_header.h b/libc/hdrgen/tests/expected_output/test_header.h
similarity index 100%
rename from libc/newhdrgen/tests/expected_output/test_header.h
rename to libc/hdrgen/tests/expected_output/test_header.h
diff --git a/libc/newhdrgen/tests/input/test_small.h.def b/libc/hdrgen/tests/input/test_small.h.def
similarity index 100%
rename from libc/newhdrgen/tests/input/test_small.h.def
rename to libc/hdrgen/tests/input/test_small.h.def
diff --git a/libc/newhdrgen/tests/input/test_small.yaml b/libc/hdrgen/tests/input/test_small.yaml
similarity index 100%
rename from libc/newhdrgen/tests/input/test_small.yaml
rename to libc/hdrgen/tests/input/test_small.yaml
diff --git a/libc/newhdrgen/tests/test_integration.py b/libc/hdrgen/tests/test_integration.py
similarity index 85%
rename from libc/newhdrgen/tests/test_integration.py
rename to libc/hdrgen/tests/test_integration.py
index 33353785af233..8ea6d8a708073 100644
--- a/libc/newhdrgen/tests/test_integration.py
+++ b/libc/hdrgen/tests/test_integration.py
@@ -9,7 +9,7 @@
 class TestHeaderGenIntegration(unittest.TestCase):
     def setUp(self):
         self.output_dir = Path(
-            args.output_dir if args.output_dir else "libc/newhdrgen/tests/output"
+            args.output_dir if args.output_dir else "libc/hdrgen/tests/output"
         )
 
         self.maxDiff = None
@@ -21,7 +21,7 @@ def run_script(self, yaml_file, h_def_file, output_dir, entry_points):
         h_def_file = self.source_dir / h_def_file
         command = [
             "python3",
-            str(self.source_dir / "libc/newhdrgen/yaml_to_classes.py"),
+            str(self.source_dir / "libc/hdrgen/yaml_to_classes.py"),
             str(yaml_file),
             "--h_def_file",
             str(h_def_file),
@@ -51,10 +51,10 @@ def compare_files(self, generated_file, expected_file):
         self.assertEqual(gen_content, exp_content)
 
     def test_generate_header(self):
-        yaml_file = "libc/newhdrgen/tests/input/test_small.yaml"
-        h_def_file = "libc/newhdrgen/tests/input/test_small.h.def"
+        yaml_file = "libc/hdrgen/tests/input/test_small.yaml"
+        h_def_file = "libc/hdrgen/tests/input/test_small.h.def"
         expected_output_file = (
-            self.source_dir / "libc/newhdrgen/tests/expected_output/test_header.h"
+            self.source_dir / "libc/hdrgen/tests/expected_output/test_header.h"
         )
         output_file = self.output_dir / "test_small.h"
         entry_points = {"func_b", "func_a", "func_c", "func_d", "func_e"}
diff --git a/libc/newhdrgen/yaml/arpa/inet.yaml b/libc/hdrgen/yaml/arpa/inet.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/arpa/inet.yaml
rename to libc/hdrgen/yaml/arpa/inet.yaml
diff --git a/libc/newhdrgen/yaml/assert.yaml b/libc/hdrgen/yaml/assert.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/assert.yaml
rename to libc/hdrgen/yaml/assert.yaml
diff --git a/libc/newhdrgen/yaml/ctype.yaml b/libc/hdrgen/yaml/ctype.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/ctype.yaml
rename to libc/hdrgen/yaml/ctype.yaml
diff --git a/libc/newhdrgen/yaml/dirent.yaml b/libc/hdrgen/yaml/dirent.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/dirent.yaml
rename to libc/hdrgen/yaml/dirent.yaml
diff --git a/libc/newhdrgen/yaml/dlfcn.yaml b/libc/hdrgen/yaml/dlfcn.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/dlfcn.yaml
rename to libc/hdrgen/yaml/dlfcn.yaml
diff --git a/libc/newhdrgen/yaml/elf.yaml b/libc/hdrgen/yaml/elf.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/elf.yaml
rename to libc/hdrgen/yaml/elf.yaml
diff --git a/libc/newhdrgen/yaml/errno.yaml b/libc/hdrgen/yaml/errno.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/errno.yaml
rename to libc/hdrgen/yaml/errno.yaml
diff --git a/libc/newhdrgen/yaml/fcntl.yaml b/libc/hdrgen/yaml/fcntl.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/fcntl.yaml
rename to libc/hdrgen/yaml/fcntl.yaml
diff --git a/libc/newhdrgen/yaml/features.yaml b/libc/hdrgen/yaml/features.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/features.yaml
rename to libc/hdrgen/yaml/features.yaml
diff --git a/libc/newhdrgen/yaml/fenv.yaml b/libc/hdrgen/yaml/fenv.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/fenv.yaml
rename to libc/hdrgen/yaml/fenv.yaml
diff --git a/libc/newhdrgen/yaml/float.yaml b/libc/hdrgen/yaml/float.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/float.yaml
rename to libc/hdrgen/yaml/float.yaml
diff --git a/libc/newhdrgen/yaml/gpu/rpc.yaml b/libc/hdrgen/yaml/gpu/rpc.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/gpu/rpc.yaml
rename to libc/hdrgen/yaml/gpu/rpc.yaml
diff --git a/libc/newhdrgen/yaml/inttypes.yaml b/libc/hdrgen/yaml/inttypes.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/inttypes.yaml
rename to libc/hdrgen/yaml/inttypes.yaml
diff --git a/libc/newhdrgen/yaml/limits.yaml b/libc/hdrgen/yaml/limits.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/limits.yaml
rename to libc/hdrgen/yaml/limits.yaml
diff --git a/libc/newhdrgen/yaml/link.yaml b/libc/hdrgen/yaml/link.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/link.yaml
rename to libc/hdrgen/yaml/link.yaml
diff --git a/libc/newhdrgen/yaml/locale.yaml b/libc/hdrgen/yaml/locale.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/locale.yaml
rename to libc/hdrgen/yaml/locale.yaml
diff --git a/libc/newhdrgen/yaml/malloc.yaml b/libc/hdrgen/yaml/malloc.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/malloc.yaml
rename to libc/hdrgen/yaml/malloc.yaml
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/hdrgen/yaml/math.yaml
similarity index 99%
rename from libc/newhdrgen/yaml/math.yaml
rename to libc/hdrgen/yaml/math.yaml
index e09f0929e45f8..00efc34789667 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/hdrgen/yaml/math.yaml
@@ -2339,6 +2339,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: sinf16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: sinhf
     standards:
       - stdc
diff --git a/libc/newhdrgen/yaml/pthread.yaml b/libc/hdrgen/yaml/pthread.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/pthread.yaml
rename to libc/hdrgen/yaml/pthread.yaml
diff --git a/libc/newhdrgen/yaml/sched.yaml b/libc/hdrgen/yaml/sched.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sched.yaml
rename to libc/hdrgen/yaml/sched.yaml
diff --git a/libc/newhdrgen/yaml/search.yaml b/libc/hdrgen/yaml/search.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/search.yaml
rename to libc/hdrgen/yaml/search.yaml
diff --git a/libc/newhdrgen/yaml/setjmp.yaml b/libc/hdrgen/yaml/setjmp.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/setjmp.yaml
rename to libc/hdrgen/yaml/setjmp.yaml
diff --git a/libc/newhdrgen/yaml/signal.yaml b/libc/hdrgen/yaml/signal.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/signal.yaml
rename to libc/hdrgen/yaml/signal.yaml
diff --git a/libc/newhdrgen/yaml/spawn.yaml b/libc/hdrgen/yaml/spawn.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/spawn.yaml
rename to libc/hdrgen/yaml/spawn.yaml
diff --git a/libc/newhdrgen/yaml/stdbit.yaml b/libc/hdrgen/yaml/stdbit.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdbit.yaml
rename to libc/hdrgen/yaml/stdbit.yaml
diff --git a/libc/newhdrgen/yaml/stdckdint.yaml b/libc/hdrgen/yaml/stdckdint.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdckdint.yaml
rename to libc/hdrgen/yaml/stdckdint.yaml
diff --git a/libc/newhdrgen/yaml/stdfix.yaml b/libc/hdrgen/yaml/stdfix.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdfix.yaml
rename to libc/hdrgen/yaml/stdfix.yaml
diff --git a/libc/newhdrgen/yaml/stdint.yaml b/libc/hdrgen/yaml/stdint.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdint.yaml
rename to libc/hdrgen/yaml/stdint.yaml
diff --git a/libc/newhdrgen/yaml/stdio.yaml b/libc/hdrgen/yaml/stdio.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdio.yaml
rename to libc/hdrgen/yaml/stdio.yaml
diff --git a/libc/newhdrgen/yaml/stdlib.yaml b/libc/hdrgen/yaml/stdlib.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/stdlib.yaml
rename to libc/hdrgen/yaml/stdlib.yaml
diff --git a/libc/newhdrgen/yaml/string.yaml b/libc/hdrgen/yaml/string.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/string.yaml
rename to libc/hdrgen/yaml/string.yaml
diff --git a/libc/newhdrgen/yaml/strings.yaml b/libc/hdrgen/yaml/strings.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/strings.yaml
rename to libc/hdrgen/yaml/strings.yaml
diff --git a/libc/newhdrgen/yaml/sys/auxv.yaml b/libc/hdrgen/yaml/sys/auxv.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/auxv.yaml
rename to libc/hdrgen/yaml/sys/auxv.yaml
diff --git a/libc/newhdrgen/yaml/sys/epoll.yaml b/libc/hdrgen/yaml/sys/epoll.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/epoll.yaml
rename to libc/hdrgen/yaml/sys/epoll.yaml
diff --git a/libc/newhdrgen/yaml/sys/ioctl.yaml b/libc/hdrgen/yaml/sys/ioctl.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/ioctl.yaml
rename to libc/hdrgen/yaml/sys/ioctl.yaml
diff --git a/libc/newhdrgen/yaml/sys/mman.yaml b/libc/hdrgen/yaml/sys/mman.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/mman.yaml
rename to libc/hdrgen/yaml/sys/mman.yaml
diff --git a/libc/newhdrgen/yaml/sys/prctl.yaml b/libc/hdrgen/yaml/sys/prctl.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/prctl.yaml
rename to libc/hdrgen/yaml/sys/prctl.yaml
diff --git a/libc/newhdrgen/yaml/sys/random.yaml b/libc/hdrgen/yaml/sys/random.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/random.yaml
rename to libc/hdrgen/yaml/sys/random.yaml
diff --git a/libc/newhdrgen/yaml/sys/resource.yaml b/libc/hdrgen/yaml/sys/resource.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/resource.yaml
rename to libc/hdrgen/yaml/sys/resource.yaml
diff --git a/libc/newhdrgen/yaml/sys/select.yaml b/libc/hdrgen/yaml/sys/select.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/select.yaml
rename to libc/hdrgen/yaml/sys/select.yaml
diff --git a/libc/newhdrgen/yaml/sys/sendfile.yaml b/libc/hdrgen/yaml/sys/sendfile.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/sendfile.yaml
rename to libc/hdrgen/yaml/sys/sendfile.yaml
diff --git a/libc/newhdrgen/yaml/sys/socket.yaml b/libc/hdrgen/yaml/sys/socket.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/socket.yaml
rename to libc/hdrgen/yaml/sys/socket.yaml
diff --git a/libc/newhdrgen/yaml/sys/stat.yaml b/libc/hdrgen/yaml/sys/stat.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/stat.yaml
rename to libc/hdrgen/yaml/sys/stat.yaml
diff --git a/libc/newhdrgen/yaml/sys/statvfs.yaml b/libc/hdrgen/yaml/sys/statvfs.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/statvfs.yaml
rename to libc/hdrgen/yaml/sys/statvfs.yaml
diff --git a/libc/newhdrgen/yaml/sys/syscall.yaml b/libc/hdrgen/yaml/sys/syscall.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/syscall.yaml
rename to libc/hdrgen/yaml/sys/syscall.yaml
diff --git a/libc/newhdrgen/yaml/sys/time.yaml b/libc/hdrgen/yaml/sys/time.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/time.yaml
rename to libc/hdrgen/yaml/sys/time.yaml
diff --git a/libc/newhdrgen/yaml/sys/types.yaml b/libc/hdrgen/yaml/sys/types.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/types.yaml
rename to libc/hdrgen/yaml/sys/types.yaml
diff --git a/libc/newhdrgen/yaml/sys/utsname.yaml b/libc/hdrgen/yaml/sys/utsname.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/utsname.yaml
rename to libc/hdrgen/yaml/sys/utsname.yaml
diff --git a/libc/newhdrgen/yaml/sys/wait.yaml b/libc/hdrgen/yaml/sys/wait.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/sys/wait.yaml
rename to libc/hdrgen/yaml/sys/wait.yaml
diff --git a/libc/newhdrgen/yaml/termios.yaml b/libc/hdrgen/yaml/termios.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/termios.yaml
rename to libc/hdrgen/yaml/termios.yaml
diff --git a/libc/newhdrgen/yaml/threads.yaml b/libc/hdrgen/yaml/threads.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/threads.yaml
rename to libc/hdrgen/yaml/threads.yaml
diff --git a/libc/newhdrgen/yaml/time.yaml b/libc/hdrgen/yaml/time.yaml
similarity index 93%
rename from libc/newhdrgen/yaml/time.yaml
rename to libc/hdrgen/yaml/time.yaml
index 69b40bef3160d..3f745e5ee3386 100644
--- a/libc/newhdrgen/yaml/time.yaml
+++ b/libc/hdrgen/yaml/time.yaml
@@ -96,3 +96,10 @@ functions:
     return_type: time_t
     arguments:
       - type: time_t *
+  - name: timespec_get
+    standard:
+      - stdc
+    return_type: int
+    arguments:
+      - type: struct timespec *
+      - type: int
diff --git a/libc/newhdrgen/yaml/uchar.yaml b/libc/hdrgen/yaml/uchar.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/uchar.yaml
rename to libc/hdrgen/yaml/uchar.yaml
diff --git a/libc/newhdrgen/yaml/unistd.yaml b/libc/hdrgen/yaml/unistd.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/unistd.yaml
rename to libc/hdrgen/yaml/unistd.yaml
diff --git a/libc/newhdrgen/yaml/wchar.yaml b/libc/hdrgen/yaml/wchar.yaml
similarity index 100%
rename from libc/newhdrgen/yaml/wchar.yaml
rename to libc/hdrgen/yaml/wchar.yaml
diff --git a/libc/newhdrgen/yaml_functions_sorted.py b/libc/hdrgen/yaml_functions_sorted.py
similarity index 100%
rename from libc/newhdrgen/yaml_functions_sorted.py
rename to libc/hdrgen/yaml_functions_sorted.py
diff --git a/libc/newhdrgen/yaml_to_classes.py b/libc/hdrgen/yaml_to_classes.py
similarity index 100%
rename from libc/newhdrgen/yaml_to_classes.py
rename to libc/hdrgen/yaml_to_classes.py
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 899a93ad72d4c..18ce8e22d1a0a 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -17,30 +17,22 @@ add_header(
     __llvm-libc-common.h
 )
 
+# TODO: Can we simplify this macro expansion?
+# https://github.com/llvm/llvm-project/issues/117254
 macro(add_header_macro TARGET_NAME YAML_FILE DEF_FILE GEN_HDR DEPENDS)
-  if (LIBC_USE_NEW_HEADER_GEN)
-    add_gen_header2(
-      ${TARGET_NAME}
-      YAML_FILE ${YAML_FILE}
-      DEF_FILE ${DEF_FILE}
-      GEN_HDR ${GEN_HDR}
-      ${DEPENDS}
-      ${ARGN}
-    )
-  else()
-    add_gen_header(
-      ${TARGET_NAME}
-      DEF_FILE ${DEF_FILE}
-      GEN_HDR ${GEN_HDR}
-      ${DEPENDS}
-      ${ARGN}
-    )
-  endif()
+  add_gen_header(
+    ${TARGET_NAME}
+    YAML_FILE ${YAML_FILE}
+    DEF_FILE ${DEF_FILE}
+    GEN_HDR ${GEN_HDR}
+    ${DEPENDS}
+    ${ARGN}
+  )
 endmacro()
 
 add_header_macro(
   ctype
-  ../libc/newhdrgen/yaml/ctype.yaml
+  ../libc/hdrgen/yaml/ctype.yaml
   ctype.h.def
   ctype.h
   DEPENDS
@@ -50,7 +42,7 @@ add_header_macro(
 
 add_header_macro(
   dirent
-  ../libc/newhdrgen/yaml/dirent.yaml
+  ../libc/hdrgen/yaml/dirent.yaml
   dirent.h.def
   dirent.h
   DEPENDS
@@ -62,7 +54,7 @@ add_header_macro(
 
 add_header_macro(
   fcntl
-  ../libc/newhdrgen/yaml/fcntl.yaml
+  ../libc/hdrgen/yaml/fcntl.yaml
   fcntl.h.def
   fcntl.h
   DEPENDS
@@ -78,7 +70,7 @@ add_header_macro(
 
 add_header_macro(
   dlfcn
-  ../libc/newhdrgen/yaml/dlfcn.yaml
+  ../libc/hdrgen/yaml/dlfcn.yaml
   dlfcn.h.def
   dlfcn.h
   DEPENDS
@@ -88,7 +80,7 @@ add_header_macro(
 
 add_header_macro(
   features
-  ../libc/newhdrgen/yaml/features.yaml
+  ../libc/hdrgen/yaml/features.yaml
   features.h.def
   features.h
   DEPENDS
@@ -98,7 +90,7 @@ add_header_macro(
 
 add_header_macro(
   fenv
-  ../libc/newhdrgen/yaml/fenv.yaml
+  ../libc/hdrgen/yaml/fenv.yaml
   fenv.h.def
   fenv.h
   DEPENDS
@@ -110,7 +102,7 @@ add_header_macro(
 
 add_header_macro(
   inttypes
-  ../libc/newhdrgen/yaml/inttypes.yaml
+  ../libc/hdrgen/yaml/inttypes.yaml
   inttypes.h.def
   inttypes.h
   DEPENDS
@@ -121,7 +113,7 @@ add_header_macro(
 
 add_header_macro(
   float
-  ../libc/newhdrgen/yaml/float.yaml
+  ../libc/hdrgen/yaml/float.yaml
   float.h.def
   float.h
   DEPENDS
@@ -130,7 +122,7 @@ add_header_macro(
 
 add_header_macro(
   stdint
-  ../libc/newhdrgen/yaml/stdint.yaml
+  ../libc/hdrgen/yaml/stdint.yaml
   stdint.h.def
   stdint.h
   DEPENDS
@@ -139,7 +131,7 @@ add_header_macro(
 
 add_header_macro(
   limits
-  ../libc/newhdrgen/yaml/limits.yaml
+  ../libc/hdrgen/yaml/limits.yaml
   limits.h.def
   limits.h
   DEPENDS
@@ -148,7 +140,7 @@ add_header_macro(
 
 add_header_macro(
   malloc
-  ../libc/newhdrgen/yaml/malloc.yaml
+  ../libc/hdrgen/yaml/malloc.yaml
   malloc.h.def
   malloc.h
   DEPENDS
@@ -158,7 +150,7 @@ add_header_macro(
 
 add_header_macro(
   math
-  ../libc/newhdrgen/yaml/math.yaml
+  ../libc/hdrgen/yaml/math.yaml
   math.h.def
   math.h
   DEPENDS
@@ -173,7 +165,7 @@ add_header_macro(
 
 add_header_macro(
   stdfix
-  ../libc/newhdrgen/yaml/stdfix.yaml
+  ../libc/hdrgen/yaml/stdfix.yaml
   stdfix.h.def
   stdfix.h
   DEPENDS
@@ -186,7 +178,7 @@ file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/arpa)
 
 add_header_macro(
   arpa_inet
-  ../libc/newhdrgen/yaml/arpa/inet.yaml
+  ../libc/hdrgen/yaml/arpa/inet.yaml
   arpa/inet.h.def
   arpa/inet.h
   DEPENDS
@@ -195,7 +187,7 @@ add_header_macro(
 
 add_header_macro(
   assert
-  ../libc/newhdrgen/yaml/assert.yaml
+  ../libc/hdrgen/yaml/assert.yaml
   assert.h.def
   assert.h
   DEPENDS
@@ -205,7 +197,7 @@ add_header_macro(
 
 add_header_macro(
   complex
-  ../libc/newhdrgen/yaml/complex.yaml
+  ../libc/hdrgen/yaml/complex.yaml
   complex.h.def
   complex.h
   DEPENDS
@@ -215,7 +207,7 @@ add_header_macro(
 
 add_header_macro(
   setjmp
-  ../libc/newhdrgen/yaml/setjmp.yaml
+  ../libc/hdrgen/yaml/setjmp.yaml
   setjmp.h.def
   setjmp.h
   DEPENDS
@@ -225,7 +217,7 @@ add_header_macro(
 
 add_header_macro(
   string
-  ../libc/newhdrgen/yaml/string.yaml
+  ../libc/hdrgen/yaml/string.yaml
   string.h.def
   string.h
   DEPENDS
@@ -236,7 +228,7 @@ add_header_macro(
 
 add_header_macro(
   strings
-  ../libc/newhdrgen/yaml/strings.yaml
+  ../libc/hdrgen/yaml/strings.yaml
   strings.h.def
   strings.h
   DEPENDS
@@ -246,7 +238,7 @@ add_header_macro(
 
 add_header_macro(
   search
-  ../libc/newhdrgen/yaml/search.yaml
+  ../libc/hdrgen/yaml/search.yaml
   search.h.def
   search.h
   DEPENDS
@@ -260,7 +252,7 @@ add_header_macro(
 
 add_header_macro(
   time
-  ../libc/newhdrgen/yaml/time.yaml
+  ../libc/hdrgen/yaml/time.yaml
   time.h.def
   time.h
   DEPENDS
@@ -276,7 +268,7 @@ add_header_macro(
 
 add_header_macro(
   threads
-  ../libc/newhdrgen/yaml/threads.yaml
+  ../libc/hdrgen/yaml/threads.yaml
   threads.h.def
   threads.h
   DEPENDS
@@ -293,7 +285,7 @@ add_header_macro(
 
 add_header_macro(
   errno
-  ../libc/newhdrgen/yaml/errno.yaml
+  ../libc/hdrgen/yaml/errno.yaml
   errno.h.def
   errno.h
   DEPENDS
@@ -303,7 +295,7 @@ add_header_macro(
 
 add_header_macro(
   signal
-  ../libc/newhdrgen/yaml/signal.yaml
+  ../libc/hdrgen/yaml/signal.yaml
   signal.h.def
   signal.h
   DEPENDS
@@ -319,7 +311,7 @@ add_header_macro(
 
 add_header_macro(
   stdbit
-  ../libc/newhdrgen/yaml/stdbit.yaml
+  ../libc/hdrgen/yaml/stdbit.yaml
   stdbit.h.def
   stdbit.h
   DEPENDS
@@ -329,7 +321,7 @@ add_header_macro(
 
 add_header_macro(
   stdckdint
-  ../libc/newhdrgen/yaml/stdckdint.yaml
+  ../libc/hdrgen/yaml/stdckdint.yaml
   stdckdint.h.def
   stdckdint.h
   DEPENDS
@@ -339,7 +331,7 @@ add_header_macro(
 
 add_header_macro(
   stdio
-  ../libc/newhdrgen/yaml/stdio.yaml
+  ../libc/hdrgen/yaml/stdio.yaml
   stdio.h.def
   stdio.h
   DEPENDS
@@ -355,7 +347,7 @@ add_header_macro(
 
 add_header_macro(
   stdlib
-  ../libc/newhdrgen/yaml/stdlib.yaml
+  ../libc/hdrgen/yaml/stdlib.yaml
   stdlib.h.def
   stdlib.h
   DEPENDS
@@ -374,7 +366,7 @@ add_header_macro(
 
 add_header_macro(
   unistd
-  ../libc/newhdrgen/yaml/unistd.yaml
+  ../libc/hdrgen/yaml/unistd.yaml
   unistd.h.def
   unistd.h
   DEPENDS
@@ -393,11 +385,11 @@ add_header_macro(
 
 add_header_macro(
   pthread
-  ../libc/newhdrgen/yaml/pthread.yaml
+  ../libc/hdrgen/yaml/pthread.yaml
   pthread.h.def
   pthread.h
   DEPENDS
-    .llvm_libc_common_h
+    .llvm-libc-macros.pthread_macros
     .llvm-libc-types.__atfork_callback_t
     .llvm-libc-types.__pthread_once_func_t
     .llvm-libc-types.__pthread_start_t
@@ -412,11 +404,12 @@ add_header_macro(
     .llvm-libc-types.pthread_rwlockattr_t
     .llvm-libc-types.pthread_spinlock_t
     .llvm-libc-types.pthread_t
+    .llvm_libc_common_h
 )
 
 add_header_macro(
   sched
-  ../libc/newhdrgen/yaml/sched.yaml
+  ../libc/hdrgen/yaml/sched.yaml
   sched.h.def
   sched.h
   DEPENDS
@@ -433,7 +426,7 @@ add_header_macro(
 
 add_header_macro(
   spawn
-  ../libc/newhdrgen/yaml/spawn.yaml
+  ../libc/hdrgen/yaml/spawn.yaml
   spawn.h.def
   spawn.h
   DEPENDS
@@ -446,7 +439,7 @@ add_header_macro(
 
 add_header_macro(
   link
-  ../libc/newhdrgen/yaml/link.yaml
+  ../libc/hdrgen/yaml/link.yaml
   link.h.def
   link.h
   DEPENDS
@@ -456,7 +449,7 @@ add_header_macro(
 
 add_header_macro(
   elf
-  ../libc/newhdrgen/yaml/elf.yaml
+  ../libc/hdrgen/yaml/elf.yaml
   elf.h.def
   elf.h
   DEPENDS
@@ -470,7 +463,7 @@ file(MAKE_DIRECTORY ${LIBC_INCLUDE_DIR}/sys)
 
 add_header_macro(
   sys_auxv
-  ../libc/newhdrgen/yaml/sys/auxv.yaml
+  ../libc/hdrgen/yaml/sys/auxv.yaml
   sys/auxv.h.def
   sys/auxv.h
   DEPENDS
@@ -480,7 +473,7 @@ add_header_macro(
 
 add_header_macro(
   sys_epoll
-  ../libc/newhdrgen/yaml/sys/epoll.yaml
+  ../libc/hdrgen/yaml/sys/epoll.yaml
   sys/epoll.h.def
   sys/epoll.h
   DEPENDS
@@ -493,7 +486,7 @@ add_header_macro(
 
 add_header_macro(
   sys_ioctl
-  ../libc/newhdrgen/yaml/sys/ioctl.yaml
+  ../libc/hdrgen/yaml/sys/ioctl.yaml
   sys/ioctl.h.def
   sys/ioctl.h
   DEPENDS
@@ -503,7 +496,7 @@ add_header_macro(
 
 add_header_macro(
   sys_mman
-  ../libc/newhdrgen/yaml/sys/mman.yaml
+  ../libc/hdrgen/yaml/sys/mman.yaml
   sys/mman.h.def
   sys/mman.h
   DEPENDS
@@ -516,7 +509,7 @@ add_header_macro(
 
 add_header_macro(
   sys_prctl
-  ../libc/newhdrgen/yaml/sys/prctl.yaml
+  ../libc/hdrgen/yaml/sys/prctl.yaml
   sys/prctl.h.def
   sys/prctl.h
   DEPENDS
@@ -533,7 +526,7 @@ add_header(
 
 add_header_macro(
   sys_random
-  ../libc/newhdrgen/yaml/sys/random.yaml
+  ../libc/hdrgen/yaml/sys/random.yaml
   sys/random.h.def
   sys/random.h
   DEPENDS
@@ -545,7 +538,7 @@ add_header_macro(
 
 add_header_macro(
   sys_resource
-  ../libc/newhdrgen/yaml/sys/resource.yaml
+  ../libc/hdrgen/yaml/sys/resource.yaml
   sys/resource.h.def
   sys/resource.h
   DEPENDS
@@ -557,7 +550,7 @@ add_header_macro(
 
 add_header_macro(
   sys_stat
-  ../libc/newhdrgen/yaml/sys/stat.yaml
+  ../libc/hdrgen/yaml/sys/stat.yaml
   sys/stat.h.def
   sys/stat.h
   DEPENDS
@@ -579,7 +572,7 @@ add_header_macro(
 
 add_header_macro(
   sys_select
-  ../libc/newhdrgen/yaml/sys/select.yaml
+  ../libc/hdrgen/yaml/sys/select.yaml
   sys/select.h.def
   sys/select.h
   DEPENDS
@@ -595,7 +588,7 @@ add_header_macro(
 
 add_header_macro(
   sys_sendfile
-  ../libc/newhdrgen/yaml/sys/sendfile.yaml
+  ../libc/hdrgen/yaml/sys/sendfile.yaml
   sys/sendfile.h.def
   sys/sendfile.h
   DEPENDS
@@ -607,7 +600,7 @@ add_header_macro(
 
 add_header_macro(
   sys_socket
-  ../libc/newhdrgen/yaml/sys/socket.yaml
+  ../libc/hdrgen/yaml/sys/socket.yaml
   sys/socket.h.def
   sys/socket.h
   DEPENDS
@@ -623,7 +616,7 @@ add_header_macro(
 
 add_header_macro(
   sys_statvfs
-  ../libc/newhdrgen/yaml/sys/statvfs.yaml
+  ../libc/hdrgen/yaml/sys/statvfs.yaml
   sys/statvfs.h.def
   sys/statvfs.h
   DEPENDS
@@ -633,7 +626,7 @@ add_header_macro(
 
 add_header_macro(
   sys_syscall
-  ../libc/newhdrgen/yaml/sys/syscall.yaml
+  ../libc/hdrgen/yaml/sys/syscall.yaml
   sys/syscall.h.def
   sys/syscall.h
   DEPENDS
@@ -641,7 +634,7 @@ add_header_macro(
 
 add_header_macro(
   sys_time
-  ../libc/newhdrgen/yaml/sys/time.yaml
+  ../libc/hdrgen/yaml/sys/time.yaml
   sys/time.h.def
   sys/time.h
   DEPENDS
@@ -652,7 +645,7 @@ add_header_macro(
 
 add_header_macro(
   sys_types
-  ../libc/newhdrgen/yaml/sys/types.yaml
+  ../libc/hdrgen/yaml/sys/types.yaml
   sys/types.h.def
   sys/types.h
   DEPENDS
@@ -682,7 +675,7 @@ add_header_macro(
 
 add_header_macro(
   sys_utsname
-  ../libc/newhdrgen/yaml/sys/utsname.yaml
+  ../libc/hdrgen/yaml/sys/utsname.yaml
   sys/utsname.h.def
   sys/utsname.h
   DEPENDS
@@ -692,7 +685,7 @@ add_header_macro(
 
 add_header_macro(
   sys_wait
-  ../libc/newhdrgen/yaml/sys/wait.yaml
+  ../libc/hdrgen/yaml/sys/wait.yaml
   sys/wait.h.def
   sys/wait.h
   DEPENDS
@@ -705,7 +698,7 @@ add_header_macro(
 
 add_header_macro(
   termios
-  ../libc/newhdrgen/yaml/termios.yaml
+  ../libc/hdrgen/yaml/termios.yaml
   termios.h.def
   termios.h
   DEPENDS
@@ -720,7 +713,7 @@ add_header_macro(
 
 add_header_macro(
   uchar
-  ../libc/newhdrgen/yaml/uchar.yaml
+  ../libc/hdrgen/yaml/uchar.yaml
   uchar.h.def
   uchar.h
   DEPENDS
@@ -733,7 +726,7 @@ add_header_macro(
 
 add_header_macro(
   wchar
-  ../libc/newhdrgen/yaml/wchar.yaml
+  ../libc/hdrgen/yaml/wchar.yaml
   wchar.h.def
   wchar.h
   DEPENDS
@@ -747,7 +740,7 @@ add_header_macro(
 
 add_header_macro(
   locale
-  ../libc/newhdrgen/yaml/locale.yaml
+  ../libc/hdrgen/yaml/locale.yaml
   locale.h.def
   locale.h
   DEPENDS
@@ -762,7 +755,7 @@ if(LIBC_TARGET_OS_IS_GPU)
 
   add_header_macro(
     gpu_rpc
-    ../libc/newhdrgen/yaml/gpu/rpc.yaml
+    ../libc/hdrgen/yaml/gpu/rpc.yaml
     gpu/rpc.h.def
     gpu/rpc.h
     DEPENDS
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 75194923a452f..9d5d9f6544288 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -315,3 +315,9 @@ add_macro_header(
   HDR
     locale-macros.h
 )
+
+add_macro_header(
+  pthread_macros
+  HDR
+    pthread-macros.h
+)
diff --git a/libc/include/llvm-libc-macros/pthread-macros.h b/libc/include/llvm-libc-macros/pthread-macros.h
new file mode 100644
index 0000000000000..8a144dbd2e611
--- /dev/null
+++ b/libc/include/llvm-libc-macros/pthread-macros.h
@@ -0,0 +1,37 @@
+//===-- Definition of pthread macros --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_PTHREAD_MACRO_H
+#define LLVM_LIBC_MACROS_PTHREAD_MACRO_H
+
+#define PTHREAD_CREATE_JOINABLE 0
+#define PTHREAD_CREATE_DETACHED 1
+
+#define PTHREAD_MUTEX_NORMAL 0
+#define PTHREAD_MUTEX_ERRORCHECK 1
+#define PTHREAD_MUTEX_RECURSIVE 2
+#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL
+
+#define PTHREAD_MUTEX_STALLED 0
+#define PTHREAD_MUTEX_ROBUST 1
+
+#define PTHREAD_ONCE_INIT {0}
+
+#define PTHREAD_PROCESS_PRIVATE 0
+#define PTHREAD_PROCESS_SHARED 1
+
+#define PTHREAD_MUTEX_INITIALIZER {0}
+#define PTHREAD_RWLOCK_INITIALIZER {0}
+
+// glibc extensions
+#define PTHREAD_STACK_MIN (1 << 14) // 16KB
+#define PTHREAD_RWLOCK_PREFER_READER_NP 0
+#define PTHREAD_RWLOCK_PREFER_WRITER_NP 1
+#define PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP 2
+
+#endif // LLVM_LIBC_MACROS_PTHREAD_MACRO_H
diff --git a/libc/include/llvm-libc-macros/time-macros.h b/libc/include/llvm-libc-macros/time-macros.h
index 6d49ed484d5d4..445d8b3e837ed 100644
--- a/libc/include/llvm-libc-macros/time-macros.h
+++ b/libc/include/llvm-libc-macros/time-macros.h
@@ -7,4 +7,9 @@
 #include "linux/time-macros.h"
 #endif
 
+#define TIME_UTC 1
+#define TIME_MONOTONIC 2
+#define TIME_ACTIVE 3
+#define TIME_THREAD_ACTIVE 4
+
 #endif // LLVM_LIBC_MACROS_TIME_MACROS_H
diff --git a/libc/include/llvm-libc-macros/windows/CMakeLists.txt b/libc/include/llvm-libc-macros/windows/CMakeLists.txt
new file mode 100644
index 0000000000000..48afc795178a0
--- /dev/null
+++ b/libc/include/llvm-libc-macros/windows/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_header(
+  time_macros_ext
+  HDR
+    time-macros-ext.h
+)
+
diff --git a/libc/include/llvm-libc-macros/windows/time-macros-ext.h b/libc/include/llvm-libc-macros/windows/time-macros-ext.h
new file mode 100644
index 0000000000000..71d914b451877
--- /dev/null
+++ b/libc/include/llvm-libc-macros/windows/time-macros-ext.h
@@ -0,0 +1,17 @@
+//===-- Windows Time Macros Extension -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_WINDOWS_TIME_MACROS_EXT_H
+#define LLVM_LIBC_MACROS_WINDOWS_TIME_MACROS_EXT_H
+
+#define CLOCK_MONOTONIC 0
+#define CLOCK_REALTIME 1
+#define CLOCK_PROCESS_CPUTIME_ID 2
+#define CLOCK_THREAD_CPUTIME_ID 3
+
+#endif // LLVM_LIBC_MACROS_WINDOWS_TIME_MACROS_EXT_H
diff --git a/libc/include/llvm-libc-types/cfloat128.h b/libc/include/llvm-libc-types/cfloat128.h
index a371671cf6235..f76a0c1c2f5af 100644
--- a/libc/include/llvm-libc-types/cfloat128.h
+++ b/libc/include/llvm-libc-types/cfloat128.h
@@ -35,6 +35,7 @@ typedef _Complex _Float128 cfloat128;
 typedef _Complex __float128 cfloat128;
 #elif (LDBL_MANT_DIG == 113)
 #define LIBC_TYPES_HAS_CFLOAT128
+#define LIBC_TYPES_CFLOAT128_IS_COMPLEX_LONG_DOUBLE
 typedef _Complex long double cfloat128;
 #endif
 
diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def
index 4dbeed6b5f321..aa1e9983e4bb9 100644
--- a/libc/include/pthread.h.def
+++ b/libc/include/pthread.h.def
@@ -10,39 +10,7 @@
 #define LLVM_LIBC_PTHREAD_H
 
 #include "__llvm-libc-common.h"
-
-// TODO: move to a pthreads-macros.h file:
-// https://github.com/llvm/llvm-project/issues/88997
-
-#define PTHREAD_STACK_MIN (1 << 14) // 16KB
-
-#define PTHREAD_MUTEX_INITIALIZER {0}
-#define PTHREAD_RWLOCK_INITIALIZER {}
-#define PTHREAD_ONCE_INIT {0}
-
-enum {
-  PTHREAD_CREATE_JOINABLE = 0x0,
-  PTHREAD_CREATE_DETACHED = 0x1,
-
-  PTHREAD_MUTEX_NORMAL = 0x0,
-  PTHREAD_MUTEX_ERRORCHECK = 0x1,
-  PTHREAD_MUTEX_RECURSIVE = 0x2,
-  PTHREAD_MUTEX_DEFAULT = PTHREAD_MUTEX_NORMAL,
-
-  PTHREAD_PROCESS_PRIVATE = 0x0,
-  PTHREAD_PROCESS_SHARED = 0x1,
-
-  PTHREAD_MUTEX_STALLED = 0x0,
-  PTHREAD_MUTEX_ROBUST = 0x1,
-};
-
-#define PTHREAD_PROCESS_PRIVATE 0
-#define PTHREAD_PROCESS_SHARED 1
-
-#define PTHREAD_RWLOCK_PREFER_READER_NP 0
-#define PTHREAD_RWLOCK_PREFER_WRITER_NP 1
-#define PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP 2
-
+#include "llvm-libc-macros/pthread-macros.h"
 
 %%public_api()
 
diff --git a/libc/newhdrgen/CMakeLists.txt b/libc/newhdrgen/CMakeLists.txt
deleted file mode 100644
index 523f8c2609124..0000000000000
--- a/libc/newhdrgen/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-if(LLVM_LIBC_FULL_BUILD)
-  enable_testing()
-
-  set(NEWHDGEN_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests)
-  set(TEST_OUTPUT_DIR ${CMAKE_BINARY_DIR}/newhdrgen/output)
-
-  add_test(
-    NAME newhdrgen_integration_test
-    COMMAND python3 ${NEWHDGEN_TESTS_DIR}/test_integration.py --output_dir ${TEST_OUTPUT_DIR}
-  )
- 
-  add_custom_target(check-newhdrgen
-    COMMAND ${CMAKE_CTEST_COMMAND} -R newhdrgen_integration_test --output-on-failure
-  )
-
-  message(STATUS "Integration test for newhdrgen added.")
-endif()
diff --git a/libc/shared/rpc_opcodes.h b/libc/shared/rpc_opcodes.h
index 430b53aa1870c..270c35dec28b8 100644
--- a/libc/shared/rpc_opcodes.h
+++ b/libc/shared/rpc_opcodes.h
@@ -9,41 +9,51 @@
 #ifndef LLVM_LIBC_SHARED_RPC_OPCODES_H
 #define LLVM_LIBC_SHARED_RPC_OPCODES_H
 
+#include "rpc.h"
+
 #define LLVM_LIBC_RPC_BASE 'c'
 #define LLVM_LIBC_OPCODE(n) (LLVM_LIBC_RPC_BASE << 24 | n)
 
 typedef enum {
-  RPC_NOOP = LLVM_LIBC_OPCODE(0),
-  RPC_EXIT = LLVM_LIBC_OPCODE(1),
-  RPC_WRITE_TO_STDOUT = LLVM_LIBC_OPCODE(2),
-  RPC_WRITE_TO_STDERR = LLVM_LIBC_OPCODE(3),
-  RPC_WRITE_TO_STREAM = LLVM_LIBC_OPCODE(4),
-  RPC_WRITE_TO_STDOUT_NEWLINE = LLVM_LIBC_OPCODE(5),
-  RPC_READ_FROM_STREAM = LLVM_LIBC_OPCODE(6),
-  RPC_READ_FGETS = LLVM_LIBC_OPCODE(7),
-  RPC_OPEN_FILE = LLVM_LIBC_OPCODE(8),
-  RPC_CLOSE_FILE = LLVM_LIBC_OPCODE(9),
-  RPC_MALLOC = LLVM_LIBC_OPCODE(10),
-  RPC_FREE = LLVM_LIBC_OPCODE(11),
-  RPC_HOST_CALL = LLVM_LIBC_OPCODE(12),
-  RPC_ABORT = LLVM_LIBC_OPCODE(13),
-  RPC_FEOF = LLVM_LIBC_OPCODE(14),
-  RPC_FERROR = LLVM_LIBC_OPCODE(15),
-  RPC_CLEARERR = LLVM_LIBC_OPCODE(16),
-  RPC_FSEEK = LLVM_LIBC_OPCODE(17),
-  RPC_FTELL = LLVM_LIBC_OPCODE(18),
-  RPC_FFLUSH = LLVM_LIBC_OPCODE(19),
-  RPC_UNGETC = LLVM_LIBC_OPCODE(20),
-  RPC_PRINTF_TO_STDOUT = LLVM_LIBC_OPCODE(21),
-  RPC_PRINTF_TO_STDERR = LLVM_LIBC_OPCODE(22),
-  RPC_PRINTF_TO_STREAM = LLVM_LIBC_OPCODE(23),
-  RPC_PRINTF_TO_STDOUT_PACKED = LLVM_LIBC_OPCODE(24),
-  RPC_PRINTF_TO_STDERR_PACKED = LLVM_LIBC_OPCODE(25),
-  RPC_PRINTF_TO_STREAM_PACKED = LLVM_LIBC_OPCODE(26),
-  RPC_REMOVE = LLVM_LIBC_OPCODE(27),
-  RPC_RENAME = LLVM_LIBC_OPCODE(28),
-  RPC_SYSTEM = LLVM_LIBC_OPCODE(29),
-  RPC_LAST = 0xFFFFFFFF,
+  LIBC_NOOP = LLVM_LIBC_OPCODE(0),
+  LIBC_EXIT = LLVM_LIBC_OPCODE(1),
+  LIBC_WRITE_TO_STDOUT = LLVM_LIBC_OPCODE(2),
+  LIBC_WRITE_TO_STDERR = LLVM_LIBC_OPCODE(3),
+  LIBC_WRITE_TO_STREAM = LLVM_LIBC_OPCODE(4),
+  LIBC_WRITE_TO_STDOUT_NEWLINE = LLVM_LIBC_OPCODE(5),
+  LIBC_READ_FROM_STREAM = LLVM_LIBC_OPCODE(6),
+  LIBC_READ_FGETS = LLVM_LIBC_OPCODE(7),
+  LIBC_OPEN_FILE = LLVM_LIBC_OPCODE(8),
+  LIBC_CLOSE_FILE = LLVM_LIBC_OPCODE(9),
+  LIBC_MALLOC = LLVM_LIBC_OPCODE(10),
+  LIBC_FREE = LLVM_LIBC_OPCODE(11),
+  LIBC_HOST_CALL = LLVM_LIBC_OPCODE(12),
+  LIBC_ABORT = LLVM_LIBC_OPCODE(13),
+  LIBC_FEOF = LLVM_LIBC_OPCODE(14),
+  LIBC_FERROR = LLVM_LIBC_OPCODE(15),
+  LIBC_CLEARERR = LLVM_LIBC_OPCODE(16),
+  LIBC_FSEEK = LLVM_LIBC_OPCODE(17),
+  LIBC_FTELL = LLVM_LIBC_OPCODE(18),
+  LIBC_FFLUSH = LLVM_LIBC_OPCODE(19),
+  LIBC_UNGETC = LLVM_LIBC_OPCODE(20),
+  LIBC_PRINTF_TO_STDOUT = LLVM_LIBC_OPCODE(21),
+  LIBC_PRINTF_TO_STDERR = LLVM_LIBC_OPCODE(22),
+  LIBC_PRINTF_TO_STREAM = LLVM_LIBC_OPCODE(23),
+  LIBC_PRINTF_TO_STDOUT_PACKED = LLVM_LIBC_OPCODE(24),
+  LIBC_PRINTF_TO_STDERR_PACKED = LLVM_LIBC_OPCODE(25),
+  LIBC_PRINTF_TO_STREAM_PACKED = LLVM_LIBC_OPCODE(26),
+  LIBC_REMOVE = LLVM_LIBC_OPCODE(27),
+  LIBC_RENAME = LLVM_LIBC_OPCODE(28),
+  LIBC_SYSTEM = LLVM_LIBC_OPCODE(29),
+  LIBC_LAST = 0xFFFFFFFF,
 } rpc_opcode_t;
 
+#undef LLVM_LIBC_OPCODE
+
+namespace rpc {
+// The implementation of this function currently lives in the utility directory
+// at 'utils/gpu/server/rpc_server.cpp'.
+rpc::Status handle_libc_opcodes(rpc::Server::Port &port, uint32_t num_lanes);
+} // namespace rpc
+
 #endif // LLVM_LIBC_SHARED_RPC_OPCODES_H
diff --git a/libc/spec/bsd_ext.td b/libc/spec/bsd_ext.td
deleted file mode 100644
index 2b91324e36db9..0000000000000
--- a/libc/spec/bsd_ext.td
+++ /dev/null
@@ -1,87 +0,0 @@
-def BsdExtensions : StandardSpec<"BSDExtensions"> {
-  HeaderSpec Math = HeaderSpec<
-      "math.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<"isnan", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"isnanf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"isnanl", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
-      ]
-  >;
-
-  HeaderSpec String = HeaderSpec<
-      "string.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "strlcat",
-            RetValSpec<SizeTType>,
-            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strlcpy",
-            RetValSpec<SizeTType>,
-            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strsep",
-            RetValSpec<CharPtr>,
-            [ArgSpec<CharRestrictedPtrPtr>, ArgSpec<ConstCharRestrictedPtr>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Strings = HeaderSpec<
-      "strings.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "strcasecmp",
-            RetValSpec<IntType>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-            "strncasecmp",
-            RetValSpec<IntType>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "index",
-            RetValSpec<CharPtr>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "rindex",
-            RetValSpec<CharPtr>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-      ]
-  >;
-
-  HeaderSpec SysWait = HeaderSpec<
-      "sys/wait.h",
-      [], // Macros
-      [StructRUsage], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "wait4",
-            RetValSpec<PidT>,
-            [ArgSpec<PidT>, ArgSpec<IntPtr>, ArgSpec<IntType>, ArgSpec<StructRUsagePtr>]
-        >
-      ]
-  >;
-
-  let Headers = [
-    Math,
-    String,
-    Strings,
-    SysWait,
-  ];
-}
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
deleted file mode 100644
index 64121aed9574f..0000000000000
--- a/libc/spec/gnu_ext.td
+++ /dev/null
@@ -1,316 +0,0 @@
-def CpuSetT : NamedType<"cpu_set_t">;
-def CpuSetPtr : PtrType<CpuSetT>;
-def ConstCpuSetPtr : ConstType<CpuSetPtr>;
-
-def QSortRCompareT : NamedType<"__qsortrcompare_t">;
-def StructHsearchData : NamedType<"struct hsearch_data">;
-def StructHsearchDataPtr : PtrType<StructHsearchData>;
-
-def GnuExtensions : StandardSpec<"GNUExtensions"> {
-  NamedType CookieIOFunctionsT = NamedType<"cookie_io_functions_t">;
-  HeaderSpec CType = HeaderSpec<
-    "ctype.h",
-    [], // Macros
-    [], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "toascii",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-    ]
-  >;
-
-  HeaderSpec Malloc = HeaderSpec<
-      "malloc.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<"mallopt", RetValSpec<IntType>, [ArgSpec<IntType>, ArgSpec<IntType>]>,
-      ]
-  >;
-
-  HeaderSpec Math = HeaderSpec<
-      "math.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "sincosf",
-            RetValSpec<VoidType>,
-            [ArgSpec<FloatType>, ArgSpec<FloatPtr>, ArgSpec<FloatPtr>]
-        >,
-        FunctionSpec<
-            "lgamma_r",
-            RetValSpec<DoubleType>,
-            [ArgSpec<DoubleType>, ArgSpec<IntPtr>]
-        >,
-        FunctionSpec<
-            "lgammaf_r",
-            RetValSpec<FloatType>,
-            [ArgSpec<FloatType>, ArgSpec<IntPtr>]
-        >,
-        FunctionSpec<
-            "lgammal_r",
-            RetValSpec<LongDoubleType>,
-            [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Sched = HeaderSpec<
-      "sched.h",
-      [], // Macros
-      [PidT, SizeTType, CpuSetT], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "sched_getaffinity",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<SizeTType>, ArgSpec<CpuSetPtr>]
-        >,
-        FunctionSpec<
-            "sched_setaffinity",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<SizeTType>, ArgSpec<ConstCpuSetPtr>]
-        >,
-      ]
-  >;
-  HeaderSpec String = HeaderSpec<
-      "string.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-         FunctionSpec<
-            "memmem",
-            RetValSpec<VoidPtr>,
-            [ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "memrchr",
-            RetValSpec<VoidPtr>,
-            [ArgSpec<ConstVoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strerror_r",
-            RetValSpec<CharPtr>,
-            [ArgSpec<IntType>, ArgSpec<CharPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strcasestr",
-            RetValSpec<CharPtr>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-            "strchrnul",
-            RetValSpec<CharPtr>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Search = HeaderSpec<
-    "search.h",
-    [], // Macros
-    [
-        StructHsearchData
-    ],
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "hcreate_r",
-            RetValSpec<IntType>,
-            [
-                ArgSpec<SizeTType>,
-                ArgSpec<StructHsearchDataPtr>
-            ]
-        >,
-        FunctionSpec<
-            "hdestroy_r",
-            RetValSpec<VoidType>,
-            [
-                ArgSpec<StructHsearchDataPtr>
-            ]
-        >,
-        FunctionSpec<
-            "hsearch_r",
-            RetValSpec<IntType>,
-            [
-                ArgSpec<EntryType>,
-                ArgSpec<ActionType>,
-                ArgSpec<EntryTypePtrPtr>,
-                ArgSpec<StructHsearchDataPtr>
-            ]
-        >,
-    ]
-  >;
-
-  HeaderSpec FEnv = HeaderSpec<
-      "fenv.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "fedisableexcept",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "feenableexcept",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "fegetexcept",
-            RetValSpec<IntType>,
-            []
-        >
-      ]
-  >;
-
-  HeaderSpec StdIO = HeaderSpec<
-      "stdio.h",
-      [], // Macros
-      [CookieIOFunctionsT], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "clearerr_unlocked",
-              RetValSpec<VoidType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "feof_unlocked",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "ferror_unlocked",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "fopencookie",
-              RetValSpec<FILEPtr>,
-              [ArgSpec<VoidPtr>, ArgSpec<ConstCharPtr>, ArgSpec<CookieIOFunctionsT>]
-          >,
-          FunctionSpec<
-              "fread_unlocked",
-              RetValSpec<SizeTType>,
-              [ArgSpec<VoidRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<SizeTType>,
-               ArgSpec<FILERestrictedPtr>]
-          >,
-          FunctionSpec<
-              "fwrite_unlocked",
-              RetValSpec<SizeTType>,
-              [ArgSpec<ConstVoidRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<SizeTType>,
-               ArgSpec<FILERestrictedPtr>]
-          >,
-          FunctionSpec<
-              "fgetc_unlocked",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec StdLib = HeaderSpec<
-      "stdlib.h",
-      [], // Macros
-      [QSortRCompareT], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-                "qsort_r",
-                RetValSpec<VoidType>,
-                [ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<SizeTType>, ArgSpec<QSortRCompareT>, ArgSpec<VoidPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec PThread = HeaderSpec<
-      "pthread.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "pthread_setname_np",
-              RetValSpec<IntType>,
-              [ArgSpec<PThreadTType>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "pthread_getname_np",
-              RetValSpec<IntType>,
-              [ArgSpec<PThreadTType>, ArgSpec<CharPtr>, ArgSpec<SizeTType>]
-          >,
-      ]
-  >;
-
-  HeaderSpec SysAuxv = HeaderSpec<
-      "sys/auxv.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "getauxval",
-              RetValSpec<UnsignedLongType>,
-              [ArgSpec<UnsignedLongType>]
-          >,
-      ]  // Functions
-  >;
-
-  HeaderSpec SendFile = HeaderSpec<
-      "sys/sendfile.h",
-      [], // Macros
-      [OffTType, SizeTType, SSizeTType,], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "sendfile",
-              RetValSpec<SSizeTType>,
-              [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<OffTPtr>, ArgSpec<SizeTType>]
-          >,
-      ]
-  >;
-
-  HeaderSpec UniStd = HeaderSpec<
-    "unistd.h",
-    [], // Macros
-    [], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "dup2",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>]
-        >,
-    ]
-  >;
-
-  let Headers = [
-    CType,
-    FEnv,
-    Malloc,
-    Math,
-    PThread,
-    Sched,
-    SendFile,
-    SysAuxv,
-    StdIO,
-    StdLib,
-    String,
-    Search,
-    UniStd,
-  ];
-}
diff --git a/libc/spec/gpu_ext.td b/libc/spec/gpu_ext.td
deleted file mode 100644
index d99531dc06bcd..0000000000000
--- a/libc/spec/gpu_ext.td
+++ /dev/null
@@ -1,18 +0,0 @@
-def GPUExtensions : StandardSpec<"GPUExtensions"> {
-  HeaderSpec RPC = HeaderSpec<
-    "gpu/rpc.h",
-    [], // Macros
-    [], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "rpc_host_call",
-            RetValSpec<UnsignedLongLongType>,
-            [ArgSpec<VoidPtr>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-    ]
-  >;
-  let Headers = [
-    RPC,
-  ];
-}
diff --git a/libc/spec/linux.td b/libc/spec/linux.td
deleted file mode 100644
index 9b5dc8e30c95e..0000000000000
--- a/libc/spec/linux.td
+++ /dev/null
@@ -1,328 +0,0 @@
-def StructEpollEvent : NamedType<"struct epoll_event">;
-def StructEpollEventPtr : PtrType<StructEpollEvent>;
-
-def StructEpollData : NamedType<"struct epoll_data">;
-
-def Linux : StandardSpec<"Linux"> {
-  HeaderSpec Errno = HeaderSpec<
-      "errno.h",
-      [
-        Macro<"ENOMEDIUM">,
-        Macro<"ENOTBLK">,
-        Macro<"EMEDIUMTYPE">,
-        Macro<"EBADSLT">,
-        Macro<"ECHRNG">,
-        Macro<"ERFKILL">,
-        Macro<"EUSERS">,
-        Macro<"EBADR">,
-        Macro<"EL3HLT">,
-        Macro<"ENOTUNIQ">,
-        Macro<"EXFULL">,
-        Macro<"EHOSTDOWN">,
-        Macro<"EL3RST">,
-        Macro<"ENOPKG">,
-        Macro<"ENOCSI">,
-        Macro<"EUNATCH">,
-        Macro<"EREMCHG">,
-        Macro<"ETOOMANYREFS">,
-        Macro<"EL2HLT">,
-        Macro<"EBADFD">,
-        Macro<"EREMOTEIO">,
-        Macro<"ENAVAIL">,
-        Macro<"ELIBEXEC">,
-        Macro<"ESHUTDOWN">,
-        Macro<"ENOKEY">,
-        Macro<"ESTRPIPE">,
-        Macro<"EKEYREJECTED">,
-        Macro<"ESRMNT">,
-        Macro<"EKEYREVOKED">,
-        Macro<"EBADE">,
-        Macro<"ELIBBAD">,
-        Macro<"EISNAM">,
-        Macro<"EBFONT">,
-        Macro<"EPFNOSUPPORT">,
-        Macro<"EREMOTE">,
-        Macro<"EDEADLOCK">,
-        Macro<"ENONET">,
-        Macro<"EDOTDOT">,
-        Macro<"EKEYEXPIRED">,
-        Macro<"ELIBSCN">,
-        Macro<"ERESTART">,
-        Macro<"EBADRQC">,
-        Macro<"EUCLEAN">,
-        Macro<"ENOANO">,
-        Macro<"ELIBACC">,
-        Macro<"EHWPOISON">,
-        Macro<"ELIBMAX">,
-        Macro<"ESOCKTNOSUPPORT">,
-        Macro<"ENOTNAM">,
-        Macro<"ELNRNG">,
-        Macro<"EL2NSYNC">,
-        Macro<"EADV">,
-        Macro<"ECOMM">,
-      ]
-  >;
-
-  HeaderSpec Sched = HeaderSpec<
-      "sched.h",
-      [
-        Macro<"SCHED_OTHER">,
-        Macro<"SCHED_FIFO">,
-        Macro<"SCHED_RR">,
-        Macro<"SCHED_BATCH">,
-        Macro<"SCHED_ISO">,
-        Macro<"SCHED_IDLE">,
-        Macro<"SCHED_DEADLINE">,
-      ],
-      [], // Types
-      [], // Enumerations
-      []  // Functions
-  >;
-
-  HeaderSpec SysMMan = HeaderSpec<
-      "sys/mman.h",
-      [Macro<"MAP_ANONYMOUS">],
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "mincore",
-            RetValSpec<IntType>,
-            [
-              ArgSpec<VoidPtr>,
-              ArgSpec<SizeTType>,
-              ArgSpec<UnsignedCharPtr>,
-            ]
-        >,
-        FunctionSpec<
-          "mlock2",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<VoidPtr>,
-            ArgSpec<SizeTType>,
-            ArgSpec<UnsignedIntType>,
-          ]
-        >,
-        FunctionSpec<
-          "remap_file_pages",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<VoidPtr>,
-            ArgSpec<SizeTType>,
-            ArgSpec<IntType>,
-            ArgSpec<SizeTType>,
-            ArgSpec<IntType>,
-          ]
-        >,
-        FunctionSpec<
-          "mremap",
-          RetValSpec<VoidPtr>,
-          [
-            ArgSpec<VoidPtr>,
-            ArgSpec<SizeTType>,
-            ArgSpec<SizeTType>,
-            ArgSpec<IntType>,
-            ArgSpec<VarArgType>,
-          ]
-        >,
-      ]  // Functions
-  >;
-
-
-  HeaderSpec SysPrctl = HeaderSpec<
-      "sys/prctl.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "prctl",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<UnsignedLongType>,
-            ArgSpec<UnsignedLongType>,
-            ArgSpec<UnsignedLongType>,
-            ArgSpec<UnsignedLongType>,
-          ]
-        >,
-      ]  // Functions
-  >;
-
-  HeaderSpec SysRandom = HeaderSpec<
-      "sys/random.h",
-      [
-        Macro<"GRND_RANDOM">,
-        Macro<"GRND_NONBLOCK">,
-        Macro<"GRND_INSECURE">,
-      ],
-      [SizeTType, SSizeTType], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "getrandom",
-          RetValSpec<SSizeTType>,
-          [
-            ArgSpec<VoidPtr>,
-            ArgSpec<SizeTType>,
-            ArgSpec<UnsignedIntType>
-          ]
-        >,
-      ]
-  >;
-
-  HeaderSpec SysTime = HeaderSpec<
-      "sys/time.h",
-      [
-        Macro<"timeradd">,
-        Macro<"timersub">,
-        Macro<"timerclear">,
-        Macro<"timerisset">,
-        Macro<"timercmp">,
-      ],
-      [StructTimevalType], // Types
-      [], // Enumerations
-      []  // Functions
-  >;
-
-
-  HeaderSpec SysEpoll = HeaderSpec<
-      "sys/epoll.h",
-      [], // Macros
-      [
-        StructEpollEvent,
-        StructEpollData,
-        SigSetType,
-        StructTimeSpec,
-      ], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "epoll_create",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>
-          ]
-        >,
-        FunctionSpec<
-          "epoll_create1",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>
-          ]
-        >,
-        FunctionSpec<
-          "epoll_ctl",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<IntType>,
-            ArgSpec<IntType>,
-            ArgSpec<StructEpollEventPtr>
-          ]
-        >,
-        FunctionSpec<
-          "epoll_wait",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<StructEpollEventPtr>,
-            ArgSpec<IntType>,
-            ArgSpec<IntType>
-          ]
-        >,
-        FunctionSpec<
-          "epoll_pwait",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<StructEpollEventPtr>,
-            ArgSpec<IntType>,
-            ArgSpec<IntType>,
-            ArgSpec<ConstSigSetPtrType>
-          ]
-        >,
-        FunctionSpec<
-          "epoll_pwait2",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<StructEpollEventPtr>,
-            ArgSpec<IntType>,
-            ArgSpec<ConstStructTimeSpecPtr>,
-            ArgSpec<ConstSigSetPtrType>
-          ]
-        >,
-      ]  // Functions
-  >;
-
-  HeaderSpec Signal = HeaderSpec<
-      "signal.h",
-      [
-        Macro<"NSIG">,
-
-        Macro<"SIGHUP">,
-        Macro<"SIGINT">,
-        Macro<"SIGQUIT">,
-        Macro<"SIGILL">,
-        Macro<"SIGTRAP">,
-        Macro<"SIGABRT">,
-        Macro<"SIGIOT">,
-        Macro<"SIGBUS">,
-        Macro<"SIGFPE">,
-        Macro<"SIGKILL">,
-        Macro<"SIGUSR1">,
-        Macro<"SIGSEGV">,
-        Macro<"SIGUSR2">,
-        Macro<"SIGPIPE">,
-        Macro<"SIGALRM">,
-        Macro<"SIGTERM">,
-        Macro<"SIGSTKFLT">,
-        Macro<"SIGCHLD">,
-        Macro<"SIGCONT">,
-        Macro<"SIGSTOP">,
-        Macro<"SIGTSTP">,
-        Macro<"SIGTTIN">,
-        Macro<"SIGTTOU">,
-        Macro<"SIGURG">,
-        Macro<"SIGXCPU">,
-        Macro<"SIGXFSZ">,
-        Macro<"SIGVTALRM">,
-        Macro<"SIGPROF">,
-        Macro<"SIGWINCH">,
-        Macro<"SIGIO">,
-        Macro<"SIGPOLL">,
-        Macro<"SIGPWR">,
-        Macro<"SIGSYS">,
-        Macro<"SIGUNUSED">,
-      ]
-  >;
-
-
-  HeaderSpec UniStd = HeaderSpec<
-    "unistd.h",
-    [], // Macros
-    [],
-    [], // Enumerations
-    [
-        FunctionSpec<
-          "pipe2",
-          RetValSpec<IntType>,
-          [ArgSpec<IntPtr>, ArgSpec<IntType>] //TODO: make this int[2]
-        >,
-    ],
-    []
-  >;
-  
-
-  let Headers = [
-    Errno,
-    SysEpoll,
-    SysMMan,
-    SysPrctl,
-    SysRandom,
-    SysTime,
-    Signal,
-    UniStd,
-  ];
-}
diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td
deleted file mode 100644
index cd63e34a44ef0..0000000000000
--- a/libc/spec/llvm_libc_ext.td
+++ /dev/null
@@ -1,116 +0,0 @@
-def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> {
-  HeaderSpec Strings = HeaderSpec<
-      "strings.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "bcopy",
-              RetValSpec<VoidType>,
-              [ArgSpec<ConstVoidPtr>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "bzero",
-              RetValSpec<VoidType>,
-              [ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "bcmp",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstVoidPtr>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Assert = HeaderSpec<
-      "assert.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "__assert_fail",
-              RetValSpec<NoReturn>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>, ArgSpec<UnsignedType>, ArgSpec<ConstCharPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Sched = HeaderSpec<
-      "sched.h",
-      [], // Macros
-      [PidT, SizeTType, CpuSetT], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "__sched_getcpucount",
-            RetValSpec<IntType>,
-            [ArgSpec<SizeTType>, ArgSpec<ConstCpuSetPtr>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Math = HeaderSpec<
-      "math.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          GuardedFunctionSpec<"daddf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"ddivf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"dfmaf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"dsqrtf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"dsubf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-
-          GuardedFunctionSpec<"f16add", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16addf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16addl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          GuardedFunctionSpec<"f16sub", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16subf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16subl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          
-          GuardedFunctionSpec<"faddf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"fdivf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"ffmaf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"fmulf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"dmulf128", RetValSpec<DoubleType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"f16mul", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16mulf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16mull", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          GuardedFunctionSpec<"f16div", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16divf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16divl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          GuardedFunctionSpec<"f16fma", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16fmaf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16fmal", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          GuardedFunctionSpec<"f16sqrt", RetValSpec<Float16Type>, [ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16sqrtf", RetValSpec<Float16Type>, [ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"f16sqrtl", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          GuardedFunctionSpec<"fsqrtf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"fsubf128", RetValSpec<FloatType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"powi", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>]>,
-          FunctionSpec<"powif", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>]>,
-      ]
-  >;
-
-  let Headers = [
-    Assert,
-    Math,
-    Sched,
-    Strings,
-  ];
-}
diff --git a/libc/spec/llvm_libc_stdfix_ext.td b/libc/spec/llvm_libc_stdfix_ext.td
deleted file mode 100644
index 7bc7ec5464081..0000000000000
--- a/libc/spec/llvm_libc_stdfix_ext.td
+++ /dev/null
@@ -1,27 +0,0 @@
-def LLVMLibcStdfixExt : StandardSpec<"llvm_libc_stdfix_ext"> {
-  HeaderSpec StdFix = HeaderSpec<
-      "stdfix.h",
-      [],  // macros
-      [],  // types
-      [],  // enums
-      [    // functions
-          GuardedFunctionSpec<"exphk", RetValSpec<ShortAccumType>, [ArgSpec<ShortAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"expk", RetValSpec<AccumType>, [ArgSpec<AccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"sqrtuhr", RetValSpec<UnsignedShortFractType>, [ArgSpec<UnsignedShortFractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"sqrtur", RetValSpec<UnsignedFractType>, [ArgSpec<UnsignedFractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"sqrtulr", RetValSpec<UnsignedLongFractType>, [ArgSpec<UnsignedLongFractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"sqrtuhk", RetValSpec<UnsignedShortAccumType>, [ArgSpec<UnsignedShortAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"sqrtuk", RetValSpec<UnsignedAccumType>, [ArgSpec<UnsignedAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"sqrtulk", RetValSpec<UnsignedLongAccumType>, [ArgSpec<UnsignedLongAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"uhksqrtus", RetValSpec<UnsignedShortAccumType>, [ArgSpec<UnsignedShortType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"uksqrtui", RetValSpec<UnsignedAccumType>, [ArgSpec<UnsignedIntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-      ]
-  >;
-
-  let Headers = [
-    StdFix,
-  ];
-}
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
deleted file mode 100644
index e354deef340f1..0000000000000
--- a/libc/spec/posix.td
+++ /dev/null
@@ -1,1867 +0,0 @@
-def SigSetType : NamedType<"sigset_t">;
-def SigSetPtrType : PtrType<SigSetType>;
-def ConstSigSetPtrType : ConstType<SigSetPtrType>;
-def RestrictedSigSetType : RestrictedPtrType<SigSetType>;
-def ConstRestrictedSigSetType : ConstType<RestrictedSigSetType>;
-
-def SigInfoType : NamedType<"siginfo_t">;
-def UnionSigVal : NamedType<"union sigval">;
-
-def StructSigaction : NamedType<"struct sigaction">;
-def StructSigactionPtr : PtrType<StructSigaction>;
-def ConstStructSigactionPtr : ConstType<StructSigactionPtr>;
-def RestrictedStructSigactionPtr : RestrictedPtrType<StructSigaction>;
-def ConstRestrictedStructSigactionPtr : ConstType<RestrictedStructSigactionPtr>;
-
-def PThreadStartT : NamedType<"__pthread_start_t">;
-def PThreadTSSDtorT : NamedType<"__pthread_tss_dtor_t">;
-def PThreadKeyT : NamedType<"pthread_key_t">;
-def PThreadKeyTPtr : PtrType<PThreadKeyT>;
-def PThreadOnceT : NamedType<"pthread_once_t">;
-def PThreadOnceTPtr : PtrType<PThreadOnceT>;
-def PThreadOnceCallback : NamedType<"__pthread_once_func_t">;
-
-def InoT : NamedType<"ino_t">;
-def UidT : NamedType<"uid_t">;
-def GidT : NamedType<"gid_t">;
-def DevT : NamedType<"dev_t">;
-def ClockIdT : NamedType<"clockid_t">;
-def RestrictedClockIdTPtr : RestrictedPtrType<ClockIdT>;
-def BlkSizeT : NamedType<"blksize_t">;
-def BlkCntT : NamedType<"blkcnt_t">;
-def NLinkT : NamedType<"nlink_t">;
-
-def StatType : NamedType<"struct stat">;
-def StatTypePtr : PtrType<StatType>;
-def RestrictedStatTypePtr : RestrictedPtrType<StatType>;
-
-def DIR : NamedType<"DIR">;
-def DIRPtr : PtrType<DIR>;
-def DIRRestrictedPtr : RestrictedPtrType<DIR>;
-def StructDirent : NamedType<"struct dirent">;
-def StructDirentPtr : PtrType<StructDirent>;
-def StructDirentPtrPtr : PtrType<StructDirentPtr>;
-def ConstStructDirentPtrPtr : ConstType<StructDirentPtrPtr>;
-
-def StructSchedParam : NamedType<"struct sched_param">;
-def StructSchedParamPtr : PtrType<StructSchedParam>;
-def ConstStructSchedParamPtr : ConstType<StructSchedParamPtr>;
-
-def ExecArgvT : NamedType<"__exec_argv_t">;
-def ExecEnvpT : NamedType<"__exec_envp_t">;
-
-def AtForkCallbackT : NamedType<"__atfork_callback_t">;
-
-def PosixSpawnFileActionsT : NamedType<"posix_spawn_file_actions_t">;
-def PosixSpawnFileActionsTPtr : PtrType<PosixSpawnFileActionsT>;
-def ConstPosixSpawnFileActionsTPtr : ConstType<PosixSpawnFileActionsTPtr>;
-def PosixSpawnFileActionsTRestrictedPtr : RestrictedPtrType<PosixSpawnFileActionsT>;
-
-def PosixSpawnAttrT : NamedType<"posix_spawnattr_t">;
-def RestrictedPosixSpawnAttrTPtrType : RestrictedPtrType<PosixSpawnAttrT>;
-
-def CcT : NamedType<"cc_t">;
-def SpeedT : NamedType<"speed_t">;
-def StructTermios : NamedType<"struct termios">;
-def StructTermiosPtr : PtrType<StructTermios>;
-def ConstStructTermiosPtr : ConstType<StructTermiosPtr>;
-def TcFlagT : NamedType<"tcflag_t">;
-
-def StackT : NamedType<"stack_t">;
-def StackTPtr : PtrType<StackT>;
-def RestrictedStackTPtr : RestrictedPtrType<StackT>;
-def ConstRestrictedStackTPtr : ConstType<RestrictedStackTPtr>;
-
-def FdSet : NamedType<"fd_set">;
-def FdSetPtr : PtrType<FdSet>;
-def RestrictedFdSetPtr : RestrictedPtrType<FdSet>;
-
-def GetoptArgvT : NamedType<"__getoptargv_t">;
-
-def SAFamilyType : NamedType<"sa_family_t">;
-def SocklenType : NamedType<"socklen_t">;
-def SocklenPtr : PtrType<SocklenType>;
-
-def StructSockAddr : NamedType<"struct sockaddr">;
-def StructSockAddrPtr : PtrType<StructSockAddr>;
-def ConstStructSockAddrPtr : ConstType<StructSockAddrPtr>;
-
-def StructMsghdr : NamedType<"struct msghdr">;
-def StructMsghdrPtr : PtrType<StructMsghdr>;
-def ConstStructMsghdrPtr : ConstType<StructMsghdrPtr>;
-
-def StructIovec : NamedType<"struct iovec">;
-def StructIovecPtr : PtrType<StructIovec>;
-def ConstStructIovecPtr : ConstType<StructIovecPtr>;
-
-def StructSockAddrUn : NamedType<"struct sockaddr_un">;
-
-def StructStatvfs : NamedType<"struct statvfs">;
-def StructStatvfsPtr : PtrType<StructStatvfs>;
-def RestrictedStructStatvfsPtr : RestrictedPtrType<StructStatvfs>;
-
-// The function pointer type for the predicate for lsearch, lfind
-def LSearchCompareT : NamedType<"__lsearchcompare_t">;
-
-def POSIX : StandardSpec<"POSIX"> {
-  PtrType CharPtr = PtrType<CharType>;
-  RestrictedPtrType RestrictedCharPtr = RestrictedPtrType<CharType>;
-  RestrictedPtrType CharRestrictedDoublePtr = RestrictedPtrType<CharPtr>;
-  ConstType ConstCharPtr = ConstType<CharPtr>;
-  ConstType ConstRestrictedCharPtr = ConstType<RestrictedCharPtr>;
-
-  NamedType ModeTType = NamedType<"mode_t">;
-
-  NamedType PThreadAttrTType = NamedType<"pthread_attr_t">;
-  PtrType PThreadAttrTPtr = PtrType<PThreadAttrTType>;
-  RestrictedPtrType RestrictedPThreadAttrTPtr = RestrictedPtrType<PThreadAttrTType>;
-  ConstType ConstPThreadAttrTPtr = ConstType<PThreadAttrTPtr>;
-  ConstType ConstRestrictedPThreadAttrTPtr = ConstType<RestrictedPThreadAttrTPtr>;
-
-  NamedType PThreadCondAttrTType = NamedType<"pthread_condattr_t">;
-  PtrType PThreadCondAttrTPtr = PtrType<PThreadCondAttrTType>;
-  ConstType ConstRestrictedPThreadCondAttrTPtr = ConstType<RestrictedPtrType<PThreadCondAttrTType>>;
-
-  NamedType PThreadRWLockAttrTType = NamedType<"pthread_rwlockattr_t">;
-  PtrType PThreadRWLockAttrTPtr = PtrType<PThreadRWLockAttrTType>;
-  ConstType ConstPThreadRWLockAttrTPtr = ConstType<PThreadRWLockAttrTPtr>;
-  RestrictedPtrType RestrictedPThreadRWLockAttrTPtr = RestrictedPtrType<PThreadRWLockAttrTType>;
-  ConstType ConstRestrictedPThreadRWLockAttrTPtr = ConstType<RestrictedPThreadRWLockAttrTPtr>;
-
-  NamedType PThreadMutexAttrTType = NamedType<"pthread_mutexattr_t">;
-  PtrType PThreadMutexAttrTPtr = PtrType<PThreadMutexAttrTType>;
-  RestrictedPtrType RestrictedPThreadMutexAttrTPtr = RestrictedPtrType<PThreadMutexAttrTType>;
-  ConstType ConstPThreadMutexAttrTPtr = ConstType<PThreadMutexAttrTPtr>;
-  ConstType ConstRestrictedPThreadMutexAttrTPtr = ConstType<RestrictedPThreadMutexAttrTPtr>;
-
-  NamedType PThreadMutexTType = NamedType<"pthread_mutex_t">;
-  PtrType PThreadMutexTPtr = PtrType<PThreadMutexTType>;
-  RestrictedPtrType RestrictedPThreadMutexTPtr = RestrictedPtrType<PThreadMutexTType>;
-  ConstType ConstPThreadMutexTPtr = ConstType<PThreadMutexTPtr>;
-  ConstType ConstRestrictedPThreadMutexTPtr = ConstType<RestrictedPThreadMutexTPtr>;
-
-  NamedType PThreadRWLockTType = NamedType<"pthread_rwlock_t">;
-  PtrType PThreadRWLockTPtr = PtrType<PThreadRWLockTType>;
-  RestrictedPtrType RestrictedPThreadRWLockTPtr = RestrictedPtrType<PThreadRWLockTType>;
-
-  NamedType PThreadSpinLockTType = NamedType<"pthread_spinlock_t">;
-  PtrType PThreadSpinLockTPtr = PtrType<PThreadSpinLockTType>;
-
-  PtrType PThreadTPtr = PtrType<PThreadTType>;
-  RestrictedPtrType RestrictedPThreadTPtr = RestrictedPtrType<PThreadTType>;
-
-  HeaderSpec Errno = HeaderSpec<
-      "errno.h",
-      [
-        Macro<"E2BIG">,
-        Macro<"EACCES">,
-        Macro<"EADDRINUSE">,
-        Macro<"EADDRNOTAVAIL">,
-        Macro<"EAFNOSUPPORT">,
-        Macro<"EAGAIN">,
-        Macro<"EALREADY">,
-        Macro<"EBADF">,
-        Macro<"EBADMSG">,
-        Macro<"EBUSY">,
-        Macro<"ECANCELED">,
-        Macro<"ECHILD">,
-        Macro<"ECONNABORTED">,
-        Macro<"ECONNREFUSED">,
-        Macro<"ECONNRESET">,
-        Macro<"EDEADLK">,
-        Macro<"EDESTADDRREQ">,
-        Macro<"EDQUOT">,
-        Macro<"EEXIST">,
-        Macro<"EFAULT">,
-        Macro<"EFBIG">,
-        Macro<"EHOSTUNREACH">,
-        Macro<"EIDRM">,
-        Macro<"EINPROGRESS">,
-        Macro<"EINTR">,
-        Macro<"EINVAL">,
-        Macro<"EIO">,
-        Macro<"EISCONN">,
-        Macro<"EISDIR">,
-        Macro<"ELOOP">,
-        Macro<"EMFILE">,
-        Macro<"EMLINK">,
-        Macro<"EMSGSIZE">,
-        Macro<"EMULTIHOP">,
-        Macro<"ENAMETOOLONG">,
-        Macro<"ENETDOWN">,
-        Macro<"ENETRESET">,
-        Macro<"ENETUNREACH">,
-        Macro<"ENFILE">,
-        Macro<"ENOBUFS">,
-        Macro<"ENODATA">,
-        Macro<"ENODEV">,
-        Macro<"ENOENT">,
-        Macro<"ENOEXEC">,
-        Macro<"ENOLCK">,
-        Macro<"ENOLINK">,
-        Macro<"ENOMEM">,
-        Macro<"ENOMSG">,
-        Macro<"ENOPROTOOPT">,
-        Macro<"ENOSPC">,
-        Macro<"ENOSR">,
-        Macro<"ENOSTR">,
-        Macro<"ENOSYS">,
-        Macro<"ENOTCONN">,
-        Macro<"ENOTDIR">,
-        Macro<"ENOTEMPTY">,
-        Macro<"ENOTRECOVERABLE">,
-        Macro<"ENOTSOCK">,
-        Macro<"ENOTSUP">,
-        Macro<"ENOTTY">,
-        Macro<"ENXIO">,
-        Macro<"EOPNOTSUPP">,
-        Macro<"EOVERFLOW">,
-        Macro<"EOWNERDEAD">,
-        Macro<"EPERM">,
-        Macro<"EPIPE">,
-        Macro<"EPROTO">,
-        Macro<"EPROTONOSUPPORT">,
-        Macro<"EPROTOTYPE">,
-        Macro<"EROFS">,
-        Macro<"ESPIPE">,
-        Macro<"ESRCH">,
-        Macro<"ESTALE">,
-        Macro<"ETIME">,
-        Macro<"ETIMEDOUT">,
-        Macro<"ETXTBSY">,
-        Macro<"EWOULDBLOCK">,
-        Macro<"EXDEV">,
-      ],
-      [], // Types
-      [], // Enumerations
-      []  // Functions
-  >;
-
-  HeaderSpec DlFcn = HeaderSpec<
-    "dlfcn.h",
-    [
-      Macro<"RTLD_LAZY">,
-      Macro<"RTLD_NOW">,
-      Macro<"RTLD_GLOBAL">,
-      Macro<"RTLD_LOCAL">,
-    ],
-    [],  // Types
-    [], // Enumerations
-    [
-      FunctionSpec<
-          "dlclose",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidPtr>]
-      >,
-      FunctionSpec<
-          "dlerror",
-          RetValSpec<CharPtr>,
-          []
-      >,
-      FunctionSpec<
-          "dlopen",
-          RetValSpec<VoidPtr>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "dlsym",
-          RetValSpec<VoidPtr>,
-          [ArgSpec<VoidRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>]
-      >,
-    ]
-  >;
-
-  HeaderSpec FCntl = HeaderSpec<
-    "fcntl.h",
-    [], // Macros
-    [
-        ModeTType,
-        OffTType,
-    ],
-    [], // Enumerations
-    [
-      FunctionSpec<
-          "creat",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ModeTType>]
-      >,
-      FunctionSpec<
-          "fcntl",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<VarArgType>]
-      >,
-      FunctionSpec<
-          "open",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<VarArgType>]
-      >,
-      FunctionSpec<
-          "openat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<VarArgType>]
-      >,
-    ]
-  >;
-
-  HeaderSpec SysMMan = HeaderSpec<
-      "sys/mman.h",
-      [
-        // TODO: Add a facility to bunch macros into bitwise-or-able groups.
-        // POSIX requires it, so such thing should be captured in this spec.
-        Macro<"PROT_EXEC">,
-        Macro<"PROT_NONE">,
-        Macro<"PROT_READ">,
-        Macro<"PROT_WRITE">,
-
-        Macro<"MAP_FIXED">,
-        Macro<"MAP_PRIVATE">,
-        Macro<"MAP_SHARED">,
-
-        Macro<"MAP_FAILED">,
-      ],
-      [
-        SizeTType,
-        OffTType,
-        ModeTType,
-      ],
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "madvise",
-            RetValSpec<IntType>,
-            [ArgSpec<VoidPtr>,
-             ArgSpec<SizeTType>,
-             ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "mmap",
-            RetValSpec<VoidPtr>,
-            [ArgSpec<VoidPtr>,
-             ArgSpec<SizeTType>,
-             ArgSpec<IntType>,
-             ArgSpec<IntType>,
-             ArgSpec<IntType>,
-             ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-            "mprotect",
-            RetValSpec<IntType>,
-            [ArgSpec<VoidPtr>,
-             ArgSpec<SizeTType>,
-             ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "munmap",
-            RetValSpec<IntType>,
-            [ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "posix_madvise",
-            RetValSpec<IntType>,
-            [ArgSpec<VoidPtr>,
-             ArgSpec<SizeTType>,
-             ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "mlock",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "munlock",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "mlockall",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "munlockall",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "msync",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "shm_open",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<ModeTType>]
-        >,
-        FunctionSpec<
-          "shm_unlink",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Signal = HeaderSpec<
-      "signal.h",
-      [], // Macros
-      [
-        SigInfoType,
-        SigSetType,
-        StackT,
-        StructSigaction,
-        UnionSigVal,
-        PidT,
-      ],
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "kill",
-          RetValSpec<IntType>,
-          [ArgSpec<PidT>,
-            ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "sigaction",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>,
-           ArgSpec<ConstRestrictedStructSigactionPtr>,
-           ArgSpec<RestrictedStructSigactionPtr>]
-        >,
-        FunctionSpec<
-          "sigaltstack",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedStackTPtr>, ArgSpec<RestrictedStackTPtr>]
-        >,
-        FunctionSpec<
-          "sigdelset",
-          RetValSpec<IntType>,
-          [ArgSpec<SigSetPtrType>,
-           ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "sigprocmask",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstRestrictedSigSetType>, ArgSpec<RestrictedSigSetType>]
-        >,
-        FunctionSpec<
-          "sigemptyset",
-          RetValSpec<IntType>,
-          [ArgSpec<SigSetPtrType>]
-        >,
-        FunctionSpec<
-          "sigaddset",
-          RetValSpec<IntType>,
-          [ArgSpec<SigSetPtrType>,
-           ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "sigfillset",
-          RetValSpec<IntType>,
-          [ArgSpec<SigSetPtrType>]
-        >,
-      ]
-  >;
-
-  HeaderSpec UniStd = HeaderSpec<
-    "unistd.h",
-    [], // Macros
-    [
-      ExecArgvT,
-      ExecEnvpT,
-      OffTType,
-      SSizeTType,
-      SizeTType,
-      PidT,
-      UidT,
-      GetoptArgvT,
-    ],
-    [], // Enumerations
-    [
-        FunctionSpec<
-          "_exit",
-          RetValSpec<NoReturn>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "access",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "chdir",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "dup",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "dup2",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "dup3",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "fchdir",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "getcwd",
-          RetValSpec<CharPtr>,
-          [ArgSpec<CharPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "close",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "execv",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ExecArgvT>]
-        >,
-        FunctionSpec<
-          "execve",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ExecArgvT>, ArgSpec<ExecEnvpT>]
-        >,
-        FunctionSpec<
-          "fork",
-          RetValSpec<PidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "fsync",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "ftruncate",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "geteuid",
-          RetValSpec<UidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getpid",
-          RetValSpec<PidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getppid",
-          RetValSpec<PidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "gettid",
-          RetValSpec<PidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getuid",
-          RetValSpec<UidT>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "isatty",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "link",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "linkat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "lseek",
-          RetValSpec<OffTType>,
-          [ArgSpec<IntType>, ArgSpec<OffTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "pread",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "pwrite",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "read",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "readlink",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "readlinkat",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "rmdir",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "getpid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "getppid",
-          RetValSpec<IntType>,
-          [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-          "link",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "linkat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "lseek",
-          RetValSpec<OffTType>,
-          [ArgSpec<IntType>, ArgSpec<OffTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "pipe",
-          RetValSpec<IntType>,
-          [ArgSpec<IntPtr>] //TODO: make this int[2]
-        >,
-        FunctionSpec<
-          "pread",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "pwrite",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "read",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "readlink",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "readlinkat",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "rmdir",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "swab",
-          RetValSpec<VoidType>,
-          [ArgSpec<ConstVoidRestrictedPtr>, ArgSpec<VoidPtr>, ArgSpec<SSizeTType>]
-        >,
-        FunctionSpec<
-          "symlink",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "symlinkat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>, ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "sysconf",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "__llvm_libc_syscall",
-          RetValSpec<LongType>,
-          [ArgSpec<LongType>,ArgSpec<LongType>,ArgSpec<LongType>,ArgSpec<LongType>,ArgSpec<LongType>,ArgSpec<LongType>,ArgSpec<LongType>]
-        >,
-        FunctionSpec<
-          "truncate",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<OffTType>]
-        >,
-        FunctionSpec<
-          "unlink",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-        FunctionSpec<
-          "unlinkat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "write",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-          "getopt",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<GetoptArgvT>, ArgSpec<ConstCharPtr>]
-        >,
-    ],
-    [
-        ObjectSpec<"environ", "char **">,
-        ObjectSpec<
-          "optarg",
-          "char *"
-        >,
-        ObjectSpec<
-          "optind",
-          "int"
-        >,
-        ObjectSpec<
-          "opterr",
-          "int"
-        >,
-        ObjectSpec<
-          "optopt",
-          "int"
-        >,
-    ]
-  >;
-
-  HeaderSpec StdLib = HeaderSpec<
-    "stdlib.h",
-    [], // Macros
-    [], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-          "getenv",
-          RetValSpec<CharPtr>,
-          [ArgSpec<ConstCharPtr>]
-        >,
-    ]
-  >;
-
-  HeaderSpec Sched = HeaderSpec<
-      "sched.h",
-      [], // Macros
-      [PidT, TimeTType, StructTimeSpec, StructSchedParam], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-            "sched_yield",
-            RetValSpec<IntType>,
-            [ArgSpec<VoidType>]
-        >,
-        FunctionSpec<
-            "sched_setparam",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<ConstStructSchedParamPtr>]
-        >,
-
-        FunctionSpec<
-            "sched_getparam",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<StructSchedParamPtr>]
-        >,
-
-        FunctionSpec<
-            "sched_setscheduler",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>]
-        >,
-
-        FunctionSpec<
-            "sched_getscheduler",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<IntType>, ArgSpec<ConstStructSchedParamPtr>]
-        >,
-
-        FunctionSpec<
-            "sched_get_priority_min",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-
-        FunctionSpec<
-            "sched_get_priority_max",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-
-        FunctionSpec<
-            "sched_rr_get_interval",
-            RetValSpec<IntType>,
-            [ArgSpec<PidT>, ArgSpec<StructTimeSpecPtr>]
-        >,
-      ]
-  >;
-
-  HeaderSpec String = HeaderSpec<
-    "string.h",
-    [
-        Macro<"NULL">,
-    ],
-    [
-        SizeTType,
-    ],
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "memccpy",
-            RetValSpec<VoidPtr>,
-            [ArgSpec<VoidRestrictedPtr>,
-             ArgSpec<ConstVoidRestrictedPtr>,
-             ArgSpec<IntType>,
-             ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "mempcpy",
-            RetValSpec<VoidPtr>,
-            [ArgSpec<VoidRestrictedPtr>,
-             ArgSpec<ConstVoidRestrictedPtr>,
-             ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "stpcpy",
-            RetValSpec<CharPtr>,
-            [ArgSpec<RestrictedCharPtr>,
-             ArgSpec<ConstRestrictedCharPtr>]
-        >,
-        FunctionSpec<
-            "stpncpy",
-            RetValSpec<CharPtr>,
-            [ArgSpec<RestrictedCharPtr>,
-             ArgSpec<ConstRestrictedCharPtr>,
-             ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strnlen",
-             RetValSpec<SizeTType>,
-             [ArgSpec<ConstCharPtr>, ArgSpec<SizeTType>]
-        >,
-        FunctionSpec<
-            "strtok_r",
-            RetValSpec<CharPtr>,
-            [ArgSpec<RestrictedCharPtr>,
-             ArgSpec<ConstRestrictedCharPtr>,
-             ArgSpec<CharRestrictedDoublePtr>]
-        >,
-        FunctionSpec<
-            "strsignal",
-            RetValSpec<CharPtr>,
-            [ArgSpec<IntType>]
-        >,
-    ]
-  >;
-
-  HeaderSpec CType = HeaderSpec<
-    "ctype.h",
-    [], // Macros
-    [], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "isascii",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>]
-        >,
-    ]
-  >;
-
-  NamedType RLimTType = NamedType<"rlim_t">;
-  NamedType StructRLimitType = NamedType<"struct rlimit">;
-  PtrType StructRLimitPtr = PtrType<StructRLimitType>;
-  ConstType ConstStructRLimitPtr = ConstType<StructRLimitType>;
-  HeaderSpec SysResource = HeaderSpec<
-    "sys/resource.h",
-    [], // Macros
-    [RLimTType, StructRLimitType], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-          "getrlimit",
-          RetValSpec<IntType>,
-          [ArgSpec<StructRLimitPtr>]
-        >,
-        FunctionSpec<
-          "setrlimit",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstStructRLimitPtr>]
-        >,
-    ]
-  >;
-
-  HeaderSpec SysStat = HeaderSpec<
-    "sys/stat.h",
-    [], // Macros
-    [
-        ModeTType,
-        DevT,
-        InoT,
-        UidT,
-        GidT,
-        StructTimeSpec,
-        StructTimevalType,
-        BlkSizeT,
-        BlkCntT,
-        OffTType,
-        NLinkT,
-        StatType,
-    ], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-          "chmod",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstCharPtr>, ArgSpec<ModeTType>]
-        >,
-        FunctionSpec<
-          "fchmod",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ModeTType>]
-        >,
-        FunctionSpec<
-          "fchmodat",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<ModeTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-            "fstat",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>, ArgSpec<StatTypePtr>]
-        >,
-        FunctionSpec<
-            "lstat",
-            RetValSpec<IntType>,
-            [ArgSpec<ConstRestrictedCharPtr>, ArgSpec<RestrictedStatTypePtr>]
-        >,
-        FunctionSpec<
-            "mkdir",
-            RetValSpec<IntType>,
-            [ArgSpec<ConstCharPtr>, ArgSpec<ModeTType>]
-        >,
-        FunctionSpec<
-            "mkdirat",
-            RetValSpec<IntType>,
-            [ArgSpec<IntType>, ArgSpec<ConstCharPtr>, ArgSpec<ModeTType>]
-        >,
-        FunctionSpec<
-            "stat",
-            RetValSpec<IntType>,
-            [ArgSpec<ConstRestrictedCharPtr>, ArgSpec<RestrictedStatTypePtr>]
-        >,
-    ]
-  >;
-
-  HeaderSpec SysStatvfs = HeaderSpec<
-      "sys/statvfs.h",
-      [], // Macros
-      [StructStatvfs], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "statvfs",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<ConstRestrictedCharPtr>,
-            ArgSpec<RestrictedStructStatvfsPtr>
-          ]
-        >,
-        FunctionSpec<
-          "fstatvfs",
-          RetValSpec<IntType>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<StructStatvfsPtr>
-          ]
-        >,
-      ]  // Functions
-    >;
-
-  NamedType StructUtsName = NamedType<"struct utsname">;
-  PtrType StructUtsNamePtr = PtrType<StructUtsName>;
-  HeaderSpec SysUtsName = HeaderSpec<
-    "sys/utsname.h",
-    [], // Macros
-    [StructUtsName], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "uname",
-            RetValSpec<IntType>,
-            [ArgSpec<StructUtsNamePtr>]
-        >,
-    ]
-  >;
-
-  HeaderSpec ArpaInet = HeaderSpec<
-      "arpa/inet.h",
-      [], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "htonl",
-              RetValSpec<UInt32Type>,
-              [ArgSpec<UInt32Type>]
-          >,
-          FunctionSpec<
-              "htons",
-              RetValSpec<UInt16Type>,
-              [ArgSpec<UInt16Type>]
-          >,
-          FunctionSpec<
-              "ntohl",
-              RetValSpec<UInt32Type>,
-              [ArgSpec<UInt32Type>]
-          >,
-          FunctionSpec<
-              "ntohs",
-              RetValSpec<UInt16Type>,
-              [ArgSpec<UInt16Type>]
-          >,
-      ]
-  >;
-
-  HeaderSpec PThread = HeaderSpec<
-    "pthread.h",
-    [], // Macros
-    [
-        AtForkCallbackT,
-        ClockIdT,
-        PThreadAttrTType,
-        PThreadCondAttrTType,
-        PThreadKeyT,
-        PThreadMutexAttrTType,
-        PThreadMutexTType,
-        PThreadOnceCallback,
-        PThreadOnceT,
-        PThreadRWLockAttrTType,
-        PThreadRWLockTType,
-        PThreadSpinLockTType,
-        PThreadStartT,
-        PThreadTSSDtorT,
-        PThreadTType,
-    ], // Types
-    [], // Enumerations
-    [
-      FunctionSpec<
-          "pthread_atfork",
-          RetValSpec<IntType>,
-          [ArgSpec<AtForkCallbackT>, ArgSpec<AtForkCallbackT>, ArgSpec<AtForkCallbackT>]
-      >,
-      FunctionSpec<
-          "pthread_attr_init",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_destroy",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_getdetachstate",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstPThreadAttrTPtr>, ArgSpec<IntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_setdetachstate",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_attr_getguardsize",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadAttrTPtr>, ArgSpec<RestrictedSizeTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_setguardsize",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>, ArgSpec<SizeTType>]
-      >,
-      FunctionSpec<
-          "pthread_attr_getstacksize",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadAttrTPtr>, ArgSpec<RestrictedSizeTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_setstacksize",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>, ArgSpec<SizeTType>]
-      >,
-      FunctionSpec<
-          "pthread_attr_getstack",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadAttrTPtr>, ArgSpec<RestrictedVoidPtrPtr>, ArgSpec<RestrictedSizeTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_attr_setstack",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadAttrTPtr>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_destroy",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadCondAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_getclock",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadCondAttrTPtr>, ArgSpec<RestrictedClockIdTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_getpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadCondAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_init",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadCondAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_setclock",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadCondAttrTPtr>, ArgSpec<ClockIdT>]
-      >,
-      FunctionSpec<
-          "pthread_condattr_setpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadCondAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_create",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadTPtr>, ArgSpec<ConstRestrictedPThreadAttrTPtr>, ArgSpec<PThreadStartT>, ArgSpec<VoidPtr>]
-      >,
-      FunctionSpec<
-          "pthread_join",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadTType>, ArgSpec<VoidPtrPtr>]
-      >,
-      FunctionSpec<
-          "pthread_detach",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadTType>]
-      >,
-      FunctionSpec<
-          "pthread_exit",
-          RetValSpec<NoReturn>,
-          [ArgSpec<VoidPtr>]
-      >,
-      FunctionSpec<
-          "pthread_self",
-          RetValSpec<PThreadTType>,
-          [ArgSpec<VoidType>]
-      >,
-      FunctionSpec<
-          "pthread_equal",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadTType>, ArgSpec<PThreadTType>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_init",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadMutexAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_destroy",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadMutexAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_gettype",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_settype",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_getrobust",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_setrobust",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_getpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_setpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_getprotocol",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_setprotocol",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_getprioceiling",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>, ArgSpec<RestrictedIntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutexattr_setprioceiling",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_mutex_init",
-          RetValSpec<IntType>,
-          [ArgSpec<RestrictedPThreadMutexTPtr>, ArgSpec<ConstRestrictedPThreadMutexAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutex_destroy",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadMutexTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutex_lock",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadMutexTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_mutex_unlock",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadMutexTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_key_create",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadKeyTPtr>, ArgSpec<PThreadTSSDtorT>]
-      >,
-      FunctionSpec<
-          "pthread_key_delete",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadKeyT>]
-      >,
-      FunctionSpec<
-          "pthread_getspecific",
-          RetValSpec<VoidPtr>,
-          [ArgSpec<PThreadKeyT>]
-      >,
-      FunctionSpec<
-          "pthread_setspecific",
-          RetValSpec<VoidPtr>,
-          [ArgSpec<PThreadKeyT>, ArgSpec<ConstVoidPtr>]
-      >,
-      FunctionSpec<
-          "pthread_once",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadOnceTPtr>, ArgSpec<PThreadOnceCallback>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_destroy",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_getkind_np",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_getpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<ConstPThreadRWLockAttrTPtr>, ArgSpec<IntPtr>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_init",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_setkind_np",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-          "pthread_rwlockattr_setpshared",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_init",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>, ArgSpec<ConstRestrictedPThreadRWLockAttrTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_tryrdlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_trywrlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_timedrdlock",
-        RetValSpec<IntType>,
-        [ArgSpec<RestrictedPThreadRWLockTPtr>, ArgSpec<ConstRestrictStructTimeSpecPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_timedwrlock",
-        RetValSpec<IntType>,
-        [ArgSpec<RestrictedPThreadRWLockTPtr>, ArgSpec<ConstRestrictStructTimeSpecPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_clockrdlock",
-        RetValSpec<IntType>,
-        [ArgSpec<RestrictedPThreadRWLockTPtr>, ArgSpec<ClockIdT>, ArgSpec<ConstRestrictStructTimeSpecPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_clockwrlock",
-        RetValSpec<IntType>,
-        [ArgSpec<RestrictedPThreadRWLockTPtr>, ArgSpec<ClockIdT>, ArgSpec<ConstRestrictStructTimeSpecPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_rdlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_wrlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_unlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_rwlock_destroy",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadRWLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_spin_init",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadSpinLockTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "pthread_spin_destroy",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadSpinLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_spin_lock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadSpinLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_spin_trylock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadSpinLockTPtr>]
-      >,
-      FunctionSpec<
-        "pthread_spin_unlock",
-        RetValSpec<IntType>,
-        [ArgSpec<PThreadSpinLockTPtr>]
-      >
-    ]
-  >;
-
-  HeaderSpec StdIO = HeaderSpec<
-      "stdio.h",
-      [], // Macros
-      [OffTType], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "flockfile",
-              RetValSpec<VoidType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "funlockfile",
-              RetValSpec<VoidType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "getc_unlocked",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "getchar_unlocked",
-              RetValSpec<IntType>,
-              [ArgSpec<VoidType>]
-          >,
-          FunctionSpec<
-            "fileno",
-            RetValSpec<IntType>,
-            [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-            "fdopen",
-            RetValSpec<FILEPtr>,
-            [ArgSpec<IntType>, ArgSpec<ConstCharPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Dirent = HeaderSpec<
-      "dirent.h",
-      [], // Macros
-      [InoT, StructDirent, DIR], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "alphasort",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstStructDirentPtrPtr>, ArgSpec<ConstStructDirentPtrPtr>]
-          >,
-          FunctionSpec<
-              "closedir",
-              RetValSpec<IntType>,
-              [ArgSpec<DIRPtr>]
-          >,
-          FunctionSpec<
-              "dirfd",
-              RetValSpec<IntType>,
-              [ArgSpec<DIRPtr>]
-          >,
-          FunctionSpec<
-              "fdopendir",
-              RetValSpec<DIRPtr>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "opendir",
-              RetValSpec<DIRPtr>,
-              [ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "readdir",
-              RetValSpec<StructDirentPtr>,
-              [ArgSpec<DIRPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Time = HeaderSpec<
-      "time.h",
-      [], // Macros
-      [ClockIdT, StructTimeSpec, StructTimevalType], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "clock_gettime",
-              RetValSpec<IntType>,
-              [ArgSpec<ClockIdT>, ArgSpec<StructTimeSpecPtr>]
-          >,
-          FunctionSpec<
-              "gettimeofday",
-              RetValSpec<IntType>,
-              [ArgSpec<StructTimevalPtr>, ArgSpec<VoidPtr>]
-          >,
-          FunctionSpec<
-              "nanosleep",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstStructTimeSpecPtr>, ArgSpec<StructTimeSpecPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec SysWait = HeaderSpec<
-    "sys/wait.h",
-    [], // Macros
-    [PidT, StructRUsage, SigInfoType],
-    [], // Enumerations
-    [
-      FunctionSpec<
-        "wait",
-        RetValSpec<PidT>,
-        [ArgSpec<IntPtr>]
-      >,
-      FunctionSpec<
-        "waitpid",
-        RetValSpec<PidT>,
-        [ArgSpec<PidT>, ArgSpec<IntPtr>, ArgSpec<IntType>]
-      >
-    ]
-  >;
-
-  HeaderSpec SysIOctl = HeaderSpec<
-    "sys/ioctl.h",
-    [
-      Macro<"TIOCGETD">,
-    ],  // Macros
-    [], // Types
-    [], // Enumerations
-    []  // Functions
-  >;
-
-  HeaderSpec Spawn = HeaderSpec<
-    "spawn.h",
-    [], // Macros
-    [ModeTType, PosixSpawnAttrT, PidT, PosixSpawnFileActionsT],
-    [], // Enumerations
-    [
-      FunctionSpec<
-        "posix_spawn_file_actions_addclose",
-        RetValSpec<IntType>,
-        [ArgSpec<PosixSpawnFileActionsTPtr>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "posix_spawn_file_actions_adddup2",
-        RetValSpec<IntType>,
-        [ArgSpec<PosixSpawnFileActionsTPtr>, ArgSpec<IntType>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "posix_spawn_file_actions_addopen",
-        RetValSpec<IntType>,
-        [ArgSpec<PosixSpawnFileActionsTRestrictedPtr>, ArgSpec<IntType>,
-         ArgSpec<ConstCharRestrictedPtr>, ArgSpec<IntType>, ArgSpec<ModeTType>]
-      >,
-      FunctionSpec<
-        "posix_spawn_file_actions_destroy",
-        RetValSpec<IntType>,
-        [ArgSpec<PosixSpawnFileActionsTPtr>]
-      >,
-      FunctionSpec<
-        "posix_spawn_file_actions_init",
-        RetValSpec<IntType>,
-        [ArgSpec<PosixSpawnFileActionsTPtr>]
-      >,
-      FunctionSpec<
-        "posix_spawn",
-        RetValSpec<IntType>,
-        [ArgSpec<RestrictedPidTPtr>, ArgSpec<ConstCharRestrictedPtr>,
-         ArgSpec<PosixSpawnFileActionsTPtr>, ArgSpec<RestrictedPosixSpawnAttrTPtrType>,
-         ArgSpec<ConstCharRestrictedPtrPtr>, ArgSpec<ConstCharRestrictedPtrPtr>]
-      >,
-    ]
-  >;
-
-  HeaderSpec Search = HeaderSpec<
-    "search.h",
-    [], // Macros
-    [
-        ActionType,
-        EntryType,
-        LSearchCompareT,
-    ], // Types
-    [], // Enumerations
-    [
-        FunctionSpec<
-            "hcreate",
-            RetValSpec<IntType>,
-            [
-                ArgSpec<SizeTType>
-            ]
-        >,
-        FunctionSpec<
-            "hdestroy",
-            RetValSpec<VoidType>,
-            [] // Args
-        >,
-        FunctionSpec<
-            "hsearch",
-            RetValSpec<EntryTypePtr>,
-            [
-                ArgSpec<EntryType>,
-                ArgSpec<ActionType>
-            ]
-        >,
-        FunctionSpec<
-            "insque",
-            RetValSpec<VoidType>,
-            [
-                ArgSpec<VoidPtr>,
-                ArgSpec<VoidPtr>
-            ]
-        >,
-        FunctionSpec<
-            "remque",
-            RetValSpec<VoidType>,
-            [
-                ArgSpec<VoidPtr>
-            ]
-        >,
-        FunctionSpec<
-            "lfind",
-            RetValSpec<VoidPtr>,
-            [
-                ArgSpec<ConstVoidPtr>,
-                ArgSpec<ConstVoidPtr>,
-                ArgSpec<SizeTPtr>,
-                ArgSpec<SizeTType>,
-                ArgSpec<LSearchCompareT>
-            ]
-        >
-    ]
-  >;
-
-  HeaderSpec Termios = HeaderSpec<
-    "termios.h",
-    [
-      Macro<"NCCS">,
-    ],
-    [CcT, PidT, SpeedT, StructTermios, TcFlagT], // Types
-    [], // Enumerations
-    [
-      FunctionSpec<
-        "cfgetispeed",
-        RetValSpec<SpeedT>,
-        [ArgSpec<ConstStructTermiosPtr>]
-      >,
-      FunctionSpec<
-        "cfgetospeed",
-        RetValSpec<SpeedT>,
-        [ArgSpec<ConstStructTermiosPtr>]
-      >,
-      FunctionSpec<
-        "cfsetispeed",
-        RetValSpec<SpeedT>,
-        [ArgSpec<StructTermiosPtr>, ArgSpec<SpeedT>]
-      >,
-      FunctionSpec<
-        "cfsetospeed",
-        RetValSpec<SpeedT>,
-        [ArgSpec<StructTermiosPtr>, ArgSpec<SpeedT>]
-      >,
-      FunctionSpec<
-        "tcdrain",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "tcflow",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "tcflush",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "tcgetattr",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>, ArgSpec<StructTermiosPtr>]
-      >,
-      FunctionSpec<
-        "tcgetsid",
-        RetValSpec<PidT>,
-        [ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "tcsendbreak",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>, ArgSpec<IntType>]
-      >,
-      FunctionSpec<
-        "tcsetattr",
-        RetValSpec<IntType>,
-        [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<StructTermiosPtr>]
-      >,
-    ]
-  >;
-
-  HeaderSpec SysSelect = HeaderSpec<
-    "sys/select.h",
-    [], // Macros
-    [FdSet, SigSetType, StructTimevalType, StructTimeSpec, SuSecondsT, TimeTType],
-    [], // Enumerations
-    [
-      FunctionSpec<
-        "select",
-        RetValSpec<IntType>,
-        [
-            ArgSpec<IntType>, ArgSpec<RestrictedFdSetPtr>, ArgSpec<RestrictedFdSetPtr>,
-            ArgSpec<RestrictedFdSetPtr>, ArgSpec<RestrictedStructTimevalPtr>
-        ]
-      >
-    ]
-  >;
-
-  HeaderSpec SysSocket = HeaderSpec<
-      "sys/socket.h",
-      [
-        Macro<"AF_UNSPEC">,
-        Macro<"AF_UNIX">,
-        Macro<"AF_LOCAL">,
-        Macro<"AF_INET">,
-        Macro<"AF_INET6">,
-        Macro<"SOCK_STREAM">,
-        Macro<"SOCK_DGRAM">,
-        Macro<"SOCK_RAW">,
-        Macro<"SOCK_RDM">,
-        Macro<"SOCK_SEQPACKET">,
-        Macro<"SOCK_PACKET">,
-      ], // Macros
-      [
-        SizeTType,
-        SSizeTType,
-        SAFamilyType,
-        StructSockAddr,
-        StructSockAddrUn,
-        SocklenType,
-        StructIovec,
-        StructMsghdr,
-      ], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "socket",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "socketpair",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntPtr>]
-        >,
-        FunctionSpec<
-          "bind",
-          RetValSpec<IntType>,
-          [ArgSpec<IntType>, ArgSpec<ConstStructSockAddrPtr>, ArgSpec<SocklenType>]
-        >,
-        FunctionSpec<
-          "send",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "sendto",
-          RetValSpec<SSizeTType>,
-          [
-            ArgSpec<IntType>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>,
-            ArgSpec<IntType>, ArgSpec<ConstStructSockAddrPtr>,
-            ArgSpec<SocklenType>
-          ]
-        >,
-        FunctionSpec<
-          "sendmsg",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<ConstStructMsghdrPtr>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "recv",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<IntType>]
-        >,
-        FunctionSpec<
-          "recvfrom",
-          RetValSpec<SSizeTType>,
-          [
-            ArgSpec<IntType>, ArgSpec<VoidPtr>, ArgSpec<SizeTType>,
-            ArgSpec<IntType>, ArgSpec<StructSockAddrPtr>,
-            ArgSpec<SocklenPtr>
-          ]
-        >,
-        FunctionSpec<
-          "recvmsg",
-          RetValSpec<SSizeTType>,
-          [ArgSpec<IntType>, ArgSpec<StructMsghdrPtr>, ArgSpec<IntType>]
-        >,
-      ]  // Functions
-  >;
-
-  HeaderSpec SysTypes = HeaderSpec<
-    "sys/types.h",
-    [], // Macros
-    [
-      BlkCntT,
-      BlkSizeT,
-      ClockIdT,
-      DevT,
-      GidT,
-      InoT,
-      ModeTType,
-      NLinkT,
-      OffTType,
-      PThreadAttrTType,
-      PThreadCondAttrTType,
-      PThreadKeyT,
-      PThreadMutexAttrTType,
-      PThreadMutexTType,
-      PThreadOnceT,
-      PThreadRWLockAttrTType,
-      PThreadRWLockTType,
-      PThreadTType,
-      PidT,
-      SSizeTType,
-      SizeTType,
-      SuSecondsT,
-      TimeTType,
-      UidT
-    ], // Types
-    [], // Enumerations
-    []  // Functions
-  >;
-
-  let Headers = [
-    ArpaInet,
-    CType,
-    Dirent,
-    DlFcn,
-    Errno,
-    FCntl,
-    PThread,
-    Sched,
-    Signal,
-    Spawn,
-    StdIO,
-    StdLib,
-    SysIOctl,
-    SysMMan,
-    SysResource,
-    SysSelect,
-    SysSocket,
-    SysStat,
-    SysStatvfs,
-    SysTypes,
-    SysUtsName,
-    SysWait,
-    Time,
-    Termios,
-    UniStd,
-    String,
-    Search,
-  ];
-}
diff --git a/libc/spec/spec.td b/libc/spec/spec.td
deleted file mode 100644
index ad9ca76846c28..0000000000000
--- a/libc/spec/spec.td
+++ /dev/null
@@ -1,253 +0,0 @@
-class Type {}
-
-class NamedType<string name> : Type {
-  string Name = name;
-}
-
-class Field<string name, Type type> {
-  string Name = name;
-  Type FieldType = type;
-}
-
-// Class to describe concrete structs specified by a standard.
-class Struct<string name> : NamedType<name> {
-  list<Field> Fields;
-}
-
-class EnumNameValue<string name, string value = "__default_enum_value__"> {
-  string Name = name;
-  string Value = value;
-}
-
-class Enum<string name, list<EnumNameValue> enumerations> : NamedType<name> {
-  list<EnumNameValue> Enumerations = enumerations;
-}
-
-class PtrType<Type type> : Type {
-  Type PointeeType = type;
-}
-
-class ConstType<Type type> : Type {
-  Type UnqualifiedType = type;
-}
-
-class RestrictedPtrType<Type type> : Type {
-  Type PointeeType = type;
-}
-
-// Builtin types.
-def VarArgType : NamedType<"...">;
-def VaListType : NamedType<"va_list">;
-def VoidType : NamedType<"void">;
-def IntType : NamedType<"int">;
-def UnsignedIntType : NamedType<"unsigned int">;
-def LongType : NamedType<"long">;
-def UnsignedLongType : NamedType<"unsigned long">;
-def LongLongType : NamedType<"long long">;
-def UnsignedLongLongType : NamedType<"unsigned long long">;
-def FloatType : NamedType<"float">;
-def DoubleType : NamedType<"double">;
-def LongDoubleType : NamedType<"long double">;
-def CharType : NamedType<"char">;
-def UnsignedCharType : NamedType<"unsigned char">;
-def UnsignedShortType : NamedType<"unsigned short">;
-def BoolType : NamedType<"bool">;
-
-def Float16Type : NamedType<"_Float16">;
-def Float128Type : NamedType<"float128">;
-
-// Common types
-def VoidPtr : PtrType<VoidType>;
-def VoidPtrPtr : PtrType<VoidPtr>;
-def RestrictedVoidPtrPtr : RestrictedPtrType<VoidPtr>;
-def ConstVoidPtr : ConstType<VoidPtr>;
-
-def SizeTType : NamedType<"size_t">;
-def SizeTPtr : PtrType<SizeTType>;
-def RestrictedSizeTPtr : RestrictedPtrType<SizeTType>;
-
-def Char8TType : NamedType<"char8_t">;
-def Char16TType : NamedType<"char16_t">;
-def Char32TType : NamedType<"char32_t">;
-def WCharType : NamedType<"wchar_t">;
-def WIntType : NamedType<"wint_t">;
-
-def LongDoublePtr : PtrType<LongDoubleType>;
-
-def IntMaxTType : NamedType<"intmax_t">;
-def UIntMaxTType : NamedType<"uintmax_t">;
-
-def UInt16Type : NamedType<"uint16_t">;
-def UInt32Type : NamedType<"uint32_t">;
-
-def OffTType : NamedType<"off_t">;
-def OffTPtr : PtrType<OffTType>;
-def SSizeTType : NamedType<"ssize_t">;
-
-// _Noreturn is really not a type, but it is convenient to treat it as a type.
-def NoReturn : NamedType<"_Noreturn void">;
-
-//types moved from stdc.td
-def VoidRestrictedPtr : RestrictedPtrType<VoidType>;
-def ConstVoidRestrictedPtr : ConstType<VoidRestrictedPtr>;
-
-def CharPtr : PtrType<CharType>;
-def ConstCharPtr : ConstType<CharPtr>;
-def CharRestrictedPtr : RestrictedPtrType<CharType>;
-def CharRestrictedPtrPtr : RestrictedPtrType<CharPtr>;
-def ConstCharRestrictedPtr : ConstType<CharRestrictedPtr>;
-def ConstCharRestrictedPtrPtr : PtrType<ConstCharRestrictedPtr>;
-
-def OnceFlagType : NamedType<"once_flag">;
-def OnceFlagTypePtr : PtrType<OnceFlagType>;
-// TODO(sivachandra): Remove this non-standard type when a formal
-// way to describe callable types is available.
-def CallOnceFuncType : NamedType<"__call_once_func_t">;
-def MtxTType : NamedType<"mtx_t">;
-def MtxTTypePtr : PtrType<MtxTType>;
-def CndTType : NamedType<"cnd_t">;
-def CndTTypePtr : PtrType<CndTType>;
-def ThrdStartTType : NamedType<"thrd_start_t">;
-def ThrdTType : NamedType<"thrd_t">;
-def ThrdTTypePtr : PtrType<ThrdTType>;
-
-def IntPtr : PtrType<IntType>;
-def RestrictedIntPtr : RestrictedPtrType<IntType>;
-def FloatPtr : PtrType<FloatType>;
-def DoublePtr : PtrType<DoubleType>;
-def Float16Ptr : PtrType<Float16Type>;
-def Float128Ptr : PtrType<Float128Type>;
-def UnsignedCharPtr : PtrType<UnsignedCharType>;
-
-def ConstDoublePtr : ConstType<DoublePtr>;
-def ConstFloatPtr : ConstType<FloatPtr>;
-def ConstLongDoublePtr : ConstType<LongDoublePtr>;
-def ConstFloat16Ptr : ConstType<Float16Ptr>;
-def ConstFloat128Ptr : ConstType<Float128Ptr>;
-
-def SigHandlerT : NamedType<"__sighandler_t">;
-
-def TimeTType : NamedType<"time_t">;
-
-def StructTimeSpec : NamedType<"struct timespec">;
-def StructTimeSpecPtr : PtrType<StructTimeSpec>;
-def ConstStructTimeSpecPtr : ConstType<StructTimeSpecPtr>;
-def RestrictStructTimeSpecPtr : RestrictedPtrType<StructTimeSpec>;
-def ConstRestrictStructTimeSpecPtr : ConstType<RestrictStructTimeSpecPtr>;
-
-def BSearchCompareT : NamedType<"__bsearchcompare_t">;
-def QSortCompareT : NamedType<"__qsortcompare_t">;
-
-def AtexitHandlerT : NamedType<"__atexithandler_t">;
-
-def FILE : NamedType<"FILE">;
-def FILEPtr : PtrType<FILE>;
-def FILERestrictedPtr : RestrictedPtrType<FILE>;
-
-def PThreadTType : NamedType<"pthread_t">;
-
-def PidT : NamedType<"pid_t">;
-def RestrictedPidTPtr : RestrictedPtrType<PidT>;
-
-def StructRUsage : NamedType<"struct rusage">;
-def StructRUsagePtr : PtrType<StructRUsage>;
-
-def StructTimevalType : NamedType<"struct timeval">;
-def StructTimevalPtr : PtrType<StructTimevalType>;
-def RestrictedStructTimevalPtr : RestrictedPtrType<StructTimevalType>;
-
-def SuSecondsT : NamedType<"suseconds_t">;
-
-//added because __assert_fail needs it.
-def UnsignedType : NamedType<"unsigned">;
-
-def ActionType : NamedType<"ACTION">;
-def EntryType : NamedType<"ENTRY">;
-def EntryTypePtr : PtrType<EntryType>;
-def EntryTypePtrPtr : PtrType<EntryTypePtr>;
-
-def MBStateTType : NamedType<"mbstate_t">;
-
-class Macro<string name> {
-  string Name = name;
-}
-
-class EnumeratedNameValue<string name, string value = "__default__"> {
-  string Name = name;
-  string Value = value;
-}
-
-class Annotation {}
-
-class RetValSpec<Type type, list<Annotation> annotations = []> {
-  Type ReturnType = type;
-  list<Annotation> Annotations = annotations;
-}
-
-class ArgSpec<Type type, list<Annotation> annotations = [], string name = ""> {
-  Type ArgType = type;
-  list<Annotation> Annotations = annotations;
-  string Name = name;
-}
-
-// The following classes are used to describe function attributes.
-// In the future, we may consider supporting parameter attributes as well.
-// https://clang.llvm.org/docs/AttributeReference.html
-class FunctionAttr<string style, string attr> {
-  string Attr = attr;
-  // The style of the attribute, e.g. "gnu", "cxx11", "declspec".
-  // - "gnu" is for GNU-style attributes: __attribute__((...))
-  // - "cxx11" is for C++11-style attributes: [[...]]
-  // - "declspec" is for Microsoft-style attributes: __declspec(...)
-  string Style = style;
-
-  // For the time being, we are only interested in identifer-like attributes.
-  // We can extend this to support function-like attributes if needed.
-  // For example, in the future, we can #define __LIBC_ATTRIBUTE_NODISCARD(...) [[nodiscard(__VA_ARGS__)]]
-  // int FunctionLike = 0;
-}
-class GnuFunctionAttr<string attr> : FunctionAttr<"gnu", attr> {}
-class Cxx11FunctionAttr<string attr, string namespace> : FunctionAttr<"cxx11", attr> {
-  // The namespace of the attribute, e.g. "gnu" or "clang". Empty string means there is no namespace.
-  string Namespace = namespace;
-}
-class DeclspecFunctionAttr<string attr> : FunctionAttr<"declspec", attr> {}
-class FunctionAttrSpec<string macro, list<FunctionAttr> instances> {
-  list<FunctionAttr> Instances = instances;
-  string Macro = macro;
-}
-
-class FunctionSpec<string name, RetValSpec return, list<ArgSpec> args, list<FunctionAttrSpec> attrs = []> {
-  string Name = name;
-  RetValSpec Return = return;
-  list<ArgSpec> Args = args;
-  list<FunctionAttrSpec> Attributes = attrs;
-}
-
-class GuardedFunctionSpec<string name, RetValSpec return, list<ArgSpec> args, string guard_macro> : FunctionSpec<name, return, args> {
-  string Guard = guard_macro;
-}
-
-class ObjectSpec<string name, string type> {
-  string Name = name;
-  string Type = type;
-}
-
-class HeaderSpec<string name,
-                list<Macro> macros = [],
-                list<Type> types = [],
-                list<EnumeratedNameValue> enumerations = [],
-                list<FunctionSpec> functions = [],
-                list<ObjectSpec> objects = []> {
-  string Name = name;
-  list<FunctionSpec> Functions = functions;
-  list<Type> Types = types;
-  list<Macro> Macros = macros;
-  list<EnumeratedNameValue> Enumerations = enumerations;
-  list<ObjectSpec> Objects = objects;
-}
-
-class StandardSpec<string name> {
-  string Name = name;
-  list<HeaderSpec> Headers;
-}
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
deleted file mode 100644
index 4fa057da1cf13..0000000000000
--- a/libc/spec/stdc.td
+++ /dev/null
@@ -1,1819 +0,0 @@
-def StdC : StandardSpec<"stdc"> {
-
-  NamedType StructTmType = NamedType<"struct tm">;
-  PtrType StructTmPtr = PtrType<StructTmType>;
-  PtrType TimeTTypePtr = PtrType<TimeTType>;
-  NamedType ClockT = NamedType<"clock_t">;
-  NamedType LocaleT = NamedType<"locale_t">;
-
-  NamedType DivTType = NamedType<"div_t">;
-  NamedType LDivTType = NamedType<"ldiv_t">;
-  NamedType LLDivTType = NamedType<"lldiv_t">;
-
-  NamedType JmpBuf = NamedType<"jmp_buf">;
-
-  NamedType TssTType = NamedType<"tss_t">;
-  PtrType TssTPtr = PtrType<TssTType>;
-  NamedType TssDtorTType = NamedType<"tss_dtor_t">;
-
-  HeaderSpec Assert = HeaderSpec<
-      "assert.h",
-      [
-          Macro<"static_assert">,
-          Macro<"assert">,
-      ],
-      [], // Types
-      [], // Enumerations
-      []
-  >;
-
-  FunctionAttrSpec ConstAttr = FunctionAttrSpec<"__LIBC_CONST_ATTR", [
-    Cxx11FunctionAttr<"const", "gnu">,
-    GnuFunctionAttr<"const">,
-  ]>;
-
-  HeaderSpec CType = HeaderSpec<
-      "ctype.h",
-      [], // Macros
-      [
-          LocaleT
-      ], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "isalnum",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isalpha",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isblank",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "iscntrl",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isdigit",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isgraph",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "islower",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isprint",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "ispunct",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isspace",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isupper",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isxdigit",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "tolower",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "toupper",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "isalnum_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isalpha_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isblank_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "iscntrl_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isdigit_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isgraph_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "islower_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isprint_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "ispunct_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isspace_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isupper_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "isxdigit_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "tolower_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "toupper_l",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<LocaleT>]
-          >,
-      ]
-  >;
-
-  NamedType FEnvT = NamedType<"fenv_t">;
-  PtrType FEnvTPtr = PtrType<FEnvT>;
-  ConstType ConstFEnvTPtr = ConstType<FEnvTPtr>;
-  NamedType FExceptT = NamedType<"fexcept_t">;
-  PtrType FExceptTPtr = PtrType<FExceptT>;
-  ConstType ConstFExceptTPtr = ConstType<FExceptTPtr>;
-  HeaderSpec Fenv = HeaderSpec<
-      "fenv.h",
-      [
-          Macro<"FE_DIVBYZERO">,
-          Macro<"FE_INEXACT">,
-          Macro<"FE_INVALID">,
-          Macro<"FE_OVERFLOW">,
-          Macro<"FE_UNDERFLOW">,
-          Macro<"FE_ALL_EXCEPT">,
-
-          Macro<"FE_DOWNWARD">,
-          Macro<"FE_TONEAREST">,
-          Macro<"FE_TOWARDZERO">,
-          Macro<"FE_UPWARD">,
-
-          Macro<"FE_DFL_ENV">
-      ],
-      [
-          FEnvT,
-          FExceptT,
-      ], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "feclearexcept",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fetestexcept",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fetestexceptflag",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstFExceptTPtr>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "feraiseexcept",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fesetround",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fegetround",
-              RetValSpec<IntType>,
-              []
-          >,
-          FunctionSpec<
-              "fegetenv",
-              RetValSpec<IntType>,
-              [ArgSpec<FEnvTPtr>]
-          >,
-          FunctionSpec<
-              "fesetenv",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstFEnvTPtr>]
-          >,
-          FunctionSpec<
-              "fegetexceptflag",
-              RetValSpec<IntType>,
-              [ArgSpec<FExceptTPtr>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fesetexcept",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fesetexceptflag",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstFExceptTPtr>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "feholdexcept",
-              RetValSpec<IntType>,
-              [ArgSpec<FEnvTPtr>]
-          >,
-          FunctionSpec<
-              "feupdateenv",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstFEnvTPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec String = HeaderSpec<
-      "string.h",
-      [
-          Macro<"NULL">,
-      ],
-      [
-          SizeTType,
-      ],
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "memcpy",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<VoidRestrictedPtr>,
-               ArgSpec<ConstVoidRestrictedPtr>,
-               ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "memmove",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<VoidPtr>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "memcmp",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstVoidPtr>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "memchr",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<ConstVoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "memset",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<VoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "memset_explicit",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<VoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strcpy",
-              RetValSpec<CharPtr>,
-              [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>]
-          >,
-          FunctionSpec<
-              "strncpy",
-              RetValSpec<CharPtr>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strcat",
-              RetValSpec<CharPtr>,
-              [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>]
-          >,
-          FunctionSpec<
-              "strncat",
-              RetValSpec<CharPtr>,
-              [ArgSpec<CharPtr>, ArgSpec<ConstCharPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strcmp",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strcoll",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strcoll_l",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>, ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "strncmp",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strxfrm",
-              RetValSpec<SizeTType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strxfrm_l",
-              RetValSpec<SizeTType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<LocaleT>]
-          >,
-          FunctionSpec<
-              "strchr",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "strcspn",
-              RetValSpec<SizeTType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strdup",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strndup",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>,ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "strpbrk",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strrchr",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "strspn",
-              RetValSpec<SizeTType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strstr",
-              RetValSpec<CharPtr>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "strtok",
-              RetValSpec<CharPtr>,
-              [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>]
-          >,
-          FunctionSpec<
-              "strerror",
-              RetValSpec<CharPtr>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "strlen",
-              RetValSpec<SizeTType>,
-              [ArgSpec<ConstCharPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Math = HeaderSpec<
-      "math.h",
-      [
-          Macro<"MATH_ERRNO">,
-          Macro<"MATH_ERREXCEPT">,
-          Macro<"math_errhandling">,
-
-          Macro<"HUGE_VAL">,
-          Macro<"INFINITY">,
-          Macro<"NAN">,
-
-          Macro<"FP_INT_UPWARD">,
-          Macro<"FP_INT_DOWNWARD">,
-          Macro<"FP_INT_TOWARDZERO">,
-          Macro<"FP_INT_TONEARESTFROMZERO">,
-          Macro<"FP_INT_TONEAREST">,
-
-          Macro<"FP_ILOGB0">,
-          Macro<"FP_ILOGBNAN">,
-
-          Macro<"isfinite">,
-          Macro<"isinf">,
-          Macro<"isnan">,
-      ],
-      [
-          NamedType<"float_t">,
-          NamedType<"double_t">,
-          NamedType<"float128">,
-      ],
-      [], // Enumerations
-      [
-          FunctionSpec<"cbrt", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"cbrtf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"copysign", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"copysignf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"copysignl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"copysignf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"copysignf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"ceil", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"ceilf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"ceill", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"ceilf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"ceilf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"daddl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          FunctionSpec<"ddivl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          FunctionSpec<"dfmal", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          FunctionSpec<"dsubl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"fabs", RetValSpec<DoubleType>, [ArgSpec<DoubleType>], [ConstAttr]>,
-          FunctionSpec<"fabsf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"fabsl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fabsf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fabsf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fadd", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"faddl", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"fdim", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fdimf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fdiml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fdimf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fdimf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fdiv", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fdivl", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"ffma", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"ffmal", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"floor", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"floorf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"floorl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"floorf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"floorf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fmin", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fminf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fminl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fminf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"fminf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"fmax", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmaxf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-          GuardedFunctionSpec<"fmaxf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-	  FunctionSpec<"fmaximum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaximumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaximuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmaximumf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fmaximumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fmaximum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaximum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmaximum_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fmaximum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fmaximum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaximum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaximum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmaximum_magf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fmaximum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fmaximum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaximum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaximum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmaximum_mag_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fmaximum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fminimum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fminimumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fminimuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fminimumf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fminimumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fminimum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fminimum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fminimum_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fminimum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fminimum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fminimum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fminimum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fminimum_magf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fminimum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-	  FunctionSpec<"fminimum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fminimum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fminimum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fminimum_mag_numf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fminimum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmaf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-
-          GuardedFunctionSpec<"f16fmaf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          FunctionSpec<"fmod", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmodf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"fmodl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"fmodf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fmodf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"frexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
-          FunctionSpec<"frexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
-          FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
-          GuardedFunctionSpec<"frexpf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"frexpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fromfp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"fromfpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"fromfpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          GuardedFunctionSpec<"fromfpf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fromfpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fromfpx", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"fromfpxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"fromfpxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          GuardedFunctionSpec<"fromfpxf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"fromfpxf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"fsub", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fsubl", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"ufromfp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"ufromfpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"ufromfpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          GuardedFunctionSpec<"ufromfpf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"ufromfpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"ufromfpx", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"ufromfpxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"ufromfpxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
-          GuardedFunctionSpec<"ufromfpxf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"ufromfpxf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"hypot", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"hypotf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-
-          FunctionSpec<"ilogb", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"ilogbf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"ilogbl", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"ilogbf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"ilogbf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"llogb", RetValSpec<LongType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"llogbf", RetValSpec<LongType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"llogbl", RetValSpec<LongType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"llogbf16", RetValSpec<LongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"llogbf128", RetValSpec<LongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"ldexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>]>,
-          FunctionSpec<"ldexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>]>,
-          FunctionSpec<"ldexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>]>,
-          GuardedFunctionSpec<"ldexpf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"ldexpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"log10", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"log10f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"log10f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"log1p", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"log1pf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"log2", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"log2f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"log2f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"log", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"logf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"logf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"logb", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"logbf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"logbl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"logbf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"logbf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"modf", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoublePtr>]>,
-          FunctionSpec<"modff", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatPtr>]>,
-          FunctionSpec<"modfl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoublePtr>]>,
-          GuardedFunctionSpec<"modff16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"modff128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Ptr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"cos", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"cosf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"sin", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"sinf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"tan", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"tanf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"erff", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"exp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"expf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"expf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"exp2", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"exp2f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"exp2f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"exp2m1f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"exp2m1f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"expm1", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"expm1f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"expm1f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"exp10", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"exp10f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"exp10f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"exp10m1f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"exp10m1f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"remainder", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"remainderf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"remainderl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"remainderf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"remainderf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"remquo", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
-          FunctionSpec<"remquof", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
-          FunctionSpec<"remquol", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
-          GuardedFunctionSpec<"remquof16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"remquof128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"round", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"roundf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"roundl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"roundf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"roundf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"roundeven", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"roundevenf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"roundevenl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"roundevenf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"roundevenf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"lround", RetValSpec<LongType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"lroundf", RetValSpec<LongType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"lroundl", RetValSpec<LongType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"lroundf16", RetValSpec<LongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"lroundf128", RetValSpec<LongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"llround", RetValSpec<LongLongType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"llroundf", RetValSpec<LongLongType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"llroundl", RetValSpec<LongLongType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"llroundf16", RetValSpec<LongLongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"llroundf128", RetValSpec<LongLongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"rint", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"rintf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"rintl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"rintf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"rintf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"lrint", RetValSpec<LongType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"lrintf", RetValSpec<LongType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"lrintl", RetValSpec<LongType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"lrintf16", RetValSpec<LongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"lrintf128", RetValSpec<LongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"llrint", RetValSpec<LongLongType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"llrintf", RetValSpec<LongLongType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"llrintl", RetValSpec<LongLongType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"llrintf16", RetValSpec<LongLongType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"llrintf128", RetValSpec<LongLongType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"sqrt", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"sqrtf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"sqrtl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"sqrtf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"sqrtf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"trunc", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"truncf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"truncl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"truncf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"truncf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"nearbyint", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"nearbyintf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"nearbyintl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"nearbyintf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"nearbyintf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"nextafterf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"nextafter", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"nextafterl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"nextafterf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"nextafterf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"nexttowardf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<LongDoubleType>]>,
-          FunctionSpec<"nexttoward", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<LongDoubleType>]>,
-          FunctionSpec<"nexttowardl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"nexttowardf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"nextdown", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"nextdownf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"nextdownl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"nextdownf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"nextdownf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"nextup", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"nextupf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"nextupl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"nextupf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"nextupf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"powf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"pow", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-
-          FunctionSpec<"coshf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"coshf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"sinhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"sinhf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"tanhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          GuardedFunctionSpec<"tanhf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-
-          FunctionSpec<"acosf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"asinf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"asin", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-
-          FunctionSpec<"atanf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"atan2", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"atan2f", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"atan2l", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"acoshf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"asinhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"atanhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-
-          FunctionSpec<"scalbln", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<LongType>]>,
-          FunctionSpec<"scalblnf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<LongType>]>,
-          FunctionSpec<"scalblnl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongType>]>,
-          GuardedFunctionSpec<"scalblnf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<LongType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"scalblnf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<LongType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"scalbn", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>]>,
-          FunctionSpec<"scalbnf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>]>,
-          FunctionSpec<"scalbnl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>]>,
-          GuardedFunctionSpec<"scalbnf16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>, ArgSpec<IntType>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"scalbnf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntType>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"nanf", RetValSpec<FloatType>, [ArgSpec<ConstCharPtr>]>,
-          FunctionSpec<"nan", RetValSpec<DoubleType>, [ArgSpec<ConstCharPtr>]>,
-          FunctionSpec<"nanl", RetValSpec<LongDoubleType>, [ArgSpec<ConstCharPtr>]>,
-          GuardedFunctionSpec<"nanf16", RetValSpec<Float16Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"nanf128", RetValSpec<Float128Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"issignaling", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"issignalingf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"issignalingl", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"issignalingf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"issignalingf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"canonicalize", RetValSpec<IntType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"canonicalizef", RetValSpec<IntType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
-          FunctionSpec<"canonicalizel", RetValSpec<IntType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"canonicalizef16", RetValSpec<IntType>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"canonicalizef128", RetValSpec<IntType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"iscanonical", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"iscanonicalf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"iscanonicall", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"iscanonicalf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"iscanonicalf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"dsqrtl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"totalorder", RetValSpec<IntType>, [ArgSpec<ConstDoublePtr>, ArgSpec<ConstDoublePtr>]>,
-	  FunctionSpec<"totalorderf", RetValSpec<IntType>, [ArgSpec<ConstFloatPtr>, ArgSpec<ConstFloatPtr>]>,
-	  FunctionSpec<"totalorderl", RetValSpec<IntType>, [ArgSpec<ConstLongDoublePtr>, ArgSpec<ConstLongDoublePtr>]>,
-	  GuardedFunctionSpec<"totalorderf16", RetValSpec<IntType>, [ArgSpec<ConstFloat16Ptr>, ArgSpec<ConstFloat16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
-	  GuardedFunctionSpec<"totalorderf128", RetValSpec<IntType>, [ArgSpec<ConstFloat128Ptr>, ArgSpec<ConstFloat128Ptr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"totalordermag", RetValSpec<IntType>, [ArgSpec<ConstDoublePtr>, ArgSpec<ConstDoublePtr>]>,
-          FunctionSpec<"totalordermagf", RetValSpec<IntType>, [ArgSpec<ConstFloatPtr>, ArgSpec<ConstFloatPtr>]>,
-          FunctionSpec<"totalordermagl", RetValSpec<IntType>, [ArgSpec<ConstLongDoublePtr>, ArgSpec<ConstLongDoublePtr>]>,
-          GuardedFunctionSpec<"totalordermagf16", RetValSpec<IntType>, [ArgSpec<ConstFloat16Ptr>, ArgSpec<ConstFloat16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"totalordermagf128", RetValSpec<IntType>, [ArgSpec<ConstFloat128Ptr>, ArgSpec<ConstFloat128Ptr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"getpayload", RetValSpec<DoubleType>, [ArgSpec<DoublePtr>]>,
-	  FunctionSpec<"getpayloadf", RetValSpec<FloatType>, [ArgSpec<FloatPtr>]>,
-          FunctionSpec<"getpayloadl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoublePtr>]>,
-          GuardedFunctionSpec<"getpayloadf16", RetValSpec<Float16Type>, [ArgSpec<Float16Ptr>], "LIBC_TYPES_HAS_FLOAT16">,
-          GuardedFunctionSpec<"getpayloadf128", RetValSpec<Float128Type>, [ArgSpec<Float128Ptr>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"setpayload", RetValSpec<IntType>, [ArgSpec<DoublePtr>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"setpayloadf", RetValSpec<IntType>, [ArgSpec<FloatPtr>, ArgSpec<FloatType>]>,
-	  FunctionSpec<"setpayloadl", RetValSpec<IntType>, [ArgSpec<LongDoublePtr>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"setpayloadf16", RetValSpec<IntType>, [ArgSpec<Float16Ptr>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-	  GuardedFunctionSpec<"setpayloadf128", RetValSpec<IntType>, [ArgSpec<Float128Ptr>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          FunctionSpec<"setpayloadsig", RetValSpec<IntType>, [ArgSpec<DoublePtr>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"setpayloadsigf", RetValSpec<IntType>, [ArgSpec<FloatPtr>, ArgSpec<FloatType>]>,
-	  FunctionSpec<"setpayloadsigl", RetValSpec<IntType>, [ArgSpec<LongDoublePtr>, ArgSpec<LongDoubleType>]>,
-          GuardedFunctionSpec<"setpayloadsigf16", RetValSpec<IntType>, [ArgSpec<Float16Ptr>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
-	  GuardedFunctionSpec<"setpayloadsigf128", RetValSpec<IntType>, [ArgSpec<Float128Ptr>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
-
-          GuardedFunctionSpec<"f16addf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          GuardedFunctionSpec<"f16subf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          FunctionSpec<"fmul", RetValSpec<FloatType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"fmull", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"dmull", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
-
-          GuardedFunctionSpec<"f16mulf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          FunctionSpec<"fsqrt", RetValSpec<FloatType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"fsqrtl", RetValSpec<FloatType>, [ArgSpec<LongDoubleType>]>,
-
-          GuardedFunctionSpec<"f16divf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          GuardedFunctionSpec<"f16sqrtf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
-
-          FunctionSpec<"lgamma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
-          FunctionSpec<"lgammaf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
-          FunctionSpec<"lgammal", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
-      ]
-  >;
-
-  HeaderSpec StdIO = HeaderSpec<
-      "stdio.h",
-      [
-          Macro<"stdin">,
-          Macro<"stderr">,
-          Macro<"stdout">,
-          Macro<"_IOFBF">,
-          Macro<"_IOLBF">,
-          Macro<"_IONBF">,
-          Macro<"EOF">,
-      ], // Macros
-      [ // Types
-          SizeTType,
-          FILE,
-      ],
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "clearerr",
-              RetValSpec<VoidType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "fclose",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "feof",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "ferror",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "fgetc",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "fgets",
-              RetValSpec<CharPtr>,
-              [
-                ArgSpec<CharRestrictedPtr>,
-                ArgSpec<IntType>,
-                ArgSpec<FILERestrictedPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "fflush",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "fopen",
-              RetValSpec<FILEPtr>,
-              [ArgSpec<ConstCharPtr>,
-               ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "fputc",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>,
-               ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "ftell",
-              RetValSpec<LongType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "getc",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "getchar",
-              RetValSpec<IntType>,
-              [ArgSpec<VoidType>]
-          >,
-          FunctionSpec<
-              "putc",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>,
-               ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "putchar",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fputs",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<FILERestrictedPtr>]
-          >,
-          FunctionSpec<
-              "puts",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>]
-          >,
-          FunctionSpec<
-              "fread",
-              RetValSpec<SizeTType>,
-              [ArgSpec<VoidRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<SizeTType>,
-               ArgSpec<FILERestrictedPtr>]
-          >,
-          FunctionSpec<
-              "fseek",
-              RetValSpec<IntType>,
-              [ArgSpec<FILEPtr>,
-               ArgSpec<LongType>,
-               ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "fwrite",
-              RetValSpec<SizeTType>,
-              [ArgSpec<ConstVoidRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<SizeTType>,
-               ArgSpec<FILERestrictedPtr>]
-          >,
-          FunctionSpec<
-              "remove",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "rename",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharPtr>, ArgSpec<ConstCharPtr>]
-          >,
-          FunctionSpec<
-              "setbuf",
-              RetValSpec<VoidType>,
-              [ArgSpec<FILERestrictedPtr>, ArgSpec<CharRestrictedPtr>]
-          >,
-          FunctionSpec<
-              "setvbuf",
-              RetValSpec<IntType>,
-              [ArgSpec<FILERestrictedPtr>, ArgSpec<CharRestrictedPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
-          >,
-          FunctionSpec<
-              "sscanf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "vsscanf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "scanf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "vscanf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "fscanf",
-              RetValSpec<IntType>,
-              [ArgSpec<FILERestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "vfscanf",
-              RetValSpec<IntType>,
-              [ArgSpec<FILERestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "sprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "snprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "printf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "fprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<FILERestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "asprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtrPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VarArgType>]
-          >,
-          FunctionSpec<
-              "vsprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "vsnprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtr>,
-               ArgSpec<SizeTType>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "vprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "vfprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<FILERestrictedPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-          FunctionSpec<
-              "ungetc",
-              RetValSpec<IntType>,
-              [ArgSpec<IntType>, ArgSpec<FILEPtr>]
-          >,
-          FunctionSpec<
-              "vasprintf",
-              RetValSpec<IntType>,
-              [ArgSpec<CharRestrictedPtrPtr>,
-               ArgSpec<ConstCharRestrictedPtr>,
-               ArgSpec<VaListType>]
-          >,
-      ],
-      [
-          ObjectSpec<
-              "stdin",
-              "FILE *"
-          >,
-          ObjectSpec<
-              "stdout",
-              "FILE *"
-          >,
-          ObjectSpec<
-              "stderr",
-              "FILE *"
-          >,
-      ]
-  >;
-
-  HeaderSpec StdBit = HeaderSpec<
-      "stdbit.h",
-      [
-        Macro<"__STDC_VERSION_STDBIT_H__">,
-        Macro<"__STDC_ENDIAN_LITTLE__">,
-        Macro<"__STDC_ENDIAN_BIG__">,
-        Macro<"__STDC_ENDIAN_NATIVE__">,
-        Macro<"stdc_leading_zeros">,
-        Macro<"stdc_leading_ones">,
-        Macro<"stdc_trailing_zeros">,
-        Macro<"stdc_trailing_ones">,
-        Macro<"stdc_first_leading_zero">,
-        Macro<"stdc_first_leading_one">,
-        Macro<"stdc_first_trailing_zero">,
-        Macro<"stdc_first_trailing_one">,
-        Macro<"stdc_count_zeros">,
-        Macro<"stdc_count_ones">,
-        Macro<"stdc_has_single_bit">,
-        Macro<"stdc_bit_width">,
-        Macro<"stdc_bit_floor">,
-        Macro<"stdc_bit_ceil">
-      ], // Macros
-      [], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<"stdc_leading_zeros_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_leading_zeros_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_leading_zeros_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_leading_zeros_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_leading_zeros_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_leading_ones_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_leading_ones_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_leading_ones_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_leading_ones_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_leading_ones_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_trailing_zeros_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_trailing_zeros_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_trailing_zeros_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_trailing_zeros_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_trailing_zeros_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_trailing_ones_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_trailing_ones_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_trailing_ones_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_trailing_ones_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_trailing_ones_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_first_leading_zero_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_first_leading_zero_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_first_leading_zero_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_first_leading_zero_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_first_leading_zero_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_first_leading_one_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_first_leading_one_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_first_leading_one_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_first_leading_one_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_first_leading_one_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_first_trailing_one_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_first_trailing_one_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_first_trailing_one_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_first_trailing_one_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_first_trailing_one_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_count_zeros_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_count_zeros_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_count_zeros_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_count_zeros_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_count_zeros_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_count_ones_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_count_ones_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_count_ones_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_count_ones_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_count_ones_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_has_single_bit_uc", RetValSpec<BoolType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_has_single_bit_us", RetValSpec<BoolType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_has_single_bit_ui", RetValSpec<BoolType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_has_single_bit_ul", RetValSpec<BoolType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_has_single_bit_ull", RetValSpec<BoolType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_bit_width_uc", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_bit_width_us", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_bit_width_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_bit_width_ul", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_bit_width_ull", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_bit_floor_uc", RetValSpec<UnsignedCharType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_bit_floor_us", RetValSpec<UnsignedShortType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_bit_floor_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_bit_floor_ul", RetValSpec<UnsignedLongType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_bit_floor_ull", RetValSpec<UnsignedLongLongType>, [ArgSpec<UnsignedLongLongType>]>,
-          FunctionSpec<"stdc_bit_ceil_uc", RetValSpec<UnsignedCharType>, [ArgSpec<UnsignedCharType>]>,
-          FunctionSpec<"stdc_bit_ceil_us", RetValSpec<UnsignedShortType>, [ArgSpec<UnsignedShortType>]>,
-          FunctionSpec<"stdc_bit_ceil_ui", RetValSpec<UnsignedIntType>, [ArgSpec<UnsignedIntType>]>,
-          FunctionSpec<"stdc_bit_ceil_ul", RetValSpec<UnsignedLongType>, [ArgSpec<UnsignedLongType>]>,
-          FunctionSpec<"stdc_bit_ceil_ull", RetValSpec<UnsignedLongLongType>, [ArgSpec<UnsignedLongLongType>]>
-      ] // Functions
-  >;
-
-  HeaderSpec StdCkdInt = HeaderSpec<
-      "stdckdint.h",
-      [
-        Macro<"__STDC_VERSION_STDCKDINT_H__">,
-        Macro<"ckd_add">,
-        Macro<"ckd_sub">,
-        Macro<"ckd_mul">
-      ], // Macros
-      [], // Types
-      [], // Enumerations
-      [] // Functions
-  >;
-
-  HeaderSpec StdLib = HeaderSpec<
-      "stdlib.h",
-      [], // Macros
-      [
-          DivTType,
-          LDivTType,
-          LLDivTType,
-          SizeTType,
-          BSearchCompareT,
-          QSortCompareT,
-          AtexitHandlerT,
-      ], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<"abort", RetValSpec<NoReturn>, [ArgSpec<VoidType>]>,
-
-          FunctionSpec<"bsearch", RetValSpec<VoidPtr>, [ArgSpec<ConstVoidPtr>, ArgSpec<ConstVoidPtr>, ArgSpec<SizeTType>, ArgSpec<SizeTType>, ArgSpec<BSearchCompareT>]>,
-
-          FunctionSpec<"abs", RetValSpec<IntType>, [ArgSpec<IntType>]>,
-          FunctionSpec<"labs", RetValSpec<LongType>, [ArgSpec<LongType>]>,
-          FunctionSpec<"llabs", RetValSpec<LongLongType>, [ArgSpec<LongLongType>]>,
-
-          FunctionSpec<"atof", RetValSpec<DoubleType>, [ArgSpec<ConstCharRestrictedPtr>]>,
-          FunctionSpec<"atoi", RetValSpec<IntType>, [ArgSpec<ConstCharPtr>]>,
-          FunctionSpec<"atol", RetValSpec<LongType>, [ArgSpec<ConstCharPtr>]>,
-          FunctionSpec<"atoll", RetValSpec<LongLongType>, [ArgSpec<ConstCharPtr>]>,
-
-          FunctionSpec<"div", RetValSpec<DivTType>, [ArgSpec<IntType>, ArgSpec<IntType>]>,
-          FunctionSpec<"ldiv", RetValSpec<LDivTType>, [ArgSpec<LongType>, ArgSpec<LongType>]>,
-          FunctionSpec<"lldiv", RetValSpec<LLDivTType>, [ArgSpec<LongLongType>, ArgSpec<LongLongType>]>,
-
-          FunctionSpec<"qsort", RetValSpec<VoidType>, [ArgSpec<VoidPtr>, ArgSpec<SizeTType>, ArgSpec<SizeTType>, ArgSpec<QSortCompareT>]>,
-
-          FunctionSpec<"rand", RetValSpec<IntType>, [ArgSpec<VoidType>]>,
-          FunctionSpec<"srand", RetValSpec<VoidType>, [ArgSpec<UnsignedIntType>]>,
-
-          FunctionSpec<"strfromf", RetValSpec<IntType>, [ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<FloatType>]>,
-          FunctionSpec<"strfromd", RetValSpec<IntType>, [ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<DoubleType>]>,
-          FunctionSpec<"strfroml", RetValSpec<IntType>, [ArgSpec<CharRestrictedPtr>, ArgSpec<SizeTType>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<LongDoubleType>]>,
-
-          FunctionSpec<"strtof", RetValSpec<FloatType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>]>,
-          FunctionSpec<"strtod", RetValSpec<DoubleType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>]>,
-          FunctionSpec<"strtold", RetValSpec<LongDoubleType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>]>,
-          FunctionSpec<"strtol", RetValSpec<LongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-          FunctionSpec<"strtoll", RetValSpec<LongLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-          FunctionSpec<"strtoul", RetValSpec<UnsignedLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-          FunctionSpec<"strtoull", RetValSpec<UnsignedLongLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-
-          FunctionSpec<"strtof_l", RetValSpec<FloatType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtod_l", RetValSpec<DoubleType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtold_l", RetValSpec<LongDoubleType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtol_l", RetValSpec<LongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtoll_l", RetValSpec<LongLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtoul_l", RetValSpec<UnsignedLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>, ArgSpec<LocaleT>]>,
-          FunctionSpec<"strtoull_l", RetValSpec<UnsignedLongLongType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>, ArgSpec<LocaleT>]>,
-
-          FunctionSpec<"malloc", RetValSpec<VoidPtr>, [ArgSpec<SizeTType>]>,
-          FunctionSpec<"calloc", RetValSpec<VoidPtr>, [ArgSpec<SizeTType>, ArgSpec<SizeTType>]>,
-          FunctionSpec<"realloc", RetValSpec<VoidPtr>, [ArgSpec<VoidPtr>, ArgSpec<SizeTType>]>,
-          FunctionSpec<"aligned_alloc", RetValSpec<VoidPtr>, [ArgSpec<SizeTType>, ArgSpec<SizeTType>]>,
-          FunctionSpec<"free", RetValSpec<VoidType>, [ArgSpec<VoidPtr>]>,
-
-          FunctionSpec<"_Exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
-          FunctionSpec<"at_quick_exit", RetValSpec<IntType>, [ArgSpec<AtexitHandlerT>]>,
-          FunctionSpec<"atexit", RetValSpec<IntType>, [ArgSpec<AtexitHandlerT>]>,
-          FunctionSpec<"exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
-          FunctionSpec<"quick_exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
-
-          FunctionSpec<"system", RetValSpec<IntType>, [ArgSpec<ConstCharPtr>]>,
-      ]
-  >;
-
-  NamedType IMaxDivTType = NamedType<"imaxdiv_t">;
-
-  HeaderSpec IntTypes = HeaderSpec<
-      "inttypes.h",
-      [
-        Macro<"__STDC_VERSION_INTTYPES_H__">,
-      ], // Macros
-      [
-        IMaxDivTType,
-      ], // Types
-      [], // Enumerations
-      [
-          FunctionSpec<"imaxabs", RetValSpec<IntMaxTType>, [ArgSpec<IntMaxTType>]>,
-          FunctionSpec<"imaxdiv", RetValSpec<IMaxDivTType>, [ArgSpec<IntMaxTType>, ArgSpec<IntMaxTType>]>,
-          FunctionSpec<"strtoimax", RetValSpec<IntMaxTType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-          FunctionSpec<"strtoumax", RetValSpec<UIntMaxTType>, [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<CharRestrictedPtrPtr>, ArgSpec<IntType>]>,
-      ]
-  >;
-
-  HeaderSpec Errno = HeaderSpec<
-      "errno.h",
-      [
-        Macro<"errno">,
-        Macro<"EDOM">,
-        Macro<"EILSEQ">,
-        Macro<"ERANGE">,
-      ]
-  >;
-
-  HeaderSpec Float = HeaderSpec<
-      "float.h",
-      [
-        Macro<"FLT_MANT_DIG">,
-        Macro<"DBL_MANT_DIG">,
-        Macro<"LDBL_MANT_DIG">,
-      ]
-  >;
-
-  HeaderSpec StdInt = HeaderSpec<"StdInt.h">;
-
-  HeaderSpec Limits = HeaderSpec<"limits.h">;
-
-  NamedType SigAtomicT = NamedType<"sig_atomic_t">;
-  HeaderSpec Signal = HeaderSpec<
-      "signal.h",
-      [
-        Macro<"SIG_BLOCK">,
-        Macro<"SIG_UNBLOCK">,
-        Macro<"SIG_SETMASK">,
-
-        Macro<"SIGABRT">,
-        Macro<"SIGFPE">,
-        Macro<"SIGILL">,
-        Macro<"SIGINT">,
-        Macro<"SIGSEGV">,
-        Macro<"SIGTERM">
-      ],
-      [
-        SizeTType,
-        SigAtomicT,
-        SigHandlerT,
-      ],
-      [], // Enumerations
-      [
-        FunctionSpec<"raise", RetValSpec<IntType>, [ArgSpec<IntType>]>,
-        FunctionSpec<
-          "signal",
-          RetValSpec<SigHandlerT>,
-          [ArgSpec<IntType>, ArgSpec<SigHandlerT>]
-        >,
-      ]
-  >;
-
-  HeaderSpec Threads = HeaderSpec<
-      "threads.h",
-      [
-          Macro<"ONCE_FLAG_INIT">,
-      ],
-      [
-          OnceFlagType,
-          CallOnceFuncType,
-          CndTType,
-          MtxTType,
-          ThrdStartTType,
-          ThrdTType,
-          TssTType,
-          TssDtorTType,
-      ],
-      [
-          EnumeratedNameValue<"mtx_plain">,
-          EnumeratedNameValue<"mtx_recursive">,
-          EnumeratedNameValue<"mtx_timed">,
-          EnumeratedNameValue<"thrd_timedout">,
-          EnumeratedNameValue<"thrd_success">,
-          EnumeratedNameValue<"thrd_busy">,
-          EnumeratedNameValue<"thrd_error">,
-          EnumeratedNameValue<"thrd_nomem">,
-      ],
-      [
-          FunctionSpec<
-              "call_once",
-              RetValSpec<VoidType>,
-              [
-                  ArgSpec<OnceFlagTypePtr>,
-                  ArgSpec<CallOnceFuncType>,
-              ]
-          >,
-          FunctionSpec<
-              "cnd_broadcast",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<CndTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "cnd_destroy",
-              RetValSpec<VoidType>,
-              [
-                  ArgSpec<CndTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "cnd_init",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<CndTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "cnd_signal",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<CndTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "cnd_wait",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<CndTTypePtr>,
-                  ArgSpec<MtxTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "mtx_init",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<MtxTTypePtr>,
-                  ArgSpec<IntType>,
-              ]
-          >,
-          FunctionSpec<
-              "mtx_destroy",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<VoidType>,
-              ]
-          >,
-          FunctionSpec<
-              "mtx_lock",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<MtxTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "mtx_unlock",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<MtxTTypePtr>,
-              ]
-          >,
-          FunctionSpec<
-              "thrd_create",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<ThrdTTypePtr>,
-                  ArgSpec<ThrdStartTType>,
-                  ArgSpec<VoidPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "thrd_join",
-              RetValSpec<IntType>,
-              [
-                  ArgSpec<ThrdTType>,
-                  ArgSpec<IntPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "thrd_detach",
-              RetValSpec<IntType>,
-              [ArgSpec<ThrdTType>]
-          >,
-          FunctionSpec<
-              "thrd_current",
-              RetValSpec<ThrdTType>,
-              [ArgSpec<VoidType>]
-          >,
-          FunctionSpec<
-              "thrd_equal",
-              RetValSpec<IntType>,
-              [ArgSpec<ThrdTType>, ArgSpec<ThrdTType>]
-          >,
-          FunctionSpec<
-              "thrd_exit",
-              RetValSpec<VoidType>,
-              [ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "tss_create",
-              RetValSpec<IntType>,
-              [ArgSpec<TssTPtr>, ArgSpec<TssDtorTType>]
-          >,
-          FunctionSpec<
-              "tss_delete",
-              RetValSpec<IntType>,
-              [ArgSpec<TssTType>]
-          >,
-          FunctionSpec<
-              "tss_get",
-              RetValSpec<VoidPtr>,
-              [ArgSpec<TssTType>]
-          >,
-          FunctionSpec<
-              "tss_set",
-              RetValSpec<IntType>,
-              [ArgSpec<TssTType>, ArgSpec<VoidPtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec Time = HeaderSpec<
-      "time.h",
-      [], // Macros
-      [ // Types
-         ClockT,
-         StructTmType,
-         StructTimeSpec,
-         TimeTType,
-         SizeTType,
-      ],
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "asctime",
-              RetValSpec<CharPtr>,
-              [ArgSpec<StructTmPtr>]
-          >,
-          FunctionSpec<
-              "asctime_r",
-              RetValSpec<CharPtr>,
-              [
-                  ArgSpec<StructTmPtr>,
-                  ArgSpec<CharPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "ctime",
-              RetValSpec<CharPtr>,
-              [ArgSpec<TimeTTypePtr>]
-          >,
-          FunctionSpec<
-              "ctime_r",
-              RetValSpec<CharPtr>,
-              [
-                  ArgSpec<TimeTTypePtr>,
-                  ArgSpec<CharPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "clock",
-              RetValSpec<ClockT>,
-              [ArgSpec<VoidType>]
-          >,
-          FunctionSpec<
-              "difftime",
-              RetValSpec<DoubleType>,
-              [
-                  ArgSpec<TimeTType>,
-                  ArgSpec<TimeTType>,
-              ]
-          >,
-          FunctionSpec<
-              "gmtime",
-              RetValSpec<StructTmPtr>,
-              [ArgSpec<TimeTTypePtr>]
-          >,
-          FunctionSpec<
-              "gmtime_r",
-              RetValSpec<StructTmPtr>,
-              [
-                  ArgSpec<TimeTTypePtr>,
-                  ArgSpec<StructTmPtr>,
-              ]
-          >,
-          FunctionSpec<
-              "mktime",
-              RetValSpec<TimeTType>,
-              [ArgSpec<StructTmPtr>]
-          >,
-          FunctionSpec<
-              "time",
-              RetValSpec<TimeTType>,
-              [ArgSpec<TimeTTypePtr>]
-          >,
-      ]
-  >;
-
-  HeaderSpec SetJmp = HeaderSpec<
-      "setjmp.h",
-      [], // Macros
-      [JmpBuf],
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "longjmp",
-              RetValSpec<NoReturn>,
-              [ArgSpec<JmpBuf>, ArgSpec<IntType>]
-          >,
-          FunctionSpec<
-              "setjmp",
-              RetValSpec<IntType>,
-              [ArgSpec<JmpBuf>]
-          >,
-          FunctionSpec<
-              "longjmp",
-              RetValSpec<VoidType>,
-              [ArgSpec<JmpBuf>, ArgSpec<IntType>]
-          >,
-      ]
-  >;
-
-  HeaderSpec UChar = HeaderSpec<
-      "uchar.h",
-      [], // Macros
-      [ //Types
-        MBStateTType,
-        Char8TType,
-        Char16TType,
-        Char32TType,
-        SizeTType,
-      ],
-      [], // Enumerations
-      []
-  >;
-
-  HeaderSpec WChar = HeaderSpec<
-      "wchar.h",
-      [ // Macros
-        Macro<"WEOF">,
-      ],
-      [ //Types
-        MBStateTType,
-        SizeTType,
-        WIntType,
-        WCharType,
-      ],
-      [], // Enumerations
-      [
-          FunctionSpec<
-              "wctob",
-              RetValSpec<IntType>,
-              [ArgSpec<WIntType>]
-          >,
-      ]
-  >;
-
-  NamedType StructLconv = NamedType<"struct lconv">;
-  PtrType StructLconvPtr = PtrType<StructLconv>;
-
-  HeaderSpec Locale = HeaderSpec<
-     "locale.h",
-      [], // Macros
-      [LocaleT, StructLconv], // Types
-      [], // Enumerations
-      [
-        FunctionSpec<
-          "duplocale",
-          RetValSpec<LocaleT>,
-          [
-            ArgSpec<LocaleT>
-          ]
-        >,
-        FunctionSpec<
-          "freelocale",
-          RetValSpec<VoidType>,
-          [
-            ArgSpec<LocaleT>
-          ]
-        >,
-        FunctionSpec<
-          "localeconv",
-          RetValSpec<StructLconvPtr>,
-          []
-        >,
-        FunctionSpec<
-          "newlocale",
-          RetValSpec<LocaleT>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<ConstCharPtr>,
-            ArgSpec<LocaleT>
-          ]
-        >,
-        FunctionSpec<
-          "setlocale",
-          RetValSpec<CharPtr>,
-          [
-            ArgSpec<IntType>,
-            ArgSpec<ConstCharPtr>
-          ]
-        >,
-        FunctionSpec<
-          "uselocale",
-          RetValSpec<LocaleT>,
-          [
-            ArgSpec<LocaleT>
-          ]
-        >
-      ]  // Functions
-  >;
-
-  let Headers = [
-    Assert,
-    CType,
-    Errno,
-    Fenv,
-    Float,
-    StdInt,
-    Limits,
-    Math,
-    String,
-    StdBit,
-    StdCkdInt,
-    StdIO,
-    StdLib,
-    IntTypes,
-    SetJmp,
-    Signal,
-    Threads,
-    Time,
-    UChar,
-    WChar,
-    Locale,
-  ];
-}
diff --git a/libc/spec/stdc_ext.td b/libc/spec/stdc_ext.td
deleted file mode 100644
index dee3b8bdf6fee..0000000000000
--- a/libc/spec/stdc_ext.td
+++ /dev/null
@@ -1,82 +0,0 @@
-// Fixed point types.
-// From ISO/IEC TR 18037:2008 standard:
-// https://standards.iso.org/ittf/PubliclyAvailableStandards/c051126_ISO_IEC_TR_18037_2008.zip
-def ShortFractType : NamedType<"short fract">;
-def FractType : NamedType<"fract">;
-def LongFractType : NamedType<"long fract">;
-def UnsignedShortFractType : NamedType<"unsigned short fract">;
-def UnsignedFractType : NamedType<"unsigned fract">;
-def UnsignedLongFractType : NamedType<"unsigned long fract">;
-
-def ShortAccumType : NamedType<"short accum">;
-def AccumType : NamedType<"accum">;
-def LongAccumType : NamedType<"long accum">;
-def UnsignedShortAccumType : NamedType<"unsigned short accum">;
-def UnsignedAccumType : NamedType<"unsigned accum">;
-def UnsignedLongAccumType : NamedType<"unsigned long accum">;
-
-def IntHrT : NamedType <"int_hr_t">;
-def IntRT : NamedType<"int_r_t">;
-def IntLrT : NamedType<"int_lr_t">;
-def IntHkT : NamedType<"int_hk_t">;
-def IntKT : NamedType<"int_k_t">;
-def IntLkT : NamedType<"int_lk_t">;
-def UIntUhrT : NamedType<"uint_uhr_t">;
-def UIntUrT : NamedType<"uint_ur_t">;
-def UIntUlrT : NamedType<"uint_ulr_t">;
-def UIntUhkT : NamedType<"uint_uhk_t">;
-def UIntUkT : NamedType<"uint_uk_t">;
-def UIntUlkT : NamedType<"uint_ulk_t">;
-
-def StdcExt : StandardSpec<"stdc_ext"> {
-  // From ISO/IEC TR 18037:2008 standard:
-  // https://standards.iso.org/ittf/PubliclyAvailableStandards/c051126_ISO_IEC_TR_18037_2008.zip
-  HeaderSpec StdFix = HeaderSpec<
-      "stdfix.h",
-      [],  // macros
-      [IntHrT,IntRT, IntLrT, IntHkT, IntKT, IntLkT, UIntUhrT, UIntUrT, UIntUlrT, UIntUhkT, UIntUkT, UIntUlkT],  // types
-      [],  // enums
-      [    // functions
-          GuardedFunctionSpec<"abshr", RetValSpec<ShortFractType>, [ArgSpec<ShortFractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"absr", RetValSpec<FractType>, [ArgSpec<FractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"abslr", RetValSpec<LongFractType>, [ArgSpec<LongFractType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"abshk", RetValSpec<ShortAccumType>, [ArgSpec<ShortAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"absk", RetValSpec<AccumType>, [ArgSpec<AccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"abslk", RetValSpec<LongAccumType>, [ArgSpec<LongAccumType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"roundhr", RetValSpec<ShortFractType>, [ArgSpec<ShortFractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundr", RetValSpec<FractType>, [ArgSpec<FractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundlr", RetValSpec<LongFractType>, [ArgSpec<LongFractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"roundhk", RetValSpec<ShortAccumType>, [ArgSpec<ShortAccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundk", RetValSpec<AccumType>, [ArgSpec<AccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundlk", RetValSpec<LongAccumType>, [ArgSpec<LongAccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"rounduhr", RetValSpec<UnsignedShortFractType>, [ArgSpec<UnsignedShortFractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundur", RetValSpec<UnsignedFractType>, [ArgSpec<UnsignedFractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundulr", RetValSpec<UnsignedLongFractType>, [ArgSpec<UnsignedLongFractType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"rounduhk", RetValSpec<UnsignedShortAccumType>, [ArgSpec<UnsignedShortAccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"rounduk", RetValSpec<UnsignedAccumType>, [ArgSpec<UnsignedAccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"roundulk", RetValSpec<UnsignedLongAccumType>, [ArgSpec<UnsignedLongAccumType>, ArgSpec<IntType>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-
-          GuardedFunctionSpec<"hrbits", RetValSpec<ShortFractType>, [ArgSpec<IntHrT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"rbits", RetValSpec<FractType>, [ArgSpec<IntRT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"lrbits", RetValSpec<LongFractType>, [ArgSpec<IntLrT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"hkbits", RetValSpec<ShortAccumType>, [ArgSpec<IntHkT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"kbits", RetValSpec<AccumType>, [ArgSpec<IntKT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"lkbits", RetValSpec<LongAccumType>, [ArgSpec<IntLkT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"uhrbits", RetValSpec<UnsignedShortFractType>, [ArgSpec<UIntUhrT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"urbits", RetValSpec<UnsignedFractType>, [ArgSpec<UIntUrT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"ukbits", RetValSpec<UnsignedAccumType>, [ArgSpec<UIntUkT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"ulrbits", RetValSpec<UnsignedLongFractType>, [ArgSpec<UIntUlrT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"uhkbits", RetValSpec<UnsignedShortAccumType>, [ArgSpec<UIntUhkT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-          GuardedFunctionSpec<"ulkbits", RetValSpec<UnsignedLongAccumType>, [ArgSpec<UIntUlkT>], "LIBC_COMPILER_HAS_FIXED_POINT">,
-      ]
-  >;
-
-  let Headers = [
-    StdFix,
-  ];
-}
diff --git a/libc/src/CMakeLists.txt b/libc/src/CMakeLists.txt
index 02c193e635362..9fc331ad18a39 100644
--- a/libc/src/CMakeLists.txt
+++ b/libc/src/CMakeLists.txt
@@ -12,7 +12,9 @@ add_subdirectory(stdfix)
 add_subdirectory(stdio)
 add_subdirectory(stdlib)
 add_subdirectory(string)
+add_subdirectory(strings)
 add_subdirectory(wchar)
+add_subdirectory(time)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(dirent)
@@ -32,13 +34,12 @@ if(NOT LLVM_LIBC_FULL_BUILD)
   return()
 endif()
 
+add_subdirectory(arpa)
 add_subdirectory(assert)
 add_subdirectory(compiler)
-add_subdirectory(network)
+add_subdirectory(locale)
 add_subdirectory(search)
 add_subdirectory(setjmp)
 add_subdirectory(signal)
 add_subdirectory(spawn)
 add_subdirectory(threads)
-add_subdirectory(time)
-add_subdirectory(locale)
diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h
index 72e7f2adde6a4..a9fc7e61610cc 100644
--- a/libc/src/__support/CPP/atomic.h
+++ b/libc/src/__support/CPP/atomic.h
@@ -40,8 +40,12 @@ enum class MemoryScope : int {
 };
 
 template <typename T> struct Atomic {
-  // For now, we will restrict to only arithmetic types.
-  static_assert(is_arithmetic_v<T>, "Only arithmetic types can be atomic.");
+  static_assert(is_trivially_copyable_v<T> && is_copy_constructible_v<T> &&
+                    is_move_constructible_v<T> && is_copy_assignable_v<T> &&
+                    is_move_assignable_v<T>,
+                "atomic<T> requires T to be trivially copyable, copy "
+                "constructible, move constructible, copy assignable, "
+                "and move assignable.");
 
 private:
   // The value stored should be appropriately aligned so that
@@ -49,6 +53,14 @@ template <typename T> struct Atomic {
   // correctly.
   static constexpr int ALIGNMENT = sizeof(T) > alignof(T) ? sizeof(T)
                                                           : alignof(T);
+  // type conversion helper to avoid long c++ style casts
+  LIBC_INLINE static int order(MemoryOrder mem_ord) {
+    return static_cast<int>(mem_ord);
+  }
+
+  LIBC_INLINE static int scope(MemoryScope mem_scope) {
+    return static_cast<int>(mem_scope);
+  }
 
 public:
   using value_type = T;
@@ -59,139 +71,155 @@ template <typename T> struct Atomic {
   // operations should be performed using the atomic methods however.
   alignas(ALIGNMENT) value_type val;
 
-  constexpr Atomic() = default;
+  LIBC_INLINE constexpr Atomic() = default;
 
   // Intializes the value without using atomic operations.
-  constexpr Atomic(value_type v) : val(v) {}
+  LIBC_INLINE constexpr Atomic(value_type v) : val(v) {}
 
-  Atomic(const Atomic &) = delete;
-  Atomic &operator=(const Atomic &) = delete;
+  LIBC_INLINE Atomic(const Atomic &) = delete;
+  LIBC_INLINE Atomic &operator=(const Atomic &) = delete;
 
   // Atomic load.
-  operator T() { return __atomic_load_n(&val, int(MemoryOrder::SEQ_CST)); }
-
-  T load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-         [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-#if __has_builtin(__scoped_atomic_load_n)
-    return __scoped_atomic_load_n(&val, int(mem_ord), (int)(mem_scope));
+  LIBC_INLINE operator T() { return load(); }
+
+  LIBC_INLINE T
+  load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+       [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    T res;
+#if __has_builtin(__scoped_atomic_load)
+    __scoped_atomic_load(&val, &res, order(mem_ord), scope(mem_scope));
 #else
-    return __atomic_load_n(&val, int(mem_ord));
+    __atomic_load(&val, &res, order(mem_ord));
 #endif
+    return res;
   }
 
   // Atomic store.
-  T operator=(T rhs) {
-    __atomic_store_n(&val, rhs, int(MemoryOrder::SEQ_CST));
+  LIBC_INLINE T operator=(T rhs) {
+    store(rhs);
     return rhs;
   }
 
-  void store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-             [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-#if __has_builtin(__scoped_atomic_store_n)
-    __scoped_atomic_store_n(&val, rhs, int(mem_ord), (int)(mem_scope));
+  LIBC_INLINE void
+  store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+        [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+#if __has_builtin(__scoped_atomic_store)
+    __scoped_atomic_store(&val, &rhs, order(mem_ord), scope(mem_scope));
 #else
-    __atomic_store_n(&val, rhs, int(mem_ord));
+    __atomic_store(&val, &rhs, order(mem_ord));
 #endif
   }
 
   // Atomic compare exchange
-  bool compare_exchange_strong(
+  LIBC_INLINE bool compare_exchange_strong(
       T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
       [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-    return __atomic_compare_exchange_n(&val, &expected, desired, false,
-                                       int(mem_ord), int(mem_ord));
+    return __atomic_compare_exchange(&val, &expected, &desired, false,
+                                     order(mem_ord), order(mem_ord));
   }
 
   // Atomic compare exchange (separate success and failure memory orders)
-  bool compare_exchange_strong(
+  LIBC_INLINE bool compare_exchange_strong(
       T &expected, T desired, MemoryOrder success_order,
       MemoryOrder failure_order,
       [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-    return __atomic_compare_exchange_n(&val, &expected, desired, false,
-                                       static_cast<int>(success_order),
-                                       static_cast<int>(failure_order));
+    return __atomic_compare_exchange(&val, &expected, &desired, false,
+                                     order(success_order),
+                                     order(failure_order));
   }
 
   // Atomic compare exchange (weak version)
-  bool compare_exchange_weak(
+  LIBC_INLINE bool compare_exchange_weak(
       T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
       [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-    return __atomic_compare_exchange_n(&val, &expected, desired, true,
-                                       static_cast<int>(mem_ord),
-                                       static_cast<int>(mem_ord));
+    return __atomic_compare_exchange(&val, &expected, &desired, true,
+                                     order(mem_ord), order(mem_ord));
   }
 
   // Atomic compare exchange (weak version with separate success and failure
   // memory orders)
-  bool compare_exchange_weak(
+  LIBC_INLINE bool compare_exchange_weak(
       T &expected, T desired, MemoryOrder success_order,
       MemoryOrder failure_order,
       [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-    return __atomic_compare_exchange_n(&val, &expected, desired, true,
-                                       static_cast<int>(success_order),
-                                       static_cast<int>(failure_order));
+    return __atomic_compare_exchange(&val, &expected, &desired, true,
+                                     order(success_order),
+                                     order(failure_order));
   }
 
-  T exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-             [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
-#if __has_builtin(__scoped_atomic_exchange_n)
-    return __scoped_atomic_exchange_n(&val, desired, int(mem_ord),
-                                      (int)(mem_scope));
+  LIBC_INLINE T
+  exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+           [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    T ret;
+#if __has_builtin(__scoped_atomic_exchange)
+    __scoped_atomic_exchange(&val, &desired, &ret, order(mem_ord),
+                             scope(mem_scope));
 #else
-    return __atomic_exchange_n(&val, desired, int(mem_ord));
+    __atomic_exchange(&val, &desired, &ret, order(mem_ord));
 #endif
+    return ret;
   }
 
-  T fetch_add(T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-              [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+  LIBC_INLINE T
+  fetch_add(T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+            [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
 #if __has_builtin(__scoped_atomic_fetch_add)
-    return __scoped_atomic_fetch_add(&val, increment, int(mem_ord),
-                                     (int)(mem_scope));
+    return __scoped_atomic_fetch_add(&val, increment, order(mem_ord),
+                                     scope(mem_scope));
 #else
-    return __atomic_fetch_add(&val, increment, int(mem_ord));
+    return __atomic_fetch_add(&val, increment, order(mem_ord));
 #endif
   }
 
-  T fetch_or(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-             [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+  LIBC_INLINE T
+  fetch_or(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+           [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
 #if __has_builtin(__scoped_atomic_fetch_or)
-    return __scoped_atomic_fetch_or(&val, mask, int(mem_ord), (int)(mem_scope));
+    return __scoped_atomic_fetch_or(&val, mask, order(mem_ord),
+                                    scope(mem_scope));
 #else
-    return __atomic_fetch_or(&val, mask, int(mem_ord));
+    return __atomic_fetch_or(&val, mask, order(mem_ord));
 #endif
   }
 
-  T fetch_and(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-              [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+  LIBC_INLINE T
+  fetch_and(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+            [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
 #if __has_builtin(__scoped_atomic_fetch_and)
-    return __scoped_atomic_fetch_and(&val, mask, int(mem_ord),
-                                     (int)(mem_scope));
+    return __scoped_atomic_fetch_and(&val, mask, order(mem_ord),
+                                     scope(mem_scope));
 #else
-    return __atomic_fetch_and(&val, mask, int(mem_ord));
+    return __atomic_fetch_and(&val, mask, order(mem_ord));
 #endif
   }
 
-  T fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
-              [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+  LIBC_INLINE T
+  fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
+            [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+    static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
 #if __has_builtin(__scoped_atomic_fetch_sub)
-    return __scoped_atomic_fetch_sub(&val, decrement, int(mem_ord),
-                                     (int)(mem_scope));
+    return __scoped_atomic_fetch_sub(&val, decrement, order(mem_ord),
+                                     scope(mem_scope));
 #else
-    return __atomic_fetch_sub(&val, decrement, int(mem_ord));
+    return __atomic_fetch_sub(&val, decrement, order(mem_ord));
 #endif
   }
 
   // Set the value without using an atomic operation. This is useful
   // in initializing atomic values without a constructor.
-  void set(T rhs) { val = rhs; }
+  LIBC_INLINE void set(T rhs) { val = rhs; }
 };
 
 // Issue a thread fence with the given memory ordering.
-LIBC_INLINE void atomic_thread_fence([[maybe_unused]] MemoryOrder mem_ord) {
-// The NVPTX backend currently does not support atomic thread fences so we use a
-// full system fence instead.
-#ifdef LIBC_TARGET_ARCH_IS_NVPTX
-  __nvvm_membar_sys();
+LIBC_INLINE void atomic_thread_fence(
+    MemoryOrder mem_ord,
+    [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
+#if __has_builtin(__scoped_atomic_thread_fence)
+  __scoped_atomic_thread_fence(static_cast<int>(mem_ord),
+                               static_cast<int>(mem_scope));
 #else
   __atomic_thread_fence(static_cast<int>(mem_ord));
 #endif
diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h
index d50b6612656db..b9bc5b8568441 100644
--- a/libc/src/__support/CPP/type_traits.h
+++ b/libc/src/__support/CPP/type_traits.h
@@ -28,6 +28,8 @@
 #include "src/__support/CPP/type_traits/is_const.h"
 #include "src/__support/CPP/type_traits/is_constant_evaluated.h"
 #include "src/__support/CPP/type_traits/is_convertible.h"
+#include "src/__support/CPP/type_traits/is_copy_assignable.h"
+#include "src/__support/CPP/type_traits/is_copy_constructible.h"
 #include "src/__support/CPP/type_traits/is_destructible.h"
 #include "src/__support/CPP/type_traits/is_enum.h"
 #include "src/__support/CPP/type_traits/is_fixed_point.h"
@@ -36,6 +38,8 @@
 #include "src/__support/CPP/type_traits/is_integral.h"
 #include "src/__support/CPP/type_traits/is_lvalue_reference.h"
 #include "src/__support/CPP/type_traits/is_member_pointer.h"
+#include "src/__support/CPP/type_traits/is_move_assignable.h"
+#include "src/__support/CPP/type_traits/is_move_constructible.h"
 #include "src/__support/CPP/type_traits/is_null_pointer.h"
 #include "src/__support/CPP/type_traits/is_object.h"
 #include "src/__support/CPP/type_traits/is_pointer.h"
diff --git a/libc/src/__support/CPP/type_traits/is_copy_assignable.h b/libc/src/__support/CPP/type_traits/is_copy_assignable.h
new file mode 100644
index 0000000000000..9beb93d14668d
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/is_copy_assignable.h
@@ -0,0 +1,32 @@
+//===-- is_copy_assignable type_traits --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_ASSIGNABLE_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_ASSIGNABLE_H
+
+#include "src/__support/CPP/type_traits/add_lvalue_reference.h"
+#include "src/__support/CPP/type_traits/integral_constant.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace cpp {
+
+// is copy assignable
+template <class T>
+struct is_copy_assignable
+    : public integral_constant<
+          bool, __is_assignable(cpp::add_lvalue_reference_t<T>,
+                                cpp::add_lvalue_reference_t<const T>)> {};
+
+template <class T>
+LIBC_INLINE_VAR constexpr bool is_copy_assignable_v =
+    is_copy_assignable<T>::value;
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_ASSIGNABLE_H
diff --git a/libc/src/__support/CPP/type_traits/is_copy_constructible.h b/libc/src/__support/CPP/type_traits/is_copy_constructible.h
new file mode 100644
index 0000000000000..d8eb9ad3507ee
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/is_copy_constructible.h
@@ -0,0 +1,31 @@
+//===-- is_copy_constructible type_traits -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_CONSTRUCTIBLE_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_CONSTRUCTIBLE_H
+
+#include "src/__support/CPP/type_traits/add_lvalue_reference.h"
+#include "src/__support/CPP/type_traits/integral_constant.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace cpp {
+
+// is copy constructible
+template <class T>
+struct is_copy_constructible
+    : public integral_constant<
+          bool, __is_constructible(T, cpp::add_lvalue_reference_t<const T>)> {};
+
+template <class T>
+LIBC_INLINE_VAR constexpr bool is_copy_constructible_v =
+    is_copy_constructible<T>::value;
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COPY_CONSTRUCTIBLE_H
diff --git a/libc/src/__support/CPP/type_traits/is_move_assignable.h b/libc/src/__support/CPP/type_traits/is_move_assignable.h
new file mode 100644
index 0000000000000..a788bd9074e32
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/is_move_assignable.h
@@ -0,0 +1,33 @@
+//===-- is_move_assignable type_traits --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_ASSIGNABLE_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_ASSIGNABLE_H
+
+#include "src/__support/CPP/type_traits/add_lvalue_reference.h"
+#include "src/__support/CPP/type_traits/add_rvalue_reference.h"
+#include "src/__support/CPP/type_traits/integral_constant.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace cpp {
+
+// is move assignable
+template <class T>
+struct is_move_assignable
+    : public integral_constant<bool, __is_assignable(
+                                         cpp::add_lvalue_reference_t<T>,
+                                         cpp::add_rvalue_reference_t<T>)> {};
+
+template <class T>
+LIBC_INLINE_VAR constexpr bool is_move_assignable_v =
+    is_move_assignable<T>::value;
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_ASSIGNABLE_H
diff --git a/libc/src/__support/CPP/type_traits/is_move_constructible.h b/libc/src/__support/CPP/type_traits/is_move_constructible.h
new file mode 100644
index 0000000000000..c898960546258
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/is_move_constructible.h
@@ -0,0 +1,31 @@
+//===-- is_move_constructible type_traits ------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_CONSTRUCTIBLE_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_CONSTRUCTIBLE_H
+
+#include "src/__support/CPP/type_traits/add_rvalue_reference.h"
+#include "src/__support/CPP/type_traits/integral_constant.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace cpp {
+
+// is move constructible
+template <class T>
+struct is_move_constructible
+    : public integral_constant<bool, __is_constructible(
+                                         T, cpp::add_rvalue_reference_t<T>)> {};
+
+template <class T>
+LIBC_INLINE_VAR constexpr bool is_move_constructible_v =
+    is_move_constructible<T>::value;
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_MOVE_CONSTRUCTIBLE_H
diff --git a/libc/src/__support/CPP/type_traits/is_trivially_copyable.h b/libc/src/__support/CPP/type_traits/is_trivially_copyable.h
index 68e56c8547834..a3e786fe1d141 100644
--- a/libc/src/__support/CPP/type_traits/is_trivially_copyable.h
+++ b/libc/src/__support/CPP/type_traits/is_trivially_copyable.h
@@ -19,6 +19,10 @@ template <class T>
 struct is_trivially_copyable
     : public integral_constant<bool, __is_trivially_copyable(T)> {};
 
+template <class T>
+LIBC_INLINE_VAR constexpr bool is_trivially_copyable_v =
+    is_trivially_copyable<T>::value;
+
 } // namespace cpp
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
index 3cea9772154fc..18b0631324f8f 100644
--- a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
@@ -26,7 +26,6 @@
 
 namespace LIBC_NAMESPACE_DECL {
 namespace fputil {
-
 struct FEnv {
   struct FPState {
     uint32_t ControlWord;
@@ -42,11 +41,11 @@ struct FEnv {
   static constexpr uint32_t DOWNWARD = 0x2;
   static constexpr uint32_t TOWARDZERO = 0x3;
 
-  static constexpr uint32_t INVALID = 0x1;
-  static constexpr uint32_t DIVBYZERO = 0x2;
-  static constexpr uint32_t OVERFLOW = 0x4;
-  static constexpr uint32_t UNDERFLOW = 0x8;
-  static constexpr uint32_t INEXACT = 0x10;
+  static constexpr uint32_t INVALID_F = 0x1;
+  static constexpr uint32_t DIVBYZERO_F = 0x2;
+  static constexpr uint32_t OVERFLOW_F = 0x4;
+  static constexpr uint32_t UNDERFLOW_F = 0x8;
+  static constexpr uint32_t INEXACT_F = 0x10;
 
   // Zero-th bit is the first bit.
   static constexpr uint32_t RoundingControlBitPosition = 22;
@@ -54,19 +53,19 @@ struct FEnv {
   static constexpr uint32_t ExceptionControlFlagsBitPosition = 8;
 
   LIBC_INLINE static uint32_t getStatusValueForExcept(int excepts) {
-    return ((excepts & FE_INVALID) ? INVALID : 0) |
-           ((excepts & FE_DIVBYZERO) ? DIVBYZERO : 0) |
-           ((excepts & FE_OVERFLOW) ? OVERFLOW : 0) |
-           ((excepts & FE_UNDERFLOW) ? UNDERFLOW : 0) |
-           ((excepts & FE_INEXACT) ? INEXACT : 0);
+    return ((excepts & FE_INVALID) ? INVALID_F : 0) |
+           ((excepts & FE_DIVBYZERO) ? DIVBYZERO_F : 0) |
+           ((excepts & FE_OVERFLOW) ? OVERFLOW_F : 0) |
+           ((excepts & FE_UNDERFLOW) ? UNDERFLOW_F : 0) |
+           ((excepts & FE_INEXACT) ? INEXACT_F : 0);
   }
 
   LIBC_INLINE static int exceptionStatusToMacro(uint32_t status) {
-    return ((status & INVALID) ? FE_INVALID : 0) |
-           ((status & DIVBYZERO) ? FE_DIVBYZERO : 0) |
-           ((status & OVERFLOW) ? FE_OVERFLOW : 0) |
-           ((status & UNDERFLOW) ? FE_UNDERFLOW : 0) |
-           ((status & INEXACT) ? FE_INEXACT : 0);
+    return ((status & INVALID_F) ? FE_INVALID : 0) |
+           ((status & DIVBYZERO_F) ? FE_DIVBYZERO : 0) |
+           ((status & OVERFLOW_F) ? FE_OVERFLOW : 0) |
+           ((status & UNDERFLOW_F) ? FE_UNDERFLOW : 0) |
+           ((status & INEXACT_F) ? FE_INEXACT : 0);
   }
 
   static uint32_t getControlWord() {
@@ -171,36 +170,36 @@ LIBC_INLINE int raise_except(int excepts) {
   uint32_t toRaise = FEnv::getStatusValueForExcept(excepts);
   int result = 0;
 
-  if (toRaise & FEnv::INVALID) {
+  if (toRaise & FEnv::INVALID_F) {
     divfunc(zero, zero);
     uint32_t statusWord = FEnv::getStatusWord();
     if (!((statusWord >> FEnv::ExceptionStatusFlagsBitPosition) &
-          FEnv::INVALID))
+          FEnv::INVALID_F))
       result = -1;
   }
 
-  if (toRaise & FEnv::DIVBYZERO) {
+  if (toRaise & FEnv::DIVBYZERO_F) {
     divfunc(one, zero);
     uint32_t statusWord = FEnv::getStatusWord();
     if (!((statusWord >> FEnv::ExceptionStatusFlagsBitPosition) &
-          FEnv::DIVBYZERO))
+          FEnv::DIVBYZERO_F))
       result = -1;
   }
-  if (toRaise & FEnv::OVERFLOW) {
+  if (toRaise & FEnv::OVERFLOW_F) {
     divfunc(largeValue, smallValue);
     uint32_t statusWord = FEnv::getStatusWord();
     if (!((statusWord >> FEnv::ExceptionStatusFlagsBitPosition) &
-          FEnv::OVERFLOW))
+          FEnv::OVERFLOW_F))
       result = -1;
   }
-  if (toRaise & FEnv::UNDERFLOW) {
+  if (toRaise & FEnv::UNDERFLOW_F) {
     divfunc(smallValue, largeValue);
     uint32_t statusWord = FEnv::getStatusWord();
     if (!((statusWord >> FEnv::ExceptionStatusFlagsBitPosition) &
-          FEnv::UNDERFLOW))
+          FEnv::UNDERFLOW_F))
       result = -1;
   }
-  if (toRaise & FEnv::INEXACT) {
+  if (toRaise & FEnv::INEXACT_F) {
     float two = 2.0f;
     float three = 3.0f;
     // 2.0 / 3.0 cannot be represented exactly in any radix 2 floating point
@@ -208,7 +207,7 @@ LIBC_INLINE int raise_except(int excepts) {
     divfunc(two, three);
     uint32_t statusWord = FEnv::getStatusWord();
     if (!((statusWord >> FEnv::ExceptionStatusFlagsBitPosition) &
-          FEnv::INEXACT))
+          FEnv::INEXACT_F))
       result = -1;
   }
   return result;
@@ -278,7 +277,6 @@ LIBC_INLINE int set_env(const fenv_t *envp) {
   FEnv::writeStatusWord(state->StatusWord);
   return 0;
 }
-
 } // namespace fputil
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/__support/GPU/allocator.cpp b/libc/src/__support/GPU/allocator.cpp
index f98e610104797..ac335a1b9aab0 100644
--- a/libc/src/__support/GPU/allocator.cpp
+++ b/libc/src/__support/GPU/allocator.cpp
@@ -17,7 +17,7 @@ namespace {
 
 void *rpc_allocate(uint64_t size) {
   void *ptr = nullptr;
-  rpc::Client::Port port = rpc::client.open<RPC_MALLOC>();
+  rpc::Client::Port port = rpc::client.open<LIBC_MALLOC>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) { buffer->data[0] = size; },
       [&](rpc::Buffer *buffer, uint32_t) {
@@ -28,7 +28,7 @@ void *rpc_allocate(uint64_t size) {
 }
 
 void rpc_free(void *ptr) {
-  rpc::Client::Port port = rpc::client.open<RPC_FREE>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FREE>();
   port.send([=](rpc::Buffer *buffer, uint32_t) {
     buffer->data[0] = reinterpret_cast<uintptr_t>(ptr);
   });
diff --git a/libc/src/__support/OSUtil/gpu/exit.cpp b/libc/src/__support/OSUtil/gpu/exit.cpp
index 0cb266a42d180..85f8183aafa98 100644
--- a/libc/src/__support/OSUtil/gpu/exit.cpp
+++ b/libc/src/__support/OSUtil/gpu/exit.cpp
@@ -18,7 +18,7 @@ namespace internal {
 
 [[noreturn]] void exit(int status) {
   // We want to first make sure the server is listening before we exit.
-  rpc::Client::Port port = rpc::client.open<RPC_EXIT>();
+  rpc::Client::Port port = rpc::client.open<LIBC_EXIT>();
   port.send_and_recv([](rpc::Buffer *, uint32_t) {},
                      [](rpc::Buffer *, uint32_t) {});
   port.send([&](rpc::Buffer *buffer, uint32_t) {
diff --git a/libc/src/__support/OSUtil/gpu/io.cpp b/libc/src/__support/OSUtil/gpu/io.cpp
index f70c2e798cfe1..5d107ab50ae88 100644
--- a/libc/src/__support/OSUtil/gpu/io.cpp
+++ b/libc/src/__support/OSUtil/gpu/io.cpp
@@ -15,7 +15,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 void write_to_stderr(cpp::string_view msg) {
-  rpc::Client::Port port = rpc::client.open<RPC_WRITE_TO_STDERR>();
+  rpc::Client::Port port = rpc::client.open<LIBC_WRITE_TO_STDERR>();
   port.send_n(msg.data(), msg.size());
   port.recv([](rpc::Buffer *, uint32_t) { /* void */ });
   port.close();
diff --git a/libc/src/__support/OSUtil/linux/exit.cpp b/libc/src/__support/OSUtil/linux/exit.cpp
index 9c64ce42be185..e26b90f6b18eb 100644
--- a/libc/src/__support/OSUtil/linux/exit.cpp
+++ b/libc/src/__support/OSUtil/linux/exit.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/OSUtil/linux/syscall.h" // syscall_impl
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "syscall.h"     // For internal syscall function.
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/__support/RPC/rpc_client.cpp b/libc/src/__support/RPC/rpc_client.cpp
index c26cf9ca2ddbe..625e474a46783 100644
--- a/libc/src/__support/RPC/rpc_client.cpp
+++ b/libc/src/__support/RPC/rpc_client.cpp
@@ -13,14 +13,10 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace rpc {
 
-/// The libc client instance used to communicate with the server.
-Client client;
-
-/// Externally visible symbol to signify the usage of an RPC client to
-/// whomever needs to run the server as well as provide a way to initialize
-/// the client with a copy..
-extern "C" [[gnu::visibility("protected")]] const void *__llvm_libc_rpc_client =
-    &client;
+/// The libc client instance used to communicate with the server. Externally
+/// visible symbol to signify the usage of an RPC client to whomever needs to
+/// run the server as well as provide a way to initialize the client.
+[[gnu::visibility("protected")]] Client client;
 
 } // namespace rpc
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/RPC/rpc_client.h b/libc/src/__support/RPC/rpc_client.h
index 7a62e4e983ad0..199803badf1a9 100644
--- a/libc/src/__support/RPC/rpc_client.h
+++ b/libc/src/__support/RPC/rpc_client.h
@@ -29,7 +29,7 @@ static_assert(cpp::is_trivially_copyable<Client>::value &&
               "The client is not trivially copyable from the server");
 
 /// The libc client instance used to communicate with the server.
-extern Client client;
+[[gnu::visibility("protected")]] extern Client client asm("__llvm_rpc_client");
 
 } // namespace rpc
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/complex_type.h b/libc/src/__support/complex_type.h
index d6b5eec8fd6b2..a6207d38d0eb5 100644
--- a/libc/src/__support/complex_type.h
+++ b/libc/src/__support/complex_type.h
@@ -10,11 +10,72 @@
 #define LLVM_LIBC_SRC___SUPPORT_COMPLEX_TYPE_H
 
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/complex_types.h"
+#include "src/__support/macros/properties/types.h"
 
 namespace LIBC_NAMESPACE_DECL {
 template <typename T> struct Complex {
   T real;
   T imag;
 };
+
+template <typename T> struct make_complex;
+
+template <> struct make_complex<float> {
+  using type = _Complex float;
+};
+template <> struct make_complex<double> {
+  using type = _Complex double;
+};
+template <> struct make_complex<long double> {
+  using type = _Complex long double;
+};
+
+#if defined(LIBC_TYPES_HAS_CFLOAT16)
+template <> struct make_complex<float16> {
+  using type = cfloat16;
+};
+#endif
+#if defined(LIBC_TYPES_HAS_CFLOAT128) &&                                       \
+    !defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
+template <> struct make_complex<float128> {
+  using type = cfloat128;
+};
+#endif
+
+template <typename T> using make_complex_t = typename make_complex<T>::type;
+
+template <typename T> struct make_real;
+
+template <> struct make_real<_Complex float> {
+  using type = float;
+};
+template <> struct make_real<_Complex double> {
+  using type = double;
+};
+template <> struct make_real<_Complex long double> {
+  using type = long double;
+};
+
+#if defined(LIBC_TYPES_HAS_CFLOAT16)
+template <> struct make_real<cfloat16> {
+  using type = float16;
+};
+#endif
+#if defined(LIBC_TYPES_HAS_CFLOAT128) &&                                       \
+    !defined(LIBC_TYPES_CFLOAT128_IS_COMPLEX_LONG_DOUBLE)
+template <> struct make_real<cfloat128> {
+  using type = float128;
+};
+#endif
+
+template <typename T> using make_real_t = typename make_real<T>::type;
+
+template <typename T> LIBC_INLINE constexpr T conjugate(T c) {
+  Complex<make_real_t<T>> c_c = cpp::bit_cast<Complex<make_real_t<T>>>(c);
+  c_c.imag = -c_c.imag;
+  return cpp::bit_cast<T>(c_c);
+}
+
 } // namespace LIBC_NAMESPACE_DECL
 #endif // LLVM_LIBC_SRC___SUPPORT_COMPLEX_TYPE_H
diff --git a/libc/src/__support/ctype_utils.h b/libc/src/__support/ctype_utils.h
index 91f6ce8cabd8d..be0f25330af9e 100644
--- a/libc/src/__support/ctype_utils.h
+++ b/libc/src/__support/ctype_utils.h
@@ -15,44 +15,567 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
-// ------------------------------------------------------
-// Rationale: Since these classification functions are
-// called in other functions, we will avoid the overhead
-// of a function call by inlining them.
-// ------------------------------------------------------
+// -----------------------------------------------------------------------------
+// ******************                 WARNING                 ******************
+// ****************** DO NOT TRY TO OPTIMIZE THESE FUNCTIONS! ******************
+// -----------------------------------------------------------------------------
+// This switch/case form is easier for the compiler to understand, and is
+// optimized into a form that is almost always the same as or better than
+// versions written by hand (see https://godbolt.org/z/qvrebqvvr). Also this
+// form makes these functions encoding independent. If you want to rewrite these
+// functions, make sure you have benchmarks to show your new solution is faster,
+// as well as a way to support non-ASCII character encodings.
 
-LIBC_INLINE static constexpr bool isalpha(unsigned ch) {
-  return (ch | 32) - 'a' < 26;
+// Similarly, do not change these functions to use case ranges. e.g.
+//  bool islower(int ch) {
+//    switch(ch) {
+//    case 'a'...'z':
+//      return true;
+//    }
+//  }
+// This assumes the character ranges are contiguous, which they aren't in
+// EBCDIC. Technically we could use some smaller ranges, but that's even harder
+// to read.
+
+LIBC_INLINE static constexpr bool islower(int ch) {
+  switch (ch) {
+  case 'a':
+  case 'b':
+  case 'c':
+  case 'd':
+  case 'e':
+  case 'f':
+  case 'g':
+  case 'h':
+  case 'i':
+  case 'j':
+  case 'k':
+  case 'l':
+  case 'm':
+  case 'n':
+  case 'o':
+  case 'p':
+  case 'q':
+  case 'r':
+  case 's':
+  case 't':
+  case 'u':
+  case 'v':
+  case 'w':
+  case 'x':
+  case 'y':
+  case 'z':
+    return true;
+  default:
+    return false;
+  }
 }
 
-LIBC_INLINE static constexpr bool isdigit(unsigned ch) {
-  return (ch - '0') < 10;
+LIBC_INLINE static constexpr bool isupper(int ch) {
+  switch (ch) {
+  case 'A':
+  case 'B':
+  case 'C':
+  case 'D':
+  case 'E':
+  case 'F':
+  case 'G':
+  case 'H':
+  case 'I':
+  case 'J':
+  case 'K':
+  case 'L':
+  case 'M':
+  case 'N':
+  case 'O':
+  case 'P':
+  case 'Q':
+  case 'R':
+  case 'S':
+  case 'T':
+  case 'U':
+  case 'V':
+  case 'W':
+  case 'X':
+  case 'Y':
+  case 'Z':
+    return true;
+  default:
+    return false;
+  }
 }
 
-LIBC_INLINE static constexpr bool isalnum(unsigned ch) {
-  return isalpha(ch) || isdigit(ch);
+LIBC_INLINE static constexpr bool isdigit(int ch) {
+  switch (ch) {
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7':
+  case '8':
+  case '9':
+    return true;
+  default:
+    return false;
+  }
 }
 
-LIBC_INLINE static constexpr bool isgraph(unsigned ch) {
-  return 0x20 < ch && ch < 0x7f;
+LIBC_INLINE static constexpr int tolower(int ch) {
+  switch (ch) {
+  case 'A':
+    return 'a';
+  case 'B':
+    return 'b';
+  case 'C':
+    return 'c';
+  case 'D':
+    return 'd';
+  case 'E':
+    return 'e';
+  case 'F':
+    return 'f';
+  case 'G':
+    return 'g';
+  case 'H':
+    return 'h';
+  case 'I':
+    return 'i';
+  case 'J':
+    return 'j';
+  case 'K':
+    return 'k';
+  case 'L':
+    return 'l';
+  case 'M':
+    return 'm';
+  case 'N':
+    return 'n';
+  case 'O':
+    return 'o';
+  case 'P':
+    return 'p';
+  case 'Q':
+    return 'q';
+  case 'R':
+    return 'r';
+  case 'S':
+    return 's';
+  case 'T':
+    return 't';
+  case 'U':
+    return 'u';
+  case 'V':
+    return 'v';
+  case 'W':
+    return 'w';
+  case 'X':
+    return 'x';
+  case 'Y':
+    return 'y';
+  case 'Z':
+    return 'z';
+  default:
+    return ch;
+  }
 }
 
-LIBC_INLINE static constexpr bool islower(unsigned ch) {
-  return (ch - 'a') < 26;
+LIBC_INLINE static constexpr int toupper(int ch) {
+  switch (ch) {
+  case 'a':
+    return 'A';
+  case 'b':
+    return 'B';
+  case 'c':
+    return 'C';
+  case 'd':
+    return 'D';
+  case 'e':
+    return 'E';
+  case 'f':
+    return 'F';
+  case 'g':
+    return 'G';
+  case 'h':
+    return 'H';
+  case 'i':
+    return 'I';
+  case 'j':
+    return 'J';
+  case 'k':
+    return 'K';
+  case 'l':
+    return 'L';
+  case 'm':
+    return 'M';
+  case 'n':
+    return 'N';
+  case 'o':
+    return 'O';
+  case 'p':
+    return 'P';
+  case 'q':
+    return 'Q';
+  case 'r':
+    return 'R';
+  case 's':
+    return 'S';
+  case 't':
+    return 'T';
+  case 'u':
+    return 'U';
+  case 'v':
+    return 'V';
+  case 'w':
+    return 'W';
+  case 'x':
+    return 'X';
+  case 'y':
+    return 'Y';
+  case 'z':
+    return 'Z';
+  default:
+    return ch;
+  }
 }
 
-LIBC_INLINE static constexpr bool isupper(unsigned ch) {
-  return (ch - 'A') < 26;
+LIBC_INLINE static constexpr bool isalpha(int ch) {
+  switch (ch) {
+  case 'a':
+  case 'b':
+  case 'c':
+  case 'd':
+  case 'e':
+  case 'f':
+  case 'g':
+  case 'h':
+  case 'i':
+  case 'j':
+  case 'k':
+  case 'l':
+  case 'm':
+  case 'n':
+  case 'o':
+  case 'p':
+  case 'q':
+  case 'r':
+  case 's':
+  case 't':
+  case 'u':
+  case 'v':
+  case 'w':
+  case 'x':
+  case 'y':
+  case 'z':
+  case 'A':
+  case 'B':
+  case 'C':
+  case 'D':
+  case 'E':
+  case 'F':
+  case 'G':
+  case 'H':
+  case 'I':
+  case 'J':
+  case 'K':
+  case 'L':
+  case 'M':
+  case 'N':
+  case 'O':
+  case 'P':
+  case 'Q':
+  case 'R':
+  case 'S':
+  case 'T':
+  case 'U':
+  case 'V':
+  case 'W':
+  case 'X':
+  case 'Y':
+  case 'Z':
+    return true;
+  default:
+    return false;
+  }
 }
 
-LIBC_INLINE static constexpr bool isspace(unsigned ch) {
-  return ch == ' ' || (ch - '\t') < 5;
+LIBC_INLINE static constexpr bool isalnum(int ch) {
+  switch (ch) {
+  case 'a':
+  case 'b':
+  case 'c':
+  case 'd':
+  case 'e':
+  case 'f':
+  case 'g':
+  case 'h':
+  case 'i':
+  case 'j':
+  case 'k':
+  case 'l':
+  case 'm':
+  case 'n':
+  case 'o':
+  case 'p':
+  case 'q':
+  case 'r':
+  case 's':
+  case 't':
+  case 'u':
+  case 'v':
+  case 'w':
+  case 'x':
+  case 'y':
+  case 'z':
+  case 'A':
+  case 'B':
+  case 'C':
+  case 'D':
+  case 'E':
+  case 'F':
+  case 'G':
+  case 'H':
+  case 'I':
+  case 'J':
+  case 'K':
+  case 'L':
+  case 'M':
+  case 'N':
+  case 'O':
+  case 'P':
+  case 'Q':
+  case 'R':
+  case 'S':
+  case 'T':
+  case 'U':
+  case 'V':
+  case 'W':
+  case 'X':
+  case 'Y':
+  case 'Z':
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7':
+  case '8':
+  case '9':
+    return true;
+  default:
+    return false;
+  }
 }
 
-LIBC_INLINE static constexpr int tolower(int ch) {
-  if (isupper(ch))
-    return ch + ('a' - 'A');
-  return ch;
+LIBC_INLINE static constexpr int b36_char_to_int(int ch) {
+  switch (ch) {
+  case '0':
+    return 0;
+  case '1':
+    return 1;
+  case '2':
+    return 2;
+  case '3':
+    return 3;
+  case '4':
+    return 4;
+  case '5':
+    return 5;
+  case '6':
+    return 6;
+  case '7':
+    return 7;
+  case '8':
+    return 8;
+  case '9':
+    return 9;
+  case 'a':
+  case 'A':
+    return 10;
+  case 'b':
+  case 'B':
+    return 11;
+  case 'c':
+  case 'C':
+    return 12;
+  case 'd':
+  case 'D':
+    return 13;
+  case 'e':
+  case 'E':
+    return 14;
+  case 'f':
+  case 'F':
+    return 15;
+  case 'g':
+  case 'G':
+    return 16;
+  case 'h':
+  case 'H':
+    return 17;
+  case 'i':
+  case 'I':
+    return 18;
+  case 'j':
+  case 'J':
+    return 19;
+  case 'k':
+  case 'K':
+    return 20;
+  case 'l':
+  case 'L':
+    return 21;
+  case 'm':
+  case 'M':
+    return 22;
+  case 'n':
+  case 'N':
+    return 23;
+  case 'o':
+  case 'O':
+    return 24;
+  case 'p':
+  case 'P':
+    return 25;
+  case 'q':
+  case 'Q':
+    return 26;
+  case 'r':
+  case 'R':
+    return 27;
+  case 's':
+  case 'S':
+    return 28;
+  case 't':
+  case 'T':
+    return 29;
+  case 'u':
+  case 'U':
+    return 30;
+  case 'v':
+  case 'V':
+    return 31;
+  case 'w':
+  case 'W':
+    return 32;
+  case 'x':
+  case 'X':
+    return 33;
+  case 'y':
+  case 'Y':
+    return 34;
+  case 'z':
+  case 'Z':
+    return 35;
+  default:
+    return 0;
+  }
+}
+
+LIBC_INLINE static constexpr int int_to_b36_char(int num) {
+  // Can't actually use LIBC_ASSERT here because it depends on integer_to_string
+  // which depends on this.
+
+  // LIBC_ASSERT(num < 36);
+  switch (num) {
+  case 0:
+    return '0';
+  case 1:
+    return '1';
+  case 2:
+    return '2';
+  case 3:
+    return '3';
+  case 4:
+    return '4';
+  case 5:
+    return '5';
+  case 6:
+    return '6';
+  case 7:
+    return '7';
+  case 8:
+    return '8';
+  case 9:
+    return '9';
+  case 10:
+    return 'a';
+  case 11:
+    return 'b';
+  case 12:
+    return 'c';
+  case 13:
+    return 'd';
+  case 14:
+    return 'e';
+  case 15:
+    return 'f';
+  case 16:
+    return 'g';
+  case 17:
+    return 'h';
+  case 18:
+    return 'i';
+  case 19:
+    return 'j';
+  case 20:
+    return 'k';
+  case 21:
+    return 'l';
+  case 22:
+    return 'm';
+  case 23:
+    return 'n';
+  case 24:
+    return 'o';
+  case 25:
+    return 'p';
+  case 26:
+    return 'q';
+  case 27:
+    return 'r';
+  case 28:
+    return 's';
+  case 29:
+    return 't';
+  case 30:
+    return 'u';
+  case 31:
+    return 'v';
+  case 32:
+    return 'w';
+  case 33:
+    return 'x';
+  case 34:
+    return 'y';
+  case 35:
+    return 'z';
+  default:
+    return '!';
+  }
+}
+
+LIBC_INLINE static constexpr bool isspace(int ch) {
+  switch (ch) {
+  case ' ':
+  case '\t':
+  case '\n':
+  case '\v':
+  case '\f':
+  case '\r':
+    return true;
+  default:
+    return false;
+  }
+}
+
+// not yet encoding independent.
+LIBC_INLINE static constexpr bool isgraph(int ch) {
+  return 0x20 < ch && ch < 0x7f;
 }
 
 } // namespace internal
diff --git a/libc/src/__support/high_precision_decimal.h b/libc/src/__support/high_precision_decimal.h
index 20088d6d79791..922dce484aa6b 100644
--- a/libc/src/__support/high_precision_decimal.h
+++ b/libc/src/__support/high_precision_decimal.h
@@ -178,9 +178,11 @@ class HighPrecisionDecimal {
       if (digit_index >= this->num_digits) {
         return new_digits - 1;
       }
-      if (this->digits[digit_index] != power_of_five[digit_index] - '0') {
+      if (this->digits[digit_index] !=
+          internal::b36_char_to_int(power_of_five[digit_index])) {
         return new_digits -
-               ((this->digits[digit_index] < power_of_five[digit_index] - '0')
+               ((this->digits[digit_index] <
+                 internal::b36_char_to_int(power_of_five[digit_index]))
                     ? 1
                     : 0);
       }
@@ -337,8 +339,8 @@ class HighPrecisionDecimal {
         }
         ++total_digits;
         if (this->num_digits < MAX_NUM_DIGITS) {
-          this->digits[this->num_digits] =
-              static_cast<uint8_t>(num_string[num_cur] - '0');
+          this->digits[this->num_digits] = static_cast<uint8_t>(
+              internal::b36_char_to_int(num_string[num_cur]));
           ++this->num_digits;
         } else if (num_string[num_cur] != '0') {
           this->truncated = true;
diff --git a/libc/src/__support/integer_literals.h b/libc/src/__support/integer_literals.h
index 4c5c4c4166681..0298ec7d088d6 100644
--- a/libc/src/__support/integer_literals.h
+++ b/libc/src/__support/integer_literals.h
@@ -13,12 +13,13 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_INTEGER_LITERALS_H
 #define LLVM_LIBC_SRC___SUPPORT_INTEGER_LITERALS_H
 
-#include "src/__support/CPP/limits.h"        // CHAR_BIT
+#include "src/__support/CPP/limits.h" // CHAR_BIT
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
 #include "src/__support/macros/config.h"
-#include "src/__support/uint128.h"           // UInt128
-#include <stddef.h>                          // size_t
-#include <stdint.h>                          // uintxx_t
+#include "src/__support/uint128.h" // UInt128
+#include <stddef.h>                // size_t
+#include <stdint.h>                // uintxx_t
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -75,26 +76,13 @@ template <typename T, int base> struct DigitBuffer {
       push(*str);
   }
 
-  // Returns the digit for a particular character.
-  // Returns INVALID_DIGIT if the character is invalid.
-  LIBC_INLINE static constexpr uint8_t get_digit_value(const char c) {
-    const auto to_lower = [](char c) { return c | 32; };
-    const auto is_digit = [](char c) { return c >= '0' && c <= '9'; };
-    const auto is_alpha = [](char c) {
-      return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
-    };
-    if (is_digit(c))
-      return static_cast<uint8_t>(c - '0');
-    if (base > 10 && is_alpha(c))
-      return static_cast<uint8_t>(to_lower(c) - 'a' + 10);
-    return INVALID_DIGIT;
-  }
-
   // Adds a single character to this buffer.
   LIBC_INLINE constexpr void push(char c) {
     if (c == '\'')
       return; // ' is valid but not taken into account.
-    const uint8_t value = get_digit_value(c);
+    const int b36_val = internal::b36_char_to_int(c);
+    const uint8_t value = static_cast<uint8_t>(
+        b36_val < base && (b36_val != 0 || c == '0') ? b36_val : INVALID_DIGIT);
     if (value == INVALID_DIGIT || size >= MAX_DIGITS) {
       // During constant evaluation `__builtin_unreachable` will halt the
       // compiler as it is not executable. This is preferable over `assert` that
diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h
index 40d45a05ceadb..ea620087584cb 100644
--- a/libc/src/__support/integer_to_string.h
+++ b/libc/src/__support/integer_to_string.h
@@ -69,6 +69,7 @@
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/big_int.h" // make_integral_or_big_int_unsigned_t
 #include "src/__support/common.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -214,9 +215,9 @@ template <typename T, typename Fmt = radix::Dec> class IntegerToString {
     using UNSIGNED_T = make_integral_or_big_int_unsigned_t<T>;
 
     LIBC_INLINE static char digit_char(uint8_t digit) {
-      if (digit < 10)
-        return '0' + static_cast<char>(digit);
-      return (Fmt::IS_UPPERCASE ? 'A' : 'a') + static_cast<char>(digit - 10);
+      const int result = internal::int_to_b36_char(digit);
+      return static_cast<char>(Fmt::IS_UPPERCASE ? internal::toupper(result)
+                                                 : result);
     }
 
     LIBC_INLINE static void
diff --git a/libc/src/__support/macros/optimization.h b/libc/src/__support/macros/optimization.h
index 41ecd2bd6d719..a2634950d431b 100644
--- a/libc/src/__support/macros/optimization.h
+++ b/libc/src/__support/macros/optimization.h
@@ -48,7 +48,7 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 
 #ifndef LIBC_MATH
 #define LIBC_MATH 0
-#else
+#endif // LIBC_MATH
 
 #if (LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS)
 #define LIBC_MATH_HAS_SKIP_ACCURATE_PASS
@@ -58,6 +58,4 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 #define LIBC_MATH_HAS_SMALL_TABLES
 #endif
 
-#endif // LIBC_MATH
-
 #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
diff --git a/libc/src/__support/macros/properties/os.h b/libc/src/__support/macros/properties/os.h
index 1c8fd5721ce67..807ce1812735c 100644
--- a/libc/src/__support/macros/properties/os.h
+++ b/libc/src/__support/macros/properties/os.h
@@ -25,18 +25,6 @@
 #define LIBC_TARGET_OS_IS_WINDOWS
 #endif
 
-#if (defined(__apple__) || defined(__APPLE__) || defined(__MACH__))
-// From https://stackoverflow.com/a/49560690
-#include "TargetConditionals.h"
-#if defined(TARGET_OS_OSX)
-#define LIBC_TARGET_OS_IS_MACOS
-#endif
-#if defined(TARGET_OS_IPHONE)
-// This is set for any non-Mac Apple products (IOS, TV, WATCH)
-#define LIBC_TARGET_OS_IS_IPHONE
-#endif
-#endif
-
 #if defined(__Fuchsia__)
 #define LIBC_TARGET_OS_IS_FUCHSIA
 #endif
diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h
index 80ea334d15c03..b4d5646822df3 100644
--- a/libc/src/__support/str_to_float.h
+++ b/libc/src/__support/str_to_float.h
@@ -909,7 +909,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT,
       cpp::numeric_limits<StorageType>::max() / BASE;
   while (true) {
     if (isdigit(src[index])) {
-      uint32_t digit = src[index] - '0';
+      uint32_t digit = b36_char_to_int(src[index]);
       seen_digit = true;
 
       if (mantissa < bitstype_max_div_by_base) {
diff --git a/libc/src/__support/str_to_integer.h b/libc/src/__support/str_to_integer.h
index 86611f9a6902d..8e569e8a7feb0 100644
--- a/libc/src/__support/str_to_integer.h
+++ b/libc/src/__support/str_to_integer.h
@@ -42,14 +42,6 @@ first_non_whitespace(const char *__restrict src,
   return src + src_cur;
 }
 
-LIBC_INLINE int b36_char_to_int(char input) {
-  if (isdigit(input))
-    return input - '0';
-  if (isalpha(input))
-    return (input | 32) + 10 - 'a';
-  return 0;
-}
-
 // checks if the next 3 characters of the string pointer are the start of a
 // hexadecimal number. Does not advance the string pointer.
 LIBC_INLINE bool
@@ -57,7 +49,7 @@ is_hex_start(const char *__restrict src,
              size_t src_len = cpp::numeric_limits<size_t>::max()) {
   if (src_len < 3)
     return false;
-  return *src == '0' && (*(src + 1) | 32) == 'x' && isalnum(*(src + 2)) &&
+  return *src == '0' && tolower(*(src + 1)) == 'x' && isalnum(*(src + 2)) &&
          b36_char_to_int(*(src + 2)) < 16;
 }
 
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index 57fcc7bb67a6a..f7aeb5b709aa3 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -162,7 +162,7 @@ class RwState {
   LIBC_INLINE constexpr bool has_active_reader() const {
     return state >= ACTIVE_READER_COUNT_UNIT;
   }
-  LIBC_INLINE constexpr bool has_acitve_owner() const {
+  LIBC_INLINE constexpr bool has_active_owner() const {
     return has_active_reader() || has_active_writer();
   }
   LIBC_INLINE constexpr bool has_last_reader() const {
@@ -193,7 +193,7 @@ class RwState {
       }
       __builtin_unreachable();
     } else
-      return !has_acitve_owner();
+      return !has_active_owner();
   }
 
   // This function check if it is possible to grow the reader count without
@@ -548,7 +548,7 @@ class RwLock {
   [[nodiscard]]
   LIBC_INLINE LockResult check_for_destroy() {
     RwState old = RwState::load(state, cpp::MemoryOrder::RELAXED);
-    if (old.has_acitve_owner())
+    if (old.has_active_owner())
       return LockResult::Busy;
     return LockResult::Success;
   }
diff --git a/libc/src/__support/time/CMakeLists.txt b/libc/src/__support/time/CMakeLists.txt
index 89ddffb099388..8247e792e8410 100644
--- a/libc/src/__support/time/CMakeLists.txt
+++ b/libc/src/__support/time/CMakeLists.txt
@@ -1,7 +1,3 @@
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
-  add_subdirectory(${LIBC_TARGET_OS})
-endif()
-
 add_header_library(
   units
   HDRS
@@ -10,3 +6,16 @@ add_header_library(
     libc.src.__support.common
     libc.hdr.types.time_t
 )
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+else()
+  return()
+endif()
+
+add_object_library(
+  clock_gettime
+  ALIAS
+  DEPENDS
+    libc.src.__support.time.${LIBC_TARGET_OS}.clock_gettime
+)
diff --git a/libc/src/__support/time/linux/clock_gettime.h b/libc/src/__support/time/clock_gettime.h
similarity index 72%
rename from libc/src/__support/time/linux/clock_gettime.h
rename to libc/src/__support/time/clock_gettime.h
index f7f996ce7c197..584bf546cd60c 100644
--- a/libc/src/__support/time/linux/clock_gettime.h
+++ b/libc/src/__support/time/clock_gettime.h
@@ -6,21 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
-#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
 
 #include "hdr/types/clockid_t.h"
 #include "hdr/types/struct_timespec.h"
 #include "src/__support/error_or.h"
 
-#if defined(SYS_clock_gettime64)
-#include <linux/time_types.h>
-#endif
-
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts);
 } // namespace internal
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/gpu/CMakeLists.txt b/libc/src/__support/time/gpu/CMakeLists.txt
new file mode 100644
index 0000000000000..fc465e0cea25f
--- /dev/null
+++ b/libc/src/__support/time/gpu/CMakeLists.txt
@@ -0,0 +1,22 @@
+add_object_library(
+  time_utils
+  SRCS
+    time_utils.cpp
+  HDRS
+    time_utils.h
+  DEPENDS
+    libc.hdr.types.clock_t
+    libc.hdr.time_macros
+)
+
+add_object_library(
+  clock_gettime
+  SRCS
+    clock_gettime.cpp
+  HDRS
+    ../clock_gettime.h
+  DEPENDS
+    libc.hdr.types.clockid_t
+    libc.hdr.types.struct_timespec
+    .time_utils
+)
diff --git a/libc/src/__support/time/gpu/clock_gettime.cpp b/libc/src/__support/time/gpu/clock_gettime.cpp
new file mode 100644
index 0000000000000..cede72a1f35da
--- /dev/null
+++ b/libc/src/__support/time/gpu/clock_gettime.cpp
@@ -0,0 +1,33 @@
+//===---------- GPU implementation of the clock_gettime function ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/clock_gettime.h"
+
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/gpu/time_utils.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+constexpr uint64_t TICKS_PER_SEC = 1000000000UL;
+
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
+  if (clockid != CLOCK_MONOTONIC || !ts)
+    return cpp::unexpected(-1);
+
+  uint64_t ns_per_tick = TICKS_PER_SEC / GPU_CLOCKS_PER_SEC;
+  uint64_t ticks = gpu::fixed_frequency_clock();
+
+  ts->tv_nsec = (ticks * ns_per_tick) % TICKS_PER_SEC;
+  ts->tv_sec = (ticks * ns_per_tick) / TICKS_PER_SEC;
+
+  return 0;
+}
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/time/gpu/time_utils.cpp b/libc/src/__support/time/gpu/time_utils.cpp
similarity index 100%
rename from libc/src/time/gpu/time_utils.cpp
rename to libc/src/__support/time/gpu/time_utils.cpp
diff --git a/libc/src/time/gpu/time_utils.h b/libc/src/__support/time/gpu/time_utils.h
similarity index 100%
rename from libc/src/time/gpu/time_utils.h
rename to libc/src/__support/time/gpu/time_utils.h
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
index 94ed09e652152..6fec7eeba99ad 100644
--- a/libc/src/__support/time/linux/CMakeLists.txt
+++ b/libc/src/__support/time/linux/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_object_library(
   clock_gettime
   HDRS
-    clock_gettime.h
+    ../clock_gettime.h
   SRCS
     clock_gettime.cpp
   DEPENDS
diff --git a/libc/src/__support/time/linux/clock_conversion.h b/libc/src/__support/time/linux/clock_conversion.h
index 7a52873263a14..ac5357d308d7c 100644
--- a/libc/src/__support/time/linux/clock_conversion.h
+++ b/libc/src/__support/time/linux/clock_conversion.h
@@ -10,7 +10,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
 
 #include "src/__support/macros/config.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/__support/time/units.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/__support/time/linux/clock_gettime.cpp b/libc/src/__support/time/linux/clock_gettime.cpp
index 3a0eca417724a..944fc0a2b80fe 100644
--- a/libc/src/__support/time/linux/clock_gettime.cpp
+++ b/libc/src/__support/time/linux/clock_gettime.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "hdr/types/clockid_t.h"
 #include "hdr/types/struct_timespec.h"
 #include "src/__support/OSUtil/linux/vdso.h"
diff --git a/libc/src/__support/time/windows/CMakeLists.txt b/libc/src/__support/time/windows/CMakeLists.txt
new file mode 100644
index 0000000000000..dd0ac2f2f79ae
--- /dev/null
+++ b/libc/src/__support/time/windows/CMakeLists.txt
@@ -0,0 +1,26 @@
+add_header_library(
+  performance_counter 
+  HDRS
+    performance_counter.h
+  DEPENDS
+    libc.src.__support.CPP.atomic
+    libc.src.__support.common
+)
+
+add_object_library(
+  clock_gettime
+  HDRS
+    ../clock_gettime.h
+  SRCS
+    clock_gettime.cpp
+  DEPENDS
+    .performance_counter
+    libc.hdr.types.struct_timespec
+    libc.hdr.types.clockid_t
+    libc.hdr.errno_macros
+    libc.src.__support.common
+    libc.src.__support.error_or
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.limits
+)
diff --git a/libc/src/__support/time/windows/clock_gettime.cpp b/libc/src/__support/time/windows/clock_gettime.cpp
new file mode 100644
index 0000000000000..813eae0df9705
--- /dev/null
+++ b/libc/src/__support/time/windows/clock_gettime.cpp
@@ -0,0 +1,134 @@
+//===--- clock_gettime windows implementation -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/time_macros.h"
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/macros/optimization.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/units.h"
+#include "src/__support/time/windows/performance_counter.h"
+
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <Windows.h>
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
+  using namespace time_units;
+  constexpr unsigned long long HNS_PER_SEC = 1_s_ns / 100ULL;
+  constexpr long long SEC_LIMIT =
+      cpp::numeric_limits<decltype(ts->tv_sec)>::max();
+  ErrorOr<int> ret = 0;
+  switch (clockid) {
+  default:
+    ret = cpp::unexpected(EINVAL);
+    break;
+
+  case CLOCK_MONOTONIC: {
+    // see
+    // https://learn.microsoft.com/en-us/windows/win32/sysinfo/acquiring-high-resolution-time-stamps
+    // Is the performance counter monotonic (non-decreasing)?
+    // Yes. performance_counter does not go backward.
+    [[clang::uninitialized]] LARGE_INTEGER buffer;
+    // On systems that run Windows XP or later, the function will always
+    // succeed and will thus never return zero.
+    ::QueryPerformanceCounter(&buffer);
+    long long freq = performance_counter::get_ticks_per_second();
+    long long ticks = buffer.QuadPart;
+    long long tv_sec = ticks / freq;
+    long long tv_nsec = (ticks % freq) * 1_s_ns / freq;
+    if (LIBC_UNLIKELY(tv_sec > SEC_LIMIT)) {
+      ret = cpp::unexpected(EOVERFLOW);
+      break;
+    }
+    ts->tv_sec = static_cast<decltype(ts->tv_sec)>(tv_sec);
+    ts->tv_nsec = static_cast<decltype(ts->tv_nsec)>(tv_nsec);
+    break;
+  }
+  case CLOCK_REALTIME: {
+    // https://learn.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimepreciseasfiletime
+    // GetSystemTimePreciseAsFileTime
+    // This function is best suited for high-resolution time-of-day
+    // measurements, or time stamps that are synchronized to UTC
+    [[clang::uninitialized]] FILETIME file_time;
+    [[clang::uninitialized]] ULARGE_INTEGER time;
+    ::GetSystemTimePreciseAsFileTime(&file_time);
+    time.LowPart = file_time.dwLowDateTime;
+    time.HighPart = file_time.dwHighDateTime;
+
+    // adjust to POSIX epoch (from Jan 1, 1601 to Jan 1, 1970)
+    constexpr unsigned long long POSIX_TIME_SHIFT =
+        (11644473600ULL * HNS_PER_SEC);
+    if (LIBC_UNLIKELY(POSIX_TIME_SHIFT > time.QuadPart)) {
+      ret = cpp::unexpected(EOVERFLOW);
+      break;
+    }
+    time.QuadPart -= (11644473600ULL * HNS_PER_SEC);
+    unsigned long long tv_sec = time.QuadPart / HNS_PER_SEC;
+    unsigned long long tv_nsec = (time.QuadPart % HNS_PER_SEC) * 100ULL;
+    if (LIBC_UNLIKELY(tv_sec > SEC_LIMIT)) {
+      ret = cpp::unexpected(EOVERFLOW);
+      break;
+    }
+    ts->tv_sec = static_cast<decltype(ts->tv_sec)>(tv_sec);
+    ts->tv_nsec = static_cast<decltype(ts->tv_nsec)>(tv_nsec);
+    break;
+  }
+  case CLOCK_PROCESS_CPUTIME_ID:
+  case CLOCK_THREAD_CPUTIME_ID: {
+    [[clang::uninitialized]] FILETIME creation_time;
+    [[clang::uninitialized]] FILETIME exit_time;
+    [[clang::uninitialized]] FILETIME kernel_time;
+    [[clang::uninitialized]] FILETIME user_time;
+    bool success;
+    if (clockid == CLOCK_PROCESS_CPUTIME_ID) {
+      // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes
+      success = ::GetProcessTimes(::GetCurrentProcess(), &creation_time,
+                                  &exit_time, &kernel_time, &user_time);
+    } else {
+      // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getthreadtimes
+      success = ::GetThreadTimes(::GetCurrentThread(), &creation_time,
+                                 &exit_time, &kernel_time, &user_time);
+    }
+    if (!success) {
+      ret = cpp::unexpected(EINVAL);
+      break;
+    }
+    // https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-filetime
+    // It is not recommended that you add and subtract values from the FILETIME
+    // structure to obtain relative times. Instead, you should copy the low- and
+    // high-order parts of the file time to a ULARGE_INTEGER structure, perform
+    // 64-bit arithmetic on the QuadPart member, and copy the LowPart and
+    // HighPart members into the FILETIME structure.
+    auto kernel_time_hns = cpp::bit_cast<ULARGE_INTEGER>(kernel_time);
+    auto user_time_hns = cpp::bit_cast<ULARGE_INTEGER>(user_time);
+    unsigned long long total_time_hns =
+        kernel_time_hns.QuadPart + user_time_hns.QuadPart;
+
+    unsigned long long tv_sec = total_time_hns / HNS_PER_SEC;
+    unsigned long long tv_nsec = (total_time_hns % HNS_PER_SEC) * 100ULL;
+
+    if (LIBC_UNLIKELY(tv_sec > SEC_LIMIT)) {
+      ret = cpp::unexpected(EOVERFLOW);
+      break;
+    }
+
+    ts->tv_sec = static_cast<decltype(ts->tv_sec)>(tv_sec);
+    ts->tv_nsec = static_cast<decltype(ts->tv_nsec)>(tv_nsec);
+
+    break;
+  }
+  }
+  return ret;
+}
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/time/windows/performance_counter.h b/libc/src/__support/time/windows/performance_counter.h
new file mode 100644
index 0000000000000..3df81b3efff40
--- /dev/null
+++ b/libc/src/__support/time/windows/performance_counter.h
@@ -0,0 +1,35 @@
+//===--- Cached Performance Counter Frequency  ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/common.h"
+
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <Windows.h>
+
+namespace LIBC_NAMESPACE_DECL {
+namespace performance_counter {
+LIBC_INLINE long long get_ticks_per_second() {
+  static cpp::Atomic<long long> frequency = 0;
+  // Relaxed ordering is enough. It is okay to record the frequency multiple
+  // times. The store operation itself is atomic and the value must propagate
+  // as required by cache coherence.
+  auto freq = frequency.load(cpp::MemoryOrder::RELAXED);
+  if (!freq) {
+    [[clang::uninitialized]] LARGE_INTEGER buffer;
+    // On systems that run Windows XP or later, the function will always
+    // succeed and will thus never return zero.
+    ::QueryPerformanceFrequency(&buffer);
+    frequency.store(buffer.QuadPart, cpp::MemoryOrder::RELAXED);
+    return buffer.QuadPart;
+  }
+  return freq;
+}
+} // namespace performance_counter
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/arpa/CMakeLists.txt b/libc/src/arpa/CMakeLists.txt
new file mode 100644
index 0000000000000..5c89828860ff8
--- /dev/null
+++ b/libc/src/arpa/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(inet)
diff --git a/libc/src/network/CMakeLists.txt b/libc/src/arpa/inet/CMakeLists.txt
similarity index 100%
rename from libc/src/network/CMakeLists.txt
rename to libc/src/arpa/inet/CMakeLists.txt
diff --git a/libc/src/network/htonl.cpp b/libc/src/arpa/inet/htonl.cpp
similarity index 95%
rename from libc/src/network/htonl.cpp
rename to libc/src/arpa/inet/htonl.cpp
index 681786adea68a..fb0404a26a244 100644
--- a/libc/src/network/htonl.cpp
+++ b/libc/src/arpa/inet/htonl.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/network/htonl.h"
+#include "src/arpa/inet/htonl.h"
 #include "src/__support/common.h"
 #include "src/__support/endian_internal.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/network/htonl.h b/libc/src/arpa/inet/htonl.h
similarity index 81%
rename from libc/src/network/htonl.h
rename to libc/src/arpa/inet/htonl.h
index 14217310a2271..e444972c771ea 100644
--- a/libc/src/network/htonl.h
+++ b/libc/src/arpa/inet/htonl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_NETWORK_HTONL_H
-#define LLVM_LIBC_SRC_NETWORK_HTONL_H
+#ifndef LLVM_LIBC_SRC_ARPA_INET_HTONL_H
+#define LLVM_LIBC_SRC_ARPA_INET_HTONL_H
 
 #include "src/__support/macros/config.h"
 #include <stdint.h>
@@ -18,4 +18,4 @@ uint32_t htonl(uint32_t hostlong);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_NETWORK_HTONL_H
+#endif // LLVM_LIBC_SRC_ARPA_INET_HTONL_H
diff --git a/libc/src/network/htons.cpp b/libc/src/arpa/inet/htons.cpp
similarity index 95%
rename from libc/src/network/htons.cpp
rename to libc/src/arpa/inet/htons.cpp
index 675f53cf4a0ad..1fcbbdf67deb0 100644
--- a/libc/src/network/htons.cpp
+++ b/libc/src/arpa/inet/htons.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/network/htons.h"
+#include "src/arpa/inet/htons.h"
 #include "src/__support/common.h"
 #include "src/__support/endian_internal.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/network/htons.h b/libc/src/arpa/inet/htons.h
similarity index 81%
rename from libc/src/network/htons.h
rename to libc/src/arpa/inet/htons.h
index 8f5c68a0a60a8..35c2acdcfae45 100644
--- a/libc/src/network/htons.h
+++ b/libc/src/arpa/inet/htons.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_NETWORK_HTONS_H
-#define LLVM_LIBC_SRC_NETWORK_HTONS_H
+#ifndef LLVM_LIBC_SRC_ARPA_INET_HTONS_H
+#define LLVM_LIBC_SRC_ARPA_INET_HTONS_H
 
 #include "src/__support/macros/config.h"
 #include <stdint.h>
@@ -18,4 +18,4 @@ uint16_t htons(uint16_t hostshort);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_NETWORK_HTONS_H
+#endif // LLVM_LIBC_SRC_ARPA_INET_HTONS_H
diff --git a/libc/src/network/ntohl.cpp b/libc/src/arpa/inet/ntohl.cpp
similarity index 95%
rename from libc/src/network/ntohl.cpp
rename to libc/src/arpa/inet/ntohl.cpp
index 6a309e97fca79..d472107a1988c 100644
--- a/libc/src/network/ntohl.cpp
+++ b/libc/src/arpa/inet/ntohl.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/network/ntohl.h"
+#include "src/arpa/inet/ntohl.h"
 #include "src/__support/common.h"
 #include "src/__support/endian_internal.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/network/ntohl.h b/libc/src/arpa/inet/ntohl.h
similarity index 81%
rename from libc/src/network/ntohl.h
rename to libc/src/arpa/inet/ntohl.h
index c325951948062..40079650d6fd1 100644
--- a/libc/src/network/ntohl.h
+++ b/libc/src/arpa/inet/ntohl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_NETWORK_NTOHL_H
-#define LLVM_LIBC_SRC_NETWORK_NTOHL_H
+#ifndef LLVM_LIBC_SRC_ARPA_INET_NTOHL_H
+#define LLVM_LIBC_SRC_ARPA_INET_NTOHL_H
 
 #include "src/__support/macros/config.h"
 #include <stdint.h>
@@ -18,4 +18,4 @@ uint32_t ntohl(uint32_t netlong);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_NETWORK_NTOHL_H
+#endif // LLVM_LIBC_SRC_ARPA_INET_NTOHL_H
diff --git a/libc/src/network/ntohs.cpp b/libc/src/arpa/inet/ntohs.cpp
similarity index 95%
rename from libc/src/network/ntohs.cpp
rename to libc/src/arpa/inet/ntohs.cpp
index b51ecb93241ca..9082ed928778e 100644
--- a/libc/src/network/ntohs.cpp
+++ b/libc/src/arpa/inet/ntohs.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/network/ntohs.h"
+#include "src/arpa/inet/ntohs.h"
 #include "src/__support/common.h"
 #include "src/__support/endian_internal.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/network/ntohs.h b/libc/src/arpa/inet/ntohs.h
similarity index 81%
rename from libc/src/network/ntohs.h
rename to libc/src/arpa/inet/ntohs.h
index f55591415f764..5fe3ebcb62520 100644
--- a/libc/src/network/ntohs.h
+++ b/libc/src/arpa/inet/ntohs.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_NETWORK_NTOHS_H
-#define LLVM_LIBC_SRC_NETWORK_NTOHS_H
+#ifndef LLVM_LIBC_SRC_ARPA_INET_NTOHS_H
+#define LLVM_LIBC_SRC_ARPA_INET_NTOHS_H
 
 #include "src/__support/macros/config.h"
 #include <stdint.h>
@@ -18,4 +18,4 @@ uint16_t ntohs(uint16_t netshort);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_NETWORK_NTOHS_H
+#endif // LLVM_LIBC_SRC_ARPA_INET_NTOHS_H
diff --git a/libc/src/complex/CMakeLists.txt b/libc/src/complex/CMakeLists.txt
index 289cce5455af7..7306e2fe925e3 100644
--- a/libc/src/complex/CMakeLists.txt
+++ b/libc/src/complex/CMakeLists.txt
@@ -24,3 +24,9 @@ add_complex_entrypoint_object(cimagf)
 add_complex_entrypoint_object(cimagl)
 add_complex_entrypoint_object(cimagf16)
 add_complex_entrypoint_object(cimagf128)
+
+add_complex_entrypoint_object(conj)
+add_complex_entrypoint_object(conjf)
+add_complex_entrypoint_object(conjl)
+add_complex_entrypoint_object(conjf16)
+add_complex_entrypoint_object(conjf128)
diff --git a/libc/src/complex/conj.h b/libc/src/complex/conj.h
new file mode 100644
index 0000000000000..2ff82d2758620
--- /dev/null
+++ b/libc/src/complex/conj.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for conj --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_COMPLEX_CONJ_H
+#define LLVM_LIBC_SRC_COMPLEX_CONJ_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+_Complex double conj(_Complex double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_COMPLEX_CONJ_H
diff --git a/libc/src/complex/conjf.h b/libc/src/complex/conjf.h
new file mode 100644
index 0000000000000..6b3bd612e5bf7
--- /dev/null
+++ b/libc/src/complex/conjf.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for conjf -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_COMPLEX_CONJF_H
+#define LLVM_LIBC_SRC_COMPLEX_CONJF_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+_Complex float conjf(_Complex float x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_COMPLEX_CONJF_H
diff --git a/libc/src/complex/conjf128.h b/libc/src/complex/conjf128.h
new file mode 100644
index 0000000000000..587c979d315ef
--- /dev/null
+++ b/libc/src/complex/conjf128.h
@@ -0,0 +1,27 @@
+//===-- Implementation header for conjf128 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/macros/properties/complex_types.h"
+#include "src/__support/macros/properties/types.h"
+
+#if defined(LIBC_TYPES_HAS_CFLOAT128)
+
+#ifndef LLVM_LIBC_SRC_COMPLEX_CONJF128_H
+#define LLVM_LIBC_SRC_COMPLEX_CONJF128_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+cfloat128 conjf128(cfloat128 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_COMPLEX_CONJF128_H
+
+#endif // LIBC_TYPES_HAS_CFLOAT128
diff --git a/libc/src/complex/conjf16.h b/libc/src/complex/conjf16.h
new file mode 100644
index 0000000000000..b15c5b3f61f4a
--- /dev/null
+++ b/libc/src/complex/conjf16.h
@@ -0,0 +1,27 @@
+//===-- Implementation header for conjf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/macros/properties/complex_types.h"
+#include "src/__support/macros/properties/types.h"
+
+#if defined(LIBC_TYPES_HAS_CFLOAT16)
+
+#ifndef LLVM_LIBC_SRC_COMPLEX_CONJF16_H
+#define LLVM_LIBC_SRC_COMPLEX_CONJF16_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+cfloat16 conjf16(cfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_COMPLEX_CONJF16_H
+
+#endif // LIBC_TYPES_HAS_CFLOAT16
diff --git a/libc/src/complex/conjl.h b/libc/src/complex/conjl.h
new file mode 100644
index 0000000000000..aec640f9433ac
--- /dev/null
+++ b/libc/src/complex/conjl.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for conjl -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_COMPLEX_CONJL_H
+#define LLVM_LIBC_SRC_COMPLEX_CONJL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+_Complex long double conjl(_Complex long double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_COMPLEX_CONJL_H
diff --git a/libc/src/complex/generic/CMakeLists.txt b/libc/src/complex/generic/CMakeLists.txt
index a3da781c60237..cc14f89122edd 100644
--- a/libc/src/complex/generic/CMakeLists.txt
+++ b/libc/src/complex/generic/CMakeLists.txt
@@ -1,3 +1,72 @@
+add_entrypoint_object(
+  conj
+  SRCS
+    conj.cpp
+  HDRS
+    ../conj.h
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    libc.src.__support.CPP.bit
+    libc.src.__support.complex_type
+)
+
+add_entrypoint_object(
+  conjf
+  SRCS
+    conjf.cpp
+  HDRS
+    ../conjf.h
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    libc.src.__support.CPP.bit
+    libc.src.__support.complex_type
+)
+
+add_entrypoint_object(
+  conjl
+  SRCS
+    conjl.cpp
+  HDRS
+    ../conjl.h
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    libc.src.__support.CPP.bit
+    libc.src.__support.complex_type
+)
+
+add_entrypoint_object(
+  conjf16
+  SRCS
+    conjf16.cpp
+  HDRS
+    ../conjf16.h
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    libc.src.__support.CPP.bit
+    libc.src.__support.complex_type
+    libc.src.__support.macros.properties.types
+    libc.src.__support.macros.properties.complex_types
+)
+
+add_entrypoint_object(
+  conjf128
+  SRCS
+    conjf128.cpp
+  HDRS
+    ../conjf128.h
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    libc.src.__support.CPP.bit
+    libc.src.__support.complex_type
+    libc.src.__support.macros.properties.types
+    libc.src.__support.macros.properties.complex_types
+)
+
 add_entrypoint_object(
   creal
   SRCS
diff --git a/libc/src/complex/generic/conj.cpp b/libc/src/complex/generic/conj.cpp
new file mode 100644
index 0000000000000..1a93bc25dc3c4
--- /dev/null
+++ b/libc/src/complex/generic/conj.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of conj function -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/complex/conj.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/common.h"
+#include "src/__support/complex_type.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(_Complex double, conj, (_Complex double x)) {
+  return conjugate<_Complex double>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/complex/generic/conjf.cpp b/libc/src/complex/generic/conjf.cpp
new file mode 100644
index 0000000000000..33cb34340a04e
--- /dev/null
+++ b/libc/src/complex/generic/conjf.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of conjf function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/complex/conjf.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/common.h"
+#include "src/__support/complex_type.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(_Complex float, conjf, (_Complex float x)) {
+  return conjugate<_Complex float>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/complex/generic/conjf128.cpp b/libc/src/complex/generic/conjf128.cpp
new file mode 100644
index 0000000000000..4e35b3d5a97b7
--- /dev/null
+++ b/libc/src/complex/generic/conjf128.cpp
@@ -0,0 +1,24 @@
+//===-- Implementation of conjf128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/complex/conjf128.h"
+#if defined(LIBC_TYPES_HAS_CFLOAT128)
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/common.h"
+#include "src/__support/complex_type.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(cfloat128, conjf128, (cfloat128 x)) {
+  return conjugate<cfloat128>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_CFLOAT128
diff --git a/libc/src/complex/generic/conjf16.cpp b/libc/src/complex/generic/conjf16.cpp
new file mode 100644
index 0000000000000..2564fe252027a
--- /dev/null
+++ b/libc/src/complex/generic/conjf16.cpp
@@ -0,0 +1,24 @@
+//===-- Implementation of conjf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/complex/conjf16.h"
+#if defined(LIBC_TYPES_HAS_CFLOAT16)
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/common.h"
+#include "src/__support/complex_type.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(cfloat16, conjf16, (cfloat16 x)) {
+  return conjugate<cfloat16>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_CFLOAT16
diff --git a/libc/src/complex/generic/conjl.cpp b/libc/src/complex/generic/conjl.cpp
new file mode 100644
index 0000000000000..dc071ab1ec51b
--- /dev/null
+++ b/libc/src/complex/generic/conjl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of conjl function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/complex/conjl.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/common.h"
+#include "src/__support/complex_type.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(_Complex long double, conjl, (_Complex long double x)) {
+  return conjugate<_Complex long double>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/ctype/isxdigit.cpp b/libc/src/ctype/isxdigit.cpp
index 6b730c354db08..81f645c6f49fc 100644
--- a/libc/src/ctype/isxdigit.cpp
+++ b/libc/src/ctype/isxdigit.cpp
@@ -16,7 +16,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, isxdigit, (int c)) {
   const unsigned ch = static_cast<unsigned>(c);
-  return static_cast<int>(internal::isdigit(ch) || (ch | 32) - 'a' < 6);
+  return static_cast<int>(internal::isalnum(ch) &&
+                          internal::b36_char_to_int(ch) < 16);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/ctype/isxdigit_l.cpp b/libc/src/ctype/isxdigit_l.cpp
index 8a5c7d4d28ab1..eddfd20a2da3b 100644
--- a/libc/src/ctype/isxdigit_l.cpp
+++ b/libc/src/ctype/isxdigit_l.cpp
@@ -16,7 +16,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, isxdigit_l, (int c, locale_t)) {
   const unsigned ch = static_cast<unsigned>(c);
-  return static_cast<int>(internal::isdigit(ch) || (ch | 32) - 'a' < 6);
+  return static_cast<int>(internal::isalnum(ch) &&
+                          internal::b36_char_to_int(ch) < 16);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/ctype/toupper.cpp b/libc/src/ctype/toupper.cpp
index b5a23fc7f588b..1e1e8fc400711 100644
--- a/libc/src/ctype/toupper.cpp
+++ b/libc/src/ctype/toupper.cpp
@@ -14,10 +14,6 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(int, toupper, (int c)) {
-  if (internal::islower(c))
-    return c - ('a' - 'A');
-  return c;
-}
+LLVM_LIBC_FUNCTION(int, toupper, (int c)) { return internal::toupper(c); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/ctype/toupper_l.cpp b/libc/src/ctype/toupper_l.cpp
index f536ff3623616..a435ca1ab5d41 100644
--- a/libc/src/ctype/toupper_l.cpp
+++ b/libc/src/ctype/toupper_l.cpp
@@ -15,9 +15,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, toupper_l, (int c, locale_t)) {
-  if (internal::islower(c))
-    return c - ('a' - 'A');
-  return c;
+  return internal::toupper(c);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/gpu/rpc_host_call.cpp b/libc/src/gpu/rpc_host_call.cpp
index 1181e9554d16e..676031d16e154 100644
--- a/libc/src/gpu/rpc_host_call.cpp
+++ b/libc/src/gpu/rpc_host_call.cpp
@@ -19,7 +19,7 @@ namespace LIBC_NAMESPACE_DECL {
 // arguments. We expect that the pointer here is a valid pointer on the server.
 LLVM_LIBC_FUNCTION(unsigned long long, rpc_host_call,
                    (void *fn, void *data, size_t size)) {
-  rpc::Client::Port port = rpc::client.open<RPC_HOST_CALL>();
+  rpc::Client::Port port = rpc::client.open<LIBC_HOST_CALL>();
   port.send_n(data, size);
   port.send([=](rpc::Buffer *buffer, uint32_t) {
     buffer->data[0] = reinterpret_cast<uintptr_t>(fn);
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 76a5e491effa0..390a59d07a28b 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -484,6 +484,7 @@ add_math_entrypoint_object(sincosf)
 
 add_math_entrypoint_object(sin)
 add_math_entrypoint_object(sinf)
+add_math_entrypoint_object(sinf16)
 add_math_entrypoint_object(sinpif)
 add_math_entrypoint_object(sinpif16)
 
diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md
index e9a6aadc6c44f..f02d502399e2b 100644
--- a/libc/src/math/docs/add_math_function.md
+++ b/libc/src/math/docs/add_math_function.md
@@ -18,7 +18,7 @@ together with its specifications:
 ```
 - Add function specs to the file:
 ```
-  libc/newhdrgen/yaml/math.yaml
+  libc/hdrgen/yaml/math.yaml
 ```
 
 ## Implementation
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index a5d17ad023f52..aeb758d4a092d 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -498,6 +498,27 @@ add_entrypoint_object(
     ${libc_opt_high_flag}
 )
 
+add_entrypoint_object(
+  sinf16
+  SRCS
+    sinf16.cpp
+  HDRS
+    ../sinf16.h
+  DEPENDS
+    .sincosf16_utils
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   sincos
   SRCS
diff --git a/libc/src/math/generic/atan2f.cpp b/libc/src/math/generic/atan2f.cpp
index a2e5499809a34..db7639396cdd7 100644
--- a/libc/src/math/generic/atan2f.cpp
+++ b/libc/src/math/generic/atan2f.cpp
@@ -21,6 +21,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 namespace {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
 // Look up tables for accurate pass:
 
 // atan(i/16) with i = 0..16, generated by Sollya with:
@@ -163,6 +165,8 @@ float atan2f_double_double(double num_d, double den_d, double q_d, int idx,
   return static_cast<float>(cpp::bit_cast<double>(rr_bits));
 }
 
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
 } // anonymous namespace
 
 // There are several range reduction steps we can take for atan2(y, x) as
@@ -283,14 +287,24 @@ LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) {
   fputil::DoubleDouble const_term = CONST_ADJ[x_sign][y_sign][recip];
   double q_d = num_d / den_d;
 
-  double k_d = fputil::nearest_integer(q_d * 0x1.0p4f);
+  double k_d = fputil::nearest_integer(q_d * 0x1.0p4);
   int idx = static_cast<int>(k_d);
+  double r;
+
+#ifdef LIBC_MATH_HAS_SMALL_TABLES
+  double p = atan_eval_no_table(num_d, den_d, k_d * 0x1.0p-4);
+  r = final_sign * (p + (const_term.hi + ATAN_K_OVER_16[idx]));
+#else
   q_d = fputil::multiply_add(k_d, -0x1.0p-4, q_d);
 
   double p = atan_eval(q_d, idx);
-  double r = final_sign *
-             fputil::multiply_add(q_d, p, const_term.hi + ATAN_COEFFS[idx][0]);
+  r = final_sign *
+      fputil::multiply_add(q_d, p, const_term.hi + ATAN_COEFFS[idx][0]);
+#endif // LIBC_MATH_HAS_SMALL_TABLES
 
+#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  return static_cast<float>(r);
+#else
   constexpr uint32_t LOWER_ERR = 4;
   // Mask sticky bits in double precision before rounding to single precision.
   constexpr uint32_t MASK =
@@ -306,6 +320,7 @@ LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) {
 
   return atan2f_double_double(num_d, den_d, q_d, idx, k_d, final_sign,
                               const_term);
+#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/inv_trigf_utils.h b/libc/src/math/generic/inv_trigf_utils.h
index bdabec9c1b6bb..8b47aba342995 100644
--- a/libc/src/math/generic/inv_trigf_utils.h
+++ b/libc/src/math/generic/inv_trigf_utils.h
@@ -17,14 +17,35 @@
 namespace LIBC_NAMESPACE_DECL {
 
 // PI and PI / 2
-constexpr double M_MATH_PI = 0x1.921fb54442d18p+1;
-constexpr double M_MATH_PI_2 = 0x1.921fb54442d18p+0;
+static constexpr double M_MATH_PI = 0x1.921fb54442d18p+1;
+static constexpr double M_MATH_PI_2 = 0x1.921fb54442d18p+0;
 
 extern double ATAN_COEFFS[17][9];
 
+// Look-up table for atan(k/16) with k = 0..16.
+static constexpr double ATAN_K_OVER_16[17] = {
+    0.0,
+    0x1.ff55bb72cfdeap-5,
+    0x1.fd5ba9aac2f6ep-4,
+    0x1.7b97b4bce5b02p-3,
+    0x1.f5b75f92c80ddp-3,
+    0x1.362773707ebccp-2,
+    0x1.6f61941e4def1p-2,
+    0x1.a64eec3cc23fdp-2,
+    0x1.dac670561bb4fp-2,
+    0x1.0657e94db30dp-1,
+    0x1.1e00babdefeb4p-1,
+    0x1.345f01cce37bbp-1,
+    0x1.4978fa3269ee1p-1,
+    0x1.5d58987169b18p-1,
+    0x1.700a7c5784634p-1,
+    0x1.819d0b7158a4dp-1,
+    0x1.921fb54442d18p-1,
+};
+
 // For |x| <= 1/32 and 0 <= i <= 16, return Q(x) such that:
 //   Q(x) ~ (atan(x + i/16) - atan(i/16)) / x.
-LIBC_INLINE double atan_eval(double x, int i) {
+LIBC_INLINE static double atan_eval(double x, unsigned i) {
   double x2 = x * x;
 
   double c0 = fputil::multiply_add(x, ATAN_COEFFS[i][2], ATAN_COEFFS[i][1]);
@@ -39,16 +60,43 @@ LIBC_INLINE double atan_eval(double x, int i) {
   return p;
 }
 
+// Evaluate atan without big lookup table.
+//   atan(n/d) - atan(k/16) = atan((n/d - k/16) / (1 + (n/d) * (k/16)))
+//                          = atan((n - d * k/16)) / (d + n * k/16))
+// So we let q = (n - d * k/16) / (d + n * k/16),
+// and approximate with Taylor polynomial:
+//   atan(q) ~ q - q^3/3 + q^5/5 - q^7/7 + q^9/9
+LIBC_INLINE static double atan_eval_no_table(double num, double den,
+                                             double k_over_16) {
+  double num_r = fputil::multiply_add(den, -k_over_16, num);
+  double den_r = fputil::multiply_add(num, k_over_16, den);
+  double q = num_r / den_r;
+
+  constexpr double ATAN_TAYLOR[] = {
+      -0x1.5555555555555p-2,
+      0x1.999999999999ap-3,
+      -0x1.2492492492492p-3,
+      0x1.c71c71c71c71cp-4,
+  };
+  double q2 = q * q;
+  double q3 = q2 * q;
+  double q4 = q2 * q2;
+  double c0 = fputil::multiply_add(q2, ATAN_TAYLOR[1], ATAN_TAYLOR[0]);
+  double c1 = fputil::multiply_add(q2, ATAN_TAYLOR[3], ATAN_TAYLOR[2]);
+  double d = fputil::multiply_add(q4, c1, c0);
+  return fputil::multiply_add(q3, d, q);
+}
+
 // > Q = fpminimax(asin(x)/x, [|0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20|],
 //                 [|1, D...|], [0, 0.5]);
-constexpr double ASIN_COEFFS[10] = {0x1.5555555540fa1p-3, 0x1.333333512edc2p-4,
-                                    0x1.6db6cc1541b31p-5, 0x1.f1caff324770ep-6,
-                                    0x1.6e43899f5f4f4p-6, 0x1.1f847cf652577p-6,
-                                    0x1.9b60f47f87146p-7, 0x1.259e2634c494fp-6,
-                                    -0x1.df946fa875ddp-8, 0x1.02311ecf99c28p-5};
+static constexpr double ASIN_COEFFS[10] = {
+    0x1.5555555540fa1p-3, 0x1.333333512edc2p-4, 0x1.6db6cc1541b31p-5,
+    0x1.f1caff324770ep-6, 0x1.6e43899f5f4f4p-6, 0x1.1f847cf652577p-6,
+    0x1.9b60f47f87146p-7, 0x1.259e2634c494fp-6, -0x1.df946fa875ddp-8,
+    0x1.02311ecf99c28p-5};
 
 // Evaluate P(x^2) - 1, where P(x^2) ~ asin(x)/x
-LIBC_INLINE double asin_eval(double xsq) {
+LIBC_INLINE static double asin_eval(double xsq) {
   double x4 = xsq * xsq;
   double r1 = fputil::polyeval(x4, ASIN_COEFFS[0], ASIN_COEFFS[2],
                                ASIN_COEFFS[4], ASIN_COEFFS[6], ASIN_COEFFS[8]);
diff --git a/libc/src/math/generic/sincosf16_utils.h b/libc/src/math/generic/sincosf16_utils.h
index 83511755a56c4..5e5edd4a8c85b 100644
--- a/libc/src/math/generic/sincosf16_utils.h
+++ b/libc/src/math/generic/sincosf16_utils.h
@@ -11,6 +11,7 @@
 
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/nearest_integer.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
@@ -46,10 +47,31 @@ LIBC_INLINE int32_t range_reduction_sincospif16(float x, float &y) {
   return static_cast<int32_t>(kf);
 }
 
-LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k,
-                                  float &sin_y, float &cosm1_y) {
-  float y;
-  int32_t k = range_reduction_sincospif16(xf, y);
+// Recall, range reduction:
+//   k = round(x * 32/pi)
+//   y = x * 32/pi - k
+//
+// The constant 0x1.45f306dc9c883p3 is 32/pi rounded to double-precision.
+// 32/pi is generated by Sollya with the following commands:
+// > display = hexadecimal;
+// > round(32/pi, D, RN);
+//
+// The precision choice of 'double' is to minimize rounding errors
+// in this initial scaling step, preserving enough bits so errors accumulated
+// while computing the subtraction: y = x * 32/pi - round(x * 32/pi)
+// are beyond the least-significant bit of single-precision used during
+// further intermediate computation.
+LIBC_INLINE int32_t range_reduction_sincosf16(float x, float &y) {
+  double prod = x * 0x1.45f306dc9c883p3;
+  double kf = fputil::nearest_integer(prod);
+  y = static_cast<float>(prod - kf);
+
+  return static_cast<int32_t>(kf);
+}
+
+static LIBC_INLINE void sincosf16_poly_eval(int32_t k, float y, float &sin_k,
+                                            float &cos_k, float &sin_y,
+                                            float &cosm1_y) {
 
   sin_k = SIN_K_PI_OVER_32[k & 63];
   cos_k = SIN_K_PI_OVER_32[(k + 16) & 63];
@@ -72,6 +94,22 @@ LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k,
                                    0x1.a6f7a2p-29f);
 }
 
+LIBC_INLINE void sincosf16_eval(float xf, float &sin_k, float &cos_k,
+                                float &sin_y, float &cosm1_y) {
+  float y;
+  int32_t k = range_reduction_sincosf16(xf, y);
+
+  sincosf16_poly_eval(k, y, sin_k, cos_k, sin_y, cosm1_y);
+}
+
+LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k,
+                                  float &sin_y, float &cosm1_y) {
+  float y;
+  int32_t k = range_reduction_sincospif16(xf, y);
+
+  sincosf16_poly_eval(k, y, sin_k, cos_k, sin_y, cosm1_y);
+}
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H
diff --git a/libc/src/math/generic/sinf16.cpp b/libc/src/math/generic/sinf16.cpp
new file mode 100644
index 0000000000000..86546348ba739
--- /dev/null
+++ b/libc/src/math/generic/sinf16.cpp
@@ -0,0 +1,108 @@
+//===-- Half-precision sin(x) function ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/sinf16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "sincosf16_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+constexpr size_t N_EXCEPTS = 4;
+
+constexpr fputil::ExceptValues<float16, N_EXCEPTS> SINF16_EXCEPTS{{
+    // (input, RZ output, RU offset, RD offset, RN offset)
+    {0x2b45, 0x2b43, 1, 0, 1},
+    {0x585c, 0x3ba3, 1, 0, 1},
+    {0x5cb0, 0xbbff, 0, 1, 0},
+    {0x51f5, 0xb80f, 0, 1, 0},
+}};
+
+LLVM_LIBC_FUNCTION(float16, sinf16, (float16 x)) {
+  using FPBits = fputil::FPBits<float16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // Range reduction:
+  // For |x| > pi/32, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * pi/32
+  //   k is an integer, |y| < 0.5
+  //
+  // This is done by performing:
+  //   k = round(x * 32/pi)
+  //   y = x * 32/pi - k
+  //
+  // Once k and y are computed, we then deduce the answer by the sine of sum
+  // formula:
+  //   sin(x) = sin((k + y) * pi/32)
+  //   	      = sin(k * pi/32) * cos(y * pi/32) +
+  //   	        sin(y * pi/32) * cos(k * pi/32)
+
+  // Handle exceptional values
+  if (LIBC_UNLIKELY(x_abs == 0x585c || x_abs == 0x5cb0 || x_abs == 0x51f5 ||
+                    x_abs == 0x2b45)) {
+    bool x_sign = x_u >> 15;
+    if (auto r = SINF16_EXCEPTS.lookup_odd(x_abs, x_sign);
+        LIBC_UNLIKELY(r.has_value()))
+      return r.value();
+  }
+
+  int rounding = fputil::quick_get_round();
+
+  // Exhaustive tests show that for |x| <= 0x1.f4p-11, 1ULP rounding errors
+  // occur. To fix this, the following apply:
+  if (LIBC_UNLIKELY(x_abs <= 0x13d0)) {
+    // sin(+/-0) = +/-0
+    if (LIBC_UNLIKELY(x_abs == 0U))
+      return x;
+
+    // When x > 0, and rounding upward, sin(x) == x.
+    // When x < 0, and rounding downward, sin(x) == x.
+    if ((rounding == FE_UPWARD && xbits.is_pos()) ||
+        (rounding == FE_DOWNWARD && xbits.is_neg()))
+      return x;
+
+    // When x < 0, and rounding upward, sin(x) == (x - 1ULP)
+    if (rounding == FE_UPWARD && xbits.is_neg()) {
+      x_u--;
+      return FPBits(x_u).get_val();
+    }
+  }
+
+  if (xbits.is_inf_or_nan()) {
+    if (xbits.is_inf()) {
+      fputil::set_errno_if_required(EDOM);
+      fputil::raise_except_if_required(FE_INVALID);
+    }
+
+    return x + FPBits::quiet_nan().get_val();
+  }
+
+  float sin_k, cos_k, sin_y, cosm1_y;
+  sincosf16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
+
+  if (LIBC_UNLIKELY(sin_y == 0 && sin_k == 0))
+    return FPBits::zero(xbits.sign()).get_val();
+
+  // Since, cosm1_y = cos_y - 1, therfore:
+  //    sin(x) = cos_k * sin_y + sin_k + (cosm1_y * sin_k)
+  return fputil::cast<float16>(fputil::multiply_add(
+      sin_y, cos_k, fputil::multiply_add(cosm1_y, sin_k, sin_k)));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/tanpif16.cpp b/libc/src/math/generic/tanpif16.cpp
index ab3c9cb2122ba..67635536ee319 100644
--- a/libc/src/math/generic/tanpif16.cpp
+++ b/libc/src/math/generic/tanpif16.cpp
@@ -21,7 +21,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 constexpr size_t N_EXCEPTS = 21;
 
-constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANF16_EXCEPTS{{
+constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANPIF16_EXCEPTS{{
     // (input, RZ output, RU offset, RD offset, RN offset)
     {0x07f2, 0x0e3d, 1, 0, 0}, {0x086a, 0x0eee, 1, 0, 1},
     {0x08db, 0x0fa0, 1, 0, 0}, {0x094c, 0x1029, 1, 0, 0},
@@ -49,7 +49,7 @@ LLVM_LIBC_FUNCTION(float16, tanpif16, (float16 x)) {
       return x;
 
     bool x_sign = x_u >> 15;
-    if (auto r = TANF16_EXCEPTS.lookup_odd(x_abs, x_sign);
+    if (auto r = TANPIF16_EXCEPTS.lookup_odd(x_abs, x_sign);
         LIBC_UNLIKELY(r.has_value()))
       return r.value();
   }
diff --git a/libc/src/math/sinf16.h b/libc/src/math/sinf16.h
new file mode 100644
index 0000000000000..23f1aa99b6233
--- /dev/null
+++ b/libc/src/math/sinf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for sinf16 ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_SINF16_H
+#define LLVM_LIBC_SRC_MATH_SINF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 sinf16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_SINF16_H
diff --git a/libc/src/stdio/gpu/clearerr.cpp b/libc/src/stdio/gpu/clearerr.cpp
index 4c631b9f946f3..4d843e4be33ee 100644
--- a/libc/src/stdio/gpu/clearerr.cpp
+++ b/libc/src/stdio/gpu/clearerr.cpp
@@ -15,7 +15,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void, clearerr, (::FILE * stream)) {
-  rpc::Client::Port port = rpc::client.open<RPC_CLEARERR>();
+  rpc::Client::Port port = rpc::client.open<LIBC_CLEARERR>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/fclose.cpp b/libc/src/stdio/gpu/fclose.cpp
index 683e0548495d1..847d6b8461729 100644
--- a/libc/src/stdio/gpu/fclose.cpp
+++ b/libc/src/stdio/gpu/fclose.cpp
@@ -18,7 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(int, fclose, (::FILE * stream)) {
   uint64_t ret = 0;
   uintptr_t file = reinterpret_cast<uintptr_t>(stream);
-  rpc::Client::Port port = rpc::client.open<RPC_CLOSE_FILE>();
+  rpc::Client::Port port = rpc::client.open<LIBC_CLOSE_FILE>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) { buffer->data[0] = file; },
       [&](rpc::Buffer *buffer, uint32_t) { ret = buffer->data[0]; });
diff --git a/libc/src/stdio/gpu/feof.cpp b/libc/src/stdio/gpu/feof.cpp
index 02adb4ce73d68..a15e4874d0e2a 100644
--- a/libc/src/stdio/gpu/feof.cpp
+++ b/libc/src/stdio/gpu/feof.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, feof, (::FILE * stream)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_FEOF>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FEOF>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/ferror.cpp b/libc/src/stdio/gpu/ferror.cpp
index ca777131fd1b3..8273820370677 100644
--- a/libc/src/stdio/gpu/ferror.cpp
+++ b/libc/src/stdio/gpu/ferror.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, ferror, (::FILE * stream)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_FERROR>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FERROR>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/fflush.cpp b/libc/src/stdio/gpu/fflush.cpp
index 577325b70c4e7..5a5137be6a4af 100644
--- a/libc/src/stdio/gpu/fflush.cpp
+++ b/libc/src/stdio/gpu/fflush.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, fflush, (::FILE * stream)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_FFLUSH>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FFLUSH>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/fgets.cpp b/libc/src/stdio/gpu/fgets.cpp
index fbc1b0cf7d1a8..d90b7aa4aa669 100644
--- a/libc/src/stdio/gpu/fgets.cpp
+++ b/libc/src/stdio/gpu/fgets.cpp
@@ -26,7 +26,7 @@ LLVM_LIBC_FUNCTION(char *, fgets,
 
   uint64_t recv_size;
   void *buf = nullptr;
-  rpc::Client::Port port = rpc::client.open<RPC_READ_FGETS>();
+  rpc::Client::Port port = rpc::client.open<LIBC_READ_FGETS>();
   port.send([=](rpc::Buffer *buffer, uint32_t) {
     buffer->data[0] = count;
     buffer->data[1] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/file.h b/libc/src/stdio/gpu/file.h
index 6ca792b454580..437ee9982438e 100644
--- a/libc/src/stdio/gpu/file.h
+++ b/libc/src/stdio/gpu/file.h
@@ -54,7 +54,7 @@ LIBC_INLINE uint64_t write_impl(::FILE *file, const void *data, size_t size) {
   uint64_t ret = 0;
   rpc::Client::Port port = rpc::client.open<opcode>();
 
-  if constexpr (opcode == RPC_WRITE_TO_STREAM) {
+  if constexpr (opcode == LIBC_WRITE_TO_STREAM) {
     port.send([&](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = reinterpret_cast<uintptr_t>(file);
     });
@@ -70,17 +70,17 @@ LIBC_INLINE uint64_t write_impl(::FILE *file, const void *data, size_t size) {
 
 LIBC_INLINE uint64_t write(::FILE *f, const void *data, size_t size) {
   if (f == stdout)
-    return write_impl<RPC_WRITE_TO_STDOUT>(f, data, size);
+    return write_impl<LIBC_WRITE_TO_STDOUT>(f, data, size);
   else if (f == stderr)
-    return write_impl<RPC_WRITE_TO_STDERR>(f, data, size);
+    return write_impl<LIBC_WRITE_TO_STDERR>(f, data, size);
   else
-    return write_impl<RPC_WRITE_TO_STREAM>(f, data, size);
+    return write_impl<LIBC_WRITE_TO_STREAM>(f, data, size);
 }
 
 LIBC_INLINE uint64_t read_from_stream(::FILE *file, void *buf, size_t size) {
   uint64_t ret = 0;
   uint64_t recv_size;
-  rpc::Client::Port port = rpc::client.open<RPC_READ_FROM_STREAM>();
+  rpc::Client::Port port = rpc::client.open<LIBC_READ_FROM_STREAM>();
   port.send([=](rpc::Buffer *buffer, uint32_t) {
     buffer->data[0] = size;
     buffer->data[1] = from_stream(file);
diff --git a/libc/src/stdio/gpu/fopen.cpp b/libc/src/stdio/gpu/fopen.cpp
index e165d2acd2109..18dd719537789 100644
--- a/libc/src/stdio/gpu/fopen.cpp
+++ b/libc/src/stdio/gpu/fopen.cpp
@@ -18,7 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(::FILE *, fopen,
                    (const char *__restrict path, const char *__restrict mode)) {
   uintptr_t file;
-  rpc::Client::Port port = rpc::client.open<RPC_OPEN_FILE>();
+  rpc::Client::Port port = rpc::client.open<LIBC_OPEN_FILE>();
   port.send_n(path, internal::string_length(path) + 1);
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
diff --git a/libc/src/stdio/gpu/fseek.cpp b/libc/src/stdio/gpu/fseek.cpp
index 37c40bc602d87..5ada5de50db8b 100644
--- a/libc/src/stdio/gpu/fseek.cpp
+++ b/libc/src/stdio/gpu/fseek.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, fseek, (::FILE * stream, long offset, int whence)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_FSEEK>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FSEEK>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/ftell.cpp b/libc/src/stdio/gpu/ftell.cpp
index 226aeda2f8ded..04bb0ddfa06ae 100644
--- a/libc/src/stdio/gpu/ftell.cpp
+++ b/libc/src/stdio/gpu/ftell.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(long, ftell, (::FILE * stream)) {
   long ret;
-  rpc::Client::Port port = rpc::client.open<RPC_FSEEK>();
+  rpc::Client::Port port = rpc::client.open<LIBC_FSEEK>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = file::from_stream(stream);
diff --git a/libc/src/stdio/gpu/puts.cpp b/libc/src/stdio/gpu/puts.cpp
index 20f7a889a378a..fc252abe52d17 100644
--- a/libc/src/stdio/gpu/puts.cpp
+++ b/libc/src/stdio/gpu/puts.cpp
@@ -18,7 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, puts, (const char *__restrict str)) {
   cpp::string_view str_view(str);
-  auto written = file::write_impl<RPC_WRITE_TO_STDOUT_NEWLINE>(stdout, str,
+  auto written = file::write_impl<LIBC_WRITE_TO_STDOUT_NEWLINE>(stdout, str,
                                                                str_view.size());
   if (written != str_view.size() + 1)
     return EOF;
diff --git a/libc/src/stdio/gpu/remove.cpp b/libc/src/stdio/gpu/remove.cpp
index 6604be1c31f2b..4bfb5d3fe8ed7 100644
--- a/libc/src/stdio/gpu/remove.cpp
+++ b/libc/src/stdio/gpu/remove.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, remove, (const char *path)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_REMOVE>();
+  rpc::Client::Port port = rpc::client.open<LIBC_REMOVE>();
   port.send_n(path, internal::string_length(path) + 1);
   port.recv([&](rpc::Buffer *buffer, uint32_t) {
     ret = static_cast<int>(buffer->data[0]);
diff --git a/libc/src/stdio/gpu/rename.cpp b/libc/src/stdio/gpu/rename.cpp
index e6396e212b8b5..589bf2456c8d0 100644
--- a/libc/src/stdio/gpu/rename.cpp
+++ b/libc/src/stdio/gpu/rename.cpp
@@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, rename, (const char *oldpath, const char *newpath)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_RENAME>();
+  rpc::Client::Port port = rpc::client.open<LIBC_RENAME>();
   port.send_n(oldpath, internal::string_length(oldpath) + 1);
   port.send_n(newpath, internal::string_length(newpath) + 1);
   port.recv([&](rpc::Buffer *buffer, uint32_t) {
diff --git a/libc/src/stdio/gpu/ungetc.cpp b/libc/src/stdio/gpu/ungetc.cpp
index dce14391b7de4..fd1db467975ee 100644
--- a/libc/src/stdio/gpu/ungetc.cpp
+++ b/libc/src/stdio/gpu/ungetc.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, ungetc, (int c, ::FILE *stream)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_UNGETC>();
+  rpc::Client::Port port = rpc::client.open<LIBC_UNGETC>();
   port.send_and_recv(
       [=](rpc::Buffer *buffer, uint32_t) {
         buffer->data[0] = c;
diff --git a/libc/src/stdio/gpu/vfprintf_utils.h b/libc/src/stdio/gpu/vfprintf_utils.h
index a0a8c39781ad6..df157d36c5166 100644
--- a/libc/src/stdio/gpu/vfprintf_utils.h
+++ b/libc/src/stdio/gpu/vfprintf_utils.h
@@ -23,8 +23,8 @@ LIBC_INLINE int vfprintf_impl(::FILE *__restrict file,
   uint64_t mask = gpu::get_lane_mask();
   rpc::Client::Port port = rpc::client.open<opcode>();
 
-  if constexpr (opcode == RPC_PRINTF_TO_STREAM ||
-                opcode == RPC_PRINTF_TO_STREAM_PACKED) {
+  if constexpr (opcode == LIBC_PRINTF_TO_STREAM ||
+                opcode == LIBC_PRINTF_TO_STREAM_PACKED) {
     port.send([&](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = reinterpret_cast<uintptr_t>(file);
     });
@@ -63,24 +63,24 @@ LIBC_INLINE int vfprintf_internal(::FILE *__restrict stream,
   // separate opcode so the server knows how much to advance the pointers.
 #if defined(LIBC_TARGET_ARCH_IS_AMDGPU)
   if (stream == stdout)
-    return vfprintf_impl<RPC_PRINTF_TO_STDOUT_PACKED>(stream, format,
-                                                      format_size, vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STDOUT_PACKED>(stream, format,
+                                                       format_size, vlist);
   else if (stream == stderr)
-    return vfprintf_impl<RPC_PRINTF_TO_STDERR_PACKED>(stream, format,
-                                                      format_size, vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STDERR_PACKED>(stream, format,
+                                                       format_size, vlist);
   else
-    return vfprintf_impl<RPC_PRINTF_TO_STREAM_PACKED>(stream, format,
-                                                      format_size, vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STREAM_PACKED>(stream, format,
+                                                       format_size, vlist);
 #else
   if (stream == stdout)
-    return vfprintf_impl<RPC_PRINTF_TO_STDOUT>(stream, format, format_size,
-                                               vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STDOUT>(stream, format, format_size,
+                                                vlist);
   else if (stream == stderr)
-    return vfprintf_impl<RPC_PRINTF_TO_STDERR>(stream, format, format_size,
-                                               vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STDERR>(stream, format, format_size,
+                                                vlist);
   else
-    return vfprintf_impl<RPC_PRINTF_TO_STREAM>(stream, format, format_size,
-                                               vlist);
+    return vfprintf_impl<LIBC_PRINTF_TO_STREAM>(stream, format, format_size,
+                                                vlist);
 #endif
 }
 
diff --git a/libc/src/stdio/printf_core/fixed_converter.h b/libc/src/stdio/printf_core/fixed_converter.h
index c8812d77b62e3..ba0a62d9fcb87 100644
--- a/libc/src/stdio/printf_core/fixed_converter.h
+++ b/libc/src/stdio/printf_core/fixed_converter.h
@@ -11,6 +11,7 @@
 
 #include "include/llvm-libc-macros/stdfix-macros.h"
 #include "src/__support/CPP/string_view.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/fixed_point/fx_bits.h"
 #include "src/__support/fixed_point/fx_rep.h"
 #include "src/__support/integer_to_string.h"
@@ -68,10 +69,6 @@ LIBC_INLINE int convert_fixed(Writer *writer, const FormatSection &to_conv) {
   using LARep = fixed_point::FXRep<unsigned long accum>;
   using StorageType = LARep::StorageType;
 
-  // All of the letters will be defined relative to variable a, which will be
-  // the appropriate case based on the name of the conversion. This converts any
-  // conversion name into the letter 'a' with the appropriate case.
-  const char a = (to_conv.conv_name & 32) | 'A';
   FormatFlags flags = to_conv.flags;
 
   bool is_negative;
@@ -179,9 +176,9 @@ LIBC_INLINE int convert_fixed(Writer *writer, const FormatSection &to_conv) {
     // unspecified.
     RoundDirection round;
     char first_digit_after = fraction_digits[precision];
-    if (first_digit_after > '5') {
+    if (internal::b36_char_to_int(first_digit_after) > 5) {
       round = RoundDirection::Up;
-    } else if (first_digit_after < '5') {
+    } else if (internal::b36_char_to_int(first_digit_after) < 5) {
       round = RoundDirection::Down;
     } else {
       // first_digit_after == '5'
@@ -204,7 +201,8 @@ LIBC_INLINE int convert_fixed(Writer *writer, const FormatSection &to_conv) {
         keep_rounding = false;
         char cur_digit = fraction_digits[digit_to_round];
         // if the digit should not be rounded up
-        if (round == RoundDirection::Even && ((cur_digit - '0') % 2) == 0) {
+        if (round == RoundDirection::Even &&
+            (internal::b36_char_to_int(cur_digit) % 2) == 0) {
           // break out of the loop
           break;
         }
@@ -246,7 +244,7 @@ LIBC_INLINE int convert_fixed(Writer *writer, const FormatSection &to_conv) {
   char sign_char = 0;
 
   // Check if the conv name is uppercase
-  if (a == 'A') {
+  if (internal::isupper(to_conv.conv_name)) {
     // These flags are only for signed conversions, so this removes them if the
     // conversion is unsigned.
     flags = FormatFlags(flags &
diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h
index e39ba6ecea8d4..d93457fcafd7f 100644
--- a/libc/src/stdio/printf_core/float_dec_converter.h
+++ b/libc/src/stdio/printf_core/float_dec_converter.h
@@ -13,6 +13,7 @@
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/big_int.h" // is_big_int_v
+#include "src/__support/ctype_utils.h"
 #include "src/__support/float_to_string.h"
 #include "src/__support/integer_to_string.h"
 #include "src/__support/libc_assert.h"
@@ -587,8 +588,6 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer,
   int exponent = float_bits.get_explicit_exponent();
   StorageType mantissa = float_bits.get_explicit_mantissa();
 
-  const char a = (to_conv.conv_name & 32) | 'A';
-
   char sign_char = 0;
 
   if (float_bits.is_neg())
@@ -734,7 +733,8 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer,
   round = get_round_direction(last_digit, truncated, float_bits.sign());
 
   RET_IF_RESULT_NEGATIVE(float_writer.write_last_block(
-      digits, maximum, round, final_exponent, a + 'E' - 'A'));
+      digits, maximum, round, final_exponent,
+      internal::islower(to_conv.conv_name) ? 'e' : 'E'));
 
   RET_IF_RESULT_NEGATIVE(float_writer.right_pad());
   return WRITE_OK;
diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/stdio/printf_core/float_hex_converter.h
index 0b3ff3dd1cbfd..b264b5cf20728 100644
--- a/libc/src/stdio/printf_core/float_hex_converter.h
+++ b/libc/src/stdio/printf_core/float_hex_converter.h
@@ -12,6 +12,7 @@
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/stdio/printf_core/converter_utils.h"
 #include "src/stdio/printf_core/core_structs.h"
@@ -28,10 +29,6 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
                                       const FormatSection &to_conv) {
   using LDBits = fputil::FPBits<long double>;
   using StorageType = LDBits::StorageType;
-  // All of the letters will be defined relative to variable a, which will be
-  // the appropriate case based on the name of the conversion. This converts any
-  // conversion name into the letter 'a' with the appropriate case.
-  const char a = (to_conv.conv_name & 32) | 'A';
 
   bool is_negative;
   int exponent;
@@ -138,9 +135,10 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   size_t mant_cur = mant_len;
   size_t first_non_zero = 1;
   for (; mant_cur > 0; --mant_cur, mantissa >>= 4) {
-    char mant_mod_16 = static_cast<char>(mantissa) & 15;
-    char new_digit = static_cast<char>(
-        (mant_mod_16 > 9) ? (mant_mod_16 - 10 + a) : (mant_mod_16 + '0'));
+    char mant_mod_16 = static_cast<char>(mantissa % 16);
+    char new_digit = static_cast<char>(internal::int_to_b36_char(mant_mod_16));
+    if (internal::isupper(to_conv.conv_name))
+      new_digit = static_cast<char>(internal::toupper(new_digit));
     mant_buffer[mant_cur - 1] = new_digit;
     if (new_digit != '0' && first_non_zero < mant_cur)
       first_non_zero = mant_cur;
@@ -168,7 +166,8 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
 
   size_t exp_cur = EXP_LEN;
   for (; exponent > 0; --exp_cur, exponent /= 10) {
-    exp_buffer[exp_cur - 1] = static_cast<char>((exponent % 10) + '0');
+    exp_buffer[exp_cur - 1] =
+        static_cast<char>(internal::int_to_b36_char(exponent % 10));
   }
   if (exp_cur == EXP_LEN) { // if nothing else was written, write a 0.
     exp_buffer[EXP_LEN - 1] = '0';
@@ -187,7 +186,7 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   constexpr size_t PREFIX_LEN = 2;
   char prefix[PREFIX_LEN];
   prefix[0] = '0';
-  prefix[1] = a + ('x' - 'a');
+  prefix[1] = internal::islower(to_conv.conv_name) ? 'x' : 'X';
   const cpp::string_view prefix_str(prefix, PREFIX_LEN);
 
   // If the precision is greater than the actual result, pad with 0s
@@ -200,7 +199,7 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   constexpr cpp::string_view HEXADECIMAL_POINT(".");
 
   // This is for the letter 'p' before the exponent.
-  const char exp_separator = a + ('p' - 'a');
+  const char exp_separator = internal::islower(to_conv.conv_name) ? 'p' : 'P';
   constexpr int EXP_SEPARATOR_LEN = 1;
 
   padding = static_cast<int>(to_conv.min_width - (sign_char > 0 ? 1 : 0) -
diff --git a/libc/src/stdio/printf_core/float_inf_nan_converter.h b/libc/src/stdio/printf_core/float_inf_nan_converter.h
index a7da682b835be..3e41612e21c9f 100644
--- a/libc/src/stdio/printf_core/float_inf_nan_converter.h
+++ b/libc/src/stdio/printf_core/float_inf_nan_converter.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H
 
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/stdio/printf_core/converter_utils.h"
 #include "src/stdio/printf_core/core_structs.h"
@@ -26,8 +27,6 @@ using StorageType = fputil::FPBits<long double>::StorageType;
 LIBC_INLINE int convert_inf_nan(Writer *writer, const FormatSection &to_conv) {
   // All of the letters will be defined relative to variable a, which will be
   // the appropriate case based on the case of the conversion.
-  const char a = (to_conv.conv_name & 32) | 'A';
-
   bool is_negative;
   StorageType mantissa;
   if (to_conv.length_modifier == LengthModifier::L) {
@@ -66,9 +65,11 @@ LIBC_INLINE int convert_inf_nan(Writer *writer, const FormatSection &to_conv) {
   if (sign_char)
     RET_IF_RESULT_NEGATIVE(writer->write(sign_char));
   if (mantissa == 0) { // inf
-    RET_IF_RESULT_NEGATIVE(writer->write(a == 'a' ? "inf" : "INF"));
+    RET_IF_RESULT_NEGATIVE(
+        writer->write(internal::islower(to_conv.conv_name) ? "inf" : "INF"));
   } else { // nan
-    RET_IF_RESULT_NEGATIVE(writer->write(a == 'a' ? "nan" : "NAN"));
+    RET_IF_RESULT_NEGATIVE(
+        writer->write(internal::islower(to_conv.conv_name) ? "nan" : "NAN"));
   }
 
   if (padding > 0 && ((to_conv.flags & FormatFlags::LEFT_JUSTIFIED) ==
diff --git a/libc/src/stdio/printf_core/int_converter.h b/libc/src/stdio/printf_core/int_converter.h
index f345e86b97a69..d0af229f89be5 100644
--- a/libc/src/stdio/printf_core/int_converter.h
+++ b/libc/src/stdio/printf_core/int_converter.h
@@ -11,6 +11,7 @@
 
 #include "src/__support/CPP/span.h"
 #include "src/__support/CPP/string_view.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/integer_to_string.h"
 #include "src/__support/macros/config.h"
 #include "src/stdio/printf_core/converter_utils.h"
@@ -23,11 +24,6 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace printf_core {
 
-// These functions only work on characters that are already known to be in the
-// alphabet. Their behavior is undefined otherwise.
-LIBC_INLINE constexpr char to_lower(char a) { return a | 32; }
-LIBC_INLINE constexpr bool is_lower(char a) { return (a & 32) > 0; }
-
 namespace details {
 
 using HexFmt = IntegerToString<uintmax_t, radix::Hex>;
@@ -49,14 +45,14 @@ LIBC_INLINE constexpr size_t num_buf_size() {
 
 LIBC_INLINE cpp::optional<cpp::string_view>
 num_to_strview(uintmax_t num, cpp::span<char> bufref, char conv_name) {
-  if (to_lower(conv_name) == 'x') {
-    if (is_lower(conv_name))
+  if (internal::tolower(conv_name) == 'x') {
+    if (internal::islower(conv_name))
       return HexFmt::format_to(bufref, num);
     else
       return HexFmtUppercase::format_to(bufref, num);
   } else if (conv_name == 'o') {
     return OctFmt::format_to(bufref, num);
-  } else if (to_lower(conv_name) == 'b') {
+  } else if (internal::tolower(conv_name) == 'b') {
     return BinFmt::format_to(bufref, num);
   } else {
     return DecFmt::format_to(bufref, num);
@@ -72,7 +68,6 @@ LIBC_INLINE int convert_int(Writer *writer, const FormatSection &to_conv) {
   uintmax_t num = static_cast<uintmax_t>(to_conv.conv_val_raw);
   bool is_negative = false;
   FormatFlags flags = to_conv.flags;
-  const char a = is_lower(to_conv.conv_name) ? 'a' : 'A';
 
   // If the conversion is signed, then handle negative values.
   if (to_conv.conv_name == 'd' || to_conv.conv_name == 'i') {
@@ -116,16 +111,16 @@ LIBC_INLINE int convert_int(Writer *writer, const FormatSection &to_conv) {
   // conversions. Since hexadecimal is unsigned these will never conflict.
   size_t prefix_len;
   char prefix[2];
-  if ((to_lower(to_conv.conv_name) == 'x') &&
+  if ((internal::tolower(to_conv.conv_name) == 'x') &&
       ((flags & FormatFlags::ALTERNATE_FORM) != 0) && num != 0) {
     prefix_len = 2;
     prefix[0] = '0';
-    prefix[1] = a + ('x' - 'a');
-  } else if ((to_lower(to_conv.conv_name) == 'b') &&
+    prefix[1] = internal::islower(to_conv.conv_name) ? 'x' : 'X';
+  } else if ((internal::tolower(to_conv.conv_name) == 'b') &&
              ((flags & FormatFlags::ALTERNATE_FORM) != 0) && num != 0) {
     prefix_len = 2;
     prefix[0] = '0';
-    prefix[1] = a + ('b' - 'a');
+    prefix[1] = internal::islower(to_conv.conv_name) ? 'b' : 'B';
   } else {
     prefix_len = (sign_char == 0 ? 0 : 1);
     prefix[0] = sign_char;
diff --git a/libc/src/stdio/scanf_core/converter_utils.h b/libc/src/stdio/scanf_core/converter_utils.h
index 61954556b838a..6f4d16cffb19c 100644
--- a/libc/src/stdio/scanf_core/converter_utils.h
+++ b/libc/src/stdio/scanf_core/converter_utils.h
@@ -19,16 +19,6 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace scanf_core {
 
-LIBC_INLINE constexpr char to_lower(char a) { return a | 32; }
-
-LIBC_INLINE constexpr int b36_char_to_int(char input) {
-  if (internal::isdigit(input))
-    return input - '0';
-  if (internal::isalpha(input))
-    return to_lower(input) + 10 - 'a';
-  return 0;
-}
-
 LIBC_INLINE void write_int_with_length(uintmax_t output_val,
                                        const FormatSection &to_conv) {
   if ((to_conv.flags & NO_WRITE) != 0) {
diff --git a/libc/src/stdio/scanf_core/float_converter.cpp b/libc/src/stdio/scanf_core/float_converter.cpp
index b2d60a249a5a7..9c714d0727214 100644
--- a/libc/src/stdio/scanf_core/float_converter.cpp
+++ b/libc/src/stdio/scanf_core/float_converter.cpp
@@ -55,11 +55,12 @@ int convert_float(Reader *reader, const FormatSection &to_conv) {
 
   // Handle inf
 
-  if (to_lower(cur_char) == inf_string[0]) {
+  if (internal::tolower(cur_char) == inf_string[0]) {
     size_t inf_index = 0;
 
-    for (; inf_index < sizeof(inf_string) && out_str.length() < max_width &&
-           to_lower(cur_char) == inf_string[inf_index];
+    for (;
+         inf_index < (sizeof(inf_string) - 1) && out_str.length() < max_width &&
+         internal::tolower(cur_char) == inf_string[inf_index];
          ++inf_index) {
       if (!out_str.append(cur_char)) {
         return ALLOCATION_FAILURE;
@@ -78,11 +79,12 @@ int convert_float(Reader *reader, const FormatSection &to_conv) {
   static const char nan_string[] = "nan";
 
   // Handle nan
-  if (to_lower(cur_char) == nan_string[0]) {
+  if (internal::tolower(cur_char) == nan_string[0]) {
     size_t nan_index = 0;
 
-    for (; nan_index < sizeof(nan_string) && out_str.length() < max_width &&
-           to_lower(cur_char) == nan_string[nan_index];
+    for (;
+         nan_index < (sizeof(nan_string) - 1) && out_str.length() < max_width &&
+         internal::tolower(cur_char) == nan_string[nan_index];
          ++nan_index) {
       if (!out_str.append(cur_char)) {
         return ALLOCATION_FAILURE;
@@ -117,7 +119,7 @@ int convert_float(Reader *reader, const FormatSection &to_conv) {
     }
 
     // If that next character is an 'x' then this is a hexadecimal number.
-    if (to_lower(cur_char) == 'x') {
+    if (internal::tolower(cur_char) == 'x') {
       base = 16;
 
       if (!out_str.append(cur_char)) {
@@ -163,7 +165,7 @@ int convert_float(Reader *reader, const FormatSection &to_conv) {
 
   // Handle the exponent, which has an exponent mark, an optional sign, and
   // decimal digits.
-  if (to_lower(cur_char) == exponent_mark) {
+  if (internal::tolower(cur_char) == exponent_mark) {
     if (!out_str.append(cur_char)) {
       return ALLOCATION_FAILURE;
     }
diff --git a/libc/src/stdio/scanf_core/int_converter.cpp b/libc/src/stdio/scanf_core/int_converter.cpp
index ecdac52e84bbd..fce817245c010 100644
--- a/libc/src/stdio/scanf_core/int_converter.cpp
+++ b/libc/src/stdio/scanf_core/int_converter.cpp
@@ -80,7 +80,8 @@ int convert_int(Reader *reader, const FormatSection &to_conv) {
     is_signed = true;
   } else if (to_conv.conv_name == 'o') {
     base = 8;
-  } else if (to_lower(to_conv.conv_name) == 'x' || to_conv.conv_name == 'p') {
+  } else if (internal::tolower(to_conv.conv_name) == 'x' ||
+             to_conv.conv_name == 'p') {
     base = 16;
   } else if (to_conv.conv_name == 'd') {
     base = 10;
@@ -122,7 +123,7 @@ int convert_int(Reader *reader, const FormatSection &to_conv) {
         return READ_OK;
       }
 
-      if (to_lower(cur_char) == 'x') {
+      if (internal::tolower(cur_char) == 'x') {
         // This is a valid hex prefix.
 
         is_number = false;
@@ -175,17 +176,18 @@ int convert_int(Reader *reader, const FormatSection &to_conv) {
 
   const uintmax_t max_div_by_base = MAX / base;
 
-  if (internal::isalnum(cur_char) && b36_char_to_int(cur_char) < base) {
+  if (internal::isalnum(cur_char) &&
+      internal::b36_char_to_int(cur_char) < base) {
     is_number = true;
   }
 
   bool has_overflow = false;
   size_t i = 0;
   for (; i < max_width && internal::isalnum(cur_char) &&
-         b36_char_to_int(cur_char) < base;
+         internal::b36_char_to_int(cur_char) < base;
        ++i, cur_char = reader->getc()) {
 
-    uintmax_t cur_digit = b36_char_to_int(cur_char);
+    uintmax_t cur_digit = internal::b36_char_to_int(cur_char);
 
     if (result == MAX) {
       has_overflow = true;
diff --git a/libc/src/stdio/scanf_core/ptr_converter.cpp b/libc/src/stdio/scanf_core/ptr_converter.cpp
index 1a42a389d74b4..37f002d3da698 100644
--- a/libc/src/stdio/scanf_core/ptr_converter.cpp
+++ b/libc/src/stdio/scanf_core/ptr_converter.cpp
@@ -8,6 +8,7 @@
 
 #include "src/stdio/scanf_core/ptr_converter.h"
 
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/stdio/scanf_core/converter_utils.h"
 #include "src/stdio/scanf_core/core_structs.h"
@@ -24,7 +25,8 @@ int convert_pointer(Reader *reader, const FormatSection &to_conv) {
   // Check if it's exactly the nullptr string, if so then it's a nullptr.
   char cur_char = reader->getc();
   size_t i = 0;
-  for (; i < sizeof(nullptr_string) && to_lower(cur_char) == nullptr_string[i];
+  for (; i < (sizeof(nullptr_string) - 1) &&
+         internal::tolower(cur_char) == nullptr_string[i];
        ++i) {
     cur_char = reader->getc();
   }
diff --git a/libc/src/stdlib/gpu/abort.cpp b/libc/src/stdlib/gpu/abort.cpp
index 3a06fb38c3f64..8a7e783990d61 100644
--- a/libc/src/stdlib/gpu/abort.cpp
+++ b/libc/src/stdlib/gpu/abort.cpp
@@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void, abort, ()) {
   // We want to first make sure the server is listening before we abort.
-  rpc::Client::Port port = rpc::client.open<RPC_ABORT>();
+  rpc::Client::Port port = rpc::client.open<LIBC_ABORT>();
   port.send_and_recv([](rpc::Buffer *, uint32_t) {},
                      [](rpc::Buffer *, uint32_t) {});
   port.send([&](rpc::Buffer *, uint32_t) {});
diff --git a/libc/src/stdlib/gpu/system.cpp b/libc/src/stdlib/gpu/system.cpp
index 1890006512de4..1677e6051c5fb 100644
--- a/libc/src/stdlib/gpu/system.cpp
+++ b/libc/src/stdlib/gpu/system.cpp
@@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, system, (const char *command)) {
   int ret;
-  rpc::Client::Port port = rpc::client.open<RPC_SYSTEM>();
+  rpc::Client::Port port = rpc::client.open<LIBC_SYSTEM>();
   port.send_n(command, internal::string_length(command) + 1);
   port.recv([&](rpc::Buffer *buffer, uint32_t) {
     ret = static_cast<int>(buffer->data[0]);
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 8fe1226dd6a7a..e3faa543e630c 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -34,24 +34,6 @@ add_header_library(
     libc.src.__support.macros.config
 )
 
-add_entrypoint_object(
-  bcopy
-  SRCS
-    bcopy.cpp
-  HDRS
-    bcopy.h
-)
-
-add_entrypoint_object(
-  index
-  SRCS
-    index.cpp
-  HDRS
-    index.h
-  DEPENDS
-    .string_utils
-)
-
 add_entrypoint_object(
   memccpy
   SRCS
@@ -98,16 +80,6 @@ add_entrypoint_object(
     memrchr.h
 )
 
-add_entrypoint_object(
-  rindex
-  SRCS
-    rindex.cpp
-  HDRS
-    rindex.h
-  DEPENDS
-    .string_utils
-)
-
 add_entrypoint_object(
   stpcpy
   SRCS
@@ -171,17 +143,6 @@ add_entrypoint_object(
     .memory_utils.inline_strcmp
 )
 
-add_entrypoint_object(
-  strcasecmp
-  SRCS
-    strcasecmp.cpp
-  HDRS
-    strcasecmp.h
-  DEPENDS
-    .memory_utils.inline_strcmp
-    libc.src.__support.ctype_utils
-)
-
 add_entrypoint_object(
   strcasestr
   SRCS
@@ -319,17 +280,6 @@ add_entrypoint_object(
     .memory_utils.inline_strcmp
 )
 
-add_entrypoint_object(
-  strncasecmp
-  SRCS
-    strncasecmp.cpp
-  HDRS
-    strncasecmp.h
-  DEPENDS
-    .memory_utils.inline_strcmp
-    libc.src.__support.ctype_utils
-)
-
 add_entrypoint_object(
   strncpy
   SRCS
@@ -475,102 +425,6 @@ add_entrypoint_object(
     .memory_utils.inline_memset
 )
 
-# Helper to define a function with multiple implementations
-# - Computes flags to satisfy required/rejected features and arch,
-# - Declares an entry point,
-# - Attach the REQUIRE_CPU_FEATURES property to the target,
-# - Add the fully qualified target to `${name}_implementations` global property for tests.
-function(add_implementation name impl_name)
-  cmake_parse_arguments(
-    "ADD_IMPL"
-    "" # Optional arguments
-    "" # Single value arguments
-    "REQUIRE;SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;MLLVM_COMPILE_OPTIONS" # Multi value arguments
-    ${ARGN})
-
-  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-    # Note that '-mllvm' needs to be prefixed with 'SHELL:' to prevent CMake flag deduplication.
-    foreach(opt IN LISTS ADD_IMPL_MLLVM_COMPILE_OPTIONS)
-      list(APPEND ADD_IMPL_COMPILE_OPTIONS "SHELL:-mllvm ${opt}")
-    endforeach()
-  endif()
-
-  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
-    # Prevent warning when passing x86 SIMD types as template arguments.
-    # e.g. "warning: ignoring attributes on template argument ‘__m128i’ [-Wignored-attributes]"
-    list(APPEND ADD_IMPL_COMPILE_OPTIONS "-Wno-ignored-attributes")
-  endif()
-
-  add_entrypoint_object(${impl_name}
-    NAME ${name}
-    SRCS ${ADD_IMPL_SRCS}
-    HDRS ${ADD_IMPL_HDRS}
-    DEPENDS ${ADD_IMPL_DEPENDS}
-    COMPILE_OPTIONS ${libc_opt_high_flag} ${ADD_IMPL_COMPILE_OPTIONS}
-  )
-  get_fq_target_name(${impl_name} fq_target_name)
-  set_target_properties(${fq_target_name} PROPERTIES REQUIRE_CPU_FEATURES "${ADD_IMPL_REQUIRE}")
-  set_property(GLOBAL APPEND PROPERTY "${name}_implementations" "${fq_target_name}")
-endfunction()
-
-# ------------------------------------------------------------------------------
-# bcmp
-# ------------------------------------------------------------------------------
-
-function(add_bcmp bcmp_name)
-  add_implementation(bcmp ${bcmp_name}
-    SRCS ${LIBC_SOURCE_DIR}/src/string/bcmp.cpp
-    HDRS ${LIBC_SOURCE_DIR}/src/string/bcmp.h
-    DEPENDS
-      .memory_utils.memory_utils
-      libc.include.string
-    ${ARGN}
-  )
-endfunction()
-
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  add_bcmp(bcmp_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
-  add_bcmp(bcmp_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
-  add_bcmp(bcmp_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
-  add_bcmp(bcmp_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512BW)
-  add_bcmp(bcmp_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
-  add_bcmp(bcmp)
-elseif(LIBC_TARGET_OS_IS_GPU)
-  add_bcmp(bcmp)
-else()
-  add_bcmp(bcmp_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
-  add_bcmp(bcmp)
-endif()
-
-# ------------------------------------------------------------------------------
-# bzero
-# ------------------------------------------------------------------------------
-
-function(add_bzero bzero_name)
-  add_implementation(bzero ${bzero_name}
-    SRCS ${LIBC_SOURCE_DIR}/src/string/bzero.cpp
-    HDRS ${LIBC_SOURCE_DIR}/src/string/bzero.h
-    DEPENDS
-      .memory_utils.inline_memset
-      libc.include.string
-    ${ARGN}
-  )
-endfunction()
-
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  add_bzero(bzero_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
-  add_bzero(bzero_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
-  add_bzero(bzero_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
-  add_bzero(bzero_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512F)
-  add_bzero(bzero_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
-  add_bzero(bzero)
-elseif(LIBC_TARGET_OS_IS_GPU)
-  add_bzero(bzero)
-else()
-  add_bzero(bzero_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
-  add_bzero(bzero)
-endif()
-
 # ------------------------------------------------------------------------------
 # memcmp
 # ------------------------------------------------------------------------------
diff --git a/libc/src/strings/CMakeLists.txt b/libc/src/strings/CMakeLists.txt
new file mode 100644
index 0000000000000..5e84c7be1f7d6
--- /dev/null
+++ b/libc/src/strings/CMakeLists.txt
@@ -0,0 +1,97 @@
+function(add_bcmp bcmp_name)
+  add_implementation(bcmp ${bcmp_name}
+    SRCS ${LIBC_SOURCE_DIR}/src/strings/bcmp.cpp
+    HDRS ${LIBC_SOURCE_DIR}/src/strings/bcmp.h
+    DEPENDS
+      libc.src.string.memory_utils.memory_utils
+    ${ARGN}
+  )
+endfunction()
+
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+  add_bcmp(bcmp_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
+  add_bcmp(bcmp_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
+  add_bcmp(bcmp_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
+  add_bcmp(bcmp_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512BW)
+  add_bcmp(bcmp_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
+  add_bcmp(bcmp)
+elseif(LIBC_TARGET_OS_IS_GPU)
+  add_bcmp(bcmp)
+else()
+  add_bcmp(bcmp_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
+  add_bcmp(bcmp)
+endif()
+
+function(add_bzero bzero_name)
+  add_implementation(bzero ${bzero_name}
+    SRCS ${LIBC_SOURCE_DIR}/src/strings/bzero.cpp
+    HDRS ${LIBC_SOURCE_DIR}/src/strings/bzero.h
+    DEPENDS
+      libc.src.string.memory_utils.inline_memset
+    ${ARGN}
+  )
+endfunction()
+
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+  add_bzero(bzero_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
+  add_bzero(bzero_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
+  add_bzero(bzero_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
+  add_bzero(bzero_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512F)
+  add_bzero(bzero_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
+  add_bzero(bzero)
+elseif(LIBC_TARGET_OS_IS_GPU)
+  add_bzero(bzero)
+else()
+  add_bzero(bzero_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
+  add_bzero(bzero)
+endif()
+
+add_entrypoint_object(
+  bcopy
+  SRCS
+    bcopy.cpp
+  HDRS
+    bcopy.h
+)
+
+add_entrypoint_object(
+  index
+  SRCS
+    index.cpp
+  HDRS
+    index.h
+  DEPENDS
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  rindex
+  SRCS
+    rindex.cpp
+  HDRS
+    rindex.h
+  DEPENDS
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  strcasecmp
+  SRCS
+    strcasecmp.cpp
+  HDRS
+    strcasecmp.h
+  DEPENDS
+    libc.src.__support.ctype_utils
+    libc.src.string.memory_utils.inline_strcmp
+)
+
+add_entrypoint_object(
+  strncasecmp
+  SRCS
+    strncasecmp.cpp
+  HDRS
+    strncasecmp.h
+  DEPENDS
+    libc.src.__support.ctype_utils
+    libc.src.string.memory_utils.inline_strcmp
+)
diff --git a/libc/src/string/bcmp.cpp b/libc/src/strings/bcmp.cpp
similarity index 95%
rename from libc/src/string/bcmp.cpp
rename to libc/src/strings/bcmp.cpp
index 6e9c9ae3b7a3b..083f13d3f95d3 100644
--- a/libc/src/string/bcmp.cpp
+++ b/libc/src/strings/bcmp.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/bcmp.h"
+#include "src/strings/bcmp.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memory_utils/inline_bcmp.h"
diff --git a/libc/src/string/bcmp.h b/libc/src/strings/bcmp.h
similarity index 83%
rename from libc/src/string/bcmp.h
rename to libc/src/strings/bcmp.h
index a82b529164d95..46b4d7163b52e 100644
--- a/libc/src/string/bcmp.h
+++ b/libc/src/strings/bcmp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_BCMP_H
-#define LLVM_LIBC_SRC_STRING_BCMP_H
+#ifndef LLVM_LIBC_SRC_STRINGS_BCMP_H
+#define LLVM_LIBC_SRC_STRINGS_BCMP_H
 
 #include "src/__support/macros/config.h"
 #include <stddef.h> // size_t
@@ -18,4 +18,4 @@ int bcmp(const void *lhs, const void *rhs, size_t count);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_BCMP_H
+#endif // LLVM_LIBC_SRC_STRINGS_BCMP_H
diff --git a/libc/src/string/bcopy.cpp b/libc/src/strings/bcopy.cpp
similarity index 95%
rename from libc/src/string/bcopy.cpp
rename to libc/src/strings/bcopy.cpp
index 89aad71e354c7..eb0358c187472 100644
--- a/libc/src/string/bcopy.cpp
+++ b/libc/src/strings/bcopy.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/bcopy.h"
+#include "src/strings/bcopy.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memory_utils/inline_memmove.h"
diff --git a/libc/src/string/bcopy.h b/libc/src/strings/bcopy.h
similarity index 83%
rename from libc/src/string/bcopy.h
rename to libc/src/strings/bcopy.h
index 4cf0263a7d639..d8ee77e83c6d8 100644
--- a/libc/src/string/bcopy.h
+++ b/libc/src/strings/bcopy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_BCOPY_H
-#define LLVM_LIBC_SRC_STRING_BCOPY_H
+#ifndef LLVM_LIBC_SRC_STRINGS_BCOPY_H
+#define LLVM_LIBC_SRC_STRINGS_BCOPY_H
 
 #include "src/__support/macros/config.h"
 #include <stddef.h> // size_t
@@ -18,4 +18,4 @@ void bcopy(const void *src, void *dest, size_t count);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_BCOPY_H
+#endif // LLVM_LIBC_SRC_STRINGS_BCOPY_H
diff --git a/libc/src/string/bzero.cpp b/libc/src/strings/bzero.cpp
similarity index 95%
rename from libc/src/string/bzero.cpp
rename to libc/src/strings/bzero.cpp
index 7bcbee3547b9b..9c2e9700442af 100644
--- a/libc/src/string/bzero.cpp
+++ b/libc/src/strings/bzero.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/bzero.h"
+#include "src/strings/bzero.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memory_utils/inline_bzero.h"
diff --git a/libc/src/string/bzero.h b/libc/src/strings/bzero.h
similarity index 82%
rename from libc/src/string/bzero.h
rename to libc/src/strings/bzero.h
index d9722197e8e8b..3e270fe41f6cd 100644
--- a/libc/src/string/bzero.h
+++ b/libc/src/strings/bzero.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_BZERO_H
-#define LLVM_LIBC_SRC_STRING_BZERO_H
+#ifndef LLVM_LIBC_SRC_STRINGS_BZERO_H
+#define LLVM_LIBC_SRC_STRINGS_BZERO_H
 
 #include "src/__support/macros/config.h"
 #include <stddef.h> // size_t
@@ -18,4 +18,4 @@ void bzero(void *ptr, size_t count);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_BZERO_H
+#endif // LLVM_LIBC_SRC_STRINGS_BZERO_H
diff --git a/libc/src/string/index.cpp b/libc/src/strings/index.cpp
similarity index 95%
rename from libc/src/string/index.cpp
rename to libc/src/strings/index.cpp
index 46cf54825f6ce..33b2be9f33e0d 100644
--- a/libc/src/string/index.cpp
+++ b/libc/src/strings/index.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/index.h"
+#include "src/strings/index.h"
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/string/index.h b/libc/src/strings/index.h
similarity index 81%
rename from libc/src/string/index.h
rename to libc/src/strings/index.h
index 9843bcdf9a573..d382cdd249aa9 100644
--- a/libc/src/string/index.h
+++ b/libc/src/strings/index.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_INDEX_H
-#define LLVM_LIBC_SRC_STRING_INDEX_H
+#ifndef LLVM_LIBC_SRC_STRINGS_INDEX_H
+#define LLVM_LIBC_SRC_STRINGS_INDEX_H
 
 #include "src/__support/macros/config.h"
 
@@ -17,4 +17,4 @@ char *index(const char *src, int c);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_INDEX_H
+#endif // LLVM_LIBC_SRC_STRINGS_INDEX_H
diff --git a/libc/src/string/rindex.cpp b/libc/src/strings/rindex.cpp
similarity index 95%
rename from libc/src/string/rindex.cpp
rename to libc/src/strings/rindex.cpp
index 25879ddb07f62..1242e0faf85fc 100644
--- a/libc/src/string/rindex.cpp
+++ b/libc/src/strings/rindex.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/rindex.h"
+#include "src/strings/rindex.h"
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
diff --git a/libc/src/string/rindex.h b/libc/src/strings/rindex.h
similarity index 81%
rename from libc/src/string/rindex.h
rename to libc/src/strings/rindex.h
index cfc35daa1f4d5..f8aa7b9be28f5 100644
--- a/libc/src/string/rindex.h
+++ b/libc/src/strings/rindex.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_RINDEX_H
-#define LLVM_LIBC_SRC_STRING_RINDEX_H
+#ifndef LLVM_LIBC_SRC_STRINGS_RINDEX_H
+#define LLVM_LIBC_SRC_STRINGS_RINDEX_H
 
 #include "src/__support/macros/config.h"
 
@@ -17,4 +17,4 @@ char *rindex(const char *src, int c);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_RINDEX_H
+#endif // LLVM_LIBC_SRC_STRINGS_RINDEX_H
diff --git a/libc/src/string/strcasecmp.cpp b/libc/src/strings/strcasecmp.cpp
similarity index 96%
rename from libc/src/string/strcasecmp.cpp
rename to libc/src/strings/strcasecmp.cpp
index 1274c047fc28e..4bbe2909df1e2 100644
--- a/libc/src/string/strcasecmp.cpp
+++ b/libc/src/strings/strcasecmp.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/strcasecmp.h"
+#include "src/strings/strcasecmp.h"
 
 #include "src/__support/common.h"
 #include "src/__support/ctype_utils.h"
diff --git a/libc/src/string/strcasecmp.h b/libc/src/strings/strcasecmp.h
similarity index 80%
rename from libc/src/string/strcasecmp.h
rename to libc/src/strings/strcasecmp.h
index 2916b8d417075..b02fb31615879 100644
--- a/libc/src/string/strcasecmp.h
+++ b/libc/src/strings/strcasecmp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_STRCASECMP_H
-#define LLVM_LIBC_SRC_STRING_STRCASECMP_H
+#ifndef LLVM_LIBC_SRC_STRINGS_STRCASECMP_H
+#define LLVM_LIBC_SRC_STRINGS_STRCASECMP_H
 
 #include "src/__support/macros/config.h"
 
@@ -17,4 +17,4 @@ int strcasecmp(const char *left, const char *right);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_STRCASECMP_H
+#endif // LLVM_LIBC_SRC_STRINGS_STRCASECMP_H
diff --git a/libc/src/string/strncasecmp.cpp b/libc/src/strings/strncasecmp.cpp
similarity index 96%
rename from libc/src/string/strncasecmp.cpp
rename to libc/src/strings/strncasecmp.cpp
index 45f82c98069b2..9c2f0ab131269 100644
--- a/libc/src/string/strncasecmp.cpp
+++ b/libc/src/strings/strncasecmp.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/strncasecmp.h"
+#include "src/strings/strncasecmp.h"
 
 #include "src/__support/common.h"
 #include "src/__support/ctype_utils.h"
diff --git a/libc/src/string/strncasecmp.h b/libc/src/strings/strncasecmp.h
similarity index 80%
rename from libc/src/string/strncasecmp.h
rename to libc/src/strings/strncasecmp.h
index 15f74994ca56d..59df517058910 100644
--- a/libc/src/string/strncasecmp.h
+++ b/libc/src/strings/strncasecmp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_STRNCASECMP_H
-#define LLVM_LIBC_SRC_STRING_STRNCASECMP_H
+#ifndef LLVM_LIBC_SRC_STRINGS_STRNCASECMP_H
+#define LLVM_LIBC_SRC_STRINGS_STRNCASECMP_H
 
 #include "src/__support/macros/config.h"
 #include <stddef.h>
@@ -18,4 +18,4 @@ int strncasecmp(const char *left, const char *right, size_t n);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_STRING_STRNCASECMP_H
+#endif // LLVM_LIBC_SRC_STRINGS_STRNCASECMP_H
diff --git a/libc/src/time/CMakeLists.txt b/libc/src/time/CMakeLists.txt
index b3318e7ca87fa..ae835dcc74274 100644
--- a/libc/src/time/CMakeLists.txt
+++ b/libc/src/time/CMakeLists.txt
@@ -106,9 +106,22 @@ add_entrypoint_object(
 
 add_entrypoint_object(
   time
+  SRCS
+    time.cpp
+  HDRS
+    time_func.h
+  DEPENDS
+    libc.hdr.time_macros
+    libc.hdr.types.time_t
+    libc.src.__support.time.clock_gettime
+    libc.src.errno.errno
+)
+
+add_entrypoint_object(
+  timespec_get
   ALIAS
   DEPENDS
-    .${LIBC_TARGET_OS}.time
+    .${LIBC_TARGET_OS}.timespec_get
 )
 
 add_entrypoint_object(
@@ -138,3 +151,10 @@ add_entrypoint_object(
   DEPENDS
     .${LIBC_TARGET_OS}.gettimeofday
 )
+
+add_entrypoint_object(
+  clock_getres
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.clock_getres
+)
diff --git a/libc/src/time/baremetal/CMakeLists.txt b/libc/src/time/baremetal/CMakeLists.txt
new file mode 100644
index 0000000000000..bf0d42e265d6f
--- /dev/null
+++ b/libc/src/time/baremetal/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_entrypoint_object(
+  timespec_get
+  SRCS
+    timespec_get.cpp
+  HDRS
+    ../timespec_get.h
+  DEPENDS
+    libc.hdr.time_macros
+    libc.hdr.types.struct_timespec
+)
diff --git a/libc/src/time/baremetal/timespec_get.cpp b/libc/src/time/baremetal/timespec_get.cpp
new file mode 100644
index 0000000000000..173fe0e06e3a9
--- /dev/null
+++ b/libc/src/time/baremetal/timespec_get.cpp
@@ -0,0 +1,27 @@
+//===-- Implementation of timespec_get for baremetal ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/timespec_get.h"
+#include "hdr/time_macros.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+extern "C" bool __llvm_libc_timespec_get_utc(struct timespec *ts);
+
+LLVM_LIBC_FUNCTION(int, timespec_get, (struct timespec * ts, int base)) {
+  if (base != TIME_UTC)
+    return 0;
+
+  if (!__llvm_libc_timespec_get_utc(ts))
+    return 0;
+  return base;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/time/clock_getres.h b/libc/src/time/clock_getres.h
new file mode 100644
index 0000000000000..c1c581c535994
--- /dev/null
+++ b/libc/src/time/clock_getres.h
@@ -0,0 +1,21 @@
+//===-- Implementation header of clock_getres -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_TIME_CLOCK_GETRES_H
+#define LLVM_LIBC_SRC_TIME_CLOCK_GETRES_H
+
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int clock_getres(clockid_t clockid, timespec *tp);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_TIME_CLOCK_GETRES_H
diff --git a/libc/src/time/gpu/CMakeLists.txt b/libc/src/time/gpu/CMakeLists.txt
index c9b4562815801..31a60595d68ff 100644
--- a/libc/src/time/gpu/CMakeLists.txt
+++ b/libc/src/time/gpu/CMakeLists.txt
@@ -1,14 +1,3 @@
-add_object_library(
-  time_utils
-  SRCS
-    time_utils.cpp
-  HDRS
-    time_utils.h
-  DEPENDS
-    libc.hdr.types.clock_t
-    libc.hdr.time_macros
-)
-
 add_entrypoint_object(
   clock
   SRCS
@@ -18,7 +7,8 @@ add_entrypoint_object(
   DEPENDS
     libc.include.time
     libc.src.__support.GPU.utils
-    .time_utils
+    libc.src.__support.time.clock_gettime
+    libc.src.__support.time.gpu.time_utils
 )
 
 add_entrypoint_object(
@@ -30,7 +20,19 @@ add_entrypoint_object(
   DEPENDS
     libc.include.time
     libc.src.__support.GPU.utils
-    .time_utils
+    libc.src.__support.time.gpu.time_utils
+)
+
+add_entrypoint_object(
+  timespec_get
+  SRCS
+    timespec_get.cpp
+  HDRS
+    ../timespec_get.h
+  DEPENDS
+    libc.hdr.time_macros
+    libc.hdr.types.struct_timespec
+    libc.src.__support.time.gpu.time_utils
 )
 
 add_entrypoint_object(
@@ -40,7 +42,9 @@ add_entrypoint_object(
   HDRS
     ../clock_gettime.h
   DEPENDS
+    libc.hdr.time_macros
     libc.hdr.types.clockid_t
     libc.hdr.types.struct_timespec
-    .time_utils
+    libc.src.__support.time.gpu.time_utils
+    libc.src.__support.time.clock_gettime
 )
diff --git a/libc/src/time/gpu/clock.cpp b/libc/src/time/gpu/clock.cpp
index 4cdb1d505aed2..add5b2725ef8f 100644
--- a/libc/src/time/gpu/clock.cpp
+++ b/libc/src/time/gpu/clock.cpp
@@ -8,7 +8,7 @@
 
 #include "src/time/clock.h"
 #include "src/__support/macros/config.h"
-#include "src/time/gpu/time_utils.h"
+#include "src/__support/time/gpu/time_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/time/gpu/clock_gettime.cpp b/libc/src/time/gpu/clock_gettime.cpp
index de7899a2a17cc..81547ef7f1ca6 100644
--- a/libc/src/time/gpu/clock_gettime.cpp
+++ b/libc/src/time/gpu/clock_gettime.cpp
@@ -10,23 +10,16 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "time_utils.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/gpu/time_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-constexpr uint64_t TICKS_PER_SEC = 1000000000UL;
-
 LLVM_LIBC_FUNCTION(int, clock_gettime, (clockid_t clockid, timespec *ts)) {
-  if (clockid != CLOCK_MONOTONIC || !ts)
-    return -1;
-
-  uint64_t ns_per_tick = TICKS_PER_SEC / GPU_CLOCKS_PER_SEC;
-  uint64_t ticks = gpu::fixed_frequency_clock();
-
-  ts->tv_nsec = (ticks * ns_per_tick) % TICKS_PER_SEC;
-  ts->tv_sec = (ticks * ns_per_tick) / TICKS_PER_SEC;
-
-  return 0;
+  ErrorOr<int> result = internal::clock_gettime(clockid, ts);
+  if (result)
+    return result.value();
+  return result.error();
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/time/gpu/nanosleep.cpp b/libc/src/time/gpu/nanosleep.cpp
index 3f4a609dd40eb..25a22d5703fa7 100644
--- a/libc/src/time/gpu/nanosleep.cpp
+++ b/libc/src/time/gpu/nanosleep.cpp
@@ -9,7 +9,7 @@
 #include "src/time/nanosleep.h"
 
 #include "src/__support/macros/config.h"
-#include "time_utils.h"
+#include "src/__support/time/gpu/time_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/time/gpu/timespec_get.cpp b/libc/src/time/gpu/timespec_get.cpp
new file mode 100644
index 0000000000000..f4ef328a83122
--- /dev/null
+++ b/libc/src/time/gpu/timespec_get.cpp
@@ -0,0 +1,29 @@
+//===-- Implementation of timespec_get for gpu ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/timespec_get.h"
+#include "hdr/time_macros.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, timespec_get, (struct timespec * ts, int base)) {
+  if (base != TIME_MONOTONIC || !ts)
+    return 0;
+
+  uint64_t ns_per_tick = TICKS_PER_SEC / GPU_CLOCKS_PER_SEC;
+  uint64_t ticks = gpu::fixed_frequency_clock();
+
+  ts->tv_nsec = (ticks * ns_per_tick) % TICKS_PER_SEC;
+  ts->tv_sec = (ticks * ns_per_tick) / TICKS_PER_SEC;
+
+  return base;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/time/linux/CMakeLists.txt b/libc/src/time/linux/CMakeLists.txt
index c15fb44ad5d12..314623f9f425e 100644
--- a/libc/src/time/linux/CMakeLists.txt
+++ b/libc/src/time/linux/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_entrypoint_object(
-  time
+  timespec_get
   SRCS
-    time.cpp
+    timespec_get.cpp
   HDRS
-    ../time_func.h
+    ../timespec_get.h
   DEPENDS
     libc.hdr.time_macros
-    libc.hdr.types.time_t
-    libc.src.__support.time.linux.clock_gettime
+    libc.hdr.types.struct_timespec
+    libc.src.__support.time.clock_gettime
     libc.src.errno.errno
 )
 
@@ -21,7 +21,7 @@ add_entrypoint_object(
     libc.hdr.time_macros
     libc.hdr.types.clock_t
     libc.src.__support.time.units
-    libc.src.__support.time.linux.clock_gettime
+    libc.src.__support.time.clock_gettime
     libc.src.__support.CPP.limits
     libc.src.errno.errno
 )
@@ -49,7 +49,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.clockid_t
     libc.hdr.types.struct_timespec
-    libc.src.__support.time.linux.clock_gettime
+    libc.src.__support.time.clock_gettime
     libc.src.errno.errno
 )
 
@@ -62,7 +62,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.time_macros
     libc.hdr.types.suseconds_t
-    libc.src.__support.time.linux.clock_gettime
+    libc.src.__support.time.clock_gettime
     libc.src.__support.time.units
     libc.src.errno.errno
 )
diff --git a/libc/src/time/linux/clock.cpp b/libc/src/time/linux/clock.cpp
index f43e1bcad6a3a..ee4fa82b4f894 100644
--- a/libc/src/time/linux/clock.cpp
+++ b/libc/src/time/linux/clock.cpp
@@ -11,7 +11,7 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
 
diff --git a/libc/src/time/linux/clock_gettime.cpp b/libc/src/time/linux/clock_gettime.cpp
index a2b20a6dbc987..743c644d65d02 100644
--- a/libc/src/time/linux/clock_gettime.cpp
+++ b/libc/src/time/linux/clock_gettime.cpp
@@ -9,7 +9,7 @@
 #include "src/time/clock_gettime.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/time/linux/gettimeofday.cpp b/libc/src/time/linux/gettimeofday.cpp
index 19d9988ae73a6..e8ddf482fc984 100644
--- a/libc/src/time/linux/gettimeofday.cpp
+++ b/libc/src/time/linux/gettimeofday.cpp
@@ -11,7 +11,7 @@
 #include "hdr/types/suseconds_t.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
 
diff --git a/libc/src/time/linux/timespec_get.cpp b/libc/src/time/linux/timespec_get.cpp
new file mode 100644
index 0000000000000..cf5174523aa4f
--- /dev/null
+++ b/libc/src/time/linux/timespec_get.cpp
@@ -0,0 +1,45 @@
+//===-- Implementation of timespec_get for Linux --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/timespec_get.h"
+#include "hdr/time_macros.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/errno/libc_errno.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, timespec_get, (struct timespec * ts, int base)) {
+  clockid_t clockid;
+  switch (base) {
+  case TIME_UTC:
+    clockid = CLOCK_REALTIME;
+    break;
+  case TIME_MONOTONIC:
+    clockid = CLOCK_MONOTONIC;
+    break;
+  case TIME_ACTIVE:
+    clockid = CLOCK_PROCESS_CPUTIME_ID;
+    break;
+  case TIME_THREAD_ACTIVE:
+    clockid = CLOCK_THREAD_CPUTIME_ID;
+    break;
+  default:
+    return 0;
+  }
+
+  auto result = internal::clock_gettime(clockid, ts);
+  if (!result.has_value()) {
+    libc_errno = result.error();
+    return 0;
+  }
+  return base;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/time/linux/time.cpp b/libc/src/time/time.cpp
similarity index 83%
rename from libc/src/time/linux/time.cpp
rename to libc/src/time/time.cpp
index 20fb86e8e29db..4a0b614a68ef5 100644
--- a/libc/src/time/linux/time.cpp
+++ b/libc/src/time/time.cpp
@@ -9,14 +9,14 @@
 #include "hdr/time_macros.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
 #include "src/time/time_func.h"
 
 namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(time_t, time, (time_t * tp)) {
-  // TODO: Use the Linux VDSO to fetch the time and avoid the syscall.
+// avoid inconsitent clang-format behavior
+using time_ptr_t = time_t *;
+LLVM_LIBC_FUNCTION(time_t, time, (time_ptr_t tp)) {
   struct timespec ts;
   auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
   if (!result.has_value()) {
diff --git a/libc/src/time/timespec_get.h b/libc/src/time/timespec_get.h
new file mode 100644
index 0000000000000..2a176ff09ce4d
--- /dev/null
+++ b/libc/src/time/timespec_get.h
@@ -0,0 +1,21 @@
+//===-- Implementation header of timespec_get -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_TIME_TIMESPEC_GET_H
+#define LLVM_LIBC_SRC_TIME_TIMESPEC_GET_H
+
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int timespec_get(struct timespec *ts, int base);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_TIME_TIMESPEC_GET_H
diff --git a/libc/src/time/windows/CMakeLists.txt b/libc/src/time/windows/CMakeLists.txt
new file mode 100644
index 0000000000000..6f242cd168f52
--- /dev/null
+++ b/libc/src/time/windows/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_entrypoint_object(
+    clock_getres
+    SRCS
+        clock_getres.cpp
+    DEPENDS
+        libc.src.__support.time.windows.performance_counter
+        libc.src.__support.time.units
+        libc.src.__support.common
+        libc.src.__support.macros.optimization
+        libc.hdr.time_macros
+        libc.hdr.types.time_t
+        libc.hdr.types.struct_timespec
+)
diff --git a/libc/src/time/windows/clock_getres.cpp b/libc/src/time/windows/clock_getres.cpp
new file mode 100644
index 0000000000000..b8c0c82aa6419
--- /dev/null
+++ b/libc/src/time/windows/clock_getres.cpp
@@ -0,0 +1,110 @@
+//===-- Windows implementation of clock_getres ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "hdr/time_macros.h"
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
+
+#include "src/__support/CPP/limits.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/optimization.h"
+#include "src/__support/time/units.h"
+#include "src/__support/time/windows/performance_counter.h"
+#include "src/errno/libc_errno.h"
+#include "src/time/clock_getres.h"
+
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <Windows.h>
+
+// add in dependencies for GetSystemTimeAdjustmentPrecise
+#pragma comment(lib, "mincore.lib")
+
+namespace LIBC_NAMESPACE_DECL {
+LLVM_LIBC_FUNCTION(int, clock_getres, (clockid_t id, struct timespec *res)) {
+  using namespace time_units;
+  // POSIX allows nullptr to be passed as res, in which case the function should
+  // do nothing.
+  if (res == nullptr)
+    return 0;
+  constexpr unsigned long long HNS_PER_SEC = 1_s_ns / 100ULL;
+  constexpr unsigned long long SEC_LIMIT =
+      cpp::numeric_limits<decltype(res->tv_sec)>::max();
+  // For CLOCK_MONOTONIC, we are using performance counter
+  // https://learn.microsoft.com/en-us/windows/win32/sysinfo/acquiring-high-resolution-time-stamps
+  // Hence, the resolution is given by the performance counter frequency.
+  // For CLOCK_REALTIME, the precision is given by
+  // GetSystemTimeAdjustmentPrecise
+  // (https://learn.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimeadjustmentprecise)
+  // For CLOCK_PROCESS_CPUTIME_ID, CLOCK_THREAD_CPUTIME_ID, the precision is
+  // given by GetSystemTimeAdjustment
+  // (https://learn.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimeadjustment)
+  switch (id) {
+  default:
+    libc_errno = EINVAL;
+    return -1;
+
+  case CLOCK_MONOTONIC: {
+    long long freq = performance_counter::get_ticks_per_second();
+    __builtin_assume(freq != 0);
+    // division of 1 second by frequency, rounded up.
+    long long tv_sec = static_cast<long long>(freq == 1);
+    long long tv_nsec =
+        LIBC_LIKELY(freq != 1) ? 1ll + ((1_s_ns - 1ll) / freq) : 0ll;
+    // not possible to overflow tv_sec, tv_nsec
+    res->tv_sec = static_cast<decltype(res->tv_sec)>(tv_sec);
+    res->tv_nsec = static_cast<decltype(res->tv_nsec)>(tv_nsec);
+    break;
+  }
+
+  case CLOCK_REALTIME: {
+    [[clang::uninitialized]] DWORD64 time_adjustment;
+    [[clang::uninitialized]] DWORD64 time_increment;
+    [[clang::uninitialized]] BOOL time_adjustment_disabled;
+    if (!::GetSystemTimeAdjustmentPrecise(&time_adjustment, &time_increment,
+                                          &time_adjustment_disabled)) {
+      libc_errno = EINVAL;
+      return -1;
+    }
+    DWORD64 tv_sec = time_increment / HNS_PER_SEC;
+    DWORD64 tv_nsec = (time_increment % HNS_PER_SEC) * 100ULL;
+    if (LIBC_UNLIKELY(tv_sec > SEC_LIMIT)) {
+      libc_errno = EOVERFLOW;
+      return -1;
+    }
+    res->tv_sec = static_cast<decltype(res->tv_sec)>(tv_sec);
+    res->tv_nsec = static_cast<decltype(res->tv_nsec)>(tv_nsec);
+    break;
+  }
+  case CLOCK_PROCESS_CPUTIME_ID:
+  case CLOCK_THREAD_CPUTIME_ID: {
+    [[clang::uninitialized]] DWORD time_adjustment;
+    [[clang::uninitialized]] DWORD time_increment;
+    [[clang::uninitialized]] BOOL time_adjustment_disabled;
+    if (!::GetSystemTimeAdjustment(&time_adjustment, &time_increment,
+                                   &time_adjustment_disabled)) {
+      libc_errno = EINVAL;
+      return -1;
+    }
+    DWORD hns_per_sec = static_cast<DWORD>(HNS_PER_SEC);
+    DWORD sec_limit = static_cast<DWORD>(SEC_LIMIT);
+    DWORD tv_sec = time_increment / hns_per_sec;
+    DWORD tv_nsec = (time_increment % hns_per_sec) * 100UL;
+    if (LIBC_UNLIKELY(tv_sec > sec_limit)) {
+      libc_errno = EOVERFLOW;
+      return -1;
+    }
+    res->tv_sec = static_cast<decltype(res->tv_sec)>(tv_sec);
+    res->tv_nsec = static_cast<decltype(res->tv_nsec)>(tv_nsec);
+    break;
+  }
+  }
+  return 0;
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/UnitTest/MemoryMatcher.cpp b/libc/test/UnitTest/MemoryMatcher.cpp
index 244f25572c378..3cd5174fd7f75 100644
--- a/libc/test/UnitTest/MemoryMatcher.cpp
+++ b/libc/test/UnitTest/MemoryMatcher.cpp
@@ -8,6 +8,7 @@
 
 #include "MemoryMatcher.h"
 
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
 #include "test/UnitTest/Test.h"
 
@@ -40,7 +41,8 @@ bool MemoryMatcher::match(MemoryView actualValue) {
 
 static void display(char C) {
   const auto print = [](unsigned char I) {
-    tlog << static_cast<char>(I < 10 ? '0' + I : 'A' + I - 10);
+    tlog << static_cast<char>(LIBC_NAMESPACE::internal::toupper(
+        LIBC_NAMESPACE::internal::int_to_b36_char(I)));
   };
   print(static_cast<unsigned char>(C) / 16);
   print(static_cast<unsigned char>(C) & 15);
diff --git a/libc/test/integration/startup/gpu/CMakeLists.txt b/libc/test/integration/startup/gpu/CMakeLists.txt
index 7555986b16df4..1eee7bcc3d18a 100644
--- a/libc/test/integration/startup/gpu/CMakeLists.txt
+++ b/libc/test/integration/startup/gpu/CMakeLists.txt
@@ -53,3 +53,36 @@ add_integration_test(
    --threads 32
    --blocks 8
 )
+
+if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
+  add_integration_test(
+    startup_rpc_lane_test_w32
+    SUITE libc-startup-tests
+    SRCS
+      rpc_lane_test
+    LOADER_ARGS
+      --threads 32
+    COMPILE_OPTIONS
+      -mno-wavefrontsize64
+  )
+
+  add_integration_test(
+    startup_rpc_lane_test_w64
+    SUITE libc-startup-tests
+    SRCS
+      rpc_lane_test.cpp
+    LOADER_ARGS
+      --threads 64
+    COMPILE_OPTIONS
+      -mwavefrontsize64
+  )
+else()
+  add_integration_test(
+    startup_rpc_lane_test_w32
+    SUITE libc-startup-tests
+    SRCS
+      rpc_lane_test.cpp
+    LOADER_ARGS
+      --threads 32
+  )
+endif()
diff --git a/libc/test/integration/startup/gpu/rpc_lane_test.cpp b/libc/test/integration/startup/gpu/rpc_lane_test.cpp
new file mode 100644
index 0000000000000..72c7109eecfd6
--- /dev/null
+++ b/libc/test/integration/startup/gpu/rpc_lane_test.cpp
@@ -0,0 +1,36 @@
+//===-- Loader test to check the RPC interface with the loader ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/llvm-libc-types/test_rpc_opcodes_t.h"
+#include "src/__support/GPU/utils.h"
+#include "src/__support/RPC/rpc_client.h"
+#include "test/IntegrationTest/test.h"
+
+using namespace LIBC_NAMESPACE;
+
+static void test_add() {
+  uint64_t cnt = gpu::get_lane_id();
+  LIBC_NAMESPACE::rpc::Client::Port port =
+      LIBC_NAMESPACE::rpc::client.open<RPC_TEST_INCREMENT>();
+  port.send_and_recv(
+      [=](LIBC_NAMESPACE::rpc::Buffer *buffer, uint32_t) {
+        reinterpret_cast<uint64_t *>(buffer->data)[0] = cnt;
+      },
+      [&](LIBC_NAMESPACE::rpc::Buffer *buffer, uint32_t) {
+        cnt = reinterpret_cast<uint64_t *>(buffer->data)[0];
+      });
+  port.close();
+  EXPECT_EQ(cnt, gpu::get_lane_id() + 1);
+  EXPECT_EQ(gpu::get_thread_id(), gpu::get_lane_id());
+}
+
+TEST_MAIN(int argc, char **argv, char **envp) {
+  test_add();
+
+  return 0;
+}
diff --git a/libc/test/integration/startup/gpu/rpc_test.cpp b/libc/test/integration/startup/gpu/rpc_test.cpp
index 3deb72b9f85da..6dce6cde9770a 100644
--- a/libc/test/integration/startup/gpu/rpc_test.cpp
+++ b/libc/test/integration/startup/gpu/rpc_test.cpp
@@ -35,7 +35,7 @@ static void test_add_simple() {
 // Test to ensure that the RPC mechanism doesn't hang on divergence.
 static void test_noop(uint8_t data) {
   LIBC_NAMESPACE::rpc::Client::Port port =
-      LIBC_NAMESPACE::rpc::client.open<RPC_NOOP>();
+      LIBC_NAMESPACE::rpc::client.open<LIBC_NOOP>();
   port.send([=](LIBC_NAMESPACE::rpc::Buffer *buffer, uint32_t) {
     buffer->data[0] = data;
   });
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index e121555bd60a9..31008508d6492 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -58,7 +58,9 @@ add_subdirectory(stdfix)
 add_subdirectory(stdio)
 add_subdirectory(stdlib)
 add_subdirectory(string)
+add_subdirectory(strings)
 add_subdirectory(wchar)
+add_subdirectory(time)
 
 # Depends on utilities in stdlib
 add_subdirectory(inttypes)
@@ -75,94 +77,15 @@ if(NOT LLVM_LIBC_FULL_BUILD)
   return()
 endif()
 
+add_subdirectory(arpa)
 add_subdirectory(assert)
 add_subdirectory(compiler)
 add_subdirectory(dirent)
-add_subdirectory(network)
+add_subdirectory(locale)
 add_subdirectory(setjmp)
 add_subdirectory(signal)
 add_subdirectory(spawn)
-add_subdirectory(time)
-add_subdirectory(locale)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(pthread)
 endif()
-
-if(LLVM_RUNTIMES_BUILD OR LIBC_HDRGEN_EXE)
-  # The public API test below uses tablegen to generate the test
-  # source file. Since tablegen is not available during a runtimes
-  # build, we will skip the test.
-  # If a different libc-hdrgen binary is being used, then also we
-  # skip the api-test as we cannot generate the test source file.
-  return()
-endif()
-
-set(public_test ${CMAKE_CURRENT_BINARY_DIR}/public_api_test.cpp)
-
-set(entrypoints_name_list "")
-foreach(entry IN LISTS TARGET_LLVMLIBC_ENTRYPOINTS)
-  get_target_property(entry_name ${entry} "ENTRYPOINT_NAME")
-  list(APPEND entrypoints_name_list ${entry_name})
-endforeach()
-
-# TODO: Remove these when they are added to the TableGen.
-list(REMOVE_ITEM entrypoints_name_list "__assert_fail" "__errno_location")
-list(TRANSFORM entrypoints_name_list PREPEND "-e=")
-
-file(GLOB spec_files ${LIBC_SOURCE_DIR}/spec/*.td)
-
-# Generate api test souce code.
-add_custom_command(
-  OUTPUT ${public_test}
-  COMMAND $<TARGET_FILE:libc-prototype-testgen> -o ${public_test}
-          ${entrypoints_name_list}
-          -I ${LIBC_SOURCE_DIR}
-          ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td
-
-  DEPENDS ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td ${spec_files}
-          libc-prototype-testgen ${TARGET_PUBLIC_HEADERS}
-          ${LIBC_TARGET}
-)
-
-add_custom_target(libc-api-test)
-add_dependencies(check-libc libc-api-test)
-
-set(
-  allocator_entrypoints
-    libc.src.stdlib.malloc
-    libc.src.stdlib.calloc
-    libc.src.stdlib.realloc
-    libc.src.stdlib.aligned_alloc
-    libc.src.stdlib.free
-)
-set(api-test-entrypoints ${TARGET_LLVMLIBC_ENTRYPOINTS})
-list(REMOVE_ITEM api-test-entrypoints ${allocator_entrypoints})
-add_integration_test(
-  api-test
-  SUITE
-    libc-api-test
-  SRCS
-    ${public_test}
-  DEPENDS
-    ${api-test-entrypoints}
-)
-
-if(COMPILER_RESOURCE_DIR AND LLVM_LIBC_ENABLE_LINTING)
-  add_custom_target(
-    libc-api-test-tidy
-    VERBATIM
-    COMMAND ${LLVM_LIBC_CLANG_TIDY} --system-headers
-      --checks=-*,llvmlibc-restrict-system-libc-headers
-      "--extra-arg=-resource-dir=${COMPILER_RESOURCE_DIR}"
-      --header-filter=.*
-      --warnings-as-errors=llvmlibc-*
-      "-config={CheckOptions: [{key: llvmlibc-restrict-system-libc-headers.Includes, value: '-*, linux/*, asm/*.h, asm-generic/*.h'}]}"
-      --quiet
-      -p ${PROJECT_BINARY_DIR}
-      ${public_test}
-    DEPENDS
-      clang-tidy ${public_test}
-  )
-  add_dependencies(libc-api-test libc-api-test-tidy)
-endif()
diff --git a/libc/test/src/__support/CPP/atomic_test.cpp b/libc/test/src/__support/CPP/atomic_test.cpp
index 5b105c8eb3d56..5c3f60e9a68cd 100644
--- a/libc/test/src/__support/CPP/atomic_test.cpp
+++ b/libc/test/src/__support/CPP/atomic_test.cpp
@@ -32,3 +32,21 @@ TEST(LlvmLibcAtomicTest, CompareExchangeStrong) {
   ASSERT_FALSE(aint.compare_exchange_strong(desired, 100));
   ASSERT_EQ(aint.load(LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED), 100);
 }
+
+struct alignas(void *) TrivialData {
+  char a;
+  char b;
+  char padding[sizeof(void *) - 2];
+};
+
+TEST(LlvmLibcAtomicTest, TrivialCompositeData) {
+  LIBC_NAMESPACE::cpp::Atomic<TrivialData> data({'a', 'b', {}});
+  ASSERT_EQ(data.load(LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED).a, 'a');
+  ASSERT_EQ(data.load(LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED).b, 'b');
+
+  auto old = data.exchange({'c', 'd', {}});
+  ASSERT_EQ(data.load(LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED).a, 'c');
+  ASSERT_EQ(data.load(LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED).b, 'd');
+  ASSERT_EQ(old.a, 'a');
+  ASSERT_EQ(old.b, 'b');
+}
diff --git a/libc/test/src/__support/CPP/stringview_test.cpp b/libc/test/src/__support/CPP/stringview_test.cpp
index 6b68f2a1c47a9..c9348243745a7 100644
--- a/libc/test/src/__support/CPP/stringview_test.cpp
+++ b/libc/test/src/__support/CPP/stringview_test.cpp
@@ -109,8 +109,6 @@ TEST(LlvmLibcStringViewTest, Observer) {
   ASSERT_EQ(ABC.back(), 'c');
 }
 
-bool isDigit(char c) { return c >= '0' && c <= '9'; }
-
 TEST(LlvmLibcStringViewTest, FindFirstOf) {
   string_view Tmp("abca");
   ASSERT_TRUE(Tmp.find_first_of('a') == 0);
@@ -236,6 +234,9 @@ TEST(LlvmLibcStringViewTest, FindFirstNotOf) {
 
 TEST(LlvmLibcStringViewTest, Contains) {
   string_view Empty;
+  static_assert(
+      'a' < 'z',
+      "This test only supports character encodings where 'a' is below 'z'");
   for (char c = 'a'; c < 'z'; ++c)
     EXPECT_FALSE(Empty.contains(c));
 
diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
index 918f5d35c94f4..dadc706421d06 100644
--- a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
+++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
@@ -12,7 +12,7 @@
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/threads/linux/raw_mutex.h"
 #include "src/__support/threads/sleep.h"
-#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/stdlib/exit.h"
 #include "src/sys/mman/mmap.h"
 #include "src/sys/mman/munmap.h"
diff --git a/libc/test/src/arpa/CMakeLists.txt b/libc/test/src/arpa/CMakeLists.txt
new file mode 100644
index 0000000000000..5c89828860ff8
--- /dev/null
+++ b/libc/test/src/arpa/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(inet)
diff --git a/libc/test/src/arpa/inet/CMakeLists.txt b/libc/test/src/arpa/inet/CMakeLists.txt
new file mode 100644
index 0000000000000..6e78e3a50e612
--- /dev/null
+++ b/libc/test/src/arpa/inet/CMakeLists.txt
@@ -0,0 +1,53 @@
+add_custom_target(libc_arpa_inet_unittests)
+
+add_libc_unittest(
+  htonl
+  SUITE
+    libc_arpa_inet_unittests
+  SRCS
+    htonl_test.cpp
+  CXX_STANDARD
+    20
+  DEPENDS
+    libc.src.arpa.inet.htonl
+    libc.src.arpa.inet.ntohl
+)
+
+add_libc_unittest(
+  htons
+  SUITE
+    libc_arpa_inet_unittests
+  SRCS
+    htons_test.cpp
+  CXX_STANDARD
+    20
+  DEPENDS
+    libc.src.arpa.inet.htons
+    libc.src.arpa.inet.ntohs
+)
+
+add_libc_unittest(
+  ntohl
+  SUITE
+    libc_arpa_inet_unittests
+  SRCS
+    ntohl_test.cpp
+  CXX_STANDARD
+    20
+  DEPENDS
+    libc.src.arpa.inet.htonl
+    libc.src.arpa.inet.ntohl
+)
+
+add_libc_unittest(
+  ntohs
+  SUITE
+    libc_arpa_inet_unittests
+  SRCS
+    ntohs_test.cpp
+  CXX_STANDARD
+    20
+  DEPENDS
+    libc.src.arpa.inet.htons
+    libc.src.arpa.inet.ntohs
+)
diff --git a/libc/test/src/network/htonl_test.cpp b/libc/test/src/arpa/inet/htonl_test.cpp
similarity index 93%
rename from libc/test/src/network/htonl_test.cpp
rename to libc/test/src/arpa/inet/htonl_test.cpp
index f2e2541312c31..4cc1e4cb4e886 100644
--- a/libc/test/src/network/htonl_test.cpp
+++ b/libc/test/src/arpa/inet/htonl_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/endian_internal.h"
-#include "src/network/htonl.h"
-#include "src/network/ntohl.h"
+#include "src/arpa/inet/htonl.h"
+#include "src/arpa/inet/ntohl.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcHtonl, SmokeTest) {
diff --git a/libc/test/src/network/htons_test.cpp b/libc/test/src/arpa/inet/htons_test.cpp
similarity index 93%
rename from libc/test/src/network/htons_test.cpp
rename to libc/test/src/arpa/inet/htons_test.cpp
index 9668162523ce5..6a95ec5587e95 100644
--- a/libc/test/src/network/htons_test.cpp
+++ b/libc/test/src/arpa/inet/htons_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/endian_internal.h"
-#include "src/network/htons.h"
-#include "src/network/ntohs.h"
+#include "src/arpa/inet/htons.h"
+#include "src/arpa/inet/ntohs.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcHtons, SmokeTest) {
diff --git a/libc/test/src/network/ntohl_test.cpp b/libc/test/src/arpa/inet/ntohl_test.cpp
similarity index 93%
rename from libc/test/src/network/ntohl_test.cpp
rename to libc/test/src/arpa/inet/ntohl_test.cpp
index b72456b7200e5..42562486d5c00 100644
--- a/libc/test/src/network/ntohl_test.cpp
+++ b/libc/test/src/arpa/inet/ntohl_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/endian_internal.h"
-#include "src/network/htonl.h"
-#include "src/network/ntohl.h"
+#include "src/arpa/inet/htonl.h"
+#include "src/arpa/inet/ntohl.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcNtohl, SmokeTest) {
diff --git a/libc/test/src/network/ntohs_test.cpp b/libc/test/src/arpa/inet/ntohs_test.cpp
similarity index 93%
rename from libc/test/src/network/ntohs_test.cpp
rename to libc/test/src/arpa/inet/ntohs_test.cpp
index 1104356076b94..38b2c8d8fe40f 100644
--- a/libc/test/src/network/ntohs_test.cpp
+++ b/libc/test/src/arpa/inet/ntohs_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/endian_internal.h"
-#include "src/network/htons.h"
-#include "src/network/ntohs.h"
+#include "src/arpa/inet/htons.h"
+#include "src/arpa/inet/ntohs.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcNtohs, SmokeTest) {
diff --git a/libc/test/src/complex/CImagTest.h b/libc/test/src/complex/CImagTest.h
index 6d2f935002669..408460d97dfc6 100644
--- a/libc/test/src/complex/CImagTest.h
+++ b/libc/test/src/complex/CImagTest.h
@@ -38,9 +38,14 @@ class CImagTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                  neg_min_denormal);
     EXPECT_FP_EQ(func(CFPT(1241.112 + max_denormal * 1.0i)), max_denormal);
     EXPECT_FP_EQ(func(CFPT(121.121 + zero * 1.0i)), zero);
-    EXPECT_FP_EQ(func(CFPT(neg_zero + zero * 1.0i)), zero);
-    EXPECT_FP_EQ(func(CFPT(neg_zero + neg_zero * 1.0i)), neg_zero);
-    EXPECT_FP_EQ(func(CFPT(zero + neg_zero * 1.0i)), neg_zero);
+    EXPECT_FP_EQ(func(CFPT(0.0 + 0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0 + 0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0 - 0.0i)), -0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0 - 0.0i)), -0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0i)), -0.0);
   }
 
   void testRoundedNumbers(CImagFunc func) {
diff --git a/libc/test/src/complex/CMakeLists.txt b/libc/test/src/complex/CMakeLists.txt
index 8f49e6d79e179..0c668d9e1e8b9 100644
--- a/libc/test/src/complex/CMakeLists.txt
+++ b/libc/test/src/complex/CMakeLists.txt
@@ -1,5 +1,65 @@
 add_custom_target(libc-complex-unittests)
 
+add_libc_test(
+  conj_test
+  SUITE
+    libc-complex-unittests
+  SRCS
+    conj_test.cpp
+  DEPENDS
+    libc.src.complex.conj
+  LINK_LIBRARIES
+      LibcFPTestHelpers
+)
+
+add_libc_test(
+  conjf_test
+  SUITE
+    libc-complex-unittests
+  SRCS
+    conjf_test.cpp
+  DEPENDS
+    libc.src.complex.conjf
+  LINK_LIBRARIES
+    LibcFPTestHelpers
+)
+
+add_libc_test(
+  conjl_test
+  SUITE
+    libc-complex-unittests
+  SRCS
+    conjl_test.cpp
+  DEPENDS
+    libc.src.complex.conjl
+  LINK_LIBRARIES
+    LibcFPTestHelpers
+)
+
+add_libc_test(
+  conjf16_test
+  SUITE
+    libc-complex-unittests
+  SRCS
+    conjf16_test.cpp
+  DEPENDS
+    libc.src.complex.conjf16
+  LINK_LIBRARIES
+    LibcFPTestHelpers
+)
+
+add_libc_test(
+  conjf128_test
+  SUITE
+    libc-complex-unittests
+  SRCS
+    conjf128_test.cpp
+  DEPENDS
+    libc.src.complex.conjf128
+  LINK_LIBRARIES
+    LibcFPTestHelpers
+)
+
 add_libc_test(
   creal_test
   SUITE
diff --git a/libc/test/src/complex/CRealTest.h b/libc/test/src/complex/CRealTest.h
index a25555b506e22..80eafc9975f4c 100644
--- a/libc/test/src/complex/CRealTest.h
+++ b/libc/test/src/complex/CRealTest.h
@@ -37,8 +37,14 @@ class CRealTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(func(CFPT(neg_min_denormal + 781.134i)), neg_min_denormal);
     EXPECT_FP_EQ(func(CFPT(max_denormal + 1241.112i)), max_denormal);
     EXPECT_FP_EQ(func(CFPT(zero + 121.121i)), zero);
-    EXPECT_FP_EQ(func(CFPT(neg_zero + neg_zero * 1.0i)), neg_zero);
-    EXPECT_FP_EQ(func(CFPT(neg_zero + zero * 1.0i)), zero);
+    EXPECT_FP_EQ(func(CFPT(0.0 + 0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0 + 0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0 - 0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0 - 0.0i)), -0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0)), -0.0);
+    EXPECT_FP_EQ(func(CFPT(0.0i)), 0.0);
+    EXPECT_FP_EQ(func(CFPT(-0.0i)), -0.0);
   }
 
   void testRoundedNumbers(CRealFunc func) {
diff --git a/libc/test/src/complex/ConjTest.h b/libc/test/src/complex/ConjTest.h
new file mode 100644
index 0000000000000..da4fb4fd137c8
--- /dev/null
+++ b/libc/test/src/complex/ConjTest.h
@@ -0,0 +1,131 @@
+//===-- Utility class to test different flavors of conj ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_COMPLEX_CONJTEST_H
+#define LLVM_LIBC_TEST_SRC_COMPLEX_CONJTEST_H
+
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include "hdr/math_macros.h"
+
+template <typename CFPT, typename FPT>
+class ConjTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  DECLARE_SPECIAL_CONSTANTS(FPT)
+
+public:
+  typedef CFPT (*ConjFunc)(CFPT);
+
+  void testSpecialNumbers(ConjFunc func) {
+    EXPECT_CFP_EQ(func(CFPT(aNaN + 67.123i)), CFPT(aNaN - 67.123i));
+    EXPECT_CFP_EQ(func(CFPT(neg_aNaN + 78.319i)), CFPT(neg_aNaN - 78.319i));
+    EXPECT_CFP_EQ(func(CFPT(sNaN + 7813.131i)), CFPT(sNaN - 7813.131i));
+    EXPECT_CFP_EQ(func(CFPT(neg_sNaN + 7824.152i)), CFPT(neg_sNaN - 7824.152i));
+    EXPECT_CFP_EQ(func(CFPT(inf + 9024.2442i)), CFPT(inf - 9024.2442i));
+    EXPECT_CFP_EQ(func(CFPT(neg_inf + 8923.124i)), CFPT(neg_inf - 8923.124i));
+    EXPECT_CFP_EQ(func(CFPT(min_normal + 782.124i)),
+                  CFPT(min_normal - 782.124i));
+    EXPECT_CFP_EQ(func(CFPT(max_normal + 2141.2352i)),
+                  CFPT(max_normal - 2141.2352i));
+    EXPECT_CFP_EQ(func(CFPT(neg_max_normal + 341.134i)),
+                  CFPT(neg_max_normal - 341.134i));
+    EXPECT_CFP_EQ(func(CFPT(min_denormal + 781.142i)),
+                  CFPT(min_denormal - 781.142i));
+    EXPECT_CFP_EQ(func(CFPT(neg_min_denormal + 781.134i)),
+                  CFPT(neg_min_denormal - 781.134i));
+    EXPECT_CFP_EQ(func(CFPT(max_denormal + 1241.112i)),
+                  CFPT(max_denormal - 1241.112i));
+    EXPECT_CFP_EQ(func(CFPT(zero + 121.121i)), CFPT(zero - 121.121i));
+    EXPECT_CFP_EQ(func(CFPT(67.123 + aNaN * 1.0i)), CFPT(67.123 - aNaN * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(78.319 + neg_aNaN * 1.0i)),
+                  CFPT(78.319 - neg_aNaN * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(7813.131 + sNaN * 1.0i)),
+                  CFPT(7813.131 - sNaN * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(7824.152 + neg_sNaN * 1.0i)),
+                  CFPT(7824.152 - neg_sNaN * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(9024.2442 + inf * 1.0i)),
+                  CFPT(9024.2442 - inf * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(8923.124 + neg_inf * 1.0i)),
+                  CFPT(8923.124 - neg_inf * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(782.124 + min_normal * 1.0i)),
+                  CFPT(782.124 - min_normal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(2141.2352 + max_normal * 1.0i)),
+                  CFPT(2141.2352 - max_normal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(341.134 + neg_max_normal * 1.0i)),
+                  CFPT(341.134 - neg_max_normal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(781.142 + min_denormal * 1.0i)),
+                  CFPT(781.142 - min_denormal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(781.134 + neg_min_denormal * 1.0i)),
+                  CFPT(781.134 - neg_min_denormal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(1241.112 + max_denormal * 1.0i)),
+                  CFPT(1241.112 - max_denormal * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(121.121 + zero * 1.0i)),
+                  CFPT(121.121 - zero * 1.0i));
+    EXPECT_CFP_EQ(func(CFPT(0.0 - 0.0i)), CFPT(0.0 + 0.0i));
+    EXPECT_CFP_EQ(func(CFPT(0.0 + 0.0i)), CFPT(0.0 - 0.0i));
+    // This test passes because the conjugate of -0.0 - 0.0i is CMPLX(-0.0, 0.0)
+    // which cannot be represented as -0.0 + 0.0i because -0.0 + 0.0i is
+    // actually CMPLX(-0.0, 0.0) + CMPLX(0.0, 0.0) = 0.0 + 0.0i so to represent
+    // CMPLX(-0.0, 0.0), we use -0.0
+    EXPECT_CFP_EQ(func(CFPT(-0.0 - 0.0i)), CFPT(-0.0));
+    // This test passes because -0.0 + 0.0i is actually
+    // CMPLX(-0.0, 0.0) + CMPLX(0.0, 0.0) = CMPLX(-0.0 + 0.0, 0.0) = 0.0 + 0.0i
+    EXPECT_CFP_EQ(func(CFPT(-0.0 + 0.0i)), CFPT(0.0 - 0.0i));
+    EXPECT_CFP_EQ(func(CFPT(0.0)), CFPT(0.0 - 0.0i));
+    EXPECT_CFP_EQ(func(CFPT(-0.0)), CFPT(-0.0 - 0.0i));
+    EXPECT_CFP_EQ(func(CFPT(0.0i)), CFPT(0.0 - 0.0i));
+    EXPECT_CFP_EQ(func(CFPT(-0.0i)), CFPT(-0.0));
+  }
+
+  void testRoundedNumbers(ConjFunc func) {
+    EXPECT_CFP_EQ(func((CFPT)(4523.1413 + 12413.1414i)),
+                  CFPT(4523.1413 - 12413.1414i));
+    EXPECT_CFP_EQ(func((CFPT)(-4523.1413 + 12413.1414i)),
+                  CFPT(-4523.1413 - 12413.1414i));
+    EXPECT_CFP_EQ(func((CFPT)(4523.1413 - 12413.1414i)),
+                  CFPT(4523.1413 + 12413.1414i));
+    EXPECT_CFP_EQ(func((CFPT)(-4523.1413 - 12413.1414i)),
+                  CFPT(-4523.1413 + 12413.1414i));
+
+    EXPECT_CFP_EQ(func((CFPT)(3210.5678 + 9876.5432i)),
+                  CFPT(3210.5678 - 9876.5432i));
+    EXPECT_CFP_EQ(func((CFPT)(-3210.5678 + 9876.5432i)),
+                  CFPT(-3210.5678 - 9876.5432i));
+    EXPECT_CFP_EQ(func((CFPT)(3210.5678 - 9876.5432i)),
+                  CFPT(3210.5678 + 9876.5432i));
+    EXPECT_CFP_EQ(func((CFPT)(-3210.5678 - 9876.5432i)),
+                  CFPT(-3210.5678 + 9876.5432i));
+
+    EXPECT_CFP_EQ(func((CFPT)(1234.4321 + 4321.1234i)),
+                  CFPT(1234.4321 - 4321.1234i));
+    EXPECT_CFP_EQ(func((CFPT)(-1234.4321 + 4321.1234i)),
+                  CFPT(-1234.4321 - 4321.1234i));
+    EXPECT_CFP_EQ(func((CFPT)(1234.4321 - 4321.1234i)),
+                  CFPT(1234.4321 + 4321.1234i));
+    EXPECT_CFP_EQ(func((CFPT)(-1234.4321 - 4321.1234i)),
+                  CFPT(-1234.4321 + 4321.1234i));
+
+    EXPECT_CFP_EQ(func((CFPT)(6789.1234 + 8765.6789i)),
+                  CFPT(6789.1234 - 8765.6789i));
+    EXPECT_CFP_EQ(func((CFPT)(-6789.1234 + 8765.6789i)),
+                  CFPT(-6789.1234 - 8765.6789i));
+    EXPECT_CFP_EQ(func((CFPT)(6789.1234 - 8765.6789i)),
+                  CFPT(6789.1234 + 8765.6789i));
+    EXPECT_CFP_EQ(func((CFPT)(-6789.1234 - 8765.6789i)),
+                  CFPT(-6789.1234 + 8765.6789i));
+  }
+};
+
+#define LIST_CONJ_TESTS(U, T, func)                                            \
+  using LlvmLibcConjTest = ConjTest<U, T>;                                     \
+  TEST_F(LlvmLibcConjTest, SpecialNumbers) { testSpecialNumbers(&func); }      \
+  TEST_F(LlvmLibcConjTest, RoundedNumbers) { testRoundedNumbers(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_COMPLEX_CONJTEST_H
diff --git a/libc/utils/HdrGen/Command.cpp b/libc/test/src/complex/conj_test.cpp
similarity index 63%
rename from libc/utils/HdrGen/Command.cpp
rename to libc/test/src/complex/conj_test.cpp
index 04462c18aac3f..97445fa065088 100644
--- a/libc/utils/HdrGen/Command.cpp
+++ b/libc/test/src/complex/conj_test.cpp
@@ -1,4 +1,4 @@
-//===-- Base class for header generation commands -------------------------===//
+//===-- Unittests for conj ------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Command.h"
+#include "ConjTest.h"
 
-namespace llvm_libc {
+#include "src/complex/conj.h"
 
-Command::~Command() {}
-
-} // namespace llvm_libc
+LIST_CONJ_TESTS(_Complex double, double, LIBC_NAMESPACE::conj)
diff --git a/libc/test/src/complex/conjf128_test.cpp b/libc/test/src/complex/conjf128_test.cpp
new file mode 100644
index 0000000000000..a1feb9ff31fdc
--- /dev/null
+++ b/libc/test/src/complex/conjf128_test.cpp
@@ -0,0 +1,17 @@
+//===-- Unittests for conjf128 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConjTest.h"
+
+#include "src/complex/conjf128.h"
+
+#if defined(LIBC_TYPES_HAS_CFLOAT128)
+
+LIST_CONJ_TESTS(cfloat128, float128, LIBC_NAMESPACE::conjf128)
+
+#endif // LIBC_TYPES_HAS_CFLOAT128
diff --git a/libc/test/src/complex/conjf16_test.cpp b/libc/test/src/complex/conjf16_test.cpp
new file mode 100644
index 0000000000000..0de9f448e8681
--- /dev/null
+++ b/libc/test/src/complex/conjf16_test.cpp
@@ -0,0 +1,17 @@
+//===-- Unittests for conjf16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConjTest.h"
+
+#include "src/complex/conjf16.h"
+
+#if defined(LIBC_TYPES_HAS_CFLOAT16)
+
+LIST_CONJ_TESTS(cfloat16, float16, LIBC_NAMESPACE::conjf16)
+
+#endif // LIBC_TYPES_HAS_CFLOAT16
diff --git a/libc/test/src/complex/conjf_test.cpp b/libc/test/src/complex/conjf_test.cpp
new file mode 100644
index 0000000000000..34b00b37996c3
--- /dev/null
+++ b/libc/test/src/complex/conjf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for conjf -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConjTest.h"
+
+#include "src/complex/conjf.h"
+
+LIST_CONJ_TESTS(_Complex float, float, LIBC_NAMESPACE::conjf)
diff --git a/libc/test/src/complex/conjl_test.cpp b/libc/test/src/complex/conjl_test.cpp
new file mode 100644
index 0000000000000..ec299f9631547
--- /dev/null
+++ b/libc/test/src/complex/conjl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for conjl -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConjTest.h"
+
+#include "src/complex/conjl.h"
+
+LIST_CONJ_TESTS(_Complex long double, long double, LIBC_NAMESPACE::conjl)
diff --git a/libc/test/src/ctype/isalnum_test.cpp b/libc/test/src/ctype/isalnum_test.cpp
index b71d36111d725..cfc54e95e922d 100644
--- a/libc/test/src/ctype/isalnum_test.cpp
+++ b/libc/test/src/ctype/isalnum_test.cpp
@@ -6,18 +6,49 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/isalnum.h"
 
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char ALNUM_ARRAY[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsAlNum, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::isalnum('a'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::isalnum('B'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::isalnum('3'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::isalnum(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalnum('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalnum('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalnum(-1), 0);
+}
+
 TEST(LlvmLibcIsAlNum, DefaultLocale) {
   // Loops through all characters, verifying that numbers and letters
   // return non-zero integer and everything else returns a zero.
-  for (int c = -255; c < 255; ++c) {
-    if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-        ('0' <= c && c <= '9'))
-      EXPECT_NE(LIBC_NAMESPACE::isalnum(c), 0);
+  for (int ch = -255; ch < 255; ++ch) {
+    if (in_span(ch, ALNUM_ARRAY))
+      EXPECT_NE(LIBC_NAMESPACE::isalnum(ch), 0);
     else
-      EXPECT_EQ(LIBC_NAMESPACE::isalnum(c), 0);
+      EXPECT_EQ(LIBC_NAMESPACE::isalnum(ch), 0);
   }
 }
diff --git a/libc/test/src/ctype/isalpha_test.cpp b/libc/test/src/ctype/isalpha_test.cpp
index 10cdb962ee2ee..1e439cf973e35 100644
--- a/libc/test/src/ctype/isalpha_test.cpp
+++ b/libc/test/src/ctype/isalpha_test.cpp
@@ -6,15 +6,47 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/isalpha.h"
 
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char ALPHA_ARRAY[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsAlpha, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::isalpha('a'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::isalpha('B'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::isalpha('3'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalpha(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalpha('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalpha('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isalpha(-1), 0);
+}
+
 TEST(LlvmLibcIsAlpha, DefaultLocale) {
   // Loops through all characters, verifying that letters return a
   // non-zero integer and everything else returns zero.
+  // TODO: encoding indep
   for (int ch = -255; ch < 255; ++ch) {
-    if (('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z'))
+    if (in_span(ch, ALPHA_ARRAY))
       EXPECT_NE(LIBC_NAMESPACE::isalpha(ch), 0);
     else
       EXPECT_EQ(LIBC_NAMESPACE::isalpha(ch), 0);
diff --git a/libc/test/src/ctype/isdigit_test.cpp b/libc/test/src/ctype/isdigit_test.cpp
index a9f84db3ef7e8..0ee132d7f8515 100644
--- a/libc/test/src/ctype/isdigit_test.cpp
+++ b/libc/test/src/ctype/isdigit_test.cpp
@@ -6,15 +6,43 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/isdigit.h"
 
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char DIGIT_ARRAY[] = {
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsDigit, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::isdigit('3'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit('a'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit('B'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isdigit(-1), 0);
+}
+
 TEST(LlvmLibcIsDigit, DefaultLocale) {
-  // Loops through all characters, verifying that numbers return a
-  // non-zero integer and everything else returns zero.
+  // Loops through all characters, verifying that numbers and letters
+  // return non-zero integer and everything else returns a zero.
   for (int ch = -255; ch < 255; ++ch) {
-    if ('0' <= ch && ch <= '9')
+    if (in_span(ch, DIGIT_ARRAY))
       EXPECT_NE(LIBC_NAMESPACE::isdigit(ch), 0);
     else
       EXPECT_EQ(LIBC_NAMESPACE::isdigit(ch), 0);
diff --git a/libc/test/src/ctype/islower_test.cpp b/libc/test/src/ctype/islower_test.cpp
index ba7caf65b6fd3..f877171abb9a3 100644
--- a/libc/test/src/ctype/islower_test.cpp
+++ b/libc/test/src/ctype/islower_test.cpp
@@ -6,14 +6,44 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/islower.h"
+
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char LOWER_ARRAY[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsLower, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::islower('a'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::islower('B'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::islower('3'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::islower(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::islower('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::islower('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::islower(-1), 0);
+}
+
 TEST(LlvmLibcIsLower, DefaultLocale) {
-  // Loops through all characters, verifying that lowercase letters
-  // return a non-zero integer and everything else returns zero.
+  // Loops through all characters, verifying that numbers and letters
+  // return non-zero integer and everything else returns a zero.
   for (int ch = -255; ch < 255; ++ch) {
-    if ('a' <= ch && ch <= 'z')
+    if (in_span(ch, LOWER_ARRAY))
       EXPECT_NE(LIBC_NAMESPACE::islower(ch), 0);
     else
       EXPECT_EQ(LIBC_NAMESPACE::islower(ch), 0);
diff --git a/libc/test/src/ctype/isupper_test.cpp b/libc/test/src/ctype/isupper_test.cpp
index 05b2fd069ef06..151ed23f0ac99 100644
--- a/libc/test/src/ctype/isupper_test.cpp
+++ b/libc/test/src/ctype/isupper_test.cpp
@@ -6,14 +6,44 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/isupper.h"
+
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char UPPER_ARRAY[] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsUpper, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::isupper('B'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::isupper('a'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isupper('3'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isupper(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isupper('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isupper('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isupper(-1), 0);
+}
+
 TEST(LlvmLibcIsUpper, DefaultLocale) {
-  // Loops through all characters, verifying that uppercase letters
-  // return a non-zero integer and everything else returns zero.
+  // Loops through all characters, verifying that numbers and letters
+  // return non-zero integer and everything else returns a zero.
   for (int ch = -255; ch < 255; ++ch) {
-    if ('A' <= ch && ch <= 'Z')
+    if (in_span(ch, UPPER_ARRAY))
       EXPECT_NE(LIBC_NAMESPACE::isupper(ch), 0);
     else
       EXPECT_EQ(LIBC_NAMESPACE::isupper(ch), 0);
diff --git a/libc/test/src/ctype/isxdigit_test.cpp b/libc/test/src/ctype/isxdigit_test.cpp
index b8f27a968540c..ec58f0da49223 100644
--- a/libc/test/src/ctype/isxdigit_test.cpp
+++ b/libc/test/src/ctype/isxdigit_test.cpp
@@ -6,13 +6,45 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/isxdigit.h"
+
 #include "test/UnitTest/Test.h"
 
-TEST(LlvmLibcIsXDigit, DefaultLocale) {
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+constexpr char XDIGIT_ARRAY[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E',
+    'F', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+};
+
+bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return true;
+  return false;
+}
+
+} // namespace
+
+TEST(LlvmLibcIsXdigit, SimpleTest) {
+  EXPECT_NE(LIBC_NAMESPACE::isxdigit('a'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::isxdigit('B'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::isxdigit('3'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::isxdigit('z'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isxdigit(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isxdigit('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isxdigit('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::isxdigit(-1), 0);
+}
+
+TEST(LlvmLibcIsXdigit, DefaultLocale) {
+  // Loops through all characters, verifying that numbers and letters
+  // return non-zero integer and everything else returns a zero.
   for (int ch = -255; ch < 255; ++ch) {
-    if (('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f') ||
-        ('A' <= ch && ch <= 'F'))
+    if (in_span(ch, XDIGIT_ARRAY))
       EXPECT_NE(LIBC_NAMESPACE::isxdigit(ch), 0);
     else
       EXPECT_EQ(LIBC_NAMESPACE::isxdigit(ch), 0);
diff --git a/libc/test/src/ctype/tolower_test.cpp b/libc/test/src/ctype/tolower_test.cpp
index 3770ce4ea68b6..fbcc5b8ee0d9d 100644
--- a/libc/test/src/ctype/tolower_test.cpp
+++ b/libc/test/src/ctype/tolower_test.cpp
@@ -6,14 +6,55 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/tolower.h"
+
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+// Invariant: UPPER_ARR and LOWER_ARR are both the complete alphabet in the same
+// order.
+constexpr char UPPER_ARR[] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+};
+constexpr char LOWER_ARR[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+};
+
+static_assert(
+    sizeof(UPPER_ARR) == sizeof(LOWER_ARR),
+    "There must be the same number of uppercase and lowercase letters.");
+
+int span_index(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return static_cast<int>(i);
+  return -1;
+}
+
+} // namespace
+
+TEST(LlvmLibcToLower, SimpleTest) {
+  EXPECT_EQ(LIBC_NAMESPACE::tolower('a'), int('a'));
+  EXPECT_EQ(LIBC_NAMESPACE::tolower('B'), int('b'));
+  EXPECT_EQ(LIBC_NAMESPACE::tolower('3'), int('3'));
+
+  EXPECT_EQ(LIBC_NAMESPACE::tolower(' '), int(' '));
+  EXPECT_EQ(LIBC_NAMESPACE::tolower('?'), int('?'));
+  EXPECT_EQ(LIBC_NAMESPACE::tolower('\0'), int('\0'));
+  EXPECT_EQ(LIBC_NAMESPACE::tolower(-1), int(-1));
+}
+
 TEST(LlvmLibcToLower, DefaultLocale) {
   for (int ch = -255; ch < 255; ++ch) {
-    // This follows pattern 'A' + 32 = 'a'.
-    if ('A' <= ch && ch <= 'Z')
-      EXPECT_EQ(LIBC_NAMESPACE::tolower(ch), ch + 32);
+    int char_index = span_index(ch, UPPER_ARR);
+    if (char_index != -1)
+      EXPECT_EQ(LIBC_NAMESPACE::tolower(ch),
+                static_cast<int>(LOWER_ARR[char_index]));
     else
       EXPECT_EQ(LIBC_NAMESPACE::tolower(ch), ch);
   }
diff --git a/libc/test/src/ctype/toupper_test.cpp b/libc/test/src/ctype/toupper_test.cpp
index 0413b43fb6009..409b3cd96ed1e 100644
--- a/libc/test/src/ctype/toupper_test.cpp
+++ b/libc/test/src/ctype/toupper_test.cpp
@@ -6,14 +6,55 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/span.h"
 #include "src/ctype/toupper.h"
+
 #include "test/UnitTest/Test.h"
 
+namespace {
+
+// TODO: Merge the ctype tests using this framework.
+// Invariant: UPPER_ARR and LOWER_ARR are both the complete alphabet in the same
+// order.
+constexpr char UPPER_ARR[] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+};
+constexpr char LOWER_ARR[] = {
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+};
+
+static_assert(
+    sizeof(UPPER_ARR) == sizeof(LOWER_ARR),
+    "There must be the same number of uppercase and lowercase letters.");
+
+int span_index(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+  for (size_t i = 0; i < arr.size(); ++i)
+    if (static_cast<int>(arr[i]) == ch)
+      return static_cast<int>(i);
+  return -1;
+}
+
+} // namespace
+
+TEST(LlvmLibcToUpper, SimpleTest) {
+  EXPECT_EQ(LIBC_NAMESPACE::toupper('a'), int('A'));
+  EXPECT_EQ(LIBC_NAMESPACE::toupper('B'), int('B'));
+  EXPECT_EQ(LIBC_NAMESPACE::toupper('3'), int('3'));
+
+  EXPECT_EQ(LIBC_NAMESPACE::toupper(' '), int(' '));
+  EXPECT_EQ(LIBC_NAMESPACE::toupper('?'), int('?'));
+  EXPECT_EQ(LIBC_NAMESPACE::toupper('\0'), int('\0'));
+  EXPECT_EQ(LIBC_NAMESPACE::toupper(-1), int(-1));
+}
+
 TEST(LlvmLibcToUpper, DefaultLocale) {
   for (int ch = -255; ch < 255; ++ch) {
-    // This follows pattern 'a' - 32 = 'A'.
-    if ('a' <= ch && ch <= 'z')
-      EXPECT_EQ(LIBC_NAMESPACE::toupper(ch), ch - 32);
+    int char_index = span_index(ch, LOWER_ARR);
+    if (char_index != -1)
+      EXPECT_EQ(LIBC_NAMESPACE::toupper(ch),
+                static_cast<int>(UPPER_ARR[char_index]));
     else
       EXPECT_EQ(LIBC_NAMESPACE::toupper(ch), ch);
   }
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 610f4d9fc1a3b..ea75720df4f43 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -85,6 +85,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  sinf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    sinf16_test.cpp
+  DEPENDS
+    libc.src.math.sinf16  
+)
+
 add_fp_unittest(
   sinpif_test
   NEED_MPFR
diff --git a/libc/test/src/math/sinf16_test.cpp b/libc/test/src/math/sinf16_test.cpp
new file mode 100644
index 0000000000000..b05501cb0f145
--- /dev/null
+++ b/libc/test/src/math/sinf16_test.cpp
@@ -0,0 +1,40 @@
+//===-- Exhaustive test for sinf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/sinf16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcSinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7c00U;
+
+// Range: [-Inf, 0]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xfc00U;
+
+TEST_F(LlvmLibcSinf16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Sin, x,
+                                   LIBC_NAMESPACE::sinf16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcSinf16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Sin, x,
+                                   LIBC_NAMESPACE::sinf16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index e9c785f7d9330..2c1c4dba73846 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -49,6 +49,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  sinf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    sinf16_test.cpp
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.sinf16
+)
+
 add_fp_unittest(
   sinpif_test
   SUITE
diff --git a/libc/test/src/math/smoke/sinf16_test.cpp b/libc/test/src/math/smoke/sinf16_test.cpp
new file mode 100644
index 0000000000000..2966c3c952fd2
--- /dev/null
+++ b/libc/test/src/math/smoke/sinf16_test.cpp
@@ -0,0 +1,33 @@
+//===-- Unittests for sinf16 ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/errno/libc_errno.h"
+#include "src/math/sinf16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LlvmLibcSinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+TEST_F(LlvmLibcSinf16Test, SpecialNumbers) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf16(aNaN));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::sinf16(zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(neg_zero, LIBC_NAMESPACE::sinf16(neg_zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf16(inf));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf16(neg_inf));
+  EXPECT_MATH_ERRNO(EDOM);
+}
diff --git a/libc/test/src/network/CMakeLists.txt b/libc/test/src/network/CMakeLists.txt
deleted file mode 100644
index 222205dfe247a..0000000000000
--- a/libc/test/src/network/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-add_custom_target(libc_network_unittests)
-
-add_libc_unittest(
-  htonl
-  SUITE
-    libc_network_unittests
-  SRCS
-    htonl_test.cpp
-  CXX_STANDARD
-    20
-  DEPENDS
-    libc.src.network.htonl
-    libc.src.network.ntohl
-)
-
-add_libc_unittest(
-  htons
-  SUITE
-    libc_network_unittests
-  SRCS
-    htons_test.cpp
-  CXX_STANDARD
-    20
-  DEPENDS
-    libc.src.network.htons
-    libc.src.network.ntohs
-)
-
-add_libc_unittest(
-  ntohl
-  SUITE
-    libc_network_unittests
-  SRCS
-    ntohl_test.cpp
-  CXX_STANDARD
-    20
-  DEPENDS
-    libc.src.network.htonl
-    libc.src.network.ntohl
-)
-
-add_libc_unittest(
-  ntohs
-  SUITE
-    libc_network_unittests
-  SRCS
-    ntohs_test.cpp
-  CXX_STANDARD
-    20
-  DEPENDS
-    libc.src.network.htons
-    libc.src.network.ntohs
-)
diff --git a/libc/test/src/stdlib/StrtolTest.h b/libc/test/src/stdlib/StrtolTest.h
index 8a67848e4c330..ed302f14d03ef 100644
--- a/libc/test/src/stdlib/StrtolTest.h
+++ b/libc/test/src/stdlib/StrtolTest.h
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/type_traits.h"
+#include "src/__support/ctype_utils.h"
 #include "src/__support/macros/properties/architectures.h"
 #include "src/errno/libc_errno.h"
 #include "test/UnitTest/Test.h"
@@ -16,14 +17,6 @@
 
 using LIBC_NAMESPACE::cpp::is_signed_v;
 
-static inline char int_to_b36_char(int input) {
-  if (input < 0 || input > 36)
-    return '0';
-  if (input < 10)
-    return static_cast<char>('0' + input);
-  return static_cast<char>('A' + input - 10);
-}
-
 template <typename ReturnT>
 struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
   using FunctionT = ReturnT (*)(const char *, char **, int);
@@ -207,7 +200,8 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     char small_string[4] = {'\0', '\0', '\0', '\0'};
     for (int base = 2; base <= 36; ++base) {
       for (int first_digit = 0; first_digit <= 36; ++first_digit) {
-        small_string[0] = int_to_b36_char(first_digit);
+        small_string[0] = static_cast<char>(
+            LIBC_NAMESPACE::internal::int_to_b36_char(first_digit));
         if (first_digit < base) {
           LIBC_NAMESPACE::libc_errno = 0;
           ASSERT_EQ(func(small_string, nullptr, base),
@@ -223,9 +217,11 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
 
     for (int base = 2; base <= 36; ++base) {
       for (int first_digit = 0; first_digit <= 36; ++first_digit) {
-        small_string[0] = int_to_b36_char(first_digit);
+        small_string[0] = static_cast<char>(
+            LIBC_NAMESPACE::internal::int_to_b36_char(first_digit));
         for (int second_digit = 0; second_digit <= 36; ++second_digit) {
-          small_string[1] = int_to_b36_char(second_digit);
+          small_string[1] = static_cast<char>(
+              LIBC_NAMESPACE::internal::int_to_b36_char(second_digit));
           if (first_digit < base && second_digit < base) {
             LIBC_NAMESPACE::libc_errno = 0;
             ASSERT_EQ(
@@ -248,11 +244,14 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
 
     for (int base = 2; base <= 36; ++base) {
       for (int first_digit = 0; first_digit <= 36; ++first_digit) {
-        small_string[0] = int_to_b36_char(first_digit);
+        small_string[0] = static_cast<char>(
+            LIBC_NAMESPACE::internal::int_to_b36_char(first_digit));
         for (int second_digit = 0; second_digit <= 36; ++second_digit) {
-          small_string[1] = int_to_b36_char(second_digit);
+          small_string[1] = static_cast<char>(
+              LIBC_NAMESPACE::internal::int_to_b36_char(second_digit));
           for (int third_digit = 0; third_digit <= limit; ++third_digit) {
-            small_string[2] = int_to_b36_char(third_digit);
+            small_string[2] = static_cast<char>(
+                LIBC_NAMESPACE::internal::int_to_b36_char(third_digit));
 
             if (first_digit < base && second_digit < base &&
                 third_digit < base) {
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index b6b59a689cc8f..a675373938e99 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -2,35 +2,12 @@ add_custom_target(libc-string-tests)
 
 add_subdirectory(memory_utils)
 
-add_libc_test(
-  bcopy_test
-  SUITE
-    libc-string-tests
-  SRCS
-    bcopy_test.cpp
-  DEPENDS
-    libc.src.string.bcopy
-  LINK_LIBRARIES
-    LibcMemoryHelpers
-)
-
 add_header_library(
   strchr_test_support
   HDRS
     StrchrTest.h
 )
 
-add_libc_test(
-  index_test
-  SUITE
-    libc-string-tests
-  SRCS
-    index_test.cpp
-  DEPENDS
-    libc.src.string.index
-    .strchr_test_support
-)
-
 add_libc_test(
   memccpy_test
   SUITE
@@ -81,17 +58,6 @@ add_libc_test(
     libc.src.string.memrchr
 )
 
-add_libc_test(
-  rindex_test
-  SUITE
-    libc-string-tests
-  SRCS
-    rindex_test.cpp
-  DEPENDS
-    libc.src.string.rindex
-    .strchr_test_support
-)
-
 add_libc_test(
   stpcpy_test
   SUITE
@@ -153,16 +119,6 @@ add_libc_test(
     libc.src.string.strcmp
 )
 
-add_libc_test(
-  strcasecmp_test
-  SUITE
-    libc-string-tests
-  SRCS
-    strcasecmp_test.cpp
-  DEPENDS
-    libc.src.string.strcasecmp
-)
-
 add_libc_test(
   strcasestr_test
   SUITE
@@ -287,16 +243,6 @@ add_libc_test(
     libc.src.string.strncmp
 )
 
-add_libc_test(
-  strncasecmp_test
-  SUITE
-    libc-string-tests
-  SRCS
-    strncasecmp_test.cpp
-  DEPENDS
-    libc.src.string.strncasecmp
-)
-
 add_libc_test(
   strncpy_test
   SUITE
@@ -428,39 +374,7 @@ add_libc_test(
     libc.src.string.memset_explicit
 )
 
-# Tests all implementations that can run on the target CPU.
-function(add_libc_multi_impl_test name)
-  get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
-  foreach(fq_config_name IN LISTS fq_implementations)
-    get_target_property(required_cpu_features ${fq_config_name} REQUIRE_CPU_FEATURES)
-    cpu_supports(can_run "${required_cpu_features}")
-    if(can_run)
-      string(FIND ${fq_config_name} "." last_dot_loc REVERSE)
-      math(EXPR name_loc "${last_dot_loc} + 1")
-      string(SUBSTRING ${fq_config_name} ${name_loc} -1 target_name)
-      add_libc_test(
-        ${target_name}_test
-        SUITE
-          libc-string-tests
-        COMPILE_OPTIONS
-          ${LIBC_COMPILE_OPTIONS_NATIVE}
-        LINK_LIBRARIES
-          LibcMemoryHelpers
-        ${ARGN}
-        DEPENDS
-          ${fq_config_name}
-          libc.src.__support.macros.sanitizer
-      )
-      get_fq_target_name(${fq_config_name}_test fq_target_name)
-    else()
-      message(STATUS "Skipping test for '${fq_config_name}' insufficient host cpu features '${required_cpu_features}'")
-    endif()
-  endforeach()
-endfunction()
-
-add_libc_multi_impl_test(bcmp SRCS bcmp_test.cpp)
-add_libc_multi_impl_test(bzero SRCS bzero_test.cpp)
-add_libc_multi_impl_test(memcmp SRCS memcmp_test.cpp)
-add_libc_multi_impl_test(memcpy SRCS memcpy_test.cpp)
-add_libc_multi_impl_test(memmove SRCS memmove_test.cpp)
-add_libc_multi_impl_test(memset SRCS memset_test.cpp)
+add_libc_multi_impl_test(memcmp libc-string-tests SRCS memcmp_test.cpp)
+add_libc_multi_impl_test(memcpy libc-string-tests SRCS memcpy_test.cpp)
+add_libc_multi_impl_test(memmove libc-string-tests SRCS memmove_test.cpp)
+add_libc_multi_impl_test(memset libc-string-tests SRCS memset_test.cpp)
diff --git a/libc/test/src/string/strcmp_test.cpp b/libc/test/src/string/strcmp_test.cpp
index ef58dc608c83b..234447610222f 100644
--- a/libc/test/src/string/strcmp_test.cpp
+++ b/libc/test/src/string/strcmp_test.cpp
@@ -25,13 +25,13 @@ TEST(LlvmLibcStrCmpTest, EmptyStringShouldNotEqualNonEmptyString) {
   const char *s2 = "abc";
   int result = LIBC_NAMESPACE::strcmp(empty, s2);
   // This should be '\0' - 'a' = -97
-  ASSERT_EQ(result, -97);
+  ASSERT_EQ(result, '\0' - 'a');
 
   // Similar case if empty string is second argument.
   const char *s3 = "123";
   result = LIBC_NAMESPACE::strcmp(s3, empty);
   // This should be '1' - '\0' = 49
-  ASSERT_EQ(result, 49);
+  ASSERT_EQ(result, '1' - '\0');
 }
 
 TEST(LlvmLibcStrCmpTest, EqualStringsShouldReturnZero) {
@@ -50,12 +50,12 @@ TEST(LlvmLibcStrCmpTest, ShouldReturnResultOfFirstDifference) {
   const char *s2 = "___C55__";
   int result = LIBC_NAMESPACE::strcmp(s1, s2);
   // This should return 'B' - 'C' = -1.
-  ASSERT_EQ(result, -1);
+  ASSERT_EQ(result, 'B' - 'C');
 
   // Verify operands reversed.
   result = LIBC_NAMESPACE::strcmp(s2, s1);
   // This should return 'C' - 'B' = 1.
-  ASSERT_EQ(result, 1);
+  ASSERT_EQ(result, 'C' - 'B');
 }
 
 TEST(LlvmLibcStrCmpTest, CapitalizedLetterShouldNotBeEqual) {
@@ -63,12 +63,12 @@ TEST(LlvmLibcStrCmpTest, CapitalizedLetterShouldNotBeEqual) {
   const char *s2 = "abCd";
   int result = LIBC_NAMESPACE::strcmp(s1, s2);
   // 'c' - 'C' = 32.
-  ASSERT_EQ(result, 32);
+  ASSERT_EQ(result, 'c' - 'C');
 
   // Verify operands reversed.
   result = LIBC_NAMESPACE::strcmp(s2, s1);
   // 'C' - 'c' = -32.
-  ASSERT_EQ(result, -32);
+  ASSERT_EQ(result, 'C' - 'c');
 }
 
 TEST(LlvmLibcStrCmpTest, UnequalLengthStringsShouldNotReturnZero) {
@@ -76,12 +76,12 @@ TEST(LlvmLibcStrCmpTest, UnequalLengthStringsShouldNotReturnZero) {
   const char *s2 = "abcd";
   int result = LIBC_NAMESPACE::strcmp(s1, s2);
   // '\0' - 'd' = -100.
-  ASSERT_EQ(result, -100);
+  ASSERT_EQ(result, -'\0' - 'd');
 
   // Verify operands reversed.
   result = LIBC_NAMESPACE::strcmp(s2, s1);
   // 'd' - '\0' = 100.
-  ASSERT_EQ(result, 100);
+  ASSERT_EQ(result, 'd' - '\0');
 }
 
 TEST(LlvmLibcStrCmpTest, StringArgumentSwapChangesSign) {
@@ -89,11 +89,11 @@ TEST(LlvmLibcStrCmpTest, StringArgumentSwapChangesSign) {
   const char *b = "b";
   int result = LIBC_NAMESPACE::strcmp(b, a);
   // 'b' - 'a' = 1.
-  ASSERT_EQ(result, 1);
+  ASSERT_EQ(result, 'b' - 'a');
 
   result = LIBC_NAMESPACE::strcmp(a, b);
   // 'a' - 'b' = -1.
-  ASSERT_EQ(result, -1);
+  ASSERT_EQ(result, 'a' - 'b');
 }
 
 TEST(LlvmLibcStrCmpTest, Case) {
diff --git a/libc/test/src/strings/CMakeLists.txt b/libc/test/src/strings/CMakeLists.txt
new file mode 100644
index 0000000000000..963a1d6d6d60c
--- /dev/null
+++ b/libc/test/src/strings/CMakeLists.txt
@@ -0,0 +1,58 @@
+add_custom_target(libc-strings-tests)
+
+add_libc_test(
+  bcopy_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    bcopy_test.cpp
+  DEPENDS
+    libc.src.strings.bcopy
+  LINK_LIBRARIES
+    LibcMemoryHelpers
+)
+
+add_libc_test(
+  index_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    index_test.cpp
+  DEPENDS
+    libc.src.strings.index
+    libc.test.src.strchr_test_support
+)
+
+add_libc_test(
+  rindex_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    rindex_test.cpp
+  DEPENDS
+    libc.src.strings.rindex
+    libc.test.src.strchr_test_support
+)
+
+add_libc_test(
+  strcasecmp_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    strcasecmp_test.cpp
+  DEPENDS
+    libc.src.strings.strcasecmp
+)
+
+add_libc_test(
+  strncasecmp_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    strncasecmp_test.cpp
+  DEPENDS
+    libc.src.strings.strncasecmp
+)
+
+add_libc_multi_impl_test(bcmp libc-strings-tests SRCS bcmp_test.cpp)
+add_libc_multi_impl_test(bzero libc-strings-tests SRCS bzero_test.cpp)
diff --git a/libc/test/src/string/bcmp_test.cpp b/libc/test/src/strings/bcmp_test.cpp
similarity index 95%
rename from libc/test/src/string/bcmp_test.cpp
rename to libc/test/src/strings/bcmp_test.cpp
index c639040685e19..794f9a1b8bd9d 100644
--- a/libc/test/src/string/bcmp_test.cpp
+++ b/libc/test/src/strings/bcmp_test.cpp
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "memory_utils/memory_check_utils.h"
 #include "src/__support/macros/config.h"
-#include "src/string/bcmp.h"
+#include "src/strings/bcmp.h"
 #include "test/UnitTest/Test.h"
 #include "test/UnitTest/TestLogger.h"
+#include "test/src/string/memory_utils/memory_check_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/test/src/string/bcopy_test.cpp b/libc/test/src/strings/bcopy_test.cpp
similarity index 97%
rename from libc/test/src/string/bcopy_test.cpp
rename to libc/test/src/strings/bcopy_test.cpp
index 04772bb5d8ad7..f6bb859b4ae9f 100644
--- a/libc/test/src/string/bcopy_test.cpp
+++ b/libc/test/src/strings/bcopy_test.cpp
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/__support/macros/config.h"
-#include "src/string/bcopy.h"
+#include "src/strings/bcopy.h"
 
-#include "memory_utils/memory_check_utils.h"
 #include "src/__support/CPP/span.h"
+#include "src/__support/macros/config.h"
 #include "test/UnitTest/MemoryMatcher.h"
 #include "test/UnitTest/Test.h"
+#include "test/src/string/memory_utils/memory_check_utils.h"
 
 using LIBC_NAMESPACE::cpp::array;
 using LIBC_NAMESPACE::cpp::span;
diff --git a/libc/test/src/string/bzero_test.cpp b/libc/test/src/strings/bzero_test.cpp
similarity index 91%
rename from libc/test/src/string/bzero_test.cpp
rename to libc/test/src/strings/bzero_test.cpp
index a24043613bed7..4d4112f4be8ee 100644
--- a/libc/test/src/string/bzero_test.cpp
+++ b/libc/test/src/strings/bzero_test.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "memory_utils/memory_check_utils.h"
 #include "src/__support/macros/config.h"
-#include "src/string/bzero.h"
+#include "src/strings/bzero.h"
 #include "test/UnitTest/Test.h"
+#include "test/src/string/memory_utils/memory_check_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/test/src/string/index_test.cpp b/libc/test/src/strings/index_test.cpp
similarity index 100%
rename from libc/test/src/string/index_test.cpp
rename to libc/test/src/strings/index_test.cpp
diff --git a/libc/test/src/string/rindex_test.cpp b/libc/test/src/strings/rindex_test.cpp
similarity index 100%
rename from libc/test/src/string/rindex_test.cpp
rename to libc/test/src/strings/rindex_test.cpp
diff --git a/libc/test/src/string/strcasecmp_test.cpp b/libc/test/src/strings/strcasecmp_test.cpp
similarity index 97%
rename from libc/test/src/string/strcasecmp_test.cpp
rename to libc/test/src/strings/strcasecmp_test.cpp
index df7888168c69e..cd29c213a7324 100644
--- a/libc/test/src/string/strcasecmp_test.cpp
+++ b/libc/test/src/strings/strcasecmp_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/strcasecmp.h"
+#include "src/strings/strcasecmp.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStrCaseCmpTest, EmptyStringsShouldReturnZero) {
diff --git a/libc/test/src/string/strncasecmp_test.cpp b/libc/test/src/strings/strncasecmp_test.cpp
similarity index 97%
rename from libc/test/src/string/strncasecmp_test.cpp
rename to libc/test/src/strings/strncasecmp_test.cpp
index b4173c455de91..870574ed9507c 100644
--- a/libc/test/src/string/strncasecmp_test.cpp
+++ b/libc/test/src/strings/strncasecmp_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/string/strncasecmp.h"
+#include "src/strings/strncasecmp.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStrNCaseCmpTest,
diff --git a/libc/test/src/time/CMakeLists.txt b/libc/test/src/time/CMakeLists.txt
index bba01f063fed2..da3903f3e0e49 100644
--- a/libc/test/src/time/CMakeLists.txt
+++ b/libc/test/src/time/CMakeLists.txt
@@ -71,11 +71,21 @@ add_libc_test(
   SUITE
     libc_time_unittests
   SRCS
-  clock_gettime_test.cpp
+    clock_gettime_test.cpp
   DEPENDS
     libc.src.time.clock_gettime
 )
 
+add_libc_test(
+  clock_getres_test
+  SUITE
+    libc_time_unittests
+  SRCS
+    clock_getres_test.cpp
+  DEPENDS
+    libc.src.time.clock_getres
+)
+
 add_libc_unittest(
   difftime_test
   SUITE
@@ -157,11 +167,21 @@ add_libc_unittest(
   SRCS
     time_test.cpp
   DEPENDS
-    libc.include.time
     libc.src.time.time
+    libc.src.__support.time.clock_gettime
     libc.src.errno.errno
 )
 
+add_libc_test(
+  timespec_get_test
+  SUITE
+    libc_time_unittests
+  SRCS
+    timespec_get_test.cpp
+  DEPENDS
+    libc.src.time.timespec_get
+)
+
 add_libc_test(
   clock_test
   SUITE
diff --git a/libc/test/src/time/clock_getres_test.cpp b/libc/test/src/time/clock_getres_test.cpp
new file mode 100644
index 0000000000000..d8b3f01b37a38
--- /dev/null
+++ b/libc/test/src/time/clock_getres_test.cpp
@@ -0,0 +1,55 @@
+//===-- Unittests for clock_getres- ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/time_macros.h"
+#include "src/time/clock_getres.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+
+TEST(LlvmLibcClockGetRes, Invalid) {
+  timespec tp;
+  EXPECT_THAT(LIBC_NAMESPACE::clock_getres(-1, &tp), Fails(EINVAL));
+}
+
+TEST(LlvmLibcClockGetRes, NullSpec) {
+  EXPECT_THAT(LIBC_NAMESPACE::clock_getres(CLOCK_REALTIME, nullptr),
+              Succeeds());
+}
+
+TEST(LlvmLibcClockGetRes, Realtime) {
+  timespec tp;
+  EXPECT_THAT(LIBC_NAMESPACE::clock_getres(CLOCK_REALTIME, &tp), Succeeds());
+  EXPECT_GE(tp.tv_sec, static_cast<decltype(tp.tv_sec)>(0));
+  EXPECT_GE(tp.tv_nsec, static_cast<decltype(tp.tv_nsec)>(0));
+}
+
+TEST(LlvmLibcClockGetRes, Monotonic) {
+  timespec tp;
+  ASSERT_THAT(LIBC_NAMESPACE::clock_getres(CLOCK_MONOTONIC, &tp), Succeeds());
+  EXPECT_GE(tp.tv_sec, static_cast<decltype(tp.tv_sec)>(0));
+  EXPECT_GE(tp.tv_nsec, static_cast<decltype(tp.tv_nsec)>(0));
+}
+
+TEST(LlvmLibcClockGetRes, ProcessCpuTime) {
+  timespec tp;
+  ASSERT_THAT(LIBC_NAMESPACE::clock_getres(CLOCK_PROCESS_CPUTIME_ID, &tp),
+              Succeeds());
+  EXPECT_GE(tp.tv_sec, static_cast<decltype(tp.tv_sec)>(0));
+  EXPECT_GE(tp.tv_nsec, static_cast<decltype(tp.tv_nsec)>(0));
+}
+
+TEST(LlvmLibcClockGetRes, ThreadCpuTime) {
+  timespec tp;
+  ASSERT_THAT(LIBC_NAMESPACE::clock_getres(CLOCK_THREAD_CPUTIME_ID, &tp),
+              Succeeds());
+  EXPECT_GE(tp.tv_sec, static_cast<decltype(tp.tv_sec)>(0));
+  EXPECT_GE(tp.tv_nsec, static_cast<decltype(tp.tv_nsec)>(0));
+}
diff --git a/libc/test/src/time/time_test.cpp b/libc/test/src/time/time_test.cpp
index d3d4dc9a28515..7cdb4e834633e 100644
--- a/libc/test/src/time/time_test.cpp
+++ b/libc/test/src/time/time_test.cpp
@@ -9,8 +9,6 @@
 #include "src/time/time_func.h"
 #include "test/UnitTest/Test.h"
 
-#include <time.h>
-
 TEST(LlvmLibcTimeTest, SmokeTest) {
   time_t t1;
   time_t t2 = LIBC_NAMESPACE::time(&t1);
diff --git a/libc/test/src/time/timespec_get_test.cpp b/libc/test/src/time/timespec_get_test.cpp
new file mode 100644
index 0000000000000..327bfefe934c1
--- /dev/null
+++ b/libc/test/src/time/timespec_get_test.cpp
@@ -0,0 +1,46 @@
+//===-- Unittests for timespec_get ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/time_macros.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/macros/properties/architectures.h"
+#include "src/time/timespec_get.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcTimespecGet, Utc) {
+  timespec ts;
+  int result;
+  result = LIBC_NAMESPACE::timespec_get(&ts, TIME_UTC);
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+  ASSERT_EQ(result, 0);
+#else
+  ASSERT_EQ(result, TIME_UTC);
+  ASSERT_GT(ts.tv_sec, time_t(0));
+#endif
+}
+
+TEST(LlvmLibcTimespecGet, Monotonic) {
+  timespec ts1, ts2;
+  int result;
+  result = LIBC_NAMESPACE::timespec_get(&ts1, TIME_MONOTONIC);
+  ASSERT_EQ(result, TIME_MONOTONIC);
+  ASSERT_GT(ts1.tv_sec, time_t(0));
+  result = LIBC_NAMESPACE::timespec_get(&ts2, TIME_MONOTONIC);
+  ASSERT_EQ(result, TIME_MONOTONIC);
+  ASSERT_GE(ts2.tv_sec, ts1.tv_sec); // The monotonic time should increase.
+  if (ts2.tv_sec == ts1.tv_sec) {
+    ASSERT_GE(ts2.tv_nsec, ts1.tv_nsec);
+  }
+}
+
+TEST(LlvmLibcTimespecGet, Unknown) {
+  timespec ts;
+  int result;
+  result = LIBC_NAMESPACE::timespec_get(&ts, 0);
+  ASSERT_EQ(result, 0);
+}
diff --git a/libc/utils/HdrGen/CMakeLists.txt b/libc/utils/HdrGen/CMakeLists.txt
deleted file mode 100644
index 47f845b9f9a5b..0000000000000
--- a/libc/utils/HdrGen/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-include(TableGen)
-
-if (NOT LLVM_LINK_LLVM_DYLIB)
-  set(LLVM_LINK_COMPONENTS Support)
-endif()
-
-add_tablegen(libc-hdrgen LIBC
-  Command.h
-  Command.cpp
-  Generator.cpp
-  Generator.h
-  IncludeFileCommand.cpp
-  IncludeFileCommand.h
-  Main.cpp
-  PublicAPICommand.cpp
-  PublicAPICommand.h
-)
-
-target_include_directories(libc-hdrgen PRIVATE ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
-target_link_libraries(libc-hdrgen PRIVATE LibcTableGenUtil)
-
-add_subdirectory(PrototypeTestGen)
diff --git a/libc/utils/HdrGen/Command.h b/libc/utils/HdrGen/Command.h
deleted file mode 100644
index 42516798ffaec..0000000000000
--- a/libc/utils/HdrGen/Command.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- Base class for header generation commands ---------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
-#define LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/SourceMgr.h"
-
-#include <cstdlib>
-
-namespace llvm {
-
-class raw_ostream;
-class RecordKeeper;
-
-} // namespace llvm
-
-namespace llvm_libc {
-
-typedef llvm::SmallVector<llvm::StringRef, 4> ArgVector;
-
-class Command {
-public:
-  class ErrorReporter {
-    llvm::SMLoc Loc;
-    const llvm::SourceMgr &SrcMgr;
-
-  public:
-    ErrorReporter(llvm::SMLoc L, llvm::SourceMgr &SM) : Loc(L), SrcMgr(SM) {}
-
-    [[noreturn]] void printFatalError(llvm::Twine Msg) const {
-      SrcMgr.PrintMessage(Loc, llvm::SourceMgr::DK_Error, Msg);
-      std::exit(1);
-    }
-  };
-
-  virtual ~Command();
-
-  virtual void run(llvm::raw_ostream &OS, const ArgVector &Args,
-                   llvm::StringRef StdHeader, const llvm::RecordKeeper &Records,
-                   const ErrorReporter &Reporter) const = 0;
-};
-
-} // namespace llvm_libc
-
-#endif // LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
diff --git a/libc/utils/HdrGen/Generator.cpp b/libc/utils/HdrGen/Generator.cpp
deleted file mode 100644
index b4e1166b1a37b..0000000000000
--- a/libc/utils/HdrGen/Generator.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-//===-- Implementation of the main header generation class ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Generator.h"
-
-#include "IncludeFileCommand.h"
-#include "PublicAPICommand.h"
-#include "utils/LibcTableGenUtil/APIIndexer.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <cstdlib>
-#include <memory>
-
-static const char CommandPrefix[] = "%%";
-static const size_t CommandPrefixSize = llvm::StringRef(CommandPrefix).size();
-
-static const char CommentPrefix[] = "<!>";
-
-static const char ParamNamePrefix[] = "${";
-static const size_t ParamNamePrefixSize =
-    llvm::StringRef(ParamNamePrefix).size();
-static const char ParamNameSuffix[] = "}";
-static const size_t ParamNameSuffixSize =
-    llvm::StringRef(ParamNameSuffix).size();
-
-namespace llvm_libc {
-
-Command *Generator::getCommandHandler(llvm::StringRef CommandName) {
-  if (CommandName == IncludeFileCommand::Name) {
-    if (!IncludeFileCmd)
-      IncludeFileCmd = std::make_unique<IncludeFileCommand>();
-    return IncludeFileCmd.get();
-  } else if (CommandName == PublicAPICommand::Name) {
-    if (!PublicAPICmd)
-      PublicAPICmd = std::make_unique<PublicAPICommand>(EntrypointNameList);
-    return PublicAPICmd.get();
-  } else {
-    return nullptr;
-  }
-}
-
-void Generator::parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args) {
-  if (!ArgStr.contains(',') && ArgStr.trim(' ').trim('\t').size() == 0) {
-    // If it is just space between the parenthesis
-    return;
-  }
-
-  ArgStr.split(Args, ",");
-  for (llvm::StringRef &A : Args) {
-    A = A.trim(' ');
-    if (A.starts_with(ParamNamePrefix) && A.ends_with(ParamNameSuffix)) {
-      A = A.drop_front(ParamNamePrefixSize).drop_back(ParamNameSuffixSize);
-      A = ArgMap[std::string(A)];
-    }
-  }
-}
-
-void Generator::generate(llvm::raw_ostream &OS,
-                         const llvm::RecordKeeper &Records) {
-  auto DefFileBuffer = llvm::MemoryBuffer::getFile(HeaderDefFile);
-  if (!DefFileBuffer) {
-    llvm::errs() << "Unable to open " << HeaderDefFile << ".\n";
-    std::exit(1);
-  }
-  llvm::SourceMgr SrcMgr;
-  unsigned DefFileID = SrcMgr.AddNewSourceBuffer(
-      std::move(DefFileBuffer.get()), llvm::SMLoc::getFromPointer(nullptr));
-
-  llvm::StringRef Content = SrcMgr.getMemoryBuffer(DefFileID)->getBuffer();
-  while (true) {
-    std::pair<llvm::StringRef, llvm::StringRef> P = Content.split('\n');
-    Content = P.second;
-
-    llvm::StringRef Line = P.first.trim(' ');
-    if (Line.starts_with(CommandPrefix)) {
-      Line = Line.drop_front(CommandPrefixSize);
-
-      P = Line.split("(");
-      // It's possible that we have windows line endings, so strip off the extra
-      // CR.
-      P.second = P.second.trim();
-      if (P.second.empty() || P.second[P.second.size() - 1] != ')') {
-        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()),
-                            llvm::SourceMgr::DK_Error,
-                            "Command argument list should begin with '(' "
-                            "and end with ')'.");
-        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()),
-                            llvm::SourceMgr::DK_Error, P.second.data());
-        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()),
-                            llvm::SourceMgr::DK_Error,
-                            std::to_string(P.second.size()));
-        std::exit(1);
-      }
-      llvm::StringRef CommandName = P.first;
-      Command *Cmd = getCommandHandler(CommandName);
-      if (Cmd == nullptr) {
-        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(CommandName.data()),
-                            llvm::SourceMgr::DK_Error,
-                            "Unknown command '%%" + CommandName + "'.");
-        std::exit(1);
-      }
-
-      llvm::StringRef ArgStr = P.second.drop_back(1);
-      ArgVector Args;
-      parseCommandArgs(ArgStr, Args);
-
-      Command::ErrorReporter Reporter(
-          llvm::SMLoc::getFromPointer(CommandName.data()), SrcMgr);
-      Cmd->run(OS, Args, StdHeader, Records, Reporter);
-    } else if (!Line.starts_with(CommentPrefix)) {
-      // There is no comment or command on this line so we just write it as is.
-      OS << P.first << "\n";
-    }
-
-    if (P.second.empty())
-      break;
-  }
-}
-
-void Generator::generateDecls(llvm::raw_ostream &OS,
-                              const llvm::RecordKeeper &Records) {
-
-  OS << "//===-- C standard declarations for " << StdHeader << " "
-     << std::string(80 - (42 + StdHeader.size()), '-') << "===//\n"
-     << "//\n"
-     << "// Part of the LLVM Project, under the Apache License v2.0 with LLVM "
-        "Exceptions.\n"
-     << "// See https://llvm.org/LICENSE.txt for license information.\n"
-     << "// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
-     << "//\n"
-     << "//"
-        "===-------------------------------------------------------------------"
-        "---===//\n\n";
-
-  std::string HeaderGuard(StdHeader.size(), '\0');
-  llvm::transform(StdHeader, HeaderGuard.begin(), [](const char C) -> char {
-    return !isalnum(C) ? '_' : llvm::toUpper(C);
-  });
-  OS << "#ifndef __LLVM_LIBC_DECLARATIONS_" << HeaderGuard << "\n"
-     << "#define __LLVM_LIBC_DECLARATIONS_" << HeaderGuard << "\n\n";
-
-  OS << "#ifndef __LIBC_ATTRS\n"
-     << "#define __LIBC_ATTRS\n"
-     << "#endif\n\n";
-
-  OS << "#ifdef __cplusplus\n"
-     << "extern \"C\" {\n"
-     << "#endif\n\n";
-
-  APIIndexer G(StdHeader, Records);
-  for (auto &Name : EntrypointNameList) {
-    // Filter out functions not exported by this header.
-    if (G.FunctionSpecMap.find(Name) == G.FunctionSpecMap.end())
-      continue;
-
-    const llvm::Record *FunctionSpec = G.FunctionSpecMap[Name];
-    const llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return");
-    const llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType");
-
-    OS << G.getTypeAsString(ReturnType) << " " << Name << "(";
-
-    auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args");
-    for (size_t i = 0; i < ArgsList.size(); ++i) {
-      const llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType");
-      OS << G.getTypeAsString(ArgType);
-      if (i < ArgsList.size() - 1)
-        OS << ", ";
-    }
-
-    OS << ") __LIBC_ATTRS;\n\n";
-  }
-
-  // Make another pass over entrypoints to emit object declarations.
-  for (const auto &Name : EntrypointNameList) {
-    if (G.ObjectSpecMap.find(Name) == G.ObjectSpecMap.end())
-      continue;
-    const llvm::Record *ObjectSpec = G.ObjectSpecMap[Name];
-    auto Type = ObjectSpec->getValueAsString("Type");
-    OS << "extern " << Type << " " << Name << " __LIBC_ATTRS;\n";
-  }
-
-  // Emit a final newline if we emitted any object declarations.
-  if (llvm::any_of(EntrypointNameList, [&](const std::string &Name) {
-        return G.ObjectSpecMap.find(Name) != G.ObjectSpecMap.end();
-      }))
-    OS << "\n";
-
-  OS << "#ifdef __cplusplus\n"
-     << "}\n"
-     << "#endif\n\n";
-  OS << "#endif\n";
-}
-
-} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/Generator.h b/libc/utils/HdrGen/Generator.h
deleted file mode 100644
index 1c149d2115640..0000000000000
--- a/libc/utils/HdrGen/Generator.h
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- The main header generation class ------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
-#define LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
-
-#include "Command.h"
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-namespace llvm {
-
-class raw_ostream;
-class RecordKeeper;
-
-} // namespace llvm
-
-namespace llvm_libc {
-
-class Command;
-
-class Generator {
-  llvm::StringRef HeaderDefFile;
-  const std::vector<std::string> &EntrypointNameList;
-  llvm::StringRef StdHeader;
-  std::unordered_map<std::string, std::string> &ArgMap;
-
-  std::unique_ptr<Command> IncludeFileCmd;
-  std::unique_ptr<Command> PublicAPICmd;
-
-  Command *getCommandHandler(llvm::StringRef CommandName);
-
-  void parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args);
-
-  void printError(llvm::StringRef Msg);
-
-public:
-  Generator(const std::string &DefFile, const std::vector<std::string> &EN,
-            const std::string &Header,
-            std::unordered_map<std::string, std::string> &Map)
-      : HeaderDefFile(DefFile), EntrypointNameList(EN), StdHeader(Header),
-        ArgMap(Map) {}
-
-  void generate(llvm::raw_ostream &OS, const llvm::RecordKeeper &Records);
-  void generateDecls(llvm::raw_ostream &OS, const llvm::RecordKeeper &Records);
-};
-
-} // namespace llvm_libc
-
-#endif // LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
diff --git a/libc/utils/HdrGen/IncludeFileCommand.cpp b/libc/utils/HdrGen/IncludeFileCommand.cpp
deleted file mode 100644
index 43efe43585eb2..0000000000000
--- a/libc/utils/HdrGen/IncludeFileCommand.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-//===-- Implementation of IncludeFileCommand ------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "IncludeFileCommand.h"
-
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SourceMgr.h"
-
-#include <cstdlib>
-
-namespace llvm_libc {
-
-const char IncludeFileCommand::Name[] = "include_file";
-
-void IncludeFileCommand::run(llvm::raw_ostream &OS, const ArgVector &Args,
-                             llvm::StringRef StdHeader,
-                             const llvm::RecordKeeper &Records,
-                             const Command::ErrorReporter &Reporter) const {
-  if (Args.size() != 1) {
-    Reporter.printFatalError(
-        "%%include_file command takes exactly 1 argument.");
-  }
-
-  llvm::StringRef IncludeFile = Args[0];
-  auto Buffer = llvm::MemoryBuffer::getFileAsStream(IncludeFile);
-  if (!Buffer)
-    Reporter.printFatalError(llvm::StringRef("Unable to open ") + IncludeFile);
-
-  llvm::StringRef Content = Buffer.get()->getBuffer();
-
-  // If the included file has %%begin() command listed, then we want to write
-  // only the content after the begin command.
-  // TODO: The way the content is split below does not allow space within the
-  // the parentheses and, before and after the command. This probably is too
-  // strict and should be relaxed.
-  auto P = Content.split("\n%%begin()\n");
-  if (P.second.empty()) {
-    // There was no %%begin in the content.
-    OS << P.first;
-  } else {
-    OS << P.second;
-  }
-}
-
-} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/IncludeFileCommand.h b/libc/utils/HdrGen/IncludeFileCommand.h
deleted file mode 100644
index b3a6ff5462ded..0000000000000
--- a/libc/utils/HdrGen/IncludeFileCommand.h
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- Class which implements the %%include_file command -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
-#define LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
-
-#include "Command.h"
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-
-#include <string>
-
-namespace llvm_libc {
-
-class IncludeFileCommand : public Command {
-public:
-  static const char Name[];
-
-  void run(llvm::raw_ostream &OS, const ArgVector &Args,
-           llvm::StringRef StdHeader, const llvm::RecordKeeper &Records,
-           const Command::ErrorReporter &Reporter) const override;
-};
-
-} // namespace llvm_libc
-
-#endif // LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
diff --git a/libc/utils/HdrGen/Main.cpp b/libc/utils/HdrGen/Main.cpp
deleted file mode 100644
index f795e96e45c57..0000000000000
--- a/libc/utils/HdrGen/Main.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
-//===-- "main" function of libc-hdrgen ------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Generator.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/TableGen/Main.h"
-
-#include <string>
-#include <unordered_map>
-
-static llvm::cl::opt<std::string>
-    HeaderDefFile("def", llvm::cl::desc("Path to the .h.def file."),
-                  llvm::cl::value_desc("<filename>"), llvm::cl::Required);
-static llvm::cl::opt<std::string> StandardHeader(
-    "header",
-    llvm::cl::desc("The standard header file which is to be generated."),
-    llvm::cl::value_desc("<header file>"));
-static llvm::cl::list<std::string> EntrypointNamesOption(
-    "e", llvm::cl::value_desc("<list of entrypoints>"),
-    llvm::cl::desc(
-        "Each --e is one entrypoint (generated from entrypoints.txt)"),
-    llvm::cl::OneOrMore);
-static llvm::cl::list<std::string> ReplacementValues(
-    "args", llvm::cl::desc("Command separated <argument name>=<value> pairs."),
-    llvm::cl::value_desc("<name=value>[,name=value]"));
-static llvm::cl::opt<bool> ExportDecls(
-    "export-decls",
-    llvm::cl::desc("Output a new header containing only the entrypoints."));
-
-static void
-ParseArgValuePairs(std::unordered_map<std::string, std::string> &Map) {
-  for (std::string &R : ReplacementValues) {
-    auto Pair = llvm::StringRef(R).split('=');
-    Map[std::string(Pair.first)] = std::string(Pair.second);
-  }
-}
-
-static bool HeaderGeneratorMain(llvm::raw_ostream &OS,
-                                const llvm::RecordKeeper &Records) {
-  std::unordered_map<std::string, std::string> ArgMap;
-  ParseArgValuePairs(ArgMap);
-  llvm_libc::Generator G(HeaderDefFile, EntrypointNamesOption, StandardHeader,
-                         ArgMap);
-  if (ExportDecls)
-    G.generateDecls(OS, Records);
-  else
-    G.generate(OS, Records);
-
-  return false;
-}
-
-int main(int argc, char *argv[]) {
-  llvm::cl::ParseCommandLineOptions(argc, argv);
-  return TableGenMain(argv[0], &HeaderGeneratorMain);
-}
diff --git a/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt b/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt
deleted file mode 100644
index 9e25c21c6b359..0000000000000
--- a/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-add_tablegen(libc-prototype-testgen LLVM_LIBC
-  PrototypeTestGen.cpp
-)
-target_link_libraries(libc-prototype-testgen PRIVATE LibcTableGenUtil)
-target_include_directories(libc-prototype-testgen PRIVATE ${LIBC_SOURCE_DIR})
diff --git a/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp b/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
deleted file mode 100644
index 7acaf75c4c1c8..0000000000000
--- a/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-//===-- PrototypeTestGen.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "utils/LibcTableGenUtil/APIIndexer.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/TableGen/Main.h"
-#include "llvm/TableGen/Record.h"
-
-namespace {
-
-llvm::cl::list<std::string>
-    EntrypointNamesOption("e", llvm::cl::desc("<list of entrypoints>"),
-                          llvm::cl::OneOrMore);
-
-} // anonymous namespace
-
-bool TestGeneratorMain(llvm::raw_ostream &OS,
-                       const llvm::RecordKeeper &records) {
-  OS << "#include \"src/__support/CPP/type_traits.h\"\n";
-  llvm_libc::APIIndexer G(records);
-  std::unordered_set<std::string> headerFileSet;
-  for (const auto &entrypoint : EntrypointNamesOption) {
-    if (entrypoint == "errno")
-      continue;
-    auto match = G.FunctionToHeaderMap.find(entrypoint);
-    if (match == G.FunctionToHeaderMap.end()) {
-      auto objectMatch = G.ObjectToHeaderMap.find(entrypoint);
-      if (objectMatch != G.ObjectToHeaderMap.end()) {
-        headerFileSet.insert(objectMatch->second);
-        continue;
-      }
-
-      llvm::errs() << "ERROR: entrypoint '" << entrypoint
-                   << "' could not be found in spec in any public header\n";
-      return true;
-    }
-    headerFileSet.insert(match->second);
-  }
-  for (const auto &header : headerFileSet)
-    OS << "#include <" << header << ">\n";
-
-  OS << '\n';
-
-  OS << "extern \"C\" int main() {\n";
-  for (const auto &entrypoint : EntrypointNamesOption) {
-    if (entrypoint == "errno")
-      continue;
-    auto match = G.FunctionSpecMap.find(entrypoint);
-    if (match == G.FunctionSpecMap.end()) {
-      auto objectMatch = G.ObjectSpecMap.find(entrypoint);
-      if (objectMatch != G.ObjectSpecMap.end()) {
-        auto entrypointPtr = entrypoint + "_ptr";
-        llvm::Record *objectSpec = G.ObjectSpecMap[entrypoint];
-        auto objectType = objectSpec->getValueAsString("Type");
-        // We just make sure that the global object is present.
-        OS << "  " << objectType << " *" << entrypointPtr << " = &"
-           << entrypoint << ";\n";
-        OS << "  ++" << entrypointPtr << ";\n"; // To avoid unused var warning.
-        continue;
-      }
-      llvm::errs() << "ERROR: entrypoint '" << entrypoint
-                   << "' could not be found in spec in any public header\n";
-      return true;
-    }
-    llvm::Record *functionSpec = match->second;
-    llvm::Record *retValSpec = functionSpec->getValueAsDef("Return");
-    std::string returnType =
-        G.getTypeAsString(retValSpec->getValueAsDef("ReturnType"));
-    // _Noreturn is an indication for the compiler that a function
-    // doesn't return, and isn't a type understood by c++ templates.
-    if (llvm::StringRef(returnType).contains("_Noreturn"))
-      returnType = "void";
-
-    OS << "  static_assert(LIBC_NAMESPACE::cpp::is_same_v<" << returnType
-       << '(';
-    auto args = functionSpec->getValueAsListOfDefs("Args");
-    for (size_t i = 0, size = args.size(); i < size; ++i) {
-      llvm::Record *argType = args[i]->getValueAsDef("ArgType");
-      OS << G.getTypeAsString(argType);
-      if (i < size - 1)
-        OS << ", ";
-    }
-    OS << ") __NOEXCEPT, decltype(" << entrypoint << ")>, ";
-    OS << '"' << entrypoint
-       << " prototype in TableGen does not match public header" << '"';
-    OS << ");\n";
-  }
-
-  OS << '\n';
-  OS << "  return 0;\n";
-  OS << "}\n\n";
-
-  return false;
-}
-
-int main(int argc, char *argv[]) {
-  llvm::cl::ParseCommandLineOptions(argc, argv);
-  return TableGenMain(argv[0], TestGeneratorMain);
-}
diff --git a/libc/utils/HdrGen/PublicAPICommand.cpp b/libc/utils/HdrGen/PublicAPICommand.cpp
deleted file mode 100644
index 5c46c82a23853..0000000000000
--- a/libc/utils/HdrGen/PublicAPICommand.cpp
+++ /dev/null
@@ -1,331 +0,0 @@
-//===-- Implementation of PublicAPICommand --------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "PublicAPICommand.h"
-
-#include "utils/LibcTableGenUtil/APIIndexer.h"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/TableGen/Record.h"
-#include <algorithm>
-#include <vector>
-
-// Text blocks for macro definitions and type decls can be indented to
-// suit the surrounding tablegen listing. We need to dedent such blocks
-// before writing them out.
-static void dedentAndWrite(llvm::StringRef Text, llvm::raw_ostream &OS) {
-  llvm::SmallVector<llvm::StringRef, 10> Lines;
-  llvm::SplitString(Text, Lines, "\n");
-  size_t shortest_indent = 1024;
-  for (llvm::StringRef L : Lines) {
-    llvm::StringRef Indent = L.take_while([](char c) { return c == ' '; });
-    size_t IndentSize = Indent.size();
-    if (Indent.size() == L.size()) {
-      // Line is all spaces so no point noting the indent.
-      continue;
-    }
-    if (IndentSize < shortest_indent)
-      shortest_indent = IndentSize;
-  }
-  for (llvm::StringRef L : Lines) {
-    if (L.size() >= shortest_indent)
-      OS << L.drop_front(shortest_indent) << '\n';
-  }
-}
-
-static std::string getTypeHdrName(const std::string &Name) {
-  llvm::SmallVector<llvm::StringRef> Parts;
-  llvm::SplitString(llvm::StringRef(Name), Parts);
-  return llvm::join(Parts.begin(), Parts.end(), "_");
-}
-
-namespace llvm_libc {
-
-static bool isAsciiStart(char C) {
-  return (C >= 'A' && C <= 'Z') || (C >= 'a' && C <= 'z') || C == '_';
-}
-
-static bool isAsciiContinue(char C) {
-  return isAsciiStart(C) || (C >= '0' && C <= '9');
-}
-
-static bool isAsciiIdentifier(llvm::StringRef S) {
-  if (S.empty())
-    return false;
-  if (!isAsciiStart(S[0]))
-    return false;
-  for (char C : S.drop_front())
-    if (!isAsciiContinue(C))
-      return false;
-  return true;
-}
-
-static AttributeStyle getAttributeStyle(const llvm::Record *Instance) {
-  llvm::StringRef Style = Instance->getValueAsString("Style");
-  return llvm::StringSwitch<AttributeStyle>(Style)
-      .Case("cxx11", AttributeStyle::Cxx11)
-      .Case("gnu", AttributeStyle::Gnu)
-      .Case("declspec", AttributeStyle::Declspec)
-      .Default(AttributeStyle::Gnu);
-}
-
-static AttributeNamespace getAttributeNamespace(const llvm::Record *Instance) {
-  llvm::StringRef Namespace = Instance->getValueAsString("Namespace");
-  return llvm::StringSwitch<AttributeNamespace>(Namespace)
-      .Case("clang", AttributeNamespace::Clang)
-      .Case("gnu", AttributeNamespace::Gnu)
-      .Default(AttributeNamespace::None);
-}
-
-using AttributeMap = llvm::DenseMap<llvm::StringRef, const llvm::Record *>;
-
-template <class SpecMap, class FuncList>
-static AttributeMap collectAttributeMacros(const SpecMap &Spec,
-                                           const FuncList &Funcs) {
-  llvm::DenseMap<llvm::StringRef, const llvm::Record *> MacroAttr;
-  for (const auto &Name : Funcs) {
-    auto Iter = Spec.find(Name);
-    if (Iter == Spec.end())
-      continue;
-
-    const llvm::Record *FunctionSpec = Iter->second;
-    for (const llvm::Record *Attr :
-         FunctionSpec->getValueAsListOfDefs("Attributes"))
-      MacroAttr[Attr->getValueAsString("Macro")] = Attr;
-  }
-  return MacroAttr;
-}
-
-static void emitAttributeMacroDecls(const AttributeMap &MacroAttr,
-                                    llvm::raw_ostream &OS) {
-  for (auto &[Macro, Attr] : MacroAttr) {
-    std::vector<const llvm::Record *> Instances =
-        Attr->getValueAsListOfDefs("Instances");
-    llvm::SmallVector<std::pair<AttributeStyle, const llvm::Record *>> Styles;
-    std::transform(Instances.begin(), Instances.end(),
-                   std::back_inserter(Styles),
-                   [&](const llvm::Record *Instance)
-                       -> std::pair<AttributeStyle, const llvm::Record *> {
-                     auto Style = getAttributeStyle(Instance);
-                     return {Style, Instance};
-                   });
-    // 1. If __cplusplus is defined and cxx11 style is provided, define the
-    // macro using cxx11 version with the following priority:
-    //    1a. If there is no namespace (so the macro is supposed to be
-    //        compiler-independent), use this version first. This macro will be
-    //        tested via __has_cpp_attribute.
-    //    1b. If the attribute is a clang attribute, check for __clang__.
-    //    1c. If the attribute is a gnu attribute, check for __GNUC__.
-    // 2. Otherwise, if __GNUC__ is defined and gnu style is provided,
-    //    define the macro using gnu version;
-    // 3. Otherwise, if _MSC_VER is defined and __declspec is provided, define
-    //    the macro using __declspec version;
-    // 4. Fallback to empty macro.
-    std::sort(Styles.begin(), Styles.end(), [&](auto &a, auto &b) {
-      if (a.first == AttributeStyle::Cxx11 && b.first == AttributeStyle::Cxx11)
-        return getAttributeNamespace(a.second) <
-               getAttributeNamespace(b.second);
-      return a.first < b.first;
-    });
-    for (auto &[Style, Instance] : Styles) {
-      llvm::StringRef Attr = Instance->getValueAsString("Attr");
-      if (Style == AttributeStyle::Cxx11) {
-        OS << "#if !defined(" << Macro << ") && defined(__cplusplus)";
-        AttributeNamespace Namespace = getAttributeNamespace(Instance);
-        if (Namespace == AttributeNamespace::Clang)
-          OS << " && defined(__clang__)\n";
-        else if (Namespace == AttributeNamespace::Gnu)
-          OS << " && defined(__GNUC__)\n";
-        else
-          OS << '\n';
-        if (isAsciiIdentifier(Attr) && Namespace != AttributeNamespace::None)
-          OS << "#if __has_attribute(" << Attr << ")\n";
-        else
-          OS << "#if __has_cpp_attribute(" << Attr << ")\n";
-        OS << "#define " << Macro << " [[";
-        if (Namespace == AttributeNamespace::Clang)
-          OS << "clang::";
-        else if (Namespace == AttributeNamespace::Gnu)
-          OS << "gnu::";
-        OS << Attr << "]]\n";
-        if (isAsciiIdentifier(Attr))
-          OS << "#endif\n";
-        OS << "#endif\n";
-      }
-      if (Style == AttributeStyle::Gnu) {
-        OS << "#if !defined(" << Macro << ") && defined(__GNUC__)\n";
-        if (isAsciiIdentifier(Attr))
-          OS << "#if __has_attribute(" << Attr << ")\n";
-        OS << "#define " << Macro << " __attribute__((";
-        OS << Attr << "))\n";
-        if (isAsciiIdentifier(Attr))
-          OS << "#endif\n";
-        OS << "#endif\n";
-      }
-      if (Style == AttributeStyle::Declspec) {
-        OS << "#if !defined(" << Macro << ") && defined(_MSC_VER)\n";
-        OS << "#define " << Macro << " __declspec(";
-        OS << Attr << ")\n";
-        OS << "#endif\n";
-      }
-    }
-    OS << "#if !defined(" << Macro << ")\n";
-    OS << "#define " << Macro << '\n';
-    OS << "#endif\n";
-  }
-
-  if (!MacroAttr.empty())
-    OS << '\n';
-}
-
-static void emitAttributeMacroForFunction(const llvm::Record *FunctionSpec,
-                                          llvm::raw_ostream &OS) {
-  std::vector<const llvm::Record *> Attributes =
-      FunctionSpec->getValueAsListOfDefs("Attributes");
-  llvm::interleave(
-      Attributes.begin(), Attributes.end(),
-      [&](const llvm::Record *Attr) { OS << Attr->getValueAsString("Macro"); },
-      [&]() { OS << ' '; });
-  if (!Attributes.empty())
-    OS << ' ';
-}
-
-static void emitUndefsForAttributeMacros(const AttributeMap &MacroAttr,
-                                         llvm::raw_ostream &OS) {
-  if (!MacroAttr.empty())
-    OS << '\n';
-  for (auto &[Macro, Attr] : MacroAttr)
-    OS << "#undef " << Macro << '\n';
-}
-
-static void writeAPIFromIndex(APIIndexer &G,
-                              std::vector<std::string> EntrypointNameList,
-                              llvm::raw_ostream &OS) {
-  for (auto &Pair : G.MacroDefsMap) {
-    const std::string &Name = Pair.first;
-    if (!G.MacroSpecMap.count(Name))
-      llvm::PrintFatalError(Name + " not found in any standard spec.\n");
-
-    const llvm::Record *MacroDef = Pair.second;
-    dedentAndWrite(MacroDef->getValueAsString("Defn"), OS);
-
-    OS << '\n';
-  }
-
-  for (auto &TypeName : G.RequiredTypes) {
-    if (!G.TypeSpecMap.count(TypeName))
-      llvm::PrintFatalError(TypeName + " not found in any standard spec.\n");
-    OS << "#include <llvm-libc-types/" << getTypeHdrName(TypeName) << ".h>\n";
-  }
-  OS << '\n';
-
-  if (G.Enumerations.size() != 0)
-    OS << "enum {" << '\n';
-  for (const auto &Name : G.Enumerations) {
-    if (!G.EnumerationSpecMap.count(Name))
-      llvm::PrintFatalError(
-          Name + " is not listed as an enumeration in any standard spec.\n");
-
-    const llvm::Record *EnumerationSpec = G.EnumerationSpecMap[Name];
-    OS << "  " << EnumerationSpec->getValueAsString("Name");
-    auto Value = EnumerationSpec->getValueAsString("Value");
-    if (Value == "__default__") {
-      OS << ",\n";
-    } else {
-      OS << " = " << Value << ",\n";
-    }
-  }
-  if (G.Enumerations.size() != 0)
-    OS << "};\n\n";
-
-  // Collect and declare macros for attributes
-  AttributeMap MacroAttr =
-      collectAttributeMacros(G.FunctionSpecMap, EntrypointNameList);
-  emitAttributeMacroDecls(MacroAttr, OS);
-
-  OS << "__BEGIN_C_DECLS\n\n";
-  for (auto &Name : EntrypointNameList) {
-    auto Iter = G.FunctionSpecMap.find(Name);
-
-    // Functions that aren't in this header file are skipped as
-    // opposed to erroring out because the list of functions being
-    // iterated over is the complete list of functions with
-    // entrypoints. Thus this is filtering out the functions that
-    // don't go to this header file, whereas the other, similar
-    // conditionals above are more of a sanity check.
-    if (Iter == G.FunctionSpecMap.end())
-      continue;
-
-    const llvm::Record *FunctionSpec = Iter->second;
-    const llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return");
-    const llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType");
-
-    // TODO: https://github.com/llvm/llvm-project/issues/81208
-    //   Ideally, we should group functions based on their guarding macros.
-    bool Guarded =
-        (FunctionSpec->getType()->getAsString() == "GuardedFunctionSpec");
-
-    if (Guarded)
-      OS << "#ifdef " << FunctionSpec->getValueAsString("Guard") << "\n";
-
-    // Emit attribute macros for the function. Space is automatically added.
-    emitAttributeMacroForFunction(FunctionSpec, OS);
-    OS << G.getTypeAsString(ReturnType) << " " << Name << "(";
-
-    auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args");
-    for (size_t i = 0; i < ArgsList.size(); ++i) {
-      const llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType");
-      OS << G.getTypeAsString(ArgType);
-      if (i < ArgsList.size() - 1)
-        OS << ", ";
-    }
-
-    OS << ") __NOEXCEPT;\n";
-
-    if (Guarded)
-      OS << "#endif // " << FunctionSpec->getValueAsString("Guard") << "\n";
-
-    OS << "\n";
-  }
-
-  // Make another pass over entrypoints to emit object declarations.
-  for (const auto &Name : EntrypointNameList) {
-    auto Iter = G.ObjectSpecMap.find(Name);
-    if (Iter == G.ObjectSpecMap.end())
-      continue;
-    const llvm::Record *ObjectSpec = Iter->second;
-    auto Type = ObjectSpec->getValueAsString("Type");
-    OS << "extern " << Type << " " << Name << ";\n";
-  }
-  OS << "__END_C_DECLS\n";
-
-  // Undef file-level attribute macros.
-  emitUndefsForAttributeMacros(MacroAttr, OS);
-}
-
-void writePublicAPI(llvm::raw_ostream &OS, const llvm::RecordKeeper &Records) {}
-
-const char PublicAPICommand::Name[] = "public_api";
-
-void PublicAPICommand::run(llvm::raw_ostream &OS, const ArgVector &Args,
-                           llvm::StringRef StdHeader,
-                           const llvm::RecordKeeper &Records,
-                           const Command::ErrorReporter &Reporter) const {
-  if (Args.size() != 0)
-    Reporter.printFatalError("public_api command does not take any arguments.");
-
-  APIIndexer G(StdHeader, Records);
-  writeAPIFromIndex(G, EntrypointNameList, OS);
-}
-
-} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/PublicAPICommand.h b/libc/utils/HdrGen/PublicAPICommand.h
deleted file mode 100644
index 49078f4857f90..0000000000000
--- a/libc/utils/HdrGen/PublicAPICommand.h
+++ /dev/null
@@ -1,48 +0,0 @@
-//===-- Implementation of PublicAPICommand ----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_HDRGEN_PUBLICAPICOMMAND_H
-#define LLVM_LIBC_UTILS_HDRGEN_PUBLICAPICOMMAND_H
-
-#include "Command.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Record.h"
-
-namespace llvm {
-
-class raw_ostream;
-class Record;
-class RecordKeeper;
-
-} // namespace llvm
-
-namespace llvm_libc {
-
-enum class AttributeStyle { Cxx11 = 0, Gnu = 1, Declspec = 2 };
-enum class AttributeNamespace { None = 0, Clang = 1, Gnu = 2 };
-
-class PublicAPICommand : public Command {
-private:
-  const std::vector<std::string> &EntrypointNameList;
-
-public:
-  static const char Name[];
-
-  PublicAPICommand(const std::vector<std::string> &EntrypointNames)
-      : EntrypointNameList(EntrypointNames) {}
-
-  void run(llvm::raw_ostream &OS, const ArgVector &Args,
-           llvm::StringRef StdHeader, const llvm::RecordKeeper &Records,
-           const Command::ErrorReporter &Reporter) const override;
-};
-
-} // namespace llvm_libc
-
-#endif // LLVM_LIBC_UTILS_HDRGEN_PUBLICAPICOMMAND_H
diff --git a/libc/utils/HdrGen/README.md b/libc/utils/HdrGen/README.md
deleted file mode 100644
index a61cf3bacbb0e..0000000000000
--- a/libc/utils/HdrGen/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# The LLVM libc header generation system
-
-LLVM libc uses a header generation scheme to generate public as well as internal
-header files. This directory contains the implementation of the header generator
-which drives this header generation scheme.
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
deleted file mode 100644
index cf66d0a7aef6e..0000000000000
--- a/libc/utils/LibcTableGenUtil/APIIndexer.cpp
+++ /dev/null
@@ -1,173 +0,0 @@
-//===-- Implementation of APIIndexer class --------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "APIIndexer.h"
-
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Record.h"
-
-namespace llvm_libc {
-
-static const char NamedTypeClassName[] = "NamedType";
-static const char PtrTypeClassName[] = "PtrType";
-static const char RestrictedPtrTypeClassName[] = "RestrictedPtrType";
-static const char ConstTypeClassName[] = "ConstType";
-static const char StructTypeClassName[] = "Struct";
-
-static const char StandardSpecClassName[] = "StandardSpec";
-static const char PublicAPIClassName[] = "PublicAPI";
-
-static bool isa(const llvm::Record *Def, const llvm::Record *TypeClass) {
-  const llvm::RecordRecTy *RecordType = Def->getType();
-  llvm::ArrayRef<const llvm::Record *> Classes = RecordType->getClasses();
-  // We want exact types. That is, we don't want the classes listed in
-  // spec.td to be subclassed. Hence, we do not want the record |Def|
-  // to be of more than one class type..
-  if (Classes.size() != 1)
-    return false;
-  return Classes[0] == TypeClass;
-}
-
-bool APIIndexer::isaNamedType(const llvm::Record *Def) {
-  return isa(Def, NamedTypeClass);
-}
-
-bool APIIndexer::isaStructType(const llvm::Record *Def) {
-  return isa(Def, StructClass);
-}
-
-bool APIIndexer::isaPtrType(const llvm::Record *Def) {
-  return isa(Def, PtrTypeClass);
-}
-
-bool APIIndexer::isaConstType(const llvm::Record *Def) {
-  return isa(Def, ConstTypeClass);
-}
-
-bool APIIndexer::isaRestrictedPtrType(const llvm::Record *Def) {
-  return isa(Def, RestrictedPtrTypeClass);
-}
-
-bool APIIndexer::isaStandardSpec(const llvm::Record *Def) {
-  return isa(Def, StandardSpecClass);
-}
-
-bool APIIndexer::isaPublicAPI(const llvm::Record *Def) {
-  return isa(Def, PublicAPIClass);
-}
-
-std::string APIIndexer::getTypeAsString(const llvm::Record *TypeRecord) {
-  if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) {
-    return std::string(TypeRecord->getValueAsString("Name"));
-  } else if (isaPtrType(TypeRecord)) {
-    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *";
-  } else if (isaConstType(TypeRecord)) {
-    return std::string("const ") +
-           getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType"));
-  } else if (isaRestrictedPtrType(TypeRecord)) {
-    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) +
-           " *__restrict";
-  } else {
-    llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n");
-  }
-}
-
-void APIIndexer::indexStandardSpecDef(const llvm::Record *StandardSpec) {
-  auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
-  for (const llvm::Record *HeaderSpec : HeaderSpecList) {
-    llvm::StringRef Header = HeaderSpec->getValueAsString("Name");
-    if (!StdHeader.has_value() || Header == StdHeader) {
-      PublicHeaders.emplace(Header);
-      auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
-      // TODO: Trigger a fatal error on duplicate specs.
-      for (const llvm::Record *MacroSpec : MacroSpecList)
-        MacroSpecMap[std::string(MacroSpec->getValueAsString("Name"))] =
-            MacroSpec;
-
-      auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
-      for (const llvm::Record *TypeSpec : TypeSpecList)
-        TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] = TypeSpec;
-
-      auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
-      for (const llvm::Record *FunctionSpec : FunctionSpecList) {
-        auto FunctionName = std::string(FunctionSpec->getValueAsString("Name"));
-        FunctionSpecMap[FunctionName] = FunctionSpec;
-        FunctionToHeaderMap[FunctionName] = std::string(Header);
-      }
-
-      auto EnumerationSpecList =
-          HeaderSpec->getValueAsListOfDefs("Enumerations");
-      for (const llvm::Record *EnumerationSpec : EnumerationSpecList) {
-        EnumerationSpecMap[std::string(
-            EnumerationSpec->getValueAsString("Name"))] = EnumerationSpec;
-      }
-
-      auto ObjectSpecList = HeaderSpec->getValueAsListOfDefs("Objects");
-      for (const llvm::Record *ObjectSpec : ObjectSpecList) {
-        auto ObjectName = std::string(ObjectSpec->getValueAsString("Name"));
-        ObjectSpecMap[ObjectName] = ObjectSpec;
-        ObjectToHeaderMap[ObjectName] = std::string(Header);
-      }
-    }
-  }
-}
-
-void APIIndexer::indexPublicAPIDef(const llvm::Record *PublicAPI) {
-  // While indexing the public API, we do not check if any of the entities
-  // requested is from an included standard. Such a check is done while
-  // generating the API.
-  auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
-  for (const llvm::Record *MacroDef : MacroDefList)
-    MacroDefsMap[std::string(MacroDef->getValueAsString("Name"))] = MacroDef;
-
-  auto TypeList = PublicAPI->getValueAsListOfStrings("Types");
-  for (llvm::StringRef TypeName : TypeList)
-    RequiredTypes.insert(std::string(TypeName));
-
-  auto StructList = PublicAPI->getValueAsListOfStrings("Structs");
-  for (llvm::StringRef StructName : StructList)
-    Structs.insert(std::string(StructName));
-
-  auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions");
-  for (llvm::StringRef FunctionName : FunctionList)
-    Functions.insert(std::string(FunctionName));
-
-  auto EnumerationList = PublicAPI->getValueAsListOfStrings("Enumerations");
-  for (llvm::StringRef EnumerationName : EnumerationList)
-    Enumerations.insert(std::string(EnumerationName));
-
-  auto ObjectList = PublicAPI->getValueAsListOfStrings("Objects");
-  for (llvm::StringRef ObjectName : ObjectList)
-    Objects.insert(std::string(ObjectName));
-}
-
-void APIIndexer::index(const llvm::RecordKeeper &Records) {
-  NamedTypeClass = Records.getClass(NamedTypeClassName);
-  PtrTypeClass = Records.getClass(PtrTypeClassName);
-  RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName);
-  StructClass = Records.getClass(StructTypeClassName);
-  ConstTypeClass = Records.getClass(ConstTypeClassName);
-  StandardSpecClass = Records.getClass(StandardSpecClassName);
-  PublicAPIClass = Records.getClass(PublicAPIClassName);
-
-  const auto &DefsMap = Records.getDefs();
-  for (auto &Pair : DefsMap) {
-    const llvm::Record *Def = Pair.second.get();
-    if (isaStandardSpec(Def))
-      indexStandardSpecDef(Def);
-    if (isaPublicAPI(Def)) {
-      if (!StdHeader.has_value() ||
-          Def->getValueAsString("HeaderName") == StdHeader)
-        indexPublicAPIDef(Def);
-    }
-  }
-}
-
-} // namespace llvm_libc
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.h b/libc/utils/LibcTableGenUtil/APIIndexer.h
deleted file mode 100644
index b8bca15ba131c..0000000000000
--- a/libc/utils/LibcTableGenUtil/APIIndexer.h
+++ /dev/null
@@ -1,86 +0,0 @@
-//===-- A class to index libc API listed in tablegen files ------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H
-#define LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/TableGen/Record.h"
-
-#include <optional>
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-
-namespace llvm_libc {
-
-class APIIndexer {
-private:
-  std::optional<llvm::StringRef> StdHeader;
-
-  // TableGen classes in spec.td.
-  const llvm::Record *NamedTypeClass;
-  const llvm::Record *PtrTypeClass;
-  const llvm::Record *RestrictedPtrTypeClass;
-  const llvm::Record *ConstTypeClass;
-  const llvm::Record *StructClass;
-  const llvm::Record *StandardSpecClass;
-  const llvm::Record *PublicAPIClass;
-
-  bool isaNamedType(const llvm::Record *Def);
-  bool isaStructType(const llvm::Record *Def);
-  bool isaPtrType(const llvm::Record *Def);
-  bool isaConstType(const llvm::Record *Def);
-  bool isaRestrictedPtrType(const llvm::Record *Def);
-  bool isaStandardSpec(const llvm::Record *Def);
-  bool isaPublicAPI(const llvm::Record *Def);
-
-  void indexStandardSpecDef(const llvm::Record *StandardSpec);
-  void indexPublicAPIDef(const llvm::Record *PublicAPI);
-  void index(const llvm::RecordKeeper &Records);
-
-public:
-  using NameToRecordMapping =
-      std::unordered_map<std::string, const llvm::Record *>;
-  using NameSet = std::unordered_set<std::string>;
-
-  // This indexes all headers, not just a specified one.
-  explicit APIIndexer(const llvm::RecordKeeper &Records)
-      : StdHeader(std::nullopt) {
-    index(Records);
-  }
-
-  APIIndexer(llvm::StringRef Header, const llvm::RecordKeeper &Records)
-      : StdHeader(Header) {
-    index(Records);
-  }
-
-  // Mapping from names to records defining them.
-  NameToRecordMapping MacroSpecMap;
-  NameToRecordMapping TypeSpecMap;
-  NameToRecordMapping EnumerationSpecMap;
-  NameToRecordMapping FunctionSpecMap;
-  NameToRecordMapping MacroDefsMap;
-  NameToRecordMapping ObjectSpecMap;
-
-  std::unordered_map<std::string, std::string> FunctionToHeaderMap;
-  std::unordered_map<std::string, std::string> ObjectToHeaderMap;
-
-  NameSet RequiredTypes;
-  NameSet Structs;
-  NameSet Enumerations;
-  NameSet Functions;
-  NameSet Objects;
-  NameSet PublicHeaders;
-
-  std::string getTypeAsString(const llvm::Record *TypeRecord);
-};
-
-} // namespace llvm_libc
-
-#endif // LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H
diff --git a/libc/utils/LibcTableGenUtil/CMakeLists.txt b/libc/utils/LibcTableGenUtil/CMakeLists.txt
deleted file mode 100644
index 9421383394a35..0000000000000
--- a/libc/utils/LibcTableGenUtil/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-if (NOT LLVM_LINK_LLVM_DYLIB)
-  set(flags "DISABLE_LLVM_LINK_LLVM_DYLIB;LINK_COMPONENTS;Support;TableGen")
-else()
-  set(flags "LINK_COMPONENTS;TableGen")
-endif()
-add_llvm_library(
-  LibcTableGenUtil
-  APIIndexer.cpp
-  APIIndexer.h
-  ${flags}
-)
-target_include_directories(LibcTableGenUtil PUBLIC ${LIBC_SOURCE_DIR})
-target_include_directories(LibcTableGenUtil PRIVATE ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
diff --git a/libc/utils/docgen/assert.json b/libc/utils/docgen/assert.json
new file mode 100644
index 0000000000000..28ec12028ef67
--- /dev/null
+++ b/libc/utils/docgen/assert.json
@@ -0,0 +1,10 @@
+{
+  "macros": {
+    "__STDC_VERSION_ASSERT_H__": {
+      "c-definition": "7.2.1"
+    },
+    "assert": {
+      "c-definition": "7.2.1"
+    }
+  }
+}
diff --git a/libc/utils/docgen/docgen.py b/libc/utils/docgen/docgen.py
index 25e22d4d58778..aa30a6e51ef87 100755
--- a/libc/utils/docgen/docgen.py
+++ b/libc/utils/docgen/docgen.py
@@ -92,7 +92,7 @@ def load_api(header: Header) -> Dict:
     return json.loads(api)
 
 
-def print_tbl_dir():
+def print_tbl_dir(name):
     print(
         f"""
 .. list-table::
@@ -100,10 +100,10 @@ def print_tbl_dir():
   :align: center
   :header-rows: 1
 
-  * - Function
+  * - {name}
     - Implemented
     - C23 Standard Section
-    - POSIX.1-2017 Standard Section"""
+    - POSIX.1-2024 Standard Section"""
     )
 
 
@@ -112,7 +112,7 @@ def print_functions_rst(header: Header, functions: Dict):
     print(tbl_hdr)
     print("=" * len(tbl_hdr))
 
-    print_tbl_dir()
+    print_tbl_dir("Function")
 
     for name in sorted(functions.keys()):
         print(f"  * - {name}")
@@ -138,7 +138,7 @@ def print_macros_rst(header: Header, macros: Dict):
     print(tbl_hdr)
     print("=" * len(tbl_hdr))
 
-    print_tbl_dir()
+    print_tbl_dir("Macro")
 
     for name in sorted(macros.keys()):
         print(f"  * - {name}")
@@ -161,7 +161,7 @@ def print_macros_rst(header: Header, macros: Dict):
 
 
 def print_impl_status_rst(header: Header, api: Dict):
-    print(".. include:: check.rst\n")
+    print(".. include:: ../check.rst\n")
 
     print("=" * len(header.name))
     print(header.name)
diff --git a/libc/utils/docgen/errno.json b/libc/utils/docgen/errno.json
new file mode 100644
index 0000000000000..aface8e42b495
--- /dev/null
+++ b/libc/utils/docgen/errno.json
@@ -0,0 +1,16 @@
+{
+  "macros": {
+    "EDOM": {
+      "c-definition": "7.5"
+    },
+    "EILSEQ": {
+      "c-definition": "7.5"
+    },
+    "ERANGE": {
+      "c-definition": "7.5"
+    },
+    "errno": {
+      "c-definition": "7.5"
+    }
+  }
+}
diff --git a/libc/utils/docgen/float.json b/libc/utils/docgen/float.json
new file mode 100644
index 0000000000000..a906cbf4fa754
--- /dev/null
+++ b/libc/utils/docgen/float.json
@@ -0,0 +1,163 @@
+{
+  "macros": {
+    "__STDC_VERSION_FLOAT_H__": {
+      "c-definition": "7.7"
+    },
+    "FLT_EVAL_METHOD" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_ROUNDS" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_EVAL_METHOD" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_HAS_SUBNORM" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_HAS_SUBNORM" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_HAS_SUBNORM" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_RADIX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MANT_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MANT_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MANT_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_DECIMAL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_DECIMAL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_DECIMAL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DECIMAL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_IS_IEC_60559" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_IS_IEC_60559" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_IS_IEC_60559" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_DIG" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MIN_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MIN_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MIN_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MIN_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MIN_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MIN_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MAX_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MAX_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MAX_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MAX_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MAX_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MAX_10_EXP" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_NORM_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_NORM_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_NORM_MAX" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_EPSILON" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_EPSILON" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_EPSILON" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_SNAN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_SNAN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_SNAN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "FLT_TRUE_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "DBL_TRUE_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "LDBL_TRUE_MIN" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "INFINITY" : {
+      "c-definition": "5.3.5.3.3"
+    },
+    "NAN" : {
+      "c-definition": "5.3.5.3.3"
+    }
+  }
+}
diff --git a/libc/utils/docgen/inttypes.json b/libc/utils/docgen/inttypes.json
new file mode 100644
index 0000000000000..001f15f27e638
--- /dev/null
+++ b/libc/utils/docgen/inttypes.json
@@ -0,0 +1,22 @@
+{
+  "functions": {
+    "imaxabs": {
+      "c-definition": "7.8.2.1"
+    },
+    "imaxdiv": {
+      "c-definition": "7.8.2.2"
+    },
+    "strtoimax": {
+      "c-definition": "7.8.2.3"
+    },
+    "strtoumax": {
+      "c-definition": "7.8.2.3"
+    },
+    "wcstoimax": {
+      "c-definition": "7.8.2.4"
+    },
+    "wcstoumax": {
+      "c-definition": "7.8.2.4"
+    }
+  }
+}
diff --git a/libc/utils/docgen/locale.json b/libc/utils/docgen/locale.json
new file mode 100644
index 0000000000000..89329f9aae5b0
--- /dev/null
+++ b/libc/utils/docgen/locale.json
@@ -0,0 +1,30 @@
+{
+  "macros": {
+    "LC_ALL": {
+      "c-definition": "7.11"
+    },
+    "LC_COLLATE": {
+      "c-definition": "7.11"
+    },
+    "LC_CTYPE": {
+      "c-definition": "7.11"
+    },
+    "LC_MONETARY": {
+      "c-definition": "7.11"
+    },
+    "LC_NUMERIC": {
+      "c-definition": "7.11"
+    },
+    "LC_TIME": {
+      "c-definition": "7.11"
+    }
+  },
+  "functions": {
+    "setlocale": {
+      "c-definition": "7.11.1.1"
+    },
+    "localeconv": {
+      "c-definition": "7.11.2.1"
+    }
+  }
+}
diff --git a/libc/utils/docgen/setjmp.json b/libc/utils/docgen/setjmp.json
index 38d4af568926a..0b9a4e65da4f6 100644
--- a/libc/utils/docgen/setjmp.json
+++ b/libc/utils/docgen/setjmp.json
@@ -1,15 +1,15 @@
 {
   "macros": {
     "__STDC_VERSION_SETJMP_H__": {
-      "defined": "7.13.2"
-    },
-    "setjmp": {
-      "defined": "7.13.1.1"
+      "c-definition": "7.13.2"
     }
   },
   "functions": {
+    "setjmp": {
+      "c-definition": "7.13.1.1"
+    },
     "longjmp": {
-      "defined": "7.13.2.1"
+      "c-definition": "7.13.2.1"
     }
   }
 }
diff --git a/libc/utils/docgen/signal.json b/libc/utils/docgen/signal.json
index 337b0c19717b7..ec83144da8576 100644
--- a/libc/utils/docgen/signal.json
+++ b/libc/utils/docgen/signal.json
@@ -2,151 +2,151 @@
   "macros": {
     "SIG_DFL": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIG_ERR": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIG_HOLD": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIG_IGN": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGRTMIN": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGRTMAX": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGABRT": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGALRM": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGBUS": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGCHLD": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGCONT": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGFPE": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGHUP": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGILL": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGINT": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGKILL": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGPIPE": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGPIPE": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGQUIT": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGSEGV": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGSTOP": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGTERM": {
       "c-definition": "7.14.3",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGTSTP": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGTTIN": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGTTOU": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGUSR1": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGUSR2": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGPOLL": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGPROF": {
-    "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+    "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGSYS": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGTRAP": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGURG": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGVTALRM": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGXCPU": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     },
     "SIGXFSZ": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html"
     }
   },
   "functions": {
     "signal": {
       "c-definition": "7.14.1.1",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/signal.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/signal.html"
     },
     "raise": {
       "c-definition": "7.14.2.1",
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/raise.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/raise.html"
     },
     "kill": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/kill.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/kill.html"
     },
     "sigaction": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaction.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaction.html"
     },
     "sigaddset": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaddset.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaddset.html"
     },
     "sigaltstack": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigaltstack.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigaltstack.html"
     },
     "sigdelset": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigdelset.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigdelset.html"
     },
     "sigemptyset": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigemptyset.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigemptyset.html"
     },
     "sigfillset": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigfillset.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigfillset.html"
     },
     "sigprocmask": {
-      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9699919799/functions/sigprocmask.html"
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/sigprocmask.html"
     }
   }
 }
diff --git a/libc/utils/docgen/stdlib.json b/libc/utils/docgen/stdlib.json
new file mode 100644
index 0000000000000..0ca508110c0f1
--- /dev/null
+++ b/libc/utils/docgen/stdlib.json
@@ -0,0 +1,174 @@
+{
+  "macros": {
+    "__STDC_VERSION_STDLIB_H__": {
+      "c-definition": "7.24"
+    },
+    "EXIT_FAILURE": {
+      "c-definition": "7.24"
+    },
+    "EXIT_SUCCESS": {
+      "c-definition": "7.24"
+    },
+    "RAND_MAX": {
+      "c-definition": "7.24"
+    },
+    "MB_CUR_MAX": {
+      "c-definition": "7.24"
+    }
+  },
+  "functions": {
+    "atof": {
+      "c-definition": "7.24.1.1"
+    },
+    "atoi": {
+      "c-definition": "7.24.1.2"
+    },
+    "atol": {
+      "c-definition": "7.24.1.2"
+    },
+    "atoll": {
+      "c-definition": "7.24.1.2"
+    },
+    "strfromd": {
+      "c-definition": "7.24.1.3"
+    },
+    "strfromf": {
+      "c-definition": "7.24.1.3"
+    },
+    "strfroml": {
+      "c-definition": "7.24.1.3"
+    },
+    "strfromd32": {
+      "c-definition": "7.24.1.4"
+    },
+    "strfromd64": {
+      "c-definition": "7.24.1.4"
+    },
+    "strfromd128": {
+      "c-definition": "7.24.1.4"
+    },
+    "strtod": {
+      "c-definition": "7.24.1.5"
+    },
+    "strtof": {
+      "c-definition": "7.24.1.5"
+    },
+    "strtold": {
+      "c-definition": "7.24.1.5"
+    },
+    "strtod32": {
+      "c-definition": "7.24.1.6"
+    },
+    "strtod64": {
+      "c-definition": "7.24.1.6"
+    },
+    "strtod128": {
+      "c-definition": "7.24.1.6"
+    },
+    "strtol": {
+      "c-definition": "7.24.1.7"
+    },
+    "strtoll": {
+      "c-definition": "7.24.1.7"
+    },
+    "strtoul": {
+      "c-definition": "7.24.1.7"
+    },
+    "strtoull": {
+      "c-definition": "7.24.1.7"
+    },
+    "rand": {
+      "c-definition": "7.24.2.1"
+    },
+    "srand": {
+      "c-definition": "7.24.2.2"
+    },
+    "aligned_alloc": {
+      "c-definition": "7.24.3.1"
+    },
+    "calloc": {
+      "c-definition": "7.24.3.2"
+    },
+    "free": {
+      "c-definition": "7.24.3.3"
+    },
+    "free_sized": {
+      "c-definition": "7.24.3.4"
+    },
+    "free_aligned_sized": {
+      "c-definition": "7.24.3.5"
+    },
+    "malloc": {
+      "c-definition": "7.24.3.6"
+    },
+    "realloc": {
+      "c-definition": "7.24.3.7"
+    },
+    "abort": {
+      "c-definition": "7.24.4.1"
+    },
+    "atexit": {
+      "c-definition": "7.24.4.2"
+    },
+    "at_quick_exit": {
+      "c-definition": "7.24.4.3"
+    },
+    "exit": {
+      "c-definition": "7.24.4.4"
+    },
+    "_Exit": {
+      "c-definition": "7.24.4.5"
+    },
+    "getenv": {
+      "c-definition": "7.24.4.6"
+    },
+    "quick_exit": {
+      "c-definition": "7.24.4.7"
+    },
+    "system": {
+      "c-definition": "7.24.4.8"
+    },
+    "bsearch": {
+      "c-definition": "7.24.5.1"
+    },
+    "qsort": {
+      "c-definition": "7.24.5.2"
+    },
+    "abs": {
+      "c-definition": "7.24.6.1"
+    },
+    "labs": {
+      "c-definition": "7.24.6.1"
+    },
+    "llabs": {
+      "c-definition": "7.24.6.1"
+    },
+    "div": {
+      "c-definition": "7.24.6.2"
+    },
+    "ldiv": {
+      "c-definition": "7.24.6.2"
+    },
+    "lldiv": {
+      "c-definition": "7.24.6.2"
+    },
+    "mblen": {
+      "c-definition": "7.24.7.1"
+    },
+    "mbtowc": {
+      "c-definition": "7.24.7.2"
+    },
+    "wctomb": {
+      "c-definition": "7.24.7.3"
+    },
+    "mbstowcs": {
+      "c-definition": "7.24.8.1"
+    },
+    "wcstombs": {
+      "c-definition": "7.24.8.2"
+    },
+    "memalignment": {
+      "c-definition": "7.24.9.1"
+    }
+  }
+}
diff --git a/libc/utils/docgen/string.json b/libc/utils/docgen/string.json
new file mode 100644
index 0000000000000..d3fd9daf186a1
--- /dev/null
+++ b/libc/utils/docgen/string.json
@@ -0,0 +1,99 @@
+{
+  "macros": {
+    "__STDC_VERSION_STRING_H__": {
+      "c-definition": "7.26.1"
+    }
+  },
+  "functions": {
+    "memcpy": {
+      "c-definition": "7.26.2.1"
+    },
+    "memccpy": {
+      "c-definition": "7.26.2.2"
+    },
+    "mempcpy": {
+      "c-definition": "TODO: glibc extension"
+    },
+    "memmove": {
+      "c-definition": "7.26.2.3"
+    },
+    "strcpy": {
+      "c-definition": "7.26.2.4"
+    },
+    "strncpy": {
+      "c-definition": "7.26.2.5"
+    },
+    "stpcpy": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/stpcpy.html"
+    },
+    "stpncpy": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/stpncpy.html"
+    },
+    "strdup": {
+      "c-definition": "7.26.2.6"
+    },
+    "strndup": {
+      "c-definition": "7.26.2.7"
+    },
+    "strcat": {
+      "c-definition": "7.26.3.1"
+    },
+    "strncat": {
+      "c-definition": "7.26.3.2"
+    },
+    "memcmp": {
+      "c-definition": "7.26.4.1"
+    },
+    "strcmp": {
+      "c-definition": "7.26.4.2"
+    },
+    "strcoll": {
+      "c-definition": "7.26.4.3"
+    },
+    "strncmp": {
+      "c-definition": "7.26.4.4"
+    },
+    "strxfrm": {
+      "c-definition": "7.26.4.5"
+    },
+    "memchr": {
+      "c-definition": "7.26.5.2"
+    },
+    "strchr": {
+      "c-definition": "7.26.5.3"
+    },
+    "strcspn": {
+      "c-definition": "7.26.5.4"
+    },
+    "strpbrk": {
+      "c-definition": "7.26.5.5"
+    },
+    "strrchr": {
+      "c-definition": "7.26.5.6"
+    },
+    "strspn": {
+      "c-definition": "7.26.5.7"
+    },
+    "strstr": {
+      "c-definition": "7.26.5.8"
+    },
+    "strtok": {
+      "c-definition": "7.26.5.9"
+    },
+    "strtok_r": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/strtok_r.html"
+    },
+    "memset": {
+      "c-definition": "7.26.6.1"
+    },
+    "memset_explicit": {
+      "c-definition": "7.26.6.2"
+    },
+    "strerror": {
+      "c-definition": "7.26.6.3"
+    },
+    "strlen": {
+      "c-definition": "7.26.6.4"
+    }
+  }
+}
diff --git a/libc/utils/docgen/strings.json b/libc/utils/docgen/strings.json
new file mode 100644
index 0000000000000..5274745f5f50d
--- /dev/null
+++ b/libc/utils/docgen/strings.json
@@ -0,0 +1,40 @@
+{
+  "functions": {
+    "bzero": {
+      "posix-definition": "removed in POSIX.1-2008"
+    },
+    "bcmp": {
+      "posix-definition": "removed in POSIX.1-2008"
+    },
+    "bcopy": {
+      "posix-definition": "removed in POSIX.1-2008"
+    },
+    "ffs": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffs.html"
+    },
+    "ffsl": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffsl.html"
+    },
+    "ffsll": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/ffsll.html"
+    },
+    "index": {
+      "posix-definition": "removed in POSIX.1-2008"
+    },
+    "rindex": {
+      "posix-definition": "removed in POSIX.1-2008"
+    },
+    "strcasecmp": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html"
+    },
+    "strcasecmp_l": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html"
+    },
+    "strncasecmp": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html"
+    },
+    "strncasecmp_l": {
+      "posix-definition": "https://pubs.opengroup.org/onlinepubs/9799919799/functions/strncasecmp.html"
+    }
+  }
+}
diff --git a/libc/utils/docgen/threads.json b/libc/utils/docgen/threads.json
index 8591cbde55a49..62c5ff881e42e 100644
--- a/libc/utils/docgen/threads.json
+++ b/libc/utils/docgen/threads.json
@@ -1,10 +1,13 @@
 {
   "macros": {
+    "__STDC_NO_THREADS__": {
+      "c-definition": "7.28.1"
+    },
     "ONCE_FLAG_INIT": {
-      "c-definition": "7.28.1.3"
+      "c-definition": "7.28.1"
     },
     "TSS_DTOR_ITERATIONS": {
-      "c-definition": "7.28.1.3"
+      "c-definition": "7.28.1"
     }
   },
   "functions": {
@@ -30,22 +33,22 @@
       "c-definition": "7.28.3.6"
     },
     "mtx_destroy": {
-      "c-definition": "7.28.4.1"
+      "c-definition": "7.28.4.2"
     },
     "mtx_init": {
-      "c-definition": "7.28.4.2"
+      "c-definition": "7.28.4.3"
     },
     "mtx_lock": {
-      "c-definition": "7.28.4.3"
+      "c-definition": "7.28.4.4"
     },
     "mtx_timedlock": {
-      "c-definition": "7.28.4.4"
+      "c-definition": "7.28.4.5"
     },
     "mtx_trylock": {
-      "c-definition": "7.28.4.5"
+      "c-definition": "7.28.4.6"
     },
     "mtx_unlock": {
-      "c-definition": "7.28.4.6"
+      "c-definition": "7.28.4.7"
     },
     "thrd_create": {
       "c-definition": "7.28.5.1"
diff --git a/libc/utils/docgen/uchar.json b/libc/utils/docgen/uchar.json
new file mode 100644
index 0000000000000..c7a8764657da6
--- /dev/null
+++ b/libc/utils/docgen/uchar.json
@@ -0,0 +1,27 @@
+{
+  "macros": {
+    "__STDC_VERSION_UCHAR_H__": {
+      "c-definition": "7.30.1"
+    }
+  },
+  "functions": {
+    "mbrtoc8": {
+      "c-definition": "7.30.2.2"
+    },
+    "c8rtomb": {
+      "c-definition": "7.30.2.3"
+    },
+    "mbrtoc16": {
+      "c-definition": "7.30.2.4"
+    },
+    "c16rtomb": {
+      "c-definition": "7.30.2.5"
+    },
+    "mbrtoc32": {
+      "c-definition": "7.30.2.6"
+    },
+    "c32rtomb": {
+      "c-definition": "7.30.2.7"
+    }
+  }
+}
diff --git a/libc/utils/docgen/wchar.json b/libc/utils/docgen/wchar.json
new file mode 100644
index 0000000000000..a44f2ef82b7cd
--- /dev/null
+++ b/libc/utils/docgen/wchar.json
@@ -0,0 +1,198 @@
+{
+  "macros": {
+    "__STDC_VERSION_WCHAR_H__": {
+      "c-definition": "7.31.1"
+    },
+    "WEOF": {
+      "c-definition": "7.31.1"
+    }
+  },
+  "functions": {
+    "fwprintf": {
+      "c-definition": "7.31.2.2"
+    },
+    "fwscanf": {
+      "c-definition": "7.31.2.3"
+    },
+    "swprintf": {
+      "c-definition": "7.31.2.4"
+    },
+    "swscanf": {
+      "c-definition": "7.31.2.5"
+    },
+    "vfwprintf": {
+      "c-definition": "7.31.2.6"
+    },
+    "vfwscanf": {
+      "c-definition": "7.31.2.7"
+    },
+    "vswprintf": {
+      "c-definition": "7.31.2.8"
+    },
+    "vswscanf": {
+      "c-definition": "7.31.2.9"
+    },
+    "vwprintf": {
+      "c-definition": "7.31.2.10"
+    },
+    "vwscanf": {
+      "c-definition": "7.31.2.11"
+    },
+    "wprintf": {
+      "c-definition": "7.31.2.12"
+    },
+    "wscanf": {
+      "c-definition": "7.31.2.13"
+    },
+    "fgetwc": {
+      "c-definition": "7.31.3.1"
+    },
+    "fgetws": {
+      "c-definition": "7.31.3.2"
+    },
+    "fputwc": {
+      "c-definition": "7.31.3.3"
+    },
+    "fputws": {
+      "c-definition": "7.31.3.4"
+    },
+    "fwide": {
+      "c-definition": "7.31.3.5"
+    },
+    "getwc": {
+      "c-definition": "7.31.3.6"
+    },
+    "getwchar": {
+      "c-definition": "7.31.3.7"
+    },
+    "putwc": {
+      "c-definition": "7.31.3.8"
+    },
+    "putwchar": {
+      "c-definition": "7.31.3.9"
+    },
+    "ungetwc": {
+      "c-definition": "7.31.3.10"
+    },
+    "wcstod": {
+      "c-definition": "7.31.4.2.2"
+    },
+    "wcstof": {
+      "c-definition": "7.31.4.2.2"
+    },
+    "wcstold": {
+      "c-definition": "7.31.4.2.2"
+    },
+    "wcstod32": {
+      "c-definition": "7.31.4.2.3"
+    },
+    "wcstod64": {
+      "c-definition": "7.31.4.2.3"
+    },
+    "wcstod128": {
+      "c-definition": "7.31.4.2.3"
+    },
+    "wcstol": {
+      "c-definition": "7.31.4.2.4"
+    },
+    "wcstoll": {
+      "c-definition": "7.31.4.2.4"
+    },
+    "wcstoul": {
+      "c-definition": "7.31.4.2.4"
+    },
+    "wcstoull": {
+      "c-definition": "7.31.4.2.4"
+    },
+    "wcscpy": {
+      "c-definition": "7.31.4.3.1"
+    },
+    "wcsncpy": {
+      "c-definition": "7.31.4.3.2"
+    },
+    "wmemcpy": {
+      "c-definition": "7.31.4.3.3"
+    },
+    "wmemmove": {
+      "c-definition": "7.31.4.3.4"
+    },
+    "wcscat": {
+      "c-definition": "7.31.4.4.1"
+    },
+    "wcsncat": {
+      "c-definition": "7.31.4.4.2"
+    },
+    "wcscmp": {
+      "c-definition": "7.31.4.5.2"
+    },
+    "wcscoll": {
+      "c-definition": "7.31.4.5.3"
+    },
+    "wcsncmp": {
+      "c-definition": "7.31.4.5.4"
+    },
+    "wcsxfrm": {
+      "c-definition": "7.31.4.5.5"
+    },
+    "wmemcmp": {
+      "c-definition": "7.31.4.5.6"
+    },
+    "wcschr": {
+      "c-definition": "7.31.4.6.2"
+    },
+    "wcscspn": {
+      "c-definition": "7.31.4.6.3"
+    },
+    "wcspbrk": {
+      "c-definition": "7.31.4.6.4"
+    },
+    "wcsrchr": {
+      "c-definition": "7.31.4.6.5"
+    },
+    "wcsspn": {
+      "c-definition": "7.31.4.6.6"
+    },
+    "wcsstr": {
+      "c-definition": "7.31.4.6.7"
+    },
+    "wcstok": {
+      "c-definition": "7.31.4.6.8"
+    },
+    "wmemchr": {
+      "c-definition": "7.31.4.6.9"
+    },
+    "wcslen": {
+      "c-definition": "7.31.4.7.1"
+    },
+    "wmemset": {
+      "c-definition": "7.31.4.7.2"
+    },
+    "wcsftime": {
+      "c-definition": "7.31.5.1"
+    },
+    "btowc": {
+      "c-definition": "7.31.6.2.1"
+    },
+    "wctob": {
+      "c-definition": "7.31.6.2.2"
+    },
+    "mbsinit": {
+      "c-definition": "7.31.6.3.1"
+    },
+    "mbrlen": {
+      "c-definition": "7.31.6.4.2"
+    },
+    "mbrtowc": {
+      "c-definition": "7.31.6.4.3"
+    },
+    "wcrtomb": {
+      "c-definition": "7.31.6.4.4"
+    },
+    "mbsrtowcs": {
+      "c-definition": "7.31.6.5.2"
+    },
+    "wcsrtombs": {
+      "c-definition": "7.31.6.5.3"
+    }
+  }
+}
diff --git a/libc/utils/docgen/wctype.json b/libc/utils/docgen/wctype.json
new file mode 100644
index 0000000000000..29bac1d5d03aa
--- /dev/null
+++ b/libc/utils/docgen/wctype.json
@@ -0,0 +1,58 @@
+{
+  "functions": {
+    "iswalnum": {
+      "c-definition": "7.32.2.1.1"
+    },
+    "iswalpha": {
+      "c-definition": "7.32.2.1.2"
+    },
+    "iswblank": {
+      "c-definition": "7.32.2.1.3"
+    },
+    "iswblank": {
+      "c-definition": "7.32.2.1.4"
+    },
+    "iswdigit": {
+      "c-definition": "7.32.2.1.5"
+    },
+    "iswgraph": {
+      "c-definition": "7.32.2.1.6"
+    },
+    "iswlower": {
+      "c-definition": "7.32.2.1.7"
+    },
+    "iswprint": {
+      "c-definition": "7.32.2.1.8"
+    },
+    "iswpunct": {
+      "c-definition": "7.32.2.1.9"
+    },
+    "iswspace": {
+      "c-definition": "7.32.2.1.10"
+    },
+    "iswupper": {
+      "c-definition": "7.32.2.1.11"
+    },
+    "iswxdigit": {
+      "c-definition": "7.32.2.1.12"
+    },
+    "iswctype": {
+      "c-definition": "7.32.2.2.1"
+    },
+    "wctype": {
+      "c-definition": "7.32.2.2.2"
+    },
+    "towlower": {
+      "c-definition": "7.32.3.1.1"
+    },
+    "towupper": {
+      "c-definition": "7.32.3.1.2"
+    },
+    "towctrans": {
+      "c-definition": "7.32.3.2.1"
+    },
+    "wctrans": {
+      "c-definition": "7.32.3.2.2"
+    }
+  }
+}
diff --git a/libc/utils/gpu/loader/Loader.h b/libc/utils/gpu/loader/Loader.h
index 497f8114728bf..0d683832855e9 100644
--- a/libc/utils/gpu/loader/Loader.h
+++ b/libc/utils/gpu/loader/Loader.h
@@ -9,8 +9,6 @@
 #ifndef LLVM_LIBC_UTILS_GPU_LOADER_LOADER_H
 #define LLVM_LIBC_UTILS_GPU_LOADER_LOADER_H
 
-#include "utils/gpu/server/llvmlibc_rpc_server.h"
-
 #include "include/llvm-libc-types/test_rpc_opcodes_t.h"
 
 #include "shared/rpc.h"
@@ -170,20 +168,20 @@ inline uint32_t handle_server(rpc::Server &server, uint32_t index,
     port->recv([&](rpc::Buffer *, uint32_t) {});
     break;
   }
-  case RPC_MALLOC: {
+  case LIBC_MALLOC: {
     port->recv_and_send([&](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = reinterpret_cast<uintptr_t>(alloc(buffer->data[0]));
     });
     break;
   }
-  case RPC_FREE: {
+  case LIBC_FREE: {
     port->recv([&](rpc::Buffer *buffer, uint32_t) {
       free(reinterpret_cast<void *>(buffer->data[0]));
     });
     break;
   }
   default:
-    status = libc_handle_rpc_port(&*port, num_lanes);
+    status = handle_libc_opcodes(*port, num_lanes);
     break;
   }
 
diff --git a/libc/utils/gpu/loader/amdgpu/CMakeLists.txt b/libc/utils/gpu/loader/amdgpu/CMakeLists.txt
index 46c5631046ce2..80c5ae357416a 100644
--- a/libc/utils/gpu/loader/amdgpu/CMakeLists.txt
+++ b/libc/utils/gpu/loader/amdgpu/CMakeLists.txt
@@ -3,6 +3,7 @@ set(LLVM_LINK_COMPONENTS
   Object
   Option
   Support
+  FrontendOffloading
   )
 
 add_llvm_executable(amdhsa-loader amdhsa-loader.cpp)
diff --git a/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp b/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
index 13a1366833547..00fde147b0abd 100644
--- a/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
+++ b/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
@@ -15,18 +15,10 @@
 
 #include "Loader.h"
 
-#if defined(__has_include)
-#if __has_include("hsa/hsa.h")
 #include "hsa/hsa.h"
 #include "hsa/hsa_ext_amd.h"
-#elif __has_include("hsa.h")
-#include "hsa.h"
-#include "hsa_ext_amd.h"
-#endif
-#else
-#include "hsa/hsa.h"
-#include "hsa/hsa_ext_amd.h"
-#endif
+
+#include "llvm/Frontend/Offloading/Utility.h"
 
 #include <atomic>
 #include <cstdio>
@@ -163,17 +155,13 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable,
                            hsa_queue_t *queue, rpc::Server &server,
                            const LaunchParameters &params,
                            const char *kernel_name, args_t kernel_args,
-                           bool print_resource_usage) {
+                           uint32_t wavefront_size, bool print_resource_usage) {
   // Look up the kernel in the loaded executable.
   hsa_executable_symbol_t symbol;
   if (hsa_status_t err = hsa_executable_get_symbol_by_name(
           executable, kernel_name, &dev_agent, &symbol))
     return err;
 
-  uint32_t wavefront_size = 0;
-  if (hsa_status_t err = hsa_agent_get_info(
-          dev_agent, HSA_AGENT_INFO_WAVEFRONT_SIZE, &wavefront_size))
-    handle_error(err);
   // Retrieve different properties of the kernel symbol used for launch.
   uint64_t kernel;
   uint32_t args_size;
@@ -419,6 +407,16 @@ int load(int argc, const char **argv, const char **envp, void *image,
               dev_agent, &coarsegrained_pool))
     handle_error(err);
 
+  // The AMDGPU target can change its wavefront size. There currently isn't a
+  // good way to look this up through the HSA API so we use the LLVM interface.
+  uint16_t abi_version;
+  llvm::StringRef image_ref(reinterpret_cast<char *>(image), size);
+  llvm::StringMap<llvm::offloading::amdgpu::AMDGPUKernelMetaData> info_map;
+  if (llvm::Error err = llvm::offloading::amdgpu::getAMDGPUMetaDataFromImage(
+          llvm::MemoryBufferRef(image_ref, ""), info_map, abi_version)) {
+    handle_error(llvm::toString(std::move(err)).c_str());
+  }
+
   // Allocate fine-grained memory on the host to hold the pointer array for the
   // copied argv and allow the GPU agent to access it.
   auto allocator = [&](uint64_t size) -> void * {
@@ -448,10 +446,10 @@ int load(int argc, const char **argv, const char **envp, void *image,
   hsa_amd_memory_fill(dev_ret, 0, /*count=*/1);
 
   // Allocate finegrained memory for the RPC server and client to share.
-  uint32_t wavefront_size = 0;
-  if (hsa_status_t err = hsa_agent_get_info(
-          dev_agent, HSA_AGENT_INFO_WAVEFRONT_SIZE, &wavefront_size))
-    handle_error(err);
+  uint32_t wavefront_size =
+      llvm::max_element(info_map, [](auto &&x, auto &&y) {
+        return x.second.WavefrontSize < y.second.WavefrontSize;
+      })->second.WavefrontSize;
 
   // Set up the RPC server.
   void *rpc_buffer;
@@ -469,15 +467,8 @@ int load(int argc, const char **argv, const char **envp, void *image,
   // device's internal pointer.
   hsa_executable_symbol_t rpc_client_sym;
   if (hsa_status_t err = hsa_executable_get_symbol_by_name(
-          executable, "__llvm_libc_rpc_client", &dev_agent, &rpc_client_sym))
-    handle_error(err);
-
-  void *rpc_client_host;
-  if (hsa_status_t err =
-          hsa_amd_memory_pool_allocate(finegrained_pool, sizeof(void *),
-                                       /*flags=*/0, &rpc_client_host))
+          executable, "__llvm_rpc_client", &dev_agent, &rpc_client_sym))
     handle_error(err);
-  hsa_amd_agents_allow_access(1, &dev_agent, nullptr, rpc_client_host);
 
   void *rpc_client_dev;
   if (hsa_status_t err = hsa_executable_symbol_get_info(
@@ -485,11 +476,6 @@ int load(int argc, const char **argv, const char **envp, void *image,
           &rpc_client_dev))
     handle_error(err);
 
-  // Copy the address of the client buffer from the device to the host.
-  if (hsa_status_t err = hsa_memcpy(rpc_client_host, host_agent, rpc_client_dev,
-                                    dev_agent, sizeof(void *)))
-    handle_error(err);
-
   void *rpc_client_buffer;
   if (hsa_status_t err =
           hsa_amd_memory_lock(&client, sizeof(rpc::Client),
@@ -498,14 +484,12 @@ int load(int argc, const char **argv, const char **envp, void *image,
 
   // Copy the RPC client buffer to the address pointed to by the symbol.
   if (hsa_status_t err =
-          hsa_memcpy(*reinterpret_cast<void **>(rpc_client_host), dev_agent,
-                     rpc_client_buffer, host_agent, sizeof(rpc::Client)))
+          hsa_memcpy(rpc_client_dev, dev_agent, rpc_client_buffer, host_agent,
+                     sizeof(rpc::Client)))
     handle_error(err);
 
   if (hsa_status_t err = hsa_amd_memory_unlock(&client))
     handle_error(err);
-  if (hsa_status_t err = hsa_amd_memory_pool_free(rpc_client_host))
-    handle_error(err);
 
   // Obtain the GPU's fixed-frequency clock rate and copy it to the GPU.
   // If the clock_freq symbol is missing, no work to do.
@@ -513,7 +497,6 @@ int load(int argc, const char **argv, const char **envp, void *image,
   if (HSA_STATUS_SUCCESS ==
       hsa_executable_get_symbol_by_name(executable, "__llvm_libc_clock_freq",
                                         &dev_agent, &freq_sym)) {
-
     void *host_clock_freq;
     if (hsa_status_t err =
             hsa_amd_memory_pool_allocate(finegrained_pool, sizeof(uint64_t),
@@ -553,16 +536,17 @@ int load(int argc, const char **argv, const char **envp, void *image,
 
   LaunchParameters single_threaded_params = {1, 1, 1, 1, 1, 1};
   begin_args_t init_args = {argc, dev_argv, dev_envp};
-  if (hsa_status_t err = launch_kernel(dev_agent, executable, kernargs_pool,
-                                       coarsegrained_pool, queue, server,
-                                       single_threaded_params, "_begin.kd",
-                                       init_args, print_resource_usage))
+  if (hsa_status_t err = launch_kernel(
+          dev_agent, executable, kernargs_pool, coarsegrained_pool, queue,
+          server, single_threaded_params, "_begin.kd", init_args,
+          info_map["_begin"].WavefrontSize, print_resource_usage))
     handle_error(err);
 
   start_args_t args = {argc, dev_argv, dev_envp, dev_ret};
   if (hsa_status_t err = launch_kernel(
           dev_agent, executable, kernargs_pool, coarsegrained_pool, queue,
-          server, params, "_start.kd", args, print_resource_usage))
+          server, params, "_start.kd", args, info_map["_start"].WavefrontSize,
+          print_resource_usage))
     handle_error(err);
 
   void *host_ret;
@@ -580,10 +564,10 @@ int load(int argc, const char **argv, const char **envp, void *image,
   int ret = *static_cast<int *>(host_ret);
 
   end_args_t fini_args = {ret};
-  if (hsa_status_t err = launch_kernel(dev_agent, executable, kernargs_pool,
-                                       coarsegrained_pool, queue, server,
-                                       single_threaded_params, "_end.kd",
-                                       fini_args, print_resource_usage))
+  if (hsa_status_t err = launch_kernel(
+          dev_agent, executable, kernargs_pool, coarsegrained_pool, queue,
+          server, single_threaded_params, "_end.kd", fini_args,
+          info_map["_end"].WavefrontSize, print_resource_usage))
     handle_error(err);
 
   if (hsa_status_t err = hsa_amd_memory_pool_free(rpc_buffer))
diff --git a/libc/utils/gpu/loader/nvptx/nvptx-loader.cpp b/libc/utils/gpu/loader/nvptx/nvptx-loader.cpp
index 0ba217451feae..7d6c176c6f360 100644
--- a/libc/utils/gpu/loader/nvptx/nvptx-loader.cpp
+++ b/libc/utils/gpu/loader/nvptx/nvptx-loader.cpp
@@ -314,15 +314,10 @@ int load(int argc, const char **argv, const char **envp, void *image,
   CUdeviceptr rpc_client_dev = 0;
   uint64_t client_ptr_size = sizeof(void *);
   if (CUresult err = cuModuleGetGlobal(&rpc_client_dev, &client_ptr_size,
-                                       binary, "__llvm_libc_rpc_client"))
+                                       binary, "__llvm_rpc_client"))
     handle_error(err);
 
-  CUdeviceptr rpc_client_host = 0;
-  if (CUresult err =
-          cuMemcpyDtoH(&rpc_client_host, rpc_client_dev, sizeof(void *)))
-    handle_error(err);
-  if (CUresult err =
-          cuMemcpyHtoD(rpc_client_host, &client, sizeof(rpc::Client)))
+  if (CUresult err = cuMemcpyHtoD(rpc_client_dev, &client, sizeof(rpc::Client)))
     handle_error(err);
 
   LaunchParameters single_threaded_params = {1, 1, 1, 1, 1, 1};
diff --git a/libc/utils/gpu/server/CMakeLists.txt b/libc/utils/gpu/server/CMakeLists.txt
index 36a4c2a8b051e..b1cada44cd327 100644
--- a/libc/utils/gpu/server/CMakeLists.txt
+++ b/libc/utils/gpu/server/CMakeLists.txt
@@ -23,9 +23,6 @@ target_compile_definitions(llvmlibc_rpc_server PUBLIC
                            LIBC_NAMESPACE=${LIBC_NAMESPACE})
 
 # Install the server and associated header.
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/llvmlibc_rpc_server.h
-        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-        COMPONENT libc-headers)
 install(TARGETS llvmlibc_rpc_server
         ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}"
         COMPONENT libc)
diff --git a/libc/utils/gpu/server/llvmlibc_rpc_server.h b/libc/utils/gpu/server/llvmlibc_rpc_server.h
deleted file mode 100644
index b7f173734345c..0000000000000
--- a/libc/utils/gpu/server/llvmlibc_rpc_server.h
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- Shared memory RPC server instantiation ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_UTILS_GPU_SERVER_RPC_SERVER_H
-#define LLVM_LIBC_UTILS_GPU_SERVER_RPC_SERVER_H
-
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int libc_handle_rpc_port(void *port, uint32_t num_lanes);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index 21af7ad7d5f1f..f724c5c82c422 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -17,8 +17,6 @@
 #include "shared/rpc.h"
 #include "shared/rpc_opcodes.h"
 
-#include "llvmlibc_rpc_server.h"
-
 #include "src/__support/arg_list.h"
 #include "src/stdio/printf_core/converter.h"
 #include "src/stdio/printf_core/parser.h"
@@ -72,13 +70,13 @@ template <bool packed, uint32_t num_lanes>
 static void handle_printf(rpc::Server::Port &port, TempStorage &temp_storage) {
   FILE *files[num_lanes] = {nullptr};
   // Get the appropriate output stream to use.
-  if (port.get_opcode() == RPC_PRINTF_TO_STREAM ||
-      port.get_opcode() == RPC_PRINTF_TO_STREAM_PACKED)
+  if (port.get_opcode() == LIBC_PRINTF_TO_STREAM ||
+      port.get_opcode() == LIBC_PRINTF_TO_STREAM_PACKED)
     port.recv([&](rpc::Buffer *buffer, uint32_t id) {
       files[id] = reinterpret_cast<FILE *>(buffer->data[0]);
     });
-  else if (port.get_opcode() == RPC_PRINTF_TO_STDOUT ||
-           port.get_opcode() == RPC_PRINTF_TO_STDOUT_PACKED)
+  else if (port.get_opcode() == LIBC_PRINTF_TO_STDOUT ||
+           port.get_opcode() == LIBC_PRINTF_TO_STDOUT_PACKED)
     std::fill(files, files + num_lanes, stdout);
   else
     std::fill(files, files + num_lanes, stderr);
@@ -232,18 +230,18 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
   TempStorage temp_storage;
 
   switch (port.get_opcode()) {
-  case RPC_WRITE_TO_STREAM:
-  case RPC_WRITE_TO_STDERR:
-  case RPC_WRITE_TO_STDOUT:
-  case RPC_WRITE_TO_STDOUT_NEWLINE: {
+  case LIBC_WRITE_TO_STREAM:
+  case LIBC_WRITE_TO_STDERR:
+  case LIBC_WRITE_TO_STDOUT:
+  case LIBC_WRITE_TO_STDOUT_NEWLINE: {
     uint64_t sizes[num_lanes] = {0};
     void *strs[num_lanes] = {nullptr};
     FILE *files[num_lanes] = {nullptr};
-    if (port.get_opcode() == RPC_WRITE_TO_STREAM) {
+    if (port.get_opcode() == LIBC_WRITE_TO_STREAM) {
       port.recv([&](rpc::Buffer *buffer, uint32_t id) {
         files[id] = reinterpret_cast<FILE *>(buffer->data[0]);
       });
-    } else if (port.get_opcode() == RPC_WRITE_TO_STDERR) {
+    } else if (port.get_opcode() == LIBC_WRITE_TO_STDERR) {
       std::fill(files, files + num_lanes, stderr);
     } else {
       std::fill(files, files + num_lanes, stdout);
@@ -254,14 +252,14 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     port.send([&](rpc::Buffer *buffer, uint32_t id) {
       flockfile(files[id]);
       buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);
-      if (port.get_opcode() == RPC_WRITE_TO_STDOUT_NEWLINE &&
+      if (port.get_opcode() == LIBC_WRITE_TO_STDOUT_NEWLINE &&
           buffer->data[0] == sizes[id])
         buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);
       funlockfile(files[id]);
     });
     break;
   }
-  case RPC_READ_FROM_STREAM: {
+  case LIBC_READ_FROM_STREAM: {
     uint64_t sizes[num_lanes] = {0};
     void *data[num_lanes] = {nullptr};
     port.recv([&](rpc::Buffer *buffer, uint32_t id) {
@@ -275,7 +273,7 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_READ_FGETS: {
+  case LIBC_READ_FGETS: {
     uint64_t sizes[num_lanes] = {0};
     void *data[num_lanes] = {nullptr};
     port.recv([&](rpc::Buffer *buffer, uint32_t id) {
@@ -287,7 +285,7 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     port.send_n(data, sizes);
     break;
   }
-  case RPC_OPEN_FILE: {
+  case LIBC_OPEN_FILE: {
     uint64_t sizes[num_lanes] = {0};
     void *paths[num_lanes] = {nullptr};
     port.recv_n(paths, sizes,
@@ -299,14 +297,14 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_CLOSE_FILE: {
+  case LIBC_CLOSE_FILE: {
     port.recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
       FILE *file = reinterpret_cast<FILE *>(buffer->data[0]);
       buffer->data[0] = fclose(file);
     });
     break;
   }
-  case RPC_EXIT: {
+  case LIBC_EXIT: {
     // Send a response to the client to signal that we are ready to exit.
     port.recv_and_send([](rpc::Buffer *, uint32_t) {});
     port.recv([](rpc::Buffer *buffer, uint32_t) {
@@ -316,14 +314,14 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_ABORT: {
+  case LIBC_ABORT: {
     // Send a response to the client to signal that we are ready to abort.
     port.recv_and_send([](rpc::Buffer *, uint32_t) {});
     port.recv([](rpc::Buffer *, uint32_t) {});
     abort();
     break;
   }
-  case RPC_HOST_CALL: {
+  case LIBC_HOST_CALL: {
     uint64_t sizes[num_lanes] = {0};
     unsigned long long results[num_lanes] = {0};
     void *args[num_lanes] = {nullptr};
@@ -339,25 +337,25 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_FEOF: {
+  case LIBC_FEOF: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = feof(to_stream(buffer->data[0]));
     });
     break;
   }
-  case RPC_FERROR: {
+  case LIBC_FERROR: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = ferror(to_stream(buffer->data[0]));
     });
     break;
   }
-  case RPC_CLEARERR: {
+  case LIBC_CLEARERR: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       clearerr(to_stream(buffer->data[0]));
     });
     break;
   }
-  case RPC_FSEEK: {
+  case LIBC_FSEEK: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] =
           fseek(to_stream(buffer->data[0]), static_cast<long>(buffer->data[1]),
@@ -365,38 +363,38 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_FTELL: {
+  case LIBC_FTELL: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = ftell(to_stream(buffer->data[0]));
     });
     break;
   }
-  case RPC_FFLUSH: {
+  case LIBC_FFLUSH: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] = fflush(to_stream(buffer->data[0]));
     });
     break;
   }
-  case RPC_UNGETC: {
+  case LIBC_UNGETC: {
     port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {
       buffer->data[0] =
           ungetc(static_cast<int>(buffer->data[0]), to_stream(buffer->data[1]));
     });
     break;
   }
-  case RPC_PRINTF_TO_STREAM_PACKED:
-  case RPC_PRINTF_TO_STDOUT_PACKED:
-  case RPC_PRINTF_TO_STDERR_PACKED: {
+  case LIBC_PRINTF_TO_STREAM_PACKED:
+  case LIBC_PRINTF_TO_STDOUT_PACKED:
+  case LIBC_PRINTF_TO_STDERR_PACKED: {
     handle_printf<true, num_lanes>(port, temp_storage);
     break;
   }
-  case RPC_PRINTF_TO_STREAM:
-  case RPC_PRINTF_TO_STDOUT:
-  case RPC_PRINTF_TO_STDERR: {
+  case LIBC_PRINTF_TO_STREAM:
+  case LIBC_PRINTF_TO_STDOUT:
+  case LIBC_PRINTF_TO_STDERR: {
     handle_printf<false, num_lanes>(port, temp_storage);
     break;
   }
-  case RPC_REMOVE: {
+  case LIBC_REMOVE: {
     uint64_t sizes[num_lanes] = {0};
     void *args[num_lanes] = {nullptr};
     port.recv_n(args, sizes,
@@ -407,7 +405,7 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_RENAME: {
+  case LIBC_RENAME: {
     uint64_t oldsizes[num_lanes] = {0};
     uint64_t newsizes[num_lanes] = {0};
     void *oldpath[num_lanes] = {nullptr};
@@ -423,7 +421,7 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_SYSTEM: {
+  case LIBC_SYSTEM: {
     uint64_t sizes[num_lanes] = {0};
     void *args[num_lanes] = {nullptr};
     port.recv_n(args, sizes,
@@ -434,7 +432,7 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
     });
     break;
   }
-  case RPC_NOOP: {
+  case LIBC_NOOP: {
     port.recv([](rpc::Buffer *, uint32_t) {});
     break;
   }
@@ -445,15 +443,19 @@ rpc::Status handle_port_impl(rpc::Server::Port &port) {
   return rpc::SUCCESS;
 }
 
-int libc_handle_rpc_port(void *port, uint32_t num_lanes) {
+namespace rpc {
+// The implementation of this function currently lives in the utility directory
+// at 'utils/gpu/server/rpc_server.cpp'.
+rpc::Status handle_libc_opcodes(rpc::Server::Port &port, uint32_t num_lanes) {
   switch (num_lanes) {
   case 1:
-    return handle_port_impl<1>(*reinterpret_cast<rpc::Server::Port *>(port));
+    return handle_port_impl<1>(port);
   case 32:
-    return handle_port_impl<32>(*reinterpret_cast<rpc::Server::Port *>(port));
+    return handle_port_impl<32>(port);
   case 64:
-    return handle_port_impl<64>(*reinterpret_cast<rpc::Server::Port *>(port));
+    return handle_port_impl<64>(port);
   default:
     return rpc::ERROR;
   }
 }
+} // namespace rpc
diff --git a/libcxx/Maintainers.md b/libcxx/Maintainers.md
new file mode 100644
index 0000000000000..2630ed9001119
--- /dev/null
+++ b/libcxx/Maintainers.md
@@ -0,0 +1,10 @@
+# libc++ Maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+libc++.
+
+# Lead maintainer
+
+Louis Dionne \
+ldionne.2@gmail.com (email), [ldionne](https://github.com/ldionne) (GitHub)
\ No newline at end of file
diff --git a/libcxx/docs/CodingGuidelines.rst b/libcxx/docs/CodingGuidelines.rst
new file mode 100644
index 0000000000000..1bb62072e886d
--- /dev/null
+++ b/libcxx/docs/CodingGuidelines.rst
@@ -0,0 +1,186 @@
+.. _CodingGuidelines:
+
+========================
+libc++ Coding Guidelines
+========================
+
+.. contents::
+  :local:
+
+Use ``__ugly_names`` for implementation details
+===============================================
+
+Libc++ uses ``__ugly_names`` or ``_UglyNames`` for implementation details. These names are reserved for implementations,
+so users may not use them in their own applications. When using a name like ``T``, a user may have defined a macro that
+changes the meaning of ``T``. By using ``__ugly_names`` we avoid that problem.
+
+This is partially enforced by the clang-tidy check ``readability-identifier-naming`` and
+``libcxx/test/libcxx/system_reserved_names.gen.py``.
+
+Don't use argument-dependent lookup unless required by the standard
+===================================================================
+
+Unqualified function calls are susceptible to
+`argument-dependent lookup (ADL) <https://en.cppreference.com/w/cpp/language/adl>`_. This means calling
+``move(UserType)`` might not call ``std::move``. Therefore, function calls must use qualified names to avoid ADL. Some
+functions in the standard library `require ADL usage <http://eel.is/c++draft/contents#3>`_. Names of classes, variables,
+concepts, and type aliases are not subject to ADL. They don't need to be qualified.
+
+Function overloading also applies to operators. Using ``&user_object`` may call a user-defined ``operator&``. Use
+``std::addressof`` instead. Similarly, to avoid invoking a user-defined ``operator,``, make sure to cast the result to
+``void`` when using the ``,`` or avoid it in the first place. For example:
+
+.. code-block:: cpp
+
+    for (; __first1 != __last1; ++__first1, (void)++__first2) {
+      ...
+    }
+
+This is mostly enforced by the clang-tidy checks ``libcpp-robust-against-adl`` and ``libcpp-qualify-declval``.
+
+Avoid including public headers
+==============================
+
+libc++ uses implementation-detail headers for most code. These are in a directory that starts with two underscores
+(e.g. ``<__type_traits/decay.h>``). These detail headers are significantly smaller than their public counterparts.
+This reduces the amount of code that is included in a single public header, which reduces compile times.
+
+Add ``_LIBCPP_HIDE_FROM_ABI`` unless you know better
+====================================================
+
+``_LIBCPP_HIDE_FROM_ABI`` should be on every function in the library unless there is a reason not to do so. The main
+reason not to add ``_LIBCPP_HIDE_FROM_ABI`` is if a function is exported from the libc++ built library. In that case the
+function should be marked with ``_LIBCPP_EXPORTED_FROM_ABI``. Virtual functions should be marked with
+``_LIBCPP_HIDE_FROM_ABI_VIRTUAL`` instead.
+
+This is mostly enforced by the clang-tidy checks ``libcpp-hide-from-abi`` and ``libcpp-avoid-abi-tag-on-virtual``.
+
+Define configuration macros to 0 or 1
+=====================================
+
+Macros should usually be defined in all configurations, instead of defining them when they're enabled and leaving them
+undefined otherwise. For example, use
+
+.. code-block:: cpp
+
+  #if SOMETHING
+  #  define _LIBCPP_SOMETHING_ENABLED 1
+  #else
+  #  define _LIBCPP_SOMETHING_ENABLED 0
+  #endif
+
+and then check for ``#if _LIBCPP_SOMETHING_ENABLED`` instead of
+
+.. code-block:: cpp
+
+  #if SOMETHING
+  #  define _LIBCPP_SOMETHING_ENABLED
+  #endif
+
+and then checking for ``#ifdef _LIBCPP_SOMETHING_ENABLED``.
+
+This makes it significantly easier to catch missing includes, since Clang and GCC will warn when using and undefined
+marco inside an ``#if`` statement when using ``-Wundef``. Some macros in libc++ don't use this style yet, so this only
+applies when introducing a new macro.
+
+This is partially enforced by the clang-tidy check ``libcpp-internal-ftms``.
+
+Use ``_LIBCPP_STD_VER``
+=======================
+
+libc++ defines the macro ``_LIBCPP_STD_VER`` for the different libc++ dialects. This should be used instead of
+``__cplusplus``.
+
+This is mostly enforced by the clang-tidy check ``libcpp-cpp-version-check``.
+
+Use ``__ugly__`` spellings of vendor attributes
+===============================================
+
+Vendor attributes should always be ``__uglified__`` to avoid naming clashes with user-defined macros. For gnu-style
+attributes this takes the form ``__attribute__((__foo__))``. C++11-style attributes look like ``[[_Clang::__foo__]]`` or
+``[[__gnu__::__foo__]]`` for Clang or GCC attributes respectively. Clang and GCC also support standard attributes in
+earlier language dialects than they were introduced. These should be spelled as ``[[__foo__]]``. MSVC currently doesn't
+provide alternative spellings for their attributes, so these should be avoided if at all possible.
+
+This is enforced by the clang-tidy check ``libcpp-uglify-attributes``.
+
+Use C++11 extensions in C++03 code if they simplify the code
+============================================================
+
+libc++ only supports Clang in C++98/03 mode. Clang provides many C++11 features in C++03, making it possible to write a
+lot of code in a simpler way than if we were restricted to C++03 features. Some use of extensions is even mandatory,
+since libc++ supports move semantics in C++03.
+
+Use ``using`` aliases instead of ``typedef``
+============================================
+
+``using`` aliases are generally easier to read and support templates. Some code in libc++ uses ``typedef`` for
+historical reasons.
+
+Write SFINAE with ``requires`` clauses in C++20-only code
+=========================================================
+
+``requires`` clauses can be significantly easier to read than ``enable_if`` and friends in some cases, since concepts
+subsume other concepts. This means that overloads based on traits can be written without negating more general cases.
+They also show intent better.
+
+Write ``enable_if`` as ``enable_if_t<conditon, int> = 0``
+=========================================================
+
+The form ``enable_if_t<condition, int> = 0`` is the only one that works in every language mode and for overload sets
+using the same template arguments otherwise. If the code must work in C++11 or C++03, the libc++-internal alias
+``__enable_if_t`` can be used instead.
+
+Prefer alias templates over class templates
+===========================================
+
+Alias templates are much more lightweight than class templates, since they don't require new instantiations for
+different types. If the only member of a class is an alias, like in type traits, alias templates should be used if
+possible. They do force more eager evaluation though, which can be a problem in some cases.
+
+Apply ``[[nodiscard]]`` where relevant
+======================================
+
+Libc++ adds ``[[nodiscard]]`` whenever relevant to catch potential bugs. The standards committee has decided to _not_
+have a recommended practice where to put them, so libc++ applies it whenever it makes sense to catch potential bugs.
+
+``[[nodiscard]]`` should be applied to functions
+
+- where discarding the return value is most likely a correctness issue. For example a locking constructor in
+  ``unique_lock``.
+
+- where discarding the return value likely points to the user wanting to do something different. For example
+  ``vector::empty()``, which probably should have been ``vector::clear()``.
+
+  This can help spotting bugs easily which otherwise may take a very long time to find.
+
+- which return a constant. For example ``numeric_limits::min()``.
+- which only observe a value. For example ``string::size()``.
+
+  Code that discards values from these kinds of functions is dead code. It can either be removed, or the programmer
+  meant to do something different.
+
+- where discarding the value is most likely a misuse of the function. For example ``std::find(first, last, val)``.
+
+  This protects programmers from assuming too much about how the internals of a function work, making code more robust
+  in the presence of future optimizations.
+
+Applications of ``[[nodiscard]]`` are code like any other code, so we aim to test them on public interfaces. This can be
+done with a ``.verify.cpp`` test. Many examples are available. Just look for tests with the suffix
+``.nodiscard.verify.cpp``.
+
+Don't use public API names for symbols on the ABI boundary
+==========================================================
+
+Most functions in libc++ are defined in headers either as templates or as ``inline`` functions. However, we sometimes
+need or want to define functions in the built library. Symbols that are declared in the headers and defined in the
+built library become part of the ABI of libc++, which must be preserved for backwards compatibility. This means that
+we can't easily remove or rename such symbols except in special cases.
+
+When adding a symbol to the built library, make sure not to use a public name directly. Instead, define a
+``_LIBCPP_HIDE_FROM_ABI`` function in the headers with the public name and have it call a private function in the built
+library. This approach makes it easier to make changes to libc++ like move something from the built library to the
+headers (which is sometimes required for ``constexpr`` support).
+
+When defining a function at the ABI boundary, it can also be useful to consider which attributes (like ``[[gnu::pure]]``
+and ``[[clang::noescape]]``) can be added to the function to improve the compiler's ability to optimize.
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index b15fc88943d59..5bcd6e3a7f03e 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -36,56 +36,10 @@ Every change in libc++ must come with appropriate tests. Libc++ has an extensive
 should be run locally by developers before submitting patches and is also run as part of our CI
 infrastructure. The documentation about writing tests and running them is :ref:`here <testing>`.
 
-Coding standards
-================
+Coding Guidelines
+=================
 
-In general, libc++ follows the `LLVM Coding Standards <https://llvm.org/docs/CodingStandards.html>`_.
-There are some deviations from these standards.
-
-Libc++ uses ``__ugly_names``. These names are reserved for implementations, so
-users may not use them in their own applications. When using a name like ``T``,
-a user may have defined a macro that changes the meaning of ``T``. By using
-``__ugly_names`` we avoid that problem. Other standard libraries and compilers
-use these names too. To avoid common clashes with other uglified names used in
-other implementations (e.g. system headers), the test in
-``libcxx/test/libcxx/system_reserved_names.gen.py`` contains the list of
-reserved names that can't be used.
-
-Unqualified function calls are susceptible to
-`argument-dependent lookup (ADL) <https://en.cppreference.com/w/cpp/language/adl>`_.
-This means calling ``move(UserType)`` might not call ``std::move``. Therefore,
-function calls must use qualified names to avoid ADL. Some functions in the
-standard library `require ADL usage <http://eel.is/c++draft/contents#3>`_.
-Names of classes, variables, concepts, and type aliases are not subject to ADL.
-They don't need to be qualified.
-
-Function overloading also applies to operators. Using ``&user_object`` may call
-a user-defined ``operator&``. Use ``std::addressof`` instead. Similarly, to
-avoid invoking a user-defined ``operator,``, make sure to cast the result to
-``void`` when using the ``,``. For example:
-
-.. code-block:: cpp
-
-    for (; __first1 != __last1; ++__first1, (void)++__first2) {
-      ...
-    }
-
-In general, try to follow the style of existing code. There are a few
-exceptions:
-
-- Prefer ``using foo = int`` over ``typedef int foo``. The compilers supported
-  by libc++ accept alias declarations in all standard modes.
-
-Other tips are:
-
-- Keep the number of formatting changes in patches minimal.
-- Provide separate patches for style fixes and for bug fixes or features. Keep in
-  mind that large formatting patches may cause merge conflicts with other patches
-  under review. In general, we prefer to avoid large reformatting patches.
-- Keep patches self-contained. Large and/or complicated patches are harder to
-  review and take a significant amount of time. It's fine to have multiple
-  patches to implement one feature if the feature can be split into
-  self-contained sub-tasks.
+libc++'s coding guidelines are documented :ref:`here <CodingGuidelines>`.
 
 
 Resources
@@ -186,6 +140,17 @@ rule -- for very simple patches, use your judgement. The `"libc++" review group
 consists of frequent libc++ contributors with a good understanding of the project's
 guidelines -- if you would like to be added to it, please reach out on Discord.
 
+Some tips:
+
+- Keep the number of formatting changes in patches minimal.
+- Provide separate patches for style fixes and for bug fixes or features. Keep in
+  mind that large formatting patches may cause merge conflicts with other patches
+  under review. In general, we prefer to avoid large reformatting patches.
+- Keep patches self-contained. Large and/or complicated patches are harder to
+  review and take a significant amount of time. It's fine to have multiple
+  patches to implement one feature if the feature can be split into
+  self-contained sub-tasks.
+
 Exporting new symbols from the library
 ======================================
 
diff --git a/libcxx/docs/DesignDocs/ExtendedCXX03Support.rst b/libcxx/docs/DesignDocs/ExtendedCXX03Support.rst
index 8c18e563e8199..42d59db2bb7a4 100644
--- a/libcxx/docs/DesignDocs/ExtendedCXX03Support.rst
+++ b/libcxx/docs/DesignDocs/ExtendedCXX03Support.rst
@@ -47,72 +47,3 @@ Provided C++11 Library Extensions
 
   This section will be updated once the libc++ developer community has further discussed the
   future of C++03 with libc++.
-
-
-Using Minimal C++11 in libc++
-=============================
-
-This section is for developers submitting patches to libc++. It describes idioms that should be
-used in libc++ code, even in C++03, and the reasons behind them.
-
-
-Use Alias Templates over Class Templates
-----------------------------------------
-
-Alias templates should be used instead of class templates in metaprogramming. Unlike class templates,
-Alias templates do not produce a new instantiation every time they are used. This significantly
-decreases the amount of memory used by the compiler.
-
-For example, libc++ should not use ``add_const`` internally. Instead it should use an alias template
-like
-
-.. code-block:: cpp
-
-  template <class _Tp>
-  using _AddConst = const _Tp;
-
-Use Default Template Parameters for SFINAE
-------------------------------------------
-
-There are three places in a function declaration that SFINAE may occur: In the template parameter list,
-in the function parameter list, and in the return type. For example:
-
-.. code-block:: cpp
-
-  template <class _Tp, class _ = enable_if_t</*...*/ >
-  void foo(_Tp); // #1
-
-  template <class _Tp>
-  void bar(_Tp, enable_if_t</*...*/>* = nullptr); // # 2
-
-  template <class _Tp>
-  enable_if_t</*...*/> baz(_Tp); // # 3
-
-Using default template parameters for SFINAE (#1) should always be preferred.
-
-Option #2 has two problems. First, users can observe and accidentally pass values to the SFINAE
-function argument. Second, the default argument creates a live variable, which causes debug
-information to be emitted containing the text of the SFINAE.
-
-Option #3 can also cause more debug information to be emitted than is needed, because the function
-return type will appear in the debug information.
-
-Use ``unique_ptr`` when allocating memory
-------------------------------------------
-
-The standard library often needs to allocate memory and then construct a user type in it.
-If the users constructor throws, the library needs to deallocate that memory. The idiomatic way to
-achieve this is with ``unique_ptr``.
-
-``__builtin_new_allocator`` is an example of this idiom. Example usage would look like:
-
-.. code-block:: cpp
-
-  template <class T>
-  T* __create() {
-    using _UniquePtr = unique_ptr<void*, __default_new_allocator::__default_new_deleter>;
-    _UniquePtr __p = __default_new_allocator::__allocate_bytes(sizeof(T), alignof(T));
-    T* __res = ::new(__p.get()) T();
-    (void)__p.release();
-    return __res;
-  }
diff --git a/libcxx/docs/DesignDocs/NodiscardPolicy.rst b/libcxx/docs/DesignDocs/NodiscardPolicy.rst
deleted file mode 100644
index afbb18b0096d7..0000000000000
--- a/libcxx/docs/DesignDocs/NodiscardPolicy.rst
+++ /dev/null
@@ -1,42 +0,0 @@
-===================================================
-Guidelines for applying ``[[nodiscard]]`` in libc++
-===================================================
-
-Libc++ adds ``[[nodiscard]]`` to functions in a lot of places. The standards
-committee has decided to not have a recommended practice where to put them, so
-this document lists where ``[[nodiscard]]`` should be applied in libc++.
-
-When should ``[[nodiscard]]`` be added to functions?
-====================================================
-
-``[[nodiscard]]`` should be applied to functions
-
-- where discarding the return value is most likely a correctness issue.
-  For example a locking constructor in ``unique_lock``.
-
-- where discarding the return value likely points to the user wanting to do
-  something different. For example ``vector::empty()``, which probably should
-  have been ``vector::clear()``.
-
-  This can help spotting bugs easily which otherwise may take a very long time
-  to find.
-
-- which return a constant. For example ``numeric_limits::min()``.
-- which only observe a value. For example ``string::size()``.
-
-  Code that discards values from these kinds of functions is dead code. It can
-  either be removed, or the programmer meant to do something different.
-
-- where discarding the value is most likely a misuse of the function. For
-  example ``find``.
-
-  This protects programmers from assuming too much about how the internals of
-  a function work, making code more robust in the presence of future
-  optimizations.
-
-What should be done when adding ``[[nodiscard]]`` to a function?
-================================================================
-
-Applications of ``[[nodiscard]]`` are code like any other code, so we aim to
-test them. This can be done with a ``.verify.cpp`` test. Many examples are
-available. Just look for tests with the suffix ``.nodiscard.verify.cpp``.
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index d520c46bae1ef..a72fd2d5db9e2 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -69,6 +69,10 @@ Improvements and New Features
 - The ``_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY`` ABI configuration was added, which allows storing valid bounds
   in ``std::array::iterator`` and detecting OOB accesses when the appropriate hardening mode is enabled.
 
+- The input iterator overload of `assign(_InputIterator, _InputIterator)` in `std::vector<_Tp, _Allocator>` has been
+  optimized, resulting in a performance improvement of up to 2x for trivial element types (e.g., `std::vector<int>`),
+  and up to 3.4x for non-trivial element types (e.g., `std::vector<std::vector<int>>`).
+
 Deprecations and Removals
 -------------------------
 
@@ -105,6 +109,10 @@ Deprecations and Removals
 - The ``_LIBCPP_DISABLE_AVAILABILITY`` macro that was used to force-disable availability markup has now been removed.
   Whether availability markup is used by the library is now solely controlled at configuration-time.
 
+- The pointer safety functions ``declare_reachable``, ``declare_no_pointers``, ``undeclare_no_pointers`` and
+  ``__undeclare_reachable`` have been removed from the library. These functions were never implemented in a non-trivial
+  way, making it very unlikely that any binary depends on them.
+
 Upcoming Deprecations and Removals
 ----------------------------------
 
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 4a4fbe9bec222..24398574064e6 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -52,7 +52,7 @@
 "`P2443R1 <https://wg21.link/P2443R1>`__","``views::chunk_by``","2022-02 (Virtual)","|Complete|","18",""
 "","","","","",""
 "`P0009R18 <https://wg21.link/P0009R18>`__","mdspan: A Non-Owning Multidimensional Array Reference","2022-07 (Virtual)","|Complete|","18",""
-"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","|In progress|","",""
+"`P0429R9 <https://wg21.link/P0429R9>`__","A Standard ``flat_map``","2022-07 (Virtual)","|In Progress|","",""
 "`P1169R4 <https://wg21.link/P1169R4>`__","``static operator()``","2022-07 (Virtual)","|Complete|","16",""
 "`P1222R4 <https://wg21.link/P1222R4>`__","A Standard ``flat_set``","2022-07 (Virtual)","","",""
 "`P1223R5 <https://wg21.link/P1223R5>`__","``ranges::find_last()``, ``ranges::find_last_if()``, and ``ranges::find_last_if_not()``","2022-07 (Virtual)","|Complete|","19",""
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index a0d2ccdc34d08..45faea0568b2e 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -76,7 +76,42 @@
 "`LWG4105 <https://wg21.link/LWG4105>`__","``ranges::ends_with``\`s Returns misses difference casting","2024-06 (St. Louis)","","",""
 "`LWG4106 <https://wg21.link/LWG4106>`__","``basic_format_args`` should not be default-constructible","2024-06 (St. Louis)","|Complete|","19",""
 "","","","","",""
+"`LWG3216 <https://wg21.link/LWG3216>`__","Rebinding the allocator before calling ``construct``/``destroy`` in ``allocate_shared``","2024-11 (Wrocław)","","",""
+"`LWG3436 <https://wg21.link/LWG3436>`__","``std::construct_at`` should support arrays","2024-11 (Wrocław)","","",""
+"`LWG3886 <https://wg21.link/LWG3886>`__","Monad mo' problems","2024-11 (Wrocław)","","",""
+"`LWG3899 <https://wg21.link/LWG3899>`__","``co_yield``\ing elements of an lvalue generator is unnecessarily inefficient","2024-11 (Wrocław)","","",""
+"`LWG3900 <https://wg21.link/LWG3900>`__","The ``allocator_arg_t`` overloads of ``generator::promise_type::operator new`` should not be constrained","2024-11 (Wrocław)","","",""
+"`LWG3918 <https://wg21.link/LWG3918>`__","``std::uninitialized_move/_n`` and guaranteed copy elision","2024-11 (Wrocław)","","",""
+"`LWG4014 <https://wg21.link/LWG4014>`__","LWG 3809 changes behavior of some existing ``std::subtract_with_carry_engine code``","2024-11 (Wrocław)","","",""
+"`LWG4024 <https://wg21.link/LWG4024>`__","Underspecified destruction of objects created in ``std::make_shared_for_overwrite``/``std::allocate_shared_for_overwrite``","2024-11 (Wrocław)","","16",""
+"`LWG4027 <https://wg21.link/LWG4027>`__","``possibly-const-range`` should prefer returning ``const R&``","2024-11 (Wrocław)","","",""
+"`LWG4044 <https://wg21.link/LWG4044>`__","Confusing requirements for ``std::print`` on POSIX platforms","2024-11 (Wrocław)","","",""
+"`LWG4064 <https://wg21.link/LWG4064>`__","Clarify that ``std::launder`` is not needed when using the result of ``std::memcpy``","2024-11 (Wrocław)","","",""
+"`LWG4072 <https://wg21.link/LWG4072>`__","``std::optional`` comparisons: constrain harder","2024-11 (Wrocław)","","",""
+"`LWG4084 <https://wg21.link/LWG4084>`__","``std::fixed`` ignores ``std::uppercase``","2024-11 (Wrocław)","","",""
+"`LWG4085 <https://wg21.link/LWG4085>`__","``ranges::generate_random``'s helper lambda should specify the return type","2024-11 (Wrocław)","","",""
+"`LWG4088 <https://wg21.link/LWG4088>`__","``println`` ignores the locale imbued in ``std::ostream``","2024-11 (Wrocław)","","",""
+"`LWG4112 <https://wg21.link/LWG4112>`__","``has-arrow`` should required ``operator->()`` to be ``const``-qualified","2024-11 (Wrocław)","","",""
+"`LWG4113 <https://wg21.link/LWG4113>`__","Disallow ``has_unique_object_representations<Incomplete[]>``","2024-11 (Wrocław)","","",""
+"`LWG4119 <https://wg21.link/LWG4119>`__","``generator::promise_type::yield_value(ranges::elements_of<R, Alloc>)``'s nested ``generator`` may be ill-formed","2024-11 (Wrocław)","","",""
+"`LWG4124 <https://wg21.link/LWG4124>`__","Cannot format ``zoned_time`` with resolution coarser than ``seconds``","2024-11 (Wrocław)","","",""
+"`LWG4126 <https://wg21.link/LWG4126>`__","Some feature-test macros for fully freestanding features are not yet marked freestanding","2024-11 (Wrocław)","","",""
+"`LWG4134 <https://wg21.link/LWG4134>`__","Issue with Philox algorithm specification","2024-11 (Wrocław)","","",""
+"`LWG4135 <https://wg21.link/LWG4135>`__","The helper lambda of ``std::erase`` for list should specify return type as ``bool``","2024-11 (Wrocław)","","",""
+"`LWG4140 <https://wg21.link/LWG4140>`__","Useless default constructors for bit reference types","2024-11 (Wrocław)","","",""
+"`LWG4141 <https://wg21.link/LWG4141>`__","Improve prohibitions on ""additional storage""","2024-11 (Wrocław)","","",""
+"`LWG4142 <https://wg21.link/LWG4142>`__","``format_parse_context::check_dynamic_spec`` should require at least one type","2024-11 (Wrocław)","","",""
+"`LWG4144 <https://wg21.link/LWG4144>`__","Disallow ``unique_ptr<T&, D>``","2024-11 (Wrocław)","","",""
+"`LWG4147 <https://wg21.link/LWG4147>`__","Precondition on ``inplace_vector::emplace``","2024-11 (Wrocław)","","",""
+"`LWG4148 <https://wg21.link/LWG4148>`__","``unique_ptr::operator*`` should not allow dangling references","2024-11 (Wrocław)","","",""
+"`LWG4153 <https://wg21.link/LWG4153>`__","Fix extra ""-1"" for ``philox_engine::max()``","2024-11 (Wrocław)","","",""
+"`LWG4154 <https://wg21.link/LWG4154>`__","The Mandates for ``std::packaged_task``'s constructor from a callable entity should consider decaying","2024-11 (Wrocław)","","",""
+"`LWG4157 <https://wg21.link/LWG4157>`__","The resolution of LWG3465 was damaged by P2167R3","2024-11 (Wrocław)","","20",""
+"`LWG4164 <https://wg21.link/LWG4164>`__","Missing guarantees for ``forward_list`` modifiers","2024-11 (Wrocław)","","",""
+"`LWG4169 <https://wg21.link/LWG4169>`__","``std::atomic<T>``'s default constructor should be constrained","2024-11 (Wrocław)","","",""
+"`LWG4170 <https://wg21.link/LWG4170>`__","``contiguous_iterator`` should require ``to_address(I{})``","2024-11 (Wrocław)","","",""
+"","","","","",""
 "`LWG3343 <https://wg21.link/LWG3343>`__","Ordering of calls to ``unlock()`` and ``notify_all()`` in Effects element of ``notify_all_at_thread_exit()`` should be reversed","Not Adopted Yet","|Complete|","16",""
 "`LWG4139 <https://wg21.link/LWG4139>`__","§[time.zone.leap] recursive constraint in <=>","Not Adopted Yet","|Complete|","20",""
-"`LWG3456 <https://wg21.link/LWG3456>`__","Pattern used by std::from_chars is underspecified (option B)",,"Not Yet Adopted","|Complete|","20",""
+"`LWG3456 <https://wg21.link/LWG3456>`__","Pattern used by std::from_chars is underspecified (option B)","Not Adopted Yet","|Complete|","20",""
 "","","","","",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 6659d815612d0..3a8a794ca4ea1 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -76,3 +76,25 @@
 "`P2422R1 <https://wg21.link/P2422R1>`__","Remove ``nodiscard`` annotations from the standard library specification","2024-06 (St. Louis)","|Complete|","19","``nodiscard`` attributes were kept as a conforming extension"
 "`P2300R10 <https://wg21.link/P2300R10>`__","``std::execution``","2024-06 (St. Louis)","","",""
 "","","","","",""
+"`P3136R1 <https://wg21.link/P3136R1>`__","Retiring niebloids","2024-11 (Wrocław)","","",""
+"`P3138R5 <https://wg21.link/P3138R5>`__","``views::cache_latest``","2024-11 (Wrocław)","","",""
+"`P3379R0 <https://wg21.link/P3379R0>`__","Constrain ``std::expected`` equality operators","2024-11 (Wrocław)","","",""
+"`P0472R2 <https://wg21.link/P0472R2>`__","Put ``std::monostate`` in ``<utility>``","2024-11 (Wrocław)","","",""
+"`P2862R1 <https://wg21.link/P2862R1>`__","``text_encoding::name()`` should never return null values","2024-11 (Wrocław)","","",""
+"`P2897R7 <https://wg21.link/P2897R7>`__","``aligned_accessor``: An ``mdspan`` accessor expressing pointer over-alignment","2024-11 (Wrocław)","","",""
+"`P3355R1 <https://wg21.link/P3355R1>`__","Fix ``submdspan`` for C++26","2024-11 (Wrocław)","","",""
+"`P3222R0 <https://wg21.link/P3222R0>`__","Fix C++26 by adding transposed special cases for P2642 layouts","2024-11 (Wrocław)","","",""
+"`P3050R2 <https://wg21.link/P3050R2>`__","Fix C++26 by optimizing ``linalg::conjugated`` for noncomplex value types","2024-11 (Wrocław)","","",""
+"`P3396R1 <https://wg21.link/P3396R1>`__","``std::execution`` wording fixes","2024-11 (Wrocław)","","",""
+"`P2835R7 <https://wg21.link/P2835R7>`__","Expose ``std::atomic_ref``'s object address","2024-11 (Wrocław)","","",""
+"`P3323R1 <https://wg21.link/P3323R1>`__","cv-qualified types in ``atomic`` and ``atomic_ref``","2024-11 (Wrocław)","","",""
+"`P3508R0 <https://wg21.link/P3508R0>`__","Wording for ""constexpr for specialized memory algorithms""","2024-11 (Wrocław)","","",""
+"`P3369R0 <https://wg21.link/P3369R0>`__","constexpr for ``uninitialized_default_construct``","2024-11 (Wrocław)","","",""
+"`P3370R1 <https://wg21.link/P3370R1>`__","Add new library headers from C23","2024-11 (Wrocław)","","",""
+"`P3309R3 <https://wg21.link/P3309R3>`__","constexpr ``atomic`` and ``atomic_ref``","2024-11 (Wrocław)","","",""
+"`P3019R11 <https://wg21.link/P3019R11>`__","``indirect`` and ``polymorphic``: Vocabulary Types for Composite Class Design","2024-11 (Wrocław)","","",""
+"`P1928R15 <https://wg21.link/P1928R15>`__","``std::simd`` — merge data-parallel types from the Parallelism TS 2","2024-11 (Wrocław)","","",""
+"`P3325R5 <https://wg21.link/P3325R5>`__","A Utility for Creating Execution Environments","2024-11 (Wrocław)","","",""
+"`P3068R6 <https://wg21.link/P3068R6>`__","Allowing exception throwing in constant-evaluation","2024-11 (Wrocław)","","",""
+"`P3247R2 <https://wg21.link/P3247R2>`__","Deprecate the notion of trivial types","2024-11 (Wrocław)","","",""
+"","","","","",""
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index 6d2c954489f62..cf092fabd046f 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -452,12 +452,7 @@ when running the benchmarks. For example,
 
 .. code-block:: bash
 
-  $ libcxx/utils/libcxx-lit <build> -sv libcxx/test/benchmarks/string.bench.cpp --param optimization=speed
-
-If you want to see where a benchmark is located (e.g. you want to store the executable
-for subsequent analysis), you can print that information by passing ``--show-all`` to
-``lit``. That will print the command-lines being executed, which includes the location
-of the executable created for that benchmark.
+  $ libcxx/utils/libcxx-lit <build> libcxx/test/benchmarks/string.bench.cpp --show-all --param optimization=speed
 
 Note that benchmarks are only dry-run when run via the ``check-cxx`` target since
 we only want to make sure they don't rot. Do not rely on the results of benchmarks
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index a9610cbb4db3a..37912e9f8d03e 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -38,6 +38,7 @@ Getting Started with libc++
    UserDocumentation
    VendorDocumentation
    Contributing
+   CodingGuidelines
    TestingLibcxx
    ImplementationDefinedBehavior
    Modules
@@ -216,7 +217,6 @@ Design Documents
    DesignDocs/FeatureTestMacros
    DesignDocs/FileTimeType
    DesignDocs/HeaderRemovalPolicy
-   DesignDocs/NodiscardPolicy
    DesignDocs/NoexceptPolicy
    DesignDocs/PSTLIntegration
    DesignDocs/ThreadingSupportAPI
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 2bb6d263340d3..561ce4b42fb25 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -212,11 +212,13 @@ set(files
   __atomic/atomic_sync.h
   __atomic/check_memory_order.h
   __atomic/contention_t.h
-  __atomic/cxx_atomic_impl.h
   __atomic/fence.h
   __atomic/is_always_lock_free.h
   __atomic/kill_dependency.h
   __atomic/memory_order.h
+  __atomic/support.h
+  __atomic/support/c11.h
+  __atomic/support/gcc.h
   __atomic/to_gcc_order.h
   __bit/bit_cast.h
   __bit/bit_ceil.h
@@ -358,6 +360,7 @@ set(files
   __filesystem/space_info.h
   __filesystem/u8path.h
   __flat_map/flat_map.h
+  __flat_map/key_value_iterator.h
   __flat_map/sorted_unique.h
   __format/buffer.h
   __format/concepts.h
diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h
index ae0475693f22b..8029b52770d26 100644
--- a/libcxx/include/__atomic/atomic.h
+++ b/libcxx/include/__atomic/atomic.h
@@ -11,12 +11,11 @@
 
 #include <__atomic/atomic_sync.h>
 #include <__atomic/check_memory_order.h>
-#include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/is_always_lock_free.h>
 #include <__atomic/memory_order.h>
+#include <__atomic/support.h>
 #include <__config>
 #include <__cstddef/ptrdiff_t.h>
-#include <__functional/operations.h>
 #include <__memory/addressof.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_floating_point.h>
@@ -376,7 +375,8 @@ struct atomic<_Tp> : __atomic_base<_Tp> {
     auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
       return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
     };
-    return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op);
+    auto __plus = [](auto __a, auto __b) { return __a + __b; };
+    return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);
   }
 
   template <class _This>
@@ -384,7 +384,8 @@ struct atomic<_Tp> : __atomic_base<_Tp> {
     auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
       return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
     };
-    return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op);
+    auto __minus = [](auto __a, auto __b) { return __a - __b; };
+    return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);
   }
 
 public:
diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h
index abebfc112cb8e..5cc6fb0c55d09 100644
--- a/libcxx/include/__atomic/atomic_flag.h
+++ b/libcxx/include/__atomic/atomic_flag.h
@@ -11,8 +11,8 @@
 
 #include <__atomic/atomic_sync.h>
 #include <__atomic/contention_t.h>
-#include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
+#include <__atomic/support.h>
 #include <__chrono/duration.h>
 #include <__config>
 #include <__memory/addressof.h>
diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h
index 08f3497fef9f4..153001e7b62e3 100644
--- a/libcxx/include/__atomic/atomic_sync.h
+++ b/libcxx/include/__atomic/atomic_sync.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
 
 #include <__atomic/contention_t.h>
-#include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
 #include <__atomic/to_gcc_order.h>
 #include <__chrono/duration.h>
@@ -57,19 +56,8 @@ struct __atomic_waitable< _Tp,
                                    decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
                                        std::declval<const _Tp&>()))> > : true_type {};
 
-template <class _AtomicWaitable, class _Poll>
-struct __atomic_wait_poll_impl {
-  const _AtomicWaitable& __a_;
-  _Poll __poll_;
-  memory_order __order_;
-
-  _LIBCPP_HIDE_FROM_ABI bool operator()() const {
-    auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a_, __order_);
-    return __poll_(__current_val);
-  }
-};
-
-#if _LIBCPP_HAS_THREADS
+#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_HAS_THREADS
 
 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
@@ -144,11 +132,16 @@ struct __atomic_wait_backoff_impl {
 // value. The predicate function must not return `false` spuriously.
 template <class _AtomicWaitable, class _Poll>
 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
-__atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
+__atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
-  __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_impl     = {__a, __poll, __order};
   __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
-  std::__libcpp_thread_poll_with_backoff(__poll_impl, __backoff_fn);
+  std::__libcpp_thread_poll_with_backoff(
+      /* poll */
+      [&]() {
+        auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
+        return __poll(__current_val);
+      },
+      /* backoff */ __backoff_fn);
 }
 
 template <class _AtomicWaitable>
@@ -163,12 +156,17 @@ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _
   std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
 }
 
-#else // _LIBCPP_HAS_THREADS
+#  else // _LIBCPP_HAS_THREADS
 
 template <class _AtomicWaitable, class _Poll>
-_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
-  __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order};
-  std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy());
+_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
+  std::__libcpp_thread_poll_with_backoff(
+      /* poll */
+      [&]() {
+        auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
+        return __poll(__current_val);
+      },
+      /* backoff */ __spinning_backoff_policy());
 }
 
 template <class _AtomicWaitable>
@@ -177,29 +175,24 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {}
 template <class _AtomicWaitable>
 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {}
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
   return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0;
 }
 
-template <class _Tp>
-struct __atomic_compare_unequal_to {
-  _Tp __val_;
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __arg) const {
-    return !std::__cxx_nonatomic_compare_equal(__arg, __val_);
-  }
-};
-
-template <class _AtomicWaitable, class _Up>
+template <class _AtomicWaitable, class _Tp>
 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
-__atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) {
+__atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) {
   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
-  __atomic_compare_unequal_to<_Up> __nonatomic_equal = {__val};
-  std::__atomic_wait_unless(__a, __nonatomic_equal, __order);
+  std::__atomic_wait_unless(__a, __order, [&](_Tp const& __current) {
+    return !std::__cxx_nonatomic_compare_equal(__current, __val);
+  });
 }
 
+#endif // C++20
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H
diff --git a/libcxx/include/__atomic/contention_t.h b/libcxx/include/__atomic/contention_t.h
index 65890f338ce99..6f2a073bc1a8e 100644
--- a/libcxx/include/__atomic/contention_t.h
+++ b/libcxx/include/__atomic/contention_t.h
@@ -9,7 +9,7 @@
 #ifndef _LIBCPP___ATOMIC_CONTENTION_T_H
 #define _LIBCPP___ATOMIC_CONTENTION_T_H
 
-#include <__atomic/cxx_atomic_impl.h>
+#include <__atomic/support.h>
 #include <__config>
 #include <cstdint>
 
diff --git a/libcxx/include/__atomic/fence.h b/libcxx/include/__atomic/fence.h
index 8c27ea54d62dd..0a63cedddb3f9 100644
--- a/libcxx/include/__atomic/fence.h
+++ b/libcxx/include/__atomic/fence.h
@@ -9,8 +9,8 @@
 #ifndef _LIBCPP___ATOMIC_FENCE_H
 #define _LIBCPP___ATOMIC_FENCE_H
 
-#include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
+#include <__atomic/support.h>
 #include <__config>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__atomic/support.h b/libcxx/include/__atomic/support.h
new file mode 100644
index 0000000000000..4b555ab483ca0
--- /dev/null
+++ b/libcxx/include/__atomic/support.h
@@ -0,0 +1,124 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_SUPPORT_H
+#define _LIBCPP___ATOMIC_SUPPORT_H
+
+#include <__config>
+#include <__type_traits/is_trivially_copyable.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+//
+// This file implements base support for atomics on the platform.
+//
+// The following operations and types must be implemented (where _Atmc
+// is __cxx_atomic_base_impl for readability):
+//
+// clang-format off
+//
+// template <class _Tp>
+// struct __cxx_atomic_base_impl;
+//
+// #define __cxx_atomic_is_lock_free(__size)
+//
+// void __cxx_atomic_thread_fence(memory_order __order) noexcept;
+// void __cxx_atomic_signal_fence(memory_order __order) noexcept;
+//
+// template <class _Tp>
+// void __cxx_atomic_init(_Atmc<_Tp> volatile* __a, _Tp __val) noexcept;
+// template <class _Tp>
+// void __cxx_atomic_init(_Atmc<_Tp>* __a, _Tp __val) noexcept;
+//
+// template <class _Tp>
+// void __cxx_atomic_store(_Atmc<_Tp> volatile* __a, _Tp __val, memory_order __order) noexcept;
+// template <class _Tp>
+// void __cxx_atomic_store(_Atmc<_Tp>* __a, _Tp __val, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_load(_Atmc<_Tp> const volatile* __a, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_load(_Atmc<_Tp> const* __a, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// void __cxx_atomic_load_inplace(_Atmc<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) noexcept;
+// template <class _Tp>
+// void __cxx_atomic_load_inplace(_Atmc<_Tp> const* __a, _Tp* __dst, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_exchange(_Atmc<_Tp> volatile* __a, _Tp __value, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_exchange(_Atmc<_Tp>* __a, _Tp __value, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// bool __cxx_atomic_compare_exchange_strong(_Atmc<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept;
+// template <class _Tp>
+// bool __cxx_atomic_compare_exchange_strong(_Atmc<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept;
+//
+// template <class _Tp>
+// bool __cxx_atomic_compare_exchange_weak(_Atmc<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept;
+// template <class _Tp>
+// bool __cxx_atomic_compare_exchange_weak(_Atmc<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_add(_Atmc<_Tp> volatile* __a, _Tp __delta, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_add(_Atmc<_Tp>* __a, _Tp __delta, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp* __cxx_atomic_fetch_add(_Atmc<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp* __cxx_atomic_fetch_add(_Atmc<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_sub(_Atmc<_Tp> volatile* __a, _Tp __delta, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_sub(_Atmc<_Tp>* __a, _Tp __delta, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp* __cxx_atomic_fetch_sub(_Atmc<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp* __cxx_atomic_fetch_sub(_Atmc<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_and(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_and(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept;
+//
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_or(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_or(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_xor(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept;
+// template <class _Tp>
+// _Tp __cxx_atomic_fetch_xor(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept;
+//
+// clang-format on
+//
+
+#if _LIBCPP_HAS_GCC_ATOMIC_IMP
+#  include <__atomic/support/gcc.h>
+#elif _LIBCPP_HAS_C_ATOMIC_IMP
+#  include <__atomic/support/c11.h>
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
+struct __cxx_atomic_impl : public _Base {
+  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
+
+  _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_SUPPORT_H
diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/support/c11.h
similarity index 52%
rename from libcxx/include/__atomic/cxx_atomic_impl.h
rename to libcxx/include/__atomic/support/c11.h
index acffb0fa8f5d9..177a075be4073 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/support/c11.h
@@ -6,276 +6,39 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
-#define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
+#define _LIBCPP___ATOMIC_SUPPORT_C11_H
 
 #include <__atomic/memory_order.h>
-#include <__atomic/to_gcc_order.h>
 #include <__config>
 #include <__cstddef/ptrdiff_t.h>
 #include <__memory/addressof.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_assignable.h>
-#include <__type_traits/is_trivially_copyable.h>
 #include <__type_traits/remove_const.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#if _LIBCPP_HAS_GCC_ATOMIC_IMP
-
-// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
-// the default operator= in an object is not volatile, a byte-by-byte copy
-// is required.
-template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
-  __a_value = __val;
-}
-template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
-  volatile char* __to         = reinterpret_cast<volatile char*>(std::addressof(__a_value));
-  volatile char* __end        = __to + sizeof(_Tp);
-  volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val));
-  while (__to != __end)
-    *__to++ = *__from++;
-}
-
-template <typename _Tp>
-struct __cxx_atomic_base_impl {
-  _LIBCPP_HIDE_FROM_ABI
-#  ifndef _LIBCPP_CXX03_LANG
-  __cxx_atomic_base_impl() _NOEXCEPT = default;
-#  else
-  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
-  }
-#  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
-  _Tp __a_value;
-};
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
-  __cxx_atomic_assign_volatile(__a->__a_value, __val);
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
-  __a->__a_value = __val;
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) {
-  __atomic_thread_fence(__to_gcc_order(__order));
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) {
-  __atomic_signal_fence(__to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
-  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
-  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
-  _Tp __ret;
-  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
-  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI void
-__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
-  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
-  _Tp __ret;
-  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
-  _Tp __ret;
-  __atomic_exchange(
-      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
-  _Tp __ret;
-  __atomic_exchange(
-      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
-  return __ret;
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    volatile __cxx_atomic_base_impl<_Tp>* __a,
-    _Tp* __expected,
-    _Tp __value,
-    memory_order __success,
-    memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      false,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      false,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    volatile __cxx_atomic_base_impl<_Tp>* __a,
-    _Tp* __expected,
-    _Tp __value,
-    memory_order __success,
-    memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      true,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
-    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      true,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
-}
-
-template <typename _Tp>
-struct __skip_amt {
-  enum { value = 1 };
-};
-
-template <typename _Tp>
-struct __skip_amt<_Tp*> {
-  enum { value = sizeof(_Tp) };
-};
-
-// FIXME: Haven't figured out what the spec says about using arrays with
-// atomic_fetch_add. Force a failure rather than creating bad behavior.
-template <typename _Tp>
-struct __skip_amt<_Tp[]> {};
-template <typename _Tp, int n>
-struct __skip_amt<_Tp[n]> {};
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp, typename _Td>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
-  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-template <typename _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
-  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
-}
-
-#  define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
+//
+// This file implements support for C11-style atomics
+//
 
-#elif _LIBCPP_HAS_C_ATOMIC_IMP
+_LIBCPP_BEGIN_NAMESPACE_STD
 
 template <typename _Tp>
 struct __cxx_atomic_base_impl {
   _LIBCPP_HIDE_FROM_ABI
-#  ifndef _LIBCPP_CXX03_LANG
+#ifndef _LIBCPP_CXX03_LANG
   __cxx_atomic_base_impl() _NOEXCEPT = default;
-#  else
+#else
   __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
   }
-#  endif // _LIBCPP_CXX03_LANG
+#endif // _LIBCPP_CXX03_LANG
   _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
   _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
 };
 
-#  define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
+#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
 
 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
   __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
@@ -496,16 +259,6 @@ __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_o
       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 }
 
-#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
-
-template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
-struct __cxx_atomic_impl : public _Base {
-  static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
-
-  _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
-};
-
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H
diff --git a/libcxx/include/__atomic/support/gcc.h b/libcxx/include/__atomic/support/gcc.h
new file mode 100644
index 0000000000000..73c1b1c8070a4
--- /dev/null
+++ b/libcxx/include/__atomic/support/gcc.h
@@ -0,0 +1,265 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ATOMIC_SUPPORT_GCC_H
+#define _LIBCPP___ATOMIC_SUPPORT_GCC_H
+
+#include <__atomic/memory_order.h>
+#include <__atomic/to_gcc_order.h>
+#include <__config>
+#include <__memory/addressof.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/remove_const.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+//
+// This file implements support for GCC-style atomics
+//
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
+// the default operator= in an object is not volatile, a byte-by-byte copy
+// is required.
+template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
+  __a_value = __val;
+}
+template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
+  volatile char* __to         = reinterpret_cast<volatile char*>(std::addressof(__a_value));
+  volatile char* __end        = __to + sizeof(_Tp);
+  volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val));
+  while (__to != __end)
+    *__to++ = *__from++;
+}
+
+template <typename _Tp>
+struct __cxx_atomic_base_impl {
+  _LIBCPP_HIDE_FROM_ABI
+#ifndef _LIBCPP_CXX03_LANG
+  __cxx_atomic_base_impl() _NOEXCEPT = default;
+#else
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+  }
+#endif // _LIBCPP_CXX03_LANG
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
+  _Tp __a_value;
+};
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+  __cxx_atomic_assign_volatile(__a->__a_value, __val);
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
+  __a->__a_value = __val;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) {
+  __atomic_thread_fence(__to_gcc_order(__order));
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) {
+  __atomic_signal_fence(__to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
+  _Tp __ret;
+  __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
+  __atomic_exchange(
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
+  _Tp __ret;
+  __atomic_exchange(
+      std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
+  return __ret;
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      false,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      false,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    volatile __cxx_atomic_base_impl<_Tp>* __a,
+    _Tp* __expected,
+    _Tp __value,
+    memory_order __success,
+    memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      true,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
+    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
+  return __atomic_compare_exchange(
+      std::addressof(__a->__a_value),
+      __expected,
+      std::addressof(__value),
+      true,
+      __to_gcc_order(__success),
+      __to_gcc_failure_order(__failure));
+}
+
+template <typename _Tp>
+struct __skip_amt {
+  enum { value = 1 };
+};
+
+template <typename _Tp>
+struct __skip_amt<_Tp*> {
+  enum { value = sizeof(_Tp) };
+};
+
+// FIXME: Haven't figured out what the spec says about using arrays with
+// atomic_fetch_add. Force a failure rather than creating bad behavior.
+template <typename _Tp>
+struct __skip_amt<_Tp[]> {};
+template <typename _Tp, int n>
+struct __skip_amt<_Tp[n]> {};
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp, typename _Td>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
+  return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
+  return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
+}
+
+#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ATOMIC_SUPPORT_GCC_H
diff --git a/libcxx/include/__configuration/availability.h b/libcxx/include/__configuration/availability.h
index efda2a04a4841..261cf9c1ae9d8 100644
--- a/libcxx/include/__configuration/availability.h
+++ b/libcxx/include/__configuration/availability.h
@@ -87,9 +87,6 @@
 #  define _LIBCPP_INTRODUCED_IN_LLVM_18 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE /* nothing */
 
-#  define _LIBCPP_INTRODUCED_IN_LLVM_17 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_17_ATTRIBUTE /* nothing */
-
 #  define _LIBCPP_INTRODUCED_IN_LLVM_16 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE /* nothing */
 
@@ -99,26 +96,17 @@
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE /* nothing */
 
-#  define _LIBCPP_INTRODUCED_IN_LLVM_13 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_13_ATTRIBUTE /* nothing */
-
 #  define _LIBCPP_INTRODUCED_IN_LLVM_12 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_11 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE /* nothing */
 
-#  define _LIBCPP_INTRODUCED_IN_LLVM_10 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE /* nothing */
-
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE      /* nothing */
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP  /* nothing */
 
-#  define _LIBCPP_INTRODUCED_IN_LLVM_8 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_8_ATTRIBUTE /* nothing */
-
 #  define _LIBCPP_INTRODUCED_IN_LLVM_4 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE /* nothing */
 
@@ -155,25 +143,6 @@
     __attribute__((availability(bridgeos, strict, introduced = 9.0)))                                             \
     __attribute__((availability(driverkit, strict, introduced = 24.0)))
 
-// LLVM 17
-#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140400) ||       \
-      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170400) ||     \
-      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 170400) ||             \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100400) ||       \
-      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 80400) ||      \
-      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 230400)
-#    define _LIBCPP_INTRODUCED_IN_LLVM_17 0
-#  else
-#    define _LIBCPP_INTRODUCED_IN_LLVM_17 1
-#  endif
-#  define _LIBCPP_INTRODUCED_IN_LLVM_17_ATTRIBUTE                                                                 \
-    __attribute__((availability(macos, strict, introduced = 14.4)))                                               \
-    __attribute__((availability(ios, strict, introduced = 17.4)))                                                 \
-    __attribute__((availability(tvos, strict, introduced = 17.4)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 10.4)))                                             \
-    __attribute__((availability(bridgeos, strict, introduced = 8.4)))                                             \
-    __attribute__((availability(driverkit, strict, introduced = 23.4)))
-
 // LLVM 16
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140000) ||       \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170000) ||     \
@@ -216,25 +185,6 @@
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14 _LIBCPP_INTRODUCED_IN_LLVM_15
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE
 
-// LLVM 13
-#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 130000) ||   \
-      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 160000) || \
-      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 160000) ||         \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90000) ||    \
-      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 70000) ||  \
-      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 220000)
-#    define _LIBCPP_INTRODUCED_IN_LLVM_13 0
-#  else
-#    define _LIBCPP_INTRODUCED_IN_LLVM_13 1
-#  endif
-#  define _LIBCPP_INTRODUCED_IN_LLVM_13_ATTRIBUTE                                                                 \
-    __attribute__((availability(macos, strict, introduced = 13.0)))                                               \
-    __attribute__((availability(ios, strict, introduced = 16.0)))                                                 \
-    __attribute__((availability(tvos, strict, introduced = 16.0)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 9.0)))                                              \
-    __attribute__((availability(bridgeos, strict, introduced = 7.0)))                                             \
-    __attribute__((availability(driverkit, strict, introduced = 22.0)))
-
 // LLVM 12
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 120300)   ||     \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 150300) ||     \
@@ -269,10 +219,6 @@
     __attribute__((availability(tvos, strict, introduced = 14.0)))                                                \
     __attribute__((availability(watchos, strict, introduced = 7.0)))
 
-// LLVM 10
-#  define _LIBCPP_INTRODUCED_IN_LLVM_10 _LIBCPP_INTRODUCED_IN_LLVM_11
-#  define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE
-
 // LLVM 9
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500) ||   \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000) || \
diff --git a/libcxx/include/__cxx03/__algorithm/adjacent_find.h b/libcxx/include/__cxx03/__algorithm/adjacent_find.h
index 88036db84de89..6add0f3fe2b53 100644
--- a/libcxx/include/__cxx03/__algorithm/adjacent_find.h
+++ b/libcxx/include/__cxx03/__algorithm/adjacent_find.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ADJACENT_FIND_H
-#define _LIBCPP___ALGORITHM_ADJACENT_FIND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ADJACENT_FIND_H
+#define _LIBCPP___CXX03___ALGORITHM_ADJACENT_FIND_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -55,4 +55,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_ADJACENT_FIND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ADJACENT_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/all_of.h b/libcxx/include/__cxx03/__algorithm/all_of.h
index b32d97241506e..fe46ee5fca43c 100644
--- a/libcxx/include/__cxx03/__algorithm/all_of.h
+++ b/libcxx/include/__cxx03/__algorithm/all_of.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ALL_OF_H
-#define _LIBCPP___ALGORITHM_ALL_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ALL_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_ALL_OF_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_ALL_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ALL_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/any_of.h b/libcxx/include/__cxx03/__algorithm/any_of.h
index bd7de96fbef40..26bf3996e8a6f 100644
--- a/libcxx/include/__cxx03/__algorithm/any_of.h
+++ b/libcxx/include/__cxx03/__algorithm/any_of.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ANY_OF_H
-#define _LIBCPP___ALGORITHM_ANY_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ANY_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_ANY_OF_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ any_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_ANY_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/binary_search.h b/libcxx/include/__cxx03/__algorithm/binary_search.h
index a93c62b32d742..a72da8e396639 100644
--- a/libcxx/include/__cxx03/__algorithm/binary_search.h
+++ b/libcxx/include/__cxx03/__algorithm/binary_search.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_BINARY_SEARCH_H
-#define _LIBCPP___ALGORITHM_BINARY_SEARCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_BINARY_SEARCH_H
+#define _LIBCPP___CXX03___ALGORITHM_BINARY_SEARCH_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -36,4 +36,4 @@ binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __va
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_BINARY_SEARCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_BINARY_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/clamp.h b/libcxx/include/__cxx03/__algorithm/clamp.h
index e5e4dbf774dc3..0e4bb0eb6b950 100644
--- a/libcxx/include/__cxx03/__algorithm/clamp.h
+++ b/libcxx/include/__cxx03/__algorithm/clamp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_CLAMP_H
-#define _LIBCPP___ALGORITHM_CLAMP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_CLAMP_H
+#define _LIBCPP___CXX03___ALGORITHM_CLAMP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__assert>
@@ -41,4 +41,4 @@ clamp(_LIBCPP_LIFETIMEBOUND const _Tp& __v,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_CLAMP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_CLAMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/comp.h b/libcxx/include/__cxx03/__algorithm/comp.h
index f01e395dc21ea..0c638b4e4a651 100644
--- a/libcxx/include/__cxx03/__algorithm/comp.h
+++ b/libcxx/include/__cxx03/__algorithm/comp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COMP_H
-#define _LIBCPP___ALGORITHM_COMP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COMP_H
+#define _LIBCPP___CXX03___ALGORITHM_COMP_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/desugars_to.h>
@@ -46,4 +46,4 @@ inline const bool __desugars_to_v<__less_tag, __less<>, _Tp, _Tp> = true;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_COMP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/comp_ref_type.h b/libcxx/include/__cxx03/__algorithm/comp_ref_type.h
index 413bf30c6a64c..ab793da0ad293 100644
--- a/libcxx/include/__cxx03/__algorithm/comp_ref_type.h
+++ b/libcxx/include/__cxx03/__algorithm/comp_ref_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
-#define _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COMP_REF_TYPE_H
+#define _LIBCPP___CXX03___ALGORITHM_COMP_REF_TYPE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -64,4 +64,4 @@ using __comp_ref_type = _Comp&;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_COMP_REF_TYPE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COMP_REF_TYPE_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy.h b/libcxx/include/__cxx03/__algorithm/copy.h
index 6de8df3f980fe..2aa0ab78b7858 100644
--- a/libcxx/include/__cxx03/__algorithm/copy.h
+++ b/libcxx/include/__cxx03/__algorithm/copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COPY_H
-#define _LIBCPP___ALGORITHM_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_COPY_H
 
 #include <__cxx03/__algorithm/copy_move_common.h>
 #include <__cxx03/__algorithm/for_each_segment.h>
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_backward.h b/libcxx/include/__cxx03/__algorithm/copy_backward.h
index dd7ff8ada5280..9262d13d6c175 100644
--- a/libcxx/include/__cxx03/__algorithm/copy_backward.h
+++ b/libcxx/include/__cxx03/__algorithm/copy_backward.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COPY_BACKWARD_H
-#define _LIBCPP___ALGORITHM_COPY_BACKWARD_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COPY_BACKWARD_H
+#define _LIBCPP___CXX03___ALGORITHM_COPY_BACKWARD_H
 
 #include <__cxx03/__algorithm/copy_move_common.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -134,4 +134,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_COPY_BACKWARD_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COPY_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_if.h b/libcxx/include/__cxx03/__algorithm/copy_if.h
index 345b12878d333..2db0c26fb86be 100644
--- a/libcxx/include/__cxx03/__algorithm/copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COPY_IF_H
-#define _LIBCPP___ALGORITHM_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_COPY_IF_H
 
 #include <__cxx03/__config>
 
@@ -31,4 +31,4 @@ copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_move_common.h b/libcxx/include/__cxx03/__algorithm/copy_move_common.h
index c598307025176..637b5a01daa75 100644
--- a/libcxx/include/__cxx03/__algorithm/copy_move_common.h
+++ b/libcxx/include/__cxx03/__algorithm/copy_move_common.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
-#define _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COPY_MOVE_COMMON_H
+#define _LIBCPP___CXX03___ALGORITHM_COPY_MOVE_COMMON_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/unwrap_iter.h>
@@ -111,4 +111,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COPY_MOVE_COMMON_H
diff --git a/libcxx/include/__cxx03/__algorithm/copy_n.h b/libcxx/include/__cxx03/__algorithm/copy_n.h
index 14f1402944335..aedb232b1bd5e 100644
--- a/libcxx/include/__cxx03/__algorithm/copy_n.h
+++ b/libcxx/include/__cxx03/__algorithm/copy_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COPY_N_H
-#define _LIBCPP___ALGORITHM_COPY_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COPY_N_H
+#define _LIBCPP___CXX03___ALGORITHM_COPY_N_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__config>
@@ -57,4 +57,4 @@ copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_COPY_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COPY_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/count.h b/libcxx/include/__cxx03/__algorithm/count.h
index b7024dc8c6318..7c1fc3e579898 100644
--- a/libcxx/include/__cxx03/__algorithm/count.h
+++ b/libcxx/include/__cxx03/__algorithm/count.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COUNT_H
-#define _LIBCPP___ALGORITHM_COUNT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COUNT_H
+#define _LIBCPP___CXX03___ALGORITHM_COUNT_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/min.h>
@@ -89,4 +89,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_COUNT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COUNT_H
diff --git a/libcxx/include/__cxx03/__algorithm/count_if.h b/libcxx/include/__cxx03/__algorithm/count_if.h
index eeb42052b08ae..d333e86189176 100644
--- a/libcxx/include/__cxx03/__algorithm/count_if.h
+++ b/libcxx/include/__cxx03/__algorithm/count_if.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_COUNT_IF_H
-#define _LIBCPP___ALGORITHM_COUNT_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_COUNT_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_COUNT_IF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -32,4 +32,4 @@ count_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_COUNT_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_COUNT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/equal.h b/libcxx/include/__cxx03/__algorithm/equal.h
index a4d0a999b1819..4e478932981e7 100644
--- a/libcxx/include/__cxx03/__algorithm/equal.h
+++ b/libcxx/include/__cxx03/__algorithm/equal.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_EQUAL_H
-#define _LIBCPP___ALGORITHM_EQUAL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_EQUAL_H
+#define _LIBCPP___CXX03___ALGORITHM_EQUAL_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/unwrap_iter.h>
@@ -130,4 +130,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_EQUAL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_EQUAL_H
diff --git a/libcxx/include/__cxx03/__algorithm/equal_range.h b/libcxx/include/__cxx03/__algorithm/equal_range.h
index 9abda2bd2e36f..4b3126a2840cf 100644
--- a/libcxx/include/__cxx03/__algorithm/equal_range.h
+++ b/libcxx/include/__cxx03/__algorithm/equal_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_EQUAL_RANGE_H
-#define _LIBCPP___ALGORITHM_EQUAL_RANGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_EQUAL_RANGE_H
+#define _LIBCPP___CXX03___ALGORITHM_EQUAL_RANGE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_EQUAL_RANGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_EQUAL_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/fill.h b/libcxx/include/__cxx03/__algorithm/fill.h
index 5da0f4457daa6..4aaf2744e8a58 100644
--- a/libcxx/include/__cxx03/__algorithm/fill.h
+++ b/libcxx/include/__cxx03/__algorithm/fill.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FILL_H
-#define _LIBCPP___ALGORITHM_FILL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FILL_H
+#define _LIBCPP___CXX03___ALGORITHM_FILL_H
 
 #include <__cxx03/__algorithm/fill_n.h>
 #include <__cxx03/__config>
@@ -42,4 +42,4 @@ fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FILL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FILL_H
diff --git a/libcxx/include/__cxx03/__algorithm/fill_n.h b/libcxx/include/__cxx03/__algorithm/fill_n.h
index fd548f27056a1..99b712c7b0360 100644
--- a/libcxx/include/__cxx03/__algorithm/fill_n.h
+++ b/libcxx/include/__cxx03/__algorithm/fill_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FILL_N_H
-#define _LIBCPP___ALGORITHM_FILL_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FILL_N_H
+#define _LIBCPP___CXX03___ALGORITHM_FILL_N_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__config>
@@ -95,4 +95,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_FILL_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FILL_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/find.h b/libcxx/include/__cxx03/__algorithm/find.h
index 7a48a449c897b..ff5ac9b8b1bd0 100644
--- a/libcxx/include/__cxx03/__algorithm/find.h
+++ b/libcxx/include/__cxx03/__algorithm/find.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_H
-#define _LIBCPP___ALGORITHM_FIND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_H
 
 #include <__cxx03/__algorithm/find_segment_if.h>
 #include <__cxx03/__algorithm/min.h>
@@ -178,4 +178,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_FIND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_end.h b/libcxx/include/__cxx03/__algorithm/find_end.h
index 34a45e2c66fae..3fa046bd0df3e 100644
--- a/libcxx/include/__cxx03/__algorithm/find_end.h
+++ b/libcxx/include/__cxx03/__algorithm/find_end.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_END_OF_H
-#define _LIBCPP___ALGORITHM_FIND_END_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_END_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_END_OF_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -222,4 +222,4 @@ find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FIND_END_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_END_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_first_of.h b/libcxx/include/__cxx03/__algorithm/find_first_of.h
index 05eb85fd663d2..b1b3e5f3be01e 100644
--- a/libcxx/include/__cxx03/__algorithm/find_first_of.h
+++ b/libcxx/include/__cxx03/__algorithm/find_first_of.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
-#define _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_FIRST_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_FIRST_OF_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__config>
@@ -52,4 +52,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Fo
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FIND_FIRST_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_FIRST_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_if.h b/libcxx/include/__cxx03/__algorithm/find_if.h
index b0150e539f9ab..ca4139c86787c 100644
--- a/libcxx/include/__cxx03/__algorithm/find_if.h
+++ b/libcxx/include/__cxx03/__algorithm/find_if.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_IF_H
-#define _LIBCPP___ALGORITHM_FIND_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_IF_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ find_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FIND_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_if_not.h b/libcxx/include/__cxx03/__algorithm/find_if_not.h
index 67d9a7deb2edd..a662dfbddfbb9 100644
--- a/libcxx/include/__cxx03/__algorithm/find_if_not.h
+++ b/libcxx/include/__cxx03/__algorithm/find_if_not.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_IF_NOT_H
-#define _LIBCPP___ALGORITHM_FIND_IF_NOT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_IF_NOT_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_IF_NOT_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ find_if_not(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FIND_IF_NOT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_IF_NOT_H
diff --git a/libcxx/include/__cxx03/__algorithm/find_segment_if.h b/libcxx/include/__cxx03/__algorithm/find_segment_if.h
index a6c016234418e..3475e9e8bdacd 100644
--- a/libcxx/include/__cxx03/__algorithm/find_segment_if.h
+++ b/libcxx/include/__cxx03/__algorithm/find_segment_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
-#define _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FIND_SEGMENT_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_FIND_SEGMENT_IF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/segmented_iterator.h>
@@ -59,4 +59,4 @@ __find_segment_if(_SegmentedIterator __first, _SegmentedIterator __last, _Pred _
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FIND_SEGMENT_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FIND_SEGMENT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/fold.h b/libcxx/include/__cxx03/__algorithm/fold.h
index 7e9c745bbbdb2..5ec0d747c009c 100644
--- a/libcxx/include/__cxx03/__algorithm/fold.h
+++ b/libcxx/include/__cxx03/__algorithm/fold.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FOLD_H
-#define _LIBCPP___ALGORITHM_FOLD_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FOLD_H
+#define _LIBCPP___CXX03___ALGORITHM_FOLD_H
 
 #include <__cxx03/__concepts/assignable.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -125,4 +125,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_FOLD_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FOLD_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each.h b/libcxx/include/__cxx03/__algorithm/for_each.h
index d2b19310c60f5..a6faf805fe9cf 100644
--- a/libcxx/include/__cxx03/__algorithm/for_each.h
+++ b/libcxx/include/__cxx03/__algorithm/for_each.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FOR_EACH_H
-#define _LIBCPP___ALGORITHM_FOR_EACH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FOR_EACH_H
+#define _LIBCPP___CXX03___ALGORITHM_FOR_EACH_H
 
 #include <__cxx03/__algorithm/for_each_segment.h>
 #include <__cxx03/__config>
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_FOR_EACH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each_n.h b/libcxx/include/__cxx03/__algorithm/for_each_n.h
index 7b8c40eacf967..486a5266ad011 100644
--- a/libcxx/include/__cxx03/__algorithm/for_each_n.h
+++ b/libcxx/include/__cxx03/__algorithm/for_each_n.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_H
-#define _LIBCPP___ALGORITHM_FOR_EACH_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FOR_EACH_N_H
+#define _LIBCPP___CXX03___ALGORITHM_FOR_EACH_N_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/convert_to_integral.h>
@@ -38,4 +38,4 @@ for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FOR_EACH_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FOR_EACH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/for_each_segment.h b/libcxx/include/__cxx03/__algorithm/for_each_segment.h
index 50c4b28deba02..02b4a1799d6a8 100644
--- a/libcxx/include/__cxx03/__algorithm/for_each_segment.h
+++ b/libcxx/include/__cxx03/__algorithm/for_each_segment.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
-#define _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_FOR_EACH_SEGMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_FOR_EACH_SEGMENT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/segmented_iterator.h>
@@ -50,4 +50,4 @@ __for_each_segment(_SegmentedIterator __first, _SegmentedIterator __last, _Funct
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_FOR_EACH_SEGMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/generate.h b/libcxx/include/__cxx03/__algorithm/generate.h
index 14da75cd44ceb..fa1929b639ad1 100644
--- a/libcxx/include/__cxx03/__algorithm/generate.h
+++ b/libcxx/include/__cxx03/__algorithm/generate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_GENERATE_H
-#define _LIBCPP___ALGORITHM_GENERATE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_GENERATE_H
+#define _LIBCPP___CXX03___ALGORITHM_GENERATE_H
 
 #include <__cxx03/__config>
 
@@ -26,4 +26,4 @@ generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __gen) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_GENERATE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_GENERATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/generate_n.h b/libcxx/include/__cxx03/__algorithm/generate_n.h
index 32cc86911815c..5a421131070e9 100644
--- a/libcxx/include/__cxx03/__algorithm/generate_n.h
+++ b/libcxx/include/__cxx03/__algorithm/generate_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_GENERATE_N_H
-#define _LIBCPP___ALGORITHM_GENERATE_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_GENERATE_N_H
+#define _LIBCPP___CXX03___ALGORITHM_GENERATE_N_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/convert_to_integral.h>
@@ -30,4 +30,4 @@ generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_GENERATE_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_GENERATE_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/half_positive.h b/libcxx/include/__cxx03/__algorithm/half_positive.h
index 4378964d9d836..a436a6086b5e9 100644
--- a/libcxx/include/__cxx03/__algorithm/half_positive.h
+++ b/libcxx/include/__cxx03/__algorithm/half_positive.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_HALF_POSITIVE_H
-#define _LIBCPP___ALGORITHM_HALF_POSITIVE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_HALF_POSITIVE_H
+#define _LIBCPP___CXX03___ALGORITHM_HALF_POSITIVE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -34,4 +34,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp __half_positive(_Tp __value) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_HALF_POSITIVE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_HALF_POSITIVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_found_result.h b/libcxx/include/__cxx03/__algorithm/in_found_result.h
index 0f4895dab437e..234e17cbd019a 100644
--- a/libcxx/include/__cxx03/__algorithm/in_found_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_found_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_FOUND_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_FOUND_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -51,4 +51,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_FOUND_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_FOUND_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_fun_result.h b/libcxx/include/__cxx03/__algorithm/in_fun_result.h
index 998d4599ceac8..402fbecc3df04 100644
--- a/libcxx/include/__cxx03/__algorithm/in_fun_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_fun_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_FUN_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_FUN_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -51,4 +51,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_FUN_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_FUN_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_in_out_result.h b/libcxx/include/__cxx03/__algorithm/in_in_out_result.h
index bb3a7e5466de0..cb14a7e654faa 100644
--- a/libcxx/include/__cxx03/__algorithm/in_in_out_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_in_out_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_IN_OUT_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_IN_OUT_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_IN_OUT_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_IN_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_in_result.h b/libcxx/include/__cxx03/__algorithm/in_in_result.h
index 12f1b572c5870..3ef349f87ff6c 100644
--- a/libcxx/include/__cxx03/__algorithm/in_in_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_in_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_IN_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_IN_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_IN_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_IN_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -53,4 +53,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_IN_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_IN_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_out_out_result.h b/libcxx/include/__cxx03/__algorithm/in_out_out_result.h
index c7d18535d1014..de67ac89fefeb 100644
--- a/libcxx/include/__cxx03/__algorithm/in_out_out_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_out_out_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_OUT_OUT_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_OUT_OUT_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_OUT_OUT_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_OUT_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/in_out_result.h b/libcxx/include/__cxx03/__algorithm/in_out_result.h
index 66bd1a1666988..e80f5fd7ba77b 100644
--- a/libcxx/include/__cxx03/__algorithm/in_out_result.h
+++ b/libcxx/include/__cxx03/__algorithm/in_out_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
-#define _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IN_OUT_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_IN_OUT_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -53,4 +53,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IN_OUT_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IN_OUT_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/includes.h b/libcxx/include/__cxx03/__algorithm/includes.h
index 6d3eb44bb8c44..24b46d1f10cb5 100644
--- a/libcxx/include/__cxx03/__algorithm/includes.h
+++ b/libcxx/include/__cxx03/__algorithm/includes.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_INCLUDES_H
-#define _LIBCPP___ALGORITHM_INCLUDES_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_INCLUDES_H
+#define _LIBCPP___CXX03___ALGORITHM_INCLUDES_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -76,4 +76,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_INCLUDES_H
+#endif // _LIBCPP___CXX03___ALGORITHM_INCLUDES_H
diff --git a/libcxx/include/__cxx03/__algorithm/inplace_merge.h b/libcxx/include/__cxx03/__algorithm/inplace_merge.h
index 3816ba5793c0d..187969dcc4029 100644
--- a/libcxx/include/__cxx03/__algorithm/inplace_merge.h
+++ b/libcxx/include/__cxx03/__algorithm/inplace_merge.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_INPLACE_MERGE_H
-#define _LIBCPP___ALGORITHM_INPLACE_MERGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_INPLACE_MERGE_H
+#define _LIBCPP___CXX03___ALGORITHM_INPLACE_MERGE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -237,4 +237,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_INPLACE_MERGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_INPLACE_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_heap.h b/libcxx/include/__cxx03/__algorithm/is_heap.h
index dde4bfb6ff2db..c19adb84ba570 100644
--- a/libcxx/include/__cxx03/__algorithm/is_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/is_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_HEAP_H
-#define _LIBCPP___ALGORITHM_IS_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_HEAP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -35,4 +35,4 @@ is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_IS_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_heap_until.h b/libcxx/include/__cxx03/__algorithm/is_heap_until.h
index d46dd343115c3..e3a6d9769fcc5 100644
--- a/libcxx/include/__cxx03/__algorithm/is_heap_until.h
+++ b/libcxx/include/__cxx03/__algorithm/is_heap_until.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
-#define _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_HEAP_UNTIL_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_HEAP_UNTIL_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -59,4 +59,4 @@ is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_IS_HEAP_UNTIL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_HEAP_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_partitioned.h b/libcxx/include/__cxx03/__algorithm/is_partitioned.h
index 8bd202f3e2429..a7dff7bf42544 100644
--- a/libcxx/include/__cxx03/__algorithm/is_partitioned.h
+++ b/libcxx/include/__cxx03/__algorithm/is_partitioned.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_PARTITIONED_H
-#define _LIBCPP___ALGORITHM_IS_PARTITIONED_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_PARTITIONED_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_PARTITIONED_H
 
 #include <__cxx03/__config>
 
@@ -34,4 +34,4 @@ is_partitioned(_InputIterator __first, _InputIterator __last, _Predicate __pred)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_IS_PARTITIONED_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_PARTITIONED_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_permutation.h b/libcxx/include/__cxx03/__algorithm/is_permutation.h
index 0274ba98fcd16..fd29e82656a3c 100644
--- a/libcxx/include/__cxx03/__algorithm/is_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/is_permutation.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_IS_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_PERMUTATION_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -305,4 +305,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_IS_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_sorted.h b/libcxx/include/__cxx03/__algorithm/is_sorted.h
index 0a003a86b9913..1318f5baf8394 100644
--- a/libcxx/include/__cxx03/__algorithm/is_sorted.h
+++ b/libcxx/include/__cxx03/__algorithm/is_sorted.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_SORTED_H
-#define _LIBCPP___ALGORITHM_IS_SORTED_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_SORTED_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_SORTED_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -35,4 +35,4 @@ is_sorted(_ForwardIterator __first, _ForwardIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_IS_SORTED_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_SORTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/is_sorted_until.h b/libcxx/include/__cxx03/__algorithm/is_sorted_until.h
index eb90e5dd88e67..f97fb7c2e53ef 100644
--- a/libcxx/include/__cxx03/__algorithm/is_sorted_until.h
+++ b/libcxx/include/__cxx03/__algorithm/is_sorted_until.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
-#define _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_IS_SORTED_UNTIL_H
+#define _LIBCPP___CXX03___ALGORITHM_IS_SORTED_UNTIL_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -48,4 +48,4 @@ is_sorted_until(_ForwardIterator __first, _ForwardIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_IS_SORTED_UNTIL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_IS_SORTED_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/iter_swap.h b/libcxx/include/__cxx03/__algorithm/iter_swap.h
index 837a603d23e32..4fcbcdcf1e050 100644
--- a/libcxx/include/__cxx03/__algorithm/iter_swap.h
+++ b/libcxx/include/__cxx03/__algorithm/iter_swap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ITER_SWAP_H
-#define _LIBCPP___ALGORITHM_ITER_SWAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ITER_SWAP_H
+#define _LIBCPP___CXX03___ALGORITHM_ITER_SWAP_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/declval.h>
@@ -28,4 +28,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void iter_swap(_Forwa
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_ITER_SWAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ITER_SWAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/iterator_operations.h b/libcxx/include/__cxx03/__algorithm/iterator_operations.h
index b4f1f9e3ffca9..24973b66e4f96 100644
--- a/libcxx/include/__cxx03/__algorithm/iterator_operations.h
+++ b/libcxx/include/__cxx03/__algorithm/iterator_operations.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
-#define _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ITERATOR_OPERATIONS_H
+#define _LIBCPP___CXX03___ALGORITHM_ITERATOR_OPERATIONS_H
 
 #include <__cxx03/__algorithm/iter_swap.h>
 #include <__cxx03/__algorithm/ranges_iterator_concept.h>
@@ -220,4 +220,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_ITERATOR_OPERATIONS_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ITERATOR_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h b/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
index e3d93be6177e7..b019e4b5021b4 100644
--- a/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
+++ b/libcxx/include/__cxx03/__algorithm/lexicographical_compare.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
-#define _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+#define _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -54,4 +54,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 boo
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
diff --git a/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h b/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
index bea67a7937659..2fc53636abbd4 100644
--- a/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
+++ b/libcxx/include/__cxx03/__algorithm/lexicographical_compare_three_way.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
-#define _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
+#define _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__algorithm/three_way_comp_ref_type.h>
@@ -122,4 +122,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_LEXICOGRAPHICAL_COMPARE_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__algorithm/lower_bound.h b/libcxx/include/__cxx03/__algorithm/lower_bound.h
index 844674cb78604..810792bfe8fe1 100644
--- a/libcxx/include/__cxx03/__algorithm/lower_bound.h
+++ b/libcxx/include/__cxx03/__algorithm/lower_bound.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_LOWER_BOUND_H
-#define _LIBCPP___ALGORITHM_LOWER_BOUND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_LOWER_BOUND_H
+#define _LIBCPP___CXX03___ALGORITHM_LOWER_BOUND_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/half_positive.h>
@@ -106,4 +106,4 @@ lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __valu
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_LOWER_BOUND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_LOWER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/make_heap.h b/libcxx/include/__cxx03/__algorithm/make_heap.h
index 5239a99083f50..35a7f7bf9779f 100644
--- a/libcxx/include/__cxx03/__algorithm/make_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/make_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MAKE_HEAP_H
-#define _LIBCPP___ALGORITHM_MAKE_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MAKE_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_MAKE_HEAP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -57,4 +57,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MAKE_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MAKE_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/make_projected.h b/libcxx/include/__cxx03/__algorithm/make_projected.h
index 8368e269385f4..ac17c2bba7ba8 100644
--- a/libcxx/include/__cxx03/__algorithm/make_projected.h
+++ b/libcxx/include/__cxx03/__algorithm/make_projected.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
-#define _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MAKE_PROJECTED_H
+#define _LIBCPP___CXX03___ALGORITHM_MAKE_PROJECTED_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_MAKE_PROJECTED_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MAKE_PROJECTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/max.h b/libcxx/include/__cxx03/__algorithm/max.h
index 983e81cd24bc4..7726d16827157 100644
--- a/libcxx/include/__cxx03/__algorithm/max.h
+++ b/libcxx/include/__cxx03/__algorithm/max.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MAX_H
-#define _LIBCPP___ALGORITHM_MAX_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MAX_H
+#define _LIBCPP___CXX03___ALGORITHM_MAX_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -55,4 +55,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MAX_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/max_element.h b/libcxx/include/__cxx03/__algorithm/max_element.h
index 686c116ab0105..20a22e74c8be7 100644
--- a/libcxx/include/__cxx03/__algorithm/max_element.h
+++ b/libcxx/include/__cxx03/__algorithm/max_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MAX_ELEMENT_H
-#define _LIBCPP___ALGORITHM_MAX_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MAX_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_MAX_ELEMENT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -48,4 +48,4 @@ max_element(_ForwardIterator __first, _ForwardIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_MAX_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/merge.h b/libcxx/include/__cxx03/__algorithm/merge.h
index 41b0c9263e2fd..90b986f747a3c 100644
--- a/libcxx/include/__cxx03/__algorithm/merge.h
+++ b/libcxx/include/__cxx03/__algorithm/merge.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MERGE_H
-#define _LIBCPP___ALGORITHM_MERGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MERGE_H
+#define _LIBCPP___CXX03___ALGORITHM_MERGE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -66,4 +66,4 @@ merge(_InputIterator1 __first1,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_MERGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/min.h b/libcxx/include/__cxx03/__algorithm/min.h
index 836f8f0028748..d4801ab3330da 100644
--- a/libcxx/include/__cxx03/__algorithm/min.h
+++ b/libcxx/include/__cxx03/__algorithm/min.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MIN_H
-#define _LIBCPP___ALGORITHM_MIN_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MIN_H
+#define _LIBCPP___CXX03___ALGORITHM_MIN_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -55,4 +55,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MIN_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MIN_H
diff --git a/libcxx/include/__cxx03/__algorithm/min_element.h b/libcxx/include/__cxx03/__algorithm/min_element.h
index d625ab0886cf7..aa4f6013e8752 100644
--- a/libcxx/include/__cxx03/__algorithm/min_element.h
+++ b/libcxx/include/__cxx03/__algorithm/min_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MIN_ELEMENT_H
-#define _LIBCPP___ALGORITHM_MIN_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MIN_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_MIN_ELEMENT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -68,4 +68,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MIN_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MIN_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/min_max_result.h b/libcxx/include/__cxx03/__algorithm/min_max_result.h
index 2b56f88f1f9fe..2098aada70a52 100644
--- a/libcxx/include/__cxx03/__algorithm/min_max_result.h
+++ b/libcxx/include/__cxx03/__algorithm/min_max_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
-#define _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MIN_MAX_RESULT_H
+#define _LIBCPP___CXX03___ALGORITHM_MIN_MAX_RESULT_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -53,4 +53,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MIN_MAX_RESULT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MIN_MAX_RESULT_H
diff --git a/libcxx/include/__cxx03/__algorithm/minmax.h b/libcxx/include/__cxx03/__algorithm/minmax.h
index a36970a0565fe..e09a5b04104aa 100644
--- a/libcxx/include/__cxx03/__algorithm/minmax.h
+++ b/libcxx/include/__cxx03/__algorithm/minmax.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MINMAX_H
-#define _LIBCPP___ALGORITHM_MINMAX_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MINMAX_H
+#define _LIBCPP___CXX03___ALGORITHM_MINMAX_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/minmax_element.h>
@@ -56,4 +56,4 @@ minmax(initializer_list<_Tp> __t) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_MINMAX_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/minmax_element.h b/libcxx/include/__cxx03/__algorithm/minmax_element.h
index 6298784092354..673c22020ef3a 100644
--- a/libcxx/include/__cxx03/__algorithm/minmax_element.h
+++ b/libcxx/include/__cxx03/__algorithm/minmax_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
-#define _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MINMAX_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_MINMAX_ELEMENT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__config>
@@ -97,4 +97,4 @@ minmax_element(_ForwardIterator __first, _ForwardIterator __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_MINMAX_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MINMAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/mismatch.h b/libcxx/include/__cxx03/__algorithm/mismatch.h
index 1eb9717c9ae6c..163bc8fb165c0 100644
--- a/libcxx/include/__cxx03/__algorithm/mismatch.h
+++ b/libcxx/include/__cxx03/__algorithm/mismatch.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MISMATCH_H
-#define _LIBCPP___ALGORITHM_MISMATCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MISMATCH_H
+#define _LIBCPP___CXX03___ALGORITHM_MISMATCH_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/min.h>
@@ -214,4 +214,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MISMATCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MISMATCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/move.h b/libcxx/include/__cxx03/__algorithm/move.h
index 11366afe99ffb..cb158e15f19f5 100644
--- a/libcxx/include/__cxx03/__algorithm/move.h
+++ b/libcxx/include/__cxx03/__algorithm/move.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MOVE_H
-#define _LIBCPP___ALGORITHM_MOVE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MOVE_H
+#define _LIBCPP___CXX03___ALGORITHM_MOVE_H
 
 #include <__cxx03/__algorithm/copy_move_common.h>
 #include <__cxx03/__algorithm/for_each_segment.h>
@@ -124,4 +124,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MOVE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/move_backward.h b/libcxx/include/__cxx03/__algorithm/move_backward.h
index 9d3e87bb9667e..d4da82382a4c7 100644
--- a/libcxx/include/__cxx03/__algorithm/move_backward.h
+++ b/libcxx/include/__cxx03/__algorithm/move_backward.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
-#define _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_MOVE_BACKWARD_H
+#define _LIBCPP___CXX03___ALGORITHM_MOVE_BACKWARD_H
 
 #include <__cxx03/__algorithm/copy_move_common.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -134,4 +134,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
+#endif // _LIBCPP___CXX03___ALGORITHM_MOVE_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/next_permutation.h b/libcxx/include/__cxx03/__algorithm/next_permutation.h
index 02aa9113d1830..7d6b2ddad5056 100644
--- a/libcxx/include/__cxx03/__algorithm/next_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/next_permutation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_NEXT_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_NEXT_PERMUTATION_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -72,4 +72,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_NEXT_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_NEXT_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/none_of.h b/libcxx/include/__cxx03/__algorithm/none_of.h
index e5d095da23482..91162ec24ab1d 100644
--- a/libcxx/include/__cxx03/__algorithm/none_of.h
+++ b/libcxx/include/__cxx03/__algorithm/none_of.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_NONE_OF_H
-#define _LIBCPP___ALGORITHM_NONE_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_NONE_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_NONE_OF_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ none_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_NONE_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_NONE_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/nth_element.h b/libcxx/include/__cxx03/__algorithm/nth_element.h
index f840864c531c5..232966e0d2670 100644
--- a/libcxx/include/__cxx03/__algorithm/nth_element.h
+++ b/libcxx/include/__cxx03/__algorithm/nth_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_NTH_ELEMENT_H
-#define _LIBCPP___ALGORITHM_NTH_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_NTH_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_NTH_ELEMENT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -258,4 +258,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_NTH_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_NTH_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/partial_sort.h b/libcxx/include/__cxx03/__algorithm/partial_sort.h
index 6984b1c91f201..04597fc32b9a2 100644
--- a/libcxx/include/__cxx03/__algorithm/partial_sort.h
+++ b/libcxx/include/__cxx03/__algorithm/partial_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PARTIAL_SORT_H
-#define _LIBCPP___ALGORITHM_PARTIAL_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -88,4 +88,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PARTIAL_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h b/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
index c8849b9f6175f..d4b5fafba9678 100644
--- a/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/partial_sort_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
-#define _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_COPY_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PARTIAL_SORT_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PARTIAL_SORT_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition.h b/libcxx/include/__cxx03/__algorithm/partition.h
index 5f26384b7c8ef..d39dbbbd0b185 100644
--- a/libcxx/include/__cxx03/__algorithm/partition.h
+++ b/libcxx/include/__cxx03/__algorithm/partition.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PARTITION_H
-#define _LIBCPP___ALGORITHM_PARTITION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PARTITION_H
+#define _LIBCPP___CXX03___ALGORITHM_PARTITION_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__config>
@@ -87,4 +87,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PARTITION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition_copy.h b/libcxx/include/__cxx03/__algorithm/partition_copy.h
index 916a1c301d6d5..18d82cfa20326 100644
--- a/libcxx/include/__cxx03/__algorithm/partition_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/partition_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PARTITION_COPY_H
-#define _LIBCPP___ALGORITHM_PARTITION_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PARTITION_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_PARTITION_COPY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -40,4 +40,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_OutputIterator1, _Outp
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_PARTITION_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PARTITION_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/partition_point.h b/libcxx/include/__cxx03/__algorithm/partition_point.h
index c28ee8cae0e6d..ccf203bbf245e 100644
--- a/libcxx/include/__cxx03/__algorithm/partition_point.h
+++ b/libcxx/include/__cxx03/__algorithm/partition_point.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PARTITION_POINT_H
-#define _LIBCPP___ALGORITHM_PARTITION_POINT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PARTITION_POINT_H
+#define _LIBCPP___CXX03___ALGORITHM_PARTITION_POINT_H
 
 #include <__cxx03/__algorithm/half_positive.h>
 #include <__cxx03/__config>
@@ -41,4 +41,4 @@ partition_point(_ForwardIterator __first, _ForwardIterator __last, _Predicate __
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_PARTITION_POINT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PARTITION_POINT_H
diff --git a/libcxx/include/__cxx03/__algorithm/pop_heap.h b/libcxx/include/__cxx03/__algorithm/pop_heap.h
index 800cabf94eaa9..5d19e902ff13b 100644
--- a/libcxx/include/__cxx03/__algorithm/pop_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/pop_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_POP_HEAP_H
-#define _LIBCPP___ALGORITHM_POP_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_POP_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_POP_HEAP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_POP_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_POP_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/prev_permutation.h b/libcxx/include/__cxx03/__algorithm/prev_permutation.h
index 2569156eac642..b050d9cf337a7 100644
--- a/libcxx/include/__cxx03/__algorithm/prev_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/prev_permutation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PREV_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_PREV_PERMUTATION_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -72,4 +72,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PREV_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PREV_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/pstl.h b/libcxx/include/__cxx03/__algorithm/pstl.h
index 8dea15e09cb44..4fc0d47996107 100644
--- a/libcxx/include/__cxx03/__algorithm/pstl.h
+++ b/libcxx/include/__cxx03/__algorithm/pstl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PSTL_H
-#define _LIBCPP___ALGORITHM_PSTL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PSTL_H
+#define _LIBCPP___CXX03___ALGORITHM_PSTL_H
 
 #include <__cxx03/__config>
 
@@ -660,4 +660,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PSTL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PSTL_H
diff --git a/libcxx/include/__cxx03/__algorithm/push_heap.h b/libcxx/include/__cxx03/__algorithm/push_heap.h
index de4dcc33fb1fd..9ef44cdb3feea 100644
--- a/libcxx/include/__cxx03/__algorithm/push_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/push_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_PUSH_HEAP_H
-#define _LIBCPP___ALGORITHM_PUSH_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_PUSH_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_PUSH_HEAP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -81,4 +81,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_PUSH_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_PUSH_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h b/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
index 26caf4fe40ae0..e62db90f83c18 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_adjacent_find.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
-#define _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ADJACENT_FIND_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ADJACENT_FIND_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -80,4 +80,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ADJACENT_FIND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ADJACENT_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_all_of.h b/libcxx/include/__cxx03/__algorithm/ranges_all_of.h
index 7e92c37b8fd99..e03f48b23c222 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_all_of.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_all_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
-#define _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ALL_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ALL_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ALL_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ALL_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_any_of.h b/libcxx/include/__cxx03/__algorithm/ranges_any_of.h
index d20177223fc6b..23b21f9a8bf13 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_any_of.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_any_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
-#define _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ANY_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ANY_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ANY_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h b/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
index 675cc1ac8074a..18ebf2b07bdb6 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_binary_search.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
-#define _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_BINARY_SEARCH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_BINARY_SEARCH_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/lower_bound.h>
@@ -70,4 +70,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_BINARY_SEARCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_BINARY_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_clamp.h b/libcxx/include/__cxx03/__algorithm/ranges_clamp.h
index 3f9f6b1f9944f..da4dda06a2255 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_clamp.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_clamp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_CLAMP_H
-#define _LIBCPP___ALGORITHM_RANGES_CLAMP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_CLAMP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_CLAMP_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_CLAMP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_CLAMP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_contains.h b/libcxx/include/__cxx03/__algorithm/ranges_contains.h
index 312de975efa5f..00b9d08c83c6c 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_contains.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_contains.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
-#define _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_H
 
 #include <__cxx03/__algorithm/ranges_find.h>
 #include <__cxx03/__config>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_CONTAINS_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h b/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
index d3302f1c915df..6ac67b43ca6cc 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_contains_subrange.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
-#define _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
 
 #include <__cxx03/__algorithm/ranges_search.h>
 #include <__cxx03/__config>
@@ -94,4 +94,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_CONTAINS_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_copy.h
index cf3f9974936ad..e94a2089a692c 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__algorithm/in_out_result.h>
@@ -68,4 +68,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
index a6bb781397dec..90928177491f5 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_backward.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
-#define _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_BACKWARD_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_BACKWARD_H
 
 #include <__cxx03/__algorithm/copy_backward.h>
 #include <__cxx03/__algorithm/in_out_result.h>
@@ -66,4 +66,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COPY_BACKWARD_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
index f0d3005a4982b..814d408cb703e 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_IF_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__config>
@@ -84,4 +84,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h b/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
index 92f10e7466e53..67639e7c19aff 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_copy_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_N_H
-#define _LIBCPP___ALGORITHM_RANGES_COPY_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_N_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_N_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__algorithm/in_out_result.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COPY_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COPY_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_count.h b/libcxx/include/__cxx03/__algorithm/ranges_count.h
index ae24b57c17d0a..b8553badfce02 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_count.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_count.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COUNT_H
-#define _LIBCPP___ALGORITHM_RANGES_COUNT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_H
 
 #include <__cxx03/__algorithm/count.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COUNT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_count_if.h b/libcxx/include/__cxx03/__algorithm/ranges_count_if.h
index ecadf6fac7128..10850c857d6df 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_count_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_count_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_IF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -76,4 +76,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_COUNT_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_COUNT_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h b/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
index 85329f8f4e58a..39dcb79559c22 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_ends_with.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
-#define _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ENDS_WITH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ENDS_WITH_H
 
 #include <__cxx03/__algorithm/ranges_equal.h>
 #include <__cxx03/__algorithm/ranges_starts_with.h>
@@ -198,4 +198,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ENDS_WITH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ENDS_WITH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_equal.h b/libcxx/include/__cxx03/__algorithm/ranges_equal.h
index b89235a299414..abc701a9f1188 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_equal.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_equal.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_EQUAL_H
-#define _LIBCPP___ALGORITHM_RANGES_EQUAL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_H
 
 #include <__cxx03/__algorithm/equal.h>
 #include <__cxx03/__algorithm/unwrap_range.h>
@@ -106,4 +106,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_EQUAL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h b/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
index d6a38e5edd312..64e686704efd2 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_equal_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
-#define _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_RANGE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_RANGE_H
 
 #include <__cxx03/__algorithm/equal_range.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -77,4 +77,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_EQUAL_RANGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_EQUAL_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_fill.h b/libcxx/include/__cxx03/__algorithm/ranges_fill.h
index 56008ec7304da..f85a1f4ed21c1 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_fill.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_fill.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FILL_H
-#define _LIBCPP___ALGORITHM_RANGES_FILL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_H
 
 #include <__cxx03/__algorithm/ranges_fill_n.h>
 #include <__cxx03/__config>
@@ -59,4 +59,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FILL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h b/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
index bb00676943a6c..07f04c293451f 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_fill_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FILL_N_H
-#define _LIBCPP___ALGORITHM_RANGES_FILL_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_N_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_N_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -50,4 +50,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FILL_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FILL_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find.h b/libcxx/include/__cxx03/__algorithm/ranges_find.h
index 896fe920fd815..0b2c865476db0 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_H
 
 #include <__cxx03/__algorithm/find.h>
 #include <__cxx03/__algorithm/ranges_find_if.h>
@@ -77,4 +77,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_end.h b/libcxx/include/__cxx03/__algorithm/ranges_find_end.h
index c71f32546afb1..117753b3d86b6 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find_end.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_end.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_END_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_END_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_END_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_END_H
 
 #include <__cxx03/__algorithm/find_end.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -100,4 +100,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_END_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_END_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h b/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
index 23ae82eb4685f..ebf01681419ca 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_first_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_FIRST_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_FIRST_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_FIRST_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_FIRST_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_if.h b/libcxx/include/__cxx03/__algorithm/ranges_find_if.h
index a518f532a73e2..a1098e0de1824 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -72,4 +72,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h b/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
index c54d565188c4f..6bc8f16395873 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_if_not.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_NOT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_NOT_H
 
 #include <__cxx03/__algorithm/ranges_find_if.h>
 #include <__cxx03/__config>
@@ -66,4 +66,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_IF_NOT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_IF_NOT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_find_last.h b/libcxx/include/__cxx03/__algorithm/ranges_find_last.h
index 3028dc5bf53cb..506452c7c001d 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_find_last.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_find_last.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
-#define _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_LAST_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_LAST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -172,4 +172,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FIND_LAST_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FIND_LAST_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_for_each.h b/libcxx/include/__cxx03/__algorithm/ranges_for_each.h
index eca0f35e49fbd..3e926e9d2c4d1 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_for_each.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_for_each.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
-#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_H
 
 #include <__cxx03/__algorithm/in_fun_result.h>
 #include <__cxx03/__config>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h b/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
index fa3051fa8a0aa..96df211298541 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_for_each_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
-#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_N_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_N_H
 
 #include <__cxx03/__algorithm/in_fun_result.h>
 #include <__cxx03/__config>
@@ -61,4 +61,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_FOR_EACH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_generate.h b/libcxx/include/__cxx03/__algorithm/ranges_generate.h
index 570ed73a29550..96d43592d2a76 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_generate.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_generate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_GENERATE_H
-#define _LIBCPP___ALGORITHM_RANGES_GENERATE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/invocable.h>
@@ -70,4 +70,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_GENERATE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h b/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
index b7c684be2e03f..dd43e21618718 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_generate_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
-#define _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_N_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_N_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/invocable.h>
@@ -62,4 +62,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_GENERATE_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_includes.h b/libcxx/include/__cxx03/__algorithm/ranges_includes.h
index 3abe69118f20a..587a0eda16da5 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_includes.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_includes.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
-#define _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_INCLUDES_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_INCLUDES_H
 
 #include <__cxx03/__algorithm/includes.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -95,4 +95,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_INCLUDES_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_INCLUDES_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h b/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
index ebb711d731209..34668f4d57baf 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_inplace_merge.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
-#define _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_INPLACE_MERGE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_INPLACE_MERGE_H
 
 #include <__cxx03/__algorithm/inplace_merge.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -81,4 +81,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_INPLACE_MERGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_INPLACE_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
index 7c6b60a56f6c7..47112bdddd057 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
-#define _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_H
 
 #include <__cxx03/__algorithm/is_heap_until.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_IS_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h b/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
index e32e802f44732..d7176f784037c 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_heap_until.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
-#define _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
 
 #include <__cxx03/__algorithm/is_heap_until.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_IS_HEAP_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h b/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
index 87d59d0410fe5..d746f55322e78 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_partitioned.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
-#define _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PARTITIONED_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PARTITIONED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -83,4 +83,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_IS_PARTITIONED_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PARTITIONED_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
index a894854a8b65a..31d85e5fb27cd 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_permutation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PERMUTATION_H
 
 #include <__cxx03/__algorithm/is_permutation.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -104,4 +104,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_IS_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_IS_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
index 201673553fd4f..f5c4e6f4191ad 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
-#define _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
+#ifndef _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_H
+#define _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_H
 
 #include <__cxx03/__algorithm/ranges_is_sorted_until.h>
 #include <__cxx03/__config>
@@ -64,4 +64,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP__ALGORITHM_RANGES_IS_SORTED_H
+#endif // _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
index 713d2664884ac..701c01749c260 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_is_sorted_until.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
-#define _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
+#ifndef _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
+#define _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -79,4 +79,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
+#endif // _LIBCPP___CXX03__ALGORITHM_RANGES_IS_SORTED_UNTIL_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h b/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
index eb2025c234ba1..1ff6328cfc150 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_iterator_concept.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
-#define _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -53,4 +53,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h b/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
index ba87a20b425b4..e3fec1c416519 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_lexicographical_compare.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
-#define _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_LEXICOGRAPHICAL_COMPARE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h b/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
index 2c312866ee930..bed4c815e5b4f 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_lower_bound.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
-#define _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_LOWER_BOUND_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_LOWER_BOUND_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/lower_bound.h>
@@ -70,4 +70,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_LOWER_BOUND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_LOWER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
index 5a00575abbd34..ba8ccb89baad6 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_make_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
-#define _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MAKE_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MAKE_HEAP_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_heap.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MAKE_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MAKE_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_max.h b/libcxx/include/__cxx03/__algorithm/ranges_max.h
index d61a54fa7fff7..58ea541d3e493 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_max.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_max.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MAX_H
-#define _LIBCPP___ALGORITHM_RANGES_MAX_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_H
 
 #include <__cxx03/__algorithm/ranges_min_element.h>
 #include <__cxx03/__assert>
@@ -100,4 +100,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MAX_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_max_element.h b/libcxx/include/__cxx03/__algorithm/ranges_max_element.h
index 6bcf77bc29d4d..cbe50d1bf4127 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_max_element.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_max_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
-#define _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_ELEMENT_H
 
 #include <__cxx03/__algorithm/ranges_min_element.h>
 #include <__cxx03/__config>
@@ -66,4 +66,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MAX_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MAX_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_merge.h b/libcxx/include/__cxx03/__algorithm/ranges_merge.h
index 697dfdae0b300..8b8b8fbbab2cb 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_merge.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_merge.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MERGE_H
-#define _LIBCPP___ALGORITHM_RANGES_MERGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MERGE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MERGE_H
 
 #include <__cxx03/__algorithm/in_in_out_result.h>
 #include <__cxx03/__algorithm/ranges_copy.h>
@@ -135,4 +135,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MERGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MERGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_min.h b/libcxx/include/__cxx03/__algorithm/ranges_min.h
index bcf0705071318..72f06e72e3003 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_min.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_min.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MIN_H
-#define _LIBCPP___ALGORITHM_RANGES_MIN_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_H
 
 #include <__cxx03/__algorithm/ranges_min_element.h>
 #include <__cxx03/__assert>
@@ -92,4 +92,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MIN_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_min_element.h b/libcxx/include/__cxx03/__algorithm/ranges_min_element.h
index 320a47a57ec6d..b3854233966b0 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_min_element.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_min_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
-#define _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_ELEMENT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MIN_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MIN_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_minmax.h b/libcxx/include/__cxx03/__algorithm/ranges_minmax.h
index 8adf1af585518..8ce4e438382e5 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_minmax.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_minmax.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MINMAX_H
-#define _LIBCPP___ALGORITHM_RANGES_MINMAX_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_H
 
 #include <__cxx03/__algorithm/min_max_result.h>
 #include <__cxx03/__algorithm/minmax_element.h>
@@ -172,4 +172,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MINMAX_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h b/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
index ebc44082f4c3a..1e92a63a9d33e 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_minmax_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MINMAX_ELEMENT_H
-#define _LIBCPP___ALGORITHM_RANGES_MINMAX_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_ELEMENT_H
 
 #include <__cxx03/__algorithm/min_max_result.h>
 #include <__cxx03/__algorithm/minmax_element.h>
@@ -75,4 +75,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MINMAX_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MINMAX_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h b/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
index ccca6c7f6eb55..d3ebb7ded9c0b 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_mismatch.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
-#define _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MISMATCH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MISMATCH_H
 
 #include <__cxx03/__algorithm/in_in_result.h>
 #include <__cxx03/__algorithm/mismatch.h>
@@ -97,4 +97,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MISMATCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MISMATCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_move.h b/libcxx/include/__cxx03/__algorithm/ranges_move.h
index 452608bef7456..e20f4d8854db2 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_move.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_move.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MOVE_H
-#define _LIBCPP___ALGORITHM_RANGES_MOVE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MOVE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h b/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
index e353d9e2ffd10..cecaa7d8d657b 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_move_backward.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
-#define _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_BACKWARD_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_BACKWARD_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -73,4 +73,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_MOVE_BACKWARD_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
index b134cfca38195..337c7a892cfba 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_next_permutation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_NEXT_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_NEXT_PERMUTATION_H
 
 #include <__cxx03/__algorithm/in_found_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -75,4 +75,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_NEXT_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_NEXT_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_none_of.h b/libcxx/include/__cxx03/__algorithm/ranges_none_of.h
index 36d5477e14dee..461fbc87c5d24 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_none_of.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_none_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
-#define _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_NONE_OF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_NONE_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -72,4 +72,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_NONE_OF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_NONE_OF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h b/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
index df428390f7a3d..27bf33e7703de 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_nth_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
-#define _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_NTH_ELEMENT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_NTH_ELEMENT_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -81,4 +81,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_NTH_ELEMENT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_NTH_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
index d19fc9b5297eb..06aab77f65f3c 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
-#define _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -79,4 +79,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
index bd2be3cb2cf2d..3fda0e58e74d0 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partial_sort_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -111,4 +111,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PARTIAL_SORT_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition.h b/libcxx/include/__cxx03/__algorithm/ranges_partition.h
index d6c147c441def..261972ada5683 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_partition.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_H
-#define _LIBCPP___ALGORITHM_RANGES_PARTITION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -85,4 +85,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
index 42cbb3a96a237..c80108022d096 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_COPY_H
 
 #include <__cxx03/__algorithm/in_out_out_result.h>
 #include <__cxx03/__config>
@@ -107,4 +107,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h b/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
index 60731b2ddcf3c..337e1261f01c5 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_partition_point.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
-#define _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_POINT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_POINT_H
 
 #include <__cxx03/__algorithm/half_positive.h>
 #include <__cxx03/__config>
@@ -90,4 +90,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PARTITION_POINT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PARTITION_POINT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
index d201d62c73575..ecbfb44888edf 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_pop_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
-#define _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_POP_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_POP_HEAP_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -83,4 +83,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_POP_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_POP_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h b/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
index 8d40d44e6ac1e..ded9eb2c47155 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_prev_permutation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
-#define _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PREV_PERMUTATION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PREV_PERMUTATION_H
 
 #include <__cxx03/__algorithm/in_found_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -75,4 +75,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PREV_PERMUTATION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PREV_PERMUTATION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
index 39d219e763dee..33479d9bfad66 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_push_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
-#define _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_PUSH_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_PUSH_HEAP_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_PUSH_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_PUSH_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove.h b/libcxx/include/__cxx03/__algorithm/ranges_remove.h
index d766a7b4a846f..13b4efb9f5c3b 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_remove.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_H
-#define _LIBCPP___ALGORITHM_RANGES_REMOVE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_H
 #include <__cxx03/__config>
 
 #include <__cxx03/__algorithm/ranges_remove_if.h>
@@ -65,4 +65,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
index fdb04cd96cf89..47c2d6c086920 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/ranges_remove_copy_if.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
index dd643a581a1d2..3674e4c4207b7 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_IF_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -92,4 +92,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h b/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
index 9d4ccd0f0b52d..546acf8b78da3 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_remove_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_IF_H
 #include <__cxx03/__config>
 
 #include <__cxx03/__algorithm/ranges_find_if.h>
@@ -86,4 +86,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REMOVE_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REMOVE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace.h b/libcxx/include/__cxx03/__algorithm/ranges_replace.h
index 774d7632e41cf..f5c030e7993c2 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_replace.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_H
-#define _LIBCPP___ALGORITHM_RANGES_REPLACE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_H
 
 #include <__cxx03/__algorithm/ranges_replace_if.h>
 #include <__cxx03/__config>
@@ -65,4 +65,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
index d90715d3661ba..387b7a42f24c8 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/ranges_replace_copy_if.h>
@@ -90,4 +90,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
index 88e5ff71c58c5..f5786d9f8948d 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_IF_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__config>
@@ -95,4 +95,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h b/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
index 6d9ff45a59c97..499cc7887b16c 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_replace_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
-#define _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_IF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REPLACE_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REPLACE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_reverse.h b/libcxx/include/__cxx03/__algorithm/ranges_reverse.h
index 78614666200ef..577a4310f2152 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_reverse.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_reverse.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REVERSE_H
-#define _LIBCPP___ALGORITHM_RANGES_REVERSE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -76,4 +76,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REVERSE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
index 7aab06d721430..d727202365b08 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_reverse_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/ranges_copy.h>
@@ -67,4 +67,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_REVERSE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_REVERSE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_rotate.h b/libcxx/include/__cxx03/__algorithm/ranges_rotate.h
index 2efa88748ff70..e6bef76b3cb74 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_rotate.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_rotate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ROTATE_H
-#define _LIBCPP___ALGORITHM_RANGES_ROTATE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/ranges_iterator_concept.h>
@@ -68,4 +68,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ROTATE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
index 7aa1fa13feddd..40975ad7213de 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_rotate_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/ranges_copy.h>
@@ -65,4 +65,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_ROTATE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_ROTATE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sample.h b/libcxx/include/__cxx03/__algorithm/ranges_sample.h
index 08a44d3a3c855..778661c2ce413 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_sample.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sample.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
-#define _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SAMPLE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SAMPLE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/sample.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SAMPLE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SAMPLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_search.h b/libcxx/include/__cxx03/__algorithm/ranges_search.h
index 68ac1fce9e7bf..a4d56b80c9452 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_search.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_search.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SEARCH_H
-#define _LIBCPP___ALGORITHM_RANGES_SEARCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/search.h>
@@ -131,4 +131,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SEARCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_search_n.h b/libcxx/include/__cxx03/__algorithm/ranges_search_n.h
index c7c670f02c20c..eac6e33a4da9e 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_search_n.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_search_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
-#define _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_N_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_N_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/search_n.h>
@@ -113,4 +113,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SEARCH_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SEARCH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h b/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
index 764384b970833..0343b68f8534d 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_difference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
-#define _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SET_DIFFERENCE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SET_DIFFERENCE_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -106,4 +106,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SET_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h b/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
index 4aebb5e195da2..ff0ae0fe1446f 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_intersection.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
-#define _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SET_INTERSECTION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SET_INTERSECTION_H
 
 #include <__cxx03/__algorithm/in_in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -111,4 +111,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SET_INTERSECTION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SET_INTERSECTION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h b/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
index 50ec574fcc301..4c7413d5a656a 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_symmetric_difference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
-#define _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
 
 #include <__cxx03/__algorithm/in_in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -111,4 +111,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_set_union.h b/libcxx/include/__cxx03/__algorithm/ranges_set_union.h
index 87d08e4cfedcc..0752ee51d37d1 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_set_union.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_set_union.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
-#define _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SET_UNION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SET_UNION_H
 
 #include <__cxx03/__algorithm/in_in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -112,4 +112,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SET_UNION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SET_UNION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h b/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
index 44e3a73e1b4b7..2728d5df97f58 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_shuffle.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
-#define _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SHUFFLE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SHUFFLE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/shuffle.h>
@@ -69,4 +69,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SHUFFLE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SHUFFLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_sort.h
index ad24cea344e8b..4fa26ad238fa1 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_sort.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SORT_H
-#define _LIBCPP___ALGORITHM_RANGES_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -81,4 +81,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h b/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
index 403917b032ddc..7c45fca156bae 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_sort_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
-#define _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_HEAP_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SORT_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SORT_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h b/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
index cd4ae07bf9c70..b7d84dc6a0ccb 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_stable_partition.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
-#define _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_PARTITION_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_PARTITION_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -89,4 +89,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_STABLE_PARTITION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h b/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
index 20c83283b9606..83ff06971d081 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_stable_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
-#define _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_SORT_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -79,4 +79,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_STABLE_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h b/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
index 0e2424a0d8a03..91117ec29413f 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_starts_with.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
-#define _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_STARTS_WITH_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_STARTS_WITH_H
 
 #include <__cxx03/__algorithm/in_in_result.h>
 #include <__cxx03/__algorithm/ranges_mismatch.h>
@@ -92,4 +92,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_STARTS_WITH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_STARTS_WITH_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h b/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
index 610e7c315486b..46a868f3a563f 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_swap_ranges.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
-#define _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_SWAP_RANGES_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_SWAP_RANGES_H
 
 #include <__cxx03/__algorithm/in_in_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -67,4 +67,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_SWAP_RANGES_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_SWAP_RANGES_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_transform.h b/libcxx/include/__cxx03/__algorithm/ranges_transform.h
index 12e4a50154aae..ba6ec5a3ea201 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_transform.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_transform.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
-#define _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_TRANSFORM_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_TRANSFORM_H
 
 #include <__cxx03/__algorithm/in_in_out_result.h>
 #include <__cxx03/__algorithm/in_out_result.h>
@@ -174,4 +174,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_TRANSFORM_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_unique.h b/libcxx/include/__cxx03/__algorithm/ranges_unique.h
index 0893127dd9d17..d54b1c5205c28 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_unique.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_unique.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
-#define _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/make_projected.h>
@@ -79,4 +79,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_UNIQUE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h b/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
index c30332a217f7e..6eed79b258998 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_unique_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
-#define _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_COPY_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -117,4 +117,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_RANGES_UNIQUE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_UNIQUE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h b/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
index 79b49ab804b3b..004b9ae654f85 100644
--- a/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
+++ b/libcxx/include/__cxx03/__algorithm/ranges_upper_bound.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
-#define _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_UPPER_BOUND_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_UPPER_BOUND_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/lower_bound.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_UPPER_BOUND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_UPPER_BOUND_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove.h b/libcxx/include/__cxx03/__algorithm/remove.h
index 7849527585b17..208351e22ac90 100644
--- a/libcxx/include/__cxx03/__algorithm/remove.h
+++ b/libcxx/include/__cxx03/__algorithm/remove.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REMOVE_H
-#define _LIBCPP___ALGORITHM_REMOVE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REMOVE_H
+#define _LIBCPP___CXX03___ALGORITHM_REMOVE_H
 
 #include <__cxx03/__algorithm/find.h>
 #include <__cxx03/__algorithm/find_if.h>
@@ -43,4 +43,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_REMOVE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REMOVE_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_copy.h b/libcxx/include/__cxx03/__algorithm/remove_copy.h
index e79c798127161..1bed25224281b 100644
--- a/libcxx/include/__cxx03/__algorithm/remove_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/remove_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REMOVE_COPY_H
-#define _LIBCPP___ALGORITHM_REMOVE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_H
 
 #include <__cxx03/__config>
 
@@ -31,4 +31,4 @@ remove_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __res
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REMOVE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_copy_if.h b/libcxx/include/__cxx03/__algorithm/remove_copy_if.h
index 7132e3c0bdb1a..3ec019dfd5912 100644
--- a/libcxx/include/__cxx03/__algorithm/remove_copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/remove_copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
-#define _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_IF_H
 
 #include <__cxx03/__config>
 
@@ -31,4 +31,4 @@ remove_copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REMOVE_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REMOVE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/remove_if.h b/libcxx/include/__cxx03/__algorithm/remove_if.h
index e6dc3d15fbc9c..c64e0aa4477e5 100644
--- a/libcxx/include/__cxx03/__algorithm/remove_if.h
+++ b/libcxx/include/__cxx03/__algorithm/remove_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REMOVE_IF_H
-#define _LIBCPP___ALGORITHM_REMOVE_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REMOVE_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_REMOVE_IF_H
 
 #include <__cxx03/__algorithm/find_if.h>
 #include <__cxx03/__config>
@@ -42,4 +42,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_REMOVE_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REMOVE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace.h b/libcxx/include/__cxx03/__algorithm/replace.h
index 2b24752f048ef..692cece1708f9 100644
--- a/libcxx/include/__cxx03/__algorithm/replace.h
+++ b/libcxx/include/__cxx03/__algorithm/replace.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REPLACE_H
-#define _LIBCPP___ALGORITHM_REPLACE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REPLACE_H
+#define _LIBCPP___CXX03___ALGORITHM_REPLACE_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ replace(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_valu
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REPLACE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REPLACE_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_copy.h b/libcxx/include/__cxx03/__algorithm/replace_copy.h
index 6c50dd4e14f57..4f8b375df2fb7 100644
--- a/libcxx/include/__cxx03/__algorithm/replace_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/replace_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REPLACE_COPY_H
-#define _LIBCPP___ALGORITHM_REPLACE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_H
 
 #include <__cxx03/__config>
 
@@ -34,4 +34,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator repla
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REPLACE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_copy_if.h b/libcxx/include/__cxx03/__algorithm/replace_copy_if.h
index c714d50037339..cfc7b0aa2d34c 100644
--- a/libcxx/include/__cxx03/__algorithm/replace_copy_if.h
+++ b/libcxx/include/__cxx03/__algorithm/replace_copy_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
-#define _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_IF_H
 
 #include <__cxx03/__config>
 
@@ -34,4 +34,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator repla
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REPLACE_COPY_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REPLACE_COPY_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/replace_if.h b/libcxx/include/__cxx03/__algorithm/replace_if.h
index 8bd9a9c0db979..f46da35714ef3 100644
--- a/libcxx/include/__cxx03/__algorithm/replace_if.h
+++ b/libcxx/include/__cxx03/__algorithm/replace_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REPLACE_IF_H
-#define _LIBCPP___ALGORITHM_REPLACE_IF_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REPLACE_IF_H
+#define _LIBCPP___CXX03___ALGORITHM_REPLACE_IF_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ replace_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REPLACE_IF_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REPLACE_IF_H
diff --git a/libcxx/include/__cxx03/__algorithm/reverse.h b/libcxx/include/__cxx03/__algorithm/reverse.h
index b97ec38490a22..868377c7b26bd 100644
--- a/libcxx/include/__cxx03/__algorithm/reverse.h
+++ b/libcxx/include/__cxx03/__algorithm/reverse.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REVERSE_H
-#define _LIBCPP___ALGORITHM_REVERSE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REVERSE_H
+#define _LIBCPP___CXX03___ALGORITHM_REVERSE_H
 
 #include <__cxx03/__algorithm/iter_swap.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -59,4 +59,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_REVERSE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REVERSE_H
diff --git a/libcxx/include/__cxx03/__algorithm/reverse_copy.h b/libcxx/include/__cxx03/__algorithm/reverse_copy.h
index 29070fec8448c..3553102a2d03c 100644
--- a/libcxx/include/__cxx03/__algorithm/reverse_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/reverse_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_REVERSE_COPY_H
-#define _LIBCPP___ALGORITHM_REVERSE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_REVERSE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_REVERSE_COPY_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _Out
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_REVERSE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_REVERSE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/rotate.h b/libcxx/include/__cxx03/__algorithm/rotate.h
index d6ca9230731ed..e41edf00e7993 100644
--- a/libcxx/include/__cxx03/__algorithm/rotate.h
+++ b/libcxx/include/__cxx03/__algorithm/rotate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ROTATE_H
-#define _LIBCPP___ALGORITHM_ROTATE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ROTATE_H
+#define _LIBCPP___CXX03___ALGORITHM_ROTATE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/move.h>
@@ -195,4 +195,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_ROTATE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ROTATE_H
diff --git a/libcxx/include/__cxx03/__algorithm/rotate_copy.h b/libcxx/include/__cxx03/__algorithm/rotate_copy.h
index c20d9571e6ffd..6970cdc5a2c56 100644
--- a/libcxx/include/__cxx03/__algorithm/rotate_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/rotate_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_ROTATE_COPY_H
-#define _LIBCPP___ALGORITHM_ROTATE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_ROTATE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_ROTATE_COPY_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__config>
@@ -26,4 +26,4 @@ rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterato
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_ROTATE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_ROTATE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/sample.h b/libcxx/include/__cxx03/__algorithm/sample.h
index e6743cf3828c0..55e34c9f24813 100644
--- a/libcxx/include/__cxx03/__algorithm/sample.h
+++ b/libcxx/include/__cxx03/__algorithm/sample.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SAMPLE_H
-#define _LIBCPP___ALGORITHM_SAMPLE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SAMPLE_H
+#define _LIBCPP___CXX03___ALGORITHM_SAMPLE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/min.h>
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SAMPLE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SAMPLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/search.h b/libcxx/include/__cxx03/__algorithm/search.h
index eb862f4ad7ca9..7ca6bdbc2beb5 100644
--- a/libcxx/include/__cxx03/__algorithm/search.h
+++ b/libcxx/include/__cxx03/__algorithm/search.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SEARCH_H
-#define _LIBCPP___ALGORITHM_SEARCH_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SEARCH_H
+#define _LIBCPP___CXX03___ALGORITHM_SEARCH_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -189,4 +189,4 @@ search(_ForwardIterator __f, _ForwardIterator __l, const _Searcher& __s) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_SEARCH_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SEARCH_H
diff --git a/libcxx/include/__cxx03/__algorithm/search_n.h b/libcxx/include/__cxx03/__algorithm/search_n.h
index d6458b8ca90d8..cf63de27666b2 100644
--- a/libcxx/include/__cxx03/__algorithm/search_n.h
+++ b/libcxx/include/__cxx03/__algorithm/search_n.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SEARCH_N_H
-#define _LIBCPP___ALGORITHM_SEARCH_N_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SEARCH_N_H
+#define _LIBCPP___CXX03___ALGORITHM_SEARCH_N_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -152,4 +152,4 @@ search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_SEARCH_N_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SEARCH_N_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_difference.h b/libcxx/include/__cxx03/__algorithm/set_difference.h
index 13d6f0249e436..1bdddd3955980 100644
--- a/libcxx/include/__cxx03/__algorithm/set_difference.h
+++ b/libcxx/include/__cxx03/__algorithm/set_difference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
-#define _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SET_DIFFERENCE_H
+#define _LIBCPP___CXX03___ALGORITHM_SET_DIFFERENCE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -76,4 +76,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SET_DIFFERENCE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SET_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_intersection.h b/libcxx/include/__cxx03/__algorithm/set_intersection.h
index f1193ec349cfd..8164ced338d20 100644
--- a/libcxx/include/__cxx03/__algorithm/set_intersection.h
+++ b/libcxx/include/__cxx03/__algorithm/set_intersection.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SET_INTERSECTION_H
-#define _LIBCPP___ALGORITHM_SET_INTERSECTION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SET_INTERSECTION_H
+#define _LIBCPP___CXX03___ALGORITHM_SET_INTERSECTION_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -214,4 +214,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SET_INTERSECTION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SET_INTERSECTION_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h b/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
index 71aca2b8ed03b..64fdf4543be9c 100644
--- a/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
+++ b/libcxx/include/__cxx03/__algorithm/set_symmetric_difference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
-#define _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
+#define _LIBCPP___CXX03___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -106,4 +106,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SET_SYMMETRIC_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__algorithm/set_union.h b/libcxx/include/__cxx03/__algorithm/set_union.h
index f60221cce3cd8..a5c6d5eabd394 100644
--- a/libcxx/include/__cxx03/__algorithm/set_union.h
+++ b/libcxx/include/__cxx03/__algorithm/set_union.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SET_UNION_H
-#define _LIBCPP___ALGORITHM_SET_UNION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SET_UNION_H
+#define _LIBCPP___CXX03___ALGORITHM_SET_UNION_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -102,4 +102,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SET_UNION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SET_UNION_H
diff --git a/libcxx/include/__cxx03/__algorithm/shift_left.h b/libcxx/include/__cxx03/__algorithm/shift_left.h
index b59a069826710..886f353f2814c 100644
--- a/libcxx/include/__cxx03/__algorithm/shift_left.h
+++ b/libcxx/include/__cxx03/__algorithm/shift_left.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SHIFT_LEFT_H
-#define _LIBCPP___ALGORITHM_SHIFT_LEFT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SHIFT_LEFT_H
+#define _LIBCPP___CXX03___ALGORITHM_SHIFT_LEFT_H
 
 #include <__cxx03/__algorithm/move.h>
 #include <__cxx03/__config>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SHIFT_LEFT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SHIFT_LEFT_H
diff --git a/libcxx/include/__cxx03/__algorithm/shift_right.h b/libcxx/include/__cxx03/__algorithm/shift_right.h
index 51d8ea613245d..4bff76af2fd99 100644
--- a/libcxx/include/__cxx03/__algorithm/shift_right.h
+++ b/libcxx/include/__cxx03/__algorithm/shift_right.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
-#define _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SHIFT_RIGHT_H
+#define _LIBCPP___CXX03___ALGORITHM_SHIFT_RIGHT_H
 
 #include <__cxx03/__algorithm/move.h>
 #include <__cxx03/__algorithm/move_backward.h>
@@ -102,4 +102,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SHIFT_RIGHT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SHIFT_RIGHT_H
diff --git a/libcxx/include/__cxx03/__algorithm/shuffle.h b/libcxx/include/__cxx03/__algorithm/shuffle.h
index 30b372ffe767a..abb96bce40236 100644
--- a/libcxx/include/__cxx03/__algorithm/shuffle.h
+++ b/libcxx/include/__cxx03/__algorithm/shuffle.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SHUFFLE_H
-#define _LIBCPP___ALGORITHM_SHUFFLE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SHUFFLE_H
+#define _LIBCPP___CXX03___ALGORITHM_SHUFFLE_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__config>
@@ -164,4 +164,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SHUFFLE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SHUFFLE_H
diff --git a/libcxx/include/__cxx03/__algorithm/sift_down.h b/libcxx/include/__cxx03/__algorithm/sift_down.h
index 85f0aa92c568d..774a6d2450d57 100644
--- a/libcxx/include/__cxx03/__algorithm/sift_down.h
+++ b/libcxx/include/__cxx03/__algorithm/sift_down.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SIFT_DOWN_H
-#define _LIBCPP___ALGORITHM_SIFT_DOWN_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SIFT_DOWN_H
+#define _LIBCPP___CXX03___ALGORITHM_SIFT_DOWN_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__assert>
@@ -115,4 +115,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SIFT_DOWN_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SIFT_DOWN_H
diff --git a/libcxx/include/__cxx03/__algorithm/simd_utils.h b/libcxx/include/__cxx03/__algorithm/simd_utils.h
index 4769a8176a885..db2e20fb65ece 100644
--- a/libcxx/include/__cxx03/__algorithm/simd_utils.h
+++ b/libcxx/include/__cxx03/__algorithm/simd_utils.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H
-#define _LIBCPP___ALGORITHM_SIMD_UTILS_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H
+#define _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__bit/bit_cast.h>
@@ -161,4 +161,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H
diff --git a/libcxx/include/__cxx03/__algorithm/sort.h b/libcxx/include/__cxx03/__algorithm/sort.h
index d14ec87b4aea8..009ebf717bbd8 100644
--- a/libcxx/include/__cxx03/__algorithm/sort.h
+++ b/libcxx/include/__cxx03/__algorithm/sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SORT_H
-#define _LIBCPP___ALGORITHM_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_SORT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -1013,4 +1013,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/sort_heap.h b/libcxx/include/__cxx03/__algorithm/sort_heap.h
index c2d218b8f5e0d..b5a341103980e 100644
--- a/libcxx/include/__cxx03/__algorithm/sort_heap.h
+++ b/libcxx/include/__cxx03/__algorithm/sort_heap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SORT_HEAP_H
-#define _LIBCPP___ALGORITHM_SORT_HEAP_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SORT_HEAP_H
+#define _LIBCPP___CXX03___ALGORITHM_SORT_HEAP_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -60,4 +60,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SORT_HEAP_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SORT_HEAP_H
diff --git a/libcxx/include/__cxx03/__algorithm/stable_partition.h b/libcxx/include/__cxx03/__algorithm/stable_partition.h
index cea18b30fbb37..c8044987e247a 100644
--- a/libcxx/include/__cxx03/__algorithm/stable_partition.h
+++ b/libcxx/include/__cxx03/__algorithm/stable_partition.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_STABLE_PARTITION_H
-#define _LIBCPP___ALGORITHM_STABLE_PARTITION_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_STABLE_PARTITION_H
+#define _LIBCPP___CXX03___ALGORITHM_STABLE_PARTITION_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__algorithm/rotate.h>
@@ -304,4 +304,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_STABLE_PARTITION_H
+#endif // _LIBCPP___CXX03___ALGORITHM_STABLE_PARTITION_H
diff --git a/libcxx/include/__cxx03/__algorithm/stable_sort.h b/libcxx/include/__cxx03/__algorithm/stable_sort.h
index 6a3cef7ec1b66..e1f89f6341b9f 100644
--- a/libcxx/include/__cxx03/__algorithm/stable_sort.h
+++ b/libcxx/include/__cxx03/__algorithm/stable_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_STABLE_SORT_H
-#define _LIBCPP___ALGORITHM_STABLE_SORT_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_STABLE_SORT_H
+#define _LIBCPP___CXX03___ALGORITHM_STABLE_SORT_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/comp_ref_type.h>
@@ -270,4 +270,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_STABLE_SORT_H
+#endif // _LIBCPP___CXX03___ALGORITHM_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__algorithm/swap_ranges.h b/libcxx/include/__cxx03/__algorithm/swap_ranges.h
index 12bc8056e3577..9a19ffe602b33 100644
--- a/libcxx/include/__cxx03/__algorithm/swap_ranges.h
+++ b/libcxx/include/__cxx03/__algorithm/swap_ranges.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_SWAP_RANGES_H
-#define _LIBCPP___ALGORITHM_SWAP_RANGES_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_SWAP_RANGES_H
+#define _LIBCPP___CXX03___ALGORITHM_SWAP_RANGES_H
 
 #include <__cxx03/__algorithm/iterator_operations.h>
 #include <__cxx03/__config>
@@ -59,4 +59,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_SWAP_RANGES_H
+#endif // _LIBCPP___CXX03___ALGORITHM_SWAP_RANGES_H
diff --git a/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h b/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
index be6a4e4d01482..4463bd2115482 100644
--- a/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
+++ b/libcxx/include/__cxx03/__algorithm/three_way_comp_ref_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
-#define _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
+#define _LIBCPP___CXX03___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__compare/ordering.h>
@@ -71,4 +71,4 @@ using __three_way_comp_ref_type = _Comp&;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_THREE_WAY_COMP_REF_TYPE_H
diff --git a/libcxx/include/__cxx03/__algorithm/transform.h b/libcxx/include/__cxx03/__algorithm/transform.h
index 1608932b050b4..4bed1ed4f8d59 100644
--- a/libcxx/include/__cxx03/__algorithm/transform.h
+++ b/libcxx/include/__cxx03/__algorithm/transform.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_TRANSFORM_H
-#define _LIBCPP___ALGORITHM_TRANSFORM_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_TRANSFORM_H
+#define _LIBCPP___CXX03___ALGORITHM_TRANSFORM_H
 
 #include <__cxx03/__config>
 
@@ -39,4 +39,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator trans
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ALGORITHM_TRANSFORM_H
+#endif // _LIBCPP___CXX03___ALGORITHM_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h b/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
index 25a9e35fd7aa7..8d3858226b992 100644
--- a/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
+++ b/libcxx/include/__cxx03/__algorithm/uniform_random_bit_generator_adaptor.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
-#define _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
+#define _LIBCPP___CXX03___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -55,4 +55,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
+#endif // _LIBCPP___CXX03___ALGORITHM_RANGES_UNIFORM_RANDOM_BIT_GENERATOR_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__algorithm/unique.h b/libcxx/include/__cxx03/__algorithm/unique.h
index 8d21c63eb2b7d..b7eb2849e4e37 100644
--- a/libcxx/include/__cxx03/__algorithm/unique.h
+++ b/libcxx/include/__cxx03/__algorithm/unique.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_UNIQUE_H
-#define _LIBCPP___ALGORITHM_UNIQUE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_UNIQUE_H
+#define _LIBCPP___CXX03___ALGORITHM_UNIQUE_H
 
 #include <__cxx03/__algorithm/adjacent_find.h>
 #include <__cxx03/__algorithm/comp.h>
@@ -61,4 +61,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_UNIQUE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_UNIQUE_H
diff --git a/libcxx/include/__cxx03/__algorithm/unique_copy.h b/libcxx/include/__cxx03/__algorithm/unique_copy.h
index cfa95a58f7b09..6d3daefaa1ca5 100644
--- a/libcxx/include/__cxx03/__algorithm/unique_copy.h
+++ b/libcxx/include/__cxx03/__algorithm/unique_copy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_UNIQUE_COPY_H
-#define _LIBCPP___ALGORITHM_UNIQUE_COPY_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_UNIQUE_COPY_H
+#define _LIBCPP___CXX03___ALGORITHM_UNIQUE_COPY_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/iterator_operations.h>
@@ -124,4 +124,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_UNIQUE_COPY_H
+#endif // _LIBCPP___CXX03___ALGORITHM_UNIQUE_COPY_H
diff --git a/libcxx/include/__cxx03/__algorithm/unwrap_iter.h b/libcxx/include/__cxx03/__algorithm/unwrap_iter.h
index b3259af17dd03..055f56b9db155 100644
--- a/libcxx/include/__cxx03/__algorithm/unwrap_iter.h
+++ b/libcxx/include/__cxx03/__algorithm/unwrap_iter.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_UNWRAP_ITER_H
-#define _LIBCPP___ALGORITHM_UNWRAP_ITER_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_UNWRAP_ITER_H
+#define _LIBCPP___CXX03___ALGORITHM_UNWRAP_ITER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_UNWRAP_ITER_H
+#endif // _LIBCPP___CXX03___ALGORITHM_UNWRAP_ITER_H
diff --git a/libcxx/include/__cxx03/__algorithm/unwrap_range.h b/libcxx/include/__cxx03/__algorithm/unwrap_range.h
index 26045ef7075af..0b81f1e805c8a 100644
--- a/libcxx/include/__cxx03/__algorithm/unwrap_range.h
+++ b/libcxx/include/__cxx03/__algorithm/unwrap_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
-#define _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_UNWRAP_RANGE_H
+#define _LIBCPP___CXX03___ALGORITHM_UNWRAP_RANGE_H
 
 #include <__cxx03/__algorithm/unwrap_iter.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -96,4 +96,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_UNWRAP_RANGE_H
+#endif // _LIBCPP___CXX03___ALGORITHM_UNWRAP_RANGE_H
diff --git a/libcxx/include/__cxx03/__algorithm/upper_bound.h b/libcxx/include/__cxx03/__algorithm/upper_bound.h
index 069dd6d253e28..3a0f1f560cb77 100644
--- a/libcxx/include/__cxx03/__algorithm/upper_bound.h
+++ b/libcxx/include/__cxx03/__algorithm/upper_bound.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_UPPER_BOUND_H
-#define _LIBCPP___ALGORITHM_UPPER_BOUND_H
+#ifndef _LIBCPP___CXX03___ALGORITHM_UPPER_BOUND_H
+#define _LIBCPP___CXX03___ALGORITHM_UPPER_BOUND_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__algorithm/half_positive.h>
@@ -65,4 +65,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_UPPER_BOUND_H
+#endif // _LIBCPP___CXX03___ALGORITHM_UPPER_BOUND_H
diff --git a/libcxx/include/__cxx03/__assert b/libcxx/include/__cxx03/__assert
index 19a322934e41d..f6c4d8e4a5d09 100644
--- a/libcxx/include/__cxx03/__assert
+++ b/libcxx/include/__cxx03/__assert
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ASSERT
-#define _LIBCPP___ASSERT
+#ifndef _LIBCPP___CXX03___ASSERT
+#define _LIBCPP___CXX03___ASSERT
 
 #include <__assertion_handler> // Note: this include is generated by CMake and is potentially vendor-provided.
 #include <__cxx03/__config>
@@ -115,4 +115,4 @@
 #endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
 // clang-format on
 
-#endif // _LIBCPP___ASSERT
+#endif // _LIBCPP___CXX03___ASSERT
diff --git a/libcxx/include/__cxx03/__atomic/aliases.h b/libcxx/include/__cxx03/__atomic/aliases.h
index b5a7685eb4fa8..918ffc35ccf63 100644
--- a/libcxx/include/__cxx03/__atomic/aliases.h
+++ b/libcxx/include/__cxx03/__atomic/aliases.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ALIASES_H
-#define _LIBCPP___ATOMIC_ALIASES_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ALIASES_H
+#define _LIBCPP___CXX03___ATOMIC_ALIASES_H
 
 #include <__cxx03/__atomic/atomic.h>
 #include <__cxx03/__atomic/atomic_lock_free.h>
@@ -107,4 +107,4 @@ using atomic_unsigned_lock_free = atomic<make_unsigned_t<__contention_t_or_large
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_ALIASES_H
+#endif // _LIBCPP___CXX03___ATOMIC_ALIASES_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic.h b/libcxx/include/__cxx03/__atomic/atomic.h
index 0e5936e2eb542..9ac800613f824 100644
--- a/libcxx/include/__cxx03/__atomic/atomic.h
+++ b/libcxx/include/__cxx03/__atomic/atomic.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_H
-#define _LIBCPP___ATOMIC_ATOMIC_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_H
 
 #include <__cxx03/__atomic/atomic_base.h>
 #include <__cxx03/__atomic/check_memory_order.h>
@@ -619,4 +619,4 @@ atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __o
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_base.h b/libcxx/include/__cxx03/__atomic/atomic_base.h
index ae6aaf4f8284f..e4d4929b8912e 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_base.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_base.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_BASE_H
-#define _LIBCPP___ATOMIC_ATOMIC_BASE_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_BASE_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_BASE_H
 
 #include <__cxx03/__atomic/atomic_sync.h>
 #include <__cxx03/__atomic/check_memory_order.h>
@@ -218,4 +218,4 @@ struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_BASE_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_flag.h b/libcxx/include/__cxx03/__atomic/atomic_flag.h
index 6a8471e1f1ec8..c119e9bc498e6 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_flag.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_flag.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_FLAG_H
-#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_FLAG_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_FLAG_H
 
 #include <__cxx03/__atomic/atomic_sync.h>
 #include <__cxx03/__atomic/contention_t.h>
@@ -186,4 +186,4 @@ atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_FLAG_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_FLAG_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_init.h b/libcxx/include/__cxx03/__atomic/atomic_init.h
index 666afb93268d6..edabfe1a12bb4 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_init.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_init.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_INIT_H
-#define _LIBCPP___ATOMIC_ATOMIC_INIT_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_INIT_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_INIT_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@
 #  pragma clang deprecated(ATOMIC_VAR_INIT)
 #endif
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_INIT_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_INIT_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_lock_free.h b/libcxx/include/__cxx03/__atomic/atomic_lock_free.h
index 17dfcfd46106e..297d148babc83 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_lock_free.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_lock_free.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
-#define _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_LOCK_FREE_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_LOCK_FREE_H
 
 #include <__cxx03/__config>
 
@@ -45,4 +45,4 @@
 #  define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
 #endif
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_LOCK_FREE_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_ref.h b/libcxx/include/__cxx03/__atomic/atomic_ref.h
index c19e6a2e5d26d..f30aac3210e27 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_ref.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_ref.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
-#define _LIBCPP___ATOMIC_ATOMIC_REF_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_REF_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_REF_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__atomic/atomic_sync.h>
@@ -375,4 +375,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
+#endif // _LIBCPP___CXX03__ATOMIC_ATOMIC_REF_H
diff --git a/libcxx/include/__cxx03/__atomic/atomic_sync.h b/libcxx/include/__cxx03/__atomic/atomic_sync.h
index 815c8a1459649..ca029f0384058 100644
--- a/libcxx/include/__cxx03/__atomic/atomic_sync.h
+++ b/libcxx/include/__cxx03/__atomic/atomic_sync.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
-#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
+#ifndef _LIBCPP___CXX03___ATOMIC_ATOMIC_SYNC_H
+#define _LIBCPP___CXX03___ATOMIC_ATOMIC_SYNC_H
 
 #include <__cxx03/__atomic/contention_t.h>
 #include <__cxx03/__atomic/cxx_atomic_impl.h>
@@ -202,4 +202,4 @@ __atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H
+#endif // _LIBCPP___CXX03___ATOMIC_ATOMIC_SYNC_H
diff --git a/libcxx/include/__cxx03/__atomic/check_memory_order.h b/libcxx/include/__cxx03/__atomic/check_memory_order.h
index 11033ef2c3b49..5661efd4bad27 100644
--- a/libcxx/include/__cxx03/__atomic/check_memory_order.h
+++ b/libcxx/include/__cxx03/__atomic/check_memory_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
-#define _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
+#ifndef _LIBCPP___CXX03___ATOMIC_CHECK_MEMORY_ORDER_H
+#define _LIBCPP___CXX03___ATOMIC_CHECK_MEMORY_ORDER_H
 
 #include <__cxx03/__config>
 
@@ -31,4 +31,4 @@
   _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel,                                 \
                            "memory order argument to atomic operation is invalid")
 
-#endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H
+#endif // _LIBCPP___CXX03___ATOMIC_CHECK_MEMORY_ORDER_H
diff --git a/libcxx/include/__cxx03/__atomic/contention_t.h b/libcxx/include/__cxx03/__atomic/contention_t.h
index 1f069b01b28e7..a97f0668da2fe 100644
--- a/libcxx/include/__cxx03/__atomic/contention_t.h
+++ b/libcxx/include/__cxx03/__atomic/contention_t.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_CONTENTION_T_H
-#define _LIBCPP___ATOMIC_CONTENTION_T_H
+#ifndef _LIBCPP___CXX03___ATOMIC_CONTENTION_T_H
+#define _LIBCPP___CXX03___ATOMIC_CONTENTION_T_H
 
 #include <__cxx03/__atomic/cxx_atomic_impl.h>
 #include <__cxx03/__config>
@@ -29,4 +29,4 @@ using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_CONTENTION_T_H
+#endif // _LIBCPP___CXX03___ATOMIC_CONTENTION_T_H
diff --git a/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h b/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
index f06627f1a8f66..990d283c62d1a 100644
--- a/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__cxx03/__atomic/cxx_atomic_impl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
-#define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+#ifndef _LIBCPP___CXX03___ATOMIC_CXX_ATOMIC_IMPL_H
+#define _LIBCPP___CXX03___ATOMIC_CXX_ATOMIC_IMPL_H
 
 #include <__cxx03/__atomic/memory_order.h>
 #include <__cxx03/__atomic/to_gcc_order.h>
@@ -507,4 +507,4 @@ struct __cxx_atomic_impl : public _Base {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
+#endif // _LIBCPP___CXX03___ATOMIC_CXX_ATOMIC_IMPL_H
diff --git a/libcxx/include/__cxx03/__atomic/fence.h b/libcxx/include/__cxx03/__atomic/fence.h
index 5200cd533bdeb..05eaa83f36406 100644
--- a/libcxx/include/__cxx03/__atomic/fence.h
+++ b/libcxx/include/__cxx03/__atomic/fence.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_FENCE_H
-#define _LIBCPP___ATOMIC_FENCE_H
+#ifndef _LIBCPP___CXX03___ATOMIC_FENCE_H
+#define _LIBCPP___CXX03___ATOMIC_FENCE_H
 
 #include <__cxx03/__atomic/cxx_atomic_impl.h>
 #include <__cxx03/__atomic/memory_order.h>
@@ -25,4 +25,4 @@ inline _LIBCPP_HIDE_FROM_ABI void atomic_signal_fence(memory_order __m) _NOEXCEP
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_FENCE_H
+#endif // _LIBCPP___CXX03___ATOMIC_FENCE_H
diff --git a/libcxx/include/__cxx03/__atomic/is_always_lock_free.h b/libcxx/include/__cxx03/__atomic/is_always_lock_free.h
index 29c42d9340fdb..45628ae53214b 100644
--- a/libcxx/include/__cxx03/__atomic/is_always_lock_free.h
+++ b/libcxx/include/__cxx03/__atomic/is_always_lock_free.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
-#define _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
+#ifndef _LIBCPP___CXX03___ATOMIC_IS_ALWAYS_LOCK_FREE_H
+#define _LIBCPP___CXX03___ATOMIC_IS_ALWAYS_LOCK_FREE_H
 
 #include <__cxx03/__config>
 
@@ -25,4 +25,4 @@ struct __libcpp_is_always_lock_free {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H
+#endif // _LIBCPP___CXX03___ATOMIC_IS_ALWAYS_LOCK_FREE_H
diff --git a/libcxx/include/__cxx03/__atomic/kill_dependency.h b/libcxx/include/__cxx03/__atomic/kill_dependency.h
index 3deb29f0dbda1..1e0dfc81e3c5c 100644
--- a/libcxx/include/__cxx03/__atomic/kill_dependency.h
+++ b/libcxx/include/__cxx03/__atomic/kill_dependency.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
-#define _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
+#ifndef _LIBCPP___CXX03___ATOMIC_KILL_DEPENDENCY_H
+#define _LIBCPP___CXX03___ATOMIC_KILL_DEPENDENCY_H
 
 #include <__cxx03/__config>
 
@@ -24,4 +24,4 @@ _LIBCPP_HIDE_FROM_ABI _Tp kill_dependency(_Tp __y) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_KILL_DEPENDENCY_H
+#endif // _LIBCPP___CXX03___ATOMIC_KILL_DEPENDENCY_H
diff --git a/libcxx/include/__cxx03/__atomic/memory_order.h b/libcxx/include/__cxx03/__atomic/memory_order.h
index 8a7564dc1553e..b8b9dda3d1150 100644
--- a/libcxx/include/__cxx03/__atomic/memory_order.h
+++ b/libcxx/include/__cxx03/__atomic/memory_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_MEMORY_ORDER_H
-#define _LIBCPP___ATOMIC_MEMORY_ORDER_H
+#ifndef _LIBCPP___CXX03___ATOMIC_MEMORY_ORDER_H
+#define _LIBCPP___CXX03___ATOMIC_MEMORY_ORDER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_same.h>
@@ -62,4 +62,4 @@ enum memory_order {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_MEMORY_ORDER_H
+#endif // _LIBCPP___CXX03___ATOMIC_MEMORY_ORDER_H
diff --git a/libcxx/include/__cxx03/__atomic/to_gcc_order.h b/libcxx/include/__cxx03/__atomic/to_gcc_order.h
index 41ada88281732..aab3c59602f11 100644
--- a/libcxx/include/__cxx03/__atomic/to_gcc_order.h
+++ b/libcxx/include/__cxx03/__atomic/to_gcc_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ATOMIC_TO_GCC_ORDER_H
-#define _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+#ifndef _LIBCPP___CXX03___ATOMIC_TO_GCC_ORDER_H
+#define _LIBCPP___CXX03___ATOMIC_TO_GCC_ORDER_H
 
 #include <__cxx03/__atomic/memory_order.h>
 #include <__cxx03/__config>
@@ -51,4 +51,4 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ATOMIC_TO_GCC_ORDER_H
+#endif // _LIBCPP___CXX03___ATOMIC_TO_GCC_ORDER_H
diff --git a/libcxx/include/__cxx03/__bit/bit_cast.h b/libcxx/include/__cxx03/__bit/bit_cast.h
index 9f88805e125dc..2613dc4aa332a 100644
--- a/libcxx/include/__cxx03/__bit/bit_cast.h
+++ b/libcxx/include/__cxx03/__bit/bit_cast.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BIT_CAST_H
-#define _LIBCPP___BIT_BIT_CAST_H
+#ifndef _LIBCPP___CXX03___BIT_BIT_CAST_H
+#define _LIBCPP___CXX03___BIT_BIT_CAST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_trivially_copyable.h>
@@ -41,4 +41,4 @@ template <class _ToType, class _FromType>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BIT_CAST_H
+#endif // _LIBCPP___CXX03___BIT_BIT_CAST_H
diff --git a/libcxx/include/__cxx03/__bit/bit_ceil.h b/libcxx/include/__cxx03/__bit/bit_ceil.h
index b5cb5cc08f422..598344b1d4268 100644
--- a/libcxx/include/__cxx03/__bit/bit_ceil.h
+++ b/libcxx/include/__cxx03/__bit/bit_ceil.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BIT_CEIL_H
-#define _LIBCPP___BIT_BIT_CEIL_H
+#ifndef _LIBCPP___CXX03___BIT_BIT_CEIL_H
+#define _LIBCPP___CXX03___BIT_BIT_CEIL_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__bit/countl.h>
@@ -51,4 +51,4 @@ template <__libcpp_unsigned_integer _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BIT_CEIL_H
+#endif // _LIBCPP___CXX03___BIT_BIT_CEIL_H
diff --git a/libcxx/include/__cxx03/__bit/bit_floor.h b/libcxx/include/__cxx03/__bit/bit_floor.h
index 93379be5e2162..b39d73a64e078 100644
--- a/libcxx/include/__cxx03/__bit/bit_floor.h
+++ b/libcxx/include/__cxx03/__bit/bit_floor.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BIT_FLOOR_H
-#define _LIBCPP___BIT_BIT_FLOOR_H
+#ifndef _LIBCPP___CXX03___BIT_BIT_FLOOR_H
+#define _LIBCPP___CXX03___BIT_BIT_FLOOR_H
 
 #include <__cxx03/__bit/bit_log2.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -31,4 +31,4 @@ template <__libcpp_unsigned_integer _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BIT_FLOOR_H
+#endif // _LIBCPP___CXX03___BIT_BIT_FLOOR_H
diff --git a/libcxx/include/__cxx03/__bit/bit_log2.h b/libcxx/include/__cxx03/__bit/bit_log2.h
index 16a7fb1ec3364..1ca80959e6dac 100644
--- a/libcxx/include/__cxx03/__bit/bit_log2.h
+++ b/libcxx/include/__cxx03/__bit/bit_log2.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BIT_LOG2_H
-#define _LIBCPP___BIT_BIT_LOG2_H
+#ifndef _LIBCPP___CXX03___BIT_BIT_LOG2_H
+#define _LIBCPP___CXX03___BIT_BIT_LOG2_H
 
 #include <__cxx03/__bit/countl.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -31,4 +31,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __bit_log2(_Tp __t) noexcept {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BIT_LOG2_H
+#endif // _LIBCPP___CXX03___BIT_BIT_LOG2_H
diff --git a/libcxx/include/__cxx03/__bit/bit_width.h b/libcxx/include/__cxx03/__bit/bit_width.h
index 4832359f75326..e5d9ffd03ca7f 100644
--- a/libcxx/include/__cxx03/__bit/bit_width.h
+++ b/libcxx/include/__cxx03/__bit/bit_width.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BIT_WIDTH_H
-#define _LIBCPP___BIT_BIT_WIDTH_H
+#ifndef _LIBCPP___CXX03___BIT_BIT_WIDTH_H
+#define _LIBCPP___CXX03___BIT_BIT_WIDTH_H
 
 #include <__cxx03/__bit/bit_log2.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -30,4 +30,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___BIT_BIT_WIDTH_H
+#endif // _LIBCPP___CXX03___BIT_BIT_WIDTH_H
diff --git a/libcxx/include/__cxx03/__bit/blsr.h b/libcxx/include/__cxx03/__bit/blsr.h
index 5375bdf9208d7..ae1d8b588925d 100644
--- a/libcxx/include/__cxx03/__bit/blsr.h
+++ b/libcxx/include/__cxx03/__bit/blsr.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BLSR_H
-#define _LIBCPP___BIT_BLSR_H
+#ifndef _LIBCPP___CXX03___BIT_BLSR_H
+#define _LIBCPP___CXX03___BIT_BLSR_H
 
 #include <__cxx03/__config>
 
@@ -31,4 +31,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR unsigned long long __libcpp_blsr(
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BLSR_H
+#endif // _LIBCPP___CXX03___BIT_BLSR_H
diff --git a/libcxx/include/__cxx03/__bit/byteswap.h b/libcxx/include/__cxx03/__bit/byteswap.h
index 6e65600512284..f40d870ca7fc2 100644
--- a/libcxx/include/__cxx03/__bit/byteswap.h
+++ b/libcxx/include/__cxx03/__bit/byteswap.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_BYTESWAP_H
-#define _LIBCPP___BIT_BYTESWAP_H
+#ifndef _LIBCPP___CXX03___BIT_BYTESWAP_H
+#define _LIBCPP___CXX03___BIT_BYTESWAP_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -50,4 +50,4 @@ template <integral _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_BYTESWAP_H
+#endif // _LIBCPP___CXX03___BIT_BYTESWAP_H
diff --git a/libcxx/include/__cxx03/__bit/countl.h b/libcxx/include/__cxx03/__bit/countl.h
index 9a4a269a7da2d..0c8181b53408c 100644
--- a/libcxx/include/__cxx03/__bit/countl.h
+++ b/libcxx/include/__cxx03/__bit/countl.h
@@ -9,8 +9,8 @@
 // TODO: __builtin_clzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
 //  refactor this code to exclusively use __builtin_clzg.
 
-#ifndef _LIBCPP___BIT_COUNTL_H
-#define _LIBCPP___BIT_COUNTL_H
+#ifndef _LIBCPP___CXX03___BIT_COUNTL_H
+#define _LIBCPP___CXX03___BIT_COUNTL_H
 
 #include <__cxx03/__bit/rotate.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -110,4 +110,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___BIT_COUNTL_H
+#endif // _LIBCPP___CXX03___BIT_COUNTL_H
diff --git a/libcxx/include/__cxx03/__bit/countr.h b/libcxx/include/__cxx03/__bit/countr.h
index 34525f591048b..d9306a234cb30 100644
--- a/libcxx/include/__cxx03/__bit/countr.h
+++ b/libcxx/include/__cxx03/__bit/countr.h
@@ -9,8 +9,8 @@
 // TODO: __builtin_ctzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
 //  refactor this code to exclusively use __builtin_ctzg.
 
-#ifndef _LIBCPP___BIT_COUNTR_H
-#define _LIBCPP___BIT_COUNTR_H
+#ifndef _LIBCPP___CXX03___BIT_COUNTR_H
+#define _LIBCPP___CXX03___BIT_COUNTR_H
 
 #include <__cxx03/__bit/rotate.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -81,4 +81,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___BIT_COUNTR_H
+#endif // _LIBCPP___CXX03___BIT_COUNTR_H
diff --git a/libcxx/include/__cxx03/__bit/endian.h b/libcxx/include/__cxx03/__bit/endian.h
index ba0a5ac4c2d08..641f827736b4f 100644
--- a/libcxx/include/__cxx03/__bit/endian.h
+++ b/libcxx/include/__cxx03/__bit/endian.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_ENDIAN_H
-#define _LIBCPP___BIT_ENDIAN_H
+#ifndef _LIBCPP___CXX03___BIT_ENDIAN_H
+#define _LIBCPP___CXX03___BIT_ENDIAN_H
 
 #include <__cxx03/__config>
 
@@ -35,4 +35,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___BIT_ENDIAN_H
+#endif // _LIBCPP___CXX03___BIT_ENDIAN_H
diff --git a/libcxx/include/__cxx03/__bit/has_single_bit.h b/libcxx/include/__cxx03/__bit/has_single_bit.h
index 8f69d343a6087..07f3f8523d585 100644
--- a/libcxx/include/__cxx03/__bit/has_single_bit.h
+++ b/libcxx/include/__cxx03/__bit/has_single_bit.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_HAS_SINGLE_BIT_H
-#define _LIBCPP___BIT_HAS_SINGLE_BIT_H
+#ifndef _LIBCPP___CXX03___BIT_HAS_SINGLE_BIT_H
+#define _LIBCPP___CXX03___BIT_HAS_SINGLE_BIT_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -34,4 +34,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___BIT_HAS_SINGLE_BIT_H
+#endif // _LIBCPP___CXX03___BIT_HAS_SINGLE_BIT_H
diff --git a/libcxx/include/__cxx03/__bit/invert_if.h b/libcxx/include/__cxx03/__bit/invert_if.h
index 0b729bc79c192..b111d702ea755 100644
--- a/libcxx/include/__cxx03/__bit/invert_if.h
+++ b/libcxx/include/__cxx03/__bit/invert_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_INVERT_IF_H
-#define _LIBCPP___BIT_INVERT_IF_H
+#ifndef _LIBCPP___CXX03___BIT_INVERT_IF_H
+#define _LIBCPP___CXX03___BIT_INVERT_IF_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -27,4 +27,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __invert_if(_Tp __v) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_INVERT_IF_H
+#endif // _LIBCPP___CXX03___BIT_INVERT_IF_H
diff --git a/libcxx/include/__cxx03/__bit/popcount.h b/libcxx/include/__cxx03/__bit/popcount.h
index 70a12ea260741..835c07703f233 100644
--- a/libcxx/include/__cxx03/__bit/popcount.h
+++ b/libcxx/include/__cxx03/__bit/popcount.h
@@ -9,8 +9,8 @@
 // TODO: __builtin_popcountg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
 //  refactor this code to exclusively use __builtin_popcountg.
 
-#ifndef _LIBCPP___BIT_POPCOUNT_H
-#define _LIBCPP___BIT_POPCOUNT_H
+#ifndef _LIBCPP___CXX03___BIT_POPCOUNT_H
+#define _LIBCPP___CXX03___BIT_POPCOUNT_H
 
 #include <__cxx03/__bit/rotate.h>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -68,4 +68,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___BIT_POPCOUNT_H
+#endif // _LIBCPP___CXX03___BIT_POPCOUNT_H
diff --git a/libcxx/include/__cxx03/__bit/rotate.h b/libcxx/include/__cxx03/__bit/rotate.h
index 02b4ac66af109..0ce3d2c7629db 100644
--- a/libcxx/include/__cxx03/__bit/rotate.h
+++ b/libcxx/include/__cxx03/__bit/rotate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_ROTATE_H
-#define _LIBCPP___BIT_ROTATE_H
+#ifndef _LIBCPP___CXX03___BIT_ROTATE_H
+#define _LIBCPP___CXX03___BIT_ROTATE_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -69,4 +69,4 @@ template <__libcpp_unsigned_integer _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___BIT_ROTATE_H
+#endif // _LIBCPP___CXX03___BIT_ROTATE_H
diff --git a/libcxx/include/__cxx03/__bit_reference b/libcxx/include/__cxx03/__bit_reference
index 7339f5bd8c9e1..bf86f9a76e24a 100644
--- a/libcxx/include/__cxx03/__bit_reference
+++ b/libcxx/include/__cxx03/__bit_reference
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___BIT_REFERENCE
-#define _LIBCPP___BIT_REFERENCE
+#ifndef _LIBCPP___CXX03___BIT_REFERENCE
+#define _LIBCPP___CXX03___BIT_REFERENCE
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__algorithm/fill_n.h>
@@ -1021,4 +1021,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___BIT_REFERENCE
+#endif // _LIBCPP___CXX03___BIT_REFERENCE
diff --git a/libcxx/include/__cxx03/__charconv/chars_format.h b/libcxx/include/__cxx03/__charconv/chars_format.h
index c206289c0c258..a886521832eff 100644
--- a/libcxx/include/__cxx03/__charconv/chars_format.h
+++ b/libcxx/include/__cxx03/__charconv/chars_format.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_CHARS_FORMAT_H
-#define _LIBCPP___CHARCONV_CHARS_FORMAT_H
+#ifndef _LIBCPP___CXX03___CHARCONV_CHARS_FORMAT_H
+#define _LIBCPP___CXX03___CHARCONV_CHARS_FORMAT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/to_underlying.h>
@@ -58,4 +58,4 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr chars_format& operator^=(chars_format& __
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_CHARS_FORMAT_H
+#endif // _LIBCPP___CXX03___CHARCONV_CHARS_FORMAT_H
diff --git a/libcxx/include/__cxx03/__charconv/from_chars_integral.h b/libcxx/include/__cxx03/__charconv/from_chars_integral.h
index a3d6e5537280d..ba3746fcf776e 100644
--- a/libcxx/include/__cxx03/__charconv/from_chars_integral.h
+++ b/libcxx/include/__cxx03/__charconv/from_chars_integral.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
-#define _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___CHARCONV_FROM_CHARS_INTEGRAL_H
+#define _LIBCPP___CXX03___CHARCONV_FROM_CHARS_INTEGRAL_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__assert>
@@ -237,4 +237,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHARCONV_FROM_CHARS_INTEGRAL_H
+#endif // _LIBCPP___CXX03___CHARCONV_FROM_CHARS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__charconv/from_chars_result.h b/libcxx/include/__cxx03/__charconv/from_chars_result.h
index 8bba738969e34..03e490fce56a9 100644
--- a/libcxx/include/__cxx03/__charconv/from_chars_result.h
+++ b/libcxx/include/__cxx03/__charconv/from_chars_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
-#define _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
+#ifndef _LIBCPP___CXX03___CHARCONV_FROM_CHARS_RESULT_H
+#define _LIBCPP___CXX03___CHARCONV_FROM_CHARS_RESULT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__system_error/errc.h>
@@ -36,4 +36,4 @@ struct _LIBCPP_EXPORTED_FROM_ABI from_chars_result {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_FROM_CHARS_RESULT_H
+#endif // _LIBCPP___CXX03___CHARCONV_FROM_CHARS_RESULT_H
diff --git a/libcxx/include/__cxx03/__charconv/tables.h b/libcxx/include/__cxx03/__charconv/tables.h
index 110352a4be0c7..8c9fd3daf170c 100644
--- a/libcxx/include/__cxx03/__charconv/tables.h
+++ b/libcxx/include/__cxx03/__charconv/tables.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TABLES
-#define _LIBCPP___CHARCONV_TABLES
+#ifndef _LIBCPP___CXX03___CHARCONV_TABLES
+#define _LIBCPP___CXX03___CHARCONV_TABLES
 
 #include <__cxx03/__config>
 #include <__cxx03/cstdint>
@@ -160,4 +160,4 @@ inline constexpr char __digits_base_10[200] = {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_TABLES
+#endif // _LIBCPP___CXX03___CHARCONV_TABLES
diff --git a/libcxx/include/__cxx03/__charconv/to_chars.h b/libcxx/include/__cxx03/__charconv/to_chars.h
index 2db35ace09439..e526ce427c22f 100644
--- a/libcxx/include/__cxx03/__charconv/to_chars.h
+++ b/libcxx/include/__cxx03/__charconv/to_chars.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TO_CHARS
-#define _LIBCPP___CHARCONV_TO_CHARS
+#ifndef _LIBCPP___CXX03___CHARCONV_TO_CHARS
+#define _LIBCPP___CXX03___CHARCONV_TO_CHARS
 
 #include <__cxx03/__charconv/to_chars_floating_point.h>
 #include <__cxx03/__charconv/to_chars_integral.h>
@@ -22,4 +22,4 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_TO_CHARS
+#endif // _LIBCPP___CXX03___CHARCONV_TO_CHARS
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_base_10.h b/libcxx/include/__cxx03/__charconv/to_chars_base_10.h
index 33baaf63caeef..b13b3e33d8f97 100644
--- a/libcxx/include/__cxx03/__charconv/to_chars_base_10.h
+++ b/libcxx/include/__cxx03/__charconv/to_chars_base_10.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
-#define _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
+#ifndef _LIBCPP___CXX03___CHARCONV_TO_CHARS_BASE_10_H
+#define _LIBCPP___CXX03___CHARCONV_TO_CHARS_BASE_10_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__assert>
@@ -185,4 +185,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHARCONV_TO_CHARS_BASE_10_H
+#endif // _LIBCPP___CXX03___CHARCONV_TO_CHARS_BASE_10_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h b/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
index 305b7e71a6b7e..167ffeae7bedd 100644
--- a/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
+++ b/libcxx/include/__cxx03/__charconv/to_chars_floating_point.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
-#define _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
+#ifndef _LIBCPP___CXX03___CHARCONV_TO_CHARS_FLOATING_POINT_H
+#define _LIBCPP___CXX03___CHARCONV_TO_CHARS_FLOATING_POINT_H
 
 #include <__cxx03/__charconv/chars_format.h>
 #include <__cxx03/__charconv/to_chars_result.h>
@@ -52,4 +52,4 @@ to_chars(char* __first, char* __last, long double __value, chars_format __fmt, i
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_TO_CHARS_FLOATING_POINT_H
+#endif // _LIBCPP___CXX03___CHARCONV_TO_CHARS_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_integral.h b/libcxx/include/__cxx03/__charconv/to_chars_integral.h
index 0b3f319e75125..2579e02229851 100644
--- a/libcxx/include/__cxx03/__charconv/to_chars_integral.h
+++ b/libcxx/include/__cxx03/__charconv/to_chars_integral.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
-#define _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___CHARCONV_TO_CHARS_INTEGRAL_H
+#define _LIBCPP___CXX03___CHARCONV_TO_CHARS_INTEGRAL_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__assert>
@@ -324,4 +324,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHARCONV_TO_CHARS_INTEGRAL_H
+#endif // _LIBCPP___CXX03___CHARCONV_TO_CHARS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__charconv/to_chars_result.h b/libcxx/include/__cxx03/__charconv/to_chars_result.h
index 72fef89dcc5f5..82a60db6f6741 100644
--- a/libcxx/include/__cxx03/__charconv/to_chars_result.h
+++ b/libcxx/include/__cxx03/__charconv/to_chars_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
-#define _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
+#ifndef _LIBCPP___CXX03___CHARCONV_TO_CHARS_RESULT_H
+#define _LIBCPP___CXX03___CHARCONV_TO_CHARS_RESULT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__system_error/errc.h>
@@ -36,4 +36,4 @@ struct _LIBCPP_EXPORTED_FROM_ABI to_chars_result {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHARCONV_TO_CHARS_RESULT_H
+#endif // _LIBCPP___CXX03___CHARCONV_TO_CHARS_RESULT_H
diff --git a/libcxx/include/__cxx03/__charconv/traits.h b/libcxx/include/__cxx03/__charconv/traits.h
index 5eafa6caa39e5..110ff9c07e7d0 100644
--- a/libcxx/include/__cxx03/__charconv/traits.h
+++ b/libcxx/include/__cxx03/__charconv/traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHARCONV_TRAITS
-#define _LIBCPP___CHARCONV_TRAITS
+#ifndef _LIBCPP___CXX03___CHARCONV_TRAITS
+#define _LIBCPP___CXX03___CHARCONV_TRAITS
 
 #include <__cxx03/__assert>
 #include <__cxx03/__bit/countl.h>
@@ -197,4 +197,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHARCONV_TRAITS
+#endif // _LIBCPP___CXX03___CHARCONV_TRAITS
diff --git a/libcxx/include/__cxx03/__chrono/calendar.h b/libcxx/include/__cxx03/__chrono/calendar.h
index 5d8e396851ae8..4113ca435cc1e 100644
--- a/libcxx/include/__cxx03/__chrono/calendar.h
+++ b/libcxx/include/__cxx03/__chrono/calendar.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_CALENDAR_H
-#define _LIBCPP___CHRONO_CALENDAR_H
+#ifndef _LIBCPP___CXX03___CHRONO_CALENDAR_H
+#define _LIBCPP___CXX03___CHRONO_CALENDAR_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -41,4 +41,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_CALENDAR_H
+#endif // _LIBCPP___CXX03___CHRONO_CALENDAR_H
diff --git a/libcxx/include/__cxx03/__chrono/concepts.h b/libcxx/include/__cxx03/__chrono/concepts.h
index 1fb5241055474..55a2c54fc3f5b 100644
--- a/libcxx/include/__cxx03/__chrono/concepts.h
+++ b/libcxx/include/__cxx03/__chrono/concepts.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_CONCEPTS_H
-#define _LIBCPP___CHRONO_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___CHRONO_CONCEPTS_H
+#define _LIBCPP___CXX03___CHRONO_CONCEPTS_H
 
 #include <__cxx03/__chrono/hh_mm_ss.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -33,4 +33,4 @@ concept __is_time_point = __is_specialization_v<_Tp, chrono::time_point>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_CONCEPTS_H
+#endif // _LIBCPP___CXX03___CHRONO_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__chrono/convert_to_timespec.h b/libcxx/include/__cxx03/__chrono/convert_to_timespec.h
index df0e7714ddd70..24d77c21807eb 100644
--- a/libcxx/include/__cxx03/__chrono/convert_to_timespec.h
+++ b/libcxx/include/__cxx03/__chrono/convert_to_timespec.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
-#define _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
+#ifndef _LIBCPP___CXX03___CHRONO_CONVERT_TO_TIMESPEC_H
+#define _LIBCPP___CXX03___CHRONO_CONVERT_TO_TIMESPEC_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__config>
@@ -48,4 +48,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHRONO_CONVERT_TO_TIMESPEC_H
+#endif // _LIBCPP___CXX03___CHRONO_CONVERT_TO_TIMESPEC_H
diff --git a/libcxx/include/__cxx03/__chrono/convert_to_tm.h b/libcxx/include/__cxx03/__chrono/convert_to_tm.h
index ccc7ee7b11973..e02ede48317b7 100644
--- a/libcxx/include/__cxx03/__chrono/convert_to_tm.h
+++ b/libcxx/include/__cxx03/__chrono/convert_to_tm.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_CONVERT_TO_TM_H
-#define _LIBCPP___CHRONO_CONVERT_TO_TM_H
+#ifndef _LIBCPP___CXX03___CHRONO_CONVERT_TO_TM_H
+#define _LIBCPP___CXX03___CHRONO_CONVERT_TO_TM_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/concepts.h>
@@ -199,4 +199,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHRONO_CONVERT_TO_TM_H
+#endif // _LIBCPP___CXX03___CHRONO_CONVERT_TO_TM_H
diff --git a/libcxx/include/__cxx03/__chrono/day.h b/libcxx/include/__cxx03/__chrono/day.h
index 6df2aa4283461..6b989ea3c6c24 100644
--- a/libcxx/include/__cxx03/__chrono/day.h
+++ b/libcxx/include/__cxx03/__chrono/day.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_DAY_H
-#define _LIBCPP___CHRONO_DAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_DAY_H
+#define _LIBCPP___CXX03___CHRONO_DAY_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__config>
@@ -96,4 +96,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_DAY_H
+#endif // _LIBCPP___CXX03___CHRONO_DAY_H
diff --git a/libcxx/include/__cxx03/__chrono/duration.h b/libcxx/include/__cxx03/__chrono/duration.h
index 5fabe08db1bfc..fd936eda44eef 100644
--- a/libcxx/include/__cxx03/__chrono/duration.h
+++ b/libcxx/include/__cxx03/__chrono/duration.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_DURATION_H
-#define _LIBCPP___CHRONO_DURATION_H
+#ifndef _LIBCPP___CXX03___CHRONO_DURATION_H
+#define _LIBCPP___CXX03___CHRONO_DURATION_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -547,4 +547,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP___CHRONO_DURATION_H
+#endif // _LIBCPP___CXX03___CHRONO_DURATION_H
diff --git a/libcxx/include/__cxx03/__chrono/exception.h b/libcxx/include/__cxx03/__chrono/exception.h
index 3e1a29c203772..4db9d8c1c4edf 100644
--- a/libcxx/include/__cxx03/__chrono/exception.h
+++ b/libcxx/include/__cxx03/__chrono/exception.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_EXCEPTION_H
-#define _LIBCPP___CHRONO_EXCEPTION_H
+#ifndef _LIBCPP___CXX03___CHRONO_EXCEPTION_H
+#define _LIBCPP___CXX03___CHRONO_EXCEPTION_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -132,4 +132,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_EXCEPTION_H
+#endif // _LIBCPP___CXX03___CHRONO_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__chrono/file_clock.h b/libcxx/include/__cxx03/__chrono/file_clock.h
index aef92057bd04a..6e2d32b4ef88a 100644
--- a/libcxx/include/__cxx03/__chrono/file_clock.h
+++ b/libcxx/include/__cxx03/__chrono/file_clock.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_FILE_CLOCK_H
-#define _LIBCPP___CHRONO_FILE_CLOCK_H
+#ifndef _LIBCPP___CXX03___CHRONO_FILE_CLOCK_H
+#define _LIBCPP___CXX03___CHRONO_FILE_CLOCK_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/system_clock.h>
@@ -77,4 +77,4 @@ struct _FilesystemClock {
 _LIBCPP_END_NAMESPACE_FILESYSTEM
 #endif // !_LIBCPP_CXX03_LANG
 
-#endif // _LIBCPP___CHRONO_FILE_CLOCK_H
+#endif // _LIBCPP___CXX03___CHRONO_FILE_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/formatter.h b/libcxx/include/__cxx03/__chrono/formatter.h
index 54e5dff6b97dc..823f6e34c5361 100644
--- a/libcxx/include/__cxx03/__chrono/formatter.h
+++ b/libcxx/include/__cxx03/__chrono/formatter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_FORMATTER_H
-#define _LIBCPP___CHRONO_FORMATTER_H
+#ifndef _LIBCPP___CXX03___CHRONO_FORMATTER_H
+#define _LIBCPP___CXX03___CHRONO_FORMATTER_H
 
 #include <__cxx03/__algorithm/ranges_copy.h>
 #include <__cxx03/__chrono/calendar.h>
@@ -987,4 +987,4 @@ struct formatter<chrono::zoned_time<_Duration, _TimeZonePtr>, _CharT> : public _
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif //  _LIBCPP___CHRONO_FORMATTER_H
+#endif //  _LIBCPP___CXX03___CHRONO_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__chrono/hh_mm_ss.h b/libcxx/include/__cxx03/__chrono/hh_mm_ss.h
index 100687064ed88..e40cfa3a58b8d 100644
--- a/libcxx/include/__cxx03/__chrono/hh_mm_ss.h
+++ b/libcxx/include/__cxx03/__chrono/hh_mm_ss.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_HH_MM_SS_H
-#define _LIBCPP___CHRONO_HH_MM_SS_H
+#ifndef _LIBCPP___CXX03___CHRONO_HH_MM_SS_H
+#define _LIBCPP___CXX03___CHRONO_HH_MM_SS_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -109,4 +109,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_HH_MM_SS_H
+#endif // _LIBCPP___CXX03___CHRONO_HH_MM_SS_H
diff --git a/libcxx/include/__cxx03/__chrono/high_resolution_clock.h b/libcxx/include/__cxx03/__chrono/high_resolution_clock.h
index 9c5104338421b..3371d16112f97 100644
--- a/libcxx/include/__cxx03/__chrono/high_resolution_clock.h
+++ b/libcxx/include/__cxx03/__chrono/high_resolution_clock.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
-#define _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
+#ifndef _LIBCPP___CXX03___CHRONO_HIGH_RESOLUTION_CLOCK_H
+#define _LIBCPP___CXX03___CHRONO_HIGH_RESOLUTION_CLOCK_H
 
 #include <__cxx03/__chrono/steady_clock.h>
 #include <__cxx03/__chrono/system_clock.h>
@@ -32,4 +32,4 @@ typedef system_clock high_resolution_clock;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_HIGH_RESOLUTION_CLOCK_H
+#endif // _LIBCPP___CXX03___CHRONO_HIGH_RESOLUTION_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/leap_second.h b/libcxx/include/__cxx03/__chrono/leap_second.h
index 900eff22aa856..d6678bc58d7b1 100644
--- a/libcxx/include/__cxx03/__chrono/leap_second.h
+++ b/libcxx/include/__cxx03/__chrono/leap_second.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_LEAP_SECOND_H
-#define _LIBCPP___CHRONO_LEAP_SECOND_H
+#ifndef _LIBCPP___CXX03___CHRONO_LEAP_SECOND_H
+#define _LIBCPP___CXX03___CHRONO_LEAP_SECOND_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -123,4 +123,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_LEAP_SECOND_H
+#endif // _LIBCPP___CXX03___CHRONO_LEAP_SECOND_H
diff --git a/libcxx/include/__cxx03/__chrono/literals.h b/libcxx/include/__cxx03/__chrono/literals.h
index d299fb97da3c9..6d0f596aa4ce5 100644
--- a/libcxx/include/__cxx03/__chrono/literals.h
+++ b/libcxx/include/__cxx03/__chrono/literals.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_LITERALS_H
-#define _LIBCPP___CHRONO_LITERALS_H
+#ifndef _LIBCPP___CXX03___CHRONO_LITERALS_H
+#define _LIBCPP___CXX03___CHRONO_LITERALS_H
 
 #include <__cxx03/__chrono/day.h>
 #include <__cxx03/__chrono/year.h>
@@ -42,4 +42,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_LITERALS_H
+#endif // _LIBCPP___CXX03___CHRONO_LITERALS_H
diff --git a/libcxx/include/__cxx03/__chrono/local_info.h b/libcxx/include/__cxx03/__chrono/local_info.h
index 8e2194c05aba1..1c475ec93c167 100644
--- a/libcxx/include/__cxx03/__chrono/local_info.h
+++ b/libcxx/include/__cxx03/__chrono/local_info.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_LOCAL_INFO_H
-#define _LIBCPP___CHRONO_LOCAL_INFO_H
+#ifndef _LIBCPP___CXX03___CHRONO_LOCAL_INFO_H
+#define _LIBCPP___CXX03___CHRONO_LOCAL_INFO_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -47,4 +47,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_LOCAL_INFO_H
+#endif // _LIBCPP___CXX03___CHRONO_LOCAL_INFO_H
diff --git a/libcxx/include/__cxx03/__chrono/month.h b/libcxx/include/__cxx03/__chrono/month.h
index fd66bfea2f60d..9ccc85991af71 100644
--- a/libcxx/include/__cxx03/__chrono/month.h
+++ b/libcxx/include/__cxx03/__chrono/month.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_MONTH_H
-#define _LIBCPP___CHRONO_MONTH_H
+#ifndef _LIBCPP___CXX03___CHRONO_MONTH_H
+#define _LIBCPP___CXX03___CHRONO_MONTH_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__config>
@@ -112,4 +112,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_MONTH_H
+#endif // _LIBCPP___CXX03___CHRONO_MONTH_H
diff --git a/libcxx/include/__cxx03/__chrono/month_weekday.h b/libcxx/include/__cxx03/__chrono/month_weekday.h
index 6595618d822c0..eedc6e9a784bb 100644
--- a/libcxx/include/__cxx03/__chrono/month_weekday.h
+++ b/libcxx/include/__cxx03/__chrono/month_weekday.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_MONTH_WEEKDAY_H
-#define _LIBCPP___CHRONO_MONTH_WEEKDAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_MONTH_WEEKDAY_H
+#define _LIBCPP___CXX03___CHRONO_MONTH_WEEKDAY_H
 
 #include <__cxx03/__chrono/month.h>
 #include <__cxx03/__chrono/weekday.h>
@@ -102,4 +102,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_MONTH_WEEKDAY_H
+#endif // _LIBCPP___CXX03___CHRONO_MONTH_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/monthday.h b/libcxx/include/__cxx03/__chrono/monthday.h
index fe85e6b8bab32..a64584ee2e5f9 100644
--- a/libcxx/include/__cxx03/__chrono/monthday.h
+++ b/libcxx/include/__cxx03/__chrono/monthday.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_MONTHDAY_H
-#define _LIBCPP___CHRONO_MONTHDAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_MONTHDAY_H
+#define _LIBCPP___CXX03___CHRONO_MONTHDAY_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/day.h>
@@ -130,4 +130,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_MONTHDAY_H
+#endif // _LIBCPP___CXX03___CHRONO_MONTHDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/ostream.h b/libcxx/include/__cxx03/__chrono/ostream.h
index 6ac6b2831e117..fed43890c39b7 100644
--- a/libcxx/include/__cxx03/__chrono/ostream.h
+++ b/libcxx/include/__cxx03/__chrono/ostream.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_OSTREAM_H
-#define _LIBCPP___CHRONO_OSTREAM_H
+#ifndef _LIBCPP___CXX03___CHRONO_OSTREAM_H
+#define _LIBCPP___CXX03___CHRONO_OSTREAM_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/day.h>
@@ -319,4 +319,4 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const zoned_time<_Duration, _Ti
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_OSTREAM_H
+#endif // _LIBCPP___CXX03___CHRONO_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h b/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
index 3a09a21ac54b3..2bec77d8c74dd 100644
--- a/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
+++ b/libcxx/include/__cxx03/__chrono/parser_std_format_spec.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
-#define _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
+#ifndef _LIBCPP___CXX03___CHRONO_PARSER_STD_FORMAT_SPEC_H
+#define _LIBCPP___CXX03___CHRONO_PARSER_STD_FORMAT_SPEC_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__format/concepts.h>
@@ -413,4 +413,4 @@ class _LIBCPP_TEMPLATE_VIS __parser_chrono {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_PARSER_STD_FORMAT_SPEC_H
+#endif // _LIBCPP___CXX03___CHRONO_PARSER_STD_FORMAT_SPEC_H
diff --git a/libcxx/include/__cxx03/__chrono/statically_widen.h b/libcxx/include/__cxx03/__chrono/statically_widen.h
index 101a903bc3f28..183078152f61c 100644
--- a/libcxx/include/__cxx03/__chrono/statically_widen.h
+++ b/libcxx/include/__cxx03/__chrono/statically_widen.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_STATICALLY_WIDEN_H
-#define _LIBCPP___CHRONO_STATICALLY_WIDEN_H
+#ifndef _LIBCPP___CXX03___CHRONO_STATICALLY_WIDEN_H
+#define _LIBCPP___CXX03___CHRONO_STATICALLY_WIDEN_H
 
 // Implements the STATICALLY-WIDEN exposition-only function. ([time.general]/2)
 
@@ -49,4 +49,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr const _CharT* __statically_widen(const char* __s
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_STATICALLY_WIDEN_H
+#endif // _LIBCPP___CXX03___CHRONO_STATICALLY_WIDEN_H
diff --git a/libcxx/include/__cxx03/__chrono/steady_clock.h b/libcxx/include/__cxx03/__chrono/steady_clock.h
index ad5b64b8142bc..f8a56954f31ad 100644
--- a/libcxx/include/__cxx03/__chrono/steady_clock.h
+++ b/libcxx/include/__cxx03/__chrono/steady_clock.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_STEADY_CLOCK_H
-#define _LIBCPP___CHRONO_STEADY_CLOCK_H
+#ifndef _LIBCPP___CXX03___CHRONO_STEADY_CLOCK_H
+#define _LIBCPP___CXX03___CHRONO_STEADY_CLOCK_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -39,4 +39,4 @@ class _LIBCPP_EXPORTED_FROM_ABI steady_clock {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_STEADY_CLOCK_H
+#endif // _LIBCPP___CXX03___CHRONO_STEADY_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/sys_info.h b/libcxx/include/__cxx03/__chrono/sys_info.h
index ed1c7cabd9b60..45c230de91cc1 100644
--- a/libcxx/include/__cxx03/__chrono/sys_info.h
+++ b/libcxx/include/__cxx03/__chrono/sys_info.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_SYS_INFO_H
-#define _LIBCPP___CHRONO_SYS_INFO_H
+#ifndef _LIBCPP___CXX03___CHRONO_SYS_INFO_H
+#define _LIBCPP___CXX03___CHRONO_SYS_INFO_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -48,4 +48,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_SYS_INFO_H
+#endif // _LIBCPP___CXX03___CHRONO_SYS_INFO_H
diff --git a/libcxx/include/__cxx03/__chrono/system_clock.h b/libcxx/include/__cxx03/__chrono/system_clock.h
index 61382ebdfbe6d..7ae053f3bb409 100644
--- a/libcxx/include/__cxx03/__chrono/system_clock.h
+++ b/libcxx/include/__cxx03/__chrono/system_clock.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_SYSTEM_CLOCK_H
-#define _LIBCPP___CHRONO_SYSTEM_CLOCK_H
+#ifndef _LIBCPP___CXX03___CHRONO_SYSTEM_CLOCK_H
+#define _LIBCPP___CXX03___CHRONO_SYSTEM_CLOCK_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -49,4 +49,4 @@ using sys_days    = sys_time<days>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CHRONO_SYSTEM_CLOCK_H
+#endif // _LIBCPP___CXX03___CHRONO_SYSTEM_CLOCK_H
diff --git a/libcxx/include/__cxx03/__chrono/time_point.h b/libcxx/include/__cxx03/__chrono/time_point.h
index a26423efb15b4..8703275a75ea7 100644
--- a/libcxx/include/__cxx03/__chrono/time_point.h
+++ b/libcxx/include/__cxx03/__chrono/time_point.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_TIME_POINT_H
-#define _LIBCPP___CHRONO_TIME_POINT_H
+#ifndef _LIBCPP___CXX03___CHRONO_TIME_POINT_H
+#define _LIBCPP___CXX03___CHRONO_TIME_POINT_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__compare/ordering.h>
@@ -217,4 +217,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHRONO_TIME_POINT_H
+#endif // _LIBCPP___CXX03___CHRONO_TIME_POINT_H
diff --git a/libcxx/include/__cxx03/__chrono/time_zone.h b/libcxx/include/__cxx03/__chrono/time_zone.h
index f14646787cf03..a49e9023c0a6a 100644
--- a/libcxx/include/__cxx03/__chrono/time_zone.h
+++ b/libcxx/include/__cxx03/__chrono/time_zone.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_TIME_ZONE_H
-#define _LIBCPP___CHRONO_TIME_ZONE_H
+#ifndef _LIBCPP___CXX03___CHRONO_TIME_ZONE_H
+#define _LIBCPP___CXX03___CHRONO_TIME_ZONE_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -179,4 +179,4 @@ _LIBCPP_POP_MACROS
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_TIME_ZONE_H
+#endif // _LIBCPP___CXX03___CHRONO_TIME_ZONE_H
diff --git a/libcxx/include/__cxx03/__chrono/time_zone_link.h b/libcxx/include/__cxx03/__chrono/time_zone_link.h
index a19226b06d866..f97851bcd9811 100644
--- a/libcxx/include/__cxx03/__chrono/time_zone_link.h
+++ b/libcxx/include/__cxx03/__chrono/time_zone_link.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_TIME_ZONE_LINK_H
-#define _LIBCPP___CHRONO_TIME_ZONE_LINK_H
+#ifndef _LIBCPP___CXX03___CHRONO_TIME_ZONE_LINK_H
+#define _LIBCPP___CXX03___CHRONO_TIME_ZONE_LINK_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -76,4 +76,4 @@ _LIBCPP_POP_MACROS
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_TIME_ZONE_LINK_H
+#endif // _LIBCPP___CXX03___CHRONO_TIME_ZONE_LINK_H
diff --git a/libcxx/include/__cxx03/__chrono/tzdb.h b/libcxx/include/__cxx03/__chrono/tzdb.h
index cb218314a8e85..69cd19042c35a 100644
--- a/libcxx/include/__cxx03/__chrono/tzdb.h
+++ b/libcxx/include/__cxx03/__chrono/tzdb.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_TZDB_H
-#define _LIBCPP___CHRONO_TZDB_H
+#ifndef _LIBCPP___CXX03___CHRONO_TZDB_H
+#define _LIBCPP___CXX03___CHRONO_TZDB_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -91,4 +91,4 @@ _LIBCPP_POP_MACROS
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_TZDB_H
+#endif // _LIBCPP___CXX03___CHRONO_TZDB_H
diff --git a/libcxx/include/__cxx03/__chrono/tzdb_list.h b/libcxx/include/__cxx03/__chrono/tzdb_list.h
index 96d52da88b108..e3fb2a01161a5 100644
--- a/libcxx/include/__cxx03/__chrono/tzdb_list.h
+++ b/libcxx/include/__cxx03/__chrono/tzdb_list.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_TZDB_LIST_H
-#define _LIBCPP___CHRONO_TZDB_LIST_H
+#ifndef _LIBCPP___CXX03___CHRONO_TZDB_LIST_H
+#define _LIBCPP___CXX03___CHRONO_TZDB_LIST_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -105,4 +105,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_TZDB_LIST_H
+#endif // _LIBCPP___CXX03___CHRONO_TZDB_LIST_H
diff --git a/libcxx/include/__cxx03/__chrono/weekday.h b/libcxx/include/__cxx03/__chrono/weekday.h
index d7a0cc8496dcd..3c152653118d4 100644
--- a/libcxx/include/__cxx03/__chrono/weekday.h
+++ b/libcxx/include/__cxx03/__chrono/weekday.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_WEEKDAY_H
-#define _LIBCPP___CHRONO_WEEKDAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_WEEKDAY_H
+#define _LIBCPP___CXX03___CHRONO_WEEKDAY_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/duration.h>
@@ -183,4 +183,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_WEEKDAY_H
+#endif // _LIBCPP___CXX03___CHRONO_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/year.h b/libcxx/include/__cxx03/__chrono/year.h
index 2850616a7aa21..3594977e2be99 100644
--- a/libcxx/include/__cxx03/__chrono/year.h
+++ b/libcxx/include/__cxx03/__chrono/year.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_YEAR_H
-#define _LIBCPP___CHRONO_YEAR_H
+#ifndef _LIBCPP___CXX03___CHRONO_YEAR_H
+#define _LIBCPP___CXX03___CHRONO_YEAR_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__config>
@@ -115,4 +115,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CHRONO_YEAR_H
+#endif // _LIBCPP___CXX03___CHRONO_YEAR_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month.h b/libcxx/include/__cxx03/__chrono/year_month.h
index 75784df9386d8..07652cfc868c1 100644
--- a/libcxx/include/__cxx03/__chrono/year_month.h
+++ b/libcxx/include/__cxx03/__chrono/year_month.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_YEAR_MONTH_H
-#define _LIBCPP___CHRONO_YEAR_MONTH_H
+#ifndef _LIBCPP___CXX03___CHRONO_YEAR_MONTH_H
+#define _LIBCPP___CXX03___CHRONO_YEAR_MONTH_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/month.h>
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_YEAR_MONTH_H
+#endif // _LIBCPP___CXX03___CHRONO_YEAR_MONTH_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month_day.h b/libcxx/include/__cxx03/__chrono/year_month_day.h
index dff7423a0ffb3..551da3e3a1ffd 100644
--- a/libcxx/include/__cxx03/__chrono/year_month_day.h
+++ b/libcxx/include/__cxx03/__chrono/year_month_day.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
-#define _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_YEAR_MONTH_DAY_H
+#define _LIBCPP___CXX03___CHRONO_YEAR_MONTH_DAY_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/day.h>
@@ -334,4 +334,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_YEAR_MONTH_DAY_H
+#endif // _LIBCPP___CXX03___CHRONO_YEAR_MONTH_DAY_H
diff --git a/libcxx/include/__cxx03/__chrono/year_month_weekday.h b/libcxx/include/__cxx03/__chrono/year_month_weekday.h
index 3177f21964862..3f6974abeb3cc 100644
--- a/libcxx/include/__cxx03/__chrono/year_month_weekday.h
+++ b/libcxx/include/__cxx03/__chrono/year_month_weekday.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
-#define _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
+#ifndef _LIBCPP___CXX03___CHRONO_YEAR_MONTH_WEEKDAY_H
+#define _LIBCPP___CXX03___CHRONO_YEAR_MONTH_WEEKDAY_H
 
 #include <__cxx03/__chrono/calendar.h>
 #include <__cxx03/__chrono/day.h>
@@ -284,4 +284,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___CHRONO_YEAR_MONTH_WEEKDAY_H
+#endif // _LIBCPP___CXX03___CHRONO_YEAR_MONTH_WEEKDAY_H
diff --git a/libcxx/include/__cxx03/__chrono/zoned_time.h b/libcxx/include/__cxx03/__chrono/zoned_time.h
index f3333a4189cc5..7f83dcfc31a84 100644
--- a/libcxx/include/__cxx03/__chrono/zoned_time.h
+++ b/libcxx/include/__cxx03/__chrono/zoned_time.h
@@ -9,8 +9,8 @@
 
 // For information see https://libcxx.llvm.org/DesignDocs/TimeZone.html
 
-#ifndef _LIBCPP___CHRONO_ZONED_TIME_H
-#define _LIBCPP___CHRONO_ZONED_TIME_H
+#ifndef _LIBCPP___CXX03___CHRONO_ZONED_TIME_H
+#define _LIBCPP___CXX03___CHRONO_ZONED_TIME_H
 
 #include <__cxx03/version>
 // Enable the contents of the header only when libc++ was built with experimental features enabled.
@@ -224,4 +224,4 @@ _LIBCPP_POP_MACROS
 
 #endif // !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB)
 
-#endif // _LIBCPP___CHRONO_ZONED_TIME_H
+#endif // _LIBCPP___CXX03___CHRONO_ZONED_TIME_H
diff --git a/libcxx/include/__cxx03/__compare/common_comparison_category.h b/libcxx/include/__cxx03/__compare/common_comparison_category.h
index 795884d13177f..22375e5fe2dcd 100644
--- a/libcxx/include/__cxx03/__compare/common_comparison_category.h
+++ b/libcxx/include/__cxx03/__compare/common_comparison_category.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
-#define _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
+#ifndef _LIBCPP___CXX03___COMPARE_COMMON_COMPARISON_CATEGORY_H
+#define _LIBCPP___CXX03___COMPARE_COMMON_COMPARISON_CATEGORY_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -83,4 +83,4 @@ using common_comparison_category_t = typename common_comparison_category<_Ts...>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMMON_COMPARISON_CATEGORY_H
+#endif // _LIBCPP___CXX03___COMPARE_COMMON_COMPARISON_CATEGORY_H
diff --git a/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
index dc939d8f9a6b8..faddc2faaa853 100644
--- a/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
+++ b/libcxx/include/__cxx03/__compare/compare_partial_order_fallback.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
-#define _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
+#ifndef _LIBCPP___CXX03___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
+#define _LIBCPP___CXX03___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/partial_order.h>
@@ -73,4 +73,4 @@ inline constexpr auto compare_partial_order_fallback = __compare_partial_order_f
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
+#endif // _LIBCPP___CXX03___COMPARE_COMPARE_PARTIAL_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
index 5ea1ce7c64d12..6ba4e49712c07 100644
--- a/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
+++ b/libcxx/include/__cxx03/__compare/compare_strong_order_fallback.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
-#define _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
+#ifndef _LIBCPP___CXX03___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
+#define _LIBCPP___CXX03___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/strong_order.h>
@@ -70,4 +70,4 @@ inline constexpr auto compare_strong_order_fallback = __compare_strong_order_fal
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
+#endif // _LIBCPP___CXX03___COMPARE_COMPARE_STRONG_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/compare_three_way.h b/libcxx/include/__cxx03/__compare/compare_three_way.h
index 258ad43103429..78354b107f213 100644
--- a/libcxx/include/__cxx03/__compare/compare_three_way.h
+++ b/libcxx/include/__cxx03/__compare/compare_three_way.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
-#define _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
+#ifndef _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_H
+#define _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_H
 
 #include <__cxx03/__compare/three_way_comparable.h>
 #include <__cxx03/__config>
@@ -37,4 +37,4 @@ struct _LIBCPP_TEMPLATE_VIS compare_three_way {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMPARE_THREE_WAY_H
+#endif // _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__compare/compare_three_way_result.h b/libcxx/include/__cxx03/__compare/compare_three_way_result.h
index 7577b9ee15f90..3b5dccc042e54 100644
--- a/libcxx/include/__cxx03/__compare/compare_three_way_result.h
+++ b/libcxx/include/__cxx03/__compare/compare_three_way_result.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
-#define _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
+#ifndef _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_RESULT_H
+#define _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_RESULT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/make_const_lvalue_ref.h>
@@ -42,4 +42,4 @@ using compare_three_way_result_t = typename compare_three_way_result<_Tp, _Up>::
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMPARE_THREE_WAY_RESULT_H
+#endif // _LIBCPP___CXX03___COMPARE_COMPARE_THREE_WAY_RESULT_H
diff --git a/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h b/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
index e12dc8eb5c9c0..ecfee76dff272 100644
--- a/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
+++ b/libcxx/include/__cxx03/__compare/compare_weak_order_fallback.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
-#define _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
+#ifndef _LIBCPP___CXX03___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
+#define _LIBCPP___CXX03___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/weak_order.h>
@@ -70,4 +70,4 @@ inline constexpr auto compare_weak_order_fallback = __compare_weak_order_fallbac
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
+#endif // _LIBCPP___CXX03___COMPARE_COMPARE_WEAK_ORDER_FALLBACK
diff --git a/libcxx/include/__cxx03/__compare/is_eq.h b/libcxx/include/__cxx03/__compare/is_eq.h
index 09cc7a2c85d4e..52370834dc408 100644
--- a/libcxx/include/__cxx03/__compare/is_eq.h
+++ b/libcxx/include/__cxx03/__compare/is_eq.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_IS_EQ_H
-#define _LIBCPP___COMPARE_IS_EQ_H
+#ifndef _LIBCPP___CXX03___COMPARE_IS_EQ_H
+#define _LIBCPP___CXX03___COMPARE_IS_EQ_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -31,4 +31,4 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gteq(partial_ordering __c) noexce
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_IS_EQ_H
+#endif // _LIBCPP___CXX03___COMPARE_IS_EQ_H
diff --git a/libcxx/include/__cxx03/__compare/ordering.h b/libcxx/include/__cxx03/__compare/ordering.h
index 71c199991f330..0a497cba682df 100644
--- a/libcxx/include/__cxx03/__compare/ordering.h
+++ b/libcxx/include/__cxx03/__compare/ordering.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_ORDERING_H
-#define _LIBCPP___COMPARE_ORDERING_H
+#ifndef _LIBCPP___CXX03___COMPARE_ORDERING_H
+#define _LIBCPP___CXX03___COMPARE_ORDERING_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -275,4 +275,4 @@ concept __comparison_category = __one_of_v<_Tp, partial_ordering, weak_ordering,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_ORDERING_H
+#endif // _LIBCPP___CXX03___COMPARE_ORDERING_H
diff --git a/libcxx/include/__cxx03/__compare/partial_order.h b/libcxx/include/__cxx03/__compare/partial_order.h
index df8b842e7bf84..119d6581f1242 100644
--- a/libcxx/include/__cxx03/__compare/partial_order.h
+++ b/libcxx/include/__cxx03/__compare/partial_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_PARTIAL_ORDER
-#define _LIBCPP___COMPARE_PARTIAL_ORDER
+#ifndef _LIBCPP___CXX03___COMPARE_PARTIAL_ORDER
+#define _LIBCPP___CXX03___COMPARE_PARTIAL_ORDER
 
 #include <__cxx03/__compare/compare_three_way.h>
 #include <__cxx03/__compare/ordering.h>
@@ -74,4 +74,4 @@ inline constexpr auto partial_order = __partial_order::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_PARTIAL_ORDER
+#endif // _LIBCPP___CXX03___COMPARE_PARTIAL_ORDER
diff --git a/libcxx/include/__cxx03/__compare/strong_order.h b/libcxx/include/__cxx03/__compare/strong_order.h
index 7522325912351..ee29de2b9260f 100644
--- a/libcxx/include/__cxx03/__compare/strong_order.h
+++ b/libcxx/include/__cxx03/__compare/strong_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_STRONG_ORDER
-#define _LIBCPP___COMPARE_STRONG_ORDER
+#ifndef _LIBCPP___CXX03___COMPARE_STRONG_ORDER
+#define _LIBCPP___CXX03___COMPARE_STRONG_ORDER
 
 #include <__cxx03/__bit/bit_cast.h>
 #include <__cxx03/__compare/compare_three_way.h>
@@ -140,4 +140,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___COMPARE_STRONG_ORDER
+#endif // _LIBCPP___CXX03___COMPARE_STRONG_ORDER
diff --git a/libcxx/include/__cxx03/__compare/synth_three_way.h b/libcxx/include/__cxx03/__compare/synth_three_way.h
index fbaaf61864299..2625ea80bf1b7 100644
--- a/libcxx/include/__cxx03/__compare/synth_three_way.h
+++ b/libcxx/include/__cxx03/__compare/synth_three_way.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
-#define _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
+#ifndef _LIBCPP___CXX03___COMPARE_SYNTH_THREE_WAY_H
+#define _LIBCPP___CXX03___COMPARE_SYNTH_THREE_WAY_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -49,4 +49,4 @@ using __synth_three_way_result = decltype(std::__synth_three_way(std::declval<_T
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_SYNTH_THREE_WAY_H
+#endif // _LIBCPP___CXX03___COMPARE_SYNTH_THREE_WAY_H
diff --git a/libcxx/include/__cxx03/__compare/three_way_comparable.h b/libcxx/include/__cxx03/__compare/three_way_comparable.h
index 9ad84d2bde498..a3539459ea25c 100644
--- a/libcxx/include/__cxx03/__compare/three_way_comparable.h
+++ b/libcxx/include/__cxx03/__compare/three_way_comparable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
-#define _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
+#ifndef _LIBCPP___CXX03___COMPARE_THREE_WAY_COMPARABLE_H
+#define _LIBCPP___CXX03___COMPARE_THREE_WAY_COMPARABLE_H
 
 #include <__cxx03/__compare/common_comparison_category.h>
 #include <__cxx03/__compare/ordering.h>
@@ -52,4 +52,4 @@ concept three_way_comparable_with =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_THREE_WAY_COMPARABLE_H
+#endif // _LIBCPP___CXX03___COMPARE_THREE_WAY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__compare/weak_order.h b/libcxx/include/__cxx03/__compare/weak_order.h
index e4c4797c06db6..32ab1b68e459a 100644
--- a/libcxx/include/__cxx03/__compare/weak_order.h
+++ b/libcxx/include/__cxx03/__compare/weak_order.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COMPARE_WEAK_ORDER
-#define _LIBCPP___COMPARE_WEAK_ORDER
+#ifndef _LIBCPP___CXX03___COMPARE_WEAK_ORDER
+#define _LIBCPP___CXX03___COMPARE_WEAK_ORDER
 
 #include <__cxx03/__compare/compare_three_way.h>
 #include <__cxx03/__compare/ordering.h>
@@ -102,4 +102,4 @@ inline constexpr auto weak_order = __weak_order::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___COMPARE_WEAK_ORDER
+#endif // _LIBCPP___CXX03___COMPARE_WEAK_ORDER
diff --git a/libcxx/include/__cxx03/__concepts/arithmetic.h b/libcxx/include/__cxx03/__concepts/arithmetic.h
index a8ef1d1532cac..6c5196bcdf790 100644
--- a/libcxx/include/__cxx03/__concepts/arithmetic.h
+++ b/libcxx/include/__cxx03/__concepts/arithmetic.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_ARITHMETIC_H
-#define _LIBCPP___CONCEPTS_ARITHMETIC_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_ARITHMETIC_H
+#define _LIBCPP___CXX03___CONCEPTS_ARITHMETIC_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_floating_point.h>
@@ -53,4 +53,4 @@ concept __libcpp_integer = __libcpp_unsigned_integer<_Tp> || __libcpp_signed_int
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_ARITHMETIC_H
+#endif // _LIBCPP___CXX03___CONCEPTS_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__concepts/assignable.h b/libcxx/include/__cxx03/__concepts/assignable.h
index 563deb5e4cd69..8b69770222b0a 100644
--- a/libcxx/include/__cxx03/__concepts/assignable.h
+++ b/libcxx/include/__cxx03/__concepts/assignable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_ASSIGNABLE_H
-#define _LIBCPP___CONCEPTS_ASSIGNABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_ASSIGNABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_ASSIGNABLE_H
 
 #include <__cxx03/__concepts/common_reference_with.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -38,4 +38,4 @@ concept assignable_from =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_ASSIGNABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/boolean_testable.h b/libcxx/include/__cxx03/__concepts/boolean_testable.h
index 1f655c1b19cdf..21e03f76d8bb1 100644
--- a/libcxx/include/__cxx03/__concepts/boolean_testable.h
+++ b/libcxx/include/__cxx03/__concepts/boolean_testable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
-#define _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_BOOLEAN_TESTABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_BOOLEAN_TESTABLE_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -35,4 +35,4 @@ concept __boolean_testable = __boolean_testable_impl<_Tp> && requires(_Tp&& __t)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_BOOLEAN_TESTABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_BOOLEAN_TESTABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/class_or_enum.h b/libcxx/include/__cxx03/__concepts/class_or_enum.h
index ef9a61cdbebf7..a8604e6d9eb61 100644
--- a/libcxx/include/__cxx03/__concepts/class_or_enum.h
+++ b/libcxx/include/__cxx03/__concepts/class_or_enum.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
-#define _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_CLASS_OR_ENUM_H
+#define _LIBCPP___CXX03___CONCEPTS_CLASS_OR_ENUM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_class.h>
@@ -32,4 +32,4 @@ concept __class_or_enum = is_class_v<_Tp> || is_union_v<_Tp> || is_enum_v<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_CLASS_OR_ENUM_H
+#endif // _LIBCPP___CXX03___CONCEPTS_CLASS_OR_ENUM_H
diff --git a/libcxx/include/__cxx03/__concepts/common_reference_with.h b/libcxx/include/__cxx03/__concepts/common_reference_with.h
index 8e0b5bafe7862..1ffe9f67be48b 100644
--- a/libcxx/include/__cxx03/__concepts/common_reference_with.h
+++ b/libcxx/include/__cxx03/__concepts/common_reference_with.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
-#define _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_COMMON_REFERENCE_WITH_H
+#define _LIBCPP___CXX03___CONCEPTS_COMMON_REFERENCE_WITH_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -33,4 +33,4 @@ concept common_reference_with =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_COMMON_REFERENCE_WITH_H
+#endif // _LIBCPP___CXX03___CONCEPTS_COMMON_REFERENCE_WITH_H
diff --git a/libcxx/include/__cxx03/__concepts/common_with.h b/libcxx/include/__cxx03/__concepts/common_with.h
index 37f4e3c30c2b2..ee100052c0c52 100644
--- a/libcxx/include/__cxx03/__concepts/common_with.h
+++ b/libcxx/include/__cxx03/__concepts/common_with.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_COMMON_WITH_H
-#define _LIBCPP___CONCEPTS_COMMON_WITH_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_COMMON_WITH_H
+#define _LIBCPP___CXX03___CONCEPTS_COMMON_WITH_H
 
 #include <__cxx03/__concepts/common_reference_with.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -49,4 +49,4 @@ concept common_with =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_COMMON_WITH_H
+#endif // _LIBCPP___CXX03___CONCEPTS_COMMON_WITH_H
diff --git a/libcxx/include/__cxx03/__concepts/constructible.h b/libcxx/include/__cxx03/__concepts/constructible.h
index 356ca47626071..2d04667f7d3e4 100644
--- a/libcxx/include/__cxx03/__concepts/constructible.h
+++ b/libcxx/include/__cxx03/__concepts/constructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
-#define _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_CONSTRUCTIBLE_H
+#define _LIBCPP___CXX03___CONCEPTS_CONSTRUCTIBLE_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__concepts/destructible.h>
@@ -52,4 +52,4 @@ concept copy_constructible =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_CONSTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__concepts/convertible_to.h b/libcxx/include/__cxx03/__concepts/convertible_to.h
index 4802621c93ef7..af27b76c9c04a 100644
--- a/libcxx/include/__cxx03/__concepts/convertible_to.h
+++ b/libcxx/include/__cxx03/__concepts/convertible_to.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
-#define _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_CONVERTIBLE_TO_H
+#define _LIBCPP___CXX03___CONCEPTS_CONVERTIBLE_TO_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_convertible.h>
@@ -30,4 +30,4 @@ concept convertible_to = is_convertible_v<_From, _To> && requires { static_cast<
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_CONVERTIBLE_TO_H
+#endif // _LIBCPP___CXX03___CONCEPTS_CONVERTIBLE_TO_H
diff --git a/libcxx/include/__cxx03/__concepts/copyable.h b/libcxx/include/__cxx03/__concepts/copyable.h
index bcadc7382f50f..8ca2d4f940834 100644
--- a/libcxx/include/__cxx03/__concepts/copyable.h
+++ b/libcxx/include/__cxx03/__concepts/copyable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_COPYABLE_H
-#define _LIBCPP___CONCEPTS_COPYABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_COPYABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_COPYABLE_H
 
 #include <__cxx03/__concepts/assignable.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -38,4 +38,4 @@ concept copyable =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_COPYABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_COPYABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/derived_from.h b/libcxx/include/__cxx03/__concepts/derived_from.h
index 861c84f0cf6ba..8112b6c2b70f9 100644
--- a/libcxx/include/__cxx03/__concepts/derived_from.h
+++ b/libcxx/include/__cxx03/__concepts/derived_from.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_DERIVED_FROM_H
-#define _LIBCPP___CONCEPTS_DERIVED_FROM_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_DERIVED_FROM_H
+#define _LIBCPP___CXX03___CONCEPTS_DERIVED_FROM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_base_of.h>
@@ -30,4 +30,4 @@ concept derived_from = is_base_of_v<_Bp, _Dp> && is_convertible_v<const volatile
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_DERIVED_FROM_H
+#endif // _LIBCPP___CXX03___CONCEPTS_DERIVED_FROM_H
diff --git a/libcxx/include/__cxx03/__concepts/destructible.h b/libcxx/include/__cxx03/__concepts/destructible.h
index 5aa9a908648cd..bd302c3acb62c 100644
--- a/libcxx/include/__cxx03/__concepts/destructible.h
+++ b/libcxx/include/__cxx03/__concepts/destructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
-#define _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_DESTRUCTIBLE_H
+#define _LIBCPP___CXX03___CONCEPTS_DESTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_nothrow_destructible.h>
@@ -29,4 +29,4 @@ concept destructible = is_nothrow_destructible_v<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_DESTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__concepts/different_from.h b/libcxx/include/__cxx03/__concepts/different_from.h
index bedf3a0bf0b7f..a5d59d47be1d4 100644
--- a/libcxx/include/__cxx03/__concepts/different_from.h
+++ b/libcxx/include/__cxx03/__concepts/different_from.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
-#define _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_DIFFERENT_FROM_H
+#define _LIBCPP___CXX03___CONCEPTS_DIFFERENT_FROM_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -28,4 +28,4 @@ concept __different_from = !same_as<remove_cvref_t<_Tp>, remove_cvref_t<_Up>>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_DIFFERENT_FROM_H
+#endif // _LIBCPP___CXX03___CONCEPTS_DIFFERENT_FROM_H
diff --git a/libcxx/include/__cxx03/__concepts/equality_comparable.h b/libcxx/include/__cxx03/__concepts/equality_comparable.h
index a77f8f3de60b2..a3a4504bcb627 100644
--- a/libcxx/include/__cxx03/__concepts/equality_comparable.h
+++ b/libcxx/include/__cxx03/__concepts/equality_comparable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
-#define _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_EQUALITY_COMPARABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_EQUALITY_COMPARABLE_H
 
 #include <__cxx03/__concepts/boolean_testable.h>
 #include <__cxx03/__concepts/common_reference_with.h>
@@ -53,4 +53,4 @@ concept equality_comparable_with =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_EQUALITY_COMPARABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_EQUALITY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/invocable.h b/libcxx/include/__cxx03/__concepts/invocable.h
index fbbbf16e32fe9..624d8b0a92465 100644
--- a/libcxx/include/__cxx03/__concepts/invocable.h
+++ b/libcxx/include/__cxx03/__concepts/invocable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_INVOCABLE_H
-#define _LIBCPP___CONCEPTS_INVOCABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_INVOCABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_INVOCABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -37,4 +37,4 @@ concept regular_invocable = invocable<_Fn, _Args...>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_INVOCABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_INVOCABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/movable.h b/libcxx/include/__cxx03/__concepts/movable.h
index 883eda880b6f0..a0c3372e88c96 100644
--- a/libcxx/include/__cxx03/__concepts/movable.h
+++ b/libcxx/include/__cxx03/__concepts/movable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_MOVABLE_H
-#define _LIBCPP___CONCEPTS_MOVABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_MOVABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_MOVABLE_H
 
 #include <__cxx03/__concepts/assignable.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -32,4 +32,4 @@ concept movable = is_object_v<_Tp> && move_constructible<_Tp> && assignable_from
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_MOVABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_MOVABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/predicate.h b/libcxx/include/__cxx03/__concepts/predicate.h
index 15007f3b09926..5268854d50269 100644
--- a/libcxx/include/__cxx03/__concepts/predicate.h
+++ b/libcxx/include/__cxx03/__concepts/predicate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_PREDICATE_H
-#define _LIBCPP___CONCEPTS_PREDICATE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_PREDICATE_H
+#define _LIBCPP___CXX03___CONCEPTS_PREDICATE_H
 
 #include <__cxx03/__concepts/boolean_testable.h>
 #include <__cxx03/__concepts/invocable.h>
@@ -31,4 +31,4 @@ concept predicate = regular_invocable<_Fn, _Args...> && __boolean_testable<invok
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_PREDICATE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_PREDICATE_H
diff --git a/libcxx/include/__cxx03/__concepts/regular.h b/libcxx/include/__cxx03/__concepts/regular.h
index faf04f2c6b943..fd0365ab70435 100644
--- a/libcxx/include/__cxx03/__concepts/regular.h
+++ b/libcxx/include/__cxx03/__concepts/regular.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_REGULAR_H
-#define _LIBCPP___CONCEPTS_REGULAR_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_REGULAR_H
+#define _LIBCPP___CXX03___CONCEPTS_REGULAR_H
 
 #include <__cxx03/__concepts/equality_comparable.h>
 #include <__cxx03/__concepts/semiregular.h>
@@ -30,4 +30,4 @@ concept regular = semiregular<_Tp> && equality_comparable<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_REGULAR_H
+#endif // _LIBCPP___CXX03___CONCEPTS_REGULAR_H
diff --git a/libcxx/include/__cxx03/__concepts/relation.h b/libcxx/include/__cxx03/__concepts/relation.h
index 0d90406012e33..67a612979daab 100644
--- a/libcxx/include/__cxx03/__concepts/relation.h
+++ b/libcxx/include/__cxx03/__concepts/relation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_RELATION_H
-#define _LIBCPP___CONCEPTS_RELATION_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_RELATION_H
+#define _LIBCPP___CXX03___CONCEPTS_RELATION_H
 
 #include <__cxx03/__concepts/predicate.h>
 #include <__cxx03/__config>
@@ -40,4 +40,4 @@ concept strict_weak_order = relation<_Rp, _Tp, _Up>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_RELATION_H
+#endif // _LIBCPP___CXX03___CONCEPTS_RELATION_H
diff --git a/libcxx/include/__cxx03/__concepts/same_as.h b/libcxx/include/__cxx03/__concepts/same_as.h
index 6c81ecfbbfe5b..5318cda953189 100644
--- a/libcxx/include/__cxx03/__concepts/same_as.h
+++ b/libcxx/include/__cxx03/__concepts/same_as.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_SAME_AS_H
-#define _LIBCPP___CONCEPTS_SAME_AS_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_SAME_AS_H
+#define _LIBCPP___CXX03___CONCEPTS_SAME_AS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_same.h>
@@ -32,4 +32,4 @@ concept same_as = __same_as_impl<_Tp, _Up> && __same_as_impl<_Up, _Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_SAME_AS_H
+#endif // _LIBCPP___CXX03___CONCEPTS_SAME_AS_H
diff --git a/libcxx/include/__cxx03/__concepts/semiregular.h b/libcxx/include/__cxx03/__concepts/semiregular.h
index 2a3eb3d667292..06505080f8d36 100644
--- a/libcxx/include/__cxx03/__concepts/semiregular.h
+++ b/libcxx/include/__cxx03/__concepts/semiregular.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_SEMIREGULAR_H
-#define _LIBCPP___CONCEPTS_SEMIREGULAR_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_SEMIREGULAR_H
+#define _LIBCPP___CXX03___CONCEPTS_SEMIREGULAR_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/copyable.h>
@@ -30,4 +30,4 @@ concept semiregular = copyable<_Tp> && default_initializable<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_SEMIREGULAR_H
+#endif // _LIBCPP___CXX03___CONCEPTS_SEMIREGULAR_H
diff --git a/libcxx/include/__cxx03/__concepts/swappable.h b/libcxx/include/__cxx03/__concepts/swappable.h
index 333a42ad0c500..95adb0f1b8dad 100644
--- a/libcxx/include/__cxx03/__concepts/swappable.h
+++ b/libcxx/include/__cxx03/__concepts/swappable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_SWAPPABLE_H
-#define _LIBCPP___CONCEPTS_SWAPPABLE_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_SWAPPABLE_H
+#define _LIBCPP___CXX03___CONCEPTS_SWAPPABLE_H
 
 #include <__cxx03/__concepts/assignable.h>
 #include <__cxx03/__concepts/class_or_enum.h>
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CONCEPTS_SWAPPABLE_H
+#endif // _LIBCPP___CXX03___CONCEPTS_SWAPPABLE_H
diff --git a/libcxx/include/__cxx03/__concepts/totally_ordered.h b/libcxx/include/__cxx03/__concepts/totally_ordered.h
index f35502b58f125..85ba0e4fdf47d 100644
--- a/libcxx/include/__cxx03/__concepts/totally_ordered.h
+++ b/libcxx/include/__cxx03/__concepts/totally_ordered.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
-#define _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
+#ifndef _LIBCPP___CXX03___CONCEPTS_TOTALLY_ORDERED_H
+#define _LIBCPP___CXX03___CONCEPTS_TOTALLY_ORDERED_H
 
 #include <__cxx03/__concepts/boolean_testable.h>
 #include <__cxx03/__concepts/equality_comparable.h>
@@ -56,4 +56,4 @@ concept totally_ordered_with =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___CONCEPTS_TOTALLY_ORDERED_H
+#endif // _LIBCPP___CXX03___CONCEPTS_TOTALLY_ORDERED_H
diff --git a/libcxx/include/__cxx03/__condition_variable/condition_variable.h b/libcxx/include/__cxx03/__condition_variable/condition_variable.h
index d96c6928fa7be..8e41ad89914f9 100644
--- a/libcxx/include/__cxx03/__condition_variable/condition_variable.h
+++ b/libcxx/include/__cxx03/__condition_variable/condition_variable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
-#define _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
+#ifndef _LIBCPP___CXX03___CONDITION_VARIABLE_CONDITION_VARIABLE_H
+#define _LIBCPP___CXX03___CONDITION_VARIABLE_CONDITION_VARIABLE_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/steady_clock.h>
@@ -241,4 +241,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___CONDITION_VARIABLE_CONDITION_VARIABLE_H
+#endif // _LIBCPP___CXX03___CONDITION_VARIABLE_CONDITION_VARIABLE_H
diff --git a/libcxx/include/__cxx03/__config b/libcxx/include/__cxx03/__config
index 6f77292387776..935fa4cc404f4 100644
--- a/libcxx/include/__cxx03/__config
+++ b/libcxx/include/__cxx03/__config
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIG
-#define _LIBCPP___CONFIG
+#ifndef _LIBCPP___CXX03___CONFIG
+#define _LIBCPP___CXX03___CONFIG
 
 #include <__config_site>
 #include <__cxx03/__configuration/abi.h>
@@ -1225,4 +1225,4 @@ typedef __char32_t char32_t;
 
 #endif // __cplusplus
 
-#endif // _LIBCPP___CONFIG
+#endif // _LIBCPP___CXX03___CONFIG
diff --git a/libcxx/include/__cxx03/__config_site.in b/libcxx/include/__cxx03/__config_site.in
index 67022146c9082..174af4be5c4e5 100644
--- a/libcxx/include/__cxx03/__config_site.in
+++ b/libcxx/include/__cxx03/__config_site.in
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIG_SITE
-#define _LIBCPP___CONFIG_SITE
+#ifndef _LIBCPP___CXX03___CONFIG_SITE
+#define _LIBCPP___CXX03___CONFIG_SITE
 
 #cmakedefine _LIBCPP_ABI_VERSION @_LIBCPP_ABI_VERSION@
 #cmakedefine _LIBCPP_ABI_NAMESPACE @_LIBCPP_ABI_NAMESPACE@
@@ -53,4 +53,4 @@
 #  pragma clang diagnostic pop
 #endif
 
-#endif // _LIBCPP___CONFIG_SITE
+#endif // _LIBCPP___CXX03___CONFIG_SITE
diff --git a/libcxx/include/__cxx03/__configuration/abi.h b/libcxx/include/__cxx03/__configuration/abi.h
index 0d11528877697..9e7f7313c8834 100644
--- a/libcxx/include/__cxx03/__configuration/abi.h
+++ b/libcxx/include/__cxx03/__configuration/abi.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIGURATION_ABI_H
-#define _LIBCPP___CONFIGURATION_ABI_H
+#ifndef _LIBCPP___CXX03___CONFIGURATION_ABI_H
+#define _LIBCPP___CXX03___CONFIGURATION_ABI_H
 
 #include <__config_site>
 #include <__cxx03/__configuration/compiler.h>
@@ -169,4 +169,4 @@
 #  endif
 #endif
 
-#endif // _LIBCPP___CONFIGURATION_ABI_H
+#endif // _LIBCPP___CXX03___CONFIGURATION_ABI_H
diff --git a/libcxx/include/__cxx03/__configuration/availability.h b/libcxx/include/__cxx03/__configuration/availability.h
index e54ba50ae9a02..ad45ce01d0a7e 100644
--- a/libcxx/include/__cxx03/__configuration/availability.h
+++ b/libcxx/include/__cxx03/__configuration/availability.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIGURATION_AVAILABILITY_H
-#define _LIBCPP___CONFIGURATION_AVAILABILITY_H
+#ifndef _LIBCPP___CXX03___CONFIGURATION_AVAILABILITY_H
+#define _LIBCPP___CXX03___CONFIGURATION_AVAILABILITY_H
 
 #include <__cxx03/__configuration/compiler.h>
 #include <__cxx03/__configuration/language.h>
@@ -397,4 +397,4 @@
 #  define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION
 #endif
 
-#endif // _LIBCPP___CONFIGURATION_AVAILABILITY_H
+#endif // _LIBCPP___CXX03___CONFIGURATION_AVAILABILITY_H
diff --git a/libcxx/include/__cxx03/__configuration/compiler.h b/libcxx/include/__cxx03/__configuration/compiler.h
index 80ece22bb50bd..2f33fb1f6ef4d 100644
--- a/libcxx/include/__cxx03/__configuration/compiler.h
+++ b/libcxx/include/__cxx03/__configuration/compiler.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIGURATION_COMPILER_H
-#define _LIBCPP___CONFIGURATION_COMPILER_H
+#ifndef _LIBCPP___CXX03___CONFIGURATION_COMPILER_H
+#define _LIBCPP___CXX03___CONFIGURATION_COMPILER_H
 
 #include <__config_site>
 
@@ -48,4 +48,4 @@
 
 #endif
 
-#endif // _LIBCPP___CONFIGURATION_COMPILER_H
+#endif // _LIBCPP___CXX03___CONFIGURATION_COMPILER_H
diff --git a/libcxx/include/__cxx03/__configuration/language.h b/libcxx/include/__cxx03/__configuration/language.h
index fa62a7b6f5c2a..604f2d2cd532a 100644
--- a/libcxx/include/__cxx03/__configuration/language.h
+++ b/libcxx/include/__cxx03/__configuration/language.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIGURATION_LANGUAGE_H
-#define _LIBCPP___CONFIGURATION_LANGUAGE_H
+#ifndef _LIBCPP___CXX03___CONFIGURATION_LANGUAGE_H
+#define _LIBCPP___CXX03___CONFIGURATION_LANGUAGE_H
 
 #include <__config_site>
 
@@ -43,4 +43,4 @@
 #  define _LIBCPP_HAS_NO_EXCEPTIONS
 #endif
 
-#endif // _LIBCPP___CONFIGURATION_LANGUAGE_H
+#endif // _LIBCPP___CXX03___CONFIGURATION_LANGUAGE_H
diff --git a/libcxx/include/__cxx03/__configuration/platform.h b/libcxx/include/__cxx03/__configuration/platform.h
index 27f68d04e8a8d..b4718986ad10d 100644
--- a/libcxx/include/__cxx03/__configuration/platform.h
+++ b/libcxx/include/__cxx03/__configuration/platform.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___CONFIGURATION_PLATFORM_H
-#define _LIBCPP___CONFIGURATION_PLATFORM_H
+#ifndef _LIBCPP___CXX03___CONFIGURATION_PLATFORM_H
+#define _LIBCPP___CXX03___CONFIGURATION_PLATFORM_H
 
 #include <__config_site>
 
@@ -51,4 +51,4 @@
 #  define _LIBCPP_BIG_ENDIAN
 #endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 
-#endif // _LIBCPP___CONFIGURATION_PLATFORM_H
+#endif // _LIBCPP___CXX03___CONFIGURATION_PLATFORM_H
diff --git a/libcxx/include/__cxx03/__coroutine/coroutine_handle.h b/libcxx/include/__cxx03/__coroutine/coroutine_handle.h
index 183d113e2e69b..f96c56bae168b 100644
--- a/libcxx/include/__cxx03/__coroutine/coroutine_handle.h
+++ b/libcxx/include/__cxx03/__coroutine/coroutine_handle.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
-#define _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
+#ifndef _LIBCPP___CXX03___COROUTINE_COROUTINE_HANDLE_H
+#define _LIBCPP___CXX03___COROUTINE_COROUTINE_HANDLE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -173,4 +173,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // __LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
+#endif // _LIBCPP___CXX03___COROUTINE_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__cxx03/__coroutine/coroutine_traits.h b/libcxx/include/__cxx03/__coroutine/coroutine_traits.h
index b4fc2f45a1ac4..2380b7a233910 100644
--- a/libcxx/include/__cxx03/__coroutine/coroutine_traits.h
+++ b/libcxx/include/__cxx03/__coroutine/coroutine_traits.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
-#define _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
+#ifndef _LIBCPP___CXX03___COROUTINE_COROUTINE_TRAITS_H
+#define _LIBCPP___CXX03___COROUTINE_COROUTINE_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/void_t.h>
@@ -45,4 +45,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // __LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
+#endif // _LIBCPP___CXX03___COROUTINE_COROUTINE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h b/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
index 2a6e25cde2f68..c6ec745dbc2fc 100644
--- a/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
+++ b/libcxx/include/__cxx03/__coroutine/noop_coroutine_handle.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
-#define _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
+#ifndef _LIBCPP___CXX03___COROUTINE_NOOP_COROUTINE_HANDLE_H
+#define _LIBCPP___CXX03___COROUTINE_NOOP_COROUTINE_HANDLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__coroutine/coroutine_handle.h>
@@ -96,4 +96,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // __LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
+#endif // _LIBCPP___CXX03___COROUTINE_NOOP_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h b/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
index 9420824b6cefc..7e34aad9cfce8 100644
--- a/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
+++ b/libcxx/include/__cxx03/__coroutine/trivial_awaitables.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
-#define __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
+#ifndef __LIBCPP___CXX03___COROUTINE_TRIVIAL_AWAITABLES_H
+#define __LIBCPP___CXX03___COROUTINE_TRIVIAL_AWAITABLES_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__coroutine/coroutine_handle.h>
@@ -37,4 +37,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // __LIBCPP_STD_VER >= 20
 
-#endif // __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
+#endif // __LIBCPP___CXX03___COROUTINE_TRIVIAL_AWAITABLES_H
diff --git a/libcxx/include/__cxx03/__debug_utils/randomize_range.h b/libcxx/include/__cxx03/__debug_utils/randomize_range.h
index 59e3a04f6d1ad..1baf4f5361ffd 100644
--- a/libcxx/include/__cxx03/__debug_utils/randomize_range.h
+++ b/libcxx/include/__cxx03/__debug_utils/randomize_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
-#define _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
+#ifndef _LIBCPP___CXX03___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
+#define _LIBCPP___CXX03___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
 
 #include <__cxx03/__config>
 
@@ -39,4 +39,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __debug_randomize_range
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
+#endif // _LIBCPP___CXX03___LIBCXX_DEBUG_RANDOMIZE_RANGE_H
diff --git a/libcxx/include/__cxx03/__debug_utils/sanitizers.h b/libcxx/include/__cxx03/__debug_utils/sanitizers.h
index cd0caa9ed0a3f..e3cda20468b53 100644
--- a/libcxx/include/__cxx03/__debug_utils/sanitizers.h
+++ b/libcxx/include/__cxx03/__debug_utils/sanitizers.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
-#define _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+#ifndef _LIBCPP___CXX03___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+#define _LIBCPP___CXX03___LIBCXX_DEBUG_UTILS_SANITIZERS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -101,4 +101,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __annotate_contiguous_c
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+#endif // _LIBCPP___CXX03___LIBCXX_DEBUG_UTILS_SANITIZERS_H
diff --git a/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h b/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
index 6aec78e46ff89..8d3a918d9b557 100644
--- a/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
+++ b/libcxx/include/__cxx03/__debug_utils/strict_weak_ordering_check.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
-#define _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
+#ifndef _LIBCPP___CXX03___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
+#define _LIBCPP___CXX03___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
 
 #include <__cxx03/__config>
 
@@ -74,4 +74,4 @@ __check_strict_weak_ordering_sorted(_RandomAccessIterator __first, _RandomAccess
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
+#endif // _LIBCPP___CXX03___LIBCXX_DEBUG_STRICT_WEAK_ORDERING_CHECK
diff --git a/libcxx/include/__cxx03/__exception/exception.h b/libcxx/include/__cxx03/__exception/exception.h
index 8557cbd4945e4..27b1e4353ec46 100644
--- a/libcxx/include/__cxx03/__exception/exception.h
+++ b/libcxx/include/__cxx03/__exception/exception.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___EXCEPTION_EXCEPTION_H
-#define _LIBCPP___EXCEPTION_EXCEPTION_H
+#ifndef _LIBCPP___CXX03___EXCEPTION_EXCEPTION_H
+#define _LIBCPP___CXX03___EXCEPTION_EXCEPTION_H
 
 #include <__cxx03/__config>
 
@@ -91,4 +91,4 @@ class _LIBCPP_EXPORTED_FROM_ABI bad_exception : public exception {
 
 } // namespace std
 
-#endif // _LIBCPP___EXCEPTION_EXCEPTION_H
+#endif // _LIBCPP___CXX03___EXCEPTION_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__exception/exception_ptr.h b/libcxx/include/__cxx03/__exception/exception_ptr.h
index 6bdb47fd395f0..33dbd40bef42c 100644
--- a/libcxx/include/__cxx03/__exception/exception_ptr.h
+++ b/libcxx/include/__cxx03/__exception/exception_ptr.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
-#define _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
+#ifndef _LIBCPP___CXX03___EXCEPTION_EXCEPTION_PTR_H
+#define _LIBCPP___CXX03___EXCEPTION_EXCEPTION_PTR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__exception/operations.h>
@@ -174,4 +174,4 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
 #endif // _LIBCPP_ABI_MICROSOFT
 } // namespace std
 
-#endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H
+#endif // _LIBCPP___CXX03___EXCEPTION_EXCEPTION_PTR_H
diff --git a/libcxx/include/__cxx03/__exception/nested_exception.h b/libcxx/include/__cxx03/__exception/nested_exception.h
index 1b889e6bf5d87..60799942fb136 100644
--- a/libcxx/include/__cxx03/__exception/nested_exception.h
+++ b/libcxx/include/__cxx03/__exception/nested_exception.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
-#define _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
+#ifndef _LIBCPP___CXX03___EXCEPTION_NESTED_EXCEPTION_H
+#define _LIBCPP___CXX03___EXCEPTION_NESTED_EXCEPTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__exception/exception_ptr.h>
@@ -96,4 +96,4 @@ inline _LIBCPP_HIDE_FROM_ABI void rethrow_if_nested(const _Ep&) {}
 
 } // namespace std
 
-#endif // _LIBCPP___EXCEPTION_NESTED_EXCEPTION_H
+#endif // _LIBCPP___CXX03___EXCEPTION_NESTED_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__exception/operations.h b/libcxx/include/__cxx03/__exception/operations.h
index cb67194553d37..9049a0f313fae 100644
--- a/libcxx/include/__cxx03/__exception/operations.h
+++ b/libcxx/include/__cxx03/__exception/operations.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___EXCEPTION_OPERATIONS_H
-#define _LIBCPP___EXCEPTION_OPERATIONS_H
+#ifndef _LIBCPP___CXX03___EXCEPTION_OPERATIONS_H
+#define _LIBCPP___CXX03___EXCEPTION_OPERATIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -38,4 +38,4 @@ _LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
 _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
 } // namespace std
 
-#endif // _LIBCPP___EXCEPTION_OPERATIONS_H
+#endif // _LIBCPP___CXX03___EXCEPTION_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__exception/terminate.h b/libcxx/include/__cxx03/__exception/terminate.h
index 78ccd16c91393..5f0cfa930f9af 100644
--- a/libcxx/include/__cxx03/__exception/terminate.h
+++ b/libcxx/include/__cxx03/__exception/terminate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___EXCEPTION_TERMINATE_H
-#define _LIBCPP___EXCEPTION_TERMINATE_H
+#ifndef _LIBCPP___CXX03___EXCEPTION_TERMINATE_H
+#define _LIBCPP___CXX03___EXCEPTION_TERMINATE_H
 
 #include <__cxx03/__config>
 
@@ -19,4 +19,4 @@ namespace std { // purposefully not using versioning namespace
 _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void terminate() _NOEXCEPT;
 } // namespace std
 
-#endif // _LIBCPP___EXCEPTION_TERMINATE_H
+#endif // _LIBCPP___CXX03___EXCEPTION_TERMINATE_H
diff --git a/libcxx/include/__cxx03/__expected/bad_expected_access.h b/libcxx/include/__cxx03/__expected/bad_expected_access.h
index 4f7e33d2c6986..d9c64976c25b9 100644
--- a/libcxx/include/__cxx03/__expected/bad_expected_access.h
+++ b/libcxx/include/__cxx03/__expected/bad_expected_access.h
@@ -6,8 +6,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
-#define _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
+#ifndef _LIBCPP___CXX03___EXPECTED_BAD_EXPECTED_ACCESS_H
+#define _LIBCPP___CXX03___EXPECTED_BAD_EXPECTED_ACCESS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__exception/exception.h>
@@ -70,4 +70,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
+#endif // _LIBCPP___CXX03___EXPECTED_BAD_EXPECTED_ACCESS_H
diff --git a/libcxx/include/__cxx03/__expected/expected.h b/libcxx/include/__cxx03/__expected/expected.h
index adadea8e4b39c..1d54bb9f6edeb 100644
--- a/libcxx/include/__cxx03/__expected/expected.h
+++ b/libcxx/include/__cxx03/__expected/expected.h
@@ -6,8 +6,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___EXPECTED_EXPECTED_H
-#define _LIBCPP___EXPECTED_EXPECTED_H
+#ifndef _LIBCPP___CXX03___EXPECTED_EXPECTED_H
+#define _LIBCPP___CXX03___EXPECTED_EXPECTED_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -1871,4 +1871,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___EXPECTED_EXPECTED_H
+#endif // _LIBCPP___CXX03___EXPECTED_EXPECTED_H
diff --git a/libcxx/include/__cxx03/__expected/unexpect.h b/libcxx/include/__cxx03/__expected/unexpect.h
index 895f053a1e333..d8de027cabc5b 100644
--- a/libcxx/include/__cxx03/__expected/unexpect.h
+++ b/libcxx/include/__cxx03/__expected/unexpect.h
@@ -6,8 +6,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___EXPECTED_UNEXPECT_H
-#define _LIBCPP___EXPECTED_UNEXPECT_H
+#ifndef _LIBCPP___CXX03___EXPECTED_UNEXPECT_H
+#define _LIBCPP___CXX03___EXPECTED_UNEXPECT_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 23
 
-#endif // _LIBCPP___EXPECTED_UNEXPECT_H
+#endif // _LIBCPP___CXX03___EXPECTED_UNEXPECT_H
diff --git a/libcxx/include/__cxx03/__expected/unexpected.h b/libcxx/include/__cxx03/__expected/unexpected.h
index fe2ab407b4151..bafa33a011eea 100644
--- a/libcxx/include/__cxx03/__expected/unexpected.h
+++ b/libcxx/include/__cxx03/__expected/unexpected.h
@@ -6,8 +6,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___EXPECTED_UNEXPECTED_H
-#define _LIBCPP___EXPECTED_UNEXPECTED_H
+#ifndef _LIBCPP___CXX03___EXPECTED_UNEXPECTED_H
+#define _LIBCPP___CXX03___EXPECTED_UNEXPECTED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conjunction.h>
@@ -124,4 +124,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___EXPECTED_UNEXPECTED_H
+#endif // _LIBCPP___CXX03___EXPECTED_UNEXPECTED_H
diff --git a/libcxx/include/__cxx03/__filesystem/copy_options.h b/libcxx/include/__cxx03/__filesystem/copy_options.h
index c9adf3cd64eb4..01abb5042b7e4 100644
--- a/libcxx/include/__cxx03/__filesystem/copy_options.h
+++ b/libcxx/include/__cxx03/__filesystem/copy_options.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
-#define _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_COPY_OPTIONS_H
+#define _LIBCPP___CXX03___FILESYSTEM_COPY_OPTIONS_H
 
 #include <__cxx03/__config>
 
@@ -66,4 +66,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_COPY_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_entry.h b/libcxx/include/__cxx03/__filesystem/directory_entry.h
index a78f43d6b1c5c..1a8ebf470bf89 100644
--- a/libcxx/include/__cxx03/__filesystem/directory_entry.h
+++ b/libcxx/include/__cxx03/__filesystem/directory_entry.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
-#define _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ENTRY_H
+#define _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ENTRY_H
 
 #include <__cxx03/__chrono/time_point.h>
 #include <__cxx03/__compare/ordering.h>
@@ -432,4 +432,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ENTRY_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_iterator.h b/libcxx/include/__cxx03/__filesystem/directory_iterator.h
index 667007f5c2cdf..226ef7a27f942 100644
--- a/libcxx/include/__cxx03/__filesystem/directory_iterator.h
+++ b/libcxx/include/__cxx03/__filesystem/directory_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
-#define _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ITERATOR_H
+#define _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ITERATOR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -148,4 +148,4 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/directory_options.h b/libcxx/include/__cxx03/__filesystem/directory_options.h
index eeb87936f2bd1..babdf9c911bee 100644
--- a/libcxx/include/__cxx03/__filesystem/directory_options.h
+++ b/libcxx/include/__cxx03/__filesystem/directory_options.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
-#define _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_OPTIONS_H
+#define _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_OPTIONS_H
 
 #include <__cxx03/__config>
 
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_DIRECTORY_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_status.h b/libcxx/include/__cxx03/__filesystem/file_status.h
index 0022518842af7..1e5ea497434d9 100644
--- a/libcxx/include/__cxx03/__filesystem/file_status.h
+++ b/libcxx/include/__cxx03/__filesystem/file_status.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_FILE_STATUS_H
-#define _LIBCPP___FILESYSTEM_FILE_STATUS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_FILE_STATUS_H
+#define _LIBCPP___CXX03___FILESYSTEM_FILE_STATUS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__filesystem/file_type.h>
@@ -64,4 +64,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_FILE_STATUS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_FILE_STATUS_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_time_type.h b/libcxx/include/__cxx03/__filesystem/file_time_type.h
index 1e964bbe32522..cd52453b0b879 100644
--- a/libcxx/include/__cxx03/__filesystem/file_time_type.h
+++ b/libcxx/include/__cxx03/__filesystem/file_time_type.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
-#define _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_FILE_TIME_TYPE_H
+#define _LIBCPP___CXX03___FILESYSTEM_FILE_TIME_TYPE_H
 
 #include <__cxx03/__chrono/file_clock.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -28,4 +28,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_FILE_TIME_TYPE_H
diff --git a/libcxx/include/__cxx03/__filesystem/file_type.h b/libcxx/include/__cxx03/__filesystem/file_type.h
index 392ca2b57c26b..34f0aba307128 100644
--- a/libcxx/include/__cxx03/__filesystem/file_type.h
+++ b/libcxx/include/__cxx03/__filesystem/file_type.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_FILE_TYPE_H
-#define _LIBCPP___FILESYSTEM_FILE_TYPE_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_FILE_TYPE_H
+#define _LIBCPP___CXX03___FILESYSTEM_FILE_TYPE_H
 
 #include <__cxx03/__config>
 
@@ -39,4 +39,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_FILE_TYPE_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_FILE_TYPE_H
diff --git a/libcxx/include/__cxx03/__filesystem/filesystem_error.h b/libcxx/include/__cxx03/__filesystem/filesystem_error.h
index 8b68bac2e36fa..ef9e0d7846436 100644
--- a/libcxx/include/__cxx03/__filesystem/filesystem_error.h
+++ b/libcxx/include/__cxx03/__filesystem/filesystem_error.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
-#define _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_FILESYSTEM_ERROR_H
+#define _LIBCPP___CXX03___FILESYSTEM_FILESYSTEM_ERROR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__filesystem/path.h>
@@ -85,4 +85,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_FILESYSTEM_ERROR_H
diff --git a/libcxx/include/__cxx03/__filesystem/operations.h b/libcxx/include/__cxx03/__filesystem/operations.h
index 90b90bbaace93..fdb3dab858f50 100644
--- a/libcxx/include/__cxx03/__filesystem/operations.h
+++ b/libcxx/include/__cxx03/__filesystem/operations.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_OPERATIONS_H
-#define _LIBCPP___FILESYSTEM_OPERATIONS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_OPERATIONS_H
+#define _LIBCPP___CXX03___FILESYSTEM_OPERATIONS_H
 
 #include <__cxx03/__chrono/time_point.h>
 #include <__cxx03/__config>
@@ -307,4 +307,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM)
 
-#endif // _LIBCPP___FILESYSTEM_OPERATIONS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/path.h b/libcxx/include/__cxx03/__filesystem/path.h
index 815d881d8a099..e5eed68e4d6c4 100644
--- a/libcxx/include/__cxx03/__filesystem/path.h
+++ b/libcxx/include/__cxx03/__filesystem/path.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_PATH_H
-#define _LIBCPP___FILESYSTEM_PATH_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_PATH_H
+#define _LIBCPP___CXX03___FILESYSTEM_PATH_H
 
 #include <__cxx03/__algorithm/replace.h>
 #include <__cxx03/__algorithm/replace_copy.h>
@@ -928,4 +928,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FILESYSTEM_PATH_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_PATH_H
diff --git a/libcxx/include/__cxx03/__filesystem/path_iterator.h b/libcxx/include/__cxx03/__filesystem/path_iterator.h
index c439782a2a6af..121626b4b6ca0 100644
--- a/libcxx/include/__cxx03/__filesystem/path_iterator.h
+++ b/libcxx/include/__cxx03/__filesystem/path_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
-#define _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_PATH_ITERATOR_H
+#define _LIBCPP___CXX03___FILESYSTEM_PATH_ITERATOR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -112,4 +112,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_PATH_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/perm_options.h b/libcxx/include/__cxx03/__filesystem/perm_options.h
index bfd7941cf1003..bebb3038bf6eb 100644
--- a/libcxx/include/__cxx03/__filesystem/perm_options.h
+++ b/libcxx/include/__cxx03/__filesystem/perm_options.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
-#define _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_PERM_OPTIONS_H
+#define _LIBCPP___CXX03___FILESYSTEM_PERM_OPTIONS_H
 
 #include <__cxx03/__config>
 
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_PERM_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__filesystem/perms.h b/libcxx/include/__cxx03/__filesystem/perms.h
index d14023730938c..139352c3a8f89 100644
--- a/libcxx/include/__cxx03/__filesystem/perms.h
+++ b/libcxx/include/__cxx03/__filesystem/perms.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_PERMS_H
-#define _LIBCPP___FILESYSTEM_PERMS_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_PERMS_H
+#define _LIBCPP___CXX03___FILESYSTEM_PERMS_H
 
 #include <__cxx03/__config>
 
@@ -77,4 +77,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_PERMS_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_PERMS_H
diff --git a/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h b/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
index 43da731cc5b9c..a459bd3db4bb2 100644
--- a/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
+++ b/libcxx/include/__cxx03/__filesystem/recursive_directory_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
-#define _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+#define _LIBCPP___CXX03___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__filesystem/directory_entry.h>
@@ -161,4 +161,4 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__filesystem/space_info.h b/libcxx/include/__cxx03/__filesystem/space_info.h
index 8e2f260249793..0a998f87329a8 100644
--- a/libcxx/include/__cxx03/__filesystem/space_info.h
+++ b/libcxx/include/__cxx03/__filesystem/space_info.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_SPACE_INFO_H
-#define _LIBCPP___FILESYSTEM_SPACE_INFO_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_SPACE_INFO_H
+#define _LIBCPP___CXX03___FILESYSTEM_SPACE_INFO_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstdint>
@@ -35,4 +35,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_SPACE_INFO_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_SPACE_INFO_H
diff --git a/libcxx/include/__cxx03/__filesystem/u8path.h b/libcxx/include/__cxx03/__filesystem/u8path.h
index 7f1fe89bf2522..f60ef6a83f5cc 100644
--- a/libcxx/include/__cxx03/__filesystem/u8path.h
+++ b/libcxx/include/__cxx03/__filesystem/u8path.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FILESYSTEM_U8PATH_H
-#define _LIBCPP___FILESYSTEM_U8PATH_H
+#ifndef _LIBCPP___CXX03___FILESYSTEM_U8PATH_H
+#define _LIBCPP___CXX03___FILESYSTEM_U8PATH_H
 
 #include <__cxx03/__algorithm/unwrap_iter.h>
 #include <__cxx03/__config>
@@ -97,4 +97,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FILESYSTEM_U8PATH_H
+#endif // _LIBCPP___CXX03___FILESYSTEM_U8PATH_H
diff --git a/libcxx/include/__cxx03/__format/buffer.h b/libcxx/include/__cxx03/__format/buffer.h
index d4162086b84f4..167b06d7fd226 100644
--- a/libcxx/include/__cxx03/__format/buffer.h
+++ b/libcxx/include/__cxx03/__format/buffer.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_BUFFER_H
-#define _LIBCPP___FORMAT_BUFFER_H
+#ifndef _LIBCPP___CXX03___FORMAT_BUFFER_H
+#define _LIBCPP___CXX03___FORMAT_BUFFER_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__algorithm/fill_n.h>
@@ -652,4 +652,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_BUFFER_H
+#endif // _LIBCPP___CXX03___FORMAT_BUFFER_H
diff --git a/libcxx/include/__cxx03/__format/concepts.h b/libcxx/include/__cxx03/__format/concepts.h
index ffe937e207ffe..90fcb7ed0b8f6 100644
--- a/libcxx/include/__cxx03/__format/concepts.h
+++ b/libcxx/include/__cxx03/__format/concepts.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_CONCEPTS_H
-#define _LIBCPP___FORMAT_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___FORMAT_CONCEPTS_H
+#define _LIBCPP___CXX03___FORMAT_CONCEPTS_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__concepts/semiregular.h>
@@ -80,4 +80,4 @@ concept __fmt_pair_like =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_CONCEPTS_H
+#endif // _LIBCPP___CXX03___FORMAT_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__format/container_adaptor.h b/libcxx/include/__cxx03/__format/container_adaptor.h
index 5d47677d8d537..42c0d14be61a8 100644
--- a/libcxx/include/__cxx03/__format/container_adaptor.h
+++ b/libcxx/include/__cxx03/__format/container_adaptor.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
-#define _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
+#ifndef _LIBCPP___CXX03___FORMAT_CONTAINER_ADAPTOR_H
+#define _LIBCPP___CXX03___FORMAT_CONTAINER_ADAPTOR_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -70,4 +70,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<stack<_Tp, _Container>, _CharT>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_CONTAINER_ADAPTOR_H
+#endif // _LIBCPP___CXX03___FORMAT_CONTAINER_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__format/enable_insertable.h b/libcxx/include/__cxx03/__format/enable_insertable.h
index 8bcae600a54ea..d52ca8cfd442c 100644
--- a/libcxx/include/__cxx03/__format/enable_insertable.h
+++ b/libcxx/include/__cxx03/__format/enable_insertable.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
-#define _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_ENABLE_INSERTABLE_H
+#define _LIBCPP___CXX03___FORMAT_ENABLE_INSERTABLE_H
 
 #include <__cxx03/__config>
 
@@ -32,4 +32,4 @@ inline constexpr bool __enable_insertable = false;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_ENABLE_INSERTABLE_H
+#endif // _LIBCPP___CXX03___FORMAT_ENABLE_INSERTABLE_H
diff --git a/libcxx/include/__cxx03/__format/escaped_output_table.h b/libcxx/include/__cxx03/__format/escaped_output_table.h
index 0039968566f88..c5142849865ba 100644
--- a/libcxx/include/__cxx03/__format/escaped_output_table.h
+++ b/libcxx/include/__cxx03/__format/escaped_output_table.h
@@ -58,8 +58,8 @@
 // use or other dealings in these Data Files or Software without prior
 // written authorization of the copyright holder.
 
-#ifndef _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
-#define _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_ESCAPED_OUTPUT_TABLE_H
+#define _LIBCPP___CXX03___FORMAT_ESCAPED_OUTPUT_TABLE_H
 
 #include <__cxx03/__algorithm/ranges_upper_bound.h>
 #include <__cxx03/__config>
@@ -860,4 +860,4 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[711] = {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_ESCAPED_OUTPUT_TABLE_H
+#endif // _LIBCPP___CXX03___FORMAT_ESCAPED_OUTPUT_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h b/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
index d83838bbc83a7..c26b870d30378 100644
--- a/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
+++ b/libcxx/include/__cxx03/__format/extended_grapheme_cluster_table.h
@@ -58,8 +58,8 @@
 // use or other dealings in these Data Files or Software without prior
 // written authorization of the copyright holder.
 
-#ifndef _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
-#define _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
+#define _LIBCPP___CXX03___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
 
 #include <__cxx03/__algorithm/ranges_upper_bound.h>
 #include <__cxx03/__config>
@@ -1660,4 +1660,4 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
+#endif // _LIBCPP___CXX03___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/format_arg.h b/libcxx/include/__cxx03/__format/format_arg.h
index 3d37555c0e2df..cf9bf75e60df9 100644
--- a/libcxx/include/__cxx03/__format/format_arg.h
+++ b/libcxx/include/__cxx03/__format/format_arg.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_ARG_H
-#define _LIBCPP___FORMAT_FORMAT_ARG_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_ARG_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_ARG_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -398,4 +398,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMAT_ARG_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_ARG_H
diff --git a/libcxx/include/__cxx03/__format/format_arg_store.h b/libcxx/include/__cxx03/__format/format_arg_store.h
index 836a5a2ffc456..1116b53edc20c 100644
--- a/libcxx/include/__cxx03/__format/format_arg_store.h
+++ b/libcxx/include/__cxx03/__format/format_arg_store.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
-#define _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_ARG_STORE_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_ARG_STORE_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -263,4 +263,4 @@ struct _LIBCPP_TEMPLATE_VIS __format_arg_store {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_ARG_STORE_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_ARG_STORE_H
diff --git a/libcxx/include/__cxx03/__format/format_args.h b/libcxx/include/__cxx03/__format/format_args.h
index 494ba91cf55ae..702b8763d31e5 100644
--- a/libcxx/include/__cxx03/__format/format_args.h
+++ b/libcxx/include/__cxx03/__format/format_args.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_ARGS_H
-#define _LIBCPP___FORMAT_FORMAT_ARGS_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_ARGS_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_ARGS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__format/format_arg.h>
@@ -75,4 +75,4 @@ basic_format_args(__format_arg_store<_Context, _Args...>) -> basic_format_args<_
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_ARGS_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_ARGS_H
diff --git a/libcxx/include/__cxx03/__format/format_context.h b/libcxx/include/__cxx03/__format/format_context.h
index 04f2d69a247db..7f1011581133b 100644
--- a/libcxx/include/__cxx03/__format/format_context.h
+++ b/libcxx/include/__cxx03/__format/format_context.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_CONTEXT_H
-#define _LIBCPP___FORMAT_FORMAT_CONTEXT_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_CONTEXT_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_CONTEXT_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -217,4 +217,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMAT_CONTEXT_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_CONTEXT_H
diff --git a/libcxx/include/__cxx03/__format/format_error.h b/libcxx/include/__cxx03/__format/format_error.h
index 701bf3ad0dcc5..c0a2959131e48 100644
--- a/libcxx/include/__cxx03/__format/format_error.h
+++ b/libcxx/include/__cxx03/__format/format_error.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_ERROR_H
-#define _LIBCPP___FORMAT_FORMAT_ERROR_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_ERROR_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_ERROR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__verbose_abort>
@@ -47,4 +47,4 @@ _LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_format_error(const ch
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_ERROR_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_ERROR_H
diff --git a/libcxx/include/__cxx03/__format/format_functions.h b/libcxx/include/__cxx03/__format/format_functions.h
index bc9056dd0bdeb..d3a1bc46c8a7d 100644
--- a/libcxx/include/__cxx03/__format/format_functions.h
+++ b/libcxx/include/__cxx03/__format/format_functions.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_FUNCTIONS
-#define _LIBCPP___FORMAT_FORMAT_FUNCTIONS
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_FUNCTIONS
+#define _LIBCPP___CXX03___FORMAT_FORMAT_FUNCTIONS
 
 #include <__cxx03/__algorithm/clamp.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -677,4 +677,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMAT_FUNCTIONS
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_FUNCTIONS
diff --git a/libcxx/include/__cxx03/__format/format_parse_context.h b/libcxx/include/__cxx03/__format/format_parse_context.h
index 1df545987659b..81338506aee69 100644
--- a/libcxx/include/__cxx03/__format/format_parse_context.h
+++ b/libcxx/include/__cxx03/__format/format_parse_context.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
-#define _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_PARSE_CONTEXT_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_PARSE_CONTEXT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__format/format_error.h>
@@ -102,4 +102,4 @@ using wformat_parse_context = basic_format_parse_context<wchar_t>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_PARSE_CONTEXT_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_PARSE_CONTEXT_H
diff --git a/libcxx/include/__cxx03/__format/format_string.h b/libcxx/include/__cxx03/__format/format_string.h
index c48719337372d..6a7276419167a 100644
--- a/libcxx/include/__cxx03/__format/format_string.h
+++ b/libcxx/include/__cxx03/__format/format_string.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_STRING_H
-#define _LIBCPP___FORMAT_FORMAT_STRING_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_STRING_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_STRING_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -157,4 +157,4 @@ __parse_arg_id(_Iterator __begin, _Iterator __end, auto& __parse_ctx) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_STRING_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_STRING_H
diff --git a/libcxx/include/__cxx03/__format/format_to_n_result.h b/libcxx/include/__cxx03/__format/format_to_n_result.h
index 8797432922eb9..65e7031f57648 100644
--- a/libcxx/include/__cxx03/__format/format_to_n_result.h
+++ b/libcxx/include/__cxx03/__format/format_to_n_result.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
-#define _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMAT_TO_N_RESULT_H
+#define _LIBCPP___CXX03___FORMAT_FORMAT_TO_N_RESULT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/incrementable_traits.h>
@@ -32,4 +32,4 @@ _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(format_to_n_result);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_TO_N_RESULT_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMAT_TO_N_RESULT_H
diff --git a/libcxx/include/__cxx03/__format/formatter.h b/libcxx/include/__cxx03/__format/formatter.h
index 98457d0a2c65e..9afc9dfce3cb5 100644
--- a/libcxx/include/__cxx03/__format/formatter.h
+++ b/libcxx/include/__cxx03/__format/formatter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_H
-#define _LIBCPP___FORMAT_FORMATTER_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/format.h>
@@ -50,4 +50,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __set_debug_format(_Tp& __formatter) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_bool.h b/libcxx/include/__cxx03/__format/formatter_bool.h
index dd8e97bfc7f48..12a29c2c56040 100644
--- a/libcxx/include/__cxx03/__format/formatter_bool.h
+++ b/libcxx/include/__cxx03/__format/formatter_bool.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_BOOL_H
-#define _LIBCPP___FORMAT_FORMATTER_BOOL_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_BOOL_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_BOOL_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__assert>
@@ -73,4 +73,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<bool, _CharT> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_BOOL_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_BOOL_H
diff --git a/libcxx/include/__cxx03/__format/formatter_char.h b/libcxx/include/__cxx03/__format/formatter_char.h
index b743433c2891d..f0a7d153ea343 100644
--- a/libcxx/include/__cxx03/__format/formatter_char.h
+++ b/libcxx/include/__cxx03/__format/formatter_char.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_CHAR_H
-#define _LIBCPP___FORMAT_FORMATTER_CHAR_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_CHAR_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_CHAR_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -90,4 +90,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<wchar_t, wchar_t> : public __formatter_cha
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_CHAR_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_CHAR_H
diff --git a/libcxx/include/__cxx03/__format/formatter_floating_point.h b/libcxx/include/__cxx03/__format/formatter_floating_point.h
index 36663fb90716b..9f7eec0addcf2 100644
--- a/libcxx/include/__cxx03/__format/formatter_floating_point.h
+++ b/libcxx/include/__cxx03/__format/formatter_floating_point.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
-#define _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_FLOATING_POINT_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_FLOATING_POINT_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__algorithm/find.h>
@@ -780,4 +780,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMATTER_FLOATING_POINT_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__format/formatter_integer.h b/libcxx/include/__cxx03/__format/formatter_integer.h
index 0d4dd15de4f9d..365b44093ede2 100644
--- a/libcxx/include/__cxx03/__format/formatter_integer.h
+++ b/libcxx/include/__cxx03/__format/formatter_integer.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_INTEGER_H
-#define _LIBCPP___FORMAT_FORMATTER_INTEGER_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGER_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGER_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -92,4 +92,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<__uint128_t, _CharT> : public __formatter_
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_INTEGER_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_integral.h b/libcxx/include/__cxx03/__format/formatter_integral.h
index 6bab831244498..fc846549dabdc 100644
--- a/libcxx/include/__cxx03/__format/formatter_integral.h
+++ b/libcxx/include/__cxx03/__format/formatter_integral.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
-#define _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGRAL_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGRAL_H
 
 #include <__cxx03/__charconv/to_chars_integral.h>
 #include <__cxx03/__charconv/to_chars_result.h>
@@ -442,4 +442,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMATTER_INTEGRAL_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__format/formatter_output.h b/libcxx/include/__cxx03/__format/formatter_output.h
index edf6909de750a..1f3ab7f2ff692 100644
--- a/libcxx/include/__cxx03/__format/formatter_output.h
+++ b/libcxx/include/__cxx03/__format/formatter_output.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
-#define _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_OUTPUT_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_OUTPUT_H
 
 #include <__cxx03/__algorithm/ranges_copy.h>
 #include <__cxx03/__algorithm/ranges_fill_n.h>
@@ -332,4 +332,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_FORMATTER_OUTPUT_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_OUTPUT_H
diff --git a/libcxx/include/__cxx03/__format/formatter_pointer.h b/libcxx/include/__cxx03/__format/formatter_pointer.h
index 166a2e93b622f..906d62a83e33b 100644
--- a/libcxx/include/__cxx03/__format/formatter_pointer.h
+++ b/libcxx/include/__cxx03/__format/formatter_pointer.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_POINTER_H
-#define _LIBCPP___FORMAT_FORMATTER_POINTER_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_POINTER_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_POINTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__format/concepts.h>
@@ -69,4 +69,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<const void*, _CharT> : public __formatter_
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_POINTER_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_POINTER_H
diff --git a/libcxx/include/__cxx03/__format/formatter_string.h b/libcxx/include/__cxx03/__format/formatter_string.h
index 9cc862a4545c7..024f7936875ec 100644
--- a/libcxx/include/__cxx03/__format/formatter_string.h
+++ b/libcxx/include/__cxx03/__format/formatter_string.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_STRING_H
-#define _LIBCPP___FORMAT_FORMATTER_STRING_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_STRING_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_STRING_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__format/concepts.h>
@@ -147,4 +147,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<basic_string_view<_CharT, _Traits>, _CharT
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_STRING_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_STRING_H
diff --git a/libcxx/include/__cxx03/__format/formatter_tuple.h b/libcxx/include/__cxx03/__format/formatter_tuple.h
index 911552f663505..0a5ce4e070a64 100644
--- a/libcxx/include/__cxx03/__format/formatter_tuple.h
+++ b/libcxx/include/__cxx03/__format/formatter_tuple.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMATTER_TUPLE_H
-#define _LIBCPP___FORMAT_FORMATTER_TUPLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_FORMATTER_TUPLE_H
+#define _LIBCPP___CXX03___FORMAT_FORMATTER_TUPLE_H
 
 #include <__cxx03/__algorithm/ranges_copy.h>
 #include <__cxx03/__chrono/statically_widen.h>
@@ -147,4 +147,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<tuple<_Args...>, _CharT>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMATTER_TUPLE_H
+#endif // _LIBCPP___CXX03___FORMAT_FORMATTER_TUPLE_H
diff --git a/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h b/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
index b02c2c324573c..37fbde2998736 100644
--- a/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
+++ b/libcxx/include/__cxx03/__format/indic_conjunct_break_table.h
@@ -58,8 +58,8 @@
 // use or other dealings in these Data Files or Software without prior
 // written authorization of the copyright holder.
 
-#ifndef _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
-#define _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
+#define _LIBCPP___CXX03___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
 
 #include <__cxx03/__algorithm/ranges_upper_bound.h>
 #include <__cxx03/__config>
@@ -347,4 +347,4 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
+#endif // _LIBCPP___CXX03___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/parser_std_format_spec.h b/libcxx/include/__cxx03/__format/parser_std_format_spec.h
index 1afb5b2c5dd14..394b6e655e726 100644
--- a/libcxx/include/__cxx03/__format/parser_std_format_spec.h
+++ b/libcxx/include/__cxx03/__format/parser_std_format_spec.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
-#define _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
+#ifndef _LIBCPP___CXX03___FORMAT_PARSER_STD_FORMAT_SPEC_H
+#define _LIBCPP___CXX03___FORMAT_PARSER_STD_FORMAT_SPEC_H
 
 /// \file Contains the std-format-spec parser.
 ///
@@ -1169,4 +1169,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_PARSER_STD_FORMAT_SPEC_H
+#endif // _LIBCPP___CXX03___FORMAT_PARSER_STD_FORMAT_SPEC_H
diff --git a/libcxx/include/__cxx03/__format/range_default_formatter.h b/libcxx/include/__cxx03/__format/range_default_formatter.h
index c0e07870430e8..ca51a0f0a8227 100644
--- a/libcxx/include/__cxx03/__format/range_default_formatter.h
+++ b/libcxx/include/__cxx03/__format/range_default_formatter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
-#define _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
+#ifndef _LIBCPP___CXX03___FORMAT_RANGE_DEFAULT_FORMATTER_H
+#define _LIBCPP___CXX03___FORMAT_RANGE_DEFAULT_FORMATTER_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -211,4 +211,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter<_Rp, _CharT> : __range_default_formatter<f
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_RANGE_DEFAULT_FORMATTER_H
+#endif // _LIBCPP___CXX03___FORMAT_RANGE_DEFAULT_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/range_formatter.h b/libcxx/include/__cxx03/__format/range_formatter.h
index 57bb5ff726ba6..0b6ee4401fc75 100644
--- a/libcxx/include/__cxx03/__format/range_formatter.h
+++ b/libcxx/include/__cxx03/__format/range_formatter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_RANGE_FORMATTER_H
-#define _LIBCPP___FORMAT_RANGE_FORMATTER_H
+#ifndef _LIBCPP___CXX03___FORMAT_RANGE_FORMATTER_H
+#define _LIBCPP___CXX03___FORMAT_RANGE_FORMATTER_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -261,4 +261,4 @@ struct _LIBCPP_TEMPLATE_VIS range_formatter {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_RANGE_FORMATTER_H
+#endif // _LIBCPP___CXX03___FORMAT_RANGE_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__format/unicode.h b/libcxx/include/__cxx03/__format/unicode.h
index 5cf1a8967ab97..e6e9b895df9a7 100644
--- a/libcxx/include/__cxx03/__format/unicode.h
+++ b/libcxx/include/__cxx03/__format/unicode.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_UNICODE_H
-#define _LIBCPP___FORMAT_UNICODE_H
+#ifndef _LIBCPP___CXX03___FORMAT_UNICODE_H
+#define _LIBCPP___CXX03___FORMAT_UNICODE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__bit/countl.h>
@@ -599,4 +599,4 @@ class __code_point_view {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_UNICODE_H
+#endif // _LIBCPP___CXX03___FORMAT_UNICODE_H
diff --git a/libcxx/include/__cxx03/__format/width_estimation_table.h b/libcxx/include/__cxx03/__format/width_estimation_table.h
index 2f0367d1c8f98..0929c4bbf0f09 100644
--- a/libcxx/include/__cxx03/__format/width_estimation_table.h
+++ b/libcxx/include/__cxx03/__format/width_estimation_table.h
@@ -58,8 +58,8 @@
 // use or other dealings in these Data Files or Software without prior
 // written authorization of the copyright holder.
 
-#ifndef _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
-#define _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
+#ifndef _LIBCPP___CXX03___FORMAT_WIDTH_ESTIMATION_TABLE_H
+#define _LIBCPP___CXX03___FORMAT_WIDTH_ESTIMATION_TABLE_H
 
 #include <__cxx03/__algorithm/ranges_upper_bound.h>
 #include <__cxx03/__config>
@@ -267,4 +267,4 @@ inline constexpr uint32_t __table_upper_bound = 0x0003fffd;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
+#endif // _LIBCPP___CXX03___FORMAT_WIDTH_ESTIMATION_TABLE_H
diff --git a/libcxx/include/__cxx03/__format/write_escaped.h b/libcxx/include/__cxx03/__format/write_escaped.h
index 82e65ca988ebf..5711e7c539d02 100644
--- a/libcxx/include/__cxx03/__format/write_escaped.h
+++ b/libcxx/include/__cxx03/__format/write_escaped.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_WRITE_ESCAPED_H
-#define _LIBCPP___FORMAT_WRITE_ESCAPED_H
+#ifndef _LIBCPP___CXX03___FORMAT_WRITE_ESCAPED_H
+#define _LIBCPP___CXX03___FORMAT_WRITE_ESCAPED_H
 
 #include <__cxx03/__algorithm/ranges_copy.h>
 #include <__cxx03/__algorithm/ranges_for_each.h>
@@ -239,4 +239,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FORMAT_WRITE_ESCAPED_H
+#endif // _LIBCPP___CXX03___FORMAT_WRITE_ESCAPED_H
diff --git a/libcxx/include/__cxx03/__functional/binary_function.h b/libcxx/include/__cxx03/__functional/binary_function.h
index 0e54bc5e3c611..cc247eacb8e37 100644
--- a/libcxx/include/__cxx03/__functional/binary_function.h
+++ b/libcxx/include/__cxx03/__functional/binary_function.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
-#define _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BINARY_FUNCTION_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BINARY_FUNCTION_H
 
 #include <__cxx03/__config>
 
@@ -51,4 +51,4 @@ using __binary_function = __binary_function_keep_layout_base<_Arg1, _Arg2, _Resu
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BINARY_FUNCTION_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BINARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/binary_negate.h b/libcxx/include/__cxx03/__functional/binary_negate.h
index 60dfb19918656..e2173d72dc404 100644
--- a/libcxx/include/__cxx03/__functional/binary_negate.h
+++ b/libcxx/include/__cxx03/__functional/binary_negate.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
-#define _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BINARY_NEGATE_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BINARY_NEGATE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -48,4 +48,4 @@ not2(const _Predicate& __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BINARY_NEGATE_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BINARY_NEGATE_H
diff --git a/libcxx/include/__cxx03/__functional/bind.h b/libcxx/include/__cxx03/__functional/bind.h
index 5a1a76a6f1f7a..70c4be05ce65e 100644
--- a/libcxx/include/__cxx03/__functional/bind.h
+++ b/libcxx/include/__cxx03/__functional/bind.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BIND_H
-#define _LIBCPP___FUNCTIONAL_BIND_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BIND_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BIND_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -293,4 +293,4 @@ bind(_Fp&& __f, _BoundArgs&&... __bound_args) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BIND_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BIND_H
diff --git a/libcxx/include/__cxx03/__functional/bind_back.h b/libcxx/include/__cxx03/__functional/bind_back.h
index fef99f1852377..2e333b1b14d6a 100644
--- a/libcxx/include/__cxx03/__functional/bind_back.h
+++ b/libcxx/include/__cxx03/__functional/bind_back.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BIND_BACK_H
-#define _LIBCPP___FUNCTIONAL_BIND_BACK_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BIND_BACK_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BIND_BACK_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -80,4 +80,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto bind_back(_Fn&& __f, _Args&&... __args) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BIND_BACK_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BIND_BACK_H
diff --git a/libcxx/include/__cxx03/__functional/bind_front.h b/libcxx/include/__cxx03/__functional/bind_front.h
index 1156f53be096e..6447c2d1bc2eb 100644
--- a/libcxx/include/__cxx03/__functional/bind_front.h
+++ b/libcxx/include/__cxx03/__functional/bind_front.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BIND_FRONT_H
-#define _LIBCPP___FUNCTIONAL_BIND_FRONT_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BIND_FRONT_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BIND_FRONT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -51,4 +51,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto bind_front(_Fn&& __f, _Args&&... __args) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BIND_FRONT_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BIND_FRONT_H
diff --git a/libcxx/include/__cxx03/__functional/binder1st.h b/libcxx/include/__cxx03/__functional/binder1st.h
index 1f2f7ebb9c853..21c136ff16d8c 100644
--- a/libcxx/include/__cxx03/__functional/binder1st.h
+++ b/libcxx/include/__cxx03/__functional/binder1st.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BINDER1ST_H
-#define _LIBCPP___FUNCTIONAL_BINDER1ST_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BINDER1ST_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BINDER1ST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/unary_function.h>
@@ -51,4 +51,4 @@ bind1st(const _Operation& __op, const _Tp& __x) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BINDER1ST_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BINDER1ST_H
diff --git a/libcxx/include/__cxx03/__functional/binder2nd.h b/libcxx/include/__cxx03/__functional/binder2nd.h
index d70fab8c93b4a..397215b3d9e11 100644
--- a/libcxx/include/__cxx03/__functional/binder2nd.h
+++ b/libcxx/include/__cxx03/__functional/binder2nd.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BINDER2ND_H
-#define _LIBCPP___FUNCTIONAL_BINDER2ND_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BINDER2ND_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BINDER2ND_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/unary_function.h>
@@ -51,4 +51,4 @@ bind2nd(const _Operation& __op, const _Tp& __x) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_BINDER2ND_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BINDER2ND_H
diff --git a/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h b/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
index e7724ce6f44c0..384c4b53cc270 100644
--- a/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
+++ b/libcxx/include/__cxx03/__functional/boyer_moore_searcher.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
-#define _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
+#define _LIBCPP___CXX03___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -303,4 +303,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_BOYER_MOORE_SEARCHER_H
diff --git a/libcxx/include/__cxx03/__functional/compose.h b/libcxx/include/__cxx03/__functional/compose.h
index 504a7e1ca1426..afb6358ad535e 100644
--- a/libcxx/include/__cxx03/__functional/compose.h
+++ b/libcxx/include/__cxx03/__functional/compose.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_COMPOSE_H
-#define _LIBCPP___FUNCTIONAL_COMPOSE_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_COMPOSE_H
+#define _LIBCPP___CXX03___FUNCTIONAL_COMPOSE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -50,4 +50,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto __compose(_Fn1&& __f1, _Fn2&& __f2) noexcep
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_COMPOSE_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_COMPOSE_H
diff --git a/libcxx/include/__cxx03/__functional/default_searcher.h b/libcxx/include/__cxx03/__functional/default_searcher.h
index 90ee48fc5e93b..f1ad66990c257 100644
--- a/libcxx/include/__cxx03/__functional/default_searcher.h
+++ b/libcxx/include/__cxx03/__functional/default_searcher.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
-#define _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_DEFAULT_SEARCHER_H
+#define _LIBCPP___CXX03___FUNCTIONAL_DEFAULT_SEARCHER_H
 
 #include <__cxx03/__algorithm/search.h>
 #include <__cxx03/__config>
@@ -51,4 +51,4 @@ _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(default_searcher);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_DEFAULT_SEARCHER_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_DEFAULT_SEARCHER_H
diff --git a/libcxx/include/__cxx03/__functional/function.h b/libcxx/include/__cxx03/__functional/function.h
index fe3cc8accac63..1d60391494da8 100644
--- a/libcxx/include/__cxx03/__functional/function.h
+++ b/libcxx/include/__cxx03/__functional/function.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_FUNCTION_H
-#define _LIBCPP___FUNCTIONAL_FUNCTION_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_FUNCTION_H
+#define _LIBCPP___CXX03___FUNCTIONAL_FUNCTION_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -1045,4 +1045,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FUNCTIONAL_FUNCTION_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/hash.h b/libcxx/include/__cxx03/__functional/hash.h
index 216d2ae4d1bc5..15473a8572435 100644
--- a/libcxx/include/__cxx03/__functional/hash.h
+++ b/libcxx/include/__cxx03/__functional/hash.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_HASH_H
-#define _LIBCPP___FUNCTIONAL_HASH_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_HASH_H
+#define _LIBCPP___CXX03___FUNCTIONAL_HASH_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/unary_function.h>
@@ -539,4 +539,4 @@ using __enable_hash_helper _LIBCPP_NODEBUG = _Type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_HASH_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_HASH_H
diff --git a/libcxx/include/__cxx03/__functional/identity.h b/libcxx/include/__cxx03/__functional/identity.h
index 7d016af84f854..61e5ebdb31e6e 100644
--- a/libcxx/include/__cxx03/__functional/identity.h
+++ b/libcxx/include/__cxx03/__functional/identity.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_IDENTITY_H
-#define _LIBCPP___FUNCTIONAL_IDENTITY_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_IDENTITY_H
+#define _LIBCPP___CXX03___FUNCTIONAL_IDENTITY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/functional.h>
@@ -62,4 +62,4 @@ struct __is_identity<reference_wrapper<const identity> > : true_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_IDENTITY_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_IDENTITY_H
diff --git a/libcxx/include/__cxx03/__functional/invoke.h b/libcxx/include/__cxx03/__functional/invoke.h
index a9c039bc0379b..d17a2d58e471f 100644
--- a/libcxx/include/__cxx03/__functional/invoke.h
+++ b/libcxx/include/__cxx03/__functional/invoke.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_INVOKE_H
-#define _LIBCPP___FUNCTIONAL_INVOKE_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_INVOKE_H
+#define _LIBCPP___CXX03___FUNCTIONAL_INVOKE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/invoke.h>
@@ -51,4 +51,4 @@ invoke_r(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_r_v<_Result
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_INVOKE_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_INVOKE_H
diff --git a/libcxx/include/__cxx03/__functional/is_transparent.h b/libcxx/include/__cxx03/__functional/is_transparent.h
index de978871404b8..b630aca633672 100644
--- a/libcxx/include/__cxx03/__functional/is_transparent.h
+++ b/libcxx/include/__cxx03/__functional/is_transparent.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
-#define _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_IS_TRANSPARENT
+#define _LIBCPP___CXX03___FUNCTIONAL_IS_TRANSPARENT
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/void_t.h>
@@ -31,4 +31,4 @@ inline const bool __is_transparent_v<_Tp, _Up, __void_t<typename _Tp::is_transpa
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
+#endif // _LIBCPP___CXX03___FUNCTIONAL_IS_TRANSPARENT
diff --git a/libcxx/include/__cxx03/__functional/mem_fn.h b/libcxx/include/__cxx03/__functional/mem_fn.h
index 31bf5a86bffdc..b3257f8302f1b 100644
--- a/libcxx/include/__cxx03/__functional/mem_fn.h
+++ b/libcxx/include/__cxx03/__functional/mem_fn.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_MEM_FN_H
-#define _LIBCPP___FUNCTIONAL_MEM_FN_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_MEM_FN_H
+#define _LIBCPP___CXX03___FUNCTIONAL_MEM_FN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -51,4 +51,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn<_Rp _Tp::*>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_MEM_FN_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_MEM_FN_H
diff --git a/libcxx/include/__cxx03/__functional/mem_fun_ref.h b/libcxx/include/__cxx03/__functional/mem_fun_ref.h
index e0e2f29b95d16..ef962231dc92b 100644
--- a/libcxx/include/__cxx03/__functional/mem_fun_ref.h
+++ b/libcxx/include/__cxx03/__functional/mem_fun_ref.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
-#define _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_MEM_FUN_REF_H
+#define _LIBCPP___CXX03___FUNCTIONAL_MEM_FUN_REF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -143,4 +143,4 @@ mem_fun_ref(_Sp (_Tp::*__f)(_Ap) const) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_MEM_FUN_REF_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_MEM_FUN_REF_H
diff --git a/libcxx/include/__cxx03/__functional/not_fn.h b/libcxx/include/__cxx03/__functional/not_fn.h
index 8b9264af64c58..a6c648bc0beb2 100644
--- a/libcxx/include/__cxx03/__functional/not_fn.h
+++ b/libcxx/include/__cxx03/__functional/not_fn.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_NOT_FN_H
-#define _LIBCPP___FUNCTIONAL_NOT_FN_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_NOT_FN_H
+#define _LIBCPP___CXX03___FUNCTIONAL_NOT_FN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -50,4 +50,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto not_fn(_Fn&& __f) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_NOT_FN_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_NOT_FN_H
diff --git a/libcxx/include/__cxx03/__functional/operations.h b/libcxx/include/__cxx03/__functional/operations.h
index c05579e35e835..ddbe71bdc8b68 100644
--- a/libcxx/include/__cxx03/__functional/operations.h
+++ b/libcxx/include/__cxx03/__functional/operations.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_OPERATIONS_H
-#define _LIBCPP___FUNCTIONAL_OPERATIONS_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_OPERATIONS_H
+#define _LIBCPP___CXX03___FUNCTIONAL_OPERATIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -538,4 +538,4 @@ struct _LIBCPP_TEMPLATE_VIS logical_or<void> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_OPERATIONS_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__functional/perfect_forward.h b/libcxx/include/__cxx03/__functional/perfect_forward.h
index 45ecb68250dc9..230e0dc21b429 100644
--- a/libcxx/include/__cxx03/__functional/perfect_forward.h
+++ b/libcxx/include/__cxx03/__functional/perfect_forward.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
-#define _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_PERFECT_FORWARD_H
+#define _LIBCPP___CXX03___FUNCTIONAL_PERFECT_FORWARD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -101,4 +101,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FUNCTIONAL_PERFECT_FORWARD_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_PERFECT_FORWARD_H
diff --git a/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h b/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
index d033f9050f0c3..72474255d7fe2 100644
--- a/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
+++ b/libcxx/include/__cxx03/__functional/pointer_to_binary_function.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
-#define _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
+#define _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -41,4 +41,4 @@ ptr_fun(_Result (*__f)(_Arg1, _Arg2)) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_BINARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h b/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
index 9e7f62915c91f..8540ad07f3ec0 100644
--- a/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
+++ b/libcxx/include/__cxx03/__functional/pointer_to_unary_function.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
-#define _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
+#define _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/unary_function.h>
@@ -41,4 +41,4 @@ ptr_fun(_Result (*__f)(_Arg)) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_POINTER_TO_UNARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/ranges_operations.h b/libcxx/include/__cxx03/__functional/ranges_operations.h
index e8a2b367c9350..e48593824852d 100644
--- a/libcxx/include/__cxx03/__functional/ranges_operations.h
+++ b/libcxx/include/__cxx03/__functional/ranges_operations.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
-#define _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_RANGES_OPERATIONS_H
+#define _LIBCPP___CXX03___FUNCTIONAL_RANGES_OPERATIONS_H
 
 #include <__cxx03/__concepts/equality_comparable.h>
 #include <__cxx03/__concepts/totally_ordered.h>
@@ -106,4 +106,4 @@ inline const bool __desugars_to_v<__less_tag, ranges::less, _Tp, _Up> = true;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_RANGES_OPERATIONS_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_RANGES_OPERATIONS_H
diff --git a/libcxx/include/__cxx03/__functional/reference_wrapper.h b/libcxx/include/__cxx03/__functional/reference_wrapper.h
index 73788e1fdb89b..e217bdf543351 100644
--- a/libcxx/include/__cxx03/__functional/reference_wrapper.h
+++ b/libcxx/include/__cxx03/__functional/reference_wrapper.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
-#define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_REFERENCE_WRAPPER_H
+#define _LIBCPP___CXX03___FUNCTIONAL_REFERENCE_WRAPPER_H
 
 #include <__cxx03/__compare/synth_three_way.h>
 #include <__cxx03/__concepts/boolean_testable.h>
@@ -151,4 +151,4 @@ void cref(const _Tp&&) = delete;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_REFERENCE_WRAPPER_H
diff --git a/libcxx/include/__cxx03/__functional/unary_function.h b/libcxx/include/__cxx03/__functional/unary_function.h
index 9b1e336f15fa4..45ba29ee2cac5 100644
--- a/libcxx/include/__cxx03/__functional/unary_function.h
+++ b/libcxx/include/__cxx03/__functional/unary_function.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
-#define _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_UNARY_FUNCTION_H
+#define _LIBCPP___CXX03___FUNCTIONAL_UNARY_FUNCTION_H
 
 #include <__cxx03/__config>
 
@@ -48,4 +48,4 @@ using __unary_function = __unary_function_keep_layout_base<_Arg, _Result>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_UNARY_FUNCTION_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_UNARY_FUNCTION_H
diff --git a/libcxx/include/__cxx03/__functional/unary_negate.h b/libcxx/include/__cxx03/__functional/unary_negate.h
index 7154d234b007f..4d527865f378c 100644
--- a/libcxx/include/__cxx03/__functional/unary_negate.h
+++ b/libcxx/include/__cxx03/__functional/unary_negate.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
-#define _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_UNARY_NEGATE_H
+#define _LIBCPP___CXX03___FUNCTIONAL_UNARY_NEGATE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/unary_function.h>
@@ -45,4 +45,4 @@ not1(const _Predicate& __pred) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_UNARY_NEGATE_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_UNARY_NEGATE_H
diff --git a/libcxx/include/__cxx03/__functional/weak_result_type.h b/libcxx/include/__cxx03/__functional/weak_result_type.h
index e1d62d634ca33..539989dda2d97 100644
--- a/libcxx/include/__cxx03/__functional/weak_result_type.h
+++ b/libcxx/include/__cxx03/__functional/weak_result_type.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
-#define _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_WEAK_RESULT_TYPE_H
+#define _LIBCPP___CXX03___FUNCTIONAL_WEAK_RESULT_TYPE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/binary_function.h>
@@ -228,4 +228,4 @@ struct __invoke_return {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_WEAK_RESULT_TYPE_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_WEAK_RESULT_TYPE_H
diff --git a/libcxx/include/__cxx03/__fwd/array.h b/libcxx/include/__cxx03/__fwd/array.h
index 36343371a2278..f2648a1986e27 100644
--- a/libcxx/include/__cxx03/__fwd/array.h
+++ b/libcxx/include/__cxx03/__fwd/array.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_ARRAY_H
-#define _LIBCPP___FWD_ARRAY_H
+#ifndef _LIBCPP___CXX03___FWD_ARRAY_H
+#define _LIBCPP___CXX03___FWD_ARRAY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -43,4 +43,4 @@ struct __is_std_array<array<_Tp, _Size> > : true_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_ARRAY_H
+#endif // _LIBCPP___CXX03___FWD_ARRAY_H
diff --git a/libcxx/include/__cxx03/__fwd/bit_reference.h b/libcxx/include/__cxx03/__fwd/bit_reference.h
index 6bda18429c392..e04a47ff97a61 100644
--- a/libcxx/include/__cxx03/__fwd/bit_reference.h
+++ b/libcxx/include/__cxx03/__fwd/bit_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_BIT_REFERENCE_H
-#define _LIBCPP___FWD_BIT_REFERENCE_H
+#ifndef _LIBCPP___CXX03___FWD_BIT_REFERENCE_H
+#define _LIBCPP___CXX03___FWD_BIT_REFERENCE_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@ class __bit_iterator;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_BIT_REFERENCE_H
+#endif // _LIBCPP___CXX03___FWD_BIT_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__fwd/complex.h b/libcxx/include/__cxx03/__fwd/complex.h
index e734f2db76fed..bb1ae20ab3bf8 100644
--- a/libcxx/include/__cxx03/__fwd/complex.h
+++ b/libcxx/include/__cxx03/__fwd/complex.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_COMPLEX_H
-#define _LIBCPP___FWD_COMPLEX_H
+#ifndef _LIBCPP___CXX03___FWD_COMPLEX_H
+#define _LIBCPP___CXX03___FWD_COMPLEX_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -39,4 +39,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const complex<_Tp>&&) noexcept;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_COMPLEX_H
+#endif // _LIBCPP___CXX03___FWD_COMPLEX_H
diff --git a/libcxx/include/__cxx03/__fwd/deque.h b/libcxx/include/__cxx03/__fwd/deque.h
index 31aa8c3799823..7d1c6f79d0d77 100644
--- a/libcxx/include/__cxx03/__fwd/deque.h
+++ b/libcxx/include/__cxx03/__fwd/deque.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_DEQUE_H
-#define _LIBCPP___FWD_DEQUE_H
+#ifndef _LIBCPP___CXX03___FWD_DEQUE_H
+#define _LIBCPP___CXX03___FWD_DEQUE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/memory.h>
@@ -23,4 +23,4 @@ class _LIBCPP_TEMPLATE_VIS deque;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_DEQUE_H
+#endif // _LIBCPP___CXX03___FWD_DEQUE_H
diff --git a/libcxx/include/__cxx03/__fwd/format.h b/libcxx/include/__cxx03/__fwd/format.h
index 2777433363270..3da3fd728266b 100644
--- a/libcxx/include/__cxx03/__fwd/format.h
+++ b/libcxx/include/__cxx03/__fwd/format.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_FORMAT_H
-#define _LIBCPP___FWD_FORMAT_H
+#ifndef _LIBCPP___CXX03___FWD_FORMAT_H
+#define _LIBCPP___CXX03___FWD_FORMAT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -35,4 +35,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_FORMAT_H
+#endif // _LIBCPP___CXX03___FWD_FORMAT_H
diff --git a/libcxx/include/__cxx03/__fwd/fstream.h b/libcxx/include/__cxx03/__fwd/fstream.h
index 717cb8c583d2c..8cc5d2b9efd4e 100644
--- a/libcxx/include/__cxx03/__fwd/fstream.h
+++ b/libcxx/include/__cxx03/__fwd/fstream.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_FSTREAM_H
-#define _LIBCPP___FWD_FSTREAM_H
+#ifndef _LIBCPP___CXX03___FWD_FSTREAM_H
+#define _LIBCPP___CXX03___FWD_FSTREAM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -50,4 +50,4 @@ class _LIBCPP_PREFERRED_NAME(fstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERR
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_FSTREAM_H
+#endif // _LIBCPP___CXX03___FWD_FSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/functional.h b/libcxx/include/__cxx03/__fwd/functional.h
index e19b802d178ca..924874ca70050 100644
--- a/libcxx/include/__cxx03/__fwd/functional.h
+++ b/libcxx/include/__cxx03/__fwd/functional.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_FUNCTIONAL_H
-#define _LIBCPP___FWD_FUNCTIONAL_H
+#ifndef _LIBCPP___CXX03___FWD_FUNCTIONAL_H
+#define _LIBCPP___CXX03___FWD_FUNCTIONAL_H
 
 #include <__cxx03/__config>
 
@@ -25,4 +25,4 @@ class _LIBCPP_TEMPLATE_VIS reference_wrapper;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_FUNCTIONAL_H
+#endif // _LIBCPP___CXX03___FWD_FUNCTIONAL_H
diff --git a/libcxx/include/__cxx03/__fwd/ios.h b/libcxx/include/__cxx03/__fwd/ios.h
index f5ba74c9067bb..dc03e8c6bab2f 100644
--- a/libcxx/include/__cxx03/__fwd/ios.h
+++ b/libcxx/include/__cxx03/__fwd/ios.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_IOS_H
-#define _LIBCPP___FWD_IOS_H
+#ifndef _LIBCPP___CXX03___FWD_IOS_H
+#define _LIBCPP___CXX03___FWD_IOS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -40,4 +40,4 @@ using streamoff = long long; // for char_traits in <string>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_IOS_H
+#endif // _LIBCPP___CXX03___FWD_IOS_H
diff --git a/libcxx/include/__cxx03/__fwd/istream.h b/libcxx/include/__cxx03/__fwd/istream.h
index 8975ac26baf6b..f3edc7859c642 100644
--- a/libcxx/include/__cxx03/__fwd/istream.h
+++ b/libcxx/include/__cxx03/__fwd/istream.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_ISTREAM_H
-#define _LIBCPP___FWD_ISTREAM_H
+#ifndef _LIBCPP___CXX03___FWD_ISTREAM_H
+#define _LIBCPP___CXX03___FWD_ISTREAM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -40,4 +40,4 @@ class _LIBCPP_PREFERRED_NAME(iostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFER
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_ISTREAM_H
+#endif // _LIBCPP___CXX03___FWD_ISTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/mdspan.h b/libcxx/include/__cxx03/__fwd/mdspan.h
index 369da43a20296..50209798a1443 100644
--- a/libcxx/include/__cxx03/__fwd/mdspan.h
+++ b/libcxx/include/__cxx03/__fwd/mdspan.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_LAYOUTS_H
-#define _LIBCPP___MDSPAN_LAYOUTS_H
+#ifndef _LIBCPP___CXX03___MDSPAN_LAYOUTS_H
+#define _LIBCPP___CXX03___MDSPAN_LAYOUTS_H
 
 #include <__cxx03/__config>
 
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_LAYOUTS_H
+#endif // _LIBCPP___CXX03___MDSPAN_LAYOUTS_H
diff --git a/libcxx/include/__cxx03/__fwd/memory.h b/libcxx/include/__cxx03/__fwd/memory.h
index 109c0f6c051cc..df76b1747c20a 100644
--- a/libcxx/include/__cxx03/__fwd/memory.h
+++ b/libcxx/include/__cxx03/__fwd/memory.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_MEMORY_H
-#define _LIBCPP___FWD_MEMORY_H
+#ifndef _LIBCPP___CXX03___FWD_MEMORY_H
+#define _LIBCPP___CXX03___FWD_MEMORY_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@ class _LIBCPP_TEMPLATE_VIS allocator;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_MEMORY_H
+#endif // _LIBCPP___CXX03___FWD_MEMORY_H
diff --git a/libcxx/include/__cxx03/__fwd/memory_resource.h b/libcxx/include/__cxx03/__fwd/memory_resource.h
index 1a12dcfdaf627..aefa00a12a835 100644
--- a/libcxx/include/__cxx03/__fwd/memory_resource.h
+++ b/libcxx/include/__cxx03/__fwd/memory_resource.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_MEMORY_RESOURCE_H
-#define _LIBCPP___FWD_MEMORY_RESOURCE_H
+#ifndef _LIBCPP___CXX03___FWD_MEMORY_RESOURCE_H
+#define _LIBCPP___CXX03___FWD_MEMORY_RESOURCE_H
 
 #include <__cxx03/__config>
 
@@ -24,4 +24,4 @@ class _LIBCPP_AVAILABILITY_PMR _LIBCPP_TEMPLATE_VIS polymorphic_allocator;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_MEMORY_RESOURCE_H
+#endif // _LIBCPP___CXX03___FWD_MEMORY_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__fwd/ostream.h b/libcxx/include/__cxx03/__fwd/ostream.h
index 5660ac50aac46..b5061e6d3bc6d 100644
--- a/libcxx/include/__cxx03/__fwd/ostream.h
+++ b/libcxx/include/__cxx03/__fwd/ostream.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_OSTREAM_H
-#define _LIBCPP___FWD_OSTREAM_H
+#ifndef _LIBCPP___CXX03___FWD_OSTREAM_H
+#define _LIBCPP___CXX03___FWD_OSTREAM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -32,4 +32,4 @@ class _LIBCPP_PREFERRED_NAME(ostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERR
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_OSTREAM_H
+#endif // _LIBCPP___CXX03___FWD_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/pair.h b/libcxx/include/__cxx03/__fwd/pair.h
index 36656b454e0ab..092374b509606 100644
--- a/libcxx/include/__cxx03/__fwd/pair.h
+++ b/libcxx/include/__cxx03/__fwd/pair.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_PAIR_H
-#define _LIBCPP___FWD_PAIR_H
+#ifndef _LIBCPP___CXX03___FWD_PAIR_H
+#define _LIBCPP___CXX03___FWD_PAIR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/tuple.h>
@@ -42,4 +42,4 @@ get(const pair<_T1, _T2>&&) _NOEXCEPT;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_PAIR_H
+#endif // _LIBCPP___CXX03___FWD_PAIR_H
diff --git a/libcxx/include/__cxx03/__fwd/queue.h b/libcxx/include/__cxx03/__fwd/queue.h
index 54afd5113ba89..c2f68089d0e33 100644
--- a/libcxx/include/__cxx03/__fwd/queue.h
+++ b/libcxx/include/__cxx03/__fwd/queue.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_QUEUE_H
-#define _LIBCPP___FWD_QUEUE_H
+#ifndef _LIBCPP___CXX03___FWD_QUEUE_H
+#define _LIBCPP___CXX03___FWD_QUEUE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/operations.h>
@@ -28,4 +28,4 @@ class _LIBCPP_TEMPLATE_VIS priority_queue;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_QUEUE_H
+#endif // _LIBCPP___CXX03___FWD_QUEUE_H
diff --git a/libcxx/include/__cxx03/__fwd/span.h b/libcxx/include/__cxx03/__fwd/span.h
index 47ed3eca5af4e..415a3c5ba8a55 100644
--- a/libcxx/include/__cxx03/__fwd/span.h
+++ b/libcxx/include/__cxx03/__fwd/span.h
@@ -7,8 +7,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_SPAN_H
-#define _LIBCPP___FWD_SPAN_H
+#ifndef _LIBCPP___CXX03___FWD_SPAN_H
+#define _LIBCPP___CXX03___FWD_SPAN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -35,4 +35,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___FWD_SPAN_H
+#endif // _LIBCPP___CXX03___FWD_SPAN_H
diff --git a/libcxx/include/__cxx03/__fwd/sstream.h b/libcxx/include/__cxx03/__fwd/sstream.h
index cbb53942c8212..32ae42cd7b020 100644
--- a/libcxx/include/__cxx03/__fwd/sstream.h
+++ b/libcxx/include/__cxx03/__fwd/sstream.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_SSTREAM_H
-#define _LIBCPP___FWD_SSTREAM_H
+#ifndef _LIBCPP___CXX03___FWD_SSTREAM_H
+#define _LIBCPP___CXX03___FWD_SSTREAM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/memory.h>
@@ -55,4 +55,4 @@ class _LIBCPP_PREFERRED_NAME(stringstream)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_SSTREAM_H
+#endif // _LIBCPP___CXX03___FWD_SSTREAM_H
diff --git a/libcxx/include/__cxx03/__fwd/stack.h b/libcxx/include/__cxx03/__fwd/stack.h
index ddd7a67615876..715356e91b411 100644
--- a/libcxx/include/__cxx03/__fwd/stack.h
+++ b/libcxx/include/__cxx03/__fwd/stack.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_STACK_H
-#define _LIBCPP___FWD_STACK_H
+#ifndef _LIBCPP___CXX03___FWD_STACK_H
+#define _LIBCPP___CXX03___FWD_STACK_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/deque.h>
@@ -23,4 +23,4 @@ class _LIBCPP_TEMPLATE_VIS stack;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_STACK_H
+#endif // _LIBCPP___CXX03___FWD_STACK_H
diff --git a/libcxx/include/__cxx03/__fwd/streambuf.h b/libcxx/include/__cxx03/__fwd/streambuf.h
index 0b448836aaef7..d22be50ce8ad1 100644
--- a/libcxx/include/__cxx03/__fwd/streambuf.h
+++ b/libcxx/include/__cxx03/__fwd/streambuf.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_STREAMBUF_H
-#define _LIBCPP___FWD_STREAMBUF_H
+#ifndef _LIBCPP___CXX03___FWD_STREAMBUF_H
+#define _LIBCPP___CXX03___FWD_STREAMBUF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -32,4 +32,4 @@ class _LIBCPP_PREFERRED_NAME(streambuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFE
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_STREAMBUF_H
+#endif // _LIBCPP___CXX03___FWD_STREAMBUF_H
diff --git a/libcxx/include/__cxx03/__fwd/string.h b/libcxx/include/__cxx03/__fwd/string.h
index 07042b205e8ce..21c2584c6e95b 100644
--- a/libcxx/include/__cxx03/__fwd/string.h
+++ b/libcxx/include/__cxx03/__fwd/string.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_STRING_H
-#define _LIBCPP___FWD_STRING_H
+#ifndef _LIBCPP___CXX03___FWD_STRING_H
+#define _LIBCPP___CXX03___FWD_STRING_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/memory.h>
@@ -104,4 +104,4 @@ class _LIBCPP_PREFERRED_NAME(string)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_STRING_H
+#endif // _LIBCPP___CXX03___FWD_STRING_H
diff --git a/libcxx/include/__cxx03/__fwd/string_view.h b/libcxx/include/__cxx03/__fwd/string_view.h
index 25ee230d20b33..76c608b46b1c6 100644
--- a/libcxx/include/__cxx03/__fwd/string_view.h
+++ b/libcxx/include/__cxx03/__fwd/string_view.h
@@ -7,8 +7,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_STRING_VIEW_H
-#define _LIBCPP___FWD_STRING_VIEW_H
+#ifndef _LIBCPP___CXX03___FWD_STRING_VIEW_H
+#define _LIBCPP___CXX03___FWD_STRING_VIEW_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/string.h>
@@ -47,4 +47,4 @@ class _LIBCPP_PREFERRED_NAME(string_view)
 // clang-format on
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_STRING_VIEW_H
+#endif // _LIBCPP___CXX03___FWD_STRING_VIEW_H
diff --git a/libcxx/include/__cxx03/__fwd/subrange.h b/libcxx/include/__cxx03/__fwd/subrange.h
index aebbd866daeb2..34ba3504d0499 100644
--- a/libcxx/include/__cxx03/__fwd/subrange.h
+++ b/libcxx/include/__cxx03/__fwd/subrange.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_SUBRANGE_H
-#define _LIBCPP___FWD_SUBRANGE_H
+#ifndef _LIBCPP___CXX03___FWD_SUBRANGE_H
+#define _LIBCPP___CXX03___FWD_SUBRANGE_H
 
 #include <__cxx03/__concepts/copyable.h>
 #include <__cxx03/__config>
@@ -46,4 +46,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___FWD_SUBRANGE_H
+#endif // _LIBCPP___CXX03___FWD_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__fwd/tuple.h b/libcxx/include/__cxx03/__fwd/tuple.h
index 75f7e6d16aecf..fed99583f199f 100644
--- a/libcxx/include/__cxx03/__fwd/tuple.h
+++ b/libcxx/include/__cxx03/__fwd/tuple.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_TUPLE_H
-#define _LIBCPP___FWD_TUPLE_H
+#ifndef _LIBCPP___CXX03___FWD_TUPLE_H
+#define _LIBCPP___CXX03___FWD_TUPLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -49,4 +49,4 @@ get(const tuple<_Tp...>&&) _NOEXCEPT;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_TUPLE_H
+#endif // _LIBCPP___CXX03___FWD_TUPLE_H
diff --git a/libcxx/include/__cxx03/__fwd/vector.h b/libcxx/include/__cxx03/__fwd/vector.h
index 3ed34e9f9ef94..4133173d765c4 100644
--- a/libcxx/include/__cxx03/__fwd/vector.h
+++ b/libcxx/include/__cxx03/__fwd/vector.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FWD_VECTOR_H
-#define _LIBCPP___FWD_VECTOR_H
+#ifndef _LIBCPP___CXX03___FWD_VECTOR_H
+#define _LIBCPP___CXX03___FWD_VECTOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/memory.h>
@@ -23,4 +23,4 @@ class _LIBCPP_TEMPLATE_VIS vector;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FWD_VECTOR_H
+#endif // _LIBCPP___CXX03___FWD_VECTOR_H
diff --git a/libcxx/include/__cxx03/__hash_table b/libcxx/include/__cxx03/__hash_table
index 348dcaf01e8c1..4a27681442c27 100644
--- a/libcxx/include/__cxx03/__hash_table
+++ b/libcxx/include/__cxx03/__hash_table
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___HASH_TABLE
-#define _LIBCPP___HASH_TABLE
+#ifndef _LIBCPP___CXX03___HASH_TABLE
+#define _LIBCPP___CXX03___HASH_TABLE
 
 #include <__cxx03/__algorithm/max.h>
 #include <__cxx03/__algorithm/min.h>
@@ -2041,4 +2041,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___HASH_TABLE
+#endif // _LIBCPP___CXX03___HASH_TABLE
diff --git a/libcxx/include/__cxx03/__ios/fpos.h b/libcxx/include/__cxx03/__ios/fpos.h
index 20904d3f5469d..5f16bfd4caec9 100644
--- a/libcxx/include/__cxx03/__ios/fpos.h
+++ b/libcxx/include/__cxx03/__ios/fpos.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___IOS_FPOS_H
-#define _LIBCPP___IOS_FPOS_H
+#ifndef _LIBCPP___CXX03___IOS_FPOS_H
+#define _LIBCPP___CXX03___IOS_FPOS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/ios.h>
@@ -73,4 +73,4 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const fpos<_StateT>& __x, const fpo
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___IOS_FPOS_H
+#endif // _LIBCPP___CXX03___IOS_FPOS_H
diff --git a/libcxx/include/__cxx03/__iterator/access.h b/libcxx/include/__cxx03/__iterator/access.h
index 2d7b4d16d47ce..2d61d4a879d4f 100644
--- a/libcxx/include/__cxx03/__iterator/access.h
+++ b/libcxx/include/__cxx03/__iterator/access.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ACCESS_H
-#define _LIBCPP___ITERATOR_ACCESS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ACCESS_H
+#define _LIBCPP___CXX03___ITERATOR_ACCESS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -92,4 +92,4 @@ _LIBCPP_HIDE_FROM_ABI typename _Cp::const_iterator end(const _Cp& __c) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ACCESS_H
+#endif // _LIBCPP___CXX03___ITERATOR_ACCESS_H
diff --git a/libcxx/include/__cxx03/__iterator/advance.h b/libcxx/include/__cxx03/__iterator/advance.h
index b481652fbed9d..ac4904829b7ee 100644
--- a/libcxx/include/__cxx03/__iterator/advance.h
+++ b/libcxx/include/__cxx03/__iterator/advance.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ADVANCE_H
-#define _LIBCPP___ITERATOR_ADVANCE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ADVANCE_H
+#define _LIBCPP___CXX03___ITERATOR_ADVANCE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/assignable.h>
@@ -202,4 +202,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_ADVANCE_H
+#endif // _LIBCPP___CXX03___ITERATOR_ADVANCE_H
diff --git a/libcxx/include/__cxx03/__iterator/aliasing_iterator.h b/libcxx/include/__cxx03/__iterator/aliasing_iterator.h
index a3cb555606cb9..93da1d28ef094 100644
--- a/libcxx/include/__cxx03/__iterator/aliasing_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/aliasing_iterator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
-#define _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ALIASING_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_ALIASING_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -124,4 +124,4 @@ using __aliasing_iterator = typename __aliasing_iterator_wrapper<_BaseT, _Alias>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_ALIASING_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/back_insert_iterator.h b/libcxx/include/__cxx03/__iterator/back_insert_iterator.h
index 2d3885951d134..962639a2746bf 100644
--- a/libcxx/include/__cxx03/__iterator/back_insert_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/back_insert_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
-#define _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_BACK_INSERT_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_BACK_INSERT_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator.h>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_BACK_INSERT_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_BACK_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/bounded_iter.h b/libcxx/include/__cxx03/__iterator/bounded_iter.h
index dc93be089b1b8..d361e74283de7 100644
--- a/libcxx/include/__cxx03/__iterator/bounded_iter.h
+++ b/libcxx/include/__cxx03/__iterator/bounded_iter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_BOUNDED_ITER_H
-#define _LIBCPP___ITERATOR_BOUNDED_ITER_H
+#ifndef _LIBCPP___CXX03___ITERATOR_BOUNDED_ITER_H
+#define _LIBCPP___CXX03___ITERATOR_BOUNDED_ITER_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__compare/ordering.h>
@@ -280,4 +280,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_BOUNDED_ITER_H
+#endif // _LIBCPP___CXX03___ITERATOR_BOUNDED_ITER_H
diff --git a/libcxx/include/__cxx03/__iterator/common_iterator.h b/libcxx/include/__cxx03/__iterator/common_iterator.h
index 5d052c0d2300e..6083a23e67a86 100644
--- a/libcxx/include/__cxx03/__iterator/common_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/common_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_COMMON_ITERATOR_H
-#define _LIBCPP___ITERATOR_COMMON_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_COMMON_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_COMMON_ITERATOR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/assignable.h>
@@ -296,4 +296,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_COMMON_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_COMMON_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/concepts.h b/libcxx/include/__cxx03/__iterator/concepts.h
index 4973ce8987ef5..af7d0c9a0541d 100644
--- a/libcxx/include/__cxx03/__iterator/concepts.h
+++ b/libcxx/include/__cxx03/__iterator/concepts.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_CONCEPTS_H
-#define _LIBCPP___ITERATOR_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_CONCEPTS_H
+#define _LIBCPP___CXX03___ITERATOR_CONCEPTS_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__concepts/assignable.h>
@@ -254,4 +254,4 @@ using __has_random_access_iterator_category_or_concept
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_CONCEPTS_H
+#endif // _LIBCPP___CXX03___ITERATOR_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__iterator/counted_iterator.h b/libcxx/include/__cxx03/__iterator/counted_iterator.h
index 161ec54a1a3b1..9d6004dc90b84 100644
--- a/libcxx/include/__cxx03/__iterator/counted_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/counted_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
-#define _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_COUNTED_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_COUNTED_ITERATOR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/assignable.h>
@@ -286,4 +286,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_COUNTED_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_COUNTED_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h b/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
index 938884e5f69ab..aed3527aee08a 100644
--- a/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
+++ b/libcxx/include/__cxx03/__iterator/cpp17_iterator_concepts.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
-#define _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
+#define _LIBCPP___CXX03___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
 
 #include <__cxx03/__concepts/boolean_testable.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -187,4 +187,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
+#endif // _LIBCPP___CXX03___ITERATOR_CPP17_ITERATOR_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__iterator/data.h b/libcxx/include/__cxx03/__iterator/data.h
index d4f26608d9c49..58047e4b6c755 100644
--- a/libcxx/include/__cxx03/__iterator/data.h
+++ b/libcxx/include/__cxx03/__iterator/data.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_DATA_H
-#define _LIBCPP___ITERATOR_DATA_H
+#ifndef _LIBCPP___CXX03___ITERATOR_DATA_H
+#define _LIBCPP___CXX03___ITERATOR_DATA_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -46,4 +46,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr const _Ep* data(initializer_list<_Ep> __il) noex
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_DATA_H
+#endif // _LIBCPP___CXX03___ITERATOR_DATA_H
diff --git a/libcxx/include/__cxx03/__iterator/default_sentinel.h b/libcxx/include/__cxx03/__iterator/default_sentinel.h
index fd05aeb59bce6..7e1099a3d11c5 100644
--- a/libcxx/include/__cxx03/__iterator/default_sentinel.h
+++ b/libcxx/include/__cxx03/__iterator/default_sentinel.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
-#define _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
+#ifndef _LIBCPP___CXX03___ITERATOR_DEFAULT_SENTINEL_H
+#define _LIBCPP___CXX03___ITERATOR_DEFAULT_SENTINEL_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ inline constexpr default_sentinel_t default_sentinel{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_DEFAULT_SENTINEL_H
+#endif // _LIBCPP___CXX03___ITERATOR_DEFAULT_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/distance.h b/libcxx/include/__cxx03/__iterator/distance.h
index 20f2771efc437..91dcc32913799 100644
--- a/libcxx/include/__cxx03/__iterator/distance.h
+++ b/libcxx/include/__cxx03/__iterator/distance.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_DISTANCE_H
-#define _LIBCPP___ITERATOR_DISTANCE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_DISTANCE_H
+#define _LIBCPP___CXX03___ITERATOR_DISTANCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -96,4 +96,4 @@ inline constexpr auto distance = __distance::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_DISTANCE_H
+#endif // _LIBCPP___CXX03___ITERATOR_DISTANCE_H
diff --git a/libcxx/include/__cxx03/__iterator/empty.h b/libcxx/include/__cxx03/__iterator/empty.h
index 0bdb951ee12dc..20f505e789c13 100644
--- a/libcxx/include/__cxx03/__iterator/empty.h
+++ b/libcxx/include/__cxx03/__iterator/empty.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_EMPTY_H
-#define _LIBCPP___ITERATOR_EMPTY_H
+#ifndef _LIBCPP___CXX03___ITERATOR_EMPTY_H
+#define _LIBCPP___CXX03___ITERATOR_EMPTY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -42,4 +42,4 @@ template <class _Ep>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_EMPTY_H
+#endif // _LIBCPP___CXX03___ITERATOR_EMPTY_H
diff --git a/libcxx/include/__cxx03/__iterator/erase_if_container.h b/libcxx/include/__cxx03/__iterator/erase_if_container.h
index 11edc9c1312c6..c83eccd133d1f 100644
--- a/libcxx/include/__cxx03/__iterator/erase_if_container.h
+++ b/libcxx/include/__cxx03/__iterator/erase_if_container.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
-#define _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ERASE_IF_CONTAINER_H
+#define _LIBCPP___CXX03___ITERATOR_ERASE_IF_CONTAINER_H
 
 #include <__cxx03/__config>
 
@@ -40,4 +40,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_ERASE_IF_CONTAINER_H
+#endif // _LIBCPP___CXX03___ITERATOR_ERASE_IF_CONTAINER_H
diff --git a/libcxx/include/__cxx03/__iterator/front_insert_iterator.h b/libcxx/include/__cxx03/__iterator/front_insert_iterator.h
index f28b82840f2d6..9c2b987806f5b 100644
--- a/libcxx/include/__cxx03/__iterator/front_insert_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/front_insert_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
-#define _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_FRONT_INSERT_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_FRONT_INSERT_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator.h>
@@ -80,4 +80,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_FRONT_INSERT_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_FRONT_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/incrementable_traits.h b/libcxx/include/__cxx03/__iterator/incrementable_traits.h
index 8eb730bfe8ad9..e861f80ecb4d6 100644
--- a/libcxx/include/__cxx03/__iterator/incrementable_traits.h
+++ b/libcxx/include/__cxx03/__iterator/incrementable_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
-#define _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_INCREMENTABLE_TRAITS_H
+#define _LIBCPP___CXX03___ITERATOR_INCREMENTABLE_TRAITS_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -76,4 +76,4 @@ using iter_difference_t =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_INCREMENTABLE_TRAITS_H
+#endif // _LIBCPP___CXX03___ITERATOR_INCREMENTABLE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/indirectly_comparable.h b/libcxx/include/__cxx03/__iterator/indirectly_comparable.h
index caef64e6f0831..56de2de09eccd 100644
--- a/libcxx/include/__cxx03/__iterator/indirectly_comparable.h
+++ b/libcxx/include/__cxx03/__iterator/indirectly_comparable.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
-#define _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_INDIRECTLY_COMPARABLE_H
+#define _LIBCPP___CXX03___ITERATOR_INDIRECTLY_COMPARABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -30,4 +30,4 @@ concept indirectly_comparable = indirect_binary_predicate<_Rp, projected<_I1, _P
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_INDIRECTLY_COMPARABLE_H
+#endif // _LIBCPP___CXX03___ITERATOR_INDIRECTLY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/insert_iterator.h b/libcxx/include/__cxx03/__iterator/insert_iterator.h
index 1f0320acda854..ef063ab73fced 100644
--- a/libcxx/include/__cxx03/__iterator/insert_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/insert_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_INSERT_ITERATOR_H
-#define _LIBCPP___ITERATOR_INSERT_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_INSERT_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_INSERT_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator.h>
@@ -92,4 +92,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_INSERT_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_INSERT_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/istream_iterator.h b/libcxx/include/__cxx03/__iterator/istream_iterator.h
index af0f477358a7f..71ca4ebc4a384 100644
--- a/libcxx/include/__cxx03/__iterator/istream_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/istream_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
-#define _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ISTREAM_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_ISTREAM_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/istream.h>
@@ -98,4 +98,4 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const istream_iterator<_Tp, _CharT,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ISTREAM_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_ISTREAM_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h b/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
index 73da595172848..768747d490cc0 100644
--- a/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/istreambuf_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
-#define _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ISTREAMBUF_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_ISTREAMBUF_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/istream.h>
@@ -106,4 +106,4 @@ operator!=(const istreambuf_iterator<_CharT, _Traits>& __a, const istreambuf_ite
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ISTREAMBUF_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_ISTREAMBUF_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/iter_move.h b/libcxx/include/__cxx03/__iterator/iter_move.h
index f45baaaf63c1e..ad33ea065f003 100644
--- a/libcxx/include/__cxx03/__iterator/iter_move.h
+++ b/libcxx/include/__cxx03/__iterator/iter_move.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ITER_MOVE_H
-#define _LIBCPP___ITERATOR_ITER_MOVE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ITER_MOVE_H
+#define _LIBCPP___CXX03___ITERATOR_ITER_MOVE_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__config>
@@ -100,4 +100,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_ITER_MOVE_H
+#endif // _LIBCPP___CXX03___ITERATOR_ITER_MOVE_H
diff --git a/libcxx/include/__cxx03/__iterator/iter_swap.h b/libcxx/include/__cxx03/__iterator/iter_swap.h
index c3a7e164ff6f6..b61092dee724f 100644
--- a/libcxx/include/__cxx03/__iterator/iter_swap.h
+++ b/libcxx/include/__cxx03/__iterator/iter_swap.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ITER_SWAP_H
-#define _LIBCPP___ITERATOR_ITER_SWAP_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ITER_SWAP_H
+#define _LIBCPP___CXX03___ITERATOR_ITER_SWAP_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__concepts/swappable.h>
@@ -105,4 +105,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_ITER_SWAP_H
+#endif // _LIBCPP___CXX03___ITERATOR_ITER_SWAP_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator.h b/libcxx/include/__cxx03/__iterator/iterator.h
index 5d9648ce6a06f..60114ffc5b4dc 100644
--- a/libcxx/include/__cxx03/__iterator/iterator.h
+++ b/libcxx/include/__cxx03/__iterator/iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ITERATOR_H
-#define _LIBCPP___ITERATOR_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -30,4 +30,4 @@ struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 iterator {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator_traits.h b/libcxx/include/__cxx03/__iterator/iterator_traits.h
index 7efd0c81c9301..66bab3ddd7e57 100644
--- a/libcxx/include/__cxx03/__iterator/iterator_traits.h
+++ b/libcxx/include/__cxx03/__iterator/iterator_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
-#define _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ITERATOR_TRAITS_H
+#define _LIBCPP___CXX03___ITERATOR_ITERATOR_TRAITS_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -525,4 +525,4 @@ using iter_value_t =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
+#endif // _LIBCPP___CXX03___ITERATOR_ITERATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/iterator_with_data.h b/libcxx/include/__cxx03/__iterator/iterator_with_data.h
index fb7aafc38a30c..e6aa66922237b 100644
--- a/libcxx/include/__cxx03/__iterator/iterator_with_data.h
+++ b/libcxx/include/__cxx03/__iterator/iterator_with_data.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
-#define _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
+#ifndef _LIBCPP___CXX03___ITERATOR_ITERATOR_WITH_DATA_H
+#define _LIBCPP___CXX03___ITERATOR_ITERATOR_WITH_DATA_H
 
 #include <__cxx03/__compare/compare_three_way_result.h>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -102,4 +102,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_ITERATOR_WITH_DATA_H
+#endif // _LIBCPP___CXX03___ITERATOR_ITERATOR_WITH_DATA_H
diff --git a/libcxx/include/__cxx03/__iterator/mergeable.h b/libcxx/include/__cxx03/__iterator/mergeable.h
index a20134defa87f..5e54fd28a33ff 100644
--- a/libcxx/include/__cxx03/__iterator/mergeable.h
+++ b/libcxx/include/__cxx03/__iterator/mergeable.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_MERGEABLE_H
-#define _LIBCPP___ITERATOR_MERGEABLE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_MERGEABLE_H
+#define _LIBCPP___CXX03___ITERATOR_MERGEABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -39,4 +39,4 @@ concept mergeable =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_MERGEABLE_H
+#endif // _LIBCPP___CXX03___ITERATOR_MERGEABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/move_iterator.h b/libcxx/include/__cxx03/__iterator/move_iterator.h
index 701f51a272da6..0fbcfdd0c1d9f 100644
--- a/libcxx/include/__cxx03/__iterator/move_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/move_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_MOVE_ITERATOR_H
-#define _LIBCPP___ITERATOR_MOVE_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_MOVE_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_MOVE_ITERATOR_H
 
 #include <__cxx03/__compare/compare_three_way_result.h>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -344,4 +344,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_MOVE_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_MOVE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/move_sentinel.h b/libcxx/include/__cxx03/__iterator/move_sentinel.h
index 8518bcf39ea96..cb4cf95a8c6ec 100644
--- a/libcxx/include/__cxx03/__iterator/move_sentinel.h
+++ b/libcxx/include/__cxx03/__iterator/move_sentinel.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_MOVE_SENTINEL_H
-#define _LIBCPP___ITERATOR_MOVE_SENTINEL_H
+#ifndef _LIBCPP___CXX03___ITERATOR_MOVE_SENTINEL_H
+#define _LIBCPP___CXX03___ITERATOR_MOVE_SENTINEL_H
 
 #include <__cxx03/__concepts/assignable.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -58,4 +58,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ITERATOR_MOVE_SENTINEL_H
+#endif // _LIBCPP___CXX03___ITERATOR_MOVE_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/next.h b/libcxx/include/__cxx03/__iterator/next.h
index 554760347606e..459702fd43743 100644
--- a/libcxx/include/__cxx03/__iterator/next.h
+++ b/libcxx/include/__cxx03/__iterator/next.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_NEXT_H
-#define _LIBCPP___ITERATOR_NEXT_H
+#ifndef _LIBCPP___CXX03___ITERATOR_NEXT_H
+#define _LIBCPP___CXX03___ITERATOR_NEXT_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -80,4 +80,4 @@ inline constexpr auto next = __next::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_NEXT_H
+#endif // _LIBCPP___CXX03___ITERATOR_NEXT_H
diff --git a/libcxx/include/__cxx03/__iterator/ostream_iterator.h b/libcxx/include/__cxx03/__iterator/ostream_iterator.h
index 40f737831f409..100cc54045bb3 100644
--- a/libcxx/include/__cxx03/__iterator/ostream_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/ostream_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
-#define _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_OSTREAM_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_OSTREAM_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/ostream.h>
@@ -72,4 +72,4 @@ class _LIBCPP_TEMPLATE_VIS ostream_iterator
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_OSTREAM_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_OSTREAM_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h b/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
index 363279ce684e1..b5472b33b7711 100644
--- a/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/ostreambuf_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
-#define _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_OSTREAMBUF_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_OSTREAMBUF_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator.h>
@@ -69,4 +69,4 @@ class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_OSTREAMBUF_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_OSTREAMBUF_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/permutable.h b/libcxx/include/__cxx03/__iterator/permutable.h
index c2641de963de7..368c4089f1316 100644
--- a/libcxx/include/__cxx03/__iterator/permutable.h
+++ b/libcxx/include/__cxx03/__iterator/permutable.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_PERMUTABLE_H
-#define _LIBCPP___ITERATOR_PERMUTABLE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_PERMUTABLE_H
+#define _LIBCPP___CXX03___ITERATOR_PERMUTABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -31,4 +31,4 @@ concept permutable =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_PERMUTABLE_H
+#endif // _LIBCPP___CXX03___ITERATOR_PERMUTABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/prev.h b/libcxx/include/__cxx03/__iterator/prev.h
index 7256f2d0a0faf..c272e52a9349b 100644
--- a/libcxx/include/__cxx03/__iterator/prev.h
+++ b/libcxx/include/__cxx03/__iterator/prev.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_PREV_H
-#define _LIBCPP___ITERATOR_PREV_H
+#ifndef _LIBCPP___CXX03___ITERATOR_PREV_H
+#define _LIBCPP___CXX03___ITERATOR_PREV_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -73,4 +73,4 @@ inline constexpr auto prev = __prev::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_PREV_H
+#endif // _LIBCPP___CXX03___ITERATOR_PREV_H
diff --git a/libcxx/include/__cxx03/__iterator/projected.h b/libcxx/include/__cxx03/__iterator/projected.h
index 582d192120620..b80b7d56da890 100644
--- a/libcxx/include/__cxx03/__iterator/projected.h
+++ b/libcxx/include/__cxx03/__iterator/projected.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_PROJECTED_H
-#define _LIBCPP___ITERATOR_PROJECTED_H
+#ifndef _LIBCPP___CXX03___ITERATOR_PROJECTED_H
+#define _LIBCPP___CXX03___ITERATOR_PROJECTED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -50,4 +50,4 @@ using projected = typename __projected_impl<_It, _Proj>::__type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_PROJECTED_H
+#endif // _LIBCPP___CXX03___ITERATOR_PROJECTED_H
diff --git a/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h b/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
index 0dddc16575390..b2333ace6d631 100644
--- a/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
+++ b/libcxx/include/__cxx03/__iterator/ranges_iterator_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
-#define _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_RANGES_ITERATOR_TRAITS_H
+#define _LIBCPP___CXX03___ITERATOR_RANGES_ITERATOR_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/pair.h>
@@ -37,4 +37,4 @@ using __range_to_alloc_type =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_RANGES_ITERATOR_TRAITS_H
+#endif // _LIBCPP___CXX03___ITERATOR_RANGES_ITERATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/readable_traits.h b/libcxx/include/__cxx03/__iterator/readable_traits.h
index 07ef5e30ae469..c7b21649d8a18 100644
--- a/libcxx/include/__cxx03/__iterator/readable_traits.h
+++ b/libcxx/include/__cxx03/__iterator/readable_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_READABLE_TRAITS_H
-#define _LIBCPP___ITERATOR_READABLE_TRAITS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_READABLE_TRAITS_H
+#define _LIBCPP___CXX03___ITERATOR_READABLE_TRAITS_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -78,4 +78,4 @@ struct indirectly_readable_traits<_Tp> : __cond_value_type<typename _Tp::value_t
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_READABLE_TRAITS_H
+#endif // _LIBCPP___CXX03___ITERATOR_READABLE_TRAITS_H
diff --git a/libcxx/include/__cxx03/__iterator/reverse_access.h b/libcxx/include/__cxx03/__iterator/reverse_access.h
index f307fd8b01b5e..bea5b1412c1f9 100644
--- a/libcxx/include/__cxx03/__iterator/reverse_access.h
+++ b/libcxx/include/__cxx03/__iterator/reverse_access.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_REVERSE_ACCESS_H
-#define _LIBCPP___ITERATOR_REVERSE_ACCESS_H
+#ifndef _LIBCPP___CXX03___ITERATOR_REVERSE_ACCESS_H
+#define _LIBCPP___CXX03___ITERATOR_REVERSE_ACCESS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/reverse_iterator.h>
@@ -77,4 +77,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto crend(const _Cp& __c) -
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_REVERSE_ACCESS_H
+#endif // _LIBCPP___CXX03___ITERATOR_REVERSE_ACCESS_H
diff --git a/libcxx/include/__cxx03/__iterator/reverse_iterator.h b/libcxx/include/__cxx03/__iterator/reverse_iterator.h
index 000da25a0f330..b95ca27ff5fb0 100644
--- a/libcxx/include/__cxx03/__iterator/reverse_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/reverse_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
-#define _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
+#ifndef _LIBCPP___CXX03___ITERATOR_REVERSE_ITERATOR_H
+#define _LIBCPP___CXX03___ITERATOR_REVERSE_ITERATOR_H
 
 #include <__cxx03/__algorithm/unwrap_iter.h>
 #include <__cxx03/__compare/compare_three_way_result.h>
@@ -343,4 +343,4 @@ struct __unwrap_iter_impl<reverse_iterator<reverse_iterator<_Iter> >, __b> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_REVERSE_ITERATOR_H
+#endif // _LIBCPP___CXX03___ITERATOR_REVERSE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/segmented_iterator.h b/libcxx/include/__cxx03/__iterator/segmented_iterator.h
index 93cd8e195eb05..7e353d620dc83 100644
--- a/libcxx/include/__cxx03/__iterator/segmented_iterator.h
+++ b/libcxx/include/__cxx03/__iterator/segmented_iterator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SEGMENTED_ITERATOR_H
-#define _LIBCPP___SEGMENTED_ITERATOR_H
+#ifndef _LIBCPP___CXX03___SEGMENTED_ITERATOR_H
+#define _LIBCPP___CXX03___SEGMENTED_ITERATOR_H
 
 // Segmented iterators are iterators over (not necessarily contiguous) sub-ranges.
 //
@@ -76,4 +76,4 @@ using __is_segmented_iterator = __has_specialization<__segmented_iterator_traits
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___SEGMENTED_ITERATOR_H
+#endif // _LIBCPP___CXX03___SEGMENTED_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__iterator/size.h b/libcxx/include/__cxx03/__iterator/size.h
index 3e8c2537f723f..b0374a6c59e1c 100644
--- a/libcxx/include/__cxx03/__iterator/size.h
+++ b/libcxx/include/__cxx03/__iterator/size.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_SIZE_H
-#define _LIBCPP___ITERATOR_SIZE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_SIZE_H
+#define _LIBCPP___CXX03___ITERATOR_SIZE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/common_type.h>
@@ -56,4 +56,4 @@ _LIBCPP_DIAGNOSTIC_POP
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_SIZE_H
+#endif // _LIBCPP___CXX03___ITERATOR_SIZE_H
diff --git a/libcxx/include/__cxx03/__iterator/sortable.h b/libcxx/include/__cxx03/__iterator/sortable.h
index 5c32629ff55dc..ae82f0f615432 100644
--- a/libcxx/include/__cxx03/__iterator/sortable.h
+++ b/libcxx/include/__cxx03/__iterator/sortable.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_SORTABLE_H
-#define _LIBCPP___ITERATOR_SORTABLE_H
+#ifndef _LIBCPP___CXX03___ITERATOR_SORTABLE_H
+#define _LIBCPP___CXX03___ITERATOR_SORTABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/identity.h>
@@ -32,4 +32,4 @@ concept sortable = permutable<_Iter> && indirect_strict_weak_order<_Comp, projec
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_SORTABLE_H
+#endif // _LIBCPP___CXX03___ITERATOR_SORTABLE_H
diff --git a/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h b/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
index 29d76409728ff..6184692b4074a 100644
--- a/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
+++ b/libcxx/include/__cxx03/__iterator/unreachable_sentinel.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
-#define _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
+#ifndef _LIBCPP___CXX03___ITERATOR_UNREACHABLE_SENTINEL_H
+#define _LIBCPP___CXX03___ITERATOR_UNREACHABLE_SENTINEL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>
@@ -34,4 +34,4 @@ inline constexpr unreachable_sentinel_t unreachable_sentinel{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_UNREACHABLE_SENTINEL_H
+#endif // _LIBCPP___CXX03___ITERATOR_UNREACHABLE_SENTINEL_H
diff --git a/libcxx/include/__cxx03/__iterator/wrap_iter.h b/libcxx/include/__cxx03/__iterator/wrap_iter.h
index 5ef909c19f4d3..8e7df13490974 100644
--- a/libcxx/include/__cxx03/__iterator/wrap_iter.h
+++ b/libcxx/include/__cxx03/__iterator/wrap_iter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ITERATOR_WRAP_ITER_H
-#define _LIBCPP___ITERATOR_WRAP_ITER_H
+#ifndef _LIBCPP___CXX03___ITERATOR_WRAP_ITER_H
+#define _LIBCPP___CXX03___ITERATOR_WRAP_ITER_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -241,4 +241,4 @@ struct _LIBCPP_TEMPLATE_VIS pointer_traits<__wrap_iter<_It> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_WRAP_ITER_H
+#endif // _LIBCPP___CXX03___ITERATOR_WRAP_ITER_H
diff --git a/libcxx/include/__cxx03/__locale b/libcxx/include/__cxx03/__locale
index 7324aa2390ce7..383b56a1942e8 100644
--- a/libcxx/include/__cxx03/__locale
+++ b/libcxx/include/__cxx03/__locale
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE
-#define _LIBCPP___LOCALE
+#ifndef _LIBCPP___CXX03___LOCALE
+#define _LIBCPP___CXX03___LOCALE
 
 #include <__cxx03/__config>
 #include <__cxx03/__locale_dir/locale_base_api.h>
@@ -1510,4 +1510,4 @@ private:
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE
+#endif // _LIBCPP___CXX03___LOCALE
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api.h
index e62ebb17765d7..a20f0952f52c3 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
-#define _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
+#ifndef _LIBCPP___CXX03___LOCALE_DIR_LOCALE_BASE_API_H
+#define _LIBCPP___CXX03___LOCALE_DIR_LOCALE_BASE_API_H
 
 #if defined(_LIBCPP_MSVCRT_LIKE)
 #  include <__cxx03/__locale_dir/locale_base_api/win32.h>
@@ -95,4 +95,4 @@ except that locale_t is used instead of the current global locale.
 The variadic functions may be implemented as templates with a parameter pack instead of variadic functions.
 */
 
-#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_H
+#endif // _LIBCPP___CXX03___LOCALE_DIR_LOCALE_BASE_API_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
index b943c82c62298..265dbf892a54f 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/android.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_ANDROID_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_ANDROID_H
 
 #include <__cxx03/stdlib.h>
 
@@ -47,4 +47,4 @@ inline _LIBCPP_HIDE_FROM_ABI double strtod_l(const char* __nptr, char** __endptr
 #  endif // __NDK_MAJOR__ <= 16
 #endif   // __has_include(<android/ndk-version.h>)
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_ANDROID_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_ANDROID_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
index 1f9607209842c..35bf30c07fde1 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_defaults.h
@@ -11,8 +11,8 @@
 // we will define the mapping from an internal macro to the real BSD symbol.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -33,4 +33,4 @@
 #define __libcpp_asprintf_l(...) asprintf_l(__VA_ARGS__)
 #define __libcpp_sscanf_l(...) sscanf_l(__VA_ARGS__)
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_DEFAULTS_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
index 129cacb317ee4..e61e950a42c77 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/bsd_locale_fallbacks.h
@@ -10,8 +10,8 @@
 // of those functions for non-BSD platforms.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
 
 #include <__cxx03/__locale_dir/locale_base_api/locale_guard.h>
 #include <__cxx03/cstdio>
@@ -123,4 +123,4 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __libcpp_sscanf_l(
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
index 74d017d07435d..6ee6a3a1c8755 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/fuchsia.h
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_FUCHSIA_H
 
 #include <__cxx03/__support/xlocale/__posix_l_fallback.h>
 #include <__cxx03/__support/xlocale/__strtonum_fallback.h>
 #include <__cxx03/cstdlib>
 #include <__cxx03/cwchar>
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_FUCHSIA_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_FUCHSIA_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
index ff4b2f8b272cd..0fab9d24d84b6 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/ibm.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_IBM_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_IBM_H
 
 #if defined(__MVS__)
 #  include <__cxx03/__support/ibm/locale_mgmt_zos.h>
@@ -105,4 +105,4 @@ _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 2, 0) int vasprintf(char** strp, const char
   return str_size;
 }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_IBM_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_IBM_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
index 17eade28f35f3..e3583634e4322 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/locale_guard.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__locale> // for locale_t
@@ -75,4 +75,4 @@ struct __libcpp_locale_guard {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_LOCALE_GUARD_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
index f67511f4a0bc5..e2d23d49c9e45 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/musl.h
@@ -14,8 +14,8 @@
 // in Musl.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_MUSL_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_MUSL_H
 
 #include <__cxx03/cstdlib>
 #include <__cxx03/cwchar>
@@ -28,4 +28,4 @@ inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, c
   return ::strtoull(__nptr, __endptr, __base);
 }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_MUSL_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_MUSL_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
index a8c1cff16e6d8..e52d90794556a 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/newlib.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_NEWLIB_H
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_NEWLIB_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_NEWLIB_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
index effd9e0d76b1c..6b64607d484d3 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/openbsd.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_OPENBSD_H
 
 #include <__cxx03/__support/xlocale/__strtonum_fallback.h>
 #include <__cxx03/clocale>
@@ -16,4 +16,4 @@
 #include <__cxx03/ctype.h>
 #include <__cxx03/cwctype>
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_OPENBSD_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_OPENBSD_H
diff --git a/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h b/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
index 60b9435039f6d..5f85acf88e6f0 100644
--- a/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
+++ b/libcxx/include/__cxx03/__locale_dir/locale_base_api/win32.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
-#define _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+#ifndef _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_WIN32_H
+#define _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_WIN32_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -232,4 +232,4 @@ _LIBCPP_EXPORTED_FROM_ABI int vasprintf_l(char** __ret, locale_t __loc, const ch
 // not-so-pressing FIXME: use locale to determine blank characters
 inline int iswblank_l(wint_t __c, locale_t /*loc*/) { return (__c == L' ' || __c == L'\t'); }
 
-#endif // _LIBCPP___LOCALE_LOCALE_BASE_API_WIN32_H
+#endif // _LIBCPP___CXX03___LOCALE_LOCALE_BASE_API_WIN32_H
diff --git a/libcxx/include/__cxx03/__math/abs.h b/libcxx/include/__cxx03/__math/abs.h
index ad543e654c48b..61fc58ac22692 100644
--- a/libcxx/include/__cxx03/__math/abs.h
+++ b/libcxx/include/__cxx03/__math/abs.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_ABS_H
-#define _LIBCPP___MATH_ABS_H
+#ifndef _LIBCPP___CXX03___MATH_ABS_H
+#define _LIBCPP___CXX03___MATH_ABS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -43,4 +43,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double fabs(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_ABS_H
+#endif // _LIBCPP___CXX03___MATH_ABS_H
diff --git a/libcxx/include/__cxx03/__math/copysign.h b/libcxx/include/__cxx03/__math/copysign.h
index e70d6ee286774..d20940604c6f9 100644
--- a/libcxx/include/__cxx03/__math/copysign.h
+++ b/libcxx/include/__cxx03/__math/copysign.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_COPYSIGN_H
-#define _LIBCPP___MATH_COPYSIGN_H
+#ifndef _LIBCPP___CXX03___MATH_COPYSIGN_H
+#define _LIBCPP___CXX03___MATH_COPYSIGN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -42,4 +42,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::typ
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_COPYSIGN_H
+#endif // _LIBCPP___CXX03___MATH_COPYSIGN_H
diff --git a/libcxx/include/__cxx03/__math/error_functions.h b/libcxx/include/__cxx03/__math/error_functions.h
index 47f506096676e..25cf059989862 100644
--- a/libcxx/include/__cxx03/__math/error_functions.h
+++ b/libcxx/include/__cxx03/__math/error_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_ERROR_FUNCTIONS_H
-#define _LIBCPP___MATH_ERROR_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_ERROR_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_ERROR_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -57,4 +57,4 @@ inline _LIBCPP_HIDE_FROM_ABI double erfc(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_ERROR_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_ERROR_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/exponential_functions.h b/libcxx/include/__cxx03/__math/exponential_functions.h
index 2e988e1709541..452d5d238f910 100644
--- a/libcxx/include/__cxx03/__math/exponential_functions.h
+++ b/libcxx/include/__cxx03/__math/exponential_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
-#define _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_EXPONENTIAL_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_EXPONENTIAL_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -168,4 +168,4 @@ inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type pow(_A1 __x, _A2
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_EXPONENTIAL_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_EXPONENTIAL_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/fdim.h b/libcxx/include/__cxx03/__math/fdim.h
index 5f0c8ae34eba3..ac6fec44efcb4 100644
--- a/libcxx/include/__cxx03/__math/fdim.h
+++ b/libcxx/include/__cxx03/__math/fdim.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_FDIM_H
-#define _LIBCPP___MATH_FDIM_H
+#ifndef _LIBCPP___CXX03___MATH_FDIM_H
+#define _LIBCPP___CXX03___MATH_FDIM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -45,4 +45,4 @@ inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type fdim(_A1 __x, _A
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_FDIM_H
+#endif // _LIBCPP___CXX03___MATH_FDIM_H
diff --git a/libcxx/include/__cxx03/__math/fma.h b/libcxx/include/__cxx03/__math/fma.h
index b58bc1a9855f7..ce7e92a330376 100644
--- a/libcxx/include/__cxx03/__math/fma.h
+++ b/libcxx/include/__cxx03/__math/fma.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_FMA_H
-#define _LIBCPP___MATH_FMA_H
+#ifndef _LIBCPP___CXX03___MATH_FMA_H
+#define _LIBCPP___CXX03___MATH_FMA_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -52,4 +52,4 @@ inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2, _A3>::type fma(_A1 __x
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_FMA_H
+#endif // _LIBCPP___CXX03___MATH_FMA_H
diff --git a/libcxx/include/__cxx03/__math/gamma.h b/libcxx/include/__cxx03/__math/gamma.h
index 613ff0104def4..4d104988e2c9e 100644
--- a/libcxx/include/__cxx03/__math/gamma.h
+++ b/libcxx/include/__cxx03/__math/gamma.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_GAMMA_H
-#define _LIBCPP___MATH_GAMMA_H
+#ifndef _LIBCPP___CXX03___MATH_GAMMA_H
+#define _LIBCPP___CXX03___MATH_GAMMA_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -59,4 +59,4 @@ inline _LIBCPP_HIDE_FROM_ABI double tgamma(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_GAMMA_H
+#endif // _LIBCPP___CXX03___MATH_GAMMA_H
diff --git a/libcxx/include/__cxx03/__math/hyperbolic_functions.h b/libcxx/include/__cxx03/__math/hyperbolic_functions.h
index 63070ee4f000d..2b765ca923cae 100644
--- a/libcxx/include/__cxx03/__math/hyperbolic_functions.h
+++ b/libcxx/include/__cxx03/__math/hyperbolic_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
-#define _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_HYPERBOLIC_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_HYPERBOLIC_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -73,4 +73,4 @@ inline _LIBCPP_HIDE_FROM_ABI double tanh(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_HYPERBOLIC_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_HYPERBOLIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/hypot.h b/libcxx/include/__cxx03/__math/hypot.h
index cb032f0fd6991..bd87fef5f5ed5 100644
--- a/libcxx/include/__cxx03/__math/hypot.h
+++ b/libcxx/include/__cxx03/__math/hypot.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_HYPOT_H
-#define _LIBCPP___MATH_HYPOT_H
+#ifndef _LIBCPP___CXX03___MATH_HYPOT_H
+#define _LIBCPP___CXX03___MATH_HYPOT_H
 
 #include <__cxx03/__algorithm/max.h>
 #include <__cxx03/__config>
@@ -106,4 +106,4 @@ _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2, _A3>::type hypot(_A1 __x, _A2
 _LIBCPP_END_NAMESPACE_STD
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MATH_HYPOT_H
+#endif // _LIBCPP___CXX03___MATH_HYPOT_H
diff --git a/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h b/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
index 0701ead8a32f1..b48689d159af0 100644
--- a/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
+++ b/libcxx/include/__cxx03/__math/inverse_hyperbolic_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
-#define _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -73,4 +73,4 @@ inline _LIBCPP_HIDE_FROM_ABI double atanh(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_INVERSE_HYPERBOLIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h b/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
index 626295321627a..573c9b1bdc878 100644
--- a/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
+++ b/libcxx/include/__cxx03/__math/inverse_trigonometric_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
-#define _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -96,4 +96,4 @@ inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type atan2(_A1 __y, _
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_INVERSE_TRIGONOMETRIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/logarithms.h b/libcxx/include/__cxx03/__math/logarithms.h
index ad067abdb3813..25473501435cc 100644
--- a/libcxx/include/__cxx03/__math/logarithms.h
+++ b/libcxx/include/__cxx03/__math/logarithms.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_LOGARITHMS_H
-#define _LIBCPP___MATH_LOGARITHMS_H
+#ifndef _LIBCPP___CXX03___MATH_LOGARITHMS_H
+#define _LIBCPP___CXX03___MATH_LOGARITHMS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -121,4 +121,4 @@ inline _LIBCPP_HIDE_FROM_ABI double logb(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_LOGARITHMS_H
+#endif // _LIBCPP___CXX03___MATH_LOGARITHMS_H
diff --git a/libcxx/include/__cxx03/__math/min_max.h b/libcxx/include/__cxx03/__math/min_max.h
index 717c8cf57e866..4df5bc1ee0f77 100644
--- a/libcxx/include/__cxx03/__math/min_max.h
+++ b/libcxx/include/__cxx03/__math/min_max.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_MIN_MAX_H
-#define _LIBCPP___MATH_MIN_MAX_H
+#ifndef _LIBCPP___CXX03___MATH_MIN_MAX_H
+#define _LIBCPP___CXX03___MATH_MIN_MAX_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -71,4 +71,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::typ
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_MIN_MAX_H
+#endif // _LIBCPP___CXX03___MATH_MIN_MAX_H
diff --git a/libcxx/include/__cxx03/__math/modulo.h b/libcxx/include/__cxx03/__math/modulo.h
index 4fe58b1bf45dd..5a6124187c5d2 100644
--- a/libcxx/include/__cxx03/__math/modulo.h
+++ b/libcxx/include/__cxx03/__math/modulo.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_MODULO_H
-#define _LIBCPP___MATH_MODULO_H
+#ifndef _LIBCPP___CXX03___MATH_MODULO_H
+#define _LIBCPP___CXX03___MATH_MODULO_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -60,4 +60,4 @@ inline _LIBCPP_HIDE_FROM_ABI long double modf(long double __x, long double* __y)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_MODULO_H
+#endif // _LIBCPP___CXX03___MATH_MODULO_H
diff --git a/libcxx/include/__cxx03/__math/remainder.h b/libcxx/include/__cxx03/__math/remainder.h
index e7d825f4c8e38..a9f750b6ddc3f 100644
--- a/libcxx/include/__cxx03/__math/remainder.h
+++ b/libcxx/include/__cxx03/__math/remainder.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_REMAINDER_H
-#define _LIBCPP___MATH_REMAINDER_H
+#ifndef _LIBCPP___CXX03___MATH_REMAINDER_H
+#define _LIBCPP___CXX03___MATH_REMAINDER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -70,4 +70,4 @@ inline _LIBCPP_HIDE_FROM_ABI typename __promote<_A1, _A2>::type remquo(_A1 __x,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_REMAINDER_H
+#endif // _LIBCPP___CXX03___MATH_REMAINDER_H
diff --git a/libcxx/include/__cxx03/__math/roots.h b/libcxx/include/__cxx03/__math/roots.h
index bb78b70dcaacb..77d9eead803c6 100644
--- a/libcxx/include/__cxx03/__math/roots.h
+++ b/libcxx/include/__cxx03/__math/roots.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_ROOTS_H
-#define _LIBCPP___MATH_ROOTS_H
+#ifndef _LIBCPP___CXX03___MATH_ROOTS_H
+#define _LIBCPP___CXX03___MATH_ROOTS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -59,4 +59,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double cbrt(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_ROOTS_H
+#endif // _LIBCPP___CXX03___MATH_ROOTS_H
diff --git a/libcxx/include/__cxx03/__math/rounding_functions.h b/libcxx/include/__cxx03/__math/rounding_functions.h
index 69b021729e0ac..e146d743e974a 100644
--- a/libcxx/include/__cxx03/__math/rounding_functions.h
+++ b/libcxx/include/__cxx03/__math/rounding_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
-#define _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_ROUNDING_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_ROUNDING_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -242,4 +242,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI double trunc(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_ROUNDING_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_ROUNDING_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/special_functions.h b/libcxx/include/__cxx03/__math/special_functions.h
index 27fb394e803e3..38832215b89d5 100644
--- a/libcxx/include/__cxx03/__math/special_functions.h
+++ b/libcxx/include/__cxx03/__math/special_functions.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
-#define _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_SPECIAL_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_SPECIAL_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__math/copysign.h>
@@ -81,4 +81,4 @@ _LIBCPP_HIDE_FROM_ABI double hermite(unsigned __n, _Integer __x) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_SPECIAL_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_SPECIAL_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__math/traits.h b/libcxx/include/__cxx03/__math/traits.h
index 60823378a835b..f62509c32c008 100644
--- a/libcxx/include/__cxx03/__math/traits.h
+++ b/libcxx/include/__cxx03/__math/traits.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_TRAITS_H
-#define _LIBCPP___MATH_TRAITS_H
+#ifndef _LIBCPP___CXX03___MATH_TRAITS_H
+#define _LIBCPP___CXX03___MATH_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -185,4 +185,4 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI bool isunordered(_A1 __x, _A2 __y
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_TRAITS_H
+#endif // _LIBCPP___CXX03___MATH_TRAITS_H
diff --git a/libcxx/include/__cxx03/__math/trigonometric_functions.h b/libcxx/include/__cxx03/__math/trigonometric_functions.h
index 9d99f409cb207..6c23ae6bc854f 100644
--- a/libcxx/include/__cxx03/__math/trigonometric_functions.h
+++ b/libcxx/include/__cxx03/__math/trigonometric_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
-#define _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___MATH_TRIGONOMETRIC_FUNCTIONS_H
+#define _LIBCPP___CXX03___MATH_TRIGONOMETRIC_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -73,4 +73,4 @@ inline _LIBCPP_HIDE_FROM_ABI double tan(_A1 __x) _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MATH_TRIGONOMETRIC_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___MATH_TRIGONOMETRIC_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__mbstate_t.h b/libcxx/include/__cxx03/__mbstate_t.h
index 302cc5ddc4db2..6c7cfaaa8fe5b 100644
--- a/libcxx/include/__cxx03/__mbstate_t.h
+++ b/libcxx/include/__cxx03/__mbstate_t.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MBSTATE_T_H
-#define _LIBCPP___MBSTATE_T_H
+#ifndef _LIBCPP___CXX03___MBSTATE_T_H
+#define _LIBCPP___CXX03___MBSTATE_T_H
 
 #include <__cxx03/__config>
 
@@ -51,4 +51,4 @@
 #  error "We don't know how to get the definition of mbstate_t without <wchar.h> on your platform."
 #endif
 
-#endif // _LIBCPP___MBSTATE_T_H
+#endif // _LIBCPP___CXX03___MBSTATE_T_H
diff --git a/libcxx/include/__cxx03/__mdspan/default_accessor.h b/libcxx/include/__cxx03/__mdspan/default_accessor.h
index 1745f077db701..1f7c331818566 100644
--- a/libcxx/include/__cxx03/__mdspan/default_accessor.h
+++ b/libcxx/include/__cxx03/__mdspan/default_accessor.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
-#define _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
+#ifndef _LIBCPP___CXX03___MDSPAN_DEFAULT_ACCESSOR_H
+#define _LIBCPP___CXX03___MDSPAN_DEFAULT_ACCESSOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_abstract.h>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_DEFAULT_ACCESSOR_H
+#endif // _LIBCPP___CXX03___MDSPAN_DEFAULT_ACCESSOR_H
diff --git a/libcxx/include/__cxx03/__mdspan/extents.h b/libcxx/include/__cxx03/__mdspan/extents.h
index b6f2b6abf3f61..d6fc8b8af75aa 100644
--- a/libcxx/include/__cxx03/__mdspan/extents.h
+++ b/libcxx/include/__cxx03/__mdspan/extents.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_EXTENTS_H
-#define _LIBCPP___MDSPAN_EXTENTS_H
+#ifndef _LIBCPP___CXX03___MDSPAN_EXTENTS_H
+#define _LIBCPP___CXX03___MDSPAN_EXTENTS_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -529,4 +529,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_EXTENTS_H
+#endif // _LIBCPP___CXX03___MDSPAN_EXTENTS_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_left.h b/libcxx/include/__cxx03/__mdspan/layout_left.h
index 017f278ab6db0..7f667238818b3 100644
--- a/libcxx/include/__cxx03/__mdspan/layout_left.h
+++ b/libcxx/include/__cxx03/__mdspan/layout_left.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_LAYOUT_LEFT_H
-#define _LIBCPP___MDSPAN_LAYOUT_LEFT_H
+#ifndef _LIBCPP___CXX03___MDSPAN_LAYOUT_LEFT_H
+#define _LIBCPP___CXX03___MDSPAN_LAYOUT_LEFT_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -201,4 +201,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_LAYOUT_LEFT_H
+#endif // _LIBCPP___CXX03___MDSPAN_LAYOUT_LEFT_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_right.h b/libcxx/include/__cxx03/__mdspan/layout_right.h
index 13d430cf6b5e4..12dc5a1b558d8 100644
--- a/libcxx/include/__cxx03/__mdspan/layout_right.h
+++ b/libcxx/include/__cxx03/__mdspan/layout_right.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
-#define _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
+#ifndef _LIBCPP___CXX03___MDSPAN_LAYOUT_RIGHT_H
+#define _LIBCPP___CXX03___MDSPAN_LAYOUT_RIGHT_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -198,4 +198,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_LAYOUT_RIGHT_H
+#endif // _LIBCPP___CXX03___MDSPAN_LAYOUT_RIGHT_H
diff --git a/libcxx/include/__cxx03/__mdspan/layout_stride.h b/libcxx/include/__cxx03/__mdspan/layout_stride.h
index 9b387ba6b9f60..d97ce1a55610e 100644
--- a/libcxx/include/__cxx03/__mdspan/layout_stride.h
+++ b/libcxx/include/__cxx03/__mdspan/layout_stride.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
-#define _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+#ifndef _LIBCPP___CXX03___MDSPAN_LAYOUT_STRIDE_H
+#define _LIBCPP___CXX03___MDSPAN_LAYOUT_STRIDE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -363,4 +363,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+#endif // _LIBCPP___CXX03___MDSPAN_LAYOUT_STRIDE_H
diff --git a/libcxx/include/__cxx03/__mdspan/mdspan.h b/libcxx/include/__cxx03/__mdspan/mdspan.h
index 253ac1cbb1c42..a1f6f00d61b71 100644
--- a/libcxx/include/__cxx03/__mdspan/mdspan.h
+++ b/libcxx/include/__cxx03/__mdspan/mdspan.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MDSPAN_MDSPAN_H
-#define _LIBCPP___MDSPAN_MDSPAN_H
+#ifndef _LIBCPP___CXX03___MDSPAN_MDSPAN_H
+#define _LIBCPP___CXX03___MDSPAN_MDSPAN_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -316,4 +316,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MDSPAN_MDSPAN_H
+#endif // _LIBCPP___CXX03___MDSPAN_MDSPAN_H
diff --git a/libcxx/include/__cxx03/__memory/addressof.h b/libcxx/include/__cxx03/__memory/addressof.h
index 0f6b875884448..6c88ec87be1ac 100644
--- a/libcxx/include/__cxx03/__memory/addressof.h
+++ b/libcxx/include/__cxx03/__memory/addressof.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ADDRESSOF_H
-#define _LIBCPP___MEMORY_ADDRESSOF_H
+#ifndef _LIBCPP___CXX03___MEMORY_ADDRESSOF_H
+#define _LIBCPP___CXX03___MEMORY_ADDRESSOF_H
 
 #include <__cxx03/__config>
 
@@ -58,4 +58,4 @@ _Tp* addressof(const _Tp&&) noexcept = delete;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ADDRESSOF_H
+#endif // _LIBCPP___CXX03___MEMORY_ADDRESSOF_H
diff --git a/libcxx/include/__cxx03/__memory/align.h b/libcxx/include/__cxx03/__memory/align.h
index 3ef7011bdb62f..2b2085f217364 100644
--- a/libcxx/include/__cxx03/__memory/align.h
+++ b/libcxx/include/__cxx03/__memory/align.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALIGN_H
-#define _LIBCPP___MEMORY_ALIGN_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALIGN_H
+#define _LIBCPP___CXX03___MEMORY_ALIGN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -22,4 +22,4 @@ _LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ALIGN_H
+#endif // _LIBCPP___CXX03___MEMORY_ALIGN_H
diff --git a/libcxx/include/__cxx03/__memory/aligned_alloc.h b/libcxx/include/__cxx03/__memory/aligned_alloc.h
index 9e864f5a7ab0d..af90baeabc140 100644
--- a/libcxx/include/__cxx03/__memory/aligned_alloc.h
+++ b/libcxx/include/__cxx03/__memory/aligned_alloc.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALIGNED_ALLOC_H
-#define _LIBCPP___MEMORY_ALIGNED_ALLOC_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALIGNED_ALLOC_H
+#define _LIBCPP___CXX03___MEMORY_ALIGNED_ALLOC_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -61,4 +61,4 @@ inline _LIBCPP_HIDE_FROM_ABI void __libcpp_aligned_free(void* __ptr) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ALIGNED_ALLOC_H
+#endif // _LIBCPP___CXX03___MEMORY_ALIGNED_ALLOC_H
diff --git a/libcxx/include/__cxx03/__memory/allocate_at_least.h b/libcxx/include/__cxx03/__memory/allocate_at_least.h
index e8b4cd3a17e5f..cb1810855a01d 100644
--- a/libcxx/include/__cxx03/__memory/allocate_at_least.h
+++ b/libcxx/include/__cxx03/__memory/allocate_at_least.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
-#define _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALLOCATE_AT_LEAST_H
+#define _LIBCPP___CXX03___MEMORY_ALLOCATE_AT_LEAST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/allocator_traits.h>
@@ -45,4 +45,4 @@ __allocate_at_least(_Alloc& __alloc, size_t __n) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ALLOCATE_AT_LEAST_H
+#endif // _LIBCPP___CXX03___MEMORY_ALLOCATE_AT_LEAST_H
diff --git a/libcxx/include/__cxx03/__memory/allocation_guard.h b/libcxx/include/__cxx03/__memory/allocation_guard.h
index a84ab2de7eb9e..0a2712d1df99b 100644
--- a/libcxx/include/__cxx03/__memory/allocation_guard.h
+++ b/libcxx/include/__cxx03/__memory/allocation_guard.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALLOCATION_GUARD_H
-#define _LIBCPP___MEMORY_ALLOCATION_GUARD_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALLOCATION_GUARD_H
+#define _LIBCPP___CXX03___MEMORY_ALLOCATION_GUARD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -105,4 +105,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_ALLOCATION_GUARD_H
+#endif // _LIBCPP___CXX03___MEMORY_ALLOCATION_GUARD_H
diff --git a/libcxx/include/__cxx03/__memory/allocator.h b/libcxx/include/__cxx03/__memory/allocator.h
index e0066c6e89b9b..06f270a0c008c 100644
--- a/libcxx/include/__cxx03/__memory/allocator.h
+++ b/libcxx/include/__cxx03/__memory/allocator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALLOCATOR_H
-#define _LIBCPP___MEMORY_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALLOCATOR_H
+#define _LIBCPP___CXX03___MEMORY_ALLOCATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -265,4 +265,4 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const allocator<_Tp>&, const alloca
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_arg_t.h b/libcxx/include/__cxx03/__memory/allocator_arg_t.h
index 5f40454dca757..3646b71e42655 100644
--- a/libcxx/include/__cxx03/__memory/allocator_arg_t.h
+++ b/libcxx/include/__cxx03/__memory/allocator_arg_t.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
-#define _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
+#ifndef _LIBCPP___CXX03___FUNCTIONAL_ALLOCATOR_ARG_T_H
+#define _LIBCPP___CXX03___FUNCTIONAL_ALLOCATOR_ARG_T_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/uses_allocator.h>
@@ -72,4 +72,4 @@ __user_alloc_construct_impl(integral_constant<int, 2>, _Tp* __storage, const _Al
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FUNCTIONAL_ALLOCATOR_ARG_T_H
+#endif // _LIBCPP___CXX03___FUNCTIONAL_ALLOCATOR_ARG_T_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_destructor.h b/libcxx/include/__cxx03/__memory/allocator_destructor.h
index e009cd1d04aa1..2c69c91561321 100644
--- a/libcxx/include/__cxx03/__memory/allocator_destructor.h
+++ b/libcxx/include/__cxx03/__memory/allocator_destructor.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
-#define _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALLOCATOR_DESTRUCTOR_H
+#define _LIBCPP___CXX03___MEMORY_ALLOCATOR_DESTRUCTOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/allocator_traits.h>
@@ -37,4 +37,4 @@ class __allocator_destructor {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ALLOCATOR_DESTRUCTOR_H
+#endif // _LIBCPP___CXX03___MEMORY_ALLOCATOR_DESTRUCTOR_H
diff --git a/libcxx/include/__cxx03/__memory/allocator_traits.h b/libcxx/include/__cxx03/__memory/allocator_traits.h
index 9833df6c50bd6..e70716c070158 100644
--- a/libcxx/include/__cxx03/__memory/allocator_traits.h
+++ b/libcxx/include/__cxx03/__memory/allocator_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
-#define _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
+#ifndef _LIBCPP___CXX03___MEMORY_ALLOCATOR_TRAITS_H
+#define _LIBCPP___CXX03___MEMORY_ALLOCATOR_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/construct_at.h>
@@ -421,4 +421,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_ALLOCATOR_TRAITS_H
+#endif // _LIBCPP___CXX03___MEMORY_ALLOCATOR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__memory/assume_aligned.h b/libcxx/include/__cxx03/__memory/assume_aligned.h
index 3b345d4c5c5c4..03a495aac580c 100644
--- a/libcxx/include/__cxx03/__memory/assume_aligned.h
+++ b/libcxx/include/__cxx03/__memory/assume_aligned.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_ASSUME_ALIGNED_H
-#define _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+#ifndef _LIBCPP___CXX03___MEMORY_ASSUME_ALIGNED_H
+#define _LIBCPP___CXX03___MEMORY_ASSUME_ALIGNED_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -47,4 +47,4 @@ template <size_t _Np, class _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+#endif // _LIBCPP___CXX03___MEMORY_ASSUME_ALIGNED_H
diff --git a/libcxx/include/__cxx03/__memory/auto_ptr.h b/libcxx/include/__cxx03/__memory/auto_ptr.h
index f5695745d22ea..d5d541e1547a5 100644
--- a/libcxx/include/__cxx03/__memory/auto_ptr.h
+++ b/libcxx/include/__cxx03/__memory/auto_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_AUTO_PTR_H
-#define _LIBCPP___MEMORY_AUTO_PTR_H
+#ifndef _LIBCPP___CXX03___MEMORY_AUTO_PTR_H
+#define _LIBCPP___CXX03___MEMORY_AUTO_PTR_H
 
 #include <__cxx03/__config>
 
@@ -89,4 +89,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
 
-#endif // _LIBCPP___MEMORY_AUTO_PTR_H
+#endif // _LIBCPP___CXX03___MEMORY_AUTO_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/builtin_new_allocator.h b/libcxx/include/__cxx03/__memory/builtin_new_allocator.h
index e0284dfcff1ee..e14a57e8a0611 100644
--- a/libcxx/include/__cxx03/__memory/builtin_new_allocator.h
+++ b/libcxx/include/__cxx03/__memory/builtin_new_allocator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
-#define _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_BUILTIN_NEW_ALLOCATOR_H
+#define _LIBCPP___CXX03___MEMORY_BUILTIN_NEW_ALLOCATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/unique_ptr.h>
@@ -64,4 +64,4 @@ struct __builtin_new_allocator {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_BUILTIN_NEW_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_BUILTIN_NEW_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/compressed_pair.h b/libcxx/include/__cxx03/__memory/compressed_pair.h
index 2af34f02772e0..086621d355d6d 100644
--- a/libcxx/include/__cxx03/__memory/compressed_pair.h
+++ b/libcxx/include/__cxx03/__memory/compressed_pair.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_COMPRESSED_PAIR_H
-#define _LIBCPP___MEMORY_COMPRESSED_PAIR_H
+#ifndef _LIBCPP___CXX03___MEMORY_COMPRESSED_PAIR_H
+#define _LIBCPP___CXX03___MEMORY_COMPRESSED_PAIR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/tuple.h>
@@ -168,4 +168,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_COMPRESSED_PAIR_H
+#endif // _LIBCPP___CXX03___MEMORY_COMPRESSED_PAIR_H
diff --git a/libcxx/include/__cxx03/__memory/concepts.h b/libcxx/include/__cxx03/__memory/concepts.h
index 85620202495eb..ac57609c0b5c1 100644
--- a/libcxx/include/__cxx03/__memory/concepts.h
+++ b/libcxx/include/__cxx03/__memory/concepts.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_CONCEPTS_H
-#define _LIBCPP___MEMORY_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___MEMORY_CONCEPTS_H
+#define _LIBCPP___CXX03___MEMORY_CONCEPTS_H
 
 #include <__cxx03/__concepts/same_as.h>
 #include <__cxx03/__config>
@@ -60,4 +60,4 @@ concept __nothrow_forward_range = __nothrow_input_range<_Rp> && __nothrow_forwar
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_CONCEPTS_H
+#endif // _LIBCPP___CXX03___MEMORY_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__memory/construct_at.h b/libcxx/include/__cxx03/__memory/construct_at.h
index 090e132a67ee5..755cdbf9743d1 100644
--- a/libcxx/include/__cxx03/__memory/construct_at.h
+++ b/libcxx/include/__cxx03/__memory/construct_at.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_CONSTRUCT_AT_H
-#define _LIBCPP___MEMORY_CONSTRUCT_AT_H
+#ifndef _LIBCPP___CXX03___MEMORY_CONSTRUCT_AT_H
+#define _LIBCPP___CXX03___MEMORY_CONSTRUCT_AT_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -125,4 +125,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_CONSTRUCT_AT_H
+#endif // _LIBCPP___CXX03___MEMORY_CONSTRUCT_AT_H
diff --git a/libcxx/include/__cxx03/__memory/destruct_n.h b/libcxx/include/__cxx03/__memory/destruct_n.h
index f5a24a67ca52c..a7dd2abd55980 100644
--- a/libcxx/include/__cxx03/__memory/destruct_n.h
+++ b/libcxx/include/__cxx03/__memory/destruct_n.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_DESTRUCT_N_H
-#define _LIBCPP___MEMORY_DESTRUCT_N_H
+#ifndef _LIBCPP___CXX03___MEMORY_DESTRUCT_N_H
+#define _LIBCPP___CXX03___MEMORY_DESTRUCT_N_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -60,4 +60,4 @@ struct __destruct_n {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_DESTRUCT_N_H
+#endif // _LIBCPP___CXX03___MEMORY_DESTRUCT_N_H
diff --git a/libcxx/include/__cxx03/__memory/inout_ptr.h b/libcxx/include/__cxx03/__memory/inout_ptr.h
index bb715e3b86c6c..4fea153874030 100644
--- a/libcxx/include/__cxx03/__memory/inout_ptr.h
+++ b/libcxx/include/__cxx03/__memory/inout_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___INOUT_PTR_H
-#define _LIBCPP___INOUT_PTR_H
+#ifndef _LIBCPP___CXX03___INOUT_PTR_H
+#define _LIBCPP___CXX03___INOUT_PTR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -106,4 +106,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___INOUT_PTR_H
+#endif // _LIBCPP___CXX03___INOUT_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/out_ptr.h b/libcxx/include/__cxx03/__memory/out_ptr.h
index 9aa9f33e293c0..053818bc1f51a 100644
--- a/libcxx/include/__cxx03/__memory/out_ptr.h
+++ b/libcxx/include/__cxx03/__memory/out_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___OUT_PTR_H
-#define _LIBCPP___OUT_PTR_H
+#ifndef _LIBCPP___CXX03___OUT_PTR_H
+#define _LIBCPP___CXX03___OUT_PTR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -98,4 +98,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___OUT_PTR_H
+#endif // _LIBCPP___CXX03___OUT_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/pointer_traits.h b/libcxx/include/__cxx03/__memory/pointer_traits.h
index 9c480af773b70..f1206836676d4 100644
--- a/libcxx/include/__cxx03/__memory/pointer_traits.h
+++ b/libcxx/include/__cxx03/__memory/pointer_traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_POINTER_TRAITS_H
-#define _LIBCPP___MEMORY_POINTER_TRAITS_H
+#ifndef _LIBCPP___CXX03___MEMORY_POINTER_TRAITS_H
+#define _LIBCPP___CXX03___MEMORY_POINTER_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -304,4 +304,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_POINTER_TRAITS_H
+#endif // _LIBCPP___CXX03___MEMORY_POINTER_TRAITS_H
diff --git a/libcxx/include/__cxx03/__memory/ranges_construct_at.h b/libcxx/include/__cxx03/__memory/ranges_construct_at.h
index f8acc9f0c21d9..db271b8572dc6 100644
--- a/libcxx/include/__cxx03/__memory/ranges_construct_at.h
+++ b/libcxx/include/__cxx03/__memory/ranges_construct_at.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
-#define _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
+#ifndef _LIBCPP___CXX03___MEMORY_RANGES_CONSTRUCT_AT_H
+#define _LIBCPP___CXX03___MEMORY_RANGES_CONSTRUCT_AT_H
 
 #include <__cxx03/__concepts/destructible.h>
 #include <__cxx03/__config>
@@ -121,4 +121,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_RANGES_CONSTRUCT_AT_H
+#endif // _LIBCPP___CXX03___MEMORY_RANGES_CONSTRUCT_AT_H
diff --git a/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
index f7af434847814..24e3dfa23180a 100644
--- a/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
+++ b/libcxx/include/__cxx03/__memory/ranges_uninitialized_algorithms.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
-#define _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+#ifndef _LIBCPP___CXX03___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+#define _LIBCPP___CXX03___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
 
 #include <__cxx03/__algorithm/in_out_result.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -322,4 +322,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+#endif // _LIBCPP___CXX03___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
diff --git a/libcxx/include/__cxx03/__memory/raw_storage_iterator.h b/libcxx/include/__cxx03/__memory/raw_storage_iterator.h
index bde1e1e6f0304..c0f6b9151e5cb 100644
--- a/libcxx/include/__cxx03/__memory/raw_storage_iterator.h
+++ b/libcxx/include/__cxx03/__memory/raw_storage_iterator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
-#define _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_RAW_STORAGE_ITERATOR_H
+#define _LIBCPP___CXX03___MEMORY_RAW_STORAGE_ITERATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator.h>
@@ -84,4 +84,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_RAW_STORAGE_ITERATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_RAW_STORAGE_ITERATOR_H
diff --git a/libcxx/include/__cxx03/__memory/shared_ptr.h b/libcxx/include/__cxx03/__memory/shared_ptr.h
index f6322d9141e30..5e1f8c1ff7d5e 100644
--- a/libcxx/include/__cxx03/__memory/shared_ptr.h
+++ b/libcxx/include/__cxx03/__memory/shared_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_SHARED_PTR_H
-#define _LIBCPP___MEMORY_SHARED_PTR_H
+#ifndef _LIBCPP___CXX03___MEMORY_SHARED_PTR_H
+#define _LIBCPP___CXX03___MEMORY_SHARED_PTR_H
 
 #include <__cxx03/__compare/compare_three_way.h>
 #include <__cxx03/__compare/ordering.h>
@@ -1691,4 +1691,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_SHARED_PTR_H
+#endif // _LIBCPP___CXX03___MEMORY_SHARED_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/swap_allocator.h b/libcxx/include/__cxx03/__memory/swap_allocator.h
index 3b463553d3ff5..2986da7365732 100644
--- a/libcxx/include/__cxx03/__memory/swap_allocator.h
+++ b/libcxx/include/__cxx03/__memory/swap_allocator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
-#define _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_SWAP_ALLOCATOR_H
+#define _LIBCPP___CXX03___MEMORY_SWAP_ALLOCATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/allocator_traits.h>
@@ -51,4 +51,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __swap_allocator
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_SWAP_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_SWAP_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/temp_value.h b/libcxx/include/__cxx03/__memory/temp_value.h
index ddf963da45de4..b777c66187bd0 100644
--- a/libcxx/include/__cxx03/__memory/temp_value.h
+++ b/libcxx/include/__cxx03/__memory/temp_value.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_TEMP_VALUE_H
-#define _LIBCPP___MEMORY_TEMP_VALUE_H
+#ifndef _LIBCPP___CXX03___MEMORY_TEMP_VALUE_H
+#define _LIBCPP___CXX03___MEMORY_TEMP_VALUE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -55,4 +55,4 @@ struct __temp_value {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_TEMP_VALUE_H
+#endif // _LIBCPP___CXX03___MEMORY_TEMP_VALUE_H
diff --git a/libcxx/include/__cxx03/__memory/temporary_buffer.h b/libcxx/include/__cxx03/__memory/temporary_buffer.h
index 11a22e6db67d9..3f584a7337f06 100644
--- a/libcxx/include/__cxx03/__memory/temporary_buffer.h
+++ b/libcxx/include/__cxx03/__memory/temporary_buffer.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
-#define _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
+#ifndef _LIBCPP___CXX03___MEMORY_TEMPORARY_BUFFER_H
+#define _LIBCPP___CXX03___MEMORY_TEMPORARY_BUFFER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/pair.h>
@@ -72,4 +72,4 @@ struct __return_temporary_buffer {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_TEMPORARY_BUFFER_H
+#endif // _LIBCPP___CXX03___MEMORY_TEMPORARY_BUFFER_H
diff --git a/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
index d74304d1d970a..d595c8c6cf49e 100644
--- a/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__cxx03/__memory/uninitialized_algorithms.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
-#define _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
+#ifndef _LIBCPP___CXX03___MEMORY_UNINITIALIZED_ALGORITHMS_H
+#define _LIBCPP___CXX03___MEMORY_UNINITIALIZED_ALGORITHMS_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__algorithm/move.h>
@@ -650,4 +650,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H
+#endif // _LIBCPP___CXX03___MEMORY_UNINITIALIZED_ALGORITHMS_H
diff --git a/libcxx/include/__cxx03/__memory/unique_ptr.h b/libcxx/include/__cxx03/__memory/unique_ptr.h
index 2576b6b37e7cb..415b002997591 100644
--- a/libcxx/include/__cxx03/__memory/unique_ptr.h
+++ b/libcxx/include/__cxx03/__memory/unique_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_UNIQUE_PTR_H
-#define _LIBCPP___MEMORY_UNIQUE_PTR_H
+#ifndef _LIBCPP___CXX03___MEMORY_UNIQUE_PTR_H
+#define _LIBCPP___CXX03___MEMORY_UNIQUE_PTR_H
 
 #include <__cxx03/__compare/compare_three_way.h>
 #include <__cxx03/__compare/compare_three_way_result.h>
@@ -690,4 +690,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_UNIQUE_PTR_H
+#endif // _LIBCPP___CXX03___MEMORY_UNIQUE_PTR_H
diff --git a/libcxx/include/__cxx03/__memory/uses_allocator.h b/libcxx/include/__cxx03/__memory/uses_allocator.h
index afdaac2d4b6fc..4fb6651c68b1d 100644
--- a/libcxx/include/__cxx03/__memory/uses_allocator.h
+++ b/libcxx/include/__cxx03/__memory/uses_allocator.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_USES_ALLOCATOR_H
-#define _LIBCPP___MEMORY_USES_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_H
+#define _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_convertible.h>
@@ -49,4 +49,4 @@ inline constexpr bool uses_allocator_v = uses_allocator<_Tp, _Alloc>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_USES_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory/uses_allocator_construction.h b/libcxx/include/__cxx03/__memory/uses_allocator_construction.h
index fae2609204b13..447d2a4e7cce1 100644
--- a/libcxx/include/__cxx03/__memory/uses_allocator_construction.h
+++ b/libcxx/include/__cxx03/__memory/uses_allocator_construction.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
-#define _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
+#ifndef _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
+#define _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/construct_at.h>
@@ -244,4 +244,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
+#endif // _LIBCPP___CXX03___MEMORY_USES_ALLOCATOR_CONSTRUCTION_H
diff --git a/libcxx/include/__cxx03/__memory/voidify.h b/libcxx/include/__cxx03/__memory/voidify.h
index b509aecc4a18f..63e03b7d7f8f8 100644
--- a/libcxx/include/__cxx03/__memory/voidify.h
+++ b/libcxx/include/__cxx03/__memory/voidify.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_VOIDIFY_H
-#define _LIBCPP___MEMORY_VOIDIFY_H
+#ifndef _LIBCPP___CXX03___MEMORY_VOIDIFY_H
+#define _LIBCPP___CXX03___MEMORY_VOIDIFY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -27,4 +27,4 @@ _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void*
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MEMORY_VOIDIFY_H
+#endif // _LIBCPP___CXX03___MEMORY_VOIDIFY_H
diff --git a/libcxx/include/__cxx03/__memory_resource/memory_resource.h b/libcxx/include/__cxx03/__memory_resource/memory_resource.h
index e4af41a9dc0b0..6bcad87edd407 100644
--- a/libcxx/include/__cxx03/__memory_resource/memory_resource.h
+++ b/libcxx/include/__cxx03/__memory_resource/memory_resource.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
-#define _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_MEMORY_RESOURCE_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_MEMORY_RESOURCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/memory_resource.h>
@@ -85,4 +85,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___MEMORY_RESOURCE_MEMORY_RESOURCE_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_MEMORY_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h b/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
index 9d418ea2289ae..f72f56ef05fc6 100644
--- a/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
+++ b/libcxx/include/__cxx03/__memory_resource/monotonic_buffer_resource.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
-#define _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -119,4 +119,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_MONOTONIC_BUFFER_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h b/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
index f615ac3811671..154f6f0f6973b 100644
--- a/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
+++ b/libcxx/include/__cxx03/__memory_resource/polymorphic_allocator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
-#define _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -228,4 +228,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_POLYMORPHIC_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__memory_resource/pool_options.h b/libcxx/include/__cxx03/__memory_resource/pool_options.h
index 50f3ea5882981..ac00e22bd7f4a 100644
--- a/libcxx/include/__cxx03/__memory_resource/pool_options.h
+++ b/libcxx/include/__cxx03/__memory_resource/pool_options.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
-#define _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_POOL_OPTIONS_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_POOL_OPTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -35,4 +35,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___MEMORY_RESOURCE_POOL_OPTIONS_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_POOL_OPTIONS_H
diff --git a/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h b/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
index f139b592eadad..c8897a3d0818e 100644
--- a/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
+++ b/libcxx/include/__cxx03/__memory_resource/synchronized_pool_resource.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
-#define _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory_resource/memory_resource.h>
@@ -89,4 +89,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_SYNCHRONIZED_POOL_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h b/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
index d3a32da069b7e..690768546ca6f 100644
--- a/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
+++ b/libcxx/include/__cxx03/__memory_resource/unsynchronized_pool_resource.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
-#define _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
+#ifndef _LIBCPP___CXX03___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
+#define _LIBCPP___CXX03___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory_resource/memory_resource.h>
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
+#endif // _LIBCPP___CXX03___MEMORY_RESOURCE_UNSYNCHRONIZED_POOL_RESOURCE_H
diff --git a/libcxx/include/__cxx03/__mutex/lock_guard.h b/libcxx/include/__cxx03/__mutex/lock_guard.h
index b20efc33b2a15..6760b1f20fa9e 100644
--- a/libcxx/include/__cxx03/__mutex/lock_guard.h
+++ b/libcxx/include/__cxx03/__mutex/lock_guard.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MUTEX_LOCK_GUARD_H
-#define _LIBCPP___MUTEX_LOCK_GUARD_H
+#ifndef _LIBCPP___CXX03___MUTEX_LOCK_GUARD_H
+#define _LIBCPP___CXX03___MUTEX_LOCK_GUARD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__mutex/tag_types.h>
@@ -45,4 +45,4 @@ _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(lock_guard);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MUTEX_LOCK_GUARD_H
+#endif // _LIBCPP___CXX03___MUTEX_LOCK_GUARD_H
diff --git a/libcxx/include/__cxx03/__mutex/mutex.h b/libcxx/include/__cxx03/__mutex/mutex.h
index 58adf2b1b2b0a..46c7546f77e32 100644
--- a/libcxx/include/__cxx03/__mutex/mutex.h
+++ b/libcxx/include/__cxx03/__mutex/mutex.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MUTEX_MUTEX_H
-#define _LIBCPP___MUTEX_MUTEX_H
+#ifndef _LIBCPP___CXX03___MUTEX_MUTEX_H
+#define _LIBCPP___CXX03___MUTEX_MUTEX_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__thread/support.h>
@@ -50,4 +50,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_HAS_NO_THREADS
 
-#endif // _LIBCPP___MUTEX_MUTEX_H
+#endif // _LIBCPP___CXX03___MUTEX_MUTEX_H
diff --git a/libcxx/include/__cxx03/__mutex/once_flag.h b/libcxx/include/__cxx03/__mutex/once_flag.h
index 9c467f7b274dd..73c2017f36854 100644
--- a/libcxx/include/__cxx03/__mutex/once_flag.h
+++ b/libcxx/include/__cxx03/__mutex/once_flag.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MUTEX_ONCE_FLAG_H
-#define _LIBCPP___MUTEX_ONCE_FLAG_H
+#ifndef _LIBCPP___CXX03___MUTEX_ONCE_FLAG_H
+#define _LIBCPP___CXX03___MUTEX_ONCE_FLAG_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -156,4 +156,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___MUTEX_ONCE_FLAG_H
+#endif // _LIBCPP___CXX03___MUTEX_ONCE_FLAG_H
diff --git a/libcxx/include/__cxx03/__mutex/tag_types.h b/libcxx/include/__cxx03/__mutex/tag_types.h
index c9665f1dd3a7a..7beb58dabc913 100644
--- a/libcxx/include/__cxx03/__mutex/tag_types.h
+++ b/libcxx/include/__cxx03/__mutex/tag_types.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MUTEX_TAG_TYPES_H
-#define _LIBCPP___MUTEX_TAG_TYPES_H
+#ifndef _LIBCPP___CXX03___MUTEX_TAG_TYPES_H
+#define _LIBCPP___CXX03___MUTEX_TAG_TYPES_H
 
 #include <__cxx03/__config>
 
@@ -41,4 +41,4 @@ constexpr adopt_lock_t adopt_lock   = adopt_lock_t();
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___MUTEX_TAG_TYPES_H
+#endif // _LIBCPP___CXX03___MUTEX_TAG_TYPES_H
diff --git a/libcxx/include/__cxx03/__mutex/unique_lock.h b/libcxx/include/__cxx03/__mutex/unique_lock.h
index 8dd620d0d5a8d..1911da3828732 100644
--- a/libcxx/include/__cxx03/__mutex/unique_lock.h
+++ b/libcxx/include/__cxx03/__mutex/unique_lock.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___MUTEX_UNIQUE_LOCK_H
-#define _LIBCPP___MUTEX_UNIQUE_LOCK_H
+#ifndef _LIBCPP___CXX03___MUTEX_UNIQUE_LOCK_H
+#define _LIBCPP___CXX03___MUTEX_UNIQUE_LOCK_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -174,4 +174,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_HAS_NO_THREADS
 
-#endif // _LIBCPP___MUTEX_UNIQUE_LOCK_H
+#endif // _LIBCPP___CXX03___MUTEX_UNIQUE_LOCK_H
diff --git a/libcxx/include/__cxx03/__node_handle b/libcxx/include/__cxx03/__node_handle
index 6b3af1507c7b4..6b272f9a49fff 100644
--- a/libcxx/include/__cxx03/__node_handle
+++ b/libcxx/include/__cxx03/__node_handle
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NODE_HANDLE
-#define _LIBCPP___NODE_HANDLE
+#ifndef _LIBCPP___CXX03___NODE_HANDLE
+#define _LIBCPP___CXX03___NODE_HANDLE
 
 /*
 
@@ -206,4 +206,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NODE_HANDLE
+#endif // _LIBCPP___CXX03___NODE_HANDLE
diff --git a/libcxx/include/__cxx03/__numeric/accumulate.h b/libcxx/include/__cxx03/__numeric/accumulate.h
index 243a4b05b97a3..28309201bb060 100644
--- a/libcxx/include/__cxx03/__numeric/accumulate.h
+++ b/libcxx/include/__cxx03/__numeric/accumulate.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_ACCUMULATE_H
-#define _LIBCPP___NUMERIC_ACCUMULATE_H
+#ifndef _LIBCPP___CXX03___NUMERIC_ACCUMULATE_H
+#define _LIBCPP___CXX03___NUMERIC_ACCUMULATE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/move.h>
@@ -50,4 +50,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_ACCUMULATE_H
+#endif // _LIBCPP___CXX03___NUMERIC_ACCUMULATE_H
diff --git a/libcxx/include/__cxx03/__numeric/adjacent_difference.h b/libcxx/include/__cxx03/__numeric/adjacent_difference.h
index a07d46a3892c1..ec7ba7c27f854 100644
--- a/libcxx/include/__cxx03/__numeric/adjacent_difference.h
+++ b/libcxx/include/__cxx03/__numeric/adjacent_difference.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
-#define _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
+#ifndef _LIBCPP___CXX03___NUMERIC_ADJACENT_DIFFERENCE_H
+#define _LIBCPP___CXX03___NUMERIC_ADJACENT_DIFFERENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -65,4 +65,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_ADJACENT_DIFFERENCE_H
+#endif // _LIBCPP___CXX03___NUMERIC_ADJACENT_DIFFERENCE_H
diff --git a/libcxx/include/__cxx03/__numeric/exclusive_scan.h b/libcxx/include/__cxx03/__numeric/exclusive_scan.h
index 70b85783ed22f..877deb22fc33d 100644
--- a/libcxx/include/__cxx03/__numeric/exclusive_scan.h
+++ b/libcxx/include/__cxx03/__numeric/exclusive_scan.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
-#define _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
+#ifndef _LIBCPP___CXX03___NUMERIC_EXCLUSIVE_SCAN_H
+#define _LIBCPP___CXX03___NUMERIC_EXCLUSIVE_SCAN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/operations.h>
@@ -55,4 +55,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_EXCLUSIVE_SCAN_H
+#endif // _LIBCPP___CXX03___NUMERIC_EXCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/gcd_lcm.h b/libcxx/include/__cxx03/__numeric/gcd_lcm.h
index 4d1a88f23bf46..e6d3b78fdfdd6 100644
--- a/libcxx/include/__cxx03/__numeric/gcd_lcm.h
+++ b/libcxx/include/__cxx03/__numeric/gcd_lcm.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_GCD_LCM_H
-#define _LIBCPP___NUMERIC_GCD_LCM_H
+#ifndef _LIBCPP___CXX03___NUMERIC_GCD_LCM_H
+#define _LIBCPP___CXX03___NUMERIC_GCD_LCM_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__assert>
@@ -129,4 +129,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_GCD_LCM_H
+#endif // _LIBCPP___CXX03___NUMERIC_GCD_LCM_H
diff --git a/libcxx/include/__cxx03/__numeric/inclusive_scan.h b/libcxx/include/__cxx03/__numeric/inclusive_scan.h
index 352161cafd498..5cac37dfc7b0d 100644
--- a/libcxx/include/__cxx03/__numeric/inclusive_scan.h
+++ b/libcxx/include/__cxx03/__numeric/inclusive_scan.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
-#define _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
+#ifndef _LIBCPP___CXX03___NUMERIC_INCLUSIVE_SCAN_H
+#define _LIBCPP___CXX03___NUMERIC_INCLUSIVE_SCAN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/operations.h>
@@ -56,4 +56,4 @@ inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___NUMERIC_INCLUSIVE_SCAN_H
+#endif // _LIBCPP___CXX03___NUMERIC_INCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/inner_product.h b/libcxx/include/__cxx03/__numeric/inner_product.h
index a5369438c7e3f..12615c3e02136 100644
--- a/libcxx/include/__cxx03/__numeric/inner_product.h
+++ b/libcxx/include/__cxx03/__numeric/inner_product.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_INNER_PRODUCT_H
-#define _LIBCPP___NUMERIC_INNER_PRODUCT_H
+#ifndef _LIBCPP___CXX03___NUMERIC_INNER_PRODUCT_H
+#define _LIBCPP___CXX03___NUMERIC_INNER_PRODUCT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/move.h>
@@ -55,4 +55,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_INNER_PRODUCT_H
+#endif // _LIBCPP___CXX03___NUMERIC_INNER_PRODUCT_H
diff --git a/libcxx/include/__cxx03/__numeric/iota.h b/libcxx/include/__cxx03/__numeric/iota.h
index fe9fb9fc37da4..f97c124700ae8 100644
--- a/libcxx/include/__cxx03/__numeric/iota.h
+++ b/libcxx/include/__cxx03/__numeric/iota.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_IOTA_H
-#define _LIBCPP___NUMERIC_IOTA_H
+#ifndef _LIBCPP___CXX03___NUMERIC_IOTA_H
+#define _LIBCPP___CXX03___NUMERIC_IOTA_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ iota(_ForwardIterator __first, _ForwardIterator __last, _Tp __value) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___NUMERIC_IOTA_H
+#endif // _LIBCPP___CXX03___NUMERIC_IOTA_H
diff --git a/libcxx/include/__cxx03/__numeric/midpoint.h b/libcxx/include/__cxx03/__numeric/midpoint.h
index 2244354c79300..91771df1f77b4 100644
--- a/libcxx/include/__cxx03/__numeric/midpoint.h
+++ b/libcxx/include/__cxx03/__numeric/midpoint.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_MIDPOINT_H
-#define _LIBCPP___NUMERIC_MIDPOINT_H
+#ifndef _LIBCPP___CXX03___NUMERIC_MIDPOINT_H
+#define _LIBCPP___CXX03___NUMERIC_MIDPOINT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -85,4 +85,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_MIDPOINT_H
+#endif // _LIBCPP___CXX03___NUMERIC_MIDPOINT_H
diff --git a/libcxx/include/__cxx03/__numeric/partial_sum.h b/libcxx/include/__cxx03/__numeric/partial_sum.h
index 1f9aa36233dd0..8dead17f1a5d1 100644
--- a/libcxx/include/__cxx03/__numeric/partial_sum.h
+++ b/libcxx/include/__cxx03/__numeric/partial_sum.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_PARTIAL_SUM_H
-#define _LIBCPP___NUMERIC_PARTIAL_SUM_H
+#ifndef _LIBCPP___CXX03___NUMERIC_PARTIAL_SUM_H
+#define _LIBCPP___CXX03___NUMERIC_PARTIAL_SUM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_PARTIAL_SUM_H
+#endif // _LIBCPP___CXX03___NUMERIC_PARTIAL_SUM_H
diff --git a/libcxx/include/__cxx03/__numeric/pstl.h b/libcxx/include/__cxx03/__numeric/pstl.h
index 365f6fabb1476..3311b40663549 100644
--- a/libcxx/include/__cxx03/__numeric/pstl.h
+++ b/libcxx/include/__cxx03/__numeric/pstl.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_PSTL_H
-#define _LIBCPP___NUMERIC_PSTL_H
+#ifndef _LIBCPP___CXX03___NUMERIC_PSTL_H
+#define _LIBCPP___CXX03___NUMERIC_PSTL_H
 
 #include <__cxx03/__config>
 
@@ -171,4 +171,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_PSTL_H
+#endif // _LIBCPP___CXX03___NUMERIC_PSTL_H
diff --git a/libcxx/include/__cxx03/__numeric/reduce.h b/libcxx/include/__cxx03/__numeric/reduce.h
index 039954498043a..628951371f959 100644
--- a/libcxx/include/__cxx03/__numeric/reduce.h
+++ b/libcxx/include/__cxx03/__numeric/reduce.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_REDUCE_H
-#define _LIBCPP___NUMERIC_REDUCE_H
+#ifndef _LIBCPP___CXX03___NUMERIC_REDUCE_H
+#define _LIBCPP___CXX03___NUMERIC_REDUCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/operations.h>
@@ -50,4 +50,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_REDUCE_H
+#endif // _LIBCPP___CXX03___NUMERIC_REDUCE_H
diff --git a/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h b/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
index 972428e36a778..d4da500beaa10 100644
--- a/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
+++ b/libcxx/include/__cxx03/__numeric/saturation_arithmetic.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
-#define _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
+#ifndef _LIBCPP___CXX03___NUMERIC_SATURATION_ARITHMETIC_H
+#define _LIBCPP___CXX03___NUMERIC_SATURATION_ARITHMETIC_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/arithmetic.h>
@@ -142,4 +142,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
+#endif // _LIBCPP___CXX03___NUMERIC_SATURATION_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h b/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
index 5857c75eb2094..034c7c929964e 100644
--- a/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
+++ b/libcxx/include/__cxx03/__numeric/transform_exclusive_scan.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
-#define _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+#ifndef _LIBCPP___CXX03___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+#define _LIBCPP___CXX03___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
 
 #include <__cxx03/__config>
 
@@ -39,4 +39,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform_ex
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+#endif // _LIBCPP___CXX03___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h b/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
index 31f7d52b4261f..00e9b1675b69f 100644
--- a/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
+++ b/libcxx/include/__cxx03/__numeric/transform_inclusive_scan.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
-#define _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+#ifndef _LIBCPP___CXX03___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+#define _LIBCPP___CXX03___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/iterator_traits.h>
@@ -49,4 +49,4 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform_in
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+#endif // _LIBCPP___CXX03___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
diff --git a/libcxx/include/__cxx03/__numeric/transform_reduce.h b/libcxx/include/__cxx03/__numeric/transform_reduce.h
index 5e494ff5d1b78..5ec1a59483f07 100644
--- a/libcxx/include/__cxx03/__numeric/transform_reduce.h
+++ b/libcxx/include/__cxx03/__numeric/transform_reduce.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
-#define _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
+#ifndef _LIBCPP___CXX03___NUMERIC_TRANSFORM_REDUCE_H
+#define _LIBCPP___CXX03___NUMERIC_TRANSFORM_REDUCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/operations.h>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___NUMERIC_TRANSFORM_REDUCE_H
+#endif // _LIBCPP___CXX03___NUMERIC_TRANSFORM_REDUCE_H
diff --git a/libcxx/include/__cxx03/__ostream/basic_ostream.h b/libcxx/include/__cxx03/__ostream/basic_ostream.h
index 1f0fb9acd3871..593ddc501c050 100644
--- a/libcxx/include/__cxx03/__ostream/basic_ostream.h
+++ b/libcxx/include/__cxx03/__ostream/basic_ostream.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___OSTREAM_BASIC_OSTREAM_H
-#define _LIBCPP___OSTREAM_BASIC_OSTREAM_H
+#ifndef _LIBCPP___CXX03___OSTREAM_BASIC_OSTREAM_H
+#define _LIBCPP___CXX03___OSTREAM_BASIC_OSTREAM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__exception/operations.h>
@@ -857,4 +857,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___OSTREAM_BASIC_OSTREAM_H
+#endif // _LIBCPP___CXX03___OSTREAM_BASIC_OSTREAM_H
diff --git a/libcxx/include/__cxx03/__ostream/print.h b/libcxx/include/__cxx03/__ostream/print.h
index b183081405184..d7943efd60f80 100644
--- a/libcxx/include/__cxx03/__ostream/print.h
+++ b/libcxx/include/__cxx03/__ostream/print.h
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___OSTREAM_PRINT_H
-#define _LIBCPP___OSTREAM_PRINT_H
+#ifndef _LIBCPP___CXX03___OSTREAM_PRINT_H
+#define _LIBCPP___CXX03___OSTREAM_PRINT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/ostream.h>
@@ -176,4 +176,4 @@ _LIBCPP_HIDE_FROM_ABI inline void println(ostream& __os) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___OSTREAM_PRINT_H
+#endif // _LIBCPP___CXX03___OSTREAM_PRINT_H
diff --git a/libcxx/include/__cxx03/__pstl/backend.h b/libcxx/include/__cxx03/__pstl/backend.h
index 9bc243746df0d..7e39c6dbc0f8d 100644
--- a/libcxx/include/__cxx03/__pstl/backend.h
+++ b/libcxx/include/__cxx03/__pstl/backend.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKEND_H
-#define _LIBCPP___PSTL_BACKEND_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKEND_H
+#define _LIBCPP___CXX03___PSTL_BACKEND_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__pstl/backend_fwd.h>
@@ -32,4 +32,4 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKEND_H
+#endif // _LIBCPP___CXX03___PSTL_BACKEND_H
diff --git a/libcxx/include/__cxx03/__pstl/backend_fwd.h b/libcxx/include/__cxx03/__pstl/backend_fwd.h
index 56c8c043e4d2c..e9abd8e99c73c 100644
--- a/libcxx/include/__cxx03/__pstl/backend_fwd.h
+++ b/libcxx/include/__cxx03/__pstl/backend_fwd.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKEND_FWD_H
-#define _LIBCPP___PSTL_BACKEND_FWD_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKEND_FWD_H
+#define _LIBCPP___CXX03___PSTL_BACKEND_FWD_H
 
 #include <__cxx03/__config>
 
@@ -298,4 +298,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKEND_FWD_H
+#endif // _LIBCPP___CXX03___PSTL_BACKEND_FWD_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/default.h b/libcxx/include/__cxx03/__pstl/backends/default.h
index 2ad388ae50d33..7f7a5325c7d1c 100644
--- a/libcxx/include/__cxx03/__pstl/backends/default.h
+++ b/libcxx/include/__cxx03/__pstl/backends/default.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKENDS_DEFAULT_H
-#define _LIBCPP___PSTL_BACKENDS_DEFAULT_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKENDS_DEFAULT_H
+#define _LIBCPP___CXX03___PSTL_BACKENDS_DEFAULT_H
 
 #include <__cxx03/__algorithm/copy_n.h>
 #include <__cxx03/__algorithm/equal.h>
@@ -500,4 +500,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKENDS_DEFAULT_H
+#endif // _LIBCPP___CXX03___PSTL_BACKENDS_DEFAULT_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/libdispatch.h b/libcxx/include/__cxx03/__pstl/backends/libdispatch.h
index fb811917025a2..80260ca5b7a5b 100644
--- a/libcxx/include/__cxx03/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__cxx03/__pstl/backends/libdispatch.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
-#define _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKENDS_LIBDISPATCH_H
+#define _LIBCPP___CXX03___PSTL_BACKENDS_LIBDISPATCH_H
 
 #include <__cxx03/__algorithm/inplace_merge.h>
 #include <__cxx03/__algorithm/lower_bound.h>
@@ -394,4 +394,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H
+#endif // _LIBCPP___CXX03___PSTL_BACKENDS_LIBDISPATCH_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/serial.h b/libcxx/include/__cxx03/__pstl/backends/serial.h
index 699c1035d9113..d0de7a1177783 100644
--- a/libcxx/include/__cxx03/__pstl/backends/serial.h
+++ b/libcxx/include/__cxx03/__pstl/backends/serial.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKENDS_SERIAL_H
-#define _LIBCPP___PSTL_BACKENDS_SERIAL_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKENDS_SERIAL_H
+#define _LIBCPP___CXX03___PSTL_BACKENDS_SERIAL_H
 
 #include <__cxx03/__algorithm/find_if.h>
 #include <__cxx03/__algorithm/for_each.h>
@@ -178,4 +178,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKENDS_SERIAL_H
+#endif // _LIBCPP___CXX03___PSTL_BACKENDS_SERIAL_H
diff --git a/libcxx/include/__cxx03/__pstl/backends/std_thread.h b/libcxx/include/__cxx03/__pstl/backends/std_thread.h
index 06cd1a1c03c46..487ddda57aa25 100644
--- a/libcxx/include/__cxx03/__pstl/backends/std_thread.h
+++ b/libcxx/include/__cxx03/__pstl/backends/std_thread.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
-#define _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
+#ifndef _LIBCPP___CXX03___PSTL_BACKENDS_STD_THREAD_H
+#define _LIBCPP___CXX03___PSTL_BACKENDS_STD_THREAD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__pstl/backend_fwd.h>
@@ -133,4 +133,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_BACKENDS_STD_THREAD_H
+#endif // _LIBCPP___CXX03___PSTL_BACKENDS_STD_THREAD_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h b/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
index b1b51d1679bf2..7828a7026e7d4 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/any_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
-#define _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_ANY_OF_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_ANY_OF_H
 
 #include <__cxx03/__algorithm/any_of.h>
 #include <__cxx03/__assert>
@@ -96,4 +96,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_ANY_OF_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h b/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
index 9538b5c94957d..770f5dd7b45da 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/cpu_traits.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
-#define _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_CPU_TRAITS_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_CPU_TRAITS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -83,4 +83,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_CPU_TRAITS_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h b/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
index 19fac90c44010..a101efb632e6c 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/fill.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_FILL_H
-#define _LIBCPP___PSTL_CPU_ALGOS_FILL_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_FILL_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_FILL_H
 
 #include <__cxx03/__algorithm/fill.h>
 #include <__cxx03/__assert>
@@ -63,4 +63,4 @@ struct __cpu_parallel_fill {
 } // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_FILL_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_FILL_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h b/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
index 2be1aad7ae6be..c7393c41c1f45 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/find_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
-#define _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H
 
 #include <__cxx03/__algorithm/find_if.h>
 #include <__cxx03/__assert>
@@ -134,4 +134,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h b/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
index 4f6f2e87342a9..9362f56b323b6 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/for_each.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
-#define _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_FOR_EACH_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_FOR_EACH_H
 
 #include <__cxx03/__algorithm/for_each.h>
 #include <__cxx03/__assert>
@@ -63,4 +63,4 @@ struct __cpu_parallel_for_each {
 } // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_FOR_EACH_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h b/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
index 0feb510dc60f7..9bab8b4838ef2 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/merge.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
-#define _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_MERGE_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_MERGE_H
 
 #include <__cxx03/__algorithm/merge.h>
 #include <__cxx03/__assert>
@@ -82,4 +82,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_MERGE_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_MERGE_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h b/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
index 63be0c7ac1669..c0ebb2cd0a1c7 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/stable_sort.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
-#define _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_STABLE_SORT_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_STABLE_SORT_H
 
 #include <__cxx03/__algorithm/stable_sort.h>
 #include <__cxx03/__config>
@@ -44,4 +44,4 @@ struct __cpu_parallel_stable_sort {
 } // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_STABLE_SORT_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_STABLE_SORT_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h b/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
index d82bbb588a998..500e110ac7e2e 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/transform.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
-#define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_H
 
 #include <__cxx03/__algorithm/transform.h>
 #include <__cxx03/__assert>
@@ -150,4 +150,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_H
diff --git a/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h b/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
index 52f0ea361b3c1..38764d963fc5e 100644
--- a/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
+++ b/libcxx/include/__cxx03/__pstl/cpu_algos/transform_reduce.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
-#define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
+#ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
+#define _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -213,4 +213,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
+#endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H
diff --git a/libcxx/include/__cxx03/__pstl/dispatch.h b/libcxx/include/__cxx03/__pstl/dispatch.h
index e223e59bb129c..39857d0eafd63 100644
--- a/libcxx/include/__cxx03/__pstl/dispatch.h
+++ b/libcxx/include/__cxx03/__pstl/dispatch.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_DISPATCH_H
-#define _LIBCPP___PSTL_DISPATCH_H
+#ifndef _LIBCPP___CXX03___PSTL_DISPATCH_H
+#define _LIBCPP___CXX03___PSTL_DISPATCH_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__pstl/backend_fwd.h>
@@ -63,4 +63,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_DISPATCH_H
+#endif // _LIBCPP___CXX03___PSTL_DISPATCH_H
diff --git a/libcxx/include/__cxx03/__pstl/handle_exception.h b/libcxx/include/__cxx03/__pstl/handle_exception.h
index 562617a1786b3..e21146fa1eaca 100644
--- a/libcxx/include/__cxx03/__pstl/handle_exception.h
+++ b/libcxx/include/__cxx03/__pstl/handle_exception.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___PSTL_HANDLE_EXCEPTION_H
-#define _LIBCPP___PSTL_HANDLE_EXCEPTION_H
+#ifndef _LIBCPP___CXX03___PSTL_HANDLE_EXCEPTION_H
+#define _LIBCPP___CXX03___PSTL_HANDLE_EXCEPTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/forward.h>
@@ -54,4 +54,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___PSTL_HANDLE_EXCEPTION_H
+#endif // _LIBCPP___CXX03___PSTL_HANDLE_EXCEPTION_H
diff --git a/libcxx/include/__cxx03/__random/bernoulli_distribution.h b/libcxx/include/__cxx03/__random/bernoulli_distribution.h
index 28536442fd1d3..3c35e8c39e237 100644
--- a/libcxx/include/__cxx03/__random/bernoulli_distribution.h
+++ b/libcxx/include/__cxx03/__random/bernoulli_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_BERNOULLI_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_BERNOULLI_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_BERNOULLI_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_BERNOULLI_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/binomial_distribution.h b/libcxx/include/__cxx03/__random/binomial_distribution.h
index 3ca98e84082e1..5b703749fc7d1 100644
--- a/libcxx/include/__cxx03/__random/binomial_distribution.h
+++ b/libcxx/include/__cxx03/__random/binomial_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_BINOMIAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_BINOMIAL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -195,4 +195,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_BINOMIAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_BINOMIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/cauchy_distribution.h b/libcxx/include/__cxx03/__random/cauchy_distribution.h
index b84ba03703143..cbec0b6f85554 100644
--- a/libcxx/include/__cxx03/__random/cauchy_distribution.h
+++ b/libcxx/include/__cxx03/__random/cauchy_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_CAUCHY_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_CAUCHY_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -136,4 +136,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_CAUCHY_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_CAUCHY_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/chi_squared_distribution.h b/libcxx/include/__cxx03/__random/chi_squared_distribution.h
index 94ba5555ce41d..531c3204130d5 100644
--- a/libcxx/include/__cxx03/__random/chi_squared_distribution.h
+++ b/libcxx/include/__cxx03/__random/chi_squared_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_CHI_SQUARED_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_CHI_SQUARED_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/gamma_distribution.h>
@@ -122,4 +122,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_CHI_SQUARED_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_CHI_SQUARED_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/clamp_to_integral.h b/libcxx/include/__cxx03/__random/clamp_to_integral.h
index c10783aab2c07..be89db8c6a807 100644
--- a/libcxx/include/__cxx03/__random/clamp_to_integral.h
+++ b/libcxx/include/__cxx03/__random/clamp_to_integral.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
-#define _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___RANDOM_CLAMP_TO_INTEGRAL_H
+#define _LIBCPP___CXX03___RANDOM_CLAMP_TO_INTEGRAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cmath>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+#endif // _LIBCPP___CXX03___RANDOM_CLAMP_TO_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__random/default_random_engine.h b/libcxx/include/__cxx03/__random/default_random_engine.h
index 10eb00b3594ff..81fbac0b301a2 100644
--- a/libcxx/include/__cxx03/__random/default_random_engine.h
+++ b/libcxx/include/__cxx03/__random/default_random_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
-#define _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_DEFAULT_RANDOM_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_DEFAULT_RANDOM_ENGINE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/linear_congruential_engine.h>
@@ -22,4 +22,4 @@ typedef minstd_rand default_random_engine;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_DEFAULT_RANDOM_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_DEFAULT_RANDOM_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/discard_block_engine.h b/libcxx/include/__cxx03/__random/discard_block_engine.h
index 2a5f877b79caa..43027f103a635 100644
--- a/libcxx/include/__cxx03/__random/discard_block_engine.h
+++ b/libcxx/include/__cxx03/__random/discard_block_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
-#define _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_DISCARD_BLOCK_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_DISCARD_BLOCK_ENGINE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_seed_sequence.h>
@@ -169,4 +169,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_DISCARD_BLOCK_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_DISCARD_BLOCK_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/discrete_distribution.h b/libcxx/include/__cxx03/__random/discrete_distribution.h
index 40b8bfb61dccf..b24948050c65c 100644
--- a/libcxx/include/__cxx03/__random/discrete_distribution.h
+++ b/libcxx/include/__cxx03/__random/discrete_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_DISCRETE_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_DISCRETE_DISTRIBUTION_H
 
 #include <__cxx03/__algorithm/upper_bound.h>
 #include <__cxx03/__config>
@@ -209,4 +209,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_DISCRETE_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_DISCRETE_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/exponential_distribution.h b/libcxx/include/__cxx03/__random/exponential_distribution.h
index 6f49c31a1bd10..d11a6672153bd 100644
--- a/libcxx/include/__cxx03/__random/exponential_distribution.h
+++ b/libcxx/include/__cxx03/__random/exponential_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_EXPONENTIAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_EXPONENTIAL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/generate_canonical.h>
@@ -129,4 +129,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_EXPONENTIAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_EXPONENTIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/extreme_value_distribution.h b/libcxx/include/__cxx03/__random/extreme_value_distribution.h
index 46c466d620d50..4207e79cdd569 100644
--- a/libcxx/include/__cxx03/__random/extreme_value_distribution.h
+++ b/libcxx/include/__cxx03/__random/extreme_value_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -136,4 +136,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_EXTREME_VALUE_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/fisher_f_distribution.h b/libcxx/include/__cxx03/__random/fisher_f_distribution.h
index 45ab47df4021b..f0050c73ca18c 100644
--- a/libcxx/include/__cxx03/__random/fisher_f_distribution.h
+++ b/libcxx/include/__cxx03/__random/fisher_f_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_FISHER_F_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_FISHER_F_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/gamma_distribution.h>
@@ -135,4 +135,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_FISHER_F_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_FISHER_F_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/gamma_distribution.h b/libcxx/include/__cxx03/__random/gamma_distribution.h
index 694c7219452a8..a5f9917fd8f93 100644
--- a/libcxx/include/__cxx03/__random/gamma_distribution.h
+++ b/libcxx/include/__cxx03/__random/gamma_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_GAMMA_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_GAMMA_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/exponential_distribution.h>
@@ -178,4 +178,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_GAMMA_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_GAMMA_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/generate_canonical.h b/libcxx/include/__cxx03/__random/generate_canonical.h
index 363f118c4f7d6..deea98fd0314b 100644
--- a/libcxx/include/__cxx03/__random/generate_canonical.h
+++ b/libcxx/include/__cxx03/__random/generate_canonical.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_GENERATE_CANONICAL_H
-#define _LIBCPP___RANDOM_GENERATE_CANONICAL_H
+#ifndef _LIBCPP___CXX03___RANDOM_GENERATE_CANONICAL_H
+#define _LIBCPP___CXX03___RANDOM_GENERATE_CANONICAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/log2.h>
@@ -48,4 +48,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_GENERATE_CANONICAL_H
+#endif // _LIBCPP___CXX03___RANDOM_GENERATE_CANONICAL_H
diff --git a/libcxx/include/__cxx03/__random/geometric_distribution.h b/libcxx/include/__cxx03/__random/geometric_distribution.h
index 3fe06792da856..85ca5d8ebbed9 100644
--- a/libcxx/include/__cxx03/__random/geometric_distribution.h
+++ b/libcxx/include/__cxx03/__random/geometric_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_GEOMETRIC_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_GEOMETRIC_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -117,4 +117,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_GEOMETRIC_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_GEOMETRIC_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/independent_bits_engine.h b/libcxx/include/__cxx03/__random/independent_bits_engine.h
index bf2350e7b0d6f..15df3f249ef82 100644
--- a/libcxx/include/__cxx03/__random/independent_bits_engine.h
+++ b/libcxx/include/__cxx03/__random/independent_bits_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
-#define _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_INDEPENDENT_BITS_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_INDEPENDENT_BITS_ENGINE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/istream.h>
@@ -202,4 +202,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_INDEPENDENT_BITS_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_INDEPENDENT_BITS_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/is_seed_sequence.h b/libcxx/include/__cxx03/__random/is_seed_sequence.h
index a924857c7ffa7..ddb9c83bb7081 100644
--- a/libcxx/include/__cxx03/__random/is_seed_sequence.h
+++ b/libcxx/include/__cxx03/__random/is_seed_sequence.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
-#define _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
+#ifndef _LIBCPP___CXX03___RANDOM_IS_SEED_SEQUENCE_H
+#define _LIBCPP___CXX03___RANDOM_IS_SEED_SEQUENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_convertible.h>
@@ -28,4 +28,4 @@ struct __is_seed_sequence {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_IS_SEED_SEQUENCE_H
+#endif // _LIBCPP___CXX03___RANDOM_IS_SEED_SEQUENCE_H
diff --git a/libcxx/include/__cxx03/__random/is_valid.h b/libcxx/include/__cxx03/__random/is_valid.h
index fc32a22bf608c..65693ceecf4d0 100644
--- a/libcxx/include/__cxx03/__random/is_valid.h
+++ b/libcxx/include/__cxx03/__random/is_valid.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_IS_VALID_H
-#define _LIBCPP___RANDOM_IS_VALID_H
+#ifndef _LIBCPP___CXX03___RANDOM_IS_VALID_H
+#define _LIBCPP___CXX03___RANDOM_IS_VALID_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -91,4 +91,4 @@ struct __libcpp_random_is_valid_urng<
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_IS_VALID_H
+#endif // _LIBCPP___CXX03___RANDOM_IS_VALID_H
diff --git a/libcxx/include/__cxx03/__random/knuth_b.h b/libcxx/include/__cxx03/__random/knuth_b.h
index b8df0078ebb1a..0ba23d7bbd2a8 100644
--- a/libcxx/include/__cxx03/__random/knuth_b.h
+++ b/libcxx/include/__cxx03/__random/knuth_b.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_KNUTH_B_H
-#define _LIBCPP___RANDOM_KNUTH_B_H
+#ifndef _LIBCPP___CXX03___RANDOM_KNUTH_B_H
+#define _LIBCPP___CXX03___RANDOM_KNUTH_B_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/linear_congruential_engine.h>
@@ -23,4 +23,4 @@ typedef shuffle_order_engine<minstd_rand0, 256> knuth_b;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_KNUTH_B_H
+#endif // _LIBCPP___CXX03___RANDOM_KNUTH_B_H
diff --git a/libcxx/include/__cxx03/__random/linear_congruential_engine.h b/libcxx/include/__cxx03/__random/linear_congruential_engine.h
index 94d140ae1a2d0..e6025571e5cc1 100644
--- a/libcxx/include/__cxx03/__random/linear_congruential_engine.h
+++ b/libcxx/include/__cxx03/__random/linear_congruential_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
-#define _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_seed_sequence.h>
@@ -384,4 +384,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_LINEAR_CONGRUENTIAL_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/log2.h b/libcxx/include/__cxx03/__random/log2.h
index 5dbaace22d86e..bea5f3c1ff4ba 100644
--- a/libcxx/include/__cxx03/__random/log2.h
+++ b/libcxx/include/__cxx03/__random/log2.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_LOG2_H
-#define _LIBCPP___RANDOM_LOG2_H
+#ifndef _LIBCPP___CXX03___RANDOM_LOG2_H
+#define _LIBCPP___CXX03___RANDOM_LOG2_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -63,4 +63,4 @@ struct __log2 {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_LOG2_H
+#endif // _LIBCPP___CXX03___RANDOM_LOG2_H
diff --git a/libcxx/include/__cxx03/__random/lognormal_distribution.h b/libcxx/include/__cxx03/__random/lognormal_distribution.h
index f1e537a21eb65..de046d9b63562 100644
--- a/libcxx/include/__cxx03/__random/lognormal_distribution.h
+++ b/libcxx/include/__cxx03/__random/lognormal_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_LOGNORMAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_LOGNORMAL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -123,4 +123,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_LOGNORMAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/mersenne_twister_engine.h b/libcxx/include/__cxx03/__random/mersenne_twister_engine.h
index 16deb8189dc46..d715b6ffd8dd4 100644
--- a/libcxx/include/__cxx03/__random/mersenne_twister_engine.h
+++ b/libcxx/include/__cxx03/__random/mersenne_twister_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
-#define _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_MERSENNE_TWISTER_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_MERSENNE_TWISTER_ENGINE_H
 
 #include <__cxx03/__algorithm/equal.h>
 #include <__cxx03/__algorithm/min.h>
@@ -911,4 +911,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_MERSENNE_TWISTER_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_MERSENNE_TWISTER_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/negative_binomial_distribution.h b/libcxx/include/__cxx03/__random/negative_binomial_distribution.h
index 6a99c87f5296b..f50b38218710f 100644
--- a/libcxx/include/__cxx03/__random/negative_binomial_distribution.h
+++ b/libcxx/include/__cxx03/__random/negative_binomial_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -154,4 +154,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_NEGATIVE_BINOMIAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/normal_distribution.h b/libcxx/include/__cxx03/__random/normal_distribution.h
index 95e4f5da49eb4..f9700c48ade05 100644
--- a/libcxx/include/__cxx03/__random/normal_distribution.h
+++ b/libcxx/include/__cxx03/__random/normal_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_NORMAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_NORMAL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/is_valid.h>
@@ -173,4 +173,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_NORMAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_NORMAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h b/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
index 5bd53b81cbfce..2036bb29ce1a6 100644
--- a/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
+++ b/libcxx/include/__cxx03/__random/piecewise_constant_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
 
 #include <__cxx03/__algorithm/upper_bound.h>
 #include <__cxx03/__config>
@@ -299,4 +299,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h b/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
index faf845321fec8..c871d01c37b41 100644
--- a/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
+++ b/libcxx/include/__cxx03/__random/piecewise_linear_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
 
 #include <__cxx03/__algorithm/upper_bound.h>
 #include <__cxx03/__config>
@@ -312,4 +312,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/poisson_distribution.h b/libcxx/include/__cxx03/__random/poisson_distribution.h
index 7fb4b9681d70d..fdb30042e4a3c 100644
--- a/libcxx/include/__cxx03/__random/poisson_distribution.h
+++ b/libcxx/include/__cxx03/__random/poisson_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_POISSON_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_POISSON_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/clamp_to_integral.h>
@@ -238,4 +238,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_POISSON_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/random_device.h b/libcxx/include/__cxx03/__random/random_device.h
index 2f95979bf5491..35874bc492a0a 100644
--- a/libcxx/include/__cxx03/__random/random_device.h
+++ b/libcxx/include/__cxx03/__random/random_device.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_RANDOM_DEVICE_H
-#define _LIBCPP___RANDOM_RANDOM_DEVICE_H
+#ifndef _LIBCPP___CXX03___RANDOM_RANDOM_DEVICE_H
+#define _LIBCPP___CXX03___RANDOM_RANDOM_DEVICE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/string>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_RANDOM_DEVICE_H
+#endif // _LIBCPP___CXX03___RANDOM_RANDOM_DEVICE_H
diff --git a/libcxx/include/__cxx03/__random/ranlux.h b/libcxx/include/__cxx03/__random/ranlux.h
index e99773cae392f..2c808f50e6382 100644
--- a/libcxx/include/__cxx03/__random/ranlux.h
+++ b/libcxx/include/__cxx03/__random/ranlux.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_RANLUX_H
-#define _LIBCPP___RANDOM_RANLUX_H
+#ifndef _LIBCPP___CXX03___RANDOM_RANLUX_H
+#define _LIBCPP___CXX03___RANDOM_RANLUX_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/discard_block_engine.h>
@@ -28,4 +28,4 @@ typedef discard_block_engine<ranlux48_base, 389, 11> ranlux48;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANDOM_RANLUX_H
+#endif // _LIBCPP___CXX03___RANDOM_RANLUX_H
diff --git a/libcxx/include/__cxx03/__random/seed_seq.h b/libcxx/include/__cxx03/__random/seed_seq.h
index 8ccd656b47e88..617ee66c819d8 100644
--- a/libcxx/include/__cxx03/__random/seed_seq.h
+++ b/libcxx/include/__cxx03/__random/seed_seq.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_SEED_SEQ_H
-#define _LIBCPP___RANDOM_SEED_SEQ_H
+#ifndef _LIBCPP___CXX03___RANDOM_SEED_SEQ_H
+#define _LIBCPP___CXX03___RANDOM_SEED_SEQ_H
 
 #include <__cxx03/__algorithm/copy.h>
 #include <__cxx03/__algorithm/fill.h>
@@ -163,4 +163,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_SEED_SEQ_H
+#endif // _LIBCPP___CXX03___RANDOM_SEED_SEQ_H
diff --git a/libcxx/include/__cxx03/__random/shuffle_order_engine.h b/libcxx/include/__cxx03/__random/shuffle_order_engine.h
index 8eb57c9ec56db..77eb0cf1fec87 100644
--- a/libcxx/include/__cxx03/__random/shuffle_order_engine.h
+++ b/libcxx/include/__cxx03/__random/shuffle_order_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
-#define _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_SHUFFLE_ORDER_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_SHUFFLE_ORDER_ENGINE_H
 
 #include <__cxx03/__algorithm/equal.h>
 #include <__cxx03/__config>
@@ -227,4 +227,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_SHUFFLE_ORDER_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_SHUFFLE_ORDER_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/student_t_distribution.h b/libcxx/include/__cxx03/__random/student_t_distribution.h
index 32a1c3c7381ff..6c71d1a1a4fb8 100644
--- a/libcxx/include/__cxx03/__random/student_t_distribution.h
+++ b/libcxx/include/__cxx03/__random/student_t_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_STUDENT_T_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_STUDENT_T_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/gamma_distribution.h>
@@ -129,4 +129,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_STUDENT_T_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_STUDENT_T_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h b/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
index c539994be1c80..3a30e41bbb460 100644
--- a/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
+++ b/libcxx/include/__cxx03/__random/subtract_with_carry_engine.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
-#define _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
+#ifndef _LIBCPP___CXX03___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
+#define _LIBCPP___CXX03___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
 
 #include <__cxx03/__algorithm/equal.h>
 #include <__cxx03/__algorithm/min.h>
@@ -273,4 +273,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
+#endif // _LIBCPP___CXX03___RANDOM_SUBTRACT_WITH_CARRY_ENGINE_H
diff --git a/libcxx/include/__cxx03/__random/uniform_int_distribution.h b/libcxx/include/__cxx03/__random/uniform_int_distribution.h
index 8e509fad15774..2e4656bd503f6 100644
--- a/libcxx/include/__cxx03/__random/uniform_int_distribution.h
+++ b/libcxx/include/__cxx03/__random/uniform_int_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_UNIFORM_INT_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_UNIFORM_INT_DISTRIBUTION_H
 
 #include <__cxx03/__bit/countl.h>
 #include <__cxx03/__config>
@@ -261,4 +261,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_UNIFORM_INT_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_UNIFORM_INT_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h b/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
index 74bbea38bf207..37c10a1ddfbd9 100644
--- a/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
+++ b/libcxx/include/__cxx03/__random/uniform_random_bit_generator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
-#define _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
+#ifndef _LIBCPP___CXX03___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
+#define _LIBCPP___CXX03___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__concepts/invocable.h>
@@ -41,4 +41,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
+#endif // _LIBCPP___CXX03___RANDOM_UNIFORM_RANDOM_BIT_GENERATOR_H
diff --git a/libcxx/include/__cxx03/__random/uniform_real_distribution.h b/libcxx/include/__cxx03/__random/uniform_real_distribution.h
index 64c978906a4ce..623344455ee31 100644
--- a/libcxx/include/__cxx03/__random/uniform_real_distribution.h
+++ b/libcxx/include/__cxx03/__random/uniform_real_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/generate_canonical.h>
@@ -135,4 +135,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_UNIFORM_REAL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__random/weibull_distribution.h b/libcxx/include/__cxx03/__random/weibull_distribution.h
index 6c5937aaad9e7..83bb88d50f626 100644
--- a/libcxx/include/__cxx03/__random/weibull_distribution.h
+++ b/libcxx/include/__cxx03/__random/weibull_distribution.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
-#define _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
+#ifndef _LIBCPP___CXX03___RANDOM_WEIBULL_DISTRIBUTION_H
+#define _LIBCPP___CXX03___RANDOM_WEIBULL_DISTRIBUTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__random/exponential_distribution.h>
@@ -129,4 +129,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANDOM_WEIBULL_DISTRIBUTION_H
+#endif // _LIBCPP___CXX03___RANDOM_WEIBULL_DISTRIBUTION_H
diff --git a/libcxx/include/__cxx03/__ranges/access.h b/libcxx/include/__cxx03/__ranges/access.h
index a6a0470afeacd..23b61946a16bb 100644
--- a/libcxx/include/__cxx03/__ranges/access.h
+++ b/libcxx/include/__cxx03/__ranges/access.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ACCESS_H
-#define _LIBCPP___RANGES_ACCESS_H
+#ifndef _LIBCPP___CXX03___RANGES_ACCESS_H
+#define _LIBCPP___CXX03___RANGES_ACCESS_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__config>
@@ -207,4 +207,4 @@ inline constexpr auto cend = __cend::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_ACCESS_H
+#endif // _LIBCPP___CXX03___RANGES_ACCESS_H
diff --git a/libcxx/include/__cxx03/__ranges/all.h b/libcxx/include/__cxx03/__ranges/all.h
index 0cb834fc41a35..57497f83478b5 100644
--- a/libcxx/include/__cxx03/__ranges/all.h
+++ b/libcxx/include/__cxx03/__ranges/all.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ALL_H
-#define _LIBCPP___RANGES_ALL_H
+#ifndef _LIBCPP___CXX03___RANGES_ALL_H
+#define _LIBCPP___CXX03___RANGES_ALL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/compose.h>         // TODO(modules): Those should not be required
@@ -75,4 +75,4 @@ using all_t = decltype(views::all(std::declval<_Range>()));
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_ALL_H
+#endif // _LIBCPP___CXX03___RANGES_ALL_H
diff --git a/libcxx/include/__cxx03/__ranges/as_rvalue_view.h b/libcxx/include/__cxx03/__ranges/as_rvalue_view.h
index 011939f19ab0f..42234dabaf081 100644
--- a/libcxx/include/__cxx03/__ranges/as_rvalue_view.h
+++ b/libcxx/include/__cxx03/__ranges/as_rvalue_view.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_AS_RVALUE_H
-#define _LIBCPP___RANGES_AS_RVALUE_H
+#ifndef _LIBCPP___CXX03___RANGES_AS_RVALUE_H
+#define _LIBCPP___CXX03___RANGES_AS_RVALUE_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -139,4 +139,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_AS_RVALUE_H
+#endif // _LIBCPP___CXX03___RANGES_AS_RVALUE_H
diff --git a/libcxx/include/__cxx03/__ranges/chunk_by_view.h b/libcxx/include/__cxx03/__ranges/chunk_by_view.h
index 8f17d56e5e93b..2b5c4d607c417 100644
--- a/libcxx/include/__cxx03/__ranges/chunk_by_view.h
+++ b/libcxx/include/__cxx03/__ranges/chunk_by_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_CHUNK_BY_VIEW_H
-#define _LIBCPP___RANGES_CHUNK_BY_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_CHUNK_BY_VIEW_H
+#define _LIBCPP___CXX03___RANGES_CHUNK_BY_VIEW_H
 
 #include <__cxx03/__algorithm/ranges_adjacent_find.h>
 #include <__cxx03/__assert>
@@ -232,4 +232,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_CHUNK_BY_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_CHUNK_BY_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/common_view.h b/libcxx/include/__cxx03/__ranges/common_view.h
index 5af8de542dad9..e7022ff63abba 100644
--- a/libcxx/include/__cxx03/__ranges/common_view.h
+++ b/libcxx/include/__cxx03/__ranges/common_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_COMMON_VIEW_H
-#define _LIBCPP___RANGES_COMMON_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_COMMON_VIEW_H
+#define _LIBCPP___CXX03___RANGES_COMMON_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/copyable.h>
@@ -139,4 +139,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_COMMON_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_COMMON_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/concepts.h b/libcxx/include/__cxx03/__ranges/concepts.h
index 238d02cc2f68f..958a81c19bfa7 100644
--- a/libcxx/include/__cxx03/__ranges/concepts.h
+++ b/libcxx/include/__cxx03/__ranges/concepts.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_CONCEPTS_H
-#define _LIBCPP___RANGES_CONCEPTS_H
+#ifndef _LIBCPP___CXX03___RANGES_CONCEPTS_H
+#define _LIBCPP___CXX03___RANGES_CONCEPTS_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/movable.h>
@@ -139,4 +139,4 @@ concept viewable_range =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_CONCEPTS_H
+#endif // _LIBCPP___CXX03___RANGES_CONCEPTS_H
diff --git a/libcxx/include/__cxx03/__ranges/container_compatible_range.h b/libcxx/include/__cxx03/__ranges/container_compatible_range.h
index 7bf8218e667d0..9e47449fe50ab 100644
--- a/libcxx/include/__cxx03/__ranges/container_compatible_range.h
+++ b/libcxx/include/__cxx03/__ranges/container_compatible_range.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
-#define _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
+#ifndef _LIBCPP___CXX03___RANGES_CONTAINER_COMPATIBLE_RANGE_H
+#define _LIBCPP___CXX03___RANGES_CONTAINER_COMPATIBLE_RANGE_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -30,4 +30,4 @@ concept _ContainerCompatibleRange =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_CONTAINER_COMPATIBLE_RANGE_H
+#endif // _LIBCPP___CXX03___RANGES_CONTAINER_COMPATIBLE_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/counted.h b/libcxx/include/__cxx03/__ranges/counted.h
index a20a4824b03cd..a4a11cd0064b9 100644
--- a/libcxx/include/__cxx03/__ranges/counted.h
+++ b/libcxx/include/__cxx03/__ranges/counted.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_COUNTED_H
-#define _LIBCPP___RANGES_COUNTED_H
+#ifndef _LIBCPP___CXX03___RANGES_COUNTED_H
+#define _LIBCPP___CXX03___RANGES_COUNTED_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__config>
@@ -86,4 +86,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_COUNTED_H
+#endif // _LIBCPP___CXX03___RANGES_COUNTED_H
diff --git a/libcxx/include/__cxx03/__ranges/dangling.h b/libcxx/include/__cxx03/__ranges/dangling.h
index fee2712ac829b..d57eca19b1c30 100644
--- a/libcxx/include/__cxx03/__ranges/dangling.h
+++ b/libcxx/include/__cxx03/__ranges/dangling.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_DANGLING_H
-#define _LIBCPP___RANGES_DANGLING_H
+#ifndef _LIBCPP___CXX03___RANGES_DANGLING_H
+#define _LIBCPP___CXX03___RANGES_DANGLING_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__ranges/access.h>
@@ -39,4 +39,4 @@ using borrowed_iterator_t = _If<borrowed_range<_Rp>, iterator_t<_Rp>, dangling>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_DANGLING_H
+#endif // _LIBCPP___CXX03___RANGES_DANGLING_H
diff --git a/libcxx/include/__cxx03/__ranges/data.h b/libcxx/include/__cxx03/__ranges/data.h
index 2e38f210804da..80e473d218614 100644
--- a/libcxx/include/__cxx03/__ranges/data.h
+++ b/libcxx/include/__cxx03/__ranges/data.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_DATA_H
-#define _LIBCPP___RANGES_DATA_H
+#ifndef _LIBCPP___CXX03___RANGES_DATA_H
+#define _LIBCPP___CXX03___RANGES_DATA_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__config>
@@ -99,4 +99,4 @@ inline constexpr auto cdata = __cdata::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_DATA_H
+#endif // _LIBCPP___CXX03___RANGES_DATA_H
diff --git a/libcxx/include/__cxx03/__ranges/drop_view.h b/libcxx/include/__cxx03/__ranges/drop_view.h
index 9d1336fc877ed..78e0a26e4c7e9 100644
--- a/libcxx/include/__cxx03/__ranges/drop_view.h
+++ b/libcxx/include/__cxx03/__ranges/drop_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_DROP_VIEW_H
-#define _LIBCPP___RANGES_DROP_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_DROP_VIEW_H
+#define _LIBCPP___CXX03___RANGES_DROP_VIEW_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__assert>
@@ -326,4 +326,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_DROP_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_DROP_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/drop_while_view.h b/libcxx/include/__cxx03/__ranges/drop_while_view.h
index 0542908a6cb5a..604dd29742a90 100644
--- a/libcxx/include/__cxx03/__ranges/drop_while_view.h
+++ b/libcxx/include/__cxx03/__ranges/drop_while_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_DROP_WHILE_VIEW_H
-#define _LIBCPP___RANGES_DROP_WHILE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_DROP_WHILE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_DROP_WHILE_VIEW_H
 
 #include <__cxx03/__algorithm/ranges_find_if_not.h>
 #include <__cxx03/__assert>
@@ -133,4 +133,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_DROP_WHILE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_DROP_WHILE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/elements_view.h b/libcxx/include/__cxx03/__ranges/elements_view.h
index 515562719039b..9db16cde83be5 100644
--- a/libcxx/include/__cxx03/__ranges/elements_view.h
+++ b/libcxx/include/__cxx03/__ranges/elements_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ELEMENTS_VIEW_H
-#define _LIBCPP___RANGES_ELEMENTS_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_ELEMENTS_VIEW_H
+#define _LIBCPP___CXX03___RANGES_ELEMENTS_VIEW_H
 
 #include <__cxx03/__compare/three_way_comparable.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -415,4 +415,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_ELEMENTS_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_ELEMENTS_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/empty.h b/libcxx/include/__cxx03/__ranges/empty.h
index 957fed827b404..c72a17ecaf19d 100644
--- a/libcxx/include/__cxx03/__ranges/empty.h
+++ b/libcxx/include/__cxx03/__ranges/empty.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_EMPTY_H
-#define _LIBCPP___RANGES_EMPTY_H
+#ifndef _LIBCPP___CXX03___RANGES_EMPTY_H
+#define _LIBCPP___CXX03___RANGES_EMPTY_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__config>
@@ -68,4 +68,4 @@ inline constexpr auto empty = __empty::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_EMPTY_H
+#endif // _LIBCPP___CXX03___RANGES_EMPTY_H
diff --git a/libcxx/include/__cxx03/__ranges/empty_view.h b/libcxx/include/__cxx03/__ranges/empty_view.h
index 265575baf678d..11a77de6a3392 100644
--- a/libcxx/include/__cxx03/__ranges/empty_view.h
+++ b/libcxx/include/__cxx03/__ranges/empty_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_EMPTY_VIEW_H
-#define _LIBCPP___RANGES_EMPTY_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_EMPTY_VIEW_H
+#define _LIBCPP___CXX03___RANGES_EMPTY_VIEW_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__ranges/enable_borrowed_range.h>
@@ -51,4 +51,4 @@ inline constexpr empty_view<_Tp> empty{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_EMPTY_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_EMPTY_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h b/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
index 81051f67c003c..462053e35535d 100644
--- a/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
+++ b/libcxx/include/__cxx03/__ranges/enable_borrowed_range.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
-#define _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
+#ifndef _LIBCPP___CXX03___RANGES_ENABLE_BORROWED_RANGE_H
+#define _LIBCPP___CXX03___RANGES_ENABLE_BORROWED_RANGE_H
 
 // These customization variables are used in <span> and <string_view>. The
 // separate header is used to avoid including the entire <ranges> header in
@@ -37,4 +37,4 @@ inline constexpr bool enable_borrowed_range = false;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_ENABLE_BORROWED_RANGE_H
+#endif // _LIBCPP___CXX03___RANGES_ENABLE_BORROWED_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/enable_view.h b/libcxx/include/__cxx03/__ranges/enable_view.h
index 4697dab872bcf..b8d78c29064b7 100644
--- a/libcxx/include/__cxx03/__ranges/enable_view.h
+++ b/libcxx/include/__cxx03/__ranges/enable_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ENABLE_VIEW_H
-#define _LIBCPP___RANGES_ENABLE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_ENABLE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_ENABLE_VIEW_H
 
 #include <__cxx03/__concepts/derived_from.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -48,4 +48,4 @@ inline constexpr bool enable_view = derived_from<_Tp, view_base> || requires {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_ENABLE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_ENABLE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/filter_view.h b/libcxx/include/__cxx03/__ranges/filter_view.h
index 5015140c39ae7..b9beadb136652 100644
--- a/libcxx/include/__cxx03/__ranges/filter_view.h
+++ b/libcxx/include/__cxx03/__ranges/filter_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_FILTER_VIEW_H
-#define _LIBCPP___RANGES_FILTER_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_FILTER_VIEW_H
+#define _LIBCPP___CXX03___RANGES_FILTER_VIEW_H
 
 #include <__cxx03/__algorithm/ranges_find_if.h>
 #include <__cxx03/__assert>
@@ -257,4 +257,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_FILTER_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_FILTER_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/from_range.h b/libcxx/include/__cxx03/__ranges/from_range.h
index 55ff79edd513d..2f0c827f827fe 100644
--- a/libcxx/include/__cxx03/__ranges/from_range.h
+++ b/libcxx/include/__cxx03/__ranges/from_range.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_FROM_RANGE_H
-#define _LIBCPP___RANGES_FROM_RANGE_H
+#ifndef _LIBCPP___CXX03___RANGES_FROM_RANGE_H
+#define _LIBCPP___CXX03___RANGES_FROM_RANGE_H
 
 #include <__cxx03/__config>
 
@@ -30,4 +30,4 @@ inline constexpr from_range_t from_range{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_FROM_RANGE_H
+#endif // _LIBCPP___CXX03___RANGES_FROM_RANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/iota_view.h b/libcxx/include/__cxx03/__ranges/iota_view.h
index 5bc2394dc3dc2..5fa1e8151af20 100644
--- a/libcxx/include/__cxx03/__ranges/iota_view.h
+++ b/libcxx/include/__cxx03/__ranges/iota_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_IOTA_VIEW_H
-#define _LIBCPP___RANGES_IOTA_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_IOTA_VIEW_H
+#define _LIBCPP___CXX03___RANGES_IOTA_VIEW_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__compare/three_way_comparable.h>
@@ -401,4 +401,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_IOTA_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_IOTA_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/istream_view.h b/libcxx/include/__cxx03/__ranges/istream_view.h
index 8db34132af59c..b54dc137690a0 100644
--- a/libcxx/include/__cxx03/__ranges/istream_view.h
+++ b/libcxx/include/__cxx03/__ranges/istream_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ISTREAM_VIEW_H
-#define _LIBCPP___RANGES_ISTREAM_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_ISTREAM_VIEW_H
+#define _LIBCPP___CXX03___RANGES_ISTREAM_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/derived_from.h>
@@ -138,4 +138,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 20
 
-#endif // _LIBCPP___RANGES_ISTREAM_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_ISTREAM_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/join_view.h b/libcxx/include/__cxx03/__ranges/join_view.h
index ea47eeefcf951..3496fc9810704 100644
--- a/libcxx/include/__cxx03/__ranges/join_view.h
+++ b/libcxx/include/__cxx03/__ranges/join_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_JOIN_VIEW_H
-#define _LIBCPP___RANGES_JOIN_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_JOIN_VIEW_H
+#define _LIBCPP___CXX03___RANGES_JOIN_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -420,4 +420,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_JOIN_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_JOIN_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/lazy_split_view.h b/libcxx/include/__cxx03/__ranges/lazy_split_view.h
index 9e70c237b4fb2..8dc05e0ec6f95 100644
--- a/libcxx/include/__cxx03/__ranges/lazy_split_view.h
+++ b/libcxx/include/__cxx03/__ranges/lazy_split_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
-#define _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_LAZY_SPLIT_VIEW_H
+#define _LIBCPP___CXX03___RANGES_LAZY_SPLIT_VIEW_H
 
 #include <__cxx03/__algorithm/ranges_find.h>
 #include <__cxx03/__algorithm/ranges_mismatch.h>
@@ -438,4 +438,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_LAZY_SPLIT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/movable_box.h b/libcxx/include/__cxx03/__ranges/movable_box.h
index ab6f407ed537b..7dc8b95b28bb4 100644
--- a/libcxx/include/__cxx03/__ranges/movable_box.h
+++ b/libcxx/include/__cxx03/__ranges/movable_box.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_MOVABLE_BOX_H
-#define _LIBCPP___RANGES_MOVABLE_BOX_H
+#ifndef _LIBCPP___CXX03___RANGES_MOVABLE_BOX_H
+#define _LIBCPP___CXX03___RANGES_MOVABLE_BOX_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/copyable.h>
@@ -244,4 +244,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_MOVABLE_BOX_H
+#endif // _LIBCPP___CXX03___RANGES_MOVABLE_BOX_H
diff --git a/libcxx/include/__cxx03/__ranges/non_propagating_cache.h b/libcxx/include/__cxx03/__ranges/non_propagating_cache.h
index 6f8c8600bab10..0db1914e592ba 100644
--- a/libcxx/include/__cxx03/__ranges/non_propagating_cache.h
+++ b/libcxx/include/__cxx03/__ranges/non_propagating_cache.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
-#define _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
+#ifndef _LIBCPP___CXX03___RANGES_NON_PROPAGATING_CACHE_H
+#define _LIBCPP___CXX03___RANGES_NON_PROPAGATING_CACHE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__iterator/concepts.h>        // indirectly_readable
@@ -100,4 +100,4 @@ struct __empty_cache {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_NON_PROPAGATING_CACHE_H
+#endif // _LIBCPP___CXX03___RANGES_NON_PROPAGATING_CACHE_H
diff --git a/libcxx/include/__cxx03/__ranges/owning_view.h b/libcxx/include/__cxx03/__ranges/owning_view.h
index ab7e4e3eeb974..5519d2c795c85 100644
--- a/libcxx/include/__cxx03/__ranges/owning_view.h
+++ b/libcxx/include/__cxx03/__ranges/owning_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_OWNING_VIEW_H
-#define _LIBCPP___RANGES_OWNING_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_OWNING_VIEW_H
+#define _LIBCPP___CXX03___RANGES_OWNING_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/movable.h>
@@ -113,4 +113,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_OWNING_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_OWNING_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/range_adaptor.h b/libcxx/include/__cxx03/__ranges/range_adaptor.h
index ef7301d695848..ef08464234b57 100644
--- a/libcxx/include/__cxx03/__ranges/range_adaptor.h
+++ b/libcxx/include/__cxx03/__ranges/range_adaptor.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_RANGE_ADAPTOR_H
-#define _LIBCPP___RANGES_RANGE_ADAPTOR_H
+#ifndef _LIBCPP___CXX03___RANGES_RANGE_ADAPTOR_H
+#define _LIBCPP___CXX03___RANGES_RANGE_ADAPTOR_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/derived_from.h>
@@ -98,4 +98,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_RANGE_ADAPTOR_H
+#endif // _LIBCPP___CXX03___RANGES_RANGE_ADAPTOR_H
diff --git a/libcxx/include/__cxx03/__ranges/rbegin.h b/libcxx/include/__cxx03/__ranges/rbegin.h
index 3e114605baf37..8f987319cc4fe 100644
--- a/libcxx/include/__cxx03/__ranges/rbegin.h
+++ b/libcxx/include/__cxx03/__ranges/rbegin.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_RBEGIN_H
-#define _LIBCPP___RANGES_RBEGIN_H
+#ifndef _LIBCPP___CXX03___RANGES_RBEGIN_H
+#define _LIBCPP___CXX03___RANGES_RBEGIN_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -117,4 +117,4 @@ inline constexpr auto crbegin = __crbegin::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_RBEGIN_H
+#endif // _LIBCPP___CXX03___RANGES_RBEGIN_H
diff --git a/libcxx/include/__cxx03/__ranges/ref_view.h b/libcxx/include/__cxx03/__ranges/ref_view.h
index 9fd2835cf385c..43c3bebe34dc3 100644
--- a/libcxx/include/__cxx03/__ranges/ref_view.h
+++ b/libcxx/include/__cxx03/__ranges/ref_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_REF_VIEW_H
-#define _LIBCPP___RANGES_REF_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_REF_VIEW_H
+#define _LIBCPP___CXX03___RANGES_REF_VIEW_H
 
 #include <__cxx03/__concepts/convertible_to.h>
 #include <__cxx03/__concepts/different_from.h>
@@ -86,4 +86,4 @@ inline constexpr bool enable_borrowed_range<ref_view<_Tp>> = true;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_REF_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_REF_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/rend.h b/libcxx/include/__cxx03/__ranges/rend.h
index 9d663e54f4619..aa7e4a0ba396a 100644
--- a/libcxx/include/__cxx03/__ranges/rend.h
+++ b/libcxx/include/__cxx03/__ranges/rend.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_REND_H
-#define _LIBCPP___RANGES_REND_H
+#ifndef _LIBCPP___CXX03___RANGES_REND_H
+#define _LIBCPP___CXX03___RANGES_REND_H
 
 #include <__cxx03/__concepts/class_or_enum.h>
 #include <__cxx03/__concepts/same_as.h>
@@ -120,4 +120,4 @@ inline constexpr auto crend = __crend::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_REND_H
+#endif // _LIBCPP___CXX03___RANGES_REND_H
diff --git a/libcxx/include/__cxx03/__ranges/repeat_view.h b/libcxx/include/__cxx03/__ranges/repeat_view.h
index fae883e01bcfe..a0cd065b9f2aa 100644
--- a/libcxx/include/__cxx03/__ranges/repeat_view.h
+++ b/libcxx/include/__cxx03/__ranges/repeat_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_REPEAT_VIEW_H
-#define _LIBCPP___RANGES_REPEAT_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_REPEAT_VIEW_H
+#define _LIBCPP___CXX03___RANGES_REPEAT_VIEW_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/constructible.h>
@@ -263,4 +263,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_REPEAT_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_REPEAT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/reverse_view.h b/libcxx/include/__cxx03/__ranges/reverse_view.h
index 7513114779c5a..53a0e9d213c5b 100644
--- a/libcxx/include/__cxx03/__ranges/reverse_view.h
+++ b/libcxx/include/__cxx03/__ranges/reverse_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_REVERSE_VIEW_H
-#define _LIBCPP___RANGES_REVERSE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_REVERSE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_REVERSE_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__config>
@@ -200,4 +200,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_REVERSE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_REVERSE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/single_view.h b/libcxx/include/__cxx03/__ranges/single_view.h
index 34054fa0f9d42..3e43302f213b1 100644
--- a/libcxx/include/__cxx03/__ranges/single_view.h
+++ b/libcxx/include/__cxx03/__ranges/single_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_SINGLE_VIEW_H
-#define _LIBCPP___RANGES_SINGLE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_SINGLE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_SINGLE_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__config>
@@ -108,4 +108,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_SINGLE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_SINGLE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/size.h b/libcxx/include/__cxx03/__ranges/size.h
index 84b165551d921..f311f76df15cc 100644
--- a/libcxx/include/__cxx03/__ranges/size.h
+++ b/libcxx/include/__cxx03/__ranges/size.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_SIZE_H
-#define _LIBCPP___RANGES_SIZE_H
+#ifndef _LIBCPP___CXX03___RANGES_SIZE_H
+#define _LIBCPP___CXX03___RANGES_SIZE_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__concepts/class_or_enum.h>
@@ -134,4 +134,4 @@ inline constexpr auto ssize = __ssize::__fn{};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_SIZE_H
+#endif // _LIBCPP___CXX03___RANGES_SIZE_H
diff --git a/libcxx/include/__cxx03/__ranges/split_view.h b/libcxx/include/__cxx03/__ranges/split_view.h
index f23e7878e349c..76ab9f485bd7c 100644
--- a/libcxx/include/__cxx03/__ranges/split_view.h
+++ b/libcxx/include/__cxx03/__ranges/split_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_SPLIT_VIEW_H
-#define _LIBCPP___RANGES_SPLIT_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_SPLIT_VIEW_H
+#define _LIBCPP___CXX03___RANGES_SPLIT_VIEW_H
 
 #include <__cxx03/__algorithm/ranges_search.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -229,4 +229,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_SPLIT_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_SPLIT_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/subrange.h b/libcxx/include/__cxx03/__ranges/subrange.h
index fa985a5c365af..e077d221ed202 100644
--- a/libcxx/include/__cxx03/__ranges/subrange.h
+++ b/libcxx/include/__cxx03/__ranges/subrange.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_SUBRANGE_H
-#define _LIBCPP___RANGES_SUBRANGE_H
+#ifndef _LIBCPP___CXX03___RANGES_SUBRANGE_H
+#define _LIBCPP___CXX03___RANGES_SUBRANGE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/constructible.h>
@@ -270,4 +270,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_SUBRANGE_H
+#endif // _LIBCPP___CXX03___RANGES_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__ranges/take_view.h b/libcxx/include/__cxx03/__ranges/take_view.h
index e909158f5c417..d3cadd55ae1c1 100644
--- a/libcxx/include/__cxx03/__ranges/take_view.h
+++ b/libcxx/include/__cxx03/__ranges/take_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_TAKE_VIEW_H
-#define _LIBCPP___RANGES_TAKE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_TAKE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_TAKE_VIEW_H
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__algorithm/ranges_min.h>
@@ -366,4 +366,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_TAKE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_TAKE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/take_while_view.h b/libcxx/include/__cxx03/__ranges/take_while_view.h
index 3968777240ed2..7dec8490e0754 100644
--- a/libcxx/include/__cxx03/__ranges/take_while_view.h
+++ b/libcxx/include/__cxx03/__ranges/take_while_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
-#define _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_TAKE_WHILE_VIEW_H
+#define _LIBCPP___CXX03___RANGES_TAKE_WHILE_VIEW_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/convertible_to.h>
@@ -167,4 +167,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_TAKE_WHILE_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_TAKE_WHILE_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/to.h b/libcxx/include/__cxx03/__ranges/to.h
index 682b811946a62..f50f43cccc53a 100644
--- a/libcxx/include/__cxx03/__ranges/to.h
+++ b/libcxx/include/__cxx03/__ranges/to.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_TO_H
-#define _LIBCPP___RANGES_TO_H
+#ifndef _LIBCPP___CXX03___RANGES_TO_H
+#define _LIBCPP___CXX03___RANGES_TO_H
 
 #include <__cxx03/__algorithm/ranges_copy.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -242,4 +242,4 @@ template <template <class...> class _Container, class... _Args>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_TO_H
+#endif // _LIBCPP___CXX03___RANGES_TO_H
diff --git a/libcxx/include/__cxx03/__ranges/transform_view.h b/libcxx/include/__cxx03/__ranges/transform_view.h
index 78012cde38854..b4e7dde7ee91b 100644
--- a/libcxx/include/__cxx03/__ranges/transform_view.h
+++ b/libcxx/include/__cxx03/__ranges/transform_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_TRANSFORM_VIEW_H
-#define _LIBCPP___RANGES_TRANSFORM_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_TRANSFORM_VIEW_H
+#define _LIBCPP___CXX03___RANGES_TRANSFORM_VIEW_H
 
 #include <__cxx03/__compare/three_way_comparable.h>
 #include <__cxx03/__concepts/constructible.h>
@@ -414,4 +414,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_TRANSFORM_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_TRANSFORM_VIEW_H
diff --git a/libcxx/include/__cxx03/__ranges/view_interface.h b/libcxx/include/__cxx03/__ranges/view_interface.h
index 23cdcdaaad43c..956049eab6d4e 100644
--- a/libcxx/include/__cxx03/__ranges/view_interface.h
+++ b/libcxx/include/__cxx03/__ranges/view_interface.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_VIEW_INTERFACE_H
-#define _LIBCPP___RANGES_VIEW_INTERFACE_H
+#ifndef _LIBCPP___CXX03___RANGES_VIEW_INTERFACE_H
+#define _LIBCPP___CXX03___RANGES_VIEW_INTERFACE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__concepts/derived_from.h>
@@ -167,4 +167,4 @@ class view_interface {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_VIEW_INTERFACE_H
+#endif // _LIBCPP___CXX03___RANGES_VIEW_INTERFACE_H
diff --git a/libcxx/include/__cxx03/__ranges/views.h b/libcxx/include/__cxx03/__ranges/views.h
index 2809271f6e1a9..91fa1a933c22a 100644
--- a/libcxx/include/__cxx03/__ranges/views.h
+++ b/libcxx/include/__cxx03/__ranges/views.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_VIEWS
-#define _LIBCPP___RANGES_VIEWS
+#ifndef _LIBCPP___CXX03___RANGES_VIEWS
+#define _LIBCPP___CXX03___RANGES_VIEWS
 
 #include <__cxx03/__config>
 
@@ -32,4 +32,4 @@ namespace views = ranges::views;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___RANGES_VIEWS
+#endif // _LIBCPP___CXX03___RANGES_VIEWS
diff --git a/libcxx/include/__cxx03/__ranges/zip_view.h b/libcxx/include/__cxx03/__ranges/zip_view.h
index 4d9f62647c8f2..3ba450db92192 100644
--- a/libcxx/include/__cxx03/__ranges/zip_view.h
+++ b/libcxx/include/__cxx03/__ranges/zip_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___RANGES_ZIP_VIEW_H
-#define _LIBCPP___RANGES_ZIP_VIEW_H
+#ifndef _LIBCPP___CXX03___RANGES_ZIP_VIEW_H
+#define _LIBCPP___CXX03___RANGES_ZIP_VIEW_H
 
 #include <__cxx03/__config>
 
@@ -513,4 +513,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___RANGES_ZIP_VIEW_H
+#endif // _LIBCPP___CXX03___RANGES_ZIP_VIEW_H
diff --git a/libcxx/include/__cxx03/__split_buffer b/libcxx/include/__cxx03/__split_buffer
index 133048bfea77b..c614704ce56b0 100644
--- a/libcxx/include/__cxx03/__split_buffer
+++ b/libcxx/include/__cxx03/__split_buffer
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SPLIT_BUFFER
-#define _LIBCPP___SPLIT_BUFFER
+#ifndef _LIBCPP___CXX03___SPLIT_BUFFER
+#define _LIBCPP___CXX03___SPLIT_BUFFER
 
 #include <__cxx03/__algorithm/max.h>
 #include <__cxx03/__algorithm/move.h>
@@ -582,4 +582,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___SPLIT_BUFFER
+#endif // _LIBCPP___CXX03___SPLIT_BUFFER
diff --git a/libcxx/include/__cxx03/__std_mbstate_t.h b/libcxx/include/__cxx03/__std_mbstate_t.h
index ac28555214188..7a81d1bd024ae 100644
--- a/libcxx/include/__cxx03/__std_mbstate_t.h
+++ b/libcxx/include/__cxx03/__std_mbstate_t.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STD_MBSTATE_T_H
-#define _LIBCPP___STD_MBSTATE_T_H
+#ifndef _LIBCPP___CXX03___STD_MBSTATE_T_H
+#define _LIBCPP___CXX03___STD_MBSTATE_T_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__mbstate_t.h>
@@ -26,4 +26,4 @@ using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STD_MBSTATE_T_H
+#endif // _LIBCPP___CXX03___STD_MBSTATE_T_H
diff --git a/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h b/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
index a8e4d2a90a53b..92b17350b3bbb 100644
--- a/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
+++ b/libcxx/include/__cxx03/__stop_token/atomic_unique_lock.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
-#define _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
+#define _LIBCPP___CXX03___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
 
 #include <__cxx03/__bit/popcount.h>
 #include <__cxx03/__config>
@@ -137,4 +137,4 @@ class _LIBCPP_AVAILABILITY_SYNC __atomic_unique_lock {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
+#endif // _LIBCPP___CXX03___STOP_TOKEN_ATOMIC_UNIQUE_GUARD_H
diff --git a/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h b/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
index ab11762802ac4..8c133ad30c609 100644
--- a/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
+++ b/libcxx/include/__cxx03/__stop_token/intrusive_list_view.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
-#define _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
+#define _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -82,4 +82,4 @@ struct __intrusive_list_view {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
+#endif // _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_LIST_VIEW_H
diff --git a/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h b/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
index 62eb946a5bbed..002cdad80f0ae 100644
--- a/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
+++ b/libcxx/include/__cxx03/__stop_token/intrusive_shared_ptr.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
-#define _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
+#define _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
 
 #include <__cxx03/__atomic/atomic.h>
 #include <__cxx03/__atomic/memory_order.h>
@@ -131,4 +131,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
+#endif // _LIBCPP___CXX03___STOP_TOKEN_INTRUSIVE_SHARED_PTR_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_callback.h b/libcxx/include/__cxx03/__stop_token/stop_callback.h
index afd194cca1d00..5a92b59e30b01 100644
--- a/libcxx/include/__cxx03/__stop_token/stop_callback.h
+++ b/libcxx/include/__cxx03/__stop_token/stop_callback.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_STOP_CALLBACK_H
-#define _LIBCPP___STOP_TOKEN_STOP_CALLBACK_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_STOP_CALLBACK_H
+#define _LIBCPP___CXX03___STOP_TOKEN_STOP_CALLBACK_H
 
 #include <__cxx03/__concepts/constructible.h>
 #include <__cxx03/__concepts/destructible.h>
@@ -99,4 +99,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+#endif // _LIBCPP___CXX03___STOP_TOKEN_STOP_CALLBACK_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_source.h b/libcxx/include/__cxx03/__stop_token/stop_source.h
index 0d8b714bfba4e..83fcfa548a859 100644
--- a/libcxx/include/__cxx03/__stop_token/stop_source.h
+++ b/libcxx/include/__cxx03/__stop_token/stop_source.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_STOP_SOURCE_H
-#define _LIBCPP___STOP_TOKEN_STOP_SOURCE_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_STOP_SOURCE_H
+#define _LIBCPP___CXX03___STOP_TOKEN_STOP_SOURCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__stop_token/intrusive_shared_ptr.h>
@@ -88,4 +88,4 @@ class _LIBCPP_AVAILABILITY_SYNC stop_source {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && !defined(_LIBCPP_HAS_NO_THREADS)
+#endif // _LIBCPP___CXX03___STOP_TOKEN_STOP_SOURCE_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_state.h b/libcxx/include/__cxx03/__stop_token/stop_state.h
index 39688917d1d47..2faee49440f8a 100644
--- a/libcxx/include/__cxx03/__stop_token/stop_state.h
+++ b/libcxx/include/__cxx03/__stop_token/stop_state.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_STOP_STATE_H
-#define _LIBCPP___STOP_TOKEN_STOP_STATE_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_STOP_STATE_H
+#define _LIBCPP___CXX03___STOP_TOKEN_STOP_STATE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -233,4 +233,4 @@ struct __intrusive_shared_ptr_traits<__stop_state> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STOP_TOKEN_STOP_STATE_H
+#endif // _LIBCPP___CXX03___STOP_TOKEN_STOP_STATE_H
diff --git a/libcxx/include/__cxx03/__stop_token/stop_token.h b/libcxx/include/__cxx03/__stop_token/stop_token.h
index bba5cd1c4796e..64f3e605c7886 100644
--- a/libcxx/include/__cxx03/__stop_token/stop_token.h
+++ b/libcxx/include/__cxx03/__stop_token/stop_token.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
-#define _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
+#ifndef _LIBCPP___CXX03___STOP_TOKEN_STOP_TOKEN_H
+#define _LIBCPP___CXX03___STOP_TOKEN_STOP_TOKEN_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__stop_token/intrusive_shared_ptr.h>
@@ -60,4 +60,4 @@ class _LIBCPP_AVAILABILITY_SYNC stop_token {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STOP_TOKEN_STOP_TOKEN_H
+#endif // _LIBCPP___CXX03___STOP_TOKEN_STOP_TOKEN_H
diff --git a/libcxx/include/__cxx03/__string/char_traits.h b/libcxx/include/__cxx03/__string/char_traits.h
index 69078d6615ea4..e962d986b2785 100644
--- a/libcxx/include/__cxx03/__string/char_traits.h
+++ b/libcxx/include/__cxx03/__string/char_traits.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STRING_CHAR_TRAITS_H
-#define _LIBCPP___STRING_CHAR_TRAITS_H
+#ifndef _LIBCPP___CXX03___STRING_CHAR_TRAITS_H
+#define _LIBCPP___CXX03___STRING_CHAR_TRAITS_H
 
 #include <__cxx03/__algorithm/fill_n.h>
 #include <__cxx03/__algorithm/find.h>
@@ -540,4 +540,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___STRING_CHAR_TRAITS_H
+#endif // _LIBCPP___CXX03___STRING_CHAR_TRAITS_H
diff --git a/libcxx/include/__cxx03/__string/constexpr_c_functions.h b/libcxx/include/__cxx03/__string/constexpr_c_functions.h
index b1e269d974087..5a0b4953de177 100644
--- a/libcxx/include/__cxx03/__string/constexpr_c_functions.h
+++ b/libcxx/include/__cxx03/__string/constexpr_c_functions.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
-#define _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
+#ifndef _LIBCPP___CXX03___STRING_CONSTEXPR_C_FUNCTIONS_H
+#define _LIBCPP___CXX03___STRING_CONSTEXPR_C_FUNCTIONS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/addressof.h>
@@ -231,4 +231,4 @@ __constexpr_memmove(_Tp* __dest, _Up* __src, __element_count __n) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___STRING_CONSTEXPR_C_FUNCTIONS_H
+#endif // _LIBCPP___CXX03___STRING_CONSTEXPR_C_FUNCTIONS_H
diff --git a/libcxx/include/__cxx03/__string/extern_template_lists.h b/libcxx/include/__cxx03/__string/extern_template_lists.h
index 09b3375aeecf6..27cfd560988cb 100644
--- a/libcxx/include/__cxx03/__string/extern_template_lists.h
+++ b/libcxx/include/__cxx03/__string/extern_template_lists.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
-#define _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
+#ifndef _LIBCPP___CXX03___STRING_EXTERN_TEMPLATE_LISTS_H
+#define _LIBCPP___CXX03___STRING_EXTERN_TEMPLATE_LISTS_H
 
 #include <__cxx03/__config>
 
@@ -130,4 +130,4 @@
 
 // clang-format on
 
-#endif // _LIBCPP___STRING_EXTERN_TEMPLATE_LISTS_H
+#endif // _LIBCPP___CXX03___STRING_EXTERN_TEMPLATE_LISTS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/gettod_zos.h b/libcxx/include/__cxx03/__support/ibm/gettod_zos.h
index 18797cc18e614..3bb9b942a3903 100644
--- a/libcxx/include/__cxx03/__support/ibm/gettod_zos.h
+++ b/libcxx/include/__cxx03/__support/ibm/gettod_zos.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
-#define _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
+#ifndef _LIBCPP___CXX03___SUPPORT_IBM_GETTOD_ZOS_H
+#define _LIBCPP___CXX03___SUPPORT_IBM_GETTOD_ZOS_H
 
 #include <__cxx03/time.h>
 
@@ -49,4 +49,4 @@ inline _LIBCPP_HIDE_FROM_ABI int gettimeofdayMonotonic(struct timespec64* Output
   return 0;
 }
 
-#endif // _LIBCPP___SUPPORT_IBM_GETTOD_ZOS_H
+#endif // _LIBCPP___CXX03___SUPPORT_IBM_GETTOD_ZOS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h b/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
index ec88704ba8604..9fa8f8c9088e1 100644
--- a/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
+++ b/libcxx/include/__cxx03/__support/ibm/locale_mgmt_zos.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
-#define _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
+#ifndef _LIBCPP___CXX03___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
+#define _LIBCPP___CXX03___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
 
 #if defined(__MVS__)
 #  include <__cxx03/locale.h>
@@ -50,4 +50,4 @@ locale_t uselocale(locale_t newloc);
 }
 #  endif
 #endif // defined(__MVS__)
-#endif // _LIBCPP___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
+#endif // _LIBCPP___CXX03___SUPPORT_IBM_LOCALE_MGMT_ZOS_H
diff --git a/libcxx/include/__cxx03/__support/ibm/nanosleep.h b/libcxx/include/__cxx03/__support/ibm/nanosleep.h
index ecf7187543f36..1b6883bbcc111 100644
--- a/libcxx/include/__cxx03/__support/ibm/nanosleep.h
+++ b/libcxx/include/__cxx03/__support/ibm/nanosleep.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
-#define _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
+#ifndef _LIBCPP___CXX03___SUPPORT_IBM_NANOSLEEP_H
+#define _LIBCPP___CXX03___SUPPORT_IBM_NANOSLEEP_H
 
 #include <__cxx03/unistd.h>
 
@@ -52,4 +52,4 @@ inline int nanosleep(const struct timespec* __req, struct timespec* __rem) {
   return 0;
 }
 
-#endif // _LIBCPP___SUPPORT_IBM_NANOSLEEP_H
+#endif // _LIBCPP___CXX03___SUPPORT_IBM_NANOSLEEP_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h b/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
index 036bd3b183506..f948a292c5f09 100644
--- a/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
+++ b/libcxx/include/__cxx03/__support/xlocale/__nop_locale_mgmt.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
-#define _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
+#ifndef _LIBCPP___CXX03___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
+#define _LIBCPP___CXX03___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
 
 #include <__cxx03/__config>
 
@@ -32,4 +32,4 @@ inline _LIBCPP_HIDE_FROM_ABI locale_t uselocale(locale_t) { return NULL; }
 #define LC_ALL_MASK                                                                                                    \
   (LC_COLLATE_MASK | LC_CTYPE_MASK | LC_MONETARY_MASK | LC_NUMERIC_MASK | LC_TIME_MASK | LC_MESSAGES_MASK)
 
-#endif // _LIBCPP___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
+#endif // _LIBCPP___CXX03___SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h b/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
index 8651c2b7ca958..03e7f15f31ac9 100644
--- a/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
+++ b/libcxx/include/__cxx03/__support/xlocale/__posix_l_fallback.h
@@ -12,8 +12,8 @@
 // Android's bionic and Newlib).
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
-#define _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
+#ifndef _LIBCPP___CXX03___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
+#define _LIBCPP___CXX03___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
 
 #include <__cxx03/__config>
 #include <__cxx03/ctype.h>
@@ -104,4 +104,4 @@ inline _LIBCPP_HIDE_FROM_ABI size_t wcsxfrm_l(wchar_t* __dest, const wchar_t* __
 }
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
-#endif // _LIBCPP___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
+#endif // _LIBCPP___CXX03___SUPPORT_XLOCALE_POSIX_L_FALLBACK_H
diff --git a/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h b/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
index 3587e2f1a3219..9ab49ed036237 100644
--- a/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
+++ b/libcxx/include/__cxx03/__support/xlocale/__strtonum_fallback.h
@@ -12,8 +12,8 @@
 // convert strings to some numeric type.
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
-#define _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
+#ifndef _LIBCPP___CXX03___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
+#define _LIBCPP___CXX03___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
 
 #include <__cxx03/__config>
 #include <__cxx03/stdlib.h>
@@ -42,4 +42,4 @@ inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, c
   return ::strtoull(__nptr, __endptr, __base);
 }
 
-#endif // _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
+#endif // _LIBCPP___CXX03___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H
diff --git a/libcxx/include/__cxx03/__system_error/errc.h b/libcxx/include/__cxx03/__system_error/errc.h
index 33e670ecd3a70..d2f7dc5f50dd3 100644
--- a/libcxx/include/__cxx03/__system_error/errc.h
+++ b/libcxx/include/__cxx03/__system_error/errc.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ERRC
-#define _LIBCPP___ERRC
+#ifndef _LIBCPP___CXX03___ERRC
+#define _LIBCPP___CXX03___ERRC
 
 /*
     system_error synopsis
@@ -263,4 +263,4 @@ _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(errc)
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ERRC
+#endif // _LIBCPP___CXX03___ERRC
diff --git a/libcxx/include/__cxx03/__system_error/error_category.h b/libcxx/include/__cxx03/__system_error/error_category.h
index cc08cf2801d43..7f9dd7171ea5c 100644
--- a/libcxx/include/__cxx03/__system_error/error_category.h
+++ b/libcxx/include/__cxx03/__system_error/error_category.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
-#define _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
+#ifndef _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CATEGORY_H
+#define _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CATEGORY_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -72,4 +72,4 @@ __attribute__((__const__)) _LIBCPP_EXPORTED_FROM_ABI const error_category& syste
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CATEGORY_H
+#endif // _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CATEGORY_H
diff --git a/libcxx/include/__cxx03/__system_error/error_code.h b/libcxx/include/__cxx03/__system_error/error_code.h
index 2fe4d299885fc..e253f88633d33 100644
--- a/libcxx/include/__cxx03/__system_error/error_code.h
+++ b/libcxx/include/__cxx03/__system_error/error_code.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
-#define _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
+#ifndef _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CODE_H
+#define _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CODE_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -140,4 +140,4 @@ struct _LIBCPP_TEMPLATE_VIS hash<error_code> : public __unary_function<error_cod
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CODE_H
+#endif // _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CODE_H
diff --git a/libcxx/include/__cxx03/__system_error/error_condition.h b/libcxx/include/__cxx03/__system_error/error_condition.h
index 87bfcd6c6350c..ba37deaf4fcb6 100644
--- a/libcxx/include/__cxx03/__system_error/error_condition.h
+++ b/libcxx/include/__cxx03/__system_error/error_condition.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
-#define _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
+#ifndef _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CONDITION_H
+#define _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CONDITION_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -127,4 +127,4 @@ struct _LIBCPP_TEMPLATE_VIS hash<error_condition> : public __unary_function<erro
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___SYSTEM_ERROR_ERROR_CONDITION_H
+#endif // _LIBCPP___CXX03___SYSTEM_ERROR_ERROR_CONDITION_H
diff --git a/libcxx/include/__cxx03/__system_error/system_error.h b/libcxx/include/__cxx03/__system_error/system_error.h
index 2487d3bafa06d..84fbba1c84364 100644
--- a/libcxx/include/__cxx03/__system_error/system_error.h
+++ b/libcxx/include/__cxx03/__system_error/system_error.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
-#define _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
+#ifndef _LIBCPP___CXX03___SYSTEM_ERROR_SYSTEM_ERROR_H
+#define _LIBCPP___CXX03___SYSTEM_ERROR_SYSTEM_ERROR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__system_error/error_category.h>
@@ -51,4 +51,4 @@ _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __throw_system_error(error_co
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___SYSTEM_ERROR_SYSTEM_ERROR_H
+#endif // _LIBCPP___CXX03___SYSTEM_ERROR_SYSTEM_ERROR_H
diff --git a/libcxx/include/__cxx03/__thread/formatter.h b/libcxx/include/__cxx03/__thread/formatter.h
index d7ecba57203ed..5d54789ee445c 100644
--- a/libcxx/include/__cxx03/__thread/formatter.h
+++ b/libcxx/include/__cxx03/__thread/formatter.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_FORMATTER_H
-#define _LIBCPP___THREAD_FORMATTER_H
+#ifndef _LIBCPP___CXX03___THREAD_FORMATTER_H
+#define _LIBCPP___CXX03___THREAD_FORMATTER_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -77,4 +77,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 23
 
-#endif // _LIBCPP___THREAD_FORMATTER_H
+#endif // _LIBCPP___CXX03___THREAD_FORMATTER_H
diff --git a/libcxx/include/__cxx03/__thread/id.h b/libcxx/include/__cxx03/__thread/id.h
index 09e7bb071c61f..8dcd68bdf39d7 100644
--- a/libcxx/include/__cxx03/__thread/id.h
+++ b/libcxx/include/__cxx03/__thread/id.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_ID_H
-#define _LIBCPP___THREAD_ID_H
+#ifndef _LIBCPP___CXX03___THREAD_ID_H
+#define _LIBCPP___CXX03___THREAD_ID_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -118,4 +118,4 @@ inline _LIBCPP_HIDE_FROM_ABI __thread_id get_id() _NOEXCEPT { return __libcpp_th
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___THREAD_ID_H
+#endif // _LIBCPP___CXX03___THREAD_ID_H
diff --git a/libcxx/include/__cxx03/__thread/jthread.h b/libcxx/include/__cxx03/__thread/jthread.h
index 840d0cae9d0a2..6272ccaa6f7ce 100644
--- a/libcxx/include/__cxx03/__thread/jthread.h
+++ b/libcxx/include/__cxx03/__thread/jthread.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_JTHREAD_H
-#define _LIBCPP___THREAD_JTHREAD_H
+#ifndef _LIBCPP___CXX03___THREAD_JTHREAD_H
+#define _LIBCPP___CXX03___THREAD_JTHREAD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -131,4 +131,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___THREAD_JTHREAD_H
+#endif // _LIBCPP___CXX03___THREAD_JTHREAD_H
diff --git a/libcxx/include/__cxx03/__thread/poll_with_backoff.h b/libcxx/include/__cxx03/__thread/poll_with_backoff.h
index d875c1465dc81..b500629c85217 100644
--- a/libcxx/include/__cxx03/__thread/poll_with_backoff.h
+++ b/libcxx/include/__cxx03/__thread/poll_with_backoff.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
-#define _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
+#ifndef _LIBCPP___CXX03___THREAD_POLL_WITH_BACKOFF_H
+#define _LIBCPP___CXX03___THREAD_POLL_WITH_BACKOFF_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__chrono/high_resolution_clock.h>
@@ -64,4 +64,4 @@ struct __spinning_backoff_policy {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___THREAD_POLL_WITH_BACKOFF_H
+#endif // _LIBCPP___CXX03___THREAD_POLL_WITH_BACKOFF_H
diff --git a/libcxx/include/__cxx03/__thread/support.h b/libcxx/include/__cxx03/__thread/support.h
index 3ef25c88c1a8a..c1cdc999a6988 100644
--- a/libcxx/include/__cxx03/__thread/support.h
+++ b/libcxx/include/__cxx03/__thread/support.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_SUPPORT_H
-#define _LIBCPP___THREAD_SUPPORT_H
+#ifndef _LIBCPP___CXX03___THREAD_SUPPORT_H
+#define _LIBCPP___CXX03___THREAD_SUPPORT_H
 
 #include <__cxx03/__config>
 
@@ -120,4 +120,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // !_LIBCPP_HAS_NO_THREADS
 
-#endif // _LIBCPP___THREAD_SUPPORT_H
+#endif // _LIBCPP___CXX03___THREAD_SUPPORT_H
diff --git a/libcxx/include/__cxx03/__thread/support/c11.h b/libcxx/include/__cxx03/__thread/support/c11.h
index 7b791388029b9..4ae6f841a9430 100644
--- a/libcxx/include/__cxx03/__thread/support/c11.h
+++ b/libcxx/include/__cxx03/__thread/support/c11.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_SUPPORT_C11_H
-#define _LIBCPP___THREAD_SUPPORT_C11_H
+#ifndef _LIBCPP___CXX03___THREAD_SUPPORT_C11_H
+#define _LIBCPP___CXX03___THREAD_SUPPORT_C11_H
 
 #include <__cxx03/__chrono/convert_to_timespec.h>
 #include <__cxx03/__chrono/duration.h>
@@ -188,4 +188,4 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void*
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___THREAD_SUPPORT_C11_H
+#endif // _LIBCPP___CXX03___THREAD_SUPPORT_C11_H
diff --git a/libcxx/include/__cxx03/__thread/support/external.h b/libcxx/include/__cxx03/__thread/support/external.h
index d4b476c435049..39b2ad467fa56 100644
--- a/libcxx/include/__cxx03/__thread/support/external.h
+++ b/libcxx/include/__cxx03/__thread/support/external.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
-#define _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
+#ifndef _LIBCPP___CXX03___THREAD_SUPPORT_EXTERNAL_H
+#define _LIBCPP___CXX03___THREAD_SUPPORT_EXTERNAL_H
 
 #include <__cxx03/__config>
 
@@ -18,4 +18,4 @@
 
 #include <__cxx03/__external_threading>
 
-#endif // _LIBCPP___THREAD_SUPPORT_EXTERNAL_H
+#endif // _LIBCPP___CXX03___THREAD_SUPPORT_EXTERNAL_H
diff --git a/libcxx/include/__cxx03/__thread/support/pthread.h b/libcxx/include/__cxx03/__thread/support/pthread.h
index 9c3646896c108..a4d7c874e6830 100644
--- a/libcxx/include/__cxx03/__thread/support/pthread.h
+++ b/libcxx/include/__cxx03/__thread/support/pthread.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_SUPPORT_PTHREAD_H
-#define _LIBCPP___THREAD_SUPPORT_PTHREAD_H
+#ifndef _LIBCPP___CXX03___THREAD_SUPPORT_PTHREAD_H
+#define _LIBCPP___CXX03___THREAD_SUPPORT_PTHREAD_H
 
 #include <__cxx03/__chrono/convert_to_timespec.h>
 #include <__cxx03/__chrono/duration.h>
@@ -218,4 +218,4 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void*
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___THREAD_SUPPORT_PTHREAD_H
+#endif // _LIBCPP___CXX03___THREAD_SUPPORT_PTHREAD_H
diff --git a/libcxx/include/__cxx03/__thread/support/windows.h b/libcxx/include/__cxx03/__thread/support/windows.h
index b7b859b060efa..45252a57efaec 100644
--- a/libcxx/include/__cxx03/__thread/support/windows.h
+++ b/libcxx/include/__cxx03/__thread/support/windows.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_SUPPORT_WINDOWS_H
-#define _LIBCPP___THREAD_SUPPORT_WINDOWS_H
+#ifndef _LIBCPP___CXX03___THREAD_SUPPORT_WINDOWS_H
+#define _LIBCPP___CXX03___THREAD_SUPPORT_WINDOWS_H
 
 #include <__cxx03/__chrono/duration.h>
 #include <__cxx03/__config>
@@ -130,4 +130,4 @@ _LIBCPP_EXPORTED_FROM_ABI int __libcpp_tls_set(__libcpp_tls_key __key, void* __p
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___THREAD_SUPPORT_WINDOWS_H
+#endif // _LIBCPP___CXX03___THREAD_SUPPORT_WINDOWS_H
diff --git a/libcxx/include/__cxx03/__thread/this_thread.h b/libcxx/include/__cxx03/__thread/this_thread.h
index fe6c19aaf9b45..ae9c37ef02fbb 100644
--- a/libcxx/include/__cxx03/__thread/this_thread.h
+++ b/libcxx/include/__cxx03/__thread/this_thread.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_THIS_THREAD_H
-#define _LIBCPP___THREAD_THIS_THREAD_H
+#ifndef _LIBCPP___CXX03___THREAD_THIS_THREAD_H
+#define _LIBCPP___CXX03___THREAD_THIS_THREAD_H
 
 #include <__cxx03/__chrono/steady_clock.h>
 #include <__cxx03/__chrono/time_point.h>
@@ -71,4 +71,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___THREAD_THIS_THREAD_H
+#endif // _LIBCPP___CXX03___THREAD_THIS_THREAD_H
diff --git a/libcxx/include/__cxx03/__thread/thread.h b/libcxx/include/__cxx03/__thread/thread.h
index fdd875e03cc68..b624a7d46fe24 100644
--- a/libcxx/include/__cxx03/__thread/thread.h
+++ b/libcxx/include/__cxx03/__thread/thread.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_THREAD_H
-#define _LIBCPP___THREAD_THREAD_H
+#ifndef _LIBCPP___CXX03___THREAD_THREAD_H
+#define _LIBCPP___CXX03___THREAD_THREAD_H
 
 #include <__cxx03/__condition_variable/condition_variable.h>
 #include <__cxx03/__config>
@@ -255,4 +255,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___THREAD_THREAD_H
+#endif // _LIBCPP___CXX03___THREAD_THREAD_H
diff --git a/libcxx/include/__cxx03/__thread/timed_backoff_policy.h b/libcxx/include/__cxx03/__thread/timed_backoff_policy.h
index eec6f225d717d..f21d88629027e 100644
--- a/libcxx/include/__cxx03/__thread/timed_backoff_policy.h
+++ b/libcxx/include/__cxx03/__thread/timed_backoff_policy.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
-#define _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
+#ifndef _LIBCPP___CXX03___THREAD_TIMED_BACKOFF_POLICY_H
+#define _LIBCPP___CXX03___THREAD_TIMED_BACKOFF_POLICY_H
 
 #include <__cxx03/__config>
 
@@ -41,4 +41,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_HAS_NO_THREADS
 
-#endif // _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
+#endif // _LIBCPP___CXX03___THREAD_TIMED_BACKOFF_POLICY_H
diff --git a/libcxx/include/__cxx03/__tree b/libcxx/include/__cxx03/__tree
index 2b8521a091536..8982baf18d1a9 100644
--- a/libcxx/include/__cxx03/__tree
+++ b/libcxx/include/__cxx03/__tree
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TREE
-#define _LIBCPP___TREE
+#ifndef _LIBCPP___CXX03___TREE
+#define _LIBCPP___CXX03___TREE
 
 #include <__cxx03/__algorithm/min.h>
 #include <__cxx03/__assert>
@@ -2296,4 +2296,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___TREE
+#endif // _LIBCPP___CXX03___TREE
diff --git a/libcxx/include/__cxx03/__tuple/find_index.h b/libcxx/include/__cxx03/__tuple/find_index.h
index 899e7f2d7b41f..a090dff91e973 100644
--- a/libcxx/include/__cxx03/__tuple/find_index.h
+++ b/libcxx/include/__cxx03/__tuple/find_index.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_FIND_INDEX_H
-#define _LIBCPP___TUPLE_FIND_INDEX_H
+#ifndef _LIBCPP___CXX03___TUPLE_FIND_INDEX_H
+#define _LIBCPP___CXX03___TUPLE_FIND_INDEX_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_same.h>
@@ -59,4 +59,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 14
 
-#endif // _LIBCPP___TUPLE_FIND_INDEX_H
+#endif // _LIBCPP___CXX03___TUPLE_FIND_INDEX_H
diff --git a/libcxx/include/__cxx03/__tuple/ignore.h b/libcxx/include/__cxx03/__tuple/ignore.h
index 2d1800ae0d708..9e00de9a8471d 100644
--- a/libcxx/include/__cxx03/__tuple/ignore.h
+++ b/libcxx/include/__cxx03/__tuple/ignore.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_IGNORE_H
-#define _LIBCPP___TUPLE_IGNORE_H
+#ifndef _LIBCPP___CXX03___TUPLE_IGNORE_H
+#define _LIBCPP___CXX03___TUPLE_IGNORE_H
 
 #include <__cxx03/__config>
 
@@ -36,4 +36,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_CXX03_LANG
 
-#endif // _LIBCPP___TUPLE_IGNORE_H
+#endif // _LIBCPP___CXX03___TUPLE_IGNORE_H
diff --git a/libcxx/include/__cxx03/__tuple/make_tuple_types.h b/libcxx/include/__cxx03/__tuple/make_tuple_types.h
index b7abfa74f6f11..d8eed10dc51ac 100644
--- a/libcxx/include/__cxx03/__tuple/make_tuple_types.h
+++ b/libcxx/include/__cxx03/__tuple/make_tuple_types.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
-#define _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
+#ifndef _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_TYPES_H
+#define _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_TYPES_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/array.h>
@@ -77,4 +77,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_CXX03_LANG
 
-#endif // _LIBCPP___TUPLE_MAKE_TUPLE_TYPES_H
+#endif // _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_TYPES_H
diff --git a/libcxx/include/__cxx03/__tuple/sfinae_helpers.h b/libcxx/include/__cxx03/__tuple/sfinae_helpers.h
index d6889364a9162..4fbe87d6e9a5b 100644
--- a/libcxx/include/__cxx03/__tuple/sfinae_helpers.h
+++ b/libcxx/include/__cxx03/__tuple/sfinae_helpers.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_SFINAE_HELPERS_H
-#define _LIBCPP___TUPLE_SFINAE_HELPERS_H
+#ifndef _LIBCPP___CXX03___TUPLE_SFINAE_HELPERS_H
+#define _LIBCPP___CXX03___TUPLE_SFINAE_HELPERS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/tuple.h>
@@ -139,4 +139,4 @@ struct __sfinae_assign_base<false, true> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_SFINAE_HELPERS_H
+#endif // _LIBCPP___CXX03___TUPLE_SFINAE_HELPERS_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_element.h b/libcxx/include/__cxx03/__tuple/tuple_element.h
index d00879a069423..56dbdc3beb39b 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_element.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_element.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_ELEMENT_H
-#define _LIBCPP___TUPLE_TUPLE_ELEMENT_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_ELEMENT_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_ELEMENT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__tuple/tuple_indices.h>
@@ -55,4 +55,4 @@ using tuple_element_t _LIBCPP_NODEBUG = typename tuple_element<_Ip, _Tp...>::typ
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_ELEMENT_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_ELEMENT_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_indices.h b/libcxx/include/__cxx03/__tuple/tuple_indices.h
index 6e51b69e906b0..0c423d99ccb63 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_indices.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_indices.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
-#define _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
+#ifndef _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_INDICES_H
+#define _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_INDICES_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__utility/integer_sequence.h>
@@ -34,4 +34,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_CXX03_LANG
 
-#endif // _LIBCPP___TUPLE_MAKE_TUPLE_INDICES_H
+#endif // _LIBCPP___CXX03___TUPLE_MAKE_TUPLE_INDICES_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like.h b/libcxx/include/__cxx03/__tuple/tuple_like.h
index 23f42d58312a6..ea596c4ba52df 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_like.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_like.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_H
-#define _LIBCPP___TUPLE_TUPLE_LIKE_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/subrange.h>
@@ -39,4 +39,4 @@ concept __tuple_like = __tuple_like_no_subrange<_Tp> || __is_ranges_subrange_v<r
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_LIKE_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like_ext.h b/libcxx/include/__cxx03/__tuple/tuple_like_ext.h
index 5bb07cc8b5e74..94470521cd068 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_like_ext.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_like_ext.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
-#define _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_EXT_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_EXT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/array.h>
@@ -49,4 +49,4 @@ struct __tuple_like_ext<__tuple_types<_Tp...> > : true_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_LIKE_EXT_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_EXT_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h b/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
index af4a14d2d535f..ff095163363b1 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_like_no_subrange.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
-#define _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/array.h>
@@ -58,4 +58,4 @@ concept __pair_like_no_subrange = __tuple_like_no_subrange<_Tp> && tuple_size<re
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_LIKE_NO_SUBRANGE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_size.h b/libcxx/include/__cxx03/__tuple/tuple_size.h
index ada7a522182e2..1f8fd22e3eaf1 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_size.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_size.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_SIZE_H
-#define _LIBCPP___TUPLE_TUPLE_SIZE_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_SIZE_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_SIZE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/tuple.h>
@@ -72,4 +72,4 @@ inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_SIZE_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_SIZE_H
diff --git a/libcxx/include/__cxx03/__tuple/tuple_types.h b/libcxx/include/__cxx03/__tuple/tuple_types.h
index 562d199a02ba2..72ee76f49412d 100644
--- a/libcxx/include/__cxx03/__tuple/tuple_types.h
+++ b/libcxx/include/__cxx03/__tuple/tuple_types.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TUPLE_TUPLE_TYPES_H
-#define _LIBCPP___TUPLE_TUPLE_TYPES_H
+#ifndef _LIBCPP___CXX03___TUPLE_TUPLE_TYPES_H
+#define _LIBCPP___CXX03___TUPLE_TUPLE_TYPES_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@ struct __tuple_types {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TUPLE_TUPLE_TYPES_H
+#endif // _LIBCPP___CXX03___TUPLE_TUPLE_TYPES_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_const.h b/libcxx/include/__cxx03/__type_traits/add_const.h
index ce670824873ee..004ce05c03996 100644
--- a/libcxx/include/__cxx03/__type_traits/add_const.h
+++ b/libcxx/include/__cxx03/__type_traits/add_const.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_CONST_H
-#define _LIBCPP___TYPE_TRAITS_ADD_CONST_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_CONST_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_CONST_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ using add_const_t = typename add_const<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_CONST_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_cv.h b/libcxx/include/__cxx03/__type_traits/add_cv.h
index 43eb05fa40487..8445b863b5c2e 100644
--- a/libcxx/include/__cxx03/__type_traits/add_cv.h
+++ b/libcxx/include/__cxx03/__type_traits/add_cv.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_CV_H
-#define _LIBCPP___TYPE_TRAITS_ADD_CV_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_CV_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_CV_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ using add_cv_t = typename add_cv<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_CV_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h b/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
index ca6ee5dc0b296..c35b366794a7d 100644
--- a/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
+++ b/libcxx/include/__cxx03/__type_traits/add_lvalue_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_referenceable.h>
@@ -51,4 +51,4 @@ using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_pointer.h b/libcxx/include/__cxx03/__type_traits/add_pointer.h
index ec3ba5d8f404f..c53b5ce7e46ac 100644
--- a/libcxx/include/__cxx03/__type_traits/add_pointer.h
+++ b/libcxx/include/__cxx03/__type_traits/add_pointer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
-#define _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_POINTER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_POINTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_referenceable.h>
@@ -52,4 +52,4 @@ using add_pointer_t = __add_pointer_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_POINTER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h b/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
index dc3be42ab0171..812a15e6090e4 100644
--- a/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
+++ b/libcxx/include/__cxx03/__type_traits/add_rvalue_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_referenceable.h>
@@ -51,4 +51,4 @@ using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/add_volatile.h b/libcxx/include/__cxx03/__type_traits/add_volatile.h
index 35ff8d7435c6d..9f649e4978d65 100644
--- a/libcxx/include/__cxx03/__type_traits/add_volatile.h
+++ b/libcxx/include/__cxx03/__type_traits/add_volatile.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ADD_VOLATILE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ADD_VOLATILE_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ using add_volatile_t = typename add_volatile<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_VOLATILE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ADD_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/aligned_storage.h b/libcxx/include/__cxx03/__type_traits/aligned_storage.h
index fc9fe4edd27ae..8383cc48bb44e 100644
--- a/libcxx/include/__cxx03/__type_traits/aligned_storage.h
+++ b/libcxx/include/__cxx03/__type_traits/aligned_storage.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
-#define _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_STORAGE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_STORAGE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -135,4 +135,4 @@ _CREATE_ALIGNED_STORAGE_SPECIALIZATION(0x4000);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ALIGNED_STORAGE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_STORAGE_H
diff --git a/libcxx/include/__cxx03/__type_traits/aligned_union.h b/libcxx/include/__cxx03/__type_traits/aligned_union.h
index d2c695d675381..0a0deb90e3991 100644
--- a/libcxx/include/__cxx03/__type_traits/aligned_union.h
+++ b/libcxx/include/__cxx03/__type_traits/aligned_union.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
-#define _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_UNION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_UNION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/aligned_storage.h>
@@ -48,4 +48,4 @@ using aligned_union_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_union<_Len,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ALIGNED_UNION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ALIGNED_UNION_H
diff --git a/libcxx/include/__cxx03/__type_traits/alignment_of.h b/libcxx/include/__cxx03/__type_traits/alignment_of.h
index f2d5317d1a192..d72e0cc43ebf6 100644
--- a/libcxx/include/__cxx03/__type_traits/alignment_of.h
+++ b/libcxx/include/__cxx03/__type_traits/alignment_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
-#define _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ALIGNMENT_OF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ALIGNMENT_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -29,4 +29,4 @@ inline constexpr size_t alignment_of_v = _LIBCPP_ALIGNOF(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ALIGNMENT_OF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ALIGNMENT_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/can_extract_key.h b/libcxx/include/__cxx03/__type_traits/can_extract_key.h
index 8069ce25f4a67..437cee6bcb9ae 100644
--- a/libcxx/include/__cxx03/__type_traits/can_extract_key.h
+++ b/libcxx/include/__cxx03/__type_traits/can_extract_key.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
-#define _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_CAN_EXTRACT_KEY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_CAN_EXTRACT_KEY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/pair.h>
@@ -50,4 +50,4 @@ struct __can_extract_map_key<_ValTy, _Key, _Key, _RawValTy> : false_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_CAN_EXTRACT_KEY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_CAN_EXTRACT_KEY_H
diff --git a/libcxx/include/__cxx03/__type_traits/common_reference.h b/libcxx/include/__cxx03/__type_traits/common_reference.h
index ecaf83323f2f1..8e1694e14e6ef 100644
--- a/libcxx/include/__cxx03/__type_traits/common_reference.h
+++ b/libcxx/include/__cxx03/__type_traits/common_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_COMMON_REFERENCE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_COMMON_REFERENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/common_type.h>
@@ -190,4 +190,4 @@ struct common_reference {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_COMMON_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/common_type.h b/libcxx/include/__cxx03/__type_traits/common_type.h
index 9f7d0cef37faf..78c36162ee4ac 100644
--- a/libcxx/include/__cxx03/__type_traits/common_type.h
+++ b/libcxx/include/__cxx03/__type_traits/common_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
-#define _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_COMMON_TYPE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_COMMON_TYPE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -99,4 +99,4 @@ using common_type_t = typename common_type<_Tp...>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_COMMON_TYPE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_COMMON_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/conditional.h b/libcxx/include/__cxx03/__type_traits/conditional.h
index e03dcf0708016..6f7ca781f0e37 100644
--- a/libcxx/include/__cxx03/__type_traits/conditional.h
+++ b/libcxx/include/__cxx03/__type_traits/conditional.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
-#define _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_CONDITIONAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_CONDITIONAL_H
 
 #include <__cxx03/__config>
 
@@ -55,4 +55,4 @@ using __conditional_t _LIBCPP_NODEBUG = typename conditional<_Bp, _If, _Then>::t
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_CONDITIONAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_CONDITIONAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/conjunction.h b/libcxx/include/__cxx03/__type_traits/conjunction.h
index 9cb6a87764a95..af67e8d0899fe 100644
--- a/libcxx/include/__cxx03/__type_traits/conjunction.h
+++ b/libcxx/include/__cxx03/__type_traits/conjunction.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
-#define _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_CONJUNCTION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_CONJUNCTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -62,4 +62,4 @@ inline constexpr bool conjunction_v = conjunction<_Args...>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_CONJUNCTION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_CONJUNCTION_H
diff --git a/libcxx/include/__cxx03/__type_traits/copy_cv.h b/libcxx/include/__cxx03/__type_traits/copy_cv.h
index 1ebda47a18bbe..46913594d6860 100644
--- a/libcxx/include/__cxx03/__type_traits/copy_cv.h
+++ b/libcxx/include/__cxx03/__type_traits/copy_cv.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_COPY_CV_H
-#define _LIBCPP___TYPE_TRAITS_COPY_CV_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_COPY_CV_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_COPY_CV_H
 
 #include <__cxx03/__config>
 
@@ -48,4 +48,4 @@ using __copy_cv_t = typename __copy_cv<_From>::template __apply<_To>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_COPY_CV_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_COPY_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/copy_cvref.h b/libcxx/include/__cxx03/__type_traits/copy_cvref.h
index d852bcd3b99c0..f2bb2be9c0a33 100644
--- a/libcxx/include/__cxx03/__type_traits/copy_cvref.h
+++ b/libcxx/include/__cxx03/__type_traits/copy_cvref.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
-#define _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_COPY_CVREF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_COPY_CVREF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -40,4 +40,4 @@ using __copy_cvref_t = typename __copy_cvref<_From, _To>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_COPY_CVREF_H
diff --git a/libcxx/include/__cxx03/__type_traits/datasizeof.h b/libcxx/include/__cxx03/__type_traits/datasizeof.h
index 7dd87e162d341..d428856d13a66 100644
--- a/libcxx/include/__cxx03/__type_traits/datasizeof.h
+++ b/libcxx/include/__cxx03/__type_traits/datasizeof.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
-#define _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_DATASIZEOF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_DATASIZEOF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_class.h>
@@ -62,4 +62,4 @@ _LIBCPP_DIAGNOSTIC_POP
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_DATASIZEOF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_DATASIZEOF_H
diff --git a/libcxx/include/__cxx03/__type_traits/decay.h b/libcxx/include/__cxx03/__type_traits/decay.h
index 3563543a30912..bf745a89a1d0f 100644
--- a/libcxx/include/__cxx03/__type_traits/decay.h
+++ b/libcxx/include/__cxx03/__type_traits/decay.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_DECAY_H
-#define _LIBCPP___TYPE_TRAITS_DECAY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_DECAY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_DECAY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_pointer.h>
@@ -70,4 +70,4 @@ using decay_t = __decay_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_DECAY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_DECAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/dependent_type.h b/libcxx/include/__cxx03/__type_traits/dependent_type.h
index fed130a8f101a..60ce814ca6b0d 100644
--- a/libcxx/include/__cxx03/__type_traits/dependent_type.h
+++ b/libcxx/include/__cxx03/__type_traits/dependent_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
-#define _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_DEPENDENT_TYPE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_DEPENDENT_TYPE_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@ struct _LIBCPP_TEMPLATE_VIS __dependent_type : public _Tp {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_DEPENDENT_TYPE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_DEPENDENT_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/desugars_to.h b/libcxx/include/__cxx03/__type_traits/desugars_to.h
index e16fb59007fe9..eb2dfefd86e2c 100644
--- a/libcxx/include/__cxx03/__type_traits/desugars_to.h
+++ b/libcxx/include/__cxx03/__type_traits/desugars_to.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
-#define _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_DESUGARS_TO_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_DESUGARS_TO_H
 
 #include <__cxx03/__config>
 
@@ -37,4 +37,4 @@ inline const bool __desugars_to_v = false;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_DESUGARS_TO_H
diff --git a/libcxx/include/__cxx03/__type_traits/disjunction.h b/libcxx/include/__cxx03/__type_traits/disjunction.h
index 717a439669e1d..0ff8218e74196 100644
--- a/libcxx/include/__cxx03/__type_traits/disjunction.h
+++ b/libcxx/include/__cxx03/__type_traits/disjunction.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
-#define _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_DISJUNCTION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_DISJUNCTION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -55,4 +55,4 @@ inline constexpr bool disjunction_v = _Or<_Args...>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_DISJUNCTION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_DISJUNCTION_H
diff --git a/libcxx/include/__cxx03/__type_traits/enable_if.h b/libcxx/include/__cxx03/__type_traits/enable_if.h
index d21c1301e0a2e..042ad139725dc 100644
--- a/libcxx/include/__cxx03/__type_traits/enable_if.h
+++ b/libcxx/include/__cxx03/__type_traits/enable_if.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
-#define _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_ENABLE_IF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_ENABLE_IF_H
 
 #include <__cxx03/__config>
 
@@ -34,4 +34,4 @@ using enable_if_t = typename enable_if<_Bp, _Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ENABLE_IF_H
diff --git a/libcxx/include/__cxx03/__type_traits/extent.h b/libcxx/include/__cxx03/__type_traits/extent.h
index d29fef59a5a9e..4d7dd83987683 100644
--- a/libcxx/include/__cxx03/__type_traits/extent.h
+++ b/libcxx/include/__cxx03/__type_traits/extent.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_EXTENT_H
-#define _LIBCPP___TYPE_TRAITS_EXTENT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_EXTENT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_EXTENT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -51,4 +51,4 @@ inline constexpr size_t extent_v = extent<_Tp, _Ip>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_EXTENT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_EXTENT_H
diff --git a/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h b/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
index 4a507fe7fcac8..8abd31973d9d2 100644
--- a/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
+++ b/libcxx/include/__cxx03/__type_traits/has_unique_object_representation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
-#define _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -37,4 +37,4 @@ inline constexpr bool has_unique_object_representations_v = __has_unique_object_
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATION_H
diff --git a/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h b/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
index 060bd1bfe58c8..58740ce91ba1f 100644
--- a/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
+++ b/libcxx/include/__cxx03/__type_traits/has_virtual_destructor.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
-#define _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool has_virtual_destructor_v = __has_virtual_destructor(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_HAS_VIRTUAL_DESTRUCTOR_H
diff --git a/libcxx/include/__cxx03/__type_traits/integral_constant.h b/libcxx/include/__cxx03/__type_traits/integral_constant.h
index bfc11c8a45163..197453747e547 100644
--- a/libcxx/include/__cxx03/__type_traits/integral_constant.h
+++ b/libcxx/include/__cxx03/__type_traits/integral_constant.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
-#define _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_INTEGRAL_CONSTANT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_INTEGRAL_CONSTANT_H
 
 #include <__cxx03/__config>
 
@@ -44,4 +44,4 @@ using bool_constant = integral_constant<bool, __b>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_INTEGRAL_CONSTANT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_INTEGRAL_CONSTANT_H
diff --git a/libcxx/include/__cxx03/__type_traits/invoke.h b/libcxx/include/__cxx03/__type_traits/invoke.h
index 9ece1c8749629..47220d6c2e629 100644
--- a/libcxx/include/__cxx03/__type_traits/invoke.h
+++ b/libcxx/include/__cxx03/__type_traits/invoke.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_INVOKE_H
-#define _LIBCPP___TYPE_TRAITS_INVOKE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_INVOKE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_INVOKE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -267,4 +267,4 @@ using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_INVOKE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_INVOKE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_abstract.h b/libcxx/include/__cxx03/__type_traits/is_abstract.h
index 785ae6772f710..a98d2b350ba29 100644
--- a/libcxx/include/__cxx03/__type_traits/is_abstract.h
+++ b/libcxx/include/__cxx03/__type_traits/is_abstract.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
-#define _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ABSTRACT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ABSTRACT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_abstract_v = __is_abstract(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ABSTRACT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ABSTRACT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_aggregate.h b/libcxx/include/__cxx03/__type_traits/is_aggregate.h
index 3da7efa0ce640..0d95780859ca8 100644
--- a/libcxx/include/__cxx03/__type_traits/is_aggregate.h
+++ b/libcxx/include/__cxx03/__type_traits/is_aggregate.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
-#define _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_AGGREGATE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_AGGREGATE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -30,4 +30,4 @@ inline constexpr bool is_aggregate_v = __is_aggregate(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_AGGREGATE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_AGGREGATE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_allocator.h b/libcxx/include/__cxx03/__type_traits/is_allocator.h
index 1b063e84e545f..b2b73ec53fe2b 100644
--- a/libcxx/include/__cxx03/__type_traits/is_allocator.h
+++ b/libcxx/include/__cxx03/__type_traits/is_allocator.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_IS_ALLOCATOR_H
-#define _LIBCPP___TYPE_IS_ALLOCATOR_H
+#ifndef _LIBCPP___CXX03___TYPE_IS_ALLOCATOR_H
+#define _LIBCPP___CXX03___TYPE_IS_ALLOCATOR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -31,4 +31,4 @@ struct __is_allocator<_Alloc,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_IS_ALLOCATOR_H
+#endif // _LIBCPP___CXX03___TYPE_IS_ALLOCATOR_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h b/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
index 6f5b6631ba40f..8fb9e519d03ea 100644
--- a/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_always_bitcastable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -82,4 +82,4 @@ struct __is_always_bitcastable {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ALWAYS_BITCASTABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_arithmetic.h b/libcxx/include/__cxx03/__type_traits/is_arithmetic.h
index 900a5b5b4176d..8d1e019ce402d 100644
--- a/libcxx/include/__cxx03/__type_traits/is_arithmetic.h
+++ b/libcxx/include/__cxx03/__type_traits/is_arithmetic.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
-#define _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ARITHMETIC_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ARITHMETIC_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -31,4 +31,4 @@ inline constexpr bool is_arithmetic_v = is_arithmetic<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ARITHMETIC_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ARITHMETIC_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_array.h b/libcxx/include/__cxx03/__type_traits/is_array.h
index d4276ed61a4f6..a8dd572ffedec 100644
--- a/libcxx/include/__cxx03/__type_traits/is_array.h
+++ b/libcxx/include/__cxx03/__type_traits/is_array.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
-#define _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ARRAY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ARRAY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -48,4 +48,4 @@ inline constexpr bool is_array_v = is_array<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_assignable.h b/libcxx/include/__cxx03/__type_traits/is_assignable.h
index ba1d5d1afcdd1..6a7364ca6eacd 100644
--- a/libcxx/include/__cxx03/__type_traits/is_assignable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_assignable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ASSIGNABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ASSIGNABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -49,4 +49,4 @@ inline constexpr bool is_move_assignable_v = is_move_assignable<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_base_of.h b/libcxx/include/__cxx03/__type_traits/is_base_of.h
index 0a31bfaa74127..67f72731903c0 100644
--- a/libcxx/include/__cxx03/__type_traits/is_base_of.h
+++ b/libcxx/include/__cxx03/__type_traits/is_base_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
-#define _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_BASE_OF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_BASE_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_base_of_v = __is_base_of(_Bp, _Dp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_BASE_OF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_BASE_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_bounded_array.h b/libcxx/include/__cxx03/__type_traits/is_bounded_array.h
index aeb2235562838..b4b3aabec381b 100644
--- a/libcxx/include/__cxx03/__type_traits/is_bounded_array.h
+++ b/libcxx/include/__cxx03/__type_traits/is_bounded_array.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
-#define _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -38,4 +38,4 @@ inline constexpr bool is_bounded_array_v = is_bounded_array<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_callable.h b/libcxx/include/__cxx03/__type_traits/is_callable.h
index 4956d26b047fa..f4f20348416ce 100644
--- a/libcxx/include/__cxx03/__type_traits/is_callable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_callable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CALLABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CALLABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -29,4 +29,4 @@ struct __is_callable : decltype(std::__is_callable_helper<_Func, _Args...>(0)) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CALLABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CALLABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_char_like_type.h b/libcxx/include/__cxx03/__type_traits/is_char_like_type.h
index 2896a5794301d..e252b6ad18879 100644
--- a/libcxx/include/__cxx03/__type_traits/is_char_like_type.h
+++ b/libcxx/include/__cxx03/__type_traits/is_char_like_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
-#define _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conjunction.h>
@@ -25,4 +25,4 @@ using _IsCharLikeType = _And<is_standard_layout<_CharT>, is_trivial<_CharT> >;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CHAR_LIKE_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_class.h b/libcxx/include/__cxx03/__type_traits/is_class.h
index c56f3679ecb35..7109198cf6598 100644
--- a/libcxx/include/__cxx03/__type_traits/is_class.h
+++ b/libcxx/include/__cxx03/__type_traits/is_class.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CLASS_H
-#define _LIBCPP___TYPE_TRAITS_IS_CLASS_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CLASS_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CLASS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_class_v = __is_class(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CLASS_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CLASS_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_compound.h b/libcxx/include/__cxx03/__type_traits/is_compound.h
index c48c88643c8e1..4a96a71a2f965 100644
--- a/libcxx/include/__cxx03/__type_traits/is_compound.h
+++ b/libcxx/include/__cxx03/__type_traits/is_compound.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
-#define _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_COMPOUND_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_COMPOUND_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -43,4 +43,4 @@ inline constexpr bool is_compound_v = is_compound<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_COMPOUND_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_COMPOUND_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_const.h b/libcxx/include/__cxx03/__type_traits/is_const.h
index 2b297ad472bd5..ebd91c5f07c1d 100644
--- a/libcxx/include/__cxx03/__type_traits/is_const.h
+++ b/libcxx/include/__cxx03/__type_traits/is_const.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CONST_H
-#define _LIBCPP___TYPE_TRAITS_IS_CONST_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CONST_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CONST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -44,4 +44,4 @@ inline constexpr bool is_const_v = is_const<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CONST_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h b/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
index 9abd5ee9ce41a..99ebef6664ed9 100644
--- a/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
+++ b/libcxx/include/__cxx03/__type_traits/is_constant_evaluated.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
-#define _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR bool __libcpp_is_constant_evaluat
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CONSTANT_EVALUATED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_constructible.h b/libcxx/include/__cxx03/__type_traits/is_constructible.h
index 18c58311ed033..435b0fcefea8a 100644
--- a/libcxx/include/__cxx03/__type_traits/is_constructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_constructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
-#define _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_IS_CONSTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_IS_CONSTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -56,4 +56,4 @@ inline constexpr bool is_default_constructible_v = __is_constructible(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_IS_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_convertible.h b/libcxx/include/__cxx03/__type_traits/is_convertible.h
index a2b2630f109be..4692b0f4ac5f4 100644
--- a/libcxx/include/__cxx03/__type_traits/is_convertible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_convertible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CONVERTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CONVERTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_convertible_v = __is_convertible(_From, _To);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CONVERTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_core_convertible.h b/libcxx/include/__cxx03/__type_traits/is_core_convertible.h
index 8088dfb7d9589..97db84044fe61 100644
--- a/libcxx/include/__cxx03/__type_traits/is_core_convertible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_core_convertible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -32,4 +32,4 @@ struct __is_core_convertible<_Tp, _Up, decltype(static_cast<void (*)(_Up)>(0)(st
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_destructible.h b/libcxx/include/__cxx03/__type_traits/is_destructible.h
index ce94e8c27f195..c2565050151c4 100644
--- a/libcxx/include/__cxx03/__type_traits/is_destructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_destructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_DESTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_DESTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -94,4 +94,4 @@ inline constexpr bool is_destructible_v = is_destructible<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_DESTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_empty.h b/libcxx/include/__cxx03/__type_traits/is_empty.h
index 89cb681712d31..767b155aa626b 100644
--- a/libcxx/include/__cxx03/__type_traits/is_empty.h
+++ b/libcxx/include/__cxx03/__type_traits/is_empty.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
-#define _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_EMPTY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_EMPTY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_empty_v = __is_empty(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_EMPTY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_EMPTY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_enum.h b/libcxx/include/__cxx03/__type_traits/is_enum.h
index da08f569586d6..54efb15b0d01e 100644
--- a/libcxx/include/__cxx03/__type_traits/is_enum.h
+++ b/libcxx/include/__cxx03/__type_traits/is_enum.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_ENUM_H
-#define _LIBCPP___TYPE_TRAITS_IS_ENUM_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_ENUM_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_ENUM_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -38,4 +38,4 @@ inline constexpr bool is_scoped_enum_v = __is_scoped_enum(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_ENUM_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_ENUM_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h b/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
index 7948482694132..da62137eaab5c 100644
--- a/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_equality_comparable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -85,4 +85,4 @@ using __libcpp_is_trivially_equality_comparable =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_EQUALITY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_execution_policy.h b/libcxx/include/__cxx03/__type_traits/is_execution_policy.h
index d0da5bb45a70f..08bf8b00d97ea 100644
--- a/libcxx/include/__cxx03/__type_traits/is_execution_policy.h
+++ b/libcxx/include/__cxx03/__type_traits/is_execution_policy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
-#define _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_EXECUTION_POLICY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/remove_cvref.h>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_EXECUTION_POLICY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_final.h b/libcxx/include/__cxx03/__type_traits/is_final.h
index b55fadce9efce..c6e5ad0893a0e 100644
--- a/libcxx/include/__cxx03/__type_traits/is_final.h
+++ b/libcxx/include/__cxx03/__type_traits/is_final.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_FINAL_H
-#define _LIBCPP___TYPE_TRAITS_IS_FINAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_FINAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_FINAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -33,4 +33,4 @@ inline constexpr bool is_final_v = __is_final(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_FINAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_FINAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_floating_point.h b/libcxx/include/__cxx03/__type_traits/is_floating_point.h
index 99898272651c4..6b25659c9ee26 100644
--- a/libcxx/include/__cxx03/__type_traits/is_floating_point.h
+++ b/libcxx/include/__cxx03/__type_traits/is_floating_point.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
-#define _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_FLOATING_POINT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_FLOATING_POINT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -36,4 +36,4 @@ inline constexpr bool is_floating_point_v = is_floating_point<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_FLOATING_POINT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_FLOATING_POINT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_function.h b/libcxx/include/__cxx03/__type_traits/is_function.h
index 0337dfac18c68..3001c4bef9f54 100644
--- a/libcxx/include/__cxx03/__type_traits/is_function.h
+++ b/libcxx/include/__cxx03/__type_traits/is_function.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
-#define _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNCTIONAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNCTIONAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_function_v = __is_function(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_FUNCTIONAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNCTIONAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_fundamental.h b/libcxx/include/__cxx03/__type_traits/is_fundamental.h
index cc57ac7b01948..329325ecca4bd 100644
--- a/libcxx/include/__cxx03/__type_traits/is_fundamental.h
+++ b/libcxx/include/__cxx03/__type_traits/is_fundamental.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
-#define _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNDAMENTAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNDAMENTAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -45,4 +45,4 @@ inline constexpr bool is_fundamental_v = is_fundamental<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_FUNDAMENTAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_FUNDAMENTAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h b/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
index b9bf0523d937c..23f1443e408f9 100644
--- a/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_implicitly_default_constructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -43,4 +43,4 @@ struct __is_implicitly_default_constructible<_Tp,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_IMPLICITLY_DEFAULT_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_integral.h b/libcxx/include/__cxx03/__type_traits/is_integral.h
index 680875c0e737d..1adcdcbce63e9 100644
--- a/libcxx/include/__cxx03/__type_traits/is_integral.h
+++ b/libcxx/include/__cxx03/__type_traits/is_integral.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
-#define _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_INTEGRAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_INTEGRAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -71,4 +71,4 @@ inline constexpr bool is_integral_v = is_integral<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_INTEGRAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_literal_type.h b/libcxx/include/__cxx03/__type_traits/is_literal_type.h
index ad7ad3f2eb3cf..6d0c9a6d6c30a 100644
--- a/libcxx/include/__cxx03/__type_traits/is_literal_type.h
+++ b/libcxx/include/__cxx03/__type_traits/is_literal_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
-#define _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_LITERAL_TYPE
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_LITERAL_TYPE
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -31,4 +31,4 @@ _LIBCPP_DEPRECATED_IN_CXX17 inline constexpr bool is_literal_type_v = __is_liter
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_LITERAL_TYPE
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_LITERAL_TYPE
diff --git a/libcxx/include/__cxx03/__type_traits/is_member_pointer.h b/libcxx/include/__cxx03/__type_traits/is_member_pointer.h
index 0eba57ddb827c..334b794f879a1 100644
--- a/libcxx/include/__cxx03/__type_traits/is_member_pointer.h
+++ b/libcxx/include/__cxx03/__type_traits/is_member_pointer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
-#define _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_MEMBER_POINTER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_MEMBER_POINTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -40,4 +40,4 @@ inline constexpr bool is_member_function_pointer_v = __is_member_function_pointe
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_MEMBER_POINTER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_MEMBER_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
index adf9be8e02518..64b525e254e1f 100644
--- a/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_assignable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -53,4 +53,4 @@ inline constexpr bool is_nothrow_move_assignable_v = is_nothrow_move_assignable<
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
index bbf0220c57153..b528cc7b491d7 100644
--- a/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_constructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -58,4 +58,4 @@ inline constexpr bool is_nothrow_default_constructible_v = __is_nothrow_construc
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
index b94727fce49e1..c62e6ccaf9fa1 100644
--- a/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_convertible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conjunction.h>
@@ -59,4 +59,4 @@ inline constexpr bool is_nothrow_convertible_v = is_nothrow_convertible<_Fm, _To
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_CONVERTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h b/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
index b84523f40b430..23a28c9bb1da0 100644
--- a/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_nothrow_destructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -60,4 +60,4 @@ inline constexpr bool is_nothrow_destructible_v = is_nothrow_destructible<_Tp>::
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_NOTHROW_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_null_pointer.h b/libcxx/include/__cxx03/__type_traits/is_null_pointer.h
index 9c605450638c3..acbf0c16a9643 100644
--- a/libcxx/include/__cxx03/__type_traits/is_null_pointer.h
+++ b/libcxx/include/__cxx03/__type_traits/is_null_pointer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
-#define _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_NULL_POINTER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_NULL_POINTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -34,4 +34,4 @@ inline constexpr bool is_null_pointer_v = __is_null_pointer_v<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_NULL_POINTER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_NULL_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_object.h b/libcxx/include/__cxx03/__type_traits/is_object.h
index c61c67dc51248..09d893d11220c 100644
--- a/libcxx/include/__cxx03/__type_traits/is_object.h
+++ b/libcxx/include/__cxx03/__type_traits/is_object.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
-#define _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_OBJECT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_OBJECT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_object_v = __is_object(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_OBJECT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_OBJECT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_pod.h b/libcxx/include/__cxx03/__type_traits/is_pod.h
index 3381a61657841..4b4e9cc3eb19f 100644
--- a/libcxx/include/__cxx03/__type_traits/is_pod.h
+++ b/libcxx/include/__cxx03/__type_traits/is_pod.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_POD_H
-#define _LIBCPP___TYPE_TRAITS_IS_POD_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_POD_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_POD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_pod_v = __is_pod(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_POD_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_POD_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_pointer.h b/libcxx/include/__cxx03/__type_traits/is_pointer.h
index 7afce1a07f281..8ce1421921499 100644
--- a/libcxx/include/__cxx03/__type_traits/is_pointer.h
+++ b/libcxx/include/__cxx03/__type_traits/is_pointer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_POINTER_H
-#define _LIBCPP___TYPE_TRAITS_IS_POINTER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_POINTER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_POINTER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -62,4 +62,4 @@ inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_POINTER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_polymorphic.h b/libcxx/include/__cxx03/__type_traits/is_polymorphic.h
index 577cfdbbf32c7..206abd871915f 100644
--- a/libcxx/include/__cxx03/__type_traits/is_polymorphic.h
+++ b/libcxx/include/__cxx03/__type_traits/is_polymorphic.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
-#define _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_POLYMORPHIC_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_POLYMORPHIC_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_polymorphic_v = __is_polymorphic(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_POLYMORPHIC_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_POLYMORPHIC_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_primary_template.h b/libcxx/include/__cxx03/__type_traits/is_primary_template.h
index 021cd9744471a..92cd5596fea3c 100644
--- a/libcxx/include/__cxx03/__type_traits/is_primary_template.h
+++ b/libcxx/include/__cxx03/__type_traits/is_primary_template.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
-#define _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -28,4 +28,4 @@ using __is_primary_template = _IsValidExpansion<__test_for_primary_template, _Tp
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_reference.h b/libcxx/include/__cxx03/__type_traits/is_reference.h
index cca4d1ccdac4b..72bdf1418f2f3 100644
--- a/libcxx/include/__cxx03/__type_traits/is_reference.h
+++ b/libcxx/include/__cxx03/__type_traits/is_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -65,4 +65,4 @@ inline constexpr bool is_rvalue_reference_v = is_rvalue_reference<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_REFERENCE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h b/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
index e3f06df804005..3cbce8faa9110 100644
--- a/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
+++ b/libcxx/include/__cxx03/__type_traits/is_reference_wrapper.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
-#define _LIBCPP___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCE_WRAPPER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/functional.h>
@@ -29,4 +29,4 @@ struct __is_reference_wrapper : public __is_reference_wrapper_impl<__remove_cv_t
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ENABLE_IF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_ENABLE_IF_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_referenceable.h b/libcxx/include/__cxx03/__type_traits/is_referenceable.h
index 13ef875538488..7bc98e8858001 100644
--- a/libcxx/include/__cxx03/__type_traits/is_referenceable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_referenceable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCEABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCEABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -38,4 +38,4 @@ struct __libcpp_is_referenceable
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_REFERENCEABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_same.h b/libcxx/include/__cxx03/__type_traits/is_same.h
index 29ff5116e8e89..d365e1235e7c6 100644
--- a/libcxx/include/__cxx03/__type_traits/is_same.h
+++ b/libcxx/include/__cxx03/__type_traits/is_same.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SAME_H
-#define _LIBCPP___TYPE_TRAITS_IS_SAME_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SAME_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SAME_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -41,4 +41,4 @@ using _IsNotSame = _BoolConstant<!__is_same(_Tp, _Up)>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SAME_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SAME_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_scalar.h b/libcxx/include/__cxx03/__type_traits/is_scalar.h
index 49d793ad06e71..b12e045f053a4 100644
--- a/libcxx/include/__cxx03/__type_traits/is_scalar.h
+++ b/libcxx/include/__cxx03/__type_traits/is_scalar.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
-#define _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SCALAR_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SCALAR_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -66,4 +66,4 @@ inline constexpr bool is_scalar_v = is_scalar<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SCALAR_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SCALAR_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_signed.h b/libcxx/include/__cxx03/__type_traits/is_signed.h
index 6f1cc05aaf373..20613cf63f7d0 100644
--- a/libcxx/include/__cxx03/__type_traits/is_signed.h
+++ b/libcxx/include/__cxx03/__type_traits/is_signed.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
-#define _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -56,4 +56,4 @@ inline constexpr bool is_signed_v = is_signed<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SIGNED_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_signed_integer.h b/libcxx/include/__cxx03/__type_traits/is_signed_integer.h
index 24b1aae258352..6a1198f24cd4c 100644
--- a/libcxx/include/__cxx03/__type_traits/is_signed_integer.h
+++ b/libcxx/include/__cxx03/__type_traits/is_signed_integer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
-#define _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_INTEGER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_INTEGER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -32,4 +32,4 @@ template <>          struct __libcpp_is_signed_integer<__int128_t>       : publi
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SIGNED_INTEGER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_specialization.h b/libcxx/include/__cxx03/__type_traits/is_specialization.h
index bccdd77df0e56..b1946df29587b 100644
--- a/libcxx/include/__cxx03/__type_traits/is_specialization.h
+++ b/libcxx/include/__cxx03/__type_traits/is_specialization.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
-#define _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SPECIALIZATION
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SPECIALIZATION
 
 // This contains parts of P2098R1 but is based on MSVC STL's implementation.
 //
@@ -42,4 +42,4 @@ inline constexpr bool __is_specialization_v<_Template<_Args...>, _Template> = tr
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SPECIALIZATION
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SPECIALIZATION
diff --git a/libcxx/include/__cxx03/__type_traits/is_standard_layout.h b/libcxx/include/__cxx03/__type_traits/is_standard_layout.h
index 1325f72478bdf..22889d5ff9a38 100644
--- a/libcxx/include/__cxx03/__type_traits/is_standard_layout.h
+++ b/libcxx/include/__cxx03/__type_traits/is_standard_layout.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
-#define _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_standard_layout_v = __is_standard_layout(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_STANDARD_LAYOUT_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_swappable.h b/libcxx/include/__cxx03/__type_traits/is_swappable.h
index 9c2e373675d8c..c3119c3de0895 100644
--- a/libcxx/include/__cxx03/__type_traits/is_swappable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_swappable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_SWAPPABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_SWAPPABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -101,4 +101,4 @@ struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable : bool_constant<is_nothrow_swap
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_SWAPPABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_SWAPPABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivial.h b/libcxx/include/__cxx03/__type_traits/is_trivial.h
index acb7a731ead52..e998255b813c5 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivial.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivial.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIAL_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_trivial_v = __is_trivial(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIAL_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIAL_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
index 757652572c81e..180e05de2b4ba 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_assignable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_const.h>
@@ -53,4 +53,4 @@ inline constexpr bool is_trivially_move_assignable_v = is_trivially_move_assigna
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h b/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
index 33110203fb422..a47e6d9031a53 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_constructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_lvalue_reference.h>
@@ -58,4 +58,4 @@ inline constexpr bool is_trivially_default_constructible_v = __is_trivially_cons
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
index c80688b38c43d..67a55e86e6b00 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_copyable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -34,4 +34,4 @@ inline constexpr bool __is_cheap_to_copy = is_trivially_copyable_v<_Tp> && sizeo
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h b/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
index 7c45619ea22c3..1dbd9e86a0691 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_destructible.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -44,4 +44,4 @@ inline constexpr bool is_trivially_destructible_v = is_trivially_destructible<_T
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_DESTRUCTIBLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
index 45043370a19e1..a5522b4427a9c 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_lexicographically_comparable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -50,4 +50,4 @@ struct __libcpp_is_trivially_lexicographically_comparable
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_LEXICOGRAPHICALLY_COMPARABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h b/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
index d134218cba3ff..062b289575bc1 100644
--- a/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
+++ b/libcxx/include/__cxx03/__type_traits/is_trivially_relocatable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -39,4 +39,4 @@ struct __libcpp_is_trivially_relocatable<_Tp,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_TRIVIALLY_RELOCATABLE_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h b/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
index e7262f5630287..d11a8094e9c13 100644
--- a/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
+++ b/libcxx/include/__cxx03/__type_traits/is_unbounded_array.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
-#define _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -37,4 +37,4 @@ inline constexpr bool is_unbounded_array_v = is_unbounded_array<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_UNBOUNDED_ARRAY_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_union.h b/libcxx/include/__cxx03/__type_traits/is_union.h
index 03733e6dac236..eaa222efde036 100644
--- a/libcxx/include/__cxx03/__type_traits/is_union.h
+++ b/libcxx/include/__cxx03/__type_traits/is_union.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_UNION_H
-#define _LIBCPP___TYPE_TRAITS_IS_UNION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_UNION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_UNION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_union_v = __is_union(_Tp);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_UNION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_UNION_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unsigned.h b/libcxx/include/__cxx03/__type_traits/is_unsigned.h
index b302380cd7838..99f888fb45bef 100644
--- a/libcxx/include/__cxx03/__type_traits/is_unsigned.h
+++ b/libcxx/include/__cxx03/__type_traits/is_unsigned.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
-#define _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -56,4 +56,4 @@ inline constexpr bool is_unsigned_v = is_unsigned<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h b/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
index 7a0508289ed64..250427cf8adf9 100644
--- a/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
+++ b/libcxx/include/__cxx03/__type_traits/is_unsigned_integer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
-#define _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -32,4 +32,4 @@ template <>          struct __libcpp_is_unsigned_integer<__uint128_t>        : p
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h b/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
index a2a4b30ff2f4f..d2b8927d78e24 100644
--- a/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
+++ b/libcxx/include/__cxx03/__type_traits/is_valid_expansion.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
-#define _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_VALID_EXPANSION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_VALID_EXPANSION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ using _IsValidExpansion _LIBCPP_NODEBUG = decltype(std::__sfinae_test_impl<_Temp
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_VALID_EXPANSION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_VALID_EXPANSION_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_void.h b/libcxx/include/__cxx03/__type_traits/is_void.h
index aa6712e19c33f..0f5e9cae179ce 100644
--- a/libcxx/include/__cxx03/__type_traits/is_void.h
+++ b/libcxx/include/__cxx03/__type_traits/is_void.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_VOID_H
-#define _LIBCPP___TYPE_TRAITS_IS_VOID_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_VOID_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_VOID_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -28,4 +28,4 @@ inline constexpr bool is_void_v = __is_same(__remove_cv(_Tp), void);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_VOID_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_VOID_H
diff --git a/libcxx/include/__cxx03/__type_traits/is_volatile.h b/libcxx/include/__cxx03/__type_traits/is_volatile.h
index 43f9d089d2600..75b6b49122ed8 100644
--- a/libcxx/include/__cxx03/__type_traits/is_volatile.h
+++ b/libcxx/include/__cxx03/__type_traits/is_volatile.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
-#define _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_IS_VOLATILE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_IS_VOLATILE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -44,4 +44,4 @@ inline constexpr bool is_volatile_v = is_volatile<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_IS_VOLATILE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_IS_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/lazy.h b/libcxx/include/__cxx03/__type_traits/lazy.h
index 4eb5893671f13..5362d98929894 100644
--- a/libcxx/include/__cxx03/__type_traits/lazy.h
+++ b/libcxx/include/__cxx03/__type_traits/lazy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_LAZY_H
-#define _LIBCPP___TYPE_TRAITS_LAZY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_LAZY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_LAZY_H
 
 #include <__cxx03/__config>
 
@@ -22,4 +22,4 @@ struct _Lazy : _Func<_Args...> {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_LAZY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_LAZY_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h b/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
index e1be7d2022edb..918a4c1db69d7 100644
--- a/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
+++ b/libcxx/include/__cxx03/__type_traits/make_32_64_or_128_bit.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
-#define _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -46,4 +46,4 @@ using __make_32_64_or_128_bit_t =
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_MAKE_32_64_OR_128_BIT_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h b/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
index cf9a46262a4b0..de7fc2a845967 100644
--- a/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
+++ b/libcxx/include/__cxx03/__type_traits/make_const_lvalue_ref.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
-#define _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/remove_reference.h>
@@ -23,4 +23,4 @@ using __make_const_lvalue_ref = const __libcpp_remove_reference_t<_Tp>&;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_MAKE_CONST_LVALUE_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_signed.h b/libcxx/include/__cxx03/__type_traits/make_signed.h
index 6fc97078d6536..fb359441ee6e5 100644
--- a/libcxx/include/__cxx03/__type_traits/make_signed.h
+++ b/libcxx/include/__cxx03/__type_traits/make_signed.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
-#define _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_MAKE_SIGNED_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_MAKE_SIGNED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/copy_cv.h>
@@ -86,4 +86,4 @@ using make_signed_t = __make_signed_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_MAKE_SIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/make_unsigned.h b/libcxx/include/__cxx03/__type_traits/make_unsigned.h
index 1b0312d61f9b4..3e2105242b665 100644
--- a/libcxx/include/__cxx03/__type_traits/make_unsigned.h
+++ b/libcxx/include/__cxx03/__type_traits/make_unsigned.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
-#define _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_MAKE_UNSIGNED_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_MAKE_UNSIGNED_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -98,4 +98,4 @@ using __copy_unsigned_t = __conditional_t<is_unsigned<_Tp>::value, __make_unsign
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_MAKE_UNSIGNED_H
diff --git a/libcxx/include/__cxx03/__type_traits/maybe_const.h b/libcxx/include/__cxx03/__type_traits/maybe_const.h
index 97928110edf24..88551f8fac1c6 100644
--- a/libcxx/include/__cxx03/__type_traits/maybe_const.h
+++ b/libcxx/include/__cxx03/__type_traits/maybe_const.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
-#define _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_MAYBE_CONST_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_MAYBE_CONST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -23,4 +23,4 @@ using __maybe_const = __conditional_t<_Const, const _Tp, _Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_MAYBE_CONST_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_MAYBE_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/nat.h b/libcxx/include/__cxx03/__type_traits/nat.h
index e476364c4950e..f45a7aadb3c79 100644
--- a/libcxx/include/__cxx03/__type_traits/nat.h
+++ b/libcxx/include/__cxx03/__type_traits/nat.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_NAT_H
-#define _LIBCPP___TYPE_TRAITS_NAT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_NAT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_NAT_H
 
 #include <__cxx03/__config>
 
@@ -28,4 +28,4 @@ struct __nat {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_NAT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_NAT_H
diff --git a/libcxx/include/__cxx03/__type_traits/negation.h b/libcxx/include/__cxx03/__type_traits/negation.h
index 56559071e62f7..d9b290e2e35d6 100644
--- a/libcxx/include/__cxx03/__type_traits/negation.h
+++ b/libcxx/include/__cxx03/__type_traits/negation.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_NEGATION_H
-#define _LIBCPP___TYPE_TRAITS_NEGATION_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_NEGATION_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_NEGATION_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -30,4 +30,4 @@ inline constexpr bool negation_v = !_Tp::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_NEGATION_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_NEGATION_H
diff --git a/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h b/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
index 4684440a5760f..d275ed585678b 100644
--- a/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
+++ b/libcxx/include/__cxx03/__type_traits/noexcept_move_assign_container.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
-#define _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/allocator_traits.h>
@@ -34,4 +34,4 @@ struct __noexcept_move_assign_container
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_NOEXCEPT_MOVE_ASSIGN_CONTAINER_H
diff --git a/libcxx/include/__cxx03/__type_traits/promote.h b/libcxx/include/__cxx03/__type_traits/promote.h
index b890aecdb5eb8..92671223e3dbd 100644
--- a/libcxx/include/__cxx03/__type_traits/promote.h
+++ b/libcxx/include/__cxx03/__type_traits/promote.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_PROMOTE_H
-#define _LIBCPP___TYPE_TRAITS_PROMOTE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_PROMOTE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_PROMOTE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -125,4 +125,4 @@ class __promote : public __promote_imp<_A1, _A2, _A3> {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_PROMOTE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_PROMOTE_H
diff --git a/libcxx/include/__cxx03/__type_traits/rank.h b/libcxx/include/__cxx03/__type_traits/rank.h
index 6f2109b6541f7..a6b2598f6d7ca 100644
--- a/libcxx/include/__cxx03/__type_traits/rank.h
+++ b/libcxx/include/__cxx03/__type_traits/rank.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_RANK_H
-#define _LIBCPP___TYPE_TRAITS_RANK_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_RANK_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_RANK_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -43,4 +43,4 @@ inline constexpr size_t rank_v = rank<_Tp>::value;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_RANK_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_RANK_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_all_extents.h b/libcxx/include/__cxx03/__type_traits/remove_all_extents.h
index 3d06a52117889..9a1fd0c7fb5bb 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_all_extents.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_all_extents.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -51,4 +51,4 @@ using remove_all_extents_t = __remove_all_extents_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_const.h b/libcxx/include/__cxx03/__type_traits/remove_const.h
index 06a70a260080c..885563bfcfbe3 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_const.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_const.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_H
 
 #include <__cxx03/__config>
 
@@ -46,4 +46,4 @@ using remove_const_t = __remove_const_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CONST_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_const_ref.h b/libcxx/include/__cxx03/__type_traits/remove_const_ref.h
index c7d058dc2a65e..24f7272880600 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_const_ref.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_const_ref.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_REF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_REF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/remove_const.h>
@@ -24,4 +24,4 @@ using __remove_const_ref_t = __remove_const_t<__libcpp_remove_reference_t<_Tp> >
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CONST_REF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CONST_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_cv.h b/libcxx/include/__cxx03/__type_traits/remove_cv.h
index 34bce7cf45ff1..1612d24464a8c 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_cv.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_cv.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CV_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CV_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/remove_const.h>
@@ -44,4 +44,4 @@ using remove_cv_t = __remove_cv_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CV_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CV_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_cvref.h b/libcxx/include/__cxx03/__type_traits/remove_cvref.h
index 38a37d730e44a..907622efe1a67 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_cvref.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_cvref.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CVREF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CVREF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_same.h>
@@ -43,4 +43,4 @@ using remove_cvref_t = __remove_cvref_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_CVREF_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_extent.h b/libcxx/include/__cxx03/__type_traits/remove_extent.h
index b1ccdc8a075d5..31a2535dca88a 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_extent.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_extent.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_EXTENT_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_EXTENT_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -51,4 +51,4 @@ using remove_extent_t = __remove_extent_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_EXTENT_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_pointer.h b/libcxx/include/__cxx03/__type_traits/remove_pointer.h
index 965ddf01a30f7..4d689f67d1765 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_pointer.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_pointer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_POINTER_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_POINTER_H
 
 #include <__cxx03/__config>
 
@@ -50,4 +50,4 @@ using remove_pointer_t = __remove_pointer_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_POINTER_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_POINTER_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_reference.h b/libcxx/include/__cxx03/__type_traits/remove_reference.h
index 768b9acd2b687..cd500ff761e28 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_reference.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_reference.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_REFERENCE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_REFERENCE_H
 
 #include <__cxx03/__config>
 
@@ -44,4 +44,4 @@ using remove_reference_t = __libcpp_remove_reference_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_REFERENCE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_REFERENCE_H
diff --git a/libcxx/include/__cxx03/__type_traits/remove_volatile.h b/libcxx/include/__cxx03/__type_traits/remove_volatile.h
index 95fc78ee8e41c..3c7e041398046 100644
--- a/libcxx/include/__cxx03/__type_traits/remove_volatile.h
+++ b/libcxx/include/__cxx03/__type_traits/remove_volatile.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
-#define _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_VOLATILE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_VOLATILE_H
 
 #include <__cxx03/__config>
 
@@ -46,4 +46,4 @@ using remove_volatile_t = __remove_volatile_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_REMOVE_VOLATILE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_REMOVE_VOLATILE_H
diff --git a/libcxx/include/__cxx03/__type_traits/result_of.h b/libcxx/include/__cxx03/__type_traits/result_of.h
index 7398f52f8965d..36d4180346547 100644
--- a/libcxx/include/__cxx03/__type_traits/result_of.h
+++ b/libcxx/include/__cxx03/__type_traits/result_of.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_RESULT_OF_H
-#define _LIBCPP___TYPE_TRAITS_RESULT_OF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_RESULT_OF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_RESULT_OF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__functional/invoke.h>
@@ -35,4 +35,4 @@ using result_of_t _LIBCPP_DEPRECATED_IN_CXX17 = typename result_of<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_RESULT_OF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_RESULT_OF_H
diff --git a/libcxx/include/__cxx03/__type_traits/strip_signature.h b/libcxx/include/__cxx03/__type_traits/strip_signature.h
index d728604ac2fec..2e7736609bf17 100644
--- a/libcxx/include/__cxx03/__type_traits/strip_signature.h
+++ b/libcxx/include/__cxx03/__type_traits/strip_signature.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
-#define _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_STRIP_SIGNATURE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_STRIP_SIGNATURE_H
 
 #include <__cxx03/__config>
 
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 17
 
-#endif // _LIBCPP___TYPE_TRAITS_STRIP_SIGNATURE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_STRIP_SIGNATURE_H
diff --git a/libcxx/include/__cxx03/__type_traits/type_identity.h b/libcxx/include/__cxx03/__type_traits/type_identity.h
index 27bbd7c51d426..1ff01002509de 100644
--- a/libcxx/include/__cxx03/__type_traits/type_identity.h
+++ b/libcxx/include/__cxx03/__type_traits/type_identity.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
-#define _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_TYPE_IDENTITY_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_TYPE_IDENTITY_H
 
 #include <__cxx03/__config>
 
@@ -36,4 +36,4 @@ using type_identity_t = typename type_identity<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_TYPE_IDENTITY_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_TYPE_IDENTITY_H
diff --git a/libcxx/include/__cxx03/__type_traits/type_list.h b/libcxx/include/__cxx03/__type_traits/type_list.h
index 31717a60b13cf..40d6f29276347 100644
--- a/libcxx/include/__cxx03/__type_traits/type_list.h
+++ b/libcxx/include/__cxx03/__type_traits/type_list.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
-#define _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_TYPE_LIST_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_TYPE_LIST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -39,4 +39,4 @@ struct __find_first<__type_list<_Hp, _Tp>, _Size, false> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_TYPE_LIST_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_TYPE_LIST_H
diff --git a/libcxx/include/__cxx03/__type_traits/underlying_type.h b/libcxx/include/__cxx03/__type_traits/underlying_type.h
index da9d3df264526..6e82ff83a6f71 100644
--- a/libcxx/include/__cxx03/__type_traits/underlying_type.h
+++ b/libcxx/include/__cxx03/__type_traits/underlying_type.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
-#define _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_UNDERLYING_TYPE_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_UNDERLYING_TYPE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_enum.h>
@@ -39,4 +39,4 @@ using underlying_type_t = typename underlying_type<_Tp>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_UNDERLYING_TYPE_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_UNDERLYING_TYPE_H
diff --git a/libcxx/include/__cxx03/__type_traits/unwrap_ref.h b/libcxx/include/__cxx03/__type_traits/unwrap_ref.h
index 034bf337518de..a5628ece8de31 100644
--- a/libcxx/include/__cxx03/__type_traits/unwrap_ref.h
+++ b/libcxx/include/__cxx03/__type_traits/unwrap_ref.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
-#define _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_UNWRAP_REF_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_UNWRAP_REF_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__fwd/functional.h>
@@ -55,4 +55,4 @@ struct __unwrap_ref_decay
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_UNWRAP_REF_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_UNWRAP_REF_H
diff --git a/libcxx/include/__cxx03/__type_traits/void_t.h b/libcxx/include/__cxx03/__type_traits/void_t.h
index 2fa557ab38178..a639a5f73d2b4 100644
--- a/libcxx/include/__cxx03/__type_traits/void_t.h
+++ b/libcxx/include/__cxx03/__type_traits/void_t.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_VOID_T_H
-#define _LIBCPP___TYPE_TRAITS_VOID_T_H
+#ifndef _LIBCPP___CXX03___TYPE_TRAITS_VOID_T_H
+#define _LIBCPP___CXX03___TYPE_TRAITS_VOID_T_H
 
 #include <__cxx03/__config>
 
@@ -27,4 +27,4 @@ using __void_t = void;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_VOID_T_H
+#endif // _LIBCPP___CXX03___TYPE_TRAITS_VOID_T_H
diff --git a/libcxx/include/__cxx03/__utility/as_const.h b/libcxx/include/__cxx03/__utility/as_const.h
index 33c4686d663e6..ba1b67216f1ce 100644
--- a/libcxx/include/__cxx03/__utility/as_const.h
+++ b/libcxx/include/__cxx03/__utility/as_const.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_AS_CONST_H
-#define _LIBCPP___UTILITY_AS_CONST_H
+#ifndef _LIBCPP___CXX03___UTILITY_AS_CONST_H
+#define _LIBCPP___CXX03___UTILITY_AS_CONST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/add_const.h>
@@ -32,4 +32,4 @@ void as_const(const _Tp&&) = delete;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_AS_CONST_H
+#endif // _LIBCPP___CXX03___UTILITY_AS_CONST_H
diff --git a/libcxx/include/__cxx03/__utility/as_lvalue.h b/libcxx/include/__cxx03/__utility/as_lvalue.h
index 95ce9497207d2..ea5404ad0bd2b 100644
--- a/libcxx/include/__cxx03/__utility/as_lvalue.h
+++ b/libcxx/include/__cxx03/__utility/as_lvalue.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_AS_LVALUE_H
-#define _LIBCPP___UTILITY_AS_LVALUE_H
+#ifndef _LIBCPP___CXX03___UTILITY_AS_LVALUE_H
+#define _LIBCPP___CXX03___UTILITY_AS_LVALUE_H
 
 #include <__cxx03/__config>
 
@@ -34,4 +34,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_AS_LVALUE_H
+#endif // _LIBCPP___CXX03___UTILITY_AS_LVALUE_H
diff --git a/libcxx/include/__cxx03/__utility/auto_cast.h b/libcxx/include/__cxx03/__utility/auto_cast.h
index 9932382c63063..c506e891121e0 100644
--- a/libcxx/include/__cxx03/__utility/auto_cast.h
+++ b/libcxx/include/__cxx03/__utility/auto_cast.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_AUTO_CAST_H
-#define _LIBCPP___UTILITY_AUTO_CAST_H
+#ifndef _LIBCPP___CXX03___UTILITY_AUTO_CAST_H
+#define _LIBCPP___CXX03___UTILITY_AUTO_CAST_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/decay.h>
@@ -19,4 +19,4 @@
 
 #define _LIBCPP_AUTO_CAST(expr) static_cast<::std::__decay_t<decltype((expr))> >(expr)
 
-#endif // _LIBCPP___UTILITY_AUTO_CAST_H
+#endif // _LIBCPP___CXX03___UTILITY_AUTO_CAST_H
diff --git a/libcxx/include/__cxx03/__utility/cmp.h b/libcxx/include/__cxx03/__utility/cmp.h
index 512a31857642a..afb4d83a2a295 100644
--- a/libcxx/include/__cxx03/__utility/cmp.h
+++ b/libcxx/include/__cxx03/__utility/cmp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_CMP_H
-#define _LIBCPP___UTILITY_CMP_H
+#ifndef _LIBCPP___CXX03___UTILITY_CMP_H
+#define _LIBCPP___CXX03___UTILITY_CMP_H
 
 #include <__cxx03/__concepts/arithmetic.h>
 #include <__cxx03/__config>
@@ -78,4 +78,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_CMP_H
+#endif // _LIBCPP___CXX03___UTILITY_CMP_H
diff --git a/libcxx/include/__cxx03/__utility/convert_to_integral.h b/libcxx/include/__cxx03/__utility/convert_to_integral.h
index b350d0fd9d934..e1cc9195de165 100644
--- a/libcxx/include/__cxx03/__utility/convert_to_integral.h
+++ b/libcxx/include/__cxx03/__utility/convert_to_integral.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
-#define _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
+#ifndef _LIBCPP___CXX03___UTILITY_CONVERT_TO_INTEGRAL_H
+#define _LIBCPP___CXX03___UTILITY_CONVERT_TO_INTEGRAL_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/enable_if.h>
@@ -65,4 +65,4 @@ __convert_to_integral(_Tp __val) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_CONVERT_TO_INTEGRAL_H
+#endif // _LIBCPP___CXX03___UTILITY_CONVERT_TO_INTEGRAL_H
diff --git a/libcxx/include/__cxx03/__utility/declval.h b/libcxx/include/__cxx03/__utility/declval.h
index f8c2c2ac4287a..8992c5a0b6dd3 100644
--- a/libcxx/include/__cxx03/__utility/declval.h
+++ b/libcxx/include/__cxx03/__utility/declval.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_DECLVAL_H
-#define _LIBCPP___UTILITY_DECLVAL_H
+#ifndef _LIBCPP___CXX03___UTILITY_DECLVAL_H
+#define _LIBCPP___CXX03___UTILITY_DECLVAL_H
 
 #include <__cxx03/__config>
 
@@ -35,4 +35,4 @@ _LIBCPP_HIDE_FROM_ABI decltype(std::__declval<_Tp>(0)) declval() _NOEXCEPT {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_DECLVAL_H
+#endif // _LIBCPP___CXX03___UTILITY_DECLVAL_H
diff --git a/libcxx/include/__cxx03/__utility/empty.h b/libcxx/include/__cxx03/__utility/empty.h
index bd169acabc489..b6623020fe50c 100644
--- a/libcxx/include/__cxx03/__utility/empty.h
+++ b/libcxx/include/__cxx03/__utility/empty.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_EMPTY_H
-#define _LIBCPP___UTILITY_EMPTY_H
+#ifndef _LIBCPP___CXX03___UTILITY_EMPTY_H
+#define _LIBCPP___CXX03___UTILITY_EMPTY_H
 
 #include <__cxx03/__config>
 
@@ -21,4 +21,4 @@ struct __empty {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_EMPTY_H
+#endif // _LIBCPP___CXX03___UTILITY_EMPTY_H
diff --git a/libcxx/include/__cxx03/__utility/exception_guard.h b/libcxx/include/__cxx03/__utility/exception_guard.h
index ec4c58fca0ff6..2cdb9b41cd0ed 100644
--- a/libcxx/include/__cxx03/__utility/exception_guard.h
+++ b/libcxx/include/__cxx03/__utility/exception_guard.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_TRANSACTION_H
-#define _LIBCPP___UTILITY_TRANSACTION_H
+#ifndef _LIBCPP___CXX03___UTILITY_TRANSACTION_H
+#define _LIBCPP___CXX03___UTILITY_TRANSACTION_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -141,4 +141,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_TRANSACTION_H
+#endif // _LIBCPP___CXX03___UTILITY_TRANSACTION_H
diff --git a/libcxx/include/__cxx03/__utility/exchange.h b/libcxx/include/__cxx03/__utility/exchange.h
index 258d4c4ce4e9e..682e7d78e65d2 100644
--- a/libcxx/include/__cxx03/__utility/exchange.h
+++ b/libcxx/include/__cxx03/__utility/exchange.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_EXCHANGE_H
-#define _LIBCPP___UTILITY_EXCHANGE_H
+#ifndef _LIBCPP___CXX03___UTILITY_EXCHANGE_H
+#define _LIBCPP___CXX03___UTILITY_EXCHANGE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_nothrow_assignable.h>
@@ -38,4 +38,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_EXCHANGE_H
+#endif // _LIBCPP___CXX03___UTILITY_EXCHANGE_H
diff --git a/libcxx/include/__cxx03/__utility/forward.h b/libcxx/include/__cxx03/__utility/forward.h
index ac63e2cafd3d8..fc0b45a0ed7ce 100644
--- a/libcxx/include/__cxx03/__utility/forward.h
+++ b/libcxx/include/__cxx03/__utility/forward.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_FORWARD_H
-#define _LIBCPP___UTILITY_FORWARD_H
+#ifndef _LIBCPP___CXX03___UTILITY_FORWARD_H
+#define _LIBCPP___CXX03___UTILITY_FORWARD_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_reference.h>
@@ -35,4 +35,4 @@ forward(_LIBCPP_LIFETIMEBOUND __libcpp_remove_reference_t<_Tp>&& __t) _NOEXCEPT
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_FORWARD_H
+#endif // _LIBCPP___CXX03___UTILITY_FORWARD_H
diff --git a/libcxx/include/__cxx03/__utility/forward_like.h b/libcxx/include/__cxx03/__utility/forward_like.h
index ed5cbee0be0c4..989a76ed5b102 100644
--- a/libcxx/include/__cxx03/__utility/forward_like.h
+++ b/libcxx/include/__cxx03/__utility/forward_like.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_FORWARD_LIKE_H
-#define _LIBCPP___UTILITY_FORWARD_LIKE_H
+#ifndef _LIBCPP___CXX03___UTILITY_FORWARD_LIKE_H
+#define _LIBCPP___CXX03___UTILITY_FORWARD_LIKE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -43,4 +43,4 @@ forward_like(_LIBCPP_LIFETIMEBOUND _Up&& __ux) noexcept -> _ForwardLike<_Tp, _Up
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_FORWARD_LIKE_H
+#endif // _LIBCPP___CXX03___UTILITY_FORWARD_LIKE_H
diff --git a/libcxx/include/__cxx03/__utility/in_place.h b/libcxx/include/__cxx03/__utility/in_place.h
index b0c257dec4652..2ab0be25579b8 100644
--- a/libcxx/include/__cxx03/__utility/in_place.h
+++ b/libcxx/include/__cxx03/__utility/in_place.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_IN_PLACE_H
-#define _LIBCPP___UTILITY_IN_PLACE_H
+#ifndef _LIBCPP___CXX03___UTILITY_IN_PLACE_H
+#define _LIBCPP___CXX03___UTILITY_IN_PLACE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/remove_cvref.h>
@@ -60,4 +60,4 @@ using __is_inplace_index = __is_inplace_index_imp<__remove_cvref_t<_Tp>>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_IN_PLACE_H
+#endif // _LIBCPP___CXX03___UTILITY_IN_PLACE_H
diff --git a/libcxx/include/__cxx03/__utility/integer_sequence.h b/libcxx/include/__cxx03/__utility/integer_sequence.h
index 5f3d1a7630018..f5fdf6325e5b2 100644
--- a/libcxx/include/__cxx03/__utility/integer_sequence.h
+++ b/libcxx/include/__cxx03/__utility/integer_sequence.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
-#define _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
+#ifndef _LIBCPP___CXX03___UTILITY_INTEGER_SEQUENCE_H
+#define _LIBCPP___CXX03___UTILITY_INTEGER_SEQUENCE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_integral.h>
@@ -87,4 +87,4 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __for_each_index_sequence(index_sequence<_I
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_INTEGER_SEQUENCE_H
+#endif // _LIBCPP___CXX03___UTILITY_INTEGER_SEQUENCE_H
diff --git a/libcxx/include/__cxx03/__utility/is_pointer_in_range.h b/libcxx/include/__cxx03/__utility/is_pointer_in_range.h
index 8773c526a0a3f..4d7d3c5e039f2 100644
--- a/libcxx/include/__cxx03/__utility/is_pointer_in_range.h
+++ b/libcxx/include/__cxx03/__utility/is_pointer_in_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
-#define _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
+#ifndef _LIBCPP___CXX03___UTILITY_IS_POINTER_IN_RANGE_H
+#define _LIBCPP___CXX03___UTILITY_IS_POINTER_IN_RANGE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__assert>
@@ -59,4 +59,4 @@ __is_pointer_in_range(const _Tp* __begin, const _Tp* __end, const _Up* __ptr) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_IS_POINTER_IN_RANGE_H
+#endif // _LIBCPP___CXX03___UTILITY_IS_POINTER_IN_RANGE_H
diff --git a/libcxx/include/__cxx03/__utility/is_valid_range.h b/libcxx/include/__cxx03/__utility/is_valid_range.h
index a7027bd582cb7..b3770c2c428b0 100644
--- a/libcxx/include/__cxx03/__utility/is_valid_range.h
+++ b/libcxx/include/__cxx03/__utility/is_valid_range.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_IS_VALID_RANGE_H
-#define _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#ifndef _LIBCPP___CXX03___UTILITY_IS_VALID_RANGE_H
+#define _LIBCPP___CXX03___UTILITY_IS_VALID_RANGE_H
 
 #include <__cxx03/__algorithm/comp.h>
 #include <__cxx03/__config>
@@ -34,4 +34,4 @@ __is_valid_range(const _Tp* __first, const _Tp* __last) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#endif // _LIBCPP___CXX03___UTILITY_IS_VALID_RANGE_H
diff --git a/libcxx/include/__cxx03/__utility/move.h b/libcxx/include/__cxx03/__utility/move.h
index 89fa877161e65..18692ba9ec128 100644
--- a/libcxx/include/__cxx03/__utility/move.h
+++ b/libcxx/include/__cxx03/__utility/move.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_MOVE_H
-#define _LIBCPP___UTILITY_MOVE_H
+#ifndef _LIBCPP___CXX03___UTILITY_MOVE_H
+#define _LIBCPP___CXX03___UTILITY_MOVE_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/conditional.h>
@@ -46,4 +46,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_MOVE_H
+#endif // _LIBCPP___CXX03___UTILITY_MOVE_H
diff --git a/libcxx/include/__cxx03/__utility/no_destroy.h b/libcxx/include/__cxx03/__utility/no_destroy.h
index 14ac437ecf713..5c3dfc5d81e65 100644
--- a/libcxx/include/__cxx03/__utility/no_destroy.h
+++ b/libcxx/include/__cxx03/__utility/no_destroy.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_NO_DESTROY_H
-#define _LIBCPP___UTILITY_NO_DESTROY_H
+#ifndef _LIBCPP___CXX03___UTILITY_NO_DESTROY_H
+#define _LIBCPP___CXX03___UTILITY_NO_DESTROY_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_constant_evaluated.h>
@@ -51,4 +51,4 @@ struct __no_destroy {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_NO_DESTROY_H
+#endif // _LIBCPP___CXX03___UTILITY_NO_DESTROY_H
diff --git a/libcxx/include/__cxx03/__utility/pair.h b/libcxx/include/__cxx03/__utility/pair.h
index 175976a8f627b..496fbbba675a4 100644
--- a/libcxx/include/__cxx03/__utility/pair.h
+++ b/libcxx/include/__cxx03/__utility/pair.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_PAIR_H
-#define _LIBCPP___UTILITY_PAIR_H
+#ifndef _LIBCPP___CXX03___UTILITY_PAIR_H
+#define _LIBCPP___CXX03___UTILITY_PAIR_H
 
 #include <__cxx03/__compare/common_comparison_category.h>
 #include <__cxx03/__compare/synth_three_way.h>
@@ -715,4 +715,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_PAIR_H
+#endif // _LIBCPP___CXX03___UTILITY_PAIR_H
diff --git a/libcxx/include/__cxx03/__utility/piecewise_construct.h b/libcxx/include/__cxx03/__utility/piecewise_construct.h
index 10ad333f81f14..a45d38070f09d 100644
--- a/libcxx/include/__cxx03/__utility/piecewise_construct.h
+++ b/libcxx/include/__cxx03/__utility/piecewise_construct.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
-#define _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
+#ifndef _LIBCPP___CXX03___UTILITY_PIECEWISE_CONSTRUCT_H
+#define _LIBCPP___CXX03___UTILITY_PIECEWISE_CONSTRUCT_H
 
 #include <__cxx03/__config>
 
@@ -29,4 +29,4 @@ constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_PIECEWISE_CONSTRUCT_H
+#endif // _LIBCPP___CXX03___UTILITY_PIECEWISE_CONSTRUCT_H
diff --git a/libcxx/include/__cxx03/__utility/priority_tag.h b/libcxx/include/__cxx03/__utility/priority_tag.h
index 8d8cf0e70056f..cefcc927b231b 100644
--- a/libcxx/include/__cxx03/__utility/priority_tag.h
+++ b/libcxx/include/__cxx03/__utility/priority_tag.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_PRIORITY_TAG_H
-#define _LIBCPP___UTILITY_PRIORITY_TAG_H
+#ifndef _LIBCPP___CXX03___UTILITY_PRIORITY_TAG_H
+#define _LIBCPP___CXX03___UTILITY_PRIORITY_TAG_H
 
 #include <__cxx03/__config>
 #include <__cxx03/cstddef>
@@ -25,4 +25,4 @@ struct __priority_tag<0> {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_PRIORITY_TAG_H
+#endif // _LIBCPP___CXX03___UTILITY_PRIORITY_TAG_H
diff --git a/libcxx/include/__cxx03/__utility/private_constructor_tag.h b/libcxx/include/__cxx03/__utility/private_constructor_tag.h
index c2198dbdd420f..2db48a6c9c060 100644
--- a/libcxx/include/__cxx03/__utility/private_constructor_tag.h
+++ b/libcxx/include/__cxx03/__utility/private_constructor_tag.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
-#define _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
+#ifndef _LIBCPP___CXX03__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
+#define _LIBCPP___CXX03__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
 
 #include <__cxx03/__config>
 
@@ -25,4 +25,4 @@ struct __private_constructor_tag {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
+#endif // _LIBCPP___CXX03__UTILITY_PRIVATE_CONSTRUCTOR_TAG_H
diff --git a/libcxx/include/__cxx03/__utility/rel_ops.h b/libcxx/include/__cxx03/__utility/rel_ops.h
index f066a816fcfb9..b8fadd4d86227 100644
--- a/libcxx/include/__cxx03/__utility/rel_ops.h
+++ b/libcxx/include/__cxx03/__utility/rel_ops.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_REL_OPS_H
-#define _LIBCPP___UTILITY_REL_OPS_H
+#ifndef _LIBCPP___CXX03___UTILITY_REL_OPS_H
+#define _LIBCPP___CXX03___UTILITY_REL_OPS_H
 
 #include <__cxx03/__config>
 
@@ -43,4 +43,4 @@ inline _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool operator>=(const _
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_REL_OPS_H
+#endif // _LIBCPP___CXX03___UTILITY_REL_OPS_H
diff --git a/libcxx/include/__cxx03/__utility/small_buffer.h b/libcxx/include/__cxx03/__utility/small_buffer.h
index 4836a739ed085..fa5e5b238ab9d 100644
--- a/libcxx/include/__cxx03/__utility/small_buffer.h
+++ b/libcxx/include/__cxx03/__utility/small_buffer.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_SMALL_BUFFER_H
-#define _LIBCPP___UTILITY_SMALL_BUFFER_H
+#ifndef _LIBCPP___CXX03___UTILITY_SMALL_BUFFER_H
+#define _LIBCPP___CXX03___UTILITY_SMALL_BUFFER_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__memory/construct_at.h>
@@ -96,4 +96,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 23
 
-#endif // _LIBCPP___UTILITY_SMALL_BUFFER_H
+#endif // _LIBCPP___CXX03___UTILITY_SMALL_BUFFER_H
diff --git a/libcxx/include/__cxx03/__utility/swap.h b/libcxx/include/__cxx03/__utility/swap.h
index 6ad367509635c..d74ec818084ca 100644
--- a/libcxx/include/__cxx03/__utility/swap.h
+++ b/libcxx/include/__cxx03/__utility/swap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_SWAP_H
-#define _LIBCPP___UTILITY_SWAP_H
+#ifndef _LIBCPP___CXX03___UTILITY_SWAP_H
+#define _LIBCPP___CXX03___UTILITY_SWAP_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/is_assignable.h>
@@ -56,4 +56,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___UTILITY_SWAP_H
+#endif // _LIBCPP___CXX03___UTILITY_SWAP_H
diff --git a/libcxx/include/__cxx03/__utility/to_underlying.h b/libcxx/include/__cxx03/__utility/to_underlying.h
index 8d33de1714c72..9006012f5a19e 100644
--- a/libcxx/include/__cxx03/__utility/to_underlying.h
+++ b/libcxx/include/__cxx03/__utility/to_underlying.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_TO_UNDERLYING_H
-#define _LIBCPP___UTILITY_TO_UNDERLYING_H
+#ifndef _LIBCPP___CXX03___UTILITY_TO_UNDERLYING_H
+#define _LIBCPP___CXX03___UTILITY_TO_UNDERLYING_H
 
 #include <__cxx03/__config>
 #include <__cxx03/__type_traits/underlying_type.h>
@@ -35,4 +35,4 @@ template <class _Tp>
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_TO_UNDERLYING_H
+#endif // _LIBCPP___CXX03___UTILITY_TO_UNDERLYING_H
diff --git a/libcxx/include/__cxx03/__utility/unreachable.h b/libcxx/include/__cxx03/__utility/unreachable.h
index fb4a775d02440..373cddc1365ca 100644
--- a/libcxx/include/__cxx03/__utility/unreachable.h
+++ b/libcxx/include/__cxx03/__utility/unreachable.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_UNREACHABLE_H
-#define _LIBCPP___UTILITY_UNREACHABLE_H
+#ifndef _LIBCPP___CXX03___UTILITY_UNREACHABLE_H
+#define _LIBCPP___CXX03___UTILITY_UNREACHABLE_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__config>
@@ -31,4 +31,4 @@ _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __libcpp_unreachable() {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_UNREACHABLE_H
+#endif // _LIBCPP___CXX03___UTILITY_UNREACHABLE_H
diff --git a/libcxx/include/__cxx03/__variant/monostate.h b/libcxx/include/__cxx03/__variant/monostate.h
index 6a136e75516b6..66b94618a0fe1 100644
--- a/libcxx/include/__cxx03/__variant/monostate.h
+++ b/libcxx/include/__cxx03/__variant/monostate.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___VARIANT_MONOSTATE_H
-#define _LIBCPP___VARIANT_MONOSTATE_H
+#ifndef _LIBCPP___CXX03___VARIANT_MONOSTATE_H
+#define _LIBCPP___CXX03___VARIANT_MONOSTATE_H
 
 #include <__cxx03/__compare/ordering.h>
 #include <__cxx03/__config>
@@ -61,4 +61,4 @@ struct _LIBCPP_TEMPLATE_VIS hash<monostate> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___VARIANT_MONOSTATE_H
+#endif // _LIBCPP___CXX03___VARIANT_MONOSTATE_H
diff --git a/libcxx/include/__cxx03/__verbose_abort b/libcxx/include/__cxx03/__verbose_abort
index 4eb994a4f2043..4fcfffa2b4dfa 100644
--- a/libcxx/include/__cxx03/__verbose_abort
+++ b/libcxx/include/__cxx03/__verbose_abort
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___VERBOSE_ABORT
-#define _LIBCPP___VERBOSE_ABORT
+#ifndef _LIBCPP___CXX03___VERBOSE_ABORT
+#define _LIBCPP___CXX03___VERBOSE_ABORT
 
 #include <__cxx03/__config>
 
@@ -49,4 +49,4 @@ void __use(const char*, ...);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___VERBOSE_ABORT
+#endif // _LIBCPP___CXX03___VERBOSE_ABORT
diff --git a/libcxx/include/__cxx03/algorithm b/libcxx/include/__cxx03/algorithm
index 929e2c5b7422a..8eff4d152adb2 100644
--- a/libcxx/include/__cxx03/algorithm
+++ b/libcxx/include/__cxx03/algorithm
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ALGORITHM
-#define _LIBCPP_ALGORITHM
+#ifndef _LIBCPP___CXX03_ALGORITHM
+#define _LIBCPP___CXX03_ALGORITHM
 
 /*
     algorithm synopsis
@@ -2046,4 +2046,4 @@ template <class BidirectionalIterator, class Compare>
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_ALGORITHM
+#endif // _LIBCPP___CXX03_ALGORITHM
diff --git a/libcxx/include/__cxx03/any b/libcxx/include/__cxx03/any
index 06ce9336abbdf..01289c5c4f0a0 100644
--- a/libcxx/include/__cxx03/any
+++ b/libcxx/include/__cxx03/any
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ANY
-#define _LIBCPP_ANY
+#ifndef _LIBCPP___CXX03_ANY
+#define _LIBCPP___CXX03_ANY
 
 /*
    any synopsis
@@ -611,4 +611,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/variant>
 #endif
 
-#endif // _LIBCPP_ANY
+#endif // _LIBCPP___CXX03_ANY
diff --git a/libcxx/include/__cxx03/array b/libcxx/include/__cxx03/array
index 2686dd85668b6..74dc287aa54d6 100644
--- a/libcxx/include/__cxx03/array
+++ b/libcxx/include/__cxx03/array
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ARRAY
-#define _LIBCPP_ARRAY
+#ifndef _LIBCPP___CXX03_ARRAY
+#define _LIBCPP___CXX03_ARRAY
 
 /*
     array synopsis
@@ -511,4 +511,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_ARRAY
+#endif // _LIBCPP___CXX03_ARRAY
diff --git a/libcxx/include/__cxx03/atomic b/libcxx/include/__cxx03/atomic
index 7a0b5bd38fb94..1b488d99f80fb 100644
--- a/libcxx/include/__cxx03/atomic
+++ b/libcxx/include/__cxx03/atomic
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ATOMIC
-#define _LIBCPP_ATOMIC
+#ifndef _LIBCPP___CXX03_ATOMIC
+#define _LIBCPP___CXX03_ATOMIC
 
 /*
     atomic synopsis
@@ -589,7 +589,7 @@ template <class T>
 
 #include <__cxx03/__config>
 
-#if _LIBCPP_STD_VER < 23 && defined(_LIBCPP_STDATOMIC_H)
+#if _LIBCPP_STD_VER < 23 && defined(_LIBCPP___CXX03_STDATOMIC_H)
 #  error <atomic> is incompatible with <stdatomic.h> before C++23. Please compile with -std=c++23.
 #endif
 
@@ -629,4 +629,4 @@ template <class T>
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_ATOMIC
+#endif // _LIBCPP___CXX03_ATOMIC
diff --git a/libcxx/include/__cxx03/barrier b/libcxx/include/__cxx03/barrier
index 5aee9e7210ec4..5e11f2439c402 100644
--- a/libcxx/include/__cxx03/barrier
+++ b/libcxx/include/__cxx03/barrier
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_BARRIER
-#define _LIBCPP_BARRIER
+#ifndef _LIBCPP___CXX03_BARRIER
+#define _LIBCPP___CXX03_BARRIER
 
 /*
     barrier synopsis
@@ -305,4 +305,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/variant>
 #endif
 
-#endif //_LIBCPP_BARRIER
+#endif //_LIBCPP___CXX03_BARRIER
diff --git a/libcxx/include/__cxx03/bit b/libcxx/include/__cxx03/bit
index e220eedfeaa92..29da3a21c8924 100644
--- a/libcxx/include/__cxx03/bit
+++ b/libcxx/include/__cxx03/bit
@@ -7,8 +7,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_BIT
-#define _LIBCPP_BIT
+#ifndef _LIBCPP___CXX03_BIT
+#define _LIBCPP___CXX03_BIT
 
 /*
     bit synopsis
@@ -98,4 +98,4 @@ namespace std {
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_BIT
+#endif // _LIBCPP___CXX03_BIT
diff --git a/libcxx/include/__cxx03/bitset b/libcxx/include/__cxx03/bitset
index f41d78ce27f08..1507d0cdaf2f0 100644
--- a/libcxx/include/__cxx03/bitset
+++ b/libcxx/include/__cxx03/bitset
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_BITSET
-#define _LIBCPP_BITSET
+#ifndef _LIBCPP___CXX03_BITSET
+#define _LIBCPP___CXX03_BITSET
 
 // clang-format off
 
@@ -966,4 +966,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_BITSET
+#endif // _LIBCPP___CXX03_BITSET
diff --git a/libcxx/include/__cxx03/ccomplex b/libcxx/include/__cxx03/ccomplex
index fb16b8fbb54af..a96d7512661c7 100644
--- a/libcxx/include/__cxx03/ccomplex
+++ b/libcxx/include/__cxx03/ccomplex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CCOMPLEX
-#define _LIBCPP_CCOMPLEX
+#ifndef _LIBCPP___CXX03_CCOMPLEX
+#define _LIBCPP___CXX03_CCOMPLEX
 
 /*
     ccomplex synopsis
@@ -23,4 +23,4 @@
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CCOMPLEX
+#endif // _LIBCPP___CXX03_CCOMPLEX
diff --git a/libcxx/include/__cxx03/cctype b/libcxx/include/__cxx03/cctype
index 1bfb66b704a27..7f1becb46341b 100644
--- a/libcxx/include/__cxx03/cctype
+++ b/libcxx/include/__cxx03/cctype
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CCTYPE
-#define _LIBCPP_CCTYPE
+#ifndef _LIBCPP___CXX03_CCTYPE
+#define _LIBCPP___CXX03_CCTYPE
 
 /*
     cctype synopsis
@@ -38,7 +38,7 @@ int toupper(int c);
 
 #include <__cxx03/ctype.h>
 
-#ifndef _LIBCPP_CTYPE_H
+#ifndef _LIBCPP___CXX03_CTYPE_H
 #   error <cctype> tried including <ctype.h> but didn't find libc++'s <ctype.h> header. \
           This usually means that your header search paths are not configured properly.  \
           The header search paths should contain the C++ Standard Library headers before \
@@ -124,4 +124,4 @@ using ::toupper _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CCTYPE
+#endif // _LIBCPP___CXX03_CCTYPE
diff --git a/libcxx/include/__cxx03/cerrno b/libcxx/include/__cxx03/cerrno
index 987e7492ce05a..acb894ea19495 100644
--- a/libcxx/include/__cxx03/cerrno
+++ b/libcxx/include/__cxx03/cerrno
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CERRNO
-#define _LIBCPP_CERRNO
+#ifndef _LIBCPP___CXX03_CERRNO
+#define _LIBCPP___CXX03_CERRNO
 
 /*
     cerrno synopsis
@@ -26,7 +26,7 @@ Macros:
 
 #include <__cxx03/errno.h>
 
-#ifndef _LIBCPP_ERRNO_H
+#ifndef _LIBCPP___CXX03_ERRNO_H
 #   error <cerrno> tried including <errno.h> but didn't find libc++'s <errno.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -45,4 +45,4 @@ Macros:
 // Based on the post commit feedback the macro are no longer deprecated.
 // Instead libc++ leaves the deprecation to the provider of errno.h.
 
-#endif // _LIBCPP_CERRNO
+#endif // _LIBCPP___CXX03_CERRNO
diff --git a/libcxx/include/__cxx03/cfenv b/libcxx/include/__cxx03/cfenv
index 59c56d29a7c4a..d707f5a65b9bc 100644
--- a/libcxx/include/__cxx03/cfenv
+++ b/libcxx/include/__cxx03/cfenv
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CFENV
-#define _LIBCPP_CFENV
+#ifndef _LIBCPP___CXX03_CFENV
+#define _LIBCPP___CXX03_CFENV
 
 /*
     cfenv synopsis
@@ -56,7 +56,7 @@ int feupdateenv(const fenv_t* envp);
 
 #include <__cxx03/fenv.h>
 
-#ifndef _LIBCPP_FENV_H
+#ifndef _LIBCPP___CXX03_FENV_H
 #   error <cfenv> tried including <fenv.h> but didn't find libc++'s <fenv.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -87,4 +87,4 @@ using ::feupdateenv _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CFENV
+#endif // _LIBCPP___CXX03_CFENV
diff --git a/libcxx/include/__cxx03/cfloat b/libcxx/include/__cxx03/cfloat
index 5f65b014ca121..5d10db6a454a6 100644
--- a/libcxx/include/__cxx03/cfloat
+++ b/libcxx/include/__cxx03/cfloat
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CFLOAT
-#define _LIBCPP_CFLOAT
+#ifndef _LIBCPP___CXX03_CFLOAT
+#define _LIBCPP___CXX03_CFLOAT
 
 /*
     cfloat synopsis
@@ -73,7 +73,7 @@ Macros:
 
 #include <__cxx03/float.h>
 
-#ifndef _LIBCPP_FLOAT_H
+#ifndef _LIBCPP___CXX03_FLOAT_H
 #   error <cfloat> tried including <float.h> but didn't find libc++'s <float.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -85,4 +85,4 @@ Macros:
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CFLOAT
+#endif // _LIBCPP___CXX03_CFLOAT
diff --git a/libcxx/include/__cxx03/charconv b/libcxx/include/__cxx03/charconv
index 0135a74fba8a5..352189aa34087 100644
--- a/libcxx/include/__cxx03/charconv
+++ b/libcxx/include/__cxx03/charconv
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CHARCONV
-#define _LIBCPP_CHARCONV
+#ifndef _LIBCPP___CXX03_CHARCONV
+#define _LIBCPP___CXX03_CHARCONV
 
 /*
     charconv synopsis
@@ -112,4 +112,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_CHARCONV
+#endif // _LIBCPP___CXX03_CHARCONV
diff --git a/libcxx/include/__cxx03/chrono b/libcxx/include/__cxx03/chrono
index f39abf1897df9..167e85203ced6 100644
--- a/libcxx/include/__cxx03/chrono
+++ b/libcxx/include/__cxx03/chrono
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CHRONO
-#define _LIBCPP_CHRONO
+#ifndef _LIBCPP___CXX03_CHRONO
+#define _LIBCPP___CXX03_CHRONO
 
 // clang-format off
 
@@ -1019,4 +1019,4 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 #  endif
 #endif
 
-#endif // _LIBCPP_CHRONO
+#endif // _LIBCPP___CXX03_CHRONO
diff --git a/libcxx/include/__cxx03/cinttypes b/libcxx/include/__cxx03/cinttypes
index 146f5b8fce0c9..a14c1a57e66f9 100644
--- a/libcxx/include/__cxx03/cinttypes
+++ b/libcxx/include/__cxx03/cinttypes
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CINTTYPES
-#define _LIBCPP_CINTTYPES
+#ifndef _LIBCPP___CXX03_CINTTYPES
+#define _LIBCPP___CXX03_CINTTYPES
 
 /*
     cinttypes synopsis
@@ -243,7 +243,7 @@ uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int
 
 #include <__cxx03/inttypes.h>
 
-#ifndef _LIBCPP_INTTYPES_H
+#ifndef _LIBCPP___CXX03_INTTYPES_H
 #   error <cinttypes> tried including <inttypes.h> but didn't find libc++'s <inttypes.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -267,4 +267,4 @@ using ::wcstoumax _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CINTTYPES
+#endif // _LIBCPP___CXX03_CINTTYPES
diff --git a/libcxx/include/__cxx03/ciso646 b/libcxx/include/__cxx03/ciso646
index 7109a31b52f24..5323d0b3393b2 100644
--- a/libcxx/include/__cxx03/ciso646
+++ b/libcxx/include/__cxx03/ciso646
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CISO646
-#define _LIBCPP_CISO646
+#ifndef _LIBCPP___CXX03_CISO646
+#define _LIBCPP___CXX03_CISO646
 
 /*
     ciso646 synopsis
@@ -21,4 +21,4 @@
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CISO646
+#endif // _LIBCPP___CXX03_CISO646
diff --git a/libcxx/include/__cxx03/climits b/libcxx/include/__cxx03/climits
index aff33b687bdef..37d8c41a0c6de 100644
--- a/libcxx/include/__cxx03/climits
+++ b/libcxx/include/__cxx03/climits
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CLIMITS
-#define _LIBCPP_CLIMITS
+#ifndef _LIBCPP___CXX03_CLIMITS
+#define _LIBCPP___CXX03_CLIMITS
 
 /*
     climits synopsis
@@ -45,4 +45,4 @@ Macros:
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CLIMITS
+#endif // _LIBCPP___CXX03_CLIMITS
diff --git a/libcxx/include/__cxx03/clocale b/libcxx/include/__cxx03/clocale
index d8fc91465ef74..497aed2486918 100644
--- a/libcxx/include/__cxx03/clocale
+++ b/libcxx/include/__cxx03/clocale
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CLOCALE
-#define _LIBCPP_CLOCALE
+#ifndef _LIBCPP___CXX03_CLOCALE
+#define _LIBCPP___CXX03_CLOCALE
 
 /*
     clocale synopsis
@@ -38,7 +38,7 @@ lconv* localeconv();
 
 #include <__cxx03/locale.h>
 
-#ifndef _LIBCPP_LOCALE_H
+#ifndef _LIBCPP___CXX03_LOCALE_H
 #   error <clocale> tried including <locale.h> but didn't find libc++'s <locale.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -58,4 +58,4 @@ using ::localeconv _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CLOCALE
+#endif // _LIBCPP___CXX03_CLOCALE
diff --git a/libcxx/include/__cxx03/cmath b/libcxx/include/__cxx03/cmath
index 659b5c965d6e9..6aa9ad9c7c4d5 100644
--- a/libcxx/include/__cxx03/cmath
+++ b/libcxx/include/__cxx03/cmath
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CMATH
-#define _LIBCPP_CMATH
+#ifndef _LIBCPP___CXX03_CMATH
+#define _LIBCPP___CXX03_CMATH
 
 /*
     cmath synopsis
@@ -327,7 +327,7 @@ constexpr long double lerp(long double a, long double b, long double t) noexcept
 #include <__cxx03/__math/special_functions.h>
 #include <__cxx03/math.h>
 
-#ifndef _LIBCPP_MATH_H
+#ifndef _LIBCPP___CXX03_MATH_H
 #   error <cmath> tried including <math.h> but didn't find libc++'s <math.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -643,4 +643,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_CMATH
+#endif // _LIBCPP___CXX03_CMATH
diff --git a/libcxx/include/__cxx03/codecvt b/libcxx/include/__cxx03/codecvt
index 827822bf12466..22a4de3790d18 100644
--- a/libcxx/include/__cxx03/codecvt
+++ b/libcxx/include/__cxx03/codecvt
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CODECVT
-#define _LIBCPP_CODECVT
+#ifndef _LIBCPP___CXX03_CODECVT
+#define _LIBCPP___CXX03_CODECVT
 
 /*
     codecvt synopsis
@@ -594,4 +594,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_CODECVT
+#endif // _LIBCPP___CXX03_CODECVT
diff --git a/libcxx/include/__cxx03/compare b/libcxx/include/__cxx03/compare
index 03b87ea1d1071..1199cc45c622b 100644
--- a/libcxx/include/__cxx03/compare
+++ b/libcxx/include/__cxx03/compare
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_COMPARE
-#define _LIBCPP_COMPARE
+#ifndef _LIBCPP___CXX03_COMPARE
+#define _LIBCPP___CXX03_COMPARE
 
 /*
     compare synopsis
@@ -175,4 +175,4 @@ namespace std {
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_COMPARE
+#endif // _LIBCPP___CXX03_COMPARE
diff --git a/libcxx/include/__cxx03/complex b/libcxx/include/__cxx03/complex
index 0bad3de200084..c68bce75bb9be 100644
--- a/libcxx/include/__cxx03/complex
+++ b/libcxx/include/__cxx03/complex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_COMPLEX
-#define _LIBCPP_COMPLEX
+#ifndef _LIBCPP___CXX03_COMPLEX
+#define _LIBCPP___CXX03_COMPLEX
 
 /*
     complex synopsis
@@ -1477,4 +1477,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_COMPLEX
+#endif // _LIBCPP___CXX03_COMPLEX
diff --git a/libcxx/include/__cxx03/complex.h b/libcxx/include/__cxx03/complex.h
index a20ab88802060..373017ff3f440 100644
--- a/libcxx/include/__cxx03/complex.h
+++ b/libcxx/include/__cxx03/complex.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_COMPLEX_H
-#define _LIBCPP_COMPLEX_H
+#ifndef _LIBCPP___CXX03_COMPLEX_H
+#define _LIBCPP___CXX03_COMPLEX_H
 
 /*
     complex.h synopsis
@@ -29,4 +29,4 @@
 #  include_next <complex.h>
 #endif
 
-#endif // _LIBCPP_COMPLEX_H
+#endif // _LIBCPP___CXX03_COMPLEX_H
diff --git a/libcxx/include/__cxx03/concepts b/libcxx/include/__cxx03/concepts
index aa6e62e4d39c8..4e129022e55a0 100644
--- a/libcxx/include/__cxx03/concepts
+++ b/libcxx/include/__cxx03/concepts
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CONCEPTS
-#define _LIBCPP_CONCEPTS
+#ifndef _LIBCPP___CXX03_CONCEPTS
+#define _LIBCPP___CXX03_CONCEPTS
 
 /*
     concepts synopsis
@@ -170,4 +170,4 @@ namespace std {
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CONCEPTS
+#endif // _LIBCPP___CXX03_CONCEPTS
diff --git a/libcxx/include/__cxx03/condition_variable b/libcxx/include/__cxx03/condition_variable
index db862c44f9d3f..5ad540c4626c0 100644
--- a/libcxx/include/__cxx03/condition_variable
+++ b/libcxx/include/__cxx03/condition_variable
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CONDITION_VARIABLE
-#define _LIBCPP_CONDITION_VARIABLE
+#ifndef _LIBCPP___CXX03_CONDITION_VARIABLE
+#define _LIBCPP___CXX03_CONDITION_VARIABLE
 
 /*
     condition_variable synopsis
@@ -366,4 +366,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_CONDITION_VARIABLE
+#endif // _LIBCPP___CXX03_CONDITION_VARIABLE
diff --git a/libcxx/include/__cxx03/coroutine b/libcxx/include/__cxx03/coroutine
index 2a8a06e63cc28..6f2af35cc4d76 100644
--- a/libcxx/include/__cxx03/coroutine
+++ b/libcxx/include/__cxx03/coroutine
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_COROUTINE
-#define _LIBCPP_COROUTINE
+#ifndef _LIBCPP___CXX03_COROUTINE
+#define _LIBCPP___CXX03_COROUTINE
 
 /**
     coroutine synopsis
@@ -64,4 +64,4 @@ struct suspend_always;
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_COROUTINE
+#endif // _LIBCPP___CXX03_COROUTINE
diff --git a/libcxx/include/__cxx03/csetjmp b/libcxx/include/__cxx03/csetjmp
index be9391506d0f4..5eeda2dda4ba6 100644
--- a/libcxx/include/__cxx03/csetjmp
+++ b/libcxx/include/__cxx03/csetjmp
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSETJMP
-#define _LIBCPP_CSETJMP
+#ifndef _LIBCPP___CXX03_CSETJMP
+#define _LIBCPP___CXX03_CSETJMP
 
 /*
     csetjmp synopsis
@@ -35,7 +35,7 @@ void longjmp(jmp_buf env, int val);
 // <setjmp.h> is not provided by libc++
 #if __has_include(<setjmp.h>)
 #  include <setjmp.h>
-#  ifdef _LIBCPP_SETJMP_H
+#  ifdef _LIBCPP___CXX03_SETJMP_H
 #    error "If libc++ starts defining <setjmp.h>, the __has_include check should move to libc++'s <setjmp.h>"
 #  endif
 #endif
@@ -51,4 +51,4 @@ using ::longjmp _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSETJMP
+#endif // _LIBCPP___CXX03_CSETJMP
diff --git a/libcxx/include/__cxx03/csignal b/libcxx/include/__cxx03/csignal
index f8f511fbca432..945278d6f2dfc 100644
--- a/libcxx/include/__cxx03/csignal
+++ b/libcxx/include/__cxx03/csignal
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSIGNAL
-#define _LIBCPP_CSIGNAL
+#ifndef _LIBCPP___CXX03_CSIGNAL
+#define _LIBCPP___CXX03_CSIGNAL
 
 /*
     csignal synopsis
@@ -44,7 +44,7 @@ int raise(int sig);
 // <signal.h> is not provided by libc++
 #if __has_include(<signal.h>)
 #  include <signal.h>
-#  ifdef _LIBCPP_SIGNAL_H
+#  ifdef _LIBCPP___CXX03_SIGNAL_H
 #    error "If libc++ starts defining <signal.h>, the __has_include check should move to libc++'s <signal.h>"
 #  endif
 #endif
@@ -61,4 +61,4 @@ using ::raise _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSIGNAL
+#endif // _LIBCPP___CXX03_CSIGNAL
diff --git a/libcxx/include/__cxx03/cstdarg b/libcxx/include/__cxx03/cstdarg
index ae04819a679ab..91a452a154f66 100644
--- a/libcxx/include/__cxx03/cstdarg
+++ b/libcxx/include/__cxx03/cstdarg
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDARG
-#define _LIBCPP_CSTDARG
+#ifndef _LIBCPP___CXX03_CSTDARG
+#define _LIBCPP___CXX03_CSTDARG
 
 /*
     cstdarg synopsis
@@ -36,7 +36,7 @@ Types:
 // <stdarg.h> is not provided by libc++
 #if __has_include(<stdarg.h>)
 #  include <stdarg.h>
-#  ifdef _LIBCPP_STDARG_H
+#  ifdef _LIBCPP___CXX03_STDARG_H
 #    error "If libc++ starts defining <stdarg.h>, the __has_include check should move to libc++'s <stdarg.h>"
 #  endif
 #endif
@@ -51,4 +51,4 @@ using ::va_list _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSTDARG
+#endif // _LIBCPP___CXX03_CSTDARG
diff --git a/libcxx/include/__cxx03/cstdbool b/libcxx/include/__cxx03/cstdbool
index 7449c40780103..16053e239c935 100644
--- a/libcxx/include/__cxx03/cstdbool
+++ b/libcxx/include/__cxx03/cstdbool
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDBOOL
-#define _LIBCPP_CSTDBOOL
+#ifndef _LIBCPP___CXX03_CSTDBOOL
+#define _LIBCPP___CXX03_CSTDBOOL
 
 /*
     cstdbool synopsis
@@ -28,4 +28,4 @@ Macros:
 #undef __bool_true_false_are_defined
 #define __bool_true_false_are_defined 1
 
-#endif // _LIBCPP_CSTDBOOL
+#endif // _LIBCPP___CXX03_CSTDBOOL
diff --git a/libcxx/include/__cxx03/cstddef b/libcxx/include/__cxx03/cstddef
index 284f2fb465617..38f89a77d1ce7 100644
--- a/libcxx/include/__cxx03/cstddef
+++ b/libcxx/include/__cxx03/cstddef
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDDEF
-#define _LIBCPP_CSTDDEF
+#ifndef _LIBCPP___CXX03_CSTDDEF
+#define _LIBCPP___CXX03_CSTDDEF
 
 /*
     cstddef synopsis
@@ -41,7 +41,7 @@ Types:
 
 #include <__cxx03/stddef.h>
 
-#ifndef _LIBCPP_STDDEF_H
+#ifndef _LIBCPP___CXX03_STDDEF_H
 #   error <cstddef> tried including <stddef.h> but didn't find libc++'s <stddef.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -130,4 +130,4 @@ template <class _Integer, __enable_if_t<is_integral<_Integer>::value, int> = 0>
 
 #endif
 
-#endif // _LIBCPP_CSTDDEF
+#endif // _LIBCPP___CXX03_CSTDDEF
diff --git a/libcxx/include/__cxx03/cstdint b/libcxx/include/__cxx03/cstdint
index 8dd7e8510130d..876abb7480bc1 100644
--- a/libcxx/include/__cxx03/cstdint
+++ b/libcxx/include/__cxx03/cstdint
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDINT
-#define _LIBCPP_CSTDINT
+#ifndef _LIBCPP___CXX03_CSTDINT
+#define _LIBCPP___CXX03_CSTDINT
 
 /*
     cstdint synopsis
@@ -144,7 +144,7 @@ Types:
 
 #include <__cxx03/stdint.h>
 
-#ifndef _LIBCPP_STDINT_H
+#ifndef _LIBCPP___CXX03_STDINT_H
 #   error <cstdint> tried including <stdint.h> but didn't find libc++'s <stdint.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -196,4 +196,4 @@ using ::uintmax_t _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSTDINT
+#endif // _LIBCPP___CXX03_CSTDINT
diff --git a/libcxx/include/__cxx03/cstdio b/libcxx/include/__cxx03/cstdio
index 3085e6d3733ac..801a58625c4ca 100644
--- a/libcxx/include/__cxx03/cstdio
+++ b/libcxx/include/__cxx03/cstdio
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDIO
-#define _LIBCPP_CSTDIO
+#ifndef _LIBCPP___CXX03_CSTDIO
+#define _LIBCPP___CXX03_CSTDIO
 
 /*
     cstdio synopsis
@@ -99,7 +99,7 @@ void perror(const char* s);
 
 #include <__cxx03/stdio.h>
 
-#ifndef _LIBCPP_STDIO_H
+#ifndef _LIBCPP___CXX03_STDIO_H
 #   error <cstdio> tried including <stdio.h> but didn't find libc++'s <stdio.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -171,4 +171,4 @@ using ::vprintf _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSTDIO
+#endif // _LIBCPP___CXX03_CSTDIO
diff --git a/libcxx/include/__cxx03/cstdlib b/libcxx/include/__cxx03/cstdlib
index 078db2b1a9115..ec4757dc3d35d 100644
--- a/libcxx/include/__cxx03/cstdlib
+++ b/libcxx/include/__cxx03/cstdlib
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTDLIB
-#define _LIBCPP_CSTDLIB
+#ifndef _LIBCPP___CXX03_CSTDLIB
+#define _LIBCPP___CXX03_CSTDLIB
 
 /*
     cstdlib synopsis
@@ -85,7 +85,7 @@ void *aligned_alloc(size_t alignment, size_t size);                       // C11
 
 #include <__cxx03/stdlib.h>
 
-#ifndef _LIBCPP_STDLIB_H
+#ifndef _LIBCPP___CXX03_STDLIB_H
 #   error <cstdlib> tried including <stdlib.h> but didn't find libc++'s <stdlib.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -151,4 +151,4 @@ using ::aligned_alloc _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSTDLIB
+#endif // _LIBCPP___CXX03_CSTDLIB
diff --git a/libcxx/include/__cxx03/cstring b/libcxx/include/__cxx03/cstring
index 693777c0f0bdf..4d57420f54f54 100644
--- a/libcxx/include/__cxx03/cstring
+++ b/libcxx/include/__cxx03/cstring
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CSTRING
-#define _LIBCPP_CSTRING
+#ifndef _LIBCPP___CXX03_CSTRING
+#define _LIBCPP___CXX03_CSTRING
 
 /*
     cstring synopsis
@@ -61,7 +61,7 @@ size_t strlen(const char* s);
 
 #include <__cxx03/string.h>
 
-#ifndef _LIBCPP_STRING_H
+#ifndef _LIBCPP___CXX03_STRING_H
 #   error <cstring> tried including <string.h> but didn't find libc++'s <string.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -101,4 +101,4 @@ using ::strlen _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CSTRING
+#endif // _LIBCPP___CXX03_CSTRING
diff --git a/libcxx/include/__cxx03/ctgmath b/libcxx/include/__cxx03/ctgmath
index 5da5f6dfb66d6..e3fa00fd68ab2 100644
--- a/libcxx/include/__cxx03/ctgmath
+++ b/libcxx/include/__cxx03/ctgmath
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CTGMATH
-#define _LIBCPP_CTGMATH
+#ifndef _LIBCPP___CXX03_CTGMATH
+#define _LIBCPP___CXX03_CTGMATH
 
 /*
     ctgmath synopsis
@@ -25,4 +25,4 @@
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_CTGMATH
+#endif // _LIBCPP___CXX03_CTGMATH
diff --git a/libcxx/include/__cxx03/ctime b/libcxx/include/__cxx03/ctime
index 84080a9c5e953..07432e6e9a43d 100644
--- a/libcxx/include/__cxx03/ctime
+++ b/libcxx/include/__cxx03/ctime
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CTIME
-#define _LIBCPP_CTIME
+#ifndef _LIBCPP___CXX03_CTIME
+#define _LIBCPP___CXX03_CTIME
 
 /*
     ctime synopsis
@@ -50,7 +50,7 @@ int timespec_get( struct timespec *ts, int base); // C++17
 // <time.h> is not provided by libc++
 #if __has_include(<time.h>)
 #  include <time.h>
-#  ifdef _LIBCPP_TIME_H
+#  ifdef _LIBCPP___CXX03_TIME_H
 #    error "If libc++ starts defining <time.h>, the __has_include check should move to libc++'s <time.h>"
 #  endif
 #endif
@@ -83,4 +83,4 @@ using ::timespec_get _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CTIME
+#endif // _LIBCPP___CXX03_CTIME
diff --git a/libcxx/include/__cxx03/ctype.h b/libcxx/include/__cxx03/ctype.h
index e169040376d4d..cd4097ab012ff 100644
--- a/libcxx/include/__cxx03/ctype.h
+++ b/libcxx/include/__cxx03/ctype.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CTYPE_H
-#define _LIBCPP_CTYPE_H
+#ifndef _LIBCPP___CXX03_CTYPE_H
+#define _LIBCPP___CXX03_CTYPE_H
 
 /*
     ctype.h synopsis
@@ -58,4 +58,4 @@ int toupper(int c);
 
 #endif
 
-#endif // _LIBCPP_CTYPE_H
+#endif // _LIBCPP___CXX03_CTYPE_H
diff --git a/libcxx/include/__cxx03/cuchar b/libcxx/include/__cxx03/cuchar
index 1029f7376756b..c9da0e7e55f0b 100644
--- a/libcxx/include/__cxx03/cuchar
+++ b/libcxx/include/__cxx03/cuchar
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CUCHAR
-#define _LIBCPP_CUCHAR
+#ifndef _LIBCPP___CXX03_CUCHAR
+#define _LIBCPP___CXX03_CUCHAR
 
 /*
     cuchar synopsis // since C++11
@@ -40,7 +40,7 @@ size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
 
 #include <__cxx03/uchar.h>
 
-#ifndef _LIBCPP_UCHAR_H
+#ifndef _LIBCPP___CXX03_UCHAR_H
 #   error <cuchar> tried including <uchar.h> but didn't find libc++'s <uchar.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -72,4 +72,4 @@ using ::c32rtomb _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CUCHAR
+#endif // _LIBCPP___CXX03_CUCHAR
diff --git a/libcxx/include/__cxx03/cwchar b/libcxx/include/__cxx03/cwchar
index 50fa5607fc4a1..2da11571df1f1 100644
--- a/libcxx/include/__cxx03/cwchar
+++ b/libcxx/include/__cxx03/cwchar
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CWCHAR
-#define _LIBCPP_CWCHAR
+#ifndef _LIBCPP___CXX03_CWCHAR
+#define _LIBCPP___CXX03_CWCHAR
 
 /*
     cwchar synopsis
@@ -112,7 +112,7 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 
 #include <__cxx03/wchar.h>
 
-#ifndef _LIBCPP_WCHAR_H
+#ifndef _LIBCPP___CXX03_WCHAR_H
 #   error <cwchar> tried including <wchar.h> but didn't find libc++'s <wchar.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -258,4 +258,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/cstddef>
 #endif
 
-#endif // _LIBCPP_CWCHAR
+#endif // _LIBCPP___CXX03_CWCHAR
diff --git a/libcxx/include/__cxx03/cwctype b/libcxx/include/__cxx03/cwctype
index 791277f9c1032..5228124936c6c 100644
--- a/libcxx/include/__cxx03/cwctype
+++ b/libcxx/include/__cxx03/cwctype
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_CWCTYPE
-#define _LIBCPP_CWCTYPE
+#ifndef _LIBCPP___CXX03_CWCTYPE
+#define _LIBCPP___CXX03_CWCTYPE
 
 /*
     cwctype synopsis
@@ -54,7 +54,7 @@ wctrans_t wctrans(const char* property);
 
 #include <__cxx03/wctype.h>
 
-#ifndef _LIBCPP_WCTYPE_H
+#ifndef _LIBCPP___CXX03_WCTYPE_H
 #   error <cwctype> tried including <wctype.h> but didn't find libc++'s <wctype.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
@@ -94,4 +94,4 @@ using ::wctrans _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_CWCTYPE
+#endif // _LIBCPP___CXX03_CWCTYPE
diff --git a/libcxx/include/__cxx03/deque b/libcxx/include/__cxx03/deque
index 7525a5b59a9bd..8e40befbb71ea 100644
--- a/libcxx/include/__cxx03/deque
+++ b/libcxx/include/__cxx03/deque
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_DEQUE
-#define _LIBCPP_DEQUE
+#ifndef _LIBCPP___CXX03_DEQUE
+#define _LIBCPP___CXX03_DEQUE
 
 /*
     deque synopsis
@@ -2618,4 +2618,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_DEQUE
+#endif // _LIBCPP___CXX03_DEQUE
diff --git a/libcxx/include/__cxx03/errno.h b/libcxx/include/__cxx03/errno.h
index 0ebd03c0b6a62..440235bef8545 100644
--- a/libcxx/include/__cxx03/errno.h
+++ b/libcxx/include/__cxx03/errno.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ERRNO_H
-#define _LIBCPP_ERRNO_H
+#ifndef _LIBCPP___CXX03_ERRNO_H
+#define _LIBCPP___CXX03_ERRNO_H
 
 /*
     errno.h synopsis
@@ -396,4 +396,4 @@ static const int __elast2 = 105;
 
 #endif // __cplusplus
 
-#endif // _LIBCPP_ERRNO_H
+#endif // _LIBCPP___CXX03_ERRNO_H
diff --git a/libcxx/include/__cxx03/exception b/libcxx/include/__cxx03/exception
index a6cb38526ff36..17a8aefd29bff 100644
--- a/libcxx/include/__cxx03/exception
+++ b/libcxx/include/__cxx03/exception
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXCEPTION
-#define _LIBCPP_EXCEPTION
+#ifndef _LIBCPP___CXX03_EXCEPTION
+#define _LIBCPP___CXX03_EXCEPTION
 
 /*
     exception synopsis
@@ -93,4 +93,4 @@ template <class E> void rethrow_if_nested(const E& e);
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_EXCEPTION
+#endif // _LIBCPP___CXX03_EXCEPTION
diff --git a/libcxx/include/__cxx03/execution b/libcxx/include/__cxx03/execution
index c04c665e6908b..b056c9c226e18 100644
--- a/libcxx/include/__cxx03/execution
+++ b/libcxx/include/__cxx03/execution
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXECUTION
-#define _LIBCPP_EXECUTION
+#ifndef _LIBCPP___CXX03_EXECUTION
+#define _LIBCPP___CXX03_EXECUTION
 
 /*
 namespace std::execution {
@@ -146,4 +146,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/cstddef>
 #endif
 
-#endif // _LIBCPP_EXECUTION
+#endif // _LIBCPP___CXX03_EXECUTION
diff --git a/libcxx/include/__cxx03/expected b/libcxx/include/__cxx03/expected
index c3bd4af17414d..6c5ebf190a898 100644
--- a/libcxx/include/__cxx03/expected
+++ b/libcxx/include/__cxx03/expected
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPECTED
-#define _LIBCPP_EXPECTED
+#ifndef _LIBCPP___CXX03_EXPECTED
+#define _LIBCPP___CXX03_EXPECTED
 
 /*
   Header <expected> synopsis
@@ -59,4 +59,4 @@ namespace std {
 #  include <__cxx03/new>
 #endif
 
-#endif // _LIBCPP_EXPECTED
+#endif // _LIBCPP___CXX03_EXPECTED
diff --git a/libcxx/include/__cxx03/experimental/__config b/libcxx/include/__cxx03/experimental/__config
index 8f6065f919c7c..027522ed35771 100644
--- a/libcxx/include/__cxx03/experimental/__config
+++ b/libcxx/include/__cxx03/experimental/__config
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_CONFIG
-#define _LIBCPP_EXPERIMENTAL_CONFIG
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_CONFIG
+#define _LIBCPP___CXX03_EXPERIMENTAL_CONFIG
 
 #include <__cxx03/__config>
 
diff --git a/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h b/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
index bbe6ee04d7c5e..ea0ddd94e4687 100644
--- a/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
+++ b/libcxx/include/__cxx03/experimental/__simd/aligned_tag.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
 
 #include <__cxx03/__memory/assume_aligned.h>
 #include <__cxx03/__type_traits/remove_const.h>
@@ -72,4 +72,4 @@ inline constexpr overaligned_tag<_Np> overaligned{};
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_ALIGNED_TAG_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/declaration.h b/libcxx/include/__cxx03/experimental/__simd/declaration.h
index 7a57e3b888916..94f068ee8b2af 100644
--- a/libcxx/include/__cxx03/experimental/__simd/declaration.h
+++ b/libcxx/include/__cxx03/experimental/__simd/declaration.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_DECLARATION_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_DECLARATION_H
 
 #include <__cxx03/cstddef>
 #include <__cxx03/experimental/__config>
@@ -78,4 +78,4 @@ class simd_mask;
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_DECLARATION_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_DECLARATION_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/reference.h b/libcxx/include/__cxx03/experimental/__simd/reference.h
index b9e0289c0ee9b..b7330d4b175cb 100644
--- a/libcxx/include/__cxx03/experimental/__simd/reference.h
+++ b/libcxx/include/__cxx03/experimental/__simd/reference.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_REFERENCE_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_REFERENCE_H
 
 #include <__cxx03/__type_traits/is_assignable.h>
 #include <__cxx03/__type_traits/is_same.h>
@@ -102,4 +102,4 @@ _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_REFERENCE_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/scalar.h b/libcxx/include/__cxx03/experimental/__simd/scalar.h
index 938fee89fa47a..5f582c6dd8ea7 100644
--- a/libcxx/include/__cxx03/experimental/__simd/scalar.h
+++ b/libcxx/include/__cxx03/experimental/__simd/scalar.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SCALAR_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SCALAR_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/cstddef>
@@ -84,4 +84,4 @@ struct __mask_operations<_Tp, simd_abi::__scalar> {
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SCALAR_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/simd.h b/libcxx/include/__cxx03/experimental/__simd/simd.h
index 4e170997fbada..8f9d94d9ae535 100644
--- a/libcxx/include/__cxx03/experimental/__simd/simd.h
+++ b/libcxx/include/__cxx03/experimental/__simd/simd.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_H
 
 #include <__cxx03/__type_traits/is_same.h>
 #include <__cxx03/__type_traits/remove_cvref.h>
@@ -99,4 +99,4 @@ using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_SIMD_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/simd_mask.h b/libcxx/include/__cxx03/experimental/__simd/simd_mask.h
index fa2f134ea7a3f..34658d0e2eeae 100644
--- a/libcxx/include/__cxx03/experimental/__simd/simd_mask.h
+++ b/libcxx/include/__cxx03/experimental/__simd/simd_mask.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_MASK_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_MASK_H
 
 #include <__cxx03/__type_traits/is_same.h>
 #include <__cxx03/cstddef>
@@ -87,4 +87,4 @@ using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_SIMD_MASK_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_SIMD_MASK_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/traits.h b/libcxx/include/__cxx03/experimental/__simd/traits.h
index 3a0480b512e2e..fbaca2660e433 100644
--- a/libcxx/include/__cxx03/experimental/__simd/traits.h
+++ b/libcxx/include/__cxx03/experimental/__simd/traits.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_TRAITS_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_TRAITS_H
 
 #include <__cxx03/__bit/bit_ceil.h>
 #include <__cxx03/__type_traits/integral_constant.h>
@@ -72,4 +72,4 @@ inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_TRAITS_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/utility.h b/libcxx/include/__cxx03/experimental/__simd/utility.h
index 803378ddcdd3d..c047059abf04a 100644
--- a/libcxx/include/__cxx03/experimental/__simd/utility.h
+++ b/libcxx/include/__cxx03/experimental/__simd/utility.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_UTILITY_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_UTILITY_H
 
 #include <__cxx03/__type_traits/is_arithmetic.h>
 #include <__cxx03/__type_traits/is_const.h>
@@ -100,4 +100,4 @@ _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_UTILITY_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_UTILITY_H
diff --git a/libcxx/include/__cxx03/experimental/__simd/vec_ext.h b/libcxx/include/__cxx03/experimental/__simd/vec_ext.h
index 99917e51a2217..a77fbc31fe836 100644
--- a/libcxx/include/__cxx03/experimental/__simd/vec_ext.h
+++ b/libcxx/include/__cxx03/experimental/__simd/vec_ext.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
-#define _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL___SIMD_VEC_EXT_H
+#define _LIBCPP___CXX03_EXPERIMENTAL___SIMD_VEC_EXT_H
 
 #include <__cxx03/__assert>
 #include <__cxx03/__bit/bit_ceil.h>
@@ -116,4 +116,4 @@ struct __mask_operations<_Tp, simd_abi::__vec_ext<_Np>> {
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL)
-#endif // _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
+#endif // _LIBCPP___CXX03_EXPERIMENTAL___SIMD_VEC_EXT_H
diff --git a/libcxx/include/__cxx03/experimental/iterator b/libcxx/include/__cxx03/experimental/iterator
index d579927c548cf..b639c5900832d 100644
--- a/libcxx/include/__cxx03/experimental/iterator
+++ b/libcxx/include/__cxx03/experimental/iterator
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_ITERATOR
-#define _LIBCPP_EXPERIMENTAL_ITERATOR
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_ITERATOR
+#define _LIBCPP___CXX03_EXPERIMENTAL_ITERATOR
 
 /*
 namespace std {
@@ -124,4 +124,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_EXPERIMENTAL_ITERATOR
+#endif // _LIBCPP___CXX03_EXPERIMENTAL_ITERATOR
diff --git a/libcxx/include/__cxx03/experimental/memory b/libcxx/include/__cxx03/experimental/memory
index 6e70a6d2f87c2..37b271087275c 100644
--- a/libcxx/include/__cxx03/experimental/memory
+++ b/libcxx/include/__cxx03/experimental/memory
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_MEMORY
-#define _LIBCPP_EXPERIMENTAL_MEMORY
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_MEMORY
+#define _LIBCPP___CXX03_EXPERIMENTAL_MEMORY
 
 /*
     experimental/memory synopsis
@@ -195,4 +195,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/limits>
 #endif
 
-#endif /* _LIBCPP_EXPERIMENTAL_MEMORY */
+#endif /* _LIBCPP___CXX03_EXPERIMENTAL_MEMORY */
diff --git a/libcxx/include/__cxx03/experimental/propagate_const b/libcxx/include/__cxx03/experimental/propagate_const
index d439f7c52c0a6..7fc9d35ba8f3d 100644
--- a/libcxx/include/__cxx03/experimental/propagate_const
+++ b/libcxx/include/__cxx03/experimental/propagate_const
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
-#define _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_PROPAGATE_CONST
+#define _LIBCPP___CXX03_EXPERIMENTAL_PROPAGATE_CONST
 
 /*
     propagate_const synopsis
@@ -487,4 +487,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_EXPERIMENTAL_PROPAGATE_CONST
+#endif // _LIBCPP___CXX03_EXPERIMENTAL_PROPAGATE_CONST
diff --git a/libcxx/include/__cxx03/experimental/simd b/libcxx/include/__cxx03/experimental/simd
index 4b0461cf4ef36..553f97594c5ce 100644
--- a/libcxx/include/__cxx03/experimental/simd
+++ b/libcxx/include/__cxx03/experimental/simd
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_SIMD
-#define _LIBCPP_EXPERIMENTAL_SIMD
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_SIMD
+#define _LIBCPP___CXX03_EXPERIMENTAL_SIMD
 
 /*
     experimental/simd synopsis
@@ -85,4 +85,4 @@ inline namespace parallelism_v2 {
 #include <__cxx03/experimental/__simd/traits.h>
 #include <__cxx03/experimental/__simd/vec_ext.h>
 
-#endif /* _LIBCPP_EXPERIMENTAL_SIMD */
+#endif /* _LIBCPP___CXX03_EXPERIMENTAL_SIMD */
diff --git a/libcxx/include/__cxx03/experimental/type_traits b/libcxx/include/__cxx03/experimental/type_traits
index 47f29d83b0e6d..c7cfef14a3c06 100644
--- a/libcxx/include/__cxx03/experimental/type_traits
+++ b/libcxx/include/__cxx03/experimental/type_traits
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_TYPE_TRAITS
-#define _LIBCPP_EXPERIMENTAL_TYPE_TRAITS
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_TYPE_TRAITS
+#define _LIBCPP___CXX03_EXPERIMENTAL_TYPE_TRAITS
 
 /**
     experimental/type_traits synopsis
@@ -150,4 +150,4 @@ _LIBCPP_END_NAMESPACE_LFTS
 
 #endif /* _LIBCPP_STD_VER >= 14 */
 
-#endif /* _LIBCPP_EXPERIMENTAL_TYPE_TRAITS */
+#endif /* _LIBCPP___CXX03_EXPERIMENTAL_TYPE_TRAITS */
diff --git a/libcxx/include/__cxx03/experimental/utility b/libcxx/include/__cxx03/experimental/utility
index 73239e7a4856a..4e8156ae8728f 100644
--- a/libcxx/include/__cxx03/experimental/utility
+++ b/libcxx/include/__cxx03/experimental/utility
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXPERIMENTAL_UTILITY
-#define _LIBCPP_EXPERIMENTAL_UTILITY
+#ifndef _LIBCPP___CXX03_EXPERIMENTAL_UTILITY
+#define _LIBCPP___CXX03_EXPERIMENTAL_UTILITY
 
 /*
     experimental/utility synopsis
@@ -43,4 +43,4 @@ struct _LIBCPP_TEMPLATE_VIS erased_type {};
 
 _LIBCPP_END_NAMESPACE_LFTS
 
-#endif /* _LIBCPP_EXPERIMENTAL_UTILITY */
+#endif /* _LIBCPP___CXX03_EXPERIMENTAL_UTILITY */
diff --git a/libcxx/include/__cxx03/ext/__hash b/libcxx/include/__cxx03/ext/__hash
index 3b363257b1a0b..8f27c7ea105c5 100644
--- a/libcxx/include/__cxx03/ext/__hash
+++ b/libcxx/include/__cxx03/ext/__hash
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_EXT_HASH
-#define _LIBCPP_EXT_HASH
+#ifndef _LIBCPP___CXX03_EXT_HASH
+#define _LIBCPP___CXX03_EXT_HASH
 
 #pragma GCC system_header
 
@@ -82,4 +82,4 @@ struct _LIBCPP_TEMPLATE_VIS hash<unsigned long> : public std::__unary_function<u
 };
 } // namespace __gnu_cxx
 
-#endif // _LIBCPP_EXT_HASH
+#endif // _LIBCPP___CXX03_EXT_HASH
diff --git a/libcxx/include/__cxx03/ext/hash_map b/libcxx/include/__cxx03/ext/hash_map
index bffb68672e438..f075ed19ea40f 100644
--- a/libcxx/include/__cxx03/ext/hash_map
+++ b/libcxx/include/__cxx03/ext/hash_map
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_HASH_MAP
-#define _LIBCPP_HASH_MAP
+#ifndef _LIBCPP___CXX03_HASH_MAP
+#define _LIBCPP___CXX03_HASH_MAP
 
 /*
 
@@ -869,4 +869,4 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const hash_multimap<_Key, _Tp, _Has
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_HASH_MAP
+#endif // _LIBCPP___CXX03_HASH_MAP
diff --git a/libcxx/include/__cxx03/ext/hash_set b/libcxx/include/__cxx03/ext/hash_set
index 3886e578ec4aa..fefad7c5833b1 100644
--- a/libcxx/include/__cxx03/ext/hash_set
+++ b/libcxx/include/__cxx03/ext/hash_set
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_HASH_SET
-#define _LIBCPP_HASH_SET
+#ifndef _LIBCPP___CXX03_HASH_SET
+#define _LIBCPP___CXX03_HASH_SET
 
 /*
 
@@ -581,4 +581,4 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const hash_multiset<_Value, _Hash,
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_HASH_SET
+#endif // _LIBCPP___CXX03_HASH_SET
diff --git a/libcxx/include/__cxx03/fenv.h b/libcxx/include/__cxx03/fenv.h
index dd9f381892b54..87fe9d18b2b77 100644
--- a/libcxx/include/__cxx03/fenv.h
+++ b/libcxx/include/__cxx03/fenv.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FENV_H
-#define _LIBCPP_FENV_H
+#ifndef _LIBCPP___CXX03_FENV_H
+#define _LIBCPP___CXX03_FENV_H
 
 /*
     fenv.h synopsis
@@ -111,4 +111,4 @@ extern "C++" {
 
 #endif // defined(__cplusplus)
 
-#endif // _LIBCPP_FENV_H
+#endif // _LIBCPP___CXX03_FENV_H
diff --git a/libcxx/include/__cxx03/filesystem b/libcxx/include/__cxx03/filesystem
index 2f1393bcbcf61..c991e15b923a9 100644
--- a/libcxx/include/__cxx03/filesystem
+++ b/libcxx/include/__cxx03/filesystem
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FILESYSTEM
-#define _LIBCPP_FILESYSTEM
+#ifndef _LIBCPP___CXX03_FILESYSTEM
+#define _LIBCPP___CXX03_FILESYSTEM
 
 /*
     filesystem synopsis
@@ -574,4 +574,4 @@ inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_direct
 #  include <__cxx03/system_error>
 #endif
 
-#endif // _LIBCPP_FILESYSTEM
+#endif // _LIBCPP___CXX03_FILESYSTEM
diff --git a/libcxx/include/__cxx03/float.h b/libcxx/include/__cxx03/float.h
index 535e73e461519..38f199067f411 100644
--- a/libcxx/include/__cxx03/float.h
+++ b/libcxx/include/__cxx03/float.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FLOAT_H
-#define _LIBCPP_FLOAT_H
+#ifndef _LIBCPP___CXX03_FLOAT_H
+#define _LIBCPP___CXX03_FLOAT_H
 
 /*
     float.h synopsis
@@ -92,4 +92,4 @@
 
 #endif // __cplusplus
 
-#endif // _LIBCPP_FLOAT_H
+#endif // _LIBCPP___CXX03_FLOAT_H
diff --git a/libcxx/include/__cxx03/format b/libcxx/include/__cxx03/format
index c8adb186f1024..0183f8374efd1 100644
--- a/libcxx/include/__cxx03/format
+++ b/libcxx/include/__cxx03/format
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FORMAT
-#define _LIBCPP_FORMAT
+#ifndef _LIBCPP___CXX03_FORMAT
+#define _LIBCPP___CXX03_FORMAT
 
 /*
 
@@ -254,4 +254,4 @@ namespace std {
 #  endif
 #endif
 
-#endif // _LIBCPP_FORMAT
+#endif // _LIBCPP___CXX03_FORMAT
diff --git a/libcxx/include/__cxx03/forward_list b/libcxx/include/__cxx03/forward_list
index c83ab2582372d..ecffb9c8f3a10 100644
--- a/libcxx/include/__cxx03/forward_list
+++ b/libcxx/include/__cxx03/forward_list
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FORWARD_LIST
-#define _LIBCPP_FORWARD_LIST
+#ifndef _LIBCPP___CXX03_FORWARD_LIST
+#define _LIBCPP___CXX03_FORWARD_LIST
 
 /*
     forward_list synopsis
@@ -1571,4 +1571,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_FORWARD_LIST
+#endif // _LIBCPP___CXX03_FORWARD_LIST
diff --git a/libcxx/include/__cxx03/fstream b/libcxx/include/__cxx03/fstream
index 6fbced3caec3e..7e43a533fd794 100644
--- a/libcxx/include/__cxx03/fstream
+++ b/libcxx/include/__cxx03/fstream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FSTREAM
-#define _LIBCPP_FSTREAM
+#ifndef _LIBCPP___CXX03_FSTREAM
+#define _LIBCPP___CXX03_FSTREAM
 
 /*
     fstream synopsis
@@ -1569,4 +1569,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_FSTREAM
+#endif // _LIBCPP___CXX03_FSTREAM
diff --git a/libcxx/include/__cxx03/functional b/libcxx/include/__cxx03/functional
index 406c2ce297259..985fcfa953204 100644
--- a/libcxx/include/__cxx03/functional
+++ b/libcxx/include/__cxx03/functional
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FUNCTIONAL
-#define _LIBCPP_FUNCTIONAL
+#ifndef _LIBCPP___CXX03_FUNCTIONAL
+#define _LIBCPP___CXX03_FUNCTIONAL
 
 /*
     functional synopsis
@@ -595,4 +595,4 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_FUNCTIONAL
+#endif // _LIBCPP___CXX03_FUNCTIONAL
diff --git a/libcxx/include/__cxx03/future b/libcxx/include/__cxx03/future
index d40749d7375ea..f92bc1266939e 100644
--- a/libcxx/include/__cxx03/future
+++ b/libcxx/include/__cxx03/future
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_FUTURE
-#define _LIBCPP_FUTURE
+#ifndef _LIBCPP___CXX03_FUTURE
+#define _LIBCPP___CXX03_FUTURE
 
 /*
     future synopsis
@@ -2059,4 +2059,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/system_error>
 #endif
 
-#endif // _LIBCPP_FUTURE
+#endif // _LIBCPP___CXX03_FUTURE
diff --git a/libcxx/include/__cxx03/initializer_list b/libcxx/include/__cxx03/initializer_list
index b891c5e51b7a1..4bb2deb01ae6d 100644
--- a/libcxx/include/__cxx03/initializer_list
+++ b/libcxx/include/__cxx03/initializer_list
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_INITIALIZER_LIST
-#define _LIBCPP_INITIALIZER_LIST
+#ifndef _LIBCPP___CXX03_INITIALIZER_LIST
+#define _LIBCPP___CXX03_INITIALIZER_LIST
 
 /*
     initializer_list synopsis
@@ -95,4 +95,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end(initia
 
 } // namespace std
 
-#endif // _LIBCPP_INITIALIZER_LIST
+#endif // _LIBCPP___CXX03_INITIALIZER_LIST
diff --git a/libcxx/include/__cxx03/inttypes.h b/libcxx/include/__cxx03/inttypes.h
index 9101fab9ab080..82acd87f5e8eb 100644
--- a/libcxx/include/__cxx03/inttypes.h
+++ b/libcxx/include/__cxx03/inttypes.h
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_INTTYPES_H
+#ifndef _LIBCPP___CXX03_INTTYPES_H
 // AIX system headers need inttypes.h to be re-enterable while _STD_TYPES_T
 // is defined until an inclusion of it without _STD_TYPES_T occurs, in which
 // case the header guard macro is defined.
 #if !defined(_AIX) || !defined(_STD_TYPES_T)
-#  define _LIBCPP_INTTYPES_H
+#  define _LIBCPP___CXX03_INTTYPES_H
 #endif // _STD_TYPES_T
 
 /*
@@ -261,4 +261,4 @@ uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int
 
 #endif // __cplusplus
 
-#endif // _LIBCPP_INTTYPES_H
+#endif // _LIBCPP___CXX03_INTTYPES_H
diff --git a/libcxx/include/__cxx03/iomanip b/libcxx/include/__cxx03/iomanip
index 768bee9f22664..8f75e23f67ac7 100644
--- a/libcxx/include/__cxx03/iomanip
+++ b/libcxx/include/__cxx03/iomanip
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_IOMANIP
-#define _LIBCPP_IOMANIP
+#ifndef _LIBCPP___CXX03_IOMANIP
+#define _LIBCPP___CXX03_IOMANIP
 
 /*
     iomanip synopsis
@@ -539,4 +539,4 @@ quoted(basic_string_view<_CharT, _Traits> __sv, _CharT __delim = _CharT('"'), _C
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_IOMANIP
+#endif // _LIBCPP___CXX03_IOMANIP
diff --git a/libcxx/include/__cxx03/ios b/libcxx/include/__cxx03/ios
index a9eeefc0883d4..ace6d666d6b58 100644
--- a/libcxx/include/__cxx03/ios
+++ b/libcxx/include/__cxx03/ios
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_IOS
-#define _LIBCPP_IOS
+#ifndef _LIBCPP___CXX03_IOS
+#define _LIBCPP___CXX03_IOS
 
 /*
     ios synopsis
@@ -886,4 +886,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_IOS
+#endif // _LIBCPP___CXX03_IOS
diff --git a/libcxx/include/__cxx03/iosfwd b/libcxx/include/__cxx03/iosfwd
index 9c4c86bc2fa27..624bd157d4962 100644
--- a/libcxx/include/__cxx03/iosfwd
+++ b/libcxx/include/__cxx03/iosfwd
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_IOSFWD
-#define _LIBCPP_IOSFWD
+#ifndef _LIBCPP___CXX03_IOSFWD
+#define _LIBCPP___CXX03_IOSFWD
 
 /*
     iosfwd synopsis
@@ -183,4 +183,4 @@ public:
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_IOSFWD
+#endif // _LIBCPP___CXX03_IOSFWD
diff --git a/libcxx/include/__cxx03/iostream b/libcxx/include/__cxx03/iostream
index c4923117822c7..d302ad2dfc150 100644
--- a/libcxx/include/__cxx03/iostream
+++ b/libcxx/include/__cxx03/iostream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_IOSTREAM
-#define _LIBCPP_IOSTREAM
+#ifndef _LIBCPP___CXX03_IOSTREAM
+#define _LIBCPP___CXX03_IOSTREAM
 
 /*
     iostream synopsis
@@ -64,4 +64,4 @@ extern _LIBCPP_EXPORTED_FROM_ABI wostream wclog;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_IOSTREAM
+#endif // _LIBCPP___CXX03_IOSTREAM
diff --git a/libcxx/include/__cxx03/istream b/libcxx/include/__cxx03/istream
index c2fc00298bd58..b1f7b0888d71b 100644
--- a/libcxx/include/__cxx03/istream
+++ b/libcxx/include/__cxx03/istream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ISTREAM
-#define _LIBCPP_ISTREAM
+#ifndef _LIBCPP___CXX03_ISTREAM
+#define _LIBCPP___CXX03_ISTREAM
 
 /*
     istream synopsis
@@ -1370,4 +1370,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_ISTREAM
+#endif // _LIBCPP___CXX03_ISTREAM
diff --git a/libcxx/include/__cxx03/iterator b/libcxx/include/__cxx03/iterator
index f415443f8dc1d..5ac8a4274b3dd 100644
--- a/libcxx/include/__cxx03/iterator
+++ b/libcxx/include/__cxx03/iterator
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_ITERATOR
-#define _LIBCPP_ITERATOR
+#ifndef _LIBCPP___CXX03_ITERATOR
+#define _LIBCPP___CXX03_ITERATOR
 
 /*
     iterator synopsis
@@ -751,4 +751,4 @@ template <class E> constexpr const E* data(initializer_list<E> il) noexcept;
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_ITERATOR
+#endif // _LIBCPP___CXX03_ITERATOR
diff --git a/libcxx/include/__cxx03/latch b/libcxx/include/__cxx03/latch
index bcfb6a51243db..cca88c594039b 100644
--- a/libcxx/include/__cxx03/latch
+++ b/libcxx/include/__cxx03/latch
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_LATCH
-#define _LIBCPP_LATCH
+#ifndef _LIBCPP___CXX03_LATCH
+#define _LIBCPP___CXX03_LATCH
 
 /*
     latch synopsis
@@ -126,4 +126,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/atomic>
 #endif
 
-#endif //_LIBCPP_LATCH
+#endif //_LIBCPP___CXX03_LATCH
diff --git a/libcxx/include/__cxx03/limits b/libcxx/include/__cxx03/limits
index d028d8256f59d..bfc3cb8e1bb19 100644
--- a/libcxx/include/__cxx03/limits
+++ b/libcxx/include/__cxx03/limits
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_LIMITS
-#define _LIBCPP_LIMITS
+#ifndef _LIBCPP___CXX03_LIMITS
+#define _LIBCPP___CXX03_LIMITS
 
 /*
     limits synopsis
@@ -581,4 +581,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_LIMITS
+#endif // _LIBCPP___CXX03_LIMITS
diff --git a/libcxx/include/__cxx03/list b/libcxx/include/__cxx03/list
index 742df50767e15..647bb770646ca 100644
--- a/libcxx/include/__cxx03/list
+++ b/libcxx/include/__cxx03/list
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_LIST
-#define _LIBCPP_LIST
+#ifndef _LIBCPP___CXX03_LIST
+#define _LIBCPP___CXX03_LIST
 
 /*
     list synopsis
@@ -1739,4 +1739,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_LIST
+#endif // _LIBCPP___CXX03_LIST
diff --git a/libcxx/include/__cxx03/locale b/libcxx/include/__cxx03/locale
index 48ae270e0949b..78c0fd7763af2 100644
--- a/libcxx/include/__cxx03/locale
+++ b/libcxx/include/__cxx03/locale
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_LOCALE
-#define _LIBCPP_LOCALE
+#ifndef _LIBCPP___CXX03_LOCALE
+#define _LIBCPP___CXX03_LOCALE
 
 /*
     locale synopsis
@@ -3764,4 +3764,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_LOCALE
+#endif // _LIBCPP___CXX03_LOCALE
diff --git a/libcxx/include/__cxx03/locale.h b/libcxx/include/__cxx03/locale.h
index b33059d7965d6..6a6773674ad93 100644
--- a/libcxx/include/__cxx03/locale.h
+++ b/libcxx/include/__cxx03/locale.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_LOCALE_H
-#define _LIBCPP_LOCALE_H
+#ifndef _LIBCPP___CXX03_LOCALE_H
+#define _LIBCPP___CXX03_LOCALE_H
 
 /*
     locale.h synopsis
@@ -43,4 +43,4 @@
 #  include_next <locale.h>
 #endif
 
-#endif // _LIBCPP_LOCALE_H
+#endif // _LIBCPP___CXX03_LOCALE_H
diff --git a/libcxx/include/__cxx03/map b/libcxx/include/__cxx03/map
index 53dcbaf4eae1d..bd03d8b741a71 100644
--- a/libcxx/include/__cxx03/map
+++ b/libcxx/include/__cxx03/map
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_MAP
-#define _LIBCPP_MAP
+#ifndef _LIBCPP___CXX03_MAP
+#define _LIBCPP___CXX03_MAP
 
 /*
 
@@ -2178,4 +2178,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_MAP
+#endif // _LIBCPP___CXX03_MAP
diff --git a/libcxx/include/__cxx03/math.h b/libcxx/include/__cxx03/math.h
index 9a2509e93d01a..ca4062072e01b 100644
--- a/libcxx/include/__cxx03/math.h
+++ b/libcxx/include/__cxx03/math.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_MATH_H
-#  define _LIBCPP_MATH_H
+#ifndef _LIBCPP___CXX03_MATH_H
+#  define _LIBCPP___CXX03_MATH_H
 
 /*
     math.h synopsis
@@ -503,7 +503,7 @@ using std::__math::trunc;
 
 #  endif // __cplusplus
 
-#else // _LIBCPP_MATH_H
+#else // _LIBCPP___CXX03_MATH_H
 
 // This include lives outside the header guard in order to support an MSVC
 // extension which allows users to do:
@@ -517,4 +517,4 @@ using std::__math::trunc;
 #    include_next <math.h>
 #  endif
 
-#endif // _LIBCPP_MATH_H
+#endif // _LIBCPP___CXX03_MATH_H
diff --git a/libcxx/include/__cxx03/mdspan b/libcxx/include/__cxx03/mdspan
index 950145b34b098..c47a63794955b 100644
--- a/libcxx/include/__cxx03/mdspan
+++ b/libcxx/include/__cxx03/mdspan
@@ -405,8 +405,8 @@ namespace std {
 }
 */
 
-#ifndef _LIBCPP_MDSPAN
-#define _LIBCPP_MDSPAN
+#ifndef _LIBCPP___CXX03_MDSPAN
+#define _LIBCPP___CXX03_MDSPAN
 
 #include <__cxx03/__config>
 
@@ -435,4 +435,4 @@ namespace std {
 #  include <__cxx03/span>
 #endif
 
-#endif // _LIBCPP_MDSPAN
+#endif // _LIBCPP___CXX03_MDSPAN
diff --git a/libcxx/include/__cxx03/memory b/libcxx/include/__cxx03/memory
index 5bbcd8513e73d..ea417bb827c78 100644
--- a/libcxx/include/__cxx03/memory
+++ b/libcxx/include/__cxx03/memory
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_MEMORY
-#define _LIBCPP_MEMORY
+#ifndef _LIBCPP___CXX03_MEMORY
+#define _LIBCPP___CXX03_MEMORY
 
 // clang-format off
 
@@ -995,4 +995,4 @@ template<class Pointer = void, class Smart, class... Args>
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_MEMORY
+#endif // _LIBCPP___CXX03_MEMORY
diff --git a/libcxx/include/__cxx03/memory_resource b/libcxx/include/__cxx03/memory_resource
index fb217e40f6301..1b56153dc843b 100644
--- a/libcxx/include/__cxx03/memory_resource
+++ b/libcxx/include/__cxx03/memory_resource
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_MEMORY_RESOURCE
-#define _LIBCPP_MEMORY_RESOURCE
+#ifndef _LIBCPP___CXX03_MEMORY_RESOURCE
+#define _LIBCPP___CXX03_MEMORY_RESOURCE
 
 /**
     memory_resource synopsis
@@ -80,4 +80,4 @@ namespace std::pmr {
 #  include <__cxx03/stdexcept>
 #endif
 
-#endif /* _LIBCPP_MEMORY_RESOURCE */
+#endif /* _LIBCPP___CXX03_MEMORY_RESOURCE */
diff --git a/libcxx/include/__cxx03/mutex b/libcxx/include/__cxx03/mutex
index 550d0f887dc49..a799a0b774ad7 100644
--- a/libcxx/include/__cxx03/mutex
+++ b/libcxx/include/__cxx03/mutex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_MUTEX
-#define _LIBCPP_MUTEX
+#ifndef _LIBCPP___CXX03_MUTEX
+#define _LIBCPP___CXX03_MUTEX
 
 /*
     mutex synopsis
@@ -513,4 +513,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/typeinfo>
 #endif
 
-#endif // _LIBCPP_MUTEX
+#endif // _LIBCPP___CXX03_MUTEX
diff --git a/libcxx/include/__cxx03/new b/libcxx/include/__cxx03/new
index 7458fb0838150..9a893e727b44b 100644
--- a/libcxx/include/__cxx03/new
+++ b/libcxx/include/__cxx03/new
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_NEW
-#define _LIBCPP_NEW
+#ifndef _LIBCPP___CXX03_NEW
+#define _LIBCPP___CXX03_NEW
 
 /*
     new synopsis
@@ -359,4 +359,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_NEW
+#endif // _LIBCPP___CXX03_NEW
diff --git a/libcxx/include/__cxx03/numbers b/libcxx/include/__cxx03/numbers
index d8ca33295d551..3ef8aa2324494 100644
--- a/libcxx/include/__cxx03/numbers
+++ b/libcxx/include/__cxx03/numbers
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_NUMBERS
-#define _LIBCPP_NUMBERS
+#ifndef _LIBCPP___CXX03_NUMBERS
+#define _LIBCPP___CXX03_NUMBERS
 
 /*
     numbers synopsis
@@ -161,4 +161,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_NUMBERS
+#endif // _LIBCPP___CXX03_NUMBERS
diff --git a/libcxx/include/__cxx03/numeric b/libcxx/include/__cxx03/numeric
index 85c756b021618..992641b8c210b 100644
--- a/libcxx/include/__cxx03/numeric
+++ b/libcxx/include/__cxx03/numeric
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_NUMERIC
-#define _LIBCPP_NUMERIC
+#ifndef _LIBCPP___CXX03_NUMERIC
+#define _LIBCPP___CXX03_NUMERIC
 
 /*
     numeric synopsis
@@ -204,4 +204,4 @@ constexpr T saturate_cast(U x) noexcept;                    // freestanding, Sin
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_NUMERIC
+#endif // _LIBCPP___CXX03_NUMERIC
diff --git a/libcxx/include/__cxx03/optional b/libcxx/include/__cxx03/optional
index 32be4b0cd4db8..e2c6fd0950c34 100644
--- a/libcxx/include/__cxx03/optional
+++ b/libcxx/include/__cxx03/optional
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_OPTIONAL
-#define _LIBCPP_OPTIONAL
+#ifndef _LIBCPP___CXX03_OPTIONAL
+#define _LIBCPP___CXX03_OPTIONAL
 
 /*
     optional synopsis
@@ -1301,4 +1301,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/variant>
 #endif
 
-#endif // _LIBCPP_OPTIONAL
+#endif // _LIBCPP___CXX03_OPTIONAL
diff --git a/libcxx/include/__cxx03/ostream b/libcxx/include/__cxx03/ostream
index 708434803542d..4400fa60ba7e7 100644
--- a/libcxx/include/__cxx03/ostream
+++ b/libcxx/include/__cxx03/ostream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_OSTREAM
-#define _LIBCPP_OSTREAM
+#ifndef _LIBCPP___CXX03_OSTREAM
+#define _LIBCPP___CXX03_OSTREAM
 
 /*
     ostream synopsis
@@ -199,4 +199,4 @@ void vprint_nonunicode(ostream& os, string_view fmt, format_args args);
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_OSTREAM
+#endif // _LIBCPP___CXX03_OSTREAM
diff --git a/libcxx/include/__cxx03/print b/libcxx/include/__cxx03/print
index bb7731abe4936..02f1a6127a21a 100644
--- a/libcxx/include/__cxx03/print
+++ b/libcxx/include/__cxx03/print
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_PRINT
-#define _LIBCPP_PRINT
+#ifndef _LIBCPP___CXX03_PRINT
+#define _LIBCPP___CXX03_PRINT
 
 /*
 namespace std {
@@ -399,4 +399,4 @@ _LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(string_view __fmt, format_ar
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_PRINT
+#endif // _LIBCPP___CXX03_PRINT
diff --git a/libcxx/include/__cxx03/queue b/libcxx/include/__cxx03/queue
index 7031e4264f40a..5f0189e61ee45 100644
--- a/libcxx/include/__cxx03/queue
+++ b/libcxx/include/__cxx03/queue
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_QUEUE
-#define _LIBCPP_QUEUE
+#ifndef _LIBCPP___CXX03_QUEUE
+#define _LIBCPP___CXX03_QUEUE
 
 /*
     queue synopsis
@@ -953,4 +953,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_QUEUE
+#endif // _LIBCPP___CXX03_QUEUE
diff --git a/libcxx/include/__cxx03/random b/libcxx/include/__cxx03/random
index 6c6f730fbac7e..71c59edecfec0 100644
--- a/libcxx/include/__cxx03/random
+++ b/libcxx/include/__cxx03/random
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_RANDOM
-#define _LIBCPP_RANDOM
+#ifndef _LIBCPP___CXX03_RANDOM
+#define _LIBCPP___CXX03_RANDOM
 
 /*
     random synopsis
@@ -1739,4 +1739,4 @@ class piecewise_linear_distribution
 #  include <__cxx03/vector>
 #endif
 
-#endif // _LIBCPP_RANDOM
+#endif // _LIBCPP___CXX03_RANDOM
diff --git a/libcxx/include/__cxx03/ranges b/libcxx/include/__cxx03/ranges
index 3c3021e30d4a5..a762a0c2ae758 100644
--- a/libcxx/include/__cxx03/ranges
+++ b/libcxx/include/__cxx03/ranges
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_RANGES
-#define _LIBCPP_RANGES
+#ifndef _LIBCPP___CXX03_RANGES
+#define _LIBCPP___CXX03_RANGES
 
 /*
 
@@ -460,4 +460,4 @@ namespace std {
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_RANGES
+#endif // _LIBCPP___CXX03_RANGES
diff --git a/libcxx/include/__cxx03/ratio b/libcxx/include/__cxx03/ratio
index e167b36b7dd17..57b5dcbf2ddce 100644
--- a/libcxx/include/__cxx03/ratio
+++ b/libcxx/include/__cxx03/ratio
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_RATIO
-#define _LIBCPP_RATIO
+#ifndef _LIBCPP___CXX03_RATIO
+#define _LIBCPP___CXX03_RATIO
 
 /*
     ratio synopsis
@@ -514,4 +514,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_RATIO
+#endif // _LIBCPP___CXX03_RATIO
diff --git a/libcxx/include/__cxx03/regex b/libcxx/include/__cxx03/regex
index 9d95db45d419d..c05c8768a89a1 100644
--- a/libcxx/include/__cxx03/regex
+++ b/libcxx/include/__cxx03/regex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_REGEX
-#define _LIBCPP_REGEX
+#ifndef _LIBCPP___CXX03_REGEX
+#define _LIBCPP___CXX03_REGEX
 
 /*
     regex synopsis
@@ -5833,4 +5833,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_REGEX
+#endif // _LIBCPP___CXX03_REGEX
diff --git a/libcxx/include/__cxx03/scoped_allocator b/libcxx/include/__cxx03/scoped_allocator
index 6c62741ca2c8d..faaca57302a60 100644
--- a/libcxx/include/__cxx03/scoped_allocator
+++ b/libcxx/include/__cxx03/scoped_allocator
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SCOPED_ALLOCATOR
-#define _LIBCPP_SCOPED_ALLOCATOR
+#ifndef _LIBCPP___CXX03_SCOPED_ALLOCATOR
+#define _LIBCPP___CXX03_SCOPED_ALLOCATOR
 
 /*
     scoped_allocator synopsis
@@ -570,4 +570,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/variant>
 #endif
 
-#endif // _LIBCPP_SCOPED_ALLOCATOR
+#endif // _LIBCPP___CXX03_SCOPED_ALLOCATOR
diff --git a/libcxx/include/__cxx03/semaphore b/libcxx/include/__cxx03/semaphore
index e8f9f072bb662..8e07f1a910d0c 100644
--- a/libcxx/include/__cxx03/semaphore
+++ b/libcxx/include/__cxx03/semaphore
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SEMAPHORE
-#define _LIBCPP_SEMAPHORE
+#ifndef _LIBCPP___CXX03_SEMAPHORE
+#define _LIBCPP___CXX03_SEMAPHORE
 
 /*
     semaphore synopsis
@@ -185,4 +185,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/atomic>
 #endif
 
-#endif //_LIBCPP_SEMAPHORE
+#endif //_LIBCPP___CXX03_SEMAPHORE
diff --git a/libcxx/include/__cxx03/set b/libcxx/include/__cxx03/set
index b7312fff21395..2b14ef8b39a9f 100644
--- a/libcxx/include/__cxx03/set
+++ b/libcxx/include/__cxx03/set
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SET
-#define _LIBCPP_SET
+#ifndef _LIBCPP___CXX03_SET
+#define _LIBCPP___CXX03_SET
 
 /*
 
@@ -1497,4 +1497,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_SET
+#endif // _LIBCPP___CXX03_SET
diff --git a/libcxx/include/__cxx03/shared_mutex b/libcxx/include/__cxx03/shared_mutex
index 99f77a8921005..7acecdafc36a9 100644
--- a/libcxx/include/__cxx03/shared_mutex
+++ b/libcxx/include/__cxx03/shared_mutex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SHARED_MUTEX
-#define _LIBCPP_SHARED_MUTEX
+#ifndef _LIBCPP___CXX03_SHARED_MUTEX
+#define _LIBCPP___CXX03_SHARED_MUTEX
 
 /*
     shared_mutex synopsis
@@ -461,4 +461,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/system_error>
 #endif
 
-#endif // _LIBCPP_SHARED_MUTEX
+#endif // _LIBCPP___CXX03_SHARED_MUTEX
diff --git a/libcxx/include/__cxx03/source_location b/libcxx/include/__cxx03/source_location
index 509a547808821..3cd36babe3bf3 100644
--- a/libcxx/include/__cxx03/source_location
+++ b/libcxx/include/__cxx03/source_location
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SOURCE_LOCATION
-#define _LIBCPP_SOURCE_LOCATION
+#ifndef _LIBCPP___CXX03_SOURCE_LOCATION
+#define _LIBCPP___CXX03_SOURCE_LOCATION
 
 /* source_location synopsis
 
@@ -82,4 +82,4 @@ public:
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_SOURCE_LOCATION
+#endif // _LIBCPP___CXX03_SOURCE_LOCATION
diff --git a/libcxx/include/__cxx03/span b/libcxx/include/__cxx03/span
index d09f96468064b..7d1ef460e5994 100644
--- a/libcxx/include/__cxx03/span
+++ b/libcxx/include/__cxx03/span
@@ -7,8 +7,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SPAN
-#define _LIBCPP_SPAN
+#ifndef _LIBCPP___CXX03_SPAN
+#define _LIBCPP___CXX03_SPAN
 
 /*
     span synopsis
@@ -633,4 +633,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_SPAN
+#endif // _LIBCPP___CXX03_SPAN
diff --git a/libcxx/include/__cxx03/sstream b/libcxx/include/__cxx03/sstream
index 2667ae9cb18e1..744c6365cebf0 100644
--- a/libcxx/include/__cxx03/sstream
+++ b/libcxx/include/__cxx03/sstream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SSTREAM
-#define _LIBCPP_SSTREAM
+#ifndef _LIBCPP___CXX03_SSTREAM
+#define _LIBCPP___CXX03_SSTREAM
 
 // clang-format off
 
@@ -1273,4 +1273,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_SSTREAM
+#endif // _LIBCPP___CXX03_SSTREAM
diff --git a/libcxx/include/__cxx03/stack b/libcxx/include/__cxx03/stack
index a868878653598..e4988362b8010 100644
--- a/libcxx/include/__cxx03/stack
+++ b/libcxx/include/__cxx03/stack
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STACK
-#define _LIBCPP_STACK
+#ifndef _LIBCPP___CXX03_STACK
+#define _LIBCPP___CXX03_STACK
 
 /*
     stack synopsis
@@ -376,4 +376,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_STACK
+#endif // _LIBCPP___CXX03_STACK
diff --git a/libcxx/include/__cxx03/stdatomic.h b/libcxx/include/__cxx03/stdatomic.h
index 15f68b93a414f..44e693aefb4ab 100644
--- a/libcxx/include/__cxx03/stdatomic.h
+++ b/libcxx/include/__cxx03/stdatomic.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STDATOMIC_H
-#define _LIBCPP_STDATOMIC_H
+#ifndef _LIBCPP___CXX03_STDATOMIC_H
+#define _LIBCPP___CXX03_STDATOMIC_H
 
 /*
     stdatomic.h synopsis
@@ -232,4 +232,4 @@ using std::atomic_thread_fence _LIBCPP_USING_IF_EXISTS;
 
 #endif // defined(__cplusplus) && _LIBCPP_STD_VER >= 23
 
-#endif // _LIBCPP_STDATOMIC_H
+#endif // _LIBCPP___CXX03_STDATOMIC_H
diff --git a/libcxx/include/__cxx03/stdbool.h b/libcxx/include/__cxx03/stdbool.h
index 0444340e24412..e6c2eae2cfd12 100644
--- a/libcxx/include/__cxx03/stdbool.h
+++ b/libcxx/include/__cxx03/stdbool.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STDBOOL_H
-#define _LIBCPP_STDBOOL_H
+#ifndef _LIBCPP___CXX03_STDBOOL_H
+#define _LIBCPP___CXX03_STDBOOL_H
 
 /*
     stdbool.h synopsis
@@ -37,4 +37,4 @@
 #  define __bool_true_false_are_defined 1
 #endif
 
-#endif // _LIBCPP_STDBOOL_H
+#endif // _LIBCPP___CXX03_STDBOOL_H
diff --git a/libcxx/include/__cxx03/stddef.h b/libcxx/include/__cxx03/stddef.h
index 38c15a3f2f247..5d46cf27c39f5 100644
--- a/libcxx/include/__cxx03/stddef.h
+++ b/libcxx/include/__cxx03/stddef.h
@@ -34,11 +34,11 @@
 //       with different defines and the underlying <stddef.h> will know how to deal with that.
 #include_next <stddef.h>
 
-#ifndef _LIBCPP_STDDEF_H
-#  define _LIBCPP_STDDEF_H
+#ifndef _LIBCPP___CXX03_STDDEF_H
+#  define _LIBCPP___CXX03_STDDEF_H
 
 #  ifdef __cplusplus
 typedef decltype(nullptr) nullptr_t;
 #  endif
 
-#endif // _LIBCPP_STDDEF_H
+#endif // _LIBCPP___CXX03_STDDEF_H
diff --git a/libcxx/include/__cxx03/stdexcept b/libcxx/include/__cxx03/stdexcept
index cd035f6df86db..772f8b543b172 100644
--- a/libcxx/include/__cxx03/stdexcept
+++ b/libcxx/include/__cxx03/stdexcept
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STDEXCEPT
-#define _LIBCPP_STDEXCEPT
+#ifndef _LIBCPP___CXX03_STDEXCEPT
+#define _LIBCPP___CXX03_STDEXCEPT
 
 /*
     stdexcept synopsis
@@ -283,4 +283,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/iosfwd>
 #endif
 
-#endif // _LIBCPP_STDEXCEPT
+#endif // _LIBCPP___CXX03_STDEXCEPT
diff --git a/libcxx/include/__cxx03/stdint.h b/libcxx/include/__cxx03/stdint.h
index 5e845a7a517a6..0510da42dd8cf 100644
--- a/libcxx/include/__cxx03/stdint.h
+++ b/libcxx/include/__cxx03/stdint.h
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STDINT_H
+#ifndef _LIBCPP___CXX03_STDINT_H
 // AIX system headers need stdint.h to be re-enterable while _STD_TYPES_T
 // is defined until an inclusion of it without _STD_TYPES_T occurs, in which
 // case the header guard macro is defined.
 #if !defined(_AIX) || !defined(_STD_TYPES_T)
-#  define _LIBCPP_STDINT_H
+#  define _LIBCPP___CXX03_STDINT_H
 #endif // _STD_TYPES_T
 
 /*
@@ -124,4 +124,4 @@
 #  include_next <stdint.h>
 #endif
 
-#endif // _LIBCPP_STDINT_H
+#endif // _LIBCPP___CXX03_STDINT_H
diff --git a/libcxx/include/__cxx03/stdio.h b/libcxx/include/__cxx03/stdio.h
index 214d9481bf4c6..6c6671e90c9a8 100644
--- a/libcxx/include/__cxx03/stdio.h
+++ b/libcxx/include/__cxx03/stdio.h
@@ -15,8 +15,8 @@
 
 #  include_next <stdio.h>
 
-#elif !defined(_LIBCPP_STDIO_H)
-#  define _LIBCPP_STDIO_H
+#elif !defined(_LIBCPP___CXX03_STDIO_H)
+#  define _LIBCPP___CXX03_STDIO_H
 
 /*
     stdio.h synopsis
@@ -120,4 +120,4 @@ void perror(const char* s);
 
 #  endif
 
-#endif // _LIBCPP_STDIO_H
+#endif // _LIBCPP___CXX03_STDIO_H
diff --git a/libcxx/include/__cxx03/stdlib.h b/libcxx/include/__cxx03/stdlib.h
index 77c85705e1911..5e7da5d90dc02 100644
--- a/libcxx/include/__cxx03/stdlib.h
+++ b/libcxx/include/__cxx03/stdlib.h
@@ -15,8 +15,8 @@
 
 #  include_next <stdlib.h>
 
-#elif !defined(_LIBCPP_STDLIB_H)
-#  define _LIBCPP_STDLIB_H
+#elif !defined(_LIBCPP___CXX03_STDLIB_H)
+#  define _LIBCPP___CXX03_STDLIB_H
 
 /*
     stdlib.h synopsis
@@ -148,4 +148,4 @@ inline _LIBCPP_HIDE_FROM_ABI lldiv_t div(long long __x, long long __y) _NOEXCEPT
 } // extern "C++"
 #  endif   // __cplusplus
 
-#endif // _LIBCPP_STDLIB_H
+#endif // _LIBCPP___CXX03_STDLIB_H
diff --git a/libcxx/include/__cxx03/stop_token b/libcxx/include/__cxx03/stop_token
index 7173162cf66af..369934d7ec6ea 100644
--- a/libcxx/include/__cxx03/stop_token
+++ b/libcxx/include/__cxx03/stop_token
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STOP_TOKEN
-#define _LIBCPP_STOP_TOKEN
+#ifndef _LIBCPP___CXX03_STOP_TOKEN
+#define _LIBCPP___CXX03_STOP_TOKEN
 
 /*
 
@@ -53,4 +53,4 @@ namespace std {
 #  include <__cxx03/iosfwd>
 #endif
 
-#endif // _LIBCPP_STOP_TOKEN
+#endif // _LIBCPP___CXX03_STOP_TOKEN
diff --git a/libcxx/include/__cxx03/streambuf b/libcxx/include/__cxx03/streambuf
index 9f60d38b4731f..8920fce1f78cc 100644
--- a/libcxx/include/__cxx03/streambuf
+++ b/libcxx/include/__cxx03/streambuf
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STREAMBUF
-#define _LIBCPP_STREAMBUF
+#ifndef _LIBCPP___CXX03_STREAMBUF
+#define _LIBCPP___CXX03_STREAMBUF
 
 /*
     streambuf synopsis
@@ -442,4 +442,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/cstdint>
 #endif
 
-#endif // _LIBCPP_STREAMBUF
+#endif // _LIBCPP___CXX03_STREAMBUF
diff --git a/libcxx/include/__cxx03/string b/libcxx/include/__cxx03/string
index 2f2f623440870..c4431dcb04d41 100644
--- a/libcxx/include/__cxx03/string
+++ b/libcxx/include/__cxx03/string
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STRING
-#define _LIBCPP_STRING
+#ifndef _LIBCPP___CXX03_STRING
+#define _LIBCPP___CXX03_STRING
 
 // clang-format off
 
@@ -4349,4 +4349,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_STRING
+#endif // _LIBCPP___CXX03_STRING
diff --git a/libcxx/include/__cxx03/string.h b/libcxx/include/__cxx03/string.h
index e1dc4cbd8f29e..49595fecc591d 100644
--- a/libcxx/include/__cxx03/string.h
+++ b/libcxx/include/__cxx03/string.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STRING_H
-#define _LIBCPP_STRING_H
+#ifndef _LIBCPP___CXX03_STRING_H
+#define _LIBCPP___CXX03_STRING_H
 
 /*
     string.h synopsis
@@ -107,4 +107,4 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strstr(char* __s1,
 } // extern "C++"
 #endif
 
-#endif // _LIBCPP_STRING_H
+#endif // _LIBCPP___CXX03_STRING_H
diff --git a/libcxx/include/__cxx03/string_view b/libcxx/include/__cxx03/string_view
index 8b21ecb3d9a8b..c6876ab5f4446 100644
--- a/libcxx/include/__cxx03/string_view
+++ b/libcxx/include/__cxx03/string_view
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STRING_VIEW
-#define _LIBCPP_STRING_VIEW
+#ifndef _LIBCPP___CXX03_STRING_VIEW
+#define _LIBCPP___CXX03_STRING_VIEW
 
 // clang-format off
 
@@ -945,4 +945,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_STRING_VIEW
+#endif // _LIBCPP___CXX03_STRING_VIEW
diff --git a/libcxx/include/__cxx03/strstream b/libcxx/include/__cxx03/strstream
index 80e166acb7515..be241a9484f60 100644
--- a/libcxx/include/__cxx03/strstream
+++ b/libcxx/include/__cxx03/strstream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_STRSTREAM
-#define _LIBCPP_STRSTREAM
+#ifndef _LIBCPP___CXX03_STRSTREAM
+#define _LIBCPP___CXX03_STRSTREAM
 
 /*
     strstream synopsis
@@ -348,4 +348,4 @@ _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
 
-#endif // _LIBCPP_STRSTREAM
+#endif // _LIBCPP___CXX03_STRSTREAM
diff --git a/libcxx/include/__cxx03/syncstream b/libcxx/include/__cxx03/syncstream
index 4e67b43f3ccb9..7aacdc6c05fd4 100644
--- a/libcxx/include/__cxx03/syncstream
+++ b/libcxx/include/__cxx03/syncstream
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SYNCSTREAM
-#define _LIBCPP_SYNCSTREAM
+#ifndef _LIBCPP___CXX03_SYNCSTREAM
+#define _LIBCPP___CXX03_SYNCSTREAM
 
 /*
     syncstream synopsis
@@ -509,4 +509,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_SYNCSTREAM
+#endif // _LIBCPP___CXX03_SYNCSTREAM
diff --git a/libcxx/include/__cxx03/system_error b/libcxx/include/__cxx03/system_error
index eb5a2dc4532e5..8d0d55c7e567b 100644
--- a/libcxx/include/__cxx03/system_error
+++ b/libcxx/include/__cxx03/system_error
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_SYSTEM_ERROR
-#define _LIBCPP_SYSTEM_ERROR
+#ifndef _LIBCPP___CXX03_SYSTEM_ERROR
+#define _LIBCPP___CXX03_SYSTEM_ERROR
 
 /*
     system_error synopsis
@@ -168,4 +168,4 @@ template <> struct hash<std::error_condition>;
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_SYSTEM_ERROR
+#endif // _LIBCPP___CXX03_SYSTEM_ERROR
diff --git a/libcxx/include/__cxx03/tgmath.h b/libcxx/include/__cxx03/tgmath.h
index 52637d1eaa048..7d81a110ed610 100644
--- a/libcxx/include/__cxx03/tgmath.h
+++ b/libcxx/include/__cxx03/tgmath.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_TGMATH_H
-#define _LIBCPP_TGMATH_H
+#ifndef _LIBCPP___CXX03_TGMATH_H
+#define _LIBCPP___CXX03_TGMATH_H
 
 /*
     tgmath.h synopsis
@@ -31,4 +31,4 @@
 #  endif
 #endif
 
-#endif // _LIBCPP_TGMATH_H
+#endif // _LIBCPP___CXX03_TGMATH_H
diff --git a/libcxx/include/__cxx03/thread b/libcxx/include/__cxx03/thread
index f6d9cc8096a74..8034854a73829 100644
--- a/libcxx/include/__cxx03/thread
+++ b/libcxx/include/__cxx03/thread
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_THREAD
-#define _LIBCPP_THREAD
+#ifndef _LIBCPP___CXX03_THREAD
+#define _LIBCPP___CXX03_THREAD
 
 /*
 
@@ -127,4 +127,4 @@ void sleep_for(const chrono::duration<Rep, Period>& rel_time);
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_THREAD
+#endif // _LIBCPP___CXX03_THREAD
diff --git a/libcxx/include/__cxx03/tuple b/libcxx/include/__cxx03/tuple
index 72375ad6c0abc..3579fd7981f00 100644
--- a/libcxx/include/__cxx03/tuple
+++ b/libcxx/include/__cxx03/tuple
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_TUPLE
-#define _LIBCPP_TUPLE
+#ifndef _LIBCPP___CXX03_TUPLE
+#define _LIBCPP___CXX03_TUPLE
 
 // clang-format off
 
@@ -1416,4 +1416,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_TUPLE
+#endif // _LIBCPP___CXX03_TUPLE
diff --git a/libcxx/include/__cxx03/type_traits b/libcxx/include/__cxx03/type_traits
index 484441cbb6700..287f6f0d32b18 100644
--- a/libcxx/include/__cxx03/type_traits
+++ b/libcxx/include/__cxx03/type_traits
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_TYPE_TRAITS
-#define _LIBCPP_TYPE_TRAITS
+#ifndef _LIBCPP___CXX03_TYPE_TRAITS
+#define _LIBCPP___CXX03_TYPE_TRAITS
 
 /*
     type_traits synopsis
@@ -520,4 +520,4 @@ namespace std
 #  pragma GCC system_header
 #endif
 
-#endif // _LIBCPP_TYPE_TRAITS
+#endif // _LIBCPP___CXX03_TYPE_TRAITS
diff --git a/libcxx/include/__cxx03/typeindex b/libcxx/include/__cxx03/typeindex
index e1d6097ebfa61..c7587e4e23057 100644
--- a/libcxx/include/__cxx03/typeindex
+++ b/libcxx/include/__cxx03/typeindex
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_TYPEINDEX
-#define _LIBCPP_TYPEINDEX
+#ifndef _LIBCPP___CXX03_TYPEINDEX
+#define _LIBCPP___CXX03_TYPEINDEX
 
 /*
 
@@ -103,4 +103,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_TYPEINDEX
+#endif // _LIBCPP___CXX03_TYPEINDEX
diff --git a/libcxx/include/__cxx03/typeinfo b/libcxx/include/__cxx03/typeinfo
index 823c3cbae1cfb..f01761bf485e3 100644
--- a/libcxx/include/__cxx03/typeinfo
+++ b/libcxx/include/__cxx03/typeinfo
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_TYPEINFO
-#define _LIBCPP_TYPEINFO
+#ifndef _LIBCPP___CXX03_TYPEINFO
+#define _LIBCPP___CXX03_TYPEINFO
 
 /*
 
@@ -386,4 +386,4 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_TYPEINFO
+#endif // _LIBCPP___CXX03_TYPEINFO
diff --git a/libcxx/include/__cxx03/uchar.h b/libcxx/include/__cxx03/uchar.h
index 29a10211ad36e..a19faea5814cf 100644
--- a/libcxx/include/__cxx03/uchar.h
+++ b/libcxx/include/__cxx03/uchar.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_UCHAR_H
-#define _LIBCPP_UCHAR_H
+#ifndef _LIBCPP___CXX03_UCHAR_H
+#define _LIBCPP___CXX03_UCHAR_H
 
 /*
     uchar.h synopsis // since C++11
@@ -53,4 +53,4 @@ size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
 
 #endif // _LIBCPP_CXX03_LANG
 
-#endif // _LIBCPP_UCHAR_H
+#endif // _LIBCPP___CXX03_UCHAR_H
diff --git a/libcxx/include/__cxx03/unordered_map b/libcxx/include/__cxx03/unordered_map
index cccfc4900895a..612ad02f77164 100644
--- a/libcxx/include/__cxx03/unordered_map
+++ b/libcxx/include/__cxx03/unordered_map
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_UNORDERED_MAP
-#define _LIBCPP_UNORDERED_MAP
+#ifndef _LIBCPP___CXX03_UNORDERED_MAP
+#define _LIBCPP___CXX03_UNORDERED_MAP
 
 /*
 
@@ -2538,4 +2538,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_UNORDERED_MAP
+#endif // _LIBCPP___CXX03_UNORDERED_MAP
diff --git a/libcxx/include/__cxx03/unordered_set b/libcxx/include/__cxx03/unordered_set
index b64d334c129de..25e9e1f82ad32 100644
--- a/libcxx/include/__cxx03/unordered_set
+++ b/libcxx/include/__cxx03/unordered_set
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_UNORDERED_SET
-#define _LIBCPP_UNORDERED_SET
+#ifndef _LIBCPP___CXX03_UNORDERED_SET
+#define _LIBCPP___CXX03_UNORDERED_SET
 
 // clang-format off
 
@@ -1808,4 +1808,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_UNORDERED_SET
+#endif // _LIBCPP___CXX03_UNORDERED_SET
diff --git a/libcxx/include/__cxx03/utility b/libcxx/include/__cxx03/utility
index c0734df8db636..ac0637f6b41ee 100644
--- a/libcxx/include/__cxx03/utility
+++ b/libcxx/include/__cxx03/utility
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_UTILITY
-#define _LIBCPP_UTILITY
+#ifndef _LIBCPP___CXX03_UTILITY
+#define _LIBCPP___CXX03_UTILITY
 
 /*
     utility synopsis
@@ -306,4 +306,4 @@ template <class T>
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_UTILITY
+#endif // _LIBCPP___CXX03_UTILITY
diff --git a/libcxx/include/__cxx03/valarray b/libcxx/include/__cxx03/valarray
index 890e0a2723c7a..1070add36abc4 100644
--- a/libcxx/include/__cxx03/valarray
+++ b/libcxx/include/__cxx03/valarray
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_VALARRAY
-#define _LIBCPP_VALARRAY
+#ifndef _LIBCPP___CXX03_VALARRAY
+#define _LIBCPP___CXX03_VALARRAY
 
 /*
     valarray synopsis
@@ -3361,4 +3361,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/type_traits>
 #endif
 
-#endif // _LIBCPP_VALARRAY
+#endif // _LIBCPP___CXX03_VALARRAY
diff --git a/libcxx/include/__cxx03/variant b/libcxx/include/__cxx03/variant
index 97e012bf81c95..cedc733e9b391 100644
--- a/libcxx/include/__cxx03/variant
+++ b/libcxx/include/__cxx03/variant
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_VARIANT
-#define _LIBCPP_VARIANT
+#ifndef _LIBCPP___CXX03_VARIANT
+#define _LIBCPP___CXX03_VARIANT
 
 /*
    variant synopsis
@@ -1647,4 +1647,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_VARIANT
+#endif // _LIBCPP___CXX03_VARIANT
diff --git a/libcxx/include/__cxx03/vector b/libcxx/include/__cxx03/vector
index 6e01ece06ab97..6ee35b4e36258 100644
--- a/libcxx/include/__cxx03/vector
+++ b/libcxx/include/__cxx03/vector
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_VECTOR
-#define _LIBCPP_VECTOR
+#ifndef _LIBCPP___CXX03_VECTOR
+#define _LIBCPP___CXX03_VECTOR
 
 // clang-format off
 
@@ -3027,4 +3027,4 @@ _LIBCPP_POP_MACROS
 #  include <__cxx03/utility>
 #endif
 
-#endif // _LIBCPP_VECTOR
+#endif // _LIBCPP___CXX03_VECTOR
diff --git a/libcxx/include/__cxx03/wchar.h b/libcxx/include/__cxx03/wchar.h
index 131948eb471d0..91975f7df49bd 100644
--- a/libcxx/include/__cxx03/wchar.h
+++ b/libcxx/include/__cxx03/wchar.h
@@ -15,8 +15,8 @@
 
 #  include_next <wchar.h>
 
-#elif !defined(_LIBCPP_WCHAR_H)
-#  define _LIBCPP_WCHAR_H
+#elif !defined(_LIBCPP___CXX03_WCHAR_H)
+#  define _LIBCPP___CXX03_WCHAR_H
 
 /*
     wchar.h synopsis
@@ -208,4 +208,4 @@ size_t wcsnrtombs(
 } // extern "C"
 #  endif // __cplusplus && (_LIBCPP_MSVCRT || __MVS__)
 
-#endif // _LIBCPP_WCHAR_H
+#endif // _LIBCPP___CXX03_WCHAR_H
diff --git a/libcxx/include/__cxx03/wctype.h b/libcxx/include/__cxx03/wctype.h
index def009c58f015..131d8b51ea1ca 100644
--- a/libcxx/include/__cxx03/wctype.h
+++ b/libcxx/include/__cxx03/wctype.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP_WCTYPE_H
-#define _LIBCPP_WCTYPE_H
+#ifndef _LIBCPP___CXX03_WCTYPE_H
+#define _LIBCPP___CXX03_WCTYPE_H
 
 /*
     wctype.h synopsis
@@ -92,4 +92,4 @@ wctrans_t wctrans(const char* property);
 
 #endif // __cplusplus
 
-#endif // _LIBCPP_WCTYPE_H
+#endif // _LIBCPP___CXX03_WCTYPE_H
diff --git a/libcxx/include/__flat_map/flat_map.h b/libcxx/include/__flat_map/flat_map.h
index 58b362ad7a706..b66bc1cb66fc1 100644
--- a/libcxx/include/__flat_map/flat_map.h
+++ b/libcxx/include/__flat_map/flat_map.h
@@ -23,11 +23,11 @@
 #include <__algorithm/remove_if.h>
 #include <__assert>
 #include <__compare/synth_three_way.h>
-#include <__concepts/convertible_to.h>
 #include <__concepts/swappable.h>
 #include <__config>
 #include <__cstddef/byte.h>
 #include <__cstddef/ptrdiff_t.h>
+#include <__flat_map/key_value_iterator.h>
 #include <__flat_map/sorted_unique.h>
 #include <__functional/invoke.h>
 #include <__functional/is_transparent.h>
@@ -38,7 +38,6 @@
 #include <__iterator/next.h>
 #include <__iterator/ranges_iterator_traits.h>
 #include <__iterator/reverse_iterator.h>
-#include <__memory/addressof.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/uses_allocator.h>
 #include <__memory/uses_allocator_construction.h>
@@ -57,8 +56,8 @@
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/is_same.h>
-#include <__type_traits/maybe_const.h>
 #include <__utility/exception_guard.h>
+#include <__utility/move.h>
 #include <__utility/pair.h>
 #include <__utility/scope_guard.h>
 #include <__vector/vector.h>
@@ -82,9 +81,6 @@ template <class _Key,
           class _KeyContainer    = vector<_Key>,
           class _MappedContainer = vector<_Tp>>
 class flat_map {
-  template <bool _Const>
-  struct __iterator;
-
   template <class, class, class, class, class>
   friend class flat_map;
 
@@ -93,6 +89,9 @@ class flat_map {
   static_assert(!is_same_v<_KeyContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
   static_assert(!is_same_v<_MappedContainer, std::vector<bool>>, "vector<bool> is not a sequence container");
 
+  template <bool _Const>
+  using __iterator = __key_value_iterator<flat_map, _KeyContainer, _MappedContainer, _Const>;
+
 public:
   // types
   using key_type               = _Key;
@@ -134,123 +133,6 @@ class flat_map {
 
   _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_compare_transparent = __is_transparent_v<_Compare, _Compare>;
 
-  template <bool _Const>
-  struct __iterator {
-  private:
-    using __key_iterator    = ranges::iterator_t<const key_container_type>;
-    using __mapped_iterator = ranges::iterator_t<__maybe_const<_Const, mapped_container_type>>;
-    using __reference       = pair<iter_reference_t<__key_iterator>, iter_reference_t<__mapped_iterator>>;
-
-    struct __arrow_proxy {
-      __reference __ref_;
-      _LIBCPP_HIDE_FROM_ABI __reference* operator->() { return std::addressof(__ref_); }
-    };
-
-    __key_iterator __key_iter_;
-    __mapped_iterator __mapped_iter_;
-
-    friend flat_map;
-
-  public:
-    using iterator_concept = random_access_iterator_tag;
-    // `flat_map::iterator` only satisfy "Cpp17InputIterator" named requirements, because
-    // its `reference` is not a reference type.
-    // However, to avoid surprising runtime behaviour when it is used with the
-    // Cpp17 algorithms or operations, iterator_category is set to random_access_iterator_tag.
-    using iterator_category = random_access_iterator_tag;
-    using value_type        = flat_map::value_type;
-    using difference_type   = flat_map::difference_type;
-
-    _LIBCPP_HIDE_FROM_ABI __iterator() = default;
-
-    _LIBCPP_HIDE_FROM_ABI __iterator(__iterator<!_Const> __i)
-      requires _Const && convertible_to<ranges::iterator_t<key_container_type>, __key_iterator> &&
-                   convertible_to<ranges::iterator_t<mapped_container_type>, __mapped_iterator>
-        : __key_iter_(std::move(__i.__key_iter_)), __mapped_iter_(std::move(__i.__mapped_iter_)) {}
-
-    _LIBCPP_HIDE_FROM_ABI __iterator(__key_iterator __key_iter, __mapped_iterator __mapped_iter)
-        : __key_iter_(std::move(__key_iter)), __mapped_iter_(std::move(__mapped_iter)) {}
-
-    _LIBCPP_HIDE_FROM_ABI __reference operator*() const { return __reference(*__key_iter_, *__mapped_iter_); }
-    _LIBCPP_HIDE_FROM_ABI __arrow_proxy operator->() const { return __arrow_proxy{**this}; }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator& operator++() {
-      ++__key_iter_;
-      ++__mapped_iter_;
-      return *this;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator operator++(int) {
-      __iterator __tmp(*this);
-      ++*this;
-      return __tmp;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator& operator--() {
-      --__key_iter_;
-      --__mapped_iter_;
-      return *this;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator operator--(int) {
-      __iterator __tmp(*this);
-      --*this;
-      return __tmp;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator& operator+=(difference_type __x) {
-      __key_iter_ += __x;
-      __mapped_iter_ += __x;
-      return *this;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __iterator& operator-=(difference_type __x) {
-      __key_iter_ -= __x;
-      __mapped_iter_ -= __x;
-      return *this;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI __reference operator[](difference_type __n) const { return *(*this + __n); }
-
-    _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
-      return __x.__key_iter_ == __y.__key_iter_;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI friend bool operator<(const __iterator& __x, const __iterator& __y) {
-      return __x.__key_iter_ < __y.__key_iter_;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI friend bool operator>(const __iterator& __x, const __iterator& __y) { return __y < __x; }
-
-    _LIBCPP_HIDE_FROM_ABI friend bool operator<=(const __iterator& __x, const __iterator& __y) { return !(__y < __x); }
-
-    _LIBCPP_HIDE_FROM_ABI friend bool operator>=(const __iterator& __x, const __iterator& __y) { return !(__x < __y); }
-
-    _LIBCPP_HIDE_FROM_ABI friend auto operator<=>(const __iterator& __x, const __iterator& __y)
-      requires three_way_comparable<__key_iterator>
-    {
-      return __x.__key_iter_ <=> __y.__key_iter_;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(const __iterator& __i, difference_type __n) {
-      auto __tmp = __i;
-      __tmp += __n;
-      return __tmp;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(difference_type __n, const __iterator& __i) { return __i + __n; }
-
-    _LIBCPP_HIDE_FROM_ABI friend __iterator operator-(const __iterator& __i, difference_type __n) {
-      auto __tmp = __i;
-      __tmp -= __n;
-      return __tmp;
-    }
-
-    _LIBCPP_HIDE_FROM_ABI friend difference_type operator-(const __iterator& __x, const __iterator& __y) {
-      return difference_type(__x.__key_iter_ - __y.__key_iter_);
-    }
-  };
-
 public:
   // [flat.map.cons], construct/copy/destroy
   _LIBCPP_HIDE_FROM_ABI flat_map() noexcept(
diff --git a/libcxx/include/__flat_map/key_value_iterator.h b/libcxx/include/__flat_map/key_value_iterator.h
new file mode 100644
index 0000000000000..987ac677a4130
--- /dev/null
+++ b/libcxx/include/__flat_map/key_value_iterator.h
@@ -0,0 +1,177 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FLAT_MAP_KEY_VALUE_ITERATOR_H
+#define _LIBCPP___FLAT_MAP_KEY_VALUE_ITERATOR_H
+
+#include <__compare/three_way_comparable.h>
+#include <__concepts/convertible_to.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+#include <__ranges/access.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/maybe_const.h>
+#include <__utility/move.h>
+#include <__utility/pair.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 23
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+/**
+ * __key_value_iterator is a proxy iterator which zips the underlying
+ * _KeyContainer::iterator and the underlying _MappedContainer::iterator.
+ * The two underlying iterators will be incremented/decremented together.
+ * And the reference is a pair of the const key reference and the value reference.
+ */
+template <class _Owner, class _KeyContainer, class _MappedContainer, bool _Const>
+struct __key_value_iterator {
+private:
+  using __key_iterator    = ranges::iterator_t<const _KeyContainer>;
+  using __mapped_iterator = ranges::iterator_t<__maybe_const<_Const, _MappedContainer>>;
+  using __reference       = _If<_Const, typename _Owner::const_reference, typename _Owner::reference>;
+
+  struct __arrow_proxy {
+    __reference __ref_;
+    _LIBCPP_HIDE_FROM_ABI __reference* operator->() { return std::addressof(__ref_); }
+  };
+
+  __key_iterator __key_iter_;
+  __mapped_iterator __mapped_iter_;
+
+  friend _Owner;
+
+  template <class, class, class, bool>
+  friend struct __key_value_iterator;
+
+public:
+  using iterator_concept = random_access_iterator_tag;
+  // `__key_value_iterator` only satisfy "Cpp17InputIterator" named requirements, because
+  // its `reference` is not a reference type.
+  // However, to avoid surprising runtime behaviour when it is used with the
+  // Cpp17 algorithms or operations, iterator_category is set to random_access_iterator_tag.
+  using iterator_category = random_access_iterator_tag;
+  using value_type        = typename _Owner::value_type;
+  using difference_type   = typename _Owner::difference_type;
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator() = default;
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator(__key_value_iterator<_Owner, _KeyContainer, _MappedContainer, !_Const> __i)
+    requires _Const && convertible_to<ranges::iterator_t<_KeyContainer>, __key_iterator> &&
+                 convertible_to<ranges::iterator_t<_MappedContainer>, __mapped_iterator>
+      : __key_iter_(std::move(__i.__key_iter_)), __mapped_iter_(std::move(__i.__mapped_iter_)) {}
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator(__key_iterator __key_iter, __mapped_iterator __mapped_iter)
+      : __key_iter_(std::move(__key_iter)), __mapped_iter_(std::move(__mapped_iter)) {}
+
+  _LIBCPP_HIDE_FROM_ABI __reference operator*() const { return __reference(*__key_iter_, *__mapped_iter_); }
+  _LIBCPP_HIDE_FROM_ABI __arrow_proxy operator->() const { return __arrow_proxy{**this}; }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator& operator++() {
+    ++__key_iter_;
+    ++__mapped_iter_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator operator++(int) {
+    __key_value_iterator __tmp(*this);
+    ++*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator& operator--() {
+    --__key_iter_;
+    --__mapped_iter_;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator operator--(int) {
+    __key_value_iterator __tmp(*this);
+    --*this;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator& operator+=(difference_type __x) {
+    __key_iter_ += __x;
+    __mapped_iter_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __key_value_iterator& operator-=(difference_type __x) {
+    __key_iter_ -= __x;
+    __mapped_iter_ -= __x;
+    return *this;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI __reference operator[](difference_type __n) const { return *(*this + __n); }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool
+  operator==(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return __x.__key_iter_ == __y.__key_iter_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator<(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return __x.__key_iter_ < __y.__key_iter_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator>(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return __y < __x;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator<=(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return !(__y < __x);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend bool operator>=(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return !(__x < __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend auto operator<=>(const __key_value_iterator& __x, const __key_value_iterator& __y)
+    requires three_way_comparable<__key_iterator>
+  {
+    return __x.__key_iter_ <=> __y.__key_iter_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend __key_value_iterator operator+(const __key_value_iterator& __i, difference_type __n) {
+    auto __tmp = __i;
+    __tmp += __n;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend __key_value_iterator operator+(difference_type __n, const __key_value_iterator& __i) {
+    return __i + __n;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend __key_value_iterator operator-(const __key_value_iterator& __i, difference_type __n) {
+    auto __tmp = __i;
+    __tmp -= __n;
+    return __tmp;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend difference_type
+  operator-(const __key_value_iterator& __x, const __key_value_iterator& __y) {
+    return difference_type(__x.__key_iter_ - __y.__key_iter_);
+  }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FLAT_MAP_KEY_VALUE_ITERATOR_H
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index 62b454c922752..2d9ab847e9f25 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -44,10 +44,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // __pointer
 template <class _Tp>
-using __pointer_member = typename _Tp::pointer;
+using __pointer_member _LIBCPP_NODEBUG = typename _Tp::pointer;
 
 template <class _Tp, class _Alloc>
-using __pointer = __detected_or_t<_Tp*, __pointer_member, __libcpp_remove_reference_t<_Alloc> >;
+using __pointer _LIBCPP_NODEBUG = __detected_or_t<_Tp*, __pointer_member, __libcpp_remove_reference_t<_Alloc> >;
 
 // __const_pointer
 _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_const_pointer, const_pointer);
@@ -58,7 +58,7 @@ struct __const_pointer {
 template <class _Tp, class _Ptr, class _Alloc>
 struct __const_pointer<_Tp, _Ptr, _Alloc, false> {
 #ifdef _LIBCPP_CXX03_LANG
-  using type = typename pointer_traits<_Ptr>::template rebind<const _Tp>::other;
+  using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<const _Tp>::other;
 #else
   using type _LIBCPP_NODEBUG = typename pointer_traits<_Ptr>::template rebind<const _Tp>;
 #endif
@@ -96,10 +96,10 @@ struct __const_void_pointer<_Ptr, _Alloc, false> {
 
 // __size_type
 template <class _Tp>
-using __size_type_member = typename _Tp::size_type;
+using __size_type_member _LIBCPP_NODEBUG = typename _Tp::size_type;
 
 template <class _Alloc, class _DiffType>
-using __size_type = __detected_or_t<__make_unsigned_t<_DiffType>, __size_type_member, _Alloc>;
+using __size_type _LIBCPP_NODEBUG = __detected_or_t<__make_unsigned_t<_DiffType>, __size_type_member, _Alloc>;
 
 // __alloc_traits_difference_type
 _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX(__has_alloc_traits_difference_type, difference_type);
@@ -114,33 +114,37 @@ struct __alloc_traits_difference_type<_Alloc, _Ptr, true> {
 
 // __propagate_on_container_copy_assignment
 template <class _Tp>
-using __propagate_on_container_copy_assignment_member = typename _Tp::propagate_on_container_copy_assignment;
+using __propagate_on_container_copy_assignment_member _LIBCPP_NODEBUG =
+    typename _Tp::propagate_on_container_copy_assignment;
 
 template <class _Alloc>
-using __propagate_on_container_copy_assignment =
+using __propagate_on_container_copy_assignment _LIBCPP_NODEBUG =
     __detected_or_t<false_type, __propagate_on_container_copy_assignment_member, _Alloc>;
 
 // __propagate_on_container_move_assignment
 template <class _Tp>
-using __propagate_on_container_move_assignment_member = typename _Tp::propagate_on_container_move_assignment;
+using __propagate_on_container_move_assignment_member _LIBCPP_NODEBUG =
+    typename _Tp::propagate_on_container_move_assignment;
 
 template <class _Alloc>
-using __propagate_on_container_move_assignment =
+using __propagate_on_container_move_assignment _LIBCPP_NODEBUG =
     __detected_or_t<false_type, __propagate_on_container_move_assignment_member, _Alloc>;
 
 // __propagate_on_container_swap
 template <class _Tp>
-using __propagate_on_container_swap_member = typename _Tp::propagate_on_container_swap;
+using __propagate_on_container_swap_member _LIBCPP_NODEBUG = typename _Tp::propagate_on_container_swap;
 
 template <class _Alloc>
-using __propagate_on_container_swap = __detected_or_t<false_type, __propagate_on_container_swap_member, _Alloc>;
+using __propagate_on_container_swap _LIBCPP_NODEBUG =
+    __detected_or_t<false_type, __propagate_on_container_swap_member, _Alloc>;
 
 // __is_always_equal
 template <class _Tp>
-using __is_always_equal_member = typename _Tp::is_always_equal;
+using __is_always_equal_member _LIBCPP_NODEBUG = typename _Tp::is_always_equal;
 
 template <class _Alloc>
-using __is_always_equal = __detected_or_t<typename is_empty<_Alloc>::type, __is_always_equal_member, _Alloc>;
+using __is_always_equal _LIBCPP_NODEBUG =
+    __detected_or_t<typename is_empty<_Alloc>::type, __is_always_equal_member, _Alloc>;
 
 // __allocator_traits_rebind
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
@@ -165,7 +169,7 @@ struct __allocator_traits_rebind<_Alloc<_Tp, _Args...>, _Up, false> {
 _LIBCPP_SUPPRESS_DEPRECATED_POP
 
 template <class _Alloc, class _Tp>
-using __allocator_traits_rebind_t = typename __allocator_traits_rebind<_Alloc, _Tp>::type;
+using __allocator_traits_rebind_t _LIBCPP_NODEBUG = typename __allocator_traits_rebind<_Alloc, _Tp>::type;
 
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 
@@ -355,12 +359,12 @@ template <class _Traits, class _Tp>
 using __rebind_alloc _LIBCPP_NODEBUG = typename _Traits::template rebind_alloc<_Tp>;
 #else
 template <class _Traits, class _Tp>
-using __rebind_alloc = typename _Traits::template rebind_alloc<_Tp>::other;
+using __rebind_alloc _LIBCPP_NODEBUG = typename _Traits::template rebind_alloc<_Tp>::other;
 #endif
 
 template <class _Alloc>
 struct __check_valid_allocator : true_type {
-  using _Traits = std::allocator_traits<_Alloc>;
+  using _Traits _LIBCPP_NODEBUG = std::allocator_traits<_Alloc>;
   static_assert(is_same<_Alloc, __rebind_alloc<_Traits, typename _Traits::value_type> >::value,
                 "[allocator.requirements] states that rebinding an allocator to the same type should result in the "
                 "original allocator");
diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h
index a7acaaff9da09..38798a21fa3c9 100644
--- a/libcxx/include/__memory/compressed_pair.h
+++ b/libcxx/include/__memory/compressed_pair.h
@@ -50,9 +50,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // member1 - offset: 0, size: 4
 // member2 - offset: 4, size: 4
 // member3 - offset: 8, size: 8
+//
+// Furthermore, that alignment must be the same as what was used in the old __compressed_pair layout, so we must
+// handle reference types specially since alignof(T&) == alignof(T).
+// See https://github.com/llvm/llvm-project/issues/118559.
 
 #ifndef _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 
+template <class _Tp>
+inline const size_t __compressed_pair_alignment = _LIBCPP_ALIGNOF(_Tp);
+
+template <class _Tp>
+inline const size_t __compressed_pair_alignment<_Tp&> = _LIBCPP_ALIGNOF(void*);
+
 template <class _ToPad,
           bool _Empty = ((is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value) ||
                          is_reference<_ToPad>::value || sizeof(_ToPad) == __datasizeof_v<_ToPad>)>
@@ -64,14 +74,15 @@ template <class _ToPad>
 class __compressed_pair_padding<_ToPad, true> {};
 
 #  define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)                                                  \
-    _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(_LIBCPP_ALIGNOF(T2)))) T1 Initializer1;                       \
+    _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1;    \
     _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);          \
     _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                         \
     _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _)
 
 #  define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3)                              \
     _LIBCPP_NO_UNIQUE_ADDRESS                                                                                          \
-    __attribute__((__aligned__(_LIBCPP_ALIGNOF(T2)), __aligned__(_LIBCPP_ALIGNOF(T3)))) T1 Initializer1;               \
+    __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>),                                                \
+                   __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1;                              \
     _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);          \
     _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                         \
     _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _);          \
diff --git a/libcxx/include/__numeric/gcd_lcm.h b/libcxx/include/__numeric/gcd_lcm.h
index 9be6cf8516b13..f15f64ea5568d 100644
--- a/libcxx/include/__numeric/gcd_lcm.h
+++ b/libcxx/include/__numeric/gcd_lcm.h
@@ -55,7 +55,8 @@ template <class _Tp>
 constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
   static_assert(!is_signed<_Tp>::value, "");
 
-  // From: https://lemire.me/blog/2013/12/26/fastest-way-to-compute-the-greatest-common-divisor
+  // Using Binary GCD algorithm https://en.wikipedia.org/wiki/Binary_GCD_algorithm, based on an implementation
+  // from https://lemire.me/blog/2024/04/13/greatest-common-divisor-the-extended-euclidean-algorithm-and-speed/
   //
   // If power of two divides both numbers, we can push it out.
   // - gcd( 2^x * a, 2^x * b) = 2^x * gcd(a, b)
@@ -76,21 +77,17 @@ constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
   if (__a == 0)
     return __b;
 
-  int __az    = std::__countr_zero(__a);
-  int __bz    = std::__countr_zero(__b);
-  int __shift = std::min(__az, __bz);
-  __a >>= __az;
-  __b >>= __bz;
+  _Tp __c     = __a | __b;
+  int __shift = std::__countr_zero(__c);
+  __a >>= std::__countr_zero(__a);
   do {
-    _Tp __diff = __a - __b;
-    if (__a > __b) {
-      __a = __b;
-      __b = __diff;
+    _Tp __t = __b >> std::__countr_zero(__b);
+    if (__a > __t) {
+      __b = __a - __t;
+      __a = __t;
     } else {
-      __b = __b - __a;
+      __b = __t - __a;
     }
-    if (__diff != 0)
-      __b >>= std::__countr_zero(__diff);
   } while (__b != 0);
   return __a << __shift;
 }
diff --git a/libcxx/include/__type_traits/detected_or.h b/libcxx/include/__type_traits/detected_or.h
index 390f368411471..49c0c7111aebf 100644
--- a/libcxx/include/__type_traits/detected_or.h
+++ b/libcxx/include/__type_traits/detected_or.h
@@ -20,16 +20,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Default, class _Void, template <class...> class _Op, class... _Args>
 struct __detector {
-  using type = _Default;
+  using type _LIBCPP_NODEBUG = _Default;
 };
 
 template <class _Default, template <class...> class _Op, class... _Args>
 struct __detector<_Default, __void_t<_Op<_Args...> >, _Op, _Args...> {
-  using type = _Op<_Args...>;
+  using type _LIBCPP_NODEBUG = _Op<_Args...>;
 };
 
 template <class _Default, template <class...> class _Op, class... _Args>
-using __detected_or_t = typename __detector<_Default, void, _Op, _Args...>::type;
+using __detected_or_t _LIBCPP_NODEBUG = typename __detector<_Default, void, _Op, _Args...>::type;
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h
index 4b9ba24e97f65..6ba7ba7bcf724 100644
--- a/libcxx/include/__vector/vector.h
+++ b/libcxx/include/__vector/vector.h
@@ -1003,9 +1003,15 @@ template <class _Tp, class _Allocator>
 template <class _Iterator, class _Sentinel>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
 vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) {
-  clear();
-  for (; __first != __last; ++__first)
-    emplace_back(*__first);
+  pointer __cur = __begin_;
+  for (; __first != __last && __cur != __end_; ++__first, (void)++__cur)
+    *__cur = *__first;
+  if (__cur != __end_) {
+    __destruct_at_end(__cur);
+  } else {
+    for (; __first != __last; ++__first)
+      emplace_back(*__first);
+  }
 }
 
 template <class _Tp, class _Allocator>
diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h
index 3b03f65b7ee54..36eb7f350ac40 100644
--- a/libcxx/include/__vector/vector_bool.h
+++ b/libcxx/include/__vector/vector_bool.h
@@ -390,18 +390,12 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
   template <class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
   __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      for (; __first != __last; ++__first)
-        push_back(*__first);
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-      if (__begin_ != nullptr)
-        __storage_traits::deallocate(__alloc_, __begin_, __cap_);
-      throw;
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+    auto __guard = std::__make_exception_guard(__destroy_vector(*this));
+
+    for (; __first != __last; ++__first)
+      push_back(*__first);
+
+    __guard.__complete();
   }
 
   template <class _Iterator, class _Sentinel>
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index 14355d63f2901..080fca8c1ca51 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -1827,232 +1827,235 @@ template <class BidirectionalIterator, class Compare>
 
 */
 
-#include <__config>
-
-#include <__algorithm/adjacent_find.h>
-#include <__algorithm/all_of.h>
-#include <__algorithm/any_of.h>
-#include <__algorithm/binary_search.h>
-#include <__algorithm/copy.h>
-#include <__algorithm/copy_backward.h>
-#include <__algorithm/copy_if.h>
-#include <__algorithm/copy_n.h>
-#include <__algorithm/count.h>
-#include <__algorithm/count_if.h>
-#include <__algorithm/equal.h>
-#include <__algorithm/equal_range.h>
-#include <__algorithm/fill.h>
-#include <__algorithm/fill_n.h>
-#include <__algorithm/find.h>
-#include <__algorithm/find_end.h>
-#include <__algorithm/find_first_of.h>
-#include <__algorithm/find_if.h>
-#include <__algorithm/find_if_not.h>
-#include <__algorithm/for_each.h>
-#include <__algorithm/generate.h>
-#include <__algorithm/generate_n.h>
-#include <__algorithm/includes.h>
-#include <__algorithm/inplace_merge.h>
-#include <__algorithm/is_heap.h>
-#include <__algorithm/is_heap_until.h>
-#include <__algorithm/is_partitioned.h>
-#include <__algorithm/is_permutation.h>
-#include <__algorithm/is_sorted.h>
-#include <__algorithm/is_sorted_until.h>
-#include <__algorithm/iter_swap.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lower_bound.h>
-#include <__algorithm/make_heap.h>
-#include <__algorithm/max.h>
-#include <__algorithm/max_element.h>
-#include <__algorithm/merge.h>
-#include <__algorithm/min.h>
-#include <__algorithm/min_element.h>
-#include <__algorithm/minmax.h>
-#include <__algorithm/minmax_element.h>
-#include <__algorithm/mismatch.h>
-#include <__algorithm/move.h>
-#include <__algorithm/move_backward.h>
-#include <__algorithm/next_permutation.h>
-#include <__algorithm/none_of.h>
-#include <__algorithm/nth_element.h>
-#include <__algorithm/partial_sort.h>
-#include <__algorithm/partial_sort_copy.h>
-#include <__algorithm/partition.h>
-#include <__algorithm/partition_copy.h>
-#include <__algorithm/partition_point.h>
-#include <__algorithm/pop_heap.h>
-#include <__algorithm/prev_permutation.h>
-#include <__algorithm/push_heap.h>
-#include <__algorithm/remove.h>
-#include <__algorithm/remove_copy.h>
-#include <__algorithm/remove_copy_if.h>
-#include <__algorithm/remove_if.h>
-#include <__algorithm/replace.h>
-#include <__algorithm/replace_copy.h>
-#include <__algorithm/replace_copy_if.h>
-#include <__algorithm/replace_if.h>
-#include <__algorithm/reverse.h>
-#include <__algorithm/reverse_copy.h>
-#include <__algorithm/rotate.h>
-#include <__algorithm/rotate_copy.h>
-#include <__algorithm/search.h>
-#include <__algorithm/search_n.h>
-#include <__algorithm/set_difference.h>
-#include <__algorithm/set_intersection.h>
-#include <__algorithm/set_symmetric_difference.h>
-#include <__algorithm/set_union.h>
-#include <__algorithm/shuffle.h>
-#include <__algorithm/sort.h>
-#include <__algorithm/sort_heap.h>
-#include <__algorithm/stable_partition.h>
-#include <__algorithm/stable_sort.h>
-#include <__algorithm/swap_ranges.h>
-#include <__algorithm/transform.h>
-#include <__algorithm/unique.h>
-#include <__algorithm/unique_copy.h>
-#include <__algorithm/upper_bound.h>
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__algorithm/clamp.h>
-#  include <__algorithm/for_each_n.h>
-#  include <__algorithm/pstl.h>
-#  include <__algorithm/sample.h>
-#endif // _LIBCPP_STD_VER >= 17
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__algorithm/in_found_result.h>
-#  include <__algorithm/in_fun_result.h>
-#  include <__algorithm/in_in_out_result.h>
-#  include <__algorithm/in_in_result.h>
-#  include <__algorithm/in_out_out_result.h>
-#  include <__algorithm/in_out_result.h>
-#  include <__algorithm/lexicographical_compare_three_way.h>
-#  include <__algorithm/min_max_result.h>
-#  include <__algorithm/ranges_adjacent_find.h>
-#  include <__algorithm/ranges_all_of.h>
-#  include <__algorithm/ranges_any_of.h>
-#  include <__algorithm/ranges_binary_search.h>
-#  include <__algorithm/ranges_clamp.h>
-#  include <__algorithm/ranges_contains.h>
-#  include <__algorithm/ranges_copy.h>
-#  include <__algorithm/ranges_copy_backward.h>
-#  include <__algorithm/ranges_copy_if.h>
-#  include <__algorithm/ranges_copy_n.h>
-#  include <__algorithm/ranges_count.h>
-#  include <__algorithm/ranges_count_if.h>
-#  include <__algorithm/ranges_equal.h>
-#  include <__algorithm/ranges_equal_range.h>
-#  include <__algorithm/ranges_fill.h>
-#  include <__algorithm/ranges_fill_n.h>
-#  include <__algorithm/ranges_find.h>
-#  include <__algorithm/ranges_find_end.h>
-#  include <__algorithm/ranges_find_first_of.h>
-#  include <__algorithm/ranges_find_if.h>
-#  include <__algorithm/ranges_find_if_not.h>
-#  include <__algorithm/ranges_for_each.h>
-#  include <__algorithm/ranges_for_each_n.h>
-#  include <__algorithm/ranges_generate.h>
-#  include <__algorithm/ranges_generate_n.h>
-#  include <__algorithm/ranges_includes.h>
-#  include <__algorithm/ranges_inplace_merge.h>
-#  include <__algorithm/ranges_is_heap.h>
-#  include <__algorithm/ranges_is_heap_until.h>
-#  include <__algorithm/ranges_is_partitioned.h>
-#  include <__algorithm/ranges_is_permutation.h>
-#  include <__algorithm/ranges_is_sorted.h>
-#  include <__algorithm/ranges_is_sorted_until.h>
-#  include <__algorithm/ranges_lexicographical_compare.h>
-#  include <__algorithm/ranges_lower_bound.h>
-#  include <__algorithm/ranges_make_heap.h>
-#  include <__algorithm/ranges_max.h>
-#  include <__algorithm/ranges_max_element.h>
-#  include <__algorithm/ranges_merge.h>
-#  include <__algorithm/ranges_min.h>
-#  include <__algorithm/ranges_min_element.h>
-#  include <__algorithm/ranges_minmax.h>
-#  include <__algorithm/ranges_minmax_element.h>
-#  include <__algorithm/ranges_mismatch.h>
-#  include <__algorithm/ranges_move.h>
-#  include <__algorithm/ranges_move_backward.h>
-#  include <__algorithm/ranges_next_permutation.h>
-#  include <__algorithm/ranges_none_of.h>
-#  include <__algorithm/ranges_nth_element.h>
-#  include <__algorithm/ranges_partial_sort.h>
-#  include <__algorithm/ranges_partial_sort_copy.h>
-#  include <__algorithm/ranges_partition.h>
-#  include <__algorithm/ranges_partition_copy.h>
-#  include <__algorithm/ranges_partition_point.h>
-#  include <__algorithm/ranges_pop_heap.h>
-#  include <__algorithm/ranges_prev_permutation.h>
-#  include <__algorithm/ranges_push_heap.h>
-#  include <__algorithm/ranges_remove.h>
-#  include <__algorithm/ranges_remove_copy.h>
-#  include <__algorithm/ranges_remove_copy_if.h>
-#  include <__algorithm/ranges_remove_if.h>
-#  include <__algorithm/ranges_replace.h>
-#  include <__algorithm/ranges_replace_copy.h>
-#  include <__algorithm/ranges_replace_copy_if.h>
-#  include <__algorithm/ranges_replace_if.h>
-#  include <__algorithm/ranges_reverse.h>
-#  include <__algorithm/ranges_reverse_copy.h>
-#  include <__algorithm/ranges_rotate.h>
-#  include <__algorithm/ranges_rotate_copy.h>
-#  include <__algorithm/ranges_sample.h>
-#  include <__algorithm/ranges_search.h>
-#  include <__algorithm/ranges_search_n.h>
-#  include <__algorithm/ranges_set_difference.h>
-#  include <__algorithm/ranges_set_intersection.h>
-#  include <__algorithm/ranges_set_symmetric_difference.h>
-#  include <__algorithm/ranges_set_union.h>
-#  include <__algorithm/ranges_shuffle.h>
-#  include <__algorithm/ranges_sort.h>
-#  include <__algorithm/ranges_sort_heap.h>
-#  include <__algorithm/ranges_stable_partition.h>
-#  include <__algorithm/ranges_stable_sort.h>
-#  include <__algorithm/ranges_swap_ranges.h>
-#  include <__algorithm/ranges_transform.h>
-#  include <__algorithm/ranges_unique.h>
-#  include <__algorithm/ranges_unique_copy.h>
-#  include <__algorithm/ranges_upper_bound.h>
-#  include <__algorithm/shift_left.h>
-#  include <__algorithm/shift_right.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__algorithm/ranges_contains_subrange.h>
-#  include <__algorithm/ranges_ends_with.h>
-#  include <__algorithm/ranges_find_last.h>
-#  include <__algorithm/ranges_fold.h>
-#  include <__algorithm/ranges_starts_with.h>
-#endif // _LIBCPP_STD_VER >= 23
-
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__algorithm/adjacent_find.h>
+#  include <__algorithm/all_of.h>
+#  include <__algorithm/any_of.h>
+#  include <__algorithm/binary_search.h>
+#  include <__algorithm/copy.h>
+#  include <__algorithm/copy_backward.h>
+#  include <__algorithm/copy_if.h>
+#  include <__algorithm/copy_n.h>
+#  include <__algorithm/count.h>
+#  include <__algorithm/count_if.h>
+#  include <__algorithm/equal.h>
+#  include <__algorithm/equal_range.h>
+#  include <__algorithm/fill.h>
+#  include <__algorithm/fill_n.h>
+#  include <__algorithm/find.h>
+#  include <__algorithm/find_end.h>
+#  include <__algorithm/find_first_of.h>
+#  include <__algorithm/find_if.h>
+#  include <__algorithm/find_if_not.h>
+#  include <__algorithm/for_each.h>
+#  include <__algorithm/generate.h>
+#  include <__algorithm/generate_n.h>
+#  include <__algorithm/includes.h>
+#  include <__algorithm/inplace_merge.h>
+#  include <__algorithm/is_heap.h>
+#  include <__algorithm/is_heap_until.h>
+#  include <__algorithm/is_partitioned.h>
+#  include <__algorithm/is_permutation.h>
+#  include <__algorithm/is_sorted.h>
+#  include <__algorithm/is_sorted_until.h>
+#  include <__algorithm/iter_swap.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lower_bound.h>
+#  include <__algorithm/make_heap.h>
+#  include <__algorithm/max.h>
+#  include <__algorithm/max_element.h>
+#  include <__algorithm/merge.h>
+#  include <__algorithm/min.h>
+#  include <__algorithm/min_element.h>
+#  include <__algorithm/minmax.h>
+#  include <__algorithm/minmax_element.h>
+#  include <__algorithm/mismatch.h>
+#  include <__algorithm/move.h>
+#  include <__algorithm/move_backward.h>
+#  include <__algorithm/next_permutation.h>
+#  include <__algorithm/none_of.h>
+#  include <__algorithm/nth_element.h>
+#  include <__algorithm/partial_sort.h>
+#  include <__algorithm/partial_sort_copy.h>
+#  include <__algorithm/partition.h>
+#  include <__algorithm/partition_copy.h>
+#  include <__algorithm/partition_point.h>
+#  include <__algorithm/pop_heap.h>
+#  include <__algorithm/prev_permutation.h>
+#  include <__algorithm/push_heap.h>
+#  include <__algorithm/remove.h>
+#  include <__algorithm/remove_copy.h>
+#  include <__algorithm/remove_copy_if.h>
+#  include <__algorithm/remove_if.h>
+#  include <__algorithm/replace.h>
+#  include <__algorithm/replace_copy.h>
+#  include <__algorithm/replace_copy_if.h>
+#  include <__algorithm/replace_if.h>
+#  include <__algorithm/reverse.h>
+#  include <__algorithm/reverse_copy.h>
+#  include <__algorithm/rotate.h>
+#  include <__algorithm/rotate_copy.h>
+#  include <__algorithm/search.h>
+#  include <__algorithm/search_n.h>
+#  include <__algorithm/set_difference.h>
+#  include <__algorithm/set_intersection.h>
+#  include <__algorithm/set_symmetric_difference.h>
+#  include <__algorithm/set_union.h>
+#  include <__algorithm/shuffle.h>
+#  include <__algorithm/sort.h>
+#  include <__algorithm/sort_heap.h>
+#  include <__algorithm/stable_partition.h>
+#  include <__algorithm/stable_sort.h>
+#  include <__algorithm/swap_ranges.h>
+#  include <__algorithm/transform.h>
+#  include <__algorithm/unique.h>
+#  include <__algorithm/unique_copy.h>
+#  include <__algorithm/upper_bound.h>
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__algorithm/clamp.h>
+#    include <__algorithm/for_each_n.h>
+#    include <__algorithm/pstl.h>
+#    include <__algorithm/sample.h>
+#  endif // _LIBCPP_STD_VER >= 17
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__algorithm/in_found_result.h>
+#    include <__algorithm/in_fun_result.h>
+#    include <__algorithm/in_in_out_result.h>
+#    include <__algorithm/in_in_result.h>
+#    include <__algorithm/in_out_out_result.h>
+#    include <__algorithm/in_out_result.h>
+#    include <__algorithm/lexicographical_compare_three_way.h>
+#    include <__algorithm/min_max_result.h>
+#    include <__algorithm/ranges_adjacent_find.h>
+#    include <__algorithm/ranges_all_of.h>
+#    include <__algorithm/ranges_any_of.h>
+#    include <__algorithm/ranges_binary_search.h>
+#    include <__algorithm/ranges_clamp.h>
+#    include <__algorithm/ranges_contains.h>
+#    include <__algorithm/ranges_copy.h>
+#    include <__algorithm/ranges_copy_backward.h>
+#    include <__algorithm/ranges_copy_if.h>
+#    include <__algorithm/ranges_copy_n.h>
+#    include <__algorithm/ranges_count.h>
+#    include <__algorithm/ranges_count_if.h>
+#    include <__algorithm/ranges_equal.h>
+#    include <__algorithm/ranges_equal_range.h>
+#    include <__algorithm/ranges_fill.h>
+#    include <__algorithm/ranges_fill_n.h>
+#    include <__algorithm/ranges_find.h>
+#    include <__algorithm/ranges_find_end.h>
+#    include <__algorithm/ranges_find_first_of.h>
+#    include <__algorithm/ranges_find_if.h>
+#    include <__algorithm/ranges_find_if_not.h>
+#    include <__algorithm/ranges_for_each.h>
+#    include <__algorithm/ranges_for_each_n.h>
+#    include <__algorithm/ranges_generate.h>
+#    include <__algorithm/ranges_generate_n.h>
+#    include <__algorithm/ranges_includes.h>
+#    include <__algorithm/ranges_inplace_merge.h>
+#    include <__algorithm/ranges_is_heap.h>
+#    include <__algorithm/ranges_is_heap_until.h>
+#    include <__algorithm/ranges_is_partitioned.h>
+#    include <__algorithm/ranges_is_permutation.h>
+#    include <__algorithm/ranges_is_sorted.h>
+#    include <__algorithm/ranges_is_sorted_until.h>
+#    include <__algorithm/ranges_lexicographical_compare.h>
+#    include <__algorithm/ranges_lower_bound.h>
+#    include <__algorithm/ranges_make_heap.h>
+#    include <__algorithm/ranges_max.h>
+#    include <__algorithm/ranges_max_element.h>
+#    include <__algorithm/ranges_merge.h>
+#    include <__algorithm/ranges_min.h>
+#    include <__algorithm/ranges_min_element.h>
+#    include <__algorithm/ranges_minmax.h>
+#    include <__algorithm/ranges_minmax_element.h>
+#    include <__algorithm/ranges_mismatch.h>
+#    include <__algorithm/ranges_move.h>
+#    include <__algorithm/ranges_move_backward.h>
+#    include <__algorithm/ranges_next_permutation.h>
+#    include <__algorithm/ranges_none_of.h>
+#    include <__algorithm/ranges_nth_element.h>
+#    include <__algorithm/ranges_partial_sort.h>
+#    include <__algorithm/ranges_partial_sort_copy.h>
+#    include <__algorithm/ranges_partition.h>
+#    include <__algorithm/ranges_partition_copy.h>
+#    include <__algorithm/ranges_partition_point.h>
+#    include <__algorithm/ranges_pop_heap.h>
+#    include <__algorithm/ranges_prev_permutation.h>
+#    include <__algorithm/ranges_push_heap.h>
+#    include <__algorithm/ranges_remove.h>
+#    include <__algorithm/ranges_remove_copy.h>
+#    include <__algorithm/ranges_remove_copy_if.h>
+#    include <__algorithm/ranges_remove_if.h>
+#    include <__algorithm/ranges_replace.h>
+#    include <__algorithm/ranges_replace_copy.h>
+#    include <__algorithm/ranges_replace_copy_if.h>
+#    include <__algorithm/ranges_replace_if.h>
+#    include <__algorithm/ranges_reverse.h>
+#    include <__algorithm/ranges_reverse_copy.h>
+#    include <__algorithm/ranges_rotate.h>
+#    include <__algorithm/ranges_rotate_copy.h>
+#    include <__algorithm/ranges_sample.h>
+#    include <__algorithm/ranges_search.h>
+#    include <__algorithm/ranges_search_n.h>
+#    include <__algorithm/ranges_set_difference.h>
+#    include <__algorithm/ranges_set_intersection.h>
+#    include <__algorithm/ranges_set_symmetric_difference.h>
+#    include <__algorithm/ranges_set_union.h>
+#    include <__algorithm/ranges_shuffle.h>
+#    include <__algorithm/ranges_sort.h>
+#    include <__algorithm/ranges_sort_heap.h>
+#    include <__algorithm/ranges_stable_partition.h>
+#    include <__algorithm/ranges_stable_sort.h>
+#    include <__algorithm/ranges_swap_ranges.h>
+#    include <__algorithm/ranges_transform.h>
+#    include <__algorithm/ranges_unique.h>
+#    include <__algorithm/ranges_unique_copy.h>
+#    include <__algorithm/ranges_upper_bound.h>
+#    include <__algorithm/shift_left.h>
+#    include <__algorithm/shift_right.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__algorithm/ranges_contains_subrange.h>
+#    include <__algorithm/ranges_ends_with.h>
+#    include <__algorithm/ranges_find_last.h>
+#    include <__algorithm/ranges_fold.h>
+#    include <__algorithm/ranges_starts_with.h>
+#  endif // _LIBCPP_STD_VER >= 23
+
+#  include <version>
 
 // standard-mandated includes
 
 // [algorithm.syn]
-#include <initializer_list>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 14
-#  include <execution>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <bit>
-#  include <concepts>
-#  include <cstdlib>
-#  include <cstring>
-#  include <iterator>
-#  include <memory>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <utility>
-#endif
+#  include <initializer_list>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 14
+#    include <execution>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <bit>
+#    include <concepts>
+#    include <cstdlib>
+#    include <cstring>
+#    include <iterator>
+#    include <memory>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_ALGORITHM
diff --git a/libcxx/include/any b/libcxx/include/any
index cae56fa376e05..c8658a468713c 100644
--- a/libcxx/include/any
+++ b/libcxx/include/any
@@ -80,41 +80,43 @@ namespace std {
 
 */
 
-#include <__config>
-#include <__memory/allocator.h>
-#include <__memory/allocator_destructor.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/unique_ptr.h>
-#include <__type_traits/add_cv_quals.h>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/aligned_storage.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_reference.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_void.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__utility/forward.h>
-#include <__utility/in_place.h>
-#include <__utility/move.h>
-#include <__utility/unreachable.h>
-#include <__verbose_abort>
-#include <initializer_list>
-#include <typeinfo>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_destructor.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/unique_ptr.h>
+#  include <__type_traits/add_cv_quals.h>
+#  include <__type_traits/add_pointer.h>
+#  include <__type_traits/aligned_storage.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/decay.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_function.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_reference.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_void.h>
+#  include <__type_traits/remove_cv.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__utility/forward.h>
+#  include <__utility/in_place.h>
+#  include <__utility/move.h>
+#  include <__utility/unreachable.h>
+#  include <__verbose_abort>
+#  include <initializer_list>
+#  include <typeinfo>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 namespace std {
 class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_ANY_CAST bad_any_cast : public bad_cast {
@@ -125,14 +127,14 @@ public:
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST void __throw_bad_any_cast() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_any_cast();
-#  else
+#    else
   _LIBCPP_VERBOSE_ABORT("bad_any_cast was thrown in -fno-exceptions mode");
-#  endif
+#    endif
 }
 
 // Forward declarations
@@ -174,10 +176,10 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr const void* __get_fallback_typeid() {
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI bool __compare_typeid(type_info const* __id, const void* __fallback_id) {
-#  if _LIBCPP_HAS_RTTI
+#    if _LIBCPP_HAS_RTTI
   if (__id && *__id == typeid(_Tp))
     return true;
-#  endif
+#    endif
   return !__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>();
 }
 
@@ -264,7 +266,7 @@ public:
   // 6.3.4 any observers
   _LIBCPP_HIDE_FROM_ABI bool has_value() const _NOEXCEPT { return __h_ != nullptr; }
 
-#  if _LIBCPP_HAS_RTTI
+#    if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI const type_info& type() const _NOEXCEPT {
     if (__h_) {
       return *static_cast<type_info const*>(this->__call(_Action::_TypeInfo));
@@ -272,7 +274,7 @@ public:
       return typeid(void);
     }
   }
-#  endif
+#    endif
 
 private:
   typedef __any_imp::_Action _Action;
@@ -370,11 +372,11 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
-#  if _LIBCPP_HAS_RTTI
+#    if _LIBCPP_HAS_RTTI
     return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
-#  else
+#    else
     return nullptr;
-#  endif
+#    endif
   }
 };
 
@@ -442,11 +444,11 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
-#  if _LIBCPP_HAS_RTTI
+#    if _LIBCPP_HAS_RTTI
     return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
-#  else
+#    else
     return nullptr;
-#  endif
+#    endif
   }
 };
 
@@ -577,37 +579,38 @@ _LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any* __any) _NOEXCEPT {
     void* __p = __any->__call(
         _Action::_Get,
         nullptr,
-#  if _LIBCPP_HAS_RTTI
+#    if _LIBCPP_HAS_RTTI
         &typeid(_ValueType),
-#  else
+#    else
         nullptr,
-#  endif
+#    endif
         __any_imp::__get_fallback_typeid<_ValueType>());
     return std::__pointer_or_func_cast<_ReturnType>(__p, is_function<_ValueType>{});
   }
   return nullptr;
 }
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
-#  include <chrono>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <iterator>
-#  include <memory>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <variant>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#    include <chrono>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <iterator>
+#    include <memory>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <variant>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_ANY
diff --git a/libcxx/include/array b/libcxx/include/array
index 99b38d049e129..b65b99533cbeb 100644
--- a/libcxx/include/array
+++ b/libcxx/include/array
@@ -111,60 +111,62 @@ template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexce
 
 */
 
-#include <__algorithm/equal.h>
-#include <__algorithm/fill_n.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__algorithm/swap_ranges.h>
-#include <__assert>
-#include <__config>
-#include <__cstddef/ptrdiff_t.h>
-#include <__fwd/array.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/static_bounded_iter.h>
-#include <__iterator/wrap_iter.h>
-#include <__tuple/sfinae_helpers.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/conjunction.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_const.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/remove_cv.h>
-#include <__utility/empty.h>
-#include <__utility/integer_sequence.h>
-#include <__utility/move.h>
-#include <__utility/unreachable.h>
-#include <stdexcept>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/equal.h>
+#  include <__algorithm/fill_n.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/swap_ranges.h>
+#  include <__assert>
+#  include <__config>
+#  include <__cstddef/ptrdiff_t.h>
+#  include <__fwd/array.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/static_bounded_iter.h>
+#  include <__iterator/wrap_iter.h>
+#  include <__tuple/sfinae_helpers.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/conjunction.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_const.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/remove_cv.h>
+#  include <__utility/empty.h>
+#  include <__utility/integer_sequence.h>
+#  include <__utility/move.h>
+#  include <__utility/unreachable.h>
+#  include <stdexcept>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [array.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
 // [tuple.helper]
-#include <__tuple/tuple_element.h>
-#include <__tuple/tuple_size.h>
+#  include <__tuple/tuple_element.h>
+#  include <__tuple/tuple_size.h>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -179,16 +181,16 @@ struct _LIBCPP_TEMPLATE_VIS array {
   using const_reference = const value_type&;
   using pointer         = value_type*;
   using const_pointer   = const value_type*;
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
   using iterator       = __static_bounded_iter<pointer, _Size>;
   using const_iterator = __static_bounded_iter<const_pointer, _Size>;
-#elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY)
+#  elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY)
   using iterator       = __wrap_iter<pointer>;
   using const_iterator = __wrap_iter<const_pointer>;
-#else
+#  else
   using iterator       = pointer;
   using const_iterator = const_pointer;
-#endif
+#  endif
   using size_type              = size_t;
   using difference_type        = ptrdiff_t;
   using reverse_iterator       = std::reverse_iterator<iterator>;
@@ -207,32 +209,32 @@ struct _LIBCPP_TEMPLATE_VIS array {
 
   // iterators:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<_Size>(data(), data());
-#else
+#  else
     return iterator(data());
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<_Size>(data(), data());
-#else
+#  else
     return const_iterator(data());
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<_Size>(data() + _Size, data());
-#else
+#  else
     return iterator(data() + _Size);
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<_Size>(data() + _Size, data());
-#else
+#  else
     return const_iterator(data() + _Size);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
@@ -302,16 +304,16 @@ struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0> {
   using const_reference = const value_type&;
   using pointer         = value_type*;
   using const_pointer   = const value_type*;
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
   using iterator       = __static_bounded_iter<pointer, 0>;
   using const_iterator = __static_bounded_iter<const_pointer, 0>;
-#elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY)
+#  elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY)
   using iterator       = __wrap_iter<pointer>;
   using const_iterator = __wrap_iter<const_pointer>;
-#else
+#  else
   using iterator       = pointer;
   using const_iterator = const_pointer;
-#endif
+#  endif
   using size_type              = size_t;
   using difference_type        = ptrdiff_t;
   using reverse_iterator       = std::reverse_iterator<iterator>;
@@ -338,32 +340,32 @@ struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0> {
 
   // iterators:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<0>(data(), data());
-#else
+#  else
     return iterator(data());
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<0>(data(), data());
-#else
+#  else
     return const_iterator(data());
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<0>(data(), data());
-#else
+#  else
     return iterator(data());
-#endif
+#  endif
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
     return std::__make_static_bounded_iter<0>(data(), data());
-#else
+#  else
     return const_iterator(data());
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
@@ -433,10 +435,10 @@ struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0> {
   }
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp, class... _Args, class = enable_if_t<__all<_IsSame<_Tp, _Args>::value...>::value> >
 array(_Tp, _Args...) -> array<_Tp, 1 + sizeof...(_Args)>;
-#endif
+#  endif
 
 template <class _Tp, size_t _Size>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
@@ -444,7 +446,7 @@ operator==(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
   return std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, size_t _Size>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
@@ -471,7 +473,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const array<_Tp, _Size>& __x, const
   return !(__x < __y);
 }
 
-#else // _LIBCPP_STD_VER <= 17
+#  else // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, size_t _Size>
 _LIBCPP_HIDE_FROM_ABI constexpr __synth_three_way_result<_Tp>
@@ -479,7 +481,7 @@ operator<=>(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) {
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, size_t _Size, __enable_if_t<_Size == 0 || __is_swappable_v<_Tp>, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(array<_Tp, _Size>& __x, array<_Tp, _Size>& __y)
@@ -520,7 +522,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&& get(const
   return std::move(__a.__elems_[_Ip]);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <typename _Tp, size_t _Size, size_t... _Index>
 _LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
@@ -550,19 +552,20 @@ to_array(_Tp (&&__arr)[_Size]) noexcept(is_nothrow_move_constructible_v<_Tp>) {
   return std::__to_array_rvalue_impl(std::move(__arr), make_index_sequence<_Size>());
 }
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iterator>
-#  include <type_traits>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iterator>
+#    include <type_traits>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_ARRAY
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index d4adf277c49c7..64635d099af92 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -587,42 +587,44 @@ template <class T>
 
 */
 
-#include <__config>
-
-#include <__atomic/aliases.h>
-#include <__atomic/atomic.h>
-#include <__atomic/atomic_flag.h>
-#include <__atomic/atomic_init.h>
-#include <__atomic/atomic_lock_free.h>
-#include <__atomic/atomic_sync.h>
-#include <__atomic/check_memory_order.h>
-#include <__atomic/contention_t.h>
-#include <__atomic/cxx_atomic_impl.h>
-#include <__atomic/fence.h>
-#include <__atomic/is_always_lock_free.h>
-#include <__atomic/kill_dependency.h>
-#include <__atomic/memory_order.h>
-#include <version>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__atomic/atomic_ref.h>
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !_LIBCPP_HAS_ATOMIC_HEADER
-#  error <atomic> is not implemented
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cmath>
-#  include <compare>
-#  include <cstddef>
-#  include <cstdlib>
-#  include <cstring>
-#  include <type_traits>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__atomic/aliases.h>
+#  include <__atomic/atomic.h>
+#  include <__atomic/atomic_flag.h>
+#  include <__atomic/atomic_init.h>
+#  include <__atomic/atomic_lock_free.h>
+#  include <__atomic/atomic_sync.h>
+#  include <__atomic/check_memory_order.h>
+#  include <__atomic/contention_t.h>
+#  include <__atomic/fence.h>
+#  include <__atomic/is_always_lock_free.h>
+#  include <__atomic/kill_dependency.h>
+#  include <__atomic/memory_order.h>
+#  include <version>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__atomic/atomic_ref.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !_LIBCPP_HAS_ATOMIC_HEADER
+#    error <atomic> is not implemented
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cmath>
+#    include <compare>
+#    include <cstddef>
+#    include <cstdlib>
+#    include <cstring>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_ATOMIC
diff --git a/libcxx/include/barrier b/libcxx/include/barrier
index 980eae06ab140..705cb7d9ac52e 100644
--- a/libcxx/include/barrier
+++ b/libcxx/include/barrier
@@ -45,30 +45,32 @@ namespace std
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_THREADS
-
-#  include <__assert>
-#  include <__atomic/atomic.h>
-#  include <__atomic/memory_order.h>
-#  include <__cstddef/ptrdiff_t.h>
-#  include <__memory/unique_ptr.h>
-#  include <__thread/poll_with_backoff.h>
-#  include <__thread/timed_backoff_policy.h>
-#  include <__utility/move.h>
-#  include <cstdint>
-#  include <limits>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_THREADS
+
+#    include <__assert>
+#    include <__atomic/atomic.h>
+#    include <__atomic/memory_order.h>
+#    include <__cstddef/ptrdiff_t.h>
+#    include <__memory/unique_ptr.h>
+#    include <__thread/poll_with_backoff.h>
+#    include <__thread/timed_backoff_policy.h>
+#    include <__utility/move.h>
+#    include <cstdint>
+#    include <limits>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -76,7 +78,7 @@ struct __empty_completion {
   inline _LIBCPP_HIDE_FROM_ABI void operator()() noexcept {}
 };
 
-#    ifndef _LIBCPP_HAS_NO_TREE_BARRIER
+#      ifndef _LIBCPP_HAS_NO_TREE_BARRIER
 
 /*
 
@@ -150,7 +152,7 @@ public:
   }
 };
 
-#    else
+#      else
 
 /*
 
@@ -251,7 +253,7 @@ public:
   }
 };
 
-#    endif // !_LIBCPP_HAS_NO_TREE_BARRIER
+#      endif // !_LIBCPP_HAS_NO_TREE_BARRIER
 
 template <class _CompletionF = __empty_completion>
 class barrier {
@@ -290,19 +292,20 @@ public:
 
 _LIBCPP_END_NAMESPACE_STD
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <iterator>
-#  include <memory>
-#  include <stdexcept>
-#  include <variant>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <iterator>
+#    include <memory>
+#    include <stdexcept>
+#    include <variant>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_BARRIER
diff --git a/libcxx/include/bit b/libcxx/include/bit
index 092aebca26a31..a5a317ebb232b 100644
--- a/libcxx/include/bit
+++ b/libcxx/include/bit
@@ -61,37 +61,40 @@ namespace std {
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__bit/bit_cast.h>
-#  include <__bit/bit_ceil.h>
-#  include <__bit/bit_floor.h>
-#  include <__bit/bit_log2.h>
-#  include <__bit/bit_width.h>
-#  include <__bit/countl.h>
-#  include <__bit/countr.h>
-#  include <__bit/endian.h>
-#  include <__bit/has_single_bit.h>
-#  include <__bit/popcount.h>
-#  include <__bit/rotate.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__bit/byteswap.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <limits>
-#  include <type_traits>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__bit/bit_cast.h>
+#    include <__bit/bit_ceil.h>
+#    include <__bit/bit_floor.h>
+#    include <__bit/bit_log2.h>
+#    include <__bit/bit_width.h>
+#    include <__bit/countl.h>
+#    include <__bit/countr.h>
+#    include <__bit/endian.h>
+#    include <__bit/has_single_bit.h>
+#    include <__bit/popcount.h>
+#    include <__bit/rotate.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__bit/byteswap.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <limits>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_BIT
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 9f14b69e7a9b4..e2dc7b459f1c4 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -126,32 +126,34 @@ template <size_t N> struct hash<std::bitset<N>>;
 
 // clang-format on
 
-#include <__algorithm/count.h>
-#include <__algorithm/fill.h>
-#include <__algorithm/fill_n.h>
-#include <__algorithm/find.h>
-#include <__bit_reference>
-#include <__config>
-#include <__functional/hash.h>
-#include <__functional/unary_function.h>
-#include <__type_traits/is_char_like_type.h>
-#include <climits>
-#include <stdexcept>
-#include <string_view>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/count.h>
+#  include <__algorithm/fill.h>
+#  include <__algorithm/fill_n.h>
+#  include <__algorithm/find.h>
+#  include <__bit_reference>
+#  include <__config>
+#  include <__functional/hash.h>
+#  include <__functional/unary_function.h>
+#  include <__type_traits/is_char_like_type.h>
+#  include <climits>
+#  include <stdexcept>
+#  include <string_view>
+#  include <version>
 
 // standard-mandated includes
 
 // [bitset.syn]
-#include <iosfwd>
-#include <string>
+#  include <iosfwd>
+#  include <string>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -222,10 +224,10 @@ protected:
   _LIBCPP_HIDE_FROM_ABI size_t __hash_code() const _NOEXCEPT;
 
 private:
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   void __init(unsigned long long __v, false_type) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI void __init(unsigned long long __v, true_type) _NOEXCEPT;
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(false_type) const;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(true_type) const;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(false_type) const;
@@ -236,16 +238,16 @@ private:
 
 template <size_t _N_words, size_t _Size>
 inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset() _NOEXCEPT
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     : __first_{0}
-#endif
+#  endif
 {
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   std::fill_n(__first_, _N_words, __storage_type(0));
-#endif
+#  endif
 }
 
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
 
 template <size_t _N_words, size_t _Size>
 void __bitset<_N_words, _Size>::__init(unsigned long long __v, false_type) _NOEXCEPT {
@@ -271,27 +273,27 @@ inline _LIBCPP_HIDE_FROM_ABI void __bitset<_N_words, _Size>::__init(unsigned lon
   std::fill(__first_ + 1, __first_ + sizeof(__first_) / sizeof(__first_[0]), __storage_type(0));
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <size_t _N_words, size_t _Size>
 inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
-#ifndef _LIBCPP_CXX03_LANG
-#  if __SIZEOF_SIZE_T__ == 8
+#  ifndef _LIBCPP_CXX03_LANG
+#    if __SIZEOF_SIZE_T__ == 8
     : __first_{__v}
-#  elif __SIZEOF_SIZE_T__ == 4
+#    elif __SIZEOF_SIZE_T__ == 4
     : __first_{static_cast<__storage_type>(__v),
                _Size >= 2 * __bits_per_word
                    ? static_cast<__storage_type>(__v >> __bits_per_word)
                    : static_cast<__storage_type>((__v >> __bits_per_word) &
                                                  (__storage_type(1) << (_Size - __bits_per_word)) - 1)}
-#  else
-#    error This constructor has not been ported to this platform
+#    else
+#      error This constructor has not been ported to this platform
+#    endif
 #  endif
-#endif
 {
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   __init(__v, integral_constant<bool, sizeof(unsigned long long) == sizeof(__storage_type)>());
-#endif
+#  endif
 }
 
 template <size_t _N_words, size_t _Size>
@@ -623,18 +625,18 @@ public:
   template <class _CharT, __enable_if_t<_IsCharLikeType<_CharT>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit bitset(
       const _CharT* __str,
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
       typename basic_string_view<_CharT>::size_type __n = basic_string_view<_CharT>::npos,
-#else
+#  else
       typename basic_string<_CharT>::size_type __n = basic_string<_CharT>::npos,
-#endif
+#  endif
       _CharT __zero = _CharT('0'),
       _CharT __one  = _CharT('1')) {
 
     size_t __rlen = std::min(__n, char_traits<_CharT>::length(__str));
     __init_from_string_view(basic_string_view<_CharT>(__str, __rlen), __zero, __one);
   }
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   template <class _CharT, class _Traits>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit bitset(
       basic_string_view<_CharT, _Traits> __str,
@@ -648,7 +650,7 @@ public:
     size_t __rlen = std::min(__n, __str.size() - __pos);
     __init_from_string_view(basic_string_view<_CharT, _Traits>(__str.data() + __pos, __rlen), __zero, __one);
   }
-#endif
+#  endif
   template <class _CharT, class _Traits, class _Allocator>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit bitset(
       const basic_string<_CharT, _Traits, _Allocator>& __str,
@@ -679,13 +681,13 @@ public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& flip(size_t __pos);
 
   // element access:
-#ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+#  ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator[](size_t __p) const { return __base::__make_ref(__p); }
-#else
+#  else
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference operator[](size_t __p) const {
     return __base::__make_ref(__p);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference operator[](size_t __p) {
     return __base::__make_ref(__p);
   }
@@ -705,9 +707,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t count() const _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_t size() const _NOEXCEPT { return _Size; }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const bitset& __rhs) const _NOEXCEPT;
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const bitset& __rhs) const _NOEXCEPT;
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool test(size_t __pos) const;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT;
@@ -880,14 +882,14 @@ bitset<_Size>::operator==(const bitset& __rhs) const _NOEXCEPT {
   return std::equal(__base::__make_iter(0), __base::__make_iter(_Size), __rhs.__make_iter(0));
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <size_t _Size>
 inline _LIBCPP_HIDE_FROM_ABI bool bitset<_Size>::operator!=(const bitset& __rhs) const _NOEXCEPT {
   return !(*this == __rhs);
 }
 
-#endif
+#  endif
 
 template <size_t _Size>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool bitset<_Size>::test(size_t __pos) const {
@@ -964,10 +966,11 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstdlib>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstdlib>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_BITSET
diff --git a/libcxx/include/cassert b/libcxx/include/cassert
index 6fec37dc63761..18e6e29353362 100644
--- a/libcxx/include/cassert
+++ b/libcxx/include/cassert
@@ -16,16 +16,19 @@ Macros:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // <assert.h> is not provided by libc++
-#if __has_include(<assert.h>)
-#  include <assert.h>
-#  ifdef _LIBCPP_ASSERT_H
-#    error "If libc++ starts defining <assert.h>, the __has_include check should move to libc++'s <assert.h>"
+#  if __has_include(<assert.h>)
+#    include <assert.h>
+#    ifdef _LIBCPP_ASSERT_H
+#      error "If libc++ starts defining <assert.h>, the __has_include check should move to libc++'s <assert.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
diff --git a/libcxx/include/ccomplex b/libcxx/include/ccomplex
index d379c9e7f0174..bd5a811b7e9fb 100644
--- a/libcxx/include/ccomplex
+++ b/libcxx/include/ccomplex
@@ -17,22 +17,25 @@
 
 */
 
-#include <complex>
+#if 0
+#else // 0
+#  include <complex>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 using __standard_header_ccomplex _LIBCPP_DEPRECATED_("removed in C++20. Include <complex> instead.") = void;
 using __use_standard_header_ccomplex = __standard_header_ccomplex;
 
-#elif _LIBCPP_STD_VER >= 17
+#  elif _LIBCPP_STD_VER >= 17
 
 using __standard_header_ccomplex _LIBCPP_DEPRECATED_("Include <complex> instead.") = void;
 using __use_standard_header_ccomplex                                               = __standard_header_ccomplex;
 
-#endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CCOMPLEX
diff --git a/libcxx/include/cctype b/libcxx/include/cctype
index d7af7e084aa23..ee3002b2cc5fe 100644
--- a/libcxx/include/cctype
+++ b/libcxx/include/cctype
@@ -34,78 +34,80 @@ int toupper(int c);
 }  // std
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <ctype.h>
+#  include <ctype.h>
 
-#ifndef _LIBCPP_CTYPE_H
+#  ifndef _LIBCPP_CTYPE_H
 #   error <cctype> tried including <ctype.h> but didn't find libc++'s <ctype.h> header. \
           This usually means that your header search paths are not configured properly.  \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifdef isalnum
-#  undef isalnum
-#endif
+#  ifdef isalnum
+#    undef isalnum
+#  endif
 
-#ifdef isalpha
-#  undef isalpha
-#endif
+#  ifdef isalpha
+#    undef isalpha
+#  endif
 
-#ifdef isblank
-#  undef isblank
-#endif
+#  ifdef isblank
+#    undef isblank
+#  endif
 
-#ifdef iscntrl
-#  undef iscntrl
-#endif
+#  ifdef iscntrl
+#    undef iscntrl
+#  endif
 
-#ifdef isdigit
-#  undef isdigit
-#endif
+#  ifdef isdigit
+#    undef isdigit
+#  endif
 
-#ifdef isgraph
-#  undef isgraph
-#endif
+#  ifdef isgraph
+#    undef isgraph
+#  endif
 
-#ifdef islower
-#  undef islower
-#endif
+#  ifdef islower
+#    undef islower
+#  endif
 
-#ifdef isprint
-#  undef isprint
-#endif
+#  ifdef isprint
+#    undef isprint
+#  endif
 
-#ifdef ispunct
-#  undef ispunct
-#endif
+#  ifdef ispunct
+#    undef ispunct
+#  endif
 
-#ifdef isspace
-#  undef isspace
-#endif
+#  ifdef isspace
+#    undef isspace
+#  endif
 
-#ifdef isupper
-#  undef isupper
-#endif
+#  ifdef isupper
+#    undef isupper
+#  endif
 
-#ifdef isxdigit
-#  undef isxdigit
-#endif
+#  ifdef isxdigit
+#    undef isxdigit
+#  endif
 
-#ifdef tolower
-#  undef tolower
-#endif
+#  ifdef tolower
+#    undef tolower
+#  endif
 
-#ifdef toupper
-#  undef toupper
-#endif
+#  ifdef toupper
+#    undef toupper
+#  endif
 
 using ::isalnum _LIBCPP_USING_IF_EXISTS;
 using ::isalpha _LIBCPP_USING_IF_EXISTS;
@@ -123,5 +125,6 @@ using ::tolower _LIBCPP_USING_IF_EXISTS;
 using ::toupper _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
+#endif // 0
 
 #endif // _LIBCPP_CCTYPE
diff --git a/libcxx/include/cerrno b/libcxx/include/cerrno
index f1295680fed6c..63017d07c019a 100644
--- a/libcxx/include/cerrno
+++ b/libcxx/include/cerrno
@@ -22,21 +22,23 @@ Macros:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <errno.h>
+#  include <errno.h>
 
-#ifndef _LIBCPP_ERRNO_H
+#  ifndef _LIBCPP_ERRNO_H
 #   error <cerrno> tried including <errno.h> but didn't find libc++'s <errno.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 // LWG3869 Deprecate std::errc constants related to UNIX STREAMS
 //
@@ -44,5 +46,6 @@ Macros:
 // deprecated in libc++ in https://github.com/llvm/llvm-project/pull/80542.
 // Based on the post commit feedback the macro are no longer deprecated.
 // Instead libc++ leaves the deprecation to the provider of errno.h.
+#endif // 0
 
 #endif // _LIBCPP_CERRNO
diff --git a/libcxx/include/cfenv b/libcxx/include/cfenv
index f8cacd562f76b..47ed700f4bee6 100644
--- a/libcxx/include/cfenv
+++ b/libcxx/include/cfenv
@@ -52,21 +52,23 @@ int feupdateenv(const fenv_t* envp);
 }  // std
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <fenv.h>
+#  include <fenv.h>
 
-#ifndef _LIBCPP_FENV_H
+#  ifndef _LIBCPP_FENV_H
 #   error <cfenv> tried including <fenv.h> but didn't find libc++'s <fenv.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -84,6 +86,7 @@ using ::fegetenv _LIBCPP_USING_IF_EXISTS;
 using ::feholdexcept _LIBCPP_USING_IF_EXISTS;
 using ::fesetenv _LIBCPP_USING_IF_EXISTS;
 using ::feupdateenv _LIBCPP_USING_IF_EXISTS;
+#endif // 0
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/cfloat b/libcxx/include/cfloat
index 5d1b38c557dca..fc37f58f4f12c 100644
--- a/libcxx/include/cfloat
+++ b/libcxx/include/cfloat
@@ -69,20 +69,23 @@ Macros:
     LDBL_TRUE_MIN       // C11
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <float.h>
+#  include <float.h>
 
-#ifndef _LIBCPP_FLOAT_H
+#  ifndef _LIBCPP_FLOAT_H
 #   error <cfloat> tried including <float.h> but didn't find libc++'s <float.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CFLOAT
diff --git a/libcxx/include/charconv b/libcxx/include/charconv
index a65b3d3527080..7415f4a5182dc 100644
--- a/libcxx/include/charconv
+++ b/libcxx/include/charconv
@@ -75,42 +75,45 @@ namespace std {
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__charconv/chars_format.h>
-#  include <__charconv/from_chars_floating_point.h>
-#  include <__charconv/from_chars_integral.h>
-#  include <__charconv/from_chars_result.h>
-#  include <__charconv/tables.h>
-#  include <__charconv/to_chars.h>
-#  include <__charconv/to_chars_base_10.h>
-#  include <__charconv/to_chars_floating_point.h>
-#  include <__charconv/to_chars_integral.h>
-#  include <__charconv/to_chars_result.h>
-#  include <__charconv/traits.h>
-#endif // _LIBCPP_STD_VER >= 17
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__charconv/chars_format.h>
+#    include <__charconv/from_chars_floating_point.h>
+#    include <__charconv/from_chars_integral.h>
+#    include <__charconv/from_chars_result.h>
+#    include <__charconv/tables.h>
+#    include <__charconv/to_chars.h>
+#    include <__charconv/to_chars_base_10.h>
+#    include <__charconv/to_chars_floating_point.h>
+#    include <__charconv/to_chars_integral.h>
+#    include <__charconv/to_chars_result.h>
+#    include <__charconv/traits.h>
+#  endif // _LIBCPP_STD_VER >= 17
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cmath>
-#  include <concepts>
-#  include <cstddef>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <cstring>
-#  include <iosfwd>
-#  include <limits>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cmath>
+#    include <concepts>
+#    include <cstddef>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <cstring>
+#    include <iosfwd>
+#    include <limits>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CHARCONV
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 42da1b4ab9756..6654a6776d397 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -939,84 +939,86 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 
 // clang-format on
 
-#include <__config>
-
-#include <__chrono/duration.h>
-#include <__chrono/file_clock.h>
-#include <__chrono/high_resolution_clock.h>
-#include <__chrono/steady_clock.h>
-#include <__chrono/system_clock.h>
-#include <__chrono/time_point.h>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__chrono/calendar.h>
-#  include <__chrono/day.h>
-#  include <__chrono/exception.h>
-#  include <__chrono/hh_mm_ss.h>
-#  include <__chrono/literals.h>
-#  include <__chrono/local_info.h>
-#  include <__chrono/month.h>
-#  include <__chrono/month_weekday.h>
-#  include <__chrono/monthday.h>
-#  include <__chrono/sys_info.h>
-#  include <__chrono/weekday.h>
-#  include <__chrono/year.h>
-#  include <__chrono/year_month.h>
-#  include <__chrono/year_month_day.h>
-#  include <__chrono/year_month_weekday.h>
-
-#  if _LIBCPP_HAS_LOCALIZATION
-#    include <__chrono/formatter.h>
-#    include <__chrono/ostream.h>
-#    include <__chrono/parser_std_format_spec.h>
-#    include <__chrono/statically_widen.h>
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__chrono/duration.h>
+#  include <__chrono/file_clock.h>
+#  include <__chrono/high_resolution_clock.h>
+#  include <__chrono/steady_clock.h>
+#  include <__chrono/system_clock.h>
+#  include <__chrono/time_point.h>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__chrono/calendar.h>
+#    include <__chrono/day.h>
+#    include <__chrono/exception.h>
+#    include <__chrono/hh_mm_ss.h>
+#    include <__chrono/literals.h>
+#    include <__chrono/local_info.h>
+#    include <__chrono/month.h>
+#    include <__chrono/month_weekday.h>
+#    include <__chrono/monthday.h>
+#    include <__chrono/sys_info.h>
+#    include <__chrono/weekday.h>
+#    include <__chrono/year.h>
+#    include <__chrono/year_month.h>
+#    include <__chrono/year_month_day.h>
+#    include <__chrono/year_month_weekday.h>
+
+#    if _LIBCPP_HAS_LOCALIZATION
+#      include <__chrono/formatter.h>
+#      include <__chrono/ostream.h>
+#      include <__chrono/parser_std_format_spec.h>
+#      include <__chrono/statically_widen.h>
+#    endif
+
+#    if _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
+#      include <__chrono/leap_second.h>
+#      include <__chrono/time_zone.h>
+#      include <__chrono/time_zone_link.h>
+#      include <__chrono/tzdb.h>
+#      include <__chrono/tzdb_list.h>
+#      include <__chrono/zoned_time.h>
+#    endif
 
-#  if _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
-#    include <__chrono/leap_second.h>
-#    include <__chrono/time_zone.h>
-#    include <__chrono/time_zone_link.h>
-#    include <__chrono/tzdb.h>
-#    include <__chrono/tzdb_list.h>
-#    include <__chrono/zoned_time.h>
 #  endif
 
-#endif
-
-#include <version>
+#  include <version>
 
 // standard-mandated includes
 
 // [time.syn]
-#include <compare>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
-#  include <cstdint>
-#  include <stdexcept>
-#  include <string_view>
-#  include <vector>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <bit>
-#  include <concepts>
-#  include <cstring>
-#  include <forward_list>
-#  include <string>
-#  include <tuple>
-#  include <vector>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 20
-#  include <charconv>
-#  if _LIBCPP_HAS_LOCALIZATION
-#    include <locale>
-#    include <ostream>
+#  include <compare>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#    include <cstdint>
+#    include <stdexcept>
+#    include <string_view>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <bit>
+#    include <concepts>
+#    include <cstring>
+#    include <forward_list>
+#    include <string>
+#    include <tuple>
+#    include <vector>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 20
+#    include <charconv>
+#    if _LIBCPP_HAS_LOCALIZATION
+#      include <locale>
+#      include <ostream>
+#    endif
 #  endif
-#endif
+#endif // 0
 
 #endif // _LIBCPP_CHRONO
diff --git a/libcxx/include/cinttypes b/libcxx/include/cinttypes
index 52663a4f35fad..053067d4d6250 100644
--- a/libcxx/include/cinttypes
+++ b/libcxx/include/cinttypes
@@ -234,26 +234,28 @@ uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int
 }  // std
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // standard-mandated includes
 
 // [cinttypes.syn]
-#include <cstdint>
+#  include <cstdint>
 
-#include <inttypes.h>
+#  include <inttypes.h>
 
-#ifndef _LIBCPP_INTTYPES_H
+#  ifndef _LIBCPP_INTTYPES_H
 #   error <cinttypes> tried including <inttypes.h> but didn't find libc++'s <inttypes.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -267,4 +269,6 @@ using ::wcstoumax _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CINTTYPES
diff --git a/libcxx/include/ciso646 b/libcxx/include/ciso646
index 5fcac79e38a7f..965b78741a0b4 100644
--- a/libcxx/include/ciso646
+++ b/libcxx/include/ciso646
@@ -15,17 +15,20 @@
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 using __standard_header_ciso646 _LIBCPP_DEPRECATED_("removed in C++20. Include <version> instead.") = void;
 using __use_standard_header_ciso646 = __standard_header_ciso646;
 
-#endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CISO646
diff --git a/libcxx/include/climits b/libcxx/include/climits
index bcd8b4a56a073..b31f77d9e859d 100644
--- a/libcxx/include/climits
+++ b/libcxx/include/climits
@@ -37,12 +37,16 @@ Macros:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <limits.h>
+#  include <limits.h>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#endif // 0
 
 #endif // _LIBCPP_CLIMITS
diff --git a/libcxx/include/clocale b/libcxx/include/clocale
index a82a166712c26..4199487a3992f 100644
--- a/libcxx/include/clocale
+++ b/libcxx/include/clocale
@@ -34,15 +34,17 @@ lconv* localeconv();
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if __has_include(<locale.h>)
-#  include <locale.h>
-#endif
+#  if __has_include(<locale.h>)
+#    include <locale.h>
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -52,4 +54,6 @@ using ::localeconv _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CLOCALE
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 5d30b151870e0..ff8e7e1bf1481 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -312,35 +312,37 @@ constexpr long double lerp(long double a, long double b, long double t) noexcept
 
 */
 
-#include <__config>
-#include <__math/hypot.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_arithmetic.h>
-#include <__type_traits/is_constant_evaluated.h>
-#include <__type_traits/is_floating_point.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/promote.h>
-#include <__type_traits/remove_cv.h>
-#include <limits>
-#include <version>
-
-#include <__math/special_functions.h>
-#include <math.h>
-
-#ifndef _LIBCPP_MATH_H
+#if 0
+#else // 0
+#  include <__config>
+#  include <__math/hypot.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_arithmetic.h>
+#  include <__type_traits/is_constant_evaluated.h>
+#  include <__type_traits/is_floating_point.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/promote.h>
+#  include <__type_traits/remove_cv.h>
+#  include <limits>
+#  include <version>
+
+#  include <__math/special_functions.h>
+#  include <math.h>
+
+#  ifndef _LIBCPP_MATH_H
 #   error <cmath> tried including <math.h> but didn't find libc++'s <math.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -556,11 +558,11 @@ using ::truncl _LIBCPP_USING_IF_EXISTS;
 
 template <class _A1, __enable_if_t<is_floating_point<_A1>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isinf(_A1 __lcpp_x) _NOEXCEPT {
-#if __has_builtin(__builtin_isinf)
+#  if __has_builtin(__builtin_isinf)
   return __builtin_isinf(__lcpp_x);
-#else
+#  else
   return isinf(__lcpp_x);
-#endif
+#  endif
 }
 
 template <class _A1, __enable_if_t<!is_floating_point<_A1>::value, int> = 0>
@@ -568,7 +570,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isinf(_A1 __lcpp_x) _NO
   return std::isinf(__lcpp_x);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <typename _Fp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Fp __lerp(_Fp __a, _Fp __b, _Fp __t) noexcept {
   if ((__a <= 0 && __b >= 0) || (__a >= 0 && __b <= 0))
@@ -605,14 +607,15 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr
       _IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value && _IsSame<_A3, __result_type>::value));
   return std::__lerp((__result_type)__a, (__result_type)__b, (__result_type)__t);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CMATH
diff --git a/libcxx/include/codecvt b/libcxx/include/codecvt
index 8e0b52576d9c4..ce6d995f37aba 100644
--- a/libcxx/include/codecvt
+++ b/libcxx/include/codecvt
@@ -54,15 +54,17 @@ class codecvt_utf8_utf16
 
 */
 
-#include <__config>
-#include <__locale>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__locale>
+#  include <version>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -73,7 +75,7 @@ enum _LIBCPP_DEPRECATED_IN_CXX17 codecvt_mode { consume_header = 4, generate_hea
 template <class _Elem>
 class __codecvt_utf8;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8<wchar_t> : public codecvt<wchar_t, char, mbstate_t> {
   unsigned long __maxcode_;
@@ -112,7 +114,7 @@ protected:
   int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
   int do_max_length() const _NOEXCEPT override;
 };
-#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#    endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
@@ -203,7 +205,7 @@ _LIBCPP_SUPPRESS_DEPRECATED_POP
 template <class _Elem, bool _LittleEndian>
 class __codecvt_utf16;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf16<wchar_t, false> : public codecvt<wchar_t, char, mbstate_t> {
   unsigned long __maxcode_;
@@ -281,7 +283,7 @@ protected:
   int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
   int do_max_length() const _NOEXCEPT override;
 };
-#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#    endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
@@ -448,7 +450,7 @@ _LIBCPP_SUPPRESS_DEPRECATED_POP
 template <class _Elem>
 class __codecvt_utf8_utf16;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 class _LIBCPP_EXPORTED_FROM_ABI __codecvt_utf8_utf16<wchar_t> : public codecvt<wchar_t, char, mbstate_t> {
   unsigned long __maxcode_;
@@ -487,7 +489,7 @@ protected:
   int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const override;
   int do_max_length() const _NOEXCEPT override;
 };
-#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#    endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
@@ -576,22 +578,23 @@ _LIBCPP_SUPPRESS_DEPRECATED_POP
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstddef>
-#  include <cstdlib>
-#  include <cstring>
-#  include <initializer_list>
-#  include <iosfwd>
-#  include <limits>
-#  include <mutex>
-#  include <new>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCPP_ENABLE_CXX26_REMOVED_CODECVT)
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstddef>
+#    include <cstdlib>
+#    include <cstring>
+#    include <initializer_list>
+#    include <iosfwd>
+#    include <limits>
+#    include <mutex>
+#    include <new>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CODECVT
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 440d4c4b4dd26..97871e679bf32 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -140,34 +140,37 @@ namespace std {
 }
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__compare/common_comparison_category.h>
-#  include <__compare/compare_partial_order_fallback.h>
-#  include <__compare/compare_strong_order_fallback.h>
-#  include <__compare/compare_three_way.h>
-#  include <__compare/compare_three_way_result.h>
-#  include <__compare/compare_weak_order_fallback.h>
-#  include <__compare/is_eq.h>
-#  include <__compare/ordering.h>
-#  include <__compare/partial_order.h>
-#  include <__compare/strong_order.h>
-#  include <__compare/synth_three_way.h>
-#  include <__compare/three_way_comparable.h>
-#  include <__compare/weak_order.h>
-#endif // _LIBCPP_STD_VER >= 20
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cmath>
-#  include <cstddef>
-#  include <type_traits>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__compare/common_comparison_category.h>
+#    include <__compare/compare_partial_order_fallback.h>
+#    include <__compare/compare_strong_order_fallback.h>
+#    include <__compare/compare_three_way.h>
+#    include <__compare/compare_three_way_result.h>
+#    include <__compare/compare_weak_order_fallback.h>
+#    include <__compare/is_eq.h>
+#    include <__compare/ordering.h>
+#    include <__compare/partial_order.h>
+#    include <__compare/strong_order.h>
+#    include <__compare/synth_three_way.h>
+#    include <__compare/three_way_comparable.h>
+#    include <__compare/weak_order.h>
+#  endif // _LIBCPP_STD_VER >= 20
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cmath>
+#    include <cstddef>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_COMPARE
diff --git a/libcxx/include/complex b/libcxx/include/complex
index 2dc6d298e44f6..bfc27bb383416 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -256,26 +256,28 @@ template<class T> complex<T> tanh (const complex<T>&);
 
 */
 
-#include <__config>
-#include <__fwd/complex.h>
-#include <__fwd/tuple.h>
-#include <__tuple/tuple_element.h>
-#include <__tuple/tuple_size.h>
-#include <__type_traits/conditional.h>
-#include <__utility/move.h>
-#include <cmath>
-#include <version>
-
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <sstream> // for std::basic_ostringstream
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__fwd/complex.h>
+#  include <__fwd/tuple.h>
+#  include <__tuple/tuple_element.h>
+#  include <__tuple/tuple_size.h>
+#  include <__type_traits/conditional.h>
+#  include <__utility/move.h>
+#  include <cmath>
+#  include <version>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+#    include <sstream> // for std::basic_ostringstream
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -374,7 +376,7 @@ public:
     return *this;
   }
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
 
@@ -386,7 +388,7 @@ public:
 
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
-#endif
+#  endif
 };
 
 template <>
@@ -404,11 +406,11 @@ using __complex_t =
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __complex_t<_Tp> __make_complex(_Tp __re, _Tp __im) {
-#if __has_builtin(__builtin_complex)
+#  if __has_builtin(__builtin_complex)
   return __builtin_complex(__re, __im);
-#else
+#  else
   return __complex_t<_Tp>{__re, __im};
-#endif
+#  endif
 }
 
 template <>
@@ -493,7 +495,7 @@ public:
     return *this;
   }
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
 
@@ -505,7 +507,7 @@ public:
 
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
-#endif
+#  endif
 };
 
 template <>
@@ -593,7 +595,7 @@ public:
     return *this;
   }
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
 
@@ -605,7 +607,7 @@ public:
 
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
-#endif
+#  endif
 };
 
 template <>
@@ -694,7 +696,7 @@ public:
     return *this;
   }
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>&) noexcept;
 
@@ -706,7 +708,7 @@ public:
 
   template <size_t _Ip, class _Xp>
   friend _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&&) noexcept;
-#endif
+#  endif
 };
 
 inline _LIBCPP_CONSTEXPR complex<float>::complex(const complex<double>& __c) : __re_(__c.real()), __im_(__c.imag()) {}
@@ -861,7 +863,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator==(const
   return __x.real() == __y && __x.imag() == 0;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator==(const _Tp& __x, const complex<_Tp>& __y) {
@@ -884,7 +886,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator!=(const
   return !(__x == __y);
 }
 
-#endif
+#  endif
 
 // 26.3.7 values:
 
@@ -1322,7 +1324,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> tan(const complex<_Tp>& __x) {
   return complex<_Tp>(__z.imag(), -__z.real());
 }
 
-#if _LIBCPP_HAS_LOCALIZATION
+#  if _LIBCPP_HAS_LOCALIZATION
 template <class _Tp, class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 operator>>(basic_istream<_CharT, _Traits>& __is, complex<_Tp>& __x) {
@@ -1379,9 +1381,9 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const complex<_Tp>& __x) {
   __s << '(' << __x.real() << ',' << __x.imag() << ')';
   return __os << __s.str();
 }
-#endif // _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_LOCALIZATION
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
 
 // [complex.tuple], tuple interface
 
@@ -1434,9 +1436,9 @@ _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&& __z) noexce
   }
 }
 
-#endif // _LIBCPP_STD_VER >= 26
+#  endif // _LIBCPP_STD_VER >= 26
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 // Literal suffix for complex number literals [complex.literals]
 inline namespace literals {
 inline namespace complex_literals {
@@ -1463,16 +1465,17 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr complex<float> operator""if(unsigned long
 }
 } // namespace complex_literals
 } // namespace literals
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <iosfwd>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <iosfwd>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_COMPLEX
diff --git a/libcxx/include/complex.h b/libcxx/include/complex.h
index 89595ae2068a6..5bc7ab435ca72 100644
--- a/libcxx/include/complex.h
+++ b/libcxx/include/complex.h
@@ -17,16 +17,19 @@
 
 */
 
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#ifdef __cplusplus
-#  include <complex>
-#elif __has_include_next(<complex.h>)
-#  include_next <complex.h>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  ifdef __cplusplus
+#    include <complex>
+#  elif __has_include_next(<complex.h>)
+#    include_next <complex.h>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_COMPLEX_H
diff --git a/libcxx/include/concepts b/libcxx/include/concepts
index 6db2caebaf5b1..2fe5400c2c715 100644
--- a/libcxx/include/concepts
+++ b/libcxx/include/concepts
@@ -129,42 +129,45 @@ namespace std {
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__concepts/arithmetic.h>
-#  include <__concepts/assignable.h>
-#  include <__concepts/boolean_testable.h>
-#  include <__concepts/class_or_enum.h>
-#  include <__concepts/common_reference_with.h>
-#  include <__concepts/common_with.h>
-#  include <__concepts/constructible.h>
-#  include <__concepts/convertible_to.h>
-#  include <__concepts/copyable.h>
-#  include <__concepts/derived_from.h>
-#  include <__concepts/destructible.h>
-#  include <__concepts/different_from.h>
-#  include <__concepts/equality_comparable.h>
-#  include <__concepts/invocable.h>
-#  include <__concepts/movable.h>
-#  include <__concepts/predicate.h>
-#  include <__concepts/regular.h>
-#  include <__concepts/relation.h>
-#  include <__concepts/same_as.h>
-#  include <__concepts/semiregular.h>
-#  include <__concepts/swappable.h>
-#  include <__concepts/totally_ordered.h>
-#endif // _LIBCPP_STD_VER >= 20
-
-#include <version>
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <type_traits>
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__concepts/arithmetic.h>
+#    include <__concepts/assignable.h>
+#    include <__concepts/boolean_testable.h>
+#    include <__concepts/class_or_enum.h>
+#    include <__concepts/common_reference_with.h>
+#    include <__concepts/common_with.h>
+#    include <__concepts/constructible.h>
+#    include <__concepts/convertible_to.h>
+#    include <__concepts/copyable.h>
+#    include <__concepts/derived_from.h>
+#    include <__concepts/destructible.h>
+#    include <__concepts/different_from.h>
+#    include <__concepts/equality_comparable.h>
+#    include <__concepts/invocable.h>
+#    include <__concepts/movable.h>
+#    include <__concepts/predicate.h>
+#    include <__concepts/regular.h>
+#    include <__concepts/relation.h>
+#    include <__concepts/same_as.h>
+#    include <__concepts/semiregular.h>
+#    include <__concepts/swappable.h>
+#    include <__concepts/totally_ordered.h>
+#  endif // _LIBCPP_STD_VER >= 20
+
+#  include <version>
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <type_traits>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CONCEPTS
diff --git a/libcxx/include/condition_variable b/libcxx/include/condition_variable
index f894407d9321b..f351f13b71389 100644
--- a/libcxx/include/condition_variable
+++ b/libcxx/include/condition_variable
@@ -118,29 +118,31 @@ public:
 
 */
 
-#include <__chrono/duration.h>
-#include <__chrono/steady_clock.h>
-#include <__chrono/time_point.h>
-#include <__condition_variable/condition_variable.h>
-#include <__config>
-#include <__memory/shared_ptr.h>
-#include <__mutex/lock_guard.h>
-#include <__mutex/mutex.h>
-#include <__mutex/tag_types.h>
-#include <__mutex/unique_lock.h>
-#include <__stop_token/stop_callback.h>
-#include <__stop_token/stop_token.h>
-#include <__utility/move.h>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__chrono/duration.h>
+#  include <__chrono/steady_clock.h>
+#  include <__chrono/time_point.h>
+#  include <__condition_variable/condition_variable.h>
+#  include <__config>
+#  include <__memory/shared_ptr.h>
+#  include <__mutex/lock_guard.h>
+#  include <__mutex/mutex.h>
+#  include <__mutex/tag_types.h>
+#  include <__mutex/unique_lock.h>
+#  include <__stop_token/stop_callback.h>
+#  include <__stop_token/stop_token.h>
+#  include <__utility/move.h>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
-#if _LIBCPP_HAS_THREADS
+#  if _LIBCPP_HAS_THREADS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -173,7 +175,7 @@ public:
   template <class _Lock, class _Rep, class _Period, class _Predicate>
   bool _LIBCPP_HIDE_FROM_ABI wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred);
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
   template <class _Lock, class _Predicate>
   _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool wait(_Lock& __lock, stop_token __stoken, _Predicate __pred);
@@ -186,7 +188,7 @@ public:
   _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
   wait_for(_Lock& __lock, stop_token __stoken, const chrono::duration<_Rep, _Period>& __rel_time, _Predicate __pred);
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 };
 
 inline condition_variable_any::condition_variable_any() : __mut_(make_shared<mutex>()) {}
@@ -260,7 +262,7 @@ condition_variable_any::wait_for(_Lock& __lock, const chrono::duration<_Rep, _Pe
   return wait_until(__lock, chrono::steady_clock::now() + __d, std::move(__pred));
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class _Lock, class _Predicate>
 bool condition_variable_any::wait(_Lock& __user_lock, stop_token __stoken, _Predicate __pred) {
@@ -341,29 +343,30 @@ bool condition_variable_any::wait_for(
   return wait_until(__lock, std::move(__stoken), chrono::steady_clock::now() + __rel_time, std::move(__pred));
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_EXPORTED_FROM_ABI void notify_all_at_thread_exit(condition_variable&, unique_lock<mutex>);
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <cstring>
-#  include <initializer_list>
-#  include <iosfwd>
-#  include <new>
-#  include <stdexcept>
-#  include <system_error>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <cstring>
+#    include <initializer_list>
+#    include <iosfwd>
+#    include <new>
+#    include <stdexcept>
+#    include <system_error>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CONDITION_VARIABLE
diff --git a/libcxx/include/coroutine b/libcxx/include/coroutine
index 18601717768c0..c39aa9a3da0ce 100644
--- a/libcxx/include/coroutine
+++ b/libcxx/include/coroutine
@@ -38,31 +38,34 @@ struct suspend_always;
 
  */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_STD_VER >= 20
-#  include <__coroutine/coroutine_handle.h>
-#  include <__coroutine/coroutine_traits.h>
-#  include <__coroutine/noop_coroutine_handle.h>
-#  include <__coroutine/trivial_awaitables.h>
-#endif // _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
+#    include <__coroutine/coroutine_handle.h>
+#    include <__coroutine/coroutine_traits.h>
+#    include <__coroutine/noop_coroutine_handle.h>
+#    include <__coroutine/trivial_awaitables.h>
+#  endif // _LIBCPP_STD_VER >= 20
 
-#include <version>
+#  include <version>
 
 // standard-mandated includes
 
 // [coroutine.syn]
-#include <compare>
+#  include <compare>
 
-#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
-#  pragma GCC system_header
-#endif
+#  ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
+#    pragma GCC system_header
+#  endif
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <iosfwd>
-#  include <limits>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <iosfwd>
+#    include <limits>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_COROUTINE
diff --git a/libcxx/include/csetjmp b/libcxx/include/csetjmp
index 7ba90068710ae..bd512720be008 100644
--- a/libcxx/include/csetjmp
+++ b/libcxx/include/csetjmp
@@ -30,19 +30,21 @@ void longjmp(jmp_buf env, int val);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // <setjmp.h> is not provided by libc++
-#if __has_include(<setjmp.h>)
-#  include <setjmp.h>
-#  ifdef _LIBCPP_SETJMP_H
-#    error "If libc++ starts defining <setjmp.h>, the __has_include check should move to libc++'s <setjmp.h>"
+#  if __has_include(<setjmp.h>)
+#    include <setjmp.h>
+#    ifdef _LIBCPP_SETJMP_H
+#      error "If libc++ starts defining <setjmp.h>, the __has_include check should move to libc++'s <setjmp.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -51,4 +53,6 @@ using ::longjmp _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSETJMP
diff --git a/libcxx/include/csignal b/libcxx/include/csignal
index 804a7f95ae968..92edf08f683e7 100644
--- a/libcxx/include/csignal
+++ b/libcxx/include/csignal
@@ -39,19 +39,21 @@ int raise(int sig);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // <signal.h> is not provided by libc++
-#if __has_include(<signal.h>)
-#  include <signal.h>
-#  ifdef _LIBCPP_SIGNAL_H
-#    error "If libc++ starts defining <signal.h>, the __has_include check should move to libc++'s <signal.h>"
+#  if __has_include(<signal.h>)
+#    include <signal.h>
+#    ifdef _LIBCPP_SIGNAL_H
+#      error "If libc++ starts defining <signal.h>, the __has_include check should move to libc++'s <signal.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -61,4 +63,6 @@ using ::raise _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSIGNAL
diff --git a/libcxx/include/cstdalign b/libcxx/include/cstdalign
index e6a2a3c717742..ab78344cd36c6 100644
--- a/libcxx/include/cstdalign
+++ b/libcxx/include/cstdalign
@@ -20,36 +20,39 @@ Macros:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // <stdalign.h> is not provided by libc++
-#if __has_include(<stdalign.h>)
-#  include <stdalign.h>
-#  ifdef _LIBCPP_STDALIGN_H
-#    error "If libc++ starts defining <stdalign.h>, the __has_include check should move to libc++'s <stdalign.h>"
+#  if __has_include(<stdalign.h>)
+#    include <stdalign.h>
+#    ifdef _LIBCPP_STDALIGN_H
+#      error "If libc++ starts defining <stdalign.h>, the __has_include check should move to libc++'s <stdalign.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#undef __alignas_is_defined
-#define __alignas_is_defined 1
+#  undef __alignas_is_defined
+#  define __alignas_is_defined 1
 
-#undef __alignof_is_defined
-#define __alignof_is_defined 1
+#  undef __alignof_is_defined
+#  define __alignof_is_defined 1
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 using __standard_header_cstdalign _LIBCPP_DEPRECATED_("removed in C++20.") = void;
 using __use_standard_header_cstdalign                                      = __standard_header_cstdalign;
 
-#elif _LIBCPP_STD_VER >= 17
+#  elif _LIBCPP_STD_VER >= 17
 
 using __standard_header_cstdalign _LIBCPP_DEPRECATED = void;
 using __use_standard_header_cstdalign                = __standard_header_cstdalign;
 
-#endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CSTDALIGN
diff --git a/libcxx/include/cstdarg b/libcxx/include/cstdarg
index 4642eb7b5258c..1930fc95cf851 100644
--- a/libcxx/include/cstdarg
+++ b/libcxx/include/cstdarg
@@ -31,19 +31,21 @@ Types:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
 // <stdarg.h> is not provided by libc++
-#if __has_include(<stdarg.h>)
-#  include <stdarg.h>
-#  ifdef _LIBCPP_STDARG_H
-#    error "If libc++ starts defining <stdarg.h>, the __has_include check should move to libc++'s <stdarg.h>"
+#  if __has_include(<stdarg.h>)
+#    include <stdarg.h>
+#    ifdef _LIBCPP_STDARG_H
+#      error "If libc++ starts defining <stdarg.h>, the __has_include check should move to libc++'s <stdarg.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -51,4 +53,6 @@ using ::va_list _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSTDARG
diff --git a/libcxx/include/cstdbool b/libcxx/include/cstdbool
index 1d627258e10c0..a436c5c851a84 100644
--- a/libcxx/include/cstdbool
+++ b/libcxx/include/cstdbool
@@ -19,25 +19,28 @@ Macros:
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#undef __bool_true_false_are_defined
-#define __bool_true_false_are_defined 1
+#  undef __bool_true_false_are_defined
+#  define __bool_true_false_are_defined 1
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 using __standard_header_cstdbool _LIBCPP_DEPRECATED_("removed in C++20.") = void;
 using __use_standard_header_cstdbool                                      = __standard_header_cstdbool;
 
-#elif _LIBCPP_STD_VER >= 17
+#  elif _LIBCPP_STD_VER >= 17
 
 using __standard_header_cstdbool _LIBCPP_DEPRECATED = void;
 using __use_standard_header_cstdbool                = __standard_header_cstdbool;
 
-#endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CSTDBOOL
diff --git a/libcxx/include/cstddef b/libcxx/include/cstddef
index 2b138d9690393..2e79d05243666 100644
--- a/libcxx/include/cstddef
+++ b/libcxx/include/cstddef
@@ -33,27 +33,30 @@ Types:
 
 */
 
-#include <__config>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <version>
 
-#include <stddef.h>
+#  include <stddef.h>
 
-#ifndef _LIBCPP_STDDEF_H
+#  ifndef _LIBCPP_STDDEF_H
 #   error <cstddef> tried including <stddef.h> but didn't find libc++'s <stddef.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#include <__cstddef/byte.h>
-#include <__cstddef/max_align_t.h>
-#include <__cstddef/nullptr_t.h>
-#include <__cstddef/ptrdiff_t.h>
-#include <__cstddef/size_t.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  include <__cstddef/byte.h>
+#  include <__cstddef/max_align_t.h>
+#  include <__cstddef/nullptr_t.h>
+#  include <__cstddef/ptrdiff_t.h>
+#  include <__cstddef/size_t.h>
+#endif // 0
 
 #endif // _LIBCPP_CSTDDEF
diff --git a/libcxx/include/cstdint b/libcxx/include/cstdint
index b3167c33f2477..594478930500c 100644
--- a/libcxx/include/cstdint
+++ b/libcxx/include/cstdint
@@ -140,15 +140,17 @@ Types:
 }  // std
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if __has_include(<stdint.h>)
-#  include <stdint.h>
-#endif
+#  if __has_include(<stdint.h>)
+#    include <stdint.h>
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -190,4 +192,6 @@ using ::uintmax_t _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSTDINT
diff --git a/libcxx/include/cstdio b/libcxx/include/cstdio
index a461c24dcc019..1b1a05d264b37 100644
--- a/libcxx/include/cstdio
+++ b/libcxx/include/cstdio
@@ -95,22 +95,24 @@ void perror(const char* s);
 }  // std
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
 
-#include <stdio.h>
+#  include <stdio.h>
 
-#ifndef _LIBCPP_STDIO_H
+#  ifndef _LIBCPP_STDIO_H
 #   error <cstdio> tried including <stdio.h> but didn't find libc++'s <stdio.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -158,9 +160,9 @@ using ::tmpfile _LIBCPP_USING_IF_EXISTS;
 using ::tmpnam _LIBCPP_USING_IF_EXISTS;
 
 using ::getchar _LIBCPP_USING_IF_EXISTS;
-#if _LIBCPP_STD_VER <= 11
+#  if _LIBCPP_STD_VER <= 11
 using ::gets _LIBCPP_USING_IF_EXISTS;
-#endif
+#  endif
 using ::scanf _LIBCPP_USING_IF_EXISTS;
 using ::vscanf _LIBCPP_USING_IF_EXISTS;
 
@@ -171,4 +173,6 @@ using ::vprintf _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSTDIO
diff --git a/libcxx/include/cstdlib b/libcxx/include/cstdlib
index be0d9b8d7227a..03e68957f71e4 100644
--- a/libcxx/include/cstdlib
+++ b/libcxx/include/cstdlib
@@ -81,22 +81,24 @@ void *aligned_alloc(size_t alignment, size_t size);                       // C11
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
 
-#include <stdlib.h>
+#  include <stdlib.h>
 
-#ifndef _LIBCPP_STDLIB_H
+#  ifndef _LIBCPP_STDLIB_H
 #   error <cstdlib> tried including <stdlib.h> but didn't find libc++'s <stdlib.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -135,20 +137,22 @@ using ::div _LIBCPP_USING_IF_EXISTS;
 using ::ldiv _LIBCPP_USING_IF_EXISTS;
 using ::lldiv _LIBCPP_USING_IF_EXISTS;
 using ::mblen _LIBCPP_USING_IF_EXISTS;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 using ::mbtowc _LIBCPP_USING_IF_EXISTS;
 using ::wctomb _LIBCPP_USING_IF_EXISTS;
 using ::mbstowcs _LIBCPP_USING_IF_EXISTS;
 using ::wcstombs _LIBCPP_USING_IF_EXISTS;
-#endif
-#if !defined(_LIBCPP_CXX03_LANG)
+#  endif
+#  if !defined(_LIBCPP_CXX03_LANG)
 using ::at_quick_exit _LIBCPP_USING_IF_EXISTS;
 using ::quick_exit _LIBCPP_USING_IF_EXISTS;
-#endif
-#if _LIBCPP_STD_VER >= 17
+#  endif
+#  if _LIBCPP_STD_VER >= 17
 using ::aligned_alloc _LIBCPP_USING_IF_EXISTS;
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSTDLIB
diff --git a/libcxx/include/cstring b/libcxx/include/cstring
index 5bb6e3e10628f..f414885864106 100644
--- a/libcxx/include/cstring
+++ b/libcxx/include/cstring
@@ -56,23 +56,25 @@ size_t strlen(const char* s);
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
-#include <__type_traits/is_constant_evaluated.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
+#  include <__type_traits/is_constant_evaluated.h>
 
-#include <string.h>
+#  include <string.h>
 
-#ifndef _LIBCPP_STRING_H
+#  ifndef _LIBCPP_STRING_H
 #   error <cstring> tried including <string.h> but didn't find libc++'s <string.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -101,4 +103,6 @@ using ::strlen _LIBCPP_USING_IF_EXISTS;
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CSTRING
diff --git a/libcxx/include/ctgmath b/libcxx/include/ctgmath
index 7dbe952f021b7..d8bb64032eca4 100644
--- a/libcxx/include/ctgmath
+++ b/libcxx/include/ctgmath
@@ -18,23 +18,27 @@
 
 */
 
-#include <cmath>
-#include <complex>
+#if 0
+#else // 0
+#  include <cmath>
+#  include <complex>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 using __standard_header_ctgmath _LIBCPP_DEPRECATED_("removed in C++20. Include <cmath> and <complex> instead.") = void;
 using __use_standard_header_ctgmath = __standard_header_ctgmath;
 
-#elif _LIBCPP_STD_VER >= 17
+#  elif _LIBCPP_STD_VER >= 17
 
 using __standard_header_ctgmath _LIBCPP_DEPRECATED_("Include <cmath> and <complex> instead.") = void;
 using __use_standard_header_ctgmath = __standard_header_ctgmath;
 
-#endif
+#  endif
+
+#endif // 0
 
 #endif // _LIBCPP_CTGMATH
diff --git a/libcxx/include/ctime b/libcxx/include/ctime
index d2d2470f2fa60..6c5b0166f9e42 100644
--- a/libcxx/include/ctime
+++ b/libcxx/include/ctime
@@ -45,29 +45,31 @@ int timespec_get( struct timespec *ts, int base); // C++17
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
 
 // <time.h> is not provided by libc++
-#if __has_include(<time.h>)
-#  include <time.h>
-#  ifdef _LIBCPP_TIME_H
-#    error "If libc++ starts defining <time.h>, the __has_include check should move to libc++'s <time.h>"
+#  if __has_include(<time.h>)
+#    include <time.h>
+#    ifdef _LIBCPP_TIME_H
+#      error "If libc++ starts defining <time.h>, the __has_include check should move to libc++'s <time.h>"
+#    endif
 #  endif
-#endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 using ::clock_t _LIBCPP_USING_IF_EXISTS;
 using ::time_t _LIBCPP_USING_IF_EXISTS;
 using ::tm _LIBCPP_USING_IF_EXISTS;
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 using ::timespec _LIBCPP_USING_IF_EXISTS;
-#endif
+#  endif
 using ::clock _LIBCPP_USING_IF_EXISTS;
 using ::difftime _LIBCPP_USING_IF_EXISTS;
 using ::mktime _LIBCPP_USING_IF_EXISTS;
@@ -77,10 +79,12 @@ using ::ctime _LIBCPP_USING_IF_EXISTS;
 using ::gmtime _LIBCPP_USING_IF_EXISTS;
 using ::localtime _LIBCPP_USING_IF_EXISTS;
 using ::strftime _LIBCPP_USING_IF_EXISTS;
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 using ::timespec_get _LIBCPP_USING_IF_EXISTS;
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CTIME
diff --git a/libcxx/include/ctype.h b/libcxx/include/ctype.h
index 448e4410c554f..611f94036a1bd 100644
--- a/libcxx/include/ctype.h
+++ b/libcxx/include/ctype.h
@@ -29,33 +29,36 @@ int tolower(int c);
 int toupper(int c);
 */
 
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if __has_include_next(<ctype.h>)
-#  include_next <ctype.h>
-#endif
-
-#ifdef __cplusplus
-
-#  undef isalnum
-#  undef isalpha
-#  undef isblank
-#  undef iscntrl
-#  undef isdigit
-#  undef isgraph
-#  undef islower
-#  undef isprint
-#  undef ispunct
-#  undef isspace
-#  undef isupper
-#  undef isxdigit
-#  undef tolower
-#  undef toupper
-
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if __has_include_next(<ctype.h>)
+#    include_next <ctype.h>
+#  endif
+
+#  ifdef __cplusplus
+
+#    undef isalnum
+#    undef isalpha
+#    undef isblank
+#    undef iscntrl
+#    undef isdigit
+#    undef isgraph
+#    undef islower
+#    undef isprint
+#    undef ispunct
+#    undef isspace
+#    undef isupper
+#    undef isxdigit
+#    undef tolower
+#    undef toupper
+
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CTYPE_H
diff --git a/libcxx/include/cuchar b/libcxx/include/cuchar
index 1d156eb7aaaf3..3ace72f1f8b1f 100644
--- a/libcxx/include/cuchar
+++ b/libcxx/include/cuchar
@@ -36,39 +36,42 @@ size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
 
-#include <uchar.h>
+#  include <uchar.h>
 
-#ifndef _LIBCPP_UCHAR_H
+#  ifndef _LIBCPP_UCHAR_H
 #   error <cuchar> tried including <uchar.h> but didn't find libc++'s <uchar.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_CXX03_LANG)
+#  if !defined(_LIBCPP_CXX03_LANG)
 
 using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
 
-#  if _LIBCPP_HAS_C8RTOMB_MBRTOC8
+#    if _LIBCPP_HAS_C8RTOMB_MBRTOC8
 using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS;
 using ::c8rtomb _LIBCPP_USING_IF_EXISTS;
-#  endif
+#    endif
 using ::mbrtoc16 _LIBCPP_USING_IF_EXISTS;
 using ::c16rtomb _LIBCPP_USING_IF_EXISTS;
 using ::mbrtoc32 _LIBCPP_USING_IF_EXISTS;
 using ::c32rtomb _LIBCPP_USING_IF_EXISTS;
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
+#endif   // 0
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/cwchar b/libcxx/include/cwchar
index f5a26664c1c3e..f97d9c43433a1 100644
--- a/libcxx/include/cwchar
+++ b/libcxx/include/cwchar
@@ -102,28 +102,30 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
-#include <__type_traits/copy_cv.h>
-#include <__type_traits/is_constant_evaluated.h>
-#include <__type_traits/is_equality_comparable.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/remove_cv.h>
-#include <cwctype>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
+#  include <__type_traits/copy_cv.h>
+#  include <__type_traits/is_constant_evaluated.h>
+#  include <__type_traits/is_equality_comparable.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/remove_cv.h>
+#  include <cwctype>
 
-#include <wchar.h>
+#  include <wchar.h>
 
-#ifndef _LIBCPP_WCHAR_H
+#  ifndef _LIBCPP_WCHAR_H
 #   error <cwchar> tried including <wchar.h> but didn't find libc++'s <wchar.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -194,9 +196,9 @@ using ::vwprintf _LIBCPP_USING_IF_EXISTS;
 using ::wprintf _LIBCPP_USING_IF_EXISTS;
 
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t __constexpr_wcslen(const wchar_t* __str) {
-#if __has_builtin(__builtin_wcslen)
+#  if __has_builtin(__builtin_wcslen)
   return __builtin_wcslen(__str);
-#else
+#  else
   if (!__libcpp_is_constant_evaluated())
     return std::wcslen(__str);
 
@@ -204,14 +206,14 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t __constexpr_wc
   for (; *__str != L'\0'; ++__str)
     ++__len;
   return __len;
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int
 __constexpr_wmemcmp(const wchar_t* __lhs, const wchar_t* __rhs, size_t __count) {
-#if __has_builtin(__builtin_wmemcmp)
+#  if __has_builtin(__builtin_wmemcmp)
   return __builtin_wmemcmp(__lhs, __rhs, __count);
-#else
+#  else
   if (!__libcpp_is_constant_evaluated())
     return std::wmemcmp(__lhs, __rhs, __count);
 
@@ -222,7 +224,7 @@ __constexpr_wmemcmp(const wchar_t* __lhs, const wchar_t* __rhs, size_t __count)
       return 1;
   }
   return 0;
-#endif
+#  endif
 }
 
 template <class _Tp, class _Up>
@@ -231,18 +233,18 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_wmemchr(_Tp
                     __libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
                 "Calling wmemchr on non-trivially equality comparable types is unsafe.");
 
-#if __has_builtin(__builtin_wmemchr)
+#  if __has_builtin(__builtin_wmemchr)
   if (!__libcpp_is_constant_evaluated()) {
     wchar_t __value_buffer = 0;
     __builtin_memcpy(&__value_buffer, &__value, sizeof(wchar_t));
     return reinterpret_cast<_Tp*>(
         __builtin_wmemchr(reinterpret_cast<__copy_cv_t<_Tp, wchar_t>*>(__str), __value_buffer, __count));
   }
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   else if constexpr (is_same_v<remove_cv_t<_Tp>, wchar_t>)
     return __builtin_wmemchr(__str, __value, __count);
-#  endif
-#endif // __has_builtin(__builtin_wmemchr)
+#    endif
+#  endif // __has_builtin(__builtin_wmemchr)
 
   for (; __count; --__count) {
     if (*__str == __value)
@@ -254,8 +256,9 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_wmemchr(_Tp
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_CWCHAR
diff --git a/libcxx/include/cwctype b/libcxx/include/cwctype
index 04abfabef5793..848aadd8a1d11 100644
--- a/libcxx/include/cwctype
+++ b/libcxx/include/cwctype
@@ -49,26 +49,28 @@ wctrans_t wctrans(const char* property);
 
 */
 
-#include <__config>
-#include <cctype>
+#if 0
+#else // 0
+#  include <__config>
+#  include <cctype>
 
-#include <wctype.h>
+#  include <wctype.h>
 
-#ifndef _LIBCPP_WCTYPE_H
+#  ifndef _LIBCPP_WCTYPE_H
 #   error <cwctype> tried including <wctype.h> but didn't find libc++'s <wctype.h> header. \
           This usually means that your header search paths are not configured properly. \
           The header search paths should contain the C++ Standard Library headers before \
           any C Standard Library, and you are probably using compiler flags that make that \
           not be the case.
-#endif
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if defined(_LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H)
+#  if defined(_LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H)
 using ::wint_t _LIBCPP_USING_IF_EXISTS;
 using ::wctrans_t _LIBCPP_USING_IF_EXISTS;
 using ::wctype_t _LIBCPP_USING_IF_EXISTS;
@@ -90,8 +92,10 @@ using ::towlower _LIBCPP_USING_IF_EXISTS;
 using ::towupper _LIBCPP_USING_IF_EXISTS;
 using ::towctrans _LIBCPP_USING_IF_EXISTS;
 using ::wctrans _LIBCPP_USING_IF_EXISTS;
-#endif // _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
+#  endif // _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_CWCTYPE
diff --git a/libcxx/include/deque b/libcxx/include/deque
index ad66750348974..2bd850a67f07c 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -177,87 +177,89 @@ template <class T, class Allocator, class Predicate>
 
 */
 
-#include <__algorithm/copy.h>
-#include <__algorithm/copy_backward.h>
-#include <__algorithm/copy_n.h>
-#include <__algorithm/equal.h>
-#include <__algorithm/fill_n.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__algorithm/max.h>
-#include <__algorithm/min.h>
-#include <__algorithm/move.h>
-#include <__algorithm/move_backward.h>
-#include <__algorithm/remove.h>
-#include <__algorithm/remove_if.h>
-#include <__algorithm/unwrap_iter.h>
-#include <__assert>
-#include <__config>
-#include <__debug_utils/sanitizers.h>
-#include <__format/enable_insertable.h>
-#include <__fwd/deque.h>
-#include <__iterator/distance.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/move_iterator.h>
-#include <__iterator/next.h>
-#include <__iterator/prev.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/segmented_iterator.h>
-#include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_destructor.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/swap_allocator.h>
-#include <__memory/temp_value.h>
-#include <__memory/unique_ptr.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__ranges/size.h>
-#include <__split_buffer>
-#include <__type_traits/conditional.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/disjunction.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <__utility/swap.h>
-#include <limits>
-#include <stdexcept>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/copy.h>
+#  include <__algorithm/copy_backward.h>
+#  include <__algorithm/copy_n.h>
+#  include <__algorithm/equal.h>
+#  include <__algorithm/fill_n.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/max.h>
+#  include <__algorithm/min.h>
+#  include <__algorithm/move.h>
+#  include <__algorithm/move_backward.h>
+#  include <__algorithm/remove.h>
+#  include <__algorithm/remove_if.h>
+#  include <__algorithm/unwrap_iter.h>
+#  include <__assert>
+#  include <__config>
+#  include <__debug_utils/sanitizers.h>
+#  include <__format/enable_insertable.h>
+#  include <__fwd/deque.h>
+#  include <__iterator/distance.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/move_iterator.h>
+#  include <__iterator/next.h>
+#  include <__iterator/prev.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/segmented_iterator.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_destructor.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/swap_allocator.h>
+#  include <__memory/temp_value.h>
+#  include <__memory/unique_ptr.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__ranges/size.h>
+#  include <__split_buffer>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/disjunction.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <__utility/swap.h>
+#  include <limits>
+#  include <stdexcept>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [deque.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -272,13 +274,13 @@ template <class _ValueType,
           class _MapPointer,
           class _DiffType,
           _DiffType _BS =
-#ifdef _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
+#  ifdef _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
               // Keep template parameter to avoid changing all template declarations thoughout
               // this file.
           0
-#else
+#  else
               __deque_block_size<_ValueType, _DiffType>::value
-#endif
+#  endif
           >
 class _LIBCPP_TEMPLATE_VIS __deque_iterator {
   typedef _MapPointer __map_iterator;
@@ -299,10 +301,10 @@ public:
   typedef _Reference reference;
 
   _LIBCPP_HIDE_FROM_ABI __deque_iterator() _NOEXCEPT
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       : __m_iter_(nullptr),
         __ptr_(nullptr)
-#endif
+#  endif
   {
   }
 
@@ -391,7 +393,7 @@ public:
     return __x.__ptr_ == __y.__ptr_;
   }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
   _LIBCPP_HIDE_FROM_ABI friend bool operator!=(const __deque_iterator& __x, const __deque_iterator& __y) {
     return !(__x == __y);
   }
@@ -411,7 +413,7 @@ public:
     return !(__x < __y);
   }
 
-#else
+#  else
 
   _LIBCPP_HIDE_FROM_ABI friend strong_ordering operator<=>(const __deque_iterator& __x, const __deque_iterator& __y) {
     if (__x.__m_iter_ < __y.__m_iter_)
@@ -433,7 +435,7 @@ public:
 
     return strong_ordering::greater;
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 private:
   _LIBCPP_HIDE_FROM_ABI explicit __deque_iterator(__map_iterator __m, pointer __p) _NOEXCEPT
@@ -621,9 +623,9 @@ public:
   }
 
   explicit _LIBCPP_HIDE_FROM_ABI deque(size_type __n);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   explicit _LIBCPP_HIDE_FROM_ABI deque(size_type __n, const _Allocator& __a);
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI deque(size_type __n, const value_type& __v);
 
   template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
@@ -639,7 +641,7 @@ public:
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI deque(_InputIter __f, _InputIter __l, const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI deque(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
       : __map_(__pointer_allocator(__a)), __start_(0), __size_(0), __alloc_(__a) {
@@ -652,14 +654,14 @@ public:
       }
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI deque(const deque& __c);
   _LIBCPP_HIDE_FROM_ABI deque(const deque& __c, const __type_identity_t<allocator_type>& __a);
 
   _LIBCPP_HIDE_FROM_ABI deque& operator=(const deque& __c);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI deque(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI deque(initializer_list<value_type> __il, const allocator_type& __a);
 
@@ -675,7 +677,7 @@ public:
                                   is_nothrow_move_assignable<allocator_type>::value);
 
   _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class _InputIter,
             __enable_if_t<__has_input_iterator_category<_InputIter>::value &&
@@ -685,7 +687,7 @@ public:
   template <class _RAIter, __enable_if_t<__has_random_access_iterator_category<_RAIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void assign(_RAIter __f, _RAIter __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
     if constexpr (ranges::random_access_range<_Range>) {
@@ -700,7 +702,7 @@ public:
       __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
 
@@ -769,25 +771,25 @@ public:
   // 23.2.2.3 modifiers:
   _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
   _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __v);
-#ifndef _LIBCPP_CXX03_LANG
-#  if _LIBCPP_STD_VER >= 17
+#  ifndef _LIBCPP_CXX03_LANG
+#    if _LIBCPP_STD_VER >= 17
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI reference emplace_back(_Args&&... __args);
-#  else
+#    else
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
-#  endif
+#    endif
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
 
   _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
   _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __v);
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
     insert_range(begin(), std::forward<_Range>(__range));
@@ -797,14 +799,14 @@ public:
   _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
     insert_range(end(), std::forward<_Range>(__range));
   }
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v);
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
     return insert(__p, __il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v);
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __v);
   template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> = 0>
@@ -815,7 +817,7 @@ public:
   template <class _BiIter, __enable_if_t<__has_bidirectional_iterator_category<_BiIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _BiIter __f, _BiIter __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI iterator insert_range(const_iterator __position, _Range&& __range) {
     if constexpr (ranges::bidirectional_range<_Range>) {
@@ -830,7 +832,7 @@ public:
       return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void pop_front();
   _LIBCPP_HIDE_FROM_ABI void pop_back();
@@ -838,11 +840,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
 
   _LIBCPP_HIDE_FROM_ABI void swap(deque& __c)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
-#else
+#  else
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
 
   _LIBCPP_HIDE_FROM_ABI bool __invariants() const {
@@ -920,7 +922,7 @@ private:
     (void)__end;
     (void)__annotation_type;
     (void)__place;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     // __beg - index of the first item to annotate
     // __end - index behind the last item to annotate (so last item + 1)
     // __annotation_type - __asan_unposion or __asan_poison
@@ -1013,23 +1015,23 @@ private:
       std::__annotate_double_ended_contiguous_container<_Allocator>(
           __mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
     }
-#endif // _LIBCPP_HAS_ASAN
+#  endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     if (__current_size == 0)
       __annotate_from_to(0, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     else {
       __annotate_from_to(0, __start_, __asan_poison, __asan_front_moved);
       __annotate_from_to(__start_ + __current_size, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     }
-#endif // _LIBCPP_HAS_ASAN
+#  endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     if (empty()) {
       for (size_t __i = 0; __i < __map_.size(); ++__i) {
         __annotate_whole_block(__i, __asan_unposion);
@@ -1038,37 +1040,37 @@ private:
       __annotate_from_to(0, __start_, __asan_unposion, __asan_front_moved);
       __annotate_from_to(__start_ + size(), __map_.size() * __block_size, __asan_unposion, __asan_back_moved);
     }
-#endif // _LIBCPP_HAS_ASAN
+#  endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_front(size_type __n) const _NOEXCEPT {
     (void)__n;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     __annotate_from_to(__start_ - __n, __start_, __asan_unposion, __asan_front_moved);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_back(size_type __n) const _NOEXCEPT {
     (void)__n;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     __annotate_from_to(__start_ + size(), __start_ + size() + __n, __asan_unposion, __asan_back_moved);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_front(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     __annotate_from_to(__old_start, __old_start + (__old_size - size()), __asan_poison, __asan_front_moved);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_back(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     __annotate_from_to(__old_start + size(), __old_start + __old_size, __asan_poison, __asan_back_moved);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_poison_block(const void* __beginning, const void* __end) const _NOEXCEPT {
@@ -1079,7 +1081,7 @@ private:
   __annotate_whole_block(size_t __block_index, __asan_annotation_type __annotation_type) const _NOEXCEPT {
     (void)__block_index;
     (void)__annotation_type;
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
     __map_const_iterator __block_it = __map_.begin() + __block_index;
     const void* __block_start       = std::__to_address(*__block_it);
     const void* __block_end         = std::__to_address(*__block_it + __block_size);
@@ -1090,9 +1092,9 @@ private:
       std::__annotate_double_ended_contiguous_container<_Allocator>(
           __block_start, __block_end, __block_start, __block_start, __block_start, __block_end);
     }
-#endif
+#  endif
   }
-#if _LIBCPP_HAS_ASAN
+#  if _LIBCPP_HAS_ASAN
 
 public:
   _LIBCPP_HIDE_FROM_ABI bool __verify_asan_annotations() const _NOEXCEPT {
@@ -1154,7 +1156,7 @@ public:
   }
 
 private:
-#endif // _LIBCPP_HAS_ASAN
+#  endif // _LIBCPP_HAS_ASAN
   _LIBCPP_HIDE_FROM_ABI bool __maybe_remove_front_spare(bool __keep_one = true) {
     if (__front_spare_blocks() >= 2 || (!__keep_one && __front_spare_blocks())) {
       __annotate_whole_block(0, __asan_unposion);
@@ -1244,7 +1246,7 @@ template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR const typename allocator_traits<_Alloc>::difference_type deque<_Tp, _Alloc>::__block_size =
     __deque_block_size<value_type, difference_type>::value;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Alloc = allocator<__iter_value_type<_InputIterator>>,
           class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
@@ -1256,14 +1258,14 @@ template <class _InputIterator,
           class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
           class = enable_if_t<__is_allocator<_Alloc>::value> >
 deque(_InputIterator, _InputIterator, _Alloc) -> deque<__iter_value_type<_InputIterator>, _Alloc>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Alloc = allocator<ranges::range_value_t<_Range>>,
           class        = enable_if_t<__is_allocator<_Alloc>::value> >
 deque(from_range_t, _Range&&, _Alloc = _Alloc()) -> deque<ranges::range_value_t<_Range>, _Alloc>;
-#endif
+#  endif
 
 template <class _Tp, class _Allocator>
 deque<_Tp, _Allocator>::deque(size_type __n) : __start_(0), __size_(0) {
@@ -1272,7 +1274,7 @@ deque<_Tp, _Allocator>::deque(size_type __n) : __start_(0), __size_(0) {
     __append(__n);
 }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 template <class _Tp, class _Allocator>
 deque<_Tp, _Allocator>::deque(size_type __n, const _Allocator& __a)
     : __map_(__pointer_allocator(__a)), __start_(0), __size_(0), __alloc_(__a) {
@@ -1280,7 +1282,7 @@ deque<_Tp, _Allocator>::deque(size_type __n, const _Allocator& __a)
   if (__n > 0)
     __append(__n);
 }
-#endif
+#  endif
 
 template <class _Tp, class _Allocator>
 deque<_Tp, _Allocator>::deque(size_type __n, const value_type& __v) : __start_(0), __size_(0) {
@@ -1330,7 +1332,7 @@ deque<_Tp, _Allocator>& deque<_Tp, _Allocator>::operator=(const deque& __c) {
   return *this;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Allocator>
 deque<_Tp, _Allocator>::deque(initializer_list<value_type> __il) : __start_(0), __size_(0) {
@@ -1398,7 +1400,7 @@ void deque<_Tp, _Allocator>::__move_assign(deque& __c,
   __move_assign(__c);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Allocator>
 template <class _InputIter,
@@ -1586,7 +1588,7 @@ void deque<_Tp, _Allocator>::push_front(const value_type& __v) {
   ++__size();
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Allocator>
 void deque<_Tp, _Allocator>::push_back(value_type&& __v) {
   allocator_type& __a = __alloc();
@@ -1600,11 +1602,11 @@ void deque<_Tp, _Allocator>::push_back(value_type&& __v) {
 
 template <class _Tp, class _Allocator>
 template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 typename deque<_Tp, _Allocator>::reference
-#  else
+#    else
 void
-#  endif
+#    endif
 deque<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
   allocator_type& __a = __alloc();
   if (__back_spare() == 0)
@@ -1613,9 +1615,9 @@ deque<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
   __annotate_increase_back(1);
   __alloc_traits::construct(__a, std::addressof(*end()), std::forward<_Args>(__args)...);
   ++__size();
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   return *--end();
-#  endif
+#    endif
 }
 
 template <class _Tp, class _Allocator>
@@ -1632,11 +1634,11 @@ void deque<_Tp, _Allocator>::push_front(value_type&& __v) {
 
 template <class _Tp, class _Allocator>
 template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 typename deque<_Tp, _Allocator>::reference
-#  else
+#    else
 void
-#  endif
+#    endif
 deque<_Tp, _Allocator>::emplace_front(_Args&&... __args) {
   allocator_type& __a = __alloc();
   if (__front_spare() == 0)
@@ -1646,9 +1648,9 @@ deque<_Tp, _Allocator>::emplace_front(_Args&&... __args) {
   __alloc_traits::construct(__a, std::addressof(*--begin()), std::forward<_Args>(__args)...);
   --__start_;
   ++__size();
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   return *begin();
-#  endif
+#    endif
 }
 
 template <class _Tp, class _Allocator>
@@ -1746,7 +1748,7 @@ typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::emplace(const_
   return begin() + __pos;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Allocator>
 typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, const value_type& __v) {
@@ -1969,11 +1971,11 @@ template <class _Tp, class _Allocator>
 template <class _InputIterator, class _Sentinel>
 _LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__append_with_sentinel(_InputIterator __f, _Sentinel __l) {
   for (; __f != __l; ++__f)
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
     push_back(*__f);
-#else
+#  else
     emplace_back(*__f);
-#endif
+#  endif
 }
 
 template <class _Tp, class _Allocator>
@@ -2125,22 +2127,22 @@ void deque<_Tp, _Allocator>::__add_front_capacity(size_type __n) {
     size_type __ds = (__nb + __back_capacity) * __block_size - __map_.empty();
     __split_buffer<pointer, __pointer_allocator&> __buf(
         std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()), 0, __map_.__alloc_);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __nb > 0; --__nb) {
         __buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
         // ASan: this is empty container, we have to poison whole block
         __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __annotate_delete();
       for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
         __alloc_traits::deallocate(__a, *__i, __block_size);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __back_capacity > 0; --__back_capacity) {
       __buf.emplace_back(__map_.back());
       __map_.pop_back();
@@ -2250,22 +2252,22 @@ void deque<_Tp, _Allocator>::__add_back_capacity(size_type __n) {
         std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()),
         __map_.size() - __front_capacity,
         __map_.__alloc_);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __nb > 0; --__nb) {
         __buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
         // ASan: this is an empty container, we have to poison the whole block
         __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __annotate_delete();
       for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
         __alloc_traits::deallocate(__a, *__i, __block_size);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __front_capacity > 0; --__front_capacity) {
       __buf.emplace_back(__map_.front());
       __map_.pop_front();
@@ -2502,11 +2504,11 @@ void deque<_Tp, _Allocator>::__erase_to_end(const_iterator __f) {
 
 template <class _Tp, class _Allocator>
 inline void deque<_Tp, _Allocator>::swap(deque& __c)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
-#else
+#  else
     _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
-#endif
+#  endif
 {
   __map_.swap(__c.__map_);
   std::swap(__start_, __c.__start_);
@@ -2542,7 +2544,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator==(const deque<_Tp, _Allocator>& __x,
   return __sz == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y) {
@@ -2569,7 +2571,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const deque<_Tp, _Allocator>& __x,
   return !(__y < __x);
 }
 
-#else // _LIBCPP_STD_VER <= 17
+#  else // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
@@ -2577,7 +2579,7 @@ operator<=>(const deque<_Tp, _Allocator>& __x, const deque<_Tp, _Allocator>& __y
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void swap(deque<_Tp, _Allocator>& __x, deque<_Tp, _Allocator>& __y)
@@ -2585,7 +2587,7 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(deque<_Tp, _Allocator>& __x, deque<_Tp, _
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Tp, class _Allocator, class _Up>
 inline _LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::size_type
 erase(deque<_Tp, _Allocator>& __c, const _Up& __v) {
@@ -2604,12 +2606,12 @@ erase_if(deque<_Tp, _Allocator>& __c, _Predicate __pred) {
 
 template <>
 inline constexpr bool __format::__enable_insertable<std::deque<char>> = true;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 inline constexpr bool __format::__enable_insertable<std::deque<wchar_t>> = true;
-#  endif
+#    endif
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _Tp, class _Allocator>
 struct __container_traits<deque<_Tp, _Allocator> > {
@@ -2624,27 +2626,28 @@ struct __container_traits<deque<_Tp, _Allocator> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _ValueT>
 using deque _LIBCPP_AVAILABILITY_PMR = std::deque<_ValueT, polymorphic_allocator<_ValueT>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <functional>
-#  include <iosfwd>
-#  include <iterator>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <functional>
+#    include <iosfwd>
+#    include <iterator>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_DEQUE
diff --git a/libcxx/include/errno.h b/libcxx/include/errno.h
index e657ad84ff44b..648d22e98846b 100644
--- a/libcxx/include/errno.h
+++ b/libcxx/include/errno.h
@@ -22,378 +22,381 @@
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if __has_include_next(<errno.h>)
-#  include_next <errno.h>
-#endif
+#  if __has_include_next(<errno.h>)
+#    include_next <errno.h>
+#  endif
 
-#ifdef __cplusplus
+#  ifdef __cplusplus
 
-#  if !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
+#    if !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
 
-#    ifdef ELAST
+#      ifdef ELAST
 
 static const int __elast1 = ELAST + 1;
 static const int __elast2 = ELAST + 2;
 
-#    else
+#      else
 
 static const int __elast1 = 104;
 static const int __elast2 = 105;
 
-#    endif
+#      endif
 
-#    ifdef ENOTRECOVERABLE
+#      ifdef ENOTRECOVERABLE
 
-#      define EOWNERDEAD __elast1
+#        define EOWNERDEAD __elast1
 
-#      ifdef ELAST
-#        undef ELAST
-#        define ELAST EOWNERDEAD
-#      endif
+#        ifdef ELAST
+#          undef ELAST
+#          define ELAST EOWNERDEAD
+#        endif
 
-#    elif defined(EOWNERDEAD)
+#      elif defined(EOWNERDEAD)
 
-#      define ENOTRECOVERABLE __elast1
-#      ifdef ELAST
-#        undef ELAST
-#        define ELAST ENOTRECOVERABLE
-#      endif
+#        define ENOTRECOVERABLE __elast1
+#        ifdef ELAST
+#          undef ELAST
+#          define ELAST ENOTRECOVERABLE
+#        endif
 
-#    else // defined(EOWNERDEAD)
+#      else // defined(EOWNERDEAD)
 
-#      define EOWNERDEAD __elast1
-#      define ENOTRECOVERABLE __elast2
-#      ifdef ELAST
-#        undef ELAST
-#        define ELAST ENOTRECOVERABLE
-#      endif
+#        define EOWNERDEAD __elast1
+#        define ENOTRECOVERABLE __elast2
+#        ifdef ELAST
+#          undef ELAST
+#          define ELAST ENOTRECOVERABLE
+#        endif
 
-#    endif // defined(EOWNERDEAD)
+#      endif // defined(EOWNERDEAD)
 
-#  endif // !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
+#    endif // !defined(EOWNERDEAD) || !defined(ENOTRECOVERABLE)
 
 //  supply errno values likely to be missing, particularly on Windows
 
-#  ifndef EAFNOSUPPORT
-#    define EAFNOSUPPORT 9901
-#  endif
+#    ifndef EAFNOSUPPORT
+#      define EAFNOSUPPORT 9901
+#    endif
 
-#  ifndef EADDRINUSE
-#    define EADDRINUSE 9902
-#  endif
+#    ifndef EADDRINUSE
+#      define EADDRINUSE 9902
+#    endif
 
-#  ifndef EADDRNOTAVAIL
-#    define EADDRNOTAVAIL 9903
-#  endif
+#    ifndef EADDRNOTAVAIL
+#      define EADDRNOTAVAIL 9903
+#    endif
 
-#  ifndef EISCONN
-#    define EISCONN 9904
-#  endif
+#    ifndef EISCONN
+#      define EISCONN 9904
+#    endif
 
-#  ifndef EBADMSG
-#    define EBADMSG 9905
-#  endif
+#    ifndef EBADMSG
+#      define EBADMSG 9905
+#    endif
 
-#  ifndef ECONNABORTED
-#    define ECONNABORTED 9906
-#  endif
+#    ifndef ECONNABORTED
+#      define ECONNABORTED 9906
+#    endif
 
-#  ifndef EALREADY
-#    define EALREADY 9907
-#  endif
+#    ifndef EALREADY
+#      define EALREADY 9907
+#    endif
 
-#  ifndef ECONNREFUSED
-#    define ECONNREFUSED 9908
-#  endif
+#    ifndef ECONNREFUSED
+#      define ECONNREFUSED 9908
+#    endif
 
-#  ifndef ECONNRESET
-#    define ECONNRESET 9909
-#  endif
+#    ifndef ECONNRESET
+#      define ECONNRESET 9909
+#    endif
 
-#  ifndef EDESTADDRREQ
-#    define EDESTADDRREQ 9910
-#  endif
+#    ifndef EDESTADDRREQ
+#      define EDESTADDRREQ 9910
+#    endif
 
-#  ifndef EHOSTUNREACH
-#    define EHOSTUNREACH 9911
-#  endif
+#    ifndef EHOSTUNREACH
+#      define EHOSTUNREACH 9911
+#    endif
 
-#  ifndef EIDRM
-#    define EIDRM 9912
-#  endif
+#    ifndef EIDRM
+#      define EIDRM 9912
+#    endif
 
-#  ifndef EMSGSIZE
-#    define EMSGSIZE 9913
-#  endif
+#    ifndef EMSGSIZE
+#      define EMSGSIZE 9913
+#    endif
 
-#  ifndef ENETDOWN
-#    define ENETDOWN 9914
-#  endif
+#    ifndef ENETDOWN
+#      define ENETDOWN 9914
+#    endif
 
-#  ifndef ENETRESET
-#    define ENETRESET 9915
-#  endif
+#    ifndef ENETRESET
+#      define ENETRESET 9915
+#    endif
 
-#  ifndef ENETUNREACH
-#    define ENETUNREACH 9916
-#  endif
+#    ifndef ENETUNREACH
+#      define ENETUNREACH 9916
+#    endif
 
-#  ifndef ENOBUFS
-#    define ENOBUFS 9917
-#  endif
+#    ifndef ENOBUFS
+#      define ENOBUFS 9917
+#    endif
 
-#  ifndef ENOLINK
-#    define ENOLINK 9918
-#  endif
+#    ifndef ENOLINK
+#      define ENOLINK 9918
+#    endif
 
-#  ifndef ENODATA
-#    define ENODATA 9919
-#  endif
+#    ifndef ENODATA
+#      define ENODATA 9919
+#    endif
 
-#  ifndef ENOMSG
-#    define ENOMSG 9920
-#  endif
+#    ifndef ENOMSG
+#      define ENOMSG 9920
+#    endif
 
-#  ifndef ENOPROTOOPT
-#    define ENOPROTOOPT 9921
-#  endif
+#    ifndef ENOPROTOOPT
+#      define ENOPROTOOPT 9921
+#    endif
 
-#  ifndef ENOSR
-#    define ENOSR 9922
-#  endif
+#    ifndef ENOSR
+#      define ENOSR 9922
+#    endif
 
-#  ifndef ENOTSOCK
-#    define ENOTSOCK 9923
-#  endif
+#    ifndef ENOTSOCK
+#      define ENOTSOCK 9923
+#    endif
 
-#  ifndef ENOSTR
-#    define ENOSTR 9924
-#  endif
+#    ifndef ENOSTR
+#      define ENOSTR 9924
+#    endif
 
-#  ifndef ENOTCONN
-#    define ENOTCONN 9925
-#  endif
+#    ifndef ENOTCONN
+#      define ENOTCONN 9925
+#    endif
 
-#  ifndef ENOTSUP
-#    define ENOTSUP 9926
-#  endif
+#    ifndef ENOTSUP
+#      define ENOTSUP 9926
+#    endif
 
-#  ifndef ECANCELED
-#    define ECANCELED 9927
-#  endif
+#    ifndef ECANCELED
+#      define ECANCELED 9927
+#    endif
 
-#  ifndef EINPROGRESS
-#    define EINPROGRESS 9928
-#  endif
+#    ifndef EINPROGRESS
+#      define EINPROGRESS 9928
+#    endif
 
-#  ifndef EOPNOTSUPP
-#    define EOPNOTSUPP 9929
-#  endif
+#    ifndef EOPNOTSUPP
+#      define EOPNOTSUPP 9929
+#    endif
 
-#  ifndef EWOULDBLOCK
-#    define EWOULDBLOCK 9930
-#  endif
+#    ifndef EWOULDBLOCK
+#      define EWOULDBLOCK 9930
+#    endif
 
-#  ifndef EOWNERDEAD
-#    define EOWNERDEAD 9931
-#  endif
+#    ifndef EOWNERDEAD
+#      define EOWNERDEAD 9931
+#    endif
 
-#  ifndef EPROTO
-#    define EPROTO 9932
-#  endif
+#    ifndef EPROTO
+#      define EPROTO 9932
+#    endif
 
-#  ifndef EPROTONOSUPPORT
-#    define EPROTONOSUPPORT 9933
-#  endif
+#    ifndef EPROTONOSUPPORT
+#      define EPROTONOSUPPORT 9933
+#    endif
 
-#  ifndef ENOTRECOVERABLE
-#    define ENOTRECOVERABLE 9934
-#  endif
+#    ifndef ENOTRECOVERABLE
+#      define ENOTRECOVERABLE 9934
+#    endif
 
-#  ifndef ETIME
-#    define ETIME 9935
-#  endif
+#    ifndef ETIME
+#      define ETIME 9935
+#    endif
 
-#  ifndef ETXTBSY
-#    define ETXTBSY 9936
-#  endif
+#    ifndef ETXTBSY
+#      define ETXTBSY 9936
+#    endif
 
-#  ifndef ETIMEDOUT
-#    define ETIMEDOUT 9938
-#  endif
+#    ifndef ETIMEDOUT
+#      define ETIMEDOUT 9938
+#    endif
 
-#  ifndef ELOOP
-#    define ELOOP 9939
-#  endif
+#    ifndef ELOOP
+#      define ELOOP 9939
+#    endif
 
-#  ifndef EOVERFLOW
-#    define EOVERFLOW 9940
-#  endif
+#    ifndef EOVERFLOW
+#      define EOVERFLOW 9940
+#    endif
 
-#  ifndef EPROTOTYPE
-#    define EPROTOTYPE 9941
-#  endif
+#    ifndef EPROTOTYPE
+#      define EPROTOTYPE 9941
+#    endif
 
-#  ifndef ENOSYS
-#    define ENOSYS 9942
-#  endif
+#    ifndef ENOSYS
+#      define ENOSYS 9942
+#    endif
 
-#  ifndef EINVAL
-#    define EINVAL 9943
-#  endif
+#    ifndef EINVAL
+#      define EINVAL 9943
+#    endif
 
-#  ifndef ERANGE
-#    define ERANGE 9944
-#  endif
+#    ifndef ERANGE
+#      define ERANGE 9944
+#    endif
 
-#  ifndef EILSEQ
-#    define EILSEQ 9945
-#  endif
+#    ifndef EILSEQ
+#      define EILSEQ 9945
+#    endif
 
 //  Windows Mobile doesn't appear to define these:
 
-#  ifndef E2BIG
-#    define E2BIG 9946
-#  endif
+#    ifndef E2BIG
+#      define E2BIG 9946
+#    endif
 
-#  ifndef EDOM
-#    define EDOM 9947
-#  endif
+#    ifndef EDOM
+#      define EDOM 9947
+#    endif
 
-#  ifndef EFAULT
-#    define EFAULT 9948
-#  endif
+#    ifndef EFAULT
+#      define EFAULT 9948
+#    endif
 
-#  ifndef EBADF
-#    define EBADF 9949
-#  endif
+#    ifndef EBADF
+#      define EBADF 9949
+#    endif
 
-#  ifndef EPIPE
-#    define EPIPE 9950
-#  endif
+#    ifndef EPIPE
+#      define EPIPE 9950
+#    endif
 
-#  ifndef EXDEV
-#    define EXDEV 9951
-#  endif
+#    ifndef EXDEV
+#      define EXDEV 9951
+#    endif
 
-#  ifndef EBUSY
-#    define EBUSY 9952
-#  endif
+#    ifndef EBUSY
+#      define EBUSY 9952
+#    endif
 
-#  ifndef ENOTEMPTY
-#    define ENOTEMPTY 9953
-#  endif
+#    ifndef ENOTEMPTY
+#      define ENOTEMPTY 9953
+#    endif
 
-#  ifndef ENOEXEC
-#    define ENOEXEC 9954
-#  endif
+#    ifndef ENOEXEC
+#      define ENOEXEC 9954
+#    endif
 
-#  ifndef EEXIST
-#    define EEXIST 9955
-#  endif
+#    ifndef EEXIST
+#      define EEXIST 9955
+#    endif
 
-#  ifndef EFBIG
-#    define EFBIG 9956
-#  endif
+#    ifndef EFBIG
+#      define EFBIG 9956
+#    endif
 
-#  ifndef ENAMETOOLONG
-#    define ENAMETOOLONG 9957
-#  endif
+#    ifndef ENAMETOOLONG
+#      define ENAMETOOLONG 9957
+#    endif
 
-#  ifndef ENOTTY
-#    define ENOTTY 9958
-#  endif
+#    ifndef ENOTTY
+#      define ENOTTY 9958
+#    endif
 
-#  ifndef EINTR
-#    define EINTR 9959
-#  endif
+#    ifndef EINTR
+#      define EINTR 9959
+#    endif
 
-#  ifndef ESPIPE
-#    define ESPIPE 9960
-#  endif
+#    ifndef ESPIPE
+#      define ESPIPE 9960
+#    endif
 
-#  ifndef EIO
-#    define EIO 9961
-#  endif
+#    ifndef EIO
+#      define EIO 9961
+#    endif
 
-#  ifndef EISDIR
-#    define EISDIR 9962
-#  endif
+#    ifndef EISDIR
+#      define EISDIR 9962
+#    endif
 
-#  ifndef ECHILD
-#    define ECHILD 9963
-#  endif
+#    ifndef ECHILD
+#      define ECHILD 9963
+#    endif
 
-#  ifndef ENOLCK
-#    define ENOLCK 9964
-#  endif
+#    ifndef ENOLCK
+#      define ENOLCK 9964
+#    endif
 
-#  ifndef ENOSPC
-#    define ENOSPC 9965
-#  endif
+#    ifndef ENOSPC
+#      define ENOSPC 9965
+#    endif
 
-#  ifndef ENXIO
-#    define ENXIO 9966
-#  endif
+#    ifndef ENXIO
+#      define ENXIO 9966
+#    endif
 
-#  ifndef ENODEV
-#    define ENODEV 9967
-#  endif
+#    ifndef ENODEV
+#      define ENODEV 9967
+#    endif
 
-#  ifndef ENOENT
-#    define ENOENT 9968
-#  endif
+#    ifndef ENOENT
+#      define ENOENT 9968
+#    endif
 
-#  ifndef ESRCH
-#    define ESRCH 9969
-#  endif
+#    ifndef ESRCH
+#      define ESRCH 9969
+#    endif
 
-#  ifndef ENOTDIR
-#    define ENOTDIR 9970
-#  endif
+#    ifndef ENOTDIR
+#      define ENOTDIR 9970
+#    endif
 
-#  ifndef ENOMEM
-#    define ENOMEM 9971
-#  endif
+#    ifndef ENOMEM
+#      define ENOMEM 9971
+#    endif
 
-#  ifndef EPERM
-#    define EPERM 9972
-#  endif
+#    ifndef EPERM
+#      define EPERM 9972
+#    endif
 
-#  ifndef EACCES
-#    define EACCES 9973
-#  endif
+#    ifndef EACCES
+#      define EACCES 9973
+#    endif
 
-#  ifndef EROFS
-#    define EROFS 9974
-#  endif
+#    ifndef EROFS
+#      define EROFS 9974
+#    endif
 
-#  ifndef EDEADLK
-#    define EDEADLK 9975
-#  endif
+#    ifndef EDEADLK
+#      define EDEADLK 9975
+#    endif
 
-#  ifndef EAGAIN
-#    define EAGAIN 9976
-#  endif
+#    ifndef EAGAIN
+#      define EAGAIN 9976
+#    endif
 
-#  ifndef ENFILE
-#    define ENFILE 9977
-#  endif
+#    ifndef ENFILE
+#      define ENFILE 9977
+#    endif
 
-#  ifndef EMFILE
-#    define EMFILE 9978
-#  endif
+#    ifndef EMFILE
+#      define EMFILE 9978
+#    endif
 
-#  ifndef EMLINK
-#    define EMLINK 9979
-#  endif
+#    ifndef EMLINK
+#      define EMLINK 9979
+#    endif
 
-#endif // __cplusplus
+#  endif // __cplusplus
+#endif   // 0
 
 #endif // _LIBCPP_ERRNO_H
diff --git a/libcxx/include/exception b/libcxx/include/exception
index 88eaaf06bf4a2..f0eaba1295fa3 100644
--- a/libcxx/include/exception
+++ b/libcxx/include/exception
@@ -76,22 +76,25 @@ template <class E> void rethrow_if_nested(const E& e);
 
 */
 
-#include <__config>
-#include <__exception/exception.h>
-#include <__exception/exception_ptr.h>
-#include <__exception/nested_exception.h>
-#include <__exception/operations.h>
-#include <__exception/terminate.h>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <type_traits>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__exception/exception.h>
+#  include <__exception/exception_ptr.h>
+#  include <__exception/nested_exception.h>
+#  include <__exception/operations.h>
+#  include <__exception/terminate.h>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_EXCEPTION
diff --git a/libcxx/include/execution b/libcxx/include/execution
index 94d434b2e4603..e8ef8b1e6ee79 100644
--- a/libcxx/include/execution
+++ b/libcxx/include/execution
@@ -32,17 +32,19 @@ namespace std {
 }
 */
 
-#include <__config>
-#include <__type_traits/is_execution_policy.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/remove_cvref.h>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__type_traits/is_execution_policy.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+#  if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -79,7 +81,7 @@ struct __unsequenced_policy {
 
 constexpr __unsequenced_policy __unseq{__disable_user_instantiations_tag{}};
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 struct unsequenced_policy {
   _LIBCPP_HIDE_FROM_ABI constexpr explicit unsequenced_policy(__disable_user_instantiations_tag) {}
@@ -89,7 +91,7 @@ struct unsequenced_policy {
 
 inline constexpr unsequenced_policy unseq{__disable_user_instantiations_tag{}};
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 } // namespace execution
 
@@ -117,14 +119,14 @@ inline constexpr bool __is_unsequenced_execution_policy_impl<execution::__unsequ
 template <>
 inline constexpr bool __is_unsequenced_execution_policy_impl<execution::parallel_unsequenced_policy> = true;
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 template <>
 inline constexpr bool is_execution_policy_v<execution::unsequenced_policy> = true;
 
 template <>
 inline constexpr bool __is_unsequenced_execution_policy_impl<execution::unsequenced_policy> = true;
 
-#  endif
+#    endif
 
 template <class _Tp>
 struct is_execution_policy : bool_constant<is_execution_policy_v<_Tp>> {};
@@ -140,10 +142,11 @@ _LIBCPP_HIDE_FROM_ABI auto __remove_parallel_policy(const _ExecutionPolicy&) {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+#  endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_EXECUTION
diff --git a/libcxx/include/expected b/libcxx/include/expected
index 3c7ef336432a1..ac2415ff5a62d 100644
--- a/libcxx/include/expected
+++ b/libcxx/include/expected
@@ -38,19 +38,22 @@ namespace std {
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__expected/bad_expected_access.h>
-#  include <__expected/expected.h>
-#  include <__expected/unexpect.h>
-#  include <__expected/unexpected.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__expected/bad_expected_access.h>
+#    include <__expected/expected.h>
+#    include <__expected/unexpect.h>
+#    include <__expected/unexpected.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_EXPECTED
diff --git a/libcxx/include/fenv.h b/libcxx/include/fenv.h
index 5647f2b339555..dddf59d47f021 100644
--- a/libcxx/include/fenv.h
+++ b/libcxx/include/fenv.h
@@ -49,66 +49,69 @@ int feupdateenv(const fenv_t* envp);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if __has_include_next(<fenv.h>)
-#  include_next <fenv.h>
-#endif
+#  if __has_include_next(<fenv.h>)
+#    include_next <fenv.h>
+#  endif
 
-#ifdef __cplusplus
+#  ifdef __cplusplus
 
 extern "C++" {
 
-#  ifdef feclearexcept
-#    undef feclearexcept
-#  endif
+#    ifdef feclearexcept
+#      undef feclearexcept
+#    endif
 
-#  ifdef fegetexceptflag
-#    undef fegetexceptflag
-#  endif
+#    ifdef fegetexceptflag
+#      undef fegetexceptflag
+#    endif
 
-#  ifdef feraiseexcept
-#    undef feraiseexcept
-#  endif
+#    ifdef feraiseexcept
+#      undef feraiseexcept
+#    endif
 
-#  ifdef fesetexceptflag
-#    undef fesetexceptflag
-#  endif
+#    ifdef fesetexceptflag
+#      undef fesetexceptflag
+#    endif
 
-#  ifdef fetestexcept
-#    undef fetestexcept
-#  endif
+#    ifdef fetestexcept
+#      undef fetestexcept
+#    endif
 
-#  ifdef fegetround
-#    undef fegetround
-#  endif
+#    ifdef fegetround
+#      undef fegetround
+#    endif
 
-#  ifdef fesetround
-#    undef fesetround
-#  endif
+#    ifdef fesetround
+#      undef fesetround
+#    endif
 
-#  ifdef fegetenv
-#    undef fegetenv
-#  endif
+#    ifdef fegetenv
+#      undef fegetenv
+#    endif
 
-#  ifdef feholdexcept
-#    undef feholdexcept
-#  endif
+#    ifdef feholdexcept
+#      undef feholdexcept
+#    endif
 
-#  ifdef fesetenv
-#    undef fesetenv
-#  endif
+#    ifdef fesetenv
+#      undef fesetenv
+#    endif
 
-#  ifdef feupdateenv
-#    undef feupdateenv
-#  endif
+#    ifdef feupdateenv
+#      undef feupdateenv
+#    endif
 
 } // extern "C++"
 
-#endif // defined(__cplusplus)
+#  endif // defined(__cplusplus)
+#endif   // 0
 
 #endif // _LIBCPP_FENV_H
diff --git a/libcxx/include/filesystem b/libcxx/include/filesystem
index 6ea04df0a089b..f6f5dcb1cc405 100644
--- a/libcxx/include/filesystem
+++ b/libcxx/include/filesystem
@@ -533,45 +533,48 @@ inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_direct
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__filesystem/copy_options.h>
-#  include <__filesystem/directory_entry.h>
-#  include <__filesystem/directory_iterator.h>
-#  include <__filesystem/directory_options.h>
-#  include <__filesystem/file_status.h>
-#  include <__filesystem/file_time_type.h>
-#  include <__filesystem/file_type.h>
-#  include <__filesystem/filesystem_error.h>
-#  include <__filesystem/operations.h>
-#  include <__filesystem/path.h>
-#  include <__filesystem/path_iterator.h>
-#  include <__filesystem/perm_options.h>
-#  include <__filesystem/perms.h>
-#  include <__filesystem/recursive_directory_iterator.h>
-#  include <__filesystem/space_info.h>
-#  include <__filesystem/u8path.h>
-#endif
-
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__filesystem/copy_options.h>
+#    include <__filesystem/directory_entry.h>
+#    include <__filesystem/directory_iterator.h>
+#    include <__filesystem/directory_options.h>
+#    include <__filesystem/file_status.h>
+#    include <__filesystem/file_time_type.h>
+#    include <__filesystem/file_type.h>
+#    include <__filesystem/filesystem_error.h>
+#    include <__filesystem/operations.h>
+#    include <__filesystem/path.h>
+#    include <__filesystem/path_iterator.h>
+#    include <__filesystem/perm_options.h>
+#    include <__filesystem/perms.h>
+#    include <__filesystem/recursive_directory_iterator.h>
+#    include <__filesystem/space_info.h>
+#    include <__filesystem/u8path.h>
+#  endif
+
+#  include <version>
 
 // standard-mandated includes
 
 // [fs.filesystem.syn]
-#include <compare>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstdlib>
-#  include <cstring>
-#  include <iosfwd>
-#  include <new>
-#  include <system_error>
-#endif
+#  include <compare>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstdlib>
+#    include <cstring>
+#    include <iosfwd>
+#    include <new>
+#    include <system_error>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FILESYSTEM
diff --git a/libcxx/include/flat_map b/libcxx/include/flat_map
index 15d79dd1ddca3..b99eebfa9fc53 100644
--- a/libcxx/include/flat_map
+++ b/libcxx/include/flat_map
@@ -37,18 +37,21 @@ namespace std {
       erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
 */
 
-#include <__assert> // all public C++ headers provide the assertion handler
-#include <__config>
-#include <__flat_map/flat_map.h>
-#include <__flat_map/sorted_unique.h>
-#include <version>
+#if 0
+#else                 // 0
+#  include <__assert> // all public C++ headers provide the assertion handler
+#  include <__config>
+#  include <__flat_map/flat_map.h>
+#  include <__flat_map/sorted_unique.h>
+#  include <version>
 
 // standard required includes
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FLAT_MAP
diff --git a/libcxx/include/float.h b/libcxx/include/float.h
index d572866c1358b..2874323538b70 100644
--- a/libcxx/include/float.h
+++ b/libcxx/include/float.h
@@ -70,26 +70,29 @@
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if __has_include_next(<float.h>)
-#  include_next <float.h>
-#endif
+#  if __has_include_next(<float.h>)
+#    include_next <float.h>
+#  endif
 
-#ifdef __cplusplus
+#  ifdef __cplusplus
 
-#  ifndef FLT_EVAL_METHOD
-#    define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
-#  endif
+#    ifndef FLT_EVAL_METHOD
+#      define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
+#    endif
 
-#  ifndef DECIMAL_DIG
-#    define DECIMAL_DIG __DECIMAL_DIG__
-#  endif
+#    ifndef DECIMAL_DIG
+#      define DECIMAL_DIG __DECIMAL_DIG__
+#    endif
 
-#endif // __cplusplus
+#  endif // __cplusplus
+#endif   // 0
 
 #endif // _LIBCPP_FLOAT_H
diff --git a/libcxx/include/format b/libcxx/include/format
index 82a2e32286838..6ac1e928b26a5 100644
--- a/libcxx/include/format
+++ b/libcxx/include/format
@@ -191,70 +191,73 @@ namespace std {
 
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__format/buffer.h>
-#  include <__format/concepts.h>
-#  include <__format/container_adaptor.h>
-#  include <__format/enable_insertable.h>
-#  include <__format/escaped_output_table.h>
-#  include <__format/extended_grapheme_cluster_table.h>
-#  include <__format/format_arg.h>
-#  include <__format/format_arg_store.h>
-#  include <__format/format_args.h>
-#  include <__format/format_context.h>
-#  include <__format/format_error.h>
-#  include <__format/format_functions.h>
-#  include <__format/format_parse_context.h>
-#  include <__format/format_string.h>
-#  include <__format/format_to_n_result.h>
-#  include <__format/formatter.h>
-#  include <__format/formatter_bool.h>
-#  include <__format/formatter_char.h>
-#  include <__format/formatter_floating_point.h>
-#  include <__format/formatter_integer.h>
-#  include <__format/formatter_pointer.h>
-#  include <__format/formatter_string.h>
-#  include <__format/formatter_tuple.h>
-#  include <__format/parser_std_format_spec.h>
-#  include <__format/range_default_formatter.h>
-#  include <__format/range_formatter.h>
-#  include <__format/unicode.h>
-#  include <__fwd/format.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <array>
-#  include <cctype>
-#  include <cerrno>
-#  include <clocale>
-#  include <cmath>
-#  include <cstddef>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <cstring>
-#  include <initializer_list>
-#  include <limits>
-#  include <locale>
-#  include <new>
-#  include <optional>
-#  include <queue>
-#  include <stack>
-#  include <stdexcept>
-#  include <string>
-#  include <string_view>
-#  include <tuple>
-
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
-#    include <cwchar>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__format/buffer.h>
+#    include <__format/concepts.h>
+#    include <__format/container_adaptor.h>
+#    include <__format/enable_insertable.h>
+#    include <__format/escaped_output_table.h>
+#    include <__format/extended_grapheme_cluster_table.h>
+#    include <__format/format_arg.h>
+#    include <__format/format_arg_store.h>
+#    include <__format/format_args.h>
+#    include <__format/format_context.h>
+#    include <__format/format_error.h>
+#    include <__format/format_functions.h>
+#    include <__format/format_parse_context.h>
+#    include <__format/format_string.h>
+#    include <__format/format_to_n_result.h>
+#    include <__format/formatter.h>
+#    include <__format/formatter_bool.h>
+#    include <__format/formatter_char.h>
+#    include <__format/formatter_floating_point.h>
+#    include <__format/formatter_integer.h>
+#    include <__format/formatter_pointer.h>
+#    include <__format/formatter_string.h>
+#    include <__format/formatter_tuple.h>
+#    include <__format/parser_std_format_spec.h>
+#    include <__format/range_default_formatter.h>
+#    include <__format/range_formatter.h>
+#    include <__format/unicode.h>
+#    include <__fwd/format.h>
 #  endif
-#endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <array>
+#    include <cctype>
+#    include <cerrno>
+#    include <clocale>
+#    include <cmath>
+#    include <cstddef>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <cstring>
+#    include <initializer_list>
+#    include <limits>
+#    include <locale>
+#    include <new>
+#    include <optional>
+#    include <queue>
+#    include <stack>
+#    include <stdexcept>
+#    include <string>
+#    include <string_view>
+#    include <tuple>
+
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      include <cwchar>
+#    endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FORMAT
diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index 57730870ddf43..bc95b30f441fa 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -195,66 +195,68 @@ template <class T, class Allocator, class Predicate>
 
 */
 
-#include <__algorithm/comp.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__algorithm/min.h>
-#include <__config>
-#include <__cstddef/nullptr_t.h>
-#include <__iterator/distance.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/move_iterator.h>
-#include <__iterator/next.h>
-#include <__memory/addressof.h>
-#include <__memory/allocation_guard.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/construct_at.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/swap_allocator.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_const.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_pointer.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <limits>
-#include <new> // __launder
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/comp.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/min.h>
+#  include <__config>
+#  include <__cstddef/nullptr_t.h>
+#  include <__iterator/distance.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/move_iterator.h>
+#  include <__iterator/next.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocation_guard.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
+#  include <__memory/construct_at.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/swap_allocator.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_const.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_pointer.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <limits>
+#  include <new> // __launder
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [forward.list.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -281,7 +283,7 @@ struct __forward_node_traits {
   typedef __rebind_pointer_t<_NodePtr, void> __void_pointer;
 
 // TODO(LLVM 22): Remove this check
-#ifndef _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
+#  ifndef _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
   static_assert(sizeof(__begin_node_pointer) == sizeof(__node_pointer) && _LIBCPP_ALIGNOF(__begin_node_pointer) ==
                     _LIBCPP_ALIGNOF(__node_pointer),
                 "It looks like you are using std::forward_list with a fancy pointer type that thas a different "
@@ -289,7 +291,7 @@ struct __forward_node_traits {
                 "pointer (both of which are implementation details of the standard library). This means that your ABI "
                 "is being broken between LLVM 19 and LLVM 20. If you don't care about your ABI being broken, define "
                 "the _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI static __begin_node_pointer __as_iter_node(__begin_node_pointer __p) { return __p; }
   _LIBCPP_HIDE_FROM_ABI static __begin_node_pointer __as_iter_node(__node_pointer __p) {
@@ -323,7 +325,7 @@ struct __forward_list_node : public __begin_node_of<_Tp, _VoidPtr> {
 
   // We allow starting the lifetime of nodes without initializing the value held by the node,
   // since that is handled by the list itself in order to be allocator-aware.
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 private:
   union {
@@ -332,14 +334,14 @@ private:
 
 public:
   _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
-#else
+#  else
 
 private:
   _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
 
 public:
   _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {}
   _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {}
@@ -514,11 +516,11 @@ protected:
       : __before_begin_(__begin_node()), __alloc_(__a) {}
 
 public:
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI
   __forward_list_base(__forward_list_base&& __x) noexcept(is_nothrow_move_constructible<__node_allocator>::value);
   _LIBCPP_HIDE_FROM_ABI __forward_list_base(__forward_list_base&& __x, const allocator_type& __a);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   __forward_list_base(const __forward_list_base&)            = delete;
   __forward_list_base& operator=(const __forward_list_base&) = delete;
@@ -562,11 +564,11 @@ protected:
 
 public:
   _LIBCPP_HIDE_FROM_ABI void swap(__forward_list_base& __x)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
-#else
+#  else
       _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>);
-#endif
+#  endif
 
 protected:
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
@@ -586,7 +588,7 @@ private:
   }
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x) noexcept(
@@ -604,7 +606,7 @@ inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base
   }
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 __forward_list_base<_Tp, _Alloc>::~__forward_list_base() {
@@ -613,11 +615,11 @@ __forward_list_base<_Tp, _Alloc>::~__forward_list_base() {
 
 template <class _Tp, class _Alloc>
 inline void __forward_list_base<_Tp, _Alloc>::swap(__forward_list_base& __x)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
-#else
+#  else
     _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
-#endif
+#  endif
 {
   std::__swap_allocator(__alloc_, __x.__alloc_);
   using std::swap;
@@ -664,19 +666,19 @@ public:
 
   typedef typename __base::iterator iterator;
   typedef typename __base::const_iterator const_iterator;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   typedef size_type __remove_return_type;
-#else
+#  else
   typedef void __remove_return_type;
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI forward_list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {
   } // = default;
   _LIBCPP_HIDE_FROM_ABI explicit forward_list(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n, const allocator_type& __a);
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v);
 
   template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
@@ -690,20 +692,20 @@ public:
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI forward_list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
       : __base(__a) {
     prepend_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x);
   _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a);
 
   _LIBCPP_HIDE_FROM_ABI forward_list& operator=(const forward_list& __x);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x) noexcept(is_nothrow_move_constructible<__base>::value)
       : __base(std::move(__x)) {}
   _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a);
@@ -718,19 +720,19 @@ public:
   _LIBCPP_HIDE_FROM_ABI forward_list& operator=(initializer_list<value_type> __il);
 
   _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   // ~forward_list() = default;
 
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
   void _LIBCPP_HIDE_FROM_ABI assign(_InputIterator __f, _InputIterator __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
     __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
 
@@ -766,28 +768,28 @@ public:
   _LIBCPP_HIDE_FROM_ABI reference front() { return __base::__before_begin()->__next_->__get_value(); }
   _LIBCPP_HIDE_FROM_ABI const_reference front() const { return __base::__before_begin()->__next_->__get_value(); }
 
-#ifndef _LIBCPP_CXX03_LANG
-#  if _LIBCPP_STD_VER >= 17
+#  ifndef _LIBCPP_CXX03_LANG
+#    if _LIBCPP_STD_VER >= 17
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
-#  else
+#    else
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
     insert_range_after(cbefore_begin(), std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void pop_front();
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace_after(const_iterator __p, _Args&&... __args);
 
@@ -795,18 +797,18 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, initializer_list<value_type> __il) {
     return insert_after(__p, __il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, const value_type& __v);
   _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, size_type __n, const value_type& __v);
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI iterator insert_range_after(const_iterator __position, _Range&& __range) {
     return __insert_after_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
   }
-#endif
+#  endif
 
   template <class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI iterator __insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l);
@@ -815,11 +817,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __f, const_iterator __l);
 
   _LIBCPP_HIDE_FROM_ABI void swap(forward_list& __x)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT
-#else
+#  else
       _NOEXCEPT_(!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
-#endif
+#  endif
   {
     __base::swap(__x);
   }
@@ -842,13 +844,13 @@ public:
   _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
   template <class _BinaryPredicate>
   _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPredicate __binary_pred);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x) { merge(__x, __less<>()); }
   template <class _Compare>
   _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x, _Compare __comp) {
     merge(__x, std::move(__comp));
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x) { merge(__x, __less<>()); }
   template <class _Compare>
   _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x, _Compare __comp);
@@ -858,11 +860,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
 
 private:
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
   _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, false_type);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class _Iter, class _Sent>
   _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iter __f, _Sent __l);
@@ -875,7 +877,7 @@ private:
   static _LIBCPP_HIDDEN __node_pointer __sort(__node_pointer __f, difference_type __sz, _Compare& __comp);
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Alloc = allocator<__iter_value_type<_InputIterator>>,
           class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
@@ -887,14 +889,14 @@ template <class _InputIterator,
           class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
           class = enable_if_t<__is_allocator<_Alloc>::value> >
 forward_list(_InputIterator, _InputIterator, _Alloc) -> forward_list<__iter_value_type<_InputIterator>, _Alloc>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Alloc = allocator<ranges::range_value_t<_Range>>,
           class        = enable_if_t<__is_allocator<_Alloc>::value> >
 forward_list(from_range_t, _Range&&, _Alloc = _Alloc()) -> forward_list<ranges::range_value_t<_Range>, _Alloc>;
-#endif
+#  endif
 
 template <class _Tp, class _Alloc>
 inline forward_list<_Tp, _Alloc>::forward_list(const allocator_type& __a) : __base(__a) {}
@@ -908,7 +910,7 @@ forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
   }
 }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 template <class _Tp, class _Alloc>
 forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __base_alloc) : __base(__base_alloc) {
   if (__n > 0) {
@@ -917,7 +919,7 @@ forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __b
     }
   }
 }
-#endif
+#  endif
 
 template <class _Tp, class _Alloc>
 forward_list<_Tp, _Alloc>::forward_list(size_type __n, const value_type& __v) {
@@ -958,7 +960,7 @@ forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(const forward_li
   return *this;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Alloc>
 forward_list<_Tp, _Alloc>::forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a)
     : __base(std::move(__x), __a) {
@@ -1010,7 +1012,7 @@ inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(initializ
   return *this;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
@@ -1045,7 +1047,7 @@ void forward_list<_Tp, _Alloc>::assign(size_type __n, const value_type& __v) {
     erase_after(__i, __e);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 inline void forward_list<_Tp, _Alloc>::assign(initializer_list<value_type> __il) {
@@ -1054,17 +1056,17 @@ inline void forward_list<_Tp, _Alloc>::assign(initializer_list<value_type> __il)
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 typename forward_list<_Tp, _Alloc>::reference
-#  else
+#    else
 void
-#  endif
+#    endif
 forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
   __base::__before_begin()->__next_ =
       this->__create_node(/* next = */ __base::__before_begin()->__next_, std::forward<_Args>(__args)...);
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   return __base::__before_begin()->__next_->__get_value();
-#  endif
+#    endif
 }
 
 template <class _Tp, class _Alloc>
@@ -1073,7 +1075,7 @@ void forward_list<_Tp, _Alloc>::push_front(value_type&& __v) {
       this->__create_node(/* next = */ __base::__before_begin()->__next_, std::move(__v));
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 void forward_list<_Tp, _Alloc>::push_front(const value_type& __v) {
@@ -1087,7 +1089,7 @@ void forward_list<_Tp, _Alloc>::pop_front() {
   this->__delete_node(__p);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
@@ -1106,7 +1108,7 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
   return iterator(__r->__next_);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 typename forward_list<_Tp, _Alloc>::iterator
@@ -1123,13 +1125,13 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const
   if (__n > 0) {
     __node_pointer __first = this->__create_node(/* next = */ nullptr, __v);
     __node_pointer __last  = __first;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, __last = __last->__next_) {
         __last->__next_ = this->__create_node(/* next = */ nullptr, __v);
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (__first != nullptr) {
         __node_pointer __next = __first->__next_;
@@ -1138,7 +1140,7 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
     __r             = static_cast<__begin_node_pointer>(__last);
@@ -1163,13 +1165,13 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
     __node_pointer __first = this->__create_node(/* next = */ nullptr, *__f);
     __node_pointer __last  = __first;
 
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (++__f; __f != __l; ++__f, ((void)(__last = __last->__next_))) {
         __last->__next_ = this->__create_node(/* next = */ nullptr, *__f);
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (__first != nullptr) {
         __node_pointer __next = __first->__next_;
@@ -1178,7 +1180,7 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
@@ -1484,7 +1486,7 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const forward_list<_Tp, _Alloc>& __x, cons
   return (__ix == __ex) == (__iy == __ey);
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -1516,7 +1518,7 @@ operator<=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>
   return !(__y < __x);
 }
 
-#else // #if _LIBCPP_STD_VER <= 17
+#  else // #if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
@@ -1524,7 +1526,7 @@ operator<=>(const forward_list<_Tp, _Allocator>& __x, const forward_list<_Tp, _A
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // #if _LIBCPP_STD_VER <= 17
+#  endif // #if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI void swap(forward_list<_Tp, _Alloc>& __x, forward_list<_Tp, _Alloc>& __y)
@@ -1532,7 +1534,7 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(forward_list<_Tp, _Alloc>& __x, forward_l
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Tp, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
 erase_if(forward_list<_Tp, _Allocator>& __c, _Predicate __pred) {
@@ -1544,7 +1546,7 @@ inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
 erase(forward_list<_Tp, _Allocator>& __c, const _Up& __v) {
   return std::erase_if(__c, [&](auto& __elem) { return __elem == __v; });
 }
-#endif
+#  endif
 
 template <class _Tp, class _Allocator>
 struct __container_traits<forward_list<_Tp, _Allocator> > {
@@ -1559,30 +1561,31 @@ struct __container_traits<forward_list<_Tp, _Allocator> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _ValueT>
 using forward_list _LIBCPP_AVAILABILITY_PMR = std::forward_list<_ValueT, polymorphic_allocator<_ValueT>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <atomic>
-#  include <concepts>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <cstring>
-#  include <functional>
-#  include <iosfwd>
-#  include <iterator>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <atomic>
+#    include <concepts>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <cstring>
+#    include <functional>
+#    include <iosfwd>
+#    include <iterator>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FORWARD_LIST
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index e188bfc9042a7..9722440808e36 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -186,46 +186,48 @@ typedef basic_fstream<wchar_t> wfstream;
 
 */
 
-#include <__algorithm/max.h>
-#include <__assert>
-#include <__config>
-#include <__filesystem/path.h>
-#include <__fwd/fstream.h>
-#include <__locale>
-#include <__memory/addressof.h>
-#include <__memory/unique_ptr.h>
-#include <__ostream/basic_ostream.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_same.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <__utility/unreachable.h>
-#include <cstdio>
-#include <istream>
-#include <streambuf>
-#include <typeinfo>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__algorithm/max.h>
+#  include <__assert>
+#  include <__config>
+#  include <__filesystem/path.h>
+#  include <__fwd/fstream.h>
+#  include <__locale>
+#  include <__memory/addressof.h>
+#  include <__memory/unique_ptr.h>
+#  include <__ostream/basic_ostream.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_same.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <__utility/unreachable.h>
+#  include <cstdio>
+#  include <istream>
+#  include <streambuf>
+#  include <typeinfo>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
-#if !defined(_LIBCPP_MSVCRT) && !defined(_NEWLIB_VERSION)
-#  define _LIBCPP_HAS_OFF_T_FUNCTIONS 1
-#else
-#  define _LIBCPP_HAS_OFF_T_FUNCTIONS 0
-#endif
+#  if !defined(_LIBCPP_MSVCRT) && !defined(_NEWLIB_VERSION)
+#    define _LIBCPP_HAS_OFF_T_FUNCTIONS 1
+#  else
+#    define _LIBCPP_HAS_OFF_T_FUNCTIONS 0
+#  endif
 
-#if _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
+#  if _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API)
+#    if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API)
 _LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file) noexcept;
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 class _LIBCPP_TEMPLATE_VIS basic_filebuf : public basic_streambuf<_CharT, _Traits> {
@@ -236,15 +238,15 @@ public:
   typedef typename traits_type::pos_type pos_type;
   typedef typename traits_type::off_type off_type;
   typedef typename traits_type::state_type state_type;
-#  if _LIBCPP_STD_VER >= 26
-#    if defined(_LIBCPP_WIN32API)
+#    if _LIBCPP_STD_VER >= 26
+#      if defined(_LIBCPP_WIN32API)
   using native_handle_type = void*; // HANDLE
-#    elif __has_include(<unistd.h>)
+#      elif __has_include(<unistd.h>)
   using native_handle_type = int; // POSIX file descriptor
-#    else
-#      error "Provide a native file handle!"
+#      else
+#        error "Provide a native file handle!"
+#      endif
 #    endif
-#  endif
 
   // 27.9.1.2 Constructors/destructor:
   basic_filebuf();
@@ -258,36 +260,36 @@ public:
   // 27.9.1.4 Members:
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   basic_filebuf* open(const char* __s, ios_base::openmode __mode);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   basic_filebuf* open(const wchar_t* __s, ios_base::openmode __mode);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI basic_filebuf* open(const string& __s, ios_base::openmode __mode);
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI basic_filebuf*
   open(const filesystem::path& __p, ios_base::openmode __mode) {
     return open(__p.c_str(), __mode);
   }
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI basic_filebuf* __open(int __fd, ios_base::openmode __mode);
   basic_filebuf* close();
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
     _LIBCPP_ASSERT_UNCATEGORIZED(this->is_open(), "File must be opened");
-#    if defined(_LIBCPP_WIN32API)
+#      if defined(_LIBCPP_WIN32API)
     return std::__filebuf_windows_native_handle(__file_);
-#    elif __has_include(<unistd.h>)
+#      elif __has_include(<unistd.h>)
     return fileno(__file_);
-#    else
-#      error "Provide a way to determine the file native handle!"
-#    endif
+#      else
+#        error "Provide a way to determine the file native handle!"
+#      endif
   }
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
   _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT;
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI inline static const wchar_t* __make_mdwstring(ios_base::openmode __mode) _NOEXCEPT;
-#  endif
+#    endif
 
 protected:
   // 27.9.1.5 Overridden virtual functions:
@@ -489,14 +491,14 @@ inline basic_filebuf<_CharT, _Traits>& basic_filebuf<_CharT, _Traits>::operator=
 
 template <class _CharT, class _Traits>
 basic_filebuf<_CharT, _Traits>::~basic_filebuf() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     close();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   if (__owns_eb_)
     delete[] __extbuf_;
   if (__owns_ib_)
@@ -616,7 +618,7 @@ const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(ios_base::openmode _
   case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
   case ios_base::in | ios_base::app | ios_base::binary:
     return "a+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   case ios_base::out | ios_base::noreplace:
   case ios_base::out | ios_base::trunc | ios_base::noreplace:
     return "wx" _LIBCPP_FOPEN_CLOEXEC_MODE;
@@ -627,14 +629,14 @@ const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(ios_base::openmode _
     return "wbx" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
     return "w+bx" _LIBCPP_FOPEN_CLOEXEC_MODE;
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
   default:
     return nullptr;
   }
   __libcpp_unreachable();
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmode __mode) _NOEXCEPT {
   switch (__mode & ~ios_base::ate) {
@@ -668,7 +670,7 @@ const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmo
   case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
   case ios_base::in | ios_base::app | ios_base::binary:
     return L"a+b";
-#    if _LIBCPP_STD_VER >= 23
+#      if _LIBCPP_STD_VER >= 23
   case ios_base::out | ios_base::noreplace:
   case ios_base::out | ios_base::trunc | ios_base::noreplace:
     return L"wx";
@@ -679,13 +681,13 @@ const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmo
     return L"wbx";
   case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
     return L"w+bx";
-#    endif // _LIBCPP_STD_VER >= 23
+#      endif // _LIBCPP_STD_VER >= 23
   default:
     return nullptr;
   }
   __libcpp_unreachable();
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
@@ -709,7 +711,7 @@ inline basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::__open(in
   return __do_open(fdopen(__fd, __mdstr), __mode);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 // This is basically the same as the char* overload except that it uses _wfopen
 // and long mode strings.
 template <class _CharT, class _Traits>
@@ -722,7 +724,7 @@ basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const wchar
 
   return __do_open(_wfopen(__s, __mdstr), __mode);
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 inline basic_filebuf<_CharT, _Traits>*
@@ -933,15 +935,15 @@ basic_filebuf<_CharT, _Traits>::seekoff(off_type __off, ios_base::seekdir __way,
   default:
     return pos_type(off_type(-1));
   }
-#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
+#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
   if (fseek(__file_, __width > 0 ? __width * __off : 0, __whence))
     return pos_type(off_type(-1));
   pos_type __r = ftell(__file_);
-#  else
+#    else
   if (::fseeko(__file_, __width > 0 ? __width * __off : 0, __whence))
     return pos_type(off_type(-1));
   pos_type __r = ftello(__file_);
-#  endif
+#    endif
   __r.state(__st_);
   return __r;
 }
@@ -951,13 +953,13 @@ typename basic_filebuf<_CharT, _Traits>::pos_type
 basic_filebuf<_CharT, _Traits>::seekpos(pos_type __sp, ios_base::openmode) {
   if (__file_ == nullptr || sync())
     return pos_type(off_type(-1));
-#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
+#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
   if (fseek(__file_, __sp, SEEK_SET))
     return pos_type(off_type(-1));
-#  else
+#    else
   if (::fseeko(__file_, __sp, SEEK_SET))
     return pos_type(off_type(-1));
-#  endif
+#    endif
   __st_ = __sp.state();
   return __sp;
 }
@@ -1004,13 +1006,13 @@ int basic_filebuf<_CharT, _Traits>::sync() {
         }
       }
     }
-#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
+#    if !_LIBCPP_HAS_OFF_T_FUNCTIONS
     if (fseek(__file_, -__c, SEEK_CUR))
       return -1;
-#  else
+#    else
     if (::fseeko(__file_, -__c, SEEK_CUR))
       return -1;
-#  endif
+#    endif
     if (__update_st)
       __st_ = __state;
     __extbufnext_ = __extbufend_ = __extbuf_;
@@ -1096,42 +1098,42 @@ public:
   typedef typename traits_type::int_type int_type;
   typedef typename traits_type::pos_type pos_type;
   typedef typename traits_type::off_type off_type;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI basic_ifstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const string& __s, ios_base::openmode __mode = ios_base::in);
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
       _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const _Tp& __p, ios_base::openmode __mode = ios_base::in)
       : basic_ifstream(__p.c_str(), __mode) {}
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI basic_ifstream(basic_ifstream&& __rhs);
   _LIBCPP_HIDE_FROM_ABI basic_ifstream& operator=(basic_ifstream&& __rhs);
   _LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream& __rhs);
 
   _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   void open(const char* __s, ios_base::openmode __mode = ios_base::in);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
-#  endif
+#    endif
   void open(const string& __s, ios_base::openmode __mode = ios_base::in);
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
   open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in) {
     return open(__p.c_str(), __mode);
   }
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI void __open(int __fd, ios_base::openmode __mode);
   _LIBCPP_HIDE_FROM_ABI void close();
@@ -1151,14 +1153,14 @@ inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const char* __s, ios_base
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_istream<char_type, traits_type>(std::addressof(__sb_)) {
   if (__sb_.open(__s, __mode | ios_base::in) == nullptr)
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 // extension
 template <class _CharT, class _Traits>
@@ -1210,7 +1212,7 @@ void basic_ifstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode _
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_ifstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode | ios_base::in))
@@ -1218,7 +1220,7 @@ void basic_ifstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmod
   else
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 void basic_ifstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
@@ -1252,45 +1254,45 @@ public:
   typedef typename traits_type::int_type int_type;
   typedef typename traits_type::pos_type pos_type;
   typedef typename traits_type::off_type off_type;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI basic_ofstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const char* __s, ios_base::openmode __mode = ios_base::out);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const string& __s, ios_base::openmode __mode = ios_base::out);
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
       _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const _Tp& __p, ios_base::openmode __mode = ios_base::out)
       : basic_ofstream(__p.c_str(), __mode) {}
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI basic_ofstream(basic_ofstream&& __rhs);
   _LIBCPP_HIDE_FROM_ABI basic_ofstream& operator=(basic_ofstream&& __rhs);
   _LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream& __rhs);
 
   _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   void open(const char* __s, ios_base::openmode __mode = ios_base::out);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
-#  endif
+#    endif
   void open(const string& __s, ios_base::openmode __mode = ios_base::out);
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
   open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::out) {
     return open(__p.c_str(), __mode);
   }
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI void __open(int __fd, ios_base::openmode __mode);
   _LIBCPP_HIDE_FROM_ABI void close();
@@ -1310,14 +1312,14 @@ inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const char* __s, ios_base
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_ostream<char_type, traits_type>(std::addressof(__sb_)) {
   if (__sb_.open(__s, __mode | ios_base::out) == nullptr)
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 // extension
 template <class _CharT, class _Traits>
@@ -1369,7 +1371,7 @@ void basic_ofstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode _
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_ofstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode | ios_base::out))
@@ -1377,7 +1379,7 @@ void basic_ofstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmod
   else
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 void basic_ofstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
@@ -1411,26 +1413,26 @@ public:
   typedef typename traits_type::int_type int_type;
   typedef typename traits_type::pos_type pos_type;
   typedef typename traits_type::off_type off_type;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type;
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI basic_fstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const char* __s,
                                                ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const wchar_t* __s,
                                                ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const string& __s,
                                                ios_base::openmode __mode = ios_base::in | ios_base::out);
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(
       const _Tp& __p, ios_base::openmode __mode = ios_base::in | ios_base::out)
       : basic_fstream(__p.c_str(), __mode) {}
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI basic_fstream(basic_fstream&& __rhs);
 
@@ -1439,22 +1441,22 @@ public:
   _LIBCPP_HIDE_FROM_ABI void swap(basic_fstream& __rhs);
 
   _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   _LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  endif
+#    endif
   _LIBCPP_HIDE_FROM_ABI void open(const string& __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
   open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in | ios_base::out) {
     return open(__p.c_str(), __mode);
   }
-#  endif // _LIBCPP_STD_VER >= 17
+#    endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI void close();
 
@@ -1473,14 +1475,14 @@ inline basic_fstream<_CharT, _Traits>::basic_fstream(const char* __s, ios_base::
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_fstream<_CharT, _Traits>::basic_fstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_iostream<char_type, traits_type>(std::addressof(__sb_)) {
   if (__sb_.open(__s, __mode) == nullptr)
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 inline basic_fstream<_CharT, _Traits>::basic_fstream(const string& __s, ios_base::openmode __mode)
@@ -1532,7 +1534,7 @@ void basic_fstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode __
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_fstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode))
@@ -1540,7 +1542,7 @@ void basic_fstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode
   else
     this->setstate(ios_base::failbit);
 }
-#  endif
+#    endif
 
 template <class _CharT, class _Traits>
 void basic_fstream<_CharT, _Traits>::open(const string& __s, ios_base::openmode __mode) {
@@ -1556,32 +1558,33 @@ inline void basic_fstream<_CharT, _Traits>::close() {
     this->setstate(ios_base::failbit);
 }
 
-#  if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
+#    if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ifstream<char>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ofstream<char>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_filebuf<char>;
-#  endif
+#    endif
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_FILESYSTEM && _LIBCPP_HAS_LOCALIZATION
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <limits>
-#  include <mutex>
-#  include <new>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
-#  include <filesystem>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <limits>
+#    include <mutex>
+#    include <new>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <filesystem>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FSTREAM
diff --git a/libcxx/include/functional b/libcxx/include/functional
index 55b50eea84b73..69f71314f0514 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -527,72 +527,75 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 
 */
 
-#include <__config>
-
-#include <__functional/binary_function.h>
-#include <__functional/binary_negate.h>
-#include <__functional/bind.h>
-#include <__functional/binder1st.h>
-#include <__functional/binder2nd.h>
-#include <__functional/hash.h>
-#include <__functional/mem_fn.h> // TODO: deprecate
-#include <__functional/mem_fun_ref.h>
-#include <__functional/operations.h>
-#include <__functional/pointer_to_binary_function.h>
-#include <__functional/pointer_to_unary_function.h>
-#include <__functional/reference_wrapper.h>
-#include <__functional/unary_function.h>
-#include <__functional/unary_negate.h>
-
-#ifndef _LIBCPP_CXX03_LANG
-#  include <__functional/function.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__functional/boyer_moore_searcher.h>
-#  include <__functional/default_searcher.h>
-#  include <__functional/invoke.h>
-#  include <__functional/not_fn.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__functional/bind_back.h>
-#  include <__functional/bind_front.h>
-#  include <__functional/identity.h>
-#  include <__functional/ranges_operations.h>
-#  include <__type_traits/unwrap_ref.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && defined(_LIBCPP_CXX03_LANG)
-#  include <limits>
-#  include <new>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
-#  include <array>
-#  include <initializer_list>
-#  include <unordered_map>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <exception>
-#  include <iosfwd>
-#  include <memory>
-#  include <stdexcept>
-#  include <tuple>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#  include <vector>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__functional/binary_function.h>
+#  include <__functional/binary_negate.h>
+#  include <__functional/bind.h>
+#  include <__functional/binder1st.h>
+#  include <__functional/binder2nd.h>
+#  include <__functional/hash.h>
+#  include <__functional/mem_fn.h> // TODO: deprecate
+#  include <__functional/mem_fun_ref.h>
+#  include <__functional/operations.h>
+#  include <__functional/pointer_to_binary_function.h>
+#  include <__functional/pointer_to_unary_function.h>
+#  include <__functional/reference_wrapper.h>
+#  include <__functional/unary_function.h>
+#  include <__functional/unary_negate.h>
+
+#  ifndef _LIBCPP_CXX03_LANG
+#    include <__functional/function.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__functional/boyer_moore_searcher.h>
+#    include <__functional/default_searcher.h>
+#    include <__functional/invoke.h>
+#    include <__functional/not_fn.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__functional/bind_back.h>
+#    include <__functional/bind_front.h>
+#    include <__functional/identity.h>
+#    include <__functional/ranges_operations.h>
+#    include <__type_traits/unwrap_ref.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && defined(_LIBCPP_CXX03_LANG)
+#    include <limits>
+#    include <new>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#    include <array>
+#    include <initializer_list>
+#    include <unordered_map>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <exception>
+#    include <iosfwd>
+#    include <memory>
+#    include <stdexcept>
+#    include <tuple>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#    include <vector>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/future b/libcxx/include/future
index 5b2e9eed88e35..62f59682d404d 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -362,59 +362,61 @@ template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_THREADS
-
-#  include <__assert>
-#  include <__chrono/duration.h>
-#  include <__chrono/steady_clock.h>
-#  include <__chrono/time_point.h>
-#  include <__condition_variable/condition_variable.h>
-#  include <__exception/exception_ptr.h>
-#  include <__memory/addressof.h>
-#  include <__memory/allocator.h>
-#  include <__memory/allocator_arg_t.h>
-#  include <__memory/allocator_destructor.h>
-#  include <__memory/allocator_traits.h>
-#  include <__memory/compressed_pair.h>
-#  include <__memory/pointer_traits.h>
-#  include <__memory/shared_count.h>
-#  include <__memory/unique_ptr.h>
-#  include <__memory/uses_allocator.h>
-#  include <__mutex/lock_guard.h>
-#  include <__mutex/mutex.h>
-#  include <__mutex/unique_lock.h>
-#  include <__system_error/error_category.h>
-#  include <__system_error/error_code.h>
-#  include <__system_error/error_condition.h>
-#  include <__thread/thread.h>
-#  include <__type_traits/add_lvalue_reference.h>
-#  include <__type_traits/aligned_storage.h>
-#  include <__type_traits/conditional.h>
-#  include <__type_traits/decay.h>
-#  include <__type_traits/enable_if.h>
-#  include <__type_traits/invoke.h>
-#  include <__type_traits/is_same.h>
-#  include <__type_traits/remove_cvref.h>
-#  include <__type_traits/remove_reference.h>
-#  include <__type_traits/strip_signature.h>
-#  include <__type_traits/underlying_type.h>
-#  include <__utility/auto_cast.h>
-#  include <__utility/forward.h>
-#  include <__utility/move.h>
-#  include <__utility/swap.h>
-#  include <new>
-#  include <stdexcept>
-#  include <tuple>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_THREADS
+
+#    include <__assert>
+#    include <__chrono/duration.h>
+#    include <__chrono/steady_clock.h>
+#    include <__chrono/time_point.h>
+#    include <__condition_variable/condition_variable.h>
+#    include <__exception/exception_ptr.h>
+#    include <__memory/addressof.h>
+#    include <__memory/allocator.h>
+#    include <__memory/allocator_arg_t.h>
+#    include <__memory/allocator_destructor.h>
+#    include <__memory/allocator_traits.h>
+#    include <__memory/compressed_pair.h>
+#    include <__memory/pointer_traits.h>
+#    include <__memory/shared_count.h>
+#    include <__memory/unique_ptr.h>
+#    include <__memory/uses_allocator.h>
+#    include <__mutex/lock_guard.h>
+#    include <__mutex/mutex.h>
+#    include <__mutex/unique_lock.h>
+#    include <__system_error/error_category.h>
+#    include <__system_error/error_code.h>
+#    include <__system_error/error_condition.h>
+#    include <__thread/thread.h>
+#    include <__type_traits/add_lvalue_reference.h>
+#    include <__type_traits/aligned_storage.h>
+#    include <__type_traits/conditional.h>
+#    include <__type_traits/decay.h>
+#    include <__type_traits/enable_if.h>
+#    include <__type_traits/invoke.h>
+#    include <__type_traits/is_same.h>
+#    include <__type_traits/remove_cvref.h>
+#    include <__type_traits/remove_reference.h>
+#    include <__type_traits/strip_signature.h>
+#    include <__type_traits/underlying_type.h>
+#    include <__utility/auto_cast.h>
+#    include <__utility/forward.h>
+#    include <__utility/move.h>
+#    include <__utility/swap.h>
+#    include <new>
+#    include <stdexcept>
+#    include <tuple>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -426,16 +428,16 @@ _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(future_errc)
 template <>
 struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<future_errc> : public true_type {};
 
-#  ifdef _LIBCPP_CXX03_LANG
+#    ifdef _LIBCPP_CXX03_LANG
 template <>
 struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<future_errc::__lx> : public true_type {};
-#  endif
+#    endif
 
 // enum class launch
 _LIBCPP_DECLARE_STRONG_ENUM(launch){async = 1, deferred = 2, any = async | deferred};
 _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(launch)
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
 
 typedef underlying_type<launch>::type __launch_underlying_type;
 
@@ -470,7 +472,7 @@ inline _LIBCPP_HIDE_FROM_ABI launch& operator^=(launch& __x, launch __y) {
   return __x;
 }
 
-#  endif // !_LIBCPP_CXX03_LANG
+#    endif // !_LIBCPP_CXX03_LANG
 
 // enum class future_status
 _LIBCPP_DECLARE_STRONG_ENUM(future_status){ready, timeout, deferred};
@@ -497,9 +499,9 @@ class _LIBCPP_EXPORTED_FROM_ABI future_error : public logic_error {
   friend class promise;
 
 public:
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI explicit future_error(future_errc __ec) : future_error(std::make_error_code(__ec)) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
 
@@ -509,12 +511,12 @@ public:
 
 // Declared above std::future_error
 void __throw_future_error(future_errc __ev) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw future_error(make_error_code(__ev));
-#  else
+#    else
   (void)__ev;
   _LIBCPP_VERBOSE_ABORT("future_error was thrown in -fno-exceptions mode");
-#  endif
+#    endif
 }
 
 class _LIBCPP_EXPORTED_FROM_ABI __assoc_sub_state : public __shared_count {
@@ -788,15 +790,15 @@ inline __deferred_assoc_state<_Rp, _Fp>::__deferred_assoc_state(_Fp&& __f) : __f
 
 template <class _Rp, class _Fp>
 void __deferred_assoc_state<_Rp, _Fp>::__execute() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->set_value(__func_());
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Fp>
@@ -818,16 +820,16 @@ inline __deferred_assoc_state<void, _Fp>::__deferred_assoc_state(_Fp&& __f) : __
 
 template <class _Fp>
 void __deferred_assoc_state<void, _Fp>::__execute() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __func_();
     this->set_value();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class _Fp>
@@ -849,15 +851,15 @@ inline __async_assoc_state<_Rp, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(s
 
 template <class _Rp, class _Fp>
 void __async_assoc_state<_Rp, _Fp>::__execute() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->set_value(__func_());
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class _Fp>
@@ -885,16 +887,16 @@ inline __async_assoc_state<void, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(
 
 template <class _Fp>
 void __async_assoc_state<void, _Fp>::__execute() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __func_();
     this->set_value();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Fp>
@@ -1622,11 +1624,11 @@ public:
   template <class _Fp, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit packaged_task(_Fp&& __f) : __f_(std::forward<_Fp>(__f)) {}
 
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
   template <class _Fp, class _Allocator, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI packaged_task(allocator_arg_t, const _Allocator& __a, _Fp&& __f)
       : __f_(allocator_arg_t(), __a, std::forward<_Fp>(__f)), __p_(allocator_arg_t(), __a) {}
-#  endif
+#    endif
   // ~packaged_task() = default;
 
   // no copy
@@ -1665,15 +1667,15 @@ void packaged_task<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __args) {
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __p_.set_value(__f_(std::forward<_ArgTypes>(__args)...));
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class... _ArgTypes>
@@ -1682,15 +1684,15 @@ void packaged_task<_Rp(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... __
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __p_.set_value_at_thread_exit(__f_(std::forward<_ArgTypes>(__args)...));
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception_at_thread_exit(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class... _ArgTypes>
@@ -1714,11 +1716,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI packaged_task() _NOEXCEPT : __p_(nullptr) {}
   template <class _Fp, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit packaged_task(_Fp&& __f) : __f_(std::forward<_Fp>(__f)) {}
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
   template <class _Fp, class _Allocator, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, packaged_task>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI packaged_task(allocator_arg_t, const _Allocator& __a, _Fp&& __f)
       : __f_(allocator_arg_t(), __a, std::forward<_Fp>(__f)), __p_(allocator_arg_t(), __a) {}
-#  endif
+#    endif
   // ~packaged_task() = default;
 
   // no copy
@@ -1751,7 +1753,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reset();
 };
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 
 template <class _Rp, class... _Args>
 packaged_task(_Rp (*)(_Args...)) -> packaged_task<_Rp(_Args...)>;
@@ -1759,7 +1761,7 @@ packaged_task(_Rp (*)(_Args...)) -> packaged_task<_Rp(_Args...)>;
 template <class _Fp, class _Stripped = typename __strip_signature<decltype(&_Fp::operator())>::type>
 packaged_task(_Fp) -> packaged_task<_Stripped>;
 
-#  endif
+#    endif
 
 template <class... _ArgTypes>
 void packaged_task<void(_ArgTypes...)>::operator()(_ArgTypes... __args) {
@@ -1767,16 +1769,16 @@ void packaged_task<void(_ArgTypes...)>::operator()(_ArgTypes... __args) {
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __f_(std::forward<_ArgTypes>(__args)...);
     __p_.set_value();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class... _ArgTypes>
@@ -1785,16 +1787,16 @@ void packaged_task<void(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... _
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __f_(std::forward<_ArgTypes>(__args)...);
     __p_.set_value_at_thread_exit();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception_at_thread_exit(current_exception());
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class... _ArgTypes>
@@ -1810,10 +1812,10 @@ swap(packaged_task<_Rp(_ArgTypes...)>& __x, packaged_task<_Rp(_ArgTypes...)>& __
   __x.swap(__y);
 }
 
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
 template <class _Callable, class _Alloc>
 struct _LIBCPP_TEMPLATE_VIS uses_allocator<packaged_task<_Callable>, _Alloc> : public true_type {};
-#  endif
+#    endif
 
 template <class _Rp, class _Fp>
 _LIBCPP_HIDE_FROM_ABI future<_Rp> __make_deferred_assoc_state(_Fp&& __f) {
@@ -1830,7 +1832,7 @@ _LIBCPP_HIDE_FROM_ABI future<_Rp> __make_async_assoc_state(_Fp&& __f) {
   return future<_Rp>(__h.get());
 }
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
 
 template <class _Fp, class... _Args>
 class _LIBCPP_HIDDEN __async_func {
@@ -1866,18 +1868,18 @@ async(launch __policy, _Fp&& __f, _Args&&... __args) {
   typedef __async_func<__decay_t<_Fp>, __decay_t<_Args>...> _BF;
   typedef typename _BF::_Rp _Rp;
 
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
   try {
-#    endif
+#      endif
     if (__does_policy_contain(__policy, launch::async))
       return std::__make_async_assoc_state<_Rp>(
           _BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     if (__policy == launch::async)
       throw;
   }
-#    endif
+#      endif
 
   if (__does_policy_contain(__policy, launch::deferred))
     return std::__make_deferred_assoc_state<_Rp>(
@@ -1891,7 +1893,7 @@ async(_Fp&& __f, _Args&&... __args) {
   return std::async(launch::any, std::forward<_Fp>(__f), std::forward<_Args>(__args)...);
 }
 
-#  endif // C++03
+#    endif // C++03
 
 // shared_future
 
@@ -2068,19 +2070,20 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
-#  include <chrono>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#    include <chrono>
+#  endif
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <cstdlib>
-#  include <exception>
-#  include <iosfwd>
-#  include <system_error>
-#  include <thread>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <cstdlib>
+#    include <exception>
+#    include <iosfwd>
+#    include <system_error>
+#    include <thread>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_FUTURE
diff --git a/libcxx/include/initializer_list b/libcxx/include/initializer_list
index 8b9325069c129..46ffb95f83a69 100644
--- a/libcxx/include/initializer_list
+++ b/libcxx/include/initializer_list
@@ -42,18 +42,20 @@ template<class E> const E* end(initializer_list<E> il) noexcept; // constexpr in
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
+#  include <version>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 namespace std // purposefully not versioned
 {
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Ep>
 class _LIBCPP_TEMPLATE_VIS initializer_list {
@@ -92,12 +94,13 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end(initia
   return __il.end();
 }
 
-#endif // !defined(_LIBCPP_CXX03_LANG)
+#  endif // !defined(_LIBCPP_CXX03_LANG)
 
 } // namespace std
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_INITIALIZER_LIST
diff --git a/libcxx/include/inttypes.h b/libcxx/include/inttypes.h
index 8664412bd52ff..552324516c11f 100644
--- a/libcxx/include/inttypes.h
+++ b/libcxx/include/inttypes.h
@@ -235,30 +235,33 @@ uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 /* C99 stdlib (e.g. glibc < 2.18) does not provide format macros needed
    for C++11 unless __STDC_FORMAT_MACROS is defined
 */
-#if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS)
-#  define __STDC_FORMAT_MACROS
-#endif
+#  if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS)
+#    define __STDC_FORMAT_MACROS
+#  endif
 
-#if __has_include_next(<inttypes.h>)
-#  include_next <inttypes.h>
-#endif
+#  if __has_include_next(<inttypes.h>)
+#    include_next <inttypes.h>
+#  endif
 
-#ifdef __cplusplus
+#  ifdef __cplusplus
 
-#  include <stdint.h>
+#    include <stdint.h>
 
-#  undef imaxabs
-#  undef imaxdiv
+#    undef imaxabs
+#    undef imaxdiv
 
-#endif // __cplusplus
+#  endif // __cplusplus
+#endif   // 0
 
 #endif // _LIBCPP_INTTYPES_H
diff --git a/libcxx/include/iomanip b/libcxx/include/iomanip
index 6118f8fb5b417..84842dc3fa6ca 100644
--- a/libcxx/include/iomanip
+++ b/libcxx/include/iomanip
@@ -42,19 +42,21 @@ template <class charT, class traits, class Allocator>
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_LOCALIZATION
+#  if _LIBCPP_HAS_LOCALIZATION
 
-#  include <__ostream/put_character_sequence.h>
-#  include <ios>
-#  include <iosfwd>
-#  include <locale>
-#  include <version>
+#    include <__ostream/put_character_sequence.h>
+#    include <ios>
+#    include <iosfwd>
+#    include <locale>
+#    include <version>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -237,9 +239,9 @@ public:
 template <class _CharT, class _Traits, class _MoneyT>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_istream<_CharT, _Traits>::sentry __s(__is);
     if (__s) {
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
@@ -249,11 +251,11 @@ operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x) {
       __mf.get(_Ip(__is), _Ip(), __x.__intl_, __is, __err, __x.__mon_);
       __is.setstate(__err);
     }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __is.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   return __is;
 }
 
@@ -286,9 +288,9 @@ public:
 template <class _CharT, class _Traits, class _MoneyT>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Op;
@@ -297,11 +299,11 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x) {
       if (__mf.put(_Op(__os), __x.__intl_, __os, __os.fill(), __x.__mon_).failed())
         __os.setstate(ios_base::badbit);
     }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -334,9 +336,9 @@ public:
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_istream<_CharT, _Traits>::sentry __s(__is);
     if (__s) {
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
@@ -346,11 +348,11 @@ operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x) {
       __tf.get(_Ip(__is), _Ip(), __is, __err, __x.__tm_, __x.__fmt_, __x.__fmt_ + _Traits::length(__x.__fmt_));
       __is.setstate(__err);
     }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __is.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   return __is;
 }
 
@@ -383,9 +385,9 @@ public:
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Op;
@@ -395,11 +397,11 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x) {
               .failed())
         __os.setstate(ios_base::badbit);
     }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -511,7 +513,7 @@ __quoted(basic_string<_CharT, _Traits, _Allocator>& __s, _CharT __delim = _CharT
   return __quoted_proxy<_CharT, _Traits, _Allocator>(__s, __delim, __escape);
 }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
 
 template <class _CharT>
 _LIBCPP_HIDE_FROM_ABI auto quoted(const _CharT* __s, _CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\')) {
@@ -541,25 +543,26 @@ quoted(basic_string_view<_CharT, _Traits> __sv, _CharT __delim = _CharT('"'), _C
   return __quoted_output_proxy<_CharT, _Traits>(__sv.data(), __sv.data() + __sv.size(), __delim, __escape);
 }
 
-#  endif // _LIBCPP_STD_VER >= 14
+#    endif // _LIBCPP_STD_VER >= 14
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_HAS_LOCALIZATION
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <array>
-#  include <bitset>
-#  include <deque>
-#  include <format>
-#  include <functional>
-#  include <istream>
-#  include <ostream>
-#  include <print>
-#  include <queue>
-#  include <stack>
-#  include <unordered_map>
-#  include <vector>
-#endif
+#  endif // _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <array>
+#    include <bitset>
+#    include <deque>
+#    include <format>
+#    include <functional>
+#    include <istream>
+#    include <ostream>
+#    include <print>
+#    include <queue>
+#    include <stack>
+#    include <unordered_map>
+#    include <vector>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_IOMANIP
diff --git a/libcxx/include/ios b/libcxx/include/ios
index 2eb32f296aafc..23e8fdf1a6075 100644
--- a/libcxx/include/ios
+++ b/libcxx/include/ios
@@ -211,37 +211,39 @@ storage-class-specifier const error_category& iostream_category() noexcept;
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_LOCALIZATION
-
-#  include <__fwd/ios.h>
-#  include <__ios/fpos.h>
-#  include <__locale>
-#  include <__memory/addressof.h>
-#  include <__system_error/error_category.h>
-#  include <__system_error/error_code.h>
-#  include <__system_error/error_condition.h>
-#  include <__system_error/system_error.h>
-#  include <__utility/swap.h>
-#  include <__verbose_abort>
-#  include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+
+#    include <__fwd/ios.h>
+#    include <__ios/fpos.h>
+#    include <__locale>
+#    include <__memory/addressof.h>
+#    include <__system_error/error_category.h>
+#    include <__system_error/error_code.h>
+#    include <__system_error/error_condition.h>
+#    include <__system_error/system_error.h>
+#    include <__utility/swap.h>
+#    include <__verbose_abort>
+#    include <version>
 
 // standard-mandated includes
 
 // [ios.syn]
-#  include <iosfwd>
+#    include <iosfwd>
 
-#  if _LIBCPP_HAS_ATOMIC_HEADER
-#    include <__atomic/atomic.h> // for __xindex_
-#  endif
+#    if _LIBCPP_HAS_ATOMIC_HEADER
+#      include <__atomic/atomic.h> // for __xindex_
+#    endif
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -284,20 +286,20 @@ public:
   static const openmode in     = 0x08;
   static const openmode out    = 0x10;
   static const openmode trunc  = 0x20;
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   static const openmode noreplace = 0x40;
-#  endif
+#    endif
 
   enum seekdir { beg, cur, end };
 
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
   typedef iostate io_state;
   typedef openmode open_mode;
   typedef seekdir seek_dir;
 
   typedef std::streamoff streamoff;
   typedef std::streampos streampos;
-#  endif
+#    endif
 
   class _LIBCPP_EXPORTED_FROM_ABI Init;
 
@@ -397,11 +399,11 @@ private:
   size_t __event_cap_;
 // TODO(EricWF): Enable this for both Clang and GCC. Currently it is only
 // enabled with clang.
-#  if _LIBCPP_HAS_C_ATOMIC_IMP && _LIBCPP_HAS_THREADS
+#    if _LIBCPP_HAS_C_ATOMIC_IMP && _LIBCPP_HAS_THREADS
   static atomic<int> __xindex_;
-#  else
+#    else
   static int __xindex_;
-#  endif
+#    endif
   long* __iarray_;
   size_t __iarray_size_;
   size_t __iarray_cap_;
@@ -417,10 +419,10 @@ _LIBCPP_DECLARE_STRONG_ENUM_EPILOG(io_errc)
 template <>
 struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<io_errc> : public true_type {};
 
-#  ifdef _LIBCPP_CXX03_LANG
+#    ifdef _LIBCPP_CXX03_LANG
 template <>
 struct _LIBCPP_TEMPLATE_VIS is_error_code_enum<io_errc::__lx> : public true_type {};
-#  endif
+#    endif
 
 _LIBCPP_EXPORTED_FROM_ABI const error_category& iostream_category() _NOEXCEPT;
 
@@ -441,11 +443,11 @@ public:
 };
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_failure(char const* __msg) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw ios_base::failure(__msg);
-#  else
+#    else
   _LIBCPP_VERBOSE_ABORT("ios_base::failure was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#  endif
+#    endif
 }
 
 class _LIBCPP_EXPORTED_FROM_ABI ios_base::Init {
@@ -569,13 +571,13 @@ public:
   static_assert(is_same<_CharT, typename traits_type::char_type>::value,
                 "traits_type::char_type must be the same type as CharT");
 
-#  ifdef _LIBCPP_CXX03_LANG
+#    ifdef _LIBCPP_CXX03_LANG
   // Preserve the ability to compare with literal 0,
   // and implicitly convert to bool, but not implicitly convert to int.
   _LIBCPP_HIDE_FROM_ABI operator void*() const { return fail() ? nullptr : (void*)this; }
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return !fail(); }
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI bool operator!() const { return fail(); }
   _LIBCPP_HIDE_FROM_ABI iostate rdstate() const { return ios_base::rdstate(); }
@@ -625,11 +627,11 @@ protected:
 private:
   basic_ostream<char_type, traits_type>* __tie_;
 
-#  if defined(_LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE)
+#    if defined(_LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE)
   using _FillType = _FillHelper<traits_type>;
-#  else
+#    else
   using _FillType = _SentinelValueFill<traits_type>;
-#  endif
+#    endif
   mutable _FillType __fill_;
 };
 
@@ -744,9 +746,9 @@ inline _LIBCPP_HIDE_FROM_ABI void basic_ios<_CharT, _Traits>::set_rdbuf(basic_st
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ios<char>;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ios<wchar_t>;
-#  endif
+#    endif
 
 _LIBCPP_HIDE_FROM_ABI inline ios_base& boolalpha(ios_base& __str) {
   __str.setf(ios_base::boolalpha);
@@ -872,22 +874,23 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_LOCALIZATION
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstddef>
-#  include <cstdlib>
-#  include <cstring>
-#  include <initializer_list>
-#  include <limits>
-#  include <mutex>
-#  include <new>
-#  include <stdexcept>
-#  include <system_error>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  endif // _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstddef>
+#    include <cstdlib>
+#    include <cstring>
+#    include <initializer_list>
+#    include <limits>
+#    include <mutex>
+#    include <new>
+#    include <stdexcept>
+#    include <system_error>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_IOS
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index 5e777c6b23735..3f8a4cd3d4d8c 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -105,21 +105,23 @@ using wosyncstream = basic_osyncstream<wchar_t>;  // C++20
 
 */
 
-#include <__config>
-#include <__fwd/fstream.h>
-#include <__fwd/ios.h>
-#include <__fwd/istream.h>
-#include <__fwd/memory.h>
-#include <__fwd/ostream.h>
-#include <__fwd/sstream.h>
-#include <__fwd/streambuf.h>
-#include <__fwd/string.h>
-#include <__std_mbstate_t.h>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__fwd/fstream.h>
+#  include <__fwd/ios.h>
+#  include <__fwd/istream.h>
+#  include <__fwd/memory.h>
+#  include <__fwd/ostream.h>
+#  include <__fwd/sstream.h>
+#  include <__fwd/streambuf.h>
+#  include <__fwd/string.h>
+#  include <__std_mbstate_t.h>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -131,34 +133,34 @@ class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator;
 template <class _State>
 class _LIBCPP_TEMPLATE_VIS fpos;
 typedef fpos<mbstate_t> streampos;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef fpos<mbstate_t> wstreampos;
-#endif
-#if _LIBCPP_HAS_CHAR8_T
+#  endif
+#  if _LIBCPP_HAS_CHAR8_T
 typedef fpos<mbstate_t> u8streampos;
-#endif
+#  endif
 typedef fpos<mbstate_t> u16streampos;
 typedef fpos<mbstate_t> u32streampos;
 
-#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
 
 template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT>>
 class basic_syncbuf;
 
 using syncbuf = basic_syncbuf<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 using wsyncbuf = basic_syncbuf<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT>>
 class basic_osyncstream;
 
 using osyncstream = basic_osyncstream<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 using wosyncstream = basic_osyncstream<wchar_t>;
-#  endif
+#    endif
 
-#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
 
 template <class _CharT, class _Traits>
 class __save_flags {
@@ -183,4 +185,6 @@ public:
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_IOSFWD
diff --git a/libcxx/include/iostream b/libcxx/include/iostream
index d3ecc6c551fdf..067a7c111f437 100644
--- a/libcxx/include/iostream
+++ b/libcxx/include/iostream
@@ -33,20 +33,22 @@ extern wostream wclog;
 
 */
 
-#include <__config>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <version>
 
 // standard-mandated includes
 
 // [iostream.syn]
-#include <ios>
-#include <istream>
-#include <ostream>
-#include <streambuf>
+#  include <ios>
+#  include <istream>
+#  include <ostream>
+#  include <streambuf>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -55,13 +57,15 @@ extern _LIBCPP_EXPORTED_FROM_ABI ostream cout;
 extern _LIBCPP_EXPORTED_FROM_ABI ostream cerr;
 extern _LIBCPP_EXPORTED_FROM_ABI ostream clog;
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 extern _LIBCPP_EXPORTED_FROM_ABI wistream wcin;
 extern _LIBCPP_EXPORTED_FROM_ABI wostream wcout;
 extern _LIBCPP_EXPORTED_FROM_ABI wostream wcerr;
 extern _LIBCPP_EXPORTED_FROM_ABI wostream wclog;
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_IOSTREAM
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 0d6d6880f8aa0..c2edc9ab22608 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -158,30 +158,32 @@ template <class Stream, class T>
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_LOCALIZATION
-
-#  include <__fwd/istream.h>
-#  include <__iterator/istreambuf_iterator.h>
-#  include <__ostream/basic_ostream.h>
-#  include <__type_traits/conjunction.h>
-#  include <__type_traits/enable_if.h>
-#  include <__type_traits/is_base_of.h>
-#  include <__type_traits/make_unsigned.h>
-#  include <__utility/declval.h>
-#  include <__utility/forward.h>
-#  include <bitset>
-#  include <ios>
-#  include <locale>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+
+#    include <__fwd/istream.h>
+#    include <__iterator/istreambuf_iterator.h>
+#    include <__ostream/basic_ostream.h>
+#    include <__type_traits/conjunction.h>
+#    include <__type_traits/enable_if.h>
+#    include <__type_traits/is_base_of.h>
+#    include <__type_traits/make_unsigned.h>
+#    include <__utility/declval.h>
+#    include <__utility/forward.h>
+#    include <bitset>
+#    include <ios>
+#    include <locale>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -357,13 +359,13 @@ __input_arithmetic(basic_istream<_CharT, _Traits>& __is, _Tp& __n) {
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __s(__is);
   if (__s) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
       typedef num_get<_CharT, _Ip> _Fp;
       std::use_facet<_Fp>(__is.getloc()).get(_Ip(__is), _Ip(), __is, __state, __n);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -371,7 +373,7 @@ __input_arithmetic(basic_istream<_CharT, _Traits>& __is, _Tp& __n) {
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
@@ -438,9 +440,9 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __s(__is);
   if (__s) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
       typedef num_get<_CharT, _Ip> _Fp;
       long __temp;
@@ -454,7 +456,7 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
       } else {
         __n = static_cast<_Tp>(__temp);
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -462,7 +464,7 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __is.setstate(__state);
   }
   return __is;
@@ -484,9 +486,9 @@ __input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n)
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       _CharT* __s               = __p;
       const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
       while (__s != __p + (__n - 1)) {
@@ -505,7 +507,7 @@ __input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n)
       __is.width(0);
       if (__s == __p)
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -513,13 +515,13 @@ __input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n)
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits, size_t _Np>
 inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
@@ -542,7 +544,7 @@ operator>>(basic_istream<char, _Traits>& __is, signed char (&__buf)[_Np]) {
   return __is >> (char(&)[_Np])__buf;
 }
 
-#  else
+#    else
 
 template <class _CharT, class _Traits>
 inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
@@ -565,22 +567,22 @@ operator>>(basic_istream<char, _Traits>& __is, signed char* __s) {
   return __is >> (char*)__s;
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& operator>>(basic_istream<_CharT, _Traits>& __is, _CharT& __c) {
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       typename _Traits::int_type __i = __is.rdbuf()->sbumpc();
       if (_Traits::eq_int_type(__i, _Traits::eof()))
         __state |= ios_base::eofbit | ios_base::failbit;
       else
         __c = _Traits::to_char_type(__i);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -588,7 +590,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& operator>>(basic_istream<_
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
@@ -614,9 +616,9 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
   sentry __s(*this, true);
   if (__s) {
     if (__sb) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
         while (true) {
           typename traits_type::int_type __i = this->rdbuf()->sgetc();
           if (traits_type::eq_int_type(__i, _Traits::eof())) {
@@ -630,7 +632,7 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
         }
         if (__gc_ == 0)
           __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         __state |= ios_base::badbit;
         if (__gc_ == 0)
@@ -641,7 +643,7 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
           throw;
         }
       }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     } else {
       __state |= ios_base::failbit;
     }
@@ -657,22 +659,22 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
   int_type __r              = traits_type::eof();
   sentry __s(*this, true);
   if (__s) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       __r = this->rdbuf()->sbumpc();
       if (traits_type::eq_int_type(__r, traits_type::eof()))
         __state |= ios_base::failbit | ios_base::eofbit;
       else
         __gc_ = 1;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       this->__setstate_nothrow(this->rdstate() | ios_base::badbit);
       if (this->exceptions() & ios_base::badbit) {
         throw;
       }
     }
-#  endif
+#    endif
     this->setstate(__state);
   }
   return __r;
@@ -685,9 +687,9 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* _
   sentry __sen(*this, true);
   if (__sen) {
     if (__n > 0) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       try {
-#  endif
+#    endif
         while (__gc_ < __n - 1) {
           int_type __i = this->rdbuf()->sgetc();
           if (traits_type::eq_int_type(__i, traits_type::eof())) {
@@ -703,7 +705,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* _
         }
         if (__gc_ == 0)
           __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         __state |= ios_base::badbit;
         this->__setstate_nothrow(__state);
@@ -713,7 +715,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* _
           throw;
         }
       }
-#  endif
+#    endif
     } else {
       __state |= ios_base::failbit;
     }
@@ -734,9 +736,9 @@ basic_istream<_CharT, _Traits>::get(basic_streambuf<char_type, traits_type>& __s
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       while (true) {
         typename traits_type::int_type __i = this->rdbuf()->sgetc();
         if (traits_type::eq_int_type(__i, traits_type::eof())) {
@@ -751,12 +753,12 @@ basic_istream<_CharT, _Traits>::get(basic_streambuf<char_type, traits_type>& __s
         __inc_gcount();
         this->rdbuf()->sbumpc();
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       // according to the spec, exceptions here are caught but not rethrown
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     if (__gc_ == 0)
       __state |= ios_base::failbit;
     this->setstate(__state);
@@ -771,9 +773,9 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       while (true) {
         typename traits_type::int_type __i = this->rdbuf()->sgetc();
         if (traits_type::eq_int_type(__i, traits_type::eof())) {
@@ -794,7 +796,7 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
         this->rdbuf()->sbumpc();
         __inc_gcount();
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -806,7 +808,7 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   }
   if (__n > 0)
     *__s = char_type();
@@ -822,9 +824,9 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (__n == numeric_limits<streamsize>::max()) {
         while (true) {
           typename traits_type::int_type __i = this->rdbuf()->sbumpc();
@@ -848,7 +850,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
             break;
         }
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -856,7 +858,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -869,13 +871,13 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
   int_type __r              = traits_type::eof();
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       __r = this->rdbuf()->sgetc();
       if (traits_type::eq_int_type(__r, traits_type::eof()))
         __state |= ios_base::eofbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -883,7 +885,7 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -895,13 +897,13 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::read(char_type*
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       __gc_ = this->rdbuf()->sgetn(__s, __n);
       if (__gc_ != __n)
         __state |= ios_base::failbit | ios_base::eofbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -909,7 +911,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::read(char_type*
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -923,9 +925,9 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       streamsize __c = this->rdbuf()->in_avail();
       switch (__c) {
       case -1:
@@ -940,7 +942,7 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
           __state |= ios_base::failbit | ios_base::eofbit;
         break;
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -948,7 +950,7 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -963,12 +965,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::putback(char_typ
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf() == nullptr || this->rdbuf()->sputbackc(__c) == traits_type::eof())
         __state |= ios_base::badbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -976,7 +978,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::putback(char_typ
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -991,12 +993,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::unget() {
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf() == nullptr || this->rdbuf()->sungetc() == traits_type::eof())
         __state |= ios_base::badbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1004,7 +1006,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::unget() {
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -1021,14 +1023,14 @@ int basic_istream<_CharT, _Traits>::sync() {
 
   int __r = 0;
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubsync() == -1) {
         __state |= ios_base::badbit;
         __r = -1;
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1036,7 +1038,7 @@ int basic_istream<_CharT, _Traits>::sync() {
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -1048,11 +1050,11 @@ typename basic_istream<_CharT, _Traits>::pos_type basic_istream<_CharT, _Traits>
   pos_type __r(-1);
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       __r = this->rdbuf()->pubseekoff(0, ios_base::cur, ios_base::in);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1060,7 +1062,7 @@ typename basic_istream<_CharT, _Traits>::pos_type basic_istream<_CharT, _Traits>
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -1072,12 +1074,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(pos_type _
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubseekpos(__pos, ios_base::in) == pos_type(-1))
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1085,7 +1087,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(pos_type _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -1097,12 +1099,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(off_type _
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubseekoff(__off, __dir, ios_base::in) == pos_type(-1))
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1110,7 +1112,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(off_type _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -1121,9 +1123,9 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
       while (true) {
         typename _Traits::int_type __i = __is.rdbuf()->sgetc();
@@ -1135,7 +1137,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
           break;
         __is.rdbuf()->sbumpc();
       }
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1143,7 +1145,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __is.setstate(__state);
   }
   return __is;
@@ -1211,9 +1213,9 @@ operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       __str.clear();
       using _Size              = typename basic_string<_CharT, _Traits, _Allocator>::size_type;
       streamsize const __width = __is.width();
@@ -1243,7 +1245,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _
       __is.width(0);
       if (__c == 0)
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1251,7 +1253,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
@@ -1263,9 +1265,9 @@ getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _All
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       __str.clear();
       streamsize __extr = 0;
       while (true) {
@@ -1286,7 +1288,7 @@ getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _All
       }
       if (__extr == 0)
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1294,7 +1296,7 @@ getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _All
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
@@ -1324,9 +1326,9 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif
+#    endif
       basic_string<_CharT, _Traits> __str;
       const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
       size_t __c                = 0;
@@ -1348,7 +1350,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
       __x = bitset<_Size>(__str);
       if (_Size > 0 && __c == 0)
         __state |= ios_base::failbit;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1356,29 +1358,31 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
         throw;
       }
     }
-#  endif
+#    endif
     __is.setstate(__state);
   }
   return __is;
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream<wchar_t>;
-#  endif
+#    endif
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_iostream<char>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_LOCALIZATION
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <iosfwd>
-#  include <ostream>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <iosfwd>
+#    include <ostream>
+#    include <type_traits>
+#  endif
 
 _LIBCPP_POP_MACROS
 
+#endif // 0
+
 #endif // _LIBCPP_ISTREAM
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index 63fbae220b604..b9bc188110685 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -679,77 +679,80 @@ template <class E> constexpr const E* data(initializer_list<E> il) noexcept;
 
 */
 
-#include <__config>
-#include <__iterator/access.h>
-#include <__iterator/advance.h>
-#include <__iterator/back_insert_iterator.h>
-#include <__iterator/distance.h>
-#include <__iterator/front_insert_iterator.h>
-#include <__iterator/insert_iterator.h>
-#include <__iterator/istream_iterator.h>
-#include <__iterator/istreambuf_iterator.h>
-#include <__iterator/iterator.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/move_iterator.h>
-#include <__iterator/next.h>
-#include <__iterator/ostream_iterator.h>
-#include <__iterator/ostreambuf_iterator.h>
-#include <__iterator/prev.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/wrap_iter.h>
-
-#if _LIBCPP_STD_VER >= 14
-#  include <__iterator/reverse_access.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__iterator/data.h>
-#  include <__iterator/empty.h>
-#  include <__iterator/size.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__iterator/common_iterator.h>
-#  include <__iterator/concepts.h>
-#  include <__iterator/counted_iterator.h>
-#  include <__iterator/default_sentinel.h>
-#  include <__iterator/incrementable_traits.h>
-#  include <__iterator/indirectly_comparable.h>
-#  include <__iterator/iter_move.h>
-#  include <__iterator/iter_swap.h>
-#  include <__iterator/mergeable.h>
-#  include <__iterator/move_sentinel.h>
-#  include <__iterator/permutable.h>
-#  include <__iterator/projected.h>
-#  include <__iterator/readable_traits.h>
-#  include <__iterator/sortable.h>
-#  include <__iterator/unreachable_sentinel.h>
-#endif
-
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__iterator/access.h>
+#  include <__iterator/advance.h>
+#  include <__iterator/back_insert_iterator.h>
+#  include <__iterator/distance.h>
+#  include <__iterator/front_insert_iterator.h>
+#  include <__iterator/insert_iterator.h>
+#  include <__iterator/istream_iterator.h>
+#  include <__iterator/istreambuf_iterator.h>
+#  include <__iterator/iterator.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/move_iterator.h>
+#  include <__iterator/next.h>
+#  include <__iterator/ostream_iterator.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__iterator/prev.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/wrap_iter.h>
+
+#  if _LIBCPP_STD_VER >= 14
+#    include <__iterator/reverse_access.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__iterator/data.h>
+#    include <__iterator/empty.h>
+#    include <__iterator/size.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__iterator/common_iterator.h>
+#    include <__iterator/concepts.h>
+#    include <__iterator/counted_iterator.h>
+#    include <__iterator/default_sentinel.h>
+#    include <__iterator/incrementable_traits.h>
+#    include <__iterator/indirectly_comparable.h>
+#    include <__iterator/iter_move.h>
+#    include <__iterator/iter_swap.h>
+#    include <__iterator/mergeable.h>
+#    include <__iterator/move_sentinel.h>
+#    include <__iterator/permutable.h>
+#    include <__iterator/projected.h>
+#    include <__iterator/readable_traits.h>
+#    include <__iterator/sortable.h>
+#    include <__iterator/unreachable_sentinel.h>
+#  endif
+
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.synopsis]
-#include <compare>
-#include <concepts>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
-#  include <variant>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <exception>
-#  include <new>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  include <compare>
+#  include <concepts>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#    include <variant>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <exception>
+#    include <new>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_ITERATOR
diff --git a/libcxx/include/latch b/libcxx/include/latch
index 1860ed816c856..d21b689cc4815 100644
--- a/libcxx/include/latch
+++ b/libcxx/include/latch
@@ -40,26 +40,28 @@ namespace std
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_THREADS
+#  if _LIBCPP_HAS_THREADS
 
-#  include <__assert>
-#  include <__atomic/atomic.h>
-#  include <__atomic/atomic_sync.h>
-#  include <__atomic/memory_order.h>
-#  include <__cstddef/ptrdiff_t.h>
-#  include <limits>
-#  include <version>
+#    include <__assert>
+#    include <__atomic/atomic.h>
+#    include <__atomic/atomic_sync.h>
+#    include <__atomic/memory_order.h>
+#    include <__cstddef/ptrdiff_t.h>
+#    include <limits>
+#    include <version>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -99,8 +101,9 @@ public:
     return try_wait_impl(__value);
   }
   inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const {
-    std::__atomic_wait_unless(
-        __a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire);
+    std::__atomic_wait_unless(__a_, memory_order_acquire, [this](ptrdiff_t& __value) -> bool {
+      return try_wait_impl(__value);
+    });
   }
   inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) {
     _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value");
@@ -116,15 +119,16 @@ private:
 
 _LIBCPP_END_NAMESPACE_STD
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <cstddef>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <cstddef>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_LATCH
diff --git a/libcxx/include/limits b/libcxx/include/limits
index da0a92d7daff4..c27950f42106d 100644
--- a/libcxx/include/limits
+++ b/libcxx/include/limits
@@ -102,18 +102,20 @@ template<> class numeric_limits<cv long double>;
 
 */
 
-#include <__config>
-#include <__type_traits/is_arithmetic.h>
-#include <__type_traits/is_signed.h>
-#include <__type_traits/remove_cv.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__type_traits/is_arithmetic.h>
+#  include <__type_traits/is_signed.h>
+#  include <__type_traits/remove_cv.h>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-#include <version>
+#  include <__undef_macros>
+#  include <version>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -227,11 +229,11 @@ protected:
   static _LIBCPP_CONSTEXPR const bool is_bounded = true;
   static _LIBCPP_CONSTEXPR const bool is_modulo  = !std::is_signed<_Tp>::value;
 
-#if defined(__i386__) || defined(__x86_64__) || defined(__pnacl__) || defined(__wasm__)
+#  if defined(__i386__) || defined(__x86_64__) || defined(__pnacl__) || defined(__wasm__)
   static _LIBCPP_CONSTEXPR const bool traps = true;
-#else
+#  else
   static _LIBCPP_CONSTEXPR const bool traps = false;
-#endif
+#  endif
   static _LIBCPP_CONSTEXPR const bool tinyness_before          = false;
   static _LIBCPP_CONSTEXPR const float_round_style round_style = round_toward_zero;
 };
@@ -332,11 +334,11 @@ protected:
   static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
 
   static _LIBCPP_CONSTEXPR const bool traps = false;
-#if (defined(__arm__) || defined(__aarch64__))
+#  if (defined(__arm__) || defined(__aarch64__))
   static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
-#else
+#  else
   static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
-#endif
+#  endif
   static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
 };
 
@@ -389,11 +391,11 @@ protected:
   static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
 
   static _LIBCPP_CONSTEXPR const bool traps = false;
-#if (defined(__arm__) || defined(__aarch64__))
+#  if (defined(__arm__) || defined(__aarch64__))
   static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
-#else
+#  else
   static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
-#endif
+#  endif
   static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
 };
 
@@ -441,20 +443,20 @@ protected:
     return __LDBL_DENORM_MIN__;
   }
 
-#if defined(__powerpc__) && defined(__LONG_DOUBLE_IBM128__)
+#  if defined(__powerpc__) && defined(__LONG_DOUBLE_IBM128__)
   static _LIBCPP_CONSTEXPR const bool is_iec559 = false;
-#else
+#  else
   static _LIBCPP_CONSTEXPR const bool is_iec559 = true;
-#endif
+#  endif
   static _LIBCPP_CONSTEXPR const bool is_bounded = true;
   static _LIBCPP_CONSTEXPR const bool is_modulo  = false;
 
   static _LIBCPP_CONSTEXPR const bool traps = false;
-#if (defined(__arm__) || defined(__aarch64__))
+#  if (defined(__arm__) || defined(__aarch64__))
   static _LIBCPP_CONSTEXPR const bool tinyness_before = true;
-#else
+#  else
   static _LIBCPP_CONSTEXPR const bool tinyness_before = false;
-#endif
+#  endif
   static _LIBCPP_CONSTEXPR const float_round_style round_style = round_to_nearest;
 };
 
@@ -530,8 +532,9 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_LIMITS
diff --git a/libcxx/include/list b/libcxx/include/list
index 22629ffb1427e..89863995379d9 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -197,69 +197,71 @@ template <class T, class Allocator, class Predicate>
 
 */
 
-#include <__algorithm/comp.h>
-#include <__algorithm/equal.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__algorithm/min.h>
-#include <__assert>
-#include <__config>
-#include <__format/enable_insertable.h>
-#include <__iterator/distance.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/move_iterator.h>
-#include <__iterator/next.h>
-#include <__iterator/prev.h>
-#include <__iterator/reverse_iterator.h>
-#include <__memory/addressof.h>
-#include <__memory/allocation_guard.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/construct_at.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/swap_allocator.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_pointer.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <cstring>
-#include <limits>
-#include <new> // __launder
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/comp.h>
+#  include <__algorithm/equal.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/min.h>
+#  include <__assert>
+#  include <__config>
+#  include <__format/enable_insertable.h>
+#  include <__iterator/distance.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/move_iterator.h>
+#  include <__iterator/next.h>
+#  include <__iterator/prev.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocation_guard.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
+#  include <__memory/construct_at.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/swap_allocator.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_pointer.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <cstring>
+#  include <limits>
+#  include <new> // __launder
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [list.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -274,7 +276,7 @@ struct __list_node_pointer_traits {
   typedef __rebind_pointer_t<_VoidPtr, __list_node_base<_Tp, _VoidPtr> > __base_pointer;
 
 // TODO(LLVM 22): Remove this check
-#ifndef _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
+#  ifndef _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
   static_assert(sizeof(__node_pointer) == sizeof(__node_pointer) && _LIBCPP_ALIGNOF(__base_pointer) ==
                     _LIBCPP_ALIGNOF(__node_pointer),
                 "It looks like you are using std::list with a fancy pointer type that thas a different representation "
@@ -282,7 +284,7 @@ struct __list_node_pointer_traits {
                 "implementation details of the standard library). This means that your ABI is being broken between "
                 "LLVM 19 and LLVM 20. If you don't care about your ABI being broken, define the "
                 "_LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
-#endif
+#  endif
 
   static _LIBCPP_HIDE_FROM_ABI __base_pointer __unsafe_link_pointer_cast(__base_pointer __p) { return __p; }
 
@@ -314,7 +316,7 @@ template <class _Tp, class _VoidPtr>
 struct __list_node : public __list_node_base<_Tp, _VoidPtr> {
   // We allow starting the lifetime of nodes without initializing the value held by the node,
   // since that is handled by the list itself in order to be allocator-aware.
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 private:
   union {
@@ -323,14 +325,14 @@ private:
 
 public:
   _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
-#else
+#  else
 
 private:
   _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
 
 public:
   _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
-#endif
+#  endif
 
   typedef __list_node_base<_Tp, _VoidPtr> __base;
   typedef typename __base::__base_pointer __base_pointer;
@@ -512,9 +514,9 @@ protected:
   _LIBCPP_HIDE_FROM_ABI __list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value);
   _LIBCPP_HIDE_FROM_ABI __list_imp(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI __list_imp(const __node_allocator& __a);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __list_imp(__node_allocator&& __a) _NOEXCEPT;
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI ~__list_imp();
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __size_ == 0; }
@@ -525,11 +527,11 @@ protected:
   _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(__end_as_link()); }
 
   _LIBCPP_HIDE_FROM_ABI void swap(__list_imp& __c)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
-#else
+#  else
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c) {
     __copy_assign_alloc(
@@ -603,12 +605,12 @@ inline __list_imp<_Tp, _Alloc>::__list_imp(const allocator_type& __a)
 template <class _Tp, class _Alloc>
 inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a) : __size_(0), __node_alloc_(__a) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Alloc>
 inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT
     : __size_(0),
       __node_alloc_(std::move(__a)) {}
-#endif
+#  endif
 
 template <class _Tp, class _Alloc>
 __list_imp<_Tp, _Alloc>::~__list_imp() {
@@ -632,11 +634,11 @@ void __list_imp<_Tp, _Alloc>::clear() _NOEXCEPT {
 
 template <class _Tp, class _Alloc>
 void __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
-#else
+#  else
     _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
-#endif
+#  endif
 {
   _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
       __alloc_traits::propagate_on_container_swap::value || this->__node_alloc_ == __c.__node_alloc_,
@@ -683,18 +685,18 @@ public:
   typedef typename __base::const_iterator const_iterator;
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   typedef size_type __remove_return_type;
-#else
+#  else
   typedef void __remove_return_type;
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {}
   _LIBCPP_HIDE_FROM_ABI explicit list(const allocator_type& __a) : __base(__a) {}
   _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n, const allocator_type& __a);
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x);
   template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x, const allocator_type& __a) : __base(__a) {
@@ -708,18 +710,18 @@ public:
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l, const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
       : __base(__a) {
     prepend_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI list(const list& __c);
   _LIBCPP_HIDE_FROM_ABI list(const list& __c, const __type_identity_t<allocator_type>& __a);
   _LIBCPP_HIDE_FROM_ABI list& operator=(const list& __c);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il, const allocator_type& __a);
 
@@ -735,17 +737,17 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void assign(_InpIter __f, _InpIter __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
     __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __x);
 
@@ -788,11 +790,11 @@ public:
     return __base::__end_.__prev_->__as_node()->__get_value();
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __x);
   _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
     insert_range(begin(), std::forward<_Range>(__range));
@@ -802,20 +804,20 @@ public:
   _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
     insert_range(end(), std::forward<_Range>(__range));
   }
-#  endif
+#    endif
 
   template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
-#  endif
+#    endif
   template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI reference emplace_back(_Args&&... __args);
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
-#  endif
+#    endif
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
 
@@ -824,19 +826,19 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
     return insert(__p, __il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __x);
   _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __x);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class _Arg>
   _LIBCPP_HIDE_FROM_ABI void __emplace_back(_Arg&& __arg) {
     emplace_back(std::forward<_Arg>(__arg));
   }
-#else
+#  else
   _LIBCPP_HIDE_FROM_ABI void __emplace_back(value_type const& __arg) { push_back(__arg); }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x);
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __x);
@@ -844,19 +846,19 @@ public:
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InpIter __f, _InpIter __l);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI iterator insert_range(const_iterator __position, _Range&& __range) {
     return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(list& __c)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT
-#else
+#  else
       _NOEXCEPT_(!__node_alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__node_allocator>)
-#endif
+#  endif
   {
     __base::swap(__c);
   }
@@ -872,13 +874,13 @@ public:
   _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __x);
 
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c) { splice(__p, __c); }
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __i) { splice(__p, __c, __i); }
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __f, const_iterator __l) {
     splice(__p, __c, __f, __l);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __i);
   _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l);
 
@@ -889,14 +891,14 @@ public:
   template <class _BinaryPred>
   _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPred __binary_pred);
   _LIBCPP_HIDE_FROM_ABI void merge(list& __c);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void merge(list&& __c) { merge(__c); }
 
   template <class _Comp>
   _LIBCPP_HIDE_FROM_ABI void merge(list&& __c, _Comp __comp) {
     merge(__c, __comp);
   }
-#endif
+#  endif
   template <class _Comp>
   _LIBCPP_HIDE_FROM_ABI void merge(list& __c, _Comp __comp);
 
@@ -928,7 +930,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, false_type);
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Alloc = allocator<__iter_value_type<_InputIterator>>,
           class        = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
@@ -940,14 +942,14 @@ template <class _InputIterator,
           class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
           class = enable_if_t<__is_allocator<_Alloc>::value> >
 list(_InputIterator, _InputIterator, _Alloc) -> list<__iter_value_type<_InputIterator>, _Alloc>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Alloc = allocator<ranges::range_value_t<_Range>>,
           class        = enable_if_t<__is_allocator<_Alloc>::value> >
 list(from_range_t, _Range&&, _Alloc = _Alloc()) -> list<ranges::range_value_t<_Range>, _Alloc>;
-#endif
+#  endif
 
 // Link in nodes [__f, __l] just prior to __p
 template <class _Tp, class _Alloc>
@@ -984,20 +986,20 @@ inline typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::__iterator(size_t
 template <class _Tp, class _Alloc>
 list<_Tp, _Alloc>::list(size_type __n) {
   for (; __n > 0; --__n)
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     emplace_back();
-#else
+#  else
     push_back(value_type());
-#endif
+#  endif
 }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 template <class _Tp, class _Alloc>
 list<_Tp, _Alloc>::list(size_type __n, const allocator_type& __a) : __base(__a) {
   for (; __n > 0; --__n)
     emplace_back();
 }
-#endif
+#  endif
 
 template <class _Tp, class _Alloc>
 list<_Tp, _Alloc>::list(size_type __n, const value_type& __x) {
@@ -1032,7 +1034,7 @@ list<_Tp, _Alloc>::list(const list& __c, const __type_identity_t<allocator_type>
     push_back(*__i);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 list<_Tp, _Alloc>::list(initializer_list<value_type> __il, const allocator_type& __a) : __base(__a) {
@@ -1087,7 +1089,7 @@ void list<_Tp, _Alloc>::__move_assign(list& __c,
   splice(end(), __c);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(const list& __c) {
@@ -1152,13 +1154,13 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
     ++__ds;
     __r          = iterator(__node->__as_link());
     iterator __e = __r;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1170,7 +1172,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
     this->__size_ += __ds;
   }
@@ -1194,13 +1196,13 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
     ++__ds;
     __r          = iterator(__node->__as_link());
     iterator __e = __r;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (++__f; __f != __l; ++__f, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, *__f)->__as_link();
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1212,7 +1214,7 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
     this->__size_ += __ds;
   }
@@ -1235,7 +1237,7 @@ void list<_Tp, _Alloc>::push_back(const value_type& __x) {
   ++this->__size_;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 void list<_Tp, _Alloc>::push_front(value_type&& __x) {
@@ -1255,38 +1257,38 @@ void list<_Tp, _Alloc>::push_back(value_type&& __x) {
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 typename list<_Tp, _Alloc>::reference
-#  else
+#    else
 void
-#  endif
+#    endif
 list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
   __node_pointer __node =
       this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
   __base_pointer __nl = __node->__as_link();
   __link_nodes_at_front(__nl, __nl);
   ++this->__size_;
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   return __node->__get_value();
-#  endif
+#    endif
 }
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 typename list<_Tp, _Alloc>::reference
-#  else
+#    else
 void
-#  endif
+#    endif
 list<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
   __node_pointer __node =
       this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
   __base_pointer __nl = __node->__as_link();
   __link_nodes_at_back(__nl, __nl);
   ++this->__size_;
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   return __node->__get_value();
-#  endif
+#    endif
 }
 
 template <class _Tp, class _Alloc>
@@ -1309,7 +1311,7 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __
   return iterator(__nl);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
 void list<_Tp, _Alloc>::pop_front() {
@@ -1365,13 +1367,13 @@ void list<_Tp, _Alloc>::resize(size_type __n) {
     ++__ds;
     iterator __r = iterator(__node->__as_link());
     iterator __e = __r;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr)->__as_link();
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1383,7 +1385,7 @@ void list<_Tp, _Alloc>::resize(size_type __n) {
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes_at_back(__r.__ptr_, __e.__ptr_);
     this->__size_ += __ds;
   }
@@ -1401,13 +1403,13 @@ void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
     __base_pointer __nl = __node->__as_link();
     iterator __r        = iterator(__nl);
     iterator __e        = __r;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
       }
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1419,7 +1421,7 @@ void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
       }
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(__base::__end_as_link(), __r.__ptr_, __e.__ptr_);
     this->__size_ += __ds;
   }
@@ -1645,7 +1647,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator==(const list<_Tp, _Alloc>& __x, const
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool operator<(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
@@ -1672,7 +1674,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const list<_Tp, _Alloc>& __x, const
   return !(__y < __x);
 }
 
-#else // _LIBCPP_STD_VER <= 17
+#  else // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
@@ -1680,7 +1682,7 @@ operator<=>(const list<_Tp, _Allocator>& __x, const list<_Tp, _Allocator>& __y)
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI void swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>& __y)
@@ -1688,7 +1690,7 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Tp, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
 erase_if(list<_Tp, _Allocator>& __c, _Predicate __pred) {
@@ -1703,12 +1705,12 @@ erase(list<_Tp, _Allocator>& __c, const _Up& __v) {
 
 template <>
 inline constexpr bool __format::__enable_insertable<std::list<char>> = true;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 inline constexpr bool __format::__enable_insertable<std::list<wchar_t>> = true;
-#  endif
+#    endif
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _Tp, class _Allocator>
 struct __container_traits<list<_Tp, _Allocator> > {
@@ -1723,29 +1725,30 @@ struct __container_traits<list<_Tp, _Allocator> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _ValueT>
 using list _LIBCPP_AVAILABILITY_PMR = std::list<_ValueT, polymorphic_allocator<_ValueT>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <atomic>
-#  include <concepts>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <functional>
-#  include <iosfwd>
-#  include <iterator>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <atomic>
+#    include <concepts>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <functional>
+#    include <iosfwd>
+#    include <iterator>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_LIST
diff --git a/libcxx/include/locale b/libcxx/include/locale
index 65f5c6ba52f73..65b1200df061b 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -187,69 +187,71 @@ template <class charT> class messages_byname;
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_LOCALIZATION
-
-#  include <__algorithm/copy.h>
-#  include <__algorithm/equal.h>
-#  include <__algorithm/find.h>
-#  include <__algorithm/max.h>
-#  include <__algorithm/reverse.h>
-#  include <__algorithm/unwrap_iter.h>
-#  include <__assert>
-#  include <__iterator/access.h>
-#  include <__iterator/back_insert_iterator.h>
-#  include <__iterator/istreambuf_iterator.h>
-#  include <__iterator/ostreambuf_iterator.h>
-#  include <__locale>
-#  include <__locale_dir/pad_and_output.h>
-#  include <__memory/unique_ptr.h>
-#  include <__type_traits/make_unsigned.h>
-#  include <cerrno>
-#  include <cstdio>
-#  include <cstdlib>
-#  include <ctime>
-#  include <ios>
-#  include <limits>
-#  include <new>
-#  include <streambuf>
-#  include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+
+#    include <__algorithm/copy.h>
+#    include <__algorithm/equal.h>
+#    include <__algorithm/find.h>
+#    include <__algorithm/max.h>
+#    include <__algorithm/reverse.h>
+#    include <__algorithm/unwrap_iter.h>
+#    include <__assert>
+#    include <__iterator/access.h>
+#    include <__iterator/back_insert_iterator.h>
+#    include <__iterator/istreambuf_iterator.h>
+#    include <__iterator/ostreambuf_iterator.h>
+#    include <__locale>
+#    include <__locale_dir/pad_and_output.h>
+#    include <__memory/unique_ptr.h>
+#    include <__type_traits/make_unsigned.h>
+#    include <cerrno>
+#    include <cstdio>
+#    include <cstdlib>
+#    include <ctime>
+#    include <ios>
+#    include <limits>
+#    include <new>
+#    include <streambuf>
+#    include <version>
 
 // TODO: Properly qualify calls now that __bsd_locale_defaults.h defines functions instead of macros
 // NOLINTBEGIN(libcpp-robust-against-adl)
 
-#  if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+#    if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
 // Most unix variants have catopen.  These are the specific ones that don't.
-#    if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
-#      define _LIBCPP_HAS_CATOPEN 1
-#      include <nl_types.h>
+#      if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
+#        define _LIBCPP_HAS_CATOPEN 1
+#        include <nl_types.h>
+#      else
+#        define _LIBCPP_HAS_CATOPEN 0
+#      endif
 #    else
 #      define _LIBCPP_HAS_CATOPEN 0
 #    endif
-#  else
-#    define _LIBCPP_HAS_CATOPEN 0
-#  endif
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#  if defined(__APPLE__) || defined(__FreeBSD__)
-#    define _LIBCPP_GET_C_LOCALE 0
-#  elif defined(__NetBSD__)
-#    define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
-#  else
-#    define _LIBCPP_GET_C_LOCALE __cloc()
+#    if defined(__APPLE__) || defined(__FreeBSD__)
+#      define _LIBCPP_GET_C_LOCALE 0
+#    elif defined(__NetBSD__)
+#      define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
+#    else
+#      define _LIBCPP_GET_C_LOCALE __cloc()
 // Get the C locale object
 _LIBCPP_EXPORTED_FROM_ABI __locale::__locale_t __cloc();
-#    define __cloc_defined
-#  endif
+#      define __cloc_defined
+#    endif
 
 // __scan_keyword
 // Scans [__b, __e) until a match is found in the basic_strings range
@@ -397,7 +399,7 @@ struct __num_get : protected __num_get_base {
       unsigned*& __g_end,
       unsigned& __dc,
       _CharT* __atoms);
-#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
   static string __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
   static int __stage2_int_loop(
       _CharT __ct,
@@ -411,7 +413,7 @@ struct __num_get : protected __num_get_base {
       unsigned*& __g_end,
       _CharT* __atoms);
 
-#  else
+#    else
   static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
     locale __loc                 = __iob.getloc();
     const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
@@ -446,10 +448,10 @@ private:
     (void)__atoms;
     return __src;
   }
-#  endif
+#    endif
 };
 
-#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
 template <class _CharT>
 string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep) {
   locale __loc = __iob.getloc();
@@ -458,7 +460,7 @@ string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _C
   __thousands_sep              = __np.thousands_sep();
   return __np.grouping();
 }
-#  endif
+#    endif
 
 template <class _CharT>
 string __num_get<_CharT>::__stage2_float_prep(
@@ -473,16 +475,16 @@ string __num_get<_CharT>::__stage2_float_prep(
 
 template <class _CharT>
 int
-#  ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+#    ifndef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
 __num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
                   unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
                   unsigned* __g, unsigned*& __g_end, _CharT* __atoms)
-#  else
+#    else
 __num_get<_CharT>::__stage2_int_loop(_CharT __ct, int __base, char* __a, char*& __a_end,
                   unsigned& __dc, _CharT __thousands_sep, const string& __grouping,
                   unsigned* __g, unsigned*& __g_end, const _CharT* __atoms)
 
-#  endif
+#    endif
 {
   if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
     *__a_end++ = __ct == __atoms[24] ? '+' : '-';
@@ -581,9 +583,9 @@ int __num_get<_CharT>::__stage2_float_loop(
 }
 
 extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS num_get : public locale::facet, private __num_get<_CharT> {
@@ -722,7 +724,7 @@ __num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostat
     __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
     errno                                                     = 0;
     char* __p2;
-    long long __ll                                               = strtoll_l(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
     __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
     if (__current_errno == 0)
       errno = __save_errno;
@@ -754,7 +756,7 @@ __num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iost
     __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
     errno                                                     = 0;
     char* __p2;
-    unsigned long long __ll                                      = strtoull_l(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
     __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
     if (__current_errno == 0)
       errno = __save_errno;
@@ -779,17 +781,17 @@ _LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
 
 template <>
 inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
-  return strtof_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+  return __locale::__strtof(__a, __p2, _LIBCPP_GET_C_LOCALE);
 }
 
 template <>
 inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
-  return strtod_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+  return __locale::__strtod(__a, __p2, _LIBCPP_GET_C_LOCALE);
 }
 
 template <>
 inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
-  return strtold_l(__a, __p2, _LIBCPP_GET_C_LOCALE);
+  return __locale::__strtold(__a, __p2, _LIBCPP_GET_C_LOCALE);
 }
 
 template <class _Tp>
@@ -853,14 +855,14 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_signed(
   // Stage 2
   char_type __thousands_sep;
   const int __atoms_size = __num_get_base::__int_chr_cnt;
-#  ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+#    ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
   char_type __atoms1[__atoms_size];
   const char_type* __atoms = this->__do_widen(__iob, __atoms1);
   string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-#  else
+#    else
   char_type __atoms[__atoms_size];
   string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
-#  endif
+#    endif
   string __buf;
   __buf.resize(__buf.capacity());
   char* __a     = &__buf[0];
@@ -902,14 +904,14 @@ _InputIterator num_get<_CharT, _InputIterator>::__do_get_unsigned(
   // Stage 2
   char_type __thousands_sep;
   const int __atoms_size = __num_get_base::__int_chr_cnt;
-#  ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
+#    ifdef _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
   char_type __atoms1[__atoms_size];
   const char_type* __atoms = this->__do_widen(__iob, __atoms1);
   string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-#  else
+#    else
   char_type __atoms[__atoms_size];
   string __grouping = this->__stage2_int_prep(__iob, __atoms, __thousands_sep);
-#  endif
+#    endif
   string __buf;
   __buf.resize(__buf.capacity());
   char* __a     = &__buf[0];
@@ -1052,9 +1054,9 @@ _InputIterator num_get<_CharT, _InputIterator>::do_get(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
-#  endif
+#    endif
 
 struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
 protected:
@@ -1170,9 +1172,9 @@ void __num_put<_CharT>::__widen_and_group_float(
 }
 
 extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS num_put : public locale::facet, private __num_put<_CharT> {
@@ -1397,9 +1399,9 @@ num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_ty
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _InputIterator>
 _LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
@@ -1464,7 +1466,7 @@ _LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
 template <>
 _LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 _LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
 template <>
@@ -1479,7 +1481,7 @@ template <>
 _LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
 template <>
 _LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
-#  endif
+#    endif
 
 template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
@@ -1933,9 +1935,9 @@ _InputIterator time_get<_CharT, _InputIterator>::do_get(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
-#  endif
+#    endif
 
 class _LIBCPP_EXPORTED_FROM_ABI __time_get {
 protected:
@@ -1971,32 +1973,32 @@ private:
   string_type __analyze(char __fmt, const ctype<_CharT>&);
 };
 
-#  define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                      \
-    template <>                                                                                                        \
-    _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;                \
-    template <>                                                                                                        \
-    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                             \
-    template <>                                                                                                        \
-    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                           \
-    template <>                                                                                                        \
-    _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                             \
-    template <>                                                                                                        \
-    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(           \
-        char, const ctype<_CharT>&);                                                                                   \
-    extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()       \
-        const;                                                                                                         \
-    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);             \
-    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);           \
-    extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);             \
-    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                  \
-    __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);                                                 \
-    /**/
+#    define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                    \
+      template <>                                                                                                      \
+      _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;              \
+      template <>                                                                                                      \
+      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                           \
+      template <>                                                                                                      \
+      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                         \
+      template <>                                                                                                      \
+      _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                           \
+      template <>                                                                                                      \
+      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(         \
+          char, const ctype<_CharT>&);                                                                                 \
+      extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()     \
+          const;                                                                                                       \
+      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);           \
+      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);         \
+      extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);           \
+      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                \
+      __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);                                               \
+      /**/
 
 _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
-#  endif
-#  undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
+#    endif
+#    undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
 
 template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS time_get_byname
@@ -2029,9 +2031,9 @@ private:
 };
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
-#  endif
+#    endif
 
 class _LIBCPP_EXPORTED_FROM_ABI __time_put {
   __locale::__locale_t __loc_;
@@ -2042,9 +2044,9 @@ protected:
   __time_put(const string& __nm);
   ~__time_put();
   void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
   void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
-#  endif
+#    endif
 };
 
 template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
@@ -2118,9 +2120,9 @@ _OutputIterator time_put<_CharT, _OutputIterator>::do_put(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS time_put_byname : public time_put<_CharT, _OutputIterator> {
@@ -2136,9 +2138,9 @@ protected:
 };
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
-#  endif
+#    endif
 
 // money_base
 
@@ -2203,10 +2205,10 @@ const bool moneypunct<_CharT, _International>::intl;
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
-#  endif
+#    endif
 
 // moneypunct_byname
 
@@ -2261,14 +2263,14 @@ _LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 _LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
 template <>
 _LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
-#  endif
+#    endif
 
 // money_get
 
@@ -2329,9 +2331,9 @@ void __money_get<_CharT>::__gather_info(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS money_get : public locale::facet, private __money_get<_CharT> {
@@ -2639,9 +2641,9 @@ _InputIterator money_get<_CharT, _InputIterator>::do_get(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
-#  endif
+#    endif
 
 // money_put
 
@@ -2817,9 +2819,9 @@ void __money_put<_CharT>::__format(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
 class _LIBCPP_TEMPLATE_VIS money_put : public locale::facet, private __money_put<_CharT> {
@@ -2963,9 +2965,9 @@ _OutputIterator money_put<_CharT, _OutputIterator>::do_put(
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
-#  endif
+#    endif
 
 // messages
 
@@ -3009,18 +3011,18 @@ locale::id messages<_CharT>::id;
 
 template <class _CharT>
 typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
-#  if _LIBCPP_HAS_CATOPEN
+#    if _LIBCPP_HAS_CATOPEN
   return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
-#  else  // !_LIBCPP_HAS_CATOPEN
+#    else  // !_LIBCPP_HAS_CATOPEN
   (void)__nm;
   return -1;
-#  endif // _LIBCPP_HAS_CATOPEN
+#    endif // _LIBCPP_HAS_CATOPEN
 }
 
 template <class _CharT>
 typename messages<_CharT>::string_type
 messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-#  if _LIBCPP_HAS_CATOPEN
+#    if _LIBCPP_HAS_CATOPEN
   string __ndflt;
   __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
       std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
@@ -3030,27 +3032,27 @@ messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type&
   string_type __w;
   __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
   return __w;
-#  else  // !_LIBCPP_HAS_CATOPEN
+#    else  // !_LIBCPP_HAS_CATOPEN
   (void)__c;
   (void)__set;
   (void)__msgid;
   return __dflt;
-#  endif // _LIBCPP_HAS_CATOPEN
+#    endif // _LIBCPP_HAS_CATOPEN
 }
 
 template <class _CharT>
 void messages<_CharT>::do_close(catalog __c) const {
-#  if _LIBCPP_HAS_CATOPEN
+#    if _LIBCPP_HAS_CATOPEN
   catclose((nl_catd)__c);
-#  else  // !_LIBCPP_HAS_CATOPEN
+#    else  // !_LIBCPP_HAS_CATOPEN
   (void)__c;
-#  endif // _LIBCPP_HAS_CATOPEN
+#    endif // _LIBCPP_HAS_CATOPEN
 }
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
-#  endif
+#    endif
 
 template <class _CharT>
 class _LIBCPP_TEMPLATE_VIS messages_byname : public messages<_CharT> {
@@ -3067,11 +3069,11 @@ protected:
 };
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
-#  endif
+#    endif
 
-#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+#    if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
 
 template <class _Codecvt,
           class _Elem      = wchar_t,
@@ -3092,19 +3094,19 @@ private:
   size_t __cvtcount_;
 
 public:
-#    ifndef _LIBCPP_CXX03_LANG
+#      ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
   _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
-#    else
+#      else
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
-#    endif
+#      endif
 
   _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
   _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
   wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
-#    ifndef _LIBCPP_CXX03_LANG
+#      ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
-#    endif
+#      endif
   _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
 
   wstring_convert(const wstring_convert& __wc)            = delete;
@@ -3149,7 +3151,7 @@ wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
   __cvtptr_ = new _Codecvt;
 }
 
-#    ifndef _LIBCPP_CXX03_LANG
+#      ifndef _LIBCPP_CXX03_LANG
 
 template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
 inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
@@ -3161,7 +3163,7 @@ inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert
   __wc.__cvtptr_ = nullptr;
 }
 
-#    endif // _LIBCPP_CXX03_LANG
+#      endif // _LIBCPP_CXX03_LANG
 
 _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
@@ -3315,14 +3317,14 @@ private:
   bool __always_noconv_;
 
 public:
-#    ifndef _LIBCPP_CXX03_LANG
+#      ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
   explicit _LIBCPP_HIDE_FROM_ABI
   wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#    else
+#      else
   _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
   wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#    endif
+#      endif
 
   _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
 
@@ -3678,7 +3680,7 @@ wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__
 
 _LIBCPP_SUPPRESS_DEPRECATED_POP
 
-#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+#    endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
 
 _LIBCPP_END_NAMESPACE_STD
 
@@ -3686,17 +3688,18 @@ _LIBCPP_POP_MACROS
 
 // NOLINTEND(libcpp-robust-against-adl)
 
-#endif // _LIBCPP_HAS_LOCALIZATION
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdarg>
-#  include <iterator>
-#  include <mutex>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  endif // _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdarg>
+#    include <iterator>
+#    include <mutex>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_LOCALE
diff --git a/libcxx/include/map b/libcxx/include/map
index fabf39512ab75..4e964d2265b47 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -571,61 +571,63 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 
 */
 
-#include <__algorithm/equal.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__assert>
-#include <__config>
-#include <__functional/binary_function.h>
-#include <__functional/is_transparent.h>
-#include <__functional/operations.h>
-#include <__iterator/erase_if_container.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/ranges_iterator_traits.h>
-#include <__iterator/reverse_iterator.h>
-#include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/unique_ptr.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__node_handle>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__tree>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/pair.h>
-#include <__utility/piecewise_construct.h>
-#include <__utility/swap.h>
-#include <new> // for std::launder
-#include <stdexcept>
-#include <tuple>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/equal.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__assert>
+#  include <__config>
+#  include <__functional/binary_function.h>
+#  include <__functional/is_transparent.h>
+#  include <__functional/operations.h>
+#  include <__iterator/erase_if_container.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/ranges_iterator_traits.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/unique_ptr.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__node_handle>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__tree>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/pair.h>
+#  include <__utility/piecewise_construct.h>
+#  include <__utility/swap.h>
+#  include <new> // for std::launder
+#  include <stdexcept>
+#  include <tuple>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [associative.map.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -654,7 +656,7 @@ public:
     swap(static_cast<_Compare&>(*this), static_cast<_Compare&>(__y));
   }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
     return static_cast<const _Compare&>(*this)(__x, __y.__get_value().first);
@@ -664,7 +666,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
     return static_cast<const _Compare&>(*this)(__x.__get_value().first, __y);
   }
-#endif
+#  endif
 };
 
 template <class _Key, class _CP, class _Compare>
@@ -692,7 +694,7 @@ public:
     swap(__comp_, __y.__comp_);
   }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
     return __comp_(__x, __y.__get_value().first);
@@ -702,7 +704,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
     return __comp_(__x.__get_value().first, __y);
   }
-#endif
+#  endif
 };
 
 template <class _Key, class _CP, class _Compare, bool __b>
@@ -732,14 +734,14 @@ public:
         __first_constructed(false),
         __second_constructed(false) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __map_node_destructor(__tree_node_destructor<allocator_type>&& __x) _NOEXCEPT
       : __na_(__x.__na_),
         __first_constructed(__x.__value_constructed),
         __second_constructed(__x.__value_constructed) {
     __x.__value_constructed = false;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   __map_node_destructor& operator=(const __map_node_destructor&) = delete;
 
@@ -760,7 +762,7 @@ class multimap;
 template <class _TreeIterator>
 class __map_const_iterator;
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp>
 struct _LIBCPP_STANDALONE_DEBUG __value_type {
@@ -775,19 +777,19 @@ private:
 
 public:
   _LIBCPP_HIDE_FROM_ABI value_type& __get_value() {
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
     return *std::launder(std::addressof(__cc_));
-#  else
+#    else
     return __cc_;
-#  endif
+#    endif
   }
 
   _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const {
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
     return *std::launder(std::addressof(__cc_));
-#  else
+#    else
     return __cc_;
-#  endif
+#    endif
   }
 
   _LIBCPP_HIDE_FROM_ABI __nc_ref_pair_type __ref() {
@@ -822,7 +824,7 @@ public:
   __value_type(__value_type&&)      = delete;
 };
 
-#else
+#  else
 
 template <class _Key, class _Tp>
 struct __value_type {
@@ -843,7 +845,7 @@ public:
   ~__value_type()                              = delete;
 };
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 struct __extract_key_value_types;
@@ -1019,10 +1021,10 @@ public:
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __map_node_handle<typename __base::__node, allocator_type> node_type;
   typedef __insert_return_type<iterator, node_type> insert_return_type;
-#endif
+#  endif
 
   template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS map;
@@ -1054,7 +1056,7 @@ public:
     insert(__f, __l);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI
   map(from_range_t,
@@ -1064,37 +1066,37 @@ public:
       : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI map(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
       : map(__f, __l, key_compare(), __a) {}
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI map(from_range_t, _Range&& __range, const allocator_type& __a)
       : map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
 
   _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) {
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     __tree_ = __m.__tree_;
-#else
+#  else
     if (this != std::addressof(__m)) {
       __tree_.clear();
       __tree_.value_comp() = __m.__tree_.value_comp();
       __tree_.__copy_assign_alloc(__m.__tree_);
       insert(__m.begin(), __m.end());
     }
-#endif
+#  endif
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
       : __tree_(std::move(__m.__tree_)) {}
@@ -1116,17 +1118,17 @@ public:
     insert(__il.begin(), __il.end());
   }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const allocator_type& __a)
       : map(__il, key_compare(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI map& operator=(initializer_list<value_type> __il) {
     __tree_.__assign_unique(__il.begin(), __il.end());
     return *this;
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit map(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
 
@@ -1157,9 +1159,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
 
   _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
   _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
@@ -1168,7 +1170,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
   _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
     return __tree_.__emplace_unique(std::forward<_Args>(__args)...);
@@ -1189,7 +1191,7 @@ public:
     return __tree_.__insert_unique(__pos.__i_, std::forward<_Pp>(__p));
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__insert_unique(__v); }
 
@@ -1197,7 +1199,7 @@ public:
     return __tree_.__insert_unique(__p.__i_, __v);
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __v) {
     return __tree_.__insert_unique(std::move(__v));
   }
@@ -1207,7 +1209,7 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
-#endif
+#  endif
 
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
@@ -1215,7 +1217,7 @@ public:
       insert(__e.__i_, *__f);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     const_iterator __end = cend();
@@ -1223,9 +1225,9 @@ public:
       insert(__end.__i_, std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
@@ -1310,7 +1312,7 @@ public:
     return __r;
   }
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p.__i_); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __tree_.erase(__p.__i_); }
@@ -1320,7 +1322,7 @@ public:
   }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to map::insert()");
@@ -1361,13 +1363,13 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     __tree_.__node_handle_merge_unique(__source.__tree_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(map& __m) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__m.__tree_); }
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
@@ -1376,27 +1378,27 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
@@ -1406,11 +1408,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
@@ -1419,7 +1421,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __tree_.__equal_range_unique(__k);
@@ -1427,7 +1429,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __tree_.__equal_range_unique(__k);
   }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -1436,7 +1438,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#endif
+#  endif
 
 private:
   typedef typename __base::__node __node;
@@ -1448,12 +1450,12 @@ private:
   typedef __map_node_destructor<__node_allocator> _Dp;
   typedef unique_ptr<__node, _Dp> __node_holder;
 
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
-#endif
+#  endif
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Compare   = less<__iter_key_type<_InputIterator>>,
           class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
@@ -1463,7 +1465,7 @@ template <class _InputIterator,
 map(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
     -> map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Compare   = less<__range_key_type<_Range>>,
           class _Allocator = allocator<__range_to_alloc_type<_Range>>,
@@ -1471,7 +1473,7 @@ template <ranges::input_range _Range,
           class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
 map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
     -> map<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key,
           class _Tp,
@@ -1493,18 +1495,18 @@ map(_InputIterator, _InputIterator, _Allocator)
            less<__iter_key_type<_InputIterator>>,
            _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 map(from_range_t, _Range&&, _Allocator)
     -> map<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 map(initializer_list<pair<_Key, _Tp>>,
     _Allocator) -> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
     : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
@@ -1535,7 +1537,7 @@ _Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](key_type&& __k) {
   // NOLINTEND(bugprone-use-after-move)
 }
 
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 typename map<_Key, _Tp, _Compare, _Allocator>::__node_holder
@@ -1562,7 +1564,7 @@ _Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
   return __r->__value_.__get_value().second;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 _Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) {
@@ -1588,7 +1590,7 @@ operator==(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp,
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -1620,7 +1622,7 @@ operator<=(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp,
   return !(__y < __x);
 }
 
-#else // #if _LIBCPP_STD_VER <= 17
+#  else // #if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<pair<const _Key, _Tp>>
@@ -1628,7 +1630,7 @@ operator<=>(const map<_Key, _Tp, _Compare, _Allocator>& __x, const map<_Key, _Tp
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // #if _LIBCPP_STD_VER <= 17
+#  endif // #if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void
@@ -1637,13 +1639,13 @@ swap(map<_Key, _Tp, _Compare, _Allocator>& __x, map<_Key, _Tp, _Compare, _Alloca
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Tp, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename map<_Key, _Tp, _Compare, _Allocator>::size_type
 erase_if(map<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 struct __container_traits<map<_Key, _Tp, _Compare, _Allocator> > {
@@ -1703,9 +1705,9 @@ public:
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __map_node_handle<typename __base::__node, allocator_type> node_type;
-#endif
+#  endif
 
   template <class _Key2, class _Value2, class _Comp2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS map;
@@ -1737,7 +1739,7 @@ public:
     insert(__f, __l);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI
   multimap(from_range_t,
@@ -1747,19 +1749,19 @@ public:
       : __tree_(__vc(__comp), typename __base::allocator_type(__a)) {
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI multimap(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
       : multimap(__f, __l, key_compare(), __a) {}
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI multimap(from_range_t, _Range&& __range, const allocator_type& __a)
       : multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
       : __tree_(__m.__tree_.value_comp(),
@@ -1768,20 +1770,20 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) {
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     __tree_ = __m.__tree_;
-#else
+#  else
     if (this != std::addressof(__m)) {
       __tree_.clear();
       __tree_.value_comp() = __m.__tree_.value_comp();
       __tree_.__copy_assign_alloc(__m.__tree_);
       insert(__m.begin(), __m.end());
     }
-#endif
+#  endif
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
       : __tree_(std::move(__m.__tree_)) {}
@@ -1804,17 +1806,17 @@ public:
     insert(__il.begin(), __il.end());
   }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const allocator_type& __a)
       : multimap(__il, key_compare(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI multimap& operator=(initializer_list<value_type> __il) {
     __tree_.__assign_multi(__il.begin(), __il.end());
     return *this;
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit multimap(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
 
@@ -1848,7 +1850,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
   _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
@@ -1878,7 +1880,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __v) { return __tree_.__insert_multi(__v); }
 
@@ -1892,7 +1894,7 @@ public:
       __tree_.__insert_multi(__e.__i_, *__f);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     const_iterator __end = cend();
@@ -1900,7 +1902,7 @@ public:
       __tree_.__insert_multi(__end.__i_, std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p.__i_); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __tree_.erase(__p.__i_); }
@@ -1909,7 +1911,7 @@ public:
     return __tree_.erase(__f.__i_, __l.__i_);
   }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to multimap::insert()");
@@ -1950,7 +1952,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     return __tree_.__node_handle_merge_multi(__source.__tree_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
 
@@ -1960,7 +1962,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
@@ -1969,27 +1971,27 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
@@ -1999,11 +2001,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
@@ -2012,7 +2014,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -2020,7 +2022,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -2029,7 +2031,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#endif
+#  endif
 
 private:
   typedef typename __base::__node __node;
@@ -2040,7 +2042,7 @@ private:
   typedef unique_ptr<__node, _Dp> __node_holder;
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Compare   = less<__iter_key_type<_InputIterator>>,
           class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
@@ -2050,7 +2052,7 @@ template <class _InputIterator,
 multimap(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
     -> multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Compare   = less<__range_key_type<_Range>>,
           class _Allocator = allocator<__range_to_alloc_type<_Range>>,
@@ -2058,7 +2060,7 @@ template <ranges::input_range _Range,
           class            = enable_if_t<__is_allocator<_Allocator>::value, void>>
 multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
     -> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key,
           class _Tp,
@@ -2080,18 +2082,18 @@ multimap(_InputIterator, _InputIterator, _Allocator)
                 less<__iter_key_type<_InputIterator>>,
                 _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 multimap(from_range_t, _Range&&, _Allocator)
     -> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 multimap(initializer_list<pair<_Key, _Tp>>,
          _Allocator) -> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const allocator_type& __a)
     : __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
@@ -2101,7 +2103,7 @@ multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const alloca
       __tree_.__insert_multi(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_.__move()));
   }
 }
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -2109,7 +2111,7 @@ operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -2141,7 +2143,7 @@ operator<=(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<
   return !(__y < __x);
 }
 
-#else // #if _LIBCPP_STD_VER <= 17
+#  else // #if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<pair<const _Key, _Tp>>
@@ -2150,7 +2152,7 @@ operator<=>(const multimap<_Key, _Tp, _Compare, _Allocator>& __x,
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), __synth_three_way);
 }
 
-#endif // #if _LIBCPP_STD_VER <= 17
+#  endif // #if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void
@@ -2159,13 +2161,13 @@ swap(multimap<_Key, _Tp, _Compare, _Allocator>& __x, multimap<_Key, _Tp, _Compar
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Tp, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename multimap<_Key, _Tp, _Compare, _Allocator>::size_type
 erase_if(multimap<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
 struct __container_traits<multimap<_Key, _Tp, _Compare, _Allocator> > {
@@ -2177,7 +2179,7 @@ struct __container_traits<multimap<_Key, _Tp, _Compare, _Allocator> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _KeyT, class _ValueT, class _CompareT = std::less<_KeyT>>
@@ -2189,17 +2191,18 @@ using multimap _LIBCPP_AVAILABILITY_PMR =
     std::multimap<_KeyT, _ValueT, _CompareT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstdlib>
-#  include <functional>
-#  include <iterator>
-#  include <type_traits>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstdlib>
+#    include <functional>
+#    include <iterator>
+#    include <type_traits>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_MAP
diff --git a/libcxx/include/math.h b/libcxx/include/math.h
index 509ecc46184da..c338d3225dd55 100644
--- a/libcxx/include/math.h
+++ b/libcxx/include/math.h
@@ -291,93 +291,95 @@ long double    truncl(long double x);
 
 */
 
-#  include <__config>
+#  if 0
+#  else // 0
+#    include <__config>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
-#  if __has_include_next(<math.h>)
-#    include_next <math.h>
-#  endif
+#    if __has_include_next(<math.h>)
+#      include_next <math.h>
+#    endif
 
-#  ifdef __cplusplus
+#    ifdef __cplusplus
 
 // We support including .h headers inside 'extern "C"' contexts, so switch
 // back to C++ linkage before including these C++ headers.
 extern "C++" {
 
-#    ifdef fpclassify
-#      undef fpclassify
-#    endif
-
-#    ifdef signbit
-#      undef signbit
-#    endif
-
-#    ifdef isfinite
-#      undef isfinite
-#    endif
-
-#    ifdef isinf
-#      undef isinf
-#    endif
-
-#    ifdef isnan
-#      undef isnan
-#    endif
-
-#    ifdef isnormal
-#      undef isnormal
-#    endif
-
-#    ifdef isgreater
-#      undef isgreater
-#    endif
-
-#    ifdef isgreaterequal
-#      undef isgreaterequal
-#    endif
-
-#    ifdef isless
-#      undef isless
-#    endif
-
-#    ifdef islessequal
-#      undef islessequal
-#    endif
-
-#    ifdef islessgreater
-#      undef islessgreater
-#    endif
-
-#    ifdef isunordered
-#      undef isunordered
-#    endif
-
-#    include <__math/abs.h>
-#    include <__math/copysign.h>
-#    include <__math/error_functions.h>
-#    include <__math/exponential_functions.h>
-#    include <__math/fdim.h>
-#    include <__math/fma.h>
-#    include <__math/gamma.h>
-#    include <__math/hyperbolic_functions.h>
-#    include <__math/hypot.h>
-#    include <__math/inverse_hyperbolic_functions.h>
-#    include <__math/inverse_trigonometric_functions.h>
-#    include <__math/logarithms.h>
-#    include <__math/min_max.h>
-#    include <__math/modulo.h>
-#    include <__math/remainder.h>
-#    include <__math/roots.h>
-#    include <__math/rounding_functions.h>
-#    include <__math/traits.h>
-#    include <__math/trigonometric_functions.h>
-#    include <__type_traits/enable_if.h>
-#    include <__type_traits/is_floating_point.h>
-#    include <__type_traits/is_integral.h>
-#    include <stdlib.h>
+#      ifdef fpclassify
+#        undef fpclassify
+#      endif
+
+#      ifdef signbit
+#        undef signbit
+#      endif
+
+#      ifdef isfinite
+#        undef isfinite
+#      endif
+
+#      ifdef isinf
+#        undef isinf
+#      endif
+
+#      ifdef isnan
+#        undef isnan
+#      endif
+
+#      ifdef isnormal
+#        undef isnormal
+#      endif
+
+#      ifdef isgreater
+#        undef isgreater
+#      endif
+
+#      ifdef isgreaterequal
+#        undef isgreaterequal
+#      endif
+
+#      ifdef isless
+#        undef isless
+#      endif
+
+#      ifdef islessequal
+#        undef islessequal
+#      endif
+
+#      ifdef islessgreater
+#        undef islessgreater
+#      endif
+
+#      ifdef isunordered
+#        undef isunordered
+#      endif
+
+#      include <__math/abs.h>
+#      include <__math/copysign.h>
+#      include <__math/error_functions.h>
+#      include <__math/exponential_functions.h>
+#      include <__math/fdim.h>
+#      include <__math/fma.h>
+#      include <__math/gamma.h>
+#      include <__math/hyperbolic_functions.h>
+#      include <__math/hypot.h>
+#      include <__math/inverse_hyperbolic_functions.h>
+#      include <__math/inverse_trigonometric_functions.h>
+#      include <__math/logarithms.h>
+#      include <__math/min_max.h>
+#      include <__math/modulo.h>
+#      include <__math/remainder.h>
+#      include <__math/roots.h>
+#      include <__math/rounding_functions.h>
+#      include <__math/traits.h>
+#      include <__math/trigonometric_functions.h>
+#      include <__type_traits/enable_if.h>
+#      include <__type_traits/is_floating_point.h>
+#      include <__type_traits/is_integral.h>
+#      include <stdlib.h>
 
 // fpclassify relies on implementation-defined constants, so we can't move it to a detail header
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -415,7 +417,7 @@ using std::__math::fpclassify;
 using std::__math::signbit;
 
 // The MSVC runtime already provides these functions as templates
-#    ifndef _LIBCPP_MSVCRT
+#      ifndef _LIBCPP_MSVCRT
 using std::__math::isfinite;
 using std::__math::isgreater;
 using std::__math::isgreaterequal;
@@ -426,7 +428,7 @@ using std::__math::islessgreater;
 using std::__math::isnan;
 using std::__math::isnormal;
 using std::__math::isunordered;
-#    endif // _LIBCPP_MSVCRT
+#      endif // _LIBCPP_MSVCRT
 
 // abs
 //
@@ -501,7 +503,8 @@ using std::__math::trunc;
 
 } // extern "C++"
 
-#  endif // __cplusplus
+#    endif // __cplusplus
+#  endif   // 0
 
 #else // _LIBCPP_MATH_H
 
diff --git a/libcxx/include/mdspan b/libcxx/include/mdspan
index d6191a197e15c..02582c6f39c15 100644
--- a/libcxx/include/mdspan
+++ b/libcxx/include/mdspan
@@ -408,22 +408,25 @@ namespace std {
 #ifndef _LIBCPP_MDSPAN
 #define _LIBCPP_MDSPAN
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__fwd/mdspan.h>
-#  include <__mdspan/default_accessor.h>
-#  include <__mdspan/extents.h>
-#  include <__mdspan/layout_left.h>
-#  include <__mdspan/layout_right.h>
-#  include <__mdspan/layout_stride.h>
-#  include <__mdspan/mdspan.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__fwd/mdspan.h>
+#    include <__mdspan/default_accessor.h>
+#    include <__mdspan/extents.h>
+#    include <__mdspan/layout_left.h>
+#    include <__mdspan/layout_right.h>
+#    include <__mdspan/layout_stride.h>
+#    include <__mdspan/mdspan.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_MDSPAN
diff --git a/libcxx/include/memory b/libcxx/include/memory
index db3386cca4800..883666b9e1bef 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -934,65 +934,68 @@ template<class Pointer = void, class Smart, class... Args>
 
 // clang-format on
 
-#include <__config>
-#include <__memory/addressof.h>
-#include <__memory/align.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_arg_t.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/auto_ptr.h>
-#include <__memory/inout_ptr.h>
-#include <__memory/out_ptr.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/raw_storage_iterator.h>
-#include <__memory/shared_ptr.h>
-#include <__memory/temporary_buffer.h>
-#include <__memory/uninitialized_algorithms.h>
-#include <__memory/unique_ptr.h>
-#include <__memory/uses_allocator.h>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__memory/addressof.h>
+#  include <__memory/align.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_arg_t.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/auto_ptr.h>
+#  include <__memory/inout_ptr.h>
+#  include <__memory/out_ptr.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/raw_storage_iterator.h>
+#  include <__memory/shared_ptr.h>
+#  include <__memory/temporary_buffer.h>
+#  include <__memory/uninitialized_algorithms.h>
+#  include <__memory/unique_ptr.h>
+#  include <__memory/uses_allocator.h>
 
 // standard-mandated includes
 
-#if _LIBCPP_STD_VER >= 17
-#  include <__memory/construct_at.h>
-#endif
+#  if _LIBCPP_STD_VER >= 17
+#    include <__memory/construct_at.h>
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
-#  include <__memory/assume_aligned.h>
-#  include <__memory/concepts.h>
-#  include <__memory/ranges_construct_at.h>
-#  include <__memory/ranges_uninitialized_algorithms.h>
-#  include <__memory/uses_allocator_construction.h>
-#endif
+#  if _LIBCPP_STD_VER >= 20
+#    include <__memory/assume_aligned.h>
+#    include <__memory/concepts.h>
+#    include <__memory/ranges_construct_at.h>
+#    include <__memory/ranges_uninitialized_algorithms.h>
+#    include <__memory/uses_allocator_construction.h>
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
-#  include <__memory/allocate_at_least.h>
-#endif
+#  if _LIBCPP_STD_VER >= 23
+#    include <__memory/allocate_at_least.h>
+#  endif
 
-#include <version>
+#  include <version>
 
 // [memory.syn]
-#include <compare>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstddef>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <cstring>
-#  include <iosfwd>
-#  include <iterator>
-#  include <new>
-#  include <stdexcept>
-#  include <tuple>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  include <compare>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstddef>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <cstring>
+#    include <iosfwd>
+#    include <iterator>
+#    include <new>
+#    include <stdexcept>
+#    include <tuple>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_MEMORY
diff --git a/libcxx/include/memory_resource b/libcxx/include/memory_resource
index e54b7e6e2473f..e852e96536c97 100644
--- a/libcxx/include/memory_resource
+++ b/libcxx/include/memory_resource
@@ -49,29 +49,32 @@ namespace std::pmr {
 
  */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__memory_resource/memory_resource.h>
-#  include <__memory_resource/monotonic_buffer_resource.h>
-#  include <__memory_resource/polymorphic_allocator.h>
-#  include <__memory_resource/pool_options.h>
-#  include <__memory_resource/synchronized_pool_resource.h>
-#  include <__memory_resource/unsynchronized_pool_resource.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER >= 17 && _LIBCPP_STD_VER <= 20
-#  include <mutex>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <stdexcept>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__memory_resource/memory_resource.h>
+#    include <__memory_resource/monotonic_buffer_resource.h>
+#    include <__memory_resource/polymorphic_allocator.h>
+#    include <__memory_resource/pool_options.h>
+#    include <__memory_resource/synchronized_pool_resource.h>
+#    include <__memory_resource/unsynchronized_pool_resource.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER >= 17 && _LIBCPP_STD_VER <= 20
+#    include <mutex>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <stdexcept>
+#  endif
+#endif // 0
 
 #endif /* _LIBCPP_MEMORY_RESOURCE */
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index ed2b7fb192164..81621f467aba7 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -856,13 +856,18 @@ module std [system] {
     }
     module check_memory_order     { header "__atomic/check_memory_order.h" }
     module contention_t           { header "__atomic/contention_t.h" }
-    module cxx_atomic_impl        { header "__atomic/cxx_atomic_impl.h" }
     module fence                  { header "__atomic/fence.h" }
     module is_always_lock_free    { header "__atomic/is_always_lock_free.h" }
     module kill_dependency        { header "__atomic/kill_dependency.h" }
     module memory_order           { header "__atomic/memory_order.h" }
     module to_gcc_order           { header "__atomic/to_gcc_order.h" }
 
+    module support {
+      header "__atomic/support.h"
+      textual header "__atomic/support/c11.h"
+      textual header "__atomic/support/gcc.h"
+    }
+
     header "atomic"
     export *
   }
@@ -1230,6 +1235,7 @@ module std [system] {
 
   module flat_map {
     module flat_map                       { header "__flat_map/flat_map.h" }
+    module key_value_iterator             { header "__flat_map/key_value_iterator.h" }
     module sorted_unique                  { header "__flat_map/sorted_unique.h" }
 
     header "flat_map"
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index ecbfc5607dcc7..b699439c5ce5a 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -186,34 +186,36 @@ template<class Callable, class ...Args>
 
 */
 
-#include <__chrono/steady_clock.h>
-#include <__chrono/time_point.h>
-#include <__condition_variable/condition_variable.h>
-#include <__config>
-#include <__mutex/lock_guard.h>
-#include <__mutex/mutex.h>
-#include <__mutex/once_flag.h>
-#include <__mutex/tag_types.h>
-#include <__mutex/unique_lock.h>
-#include <__thread/id.h>
-#include <__thread/support.h>
-#include <__utility/forward.h>
-#include <limits>
-#ifndef _LIBCPP_CXX03_LANG
-#  include <tuple>
-#endif
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__chrono/steady_clock.h>
+#  include <__chrono/time_point.h>
+#  include <__condition_variable/condition_variable.h>
+#  include <__config>
+#  include <__mutex/lock_guard.h>
+#  include <__mutex/mutex.h>
+#  include <__mutex/once_flag.h>
+#  include <__mutex/tag_types.h>
+#  include <__mutex/unique_lock.h>
+#  include <__thread/id.h>
+#  include <__thread/support.h>
+#  include <__utility/forward.h>
+#  include <limits>
+#  ifndef _LIBCPP_CXX03_LANG
+#    include <tuple>
+#  endif
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_HAS_THREADS
+#  if _LIBCPP_HAS_THREADS
 
 class _LIBCPP_EXPORTED_FROM_ABI recursive_mutex {
   __libcpp_recursive_mutex_t __m_;
@@ -333,7 +335,7 @@ _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) {
   return 0;
 }
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
 
 template <class _L0, class _L1, class _L2, class... _L3>
 _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
@@ -349,7 +351,7 @@ _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3
   return __r;
 }
 
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
 template <class _L0, class _L1>
 _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1) {
@@ -373,7 +375,7 @@ _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1) {
   }
 }
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
 
 template <class _L0, class _L1, class _L2, class... _L3>
 void __lock_first(int __i, _L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
@@ -416,9 +418,9 @@ inline _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&...
   std::__lock_first(0, __l0, __l1, __l2, __l3...);
 }
 
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 template <class... _Mutexes>
 class _LIBCPP_TEMPLATE_VIS scoped_lock;
 
@@ -489,26 +491,27 @@ private:
 };
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(scoped_lock);
 
-#  endif // _LIBCPP_STD_VER >= 17
-#endif   // _LIBCPP_HAS_THREADS
+#    endif // _LIBCPP_STD_VER >= 17
+#  endif   // _LIBCPP_HAS_THREADS
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <cstring>
-#  include <ctime>
-#  include <initializer_list>
-#  include <iosfwd>
-#  include <new>
-#  include <stdexcept>
-#  include <system_error>
-#  include <type_traits>
-#  include <typeinfo>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <cstring>
+#    include <ctime>
+#    include <initializer_list>
+#    include <iosfwd>
+#    include <new>
+#    include <stdexcept>
+#    include <system_error>
+#    include <type_traits>
+#    include <typeinfo>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_MUTEX
diff --git a/libcxx/include/new b/libcxx/include/new
index 49e500809e158..8bb7b300d08bc 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -86,45 +86,47 @@ void  operator delete[](void* ptr, void*) noexcept;
 
 */
 
-#include <__config>
-#include <__cstddef/max_align_t.h>
-#include <__cstddef/size_t.h>
-#include <__exception/exception.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_void.h>
-#include <__verbose_abort>
-#include <version>
-
-#if defined(_LIBCPP_ABI_VCRUNTIME)
-#  include <new.h>
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
-#  define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 1
-#else
-#  define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 0
-#endif
-
-#if _LIBCPP_STD_VER >= 14 || _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
-#  define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 1
-#else
-#  define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 0
-#endif
-
-#if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION && _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
-#  define _LIBCPP_HAS_SIZED_DEALLOCATION 1
-#else
-#  define _LIBCPP_HAS_SIZED_DEALLOCATION 0
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/max_align_t.h>
+#  include <__cstddef/size_t.h>
+#  include <__exception/exception.h>
+#  include <__type_traits/is_function.h>
+#  include <__type_traits/is_void.h>
+#  include <__verbose_abort>
+#  include <version>
+
+#  if defined(_LIBCPP_ABI_VCRUNTIME)
+#    include <new.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
+#    define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 1
+#  else
+#    define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 0
+#  endif
+
+#  if _LIBCPP_STD_VER >= 14 || _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
+#    define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 1
+#  else
+#    define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 0
+#  endif
+
+#  if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION && _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
+#    define _LIBCPP_HAS_SIZED_DEALLOCATION 1
+#  else
+#    define _LIBCPP_HAS_SIZED_DEALLOCATION 0
+#  endif
 
 namespace std // purposefully not using versioning namespace
 {
 
-#if !defined(_LIBCPP_ABI_VCRUNTIME)
+#  if !defined(_LIBCPP_ABI_VCRUNTIME)
 struct _LIBCPP_EXPORTED_FROM_ABI nothrow_t {
   explicit nothrow_t() = default;
 };
@@ -152,7 +154,7 @@ typedef void (*new_handler)();
 _LIBCPP_EXPORTED_FROM_ABI new_handler set_new_handler(new_handler) _NOEXCEPT;
 _LIBCPP_EXPORTED_FROM_ABI new_handler get_new_handler() _NOEXCEPT;
 
-#elif defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0 // !_LIBCPP_ABI_VCRUNTIME
+#  elif defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0 // !_LIBCPP_ABI_VCRUNTIME
 
 // When _HAS_EXCEPTIONS == 0, these complete definitions are needed,
 // since they would normally be provided in vcruntime_exception.h
@@ -170,72 +172,72 @@ class bad_array_new_length : public bad_alloc {
 public:
   bad_array_new_length() noexcept : bad_alloc("bad array new length") {}
 };
-#endif // defined(_LIBCPP_ABI_VCRUNTIME) && defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0
+#  endif // defined(_LIBCPP_ABI_VCRUNTIME) && defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0
 
 [[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void __throw_bad_alloc(); // not in C++ spec
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_array_new_length() {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_array_new_length();
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("bad_array_new_length was thrown in -fno-exceptions mode");
-#endif
+#  endif
 }
 
-#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION && !defined(_LIBCPP_ABI_VCRUNTIME)
-#  ifndef _LIBCPP_CXX03_LANG
+#  if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION && !defined(_LIBCPP_ABI_VCRUNTIME)
+#    ifndef _LIBCPP_CXX03_LANG
 enum class align_val_t : size_t {};
-#  else
+#    else
 enum align_val_t { __zero = 0, __max = (size_t)-1 };
+#    endif
 #  endif
-#endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 // Enable the declaration even if the compiler doesn't support the language
 // feature.
 struct destroying_delete_t {
   explicit destroying_delete_t() = default;
 };
 inline constexpr destroying_delete_t destroying_delete{};
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 } // namespace std
 
-#if defined(_LIBCPP_CXX03_LANG)
-#  define _THROW_BAD_ALLOC throw(std::bad_alloc)
-#else
-#  define _THROW_BAD_ALLOC
-#endif
+#  if defined(_LIBCPP_CXX03_LANG)
+#    define _THROW_BAD_ALLOC throw(std::bad_alloc)
+#  else
+#    define _THROW_BAD_ALLOC
+#  endif
 
-#if !defined(_LIBCPP_ABI_VCRUNTIME)
+#  if !defined(_LIBCPP_ABI_VCRUNTIME)
 
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz) _THROW_BAD_ALLOC;
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t&) _NOEXCEPT
     _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT;
-#  if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
+#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _NOEXCEPT;
-#  endif
+#    endif
 
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz) _THROW_BAD_ALLOC;
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t&) _NOEXCEPT
     _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT;
-#  if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
+#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz) _NOEXCEPT;
-#  endif
+#    endif
 
-#  if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
+#    if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void*
 operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
-#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
+#      if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
-#    endif
+#      endif
 
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void*
 operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
@@ -243,10 +245,10 @@ operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
 operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
-#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
+#      if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
+#      endif
 #    endif
-#  endif
 
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void*
 operator new(std::size_t, void* __p) _NOEXCEPT {
@@ -259,43 +261,43 @@ operator new[](std::size_t, void* __p) _NOEXCEPT {
 inline _LIBCPP_HIDE_FROM_ABI void operator delete(void*, void*) _NOEXCEPT {}
 inline _LIBCPP_HIDE_FROM_ABI void operator delete[](void*, void*) _NOEXCEPT {}
 
-#endif // !_LIBCPP_ABI_VCRUNTIME
+#  endif // !_LIBCPP_ABI_VCRUNTIME
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 _LIBCPP_CONSTEXPR inline _LIBCPP_HIDE_FROM_ABI bool __is_overaligned_for_new(size_t __align) _NOEXCEPT {
-#ifdef __STDCPP_DEFAULT_NEW_ALIGNMENT__
+#  ifdef __STDCPP_DEFAULT_NEW_ALIGNMENT__
   return __align > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
-#else
+#  else
   return __align > _LIBCPP_ALIGNOF(max_align_t);
-#endif
+#  endif
 }
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void* __libcpp_operator_new(_Args... __args) {
-#if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
+#  if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
   return __builtin_operator_new(__args...);
-#else
+#  else
   return ::operator new(__args...);
-#endif
+#  endif
 }
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) _NOEXCEPT {
-#if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
+#  if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
   __builtin_operator_delete(__args...);
-#else
+#  else
   ::operator delete(__args...);
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __align) {
-#if _LIBCPP_HAS_ALIGNED_ALLOCATION
+#  if _LIBCPP_HAS_ALIGNED_ALLOCATION
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
     return __libcpp_operator_new(__size, __align_val);
   }
-#endif
+#  endif
 
   (void)__align;
   return __libcpp_operator_new(__size);
@@ -303,40 +305,40 @@ inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __ali
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void __do_deallocate_handle_size(void* __ptr, size_t __size, _Args... __args) _NOEXCEPT {
-#if !_LIBCPP_HAS_SIZED_DEALLOCATION
+#  if !_LIBCPP_HAS_SIZED_DEALLOCATION
   (void)__size;
   return std::__libcpp_operator_delete(__ptr, __args...);
-#else
+#  else
   return std::__libcpp_operator_delete(__ptr, __size, __args...);
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) _NOEXCEPT {
-#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
+#  if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
   return __do_deallocate_handle_size(__ptr, __size);
-#else
+#  else
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
     return __do_deallocate_handle_size(__ptr, __size, __align_val);
   } else {
     return __do_deallocate_handle_size(__ptr, __size);
   }
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate_unsized(void* __ptr, size_t __align) _NOEXCEPT {
-#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
+#  if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
   return __libcpp_operator_delete(__ptr);
-#else
+#  else
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
     return __libcpp_operator_delete(__ptr, __align_val);
   } else {
     return __libcpp_operator_delete(__ptr);
   }
-#endif
+#  endif
 }
 
 template <class _Tp>
@@ -346,30 +348,31 @@ template <class _Tp>
   return __builtin_launder(__p);
 }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 [[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp* launder(_Tp* __p) noexcept {
   return std::__launder(__p);
 }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
-#  if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
+#    if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
 
 inline constexpr size_t hardware_destructive_interference_size  = __GCC_DESTRUCTIVE_SIZE;
 inline constexpr size_t hardware_constructive_interference_size = __GCC_CONSTRUCTIVE_SIZE;
 
-#  endif // defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
+#    endif // defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_NEW
diff --git a/libcxx/include/numbers b/libcxx/include/numbers
index 191563c3d8a5f..41ee888ba8a9c 100644
--- a/libcxx/include/numbers
+++ b/libcxx/include/numbers
@@ -58,15 +58,17 @@ namespace std::numbers {
 }
 */
 
-#include <__concepts/arithmetic.h>
-#include <__config>
-#include <version>
+#if 0
+#else // 0
+#  include <__concepts/arithmetic.h>
+#  include <__config>
+#  include <version>
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -154,12 +156,13 @@ inline constexpr double phi        = phi_v<double>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstddef>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstddef>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_NUMBERS
diff --git a/libcxx/include/numeric b/libcxx/include/numeric
index 6b92ce3a07123..6055524b04997 100644
--- a/libcxx/include/numeric
+++ b/libcxx/include/numeric
@@ -156,52 +156,55 @@ constexpr T saturate_cast(U x) noexcept;                    // freestanding, Sin
 
 */
 
-#include <__config>
-
-#include <__numeric/accumulate.h>
-#include <__numeric/adjacent_difference.h>
-#include <__numeric/inner_product.h>
-#include <__numeric/iota.h>
-#include <__numeric/partial_sum.h>
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__numeric/exclusive_scan.h>
-#  include <__numeric/gcd_lcm.h>
-#  include <__numeric/inclusive_scan.h>
-#  include <__numeric/pstl.h>
-#  include <__numeric/reduce.h>
-#  include <__numeric/transform_exclusive_scan.h>
-#  include <__numeric/transform_inclusive_scan.h>
-#  include <__numeric/transform_reduce.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__numeric/midpoint.h>
-#  include <__numeric/saturation_arithmetic.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
-#  include <initializer_list>
-#  include <limits>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <climits>
-#  include <cmath>
-#  include <concepts>
-#  include <cstdint>
-#  include <execution>
-#  include <functional>
-#  include <iterator>
-#  include <new>
-#  include <optional>
-#  include <type_traits>
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__numeric/accumulate.h>
+#  include <__numeric/adjacent_difference.h>
+#  include <__numeric/inner_product.h>
+#  include <__numeric/iota.h>
+#  include <__numeric/partial_sum.h>
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__numeric/exclusive_scan.h>
+#    include <__numeric/gcd_lcm.h>
+#    include <__numeric/inclusive_scan.h>
+#    include <__numeric/pstl.h>
+#    include <__numeric/reduce.h>
+#    include <__numeric/transform_exclusive_scan.h>
+#    include <__numeric/transform_inclusive_scan.h>
+#    include <__numeric/transform_reduce.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__numeric/midpoint.h>
+#    include <__numeric/saturation_arithmetic.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#    include <initializer_list>
+#    include <limits>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <climits>
+#    include <cmath>
+#    include <concepts>
+#    include <cstdint>
+#    include <execution>
+#    include <functional>
+#    include <iterator>
+#    include <new>
+#    include <optional>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_NUMERIC
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 7ad6a9e116941..b19c7a21aea37 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -177,69 +177,71 @@ namespace std {
 
 */
 
-#include <__assert>
-#include <__compare/compare_three_way_result.h>
-#include <__compare/ordering.h>
-#include <__compare/three_way_comparable.h>
-#include <__concepts/invocable.h>
-#include <__config>
-#include <__exception/exception.h>
-#include <__functional/hash.h>
-#include <__functional/invoke.h>
-#include <__functional/unary_function.h>
-#include <__fwd/functional.h>
-#include <__memory/addressof.h>
-#include <__memory/construct_at.h>
-#include <__tuple/sfinae_helpers.h>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/conjunction.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/disjunction.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/invoke.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_assignable.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_destructible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_object.h>
-#include <__type_traits/is_reference.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_scalar.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_assignable.h>
-#include <__type_traits/is_trivially_constructible.h>
-#include <__type_traits/is_trivially_destructible.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/negation.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/in_place.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <__verbose_abort>
-#include <initializer_list>
-#include <new>
-#include <version>
+#if 0
+#else // 0
+#  include <__assert>
+#  include <__compare/compare_three_way_result.h>
+#  include <__compare/ordering.h>
+#  include <__compare/three_way_comparable.h>
+#  include <__concepts/invocable.h>
+#  include <__config>
+#  include <__exception/exception.h>
+#  include <__functional/hash.h>
+#  include <__functional/invoke.h>
+#  include <__functional/unary_function.h>
+#  include <__fwd/functional.h>
+#  include <__memory/addressof.h>
+#  include <__memory/construct_at.h>
+#  include <__tuple/sfinae_helpers.h>
+#  include <__type_traits/add_pointer.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/conjunction.h>
+#  include <__type_traits/decay.h>
+#  include <__type_traits/disjunction.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_assignable.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_destructible.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_object.h>
+#  include <__type_traits/is_reference.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_scalar.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/is_trivially_assignable.h>
+#  include <__type_traits/is_trivially_constructible.h>
+#  include <__type_traits/is_trivially_destructible.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/negation.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/remove_cv.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/in_place.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <__verbose_abort>
+#  include <initializer_list>
+#  include <new>
+#  include <version>
 
 // standard-mandated includes
 
 // [optional.syn]
-#include <compare>
+#  include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 namespace std // purposefully not using versioning namespace
 {
@@ -256,17 +258,17 @@ public:
 
 } // namespace std
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS void
 __throw_bad_optional_access() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_optional_access();
-#  else
+#    else
   _LIBCPP_VERBOSE_ABORT("bad_optional_access was thrown in -fno-exceptions mode");
-#  endif
+#    endif
 }
 
 struct nullopt_t {
@@ -304,12 +306,12 @@ struct __optional_destruct_base<_Tp, false> {
   _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(in_place_t, _Args&&... __args)
       : __val_(std::forward<_Args>(__args)...), __engaged_(true) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class _Fp, class... _Args>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(
       __optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
       : __val_(std::invoke(std::forward<_Fp>(__f), std::forward<_Args>(__args)...)), __engaged_(true) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept {
     if (__engaged_) {
@@ -335,12 +337,12 @@ struct __optional_destruct_base<_Tp, true> {
   _LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_destruct_base(in_place_t, _Args&&... __args)
       : __val_(std::forward<_Args>(__args)...), __engaged_(true) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class _Fp, class... _Args>
   _LIBCPP_HIDE_FROM_ABI constexpr __optional_destruct_base(
       __optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
       : __val_(std::invoke(std::forward<_Fp>(__f), std::forward<_Args>(__args)...)), __engaged_(true) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept {
     if (__engaged_) {
@@ -564,12 +566,12 @@ using __optional_sfinae_assign_base_t =
 template <class _Tp>
 class optional;
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class _Tp>
 concept __is_derived_from_optional = requires(const _Tp& __t) { []<class _Up>(const optional<_Up>&) {}(__t); };
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 template <class _Tp>
 struct __is_std_optional : false_type {};
@@ -711,14 +713,14 @@ public:
     this->__construct_from(std::move(__v));
   }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class _Tag,
             class _Fp,
             class... _Args,
             __enable_if_t<_IsSame<_Tag, __optional_construct_from_invoke_tag>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_Tag, _Fp&& __f, _Args&&... __args)
       : __base(__optional_construct_from_invoke_tag{}, std::forward<_Fp>(__f), std::forward<_Args>(__args)...) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(nullopt_t) noexcept {
     reset();
@@ -864,7 +866,7 @@ public:
     return this->has_value() ? std::move(this->__get()) : static_cast<value_type>(std::forward<_Up>(__v));
   }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class _Func>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) & {
     using _Up = invoke_result_t<_Func, value_type&>;
@@ -974,15 +976,15 @@ public:
       return std::move(*this);
     return std::forward<_Func>(__f)();
   }
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   using __base::reset;
 };
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 optional(_Tp) -> optional<_Tp>;
-#  endif
+#    endif
 
 // Comparisons between optionals
 template <class _Tp, class _Up>
@@ -1057,7 +1059,7 @@ operator>=(const optional<_Tp>& __x, const optional<_Up>& __y) {
   return *__x >= *__y;
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class _Tp, three_way_comparable_with<_Tp> _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Tp, _Up>
@@ -1067,7 +1069,7 @@ operator<=>(const optional<_Tp>& __x, const optional<_Up>& __y) {
   return __x.has_value() <=> __y.has_value();
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 // Comparisons with nullopt
 template <class _Tp>
@@ -1075,7 +1077,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const optional<_Tp>& __x, nullop
   return !static_cast<bool>(__x);
 }
 
-#  if _LIBCPP_STD_VER <= 17
+#    if _LIBCPP_STD_VER <= 17
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(nullopt_t, const optional<_Tp>& __x) noexcept {
@@ -1132,14 +1134,14 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(nullopt_t, const optional<_Tp>&
   return !static_cast<bool>(__x);
 }
 
-#  else // _LIBCPP_STD_VER <= 17
+#    else // _LIBCPP_STD_VER <= 17
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const optional<_Tp>& __x, nullopt_t) noexcept {
   return __x.has_value() <=> false;
 }
 
-#  endif // _LIBCPP_STD_VER <= 17
+#    endif // _LIBCPP_STD_VER <= 17
 
 // Comparisons with T
 template <class _Tp, class _Up>
@@ -1238,7 +1240,7 @@ operator>=(const _Tp& __v, const optional<_Up>& __x) {
   return static_cast<bool>(__x) ? __v >= *__x : true;
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class _Tp, class _Up>
   requires(!__is_derived_from_optional<_Up>) && three_way_comparable_with<_Tp, _Up>
@@ -1247,7 +1249,7 @@ operator<=>(const optional<_Tp>& __x, const _Up& __v) {
   return __x.has_value() ? *__x <=> __v : strong_ordering::less;
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI
@@ -1273,10 +1275,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(initializer_list<_Up
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS hash< __enable_hash_helper<optional<_Tp>, remove_const_t<_Tp>> > {
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS)
   _LIBCPP_DEPRECATED_IN_CXX17 typedef optional<_Tp> argument_type;
   _LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const optional<_Tp>& __opt) const {
     return static_cast<bool>(__opt) ? hash<remove_const_t<_Tp>>()(*__opt) : 0;
@@ -1285,25 +1287,26 @@ struct _LIBCPP_TEMPLATE_VIS hash< __enable_hash_helper<optional<_Tp>, remove_con
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <climits>
-#  include <concepts>
-#  include <ctime>
-#  include <iterator>
-#  include <limits>
-#  include <memory>
-#  include <ratio>
-#  include <stdexcept>
-#  include <tuple>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#  include <variant>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <climits>
+#    include <concepts>
+#    include <ctime>
+#    include <iterator>
+#    include <limits>
+#    include <memory>
+#    include <ratio>
+#    include <stdexcept>
+#    include <tuple>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#    include <variant>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_OPTIONAL
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index a58ac9b944fa2..900863c035be7 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -172,35 +172,38 @@ void vprint_nonunicode(ostream& os, string_view fmt, format_args args);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_LOCALIZATION
+#  if _LIBCPP_HAS_LOCALIZATION
 
-#  include <__ostream/basic_ostream.h>
+#    include <__ostream/basic_ostream.h>
 
-#  if _LIBCPP_STD_VER >= 23
-#    include <__ostream/print.h>
-#  endif
+#    if _LIBCPP_STD_VER >= 23
+#      include <__ostream/print.h>
+#    endif
 
-#  include <version>
+#    include <version>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
-#endif // _LIBCPP_HAS_LOCALIZATION
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdio>
-#  include <cstdlib>
-#  include <format>
-#  include <iosfwd>
-#  include <iterator>
-#  include <print>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
+#  endif // _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdio>
+#    include <cstdlib>
+#    include <format>
+#    include <iosfwd>
+#    include <iterator>
+#    include <print>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_OSTREAM
diff --git a/libcxx/include/print b/libcxx/include/print
index 8a8b686d18f34..3d846aeab3a88 100644
--- a/libcxx/include/print
+++ b/libcxx/include/print
@@ -33,28 +33,30 @@ namespace std {
 }
 */
 
-#include <__assert>
-#include <__concepts/same_as.h>
-#include <__config>
-#include <__system_error/throw_system_error.h>
-#include <__utility/forward.h>
-#include <cerrno>
-#include <cstdio>
-#include <format>
-#include <string>
-#include <string_view>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__assert>
+#  include <__concepts/same_as.h>
+#  include <__config>
+#  include <__system_error/throw_system_error.h>
+#  include <__utility/forward.h>
+#  include <cerrno>
+#  include <cstdio>
+#  include <format>
+#  include <string>
+#  include <string_view>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifdef _LIBCPP_WIN32API
+#  ifdef _LIBCPP_WIN32API
 _LIBCPP_EXPORTED_FROM_ABI bool __is_windows_terminal(FILE* __stream);
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 // A wrapper for WriteConsoleW which is used to write to the Windows
 // console. This function is in the dylib to avoid pulling in windows.h
 // in the library headers. The function itself uses some private parts
@@ -65,14 +67,14 @@ _LIBCPP_EXPORTED_FROM_ABI bool __is_windows_terminal(FILE* __stream);
 //
 // Note the function is only implemented on the Windows platform.
 _LIBCPP_EXPORTED_FROM_ABI void __write_to_windows_console(FILE* __stream, wstring_view __view);
-#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
-#elif __has_include(<unistd.h>)
+#    endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#  elif __has_include(<unistd.h>)
 _LIBCPP_EXPORTED_FROM_ABI bool __is_posix_terminal(FILE* __stream);
-#endif // _LIBCPP_WIN32API
+#  endif // _LIBCPP_WIN32API
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 
-#  if _LIBCPP_HAS_UNICODE
+#    if _LIBCPP_HAS_UNICODE
 // This is the code to transcode UTF-8 to UTF-16. This is used on
 // Windows for the native Unicode API. The code is modeled to make it
 // easier to extend to
@@ -86,27 +88,27 @@ namespace __unicode {
 // The names of these concepts are modelled after P2728R0, but the
 // implementation is not. char16_t may contain 32-bits so depending on the
 // number of bits is an issue.
-#    ifdef _LIBCPP_SHORT_WCHAR
+#      ifdef _LIBCPP_SHORT_WCHAR
 template <class _Tp>
 concept __utf16_code_unit =
     same_as<_Tp, char16_t>
-#      if _LIBCPP_HAS_WIDE_CHARACTERS
+#        if _LIBCPP_HAS_WIDE_CHARACTERS
     || same_as<_Tp, wchar_t>
-#      endif
+#        endif
     ;
 template <class _Tp>
 concept __utf32_code_unit = same_as<_Tp, char32_t>;
-#    else // _LIBCPP_SHORT_WCHAR
+#      else // _LIBCPP_SHORT_WCHAR
 template <class _Tp>
 concept __utf16_code_unit = same_as<_Tp, char16_t>;
 template <class _Tp>
 concept __utf32_code_unit =
     same_as<_Tp, char32_t>
-#      if _LIBCPP_HAS_WIDE_CHARACTERS
+#        if _LIBCPP_HAS_WIDE_CHARACTERS
     || same_as<_Tp, wchar_t>
-#      endif
+#        endif
     ;
-#    endif // _LIBCPP_SHORT_WCHAR
+#      endif // _LIBCPP_SHORT_WCHAR
 
 // Pass by reference since an output_iterator may not be copyable.
 template <class _OutIt>
@@ -164,7 +166,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr _OutIt __transcode(_InIt __first, _InIt __last,
 
 } // namespace __unicode
 
-#  endif //  _LIBCPP_HAS_UNICODE
+#    endif //  _LIBCPP_HAS_UNICODE
 
 namespace __print {
 
@@ -184,30 +186,30 @@ namespace __print {
 //   (note at the time of writing Clang is hard-coded to UTF-8.)
 //
 
-#  if !_LIBCPP_HAS_UNICODE
+#    if !_LIBCPP_HAS_UNICODE
 inline constexpr bool __use_unicode_execution_charset = false;
-#  elif defined(_MSVC_EXECUTION_CHARACTER_SET)
+#    elif defined(_MSVC_EXECUTION_CHARACTER_SET)
 // This is the same test MSVC STL uses in their implementation of <print>
 // See: https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
 inline constexpr bool __use_unicode_execution_charset = _MSVC_EXECUTION_CHARACTER_SET == 65001;
-#  else
+#    else
 inline constexpr bool __use_unicode_execution_charset = true;
-#  endif
+#    endif
 
 _LIBCPP_HIDE_FROM_ABI inline bool __is_terminal([[maybe_unused]] FILE* __stream) {
   // The macro _LIBCPP_TESTING_PRINT_IS_TERMINAL is used to change
   // the behavior in the test. This is not part of the public API.
-#  ifdef _LIBCPP_TESTING_PRINT_IS_TERMINAL
+#    ifdef _LIBCPP_TESTING_PRINT_IS_TERMINAL
   return _LIBCPP_TESTING_PRINT_IS_TERMINAL(__stream);
-#  elif _LIBCPP_AVAILABILITY_HAS_PRINT == 0 || !_LIBCPP_HAS_TERMINAL
+#    elif _LIBCPP_AVAILABILITY_HAS_PRINT == 0 || !_LIBCPP_HAS_TERMINAL
   return false;
-#  elif defined(_LIBCPP_WIN32API)
+#    elif defined(_LIBCPP_WIN32API)
   return std::__is_windows_terminal(__stream);
-#  elif __has_include(<unistd.h>)
+#    elif __has_include(<unistd.h>)
   return std::__is_posix_terminal(__stream);
-#  else
-#    error "Provide a way to determine whether a FILE* is a terminal"
-#  endif
+#    else
+#      error "Provide a way to determine whether a FILE* is a terminal"
+#    endif
 }
 
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
@@ -226,7 +228,7 @@ __vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args, bool
   }
 }
 
-#  if _LIBCPP_HAS_UNICODE
+#    if _LIBCPP_HAS_UNICODE
 
 // Note these helper functions are mainly used to aid testing.
 // On POSIX systems and Windows the output is no longer considered a
@@ -243,7 +245,7 @@ __vprint_unicode_posix(FILE* __stream, string_view __fmt, format_args __args, bo
   __print::__vprint_nonunicode(__stream, __fmt, __args, __write_nl);
 }
 
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
 _LIBCPP_HIDE_FROM_ABI inline void
 __vprint_unicode_windows(FILE* __stream, string_view __fmt, format_args __args, bool __write_nl, bool __is_terminal) {
@@ -272,16 +274,16 @@ __vprint_unicode_windows(FILE* __stream, string_view __fmt, format_args __args,
 
   // The macro _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION is used to change
   // the behavior in the test. This is not part of the public API.
-#      ifdef _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION
+#        ifdef _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION
   _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION(__stream, __view);
-#      elif defined(_LIBCPP_WIN32API)
+#        elif defined(_LIBCPP_WIN32API)
   std::__write_to_windows_console(__stream, __view);
-#      else
+#        else
   std::__throw_runtime_error("No defintion of _LIBCPP_TESTING_PRINT_WRITE_TO_WINDOWS_CONSOLE_FUNCTION and "
                              "__write_to_windows_console is not available.");
-#      endif
+#        endif
 }
-#    endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#      endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
 _LIBCPP_HIDE_FROM_ABI inline void
@@ -312,29 +314,29 @@ __vprint_unicode([[maybe_unused]] FILE* __stream,
   // so there the call can be forwarded to the non_unicode API. On
   // Windows there is a different API. This API requires transcoding.
 
-#    ifndef _LIBCPP_WIN32API
+#      ifndef _LIBCPP_WIN32API
   __print::__vprint_unicode_posix(__stream, __fmt, __args, __write_nl, __print::__is_terminal(__stream));
-#    elif _LIBCPP_HAS_WIDE_CHARACTERS
+#      elif _LIBCPP_HAS_WIDE_CHARACTERS
   __print::__vprint_unicode_windows(__stream, __fmt, __args, __write_nl, __print::__is_terminal(__stream));
-#    else
-#      error "Windows builds with wchar_t disabled are not supported."
-#    endif
+#      else
+#        error "Windows builds with wchar_t disabled are not supported."
+#      endif
 }
 
-#  endif // _LIBCPP_HAS_UNICODE
+#    endif // _LIBCPP_HAS_UNICODE
 
 } // namespace __print
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void print(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
-#  if _LIBCPP_HAS_UNICODE
+#    if _LIBCPP_HAS_UNICODE
   if constexpr (__print::__use_unicode_execution_charset)
     __print::__vprint_unicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
   else
     __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
-#  else  // _LIBCPP_HAS_UNICODE
+#    else  // _LIBCPP_HAS_UNICODE
   __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
-#  endif // _LIBCPP_HAS_UNICODE
+#    endif // _LIBCPP_HAS_UNICODE
 }
 
 template <class... _Args>
@@ -344,7 +346,7 @@ _LIBCPP_HIDE_FROM_ABI void print(format_string<_Args...> __fmt, _Args&&... __arg
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
-#  if _LIBCPP_HAS_UNICODE
+#    if _LIBCPP_HAS_UNICODE
   // Note the wording in the Standard is inefficient. The output of
   // std::format is a std::string which is then copied. This solution
   // just appends a newline at the end of the output.
@@ -352,9 +354,9 @@ _LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, format_string<_Args...> __fmt
     __print::__vprint_unicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
   else
     __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
-#  else  // _LIBCPP_HAS_UNICODE
+#    else  // _LIBCPP_HAS_UNICODE
   __print::__vprint_nonunicode(__stream, __fmt.get(), std::make_format_args(__args...), true);
-#  endif // _LIBCPP_HAS_UNICODE
+#    endif // _LIBCPP_HAS_UNICODE
 }
 
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
@@ -372,7 +374,7 @@ _LIBCPP_HIDE_FROM_ABI void println(format_string<_Args...> __fmt, _Args&&... __a
   std::println(stdout, __fmt, std::forward<_Args>(__args)...);
 }
 
-#  if _LIBCPP_HAS_UNICODE
+#    if _LIBCPP_HAS_UNICODE
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
 _LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(FILE* __stream, string_view __fmt, format_args __args) {
   __print::__vprint_unicode(__stream, __fmt, __args, false);
@@ -383,7 +385,7 @@ _LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(string_view __fmt, format_args
   std::vprint_unicode(stdout, __fmt, __args);
 }
 
-#  endif // _LIBCPP_HAS_UNICODE
+#    endif // _LIBCPP_HAS_UNICODE
 
 template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
 _LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args) {
@@ -395,8 +397,10 @@ _LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(string_view __fmt, format_ar
   std::vprint_nonunicode(stdout, __fmt, __args);
 }
 
-#endif // _LIBCPP_STD_VER >= 23
+#  endif // _LIBCPP_STD_VER >= 23
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_PRINT
diff --git a/libcxx/include/queue b/libcxx/include/queue
index db9ad26eaeddf..bebea92fc7425 100644
--- a/libcxx/include/queue
+++ b/libcxx/include/queue
@@ -254,38 +254,40 @@ template <class T, class Container, class Compare>
 
 */
 
-#include <__algorithm/make_heap.h>
-#include <__algorithm/pop_heap.h>
-#include <__algorithm/push_heap.h>
-#include <__algorithm/ranges_copy.h>
-#include <__config>
-#include <__functional/operations.h>
-#include <__fwd/deque.h>
-#include <__fwd/queue.h>
-#include <__iterator/back_insert_iterator.h>
-#include <__iterator/iterator_traits.h>
-#include <__memory/uses_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__utility/forward.h>
-#include <deque>
-#include <vector>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/make_heap.h>
+#  include <__algorithm/pop_heap.h>
+#  include <__algorithm/push_heap.h>
+#  include <__algorithm/ranges_copy.h>
+#  include <__config>
+#  include <__functional/operations.h>
+#  include <__fwd/deque.h>
+#  include <__fwd/queue.h>
+#  include <__iterator/back_insert_iterator.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__memory/uses_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__utility/forward.h>
+#  include <deque>
+#  include <vector>
+#  include <version>
 
 // standard-mandated includes
 
 // [queue.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -313,7 +315,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI queue(const queue& __q) : c(__q.c) {}
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI queue(_InputIterator __first, _InputIterator __last) : c(__first, __last) {}
 
@@ -333,14 +335,14 @@ public:
   _LIBCPP_HIDE_FROM_ABI queue(from_range_t, _Range&& __range, const _Alloc& __alloc)
       : c(from_range, std::forward<_Range>(__range), __alloc) {}
 
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI queue& operator=(const queue& __q) {
     c = __q.c;
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI queue(queue&& __q) noexcept(is_nothrow_move_constructible<container_type>::value)
       : c(std::move(__q.c)) {}
 
@@ -348,12 +350,12 @@ public:
     c = std::move(__q.c);
     return *this;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit queue(const container_type& __c) : c(__c) {}
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI explicit queue(container_type&& __c) : c(std::move(__c)) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit queue(const _Alloc& __a) : c(__a) {}
@@ -364,13 +366,13 @@ public:
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI queue(const container_type& __c, const _Alloc& __a) : c(__c, __a) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI queue(container_type&& __c, const _Alloc& __a) : c(std::move(__c), __a) {}
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI queue(queue&& __q, const _Alloc& __a) : c(std::move(__q.c), __a) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
   _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
@@ -381,10 +383,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_reference back() const { return c.back(); }
 
   _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v) { c.push_back(std::move(__v)); }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
     if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
@@ -393,22 +395,22 @@ public:
       ranges::copy(std::forward<_Range>(__range), std::back_inserter(c));
     }
   }
-#  endif
+#    endif
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   decltype(auto)
   emplace(_Args&&... __args) {
     return c.emplace_back(std::forward<_Args>(__args)...);
   }
-#  else
+#    else
   void
   emplace(_Args&&... __args) {
     c.emplace_back(std::forward<_Args>(__args)...);
   }
-#  endif
-#endif // _LIBCPP_CXX03_LANG
+#    endif
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void pop() { c.pop_front(); }
 
   _LIBCPP_HIDE_FROM_ABI void swap(queue& __q) _NOEXCEPT_(__is_nothrow_swappable_v<container_type>) {
@@ -427,7 +429,7 @@ public:
   operator<(const queue<_T1, _OtherContainer>& __x, const queue<_T1, _OtherContainer>& __y);
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
 queue(_Container) -> queue<typename _Container::value_type, _Container>;
 
@@ -436,9 +438,9 @@ template <class _Container,
           class = enable_if_t<!__is_allocator<_Container>::value>,
           class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
 queue(_Container, _Alloc) -> queue<typename _Container::value_type, _Container>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
 queue(_InputIterator, _InputIterator) -> queue<__iter_value_type<_InputIterator>>;
 
@@ -457,7 +459,7 @@ template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator
 queue(from_range_t,
       _Range&&,
       _Alloc) -> queue<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
-#endif
+#  endif
 
 template <class _Tp, class _Container>
 inline _LIBCPP_HIDE_FROM_ABI bool operator==(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y) {
@@ -489,7 +491,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const queue<_Tp, _Container>& __x,
   return !(__y < __x);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <class _Tp, three_way_comparable _Container>
 _LIBCPP_HIDE_FROM_ABI compare_three_way_result_t<_Container>
@@ -498,7 +500,7 @@ operator<=>(const queue<_Tp, _Container>& __x, const queue<_Tp, _Container>& __y
   return __x.__get_container() <=> __y.__get_container();
 }
 
-#endif
+#  endif
 
 template <class _Tp, class _Container, __enable_if_t<__is_swappable_v<_Container>, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI void swap(queue<_Tp, _Container>& __x, queue<_Tp, _Container>& __y)
@@ -538,7 +540,7 @@ public:
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q) noexcept(
       is_nothrow_move_constructible<container_type>::value && is_nothrow_move_constructible<value_compare>::value)
       : c(std::move(__q.c)), comp(std::move(__q.comp)) {}
@@ -549,13 +551,13 @@ public:
     comp = std::move(__q.comp);
     return *this;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const value_compare& __comp) : c(), comp(__comp) {}
   _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const container_type& __c);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c);
-#endif
+#  endif
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp = value_compare());
 
@@ -563,19 +565,19 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const value_compare& __comp = value_compare())
       : c(from_range, std::forward<_Range>(__range)), comp(__comp) {
     std::make_heap(c.begin(), c.end(), comp);
   }
-#endif
+#  endif
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const _Alloc& __a);
@@ -589,13 +591,13 @@ public:
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q, const _Alloc& __a);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c, const _Alloc& __a);
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q, const _Alloc& __a);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <
       class _InputIter,
@@ -619,7 +621,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI priority_queue(
       _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c, const _Alloc& __a);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <
       class _InputIter,
       class _Alloc,
@@ -627,9 +629,9 @@ public:
                     int> = 0>
   _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c, const _Alloc& __a);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 
   template <_ContainerCompatibleRange<_Tp> _Range,
             class _Alloc,
@@ -647,17 +649,17 @@ public:
     std::make_heap(c.begin(), c.end(), comp);
   }
 
-#endif
+#  endif
 
   [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
   _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
   _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.front(); }
 
   _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v);
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
     if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
@@ -668,11 +670,11 @@ public:
 
     std::make_heap(c.begin(), c.end(), comp);
   }
-#  endif
+#    endif
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI void emplace(_Args&&... __args);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void pop();
 
   _LIBCPP_HIDE_FROM_ABI void swap(priority_queue& __q)
@@ -681,7 +683,7 @@ public:
   [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const { return c; }
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Compare,
           class _Container,
           class = enable_if_t<!__is_allocator<_Compare>::value>,
@@ -735,9 +737,9 @@ template <class _InputIterator,
           class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
 priority_queue(_InputIterator, _InputIterator, _Compare, _Container, _Alloc)
     -> priority_queue<typename _Container::value_type, _Container, _Compare>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 
 template <ranges::input_range _Range,
           class _Compare = less<ranges::range_value_t<_Range>>,
@@ -757,7 +759,7 @@ template <ranges::input_range _Range, class _Alloc, class = enable_if_t<__is_all
 priority_queue(from_range_t, _Range&&, _Alloc)
     -> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>, _Alloc>>;
 
-#endif
+#  endif
 
 template <class _Tp, class _Container, class _Compare>
 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Compare& __comp, const container_type& __c)
@@ -765,7 +767,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Compare&
   std::make_heap(c.begin(), c.end(), comp);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_compare& __comp, container_type&& __c)
@@ -773,7 +775,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_com
   std::make_heap(c.begin(), c.end(), comp);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
@@ -792,7 +794,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
   std::make_heap(c.begin(), c.end(), comp);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
@@ -803,7 +805,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
   std::make_heap(c.begin(), c.end(), comp);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
@@ -827,7 +829,7 @@ template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value,
 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const priority_queue& __q, const _Alloc& __a)
     : c(__q.c, __a), comp(__q.comp) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
@@ -842,7 +844,7 @@ template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value,
 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(priority_queue&& __q, const _Alloc& __a)
     : c(std::move(__q.c), __a), comp(std::move(__q.comp)) {}
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <
@@ -877,7 +879,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
   std::make_heap(c.begin(), c.end(), comp);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Container, class _Compare>
 template <
     class _InputIter,
@@ -889,7 +891,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
   c.insert(c.end(), __f, __l);
   std::make_heap(c.begin(), c.end(), comp);
 }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __v) {
@@ -897,7 +899,7 @@ inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __
   std::push_heap(c.begin(), c.end(), comp);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 inline void priority_queue<_Tp, _Container, _Compare>::push(value_type&& __v) {
@@ -912,7 +914,7 @@ inline void priority_queue<_Tp, _Container, _Compare>::emplace(_Args&&... __args
   std::push_heap(c.begin(), c.end(), comp);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 inline void priority_queue<_Tp, _Container, _Compare>::pop() {
@@ -946,11 +948,12 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstdlib>
-#  include <functional>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstdlib>
+#    include <functional>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_QUEUE
diff --git a/libcxx/include/random b/libcxx/include/random
index 6cc3760c20e16..08e9c384538f5 100644
--- a/libcxx/include/random
+++ b/libcxx/include/random
@@ -1677,66 +1677,69 @@ class piecewise_linear_distribution
 } // std
 */
 
-#include <__config>
-#include <__random/bernoulli_distribution.h>
-#include <__random/binomial_distribution.h>
-#include <__random/cauchy_distribution.h>
-#include <__random/chi_squared_distribution.h>
-#include <__random/default_random_engine.h>
-#include <__random/discard_block_engine.h>
-#include <__random/discrete_distribution.h>
-#include <__random/exponential_distribution.h>
-#include <__random/extreme_value_distribution.h>
-#include <__random/fisher_f_distribution.h>
-#include <__random/gamma_distribution.h>
-#include <__random/generate_canonical.h>
-#include <__random/geometric_distribution.h>
-#include <__random/independent_bits_engine.h>
-#include <__random/is_seed_sequence.h>
-#include <__random/knuth_b.h>
-#include <__random/linear_congruential_engine.h>
-#include <__random/lognormal_distribution.h>
-#include <__random/mersenne_twister_engine.h>
-#include <__random/negative_binomial_distribution.h>
-#include <__random/normal_distribution.h>
-#include <__random/piecewise_constant_distribution.h>
-#include <__random/piecewise_linear_distribution.h>
-#include <__random/poisson_distribution.h>
-#include <__random/random_device.h>
-#include <__random/ranlux.h>
-#include <__random/seed_seq.h>
-#include <__random/shuffle_order_engine.h>
-#include <__random/student_t_distribution.h>
-#include <__random/subtract_with_carry_engine.h>
-#include <__random/uniform_int_distribution.h>
-#include <__random/uniform_random_bit_generator.h>
-#include <__random/uniform_real_distribution.h>
-#include <__random/weibull_distribution.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__random/bernoulli_distribution.h>
+#  include <__random/binomial_distribution.h>
+#  include <__random/cauchy_distribution.h>
+#  include <__random/chi_squared_distribution.h>
+#  include <__random/default_random_engine.h>
+#  include <__random/discard_block_engine.h>
+#  include <__random/discrete_distribution.h>
+#  include <__random/exponential_distribution.h>
+#  include <__random/extreme_value_distribution.h>
+#  include <__random/fisher_f_distribution.h>
+#  include <__random/gamma_distribution.h>
+#  include <__random/generate_canonical.h>
+#  include <__random/geometric_distribution.h>
+#  include <__random/independent_bits_engine.h>
+#  include <__random/is_seed_sequence.h>
+#  include <__random/knuth_b.h>
+#  include <__random/linear_congruential_engine.h>
+#  include <__random/lognormal_distribution.h>
+#  include <__random/mersenne_twister_engine.h>
+#  include <__random/negative_binomial_distribution.h>
+#  include <__random/normal_distribution.h>
+#  include <__random/piecewise_constant_distribution.h>
+#  include <__random/piecewise_linear_distribution.h>
+#  include <__random/poisson_distribution.h>
+#  include <__random/random_device.h>
+#  include <__random/ranlux.h>
+#  include <__random/seed_seq.h>
+#  include <__random/shuffle_order_engine.h>
+#  include <__random/student_t_distribution.h>
+#  include <__random/subtract_with_carry_engine.h>
+#  include <__random/uniform_int_distribution.h>
+#  include <__random/uniform_random_bit_generator.h>
+#  include <__random/uniform_real_distribution.h>
+#  include <__random/weibull_distribution.h>
+#  include <version>
 
 // standard-mandated includes
 
 // [rand.synopsis]
-#include <initializer_list>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <climits>
-#  include <cmath>
-#  include <concepts>
-#  include <cstddef>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <limits>
-#  include <numeric>
-#  include <string>
-#  include <type_traits>
-#  include <vector>
-#endif
+#  include <initializer_list>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <climits>
+#    include <cmath>
+#    include <concepts>
+#    include <cstddef>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <limits>
+#    include <numeric>
+#    include <string>
+#    include <type_traits>
+#    include <vector>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_RANDOM
diff --git a/libcxx/include/ranges b/libcxx/include/ranges
index d8ee6f75e8b23..2c6057e30420a 100644
--- a/libcxx/include/ranges
+++ b/libcxx/include/ranges
@@ -380,76 +380,79 @@ namespace std {
 }
 */
 
-#include <__config>
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__ranges/access.h>
-#  include <__ranges/all.h>
-#  include <__ranges/common_view.h>
-#  include <__ranges/concepts.h>
-#  include <__ranges/counted.h>
-#  include <__ranges/dangling.h>
-#  include <__ranges/data.h>
-#  include <__ranges/drop_view.h>
-#  include <__ranges/drop_while_view.h>
-#  include <__ranges/elements_view.h>
-#  include <__ranges/empty.h>
-#  include <__ranges/empty_view.h>
-#  include <__ranges/enable_borrowed_range.h>
-#  include <__ranges/enable_view.h>
-#  include <__ranges/filter_view.h>
-#  include <__ranges/iota_view.h>
-#  include <__ranges/join_view.h>
-#  include <__ranges/lazy_split_view.h>
-#  include <__ranges/rbegin.h>
-#  include <__ranges/ref_view.h>
-#  include <__ranges/rend.h>
-#  include <__ranges/reverse_view.h>
-#  include <__ranges/single_view.h>
-#  include <__ranges/size.h>
-#  include <__ranges/split_view.h>
-#  include <__ranges/subrange.h>
-#  include <__ranges/take_view.h>
-#  include <__ranges/take_while_view.h>
-#  include <__ranges/transform_view.h>
-#  include <__ranges/view_interface.h>
-#  include <__ranges/views.h>
-
-#  if _LIBCPP_HAS_LOCALIZATION
-#    include <__ranges/istream_view.h>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__ranges/access.h>
+#    include <__ranges/all.h>
+#    include <__ranges/common_view.h>
+#    include <__ranges/concepts.h>
+#    include <__ranges/counted.h>
+#    include <__ranges/dangling.h>
+#    include <__ranges/data.h>
+#    include <__ranges/drop_view.h>
+#    include <__ranges/drop_while_view.h>
+#    include <__ranges/elements_view.h>
+#    include <__ranges/empty.h>
+#    include <__ranges/empty_view.h>
+#    include <__ranges/enable_borrowed_range.h>
+#    include <__ranges/enable_view.h>
+#    include <__ranges/filter_view.h>
+#    include <__ranges/iota_view.h>
+#    include <__ranges/join_view.h>
+#    include <__ranges/lazy_split_view.h>
+#    include <__ranges/rbegin.h>
+#    include <__ranges/ref_view.h>
+#    include <__ranges/rend.h>
+#    include <__ranges/reverse_view.h>
+#    include <__ranges/single_view.h>
+#    include <__ranges/size.h>
+#    include <__ranges/split_view.h>
+#    include <__ranges/subrange.h>
+#    include <__ranges/take_view.h>
+#    include <__ranges/take_while_view.h>
+#    include <__ranges/transform_view.h>
+#    include <__ranges/view_interface.h>
+#    include <__ranges/views.h>
+
+#    if _LIBCPP_HAS_LOCALIZATION
+#      include <__ranges/istream_view.h>
+#    endif
 #  endif
-#endif
 
-#if _LIBCPP_STD_VER >= 23
-#  include <__ranges/as_rvalue_view.h>
-#  include <__ranges/chunk_by_view.h>
-#  include <__ranges/from_range.h>
-#  include <__ranges/repeat_view.h>
-#  include <__ranges/to.h>
-#  include <__ranges/zip_view.h>
-#endif
+#  if _LIBCPP_STD_VER >= 23
+#    include <__ranges/as_rvalue_view.h>
+#    include <__ranges/chunk_by_view.h>
+#    include <__ranges/from_range.h>
+#    include <__ranges/repeat_view.h>
+#    include <__ranges/to.h>
+#    include <__ranges/zip_view.h>
+#  endif
 
-#include <version>
+#  include <version>
 
 // standard-mandated includes
 
 // [ranges.syn]
-#include <compare>
-#include <initializer_list>
-#include <iterator>
+#  include <compare>
+#  include <initializer_list>
+#  include <iterator>
 
 // [tuple.helper]
-#include <__tuple/tuple_element.h>
-#include <__tuple/tuple_size.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <type_traits>
-#endif
+#  include <__tuple/tuple_element.h>
+#  include <__tuple/tuple_size.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_RANGES
diff --git a/libcxx/include/ratio b/libcxx/include/ratio
index bd262437db34f..9e56eacae0f2d 100644
--- a/libcxx/include/ratio
+++ b/libcxx/include/ratio
@@ -81,18 +81,20 @@ using quetta = ratio <1'000'000'000'000'000'000'000'000'000'000, 1>; // Since C+
 }
 */
 
-#include <__config>
-#include <__type_traits/integral_constant.h>
-#include <climits>
-#include <cstdint>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__type_traits/integral_constant.h>
+#  include <climits>
+#  include <cstdint>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -279,17 +281,17 @@ public:
                           __ll_mul<_R2::den / __gcd_n1_d2, _R1::den / __gcd_d1_n2>::value >::type type;
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 using ratio_multiply = typename __ratio_multiply<_R1, _R2>::type;
 
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct _LIBCPP_TEMPLATE_VIS ratio_multiply : public __ratio_multiply<_R1, _R2>::type {};
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct __ratio_divide {
@@ -305,17 +307,17 @@ public:
                           __ll_mul<_R2::num / __gcd_n1_n2, _R1::den / __gcd_d1_d2>::value >::type type;
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 using ratio_divide = typename __ratio_divide<_R1, _R2>::type;
 
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct _LIBCPP_TEMPLATE_VIS ratio_divide : public __ratio_divide<_R1, _R2>::type {};
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct __ratio_add {
@@ -334,17 +336,17 @@ public:
              _R2::den > >::type type;
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 using ratio_add = typename __ratio_add<_R1, _R2>::type;
 
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct _LIBCPP_TEMPLATE_VIS ratio_add : public __ratio_add<_R1, _R2>::type {};
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct __ratio_subtract {
@@ -363,17 +365,17 @@ public:
              _R2::den > >::type type;
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 using ratio_subtract = typename __ratio_subtract<_R1, _R2>::type;
 
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _R1, class _R2>
 struct _LIBCPP_TEMPLATE_VIS ratio_subtract : public __ratio_subtract<_R1, _R2>::type {};
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 // ratio_equal
 
@@ -464,7 +466,7 @@ struct _LIBCPP_TEMPLATE_VIS ratio_greater_equal : _BoolConstant<!ratio_less<_R1,
 template <class _R1, class _R2>
 using __ratio_gcd = ratio<__static_gcd<_R1::num, _R2::num>, __static_lcm<_R1::den, _R2::den> >;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _R1, class _R2>
 inline constexpr bool ratio_equal_v = ratio_equal<_R1, _R2>::value;
 
@@ -482,14 +484,15 @@ inline constexpr bool ratio_greater_v = ratio_greater<_R1, _R2>::value;
 
 template <class _R1, class _R2>
 inline constexpr bool ratio_greater_equal_v = ratio_greater_equal<_R1, _R2>::value;
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_RATIO
diff --git a/libcxx/include/regex b/libcxx/include/regex
index bd75e8de7db43..1983ca9d9ff80 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -789,48 +789,50 @@ typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
 } // std
 */
 
-#include <__algorithm/find.h>
-#include <__algorithm/search.h>
-#include <__assert>
-#include <__config>
-#include <__iterator/back_insert_iterator.h>
-#include <__iterator/default_sentinel.h>
-#include <__iterator/wrap_iter.h>
-#include <__locale>
-#include <__memory/shared_ptr.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__type_traits/is_swappable.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <__utility/swap.h>
-#include <__verbose_abort>
-#include <deque>
-#include <stdexcept>
-#include <string>
-#include <vector>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/find.h>
+#  include <__algorithm/search.h>
+#  include <__assert>
+#  include <__config>
+#  include <__iterator/back_insert_iterator.h>
+#  include <__iterator/default_sentinel.h>
+#  include <__iterator/wrap_iter.h>
+#  include <__locale>
+#  include <__memory/shared_ptr.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <__utility/swap.h>
+#  include <__verbose_abort>
+#  include <deque>
+#  include <stdexcept>
+#  include <string>
+#  include <vector>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [re.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
-#define _LIBCPP_REGEX_COMPLEXITY_FACTOR 4096
+#  define _LIBCPP_REGEX_COMPLEXITY_FACTOR 4096
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -843,11 +845,11 @@ enum syntax_option_type {
   nosubs   = 1 << 1,
   optimize = 1 << 2,
   collate  = 1 << 3,
-#ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+#  ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
   ECMAScript = 1 << 9,
-#else
+#  else
   ECMAScript = 0,
-#endif
+#  endif
   basic    = 1 << 4,
   extended = 1 << 5,
   awk      = 1 << 6,
@@ -858,11 +860,11 @@ enum syntax_option_type {
 };
 
 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR syntax_option_type __get_grammar(syntax_option_type __g) {
-#ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+#  ifdef _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
   return static_cast<syntax_option_type>(__g & 0x3F0);
-#else
+#  else
   return static_cast<syntax_option_type>(__g & 0x1F0);
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR syntax_option_type operator~(syntax_option_type __x) {
@@ -984,11 +986,11 @@ public:
 
 template <regex_constants::error_type _Ev>
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_regex_error() {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw regex_error(_Ev);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("regex_error was thrown in -fno-exceptions mode");
-#endif
+#  endif
 }
 
 template <class _CharT>
@@ -997,7 +999,7 @@ public:
   typedef _CharT char_type;
   typedef basic_string<char_type> string_type;
   typedef locale locale_type;
-#if defined(__BIONIC__) || defined(_NEWLIB_VERSION)
+#  if defined(__BIONIC__) || defined(_NEWLIB_VERSION)
   // Originally bionic's ctype_base used its own ctype masks because the
   // builtin ctype implementation wasn't in libc++ yet. Bionic's ctype mask
   // was only 8 bits wide and already saturated, so it used a wider type here
@@ -1012,9 +1014,9 @@ public:
   // often used for space constrained environments, so it makes sense not to
   // duplicate the ctype table.
   typedef uint16_t char_class_type;
-#else
+#  else
   typedef ctype_base::mask char_class_type;
-#endif
+#  endif
 
   static const char_class_type __regex_word = ctype_base::__regex_word;
 
@@ -1054,30 +1056,30 @@ private:
 
   template <class _ForwardIterator>
   string_type __transform_primary(_ForwardIterator __f, _ForwardIterator __l, char) const;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
   template <class _ForwardIterator>
   string_type __transform_primary(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const;
-#endif
+#  endif
   template <class _ForwardIterator>
   string_type __lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, char) const;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
   template <class _ForwardIterator>
   string_type __lookup_collatename(_ForwardIterator __f, _ForwardIterator __l, wchar_t) const;
-#endif
+#  endif
   template <class _ForwardIterator>
   char_class_type __lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, char) const;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
   template <class _ForwardIterator>
   char_class_type __lookup_classname(_ForwardIterator __f, _ForwardIterator __l, bool __icase, wchar_t) const;
-#endif
+#  endif
 
   static int __regex_traits_value(unsigned char __ch, int __radix);
   _LIBCPP_HIDE_FROM_ABI int __regex_traits_value(char __ch, int __radix) const {
     return __regex_traits_value(static_cast<unsigned char>(__ch), __radix);
   }
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
   _LIBCPP_HIDE_FROM_ABI int __regex_traits_value(wchar_t __ch, int __radix) const;
-#endif
+#  endif
 };
 
 template <class _CharT>
@@ -1136,7 +1138,7 @@ regex_traits<_CharT>::__transform_primary(_ForwardIterator __f, _ForwardIterator
   return __d;
 }
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class _CharT>
 template <class _ForwardIterator>
 typename regex_traits<_CharT>::string_type
@@ -1155,7 +1157,7 @@ regex_traits<_CharT>::__transform_primary(_ForwardIterator __f, _ForwardIterator
   }
   return __d;
 }
-#endif
+#  endif
 
 // lookup_collatename is very FreeBSD-specific
 
@@ -1180,7 +1182,7 @@ regex_traits<_CharT>::__lookup_collatename(_ForwardIterator __f, _ForwardIterato
   return __r;
 }
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class _CharT>
 template <class _ForwardIterator>
 typename regex_traits<_CharT>::string_type
@@ -1208,7 +1210,7 @@ regex_traits<_CharT>::__lookup_collatename(_ForwardIterator __f, _ForwardIterato
   }
   return __r;
 }
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 // lookup_classname
 
@@ -1223,7 +1225,7 @@ regex_traits<_CharT>::__lookup_classname(_ForwardIterator __f, _ForwardIterator
   return std::__get_classname(__s.c_str(), __icase);
 }
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class _CharT>
 template <class _ForwardIterator>
 typename regex_traits<_CharT>::char_class_type
@@ -1239,7 +1241,7 @@ regex_traits<_CharT>::__lookup_classname(_ForwardIterator __f, _ForwardIterator
   }
   return __get_classname(__n.c_str(), __icase);
 }
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 template <class _CharT>
 bool regex_traits<_CharT>::isctype(char_type __c, char_class_type __m) const {
@@ -1250,28 +1252,28 @@ bool regex_traits<_CharT>::isctype(char_type __c, char_class_type __m) const {
 
 inline _LIBCPP_HIDE_FROM_ABI bool __is_07(unsigned char __c) {
   return (__c & 0xF8u) ==
-#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+#  if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
          0xF0;
-#else
+#  else
          0x30;
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI bool __is_89(unsigned char __c) {
   return (__c & 0xFEu) ==
-#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+#  if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
          0xF8;
-#else
+#  else
          0x38;
-#endif
+#  endif
 }
 
 inline _LIBCPP_HIDE_FROM_ABI unsigned char __to_lower(unsigned char __c) {
-#if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
+#  if defined(__MVS__) && !defined(__NATIVE_ASCII_F)
   return __c & 0xBF;
-#else
+#  else
   return __c | 0x20;
-#endif
+#  endif
 }
 
 template <class _CharT>
@@ -1290,12 +1292,12 @@ int regex_traits<_CharT>::__regex_traits_value(unsigned char __ch, int __radix)
   return -1;
 }
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class _CharT>
 inline int regex_traits<_CharT>::__regex_traits_value(wchar_t __ch, int __radix) const {
   return __regex_traits_value(static_cast<unsigned char>(__ct_->narrow(__ch, char_type())), __radix);
 }
-#endif
+#  endif
 
 template <class _CharT>
 class __node;
@@ -1938,10 +1940,10 @@ public:
 
 template <>
 _LIBCPP_EXPORTED_FROM_ABI void __match_any_but_newline<char>::__exec(__state&) const;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 _LIBCPP_EXPORTED_FROM_ABI void __match_any_but_newline<wchar_t>::__exec(__state&) const;
-#endif
+#  endif
 
 // __match_char
 
@@ -2262,9 +2264,9 @@ template <class _CharT, class _Traits = regex_traits<_CharT> >
 class _LIBCPP_TEMPLATE_VIS basic_regex;
 
 typedef basic_regex<char> regex;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef basic_regex<wchar_t> wregex;
-#endif
+#  endif
 
 template <class _CharT, class _Traits>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(regex)
@@ -2335,21 +2337,21 @@ public:
       : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
     __init(__first, __last);
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI basic_regex(initializer_list<value_type> __il, flag_type __f = regex_constants::ECMAScript)
       : __flags_(__f), __marked_count_(0), __loop_count_(0), __open_count_(0), __end_(nullptr) {
     __init(__il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   //    ~basic_regex() = default;
 
   //     basic_regex& operator=(const basic_regex&) = default;
   //     basic_regex& operator=(basic_regex&&) = default;
   _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(const value_type* __p) { return assign(__p); }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(initializer_list<value_type> __il) { return assign(__il); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   template <class _ST, class _SA>
   _LIBCPP_HIDE_FROM_ABI basic_regex& operator=(const basic_string<value_type, _ST, _SA>& __p) {
     return assign(__p);
@@ -2357,9 +2359,9 @@ public:
 
   // assign:
   _LIBCPP_HIDE_FROM_ABI basic_regex& assign(const basic_regex& __that) { return *this = __that; }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI basic_regex& assign(basic_regex&& __that) _NOEXCEPT { return *this = std::move(__that); }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI basic_regex& assign(const value_type* __p, flag_type __f = regex_constants::ECMAScript) {
     return assign(__p, __p + __traits_.length(__p), __f);
   }
@@ -2396,14 +2398,14 @@ public:
     return assign(basic_regex(__first, __last, __f));
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI basic_regex&
   assign(initializer_list<value_type> __il, flag_type __f = regex_constants::ECMAScript) {
     return assign(__il.begin(), __il.end(), __f);
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   // const operations:
   _LIBCPP_HIDE_FROM_ABI unsigned mark_count() const { return __marked_count_; }
@@ -2644,11 +2646,11 @@ private:
   friend class __lookahead;
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
 basic_regex(_ForwardIterator, _ForwardIterator, regex_constants::syntax_option_type = regex_constants::ECMAScript)
     -> basic_regex<typename iterator_traits<_ForwardIterator>::value_type>;
-#endif
+#  endif
 
 template <class _CharT, class _Traits>
 const regex_constants::syntax_option_type basic_regex<_CharT, _Traits>::icase;
@@ -4181,10 +4183,10 @@ void basic_regex<_CharT, _Traits>::__push_lookahead(const basic_regex& __exp, bo
 
 typedef sub_match<const char*> csub_match;
 typedef sub_match<string::const_iterator> ssub_match;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef sub_match<const wchar_t*> wcsub_match;
 typedef sub_match<wstring::const_iterator> wssub_match;
-#endif
+#  endif
 
 template <class _BidirectionalIterator>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(csub_match)
@@ -4224,7 +4226,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator==(const sub_match<_BiIter>& __x, cons
   return __x.compare(__y) == 0;
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 using __sub_match_cat = compare_three_way_result_t<basic_string<typename iterator_traits<_BiIter>::value_type>>;
 
@@ -4232,7 +4234,7 @@ template <class _BiIter>
 _LIBCPP_HIDE_FROM_ABI auto operator<=>(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
   return static_cast<__sub_match_cat<_BiIter>>(__x.compare(__y) <=> 0);
 }
-#else  // _LIBCPP_STD_VER >= 20
+#  else  // _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const sub_match<_BiIter>& __x, const sub_match<_BiIter>& __y) {
   return !(__x == __y);
@@ -4299,7 +4301,7 @@ operator<=(const basic_string<typename iterator_traits<_BiIter>::value_type, _ST
            const sub_match<_BiIter>& __y) {
   return !(__y < __x);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _BiIter, class _ST, class _SA>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -4308,7 +4310,7 @@ operator==(const sub_match<_BiIter>& __x,
   return __x.compare(typename sub_match<_BiIter>::string_type(__y.data(), __y.size())) == 0;
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _BiIter, class _ST, class _SA>
 _LIBCPP_HIDE_FROM_ABI auto
 operator<=>(const sub_match<_BiIter>& __x,
@@ -4316,7 +4318,7 @@ operator<=>(const sub_match<_BiIter>& __x,
   return static_cast<__sub_match_cat<_BiIter>>(
       __x.compare(typename sub_match<_BiIter>::string_type(__y.data(), __y.size())) <=> 0);
 }
-#else  // _LIBCPP_STD_VER >= 20
+#  else  // _LIBCPP_STD_VER >= 20
 template <class _BiIter, class _ST, class _SA>
 inline _LIBCPP_HIDE_FROM_ABI bool
 operator!=(const sub_match<_BiIter>& __x,
@@ -4387,7 +4389,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool
 operator<=(typename iterator_traits<_BiIter>::value_type const* __x, const sub_match<_BiIter>& __y) {
   return !(__y < __x);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -4395,13 +4397,13 @@ operator==(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::val
   return __x.compare(__y) == 0;
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 _LIBCPP_HIDE_FROM_ABI auto
 operator<=>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
   return static_cast<__sub_match_cat<_BiIter>>(__x.compare(__y) <=> 0);
 }
-#else  // _LIBCPP_STD_VER >= 20
+#  else  // _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI bool
 operator!=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const* __y) {
@@ -4469,7 +4471,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool
 operator<=(typename iterator_traits<_BiIter>::value_type const& __x, const sub_match<_BiIter>& __y) {
   return !(__y < __x);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -4478,14 +4480,14 @@ operator==(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::val
   return __x.compare(string_type(1, __y)) == 0;
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 _LIBCPP_HIDE_FROM_ABI auto
 operator<=>(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
   using string_type = basic_string<typename iterator_traits<_BiIter>::value_type>;
   return static_cast<__sub_match_cat<_BiIter>>(__x.compare(string_type(1, __y)) <=> 0);
 }
-#else  // _LIBCPP_STD_VER >= 20
+#  else  // _LIBCPP_STD_VER >= 20
 template <class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI bool
 operator!=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
@@ -4516,7 +4518,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool
 operator<=(const sub_match<_BiIter>& __x, typename iterator_traits<_BiIter>::value_type const& __y) {
   return !(__y < __x);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _ST, class _BiIter>
 inline _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _ST>&
@@ -4526,10 +4528,10 @@ operator<<(basic_ostream<_CharT, _ST>& __os, const sub_match<_BiIter>& __m) {
 
 typedef match_results<const char*> cmatch;
 typedef match_results<string::const_iterator> smatch;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef match_results<const wchar_t*> wcmatch;
 typedef match_results<wstring::const_iterator> wsmatch;
-#endif
+#  endif
 
 template <class _BidirectionalIterator, class _Allocator>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cmatch) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wcmatch))
@@ -4559,12 +4561,12 @@ public:
   typedef basic_string<char_type> string_type;
 
   // construct/copy/destroy:
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   match_results() : match_results(allocator_type()) {}
   explicit match_results(const allocator_type& __a);
-#else
+#  else
   explicit match_results(const allocator_type& __a = allocator_type());
-#endif
+#  endif
 
   //    match_results(const match_results&) = default;
   //    match_results& operator=(const match_results&) = default;
@@ -4814,13 +4816,13 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const match_results<_BidirectionalIterator
   return __x.__matches_ == __y.__matches_ && __x.__prefix_ == __y.__prefix_ && __x.__suffix_ == __y.__suffix_;
 }
 
-#if _LIBCPP_STD_VER < 20
+#  if _LIBCPP_STD_VER < 20
 template <class _BidirectionalIterator, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const match_results<_BidirectionalIterator, _Allocator>& __x,
                                              const match_results<_BidirectionalIterator, _Allocator>& __y) {
   return !(__x == __y);
 }
-#endif
+#  endif
 
 template <class _BidirectionalIterator, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void
@@ -5232,13 +5234,13 @@ regex_search(const basic_string<_CharT, _ST, _SA>& __s,
   return __r;
 }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 template <class _ST, class _SA, class _Ap, class _Cp, class _Tp>
 bool regex_search(const basic_string<_Cp, _ST, _SA>&& __s,
                   match_results<typename basic_string<_Cp, _ST, _SA>::const_iterator, _Ap>&,
                   const basic_regex<_Cp, _Tp>& __e,
                   regex_constants::match_flag_type __flags = regex_constants::match_default) = delete;
-#endif
+#  endif
 
 // regex_match
 
@@ -5287,14 +5289,14 @@ regex_match(const basic_string<_CharT, _ST, _SA>& __s,
   return std::regex_match(__s.begin(), __s.end(), __m, __e, __flags);
 }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 template <class _ST, class _SA, class _Allocator, class _CharT, class _Traits>
 inline _LIBCPP_HIDE_FROM_ABI bool
 regex_match(const basic_string<_CharT, _ST, _SA>&& __s,
             match_results<typename basic_string<_CharT, _ST, _SA>::const_iterator, _Allocator>& __m,
             const basic_regex<_CharT, _Traits>& __e,
             regex_constants::match_flag_type __flags = regex_constants::match_default) = delete;
-#endif
+#  endif
 
 template <class _CharT, class _Traits>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -5321,10 +5323,10 @@ class _LIBCPP_TEMPLATE_VIS regex_iterator;
 
 typedef regex_iterator<const char*> cregex_iterator;
 typedef regex_iterator<string::const_iterator> sregex_iterator;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef regex_iterator<const wchar_t*> wcregex_iterator;
 typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
-#endif
+#  endif
 
 template <class _BidirectionalIterator, class _CharT, class _Traits>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cregex_iterator)
@@ -5337,9 +5339,9 @@ public:
   typedef const value_type* pointer;
   typedef const value_type& reference;
   typedef forward_iterator_tag iterator_category;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   typedef input_iterator_tag iterator_concept;
-#endif
+#  endif
 
 private:
   _BidirectionalIterator __begin_;
@@ -5354,20 +5356,20 @@ public:
                  _BidirectionalIterator __b,
                  const regex_type& __re,
                  regex_constants::match_flag_type __m = regex_constants::match_default);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   regex_iterator(_BidirectionalIterator __a,
                  _BidirectionalIterator __b,
                  const regex_type&& __re,
                  regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI bool operator==(const regex_iterator& __x) const;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const { return *this == regex_iterator(); }
-#endif
-#if _LIBCPP_STD_VER < 20
+#  endif
+#  if _LIBCPP_STD_VER < 20
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_iterator& __x) const { return !(*this == __x); }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __match_; }
   _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return std::addressof(__match_); }
@@ -5451,10 +5453,10 @@ class _LIBCPP_TEMPLATE_VIS regex_token_iterator;
 
 typedef regex_token_iterator<const char*> cregex_token_iterator;
 typedef regex_token_iterator<string::const_iterator> sregex_token_iterator;
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 typedef regex_token_iterator<const wchar_t*> wcregex_token_iterator;
 typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
-#endif
+#  endif
 
 template <class _BidirectionalIterator, class _CharT, class _Traits>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_PREFERRED_NAME(cregex_token_iterator)
@@ -5468,9 +5470,9 @@ public:
   typedef const value_type* pointer;
   typedef const value_type& reference;
   typedef forward_iterator_tag iterator_category;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   typedef input_iterator_tag iterator_concept;
-#endif
+#  endif
 
 private:
   typedef regex_iterator<_BidirectionalIterator, _CharT, _Traits> _Position;
@@ -5488,69 +5490,69 @@ public:
                        const regex_type& __re,
                        int __submatch                       = 0,
                        regex_constants::match_flag_type __m = regex_constants::match_default);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type&& __re,
                        int __submatch                       = 0,
                        regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
-#endif
+#  endif
 
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type& __re,
                        const vector<int>& __submatches,
                        regex_constants::match_flag_type __m = regex_constants::match_default);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type&& __re,
                        const vector<int>& __submatches,
                        regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type& __re,
                        initializer_list<int> __submatches,
                        regex_constants::match_flag_type __m = regex_constants::match_default);
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type&& __re,
                        initializer_list<int> __submatches,
                        regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
-#  endif
-#endif // _LIBCPP_CXX03_LANG
+#    endif
+#  endif // _LIBCPP_CXX03_LANG
   template <size_t _Np>
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type& __re,
                        const int (&__submatches)[_Np],
                        regex_constants::match_flag_type __m = regex_constants::match_default);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <size_t _Np>
   regex_token_iterator(_BidirectionalIterator __a,
                        _BidirectionalIterator __b,
                        const regex_type&& __re,
                        const int (&__submatches)[_Np],
                        regex_constants::match_flag_type __m = regex_constants::match_default) = delete;
-#endif
+#  endif
 
   regex_token_iterator(const regex_token_iterator&);
   regex_token_iterator& operator=(const regex_token_iterator&);
 
   _LIBCPP_HIDE_FROM_ABI bool operator==(const regex_token_iterator& __x) const;
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI bool operator==(default_sentinel_t) const {
     return *this == regex_token_iterator();
   }
-#endif
-#if _LIBCPP_STD_VER < 20
+#  endif
+#  if _LIBCPP_STD_VER < 20
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_token_iterator& __x) const { return !(*this == __x); }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI const value_type& operator*() const { return *__result_; }
   _LIBCPP_HIDE_FROM_ABI const value_type* operator->() const { return __result_; }
@@ -5612,7 +5614,7 @@ regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_itera
   __init(__a, __b);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _BidirectionalIterator, class _CharT, class _Traits>
 regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_iterator(
@@ -5625,7 +5627,7 @@ regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::regex_token_itera
   __init(__a, __b);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _BidirectionalIterator, class _CharT, class _Traits>
 template <size_t _Np>
@@ -5800,7 +5802,7 @@ regex_replace(const _CharT* __s,
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _BidirT>
@@ -5810,27 +5812,28 @@ using match_results _LIBCPP_AVAILABILITY_PMR =
 using cmatch _LIBCPP_AVAILABILITY_PMR = match_results<const char*>;
 using smatch _LIBCPP_AVAILABILITY_PMR = match_results<std::pmr::string::const_iterator>;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 using wcmatch _LIBCPP_AVAILABILITY_PMR = match_results<const wchar_t*>;
 using wsmatch _LIBCPP_AVAILABILITY_PMR = match_results<std::pmr::wstring::const_iterator>;
-#  endif
+#    endif
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <iterator>
-#  include <mutex>
-#  include <new>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <iterator>
+#    include <mutex>
+#    include <new>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_REGEX
diff --git a/libcxx/include/scoped_allocator b/libcxx/include/scoped_allocator
index 13e43c2f15ac8..c207ac07bdc4d 100644
--- a/libcxx/include/scoped_allocator
+++ b/libcxx/include/scoped_allocator
@@ -109,32 +109,34 @@ template <class OuterA1, class OuterA2, class... InnerAllocs>
 
 */
 
-#include <__config>
-#include <__memory/allocator_traits.h>
-#include <__memory/uses_allocator_construction.h>
-#include <__type_traits/common_type.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/integral_constant.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/remove_reference.h>
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <__utility/piecewise_construct.h>
-#include <tuple>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/uses_allocator_construction.h>
+#  include <__type_traits/common_type.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/integral_constant.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <__utility/piecewise_construct.h>
+#  include <tuple>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_CXX03_LANG)
+#  if !defined(_LIBCPP_CXX03_LANG)
 
 // scoped_allocator_adaptor
 
@@ -404,7 +406,7 @@ public:
     return allocator_traits<outer_allocator_type>::max_size(outer_allocator());
   }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _Type, class... _Args>
   _LIBCPP_HIDE_FROM_ABI void construct(_Type* __ptr, _Args&&... __args) {
     using _OM = __outermost<outer_allocator_type>;
@@ -415,7 +417,7 @@ public:
         },
         std::uses_allocator_construction_args<_Type>(inner_allocator(), std::forward<_Args>(__args)...));
   }
-#  else
+#    else
   template <class _Tp, class... _Args>
   _LIBCPP_HIDE_FROM_ABI void construct(_Tp* __p, _Args&&... __args) {
     __construct(__uses_alloc_ctor<_Tp, inner_allocator_type&, _Args...>(), __p, std::forward<_Args>(__args)...);
@@ -462,7 +464,7 @@ public:
               std::forward_as_tuple(std::forward<_Up>(__x.first)),
               std::forward_as_tuple(std::forward<_Vp>(__x.second)));
   }
-#  endif
+#    endif
 
   template <class _Tp>
   _LIBCPP_HIDE_FROM_ABI void destroy(_Tp* __p) {
@@ -522,10 +524,10 @@ private:
   friend class __scoped_allocator_storage;
 };
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 template <class _OuterAlloc, class... _InnerAllocs>
 scoped_allocator_adaptor(_OuterAlloc, _InnerAllocs...) -> scoped_allocator_adaptor<_OuterAlloc, _InnerAllocs...>;
-#  endif
+#    endif
 
 template <class _OuterA1, class _OuterA2>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -540,7 +542,7 @@ operator==(const scoped_allocator_adaptor<_OuterA1, _InnerA0, _InnerAllocs...>&
   return __a.outer_allocator() == __b.outer_allocator() && __a.inner_allocator() == __b.inner_allocator();
 }
 
-#  if _LIBCPP_STD_VER <= 17
+#    if _LIBCPP_STD_VER <= 17
 
 template <class _OuterA1, class _OuterA2, class... _InnerAllocs>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const scoped_allocator_adaptor<_OuterA1, _InnerAllocs...>& __a,
@@ -548,26 +550,27 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const scoped_allocator_adaptor<_Out
   return !(__a == __b);
 }
 
-#  endif // _LIBCPP_STD_VER <= 17
+#    endif // _LIBCPP_STD_VER <= 17
 
-#endif // !defined(_LIBCPP_CXX03_LANG)
+#  endif // !defined(_LIBCPP_CXX03_LANG)
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <climits>
-#  include <concepts>
-#  include <cstring>
-#  include <ctime>
-#  include <iterator>
-#  include <memory>
-#  include <ratio>
-#  include <stdexcept>
-#  include <type_traits>
-#  include <variant>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <climits>
+#    include <concepts>
+#    include <cstring>
+#    include <ctime>
+#    include <iterator>
+#    include <memory>
+#    include <ratio>
+#    include <stdexcept>
+#    include <type_traits>
+#    include <variant>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SCOPED_ALLOCATOR
diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore
index c594df459c93f..82f9726a96f62 100644
--- a/libcxx/include/semaphore
+++ b/libcxx/include/semaphore
@@ -45,30 +45,32 @@ using binary_semaphore = counting_semaphore<1>; // since C++20
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_THREADS
-
-#  include <__assert>
-#  include <__atomic/atomic.h>
-#  include <__atomic/atomic_sync.h>
-#  include <__atomic/memory_order.h>
-#  include <__chrono/time_point.h>
-#  include <__cstddef/ptrdiff_t.h>
-#  include <__thread/poll_with_backoff.h>
-#  include <__thread/support.h>
-#  include <__thread/timed_backoff_policy.h>
-#  include <limits>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_THREADS
+
+#    include <__assert>
+#    include <__atomic/atomic.h>
+#    include <__atomic/atomic_sync.h>
+#    include <__atomic/memory_order.h>
+#    include <__chrono/time_point.h>
+#    include <__cstddef/ptrdiff_t.h>
+#    include <__thread/poll_with_backoff.h>
+#    include <__thread/support.h>
+#    include <__thread/timed_backoff_policy.h>
+#    include <limits>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -80,7 +82,7 @@ functions. It avoids contention against users' own use of those facilities.
 
 */
 
-#    define _LIBCPP_SEMAPHORE_MAX (numeric_limits<ptrdiff_t>::max())
+#      define _LIBCPP_SEMAPHORE_MAX (numeric_limits<ptrdiff_t>::max())
 
 class __atomic_semaphore_base {
   atomic<ptrdiff_t> __a_;
@@ -96,8 +98,9 @@ public:
     }
   }
   _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() {
-    std::__atomic_wait_unless(
-        __a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed);
+    std::__atomic_wait_unless(__a_, memory_order_relaxed, [this](ptrdiff_t& __old) {
+      return __try_acquire_impl(__old);
+    });
   }
   template <class _Rep, class _Period>
   _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
@@ -173,15 +176,16 @@ using binary_semaphore = counting_semaphore<1>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <atomic>
-#  include <cstddef>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <atomic>
+#    include <cstddef>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SEMAPHORE
diff --git a/libcxx/include/set b/libcxx/include/set
index 5db0db8086dec..c4b0b018f27c4 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -512,57 +512,59 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 
 */
 
-#include <__algorithm/equal.h>
-#include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
-#include <__assert>
-#include <__config>
-#include <__functional/is_transparent.h>
-#include <__functional/operations.h>
-#include <__iterator/erase_if_container.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/ranges_iterator_traits.h>
-#include <__iterator/reverse_iterator.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__node_handle>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__tree>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/equal.h>
+#  include <__algorithm/lexicographical_compare.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__assert>
+#  include <__config>
+#  include <__functional/is_transparent.h>
+#  include <__functional/operations.h>
+#  include <__iterator/erase_if_container.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/ranges_iterator_traits.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__node_handle>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__tree>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [associative.set.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -602,10 +604,10 @@ public:
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __set_node_handle<typename __base::__node, allocator_type> node_type;
   typedef __insert_return_type<iterator, node_type> insert_return_type;
-#endif
+#  endif
 
   template <class _Key2, class _Compare2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS set;
@@ -635,7 +637,7 @@ public:
     insert(__f, __l);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI
   set(from_range_t,
@@ -645,19 +647,19 @@ public:
       : __tree_(__comp, __a) {
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI set(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
       : set(__f, __l, key_compare(), __a) {}
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI set(from_range_t, _Range&& __range, const allocator_type& __a)
       : set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
 
@@ -666,10 +668,10 @@ public:
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
       : __tree_(std::move(__s.__tree_)) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
 
@@ -677,7 +679,7 @@ public:
     insert(__s.begin(), __s.end());
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __a);
 
   _LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
@@ -690,10 +692,10 @@ public:
     insert(__il.begin(), __il.end());
   }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const allocator_type& __a)
       : set(__il, key_compare(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI set& operator=(initializer_list<value_type> __il) {
     __tree_.__assign_unique(__il.begin(), __il.end());
@@ -704,7 +706,7 @@ public:
     __tree_ = std::move(__s.__tree_);
     return *this;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI ~set() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
 
@@ -728,7 +730,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
 
   // modifiers:
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
     return __tree_.__emplace_unique(std::forward<_Args>(__args)...);
@@ -737,7 +739,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
     return __tree_.__emplace_hint_unique(__p, std::forward<_Args>(__args)...);
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__insert_unique(__v); }
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
@@ -750,7 +752,7 @@ public:
       __tree_.__insert_unique(__e, *__f);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     const_iterator __end = cend();
@@ -758,9 +760,9 @@ public:
       __tree_.__insert_unique(__end, std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __v) {
     return __tree_.__insert_unique(std::move(__v));
   }
@@ -770,14 +772,14 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p); }
   _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_unique(__k); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) { return __tree_.erase(__f, __l); }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to set::insert()");
@@ -818,7 +820,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     __tree_.__node_handle_merge_unique(__source.__tree_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(set& __s) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__s.__tree_); }
 
@@ -829,7 +831,7 @@ public:
   // set operations:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
@@ -838,27 +840,27 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
@@ -868,11 +870,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
@@ -881,7 +883,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __tree_.__equal_range_unique(__k);
@@ -889,7 +891,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __tree_.__equal_range_unique(__k);
   }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -898,10 +900,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#endif
+#  endif
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Compare   = less<__iter_value_type<_InputIterator>>,
           class _Allocator = allocator<__iter_value_type<_InputIterator>>,
@@ -911,7 +913,7 @@ template <class _InputIterator,
 set(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
     -> set<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Compare   = less<ranges::range_value_t<_Range>>,
           class _Allocator = allocator<ranges::range_value_t<_Range>>,
@@ -919,7 +921,7 @@ template <ranges::input_range _Range,
           class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
 set(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
     -> set<ranges::range_value_t<_Range>, _Compare, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key,
           class _Compare   = less<_Key>,
@@ -936,18 +938,18 @@ set(_InputIterator,
     _InputIterator,
     _Allocator) -> set<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 set(from_range_t,
     _Range&&,
     _Allocator) -> set<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Compare, class _Allocator>
 set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {
@@ -958,7 +960,7 @@ set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __t
   }
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -966,7 +968,7 @@ operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare,
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -998,7 +1000,7 @@ operator<=(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare,
   return !(__y < __x);
 }
 
-#else // _LIBCPP_STD_VER <= 17
+#  else // _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Key>
@@ -1006,7 +1008,7 @@ operator<=>(const set<_Key, _Allocator>& __x, const set<_Key, _Allocator>& __y)
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
 
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 // specialized algorithms:
 template <class _Key, class _Compare, class _Allocator>
@@ -1015,13 +1017,13 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(set<_Key, _Compare, _Allocator>& __x, set
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename set<_Key, _Compare, _Allocator>::size_type
 erase_if(set<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Compare, class _Allocator>
 struct __container_traits<set<_Key, _Compare, _Allocator> > {
@@ -1064,9 +1066,9 @@ public:
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __set_node_handle<typename __base::__node, allocator_type> node_type;
-#endif
+#  endif
 
   template <class _Key2, class _Compare2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS set;
@@ -1091,11 +1093,11 @@ public:
     insert(__f, __l);
   }
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI multiset(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
       : multiset(__f, __l, key_compare(), __a) {}
-#endif
+#  endif
 
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI
@@ -1104,7 +1106,7 @@ public:
     insert(__f, __l);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI
   multiset(from_range_t,
@@ -1118,7 +1120,7 @@ public:
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI multiset(from_range_t, _Range&& __range, const allocator_type& __a)
       : multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
       : __tree_(__s.__tree_.value_comp(),
@@ -1131,19 +1133,19 @@ public:
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
       : __tree_(std::move(__s.__tree_)) {}
 
   _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI explicit multiset(const allocator_type& __a) : __tree_(__a) {}
   _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a)
       : __tree_(__s.__tree_.value_comp(), __a) {
     insert(__s.begin(), __s.end());
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
       : __tree_(__comp) {
     insert(__il.begin(), __il.end());
@@ -1155,10 +1157,10 @@ public:
     insert(__il.begin(), __il.end());
   }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const allocator_type& __a)
       : multiset(__il, key_compare(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI multiset& operator=(initializer_list<value_type> __il) {
     __tree_.__assign_multi(__il.begin(), __il.end());
@@ -1169,7 +1171,7 @@ public:
     __tree_ = std::move(__s.__tree_);
     return *this;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI ~multiset() { static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
 
@@ -1193,7 +1195,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
 
   // modifiers:
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
     return __tree_.__emplace_multi(std::forward<_Args>(__args)...);
@@ -1202,7 +1204,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
     return __tree_.__emplace_hint_multi(__p, std::forward<_Args>(__args)...);
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __v) { return __tree_.__insert_multi(__v); }
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
@@ -1215,7 +1217,7 @@ public:
       __tree_.__insert_multi(__e, *__f);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     const_iterator __end = cend();
@@ -1223,9 +1225,9 @@ public:
       __tree_.__insert_multi(__end, std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __v) { return __tree_.__insert_multi(std::move(__v)); }
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
@@ -1233,14 +1235,14 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __tree_.erase(__p); }
   _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __tree_.__erase_multi(__k); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l) { return __tree_.erase(__f, __l); }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __tree_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to multiset::insert()");
@@ -1281,7 +1283,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     __tree_.__node_handle_merge_multi(__source.__tree_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(multiset& __s) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) {
     __tree_.swap(__s.__tree_);
@@ -1294,7 +1296,7 @@ public:
   // set operations:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
@@ -1303,27 +1305,27 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
@@ -1333,11 +1335,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
@@ -1346,7 +1348,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -1354,7 +1356,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
@@ -1363,10 +1365,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
-#endif
+#  endif
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Compare   = less<__iter_value_type<_InputIterator>>,
           class _Allocator = allocator<__iter_value_type<_InputIterator>>,
@@ -1376,7 +1378,7 @@ template <class _InputIterator,
 multiset(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
     -> multiset<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Compare   = less<ranges::range_value_t<_Range>>,
           class _Allocator = allocator<ranges::range_value_t<_Range>>,
@@ -1384,7 +1386,7 @@ template <ranges::input_range _Range,
           class            = enable_if_t<!__is_allocator<_Compare>::value, void>>
 multiset(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
     -> multiset<ranges::range_value_t<_Range>, _Compare, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key,
           class _Compare   = less<_Key>,
@@ -1402,18 +1404,18 @@ template <class _InputIterator,
 multiset(_InputIterator, _InputIterator, _Allocator)
     -> multiset<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 multiset(from_range_t,
          _Range&&,
          _Allocator) -> multiset<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
 multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Compare, class _Allocator>
 multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_type& __a)
@@ -1425,7 +1427,7 @@ multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_t
   }
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -1433,7 +1435,7 @@ operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key,
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
@@ -1465,7 +1467,7 @@ operator<=(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key,
   return !(__y < __x);
 }
 
-#else // _LIBCPP_STD_VER <= 17
+#  else // _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Key>
@@ -1473,7 +1475,7 @@ operator<=>(const multiset<_Key, _Allocator>& __x, const multiset<_Key, _Allocat
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), __synth_three_way);
 }
 
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Compare, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void
@@ -1482,13 +1484,13 @@ swap(multiset<_Key, _Compare, _Allocator>& __x, multiset<_Key, _Compare, _Alloca
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename multiset<_Key, _Compare, _Allocator>::size_type
 erase_if(multiset<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Compare, class _Allocator>
 struct __container_traits<multiset<_Key, _Compare, _Allocator> > {
@@ -1500,7 +1502,7 @@ struct __container_traits<multiset<_Key, _Compare, _Allocator> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _KeyT, class _CompareT = std::less<_KeyT>>
@@ -1510,17 +1512,18 @@ template <class _KeyT, class _CompareT = std::less<_KeyT>>
 using multiset _LIBCPP_AVAILABILITY_PMR = std::multiset<_KeyT, _CompareT, polymorphic_allocator<_KeyT>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <cstdlib>
-#  include <functional>
-#  include <iterator>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <cstdlib>
+#    include <functional>
+#    include <iterator>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SET
diff --git a/libcxx/include/shared_mutex b/libcxx/include/shared_mutex
index 2552119405419..a6615ea8962fd 100644
--- a/libcxx/include/shared_mutex
+++ b/libcxx/include/shared_mutex
@@ -122,31 +122,33 @@ template <class Mutex>
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_THREADS
-
-#  include <__chrono/duration.h>
-#  include <__chrono/steady_clock.h>
-#  include <__chrono/time_point.h>
-#  include <__condition_variable/condition_variable.h>
-#  include <__memory/addressof.h>
-#  include <__mutex/mutex.h>
-#  include <__mutex/tag_types.h>
-#  include <__mutex/unique_lock.h>
-#  include <__system_error/throw_system_error.h>
-#  include <__utility/swap.h>
-#  include <cerrno>
-#  include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_THREADS
+
+#    include <__chrono/duration.h>
+#    include <__chrono/steady_clock.h>
+#    include <__chrono/time_point.h>
+#    include <__condition_variable/condition_variable.h>
+#    include <__memory/addressof.h>
+#    include <__mutex/mutex.h>
+#    include <__mutex/tag_types.h>
+#    include <__mutex/unique_lock.h>
+#    include <__system_error/throw_system_error.h>
+#    include <__utility/swap.h>
+#    include <cerrno>
+#    include <version>
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
 
-#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#      pragma GCC system_header
-#    endif
+#      if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#        pragma GCC system_header
+#      endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -179,7 +181,7 @@ struct _LIBCPP_EXPORTED_FROM_ABI __shared_mutex_base {
   //     native_handle_type native_handle(); // See 30.2.3
 };
 
-#    if _LIBCPP_STD_VER >= 17
+#      if _LIBCPP_STD_VER >= 17
 class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_mutex")) shared_mutex {
   __shared_mutex_base __base_;
 
@@ -216,7 +218,7 @@ public:
   //     typedef __shared_mutex_base::native_handle_type native_handle_type;
   //     _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return __base::unlock_shared(); }
 };
-#    endif
+#      endif
 
 class _LIBCPP_EXPORTED_FROM_ABI
 _LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_timed_mutex")) shared_timed_mutex {
@@ -451,14 +453,15 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(shared_lock<_Mutex>& __x, shared_lock<_Mu
 
 _LIBCPP_END_NAMESPACE_STD
 
-#  endif // _LIBCPP_STD_VER >= 14
+#    endif // _LIBCPP_STD_VER >= 14
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <system_error>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <system_error>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SHARED_MUTEX
diff --git a/libcxx/include/source_location b/libcxx/include/source_location
index d16e3c46fce55..faa6b0eeb6c00 100644
--- a/libcxx/include/source_location
+++ b/libcxx/include/source_location
@@ -25,17 +25,19 @@ namespace std {
 }
 */
 
-#include <__config>
-#include <cstdint>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <cstdint>
+#  include <version>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 class source_location {
   // The names source_location::__impl, _M_file_name, _M_function_name, _M_line, and _M_column
@@ -78,8 +80,10 @@ public:
   }
 };
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
 
+#endif // 0
+
 #endif // _LIBCPP_SOURCE_LOCATION
diff --git a/libcxx/include/span b/libcxx/include/span
index 896a3cd890186..2d43d1d1079e4 100644
--- a/libcxx/include/span
+++ b/libcxx/include/span
@@ -144,60 +144,62 @@ template<class R>
 
 */
 
-#include <__assert>
-#include <__concepts/convertible_to.h>
-#include <__concepts/equality_comparable.h>
-#include <__config>
-#include <__cstddef/byte.h>
-#include <__cstddef/ptrdiff_t.h>
-#include <__fwd/array.h>
-#include <__fwd/span.h>
-#include <__iterator/bounded_iter.h>
-#include <__iterator/concepts.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/wrap_iter.h>
-#include <__memory/pointer_traits.h>
-#include <__ranges/concepts.h>
-#include <__ranges/data.h>
-#include <__ranges/enable_borrowed_range.h>
-#include <__ranges/enable_view.h>
-#include <__ranges/size.h>
-#include <__type_traits/integral_constant.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_const.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_integral.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <initializer_list>
-#include <stdexcept>
-#include <version>
+#if 0
+#else // 0
+#  include <__assert>
+#  include <__concepts/convertible_to.h>
+#  include <__concepts/equality_comparable.h>
+#  include <__config>
+#  include <__cstddef/byte.h>
+#  include <__cstddef/ptrdiff_t.h>
+#  include <__fwd/array.h>
+#  include <__fwd/span.h>
+#  include <__iterator/bounded_iter.h>
+#  include <__iterator/concepts.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/wrap_iter.h>
+#  include <__memory/pointer_traits.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/data.h>
+#  include <__ranges/enable_borrowed_range.h>
+#  include <__ranges/enable_view.h>
+#  include <__ranges/size.h>
+#  include <__type_traits/integral_constant.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_const.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_integral.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/remove_cv.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <initializer_list>
+#  include <stdexcept>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <class _Tp>
 struct __is_std_span : false_type {};
@@ -237,11 +239,11 @@ public:
   using const_pointer   = const _Tp*;
   using reference       = _Tp&;
   using const_reference = const _Tp&;
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
   using iterator = __bounded_iter<pointer>;
-#  else
+#    else
   using iterator = __wrap_iter<pointer>;
-#  endif
+#    endif
   using reverse_iterator = std::reverse_iterator<iterator>;
 
   static constexpr size_type extent = _Extent;
@@ -251,14 +253,14 @@ public:
     requires(_Sz == 0)
   _LIBCPP_HIDE_FROM_ABI constexpr span() noexcept : __data_{nullptr} {}
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI constexpr explicit span(std::initializer_list<value_type> __il)
     requires is_const_v<element_type>
       : __data_{__il.begin()} {
     _LIBCPP_ASSERT_VALID_INPUT_RANGE(
         _Extent == __il.size(), "Size mismatch in span's constructor _Extent != __il.size().");
   }
-#  endif
+#    endif
 
   constexpr span(const span&) noexcept            = default;
   constexpr span& operator=(const span&) noexcept = default;
@@ -267,6 +269,8 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr explicit span(_It __first, size_type __count) : __data_{std::to_address(__first)} {
     (void)__count;
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Extent == __count, "size mismatch in span's constructor (iterator, len)");
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__count == 0 || std::to_address(__first) != nullptr,
+                                     "passed nullptr with non-zero length in span's constructor (iterator, len)");
   }
 
   template <__span_compatible_iterator<element_type> _It, __span_compatible_sentinel_for<_It> _End>
@@ -356,13 +360,13 @@ public:
     return __data_[__idx];
   }
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
     if (__index >= size())
       std::__throw_out_of_range("span");
     return __data_[__index];
   }
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T, N>::front() on empty span");
@@ -378,18 +382,18 @@ public:
 
   // [span.iter], span iterator support
   _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data(), data(), data() + size());
-#  else
+#    else
     return iterator(data());
-#  endif
+#    endif
   }
   _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data() + size(), data(), data() + size());
-#  else
+#    else
     return iterator(data() + size());
-#  endif
+#    endif
   }
   _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
   _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
@@ -418,11 +422,11 @@ public:
   using const_pointer   = const _Tp*;
   using reference       = _Tp&;
   using const_reference = const _Tp&;
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
   using iterator = __bounded_iter<pointer>;
-#  else
+#    else
   using iterator = __wrap_iter<pointer>;
-#  endif
+#    endif
   using reverse_iterator = std::reverse_iterator<iterator>;
 
   static constexpr size_type extent = dynamic_extent;
@@ -430,18 +434,21 @@ public:
   // [span.cons], span constructors, copy, assignment, and destructor
   _LIBCPP_HIDE_FROM_ABI constexpr span() noexcept : __data_{nullptr}, __size_{0} {}
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI constexpr span(std::initializer_list<value_type> __il)
     requires is_const_v<element_type>
       : __data_{__il.begin()}, __size_{__il.size()} {}
-#  endif
+#    endif
 
   constexpr span(const span&) noexcept            = default;
   constexpr span& operator=(const span&) noexcept = default;
 
   template <__span_compatible_iterator<element_type> _It>
   _LIBCPP_HIDE_FROM_ABI constexpr span(_It __first, size_type __count)
-      : __data_{std::to_address(__first)}, __size_{__count} {}
+      : __data_{std::to_address(__first)}, __size_{__count} {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__count == 0 || std::to_address(__first) != nullptr,
+                                     "passed nullptr with non-zero length in span's constructor (iterator, len)");
+  }
 
   template <__span_compatible_iterator<element_type> _It, __span_compatible_sentinel_for<_It> _End>
   _LIBCPP_HIDE_FROM_ABI constexpr span(_It __first, _End __last)
@@ -518,13 +525,13 @@ public:
     return __data_[__idx];
   }
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
     if (__index >= size())
       std::__throw_out_of_range("span");
     return __data_[__index];
   }
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T>::front() on empty span");
@@ -540,18 +547,18 @@ public:
 
   // [span.iter], span iterator support
   _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data(), data(), data() + size());
-#  else
+#    else
     return iterator(data());
-#  endif
+#    endif
   }
   _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
-#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#    ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data() + size(), data(), data() + size());
-#  else
+#    else
     return iterator(data() + size());
-#  endif
+#    endif
   }
   _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
   _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
@@ -587,7 +594,7 @@ _LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept {
   return __s.__as_writable_bytes();
 }
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 template <class _Tp>
 concept __integral_constant_like =
     is_integral_v<decltype(_Tp::value)> && !is_same_v<bool, remove_const_t<decltype(_Tp::value)>> &&
@@ -603,10 +610,10 @@ inline constexpr size_t __maybe_static_ext<_Tp> = {_Tp::value};
 
 template <contiguous_iterator _It, class _EndOrSize>
 span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>, __maybe_static_ext<_EndOrSize>>;
-#  else
+#    else
 template <contiguous_iterator _It, class _EndOrSize>
 span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;
-#  endif
+#    endif
 
 template <class _Tp, size_t _Sz>
 span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>;
@@ -620,18 +627,19 @@ span(const array<_Tp, _Sz>&) -> span<const _Tp, _Sz>;
 template <ranges::contiguous_range _Range>
 span(_Range&&) -> span<remove_reference_t<ranges::range_reference_t<_Range>>>;
 
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <array>
-#  include <concepts>
-#  include <functional>
-#  include <iterator>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <array>
+#    include <concepts>
+#    include <functional>
+#    include <iterator>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SPAN
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index 4f458c5dfce98..920c29dee62f3 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -312,27 +312,29 @@ typedef basic_stringstream<wchar_t> wstringstream;
 
 // clang-format on
 
-#include <__config>
-
-#if _LIBCPP_HAS_LOCALIZATION
-
-#  include <__fwd/sstream.h>
-#  include <__ostream/basic_ostream.h>
-#  include <__type_traits/is_convertible.h>
-#  include <__utility/swap.h>
-#  include <ios>
-#  include <istream>
-#  include <locale>
-#  include <string>
-#  include <string_view>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+
+#    include <__fwd/sstream.h>
+#    include <__ostream/basic_ostream.h>
+#    include <__type_traits/is_convertible.h>
+#    include <__utility/swap.h>
+#    include <ios>
+#    include <istream>
+#    include <locale>
+#    include <string>
+#    include <string_view>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -375,7 +377,7 @@ public:
     str(__s);
   }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const allocator_type& __a)
       : basic_stringbuf(ios_base::in | ios_base::out, __a) {}
 
@@ -409,9 +411,9 @@ public:
       : __str_(__s), __hm_(nullptr), __mode_(__wch) {
     __init_buf_ptrs();
   }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
@@ -433,37 +435,37 @@ public:
     __init_buf_ptrs();
   }
 
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
   basic_stringbuf(const basic_stringbuf&) = delete;
   basic_stringbuf(basic_stringbuf&& __rhs) : __mode_(__rhs.__mode_) { __move_init(std::move(__rhs)); }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI basic_stringbuf(basic_stringbuf&& __rhs, const allocator_type& __a)
       : basic_stringbuf(__rhs.__mode_, __a) {
     __move_init(std::move(__rhs));
   }
-#  endif
+#    endif
 
   // [stringbuf.assign] Assign and swap:
   basic_stringbuf& operator=(const basic_stringbuf&) = delete;
   basic_stringbuf& operator=(basic_stringbuf&& __rhs);
   void swap(basic_stringbuf& __rhs)
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
       noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value ||
                allocator_traits<allocator_type>::is_always_equal::value)
-#  endif
+#    endif
           ;
 
   // [stringbuf.members] Member functions:
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
-#  endif
+#    endif
 
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
   string_type str() const;
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI string_type str() const& { return str(__str_.get_allocator()); }
 
   _LIBCPP_HIDE_FROM_ABI string_type str() && {
@@ -477,9 +479,9 @@ public:
     __init_buf_ptrs();
     return __result;
   }
-#  endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
     requires __is_allocator<_SAlloc>::value
   _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
@@ -487,14 +489,14 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept;
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
   void str(const string_type& __s) {
     __str_ = __s;
     __init_buf_ptrs();
   }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
     requires(!is_same_v<_SAlloc, allocator_type>)
   _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
@@ -506,9 +508,9 @@ public:
     __str_ = std::move(__s);
     __init_buf_ptrs();
   }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
@@ -518,7 +520,7 @@ public:
     __init_buf_ptrs();
   }
 
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
 protected:
   // [stringbuf.virtuals] Overridden virtual functions:
@@ -614,10 +616,10 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::operator=(basic_stringbuf&& __rhs)
 
 template <class _CharT, class _Traits, class _Allocator>
 void basic_stringbuf<_CharT, _Traits, _Allocator>::swap(basic_stringbuf& __rhs)
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
     noexcept(allocator_traits<_Allocator>::propagate_on_container_swap::value ||
              allocator_traits<_Allocator>::is_always_equal::value)
-#  endif
+#    endif
 {
   char_type* __p    = const_cast<char_type*>(__rhs.__str_.data());
   ptrdiff_t __rbinp = -1;
@@ -687,14 +689,14 @@ void basic_stringbuf<_CharT, _Traits, _Allocator>::swap(basic_stringbuf& __rhs)
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI void
 swap(basic_stringbuf<_CharT, _Traits, _Allocator>& __x, basic_stringbuf<_CharT, _Traits, _Allocator>& __y)
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
     noexcept(noexcept(__x.swap(__y)))
-#  endif
+#    endif
 {
   __x.swap(__y);
 }
 
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator> basic_stringbuf<_CharT, _Traits, _Allocator>::str() const {
   if (__mode_ & ios_base::out) {
@@ -705,7 +707,7 @@ basic_string<_CharT, _Traits, _Allocator> basic_stringbuf<_CharT, _Traits, _Allo
     return string_type(this->eback(), this->egptr(), __str_.get_allocator());
   return string_type(__str_.get_allocator());
 }
-#  endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI void basic_stringbuf<_CharT, _Traits, _Allocator>::__init_buf_ptrs() {
@@ -731,7 +733,7 @@ _LIBCPP_HIDE_FROM_ABI void basic_stringbuf<_CharT, _Traits, _Allocator>::__init_
   }
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT, _Traits>
 basic_stringbuf<_CharT, _Traits, _Allocator>::view() const noexcept {
@@ -743,7 +745,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::view() const noexcept {
     return basic_string_view<_CharT, _Traits>(this->eback(), this->egptr());
   return basic_string_view<_CharT, _Traits>();
 }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits, class _Allocator>
 typename basic_stringbuf<_CharT, _Traits, _Allocator>::int_type
@@ -786,9 +788,9 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) {
     if (this->pptr() == this->epptr()) {
       if (!(__mode_ & ios_base::out))
         return traits_type::eof();
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
         ptrdiff_t __nout = this->pptr() - this->pbase();
         ptrdiff_t __hm   = __hm_ - this->pbase();
         __str_.push_back(char_type());
@@ -797,11 +799,11 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) {
         this->setp(__p, __p + __str_.size());
         this->__pbump(__nout);
         __hm_ = this->pbase() + __hm;
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return traits_type::eof();
       }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     }
     __hm_ = std::max(this->pptr() + 1, __hm_);
     if (__mode_ & ios_base::in) {
@@ -886,7 +888,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const string_type& __s, ios_base::openmode __wch = ios_base::in)
       : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in) {}
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI basic_istringstream(ios_base::openmode __wch, const _Allocator& __a)
       : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch | ios_base::in, __a) {}
 
@@ -906,9 +908,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
                                                      ios_base::openmode __wch = ios_base::in)
       : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in) {}
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
@@ -925,7 +927,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI basic_istringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
       : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::in, __a) {}
 
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
   basic_istringstream(const basic_istringstream&) = delete;
   _LIBCPP_HIDE_FROM_ABI basic_istringstream(basic_istringstream&& __rhs)
@@ -950,15 +952,15 @@ public:
     return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
   }
 
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
   _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
 
   _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
-#  endif
+#    endif
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
     requires __is_allocator<_SAlloc>::value
   _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
@@ -966,26 +968,26 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
   _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
     __sb_.str(__s);
   }
 
   _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
   _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
     rdbuf()->str(__t);
   }
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 };
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -1022,7 +1024,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const string_type& __s, ios_base::openmode __wch = ios_base::out)
       : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out) {}
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI basic_ostringstream(ios_base::openmode __wch, const _Allocator& __a)
       : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch | ios_base::out, __a) {}
 
@@ -1043,9 +1045,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
                                                      ios_base::openmode __wch = ios_base::out)
       : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out) {}
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
@@ -1062,7 +1064,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
       : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::out, __a) {}
 
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
   basic_ostringstream(const basic_ostringstream&) = delete;
   _LIBCPP_HIDE_FROM_ABI basic_ostringstream(basic_ostringstream&& __rhs)
@@ -1088,15 +1090,15 @@ public:
     return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
   }
 
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
   _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
 
   _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
-#  endif
+#    endif
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
     requires __is_allocator<_SAlloc>::value
   _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
@@ -1104,26 +1106,26 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
   _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
     __sb_.str(__s);
   }
 
   _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
   _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
     rdbuf()->str(__t);
   }
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 };
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -1161,7 +1163,7 @@ public:
                                                     ios_base::openmode __wch = ios_base::in | ios_base::out)
       : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch) {}
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI basic_stringstream(ios_base::openmode __wch, const _Allocator& __a)
       : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__wch, __a) {}
 
@@ -1183,9 +1185,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(const basic_string<_CharT, _Traits, _SAlloc>& __s,
                                                     ios_base::openmode __wch = ios_base::out | ios_base::in)
       : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch) {}
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
 
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
@@ -1203,7 +1205,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI basic_stringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a)
       : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which, __a) {}
 
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 
   basic_stringstream(const basic_stringstream&) = delete;
   _LIBCPP_HIDE_FROM_ABI basic_stringstream(basic_stringstream&& __rhs)
@@ -1228,15 +1230,15 @@ public:
     return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
   }
 
-#  if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
+#    if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_BUILDING_LIBRARY)
   _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
-#  else
+#    else
   _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
 
   _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
-#  endif
+#    endif
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
     requires __is_allocator<_SAlloc>::value
   _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
@@ -1244,26 +1246,26 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _SAlloc>
   _LIBCPP_HIDE_FROM_ABI void str(const basic_string<char_type, traits_type, _SAlloc>& __s) {
     __sb_.str(__s);
   }
 
   _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); }
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
-#  if _LIBCPP_STD_VER >= 26
+#    if _LIBCPP_STD_VER >= 26
   template <class _Tp>
     requires is_convertible_v<const _Tp&, basic_string_view<_CharT, _Traits>>
   _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) {
     rdbuf()->str(__t);
   }
-#  endif //  _LIBCPP_STD_VER >= 26
+#    endif //  _LIBCPP_STD_VER >= 26
 };
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -1272,22 +1274,23 @@ swap(basic_stringstream<_CharT, _Traits, _Allocator>& __x, basic_stringstream<_C
   __x.swap(__y);
 }
 
-#  if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
+#    if _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_stringbuf<char>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_stringstream<char>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostringstream<char>;
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istringstream<char>;
-#  endif
+#    endif
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_LOCALIZATION
 
-#if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
-#  include <ostream>
-#  include <type_traits>
-#endif
+#  if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
+#    include <ostream>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SSTREAM
diff --git a/libcxx/include/stack b/libcxx/include/stack
index f75769f8a4206..da3dcf0216aea 100644
--- a/libcxx/include/stack
+++ b/libcxx/include/stack
@@ -113,33 +113,35 @@ template <class T, class Container>
 
 */
 
-#include <__algorithm/ranges_copy.h>
-#include <__config>
-#include <__fwd/stack.h>
-#include <__iterator/back_insert_iterator.h>
-#include <__iterator/iterator_traits.h>
-#include <__memory/uses_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__type_traits/is_same.h>
-#include <__utility/forward.h>
-#include <deque>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/ranges_copy.h>
+#  include <__config>
+#  include <__fwd/stack.h>
+#  include <__iterator/back_insert_iterator.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__memory/uses_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__type_traits/is_same.h>
+#  include <__utility/forward.h>
+#  include <deque>
+#  include <version>
 
 // standard-mandated includes
 
 // [stack.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -172,7 +174,7 @@ public:
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI stack(stack&& __q) noexcept(is_nothrow_move_constructible<container_type>::value)
       : c(std::move(__q.c)) {}
 
@@ -182,7 +184,7 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI explicit stack(container_type&& __c) : c(std::move(__c)) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit stack(const container_type& __c) : c(__c) {}
 
@@ -198,7 +200,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   stack(const stack& __s, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
       : c(__s.c, __a) {}
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class _Alloc>
   _LIBCPP_HIDE_FROM_ABI
   stack(container_type&& __c, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
@@ -207,9 +209,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   stack(stack&& __s, const _Alloc& __a, __enable_if_t<uses_allocator<container_type, _Alloc>::value>* = 0)
       : c(std::move(__s.c), __a) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI stack(_InputIterator __first, _InputIterator __last) : c(__first, __last) {}
 
@@ -229,7 +231,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI stack(from_range_t, _Range&& __range, const _Alloc& __alloc)
       : c(from_range, std::forward<_Range>(__range), __alloc) {}
 
-#endif
+#  endif
 
   [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
   _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
@@ -237,10 +239,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.back(); }
 
   _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v) { c.push_back(std::move(__v)); }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
   _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
     if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
@@ -249,22 +251,22 @@ public:
       ranges::copy(std::forward<_Range>(__range), std::back_inserter(c));
     }
   }
-#  endif
+#    endif
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
   decltype(auto)
   emplace(_Args&&... __args) {
     return c.emplace_back(std::forward<_Args>(__args)...);
   }
-#  else
+#    else
   void
   emplace(_Args&&... __args) {
     c.emplace_back(std::forward<_Args>(__args)...);
   }
-#  endif
-#endif // _LIBCPP_CXX03_LANG
+#    endif
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI void pop() { c.pop_back(); }
 
@@ -282,7 +284,7 @@ public:
   friend bool operator<(const stack<_T1, _OtherContainer>& __x, const stack<_T1, _OtherContainer>& __y);
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
 stack(_Container) -> stack<typename _Container::value_type, _Container>;
 
@@ -291,9 +293,9 @@ template <class _Container,
           class = enable_if_t<!__is_allocator<_Container>::value>,
           class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
 stack(_Container, _Alloc) -> stack<typename _Container::value_type, _Container>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
 stack(_InputIterator, _InputIterator) -> stack<__iter_value_type<_InputIterator>>;
 
@@ -313,7 +315,7 @@ stack(from_range_t,
       _Range&&,
       _Alloc) -> stack<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
 
-#endif
+#  endif
 
 template <class _Tp, class _Container>
 inline _LIBCPP_HIDE_FROM_ABI bool operator==(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y) {
@@ -345,7 +347,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const stack<_Tp, _Container>& __x,
   return !(__y < __x);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <class _Tp, three_way_comparable _Container>
 _LIBCPP_HIDE_FROM_ABI compare_three_way_result_t<_Container>
@@ -354,7 +356,7 @@ operator<=>(const stack<_Tp, _Container>& __x, const stack<_Tp, _Container>& __y
   return __x.__get_container() <=> __y.__get_container();
 }
 
-#endif
+#  endif
 
 template <class _Tp, class _Container, __enable_if_t<__is_swappable_v<_Container>, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI void swap(stack<_Tp, _Container>& __x, stack<_Tp, _Container>& __y)
@@ -370,10 +372,11 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <concepts>
-#  include <functional>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <concepts>
+#    include <functional>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STACK
diff --git a/libcxx/include/stdatomic.h b/libcxx/include/stdatomic.h
index f4e9b59c5fe59..8c96bd636dd45 100644
--- a/libcxx/include/stdatomic.h
+++ b/libcxx/include/stdatomic.h
@@ -117,22 +117,24 @@ using std::atomic_signal_fence                         // see below
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if defined(__cplusplus)
+#  if defined(__cplusplus)
 
-#  include <atomic>
-#  include <version>
+#    include <atomic>
+#    include <version>
 
-#  ifdef _Atomic
-#    undef _Atomic
-#  endif
+#    ifdef _Atomic
+#      undef _Atomic
+#    endif
 
-#  define _Atomic(_Tp) ::std::atomic<_Tp>
+#    define _Atomic(_Tp) ::std::atomic<_Tp>
 
 using std::memory_order _LIBCPP_USING_IF_EXISTS;
 using std::memory_order_relaxed _LIBCPP_USING_IF_EXISTS;
@@ -156,14 +158,14 @@ using std::atomic_long _LIBCPP_USING_IF_EXISTS;
 using std::atomic_ulong _LIBCPP_USING_IF_EXISTS;
 using std::atomic_llong _LIBCPP_USING_IF_EXISTS;
 using std::atomic_ullong _LIBCPP_USING_IF_EXISTS;
-#  if _LIBCPP_HAS_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
 using std::atomic_char8_t _LIBCPP_USING_IF_EXISTS;
-#  endif
+#    endif
 using std::atomic_char16_t _LIBCPP_USING_IF_EXISTS;
 using std::atomic_char32_t _LIBCPP_USING_IF_EXISTS;
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 using std::atomic_wchar_t _LIBCPP_USING_IF_EXISTS;
-#  endif
+#    endif
 
 using std::atomic_int8_t _LIBCPP_USING_IF_EXISTS;
 using std::atomic_uint8_t _LIBCPP_USING_IF_EXISTS;
@@ -228,12 +230,13 @@ using std::atomic_store_explicit _LIBCPP_USING_IF_EXISTS;
 using std::atomic_signal_fence _LIBCPP_USING_IF_EXISTS;
 using std::atomic_thread_fence _LIBCPP_USING_IF_EXISTS;
 
-#else
+#  else
 
-#  if __has_include_next(<stdatomic.h>)
-#    include_next <stdatomic.h>
-#  endif
+#    if __has_include_next(<stdatomic.h>)
+#      include_next <stdatomic.h>
+#    endif
 
-#endif // defined(__cplusplus)
+#  endif // defined(__cplusplus)
+#endif   // 0
 
 #endif // _LIBCPP_STDATOMIC_H
diff --git a/libcxx/include/stdbool.h b/libcxx/include/stdbool.h
index e74d91f459459..9f1779e992c72 100644
--- a/libcxx/include/stdbool.h
+++ b/libcxx/include/stdbool.h
@@ -19,22 +19,25 @@
 
 */
 
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if __has_include_next(<stdbool.h>)
-#  include_next <stdbool.h>
-#endif
-
-#ifdef __cplusplus
-#  undef bool
-#  undef true
-#  undef false
-#  undef __bool_true_false_are_defined
-#  define __bool_true_false_are_defined 1
-#endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if __has_include_next(<stdbool.h>)
+#    include_next <stdbool.h>
+#  endif
+
+#  ifdef __cplusplus
+#    undef bool
+#    undef true
+#    undef false
+#    undef __bool_true_false_are_defined
+#    define __bool_true_false_are_defined 1
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STDBOOL_H
diff --git a/libcxx/include/stddef.h b/libcxx/include/stddef.h
index 1583e78e3739b..66ad85fe631de 100644
--- a/libcxx/include/stddef.h
+++ b/libcxx/include/stddef.h
@@ -24,21 +24,24 @@
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 // Note: This include is outside of header guards because we sometimes get included multiple times
 //       with different defines and the underlying <stddef.h> will know how to deal with that.
-#include_next <stddef.h>
+#  include_next <stddef.h>
 
-#ifndef _LIBCPP_STDDEF_H
-#  define _LIBCPP_STDDEF_H
+#  ifndef _LIBCPP_STDDEF_H
+#    define _LIBCPP_STDDEF_H
 
-#  ifdef __cplusplus
+#    ifdef __cplusplus
 typedef decltype(nullptr) nullptr_t;
-#  endif
+#    endif
+#  endif // 0
 
 #endif // _LIBCPP_STDDEF_H
diff --git a/libcxx/include/stdexcept b/libcxx/include/stdexcept
index 8415d3339f7e3..b3d805437a02f 100644
--- a/libcxx/include/stdexcept
+++ b/libcxx/include/stdexcept
@@ -41,18 +41,20 @@ public:
 
 */
 
-#include <__config>
-#include <__exception/exception.h>
-#include <__fwd/string.h>
-#include <__verbose_abort>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__exception/exception.h>
+#  include <__fwd/string.h>
+#  include <__verbose_abort>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
 class _LIBCPP_HIDDEN __libcpp_refstring {
   const char* __imp_;
 
@@ -66,7 +68,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI const char* c_str() const _NOEXCEPT { return __imp_; }
 };
-#endif // !_LIBCPP_ABI_VCRUNTIME
+#  endif // !_LIBCPP_ABI_VCRUNTIME
 
 _LIBCPP_END_NAMESPACE_STD
 
@@ -74,7 +76,7 @@ namespace std // purposefully not using versioning namespace
 {
 
 class _LIBCPP_EXPORTED_FROM_ABI logic_error : public exception {
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
 
 private:
   std::__libcpp_refstring __imp_;
@@ -89,16 +91,16 @@ public:
   ~logic_error() _NOEXCEPT override;
 
   const char* what() const _NOEXCEPT override;
-#else
+#  else
 
 public:
   explicit logic_error(const std::string&); // Symbol uses versioned std::string
   _LIBCPP_HIDE_FROM_ABI explicit logic_error(const char* __s) : exception(__s) {}
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI runtime_error : public exception {
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
 
 private:
   std::__libcpp_refstring __imp_;
@@ -113,12 +115,12 @@ public:
   ~runtime_error() _NOEXCEPT override;
 
   const char* what() const _NOEXCEPT override;
-#else
+#  else
 
 public:
   explicit runtime_error(const std::string&); // Symbol uses versioned std::string
   _LIBCPP_HIDE_FROM_ABI explicit runtime_error(const char* __s) : exception(__s) {}
-#endif // _LIBCPP_ABI_VCRUNTIME
+#  endif // _LIBCPP_ABI_VCRUNTIME
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI domain_error : public logic_error {
@@ -126,11 +128,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit domain_error(const string& __s) : logic_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit domain_error(const char* __s) : logic_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI domain_error(const domain_error&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI domain_error& operator=(const domain_error&) _NOEXCEPT = default;
   ~domain_error() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI invalid_argument : public logic_error {
@@ -138,22 +140,22 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit invalid_argument(const string& __s) : logic_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit invalid_argument(const char* __s) : logic_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI invalid_argument(const invalid_argument&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI invalid_argument& operator=(const invalid_argument&) _NOEXCEPT = default;
   ~invalid_argument() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI length_error : public logic_error {
 public:
   _LIBCPP_HIDE_FROM_ABI explicit length_error(const string& __s) : logic_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit length_error(const char* __s) : logic_error(__s) {}
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI length_error(const length_error&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI length_error& operator=(const length_error&) _NOEXCEPT = default;
   ~length_error() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI out_of_range : public logic_error {
@@ -161,11 +163,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit out_of_range(const string& __s) : logic_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit out_of_range(const char* __s) : logic_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI out_of_range(const out_of_range&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI out_of_range& operator=(const out_of_range&) _NOEXCEPT = default;
   ~out_of_range() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI range_error : public runtime_error {
@@ -173,11 +175,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit range_error(const string& __s) : runtime_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit range_error(const char* __s) : runtime_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI range_error(const range_error&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI range_error& operator=(const range_error&) _NOEXCEPT = default;
   ~range_error() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI overflow_error : public runtime_error {
@@ -185,11 +187,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit overflow_error(const string& __s) : runtime_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit overflow_error(const char* __s) : runtime_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI overflow_error(const overflow_error&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI overflow_error& operator=(const overflow_error&) _NOEXCEPT = default;
   ~overflow_error() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 class _LIBCPP_EXPORTED_FROM_ABI underflow_error : public runtime_error {
@@ -197,11 +199,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI explicit underflow_error(const string& __s) : runtime_error(__s) {}
   _LIBCPP_HIDE_FROM_ABI explicit underflow_error(const char* __s) : runtime_error(__s) {}
 
-#ifndef _LIBCPP_ABI_VCRUNTIME
+#  ifndef _LIBCPP_ABI_VCRUNTIME
   _LIBCPP_HIDE_FROM_ABI underflow_error(const underflow_error&) _NOEXCEPT            = default;
   _LIBCPP_HIDE_FROM_ABI underflow_error& operator=(const underflow_error&) _NOEXCEPT = default;
   ~underflow_error() _NOEXCEPT override;
-#endif
+#  endif
 };
 
 } // namespace std
@@ -212,76 +214,77 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 [[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void __throw_runtime_error(const char*);
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_logic_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw logic_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("logic_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_domain_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw domain_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("domain_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_invalid_argument(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw invalid_argument(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("invalid_argument was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_length_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw length_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("length_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw out_of_range(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("out_of_range was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_range_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw range_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("range_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_overflow_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw overflow_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("overflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_underflow_error(const char* __msg) {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw underflow_error(__msg);
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("underflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
-#endif
+#  endif
 }
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <exception>
-#  include <iosfwd>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <exception>
+#    include <iosfwd>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STDEXCEPT
diff --git a/libcxx/include/stdio.h b/libcxx/include/stdio.h
index 3aa559393f185..dcf16dfb63782 100644
--- a/libcxx/include/stdio.h
+++ b/libcxx/include/stdio.h
@@ -98,26 +98,29 @@ int ferror(FILE* stream);
 void perror(const char* s);
 */
 
-#  include <__config>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
-
-#  if __has_include_next(<stdio.h>)
-#    include_next <stdio.h>
-#  endif
-
-#  ifdef __cplusplus
-
-#    undef getc
-#    undef putc
-#    undef clearerr
-#    undef feof
-#    undef ferror
-#    undef putchar
-#    undef getchar
-
-#  endif
+#  if 0
+#  else // 0
+#    include <__config>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
+
+#    if __has_include_next(<stdio.h>)
+#      include_next <stdio.h>
+#    endif
+
+#    ifdef __cplusplus
+
+#      undef getc
+#      undef putc
+#      undef clearerr
+#      undef feof
+#      undef ferror
+#      undef putchar
+#      undef getchar
+
+#    endif
+#  endif // 0
 
 #endif // _LIBCPP_STDIO_H
diff --git a/libcxx/include/stdlib.h b/libcxx/include/stdlib.h
index 358b10c0392a5..dca4c3a92de14 100644
--- a/libcxx/include/stdlib.h
+++ b/libcxx/include/stdlib.h
@@ -84,35 +84,37 @@ void *aligned_alloc(size_t alignment, size_t size);                       // C11
 
 */
 
-#  include <__config>
+#  if 0
+#  else // 0
+#    include <__config>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
-#  if __has_include_next(<stdlib.h>)
-#    include_next <stdlib.h>
-#  endif
+#    if __has_include_next(<stdlib.h>)
+#      include_next <stdlib.h>
+#    endif
 
-#  ifdef __cplusplus
+#    ifdef __cplusplus
 extern "C++" {
 // abs
 
-#    ifdef abs
-#      undef abs
-#    endif
-#    ifdef labs
-#      undef labs
-#    endif
-#    ifdef llabs
-#      undef llabs
-#    endif
+#      ifdef abs
+#        undef abs
+#      endif
+#      ifdef labs
+#        undef labs
+#      endif
+#      ifdef llabs
+#        undef llabs
+#      endif
 
 // MSVCRT already has the correct prototype in <stdlib.h> if __cplusplus is defined
-#    if !defined(_LIBCPP_MSVCRT)
+#      if !defined(_LIBCPP_MSVCRT)
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long abs(long __x) _NOEXCEPT { return __builtin_labs(__x); }
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long long abs(long long __x) _NOEXCEPT { return __builtin_llabs(__x); }
-#    endif // !defined(_LIBCPP_MSVCRT)
+#      endif // !defined(_LIBCPP_MSVCRT)
 
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI float abs(float __lcpp_x) _NOEXCEPT {
   return __builtin_fabsf(__lcpp_x); // Use builtins to prevent needing math.h
@@ -128,24 +130,25 @@ extern "C++" {
 
 // div
 
-#    ifdef div
-#      undef div
-#    endif
-#    ifdef ldiv
-#      undef ldiv
-#    endif
-#    ifdef lldiv
-#      undef lldiv
-#    endif
+#      ifdef div
+#        undef div
+#      endif
+#      ifdef ldiv
+#        undef ldiv
+#      endif
+#      ifdef lldiv
+#        undef lldiv
+#      endif
 
 // MSVCRT already has the correct prototype in <stdlib.h> if __cplusplus is defined
-#    if !defined(_LIBCPP_MSVCRT)
+#      if !defined(_LIBCPP_MSVCRT)
 inline _LIBCPP_HIDE_FROM_ABI ldiv_t div(long __x, long __y) _NOEXCEPT { return ::ldiv(__x, __y); }
-#      if !(defined(__FreeBSD__) && !defined(__LONG_LONG_SUPPORTED))
+#        if !(defined(__FreeBSD__) && !defined(__LONG_LONG_SUPPORTED))
 inline _LIBCPP_HIDE_FROM_ABI lldiv_t div(long long __x, long long __y) _NOEXCEPT { return ::lldiv(__x, __y); }
-#      endif
-#    endif // _LIBCPP_MSVCRT
+#        endif
+#      endif // _LIBCPP_MSVCRT
 } // extern "C++"
-#  endif   // __cplusplus
+#    endif   // __cplusplus
+#  endif     // 0
 
 #endif // _LIBCPP_STDLIB_H
diff --git a/libcxx/include/stop_token b/libcxx/include/stop_token
index 58e3649c07056..6a6d1330248c7 100644
--- a/libcxx/include/stop_token
+++ b/libcxx/include/stop_token
@@ -31,27 +31,30 @@ namespace std {
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_THREADS
+#  if _LIBCPP_HAS_THREADS
 
-#  if _LIBCPP_STD_VER >= 20
-#    include <__stop_token/stop_callback.h>
-#    include <__stop_token/stop_source.h>
-#    include <__stop_token/stop_token.h>
-#  endif
+#    if _LIBCPP_STD_VER >= 20
+#      include <__stop_token/stop_callback.h>
+#      include <__stop_token/stop_source.h>
+#      include <__stop_token/stop_token.h>
+#    endif
 
-#  include <version>
+#    include <version>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <iosfwd>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <iosfwd>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STOP_TOKEN
diff --git a/libcxx/include/streambuf b/libcxx/include/streambuf
index 3359c1b5bb8e9..12f6b319f70e9 100644
--- a/libcxx/include/streambuf
+++ b/libcxx/include/streambuf
@@ -107,26 +107,28 @@ protected:
 
 */
 
-#include <__config>
-
-#if _LIBCPP_HAS_LOCALIZATION
-
-#  include <__assert>
-#  include <__fwd/streambuf.h>
-#  include <__locale>
-#  include <__type_traits/is_same.h>
-#  include <__utility/is_valid_range.h>
-#  include <climits>
-#  include <ios>
-#  include <iosfwd>
-#  include <version>
-
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#if 0
+#else // 0
+#  include <__config>
+
+#  if _LIBCPP_HAS_LOCALIZATION
+
+#    include <__assert>
+#    include <__fwd/streambuf.h>
+#    include <__locale>
+#    include <__type_traits/is_same.h>
+#    include <__utility/is_valid_range.h>
+#    include <climits>
+#    include <ios>
+#    include <iosfwd>
+#    include <version>
+
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -433,18 +435,19 @@ typename basic_streambuf<_CharT, _Traits>::int_type basic_streambuf<_CharT, _Tra
 
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf<char>;
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf<wchar_t>;
-#  endif
+#    endif
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_LOCALIZATION
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstdint>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstdint>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STREAMBUF
diff --git a/libcxx/include/string b/libcxx/include/string
index bf7fc3c37ecd7..17bf4b3b98bf3 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -586,103 +586,105 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 
 // clang-format on
 
-#include <__algorithm/max.h>
-#include <__algorithm/min.h>
-#include <__algorithm/remove.h>
-#include <__algorithm/remove_if.h>
-#include <__assert>
-#include <__config>
-#include <__debug_utils/sanitizers.h>
-#include <__format/enable_insertable.h>
-#include <__functional/hash.h>
-#include <__functional/unary_function.h>
-#include <__fwd/string.h>
-#include <__ios/fpos.h>
-#include <__iterator/bounded_iter.h>
-#include <__iterator/distance.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/wrap_iter.h>
-#include <__memory/addressof.h>
-#include <__memory/allocate_at_least.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/construct_at.h>
-#include <__memory/noexcept_move_assign_container.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/swap_allocator.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__ranges/access.h>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__ranges/size.h>
-#include <__string/char_traits.h>
-#include <__string/extern_template_lists.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_standard_layout.h>
-#include <__type_traits/is_trivial.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/void_t.h>
-#include <__utility/auto_cast.h>
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/is_pointer_in_range.h>
-#include <__utility/move.h>
-#include <__utility/scope_guard.h>
-#include <__utility/swap.h>
-#include <__utility/unreachable.h>
-#include <climits>
-#include <cstdio> // EOF
-#include <cstring>
-#include <limits>
-#include <stdexcept>
-#include <string_view>
-#include <version>
-
-#if _LIBCPP_HAS_WIDE_CHARACTERS
-#  include <cwchar>
-#endif
+#if 0
+#else // 0
+#  include <__algorithm/max.h>
+#  include <__algorithm/min.h>
+#  include <__algorithm/remove.h>
+#  include <__algorithm/remove_if.h>
+#  include <__assert>
+#  include <__config>
+#  include <__debug_utils/sanitizers.h>
+#  include <__format/enable_insertable.h>
+#  include <__functional/hash.h>
+#  include <__functional/unary_function.h>
+#  include <__fwd/string.h>
+#  include <__ios/fpos.h>
+#  include <__iterator/bounded_iter.h>
+#  include <__iterator/distance.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/wrap_iter.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocate_at_least.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
+#  include <__memory/construct_at.h>
+#  include <__memory/noexcept_move_assign_container.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/swap_allocator.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__ranges/access.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__ranges/size.h>
+#  include <__string/char_traits.h>
+#  include <__string/extern_template_lists.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_standard_layout.h>
+#  include <__type_traits/is_trivial.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/void_t.h>
+#  include <__utility/auto_cast.h>
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/is_pointer_in_range.h>
+#  include <__utility/move.h>
+#  include <__utility/scope_guard.h>
+#  include <__utility/swap.h>
+#  include <__utility/unreachable.h>
+#  include <climits>
+#  include <cstdio> // EOF
+#  include <cstring>
+#  include <limits>
+#  include <stdexcept>
+#  include <string_view>
+#  include <version>
+
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    include <cwchar>
+#  endif
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [string.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
-#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#    define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
 // This macro disables AddressSanitizer (ASan) instrumentation for a specific function,
 // allowing memory accesses that would normally trigger ASan errors to proceed without crashing.
 // This is useful for accessing parts of objects memory, which should not be accessed,
 // such as unused bytes in short strings, that should never be accessed
 // by other parts of the program.
-#else
-#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
-#endif
+#  else
+#    define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -708,7 +710,7 @@ template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
 operator+(const basic_string<_CharT, _Traits, _Allocator>& __x, _CharT __y);
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
@@ -728,7 +730,7 @@ template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
 operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs, basic_string<_CharT, _Traits, _Allocator>&& __rhs);
 
-#endif
+#  endif
 
 extern template _LIBCPP_EXPORTED_FROM_ABI string operator+
     <char, char_traits<char>, allocator<char> >(char const*, string const&);
@@ -786,7 +788,7 @@ public:
   //
   // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
   // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
   // When compiling with AddressSanitizer (ASan), basic_string cannot be trivially
   // relocatable. Because the object's memory might be poisoned when its content
   // is kept inside objects memory (short string optimization), instead of in allocated
@@ -794,14 +796,14 @@ public:
   // the memory to avoid triggering false positives.
   // Therefore it's crucial to ensure the destructor is called.
   using __trivially_relocatable = void;
-#else
+#  else
   using __trivially_relocatable = __conditional_t<
       __libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
       basic_string,
       void>;
-#endif
+#  endif
 
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
     if (__libcpp_is_constant_evaluated())
       return __ptr;
@@ -820,10 +822,10 @@ public:
 
     return const_cast<const_pointer&>(__copy_ptr);
   }
-#  define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) __asan_volatile_wrapper(PTR)
-#else
-#  define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) PTR
-#endif
+#    define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) __asan_volatile_wrapper(PTR)
+#  else
+#    define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) PTR
+#  endif
 
   static_assert(!is_array<value_type>::value, "Character type of basic_string must not be an array");
   static_assert(is_standard_layout<value_type>::value, "Character type of basic_string must be standard-layout");
@@ -834,23 +836,23 @@ public:
                 "Allocator::value_type must be same type as value_type");
   static_assert(__check_valid_allocator<allocator_type>::value, "");
 
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
   // Users might provide custom allocators, and prior to C++20 we have no existing way to detect whether the allocator's
   // pointer type is contiguous (though it has to be by the Standard). Using the wrapper type ensures the iterator is
   // considered contiguous.
   typedef __bounded_iter<__wrap_iter<pointer> > iterator;
   typedef __bounded_iter<__wrap_iter<const_pointer> > const_iterator;
-#else
+#  else
   typedef __wrap_iter<pointer> iterator;
   typedef __wrap_iter<const_pointer> const_iterator;
-#endif
+#  endif
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
 private:
   static_assert(CHAR_BIT == 8, "This implementation assumes that one byte contains 8 bits");
 
-#ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
   struct __long {
     pointer __data_;
@@ -881,19 +883,19 @@ private:
   // This does not impact the short string representation, since we never need the MSB
   // for representing the size of a short string anyway.
 
-#  ifdef _LIBCPP_BIG_ENDIAN
+#    ifdef _LIBCPP_BIG_ENDIAN
   static const size_type __endian_factor = 2;
-#  else
+#    else
   static const size_type __endian_factor = 1;
-#  endif
+#    endif
 
-#else // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  else // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
-#  ifdef _LIBCPP_BIG_ENDIAN
+#    ifdef _LIBCPP_BIG_ENDIAN
   static const size_type __endian_factor = 1;
-#  else
+#    else
   static const size_type __endian_factor = 2;
-#  endif
+#    endif
 
   // Attribute 'packed' is used to keep the layout compatible with the
   // previous definition that did not use bit fields. This is because on
@@ -919,7 +921,7 @@ private:
     value_type __data_[__min_cap];
   };
 
-#endif // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  endif // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
   static_assert(sizeof(__short) == (sizeof(value_type) * (__min_cap + 1)), "__short has an unexpected size.");
 
@@ -969,7 +971,7 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator __make_iterator(pointer __p) {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
     // Bound the iterator according to the size (and not the capacity, unlike vector).
     //
     // By the Standard, string iterators are generally not guaranteed to stay valid when the container is modified,
@@ -980,21 +982,21 @@ private:
         std::__wrap_iter<pointer>(__p),
         std::__wrap_iter<pointer>(__get_pointer()),
         std::__wrap_iter<pointer>(__get_pointer() + size()));
-#else
+#  else
     return iterator(__p);
-#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+#  endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator __make_const_iterator(const_pointer __p) const {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
     // Bound the iterator according to the size (and not the capacity, unlike vector).
     return std::__make_bounded_iter(
         std::__wrap_iter<const_pointer>(__p),
         std::__wrap_iter<const_pointer>(__get_pointer()),
         std::__wrap_iter<const_pointer>(__get_pointer() + size()));
-#else
+#  else
     return const_iterator(__p);
-#endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
+#  endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
   }
 
 public:
@@ -1007,11 +1009,11 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const allocator_type& __a)
-#if _LIBCPP_STD_VER <= 14
+#  if _LIBCPP_STD_VER <= 14
       _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value)
-#else
+#  else
       _NOEXCEPT
-#endif
+#  endif
       : __rep_(), __alloc_(__a) {
     __annotate_new(0);
   }
@@ -1035,13 +1037,13 @@ public:
       __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str)
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
       _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
-#  else
+#    else
       _NOEXCEPT
-#  endif
+#    endif
       // Turning off ASan instrumentation for variable initialization with _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
       // does not work consistently during initialization of __r_, so we instead unpoison __str's memory manually first.
       // __str's memory needs to be unpoisoned only in the case where it's a short string.
@@ -1073,7 +1075,7 @@ public:
         __annotate_new(size());
     }
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s) {
@@ -1088,9 +1090,9 @@ public:
     __init(__s, traits_type::length(__s));
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   basic_string(nullptr_t) = delete;
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s, size_type __n) {
     _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "basic_string(const char*, n) detected nullptr");
@@ -1106,7 +1108,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(size_type __n, _CharT __c) { __init(__n, __c); }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string(
       basic_string&& __str, size_type __pos, const _Allocator& __alloc = _Allocator())
       : basic_string(std::move(__str), __pos, npos, __alloc) {}
@@ -1125,7 +1127,7 @@ public:
       __init(__str.data() + __pos, __len);
     }
   }
-#endif
+#  endif
 
   template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(size_type __n, _CharT __c, const _Allocator& __a)
@@ -1196,7 +1198,7 @@ public:
     __init(__first, __last);
   }
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string(
       from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
@@ -1207,9 +1209,9 @@ public:
       __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
     }
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(initializer_list<_CharT> __il) {
     __init(__il.begin(), __il.end());
   }
@@ -1218,7 +1220,7 @@ public:
       : __alloc_(__a) {
     __init(__il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() {
     __annotate_delete();
@@ -1242,7 +1244,7 @@ public:
     return assign(__sv);
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   operator=(basic_string&& __str) noexcept(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
     __move_assign(__str, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
@@ -1252,13 +1254,13 @@ public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(initializer_list<value_type> __il) {
     return assign(__il.begin(), __il.size());
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const value_type* __s) {
     return assign(__s);
   }
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   basic_string& operator=(nullptr_t) = delete;
-#endif
+#  endif
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(value_type __c);
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT {
@@ -1318,19 +1320,19 @@ public:
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __requested_capacity);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <class _Op>
   _LIBCPP_HIDE_FROM_ABI constexpr void resize_and_overwrite(size_type __n, _Op __op) {
     __resize_default_init(__n);
     __erase_to_end(std::move(__op)(data(), _LIBCPP_AUTO_CAST(__n)));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __resize_default_init(size_type __n);
 
-#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRING_RESERVE)
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRING_RESERVE)
   _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve() _NOEXCEPT { shrink_to_fit(); }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void clear() _NOEXCEPT;
 
@@ -1380,11 +1382,11 @@ public:
     return *this;
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(initializer_list<value_type> __il) {
     return append(__il);
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const basic_string& __str) {
     return append(__str.data(), __str.size());
@@ -1429,19 +1431,19 @@ public:
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   append(_ForwardIterator __first, _ForwardIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string& append_range(_Range&& __range) {
     insert_range(end(), std::forward<_Range>(__range));
     return *this;
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(initializer_list<value_type> __il) {
     return append(__il.begin(), __il.size());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(value_type __c);
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back();
@@ -1473,7 +1475,7 @@ public:
     return assign(__sv.data(), __sv.size());
   }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI constexpr void __move_assign(basic_string&& __str, size_type __pos, size_type __len) {
     // Pilfer the allocation from __str.
     _LIBCPP_ASSERT_INTERNAL(__alloc_ == __str.__alloc_, "__move_assign called with wrong allocator");
@@ -1494,18 +1496,18 @@ public:
       __annotate_shrink(__old_sz);
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const basic_string& __str) {
     return *this = __str;
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   assign(basic_string&& __str) noexcept(__noexcept_move_assign_container<_Allocator, __alloc_traits>::value) {
     *this = std::move(__str);
     return *this;
   }
-#endif
+#  endif
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const basic_string& __str, size_type __pos, size_type __n = npos);
 
   template <class _Tp,
@@ -1526,7 +1528,7 @@ public:
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   assign(_ForwardIterator __first, _ForwardIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string& assign_range(_Range&& __range) {
     if constexpr (__string_is_trivial_iterator<ranges::iterator_t<_Range>>::value &&
@@ -1540,13 +1542,13 @@ public:
 
     return *this;
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(initializer_list<value_type> __il) {
     return assign(__il.begin(), __il.size());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   insert(size_type __pos1, const basic_string& __str) {
@@ -1574,7 +1576,7 @@ public:
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, size_type __n, value_type __c);
   _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __pos, value_type __c);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
     if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
@@ -1586,7 +1588,7 @@ public:
       return insert(__position, __temp.data(), __temp.data() + __temp.size());
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
   insert(const_iterator __pos, size_type __n, value_type __c) {
@@ -1603,12 +1605,12 @@ public:
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
   insert(const_iterator __pos, _ForwardIterator __first, _ForwardIterator __last);
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
   insert(const_iterator __pos, initializer_list<value_type> __il) {
     return insert(__pos, __il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& erase(size_type __pos = 0, size_type __n = npos);
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator erase(const_iterator __pos);
@@ -1673,30 +1675,30 @@ public:
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   replace(const_iterator __i1, const_iterator __i2, _InputIterator __j1, _InputIterator __j2);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string&
   replace_with_range(const_iterator __i1, const_iterator __i2, _Range&& __range) {
     basic_string __temp(from_range, std::forward<_Range>(__range), __alloc_);
     return replace(__i1, __i2, __temp);
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   replace(const_iterator __i1, const_iterator __i2, initializer_list<value_type> __il) {
     return replace(__i1, __i2, __il.begin(), __il.end());
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type copy(value_type* __s, size_type __n, size_type __pos = 0) const;
 
-#if _LIBCPP_STD_VER <= 20
+#  if _LIBCPP_STD_VER <= 20
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string
   substr(size_type __pos = 0, size_type __n = npos) const {
     return basic_string(*this, __pos, __n);
   }
-#else
+#  else
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) const& {
     return basic_string(*this, __pos, __n);
   }
@@ -1704,24 +1706,24 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) && {
     return basic_string(std::move(*this), __pos, __n);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(basic_string& __str)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
-#else
+#  else
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* c_str() const _NOEXCEPT { return data(); }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* data() const _NOEXCEPT {
     return std::__to_address(__get_pointer());
   }
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 value_type* data() _NOEXCEPT {
     return std::__to_address(__get_pointer());
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
     return __alloc_;
@@ -1834,7 +1836,7 @@ public:
   _LIBCPP_CONSTEXPR_SINCE_CXX20 int
   compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const;
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(__self_view __sv) const noexcept {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).starts_with(__sv);
   }
@@ -1858,9 +1860,9 @@ public:
   constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* __s) const noexcept {
     return ends_with(__self_view(__s));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(__self_view __sv) const noexcept {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__sv);
   }
@@ -1872,7 +1874,7 @@ public:
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* __s) const {
     return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__s);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __invariants() const;
 
@@ -1895,15 +1897,15 @@ private:
   }
 
   static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __begin_lifetime(pointer __begin, size_type __n) {
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
     if (__libcpp_is_constant_evaluated()) {
       for (size_type __i = 0; __i != __n; ++__i)
         std::construct_at(std::addressof(__begin[__i]));
     }
-#else
+#  else
     (void)__begin;
     (void)__n;
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
   }
 
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI static bool __fits_in_sso(size_type __sz) { return __sz < __min_cap; }
@@ -1919,7 +1921,7 @@ private:
   template <class _ForwardIter, class _Sent>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static value_type*
   __copy_non_overlapping_range(_ForwardIter __first, _Sent __last, value_type* __dest) {
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value &&
                   is_same<value_type, __iter_value_type<_ForwardIter>>::value && is_same<_ForwardIter, _Sent>::value) {
       _LIBCPP_ASSERT_INTERNAL(
@@ -1928,7 +1930,7 @@ private:
       traits_type::copy(__dest, std::__to_address(__first), __last - __first);
       return __dest + (__last - __first);
     }
-#endif
+#  endif
 
     for (; __first != __last; ++__first)
       traits_type::assign(*__dest++, *__first);
@@ -2039,45 +2041,45 @@ private:
   __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
     (void)__old_mid;
     (void)__new_mid;
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
-#  if defined(__APPLE__)
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#    if defined(__APPLE__)
     // TODO: remove after addressing issue #96099 (https://github.com/llvm/llvm-project/issues/96099)
     if (!__is_long())
       return;
-#  endif
+#    endif
     std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity() + 1, __old_mid, __new_mid);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
     (void)__n;
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
-#endif
+#  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
     (void)__old_size;
-#if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
-#endif
+#  endif
   }
 
   template <size_type __a>
@@ -2126,9 +2128,9 @@ private:
   __init_with_size(_InputIterator __first, _Sentinel __last, size_type __sz);
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20
-#if _LIBCPP_ABI_VERSION >= 2 //  We want to use the function in the dylib in ABIv1
+#  if _LIBCPP_ABI_VERSION >= 2 //  We want to use the function in the dylib in ABIv1
   _LIBCPP_HIDE_FROM_ABI
-#endif
+#  endif
   _LIBCPP_DEPRECATED_("use __grow_by_without_replace") void __grow_by(
       size_type __old_cap,
       size_type __delta_cap,
@@ -2198,17 +2200,17 @@ private:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
   __copy_assign_alloc(const basic_string&, false_type) _NOEXCEPT {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
   __move_assign(basic_string& __str, false_type) noexcept(__alloc_traits::is_always_equal::value);
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
   __move_assign(basic_string& __str, true_type)
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
       noexcept;
-#  else
+#    else
       noexcept(is_nothrow_move_assignable<allocator_type>::value);
+#    endif
 #  endif
-#endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string& __str)
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_move_assignment::value ||
@@ -2271,10 +2273,10 @@ private:
   friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(value_type, const basic_string&);
   friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const basic_string&, const value_type*);
   friend _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string operator+ <>(const basic_string&, value_type);
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
   friend constexpr basic_string operator+ <>(const basic_string&, type_identity_t<__self_view>);
   friend constexpr basic_string operator+ <>(type_identity_t<__self_view>, const basic_string&);
-#endif
+#  endif
 
   template <class _CharT2, class _Traits2, class _Allocator2>
   friend inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
@@ -2283,21 +2285,21 @@ private:
 
 // These declarations must appear before any functions are implicitly used
 // so that they have the correct visibility specifier.
-#define _LIBCPP_DECLARE(...) extern template __VA_ARGS__;
-#ifdef _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
+#  define _LIBCPP_DECLARE(...) extern template __VA_ARGS__;
+#  ifdef _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
 _LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, char)
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 _LIBCPP_STRING_UNSTABLE_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, wchar_t)
-#  endif
-#else
+#    endif
+#  else
 _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, char)
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, wchar_t)
+#    endif
 #  endif
-#endif
-#undef _LIBCPP_DECLARE
+#  undef _LIBCPP_DECLARE
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _CharT     = __iter_value_type<_InputIterator>,
           class _Allocator = allocator<_CharT>,
@@ -2320,15 +2322,15 @@ template <class _CharT,
           class _Sz        = typename allocator_traits<_Allocator>::size_type >
 basic_string(basic_string_view<_CharT, _Traits>, _Sz, _Sz, const _Allocator& = _Allocator())
     -> basic_string<_CharT, _Traits, _Allocator>;
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Allocator = allocator<ranges::range_value_t<_Range>>,
           class            = enable_if_t<__is_allocator<_Allocator>::value> >
 basic_string(from_range_t, _Range&&, _Allocator = _Allocator())
     -> basic_string<ranges::range_value_t<_Range>, char_traits<ranges::range_value_t<_Range>>, _Allocator>;
-#endif
+#  endif
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void
@@ -2440,19 +2442,19 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator _
   __rep_ = __rep();
   __annotate_new(0);
 
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __first != __last; ++__first)
       push_back(*__first);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __annotate_delete();
     if (__is_long())
       __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2487,18 +2489,18 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __fir
     __set_long_size(__sz);
   }
 
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__p));
     traits_type::assign(*__end, value_type());
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     if (__is_long())
       __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   __annotate_new(__sz);
 }
 
@@ -2544,9 +2546,9 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__
 // not removed or changed to avoid breaking the ABI.
 template <class _CharT, class _Traits, class _Allocator>
 void _LIBCPP_CONSTEXPR_SINCE_CXX20
-#if _LIBCPP_ABI_VERSION >= 2 // We want to use the function in the dylib in ABIv1
+#  if _LIBCPP_ABI_VERSION >= 2 // We want to use the function in the dylib in ABIv1
 _LIBCPP_HIDE_FROM_ABI
-#endif
+#  endif
 _LIBCPP_DEPRECATED_("use __grow_by_without_replace") basic_string<_CharT, _Traits, _Allocator>::__grow_by(
     size_type __old_cap,
     size_type __delta_cap,
@@ -2701,7 +2703,7 @@ basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
   return *this;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__move_assign(
@@ -2715,22 +2717,22 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
 basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, true_type)
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
     noexcept
-#  else
+#    else
     noexcept(is_nothrow_move_assignable<allocator_type>::value)
-#  endif
+#    endif
 {
   __annotate_delete();
   if (__is_long()) {
     __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
-#  if _LIBCPP_STD_VER <= 14
+#    if _LIBCPP_STD_VER <= 14
     if (!is_nothrow_move_assignable<allocator_type>::value) {
       __set_short_size(0);
       traits_type::assign(__get_short_pointer()[0], value_type());
       __annotate_new(0);
     }
-#  endif
+#    endif
   }
   size_type __str_old_size = __str.size();
   bool __str_was_short     = !__str.__is_long();
@@ -2759,7 +2761,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr
     __annotate_new(__get_short_size());
 }
 
-#endif
+#  endif
 
 template <class _CharT, class _Traits, class _Allocator>
 template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
@@ -3396,9 +3398,9 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
       // Shrink
       // - called from shrink_to_fit should not throw.
       // - called from reserve may throw but is not required to.
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
         auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
 
         // The Standard mandates shrink_to_fit() does not increase the capacity.
@@ -3410,11 +3412,11 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
         }
         __new_data        = __allocation.ptr;
         __target_capacity = __allocation.count - 1;
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return;
       }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     }
     __begin_lifetime(__new_data, __target_capacity + 1);
     __now_long = true;
@@ -3461,11 +3463,11 @@ basic_string<_CharT, _Traits, _Allocator>::copy(value_type* __s, size_type __n,
 
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str)
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
-#else
+#  else
     _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>)
-#endif
+#  endif
 {
   _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
       __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value ||
@@ -3840,12 +3842,12 @@ template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
 operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
            const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   return basic_string_view<_CharT, _Traits>(__lhs) == basic_string_view<_CharT, _Traits>(__rhs);
-#else
+#  else
   size_t __lhs_sz = __lhs.size();
   return __lhs_sz == __rhs.size() && _Traits::compare(__lhs.data(), __rhs.data(), __lhs_sz) == 0;
-#endif
+#  endif
 }
 
 template <class _Allocator>
@@ -3858,7 +3860,7 @@ operator==(const basic_string<char, char_traits<char>, _Allocator>& __lhs,
   return char_traits<char>::compare(__lhs.data(), __rhs.data(), __sz) == 0;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool
 operator==(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
@@ -3869,7 +3871,7 @@ operator==(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>&
     return false;
   return __rhs.compare(0, _String::npos, __lhs, __lhs_len) == 0;
 }
-#endif // _LIBCPP_STD_VER <= 17
+#  endif // _LIBCPP_STD_VER <= 17
 
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
@@ -3888,7 +3890,7 @@ operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT*
   return __lhs.compare(0, _String::npos, __rhs, __rhs_len) == 0;
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
@@ -3902,7 +3904,7 @@ operator<=>(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT
   return basic_string_view<_CharT, _Traits>(__lhs) <=> basic_string_view<_CharT, _Traits>(__rhs);
 }
 
-#else  // _LIBCPP_STD_VER >= 20
+#  else  // _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
@@ -4001,7 +4003,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool
 operator>=(const _CharT* __lhs, const basic_string<_CharT, _Traits, _Allocator>& __rhs) _NOEXCEPT {
   return !(__lhs < __rhs);
 }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 // operator +
 
@@ -4084,7 +4086,7 @@ operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs, _CharT __rhs)
   return __r;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>
@@ -4130,9 +4132,9 @@ operator+(basic_string<_CharT, _Traits, _Allocator>&& __lhs, _CharT __rhs) {
   return std::move(__lhs);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
-#if _LIBCPP_STD_VER >= 26
+#  if _LIBCPP_STD_VER >= 26
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI constexpr basic_string<_CharT, _Traits, _Allocator>
@@ -4184,7 +4186,7 @@ operator+(type_identity_t<basic_string_view<_CharT, _Traits>> __lhs,
   return std::move(__rhs);
 }
 
-#endif // _LIBCPP_STD_VER >= 26
+#  endif // _LIBCPP_STD_VER >= 26
 
 // swap
 
@@ -4215,7 +4217,7 @@ _LIBCPP_EXPORTED_FROM_ABI string to_string(float __val);
 _LIBCPP_EXPORTED_FROM_ABI string to_string(double __val);
 _LIBCPP_EXPORTED_FROM_ABI string to_string(long double __val);
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 _LIBCPP_EXPORTED_FROM_ABI int stoi(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
 _LIBCPP_EXPORTED_FROM_ABI long stol(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
 _LIBCPP_EXPORTED_FROM_ABI unsigned long stoul(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
@@ -4235,7 +4237,7 @@ _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long long __val);
 _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(float __val);
 _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(double __val);
 _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long double __val);
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
+#  endif // _LIBCPP_HAS_WIDE_CHARACTERS
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_TEMPLATE_DATA_VIS const typename basic_string<_CharT, _Traits, _Allocator>::size_type
@@ -4252,10 +4254,10 @@ struct __string_hash : public __unary_function<basic_string<_CharT, char_traits<
 template <class _Allocator>
 struct hash<basic_string<char, char_traits<char>, _Allocator> > : __string_hash<char, _Allocator> {};
 
-#if _LIBCPP_HAS_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 template <class _Allocator>
 struct hash<basic_string<char8_t, char_traits<char8_t>, _Allocator> > : __string_hash<char8_t, _Allocator> {};
-#endif
+#  endif
 
 template <class _Allocator>
 struct hash<basic_string<char16_t, char_traits<char16_t>, _Allocator> > : __string_hash<char16_t, _Allocator> {};
@@ -4263,10 +4265,10 @@ struct hash<basic_string<char16_t, char_traits<char16_t>, _Allocator> > : __stri
 template <class _Allocator>
 struct hash<basic_string<char32_t, char_traits<char32_t>, _Allocator> > : __string_hash<char32_t, _Allocator> {};
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <class _Allocator>
 struct hash<basic_string<wchar_t, char_traits<wchar_t>, _Allocator> > : __string_hash<wchar_t, _Allocator> {};
-#endif
+#  endif
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
@@ -4292,7 +4294,7 @@ template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 getline(basic_istream<_CharT, _Traits>&& __is, basic_string<_CharT, _Traits, _Allocator>& __str);
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _CharT, class _Traits, class _Allocator, class _Up>
 inline _LIBCPP_HIDE_FROM_ABI typename basic_string<_CharT, _Traits, _Allocator>::size_type
 erase(basic_string<_CharT, _Traits, _Allocator>& __str, const _Up& __v) {
@@ -4308,9 +4310,9 @@ erase_if(basic_string<_CharT, _Traits, _Allocator>& __str, _Predicate __pred) {
   __str.erase(std::remove_if(__str.begin(), __str.end(), __pred), __str.end());
   return __old_size - __str.size();
 }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 // Literal suffixes for basic_string [basic.string.literals]
 inline namespace literals {
 inline namespace string_literals {
@@ -4319,18 +4321,18 @@ operator""s(const char* __str, size_t __len) {
   return basic_string<char>(__str, __len);
 }
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<wchar_t>
 operator""s(const wchar_t* __str, size_t __len) {
   return basic_string<wchar_t>(__str, __len);
 }
-#  endif
+#    endif
 
-#  if _LIBCPP_HAS_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string<char8_t> operator""s(const char8_t* __str, size_t __len) {
   return basic_string<char8_t>(__str, __len);
 }
-#  endif
+#    endif
 
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char16_t>
 operator""s(const char16_t* __str, size_t __len) {
@@ -4344,30 +4346,31 @@ operator""s(const char32_t* __str, size_t __len) {
 } // namespace string_literals
 } // namespace literals
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 template <>
 inline constexpr bool __format::__enable_insertable<std::basic_string<char>> = true;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 inline constexpr bool __format::__enable_insertable<std::basic_string<wchar_t>> = true;
+#      endif
 #    endif
-#  endif
 
-#endif
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iterator>
-#  include <new>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iterator>
+#    include <new>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STRING
diff --git a/libcxx/include/string.h b/libcxx/include/string.h
index ae10d5ad2c76f..58a4bcbc3e704 100644
--- a/libcxx/include/string.h
+++ b/libcxx/include/string.h
@@ -51,24 +51,27 @@ size_t strlen(const char* s);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if __has_include_next(<string.h>)
-#  include_next <string.h>
-#endif
+#  if __has_include_next(<string.h>)
+#    include_next <string.h>
+#  endif
 
 // MSVCRT, GNU libc and its derivates may already have the correct prototype in
 // <string.h>. This macro can be defined by users if their C library provides
 // the right signature.
-#if defined(__CORRECT_ISO_CPP_STRING_H_PROTO) || defined(_LIBCPP_MSVCRT) || defined(_STRING_H_CPLUSPLUS_98_CONFORMANCE_)
-#  define _LIBCPP_STRING_H_HAS_CONST_OVERLOADS
-#endif
+#  if defined(__CORRECT_ISO_CPP_STRING_H_PROTO) || defined(_LIBCPP_MSVCRT) ||                                          \
+      defined(_STRING_H_CPLUSPLUS_98_CONFORMANCE_)
+#    define _LIBCPP_STRING_H_HAS_CONST_OVERLOADS
+#  endif
 
-#if defined(__cplusplus) && !defined(_LIBCPP_STRING_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
+#  if defined(__cplusplus) && !defined(_LIBCPP_STRING_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
 extern "C++" {
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD const char* strchr(const char* __s, int __c) {
   return __builtin_strchr(__s, __c);
@@ -105,6 +108,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD char* strstr(char* __s1,
   return __builtin_strstr(__s1, __s2);
 }
 } // extern "C++"
-#endif
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STRING_H
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 6e05844b72cad..27b9f152ea290 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -205,60 +205,62 @@ namespace std {
 
 // clang-format on
 
-#include <__algorithm/min.h>
-#include <__assert>
-#include <__config>
-#include <__cstddef/nullptr_t.h>
-#include <__cstddef/ptrdiff_t.h>
-#include <__cstddef/size_t.h>
-#include <__functional/hash.h>
-#include <__functional/unary_function.h>
-#include <__fwd/ostream.h>
-#include <__fwd/string.h>
-#include <__fwd/string_view.h>
-#include <__iterator/bounded_iter.h>
-#include <__iterator/concepts.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/reverse_iterator.h>
-#include <__iterator/wrap_iter.h>
-#include <__memory/pointer_traits.h>
-#include <__ranges/concepts.h>
-#include <__ranges/data.h>
-#include <__ranges/enable_borrowed_range.h>
-#include <__ranges/enable_view.h>
-#include <__ranges/size.h>
-#include <__string/char_traits.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_standard_layout.h>
-#include <__type_traits/is_trivial.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__type_traits/type_identity.h>
-#include <iosfwd>
-#include <limits>
-#include <stdexcept>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/min.h>
+#  include <__assert>
+#  include <__config>
+#  include <__cstddef/nullptr_t.h>
+#  include <__cstddef/ptrdiff_t.h>
+#  include <__cstddef/size_t.h>
+#  include <__functional/hash.h>
+#  include <__functional/unary_function.h>
+#  include <__fwd/ostream.h>
+#  include <__fwd/string.h>
+#  include <__fwd/string_view.h>
+#  include <__iterator/bounded_iter.h>
+#  include <__iterator/concepts.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/reverse_iterator.h>
+#  include <__iterator/wrap_iter.h>
+#  include <__memory/pointer_traits.h>
+#  include <__ranges/concepts.h>
+#  include <__ranges/data.h>
+#  include <__ranges/enable_borrowed_range.h>
+#  include <__ranges/enable_view.h>
+#  include <__ranges/size.h>
+#  include <__string/char_traits.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_standard_layout.h>
+#  include <__type_traits/is_trivial.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__type_traits/type_identity.h>
+#  include <iosfwd>
+#  include <limits>
+#  include <stdexcept>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [string.view.synop]
-#include <compare>
+#  include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -283,13 +285,13 @@ public:
   using const_pointer   = const _CharT*;
   using reference       = _CharT&;
   using const_reference = const _CharT&;
-#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS)
+#  if defined(_LIBCPP_ABI_BOUNDED_ITERATORS)
   using const_iterator = __bounded_iter<const_pointer>;
-#elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW)
+#  elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW)
   using const_iterator = __wrap_iter<const_pointer>;
-#else
+#  else
   using const_iterator = const_pointer;
-#endif
+#  endif
   using iterator                                = const_iterator;
   using const_reverse_iterator                  = std::reverse_iterator<const_iterator>;
   using reverse_iterator                        = const_reverse_iterator;
@@ -313,7 +315,7 @@ public:
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s, size_type __len) _NOEXCEPT
       : __data_(__s),
         __size_(__len) {
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
     // Allocations must fit in `ptrdiff_t` for pointer arithmetic to work. If `__len` exceeds it, the input
     // range could not have been valid. Most likely the caller underflowed some arithmetic and inadvertently
     // passed in a negative length.
@@ -322,10 +324,10 @@ public:
         "string_view::string_view(_CharT *, size_t): length does not fit in difference_type");
     _LIBCPP_ASSERT_NON_NULL(
         __len == 0 || __s != nullptr, "string_view::string_view(_CharT *, size_t): received nullptr");
-#endif
+#  endif
   }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <contiguous_iterator _It, sized_sentinel_for<_It> _End>
     requires(is_same_v<iter_value_t<_It>, _CharT> && !is_convertible_v<_End, size_type>)
   constexpr _LIBCPP_HIDE_FROM_ABI basic_string_view(_It __begin, _End __end)
@@ -333,9 +335,9 @@ public:
     _LIBCPP_ASSERT_VALID_INPUT_RANGE(
         (__end - __begin) >= 0, "std::string_view::string_view(iterator, sentinel) received invalid range");
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <class _Range>
     requires(!is_same_v<remove_cvref_t<_Range>, basic_string_view> && ranges::contiguous_range<_Range> &&
              ranges::sized_range<_Range> && is_same_v<ranges::range_value_t<_Range>, _CharT> &&
@@ -343,14 +345,14 @@ public:
              (!requires(remove_cvref_t<_Range>& __d) { __d.operator std::basic_string_view<_CharT, _Traits>(); }))
   constexpr explicit _LIBCPP_HIDE_FROM_ABI basic_string_view(_Range&& __r)
       : __data_(ranges::data(__r)), __size_(ranges::size(__r)) {}
-#endif // _LIBCPP_STD_VER >= 23
+#  endif // _LIBCPP_STD_VER >= 23
 
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s)
       : __data_(__s), __size_(std::__char_traits_length_checked<_Traits>(__s)) {}
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   basic_string_view(nullptr_t) = delete;
-#endif
+#  endif
 
   // [string.view.iterators], iterators
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return cbegin(); }
@@ -358,19 +360,19 @@ public:
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return cend(); }
 
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data(), data(), data() + size());
-#else
+#  else
     return const_iterator(__data_);
-#endif
+#  endif
   }
 
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
-#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
+#  ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
     return std::__make_bounded_iter(data() + size(), data(), data() + size());
-#else
+#  else
     return const_iterator(__data_ + __size_);
-#endif
+#  endif
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
@@ -645,7 +647,7 @@ public:
         data(), size(), __s, __pos, traits_type::length(__s));
   }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(basic_string_view __s) const noexcept {
     return size() >= __s.size() && compare(0, __s.size(), __s) == 0;
   }
@@ -669,15 +671,15 @@ public:
   constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* __s) const noexcept {
     return ends_with(basic_string_view(__s));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(basic_string_view __sv) const noexcept { return find(__sv) != npos; }
 
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept { return find(__c) != npos; }
 
   constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* __s) const { return find(__s) != npos; }
-#endif
+#  endif
 
 private:
   struct __assume_valid {};
@@ -698,29 +700,29 @@ private:
 };
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(basic_string_view);
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _CharT, class _Traits>
 inline constexpr bool ranges::enable_view<basic_string_view<_CharT, _Traits>> = true;
 
 template <class _CharT, class _Traits>
 inline constexpr bool ranges::enable_borrowed_range<basic_string_view<_CharT, _Traits> > = true;
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
 // [string.view.deduct]
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <contiguous_iterator _It, sized_sentinel_for<_It> _End>
 basic_string_view(_It, _End) -> basic_string_view<iter_value_t<_It>>;
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
 template <ranges::contiguous_range _Range>
 basic_string_view(_Range) -> basic_string_view<ranges::range_value_t<_Range>>;
-#endif
+#  endif
 
 // [string.view.comparison]
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(basic_string_view<_CharT, _Traits> __lhs,
@@ -743,7 +745,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(basic_string_view<_CharT, _Trai
   }
 }
 
-#else
+#  else
 
 // operator ==
 
@@ -886,7 +888,7 @@ operator>=(__type_identity_t<basic_string_view<_CharT, _Traits> > __lhs,
   return __lhs.compare(__rhs) >= 0;
 }
 
-#endif //  _LIBCPP_STD_VER >= 20
+#  endif //  _LIBCPP_STD_VER >= 20
 
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
@@ -903,10 +905,10 @@ struct __string_view_hash : public __unary_function<basic_string_view<_CharT, ch
 template <>
 struct hash<basic_string_view<char, char_traits<char> > > : __string_view_hash<char> {};
 
-#if _LIBCPP_HAS_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 template <>
 struct hash<basic_string_view<char8_t, char_traits<char8_t> > > : __string_view_hash<char8_t> {};
-#endif
+#  endif
 
 template <>
 struct hash<basic_string_view<char16_t, char_traits<char16_t> > > : __string_view_hash<char16_t> {};
@@ -914,31 +916,31 @@ struct hash<basic_string_view<char16_t, char_traits<char16_t> > > : __string_vie
 template <>
 struct hash<basic_string_view<char32_t, char_traits<char32_t> > > : __string_view_hash<char32_t> {};
 
-#if _LIBCPP_HAS_WIDE_CHARACTERS
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
 template <>
 struct hash<basic_string_view<wchar_t, char_traits<wchar_t> > > : __string_view_hash<wchar_t> {};
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
 inline namespace literals {
 inline namespace string_view_literals {
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char> operator""sv(const char* __str, size_t __len) noexcept {
   return basic_string_view<char>(__str, __len);
 }
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<wchar_t>
 operator""sv(const wchar_t* __str, size_t __len) noexcept {
   return basic_string_view<wchar_t>(__str, __len);
 }
-#  endif
+#    endif
 
-#  if _LIBCPP_HAS_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char8_t>
 operator""sv(const char8_t* __str, size_t __len) noexcept {
   return basic_string_view<char8_t>(__str, __len);
 }
-#  endif
+#    endif
 
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char16_t>
 operator""sv(const char16_t* __str, size_t __len) noexcept {
@@ -951,17 +953,18 @@ operator""sv(const char32_t* __str, size_t __len) noexcept {
 }
 } // namespace string_view_literals
 } // namespace literals
-#endif
+#  endif
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iterator>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iterator>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_STRING_VIEW
diff --git a/libcxx/include/strstream b/libcxx/include/strstream
index b263fe1a76ad2..81dbdf14d9e6a 100644
--- a/libcxx/include/strstream
+++ b/libcxx/include/strstream
@@ -129,31 +129,33 @@ private:
 
 */
 
-#include <__config>
-#include <__ostream/basic_ostream.h>
-#include <istream>
-#include <streambuf>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__ostream/basic_ostream.h>
+#  include <istream>
+#  include <streambuf>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI strstreambuf : public streambuf {
 public:
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI strstreambuf() : strstreambuf(0) {}
   explicit strstreambuf(streamsize __alsize);
-#  else
+#    else
   explicit strstreambuf(streamsize __alsize = 0);
-#  endif
+#    endif
   strstreambuf(void* (*__palloc)(size_t), void (*__pfree)(void*));
   strstreambuf(char* __gnext, streamsize __n, char* __pbeg = nullptr);
   strstreambuf(const char* __gnext, streamsize __n);
@@ -163,10 +165,10 @@ public:
   strstreambuf(unsigned char* __gnext, streamsize __n, unsigned char* __pbeg = nullptr);
   strstreambuf(const unsigned char* __gnext, streamsize __n);
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI strstreambuf(strstreambuf&& __rhs);
   _LIBCPP_HIDE_FROM_ABI strstreambuf& operator=(strstreambuf&& __rhs);
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
   ~strstreambuf() override;
 
@@ -200,7 +202,7 @@ private:
   void __init(char* __gnext, streamsize __n, char* __pbeg);
 };
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
 
 inline _LIBCPP_HIDE_FROM_ABI strstreambuf::strstreambuf(strstreambuf&& __rhs)
     : streambuf(__rhs),
@@ -229,7 +231,7 @@ inline _LIBCPP_HIDE_FROM_ABI strstreambuf& strstreambuf::operator=(strstreambuf&
   return *this;
 }
 
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
 class _LIBCPP_DEPRECATED _LIBCPP_EXPORTED_FROM_ABI istrstream : public istream {
 public:
@@ -238,7 +240,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI istrstream(const char* __s, streamsize __n) : istream(&__sb_), __sb_(__s, __n) {}
   _LIBCPP_HIDE_FROM_ABI istrstream(char* __s, streamsize __n) : istream(&__sb_), __sb_(__s, __n) {}
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI istrstream(istrstream&& __rhs) // extension
       : istream(std::move(static_cast<istream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
     istream::set_rdbuf(&__sb_);
@@ -249,7 +251,7 @@ public:
     istream::operator=(std::move(__rhs));
     return *this;
   }
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
   ~istrstream() override;
 
@@ -271,7 +273,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI ostrstream(char* __s, int __n, ios_base::openmode __mode = ios_base::out)
       : ostream(&__sb_), __sb_(__s, __n, __s + (__mode & ios::app ? std::strlen(__s) : 0)) {}
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI ostrstream(ostrstream&& __rhs) // extension
       : ostream(std::move(static_cast<ostream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
     ostream::set_rdbuf(&__sb_);
@@ -282,7 +284,7 @@ public:
     ostream::operator=(std::move(__rhs));
     return *this;
   }
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
   ~ostrstream() override;
 
@@ -313,7 +315,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI strstream(char* __s, int __n, ios_base::openmode __mode = ios_base::in | ios_base::out)
       : iostream(&__sb_), __sb_(__s, __n, __s + (__mode & ios::app ? std::strlen(__s) : 0)) {}
 
-#  ifndef _LIBCPP_CXX03_LANG
+#    ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI strstream(strstream&& __rhs) // extension
       : iostream(std::move(static_cast<iostream&>(__rhs))), __sb_(std::move(__rhs.__sb_)) {
     iostream::set_rdbuf(&__sb_);
@@ -324,7 +326,7 @@ public:
     iostream::operator=(std::move(__rhs));
     return *this;
   }
-#  endif // _LIBCPP_CXX03_LANG
+#    endif // _LIBCPP_CXX03_LANG
 
   ~strstream() override;
 
@@ -347,6 +349,7 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM) || defined(_LIBCPP_BUILDING_LIBRARY)
+#endif   // 0
 
 #endif // _LIBCPP_STRSTREAM
diff --git a/libcxx/include/syncstream b/libcxx/include/syncstream
index 2699a4b3a6fbb..f801132d436b4 100644
--- a/libcxx/include/syncstream
+++ b/libcxx/include/syncstream
@@ -117,37 +117,39 @@ namespace std {
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_LOCALIZATION
+#  if _LIBCPP_HAS_LOCALIZATION
 
-#  include <__mutex/lock_guard.h>
-#  include <__utility/move.h>
-#  include <ios>
-#  include <iosfwd> // required for declaration of default arguments
-#  include <streambuf>
-#  include <string>
+#    include <__mutex/lock_guard.h>
+#    include <__utility/move.h>
+#    include <ios>
+#    include <iosfwd> // required for declaration of default arguments
+#    include <streambuf>
+#    include <string>
 
-#  if _LIBCPP_HAS_THREADS
-#    include <map>
-#    include <shared_mutex>
-#  endif
+#    if _LIBCPP_HAS_THREADS
+#      include <map>
+#      include <shared_mutex>
+#    endif
 
 // standard-mandated includes
 
 // [syncstream.syn]
-#  include <ostream>
+#    include <ostream>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 _LIBCPP_PUSH_MACROS
-#  include <__undef_macros>
+#    include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#  if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+#    if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
 
 // [syncstream.syncbuf.overview]/1
 //   Class template basic_syncbuf stores character data written to it,
@@ -160,7 +162,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 //
 // This helper singleton is used to implement the required
 // synchronisation guarantees.
-#    if _LIBCPP_HAS_THREADS
+#      if _LIBCPP_HAS_THREADS
 class __wrapped_streambuf_mutex {
   _LIBCPP_HIDE_FROM_ABI __wrapped_streambuf_mutex() = default;
 
@@ -233,7 +235,7 @@ private:
     return __it;
   }
 };
-#    endif // _LIBCPP_HAS_THREADS
+#      endif // _LIBCPP_HAS_THREADS
 
 // basic_syncbuf
 
@@ -273,14 +275,14 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI ~basic_syncbuf() {
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
     try {
-#    endif // _LIBCPP_HAS_EXCEPTIONS
+#      endif // _LIBCPP_HAS_EXCEPTIONS
       emit();
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#    endif // _LIBCPP_HAS_EXCEPTIONS
+#      endif // _LIBCPP_HAS_EXCEPTIONS
     __dec_reference();
   }
 
@@ -337,9 +339,9 @@ protected:
       return traits_type::not_eof(__c);
 
     if (this->pptr() == this->epptr()) {
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
       try {
-#    endif
+#      endif
         size_t __size = __str_.size();
         __str_.resize(__str_.capacity() + 1);
         _LIBCPP_ASSERT_INTERNAL(__str_.size() > __size, "the buffer hasn't grown");
@@ -348,11 +350,11 @@ protected:
         this->setp(__p, __p + __str_.size());
         this->pbump(__size);
 
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return traits_type::eof();
       }
-#    endif
+#      endif
     }
 
     return this->sputc(traits_type::to_char_type(__c));
@@ -373,9 +375,9 @@ private:
     if (!__wrapped_)
       return false;
 
-#    if _LIBCPP_HAS_THREADS
+#      if _LIBCPP_HAS_THREADS
     lock_guard<mutex> __lock = __wrapped_streambuf_mutex::__instance().__get_lock(__wrapped_);
-#    endif
+#      endif
 
     bool __result = true;
     if (this->pptr() != this->pbase()) {
@@ -407,24 +409,24 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI void __inc_reference() {
-#    if _LIBCPP_HAS_THREADS
+#      if _LIBCPP_HAS_THREADS
     if (__wrapped_)
       __wrapped_streambuf_mutex::__instance().__inc_reference(__wrapped_);
-#    endif
+#      endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __dec_reference() noexcept {
-#    if _LIBCPP_HAS_THREADS
+#      if _LIBCPP_HAS_THREADS
     if (__wrapped_)
       __wrapped_streambuf_mutex::__instance().__dec_reference(__wrapped_);
-#    endif
+#      endif
   }
 };
 
 using std::syncbuf;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      if _LIBCPP_HAS_WIDE_CHARACTERS
 using std::wsyncbuf;
-#    endif
+#      endif
 
 // [syncstream.syncbuf.special], specialized algorithms
 template <class _CharT, class _Traits, class _Allocator>
@@ -480,17 +482,17 @@ public:
     // TODO validate other unformatted output functions.
     typename basic_ostream<char_type, traits_type>::sentry __s(*this);
     if (__s) {
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
       try {
-#    endif
+#      endif
 
         if (__sb_.emit() == false)
           this->setstate(ios::badbit);
-#    if _LIBCPP_HAS_EXCEPTIONS
+#      if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         this->__set_badbit_and_consider_rethrow();
       }
-#    endif
+#      endif
     }
   }
 
@@ -505,16 +507,17 @@ private:
 };
 
 using std::osyncstream;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      if _LIBCPP_HAS_WIDE_CHARACTERS
 using std::wosyncstream;
-#    endif
+#      endif
 
-#  endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
+#    endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM)
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP_HAS_LOCALIZATION
+#  endif // _LIBCPP_HAS_LOCALIZATION
+#endif   // 0
 
 #endif // _LIBCPP_SYNCSTREAM
diff --git a/libcxx/include/system_error b/libcxx/include/system_error
index eeab347788a9a..351e26fd47350 100644
--- a/libcxx/include/system_error
+++ b/libcxx/include/system_error
@@ -144,28 +144,31 @@ template <> struct hash<std::error_condition>;
 
 */
 
-#include <__config>
-#include <__system_error/errc.h>
-#include <__system_error/error_category.h>
-#include <__system_error/error_code.h>
-#include <__system_error/error_condition.h>
-#include <__system_error/system_error.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__system_error/errc.h>
+#  include <__system_error/error_category.h>
+#  include <__system_error/error_code.h>
+#  include <__system_error/error_condition.h>
+#  include <__system_error/system_error.h>
+#  include <version>
 
 // standard-mandated includes
 
 // [system.error.syn]
-#include <compare>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstdint>
-#  include <cstring>
-#  include <limits>
-#  include <type_traits>
-#endif
+#  include <compare>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstdint>
+#    include <cstring>
+#    include <limits>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_SYSTEM_ERROR
diff --git a/libcxx/include/tgmath.h b/libcxx/include/tgmath.h
index 1c5058cb065a9..41cae5c2318c1 100644
--- a/libcxx/include/tgmath.h
+++ b/libcxx/include/tgmath.h
@@ -17,19 +17,22 @@
 
 */
 
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#ifdef __cplusplus
-#  include <cmath>
-#  include <complex>
-#else
-#  if __has_include_next(<tgmath.h>)
-#    include_next <tgmath.h>
+#if 0
+#else // 0
+#  include <__config>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  ifdef __cplusplus
+#    include <cmath>
+#    include <complex>
+#  else
+#    if __has_include_next(<tgmath.h>)
+#      include_next <tgmath.h>
+#    endif
 #  endif
-#endif
+#endif // 0
 
 #endif // _LIBCPP_TGMATH_H
diff --git a/libcxx/include/thread b/libcxx/include/thread
index d7c3f704ad672..719ebe9a7a237 100644
--- a/libcxx/include/thread
+++ b/libcxx/include/thread
@@ -86,44 +86,47 @@ void sleep_for(const chrono::duration<Rep, Period>& rel_time);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if _LIBCPP_HAS_THREADS
+#  if _LIBCPP_HAS_THREADS
 
-#  include <__thread/this_thread.h>
-#  include <__thread/thread.h>
+#    include <__thread/this_thread.h>
+#    include <__thread/thread.h>
 
-#  if _LIBCPP_STD_VER >= 20
-#    include <__thread/jthread.h>
-#  endif
+#    if _LIBCPP_STD_VER >= 20
+#      include <__thread/jthread.h>
+#    endif
 
-#  if _LIBCPP_STD_VER >= 23
-#    include <__thread/formatter.h>
-#  endif
+#    if _LIBCPP_STD_VER >= 23
+#      include <__thread/formatter.h>
+#    endif
 
-#  include <version>
+#    include <version>
 
 // standard-mandated includes
 
 // [thread.syn]
-#  include <compare>
+#    include <compare>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
-#endif // _LIBCPP_HAS_THREADS
+#  endif // _LIBCPP_HAS_THREADS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
-#  include <chrono>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#    include <chrono>
+#  endif
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstring>
-#  include <functional>
-#  include <new>
-#  include <system_error>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstring>
+#    include <functional>
+#    include <new>
+#    include <system_error>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_THREAD
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index c3f7b8041686d..035c67a12f856 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -210,76 +210,78 @@ template <class... Types>
 
 // clang-format on
 
-#include <__compare/common_comparison_category.h>
-#include <__compare/ordering.h>
-#include <__compare/synth_three_way.h>
-#include <__config>
-#include <__cstddef/size_t.h>
-#include <__fwd/array.h>
-#include <__fwd/pair.h>
-#include <__fwd/tuple.h>
-#include <__memory/allocator_arg_t.h>
-#include <__memory/uses_allocator.h>
-#include <__tuple/find_index.h>
-#include <__tuple/ignore.h>
-#include <__tuple/make_tuple_types.h>
-#include <__tuple/sfinae_helpers.h>
-#include <__tuple/tuple_element.h>
-#include <__tuple/tuple_indices.h>
-#include <__tuple/tuple_like_ext.h>
-#include <__tuple/tuple_size.h>
-#include <__tuple/tuple_types.h>
-#include <__type_traits/common_reference.h>
-#include <__type_traits/common_type.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/conjunction.h>
-#include <__type_traits/copy_cvref.h>
-#include <__type_traits/disjunction.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/invoke.h>
-#include <__type_traits/is_arithmetic.h>
-#include <__type_traits/is_assignable.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_empty.h>
-#include <__type_traits/is_final.h>
-#include <__type_traits/is_implicitly_default_constructible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_reference.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/lazy.h>
-#include <__type_traits/maybe_const.h>
-#include <__type_traits/nat.h>
-#include <__type_traits/negation.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__type_traits/unwrap_ref.h>
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/integer_sequence.h>
-#include <__utility/move.h>
-#include <__utility/piecewise_construct.h>
-#include <__utility/swap.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__compare/common_comparison_category.h>
+#  include <__compare/ordering.h>
+#  include <__compare/synth_three_way.h>
+#  include <__config>
+#  include <__cstddef/size_t.h>
+#  include <__fwd/array.h>
+#  include <__fwd/pair.h>
+#  include <__fwd/tuple.h>
+#  include <__memory/allocator_arg_t.h>
+#  include <__memory/uses_allocator.h>
+#  include <__tuple/find_index.h>
+#  include <__tuple/ignore.h>
+#  include <__tuple/make_tuple_types.h>
+#  include <__tuple/sfinae_helpers.h>
+#  include <__tuple/tuple_element.h>
+#  include <__tuple/tuple_indices.h>
+#  include <__tuple/tuple_like_ext.h>
+#  include <__tuple/tuple_size.h>
+#  include <__tuple/tuple_types.h>
+#  include <__type_traits/common_reference.h>
+#  include <__type_traits/common_type.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/conjunction.h>
+#  include <__type_traits/copy_cvref.h>
+#  include <__type_traits/disjunction.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_arithmetic.h>
+#  include <__type_traits/is_assignable.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_empty.h>
+#  include <__type_traits/is_final.h>
+#  include <__type_traits/is_implicitly_default_constructible.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_reference.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/lazy.h>
+#  include <__type_traits/maybe_const.h>
+#  include <__type_traits/nat.h>
+#  include <__type_traits/negation.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__type_traits/unwrap_ref.h>
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/integer_sequence.h>
+#  include <__utility/move.h>
+#  include <__utility/piecewise_construct.h>
+#  include <__utility/swap.h>
+#  include <version>
 
 // standard-mandated includes
 
 // [tuple.syn]
-#include <compare>
+#  include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 // __tuple_leaf
 
@@ -305,11 +307,11 @@ class __tuple_leaf {
 
   template <class _Tp>
   static _LIBCPP_HIDE_FROM_ABI constexpr bool __can_bind_reference() {
-#  if __has_keyword(__reference_binds_to_temporary)
+#    if __has_keyword(__reference_binds_to_temporary)
     return !__reference_binds_to_temporary(_Hp, _Tp);
-#  else
+#    else
     return true;
-#  endif
+#    endif
   }
 
 public:
@@ -693,7 +695,7 @@ public:
       tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
       : __base_(allocator_arg_t(), __a, __t) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   // tuple(tuple<U...>&) constructors (including allocator_arg_t variants)
 
   template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
@@ -704,7 +706,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, tuple<_Up...>& __t)
       : __base_(allocator_arg_t(), __alloc, __t) {}
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   // tuple(tuple<U...>&&) constructors (including allocator_arg_t variants)
   template <class... _Up, __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
@@ -719,7 +721,7 @@ public:
       tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
       : __base_(allocator_arg_t(), __a, std::move(__t)) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   // tuple(const tuple<U...>&&) constructors (including allocator_arg_t variants)
 
   template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
@@ -733,7 +735,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, const tuple<_Up...>&& __t)
       : __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   // tuple(const pair<U1, U2>&) constructors (including allocator_arg_t variants)
 
@@ -779,7 +781,7 @@ public:
       tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
       : __base_(allocator_arg_t(), __a, __p) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   // tuple(pair<U1, U2>&) constructors (including allocator_arg_t variants)
 
   template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<pair<_U1, _U2>&>::value>* = nullptr>
@@ -794,7 +796,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, pair<_U1, _U2>& __p)
       : __base_(allocator_arg_t(), __alloc, __p) {}
-#  endif
+#    endif
 
   // tuple(pair<U1, U2>&&) constructors (including allocator_arg_t variants)
 
@@ -817,7 +819,7 @@ public:
       tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
       : __base_(allocator_arg_t(), __a, std::move(__p)) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   // tuple(const pair<U1, U2>&&) constructors (including allocator_arg_t variants)
 
   template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
@@ -832,7 +834,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, const pair<_U1, _U2>&& __p)
       : __base_(allocator_arg_t(), __alloc, std::move(__p)) {}
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   // [tuple.assign]
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
@@ -842,7 +844,7 @@ public:
     return *this;
   }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple const& __tuple) const
     requires(_And<is_copy_assignable<const _Tp>...>::value)
   {
@@ -857,7 +859,7 @@ public:
         *this, std::move(__tuple), __tuple_types<_Tp...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
     return *this;
   }
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
   operator=(_If<_And<is_move_assignable<_Tp>...>::value, tuple, __nat>&& __tuple) noexcept(
@@ -887,7 +889,7 @@ public:
     return *this;
   }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class... _UTypes,
             enable_if_t< _And<_BoolConstant<sizeof...(_Tp) == sizeof...(_UTypes)>,
                               is_assignable<const _Tp&, const _UTypes&>...>::value>* = nullptr>
@@ -904,7 +906,7 @@ public:
         *this, __u, __tuple_types<_UTypes...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
     return *this;
   }
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   template <template <class...> class _Pred,
             bool _Const,
@@ -924,7 +926,7 @@ public:
   template <bool _Const, class _Pair>
   struct _NothrowAssignFromPair : _AssignPredicateFromPair<is_nothrow_assignable, _Const, _Pair> {};
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <class _U1, class _U2, enable_if_t< _EnableAssignFromPair<true, const pair<_U1, _U2>&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(const pair<_U1, _U2>& __pair) const
       noexcept(_NothrowAssignFromPair<true, const pair<_U1, _U2>&>::value) {
@@ -940,7 +942,7 @@ public:
     std::get<1>(*this) = std::move(__pair.second);
     return *this;
   }
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
   template <class _Up1,
             class _Up2,
@@ -992,12 +994,12 @@ public:
     __base_.swap(__t.__base_);
   }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   _LIBCPP_HIDE_FROM_ABI constexpr void swap(const tuple& __t) const
       noexcept(__all<is_nothrow_swappable_v<const _Tp&>...>::value) {
     __base_.swap(__t.__base_);
   }
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 };
 
 template <>
@@ -1013,12 +1015,12 @@ public:
   template <class _Alloc, class _Up>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc&, array<_Up, 0>) _NOEXCEPT {}
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(tuple&) _NOEXCEPT {}
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   _LIBCPP_HIDE_FROM_ABI constexpr void swap(const tuple&) const noexcept {}
-#  endif
+#    endif
 };
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <class... _TTypes, class... _UTypes, template <class> class _TQual, template <class> class _UQual>
   requires requires { typename tuple<common_reference_t<_TQual<_TTypes>, _UQual<_UTypes>>...>; }
 struct basic_common_reference<tuple<_TTypes...>, tuple<_UTypes...>, _TQual, _UQual> {
@@ -1030,9 +1032,9 @@ template <class... _TTypes, class... _UTypes>
 struct common_type<tuple<_TTypes...>, tuple<_UTypes...>> {
   using type = tuple<common_type_t<_TTypes, _UTypes>...>;
 };
-#  endif // _LIBCPP_STD_VER >= 23
+#    endif // _LIBCPP_STD_VER >= 23
 
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
 template <class... _Tp>
 tuple(_Tp...) -> tuple<_Tp...>;
 template <class _Tp1, class _Tp2>
@@ -1043,7 +1045,7 @@ template <class _Alloc, class _Tp1, class _Tp2>
 tuple(allocator_arg_t, _Alloc, pair<_Tp1, _Tp2>) -> tuple<_Tp1, _Tp2>;
 template <class _Alloc, class... _Tp>
 tuple(allocator_arg_t, _Alloc, tuple<_Tp...>) -> tuple<_Tp...>;
-#  endif
+#    endif
 
 template <class... _Tp, __enable_if_t<__all<__is_swappable_v<_Tp>...>::value, int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
@@ -1051,14 +1053,14 @@ swap(tuple<_Tp...>& __t, tuple<_Tp...>& __u) noexcept(__all<__is_nothrow_swappab
   __t.swap(__u);
 }
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <class... _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<__all<is_swappable_v<const _Tp>...>::value, void>
 swap(const tuple<_Tp...>& __lhs,
      const tuple<_Tp...>& __rhs) noexcept(__all<is_nothrow_swappable_v<const _Tp>...>::value) {
   __lhs.swap(__rhs);
 }
-#  endif
+#    endif
 
 // get
 
@@ -1090,7 +1092,7 @@ get(const tuple<_Tp...>&& __t) _NOEXCEPT {
   return static_cast<const type&&>(static_cast<const __tuple_leaf<_Ip, type>&&>(__t.__base_).get());
 }
 
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
 
 template <class _T1, class... _Args>
 inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(tuple<_Args...>& __tup) noexcept {
@@ -1112,7 +1114,7 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(tuple<_Args...> const&& _
   return std::get<__find_exactly_one_t<_T1, _Args...>::value>(std::move(__tup));
 }
 
-#  endif
+#    endif
 
 // tie
 
@@ -1155,7 +1157,7 @@ operator==(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
   return __tuple_equal<sizeof...(_Tp)>()(__x, __y);
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 // operator<=>
 
@@ -1175,7 +1177,7 @@ operator<=>(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
   return std::__tuple_compare_three_way(__x, __y, index_sequence_for<_Tp...>{});
 }
 
-#  else // _LIBCPP_STD_VER >= 20
+#    else // _LIBCPP_STD_VER >= 20
 
 template <class... _Tp, class... _Up>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
@@ -1229,7 +1231,7 @@ operator<=(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
   return !(__y < __x);
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 // tuple_cat
 
@@ -1345,9 +1347,9 @@ tuple_cat(_Tuple0&& __t0, _Tuples&&... __tpls) {
 template <class... _Tp, class _Alloc>
 struct _LIBCPP_TEMPLATE_VIS uses_allocator<tuple<_Tp...>, _Alloc> : true_type {};
 
-#  if _LIBCPP_STD_VER >= 17
-#    define _LIBCPP_NOEXCEPT_RETURN(...)                                                                               \
-      noexcept(noexcept(__VA_ARGS__)) { return __VA_ARGS__; }
+#    if _LIBCPP_STD_VER >= 17
+#      define _LIBCPP_NOEXCEPT_RETURN(...)                                                                             \
+        noexcept(noexcept(__VA_ARGS__)) { return __VA_ARGS__; }
 
 // The _LIBCPP_NOEXCEPT_RETURN macro breaks formatting.
 // clang-format off
@@ -1410,14 +1412,15 @@ _LIBCPP_POP_MACROS
 
 // clang-format on
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <exception>
-#  include <iosfwd>
-#  include <new>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <exception>
+#    include <iosfwd>
+#    include <new>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_TUPLE
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index cc2b7511d24d3..d7fa8c3ad8f2e 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -424,108 +424,111 @@ namespace std
 
 */
 
-#include <__config>
-#include <__type_traits/add_cv_quals.h>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/add_rvalue_reference.h>
-#include <__type_traits/aligned_storage.h>
-#include <__type_traits/aligned_union.h>
-#include <__type_traits/alignment_of.h>
-#include <__type_traits/common_type.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/extent.h>
-#include <__type_traits/has_virtual_destructor.h>
-#include <__type_traits/integral_constant.h>
-#include <__type_traits/is_abstract.h>
-#include <__type_traits/is_arithmetic.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_assignable.h>
-#include <__type_traits/is_base_of.h>
-#include <__type_traits/is_class.h>
-#include <__type_traits/is_compound.h>
-#include <__type_traits/is_const.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_destructible.h>
-#include <__type_traits/is_empty.h>
-#include <__type_traits/is_enum.h>
-#include <__type_traits/is_floating_point.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_fundamental.h>
-#include <__type_traits/is_integral.h>
-#include <__type_traits/is_literal_type.h>
-#include <__type_traits/is_member_pointer.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_nothrow_destructible.h>
-#include <__type_traits/is_object.h>
-#include <__type_traits/is_pod.h>
-#include <__type_traits/is_pointer.h>
-#include <__type_traits/is_polymorphic.h>
-#include <__type_traits/is_reference.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_scalar.h>
-#include <__type_traits/is_signed.h>
-#include <__type_traits/is_standard_layout.h>
-#include <__type_traits/is_trivial.h>
-#include <__type_traits/is_trivially_assignable.h>
-#include <__type_traits/is_trivially_constructible.h>
-#include <__type_traits/is_trivially_copyable.h>
-#include <__type_traits/is_trivially_destructible.h>
-#include <__type_traits/is_union.h>
-#include <__type_traits/is_unsigned.h>
-#include <__type_traits/is_void.h>
-#include <__type_traits/is_volatile.h>
-#include <__type_traits/make_signed.h>
-#include <__type_traits/make_unsigned.h>
-#include <__type_traits/rank.h>
-#include <__type_traits/remove_all_extents.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_extent.h>
-#include <__type_traits/remove_pointer.h>
-#include <__type_traits/remove_reference.h>
-#include <__type_traits/remove_volatile.h>
-#include <__type_traits/result_of.h>
-#include <__type_traits/underlying_type.h>
-
-#if _LIBCPP_STD_VER >= 14
-#  include <__type_traits/is_final.h>
-#  include <__type_traits/is_null_pointer.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__type_traits/conjunction.h>
-#  include <__type_traits/disjunction.h>
-#  include <__type_traits/has_unique_object_representation.h>
-#  include <__type_traits/invoke.h>
-#  include <__type_traits/is_aggregate.h>
-#  include <__type_traits/is_swappable.h>
-#  include <__type_traits/negation.h>
-#  include <__type_traits/void_t.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__type_traits/common_reference.h>
-#  include <__type_traits/is_bounded_array.h>
-#  include <__type_traits/is_constant_evaluated.h>
-#  include <__type_traits/is_nothrow_convertible.h>
-#  include <__type_traits/is_unbounded_array.h>
-#  include <__type_traits/type_identity.h>
-#  include <__type_traits/unwrap_ref.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__type_traits/is_implicit_lifetime.h>
-#endif
-
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#if 0
+#else // 0
+#  include <__config>
+#  include <__type_traits/add_cv_quals.h>
+#  include <__type_traits/add_lvalue_reference.h>
+#  include <__type_traits/add_pointer.h>
+#  include <__type_traits/add_rvalue_reference.h>
+#  include <__type_traits/aligned_storage.h>
+#  include <__type_traits/aligned_union.h>
+#  include <__type_traits/alignment_of.h>
+#  include <__type_traits/common_type.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/decay.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/extent.h>
+#  include <__type_traits/has_virtual_destructor.h>
+#  include <__type_traits/integral_constant.h>
+#  include <__type_traits/is_abstract.h>
+#  include <__type_traits/is_arithmetic.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_assignable.h>
+#  include <__type_traits/is_base_of.h>
+#  include <__type_traits/is_class.h>
+#  include <__type_traits/is_compound.h>
+#  include <__type_traits/is_const.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_destructible.h>
+#  include <__type_traits/is_empty.h>
+#  include <__type_traits/is_enum.h>
+#  include <__type_traits/is_floating_point.h>
+#  include <__type_traits/is_function.h>
+#  include <__type_traits/is_fundamental.h>
+#  include <__type_traits/is_integral.h>
+#  include <__type_traits/is_literal_type.h>
+#  include <__type_traits/is_member_pointer.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_nothrow_destructible.h>
+#  include <__type_traits/is_object.h>
+#  include <__type_traits/is_pod.h>
+#  include <__type_traits/is_pointer.h>
+#  include <__type_traits/is_polymorphic.h>
+#  include <__type_traits/is_reference.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_scalar.h>
+#  include <__type_traits/is_signed.h>
+#  include <__type_traits/is_standard_layout.h>
+#  include <__type_traits/is_trivial.h>
+#  include <__type_traits/is_trivially_assignable.h>
+#  include <__type_traits/is_trivially_constructible.h>
+#  include <__type_traits/is_trivially_copyable.h>
+#  include <__type_traits/is_trivially_destructible.h>
+#  include <__type_traits/is_union.h>
+#  include <__type_traits/is_unsigned.h>
+#  include <__type_traits/is_void.h>
+#  include <__type_traits/is_volatile.h>
+#  include <__type_traits/make_signed.h>
+#  include <__type_traits/make_unsigned.h>
+#  include <__type_traits/rank.h>
+#  include <__type_traits/remove_all_extents.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/remove_cv.h>
+#  include <__type_traits/remove_extent.h>
+#  include <__type_traits/remove_pointer.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__type_traits/remove_volatile.h>
+#  include <__type_traits/result_of.h>
+#  include <__type_traits/underlying_type.h>
+
+#  if _LIBCPP_STD_VER >= 14
+#    include <__type_traits/is_final.h>
+#    include <__type_traits/is_null_pointer.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__type_traits/conjunction.h>
+#    include <__type_traits/disjunction.h>
+#    include <__type_traits/has_unique_object_representation.h>
+#    include <__type_traits/invoke.h>
+#    include <__type_traits/is_aggregate.h>
+#    include <__type_traits/is_swappable.h>
+#    include <__type_traits/negation.h>
+#    include <__type_traits/void_t.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__type_traits/common_reference.h>
+#    include <__type_traits/is_bounded_array.h>
+#    include <__type_traits/is_constant_evaluated.h>
+#    include <__type_traits/is_nothrow_convertible.h>
+#    include <__type_traits/is_unbounded_array.h>
+#    include <__type_traits/type_identity.h>
+#    include <__type_traits/unwrap_ref.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__type_traits/is_implicit_lifetime.h>
+#  endif
+
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_TYPE_TRAITS
diff --git a/libcxx/include/typeindex b/libcxx/include/typeindex
index 9f8e65befcba4..5ecba006ff1c1 100644
--- a/libcxx/include/typeindex
+++ b/libcxx/include/typeindex
@@ -45,17 +45,19 @@ struct hash<type_index>
 
 */
 
-#include <__config>
-#include <__functional/unary_function.h>
-#include <typeinfo>
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+#  include <__functional/unary_function.h>
+#  include <typeinfo>
+#  include <version>
 
 // standard-mandated includes
-#include <compare>
+#  include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -66,14 +68,14 @@ public:
   _LIBCPP_HIDE_FROM_ABI type_index(const type_info& __y) _NOEXCEPT : __t_(&__y) {}
 
   _LIBCPP_HIDE_FROM_ABI bool operator==(const type_index& __y) const _NOEXCEPT { return *__t_ == *__y.__t_; }
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_index& __y) const _NOEXCEPT { return *__t_ != *__y.__t_; }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI bool operator<(const type_index& __y) const _NOEXCEPT { return __t_->before(*__y.__t_); }
   _LIBCPP_HIDE_FROM_ABI bool operator<=(const type_index& __y) const _NOEXCEPT { return !__y.__t_->before(*__t_); }
   _LIBCPP_HIDE_FROM_ABI bool operator>(const type_index& __y) const _NOEXCEPT { return __y.__t_->before(*__t_); }
   _LIBCPP_HIDE_FROM_ABI bool operator>=(const type_index& __y) const _NOEXCEPT { return !__t_->before(*__y.__t_); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const type_index& __y) const noexcept {
     if (*__t_ == *__y.__t_)
       return strong_ordering::equal;
@@ -81,7 +83,7 @@ public:
       return strong_ordering::less;
     return strong_ordering::greater;
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __t_->hash_code(); }
   _LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __t_->name(); }
@@ -97,11 +99,12 @@ struct _LIBCPP_TEMPLATE_VIS hash<type_index> : public __unary_function<type_inde
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <iosfwd>
-#  include <new>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <iosfwd>
+#    include <new>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_TYPEINDEX
diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo
index 28713077c688c..8d3ebea08d8dd 100644
--- a/libcxx/include/typeinfo
+++ b/libcxx/include/typeinfo
@@ -56,27 +56,29 @@ public:
 
 */
 
-#include <__config>
-#include <__cstddef/size_t.h>
-#include <__exception/exception.h>
-#include <__type_traits/integral_constant.h>
-#include <__type_traits/is_constant_evaluated.h>
-#include <__verbose_abort>
-#include <cstdint>
-#include <version>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if defined(_LIBCPP_ABI_VCRUNTIME)
-#  include <vcruntime_typeinfo.h>
-#else
+#if 0
+#else // 0
+#  include <__config>
+#  include <__cstddef/size_t.h>
+#  include <__exception/exception.h>
+#  include <__type_traits/integral_constant.h>
+#  include <__type_traits/is_constant_evaluated.h>
+#  include <__verbose_abort>
+#  include <cstdint>
+#  include <version>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if defined(_LIBCPP_ABI_VCRUNTIME)
+#    include <vcruntime_typeinfo.h>
+#  else
 
 namespace std // purposefully not using versioning namespace
 {
 
-#  if defined(_LIBCPP_ABI_MICROSOFT)
+#    if defined(_LIBCPP_ABI_MICROSOFT)
 
 class _LIBCPP_EXPORTED_FROM_ABI type_info {
   type_info& operator=(const type_info&);
@@ -107,12 +109,12 @@ public:
     return __compare(__arg) == 0;
   }
 
-#    if _LIBCPP_STD_VER <= 17
+#      if _LIBCPP_STD_VER <= 17
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_info& __arg) const _NOEXCEPT { return !operator==(__arg); }
-#    endif
+#      endif
 };
 
-#  else // !defined(_LIBCPP_ABI_MICROSOFT)
+#    else // !defined(_LIBCPP_ABI_MICROSOFT)
 
 // ========================================================================== //
 //                           Implementations
@@ -167,21 +169,21 @@ public:
 
 // This value can be overriden in the __config_site. When it's not overriden,
 // we pick a default implementation based on the platform here.
-#    ifndef _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
+#      ifndef _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
 
 // Windows and AIX binaries can't merge typeinfos, so use the NonUnique implementation.
-#      if defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF)
-#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 2
+#        if defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF)
+#          define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 2
 
 // On arm64 on Apple platforms, use the special NonUniqueARMRTTIBit implementation.
-#      elif defined(__APPLE__) && defined(__LP64__) && !defined(__x86_64__)
-#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 3
+#        elif defined(__APPLE__) && defined(__LP64__) && !defined(__x86_64__)
+#          define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 3
 
 // On all other platforms, assume the Itanium C++ ABI and use the Unique implementation.
-#      else
-#        define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 1
+#        else
+#          define _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION 1
+#        endif
 #      endif
-#    endif
 
 struct __type_info_implementations {
   struct __string_impl_base {
@@ -265,30 +267,30 @@ struct __type_info_implementations {
   };
 
   typedef
-#    if _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 1
+#      if _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 1
       __unique_impl
-#    elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 2
+#      elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 2
       __non_unique_impl
-#    elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 3
+#      elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 3
       __non_unique_arm_rtti_bit_impl
-#    else
-#      error invalid configuration for _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
-#    endif
+#      else
+#        error invalid configuration for _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
+#      endif
           __impl;
 };
 
-#    if __has_cpp_attribute(_Clang::__ptrauth_vtable_pointer__)
-#      if __has_feature(ptrauth_type_info_vtable_pointer_discrimination)
-#        define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                  \
-          [[_Clang::__ptrauth_vtable_pointer__(process_independent, address_discrimination, type_discrimination)]]
+#      if __has_cpp_attribute(_Clang::__ptrauth_vtable_pointer__)
+#        if __has_feature(ptrauth_type_info_vtable_pointer_discrimination)
+#          define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                \
+            [[_Clang::__ptrauth_vtable_pointer__(process_independent, address_discrimination, type_discrimination)]]
+#        else
+#          define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                \
+            [[_Clang::__ptrauth_vtable_pointer__(                                                                      \
+                process_independent, no_address_discrimination, no_extra_discrimination)]]
+#        endif
 #      else
-#        define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH                                                                  \
-          [[_Clang::__ptrauth_vtable_pointer__(                                                                        \
-              process_independent, no_address_discrimination, no_extra_discrimination)]]
+#        define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH
 #      endif
-#    else
-#      define _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH
-#    endif
 
 class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH type_info {
   type_info& operator=(const type_info&);
@@ -321,11 +323,11 @@ public:
     return __impl::__eq(__type_name, __arg.__type_name);
   }
 
-#    if _LIBCPP_STD_VER <= 17
+#      if _LIBCPP_STD_VER <= 17
   _LIBCPP_HIDE_FROM_ABI bool operator!=(const type_info& __arg) const _NOEXCEPT { return !operator==(__arg); }
-#    endif
+#      endif
 };
-#  endif // defined(_LIBCPP_ABI_MICROSOFT)
+#    endif // defined(_LIBCPP_ABI_MICROSOFT)
 
 class _LIBCPP_EXPORTED_FROM_ABI bad_cast : public exception {
 public:
@@ -347,9 +349,9 @@ public:
 
 } // namespace std
 
-#endif // defined(_LIBCPP_ABI_VCRUNTIME)
+#  endif // defined(_LIBCPP_ABI_VCRUNTIME)
 
-#if defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
+#  if defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
 
 namespace std {
 
@@ -371,22 +373,23 @@ private:
 
 } // namespace std
 
-#endif // defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
+#  endif // defined(_LIBCPP_ABI_VCRUNTIME) && _HAS_EXCEPTIONS == 0
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_cast() {
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_cast();
-#else
+#  else
   _LIBCPP_VERBOSE_ABORT("bad_cast was thrown in -fno-exceptions mode");
-#endif
+#  endif
 }
 _LIBCPP_END_NAMESPACE_STD
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_TYPEINFO
diff --git a/libcxx/include/uchar.h b/libcxx/include/uchar.h
index 07b78611406d5..440e00511a660 100644
--- a/libcxx/include/uchar.h
+++ b/libcxx/include/uchar.h
@@ -32,25 +32,28 @@ size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
-#if !defined(_LIBCPP_CXX03_LANG)
+#  if !defined(_LIBCPP_CXX03_LANG)
 
 // Some platforms don't implement <uchar.h> and we don't want to give a hard
 // error on those platforms. When the platform doesn't provide <uchar.h>, at
 // least include <stddef.h> so we get the declaration for size_t, and try to
 // get the declaration of mbstate_t too.
-#  if __has_include_next(<uchar.h>)
-#    include_next <uchar.h>
-#  else
-#    include <__mbstate_t.h>
-#    include <stddef.h>
-#  endif
-
-#endif // _LIBCPP_CXX03_LANG
+#    if __has_include_next(<uchar.h>)
+#      include_next <uchar.h>
+#    else
+#      include <__mbstate_t.h>
+#      include <stddef.h>
+#    endif
+
+#  endif // _LIBCPP_CXX03_LANG
+#endif   // 0
 
 #endif // _LIBCPP_UCHAR_H
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 05aa01a3b7c30..5d5979b28ff0e 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -583,60 +583,62 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
 
 */
 
-#include <__algorithm/is_permutation.h>
-#include <__assert>
-#include <__config>
-#include <__functional/hash.h>
-#include <__functional/is_transparent.h>
-#include <__functional/operations.h>
-#include <__hash_table>
-#include <__iterator/distance.h>
-#include <__iterator/erase_if_container.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/ranges_iterator_traits.h>
-#include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/pointer_traits.h>
-#include <__memory/unique_ptr.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__node_handle>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/invoke.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_integral.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/pair.h>
-#include <new> // launder
-#include <stdexcept>
-#include <tuple>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/is_permutation.h>
+#  include <__assert>
+#  include <__config>
+#  include <__functional/hash.h>
+#  include <__functional/is_transparent.h>
+#  include <__functional/operations.h>
+#  include <__hash_table>
+#  include <__iterator/distance.h>
+#  include <__iterator/erase_if_container.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/ranges_iterator_traits.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory/pointer_traits.h>
+#  include <__memory/unique_ptr.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__node_handle>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_integral.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/pair.h>
+#  include <new> // launder
+#  include <stdexcept>
+#  include <tuple>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [unord.map.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -655,12 +657,12 @@ public:
     return static_cast<const _Hash&>(*this)(__x.__get_value().first);
   }
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return static_cast<const _Hash&>(*this)(__x); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
     return static_cast<const _Hash&>(*this)(__x);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
     using std::swap;
     swap(static_cast<_Hash&>(*this), static_cast<_Hash&>(__y));
@@ -679,12 +681,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return __hash_; }
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return __hash_(__x.__get_value().first); }
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return __hash_(__x); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
     return __hash_(__x);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
     using std::swap;
     swap(__hash_, __y.__hash_);
@@ -718,7 +720,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
     return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
     return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y);
@@ -735,7 +737,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
     return static_cast<const _Pred&>(*this)(__x, __y);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
     using std::swap;
     swap(static_cast<_Pred&>(*this), static_cast<_Pred&>(__y));
@@ -761,7 +763,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
     return __pred_(__x, __y.__get_value().first);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
     return __pred_(__x.__get_value().first, __y);
@@ -778,7 +780,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
     return __pred_(__x, __y);
   }
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
     using std::swap;
     swap(__pred_, __y.__pred_);
@@ -814,19 +816,19 @@ public:
         __first_constructed(false),
         __second_constructed(false) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(__hash_node_destructor<allocator_type>&& __x) _NOEXCEPT
       : __na_(__x.__na_),
         __first_constructed(__x.__value_constructed),
         __second_constructed(__x.__value_constructed) {
     __x.__value_constructed = false;
   }
-#else  // _LIBCPP_CXX03_LANG
+#  else  // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __hash_map_node_destructor(const __hash_node_destructor<allocator_type>& __x)
       : __na_(__x.__na_), __first_constructed(__x.__value_constructed), __second_constructed(__x.__value_constructed) {
     const_cast<bool&>(__x.__value_constructed) = false;
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
     if (__second_constructed)
@@ -838,7 +840,7 @@ public:
   }
 };
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 template <class _Key, class _Tp>
 struct _LIBCPP_STANDALONE_DEBUG __hash_value_type {
   typedef _Key key_type;
@@ -852,19 +854,19 @@ private:
 
 public:
   _LIBCPP_HIDE_FROM_ABI value_type& __get_value() {
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
     return *std::launder(std::addressof(__cc_));
-#  else
+#    else
     return __cc_;
-#  endif
+#    endif
   }
 
   _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const {
-#  if _LIBCPP_STD_VER >= 17
+#    if _LIBCPP_STD_VER >= 17
     return *std::launder(std::addressof(__cc_));
-#  else
+#    else
     return __cc_;
-#  endif
+#    endif
   }
 
   _LIBCPP_HIDE_FROM_ABI __nc_ref_pair_type __ref() {
@@ -901,7 +903,7 @@ public:
   ~__hash_value_type() = delete;
 };
 
-#else
+#  else
 
 template <class _Key, class _Tp>
 struct __hash_value_type {
@@ -919,7 +921,7 @@ public:
   ~__hash_value_type() = delete;
 };
 
-#endif
+#  endif
 
 template <class _HashIterator>
 class _LIBCPP_TEMPLATE_VIS __hash_map_iterator {
@@ -954,11 +956,11 @@ public:
   friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
     return __x.__i_ == __y.__i_;
   }
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
   friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_map_iterator& __x, const __hash_map_iterator& __y) {
     return __x.__i_ != __y.__i_;
   }
-#endif
+#  endif
 
   template <class, class, class, class, class>
   friend class _LIBCPP_TEMPLATE_VIS unordered_map;
@@ -1009,12 +1011,12 @@ public:
   operator==(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
     return __x.__i_ == __y.__i_;
   }
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
   friend _LIBCPP_HIDE_FROM_ABI bool
   operator!=(const __hash_map_const_iterator& __x, const __hash_map_const_iterator& __y) {
     return __x.__i_ != __y.__i_;
   }
-#endif
+#  endif
 
   template <class, class, class, class, class>
   friend class _LIBCPP_TEMPLATE_VIS unordered_map;
@@ -1084,10 +1086,10 @@ public:
   typedef __hash_map_iterator<typename __table::local_iterator> local_iterator;
   typedef __hash_map_const_iterator<typename __table::const_local_iterator> const_local_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __map_node_handle<__node, allocator_type> node_type;
   typedef __insert_return_type<iterator, node_type> insert_return_type;
-#endif
+#  endif
 
   template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS unordered_map;
@@ -1117,7 +1119,7 @@ public:
       const key_equal& __eql,
       const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_map(
       from_range_t,
@@ -1132,12 +1134,12 @@ public:
     }
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
   _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
@@ -1152,8 +1154,8 @@ public:
       const hasher& __hf,
       const key_equal& __eql,
       const allocator_type& __a);
-#endif // _LIBCPP_CXX03_LANG
-#if _LIBCPP_STD_VER >= 14
+#  endif // _LIBCPP_CXX03_LANG
+#  if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI unordered_map(size_type __n, const allocator_type& __a)
       : unordered_map(__n, hasher(), key_equal(), __a) {}
   _LIBCPP_HIDE_FROM_ABI unordered_map(size_type __n, const hasher& __hf, const allocator_type& __a)
@@ -1167,7 +1169,7 @@ public:
       _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_map(__first, __last, __n, __hf, key_equal(), __a) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_map(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
       : unordered_map(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
@@ -1176,22 +1178,22 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   unordered_map(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_map(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
       : unordered_map(__il, __n, hasher(), key_equal(), __a) {}
   _LIBCPP_HIDE_FROM_ABI
   unordered_map(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_map(__il, __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI ~unordered_map() {
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
   }
 
   _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     __table_ = __u.__table_;
-#else
+#  else
     if (this != std::addressof(__u)) {
       __table_.clear();
       __table_.hash_function()   = __u.__table_.hash_function();
@@ -1200,14 +1202,14 @@ public:
       __table_.__copy_assign_alloc(__u.__table_);
       insert(__u.begin(), __u.end());
     }
-#endif
+#  endif
     return *this;
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
       _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
     return allocator_type(__table_.__node_alloc());
@@ -1231,16 +1233,16 @@ public:
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     for (auto&& __element : __range) {
       __table_.__insert_unique(std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) {
@@ -1271,9 +1273,9 @@ public:
     return __table_.__emplace_unique(std::forward<_Args>(__args)...).first;
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
     return __table_.__emplace_unique_key_args(
@@ -1326,7 +1328,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator, key_type&& __k, _Vp&& __v) {
     return insert_or_assign(std::move(__k), std::forward<_Vp>(__v)).first;
   }
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p.__i_); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __table_.erase(__p.__i_); }
@@ -1336,7 +1338,7 @@ public:
   }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to unordered_map::insert()");
@@ -1378,7 +1380,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     return __table_.__node_handle_merge_unique(__source.__table_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(unordered_map& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
     __table_.swap(__u.__table_);
@@ -1389,7 +1391,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
@@ -1398,24 +1400,24 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_unique(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __table_.__equal_range_unique(__k);
@@ -1423,7 +1425,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __table_.__equal_range_unique(__k);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_unique(__k);
@@ -1432,12 +1434,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_unique(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
   _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
@@ -1462,12 +1464,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
 
 private:
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
-#endif
+#  endif
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Hash      = hash<__iter_key_type<_InputIterator>>,
           class _Pred      = equal_to<__iter_key_type<_InputIterator>>,
@@ -1485,7 +1487,7 @@ unordered_map(_InputIterator,
               _Allocator                                       = _Allocator())
     -> unordered_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Hash, _Pred, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Hash      = hash<__range_key_type<_Range>>,
           class _Pred      = equal_to<__range_key_type<_Range>>,
@@ -1501,7 +1503,7 @@ unordered_map(from_range_t,
               _Pred                                            = _Pred(),
               _Allocator                                       = _Allocator())
     -> unordered_map<__range_key_type<_Range>, __range_mapped_type<_Range>, _Hash, _Pred, _Allocator>; // C++23
-#  endif
+#    endif
 
 template <class _Key,
           class _Tp,
@@ -1554,7 +1556,7 @@ unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocat
                      equal_to<__iter_key_type<_InputIterator>>,
                      _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1585,7 +1587,7 @@ unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::siz
                      equal_to<__range_key_type<_Range>>,
                      _Allocator>;
 
-#  endif
+#    endif
 
 template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1604,7 +1606,7 @@ template <class _Key,
           class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_map<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(size_type __n, const hasher& __hf, const key_equal& __eql)
@@ -1665,7 +1667,7 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_ma
   insert(__u.begin(), __u.end());
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
@@ -1723,7 +1725,7 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value
   return *this;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 template <class _InputIterator>
@@ -1732,7 +1734,7 @@ inline void unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterato
     __table_.__insert_unique(*__first);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
@@ -1750,7 +1752,7 @@ _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](key_type&& __k)
       .first->__get_value()
       .second;
 }
-#else // _LIBCPP_CXX03_LANG
+#  else // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
@@ -1775,7 +1777,7 @@ _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type&
   return __r.first->second;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::at(const key_type& __k) {
@@ -1800,13 +1802,13 @@ swap(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, unordered_map<_Key, _T
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
 erase_if(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
@@ -1822,7 +1824,7 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_map<_Key, _Tp, _Hash, _Pre
   return true;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
@@ -1830,7 +1832,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_map<_Key, _Tp, _Has
   return !(__x == __y);
 }
 
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 struct __container_traits<unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc> > {
@@ -1893,9 +1895,9 @@ public:
   typedef __hash_map_iterator<typename __table::local_iterator> local_iterator;
   typedef __hash_map_const_iterator<typename __table::const_local_iterator> const_local_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __map_node_handle<__node, allocator_type> node_type;
-#endif
+#  endif
 
   template <class _Key2, class _Tp2, class _Hash2, class _Pred2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS unordered_map;
@@ -1925,7 +1927,7 @@ public:
       const key_equal& __eql,
       const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(
       from_range_t,
@@ -1940,12 +1942,12 @@ public:
     }
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
       _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
@@ -1961,8 +1963,8 @@ public:
       const hasher& __hf,
       const key_equal& __eql,
       const allocator_type& __a);
-#endif // _LIBCPP_CXX03_LANG
-#if _LIBCPP_STD_VER >= 14
+#  endif // _LIBCPP_CXX03_LANG
+#  if _LIBCPP_STD_VER >= 14
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(size_type __n, const allocator_type& __a)
       : unordered_multimap(__n, hasher(), key_equal(), __a) {}
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(size_type __n, const hasher& __hf, const allocator_type& __a)
@@ -1976,7 +1978,7 @@ public:
       _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multimap(__first, __last, __n, __hf, key_equal(), __a) {}
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
       : unordered_multimap(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
@@ -1985,22 +1987,22 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   unordered_multimap(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multimap(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
-#  endif
+#    endif
 
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
       : unordered_multimap(__il, __n, hasher(), key_equal(), __a) {}
   _LIBCPP_HIDE_FROM_ABI
   unordered_multimap(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multimap(__il, __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI ~unordered_multimap() {
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
   }
 
   _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) {
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
     __table_ = __u.__table_;
-#else
+#  else
     if (this != std::addressof(__u)) {
       __table_.clear();
       __table_.hash_function()   = __u.__table_.hash_function();
@@ -2009,14 +2011,14 @@ public:
       __table_.__copy_assign_alloc(__u.__table_);
       insert(__u.begin(), __u.end());
     }
-#endif
+#  endif
     return *this;
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
       _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
     return allocator_type(__table_.__node_alloc());
@@ -2042,16 +2044,16 @@ public:
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     for (auto&& __element : __range) {
       __table_.__insert_multi(std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
   _LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return __table_.__insert_multi(std::move(__x)); }
 
@@ -2078,7 +2080,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
     return __table_.__emplace_hint_multi(__p.__i_, std::forward<_Args>(__args)...);
   }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p.__i_); }
   _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __p) { return __table_.erase(__p.__i_); }
@@ -2088,7 +2090,7 @@ public:
   }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to unordered_multimap::insert()");
@@ -2130,7 +2132,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     return __table_.__node_handle_merge_multi(__source.__table_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(unordered_multimap& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
     __table_.swap(__u.__table_);
@@ -2141,7 +2143,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
@@ -2150,24 +2152,24 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_multi(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __table_.__equal_range_multi(__k);
@@ -2175,7 +2177,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __table_.__equal_range_multi(__k);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_multi(__k);
@@ -2184,7 +2186,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_multi(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
   _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
@@ -2206,7 +2208,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Hash      = hash<__iter_key_type<_InputIterator>>,
           class _Pred      = equal_to<__iter_key_type<_InputIterator>>,
@@ -2228,7 +2230,7 @@ unordered_multimap(_InputIterator,
                           _Pred,
                           _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Hash      = hash<__range_key_type<_Range>>,
           class _Pred      = equal_to<__range_key_type<_Range>>,
@@ -2244,7 +2246,7 @@ unordered_multimap(from_range_t,
                    _Pred                                            = _Pred(),
                    _Allocator                                       = _Allocator())
     -> unordered_multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, _Hash, _Pred, _Allocator>;
-#  endif
+#    endif
 
 template <class _Key,
           class _Tp,
@@ -2298,7 +2300,7 @@ unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Al
                           equal_to<__iter_key_type<_InputIterator>>,
                           _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -2329,7 +2331,7 @@ unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>
                           equal_to<__range_key_type<_Range>>,
                           _Allocator>;
 
-#  endif
+#    endif
 
 template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_multimap(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -2357,7 +2359,7 @@ template <class _Key,
 unordered_multimap(
     initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
@@ -2421,7 +2423,7 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
   insert(__u.begin(), __u.end());
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
@@ -2480,7 +2482,7 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<
   return *this;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 template <class _InputIterator>
@@ -2496,13 +2498,13 @@ swap(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, unordered_multima
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
 erase_if(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
@@ -2522,7 +2524,7 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multimap<_Key, _Tp, _Hash,
   return true;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
@@ -2530,7 +2532,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multimap<_Key, _Tp,
   return !(__x == __y);
 }
 
-#endif
+#  endif
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 struct __container_traits<unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc> > {
@@ -2544,7 +2546,7 @@ struct __container_traits<unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc> >
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _KeyT, class _ValueT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
@@ -2556,18 +2558,19 @@ using unordered_multimap _LIBCPP_AVAILABILITY_PMR =
     std::unordered_multimap<_KeyT, _ValueT, _HashT, _PredT, polymorphic_allocator<std::pair<const _KeyT, _ValueT>>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <bit>
-#  include <cmath>
-#  include <concepts>
-#  include <cstdlib>
-#  include <iterator>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <bit>
+#    include <cmath>
+#    include <concepts>
+#    include <cstdlib>
+#    include <iterator>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_UNORDERED_MAP
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 7ab1c651b8c95..6f243e0235ce9 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -531,59 +531,61 @@ template <class Value, class Hash, class Pred, class Alloc>
 
 // clang-format on
 
-#include <__algorithm/is_permutation.h>
-#include <__assert>
-#include <__config>
-#include <__functional/hash.h>
-#include <__functional/is_transparent.h>
-#include <__functional/operations.h>
-#include <__hash_table>
-#include <__iterator/distance.h>
-#include <__iterator/erase_if_container.h>
-#include <__iterator/iterator_traits.h>
-#include <__iterator/ranges_iterator_traits.h>
-#include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_traits.h>
-#include <__memory_resource/polymorphic_allocator.h>
-#include <__node_handle>
-#include <__ranges/concepts.h>
-#include <__ranges/container_compatible_range.h>
-#include <__ranges/from_range.h>
-#include <__type_traits/container_traits.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/invoke.h>
-#include <__type_traits/is_allocator.h>
-#include <__type_traits/is_integral.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/type_identity.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/is_permutation.h>
+#  include <__assert>
+#  include <__config>
+#  include <__functional/hash.h>
+#  include <__functional/is_transparent.h>
+#  include <__functional/operations.h>
+#  include <__hash_table>
+#  include <__iterator/distance.h>
+#  include <__iterator/erase_if_container.h>
+#  include <__iterator/iterator_traits.h>
+#  include <__iterator/ranges_iterator_traits.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/allocator_traits.h>
+#  include <__memory_resource/polymorphic_allocator.h>
+#  include <__node_handle>
+#  include <__ranges/concepts.h>
+#  include <__ranges/container_compatible_range.h>
+#  include <__ranges/from_range.h>
+#  include <__type_traits/container_traits.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_allocator.h>
+#  include <__type_traits/is_integral.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/type_identity.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [unord.set.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -621,10 +623,10 @@ public:
   typedef typename __table::const_local_iterator local_iterator;
   typedef typename __table::const_local_iterator const_local_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __set_node_handle<typename __table::__node, allocator_type> node_type;
   typedef __insert_return_type<iterator, node_type> insert_return_type;
-#endif
+#  endif
 
   template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS unordered_set;
@@ -634,12 +636,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI unordered_set() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) {}
   explicit _LIBCPP_HIDE_FROM_ABI
   unordered_set(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   inline _LIBCPP_HIDE_FROM_ABI unordered_set(size_type __n, const allocator_type& __a)
       : unordered_set(__n, hasher(), key_equal(), __a) {}
   inline _LIBCPP_HIDE_FROM_ABI unordered_set(size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_set(__n, __hf, key_equal(), __a) {}
-#endif
+#  endif
   _LIBCPP_HIDE_FROM_ABI
   unordered_set(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
   template <class _InputIterator>
@@ -660,7 +662,7 @@ public:
       const key_equal& __eql,
       const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_set(
       from_range_t,
@@ -675,9 +677,9 @@ public:
     }
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_set(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
@@ -686,9 +688,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI unordered_set(
       _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_set(__first, __last, __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_set(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
       : unordered_set(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
@@ -697,12 +699,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   unordered_set(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_set(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_set(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u);
   _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u, const allocator_type& __a);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_set(initializer_list<value_type> __il);
@@ -717,15 +719,15 @@ public:
       const hasher& __hf,
       const key_equal& __eql,
       const allocator_type& __a);
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_set(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
       : unordered_set(__il, __n, hasher(), key_equal(), __a) {}
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_set(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_set(__il, __n, __hf, key_equal(), __a) {}
-#  endif
-#endif // _LIBCPP_CXX03_LANG
+#    endif
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI ~unordered_set() {
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
   }
@@ -734,11 +736,11 @@ public:
     __table_ = __u.__table_;
     return *this;
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u)
       _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
     return allocator_type(__table_.__node_alloc());
@@ -755,7 +757,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
   _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
     return __table_.__emplace_unique(std::forward<_Args>(__args)...);
@@ -771,21 +773,21 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, value_type&& __x) { return insert(std::move(__x)).first; }
 
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return __table_.__insert_unique(__x); }
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x).first; }
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     for (auto&& __element : __range) {
       __table_.__insert_unique(std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p); }
   _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_unique(__k); }
@@ -794,7 +796,7 @@ public:
   }
   _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __table_.clear(); }
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to unordered_set::insert()");
@@ -836,7 +838,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     __table_.__node_handle_merge_unique(__source.__table_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI void swap(unordered_set& __u) _NOEXCEPT_(__is_nothrow_swappable_v<__table>) {
     __table_.swap(__u.__table_);
@@ -847,7 +849,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
@@ -856,24 +858,24 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_unique(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __table_.__equal_range_unique(__k);
@@ -881,7 +883,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __table_.__equal_range_unique(__k);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_unique(__k);
@@ -890,7 +892,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_unique(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
   _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
@@ -912,7 +914,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Hash      = hash<__iter_value_type<_InputIterator>>,
           class _Pred      = equal_to<__iter_value_type<_InputIterator>>,
@@ -929,7 +931,7 @@ unordered_set(_InputIterator,
               _Pred                                            = _Pred(),
               _Allocator = _Allocator()) -> unordered_set<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Hash      = hash<ranges::range_value_t<_Range>>,
           class _Pred      = equal_to<ranges::range_value_t<_Range>>,
@@ -945,7 +947,7 @@ unordered_set(
     _Hash                                            = _Hash(),
     _Pred                                            = _Pred(),
     _Allocator = _Allocator()) -> unordered_set<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
-#  endif
+#    endif
 
 template <class _Tp,
           class _Hash      = hash<_Tp>,
@@ -981,7 +983,7 @@ template <class _InputIterator,
 unordered_set(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_set<__iter_value_type<_InputIterator>, _Hash, equal_to<__iter_value_type<_InputIterator>>, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1006,7 +1008,7 @@ template <ranges::input_range _Range,
 unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_set<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
 
-#  endif
+#    endif
 
 template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1020,7 +1022,7 @@ template <class _Tp,
           class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_set<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(size_type __n, const hasher& __hf, const key_equal& __eql)
@@ -1080,7 +1082,7 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set&
   insert(__u.begin(), __u.end());
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u)
@@ -1137,7 +1139,7 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_ty
   return *this;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 template <class _InputIterator>
@@ -1153,13 +1155,13 @@ swap(unordered_set<_Value, _Hash, _Pred, _Alloc>& __x, unordered_set<_Value, _Ha
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Value, class _Hash, class _Pred, class _Alloc, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename unordered_set<_Value, _Hash, _Pred, _Alloc>::size_type
 erase_if(unordered_set<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
@@ -1175,7 +1177,7 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_set<_Value, _Hash, _Pred,
   return true;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
@@ -1183,7 +1185,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_set<_Value, _Hash,
   return !(__x == __y);
 }
 
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 struct __container_traits<unordered_set<_Value, _Hash, _Pred, _Alloc> > {
@@ -1225,9 +1227,9 @@ public:
   typedef typename __table::const_local_iterator local_iterator;
   typedef typename __table::const_local_iterator const_local_iterator;
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   typedef __set_node_handle<typename __table::__node, allocator_type> node_type;
-#endif
+#  endif
 
   template <class _Value2, class _Hash2, class _Pred2, class _Alloc2>
   friend class _LIBCPP_TEMPLATE_VIS unordered_set;
@@ -1239,12 +1241,12 @@ public:
   unordered_multiset(size_type __n, const hasher& __hf = hasher(), const key_equal& __eql = key_equal());
   _LIBCPP_HIDE_FROM_ABI
   unordered_multiset(size_type __n, const hasher& __hf, const key_equal& __eql, const allocator_type& __a);
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(size_type __n, const allocator_type& __a)
       : unordered_multiset(__n, hasher(), key_equal(), __a) {}
   inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multiset(__n, __hf, key_equal(), __a) {}
-#endif
+#  endif
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(_InputIterator __first, _InputIterator __last);
   template <class _InputIterator>
@@ -1263,7 +1265,7 @@ public:
       const key_equal& __eql,
       const allocator_type& __a);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(
       from_range_t,
@@ -1278,9 +1280,9 @@ public:
     }
     insert_range(std::forward<_Range>(__range));
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 14
+#  if _LIBCPP_STD_VER >= 14
   template <class _InputIterator>
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_multiset(_InputIterator __first, _InputIterator __last, size_type __n, const allocator_type& __a)
@@ -1289,9 +1291,9 @@ public:
   inline _LIBCPP_HIDE_FROM_ABI unordered_multiset(
       _InputIterator __first, _InputIterator __last, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multiset(__first, __last, __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(from_range_t, _Range&& __range, size_type __n, const allocator_type& __a)
       : unordered_multiset(from_range, std::forward<_Range>(__range), __n, hasher(), key_equal(), __a) {}
@@ -1300,12 +1302,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   unordered_multiset(from_range_t, _Range&& __range, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multiset(from_range, std::forward<_Range>(__range), __n, __hf, key_equal(), __a) {}
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_multiset(const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u, const allocator_type& __a);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u)
       _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u, const allocator_type& __a);
@@ -1321,15 +1323,15 @@ public:
       const hasher& __hf,
       const key_equal& __eql,
       const allocator_type& __a);
-#  if _LIBCPP_STD_VER >= 14
+#    if _LIBCPP_STD_VER >= 14
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_multiset(initializer_list<value_type> __il, size_type __n, const allocator_type& __a)
       : unordered_multiset(__il, __n, hasher(), key_equal(), __a) {}
   inline _LIBCPP_HIDE_FROM_ABI
   unordered_multiset(initializer_list<value_type> __il, size_type __n, const hasher& __hf, const allocator_type& __a)
       : unordered_multiset(__il, __n, __hf, key_equal(), __a) {}
-#  endif
-#endif // _LIBCPP_CXX03_LANG
+#    endif
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI ~unordered_multiset() {
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
   }
@@ -1338,11 +1340,11 @@ public:
     __table_ = __u.__table_;
     return *this;
   }
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u)
       _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
     return allocator_type(__table_.__node_alloc());
@@ -1359,7 +1361,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
   _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
     return __table_.__emplace_multi(std::forward<_Args>(__args)...);
@@ -1374,7 +1376,7 @@ public:
     return __table_.__insert_multi(__p, std::move(__x));
   }
   _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__insert_multi(__x); }
 
@@ -1385,16 +1387,16 @@ public:
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last);
 
-#if _LIBCPP_STD_VER >= 23
+#  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
     for (auto&& __element : __range) {
       __table_.__insert_multi(std::forward<decltype(__element)>(__element));
     }
   }
-#endif
+#  endif
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
   _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) {
     _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(),
                                         "node_type with incompatible allocator passed to unordered_multiset::insert()");
@@ -1436,7 +1438,7 @@ public:
         __source.get_allocator() == get_allocator(), "merging container with incompatible allocator");
     return __table_.__node_handle_merge_multi(__source.__table_);
   }
-#endif
+#  endif
 
   _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p) { return __table_.erase(__p); }
   _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __k) { return __table_.__erase_multi(__k); }
@@ -1454,7 +1456,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
@@ -1463,24 +1465,24 @@ public:
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_multi(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
     return __table_.__equal_range_multi(__k);
@@ -1488,7 +1490,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
     return __table_.__equal_range_multi(__k);
   }
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
   template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_multi(__k);
@@ -1497,7 +1499,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_multi(__k);
   }
-#endif // _LIBCPP_STD_VER >= 20
+#  endif // _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
   _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
@@ -1519,7 +1521,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _InputIterator,
           class _Hash      = hash<__iter_value_type<_InputIterator>>,
           class _Pred      = equal_to<__iter_value_type<_InputIterator>>,
@@ -1537,7 +1539,7 @@ unordered_multiset(
     _Pred                                            = _Pred(),
     _Allocator = _Allocator()) -> unordered_multiset<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 template <ranges::input_range _Range,
           class _Hash      = hash<ranges::range_value_t<_Range>>,
           class _Pred      = equal_to<ranges::range_value_t<_Range>>,
@@ -1553,7 +1555,7 @@ unordered_multiset(
     _Hash                                            = _Hash(),
     _Pred                                            = _Pred(),
     _Allocator = _Allocator()) -> unordered_multiset<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
-#  endif
+#    endif
 
 template <class _Tp,
           class _Hash      = hash<_Tp>,
@@ -1592,7 +1594,7 @@ unordered_multiset(_InputIterator, _InputIterator, typename allocator_traits<_Al
                           equal_to<__iter_value_type<_InputIterator>>,
                           _Allocator>;
 
-#  if _LIBCPP_STD_VER >= 23
+#    if _LIBCPP_STD_VER >= 23
 
 template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1617,7 +1619,7 @@ template <ranges::input_range _Range,
 unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_multiset<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
 
-#  endif
+#    endif
 
 template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
@@ -1631,7 +1633,7 @@ template <class _Tp,
           class = enable_if_t<__is_allocator<_Allocator>::value>>
 unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
     -> unordered_multiset<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
@@ -1695,7 +1697,7 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
   insert(__u.begin(), __u.end());
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(unordered_multiset&& __u)
@@ -1753,7 +1755,7 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<val
   return *this;
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 template <class _InputIterator>
@@ -1769,13 +1771,13 @@ swap(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x, unordered_multiset<_
   __x.swap(__y);
 }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 template <class _Value, class _Hash, class _Pred, class _Alloc, class _Predicate>
 inline _LIBCPP_HIDE_FROM_ABI typename unordered_multiset<_Value, _Hash, _Pred, _Alloc>::size_type
 erase_if(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) {
   return std::__libcpp_erase_if_container(__c, __pred);
 }
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
@@ -1795,7 +1797,7 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const unordered_multiset<_Value, _Hash, _P
   return true;
 }
 
-#if _LIBCPP_STD_VER <= 17
+#  if _LIBCPP_STD_VER <= 17
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
@@ -1803,7 +1805,7 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multiset<_Value, _H
   return !(__x == __y);
 }
 
-#endif
+#  endif
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 struct __container_traits<unordered_multiset<_Value, _Hash, _Pred, _Alloc> > {
@@ -1817,7 +1819,7 @@ struct __container_traits<unordered_multiset<_Value, _Hash, _Pred, _Alloc> > {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 _LIBCPP_BEGIN_NAMESPACE_STD
 namespace pmr {
 template <class _KeyT, class _HashT = std::hash<_KeyT>, class _PredT = std::equal_to<_KeyT>>
@@ -1828,18 +1830,19 @@ using unordered_multiset _LIBCPP_AVAILABILITY_PMR =
     std::unordered_multiset<_KeyT, _HashT, _PredT, polymorphic_allocator<_KeyT>>;
 } // namespace pmr
 _LIBCPP_END_NAMESPACE_STD
-#endif
+#  endif
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cmath>
-#  include <concepts>
-#  include <cstdlib>
-#  include <functional>
-#  include <iterator>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cmath>
+#    include <concepts>
+#    include <cstdlib>
+#    include <functional>
+#    include <iterator>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_UNORDERED_SET
diff --git a/libcxx/include/utility b/libcxx/include/utility
index 138cc3ba31893..c761a83000f34 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -246,65 +246,68 @@ template <class T>
 
 */
 
-#include <__config>
-
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/pair.h>
-#include <__utility/piecewise_construct.h>
-#include <__utility/rel_ops.h>
-#include <__utility/swap.h>
-
-#if _LIBCPP_STD_VER >= 14
-#  include <__utility/exchange.h>
-#  include <__utility/integer_sequence.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 17
-#  include <__utility/as_const.h>
-#  include <__utility/in_place.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 20
-#  include <__utility/cmp.h>
-#endif
-
-#if _LIBCPP_STD_VER >= 23
-#  include <__utility/forward_like.h>
-#  include <__utility/to_underlying.h>
-#  include <__utility/unreachable.h>
-#endif
-
-#include <version>
+#if 0
+#else // 0
+#  include <__config>
+
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/move.h>
+#  include <__utility/pair.h>
+#  include <__utility/piecewise_construct.h>
+#  include <__utility/rel_ops.h>
+#  include <__utility/swap.h>
+
+#  if _LIBCPP_STD_VER >= 14
+#    include <__utility/exchange.h>
+#    include <__utility/integer_sequence.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 17
+#    include <__utility/as_const.h>
+#    include <__utility/in_place.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 20
+#    include <__utility/cmp.h>
+#  endif
+
+#  if _LIBCPP_STD_VER >= 23
+#    include <__utility/forward_like.h>
+#    include <__utility/to_underlying.h>
+#    include <__utility/unreachable.h>
+#  endif
+
+#  include <version>
 
 // standard-mandated includes
 
 // [utility.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
 // [tuple.creation]
 
-#include <__tuple/ignore.h>
+#  include <__tuple/ignore.h>
 
 // [tuple.helper]
-#include <__tuple/tuple_element.h>
-#include <__tuple/tuple_size.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <limits>
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <cstdlib>
-#  include <iosfwd>
-#  include <type_traits>
-#endif
+#  include <__tuple/tuple_element.h>
+#  include <__tuple/tuple_size.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <limits>
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <cstdlib>
+#    include <iosfwd>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_UTILITY
diff --git a/libcxx/include/valarray b/libcxx/include/valarray
index 2f7a1a7c5b49e..af916096b5ef9 100644
--- a/libcxx/include/valarray
+++ b/libcxx/include/valarray
@@ -343,39 +343,41 @@ template <class T> unspecified2 end(const valarray<T>& v);
 
 */
 
-#include <__algorithm/copy.h>
-#include <__algorithm/count.h>
-#include <__algorithm/fill.h>
-#include <__algorithm/max_element.h>
-#include <__algorithm/min.h>
-#include <__algorithm/min_element.h>
-#include <__algorithm/unwrap_iter.h>
-#include <__assert>
-#include <__config>
-#include <__cstddef/ptrdiff_t.h>
-#include <__functional/operations.h>
-#include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/uninitialized_algorithms.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/remove_reference.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <cmath>
-#include <new>
-#include <version>
+#if 0
+#else // 0
+#  include <__algorithm/copy.h>
+#  include <__algorithm/count.h>
+#  include <__algorithm/fill.h>
+#  include <__algorithm/max_element.h>
+#  include <__algorithm/min.h>
+#  include <__algorithm/min_element.h>
+#  include <__algorithm/unwrap_iter.h>
+#  include <__assert>
+#  include <__config>
+#  include <__cstddef/ptrdiff_t.h>
+#  include <__functional/operations.h>
+#  include <__memory/addressof.h>
+#  include <__memory/allocator.h>
+#  include <__memory/uninitialized_algorithms.h>
+#  include <__type_traits/decay.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <cmath>
+#  include <new>
+#  include <version>
 
 // standard-mandated includes
 
 // [valarray.syn]
-#include <initializer_list>
+#  include <initializer_list>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -397,13 +399,13 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
   _LIBCPP_HIDE_FROM_ABI size_t stride() const { return __stride_; }
 
-#if _LIBCPP_STD_VER >= 20
+#  if _LIBCPP_STD_VER >= 20
 
   _LIBCPP_HIDE_FROM_ABI friend bool operator==(const slice& __x, const slice& __y) {
     return __x.start() == __y.start() && __x.size() == __y.size() && __x.stride() == __y.stride();
   }
 
-#endif
+#  endif
 };
 
 template <class _Tp>
@@ -794,10 +796,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI valarray(const value_type& __x, size_t __n);
   valarray(const value_type* __p, size_t __n);
   valarray(const valarray& __v);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI valarray(valarray&& __v) _NOEXCEPT;
   valarray(initializer_list<value_type> __il);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   valarray(const slice_array<value_type>& __sa);
   valarray(const gslice_array<value_type>& __ga);
   valarray(const mask_array<value_type>& __ma);
@@ -806,10 +808,10 @@ public:
 
   // assignment:
   valarray& operator=(const valarray& __v);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI valarray& operator=(valarray&& __v) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI valarray& operator=(initializer_list<value_type>);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI valarray& operator=(const value_type& __x);
   _LIBCPP_HIDE_FROM_ABI valarray& operator=(const slice_array<value_type>& __sa);
   _LIBCPP_HIDE_FROM_ABI valarray& operator=(const gslice_array<value_type>& __ga);
@@ -828,22 +830,22 @@ public:
   _LIBCPP_HIDE_FROM_ABI slice_array<value_type> operator[](slice __s);
   _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const gslice& __gs) const;
   _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](const gslice& __gs);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](gslice&& __gs) const;
   _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](gslice&& __gs);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](const valarray<bool>& __vb) const;
   _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](const valarray<bool>& __vb);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](valarray<bool>&& __vb) const;
   _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](valarray<bool>&& __vb);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const valarray<size_t>& __vs) const;
   _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](const valarray<size_t>& __vs);
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](valarray<size_t>&& __vs) const;
   _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](valarray<size_t>&& __vs);
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   // unary operators:
   _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__unary_plus<_Tp>, const valarray&> > operator+() const;
@@ -942,10 +944,10 @@ private:
   valarray& __assign_range(const value_type* __f, const value_type* __l);
 };
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp, size_t _Size>
 valarray(const _Tp (&)[_Size], size_t) -> valarray<_Tp>;
-#endif
+#  endif
 
 template <class _Expr,
           __enable_if_t<__is_val_expr<_Expr>::value && __val_expr_use_member_functions<_Expr>::value, int> = 0>
@@ -1221,7 +1223,7 @@ public:
     __init(__start);
   }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI gslice(size_t __start, const valarray<size_t>& __size, valarray<size_t>&& __stride)
       : __size_(__size), __stride_(std::move(__stride)) {
@@ -1238,7 +1240,7 @@ public:
     __init(__start);
   }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI size_t start() const { return __1d_.size() ? __1d_[0] : 0; }
 
@@ -1318,10 +1320,10 @@ private:
   gslice_array(const gslice& __gs, const valarray<value_type>& __v)
       : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(__gs.__1d_) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
   gslice_array(gslice&& __gs, const valarray<value_type>& __v)
       : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(std::move(__gs.__1d_)) {}
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class>
   friend class valarray;
@@ -1708,12 +1710,12 @@ private:
   _LIBCPP_HIDE_FROM_ABI indirect_array(const valarray<size_t>& __ia, const valarray<value_type>& __v)
       : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(__ia) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI indirect_array(valarray<size_t>&& __ia, const valarray<value_type>& __v)
       : __vp_(const_cast<value_type*>(__v.__begin_)), __1d_(std::move(__ia)) {}
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
   template <class>
   friend class valarray;
@@ -1837,12 +1839,12 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI __indirect_expr(const valarray<size_t>& __ia, const _RmExpr& __e) : __expr_(__e), __1d_(__ia) {}
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI __indirect_expr(valarray<size_t>&& __ia, const _RmExpr& __e)
       : __expr_(__e), __1d_(std::move(__ia)) {}
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 public:
   _LIBCPP_HIDE_FROM_ABI __result_type operator[](size_t __i) const { return __expr_[__1d_[__i]]; }
@@ -1984,17 +1986,17 @@ template <class _Tp>
 inline valarray<_Tp>::valarray(size_t __n) : __begin_(nullptr), __end_(nullptr) {
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
         ::new ((void*)__end_) value_type();
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2007,17 +2009,17 @@ template <class _Tp>
 valarray<_Tp>::valarray(const value_type* __p, size_t __n) : __begin_(nullptr), __end_(nullptr) {
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2025,21 +2027,21 @@ template <class _Tp>
 valarray<_Tp>::valarray(const valarray& __v) : __begin_(nullptr), __end_(nullptr) {
   if (__v.size()) {
     __begin_ = __end_ = allocator<value_type>().allocate(__v.size());
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (value_type* __p = __v.__begin_; __p != __v.__end_; ++__end_, ++__p)
         ::new ((void*)__end_) value_type(*__p);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__v.size());
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline valarray<_Tp>::valarray(valarray&& __v) _NOEXCEPT : __begin_(__v.__begin_), __end_(__v.__end_) {
@@ -2051,40 +2053,40 @@ valarray<_Tp>::valarray(initializer_list<value_type> __il) : __begin_(nullptr),
   const size_t __n = __il.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       size_t __n_left = __n;
       for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#  endif // _LIBCPP_HAS_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 valarray<_Tp>::valarray(const slice_array<value_type>& __sa) : __begin_(nullptr), __end_(nullptr) {
   const size_t __n = __sa.__size_;
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       size_t __n_left = __n;
       for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2093,19 +2095,19 @@ valarray<_Tp>::valarray(const gslice_array<value_type>& __ga) : __begin_(nullptr
   const size_t __n = __ga.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ga.__vp_;
       for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2114,19 +2116,19 @@ valarray<_Tp>::valarray(const mask_array<value_type>& __ma) : __begin_(nullptr),
   const size_t __n = __ma.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ma.__vp_;
       for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2135,19 +2137,19 @@ valarray<_Tp>::valarray(const indirect_array<value_type>& __ia) : __begin_(nullp
   const size_t __n = __ia.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ia.__vp_;
       for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2177,7 +2179,7 @@ valarray<_Tp>& valarray<_Tp>::operator=(const valarray& __v) {
   return *this;
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline valarray<_Tp>& valarray<_Tp>::operator=(valarray&& __v) _NOEXCEPT {
@@ -2194,7 +2196,7 @@ inline valarray<_Tp>& valarray<_Tp>::operator=(initializer_list<value_type> __il
   return __assign_range(__il.begin(), __il.end());
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline valarray<_Tp>& valarray<_Tp>::operator=(const value_type& __x) {
@@ -2273,7 +2275,7 @@ inline gslice_array<_Tp> valarray<_Tp>::operator[](const gslice& __gs) {
   return gslice_array<value_type>(__gs, *this);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<__indirect_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](gslice&& __gs) const {
@@ -2285,7 +2287,7 @@ inline gslice_array<_Tp> valarray<_Tp>::operator[](gslice&& __gs) {
   return gslice_array<value_type>(std::move(__gs), *this);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<__mask_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](const valarray<bool>& __vb) const {
@@ -2297,7 +2299,7 @@ inline mask_array<_Tp> valarray<_Tp>::operator[](const valarray<bool>& __vb) {
   return mask_array<value_type>(__vb, *this);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<__mask_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](valarray<bool>&& __vb) const {
@@ -2309,7 +2311,7 @@ inline mask_array<_Tp> valarray<_Tp>::operator[](valarray<bool>&& __vb) {
   return mask_array<value_type>(std::move(__vb), *this);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<__indirect_expr<const valarray<_Tp>&> >
@@ -2322,7 +2324,7 @@ inline indirect_array<_Tp> valarray<_Tp>::operator[](const valarray<size_t>& __v
   return indirect_array<value_type>(__vs, *this);
 }
 
-#ifndef _LIBCPP_CXX03_LANG
+#  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<__indirect_expr<const valarray<_Tp>&> > valarray<_Tp>::operator[](valarray<size_t>&& __vs) const {
@@ -2334,7 +2336,7 @@ inline indirect_array<_Tp> valarray<_Tp>::operator[](valarray<size_t>&& __vs) {
   return indirect_array<value_type>(std::move(__vs), *this);
 }
 
-#endif // _LIBCPP_CXX03_LANG
+#  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp>
 inline __val_expr<_UnaryOp<__unary_plus<_Tp>, const valarray<_Tp>&> > valarray<_Tp>::operator+() const {
@@ -2636,17 +2638,17 @@ void valarray<_Tp>::resize(size_t __n, value_type __x) {
   __clear(size());
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
         ::new ((void*)__end_) value_type(__x);
-#if _LIBCPP_HAS_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -3351,14 +3353,15 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <concepts>
-#  include <cstdlib>
-#  include <cstring>
-#  include <functional>
-#  include <stdexcept>
-#  include <type_traits>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <concepts>
+#    include <cstdlib>
+#    include <cstring>
+#    include <functional>
+#    include <stdexcept>
+#    include <type_traits>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_VALARRAY
diff --git a/libcxx/include/variant b/libcxx/include/variant
index f604527cd2256..9006e7ca35b09 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -212,74 +212,76 @@ namespace std {
 
 */
 
-#include <__compare/common_comparison_category.h>
-#include <__compare/compare_three_way_result.h>
-#include <__compare/ordering.h>
-#include <__compare/three_way_comparable.h>
-#include <__config>
-#include <__exception/exception.h>
-#include <__functional/hash.h>
-#include <__functional/operations.h>
-#include <__functional/unary_function.h>
-#include <__fwd/variant.h>
-#include <__memory/addressof.h>
-#include <__memory/construct_at.h>
-#include <__tuple/find_index.h>
-#include <__tuple/sfinae_helpers.h>
-#include <__type_traits/add_cv_quals.h>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/common_type.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/conjunction.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/dependent_type.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/invoke.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_assignable.h>
-#include <__type_traits/is_constructible.h>
-#include <__type_traits/is_convertible.h>
-#include <__type_traits/is_destructible.h>
-#include <__type_traits/is_nothrow_assignable.h>
-#include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_reference.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__type_traits/is_trivially_assignable.h>
-#include <__type_traits/is_trivially_constructible.h>
-#include <__type_traits/is_trivially_destructible.h>
-#include <__type_traits/is_trivially_relocatable.h>
-#include <__type_traits/is_void.h>
-#include <__type_traits/remove_const.h>
-#include <__type_traits/remove_cvref.h>
-#include <__type_traits/remove_reference.h>
-#include <__type_traits/type_identity.h>
-#include <__type_traits/void_t.h>
-#include <__utility/declval.h>
-#include <__utility/forward.h>
-#include <__utility/forward_like.h>
-#include <__utility/in_place.h>
-#include <__utility/integer_sequence.h>
-#include <__utility/move.h>
-#include <__utility/swap.h>
-#include <__variant/monostate.h>
-#include <__verbose_abort>
-#include <initializer_list>
-#include <limits>
-#include <new>
-#include <version>
+#if 0
+#else // 0
+#  include <__compare/common_comparison_category.h>
+#  include <__compare/compare_three_way_result.h>
+#  include <__compare/ordering.h>
+#  include <__compare/three_way_comparable.h>
+#  include <__config>
+#  include <__exception/exception.h>
+#  include <__functional/hash.h>
+#  include <__functional/operations.h>
+#  include <__functional/unary_function.h>
+#  include <__fwd/variant.h>
+#  include <__memory/addressof.h>
+#  include <__memory/construct_at.h>
+#  include <__tuple/find_index.h>
+#  include <__tuple/sfinae_helpers.h>
+#  include <__type_traits/add_cv_quals.h>
+#  include <__type_traits/add_pointer.h>
+#  include <__type_traits/common_type.h>
+#  include <__type_traits/conditional.h>
+#  include <__type_traits/conjunction.h>
+#  include <__type_traits/decay.h>
+#  include <__type_traits/dependent_type.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/invoke.h>
+#  include <__type_traits/is_array.h>
+#  include <__type_traits/is_assignable.h>
+#  include <__type_traits/is_constructible.h>
+#  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_destructible.h>
+#  include <__type_traits/is_nothrow_assignable.h>
+#  include <__type_traits/is_nothrow_constructible.h>
+#  include <__type_traits/is_reference.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/is_swappable.h>
+#  include <__type_traits/is_trivially_assignable.h>
+#  include <__type_traits/is_trivially_constructible.h>
+#  include <__type_traits/is_trivially_destructible.h>
+#  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/is_void.h>
+#  include <__type_traits/remove_const.h>
+#  include <__type_traits/remove_cvref.h>
+#  include <__type_traits/remove_reference.h>
+#  include <__type_traits/type_identity.h>
+#  include <__type_traits/void_t.h>
+#  include <__utility/declval.h>
+#  include <__utility/forward.h>
+#  include <__utility/forward_like.h>
+#  include <__utility/in_place.h>
+#  include <__utility/integer_sequence.h>
+#  include <__utility/move.h>
+#  include <__utility/swap.h>
+#  include <__variant/monostate.h>
+#  include <__verbose_abort>
+#  include <initializer_list>
+#  include <limits>
+#  include <new>
+#  include <version>
 
 // standard-mandated includes
 
 // [variant.syn]
-#include <compare>
+#  include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 namespace std { // explicitly not using versioning namespace
 
@@ -292,7 +294,7 @@ public:
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
 // Light N-dimensional array of function pointers. Used in place of std::array to avoid
 // adding a dependency.
@@ -306,11 +308,11 @@ struct __farray {
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS void
 __throw_bad_variant_access() {
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_variant_access();
-#  else
+#    else
   _LIBCPP_VERBOSE_ABORT("bad_variant_access was thrown in -fno-exceptions mode");
-#  endif
+#    endif
 }
 
 // variant_size
@@ -344,13 +346,13 @@ struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, variant<_Types...>> {
 
 template <size_t _NumAlternatives>
 _LIBCPP_HIDE_FROM_ABI constexpr auto __choose_index_type() {
-#  ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+#    ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
   if constexpr (_NumAlternatives < numeric_limits<unsigned char>::max())
     return static_cast<unsigned char>(0);
   else if constexpr (_NumAlternatives < numeric_limits<unsigned short>::max())
     return static_cast<unsigned short>(0);
   else
-#  endif // _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+#    endif // _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
     return static_cast<unsigned int>(0);
 }
 
@@ -595,12 +597,12 @@ struct __variant {
     return __visit_alt(__make_value_visitor(std::forward<_Visitor>(__visitor)), std::forward<_Vs>(__vs)...);
   }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _Rp, class _Visitor, class... _Vs>
   _LIBCPP_HIDE_FROM_ABI static constexpr _Rp __visit_value(_Visitor&& __visitor, _Vs&&... __vs) {
     return __visit_alt(__make_value_visitor<_Rp>(std::forward<_Visitor>(__visitor)), std::forward<_Vs>(__vs)...);
   }
-#  endif
+#    endif
 
 private:
   template <class _Visitor, class... _Values>
@@ -618,7 +620,7 @@ private:
     _Visitor&& __visitor;
   };
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _Rp, class _Visitor>
   struct __value_visitor_return_type {
     template <class... _Alts>
@@ -633,26 +635,26 @@ private:
 
     _Visitor&& __visitor;
   };
-#  endif
+#    endif
 
   template <class _Visitor>
   _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
     return __value_visitor<_Visitor>{std::forward<_Visitor>(__visitor)};
   }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
   template <class _Rp, class _Visitor>
   _LIBCPP_HIDE_FROM_ABI static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
     return __value_visitor_return_type<_Rp, _Visitor>{std::forward<_Visitor>(__visitor)};
   }
-#  endif
+#    endif
 };
 
 } // namespace __visitation
 
 // Adding semi-colons in macro expansions helps clang-format to do a better job.
 // This macro is used to avoid compilation errors due to "stray" semi-colons.
-#  define _LIBCPP_EAT_SEMICOLON static_assert(true, "")
+#    define _LIBCPP_EAT_SEMICOLON static_assert(true, "")
 
 template <size_t _Index, class _Tp>
 struct _LIBCPP_TEMPLATE_VIS __alt {
@@ -672,33 +674,33 @@ union _LIBCPP_TEMPLATE_VIS __union;
 template <_Trait _DestructibleTrait, size_t _Index>
 union _LIBCPP_TEMPLATE_VIS __union<_DestructibleTrait, _Index> {};
 
-#  define _LIBCPP_VARIANT_UNION(destructible_trait, destructor_definition)                                             \
-    template <size_t _Index, class _Tp, class... _Types>                                                               \
-    union _LIBCPP_TEMPLATE_VIS __union<destructible_trait, _Index, _Tp, _Types...> {                                   \
-    public:                                                                                                            \
-      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(__valueless_t) noexcept : __dummy{} {}                          \
+#    define _LIBCPP_VARIANT_UNION(destructible_trait, destructor_definition)                                           \
+      template <size_t _Index, class _Tp, class... _Types>                                                             \
+      union _LIBCPP_TEMPLATE_VIS __union<destructible_trait, _Index, _Tp, _Types...> {                                 \
+      public:                                                                                                          \
+        _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(__valueless_t) noexcept : __dummy{} {}                        \
                                                                                                                        \
-      template <class... _Args>                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<0>, _Args&&... __args)                         \
-          : __head(in_place, std::forward<_Args>(__args)...) {}                                                        \
+        template <class... _Args>                                                                                      \
+        _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<0>, _Args&&... __args)                       \
+            : __head(in_place, std::forward<_Args>(__args)...) {}                                                      \
                                                                                                                        \
-      template <size_t _Ip, class... _Args>                                                                            \
-      _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<_Ip>, _Args&&... __args)                       \
-          : __tail(in_place_index<_Ip - 1>, std::forward<_Args>(__args)...) {}                                         \
+        template <size_t _Ip, class... _Args>                                                                          \
+        _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<_Ip>, _Args&&... __args)                     \
+            : __tail(in_place_index<_Ip - 1>, std::forward<_Args>(__args)...) {}                                       \
                                                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI __union(const __union&)            = default;                                              \
-      _LIBCPP_HIDE_FROM_ABI __union(__union&&)                 = default;                                              \
-      _LIBCPP_HIDE_FROM_ABI __union& operator=(const __union&) = default;                                              \
-      _LIBCPP_HIDE_FROM_ABI __union& operator=(__union&&)      = default;                                              \
-      destructor_definition;                                                                                           \
+        _LIBCPP_HIDE_FROM_ABI __union(const __union&)            = default;                                            \
+        _LIBCPP_HIDE_FROM_ABI __union(__union&&)                 = default;                                            \
+        _LIBCPP_HIDE_FROM_ABI __union& operator=(const __union&) = default;                                            \
+        _LIBCPP_HIDE_FROM_ABI __union& operator=(__union&&)      = default;                                            \
+        destructor_definition;                                                                                         \
                                                                                                                        \
-    private:                                                                                                           \
-      char __dummy;                                                                                                    \
-      __alt<_Index, _Tp> __head;                                                                                       \
-      __union<destructible_trait, _Index + 1, _Types...> __tail;                                                       \
+      private:                                                                                                         \
+        char __dummy;                                                                                                  \
+        __alt<_Index, _Tp> __head;                                                                                     \
+        __union<destructible_trait, _Index + 1, _Types...> __tail;                                                     \
                                                                                                                        \
-      friend struct __access::__union;                                                                                 \
-    }
+        friend struct __access::__union;                                                                               \
+      }
 
 _LIBCPP_VARIANT_UNION(_Trait::_TriviallyAvailable,
                       _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = default);
@@ -706,7 +708,7 @@ _LIBCPP_VARIANT_UNION(
     _Trait::_Available, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() {} _LIBCPP_EAT_SEMICOLON);
 _LIBCPP_VARIANT_UNION(_Trait::_Unavailable, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = delete);
 
-#  undef _LIBCPP_VARIANT_UNION
+#    undef _LIBCPP_VARIANT_UNION
 
 template <_Trait _DestructibleTrait, class... _Types>
 class _LIBCPP_TEMPLATE_VIS __base {
@@ -747,25 +749,25 @@ protected:
 template <class _Traits, _Trait = _Traits::__destructible_trait>
 class _LIBCPP_TEMPLATE_VIS __dtor;
 
-#  define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor_definition, destroy)                               \
-    template <class... _Types>                                                                                         \
-    class _LIBCPP_TEMPLATE_VIS __dtor<__traits<_Types...>, destructible_trait>                                         \
-        : public __base<destructible_trait, _Types...> {                                                               \
-      using __base_type = __base<destructible_trait, _Types...>;                                                       \
-      using __index_t   = typename __base_type::__index_t;                                                             \
+#    define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor_definition, destroy)                             \
+      template <class... _Types>                                                                                       \
+      class _LIBCPP_TEMPLATE_VIS __dtor<__traits<_Types...>, destructible_trait>                                       \
+          : public __base<destructible_trait, _Types...> {                                                             \
+        using __base_type = __base<destructible_trait, _Types...>;                                                     \
+        using __index_t   = typename __base_type::__index_t;                                                           \
                                                                                                                        \
-    public:                                                                                                            \
-      using __base_type::__base_type;                                                                                  \
-      using __base_type::operator=;                                                                                    \
-      _LIBCPP_HIDE_FROM_ABI __dtor(const __dtor&)            = default;                                                \
-      _LIBCPP_HIDE_FROM_ABI __dtor(__dtor&&)                 = default;                                                \
-      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(const __dtor&) = default;                                                \
-      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(__dtor&&)      = default;                                                \
-      destructor_definition;                                                                                           \
+      public:                                                                                                          \
+        using __base_type::__base_type;                                                                                \
+        using __base_type::operator=;                                                                                  \
+        _LIBCPP_HIDE_FROM_ABI __dtor(const __dtor&)            = default;                                              \
+        _LIBCPP_HIDE_FROM_ABI __dtor(__dtor&&)                 = default;                                              \
+        _LIBCPP_HIDE_FROM_ABI __dtor& operator=(const __dtor&) = default;                                              \
+        _LIBCPP_HIDE_FROM_ABI __dtor& operator=(__dtor&&)      = default;                                              \
+        destructor_definition;                                                                                         \
                                                                                                                        \
-    protected:                                                                                                         \
-      destroy;                                                                                                         \
-    }
+      protected:                                                                                                       \
+        destroy;                                                                                                       \
+      }
 
 _LIBCPP_VARIANT_DESTRUCTOR(
     _Trait::_TriviallyAvailable,
@@ -793,7 +795,7 @@ _LIBCPP_VARIANT_DESTRUCTOR(_Trait::_Unavailable,
                            _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor()                 = delete,
                            _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept = delete);
 
-#  undef _LIBCPP_VARIANT_DESTRUCTOR
+#    undef _LIBCPP_VARIANT_DESTRUCTOR
 
 template <class _Traits>
 class _LIBCPP_TEMPLATE_VIS __ctor : public __dtor<_Traits> {
@@ -825,22 +827,22 @@ protected:
 template <class _Traits, _Trait = _Traits::__move_constructible_trait>
 class _LIBCPP_TEMPLATE_VIS __move_constructor;
 
-#  define _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor_definition)                      \
-    template <class... _Types>                                                                                         \
-    class _LIBCPP_TEMPLATE_VIS __move_constructor<__traits<_Types...>, move_constructible_trait>                       \
-        : public __ctor<__traits<_Types...>> {                                                                         \
-      using __base_type = __ctor<__traits<_Types...>>;                                                                 \
+#    define _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor_definition)                    \
+      template <class... _Types>                                                                                       \
+      class _LIBCPP_TEMPLATE_VIS __move_constructor<__traits<_Types...>, move_constructible_trait>                     \
+          : public __ctor<__traits<_Types...>> {                                                                       \
+        using __base_type = __ctor<__traits<_Types...>>;                                                               \
                                                                                                                        \
-    public:                                                                                                            \
-      using __base_type::__base_type;                                                                                  \
-      using __base_type::operator=;                                                                                    \
+      public:                                                                                                          \
+        using __base_type::__base_type;                                                                                \
+        using __base_type::operator=;                                                                                  \
                                                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI __move_constructor(const __move_constructor&)            = default;                        \
-      _LIBCPP_HIDE_FROM_ABI ~__move_constructor()                                    = default;                        \
-      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(const __move_constructor&) = default;                        \
-      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(__move_constructor&&)      = default;                        \
-      move_constructor_definition;                                                                                     \
-    }
+        _LIBCPP_HIDE_FROM_ABI __move_constructor(const __move_constructor&)            = default;                      \
+        _LIBCPP_HIDE_FROM_ABI ~__move_constructor()                                    = default;                      \
+        _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(const __move_constructor&) = default;                      \
+        _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(__move_constructor&&)      = default;                      \
+        move_constructor_definition;                                                                                   \
+      }
 
 _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
     _Trait::_TriviallyAvailable,
@@ -858,27 +860,27 @@ _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
     _Trait::_Unavailable,
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&&) = delete);
 
-#  undef _LIBCPP_VARIANT_MOVE_CONSTRUCTOR
+#    undef _LIBCPP_VARIANT_MOVE_CONSTRUCTOR
 
 template <class _Traits, _Trait = _Traits::__copy_constructible_trait>
 class _LIBCPP_TEMPLATE_VIS __copy_constructor;
 
-#  define _LIBCPP_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor_definition)                      \
-    template <class... _Types>                                                                                         \
-    class _LIBCPP_TEMPLATE_VIS __copy_constructor<__traits<_Types...>, copy_constructible_trait>                       \
-        : public __move_constructor<__traits<_Types...>> {                                                             \
-      using __base_type = __move_constructor<__traits<_Types...>>;                                                     \
+#    define _LIBCPP_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor_definition)                    \
+      template <class... _Types>                                                                                       \
+      class _LIBCPP_TEMPLATE_VIS __copy_constructor<__traits<_Types...>, copy_constructible_trait>                     \
+          : public __move_constructor<__traits<_Types...>> {                                                           \
+        using __base_type = __move_constructor<__traits<_Types...>>;                                                   \
                                                                                                                        \
-    public:                                                                                                            \
-      using __base_type::__base_type;                                                                                  \
-      using __base_type::operator=;                                                                                    \
+      public:                                                                                                          \
+        using __base_type::__base_type;                                                                                \
+        using __base_type::operator=;                                                                                  \
                                                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI __copy_constructor(__copy_constructor&&)                 = default;                        \
-      _LIBCPP_HIDE_FROM_ABI ~__copy_constructor()                                    = default;                        \
-      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(const __copy_constructor&) = default;                        \
-      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(__copy_constructor&&)      = default;                        \
-      copy_constructor_definition;                                                                                     \
-    }
+        _LIBCPP_HIDE_FROM_ABI __copy_constructor(__copy_constructor&&)                 = default;                      \
+        _LIBCPP_HIDE_FROM_ABI ~__copy_constructor()                                    = default;                      \
+        _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(const __copy_constructor&) = default;                      \
+        _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(__copy_constructor&&)      = default;                      \
+        copy_constructor_definition;                                                                                   \
+      }
 
 _LIBCPP_VARIANT_COPY_CONSTRUCTOR(
     _Trait::_TriviallyAvailable,
@@ -893,7 +895,7 @@ _LIBCPP_VARIANT_COPY_CONSTRUCTOR(
     _Trait::_Unavailable,
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor&) = delete);
 
-#  undef _LIBCPP_VARIANT_COPY_CONSTRUCTOR
+#    undef _LIBCPP_VARIANT_COPY_CONSTRUCTOR
 
 template <class _Traits>
 class _LIBCPP_TEMPLATE_VIS __assignment : public __copy_constructor<_Traits> {
@@ -952,22 +954,22 @@ protected:
 template <class _Traits, _Trait = _Traits::__move_assignable_trait>
 class _LIBCPP_TEMPLATE_VIS __move_assignment;
 
-#  define _LIBCPP_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment_definition)                           \
-    template <class... _Types>                                                                                         \
-    class _LIBCPP_TEMPLATE_VIS __move_assignment<__traits<_Types...>, move_assignable_trait>                           \
-        : public __assignment<__traits<_Types...>> {                                                                   \
-      using __base_type = __assignment<__traits<_Types...>>;                                                           \
+#    define _LIBCPP_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment_definition)                         \
+      template <class... _Types>                                                                                       \
+      class _LIBCPP_TEMPLATE_VIS __move_assignment<__traits<_Types...>, move_assignable_trait>                         \
+          : public __assignment<__traits<_Types...>> {                                                                 \
+        using __base_type = __assignment<__traits<_Types...>>;                                                         \
                                                                                                                        \
-    public:                                                                                                            \
-      using __base_type::__base_type;                                                                                  \
-      using __base_type::operator=;                                                                                    \
+      public:                                                                                                          \
+        using __base_type::__base_type;                                                                                \
+        using __base_type::operator=;                                                                                  \
                                                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI __move_assignment(const __move_assignment&)            = default;                          \
-      _LIBCPP_HIDE_FROM_ABI __move_assignment(__move_assignment&&)                 = default;                          \
-      _LIBCPP_HIDE_FROM_ABI ~__move_assignment()                                   = default;                          \
-      _LIBCPP_HIDE_FROM_ABI __move_assignment& operator=(const __move_assignment&) = default;                          \
-      move_assignment_definition;                                                                                      \
-    }
+        _LIBCPP_HIDE_FROM_ABI __move_assignment(const __move_assignment&)            = default;                        \
+        _LIBCPP_HIDE_FROM_ABI __move_assignment(__move_assignment&&)                 = default;                        \
+        _LIBCPP_HIDE_FROM_ABI ~__move_assignment()                                   = default;                        \
+        _LIBCPP_HIDE_FROM_ABI __move_assignment& operator=(const __move_assignment&) = default;                        \
+        move_assignment_definition;                                                                                    \
+      }
 
 _LIBCPP_VARIANT_MOVE_ASSIGNMENT(_Trait::_TriviallyAvailable,
                                 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(
@@ -986,27 +988,27 @@ _LIBCPP_VARIANT_MOVE_ASSIGNMENT(
     _Trait::_Unavailable,
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(__move_assignment&&) = delete);
 
-#  undef _LIBCPP_VARIANT_MOVE_ASSIGNMENT
+#    undef _LIBCPP_VARIANT_MOVE_ASSIGNMENT
 
 template <class _Traits, _Trait = _Traits::__copy_assignable_trait>
 class _LIBCPP_TEMPLATE_VIS __copy_assignment;
 
-#  define _LIBCPP_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment_definition)                           \
-    template <class... _Types>                                                                                         \
-    class _LIBCPP_TEMPLATE_VIS __copy_assignment<__traits<_Types...>, copy_assignable_trait>                           \
-        : public __move_assignment<__traits<_Types...>> {                                                              \
-      using __base_type = __move_assignment<__traits<_Types...>>;                                                      \
+#    define _LIBCPP_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment_definition)                         \
+      template <class... _Types>                                                                                       \
+      class _LIBCPP_TEMPLATE_VIS __copy_assignment<__traits<_Types...>, copy_assignable_trait>                         \
+          : public __move_assignment<__traits<_Types...>> {                                                            \
+        using __base_type = __move_assignment<__traits<_Types...>>;                                                    \
                                                                                                                        \
-    public:                                                                                                            \
-      using __base_type::__base_type;                                                                                  \
-      using __base_type::operator=;                                                                                    \
+      public:                                                                                                          \
+        using __base_type::__base_type;                                                                                \
+        using __base_type::operator=;                                                                                  \
                                                                                                                        \
-      _LIBCPP_HIDE_FROM_ABI __copy_assignment(const __copy_assignment&)       = default;                               \
-      _LIBCPP_HIDE_FROM_ABI __copy_assignment(__copy_assignment&&)            = default;                               \
-      _LIBCPP_HIDE_FROM_ABI ~__copy_assignment()                              = default;                               \
-      _LIBCPP_HIDE_FROM_ABI __copy_assignment& operator=(__copy_assignment&&) = default;                               \
-      copy_assignment_definition;                                                                                      \
-    }
+        _LIBCPP_HIDE_FROM_ABI __copy_assignment(const __copy_assignment&)       = default;                             \
+        _LIBCPP_HIDE_FROM_ABI __copy_assignment(__copy_assignment&&)            = default;                             \
+        _LIBCPP_HIDE_FROM_ABI ~__copy_assignment()                              = default;                             \
+        _LIBCPP_HIDE_FROM_ABI __copy_assignment& operator=(__copy_assignment&&) = default;                             \
+        copy_assignment_definition;                                                                                    \
+      }
 
 _LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_TriviallyAvailable,
                                 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
@@ -1024,7 +1026,7 @@ _LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_Unavailable,
                                 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
                                     const __copy_assignment&) = delete);
 
-#  undef _LIBCPP_VARIANT_COPY_ASSIGNMENT
+#    undef _LIBCPP_VARIANT_COPY_ASSIGNMENT
 
 template <class... _Types>
 class _LIBCPP_TEMPLATE_VIS __impl : public __copy_assignment<__traits<_Types...>> {
@@ -1061,7 +1063,7 @@ public:
         std::swap(__lhs, __rhs);
       }
       __impl __tmp(std::move(*__rhs));
-#  if _LIBCPP_HAS_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       if constexpr (__all<is_nothrow_move_constructible_v<_Types>...>::value) {
         this->__generic_construct(*__rhs, std::move(*__lhs));
       } else {
@@ -1077,11 +1079,11 @@ public:
           throw;
         }
       }
-#  else
+#    else
       // this isn't consolidated with the `if constexpr` branch above due to
       // `throw` being ill-formed with exceptions disabled even when discarded.
       this->__generic_construct(*__rhs, std::move(*__lhs));
-#  endif
+#    endif
       this->__generic_construct(*__lhs, std::move(__tmp));
     }
   }
@@ -1144,14 +1146,14 @@ template <class _Visitor, class... _Vs, typename = void_t<decltype(std::__as_var
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto)
 visit(_Visitor&& __visitor, _Vs&&... __vs);
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 template <class _Rp,
           class _Visitor,
           class... _Vs,
           typename = void_t<decltype(std::__as_variant(std::declval<_Vs>()))...>>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
 visit(_Visitor&& __visitor, _Vs&&... __vs);
-#  endif
+#    endif
 
 template <class... _Types>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant
@@ -1299,7 +1301,7 @@ public:
     __impl_.__swap(__that.__impl_);
   }
 
-#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
+#    if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
   // Helper class to implement [variant.visit]/10
   //   Constraints: The call to visit does not use an explicit template-argument-list
   //   that begins with a type template-argument.
@@ -1318,7 +1320,7 @@ public:
     using _VariantT = _OverrideRef<_Self&&, _CopyConst<remove_reference_t<_Self>, variant>>;
     return std::visit<_Rp>(std::forward<_Visitor>(__visitor), (_VariantT)__self);
   }
-#  endif
+#    endif
 
 private:
   __variant_detail::__impl<_Types...> __impl_;
@@ -1462,7 +1464,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const variant<_Types...>& __lhs,
   return __variant::__visit_value_at(__lhs.index(), __convert_to_bool<equal_to<>>{}, __lhs, __rhs);
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 
 template <class... _Types>
   requires(three_way_comparable<_Types> && ...)
@@ -1482,7 +1484,7 @@ operator<=>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   return __variant::__visit_value_at(__lhs.index(), __three_way, __lhs, __rhs);
 }
 
-#  endif // _LIBCPP_STD_VER >= 20
+#    endif // _LIBCPP_STD_VER >= 20
 
 template <class... _Types>
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
@@ -1566,7 +1568,7 @@ visit(_Visitor&& __visitor, _Vs&&... __vs) {
   return __variant::__visit_value(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
 }
 
-#  if _LIBCPP_STD_VER >= 20
+#    if _LIBCPP_STD_VER >= 20
 template < class _Rp, class _Visitor, class... _Vs, typename>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
 visit(_Visitor&& __visitor, _Vs&&... __vs) {
@@ -1574,7 +1576,7 @@ visit(_Visitor&& __visitor, _Vs&&... __vs) {
   std::__throw_if_valueless(std::forward<_Vs>(__vs)...);
   return __variant::__visit_value<_Rp>(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
 }
-#  endif
+#    endif
 
 template <class... _Types>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto
@@ -1623,19 +1625,20 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __unchecked_get(variant<_Types...>& __v)
   return std::__unchecked_get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
 }
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
 
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <cstddef>
-#  include <exception>
-#  include <tuple>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <cstddef>
+#    include <exception>
+#    include <tuple>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
+#  endif
+#endif // 0
 
 #endif // _LIBCPP_VARIANT
diff --git a/libcxx/include/vector b/libcxx/include/vector
index f4fa758f6f6d1..a35686f56b0d5 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -304,67 +304,70 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 
 // clang-format on
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#include <__vector/comparison.h>
-#include <__vector/swap.h>
-#include <__vector/vector.h>
-#include <__vector/vector_bool.h>
+#  include <__vector/comparison.h>
+#  include <__vector/swap.h>
+#  include <__vector/vector.h>
+#  include <__vector/vector_bool.h>
 
-#if _LIBCPP_STD_VER >= 17
-#  include <__vector/pmr.h>
-#endif
+#  if _LIBCPP_STD_VER >= 17
+#    include <__vector/pmr.h>
+#  endif
 
-#if _LIBCPP_STD_VER >= 20
-#  include <__vector/erase.h>
-#endif
+#  if _LIBCPP_STD_VER >= 20
+#    include <__vector/erase.h>
+#  endif
 
-#if _LIBCPP_STD_VER >= 23
-#  include <__vector/vector_bool_formatter.h>
-#endif
+#  if _LIBCPP_STD_VER >= 23
+#    include <__vector/vector_bool_formatter.h>
+#  endif
 
-#include <version>
+#  include <version>
 
 // standard-mandated includes
 
 // [iterator.range]
-#include <__iterator/access.h>
-#include <__iterator/data.h>
-#include <__iterator/empty.h>
-#include <__iterator/reverse_access.h>
-#include <__iterator/size.h>
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
 
 // [vector.syn]
-#include <compare>
-#include <initializer_list>
+#  include <compare>
+#  include <initializer_list>
 
 // [vector.syn], [unord.hash]
-#include <__functional/hash.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
-#  include <algorithm>
-#  include <array>
-#  include <atomic>
-#  include <cctype>
-#  include <cerrno>
-#  include <clocale>
-#  include <concepts>
-#  include <cstdint>
-#  include <cstdlib>
-#  include <iosfwd>
-#  if _LIBCPP_HAS_LOCALIZATION
-#    include <locale>
+#  include <__functional/hash.h>
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <algorithm>
+#    include <array>
+#    include <atomic>
+#    include <cctype>
+#    include <cerrno>
+#    include <clocale>
+#    include <concepts>
+#    include <cstdint>
+#    include <cstdlib>
+#    include <iosfwd>
+#    if _LIBCPP_HAS_LOCALIZATION
+#      include <locale>
+#    endif
+#    include <string>
+#    include <string_view>
+#    include <tuple>
+#    include <type_traits>
+#    include <typeinfo>
+#    include <utility>
 #  endif
-#  include <string>
-#  include <string_view>
-#  include <tuple>
-#  include <type_traits>
-#  include <typeinfo>
-#  include <utility>
-#endif
+#endif // 0
 
 #endif // _LIBCPP_VECTOR
diff --git a/libcxx/include/wchar.h b/libcxx/include/wchar.h
index b7cd5d7fc62cd..c91d52c0ca6db 100644
--- a/libcxx/include/wchar.h
+++ b/libcxx/include/wchar.h
@@ -105,40 +105,42 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 
 */
 
-#  include <__config>
-#  include <stddef.h>
+#  if 0
+#  else // 0
+#    include <__config>
+#    include <stddef.h>
 
-#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#    pragma GCC system_header
-#  endif
+#    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#      pragma GCC system_header
+#    endif
 
 // We define this here to support older versions of glibc <wchar.h> that do
 // not define this for clang.
-#  ifdef __cplusplus
-#    define __CORRECT_ISO_CPP_WCHAR_H_PROTO
-#  endif
+#    ifdef __cplusplus
+#      define __CORRECT_ISO_CPP_WCHAR_H_PROTO
+#    endif
 
-#  if __has_include_next(<wchar.h>)
-#    include_next <wchar.h>
-#  else
-#    include <__mbstate_t.h> // make sure we have mbstate_t regardless of the existence of <wchar.h>
-#  endif
+#    if __has_include_next(<wchar.h>)
+#      include_next <wchar.h>
+#    else
+#      include <__mbstate_t.h> // make sure we have mbstate_t regardless of the existence of <wchar.h>
+#    endif
 
 // Determine whether we have const-correct overloads for wcschr and friends.
-#  if defined(_WCHAR_H_CPLUSPLUS_98_CONFORMANCE_)
-#    define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
-#  elif defined(__GLIBC_PREREQ)
-#    if __GLIBC_PREREQ(2, 10)
-#      define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
-#    endif
-#  elif defined(_LIBCPP_MSVCRT)
-#    if defined(_CRT_CONST_CORRECT_OVERLOADS)
+#    if defined(_WCHAR_H_CPLUSPLUS_98_CONFORMANCE_)
 #      define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#    elif defined(__GLIBC_PREREQ)
+#      if __GLIBC_PREREQ(2, 10)
+#        define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#      endif
+#    elif defined(_LIBCPP_MSVCRT)
+#      if defined(_CRT_CONST_CORRECT_OVERLOADS)
+#        define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1
+#      endif
 #    endif
-#  endif
 
-#  if _LIBCPP_HAS_WIDE_CHARACTERS
-#    if defined(__cplusplus) && !defined(_LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
+#    if _LIBCPP_HAS_WIDE_CHARACTERS
+#      if defined(__cplusplus) && !defined(_LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD)
 extern "C++" {
 inline _LIBCPP_HIDE_FROM_ABI wchar_t* __libcpp_wcschr(const wchar_t* __s, wchar_t __c) {
   return (wchar_t*)wcschr(__s, __c);
@@ -193,16 +195,17 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD wchar_t* wmemchr(wchar_t
   return __libcpp_wmemchr(__s, __c, __n);
 }
 }
-#    endif
+#      endif
 
-#    if defined(__cplusplus) && (defined(_LIBCPP_MSVCRT_LIKE) || defined(__MVS__))
+#      if defined(__cplusplus) && (defined(_LIBCPP_MSVCRT_LIKE) || defined(__MVS__))
 extern "C" {
 size_t mbsnrtowcs(
     wchar_t* __restrict __dst, const char** __restrict __src, size_t __nmc, size_t __len, mbstate_t* __restrict __ps);
 size_t wcsnrtombs(
     char* __restrict __dst, const wchar_t** __restrict __src, size_t __nwc, size_t __len, mbstate_t* __restrict __ps);
 } // extern "C"
-#    endif // __cplusplus && (_LIBCPP_MSVCRT || __MVS__)
-#  endif   // _LIBCPP_HAS_WIDE_CHARACTERS
+#      endif // __cplusplus && (_LIBCPP_MSVCRT || __MVS__)
+#    endif   // _LIBCPP_HAS_WIDE_CHARACTERS
+#  endif     // 0
 
 #endif // _LIBCPP_WCHAR_H
diff --git a/libcxx/include/wctype.h b/libcxx/include/wctype.h
index 9f378eaf8202a..fb48d495e0392 100644
--- a/libcxx/include/wctype.h
+++ b/libcxx/include/wctype.h
@@ -44,11 +44,13 @@ wctrans_t wctrans(const char* property);
 
 */
 
-#include <__config>
+#if 0
+#else // 0
+#  include <__config>
 
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
 
 // TODO:
 // In the future, we should unconditionally include_next <wctype.h> here and instead
@@ -59,32 +61,33 @@ wctrans_t wctrans(const char* property);
 // nothing (with using_if_exists), and if we include another header that defines one
 // of these declarations (e.g. <wchar.h>), the second `using ::wint_t` with using_if_exists
 // will fail because it does not refer to the same declaration.
-#if __has_include_next(<wctype.h>)
-#  include_next <wctype.h>
-#  define _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
-#endif
+#  if __has_include_next(<wctype.h>)
+#    include_next <wctype.h>
+#    define _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
+#  endif
 
-#ifdef __cplusplus
+#  ifdef __cplusplus
 
-#  undef iswalnum
-#  undef iswalpha
-#  undef iswblank
-#  undef iswcntrl
-#  undef iswdigit
-#  undef iswgraph
-#  undef iswlower
-#  undef iswprint
-#  undef iswpunct
-#  undef iswspace
-#  undef iswupper
-#  undef iswxdigit
-#  undef iswctype
-#  undef wctype
-#  undef towlower
-#  undef towupper
-#  undef towctrans
-#  undef wctrans
+#    undef iswalnum
+#    undef iswalpha
+#    undef iswblank
+#    undef iswcntrl
+#    undef iswdigit
+#    undef iswgraph
+#    undef iswlower
+#    undef iswprint
+#    undef iswpunct
+#    undef iswspace
+#    undef iswupper
+#    undef iswxdigit
+#    undef iswctype
+#    undef wctype
+#    undef towlower
+#    undef towupper
+#    undef towctrans
+#    undef wctrans
 
-#endif // __cplusplus
+#  endif // __cplusplus
+#endif   // 0
 
 #endif // _LIBCPP_WCTYPE_H
diff --git a/libcxx/lib/abi/CHANGELOG.TXT b/libcxx/lib/abi/CHANGELOG.TXT
index e27eb5fa046ff..f630a5a4046df 100644
--- a/libcxx/lib/abi/CHANGELOG.TXT
+++ b/libcxx/lib/abi/CHANGELOG.TXT
@@ -16,6 +16,19 @@ New entries should be added directly below the "Version" header.
 Version 20.0
 ------------
 
+* [libc++] Remove the pointer safety functions from the dylib
+
+  The pointer safety functions were never implemented by anyone in a non-trivial
+  way, which lead us to removing them entirely from the headers when they were
+  removed in C++23. This also means that just about nobody ever called these
+  functions, especially not in production code. Because of that, we expect there
+  to be no references in the wild to the functions in the dylib.
+
+  Symbol removed: _ZNSt3__117declare_reachableEPv
+  Symbol removed: _ZNSt3__119declare_no_pointersEPcm
+  Symbol removed: _ZNSt3__121__undeclare_reachableEPv
+  Symbol removed: _ZNSt3__121undeclare_no_pointersEPcm
+
 * [libcxx][libc] Implements from_chars floating-point
 
   All platforms
diff --git a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 79f999b3e02bb..63a9872ee5558 100644
--- a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1507,7 +1507,6 @@
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1561,13 +1560,11 @@
 {'is_defined': True, 'name': '__ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEEx', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKvx', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1575,7 +1572,6 @@
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index 9efdf11940a77..4bb93b8f3105e 100644
--- a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1143,7 +1143,6 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1197,13 +1196,11 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk119declare_no_pointersEPcj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1211,7 +1208,6 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk121undeclare_no_pointersEPcj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index 7fde4b905fc50..0fbe2aa69fb75 100644
--- a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -502,7 +502,6 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117declare_reachableEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -550,13 +549,11 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__shared_weak_countD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__shared_weak_countD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119declare_no_pointersEPcm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvi', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__throw_system_errorEiPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121__throw_runtime_errorEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121__undeclare_reachableEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex4lockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex6unlockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex8try_lockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -564,7 +561,6 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121undeclare_no_pointersEPcm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index da30346257f95..bba9e49f23f3a 100644
--- a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -502,7 +502,6 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117declare_reachableEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -550,13 +549,11 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__shared_weak_countD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__shared_weak_countD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__119declare_no_pointersEPcm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEEl', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvl', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__120__throw_system_errorEiPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121__throw_runtime_errorEPKc', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121__undeclare_reachableEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex4lockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex6unlockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex8try_lockEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -564,7 +561,6 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
-{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__121undeclare_no_pointersEPcm', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index e1dc6e778b57c..c0045384d6625 100644
--- a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1507,7 +1507,6 @@
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1561,13 +1560,11 @@
 {'is_defined': True, 'name': '__ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEEx', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__libcpp_atomic_waitEPVKvx', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1575,7 +1572,6 @@
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index ceeeffe4d9792..80fdddda7cabb 100644
--- a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1143,7 +1143,6 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1197,13 +1196,11 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1211,7 +1208,6 @@
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt6__ndk121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt6__ndk123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
index d3670d237b239..3225e5edbcdc3 100644
--- a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1158,7 +1158,6 @@
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1212,13 +1211,11 @@
 {'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEEl', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvl', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1226,7 +1223,6 @@
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIlNS_22__cxx_atomic_base_implIlEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
index 2c21a03d41a0d..ec8217067f352 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -1156,7 +1156,6 @@
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1210,13 +1209,11 @@
 {'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1224,7 +1221,6 @@
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
index 0d4c509509087..6e82c97963a13 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
@@ -1127,7 +1127,6 @@
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__117declare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117iostream_categoryEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__117moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
@@ -1181,13 +1180,11 @@
 {'is_defined': True, 'name': '_ZNSt3__119__thread_local_dataEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEOS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__119declare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__get_collation_nameEPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEEi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__libcpp_atomic_waitEPVKvi', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__120__throw_system_errorEiPKc', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121__undeclare_reachableEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex4lockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
@@ -1195,7 +1192,6 @@
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexC2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__121recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '_ZNSt3__121undeclare_no_pointersEPcm', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__122__libcpp_verbose_abortEPKcz', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIiNS_22__cxx_atomic_base_implIiEEEE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__123__cxx_atomic_notify_allEPVKv', 'type': 'FUNC'}
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index cce8b8976f73c..41ab8cad5b7da 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -1,5 +1,3 @@
-set(LIBCXX_LIB_CMAKEFILES_DIR "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}"  PARENT_SCOPE)
-
 # Get sources
 set(LIBCXX_SOURCES
   algorithm.cpp
@@ -32,7 +30,6 @@ set(LIBCXX_SOURCES
   include/ryu/ryu.h
   include/to_chars_floating_point.h
   include/from_chars_floating_point.h
-  legacy_pointer_safety.cpp
   memory.cpp
   memory_resource.cpp
   new_handler.cpp
diff --git a/libcxx/src/legacy_pointer_safety.cpp b/libcxx/src/legacy_pointer_safety.cpp
deleted file mode 100644
index a9361ca879bb5..0000000000000
--- a/libcxx/src/legacy_pointer_safety.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <__config>
-#include <memory>
-
-// Support for garbage collection was removed in C++23 by https://wg21.link/P2186R2. Libc++ implements
-// that removal as an extension in all Standard versions. However, we still define the functions that
-// were once part of the library's ABI for backwards compatibility.
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-_LIBCPP_EXPORTED_FROM_ABI void declare_reachable(void*) {}
-_LIBCPP_EXPORTED_FROM_ABI void declare_no_pointers(char*, size_t) {}
-_LIBCPP_EXPORTED_FROM_ABI void undeclare_no_pointers(char*, size_t) {}
-_LIBCPP_EXPORTED_FROM_ABI void* __undeclare_reachable(void* p) { return p; }
-
-_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/benchmarks/ContainerBenchmarks.h b/libcxx/test/benchmarks/ContainerBenchmarks.h
index 38e11777f488b..458134c8191d1 100644
--- a/libcxx/test/benchmarks/ContainerBenchmarks.h
+++ b/libcxx/test/benchmarks/ContainerBenchmarks.h
@@ -14,8 +14,9 @@
 #include <iterator>
 #include <utility>
 
-#include "Utilities.h"
 #include "benchmark/benchmark.h"
+#include "Utilities.h"
+#include "test_iterators.h"
 
 namespace ContainerBenchmarks {
 
@@ -50,6 +51,19 @@ void BM_Assignment(benchmark::State& st, Container) {
   }
 }
 
+template <std::size_t... sz, typename Container, typename GenInputs>
+void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
+  auto v = gen(1, sz...);
+  c.resize(st.range(0), v[0]);
+  auto in = gen(st.range(1), sz...);
+  benchmark::DoNotOptimize(&in);
+  benchmark::DoNotOptimize(&c);
+  for (auto _ : st) {
+    c.assign(cpp17_input_iterator(in.begin()), cpp17_input_iterator(in.end()));
+    benchmark::ClobberMemory();
+  }
+}
+
 template <class Container>
 void BM_ConstructSizeValue(benchmark::State& st, Container, typename Container::value_type const& val) {
   const auto size = st.range(0);
diff --git a/libcxx/test/benchmarks/GenerateInput.h b/libcxx/test/benchmarks/GenerateInput.h
index db376614382bc..67ce798071eaa 100644
--- a/libcxx/test/benchmarks/GenerateInput.h
+++ b/libcxx/test/benchmarks/GenerateInput.h
@@ -108,16 +108,29 @@ std::vector<IntT> getRandomIntegerInputs(std::size_t N) {
   return inputs;
 }
 
+inline std::vector<std::string> getRandomStringInputsWithLength(std::size_t N, std::size_t len) { // N-by-len
+  std::vector<std::string> inputs;
+  inputs.reserve(N);
+  for (std::size_t i = 0; i < N; ++i)
+    inputs.push_back(getRandomString(len));
+  return inputs;
+}
+
 inline std::vector<std::string> getDuplicateStringInputs(std::size_t N) {
   std::vector<std::string> inputs(N, getRandomString(1024));
   return inputs;
 }
 
 inline std::vector<std::string> getRandomStringInputs(std::size_t N) {
-  std::vector<std::string> inputs;
+  return getRandomStringInputsWithLength(N, 1024);
+}
+
+template <class IntT>
+std::vector<std::vector<IntT>> getRandomIntegerInputsWithLength(std::size_t N, std::size_t len) { // N-by-len
+  std::vector<std::vector<IntT>> inputs;
   inputs.reserve(N);
   for (std::size_t i = 0; i < N; ++i)
-    inputs.push_back(getRandomString(1024));
+    inputs.push_back(getRandomIntegerInputs<IntT>(len));
   return inputs;
 }
 
diff --git a/libcxx/test/benchmarks/allocation.bench.cpp b/libcxx/test/benchmarks/allocation.bench.cpp
index 66a9b88793412..bdccaa2c133bd 100644
--- a/libcxx/test/benchmarks/allocation.bench.cpp
+++ b/libcxx/test/benchmarks/allocation.bench.cpp
@@ -6,8 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: -fsized-deallocation
-// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation
+// UNSUPPORTED: c++03, c++11
+
+// These compiler versions and platforms don't enable sized deallocation by default.
+// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
 
 #include "benchmark/benchmark.h"
 
diff --git a/libcxx/test/benchmarks/vector_operations.bench.cpp b/libcxx/test/benchmarks/vector_operations.bench.cpp
index 1855861263324..3a72eaec4dd57 100644
--- a/libcxx/test/benchmarks/vector_operations.bench.cpp
+++ b/libcxx/test/benchmarks/vector_operations.bench.cpp
@@ -18,7 +18,6 @@
 #include <vector>
 
 #include "benchmark/benchmark.h"
-
 #include "ContainerBenchmarks.h"
 #include "GenerateInput.h"
 
@@ -79,4 +78,17 @@ BENCHMARK(bm_grow<std::string>);
 BENCHMARK(bm_grow<std::unique_ptr<int>>);
 BENCHMARK(bm_grow<std::deque<int>>);
 
+BENCHMARK_CAPTURE(BM_AssignInputIterIter, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
+    ->Args({TestNumInputs, TestNumInputs});
+
+BENCHMARK_CAPTURE(
+    BM_AssignInputIterIter<32>, vector_string, std::vector<std::string>{}, getRandomStringInputsWithLength)
+    ->Args({TestNumInputs, TestNumInputs});
+
+BENCHMARK_CAPTURE(BM_AssignInputIterIter<100>,
+                  vector_vector_int,
+                  std::vector<std::vector<int>>{},
+                  getRandomIntegerInputsWithLength<int>)
+    ->Args({TestNumInputs, TestNumInputs});
+
 BENCHMARK_MAIN();
diff --git a/libcxx/test/configs/armv7m-picolibc-libc++.cfg.in b/libcxx/test/configs/armv7m-picolibc-libc++.cfg.in
index 7588b5c2a44cb..7aedfde89916c 100644
--- a/libcxx/test/configs/armv7m-picolibc-libc++.cfg.in
+++ b/libcxx/test/configs/armv7m-picolibc-libc++.cfg.in
@@ -7,7 +7,7 @@ config.substitutions.append(('%{flags}', '--sysroot=@CMAKE_INSTALL_PREFIX@'))
 config.substitutions.append(('%{compile_flags}',
     '-nostdinc++ -I %{include-dir} -I %{target-include-dir} -I %{libcxx-dir}/test/support'
 
-    # Disable warnings in cxx_atomic_impl.h:
+    # Disable warnings in the <atomic> implementation:
     # "large atomic operation may incur significant performance penalty; the
     # access size (4 bytes) exceeds the max lock-free size (0 bytes)"
     ' -Wno-atomic-alignment'
diff --git a/libcxx/test/libcxx/clang_tidy.gen.py b/libcxx/test/libcxx/clang_tidy.gen.py
index 5e84fbbb9913f..0db9c0d14b196 100644
--- a/libcxx/test/libcxx/clang_tidy.gen.py
+++ b/libcxx/test/libcxx/clang_tidy.gen.py
@@ -15,7 +15,7 @@
 
 import sys
 sys.path.append(sys.argv[1])
-from libcxx.header_information import lit_header_restrictions, public_headers
+from libcxx.header_information import lit_header_restrictions, lit_header_undeprecations, public_headers
 
 for header in public_headers:
   print(f"""\
@@ -30,6 +30,7 @@
 // UNSUPPORTED: clang-17
 
 {lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
 
 // TODO: run clang-tidy with modules enabled once they are supported
 // RUN: %{{clang-tidy}} %s --warnings-as-errors=* -header-filter=.* --checks='-*,libcpp-*' --load=%{{test-tools-dir}}/clang_tidy_checks/libcxx-tidy.plugin -- %{{compile_flags}} -fno-modules
diff --git a/libcxx/test/libcxx/containers/associative/reference_comparator_abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/reference_comparator_abi.compile.pass.cpp
new file mode 100644
index 0000000000000..f364fc817c164
--- /dev/null
+++ b/libcxx/test/libcxx/containers/associative/reference_comparator_abi.compile.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Pin down the ABI of associative containers with respect to their size and alignment
+// when passed a comparator that is a reference.
+//
+// While it's not even clear that reference comparators are legal in containers, an
+// unintended ABI break was discovered after implementing the new compressed pair
+// mechanism based on [[no_unique_address]], and this is a regression test for that.
+// If we decide to make reference comparators ill-formed, this test would become
+// unnecessary.
+//
+// See https://github.com/llvm/llvm-project/issues/118559 for more details.
+
+#include <set>
+#include <map>
+
+#include "test_macros.h"
+
+struct TEST_ALIGNAS(16) Cmp {
+  bool operator()(int, int) const;
+};
+
+template <class Compare>
+struct Set {
+  char b;
+  std::set<int, Compare> s;
+};
+
+template <class Compare>
+struct Multiset {
+  char b;
+  std::multiset<int, Compare> s;
+};
+
+template <class Compare>
+struct Map {
+  char b;
+  std::map<int, char, Compare> s;
+};
+
+template <class Compare>
+struct Multimap {
+  char b;
+  std::multimap<int, char, Compare> s;
+};
+
+static_assert(sizeof(Set<Cmp&>) == sizeof(Set<bool (*)(int, int)>), "");
+static_assert(sizeof(Multiset<Cmp&>) == sizeof(Multiset<bool (*)(int, int)>), "");
+
+static_assert(sizeof(Map<Cmp&>) == sizeof(Map<bool (*)(int, int)>), "");
+static_assert(sizeof(Multimap<Cmp&>) == sizeof(Multimap<bool (*)(int, int)>), "");
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
index 4461bad8ff504..c8c6e3743bd21 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
@@ -25,13 +25,48 @@
 #include "check_assertion.h"
 
 int main(int, char**) {
-    std::array<int, 3> array{0, 1, 2};
+  std::array<int, 3> array{0, 1, 2};
 
-    auto too_large = [&] { std::span<int, 3> const s(array.data(), 4); (void)s; };
-    TEST_LIBCPP_ASSERT_FAILURE(too_large(), "size mismatch in span's constructor (iterator, len)");
+  // Input range too large (exceeds the span extent)
+  {
+    auto f = [&] {
+      std::span<int, 3> const s(array.data(), 4);
+      (void)s;
+    };
+    TEST_LIBCPP_ASSERT_FAILURE(f(), "size mismatch in span's constructor (iterator, len)");
+  }
 
-    auto too_small = [&] { std::span<int, 3> const s(array.data(), 2); (void)s; };
-    TEST_LIBCPP_ASSERT_FAILURE(too_small(), "size mismatch in span's constructor (iterator, len)");
+  // Input range too small (doesn't fill the span)
+  {
+    auto f = [&] {
+      std::span<int, 3> const s(array.data(), 2);
+      (void)s;
+    };
+    TEST_LIBCPP_ASSERT_FAILURE(f(), "size mismatch in span's constructor (iterator, len)");
+  }
 
-    return 0;
+  // Input range is non-empty but starts with a null pointer
+  {
+    // static extent
+    {
+      auto f = [&] {
+        int* p = nullptr;
+        std::span<int, 3> const s(p, 3);
+        (void)s;
+      };
+      TEST_LIBCPP_ASSERT_FAILURE(f(), "passed nullptr with non-zero length in span's constructor (iterator, len)");
+    }
+
+    // dynamic extent
+    {
+      auto f = [&] {
+        int* p = nullptr;
+        std::span<int, std::dynamic_extent> const s(p, 1);
+        (void)s;
+      };
+      TEST_LIBCPP_ASSERT_FAILURE(f(), "passed nullptr with non-zero length in span's constructor (iterator, len)");
+    }
+  }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
deleted file mode 100644
index 87f4783e12973..0000000000000
--- a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
+++ /dev/null
@@ -1,108 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// test libc++'s implementation of align_val_t, and the relevant new/delete
-// overloads in all dialects when -faligned-allocation is present.
-
-// Libc++ when built for z/OS doesn't contain the aligned allocation functions,
-// nor does the dynamic library shipped with z/OS.
-// XFAIL: target={{.+}}-zos{{.*}}
-
-// REQUIRES: -faligned-allocation
-// ADDITIONAL_COMPILE_FLAGS: -faligned-allocation
-
-#include <cassert>
-#include <new>
-#include <string>
-#include <type_traits>
-#include <typeinfo>
-
-#include "test_macros.h"
-
-static void test_allocations(std::size_t size, size_t alignment) {
-  {
-    void* ptr = ::operator new(size, std::align_val_t(alignment));
-    assert(ptr);
-    assert(reinterpret_cast<std::uintptr_t>(ptr) % alignment == 0);
-    ::operator delete(ptr, std::align_val_t(alignment));
-  }
-  {
-    void* ptr = ::operator new(size, std::align_val_t(alignment), std::nothrow);
-    assert(ptr);
-    assert(reinterpret_cast<std::uintptr_t>(ptr) % alignment == 0);
-    ::operator delete(ptr, std::align_val_t(alignment), std::nothrow);
-  }
-  {
-    void* ptr = ::operator new[](size, std::align_val_t(alignment));
-    assert(ptr);
-    assert(reinterpret_cast<std::uintptr_t>(ptr) % alignment == 0);
-    ::operator delete[](ptr, std::align_val_t(alignment));
-  }
-  {
-    void* ptr = ::operator new[](size, std::align_val_t(alignment), std::nothrow);
-    assert(ptr);
-    assert(reinterpret_cast<std::uintptr_t>(ptr) % alignment == 0);
-    ::operator delete[](ptr, std::align_val_t(alignment), std::nothrow);
-  }
-}
-
-int main(int, char**) {
-  {
-    static_assert(std::is_enum<std::align_val_t>::value, "");
-    typedef std::underlying_type<std::align_val_t>::type UT;
-    static_assert((std::is_same<UT, std::size_t>::value), "");
-  }
-  {
-    static_assert((!std::is_constructible<std::align_val_t, std::size_t>::value), "");
-#if TEST_STD_VER >= 11
-    static_assert(!std::is_constructible<std::size_t, std::align_val_t>::value, "");
-#else
-    static_assert((std::is_constructible<std::size_t, std::align_val_t>::value), "");
-#endif
-  }
-  {
-    std::align_val_t a = std::align_val_t(0);
-    std::align_val_t b = std::align_val_t(32);
-    assert(a != b);
-    assert(a == std::align_val_t(0));
-    assert(b == std::align_val_t(32));
-  }
-  // First, check the basic case, a large allocation with alignment==size.
-  test_allocations(64, 64);
-  // Size being a multiple of alignment also needs to be supported.
-  test_allocations(64, 32);
-  // When aligned allocation is implemented using aligned_alloc,
-  // that function requires a minimum alignment of sizeof(void*).
-  // Check that we can also create overaligned allocations with
-  // an alignment argument less than sizeof(void*).
-  test_allocations(2, 2);
-  // When implemented using the C11 aligned_alloc() function,
-  // that requires that size be a multiple of alignment.
-  // However, the C++ operator new has no such requirements.
-  // Check that we can create an overaligned allocation that does
-  // adhere to not have this constraint.
-  test_allocations(1, 128);
-  // Finally, test size > alignment, but with size not being
-  // a multiple of alignment.
-  test_allocations(65, 32);
-#ifndef TEST_HAS_NO_RTTI
-  {
-    // Check that libc++ doesn't define align_val_t in a versioning namespace.
-    // And that it mangles the same in C++03 through C++17
-#ifdef _MSC_VER
-    // MSVC uses a different C++ ABI with a different name mangling scheme.
-    // The type id name doesn't seem to contain the mangled form at all.
-    assert(typeid(std::align_val_t).name() == std::string("enum std::align_val_t"));
-#else
-    assert(typeid(std::align_val_t).name() == std::string("St11align_val_t"));
-#endif
-  }
-#endif
-
-  return 0;
-}
diff --git a/libcxx/test/libcxx/memory/shared_ptr_array.pass.cpp b/libcxx/test/libcxx/memory/shared_ptr_array.pass.cpp
index 772198304b415..cc8c743830826 100644
--- a/libcxx/test/libcxx/memory/shared_ptr_array.pass.cpp
+++ b/libcxx/test/libcxx/memory/shared_ptr_array.pass.cpp
@@ -8,8 +8,14 @@
 //
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: -fsized-deallocation
-// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation
+
+// These compiler versions and platforms don't enable sized deallocation by default.
+// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
 
 // This test will fail with ASan if the implementation passes different sizes
 // to corresponding allocation and deallocation functions.
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
index 1ce859f6c737e..14189840ce660 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter_stability.pass.cpp
@@ -23,6 +23,7 @@
 #include <flat_map>
 #include <random>
 #include <map>
+#include <vector>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
index 09e18986a7e81..fc3949d70745f 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.iterators/reverse_iterator.pass.cpp
@@ -23,7 +23,7 @@
 #include <deque>
 #include <flat_map>
 #include <functional>
-#include <string>
+#include <vector>
 
 #include <iterator>
 
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
index 81c590ba73a15..76461951f0d3d 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp
@@ -14,6 +14,7 @@
 // type.
 
 #include <flat_map>
+#include <vector>
 
 struct A {
   using Map = std::flat_map<A, A>;
diff --git a/libcxx/test/std/containers/sequences/vector.bool/ctor_exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/ctor_exceptions.pass.cpp
index 928fceab1f078..cb2b7d124edc1 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/ctor_exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/ctor_exceptions.pass.cpp
@@ -12,121 +12,80 @@
 // Check that vector<bool> constructors don't leak memory when an operation inside the constructor throws an exception
 
 #include <cstddef>
-#include <type_traits>
 #include <memory>
+#include <type_traits>
 #include <vector>
 
+#include "../vector/common.h"
 #include "count_new.h"
 #include "test_iterators.h"
 
-template <class T>
-struct Allocator {
-  using value_type      = T;
-  using is_always_equal = std::false_type;
-
-  template <class U>
-  Allocator(const Allocator<U>&) {}
-
-  Allocator(bool should_throw = true) {
-    if (should_throw)
-      throw 0;
-  }
-
-  T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
-  void deallocate(T* ptr, std::size_t n) { std::allocator<T>().deallocate(ptr, n); }
-
-  template <class U>
-  friend bool operator==(const Allocator&, const Allocator<U>&) { return true; }
-};
-
-template <class IterCat>
-struct Iterator {
-  using iterator_category = IterCat;
-  using difference_type   = std::ptrdiff_t;
-  using value_type        = bool;
-  using reference         = bool&;
-  using pointer           = bool*;
-
-  int i_;
-  bool b_ = true;
-  Iterator(int i = 0) : i_(i) {}
-  bool& operator*() {
-    if (i_ == 1)
-      throw 1;
-    return b_;
-  }
-
-  friend bool operator==(const Iterator& lhs, const Iterator& rhs) { return lhs.i_ == rhs.i_; }
-
-  friend bool operator!=(const Iterator& lhs, const Iterator& rhs) { return lhs.i_ != rhs.i_; }
-
-  Iterator& operator++() {
-    ++i_;
-    return *this;
-  }
+int main(int, char**) {
+  using AllocVec = std::vector<bool, throwing_allocator<bool> >;
 
-  Iterator operator++(int) {
-    auto tmp = *this;
-    ++i_;
-    return tmp;
+  try { // Throw in vector() from allocator
+    AllocVec vec;
+  } catch (int) {
   }
-};
-
-void check_new_delete_called() {
-  assert(globalMemCounter.new_called == globalMemCounter.delete_called);
-  assert(globalMemCounter.new_array_called == globalMemCounter.delete_array_called);
-  assert(globalMemCounter.aligned_new_called == globalMemCounter.aligned_delete_called);
-  assert(globalMemCounter.aligned_new_array_called == globalMemCounter.aligned_delete_array_called);
-}
-
-int main(int, char**) {
-  using AllocVec = std::vector<bool, Allocator<bool> >;
+  check_new_delete_called();
 
 #if TEST_STD_VER >= 14
   try { // Throw in vector(size_type, const allocator_type&) from allocator
-    Allocator<bool> alloc(false);
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec get_alloc(0, alloc);
   } catch (int) {
   }
   check_new_delete_called();
-#endif  // TEST_STD_VER >= 14
+#endif // TEST_STD_VER >= 14
+
+  try { // Throw in vector(size_type, const value_type&, const allocator_type&) from allocator
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
+    AllocVec get_alloc(0, true, alloc);
+  } catch (int) {
+  }
+  check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator) from input iterator
-    std::vector<bool> vec((Iterator<std::input_iterator_tag>()), Iterator<std::input_iterator_tag>(2));
+    std::vector<bool> vec(
+        throwing_iterator<bool, std::input_iterator_tag>(), throwing_iterator<bool, std::input_iterator_tag>(2));
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator) from forward iterator
-    std::vector<bool> vec((Iterator<std::forward_iterator_tag>()), Iterator<std::forward_iterator_tag>(2));
+    std::vector<bool> vec(
+        throwing_iterator<bool, std::forward_iterator_tag>(), throwing_iterator<bool, std::forward_iterator_tag>(2));
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator) from allocator
-    int a[] = {1, 2};
-    AllocVec vec(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + 2));
+    bool a[] = {true, true};
+    AllocVec vec(cpp17_input_iterator<bool*>(a), cpp17_input_iterator<bool*>(a + 2));
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from input iterator
     std::allocator<bool> alloc;
-    std::vector<bool> vec(Iterator<std::input_iterator_tag>(), Iterator<std::input_iterator_tag>(2), alloc);
+    std::vector<bool> vec(
+        throwing_iterator<bool, std::input_iterator_tag>(), throwing_iterator<bool, std::input_iterator_tag>(2), alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from forward iterator
     std::allocator<bool> alloc;
-    std::vector<bool> vec(Iterator<std::forward_iterator_tag>(), Iterator<std::forward_iterator_tag>(2), alloc);
+    std::vector<bool> vec(throwing_iterator<bool, std::forward_iterator_tag>(),
+                          throwing_iterator<bool, std::forward_iterator_tag>(2),
+                          alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from allocator
     bool a[] = {true, false};
-    Allocator<bool> alloc(false);
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec vec(cpp17_input_iterator<bool*>(a), cpp17_input_iterator<bool*>(a + 2), alloc);
   } catch (int) {
   }
@@ -134,11 +93,40 @@ int main(int, char**) {
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from allocator
     bool a[] = {true, false};
-    Allocator<bool> alloc(false);
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec vec(forward_iterator<bool*>(a), forward_iterator<bool*>(a + 2), alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
+#if TEST_STD_VER >= 11
+  try { // Throw in vector(const vector&, const allocator_type&) from allocator
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ false);
+    AllocVec vec(alloc);
+    vec.push_back(true);
+    alloc.throw_on_copy_ = true;
+    AllocVec vec2(vec, alloc);
+  } catch (int) {
+  }
+  check_new_delete_called();
+
+  try { // Throw in vector(vector&&, const allocator_type&) from allocator
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ false);
+    AllocVec vec(alloc);
+    vec.push_back(true);
+    alloc.throw_on_copy_ = true;
+    AllocVec vec2(std::move(vec), alloc);
+  } catch (int) {
+  }
+  check_new_delete_called();
+
+  try { // Throw in vector(initializer_list<value_type>, const allocator_type&) constructor from allocator
+    throwing_allocator<bool> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
+    AllocVec vec({true, true}, alloc);
+  } catch (int) {
+  }
+  check_new_delete_called();
+#endif // TEST_STD_VER >= 11
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector.bool/shrink_to_fit.pass.cpp b/libcxx/test/std/containers/sequences/vector.bool/shrink_to_fit.pass.cpp
index f8bcee31964bb..3bc639d0479f9 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/shrink_to_fit.pass.cpp
@@ -11,59 +11,35 @@
 
 // void shrink_to_fit();
 
-#include <vector>
 #include <cassert>
+#include <vector>
 
-#include "test_macros.h"
+#include "increasing_allocator.h"
 #include "min_allocator.h"
+#include "test_macros.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool> v(100);
-        v.push_back(1);
-        v.shrink_to_fit();
-        assert(v.capacity() >= 101);
-        assert(v.size() >= 101);
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  {
+    std::vector<bool> v(100);
+    v.push_back(1);
+    v.shrink_to_fit();
+    assert(v.capacity() >= 101);
+    assert(v.size() >= 101);
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool>> v(100);
-        v.push_back(1);
-        v.shrink_to_fit();
-        assert(v.capacity() >= 101);
-        assert(v.size() >= 101);
-    }
-#endif
-
-    return true;
-}
-
-#if TEST_STD_VER >= 23
-template <typename T>
-struct increasing_allocator {
-  using value_type         = T;
-  std::size_t min_elements = 1000;
-  increasing_allocator()   = default;
-
-  template <typename U>
-  constexpr increasing_allocator(const increasing_allocator<U>& other) noexcept : min_elements(other.min_elements) {}
-
-  constexpr std::allocation_result<T*> allocate_at_least(std::size_t n) {
-    if (n < min_elements)
-      n = min_elements;
-    min_elements += 1000;
-    return std::allocator<T>{}.allocate_at_least(n);
+  {
+    std::vector<bool, min_allocator<bool>> v(100);
+    v.push_back(1);
+    v.shrink_to_fit();
+    assert(v.capacity() >= 101);
+    assert(v.size() >= 101);
   }
-  constexpr T* allocate(std::size_t n) { return allocate_at_least(n).ptr; }
-  constexpr void deallocate(T* p, std::size_t n) noexcept { std::allocator<T>{}.deallocate(p, n); }
-};
+#endif
 
-template <typename T, typename U>
-bool operator==(increasing_allocator<T>, increasing_allocator<U>) {
   return true;
 }
 
+#if TEST_STD_VER >= 23
 // https://github.com/llvm/llvm-project/issues/95161
 constexpr bool test_increasing_allocator() {
   std::vector<bool, increasing_allocator<bool>> v;
@@ -77,16 +53,15 @@ constexpr bool test_increasing_allocator() {
 }
 #endif // TEST_STD_VER >= 23
 
-int main(int, char**)
-{
+int main(int, char**) {
   tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
 #if TEST_STD_VER >= 23
-    test_increasing_allocator();
-    static_assert(test_increasing_allocator());
+  test_increasing_allocator();
+  static_assert(test_increasing_allocator());
 #endif // TEST_STD_VER >= 23
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/common.h b/libcxx/test/std/containers/sequences/vector/common.h
new file mode 100644
index 0000000000000..ff8147ef6b838
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/vector/common.h
@@ -0,0 +1,128 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_STD_CONTAINERS_SEQUENCES_VECTOR_COMMON_H
+#define TEST_STD_CONTAINERS_SEQUENCES_VECTOR_COMMON_H
+
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <type_traits>
+
+#include "count_new.h"
+
+struct throwing_t {
+  int* throw_after_n_ = nullptr;
+  throwing_t() { throw 0; }
+
+  explicit throwing_t(int& throw_after_n) : throw_after_n_(&throw_after_n) {
+    if (throw_after_n == 0)
+      throw 0;
+    --throw_after_n;
+  }
+
+  throwing_t(const throwing_t& rhs) : throw_after_n_(rhs.throw_after_n_) {
+    if (throw_after_n_ == nullptr || *throw_after_n_ == 0)
+      throw 1;
+    --*throw_after_n_;
+  }
+
+  throwing_t& operator=(const throwing_t& rhs) {
+    throw_after_n_ = rhs.throw_after_n_;
+    if (throw_after_n_ == nullptr || *throw_after_n_ == 0)
+      throw 1;
+    --*throw_after_n_;
+    return *this;
+  }
+
+  friend bool operator==(const throwing_t& lhs, const throwing_t& rhs) {
+    return lhs.throw_after_n_ == rhs.throw_after_n_;
+  }
+  friend bool operator!=(const throwing_t& lhs, const throwing_t& rhs) {
+    return lhs.throw_after_n_ != rhs.throw_after_n_;
+  }
+};
+
+template <class T>
+struct throwing_allocator {
+  using value_type = T;
+
+  bool throw_on_copy_ = false;
+
+  explicit throwing_allocator(bool throw_on_ctor = true) {
+    if (throw_on_ctor)
+      throw 0;
+  }
+
+  explicit throwing_allocator(bool throw_on_ctor, bool throw_on_copy) : throw_on_copy_(throw_on_copy) {
+    if (throw_on_ctor)
+      throw 0;
+  }
+
+  throwing_allocator(const throwing_allocator& rhs) : throw_on_copy_(rhs.throw_on_copy_) {
+    if (throw_on_copy_)
+      throw 0;
+  }
+
+  template <class U>
+  throwing_allocator(const throwing_allocator<U>& rhs) : throw_on_copy_(rhs.throw_on_copy_) {
+    if (throw_on_copy_)
+      throw 0;
+  }
+
+  T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
+  void deallocate(T* ptr, std::size_t n) { std::allocator<T>().deallocate(ptr, n); }
+
+  template <class U>
+  friend bool operator==(const throwing_allocator&, const throwing_allocator<U>&) {
+    return true;
+  }
+};
+
+template <class T, class IterCat>
+struct throwing_iterator {
+  using iterator_category = IterCat;
+  using difference_type   = std::ptrdiff_t;
+  using value_type        = T;
+  using reference         = T&;
+  using pointer           = T*;
+
+  int i_;
+  T v_;
+
+  explicit throwing_iterator(int i = 0, const T& v = T()) : i_(i), v_(v) {}
+
+  reference operator*() {
+    if (i_ == 1)
+      throw 1;
+    return v_;
+  }
+
+  friend bool operator==(const throwing_iterator& lhs, const throwing_iterator& rhs) { return lhs.i_ == rhs.i_; }
+  friend bool operator!=(const throwing_iterator& lhs, const throwing_iterator& rhs) { return lhs.i_ != rhs.i_; }
+
+  throwing_iterator& operator++() {
+    ++i_;
+    return *this;
+  }
+
+  throwing_iterator operator++(int) {
+    auto tmp = *this;
+    ++i_;
+    return tmp;
+  }
+};
+
+inline void check_new_delete_called() {
+  assert(globalMemCounter.new_called == globalMemCounter.delete_called);
+  assert(globalMemCounter.new_array_called == globalMemCounter.delete_array_called);
+  assert(globalMemCounter.aligned_new_called == globalMemCounter.aligned_delete_called);
+  assert(globalMemCounter.aligned_new_array_called == globalMemCounter.aligned_delete_array_called);
+}
+
+#endif // TEST_STD_CONTAINERS_SEQUENCES_VECTOR_COMMON_H
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
index e39afb2d48f0a..48eb6ed85d47c 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit.pass.cpp
@@ -10,92 +10,70 @@
 
 // void shrink_to_fit();
 
-#include <vector>
 #include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
-#include "min_allocator.h"
+#include <vector>
+
 #include "asan_testing.h"
+#include "increasing_allocator.h"
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
 TEST_CONSTEXPR_CXX20 bool tests() {
-    {
-        std::vector<int> v(100);
-        v.push_back(1);
-        assert(is_contiguous_container_asan_correct(v));
-        v.shrink_to_fit();
-        assert(v.capacity() == 101);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-        std::vector<int, limited_allocator<int, 401> > v(100);
-        v.push_back(1);
-        assert(is_contiguous_container_asan_correct(v));
-        v.shrink_to_fit();
-        assert(v.capacity() == 101);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+  {
+    std::vector<int> v(100);
+    v.push_back(1);
+    assert(is_contiguous_container_asan_correct(v));
+    v.shrink_to_fit();
+    assert(v.capacity() == 101);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, limited_allocator<int, 401> > v(100);
+    v.push_back(1);
+    assert(is_contiguous_container_asan_correct(v));
+    v.shrink_to_fit();
+    assert(v.capacity() == 101);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #ifndef TEST_HAS_NO_EXCEPTIONS
-    if (!TEST_IS_CONSTANT_EVALUATED) {
-        std::vector<int, limited_allocator<int, 400> > v(100);
-        v.push_back(1);
-        assert(is_contiguous_container_asan_correct(v));
-        v.shrink_to_fit();
-        LIBCPP_ASSERT(v.capacity() == 200); // assumes libc++'s 2x growth factor
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    std::vector<int, limited_allocator<int, 400> > v(100);
+    v.push_back(1);
+    assert(is_contiguous_container_asan_correct(v));
+    v.shrink_to_fit();
+    LIBCPP_ASSERT(v.capacity() == 200); // assumes libc++'s 2x growth factor
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
 #if TEST_STD_VER >= 11
-    {
-        std::vector<int, min_allocator<int>> v(100);
-        v.push_back(1);
-        assert(is_contiguous_container_asan_correct(v));
-        v.shrink_to_fit();
-        assert(v.capacity() == 101);
-        assert(v.size() == 101);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-    {
-      std::vector<int, safe_allocator<int>> v(100);
-      v.push_back(1);
-      assert(is_contiguous_container_asan_correct(v));
-      v.shrink_to_fit();
-      assert(v.capacity() == 101);
-      assert(v.size() == 101);
-      assert(is_contiguous_container_asan_correct(v));
-    }
-#endif
-
-    return true;
-}
-
-#if TEST_STD_VER >= 23
-template <typename T>
-struct increasing_allocator {
-  using value_type         = T;
-  std::size_t min_elements = 1000;
-  increasing_allocator()   = default;
-
-  template <typename U>
-  constexpr increasing_allocator(const increasing_allocator<U>& other) noexcept : min_elements(other.min_elements) {}
-
-  constexpr std::allocation_result<T*> allocate_at_least(std::size_t n) {
-    if (n < min_elements)
-      n = min_elements;
-    min_elements += 1000;
-    return std::allocator<T>{}.allocate_at_least(n);
+  {
+    std::vector<int, min_allocator<int>> v(100);
+    v.push_back(1);
+    assert(is_contiguous_container_asan_correct(v));
+    v.shrink_to_fit();
+    assert(v.capacity() == 101);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    std::vector<int, safe_allocator<int>> v(100);
+    v.push_back(1);
+    assert(is_contiguous_container_asan_correct(v));
+    v.shrink_to_fit();
+    assert(v.capacity() == 101);
+    assert(v.size() == 101);
+    assert(is_contiguous_container_asan_correct(v));
   }
-  constexpr T* allocate(std::size_t n) { return allocate_at_least(n).ptr; }
-  constexpr void deallocate(T* p, std::size_t n) noexcept { std::allocator<T>{}.deallocate(p, n); }
-};
+#endif
 
-template <typename T, typename U>
-bool operator==(increasing_allocator<T>, increasing_allocator<U>) {
   return true;
 }
 
+#if TEST_STD_VER >= 23
 // https://github.com/llvm/llvm-project/issues/95161
 constexpr bool test_increasing_allocator() {
   std::vector<int, increasing_allocator<int>> v;
@@ -111,16 +89,15 @@ constexpr bool test_increasing_allocator() {
 }
 #endif // TEST_STD_VER >= 23
 
-int main(int, char**)
-{
+int main(int, char**) {
   tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
 #if TEST_STD_VER >= 23
-    test_increasing_allocator();
-    static_assert(test_increasing_allocator());
+  test_increasing_allocator();
+  static_assert(test_increasing_allocator());
 #endif
 
-    return 0;
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
index 09355688042f9..679eec2413793 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/exceptions.pass.cpp
@@ -16,98 +16,13 @@
 #include <type_traits>
 #include <vector>
 
+#include "../common.h"
 #include "count_new.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
 
-template <class T>
-struct Allocator {
-  using value_type      = T;
-  using is_always_equal = std::false_type;
-
-  template <class U>
-  Allocator(const Allocator<U>&) {}
-
-  Allocator(bool should_throw = true) {
-    if (should_throw)
-      throw 0;
-  }
-
-  T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
-  void deallocate(T* ptr, std::size_t n) { std::allocator<T>().deallocate(ptr, n); }
-
-  template <class U>
-  friend bool operator==(const Allocator&, const Allocator<U>&) {
-    return true;
-  }
-};
-
-struct ThrowingT {
-  int* throw_after_n_ = nullptr;
-  ThrowingT() { throw 0; }
-
-  ThrowingT(int& throw_after_n) : throw_after_n_(&throw_after_n) {
-    if (throw_after_n == 0)
-      throw 0;
-    --throw_after_n;
-  }
-
-  ThrowingT(const ThrowingT& rhs) : throw_after_n_(rhs.throw_after_n_) {
-    if (throw_after_n_ == nullptr || *throw_after_n_ == 0)
-      throw 1;
-    --*throw_after_n_;
-  }
-
-  ThrowingT& operator=(const ThrowingT& rhs) {
-    throw_after_n_ = rhs.throw_after_n_;
-    if (throw_after_n_ == nullptr || *throw_after_n_ == 0)
-      throw 1;
-    --*throw_after_n_;
-    return *this;
-  }
-};
-
-template <class IterCat>
-struct Iterator {
-  using iterator_category = IterCat;
-  using difference_type   = std::ptrdiff_t;
-  using value_type        = int;
-  using reference         = int&;
-  using pointer           = int*;
-
-  int i_;
-  Iterator(int i = 0) : i_(i) {}
-  int& operator*() {
-    if (i_ == 1)
-      throw 1;
-    return i_;
-  }
-
-  friend bool operator==(const Iterator& lhs, const Iterator& rhs) { return lhs.i_ == rhs.i_; }
-
-  friend bool operator!=(const Iterator& lhs, const Iterator& rhs) { return lhs.i_ != rhs.i_; }
-
-  Iterator& operator++() {
-    ++i_;
-    return *this;
-  }
-
-  Iterator operator++(int) {
-    auto tmp = *this;
-    ++i_;
-    return tmp;
-  }
-};
-
-void check_new_delete_called() {
-  assert(globalMemCounter.new_called == globalMemCounter.delete_called);
-  assert(globalMemCounter.new_array_called == globalMemCounter.delete_array_called);
-  assert(globalMemCounter.aligned_new_called == globalMemCounter.aligned_delete_called);
-  assert(globalMemCounter.aligned_new_array_called == globalMemCounter.aligned_delete_array_called);
-}
-
 int main(int, char**) {
-  using AllocVec = std::vector<int, Allocator<int> >;
+  using AllocVec = std::vector<int, throwing_allocator<int> >;
   try { // vector()
     AllocVec vec;
   } catch (int) {
@@ -115,7 +30,7 @@ int main(int, char**) {
   check_new_delete_called();
 
   try { // Throw in vector(size_type) from type
-    std::vector<ThrowingT> get_alloc(1);
+    std::vector<throwing_t> get_alloc(1);
   } catch (int) {
   }
   check_new_delete_called();
@@ -123,21 +38,21 @@ int main(int, char**) {
 #if TEST_STD_VER >= 14
   try { // Throw in vector(size_type, value_type) from type
     int throw_after = 1;
-    ThrowingT v(throw_after);
-    std::vector<ThrowingT> get_alloc(1, v);
+    throwing_t v(throw_after);
+    std::vector<throwing_t> get_alloc(1, v);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(size_type, const allocator_type&) from allocator
-    Allocator<int> alloc(false);
+    throwing_allocator<int> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec get_alloc(0, alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(size_type, const allocator_type&) from the type
-    std::vector<ThrowingT> vec(1, std::allocator<ThrowingT>());
+    std::vector<throwing_t> vec(1, std::allocator<throwing_t>());
   } catch (int) {
   }
   check_new_delete_called();
@@ -145,20 +60,22 @@ int main(int, char**) {
 
   try { // Throw in vector(size_type, value_type, const allocator_type&) from the type
     int throw_after = 1;
-    ThrowingT v(throw_after);
-    std::vector<ThrowingT> vec(1, v, std::allocator<ThrowingT>());
+    throwing_t v(throw_after);
+    std::vector<throwing_t> vec(1, v, std::allocator<throwing_t>());
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator) from input iterator
-    std::vector<int> vec((Iterator<std::input_iterator_tag>()), Iterator<std::input_iterator_tag>(2));
+    std::vector<int> vec(
+        (throwing_iterator<int, std::input_iterator_tag>()), throwing_iterator<int, std::input_iterator_tag>(2));
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator) from forward iterator
-    std::vector<int> vec((Iterator<std::forward_iterator_tag>()), Iterator<std::forward_iterator_tag>(2));
+    std::vector<int> vec(
+        (throwing_iterator<int, std::forward_iterator_tag>()), throwing_iterator<int, std::forward_iterator_tag>(2));
   } catch (int) {
   }
   check_new_delete_called();
@@ -172,39 +89,40 @@ int main(int, char**) {
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from input iterator
     std::allocator<int> alloc;
-    std::vector<int> vec(Iterator<std::input_iterator_tag>(), Iterator<std::input_iterator_tag>(2), alloc);
+    std::vector<int> vec(
+        throwing_iterator<int, std::input_iterator_tag>(), throwing_iterator<int, std::input_iterator_tag>(2), alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from forward iterator
     std::allocator<int> alloc;
-    std::vector<int> vec(Iterator<std::forward_iterator_tag>(), Iterator<std::forward_iterator_tag>(2), alloc);
+    std::vector<int> vec(throwing_iterator<int, std::forward_iterator_tag>(),
+                         throwing_iterator<int, std::forward_iterator_tag>(2),
+                         alloc);
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from allocator
     int a[] = {1, 2};
-    Allocator<int> alloc(false);
+    throwing_allocator<int> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec vec(cpp17_input_iterator<int*>(a), cpp17_input_iterator<int*>(a + 2), alloc);
   } catch (int) {
-    // FIXME: never called.
   }
   check_new_delete_called();
 
   try { // Throw in vector(InputIterator, InputIterator, const allocator_type&) from allocator
     int a[] = {1, 2};
-    Allocator<int> alloc(false);
+    throwing_allocator<int> alloc(/*throw_on_ctor = */ false, /*throw_on_copy = */ true);
     AllocVec vec(forward_iterator<int*>(a), forward_iterator<int*>(a + 2), alloc);
   } catch (int) {
-    // FIXME: never called.
   }
   check_new_delete_called();
 
   try { // Throw in vector(const vector&) from type
-    std::vector<ThrowingT> vec;
-    int throw_after = 0;
+    std::vector<throwing_t> vec;
+    int throw_after = 1;
     vec.emplace_back(throw_after);
     auto vec2 = vec;
   } catch (int) {
@@ -212,20 +130,20 @@ int main(int, char**) {
   check_new_delete_called();
 
   try { // Throw in vector(const vector&, const allocator_type&) from type
-    std::vector<ThrowingT> vec;
+    std::vector<throwing_t> vec;
     int throw_after = 1;
     vec.emplace_back(throw_after);
-    std::vector<ThrowingT> vec2(vec, std::allocator<int>());
+    std::vector<throwing_t> vec2(vec, std::allocator<int>());
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(vector&&, const allocator_type&) from type during element-wise move
-    std::vector<ThrowingT, test_allocator<ThrowingT> > vec(test_allocator<ThrowingT>(1));
+    std::vector<throwing_t, test_allocator<throwing_t> > vec(test_allocator<throwing_t>(1));
     int throw_after = 10;
-    ThrowingT v(throw_after);
+    throwing_t v(throw_after);
     vec.insert(vec.end(), 6, v);
-    std::vector<ThrowingT, test_allocator<ThrowingT> > vec2(std::move(vec), test_allocator<ThrowingT>(2));
+    std::vector<throwing_t, test_allocator<throwing_t> > vec2(std::move(vec), test_allocator<throwing_t>(2));
   } catch (int) {
   }
   check_new_delete_called();
@@ -233,14 +151,14 @@ int main(int, char**) {
 #if TEST_STD_VER >= 11
   try { // Throw in vector(initializer_list<value_type>) from type
     int throw_after = 1;
-    std::vector<ThrowingT> vec({ThrowingT(throw_after)});
+    std::vector<throwing_t> vec({throwing_t(throw_after)});
   } catch (int) {
   }
   check_new_delete_called();
 
   try { // Throw in vector(initializer_list<value_type>, const allocator_type&) constructor from type
     int throw_after = 1;
-    std::vector<ThrowingT> vec({ThrowingT(throw_after)}, std::allocator<ThrowingT>());
+    std::vector<throwing_t> vec({throwing_t(throw_after)}, std::allocator<throwing_t>());
   } catch (int) {
   }
   check_new_delete_called();
diff --git a/libcxx/test/std/language.support/support.dynamic/align_val_t.pass.cpp b/libcxx/test/std/language.support/support.dynamic/align_val_t.pass.cpp
index 28c72f0be7aed..b898b37ebc7e9 100644
--- a/libcxx/test/std/language.support/support.dynamic/align_val_t.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/align_val_t.pass.cpp
@@ -15,26 +15,50 @@
 // XFAIL: target={{.+}}-zos{{.*}}
 
 #include <new>
+#include <cassert>
+#include <cstddef>
+#include <string>
 #include <type_traits>
+#include <typeinfo>
 
 #include "test_macros.h"
 
-int main(int, char**) {
+constexpr bool test() {
+  static_assert(std::is_enum<std::align_val_t>::value, "");
+  static_assert(std::is_same<std::underlying_type<std::align_val_t>::type, std::size_t>::value, "");
+  static_assert(!std::is_constructible<std::align_val_t, std::size_t>::value, "");
+  static_assert(!std::is_constructible<std::size_t, std::align_val_t>::value, "");
+
   {
-    static_assert(std::is_enum<std::align_val_t>::value, "");
-    static_assert(std::is_same<std::underlying_type<std::align_val_t>::type, std::size_t>::value, "");
-    static_assert(!std::is_constructible<std::align_val_t, std::size_t>::value, "");
-    static_assert(!std::is_constructible<std::size_t, std::align_val_t>::value, "");
+    auto a = std::align_val_t(0);
+    auto b = std::align_val_t(32);
+    auto c = std::align_val_t(-1);
+    assert(a != b);
+    assert(a == std::align_val_t(0));
+    assert(b == std::align_val_t(32));
+    assert(static_cast<std::size_t>(c) == static_cast<std::size_t>(-1));
   }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test(), "");
+
+#if defined(_LIBCPP_VERSION) && !defined(TEST_HAS_NO_RTTI)
   {
-    constexpr auto a = std::align_val_t(0);
-    constexpr auto b = std::align_val_t(32);
-    constexpr auto c = std::align_val_t(-1);
-    static_assert(a != b, "");
-    static_assert(a == std::align_val_t(0), "");
-    static_assert(b == std::align_val_t(32), "");
-    static_assert(static_cast<std::size_t>(c) == (std::size_t)-1, "");
+    // Check that libc++ doesn't define align_val_t in a versioning namespace.
+    // And that it mangles the same in C++03 through C++17
+#  ifdef _MSC_VER
+    // MSVC uses a different C++ ABI with a different name mangling scheme.
+    // The type id name doesn't seem to contain the mangled form at all.
+    assert(typeid(std::align_val_t).name() == std::string("enum std::align_val_t"));
+#  else
+    assert(typeid(std::align_val_t).name() == std::string("St11align_val_t"));
+#  endif
   }
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
index dd8090aca5b28..c9b59ecaff396 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
@@ -34,13 +34,12 @@ void my_new_handler() {
 }
 
 int main(int, char**) {
-    // Test that we can call the function directly
-    {
-        void* x = operator new[](10, static_cast<std::align_val_t>(64));
+    test_with_interesting_alignments([](std::size_t size, std::size_t alignment) {
+        void* x = operator new[](size, static_cast<std::align_val_t>(alignment));
         assert(x != nullptr);
-        assert(reinterpret_cast<std::uintptr_t>(x) % 64 == 0);
-        operator delete[](x, static_cast<std::align_val_t>(64));
-    }
+        assert(reinterpret_cast<std::uintptr_t>(x) % alignment == 0);
+        operator delete[](x, static_cast<std::align_val_t>(alignment));
+    });
 
     // Test that the new handler is called if allocation fails
     {
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align_nothrow.pass.cpp
index 6ae8ceaf534e4..1bdae8d597f00 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align_nothrow.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align_nothrow.pass.cpp
@@ -34,13 +34,12 @@ void my_new_handler() {
 }
 
 int main(int, char**) {
-    // Test that we can call the function directly
-    {
-        void* x = operator new[](10, static_cast<std::align_val_t>(64), std::nothrow);
+    test_with_interesting_alignments([](std::size_t size, std::size_t alignment) {
+        void* x = operator new[](size, static_cast<std::align_val_t>(alignment), std::nothrow);
         assert(x != nullptr);
-        assert(reinterpret_cast<std::uintptr_t>(x) % 64 == 0);
-        operator delete[](x, static_cast<std::align_val_t>(64), std::nothrow);
-    }
+        assert(reinterpret_cast<std::uintptr_t>(x) % alignment == 0);
+        operator delete[](x, static_cast<std::align_val_t>(alignment), std::nothrow);
+    });
 
     // Test that the new handler is called and we return nullptr if allocation fails
     {
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
similarity index 67%
rename from libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp
rename to libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
index 6077318278a1b..f0ad2c0e67df3 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array.pass.cpp
@@ -6,15 +6,29 @@
 //
 //===----------------------------------------------------------------------===//
 
-// test sized operator delete[] replacement.
+// Test sized operator delete[] replacement.
 
-// Note that sized delete operator definitions below are simply ignored
-// when sized deallocation is not supported, e.g., prior to C++14.
+// UNSUPPORTED: c++03, c++11
+
+// These compiler versions and platforms don't enable sized deallocation by default.
+// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
+
+// Android clang-r536225 identifies as clang-19.0 but it predates the real
+// LLVM 19.0.0, so it also leaves sized deallocation off by default.
+// UNSUPPORTED: android && clang-19.0
 
 // UNSUPPORTED: sanitizer-new-delete
 
-// REQUIRES: -fsized-deallocation
-// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation
+// Sized deallocation was introduced in LLVM 11
+// XFAIL: using-built-library-before-llvm-11
+
+// AIX, and z/OS default to -fno-sized-deallocation.
+// XFAIL: target={{.+}}-aix{{.*}}, target={{.+}}-zos{{.*}}
 
 #if !defined(__cpp_sized_deallocation)
 # error __cpp_sized_deallocation should be defined
@@ -76,5 +90,5 @@ int main(int, char**)
     assert(0 == unsized_delete_nothrow_called);
     assert(1 == sized_delete_called);
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp
deleted file mode 100644
index 52c4e3e0f69b6..0000000000000
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// test sized operator delete[] replacement.
-
-// These compiler versions don't enable sized deallocation by default.
-// UNSUPPORTED: clang-17, clang-18
-
-// Android clang-r536225 identifies as clang-19.0 but it predates the real
-// LLVM 19.0.0, so it also leaves sized deallocation off by default.
-// UNSUPPORTED: android && clang-19.0
-
-// UNSUPPORTED: sanitizer-new-delete, c++03, c++11
-// XFAIL: apple-clang
-// XFAIL: using-built-library-before-llvm-11
-
-// AIX, z/OS, and MinGW default to -fno-sized-deallocation.
-// XFAIL: target={{.+}}-aix{{.*}}, target={{.+}}-zos{{.*}}, target={{.+}}-windows-gnu
-
-#include <new>
-#include <cstddef>
-#include <cstdlib>
-#include <cassert>
-
-#include "test_macros.h"
-
-int unsized_delete_called = 0;
-int unsized_delete_nothrow_called = 0;
-int sized_delete_called = 0;
-
-void operator delete[](void* p) TEST_NOEXCEPT
-{
-    ++unsized_delete_called;
-    std::free(p);
-}
-
-void operator delete[](void* p, const std::nothrow_t&) TEST_NOEXCEPT
-{
-    ++unsized_delete_nothrow_called;
-    std::free(p);
-}
-
-void operator delete[](void* p, std::size_t) TEST_NOEXCEPT
-{
-    ++sized_delete_called;
-    std::free(p);
-}
-
-// NOTE: Use a class with a non-trivial destructor as the test type in order
-// to ensure the correct overload is called.
-// C++14 5.3.5 [expr.delete]p10
-// - If the type is complete and if, for the second alternative (delete array)
-//   only, the operand is a pointer to a class type with a non-trivial
-//   destructor or a (possibly multi-dimensional) array thereof, the function
-//   with two parameters is selected.
-// - Otherwise, it is unspecified which of the two deallocation functions is
-//   selected.
-struct A { ~A() {} };
-
-int main(int, char**)
-{
-    A* x = new A[3];
-    assert(0 == unsized_delete_called);
-    assert(0 == unsized_delete_nothrow_called);
-    assert(0 == sized_delete_called);
-
-    delete [] x;
-    assert(0 == unsized_delete_called);
-    assert(0 == unsized_delete_nothrow_called);
-    assert(1 == sized_delete_called);
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
index dbb10a76ad9e9..5d321f08282b2 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
@@ -34,13 +34,12 @@ void my_new_handler() {
 }
 
 int main(int, char**) {
-    // Test that we can call the function directly
-    {
-        void* x = operator new(10, static_cast<std::align_val_t>(64));
+    test_with_interesting_alignments([](std::size_t size, std::size_t alignment) {
+        void* x = operator new(size, static_cast<std::align_val_t>(alignment));
         assert(x != nullptr);
-        assert(reinterpret_cast<std::uintptr_t>(x) % 64 == 0);
-        operator delete(x, static_cast<std::align_val_t>(64));
-    }
+        assert(reinterpret_cast<std::uintptr_t>(x) % alignment == 0);
+        operator delete(x, static_cast<std::align_val_t>(alignment));
+    });
 
     // Test that the new handler is called if allocation fails
     {
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp
index b9d8ea2f4e494..3e5ba574110fd 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp
@@ -34,13 +34,12 @@ void my_new_handler() {
 }
 
 int main(int, char**) {
-    // Test that we can call the function directly
-    {
-        void* x = operator new(10, static_cast<std::align_val_t>(64), std::nothrow);
+    test_with_interesting_alignments([](std::size_t size, std::size_t alignment) {
+        void* x = operator new(size, static_cast<std::align_val_t>(alignment), std::nothrow);
         assert(x != nullptr);
-        assert(reinterpret_cast<std::uintptr_t>(x) % 64 == 0);
-        operator delete(x, static_cast<std::align_val_t>(64), std::nothrow);
-    }
+        assert(reinterpret_cast<std::uintptr_t>(x) % alignment == 0);
+        operator delete(x, static_cast<std::align_val_t>(alignment), std::nothrow);
+    });
 
     // Test that the new handler is called and we return nullptr if allocation fails
     {
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
similarity index 62%
rename from libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp
rename to libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
index e00339761ec24..fd52df451afc8 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete.pass.cpp
@@ -8,19 +8,35 @@
 
 // Test sized operator delete replacement.
 
-// These compiler versions do not enable sized deallocation by default.
-// UNSUPPORTED: clang-17, clang-18
+// UNSUPPORTED: c++03, c++11
+
+// These compiler versions and platforms don't enable sized deallocation by default.
+// ADDITIONAL_COMPILE_FLAGS(clang-17): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(clang-18): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-15): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(apple-clang-16): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=x86_64-w64-windows-gnu): -fsized-deallocation
+// ADDITIONAL_COMPILE_FLAGS(target=i686-w64-windows-gnu): -fsized-deallocation
 
 // Android clang-r536225 identifies as clang-19.0 but it predates the real
 // LLVM 19.0.0, so it also leaves sized deallocation off by default.
 // UNSUPPORTED: android && clang-19.0
 
-// UNSUPPORTED: sanitizer-new-delete, c++03, c++11
-// XFAIL: apple-clang
+// UNSUPPORTED: sanitizer-new-delete
+
+// Sized deallocation was introduced in LLVM 11
 // XFAIL: using-built-library-before-llvm-11
 
-// AIX, z/OS, and MinGW default to -fno-sized-deallocation.
-// XFAIL: target={{.+}}-aix{{.*}}, target={{.+}}-zos{{.*}}, target={{.+}}-windows-gnu
+// AIX, and z/OS default to -fno-sized-deallocation.
+// XFAIL: target={{.+}}-aix{{.*}}, target={{.+}}-zos{{.*}}
+
+#if !defined(__cpp_sized_deallocation)
+# error __cpp_sized_deallocation should be defined
+#endif
+
+#if !(__cpp_sized_deallocation >= 201309L)
+# error expected __cpp_sized_deallocation >= 201309L
+#endif
 
 #include <new>
 #include <cstddef>
@@ -65,5 +81,5 @@ int main(int, char**)
     assert(1 == sized_delete_called);
     assert(0 == unsized_delete_nothrow_called);
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp
deleted file mode 100644
index ff88662d20875..0000000000000
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp
+++ /dev/null
@@ -1,71 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// test sized operator delete replacement.
-
-// Note that sized delete operator definitions below are simply ignored
-// when sized deallocation is not supported, e.g., prior to C++14.
-
-// UNSUPPORTED: sanitizer-new-delete
-
-// REQUIRES: -fsized-deallocation
-// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation -O3
-
-#if !defined(__cpp_sized_deallocation)
-# error __cpp_sized_deallocation should be defined
-#endif
-
-#if !(__cpp_sized_deallocation >= 201309L)
-# error expected __cpp_sized_deallocation >= 201309L
-#endif
-
-#include <new>
-#include <cstddef>
-#include <cstdlib>
-#include <cassert>
-
-#include "test_macros.h"
-
-int unsized_delete_called = 0;
-int unsized_delete_nothrow_called = 0;
-int sized_delete_called = 0;
-
-void operator delete(void* p) TEST_NOEXCEPT
-{
-    ++unsized_delete_called;
-    std::free(p);
-}
-
-void operator delete(void* p, const std::nothrow_t&) TEST_NOEXCEPT
-{
-    ++unsized_delete_nothrow_called;
-    std::free(p);
-}
-
-void operator delete(void* p, std::size_t) TEST_NOEXCEPT
-{
-    ++sized_delete_called;
-    std::free(p);
-}
-
-int main(int, char**)
-{
-    int *x = new int(42);
-    DoNotOptimize(x);
-    assert(0 == sized_delete_called);
-    assert(0 == unsized_delete_called);
-    assert(0 == unsized_delete_nothrow_called);
-
-    delete x;
-    DoNotOptimize(x);
-    assert(1 == sized_delete_called);
-    assert(0 == unsized_delete_called);
-    assert(0 == unsized_delete_nothrow_called);
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/types.h b/libcxx/test/std/language.support/support.dynamic/new.delete/types.h
index f19de73bf4417..46a2d6ce7f011 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/types.h
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/types.h
@@ -60,4 +60,35 @@ struct alignas(std::max_align_t) MaxAligned {
 };
 #endif // TEST_STD_VER >= 17
 
+template <class F>
+void test_with_interesting_alignments(F f) {
+  // First, check the basic case, a large allocation with alignment == size.
+  f(/* size */ 64, /* alignment */ 64);
+
+  // Size being a multiple of alignment also needs to be supported.
+  f(/* size */ 64, /* alignment */ 32);
+
+  // Test with a non power-of-two size.
+  f(/* size */ 10, /* alignment */ 64);
+
+  // When aligned allocation is implemented using aligned_alloc,
+  // that function requires a minimum alignment of sizeof(void*).
+  //
+  // Check that we can also create overaligned allocations with
+  // an alignment argument less than sizeof(void*).
+  f(/* size */ 2, /* alignment */ 2);
+
+  // When implemented using the C11 aligned_alloc() function,
+  // that requires that size be a multiple of alignment.
+  // However, the C++ operator new has no such requirements.
+  //
+  // Check that we can create an overaligned allocation that does
+  // adhere to not have this constraint.
+  f(/* size */ 1, /* alignment */ 128);
+
+  // Finally, test size > alignment, but with size not being
+  // a multiple of alignment.
+  f(/* size */ 65, /* alignment */ 32);
+}
+
 #endif // TEST_STD_LANGUAGE_SUPPORT_SUPPORT_DYNAMIC_NEW_DELETE_TYPES_H
diff --git a/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp b/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
index 6a9ec1a2ffec2..975b53a763afa 100644
--- a/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
@@ -27,17 +27,7 @@ constexpr struct {
   int x;
   int y;
   int expect;
-} Cases[] = {
-    {0, 0, 0},
-    {1, 0, 1},
-    {0, 1, 1},
-    {1, 1, 1},
-    {2, 3, 1},
-    {2, 4, 2},
-    {36, 17, 1},
-    {36, 18, 18}
-};
-
+} Cases[] = {{0, 0, 0}, {1, 0, 1}, {0, 1, 1}, {1, 1, 1}, {2, 3, 1}, {2, 4, 2}, {11, 9, 1}, {36, 17, 1}, {36, 18, 18}};
 
 template <typename Input1, typename Input2, typename Output>
 constexpr bool test0(int in1, int in2, int out)
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
index 6f5e43d1341f5..2d901e7afe2b6 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
@@ -10,12 +10,13 @@
 
 // void shrink_to_fit(); // constexpr since C++20
 
-#include <string>
 #include <cassert>
+#include <string>
 
-#include "test_macros.h"
-#include "min_allocator.h"
 #include "asan_testing.h"
+#include "increasing_allocator.h"
+#include "min_allocator.h"
+#include "test_macros.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s) {
@@ -64,30 +65,6 @@ TEST_CONSTEXPR_CXX20 bool test() {
 }
 
 #if TEST_STD_VER >= 23
-std::size_t min_bytes = 1000;
-
-template <typename T>
-struct increasing_allocator {
-  using value_type       = T;
-  increasing_allocator() = default;
-  template <typename U>
-  increasing_allocator(const increasing_allocator<U>&) noexcept {}
-  std::allocation_result<T*> allocate_at_least(std::size_t n) {
-    std::size_t allocation_amount = n * sizeof(T);
-    if (allocation_amount < min_bytes)
-      allocation_amount = min_bytes;
-    min_bytes += 1000;
-    return {static_cast<T*>(::operator new(allocation_amount)), allocation_amount / sizeof(T)};
-  }
-  T* allocate(std::size_t n) { return allocate_at_least(n).ptr; }
-  void deallocate(T* p, std::size_t) noexcept { ::operator delete(static_cast<void*>(p)); }
-};
-
-template <typename T, typename U>
-bool operator==(increasing_allocator<T>, increasing_allocator<U>) {
-  return true;
-}
-
 // https://github.com/llvm/llvm-project/issues/95161
 void test_increasing_allocator() {
   std::basic_string<char, std::char_traits<char>, increasing_allocator<char>> s{
diff --git a/libcxx/test/std/utilities/expected/expected.expected/monadic/and_then.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/monadic/and_then.pass.cpp
index 293c2d39d4a32..3ac58b34eff8c 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/monadic/and_then.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/monadic/and_then.pass.cpp
@@ -263,8 +263,8 @@ constexpr void test_val_types() {
 constexpr void test_sfinae() {
   std::expected<NonConst, int> e(std::unexpected<int>(2));
   auto l = [](auto&& x) { return x.non_const(); };
-  e.and_then(l);
-  std::move(e).and_then(l);
+  (void)e.and_then(l);
+  (void)std::move(e).and_then(l);
 }
 
 constexpr void test_move_only_error_type() {
@@ -272,14 +272,14 @@ constexpr void test_move_only_error_type() {
   {
     std::expected<int, MoveOnlyErrorType> e;
     auto l = [](int) { return std::expected<int, MoveOnlyErrorType>{}; };
-    std::move(e).and_then(l);
+    (void)std::move(e).and_then(l);
   }
 
   // Test const&&
   {
     const std::expected<int, MoveOnlyErrorType> e;
     auto l = [](const int) { return std::expected<int, MoveOnlyErrorType>{}; };
-    std::move(e).and_then(l);
+    (void)std::move(e).and_then(l);
   }
 }
 
@@ -296,10 +296,10 @@ constexpr bool test() {
     return std::expected<int, int>();
   };
 
-  e.and_then(never_called);
-  std::move(e).and_then(never_called);
-  ce.and_then(never_called);
-  std::move(ce).and_then(never_called);
+  (void)e.and_then(never_called);
+  (void)std::move(e).and_then(never_called);
+  (void)ce.and_then(never_called);
+  (void)std::move(ce).and_then(never_called);
 
   return true;
 }
diff --git a/libcxx/test/std/utilities/expected/expected.expected/monadic/or_else.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/monadic/or_else.pass.cpp
index b472a2756be9f..2587ca60a8efd 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/monadic/or_else.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/monadic/or_else.pass.cpp
@@ -178,8 +178,8 @@ struct NonConst {
 constexpr void test_sfinae() {
   std::expected<int, NonConst> e{1};
   auto l = [](auto&& x) { return x.non_const(); };
-  e.or_else(l);
-  std::move(e).or_else(l);
+  (void)e.or_else(l);
+  (void)std::move(e).or_else(l);
 }
 
 constexpr void test_move_only_error_type() {
@@ -187,28 +187,28 @@ constexpr void test_move_only_error_type() {
   {
       std::expected<int, MoveOnlyErrorType> e;
       auto l = [](MoveOnlyErrorType&) { return std::expected<int, int>{}; };
-      e.or_else(l);
+      (void)e.or_else(l);
   }
 
   // Test const&
   {
       const std::expected<int, MoveOnlyErrorType> e;
       auto l = [](const MoveOnlyErrorType&) { return std::expected<int, int>{}; };
-      e.or_else(l);
+      (void)e.or_else(l);
   }
 
   // Test &&
   {
       std::expected<int, MoveOnlyErrorType> e;
       auto l = [](MoveOnlyErrorType&&) { return std::expected<int, int>{}; };
-      std::move(e).or_else(l);
+      (void)std::move(e).or_else(l);
   }
 
   // Test const&&
   {
       const std::expected<int, MoveOnlyErrorType> e;
       auto l = [](const MoveOnlyErrorType&&) { return std::expected<int, int>{}; };
-      std::move(e).or_else(l);
+      (void)std::move(e).or_else(l);
   }
 }
 
@@ -225,10 +225,10 @@ constexpr bool test() {
     return std::expected<int, int>();
   };
 
-  e.or_else(never_called);
-  std::move(e).or_else(never_called);
-  ce.or_else(never_called);
-  std::move(ce).or_else(never_called);
+  (void)e.or_else(never_called);
+  (void)std::move(e).or_else(never_called);
+  (void)ce.or_else(never_called);
+  (void)std::move(ce).or_else(never_called);
   return true;
 }
 
diff --git a/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
index 4fb21374aebe3..cbd54d623c0f4 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
@@ -187,13 +187,13 @@ constexpr void test_val_types() {
 constexpr void test_take_val_return_void() {
   std::expected<int, int> e(1);
   int val = 0;
-  e.transform([&val]<typename T>(T&&) -> void {
+  (void)e.transform([&val]<typename T>(T&&) -> void {
     static_assert(std::is_same_v<T, int&>);
     assert(val == 0);
     val = 1;
   });
   assert(val == 1);
-  std::move(e).transform([&val]<typename T>(T&&) -> void {
+  (void)std::move(e).transform([&val]<typename T>(T&&) -> void {
     static_assert(std::is_same_v<T, int>);
     assert(val == 1);
     val = 2;
@@ -201,13 +201,13 @@ constexpr void test_take_val_return_void() {
 
   const auto& ce = e;
   assert(val == 2);
-  ce.transform([&val]<typename T>(T&&) -> void {
+  (void)ce.transform([&val]<typename T>(T&&) -> void {
     static_assert(std::is_same_v<T, const int&>);
     assert(val == 2);
     val = 3;
   });
   assert(val == 3);
-  std::move(ce).transform([&val]<typename T>(T&&) -> void {
+  (void)std::move(ce).transform([&val]<typename T>(T&&) -> void {
     static_assert(std::is_same_v<T, const int>);
     assert(val == 3);
     val = 4;
@@ -227,8 +227,8 @@ constexpr void test_direct_non_list_init() {
 constexpr void test_sfinae() {
   std::expected<NonConst, int> e(std::unexpected<int>(2));
   auto l = [](auto&& x) { return x.non_const(); };
-  e.transform(l);
-  std::move(e).transform(l);
+  (void)e.transform(l);
+  (void)std::move(e).transform(l);
 
   std::expected<int, int> e1(std::unexpected<int>(1));
   const auto& ce1         = e1;
@@ -237,10 +237,10 @@ constexpr void test_sfinae() {
     return std::expected<int, int>();
   };
 
-  e1.transform(never_called);
-  std::move(e1).transform(never_called);
-  ce1.and_then(never_called);
-  std::move(ce1).transform(never_called);
+  (void)e1.transform(never_called);
+  (void)std::move(e1).transform(never_called);
+  (void)ce1.transform(never_called);
+  (void)std::move(ce1).transform(never_called);
 }
 
 constexpr void test_move_only_error_type() {
@@ -248,14 +248,14 @@ constexpr void test_move_only_error_type() {
   {
       std::expected<int, MoveOnlyErrorType> e;
       auto l = [](int) { return 0; };
-      std::move(e).transform(l);
+      (void)std::move(e).transform(l);
   }
 
   // Test const&&
   {
       const std::expected<int, MoveOnlyErrorType> e;
       auto l = [](const int) { return 0; };
-      std::move(e).transform(l);
+      (void)std::move(e).transform(l);
   }
 }
 
diff --git a/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
index d35788d9fef25..a19e17b01f6a9 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
@@ -197,8 +197,8 @@ constexpr void test_direct_non_list_init() {
 constexpr void test_sfinae() {
   std::expected<int, NonConst> e(2);
   auto l = [](auto&& x) { return x.non_const(); };
-  e.transform_error(l);
-  std::move(e).transform_error(l);
+  (void)e.transform_error(l);
+  (void)std::move(e).transform_error(l);
 
   std::expected<int, int> e1;
   const auto& ce1 = e1;
@@ -208,10 +208,10 @@ constexpr void test_sfinae() {
     return 0;
   };
 
-  e1.transform_error(never_called);
-  std::move(e1).transform_error(never_called);
-  ce1.transform_error(never_called);
-  std::move(ce1).transform_error(never_called);
+  (void)e1.transform_error(never_called);
+  (void)std::move(e1).transform_error(never_called);
+  (void)ce1.transform_error(never_called);
+  (void)std::move(ce1).transform_error(never_called);
 }
 
 constexpr void test_move_only_error_type() {
@@ -219,28 +219,28 @@ constexpr void test_move_only_error_type() {
   {
       std::expected<int, MoveOnlyErrorType> e;
       auto l = [](MoveOnlyErrorType&) { return 0; };
-      e.transform_error(l);
+      (void)e.transform_error(l);
   }
 
   // Test const&
   {
       const std::expected<int, MoveOnlyErrorType> e;
       auto l = [](const MoveOnlyErrorType&) { return 0; };
-      e.transform_error(l);
+      (void)e.transform_error(l);
   }
 
   // Test &&
   {
       std::expected<int, MoveOnlyErrorType> e;
       auto l = [](MoveOnlyErrorType&&) { return 0; };
-      std::move(e).transform_error(l);
+      (void)std::move(e).transform_error(l);
   }
 
   // Test const&&
   {
       const std::expected<int, MoveOnlyErrorType> e;
       auto l = [](const MoveOnlyErrorType&&) { return 0; };
-      std::move(e).transform_error(l);
+      (void)std::move(e).transform_error(l);
   }
 }
 
diff --git a/libcxx/test/support/increasing_allocator.h b/libcxx/test/support/increasing_allocator.h
new file mode 100644
index 0000000000000..30bd6f40c8dad
--- /dev/null
+++ b/libcxx/test/support/increasing_allocator.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_SUPPORT_INCREASING_ALLOCATOR_H
+#define TEST_SUPPORT_INCREASING_ALLOCATOR_H
+
+#include <cstddef>
+#include <memory>
+
+#include "test_macros.h"
+
+// The increasing_allocator is a custom allocator that enforces an increasing minimum allocation size,
+// ensuring that it allocates an increasing amount of memory, possibly exceeding the requested amount.
+// This unique behavior is particularly useful for testing the shrink_to_fit functionality in std::vector,
+// vector<bool>, and std::basic_string, ensuring that shrink_to_fit does not increase the capacity of
+// the allocated memory.
+
+template <typename T>
+struct increasing_allocator {
+  using value_type         = T;
+  std::size_t min_elements = 1000;
+  increasing_allocator()   = default;
+
+  template <typename U>
+  TEST_CONSTEXPR_CXX20 increasing_allocator(const increasing_allocator<U>& other) TEST_NOEXCEPT
+      : min_elements(other.min_elements) {}
+
+#if TEST_STD_VER >= 23
+  TEST_CONSTEXPR_CXX23 std::allocation_result<T*> allocate_at_least(std::size_t n) {
+    if (n < min_elements)
+      n = min_elements;
+    min_elements += 1000;
+    return std::allocator<T>{}.allocate_at_least(n);
+  }
+#endif // TEST_STD_VER >= 23
+
+  TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
+
+  TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) TEST_NOEXCEPT { std::allocator<T>().deallocate(p, n); }
+};
+
+template <typename T, typename U>
+TEST_CONSTEXPR_CXX20 bool operator==(increasing_allocator<T>, increasing_allocator<U>) TEST_NOEXCEPT {
+  return true;
+}
+
+#endif // TEST_SUPPORT_INCREASING_ALLOCATOR_H
diff --git a/libcxx/test/support/test_allocator.h b/libcxx/test/support/test_allocator.h
index dcd15332ca304..f8b622d7f9520 100644
--- a/libcxx/test/support/test_allocator.h
+++ b/libcxx/test/support/test_allocator.h
@@ -27,45 +27,45 @@ TEST_CONSTEXPR_CXX20 inline typename std::allocator_traits<Alloc>::size_type all
 }
 
 struct test_allocator_statistics {
-  int time_to_throw = 0;
-  int throw_after = INT_MAX;
+  int time_to_throw   = 0;
+  int throw_after     = INT_MAX;
   int count           = 0; // the number of active instances
   int alloc_count     = 0; // the number of allocations not deallocating
   int allocated_size  = 0; // the size of allocated elements
   int construct_count = 0; // the number of times that ::construct was called
-  int destroy_count = 0; // the number of times that ::destroy was called
-  int copied = 0;
-  int moved = 0;
-  int converted = 0;
+  int destroy_count   = 0; // the number of times that ::destroy was called
+  int copied          = 0;
+  int moved           = 0;
+  int converted       = 0;
 
   TEST_CONSTEXPR_CXX14 void clear() {
     assert(count == 0 && "clearing leaking allocator data?");
-    count = 0;
-    time_to_throw = 0;
-    alloc_count = 0;
+    count           = 0;
+    time_to_throw   = 0;
+    alloc_count     = 0;
     allocated_size  = 0;
     construct_count = 0;
-    destroy_count = 0;
-    throw_after = INT_MAX;
+    destroy_count   = 0;
+    throw_after     = INT_MAX;
     clear_ctor_counters();
   }
 
   TEST_CONSTEXPR_CXX14 void clear_ctor_counters() {
-    copied = 0;
-    moved = 0;
+    copied    = 0;
+    moved     = 0;
     converted = 0;
   }
 };
 
 struct test_alloc_base {
   TEST_CONSTEXPR static const int destructed_value = -1;
-  TEST_CONSTEXPR static const int moved_value = INT_MAX;
+  TEST_CONSTEXPR static const int moved_value      = INT_MAX;
 };
 
 template <class T>
 class test_allocator {
-  int data_ = 0; // participates in equality
-  int id_ = 0;   // unique identifier, doesn't participate in equality
+  int data_                         = 0; // participates in equality
+  int id_                           = 0; // unique identifier, doesn't participate in equality
   test_allocator_statistics* stats_ = nullptr;
 
   template <class U>
@@ -95,7 +95,8 @@ class test_allocator {
   TEST_CONSTEXPR explicit test_allocator(int data) TEST_NOEXCEPT : data_(data) {}
 
   TEST_CONSTEXPR_CXX14 explicit test_allocator(int data, test_allocator_statistics* stats) TEST_NOEXCEPT
-      : data_(data), stats_(stats) {
+      : data_(data),
+        stats_(stats) {
     if (stats != nullptr)
       ++stats_->count;
   }
@@ -103,13 +104,17 @@ class test_allocator {
   TEST_CONSTEXPR explicit test_allocator(int data, int id) TEST_NOEXCEPT : data_(data), id_(id) {}
 
   TEST_CONSTEXPR_CXX14 explicit test_allocator(int data, int id, test_allocator_statistics* stats) TEST_NOEXCEPT
-      : data_(data), id_(id), stats_(stats) {
+      : data_(data),
+        id_(id),
+        stats_(stats) {
     if (stats_ != nullptr)
       ++stats_->count;
   }
 
   TEST_CONSTEXPR_CXX14 test_allocator(const test_allocator& a) TEST_NOEXCEPT
-    : data_(a.data_), id_(a.id_), stats_(a.stats_) {
+      : data_(a.data_),
+        id_(a.id_),
+        stats_(a.stats_) {
     assert(a.data_ != test_alloc_base::destructed_value && a.id_ != test_alloc_base::destructed_value &&
            "copying from destroyed allocator");
     if (stats_ != nullptr) {
@@ -130,7 +135,9 @@ class test_allocator {
 
   template <class U>
   TEST_CONSTEXPR_CXX14 test_allocator(const test_allocator<U>& a) TEST_NOEXCEPT
-      : data_(a.data_), id_(a.id_), stats_(a.stats_) {
+      : data_(a.data_),
+        id_(a.id_),
+        stats_(a.stats_) {
     if (stats_ != nullptr) {
       ++stats_->count;
       ++stats_->converted;
@@ -143,7 +150,7 @@ class test_allocator {
     if (stats_ != nullptr)
       --stats_->count;
     data_ = test_alloc_base::destructed_value;
-    id_ = test_alloc_base::destructed_value;
+    id_   = test_alloc_base::destructed_value;
   }
 
   TEST_CONSTEXPR pointer address(reference x) const { return &x; }
@@ -197,8 +204,8 @@ class test_allocator {
 
 template <>
 class test_allocator<void> {
-  int data_ = 0;
-  int id_ = 0;
+  int data_                         = 0;
+  int id_                           = 0;
   test_allocator_statistics* stats_ = nullptr;
 
   template <class U>
@@ -223,27 +230,30 @@ class test_allocator<void> {
   TEST_CONSTEXPR explicit test_allocator(int data) TEST_NOEXCEPT : data_(data) {}
 
   TEST_CONSTEXPR explicit test_allocator(int data, test_allocator_statistics* stats) TEST_NOEXCEPT
-      : data_(data), stats_(stats)
-  {}
+      : data_(data),
+        stats_(stats) {}
 
   TEST_CONSTEXPR explicit test_allocator(int data, int id) : data_(data), id_(id) {}
 
   TEST_CONSTEXPR_CXX14 explicit test_allocator(int data, int id, test_allocator_statistics* stats) TEST_NOEXCEPT
-      : data_(data), id_(id), stats_(stats)
-  {}
+      : data_(data),
+        id_(id),
+        stats_(stats) {}
 
   TEST_CONSTEXPR_CXX14 explicit test_allocator(const test_allocator& a) TEST_NOEXCEPT
-      : data_(a.data_), id_(a.id_), stats_(a.stats_)
-  {}
+      : data_(a.data_),
+        id_(a.id_),
+        stats_(a.stats_) {}
 
   template <class U>
   TEST_CONSTEXPR_CXX14 test_allocator(const test_allocator<U>& a) TEST_NOEXCEPT
-      : data_(a.data_), id_(a.id_), stats_(a.stats_)
-  {}
+      : data_(a.data_),
+        id_(a.id_),
+        stats_(a.stats_) {}
 
   TEST_CONSTEXPR_CXX20 ~test_allocator() TEST_NOEXCEPT {
     data_ = test_alloc_base::destructed_value;
-    id_ = test_alloc_base::destructed_value;
+    id_   = test_alloc_base::destructed_value;
   }
 
   TEST_CONSTEXPR int get_id() const { return id_; }
@@ -310,9 +320,9 @@ struct Tag_X {
   TEST_CONSTEXPR Tag_X(Ctor_Tag, Args&&...) {}
 
   // not DefaultConstructible, CopyConstructible or MoveConstructible.
-  Tag_X() = delete;
+  Tag_X()             = delete;
   Tag_X(const Tag_X&) = delete;
-  Tag_X(Tag_X&&) = delete;
+  Tag_X(Tag_X&&)      = delete;
 
   // CopyAssignable.
   TEST_CONSTEXPR_CXX14 Tag_X& operator=(const Tag_X&) { return *this; };
@@ -329,7 +339,7 @@ struct Tag_X {
 template <typename T>
 class TaggingAllocator {
 public:
-  using value_type = T;
+  using value_type   = T;
   TaggingAllocator() = default;
 
   template <typename U>
@@ -356,13 +366,13 @@ class TaggingAllocator {
 template <std::size_t MaxAllocs>
 struct limited_alloc_handle {
   std::size_t outstanding_ = 0;
-  void* last_alloc_ = nullptr;
+  void* last_alloc_        = nullptr;
 
   template <class T>
   TEST_CONSTEXPR_CXX20 T* allocate(std::size_t N) {
     if (N + outstanding_ > MaxAllocs)
       TEST_THROW(std::bad_alloc());
-    auto alloc = std::allocator<T>().allocate(N);
+    auto alloc  = std::allocator<T>().allocate(N);
     last_alloc_ = alloc;
     outstanding_ += N;
     return alloc;
@@ -467,7 +477,7 @@ class limited_allocator {
   TEST_CONSTEXPR_CXX20 pointer allocate(size_type n) { return handle_->template allocate<T>(n); }
   TEST_CONSTEXPR_CXX20 void deallocate(pointer p, size_type n) { handle_->template deallocate<T>(p, n); }
   TEST_CONSTEXPR size_type max_size() const { return N; }
-  TEST_CONSTEXPR BuffT* getHandle() const { return handle_.get(); }
+  TEST_CONSTEXPR const BuffT* getHandle() const { return handle_.get(); }
 };
 
 template <class T, class U, std::size_t N>
diff --git a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
index efe63e0f625f2..05c44e49b448c 100644
--- a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
+++ b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
@@ -1,6 +1,3 @@
-# TODO: Re-enable the tests once the CI is back under control
-return()
-
 # The find_package changes these variables. This leaves the build in an odd
 # state. Calling cmake a second time tries to write site config information in
 # the system's libc++. Restoring these setting after testing fixes this issue.
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index b8e982b653d39..df0e5d4d12f9f 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -115,17 +115,19 @@ steps:
       os: aix
     <<: *common
 
-- group: ':freebsd: FreeBSD'
-  steps:
-  - label: FreeBSD 13 amd64
-    command: libcxx/utils/ci/run-buildbot generic-cxx26
-    env:
-      CC: clang17
-      CXX: clang++17
-    agents:
-      queue: libcxx-builders
-      os: freebsd
-    <<: *common
+# TODO: Re-enable this when the FreeBSD runners are back online
+#       https://github.com/llvm/llvm-project/issues/117780
+# - group: ':freebsd: FreeBSD'
+#   steps:
+#   - label: FreeBSD 13 amd64
+#     command: libcxx/utils/ci/run-buildbot generic-cxx26
+#     env:
+#       CC: clang17
+#       CXX: clang++17
+#     agents:
+#       queue: libcxx-builders
+#       os: freebsd
+#     <<: *common
 
 - group: ':android: Android'
   steps:
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 3df7b00a8aa09..a832380e16cd8 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -387,16 +387,16 @@ bootstrapping-build)
           -DLLVM_ENABLE_ASSERTIONS=ON \
           -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests"
 
-    step "Running the LLDB libc++ data formatter tests"
-    ${NINJA} -vC "${BUILD_DIR}" lldb-api-test-deps
-    ${BUILD_DIR}/bin/llvm-lit -sv --param dotest-args='--category libc++' "${MONOREPO_ROOT}/lldb/test/API"
-
     step "Running the libc++ and libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}" check-runtimes
 
     step "Installing libc++ and libc++abi to a fake location"
     ${NINJA} -vC "${BUILD_DIR}" install-runtimes
 
+    step "Running the LLDB libc++ data formatter tests"
+    ${NINJA} -vC "${BUILD_DIR}" lldb-api-test-deps
+    ${BUILD_DIR}/bin/llvm-lit -sv --param dotest-args='--category libc++' "${MONOREPO_ROOT}/lldb/test/API"
+
     ccache -s
 ;;
 generic-static)
diff --git a/libcxx/utils/ci/run-buildbot-container b/libcxx/utils/ci/run-buildbot-container
index 74e7dab81d734..b4391455e2725 100755
--- a/libcxx/utils/ci/run-buildbot-container
+++ b/libcxx/utils/ci/run-buildbot-container
@@ -26,6 +26,6 @@ if [[ ! -d "${MONOREPO_ROOT}/libcxx/utils/ci" ]]; then
     echo "Was unable to find the root of the LLVM monorepo; are you running from within the monorepo?"
     exit 1
 fi
-docker pull ghcr.io/libcxx/actions-builder
-docker run -it --volume "${MONOREPO_ROOT}:/llvm" --workdir "/llvm" --cap-add=SYS_PTRACE ghcr.io/libcxx/actions-builder \
+docker pull ghcr.io/llvm/libcxx-linux-builder:0fd6f684b9c84c32d6cbfd9742402e788b2879f1
+docker run -it --volume "${MONOREPO_ROOT}:/llvm" --workdir "/llvm" --cap-add=SYS_PTRACE ghcr.io/llvm/libcxx-linux-builder:0fd6f684b9c84c32d6cbfd9742402e788b2879f1 \
     bash -c 'git config --global --add safe.directory /llvm ; exec bash'
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 49b61c8935e1b..e4b413deff4db 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -159,14 +159,6 @@ def _mingwSupportsModules(cfg):
         when=lambda cfg: hasCompileFlag(cfg, "-fconstexpr-ops-limit=1"),
     ),
     Feature(name="has-fblocks", when=lambda cfg: hasCompileFlag(cfg, "-fblocks")),
-    Feature(
-        name="-fsized-deallocation",
-        when=lambda cfg: hasCompileFlag(cfg, "-fsized-deallocation"),
-    ),
-    Feature(
-        name="-faligned-allocation",
-        when=lambda cfg: hasCompileFlag(cfg, "-faligned-allocation"),
-    ),
     Feature(
         name="fdelayed-template-parsing",
         when=lambda cfg: hasCompileFlag(cfg, "-fdelayed-template-parsing"),
diff --git a/libcxx/utils/synchronize_csv_status_files.py b/libcxx/utils/synchronize_csv_status_files.py
index 8317dde770baf..3132857434e7a 100755
--- a/libcxx/utils/synchronize_csv_status_files.py
+++ b/libcxx/utils/synchronize_csv_status_files.py
@@ -281,12 +281,64 @@ def write_csv(output: pathlib.Path, rows: List[Tuple]):
         for row in rows:
             writer.writerow(row)
 
-def sync_csv(rows: List[Tuple], from_github: List[PaperInfo]) -> List[Tuple]:
+def create_github_issue(paper: PaperInfo, labels: List[str]) -> None:
+    """
+    Create a new Github issue representing the given PaperInfo.
+    """
+    paper_name = paper.paper_name.replace('``', '`').replace('\\', '')
+
+    create_cli = ['gh', 'issue', 'create', '--repo', 'llvm/llvm-project',
+                    '--title', f'{paper.paper_number}: {paper_name}',
+                    '--body', f'**Link:** https://wg21.link/{paper.paper_number}',
+                    '--project', 'libc++ Standards Conformance',
+                    '--label', 'libc++']
+
+    for label in labels:
+        create_cli += ['--label', label]
+
+    print("Do you want to create the following issue?")
+    print(create_cli)
+    answer = input("y/n: ")
+    if answer == 'n':
+        print("Not creating issue")
+        return
+    elif answer != 'y':
+        print(f"Invalid answer {answer}, skipping")
+        return
+
+    print("Creating issue")
+    issue_link = subprocess.check_output(create_cli).decode().strip()
+    print(f"Created tracking issue for {paper.paper_number}: {issue_link}")
+
+    # Retrieve the "Github project item ID" by re-adding the issue to the project again,
+    # even though we created it inside the project in the first place.
+    item_add_cli = ['gh', 'project', 'item-add', LIBCXX_CONFORMANCE_PROJECT, '--owner', 'llvm', '--url', issue_link, '--format', 'json']
+    item = json.loads(subprocess.check_output(item_add_cli).decode().strip())
+
+    # Then, adjust the 'Meeting Voted' field of that item.
+    meeting_voted_cli = ['gh', 'project', 'item-edit',
+                                '--project-id', 'PVT_kwDOAQWwKc4AlOgt',
+                                '--field-id', 'PVTF_lADOAQWwKc4AlOgtzgdUEXI', '--text', paper.meeting,
+                                '--id', item['id']]
+    subprocess.check_call(meeting_voted_cli)
+
+    # And also adjust the 'Status' field of the item to 'To Do'.
+    status_cli = ['gh', 'project', 'item-edit',
+                                '--project-id', 'PVT_kwDOAQWwKc4AlOgt',
+                                '--field-id', 'PVTSSF_lADOAQWwKc4AlOgtzgdUBak', '--single-select-option-id', 'f75ad846',
+                                '--id', item['id']]
+    subprocess.check_call(status_cli)
+
+def sync_csv(rows: List[Tuple], from_github: List[PaperInfo], create_new: bool, labels: List[str] = None) -> List[Tuple]:
     """
     Given a list of CSV rows representing an existing status file and a list of PaperInfos representing
     up-to-date (but potentially incomplete) tracking information from Github, this function returns the
     new CSV rows synchronized with the up-to-date information.
 
+    If `create_new` is True and a paper from the CSV file is not tracked on Github yet, this also prompts
+    to create a new issue on Github for tracking it. In that case the created issue is tagged with the
+    provided labels.
+
     Note that this only tracks changes from 'not implemented' issues to 'implemented'. If an up-to-date
     PaperInfo reports that a paper is not implemented but the existing CSV rows report it as implemented,
     it is an error (i.e. the result is not a CSV row where the paper is *not* implemented).
@@ -305,16 +357,20 @@ def sync_csv(rows: List[Tuple], from_github: List[PaperInfo]) -> List[Tuple]:
         # issue tracking it, which we validate below.
         tracking = [gh for gh in from_github if paper.paper_number == gh.paper_number]
 
-        # If there is no tracking issue for that row in the CSV, this is an error since we're
-        # missing a Github issue.
-        if len(tracking) == 0:
-            print(f"Can't find any Github issue for CSV row: {row}")
+        # If there's more than one tracking issue, something is weird.
+        if len(tracking) > 1:
+            print(f"Found a row with more than one tracking issue: {row}\ntracked by: {tracking}")
             results.append(row)
             continue
 
-        # If there's more than one tracking issue, something is weird too.
-        if len(tracking) > 1:
-            print(f"Found a row with more than one tracking issue: {row}\ntracked by: {tracking}")
+        # If there is no tracking issue for that row and we are creating new issues, do that.
+        # Otherwise just log that we're missing an issue.
+        if len(tracking) == 0:
+            if create_new:
+                assert labels is not None, "Missing labels when creating new Github issues"
+                create_github_issue(paper, labels=labels)
+            else:
+                print(f"Can't find any Github issue for CSV row: {row}")
             results.append(row)
             continue
 
@@ -322,32 +378,61 @@ def sync_csv(rows: List[Tuple], from_github: List[PaperInfo]) -> List[Tuple]:
 
     return results
 
-CSV_FILES_TO_SYNC = [
-    'Cxx17Issues.csv',
-    'Cxx17Papers.csv',
-    'Cxx20Issues.csv',
-    'Cxx20Papers.csv',
-    'Cxx23Issues.csv',
-    'Cxx23Papers.csv',
-    'Cxx2cIssues.csv',
-    'Cxx2cPapers.csv',
-]
-
-def main():
+CSV_FILES_TO_SYNC = {
+    'Cxx17Issues.csv': ['c++17', 'lwg-issue'],
+    'Cxx17Papers.csv': ['c++17', 'wg21 paper'],
+    'Cxx20Issues.csv': ['c++20', 'lwg-issue'],
+    'Cxx20Papers.csv': ['c++20', 'wg21 paper'],
+    'Cxx23Issues.csv': ['c++23', 'lwg-issue'],
+    'Cxx23Papers.csv': ['c++23', 'wg21 paper'],
+    'Cxx2cIssues.csv': ['c++26', 'lwg-issue'],
+    'Cxx2cPapers.csv': ['c++26', 'wg21 paper'],
+}
+
+def main(argv):
+    import argparse
+    parser = argparse.ArgumentParser(prog='synchronize-status-files',
+        description='Synchronize the libc++ conformance status files with Github issues')
+    parser.add_argument('--validate-only', action='store_true',
+        help="Only perform the data validation of CSV files.")
+    parser.add_argument('--create-new', action='store_true',
+        help="Create new Github issues for CSV rows that do not correspond to any existing Github issue.")
+    parser.add_argument('--load-github-from', type=str,
+        help="A json file to load the Github project information from instead of querying the API. This is useful for testing to avoid rate limiting.")
+    args = parser.parse_args(argv)
+
     libcxx_root = pathlib.Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-    # Extract the list of PaperInfos from issues we're tracking on Github.
-    print("Loading all issues from Github")
-    gh_command_line = ['gh', 'project', 'item-list', LIBCXX_CONFORMANCE_PROJECT, '--owner', 'llvm', '--format', 'json', '--limit', '9999999']
-    project_info = json.loads(subprocess.check_output(gh_command_line))
+    # Perform data validation for all the CSV files.
+    print("Performing data validation of the CSV files")
+    for filename in CSV_FILES_TO_SYNC:
+        csv = load_csv(libcxx_root / 'docs' / 'Status' / filename)
+        for row in csv[1:]: # Skip the header
+            if row[0] != "": # Skip separator rows
+                PaperInfo.from_csv_row(row)
+
+    if args.validate_only:
+        return
+
+    # Load all the Github issues tracking papers from Github.
+    if args.load_github_from:
+        print(f"Loading all issues from {args.load_github_from}")
+        with open(args.load_github_from, 'r') as f:
+            project_info = json.load(f)
+    else:
+        print("Loading all issues from Github")
+        gh_command_line = ['gh', 'project', 'item-list', LIBCXX_CONFORMANCE_PROJECT, '--owner', 'llvm', '--format', 'json', '--limit', '9999999']
+        project_info = json.loads(subprocess.check_output(gh_command_line))
     from_github = [PaperInfo.from_github_issue(i) for i in project_info['items']]
 
-    for filename in CSV_FILES_TO_SYNC:
+    # Synchronize CSV files with the Github issues.
+    for (filename, labels) in CSV_FILES_TO_SYNC.items():
         print(f"Synchronizing {filename} with Github issues")
         file = libcxx_root / 'docs' / 'Status' / filename
         csv = load_csv(file)
-        synced = sync_csv(csv, from_github)
+        synced = sync_csv(csv, from_github, create_new=args.create_new, labels=labels)
         write_csv(file, synced)
 
 if __name__ == '__main__':
-    main()
+    import sys
+    main(sys.argv[1:])
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 84fe2784bec5c..21dda44e09976 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -75,6 +75,12 @@ if (NOT APPLE) # On Apple platforms, we always use -nostdlib++ so we don't need
   add_library_flags_if(LIBCXXABI_HAS_C_LIB c)
 endif()
 
+if (FUCHSIA)
+    # TODO: Use CMAKE_LINK_LIBRARY_USING_FEATURE once our minimum CMake is at least 3.24
+    # https://cmake.org/cmake/help/latest/variable/CMAKE_LINK_LIBRARY_USING_FEATURE.html 
+    add_link_flags("-Wl,--push-state,--as-needed,-lzircon,--pop-state")
+endif()
+
 if (NOT LIBCXXABI_USE_COMPILER_RT)
   add_library_flags_if(LIBCXXABI_HAS_GCC_LIB gcc)
 endif()
diff --git a/libcxxabi/src/cxa_guard_impl.h b/libcxxabi/src/cxa_guard_impl.h
index 7b05bf32f3eda..191a589176b1a 100644
--- a/libcxxabi/src/cxa_guard_impl.h
+++ b/libcxxabi/src/cxa_guard_impl.h
@@ -156,12 +156,12 @@ class AtomicInt {
 //                       PlatformGetThreadID
 //===----------------------------------------------------------------------===//
 
-#if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#if defined(__APPLE__) && _LIBCPP_HAS_THREAD_API_PTHREAD
 uint32_t PlatformThreadID() {
   static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
   return static_cast<uint32_t>(pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
 }
-#elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#elif defined(SYS_gettid) && _LIBCPP_HAS_THREAD_API_PTHREAD
 uint32_t PlatformThreadID() {
   static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
   return static_cast<uint32_t>(syscall(SYS_gettid));
diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index 52755e6f85103..e4752bed6da8b 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -2077,17 +2077,23 @@ class SizeofParamPackExpr : public Node {
 class CallExpr : public Node {
   const Node *Callee;
   NodeArray Args;
+  bool IsParen; // (func)(args ...) ?
 
 public:
-  CallExpr(const Node *Callee_, NodeArray Args_, Prec Prec_)
-      : Node(KCallExpr, Prec_), Callee(Callee_), Args(Args_) {}
+  CallExpr(const Node *Callee_, NodeArray Args_, bool IsParen_, Prec Prec_)
+      : Node(KCallExpr, Prec_), Callee(Callee_), Args(Args_),
+        IsParen(IsParen_) {}
 
   template <typename Fn> void match(Fn F) const {
-    F(Callee, Args, getPrecedence());
+    F(Callee, Args, IsParen, getPrecedence());
   }
 
   void printLeft(OutputBuffer &OB) const override {
+    if (IsParen)
+      OB.printOpen();
     Callee->print(OB);
+    if (IsParen)
+      OB.printClose();
     OB.printOpen();
     Args.printWithComma(OB);
     OB.printClose();
@@ -3354,9 +3360,12 @@ const typename AbstractManglingParser<
      "operator co_await"},
     {"az", OperatorInfo::OfIdOp, /*Type*/ false, Node::Prec::Unary, "alignof "},
     {"cc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, "const_cast"},
-    {"cl", OperatorInfo::Call, false, Node::Prec::Postfix, "operator()"},
+    {"cl", OperatorInfo::Call, /*Paren*/ false, Node::Prec::Postfix,
+     "operator()"},
     {"cm", OperatorInfo::Binary, false, Node::Prec::Comma, "operator,"},
     {"co", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator~"},
+    {"cp", OperatorInfo::Call, /*Paren*/ true, Node::Prec::Postfix,
+     "operator()"},
     {"cv", OperatorInfo::CCast, false, Node::Prec::Cast, "operator"}, // C Cast
     {"dV", OperatorInfo::Binary, false, Node::Prec::Assign, "operator/="},
     {"da", OperatorInfo::Del, /*Ary*/ true, Node::Prec::Unary,
@@ -5099,6 +5108,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseRequiresExpr() {
 //              ::= <binary operator-name> <expression> <expression>
 //              ::= <ternary operator-name> <expression> <expression> <expression>
 //              ::= cl <expression>+ E                                   # call
+//              ::= cp <base-unresolved-name> <expression>* E            # (name) (expr-list), call that would use argument-dependent lookup but for the parentheses
 //              ::= cv <type> <expression>                               # conversion with one argument
 //              ::= cv <type> _ <expression>* E                          # conversion with a different number of arguments
 //              ::= [gs] nw <expression>* _ <type> E                     # new (expr-list) type
@@ -5234,7 +5244,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() {
         Names.push_back(E);
       }
       return make<CallExpr>(Callee, popTrailingNodeArray(ExprsBegin),
-                            Op->getPrecedence());
+                            /*IsParen=*/Op->getFlag(), Op->getPrecedence());
     }
     case OperatorInfo::CCast: {
       // C Cast: (type)expr
@@ -5421,7 +5431,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() {
       }
     }
     return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin),
-                          Node::Prec::Postfix);
+                          /*IsParen=*/false, Node::Prec::Postfix);
   }
 
   // Only unresolved names remain.
diff --git a/libcxxabi/test/forced_unwind2.pass.cpp b/libcxxabi/test/forced_unwind2.pass.cpp
index 6ec27e4a65557..edd0a3b175f06 100644
--- a/libcxxabi/test/forced_unwind2.pass.cpp
+++ b/libcxxabi/test/forced_unwind2.pass.cpp
@@ -44,10 +44,8 @@ struct Stop<R (*)(Args...)> {
 };
 
 static void forced_unwind() {
-  _Unwind_Exception* exc = new _Unwind_Exception;
-  memset(&exc->exception_class, 0, sizeof(exc->exception_class));
-  exc->exception_cleanup = 0;
-  _Unwind_ForcedUnwind(exc, Stop<_Unwind_Stop_Fn>::stop, 0);
+  static _Unwind_Exception exc = {};
+  _Unwind_ForcedUnwind(&exc, Stop<_Unwind_Stop_Fn>::stop, 0);
   abort();
 }
 
diff --git a/libcxxabi/test/test_demangle.pass.cpp b/libcxxabi/test/test_demangle.pass.cpp
index 36f9e07eb4bef..9e8a7c358fbcb 100644
--- a/libcxxabi/test/test_demangle.pass.cpp
+++ b/libcxxabi/test/test_demangle.pass.cpp
@@ -29834,6 +29834,7 @@ const char* cases[][2] =
     {"_ZN5Casts5auto_IiEEvDTnw_DapicvT__EEE", "void Casts::auto_<int>(decltype(new auto((int)())))"},
     {"_ZN5Casts7scalar_IiEEvDTcmcvT__Ecvi_EE", "void Casts::scalar_<int>(decltype((int)(), (int)()))"},
     {"_ZN5test11aIsEEDTcl3foocvT__EEES1_", "decltype(foo((short)())) test1::a<short>(short)"},
+    {"_ZN5test11bIsEEDTcp3foocvT__EEES1_", "decltype((foo)((short)())) test1::b<short>(short)"},
     {"_ZN5test21aIPFfvEEEvT_DTclfL0p_EE", "void test2::a<float (*)()>(float (*)(), decltype(fp()))"},
     {"_ZN5test21bIPFfvEEEDTclfp_EET_", "decltype(fp()) test2::b<float (*)()>(float (*)())"},
     {"_ZN5test21cIPFfvEEEvT_PFvDTclfL1p_EEE", "void test2::c<float (*)()>(float (*)(), void (*)(decltype(fp())))"},
@@ -30397,24 +30398,24 @@ void test_invalid_cases()
     assert(!passed && "demangle did not fail");
 }
 
-const char *xfail_cases[] = {
-    // FIXME: Why does clang generate the "cp" expr?
-    "_ZN5test11bIsEEDTcp3foocvT__EEES1_",
+const char *const xfail_cases[] = {
+    // Sentinel value
+    nullptr
 };
 
-const size_t num_xfails = sizeof(xfail_cases) / sizeof(xfail_cases[0]);
-
 void test_xfail_cases()
 {
     std::size_t len = 0;
     char* buf = nullptr;
-    for (std::size_t i = 0; i < num_xfails; ++i)
+    for (const char *c_str : xfail_cases)
     {
+        if (!c_str)
+            break;
         int status;
-        char* demang = __cxxabiv1::__cxa_demangle(xfail_cases[i], buf, &len, &status);
+        char* demang = __cxxabiv1::__cxa_demangle(c_str, buf, &len, &status);
         if (status != -2)
         {
-            std::printf("%s was documented as xfail but passed\n", xfail_cases[i]);
+            std::printf("%s was documented as xfail but passed\n", c_str);
             std::printf("Got status = %d\n", status);
             assert(status == -2);
         }
diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt
index 2e18b10965633..e7ea57734cca9 100644
--- a/libunwind/src/CMakeLists.txt
+++ b/libunwind/src/CMakeLists.txt
@@ -113,6 +113,22 @@ if (HAIKU)
 
   add_compile_flags("-D_DEFAULT_SOURCE")
   add_compile_flags("-DPT_GNU_EH_FRAME=PT_EH_FRAME")
+
+  find_path(LIBUNWIND_HAIKU_PRIVATE_HEADERS
+            "commpage_defs.h"
+            PATHS ${CMAKE_SYSTEM_INCLUDE_PATH}
+            PATH_SUFFIXES "/private/system"
+            NO_DEFAULT_PATH
+            REQUIRED)
+
+  include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}")
+  if (LIBUNWIND_TARGET_TRIPLE)
+    if (${LIBUNWIND_TARGET_TRIPLE} MATCHES "^x86_64")
+      include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}/arch/x86_64")
+    endif()
+  else()
+    include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}/arch/${CMAKE_SYSTEM_PROCESSOR}")
+  endif()
 endif ()
 
 string(REPLACE ";" " " LIBUNWIND_COMPILE_FLAGS "${LIBUNWIND_COMPILE_FLAGS}")
diff --git a/libunwind/src/Unwind-sjlj.c b/libunwind/src/Unwind-sjlj.c
index 7e8faf098fe14..a3551b328950d 100644
--- a/libunwind/src/Unwind-sjlj.c
+++ b/libunwind/src/Unwind-sjlj.c
@@ -408,7 +408,7 @@ _LIBUNWIND_EXPORT uintptr_t
 _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) {
   _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context;
   _LIBUNWIND_TRACE_API("_Unwind_GetLanguageSpecificData(context=%p) "
-                       "=> 0x%" PRIuPTR,
+                       "=> 0x%" PRIxPTR,
                        (void *)context, ufc->lsda);
   return ufc->lsda;
 }
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index 32e6fb43d988f..3831d8e071ef3 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -1010,6 +1010,9 @@ class UnwindCursor : public AbstractUnwindCursor{
   template <typename Registers> int stepThroughSigReturn(Registers &) {
     return UNW_STEP_END;
   }
+#elif defined(_LIBUNWIND_TARGET_HAIKU)
+  bool setInfoForSigReturn();
+  int stepThroughSigReturn();
 #endif
 
 #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
@@ -1313,7 +1316,8 @@ class UnwindCursor : public AbstractUnwindCursor{
   unw_proc_info_t  _info;
   bool             _unwindInfoMissing;
   bool             _isSignalFrame;
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   bool             _isSigReturn = false;
 #endif
 };
@@ -2549,7 +2553,8 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
 
 template <typename A, typename R>
 void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   _isSigReturn = false;
 #endif
 
@@ -2673,7 +2678,8 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
   }
 #endif // #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
 
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   if (setInfoForSigReturn())
     return;
 #endif
@@ -2749,6 +2755,63 @@ int UnwindCursor<A, R>::stepThroughSigReturn(Registers_arm64 &) {
   _isSignalFrame = true;
   return UNW_STEP_SUCCESS;
 }
+
+#elif defined(_LIBUNWIND_TARGET_HAIKU) && defined(_LIBUNWIND_TARGET_X86_64)
+#include <commpage_defs.h>
+#include <signal.h>
+
+extern "C" {
+extern void *__gCommPageAddress;
+}
+
+template <typename A, typename R>
+bool UnwindCursor<A, R>::setInfoForSigReturn() {
+#if defined(_LIBUNWIND_TARGET_X86_64)
+  addr_t signal_handler =
+      (((addr_t *)__gCommPageAddress)[COMMPAGE_ENTRY_X86_SIGNAL_HANDLER] +
+       (addr_t)__gCommPageAddress);
+  addr_t signal_handler_ret = signal_handler + 45;
+#endif
+  pint_t pc = static_cast<pint_t>(this->getReg(UNW_REG_IP));
+  if (pc == signal_handler_ret) {
+    _info = {};
+    _info.start_ip = signal_handler;
+    _info.end_ip = signal_handler_ret;
+    _isSigReturn = true;
+    return true;
+  }
+  return false;
+}
+
+template <typename A, typename R>
+int UnwindCursor<A, R>::stepThroughSigReturn() {
+  _isSignalFrame = true;
+  pint_t sp = _registers.getSP();
+#if defined(_LIBUNWIND_TARGET_X86_64)
+  vregs *regs = (vregs *)(sp + 0x70);
+
+  _registers.setRegister(UNW_REG_IP, regs->rip);
+  _registers.setRegister(UNW_REG_SP, regs->rsp);
+  _registers.setRegister(UNW_X86_64_RAX, regs->rax);
+  _registers.setRegister(UNW_X86_64_RDX, regs->rdx);
+  _registers.setRegister(UNW_X86_64_RCX, regs->rcx);
+  _registers.setRegister(UNW_X86_64_RBX, regs->rbx);
+  _registers.setRegister(UNW_X86_64_RSI, regs->rsi);
+  _registers.setRegister(UNW_X86_64_RDI, regs->rdi);
+  _registers.setRegister(UNW_X86_64_RBP, regs->rbp);
+  _registers.setRegister(UNW_X86_64_R8, regs->r8);
+  _registers.setRegister(UNW_X86_64_R9, regs->r9);
+  _registers.setRegister(UNW_X86_64_R10, regs->r10);
+  _registers.setRegister(UNW_X86_64_R11, regs->r11);
+  _registers.setRegister(UNW_X86_64_R12, regs->r12);
+  _registers.setRegister(UNW_X86_64_R13, regs->r13);
+  _registers.setRegister(UNW_X86_64_R14, regs->r14);
+  _registers.setRegister(UNW_X86_64_R15, regs->r15);
+  // TODO: XMM
+#endif
+
+  return UNW_STEP_SUCCESS;
+}
 #endif // defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&
        // defined(_LIBUNWIND_TARGET_AARCH64)
 
@@ -2917,7 +2980,8 @@ template <typename A, typename R> int UnwindCursor<A, R>::step(bool stage2) {
 
   // Use unwinding info to modify register set as if function returned.
   int result;
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   if (_isSigReturn) {
     result = this->stepThroughSigReturn();
   } else
diff --git a/lld/CODE_OWNERS.TXT b/lld/CODE_OWNERS.TXT
deleted file mode 100644
index 44972c0d345a8..0000000000000
--- a/lld/CODE_OWNERS.TXT
+++ /dev/null
@@ -1,26 +0,0 @@
-This file is a list of the people responsible for ensuring that patches for a
-particular part of LLD are reviewed, either by themself or by someone else.
-They are also the gatekeepers for their part of LLD, with the final word on
-what goes in or not.
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts.  The fields are: name (N), email (E), web-address
-(W), PGP key ID and fingerprint (P), description (D), and snail-mail address
-(S). Each entry should contain at least the (N), (E) and (D) fields.
-
-
-N: Rui Ueyama
-E: ruiu@google.com
-D: COFF, ELF backends (COFF/* ELF/*)
-
-N: Lang Hames, Nick Kledzik
-E: lhames@gmail.com, kledzik@apple.com
-D: Old Mach-O backend
-
-N: Sam Clegg
-E: sbc@chromium.org
-D: WebAssembly backend (wasm/*)
-
-N: Jez Ng, Greg McGary, Shoaib Meenai
-E: jezng@fb.com, gkm@fb.com, smeenai@fb.com
-D: New Mach-O backend
diff --git a/lld/COFF/COFFLinkerContext.h b/lld/COFF/COFFLinkerContext.h
index 059d4aeddc6e5..5d89e97a7f776 100644
--- a/lld/COFF/COFFLinkerContext.h
+++ b/lld/COFF/COFFLinkerContext.h
@@ -88,6 +88,8 @@ class COFFLinkerContext : public CommonLinkerContext {
   Timer diskCommitTimer;
 
   Configuration config;
+
+  DynamicRelocsChunk *dynamicRelocs = nullptr;
 };
 
 } // namespace lld::coff
diff --git a/lld/COFF/CallGraphSort.cpp b/lld/COFF/CallGraphSort.cpp
index e83996b035f63..9cd60dea6e377 100644
--- a/lld/COFF/CallGraphSort.cpp
+++ b/lld/COFF/CallGraphSort.cpp
@@ -49,7 +49,7 @@ struct Cluster {
 
 class CallGraphSort {
 public:
-  CallGraphSort(const COFFLinkerContext &ctx);
+  CallGraphSort(COFFLinkerContext &ctx);
 
   DenseMap<const SectionChunk *, int> run();
 
@@ -57,7 +57,7 @@ class CallGraphSort {
   std::vector<Cluster> clusters;
   std::vector<const SectionChunk *> sections;
 
-  const COFFLinkerContext &ctx;
+  COFFLinkerContext &ctx;
 };
 
 // Maximum amount the combined cluster density can be worse than the original
@@ -73,7 +73,7 @@ using SectionPair = std::pair<const SectionChunk *, const SectionChunk *>;
 // Take the edge list in Config->CallGraphProfile, resolve symbol names to
 // Symbols, and generate a graph between InputSections with the provided
 // weights.
-CallGraphSort::CallGraphSort(const COFFLinkerContext &ctx) : ctx(ctx) {
+CallGraphSort::CallGraphSort(COFFLinkerContext &ctx) : ctx(ctx) {
   const MapVector<SectionPair, uint64_t> &profile = ctx.config.callGraphProfile;
   DenseMap<const SectionChunk *, int> secToCluster;
 
@@ -211,7 +211,8 @@ DenseMap<const SectionChunk *, int> CallGraphSort::run() {
     std::error_code ec;
     raw_fd_ostream os(ctx.config.printSymbolOrder, ec, sys::fs::OF_None);
     if (ec) {
-      error("cannot open " + ctx.config.printSymbolOrder + ": " + ec.message());
+      Err(ctx) << "cannot open " << ctx.config.printSymbolOrder << ": "
+               << ec.message();
       return orderMap;
     }
     // Print the symbols ordered by C3, in the order of increasing curOrder
@@ -244,6 +245,6 @@ DenseMap<const SectionChunk *, int> CallGraphSort::run() {
 // according to the C³ heuristic. All clusters are then sorted by a density
 // metric to further improve locality.
 DenseMap<const SectionChunk *, int>
-coff::computeCallGraphProfileOrder(const COFFLinkerContext &ctx) {
+coff::computeCallGraphProfileOrder(COFFLinkerContext &ctx) {
   return CallGraphSort(ctx).run();
 }
diff --git a/lld/COFF/CallGraphSort.h b/lld/COFF/CallGraphSort.h
index 60f2941d37a82..76aa093740a19 100644
--- a/lld/COFF/CallGraphSort.h
+++ b/lld/COFF/CallGraphSort.h
@@ -16,7 +16,7 @@ class SectionChunk;
 class COFFLinkerContext;
 
 llvm::DenseMap<const SectionChunk *, int>
-computeCallGraphProfileOrder(const COFFLinkerContext &ctx);
+computeCallGraphProfileOrder(COFFLinkerContext &ctx);
 } // namespace lld::coff
 
 #endif
diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index a82e37ff05230..23fab0e66bb67 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -25,6 +25,7 @@
 
 using namespace llvm;
 using namespace llvm::object;
+using namespace llvm::support;
 using namespace llvm::support::endian;
 using namespace llvm::COFF;
 using llvm::support::ulittle32_t;
@@ -1147,4 +1148,81 @@ uint32_t ImportThunkChunkARM64EC::extendRanges() {
   return sizeof(arm64Thunk) - sizeof(uint32_t);
 }
 
+size_t Arm64XDynamicRelocEntry::getSize() const {
+  switch (type) {
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE:
+    return sizeof(uint16_t) + size; // A header and a payload.
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA:
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
+    llvm_unreachable("unsupported type");
+  }
+}
+
+void Arm64XDynamicRelocEntry::writeTo(uint8_t *buf) const {
+  auto out = reinterpret_cast<ulittle16_t *>(buf);
+  *out = (offset & 0xfff) | (type << 12);
+
+  switch (type) {
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE:
+    *out |= ((bit_width(size) - 1) << 14); // Encode the size.
+    switch (size) {
+    case 2:
+      out[1] = value;
+      break;
+    case 4:
+      *reinterpret_cast<ulittle32_t *>(out + 1) = value;
+      break;
+    case 8:
+      *reinterpret_cast<ulittle64_t *>(out + 1) = value;
+      break;
+    default:
+      llvm_unreachable("invalid size");
+    }
+    break;
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_DELTA:
+  case IMAGE_DVRT_ARM64X_FIXUP_TYPE_ZEROFILL:
+    llvm_unreachable("unsupported type");
+  }
+}
+
+void DynamicRelocsChunk::finalize() {
+  llvm::stable_sort(arm64xRelocs, [=](const Arm64XDynamicRelocEntry &a,
+                                      const Arm64XDynamicRelocEntry &b) {
+    return a.offset < b.offset;
+  });
+
+  size = sizeof(coff_dynamic_reloc_table) + sizeof(coff_dynamic_relocation64) +
+         sizeof(coff_base_reloc_block_header);
+
+  for (const Arm64XDynamicRelocEntry &entry : arm64xRelocs) {
+    assert(!(entry.offset & ~0xfff)); // Not yet supported.
+    size += entry.getSize();
+  }
+
+  size = alignTo(size, sizeof(uint32_t));
+}
+
+void DynamicRelocsChunk::writeTo(uint8_t *buf) const {
+  auto table = reinterpret_cast<coff_dynamic_reloc_table *>(buf);
+  table->Version = 1;
+  table->Size = sizeof(coff_dynamic_relocation64);
+  buf += sizeof(*table);
+
+  auto header = reinterpret_cast<coff_dynamic_relocation64 *>(buf);
+  header->Symbol = IMAGE_DYNAMIC_RELOCATION_ARM64X;
+  buf += sizeof(*header);
+
+  auto pageHeader = reinterpret_cast<coff_base_reloc_block_header *>(buf);
+  pageHeader->BlockSize = sizeof(*pageHeader);
+  for (const Arm64XDynamicRelocEntry &entry : arm64xRelocs) {
+    entry.writeTo(buf + pageHeader->BlockSize);
+    pageHeader->BlockSize += entry.getSize();
+  }
+  pageHeader->BlockSize = alignTo(pageHeader->BlockSize, sizeof(uint32_t));
+
+  header->BaseRelocSize = pageHeader->BlockSize;
+  table->Size += header->BaseRelocSize;
+  assert(size == sizeof(*table) + sizeof(*header) + header->BaseRelocSize);
+}
+
 } // namespace lld::coff
diff --git a/lld/COFF/Chunks.h b/lld/COFF/Chunks.h
index 42284f485e5c0..0d2b2ac0f15ea 100644
--- a/lld/COFF/Chunks.h
+++ b/lld/COFF/Chunks.h
@@ -835,6 +835,42 @@ class ECExportThunkChunk : public NonSectionCodeChunk {
   Defined *target;
 };
 
+// ARM64X entry for dynamic relocations.
+class Arm64XDynamicRelocEntry {
+public:
+  Arm64XDynamicRelocEntry(llvm::COFF::Arm64XFixupType type, uint8_t size,
+                          uint32_t offset, uint64_t value)
+      : offset(offset), value(value), type(type), size(size) {}
+
+  size_t getSize() const;
+  void writeTo(uint8_t *buf) const;
+
+  uint32_t offset;
+  uint64_t value;
+
+private:
+  llvm::COFF::Arm64XFixupType type;
+  uint8_t size;
+};
+
+// Dynamic relocation chunk containing ARM64X relocations for the hybrid image.
+class DynamicRelocsChunk : public NonSectionChunk {
+public:
+  DynamicRelocsChunk() {}
+  size_t getSize() const override { return size; }
+  void writeTo(uint8_t *buf) const override;
+  void finalize();
+
+  void add(llvm::COFF::Arm64XFixupType type, uint8_t size, uint32_t offset,
+           uint64_t value) {
+    arm64xRelocs.emplace_back(type, size, offset, value);
+  }
+
+private:
+  std::vector<Arm64XDynamicRelocEntry> arm64xRelocs;
+  size_t size;
+};
+
 // MinGW specific, for the "automatic import of variables from DLLs" feature.
 // This provides the table of runtime pseudo relocations, for variable
 // references that turned out to need to be imported from a DLL even though
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 57cb443798cd8..9e6b17e87c9e7 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -9,6 +9,7 @@
 #ifndef LLD_COFF_CONFIG_H
 #define LLD_COFF_CONFIG_H
 
+#include "lld/Common/ErrorHandler.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
@@ -27,6 +28,7 @@ namespace lld::coff {
 using llvm::COFF::IMAGE_FILE_MACHINE_UNKNOWN;
 using llvm::COFF::WindowsSubsystem;
 using llvm::StringRef;
+class COFFLinkerContext;
 class DefinedAbsolute;
 class StringChunk;
 class Symbol;
@@ -332,6 +334,48 @@ struct Configuration {
   BuildIDHash buildIDHash = BuildIDHash::None;
 };
 
+struct COFFSyncStream : SyncStream {
+  COFFLinkerContext &ctx;
+  COFFSyncStream(COFFLinkerContext &ctx, DiagLevel level);
+};
+
+template <typename T>
+std::enable_if_t<!std::is_pointer_v<std::remove_reference_t<T>>,
+                 const COFFSyncStream &>
+operator<<(const COFFSyncStream &s, T &&v) {
+  s.os << std::forward<T>(v);
+  return s;
+}
+
+inline const COFFSyncStream &operator<<(const COFFSyncStream &s,
+                                        const char *v) {
+  s.os << v;
+  return s;
+}
+
+inline const COFFSyncStream &operator<<(const COFFSyncStream &s, Error v) {
+  s.os << llvm::toString(std::move(v));
+  return s;
+}
+
+// Report a log if -verbose is specified.
+COFFSyncStream Log(COFFLinkerContext &ctx);
+
+// Print a message to stdout.
+COFFSyncStream Msg(COFFLinkerContext &ctx);
+
+// Report a warning. Upgraded to an error if /WX is specified.
+COFFSyncStream Warn(COFFLinkerContext &ctx);
+
+// Report an error that will suppress the output file generation.
+COFFSyncStream Err(COFFLinkerContext &ctx);
+
+// Report a fatal error that exits immediately. This should generally be avoided
+// in favor of Err.
+COFFSyncStream Fatal(COFFLinkerContext &ctx);
+
+uint64_t errCount(COFFLinkerContext &ctx);
+
 } // namespace lld::coff
 
 #endif
diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp
index 7689ad163a657..d4d80bfd92efb 100644
--- a/lld/COFF/DebugTypes.cpp
+++ b/lld/COFF/DebugTypes.cpp
@@ -231,7 +231,7 @@ void TpiSource::remapRecord(MutableArrayRef<uint8_t> rec,
   for (const TiReference &ref : typeRefs) {
     unsigned byteSize = ref.Count * sizeof(TypeIndex);
     if (contents.size() < ref.Offset + byteSize)
-      fatal("symbol record too short");
+      Fatal(ctx) << "symbol record too short";
 
     MutableArrayRef<TypeIndex> indices(
         reinterpret_cast<TypeIndex *>(contents.data() + ref.Offset), ref.Count);
@@ -241,10 +241,10 @@ void TpiSource::remapRecord(MutableArrayRef<uint8_t> rec,
           uint16_t kind =
               reinterpret_cast<const RecordPrefix *>(rec.data())->RecordKind;
           StringRef fname = file ? file->getName() : "<unknown PDB>";
-          log("failed to remap type index in record of kind 0x" +
-              utohexstr(kind) + " in " + fname + " with bad " +
-              (ref.Kind == TiRefKind::IndexRef ? "item" : "type") +
-              " index 0x" + utohexstr(ti.getIndex()));
+          Log(ctx) << "failed to remap type index in record of kind 0x"
+                   << utohexstr(kind) << " in " << fname << " with bad "
+                   << (ref.Kind == TiRefKind::IndexRef ? "item" : "type")
+                   << " index 0x" << utohexstr(ti.getIndex());
         }
         ti = TypeIndex(SimpleTypeKind::NotTranslated);
         continue;
@@ -320,8 +320,8 @@ Error TpiSource::mergeDebugT(TypeMerger *m) {
   std::optional<PCHMergerInfo> pchInfo;
   if (auto err = mergeTypeAndIdRecords(m->idTable, m->typeTable,
                                        indexMapStorage, types, pchInfo))
-    fatal("codeview::mergeTypeAndIdRecords failed: " +
-          toString(std::move(err)));
+    Fatal(ctx) << "codeview::mergeTypeAndIdRecords failed: "
+               << toString(std::move(err));
   if (pchInfo) {
     file->pchSignature = pchInfo->PCHSignature;
     endPrecompIdx = pchInfo->EndPrecompIndex;
@@ -364,26 +364,30 @@ Error TypeServerSource::mergeDebugT(TypeMerger *m) {
   pdb::PDBFile &pdbFile = pdbInputFile->session->getPDBFile();
   Expected<pdb::TpiStream &> expectedTpi = pdbFile.getPDBTpiStream();
   if (auto e = expectedTpi.takeError())
-    fatal("Type server does not have TPI stream: " + toString(std::move(e)));
+    Fatal(ctx) << "Type server does not have TPI stream: "
+               << toString(std::move(e));
   pdb::TpiStream *maybeIpi = nullptr;
   if (pdbFile.hasPDBIpiStream()) {
     Expected<pdb::TpiStream &> expectedIpi = pdbFile.getPDBIpiStream();
     if (auto e = expectedIpi.takeError())
-      fatal("Error getting type server IPI stream: " + toString(std::move(e)));
+      Fatal(ctx) << "Error getting type server IPI stream: "
+                 << toString(std::move(e));
     maybeIpi = &*expectedIpi;
   }
 
   // Merge TPI first, because the IPI stream will reference type indices.
   if (auto err = mergeTypeRecords(m->typeTable, indexMapStorage,
                                   expectedTpi->typeArray()))
-    fatal("codeview::mergeTypeRecords failed: " + toString(std::move(err)));
+    Fatal(ctx) << "codeview::mergeTypeRecords failed: "
+               << toString(std::move(err));
   tpiMap = indexMapStorage;
 
   // Merge IPI.
   if (maybeIpi) {
     if (auto err = mergeIdRecords(m->idTable, tpiMap, ipiSrc->indexMapStorage,
                                   maybeIpi->typeArray()))
-      fatal("codeview::mergeIdRecords failed: " + toString(std::move(err)));
+      Fatal(ctx) << "codeview::mergeIdRecords failed: "
+                 << toString(std::move(err));
     ipiMap = ipiSrc->indexMapStorage;
   }
 
@@ -572,8 +576,10 @@ void PrecompSource::registerMapping() {
   if (file->pchSignature && *file->pchSignature) {
     auto it = ctx.precompSourceMappings.emplace(*file->pchSignature, this);
     if (!it.second)
-      fatal("a PCH object with the same signature has already been provided (" +
-            toString(it.first->second->file) + " and " + toString(file) + ")");
+      Fatal(ctx)
+          << "a PCH object with the same signature has already been provided ("
+          << toString(it.first->second->file) << " and " << toString(file)
+          << ")";
     registered = true;
   }
 }
@@ -674,8 +680,8 @@ void TpiSource::mergeTypeRecord(TypeIndex curIndex, CVType ty) {
       funcIdToType.push_back({funcId, funcType});
     } else {
       StringRef fname = file ? file->getName() : "<unknown PDB>";
-      warn("corrupt LF_[M]FUNC_ID record 0x" + utohexstr(curIndex.getIndex()) +
-           " in " + fname);
+      Warn(ctx) << "corrupt LF_[M]FUNC_ID record 0x"
+                << utohexstr(curIndex.getIndex()) << " in " << fname;
     }
   }
 }
@@ -756,7 +762,8 @@ void TypeServerSource::loadGHashes() {
   // Hash TPI stream.
   Expected<pdb::TpiStream &> expectedTpi = pdbFile.getPDBTpiStream();
   if (auto e = expectedTpi.takeError())
-    fatal("Type server does not have TPI stream: " + toString(std::move(e)));
+    Fatal(ctx) << "Type server does not have TPI stream: "
+               << toString(std::move(e));
   assignGHashesFromVector(
       GloballyHashedType::hashTypes(expectedTpi->typeArray()));
   isItemIndex.resize(ghashes.size());
@@ -766,7 +773,7 @@ void TypeServerSource::loadGHashes() {
     return;
   Expected<pdb::TpiStream &> expectedIpi = pdbFile.getPDBIpiStream();
   if (auto e = expectedIpi.takeError())
-    fatal("error retrieving IPI stream: " + toString(std::move(e)));
+    Fatal(ctx) << "error retrieving IPI stream: " << toString(std::move(e));
   ipiSrc->assignGHashesFromVector(
       GloballyHashedType::hashIds(expectedIpi->typeArray(), ghashes));
 
@@ -836,7 +843,7 @@ void UseTypeServerSource::remapTpiWithGHashes(GHashState *g) {
 
 void PrecompSource::loadGHashes() {
   if (getDebugH(file)) {
-    warn("ignoring .debug$H section; pch with ghash is not implemented");
+    Warn(ctx) << "ignoring .debug$H section; pch with ghash is not implemented";
   }
 
   uint32_t ghashIdx = 0;
@@ -864,7 +871,7 @@ void PrecompSource::loadGHashes() {
 void UsePrecompSource::loadGHashes() {
   auto e = findPrecompMap(file, precompDependency);
   if (!e) {
-    warn(toString(e.takeError()));
+    Warn(ctx) << e.takeError();
     return;
   }
 
@@ -1137,9 +1144,10 @@ void TypeMerger::mergeTypesWithGHash() {
       entries.push_back(cell);
   }
   parallelSort(entries, std::less<GHashCell>());
-  log(formatv("ghash table load factor: {0:p} (size {1} / capacity {2})\n",
-              tableSize ? double(entries.size()) / tableSize : 0,
-              entries.size(), tableSize));
+  Log(ctx) << formatv(
+      "ghash table load factor: {0:p} (size {1} / capacity {2})\n",
+      tableSize ? double(entries.size()) / tableSize : 0, entries.size(),
+      tableSize);
 
   // Find out how many type and item indices there are.
   auto mid = llvm::lower_bound(entries, GHashCell(true, 0, 0));
@@ -1148,8 +1156,8 @@ void TypeMerger::mergeTypesWithGHash() {
          "midpoint is not midpoint");
   uint32_t numTypes = std::distance(entries.begin(), mid);
   uint32_t numItems = std::distance(mid, entries.end());
-  log("Tpi record count: " + Twine(numTypes));
-  log("Ipi record count: " + Twine(numItems));
+  Log(ctx) << "Tpi record count: " << numTypes;
+  Log(ctx) << "Ipi record count: " << numItems;
 
   // Make a list of the "unique" type records to merge for each tpi source. Type
   // merging will skip indices not on this list. Store the destination PDB type
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index a0bff69c6302a..714de67e88b06 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -56,11 +56,33 @@
 #include <optional>
 #include <tuple>
 
+using namespace lld;
+using namespace lld::coff;
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::COFF;
 using namespace llvm::sys;
 
+COFFSyncStream::COFFSyncStream(COFFLinkerContext &ctx, DiagLevel level)
+    : SyncStream(ctx.e, level), ctx(ctx) {}
+
+COFFSyncStream coff::Log(COFFLinkerContext &ctx) {
+  return {ctx, DiagLevel::Log};
+}
+COFFSyncStream coff::Msg(COFFLinkerContext &ctx) {
+  return {ctx, DiagLevel::Msg};
+}
+COFFSyncStream coff::Warn(COFFLinkerContext &ctx) {
+  return {ctx, DiagLevel::Warn};
+}
+COFFSyncStream coff::Err(COFFLinkerContext &ctx) {
+  return {ctx, DiagLevel::Err};
+}
+COFFSyncStream coff::Fatal(COFFLinkerContext &ctx) {
+  return {ctx, DiagLevel::Fatal};
+}
+uint64_t coff::errCount(COFFLinkerContext &ctx) { return ctx.e.errorCount; }
+
 namespace lld::coff {
 
 bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
@@ -75,12 +97,12 @@ bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
 
   ctx->driver.linkerMain(args);
 
-  return errorCount() == 0;
+  return errCount(*ctx) == 0;
 }
 
 // Parse options of the form "old;new".
-static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
-                                                        unsigned id) {
+static std::pair<StringRef, StringRef>
+getOldNewOptions(COFFLinkerContext &ctx, opt::InputArgList &args, unsigned id) {
   auto *arg = args.getLastArg(id);
   if (!arg)
     return {"", ""};
@@ -88,14 +110,16 @@ static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
   StringRef s = arg->getValue();
   std::pair<StringRef, StringRef> ret = s.split(';');
   if (ret.second.empty())
-    error(arg->getSpelling() + " expects 'old;new' format, but got " + s);
+    Err(ctx) << arg->getSpelling() << " expects 'old;new' format, but got "
+             << s;
   return ret;
 }
 
 // Parse options of the form "old;new[;extra]".
 static std::tuple<StringRef, StringRef, StringRef>
-getOldNewOptionsExtra(opt::InputArgList &args, unsigned id) {
-  auto [oldDir, second] = getOldNewOptions(args, id);
+getOldNewOptionsExtra(COFFLinkerContext &ctx, opt::InputArgList &args,
+                      unsigned id) {
+  auto [oldDir, second] = getOldNewOptions(ctx, args, id);
   auto [newDir, extraDir] = second.split(';');
   return {oldDir, newDir, extraDir};
 }
@@ -195,7 +219,7 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
       make<std::unique_ptr<Archive>>(std::move(file)); // take ownership
 
       int memberIndex = 0;
-      for (MemoryBufferRef m : getArchiveMembers(archive))
+      for (MemoryBufferRef m : getArchiveMembers(ctx, archive))
         addArchiveBuffer(m, "<whole-archive>", filename, memberIndex++);
       return;
     }
@@ -212,7 +236,8 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
     ctx.symtab.addFile(make<PDBInputFile>(ctx, mbref));
     break;
   case file_magic::coff_cl_gl_object:
-    error(filename + ": is not a native COFF file. Recompile without /GL");
+    Err(ctx) << filename
+             << ": is not a native COFF file. Recompile without /GL";
     break;
   case file_magic::pecoff_executable:
     if (ctx.config.mingw) {
@@ -220,13 +245,14 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb,
       break;
     }
     if (filename.ends_with_insensitive(".dll")) {
-      error(filename + ": bad file type. Did you specify a DLL instead of an "
-                       "import library?");
+      Err(ctx) << filename
+               << ": bad file type. Did you specify a DLL instead of an "
+                  "import library?";
       break;
     }
     [[fallthrough]];
   default:
-    error(mbref.getBufferIdentifier() + ": unknown file type");
+    Err(ctx) << mbref.getBufferIdentifier() << ": unknown file type";
     break;
   }
 }
@@ -266,9 +292,9 @@ void LinkerDriver::enqueuePath(StringRef path, bool wholeArchive, bool lazy) {
       // directory.
       std::string nearest;
       if (ctx.optTable.findNearest(pathStr, nearest) > 1)
-        error(msg);
+        Err(ctx) << msg;
       else
-        error(msg + "; did you mean '" + nearest + "'");
+        Err(ctx) << msg << "; did you mean '" << nearest << "'";
     } else
       ctx.driver.addBuffer(std::move(mb), wholeArchive, lazy);
   });
@@ -292,17 +318,17 @@ void LinkerDriver::addArchiveBuffer(MemoryBufferRef mb, StringRef symName,
     obj =
         make<BitcodeFile>(ctx, mb, parentName, offsetInArchive, /*lazy=*/false);
   } else if (magic == file_magic::coff_cl_gl_object) {
-    error(mb.getBufferIdentifier() +
-          ": is not a native COFF file. Recompile without /GL?");
+    Err(ctx) << mb.getBufferIdentifier()
+             << ": is not a native COFF file. Recompile without /GL?";
     return;
   } else {
-    error("unknown file type: " + mb.getBufferIdentifier());
+    Err(ctx) << "unknown file type: " << mb.getBufferIdentifier();
     return;
   }
 
   obj->parentName = parentName;
   ctx.symtab.addFile(obj);
-  log("Loaded " + toString(obj) + " for " + symName);
+  Log(ctx) << "Loaded " << obj << " for " << symName;
 }
 
 void LinkerDriver::enqueueArchiveMember(const Archive::Child &c,
@@ -310,9 +336,9 @@ void LinkerDriver::enqueueArchiveMember(const Archive::Child &c,
                                         StringRef parentName) {
 
   auto reportBufferError = [=](Error &&e, StringRef childName) {
-    fatal("could not get the buffer for the member defining symbol " +
-          toCOFFString(ctx, sym) + ": " + parentName + "(" + childName +
-          "): " + toString(std::move(e)));
+    Fatal(ctx) << "could not get the buffer for the member defining symbol "
+               << &sym << ": " << parentName << "(" << childName
+               << "): " << std::move(e);
   };
 
   if (!c.getParent()->isThin()) {
@@ -361,7 +387,7 @@ void LinkerDriver::parseDirectives(InputFile *file) {
   if (s.empty())
     return;
 
-  log("Directives: " + toString(file) + ": " + s);
+  Log(ctx) << "Directives: " << file << ": " << s;
 
   ArgParser parser(ctx);
   // .drectve is always tokenized using Windows shell rules.
@@ -414,7 +440,7 @@ void LinkerDriver::parseDirectives(InputFile *file) {
       break;
     case OPT_entry:
       if (!arg->getValue()[0])
-        fatal("missing entry point symbol name");
+        Fatal(ctx) << "missing entry point symbol name";
       ctx.config.entry = addUndefined(mangle(arg->getValue()), true);
       break;
     case OPT_failifmismatch:
@@ -462,8 +488,8 @@ void LinkerDriver::parseDirectives(InputFile *file) {
     case OPT_inferasanlibs_no:
       break;
     default:
-      error(arg->getSpelling() + " is not allowed in .drectve (" +
-            toString(file) + ")");
+      Err(ctx) << arg->getSpelling() << " is not allowed in .drectve ("
+               << toString(file) << ")";
     }
   }
 }
@@ -728,7 +754,7 @@ Symbol *LinkerDriver::addUndefined(StringRef name, bool aliasEC) {
 void LinkerDriver::addUndefinedGlob(StringRef arg) {
   Expected<GlobPattern> pat = GlobPattern::create(arg);
   if (!pat) {
-    error("/includeglob: " + toString(pat.takeError()));
+    Err(ctx) << "/includeglob: " << toString(pat.takeError());
     return;
   }
 
@@ -756,7 +782,7 @@ StringRef LinkerDriver::mangleMaybe(Symbol *s) {
 
   // If we find a similar mangled symbol, make this an alias to it and return
   // its name.
-  log(unmangled->getName() + " aliased to " + mangled->getName());
+  Log(ctx) << unmangled->getName() << " aliased to " << mangled->getName();
   unmangled->setWeakAlias(ctx.symtab.addUndefined(mangled->getName()));
   return mangled->getName();
 }
@@ -779,14 +805,14 @@ StringRef LinkerDriver::findDefaultEntry() {
     if (findUnderscoreMangle("wWinMain")) {
       if (!findUnderscoreMangle("WinMain"))
         return mangle("wWinMainCRTStartup");
-      warn("found both wWinMain and WinMain; using latter");
+      Warn(ctx) << "found both wWinMain and WinMain; using latter";
     }
     return mangle("WinMainCRTStartup");
   }
   if (findUnderscoreMangle("wmain")) {
     if (!findUnderscoreMangle("main"))
       return mangle("wmainCRTStartup");
-    warn("found both wmain and main; using latter");
+    Warn(ctx) << "found both wmain and main; using latter";
   }
   return mangle("mainCRTStartup");
 }
@@ -805,9 +831,9 @@ WindowsSubsystem LinkerDriver::inferSubsystem() {
   bool haveWWinMain = findUnderscoreMangle("wWinMain");
   if (haveMain || haveWMain) {
     if (haveWinMain || haveWWinMain) {
-      warn(std::string("found ") + (haveMain ? "main" : "wmain") + " and " +
-           (haveWinMain ? "WinMain" : "wWinMain") +
-           "; defaulting to /subsystem:console");
+      Warn(ctx) << "found " << (haveMain ? "main" : "wmain") << " and "
+                << (haveWinMain ? "WinMain" : "wWinMain")
+                << "; defaulting to /subsystem:console";
     }
     return IMAGE_SUBSYSTEM_WINDOWS_CUI;
   }
@@ -887,7 +913,8 @@ static std::string createResponseFile(const opt::InputArgList &args,
   return std::string(data);
 }
 
-static unsigned parseDebugTypes(const opt::InputArgList &args) {
+static unsigned parseDebugTypes(COFFLinkerContext &ctx,
+                                const opt::InputArgList &args) {
   unsigned debugTypes = static_cast<unsigned>(DebugType::None);
 
   if (auto *a = args.getLastArg(OPT_debugtype)) {
@@ -902,7 +929,7 @@ static unsigned parseDebugTypes(const opt::InputArgList &args) {
                        .Case("fixup", static_cast<unsigned>(DebugType::Fixup))
                        .Default(0);
       if (v == 0) {
-        warn("/debugtype: unknown option '" + type + "'");
+        Warn(ctx) << "/debugtype: unknown option '" << type << "'";
         continue;
       }
       debugTypes |= v;
@@ -1006,8 +1033,8 @@ void LinkerDriver::createImportLibrary(bool asLib) {
   SmallString<128> tmpName;
   if (std::error_code ec =
           sys::fs::createUniqueFile(path + ".tmp-%%%%%%%%.lib", tmpName))
-    fatal("cannot create temporary file for import library " + path + ": " +
-          ec.message());
+    Fatal(ctx) << "cannot create temporary file for import library " << path
+               << ": " << ec.message();
 
   if (Error e = writeImportLibrary(libName, tmpName, exports,
                                    ctx.config.machine, ctx.config.mingw)) {
@@ -1109,7 +1136,7 @@ void LinkerDriver::parseOrderFile(StringRef arg) {
   // For some reason, the MSVC linker requires a filename to be
   // preceded by "@".
   if (!arg.starts_with("@")) {
-    error("malformed /order option: '@' missing");
+    Err(ctx) << "malformed /order option: '@' missing";
     return;
   }
 
@@ -1139,7 +1166,8 @@ void LinkerDriver::parseOrderFile(StringRef arg) {
 
     if (set.count(s) == 0) {
       if (ctx.config.warnMissingOrderSymbol)
-        warn("/order:" + arg + ": missing symbol: " + s + " [LNK4037]");
+        Warn(ctx) << "/order:" << arg << ": missing symbol: " << s
+                  << " [LNK4037]";
     } else
       ctx.config.order[s] = INT_MIN + ctx.config.order.size();
   }
@@ -1166,7 +1194,7 @@ void LinkerDriver::parseCallGraphFile(StringRef path) {
     Symbol *sym = map.lookup(name);
     if (!sym) {
       if (ctx.config.warnMissingOrderSymbol)
-        warn(path + ": no such symbol: " + name);
+        Warn(ctx) << path << ": no such symbol: " << name;
       return nullptr;
     }
 
@@ -1181,7 +1209,7 @@ void LinkerDriver::parseCallGraphFile(StringRef path) {
     uint64_t count;
 
     if (fields.size() != 3 || !to_integer(fields[2], count)) {
-      error(path + ": parse error");
+      Err(ctx) << path << ": parse error";
       return;
     }
 
@@ -1205,11 +1233,11 @@ static void readCallGraphsFromObjectFiles(COFFLinkerContext &ctx) {
         uint32_t fromIndex, toIndex;
         uint64_t count;
         if (Error err = reader.readInteger(fromIndex))
-          fatal(toString(obj) + ": Expected 32-bit integer");
+          Fatal(ctx) << toString(obj) << ": Expected 32-bit integer";
         if (Error err = reader.readInteger(toIndex))
-          fatal(toString(obj) + ": Expected 32-bit integer");
+          Fatal(ctx) << toString(obj) << ": Expected 32-bit integer";
         if (Error err = reader.readInteger(count))
-          fatal(toString(obj) + ": Expected 64-bit integer");
+          Fatal(ctx) << toString(obj) << ": Expected 64-bit integer";
         auto *fromSym = dyn_cast_or_null<Defined>(obj->getSymbol(fromIndex));
         auto *toSym = dyn_cast_or_null<Defined>(obj->getSymbol(toIndex));
         if (!fromSym || !toSym)
@@ -1251,9 +1279,11 @@ static void findKeepUniqueSections(COFFLinkerContext &ctx) {
         const char *err = nullptr;
         uint64_t symIndex = decodeULEB128(cur, &size, contents.end(), &err);
         if (err)
-          fatal(toString(obj) + ": could not decode addrsig section: " + err);
+          Fatal(ctx) << toString(obj)
+                     << ": could not decode addrsig section: " << err;
         if (symIndex >= syms.size())
-          fatal(toString(obj) + ": invalid symbol index in addrsig section");
+          Fatal(ctx) << toString(obj)
+                     << ": invalid symbol index in addrsig section";
         markAddrsig(syms[symIndex]);
         cur += size;
       }
@@ -1309,8 +1339,8 @@ void LinkerDriver::parsePDBAltPath() {
     else if (var.equals_insensitive("%_ext%"))
       buf.append(binaryExtension);
     else {
-      warn("only %_PDB% and %_EXT% supported in /pdbaltpath:, keeping " + var +
-           " as literal");
+      Warn(ctx) << "only %_PDB% and %_EXT% supported in /pdbaltpath:, keeping "
+                << var << " as literal";
       buf.append(var);
     }
 
@@ -1335,10 +1365,11 @@ void LinkerDriver::convertResources() {
   if (!ctx.config.mingw &&
       (resourceObjFiles.size() > 1 ||
        (resourceObjFiles.size() == 1 && !resources.empty()))) {
-    error((!resources.empty() ? "internal .obj file created from .res files"
-                              : toString(resourceObjFiles[1])) +
-          ": more than one resource obj file not allowed, already got " +
-          toString(resourceObjFiles.front()));
+    Err(ctx) << (!resources.empty()
+                     ? "internal .obj file created from .res files"
+                     : toString(resourceObjFiles[1]))
+             << ": more than one resource obj file not allowed, already got "
+             << resourceObjFiles.front();
     return;
   }
 
@@ -1503,7 +1534,7 @@ std::optional<std::string> getReproduceFile(const opt::InputArgList &args) {
 }
 
 static std::unique_ptr<llvm::vfs::FileSystem>
-getVFS(const opt::InputArgList &args) {
+getVFS(COFFLinkerContext &ctx, const opt::InputArgList &args) {
   using namespace llvm::vfs;
 
   const opt::Arg *arg = args.getLastArg(OPT_vfsoverlay);
@@ -1520,7 +1551,7 @@ getVFS(const opt::InputArgList &args) {
                                      /*DiagHandler*/ nullptr, arg->getValue()))
     return ret;
 
-  error("Invalid vfs overlay");
+  Err(ctx) << "Invalid vfs overlay";
   return nullptr;
 }
 
@@ -1545,7 +1576,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       (StringRef(argsArr[1]).equals_insensitive("/lib") ||
        StringRef(argsArr[1]).equals_insensitive("-lib"))) {
     if (llvm::libDriverMain(argsArr.slice(1)) != 0)
-      fatal("lib failed");
+      Fatal(ctx) << "lib failed";
     return;
   }
 
@@ -1581,11 +1612,11 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     int n = 20;
     StringRef s = arg->getValue();
     if (s.getAsInteger(10, n))
-      error(arg->getSpelling() + " number expected, but got " + s);
+      Err(ctx) << arg->getSpelling() << " number expected, but got " << s;
     ctx.e.errorLimit = n;
   }
 
-  config->vfs = getVFS(args);
+  config->vfs = getVFS(ctx, args);
 
   // Handle /help
   if (args.hasArg(OPT_help)) {
@@ -1599,8 +1630,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     StringRef v(arg->getValue());
     unsigned threads = 0;
     if (!llvm::to_integer(v, threads, 0) || threads == 0)
-      error(arg->getSpelling() + ": expected a positive integer, but got '" +
-            arg->getValue() + "'");
+      Err(ctx) << arg->getSpelling()
+               << ": expected a positive integer, but got '" << arg->getValue()
+               << "'";
     parallel::strategy = hardware_concurrency(threads);
     config->thinLTOJobs = v.str();
   }
@@ -1615,7 +1647,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // because it doesn't start with "/", but we deliberately chose "--" to
   // avoid conflict with /version and for compatibility with clang-cl.
   if (args.hasArg(OPT_dash_dash_version)) {
-    message(getLLDVersion());
+    Msg(ctx) << getLLDVersion();
     return;
   }
 
@@ -1636,8 +1668,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       if (errOrWriter) {
         tar = std::move(*errOrWriter);
       } else {
-        error("/linkrepro: failed to open " + *path + ": " +
-              toString(errOrWriter.takeError()));
+        Err(ctx) << "/linkrepro: failed to open " << *path << ": "
+                 << toString(errOrWriter.takeError());
       }
     }
   }
@@ -1646,7 +1678,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     if (args.hasArg(OPT_deffile))
       config->noEntry = true;
     else
-      fatal("no input files");
+      Fatal(ctx) << "no input files";
   }
 
   // Construct search path list.
@@ -1668,7 +1700,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         addLibSearchPaths();
     } else {
       if (args.hasArg(OPT_vctoolsdir, OPT_winsysroot))
-        warn("ignoring /vctoolsdir or /winsysroot flags in MinGW mode");
+        Warn(ctx) << "ignoring /vctoolsdir or /winsysroot flags in MinGW mode";
     }
   }
 
@@ -1729,7 +1761,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     StringRef(str).split(vec, ',');
     for (StringRef s : vec) {
       if (s == "fastlink") {
-        warn("/debug:fastlink unsupported; using /debug:full");
+        Warn(ctx) << "/debug:fastlink unsupported; using /debug:full";
         s = "full";
       }
       if (s == "none") {
@@ -1763,7 +1795,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       } else if (s == "nosymtab") {
         config->writeSymtab = false;
       } else {
-        error("/debug: unknown option: " + s);
+        Err(ctx) << "/debug: unknown option: " << s;
       }
     }
   }
@@ -1772,7 +1804,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   config->demangle = args.hasFlag(OPT_demangle, OPT_demangle_no, true);
 
   // Handle /debugtype
-  config->debugTypes = parseDebugTypes(args);
+  config->debugTypes = parseDebugTypes(ctx, args);
 
   // Handle /driver[:uponly|:wdm].
   config->driverUponly = args.hasArg(OPT_driver_uponly) ||
@@ -1809,14 +1841,14 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   // Handle /pdbstripped
   if (args.hasArg(OPT_pdbstripped))
-    warn("ignoring /pdbstripped flag, it is not yet supported");
+    Warn(ctx) << "ignoring /pdbstripped flag, it is not yet supported";
 
   // Handle /noentry
   if (args.hasArg(OPT_noentry)) {
     if (args.hasArg(OPT_dll))
       config->noEntry = true;
     else
-      error("/noentry must be specified with /dll");
+      Err(ctx) << "/noentry must be specified with /dll";
   }
 
   // Handle /dll
@@ -1840,7 +1872,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   if (fixed) {
     if (dynamicBaseArg &&
         dynamicBaseArg->getOption().getID() == OPT_dynamicbase) {
-      error("/fixed must not be specified with /dynamicbase");
+      Err(ctx) << "/fixed must not be specified with /dynamicbase";
     } else {
       config->relocatable = false;
       config->dynamicBase = false;
@@ -1857,7 +1889,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     if (auto *arg = args.getLastArg(OPT_machine)) {
       config->machine = getMachineType(arg->getValue());
       if (config->machine == IMAGE_FILE_MACHINE_UNKNOWN)
-        fatal(Twine("unknown /machine argument: ") + arg->getValue());
+        Fatal(ctx) << "unknown /machine argument: " << arg->getValue();
       addWinSysRootLibSearchPaths();
     }
   }
@@ -1881,7 +1913,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   if (auto *arg = args.getLastArg(OPT_filealign)) {
     parseNumbers(arg->getValue(), &config->fileAlign);
     if (!isPowerOf2_64(config->fileAlign))
-      error("/filealign: not a power of two: " + Twine(config->fileAlign));
+      Err(ctx) << "/filealign: not a power of two: " << config->fileAlign;
   }
 
   // Handle /stack
@@ -1925,8 +1957,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       config->repro = false;
       StringRef value(arg->getValue());
       if (value.getAsInteger(0, config->timestamp))
-        fatal(Twine("invalid timestamp: ") + value +
-              ".  Expected 32-bit integer");
+        Fatal(ctx) << "invalid timestamp: " << value
+                   << ".  Expected 32-bit integer";
     }
   } else {
     config->repro = false;
@@ -1934,8 +1966,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
             Process::GetEnv("SOURCE_DATE_EPOCH")) {
       StringRef value(*epoch);
       if (value.getAsInteger(0, config->timestamp))
-        fatal(Twine("invalid SOURCE_DATE_EPOCH timestamp: ") + value +
-              ".  Expected 32-bit integer");
+        Fatal(ctx) << "invalid SOURCE_DATE_EPOCH timestamp: " << value
+                   << ".  Expected 32-bit integer";
     } else {
       config->timestamp = time(nullptr);
     }
@@ -1988,21 +2020,22 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         ltoDebugPM = false;
       } else if (s.consume_front("lldlto=")) {
         if (s.getAsInteger(10, config->ltoo) || config->ltoo > 3)
-          error("/opt:lldlto: invalid optimization level: " + s);
+          Err(ctx) << "/opt:lldlto: invalid optimization level: " << s;
       } else if (s.consume_front("lldltocgo=")) {
         config->ltoCgo.emplace();
         if (s.getAsInteger(10, *config->ltoCgo) || *config->ltoCgo > 3)
-          error("/opt:lldltocgo: invalid codegen optimization level: " + s);
+          Err(ctx) << "/opt:lldltocgo: invalid codegen optimization level: "
+                   << s;
       } else if (s.consume_front("lldltojobs=")) {
         if (!get_threadpool_strategy(s))
-          error("/opt:lldltojobs: invalid job count: " + s);
+          Err(ctx) << "/opt:lldltojobs: invalid job count: " << s;
         config->thinLTOJobs = s.str();
       } else if (s.consume_front("lldltopartitions=")) {
         if (s.getAsInteger(10, config->ltoPartitions) ||
             config->ltoPartitions == 0)
-          error("/opt:lldltopartitions: invalid partition count: " + s);
+          Err(ctx) << "/opt:lldltopartitions: invalid partition count: " << s;
       } else if (s != "lbr" && s != "nolbr")
-        error("/opt: unknown option: " + s);
+        Err(ctx) << "/opt: unknown option: " << s;
     }
   }
 
@@ -2024,7 +2057,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       if (llvm::is_contained(lldsaveTempsValues, s))
         config->saveTempsArgs.insert(s);
       else
-        error("unknown /lldsavetemps value: " + s);
+        Err(ctx) << "unknown /lldsavetemps value: " << s;
     }
   }
 
@@ -2038,7 +2071,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     else if (s == "asm")
       config->emit = EmitKind::ASM;
     else
-      error("/lldemit: unknown option: " + s);
+      Err(ctx) << "/lldemit: unknown option: " << s;
   }
 
   // Handle /kill-at
@@ -2089,9 +2122,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   if (auto *arg = args.getLastArg(OPT_align)) {
     parseNumbers(arg->getValue(), &config->align);
     if (!isPowerOf2_64(config->align))
-      error("/align: not a power of two: " + StringRef(arg->getValue()));
+      Err(ctx) << "/align: not a power of two: " << StringRef(arg->getValue());
     if (!args.hasArg(OPT_driver))
-      warn("/align specified without /driver; image may not run");
+      Warn(ctx) << "/align specified without /driver; image may not run";
   }
 
   // Handle /aligncomm
@@ -2124,7 +2157,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   if (!config->manifestInput.empty() &&
       config->manifest != Configuration::Embed) {
-    fatal("/manifestinput: requires /manifest:embed");
+    Fatal(ctx) << "/manifestinput: requires /manifest:embed";
   }
 
   // Handle /dwodir
@@ -2137,9 +2170,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       args.getLastArgValue(OPT_thinlto_index_only_arg);
   std::tie(config->thinLTOPrefixReplaceOld, config->thinLTOPrefixReplaceNew,
            config->thinLTOPrefixReplaceNativeObject) =
-      getOldNewOptionsExtra(args, OPT_thinlto_prefix_replace);
+      getOldNewOptionsExtra(ctx, args, OPT_thinlto_prefix_replace);
   config->thinLTOObjectSuffixReplace =
-      getOldNewOptions(args, OPT_thinlto_object_suffix_replace);
+      getOldNewOptions(ctx, args, OPT_thinlto_object_suffix_replace);
   config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path);
   config->ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate);
   config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
@@ -2176,31 +2209,33 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
                    OPT_lld_allow_duplicate_weak_no, config->mingw);
 
   if (args.hasFlag(OPT_inferasanlibs, OPT_inferasanlibs_no, false))
-    warn("ignoring '/inferasanlibs', this flag is not supported");
+    Warn(ctx) << "ignoring '/inferasanlibs', this flag is not supported";
 
   if (config->incremental && args.hasArg(OPT_profile)) {
-    warn("ignoring '/incremental' due to '/profile' specification");
+    Warn(ctx) << "ignoring '/incremental' due to '/profile' specification";
     config->incremental = false;
   }
 
   if (config->incremental && args.hasArg(OPT_order)) {
-    warn("ignoring '/incremental' due to '/order' specification");
+    Warn(ctx) << "ignoring '/incremental' due to '/order' specification";
     config->incremental = false;
   }
 
   if (config->incremental && config->doGC) {
-    warn("ignoring '/incremental' because REF is enabled; use '/opt:noref' to "
-         "disable");
+    Warn(ctx) << "ignoring '/incremental' because REF is enabled; use "
+                 "'/opt:noref' to "
+                 "disable";
     config->incremental = false;
   }
 
   if (config->incremental && config->doICF != ICFLevel::None) {
-    warn("ignoring '/incremental' because ICF is enabled; use '/opt:noicf' to "
-         "disable");
+    Warn(ctx) << "ignoring '/incremental' because ICF is enabled; use "
+                 "'/opt:noicf' to "
+                 "disable";
     config->incremental = false;
   }
 
-  if (errorCount())
+  if (errCount(ctx))
     return;
 
   std::set<sys::fs::UniqueID> wholeArchives;
@@ -2231,12 +2266,12 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       switch (arg->getOption().getID()) {
       case OPT_end_lib:
         if (!inLib)
-          error("stray " + arg->getSpelling());
+          Err(ctx) << "stray " << arg->getSpelling();
         inLib = false;
         break;
       case OPT_start_lib:
         if (inLib)
-          error("nested " + arg->getSpelling());
+          Err(ctx) << "nested " << arg->getSpelling();
         inLib = true;
         break;
       case OPT_wholearchive_file:
@@ -2262,7 +2297,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // We should have inferred a machine type by now from the input files, but if
   // not we assume x64.
   if (config->machine == IMAGE_FILE_MACHINE_UNKNOWN) {
-    warn("/machine is not specified. x64 is assumed");
+    Warn(ctx) << "/machine is not specified. x64 is assumed";
     config->machine = AMD64;
     addWinSysRootLibSearchPaths();
   }
@@ -2279,7 +2314,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       stream << "  " << path << "\n";
     }
 
-    message(buffer);
+    Msg(ctx) << buffer;
   }
 
   // Process files specified as /defaultlib. These must be processed after
@@ -2328,8 +2363,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   if (!config->dynamicBase &&
       (config->machine == ARMNT || isAnyArm64(config->machine)))
-    error("/dynamicbase:no is not compatible with " +
-          machineToStr(config->machine));
+    Err(ctx) << "/dynamicbase:no is not compatible with "
+             << machineToStr(config->machine);
 
   // Handle /export
   {
@@ -2367,7 +2402,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     llvm::TimeTraceScope timeScope("Infer subsystem");
     config->subsystem = inferSubsystem();
     if (config->subsystem == IMAGE_SUBSYSTEM_UNKNOWN)
-      fatal("subsystem must be defined");
+      Fatal(ctx) << "subsystem must be defined";
   }
 
   // Handle /entry and /dll
@@ -2375,7 +2410,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     llvm::TimeTraceScope timeScope("Entry point");
     if (auto *arg = args.getLastArg(OPT_entry)) {
       if (!arg->getValue()[0])
-        fatal("missing entry point symbol name");
+        Fatal(ctx) << "missing entry point symbol name";
       config->entry = addUndefined(mangle(arg->getValue()), true);
     } else if (!config->entry && !config->noEntry) {
       if (args.hasArg(OPT_dll)) {
@@ -2390,9 +2425,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         // infer that from user-defined entry name.
         StringRef s = findDefaultEntry();
         if (s.empty())
-          fatal("entry point must be defined");
+          Fatal(ctx) << "entry point must be defined";
         config->entry = addUndefined(s, true);
-        log("Entry name inferred: " + s);
+        Log(ctx) << "Entry name inferred: " << s;
       }
     }
   }
@@ -2419,7 +2454,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   // Fail early if an output file is not writable.
   if (auto e = tryCreateFile(config->outputFile)) {
-    error("cannot open output file " + config->outputFile + ": " + e.message());
+    Err(ctx) << "cannot open output file " << config->outputFile << ": "
+             << e.message();
     return;
   }
 
@@ -2432,13 +2468,13 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       if (s == "exports")
         config->mapInfo = true;
       else
-        error("unknown option: /mapinfo:" + s);
+        Err(ctx) << "unknown option: /mapinfo:" << s;
     }
   }
 
   if (config->lldmapFile != "" && config->lldmapFile == config->mapFile) {
-    warn("/lldmap and /map have the same output file '" + config->mapFile +
-         "'.\n>>> ignoring /lldmap");
+    Warn(ctx) << "/lldmap and /map have the same output file '"
+              << config->mapFile << "'.\n>>> ignoring /lldmap";
     config->lldmapFile.clear();
   }
 
@@ -2709,7 +2745,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   // Handle /output-def (MinGW specific).
   if (auto *arg = args.getLastArg(OPT_output_def))
-    writeDefFile(arg->getValue(), config->exports);
+    writeDefFile(ctx, arg->getValue(), config->exports);
 
   // Set extra alignment for .comm symbols
   for (auto pair : config->alignComm) {
@@ -2718,7 +2754,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
     Symbol *sym = ctx.symtab.find(name);
     if (!sym) {
-      warn("/aligncomm symbol " + name + " not found");
+      Warn(ctx) << "/aligncomm symbol " << name << " not found";
       continue;
     }
 
@@ -2747,7 +2783,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   // functions.
   if (auto *arg = args.getLastArg(OPT_order)) {
     if (args.hasArg(OPT_call_graph_ordering_file))
-      error("/order and /call-graph-order-file may not be used together");
+      Err(ctx) << "/order and /call-graph-order-file may not be used together";
     parseOrderFile(arg->getValue());
     config->callGraphProfileSort = false;
   }
diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index 8a72d74bd27ba..c1b3272a1f49e 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -83,9 +83,9 @@ class Executor {
 void LinkerDriver::parseNumbers(StringRef arg, uint64_t *addr, uint64_t *size) {
   auto [s1, s2] = arg.split(',');
   if (s1.getAsInteger(0, *addr))
-    fatal("invalid number: " + s1);
+    Fatal(ctx) << "invalid number: " << s1;
   if (size && !s2.empty() && s2.getAsInteger(0, *size))
-    fatal("invalid number: " + s2);
+    Fatal(ctx) << "invalid number: " << s2;
 }
 
 // Parses a string in the form of "<integer>[.<integer>]".
@@ -94,10 +94,10 @@ void LinkerDriver::parseVersion(StringRef arg, uint32_t *major,
                                 uint32_t *minor) {
   auto [s1, s2] = arg.split('.');
   if (s1.getAsInteger(10, *major))
-    fatal("invalid number: " + s1);
+    Fatal(ctx) << "invalid number: " << s1;
   *minor = 0;
   if (!s2.empty() && s2.getAsInteger(10, *minor))
-    fatal("invalid number: " + s2);
+    Fatal(ctx) << "invalid number: " << s2;
 }
 
 void LinkerDriver::parseGuard(StringRef fullArg) {
@@ -115,7 +115,7 @@ void LinkerDriver::parseGuard(StringRef fullArg) {
     else if (arg.equals_insensitive("ehcont"))
       ctx.config.guardCF |= GuardCFLevel::CF | GuardCFLevel::EHCont;
     else
-      fatal("invalid argument to /guard: " + arg);
+      Fatal(ctx) << "invalid argument to /guard: " << arg;
   }
 }
 
@@ -138,7 +138,7 @@ void LinkerDriver::parseSubsystem(StringRef arg, WindowsSubsystem *sys,
     .Case("windows", IMAGE_SUBSYSTEM_WINDOWS_GUI)
     .Default(IMAGE_SUBSYSTEM_UNKNOWN);
   if (*sys == IMAGE_SUBSYSTEM_UNKNOWN && sysStrLower != "default")
-    fatal("unknown subsystem: " + sysStr);
+    Fatal(ctx) << "unknown subsystem: " << sysStr;
   if (!ver.empty())
     parseVersion(ver, major, minor);
   if (gotVersion)
@@ -150,10 +150,10 @@ void LinkerDriver::parseSubsystem(StringRef arg, WindowsSubsystem *sys,
 void LinkerDriver::parseAlternateName(StringRef s) {
   auto [from, to] = s.split('=');
   if (from.empty() || to.empty())
-    fatal("/alternatename: invalid argument: " + s);
+    Fatal(ctx) << "/alternatename: invalid argument: " << s;
   auto it = ctx.config.alternateNames.find(from);
   if (it != ctx.config.alternateNames.end() && it->second != to)
-    fatal("/alternatename: conflicts: " + s);
+    Fatal(ctx) << "/alternatename: conflicts: " << s;
   ctx.config.alternateNames.insert(it, std::make_pair(from, to));
 }
 
@@ -162,35 +162,35 @@ void LinkerDriver::parseAlternateName(StringRef s) {
 void LinkerDriver::parseMerge(StringRef s) {
   auto [from, to] = s.split('=');
   if (from.empty() || to.empty())
-    fatal("/merge: invalid argument: " + s);
+    Fatal(ctx) << "/merge: invalid argument: " << s;
   if (from == ".rsrc" || to == ".rsrc")
-    fatal("/merge: cannot merge '.rsrc' with any section");
+    Fatal(ctx) << "/merge: cannot merge '.rsrc' with any section";
   if (from == ".reloc" || to == ".reloc")
-    fatal("/merge: cannot merge '.reloc' with any section");
+    Fatal(ctx) << "/merge: cannot merge '.reloc' with any section";
   auto pair = ctx.config.merge.insert(std::make_pair(from, to));
   bool inserted = pair.second;
   if (!inserted) {
     StringRef existing = pair.first->second;
     if (existing != to)
-      warn(s + ": already merged into " + existing);
+      Warn(ctx) << s << ": already merged into " << existing;
   }
 }
 
 void LinkerDriver::parsePDBPageSize(StringRef s) {
   int v;
   if (s.getAsInteger(0, v)) {
-    error("/pdbpagesize: invalid argument: " + s);
+    Err(ctx) << "/pdbpagesize: invalid argument: " << s;
     return;
   }
   if (v != 4096 && v != 8192 && v != 16384 && v != 32768) {
-    error("/pdbpagesize: invalid argument: " + s);
+    Err(ctx) << "/pdbpagesize: invalid argument: " << s;
     return;
   }
 
   ctx.config.pdbPageSize = v;
 }
 
-static uint32_t parseSectionAttributes(StringRef s) {
+static uint32_t parseSectionAttributes(COFFLinkerContext &ctx, StringRef s) {
   uint32_t ret = 0;
   for (char c : s.lower()) {
     switch (c) {
@@ -216,7 +216,7 @@ static uint32_t parseSectionAttributes(StringRef s) {
       ret |= IMAGE_SCN_MEM_WRITE;
       break;
     default:
-      fatal("/section: invalid argument: " + s);
+      Fatal(ctx) << "/section: invalid argument: " << s;
     }
   }
   return ret;
@@ -226,20 +226,20 @@ static uint32_t parseSectionAttributes(StringRef s) {
 void LinkerDriver::parseSection(StringRef s) {
   auto [name, attrs] = s.split(',');
   if (name.empty() || attrs.empty())
-    fatal("/section: invalid argument: " + s);
-  ctx.config.section[name] = parseSectionAttributes(attrs);
+    Fatal(ctx) << "/section: invalid argument: " << s;
+  ctx.config.section[name] = parseSectionAttributes(ctx, attrs);
 }
 
 // Parses /aligncomm option argument.
 void LinkerDriver::parseAligncomm(StringRef s) {
   auto [name, align] = s.split(',');
   if (name.empty() || align.empty()) {
-    error("/aligncomm: invalid argument: " + s);
+    Err(ctx) << "/aligncomm: invalid argument: " << s;
     return;
   }
   int v;
   if (align.getAsInteger(0, v)) {
-    error("/aligncomm: invalid argument: " + s);
+    Err(ctx) << "/aligncomm: invalid argument: " << s;
     return;
   }
   ctx.config.alignComm[std::string(name)] =
@@ -252,7 +252,7 @@ void LinkerDriver::parseFunctionPadMin(llvm::opt::Arg *a) {
   if (!arg.empty()) {
     // Optional padding in bytes is given.
     if (arg.getAsInteger(0, ctx.config.functionPadMin))
-      error("/functionpadmin: invalid argument: " + arg);
+      Err(ctx) << "/functionpadmin: invalid argument: " << arg;
     return;
   }
   // No optional argument given.
@@ -263,7 +263,7 @@ void LinkerDriver::parseFunctionPadMin(llvm::opt::Arg *a) {
   } else if (ctx.config.machine == AMD64) {
     ctx.config.functionPadMin = 6;
   } else {
-    error("/functionpadmin: invalid argument for this machine: " + arg);
+    Err(ctx) << "/functionpadmin: invalid argument for this machine: " << arg;
   }
 }
 
@@ -272,12 +272,12 @@ void LinkerDriver::parseDependentLoadFlags(llvm::opt::Arg *a) {
   StringRef arg = a->getNumValues() ? a->getValue() : "";
   if (!arg.empty()) {
     if (arg.getAsInteger(0, ctx.config.dependentLoadFlags))
-      error("/dependentloadflag: invalid argument: " + arg);
+      Err(ctx) << "/dependentloadflag: invalid argument: " << arg;
     return;
   }
   // MSVC linker reports error "no argument specified", although MSDN describes
   // argument as optional.
-  error("/dependentloadflag: no argument specified");
+  Err(ctx) << "/dependentloadflag: no argument specified";
 }
 
 // Parses a string in the form of "EMBED[,=<integer>]|NO".
@@ -289,16 +289,16 @@ void LinkerDriver::parseManifest(StringRef arg) {
     return;
   }
   if (!arg.starts_with_insensitive("embed"))
-    fatal("invalid option " + arg);
+    Fatal(ctx) << "invalid option " << arg;
   ctx.config.manifest = Configuration::Embed;
   arg = arg.substr(strlen("embed"));
   if (arg.empty())
     return;
   if (!arg.starts_with_insensitive(",id="))
-    fatal("invalid option " + arg);
+    Fatal(ctx) << "invalid option " << arg;
   arg = arg.substr(strlen(",id="));
   if (arg.getAsInteger(0, ctx.config.manifestID))
-    fatal("invalid option " + arg);
+    Fatal(ctx) << "invalid option " << arg;
 }
 
 // Parses a string in the form of "level=<string>|uiAccess=<string>|NO".
@@ -320,7 +320,7 @@ void LinkerDriver::parseManifestUAC(StringRef arg) {
       std::tie(ctx.config.manifestUIAccess, arg) = arg.split(" ");
       continue;
     }
-    fatal("invalid option " + arg);
+    Fatal(ctx) << "invalid option " << arg;
   }
 }
 
@@ -334,12 +334,12 @@ void LinkerDriver::parseSwaprun(StringRef arg) {
     else if (swaprun.equals_insensitive("net"))
       ctx.config.swaprunNet = true;
     else if (swaprun.empty())
-      error("/swaprun: missing argument");
+      Err(ctx) << "/swaprun: missing argument";
     else
-      error("/swaprun: invalid argument: " + swaprun);
+      Err(ctx) << "/swaprun: invalid argument: " << swaprun;
     // To catch trailing commas, e.g. `/spawrun:cd,`
     if (newArg.empty() && arg.ends_with(","))
-      error("/swaprun: missing argument");
+      Err(ctx) << "/swaprun: missing argument";
     arg = newArg;
   } while (!arg.empty());
 }
@@ -348,28 +348,32 @@ void LinkerDriver::parseSwaprun(StringRef arg) {
 namespace {
 class TemporaryFile {
 public:
-  TemporaryFile(StringRef prefix, StringRef extn, StringRef contents = "") {
+  TemporaryFile(COFFLinkerContext &ctx, StringRef prefix, StringRef extn,
+                StringRef contents = "")
+      : ctx(ctx) {
     SmallString<128> s;
     if (auto ec = sys::fs::createTemporaryFile("lld-" + prefix, extn, s))
-      fatal("cannot create a temporary file: " + ec.message());
+      Fatal(ctx) << "cannot create a temporary file: " << ec.message();
     path = std::string(s);
 
     if (!contents.empty()) {
       std::error_code ec;
       raw_fd_ostream os(path, ec, sys::fs::OF_None);
       if (ec)
-        fatal("failed to open " + path + ": " + ec.message());
+        Fatal(ctx) << "failed to open " << path << ": " << ec.message();
       os << contents;
     }
   }
 
-  TemporaryFile(TemporaryFile &&obj) noexcept { std::swap(path, obj.path); }
+  TemporaryFile(TemporaryFile &&obj) noexcept : ctx(obj.ctx) {
+    std::swap(path, obj.path);
+  }
 
   ~TemporaryFile() {
     if (path.empty())
       return;
     if (sys::fs::remove(path))
-      fatal("failed to remove " + path);
+      Fatal(ctx) << "failed to remove " << path;
   }
 
   // Returns a memory buffer of this temporary file.
@@ -384,6 +388,7 @@ class TemporaryFile {
                  "could not open " + path);
   }
 
+  COFFLinkerContext &ctx;
   std::string path;
 };
 }
@@ -425,16 +430,16 @@ LinkerDriver::createManifestXmlWithInternalMt(StringRef defaultXml) {
 
   windows_manifest::WindowsManifestMerger merger;
   if (auto e = merger.merge(*defaultXmlCopy.get()))
-    fatal("internal manifest tool failed on default xml: " +
-          toString(std::move(e)));
+    Fatal(ctx) << "internal manifest tool failed on default xml: "
+               << toString(std::move(e));
 
   for (StringRef filename : ctx.config.manifestInput) {
     std::unique_ptr<MemoryBuffer> manifest =
         check(MemoryBuffer::getFile(filename));
     // Call takeBuffer to include in /reproduce: output if applicable.
     if (auto e = merger.merge(takeBuffer(std::move(manifest))))
-      fatal("internal manifest tool failed on file " + filename + ": " +
-            toString(std::move(e)));
+      Fatal(ctx) << "internal manifest tool failed on file " << filename << ": "
+                 << toString(std::move(e));
   }
 
   return std::string(merger.getMergedManifest().get()->getBuffer());
@@ -443,17 +448,17 @@ LinkerDriver::createManifestXmlWithInternalMt(StringRef defaultXml) {
 std::string
 LinkerDriver::createManifestXmlWithExternalMt(StringRef defaultXml) {
   // Create the default manifest file as a temporary file.
-  TemporaryFile Default("defaultxml", "manifest");
+  TemporaryFile Default(ctx, "defaultxml", "manifest");
   std::error_code ec;
   raw_fd_ostream os(Default.path, ec, sys::fs::OF_TextWithCRLF);
   if (ec)
-    fatal("failed to open " + Default.path + ": " + ec.message());
+    Fatal(ctx) << "failed to open " << Default.path << ": " << ec.message();
   os << defaultXml;
   os.close();
 
   // Merge user-supplied manifests if they are given.  Since libxml2 is not
   // enabled, we must shell out to Microsoft's mt.exe tool.
-  TemporaryFile user("user", "manifest");
+  TemporaryFile user(ctx, "user", "manifest");
 
   Executor e("mt.exe");
   e.add("/manifest");
@@ -555,7 +560,7 @@ void LinkerDriver::createSideBySideManifest() {
   std::error_code ec;
   raw_fd_ostream out(path, ec, sys::fs::OF_TextWithCRLF);
   if (ec)
-    fatal("failed to create manifest: " + ec.message());
+    Fatal(ctx) << "failed to create manifest: " << ec.message();
   out << createManifestXml();
 }
 
@@ -614,7 +619,7 @@ Export LinkerDriver::parseExport(StringRef arg) {
       if (!rest.empty() && !rest.contains(','))
         e.exportAs = rest;
       else
-        error("invalid EXPORTAS value: " + rest);
+        Err(ctx) << "invalid EXPORTAS value: " << rest;
       break;
     }
     if (tok.starts_with("@")) {
@@ -631,7 +636,8 @@ Export LinkerDriver::parseExport(StringRef arg) {
   return e;
 
 err:
-  fatal("invalid /export: " + arg);
+  Fatal(ctx) << "invalid /export: " << arg;
+  llvm_unreachable("");
 }
 
 // Convert stdcall/fastcall style symbols into unsuffixed symbols,
@@ -677,7 +683,7 @@ void LinkerDriver::fixupExports() {
     if (e.ordinal == 0)
       continue;
     if (!ords.insert(e.ordinal).second)
-      fatal("duplicate export ordinal: " + e.name);
+      Fatal(ctx) << "duplicate export ordinal: " << e.name;
   }
 
   for (Export &e : ctx.config.exports) {
@@ -741,12 +747,12 @@ void LinkerDriver::fixupExports() {
       continue;
     }
     if (existing->source == e.source) {
-      warn(Twine("duplicate ") + exportSourceName(existing->source) +
-           " option: " + e.name);
+      Warn(ctx) << "duplicate " << exportSourceName(existing->source)
+                << " option: " << e.name;
     } else {
-      warn("duplicate export: " + e.name +
-           Twine(" first seen in " + exportSourceName(existing->source) +
-                 Twine(", now in " + exportSourceName(e.source))));
+      Warn(ctx) << "duplicate export: " << e.name << " first seen in "
+                << exportSourceName(existing->source) << ", now in "
+                << exportSourceName(e.source);
     }
   }
   ctx.config.exports = std::move(v);
@@ -766,8 +772,8 @@ void LinkerDriver::assignExportOrdinals() {
     if (e.ordinal == 0)
       e.ordinal = ++max;
   if (max > std::numeric_limits<uint16_t>::max())
-    fatal("too many exported symbols (got " + Twine(max) + ", max " +
-          Twine(std::numeric_limits<uint16_t>::max()) + ")");
+    Fatal(ctx) << "too many exported symbols (got " << max << ", max "
+               << Twine(std::numeric_limits<uint16_t>::max()) << ")";
 }
 
 // Parses a string in the form of "key=value" and check
@@ -775,15 +781,15 @@ void LinkerDriver::assignExportOrdinals() {
 void LinkerDriver::checkFailIfMismatch(StringRef arg, InputFile *source) {
   auto [k, v] = arg.split('=');
   if (k.empty() || v.empty())
-    fatal("/failifmismatch: invalid argument: " + arg);
+    Fatal(ctx) << "/failifmismatch: invalid argument: " << arg;
   std::pair<StringRef, InputFile *> existing = ctx.config.mustMatch[k];
   if (!existing.first.empty() && v != existing.first) {
     std::string sourceStr = source ? toString(source) : "cmd-line";
     std::string existingStr =
         existing.second ? toString(existing.second) : "cmd-line";
-    fatal("/failifmismatch: mismatch detected for '" + k + "':\n>>> " +
-          existingStr + " has value " + existing.first + "\n>>> " + sourceStr +
-          " has value " + v);
+    Fatal(ctx) << "/failifmismatch: mismatch detected for '" << k << "':\n>>> "
+               << existingStr << " has value " << existing.first << "\n>>> "
+               << sourceStr << " has value " << v;
   }
   ctx.config.mustMatch[k] = {v, source};
 }
@@ -799,10 +805,10 @@ MemoryBufferRef LinkerDriver::convertResToCOFF(ArrayRef<MemoryBufferRef> mbs,
     std::unique_ptr<object::Binary> bin = check(object::createBinary(mb));
     object::WindowsResource *rf = dyn_cast<object::WindowsResource>(bin.get());
     if (!rf)
-      fatal("cannot compile non-resource file as resource");
+      Fatal(ctx) << "cannot compile non-resource file as resource";
 
     if (auto ec = parser.parse(rf, duplicates))
-      fatal(toString(std::move(ec)));
+      Fatal(ctx) << toString(std::move(ec));
   }
 
   // Note: This processes all .res files before all objs. Ideally they'd be
@@ -811,10 +817,10 @@ MemoryBufferRef LinkerDriver::convertResToCOFF(ArrayRef<MemoryBufferRef> mbs,
   for (ObjFile *f : objs) {
     object::ResourceSectionRef rsf;
     if (auto ec = rsf.load(f->getCOFFObj()))
-      fatal(toString(f) + ": " + toString(std::move(ec)));
+      Fatal(ctx) << toString(f) << ": " << toString(std::move(ec));
 
     if (auto ec = parser.parse(rsf, f->getName(), duplicates))
-      fatal(toString(std::move(ec)));
+      Fatal(ctx) << toString(std::move(ec));
   }
 
   if (ctx.config.mingw)
@@ -822,15 +828,15 @@ MemoryBufferRef LinkerDriver::convertResToCOFF(ArrayRef<MemoryBufferRef> mbs,
 
   for (const auto &dupeDiag : duplicates)
     if (ctx.config.forceMultipleRes)
-      warn(dupeDiag);
+      Warn(ctx) << dupeDiag;
     else
-      error(dupeDiag);
+      Err(ctx) << dupeDiag;
 
   Expected<std::unique_ptr<MemoryBuffer>> e =
       llvm::object::writeWindowsResourceCOFF(ctx.config.machine, parser,
                                              ctx.config.timestamp);
   if (!e)
-    fatal("failed to write .res to COFF: " + toString(e.takeError()));
+    Fatal(ctx) << "failed to write .res to COFF: " << toString(e.takeError());
 
   MemoryBufferRef mbref = **e;
   make<std::unique_ptr<MemoryBuffer>>(std::move(*e)); // take ownership
@@ -875,15 +881,16 @@ static void handleColorDiagnostics(COFFLinkerContext &ctx,
     else if (s == "never")
       ctx.e.errs().enable_colors(false);
     else if (s != "auto")
-      error("unknown option: --color-diagnostics=" + s);
+      Err(ctx) << "unknown option: --color-diagnostics=" << s;
   }
 }
 
-static cl::TokenizerCallback getQuotingStyle(opt::InputArgList &args) {
+static cl::TokenizerCallback getQuotingStyle(COFFLinkerContext &ctx,
+                                             opt::InputArgList &args) {
   if (auto *arg = args.getLastArg(OPT_rsp_quoting)) {
     StringRef s = arg->getValue();
     if (s != "windows" && s != "posix")
-      error("invalid response file quoting: " + s);
+      Err(ctx) << "invalid response file quoting: " << s;
     if (s == "windows")
       return cl::TokenizeWindowsCommandLine;
     return cl::TokenizeGNUCommandLine;
@@ -913,7 +920,7 @@ opt::InputArgList ArgParser::parse(ArrayRef<const char *> argv) {
                                               argv.data() + argv.size());
   if (!args.hasArg(OPT_lldignoreenv))
     addLINK(expandedArgv);
-  cl::ExpandResponseFiles(saver(), getQuotingStyle(args), expandedArgv);
+  cl::ExpandResponseFiles(saver(), getQuotingStyle(ctx, args), expandedArgv);
   args = ctx.optTable.ParseArgs(ArrayRef(expandedArgv).drop_front(),
                                 missingIndex, missingCount);
 
@@ -922,7 +929,7 @@ opt::InputArgList ArgParser::parse(ArrayRef<const char *> argv) {
     std::string msg = "Command line:";
     for (const char *s : expandedArgv)
       msg += " " + std::string(s);
-    message(msg);
+    Msg(ctx) << msg;
   }
 
   // Save the command line after response file expansion so we can write it to
@@ -938,21 +945,22 @@ opt::InputArgList ArgParser::parse(ArrayRef<const char *> argv) {
   ctx.e.fatalWarnings = args.hasFlag(OPT_WX, OPT_WX_no, false);
 
   if (missingCount)
-    fatal(Twine(args.getArgString(missingIndex)) + ": missing argument");
+    Fatal(ctx) << args.getArgString(missingIndex) << ": missing argument";
 
   handleColorDiagnostics(ctx, args);
 
   for (opt::Arg *arg : args.filtered(OPT_UNKNOWN)) {
     std::string nearest;
     if (ctx.optTable.findNearest(arg->getAsString(args), nearest) > 1)
-      warn("ignoring unknown argument '" + arg->getAsString(args) + "'");
+      Warn(ctx) << "ignoring unknown argument '" << arg->getAsString(args)
+                << "'";
     else
-      warn("ignoring unknown argument '" + arg->getAsString(args) +
-           "', did you mean '" + nearest + "'");
+      Warn(ctx) << "ignoring unknown argument '" << arg->getAsString(args)
+                << "', did you mean '" << nearest << "'";
   }
 
   if (args.hasArg(OPT_lib))
-    warn("ignoring /lib since it's not the first argument");
+    Warn(ctx) << "ignoring /lib since it's not the first argument";
 
   return args;
 }
@@ -992,9 +1000,10 @@ ParsedDirectives ArgParser::parseDirectives(StringRef s) {
   result.args = ctx.optTable.ParseArgs(rest, missingIndex, missingCount);
 
   if (missingCount)
-    fatal(Twine(result.args.getArgString(missingIndex)) + ": missing argument");
+    Fatal(ctx) << result.args.getArgString(missingIndex)
+               << ": missing argument";
   for (auto *arg : result.args.filtered(OPT_UNKNOWN))
-    warn("ignoring unknown argument: " + arg->getAsString(result.args));
+    Warn(ctx) << "ignoring unknown argument: " << arg->getAsString(result.args);
   return result;
 }
 
diff --git a/lld/COFF/ICF.cpp b/lld/COFF/ICF.cpp
index 3140d094ae0f0..796d3a4108ba4 100644
--- a/lld/COFF/ICF.cpp
+++ b/lld/COFF/ICF.cpp
@@ -314,16 +314,16 @@ void ICF::run() {
         [&](size_t begin, size_t end) { segregate(begin, end, false); });
   } while (repeat);
 
-  log("ICF needed " + Twine(cnt) + " iterations");
+  Log(ctx) << "ICF needed " << Twine(cnt) << " iterations";
 
   // Merge sections in the same classes.
   forEachClass([&](size_t begin, size_t end) {
     if (end - begin == 1)
       return;
 
-    log("Selected " + chunks[begin]->getDebugName());
+    Log(ctx) << "Selected " << chunks[begin]->getDebugName();
     for (size_t i = begin + 1; i < end; ++i) {
-      log("  Removed " + chunks[i]->getDebugName());
+      Log(ctx) << "  Removed " << chunks[i]->getDebugName();
       chunks[begin]->replace(chunks[i]);
     }
   });
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 6b5efb34b3f3e..f32bc5bbbc35f 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -70,6 +70,11 @@ std::string lld::toString(const coff::InputFile *file) {
       .str();
 }
 
+const COFFSyncStream &coff::operator<<(const COFFSyncStream &s,
+                                       const InputFile *f) {
+  return s << toString(f);
+}
+
 /// Checks that Source is compatible with being a weak alias to Target.
 /// If Source is Undefined and has no weak alias set, makes it a weak
 /// alias to Target.
@@ -139,7 +144,8 @@ void ArchiveFile::addMember(const Archive::Symbol &sym) {
   ctx.driver.enqueueArchiveMember(c, sym, getName());
 }
 
-std::vector<MemoryBufferRef> lld::coff::getArchiveMembers(Archive *file) {
+std::vector<MemoryBufferRef>
+lld::coff::getArchiveMembers(COFFLinkerContext &ctx, Archive *file) {
   std::vector<MemoryBufferRef> v;
   Error err = Error::success();
   for (const Archive::Child &c : file->children(err)) {
@@ -150,8 +156,8 @@ std::vector<MemoryBufferRef> lld::coff::getArchiveMembers(Archive *file) {
     v.push_back(mbref);
   }
   if (err)
-    fatal(file->getFileName() +
-          ": Archive::children failed: " + toString(std::move(err)));
+    Fatal(ctx) << file->getFileName()
+               << ": Archive::children failed: " << toString(std::move(err));
   return v;
 }
 
@@ -182,7 +188,7 @@ struct ECMapEntry {
 void ObjFile::initializeECThunks() {
   for (SectionChunk *chunk : hybmpChunks) {
     if (chunk->getContents().size() % sizeof(ECMapEntry)) {
-      error("Invalid .hybmp chunk size " + Twine(chunk->getContents().size()));
+      Err(ctx) << "Invalid .hybmp chunk size " << chunk->getContents().size();
       continue;
     }
 
@@ -201,7 +207,7 @@ void ObjFile::initializeECThunks() {
       case Arm64ECThunkType::GuestExit:
         break;
       default:
-        warn("Ignoring unknown EC thunk type " + Twine(entry->type));
+        Warn(ctx) << "Ignoring unknown EC thunk type " << entry->type;
       }
     }
   }
@@ -215,7 +221,7 @@ void ObjFile::parse() {
     bin.release();
     coffObj.reset(obj);
   } else {
-    fatal(toString(this) + " is not a COFF file");
+    Fatal(ctx) << toString(this) << " is not a COFF file";
   }
 
   // Read section and symbol tables.
@@ -229,7 +235,7 @@ void ObjFile::parse() {
 const coff_section *ObjFile::getSection(uint32_t i) {
   auto sec = coffObj->getSection(i);
   if (!sec)
-    fatal("getSection failed: #" + Twine(i) + ": " + toString(sec.takeError()));
+    Fatal(ctx) << "getSection failed: #" << i << ": " << sec.takeError();
   return *sec;
 }
 
@@ -262,8 +268,8 @@ SectionChunk *ObjFile::readSection(uint32_t sectionNumber,
   if (Expected<StringRef> e = coffObj->getSectionName(sec))
     name = *e;
   else
-    fatal("getSectionName failed: #" + Twine(sectionNumber) + ": " +
-          toString(e.takeError()));
+    Fatal(ctx) << "getSectionName failed: #" << sectionNumber << ": "
+               << e.takeError();
 
   if (name == ".drectve") {
     ArrayRef<uint8_t> data;
@@ -358,9 +364,9 @@ void ObjFile::readAssociativeDefinition(COFFSymbolRef sym,
     const coff_section *parentSec = getSection(parentIndex);
     if (Expected<StringRef> e = coffObj->getSectionName(parentSec))
       parentName = *e;
-    error(toString(this) + ": associative comdat " + name + " (sec " +
-          Twine(sectionNumber) + ") has invalid reference to section " +
-          parentName + " (sec " + Twine(parentIndex) + ")");
+    Err(ctx) << toString(this) << ": associative comdat " << name << " (sec "
+             << sectionNumber << ") has invalid reference to section "
+             << parentName << " (sec " << parentIndex << ")";
   };
 
   if (parent == pendingComdat) {
@@ -513,8 +519,8 @@ void ObjFile::initializeSymbols() {
     }
     if (sparseChunks[sym.getSectionNumber()] == pendingComdat) {
       StringRef name = check(coffObj->getSymbolName(sym));
-      log("comdat section " + name +
-          " without leader and unassociated, discarding");
+      Log(ctx) << "comdat section " << name
+               << " without leader and unassociated, discarding";
       continue;
     }
     symbols[i] = createRegular(sym);
@@ -615,10 +621,9 @@ void ObjFile::handleComdatSelection(
   // seems better though.
   // (This behavior matches ModuleLinker::getComdatResult().)
   if (selection != leaderSelection) {
-    log(("conflicting comdat type for " + toString(ctx, *leader) + ": " +
-         Twine((int)leaderSelection) + " in " + toString(leader->getFile()) +
-         " and " + Twine((int)selection) + " in " + toString(this))
-            .str());
+    Log(ctx) << "conflicting comdat type for " << leader << ": "
+             << (int)leaderSelection << " in " << leader->getFile() << " and "
+             << (int)selection << " in " << this;
     ctx.symtab.reportDuplicate(leader, this);
     return;
   }
@@ -719,12 +724,14 @@ std::optional<Symbol *> ObjFile::createDefined(
     return nullptr;
 
   if (llvm::COFF::isReservedSectionNumber(sectionNumber))
-    fatal(toString(this) + ": " + getName() +
-          " should not refer to special section " + Twine(sectionNumber));
+    Fatal(ctx) << toString(this) << ": " << getName()
+               << " should not refer to special section "
+               << Twine(sectionNumber);
 
   if ((uint32_t)sectionNumber >= sparseChunks.size())
-    fatal(toString(this) + ": " + getName() +
-          " should not refer to non-existent section " + Twine(sectionNumber));
+    Fatal(ctx) << toString(this) << ": " << getName()
+               << " should not refer to non-existent section "
+               << Twine(sectionNumber);
 
   // Comdat handling.
   // A comdat symbol consists of two symbol table entries.
@@ -754,8 +761,9 @@ std::optional<Symbol *> ObjFile::createDefined(
         // Intentionally ends at IMAGE_COMDAT_SELECT_LARGEST: link.exe
         // doesn't understand IMAGE_COMDAT_SELECT_NEWEST either.
         def->Selection > (int)IMAGE_COMDAT_SELECT_LARGEST) {
-      fatal("unknown comdat type " + std::to_string((int)def->Selection) +
-            " for " + getName() + " in " + toString(this));
+      Fatal(ctx) << "unknown comdat type "
+                 << std::to_string((int)def->Selection) << " for " << getName()
+                 << " in " << toString(this);
     }
     COMDATType selection = (COMDATType)def->Selection;
 
@@ -1080,7 +1088,7 @@ void ImportFile::parse() {
   // Check if the total size is valid.
   if (mb.getBufferSize() < sizeof(*hdr) ||
       mb.getBufferSize() != sizeof(*hdr) + hdr->SizeOfData)
-    fatal("broken import library");
+    Fatal(ctx) << "broken import library";
 
   // Read names and create an __imp_ symbol.
   StringRef buf = mb.getBuffer().substr(sizeof(*hdr));
@@ -1321,12 +1329,12 @@ void DLLFile::parse() {
     bin.release();
     coffObj.reset(obj);
   } else {
-    error(toString(this) + " is not a COFF file");
+    Err(ctx) << toString(this) << " is not a COFF file";
     return;
   }
 
   if (!coffObj->getPE32Header() && !coffObj->getPE32PlusHeader()) {
-    error(toString(this) + " is not a PE-COFF executable");
+    Err(ctx) << toString(this) << " is not a PE-COFF executable";
     return;
   }
 
diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h
index 77f7e298166ee..3c48e778ac5b5 100644
--- a/lld/COFF/InputFiles.h
+++ b/lld/COFF/InputFiles.h
@@ -40,7 +40,10 @@ class DWARFCache;
 namespace coff {
 class COFFLinkerContext;
 
-std::vector<MemoryBufferRef> getArchiveMembers(llvm::object::Archive *file);
+const COFFSyncStream &operator<<(const COFFSyncStream &, const InputFile *);
+
+std::vector<MemoryBufferRef> getArchiveMembers(COFFLinkerContext &,
+                                               llvm::object::Archive *file);
 
 using llvm::COFF::IMAGE_FILE_MACHINE_UNKNOWN;
 using llvm::COFF::MachineTypes;
diff --git a/lld/COFF/MapFile.cpp b/lld/COFF/MapFile.cpp
index 55a1cd942ab85..e3531c04e7747 100644
--- a/lld/COFF/MapFile.cpp
+++ b/lld/COFF/MapFile.cpp
@@ -209,7 +209,7 @@ void lld::coff::writeMapFile(COFFLinkerContext &ctx) {
   std::error_code ec;
   raw_fd_ostream os(ctx.config.mapFile, ec, sys::fs::OF_None);
   if (ec)
-    fatal("cannot open " + ctx.config.mapFile + ": " + ec.message());
+    Fatal(ctx) << "cannot open " << ctx.config.mapFile << ": " << ec.message();
 
   ScopedTimer t1(ctx.totalMapTimer);
 
diff --git a/lld/COFF/MinGW.cpp b/lld/COFF/MinGW.cpp
index 29c01da9e28f6..76f5a0a7500b9 100644
--- a/lld/COFF/MinGW.cpp
+++ b/lld/COFF/MinGW.cpp
@@ -170,13 +170,13 @@ bool AutoExporter::shouldExport(Defined *sym) const {
   return !excludeObjects.count(fileName);
 }
 
-void lld::coff::writeDefFile(StringRef name,
+void lld::coff::writeDefFile(COFFLinkerContext &ctx, StringRef name,
                              const std::vector<Export> &exports) {
   llvm::TimeTraceScope timeScope("Write .def file");
   std::error_code ec;
   raw_fd_ostream os(name, ec, sys::fs::OF_None);
   if (ec)
-    fatal("cannot open " + name + ": " + ec.message());
+    Fatal(ctx) << "cannot open " << name << ": " << ec.message();
 
   os << "EXPORTS\n";
   for (const Export &e : exports) {
diff --git a/lld/COFF/MinGW.h b/lld/COFF/MinGW.h
index aa5e53278ca4b..ffa500b234777 100644
--- a/lld/COFF/MinGW.h
+++ b/lld/COFF/MinGW.h
@@ -45,7 +45,8 @@ class AutoExporter {
   COFFLinkerContext &ctx;
 };
 
-void writeDefFile(StringRef name, const std::vector<Export> &exports);
+void writeDefFile(COFFLinkerContext &, StringRef name,
+                  const std::vector<Export> &exports);
 
 // The -wrap option is a feature to rename symbols so that you can write
 // wrappers for existing functions. If you pass `-wrap:foo`, all
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index c20b54a5d42e4..4382dd677ff59 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -569,7 +569,8 @@ void PDBLinker::writeSymbolRecord(SectionChunk *debugChunk,
   // Re-map all the type index references.
   TpiSource *source = debugChunk->file->debugTypesObj;
   if (!source->remapTypesInSymbolRecord(recordBytes)) {
-    log("ignoring unknown symbol record with kind 0x" + utohexstr(sym.kind()));
+    Log(ctx) << "ignoring unknown symbol record with kind 0x"
+             << utohexstr(sym.kind());
     replaceWithSkipRecord(recordBytes);
   }
 
@@ -1006,11 +1007,10 @@ static void warnUnusable(InputFile *f, Error e, bool shouldWarn) {
     consumeError(std::move(e));
     return;
   }
-  auto msg = "Cannot use debug info for '" + toString(f) + "' [LNK4099]";
+  auto diag = Warn(f->ctx);
+  diag << "Cannot use debug info for '" << f << "' [LNK4099]";
   if (e)
-    warn(msg + "\n>>> failed to load reference " + toString(std::move(e)));
-  else
-    warn(msg);
+    diag << "\n>>> failed to load reference " << std::move(e);
 }
 
 // Allocate memory for a .debug$S / .debug$F section and relocate it.
@@ -1317,7 +1317,7 @@ void PDBLinker::printStats() {
     printLargeInputTypeRecs("IPI", tMerger.ipiCounts, tMerger.getIDTable());
   }
 
-  message(buffer);
+  Msg(ctx) << buffer;
 }
 
 void PDBLinker::addNatvisFiles() {
@@ -1725,14 +1725,13 @@ void PDBLinker::commit(codeview::GUID *guid) {
   // the user can see the output of /time and /summary, which is very helpful
   // when trying to figure out why a PDB file is too large.
   if (Error e = builder.commit(ctx.config.pdbPath, guid)) {
-    e = handleErrors(std::move(e),
-        [](const llvm::msf::MSFError &me) {
-          error(me.message());
-          if (me.isPageOverflow())
-            error("try setting a larger /pdbpagesize");
-        });
+    e = handleErrors(std::move(e), [&](const llvm::msf::MSFError &me) {
+      Err(ctx) << me.message();
+      if (me.isPageOverflow())
+        Err(ctx) << "try setting a larger /pdbpagesize";
+    });
     checkError(std::move(e));
-    error("failed to write PDB file " + Twine(ctx.config.pdbPath));
+    Err(ctx) << "failed to write PDB file " << Twine(ctx.config.pdbPath);
   }
 }
 
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index 0c6df701284b7..26ace0e75c493 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -54,7 +54,7 @@ static bool compatibleMachineType(COFFLinkerContext &ctx, MachineTypes mt) {
 }
 
 void SymbolTable::addFile(InputFile *file) {
-  log("Reading " + toString(file));
+  Log(ctx) << "Reading " << toString(file);
   if (file->lazy) {
     if (auto *f = dyn_cast<BitcodeFile>(file))
       f->parseLazy();
@@ -66,8 +66,9 @@ void SymbolTable::addFile(InputFile *file) {
       ctx.objFileInstances.push_back(f);
     } else if (auto *f = dyn_cast<BitcodeFile>(file)) {
       if (ltoCompilationDone) {
-        error("LTO object file " + toString(file) + " linked in after "
-              "doing LTO compilation.");
+        Err(ctx) << "LTO object file " << toString(file)
+                 << " linked in after "
+                    "doing LTO compilation.";
       }
       ctx.bitcodeFileInstances.push_back(f);
     } else if (auto *f = dyn_cast<ImportFile>(file)) {
@@ -81,13 +82,14 @@ void SymbolTable::addFile(InputFile *file) {
       (ctx.config.machine == IMAGE_FILE_MACHINE_UNKNOWN ||
        (ctx.config.machineInferred &&
         (ctx.config.machine == ARM64 || ctx.config.machine == AMD64)))) {
-    error(toString(file) + ": machine type arm64ec is ambiguous and cannot be "
-                           "inferred, use /machine:arm64ec or /machine:arm64x");
+    Err(ctx) << toString(file)
+             << ": machine type arm64ec is ambiguous and cannot be "
+                "inferred, use /machine:arm64ec or /machine:arm64x";
     return;
   }
   if (!compatibleMachineType(ctx, mt)) {
-    error(toString(file) + ": machine type " + machineToStr(mt) +
-          " conflicts with " + machineToStr(ctx.config.machine));
+    Err(ctx) << toString(file) << ": machine type " << machineToStr(mt)
+             << " conflicts with " << machineToStr(ctx.config.machine);
     return;
   }
   if (ctx.config.machine == IMAGE_FILE_MACHINE_UNKNOWN &&
@@ -100,11 +102,8 @@ void SymbolTable::addFile(InputFile *file) {
   ctx.driver.parseDirectives(file);
 }
 
-static void errorOrWarn(const Twine &s, bool forceUnresolved) {
-  if (forceUnresolved)
-    warn(s);
-  else
-    error(s);
+static COFFSyncStream errorOrWarn(COFFLinkerContext &ctx) {
+  return {ctx, ctx.config.forceUnresolved ? DiagLevel::Warn : DiagLevel::Err};
 }
 
 // Causes the file associated with a lazy symbol to be linked in.
@@ -271,11 +270,10 @@ struct UndefinedDiag {
   std::vector<File> files;
 };
 
-static void reportUndefinedSymbol(const COFFLinkerContext &ctx,
+static void reportUndefinedSymbol(COFFLinkerContext &ctx,
                                   const UndefinedDiag &undefDiag) {
-  std::string out;
-  llvm::raw_string_ostream os(out);
-  os << "undefined symbol: " << toString(ctx, *undefDiag.sym);
+  auto diag = errorOrWarn(ctx);
+  diag << "undefined symbol: " << undefDiag.sym;
 
   const size_t maxUndefReferences = 3;
   size_t numDisplayedRefs = 0, numRefs = 0;
@@ -285,13 +283,11 @@ static void reportUndefinedSymbol(const COFFLinkerContext &ctx,
 
     numRefs += totalLocations;
     numDisplayedRefs += symbolLocations.size();
-    for (const std::string &s : symbolLocations) {
-      os << s;
-    }
+    for (const std::string &s : symbolLocations)
+      diag << s;
   }
   if (numDisplayedRefs < numRefs)
-    os << "\n>>> referenced " << numRefs - numDisplayedRefs << " more times";
-  errorOrWarn(out, ctx.config.forceUnresolved);
+    diag << "\n>>> referenced " << numRefs - numDisplayedRefs << " more times";
 }
 
 void SymbolTable::loadMinGWSymbols() {
@@ -320,16 +316,18 @@ void SymbolTable::loadMinGWSymbols() {
       if (newName != origName && (l = find(newName)) != nullptr) {
         // If we found a symbol and it is lazy; load it.
         if (l->isLazy() && !l->pendingArchiveLoad) {
-          log("Loading lazy " + l->getName() + " from " +
-              l->getFile()->getName() + " for stdcall fixup");
+          Log(ctx) << "Loading lazy " << l->getName() << " from "
+                   << l->getFile()->getName() << " for stdcall fixup";
           forceLazy(l);
         }
         // If it's lazy or already defined, hook it up as weak alias.
         if (l->isLazy() || isa<Defined>(l)) {
           if (ctx.config.warnStdcallFixup)
-            warn("Resolving " + origName + " by linking to " + newName);
+            Warn(ctx) << "Resolving " << origName << " by linking to "
+                      << newName;
           else
-            log("Resolving " + origName + " by linking to " + newName);
+            Log(ctx) << "Resolving " << origName << " by linking to "
+                     << newName;
           undef->setWeakAlias(l);
           continue;
         }
@@ -345,8 +343,8 @@ void SymbolTable::loadMinGWSymbols() {
       if (!l || l->pendingArchiveLoad || !l->isLazy())
         continue;
 
-      log("Loading lazy " + l->getName() + " from " + l->getFile()->getName() +
-          " for automatic import");
+      Log(ctx) << "Loading lazy " << l->getName() << " from "
+               << l->getFile()->getName() << " for automatic import";
       forceLazy(l);
     }
   }
@@ -371,17 +369,17 @@ bool SymbolTable::handleMinGWAutomaticImport(Symbol *sym, StringRef name) {
   // reference itself to point at the IAT entry.
   size_t impSize = 0;
   if (isa<DefinedImportData>(imp)) {
-    log("Automatically importing " + name + " from " +
-        cast<DefinedImportData>(imp)->getDLLName());
+    Log(ctx) << "Automatically importing " << name << " from "
+             << cast<DefinedImportData>(imp)->getDLLName();
     impSize = sizeof(DefinedImportData);
   } else if (isa<DefinedRegular>(imp)) {
-    log("Automatically importing " + name + " from " +
-        toString(cast<DefinedRegular>(imp)->file));
+    Log(ctx) << "Automatically importing " << name << " from "
+             << toString(cast<DefinedRegular>(imp)->file);
     impSize = sizeof(DefinedRegular);
   } else {
-    warn("unable to automatically import " + name + " from " + imp->getName() +
-         " from " + toString(cast<DefinedRegular>(imp)->file) +
-         "; unexpected symbol type");
+    Warn(ctx) << "unable to automatically import " << name << " from "
+              << imp->getName() << " from " << cast<DefinedRegular>(imp)->file
+              << "; unexpected symbol type";
     return false;
   }
   sym->replaceKeepingName(imp, impSize);
@@ -397,7 +395,7 @@ bool SymbolTable::handleMinGWAutomaticImport(Symbol *sym, StringRef name) {
   if (refptr && refptr->getChunk()->getSize() == ctx.config.wordsize) {
     SectionChunk *sc = dyn_cast_or_null<SectionChunk>(refptr->getChunk());
     if (sc && sc->getRelocs().size() == 1 && *sc->symbols().begin() == sym) {
-      log("Replacing .refptr." + name + " with " + imp->getName());
+      Log(ctx) << "Replacing .refptr." << name << " with " << imp->getName();
       refptr->getChunk()->live = false;
       refptr->replaceKeepingName(imp, impSize);
     }
@@ -412,7 +410,7 @@ bool SymbolTable::handleMinGWAutomaticImport(Symbol *sym, StringRef name) {
 /// objFiles and bitcodeFiles (if not nullptr) are used to report where
 /// undefined symbols are referenced.
 static void reportProblemSymbols(
-    const COFFLinkerContext &ctx, const SmallPtrSetImpl<Symbol *> &undefs,
+    COFFLinkerContext &ctx, const SmallPtrSetImpl<Symbol *> &undefs,
     const DenseMap<Symbol *, Symbol *> *localImports, bool needBitcodeFiles) {
   // Return early if there is nothing to report (which should be
   // the common case).
@@ -421,12 +419,12 @@ static void reportProblemSymbols(
 
   for (Symbol *b : ctx.config.gcroot) {
     if (undefs.count(b))
-      errorOrWarn("<root>: undefined symbol: " + toString(ctx, *b),
-                  ctx.config.forceUnresolved);
+      errorOrWarn(ctx) << "<root>: undefined symbol: " << b;
     if (localImports)
       if (Symbol *imp = localImports->lookup(b))
-        warn("<root>: locally defined symbol imported: " + toString(ctx, *imp) +
-             " (defined in " + toString(imp->getFile()) + ") [LNK4217]");
+        Warn(ctx) << "<root>: locally defined symbol imported: " << imp
+                  << " (defined in " << toString(imp->getFile())
+                  << ") [LNK4217]";
   }
 
   std::vector<UndefinedDiag> undefDiags;
@@ -447,9 +445,8 @@ static void reportProblemSymbols(
       }
       if (localImports)
         if (Symbol *imp = localImports->lookup(sym))
-          warn(toString(file) +
-               ": locally defined symbol imported: " + toString(ctx, *imp) +
-               " (defined in " + toString(imp->getFile()) + ") [LNK4217]");
+          Warn(ctx) << file << ": locally defined symbol imported: " << imp
+                    << " (defined in " << imp->getFile() << ") [LNK4217]";
     }
   };
 
@@ -621,7 +618,7 @@ void SymbolTable::initializeECThunks() {
     // feasible, functions are required to be COMDAT symbols with no offset.
     if (!from || !from->getChunk()->isCOMDAT() ||
         cast<DefinedRegular>(from)->getValue()) {
-      error("non COMDAT symbol '" + from->getName() + "' in hybrid map");
+      Err(ctx) << "non COMDAT symbol '" << from->getName() << "' in hybrid map";
       continue;
     }
     from->getChunk()->setEntryThunk(to);
@@ -799,24 +796,19 @@ static std::string getSourceLocation(InputFile *file, SectionChunk *sc,
 void SymbolTable::reportDuplicate(Symbol *existing, InputFile *newFile,
                                   SectionChunk *newSc,
                                   uint32_t newSectionOffset) {
-  std::string msg;
-  llvm::raw_string_ostream os(msg);
-  os << "duplicate symbol: " << toString(ctx, *existing);
+  COFFSyncStream diag(ctx, ctx.config.forceMultiple ? DiagLevel::Warn
+                                                    : DiagLevel::Err);
+  diag << "duplicate symbol: " << existing;
 
   DefinedRegular *d = dyn_cast<DefinedRegular>(existing);
   if (d && isa<ObjFile>(d->getFile())) {
-    os << getSourceLocation(d->getFile(), d->getChunk(), d->getValue(),
-                            existing->getName());
+    diag << getSourceLocation(d->getFile(), d->getChunk(), d->getValue(),
+                              existing->getName());
   } else {
-    os << getSourceLocation(existing->getFile(), nullptr, 0, "");
+    diag << getSourceLocation(existing->getFile(), nullptr, 0, "");
   }
-  os << getSourceLocation(newFile, newSc, newSectionOffset,
-                          existing->getName());
-
-  if (ctx.config.forceMultiple)
-    warn(msg);
-  else
-    error(msg);
+  diag << getSourceLocation(newFile, newSc, newSectionOffset,
+                            existing->getName());
 }
 
 Symbol *SymbolTable::addAbsolute(StringRef n, COFFSymbolRef sym) {
diff --git a/lld/COFF/Symbols.cpp b/lld/COFF/Symbols.cpp
index f2fa2392ecbbc..7de2c3829d1b0 100644
--- a/lld/COFF/Symbols.cpp
+++ b/lld/COFF/Symbols.cpp
@@ -53,6 +53,17 @@ std::string toCOFFString(const COFFLinkerContext &ctx,
   return maybeDemangleSymbol(ctx, b.getName());
 }
 
+const COFFSyncStream &
+coff::operator<<(const COFFSyncStream &s,
+                 const llvm::object::Archive::Symbol *sym) {
+  s << maybeDemangleSymbol(s.ctx, sym->getName());
+  return s;
+}
+
+const COFFSyncStream &coff::operator<<(const COFFSyncStream &s, Symbol *sym) {
+  return s << maybeDemangleSymbol(s.ctx, sym->getName());
+}
+
 namespace coff {
 
 void Symbol::computeName() {
diff --git a/lld/COFF/Symbols.h b/lld/COFF/Symbols.h
index 203a542466c68..3b367a38c889f 100644
--- a/lld/COFF/Symbols.h
+++ b/lld/COFF/Symbols.h
@@ -33,8 +33,13 @@ class ArchiveFile;
 class COFFLinkerContext;
 class InputFile;
 class ObjFile;
+class Symbol;
 class SymbolTable;
 
+const COFFSyncStream &operator<<(const COFFSyncStream &,
+                                 const llvm::object::Archive::Symbol *);
+const COFFSyncStream &operator<<(const COFFSyncStream &, Symbol *);
+
 // The base class for real symbol classes.
 class Symbol {
 public:
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index d3e326378ed2d..3d38c3b6c241c 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -79,6 +79,11 @@ static_assert(sizeof(dosProgram) % 8 == 0,
 
 static const int dosStubSize = sizeof(dos_header) + sizeof(dosProgram);
 static_assert(dosStubSize % 8 == 0, "DOSStub size must be multiple of 8");
+static const uint32_t coffHeaderOffset = dosStubSize + sizeof(PEMagic);
+static const uint32_t peHeaderOffset =
+    coffHeaderOffset + sizeof(coff_file_header);
+static const uint32_t dataDirOffset64 =
+    peHeaderOffset + sizeof(pe32plus_header);
 
 static const int numberOfDataDirectory = 16;
 
@@ -272,12 +277,12 @@ class Writer {
   OutputSection *findSection(StringRef name);
   void addBaserels();
   void addBaserelBlocks(std::vector<Baserel> &v);
+  void createDynamicRelocs();
 
   uint32_t getSizeOfInitializedData();
 
   void prepareLoadConfig();
   template <typename T> void prepareLoadConfig(T *loadConfig);
-  template <typename T> void checkLoadConfigGuardData(const T *loadConfig);
 
   std::unique_ptr<FileOutputBuffer> &buffer;
   std::map<PartialSectionKey, PartialSection *> partialSections;
@@ -671,13 +676,14 @@ void Writer::finalizeAddresses() {
     }
     if (rangesOk) {
       if (pass > 0)
-        log("Added " + Twine(numChunks - origNumChunks) + " thunks with " +
-            "margin " + Twine(margin) + " in " + Twine(pass) + " passes");
+        Log(ctx) << "Added " << (numChunks - origNumChunks) << " thunks with "
+                 << "margin " << margin << " in " << pass << " passes";
       return;
     }
 
     if (pass >= 10)
-      fatal("adding thunks hasn't converged after " + Twine(pass) + " passes");
+      Fatal(ctx) << "adding thunks hasn't converged after " << pass
+                 << " passes";
 
     if (pass > 0) {
       // If the previous pass didn't work out, reset everything back to the
@@ -754,6 +760,8 @@ void Writer::run() {
     llvm::TimeTraceScope timeScope("Write PE");
     ScopedTimer t1(ctx.codeLayoutTimer);
 
+    if (ctx.config.machine == ARM64X)
+      ctx.dynamicRelocs = make<DynamicRelocsChunk>();
     createImportTables();
     createSections();
     appendImportThunks();
@@ -764,6 +772,7 @@ void Writer::run() {
     mergeSections();
     sortECChunks();
     appendECImportTables();
+    createDynamicRelocs();
     removeUnusedSections();
     finalizeAddresses();
     removeEmptySections();
@@ -773,8 +782,8 @@ void Writer::run() {
     createSymbolAndStringTable();
 
     if (fileSize > UINT32_MAX)
-      fatal("image size (" + Twine(fileSize) + ") " +
-            "exceeds maximum allowable size (" + Twine(UINT32_MAX) + ")");
+      Fatal(ctx) << "image size (" << fileSize << ") "
+                 << "exceeds maximum allowable size (" << UINT32_MAX << ")";
 
     openFile(ctx.config.outputFile);
     if (ctx.config.is64()) {
@@ -809,8 +818,8 @@ void Writer::run() {
   llvm::TimeTraceScope timeScope("Commit PE to disk");
   ScopedTimer t2(ctx.outputCommitTimer);
   if (auto e = buffer->commit())
-    fatal("failed to write output '" + buffer->getPath() +
-          "': " + toString(std::move(e)));
+    Fatal(ctx) << "failed to write output '" << buffer->getPath()
+               << "': " << toString(std::move(e));
 }
 
 static StringRef getOutputSectionName(StringRef name) {
@@ -1117,7 +1126,7 @@ void Writer::createSections() {
       // special case for all architectures.
       outChars = data | r;
 
-      log("Processing section " + pSec->name + " -> " + name);
+      Log(ctx) << "Processing section " << pSec->name << " -> " << name;
 
       sortCRTSectionChunks(pSec->chunks);
     }
@@ -1247,12 +1256,12 @@ void Writer::createImportTables() {
       ctx.config.dllOrder[dll] = ctx.config.dllOrder.size();
 
     if (file->impSym && !isa<DefinedImportData>(file->impSym))
-      fatal(toString(ctx, *file->impSym) + " was replaced");
+      Fatal(ctx) << file->impSym << " was replaced";
     DefinedImportData *impSym = cast_or_null<DefinedImportData>(file->impSym);
     if (ctx.config.delayLoads.count(StringRef(file->dllName).lower())) {
       if (!file->thunkSym)
-        fatal("cannot delay-load " + toString(file) +
-              " due to import of data: " + toString(ctx, *impSym));
+        Fatal(ctx) << "cannot delay-load " << toString(file)
+                   << " due to import of data: " << impSym;
       delayIdata.add(impSym);
     } else {
       idata.add(impSym);
@@ -1271,7 +1280,7 @@ void Writer::appendImportThunks() {
 
     if (file->thunkSym) {
       if (!isa<DefinedImportThunk>(file->thunkSym))
-        fatal(toString(ctx, *file->thunkSym) + " was replaced");
+        Fatal(ctx) << file->thunkSym << " was replaced";
       auto *chunk = cast<DefinedImportThunk>(file->thunkSym)->getChunk();
       if (chunk->live)
         textSec->addChunk(chunk);
@@ -1279,7 +1288,7 @@ void Writer::appendImportThunks() {
 
     if (file->auxThunkSym) {
       if (!isa<DefinedImportThunk>(file->auxThunkSym))
-        fatal(toString(ctx, *file->auxThunkSym) + " was replaced");
+        Fatal(ctx) << file->auxThunkSym << " was replaced";
       auto *chunk = cast<DefinedImportThunk>(file->auxThunkSym)->getChunk();
       if (chunk->live)
         textSec->addChunk(chunk);
@@ -1313,7 +1322,7 @@ void Writer::createExportTable() {
     // Allow using a custom built export table from input object files, instead
     // of having the linker synthesize the tables.
     if (ctx.config.hadExplicitExports)
-      warn("literal .edata sections override exports");
+      Warn(ctx) << "literal .edata sections override exports";
   } else if (!ctx.config.exports.empty()) {
     for (Chunk *c : edata.chunks)
       edataSec->addChunk(c);
@@ -1325,7 +1334,7 @@ void Writer::createExportTable() {
   // Warn on exported deleting destructor.
   for (auto e : ctx.config.exports)
     if (e.sym && e.sym->getName().starts_with("??_G"))
-      warn("export of deleting dtor: " + toString(ctx, *e.sym));
+      Warn(ctx) << "export of deleting dtor: " << e.sym;
 }
 
 void Writer::removeUnusedSections() {
@@ -1457,9 +1466,10 @@ void Writer::createSymbolAndStringTable() {
     if ((sec->header.Characteristics & IMAGE_SCN_MEM_DISCARDABLE) == 0)
       continue;
     if (ctx.config.warnLongSectionNames) {
-      warn("section name " + sec->name +
-           " is longer than 8 characters and will use a non-standard string "
-           "table");
+      Warn(ctx)
+          << "section name " << sec->name
+          << " is longer than 8 characters and will use a non-standard string "
+             "table";
     }
     sec->setStringTableOff(addEntryToStringTable(sec->name));
   }
@@ -1538,7 +1548,7 @@ void Writer::mergeSections() {
     StringSet<> names;
     while (true) {
       if (!names.insert(toName).second)
-        fatal("/merge: cycle found for section '" + p.first + "'");
+        Fatal(ctx) << "/merge: cycle found for section '" << p.first << "'";
       auto i = ctx.config.merge.find(toName);
       if (i == ctx.config.merge.end())
         break;
@@ -1596,8 +1606,14 @@ void Writer::assignAddresses() {
 
   for (OutputSection *sec : ctx.outputSections) {
     llvm::TimeTraceScope timeScope("Section: ", sec->name);
-    if (sec == relocSec)
+    if (sec == relocSec) {
+      sec->chunks.clear();
       addBaserels();
+      if (ctx.dynamicRelocs) {
+        ctx.dynamicRelocs->finalize();
+        relocSec->addChunk(ctx.dynamicRelocs);
+      }
+    }
     uint64_t rawSize = 0, virtualSize = 0;
     sec->header.VirtualAddress = rva;
 
@@ -1628,7 +1644,7 @@ void Writer::assignAddresses() {
         rawSize = alignTo(virtualSize, config->fileAlign);
     }
     if (virtualSize > UINT32_MAX)
-      error("section larger than 4 GiB: " + sec->name);
+      Err(ctx) << "section larger than 4 GiB: " << sec->name;
     sec->header.VirtualSize = virtualSize;
     sec->header.SizeOfRawData = rawSize;
     if (rawSize != 0)
@@ -1672,6 +1688,7 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
   buf += sizeof(PEMagic);
 
   // Write COFF header
+  assert(coffHeaderOffset == buf - buffer->getBufferStart());
   auto *coff = reinterpret_cast<coff_file_header *>(buf);
   buf += sizeof(*coff);
   switch (config->machine) {
@@ -1704,6 +1721,7 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
       sizeof(PEHeaderTy) + sizeof(data_directory) * numberOfDataDirectory;
 
   // Write PE header
+  assert(peHeaderOffset == buf - buffer->getBufferStart());
   auto *pe = reinterpret_cast<PEHeaderTy *>(buf);
   buf += sizeof(*pe);
   pe->Magic = config->is64() ? PE32Header::PE32_PLUS : PE32Header::PE32;
@@ -1769,6 +1787,8 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
   pe->SizeOfInitializedData = getSizeOfInitializedData();
 
   // Write data directory
+  assert(!ctx.config.is64() ||
+         dataDirOffset64 == buf - buffer->getBufferStart());
   auto *dir = reinterpret_cast<data_directory *>(buf);
   buf += sizeof(*dir) * numberOfDataDirectory;
   if (edataStart) {
@@ -1798,9 +1818,12 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
                                 exceptionTable.last->getSize() -
                                 exceptionTable.first->getRVA();
   }
-  if (relocSec->getVirtualSize()) {
+  size_t relocSize = relocSec->getVirtualSize();
+  if (ctx.dynamicRelocs)
+    relocSize -= ctx.dynamicRelocs->getSize();
+  if (relocSize) {
     dir[BASE_RELOCATION_TABLE].RelativeVirtualAddress = relocSec->getRVA();
-    dir[BASE_RELOCATION_TABLE].Size = relocSec->getVirtualSize();
+    dir[BASE_RELOCATION_TABLE].Size = relocSize;
   }
   if (Symbol *sym = ctx.symtab.findUnderscore("_tls_used")) {
     if (Defined *b = dyn_cast<Defined>(sym)) {
@@ -1820,13 +1843,13 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
       assert(b->getRVA() >= sc->getRVA());
       uint64_t offsetInChunk = b->getRVA() - sc->getRVA();
       if (!sc->hasData || offsetInChunk + 4 > sc->getSize())
-        fatal("_load_config_used is malformed");
+        Fatal(ctx) << "_load_config_used is malformed";
 
       ArrayRef<uint8_t> secContents = sc->getContents();
       uint32_t loadConfigSize =
           *reinterpret_cast<const ulittle32_t *>(&secContents[offsetInChunk]);
       if (offsetInChunk + loadConfigSize > sc->getSize())
-        fatal("_load_config_used is too large");
+        Fatal(ctx) << "_load_config_used is too large";
       dir[LOAD_CONFIG_TABLE].RelativeVirtualAddress = b->getRVA();
       dir[LOAD_CONFIG_TABLE].Size = loadConfigSize;
     }
@@ -1873,7 +1896,8 @@ void Writer::createSEHTable() {
   SymbolRVASet handlers;
   for (ObjFile *file : ctx.objFileInstances) {
     if (!file->hasSafeSEH())
-      error("/safeseh: " + file->getName() + " is not compatible with SEH");
+      Err(ctx) << "/safeseh: " << file->getName()
+               << " is not compatible with SEH";
     markSymbolsForRVATable(file, file->getSXDataChunks(), handlers);
   }
 
@@ -2086,8 +2110,8 @@ void Writer::getSymbolsFromSections(ObjFile *file,
     // Validate that the contents look like symbol table indices.
     ArrayRef<uint8_t> data = c->getContents();
     if (data.size() % 4 != 0) {
-      warn("ignoring " + c->getSectionName() +
-           " symbol table index section in object " + toString(file));
+      Warn(ctx) << "ignoring " << c->getSectionName()
+                << " symbol table index section in object " << file;
       continue;
     }
 
@@ -2098,8 +2122,8 @@ void Writer::getSymbolsFromSections(ObjFile *file,
     ArrayRef<Symbol *> objSymbols = file->getSymbols();
     for (uint32_t symIndex : symIndices) {
       if (symIndex >= objSymbols.size()) {
-        warn("ignoring invalid symbol table index in section " +
-             c->getSectionName() + " in object " + toString(file));
+        Warn(ctx) << "ignoring invalid symbol table index in section "
+                  << c->getSectionName() << " in object " << file;
         continue;
       }
       if (Symbol *s = objSymbols[symIndex]) {
@@ -2215,19 +2239,20 @@ void Writer::createRuntimePseudoRelocs() {
     // Not writing any pseudo relocs; if some were needed, error out and
     // indicate what required them.
     for (const RuntimePseudoReloc &rpr : rels)
-      error("automatic dllimport of " + rpr.sym->getName() + " in " +
-            toString(rpr.target->file) + " requires pseudo relocations");
+      Err(ctx) << "automatic dllimport of " << rpr.sym->getName() << " in "
+               << toString(rpr.target->file) << " requires pseudo relocations";
     return;
   }
 
   if (!rels.empty()) {
-    log("Writing " + Twine(rels.size()) + " runtime pseudo relocations");
+    Log(ctx) << "Writing " << rels.size() << " runtime pseudo relocations";
     const char *symbolName = "_pei386_runtime_relocator";
     Symbol *relocator = ctx.symtab.findUnderscore(symbolName);
     if (!relocator)
-      error("output image has runtime pseudo relocations, but the function " +
-            Twine(symbolName) +
-            " is missing; it is needed for fixing the relocations at runtime");
+      Err(ctx)
+          << "output image has runtime pseudo relocations, but the function "
+          << symbolName
+          << " is missing; it is needed for fixing the relocations at runtime";
   }
 
   PseudoRelocTableChunk *table = make<PseudoRelocTableChunk>(rels);
@@ -2427,8 +2452,8 @@ void Writer::sortExceptionTable(ChunkRange &exceptionTable) {
   uint8_t *begin = bufAddr(exceptionTable.first);
   uint8_t *end = bufAddr(exceptionTable.last) + exceptionTable.last->getSize();
   if ((end - begin) % sizeof(T) != 0) {
-    fatal("unexpected .pdata size: " + Twine(end - begin) +
-          " is not a multiple of " + Twine(sizeof(T)));
+    Fatal(ctx) << "unexpected .pdata size: " << (end - begin)
+               << " is not a multiple of " << sizeof(T);
   }
 
   parallelSort(MutableArrayRef<T>(reinterpret_cast<T *>(begin),
@@ -2496,8 +2521,8 @@ void Writer::sortCRTSectionChunks(std::vector<Chunk *> &chunks) {
   if (ctx.config.verbose) {
     for (auto &c : chunks) {
       auto sc = dyn_cast<SectionChunk>(c);
-      log("  " + sc->file->mb.getBufferIdentifier().str() +
-          ", SectionID: " + Twine(sc->getSectionNumber()));
+      Log(ctx) << "  " << sc->file->mb.getBufferIdentifier().str()
+               << ", SectionID: " << sc->getSectionNumber();
     }
   }
 }
@@ -2521,7 +2546,6 @@ uint32_t Writer::getSizeOfInitializedData() {
 void Writer::addBaserels() {
   if (!ctx.config.relocatable)
     return;
-  relocSec->chunks.clear();
   std::vector<Baserel> v;
   for (OutputSection *sec : ctx.outputSections) {
     if (sec->header.Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
@@ -2555,6 +2579,29 @@ void Writer::addBaserelBlocks(std::vector<Baserel> &v) {
   relocSec->addChunk(make<BaserelChunk>(page, &v[i], &v[0] + j));
 }
 
+void Writer::createDynamicRelocs() {
+  if (!ctx.dynamicRelocs)
+    return;
+
+  // Adjust the Machine field in the COFF header to AMD64.
+  ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE, sizeof(uint16_t),
+                         coffHeaderOffset + offsetof(coff_file_header, Machine),
+                         AMD64);
+
+  // Clear the load config directory.
+  // FIXME: Use the hybrid load config value instead.
+  ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE, sizeof(uint32_t),
+                         dataDirOffset64 +
+                             LOAD_CONFIG_TABLE * sizeof(data_directory) +
+                             offsetof(data_directory, RelativeVirtualAddress),
+                         0);
+  ctx.dynamicRelocs->add(IMAGE_DVRT_ARM64X_FIXUP_TYPE_VALUE, sizeof(uint32_t),
+                         dataDirOffset64 +
+                             LOAD_CONFIG_TABLE * sizeof(data_directory) +
+                             offsetof(data_directory, Size),
+                         0);
+}
+
 PartialSection *Writer::createPartialSection(StringRef name,
                                              uint32_t outChars) {
   PartialSection *&pSec = partialSections[{name, outChars}];
@@ -2588,7 +2635,7 @@ void Writer::fixTlsAlignment() {
                                : sizeof(object::coff_tls_directory32);
 
   if (tlsOffset + directorySize > sec->getRawSize())
-    fatal("_tls_used sym is malformed");
+    Fatal(ctx) << "_tls_used sym is malformed";
 
   if (ctx.config.is64()) {
     object::coff_tls_directory64 *tlsDir =
@@ -2606,7 +2653,11 @@ void Writer::prepareLoadConfig() {
   auto *b = cast_if_present<DefinedRegular>(sym);
   if (!b) {
     if (ctx.config.guardCF != GuardCFLevel::Off)
-      warn("Control Flow Guard is enabled but '_load_config_used' is missing");
+      Warn(ctx)
+          << "Control Flow Guard is enabled but '_load_config_used' is missing";
+    if (ctx.config.dependentLoadFlags)
+      Warn(ctx) << "_load_config_used not found, /dependentloadflag will have "
+                   "no effect";
     return;
   }
 
@@ -2616,13 +2667,13 @@ void Writer::prepareLoadConfig() {
   uint8_t *symBuf = secBuf + (b->getRVA() - sec->getRVA());
   uint32_t expectedAlign = ctx.config.is64() ? 8 : 4;
   if (b->getChunk()->getAlignment() < expectedAlign)
-    warn("'_load_config_used' is misaligned (expected alignment to be " +
-         Twine(expectedAlign) + " bytes, got " +
-         Twine(b->getChunk()->getAlignment()) + " instead)");
+    Warn(ctx) << "'_load_config_used' is misaligned (expected alignment to be "
+              << expectedAlign << " bytes, got "
+              << b->getChunk()->getAlignment() << " instead)";
   else if (!isAligned(Align(expectedAlign), b->getRVA()))
-    warn("'_load_config_used' is misaligned (RVA is 0x" +
-         Twine::utohexstr(b->getRVA()) + " not aligned to " +
-         Twine(expectedAlign) + " bytes)");
+    Warn(ctx) << "'_load_config_used' is misaligned (RVA is 0x"
+              << Twine::utohexstr(b->getRVA()) << " not aligned to "
+              << expectedAlign << " bytes)";
 
   if (ctx.config.is64())
     prepareLoadConfig(reinterpret_cast<coff_load_configuration64 *>(symBuf));
@@ -2631,14 +2682,6 @@ void Writer::prepareLoadConfig() {
 }
 
 template <typename T> void Writer::prepareLoadConfig(T *loadConfig) {
-  if (ctx.config.dependentLoadFlags)
-    loadConfig->DependentLoadFlags = ctx.config.dependentLoadFlags;
-
-  checkLoadConfigGuardData(loadConfig);
-}
-
-template <typename T>
-void Writer::checkLoadConfigGuardData(const T *loadConfig) {
   size_t loadConfigSize = loadConfig->Size;
 
 #define RETURN_IF_NOT_CONTAINS(field)                                          \
@@ -2660,6 +2703,23 @@ void Writer::checkLoadConfigGuardData(const T *loadConfig) {
     if (loadConfig->field != s->getVA())                                       \
       warn(#field " not set correctly in '_load_config_used'");
 
+  if (ctx.config.dependentLoadFlags) {
+    RETURN_IF_NOT_CONTAINS(DependentLoadFlags)
+    loadConfig->DependentLoadFlags = ctx.config.dependentLoadFlags;
+  }
+
+  if (ctx.dynamicRelocs) {
+    IF_CONTAINS(DynamicValueRelocTableSection) {
+      loadConfig->DynamicValueRelocTableSection = relocSec->sectionIndex;
+      loadConfig->DynamicValueRelocTableOffset =
+          ctx.dynamicRelocs->getRVA() - relocSec->getRVA();
+    }
+    else {
+      warn("'_load_config_used' structure too small to include dynamic "
+           "relocations");
+    }
+  }
+
   if (ctx.config.guardCF == GuardCFLevel::Off)
     return;
   RETURN_IF_NOT_CONTAINS(GuardFlags)
diff --git a/lld/Common/ErrorHandler.cpp b/lld/Common/ErrorHandler.cpp
index 6b60ebb18e821..716bce54258ce 100644
--- a/lld/Common/ErrorHandler.cpp
+++ b/lld/Common/ErrorHandler.cpp
@@ -219,6 +219,9 @@ void ErrorHandler::reportDiagnostic(StringRef location, Colors c,
   }
   os << msg << '\n';
   errs() << buf;
+  // If msg contains a newline, ensure that the next diagnostic is preceded by
+  // a blank line separator.
+  sep = getSeparator(msg);
 }
 
 void ErrorHandler::log(const Twine &msg) {
@@ -247,7 +250,6 @@ void ErrorHandler::warn(const Twine &msg) {
 
   std::lock_guard<std::mutex> lock(mu);
   reportDiagnostic(getLocation(msg), Colors::MAGENTA, "warning", msg);
-  sep = getSeparator(msg);
 }
 
 void ErrorHandler::error(const Twine &msg) {
@@ -278,7 +280,6 @@ void ErrorHandler::error(const Twine &msg) {
       exit = exitEarly;
     }
 
-    sep = getSeparator(msg);
     ++errorCount;
   }
 
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 5b5ad482ea127..99fc750486e4b 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -154,9 +154,9 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_MOVW_UABS_G3:
     return R_ABS;
   case R_AARCH64_AUTH_ABS64:
-    return R_AARCH64_AUTH;
+    return RE_AARCH64_AUTH;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
-    return R_AARCH64_TLSDESC_PAGE;
+    return RE_AARCH64_TLSDESC_PAGE;
   case R_AARCH64_TLSDESC_LD64_LO12:
   case R_AARCH64_TLSDESC_ADD_LO12:
     return R_TLSDESC;
@@ -198,15 +198,15 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
     return R_PC;
   case R_AARCH64_ADR_PREL_PG_HI21:
   case R_AARCH64_ADR_PREL_PG_HI21_NC:
-    return R_AARCH64_PAGE_PC;
+    return RE_AARCH64_PAGE_PC;
   case R_AARCH64_LD64_GOT_LO12_NC:
   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
     return R_GOT;
   case R_AARCH64_LD64_GOTPAGE_LO15:
-    return R_AARCH64_GOT_PAGE;
+    return RE_AARCH64_GOT_PAGE;
   case R_AARCH64_ADR_GOT_PAGE:
   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
-    return R_AARCH64_GOT_PAGE_PC;
+    return RE_AARCH64_GOT_PAGE_PC;
   case R_AARCH64_GOTPCREL32:
   case R_AARCH64_GOT_LD_PREL19:
     return R_GOT_PC;
@@ -222,7 +222,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {
   if (expr == R_RELAX_TLS_GD_TO_IE) {
     if (type == R_AARCH64_TLSDESC_ADR_PAGE21)
-      return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
+      return RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
     return R_RELAX_TLS_GD_TO_IE_ABS;
   }
   return expr;
@@ -877,7 +877,7 @@ bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,
   if (val != llvm::SignExtend64(val, 33))
     return false;
 
-  Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21,
+  Relocation adrpSymRel = {RE_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21,
                            adrpRel.offset, /*addend=*/0, &sym};
   Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset,
                        /*addend=*/0, &sym};
@@ -922,21 +922,21 @@ void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
     }
 
     switch (rel.expr) {
-    case R_AARCH64_GOT_PAGE_PC:
+    case RE_AARCH64_GOT_PAGE_PC:
       if (i + 1 < size &&
           relaxer.tryRelaxAdrpLdr(rel, sec.relocs()[i + 1], secAddr, buf)) {
         ++i;
         continue;
       }
       break;
-    case R_AARCH64_PAGE_PC:
+    case RE_AARCH64_PAGE_PC:
       if (i + 1 < size &&
           relaxer.tryRelaxAdrpAdd(rel, sec.relocs()[i + 1], secAddr, buf)) {
         ++i;
         continue;
       }
       break;
-    case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
+    case RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
     case R_RELAX_TLS_GD_TO_IE_ABS:
       relaxTlsGdToIe(loc, rel, val);
       continue;
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index 62685b1e7dede..29a72d35af666 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -136,7 +136,7 @@ RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
     // GOT(S) + A - P
     return R_GOT_PC;
   case R_ARM_SBREL32:
-    return R_ARM_SBREL;
+    return RE_ARM_SBREL;
   case R_ARM_TARGET1:
     return ctx.arg.target1Rel ? R_PC : R_ABS;
   case R_ARM_TARGET2:
@@ -176,14 +176,14 @@ RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
   case R_ARM_THM_ALU_PREL_11_0:
   case R_ARM_THM_PC8:
   case R_ARM_THM_PC12:
-    return R_ARM_PCA;
+    return RE_ARM_PCA;
   case R_ARM_MOVW_BREL_NC:
   case R_ARM_MOVW_BREL:
   case R_ARM_MOVT_BREL:
   case R_ARM_THM_MOVW_BREL_NC:
   case R_ARM_THM_MOVW_BREL:
   case R_ARM_THM_MOVT_BREL:
-    return R_ARM_SBREL;
+    return RE_ARM_SBREL;
   case R_ARM_NONE:
     return R_NONE;
   case R_ARM_TLS_LE32:
diff --git a/lld/ELF/Arch/Hexagon.cpp b/lld/ELF/Arch/Hexagon.cpp
index 3c6fe0daee6bd..23b60672f6317 100644
--- a/lld/ELF/Arch/Hexagon.cpp
+++ b/lld/ELF/Arch/Hexagon.cpp
@@ -199,7 +199,7 @@ static uint32_t findMaskR6(Ctx &ctx, uint32_t insn) {
       return i.relocMask;
 
   Err(ctx) << "unrecognized instruction for 6_X relocation: 0x"
-           << utohexstr(insn);
+           << utohexstr(insn, true);
   return 0;
 }
 
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index ebfdbafc9983e..3280c34cb6ed0 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -428,7 +428,7 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
   case R_LARCH_SUB_ULEB128:
     // The LoongArch add/sub relocs behave like the RISCV counterparts; reuse
     // the RelExpr to avoid code duplication.
-    return R_RISCV_ADD;
+    return RE_RISCV_ADD;
   case R_LARCH_32_PCREL:
   case R_LARCH_64_PCREL:
   case R_LARCH_PCREL20_S2:
@@ -444,17 +444,17 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
   case R_LARCH_TLS_IE_PC_HI20:
   case R_LARCH_TLS_IE64_PC_LO20:
   case R_LARCH_TLS_IE64_PC_HI12:
-    return R_LOONGARCH_GOT_PAGE_PC;
+    return RE_LOONGARCH_GOT_PAGE_PC;
   case R_LARCH_GOT_PC_LO12:
   case R_LARCH_TLS_IE_PC_LO12:
-    return R_LOONGARCH_GOT;
+    return RE_LOONGARCH_GOT;
   case R_LARCH_TLS_LD_PC_HI20:
   case R_LARCH_TLS_GD_PC_HI20:
-    return R_LOONGARCH_TLSGD_PAGE_PC;
+    return RE_LOONGARCH_TLSGD_PAGE_PC;
   case R_LARCH_PCALA_HI20:
-    // Why not R_LOONGARCH_PAGE_PC, majority of references don't go through PLT
-    // anyway so why waste time checking only to get everything relaxed back to
-    // it?
+    // Why not RE_LOONGARCH_PAGE_PC, majority of references don't go through
+    // PLT anyway so why waste time checking only to get everything relaxed back
+    // to it?
     //
     // This is again due to the R_LARCH_PCALA_LO12 on JIRL case, where we want
     // both the HI20 and LO12 to potentially refer to the PLT. But in reality
@@ -474,12 +474,12 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
     //
     // So, unfortunately we have to again workaround this quirk the same way as
     // BFD: assuming every R_LARCH_PCALA_HI20 is potentially PLT-needing, only
-    // relaxing back to R_LOONGARCH_PAGE_PC if it's known not so at a later
+    // relaxing back to RE_LOONGARCH_PAGE_PC if it's known not so at a later
     // stage.
-    return R_LOONGARCH_PLT_PAGE_PC;
+    return RE_LOONGARCH_PLT_PAGE_PC;
   case R_LARCH_PCALA64_LO20:
   case R_LARCH_PCALA64_HI12:
-    return R_LOONGARCH_PAGE_PC;
+    return RE_LOONGARCH_PAGE_PC;
   case R_LARCH_GOT_HI20:
   case R_LARCH_GOT_LO12:
   case R_LARCH_GOT64_LO20:
@@ -501,7 +501,7 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s,
   case R_LARCH_TLS_DESC_PC_HI20:
   case R_LARCH_TLS_DESC64_PC_LO20:
   case R_LARCH_TLS_DESC64_PC_HI12:
-    return R_LOONGARCH_TLSDESC_PAGE_PC;
+    return RE_LOONGARCH_TLSDESC_PAGE_PC;
   case R_LARCH_TLS_DESC_PC_LO12:
   case R_LARCH_TLS_DESC_LD:
   case R_LARCH_TLS_DESC_HI20:
diff --git a/lld/ELF/Arch/Mips.cpp b/lld/ELF/Arch/Mips.cpp
index da76820de240d..c4848fecc6f28 100644
--- a/lld/ELF/Arch/Mips.cpp
+++ b/lld/ELF/Arch/Mips.cpp
@@ -105,7 +105,7 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s,
   case R_MIPS_GPREL32:
   case R_MICROMIPS_GPREL16:
   case R_MICROMIPS_GPREL7_S2:
-    return R_MIPS_GOTREL;
+    return RE_MIPS_GOTREL;
   case R_MIPS_26:
   case R_MICROMIPS_26_S1:
     return R_PLT;
@@ -122,9 +122,9 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s,
     // equal to the start of .got section. In that case we consider these
     // relocations as relative.
     if (&s == ctx.sym.mipsGpDisp)
-      return R_MIPS_GOT_GP_PC;
+      return RE_MIPS_GOT_GP_PC;
     if (&s == ctx.sym.mipsLocalGp)
-      return R_MIPS_GOT_GP;
+      return RE_MIPS_GOT_GP;
     [[fallthrough]];
   case R_MIPS_32:
   case R_MIPS_64:
@@ -163,14 +163,14 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s,
   case R_MIPS_GOT16:
   case R_MICROMIPS_GOT16:
     if (s.isLocal())
-      return R_MIPS_GOT_LOCAL_PAGE;
+      return RE_MIPS_GOT_LOCAL_PAGE;
     [[fallthrough]];
   case R_MIPS_CALL16:
   case R_MIPS_GOT_DISP:
   case R_MIPS_TLS_GOTTPREL:
   case R_MICROMIPS_CALL16:
   case R_MICROMIPS_TLS_GOTTPREL:
-    return R_MIPS_GOT_OFF;
+    return RE_MIPS_GOT_OFF;
   case R_MIPS_CALL_HI16:
   case R_MIPS_CALL_LO16:
   case R_MIPS_GOT_HI16:
@@ -179,15 +179,15 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s,
   case R_MICROMIPS_CALL_LO16:
   case R_MICROMIPS_GOT_HI16:
   case R_MICROMIPS_GOT_LO16:
-    return R_MIPS_GOT_OFF32;
+    return RE_MIPS_GOT_OFF32;
   case R_MIPS_GOT_PAGE:
-    return R_MIPS_GOT_LOCAL_PAGE;
+    return RE_MIPS_GOT_LOCAL_PAGE;
   case R_MIPS_TLS_GD:
   case R_MICROMIPS_TLS_GD:
-    return R_MIPS_TLSGD;
+    return RE_MIPS_TLSGD;
   case R_MIPS_TLS_LDM:
   case R_MICROMIPS_TLS_LDM:
-    return R_MIPS_TLSLD;
+    return RE_MIPS_TLSLD;
   case R_MIPS_NONE:
     return R_NONE;
   default:
@@ -369,7 +369,7 @@ bool MIPS<ELFT>::needsThunk(RelExpr expr, RelType type, const InputFile *file,
   if (type != R_MIPS_26 && type != R_MIPS_PC26_S2 &&
       type != R_MICROMIPS_26_S1 && type != R_MICROMIPS_PC26_S1)
     return false;
-  auto *f = dyn_cast_or_null<ObjFile<ELFT>>(file);
+  auto *f = dyn_cast<ObjFile<ELFT>>(file);
   if (!f)
     return false;
   // If current file has PIC code, LA25 stub is not required.
diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index 417401374436a..3203e27d82fa2 100644
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -250,7 +250,7 @@ RelExpr PPC::getRelExpr(RelType type, const Symbol &s,
   case R_PPC_REL24:
     return R_PLT_PC;
   case R_PPC_PLTREL24:
-    return R_PPC32_PLTREL;
+    return RE_PPC32_PLTREL;
   case R_PPC_GOT_TLSGD16:
     return R_TLSGD_GOT;
   case R_PPC_GOT_TLSLD16:
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 8dd1735ee1e88..4edb6af4f09ad 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -695,7 +695,7 @@ void PPC64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     if (pcRelInsn == UINT64_C(-1)) {
       Err(ctx)
           << "unrecognized instruction for R_PPC64_PCREL_OPT relaxation: 0x"
-          << Twine::utohexstr(accessInsn);
+          << utohexstr(accessInsn, true);
       break;
     }
 
@@ -1029,12 +1029,12 @@ RelExpr PPC64::getRelExpr(RelType type, const Symbol &s,
     return R_GOT_PC;
   case R_PPC64_TOC16_HA:
   case R_PPC64_TOC16_LO_DS:
-    return ctx.arg.tocOptimize ? R_PPC64_RELAX_TOC : R_GOTREL;
+    return ctx.arg.tocOptimize ? RE_PPC64_RELAX_TOC : R_GOTREL;
   case R_PPC64_TOC:
-    return R_PPC64_TOCBASE;
+    return RE_PPC64_TOCBASE;
   case R_PPC64_REL14:
   case R_PPC64_REL24:
-    return R_PPC64_CALL_PLT;
+    return RE_PPC64_CALL_PLT;
   case R_PPC64_REL24_NOTOC:
     return R_PLT_PC;
   case R_PPC64_REL16_LO:
@@ -1337,7 +1337,7 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       if (isInstructionUpdateForm(insn))
         Err(ctx) << getErrorLoc(ctx, loc)
                  << "can't toc-optimize an update instruction: 0x"
-                 << utohexstr(insn);
+                 << utohexstr(insn, true);
       writeFromHalf16(ctx, loc, (insn & 0xffe00000) | 0x00020000 | lo(val));
     } else {
       write16(ctx, loc, lo(val));
@@ -1356,8 +1356,8 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       // pointer register r2, as the base register.
       if (isInstructionUpdateForm(insn))
         Err(ctx) << getErrorLoc(ctx, loc)
-                 << "Can't toc-optimize an update instruction: 0x"
-                 << Twine::utohexstr(insn);
+                 << "can't toc-optimize an update instruction: 0x"
+                 << utohexstr(insn, true);
       insn &= 0xffe00000 | mask;
       writeFromHalf16(ctx, loc, insn | 0x00020000 | lo(val));
     } else {
@@ -1452,7 +1452,7 @@ bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
 
   // If the offset exceeds the range of the branch type then it will need
   // a range-extending thunk.
-  // See the comment in getRelocTargetVA() about R_PPC64_CALL.
+  // See the comment in getRelocTargetVA() about RE_PPC64_CALL.
   return !inBranchRange(
       type, branchAddr,
       s.getVA(ctx, a) + getPPC64GlobalEntryToLocalEntryOffset(ctx, s.stOther));
@@ -1490,7 +1490,7 @@ RelExpr PPC64::adjustGotPcExpr(RelType type, int64_t addend,
     // It only makes sense to optimize pld since paddi means that the address
     // of the object in the GOT is required rather than the object itself.
     if ((readPrefixedInst(ctx, loc) & 0xfc000000) == 0xe4000000)
-      return R_PPC64_RELAX_GOT_PC;
+      return RE_PPC64_RELAX_GOT_PC;
   }
   return R_GOT_PC;
 }
@@ -1574,7 +1574,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
     uint8_t *loc = buf + rel.offset;
     const uint64_t val = sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset);
     switch (rel.expr) {
-    case R_PPC64_RELAX_GOT_PC: {
+    case RE_PPC64_RELAX_GOT_PC: {
       // The R_PPC64_PCREL_OPT relocation must appear immediately after
       // R_PPC64_GOT_PCREL34 in the relocations table at the same offset.
       // We can only relax R_PPC64_PCREL_OPT if we have also relaxed
@@ -1588,7 +1588,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
       relaxGot(loc, rel, val);
       break;
     }
-    case R_PPC64_RELAX_TOC:
+    case RE_PPC64_RELAX_TOC:
       // rel.sym refers to the STT_SECTION symbol associated to the .toc input
       // section. If an R_PPC64_TOC16_LO (.toc + addend) references the TOC
       // entry, there may be R_PPC64_TOC16_HA not paired with
@@ -1598,7 +1598,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
           !tryRelaxPPC64TocIndirection(ctx, rel, loc))
         relocate(loc, rel, val);
       break;
-    case R_PPC64_CALL:
+    case RE_PPC64_CALL:
       // If this is a call to __tls_get_addr, it may be part of a TLS
       // sequence that has been relaxed and turned into a nop. In this
       // case, we don't want to handle it as a call.
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 58a71fd9545c5..36ae31be6ed2a 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -267,7 +267,6 @@ RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
   case R_RISCV_HI20:
   case R_RISCV_LO12_I:
   case R_RISCV_LO12_S:
-  case R_RISCV_RVC_LUI:
     return R_ABS;
   case R_RISCV_ADD8:
   case R_RISCV_ADD16:
@@ -282,7 +281,7 @@ RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
   case R_RISCV_SUB16:
   case R_RISCV_SUB32:
   case R_RISCV_SUB64:
-    return R_RISCV_ADD;
+    return RE_RISCV_ADD;
   case R_RISCV_JAL:
   case R_RISCV_BRANCH:
   case R_RISCV_PCREL_HI20:
@@ -299,7 +298,7 @@ RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
     return R_GOT_PC;
   case R_RISCV_PCREL_LO12_I:
   case R_RISCV_PCREL_LO12_S:
-    return R_RISCV_PC_INDIRECT;
+    return RE_RISCV_PC_INDIRECT;
   case R_RISCV_TLSDESC_HI20:
   case R_RISCV_TLSDESC_LOAD_LO12:
   case R_RISCV_TLSDESC_ADD_LO12:
@@ -321,7 +320,7 @@ RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
     return ctx.arg.relax ? R_RELAX_HINT : R_NONE;
   case R_RISCV_SET_ULEB128:
   case R_RISCV_SUB_ULEB128:
-    return R_RISCV_LEB128;
+    return RE_RISCV_LEB128;
   default:
     Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v
              << ") against symbol " << &s;
@@ -373,19 +372,6 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     return;
   }
 
-  case R_RISCV_RVC_LUI: {
-    int64_t imm = SignExtend64(val + 0x800, bits) >> 12;
-    checkInt(ctx, loc, imm, 6, rel);
-    if (imm == 0) { // `c.lui rd, 0` is illegal, convert to `c.li rd, 0`
-      write16le(loc, (read16le(loc) & 0x0F83) | 0x4000);
-    } else {
-      uint16_t imm17 = extractBits(val + 0x800, 17, 17) << 12;
-      uint16_t imm16_12 = extractBits(val + 0x800, 16, 12) << 2;
-      write16le(loc, (read16le(loc) & 0xEF83) | imm17 | imm16_12);
-    }
-    return;
-  }
-
   case R_RISCV_JAL: {
     checkInt(ctx, loc, val, 21, rel);
     checkAlignment(ctx, loc, val, 2, rel);
@@ -650,7 +636,7 @@ void RISCV::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
       else
         tlsdescToIe(ctx, loc, rel, val);
       continue;
-    case R_RISCV_LEB128:
+    case RE_RISCV_LEB128:
       if (i + 1 < size) {
         const Relocation &rel1 = relocs[i + 1];
         if (rel.type == R_RISCV_SET_ULEB128 &&
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index dd0dc83189d26..0c4fd00cab65c 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -401,6 +401,7 @@ RelExpr X86_64::getRelExpr(RelType type, const Symbol &s,
   case R_X86_64_CODE_4_GOTPCRELX:
   case R_X86_64_GOTTPOFF:
   case R_X86_64_CODE_4_GOTTPOFF:
+  case R_X86_64_CODE_6_GOTTPOFF:
     return R_GOT_PC;
   case R_X86_64_GOTOFF64:
     return R_GOTPLTREL;
@@ -562,8 +563,9 @@ void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
   }
 }
 
-// In some conditions, R_X86_64_GOTTPOFF/R_X86_64_CODE_4_GOTTPOFF relocation can
-// be optimized to R_X86_64_TPOFF32 so that it does not use GOT.
+// In some conditions,
+// R_X86_64_GOTTPOFF/R_X86_64_CODE_4_GOTTPOFF/R_X86_64_CODE_6_GOTTPOFF
+// relocation can be optimized to R_X86_64_TPOFF32 so that it does not use GOT.
 void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
                             uint64_t val) const {
   uint8_t *inst = loc - 3;
@@ -605,7 +607,7 @@ void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
   } else if (rel.type == R_X86_64_CODE_4_GOTTPOFF) {
     if (loc[-4] != 0xd5) {
       Err(ctx) << getErrorLoc(ctx, loc - 4)
-               << "Invalid prefix with R_X86_64_CODE_4_GOTTPOFF!";
+               << "invalid prefix with R_X86_64_CODE_4_GOTTPOFF!";
       return;
     }
     const uint8_t rex = loc[-3];
@@ -623,6 +625,41 @@ void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
                << "R_X86_64_CODE_4_GOTTPOFF must be used in MOVQ or ADDQ "
                   "instructions only";
     }
+  } else if (rel.type == R_X86_64_CODE_6_GOTTPOFF) {
+    if (loc[-6] != 0x62) {
+      Err(ctx) << getErrorLoc(ctx, loc - 6)
+               << "invalid prefix with R_X86_64_CODE_6_GOTTPOFF!";
+      return;
+    }
+    // Check bits are satisfied:
+    //   loc[-5]: X==1 (inverted polarity), (loc[-5] & 0x7) == 0x4
+    //   loc[-4]: W==1, X2==1 (inverted polarity), pp==0b00(NP)
+    //   loc[-3]: NF==1 or ND==1
+    //   loc[-2]: opcode==0x1 or opcode==0x3
+    //   loc[-1]: Mod==0b00, RM==0b101
+    if (((loc[-5] & 0x47) == 0x44) && ((loc[-4] & 0x87) == 0x84) &&
+        ((loc[-3] & 0x14) != 0) && (loc[-2] == 0x1 || loc[-2] == 0x3) &&
+        ((loc[-1] & 0xc7) == 0x5)) {
+      // "addq %reg1, foo@GOTTPOFF(%rip), %reg2" -> "addq $foo, %reg1, %reg2"
+      // "addq foo@GOTTPOFF(%rip), %reg1, %reg2" -> "addq $foo, %reg1, %reg2"
+      // "{nf} addq %reg1, foo@GOTTPOFF(%rip), %reg2"
+      //   -> "{nf} addq $foo, %reg1, %reg2"
+      // "{nf} addq name@GOTTPOFF(%rip), %reg1, %reg2"
+      //    -> "{nf} addq $foo, %reg1, %reg2"
+      // "{nf} addq name@GOTTPOFF(%rip), %reg" -> "{nf} addq $foo, %reg"
+      loc[-2] = 0x81;
+      // Move R bits to B bits in EVEX payloads and ModRM byte.
+      const uint8_t evexPayload0 = loc[-5];
+      if ((evexPayload0 & (1 << 7)) == 0)
+        loc[-5] = (evexPayload0 | (1 << 7)) & ~(1 << 5);
+      if ((evexPayload0 & (1 << 4)) == 0)
+        loc[-5] = evexPayload0 | (1 << 4) | (1 << 3);
+      *regSlot = 0xc0 | reg;
+    } else {
+      Err(ctx) << getErrorLoc(ctx, loc - 6)
+               << "R_X86_64_CODE_6_GOTTPOFF must be used in ADDQ instructions "
+                  "with NDD/NF/NDD+NF only";
+    }
   } else {
     llvm_unreachable("Unsupported relocation type!");
   }
@@ -782,6 +819,7 @@ int64_t X86_64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_X86_64_PC32:
   case R_X86_64_GOTTPOFF:
   case R_X86_64_CODE_4_GOTTPOFF:
+  case R_X86_64_CODE_6_GOTTPOFF:
   case R_X86_64_PLT32:
   case R_X86_64_TLSGD:
   case R_X86_64_TLSLD:
@@ -893,6 +931,7 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     break;
   case R_X86_64_GOTTPOFF:
   case R_X86_64_CODE_4_GOTTPOFF:
+  case R_X86_64_CODE_6_GOTTPOFF:
     if (rel.expr == R_RELAX_TLS_IE_TO_LE) {
       relaxTlsIeToLe(loc, rel, val);
     } else {
@@ -1044,7 +1083,7 @@ static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) {
   if (op != 0xff) {
     // We are relaxing a rip relative to an absolute, so compensate
     // for the old -4 addend.
-    assert(!rel.sym->file || !rel.sym->file->ctx.arg.isPic);
+    assert(!rel.sym->file->ctx.arg.isPic);
     relaxGotNoPic(loc, val + 4, op, modRm,
                   rel.type == R_X86_64_CODE_4_GOTPCRELX);
     return;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index bc4b967ccbbbb..3c553e5043180 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2452,7 +2452,7 @@ static void readSymbolPartitionSection(Ctx &ctx, InputSectionBase *s) {
     sym = readEntry(s->file, rels.rels);
   else
     sym = readEntry(s->file, rels.relas);
-  if (!isa_and_nonnull<Defined>(sym) || !sym->includeInDynsym(ctx))
+  if (!isa_and_nonnull<Defined>(sym) || !sym->isExported)
     return;
 
   StringRef partName = reinterpret_cast<const char *>(s->content().data());
@@ -2987,6 +2987,8 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   if (!ctx.arg.relocatable) {
     llvm::TimeTraceScope timeScope("Process symbol versions");
     ctx.symtab->scanVersionScript();
+
+    parseVersionAndComputeIsPreemptible(ctx);
   }
 
   // Skip the normal linked output if some LTO options are specified.
diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
index 606953e94bbad..1cdcf6be9d8a9 100644
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -459,20 +459,8 @@ static void combineRelocHashes(unsigned cnt, InputSection *isec,
   isec->eqClass[(cnt + 1) % 2] = hash | (1U << 31);
 }
 
-static void print(Ctx &ctx, const Twine &s) {
-  if (ctx.arg.printIcfSections)
-    Msg(ctx) << s;
-}
-
 // The main function of ICF.
 template <class ELFT> void ICF<ELFT>::run() {
-  // Compute isPreemptible early. We may add more symbols later, so this loop
-  // cannot be merged with the later computeIsPreemptible() pass which is used
-  // by scanRelocations().
-  if (ctx.arg.hasDynSymTab)
-    for (Symbol *sym : ctx.symtab->getSymbols())
-      sym->isPreemptible = computeIsPreemptible(ctx, *sym);
-
   // Two text sections may have identical content and relocations but different
   // LSDA, e.g. the two functions may have catch blocks of different types. If a
   // text section is referenced by a .eh_frame FDE with LSDA, it is not
@@ -544,13 +532,16 @@ template <class ELFT> void ICF<ELFT>::run() {
 
   Log(ctx) << "ICF needed " << cnt << " iterations";
 
+  auto print = [&ctx = ctx]() -> ELFSyncStream {
+    return {ctx, ctx.arg.printIcfSections ? DiagLevel::Msg : DiagLevel::None};
+  };
   // Merge sections by the equivalence class.
   forEachClassRange(0, sections.size(), [&](size_t begin, size_t end) {
     if (end - begin == 1)
       return;
-    print(ctx, "selected section " + toStr(ctx, sections[begin]));
+    print() << "selected section " << sections[begin];
     for (size_t i = begin + 1; i < end; ++i) {
-      print(ctx, "  removing identical section " + toStr(ctx, sections[i]));
+      print() << "  removing identical section " << sections[i];
       sections[begin]->replace(sections[i]);
 
       // At this point we know sections merged are fully identical and hence
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 83a25e1b66cff..2084fcfd4d651 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -367,49 +367,6 @@ void elf::parseFiles(Ctx &ctx,
 }
 
 // Concatenates arguments to construct a string representing an error location.
-static std::string createFileLineMsg(StringRef path, unsigned line) {
-  std::string filename = std::string(path::filename(path));
-  std::string lineno = ":" + std::to_string(line);
-  if (filename == path)
-    return filename + lineno;
-  return filename + lineno + " (" + path.str() + lineno + ")";
-}
-
-template <class ELFT>
-static std::string getSrcMsgAux(ObjFile<ELFT> &file, const Symbol &sym,
-                                const InputSectionBase &sec, uint64_t offset) {
-  // In DWARF, functions and variables are stored to different places.
-  // First, look up a function for a given offset.
-  if (std::optional<DILineInfo> info = file.getDILineInfo(&sec, offset))
-    return createFileLineMsg(info->FileName, info->Line);
-
-  // If it failed, look up again as a variable.
-  if (std::optional<std::pair<std::string, unsigned>> fileLine =
-          file.getVariableLoc(sym.getName()))
-    return createFileLineMsg(fileLine->first, fileLine->second);
-
-  // File.sourceFile contains STT_FILE symbol, and that is a last resort.
-  return std::string(file.sourceFile);
-}
-
-std::string InputFile::getSrcMsg(const Symbol &sym, const InputSectionBase &sec,
-                                 uint64_t offset) {
-  if (kind() != ObjKind)
-    return "";
-  switch (ekind) {
-  default:
-    llvm_unreachable("Invalid kind");
-  case ELF32LEKind:
-    return getSrcMsgAux(cast<ObjFile<ELF32LE>>(*this), sym, sec, offset);
-  case ELF32BEKind:
-    return getSrcMsgAux(cast<ObjFile<ELF32BE>>(*this), sym, sec, offset);
-  case ELF64LEKind:
-    return getSrcMsgAux(cast<ObjFile<ELF64LE>>(*this), sym, sec, offset);
-  case ELF64BEKind:
-    return getSrcMsgAux(cast<ObjFile<ELF64BE>>(*this), sym, sec, offset);
-  }
-}
-
 StringRef InputFile::getNameForScript() const {
   if (archiveName.empty())
     return getName();
@@ -480,43 +437,32 @@ static void handleSectionGroup(ArrayRef<InputSectionBase *> sections,
     prev->nextInSectionGroup = head;
 }
 
-template <class ELFT> DWARFCache *ObjFile<ELFT>::getDwarf() {
-  llvm::call_once(initDwarf, [this]() {
-    dwarf = std::make_unique<DWARFCache>(std::make_unique<DWARFContext>(
-        std::make_unique<LLDDwarfObj<ELFT>>(this), "",
-        [&](Error err) { Warn(ctx) << getName() + ": " << std::move(err); },
-        [&](Error warning) {
-          Warn(ctx) << getName() << ": " << std::move(warning);
-        }));
-  });
-
-  return dwarf.get();
-}
-
-// Returns the pair of file name and line number describing location of data
-// object (variable, array, etc) definition.
-template <class ELFT>
-std::optional<std::pair<std::string, unsigned>>
-ObjFile<ELFT>::getVariableLoc(StringRef name) {
-  return getDwarf()->getVariableLoc(name);
+template <class ELFT> void ObjFile<ELFT>::initDwarf() {
+  dwarf = std::make_unique<DWARFCache>(std::make_unique<DWARFContext>(
+      std::make_unique<LLDDwarfObj<ELFT>>(this), "",
+      [&](Error err) { Warn(ctx) << getName() + ": " << std::move(err); },
+      [&](Error warning) {
+        Warn(ctx) << getName() << ": " << std::move(warning);
+      }));
 }
 
-// Returns source line information for a given offset
-// using DWARF debug info.
-template <class ELFT>
-std::optional<DILineInfo>
-ObjFile<ELFT>::getDILineInfo(const InputSectionBase *s, uint64_t offset) {
-  // Detect SectionIndex for specified section.
-  uint64_t sectionIndex = object::SectionedAddress::UndefSection;
-  ArrayRef<InputSectionBase *> sections = s->file->getSections();
-  for (uint64_t curIndex = 0; curIndex < sections.size(); ++curIndex) {
-    if (s == sections[curIndex]) {
-      sectionIndex = curIndex;
-      break;
+DWARFCache *ELFFileBase::getDwarf() {
+  assert(fileKind == ObjKind);
+  llvm::call_once(initDwarf, [this]() {
+    switch (ekind) {
+    default:
+      llvm_unreachable("");
+    case ELF32LEKind:
+      return cast<ObjFile<ELF32LE>>(this)->initDwarf();
+    case ELF32BEKind:
+      return cast<ObjFile<ELF32BE>>(this)->initDwarf();
+    case ELF64LEKind:
+      return cast<ObjFile<ELF64LE>>(this)->initDwarf();
+    case ELF64BEKind:
+      return cast<ObjFile<ELF64BE>>(this)->initDwarf();
     }
-  }
-
-  return getDwarf()->getDILineInfo(offset, sectionIndex);
+  });
+  return dwarf.get();
 }
 
 ELFFileBase::ELFFileBase(Ctx &ctx, Kind k, ELFKind ekind, MemoryBufferRef mb)
@@ -524,6 +470,8 @@ ELFFileBase::ELFFileBase(Ctx &ctx, Kind k, ELFKind ekind, MemoryBufferRef mb)
   this->ekind = ekind;
 }
 
+ELFFileBase::~ELFFileBase() {}
+
 template <typename Elf_Shdr>
 static const Elf_Shdr *findSection(ArrayRef<Elf_Shdr> sections, uint32_t type) {
   for (const Elf_Shdr &sec : sections)
@@ -580,7 +528,7 @@ template <class ELFT> void ELFFileBase::init(InputFile::Kind k) {
     Fatal(ctx) << this << ": invalid sh_info in symbol table";
 
   elfSyms = reinterpret_cast<const void *>(eSyms.data());
-  numELFSyms = uint32_t(eSyms.size());
+  numSymbols = eSyms.size();
   stringTable = CHECK2(obj.getStringTableForSymtab(*symtabSec, sections), this);
 }
 
@@ -608,12 +556,45 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
   sections.resize(size);
   for (size_t i = 0; i != size; ++i) {
     const Elf_Shdr &sec = objSections[i];
+    if (LLVM_LIKELY(sec.sh_type == SHT_PROGBITS))
+      continue;
+    if (LLVM_LIKELY(sec.sh_type == SHT_GROUP)) {
+      StringRef signature = getShtGroupSignature(objSections, sec);
+      ArrayRef<Elf_Word> entries =
+          CHECK2(obj.template getSectionContentsAsArray<Elf_Word>(sec), this);
+      if (entries.empty())
+        Fatal(ctx) << this << ": empty SHT_GROUP";
+
+      Elf_Word flag = entries[0];
+      if (flag && flag != GRP_COMDAT)
+        Fatal(ctx) << this << ": unsupported SHT_GROUP format";
+
+      bool keepGroup = !flag || ignoreComdats ||
+                       ctx.symtab->comdatGroups
+                           .try_emplace(CachedHashStringRef(signature), this)
+                           .second;
+      if (keepGroup) {
+        if (!ctx.arg.resolveGroups)
+          sections[i] = createInputSection(
+              i, sec, check(obj.getSectionName(sec, shstrtab)));
+      } else {
+        // Otherwise, discard group members.
+        for (uint32_t secIndex : entries.slice(1)) {
+          if (secIndex >= size)
+            Fatal(ctx) << this
+                       << ": invalid section index in group: " << secIndex;
+          sections[secIndex] = &InputSection::discarded;
+        }
+      }
+      continue;
+    }
+
     if (sec.sh_type == SHT_LLVM_DEPENDENT_LIBRARIES && !ctx.arg.relocatable) {
       StringRef name = check(obj.getSectionName(sec, shstrtab));
       ArrayRef<char> data = CHECK2(
           this->getObj().template getSectionContentsAsArray<char>(sec), this);
       if (!data.empty() && data.back() != '\0') {
-        ErrAlways(ctx)
+        Err(ctx)
             << this
             << ": corrupted dependent libraries section (unterminated string): "
             << name;
@@ -624,74 +605,48 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
           d += s.size() + 1;
         }
       }
-      this->sections[i] = &InputSection::discarded;
+      sections[i] = &InputSection::discarded;
       continue;
     }
 
-    if (sec.sh_type == SHT_ARM_ATTRIBUTES && ctx.arg.emachine == EM_ARM) {
-      ARMAttributeParser attributes;
-      ArrayRef<uint8_t> contents =
-          check(this->getObj().getSectionContents(sec));
-      StringRef name = check(obj.getSectionName(sec, shstrtab));
-      this->sections[i] = &InputSection::discarded;
-      if (Error e = attributes.parse(contents, ekind == ELF32LEKind
-                                                   ? llvm::endianness::little
-                                                   : llvm::endianness::big)) {
-        InputSection isec(*this, sec, name);
-        Warn(ctx) << &isec << ": " << std::move(e);
-      } else {
-        updateSupportedARMFeatures(ctx, attributes);
-        updateARMVFPArgs(ctx, attributes, this);
-
-        // FIXME: Retain the first attribute section we see. The eglibc ARM
-        // dynamic loaders require the presence of an attribute section for
-        // dlopen to work. In a full implementation we would merge all attribute
-        // sections.
-        if (ctx.in.attributes == nullptr) {
-          ctx.in.attributes = std::make_unique<InputSection>(*this, sec, name);
-          this->sections[i] = ctx.in.attributes.get();
+    switch (ctx.arg.emachine) {
+    case EM_ARM:
+      if (sec.sh_type == SHT_ARM_ATTRIBUTES) {
+        ARMAttributeParser attributes;
+        ArrayRef<uint8_t> contents =
+            check(this->getObj().getSectionContents(sec));
+        StringRef name = check(obj.getSectionName(sec, shstrtab));
+        sections[i] = &InputSection::discarded;
+        if (Error e = attributes.parse(contents, ekind == ELF32LEKind
+                                                     ? llvm::endianness::little
+                                                     : llvm::endianness::big)) {
+          InputSection isec(*this, sec, name);
+          Warn(ctx) << &isec << ": " << std::move(e);
+        } else {
+          updateSupportedARMFeatures(ctx, attributes);
+          updateARMVFPArgs(ctx, attributes, this);
+
+          // FIXME: Retain the first attribute section we see. The eglibc ARM
+          // dynamic loaders require the presence of an attribute section for
+          // dlopen to work. In a full implementation we would merge all
+          // attribute sections.
+          if (ctx.in.attributes == nullptr) {
+            ctx.in.attributes =
+                std::make_unique<InputSection>(*this, sec, name);
+            sections[i] = ctx.in.attributes.get();
+          }
         }
       }
-    }
-
-    // Producing a static binary with MTE globals is not currently supported,
-    // remove all SHT_AARCH64_MEMTAG_GLOBALS_STATIC sections as they're unused
-    // medatada, and we don't want them to end up in the output file for static
-    // executables.
-    if (sec.sh_type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC &&
-        !canHaveMemtagGlobals(ctx)) {
-      this->sections[i] = &InputSection::discarded;
-      continue;
-    }
-
-    if (sec.sh_type != SHT_GROUP)
-      continue;
-    StringRef signature = getShtGroupSignature(objSections, sec);
-    ArrayRef<Elf_Word> entries =
-        CHECK2(obj.template getSectionContentsAsArray<Elf_Word>(sec), this);
-    if (entries.empty())
-      Fatal(ctx) << this << ": empty SHT_GROUP";
-
-    Elf_Word flag = entries[0];
-    if (flag && flag != GRP_COMDAT)
-      Fatal(ctx) << this << ": unsupported SHT_GROUP format";
-
-    bool keepGroup = (flag & GRP_COMDAT) == 0 || ignoreComdats ||
-                     ctx.symtab->comdatGroups
-                         .try_emplace(CachedHashStringRef(signature), this)
-                         .second;
-    if (keepGroup) {
-      if (!ctx.arg.resolveGroups)
-        this->sections[i] = createInputSection(
-            i, sec, check(obj.getSectionName(sec, shstrtab)));
-      continue;
-    }
-
-    // Otherwise, discard group members.
-    for (uint32_t secIndex : entries.slice(1)) {
-      if (secIndex >= size)
-        Fatal(ctx) << this << ": invalid section index in group: " << secIndex;
-      this->sections[secIndex] = &InputSection::discarded;
+      break;
+    case EM_AARCH64:
+      // Producing a static binary with MTE globals is not currently supported,
+      // remove all SHT_AARCH64_MEMTAG_GLOBALS_STATIC sections as they're unused
+      // medatada, and we don't want them to end up in the output file for
+      // static executables.
+      if (sec.sh_type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC &&
+          !canHaveMemtagGlobals(ctx))
+        sections[i] = &InputSection::discarded;
+      break;
     }
   }
 
@@ -1134,10 +1089,8 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
 template <class ELFT>
 void ObjFile<ELFT>::initializeSymbols(const object::ELFFile<ELFT> &obj) {
   ArrayRef<Elf_Sym> eSyms = this->getELFSyms<ELFT>();
-  if (numSymbols == 0) {
-    numSymbols = eSyms.size();
+  if (!symbols)
     symbols = std::make_unique<Symbol *[]>(numSymbols);
-  }
 
   // Some entries have been filled by LazyObjFile.
   auto *symtab = ctx.symtab.get();
@@ -1264,14 +1217,17 @@ template <class ELFT> void ObjFile<ELFT>::postParse() {
 
     // Handle non-COMMON defined symbol below. !sym.file allows a symbol
     // assignment to redefine a symbol without an error.
-    if (!sym.file || !sym.isDefined() || secIdx == SHN_UNDEF ||
-        secIdx == SHN_COMMON)
+    if (!sym.isDefined() || secIdx == SHN_UNDEF)
       continue;
+    if (LLVM_UNLIKELY(secIdx >= SHN_LORESERVE)) {
+      if (secIdx == SHN_COMMON)
+        continue;
+      if (secIdx == SHN_XINDEX)
+        secIdx = check(getExtendedSymbolTableIndex<ELFT>(eSym, i, shndxTable));
+      else
+        secIdx = 0;
+    }
 
-    if (LLVM_UNLIKELY(secIdx == SHN_XINDEX))
-      secIdx = check(getExtendedSymbolTableIndex<ELFT>(eSym, i, shndxTable));
-    else if (secIdx >= SHN_LORESERVE)
-      secIdx = 0;
     if (LLVM_UNLIKELY(secIdx >= sections.size()))
       Fatal(ctx) << this << ": invalid section index: " << secIdx;
     InputSectionBase *sec = sections[secIdx];
@@ -1473,6 +1429,7 @@ template <class ELFT> void SharedFile::parse() {
   const Elf_Shdr *versymSec = nullptr;
   const Elf_Shdr *verdefSec = nullptr;
   const Elf_Shdr *verneedSec = nullptr;
+  symbols = std::make_unique<Symbol *[]>(numSymbols);
 
   // Search for .dynsym, .dynamic, .symtab, .gnu.version and .gnu.version_d.
   for (const Elf_Shdr &sec : sections) {
@@ -1495,7 +1452,7 @@ template <class ELFT> void SharedFile::parse() {
     }
   }
 
-  if (versymSec && numELFSyms == 0) {
+  if (versymSec && numSymbols == 0) {
     ErrAlways(ctx) << "SHT_GNU_versym should be associated with symbol table";
     return;
   }
@@ -1538,7 +1495,7 @@ template <class ELFT> void SharedFile::parse() {
   // Parse ".gnu.version" section which is a parallel array for the symbol
   // table. If a given file doesn't have a ".gnu.version" section, we use
   // VER_NDX_GLOBAL.
-  size_t size = numELFSyms - firstGlobal;
+  size_t size = numSymbols - firstGlobal;
   std::vector<uint16_t> versyms(size, VER_NDX_GLOBAL);
   if (versymSec) {
     ArrayRef<Elf_Versym> versym =
@@ -1784,8 +1741,11 @@ static void createBitcodeSymbol(Ctx &ctx, Symbol *&sym,
   } else {
     Defined newSym(ctx, &f, StringRef(), binding, visibility, type, 0, 0,
                    nullptr);
-    if (objSym.canBeOmittedFromSymbolTable())
-      newSym.exportDynamic = false;
+    // The definition can be omitted if all bitcode definitions satisfy
+    // `canBeOmittedFromSymbolTable()` and isUsedInRegularObj is false.
+    // The latter condition is tested in Symbol::includeInDynsym.
+    sym->ltoCanOmit = objSym.canBeOmittedFromSymbolTable() &&
+                      (!sym->isDefined() || sym->ltoCanOmit);
     sym->resolve(ctx, newSym);
   }
 }
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 9161fa13ad983..0b186db1ba0d1 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -53,7 +53,7 @@ class InputFile {
 
 protected:
   std::unique_ptr<Symbol *[]> symbols;
-  uint32_t numSymbols = 0;
+  size_t numSymbols = 0;
   SmallVector<InputSectionBase *, 0> sections;
 
 public:
@@ -147,9 +147,6 @@ class InputFile {
   // True if this is an argument for --just-symbols. Usually false.
   bool justSymbols = false;
 
-  std::string getSrcMsg(const Symbol &sym, const InputSectionBase &sec,
-                        uint64_t offset);
-
   // On PPC64 we need to keep track of which files contain small code model
   // relocations that access the .toc section. To minimize the chance of a
   // relocation overflow, files that do contain said relocations should have
@@ -181,6 +178,7 @@ class InputFile {
 class ELFFileBase : public InputFile {
 public:
   ELFFileBase(Ctx &ctx, Kind k, ELFKind ekind, MemoryBufferRef m);
+  ~ELFFileBase();
   static bool classof(const InputFile *f) { return f->isElf(); }
 
   void init();
@@ -210,12 +208,15 @@ class ELFFileBase : public InputFile {
   }
   template <typename ELFT> typename ELFT::SymRange getELFSyms() const {
     return typename ELFT::SymRange(
-        reinterpret_cast<const typename ELFT::Sym *>(elfSyms), numELFSyms);
+        reinterpret_cast<const typename ELFT::Sym *>(elfSyms), numSymbols);
   }
   template <typename ELFT> typename ELFT::SymRange getGlobalELFSyms() const {
     return getELFSyms<ELFT>().slice(firstGlobal);
   }
 
+  // Get cached DWARF information.
+  DWARFCache *getDwarf();
+
 protected:
   // Initializes this class's member variables.
   template <typename ELFT> void init(InputFile::Kind k);
@@ -224,10 +225,20 @@ class ELFFileBase : public InputFile {
   const void *elfShdrs = nullptr;
   const void *elfSyms = nullptr;
   uint32_t numELFShdrs = 0;
-  uint32_t numELFSyms = 0;
   uint32_t firstGlobal = 0;
 
+  // Below are ObjFile specific members.
+
+  // Debugging information to retrieve source file and line for error
+  // reporting. Linker may find reasonable number of errors in a
+  // single object file, so we cache debugging information in order to
+  // parse it only once for each object file we link.
+  llvm::once_flag initDwarf;
+  std::unique_ptr<DWARFCache> dwarf;
+
 public:
+  // Name of source file obtained from STT_FILE, if present.
+  StringRef sourceFile;
   uint32_t andFeatures = 0;
   bool hasCommonSyms = false;
   ArrayRef<uint8_t> aarch64PauthAbiCoreInfo;
@@ -257,15 +268,6 @@ template <class ELFT> class ObjFile : public ELFFileBase {
 
   uint32_t getSectionIndex(const Elf_Sym &sym) const;
 
-  std::optional<llvm::DILineInfo> getDILineInfo(const InputSectionBase *,
-                                                uint64_t);
-  std::optional<std::pair<std::string, unsigned>>
-  getVariableLoc(StringRef name);
-
-  // Name of source file obtained from STT_FILE symbol value,
-  // or empty string if there is no such symbol in object file
-  // symbol table.
-  StringRef sourceFile;
 
   // Pointer to this input file's .llvm_addrsig section, if it has one.
   const Elf_Shdr *addrsigSec = nullptr;
@@ -286,8 +288,7 @@ template <class ELFT> class ObjFile : public ELFFileBase {
   // but had one or more functions with the no_split_stack attribute.
   bool someNoSplitStack = false;
 
-  // Get cached DWARF information.
-  DWARFCache *getDwarf();
+  void initDwarf();
 
   void initSectionsAndLocalSyms(bool ignoreComdats);
   void postParse();
@@ -318,13 +319,6 @@ template <class ELFT> class ObjFile : public ELFFileBase {
   // The following variable contains the contents of .symtab_shndx.
   // If the section does not exist (which is common), the array is empty.
   ArrayRef<Elf_Word> shndxTable;
-
-  // Debugging information to retrieve source file and line for error
-  // reporting. Linker may find reasonable number of errors in a
-  // single object file, so we cache debugging information in order to
-  // parse it only once for each object file we link.
-  std::unique_ptr<DWARFCache> dwarf;
-  llvm::once_flag initDwarf;
 };
 
 class BitcodeFile : public InputFile {
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 75121285b7b23..7e76bae19fc6a 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -16,6 +16,7 @@
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "lld/Common/CommonLinkerContext.h"
+#include "lld/Common/DWARF.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
@@ -127,15 +128,9 @@ static void decompressAux(Ctx &ctx, const InputSectionBase &sec, uint8_t *out,
 
 void InputSectionBase::decompress() const {
   Ctx &ctx = getCtx();
-  uint8_t *uncompressedBuf;
-  {
-    static std::mutex mu;
-    std::lock_guard<std::mutex> lock(mu);
-    uncompressedBuf = ctx.bAlloc.Allocate<uint8_t>(size);
-  }
-
-  invokeELFT(decompressAux, ctx, *this, uncompressedBuf, size);
-  content_ = uncompressedBuf;
+  uint8_t *buf = makeThreadLocalN<uint8_t>(size);
+  invokeELFT(decompressAux, ctx, *this, buf, size);
+  content_ = buf;
   compressed = false;
 }
 
@@ -316,15 +311,40 @@ std::string InputSectionBase::getLocation(uint64_t offset) const {
   return filename + ":(" + secAndOffset;
 }
 
-// This function is intended to be used for constructing an error message.
-// The returned message looks like this:
+static void printFileLine(const ELFSyncStream &s, StringRef path,
+                          unsigned line) {
+  StringRef filename = path::filename(path);
+  s << filename << ':' << line;
+  if (filename != path)
+    s << " (" << path << ':' << line << ')';
+}
+
+// Print an error message that looks like this:
 //
 //   foo.c:42 (/home/alice/possibly/very/long/path/foo.c:42)
-//
-//  Returns an empty string if there's no way to get line info.
-std::string InputSectionBase::getSrcMsg(const Symbol &sym,
-                                        uint64_t offset) const {
-  return file->getSrcMsg(sym, *this, offset);
+const ELFSyncStream &elf::operator<<(const ELFSyncStream &s,
+                                     InputSectionBase::SrcMsg &&msg) {
+  auto &sec = msg.sec;
+  if (sec.file->kind() != InputFile::ObjKind)
+    return s;
+  auto &file = cast<ELFFileBase>(*sec.file);
+
+  // First, look up the DWARF line table.
+  ArrayRef<InputSectionBase *> sections = file.getSections();
+  auto it = llvm::find(sections, &sec);
+  uint64_t sectionIndex = it != sections.end()
+                              ? it - sections.begin()
+                              : object::SectionedAddress::UndefSection;
+  DWARFCache *dwarf = file.getDwarf();
+  if (auto info = dwarf->getDILineInfo(msg.offset, sectionIndex))
+    printFileLine(s, info->FileName, info->Line);
+  else if (auto fileLine = dwarf->getVariableLoc(msg.sym.getName()))
+    // If it failed, look up again as a variable.
+    printFileLine(s, fileLine->first, fileLine->second);
+  else
+    // File.sourceFile contains STT_FILE symbol, and that is a last resort.
+    s << file.sourceFile;
+  return s;
 }
 
 // Returns a filename string along with an optional section name. This
@@ -336,22 +356,22 @@ std::string InputSectionBase::getSrcMsg(const Symbol &sym,
 // or
 //
 //   path/to/foo.o:(function bar) in archive path/to/bar.a
-std::string InputSectionBase::getObjMsg(uint64_t off) const {
-  std::string filename = std::string(file->getName());
-
-  std::string archive;
-  if (!file->archiveName.empty())
-    archive = (" in archive " + file->archiveName).str();
+const ELFSyncStream &elf::operator<<(const ELFSyncStream &s,
+                                     InputSectionBase::ObjMsg &&msg) {
+  auto *sec = msg.sec;
+  s << sec->file->getName() << ":(";
 
   // Find a symbol that encloses a given location. getObjMsg may be called
   // before ObjFile::initSectionsAndLocalSyms where local symbols are
   // initialized.
-  if (Defined *d = getEnclosingSymbol(off))
-    return filename + ":(" + toStr(getCtx(), *d) + ")" + archive;
-
-  // If there's no symbol, print out the offset in the section.
-  return (filename + ":(" + name + "+0x" + utohexstr(off) + ")" + archive)
-      .str();
+  if (Defined *d = sec->getEnclosingSymbol(msg.offset))
+    s << d;
+  else
+    s << sec->name << "+0x" << Twine::utohexstr(msg.offset);
+  s << ')';
+  if (!sec->file->archiveName.empty())
+    s << (" in archive " + sec->file->archiveName).str();
+  return s;
 }
 
 PotentialSpillSection::PotentialSpillSection(const InputSectionBase &source,
@@ -503,7 +523,7 @@ void InputSection::copyRelocations(Ctx &ctx, uint8_t *buf,
         addend = target.getImplicitAddend(bufLoc, type);
 
       if (ctx.arg.emachine == EM_MIPS &&
-          target.getRelExpr(type, sym, bufLoc) == R_MIPS_GOTREL) {
+          target.getRelExpr(type, sym, bufLoc) == RE_MIPS_GOTREL) {
         // Some MIPS relocations depend on "gp" value. By default,
         // this value has 0x7ff0 offset from a .got section. But
         // relocatable files produced by a compiler or a linker
@@ -635,7 +655,7 @@ static uint64_t getARMStaticBase(const Symbol &sym) {
   return os->ptLoad->firstSec->addr;
 }
 
-// For R_RISCV_PC_INDIRECT (R_RISCV_PCREL_LO12_{I,S}), the symbol actually
+// For RE_RISCV_PC_INDIRECT (R_RISCV_PCREL_LO12_{I,S}), the symbol actually
 // points the corresponding R_RISCV_PCREL_HI20 relocation, and the target VA
 // is calculated using PCREL_HI20's symbol.
 //
@@ -752,25 +772,25 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_DTPREL:
   case R_RELAX_TLS_LD_TO_LE_ABS:
   case R_RELAX_GOT_PC_NOPIC:
-  case R_AARCH64_AUTH:
-  case R_RISCV_ADD:
-  case R_RISCV_LEB128:
+  case RE_AARCH64_AUTH:
+  case RE_RISCV_ADD:
+  case RE_RISCV_LEB128:
     return r.sym->getVA(ctx, a);
   case R_ADDEND:
     return a;
   case R_RELAX_HINT:
     return 0;
-  case R_ARM_SBREL:
+  case RE_ARM_SBREL:
     return r.sym->getVA(ctx, a) - getARMStaticBase(*r.sym);
   case R_GOT:
   case R_RELAX_TLS_GD_TO_IE_ABS:
     return r.sym->getGotVA(ctx) + a;
-  case R_LOONGARCH_GOT:
+  case RE_LOONGARCH_GOT:
     // The LoongArch TLS GD relocs reuse the R_LARCH_GOT_PC_LO12 reloc r.type
     // for their page offsets. The arithmetics are different in the TLS case
     // so we have to duplicate some logic here.
     if (r.sym->hasFlag(NEEDS_TLSGD) && r.type != R_LARCH_TLS_IE_PC_LO12)
-      // Like R_LOONGARCH_TLSGD_PAGE_PC but taking the absolute value.
+      // Like RE_LOONGARCH_TLSGD_PAGE_PC but taking the absolute value.
       return ctx.in.got->getGlobalDynAddr(*r.sym) + a;
     return r.sym->getGotVA(ctx) + a;
   case R_GOTONLY_PC:
@@ -778,7 +798,7 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_GOTPLTONLY_PC:
     return ctx.in.gotPlt->getVA() + a - p;
   case R_GOTREL:
-  case R_PPC64_RELAX_TOC:
+  case RE_PPC64_RELAX_TOC:
     return r.sym->getVA(ctx, a) - ctx.in.got->getVA();
   case R_GOTPLTREL:
     return r.sym->getVA(ctx, a) - ctx.in.gotPlt->getVA();
@@ -789,10 +809,10 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_GOT_OFF:
   case R_RELAX_TLS_GD_TO_IE_GOT_OFF:
     return r.sym->getGotOffset(ctx) + a;
-  case R_AARCH64_GOT_PAGE_PC:
-  case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
+  case RE_AARCH64_GOT_PAGE_PC:
+  case RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
     return getAArch64Page(r.sym->getGotVA(ctx) + a) - getAArch64Page(p);
-  case R_AARCH64_GOT_PAGE:
+  case RE_AARCH64_GOT_PAGE:
     return r.sym->getGotVA(ctx) + a - getAArch64Page(ctx.in.got->getVA());
   case R_GOT_PC:
   case R_RELAX_TLS_GD_TO_IE:
@@ -801,17 +821,17 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
     return r.sym->getGotPltVA(ctx) + a - ctx.in.got->getVA();
   case R_GOTPLT_PC:
     return r.sym->getGotPltVA(ctx) + a - p;
-  case R_LOONGARCH_GOT_PAGE_PC:
+  case RE_LOONGARCH_GOT_PAGE_PC:
     if (r.sym->hasFlag(NEEDS_TLSGD))
       return getLoongArchPageDelta(ctx.in.got->getGlobalDynAddr(*r.sym) + a, p,
                                    r.type);
     return getLoongArchPageDelta(r.sym->getGotVA(ctx) + a, p, r.type);
-  case R_MIPS_GOTREL:
+  case RE_MIPS_GOTREL:
     return r.sym->getVA(ctx, a) - ctx.in.mipsGot->getGp(file);
-  case R_MIPS_GOT_GP:
+  case RE_MIPS_GOT_GP:
     return ctx.in.mipsGot->getGp(file) + a;
-  case R_MIPS_GOT_GP_PC: {
-    // R_MIPS_LO16 expression has R_MIPS_GOT_GP_PC r.type iif the target
+  case RE_MIPS_GOT_GP_PC: {
+    // R_MIPS_LO16 expression has RE_MIPS_GOT_GP_PC r.type iif the target
     // is _gp_disp symbol. In that case we should use the following
     // formula for calculation "AHL + GP - P + 4". For details see p. 4-19 at
     // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
@@ -825,43 +845,43 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
       v -= 1;
     return v;
   }
-  case R_MIPS_GOT_LOCAL_PAGE:
+  case RE_MIPS_GOT_LOCAL_PAGE:
     // If relocation against MIPS local symbol requires GOT entry, this entry
     // should be initialized by 'page address'. This address is high 16-bits
     // of sum the symbol's value and the addend.
     return ctx.in.mipsGot->getVA() +
            ctx.in.mipsGot->getPageEntryOffset(file, *r.sym, a) -
            ctx.in.mipsGot->getGp(file);
-  case R_MIPS_GOT_OFF:
-  case R_MIPS_GOT_OFF32:
+  case RE_MIPS_GOT_OFF:
+  case RE_MIPS_GOT_OFF32:
     // In case of MIPS if a GOT relocation has non-zero addend this addend
     // should be applied to the GOT entry content not to the GOT entry offset.
     // That is why we use separate expression r.type.
     return ctx.in.mipsGot->getVA() +
            ctx.in.mipsGot->getSymEntryOffset(file, *r.sym, a) -
            ctx.in.mipsGot->getGp(file);
-  case R_MIPS_TLSGD:
+  case RE_MIPS_TLSGD:
     return ctx.in.mipsGot->getVA() +
            ctx.in.mipsGot->getGlobalDynOffset(file, *r.sym) -
            ctx.in.mipsGot->getGp(file);
-  case R_MIPS_TLSLD:
+  case RE_MIPS_TLSLD:
     return ctx.in.mipsGot->getVA() + ctx.in.mipsGot->getTlsIndexOffset(file) -
            ctx.in.mipsGot->getGp(file);
-  case R_AARCH64_PAGE_PC: {
+  case RE_AARCH64_PAGE_PC: {
     uint64_t val = r.sym->isUndefWeak() ? p + a : r.sym->getVA(ctx, a);
     return getAArch64Page(val) - getAArch64Page(p);
   }
-  case R_RISCV_PC_INDIRECT: {
+  case RE_RISCV_PC_INDIRECT: {
     if (const Relocation *hiRel = getRISCVPCRelHi20(ctx, this, r))
       return getRelocTargetVA(ctx, *hiRel, r.sym->getVA(ctx));
     return 0;
   }
-  case R_LOONGARCH_PAGE_PC:
+  case RE_LOONGARCH_PAGE_PC:
     return getLoongArchPageDelta(r.sym->getVA(ctx, a), p, r.type);
   case R_PC:
-  case R_ARM_PCA: {
+  case RE_ARM_PCA: {
     uint64_t dest;
-    if (r.expr == R_ARM_PCA)
+    if (r.expr == RE_ARM_PCA)
       // Some PC relative ARM (Thumb) relocations align down the place.
       p = p & 0xfffffffc;
     if (r.sym->isUndefined()) {
@@ -889,20 +909,20 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_PLT:
     return r.sym->getPltVA(ctx) + a;
   case R_PLT_PC:
-  case R_PPC64_CALL_PLT:
+  case RE_PPC64_CALL_PLT:
     return r.sym->getPltVA(ctx) + a - p;
-  case R_LOONGARCH_PLT_PAGE_PC:
+  case RE_LOONGARCH_PLT_PAGE_PC:
     return getLoongArchPageDelta(r.sym->getPltVA(ctx) + a, p, r.type);
   case R_PLT_GOTPLT:
     return r.sym->getPltVA(ctx) + a - ctx.in.gotPlt->getVA();
   case R_PLT_GOTREL:
     return r.sym->getPltVA(ctx) + a - ctx.in.got->getVA();
-  case R_PPC32_PLTREL:
+  case RE_PPC32_PLTREL:
     // R_PPC_PLTREL24 uses the addend (usually 0 or 0x8000) to indicate r30
     // stores _GLOBAL_OFFSET_TABLE_ or .got2+0x8000. The addend is ignored for
     // target VA computation.
     return r.sym->getPltVA(ctx) - p;
-  case R_PPC64_CALL: {
+  case RE_PPC64_CALL: {
     uint64_t symVA = r.sym->getVA(ctx, a);
     // If we have an undefined weak symbol, we might get here with a symbol
     // address of zero. That could overflow, but the code must be unreachable,
@@ -919,10 +939,10 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
     return symVA - p +
            getPPC64GlobalEntryToLocalEntryOffset(ctx, r.sym->stOther);
   }
-  case R_PPC64_TOCBASE:
+  case RE_PPC64_TOCBASE:
     return getPPC64TocBase(ctx) + a;
   case R_RELAX_GOT_PC:
-  case R_PPC64_RELAX_GOT_PC:
+  case RE_PPC64_RELAX_GOT_PC:
     return r.sym->getVA(ctx, a) - p;
   case R_RELAX_TLS_GD_TO_LE:
   case R_RELAX_TLS_IE_TO_LE:
@@ -948,10 +968,10 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
     return ctx.in.got->getTlsDescAddr(*r.sym) + a - p;
   case R_TLSDESC_GOTPLT:
     return ctx.in.got->getTlsDescAddr(*r.sym) + a - ctx.in.gotPlt->getVA();
-  case R_AARCH64_TLSDESC_PAGE:
+  case RE_AARCH64_TLSDESC_PAGE:
     return getAArch64Page(ctx.in.got->getTlsDescAddr(*r.sym) + a) -
            getAArch64Page(p);
-  case R_LOONGARCH_TLSDESC_PAGE_PC:
+  case RE_LOONGARCH_TLSDESC_PAGE_PC:
     return getLoongArchPageDelta(ctx.in.got->getTlsDescAddr(*r.sym) + a, p,
                                  r.type);
   case R_TLSGD_GOT:
@@ -960,7 +980,7 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
     return ctx.in.got->getGlobalDynAddr(*r.sym) + a - ctx.in.gotPlt->getVA();
   case R_TLSGD_PC:
     return ctx.in.got->getGlobalDynAddr(*r.sym) + a - p;
-  case R_LOONGARCH_TLSGD_PAGE_PC:
+  case RE_LOONGARCH_TLSGD_PAGE_PC:
     return getLoongArchPageDelta(ctx.in.got->getGlobalDynAddr(*r.sym) + a, p,
                                  r.type);
   case R_TLSLD_GOTPLT:
@@ -1094,7 +1114,7 @@ void InputSection::relocateNonAlloc(Ctx &ctx, uint8_t *buf,
     // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC
     // sections.
     if (LLVM_LIKELY(expr == R_ABS) || expr == R_DTPREL || expr == R_GOTPLTREL ||
-        expr == R_RISCV_ADD || expr == R_ARM_SBREL) {
+        expr == RE_RISCV_ADD || expr == RE_ARM_SBREL) {
       target.relocateNoSym(bufLoc, type,
                            SignExtend64<bits>(sym.getVA(ctx, addend)));
       continue;
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 268caa547ffed..60988dfacbd74 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -143,6 +143,16 @@ struct RelaxAux {
 // This corresponds to a section of an input file.
 class InputSectionBase : public SectionBase {
 public:
+  struct ObjMsg {
+    const InputSectionBase *sec;
+    uint64_t offset;
+  };
+  struct SrcMsg {
+    const InputSectionBase &sec;
+    const Symbol &sym;
+    uint64_t offset;
+  };
+
   template <class ELFT>
   InputSectionBase(ObjFile<ELFT> &file, const typename ELFT::Shdr &header,
                    StringRef name, Kind sectionKind);
@@ -247,8 +257,10 @@ class InputSectionBase : public SectionBase {
 
   // Returns a source location string. Used to construct an error message.
   std::string getLocation(uint64_t offset) const;
-  std::string getSrcMsg(const Symbol &sym, uint64_t offset) const;
-  std::string getObjMsg(uint64_t offset) const;
+  ObjMsg getObjMsg(uint64_t offset) const { return {this, offset}; }
+  SrcMsg getSrcMsg(const Symbol &sym, uint64_t offset) const {
+    return {*this, sym, offset};
+  }
 
   // Each section knows how to relocate itself. These functions apply
   // relocations, assuming that Buf points to this section's copy in
@@ -515,6 +527,10 @@ inline bool isDebugSection(const InputSectionBase &sec) {
 std::string toStr(elf::Ctx &, const elf::InputSectionBase *);
 const ELFSyncStream &operator<<(const ELFSyncStream &,
                                 const InputSectionBase *);
+const ELFSyncStream &operator<<(const ELFSyncStream &,
+                                InputSectionBase::ObjMsg &&);
+const ELFSyncStream &operator<<(const ELFSyncStream &,
+                                InputSectionBase::SrcMsg &&);
 } // namespace elf
 } // namespace lld
 
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 5ed69c8c3befa..195526bf390d2 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -250,13 +250,12 @@ void BitcodeCompiler::add(BitcodeFile &f) {
     // 5) Symbols that will be referenced after linker wrapping is performed.
     r.VisibleToRegularObj = ctx.arg.relocatable || sym->isUsedInRegularObj ||
                             sym->referencedAfterWrap ||
-                            (r.Prevailing && sym->includeInDynsym(ctx)) ||
+                            (r.Prevailing && sym->isExported) ||
                             usedStartStop.count(objSym.getSectionName());
     // Identify symbols exported dynamically, and that therefore could be
     // referenced by a shared library not visible to the linker.
-    r.ExportDynamic =
-        sym->computeBinding(ctx) != STB_LOCAL &&
-        (ctx.arg.exportDynamic || sym->exportDynamic || sym->inDynamicList);
+    r.ExportDynamic = sym->computeBinding(ctx) != STB_LOCAL &&
+                      (ctx.arg.exportDynamic || sym->isExported);
     const auto *dr = dyn_cast<Defined>(sym);
     r.FinalDefinitionInLinkageUnit =
         (isExec || sym->visibility() != STV_DEFAULT) && dr &&
diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp
index 08d9ecedf2cc0..b6c22884d9176 100644
--- a/lld/ELF/MarkLive.cpp
+++ b/lld/ELF/MarkLive.cpp
@@ -221,7 +221,7 @@ template <class ELFT> void MarkLive<ELFT>::run() {
   // Preserve externally-visible symbols if the symbols defined by this
   // file can interpose other ELF file's symbols at runtime.
   for (Symbol *sym : ctx.symtab->getSymbols())
-    if (sym->includeInDynsym(ctx) && sym->partition == partition)
+    if (sym->isExported && sym->partition == partition)
       markSymbol(sym);
 
   // If this isn't the main partition, that's all that we need to preserve.
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 31d14df9be71e..a2da5543d5867 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -155,9 +155,9 @@ void OutputSection::commitSection(InputSection *isec) {
     // Otherwise, check if new type or flags are compatible with existing ones.
     if ((flags ^ isec->flags) & SHF_TLS)
       ErrAlways(ctx) << "incompatible section flags for " << name << "\n>>> "
-                     << isec << ": 0x" << utohexstr(isec->flags)
+                     << isec << ": 0x" << utohexstr(isec->flags, true)
                      << "\n>>> output section " << name << ": 0x"
-                     << utohexstr(flags);
+                     << utohexstr(flags, true);
   }
 
   isec->parent = this;
@@ -253,8 +253,20 @@ void OutputSection::finalizeInputSections() {
     for (InputSection *s : isd->sections)
       commitSection(s);
   }
-  for (auto *ms : mergeSections)
+  for (auto *ms : mergeSections) {
+    // Merging may have increased the alignment of a spillable section. Update
+    // the alignment of potential spill sections and their containing output
+    // sections.
+    if (auto it = script->potentialSpillLists.find(ms);
+        it != script->potentialSpillLists.end()) {
+      for (PotentialSpillSection *s = it->second.head; s; s = s->next) {
+        s->addralign = std::max(s->addralign, ms->addralign);
+        s->parent->addralign = std::max(s->parent->addralign, s->addralign);
+      }
+    }
+
     ms->finalizeContents();
+  }
 }
 
 static void sortByOrder(MutableArrayRef<InputSection *> in,
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index d311dba41741c..41c04b2903456 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -65,21 +65,8 @@ using namespace llvm::support::endian;
 using namespace lld;
 using namespace lld::elf;
 
-static std::optional<std::string> getLinkerScriptLocation(Ctx &ctx,
-                                                          const Symbol &sym) {
-  for (SectionCommand *cmd : ctx.script->sectionCommands)
-    if (auto *assign = dyn_cast<SymbolAssignment>(cmd))
-      if (assign->sym == &sym)
-        return assign->location;
-  return std::nullopt;
-}
-
 static void printDefinedLocation(ELFSyncStream &s, const Symbol &sym) {
-  s << "\n>>> defined in ";
-  if (sym.file)
-    return void(s << sym.file);
-  if (std::optional<std::string> loc = getLinkerScriptLocation(s.ctx, sym))
-    return void(s << *loc);
+  s << "\n>>> defined in " << sym.file;
 }
 
 // Construct a message in the following format.
@@ -91,9 +78,10 @@ static void printLocation(ELFSyncStream &s, InputSectionBase &sec,
                           const Symbol &sym, uint64_t off) {
   printDefinedLocation(s, sym);
   s << "\n>>> referenced by ";
-  std::string src = sec.getSrcMsg(sym, off);
-  if (!src.empty())
-    s << src << "\n>>>               ";
+  auto tell = s.tell();
+  s << sec.getSrcMsg(sym, off);
+  if (tell != s.tell())
+    s << "\n>>>               ";
   s << sec.getObjMsg(off);
 }
 
@@ -204,32 +192,32 @@ static bool isAbsoluteValue(const Symbol &sym) {
 // Returns true if Expr refers a PLT entry.
 static bool needsPlt(RelExpr expr) {
   return oneof<R_PLT, R_PLT_PC, R_PLT_GOTREL, R_PLT_GOTPLT, R_GOTPLT_GOTREL,
-               R_GOTPLT_PC, R_LOONGARCH_PLT_PAGE_PC, R_PPC32_PLTREL,
-               R_PPC64_CALL_PLT>(expr);
+               R_GOTPLT_PC, RE_LOONGARCH_PLT_PAGE_PC, RE_PPC32_PLTREL,
+               RE_PPC64_CALL_PLT>(expr);
 }
 
 bool lld::elf::needsGot(RelExpr expr) {
-  return oneof<R_GOT, R_GOT_OFF, R_MIPS_GOT_LOCAL_PAGE, R_MIPS_GOT_OFF,
-               R_MIPS_GOT_OFF32, R_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTPLT,
-               R_AARCH64_GOT_PAGE, R_LOONGARCH_GOT, R_LOONGARCH_GOT_PAGE_PC>(
+  return oneof<R_GOT, R_GOT_OFF, RE_MIPS_GOT_LOCAL_PAGE, RE_MIPS_GOT_OFF,
+               RE_MIPS_GOT_OFF32, RE_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTPLT,
+               RE_AARCH64_GOT_PAGE, RE_LOONGARCH_GOT, RE_LOONGARCH_GOT_PAGE_PC>(
       expr);
 }
 
 // True if this expression is of the form Sym - X, where X is a position in the
 // file (PC, or GOT for example).
 static bool isRelExpr(RelExpr expr) {
-  return oneof<R_PC, R_GOTREL, R_GOTPLTREL, R_ARM_PCA, R_MIPS_GOTREL,
-               R_PPC64_CALL, R_PPC64_RELAX_TOC, R_AARCH64_PAGE_PC,
-               R_RELAX_GOT_PC, R_RISCV_PC_INDIRECT, R_PPC64_RELAX_GOT_PC,
-               R_LOONGARCH_PAGE_PC>(expr);
+  return oneof<R_PC, R_GOTREL, R_GOTPLTREL, RE_ARM_PCA, RE_MIPS_GOTREL,
+               RE_PPC64_CALL, RE_PPC64_RELAX_TOC, RE_AARCH64_PAGE_PC,
+               R_RELAX_GOT_PC, RE_RISCV_PC_INDIRECT, RE_PPC64_RELAX_GOT_PC,
+               RE_LOONGARCH_PAGE_PC>(expr);
 }
 
 static RelExpr toPlt(RelExpr expr) {
   switch (expr) {
-  case R_LOONGARCH_PAGE_PC:
-    return R_LOONGARCH_PLT_PAGE_PC;
-  case R_PPC64_CALL:
-    return R_PPC64_CALL_PLT;
+  case RE_LOONGARCH_PAGE_PC:
+    return RE_LOONGARCH_PLT_PAGE_PC;
+  case RE_PPC64_CALL:
+    return RE_PPC64_CALL_PLT;
   case R_PC:
     return R_PLT_PC;
   case R_ABS:
@@ -246,12 +234,12 @@ static RelExpr fromPlt(RelExpr expr) {
   // reference to the symbol itself.
   switch (expr) {
   case R_PLT_PC:
-  case R_PPC32_PLTREL:
+  case RE_PPC32_PLTREL:
     return R_PC;
-  case R_LOONGARCH_PLT_PAGE_PC:
-    return R_LOONGARCH_PAGE_PC;
-  case R_PPC64_CALL_PLT:
-    return R_PPC64_CALL;
+  case RE_LOONGARCH_PLT_PAGE_PC:
+    return RE_LOONGARCH_PAGE_PC;
+  case RE_PPC64_CALL_PLT:
+    return RE_PPC64_CALL;
   case R_PLT:
     return R_ABS;
   case R_PLT_GOTPLT:
@@ -322,7 +310,6 @@ static void replaceWithDefined(Ctx &ctx, Symbol &sym, SectionBase &sec,
       .overwrite(sym);
 
   sym.versionId = old.versionId;
-  sym.exportDynamic = true;
   sym.isUsedInRegularObj = true;
   // A copy relocated alias may need a GOT entry.
   sym.flags.store(old.flags.load(std::memory_order_relaxed) & NEEDS_GOT,
@@ -494,7 +481,7 @@ class RelocationScanner {
 template <class ELFT, class RelTy>
 int64_t RelocationScanner::computeMipsAddend(const RelTy &rel, RelExpr expr,
                                              bool isLocal) const {
-  if (expr == R_MIPS_GOTREL && isLocal)
+  if (expr == RE_MIPS_GOTREL && isLocal)
     return sec->getFile<ELFT>()->mipsGp0;
 
   // The ABI says that the paired relocation is used only for REL.
@@ -525,7 +512,7 @@ int64_t RelocationScanner::computeMipsAddend(const RelTy &rel, RelExpr expr,
 // Custom error message if Sym is defined in a discarded section.
 template <class ELFT>
 static void maybeReportDiscarded(Ctx &ctx, ELFSyncStream &msg, Undefined &sym) {
-  auto *file = dyn_cast_or_null<ObjFile<ELFT>>(sym.file);
+  auto *file = dyn_cast<ObjFile<ELFT>>(sym.file);
   if (!file || !sym.discardedSecIdx)
     return;
   ArrayRef<typename ELFT::Shdr> objSections =
@@ -581,11 +568,11 @@ static const Symbol *getAlternativeSpelling(Ctx &ctx, const Undefined &sym,
                                             std::string &pre_hint,
                                             std::string &post_hint) {
   DenseMap<StringRef, const Symbol *> map;
-  if (sym.file && sym.file->kind() == InputFile::ObjKind) {
+  if (sym.file->kind() == InputFile::ObjKind) {
     auto *file = cast<ELFFileBase>(sym.file);
     // If sym is a symbol defined in a discarded section, maybeReportDiscarded()
     // will give an error. Don't suggest an alternative spelling.
-    if (file && sym.discardedSecIdx != 0 &&
+    if (sym.discardedSecIdx != 0 &&
         file->getSections()[sym.discardedSecIdx] == &InputSection::discarded)
       return nullptr;
 
@@ -738,9 +725,12 @@ static void reportUndefinedSymbol(Ctx &ctx, const UndefinedDiag &undef,
     // In the absence of line number information, utilize DW_TAG_variable (if
     // present) for the enclosing symbol (e.g. var in `int *a[] = {&undef};`).
     Symbol *enclosing = sec.getEnclosingSymbol(offset);
-    std::string src = sec.getSrcMsg(enclosing ? *enclosing : sym, offset);
-    if (!src.empty())
-      msg << src << "\n>>>               ";
+
+    ELFSyncStream msg1(ctx, DiagLevel::None);
+    auto tell = msg.tell();
+    msg << sec.getSrcMsg(enclosing ? *enclosing : sym, offset);
+    if (tell != msg.tell())
+      msg << "\n>>>               ";
     msg << sec.getObjMsg(offset);
   }
 
@@ -752,9 +742,8 @@ static void reportUndefinedSymbol(Ctx &ctx, const UndefinedDiag &undef,
     std::string pre_hint = ": ", post_hint;
     if (const Symbol *corrected =
             getAlternativeSpelling(ctx, sym, pre_hint, post_hint)) {
-      msg << "\n>>> did you mean" << pre_hint << corrected << post_hint;
-      if (corrected->file)
-        msg << "\n>>> defined in: " << corrected->file;
+      msg << "\n>>> did you mean" << pre_hint << corrected << post_hint
+          << "\n>>> defined in: " << corrected->file;
     }
   }
 
@@ -965,13 +954,14 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type,
                                                  const Symbol &sym,
                                                  uint64_t relOff) const {
   // These expressions always compute a constant
-  if (oneof<R_GOTPLT, R_GOT_OFF, R_RELAX_HINT, R_MIPS_GOT_LOCAL_PAGE,
-            R_MIPS_GOTREL, R_MIPS_GOT_OFF, R_MIPS_GOT_OFF32, R_MIPS_GOT_GP_PC,
-            R_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTONLY_PC, R_GOTPLTONLY_PC,
-            R_PLT_PC, R_PLT_GOTREL, R_PLT_GOTPLT, R_GOTPLT_GOTREL, R_GOTPLT_PC,
-            R_PPC32_PLTREL, R_PPC64_CALL_PLT, R_PPC64_RELAX_TOC, R_RISCV_ADD,
-            R_AARCH64_GOT_PAGE, R_LOONGARCH_PLT_PAGE_PC, R_LOONGARCH_GOT,
-            R_LOONGARCH_GOT_PAGE_PC>(e))
+  if (oneof<R_GOTPLT, R_GOT_OFF, R_RELAX_HINT, RE_MIPS_GOT_LOCAL_PAGE,
+            RE_MIPS_GOTREL, RE_MIPS_GOT_OFF, RE_MIPS_GOT_OFF32,
+            RE_MIPS_GOT_GP_PC, RE_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTONLY_PC,
+            R_GOTPLTONLY_PC, R_PLT_PC, R_PLT_GOTREL, R_PLT_GOTPLT,
+            R_GOTPLT_GOTREL, R_GOTPLT_PC, RE_PPC32_PLTREL, RE_PPC64_CALL_PLT,
+            RE_PPC64_RELAX_TOC, RE_RISCV_ADD, RE_AARCH64_GOT_PAGE,
+            RE_LOONGARCH_PLT_PAGE_PC, RE_LOONGARCH_GOT,
+            RE_LOONGARCH_GOT_PAGE_PC>(e))
     return true;
 
   // These never do, except if the entire file is position dependent or if
@@ -980,13 +970,13 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type,
     return ctx.target->usesOnlyLowPageBits(type) || !ctx.arg.isPic;
 
   // R_AARCH64_AUTH_ABS64 requires a dynamic relocation.
-  if (sym.isPreemptible || e == R_AARCH64_AUTH)
+  if (sym.isPreemptible || e == RE_AARCH64_AUTH)
     return false;
   if (!ctx.arg.isPic)
     return true;
 
   // Constant when referencing a non-preemptible symbol.
-  if (e == R_SIZE || e == R_RISCV_LEB128)
+  if (e == R_SIZE || e == RE_RISCV_LEB128)
     return true;
 
   // For the target and the relocation, we want to know if they are
@@ -1043,7 +1033,7 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
     if (expr != R_GOT_PC) {
       // The 0x8000 bit of r_addend of R_PPC_PLTREL24 is used to choose call
       // stub type. It should be ignored if optimized to R_PC.
-      if (ctx.arg.emachine == EM_PPC && expr == R_PPC32_PLTREL)
+      if (ctx.arg.emachine == EM_PPC && expr == RE_PPC32_PLTREL)
         addend &= ~0x8000;
       // R_HEX_GD_PLT_B22_PCREL (call a@GDPLT) is transformed into
       // call __tls_get_addr even if the symbol is non-preemptible.
@@ -1066,7 +1056,7 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
   // direct relocation on through.
   if (LLVM_UNLIKELY(isIfunc) && ctx.arg.zIfuncNoplt) {
     std::lock_guard<std::mutex> lock(ctx.relocMutex);
-    sym.exportDynamic = true;
+    sym.isExported = true;
     ctx.mainPart->relaDyn->addSymbolReloc(type, *sec, offset, sym, addend,
                                           type);
     return;
@@ -1083,7 +1073,7 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
       // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
       ctx.in.mipsGot->addEntry(*sec->file, sym, addend, expr);
     } else if (!sym.isTls() || ctx.arg.emachine != EM_LOONGARCH) {
-      // Many LoongArch TLS relocs reuse the R_LOONGARCH_GOT type, in which
+      // Many LoongArch TLS relocs reuse the RE_LOONGARCH_GOT type, in which
       // case the NEEDS_GOT flag shouldn't get set.
       sym.setFlags(NEEDS_GOT);
     }
@@ -1124,7 +1114,7 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
                     (isa<EhInputSection>(sec) && ctx.arg.emachine != EM_MIPS));
   if (canWrite) {
     RelType rel = ctx.target->getDynRel(type);
-    if (oneof<R_GOT, R_LOONGARCH_GOT>(expr) ||
+    if (oneof<R_GOT, RE_LOONGARCH_GOT>(expr) ||
         (rel == ctx.target->symbolicRel && !sym.isPreemptible)) {
       addRelativeReloc<true>(ctx, *sec, offset, sym, addend, expr, type);
       return;
@@ -1263,12 +1253,12 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
 static unsigned handleMipsTlsRelocation(Ctx &ctx, RelType type, Symbol &sym,
                                         InputSectionBase &c, uint64_t offset,
                                         int64_t addend, RelExpr expr) {
-  if (expr == R_MIPS_TLSLD) {
+  if (expr == RE_MIPS_TLSLD) {
     ctx.in.mipsGot->addTlsIndex(*c.file);
     c.addReloc({expr, type, offset, addend, &sym});
     return 1;
   }
-  if (expr == R_MIPS_TLSGD) {
+  if (expr == RE_MIPS_TLSGD) {
     ctx.in.mipsGot->addDynTlsEntry(*c.file, sym);
     c.addReloc({expr, type, offset, addend, &sym});
     return 1;
@@ -1303,7 +1293,7 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
   // LoongArch does not yet implement transition from TLSDESC to LE/IE, so
   // generate TLSDESC dynamic relocation for the dynamic linker to handle.
   if (ctx.arg.emachine == EM_LOONGARCH &&
-      oneof<R_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
+      oneof<RE_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
             R_TLSDESC_CALL>(expr)) {
     if (expr != R_TLSDESC_CALL) {
       sym.setFlags(NEEDS_TLSDESC);
@@ -1314,7 +1304,7 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
 
   bool isRISCV = ctx.arg.emachine == EM_RISCV;
 
-  if (oneof<R_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
+  if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT>(expr) &&
       ctx.arg.shared) {
     // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12_I,CALL} reference a label. Do not
@@ -1383,9 +1373,9 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return 1;
   }
 
-  if (oneof<R_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
+  if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT, R_TLSGD_GOT, R_TLSGD_GOTPLT, R_TLSGD_PC,
-            R_LOONGARCH_TLSGD_PAGE_PC>(expr)) {
+            RE_LOONGARCH_TLSGD_PAGE_PC>(expr)) {
     if (!execOptimize) {
       sym.setFlags(NEEDS_TLSGD);
       sec->addReloc({expr, type, offset, addend, &sym});
@@ -1409,8 +1399,8 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return ctx.target->getTlsGdRelaxSkip(type);
   }
 
-  if (oneof<R_GOT, R_GOTPLT, R_GOT_PC, R_AARCH64_GOT_PAGE_PC,
-            R_LOONGARCH_GOT_PAGE_PC, R_GOT_OFF, R_TLSIE_HINT>(expr)) {
+  if (oneof<R_GOT, R_GOTPLT, R_GOT_PC, RE_AARCH64_GOT_PAGE_PC,
+            RE_LOONGARCH_GOT_PAGE_PC, R_GOT_OFF, R_TLSIE_HINT>(expr)) {
     ctx.hasTlsIe.store(true, std::memory_order_relaxed);
     // Initial-Exec relocs can be optimized to Local-Exec if the symbol is
     // locally defined.  This is not supported on SystemZ.
@@ -1520,8 +1510,8 @@ void RelocationScanner::scanOne(typename Relocs<RelTy>::const_iterator &i) {
   if (oneof<R_GOTPLTONLY_PC, R_GOTPLTREL, R_GOTPLT, R_PLT_GOTPLT,
             R_TLSDESC_GOTPLT, R_TLSGD_GOTPLT>(expr)) {
     ctx.in.gotPlt->hasGotPltOffRel.store(true, std::memory_order_relaxed);
-  } else if (oneof<R_GOTONLY_PC, R_GOTREL, R_PPC32_PLTREL, R_PPC64_TOCBASE,
-                   R_PPC64_RELAX_TOC>(expr)) {
+  } else if (oneof<R_GOTONLY_PC, R_GOTREL, RE_PPC32_PLTREL, RE_PPC64_TOCBASE,
+                   RE_PPC64_RELAX_TOC>(expr)) {
     ctx.in.got->hasGotOffRel.store(true, std::memory_order_relaxed);
   }
 
@@ -1682,6 +1672,17 @@ template <class ELFT> void elf::scanRelocations(Ctx &ctx) {
     outerFn();
 }
 
+RelocationBaseSection &elf::getIRelativeSection(Ctx &ctx) {
+  // Prior to Android V, there was a bug that caused RELR relocations to be
+  // applied after packed relocations. This meant that resolvers referenced by
+  // IRELATIVE relocations in the packed relocation section would read
+  // unrelocated globals with RELR relocations when
+  // --pack-relative-relocs=android+relr is enabled. Work around this by placing
+  // IRELATIVE in .rela.plt.
+  return ctx.arg.androidPackDynRelocs ? *ctx.in.relaPlt
+                                      : *ctx.mainPart->relaDyn;
+}
+
 static bool handleNonPreemptibleIfunc(Ctx &ctx, Symbol &sym, uint16_t flags) {
   // Handle a reference to a non-preemptible ifunc. These are special in a
   // few ways:
@@ -1731,17 +1732,9 @@ static bool handleNonPreemptibleIfunc(Ctx &ctx, Symbol &sym, uint16_t flags) {
   // original section/value pairs. For non-GOT non-PLT relocation case below, we
   // may alter section/value, so create a copy of the symbol to make
   // section/value fixed.
-  //
-  // Prior to Android V, there was a bug that caused RELR relocations to be
-  // applied after packed relocations. This meant that resolvers referenced by
-  // IRELATIVE relocations in the packed relocation section would read
-  // unrelocated globals with RELR relocations when
-  // --pack-relative-relocs=android+relr is enabled. Work around this by placing
-  // IRELATIVE in .rela.plt.
   auto *directSym = makeDefined(cast<Defined>(sym));
   directSym->allocateAux(ctx);
-  auto &dyn =
-      ctx.arg.androidPackDynRelocs ? *ctx.in.relaPlt : *ctx.mainPart->relaDyn;
+  auto &dyn = getIRelativeSection(ctx);
   addPltEntry(ctx, *ctx.in.iplt, *ctx.in.igotPlt, dyn, ctx.target->iRelativeRel,
               *directSym);
   sym.allocateAux(ctx);
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index 041bd48048587..7ca203257ea87 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -22,6 +22,7 @@ class Symbol;
 class InputSection;
 class InputSectionBase;
 class OutputSection;
+class RelocationBaseSection;
 class SectionBase;
 
 // Represents a relocation type, such as R_X86_64_PC32 or R_ARM_THM_CALL.
@@ -89,42 +90,42 @@ enum RelExpr {
   //
   // Even though RelExpr is intended to be a target-neutral representation
   // of a relocation type, there are some relocations whose semantics are
-  // unique to a target. Such relocation are marked with R_<TARGET_NAME>.
-  R_AARCH64_GOT_PAGE_PC,
-  R_AARCH64_GOT_PAGE,
-  R_AARCH64_PAGE_PC,
-  R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC,
-  R_AARCH64_TLSDESC_PAGE,
-  R_AARCH64_AUTH,
-  R_ARM_PCA,
-  R_ARM_SBREL,
-  R_MIPS_GOTREL,
-  R_MIPS_GOT_GP,
-  R_MIPS_GOT_GP_PC,
-  R_MIPS_GOT_LOCAL_PAGE,
-  R_MIPS_GOT_OFF,
-  R_MIPS_GOT_OFF32,
-  R_MIPS_TLSGD,
-  R_MIPS_TLSLD,
-  R_PPC32_PLTREL,
-  R_PPC64_CALL,
-  R_PPC64_CALL_PLT,
-  R_PPC64_RELAX_TOC,
-  R_PPC64_TOCBASE,
-  R_PPC64_RELAX_GOT_PC,
-  R_RISCV_ADD,
-  R_RISCV_LEB128,
-  R_RISCV_PC_INDIRECT,
+  // unique to a target. Such relocation are marked with RE_<TARGET_NAME>.
+  RE_AARCH64_GOT_PAGE_PC,
+  RE_AARCH64_GOT_PAGE,
+  RE_AARCH64_PAGE_PC,
+  RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC,
+  RE_AARCH64_TLSDESC_PAGE,
+  RE_AARCH64_AUTH,
+  RE_ARM_PCA,
+  RE_ARM_SBREL,
+  RE_MIPS_GOTREL,
+  RE_MIPS_GOT_GP,
+  RE_MIPS_GOT_GP_PC,
+  RE_MIPS_GOT_LOCAL_PAGE,
+  RE_MIPS_GOT_OFF,
+  RE_MIPS_GOT_OFF32,
+  RE_MIPS_TLSGD,
+  RE_MIPS_TLSLD,
+  RE_PPC32_PLTREL,
+  RE_PPC64_CALL,
+  RE_PPC64_CALL_PLT,
+  RE_PPC64_RELAX_TOC,
+  RE_PPC64_TOCBASE,
+  RE_PPC64_RELAX_GOT_PC,
+  RE_RISCV_ADD,
+  RE_RISCV_LEB128,
+  RE_RISCV_PC_INDIRECT,
   // Same as R_PC but with page-aligned semantics.
-  R_LOONGARCH_PAGE_PC,
+  RE_LOONGARCH_PAGE_PC,
   // Same as R_PLT_PC but with page-aligned semantics.
-  R_LOONGARCH_PLT_PAGE_PC,
+  RE_LOONGARCH_PLT_PAGE_PC,
   // In addition to having page-aligned semantics, LoongArch GOT relocs are
   // also reused for TLS, making the semantics differ from other architectures.
-  R_LOONGARCH_GOT,
-  R_LOONGARCH_GOT_PAGE_PC,
-  R_LOONGARCH_TLSGD_PAGE_PC,
-  R_LOONGARCH_TLSDESC_PAGE_PC,
+  RE_LOONGARCH_GOT,
+  RE_LOONGARCH_GOT_PAGE_PC,
+  RE_LOONGARCH_TLSGD_PAGE_PC,
+  RE_LOONGARCH_TLSDESC_PAGE_PC,
 };
 
 // Architecture-neutral representation of relocation.
@@ -356,6 +357,8 @@ sortRels(Relocs<llvm::object::Elf_Crel_Impl<is64>> rels,
   return {};
 }
 
+RelocationBaseSection &getIRelativeSection(Ctx &ctx);
+
 // Returns true if Expr refers a GOT entry. Note that this function returns
 // false for TLS variables even though they need GOT, because TLS variables uses
 // GOT differently than the regular variables.
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index f76ca0a9faf52..d5b2200b50227 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -203,7 +203,7 @@ void SymbolTable::handleDynamicList() {
       syms = findByVersion(ver);
 
     for (Symbol *sym : syms)
-      sym->inDynamicList = true;
+      sym->exportDynamic = sym->inDynamicList = true;
   }
 }
 
@@ -350,13 +350,6 @@ void SymbolTable::scanVersionScript() {
         assignAsterisk(pat, &v, true);
   }
 
-  // Symbol themselves might know their versions because symbols
-  // can contain versions in the form of <name>@<version>.
-  // Let them parse and update their names to exclude version suffix.
-  for (Symbol *sym : symVector)
-    if (sym->hasVersionSuffix)
-      sym->parseSymbolVersion(ctx);
-
   // isPreemptible is false at this point. To correctly compute the binding of a
   // Defined (which is used by includeInDynsym(ctx)), we need to know if it is
   // VER_NDX_LOCAL or not. Compute symbol versions before handling
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index faf7d61ffbe0b..ce1e89f2d0801 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -11,6 +11,7 @@
 #include "InputFiles.h"
 #include "InputSection.h"
 #include "OutputSections.h"
+#include "SymbolTable.h"
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "Writer.h"
@@ -278,7 +279,8 @@ bool Symbol::includeInDynsym(Ctx &ctx) const {
     // __pthread_initialize_minimal reference in csu/libc-start.c.
     return !(isUndefWeak() && ctx.arg.noDynamicLinker);
 
-  return exportDynamic || inDynamicList;
+  return exportDynamic ||
+         (ctx.arg.exportDynamic && (isUsedInRegularObj || !ltoCanOmit));
 }
 
 // Print out a log message for --trace-symbol.
@@ -344,7 +346,7 @@ bool elf::computeIsPreemptible(Ctx &ctx, const Symbol &sym) {
 
   // Only symbols with default visibility that appear in dynsym can be
   // preempted. Symbols with protected visibility cannot be preempted.
-  if (!sym.includeInDynsym(ctx) || sym.visibility() != STV_DEFAULT)
+  if (sym.visibility() != STV_DEFAULT)
     return false;
 
   // At this point copy relocations have not been created yet, so any
@@ -369,15 +371,26 @@ bool elf::computeIsPreemptible(Ctx &ctx, const Symbol &sym) {
   return true;
 }
 
+void elf::parseVersionAndComputeIsPreemptible(Ctx &ctx) {
+  // Symbol themselves might know their versions because symbols
+  // can contain versions in the form of <name>@<version>.
+  // Let them parse and update their names to exclude version suffix.
+  bool hasDynSymTab = ctx.arg.hasDynSymTab;
+  for (Symbol *sym : ctx.symtab->getSymbols()) {
+    if (sym->hasVersionSuffix)
+      sym->parseSymbolVersion(ctx);
+    sym->isExported = sym->includeInDynsym(ctx);
+    if (hasDynSymTab)
+      sym->isPreemptible = sym->isExported && computeIsPreemptible(ctx, *sym);
+  }
+}
+
 // Merge symbol properties.
 //
 // When we have many symbols of the same name, we choose one of them,
 // and that's the result of symbol resolution. However, symbols that
 // were not chosen still affect some symbol properties.
 void Symbol::mergeProperties(const Symbol &other) {
-  if (other.exportDynamic)
-    exportDynamic = true;
-
   // DSO symbols do not affect visibility in the output.
   if (!other.isShared() && other.visibility() != STV_DEFAULT) {
     uint8_t v = visibility(), ov = other.visibility();
@@ -463,8 +476,7 @@ void Symbol::resolve(Ctx &ctx, const Undefined &other) {
     // A forms group 0. B form group 1. C and D (including their member object
     // files) form group 2. E forms group 3. I think that you can see how this
     // group assignment rule simulates the traditional linker's semantics.
-    bool backref = ctx.arg.warnBackrefs && other.file &&
-                   file->groupId < other.file->groupId;
+    bool backref = ctx.arg.warnBackrefs && file->groupId < other.file->groupId;
     extract(ctx);
 
     if (!ctx.arg.whyExtract.empty())
@@ -477,7 +489,7 @@ void Symbol::resolve(Ctx &ctx, const Undefined &other) {
     // sandwich), where def2 may or may not be the same as def1. We don't want
     // to warn for this case, so dismiss the warning if we see a subsequent lazy
     // definition. this->file needs to be saved because in the case of LTO it
-    // may be reset to nullptr or be replaced with a file named lto.tmp.
+    // may be reset to internalFile or be replaced with a file named lto.tmp.
     if (backref && !isWeak())
       ctx.backwardReferences.try_emplace(this,
                                          std::make_pair(other.file, file));
@@ -485,7 +497,7 @@ void Symbol::resolve(Ctx &ctx, const Undefined &other) {
   }
 
   // Undefined symbols in a SharedFile do not change the binding.
-  if (isa_and_nonnull<SharedFile>(other.file))
+  if (isa<SharedFile>(other.file))
     return;
 
   if (isUndefined() || isShared()) {
@@ -543,19 +555,18 @@ void elf::reportDuplicate(Ctx &ctx, const Symbol &sym, const InputFile *newFile,
   //   >>> defined at baz.c:563
   //   >>>            baz.o in archive libbaz.a
   auto *sec1 = cast<InputSectionBase>(d->section);
-  std::string src1 = sec1->getSrcMsg(sym, d->value);
-  std::string obj1 = sec1->getObjMsg(d->value);
-  std::string src2 = errSec->getSrcMsg(sym, errOffset);
-  std::string obj2 = errSec->getObjMsg(errOffset);
-
   auto diag = Err(ctx);
   diag << "duplicate symbol: " << &sym << "\n>>> defined at ";
-  if (!src1.empty())
-    diag << src1 << "\n>>>            ";
-  diag << obj1 << "\n>>> defined at ";
-  if (!src2.empty())
-    diag << src2 << "\n>>>            ";
-  diag << obj2;
+  auto tell = diag.tell();
+  diag << sec1->getSrcMsg(sym, d->value);
+  if (tell != diag.tell())
+    diag << "\n>>>            ";
+  diag << sec1->getObjMsg(d->value) << "\n>>> defined at ";
+  tell = diag.tell();
+  diag << errSec->getSrcMsg(sym, errOffset);
+  if (tell != diag.tell())
+    diag << "\n>>>            ";
+  diag << errSec->getObjMsg(errOffset);
 }
 
 void Symbol::checkDuplicate(Ctx &ctx, const Defined &other) const {
@@ -566,8 +577,6 @@ void Symbol::checkDuplicate(Ctx &ctx, const Defined &other) const {
 }
 
 void Symbol::resolve(Ctx &ctx, const CommonSymbol &other) {
-  if (other.exportDynamic)
-    exportDynamic = true;
   if (other.visibility() != STV_DEFAULT) {
     uint8_t v = visibility(), ov = other.visibility();
     setVisibility(v == STV_DEFAULT ? ov : std::min(v, ov));
@@ -604,8 +613,6 @@ void Symbol::resolve(Ctx &ctx, const CommonSymbol &other) {
 }
 
 void Symbol::resolve(Ctx &ctx, const Defined &other) {
-  if (other.exportDynamic)
-    exportDynamic = true;
   if (other.visibility() != STV_DEFAULT) {
     uint8_t v = visibility(), ov = other.visibility();
     setVisibility(v == STV_DEFAULT ? ov : std::min(v, ov));
@@ -620,20 +627,18 @@ void Symbol::resolve(Ctx &ctx, const LazySymbol &other) {
     return;
   }
 
-  // For common objects, we want to look for global or weak definitions that
-  // should be extracted as the canonical definition instead.
-  if (LLVM_UNLIKELY(isCommon()) && ctx.arg.fortranCommon &&
-      other.file->shouldExtractForCommon(getName())) {
-    ctx.backwardReferences.erase(this);
-    other.overwrite(*this);
-    other.extract(ctx);
-    return;
-  }
-
-  if (!isUndefined()) {
-    // See the comment in resolveUndefined().
-    if (isDefined())
+  if (LLVM_UNLIKELY(!isUndefined())) {
+    // See the comment in resolve(Ctx &, const Undefined &).
+    if (isDefined()) {
       ctx.backwardReferences.erase(this);
+    } else if (isCommon() && ctx.arg.fortranCommon &&
+               other.file->shouldExtractForCommon(getName())) {
+      // For common objects, we want to look for global or weak definitions that
+      // should be extracted as the canonical definition instead.
+      ctx.backwardReferences.erase(this);
+      other.overwrite(*this);
+      other.extract(ctx);
+    }
     return;
   }
 
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index 18d078c58fa55..fac957a724885 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -127,26 +127,18 @@ class Symbol {
   // - If -shared or --export-dynamic is specified, any symbol in an object
   //   file/bitcode sets this property, unless suppressed by LTO
   //   canBeOmittedFromSymbolTable().
+  //
+  // Primarily set in two locations, (a) after parseSymbolVersion and
+  // (b) during demoteSymbols.
   LLVM_PREFERRED_TYPE(bool)
-  uint8_t exportDynamic : 1;
-
-  // True if the symbol is in the --dynamic-list file. A Defined symbol with
-  // protected or default visibility with this property is required to be
-  // exported into .dynsym.
-  LLVM_PREFERRED_TYPE(bool)
-  uint8_t inDynamicList : 1;
+  uint8_t isExported : 1;
 
-  // Used to track if there has been at least one undefined reference to the
-  // symbol. For Undefined and SharedSymbol, the binding may change to STB_WEAK
-  // if the first undefined reference from a non-shared object is weak.
+  // Used to compute isExported. Set when defined or referenced by a SharedFile.
   LLVM_PREFERRED_TYPE(bool)
-  uint8_t referenced : 1;
+  uint8_t exportDynamic : 1;
 
-  // Used to track if this symbol will be referenced after wrapping is performed
-  // (i.e. this will be true for foo if __real_foo is referenced, and will be
-  // true for __wrap_foo if foo is referenced).
   LLVM_PREFERRED_TYPE(bool)
-  uint8_t referencedAfterWrap : 1;
+  uint8_t ltoCanOmit : 1;
 
   // True if this symbol is specified by --trace-symbol option.
   LLVM_PREFERRED_TYPE(bool)
@@ -252,7 +244,7 @@ class Symbol {
          uint8_t stOther, uint8_t type)
       : file(file), nameData(name.data()), nameSize(name.size()), type(type),
         binding(binding), stOther(stOther), symbolKind(k), exportDynamic(false),
-        archSpecificBit(false) {}
+        ltoCanOmit(false), archSpecificBit(false) {}
 
   void overwrite(Symbol &sym, Kind k) const {
     if (sym.traced)
@@ -330,6 +322,24 @@ class Symbol {
   LLVM_PREFERRED_TYPE(bool)
   uint8_t thunkAccessed : 1;
 
+  // True if the symbol is in the --dynamic-list file. A Defined symbol with
+  // protected or default visibility with this property is required to be
+  // exported into .dynsym.
+  LLVM_PREFERRED_TYPE(bool)
+  uint8_t inDynamicList : 1;
+
+  // Used to track if there has been at least one undefined reference to the
+  // symbol. For Undefined and SharedSymbol, the binding may change to STB_WEAK
+  // if the first undefined reference from a non-shared object is weak.
+  LLVM_PREFERRED_TYPE(bool)
+  uint8_t referenced : 1;
+
+  // Used to track if this symbol will be referenced after wrapping is performed
+  // (i.e. this will be true for foo if __real_foo is referenced, and will be
+  // true for __wrap_foo if foo is referenced).
+  LLVM_PREFERRED_TYPE(bool)
+  uint8_t referencedAfterWrap : 1;
+
   void setFlags(uint16_t bits) {
     flags.fetch_or(bits, std::memory_order_relaxed);
   }
@@ -365,7 +375,6 @@ class Defined : public Symbol {
           SectionBase *section)
       : Symbol(DefinedKind, file, name, binding, stOther, type), value(value),
         size(size), section(section) {
-    exportDynamic = ctx.arg.exportDynamic;
   }
   void overwrite(Symbol &sym) const;
 
@@ -403,7 +412,6 @@ class CommonSymbol : public Symbol {
                uint8_t stOther, uint8_t type, uint64_t alignment, uint64_t size)
       : Symbol(CommonKind, file, name, binding, stOther, type),
         alignment(alignment), size(size) {
-    exportDynamic = ctx.arg.exportDynamic;
   }
   void overwrite(Symbol &sym) const {
     Symbol::overwrite(sym, CommonKind);
@@ -447,7 +455,6 @@ class SharedSymbol : public Symbol {
                uint32_t alignment)
       : Symbol(SharedKind, &file, name, binding, stOther, type), value(value),
         size(size), alignment(alignment) {
-    exportDynamic = true;
     dsoProtected = visibility() == llvm::ELF::STV_PROTECTED;
     // GNU ifunc is a mechanism to allow user-supplied functions to
     // resolve PLT slot values at load-time. This is contrary to the
@@ -527,6 +534,7 @@ void reportDuplicate(Ctx &, const Symbol &sym, const InputFile *newFile,
                      InputSectionBase *errSec, uint64_t errOffset);
 void maybeWarnUnorderableSymbol(Ctx &, const Symbol *sym);
 bool computeIsPreemptible(Ctx &, const Symbol &sym);
+void parseVersionAndComputeIsPreemptible(Ctx &);
 
 } // namespace lld::elf
 
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 21fe2a25fa1bd..6c5f2a614639c 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -749,7 +749,7 @@ MipsGotSection::MipsGotSection(Ctx &ctx)
 void MipsGotSection::addEntry(InputFile &file, Symbol &sym, int64_t addend,
                               RelExpr expr) {
   FileGot &g = getGot(file);
-  if (expr == R_MIPS_GOT_LOCAL_PAGE) {
+  if (expr == RE_MIPS_GOT_LOCAL_PAGE) {
     if (const OutputSection *os = sym.getOutputSection())
       g.pagesMap.insert({os, {}});
     else
@@ -760,7 +760,7 @@ void MipsGotSection::addEntry(InputFile &file, Symbol &sym, int64_t addend,
     g.relocs.insert({&sym, 0});
   else if (sym.isPreemptible)
     g.global.insert({&sym, 0});
-  else if (expr == R_MIPS_GOT_OFF32)
+  else if (expr == RE_MIPS_GOT_OFF32)
     g.local32.insert({{&sym, addend}, 0});
   else
     g.local16.insert({{&sym, addend}, 0});
diff --git a/lld/ELF/Target.cpp b/lld/ELF/Target.cpp
index 63d813e550f93..c90ef8505aadd 100644
--- a/lld/ELF/Target.cpp
+++ b/lld/ELF/Target.cpp
@@ -106,10 +106,11 @@ ErrorPlace elf::getErrorPlace(Ctx &ctx, const uint8_t *loc) {
     if (isecLoc <= loc && loc < isecLoc + isec->getSize()) {
       std::string objLoc = isec->getLocation(loc - isecLoc);
       // Return object file location and source file location.
-      // TODO: Refactor getSrcMsg not to take a variable.
       Undefined dummy(ctx.internalFile, "", STB_LOCAL, 0, 0);
-      return {isec, objLoc + ": ",
-              isec->file ? isec->getSrcMsg(dummy, loc - isecLoc) : ""};
+      ELFSyncStream msg(ctx, DiagLevel::None);
+      if (isec->file)
+        msg << isec->getSrcMsg(dummy, loc - isecLoc);
+      return {isec, objLoc + ": ", std::string(msg.str())};
     }
   }
   return {};
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 629ce356ce2e7..4e4e0684a3f59 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -792,7 +792,14 @@ bool ThumbThunk::getMayUseShortThunk() {
   if (!mayUseShortThunk)
     return false;
   uint64_t s = getARMThunkDestVA(ctx, destination);
-  if ((s & 1) == 0 || !ctx.arg.armJ1J2BranchEncoding) {
+  // To use a short thunk the destination must be Thumb and the target must
+  // have the wide branch instruction B.w. This instruction is included when
+  // Thumb 2 is present, or in v8-M (and above) baseline architectures.
+  // armJ1J2BranchEncoding is available in all architectures with a profile and
+  // the one v6 CPU that implements Thumb 2 (Arm1156t2-s).
+  // Movt and Movw instructions require Thumb 2 or v8-M baseline.
+  if ((s & 1) == 0 || !ctx.arg.armJ1J2BranchEncoding ||
+      !ctx.arg.armHasMovtMovw) {
     mayUseShortThunk = false;
     addLongMapSyms();
     return false;
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index a7fbdc0790704..6c16549bfa6c0 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -297,8 +297,9 @@ static void demoteSymbolsAndComputeIsPreemptible(Ctx &ctx) {
       }
     }
 
+    sym->isExported = sym->includeInDynsym(ctx);
     if (ctx.arg.hasDynSymTab)
-      sym->isPreemptible = computeIsPreemptible(ctx, *sym);
+      sym->isPreemptible = sym->isExported && computeIsPreemptible(ctx, *sym);
   }
 }
 
@@ -844,11 +845,15 @@ template <class ELFT> void Writer<ELFT>::setReservedSymbolSections() {
     ctx.sym.globalOffsetTable->section = sec;
   }
 
-  // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn.
-  if (ctx.sym.relaIpltStart && ctx.mainPart->relaDyn->isNeeded()) {
-    ctx.sym.relaIpltStart->section = ctx.mainPart->relaDyn.get();
-    ctx.sym.relaIpltEnd->section = ctx.mainPart->relaDyn.get();
-    ctx.sym.relaIpltEnd->value = ctx.mainPart->relaDyn->getSize();
+  // .rela_iplt_{start,end} mark the start and the end of the section containing
+  // IRELATIVE relocations.
+  if (ctx.sym.relaIpltStart) {
+    auto &dyn = getIRelativeSection(ctx);
+    if (dyn.isNeeded()) {
+      ctx.sym.relaIpltStart->section = &dyn;
+      ctx.sym.relaIpltEnd->section = &dyn;
+      ctx.sym.relaIpltEnd->value = dyn.getSize();
+    }
   }
 
   PhdrEntry *last = nullptr;
@@ -1884,9 +1889,11 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
       if (ctx.in.symTab)
         ctx.in.symTab->addSymbol(sym);
 
-      if (sym->includeInDynsym(ctx)) {
+      // computeBinding might localize a linker-synthesized hidden symbol
+      // (e.g. __global_pointer$) that was considered exported.
+      if (sym->isExported && !sym->isLocal()) {
         ctx.partitions[sym->partition - 1].dynSymTab->addSymbol(sym);
-        if (auto *file = dyn_cast_or_null<SharedFile>(sym->file))
+        if (auto *file = dyn_cast<SharedFile>(sym->file))
           if (file->isNeeded && !sym->isUndefined())
             addVerneed(ctx, *sym);
       }
@@ -2672,8 +2679,9 @@ template <class ELFT> void Writer<ELFT>::checkSections() {
   for (OutputSection *os : ctx.outputSections)
     if ((os->addr + os->size < os->addr) ||
         (!ELFT::Is64Bits && os->addr + os->size > uint64_t(UINT32_MAX) + 1))
-      Err(ctx) << "section " << os->name << " at 0x" << utohexstr(os->addr)
-               << " of size 0x" << utohexstr(os->size)
+      Err(ctx) << "section " << os->name << " at 0x"
+               << utohexstr(os->addr, true) << " of size 0x"
+               << utohexstr(os->size, true)
                << " exceeds available address space";
 
   // Check for overlapping file offsets. In this case we need to skip any
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index a03561be925a8..d41ca5382c692 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -204,6 +204,7 @@ struct Configuration {
   std::vector<llvm::StringRef> frameworkSearchPaths;
   bool warnDuplicateRpath = true;
   llvm::SmallVector<llvm::StringRef, 0> runtimePaths;
+  llvm::SmallVector<llvm::StringRef, 0> allowableClients;
   std::vector<std::string> astPaths;
   std::vector<Symbol *> explicitUndefineds;
   llvm::StringSet<> explicitDynamicLookups;
@@ -221,11 +222,12 @@ struct Configuration {
   bool callGraphProfileSort = false;
   llvm::StringRef printSymbolOrder;
 
-  llvm::StringRef irpgoProfileSortProfilePath;
-  bool compressionSortStartupFunctions = false;
-  bool functionOrderForCompression = false;
-  bool dataOrderForCompression = false;
-  bool verboseBpSectionOrderer = false;
+  llvm::StringRef irpgoProfilePath;
+  bool bpStartupFunctionSort = false;
+  bool bpCompressionSortStartupFunctions = false;
+  bool bpFunctionOrderForCompression = false;
+  bool bpDataOrderForCompression = false;
+  bool bpVerboseSectionOrderer = false;
 
   SectionRenameMap sectionRenameMap;
   SegmentRenameMap segmentRenameMap;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 4f39613ac17de..7550b0b9fa531 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1528,6 +1528,17 @@ static SmallVector<StringRef, 0> getRuntimePaths(opt::InputArgList &args) {
   return vals;
 }
 
+static SmallVector<StringRef, 0> getAllowableClients(opt::InputArgList &args) {
+  SmallVector<StringRef, 0> vals;
+  DenseSet<StringRef> seen;
+  for (const Arg *arg : args.filtered(OPT_allowable_client)) {
+    StringRef val = arg->getValue();
+    if (seen.insert(val).second)
+      vals.push_back(val);
+  }
+  return vals;
+}
+
 namespace lld {
 namespace macho {
 bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
@@ -1772,6 +1783,7 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
   config->warnDuplicateRpath =
       args.hasFlag(OPT_warn_duplicate_rpath, OPT_no_warn_duplicate_rpath, true);
   config->runtimePaths = getRuntimePaths(args);
+  config->allowableClients = getAllowableClients(args);
   config->allLoad = args.hasFlag(OPT_all_load, OPT_noall_load, false);
   config->archMultiple = args.hasArg(OPT_arch_multiple);
   config->applicationExtension = args.hasFlag(
@@ -1826,26 +1838,47 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
       if (const Arg *arg = args.getLastArgNoClaim(OPT_call_graph_profile_sort))
         error(firstArgStr + " is incompatible with " + arg->getSpelling());
   };
+  if (args.hasArg(OPT_irpgo_profile_sort) ||
+      args.hasArg(OPT_irpgo_profile_sort_eq))
+    warn("--irpgo-profile-sort is deprecated. Please use "
+         "--bp-startup-sort=function");
+  if (const Arg *arg = args.getLastArg(OPT_irpgo_profile_eq))
+    config->irpgoProfilePath = arg->getValue();
+
   if (const Arg *arg = args.getLastArg(OPT_irpgo_profile_sort)) {
-    config->irpgoProfileSortProfilePath = arg->getValue();
+    config->irpgoProfilePath = arg->getValue();
+    config->bpStartupFunctionSort = true;
     IncompatWithCGSort(arg->getSpelling());
   }
-  config->compressionSortStartupFunctions =
-      args.hasFlag(OPT_compression_sort_startup_functions,
-                   OPT_no_compression_sort_startup_functions, false);
-  if (config->irpgoProfileSortProfilePath.empty() &&
-      config->compressionSortStartupFunctions)
-    error("--compression-sort-startup-functions must be used with "
-          "--irpgo-profile-sort");
-  if (const Arg *arg = args.getLastArg(OPT_compression_sort)) {
+  config->bpCompressionSortStartupFunctions =
+      args.hasFlag(OPT_bp_compression_sort_startup_functions,
+                   OPT_no_bp_compression_sort_startup_functions, false);
+  if (const Arg *arg = args.getLastArg(OPT_bp_startup_sort)) {
+    StringRef startupSortStr = arg->getValue();
+    if (startupSortStr == "function") {
+      config->bpStartupFunctionSort = true;
+    } else if (startupSortStr != "none") {
+      error("unknown value `" + startupSortStr + "` for " + arg->getSpelling());
+    }
+    if (startupSortStr != "none")
+      IncompatWithCGSort(arg->getSpelling());
+  }
+  if (!config->bpStartupFunctionSort &&
+      config->bpCompressionSortStartupFunctions)
+    error("--bp-compression-sort-startup-functions must be used with "
+          "--bp-startup-sort=function");
+  if (config->irpgoProfilePath.empty() && config->bpStartupFunctionSort)
+    error("--bp-startup-sort=function must be used with "
+          "--irpgo-profile");
+  if (const Arg *arg = args.getLastArg(OPT_bp_compression_sort)) {
     StringRef compressionSortStr = arg->getValue();
     if (compressionSortStr == "function") {
-      config->functionOrderForCompression = true;
+      config->bpFunctionOrderForCompression = true;
     } else if (compressionSortStr == "data") {
-      config->dataOrderForCompression = true;
+      config->bpDataOrderForCompression = true;
     } else if (compressionSortStr == "both") {
-      config->functionOrderForCompression = true;
-      config->dataOrderForCompression = true;
+      config->bpFunctionOrderForCompression = true;
+      config->bpDataOrderForCompression = true;
     } else if (compressionSortStr != "none") {
       error("unknown value `" + compressionSortStr + "` for " +
             arg->getSpelling());
@@ -1853,7 +1886,7 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
     if (compressionSortStr != "none")
       IncompatWithCGSort(arg->getSpelling());
   }
-  config->verboseBpSectionOrderer = args.hasArg(OPT_verbose_bp_section_orderer);
+  config->bpVerboseSectionOrderer = args.hasArg(OPT_verbose_bp_section_orderer);
 
   for (const Arg *arg : args.filtered(OPT_alias)) {
     config->aliasedSymbols.push_back(
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index d453ea3b9e11d..1d7f1d806cc7f 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -126,19 +126,34 @@ def no_call_graph_profile_sort : Flag<["--"], "no-call-graph-profile-sort">,
 def print_symbol_order_eq: Joined<["--"], "print-symbol-order=">,
     HelpText<"Print a symbol order specified by --call-graph-profile-sort into the specified file">,
     Group<grp_lld>;
+def irpgo_profile_eq: Joined<["--"], "irpgo-profile=">,
+    HelpText<"Read the IRPGO profile for use with -bp-startup-sort and other profile-guided optimizations">,
+    Group<grp_lld>;
+def bp_startup_sort: Joined<["--"], "bp-startup-sort=">,
+    MetaVarName<"[none,function]">,
+    HelpText<"Order sections based on profile data to improve startup time">,
+    Group<grp_lld>;
+def bp_compression_sort_startup_functions: Flag<["--"], "bp-compression-sort-startup-functions">,
+    HelpText<"Order startup functions to improve compressed size in addition to startup time">,
+    Group<grp_lld>;
+def no_bp_compression_sort_startup_functions: Flag<["--"], "no-bp-compression-sort-startup-functions">,
+    HelpText<"Do not order startup function for compression">, Group<grp_lld>;
+def bp_compression_sort: Joined<["--"], "bp-compression-sort=">,
+    MetaVarName<"[none,function,data,both]">,
+    HelpText<"Order sections to improve compressed size">, Group<grp_lld>;
 def irpgo_profile_sort: Separate<["--"], "irpgo-profile-sort">, Group<grp_lld>;
 def irpgo_profile_sort_eq: Joined<["--"], "irpgo-profile-sort=">,
     Alias<!cast<Separate>(irpgo_profile_sort)>, MetaVarName<"<profile>">,
-    HelpText<"Read the IRPGO profile at <profile> to order sections to improve startup time">,
+    HelpText<"Deprecated. Please use --irpgo-profile and --bp-startup-sort=function">,
     Group<grp_lld>;
 def compression_sort_startup_functions: Flag<["--"], "compression-sort-startup-functions">,
-    HelpText<"Order startup functions to improve compressed size in addition to startup time">,
-    Group<grp_lld>;
+    Alias<!cast<Flag>(bp_compression_sort_startup_functions)>, HelpText<"Deprecated. Please use --bp-compression-sort-startup-functions">, Group<grp_lld>;
 def no_compression_sort_startup_functions: Flag<["--"], "no-compression-sort-startup-functions">,
-    HelpText<"Do not order startup function for compression">, Group<grp_lld>;
+    Alias<!cast<Flag>(no_bp_compression_sort_startup_functions)>, HelpText<"Deprecated. Please use --no-bp-compression-sort-startup-functions">, Group<grp_lld>;
 def compression_sort: Joined<["--"], "compression-sort=">,
     MetaVarName<"[none,function,data,both]">,
-    HelpText<"Order sections to improve compressed size">, Group<grp_lld>;
+    Alias<!cast<Joined>(bp_compression_sort)>,
+    HelpText<"Deprecated. Please use --bp-compression-sort=">, Group<grp_lld>;
 def verbose_bp_section_orderer: Flag<["--"], "verbose-bp-section-orderer">,
     HelpText<"Print information on how many sections were ordered by balanced partitioning and a measure of the expected number of page faults">,
     Group<grp_lld>;
@@ -876,8 +891,7 @@ def sub_umbrella : Separate<["-"], "sub_umbrella">,
     Group<grp_rare>;
 def allowable_client : Separate<["-"], "allowable_client">,
     MetaVarName<"<name>">,
-    HelpText<"Specify <name> of a dylib or framework that is allowed to link to this dylib">,
-    Flags<[HelpHidden]>,
+    HelpText<"Specify <name> of a dylib, framework, or executable that is allowed to link to this dylib">,
     Group<grp_rare>;
 def client_name : Separate<["-"], "client_name">,
     MetaVarName<"<name>">,
diff --git a/lld/MachO/SectionPriorities.cpp b/lld/MachO/SectionPriorities.cpp
index 1e7fb5973b808..0a15112c1250d 100644
--- a/lld/MachO/SectionPriorities.cpp
+++ b/lld/MachO/SectionPriorities.cpp
@@ -353,14 +353,16 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
 DenseMap<const InputSection *, size_t>
 macho::PriorityBuilder::buildInputSectionPriorities() {
   DenseMap<const InputSection *, size_t> sectionPriorities;
-  if (!config->irpgoProfileSortProfilePath.empty() ||
-      config->functionOrderForCompression || config->dataOrderForCompression) {
+  if (config->bpStartupFunctionSort || config->bpFunctionOrderForCompression ||
+      config->bpDataOrderForCompression) {
     TimeTraceScope timeScope("Balanced Partitioning Section Orderer");
     sectionPriorities = runBalancedPartitioning(
-        highestAvailablePriority, config->irpgoProfileSortProfilePath,
-        config->functionOrderForCompression, config->dataOrderForCompression,
-        config->compressionSortStartupFunctions,
-        config->verboseBpSectionOrderer);
+        highestAvailablePriority,
+        config->bpStartupFunctionSort ? config->irpgoProfilePath : "",
+        config->bpFunctionOrderForCompression,
+        config->bpDataOrderForCompression,
+        config->bpCompressionSortStartupFunctions,
+        config->bpVerboseSectionOrderer);
   } else if (config->callGraphProfileSort) {
     // Sort sections by the profile data provided by __LLVM,__cg_profile
     // sections.
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 0eb809282af28..6a1dd0ae7ecaf 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -409,6 +409,31 @@ class LCRPath final : public LoadCommand {
   StringRef path;
 };
 
+class LCSubClient final : public LoadCommand {
+public:
+  explicit LCSubClient(StringRef client) : client(client) {}
+
+  uint32_t getSize() const override {
+    return alignToPowerOf2(sizeof(sub_client_command) + client.size() + 1,
+                           target->wordSize);
+  }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<sub_client_command *>(buf);
+    buf += sizeof(sub_client_command);
+
+    c->cmd = LC_SUB_CLIENT;
+    c->cmdsize = getSize();
+    c->client = sizeof(sub_client_command);
+
+    memcpy(buf, client.data(), client.size());
+    buf[client.size()] = '\0';
+  }
+
+private:
+  StringRef client;
+};
+
 class LCDyldEnv final : public LoadCommand {
 public:
   explicit LCDyldEnv(StringRef name) : name(name) {}
@@ -822,6 +847,8 @@ template <class LP> void Writer::createLoadCommands() {
     in.header->addLoadCommand(make<LCDylib>(LC_ID_DYLIB, config->installName,
                                             config->dylibCompatibilityVersion,
                                             config->dylibCurrentVersion));
+    for (StringRef client : config->allowableClients)
+      in.header->addLoadCommand(make<LCSubClient>(client));
     break;
   case MH_BUNDLE:
     break;
diff --git a/lld/Maintainers.md b/lld/Maintainers.md
new file mode 100644
index 0000000000000..d210246267656
--- /dev/null
+++ b/lld/Maintainers.md
@@ -0,0 +1,40 @@
+# LLD Maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+LLD.
+
+## Current Maintainers
+
+The following people are the active maintainers for the project. Please reach
+out to them for code reviews, questions about their area of expertise, or other
+assistance.
+
+### COFF, ELF backends (COFF/* ELF/*)
+
+Rui Ueyama \
+ruiu@google.com (email)
+
+### Old Mach-O backend
+
+Lang Hames \
+lhames@gmail.com (email)
+
+Nick Kledzik \
+kledzik@apple.com (email)
+
+### WebAssembly backend (wasm/*)
+
+Sam Clegg \
+sbc@chromium.org (email)
+
+### New Mach-O backend
+
+Jez Ng \
+jezng@fb.com (email)
+
+Greg McGary \
+gkm@fb.com (email)
+
+Shoaib Meenai \
+smeenai@fb.com (email)
diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp
index 72d612ceed225..b4371b5b2240d 100644
--- a/lld/MinGW/Driver.cpp
+++ b/lld/MinGW/Driver.cpp
@@ -407,6 +407,9 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
                    OPT_no_allow_multiple_definition, false))
     add("-force:multiple");
 
+  if (auto *a = args.getLastArg(OPT_dependent_load_flag))
+    add("-dependentloadflag:" + StringRef(a->getValue()));
+
   if (auto *a = args.getLastArg(OPT_icf)) {
     StringRef s = a->getValue();
     if (s == "all")
diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td
index 1ff0f7bedc18a..01b01972112a1 100644
--- a/lld/MinGW/Options.td
+++ b/lld/MinGW/Options.td
@@ -202,6 +202,7 @@ def _HASH_HASH_HASH : Flag<["-"], "###">,
     HelpText<"Print (but do not run) the commands to run for this compilation">;
 def appcontainer: F<"appcontainer">, HelpText<"Set the appcontainer flag in the executable">;
 defm delayload: Eq<"delayload", "DLL to load only on demand">;
+defm dependent_load_flag: EEq<"dependent-load-flag", "Override default LibraryLoad flags">;
 defm mllvm: EqNoHelp<"mllvm">;
 defm pdb: Eq<"pdb", "Output PDB debug info file, chosen implicitly if the argument is empty">;
 defm Xlink : Eq<"Xlink", "Pass <arg> to the COFF linker">, MetaVarName<"<arg>">;
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index e8b27aaf84058..0c24156ae1679 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -35,10 +35,19 @@ ELF Improvements
   globally changing the semantics of section matching. It also independently
   increases the expressive power of linker scripts.
   (`#95323 <https://github.com/llvm/llvm-project/pull/95323>`_)
+* Supported relocation types for x86-64 target:
+  * ``R_X86_64_CODE_4_GOTPCRELX`` (`#109783 <https://github.com/llvm/llvm-project/pull/109783>`_) (`#116737 <https://github.com/llvm/llvm-project/pull/116737>`_)
+  * ``R_X86_64_CODE_4_GOTTPOFF`` (`#116634 <https://github.com/llvm/llvm-project/pull/116634>`_)
+  * ``R_X86_64_CODE_4_GOTPC32_TLSDESC`` (`#116909 <https://github.com/llvm/llvm-project/pull/116909>`_)
+  * ``R_X86_64_CODE_6_GOTTPOFF``  (`#117675 <https://github.com/llvm/llvm-project/pull/117675>`_)
 
 Breaking changes
 ----------------
 
+* Removed support for the (deprecated) `R_RISCV_RVC_LUI` relocation. This
+  was a binutils-internal relocation used during relaxation, and was not
+  emitted by compilers/assemblers.
+
 COFF Improvements
 -----------------
 * ``/includeglob`` has been implemented to match the behavior of ``--undefined-glob`` available for ELF.
diff --git a/lld/include/lld/Common/ErrorHandler.h b/lld/include/lld/Common/ErrorHandler.h
index 79e20be2bb6be..4db1fec268d76 100644
--- a/lld/include/lld/Common/ErrorHandler.h
+++ b/lld/include/lld/Common/ErrorHandler.h
@@ -167,6 +167,7 @@ class SyncStream {
   SyncStream(SyncStream &&o) : e(o.e), level(o.level), buf(std::move(o.buf)) {}
   ~SyncStream();
   StringRef str() { return os.str(); }
+  uint64_t tell() { return os.tell(); }
 };
 
 [[noreturn]] void exitLld(int val);
diff --git a/lld/test/COFF/arm64x-loadconfig.s b/lld/test/COFF/arm64x-loadconfig.s
new file mode 100644
index 0000000000000..0d4fe0ed6d6e0
--- /dev/null
+++ b/lld/test/COFF/arm64x-loadconfig.s
@@ -0,0 +1,68 @@
+// REQUIRES: aarch64
+// RUN: split-file %s %t.dir && cd %t.dir
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows test.s -o test.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows loadconfig.s -o loadconfig.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows loadconfig-short.s -o loadconfig-short.obj
+
+// RUN: lld-link -machine:arm64x -out:out.dll -dll -noentry loadconfig.obj test.obj
+
+// RUN: llvm-readobj --coff-load-config out.dll | FileCheck -check-prefix=DYNRELOCS %s
+// DYNRELOCS:      DynamicValueRelocTableOffset: 0xC
+// DYNRELOCS-NEXT: DynamicValueRelocTableSection: 4
+// DYNRELOCS:      DynamicRelocations [
+// DYNRELOCS-NEXT:   Version: 0x1
+// DYNRELOCS-NEXT:   Arm64X [
+// DYNRELOCS-NEXT:     Entry [
+// DYNRELOCS-NEXT:       RVA: 0x7C
+// DYNRELOCS-NEXT:       Type: VALUE
+// DYNRELOCS-NEXT:       Size: 0x2
+// DYNRELOCS-NEXT:       Value: 0x8664
+// DYNRELOCS-NEXT:     ]
+// DYNRELOCS-NEXT:     Entry [
+// DYNRELOCS-NEXT:       RVA: 0x150
+// DYNRELOCS-NEXT:       Type: VALUE
+// DYNRELOCS-NEXT:       Size: 0x4
+// DYNRELOCS-NEXT:       Value: 0x0
+// DYNRELOCS-NEXT:     ]
+// DYNRELOCS-NEXT:     Entry [
+// DYNRELOCS-NEXT:       RVA: 0x154
+// DYNRELOCS-NEXT:       Type: VALUE
+// DYNRELOCS-NEXT:       Size: 0x4
+// DYNRELOCS-NEXT:       Value: 0x0
+// DYNRELOCS-NEXT:     ]
+// DYNRELOCS-NEXT:   ]
+// DYNRELOCS-NEXT: ]
+
+// RUN: llvm-readobj --headers out.dll | FileCheck -check-prefix=HEADERS %s
+// HEADERS:      BaseRelocationTableRVA: 0x4000
+// HEADERS-NEXT: BaseRelocationTableSize: 0xC
+// HEADERS:      LoadConfigTableRVA: 0x1000
+// HEADERS-NEXT: LoadConfigTableSize: 0x140
+// HEADERS:      Name: .reloc (2E 72 65 6C 6F 63 00 00)
+// HEADERS-NEXT: VirtualSize: 0x38
+
+// RUN: lld-link -machine:arm64x -out:out-short.dll -dll -noentry loadconfig-short.obj 2>&1 | FileCheck --check-prefix=WARN-RELOC-SIZE %s
+// WARN-RELOC-SIZE: lld-link: warning: '_load_config_used' structure too small to include dynamic relocations
+
+#--- test.s
+        .data
+sym:
+        // Emit a basereloc to make the loadconfig test more meaningful.
+        .xword sym
+
+#--- loadconfig.s
+        .section .rdata,"dr"
+        .globl _load_config_used
+        .p2align 3, 0
+_load_config_used:
+        .word 0x140
+        .fill 0x13c,1,0
+
+#--- loadconfig-short.s
+        .section .rdata,"dr"
+        .globl _load_config_used
+        .p2align 3, 0
+_load_config_used:
+        .word 0xe4
+        .fill 0xe0,1,0
diff --git a/lld/test/COFF/dependentflags.test b/lld/test/COFF/dependentflags.test
index 3e90511591c75..2ea974c268d5c 100644
--- a/lld/test/COFF/dependentflags.test
+++ b/lld/test/COFF/dependentflags.test
@@ -20,6 +20,10 @@ RUN: not lld-link %S/Inputs/precomp-a.obj %t.ldcfg.obj /out:%t.exe /nodefaultlib
 RUN: not lld-link %S/Inputs/precomp-a.obj %t.ldcfg.obj /out:%t.exe /nodefaultlib /force /dependentloadflag:zz 2>&1 | FileCheck %s --check-prefix FAIL
 RUN: not lld-link %S/Inputs/precomp-a.obj %t.ldcfg.obj /out:%t.exe /nodefaultlib /force /dependentloadflag:0xf0000 2>&1 | FileCheck %s --check-prefix FAIL-RANGE
 
+// ---- Warn if _load_config_used is missing
+RUN: llvm-lib %t.ldcfg.obj /out:%t.lib
+RUN: lld-link %S/Inputs/precomp-a.obj /out:%t.exe /nodefaultlib /force /dependentloadflag:0x800 2>&1 | FileCheck %s --check-prefix WARN-NOBASE
+RUN: lld-link %S/Inputs/precomp-a.obj %t.lib /out:%t.exe /nodefaultlib /force /dependentloadflag:0x800 2>&1 | FileCheck %s --allow-empty --check-prefix NO-WARN
 
 BASE: DependentLoadFlags: 0x0
 FLAGS-800: DependentLoadFlags: 0x800
@@ -29,3 +33,5 @@ FAIL: lld-link: error: /dependentloadflag: invalid argument: zz
 FAIL-RANGE: lld-link: error: /dependentloadflag: invalid argument: 0xf0000
 FAIL-NOARG: lld-link: error: /dependentloadflag: no argument specified
 
+WARN-NOBASE: lld-link: warning: _load_config_used not found, /dependentloadflag will have no effect
+NO-WARN-NOT: lld-link: warning: _load_config_used not found, /dependentloadflag will have no effect
diff --git a/lld/test/COFF/deploadflag-cfg-short.s b/lld/test/COFF/deploadflag-cfg-short.s
new file mode 100644
index 0000000000000..9cc5248044d8a
--- /dev/null
+++ b/lld/test/COFF/deploadflag-cfg-short.s
@@ -0,0 +1,12 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -triple x86_64-windows-msvc -filetype=obj %s -o %t.obj
+# RUN: lld-link %t.obj -out:%t.dll -dll -noentry -nodefaultlib -dependentloadflag:0x800 2>&1 | FileCheck %s
+# CHECK: lld-link: warning: '_load_config_used' structure too small to include DependentLoadFlags
+
+        .section .rdata,"dr"
+        .balign 8
+.globl _load_config_used
+_load_config_used:
+        .long 0x4c
+        .fill 0x48, 1, 0
diff --git a/lld/test/ELF/arm-thumb-thunk-v6m-noshort.s b/lld/test/ELF/arm-thumb-thunk-v6m-noshort.s
new file mode 100644
index 0000000000000..43d6fc27f55c9
--- /dev/null
+++ b/lld/test/ELF/arm-thumb-thunk-v6m-noshort.s
@@ -0,0 +1,57 @@
+// REQUIRES: arm
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: llvm-mc -filetype=obj -arm-add-build-attributes -triple=armv6m-none-eabi asm -o v6m.o
+// RUN: ld.lld --script=lds v6m.o -o v6m
+// RUN: llvm-objdump --no-print-imm-hex --no-show-raw-insn -d v6m --triple=armv6m-none-eabi | FileCheck %s
+
+// RUN: llvm-mc -filetype=obj -arm-add-build-attributes -triple=armv8m.base-none-eabi asm -o v8m.o
+// RUN: ld.lld --script=lds v8m.o -o v8m
+// RUN: llvm-objdump --no-print-imm-hex --no-show-raw-insn -d v8m --triple=armv8m.base-none-eabi | FileCheck --check-prefix=CHECKV8BASE %s
+
+/// Test that short thunks are not generated for v6-m as this architecture
+/// does not have the B.w instruction.
+
+//--- asm
+ .syntax unified
+
+ .section .text_low, "ax", %progbits
+ .thumb
+ .type _start, %function
+ .balign 4
+ .globl _start
+_start:
+ bl far
+ .space 0x1000 - (. - _start)
+
+/// Thunks will be inserted here. They are in short thunk range for a B.w
+/// instruction. Expect v6-M to use a long thunk as v6-M does not have B.w.
+/// Expect v8-m.base to use a short thunk as despite not having Thumb 2 it
+/// does have B.w.
+
+// CHECK-LABEL: <__Thumbv6MABSLongThunk_far>:
+// CHECK-NEXT: 2000: push    {r0, r1}
+// CHECK-NEXT:       ldr     r0, [pc, #4]
+// CHECK-NEXT:       str     r0, [sp, #4]
+// CHECK-NEXT:       pop     {r0, pc}
+// CHECK-NEXT:   01 20 00 01   .word   0x01002001
+
+// CHECKV8BASE-LABEL: <__Thumbv7ABSLongThunk_far>:
+// CHECKV8BASE-NEXT: 2000: b.w     0x1002000 <far>
+
+ .section .text_high, "ax", %progbits
+ .globl far
+ .type far, %function
+ .balign 4
+far:
+ bx lr
+
+//--- lds
+
+PHDRS {
+  low PT_LOAD FLAGS(0x1 | 0x4);
+  high PT_LOAD FLAGS(0x1 | 0x4);
+}
+SECTIONS {
+  .text_low  0x1000 : { *(.text_low) }
+  .text_high 0x1002000  : { *(.text_high) }
+}
diff --git a/lld/test/ELF/export-dynamic-symbol.s b/lld/test/ELF/export-dynamic-symbol.s
index 3a00f07a420cf..66050555e0c3d 100644
--- a/lld/test/ELF/export-dynamic-symbol.s
+++ b/lld/test/ELF/export-dynamic-symbol.s
@@ -10,15 +10,16 @@
 # so that we have time to investigate the issue without losing postcommit CI.
 # UNSUPPORTED: system-windows
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o %t.o
 
 ## For an executable, --export-dynamic-symbol exports a symbol if it is non-local and defined.
-# RUN: ld.lld -pie --export-dynamic-symbol foo --export-dynamic-symbol qux %t.o -o %t
-# RUN: llvm-nm -D -p %t | FileCheck %s
+# RUN: ld.lld -pie --export-dynamic-symbol foo --export-dynamic-symbol qux %t.o -o out
+# RUN: llvm-nm -D -p out | FileCheck %s
 # RUN: echo '{ foo; };' > %t1.list
 # RUN: echo '{ foo; qux; };' > %t2.list
-# RUN: ld.lld -pie --export-dynamic-symbol-list=%t2.list %t.o -o %t
-# RUN: llvm-nm -D -p %t | FileCheck %s
+# RUN: ld.lld -pie --export-dynamic-symbol-list=%t2.list %t.o -o out
+# RUN: llvm-nm -D -p out | FileCheck %s
 
 ## --export-dynamic exports all non-local defined symbols.
 ## --export-dynamic-symbol is shadowed.
@@ -53,6 +54,20 @@
 # RUN: ld.lld -shared -Bsymbolic --export-dynamic-symbol-list %t1.list %t.o -o %t.preempt
 # RUN: llvm-objdump -d %t.preempt | FileCheck --check-prefix=PLT1 %s
 
+## Hidden symbols cannot be exported by --export-dynamic-symbol family options.
+# RUN: llvm-mc -filetype=obj -triple=x86_64 hidden.s -o hidden.o
+# RUN: ld.lld -pie %t.o hidden.o --dynamic-list hidden.list -o out.hidden
+# RUN: llvm-readelf -s out.hidden | FileCheck %s --check-prefix=HIDDEN
+
+# HIDDEN:      '.dynsym' contains 2 entries:
+# HIDDEN:      NOTYPE GLOBAL DEFAULT [[#]] _end
+# HIDDEN:      '.symtab' contains 6 entries:
+# HIDDEN:      FUNC    LOCAL  HIDDEN  [[#]] foo
+# HIDDEN-NEXT: NOTYPE  LOCAL  HIDDEN  [[#]] _DYNAMIC
+# HIDDEN-NEXT: NOTYPE  GLOBAL DEFAULT [[#]] _start
+# HIDDEN-NEXT: FUNC    GLOBAL DEFAULT [[#]] qux
+# HIDDEN-NEXT: NOTYPE  GLOBAL DEFAULT [[#]] _end
+
 ## 'nomatch' does not match any symbol. Don't warn.
 # RUN: ld.lld --fatal-warnings -shared -Bsymbolic-functions --export-dynamic-symbol nomatch %t.o -o %t.nopreempt2
 # RUN: llvm-objdump -d %t.nopreempt2 | FileCheck --check-prefix=NOPLT %s
@@ -80,6 +95,7 @@
 # NOPLT-NOT: <foo@plt>
 # NOPLT-NOT: <qux@plt>
 
+#--- a.s
 .global _start, foo, qux
 .type foo, @function
 .type qux, @function
@@ -88,3 +104,13 @@ _start:
   call qux
 foo:
 qux:
+
+#--- hidden.s
+.hidden foo
+
+.data
+.quad _DYNAMIC
+.quad _end
+
+#--- hidden.list
+{foo;_end;_DYNAMIC;};
diff --git a/lld/test/ELF/invalid-eh-frame3.s b/lld/test/ELF/invalid-eh-frame3.s
index 7cc27fc422294..6f7c7d957ab69 100644
--- a/lld/test/ELF/invalid-eh-frame3.s
+++ b/lld/test/ELF/invalid-eh-frame3.s
@@ -4,7 +4,7 @@
 # RUN: not ld.lld --eh-frame-hdr %t -o /dev/null 2>&1 | FileCheck %s
 
 # CHECK:      error: corrupted .eh_frame: corrupted CIE (failed to read LEB128)
-# CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0xC)
+# CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0xc)
 
 .section .eh_frame,"a",@unwind
   .byte 0x08
diff --git a/lld/test/ELF/invalid-eh-frame6.s b/lld/test/ELF/invalid-eh-frame6.s
index 68b79c725d0c6..77be15f54e6b1 100644
--- a/lld/test/ELF/invalid-eh-frame6.s
+++ b/lld/test/ELF/invalid-eh-frame6.s
@@ -4,7 +4,7 @@
 # RUN: not ld.lld --eh-frame-hdr %t -o /dev/null 2>&1 | FileCheck %s
 
 # CHECK: error: corrupted .eh_frame: unknown FDE encoding
-# CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0xE)
+# CHECK-NEXT: >>> defined in {{.*}}:(.eh_frame+0xe)
 
 .section .eh_frame,"a",@unwind
   .byte 0x0E
diff --git a/lld/test/ELF/invalid/broken-relaxation-x64.test b/lld/test/ELF/invalid/broken-relaxation-x64.test
index 97a977e2c03ac..daa0b38ec3971 100644
--- a/lld/test/ELF/invalid/broken-relaxation-x64.test
+++ b/lld/test/ELF/invalid/broken-relaxation-x64.test
@@ -1,15 +1,15 @@
 # REQUIRES: x86
 
-# RUN: yaml2obj %s -o %t.o
-# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
-# ERR: R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only
-# ERR: R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only
+# RUN: yaml2obj --docnum=1 %s -o %t1.o
+# RUN: not ld.lld %t1.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
+# ERR: error: {{.*}}: R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only
+# ERR: error: {{.*}}: R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only
 
 ## YAML below contains 2 relocations of type R_X86_64_GOTTPOFF, and a .text
 ## with fake content filled by 0xFF. That means instructions for relaxation are
 ## "broken", so they does not match any known valid relaxations. We also generate
 ## .tls section because we need it for correct processing of STT_TLS symbol.
-!ELF
+--- !ELF
 FileHeader:
   Class:           ELFCLASS64
   Data:            ELFDATA2LSB
@@ -44,4 +44,131 @@ Symbols:
     Value:    0x12345
     Size:     4
     Binding:  STB_GLOBAL
- 
+
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: not ld.lld %t2.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR2 %s
+# ERR2: error: {{.*}}: invalid prefix with R_X86_64_CODE_4_GOTTPOFF!
+# ERR2: error: {{.*}}: invalid prefix with R_X86_64_CODE_6_GOTTPOFF!
+
+## YAML below contains 2 relocations of
+## R_X86_64_CODE_4_GOTTPOFF/R_X86_64_CODE_6_GOTTPOFF type, and a .text with
+## fake content filled by 0xFF. It's expected to get "invalid prefix" error
+##  message as above.
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  OSABI:           ELFOSABI_FREEBSD
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Type:            SHT_PROGBITS
+    Name:            .text
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x04
+    Content:         "FFFFFFFFFFFFFFFFFFFF"
+  - Type:            SHT_PROGBITS
+    Name:            .tls
+    Flags:           [ SHF_ALLOC, SHF_TLS ]
+  - Type:            SHT_REL
+    Name:            .rel.text
+    Link:            .symtab
+    Info:            .text
+    AddressAlign:    0x04
+    Relocations:
+      - Offset:          4
+        Symbol:          foo
+        Type:            R_X86_64_CODE_4_GOTTPOFF
+      - Offset:          6
+        Symbol:          foo
+        Type:            R_X86_64_CODE_6_GOTTPOFF
+Symbols:
+  - Name:     foo
+    Type:     STT_TLS
+    Section:  .text
+    Value:    0x12345
+    Size:     4
+    Binding:  STB_GLOBAL
+
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: not ld.lld %t3.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR3 %s
+# ERR3: error: {{.*}}: R_X86_64_CODE_4_GOTTPOFF must be used in MOVQ or ADDQ instructions only
+
+## YAML below contains R_X86_64_CODE_4_GOTTPOFF relocation type, and a .text
+## with fake content filled by 0xd5, 0xFF, ... and 0xFF. It's expected to get
+## the error message as above.
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  OSABI:           ELFOSABI_FREEBSD
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Type:            SHT_PROGBITS
+    Name:            .text
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x04
+    Content:         "d5FFFFFFFFFFFFFFFFFF"
+  - Type:            SHT_PROGBITS
+    Name:            .tls
+    Flags:           [ SHF_ALLOC, SHF_TLS ]
+  - Type:            SHT_REL
+    Name:            .rel.text
+    Link:            .symtab
+    Info:            .text
+    AddressAlign:    0x04
+    Relocations:
+      - Offset:          4
+        Symbol:          foo
+        Type:            R_X86_64_CODE_4_GOTTPOFF
+Symbols:
+  - Name:     foo
+    Type:     STT_TLS
+    Section:  .text
+    Value:    0x12345
+    Size:     4
+    Binding:  STB_GLOBAL
+
+
+# RUN: yaml2obj --docnum=4 %s -o %t4.o
+# RUN: not ld.lld %t4.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR4 %s
+# ERR4: error: {{.*}}: R_X86_64_CODE_6_GOTTPOFF must be used in ADDQ instructions with NDD/NF/NDD+NF only
+
+## YAML below contains R_X86_64_CODE_6_GOTTPOFF relocation type, and a .text
+## with fake content filled by 0x62, 0xFF, ... and 0xFF. It's expected to get
+## the error message as above.
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  OSABI:           ELFOSABI_FREEBSD
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Type:            SHT_PROGBITS
+    Name:            .text
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x04
+    Content:         "62FFFFFFFFFFFFFFFFFF"
+  - Type:            SHT_PROGBITS
+    Name:            .tls
+    Flags:           [ SHF_ALLOC, SHF_TLS ]
+  - Type:            SHT_REL
+    Name:            .rel.text
+    Link:            .symtab
+    Info:            .text
+    AddressAlign:    0x04
+    Relocations:
+      - Offset:          6
+        Symbol:          foo
+        Type:            R_X86_64_CODE_6_GOTTPOFF
+Symbols:
+  - Name:     foo
+    Type:     STT_TLS
+    Section:  .text
+    Value:    0x12345
+    Size:     4
+    Binding:  STB_GLOBAL
diff --git a/lld/test/ELF/linkerscript/i386-sections-max-va-overflow.s b/lld/test/ELF/linkerscript/i386-sections-max-va-overflow.s
index 25bef1575cce6..80b02be7dbb80 100644
--- a/lld/test/ELF/linkerscript/i386-sections-max-va-overflow.s
+++ b/lld/test/ELF/linkerscript/i386-sections-max-va-overflow.s
@@ -7,7 +7,7 @@
 
 ## .bar section has data in [0xfffffff1, 0xfffffff1 + 0x10] == [0xfffffff1, 0x1].
 ## Check we can catch this overflow.
-# ERR: error: section .bar at 0xFFFFFFF1 of size 0x10 exceeds available address space
+# ERR: error: section .bar at 0xfffffff1 of size 0x10 exceeds available address space
 
 ## [0xfffffff1, 0x100000000) is allowed.
 # RUN: echo "SECTIONS { . = 0xfffffff0;" > %t.script
diff --git a/lld/test/ELF/linkerscript/locationcountererr-arm-exidx.test b/lld/test/ELF/linkerscript/locationcountererr-arm-exidx.test
index 2ce35494ca148..c82a93efc1aae 100644
--- a/lld/test/ELF/linkerscript/locationcountererr-arm-exidx.test
+++ b/lld/test/ELF/linkerscript/locationcountererr-arm-exidx.test
@@ -12,9 +12,9 @@
 # ERR-NEXT: error: a.t:10: unable to move location counter (0x2000) backward to 0x1f6c for section 'dummy2'
 # ERR-NEXT: error: a.t:14: unable to move location counter (0x4104) backward to 0x4070 for section 'code.unused_space'
 # ERR-NEXT: error: section '.ARM.exidx' will not fit in region 'CODE': overflowed by 148 bytes
-# ERR-NEXT: error: section dummy1 at 0x1000 of size 0xFFFFFFFFFFFFFF6C exceeds available address space
-# ERR-NEXT: error: section dummy2 at 0x2000 of size 0xFFFFFFFFFFFFFF6C exceeds available address space
-# ERR-NEXT: error: section code.unused_space at 0x4104 of size 0xFFFFFFFFFFFFFF6C exceeds available address space
+# ERR-NEXT: error: section dummy1 at 0x1000 of size 0xffffffffffffff6c exceeds available address space
+# ERR-NEXT: error: section dummy2 at 0x2000 of size 0xffffffffffffff6c exceeds available address space
+# ERR-NEXT: error: section code.unused_space at 0x4104 of size 0xffffffffffffff6c exceeds available address space
 
 ## If we merge adjacent duplicate entries, we will have enough space. Don't report
 ## a spurious error https://github.com/llvm/llvm-project/issues/66836
diff --git a/lld/test/ELF/linkerscript/section-class.test b/lld/test/ELF/linkerscript/section-class.test
index 7fce13bfe3e02..5c30b6343653e 100644
--- a/lld/test/ELF/linkerscript/section-class.test
+++ b/lld/test/ELF/linkerscript/section-class.test
@@ -310,6 +310,7 @@ SECTIONS {
 .byte 0x12, 0x34
 
 .section .b,"aM",@progbits,1
+.p2align 3
 .byte 0x12
 
 # RUN: llvm-mc -n -filetype=obj -triple=x86_64 merge.s -o merge.o
@@ -317,10 +318,11 @@ SECTIONS {
 #--- spill-merge.lds
 ## SHF_MERGE sections are spilled according to the class refs of the first
 ## merged input section (the one giving the resulting section its name).
+## Spills take into account increases in section alignment due to merging.
 MEMORY {
   a : ORIGIN = 0, LENGTH = 1
-  b : ORIGIN = 1, LENGTH = 2
-  c : ORIGIN = 3, LENGTH = 2
+  b : ORIGIN = 1, LENGTH = 16
+  c : ORIGIN = 17, LENGTH = 16
 }
 
 SECTIONS {
@@ -332,12 +334,14 @@ SECTIONS {
 }
 
 # RUN: ld.lld -T spill-merge.lds merge.o -o spill-merge
-# RUN: llvm-readelf -S spill-merge | FileCheck %s --check-prefix=SPILL-MERGE
+# RUN: llvm-readelf -S -x .second spill-merge | FileCheck %s --check-prefix=SPILL-MERGE
 
-# SPILL-MERGE:      Name          Type     Address          Off    Size
+# SPILL-MERGE:      Name    Type     Address          Off    Size
 # SPILL-MERGE:      .first  PROGBITS 0000000000000000 000190 000000
-# SPILL-MERGE-NEXT: .second PROGBITS 0000000000000001 001001 000002
-# SPILL-MERGE-NEXT: .third  PROGBITS 0000000000000003 001003 000000
+# SPILL-MERGE-NEXT: .second PROGBITS 0000000000000008 001008 000009
+# SPILL-MERGE-NEXT: .third  PROGBITS 0000000000000018 001018 000000
+# SPILL-MERGE:      Hex dump of section '.second':
+# SPILL-MERGE-NEXT: 0x00000008 12000000 00000000 34 .
 
 #--- link-order.s
 .section .a,"a",@progbits
diff --git a/lld/test/ELF/linkerscript/sections-max-va-overflow.s b/lld/test/ELF/linkerscript/sections-max-va-overflow.s
index ce771b4784c44..a60da8dfb8804 100644
--- a/lld/test/ELF/linkerscript/sections-max-va-overflow.s
+++ b/lld/test/ELF/linkerscript/sections-max-va-overflow.s
@@ -7,7 +7,7 @@
 
 ## .bar section has data in [0xfffffffffffffff1, 0xfffffffffffffff1 + 0x10] ==
 ## [0xfffffffffffffff1, 0x1]. Check we can catch this overflow.
-# ERR: error: section .bar at 0xFFFFFFFFFFFFFFF1 of size 0x10 exceeds available address space
+# ERR: error: section .bar at 0xfffffffffffffff1 of size 0x10 exceeds available address space
 
 .section .bar,"ax",@progbits
 .zero 0x10
diff --git a/lld/test/ELF/lto/Inputs/internalize-exportdyn.ll b/lld/test/ELF/lto/Inputs/internalize-exportdyn.ll
index 585b99ae5a513..5bfe15c92b3f1 100644
--- a/lld/test/ELF/lto/Inputs/internalize-exportdyn.ll
+++ b/lld/test/ELF/lto/Inputs/internalize-exportdyn.ll
@@ -1,6 +1,31 @@
 target triple = "x86_64-unknown-linux-gnu"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
+$ext_and_ext = comdat any
+$lo_and_ext = comdat any
+$lo_and_wo = comdat any
+$wo_and_lo = comdat any
+
+declare void @foo(i64)
+
 define weak_odr void @bah() {
   ret void
 }
+
+define void @ext_and_ext() local_unnamed_addr comdat {
+  call void @foo(i64 2)
+  ret void
+}
+
+define linkonce_odr void @lo_and_ext() local_unnamed_addr comdat {
+  call void @foo(i64 2)
+  ret void
+}
+
+define weak_odr void @lo_and_wo() local_unnamed_addr comdat {
+  ret void
+}
+
+define linkonce_odr void @wo_and_lo() local_unnamed_addr comdat {
+  ret void
+}
diff --git a/lld/test/ELF/lto/internalize-exportdyn.ll b/lld/test/ELF/lto/internalize-exportdyn.ll
index f02d3b375dad5..7c670a548025f 100644
--- a/lld/test/ELF/lto/internalize-exportdyn.ll
+++ b/lld/test/ELF/lto/internalize-exportdyn.ll
@@ -1,14 +1,22 @@
 ; REQUIRES: x86
-; RUN: llvm-as %s -o %t.o
-; RUN: llvm-as %p/Inputs/internalize-exportdyn.ll -o %t2.o
-; RUN: ld.lld %t.o %t2.o -o %t2 --export-dynamic -save-temps
-; RUN: llvm-dis < %t2.0.2.internalize.bc | FileCheck %s
-; RUN: ld.lld %t.o %t2.o -o %t3 -shared -save-temps
-; RUN: llvm-dis < %t3.0.2.internalize.bc | FileCheck %s --check-prefix=DSO
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: llvm-as a.ll -o a.bc
+; RUN: llvm-as %p/Inputs/internalize-exportdyn.ll -o b.bc
+; RUN: llvm-mc -filetype=obj -triple=x86_64 lib.s -o lib.o
+; RUN: ld.lld a.bc b.bc lib.o -o out --export-dynamic -save-temps
+; RUN: llvm-dis < out.0.2.internalize.bc | FileCheck %s
+; RUN: ld.lld a.bc b.bc lib.o -o out2 -shared -save-temps
+; RUN: llvm-dis < out2.0.2.internalize.bc | FileCheck %s --check-prefix=DSO
 
+;--- a.ll
 target triple = "x86_64-unknown-linux-gnu"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
+$ext_and_ext = comdat any
+$lo_and_ext = comdat any
+$lo_and_wo = comdat any
+$wo_and_lo = comdat any
+
 @c = linkonce_odr constant i32 1
 @g = linkonce_odr global i32 1
 @u_c = linkonce_odr unnamed_addr constant i32 1
@@ -16,6 +24,8 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 @lu_c = linkonce_odr local_unnamed_addr constant i32 1
 @lu_g = linkonce_odr local_unnamed_addr global i32 1
 
+declare void @lib(i64)
+
 define void @_start() {
   ret void
 }
@@ -46,6 +56,24 @@ define linkonce_odr void @baz() {
 
 @use_baz = global ptr @baz
 
+define void @ext_and_ext() local_unnamed_addr comdat {
+  call void @foo(i64 1)
+  ret void
+}
+
+define linkonce_odr void @lo_and_ext() local_unnamed_addr comdat {
+  call void @foo(i64 1)
+  ret void
+}
+
+define linkonce_odr void @lo_and_wo() local_unnamed_addr comdat {
+  ret void
+}
+
+define weak_odr void @wo_and_lo() local_unnamed_addr comdat {
+  ret void
+}
+
 ; Check what gets internalized.
 ; CHECK: @c = weak_odr dso_local constant i32 1
 ; CHECK: @g = weak_odr dso_local global i32 1
@@ -60,6 +88,12 @@ define linkonce_odr void @baz() {
 ; CHECK: define internal void @zed2()
 ; CHECK: define weak_odr dso_local void @bah()
 ; CHECK: define weak_odr dso_local void @baz()
+; CHECK: define dso_local void @ext_and_ext() comdat
+; CHECK-NEXT: call void @foo(i64 1)
+; CHECK: define internal void @lo_and_ext() comdat
+; CHECK-NEXT: call void @foo(i64 1)
+; CHECK: define internal void @lo_and_wo() comdat
+; CHECK: define weak_odr dso_local void @wo_and_lo() comdat
 
 ; DSO: @c = weak_odr constant i32 1
 ; DSO: @g = weak_odr global i32 1
@@ -74,3 +108,11 @@ define linkonce_odr void @baz() {
 ; DSO: define internal void @zed2()
 ; DSO: define weak_odr void @bah()
 ; DSO: define weak_odr void @baz()
+; DSO: define void @ext_and_ext() comdat
+; DSO: define internal void @lo_and_ext() comdat
+; DSO: define internal void @lo_and_wo() comdat
+; DSO: define weak_odr void @wo_and_lo() comdat
+
+;--- lib.s
+.globl lib
+lib:
diff --git a/lld/test/ELF/pack-dyn-relocs-ifunc.s b/lld/test/ELF/pack-dyn-relocs-ifunc.s
index 6168d06f99d9e..b6000df3d6160 100644
--- a/lld/test/ELF/pack-dyn-relocs-ifunc.s
+++ b/lld/test/ELF/pack-dyn-relocs-ifunc.s
@@ -47,3 +47,28 @@ _start:
 .globl bar
 bar:
   ret
+
+#--- c.s
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-android c.s -o c.o
+# RUN: ld.lld --pack-dyn-relocs=android c.o -o c
+# RUN: llvm-readelf -sS c | FileCheck --check-prefix=STATIC %s
+
+# STATIC: .rela.plt         RELA            0000000000200158 000158 000018 18  AI  0   5  8
+# STATIC: 0000000000200158     0 NOTYPE  LOCAL  HIDDEN      1 __rela_iplt_start
+# STATIC: 0000000000200170     0 NOTYPE  LOCAL  HIDDEN      1 __rela_iplt_end
+
+.text
+.type foo, %gnu_indirect_function
+.globl foo
+foo:
+  ret
+
+.globl _start
+_start:
+  bl foo
+
+.data
+.balign 8
+.quad __rela_iplt_start
+.quad __rela_iplt_end
diff --git a/lld/test/ELF/pack-dyn-relocs-tls-x86-64.s b/lld/test/ELF/pack-dyn-relocs-tls-x86-64.s
index c6464b4bece09..c54d6f705080d 100644
--- a/lld/test/ELF/pack-dyn-relocs-tls-x86-64.s
+++ b/lld/test/ELF/pack-dyn-relocs-tls-x86-64.s
@@ -13,6 +13,7 @@
 foo:
     movq    tlsvar@GOTTPOFF(%rip), %rcx
     movq    tlsvar2@GOTTPOFF(%rip), %r31
+    addq    tlsvar3@GOTTPOFF(%rip), %rcx, %r16
 
 
     .section    .tdata,"awT",@progbits
@@ -21,7 +22,11 @@ tlsvar:
     .word   42
 tlsvar2:
     .word   42
+tlsvar3:
+    .word   42
+
 // CHECK:          Section ({{.+}}) .rela.dyn {
 // CHECK-NEXT:     R_X86_64_TPOFF64 - 0x1234
 // CHECK-NEXT:     R_X86_64_TPOFF64 - 0x1236
+// CHECK-NEXT:     R_X86_64_TPOFF64 - 0x1238
 // CHECK-NEXT:     }
diff --git a/lld/test/ELF/systemz-gotent-relax-und-dso.s b/lld/test/ELF/systemz-gotent-relax-und-dso.s
index 5a1bd7f949f89..e8b88056299cb 100644
--- a/lld/test/ELF/systemz-gotent-relax-und-dso.s
+++ b/lld/test/ELF/systemz-gotent-relax-und-dso.s
@@ -14,9 +14,9 @@
 # DISASM:      Disassembly of section .text:
 # DISASM-EMPTY:
 # DISASM-NEXT: <foo>:
-# DISASM-NEXT:     nop     0
+# DISASM-NEXT:     nop
 # DISASM:      <hid>:
-# DISASM-NEXT:     nop     0
+# DISASM-NEXT:     nop
 # DISASM:      <_start>:
 # DISASM-NEXT:    lgrl    %r1, 0x2400
 # DISASM-NEXT:    lgrl    %r1, 0x2400
diff --git a/lld/test/ELF/systemz-gotent-relax.s b/lld/test/ELF/systemz-gotent-relax.s
index e84fd8d4653e9..88b43a4e9d29c 100644
--- a/lld/test/ELF/systemz-gotent-relax.s
+++ b/lld/test/ELF/systemz-gotent-relax.s
@@ -30,9 +30,9 @@
 
 # DISASM:      Disassembly of section .text:
 # DISASM: 00000000010011e0 <foo>:
-# DISASM-NEXT:   nop 0
+# DISASM-NEXT:   nop
 # DISASM: 00000000010011e4 <hid>:
-# DISASM-NEXT:   nop 0
+# DISASM-NEXT:   nop
 # DISASM: 00000000010011e8 <ifunc>:
 # DISASM-NEXT:   br      %r14
 # DISASM: 00000000010011ea <_start>:
diff --git a/lld/test/ELF/systemz-plt.s b/lld/test/ELF/systemz-plt.s
index c7563cd18c274..717343ce4c4d5 100644
--- a/lld/test/ELF/systemz-plt.s
+++ b/lld/test/ELF/systemz-plt.s
@@ -48,9 +48,9 @@
 # DIS-NEXT: 100102c: d2 07 f0 30 10 08    	mvc	48(8,%r15), 8(%r1)
 # DIS-NEXT: 1001032: e3 10 10 10 00 04    	lg	%r1, 16(%r1)
 # DIS-NEXT: 1001038: 07 f1        	br	%r1
-# DIS-NEXT: 100103a: 07 00        	nopr   %r0
-# DIS-NEXT: 100103c: 07 00        	nopr   %r0
-# DIS-NEXT: 100103e: 07 00        	nopr   %r0
+# DIS-NEXT: 100103a: 07 00        	nopr
+# DIS-NEXT: 100103c: 07 00        	nopr
+# DIS-NEXT: 100103e: 07 00        	nopr
 # DIS-NEXT: 1001040: c0 10 00 00 10 54    	larl	%r1, 0x10030e8
 # DIS-NEXT: 1001046: e3 10 10 00 00 04    	lg	%r1, 0(%r1)
 # DIS-NEXT: 100104c: 07 f1        	br	%r1
diff --git a/lld/test/ELF/systemz-tls-gd.s b/lld/test/ELF/systemz-tls-gd.s
index 3976f55a6ae39..742797e2d62e4 100644
--- a/lld/test/ELF/systemz-tls-gd.s
+++ b/lld/test/ELF/systemz-tls-gd.s
@@ -58,17 +58,17 @@
 
 ## TP offset of a is at 0x1002218
 # LE-NEXT: lgrl    %r2, 0x1002218
-# LE-NEXT: brcl    0,
+# LE-NEXT: jgnop
 # LE-NEXT: lgf     %r2, 0(%r2,%r7)
 
 ## TP offset of b is at 0x1002220
 # LE-NEXT: lgrl    %r2, 0x1002220
-# LE-NEXT: brcl    0,
+# LE-NEXT: jgnop
 # LE-NEXT: lgf     %r2, 0(%r2,%r7)
 
 ## TP offset of c is at 0x1002228
 # LE-NEXT: lgrl    %r2, 0x1002228
-# LE-NEXT: brcl    0,
+# LE-NEXT: jgnop
 # LE-NEXT: lgf     %r2, 0(%r2,%r7)
 
 ## TP offsets
@@ -88,7 +88,7 @@
 
 ## TP offset of a is at 0x1002340
 # IE-NEXT: lgrl    %r2, 0x1002340
-# IE-NEXT: brcl    0,
+# IE-NEXT: jgnop
 # IE-NEXT: lgf     %r2, 0(%r2,%r7)
 
 ## GOT offset of the TP offset for b is at 0x1002348
diff --git a/lld/test/ELF/systemz-tls-ld.s b/lld/test/ELF/systemz-tls-ld.s
index 2cb36d7294f2b..ef104b82644ce 100644
--- a/lld/test/ELF/systemz-tls-ld.s
+++ b/lld/test/ELF/systemz-tls-ld.s
@@ -49,7 +49,7 @@
 
 ## GOT offset of the LDM TLS module ID is at 0x1002210
 # LE-NEXT: lgrl    %r2, 0x1002210
-# LE-NEXT: brcl    0,
+# LE-NEXT: jgnop
 # LE-NEXT: la      %r2, 0(%r2,%r7)
 
 ## TP offset for a is at 0x1002218
diff --git a/lld/test/ELF/tls-opt.s b/lld/test/ELF/tls-opt.s
index 818203ee19cb7..08cc52afd6411 100644
--- a/lld/test/ELF/tls-opt.s
+++ b/lld/test/ELF/tls-opt.s
@@ -20,12 +20,25 @@
 // DISASM-NEXT:   leaq -4(%r15), %r15
 // DISASM-NEXT:   addq $-4, %rsp
 // DISASM-NEXT:   addq $-4, %r12
+# EGPR
 // DISASM-NEXT:   movq $-8, %r16
 // DISASM-NEXT:   movq $-8, %r20
 // DISASM-NEXT:   movq $-4, %r16
 // DISASM-NEXT:   addq $-8, %r16
 // DISASM-NEXT:   addq $-8, %r28
 // DISASM-NEXT:   addq $-4, %r16
+# NDD
+// DISASM-NEXT:   addq $-10, %r16, %r16
+// DISASM-NEXT:   addq $-10, %r16, %r20
+// DISASM-NEXT:   addq $-10, %r16, %rax
+// DISASM-NEXT:   addq $-10, %rax, %r16
+// DISASM-NEXT:   addq $-10, %r8, %r16
+// DISASM-NEXT:   addq $-10, %rax, %r12
+# NDD + NF
+// DISASM-NEXT:   {nf} addq $-10, %r8, %r16
+// DISASM-NEXT:   {nf} addq $-10, %rax, %r12
+# NF
+// DISASM-NEXT:   {nf} addq $-10, %r12
 
 // LD to LE:
 // DISASM-NEXT:   movq %fs:0, %rax
@@ -82,6 +95,18 @@ _start:
  addq tls0@GOTTPOFF(%rip), %r16
  addq tls0@GOTTPOFF(%rip), %r28
  addq tls1@GOTTPOFF(%rip), %r16
+ # NDD
+ addq tls0@GOTTPOFF(%rip), %r16, %r16
+ addq tls0@GOTTPOFF(%rip), %r16, %r20
+ addq tls0@GOTTPOFF(%rip), %r16, %rax
+ addq tls0@GOTTPOFF(%rip), %rax, %r16
+ addq %r8, tls0@GOTTPOFF(%rip), %r16
+ addq tls0@GOTTPOFF(%rip), %rax, %r12
+ # NDD + NF
+ {nf} addq %r8, tls0@GOTTPOFF(%rip), %r16
+ {nf} addq tls0@GOTTPOFF(%rip), %rax, %r12
+ # NF
+ {nf} addq tls0@GOTTPOFF(%rip), %r12
 
  // LD to LE
  leaq tls0@tlsld(%rip), %rdi
diff --git a/lld/test/ELF/tls.s b/lld/test/ELF/tls.s
index 4f50e9d29c030..63118905911a7 100644
--- a/lld/test/ELF/tls.s
+++ b/lld/test/ELF/tls.s
@@ -14,7 +14,7 @@
 // ERR-EMPTY:
 // ERR-NEXT:  error: relocation R_X86_64_TPOFF32 against b cannot be used with -shared
 // ERR-NEXT:  defined in {{.*}}
-// ERR-NEXT:  referenced by {{.*}}:(.text+0xC)
+// ERR-NEXT:  referenced by {{.*}}:(.text+0xc)
 // ERR-EMPTY:
 // ERR-NEXT:  error: relocation R_X86_64_TPOFF32 against c cannot be used with -shared
 // ERR-NEXT:  defined in {{.*}}
@@ -22,7 +22,7 @@
 // ERR-EMPTY:
 // ERR-NEXT:  error: relocation R_X86_64_TPOFF32 against d cannot be used with -shared
 // ERR-NEXT:  defined in {{.*}}
-// ERR-NEXT:  referenced by {{.*}}:(.text+0x1C)
+// ERR-NEXT:  referenced by {{.*}}:(.text+0x1c)
 
 .global _start
 _start:
diff --git a/lld/test/ELF/undef-multi.s b/lld/test/ELF/undef-multi.s
index af21693d08913..68f6a9ca79a71 100644
--- a/lld/test/ELF/undef-multi.s
+++ b/lld/test/ELF/undef-multi.s
@@ -9,7 +9,7 @@
 # CHECK-NEXT: >>> referenced by undef-multi.s
 # CHECK-NEXT: >>>               {{.*}}:(.text+0x6)
 # CHECK-NEXT: >>> referenced by undef-multi.s
-# CHECK-NEXT: >>>               {{.*}}:(.text+0xB)
+# CHECK-NEXT: >>>               {{.*}}:(.text+0xb)
 # CHECK-NEXT: >>> referenced 2 more times
 
 # All references to a single undefined symbol count as a single error -- but
@@ -33,7 +33,7 @@
 # LIMIT-NEXT: >>> referenced by undef-multi.s
 # LIMIT-NEXT: >>>               {{.*}}:(.text+0x6)
 # LIMIT-NEXT: >>> referenced by undef-multi.s
-# LIMIT-NEXT: >>>               {{.*}}:(.text+0xB)
+# LIMIT-NEXT: >>>               {{.*}}:(.text+0xb)
 # LIMIT-NEXT: >>> referenced 9 more times
 
 .file "undef-multi.s"
diff --git a/lld/test/ELF/undef.s b/lld/test/ELF/undef.s
index 009b8bf6d23b4..5762d3b7a689c 100644
--- a/lld/test/ELF/undef.s
+++ b/lld/test/ELF/undef.s
@@ -30,11 +30,11 @@
 
 # CHECK:      error: undefined symbol: foo(int)
 # CHECK-NEXT: >>> referenced by undef.s
-# CHECK-NEXT: >>>               {{.*}}:(.text+0x1A)
+# CHECK-NEXT: >>>               {{.*}}:(.text+0x1a)
 
 # CHECK:      error: undefined symbol: Pi
 # CHECK-NEXT: >>> referenced by undef.s
-# CHECK-NEXT: >>>               {{.*}}:(.text+0x1F)
+# CHECK-NEXT: >>>               {{.*}}:(.text+0x1f)
 
 # CHECK:      error: undefined symbol: D main
 # CHECK-NEXT: >>> referenced by undef.s
diff --git a/lld/test/ELF/x86-64-tls-ie-err.s b/lld/test/ELF/x86-64-tls-ie-err.s
new file mode 100644
index 0000000000000..3cc7cb0d41737
--- /dev/null
+++ b/lld/test/ELF/x86-64-tls-ie-err.s
@@ -0,0 +1,35 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck -DFILE=%t.o %s
+
+# CHECK: error: [[FILE]]:(.text+0x2): invalid prefix with R_X86_64_CODE_4_GOTTPOFF!
+# CHECK-NEXT: error: [[FILE]]:(.text+0x8): invalid prefix with R_X86_64_CODE_6_GOTTPOFF!
+# CHECK-NEXT: error: [[FILE]]:(.text+0x12): R_X86_64_CODE_4_GOTTPOFF must be used in MOVQ or ADDQ instructions only
+# CHECK-NEXT: error: [[FILE]]:(.text+0x1a): R_X86_64_CODE_6_GOTTPOFF must be used in ADDQ instructions with NDD/NF/NDD+NF only
+
+## These negative tests are to check if the invalid prefix and unsupported
+## instructions for TLS relocation types with APX instructions are handled as
+## errors.
+
+.type tls0,@object
+.section .tbss,"awT",@nobits
+.globl tls0
+.align 4
+tls0:
+ .long 0
+ .size tls0, 4
+
+.text
+.globl _start
+_start:
+  addq 0(%rip), %rax, %r16
+  .reloc .-4, R_X86_64_CODE_4_GOTTPOFF, tls0-4
+
+  movq 0(%rip), %r16
+  .reloc .-4, R_X86_64_CODE_6_GOTTPOFF, tls0-4
+
+  andq 0(%rip), %r16
+  .reloc .-4, R_X86_64_CODE_4_GOTTPOFF, tls0-4
+
+  andq 0(%rip), %rax, %r16
+  .reloc .-4, R_X86_64_CODE_6_GOTTPOFF, tls0-4
diff --git a/lld/test/ELF/x86-64-tls-ie-local.s b/lld/test/ELF/x86-64-tls-ie-local.s
index 340a654ef9c28..0a104e7d67277 100644
--- a/lld/test/ELF/x86-64-tls-ie-local.s
+++ b/lld/test/ELF/x86-64-tls-ie-local.s
@@ -5,29 +5,47 @@
 # RUN: llvm-readobj -r %t.so | FileCheck --check-prefix=REL %s
 # RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t.so | FileCheck %s
 
-# SEC: .got PROGBITS 0000000000002348 000348 000010 00 WA 0 0 8
+# SEC: .got PROGBITS 0000000000002378 000378 000010 00 WA 0 0 8
 
 ## Dynamic relocations for non-preemptable symbols in a shared object have section index 0.
 # REL:      .rela.dyn {
-# REL-NEXT:   0x2348 R_X86_64_TPOFF64 - 0x0
-# REL-NEXT:   0x2350 R_X86_64_TPOFF64 - 0x4
+# REL-NEXT:   0x2378 R_X86_64_TPOFF64 - 0x0
+# REL-NEXT:   0x2380 R_X86_64_TPOFF64 - 0x4
 # REL-NEXT: }
 
-## &.got[0] - 0x127f = 0x2348 - 0x127f = 4297
-## &.got[1] - 0x1286 = 0x2350 - 0x1286 = 4298
-## &.got[2] - 0x128e = 0x2348 - 0x128e = 4282
-## &.got[3] - 0x1296 = 0x2350 - 0x1296 = 4282
+## &.got[0] - 0x127f = 0x2378 - 0x127f = 4345
+## &.got[1] - 0x1286 = 0x2380 - 0x1286 = 4346
+## &.got[2] - 0x128e = 0x2378 - 0x128e = 4330
+## &.got[3] - 0x1296 = 0x2380 - 0x1296 = 4330
+## &.got[0] - 0x12a0 = 0x2376 - 0x12a0 = 4310
+## &.got[1] - 0x12aa = 0x237e - 0x12aa = 4308
+## &.got[0] - 0x12b4 = 0x2376 - 0x12b4 = 4290
+## &.got[1] - 0x12be = 0x237e - 0x12be = 4288
+## &.got[0] - 0x12c8 = 0x2376 - 0x12c8 = 4270
 
-# CHECK:      1278:       addq 4297(%rip), %rax
-# CHECK-NEXT: 127f:       addq 4298(%rip), %rax
-# CHECK-NEXT: 1286:       addq 4282(%rip), %r16
-# CHECK-NEXT: 128e:       addq 4282(%rip), %r16
+# CHECK:      1278:       addq 4345(%rip), %rax
+# CHECK-NEXT: 127f:       addq 4346(%rip), %rax
+# CHECK-NEXT: 1286:       addq 4330(%rip), %r16
+# CHECK-NEXT: 128e:       addq 4330(%rip), %r16
+# CHECK-NEXT: 1296:       addq %r8, 4310(%rip), %r16
+# CHECK-NEXT: 12a0:       addq 4308(%rip), %rax, %r12
+# CHECK-NEXT: 12aa:       {nf} addq %r8, 4290(%rip), %r16
+# CHECK-NEXT: 12b4:       {nf} addq 4288(%rip), %rax, %r12
+# CHECK-NEXT: 12be:       {nf} addq 4270(%rip), %r12
 
 addq foo@GOTTPOFF(%rip), %rax
 addq bar@GOTTPOFF(%rip), %rax
+# EGPR
 addq foo@GOTTPOFF(%rip), %r16
 addq bar@GOTTPOFF(%rip), %r16
-
+# NDD
+addq %r8, foo@GOTTPOFF(%rip), %r16
+addq bar@GOTTPOFF(%rip), %rax, %r12
+# NDD + NF
+{nf} addq %r8, foo@GOTTPOFF(%rip), %r16
+{nf} addq bar@GOTTPOFF(%rip), %rax, %r12
+# NF
+{nf} addq foo@GOTTPOFF(%rip), %r12
 
 .section .tbss,"awT",@nobits
 foo:
diff --git a/lld/test/MachO/Inputs/liballowable_client.dylib b/lld/test/MachO/Inputs/liballowable_client.dylib
deleted file mode 100755
index 7c174a8a72a4c..0000000000000
Binary files a/lld/test/MachO/Inputs/liballowable_client.dylib and /dev/null differ
diff --git a/lld/test/MachO/allowable-client.s b/lld/test/MachO/allowable-client.s
index 3341dc59c1d81..e87e3fabb262c 100644
--- a/lld/test/MachO/allowable-client.s
+++ b/lld/test/MachO/allowable-client.s
@@ -1,16 +1,28 @@
 # REQUIRES: x86
 # RUN: rm -rf %t; split-file %s %t
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+# RUN: touch %t/empty.s; llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/empty.s -o %t/empty.o
 
-# Check linking against a .dylib
-# RUN: not %lld -o %t/test %t/test.o -L%S/Inputs -lallowable_client 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-IMPLICIT
-# RUN: not %lld -o %t/libtest_debug.exe %t/test.o -L%S/Inputs -lallowable_client 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-IMPLICIT
-# RUN: not %lld -o %t/test %t/test.o -L%S/Inputs -lallowable_client -client_name notallowed 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-EXPLICIT
-# RUN: %lld -o %t/test %t/test.o -L%S/Inputs -lallowable_client -client_name allowed
-# RUN: %lld -o %t/test %t/test.o -L%S/Inputs -lallowable_client -client_name all
-# RUN: %lld -o %t/all %t/test.o -L%S/Inputs -lallowable_client
-# RUN: %lld -o %t/allowed %t/test.o -L%S/Inputs -lallowable_client
-# RUN: %lld -o %t/liballowed_debug.exe %t/test.o -L%S/Inputs -lallowable_client
+# Check that `-allowable_client` generates LC_SUB_CLIENT.
+# We create our .dylib in a `lib` subdirectory to make sure we test linking against the `.dylib` instead of the `.tbd` below.
+# RUN: mkdir -p %t/lib; %lld -dylib -o %t/lib/liballowable_client.dylib %t/empty.o -allowable_client allowed -allowable_client also_allowed
+# RUN: llvm-objdump --macho --all-headers %t/lib/liballowable_client.dylib | FileCheck %s
+# CHECK:      LC_SUB_CLIENT
+# CHECK-NEXT: cmdsize 24
+# CHECK-NEXT: client allowed
+# CHECK:      LC_SUB_CLIENT
+# CHECK-NEXT: cmdsize 32
+# CHECK-NEXT: client also_allowed
+
+# Check linking against the .dylib we created above
+# RUN: not %lld -o %t/test %t/test.o -L%t/lib -lallowable_client 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-IMPLICIT
+# RUN: not %lld -o %t/libtest_debug.exe %t/test.o -L%t/lib -lallowable_client 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-IMPLICIT
+# RUN: not %lld -o %t/test %t/test.o -L%t/lib -lallowable_client -client_name notallowed 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-EXPLICIT
+# RUN: %lld -o %t/test %t/test.o -L%t/lib -lallowable_client -client_name allowed
+# RUN: %lld -o %t/test %t/test.o -L%t/lib -lallowable_client -client_name all
+# RUN: %lld -o %t/all %t/test.o -L%t/lib -lallowable_client
+# RUN: %lld -o %t/allowed %t/test.o -L%t/lib -lallowable_client
+# RUN: %lld -o %t/liballowed_debug.exe %t/test.o -L%t/lib -lallowable_client
 
 # Check linking against a .tbd
 # RUN: not %lld -o %t/test %t/test.o -L%t -lallowable_client 2>&1 | FileCheck %s --check-prefix=NOTALLOWED-IMPLICIT
diff --git a/lld/test/MachO/bp-section-orderer-errs.s b/lld/test/MachO/bp-section-orderer-errs.s
index 682eb0c08bf1f..8d19e01c716ea 100644
--- a/lld/test/MachO/bp-section-orderer-errs.s
+++ b/lld/test/MachO/bp-section-orderer-errs.s
@@ -1,12 +1,21 @@
-# RUN: not %lld -o /dev/null --irpgo-profile-sort %s --call-graph-profile-sort 2>&1 | FileCheck %s --check-prefix=IRPGO-ERR
-# RUN: not %lld -o /dev/null --irpgo-profile-sort=%s --call-graph-profile-sort 2>&1 | FileCheck %s --check-prefix=IRPGO-ERR
+# RUN: not %no-fatal-warnings-lld -o /dev/null --irpgo-profile-sort %s --call-graph-profile-sort 2>&1 | FileCheck %s --check-prefix=IRPGO-ERR
+# RUN: not %no-fatal-warnings-lld -o /dev/null --irpgo-profile-sort=%s --call-graph-profile-sort 2>&1 | FileCheck %s --check-prefix=IRPGO-ERR
 # IRPGO-ERR: --irpgo-profile-sort is incompatible with --call-graph-profile-sort
 
 # RUN: not %lld -o /dev/null --compression-sort=function --call-graph-profile-sort %s 2>&1 | FileCheck %s --check-prefix=COMPRESSION-ERR
-# COMPRESSION-ERR: --compression-sort= is incompatible with --call-graph-profile-sort
+# RUN: not %lld -o /dev/null --bp-compression-sort=function --call-graph-profile-sort %s 2>&1 | FileCheck %s --check-prefix=COMPRESSION-ERR
+# COMPRESSION-ERR: --bp-compression-sort= is incompatible with --call-graph-profile-sort
 
 # RUN: not %lld -o /dev/null --compression-sort=malformed 2>&1 | FileCheck %s --check-prefix=COMPRESSION-MALFORM
-# COMPRESSION-MALFORM: unknown value `malformed` for --compression-sort=
+# RUN: not %lld -o /dev/null --bp-compression-sort=malformed 2>&1 | FileCheck %s --check-prefix=COMPRESSION-MALFORM
+# COMPRESSION-MALFORM: unknown value `malformed` for --bp-compression-sort=
 
 # RUN: not %lld -o /dev/null --compression-sort-startup-functions 2>&1 | FileCheck %s --check-prefix=STARTUP
-# STARTUP: --compression-sort-startup-functions must be used with --irpgo-profile-sort
+# RUN: not %lld -o /dev/null --bp-compression-sort-startup-functions 2>&1 | FileCheck %s --check-prefix=STARTUP
+# STARTUP: --bp-compression-sort-startup-functions must be used with --bp-startup-sort=function
+
+# RUN: not %lld -o /dev/null --irpgo-profile=%s --bp-startup-sort=function --call-graph-profile-sort 2>&1 | FileCheck %s --check-prefix=IRPGO-STARTUP
+# IRPGO-STARTUP: --bp-startup-sort= is incompatible with --call-graph-profile-sort
+
+# RUN: not %lld -o /dev/null --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=STARTUP-COMPRESSION
+# STARTUP-COMPRESSION: --bp-startup-sort=function must be used with --irpgo-profile
\ No newline at end of file
diff --git a/lld/test/MachO/bp-section-orderer-stress.s b/lld/test/MachO/bp-section-orderer-stress.s
index 986e2d8fd1069..0bfd99eb3dd86 100644
--- a/lld/test/MachO/bp-section-orderer-stress.s
+++ b/lld/test/MachO/bp-section-orderer-stress.s
@@ -7,10 +7,13 @@
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t.s -o %t.o
 # RUN: llvm-profdata merge %t.proftext -o %t.profdata
 
-# RUN: %lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile-sort=%t.profdata --compression-sort-startup-functions --compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order1.txt
-# RUN: %lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile-sort=%t.profdata --compression-sort-startup-functions --compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order2.txt
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile-sort=%t.profdata --compression-sort-startup-functions --compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order1.txt
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile-sort=%t.profdata --compression-sort-startup-functions --compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order2.txt
 # RUN: diff %t.order1.txt %t.order2.txt
 
+# RUN: %lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile=%t.profdata --bp-startup-sort=function --bp-compression-sort-startup-functions --bp-compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order1.txt
+# RUN: %lld -arch arm64 -lSystem -e _main --icf=all -o - %t.o --irpgo-profile=%t.profdata --bp-startup-sort=function --bp-compression-sort-startup-functions --bp-compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - > %t.order2.txt
+# RUN: diff %t.order1.txt %t.order2.txt
 import random
 import sys
 
diff --git a/lld/test/MachO/bp-section-orderer.s b/lld/test/MachO/bp-section-orderer.s
index 407787025150d..e5d0e7137b30d 100644
--- a/lld/test/MachO/bp-section-orderer.s
+++ b/lld/test/MachO/bp-section-orderer.s
@@ -4,13 +4,21 @@
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/a.s -o %t/a.o
 # RUN: llvm-profdata merge %t/a.proftext -o %t/a.profdata
 
-# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile-sort=%t/a.profdata --verbose-bp-section-orderer 2>&1 | FileCheck %s --check-prefix=STARTUP
-# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile-sort=%t/a.profdata --verbose-bp-section-orderer --icf=all --compression-sort=none 2>&1 | FileCheck %s --check-prefix=STARTUP
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile-sort=%t/a.profdata --verbose-bp-section-orderer 2>&1 | FileCheck %s --check-prefix=STARTUP
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile-sort=%t/a.profdata --verbose-bp-section-orderer --icf=all --compression-sort=none 2>&1 | FileCheck %s --check-prefix=STARTUP
 
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile=%t/a.profdata --bp-startup-sort=function --verbose-bp-section-orderer 2>&1 | FileCheck %s --check-prefix=STARTUP
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --irpgo-profile=%t/a.profdata --bp-startup-sort=function --verbose-bp-section-orderer --icf=all --bp-compression-sort=none 2>&1 | FileCheck %s --check-prefix=STARTUP
 # STARTUP: Ordered 3 sections using balanced partitioning
 
-# RUN: %lld -arch arm64 -lSystem -e _main -o - %t/a.o --irpgo-profile-sort=%t/a.profdata -order_file %t/a.orderfile | llvm-nm --numeric-sort --format=just-symbols - | FileCheck %s --check-prefix=ORDERFILE
+# Check that orderfiles take precedence over BP
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main -o - %t/a.o -order_file %t/a.orderfile --irpgo-profile-sort=%t/a.profdata  | llvm-nm --numeric-sort --format=just-symbols - | FileCheck %s --check-prefix=ORDERFILE
+# RUN: %lld -arch arm64 -lSystem -e _main -o - %t/a.o -order_file %t/a.orderfile --compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - | FileCheck %s --check-prefix=ORDERFILE
 
+# RUN: %lld -arch arm64 -lSystem -e _main -o - %t/a.o -order_file %t/a.orderfile --irpgo-profile=%t/a.profdata --bp-startup-sort=function | llvm-nm --numeric-sort --format=just-symbols - | FileCheck %s --check-prefix=ORDERFILE
+# RUN: %lld -arch arm64 -lSystem -e _main -o - %t/a.o -order_file %t/a.orderfile --bp-compression-sort=both | llvm-nm --numeric-sort --format=just-symbols - | FileCheck %s --check-prefix=ORDERFILE
+
+# Functions
 # ORDERFILE: A
 # ORDERFILE: F
 # ORDERFILE: E
@@ -18,19 +26,29 @@
 # ORDERFILE-DAG: _main
 # ORDERFILE-DAG: _B
 # ORDERFILE-DAG: l_C
+
+# Data
+# ORDERFILE: s3
+# ORDERFILE: r3
+# ORDERFILE: r2
 # ORDERFILE-DAG: s1
 # ORDERFILE-DAG: s2
 # ORDERFILE-DAG: r1
-# ORDERFILE-DAG: r2
+# ORDERFILE-DAG: r4
 
 # RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --compression-sort=function 2>&1 | FileCheck %s --check-prefix=COMPRESSION-FUNC
 # RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --compression-sort=data 2>&1 | FileCheck %s --check-prefix=COMPRESSION-DATA
 # RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --compression-sort=both 2>&1 | FileCheck %s --check-prefix=COMPRESSION-BOTH
-# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --compression-sort=both --irpgo-profile-sort=%t/a.profdata 2>&1 | FileCheck %s --check-prefix=COMPRESSION-BOTH
+# RUN: %no-fatal-warnings-lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --compression-sort=both --irpgo-profile-sort=%t/a.profdata 2>&1 | FileCheck %s --check-prefix=COMPRESSION-BOTH
+
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=COMPRESSION-FUNC
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=COMPRESSION-DATA
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=COMPRESSION-BOTH
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=%t/a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=COMPRESSION-BOTH
 
 # COMPRESSION-FUNC: Ordered 7 sections using balanced partitioning
-# COMPRESSION-DATA: Ordered 4 sections using balanced partitioning
-# COMPRESSION-BOTH: Ordered 11 sections using balanced partitioning
+# COMPRESSION-DATA: Ordered 7 sections using balanced partitioning
+# COMPRESSION-BOTH: Ordered 14 sections using balanced partitioning
 
 #--- a.s
 .text
@@ -66,10 +84,16 @@ s1:
   .ascii "hello world"
 s2:
   .ascii "i am a string"
+s3:
+  .ascii "this is s3"
 r1:
   .quad s1
 r2:
   .quad r1
+r3:
+  .quad r2
+r4:
+  .quad s2
 
 .subsections_via_symbols
 
@@ -121,3 +145,6 @@ A
 F
 E
 D
+s3
+r3
+r2
diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test
index ed6bc880f8f2d..907d2d87dda5c 100644
--- a/lld/test/MinGW/driver.test
+++ b/lld/test/MinGW/driver.test
@@ -462,3 +462,6 @@ RUN: ld.lld -### foo.o -m i386pep --rpath foo 2>&1 | FileCheck -check-prefix=WAR
 RUN: ld.lld -### foo.o -m i386pep -rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
 RUN: ld.lld -### foo.o -m i386pep --rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
 WARN_RPATH: warning: parameter -{{-?}}rpath has no effect on PE/COFF targets
+
+RUN: ld.lld -### foo.o -m i386pe --dependent-load-flag=0x800 2>&1 | FileCheck -check-prefix=DEPENDENT_LOAD_FLAG %s
+DEPENDENT_LOAD_FLAG: -dependentloadflag:0x800
diff --git a/lld/test/wasm/compress-relocs.ll b/lld/test/wasm/compress-relocs.ll
index f1faab754cb76..cea9f3476e996 100644
--- a/lld/test/wasm/compress-relocs.ll
+++ b/lld/test/wasm/compress-relocs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -filetype=obj %s -o %t.o
-; RUN: llvm-mc -mattr=+reference-types -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/call-indirect.s -o %t2.o
+; RUN: llvm-mc -mattr=+call-indirect-overlong -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/call-indirect.s -o %t2.o
 ; RUN: wasm-ld --export-dynamic -o %t.wasm %t2.o %t.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 ; RUN: wasm-ld --export-dynamic -O2 -o %t-opt.wasm %t2.o %t.o
diff --git a/lld/test/wasm/import-table-explicit.s b/lld/test/wasm/import-table-explicit.s
index 1dc21beba0629..701b7a1dc3e16 100644
--- a/lld/test/wasm/import-table-explicit.s
+++ b/lld/test/wasm/import-table-explicit.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -mattr=+reference-types -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
+# RUN: llvm-mc -mattr=+call-indirect-overlong -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
 # RUN: wasm-ld --import-table -o %t.wasm %t.o
 # RUN: obj2yaml %t.wasm | FileCheck %s
 
diff --git a/lld/test/wasm/invalid-mvp-table-use.s b/lld/test/wasm/invalid-mvp-table-use.s
index b4f12a7eeb9a4..58c472e29d1ad 100644
--- a/lld/test/wasm/invalid-mvp-table-use.s
+++ b/lld/test/wasm/invalid-mvp-table-use.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s
 #
 # If any table is defined or declared besides the __indirect_function_table,
-# the compilation unit should be compiled with -mattr=+reference-types,
+# the compilation unit should be compiled with -mattr=+call-indirect-overlong,
 # causing symbol table entries to be emitted for all tables.
 # RUN: not wasm-ld --no-entry %t.o -o %t.wasm 2>&1 | FileCheck -check-prefix=CHECK-ERR %s
 
diff --git a/lld/test/wasm/lto/Inputs/libcall-archive.ll b/lld/test/wasm/lto/Inputs/libcall-archive.ll
index 7d8c34196dfe4..30764af83e673 100644
--- a/lld/test/wasm/lto/Inputs/libcall-archive.ll
+++ b/lld/test/wasm/lto/Inputs/libcall-archive.ll
@@ -5,4 +5,4 @@ define void @memcpy() #0 {
   ret void
 }
 
-attributes #0 = { "target-features"="-bulk-memory" }
+attributes #0 = { "target-features"="-bulk-memory,-bulk-memory-opt" }
diff --git a/lld/test/wasm/lto/libcall-archive.ll b/lld/test/wasm/lto/libcall-archive.ll
index 5c46d2f7ed783..0cee9a5de29f6 100644
--- a/lld/test/wasm/lto/libcall-archive.ll
+++ b/lld/test/wasm/lto/libcall-archive.ll
@@ -16,7 +16,7 @@ entry:
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1)
 
-attributes #0 = { "target-features"="-bulk-memory" }
+attributes #0 = { "target-features"="-bulk-memory,-bulk-memory-opt" }
 
 ; CHECK:       - Type:            CUSTOM
 ; CHECK-NEXT:    Name:            name
diff --git a/lld/test/wasm/lto/stub-library-libcall.s b/lld/test/wasm/lto/stub-library-libcall.s
index d65983c0cf5bf..40e15933f7bc3 100644
--- a/lld/test/wasm/lto/stub-library-libcall.s
+++ b/lld/test/wasm/lto/stub-library-libcall.s
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t_main.o %t/main.s
 # RUN: llvm-as %S/Inputs/foo.ll -o %t_foo.o
 # RUN: llvm-as %S/Inputs/libcall.ll -o %t_libcall.o
-# RUN: wasm-ld -mllvm -mattr=-bulk-memory %t_main.o %t_libcall.o %t_foo.o %p/Inputs/stub.so -o %t.wasm
+# RUN: wasm-ld -mllvm -mattr=-bulk-memory,-bulk-memory-opt %t_main.o %t_libcall.o %t_foo.o %p/Inputs/stub.so -o %t.wasm
 # RUN: obj2yaml %t.wasm | FileCheck %s
 
 # The function `func_with_libcall` will generate an undefined reference to
@@ -12,7 +12,7 @@
 # If %t_foo.o is not included in the link we get an undefined symbol reported
 # to the dependency of memcpy on the foo export:
 
-# RUN: not wasm-ld -mllvm -mattr=-bulk-memory %t_main.o %t_libcall.o %p/Inputs/stub.so -o %t.wasm 2>&1 | FileCheck --check-prefix=MISSING %s
+# RUN: not wasm-ld -mllvm -mattr=-bulk-memory,-bulk-memory-opt %t_main.o %t_libcall.o %p/Inputs/stub.so -o %t.wasm 2>&1 | FileCheck --check-prefix=MISSING %s
 # MISSING: stub.so: undefined symbol: foo. Required by memcpy
 
 #--- main.s
diff --git a/lld/test/wasm/multi-table.s b/lld/test/wasm/multi-table.s
index bf905ac748f9f..afe8ddac49768 100644
--- a/lld/test/wasm/multi-table.s
+++ b/lld/test/wasm/multi-table.s
@@ -26,7 +26,7 @@ call_indirect_explicit_tables:
   call_indirect table_b, () -> ()
   end_function
 
-# RT-MVP: wasm-ld: error: object file not built with 'reference-types' feature conflicts with import of table table_a by file
+# RT-MVP: wasm-ld: error: object file not built with 'reference-types' or 'call-indirect-overlong' feature conflicts with import of table table_a by file
 
 # CHECK:      --- !WASM
 # CHECK-NEXT: FileHeader:
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 8d01ff839ddfc..37a0156c728f6 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -59,6 +59,7 @@ void Ctx::reset() {
   stubFiles.clear();
   sharedFiles.clear();
   bitcodeFiles.clear();
+  lazyBitcodeFiles.clear();
   syntheticFunctions.clear();
   syntheticGlobals.clear();
   syntheticTables.clear();
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index fd06788457966..221f02aa1c157 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -255,13 +255,14 @@ static void setRelocs(const std::vector<T *> &chunks,
   }
 }
 
-// An object file can have two approaches to tables.  With the reference-types
-// feature enabled, input files that define or use tables declare the tables
-// using symbols, and record each use with a relocation.  This way when the
-// linker combines inputs, it can collate the tables used by the inputs,
-// assigning them distinct table numbers, and renumber all the uses as
-// appropriate.  At the same time, the linker has special logic to build the
-// indirect function table if it is needed.
+// An object file can have two approaches to tables.  With the
+// reference-types feature or call-indirect-overlong feature enabled
+// (explicitly, or implied by the reference-types feature), input files that
+// define or use tables declare the tables using symbols, and record each use
+// with a relocation.  This way when the linker combines inputs, it can collate
+// the tables used by the inputs, assigning them distinct table numbers, and
+// renumber all the uses as appropriate.  At the same time, the linker has
+// special logic to build the indirect function table if it is needed.
 //
 // However, MVP object files (those that target WebAssembly 1.0, the "minimum
 // viable product" version of WebAssembly) neither write table symbols nor
@@ -284,9 +285,9 @@ void ObjFile::addLegacyIndirectFunctionTableIfNeeded(
     return;
 
   // It's possible for an input to define tables and also use the indirect
-  // function table, but forget to compile with -mattr=+reference-types.
-  // For these newer files, we require symbols for all tables, and
-  // relocations for all of their uses.
+  // function table, but forget to compile with -mattr=+call-indirect-overlong
+  // or -mattr=+reference-types. For these newer files, we require symbols for
+  // all tables, and relocations for all of their uses.
   if (tableSymbolCount != 0) {
     error(toString(this) +
           ": expected one symbol table entry for each of the " +
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index 1454c3324af98..6b32d12ebeb45 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -326,8 +326,9 @@ void TableSection::addTable(InputTable *table) {
       // to assign table number 0 to the indirect function table.
       for (const auto *culprit : out.importSec->importedSymbols) {
         if (isa<UndefinedTable>(culprit)) {
-          error("object file not built with 'reference-types' feature "
-                "conflicts with import of table " +
+          error("object file not built with 'reference-types' or "
+                "'call-indirect-overlong' feature conflicts with import of "
+                "table " +
                 culprit->getName() + " by file " +
                 toString(culprit->getFile()));
           return;
diff --git a/lldb/bindings/interface/SBMemoryRegionInfoListExtensions.i b/lldb/bindings/interface/SBMemoryRegionInfoListExtensions.i
index 49d49110de7ff..29c0179c0ffe3 100644
--- a/lldb/bindings/interface/SBMemoryRegionInfoListExtensions.i
+++ b/lldb/bindings/interface/SBMemoryRegionInfoListExtensions.i
@@ -7,7 +7,12 @@
 
     def __iter__(self):
       '''Iterate over all the memory regions in a lldb.SBMemoryRegionInfoList object.'''
-      return lldb_iter(self, 'GetSize', 'GetMemoryRegionAtIndex')
+      import lldb
+      size = self.GetSize()
+      region = lldb.SBMemoryRegionInfo()
+      for i in range(size):
+        self.GetMemoryRegionAtIndex(i, region)
+        yield region
     %}
 #endif
 }
diff --git a/lldb/bindings/interface/SBValueExtensions.i b/lldb/bindings/interface/SBValueExtensions.i
index bee9c27775d45..f0c8797bd5f78 100644
--- a/lldb/bindings/interface/SBValueExtensions.i
+++ b/lldb/bindings/interface/SBValueExtensions.i
@@ -7,7 +7,7 @@ STRING_EXTENSION_OUTSIDE(SBValue)
             return self.GetDynamicValue (eDynamicCanRunTarget)
 
         class children_access(object):
-            '''A helper object that will lazily hand out thread for a process when supplied an index.'''
+            '''A helper object that will lazily hand out child values when supplied an index.'''
 
             def __init__(self, sbvalue):
                 self.sbvalue = sbvalue
@@ -29,6 +29,19 @@ STRING_EXTENSION_OUTSIDE(SBValue)
             '''An accessor function that returns a children_access() object which allows lazy member variable access from a lldb.SBValue object.'''
             return self.children_access (self)
 
+        def get_member_access_object(self):
+            '''An accessor function that returns an interface which provides subscript based lookup of child members.'''
+            class member_access:
+                def __init__(self, valobj):
+                    self.valobj = valobj
+
+                def __getitem__(self, key):
+                    if isinstance(key, str):
+                        return self.valobj.GetChildMemberWithName(key)
+                    raise TypeError("member key must be a string")
+
+            return member_access(self)
+
         def get_value_child_list(self):
             '''An accessor function that returns a list() that contains all children in a lldb.SBValue object.'''
             children = []
@@ -50,6 +63,7 @@ STRING_EXTENSION_OUTSIDE(SBValue)
 
         children = property(get_value_child_list, None, doc='''A read only property that returns a list() of lldb.SBValue objects for the children of the value.''')
         child = property(get_child_access_object, None, doc='''A read only property that returns an object that can access children of a variable by index (child_value = value.children[12]).''')
+        member = property(get_member_access_object, None, doc='''A read only property that returns an object that can access child members by name.''')
         name = property(GetName, None, doc='''A read only property that returns the name of this value as a string.''')
         type = property(GetType, None, doc='''A read only property that returns a lldb.SBType object that represents the type for this value.''')
         size = property(GetByteSize, None, doc='''A read only property that returns the size in bytes of this value.''')
diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index 93ccd9c479c2b..ee4c2630d32e2 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -239,10 +239,6 @@ if (LLDB_ENABLE_LZMA)
   include_directories(${LIBLZMA_INCLUDE_DIRS})
 endif()
 
-if (LLDB_ENABLE_LIBXML2)
-  include_directories(${LIBXML2_INCLUDE_DIR})
-endif()
-
 include_directories(BEFORE
   ${CMAKE_CURRENT_BINARY_DIR}/include
   ${CMAKE_CURRENT_SOURCE_DIR}/include
@@ -283,7 +279,6 @@ if (APPLE)
   find_library(FOUNDATION_LIBRARY Foundation)
   find_library(CORE_FOUNDATION_LIBRARY CoreFoundation)
   find_library(SECURITY_LIBRARY Security)
-  include_directories(${LIBXML2_INCLUDE_DIR})
 endif()
 
 if( WIN32 AND NOT CYGWIN )
diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index fb22bdecad37e..5686a33e94c93 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -164,6 +164,7 @@ interesting areas to contribute to lldb.
    resources/fuzzing
    resources/sbapi
    resources/dataformatters
+   resources/formatterbytecode
    resources/extensions
    resources/lldbgdbremote
    resources/lldbplatformpackets
diff --git a/lldb/docs/resources/formatterbytecode.rst b/lldb/docs/resources/formatterbytecode.rst
new file mode 100644
index 0000000000000..a4fd2bbe804b2
--- /dev/null
+++ b/lldb/docs/resources/formatterbytecode.rst
@@ -0,0 +1,238 @@
+Formatter Bytecode
+==================
+
+Background
+----------
+
+LLDB provides rich customization options to display data types (see :doc:`/use/variable/`). To use custom data formatters, developers need to edit the global `~/.lldbinit` file to make sure they are found and loaded. In addition to this rather manual workflow, developers or library authors can ship ship data formatters with their code in a format that allows LLDB automatically find them and run them securely.
+
+An end-to-end example of such a workflow is the Swift `DebugDescription` macro (see https://www.swift.org/blog/announcing-swift-6/#debugging ) that translates Swift string interpolation into LLDB summary strings, and puts them into a `.lldbsummaries` section, where LLDB can find them.
+
+This document describes a minimal bytecode tailored to running LLDB formatters. It defines a human-readable assembler representation for the language, an efficient binary encoding, a virtual machine for evaluating it, and format for embedding formatters into binary containers.
+
+Goals
+~~~~~
+
+Provide an efficient and secure encoding for data formatters that can be used as a compilation target from user-friendly representations (such as DIL, Swift DebugDescription, or NatVis).
+
+Non-goals
+~~~~~~~~~
+
+While humans could write the assembler syntax, making it user-friendly is not a goal. It is meant to be used as a compilation target for higher-level, language-specific affordances.
+
+Design of the virtual machine
+-----------------------------
+
+The LLDB formatter virtual machine uses a stack-based bytecode, comparable with DWARF expressions, but with higher-level data types and functions.
+
+The virtual machine has two stacks, a data and a control stack. The control stack is kept separate to make it easier to reason about the security aspects of the virtual machine.
+
+Data types
+~~~~~~~~~~
+
+All objects on the data stack must have one of the following data types. These data types are "host" data types, in LLDB parlance.
+
+* *String* (UTF-8)
+* *Int* (64 bit)
+* *UInt* (64 bit)
+* *Object* (Basically an `SBValue`)
+* *Type* (Basically an `SBType`)
+* *Selector* (One of the predefine functions)
+
+*Object* and *Type* are opaque, they can only be used as a parameters of `call`.
+
+Instruction set
+---------------
+
+Stack operations
+~~~~~~~~~~~~~~~~
+
+These instructions manipulate the data stack directly.
+
+========  ==========  ===========================
+ Opcode    Mnemonic    Stack effect
+--------  ----------  ---------------------------
+ 0x00      `dup`       `(x -> x x)`
+ 0x01      `drop`      `(x y -> x)`
+ 0x02      `pick`      `(x ... UInt -> x ... x)`
+ 0x03      `over`      `(x y -> x y x)`
+ 0x04      `swap`      `(x y -> y x)`
+ 0x05      `rot`       `(x y z -> z x y)`
+=======  ==========  ===========================
+
+Control flow
+~~~~~~~~~~~~
+
+These manipulate the control stack and program counter. Both `if` and `ifelse` expect a `UInt` at the top of the data stack to represent the condition.
+
+========  ==========  ============================================================
+ Opcode    Mnemonic    Description
+--------  ----------  ------------------------------------------------------------
+ 0x10       `{`        push a code block address onto the control stack
+  --        `}`        (technically not an opcode) syntax for end of code block
+ 0x11      `if`        `(UInt -> )` pop a block from the control stack,
+                       if the top of the data stack is nonzero, execute it
+ 0x12      `ifelse`    `(UInt -> )` pop two blocks from the control stack, if
+                       the top of the data stack is nonzero, execute the first,
+                       otherwise the second.
+========  ==========  ============================================================
+
+Literals for basic types
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+========  ===========  ============================================================
+ Opcode    Mnemonic    Description
+--------  -----------  ------------------------------------------------------------
+ 0x20      `123u`      `( -> UInt)` push an unsigned 64-bit host integer
+ 0x21      `123`       `( -> Int)` push a signed 64-bit host integer
+ 0x22      `"abc"`     `( -> String)` push a UTF-8 host string
+ 0x23      `@strlen`   `( -> Selector)` push one of the predefined function
+                       selectors. See `call`.
+========  ===========  ============================================================
+
+Conversion operations
+~~~~~~~~~~~~~~~~~~~~~
+
+========  ===========  ================================================================
+ Opcode    Mnemonic    Description
+--------  -----------  ----------------------------------------------------------------
+ 0x2a      `as_int`   `( UInt -> Int)` reinterpret a UInt as an Int
+ 0x2b      `as_uint`  `( Int -> UInt)` reinterpret an Int as a UInt
+ 0x2c      `is_null`  `( Object -> UInt )` check an object for null `(object ? 0 : 1)`
+========  ===========  ================================================================
+
+
+Arithmetic, logic, and comparison operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All of these operations are only defined for `Int` and `UInt` and both operands need to be of the same type. The `>>` operator is an arithmetic shift if the parameters are of type `Int`, otherwise it's a logical shift to the right.
+
+========  ==========  ===========================
+ Opcode    Mnemonic    Stack effect
+--------  ----------  ---------------------------
+ 0x30      `+`         `(x y -> [x+y])`
+ 0x31      `-`          etc ...
+ 0x32      `*`
+ 0x33      `/`
+ 0x34      `%`
+ 0x35      `<<`
+ 0x36      `>>`
+ 0x40      `~`
+ 0x41      `|`
+ 0x42      `^`
+ 0x50      `=`
+ 0x51      `!=`
+ 0x52      `<`
+ 0x53      `>`
+ 0x54      `=<`
+ 0x55      `>=`
+========  ==========  ===========================
+
+Function calls
+~~~~~~~~~~~~~~
+
+For security reasons the list of functions callable with `call` is predefined. The supported functions are either existing methods on `SBValue`, or string formatting operations.
+
+========  ==========  ============================================
+ Opcode    Mnemonic    Stack effect
+--------  ----------  --------------------------------------------
+ 0x60      `call`      `(Object argN ... arg0 Selector -> retval)`
+========  ==========  ============================================
+
+Method is one of a predefined set of *Selectors*.
+
+====  ============================  ===================================================  ==================================
+Sel.  Mnemonic                      Stack Effect                                         Description
+----  ----------------------------  ---------------------------------------------------  ----------------------------------
+0x00  `summary`                     `(Object @summary -> String)`                        `SBValue::GetSummary`
+0x01  `type_summary`                `(Object @type_summary -> String)`                   `SBValue::GetTypeSummary`
+0x10  `get_num_children`            `(Object @get_num_children -> UInt)`                 `SBValue::GetNumChildren`
+0x11  `get_child_at_index`          `(Object UInt @get_child_at_index -> Object)`        `SBValue::GetChildAtIndex`
+0x12  `get_child_with_name`         `(Object String @get_child_with_name -> Object)`     `SBValue::GetChildAtIndex`
+0x13  `get_child_index`             `(Object String @get_child_index -> UInt)`           `SBValue::GetChildIndex`
+0x15  `get_type`                    `(Object @get_type -> Type)`                         `SBValue::GetType`
+0x16  `get_template_argument_type`  `(Object UInt @get_template_argument_type -> Type)`  `SBValue::GetTemplateArgumentType`
+0x17  `cast`                        `(Object Type @cast -> Object)`                      `SBValue::Cast`
+0x20  `get_value`                   `(Object @get_value -> Object)`                      `SBValue::GetValue`
+0x21  `get_value_as_unsigned`       `(Object @get_value_as_unsigned -> UInt)`            `SBValue::GetValueAsUnsigned`
+0x22  `get_value_as_signed`         `(Object @get_value_as_signed -> Int)`               `SBValue::GetValueAsSigned`
+0x23  `get_value_as_address`        `(Object @get_value_as_address -> UInt)`             `SBValue::GetValueAsAddress`
+0x40  `read_memory_byte`            `(UInt @read_memory_byte -> UInt)`                   `Target::ReadMemory`
+0x41  `read_memory_uint32`          `(UInt @read_memory_uint32 -> UInt)`                 `Target::ReadMemory`
+0x42  `read_memory_int32`           `(UInt @read_memory_int32 -> Int)`                   `Target::ReadMemory`
+0x43  `read_memory_uint64`          `(UInt @read_memory_uint64 -> UInt)`                 `Target::ReadMemory`
+0x44  `read_memory_int64`           `(UInt @read_memory_int64 -> Int)`                   `Target::ReadMemory`
+0x45  `read_memory_address`         `(UInt @read_memory_uint64 -> UInt)`                 `Target::ReadMemory`
+0x46  `read_memory`                 `(UInt Type @read_memory -> Object)`                 `Target::ReadMemory`
+0x50  `fmt`                         `(String arg0 ... @fmt -> String)`                   `llvm::format`
+0x51  `sprintf`                     `(String arg0 ... sprintf -> String)`                `sprintf`
+0x52  `strlen`                      `(String strlen -> String)`                          `strlen in bytes`
+====  ============================  ===================================================  ==================================
+
+Byte Code
+~~~~~~~~~
+
+Most instructions are just a single byte opcode. The only exceptions are the literals:
+
+* *String*: Length in bytes encoded as ULEB128, followed length bytes
+* *Int*: LEB128
+* *UInt*: ULEB128
+* *Selector*: ULEB128
+
+Embedding
+~~~~~~~~~
+
+Expression programs are embedded into an `.lldbformatters` section (an evolution of the Swift `.lldbsummaries` section) that is a dictionary of type names/regexes and descriptions. It consists of a list of records. Each record starts with the following header:
+
+* Version number (ULEB128)
+* Remaining size of the record (minus the header) (ULEB128)
+
+The version number is increased whenever an incompatible change is made. Adding new opcodes or selectors is not an incompatible change since consumers can unambiguously detect this and report an error.
+
+Space between two records may be padded with NULL bytes.
+
+In version 1, a record consists of a dictionary key, which is a type name or regex.
+
+* Length of the key in bytes (ULEB128)
+* The key (UTF-8)
+
+A regex has to start with `^`, which is part of the regular expression.
+
+After this comes a flag bitfield, which is a ULEB-encoded `lldb::TypeOptions` bitfield.
+
+* Flags (ULEB128)
+
+
+This is followed by one or more dictionary values that immediately follow each other and entirely fill out the record size from the header. Each expression program has the following layout:
+
+* Function signature (1 byte)
+* Length of the program (ULEB128)
+* The program bytecode
+
+The possible function signatures are:
+
+=========  ====================== ==========================
+Signature    Mnemonic             Stack Effect
+---------  ---------------------- --------------------------
+  0x00     `@summary`             `(Object -> String)`
+  0x01     `@init`                `(Object -> Object+)`
+  0x02     `@get_num_children`    `(Object+ -> UInt)`
+  0x03     `@get_child_index`     `(Object+ String -> UInt)`
+  0x04     `@get_child_at_index`  `(Object+ UInt -> Object)`
+  0x05     `@get_value`           `(Object+ -> String)`
+=========  ====================== ==========================
+
+If not specified, the init function defaults to an empty function that just passes the Object along. Its results may be cached and allow common prep work to be done for an Object that can be reused by subsequent calls to the other methods. This way subsequent calls to `@get_child_at_index` can avoid recomputing shared information, for example.
+
+While it is more efficient to store multiple programs per type key, this is not a requirement. LLDB will merge all entries. If there are conflicts the result is undefined.
+
+Execution model
+~~~~~~~~~~~~~~~
+
+Execution begins at the first byte in the program. The program counter of the virtual machine starts at offset 0 of the bytecode and may never move outside the range of the program as defined in the header. The data stack starts with one Object or the result of the `@init` function (`Object+` in the table above).
+
+Error handling
+~~~~~~~~~~~~~~
+
+In version 1 errors are unrecoverable, the entire expression will fail if any kind of error is encountered.
+
diff --git a/lldb/examples/python/formatter_bytecode.py b/lldb/examples/python/formatter_bytecode.py
new file mode 100644
index 0000000000000..ccd0c68a75483
--- /dev/null
+++ b/lldb/examples/python/formatter_bytecode.py
@@ -0,0 +1,524 @@
+"""
+Specification, compiler, disassembler, and interpreter
+for LLDB dataformatter bytecode.
+
+See https://lldb.llvm.org/resources/formatterbytecode.html for more details.
+"""
+
+from __future__ import annotations
+
+# Types
+type_String = 1
+type_Int = 2
+type_UInt = 3
+type_Object = 4
+type_Type = 5
+
+# Opcodes
+opcode = dict()
+
+
+def define_opcode(n, mnemonic, name):
+    globals()["op_" + name] = n
+    if mnemonic:
+        opcode[mnemonic] = n
+    opcode[n] = mnemonic
+
+
+define_opcode(1, "dup", "dup")
+define_opcode(2, "drop", "drop")
+define_opcode(3, "pick", "pick")
+define_opcode(4, "over", "over")
+define_opcode(5, "swap", "swap")
+define_opcode(6, "rot", "rot")
+
+define_opcode(0x10, "{", "begin")
+define_opcode(0x11, "if", "if")
+define_opcode(0x12, "ifelse", "ifelse")
+
+define_opcode(0x20, None, "lit_uint")
+define_opcode(0x21, None, "lit_int")
+define_opcode(0x22, None, "lit_string")
+define_opcode(0x23, None, "lit_selector")
+
+define_opcode(0x2A, "as_int", "as_int")
+define_opcode(0x2B, "as_uint", "as_uint")
+define_opcode(0x2C, "is_null", "is_null")
+
+define_opcode(0x30, "+", "plus")
+define_opcode(0x31, "-", "minus")
+define_opcode(0x32, "*", "mul")
+define_opcode(0x33, "/", "div")
+define_opcode(0x34, "%", "mod")
+define_opcode(0x35, "<<", "shl")
+define_opcode(0x36, ">>", "shr")
+
+define_opcode(0x40, "&", "and")
+define_opcode(0x41, "|", "or")
+define_opcode(0x42, "^", "xor")
+define_opcode(0x43, "~", "not")
+
+define_opcode(0x50, "=", "eq")
+define_opcode(0x51, "!=", "neq")
+define_opcode(0x52, "<", "lt")
+define_opcode(0x53, ">", "gt")
+define_opcode(0x54, "=<", "le")
+define_opcode(0x55, ">=", "ge")
+
+define_opcode(0x60, "call", "call")
+
+# Function signatures
+sig_summary = 0
+sig_init = 1
+sig_get_num_children = 2
+sig_get_child_index = 3
+sig_get_child_at_index = 4
+
+# Selectors
+selector = dict()
+
+
+def define_selector(n, name):
+    globals()["sel_" + name] = n
+    selector["@" + name] = n
+    selector[n] = "@" + name
+
+
+define_selector(0, "summary")
+define_selector(1, "type_summary")
+
+define_selector(0x10, "get_num_children")
+define_selector(0x11, "get_child_at_index")
+define_selector(0x12, "get_child_with_name")
+define_selector(0x13, "get_child_index")
+define_selector(0x15, "get_type")
+define_selector(0x16, "get_template_argument_type")
+define_selector(0x17, "cast")
+define_selector(0x20, "get_value")
+define_selector(0x21, "get_value_as_unsigned")
+define_selector(0x22, "get_value_as_signed")
+define_selector(0x23, "get_value_as_address")
+
+define_selector(0x40, "read_memory_byte")
+define_selector(0x41, "read_memory_uint32")
+define_selector(0x42, "read_memory_int32")
+define_selector(0x43, "read_memory_unsigned")
+define_selector(0x44, "read_memory_signed")
+define_selector(0x45, "read_memory_address")
+define_selector(0x46, "read_memory")
+
+define_selector(0x50, "fmt")
+define_selector(0x51, "sprintf")
+define_selector(0x52, "strlen")
+
+
+################################################################################
+# Compiler.
+################################################################################
+
+
+def compile(assembler: str) -> bytearray:
+    """Compile assembler into bytecode"""
+    # This is a stack of all in-flight/unterminated blocks.
+    bytecode = [bytearray()]
+
+    def emit(byte):
+        bytecode[-1].append(byte)
+
+    tokens = list(assembler.split(" "))
+    tokens.reverse()
+    while tokens:
+        tok = tokens.pop()
+        if tok == "":
+            pass
+        elif tok == "{":
+            bytecode.append(bytearray())
+        elif tok == "}":
+            block = bytecode.pop()
+            emit(op_begin)
+            emit(len(block))  # FIXME: uleb
+            bytecode[-1].extend(block)
+        elif tok[0].isdigit():
+            if tok[-1] == "u":
+                emit(op_lit_uint)
+                emit(int(tok[:-1]))  # FIXME
+            else:
+                emit(op_lit_int)
+                emit(int(tok))  # FIXME
+        elif tok[0] == "@":
+            emit(op_lit_selector)
+            emit(selector[tok])
+        elif tok[0] == '"':
+            s = bytearray()
+            done = False
+            chrs = tok[1:]
+            while not done:
+                quoted = False
+                for c in chrs:
+                    if quoted:
+                        s.append(ord(c))  # FIXME
+                        quoted = False
+                    elif c == "\\":
+                        quoted = True
+                    elif c == '"':
+                        done = True
+                        break
+                        # FIXME assert this is last in token
+                    else:
+                        s.append(ord(c))
+                if not done:
+                    s.append(ord(" "))
+                    chrs = tokens.pop()
+
+            emit(op_lit_string)
+            emit(len(s))
+            bytecode[-1].extend(s)
+        else:
+            emit(opcode[tok])
+    assert len(bytecode) == 1  # unterminated {
+    return bytecode[0]
+
+
+################################################################################
+# Disassembler.
+################################################################################
+
+
+def disassemble(bytecode: bytearray) -> (str, int):
+    """Disassemble bytecode into (assembler, token starts)"""
+    asm = ""
+    all_bytes = list(bytecode)
+    all_bytes.reverse()
+    blocks = []
+    tokens = [0]
+
+    def next_byte():
+        """Fetch the next byte in the bytecode and keep track of all
+        in-flight blocks"""
+        for i in range(len(blocks)):
+            blocks[i] -= 1
+        tokens.append(len(asm))
+        return all_bytes.pop()
+
+    while all_bytes:
+        b = next_byte()
+        if b == op_begin:
+            asm += "{"
+            length = next_byte()
+            blocks.append(length)
+        elif b == op_lit_uint:
+            b = next_byte()
+            asm += str(b)  # FIXME uleb
+            asm += "u"
+        elif b == op_lit_int:
+            b = next_byte()
+            asm += str(b)
+        elif b == op_lit_selector:
+            b = next_byte()
+            asm += selector[b]
+        elif b == op_lit_string:
+            length = next_byte()
+            s = "'"
+            while length:
+                s += chr(next_byte())
+                length -= 1
+            asm += '"' + repr(s)[2:]
+        else:
+            asm += opcode[b]
+
+        while blocks and blocks[-1] == 0:
+            asm += " }"
+            blocks.pop()
+
+        if all_bytes:
+            asm += " "
+
+    if blocks:
+        asm += "ERROR"
+    return asm, tokens
+
+
+################################################################################
+# Interpreter.
+################################################################################
+
+
+def count_fmt_params(fmt: str) -> int:
+    """Count the number of parameters in a format string"""
+    from string import Formatter
+
+    f = Formatter()
+    n = 0
+    for _, name, _, _ in f.parse(fmt):
+        if name > n:
+            n = name
+    return n
+
+
+def interpret(bytecode: bytearray, control: list, data: list, tracing: bool = False):
+    """Interpret bytecode"""
+    frame = []
+    frame.append((0, len(bytecode)))
+
+    def trace():
+        """print a trace of the execution for debugging purposes"""
+
+        def fmt(d):
+            if isinstance(d, int):
+                return str(d)
+            if isinstance(d, str):
+                return d
+            return repr(type(d))
+
+        pc, end = frame[-1]
+        asm, tokens = disassemble(bytecode)
+        print(
+            "=== frame = {1}, data = {2}, opcode = {0}".format(
+                opcode[b], frame, [fmt(d) for d in data]
+            )
+        )
+        print(asm)
+        print(" " * (tokens[pc]) + "^")
+
+    def next_byte():
+        """Fetch the next byte and update the PC"""
+        pc, end = frame[-1]
+        assert pc < len(bytecode)
+        b = bytecode[pc]
+        frame[-1] = pc + 1, end
+        # At the end of a block?
+        while pc >= end:
+            frame.pop()
+            if not frame:
+                return None
+            pc, end = frame[-1]
+            if pc >= end:
+                return None
+            b = bytecode[pc]
+            frame[-1] = pc + 1, end
+        return b
+
+    while frame[-1][0] < len(bytecode):
+        b = next_byte()
+        if b == None:
+            break
+        if tracing:
+            trace()
+        # Data stack manipulation.
+        if b == op_dup:
+            data.append(data[-1])
+        elif b == op_drop:
+            data.pop()
+        elif b == op_pick:
+            data.append(data[data.pop()])
+        elif b == op_over:
+            data.append(data[-2])
+        elif b == op_swap:
+            x = data.pop()
+            y = data.pop()
+            data.append(x)
+            data.append(y)
+        elif b == op_rot:
+            z = data.pop()
+            y = data.pop()
+            x = data.pop()
+            data.append(z)
+            data.append(x)
+            data.append(y)
+
+        # Control stack manipulation.
+        elif b == op_begin:
+            length = next_byte()
+            pc, end = frame[-1]
+            control.append((pc, pc + length))
+            frame[-1] = pc + length, end
+        elif b == op_if:
+            if data.pop():
+                frame.append(control.pop())
+        elif b == op_ifelse:
+            if data.pop():
+                control.pop()
+                frame.append(control.pop())
+            else:
+                frame.append(control.pop())
+                control.pop()
+
+        # Literals.
+        elif b == op_lit_uint:
+            b = next_byte()  # FIXME uleb
+            data.append(int(b))
+        elif b == op_lit_int:
+            b = next_byte()  # FIXME uleb
+            data.append(int(b))
+        elif b == op_lit_selector:
+            b = next_byte()
+            data.append(b)
+        elif b == op_lit_string:
+            length = next_byte()
+            s = ""
+            while length:
+                s += chr(next_byte())
+                length -= 1
+            data.append(s)
+
+        elif b == op_as_uint:
+            pass
+        elif b == op_as_int:
+            pass
+        elif b == op_is_null:
+            data.append(1 if data.pop() == None else 0)
+
+        # Arithmetic, logic, etc.
+        elif b == op_plus:
+            data.append(data.pop() + data.pop())
+        elif b == op_minus:
+            data.append(-data.pop() + data.pop())
+        elif b == op_mul:
+            data.append(data.pop() * data.pop())
+        elif b == op_div:
+            y = data.pop()
+            data.append(data.pop() / y)
+        elif b == op_mod:
+            y = data.pop()
+            data.append(data.pop() % y)
+        elif b == op_shl:
+            y = data.pop()
+            data.append(data.pop() << y)
+        elif b == op_shr:
+            y = data.pop()
+            data.append(data.pop() >> y)
+        elif b == op_and:
+            data.append(data.pop() & data.pop())
+        elif b == op_or:
+            data.append(data.pop() | data.pop())
+        elif b == op_xor:
+            data.append(data.pop() ^ data.pop())
+        elif b == op_not:
+            data.append(not data.pop())
+        elif b == op_eq:
+            data.append(data.pop() == data.pop())
+        elif b == op_neq:
+            data.append(data.pop() != data.pop())
+        elif b == op_lt:
+            data.append(data.pop() > data.pop())
+        elif b == op_gt:
+            data.append(data.pop() < data.pop())
+        elif b == op_le:
+            data.append(data.pop() >= data.pop())
+        elif b == op_ge:
+            data.append(data.pop() <= data.pop())
+
+        # Function calls.
+        elif b == op_call:
+            sel = data.pop()
+            if sel == sel_summary:
+                data.append(data.pop().GetSummary())
+            elif sel == sel_get_num_children:
+                data.append(data.pop().GetNumChildren())
+            elif sel == sel_get_child_at_index:
+                index = data.pop()
+                valobj = data.pop()
+                data.append(valobj.GetChildAtIndex(index))
+            elif sel == sel_get_child_with_name:
+                name = data.pop()
+                valobj = data.pop()
+                data.append(valobj.GetChildMemberWithName(name))
+            elif sel == sel_get_child_index:
+                name = data.pop()
+                valobj = data.pop()
+                data.append(valobj.GetIndexOfChildWithName(name))
+            elif sel == sel_get_type:
+                data.append(data.pop().GetType())
+            elif sel == sel_get_template_argument_type:
+                n = data.pop()
+                valobj = data.pop()
+                data.append(valobj.GetTemplateArgumentType(n))
+            elif sel == sel_get_value:
+                data.append(data.pop().GetValue())
+            elif sel == sel_get_value_as_unsigned:
+                data.append(data.pop().GetValueAsUnsigned())
+            elif sel == sel_get_value_as_signed:
+                data.append(data.pop().GetValueAsSigned())
+            elif sel == sel_get_value_as_address:
+                data.append(data.pop().GetValueAsAddress())
+            elif sel == sel_cast:
+                sbtype = data.pop()
+                valobj = data.pop()
+                data.append(valobj.Cast(sbtype))
+            elif sel == sel_strlen:
+                s = data.pop()
+                data.append(len(s) if s else 0)
+            elif sel == sel_fmt:
+                fmt = data.pop()
+                n = count_fmt_params(fmt)
+                args = []
+                for i in range(n):
+                    args.append(data.pop())
+                data.append(fmt.format(*args))
+            else:
+                print("not implemented: " + selector[sel])
+                assert False
+                pass
+    return data[-1]
+
+
+if __name__ == "__main__":
+    # Work around the fact that one of the local files is called
+    # types.py, which breaks some versions of python.
+    import os, sys
+
+    path = os.path.abspath(os.path.dirname(__file__))
+    sys.path.remove(path)
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="""
+    Compiler, disassembler, and interpreter for LLDB dataformatter bytecode.
+    See https://lldb.llvm.org/resources/formatterbytecode.html for more details.
+    """
+    )
+    parser.add_argument(
+        "-c", "--compile", type=str, help="compile assembler into bytecode"
+    )
+    parser.add_argument("-d", "--disassemble", type=str, help="disassemble bytecode")
+    parser.add_argument("-t", "--test", action="store_true", help="run unit tests")
+    args = parser.parse_args()
+    if args.compile:
+        print(compile(str(args.compile)).hex())
+
+    if args.disassemble:
+        print(disassemble(bytearray.fromhex(str(args.disassemble))))
+
+    ############################################################################
+    # Tests.
+    ############################################################################
+    if args.test:
+        import unittest
+
+        class TestCompiler(unittest.TestCase):
+            def test(self):
+                self.assertEqual(compile("1u dup").hex(), "200101")
+                self.assertEqual(compile('"1u dup"').hex(), "2206317520647570")
+                self.assertEqual(compile("16 < { dup } if").hex(), "21105210010111")
+                self.assertEqual(compile('{ { " } " } }').hex(), "100710052203207d20")
+
+                def roundtrip(asm):
+                    self.assertEqual(disassemble(compile(asm))[0], asm)
+
+                roundtrip("1u dup")
+                roundtrip('1u dup "1u dup"')
+                roundtrip("16 < { dup } if")
+                roundtrip('{ { " } " } }')
+
+                self.assertEqual(interpret(compile("1 1 +"), [], []), 2)
+                self.assertEqual(interpret(compile("2 1 1 + *"), [], []), 4)
+                self.assertEqual(
+                    interpret(compile('2 1 > { "yes" } { "no" } ifelse'), [], []), "yes"
+                )
+
+                import sys
+
+                sys.argv.pop()
+                path = os.path.dirname(__file__)
+                sys.path.remove
+                unittest.main()
diff --git a/lldb/include/lldb/API/SBAddressRangeList.h b/lldb/include/lldb/API/SBAddressRangeList.h
index 5a4eeecf37dc9..41085b1edf8d7 100644
--- a/lldb/include/lldb/API/SBAddressRangeList.h
+++ b/lldb/include/lldb/API/SBAddressRangeList.h
@@ -45,6 +45,7 @@ class LLDB_API SBAddressRangeList {
 private:
   friend class SBBlock;
   friend class SBProcess;
+  friend class SBFunction;
 
   lldb_private::AddressRangeListImpl &ref() const;
 
diff --git a/lldb/include/lldb/API/SBFunction.h b/lldb/include/lldb/API/SBFunction.h
index df607fdc7ebf5..0a8aeeff1ea5a 100644
--- a/lldb/include/lldb/API/SBFunction.h
+++ b/lldb/include/lldb/API/SBFunction.h
@@ -43,6 +43,8 @@ class LLDB_API SBFunction {
 
   lldb::SBAddress GetStartAddress();
 
+  LLDB_DEPRECATED_FIXME("Not compatible with discontinuous functions.",
+                        "GetRanges()")
   lldb::SBAddress GetEndAddress();
 
   lldb::SBAddressRangeList GetRanges();
diff --git a/lldb/include/lldb/Core/AddressRangeListImpl.h b/lldb/include/lldb/Core/AddressRangeListImpl.h
index 6742e6ead87de..6b88f9b1ac179 100644
--- a/lldb/include/lldb/Core/AddressRangeListImpl.h
+++ b/lldb/include/lldb/Core/AddressRangeListImpl.h
@@ -24,9 +24,8 @@ class AddressRangeListImpl {
 public:
   AddressRangeListImpl();
 
-  AddressRangeListImpl(const AddressRangeListImpl &rhs) = default;
-
-  AddressRangeListImpl &operator=(const AddressRangeListImpl &rhs);
+  explicit AddressRangeListImpl(AddressRanges ranges)
+      : m_ranges(std::move(ranges)) {}
 
   size_t GetSize() const;
 
diff --git a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
index c7f8cccc116c4..ce15963ab5662 100644
--- a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
+++ b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
@@ -22,13 +22,12 @@ namespace lldb_private {
 class DumpValueObjectOptions {
 public:
   struct PointerDepth {
-    enum class Mode { Always, Default, Never } m_mode;
-    uint32_t m_count;
+    uint32_t m_count = 0;
 
     PointerDepth Decremented() const {
       if (m_count > 0)
-        return PointerDepth{m_mode, m_count - 1};
-      return PointerDepth{m_mode, m_count};
+        return {m_count - 1};
+      return *this;
     }
 
     bool CanAllowExpansion() const;
@@ -65,8 +64,7 @@ class DumpValueObjectOptions {
 
   DumpValueObjectOptions(ValueObject &valobj);
 
-  DumpValueObjectOptions &
-  SetMaximumPointerDepth(PointerDepth depth = {PointerDepth::Mode::Never, 0});
+  DumpValueObjectOptions &SetMaximumPointerDepth(uint32_t depth);
 
   DumpValueObjectOptions &SetMaximumDepth(uint32_t depth, bool is_default);
 
diff --git a/lldb/include/lldb/DataFormatters/FormatterSection.h b/lldb/include/lldb/DataFormatters/FormatterSection.h
new file mode 100644
index 0000000000000..0613fba6c3a60
--- /dev/null
+++ b/lldb/include/lldb/DataFormatters/FormatterSection.h
@@ -0,0 +1,26 @@
+//===-- FormattersSection.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_DATAFORMATTERS_FORMATTERSECTION_H
+#define LLDB_DATAFORMATTERS_FORMATTERSECTION_H
+
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
+
+namespace lldb_private {
+/// Load type summaries embedded in the binary. These are type summaries
+/// provided by the authors of the code.
+void LoadTypeSummariesForModule(lldb::ModuleSP module_sp);
+
+/// Load data formatters embedded in the binary. These are formatters provided
+/// by the authors of the code using LLDB formatter bytecode.
+void LoadFormattersForModule(lldb::ModuleSP module_sp);
+
+} // namespace lldb_private
+
+#endif // LLDB_DATAFORMATTERS_FORMATTERSECTION_H
diff --git a/lldb/include/lldb/DataFormatters/FormattersHelpers.h b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
index a2e8521d96651..a98042fd40f93 100644
--- a/lldb/include/lldb/DataFormatters/FormattersHelpers.h
+++ b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
@@ -1,5 +1,4 @@
-//===-- FormattersHelpers.h --------------------------------------*- C++
-//-*-===//
+//===-- FormattersHelpers.h -------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/lldb/include/lldb/DataFormatters/TypeSummary.h b/lldb/include/lldb/DataFormatters/TypeSummary.h
index 382824aa2813d..f4d5563516ecb 100644
--- a/lldb/include/lldb/DataFormatters/TypeSummary.h
+++ b/lldb/include/lldb/DataFormatters/TypeSummary.h
@@ -22,6 +22,10 @@
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StructuredData.h"
 
+namespace llvm {
+class MemoryBuffer;
+}
+
 namespace lldb_private {
 class TypeSummaryOptions {
 public:
@@ -44,7 +48,7 @@ class TypeSummaryOptions {
 
 class TypeSummaryImpl {
 public:
-  enum class Kind { eSummaryString, eScript, eCallback, eInternal };
+  enum class Kind { eSummaryString, eScript, eBytecode, eCallback, eInternal };
 
   virtual ~TypeSummaryImpl() = default;
 
@@ -409,6 +413,23 @@ struct ScriptSummaryFormat : public TypeSummaryImpl {
   ScriptSummaryFormat(const ScriptSummaryFormat &) = delete;
   const ScriptSummaryFormat &operator=(const ScriptSummaryFormat &) = delete;
 };
+
+/// A summary formatter that is defined in LLDB formmater bytecode.
+class BytecodeSummaryFormat : public TypeSummaryImpl {
+  std::unique_ptr<llvm::MemoryBuffer> m_bytecode;
+
+public:
+  BytecodeSummaryFormat(const TypeSummaryImpl::Flags &flags,
+                        std::unique_ptr<llvm::MemoryBuffer> bytecode);
+  bool FormatObject(ValueObject *valobj, std::string &dest,
+                    const TypeSummaryOptions &options) override;
+  std::string GetDescription() override;
+  std::string GetName() override;
+  static bool classof(const TypeSummaryImpl *S) {
+    return S->GetKind() == Kind::eBytecode;
+  }
+};
+
 } // namespace lldb_private
 
 #endif // LLDB_DATAFORMATTERS_TYPESUMMARY_H
diff --git a/lldb/include/lldb/Host/Socket.h b/lldb/include/lldb/Host/Socket.h
index 14468c98ac5a3..4585eac12efb9 100644
--- a/lldb/include/lldb/Host/Socket.h
+++ b/lldb/include/lldb/Host/Socket.h
@@ -11,8 +11,10 @@
 
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "lldb/Host/MainLoopBase.h"
+#include "lldb/Utility/Timeout.h"
 #include "lldb/lldb-private.h"
 
 #include "lldb/Host/SocketAddress.h"
@@ -93,7 +95,6 @@ class Socket : public IOObject {
   static void Terminate();
 
   static std::unique_ptr<Socket> Create(const SocketProtocol protocol,
-                                        bool child_processes_inherit,
                                         Status &error);
 
   virtual Status Connect(llvm::StringRef name) = 0;
@@ -108,20 +109,19 @@ class Socket : public IOObject {
 
   // Accept a single connection and "return" it in the pointer argument. This
   // function blocks until the connection arrives.
-  virtual Status Accept(Socket *&socket);
+  virtual Status Accept(const Timeout<std::micro> &timeout, Socket *&socket);
 
   // Initialize a Tcp Socket object in listening mode.  listen and accept are
   // implemented separately because the caller may wish to manipulate or query
   // the socket after it is initialized, but before entering a blocking accept.
   static llvm::Expected<std::unique_ptr<TCPSocket>>
-  TcpListen(llvm::StringRef host_and_port, bool child_processes_inherit,
-            int backlog = 5);
+  TcpListen(llvm::StringRef host_and_port, int backlog = 5);
 
   static llvm::Expected<std::unique_ptr<Socket>>
-  TcpConnect(llvm::StringRef host_and_port, bool child_processes_inherit);
+  TcpConnect(llvm::StringRef host_and_port);
 
   static llvm::Expected<std::unique_ptr<UDPSocket>>
-  UdpConnect(llvm::StringRef host_and_port, bool child_processes_inherit);
+  UdpConnect(llvm::StringRef host_and_port);
 
   static int GetOption(NativeSocket sockfd, int level, int option_name,
                        int &option_value);
@@ -152,9 +152,13 @@ class Socket : public IOObject {
   // If this Socket is connected then return the URI used to connect.
   virtual std::string GetRemoteConnectionURI() const { return ""; };
 
+  // If the Socket is listening then return the URI for clients to connect.
+  virtual std::vector<std::string> GetListeningConnectionURI() const {
+    return {};
+  }
+
 protected:
-  Socket(SocketProtocol protocol, bool should_close,
-         bool m_child_process_inherit);
+  Socket(SocketProtocol protocol, bool should_close);
 
   virtual size_t Send(const void *buf, const size_t num_bytes);
 
@@ -162,15 +166,12 @@ class Socket : public IOObject {
   static Status GetLastError();
   static void SetLastError(Status &error);
   static NativeSocket CreateSocket(const int domain, const int type,
-                                   const int protocol,
-                                   bool child_processes_inherit, Status &error);
+                                   const int protocol, Status &error);
   static NativeSocket AcceptSocket(NativeSocket sockfd, struct sockaddr *addr,
-                                   socklen_t *addrlen,
-                                   bool child_processes_inherit, Status &error);
+                                   socklen_t *addrlen, Status &error);
 
   SocketProtocol m_protocol;
   NativeSocket m_socket;
-  bool m_child_processes_inherit;
   bool m_should_close_fd;
 };
 
diff --git a/lldb/include/lldb/Host/common/TCPSocket.h b/lldb/include/lldb/Host/common/TCPSocket.h
index eefe0240fe4a9..cb950c0015ea6 100644
--- a/lldb/include/lldb/Host/common/TCPSocket.h
+++ b/lldb/include/lldb/Host/common/TCPSocket.h
@@ -13,13 +13,14 @@
 #include "lldb/Host/Socket.h"
 #include "lldb/Host/SocketAddress.h"
 #include <map>
+#include <string>
+#include <vector>
 
 namespace lldb_private {
 class TCPSocket : public Socket {
 public:
-  TCPSocket(bool should_close, bool child_processes_inherit);
-  TCPSocket(NativeSocket socket, bool should_close,
-            bool child_processes_inherit);
+  explicit TCPSocket(bool should_close);
+  TCPSocket(NativeSocket socket, bool should_close);
   ~TCPSocket() override;
 
   // returns port number or 0 if error
@@ -53,6 +54,8 @@ class TCPSocket : public Socket {
 
   std::string GetRemoteConnectionURI() const override;
 
+  std::vector<std::string> GetListeningConnectionURI() const override;
+
 private:
   TCPSocket(NativeSocket socket, const TCPSocket &listen_socket);
 
diff --git a/lldb/include/lldb/Host/common/UDPSocket.h b/lldb/include/lldb/Host/common/UDPSocket.h
index 7348010d02ada..63012d4fb6025 100644
--- a/lldb/include/lldb/Host/common/UDPSocket.h
+++ b/lldb/include/lldb/Host/common/UDPSocket.h
@@ -14,10 +14,10 @@
 namespace lldb_private {
 class UDPSocket : public Socket {
 public:
-  UDPSocket(bool should_close, bool child_processes_inherit);
+  explicit UDPSocket(bool should_close);
 
   static llvm::Expected<std::unique_ptr<UDPSocket>>
-  Connect(llvm::StringRef name, bool child_processes_inherit);
+  CreateConnected(llvm::StringRef name);
 
   std::string GetRemoteConnectionURI() const override;
 
diff --git a/lldb/include/lldb/Host/linux/AbstractSocket.h b/lldb/include/lldb/Host/linux/AbstractSocket.h
index 78a567a6b9095..accfd01457a5e 100644
--- a/lldb/include/lldb/Host/linux/AbstractSocket.h
+++ b/lldb/include/lldb/Host/linux/AbstractSocket.h
@@ -14,7 +14,7 @@
 namespace lldb_private {
 class AbstractSocket : public DomainSocket {
 public:
-  AbstractSocket(bool child_processes_inherit);
+  AbstractSocket();
 
 protected:
   size_t GetNameOffset() const override;
diff --git a/lldb/include/lldb/Host/posix/DomainSocket.h b/lldb/include/lldb/Host/posix/DomainSocket.h
index 983f43bd63371..3dbe6206da2c5 100644
--- a/lldb/include/lldb/Host/posix/DomainSocket.h
+++ b/lldb/include/lldb/Host/posix/DomainSocket.h
@@ -10,13 +10,14 @@
 #define LLDB_HOST_POSIX_DOMAINSOCKET_H
 
 #include "lldb/Host/Socket.h"
+#include <string>
+#include <vector>
 
 namespace lldb_private {
 class DomainSocket : public Socket {
 public:
-  DomainSocket(NativeSocket socket, bool should_close,
-               bool child_processes_inherit);
-  DomainSocket(bool should_close, bool child_processes_inherit);
+  DomainSocket(NativeSocket socket, bool should_close);
+  explicit DomainSocket(bool should_close);
 
   Status Connect(llvm::StringRef name) override;
   Status Listen(llvm::StringRef name, int backlog) override;
@@ -28,8 +29,10 @@ class DomainSocket : public Socket {
 
   std::string GetRemoteConnectionURI() const override;
 
+  std::vector<std::string> GetListeningConnectionURI() const override;
+
 protected:
-  DomainSocket(SocketProtocol protocol, bool child_processes_inherit);
+  DomainSocket(SocketProtocol protocol);
 
   virtual size_t GetNameOffset() const;
   virtual void DeleteSocketFile(llvm::StringRef name);
diff --git a/lldb/include/lldb/Symbol/Block.h b/lldb/include/lldb/Symbol/Block.h
index c9c4d5ad767d7..7c7a66de83199 100644
--- a/lldb/include/lldb/Symbol/Block.h
+++ b/lldb/include/lldb/Symbol/Block.h
@@ -43,11 +43,13 @@ class Block : public UserID, public SymbolContextScope {
   typedef RangeVector<uint32_t, uint32_t, 1> RangeList;
   typedef RangeList::Entry Range;
 
-  /// Construct with a User ID \a uid, \a depth.
-  ///
-  /// Initialize this block with the specified UID \a uid. The \a depth in the
-  /// \a block_list is used to represent the parent, sibling, and child block
-  /// information and also allows for partial parsing at the block level.
+  // Creates a block representing the whole function. Only meant to be used from
+  // the Function class.
+  Block(Function &function, lldb::user_id_t function_uid);
+
+  ~Block() override;
+
+  /// Creates a block with the specified UID \a uid.
   ///
   /// \param[in] uid
   ///     The UID for a given block. This value is given by the
@@ -56,19 +58,7 @@ class Block : public UserID, public SymbolContextScope {
   ///     information data that it parses for further or more in
   ///     depth parsing. Common values would be the index into a
   ///     table, or an offset into the debug information.
-  ///
-  /// \see BlockList
-  Block(lldb::user_id_t uid);
-
-  /// Destructor.
-  ~Block() override;
-
-  /// Add a child to this object.
-  ///
-  /// \param[in] child_block_sp
-  ///     A shared pointer to a child block that will get added to
-  ///     this block.
-  void AddChild(const lldb::BlockSP &child_block_sp);
+  lldb::BlockSP CreateChild(lldb::user_id_t uid);
 
   /// Add a new offset range to this block.
   void AddRange(const Range &range);
@@ -317,10 +307,6 @@ class Block : public UserID, public SymbolContextScope {
                               const Declaration *decl_ptr,
                               const Declaration *call_decl_ptr);
 
-  void SetParentScope(SymbolContextScope *parent_scope) {
-    m_parent_scope = parent_scope;
-  }
-
   /// Set accessor for the variable list.
   ///
   /// Called by the SymbolFile plug-ins after they have parsed the variable
@@ -364,7 +350,7 @@ class Block : public UserID, public SymbolContextScope {
 protected:
   typedef std::vector<lldb::BlockSP> collection;
   // Member variables.
-  SymbolContextScope *m_parent_scope;
+  SymbolContextScope &m_parent_scope;
   collection m_children;
   RangeList m_ranges;
   lldb::InlineFunctionInfoSP m_inlineInfoSP; ///< Inlined function information.
@@ -382,6 +368,8 @@ class Block : public UserID, public SymbolContextScope {
 private:
   Block(const Block &) = delete;
   const Block &operator=(const Block &) = delete;
+
+  Block(lldb::user_id_t uid, SymbolContextScope &parent_scope);
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Symbol/Function.h b/lldb/include/lldb/Symbol/Function.h
index 70f51a846f8d9..e4118c1f9be86 100644
--- a/lldb/include/lldb/Symbol/Function.h
+++ b/lldb/include/lldb/Symbol/Function.h
@@ -444,8 +444,11 @@ class Function : public UserID, public SymbolContextScope {
 
   Function *CalculateSymbolContextFunction() override;
 
+  /// DEPRECATED: Use GetAddressRanges instead.
   const AddressRange &GetAddressRange() { return m_range; }
 
+  AddressRanges GetAddressRanges() { return m_block.GetRanges(); }
+
   lldb::LanguageType GetLanguage() const;
   /// Find the file and line number of the source location of the start of the
   /// function.  This will use the declaration if present and fall back on the
@@ -650,9 +653,6 @@ class Function : public UserID, public SymbolContextScope {
   /// All lexical blocks contained in this function.
   Block m_block;
 
-  /// List of address ranges belonging to the function.
-  AddressRanges m_ranges;
-
   /// The function address range that covers the widest range needed to contain
   /// all blocks. DEPRECATED: do not use this field in new code as the range may
   /// include addresses belonging to other functions.
diff --git a/lldb/include/lldb/Target/Platform.h b/lldb/include/lldb/Target/Platform.h
index 920f80bc73317..a702abb540fd9 100644
--- a/lldb/include/lldb/Target/Platform.h
+++ b/lldb/include/lldb/Target/Platform.h
@@ -473,7 +473,31 @@ class Platform : public PluginInterface {
                       LLVM_PRETTY_FUNCTION, GetName()));
   }
 
-  const std::string &GetRemoteURL() const { return m_remote_url; }
+  /// Search CU for the SDK path the CUs was compiled against.
+  ///
+  /// \param[in] unit The CU
+  ///
+  /// \returns A parsed XcodeSDK object if successful, an Error otherwise. 
+  virtual llvm::Expected<XcodeSDK> GetSDKPathFromDebugInfo(CompileUnit &unit) {
+    return llvm::createStringError(
+        llvm::formatv("{0} not implemented for '{1}' platform.",
+                      LLVM_PRETTY_FUNCTION, GetName()));
+  }
+
+  /// Returns the full path of the most appropriate SDK for the
+  /// specified compile unit. This function gets this path by parsing
+  /// debug-info (see \ref `GetSDKPathFromDebugInfo`).
+  ///
+  /// \param[in] unit The CU to scan.
+  ///
+  /// \returns If successful, returns the full path to an
+  ///          Xcode SDK.
+  virtual llvm::Expected<std::string>
+  ResolveSDKPathFromDebugInfo(CompileUnit &unit) {
+    return llvm::createStringError(
+        llvm::formatv("{0} not implemented for '{1}' platform.",
+                      LLVM_PRETTY_FUNCTION, GetName()));
+  }
 
   bool IsHost() const {
     return m_is_host; // Is this the default host platform?
@@ -977,7 +1001,6 @@ class Platform : public PluginInterface {
   std::string m_sdk_build;
   FileSpec m_working_dir; // The working directory which is used when installing
                           // modules that have no install path set
-  std::string m_remote_url;
   std::string m_hostname;
   llvm::VersionTuple m_os_version;
   ArchSpec
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index a9a66ea2662f3..0094fcd596fdf 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -763,6 +763,8 @@ enum SectionType {
   eSectionTypeDWARFDebugLocListsDwo,
   eSectionTypeDWARFDebugTuIndex,
   eSectionTypeCTF,
+  eSectionTypeLLDBTypeSummaries,
+  eSectionTypeLLDBFormatters,
   eSectionTypeSwiftModules,
 };
 
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 8884ef5933ada..1338d16a9171e 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -865,13 +865,9 @@ def setUp(self):
         session_file = self.getLogBasenameForCurrentTest() + ".log"
         self.log_files.append(session_file)
 
-        # Python 3 doesn't support unbuffered I/O in text mode.  Open buffered.
-        self.session = encoded_file.open(session_file, "utf-8", mode="w")
-
         # Optimistically set __errored__, __failed__, __expected__ to False
         # initially.  If the test errored/failed, the session info
-        # (self.session) is then dumped into a session specific file for
-        # diagnosis.
+        # is then dumped into a session specific file for diagnosis.
         self.__cleanup_errored__ = False
         self.__errored__ = False
         self.__failed__ = False
@@ -1235,20 +1231,25 @@ def dumpSessionInfo(self):
         else:
             prefix = "Success"
 
+        session_file = self.getLogBasenameForCurrentTest() + ".log"
+
+        # Python 3 doesn't support unbuffered I/O in text mode.  Open buffered.
+        session = encoded_file.open(session_file, "utf-8", mode="w")
+
         if not self.__unexpected__ and not self.__skipped__:
             for test, traceback in pairs:
                 if test is self:
-                    print(traceback, file=self.session)
+                    print(traceback, file=session)
 
         import datetime
 
         print(
             "Session info generated @",
             datetime.datetime.now().ctime(),
-            file=self.session,
+            file=session,
         )
-        self.session.close()
-        del self.session
+        session.close()
+        del session
 
         # process the log files
         if prefix != "Success" or lldbtest_config.log_success:
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
index 8c8e4abed0b45..d6cb68f55bf29 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
@@ -185,7 +185,7 @@ def setUpServerLogging(self, is_llgs):
             ]
 
     def get_next_port(self):
-        return 12000 + random.randint(0, 3999)
+        return 12000 + random.randint(0, 7999)
 
     def reset_test_sequence(self):
         self.test_sequence = GdbRemoteTestSequence(self.logger)
@@ -388,7 +388,8 @@ def connect_to_debug_monitor(self, attach_pid=None):
         # We're using a random port algorithm to try not to collide with other ports,
         # and retry a max # times.
         attempts = 0
-        MAX_ATTEMPTS = 20
+        MAX_ATTEMPTS = 10
+        attempt_wait = 3
 
         while attempts < MAX_ATTEMPTS:
             server = self.launch_debug_monitor(attach_pid=attach_pid)
@@ -424,7 +425,8 @@ def connect_to_debug_monitor(self, attach_pid=None):
 
             # And wait a random length of time before next attempt, to avoid
             # collisions.
-            time.sleep(random.randint(1, 5))
+            time.sleep(attempt_wait)
+            attempt_wait *= 1.2
 
             # Now grab a new port number.
             self.port = self.get_next_port()
diff --git a/lldb/source/API/SBFunction.cpp b/lldb/source/API/SBFunction.cpp
index ac61220ec8736..3f6b4eea98318 100644
--- a/lldb/source/API/SBFunction.cpp
+++ b/lldb/source/API/SBFunction.cpp
@@ -10,6 +10,7 @@
 #include "lldb/API/SBAddressRange.h"
 #include "lldb/API/SBProcess.h"
 #include "lldb/API/SBStream.h"
+#include "lldb/Core/AddressRangeListImpl.h"
 #include "lldb/Core/Disassembler.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Symbol/CompileUnit.h"
@@ -153,10 +154,11 @@ SBAddress SBFunction::GetEndAddress() {
 
   SBAddress addr;
   if (m_opaque_ptr) {
-    addr_t byte_size = m_opaque_ptr->GetAddressRange().GetByteSize();
-    if (byte_size > 0) {
-      addr.SetAddress(m_opaque_ptr->GetAddressRange().GetBaseAddress());
-      addr->Slide(byte_size);
+    AddressRanges ranges = m_opaque_ptr->GetAddressRanges();
+    if (!ranges.empty()) {
+      // Return the end of the first range, use GetRanges to get all ranges.
+      addr.SetAddress(ranges.front().GetBaseAddress());
+      addr->Slide(ranges.front().GetByteSize());
     }
   }
   return addr;
@@ -166,11 +168,8 @@ lldb::SBAddressRangeList SBFunction::GetRanges() {
   LLDB_INSTRUMENT_VA(this);
 
   lldb::SBAddressRangeList ranges;
-  if (m_opaque_ptr) {
-    lldb::SBAddressRange range;
-    (*range.m_opaque_up) = m_opaque_ptr->GetAddressRange();
-    ranges.Append(std::move(range));
-  }
+  if (m_opaque_ptr)
+    ranges.ref() = AddressRangeListImpl(m_opaque_ptr->GetAddressRanges());
 
   return ranges;
 }
diff --git a/lldb/source/API/SBTypeSummary.cpp b/lldb/source/API/SBTypeSummary.cpp
index a5e87188c9f63..856ee0ed3175b 100644
--- a/lldb/source/API/SBTypeSummary.cpp
+++ b/lldb/source/API/SBTypeSummary.cpp
@@ -343,6 +343,7 @@ bool SBTypeSummary::IsEqualTo(lldb::SBTypeSummary &rhs) {
   case TypeSummaryImpl::Kind::eCallback:
     return llvm::dyn_cast<CXXFunctionSummaryFormat>(m_opaque_sp.get()) ==
            llvm::dyn_cast<CXXFunctionSummaryFormat>(rhs.m_opaque_sp.get());
+  case TypeSummaryImpl::Kind::eBytecode:
   case TypeSummaryImpl::Kind::eScript:
     if (IsFunctionCode() != rhs.IsFunctionCode())
       return false;
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index 62c4e74d853ad..d4d038d28f675 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -151,10 +151,24 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
     result.SetStatus(eReturnStatusSuccessFinishResult);
   };
 
-  // First, try `expr` as the name of a frame variable.
-  if (frame) {
-    auto valobj_sp = frame->FindVariable(ConstString(expr));
-    if (valobj_sp && valobj_sp->GetError().Success()) {
+  // First, try `expr` as a _limited_ frame variable expression path: only the
+  // dot operator (`.`) is permitted for this case.
+  //
+  // This is limited to support only unambiguous expression paths. Of note,
+  // expression paths are not attempted if the expression contain either the
+  // arrow operator (`->`) or the subscript operator (`[]`). This is because
+  // both operators can be overloaded in C++, and could result in ambiguity in
+  // how the expression is handled. Additionally, `*` and `&` are not supported.
+  const bool try_variable_path =
+      expr.find_first_of("*&->[]") == StringRef::npos;
+  if (frame && try_variable_path) {
+    VariableSP var_sp;
+    Status status;
+    auto valobj_sp = frame->GetValueForVariableExpressionPath(
+        expr, eval_options.GetUseDynamic(),
+        StackFrame::eExpressionPathOptionsAllowDirectIVarAccess, var_sp,
+        status);
+    if (valobj_sp && status.Success() && valobj_sp->GetError().Success()) {
       if (!suppress_result) {
         if (auto persisted_valobj = valobj_sp->Persist())
           valobj_sp = persisted_valobj;
diff --git a/lldb/source/Core/AddressRangeListImpl.cpp b/lldb/source/Core/AddressRangeListImpl.cpp
index d405cf0fa3ec3..257824a0551e1 100644
--- a/lldb/source/Core/AddressRangeListImpl.cpp
+++ b/lldb/source/Core/AddressRangeListImpl.cpp
@@ -13,14 +13,6 @@ using namespace lldb_private;
 
 AddressRangeListImpl::AddressRangeListImpl() : m_ranges() {}
 
-AddressRangeListImpl &
-AddressRangeListImpl::operator=(const AddressRangeListImpl &rhs) {
-  if (this == &rhs)
-    return *this;
-  m_ranges = rhs.m_ranges;
-  return *this;
-}
-
 size_t AddressRangeListImpl::GetSize() const { return m_ranges.size(); }
 
 void AddressRangeListImpl::Reserve(size_t capacity) {
diff --git a/lldb/source/Core/Section.cpp b/lldb/source/Core/Section.cpp
index 0763e88d4608f..31273ede618f2 100644
--- a/lldb/source/Core/Section.cpp
+++ b/lldb/source/Core/Section.cpp
@@ -147,10 +147,14 @@ const char *Section::GetTypeAsCString() const {
     return "dwarf-gnu-debugaltlink";
   case eSectionTypeCTF:
     return "ctf";
-  case eSectionTypeOther:
-    return "regular";
+  case eSectionTypeLLDBTypeSummaries:
+    return "lldb-type-summaries";
+  case eSectionTypeLLDBFormatters:
+    return "lldb-formatters";
   case eSectionTypeSwiftModules:
     return "swift-modules";
+  case eSectionTypeOther:
+    return "regular";
   }
   return "unknown";
 }
@@ -457,6 +461,8 @@ bool Section::ContainsOnlyDebugInfo() const {
   case eSectionTypeDWARFAppleObjC:
   case eSectionTypeDWARFGNUDebugAltLink:
   case eSectionTypeCTF:
+  case eSectionTypeLLDBTypeSummaries:
+  case eSectionTypeLLDBFormatters:
   case eSectionTypeSwiftModules:
     return true;
   }
diff --git a/lldb/source/DataFormatters/CMakeLists.txt b/lldb/source/DataFormatters/CMakeLists.txt
index 13faf65227d2c..91b10ba9e0ac8 100644
--- a/lldb/source/DataFormatters/CMakeLists.txt
+++ b/lldb/source/DataFormatters/CMakeLists.txt
@@ -5,7 +5,9 @@ add_lldb_library(lldbDataFormatters NO_PLUGIN_DEPENDENCIES
   FormatCache.cpp
   FormatClasses.cpp
   FormatManager.cpp
+  FormatterBytecode.cpp
   FormattersHelpers.cpp
+  FormatterSection.cpp
   LanguageCategory.cpp
   StringPrinter.cpp
   TypeCategory.cpp
diff --git a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
index 18d590d47d9a0..b952fb643f13e 100644
--- a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
+++ b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
@@ -14,10 +14,8 @@ using namespace lldb;
 using namespace lldb_private;
 
 DumpValueObjectOptions::DumpValueObjectOptions()
-    : m_summary_sp(), m_root_valobj_name(),
-      m_max_ptr_depth(PointerDepth{PointerDepth::Mode::Default, 0}),
-      m_decl_printing_helper(), m_child_printing_decider(),
-      m_pointer_as_array(), m_use_synthetic(true),
+    : m_summary_sp(), m_root_valobj_name(), m_decl_printing_helper(),
+      m_child_printing_decider(), m_pointer_as_array(), m_use_synthetic(true),
       m_scope_already_checked(false), m_flat_output(false), m_ignore_cap(false),
       m_show_types(false), m_show_location(false), m_use_objc(false),
       m_hide_root_type(false), m_hide_root_name(false), m_hide_name(false),
@@ -33,8 +31,8 @@ DumpValueObjectOptions::DumpValueObjectOptions(ValueObject &valobj)
 }
 
 DumpValueObjectOptions &
-DumpValueObjectOptions::SetMaximumPointerDepth(PointerDepth depth) {
-  m_max_ptr_depth = depth;
+DumpValueObjectOptions::SetMaximumPointerDepth(uint32_t depth) {
+  m_max_ptr_depth = {depth};
   return *this;
 }
 
diff --git a/lldb/source/DataFormatters/FormatterBytecode.cpp b/lldb/source/DataFormatters/FormatterBytecode.cpp
new file mode 100644
index 0000000000000..f344fbaff6f02
--- /dev/null
+++ b/lldb/source/DataFormatters/FormatterBytecode.cpp
@@ -0,0 +1,573 @@
+//===-- FormatterBytecode.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FormatterBytecode.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/ValueObject/ValueObject.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/FormatProviders.h"
+#include "llvm/Support/FormatVariadicDetails.h"
+
+using namespace lldb;
+namespace lldb_private {
+
+std::string toString(FormatterBytecode::OpCodes op) {
+  switch (op) {
+#define DEFINE_OPCODE(OP, MNEMONIC, NAME)                                      \
+  case OP: {                                                                   \
+    const char *s = MNEMONIC;                                                  \
+    return s ? s : #NAME;                                                      \
+  }
+#include "FormatterBytecode.def"
+#undef DEFINE_SIGNATURE
+  }
+  return llvm::utostr(op);
+}
+
+std::string toString(FormatterBytecode::Selectors sel) {
+  switch (sel) {
+#define DEFINE_SELECTOR(ID, NAME)                                              \
+  case ID:                                                                     \
+    return "@" #NAME;
+#include "FormatterBytecode.def"
+#undef DEFINE_SIGNATURE
+  }
+  return "@" + llvm::utostr(sel);
+}
+
+std::string toString(FormatterBytecode::Signatures sig) {
+  switch (sig) {
+#define DEFINE_SIGNATURE(ID, NAME)                                             \
+  case ID:                                                                     \
+    return "@" #NAME;
+#include "FormatterBytecode.def"
+#undef DEFINE_SIGNATURE
+  }
+  return llvm::utostr(sig);
+}
+
+std::string toString(const FormatterBytecode::DataStack &data) {
+  std::string s;
+  llvm::raw_string_ostream os(s);
+  os << "[ ";
+  for (auto &d : data) {
+    if (auto s = std::get_if<std::string>(&d))
+      os << '"' << *s << '"';
+    else if (auto u = std::get_if<uint64_t>(&d))
+      os << *u << 'u';
+    else if (auto i = std::get_if<int64_t>(&d))
+      os << *i;
+    else if (auto valobj = std::get_if<ValueObjectSP>(&d)) {
+      if (!valobj->get())
+        os << "null";
+      else
+        os << "object(" << valobj->get()->GetValueAsCString() << ')';
+    } else if (auto type = std::get_if<CompilerType>(&d)) {
+      os << '(' << type->GetTypeName(true) << ')';
+    } else if (auto sel = std::get_if<FormatterBytecode::Selectors>(&d)) {
+      os << toString(*sel);
+    }
+    os << ' ';
+  }
+  os << ']';
+  return s;
+}
+
+namespace FormatterBytecode {
+
+/// Implement the @format function.
+static llvm::Error FormatImpl(DataStack &data) {
+  auto fmt = data.Pop<std::string>();
+  auto replacements =
+      llvm::formatv_object_base::parseFormatString(fmt, 0, false);
+  std::string s;
+  llvm::raw_string_ostream os(s);
+  unsigned num_args = 0;
+  for (const auto &r : replacements)
+    if (r.Type == llvm::ReplacementType::Format)
+      num_args = std::max(num_args, r.Index + 1);
+
+  if (data.size() < num_args)
+    return llvm::createStringError("not enough arguments");
+
+  for (const auto &r : replacements) {
+    if (r.Type == llvm::ReplacementType::Literal) {
+      os << r.Spec;
+      continue;
+    }
+    using namespace llvm::support::detail;
+    auto arg = data[data.size() - num_args + r.Index];
+    auto format = [&](format_adapter &&adapter) {
+      llvm::FmtAlign Align(adapter, r.Where, r.Width, r.Pad);
+      Align.format(os, r.Options);
+    };
+
+    if (auto s = std::get_if<std::string>(&arg))
+      format(build_format_adapter(s->c_str()));
+    else if (auto u = std::get_if<uint64_t>(&arg))
+      format(build_format_adapter(u));
+    else if (auto i = std::get_if<int64_t>(&arg))
+      format(build_format_adapter(i));
+    else if (auto valobj = std::get_if<ValueObjectSP>(&arg)) {
+      if (!valobj->get())
+        format(build_format_adapter("null object"));
+      else
+        format(build_format_adapter(valobj->get()->GetValueAsCString()));
+    } else if (auto type = std::get_if<CompilerType>(&arg))
+      format(build_format_adapter(type->GetDisplayTypeName()));
+    else if (auto sel = std::get_if<FormatterBytecode::Selectors>(&arg))
+      format(build_format_adapter(toString(*sel)));
+  }
+  data.Push(s);
+  return llvm::Error::success();
+}
+
+static llvm::Error TypeCheck(llvm::ArrayRef<DataStackElement> data,
+                             DataType type) {
+  if (data.size() < 1)
+    return llvm::createStringError("not enough elements on data stack");
+
+  auto &elem = data.back();
+  switch (type) {
+  case Any:
+    break;
+  case String:
+    if (!std::holds_alternative<std::string>(elem))
+      return llvm::createStringError("expected String");
+    break;
+  case UInt:
+    if (!std::holds_alternative<uint64_t>(elem))
+      return llvm::createStringError("expected UInt");
+    break;
+  case Int:
+    if (!std::holds_alternative<int64_t>(elem))
+      return llvm::createStringError("expected Int");
+    break;
+  case Object:
+    if (!std::holds_alternative<ValueObjectSP>(elem))
+      return llvm::createStringError("expected Object");
+    break;
+  case Type:
+    if (!std::holds_alternative<CompilerType>(elem))
+      return llvm::createStringError("expected Type");
+    break;
+  case Selector:
+    if (!std::holds_alternative<Selectors>(elem))
+      return llvm::createStringError("expected Selector");
+    break;
+  }
+  return llvm::Error::success();
+}
+
+static llvm::Error TypeCheck(llvm::ArrayRef<DataStackElement> data,
+                             DataType type1, DataType type2) {
+  if (auto error = TypeCheck(data, type2))
+    return error;
+  return TypeCheck(data.drop_back(), type1);
+}
+
+static llvm::Error TypeCheck(llvm::ArrayRef<DataStackElement> data,
+                             DataType type1, DataType type2, DataType type3) {
+  if (auto error = TypeCheck(data, type3))
+    return error;
+  return TypeCheck(data.drop_back(1), type2, type1);
+}
+
+llvm::Error Interpret(std::vector<ControlStackElement> &control,
+                      DataStack &data, Selectors sel) {
+  if (control.empty())
+    return llvm::Error::success();
+  // Since the only data types are single endian and ULEBs, the
+  // endianness should not matter.
+  llvm::DataExtractor cur_block(control.back(), true, 64);
+  llvm::DataExtractor::Cursor pc(0);
+
+  while (!control.empty()) {
+    /// Activate the top most block from the control stack.
+    auto activate_block = [&]() {
+      // Save the return address.
+      if (control.size() > 1)
+        control[control.size() - 2] = cur_block.getData().drop_front(pc.tell());
+      cur_block = llvm::DataExtractor(control.back(), true, 64);
+      if (pc)
+        pc = llvm::DataExtractor::Cursor(0);
+    };
+
+    /// Fetch the next byte in the instruction stream.
+    auto next_byte = [&]() -> uint8_t {
+      // At the end of the current block?
+      while (pc.tell() >= cur_block.size() && !control.empty()) {
+        if (control.size() == 1) {
+          control.pop_back();
+          return 0;
+        }
+        control.pop_back();
+        activate_block();
+      }
+
+      // Fetch the next instruction.
+      return cur_block.getU8(pc);
+    };
+
+    // Fetch the next opcode.
+    OpCodes opcode = (OpCodes)next_byte();
+    if (control.empty() || !pc)
+      return pc.takeError();
+
+    LLDB_LOGV(GetLog(LLDBLog::DataFormatters),
+              "[eval {0}] opcode={1}, control={2}, data={3}", toString(sel),
+              toString(opcode), control.size(), toString(data));
+
+    // Various shorthands to improve the readability of error handling.
+#define TYPE_CHECK(...)                                                        \
+  if (auto error = TypeCheck(data, __VA_ARGS__))                               \
+    return error;
+
+    auto error = [&](llvm::Twine msg) {
+      return llvm::createStringError(msg + "(opcode=" + toString(opcode) + ")");
+    };
+
+    switch (opcode) {
+    // Data stack manipulation.
+    case op_dup:
+      TYPE_CHECK(Any);
+      data.Push(data.back());
+      continue;
+    case op_drop:
+      TYPE_CHECK(Any);
+      data.pop_back();
+      continue;
+    case op_pick: {
+      TYPE_CHECK(UInt);
+      uint64_t idx = data.Pop<uint64_t>();
+      if (idx >= data.size())
+        return error("index out of bounds");
+      data.Push(data[idx]);
+      continue;
+    }
+    case op_over:
+      TYPE_CHECK(Any, Any);
+      data.Push(data[data.size() - 2]);
+      continue;
+    case op_swap: {
+      TYPE_CHECK(Any, Any);
+      auto x = data.PopAny();
+      auto y = data.PopAny();
+      data.Push(x);
+      data.Push(y);
+      continue;
+    }
+    case op_rot: {
+      TYPE_CHECK(Any, Any, Any);
+      auto z = data.PopAny();
+      auto y = data.PopAny();
+      auto x = data.PopAny();
+      data.Push(z);
+      data.Push(x);
+      data.Push(y);
+      continue;
+    }
+
+    // Control stack manipulation.
+    case op_begin: {
+      uint64_t length = cur_block.getULEB128(pc);
+      if (!pc)
+        return pc.takeError();
+      llvm::StringRef block = cur_block.getBytes(pc, length);
+      if (!pc)
+        return pc.takeError();
+      control.push_back(block);
+      continue;
+    }
+    case op_if:
+      TYPE_CHECK(UInt);
+      if (data.Pop<uint64_t>() != 0) {
+        if (!cur_block.size())
+          return error("empty control stack");
+        activate_block();
+      } else
+        control.pop_back();
+      continue;
+    case op_ifelse:
+      TYPE_CHECK(UInt);
+      if (cur_block.size() < 2)
+        return error("empty control stack");
+      if (data.Pop<uint64_t>() == 0)
+        control[control.size() - 2] = control.back();
+      control.pop_back();
+      activate_block();
+      continue;
+
+    // Literals.
+    case op_lit_uint:
+      data.Push(cur_block.getULEB128(pc));
+      continue;
+    case op_lit_int:
+      data.Push(cur_block.getSLEB128(pc));
+      continue;
+    case op_lit_selector:
+      data.Push(Selectors(cur_block.getU8(pc)));
+      continue;
+    case op_lit_string: {
+      uint64_t length = cur_block.getULEB128(pc);
+      llvm::StringRef bytes = cur_block.getBytes(pc, length);
+      data.Push(bytes.str());
+      continue;
+    }
+    case op_as_uint: {
+      TYPE_CHECK(Int);
+      uint64_t casted;
+      int64_t val = data.Pop<int64_t>();
+      memcpy(&casted, &val, sizeof(val));
+      data.Push(casted);
+      continue;
+    }
+    case op_as_int: {
+      TYPE_CHECK(UInt);
+      int64_t casted;
+      uint64_t val = data.Pop<uint64_t>();
+      memcpy(&casted, &val, sizeof(val));
+      data.Push(casted);
+      continue;
+    }
+    case op_is_null: {
+      TYPE_CHECK(Object);
+      data.Push(data.Pop<ValueObjectSP>() ? (uint64_t)0 : (uint64_t)1);
+      continue;
+    }
+
+    // Arithmetic, logic, etc.
+#define BINOP_IMPL(OP, CHECK_ZERO)                                             \
+  {                                                                            \
+    TYPE_CHECK(Any, Any);                                                      \
+    auto y = data.PopAny();                                                    \
+    if (std::holds_alternative<uint64_t>(y)) {                                 \
+      if (CHECK_ZERO && !std::get<uint64_t>(y))                                \
+        return error(#OP " by zero");                                          \
+      TYPE_CHECK(UInt);                                                        \
+      data.Push((uint64_t)(data.Pop<uint64_t>() OP std::get<uint64_t>(y)));    \
+    } else if (std::holds_alternative<int64_t>(y)) {                           \
+      if (CHECK_ZERO && !std::get<int64_t>(y))                                 \
+        return error(#OP " by zero");                                          \
+      TYPE_CHECK(Int);                                                         \
+      data.Push((int64_t)(data.Pop<int64_t>() OP std::get<int64_t>(y)));       \
+    } else                                                                     \
+      return error("unsupported data types");                                  \
+  }
+#define BINOP(OP) BINOP_IMPL(OP, false)
+#define BINOP_CHECKZERO(OP) BINOP_IMPL(OP, true)
+    case op_plus:
+      BINOP(+);
+      continue;
+    case op_minus:
+      BINOP(-);
+      continue;
+    case op_mul:
+      BINOP(*);
+      continue;
+    case op_div:
+      BINOP_CHECKZERO(/);
+      continue;
+    case op_mod:
+      BINOP_CHECKZERO(%);
+      continue;
+    case op_shl:
+#define SHIFTOP(OP)                                                            \
+  {                                                                            \
+    TYPE_CHECK(Any, UInt);                                                     \
+    uint64_t y = data.Pop<uint64_t>();                                         \
+    if (y > 64)                                                                \
+      return error("shift out of bounds");                                     \
+    if (std::holds_alternative<uint64_t>(data.back())) {                       \
+      uint64_t x = data.Pop<uint64_t>();                                       \
+      data.Push(x OP y);                                                       \
+    } else if (std::holds_alternative<int64_t>(data.back())) {                 \
+      int64_t x = data.Pop<int64_t>();                                         \
+      if (y > 64)                                                              \
+        return error("shift out of bounds");                                   \
+      data.Push(x OP y);                                                       \
+    } else                                                                     \
+      return error("unsupported data types");                                  \
+  }
+      SHIFTOP(<<);
+      continue;
+    case op_shr:
+      SHIFTOP(<<);
+      continue;
+    case op_and:
+      BINOP(&);
+      continue;
+    case op_or:
+      BINOP(|);
+      continue;
+    case op_xor:
+      BINOP(^);
+      continue;
+    case op_not:
+      TYPE_CHECK(UInt);
+      data.Push(~data.Pop<uint64_t>());
+      continue;
+    case op_eq:
+      BINOP(==);
+      continue;
+    case op_neq:
+      BINOP(!=);
+      continue;
+    case op_lt:
+      BINOP(<);
+      continue;
+    case op_gt:
+      BINOP(>);
+      continue;
+    case op_le:
+      BINOP(<=);
+      continue;
+    case op_ge:
+      BINOP(>=);
+      continue;
+    case op_call: {
+      TYPE_CHECK(Selector);
+      Selectors sel = data.Pop<Selectors>();
+
+      // Shorthand to improve readability.
+#define POP_VALOBJ(VALOBJ)                                                     \
+  auto VALOBJ = data.Pop<ValueObjectSP>();                                     \
+  if (!VALOBJ)                                                                 \
+    return error("null object");
+
+      auto sel_error = [&](const char *msg) {
+        return llvm::createStringError("{0} (opcode={1}, selector={2})", msg,
+                                       toString(opcode).c_str(),
+                                       toString(sel).c_str());
+      };
+
+      switch (sel) {
+      case sel_summary: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        const char *summary = valobj->GetSummaryAsCString();
+        data.Push(summary ? std::string(valobj->GetSummaryAsCString())
+                          : std::string());
+        break;
+      }
+      case sel_get_num_children: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        auto result = valobj->GetNumChildren();
+        if (!result)
+          return result.takeError();
+        data.Push((uint64_t)*result);
+        break;
+      }
+      case sel_get_child_at_index: {
+        TYPE_CHECK(Object, UInt);
+        auto index = data.Pop<uint64_t>();
+        POP_VALOBJ(valobj);
+        data.Push(valobj->GetChildAtIndex(index));
+        break;
+      }
+      case sel_get_child_with_name: {
+        TYPE_CHECK(Object, String);
+        auto name = data.Pop<std::string>();
+        POP_VALOBJ(valobj);
+        data.Push(valobj->GetChildMemberWithName(name));
+        break;
+      }
+      case sel_get_child_index: {
+        TYPE_CHECK(Object, String);
+        auto name = data.Pop<std::string>();
+        POP_VALOBJ(valobj);
+        data.Push((uint64_t)valobj->GetIndexOfChildWithName(name));
+        break;
+      }
+      case sel_get_type: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        // FIXME: do we need to control dynamic type resolution?
+        data.Push(valobj->GetTypeImpl().GetCompilerType(false));
+        break;
+      }
+      case sel_get_template_argument_type: {
+        TYPE_CHECK(Type, UInt);
+        auto index = data.Pop<uint64_t>();
+        auto type = data.Pop<CompilerType>();
+        // FIXME: There is more code in SBType::GetTemplateArgumentType().
+        data.Push(type.GetTypeTemplateArgument(index, true));
+        break;
+      }
+      case sel_get_value: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        data.Push(std::string(valobj->GetValueAsCString()));
+        break;
+      }
+      case sel_get_value_as_unsigned: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        bool success;
+        uint64_t val = valobj->GetValueAsUnsigned(0, &success);
+        data.Push(val);
+        if (!success)
+          return sel_error("failed to get value");
+        break;
+      }
+      case sel_get_value_as_signed: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        bool success;
+        int64_t val = valobj->GetValueAsSigned(0, &success);
+        data.Push(val);
+        if (!success)
+          return sel_error("failed to get value");
+        break;
+      }
+      case sel_get_value_as_address: {
+        TYPE_CHECK(Object);
+        POP_VALOBJ(valobj);
+        bool success;
+        uint64_t addr = valobj->GetValueAsUnsigned(0, &success);
+        if (!success)
+          return sel_error("failed to get value");
+        if (auto process_sp = valobj->GetProcessSP())
+          addr = process_sp->FixDataAddress(addr);
+        data.Push(addr);
+        break;
+      }
+      case sel_cast: {
+        TYPE_CHECK(Object, Type);
+        auto type = data.Pop<CompilerType>();
+        POP_VALOBJ(valobj);
+        data.Push(valobj->Cast(type));
+        break;
+      }
+      case sel_strlen: {
+        TYPE_CHECK(String);
+        data.Push((uint64_t)data.Pop<std::string>().size());
+        break;
+      }
+      case sel_fmt: {
+        TYPE_CHECK(String);
+        if (auto error = FormatImpl(data))
+          return error;
+        break;
+      }
+      default:
+        return sel_error("selector not implemented");
+      }
+      continue;
+    }
+    }
+    return error("opcode not implemented");
+  }
+  return pc.takeError();
+}
+} // namespace FormatterBytecode
+
+} // namespace lldb_private
diff --git a/lldb/source/DataFormatters/FormatterBytecode.def b/lldb/source/DataFormatters/FormatterBytecode.def
new file mode 100644
index 0000000000000..c6645631fa006
--- /dev/null
+++ b/lldb/source/DataFormatters/FormatterBytecode.def
@@ -0,0 +1,101 @@
+//===-- FormatterBytecode.def -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DEFINE_OPCODE
+#define DEFINE_OPCODE(OP, MNEMONIC, NAME)
+#endif
+#ifndef DEFINE_SELECTOR
+#define DEFINE_SELECTOR(ID, NAME)
+#endif
+#ifndef DEFINE_SIGNATURE
+#define DEFINE_SIGNATURE(ID, NAME)
+#endif
+
+// Opcodes.
+DEFINE_OPCODE(0x01, "dup",  dup)
+DEFINE_OPCODE(0x02, "drop", drop)
+DEFINE_OPCODE(0x03, "pick", pick)
+DEFINE_OPCODE(0x04, "over", over)
+DEFINE_OPCODE(0x05, "swap", swap)
+DEFINE_OPCODE(0x06, "rot",  rot)
+
+DEFINE_OPCODE(0x10, "{", begin)
+DEFINE_OPCODE(0x11, "if", if)
+DEFINE_OPCODE(0x12, "ifelse", ifelse)
+
+DEFINE_OPCODE(0x20, nullptr, lit_uint)
+DEFINE_OPCODE(0x21, nullptr, lit_int)
+DEFINE_OPCODE(0x22, nullptr, lit_string)
+DEFINE_OPCODE(0x23, nullptr, lit_selector)
+
+DEFINE_OPCODE(0x2a, "as_int", as_int)
+DEFINE_OPCODE(0x2b, "as_uint", as_uint)
+DEFINE_OPCODE(0x2c, "is_null", is_null)
+
+DEFINE_OPCODE(0x30, "+", plus)
+DEFINE_OPCODE(0x31, "-", minus)
+DEFINE_OPCODE(0x32, "*", mul)
+DEFINE_OPCODE(0x33, "/", div)
+DEFINE_OPCODE(0x34, "%", mod)
+DEFINE_OPCODE(0x35, "<<", shl)
+DEFINE_OPCODE(0x36, ">>", shr)
+
+DEFINE_OPCODE(0x40, "&", and)
+DEFINE_OPCODE(0x41, "|", or)
+DEFINE_OPCODE(0x42, "^", xor)
+DEFINE_OPCODE(0x43, "~", not)
+
+DEFINE_OPCODE(0x50, "=", eq)
+DEFINE_OPCODE(0x51, "!=", neq)
+DEFINE_OPCODE(0x52, "<", lt)
+DEFINE_OPCODE(0x53, ">", gt)
+DEFINE_OPCODE(0x54, "=<", le)
+DEFINE_OPCODE(0x55, ">=", ge)
+
+DEFINE_OPCODE(0x60, "call", call)
+
+// Selectors.
+DEFINE_SELECTOR(0x00, summary)
+DEFINE_SELECTOR(0x01, type_summary)
+
+DEFINE_SELECTOR(0x10, get_num_children)
+DEFINE_SELECTOR(0x11, get_child_at_index)
+DEFINE_SELECTOR(0x12, get_child_with_name)
+DEFINE_SELECTOR(0x13, get_child_index)
+DEFINE_SELECTOR(0x15, get_type)
+DEFINE_SELECTOR(0x16, get_template_argument_type)
+DEFINE_SELECTOR(0x17, cast)
+
+DEFINE_SELECTOR(0x20, get_value)
+DEFINE_SELECTOR(0x21, get_value_as_unsigned)
+DEFINE_SELECTOR(0x22, get_value_as_signed)
+DEFINE_SELECTOR(0x23, get_value_as_address)
+
+DEFINE_SELECTOR(0x40, read_memory_byte)
+DEFINE_SELECTOR(0x41, read_memory_uint32)
+DEFINE_SELECTOR(0x42, read_memory_int32)
+DEFINE_SELECTOR(0x43, read_memory_unsigned)
+DEFINE_SELECTOR(0x44, read_memory_signed)
+DEFINE_SELECTOR(0x45, read_memory_address)
+DEFINE_SELECTOR(0x46, read_memory)
+
+DEFINE_SELECTOR(0x50, fmt)
+DEFINE_SELECTOR(0x51, sprintf)
+DEFINE_SELECTOR(0x52, strlen)
+
+// Formatter signatures.
+DEFINE_SIGNATURE(0, summary)
+DEFINE_SIGNATURE(1, init)
+DEFINE_SIGNATURE(2, get_num_children)
+DEFINE_SIGNATURE(3, get_child_index)
+DEFINE_SIGNATURE(4, get_child_at_index)
+DEFINE_SIGNATURE(5, get_value)
+
+#undef DEFINE_OPCODE
+#undef DEFINE_SELECTOR
+#undef DEFINE_SIGNATURE
diff --git a/lldb/source/DataFormatters/FormatterBytecode.h b/lldb/source/DataFormatters/FormatterBytecode.h
new file mode 100644
index 0000000000000..21454d9c7e231
--- /dev/null
+++ b/lldb/source/DataFormatters/FormatterBytecode.h
@@ -0,0 +1,64 @@
+//===-- FormatterBytecode.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/DataFormatters/TypeSummary.h"
+#include "lldb/Symbol/CompilerType.h"
+
+namespace lldb_private {
+
+namespace FormatterBytecode {
+
+enum DataType : uint8_t { Any, String, Int, UInt, Object, Type, Selector };
+
+enum OpCodes : uint8_t {
+#define DEFINE_OPCODE(OP, MNEMONIC, NAME) op_##NAME = OP,
+#include "FormatterBytecode.def"
+#undef DEFINE_OPCODE
+};
+
+enum Selectors : uint8_t {
+#define DEFINE_SELECTOR(ID, NAME) sel_##NAME = ID,
+#include "FormatterBytecode.def"
+#undef DEFINE_SELECTOR
+};
+
+enum Signatures : uint8_t {
+#define DEFINE_SIGNATURE(ID, NAME) sig_##NAME = ID,
+#include "FormatterBytecode.def"
+#undef DEFINE_SIGNATURE
+};
+
+using ControlStackElement = llvm::StringRef;
+using DataStackElement =
+    std::variant<std::string, uint64_t, int64_t, lldb::ValueObjectSP,
+                 CompilerType, Selectors>;
+struct DataStack : public std::vector<DataStackElement> {
+  DataStack() = default;
+  DataStack(lldb::ValueObjectSP initial_value)
+      : std::vector<DataStackElement>({initial_value}) {}
+  void Push(DataStackElement el) { push_back(el); }
+  template <typename T> T Pop() {
+    T el = std::get<T>(back());
+    pop_back();
+    return el;
+  }
+  DataStackElement PopAny() {
+    DataStackElement el = back();
+    pop_back();
+    return el;
+  }
+};
+llvm::Error Interpret(std::vector<ControlStackElement> &control,
+                      DataStack &data, Selectors sel);
+} // namespace FormatterBytecode
+
+std::string toString(FormatterBytecode::OpCodes op);
+std::string toString(FormatterBytecode::Selectors sel);
+std::string toString(FormatterBytecode::Signatures sig);
+
+} // namespace lldb_private
diff --git a/lldb/source/DataFormatters/FormatterSection.cpp b/lldb/source/DataFormatters/FormatterSection.cpp
new file mode 100644
index 0000000000000..f70f41fdeb736
--- /dev/null
+++ b/lldb/source/DataFormatters/FormatterSection.cpp
@@ -0,0 +1,163 @@
+//===-- FormatterBytecode.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FormatterBytecode.h"
+#include "lldb/Core/Module.h"
+#include "lldb/DataFormatters/DataVisualization.h"
+#include "lldb/Utility/LLDBLog.h"
+
+using namespace lldb;
+
+namespace lldb_private {
+static void ForEachFormatterInModule(
+    Module &module, SectionType section_type,
+    std::function<void(llvm::DataExtractor, llvm::StringRef)> fn) {
+  auto *sections = module.GetSectionList();
+  if (!sections)
+    return;
+
+  auto section_sp = sections->FindSectionByType(section_type, true);
+  if (!section_sp)
+    return;
+
+  TypeCategoryImplSP category;
+  DataVisualization::Categories::GetCategory(ConstString("default"), category);
+
+  // The type summary record is serialized as follows.
+  //
+  // Each record contains, in order:
+  //   * Version number of the record format
+  //   * The remaining size of the record
+  //   * The size of the type identifier
+  //   * The type identifier, either a type name, or a regex
+  //   * The size of the entry
+  //   * The entry
+  //
+  // Integers are encoded using ULEB.
+  //
+  // Strings are encoded with first a length (ULEB), then the string contents,
+  // and lastly a null terminator. The length includes the null.
+
+  DataExtractor lldb_extractor;
+  auto section_size = section_sp->GetSectionData(lldb_extractor);
+  llvm::DataExtractor section = lldb_extractor.GetAsLLVM();
+  bool le = section.isLittleEndian();
+  uint8_t addr_size = section.getAddressSize();
+  llvm::DataExtractor::Cursor cursor(0);
+  while (cursor && cursor.tell() < section_size) {
+    uint64_t version = section.getULEB128(cursor);
+    uint64_t record_size = section.getULEB128(cursor);
+    if (version == 1) {
+      llvm::DataExtractor record(section.getData().drop_front(cursor.tell()),
+                                 le, addr_size);
+      llvm::DataExtractor::Cursor cursor(0);
+      uint64_t type_size = record.getULEB128(cursor);
+      llvm::StringRef type_name = record.getBytes(cursor, type_size);
+      llvm::Error error = cursor.takeError();
+      if (!error)
+        fn(llvm::DataExtractor(record.getData().drop_front(cursor.tell()), le,
+                               addr_size),
+           type_name);
+      else
+        LLDB_LOG_ERROR(GetLog(LLDBLog::DataFormatters), std::move(error),
+                       "{0}");
+    } else {
+      // Skip unsupported record.
+      LLDB_LOG(
+          GetLog(LLDBLog::DataFormatters),
+          "Skipping unsupported embedded type summary of version {0} in {1}.",
+          version, module.GetFileSpec());
+    }
+    section.skip(cursor, record_size);
+  }
+  if (!cursor)
+    LLDB_LOG_ERROR(GetLog(LLDBLog::DataFormatters), cursor.takeError(), "{0}");
+}
+
+void LoadTypeSummariesForModule(ModuleSP module_sp) {
+  ForEachFormatterInModule(
+      *module_sp, eSectionTypeLLDBTypeSummaries,
+      [&](llvm::DataExtractor extractor, llvm::StringRef type_name) {
+        TypeCategoryImplSP category;
+        DataVisualization::Categories::GetCategory(ConstString("default"),
+                                                   category);
+        // The type summary record is serialized as follows.
+        //
+        //   * The size of the summary string
+        //   * The summary string
+        //
+        // Integers are encoded using ULEB.
+        llvm::DataExtractor::Cursor cursor(0);
+        uint64_t summary_size = extractor.getULEB128(cursor);
+        llvm::StringRef summary_string =
+            extractor.getBytes(cursor, summary_size);
+        if (!cursor) {
+          LLDB_LOG_ERROR(GetLog(LLDBLog::DataFormatters), cursor.takeError(),
+                         "{0}");
+          return;
+        }
+        if (type_name.empty() || summary_string.empty()) {
+          LLDB_LOG(GetLog(LLDBLog::DataFormatters),
+                   "Missing string(s) in embedded type summary in {0}, "
+                   "type_name={1}, summary={2}",
+                   module_sp->GetFileSpec(), type_name, summary_string);
+          return;
+        }
+        TypeSummaryImpl::Flags flags;
+        auto summary_sp = std::make_shared<StringSummaryFormat>(
+            flags, summary_string.str().c_str());
+        FormatterMatchType match_type = eFormatterMatchExact;
+        if (type_name.front() == '^')
+          match_type = eFormatterMatchRegex;
+        category->AddTypeSummary(type_name, match_type, summary_sp);
+        LLDB_LOG(GetLog(LLDBLog::DataFormatters),
+                 "Loaded embedded type summary for '{0}' from {1}.", type_name,
+                 module_sp->GetFileSpec());
+      });
+}
+
+void LoadFormattersForModule(ModuleSP module_sp) {
+  ForEachFormatterInModule(
+      *module_sp, eSectionTypeLLDBFormatters,
+      [&](llvm::DataExtractor extractor, llvm::StringRef type_name) {
+        // * Function signature (1 byte)
+        // * Length of the program (ULEB128)
+        // * The program bytecode
+        TypeCategoryImplSP category;
+        DataVisualization::Categories::GetCategory(ConstString("default"),
+                                                   category);
+        llvm::DataExtractor::Cursor cursor(0);
+        uint64_t flags = extractor.getULEB128(cursor);
+        while (cursor && cursor.tell() < extractor.size()) {
+          uint8_t signature = extractor.getU8(cursor);
+          uint64_t size = extractor.getULEB128(cursor);
+          llvm::StringRef bytecode = extractor.getBytes(cursor, size);
+          if (!cursor) {
+            LLDB_LOG_ERROR(GetLog(LLDBLog::DataFormatters), cursor.takeError(),
+                           "{0}");
+            return;
+          }
+          if (signature == 0) {
+            auto summary_sp = std::make_shared<BytecodeSummaryFormat>(
+                TypeSummaryImpl::Flags(flags),
+                llvm::MemoryBuffer::getMemBufferCopy(bytecode));
+            FormatterMatchType match_type = eFormatterMatchExact;
+            if (type_name.front() == '^')
+              match_type = eFormatterMatchRegex;
+            category->AddTypeSummary(type_name, match_type, summary_sp);
+            LLDB_LOG(GetLog(LLDBLog::DataFormatters),
+                     "Loaded embedded type summary for '{0}' from {1}.",
+                     type_name, module_sp->GetFileSpec());
+          } else
+            LLDB_LOG(GetLog(LLDBLog::DataFormatters),
+                     "Unsupported formatter signature {0} for '{1}' in {2}",
+                     signature, type_name, module_sp->GetFileSpec());
+        }
+      });
+}
+} // namespace lldb_private
diff --git a/lldb/source/DataFormatters/TypeSummary.cpp b/lldb/source/DataFormatters/TypeSummary.cpp
index 3538c1b60c8e9..2c863b364538f 100644
--- a/lldb/source/DataFormatters/TypeSummary.cpp
+++ b/lldb/source/DataFormatters/TypeSummary.cpp
@@ -8,9 +8,7 @@
 
 #include "lldb/DataFormatters/TypeSummary.h"
 
-
-
-
+#include "FormatterBytecode.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-public.h"
 
@@ -58,6 +56,8 @@ std::string TypeSummaryImpl::GetSummaryKindName() {
     return "python";
   case Kind::eInternal:
     return "c++";
+  case Kind::eBytecode:
+    return "bytecode";
   }
 }
 
@@ -230,3 +230,74 @@ std::string ScriptSummaryFormat::GetDescription() {
 }
 
 std::string ScriptSummaryFormat::GetName() { return m_script_formatter_name; }
+
+BytecodeSummaryFormat::BytecodeSummaryFormat(
+    const TypeSummaryImpl::Flags &flags,
+    std::unique_ptr<llvm::MemoryBuffer> bytecode)
+    : TypeSummaryImpl(Kind::eBytecode, flags), m_bytecode(std::move(bytecode)) {
+}
+
+bool BytecodeSummaryFormat::FormatObject(ValueObject *valobj,
+                                         std::string &retval,
+                                         const TypeSummaryOptions &options) {
+  if (!valobj)
+    return false;
+
+  TargetSP target_sp(valobj->GetTargetSP());
+
+  if (!target_sp) {
+    retval.assign("error: no target");
+    return false;
+  }
+
+  std::vector<FormatterBytecode::ControlStackElement> control(
+      {m_bytecode->getBuffer()});
+  FormatterBytecode::DataStack data({valobj->GetSP()});
+  llvm::Error error = FormatterBytecode::Interpret(
+      control, data, FormatterBytecode::sel_summary);
+  if (error) {
+    retval = llvm::toString(std::move(error));
+    return false;
+  }
+  if (!data.size()) {
+    retval = "empty stack";
+    return false;
+  }
+  auto &top = data.back();
+  retval = "";
+  llvm::raw_string_ostream os(retval);
+  if (auto s = std::get_if<std::string>(&top))
+    os << *s;
+  else if (auto u = std::get_if<uint64_t>(&top))
+    os << *u;
+  else if (auto i = std::get_if<int64_t>(&top))
+    os << *i;
+  else if (auto valobj = std::get_if<ValueObjectSP>(&top)) {
+    if (!valobj->get())
+      os << "empty object";
+    else
+      os << valobj->get()->GetValueAsCString();
+  } else if (auto type = std::get_if<CompilerType>(&top)) {
+    os << type->TypeDescription();
+  } else if (auto sel = std::get_if<FormatterBytecode::Selectors>(&top)) {
+    os << toString(*sel);
+  }
+  return true;
+}
+
+std::string BytecodeSummaryFormat::GetDescription() {
+  StreamString sstr;
+  sstr.Printf("%s%s%s%s%s%s%s\n  ", Cascades() ? "" : " (not cascading)",
+              !DoesPrintChildren(nullptr) ? "" : " (show children)",
+              !DoesPrintValue(nullptr) ? " (hide value)" : "",
+              IsOneLiner() ? " (one-line printout)" : "",
+              SkipsPointers() ? " (skip pointers)" : "",
+              SkipsReferences() ? " (skip references)" : "",
+              HideNames(nullptr) ? " (hide member names)" : "");
+  // FIXME: sstr.PutCString(disassembly);
+  return std::string(sstr.GetString());
+}
+
+std::string BytecodeSummaryFormat::GetName() {
+  return "LLDB bytecode formatter";
+}
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index face38253efab..01e604e019f25 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -503,14 +503,7 @@ ValueObjectPrinter::PrintObjectDescriptionIfNeeded(bool value_printed,
 }
 
 bool DumpValueObjectOptions::PointerDepth::CanAllowExpansion() const {
-  switch (m_mode) {
-  case Mode::Always:
-  case Mode::Default:
-    return m_count > 0;
-  case Mode::Never:
-    return false;
-  }
-  return false;
+  return m_count > 0;
 }
 
 bool ValueObjectPrinter::ShouldPrintChildren(
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index d69eb60820403..0ccff41a55206 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -180,12 +180,9 @@ bool Socket::FindProtocolByScheme(const char *scheme,
   return false;
 }
 
-Socket::Socket(SocketProtocol protocol, bool should_close,
-               bool child_processes_inherit)
+Socket::Socket(SocketProtocol protocol, bool should_close)
     : IOObject(eFDTypeSocket), m_protocol(protocol),
-      m_socket(kInvalidSocketValue),
-      m_child_processes_inherit(child_processes_inherit),
-      m_should_close_fd(should_close) {}
+      m_socket(kInvalidSocketValue), m_should_close_fd(should_close) {}
 
 Socket::~Socket() { Close(); }
 
@@ -214,24 +211,21 @@ void Socket::Terminate() {
 }
 
 std::unique_ptr<Socket> Socket::Create(const SocketProtocol protocol,
-                                       bool child_processes_inherit,
                                        Status &error) {
   error.Clear();
 
+  const bool should_close = true;
   std::unique_ptr<Socket> socket_up;
   switch (protocol) {
   case ProtocolTcp:
-    socket_up =
-        std::make_unique<TCPSocket>(true, child_processes_inherit);
+    socket_up = std::make_unique<TCPSocket>(should_close);
     break;
   case ProtocolUdp:
-    socket_up =
-        std::make_unique<UDPSocket>(true, child_processes_inherit);
+    socket_up = std::make_unique<UDPSocket>(should_close);
     break;
   case ProtocolUnixDomain:
 #if LLDB_ENABLE_POSIX
-    socket_up =
-        std::make_unique<DomainSocket>(true, child_processes_inherit);
+    socket_up = std::make_unique<DomainSocket>(should_close);
 #else
     error = Status::FromErrorString(
         "Unix domain sockets are not supported on this platform.");
@@ -239,8 +233,7 @@ std::unique_ptr<Socket> Socket::Create(const SocketProtocol protocol,
     break;
   case ProtocolUnixAbstract:
 #ifdef __linux__
-    socket_up =
-        std::make_unique<AbstractSocket>(child_processes_inherit);
+    socket_up = std::make_unique<AbstractSocket>();
 #else
     error = Status::FromErrorString(
         "Abstract domain sockets are not supported on this platform.");
@@ -255,14 +248,12 @@ std::unique_ptr<Socket> Socket::Create(const SocketProtocol protocol,
 }
 
 llvm::Expected<std::unique_ptr<Socket>>
-Socket::TcpConnect(llvm::StringRef host_and_port,
-                   bool child_processes_inherit) {
+Socket::TcpConnect(llvm::StringRef host_and_port) {
   Log *log = GetLog(LLDBLog::Connection);
   LLDB_LOG(log, "host_and_port = {0}", host_and_port);
 
   Status error;
-  std::unique_ptr<Socket> connect_socket(
-      Create(ProtocolTcp, child_processes_inherit, error));
+  std::unique_ptr<Socket> connect_socket = Create(ProtocolTcp, error);
   if (error.Fail())
     return error.ToError();
 
@@ -274,13 +265,12 @@ Socket::TcpConnect(llvm::StringRef host_and_port,
 }
 
 llvm::Expected<std::unique_ptr<TCPSocket>>
-Socket::TcpListen(llvm::StringRef host_and_port, bool child_processes_inherit,
-                  int backlog) {
+Socket::TcpListen(llvm::StringRef host_and_port, int backlog) {
   Log *log = GetLog(LLDBLog::Connection);
   LLDB_LOG(log, "host_and_port = {0}", host_and_port);
 
   std::unique_ptr<TCPSocket> listen_socket(
-      new TCPSocket(true, child_processes_inherit));
+      new TCPSocket(/*should_close=*/true));
 
   Status error = listen_socket->Listen(host_and_port, backlog);
   if (error.Fail())
@@ -290,9 +280,8 @@ Socket::TcpListen(llvm::StringRef host_and_port, bool child_processes_inherit,
 }
 
 llvm::Expected<std::unique_ptr<UDPSocket>>
-Socket::UdpConnect(llvm::StringRef host_and_port,
-                   bool child_processes_inherit) {
-  return UDPSocket::Connect(host_and_port, child_processes_inherit);
+Socket::UdpConnect(llvm::StringRef host_and_port) {
+  return UDPSocket::CreateConnected(host_and_port);
 }
 
 llvm::Expected<Socket::HostAndPort> Socket::DecodeHostAndPort(llvm::StringRef host_and_port) {
@@ -445,13 +434,11 @@ int Socket::CloseSocket(NativeSocket sockfd) {
 }
 
 NativeSocket Socket::CreateSocket(const int domain, const int type,
-                                  const int protocol,
-                                  bool child_processes_inherit, Status &error) {
+                                  const int protocol, Status &error) {
   error.Clear();
   auto socket_type = type;
 #ifdef SOCK_CLOEXEC
-  if (!child_processes_inherit)
-    socket_type |= SOCK_CLOEXEC;
+  socket_type |= SOCK_CLOEXEC;
 #endif
   auto sock = ::socket(domain, socket_type, protocol);
   if (sock == kInvalidSocketValue)
@@ -460,7 +447,8 @@ NativeSocket Socket::CreateSocket(const int domain, const int type,
   return sock;
 }
 
-Status Socket::Accept(Socket *&socket) {
+Status Socket::Accept(const Timeout<std::micro> &timeout, Socket *&socket) {
+  socket = nullptr;
   MainLoop accept_loop;
   llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>> expected_handles =
       Accept(accept_loop,
@@ -470,12 +458,19 @@ Status Socket::Accept(Socket *&socket) {
              });
   if (!expected_handles)
     return Status::FromError(expected_handles.takeError());
-  return accept_loop.Run();
+  if (timeout) {
+    accept_loop.AddCallback(
+        [](MainLoopBase &loop) { loop.RequestTermination(); }, *timeout);
+  }
+  if (Status status = accept_loop.Run(); status.Fail())
+    return status;
+  if (socket)
+    return Status();
+  return Status(std::make_error_code(std::errc::timed_out));
 }
 
 NativeSocket Socket::AcceptSocket(NativeSocket sockfd, struct sockaddr *addr,
-                                  socklen_t *addrlen,
-                                  bool child_processes_inherit, Status &error) {
+                                  socklen_t *addrlen, Status &error) {
   error.Clear();
 #if defined(ANDROID_USE_ACCEPT_WORKAROUND)
   // Hack:
@@ -485,7 +480,7 @@ NativeSocket Socket::AcceptSocket(NativeSocket sockfd, struct sockaddr *addr,
   // available in older kernels. Using an older libc would fix this issue, but
   // introduce other ones, as the old libraries were quite buggy.
   int fd = syscall(__NR_accept, sockfd, addr, addrlen);
-  if (fd >= 0 && !child_processes_inherit) {
+  if (fd >= 0) {
     int flags = ::fcntl(fd, F_GETFD);
     if (flags != -1 && ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC) != -1)
       return fd;
@@ -494,10 +489,7 @@ NativeSocket Socket::AcceptSocket(NativeSocket sockfd, struct sockaddr *addr,
   }
   return fd;
 #elif defined(SOCK_CLOEXEC) && defined(HAVE_ACCEPT4)
-  int flags = 0;
-  if (!child_processes_inherit) {
-    flags |= SOCK_CLOEXEC;
-  }
+  int flags = SOCK_CLOEXEC;
   NativeSocket fd = llvm::sys::RetryAfterSignal(
       static_cast<NativeSocket>(-1), ::accept4, sockfd, addr, addrlen, flags);
 #else
diff --git a/lldb/source/Host/common/TCPSocket.cpp b/lldb/source/Host/common/TCPSocket.cpp
index 2d16b605af949..d3282ab58b818 100644
--- a/lldb/source/Host/common/TCPSocket.cpp
+++ b/lldb/source/Host/common/TCPSocket.cpp
@@ -38,18 +38,15 @@ using namespace lldb_private;
 
 static const int kType = SOCK_STREAM;
 
-TCPSocket::TCPSocket(bool should_close, bool child_processes_inherit)
-    : Socket(ProtocolTcp, should_close, child_processes_inherit) {}
+TCPSocket::TCPSocket(bool should_close) : Socket(ProtocolTcp, should_close) {}
 
 TCPSocket::TCPSocket(NativeSocket socket, const TCPSocket &listen_socket)
-    : Socket(ProtocolTcp, listen_socket.m_should_close_fd,
-             listen_socket.m_child_processes_inherit) {
+    : Socket(ProtocolTcp, listen_socket.m_should_close_fd) {
   m_socket = socket;
 }
 
-TCPSocket::TCPSocket(NativeSocket socket, bool should_close,
-                     bool child_processes_inherit)
-    : Socket(ProtocolTcp, should_close, child_processes_inherit) {
+TCPSocket::TCPSocket(NativeSocket socket, bool should_close)
+    : Socket(ProtocolTcp, should_close) {
   m_socket = socket;
 }
 
@@ -118,14 +115,21 @@ std::string TCPSocket::GetRemoteConnectionURI() const {
   return "";
 }
 
+std::vector<std::string> TCPSocket::GetListeningConnectionURI() const {
+  std::vector<std::string> URIs;
+  for (const auto &[fd, addr] : m_listen_sockets)
+    URIs.emplace_back(llvm::formatv("connection://[{0}]:{1}",
+                                    addr.GetIPAddress(), addr.GetPort()));
+  return URIs;
+}
+
 Status TCPSocket::CreateSocket(int domain) {
   Status error;
   if (IsValid())
     error = Close();
   if (error.Fail())
     return error;
-  m_socket = Socket::CreateSocket(domain, kType, IPPROTO_TCP,
-                                  m_child_processes_inherit, error);
+  m_socket = Socket::CreateSocket(domain, kType, IPPROTO_TCP, error);
   return error;
 }
 
@@ -183,8 +187,8 @@ Status TCPSocket::Listen(llvm::StringRef name, int backlog) {
   std::vector<SocketAddress> addresses = SocketAddress::GetAddressInfo(
       host_port->hostname.c_str(), nullptr, AF_UNSPEC, SOCK_STREAM, IPPROTO_TCP);
   for (SocketAddress &address : addresses) {
-    int fd = Socket::CreateSocket(address.GetFamily(), kType, IPPROTO_TCP,
-                                  m_child_processes_inherit, error);
+    int fd =
+        Socket::CreateSocket(address.GetFamily(), kType, IPPROTO_TCP, error);
     if (error.Fail() || fd < 0)
       continue;
 
@@ -212,11 +216,11 @@ Status TCPSocket::Listen(llvm::StringRef name, int backlog) {
     }
 
     if (host_port->port == 0) {
-      socklen_t sa_len = address.GetLength();
-      if (getsockname(fd, &address.sockaddr(), &sa_len) == 0)
-        host_port->port = address.GetPort();
+      socklen_t sa_len = listen_address.GetLength();
+      if (getsockname(fd, &listen_address.sockaddr(), &sa_len) == 0)
+        host_port->port = listen_address.GetPort();
     }
-    m_listen_sockets[fd] = address;
+    m_listen_sockets[fd] = listen_address;
   }
 
   if (m_listen_sockets.empty()) {
@@ -241,14 +245,13 @@ TCPSocket::Accept(MainLoopBase &loop,
   std::vector<MainLoopBase::ReadHandleUP> handles;
   for (auto socket : m_listen_sockets) {
     auto fd = socket.first;
-    auto io_sp =
-        std::make_shared<TCPSocket>(fd, false, this->m_child_processes_inherit);
+    auto io_sp = std::make_shared<TCPSocket>(fd, false);
     auto cb = [this, fd, sock_cb](MainLoopBase &loop) {
       lldb_private::SocketAddress AcceptAddr;
       socklen_t sa_len = AcceptAddr.GetMaxLength();
       Status error;
-      NativeSocket sock = AcceptSocket(fd, &AcceptAddr.sockaddr(), &sa_len,
-                                       m_child_processes_inherit, error);
+      NativeSocket sock =
+          AcceptSocket(fd, &AcceptAddr.sockaddr(), &sa_len, error);
       Log *log = GetLog(LLDBLog::Host);
       if (error.Fail()) {
         LLDB_LOG(log, "AcceptSocket({0}): {1}", fd, error);
diff --git a/lldb/source/Host/common/UDPSocket.cpp b/lldb/source/Host/common/UDPSocket.cpp
index 05d7b2e650602..f394bda346e1d 100644
--- a/lldb/source/Host/common/UDPSocket.cpp
+++ b/lldb/source/Host/common/UDPSocket.cpp
@@ -27,12 +27,12 @@ static const int kType = SOCK_DGRAM;
 
 static const char *g_not_supported_error = "Not supported";
 
-UDPSocket::UDPSocket(NativeSocket socket) : Socket(ProtocolUdp, true, true) {
+UDPSocket::UDPSocket(NativeSocket socket)
+    : Socket(ProtocolUdp, /*should_close=*/true) {
   m_socket = socket;
 }
 
-UDPSocket::UDPSocket(bool should_close, bool child_processes_inherit)
-    : Socket(ProtocolUdp, should_close, child_processes_inherit) {}
+UDPSocket::UDPSocket(bool should_close) : Socket(ProtocolUdp, should_close) {}
 
 size_t UDPSocket::Send(const void *buf, const size_t num_bytes) {
   return ::sendto(m_socket, static_cast<const char *>(buf), num_bytes, 0,
@@ -48,7 +48,7 @@ Status UDPSocket::Listen(llvm::StringRef name, int backlog) {
 }
 
 llvm::Expected<std::unique_ptr<UDPSocket>>
-UDPSocket::Connect(llvm::StringRef name, bool child_processes_inherit) {
+UDPSocket::CreateConnected(llvm::StringRef name) {
   std::unique_ptr<UDPSocket> socket;
 
   Log *log = GetLog(LLDBLog::Connection);
@@ -84,9 +84,9 @@ UDPSocket::Connect(llvm::StringRef name, bool child_processes_inherit) {
   for (struct addrinfo *service_info_ptr = service_info_list;
        service_info_ptr != nullptr;
        service_info_ptr = service_info_ptr->ai_next) {
-    auto send_fd = CreateSocket(
-        service_info_ptr->ai_family, service_info_ptr->ai_socktype,
-        service_info_ptr->ai_protocol, child_processes_inherit, error);
+    auto send_fd =
+        CreateSocket(service_info_ptr->ai_family, service_info_ptr->ai_socktype,
+                     service_info_ptr->ai_protocol, error);
     if (error.Success()) {
       socket.reset(new UDPSocket(send_fd));
       socket->m_sockaddr = service_info_ptr;
diff --git a/lldb/source/Host/linux/AbstractSocket.cpp b/lldb/source/Host/linux/AbstractSocket.cpp
index 833234028c865..8393a80e86e72 100644
--- a/lldb/source/Host/linux/AbstractSocket.cpp
+++ b/lldb/source/Host/linux/AbstractSocket.cpp
@@ -13,8 +13,7 @@
 using namespace lldb;
 using namespace lldb_private;
 
-AbstractSocket::AbstractSocket(bool child_processes_inherit)
-    : DomainSocket(ProtocolUnixAbstract, child_processes_inherit) {}
+AbstractSocket::AbstractSocket() : DomainSocket(ProtocolUnixAbstract) {}
 
 size_t AbstractSocket::GetNameOffset() const { return 1; }
 
diff --git a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
index 8a03e47ef3d90..6bdc33f892328 100644
--- a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
+++ b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
@@ -535,7 +535,7 @@ lldb::ConnectionStatus ConnectionFileDescriptor::AcceptSocket(
     Status *error_ptr) {
   Status error;
   std::unique_ptr<Socket> listening_socket =
-      Socket::Create(socket_protocol, /*child_processes_inherit=*/false, error);
+      Socket::Create(socket_protocol, error);
   Socket *accepted_socket;
 
   if (!error.Fail())
@@ -543,7 +543,7 @@ lldb::ConnectionStatus ConnectionFileDescriptor::AcceptSocket(
 
   if (!error.Fail()) {
     post_listen_callback(*listening_socket);
-    error = listening_socket->Accept(accepted_socket);
+    error = listening_socket->Accept(/*timeout=*/std::nullopt, accepted_socket);
   }
 
   if (!error.Fail()) {
@@ -562,8 +562,7 @@ ConnectionFileDescriptor::ConnectSocket(Socket::SocketProtocol socket_protocol,
                                         llvm::StringRef socket_name,
                                         Status *error_ptr) {
   Status error;
-  std::unique_ptr<Socket> socket =
-      Socket::Create(socket_protocol, /*child_processes_inherit=*/false, error);
+  std::unique_ptr<Socket> socket = Socket::Create(socket_protocol, error);
 
   if (!error.Fail())
     error = socket->Connect(socket_name);
@@ -644,8 +643,7 @@ ConnectionFileDescriptor::ConnectUDP(llvm::StringRef s,
                                      Status *error_ptr) {
   if (error_ptr)
     *error_ptr = Status();
-  llvm::Expected<std::unique_ptr<UDPSocket>> socket =
-      Socket::UdpConnect(s, /*child_processes_inherit=*/false);
+  llvm::Expected<std::unique_ptr<UDPSocket>> socket = Socket::UdpConnect(s);
   if (!socket) {
     if (error_ptr)
       *error_ptr = Status::FromError(socket.takeError());
@@ -690,7 +688,7 @@ ConnectionFileDescriptor::ConnectFD(llvm::StringRef s,
       // this. For now, we assume we must assume we don't own it.
 
       std::unique_ptr<TCPSocket> tcp_socket;
-      tcp_socket = std::make_unique<TCPSocket>(fd, false, false);
+      tcp_socket = std::make_unique<TCPSocket>(fd, /*should_close=*/false);
       // Try and get a socket option from this file descriptor to see if
       // this is a socket and set m_is_socket accordingly.
       int resuse;
diff --git a/lldb/source/Host/posix/DomainSocket.cpp b/lldb/source/Host/posix/DomainSocket.cpp
index 369123f223930..9a0b385d998bf 100644
--- a/lldb/source/Host/posix/DomainSocket.cpp
+++ b/lldb/source/Host/posix/DomainSocket.cpp
@@ -58,24 +58,20 @@ static bool SetSockAddr(llvm::StringRef name, const size_t name_offset,
   return true;
 }
 
-DomainSocket::DomainSocket(bool should_close, bool child_processes_inherit)
-    : DomainSocket(kInvalidSocketValue, should_close, child_processes_inherit) {
-}
+DomainSocket::DomainSocket(bool should_close)
+    : DomainSocket(kInvalidSocketValue, should_close) {}
 
-DomainSocket::DomainSocket(NativeSocket socket, bool should_close,
-                           bool child_processes_inherit)
-    : Socket(ProtocolUnixDomain, should_close, child_processes_inherit) {
+DomainSocket::DomainSocket(NativeSocket socket, bool should_close)
+    : Socket(ProtocolUnixDomain, should_close) {
   m_socket = socket;
 }
 
-DomainSocket::DomainSocket(SocketProtocol protocol,
-                           bool child_processes_inherit)
-    : Socket(protocol, true, child_processes_inherit) {}
+DomainSocket::DomainSocket(SocketProtocol protocol)
+    : Socket(protocol, /*should_close=*/true) {}
 
 DomainSocket::DomainSocket(NativeSocket socket,
                            const DomainSocket &listen_socket)
-    : Socket(ProtocolUnixDomain, listen_socket.m_should_close_fd,
-             listen_socket.m_child_processes_inherit) {
+    : Socket(ProtocolUnixDomain, listen_socket.m_should_close_fd) {
   m_socket = socket;
 }
 
@@ -86,7 +82,7 @@ Status DomainSocket::Connect(llvm::StringRef name) {
     return Status::FromErrorString("Failed to set socket address");
 
   Status error;
-  m_socket = CreateSocket(kDomain, kType, 0, m_child_processes_inherit, error);
+  m_socket = CreateSocket(kDomain, kType, 0, error);
   if (error.Fail())
     return error;
   if (llvm::sys::RetryAfterSignal(-1, ::connect, GetNativeSocket(),
@@ -105,7 +101,7 @@ Status DomainSocket::Listen(llvm::StringRef name, int backlog) {
   DeleteSocketFile(name);
 
   Status error;
-  m_socket = CreateSocket(kDomain, kType, 0, m_child_processes_inherit, error);
+  m_socket = CreateSocket(kDomain, kType, 0, error);
   if (error.Fail())
     return error;
   if (::bind(GetNativeSocket(), (struct sockaddr *)&saddr_un, saddr_un_len) ==
@@ -121,13 +117,11 @@ llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>> DomainSocket::Accept(
     MainLoopBase &loop,
     std::function<void(std::unique_ptr<Socket> socket)> sock_cb) {
   // TODO: Refactor MainLoop to avoid the shared_ptr requirement.
-  auto io_sp = std::make_shared<DomainSocket>(GetNativeSocket(), false,
-                                              m_child_processes_inherit);
+  auto io_sp = std::make_shared<DomainSocket>(GetNativeSocket(), false);
   auto cb = [this, sock_cb](MainLoopBase &loop) {
     Log *log = GetLog(LLDBLog::Host);
     Status error;
-    auto conn_fd = AcceptSocket(GetNativeSocket(), nullptr, nullptr,
-                                m_child_processes_inherit, error);
+    auto conn_fd = AcceptSocket(GetNativeSocket(), nullptr, nullptr, error);
     if (error.Fail()) {
       LLDB_LOG(log, "AcceptSocket({0}): {1}", GetNativeSocket(), error);
       return;
@@ -181,3 +175,17 @@ std::string DomainSocket::GetRemoteConnectionURI() const {
       "{0}://{1}",
       GetNameOffset() == 0 ? "unix-connect" : "unix-abstract-connect", name);
 }
+
+std::vector<std::string> DomainSocket::GetListeningConnectionURI() const {
+  if (m_socket == kInvalidSocketValue)
+    return {};
+
+  struct sockaddr_un addr;
+  bzero(&addr, sizeof(struct sockaddr_un));
+  addr.sun_family = AF_UNIX;
+  socklen_t addr_len = sizeof(struct sockaddr_un);
+  if (::getsockname(m_socket, (struct sockaddr *)&addr, &addr_len) != 0)
+    return {};
+
+  return {llvm::formatv("unix-connect://{0}", addr.sun_path)};
+}
diff --git a/lldb/source/Host/windows/MainLoopWindows.cpp b/lldb/source/Host/windows/MainLoopWindows.cpp
index 0a5a35e9db9dd..f3ab2a710cd01 100644
--- a/lldb/source/Host/windows/MainLoopWindows.cpp
+++ b/lldb/source/Host/windows/MainLoopWindows.cpp
@@ -28,7 +28,7 @@ static DWORD ToTimeout(std::optional<MainLoopWindows::TimePoint> point) {
     return WSA_INFINITE;
 
   nanoseconds dur = (std::max)(*point - steady_clock::now(), nanoseconds(0));
-  return duration_cast<milliseconds>(dur).count();
+  return ceil<milliseconds>(dur).count();
 }
 
 MainLoopWindows::MainLoopWindows() {
diff --git a/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp b/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
index 0e8c1f4b5f1d9..d633c469e603e 100644
--- a/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
+++ b/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
@@ -190,8 +190,7 @@ DumpValueObjectOptions OptionGroupValueObjectDisplay::GetAsDumpOptions(
     LanguageRuntimeDescriptionDisplayVerbosity lang_descr_verbosity,
     lldb::Format format, lldb::TypeSummaryImplSP summary_sp) {
   DumpValueObjectOptions options;
-  options.SetMaximumPointerDepth(
-      {DumpValueObjectOptions::PointerDepth::Mode::Always, ptr_depth});
+  options.SetMaximumPointerDepth(ptr_depth);
   if (use_objc)
     options.SetShowSummary(false);
   else
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
index 5669c31f1c402..d5c68a436e090 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
@@ -75,6 +75,17 @@ class ExternalASTSourceWrapper : public clang::ExternalSemaSource {
     return m_Source->FindExternalVisibleDeclsByName(DC, Name);
   }
 
+  bool LoadExternalSpecializations(const clang::Decl *D,
+                                   bool OnlyPartial) override {
+    return m_Source->LoadExternalSpecializations(D, OnlyPartial);
+  }
+
+  bool LoadExternalSpecializations(
+      const clang::Decl *D,
+      llvm::ArrayRef<clang::TemplateArgument> TemplateArgs) override {
+    return m_Source->LoadExternalSpecializations(D, TemplateArgs);
+  }
+
   void completeVisibleDeclsMap(const clang::DeclContext *DC) override {
     m_Source->completeVisibleDeclsMap(DC);
   }
@@ -305,6 +316,23 @@ class SemaSourceWithPriorities : public clang::ExternalSemaSource {
     return nullptr;
   }
 
+  bool LoadExternalSpecializations(const clang::Decl *D,
+                                   bool OnlyPartial) override {
+    bool newDeclFound = false;
+    for (size_t i = 0; i < Sources.size(); ++i)
+      newDeclFound |= Sources[i]->LoadExternalSpecializations(D, OnlyPartial);
+    return newDeclFound;
+  }
+
+  bool LoadExternalSpecializations(
+      const clang::Decl *D,
+      llvm::ArrayRef<clang::TemplateArgument> TemplateArgs) override {
+    bool newDeclFound = false;
+    for (size_t i = 0; i < Sources.size(); ++i)
+      newDeclFound |= Sources[i]->LoadExternalSpecializations(D, TemplateArgs);
+    return newDeclFound;
+  }
+
   void CompleteRedeclChain(const clang::Decl *D) override {
     for (size_t i = 0; i < Sources.size(); ++i)
       Sources[i]->CompleteRedeclChain(D);
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index 0083b49965697..c43871b08191d 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -3278,7 +3278,7 @@ bool AppleObjCRuntimeV2::NonPointerISACache::EvaluateNonPointerISA(
       }
 
       // If the index is still out of range then this isn't a pointer.
-      if (index > m_indexed_isa_cache.size())
+      if (index >= m_indexed_isa_cache.size())
         return false;
 
       LLDB_LOGF(log, "AOCRT::NPI Evaluate(ret_isa = 0x%" PRIx64 ")",
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index eac9ab4577d3e..b16cb114bec88 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -1696,6 +1696,8 @@ static SectionType GetSectionTypeFromName(llvm::StringRef Name) {
       .Case(".gnu_debugaltlink", eSectionTypeDWARFGNUDebugAltLink)
       .Case(".gosymtab", eSectionTypeGoSymtab)
       .Case(".text", eSectionTypeCode)
+      .Case(".lldbsummaries", lldb::eSectionTypeLLDBTypeSummaries)
+      .Case(".lldbformatters", lldb::eSectionTypeLLDBFormatters)
       .Case(".swift_ast", eSectionTypeSwiftModules)
       .Default(eSectionTypeOther);
 }
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 079fd905037d4..488c9bd1e54af 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1209,6 +1209,8 @@ AddressClass ObjectFileMachO::GetAddressClass(lldb::addr_t file_addr) {
         case eSectionTypeDWARFAppleObjC:
         case eSectionTypeDWARFGNUDebugAltLink:
         case eSectionTypeCTF:
+        case eSectionTypeLLDBTypeSummaries:
+        case eSectionTypeLLDBFormatters:
         case eSectionTypeSwiftModules:
           return AddressClass::eDebug;
 
@@ -1484,6 +1486,8 @@ static lldb::SectionType GetSectionType(uint32_t flags,
   static ConstString g_sect_name_data("__data");
   static ConstString g_sect_name_go_symtab("__gosymtab");
   static ConstString g_sect_name_ctf("__ctf");
+  static ConstString g_sect_name_lldb_summaries("__lldbsummaries");
+  static ConstString g_sect_name_lldb_formatters("__lldbformatters");
   static ConstString g_sect_name_swift_ast("__swift_ast");
 
   if (section_name == g_sect_name_dwarf_debug_abbrev)
@@ -1564,6 +1568,10 @@ static lldb::SectionType GetSectionType(uint32_t flags,
     return eSectionTypeGoSymtab;
   if (section_name == g_sect_name_ctf)
     return eSectionTypeCTF;
+  if (section_name == g_sect_name_lldb_summaries)
+    return lldb::eSectionTypeLLDBTypeSummaries;
+  if (section_name == g_sect_name_lldb_formatters)
+    return lldb::eSectionTypeLLDBFormatters;
   if (section_name == g_sect_name_swift_ast)
     return eSectionTypeSwiftModules;
   if (section_name == g_sect_name_objc_data ||
@@ -2235,11 +2243,11 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
   LLDB_LOG(log, "Parsing symbol table for {0}", file_name);
   Progress progress("Parsing symbol table", file_name);
 
-  llvm::MachO::symtab_command symtab_load_command = {0, 0, 0, 0, 0, 0};
   llvm::MachO::linkedit_data_command function_starts_load_command = {0, 0, 0, 0};
   llvm::MachO::linkedit_data_command exports_trie_load_command = {0, 0, 0, 0};
   llvm::MachO::dyld_info_command dyld_info = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
   llvm::MachO::dysymtab_command dysymtab = m_dysymtab;
+  SymtabCommandLargeOffsets symtab_load_command;
   // The data element of type bool indicates that this entry is thumb
   // code.
   typedef AddressDataArray<lldb::addr_t, bool, 100> FunctionStarts;
@@ -2276,12 +2284,20 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
     // Watch for the symbol table load command
     switch (lc.cmd) {
     case LC_SYMTAB:
+      // struct symtab_command {
+      //   uint32_t        cmd;            /* LC_SYMTAB */
+      //   uint32_t        cmdsize;        /* sizeof(struct symtab_command) */
+      //   uint32_t        symoff;         /* symbol table offset */
+      //   uint32_t        nsyms;          /* number of symbol table entries */
+      //   uint32_t        stroff;         /* string table offset */
+      //   uint32_t        strsize;        /* string table size in bytes */
+      // };
       symtab_load_command.cmd = lc.cmd;
       symtab_load_command.cmdsize = lc.cmdsize;
-      // Read in the rest of the symtab load command
-      if (m_data.GetU32(&offset, &symtab_load_command.symoff, 4) ==
-          nullptr) // fill in symoff, nsyms, stroff, strsize fields
-        return;
+      symtab_load_command.symoff = m_data.GetU32(&offset);
+      symtab_load_command.nsyms = m_data.GetU32(&offset);
+      symtab_load_command.stroff = m_data.GetU32(&offset);
+      symtab_load_command.strsize = m_data.GetU32(&offset);
       break;
 
     case LC_DYLD_INFO:
@@ -3767,6 +3783,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
 
       SymbolType type = eSymbolTypeInvalid;
       SectionSP symbol_section;
+      lldb::addr_t symbol_byte_size = 0;
       bool add_nlist = true;
       bool is_gsym = false;
       bool demangled_is_synthesized = false;
@@ -4352,6 +4369,47 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
 
       if (symbol_section) {
         const addr_t section_file_addr = symbol_section->GetFileAddress();
+        if (symbol_byte_size == 0 && function_starts_count > 0) {
+          addr_t symbol_lookup_file_addr = nlist.n_value;
+          // Do an exact address match for non-ARM addresses, else get the
+          // closest since the symbol might be a thumb symbol which has an
+          // address with bit zero set.
+          FunctionStarts::Entry *func_start_entry =
+              function_starts.FindEntry(symbol_lookup_file_addr, !is_arm);
+          if (is_arm && func_start_entry) {
+            // Verify that the function start address is the symbol address
+            // (ARM) or the symbol address + 1 (thumb).
+            if (func_start_entry->addr != symbol_lookup_file_addr &&
+                func_start_entry->addr != (symbol_lookup_file_addr + 1)) {
+              // Not the right entry, NULL it out...
+              func_start_entry = nullptr;
+            }
+          }
+          if (func_start_entry) {
+            func_start_entry->data = true;
+
+            addr_t symbol_file_addr = func_start_entry->addr;
+            if (is_arm)
+              symbol_file_addr &= THUMB_ADDRESS_BIT_MASK;
+
+            const FunctionStarts::Entry *next_func_start_entry =
+                function_starts.FindNextEntry(func_start_entry);
+            const addr_t section_end_file_addr =
+                section_file_addr + symbol_section->GetByteSize();
+            if (next_func_start_entry) {
+              addr_t next_symbol_file_addr = next_func_start_entry->addr;
+              // Be sure the clear the Thumb address bit when we calculate the
+              // size from the current and next address
+              if (is_arm)
+                next_symbol_file_addr &= THUMB_ADDRESS_BIT_MASK;
+              symbol_byte_size = std::min<lldb::addr_t>(
+                  next_symbol_file_addr - symbol_file_addr,
+                  section_end_file_addr - symbol_file_addr);
+            } else {
+              symbol_byte_size = section_end_file_addr - symbol_file_addr;
+            }
+          }
+        }
         symbol_value -= section_file_addr;
       }
 
@@ -4458,6 +4516,9 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
       if (nlist.n_desc & N_WEAK_REF)
         sym[sym_idx].SetIsWeak(true);
 
+      if (symbol_byte_size > 0)
+        sym[sym_idx].SetByteSize(symbol_byte_size);
+
       if (demangled_is_synthesized)
         sym[sym_idx].SetDemangledNameIsSynthesized(true);
 
@@ -4576,7 +4637,23 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
           Address symbol_addr;
           if (module_sp->ResolveFileAddress(symbol_file_addr, symbol_addr)) {
             SectionSP symbol_section(symbol_addr.GetSection());
+            uint32_t symbol_byte_size = 0;
             if (symbol_section) {
+              const addr_t section_file_addr = symbol_section->GetFileAddress();
+              const FunctionStarts::Entry *next_func_start_entry =
+                  function_starts.FindNextEntry(func_start_entry);
+              const addr_t section_end_file_addr =
+                  section_file_addr + symbol_section->GetByteSize();
+              if (next_func_start_entry) {
+                addr_t next_symbol_file_addr = next_func_start_entry->addr;
+                if (is_arm)
+                  next_symbol_file_addr &= THUMB_ADDRESS_BIT_MASK;
+                symbol_byte_size = std::min<lldb::addr_t>(
+                    next_symbol_file_addr - symbol_file_addr,
+                    section_end_file_addr - symbol_file_addr);
+              } else {
+                symbol_byte_size = section_end_file_addr - symbol_file_addr;
+              }
               sym[sym_idx].SetID(synthetic_sym_id++);
               // Don't set the name for any synthetic symbols, the Symbol
               // object will generate one if needed when the name is accessed
@@ -4588,6 +4665,8 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) {
               add_symbol_addr(symbol_addr.GetFileAddress());
               if (symbol_flags)
                 sym[sym_idx].SetFlags(symbol_flags);
+              if (symbol_byte_size)
+                sym[sym_idx].SetByteSize(symbol_byte_size);
               ++sym_idx;
             }
           }
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index be87112df7d89..27b2078b5a3fc 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -256,6 +256,19 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
     bool IsValid() { return all_image_infos.size() > 0; }
   };
 
+  // The LC_SYMTAB's symtab_command structure uses 32-bit file offsets
+  // for two fields, but ObjectFileMachO needs to calculate the offsets
+  // in virtual address layout from the start of the TEXT segment, and
+  // that span may be larger than 4GB.
+  struct SymtabCommandLargeOffsets {
+    uint32_t cmd = 0;          /* LC_SYMTAB */
+    uint32_t cmdsize = 0;      /* sizeof(struct symtab_command) */
+    lldb::offset_t symoff = 0; /* symbol table offset */
+    uint32_t nsyms = 0;        /* number of symbol table entries */
+    lldb::offset_t stroff = 0; /* string table offset */
+    uint32_t strsize = 0;      /* string table size in bytes */
+  };
+
   /// Get the list of binary images that were present in the process
   /// when the corefile was produced.
   /// \return
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 8d9c919bc9b10..bfdb8140e40af 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -1010,6 +1010,8 @@ SectionType ObjectFilePECOFF::GetSectionType(llvm::StringRef sect_name,
           // .eh_frame can be truncated to 8 chars.
           .Cases(".eh_frame", ".eh_fram", eSectionTypeEHFrame)
           .Case(".gosymtab", eSectionTypeGoSymtab)
+          .Case(".lldbsummaries", lldb::eSectionTypeLLDBTypeSummaries)
+          .Case(".lldbformatters", lldb::eSectionTypeLLDBFormatters)
           .Case("swiftast", eSectionTypeSwiftModules)
           .Default(eSectionTypeInvalid);
   if (section_type != eSectionTypeInvalid)
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index ed0c614cb3576..8baf3a8d60c37 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -29,6 +29,7 @@
 #include "lldb/Interpreter/OptionValueProperties.h"
 #include "lldb/Interpreter/OptionValueString.h"
 #include "lldb/Interpreter/Options.h"
+#include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Symbol/SymbolVendor.h"
@@ -1429,3 +1430,39 @@ PlatformDarwin::ResolveSDKPathFromDebugInfo(Module &module) {
 
   return path_or_err->str();
 }
+
+llvm::Expected<XcodeSDK>
+PlatformDarwin::GetSDKPathFromDebugInfo(CompileUnit &unit) {
+  ModuleSP module_sp = unit.CalculateSymbolContextModule();
+  if (!module_sp)
+    return llvm::createStringError("compile unit has no module");
+  SymbolFile *sym_file = module_sp->GetSymbolFile();
+  if (!sym_file)
+    return llvm::createStringError(
+        llvm::formatv("No symbol file available for module '{0}'",
+                      module_sp->GetFileSpec().GetFilename()));
+
+  return sym_file->ParseXcodeSDK(unit);
+}
+
+llvm::Expected<std::string>
+PlatformDarwin::ResolveSDKPathFromDebugInfo(CompileUnit &unit) {
+  auto sdk_or_err = GetSDKPathFromDebugInfo(unit);
+  if (!sdk_or_err)
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        llvm::formatv("Failed to parse SDK path from debug-info: {0}",
+                      llvm::toString(sdk_or_err.takeError())));
+
+  auto sdk = std::move(*sdk_or_err);
+
+  auto path_or_err = HostInfo::GetSDKRoot(HostInfo::SDKOptions{sdk});
+  if (!path_or_err)
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        llvm::formatv("Error while searching for SDK (XcodeSDK '{0}'): {1}",
+                      sdk.GetString(),
+                      llvm::toString(path_or_err.takeError())));
+
+  return path_or_err->str();
+}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
index 66a26d2f49677..e2d4cb6726d58 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
@@ -130,6 +130,11 @@ class PlatformDarwin : public PlatformPOSIX {
   llvm::Expected<std::string>
   ResolveSDKPathFromDebugInfo(Module &module) override;
 
+  llvm::Expected<XcodeSDK> GetSDKPathFromDebugInfo(CompileUnit &unit) override;
+
+  llvm::Expected<std::string>
+  ResolveSDKPathFromDebugInfo(CompileUnit &unit) override;
+
 protected:
   static const char *GetCompatibleArch(ArchSpec::Core core, size_t idx);
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 7eacd605362e7..f746ced1ff413 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -1216,17 +1216,12 @@ void GDBRemoteCommunication::DumpHistory(Stream &strm) { m_history.Dump(strm); }
 llvm::Error
 GDBRemoteCommunication::ConnectLocally(GDBRemoteCommunication &client,
                                        GDBRemoteCommunication &server) {
-  const bool child_processes_inherit = false;
   const int backlog = 5;
-  TCPSocket listen_socket(true, child_processes_inherit);
+  TCPSocket listen_socket(true);
   if (llvm::Error error =
           listen_socket.Listen("localhost:0", backlog).ToError())
     return error;
 
-  Socket *accept_socket = nullptr;
-  std::future<Status> accept_status = std::async(
-      std::launch::async, [&] { return listen_socket.Accept(accept_socket); });
-
   llvm::SmallString<32> remote_addr;
   llvm::raw_svector_ostream(remote_addr)
       << "connect://localhost:" << listen_socket.GetLocalPortNumber();
@@ -1238,10 +1233,15 @@ GDBRemoteCommunication::ConnectLocally(GDBRemoteCommunication &client,
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "Unable to connect: %s", status.AsCString());
 
-  client.SetConnection(std::move(conn_up));
-  if (llvm::Error error = accept_status.get().ToError())
-    return error;
+  // The connection was already established above, so a short timeout is
+  // sufficient.
+  Socket *accept_socket = nullptr;
+  if (Status accept_status =
+          listen_socket.Accept(std::chrono::seconds(1), accept_socket);
+      accept_status.Fail())
+    return accept_status.takeError();
 
+  client.SetConnection(std::move(conn_up));
   server.SetConnection(
       std::make_unique<ConnectionFileDescriptor>(accept_socket));
   return llvm::Error::success();
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
index fadc19676609b..bc886259d6fa5 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
@@ -299,9 +299,7 @@ size_t SymbolFileBreakpad::ParseBlocksRecursive(Function &func) {
   // "INLINE 0 ...", the current level is 0 and its parent block is the
   // function block at index 0.
   std::vector<Block *> blocks;
-  Block &block = func.GetBlock(false);
-  block.AddRange(Block::Range(0, func.GetAddressRange().GetByteSize()));
-  blocks.push_back(&block);
+  blocks.push_back(&func.GetBlock(false));
 
   size_t blocks_added = 0;
   addr_t func_base = func.GetAddressRange().GetBaseAddress().GetOffset();
@@ -315,7 +313,8 @@ size_t SymbolFileBreakpad::ParseBlocksRecursive(Function &func) {
       if (record->InlineNestLevel == 0 ||
           record->InlineNestLevel <= last_added_nest_level + 1) {
         last_added_nest_level = record->InlineNestLevel;
-        BlockSP block_sp = std::make_shared<Block>(It.GetBookmark().offset);
+        BlockSP block_sp = blocks[record->InlineNestLevel]->CreateChild(
+            It.GetBookmark().offset);
         FileSpec callsite_file;
         if (record->CallSiteFileNum < m_files->size())
           callsite_file = (*m_files)[record->CallSiteFileNum];
@@ -333,7 +332,6 @@ size_t SymbolFileBreakpad::ParseBlocksRecursive(Function &func) {
         }
         block_sp->FinalizeRanges();
 
-        blocks[record->InlineNestLevel]->AddChild(block_sp);
         if (record->InlineNestLevel + 1 >= blocks.size()) {
           blocks.resize(blocks.size() + 1);
         }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index d9bdeb560e122..37c1132c1c9f9 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -45,6 +45,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/DebugInfo/DWARF/DWARFTypePrinter.h"
 #include "llvm/Demangle/Demangle.h"
 
 #include <map>
@@ -826,11 +827,11 @@ std::string DWARFASTParserClang::GetDIEClassTemplateParams(DWARFDIE die) {
   if (llvm::StringRef(die.GetName()).contains("<"))
     return {};
 
-  TypeSystemClang::TemplateParameterInfos template_param_infos;
-  if (ParseTemplateParameterInfos(die, template_param_infos))
-    return m_ast.PrintTemplateParams(template_param_infos);
-
-  return {};
+  std::string name;
+  llvm::raw_string_ostream os(name);
+  llvm::DWARFTypePrinter<DWARFDIE> type_printer(os);
+  type_printer.appendAndTerminateTemplateParameters(die);
+  return name;
 }
 
 void DWARFASTParserClang::MapDeclDIEToDefDIE(
@@ -1618,9 +1619,9 @@ void DWARFASTParserClang::GetUniqueTypeNameAndDeclaration(
     case DW_TAG_structure_type:
     case DW_TAG_union_type: {
       if (const char *class_union_struct_name = parent_decl_ctx_die.GetName()) {
-        qualified_name.insert(
-            0, GetDIEClassTemplateParams(parent_decl_ctx_die));
         qualified_name.insert(0, "::");
+        qualified_name.insert(0,
+                              GetDIEClassTemplateParams(parent_decl_ctx_die));
         qualified_name.insert(0, class_union_struct_name);
       }
       parent_decl_ctx_die = parent_decl_ctx_die.GetParentDeclContextDIE();
@@ -1673,6 +1674,12 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   if (attrs.name) {
     GetUniqueTypeNameAndDeclaration(die, cu_language, unique_typename,
                                     unique_decl);
+    if (log) {
+      dwarf->GetObjectFile()->GetModule()->LogMessage(
+          log, "SymbolFileDWARF({0:p}) - {1:x16}: {2} has unique name: {3} ",
+          static_cast<void *>(this), die.GetID(), DW_TAG_value_to_name(tag),
+          unique_typename.AsCString());
+    }
     if (UniqueDWARFASTType *unique_ast_entry_type =
             dwarf->GetUniqueDWARFASTTypeMap().Find(
                 unique_typename, die, unique_decl, byte_size,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
index 235343d227122..d92de658a49e8 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
@@ -24,9 +24,11 @@ class DWARFUnit;
 class DWARFDebugInfoEntry;
 class DWARFDeclContext;
 class SymbolFileDWARF;
+class DWARFFormValue;
 
 class DWARFBaseDIE {
 public:
+  using DWARFFormValue = dwarf::DWARFFormValue;
   DWARFBaseDIE() = default;
 
   DWARFBaseDIE(DWARFUnit *cu, DWARFDebugInfoEntry *die)
@@ -117,6 +119,12 @@ class DWARFBaseDIE {
   enum class Recurse : bool { no, yes };
   DWARFAttributes GetAttributes(Recurse recurse = Recurse::yes) const;
 
+  // The following methods use LLVM naming convension in order to be are used by
+  // LLVM libraries.
+  dw_tag_t getTag() const { return Tag(); }
+
+  const char *getShortName() const { return GetName(); }
+
 protected:
   DWARFUnit *m_cu = nullptr;
   DWARFDebugInfoEntry *m_die = nullptr;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
index 4c9f1d8505f6e..362f4c44240c7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
@@ -572,6 +572,43 @@ bool DWARFDIE::GetDIENamesAndRanges(
     return false;
 }
 
+// The following methods use LLVM naming convension in order to be are used by
+// LLVM libraries.
 llvm::iterator_range<DWARFDIE::child_iterator> DWARFDIE::children() const {
   return llvm::make_range(child_iterator(*this), child_iterator());
 }
+
+DWARFDIE::child_iterator DWARFDIE::begin() const {
+  return child_iterator(*this);
+}
+
+DWARFDIE::child_iterator DWARFDIE::end() const { return child_iterator(); }
+
+std::optional<DWARFFormValue> DWARFDIE::find(const dw_attr_t attr) const {
+  DWARFFormValue form_value;
+  if (m_die->GetAttributeValue(m_cu, attr, form_value, nullptr, false))
+    return form_value;
+  return std::nullopt;
+}
+
+std::optional<uint64_t> DWARFDIE::getLanguage() const {
+  if (IsValid())
+    return m_cu->GetDWARFLanguageType();
+  return std::nullopt;
+}
+
+DWARFDIE DWARFDIE::resolveReferencedType(dw_attr_t attr) const {
+  return GetReferencedDIE(attr);
+}
+
+DWARFDIE DWARFDIE::resolveReferencedType(DWARFFormValue v) const {
+  if (IsValid())
+    return v.Reference();
+  return {};
+}
+
+DWARFDIE DWARFDIE::resolveTypeUnitReference() const {
+  if (DWARFDIE reference = GetReferencedDIE(DW_AT_signature))
+    return reference;
+  return *this;
+}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
index 077b78eb26d0c..5c1d381930c4e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
@@ -103,8 +103,25 @@ class DWARFDIE : public DWARFBaseDIE {
       std::optional<int> &call_line, std::optional<int> &call_column,
       DWARFExpressionList *frame_base) const;
 
+  // The following methods use LLVM naming convension in order to be are used by
+  // LLVM libraries.
+  std::optional<uint64_t> getLanguage() const;
+
+  DWARFDIE getParent() const { return GetParent(); }
+
+  DWARFDIE resolveReferencedType(dw_attr_t attr) const;
+
+  DWARFDIE resolveReferencedType(DWARFFormValue v) const;
+
+  DWARFDIE resolveTypeUnitReference() const;
+
+  std::optional<DWARFFormValue> find(const dw_attr_t attr) const;
+
   /// The range of all the children of this DIE.
   llvm::iterator_range<child_iterator> children() const;
+
+  child_iterator begin() const;
+  child_iterator end() const;
 };
 
 class DWARFDIE::child_iterator
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
index 404e50d57a925..fd3d45cef4c5e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
@@ -574,6 +574,31 @@ uint64_t DWARFFormValue::Reference(dw_offset_t base_offset) const {
   }
 }
 
+std::optional<uint64_t> DWARFFormValue::getAsUnsignedConstant() const {
+  if ((!IsDataForm(m_form)) || m_form == lldb_private::dwarf::DW_FORM_sdata)
+    return std::nullopt;
+  return m_value.uval;
+}
+
+std::optional<int64_t> DWARFFormValue::getAsSignedConstant() const {
+  if ((!IsDataForm(m_form)) ||
+      (m_form == lldb_private::dwarf::DW_FORM_udata &&
+       uint64_t(std::numeric_limits<int64_t>::max()) < m_value.uval))
+    return std::nullopt;
+  switch (m_form) {
+  case lldb_private::dwarf::DW_FORM_data4:
+    return int32_t(m_value.uval);
+  case lldb_private::dwarf::DW_FORM_data2:
+    return int16_t(m_value.uval);
+  case lldb_private::dwarf::DW_FORM_data1:
+    return int8_t(m_value.uval);
+  case lldb_private::dwarf::DW_FORM_sdata:
+  case lldb_private::dwarf::DW_FORM_data8:
+  default:
+    return m_value.sval;
+  }
+}
+
 const uint8_t *DWARFFormValue::BlockData() const { return m_value.data; }
 
 bool DWARFFormValue::IsBlockForm(const dw_form_t form) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
index 8ab9163e645fe..613948f2f3c9b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
@@ -76,6 +76,12 @@ class DWARFFormValue {
   void Clear();
   static bool FormIsSupported(dw_form_t form);
 
+  // The following methods use LLVM naming convension in order to be are used by
+  // LLVM libraries.
+  std::optional<uint64_t> getAsUnsignedConstant() const;
+  std::optional<int64_t> getAsSignedConstant() const;
+  const char *getAsCString() const { return AsCString(); }
+
 protected:
   // Compile unit where m_value was located.
   // It may be different from compile unit where m_value refers to.
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
index 6f2cb455ec00e..c71c2dd47344a 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
@@ -527,7 +527,7 @@ void DebugNamesDWARFIndex::GetTypesWithQuery(
   ConstString name = query.GetTypeBasename();
   std::vector<lldb_private::CompilerContext> query_context =
       query.GetContextRef();
-  if (query_context.size() <= 1)
+  if (query_context.size() <= 1 && !query.GetExactMatch())
     return GetTypes(name, callback);
 
   llvm::SmallVector<CompilerContext> parent_contexts =
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
index 1220e6115a2a9..5b325e30bef43 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
@@ -23,6 +23,7 @@
 #include "lldb/Utility/Timer.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/ThreadPool.h"
+#include <atomic>
 #include <optional>
 
 using namespace lldb_private;
@@ -81,44 +82,53 @@ void ManualDWARFIndex::Index() {
   Progress progress("Manually indexing DWARF", module_desc.GetData(),
                     total_progress);
 
-  std::vector<IndexSet> sets(units_to_index.size());
-
-  // Keep memory down by clearing DIEs for any units if indexing
-  // caused us to load the unit's DIEs.
-  std::vector<std::optional<DWARFUnit::ScopedExtractDIEs>> clear_cu_dies(
-      units_to_index.size());
-  auto parser_fn = [&](size_t cu_idx) {
-    IndexUnit(*units_to_index[cu_idx], dwp_dwarf, sets[cu_idx]);
-    progress.Increment();
-  };
-
-  auto extract_fn = [&](size_t cu_idx) {
-    clear_cu_dies[cu_idx] = units_to_index[cu_idx]->ExtractDIEsScoped();
-    progress.Increment();
-  };
-
   // Share one thread pool across operations to avoid the overhead of
   // recreating the threads.
   llvm::ThreadPoolTaskGroup task_group(Debugger::GetThreadPool());
+  const size_t num_threads = Debugger::GetThreadPool().getMaxConcurrency();
+
+  // Run a function for each compile unit in parallel using as many threads as
+  // are available. This is significantly faster than submiting a new task for
+  // each unit.
+  auto for_each_unit = [&](auto &&fn) {
+    std::atomic<size_t> next_cu_idx = 0;
+    auto wrapper = [&fn, &next_cu_idx, &units_to_index,
+                    &progress](size_t worker_id) {
+      size_t cu_idx;
+      while ((cu_idx = next_cu_idx.fetch_add(1, std::memory_order_relaxed)) <
+             units_to_index.size()) {
+        fn(worker_id, cu_idx, units_to_index[cu_idx]);
+        progress.Increment();
+      }
+    };
 
-  // Create a task runner that extracts dies for each DWARF unit in a
-  // separate thread.
-  // First figure out which units didn't have their DIEs already
-  // parsed and remember this.  If no DIEs were parsed prior to this index
-  // function call, we are going to want to clear the CU dies after we are
-  // done indexing to make sure we don't pull in all DWARF dies, but we need
-  // to wait until all units have been indexed in case a DIE in one
-  // unit refers to another and the indexes accesses those DIEs.
-  for (size_t i = 0; i < units_to_index.size(); ++i)
-    task_group.async(extract_fn, i);
-  task_group.wait();
+    for (size_t i = 0; i < num_threads; ++i)
+      task_group.async(wrapper, i);
 
-  // Now create a task runner that can index each DWARF unit in a
-  // separate thread so we can index quickly.
-  for (size_t i = 0; i < units_to_index.size(); ++i)
-    task_group.async(parser_fn, i);
-  task_group.wait();
+    task_group.wait();
+  };
 
+  // Extract dies for all DWARFs unit in parallel.  Figure out which units
+  // didn't have their DIEs already parsed and remember this.  If no DIEs were
+  // parsed prior to this index function call, we are going to want to clear the
+  // CU dies after we are done indexing to make sure we don't pull in all DWARF
+  // dies, but we need to wait until all units have been indexed in case a DIE
+  // in one unit refers to another and the indexes accesses those DIEs.
+  std::vector<std::optional<DWARFUnit::ScopedExtractDIEs>> clear_cu_dies(
+      units_to_index.size());
+  for_each_unit([&clear_cu_dies](size_t, size_t idx, DWARFUnit *unit) {
+    clear_cu_dies[idx] = unit->ExtractDIEsScoped();
+  });
+
+  // Now index all DWARF unit in parallel.
+  std::vector<IndexSet> sets(num_threads);
+  for_each_unit(
+      [this, dwp_dwarf, &sets](size_t worker_id, size_t, DWARFUnit *unit) {
+        IndexUnit(*unit, dwp_dwarf, sets[worker_id]);
+      });
+
+  // Merge partial indexes into a single index. Process each index in a set in
+  // parallel.
   auto finalize_fn = [this, &sets, &progress](NameToDIE(IndexSet::*index)) {
     NameToDIE &result = m_set.*index;
     for (auto &set : sets)
@@ -706,6 +716,11 @@ bool ManualDWARFIndex::Encode(DataEncoder &encoder) const {
   return true;
 }
 
+bool ManualDWARFIndex::IsPartial() const {
+  // If we have units or type units to skip, then this index is partial.
+  return !m_units_to_avoid.empty() || !m_type_sigs_to_avoid.empty();
+}
+
 std::string ManualDWARFIndex::GetCacheKey() {
   std::string key;
   llvm::raw_string_ostream strm(key);
@@ -713,9 +728,26 @@ std::string ManualDWARFIndex::GetCacheKey() {
   // module can have one object file as the main executable and might have
   // another object file in a separate symbol file, or we might have a .dwo file
   // that claims its module is the main executable.
+
+  // This class can be used to index all of the DWARF, or part of the DWARF
+  // when there is a .debug_names index where some compile or type units were
+  // built without .debug_names. So we need to know when we have a full manual
+  // DWARF index or a partial manual DWARF index and save them to different
+  // cache files. Before this fix we might end up debugging a binary with
+  // .debug_names where some of the compile or type units weren't indexed, and
+  // find an issue with the .debug_names tables (bugs or being incomplete), and
+  // then we disable loading the .debug_names by setting a setting in LLDB by
+  // running "settings set plugin.symbol-file.dwarf.ignore-file-indexes 0" in
+  // another LLDB instance. The problem arose when there was an index cache from
+  // a previous run where .debug_names was enabled and it had saved a cache file
+  // that only covered the missing compile and type units from the .debug_names,
+  // and with the setting that disables the loading of the cache files we would
+  // load partial cache index cache. So we need to pick a unique cache suffix
+  // name that indicates if the cache is partial or full to avoid this problem.
+  llvm::StringRef dwarf_index_suffix(IsPartial() ? "partial-" : "full-");
   ObjectFile *objfile = m_dwarf->GetObjectFile();
   strm << objfile->GetModule()->GetCacheKey() << "-dwarf-index-"
-      << llvm::format_hex(objfile->GetCacheHash(), 10);
+       << dwarf_index_suffix << llvm::format_hex(objfile->GetCacheHash(), 10);
   return key;
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
index d8c4a22ab21f7..6a52c88a99220 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
@@ -168,6 +168,16 @@ class ManualDWARFIndex : public DWARFIndex {
                             const lldb::LanguageType cu_language,
                             IndexSet &set);
 
+  /// Return true if this manual DWARF index is covering only part of the DWARF.
+  ///
+  /// An instance of this class will be used to index all of the DWARF, but also
+  /// when we have .debug_names we will use one to index any compile or type
+  /// units that are not covered by the .debug_names table.
+  ///
+  /// \return
+  ///   True if this index is a partial index, false otherwise.
+  bool IsPartial() const;
+
   /// The DWARF file which we are indexing.
   SymbolFileDWARF *m_dwarf;
   /// Which dwarf units should we skip while building the index.
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index c900f330b481b..b2128a34d8372 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -9,6 +9,7 @@
 #include "SymbolFileDWARF.h"
 
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
+#include "llvm/DebugInfo/DWARF/DWARFTypePrinter.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Format.h"
@@ -1305,123 +1306,76 @@ bool SymbolFileDWARF::ParseDebugMacros(CompileUnit &comp_unit) {
   return true;
 }
 
-size_t SymbolFileDWARF::ParseBlocksRecursive(
-    lldb_private::CompileUnit &comp_unit, Block *parent_block,
-    const DWARFDIE &orig_die, addr_t subprogram_low_pc, uint32_t depth) {
+size_t SymbolFileDWARF::ParseBlocksRecursive(CompileUnit &comp_unit,
+                                             Block *parent_block, DWARFDIE die,
+                                             addr_t subprogram_low_pc) {
   size_t blocks_added = 0;
-  DWARFDIE die = orig_die;
-  while (die) {
+  for (; die; die = die.GetSibling()) {
     dw_tag_t tag = die.Tag();
 
-    switch (tag) {
-    case DW_TAG_inlined_subroutine:
-    case DW_TAG_subprogram:
-    case DW_TAG_lexical_block: {
-      Block *block = nullptr;
-      if (tag == DW_TAG_subprogram) {
-        // Skip any DW_TAG_subprogram DIEs that are inside of a normal or
-        // inlined functions. These will be parsed on their own as separate
-        // entities.
-
-        if (depth > 0)
-          break;
-
-        block = parent_block;
-      } else {
-        BlockSP block_sp(new Block(die.GetID()));
-        parent_block->AddChild(block_sp);
-        block = block_sp.get();
-      }
-      DWARFRangeList ranges;
-      const char *name = nullptr;
-      const char *mangled_name = nullptr;
-
-      std::optional<int> decl_file;
-      std::optional<int> decl_line;
-      std::optional<int> decl_column;
-      std::optional<int> call_file;
-      std::optional<int> call_line;
-      std::optional<int> call_column;
-      if (die.GetDIENamesAndRanges(name, mangled_name, ranges, decl_file,
-                                   decl_line, decl_column, call_file, call_line,
-                                   call_column, nullptr)) {
-        if (tag == DW_TAG_subprogram) {
-          assert(subprogram_low_pc == LLDB_INVALID_ADDRESS);
-          subprogram_low_pc = ranges.GetMinRangeBase(0);
-        } else if (tag == DW_TAG_inlined_subroutine) {
-          // We get called here for inlined subroutines in two ways. The first
-          // time is when we are making the Function object for this inlined
-          // concrete instance.  Since we're creating a top level block at
-          // here, the subprogram_low_pc will be LLDB_INVALID_ADDRESS.  So we
-          // need to adjust the containing address. The second time is when we
-          // are parsing the blocks inside the function that contains the
-          // inlined concrete instance.  Since these will be blocks inside the
-          // containing "real" function the offset will be for that function.
-          if (subprogram_low_pc == LLDB_INVALID_ADDRESS) {
-            subprogram_low_pc = ranges.GetMinRangeBase(0);
-          }
-        }
+    if (tag != DW_TAG_inlined_subroutine && tag != DW_TAG_lexical_block)
+      continue;
 
-        const size_t num_ranges = ranges.GetSize();
-        for (size_t i = 0; i < num_ranges; ++i) {
-          const DWARFRangeList::Entry &range = ranges.GetEntryRef(i);
-          const addr_t range_base = range.GetRangeBase();
-          if (range_base >= subprogram_low_pc)
-            block->AddRange(Block::Range(range_base - subprogram_low_pc,
-                                         range.GetByteSize()));
-          else {
-            GetObjectFile()->GetModule()->ReportError(
-                "{0:x8}: adding range [{1:x16}-{2:x16}) which has a base "
-                "that is less than the function's low PC {3:x16}. Please file "
-                "a bug and attach the file at the "
-                "start of this error message",
-                block->GetID(), range_base, range.GetRangeEnd(),
-                subprogram_low_pc);
-          }
-        }
-        block->FinalizeRanges();
-
-        if (tag != DW_TAG_subprogram &&
-            (name != nullptr || mangled_name != nullptr)) {
-          std::unique_ptr<Declaration> decl_up;
-          if (decl_file || decl_line || decl_column)
-            decl_up = std::make_unique<Declaration>(
-                comp_unit.GetSupportFiles().GetFileSpecAtIndex(
-                    decl_file ? *decl_file : 0),
-                decl_line ? *decl_line : 0, decl_column ? *decl_column : 0);
-
-          std::unique_ptr<Declaration> call_up;
-          if (call_file || call_line || call_column)
-            call_up = std::make_unique<Declaration>(
-                comp_unit.GetSupportFiles().GetFileSpecAtIndex(
-                    call_file ? *call_file : 0),
-                call_line ? *call_line : 0, call_column ? *call_column : 0);
-
-          block->SetInlinedFunctionInfo(name, mangled_name, decl_up.get(),
-                                        call_up.get());
+    Block *block = parent_block->CreateChild(die.GetID()).get();
+    DWARFRangeList ranges;
+    const char *name = nullptr;
+    const char *mangled_name = nullptr;
+
+    std::optional<int> decl_file;
+    std::optional<int> decl_line;
+    std::optional<int> decl_column;
+    std::optional<int> call_file;
+    std::optional<int> call_line;
+    std::optional<int> call_column;
+    if (die.GetDIENamesAndRanges(name, mangled_name, ranges, decl_file,
+                                 decl_line, decl_column, call_file, call_line,
+                                 call_column, nullptr)) {
+      const size_t num_ranges = ranges.GetSize();
+      for (size_t i = 0; i < num_ranges; ++i) {
+        const DWARFRangeList::Entry &range = ranges.GetEntryRef(i);
+        const addr_t range_base = range.GetRangeBase();
+        if (range_base >= subprogram_low_pc)
+          block->AddRange(Block::Range(range_base - subprogram_low_pc,
+                                       range.GetByteSize()));
+        else {
+          GetObjectFile()->GetModule()->ReportError(
+              "{0:x8}: adding range [{1:x16}-{2:x16}) which has a base "
+              "that is less than the function's low PC {3:x16}. Please file "
+              "a bug and attach the file at the "
+              "start of this error message",
+              block->GetID(), range_base, range.GetRangeEnd(),
+              subprogram_low_pc);
         }
+      }
+      block->FinalizeRanges();
+
+      if (tag != DW_TAG_subprogram &&
+          (name != nullptr || mangled_name != nullptr)) {
+        std::unique_ptr<Declaration> decl_up;
+        if (decl_file || decl_line || decl_column)
+          decl_up = std::make_unique<Declaration>(
+              comp_unit.GetSupportFiles().GetFileSpecAtIndex(
+                  decl_file ? *decl_file : 0),
+              decl_line ? *decl_line : 0, decl_column ? *decl_column : 0);
+
+        std::unique_ptr<Declaration> call_up;
+        if (call_file || call_line || call_column)
+          call_up = std::make_unique<Declaration>(
+              comp_unit.GetSupportFiles().GetFileSpecAtIndex(
+                  call_file ? *call_file : 0),
+              call_line ? *call_line : 0, call_column ? *call_column : 0);
+
+        block->SetInlinedFunctionInfo(name, mangled_name, decl_up.get(),
+                                      call_up.get());
+      }
 
-        ++blocks_added;
+      ++blocks_added;
 
-        if (die.HasChildren()) {
-          blocks_added +=
-              ParseBlocksRecursive(comp_unit, block, die.GetFirstChild(),
-                                   subprogram_low_pc, depth + 1);
-        }
+      if (die.HasChildren()) {
+        blocks_added += ParseBlocksRecursive(
+            comp_unit, block, die.GetFirstChild(), subprogram_low_pc);
       }
-    } break;
-    default:
-      break;
     }
-
-    // Only parse siblings of the block if we are not at depth zero. A depth of
-    // zero indicates we are currently parsing the top level DW_TAG_subprogram
-    // DIE
-
-    if (depth == 0)
-      die.Clear();
-    else
-      die = die.GetSibling();
   }
   return blocks_added;
 }
@@ -2773,22 +2727,20 @@ void SymbolFileDWARF::FindTypes(const TypeQuery &query, TypeResults &results) {
       // Copy our match's context and update the basename we are looking for
       // so we can use this only to compare the context correctly.
       m_index->GetTypesWithQuery(query_simple, [&](DWARFDIE die) {
-        if (Type *matching_type = ResolveType(die, true, true)) {
-          ConstString name = matching_type->GetQualifiedName();
-          // We have found a type that still might not match due to template
-          // parameters. If we create a new TypeQuery that uses the new type's
-          // fully qualified name, we can find out if this type matches at all
-          // context levels. We can't use just the "match_simple" context
-          // because all template parameters were stripped off. The fully
-          // qualified name of the type will have the template parameters and
-          // will allow us to make sure it matches correctly.
-          TypeQuery die_query(name.GetStringRef(),
-                              TypeQueryOptions::e_exact_match);
-          if (!query.ContextMatches(die_query.GetContextRef()))
-            return true; // Keep iterating over index types, context mismatch.
-
-          results.InsertUnique(matching_type->shared_from_this());
+        // Check the language, but only if we have a language filter.
+        if (query.HasLanguage()) {
+          if (!query.LanguageMatches(GetLanguageFamily(*die.GetCU())))
+            return true; // Keep iterating over index types, language mismatch.
         }
+
+        std::string qualified_name;
+        llvm::raw_string_ostream os(qualified_name);
+        llvm::DWARFTypePrinter<DWARFDIE> type_printer(os);
+        type_printer.appendQualifiedName(die);
+        TypeQuery die_query(qualified_name, e_exact_match);
+        if (query.ContextMatches(die_query.GetContextRef()))
+          if (Type *matching_type = ResolveType(die, true, true))
+            results.InsertUnique(matching_type->shared_from_this());
         return !results.Done(query); // Keep iterating if we aren't done.
       });
       if (results.Done(query)) {
@@ -3242,8 +3194,16 @@ size_t SymbolFileDWARF::ParseBlocksRecursive(Function &func) {
   DWARFDIE function_die =
       dwarf_cu->GetNonSkeletonUnit().GetDIE(function_die_offset);
   if (function_die) {
-    ParseBlocksRecursive(*comp_unit, &func.GetBlock(false), function_die,
-                         LLDB_INVALID_ADDRESS, 0);
+    // We can't use the file address from the Function object as (in the OSO
+    // case) it will already be remapped to the main module.
+    DWARFRangeList ranges = function_die.GetDIE()->GetAttributeAddressRanges(
+        function_die.GetCU(),
+        /*check_hi_lo_pc=*/true);
+    lldb::addr_t function_file_addr =
+        ranges.GetMinRangeBase(LLDB_INVALID_ADDRESS);
+    if (function_file_addr != LLDB_INVALID_ADDRESS)
+      ParseBlocksRecursive(*comp_unit, &func.GetBlock(false),
+                           function_die.GetFirstChild(), function_file_addr);
   }
 
   return functions_added;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index ac25a0c48ee7d..76f4188fdf4af 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -395,8 +395,7 @@ class SymbolFileDWARF : public SymbolFileCommon {
   Function *ParseFunction(CompileUnit &comp_unit, const DWARFDIE &die);
 
   size_t ParseBlocksRecursive(CompileUnit &comp_unit, Block *parent_block,
-                              const DWARFDIE &die,
-                              lldb::addr_t subprogram_low_pc, uint32_t depth);
+                              DWARFDIE die, lldb::addr_t subprogram_low_pc);
 
   size_t ParseTypes(const SymbolContext &sc, const DWARFDIE &die,
                     bool parse_siblings, bool parse_children);
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index 0f77b2e28004e..27d51bbd1cb56 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -394,18 +394,12 @@ Block *SymbolFileNativePDB::CreateBlock(PdbCompilandSymId block_id) {
 
   switch (sym.kind()) {
   case S_GPROC32:
-  case S_LPROC32: {
+  case S_LPROC32:
     // This is a function.  It must be global.  Creating the Function entry
     // for it automatically creates a block for it.
-    FunctionSP func = GetOrCreateFunction(block_id, *comp_unit);
-    if (func) {
-      Block &block = func->GetBlock(false);
-      if (block.GetNumRanges() == 0)
-        block.AddRange(Block::Range(0, func->GetAddressRange().GetByteSize()));
-      return &block;
-    }
+    if (FunctionSP func = GetOrCreateFunction(block_id, *comp_unit))
+      return &func->GetBlock(false);
     break;
-  }
   case S_BLOCK32: {
     // This is a block.  Its parent is either a function or another block.  In
     // either case, its parent can be viewed as a block (e.g. a function
@@ -424,7 +418,7 @@ Block *SymbolFileNativePDB::CreateBlock(PdbCompilandSymId block_id) {
         m_index->MakeVirtualAddress(block.Segment, block.CodeOffset);
     lldb::addr_t func_base =
         func->GetAddressRange().GetBaseAddress().GetFileAddress();
-    BlockSP child_block = std::make_shared<Block>(opaque_block_uid);
+    BlockSP child_block = parent_block->CreateChild(opaque_block_uid);
     if (block_base >= func_base)
       child_block->AddRange(Block::Range(block_base - func_base, block.CodeSize));
     else {
@@ -437,7 +431,6 @@ Block *SymbolFileNativePDB::CreateBlock(PdbCompilandSymId block_id) {
           block_id.modi, block_id.offset, block_base,
           block_base + block.CodeSize, func_base);
     }
-    parent_block->AddChild(child_block);
     ast_builder->GetOrCreateBlockDecl(block_id);
     m_blocks.insert({opaque_block_uid, child_block});
     break;
@@ -450,8 +443,7 @@ Block *SymbolFileNativePDB::CreateBlock(PdbCompilandSymId block_id) {
     Block *parent_block = GetOrCreateBlock(inline_site->parent_id);
     if (!parent_block)
       return nullptr;
-    BlockSP child_block = std::make_shared<Block>(opaque_block_uid);
-    parent_block->AddChild(child_block);
+    BlockSP child_block = parent_block->CreateChild(opaque_block_uid);
     ast_builder->GetOrCreateInlinedFunctionDecl(block_id);
     // Copy ranges from InlineSite to Block.
     for (size_t i = 0; i < inline_site->ranges.GetSize(); ++i) {
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index c7b23aedfdcca..b7854c05d345a 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -402,46 +402,32 @@ static size_t ParseFunctionBlocksForPDBSymbol(
   assert(pdb_symbol && parent_block);
 
   size_t num_added = 0;
-  switch (pdb_symbol->getSymTag()) {
-  case PDB_SymType::Block:
-  case PDB_SymType::Function: {
-    Block *block = nullptr;
-    auto &raw_sym = pdb_symbol->getRawSymbol();
-    if (auto *pdb_func = llvm::dyn_cast<PDBSymbolFunc>(pdb_symbol)) {
-      if (pdb_func->hasNoInlineAttribute())
-        break;
-      if (is_top_parent)
-        block = parent_block;
-      else
-        break;
-    } else if (llvm::isa<PDBSymbolBlock>(pdb_symbol)) {
-      auto uid = pdb_symbol->getSymIndexId();
-      if (parent_block->FindBlockByID(uid))
-        break;
-      if (raw_sym.getVirtualAddress() < func_file_vm_addr)
-        break;
 
-      auto block_sp = std::make_shared<Block>(pdb_symbol->getSymIndexId());
-      parent_block->AddChild(block_sp);
-      block = block_sp.get();
-    } else
-      llvm_unreachable("Unexpected PDB symbol!");
+  if (!is_top_parent) {
+    // Ranges for the top block were parsed together with the function.
+    if (pdb_symbol->getSymTag() != PDB_SymType::Block)
+      return num_added;
 
+    auto &raw_sym = pdb_symbol->getRawSymbol();
+    assert(llvm::isa<PDBSymbolBlock>(pdb_symbol));
+    auto uid = pdb_symbol->getSymIndexId();
+    if (parent_block->FindBlockByID(uid))
+      return num_added;
+    if (raw_sym.getVirtualAddress() < func_file_vm_addr)
+      return num_added;
+
+    Block *block = parent_block->CreateChild(pdb_symbol->getSymIndexId()).get();
     block->AddRange(Block::Range(
         raw_sym.getVirtualAddress() - func_file_vm_addr, raw_sym.getLength()));
     block->FinalizeRanges();
-    ++num_added;
+  }
+  auto results_up = pdb_symbol->findAllChildren();
+  if (!results_up)
+    return num_added;
 
-    auto results_up = pdb_symbol->findAllChildren();
-    if (!results_up)
-      break;
-    while (auto symbol_up = results_up->getNext()) {
-      num_added += ParseFunctionBlocksForPDBSymbol(
-          func_file_vm_addr, symbol_up.get(), block, false);
-    }
-  } break;
-  default:
-    break;
+  while (auto symbol_up = results_up->getNext()) {
+    num_added += ParseFunctionBlocksForPDBSymbol(
+        func_file_vm_addr, symbol_up.get(), parent_block, false);
   }
   return num_added;
 }
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index f676364c54619..0519df329cff8 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -1403,26 +1403,6 @@ static TemplateParameterList *CreateTemplateParameterList(
   return template_param_list;
 }
 
-std::string TypeSystemClang::PrintTemplateParams(
-    const TemplateParameterInfos &template_param_infos) {
-  llvm::SmallVector<NamedDecl *, 8> ignore;
-  clang::TemplateParameterList *template_param_list =
-      CreateTemplateParameterList(getASTContext(), template_param_infos,
-                                  ignore);
-  llvm::SmallVector<clang::TemplateArgument, 2> args(
-      template_param_infos.GetArgs());
-  if (template_param_infos.hasParameterPack()) {
-    llvm::ArrayRef<TemplateArgument> pack_args =
-        template_param_infos.GetParameterPackArgs();
-    args.append(pack_args.begin(), pack_args.end());
-  }
-  std::string str;
-  llvm::raw_string_ostream os(str);
-  clang::printTemplateArgumentList(os, args, GetTypePrintingPolicy(),
-                                   template_param_list);
-  return str;
-}
-
 clang::FunctionTemplateDecl *TypeSystemClang::CreateFunctionTemplateDecl(
     clang::DeclContext *decl_ctx, OptionalClangModuleID owning_module,
     clang::FunctionDecl *func_decl,
@@ -6769,12 +6749,12 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
           llvm::StringRef field_name = field->getName();
           if (field_name.empty()) {
             CompilerType field_type = GetType(field->getType());
+            std::vector<uint32_t> save_indices = child_indexes;
             child_indexes.push_back(child_idx);
             if (field_type.GetIndexOfChildMemberWithName(
                     name, omit_empty_base_classes, child_indexes))
               return child_indexes.size();
-            child_indexes.pop_back();
-
+            child_indexes = std::move(save_indices);
           } else if (field_name == name) {
             // We have to add on the number of base classes to this index!
             child_indexes.push_back(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index e39aedec7e390..678eaed381fd4 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -1148,10 +1148,6 @@ class TypeSystemClang : public TypeSystem {
 
   bool SetDeclIsForcefullyCompleted(const clang::TagDecl *td);
 
-  /// Return the template parameters (including surrounding <>) in string form.
-  std::string
-  PrintTemplateParams(const TemplateParameterInfos &template_param_infos);
-
 private:
   /// Returns the PrintingPolicy used when generating the internal type names.
   /// These type names are mostly used for the formatter selection.
diff --git a/lldb/source/Symbol/Block.cpp b/lldb/source/Symbol/Block.cpp
index 8746a25e3fad5..5cc87240f552a 100644
--- a/lldb/source/Symbol/Block.cpp
+++ b/lldb/source/Symbol/Block.cpp
@@ -21,9 +21,11 @@
 using namespace lldb;
 using namespace lldb_private;
 
-Block::Block(lldb::user_id_t uid)
-    : UserID(uid), m_parent_scope(nullptr), m_children(), m_ranges(),
-      m_inlineInfoSP(), m_variable_list_sp(), m_parsed_block_info(false),
+Block::Block(Function &function, user_id_t function_uid)
+    : Block(function_uid, function) {}
+
+Block::Block(lldb::user_id_t uid, SymbolContextScope &parent_scope)
+    : UserID(uid), m_parent_scope(parent_scope), m_parsed_block_info(false),
       m_parsed_block_variables(false), m_parsed_child_blocks(false) {}
 
 Block::~Block() = default;
@@ -134,27 +136,20 @@ Block *Block::FindInnermostBlockByOffset(const lldb::addr_t offset) {
 }
 
 void Block::CalculateSymbolContext(SymbolContext *sc) {
-  if (m_parent_scope)
-    m_parent_scope->CalculateSymbolContext(sc);
+  m_parent_scope.CalculateSymbolContext(sc);
   sc->block = this;
 }
 
 lldb::ModuleSP Block::CalculateSymbolContextModule() {
-  if (m_parent_scope)
-    return m_parent_scope->CalculateSymbolContextModule();
-  return lldb::ModuleSP();
+  return m_parent_scope.CalculateSymbolContextModule();
 }
 
 CompileUnit *Block::CalculateSymbolContextCompileUnit() {
-  if (m_parent_scope)
-    return m_parent_scope->CalculateSymbolContextCompileUnit();
-  return nullptr;
+  return m_parent_scope.CalculateSymbolContextCompileUnit();
 }
 
 Function *Block::CalculateSymbolContextFunction() {
-  if (m_parent_scope)
-    return m_parent_scope->CalculateSymbolContextFunction();
-  return nullptr;
+  return m_parent_scope.CalculateSymbolContextFunction();
 }
 
 Block *Block::CalculateSymbolContextBlock() { return this; }
@@ -200,9 +195,7 @@ bool Block::Contains(const Range &range) const {
 }
 
 Block *Block::GetParent() const {
-  if (m_parent_scope)
-    return m_parent_scope->CalculateSymbolContextBlock();
-  return nullptr;
+  return m_parent_scope.CalculateSymbolContextBlock();
 }
 
 Block *Block::GetContainingInlinedBlock() {
@@ -354,8 +347,8 @@ void Block::AddRange(const Range &range) {
   if (parent_block && !parent_block->Contains(range)) {
     Log *log = GetLog(LLDBLog::Symbols);
     if (log) {
-      ModuleSP module_sp(m_parent_scope->CalculateSymbolContextModule());
-      Function *function = m_parent_scope->CalculateSymbolContextFunction();
+      ModuleSP module_sp(m_parent_scope.CalculateSymbolContextModule());
+      Function *function = m_parent_scope.CalculateSymbolContextFunction();
       const addr_t function_file_addr =
           function->GetAddressRange().GetBaseAddress().GetFileAddress();
       const addr_t block_start_addr = function_file_addr + range.GetRangeBase();
@@ -399,11 +392,9 @@ size_t Block::MemorySize() const {
   return mem_size;
 }
 
-void Block::AddChild(const BlockSP &child_block_sp) {
-  if (child_block_sp) {
-    child_block_sp->SetParentScope(this);
-    m_children.push_back(child_block_sp);
-  }
+BlockSP Block::CreateChild(user_id_t uid) {
+  m_children.push_back(std::shared_ptr<Block>(new Block(uid, *this)));
+  return m_children.back();
 }
 
 void Block::SetInlinedFunctionInfo(const char *name, const char *mangled,
@@ -520,13 +511,11 @@ void Block::SetDidParseVariables(bool b, bool set_children) {
 }
 
 Block *Block::GetSibling() const {
-  if (m_parent_scope) {
-    Block *parent_block = GetParent();
-    if (parent_block)
-      return parent_block->GetSiblingForChild(this);
-  }
+  if (Block *parent_block = GetParent())
+    return parent_block->GetSiblingForChild(this);
   return nullptr;
 }
+
 // A parent of child blocks can be asked to find a sibling block given
 // one of its child blocks
 Block *Block::GetSiblingForChild(const Block *child_block) const {
diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp
index 0186d63e72879..4f07b946353a4 100644
--- a/lldb/source/Symbol/Function.cpp
+++ b/lldb/source/Symbol/Function.cpp
@@ -278,11 +278,15 @@ Function::Function(CompileUnit *comp_unit, lldb::user_id_t func_uid,
                    lldb::user_id_t type_uid, const Mangled &mangled, Type *type,
                    AddressRanges ranges)
     : UserID(func_uid), m_comp_unit(comp_unit), m_type_uid(type_uid),
-      m_type(type), m_mangled(mangled), m_block(func_uid),
-      m_ranges(std::move(ranges)), m_range(CollapseRanges(m_ranges)),
-      m_frame_base(), m_flags(), m_prologue_byte_size(0) {
-  m_block.SetParentScope(this);
+      m_type(type), m_mangled(mangled), m_block(*this, func_uid),
+      m_range(CollapseRanges(ranges)), m_prologue_byte_size(0) {
   assert(comp_unit != nullptr);
+  lldb::addr_t base_file_addr = m_range.GetBaseAddress().GetFileAddress();
+  for (const AddressRange &range : ranges)
+    m_block.AddRange(
+        Block::Range(range.GetBaseAddress().GetFileAddress() - base_file_addr,
+                     range.GetByteSize()));
+  m_block.FinalizeRanges();
 }
 
 Function::~Function() = default;
@@ -427,13 +431,16 @@ void Function::GetDescription(Stream *s, lldb::DescriptionLevel level,
     llvm::interleaveComma(decl_context, *s, [&](auto &ctx) { ctx.Dump(*s); });
     *s << "}";
   }
-  *s << ", range" << (m_ranges.size() > 1 ? "s" : "") << " = ";
+  *s << ", range" << (m_block.GetNumRanges() > 1 ? "s" : "") << " = ";
   Address::DumpStyle fallback_style =
       level == eDescriptionLevelVerbose
           ? Address::DumpStyleModuleWithFileAddress
           : Address::DumpStyleFileAddress;
-  for (const AddressRange &range : m_ranges)
+  for (unsigned idx = 0; idx < m_block.GetNumRanges(); ++idx) {
+    AddressRange range;
+    m_block.GetRangeAtIndex(idx, range);
     range.Dump(s, target, Address::DumpStyleLoadAddress, fallback_style);
+  }
 }
 
 void Function::Dump(Stream *s, bool show_context) const {
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index 35317d209de1f..d3881f8ccf7fe 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -366,6 +366,8 @@ AddressClass ObjectFile::GetAddressClass(addr_t file_addr) {
           case eSectionTypeDWARFAppleObjC:
           case eSectionTypeDWARFGNUDebugAltLink:
           case eSectionTypeCTF:
+          case eSectionTypeLLDBFormatters:
+          case eSectionTypeLLDBTypeSummaries:
           case eSectionTypeSwiftModules:
             return AddressClass::eDebug;
           case eSectionTypeEHFrame:
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index dfbac5a572d00..2633c976c13bf 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -670,8 +670,8 @@ ValueObjectSP StackFrame::LegacyGetValueForVariableExpressionPath(
         }
       }
 
-      // If we have a non pointer type with a sythetic value then lets check if
-      // we have an sythetic dereference specified.
+      // If we have a non-pointer type with a synthetic value then lets check if
+      // we have a synthetic dereference specified.
       if (!valobj_sp->IsPointerType() && valobj_sp->HasSyntheticValue()) {
         Status deref_error;
         if (valobj_sp->GetCompilerType().IsReferenceType()) {
@@ -686,13 +686,13 @@ ValueObjectSP StackFrame::LegacyGetValueForVariableExpressionPath(
         valobj_sp = valobj_sp->Dereference(deref_error);
         if (!valobj_sp || deref_error.Fail()) {
           error = Status::FromErrorStringWithFormatv(
-              "Failed to dereference sythetic value: {0}", deref_error);
+              "Failed to dereference synthetic value: {0}", deref_error);
           return ValueObjectSP();
         }
         // Some synthetic plug-ins fail to set the error in Dereference
         if (!valobj_sp) {
           error =
-              Status::FromErrorString("Failed to dereference sythetic value");
+              Status::FromErrorString("Failed to dereference synthetic value");
           return ValueObjectSP();
         }
         expr_is_ptr = false;
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 4bac94f35d6cf..46216ba2d566d 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -24,6 +24,7 @@
 #include "lldb/Core/Section.h"
 #include "lldb/Core/SourceManager.h"
 #include "lldb/Core/StructuredDataImpl.h"
+#include "lldb/DataFormatters/FormatterSection.h"
 #include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/REPL.h"
@@ -63,8 +64,6 @@
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
-#include "lldb/ValueObject/ValueObject.h"
-#include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SetVector.h"
@@ -1818,6 +1817,8 @@ void Target::ModulesDidLoad(ModuleList &module_list) {
     for (size_t idx = 0; idx < num_images; ++idx) {
       ModuleSP module_sp(module_list.GetModuleAtIndex(idx));
       LoadScriptingResourceForModule(module_sp, this);
+      LoadTypeSummariesForModule(module_sp);
+      LoadFormattersForModule(module_sp);
     }
     m_breakpoint_list.UpdateBreakpoints(module_list, true, false);
     m_internal_breakpoint_list.UpdateBreakpoints(module_list, true, false);
diff --git a/lldb/source/ValueObject/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp
index 86172ad1b561f..2864af107b925 100644
--- a/lldb/source/ValueObject/ValueObject.cpp
+++ b/lldb/source/ValueObject/ValueObject.cpp
@@ -3194,17 +3194,17 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
       GetCompilerType().IsPointerType() || GetCompilerType().IsNullPtrType();
   bool is_float = GetCompilerType().IsFloat();
   bool is_integer = GetCompilerType().IsInteger();
+  ExecutionContext exe_ctx(GetExecutionContextRef());
 
-  if (!type.IsScalarType()) {
-    m_error = Status::FromErrorString("target type must be a scalar");
-    return GetSP();
-  }
+  if (!type.IsScalarType())
+    return ValueObjectConstResult::Create(
+        exe_ctx.GetBestExecutionContextScope(),
+        Status::FromErrorString("target type must be a scalar"));
 
-  if (!is_scalar && !is_enum && !is_pointer) {
-    m_error =
-        Status::FromErrorString("argument must be a scalar, enum, or pointer");
-    return GetSP();
-  }
+  if (!is_scalar && !is_enum && !is_pointer)
+    return ValueObjectConstResult::Create(
+        exe_ctx.GetBestExecutionContextScope(),
+        Status::FromErrorString("argument must be a scalar, enum, or pointer"));
 
   lldb::TargetSP target = GetTargetSP();
   uint64_t type_byte_size = 0;
@@ -3215,16 +3215,15 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
     val_byte_size = temp.value();
 
   if (is_pointer) {
-    if (!type.IsInteger() && !type.IsBoolean()) {
-      m_error =
-          Status::FromErrorString("target type must be an integer or boolean");
-      return GetSP();
-    }
-    if (!type.IsBoolean() && type_byte_size < val_byte_size) {
-      m_error = Status::FromErrorString(
-          "target type cannot be smaller than the pointer type");
-      return GetSP();
-    }
+    if (!type.IsInteger() && !type.IsBoolean())
+      return ValueObjectConstResult::Create(
+          exe_ctx.GetBestExecutionContextScope(),
+          Status::FromErrorString("target type must be an integer or boolean"));
+    if (!type.IsBoolean() && type_byte_size < val_byte_size)
+      return ValueObjectConstResult::Create(
+          exe_ctx.GetBestExecutionContextScope(),
+          Status::FromErrorString(
+              "target type cannot be smaller than the pointer type"));
   }
 
   if (type.IsBoolean()) {
@@ -3236,12 +3235,12 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
       if (float_value_or_err)
         return ValueObject::CreateValueObjectFromBool(
             target, !float_value_or_err->isZero(), "result");
-      else {
-        m_error = Status::FromErrorStringWithFormat(
-            "cannot get value as APFloat: %s",
-            llvm::toString(float_value_or_err.takeError()).c_str());
-        return GetSP();
-      }
+      else
+        return ValueObjectConstResult::Create(
+            exe_ctx.GetBestExecutionContextScope(),
+            Status::FromErrorStringWithFormat(
+                "cannot get value as APFloat: %s",
+                llvm::toString(float_value_or_err.takeError()).c_str()));
     }
   }
 
@@ -3255,13 +3254,12 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
             int_value_or_err->extOrTrunc(type_byte_size * CHAR_BIT);
         return ValueObject::CreateValueObjectFromAPInt(target, ext, type,
                                                        "result");
-      } else {
-        m_error = Status::FromErrorStringWithFormat(
-            "cannot get value as APSInt: %s",
-            llvm::toString(int_value_or_err.takeError()).c_str());
-        ;
-        return GetSP();
-      }
+      } else
+        return ValueObjectConstResult::Create(
+            exe_ctx.GetBestExecutionContextScope(),
+            Status::FromErrorStringWithFormat(
+                "cannot get value as APSInt: %s",
+                llvm::toString(int_value_or_err.takeError()).c_str()));
     } else if (is_scalar && is_float) {
       llvm::APSInt integer(type_byte_size * CHAR_BIT, !type.IsSigned());
       bool is_exact;
@@ -3273,12 +3271,12 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
 
         // Casting floating point values that are out of bounds of the target
         // type is undefined behaviour.
-        if (status & llvm::APFloatBase::opInvalidOp) {
-          m_error = Status::FromErrorStringWithFormat(
-              "invalid type cast detected: %s",
-              llvm::toString(float_value_or_err.takeError()).c_str());
-          return GetSP();
-        }
+        if (status & llvm::APFloatBase::opInvalidOp)
+          return ValueObjectConstResult::Create(
+              exe_ctx.GetBestExecutionContextScope(),
+              Status::FromErrorStringWithFormat(
+                  "invalid type cast detected: %s",
+                  llvm::toString(float_value_or_err.takeError()).c_str()));
         return ValueObject::CreateValueObjectFromAPInt(target, integer, type,
                                                        "result");
       }
@@ -3297,10 +3295,11 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
         return ValueObject::CreateValueObjectFromAPFloat(target, f, type,
                                                          "result");
       } else {
-        m_error = Status::FromErrorStringWithFormat(
-            "cannot get value as APSInt: %s",
-            llvm::toString(int_value_or_err.takeError()).c_str());
-        return GetSP();
+        return ValueObjectConstResult::Create(
+            exe_ctx.GetBestExecutionContextScope(),
+            Status::FromErrorStringWithFormat(
+                "cannot get value as APSInt: %s",
+                llvm::toString(int_value_or_err.takeError()).c_str()));
       }
     } else {
       if (is_integer) {
@@ -3312,10 +3311,11 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
           return ValueObject::CreateValueObjectFromAPFloat(target, f, type,
                                                            "result");
         } else {
-          m_error = Status::FromErrorStringWithFormat(
-              "cannot get value as APSInt: %s",
-              llvm::toString(int_value_or_err.takeError()).c_str());
-          return GetSP();
+          return ValueObjectConstResult::Create(
+              exe_ctx.GetBestExecutionContextScope(),
+              Status::FromErrorStringWithFormat(
+                  "cannot get value as APSInt: %s",
+                  llvm::toString(int_value_or_err.takeError()).c_str()));
         }
       }
       if (is_float) {
@@ -3327,34 +3327,37 @@ lldb::ValueObjectSP ValueObject::CastToBasicType(CompilerType type) {
           return ValueObject::CreateValueObjectFromAPFloat(target, f, type,
                                                            "result");
         } else {
-          m_error = Status::FromErrorStringWithFormat(
-              "cannot get value as APFloat: %s",
-              llvm::toString(float_value_or_err.takeError()).c_str());
-          return GetSP();
+          return ValueObjectConstResult::Create(
+              exe_ctx.GetBestExecutionContextScope(),
+              Status::FromErrorStringWithFormat(
+                  "cannot get value as APFloat: %s",
+                  llvm::toString(float_value_or_err.takeError()).c_str()));
         }
       }
     }
   }
 
-  m_error = Status::FromErrorString("Unable to perform requested cast");
-  return GetSP();
+  return ValueObjectConstResult::Create(
+      exe_ctx.GetBestExecutionContextScope(),
+      Status::FromErrorString("Unable to perform requested cast"));
 }
 
 lldb::ValueObjectSP ValueObject::CastToEnumType(CompilerType type) {
   bool is_enum = GetCompilerType().IsEnumerationType();
   bool is_integer = GetCompilerType().IsInteger();
   bool is_float = GetCompilerType().IsFloat();
+  ExecutionContext exe_ctx(GetExecutionContextRef());
 
-  if (!is_enum && !is_integer && !is_float) {
-    m_error = Status::FromErrorString(
-        "argument must be an integer, a float, or an enum");
-    return GetSP();
-  }
+  if (!is_enum && !is_integer && !is_float)
+    return ValueObjectConstResult::Create(
+        exe_ctx.GetBestExecutionContextScope(),
+        Status::FromErrorString(
+            "argument must be an integer, a float, or an enum"));
 
-  if (!type.IsEnumerationType()) {
-    m_error = Status::FromErrorString("target type must be an enum");
-    return GetSP();
-  }
+  if (!type.IsEnumerationType())
+    return ValueObjectConstResult::Create(
+        exe_ctx.GetBestExecutionContextScope(),
+        Status::FromErrorString("target type must be an enum"));
 
   lldb::TargetSP target = GetTargetSP();
   uint64_t byte_size = 0;
@@ -3371,18 +3374,18 @@ lldb::ValueObjectSP ValueObject::CastToEnumType(CompilerType type) {
 
       // Casting floating point values that are out of bounds of the target
       // type is undefined behaviour.
-      if (status & llvm::APFloatBase::opInvalidOp) {
-        m_error = Status::FromErrorStringWithFormat(
-            "invalid type cast detected: %s",
-            llvm::toString(value_or_err.takeError()).c_str());
-        return GetSP();
-      }
+      if (status & llvm::APFloatBase::opInvalidOp)
+        return ValueObjectConstResult::Create(
+            exe_ctx.GetBestExecutionContextScope(),
+            Status::FromErrorStringWithFormat(
+                "invalid type cast detected: %s",
+                llvm::toString(value_or_err.takeError()).c_str()));
       return ValueObject::CreateValueObjectFromAPInt(target, integer, type,
                                                      "result");
-    } else {
-      m_error = Status::FromErrorString("cannot get value as APFloat");
-      return GetSP();
-    }
+    } else
+      return ValueObjectConstResult::Create(
+          exe_ctx.GetBestExecutionContextScope(),
+          Status::FromErrorString("cannot get value as APFloat"));
   } else {
     // Get the value as APSInt and extend or truncate it to the requested size.
     auto value_or_err = GetValueAsAPSInt();
@@ -3390,15 +3393,16 @@ lldb::ValueObjectSP ValueObject::CastToEnumType(CompilerType type) {
       llvm::APSInt ext = value_or_err->extOrTrunc(byte_size * CHAR_BIT);
       return ValueObject::CreateValueObjectFromAPInt(target, ext, type,
                                                      "result");
-    } else {
-      m_error = Status::FromErrorStringWithFormat(
-          "cannot get value as APSInt: %s",
-          llvm::toString(value_or_err.takeError()).c_str());
-      return GetSP();
-    }
+    } else
+      return ValueObjectConstResult::Create(
+          exe_ctx.GetBestExecutionContextScope(),
+          Status::FromErrorStringWithFormat(
+              "cannot get value as APSInt: %s",
+              llvm::toString(value_or_err.takeError()).c_str()));
   }
-  m_error = Status::FromErrorString("Cannot perform requested cast");
-  return GetSP();
+  return ValueObjectConstResult::Create(
+      exe_ctx.GetBestExecutionContextScope(),
+      Status::FromErrorString("Cannot perform requested cast"));
 }
 
 ValueObject::EvaluationPoint::EvaluationPoint() : m_mod_id(), m_exe_ctx_ref() {}
diff --git a/lldb/test/API/commands/dwim-print/Makefile b/lldb/test/API/commands/dwim-print/Makefile
index 10495940055b6..99998b20bcb05 100644
--- a/lldb/test/API/commands/dwim-print/Makefile
+++ b/lldb/test/API/commands/dwim-print/Makefile
@@ -1,3 +1,3 @@
-C_SOURCES := main.c
+CXX_SOURCES := main.cpp
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py
index b40924a182e37..492d49f008a9e 100644
--- a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py
+++ b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py
@@ -16,7 +16,7 @@ def _run_cmd(self, cmd: str) -> str:
         self.ci.HandleCommand(cmd, result)
         return result.GetOutput().rstrip()
 
-    VAR_IDENT = re.compile(r"(?:\$\d+|\w+) = ")
+    VAR_IDENT = re.compile(r"(?:\$\d+|[\w.]+) = ")
 
     def _strip_result_var(self, string: str) -> str:
         """
@@ -121,30 +121,39 @@ def test_empty_expression(self):
     def test_nested_values(self):
         """Test dwim-print with nested values (structs, etc)."""
         self.build()
-        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
         self.runCmd("settings set auto-one-line-summaries false")
         self._expect_cmd(f"dwim-print s", "frame variable")
         self._expect_cmd(f"dwim-print (struct Structure)s", "expression")
 
     def test_summary_strings(self):
-        """Test dwim-print with nested values (structs, etc)."""
+        """Test dwim-print with type summaries."""
         self.build()
-        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
         self.runCmd("settings set auto-one-line-summaries false")
         self.runCmd("type summary add -e -s 'stub summary' Structure")
         self._expect_cmd(f"dwim-print s", "frame variable")
         self._expect_cmd(f"dwim-print (struct Structure)s", "expression")
+        self.runCmd("type summary delete Structure")
 
     def test_void_result(self):
         """Test dwim-print does not surface an error message for void expressions."""
         self.build()
-        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
         self.expect("dwim-print (void)15", matching=False, patterns=["(?i)error"])
 
     def test_preserves_persistent_variables(self):
         """Test dwim-print does not delete persistent variables."""
         self.build()
-        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
         self.expect("dwim-print int $i = 15")
         # Run the same expression twice and verify success. This ensures the
         # first command does not delete the persistent variable.
@@ -154,5 +163,25 @@ def test_preserves_persistent_variables(self):
     def test_missing_type(self):
         """The expected output of po opaque is its address (no error)"""
         self.build()
-        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
         self.expect("dwim-print -O -- opaque", substrs=["0x"])
+
+    def test_variable_expression_path(self):
+        """Test dwim-print supports certain variable expression paths."""
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
+        self.runCmd("settings set auto-one-line-summaries false")
+        self._expect_cmd("dwim-print w.s", "frame variable")
+        self._expect_cmd("dwim-print wp->s", "expression")
+
+    def test_direct_child_access(self):
+        """Test dwim-print supports accessing members/ivars without qualification."""
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "break inside", lldb.SBFileSpec("main.cpp")
+        )
+        self._expect_cmd("dwim-print number", "frame variable")
diff --git a/lldb/test/API/commands/dwim-print/main.c b/lldb/test/API/commands/dwim-print/main.c
deleted file mode 100644
index 6bfe645c7c6e0..0000000000000
--- a/lldb/test/API/commands/dwim-print/main.c
+++ /dev/null
@@ -1,14 +0,0 @@
-struct Structure {
-  int number;
-};
-
-struct Opaque;
-int puts(const char *s);
-
-int main(int argc, char **argv) {
-  struct Structure s;
-  s.number = 30;
-  struct Opaque *opaque = &s;
-  puts("break here");
-  return 0;
-}
diff --git a/lldb/test/API/commands/dwim-print/main.cpp b/lldb/test/API/commands/dwim-print/main.cpp
new file mode 100644
index 0000000000000..d1abb5a85dd45
--- /dev/null
+++ b/lldb/test/API/commands/dwim-print/main.cpp
@@ -0,0 +1,22 @@
+extern "C" int puts(const char *s);
+
+struct Structure {
+  int number = 30;
+  void f() { puts("break inside"); }
+};
+
+struct Wrapper {
+  Structure s;
+};
+
+struct Opaque;
+
+int main(int argc, char **argv) {
+  Structure s;
+  Wrapper w;
+  Wrapper *wp = &w;
+  Opaque *opaque = (Opaque *)(void *)&s;
+  puts("break here");
+  s.f();
+  return 0;
+}
diff --git a/lldb/test/API/commands/register/register/aarch64_sme_z_registers/za_dynamic_resize/Makefile b/lldb/test/API/commands/register/register/aarch64_sme_z_registers/za_dynamic_resize/Makefile
index 57d926b37d45c..bee03ac62a60f 100644
--- a/lldb/test/API/commands/register/register/aarch64_sme_z_registers/za_dynamic_resize/Makefile
+++ b/lldb/test/API/commands/register/register/aarch64_sme_z_registers/za_dynamic_resize/Makefile
@@ -1,5 +1,6 @@
 C_SOURCES := main.c
 
-CFLAGS_EXTRAS := -march=armv8-a+sve+sme -lpthread
+CFLAGS_EXTRAS := -march=armv8-a+sve+sme
+ENABLE_THREADS := YES
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/register/register/aarch64_sve_registers/rw_access_dynamic_resize/Makefile b/lldb/test/API/commands/register/register/aarch64_sve_registers/rw_access_dynamic_resize/Makefile
index efa5ca913f6e2..1c65300b73738 100644
--- a/lldb/test/API/commands/register/register/aarch64_sve_registers/rw_access_dynamic_resize/Makefile
+++ b/lldb/test/API/commands/register/register/aarch64_sve_registers/rw_access_dynamic_resize/Makefile
@@ -1,5 +1,6 @@
 C_SOURCES := main.c
 
-CFLAGS_EXTRAS := -march=armv8-a+sve -lpthread
+CFLAGS_EXTRAS := -march=armv8-a+sve
+ENABLE_THREADS := YES
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/target/anon-struct/Makefile b/lldb/test/API/commands/target/anon-struct/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/commands/target/anon-struct/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/target/anon-struct/TestTargetVarAnonStruct.py b/lldb/test/API/commands/target/anon-struct/TestTargetVarAnonStruct.py
new file mode 100644
index 0000000000000..869081dc6e5e4
--- /dev/null
+++ b/lldb/test/API/commands/target/anon-struct/TestTargetVarAnonStruct.py
@@ -0,0 +1,33 @@
+"""
+Test handling of Anonymous Structs, especially that they don't crash lldb.
+"""
+
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+import os
+import shutil
+import time
+
+
+class TestFrameVarAnonStruct(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        self.do_test()
+
+    def do_test(self):
+        target = self.createTestTarget()
+
+        # Verify that we don't crash in this case.
+        self.expect(
+            "target variable 'b.x'",
+            error=True,
+            substrs=["can't find global variable 'b.x'"],
+        )
diff --git a/lldb/test/API/commands/target/anon-struct/main.cpp b/lldb/test/API/commands/target/anon-struct/main.cpp
new file mode 100644
index 0000000000000..fbe26ea0ab875
--- /dev/null
+++ b/lldb/test/API/commands/target/anon-struct/main.cpp
@@ -0,0 +1,14 @@
+struct A {
+  struct {
+    int x = 1;
+  };
+} a;
+
+struct B {
+  // Anonymous struct inherits another struct.
+  struct : public A {
+    int z = 3;
+  };
+} b;
+
+int main(int argc, char **argv) { return 0; }
diff --git a/lldb/test/API/functionalities/data-formatter/bytecode-summary/Makefile b/lldb/test/API/functionalities/data-formatter/bytecode-summary/Makefile
new file mode 100644
index 0000000000000..c9319d6e6888a
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/bytecode-summary/Makefile
@@ -0,0 +1,2 @@
+C_SOURCES := main.c
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/bytecode-summary/TestBytecodeSummary.py b/lldb/test/API/functionalities/data-formatter/bytecode-summary/TestBytecodeSummary.py
new file mode 100644
index 0000000000000..2b27bd3cdcda7
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/bytecode-summary/TestBytecodeSummary.py
@@ -0,0 +1,17 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    @skipUnlessDarwin
+    def test(self):
+        self.build()
+        if self.TraceOn():
+            self.expect("log enable -v lldb formatters")
+        lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.cpp")
+        )
+        self.expect("v x", substrs=["(MyOptional<int>) x = None"])
+        self.expect("v y", substrs=["(MyOptional<int>) y = 42"])
diff --git a/lldb/test/API/functionalities/data-formatter/bytecode-summary/main.cpp b/lldb/test/API/functionalities/data-formatter/bytecode-summary/main.cpp
new file mode 100644
index 0000000000000..35406acd6d0c4
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/bytecode-summary/main.cpp
@@ -0,0 +1,43 @@
+// A bare-bones llvm::Optional reimplementation.
+
+template <typename T> struct MyOptionalStorage {
+  MyOptionalStorage(T val) : value(val), hasVal(true) {}
+  MyOptionalStorage() {}
+  T value;
+  bool hasVal = false;
+};
+
+template <typename T> struct MyOptional {
+  MyOptionalStorage<T> Storage;
+  MyOptional(T val) : Storage(val) {}
+  MyOptional() {}
+  T &operator*() { return Storage.value; }
+};
+
+void stop() {}
+
+int main(int argc, char **argv) {
+  MyOptional<int> x, y = 42;
+  stop(); // break here
+  return *y;
+}
+
+// Produced from the assembler in
+// Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/formatter.py
+__attribute__((used, section("__DATA_CONST,__lldbformatters"))) unsigned char
+    _MyOptional_type_summary[] =
+        "\x01"             // version
+        "\xa4"             // record size
+        "\x01"             // record size
+        "\x10"             // type name size
+        "^MyOptional<.+>$" // type name
+        "\x00"             // flags
+        "\x00"             // sig_summary
+        "\x8d"             // program size
+        "\x01"             // program size
+        "\x1\x22\x7Storage#\x12\x60\x1,C\x10\x1\x5\x11\x2\x1\x22\x6hasVal#"
+        "\x12\x60\x1,\x10\x1e\x2\x22\x1b<could not read MyOptional>\x10G#!\x60 "
+        "\x0P\x10\x6\x22\x4None\x10\x36\x1#\x15\x60 "
+        "\x0#\x16\x60\x5\x22\x5value#\x12\x60\x5#\x17\x60\x1,"
+        "\x10\x6\x22\x4None\x10\x11\x1#\x0\x60\x1#R\x60\x10\x3# "
+        "\x60\x10\x1\x2\x12\x12\x12\x12"; // summary function
diff --git a/lldb/test/API/functionalities/data-formatter/embedded-summary/Makefile b/lldb/test/API/functionalities/data-formatter/embedded-summary/Makefile
new file mode 100644
index 0000000000000..c9319d6e6888a
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/embedded-summary/Makefile
@@ -0,0 +1,2 @@
+C_SOURCES := main.c
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/embedded-summary/TestEmbeddedTypeSummary.py b/lldb/test/API/functionalities/data-formatter/embedded-summary/TestEmbeddedTypeSummary.py
new file mode 100644
index 0000000000000..b8ce7d9f76eb9
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/embedded-summary/TestEmbeddedTypeSummary.py
@@ -0,0 +1,12 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    @skipUnlessDarwin
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        self.expect("v player", substrs=['"Dirk" (41)'])
diff --git a/lldb/test/API/functionalities/data-formatter/embedded-summary/main.c b/lldb/test/API/functionalities/data-formatter/embedded-summary/main.c
new file mode 100644
index 0000000000000..9ddd64246f726
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/embedded-summary/main.c
@@ -0,0 +1,22 @@
+#include <stdio.h>
+
+struct Player {
+  char *name;
+  int number;
+};
+
+__attribute__((used, section("__DATA_CONST,__lldbsummaries"))) unsigned char
+    _Player_type_summary[] = "\x01"     // version
+                             "\x25"     // record size
+                             "\x07"     // type name size
+                             "Player\0" // type name
+                             "\x1c"     // summary string size
+                             "${var.name} (${var.number})"; // summary string
+
+int main() {
+  struct Player player;
+  player.name = "Dirk";
+  player.number = 41;
+  puts("break here");
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/Makefile b/lldb/test/API/functionalities/process_save_core_minidump/Makefile
index 2d177981fdde1..e9a26189f5dad 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/Makefile
+++ b/lldb/test/API/functionalities/process_save_core_minidump/Makefile
@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS := -lpthread
+ENABLE_THREADS := YES
 
 include Makefile.rules
 
diff --git a/lldb/test/API/functionalities/vtable/TestVTableValue.py b/lldb/test/API/functionalities/vtable/TestVTableValue.py
index bfc910614afa9..f0076ea28f759 100644
--- a/lldb/test/API/functionalities/vtable/TestVTableValue.py
+++ b/lldb/test/API/functionalities/vtable/TestVTableValue.py
@@ -2,7 +2,6 @@
 Make sure the getting a variable path works and doesn't crash.
 """
 
-
 import lldb
 import lldbsuite.test.lldbutil as lldbutil
 from lldbsuite.test.decorators import *
@@ -142,7 +141,12 @@ def test_overwrite_vtable(self):
             "\x01\x01\x01\x01\x01\x01\x01\x01" if is_64bit else "\x01\x01\x01\x01"
         )
         error = lldb.SBError()
-        process.WriteMemory(vtable_addr, data, error)
+        bytes_written = process.WriteMemory(vtable_addr, data, error)
+
+        self.assertSuccess(error)
+        self.assertGreater(
+            bytes_written, 0, "Failed to overwrite first entry in vtable"
+        )
 
         scribbled_child = vtable.GetChildAtIndex(0)
         self.assertEqual(
diff --git a/lldb/test/API/python_api/find_in_memory/TestFindInMemory.py b/lldb/test/API/python_api/find_in_memory/TestFindInMemory.py
index 04e807c5c6201..1ef37d2ec9898 100644
--- a/lldb/test/API/python_api/find_in_memory/TestFindInMemory.py
+++ b/lldb/test/API/python_api/find_in_memory/TestFindInMemory.py
@@ -152,3 +152,16 @@ def test_find_in_memory_unaligned(self):
         )
         self.assertSuccess(error)
         self.assertEqual(addr, lldb.LLDB_INVALID_ADDRESS)
+
+    def test_memory_info_list_iterable(self):
+        """Make sure the SBMemoryRegionInfoList is iterable"""
+        self.assertTrue(self.process, PROCESS_IS_VALID)
+        self.assertState(self.process.GetState(), lldb.eStateStopped, PROCESS_STOPPED)
+
+        info_list = self.process.GetMemoryRegions()
+        self.assertTrue(info_list.GetSize() > 0)
+        try:
+            for info in info_list:
+                pass
+        except Exception:
+            self.fail("SBMemoryRegionInfoList is not iterable")
diff --git a/lldb/test/API/python_api/value/TestValueAPI.py b/lldb/test/API/python_api/value/TestValueAPI.py
index 512100912d6fe..9eaf2c994d846 100644
--- a/lldb/test/API/python_api/value/TestValueAPI.py
+++ b/lldb/test/API/python_api/value/TestValueAPI.py
@@ -140,10 +140,8 @@ def test(self):
         val_i = target.EvaluateExpression("i")
         val_s = target.EvaluateExpression("s")
         val_a = target.EvaluateExpression("a")
-        self.assertTrue(
-            val_s.GetChildMemberWithName("a").GetAddress().IsValid(), VALID_VARIABLE
-        )
-        self.assertTrue(val_s.GetChildMemberWithName("a").AddressOf(), VALID_VARIABLE)
+        self.assertTrue(val_s.member["a"].GetAddress().IsValid(), VALID_VARIABLE)
+        self.assertTrue(val_s.member["a"].AddressOf(), VALID_VARIABLE)
         self.assertTrue(val_a.Cast(val_i.GetType()).AddressOf(), VALID_VARIABLE)
 
         # Test some other cases of the Cast API.  We allow casts from one struct type
@@ -210,7 +208,7 @@ def test(self):
         weird_cast = f_var.Cast(val_s.GetType())
         self.assertSuccess(weird_cast.GetError(), "Can cast from a larger to a smaller")
         self.assertEqual(
-            weird_cast.GetChildMemberWithName("a").GetValueAsSigned(0),
+            weird_cast.member["a"].GetValueAsSigned(0),
             33,
             "Got the right value",
         )
diff --git a/lldb/test/API/tools/lldb-dap/threads/Makefile b/lldb/test/API/tools/lldb-dap/threads/Makefile
index 121868fa8ec33..aa6b054685d61 100644
--- a/lldb/test/API/tools/lldb-dap/threads/Makefile
+++ b/lldb/test/API/tools/lldb-dap/threads/Makefile
@@ -1,4 +1,5 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS := -lpthread
+
+ENABLE_THREADS := YES
 
 include Makefile.rules
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/MyOptional.cpp b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/MyOptional.cpp
new file mode 100644
index 0000000000000..abba34439d88f
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/MyOptional.cpp
@@ -0,0 +1,23 @@
+// A bare-bones llvm::Optional reimplementation.
+
+template <typename T> struct MyOptionalStorage {
+  MyOptionalStorage(T val) : value(val), hasVal(true) {}
+  MyOptionalStorage() {}
+  T value;
+  bool hasVal = false;
+};
+
+template <typename T> struct MyOptional {
+  MyOptionalStorage<T> Storage;
+  MyOptional(T val) : Storage(val) {}
+  MyOptional() {}
+  T &operator*() { return Storage.value; }
+};
+
+void stop() {}
+
+int main(int argc, char **argv) {
+  MyOptional<int> x, y = 42;
+  stop(); // break here
+  return *y;
+}
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/formatter.py b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/formatter.py
new file mode 100644
index 0000000000000..cc8778bd51c74
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/formatter.py
@@ -0,0 +1,167 @@
+"""
+This is the llvm::Optional data formatter from llvm/utils/lldbDataFormatters.py
+with the implementation replaced by bytecode.
+"""
+
+from __future__ import annotations
+from formatter_bytecode import *
+import lldb
+
+
+def __lldb_init_module(debugger, internal_dict):
+    debugger.HandleCommand(
+        "type synthetic add -w llvm "
+        f"-l {__name__}.MyOptionalSynthProvider "
+        '-x "^MyOptional<.+>$"'
+    )
+    debugger.HandleCommand(
+        "type summary add -w llvm "
+        f"-e -F {__name__}.MyOptionalSummaryProvider "
+        '-x "^MyOptional<.+>$"'
+    )
+
+
+def stringify(bytecode: bytearray) -> str:
+    s = ""
+    in_hex = False
+    for b in bytecode:
+        if (b < 32 or b > 127 or chr(b) in ['"', "`", "'"]) or (
+            in_hex
+            and chr(b).lower()
+            in [
+                "a",
+                "b",
+                "c",
+                "d",
+                "e",
+                "f",
+                "0",
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6",
+                "7",
+                "8",
+                "9",
+            ]
+        ):
+            s += r"\x" + hex(b)[2:]
+            in_hex = True
+        else:
+            s += chr(b)
+            in_hex = False
+    return s
+
+
+def evaluate(assembler: str, data: list):
+    bytecode = compile(assembler)
+    trace = True
+    if trace:
+        print(
+            "Compiled to {0} bytes of bytecode:\n{1}".format(
+                len(bytecode), stringify(bytecode)
+            )
+        )
+    result = interpret(bytecode, [], data, False)  # trace)
+    if trace:
+        print("--> {0}".format(result))
+    return result
+
+
+# def GetOptionalValue(valobj):
+#    storage = valobj.GetChildMemberWithName("Storage")
+#    if not storage:
+#        storage = valobj
+#
+#    failure = 2
+#    hasVal = storage.GetChildMemberWithName("hasVal").GetValueAsUnsigned(failure)
+#    if hasVal == failure:
+#        return "<could not read MyOptional>"
+#
+#    if hasVal == 0:
+#        return None
+#
+#    underlying_type = storage.GetType().GetTemplateArgumentType(0)
+#    storage = storage.GetChildMemberWithName("value")
+#    return storage.Cast(underlying_type)
+
+
+def MyOptionalSummaryProvider(valobj, internal_dict):
+    #    val = GetOptionalValue(valobj)
+    #    if val is None:
+    #        return "None"
+    #    if val.summary:
+    #        return val.summary
+    #    return val.GetValue()
+    summary = ""
+    summary += ' dup "Storage" @get_child_with_name call'  # valobj storage
+    summary += " dup is_null ~ { swap } if drop"  # storage
+    summary += ' dup "hasVal" @get_child_with_name call'  # storage obj(hasVal)
+    summary += ' dup is_null { drop "<could not read MyOptional>" } {'
+    summary += "   @get_value_as_unsigned call"  # storage int(hasVal)
+    summary += '   0u = { "None" } {'
+    summary += "     dup @get_type call"
+    summary += "     0u @get_template_argument_type call"  # storage type
+    summary += "     swap"  # type storage
+    summary += '     "value" @get_child_with_name call'  # type value
+    summary += "     swap @cast call"  # type(value)
+    summary += '     dup is_null { "None" } {'
+    summary += (
+        "       dup @summary call dup @strlen call { @get_value call } { drop } ifelse"
+    )
+    summary += "     } ifelse"
+    summary += "   } ifelse"
+    summary += " } ifelse"
+    return evaluate(summary, [valobj])
+
+
+class MyOptionalSynthProvider:
+    """Provides deref support to llvm::Optional<T>"""
+
+    def __init__(self, valobj, internal_dict):
+        self.valobj = valobj
+
+    def num_children(self):
+        # return self.valobj.num_children
+        num_children = " @get_num_children call"
+        return evaluate(num_children, [self.valobj])
+
+    def get_child_index(self, name):
+        # if name == "$$dereference$$":
+        #    return self.valobj.num_children
+        # return self.valobj.GetIndexOfChildWithName(name)
+        get_child_index = ' dup "$$dereference$$" ='
+        get_child_index += " { drop @get_num_children call } {"  # obj name
+        get_child_index += "   @get_child_index call"  # index
+        get_child_index += " } ifelse"
+        return evaluate(get_child_index, [self.valobj, name])
+
+    def get_child_at_index(self, index):
+        # if index < self.valobj.num_children:
+        #    return self.valobj.GetChildAtIndex(index)
+        # return GetOptionalValue(self.valobj) or lldb.SBValue()
+        get_child_at_index = " over over swap"  # obj index index obj
+        get_child_at_index += " @get_num_children call"  # obj index index n
+        get_child_at_index += " < { @get_child_at_index call } {"  # obj index
+
+        get_opt_val = ' dup "Storage" @get_child_with_name call'  # valobj storage
+        get_opt_val += " dup { swap } if drop"  # storage
+        get_opt_val += ' dup "hasVal" @get_child_with_name call'  # storage
+        get_opt_val += " @get_value_as_unsigned call"  # storage int(hasVal)
+        get_opt_val += ' dup 2 = { drop "<could not read MyOptional>" } {'
+        get_opt_val += '   0 = { "None" } {'
+        get_opt_val += (
+            "     dup @get_type call 0 @get_template_argument_type call"  # storage type
+        )
+        get_opt_val += "     swap"  # type storage
+        get_opt_val += '     "value" @get_child_with_name call'  # type value
+        get_opt_val += "     swap @cast call"  # type(value)
+        get_opt_val += "   } ifelse"
+        get_opt_val += " } ifelse"
+
+        get_child_at_index += get_opt_val
+        get_child_at_index += " } ifelse"
+
+        return evaluate(get_child_at_index, [self.valobj, index])
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/bytecode.test b/lldb/test/Shell/ScriptInterpreter/Python/bytecode.test
new file mode 100644
index 0000000000000..2745680292663
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/bytecode.test
@@ -0,0 +1,16 @@
+# RUN: %python %S/../../../../examples/python/formatter_bytecode.py --test
+# RUN: %python %S/../../../../examples/python/formatter_bytecode.py --compile "1u dup" | FileCheck %s --check-prefix=COMPILE
+# RUN: %python %S/../../../../examples/python/formatter_bytecode.py --disassemble "200101" | FileCheck %s --check-prefix=DISASSEMBLE
+# COMPILE: 200101
+# DISASSEMBLE: 1u dup
+
+# RUN: %clang_host -std=c++17 -g %S/Inputs/FormatterBytecode/MyOptional.cpp -o %t.exe
+# RUN: %lldb %t.exe -o "command script import %S/../../../../examples/python/formatter_bytecode.py" -o "command script import %S/Inputs/FormatterBytecode/formatter.py" -o "b -p here" -o "r" -o "v x" -o "v y" -o q | FileCheck %s --check-prefix=OPTIONAL
+# OPTIONAL: (lldb) v x
+# OPTIONAL: (MyOptional<int>) x = {
+# OPTIONAL: hasVal = false
+# OPTIONAL: }
+# OPTIONAL: (lldb) v y
+# OPTIONAL: (MyOptional<int>) y = {
+# OPTIONAL:   Storage = (value = 42, hasVal = true)
+# OPTIONAL: }
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/sb_function_ranges.s b/lldb/test/Shell/ScriptInterpreter/Python/sb_function_ranges.s
new file mode 100644
index 0000000000000..a9e4104f2aaf7
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/sb_function_ranges.s
@@ -0,0 +1,182 @@
+# REQUIRES: x86
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %t/input.s -o %t/input.o
+# RUN: %lldb %t/input.o -o "command script import %t/script.py" -o exit | FileCheck %s
+
+# CHECK: Found 1 function(s).
+# CHECK: foo: [input.o[0x0-0xe), input.o[0x14-0x1c)]
+
+#--- script.py
+import lldb
+
+def __lldb_init_module(debugger, internal_dict):
+  target = debugger.GetSelectedTarget()
+  sym_ctxs = target.FindFunctions("foo")
+  print(f"Found {len(sym_ctxs)} function(s).")
+  for ctx in sym_ctxs:
+    fn = ctx.function
+    print(f"{fn.name}: {fn.GetRanges()}")
+
+#--- input.s
+# An example of a function which has been split into two parts. Roughly
+# corresponds to this C code.
+# int baz();
+# int bar() { return 47; }
+# int foo(int flag) { return flag ? bar() : baz(); }
+# The function bar has been placed "in the middle" of foo.
+
+        .text
+
+        .type   foo,@function
+foo:
+        .cfi_startproc
+        cmpl    $0, %edi
+        je      foo.__part.2
+        jmp     foo.__part.1
+        .cfi_endproc
+.Lfoo_end:
+        .size   foo, .Lfoo_end-foo
+
+foo.__part.1:
+        .cfi_startproc
+        callq   bar
+        jmp     foo.__part.3
+.Lfoo.__part.1_end:
+        .size   foo.__part.1, .Lfoo.__part.1_end-foo.__part.1
+        .cfi_endproc
+
+bar:
+        .cfi_startproc
+        movl    $47, %eax
+        retq
+        .cfi_endproc
+.Lbar_end:
+        .size   bar, .Lbar_end-bar
+
+foo.__part.2:
+        .cfi_startproc
+        callq   baz
+        jmp     foo.__part.3
+.Lfoo.__part.2_end:
+        .size   foo.__part.2, .Lfoo.__part.2_end-foo.__part.2
+        .cfi_endproc
+
+foo.__part.3:
+        .cfi_startproc
+        retq
+.Lfoo.__part.3_end:
+        .size   foo.__part.3, .Lfoo.__part.3_end-foo.__part.3
+        .cfi_endproc
+
+
+        .section        .debug_abbrev,"",@progbits
+        .byte   1                               # Abbreviation Code
+        .byte   17                              # DW_TAG_compile_unit
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   37                              # DW_AT_producer
+        .byte   8                               # DW_FORM_string
+        .byte   19                              # DW_AT_language
+        .byte   5                               # DW_FORM_data2
+        .byte   17                              # DW_AT_low_pc
+        .byte   1                               # DW_FORM_addr
+        .byte   85                              # DW_AT_ranges
+        .byte   35                              # DW_FORM_rnglistx
+        .byte   116                             # DW_AT_rnglists_base
+        .byte   23                              # DW_FORM_sec_offset
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   2                               # Abbreviation Code
+        .byte   46                              # DW_TAG_subprogram
+        .byte   0                               # DW_CHILDREN_no
+        .byte   17                              # DW_AT_low_pc
+        .byte   1                               # DW_FORM_addr
+        .byte   18                              # DW_AT_high_pc
+        .byte   1                               # DW_FORM_addr
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   3                               # Abbreviation Code
+        .byte   46                              # DW_TAG_subprogram
+        .byte   0                               # DW_CHILDREN_no
+        .byte   85                              # DW_AT_ranges
+        .byte   35                              # DW_FORM_rnglistx
+        .byte   64                              # DW_AT_frame_base
+        .byte   24                              # DW_FORM_exprloc
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   0                               # EOM(3)
+
+        .section        .debug_info,"",@progbits
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  5                               # DWARF version number
+        .byte   1                               # DWARF Unit Type
+        .byte   8                               # Address Size (in bytes)
+        .long   .debug_abbrev                   # Offset Into Abbrev. Section
+        .byte   1                               # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"            # DW_AT_producer
+        .short  29                              # DW_AT_language
+        .quad   0                               # DW_AT_low_pc
+        .byte   1                               # DW_AT_ranges
+        .long   .Lrnglists_table_base0          # DW_AT_rnglists_base
+        .byte   2                               # Abbrev [2] DW_TAG_subprogram
+        .quad   bar                             # DW_AT_low_pc
+        .quad   .Lbar_end                       # DW_AT_high_pc
+        .asciz  "bar"                           # DW_AT_name
+        .byte   3                               # Abbrev [3] DW_TAG_subprogram
+        .byte   0                               # DW_AT_ranges
+        .byte   1                               # DW_AT_frame_base
+        .byte   86
+        .asciz  "foo"                           # DW_AT_name
+        .byte   0                               # End Of Children Mark
+.Ldebug_info_end0:
+
+        .section        .debug_rnglists,"",@progbits
+        .long   .Ldebug_list_header_end0-.Ldebug_list_header_start0 # Length
+.Ldebug_list_header_start0:
+        .short  5                               # Version
+        .byte   8                               # Address size
+        .byte   0                               # Segment selector size
+        .long   2                               # Offset entry count
+.Lrnglists_table_base0:
+        .long   .Ldebug_ranges0-.Lrnglists_table_base0
+        .long   .Ldebug_ranges1-.Lrnglists_table_base0
+.Ldebug_ranges0:
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo
+        .quad   .Lfoo_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.1
+        .quad   .Lfoo.__part.1_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.2
+        .quad   .Lfoo.__part.2_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.3
+        .quad   .Lfoo.__part.3_end
+        .byte   0                               # DW_RLE_end_of_list
+.Ldebug_ranges1:
+        .byte   6                               # DW_RLE_start_end
+        .quad   bar
+        .quad   .Lbar_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.1
+        .quad   .Lfoo.__part.1_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.2
+        .quad   .Lfoo.__part.2_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo.__part.3
+        .quad   .Lfoo.__part.3_end
+        .byte   6                               # DW_RLE_start_end
+        .quad   foo
+        .quad   .Lfoo_end
+        .byte   0                               # DW_RLE_end_of_list
+.Ldebug_list_header_end0:
+
+        .section        ".note.GNU-stack","",@progbits
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s b/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
index 2584158207cc8..b03d5d12ad2a1 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/discontinuous-function.s
@@ -10,7 +10,7 @@
 
 # CHECK: 1 match found in {{.*}}
 # CHECK: Summary: {{.*}}`foo
-# CHECK: Function: id = {{.*}}, name = "foo", ranges = [0x0000000000000000-0x0000000000000007)[0x0000000000000007-0x000000000000000e)[0x0000000000000014-0x000000000000001b)[0x000000000000001b-0x000000000000001c)
+# CHECK: Function: id = {{.*}}, name = "foo", ranges = [0x0000000000000000-0x000000000000000e)[0x0000000000000014-0x000000000000001c)
 
         .text
 
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-index-cache.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-index-cache.cpp
new file mode 100644
index 0000000000000..3e97c3fb1ebc2
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-index-cache.cpp
@@ -0,0 +1,62 @@
+// REQUIRES: lld
+
+// Test if we build a mixed binary where one .o file has a .debug_names and
+// another doesn't have one, that we save a full or partial index cache.
+// Previous versions of LLDB would have ManualDWARFIndex.cpp that would save out
+// an index cache to the same file regardless of wether the index cache was a
+// full DWARF manual index, or just the CUs and TUs that were missing from any
+// .debug_names tables. If the user had a .debug_names table and debugged once
+// with index caching enabled, then debugged again but set the setting to ignore
+// .debug_names ('settings set plugin.symbol-file.dwarf.ignore-file-indexes 1')
+// this could cause LLDB to load the index cache from the previous run which
+// was incomplete and it only contained the manually indexed DWARF from the run
+// where we used .debug_names, but it would now load it as if it were the
+// complete DWARF index.
+
+// Test that if we don't have .debug_names, that we save a full DWARF index.
+// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -gdwarf-5 -DMAIN=1 -c %s -o %t.main.o
+// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -gdwarf-5 -DMAIN=0 -c %s -o %t.foo.o
+// RUN: ld.lld %t.main.o %t.foo.o -o %t.nonames
+// RUN: llvm-dwp %t.main.dwo %t.foo.dwo -o %t.nonames.dwp
+// RUN: rm %t.main.dwo %t.foo.dwo
+// Run one time with the index cache enabled to populate the index cache. When
+// we populate the index cache we have to parse all of the DWARF debug info
+// and it is always available.
+// RUN: rm -rf %t.lldb-index-cache
+// RUN: %lldb \
+// RUN:   -O 'settings set symbols.enable-lldb-index-cache true' \
+// RUN:   -O 'settings set symbols.lldb-index-cache-path %t.lldb-index-cache' \
+// RUN:   -O 'settings set target.preload-symbols true' \
+// RUN:   %t.nonames -b
+
+// Make sure there is a file with "dwarf-index-full" in its filename
+// RUN: ls %t.lldb-index-cache | FileCheck %s -check-prefix=FULL
+// FULL: {{dwp-index-cache.cpp.tmp.nonames.*-dwarf-index-full-}}
+
+// Test that if we have one .o file with .debug_names and one without, that we
+// save a partial DWARF index.
+// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -gdwarf-5 -DMAIN=1 -c %s -o %t.main.o -gpubnames
+// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -gdwarf-5 -DMAIN=0 -c %s -o %t.foo.o
+// RUN: ld.lld %t.main.o %t.foo.o -o %t.somenames
+// RUN: llvm-dwp %t.main.dwo %t.foo.dwo -o %t.somenames.dwp
+// RUN: rm %t.main.dwo %t.foo.dwo
+// Run one time with the index cache enabled to populate the index cache. When
+// we populate the index cache we have to parse all of the DWARF debug info
+// and it is always available.
+// RUN: rm -rf %t.lldb-index-cache
+// RUN: %lldb \
+// RUN:   -O 'settings set symbols.enable-lldb-index-cache true' \
+// RUN:   -O 'settings set symbols.lldb-index-cache-path %t.lldb-index-cache' \
+// RUN:   -O 'settings set target.preload-symbols true' \
+// RUN:   %t.somenames -b
+
+// Make sure there is a file with "dwarf-index-full" in its filename
+// RUN: ls %t.lldb-index-cache | FileCheck %s -check-prefix=PARTIAL
+// PARTIAL: {{dwp-index-cache.cpp.tmp.somenames.*-dwarf-index-partial-}}
+
+#if MAIN
+extern int foo();
+int main() { return foo(); }
+#else
+int foo() { return 0; }
+#endif
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/find-basic-type.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/find-basic-type.cpp
index 2ed7b219d8da3..af49206608723 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/find-basic-type.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/find-basic-type.cpp
@@ -4,6 +4,8 @@
 // RUN: ld.lld %t.o -o %t
 // RUN: lldb-test symbols --name=foo --find=type %t | \
 // RUN:   FileCheck --check-prefix=NAME %s
+// RUN: lldb-test symbols --name=::foo --find=type %t | \
+// RUN:   FileCheck --check-prefix=EXACT %s
 // RUN: lldb-test symbols --name=foo --context=context --find=type %t | \
 // RUN:   FileCheck --check-prefix=CONTEXT %s
 // RUN: lldb-test symbols --name=not_there --find=type %t | \
@@ -12,6 +14,8 @@
 // RUN: %clang %s -g -c -o %t --target=x86_64-apple-macosx
 // RUN: lldb-test symbols --name=foo --find=type %t | \
 // RUN:   FileCheck --check-prefix=NAME %s
+// RUN: lldb-test symbols --name=::foo --find=type %t | \
+// RUN:   FileCheck --check-prefix=EXACT %s
 // RUN: lldb-test symbols --name=foo --context=context --find=type %t | \
 // RUN:   FileCheck --check-prefix=CONTEXT %s
 // RUN: lldb-test symbols --name=not_there --find=type %t | \
@@ -22,6 +26,8 @@
 // RUN: llvm-readobj --sections %t | FileCheck %s --check-prefix NAMES
 // RUN: lldb-test symbols --name=foo --find=type %t | \
 // RUN:   FileCheck --check-prefix=NAME %s
+// RUN: lldb-test symbols --name=::foo --find=type %t | \
+// RUN:   FileCheck --check-prefix=EXACT %s
 // RUN: lldb-test symbols --name=foo --context=context --find=type %t | \
 // RUN:   FileCheck --check-prefix=CONTEXT %s
 // RUN: lldb-test symbols --name=not_there --find=type %t | \
@@ -31,9 +37,11 @@
 
 // EMPTY: Found 0 types:
 // NAME: Found 4 types:
+// EXACT: Found 1 types:
 // CONTEXT: Found 1 types:
 struct foo { };
 // NAME-DAG: name = "foo", {{.*}} decl = find-basic-type.cpp:[[@LINE-1]]
+// EXACT-DAG: name = "foo", {{.*}} decl = find-basic-type.cpp:[[@LINE-2]]
 
 namespace bar {
 int context;
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/simplified-template-names.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/simplified-template-names.cpp
new file mode 100644
index 0000000000000..328d6d2e16d59
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/simplified-template-names.cpp
@@ -0,0 +1,36 @@
+// Test lldb is able to compute the fully qualified names on templates with
+// -gsimple-template-names and -fdebug-types-section.
+
+// REQUIRES: lld
+
+// Test against logging to see if we print the fully qualified names correctly.
+// RUN: %clangxx --target=x86_64-pc-linux -g -gsimple-template-names %s -c -o %t1.o
+// RUN: ld.lld %t1.o -o %t1
+// RUN: %lldb %t1 -o "log enable dwarf comp" -o "target variable v3" -o exit | FileCheck %s --check-prefix=LOG
+
+// Test that we following DW_AT_signature correctly. If not, lldb might confuse the types of v1 and v2.
+// RUN: %clangxx --target=x86_64-pc-linux -g -gsimple-template-names -fdebug-types-section %s -c -o %t2.o
+// RUN: ld.lld %t2.o -o %t2
+// RUN: %lldb %t2 -o "target variable v1 v2" -o exit | FileCheck %s --check-prefix=TYPE
+
+// LOG: unique name: t3<t2<int> >::t4
+
+// TYPE:      (t2<outer_struct1::t1<int> >) v1 = {}
+// TYPE-NEXT: (t2<outer_struct2::t1<int> >) v2 = {}
+
+struct outer_struct1 {
+  template <typename> struct t1 {};
+};
+
+struct outer_struct2 {
+  template <typename> struct t1 {};
+};
+
+template <typename> struct t2 {};
+t2<outer_struct1::t1<int>> v1;
+t2<outer_struct2::t1<int>> v2;
+
+template <typename> struct t3 {
+  struct t4 {};
+};
+t3<t2<int>>::t4 v3;
diff --git a/lldb/test/Shell/SymbolFile/PDB/function-nested-block.test b/lldb/test/Shell/SymbolFile/PDB/function-nested-block.test
index 1cb20a4036382..9057d01c25840 100644
--- a/lldb/test/Shell/SymbolFile/PDB/function-nested-block.test
+++ b/lldb/test/Shell/SymbolFile/PDB/function-nested-block.test
@@ -2,7 +2,6 @@ REQUIRES: system-windows, lld
 RUN: %build --compiler=clang-cl --nodefaultlib --output=%t.exe %S/Inputs/FunctionNestedBlockTest.cpp
 RUN: lldb-test symbols -find=function -file FunctionNestedBlockTest.cpp -line 4 %t.exe | FileCheck --check-prefix=CHECK-FUNCTION %s
 RUN: lldb-test symbols -find=block -file FunctionNestedBlockTest.cpp -line 4 %t.exe | FileCheck --check-prefix=CHECK-BLOCK %s
-XFAIL: *
 
 CHECK-FUNCTION: Found 1 functions:
 CHECK-FUNCTION: name = "{{.*}}", mangled = "{{_?}}main"
diff --git a/lldb/tools/lldb-dap/ProgressEvent.h b/lldb/tools/lldb-dap/ProgressEvent.h
index dac21977add2d..72317b879c803 100644
--- a/lldb/tools/lldb-dap/ProgressEvent.h
+++ b/lldb/tools/lldb-dap/ProgressEvent.h
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <atomic>
+#include <chrono>
 #include <mutex>
 #include <optional>
 #include <queue>
diff --git a/lldb/tools/lldb-dap/README.md b/lldb/tools/lldb-dap/README.md
index 42b5f501e32c6..123869a033724 100644
--- a/lldb/tools/lldb-dap/README.md
+++ b/lldb/tools/lldb-dap/README.md
@@ -6,9 +6,9 @@ The extension requires the `lldb-dap` (formerly `lldb-vscode`) binary.
 This binary is not packaged with the VS Code extension.
 
 There are multiple ways to obtain this binary:
-* use the binary provided by your toolchain (for example `xcrun -f lldb-dap` on macOS) or contact your toolchain vendor to include it.
-* download one of the relase packages from the [LLVM release page](https://github.com/llvm/llvm-project/releases/). The `LLVM-19.1.0-{operating_system}.tar.xz` packages contain a prebuilt `lldb-dap` binary.
-* build it from source (see [LLDB's build instructions](https://lldb.llvm.org/resources/build.html))
+* Use the binary provided by your toolchain (for example `xcrun -f lldb-dap` on macOS) or contact your toolchain vendor to include it.
+* Download one of the relase packages from the [LLVM release page](https://github.com/llvm/llvm-project/releases/). The `LLVM-19.1.0-{operating_system}.tar.xz` packages contain a prebuilt `lldb-dap` binary.
+* Build it from source (see [LLDB's build instructions](https://lldb.llvm.org/resources/build.html)).
 
 By default, the VS Code extension will expect to find `lldb-dap` in your `PATH`.
 Alternatively, you can explictly specify the location of the `lldb-dap` binary using the `lldb-dap.executable-path` setting.
@@ -179,26 +179,26 @@ The default hostname being used `localhost`.
 For both launch and attach configurations, lldb-dap accepts the following `lldb-dap`
 specific key/value pairs:
 
-|parameter          |type|req |         |
-|-------------------|----|:--:|---------|
-|**name**           |string|Y| A configuration name that will be displayed in the IDE.
-|**type**           |string|Y| Must be "lldb-dap".
-|**request**        |string|Y| Must be "launch" or "attach".
-|**program**        |string|Y| Path to the executable to launch.
-|**sourcePath**     |string| | Specify a source path to remap \"./\" to allow full paths to be used when setting breakpoints in binaries that have relative source paths.
-|**sourceMap**      |[string[2]]| | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname. Overrides sourcePath.
-|**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-dap. If the debug information in your executable contains relative paths, this option can be used so that `lldb-dap` can find source files and object files that have relative paths.
-|**commandEscapePrefix** | string | | The escape prefix to use for executing regular LLDB commands in the Debug Console, instead of printing variables. Defaults to a backtick. If it's an empty string, then all expression in the Debug Console are treated as regular LLDB commands.
-|**customFrameFormat** | string | | If non-empty, stack frames will have descriptions generated based on the provided format. See https://lldb.llvm.org/use/formatting.html for an explanation on format strings for frames. If the format string contains errors, an error message will be displayed on the Debug Console and the default frame names will be used. This might come with a performance cost because debug information might need to be processed to generate the description.
-|**customThreadFormat** | string | | Same as `customFrameFormat`, but for threads instead of stack frames.
-|**displayExtendedBacktrace**|bool| | Enable language specific extended backtraces.
-|**enableAutoVariableSummaries**|bool| | Enable auto generated summaries for variables when no summaries exist for a given type. This feature can cause performance delays in large projects when viewing variables.
-|**enableSyntheticChildDebugging**|bool| | If a variable is displayed using a synthetic children, also display the actual contents of the variable at the end under a [raw] entry. This is useful when creating sythetic child plug-ins as it lets you see the actual contents of the variable.
-|**initCommands**   |[string]| | LLDB commands executed upon debugger startup prior to creating the LLDB target.
-|**preRunCommands** |[string]| | LLDB commands executed just before launching/attaching, after the LLDB target has been created.
-|**stopCommands**   |[string]| | LLDB commands executed just after each stop.
-|**exitCommands**   |[string]| | LLDB commands executed when the program exits.
-|**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends.
+| Parameter                         | Type        | Req |         |
+|-----------------------------------|-------------|:---:|---------|
+| **name**                          | string      | Y   | A configuration name that will be displayed in the IDE.
+| **type**                          | string      | Y   | Must be "lldb-dap".
+| **request**                       | string      | Y   | Must be "launch" or "attach".
+| **program**                       | string      | Y   | Path to the executable to launch.
+| **sourcePath**                    | string      |     | Specify a source path to remap \"./\" to allow full paths to be used when setting breakpoints in binaries that have relative source paths.
+| **sourceMap**                     | [string[2]] |     | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname. Overrides sourcePath.
+| **debuggerRoot**                  | string      |     | Specify a working directory to use when launching lldb-dap. If the debug information in your executable contains relative paths, this option can be used so that `lldb-dap` can find source files and object files that have relative paths.
+| **commandEscapePrefix**           | string      |     | The escape prefix to use for executing regular LLDB commands in the Debug Console, instead of printing variables. Defaults to a backtick. If it's an empty string, then all expression in the Debug Console are treated as regular LLDB commands.
+| **customFrameFormat**             | string      |     | If non-empty, stack frames will have descriptions generated based on the provided format. See https://lldb.llvm.org/use/formatting.html for an explanation on format strings for frames. If the format string contains errors, an error message will be displayed on the Debug Console and the default frame names will be used. This might come with a performance cost because debug information might need to be processed to generate the description.
+| **customThreadFormat**            | string      |     | Same as `customFrameFormat`, but for threads instead of stack frames.
+| **displayExtendedBacktrace**      | bool        |     | Enable language specific extended backtraces.
+| **enableAutoVariableSummaries**   | bool        |     | Enable auto generated summaries for variables when no summaries exist for a given type. This feature can cause performance delays in large projects when viewing variables.
+| **enableSyntheticChildDebugging** | bool        |     | If a variable is displayed using a synthetic children, also display the actual contents of the variable at the end under a [raw] entry. This is useful when creating sythetic child plug-ins as it lets you see the actual contents of the variable.
+| **initCommands**                  | [string]    |     | LLDB commands executed upon debugger startup prior to creating the LLDB target.
+| **preRunCommands**                | [string]    |     | LLDB commands executed just before launching/attaching, after the LLDB target has been created.
+| **stopCommands**                  | [string]    |     | LLDB commands executed just after each stop.
+| **exitCommands**                  | [string]    |     | LLDB commands executed when the program exits.
+| **terminateCommands**             | [string]    |     | LLDB commands executed when the debugging session ends.
 
 All commands and command outputs will be sent to the debugger console when they are executed.
 Commands can be prefixed with `?` or `!` to modify their behavior:
@@ -208,25 +208,25 @@ Commands can be prefixed with `?` or `!` to modify their behavior:
 For JSON configurations of `"type": "launch"`, the JSON configuration can additionally
 contain the following key/value pairs:
 
-|parameter          |type|req |         |
-|-------------------|----|:--:|---------|
-|**program**        |string|Y| Path to the executable to launch.
-|**args**           |[string]|| An array of command line argument strings to be passed to the program being launched.
-|**cwd**            |string| | The program working directory.
-|**env**            |dictionary| | Environment variables to set when launching the program. The format of each environment variable string is "VAR=VALUE" for environment variables with values or just "VAR" for environment variables with no values.
-|**stopOnEntry**    |boolean| | Whether to stop program immediately after launching.
-|**runInTerminal**  |boolean| | Launch the program inside an integrated terminal in the IDE. Useful for debugging interactive command line programs.
-|**launchCommands** |[string]| | LLDB commands executed to launch the program.
+| Parameter                         | Type        | Req |         |
+|-----------------------------------|-------------|:---:|---------|
+| **program**                       | string      | Y   | Path to the executable to launch.
+| **args**                          | [string]    |     | An array of command line argument strings to be passed to the program being launched.
+| **cwd**                           | string      |     | The program working directory.
+| **env**                           | dictionary  |     | Environment variables to set when launching the program. The format of each environment variable string is "VAR=VALUE" for environment variables with values or just "VAR" for environment variables with no values.
+| **stopOnEntry**                   | boolean     |     | Whether to stop program immediately after launching.
+| **runInTerminal**                 | boolean     |     | Launch the program inside an integrated terminal in the IDE. Useful for debugging interactive command line programs.
+| **launchCommands**                | [string]    |     | LLDB commands executed to launch the program.
 
 For JSON configurations of `"type": "attach"`, the JSON configuration can contain
 the following `lldb-dap` specific key/value pairs:
 
-|parameter          |type    |req |         |
-|-------------------|--------|:--:|---------|
-|**program**        |string  | | Path to the executable to attach to. This value is optional but can help to resolve breakpoints prior the attaching to the program.
-|**pid**            |number  | | The process id of the process you wish to attach to. If **pid** is omitted, the debugger will attempt to attach to the program by finding a process whose file name matches the file name from **porgram**. Setting this value to `${command:pickMyProcess}` will allow interactive process selection in the IDE.
-|**waitFor**        |boolean | | Wait for the process to launch.
-|**attachCommands** |[string]| | LLDB commands that will be executed after **preRunCommands** which take place of the code that normally does the attach. The commands can create a new target and attach or launch it however desired. This allows custom launch and attach configurations. Core files can use `target create --core /path/to/core` to attach to core files.
+| Parameter                         | Type        | Req |         |
+|-----------------------------------|-------------|:---:|---------|
+| **program**                       | string      |     | Path to the executable to attach to. This value is optional but can help to resolve breakpoints prior the attaching to the program.
+| **pid**                           | number      |     | The process id of the process you wish to attach to. If **pid** is omitted, the debugger will attempt to attach to the program by finding a process whose file name matches the file name from **porgram**. Setting this value to `${command:pickMyProcess}` will allow interactive process selection in the IDE.
+| **waitFor**                       | boolean     |     | Wait for the process to launch.
+| **attachCommands**                | [string]    |     | LLDB commands that will be executed after **preRunCommands** which take place of the code that normally does the attach. The commands can create a new target and attach or launch it however desired. This allows custom launch and attach configurations. Core files can use `target create --core /path/to/core` to attach to core files.
 
 ## Debug Console
 
@@ -295,7 +295,7 @@ and may also be adjusted at runtime using the lldb command
 lldb-dap includes a command to trigger a Debug Adapter Protocol event
 from a script.
 
-The event maybe a custom DAP event or a standard event, if the event is not 
+The event maybe a custom DAP event or a standard event, if the event is not
 handled internally by `lldb-dap`.
 
 This command has the format:
@@ -316,16 +316,16 @@ For example you can use a launch configuration hook to trigger custom events lik
 }
 ```
 
-[See the specification](https://microsoft.github.io/debug-adapter-protocol/specification#Base_Protocol_Event) 
-for more details on Debug Adapter Protocol events and the VS Code 
-[debug.onDidReceiveDebugSessionCustomEvent](https://code.visualstudio.com/api/references/vscode-api#debug.onDidReceiveDebugSessionCustomEvent) 
+[See the specification](https://microsoft.github.io/debug-adapter-protocol/specification#Base_Protocol_Event)
+for more details on Debug Adapter Protocol events and the VS Code
+[debug.onDidReceiveDebugSessionCustomEvent](https://code.visualstudio.com/api/references/vscode-api#debug.onDidReceiveDebugSessionCustomEvent)
 API for handling a custom event from an extension.
 
 ## Contributing
 
 `lldb-dap` and `lldb` are developed under the umbrella of the [LLVM project](https://llvm.org/).
-The source code is [part of the LLVM repository](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap) on Github.
-We use Github's [issue tracker](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap) and patches can be submitted via [pull requests](https://github.com/llvm/llvm-project/pulls).
+The source code is part of the [LLVM repository](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap) on Github.
+We use Github's [issue tracker](https://github.com/llvm/llvm-project/issues?q=label%3Alldb-dap) and patches can be submitted via [pull requests](https://github.com/llvm/llvm-project/pulls?q=label%3Alldb-dap).
 Furthermore, there is a [LLDB category](https://discourse.llvm.org/c/subprojects/lldb/8) on the LLVM discourse forum.
 
 For instructions on how to get started with development on lldb-dap, see the "[Contributing to lldb-dap](https://lldb.llvm.org/resources/lldbdap.html)"
diff --git a/lldb/tools/lldb-dap/package-lock.json b/lldb/tools/lldb-dap/package-lock.json
index 8663659715590..d1cb6d00ecf56 100644
--- a/lldb/tools/lldb-dap/package-lock.json
+++ b/lldb/tools/lldb-dap/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "lldb-dap",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lldb-dap",
-      "version": "0.2.6",
+      "version": "0.2.8",
       "license": "Apache 2.0 License with LLVM exceptions",
       "devDependencies": {
         "@types/node": "^18.11.18",
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 9155163c65ba5..bbe65e1f73fd8 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
@@ -44,7 +44,7 @@
     "format": "npx prettier './src-ts/' --write",
     "package": "vsce package --out ./out/lldb-dap.vsix",
     "publish": "vsce publish",
-    "vscode-uninstall": "code --uninstall-extension llvm.lldb-dap",
+    "vscode-uninstall": "code --uninstall-extension llvm-vs-code-extensions.lldb-dap",
     "vscode-install": "code --install-extension ./out/lldb-dap.vsix"
   },
   "contributes": {
@@ -141,7 +141,7 @@
     "debuggers": [
       {
         "type": "lldb-dap",
-        "label": "Native LLDB Debugger",
+        "label": "LLDB DAP Debugger",
         "program": "./bin/lldb-dap",
         "windows": {
           "program": "./bin/lldb-dap.exe"
@@ -508,9 +508,31 @@
               "env": [],
               "cwd": "^\"\\${workspaceRoot}\""
             }
+          },
+          {
+            "label": "LLDB: Attach",
+            "description": "",
+            "body": {
+              "type": "lldb-dap",
+              "request": "attach",
+              "name": "${2:Attach}",
+              "program": "${1:<your program>}",
+              "waitFor": true
+            }
+          },
+          {
+            "label": "LLDB: Load Coredump",
+            "description": "",
+            "body": {
+              "type": "lldb-dap",
+              "request": "attach",
+              "name": "${2:Core}",
+              "program": "${1:<your program>}",
+              "coreFile": "${1:<your program>}.core"
+            }
           }
         ]
       }
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
index 2be21bfdf0dd6..55c2f3e9f7deb 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
@@ -1,33 +1,99 @@
+import * as path from "path";
+import * as util from "util";
 import * as vscode from "vscode";
-import { LLDBDapOptions } from "./types";
+import * as child_process from "child_process";
+import * as fs from "node:fs/promises";
 
-/**
- * This class defines a factory used to find the lldb-dap binary to use
- * depending on the session configuration.
- */
-export class LLDBDapDescriptorFactory
-  implements vscode.DebugAdapterDescriptorFactory
-{
-  private lldbDapOptions: LLDBDapOptions;
-
-  constructor(lldbDapOptions: LLDBDapOptions) {
-    this.lldbDapOptions = lldbDapOptions;
+export async function isExecutable(path: string): Promise<Boolean> {
+  try {
+    await fs.access(path, fs.constants.X_OK);
+  } catch {
+    return false;
   }
+  return true;
+}
 
-  static async isValidDebugAdapterPath(
-    pathUri: vscode.Uri,
-  ): Promise<Boolean> {
+async function findWithXcrun(executable: string): Promise<string | undefined> {
+  if (process.platform === "darwin") {
     try {
-      const fileStats = await vscode.workspace.fs.stat(pathUri);
-      if (!(fileStats.type & vscode.FileType.File)) {
-        return false;
+      const exec = util.promisify(child_process.execFile);
+      let { stdout, stderr } = await exec("/usr/bin/xcrun", [
+        "-find",
+        executable,
+      ]);
+      if (stdout) {
+        return stdout.toString().trimEnd();
       }
-    } catch (err) {
-      return false;
+    } catch (error) {}
+  }
+  return undefined;
+}
+
+async function findInPath(executable: string): Promise<string | undefined> {
+  const env_path =
+    process.platform === "win32" ? process.env["Path"] : process.env["PATH"];
+  if (!env_path) {
+    return undefined;
+  }
+
+  const paths = env_path.split(path.delimiter);
+  for (const p of paths) {
+    const exe_path = path.join(p, executable);
+    if (await isExecutable(exe_path)) {
+      return exe_path;
     }
-    return true;
+  }
+  return undefined;
+}
+
+async function findDAPExecutable(): Promise<string | undefined> {
+  const executable = process.platform === "win32" ? "lldb-dap.exe" : "lldb-dap";
+
+  // Prefer lldb-dap from Xcode on Darwin.
+  const xcrun_dap = findWithXcrun(executable);
+  if (xcrun_dap) {
+    return xcrun_dap;
+  }
+
+  // Find lldb-dap in the user's path.
+  const path_dap = findInPath(executable);
+  if (path_dap) {
+    return path_dap;
   }
 
+  return undefined;
+}
+
+async function getDAPExecutable(
+  session: vscode.DebugSession,
+): Promise<string | undefined> {
+  const config = vscode.workspace.getConfiguration(
+    "lldb-dap",
+    session.workspaceFolder,
+  );
+
+  // Prefer the explicitly specified path in the extension's configuration.
+  const configPath = config.get<string>("executable-path");
+  if (configPath && configPath.length !== 0) {
+    return configPath;
+  }
+
+  // Try finding the lldb-dap binary.
+  const foundPath = await findDAPExecutable();
+  if (foundPath) {
+    return foundPath;
+  }
+
+  return undefined;
+}
+
+/**
+ * This class defines a factory used to find the lldb-dap binary to use
+ * depending on the session configuration.
+ */
+export class LLDBDapDescriptorFactory
+  implements vscode.DebugAdapterDescriptorFactory
+{
   async createDebugAdapterDescriptor(
     session: vscode.DebugSession,
     executable: vscode.DebugAdapterExecutable | undefined,
@@ -36,14 +102,40 @@ export class LLDBDapDescriptorFactory
       "lldb-dap",
       session.workspaceFolder,
     );
-    const customPath = config.get<string>("executable-path");
-    const path: string = customPath || executable!!.command;
 
-    const fileUri = vscode.Uri.file(path);
-    if (!(await LLDBDapDescriptorFactory.isValidDebugAdapterPath(fileUri))) {
-      LLDBDapDescriptorFactory.showLLDBDapNotFoundMessage(fileUri.path);
+    const log_path = config.get<string>("log-path");
+    let env: { [key: string]: string } = {};
+    if (log_path) {
+      env["LLDBDAP_LOG"] = log_path;
+    }
+    const configEnvironment =
+      config.get<{ [key: string]: string }>("environment") || {};
+    const dapPath = await getDAPExecutable(session);
+    const dbgOptions = {
+      env: {
+        ...executable?.options?.env,
+        ...configEnvironment,
+        ...env,
+      },
+    };
+    if (dapPath) {
+      if (!(await isExecutable(dapPath))) {
+        LLDBDapDescriptorFactory.showLLDBDapNotFoundMessage(dapPath);
+        return undefined;
+      }
+      return new vscode.DebugAdapterExecutable(dapPath, [], dbgOptions);
+    } else if (executable) {
+      if (!(await isExecutable(executable.command))) {
+        LLDBDapDescriptorFactory.showLLDBDapNotFoundMessage(executable.command);
+        return undefined;
+      }
+      return new vscode.DebugAdapterExecutable(
+        executable.command,
+        executable.args,
+        dbgOptions,
+      );
     }
-    return this.lldbDapOptions.createDapExecutableCommand(session, executable);
+    return undefined;
   }
 
   /**
diff --git a/lldb/tools/lldb-dap/src-ts/extension.ts b/lldb/tools/lldb-dap/src-ts/extension.ts
index 36d3dfba18c14..71fd48298f8f5 100644
--- a/lldb/tools/lldb-dap/src-ts/extension.ts
+++ b/lldb/tools/lldb-dap/src-ts/extension.ts
@@ -1,74 +1,24 @@
+import * as path from "path";
+import * as util from "util";
 import * as vscode from "vscode";
-import { LLDBDapOptions } from "./types";
-import { DisposableContext } from "./disposable-context";
-import { LLDBDapDescriptorFactory } from "./debug-adapter-factory";
-
-/**
- * This creates the configurations for this project if used as a standalone
- * extension.
- */
-function createDefaultLLDBDapOptions(): LLDBDapOptions {
-  return {
-    debuggerType: "lldb-dap",
-    async createDapExecutableCommand(
-      session: vscode.DebugSession,
-      packageJSONExecutable: vscode.DebugAdapterExecutable | undefined,
-    ): Promise<vscode.DebugAdapterExecutable | undefined> {
-      const config = vscode.workspace.getConfiguration(
-        "lldb-dap",
-        session.workspaceFolder,
-      );
-      const path = config.get<string>("executable-path");
-      const log_path = config.get<string>("log-path");
 
-      let env: { [key: string]: string } = {};
-      if (log_path) {
-        env["LLDBDAP_LOG"] = log_path;
-      }
-      const configEnvironment = config.get<{ [key: string]: string }>("environment") || {};
-      if (path) {
-        const dbgOptions = {
-          env: {
-            ...configEnvironment,
-            ...env,
-          }
-        };
-        return new vscode.DebugAdapterExecutable(path, [], dbgOptions);
-      } else if (packageJSONExecutable) {
-        return new vscode.DebugAdapterExecutable(
-          packageJSONExecutable.command,
-          packageJSONExecutable.args,
-          {
-            ...packageJSONExecutable.options,
-            env: {
-              ...packageJSONExecutable.options?.env,
-              ...configEnvironment,
-              ...env,
-            },
-          },
-        );
-      } else {
-        return undefined;
-      }
-    },
-  };
-}
+import {
+  LLDBDapDescriptorFactory,
+  isExecutable,
+} from "./debug-adapter-factory";
+import { DisposableContext } from "./disposable-context";
 
 /**
  * This class represents the extension and manages its life cycle. Other extensions
  * using it as as library should use this class as the main entry point.
  */
 export class LLDBDapExtension extends DisposableContext {
-  private lldbDapOptions: LLDBDapOptions;
-
-  constructor(lldbDapOptions: LLDBDapOptions) {
+  constructor() {
     super();
-    this.lldbDapOptions = lldbDapOptions;
-
     this.pushSubscription(
       vscode.debug.registerDebugAdapterDescriptorFactory(
-        this.lldbDapOptions.debuggerType,
-        new LLDBDapDescriptorFactory(this.lldbDapOptions),
+        "lldb-dap",
+        new LLDBDapDescriptorFactory(),
       ),
     );
 
@@ -80,10 +30,7 @@ export class LLDBDapExtension extends DisposableContext {
             .get<string>("executable-path");
 
           if (dapPath) {
-            const fileUri = vscode.Uri.file(dapPath);
-            if (
-              await LLDBDapDescriptorFactory.isValidDebugAdapterPath(fileUri)
-            ) {
+            if (await isExecutable(dapPath)) {
               return;
             }
           }
@@ -98,7 +45,5 @@ export class LLDBDapExtension extends DisposableContext {
  * This is the entry point when initialized by VS Code.
  */
 export function activate(context: vscode.ExtensionContext) {
-  context.subscriptions.push(
-    new LLDBDapExtension(createDefaultLLDBDapOptions()),
-  );
+  context.subscriptions.push(new LLDBDapExtension());
 }
diff --git a/lldb/tools/lldb-dap/src-ts/types.ts b/lldb/tools/lldb-dap/src-ts/types.ts
deleted file mode 100644
index 63a8c73982caf..0000000000000
--- a/lldb/tools/lldb-dap/src-ts/types.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import * as vscode from "vscode";
-
-/**
- * Callback used to generate the actual command to be executed to launch the lldb-dap binary.
- *
- * @param session - The information of the debug session to be launched.
- *
- * @param packageJSONExecutable - An optional {@link vscode.DebugAdapterExecutable executable} for
- * lldb-dap if specified in the package.json file.
- */
-export type LLDBDapCreateDAPExecutableCommand = (
-  session: vscode.DebugSession,
-  packageJSONExecutable: vscode.DebugAdapterExecutable | undefined,
-) => Promise<vscode.DebugAdapterExecutable | undefined>;
-
-/**
- * The options that this extension accepts.
- */
-export interface LLDBDapOptions {
-  createDapExecutableCommand: LLDBDapCreateDAPExecutableCommand;
-  // The name of the debugger type as specified in the package.json file.
-  debuggerType: string;
-}
diff --git a/lldb/tools/lldb-server/lldb-gdbserver.cpp b/lldb/tools/lldb-server/lldb-gdbserver.cpp
index bc93c837a107b..ed10c161b6b2f 100644
--- a/lldb/tools/lldb-server/lldb-gdbserver.cpp
+++ b/lldb/tools/lldb-server/lldb-gdbserver.cpp
@@ -210,9 +210,8 @@ void ConnectToRemote(MainLoop &mainloop,
                                     error.AsCString());
       exit(-1);
     }
-    connection_up =
-        std::unique_ptr<Connection>(new ConnectionFileDescriptor(new TCPSocket(
-            sockfd, /*should_close=*/true, /*child_processes_inherit=*/false)));
+    connection_up = std::unique_ptr<Connection>(new ConnectionFileDescriptor(
+        new TCPSocket(sockfd, /*should_close=*/true)));
 #else
     url = llvm::formatv("fd://{0}", connection_fd).str();
 
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index 735a558810da5..880b45b989b9c 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -181,8 +181,7 @@ static Status ListenGdbConnectionsIfNeeded(
   if (protocol != Socket::ProtocolTcp)
     return Status();
 
-  gdb_sock = std::make_unique<TCPSocket>(
-      /*should_close=*/true, /*child_processes_inherit=*/false);
+  gdb_sock = std::make_unique<TCPSocket>(/*should_close=*/true);
   Status error = gdb_sock->Listen(gdb_address, backlog);
   if (error.Fail())
     return error;
@@ -474,12 +473,10 @@ int main_platform(int argc, char *argv[]) {
     GDBRemoteCommunicationServerPlatform platform(protocol, gdbserver_port);
     Socket *socket;
     if (protocol == Socket::ProtocolTcp)
-      socket = new TCPSocket(sockfd, /*should_close=*/true,
-                             /*child_processes_inherit=*/false);
+      socket = new TCPSocket(sockfd, /*should_close=*/true);
     else {
 #if LLDB_ENABLE_POSIX
-      socket = new DomainSocket(sockfd, /*should_close=*/true,
-                                /*child_processes_inherit=*/false);
+      socket = new DomainSocket(sockfd, /*should_close=*/true);
 #else
       WithColor::error() << "lldb-platform child: Unix domain sockets are not "
                             "supported on this platform.";
@@ -510,8 +507,7 @@ int main_platform(int argc, char *argv[]) {
     return socket_error;
   }
 
-  std::unique_ptr<Socket> platform_sock =
-      Socket::Create(protocol, /*child_processes_inherit=*/false, error);
+  std::unique_ptr<Socket> platform_sock = Socket::Create(protocol, error);
   if (error.Fail()) {
     printf("Failed to create platform socket: %s\n", error.AsCString());
     return socket_error;
diff --git a/lldb/unittests/DataFormatter/CMakeLists.txt b/lldb/unittests/DataFormatter/CMakeLists.txt
index 9d967a72bfd1f..b858db1c71634 100644
--- a/lldb/unittests/DataFormatter/CMakeLists.txt
+++ b/lldb/unittests/DataFormatter/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_unittest(LLDBFormatterTests
   FormatManagerTests.cpp
   FormattersContainerTest.cpp
+  FormatterBytecodeTest.cpp
   StringPrinterTests.cpp
 
   LINK_LIBS
diff --git a/lldb/unittests/DataFormatter/FormatterBytecodeTest.cpp b/lldb/unittests/DataFormatter/FormatterBytecodeTest.cpp
new file mode 100644
index 0000000000000..15d9229de0033
--- /dev/null
+++ b/lldb/unittests/DataFormatter/FormatterBytecodeTest.cpp
@@ -0,0 +1,261 @@
+#include "DataFormatters/FormatterBytecode.h"
+#include "lldb/Utility/StreamString.h"
+
+#include "gtest/gtest.h"
+
+using namespace lldb_private;
+using namespace lldb;
+using namespace FormatterBytecode;
+using llvm::StringRef;
+
+namespace {
+class FormatterBytecodeTest : public ::testing::Test {};
+
+bool Interpret(std::vector<uint8_t> code, DataStack &data) {
+  auto buf =
+      StringRef(reinterpret_cast<const char *>(code.data()), code.size());
+  std::vector<ControlStackElement> control({buf});
+  if (auto error = Interpret(control, data, sel_summary)) {
+#ifndef NDEBUG
+    llvm::errs() << llvm::toString(std::move(error)) << '\n';
+#else
+    llvm::consumeError(std::move(error));
+#endif
+    return false;
+  }
+  return true;
+}
+
+} // namespace
+
+TEST_F(FormatterBytecodeTest, StackOps) {
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 23, op_dup, op_plus}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 46u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_drop}, data));
+    ASSERT_EQ(data.size(), 0u);
+  }
+  {
+    for (unsigned char i = 0; i < 3; ++i) {
+      DataStack data;
+
+      ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_lit_uint, 2,
+                             op_lit_uint, i, op_pick},
+                            data));
+      ASSERT_EQ(data.Pop<uint64_t>(), i);
+    }
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_over}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_swap}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret(
+        {op_lit_uint, 0, op_lit_uint, 1, op_lit_uint, 2, op_rot}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_EQ(data.Pop<uint64_t>(), 2u);
+  }
+}
+
+TEST_F(FormatterBytecodeTest, ControlOps) {
+  {
+    DataStack data;
+    ASSERT_TRUE(
+        Interpret({op_lit_uint, 0, op_begin, 2, op_lit_uint, 42, op_if}, data));
+    ASSERT_EQ(data.size(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(
+        Interpret({op_lit_uint, 1, op_begin, 2, op_lit_uint, 42, op_if}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 42u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_begin, 2, op_lit_uint, 42,
+                           op_begin, 2, op_lit_uint, 23, op_ifelse},
+                          data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 23u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_begin, 2, op_lit_uint, 42,
+                           op_begin, 2, op_lit_uint, 23, op_ifelse},
+                          data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 42u);
+  }
+  {
+    DataStack data(lldb::ValueObjectSP{});
+    ASSERT_TRUE(Interpret({op_is_null}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1u, op_as_int}, data));
+    ASSERT_EQ(data.Pop<int64_t>(), 1);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_int, 126, op_as_uint}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), ~1ULL);
+  }
+}
+
+TEST_F(FormatterBytecodeTest, ArithOps) {
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 2, op_lit_uint, 3, op_plus}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 5u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 3, op_lit_uint, 2, op_minus}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 3, op_lit_uint, 2, op_mul}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 6u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 6, op_lit_uint, 2, op_div}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 3u);
+  }
+  {
+    DataStack data;
+    ASSERT_FALSE(Interpret({op_lit_uint, 23, op_lit_uint, 0, op_div}, data));
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 2, op_shl}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 4u);
+  }
+  {
+    DataStack data;
+    unsigned char minus_one = 127;
+    ASSERT_TRUE(
+        Interpret({op_lit_int, minus_one, op_lit_uint, 2, op_shl}, data));
+    ASSERT_EQ(data.Pop<int64_t>(), -4);
+  }
+  {
+    DataStack data;
+    ASSERT_FALSE(Interpret({op_lit_uint, 1, op_lit_uint, 65, op_shl}, data));
+    ASSERT_FALSE(Interpret({op_lit_uint, 1, op_lit_uint, 65, op_shr}, data));
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_and}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_and}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_or}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_or}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 0, op_or}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_xor}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_xor}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 0, op_xor}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_not}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0xffffffffffffffff);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_eq}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 0, op_eq}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_neq}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 0, op_neq}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_lt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 0, op_lt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_lt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_gt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 0, op_gt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_gt}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_le}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 0, op_le}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_le}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+  {
+    DataStack data;
+    ASSERT_TRUE(Interpret({op_lit_uint, 0, op_lit_uint, 1, op_ge}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 0u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 0, op_ge}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+    ASSERT_TRUE(Interpret({op_lit_uint, 1, op_lit_uint, 1, op_ge}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 1u);
+  }
+}
+
+TEST_F(FormatterBytecodeTest, CallOps) {
+  {
+    DataStack data;
+    data.Push(std::string{"hello"});
+    ASSERT_TRUE(Interpret({op_lit_selector, sel_strlen, op_call}, data));
+    ASSERT_EQ(data.Pop<uint64_t>(), 5u);
+  }
+  {
+    DataStack data;
+    data.Push(std::string{"A"});
+    data.Push(std::string{"B"});
+    data.Push(std::string{"{1}{0}"});
+    ASSERT_TRUE(Interpret({op_lit_selector, sel_fmt, op_call}, data));
+    ASSERT_EQ(data.Pop<std::string>(), "BA");
+  }
+  {
+    DataStack data;
+    data.Push(std::string{"{0}"});
+    ASSERT_FALSE(Interpret({op_lit_selector, sel_fmt, op_call}, data));
+  }
+}
diff --git a/lldb/unittests/Host/MainLoopTest.cpp b/lldb/unittests/Host/MainLoopTest.cpp
index e7425b737a6da..e18489978e90c 100644
--- a/lldb/unittests/Host/MainLoopTest.cpp
+++ b/lldb/unittests/Host/MainLoopTest.cpp
@@ -27,22 +27,20 @@ class MainLoopTest : public testing::Test {
   SubsystemRAII<FileSystem, Socket> subsystems;
 
   void SetUp() override {
-    bool child_processes_inherit = false;
     Status error;
-    std::unique_ptr<TCPSocket> listen_socket_up(
-        new TCPSocket(true, child_processes_inherit));
+    auto listen_socket_up = std::make_unique<TCPSocket>(true);
     ASSERT_TRUE(error.Success());
     error = listen_socket_up->Listen("localhost:0", 5);
     ASSERT_TRUE(error.Success());
 
     Socket *accept_socket;
-    std::unique_ptr<TCPSocket> connect_socket_up(
-        new TCPSocket(true, child_processes_inherit));
+    auto connect_socket_up = std::make_unique<TCPSocket>(true);
     error = connect_socket_up->Connect(
         llvm::formatv("localhost:{0}", listen_socket_up->GetLocalPortNumber())
             .str());
     ASSERT_TRUE(error.Success());
-    ASSERT_TRUE(listen_socket_up->Accept(accept_socket).Success());
+    ASSERT_TRUE(listen_socket_up->Accept(std::chrono::seconds(1), accept_socket)
+                    .Success());
 
     callback_count = 0;
     socketpair[0] = std::move(connect_socket_up);
diff --git a/lldb/unittests/Host/SocketTest.cpp b/lldb/unittests/Host/SocketTest.cpp
index 4befddc0e21ce..9cbf3e26b0883 100644
--- a/lldb/unittests/Host/SocketTest.cpp
+++ b/lldb/unittests/Host/SocketTest.cpp
@@ -12,7 +12,9 @@
 #include "lldb/Host/MainLoop.h"
 #include "lldb/Utility/UriParser.h"
 #include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <chrono>
 
 using namespace lldb_private;
 
@@ -33,7 +35,7 @@ class SocketTest : public testing::TestWithParam<SocketTestParams> {
   }
 };
 
-TEST_P(SocketTest, DecodeHostAndPort) {
+TEST_F(SocketTest, DecodeHostAndPort) {
   EXPECT_THAT_EXPECTED(Socket::DecodeHostAndPort("localhost:1138"),
                        llvm::HasValue(Socket::HostAndPort{"localhost", 1138}));
 
@@ -61,9 +63,8 @@ TEST_P(SocketTest, DecodeHostAndPort) {
   EXPECT_THAT_EXPECTED(Socket::DecodeHostAndPort("*:65535"),
                        llvm::HasValue(Socket::HostAndPort{"*", 65535}));
 
-  EXPECT_THAT_EXPECTED(
-      Socket::DecodeHostAndPort("[::1]:12345"),
-      llvm::HasValue(Socket::HostAndPort{"::1", 12345}));
+  EXPECT_THAT_EXPECTED(Socket::DecodeHostAndPort("[::1]:12345"),
+                       llvm::HasValue(Socket::HostAndPort{"::1", 12345}));
 
   EXPECT_THAT_EXPECTED(
       Socket::DecodeHostAndPort("[abcd:12fg:AF58::1]:12345"),
@@ -71,9 +72,10 @@ TEST_P(SocketTest, DecodeHostAndPort) {
 }
 
 #if LLDB_ENABLE_POSIX
-TEST_P(SocketTest, DomainListenConnectAccept) {
+TEST_F(SocketTest, DomainListenConnectAccept) {
   llvm::SmallString<64> Path;
-  std::error_code EC = llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", Path);
+  std::error_code EC =
+      llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", Path);
   ASSERT_FALSE(EC);
   llvm::sys::path::append(Path, "test");
 
@@ -86,7 +88,7 @@ TEST_P(SocketTest, DomainListenConnectAccept) {
   CreateDomainConnectedSockets(Path, &socket_a_up, &socket_b_up);
 }
 
-TEST_P(SocketTest, DomainMainLoopAccept) {
+TEST_F(SocketTest, DomainListenGetListeningConnectionURI) {
   llvm::SmallString<64> Path;
   std::error_code EC =
       llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", Path);
@@ -98,7 +100,29 @@ TEST_P(SocketTest, DomainMainLoopAccept) {
     return;
 
   auto listen_socket_up = std::make_unique<DomainSocket>(
-      /*should_close=*/true, /*child_process_inherit=*/false);
+      /*should_close=*/true);
+  Status error = listen_socket_up->Listen(Path, 5);
+  ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
+  ASSERT_TRUE(listen_socket_up->IsValid());
+
+  ASSERT_THAT(
+      listen_socket_up->GetListeningConnectionURI(),
+      testing::ElementsAre(llvm::formatv("unix-connect://{0}", Path).str()));
+}
+
+TEST_F(SocketTest, DomainMainLoopAccept) {
+  llvm::SmallString<64> Path;
+  std::error_code EC =
+      llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", Path);
+  ASSERT_FALSE(EC);
+  llvm::sys::path::append(Path, "test");
+
+  // Skip the test if the $TMPDIR is too long to hold a domain socket.
+  if (Path.size() > 107u)
+    return;
+
+  auto listen_socket_up = std::make_unique<DomainSocket>(
+      /*should_close=*/true);
   Status error = listen_socket_up->Listen(Path, 5);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
   ASSERT_TRUE(listen_socket_up->IsValid());
@@ -113,7 +137,7 @@ TEST_P(SocketTest, DomainMainLoopAccept) {
   ASSERT_THAT_EXPECTED(expected_handles, llvm::Succeeded());
 
   auto connect_socket_up = std::make_unique<DomainSocket>(
-      /*should_close=*/true, /*child_process_inherit=*/false);
+      /*should_close=*/true);
   ASSERT_THAT_ERROR(connect_socket_up->Connect(Path).ToError(),
                     llvm::Succeeded());
   ASSERT_TRUE(connect_socket_up->IsValid());
@@ -133,7 +157,7 @@ TEST_P(SocketTest, TCPListen0ConnectAccept) {
                             &socket_b_up);
 }
 
-TEST_P(SocketTest, TCPMainLoopAccept) {
+TEST_P(SocketTest, TCPAcceptTimeout) {
   if (!HostSupportsProtocol())
     return;
 
@@ -145,6 +169,25 @@ TEST_P(SocketTest, TCPMainLoopAccept) {
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
   ASSERT_TRUE(listen_socket_up->IsValid());
 
+  Socket *socket;
+  ASSERT_THAT_ERROR(
+      listen_socket_up->Accept(std::chrono::milliseconds(10), socket)
+          .takeError(),
+      llvm::Failed<llvm::ErrorInfoBase>(
+          testing::Property(&llvm::ErrorInfoBase::convertToErrorCode,
+                            std::make_error_code(std::errc::timed_out))));
+}
+
+TEST_P(SocketTest, TCPMainLoopAccept) {
+  if (!HostSupportsProtocol())
+    return;
+
+  auto listen_socket_up = std::make_unique<TCPSocket>(true);
+  Status error = listen_socket_up->Listen(
+      llvm::formatv("[{0}]:0", GetParam().localhost_ip).str(), 5);
+  ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
+  ASSERT_TRUE(listen_socket_up->IsValid());
+
   MainLoop loop;
   std::unique_ptr<Socket> accepted_socket_up;
   auto expected_handles = listen_socket_up->Accept(
@@ -154,8 +197,7 @@ TEST_P(SocketTest, TCPMainLoopAccept) {
       });
   ASSERT_THAT_EXPECTED(expected_handles, llvm::Succeeded());
 
-  std::unique_ptr<TCPSocket> connect_socket_up(
-      new TCPSocket(true, child_processes_inherit));
+  auto connect_socket_up = std::make_unique<TCPSocket>(true);
   ASSERT_THAT_ERROR(
       connect_socket_up
           ->Connect(llvm::formatv("[{0}]:{1}", GetParam().localhost_ip,
@@ -195,7 +237,7 @@ TEST_P(SocketTest, UDPConnect) {
   if (!HostSupportsIPv4())
     return;
   llvm::Expected<std::unique_ptr<UDPSocket>> socket =
-      UDPSocket::Connect("127.0.0.1:0", /*child_processes_inherit=*/false);
+      UDPSocket::CreateConnected("127.0.0.1:0");
 
   ASSERT_THAT_EXPECTED(socket, llvm::Succeeded());
   EXPECT_TRUE(socket.get()->IsValid());
@@ -205,12 +247,48 @@ TEST_P(SocketTest, TCPListen0GetPort) {
   if (!HostSupportsIPv4())
     return;
   llvm::Expected<std::unique_ptr<TCPSocket>> sock =
-      Socket::TcpListen("10.10.12.3:0", false);
+      Socket::TcpListen("10.10.12.3:0", 5);
   ASSERT_THAT_EXPECTED(sock, llvm::Succeeded());
   ASSERT_TRUE(sock.get()->IsValid());
   EXPECT_NE(sock.get()->GetLocalPortNumber(), 0);
 }
 
+TEST_P(SocketTest, TCPListen0GetListeningConnectionURI) {
+  if (!HostSupportsProtocol())
+    return;
+
+  std::string addr = llvm::formatv("[{0}]:0", GetParam().localhost_ip).str();
+  llvm::Expected<std::unique_ptr<TCPSocket>> sock = Socket::TcpListen(addr);
+  ASSERT_THAT_EXPECTED(sock, llvm::Succeeded());
+  ASSERT_TRUE(sock.get()->IsValid());
+
+  EXPECT_THAT(
+      sock.get()->GetListeningConnectionURI(),
+      testing::ElementsAre(llvm::formatv("connection://[{0}]:{1}",
+                                         GetParam().localhost_ip,
+                                         sock->get()->GetLocalPortNumber())
+                               .str()));
+}
+
+TEST_F(SocketTest, TCPListen0MultiListenerGetListeningConnectionURI) {
+  if (!HostSupportsLocalhostToIPv4() || !HostSupportsLocalhostToIPv6())
+    return;
+
+  llvm::Expected<std::unique_ptr<TCPSocket>> sock =
+      Socket::TcpListen("localhost:0", 5);
+  ASSERT_THAT_EXPECTED(sock, llvm::Succeeded());
+  ASSERT_TRUE(sock.get()->IsValid());
+
+  EXPECT_THAT(sock.get()->GetListeningConnectionURI(),
+              testing::UnorderedElementsAre(
+                  llvm::formatv("connection://[::1]:{0}",
+                                sock->get()->GetLocalPortNumber())
+                      .str(),
+                  llvm::formatv("connection://[127.0.0.1]:{0}",
+                                sock->get()->GetLocalPortNumber())
+                      .str()));
+}
+
 TEST_P(SocketTest, TCPGetConnectURI) {
   std::unique_ptr<TCPSocket> socket_a_up;
   std::unique_ptr<TCPSocket> socket_b_up;
@@ -230,7 +308,7 @@ TEST_P(SocketTest, UDPGetConnectURI) {
   if (!HostSupportsIPv4())
     return;
   llvm::Expected<std::unique_ptr<UDPSocket>> socket =
-      UDPSocket::Connect("127.0.0.1:0", /*child_processes_inherit=*/false);
+      UDPSocket::CreateConnected("127.0.0.1:0");
   ASSERT_THAT_EXPECTED(socket, llvm::Succeeded());
 
   std::string uri = socket.get()->GetRemoteConnectionURI();
@@ -238,10 +316,10 @@ TEST_P(SocketTest, UDPGetConnectURI) {
 }
 
 #if LLDB_ENABLE_POSIX
-TEST_P(SocketTest, DomainGetConnectURI) {
+TEST_F(SocketTest, DomainGetConnectURI) {
   llvm::SmallString<64> domain_path;
-  std::error_code EC =
-      llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", domain_path);
+  std::error_code EC = llvm::sys::fs::createUniqueDirectory(
+      "DomainListenConnectAccept", domain_path);
   ASSERT_FALSE(EC);
   llvm::sys::path::append(domain_path, "test");
 
@@ -266,7 +344,7 @@ INSTANTIATE_TEST_SUITE_P(
     testing::Values(SocketTestParams{/*is_ipv6=*/false,
                                      /*localhost_ip=*/"127.0.0.1"},
                     SocketTestParams{/*is_ipv6=*/true, /*localhost_ip=*/"::1"}),
-    // Prints "SocketTests/SocketTest.DecodeHostAndPort/ipv4" etc. in test logs.
+    // Prints "SocketTests/SocketTest.TCPGetAddress/ipv4" etc. in test logs.
     [](const testing::TestParamInfo<SocketTestParams> &info) {
       return info.param.is_ipv6 ? "ipv6" : "ipv4";
     });
diff --git a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
index f1998a92f19b0..fc008ca7011e4 100644
--- a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
@@ -268,6 +268,13 @@ TEST_P(SDKPathParsingMultiparamTests, TestSDKPathFromDebugInfo) {
   EXPECT_EQ(found_mismatch, expect_mismatch);
   EXPECT_EQ(sdk.IsAppleInternalSDK(), expect_internal_sdk);
   EXPECT_NE(sdk.GetString().find(expect_sdk_path_pattern), std::string::npos);
+
+  {
+    auto sdk_or_err =
+        platform_sp->GetSDKPathFromDebugInfo(*dwarf_cu->GetLLDBCompUnit());
+    ASSERT_TRUE(static_cast<bool>(sdk_or_err));
+    EXPECT_EQ(sdk.IsAppleInternalSDK(), expect_internal_sdk);
+  }
 }
 
 SDKPathParsingTestData sdkPathParsingTestCases[] = {
diff --git a/lldb/unittests/TestingSupport/Host/SocketTestUtilities.cpp b/lldb/unittests/TestingSupport/Host/SocketTestUtilities.cpp
index 2455a4f6f5d49..72ecde845567f 100644
--- a/lldb/unittests/TestingSupport/Host/SocketTestUtilities.cpp
+++ b/lldb/unittests/TestingSupport/Host/SocketTestUtilities.cpp
@@ -19,34 +19,20 @@
 
 using namespace lldb_private;
 
-static void AcceptThread(Socket *listen_socket, bool child_processes_inherit,
-                         Socket **accept_socket, Status *error) {
-  *error = listen_socket->Accept(*accept_socket);
-}
-
 template <typename SocketType>
 void lldb_private::CreateConnectedSockets(
     llvm::StringRef listen_remote_address,
     const std::function<std::string(const SocketType &)> &get_connect_addr,
     std::unique_ptr<SocketType> *a_up, std::unique_ptr<SocketType> *b_up) {
-  bool child_processes_inherit = false;
   Status error;
-  std::unique_ptr<SocketType> listen_socket_up(
-      new SocketType(true, child_processes_inherit));
+  auto listen_socket_up = std::make_unique<SocketType>(true);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
   error = listen_socket_up->Listen(listen_remote_address, 5);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
   ASSERT_TRUE(listen_socket_up->IsValid());
 
-  Status accept_error;
-  Socket *accept_socket;
-  std::thread accept_thread(AcceptThread, listen_socket_up.get(),
-                            child_processes_inherit, &accept_socket,
-                            &accept_error);
-
   std::string connect_remote_address = get_connect_addr(*listen_socket_up);
-  std::unique_ptr<SocketType> connect_socket_up(
-      new SocketType(true, child_processes_inherit));
+  auto connect_socket_up = std::make_unique<SocketType>(true);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
   error = connect_socket_up->Connect(connect_remote_address);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
@@ -55,9 +41,13 @@ void lldb_private::CreateConnectedSockets(
   a_up->swap(connect_socket_up);
   ASSERT_TRUE((*a_up)->IsValid());
 
-  accept_thread.join();
+  Socket *accept_socket;
+  ASSERT_THAT_ERROR(
+      listen_socket_up->Accept(std::chrono::seconds(1), accept_socket)
+          .takeError(),
+      llvm::Succeeded());
+
   b_up->reset(static_cast<SocketType *>(accept_socket));
-  ASSERT_THAT_ERROR(accept_error.ToError(), llvm::Succeeded());
   ASSERT_NE(nullptr, b_up->get());
   ASSERT_TRUE((*b_up)->IsValid());
 
@@ -92,8 +82,7 @@ void lldb_private::CreateDomainConnectedSockets(
 #endif
 
 static bool CheckIPSupport(llvm::StringRef Proto, llvm::StringRef Addr) {
-  llvm::Expected<std::unique_ptr<TCPSocket>> Sock = Socket::TcpListen(
-      Addr, /*child_processes_inherit=*/false);
+  llvm::Expected<std::unique_ptr<TCPSocket>> Sock = Socket::TcpListen(Addr);
   if (Sock)
     return true;
   llvm::Error Err = Sock.takeError();
@@ -127,6 +116,28 @@ bool lldb_private::HostSupportsIPv6() {
   return CheckIPSupport("IPv6", "[::1]:0");
 }
 
+bool lldb_private::HostSupportsLocalhostToIPv4() {
+  if (!HostSupportsIPv4())
+    return false;
+
+  auto addresses = SocketAddress::GetAddressInfo(
+      "localhost", nullptr, AF_UNSPEC, SOCK_STREAM, IPPROTO_TCP);
+  return llvm::any_of(addresses, [](const SocketAddress &addr) {
+    return addr.GetFamily() == AF_INET;
+  });
+}
+
+bool lldb_private::HostSupportsLocalhostToIPv6() {
+  if (!HostSupportsIPv6())
+    return false;
+
+  auto addresses = SocketAddress::GetAddressInfo(
+      "localhost", nullptr, AF_UNSPEC, SOCK_STREAM, IPPROTO_TCP);
+  return llvm::any_of(addresses, [](const SocketAddress &addr) {
+    return addr.GetFamily() == AF_INET6;
+  });
+}
+
 llvm::Expected<std::string> lldb_private::GetLocalhostIP() {
   if (HostSupportsIPv4())
     return "127.0.0.1";
diff --git a/lldb/unittests/TestingSupport/Host/SocketTestUtilities.h b/lldb/unittests/TestingSupport/Host/SocketTestUtilities.h
index efd17428ceefb..e5bab163cf82e 100644
--- a/lldb/unittests/TestingSupport/Host/SocketTestUtilities.h
+++ b/lldb/unittests/TestingSupport/Host/SocketTestUtilities.h
@@ -43,6 +43,11 @@ void CreateDomainConnectedSockets(llvm::StringRef path,
 bool HostSupportsIPv6();
 bool HostSupportsIPv4();
 
+/// Returns true if the name `localhost` maps to a loopback IPv4 address.
+bool HostSupportsLocalhostToIPv4();
+/// Returns true if the name `localhost` maps to a loopback IPv6 address.
+bool HostSupportsLocalhostToIPv6();
+
 /// Return an IP for localhost based on host support.
 ///
 /// This will return either "127.0.0.1" if IPv4 is detected, or "[::1]" if IPv6
diff --git a/lldb/unittests/debugserver/RNBSocketTest.cpp b/lldb/unittests/debugserver/RNBSocketTest.cpp
index 75f236b4d4180..48f747b09587f 100644
--- a/lldb/unittests/debugserver/RNBSocketTest.cpp
+++ b/lldb/unittests/debugserver/RNBSocketTest.cpp
@@ -26,10 +26,10 @@ std::string goodbye = "Goodbye!";
 static void ServerCallbackv4(const void *baton, in_port_t port) {
   auto child_pid = fork();
   if (child_pid == 0) {
-    char addr_buffer[256];
-    sprintf(addr_buffer, "%s:%d", (const char *)baton, port);
+    std::string addr_buffer =
+        llvm::formatv("{0}:{1}", (const char *)baton, port).str();
     llvm::Expected<std::unique_ptr<Socket>> socket_or_err =
-        Socket::TcpConnect(addr_buffer, false);
+        Socket::TcpConnect(addr_buffer);
     ASSERT_THAT_EXPECTED(socket_or_err, llvm::Succeeded());
     Socket *client_socket = socket_or_err->get();
 
@@ -59,15 +59,12 @@ void TestSocketListen(const char *addr) {
   if (addresses.size() == 0)
     return;
 
-  char addr_wrap[256];
-  if (addresses.front().GetFamily() == AF_INET6)
-    sprintf(addr_wrap, "[%s]", addr);
-  else
-    sprintf(addr_wrap, "%s", addr);
+  const char *fmt = addresses.front().GetFamily() == AF_INET6 ? "[{0}]" : "{0}";
+  std::string addr_wrap = llvm::formatv(fmt, addr).str();
 
   RNBSocket server_socket;
-  auto result =
-      server_socket.Listen(addr, 0, ServerCallbackv4, (const void *)addr_wrap);
+  auto result = server_socket.Listen(addr, 0, ServerCallbackv4,
+                                     (const void *)addr_wrap.c_str());
   ASSERT_TRUE(result == rnb_success);
   result = server_socket.Write(hello.c_str(), hello.length());
   ASSERT_TRUE(result == rnb_success);
@@ -94,11 +91,9 @@ void TestSocketConnect(const char *addr) {
   if (addresses.size() == 0)
     return;
 
-  char addr_wrap[256];
-  if (addresses.front().GetFamily() == AF_INET6)
-    sprintf(addr_wrap, "[%s]:0", addr);
-  else
-    sprintf(addr_wrap, "%s:0", addr);
+  const char *fmt =
+      addresses.front().GetFamily() == AF_INET6 ? "[{0}]:0" : "{0}:0";
+  std::string addr_wrap = llvm::formatv(fmt, addr).str();
 
   Socket *server_socket;
   llvm::Expected<std::unique_ptr<Socket>> socket_or_err =
@@ -120,7 +115,8 @@ void TestSocketConnect(const char *addr) {
     ASSERT_EQ(bye, goodbye);
   } else {
     Socket *connected_socket;
-    Status err = server_socket->Accept(connected_socket);
+    Status err =
+        server_socket->Accept(std::chrono::seconds(10), connected_socket);
     if (err.Fail()) {
       llvm::errs() << err.AsCString();
       abort();
diff --git a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
index a6f2dc32c6d0c..48e089bc0fcff 100644
--- a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
+++ b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
@@ -26,9 +26,13 @@ using namespace lldb_private;
 using namespace llvm;
 using namespace llgs_tests;
 
+static std::chrono::seconds GetDefaultTimeout() {
+  return std::chrono::seconds{10};
+}
+
 TestClient::TestClient(std::unique_ptr<Connection> Conn) {
   SetConnection(std::move(Conn));
-  SetPacketTimeout(std::chrono::seconds(10));
+  SetPacketTimeout(GetDefaultTimeout());
 }
 
 TestClient::~TestClient() {
@@ -83,7 +87,7 @@ TestClient::launchCustom(StringRef Log, bool disable_stdio,
   const std::string &LocalhostIP = *LocalhostIPOrErr;
 
   Status status;
-  TCPSocket listen_socket(true, false);
+  TCPSocket listen_socket(true);
   status = listen_socket.Listen(LocalhostIP + ":0", 5);
   if (status.Fail())
     return status.ToError();
@@ -117,7 +121,10 @@ TestClient::launchCustom(StringRef Log, bool disable_stdio,
     return status.ToError();
 
   Socket *accept_socket;
-  listen_socket.Accept(accept_socket);
+  if (llvm::Error E =
+          listen_socket.Accept(2 * GetDefaultTimeout(), accept_socket)
+              .takeError())
+    return E;
   auto Conn = std::make_unique<ConnectionFileDescriptor>(accept_socket);
   auto Client = std::unique_ptr<TestClient>(new TestClient(std::move(Conn)));
 
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index b1de31fac72e9..3e7b1ceeee875 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -151,6 +151,13 @@ if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
   endif()
 endif()
 
+if ("libc" IN_LIST LLVM_ENABLE_PROJECTS)
+  message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated now, and will "
+    "become a fatal error in the LLVM 21 release.  Please use "
+    "-DLLVM_ENABLE_RUNTIMES=libc or see the instructions at "
+    "https://libc.llvm.org/ for building the runtimes.")
+endif()
+
 # Select the runtimes to build
 #
 # As we migrate runtimes to using the bootstrapping build, the set of default runtimes
@@ -180,13 +187,8 @@ if(LIBC_GPU_BUILD)
   list(APPEND RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES "libc")
 endif()
 
-set(NEED_LIBC_HDRGEN FALSE)
-if("libc" IN_LIST LLVM_ENABLE_RUNTIMES)
-  set(NEED_LIBC_HDRGEN TRUE)
-endif()
 foreach(_name ${LLVM_RUNTIME_TARGETS})
   if("libc" IN_LIST RUNTIMES_${_name}_LLVM_ENABLE_RUNTIMES)
-    set(NEED_LIBC_HDRGEN TRUE)
     if("${_name}" STREQUAL "amdgcn-amd-amdhsa" OR "${_name}" STREQUAL "nvptx64-nvidia-cuda")
       set(LLVM_LIBC_GPU_BUILD ON)
     endif()
@@ -196,27 +198,11 @@ if("${LIBC_TARGET_TRIPLE}" STREQUAL "amdgcn-amd-amdhsa" OR
    "${LIBC_TARGET_TRIPLE}" STREQUAL "nvptx64-nvidia-cuda")
   set(LLVM_LIBC_GPU_BUILD ON)
 endif()
-if(NEED_LIBC_HDRGEN)
-  # To build the libc runtime, we need to be able to build few libc build
-  # tools from the "libc" project. So, we add it to the list of enabled
-  # projects.
-  if (NOT "libc" IN_LIST LLVM_ENABLE_PROJECTS)
-    message(STATUS "Enabling libc project to build libc build tools")
-    list(APPEND LLVM_ENABLE_PROJECTS "libc")
-  endif()
+if (NOT "libc" IN_LIST LLVM_ENABLE_PROJECTS AND LLVM_LIBC_GPU_BUILD)
+  message(STATUS "Enabling libc project to build libc testing tools")
+  list(APPEND LLVM_ENABLE_PROJECTS "libc")
 endif()
 
-foreach(proj IN LISTS LLVM_ENABLE_RUNTIMES)
-  if("${proj}" IN_LIST LLVM_ENABLE_PROJECTS)
-    # The 'libc' project bootstraps a few executables via the project build and
-    # should not emit an error currently.
-    if(NOT (NEED_LIBC_HDRGEN AND "${proj}" STREQUAL "libc"))
-      message(FATAL_ERROR "Runtime project \"${proj}\" found in LLVM_ENABLE_PROJECTS and LLVM_ENABLE_RUNTIMES. It must only appear in one of them and that one should almost always be LLVM_ENABLE_RUNTIMES.")
-    endif()
-  endif()
-endforeach()
-unset(NEED_LIBC_HDRGEN)
-
 # LLVM_ENABLE_PROJECTS_USED is `ON` if the user has ever used the
 # `LLVM_ENABLE_PROJECTS` CMake cache variable.  This exists for
 # several reasons:
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index 9744aa1aa43be..8da9b150c13d9 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -95,15 +95,10 @@ chandlerc@gmail.com, chandlerc@google.com (email), [chandlerc](https://github.co
 
 ### Instrumentation and sanitizers
 
-#### AddressSanitizer, ThreadSanitizer
+#### Sanitizers not covered by someone else
 
-Kostya Serebryany \
-kcc@google.com (email), [kcc](https://github.com/kcc) (GitHub)
-
-#### MemorySanitizer
-
-Evgeniy Stepanov \
-eugenis@google.com (email), [eugenis](https://github.com/eugenis) (GitHub)
+Vitaly Buka \
+vitalybuka@google.com (email), [vitalybuka](https://github.com/vitalybuka) (GitHub)
 
 #### NumericalStabilitySanitizer
 
@@ -424,40 +419,31 @@ gkistanova@gmail.com (email), [gkistanova](https://github.com/gkistanova) (GitHu
 Some subprojects maintain their own list of per-component maintainers.
 Others only have a lead maintainer listed here.
 
-[Bolt maintainers](https://github.com/llvm/llvm-project/blob/main/bolt/CODE_OWNERS.TXT)
+[Bolt maintainers](https://github.com/llvm/llvm-project/blob/main/bolt/Maintainers.txt)
 
 [Clang maintainers](https://github.com/llvm/llvm-project/blob/main/clang/Maintainers.rst)
 
-[Clang-tools-extra maintainers](https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/CODE_OWNERS.TXT)
+[Clang-tools-extra maintainers](https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/Maintainers.txt)
 
-[Compiler-rt maintainers](https://github.com/llvm/llvm-project/blob/main/compiler-rt/CODE_OWNERS.TXT)
+[Compiler-rt maintainers](https://github.com/llvm/llvm-project/blob/main/compiler-rt/Maintainers.md)
 
 [Flang maintainers](https://github.com/llvm/llvm-project/blob/main/flang/Maintainers.txt)
 
-[LLD maintainers](https://github.com/llvm/llvm-project/blob/main/lld/CODE_OWNERS.TXT)
+[libc++ maintainers](https://github.com/llvm/llvm-project/blob/main/libcxx/Maintainers.md)
+
+[LLD maintainers](https://github.com/llvm/llvm-project/blob/main/lld/Maintainers.md)
 
 [LLDB maintainers](https://github.com/llvm/llvm-project/blob/main/lldb/Maintainers.rst)
 
-#### libc++
+[LLVM OpenMP Library maintainers](https://github.com/llvm/llvm-project/blob/main/openmp/Maintainers.md)
 
-Louis Dionne \
-ldionne.2@gmail.com (email), [ldionne](https://github.com/ldionne) (GitHub)
+[Polly maintainers](https://github.com/llvm/llvm-project/blob/main/polly/Maintainers.md)
 
 #### libclc
 
 Tom Stellard \
 tstellar@redhat.com (email), [tstellar](https://github.com/tstellar) (GitHub)
 
-#### OpenMP (runtime library)
-
-Andrey Churbanov \
-andrey.churbanov@intel.com (email), [AndreyChurbanov](https://github.com/AndreyChurbanov) (GitHub)
-
-#### Polly
-
-Tobias Grosser \
-tobias@grosser.es (email), [tobiasgrosser](https://github.com/tobiasgrosser) (GitHub)
-
 ## Inactive Maintainers
 
 The following people have graciously spent time performing maintainer
@@ -476,6 +462,8 @@ Evan Cheng (evan.cheng@apple.com) -- Parts of code generator not covered by some
 Renato Golin (rengolin@systemcall.eu, [rengolin](https://github.com/rengolin)) -- ARM backend \
 Anton Korobeynikov (anton@korobeynikov.info, [asl](https://github.com/asl)) -- ARM EABI \
 Hans Wennborg (hans@chromium.org, [zmodem](https://github.com/zmodem)) -- Release management \
+Kostya Serebryany ([kcc](https://github.com/kcc)) -- Sanitizers \
+Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers
 
 ### Former maintainers of removed components
 
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 9f5e4344b898a..006dfb6de3a19 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -2618,13 +2618,16 @@ function(get_host_tool_path tool_name setting_name exe_var_name target_var_name)
     set(target_name "")
   elseif(LLVM_USE_HOST_TOOLS)
     get_native_tool_path(${tool_name} exe_name)
-    set(target_name ${exe_name})
+    set(target_name host_${tool_name})
   else()
     set(exe_name $<TARGET_FILE:${tool_name}>)
     set(target_name ${tool_name})
   endif()
-  set(${exe_var_name} "${exe_name}" CACHE STRING "")
-  set(${target_var_name} "${target_name}" CACHE STRING "")
+  # Force setting the cache variable because they are only used for being
+  # global in scope. Using regular variables would require careful audit of the
+  # code with several parent scope set commands.
+  set(${exe_var_name} "${exe_name}" CACHE STRING "" FORCE)
+  set(${target_var_name} "${target_name}" CACHE STRING "" FORCE)
 endfunction()
 
 function(setup_host_tool tool_name setting_name exe_var_name target_var_name)
@@ -2632,8 +2635,8 @@ function(setup_host_tool tool_name setting_name exe_var_name target_var_name)
   # Set up a native tool build if necessary
   if(LLVM_USE_HOST_TOOLS AND NOT ${setting_name})
     build_native_tool(${tool_name} exe_name DEPENDS ${tool_name})
-    add_custom_target(${target_var_name} DEPENDS ${exe_name})
+    add_custom_target(${${target_var_name}} DEPENDS ${exe_name})
     get_subproject_title(subproject_title)
-    set_target_properties(${target_var_name} PROPERTIES FOLDER "${subproject_title}/Native")
+    set_target_properties(${${target_var_name}} PROPERTIES FOLDER "${subproject_title}/Native")
   endif()
 endfunction()
diff --git a/llvm/cmake/modules/CheckCompilerVersion.cmake b/llvm/cmake/modules/CheckCompilerVersion.cmake
index 4b3a0eb9a4433..c550df7b08c84 100644
--- a/llvm/cmake/modules/CheckCompilerVersion.cmake
+++ b/llvm/cmake/modules/CheckCompilerVersion.cmake
@@ -12,9 +12,9 @@ set(APPLECLANG_MIN 10.0)
 set(APPLECLANG_SOFT_ERROR 10.0)
 
 # https://en.wikipedia.org/wiki/Microsoft_Visual_C#Internal_version_numbering
-# _MSC_VER == 1927 MSVC++ 14.27 Visual Studio 2019 Version 16.7
-set(MSVC_MIN 19.27)
-set(MSVC_SOFT_ERROR 19.27)
+# _MSC_VER == 1928 MSVC++ 19.27 Visual Studio 2019 Version 16.8 + 16.9
+set(MSVC_MIN 19.28)
+set(MSVC_SOFT_ERROR 19.28)
 
 set(LIBSTDCXX_MIN 7)
 set(LIBSTDCXX_SOFT_ERROR 7)
diff --git a/llvm/cmake/modules/CrossCompile.cmake b/llvm/cmake/modules/CrossCompile.cmake
index e36a71f522d82..c2cc84bec1521 100644
--- a/llvm/cmake/modules/CrossCompile.cmake
+++ b/llvm/cmake/modules/CrossCompile.cmake
@@ -69,14 +69,8 @@ function(llvm_create_cross_target project_name target_name toolchain buildtype)
          "-DLLVM_EXTERNAL_${name}_SOURCE_DIR=${LLVM_EXTERNAL_${name}_SOURCE_DIR}")
   endforeach()
 
-  if("libc" IN_LIST LLVM_ENABLE_PROJECTS AND NOT LIBC_HDRGEN_EXE)
-    set(libc_flags -DLLVM_LIBC_FULL_BUILD=ON -DLIBC_HDRGEN_ONLY=ON)
-    if(MSVC)
-      # Due to some issues mentioned in llvm/projects/CMakeLists.txt, libc build is disabled by
-      # default in the cross target when building with MSVC compatible compilers on Windows. Add
-      # LLVM_FORCE_BUILD_RUNTIME to bypass this issue and force its building on Windows.
-      list(APPEND libc_flags -DLLVM_FORCE_BUILD_RUNTIME=ON)
-    endif()
+  if(LLVM_LIBC_GPU_BUILD)
+    set(libc_flags -DLLVM_LIBC_GPU_BUILD=ON)
   endif()
 
   add_custom_command(OUTPUT ${${project_name}_${target_name}_BUILD}/CMakeCache.txt
diff --git a/llvm/cmake/modules/GetHostTriple.cmake b/llvm/cmake/modules/GetHostTriple.cmake
index 1be13bc01ab9b..e58d5b1ef14d4 100644
--- a/llvm/cmake/modules/GetHostTriple.cmake
+++ b/llvm/cmake/modules/GetHostTriple.cmake
@@ -2,7 +2,7 @@
 # Invokes config.guess
 
 function( get_host_triple var )
-  if( MSVC )
+  if( MSVC OR (CMAKE_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_C_COMPILER_ID MATCHES "Clang" AND NOT MINGW AND NOT MSYS))
     if( CMAKE_C_COMPILER_ARCHITECTURE_ID MATCHES "ARM64.*" )
       set( value "aarch64-pc-windows-msvc" )
     elseif( CMAKE_C_COMPILER_ARCHITECTURE_ID MATCHES "ARM.*" )
@@ -41,7 +41,7 @@ function( get_host_triple var )
     else()
       set( value "powerpc-ibm-aix" )
     endif()
-  else( MSVC )
+  else()
     if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows AND NOT MSYS)
       message(WARNING "unable to determine host target triple")
     else()
@@ -55,6 +55,6 @@ function( get_host_triple var )
       endif( NOT TT_RV EQUAL 0 )
       set( value ${TT_OUT} )
     endif()
-  endif( MSVC )
+  endif()
   set( ${var} ${value} PARENT_SCOPE )
 endfunction( get_host_triple var )
diff --git a/llvm/cmake/platforms/WinMsvc.cmake b/llvm/cmake/platforms/WinMsvc.cmake
index 00746c0826929..dfaac5048fa7a 100644
--- a/llvm/cmake/platforms/WinMsvc.cmake
+++ b/llvm/cmake/platforms/WinMsvc.cmake
@@ -277,7 +277,7 @@ set(CROSS_TOOLCHAIN_FLAGS_NATIVE "${_CTF_NATIVE_DEFAULT}" CACHE STRING "")
 set(COMPILE_FLAGS
     -D_CRT_SECURE_NO_WARNINGS
     --target=${TRIPLE_ARCH}-windows-msvc
-    -fms-compatibility-version=19.27
+    -fms-compatibility-version=19.28
     -vctoolsversion ${MSVC_VER}
     -winsdkversion ${WINSDK_VER}
     -winsysroot ${LLVM_WINSYSROOT})
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 411a1209ef947..5c6034753eb4a 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -586,7 +586,7 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor
      ``gfx9-4-generic``   ``amdgcn``     - ``gfx940``      - xnack            - Absolute flat   FP8 and BF8 instructions,
                                          - ``gfx941``      - sramecc            scratch         FP8 and BF8 conversion instructions,
                                          - ``gfx942``                                           as well as instructions with XF32 format support
-                                                                                                are not available.
+                                         - ``gfx950``                                           are not available.
 
 
      ``gfx10-1-generic``  ``amdgcn``     - ``gfx1010``     - xnack            - Absolute flat   - The following instructions are
@@ -1376,6 +1376,8 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
 
                                                    0. Interleave DS and MFMA instructions for small GEMM kernels.
                                                    1. Interleave DS and MFMA instructions for single wave small GEMM kernels.
+                                                   2. Interleave TRANS and MFMA instructions, as well as their VALU and DS predecessors, for attention kernels.
+                                                   3. Interleave TRANS and MFMA instructions, with no predecessor interleaving, for attention kernels.
 
                                                    Only one iglp_opt intrinsic may be used in a scheduling region. The iglp_opt intrinsic
                                                    cannot be combined with sched_barrier or sched_group_barrier.
@@ -5754,9 +5756,6 @@ SGPR register initial state is defined in
      then       Flat Scratch Init          2      See
                 (enable_sgpr_flat_scratch         :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`.
                 _init)
-     then       Preloaded Kernargs         N/A    See
-                (kernarg_preload_spec             :ref:`amdgpu-amdhsa-kernarg-preload`.
-                _length)
      then       Private Segment Size       1      The 32-bit byte size of a
                 (enable_sgpr_private              single work-item's memory
                 _segment_size)                    allocation. This is the
@@ -5777,6 +5776,9 @@ SGPR register initial state is defined in
                                                   may be needed for GFX9-GFX11 which
                                                   changes the meaning of the
                                                   Flat Scratch Init value.
+     then       Preloaded Kernargs         N/A    See
+                (kernarg_preload_spec             :ref:`amdgpu-amdhsa-kernarg-preload`.
+                _length)
      then       Work-Group Id X            1      32-bit work-group id in X
                 (enable_sgpr_workgroup_id         dimension of grid for
                 _X)                               wavefront.
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 6e90efe0f1119..d6d6ecf759cf3 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -177,7 +177,7 @@ the file. The standard header looks like this:
 
 .. code-block:: c++
 
-  //===-- llvm/Instruction.h - Instruction class definition -------*- C++ -*-===//
+  //===----------------------------------------------------------------------===//
   //
   // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   // See https://llvm.org/LICENSE.txt for license information.
@@ -191,17 +191,7 @@ the file. The standard header looks like this:
   ///
   //===----------------------------------------------------------------------===//
 
-A few things to note about this particular format: The "``-*- C++ -*-``" string
-on the first line is there to tell Emacs that the source file is a C++ file, not
-a C file (Emacs assumes ``.h`` files are C files by default).
-
-.. note::
-
-    This tag is not necessary in ``.cpp`` files.  The name of the file is also
-    on the first line, along with a very short description of the purpose of the
-    file.
-
-The next section in the file is a concise note that defines the license that the
+The first section in the file is a concise note that defines the license that the
 file is released under.  This makes it perfectly clear what terms the source
 code can be distributed under and should not be modified in any way.
 
diff --git a/llvm/docs/ExceptionHandling.rst b/llvm/docs/ExceptionHandling.rst
index c2afb25c6ccd4..bb72e5a71a77b 100644
--- a/llvm/docs/ExceptionHandling.rst
+++ b/llvm/docs/ExceptionHandling.rst
@@ -374,10 +374,12 @@ overall functioning of this intrinsic is compatible with the GCC
 to interoperate.
 
 The single parameter is a pointer to a five word buffer in which the calling
-context is saved. The front end places the frame pointer in the first word, and
-the target implementation of this intrinsic should place the destination address
-for a `llvm.eh.sjlj.longjmp`_ in the second word. The following three words are
-available for use in a target-specific manner.
+context is saved. The format and contents of the buffer are target-specific.
+On certain targets (ARM, PowerPC, VE, X86), the front end places the 
+frame pointer in the first word and the stack pointer in the third word, 
+while the target implementation of this intrinsic fills in the remaining 
+words.  On other targets (SystemZ), saving the calling context to the buffer 
+is left completely to the target implementation. 
 
 .. _llvm.eh.sjlj.longjmp:
 
diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
index 1f4f5d577efda..edb3b5d9c2ad5 100644
--- a/llvm/docs/GettingInvolved.rst
+++ b/llvm/docs/GettingInvolved.rst
@@ -311,11 +311,6 @@ The :doc:`CodeOfConduct` applies to all office hours.
     - Monthly, last Monday of the month at 18:00 London time (typically 9am PT), for 30 minutes.
     - `Video Call <https://meet.grosser.science/LLVMOfficeHours>`__
     - English, German, Spanish, French
-  * - Anastasia Stulova
-    - Clang internals for C/C++ language extensions and dialects, OpenCL, GPU, SPIR-V, how to contribute, women in compilers.
-    - Monthly, 1st Tuesday of the month at 17:00 BST - London time (9:00am PT except for 2 weeks in spring), 30 mins slot.
-    - `GoogleMeet <https://meet.google.com/kdy-fdbv-nuk>`__
-    - English, Russian, German (not fluently)
   * - Alexey Bader
     - SYCL compiler, offload tools, OpenCL and SPIR-V, how to contribute.
     - Monthly, 2nd Monday of the month at 9:30am PT, for 30 minutes.
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 8ef1f85d6b6fd..7f4b62f63957b 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -356,7 +356,7 @@ popular host toolchains for specific minimum versions in our build systems:
 * Clang 5.0
 * Apple Clang 10.0
 * GCC 7.4
-* Visual Studio 2019 16.7
+* Visual Studio 2019 16.8
 
 Anything older than these toolchains *may* work, but will require forcing the
 build system with a special option and is not really a supported host platform.
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 076dc7fa93e56..8b76a407c34f8 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -474,6 +474,16 @@ undefined.
   %2:_(s33) = G_CTLZ_ZERO_UNDEF %1
   %2:_(s33) = G_CTTZ_ZERO_UNDEF %1
 
+G_ABDS, G_ABDU
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Compute the absolute difference (signed and unsigned), e.g. abs(x-y).
+
+.. code-block:: none
+
+  %0:_(s33) = G_ABDS %2, %3
+  %1:_(s33) = G_ABDU %4, %5
+
 Floating Point Operations
 -------------------------
 
@@ -1124,3 +1134,6 @@ G_CONSTANT_FOLD_BARRIER
 This operation is used as an opaque barrier to prevent constant folding. Combines
 and other transformations should not look through this. These have no other
 semantics and can be safely eliminated if a target chooses.
+
+
+Unlisted: G_STACKSAVE, G_STACKRESTORE, G_FSHL, G_FSHR, G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT, G_SDIVFIX, G_UDIVFIX, G_SDIVFIXSAT, G_UDIVFIXSAT, G_FPOWI, G_FEXP10, G_FLDEXP, G_FFREXP, G_GET_FPENV, G_SET_FPENV, G_RESET_FPENV, G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE, G_INTRINSIC_FPTRUNC_ROUND, G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT, G_INTRINSIC_ROUNDEVEN, G_READCYCLECOUNTER, G_READSTEADYCOUNTER, G_PREFETCH, G_READ_REGISTER, G_WRITE_REGISTER, G_STRICT_FADD, G_STRICT_FSUB, G_STRICT_FMUL, G_STRICT_FDIV, G_STRICT_FREM, G_STRICT_FMA, G_STRICT_FSQRT, G_STRICT_FLDEXP, G_ASSERT_ALIGN
diff --git a/llvm/docs/GlobalISel/Legalizer.rst b/llvm/docs/GlobalISel/Legalizer.rst
index 1ff7b304b3a01..0e256ac7781cc 100644
--- a/llvm/docs/GlobalISel/Legalizer.rst
+++ b/llvm/docs/GlobalISel/Legalizer.rst
@@ -338,8 +338,8 @@ G_BUILD_VECTOR_TRUNC, G_CONCAT_VECTORS, G_UNMERGE_VALUES, G_PTRTOINT, and
 G_INTTOPTR have already been noted above. In addition to those, the following
 operations have requirements:
 
-* At least one G_IMPLICIT_DEF must be legal. This is usually trivial as it
-  requires no code to be selected.
+* For every type that can be produced by any instruction, G_IMPLICIT_DEF must be
+  legal.
 * G_PHI must be legal for all types in the producer and consumer typesets. This
   is usually trivial as it requires no code to be selected.
 * At least one G_FRAME_INDEX must be legal
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 91cb56fac9594..76139eb7a2d2c 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3732,10 +3732,10 @@ Fast-Math Flags
 
 LLVM IR floating-point operations (:ref:`fneg <i_fneg>`, :ref:`fadd <i_fadd>`,
 :ref:`fsub <i_fsub>`, :ref:`fmul <i_fmul>`, :ref:`fdiv <i_fdiv>`,
-:ref:`frem <i_frem>`, :ref:`fcmp <i_fcmp>`), and :ref:`phi <i_phi>`,
-:ref:`select <i_select>`, or :ref:`call <i_call>` instructions that return
-floating-point types may use the following flags to enable otherwise unsafe
-floating-point transformations.
+:ref:`frem <i_frem>`, :ref:`fcmp <i_fcmp>`, :ref:`fptrunc <i_fptrunc>`,
+:ref:`fpext <i_fpext>`), and :ref:`phi <i_phi>`, :ref:`select <i_select>`, or
+:ref:`call <i_call>` instructions that return floating-point types may use the
+following flags to enable otherwise unsafe floating-point transformations.
 
 ``fast``
    This flag is a shorthand for specifying all fast-math flags at once, and
@@ -10827,9 +10827,9 @@ Example:
 
 .. code-block:: llvm
 
-      %agg1 = insertvalue {i32, float} undef, i32 1, 0              ; yields {i32 1, float undef}
-      %agg2 = insertvalue {i32, float} %agg1, float %val, 1         ; yields {i32 1, float %val}
-      %agg3 = insertvalue {i32, {float}} undef, float %val, 1, 0    ; yields {i32 undef, {float %val}}
+      %agg1 = insertvalue {i32, float} poison, i32 1, 0              ; yields {i32 1, float poison}
+      %agg2 = insertvalue {i32, float} %agg1, float %val, 1          ; yields {i32 1, float %val}
+      %agg3 = insertvalue {i32, {float}} poison, float %val, 1, 0    ; yields {i32 poison, {float %val}}
 
 .. _memoryops:
 
@@ -11833,6 +11833,8 @@ Example:
       %Y = sext i1 true to i32             ; yields i32:-1
       %Z = sext <2 x i16> <i16 8, i16 7> to <2 x i32> ; yields <i32 8, i32 7>
 
+.. _i_fptrunc:
+
 '``fptrunc .. to``' Instruction
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -11841,7 +11843,7 @@ Syntax:
 
 ::
 
-      <result> = fptrunc <ty> <value> to <ty2>             ; yields ty2
+      <result> = fptrunc [fast-math flags]* <ty> <value> to <ty2> ; yields ty2
 
 Overview:
 """""""""
@@ -11873,6 +11875,10 @@ the low order bits leads to an all-0 payload, this cannot be represented as a
 signaling NaN (it would represent an infinity instead), so in that case
 "Unchanged NaN propagation" is not possible.
 
+This instruction can also take any number of :ref:`fast-math
+flags <fastmath>`, which are optimization hints to enable otherwise
+unsafe floating-point optimizations.
+
 Example:
 """"""""
 
@@ -11881,6 +11887,8 @@ Example:
       %X = fptrunc double 16777217.0 to float    ; yields float:16777216.0
       %Y = fptrunc double 1.0E+300 to half       ; yields half:+infinity
 
+.. _i_fpext:
+
 '``fpext .. to``' Instruction
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -11889,7 +11897,7 @@ Syntax:
 
 ::
 
-      <result> = fpext <ty> <value> to <ty2>             ; yields ty2
+      <result> = fpext [fast-math flags]* <ty> <value> to <ty2> ; yields ty2
 
 Overview:
 """""""""
@@ -11918,6 +11926,10 @@ NaN payload is propagated from the input ("Quieting NaN propagation" or
 "Unchanged NaN propagation" cases), then it is copied to the high order bits of
 the resulting payload, and the remaining low order bits are zero.
 
+This instruction can also take any number of :ref:`fast-math
+flags <fastmath>`, which are optimization hints to enable otherwise
+unsafe floating-point optimizations.
+
 Example:
 """"""""
 
@@ -20876,7 +20888,7 @@ Example:
 
       ;;; Expansion.
       ;; Lanes at and above %pivot are taken from %on_false
-      %atfirst = insertelement <4 x i32> undef, i32 %pivot, i32 0
+      %atfirst = insertelement <4 x i32> poison, i32 %pivot, i32 0
       %splat = shufflevector <4 x i32> %atfirst, <4 x i32> poison, <4 x i32> zeroinitializer
       %pivotmask = icmp ult <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> %splat
       %mergemask = and <4 x i1> %cond, <4 x i1> %pivotmask
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index bac267591e015..3978332b1149a 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -426,6 +426,15 @@ The current vendor extensions supported are:
 ``Xwchc``
   LLVM implements `the custom compressed opcodes present in some QingKe cores` by WCH / Nanjing Qinheng Microelectronics. The vendor refers to these opcodes by the name "XW".
 
+``experimental-Xqcia``
+  LLVM implements `version 0.2 of the Qualcomm uC Arithmetic extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
+
+``experimental-Xqcicsr``
+  LLVM implements `version 0.2 of the Qualcomm uC CSR extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
+
+``experimental-Xqcisls``
+  LLVM implements `version 0.2 of the Qualcomm uC Scaled Load Store extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
+
 Experimental C Intrinsics
 =========================
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index dd68d5296376f..a5805e050bfdb 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -105,12 +105,16 @@ Changes to the LLVM IR
 
 * Operand bundle values can now be metadata strings.
 
+* Fast math flags are now permitted on `fptrunc` and `fpext`.
+
 Changes to LLVM infrastructure
 ------------------------------
 
 Changes to building LLVM
 ------------------------
 
+* Raised the minimum MSVC version to Visual Studio 2019 16.8.
+
 Changes to TableGen
 -------------------
 
@@ -211,7 +215,12 @@ Changes to the RISC-V Backend
 * `f` and `cf` inline assembly constraints, when using F-/D-/H-in-X extensions,
   will use the relevant GPR rather than FPR. This makes inline assembly portable
   between e.g. F and Zfinx code.
-
+* Adds experimental assembler support for the Qualcomm uC 'Xqcicsr` (CSR)
+  extension.
+* Adds experimental assembler support for the Qualcomm uC 'Xqcisls` (Scaled Load Store)
+  extension.
+* Adds experimental assembler support for the Qualcomm uC 'Xqcia` (Arithmetic)
+  extension.
 
 Changes to the WebAssembly Backend
 ----------------------------------
@@ -221,9 +230,15 @@ and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
 and [Non-trapping float-to-int Conversions] language features, which are
 [widely implemented in engines].
 
+A new Lime1 target CPU is added, -mcpu=lime1. This CPU follows the definition of
+the Lime1 CPU [here], and enables -mmultivalue, -mmutable-globals,
+-mcall-indirect-overlong, -msign-ext, -mbulk-memory-opt, -mnontrapping-fptoint,
+and -mextended-const.
+
 [Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
 [Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
 [widely implemented in engines]: https://webassembly.org/features/
+[here]: https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1
 
 Changes to the Windows Target
 -----------------------------
@@ -254,6 +269,13 @@ Changes to the X86 Backend
 
 * Supported ``-mcpu=diamondrapids``
 
+* Supported emitting relocation types for x86-64 target:
+  * `R_X86_64_CODE_4_GOTPCRELX`
+  * `R_X86_64_CODE_4_GOTTPOFF`
+  * `R_X86_64_CODE_4_GOTPC32_TLSDESC`
+  * `R_X86_64_CODE_6_GOTTPOFF`
+
+
 Changes to the OCaml bindings
 -----------------------------
 
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index ffb04dca00fc5..8f7ac71f8026b 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -141,16 +141,18 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
 
    * - Extension Name
      - Description
+   * - ``SPV_EXT_arithmetic_fence``
+     - Adds an instruction that prevents fast-math optimizations between its argument and the expression that contains it.
+   * - ``SPV_EXT_demote_to_helper_invocation``
+     - Adds an instruction that demotes a fragment shader invocation to a helper invocation.
+   * - ``SPV_EXT_optnone``
+     - Adds OptNoneEXT value for Function Control mask that indicates a request to not optimize the function.
    * - ``SPV_EXT_shader_atomic_float16_add``
      - Extends the SPV_EXT_shader_atomic_float_add extension to support atomically adding to 16-bit floating-point numbers in memory.
    * - ``SPV_EXT_shader_atomic_float_add``
      - Adds atomic add instruction on floating-point numbers.
    * - ``SPV_EXT_shader_atomic_float_min_max``
      - Adds atomic min and max instruction on floating-point numbers.
-   * - ``SPV_EXT_arithmetic_fence``
-     - Adds an instruction that prevents fast-math optimizations between its argument and the expression that contains it.
-   * - ``SPV_EXT_demote_to_helper_invocation``
-     - Adds an instruction that demotes a fragment shader invocation to a helper invocation.
    * - ``SPV_INTEL_arbitrary_precision_integers``
      - Allows generating arbitrary width integer types.
    * - ``SPV_INTEL_bfloat16_conversion``
@@ -165,6 +167,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
      - Adds decorations that can be applied to global (module scope) variables.
    * - ``SPV_INTEL_global_variable_fpga_decorations``
      - Adds decorations that can be applied to global (module scope) variables to help code generation for FPGA devices.
+   * - ``SPV_INTEL_media_block_io``
+     - Adds additional subgroup block read and write functionality that allow applications to flexibly specify the width and height of the block to read from or write to a 2D image.
    * - ``SPV_INTEL_optnone``
      - Adds OptNoneINTEL value for Function Control mask that indicates a request to not optimize the function.
    * - ``SPV_INTEL_split_barrier``
@@ -175,6 +179,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
      - Introduces two new storage classes that are subclasses of the CrossWorkgroup storage class that provides additional information that can enable optimization.
    * - ``SPV_INTEL_variable_length_array``
      - Allows to allocate local arrays whose number of elements is unknown at compile time.
+   * - ``SPV_INTEL_joint_matrix``
+     - Adds few matrix capabilities on top of SPV_KHR_cooperative_matrix extension, such as matrix prefetch, get element coordinate and checked load/store/construct instructions, tensor float 32 and bfloat type interpretations for multuply-add instruction.
    * - ``SPV_KHR_bit_instructions``
      - Enables bit instructions to be used by SPIR-V modules without requiring the Shader capability.
    * - ``SPV_KHR_expect_assume``
diff --git a/llvm/docs/Vectorizers.rst b/llvm/docs/Vectorizers.rst
index c2beb1669c4e3..a4462e53edda0 100644
--- a/llvm/docs/Vectorizers.rst
+++ b/llvm/docs/Vectorizers.rst
@@ -345,6 +345,16 @@ instruction is available.
       f[i] = floorf(f[i]);
   }
 
+Many of these math functions are only vectorizable if the file has been built
+with a specified target vector library that provides a vector implemention
+of that math function. Using clang, this is handled by the "-fveclib" command
+line option with one of the following vector libraries:
+"accelerate,libmvec,massv,svml,sleef,darwin_libsystem_m,armpl,amdlibm"
+
+.. code-block:: console
+
+   $ clang ... -fno-math-errno -fveclib=libmvec file.c
+
 Partial unrolling during vectorization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/XRay.rst b/llvm/docs/XRay.rst
index acfb83c5374ed..ceb5b88ab9108 100644
--- a/llvm/docs/XRay.rst
+++ b/llvm/docs/XRay.rst
@@ -157,7 +157,8 @@ Also by default the filename of the XRay trace is ``xray-log.XXXXXX`` where the
 ``XXXXXX`` part is randomly generated.
 
 These options can be controlled through the ``XRAY_OPTIONS`` environment
-variable, where we list down the options and their defaults below.
+variable during program run-time, where we list down the options and their
+defaults below.
 
 +-------------------+-----------------+---------------+------------------------+
 | Option            | Type            | Default       | Description            |
@@ -178,6 +179,31 @@ variable, where we list down the options and their defaults below.
 +-------------------+-----------------+---------------+------------------------+
 
 
+In addition to environment variable, you can also provide your own definition of
+``const char *__xray_default_options(void)`` function, which returns the option
+strings. This method effectively provides default options during program build
+time. For example, you can create an additional source file (e.g. ``xray-opt.c``
+) with the following ``__xray_default_options`` definition:
+
+.. code-block:: c
+
+  __attribute__((xray_never_instrument))
+  const char *__xray_default_options() {
+    return "patch_premain=true,xray_mode=xray-basic";
+  }
+
+And link it with the program you want to instrument:
+
+::
+
+  clang -fxray-instrument prog.c xray-opt.c ...
+
+The instrumented binary will use 'patch_premain=true,xray_mode=xray-basic' by
+default even without setting ``XRAY_OPTIONS``.
+
+Note that you still can override options designated by ``__xray_default_options``
+using ``XRAY_OPTIONS`` during run-time.
+
 If you choose to not use the default logging implementation that comes with the
 XRay runtime and/or control when/how the XRay instrumentation runs, you may use
 the XRay APIs directly for doing so. To do this, you'll need to include the
diff --git a/llvm/examples/OrcV2Examples/LLJITWithObjectLinkingLayerPlugin/CMakeLists.txt b/llvm/examples/OrcV2Examples/LLJITWithObjectLinkingLayerPlugin/CMakeLists.txt
index 54814621a5a5f..0e07e24c96a1f 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithObjectLinkingLayerPlugin/CMakeLists.txt
+++ b/llvm/examples/OrcV2Examples/LLJITWithObjectLinkingLayerPlugin/CMakeLists.txt
@@ -3,6 +3,7 @@ set(LLVM_LINK_COMPONENTS
   IRReader
   JITLink
   OrcJIT
+  OrcShared
   Support
   nativecodegen
   )
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index 4ca928bf4f49e..bf80fa5a06580 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -155,7 +155,41 @@ struct APFloatBase {
     S_IEEEsingle,
     S_IEEEdouble,
     S_IEEEquad,
+    // The IBM double-double semantics. Such a number consists of a pair of
+    // IEEE 64-bit doubles (Hi, Lo), where |Hi| > |Lo|, and if normal,
+    // (double)(Hi + Lo) == Hi. The numeric value it's modeling is Hi + Lo.
+    // Therefore it has two 53-bit mantissa parts that aren't necessarily
+    // adjacent to each other, and two 11-bit exponents.
+    //
+    // Note: we need to make the value different from semBogus as otherwise
+    // an unsafe optimization may collapse both values to a single address,
+    // and we heavily rely on them having distinct addresses.
     S_PPCDoubleDouble,
+    // These are legacy semantics for the fallback, inaccurate implementation
+    // of IBM double-double, if the accurate semPPCDoubleDouble doesn't handle
+    // the operation. It's equivalent to having an IEEE number with consecutive
+    // 106 bits of mantissa and 11 bits of exponent.
+    //
+    // It's not equivalent to IBM double-double. For example, a legit IBM
+    // double-double, 1 + epsilon:
+    //
+    // 1 + epsilon = 1 + (1 >> 1076)
+    //
+    // is not representable by a consecutive 106 bits of mantissa.
+    //
+    // Currently, these semantics are used in the following way:
+    //
+    //   semPPCDoubleDouble -> (IEEEdouble, IEEEdouble) ->
+    //   (64-bit APInt, 64-bit APInt) -> (128-bit APInt) ->
+    //   semPPCDoubleDoubleLegacy -> IEEE operations
+    //
+    // We use bitcastToAPInt() to get the bit representation (in APInt) of the
+    // underlying IEEEdouble, then use the APInt constructor to construct the
+    // legacy IEEE float.
+    //
+    // TODO: Implement all operations in semPPCDoubleDouble, and delete these
+    // semantics.
+    S_PPCDoubleDoubleLegacy,
     // 8-bit floating point number following IEEE-754 conventions with bit
     // layout S1E5M2 as described in https://arxiv.org/abs/2209.05433.
     S_Float8E5M2,
@@ -214,7 +248,7 @@ struct APFloatBase {
     // types, there are no infinity or NaN values. The format is detailed in
     // https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
     S_Float4E2M1FN,
-
+    // TODO: Documentation is missing.
     S_x87DoubleExtended,
     S_MaxSemantics = S_x87DoubleExtended,
   };
@@ -228,6 +262,7 @@ struct APFloatBase {
   static const fltSemantics &IEEEdouble() LLVM_READNONE;
   static const fltSemantics &IEEEquad() LLVM_READNONE;
   static const fltSemantics &PPCDoubleDouble() LLVM_READNONE;
+  static const fltSemantics &PPCDoubleDoubleLegacy() LLVM_READNONE;
   static const fltSemantics &Float8E5M2() LLVM_READNONE;
   static const fltSemantics &Float8E5M2FNUZ() LLVM_READNONE;
   static const fltSemantics &Float8E4M3() LLVM_READNONE;
@@ -688,7 +723,7 @@ class IEEEFloat final {
   APInt convertDoubleAPFloatToAPInt() const;
   APInt convertQuadrupleAPFloatToAPInt() const;
   APInt convertF80LongDoubleAPFloatToAPInt() const;
-  APInt convertPPCDoubleDoubleAPFloatToAPInt() const;
+  APInt convertPPCDoubleDoubleLegacyAPFloatToAPInt() const;
   APInt convertFloat8E5M2APFloatToAPInt() const;
   APInt convertFloat8E5M2FNUZAPFloatToAPInt() const;
   APInt convertFloat8E4M3APFloatToAPInt() const;
@@ -709,7 +744,7 @@ class IEEEFloat final {
   void initFromDoubleAPInt(const APInt &api);
   void initFromQuadrupleAPInt(const APInt &api);
   void initFromF80LongDoubleAPInt(const APInt &api);
-  void initFromPPCDoubleDoubleAPInt(const APInt &api);
+  void initFromPPCDoubleDoubleLegacyAPInt(const APInt &api);
   void initFromFloat8E5M2APInt(const APInt &api);
   void initFromFloat8E5M2FNUZAPInt(const APInt &api);
   void initFromFloat8E4M3APInt(const APInt &api);
diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h
index c87e7cac65a5b..dcb13bd8ba51a 100644
--- a/llvm/include/llvm/ADT/BitmaskEnum.h
+++ b/llvm/include/llvm/ADT/BitmaskEnum.h
@@ -85,9 +85,13 @@
   using ::llvm::BitmaskEnumDetail::operator|;                                  \
   using ::llvm::BitmaskEnumDetail::operator&;                                  \
   using ::llvm::BitmaskEnumDetail::operator^;                                  \
+  using ::llvm::BitmaskEnumDetail::operator<<;                                 \
+  using ::llvm::BitmaskEnumDetail::operator>>;                                 \
   using ::llvm::BitmaskEnumDetail::operator|=;                                 \
   using ::llvm::BitmaskEnumDetail::operator&=;                                 \
   using ::llvm::BitmaskEnumDetail::operator^=;                                 \
+  using ::llvm::BitmaskEnumDetail::operator<<=;                                \
+  using ::llvm::BitmaskEnumDetail::operator>>=;                                \
   /* Force a semicolon at the end of this macro. */                            \
   using ::llvm::BitmaskEnumDetail::any
 
@@ -162,6 +166,16 @@ constexpr E operator^(E LHS, E RHS) {
   return static_cast<E>(Underlying(LHS) ^ Underlying(RHS));
 }
 
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
+constexpr E operator<<(E LHS, E RHS) {
+  return static_cast<E>(Underlying(LHS) << Underlying(RHS));
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
+constexpr E operator>>(E LHS, E RHS) {
+  return static_cast<E>(Underlying(LHS) >> Underlying(RHS));
+}
+
 // |=, &=, and ^= return a reference to LHS, to match the behavior of the
 // operators on builtin types.
 
@@ -183,6 +197,18 @@ E &operator^=(E &LHS, E RHS) {
   return LHS;
 }
 
+template <typename e, typename = std::enable_if_t<is_bitmask_enum<e>::value>>
+e &operator<<=(e &lhs, e rhs) {
+  lhs = lhs << rhs;
+  return lhs;
+}
+
+template <typename e, typename = std::enable_if_t<is_bitmask_enum<e>::value>>
+e &operator>>=(e &lhs, e rhs) {
+  lhs = lhs >> rhs;
+  return lhs;
+}
+
 } // namespace BitmaskEnumDetail
 
 // Enable bitmask enums in namespace ::llvm and all nested namespaces.
diff --git a/llvm/include/llvm/ADT/IntervalMap.h b/llvm/include/llvm/ADT/IntervalMap.h
index 99bf308772233..6b21e6c6d75a8 100644
--- a/llvm/include/llvm/ADT/IntervalMap.h
+++ b/llvm/include/llvm/ADT/IntervalMap.h
@@ -222,7 +222,7 @@ using IdxPair = std::pair<unsigned,unsigned>;
 template <typename T1, typename T2, unsigned N>
 class NodeBase {
 public:
-  enum { Capacity = N };
+  static constexpr unsigned Capacity = N;
 
   T1 first[N];
   T2 second[N];
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 1fc2318342ae7..cf6abc9129b40 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -53,10 +53,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   friend class SmallPtrSetIteratorImpl;
 
 protected:
-  /// SmallArray - Points to a fixed size set of buckets, used in 'small mode'.
-  const void **SmallArray;
-  /// CurArray - This is the current set of buckets.  If equal to SmallArray,
-  /// then the set is in 'small mode'.
+  /// The current set of buckets, in either small or big representation.
   const void **CurArray;
   /// CurArraySize - The allocated size of CurArray, always a power of two.
   unsigned CurArraySize;
@@ -67,16 +64,18 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   unsigned NumNonEmpty;
   /// Number of tombstones in CurArray.
   unsigned NumTombstones;
+  /// Whether the set is in small representation.
+  bool IsSmall;
 
   // Helpers to copy and move construct a SmallPtrSet.
   SmallPtrSetImplBase(const void **SmallStorage,
                       const SmallPtrSetImplBase &that);
   SmallPtrSetImplBase(const void **SmallStorage, unsigned SmallSize,
-                      SmallPtrSetImplBase &&that);
+                      const void **RHSSmallStorage, SmallPtrSetImplBase &&that);
 
   explicit SmallPtrSetImplBase(const void **SmallStorage, unsigned SmallSize)
-      : SmallArray(SmallStorage), CurArray(SmallStorage),
-        CurArraySize(SmallSize), NumNonEmpty(0), NumTombstones(0) {
+      : CurArray(SmallStorage), CurArraySize(SmallSize), NumNonEmpty(0),
+        NumTombstones(0), IsSmall(true) {
     assert(SmallSize && (SmallSize & (SmallSize-1)) == 0 &&
            "Initial size must be a power of two!");
   }
@@ -150,7 +149,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   std::pair<const void *const *, bool> insert_imp(const void *Ptr) {
     if (isSmall()) {
       // Check to see if it is already in the set.
-      for (const void **APtr = SmallArray, **E = SmallArray + NumNonEmpty;
+      for (const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
            APtr != E; ++APtr) {
         const void *Value = *APtr;
         if (Value == Ptr)
@@ -159,9 +158,9 @@ class SmallPtrSetImplBase : public DebugEpochBase {
 
       // Nope, there isn't.  If we stay small, just 'pushback' now.
       if (NumNonEmpty < CurArraySize) {
-        SmallArray[NumNonEmpty++] = Ptr;
+        CurArray[NumNonEmpty++] = Ptr;
         incrementEpoch();
-        return std::make_pair(SmallArray + (NumNonEmpty - 1), true);
+        return std::make_pair(CurArray + (NumNonEmpty - 1), true);
       }
       // Otherwise, hit the big set case, which will call grow.
     }
@@ -174,10 +173,10 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   /// in.
   bool erase_imp(const void * Ptr) {
     if (isSmall()) {
-      for (const void **APtr = SmallArray, **E = SmallArray + NumNonEmpty;
+      for (const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
            APtr != E; ++APtr) {
         if (*APtr == Ptr) {
-          *APtr = SmallArray[--NumNonEmpty];
+          *APtr = CurArray[--NumNonEmpty];
           incrementEpoch();
           return true;
         }
@@ -203,8 +202,9 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   const void *const * find_imp(const void * Ptr) const {
     if (isSmall()) {
       // Linear search for the item.
-      for (const void *const *APtr = SmallArray,
-                      *const *E = SmallArray + NumNonEmpty; APtr != E; ++APtr)
+      for (const void *const *APtr = CurArray, *const *E =
+                                                   CurArray + NumNonEmpty;
+           APtr != E; ++APtr)
         if (*APtr == Ptr)
           return APtr;
       return EndPointer();
@@ -216,7 +216,21 @@ class SmallPtrSetImplBase : public DebugEpochBase {
     return EndPointer();
   }
 
-  bool isSmall() const { return CurArray == SmallArray; }
+  bool contains_imp(const void *Ptr) const {
+    if (isSmall()) {
+      // Linear search for the item.
+      const void *const *APtr = CurArray;
+      const void *const *E = CurArray + NumNonEmpty;
+      for (; APtr != E; ++APtr)
+        if (*APtr == Ptr)
+          return true;
+      return false;
+    }
+
+    return doFind(Ptr) != nullptr;
+  }
+
+  bool isSmall() const { return IsSmall; }
 
 private:
   std::pair<const void *const *, bool> insert_imp_big(const void *Ptr);
@@ -231,16 +245,19 @@ class SmallPtrSetImplBase : public DebugEpochBase {
 protected:
   /// swap - Swaps the elements of two sets.
   /// Note: This method assumes that both sets have the same small size.
-  void swap(SmallPtrSetImplBase &RHS);
+  void swap(const void **SmallStorage, const void **RHSSmallStorage,
+            SmallPtrSetImplBase &RHS);
 
-  void CopyFrom(const SmallPtrSetImplBase &RHS);
-  void MoveFrom(unsigned SmallSize, SmallPtrSetImplBase &&RHS);
+  void copyFrom(const void **SmallStorage, const SmallPtrSetImplBase &RHS);
+  void moveFrom(const void **SmallStorage, unsigned SmallSize,
+                const void **RHSSmallStorage, SmallPtrSetImplBase &&RHS);
 
 private:
-  /// Code shared by MoveFrom() and move constructor.
-  void MoveHelper(unsigned SmallSize, SmallPtrSetImplBase &&RHS);
-  /// Code shared by CopyFrom() and copy constructor.
-  void CopyHelper(const SmallPtrSetImplBase &RHS);
+  /// Code shared by moveFrom() and move constructor.
+  void moveHelper(const void **SmallStorage, unsigned SmallSize,
+                  const void **RHSSmallStorage, SmallPtrSetImplBase &&RHS);
+  /// Code shared by copyFrom() and copy constructor.
+  void copyHelper(const SmallPtrSetImplBase &RHS);
 };
 
 /// SmallPtrSetIteratorImpl - This is the common base class shared between all
@@ -401,7 +418,7 @@ class SmallPtrSetImpl : public SmallPtrSetImplBase {
   bool remove_if(UnaryPredicate P) {
     bool Removed = false;
     if (isSmall()) {
-      const void **APtr = SmallArray, **E = SmallArray + NumNonEmpty;
+      const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
       while (APtr != E) {
         PtrType Ptr = PtrTraits::getFromVoidPointer(const_cast<void *>(*APtr));
         if (P(Ptr)) {
@@ -433,13 +450,13 @@ class SmallPtrSetImpl : public SmallPtrSetImplBase {
 
   /// count - Return 1 if the specified pointer is in the set, 0 otherwise.
   size_type count(ConstPtrType Ptr) const {
-    return find_imp(ConstPtrTraits::getAsVoidPointer(Ptr)) != EndPointer();
+    return contains_imp(ConstPtrTraits::getAsVoidPointer(Ptr));
   }
   iterator find(ConstPtrType Ptr) const {
     return makeIterator(find_imp(ConstPtrTraits::getAsVoidPointer(Ptr)));
   }
   bool contains(ConstPtrType Ptr) const {
-    return find_imp(ConstPtrTraits::getAsVoidPointer(Ptr)) != EndPointer();
+    return contains_imp(ConstPtrTraits::getAsVoidPointer(Ptr));
   }
 
   template <typename IterT>
@@ -526,7 +543,8 @@ class SmallPtrSet : public SmallPtrSetImpl<PtrType> {
   SmallPtrSet() : BaseT(SmallStorage, SmallSizePowTwo) {}
   SmallPtrSet(const SmallPtrSet &that) : BaseT(SmallStorage, that) {}
   SmallPtrSet(SmallPtrSet &&that)
-      : BaseT(SmallStorage, SmallSizePowTwo, std::move(that)) {}
+      : BaseT(SmallStorage, SmallSizePowTwo, that.SmallStorage,
+              std::move(that)) {}
 
   template<typename It>
   SmallPtrSet(It I, It E) : BaseT(SmallStorage, SmallSizePowTwo) {
@@ -541,14 +559,15 @@ class SmallPtrSet : public SmallPtrSetImpl<PtrType> {
   SmallPtrSet<PtrType, SmallSize> &
   operator=(const SmallPtrSet<PtrType, SmallSize> &RHS) {
     if (&RHS != this)
-      this->CopyFrom(RHS);
+      this->copyFrom(SmallStorage, RHS);
     return *this;
   }
 
   SmallPtrSet<PtrType, SmallSize> &
   operator=(SmallPtrSet<PtrType, SmallSize> &&RHS) {
     if (&RHS != this)
-      this->MoveFrom(SmallSizePowTwo, std::move(RHS));
+      this->moveFrom(SmallStorage, SmallSizePowTwo, RHS.SmallStorage,
+                     std::move(RHS));
     return *this;
   }
 
@@ -561,7 +580,7 @@ class SmallPtrSet : public SmallPtrSetImpl<PtrType> {
 
   /// swap - Swaps the elements of two sets.
   void swap(SmallPtrSet<PtrType, SmallSize> &RHS) {
-    SmallPtrSetImplBase::swap(RHS);
+    SmallPtrSetImplBase::swap(SmallStorage, RHS.SmallStorage, RHS);
   }
 };
 
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 60875577561dc..acc580f92b40a 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -875,6 +875,13 @@ bool isIdentifiedObject(const Value *V);
 /// IdentifiedObjects.
 bool isIdentifiedFunctionLocal(const Value *V);
 
+/// Return true if we know V to the base address of the corresponding memory
+/// object.  This implies that any address less than V must be out of bounds
+/// for the underlying object.  Note that just being isIdentifiedObject() is
+/// not enough - For example, a negative offset from a noalias argument or call
+/// can be inbounds w.r.t the actual underlying object.
+bool isBaseOfObject(const Value *V);
+
 /// Returns true if the pointer is one which would have been considered an
 /// escape by isNonEscapingLocalObject.
 bool isEscapeSource(const Value *V);
diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h
index 449852343964c..01eeadb17db9f 100644
--- a/llvm/include/llvm/Analysis/ConstraintSystem.h
+++ b/llvm/include/llvm/Analysis/ConstraintSystem.h
@@ -113,7 +113,9 @@ class ConstraintSystem {
   static SmallVector<int64_t, 8> negate(SmallVector<int64_t, 8> R) {
     // The negated constraint R is obtained by multiplying by -1 and adding 1 to
     // the constant.
-    R[0] += 1;
+    if (AddOverflow(R[0], int64_t(1), R[0]))
+      return {};
+
     return negateOrEqual(R);
   }
 
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 22e0b1bc901a4..ed54b0c077b4a 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -318,6 +318,7 @@ std::optional<int> getInliningCostEstimate(
     CallBase &Call, TargetTransformInfo &CalleeTTI,
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
+    function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
     ProfileSummaryInfo *PSI = nullptr,
     OptimizationRemarkEmitter *ORE = nullptr);
 
@@ -327,6 +328,7 @@ std::optional<InlineCostFeatures> getInliningCostFeatures(
     CallBase &Call, TargetTransformInfo &CalleeTTI,
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
+    function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
     ProfileSummaryInfo *PSI = nullptr,
     OptimizationRemarkEmitter *ORE = nullptr);
 
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 430c3edc2f0dc..d4ae4dcc918cf 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -22,6 +22,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/TargetFolder.h"
+#include "llvm/IR/CmpPredicate.h"
 #include "llvm/IR/IRBuilderFolder.h"
 #include "llvm/IR/Instruction.h"
 
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index cf7d3e044188a..fa291eeef198b 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -44,6 +44,7 @@ class DataLayout;
 class DominatorTree;
 class Function;
 class Instruction;
+class CmpPredicate;
 class LoadInst;
 struct LoopStandardAnalysisResults;
 class Pass;
@@ -152,11 +153,11 @@ Value *simplifyOrInst(Value *LHS, Value *RHS, const SimplifyQuery &Q);
 Value *simplifyXorInst(Value *LHS, Value *RHS, const SimplifyQuery &Q);
 
 /// Given operands for an ICmpInst, fold the result or return null.
-Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
                         const SimplifyQuery &Q);
 
 /// Given operands for an FCmpInst, fold the result or return null.
-Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                         FastMathFlags FMF, const SimplifyQuery &Q);
 
 /// Given operands for a SelectInst, fold the result or return null.
@@ -200,7 +201,7 @@ Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
 //=== Helper functions for higher up the class hierarchy.
 
 /// Given operands for a CmpInst, fold the result or return null.
-Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                        const SimplifyQuery &Q);
 
 /// Given operand for a UnaryOperator, fold the result or return null.
diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
index 055feceefb054..b8e08f4b7842f 100644
--- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h
+++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
@@ -260,23 +260,32 @@ class MemorySSAUpdater {
   MemoryAccess *tryRemoveTrivialPhi(MemoryPhi *Phi, RangeType &Operands);
   void tryRemoveTrivialPhis(ArrayRef<WeakVH> UpdatedPHIs);
   void fixupDefs(const SmallVectorImpl<WeakVH> &);
-  // Clone all uses and defs from BB to NewBB given a 1:1 map of all
-  // instructions and blocks cloned, and a map of MemoryPhi : Definition
-  // (MemoryAccess Phi or Def). VMap maps old instructions to cloned
-  // instructions and old blocks to cloned blocks. MPhiMap, is created in the
-  // caller of this private method, and maps existing MemoryPhis to new
-  // definitions that new MemoryAccesses must point to. These definitions may
-  // not necessarily be MemoryPhis themselves, they may be MemoryDefs. As such,
-  // the map is between MemoryPhis and MemoryAccesses, where the MemoryAccesses
-  // may be MemoryPhis or MemoryDefs and not MemoryUses.
-  // If CloneWasSimplified = true, the clone was exact. Otherwise, assume that
-  // the clone involved simplifications that may have: (1) turned a MemoryUse
-  // into an instruction that MemorySSA has no representation for, or (2) turned
-  // a MemoryDef into a MemoryUse or an instruction that MemorySSA has no
-  // representation for. No other cases are supported.
+  /// Clone all uses and defs from BB to NewBB given a 1:1 map of all
+  /// instructions and blocks cloned, and a map of MemoryPhi : Definition
+  /// (MemoryAccess Phi or Def).
+  ///
+  /// \param VMap Maps old instructions to cloned instructions and old blocks
+  ///        to cloned blocks
+  /// \param MPhiMap, is created in the caller of this private method, and maps
+  ///        existing MemoryPhis to new definitions that new MemoryAccesses
+  ///        must point to. These definitions may not necessarily be MemoryPhis
+  ///        themselves, they may be MemoryDefs. As such, the map is between
+  ///        MemoryPhis and MemoryAccesses, where the MemoryAccesses may be
+  ///        MemoryPhis or MemoryDefs and not MemoryUses.
+  /// \param IsInClonedRegion Determines whether a basic block was cloned.
+  ///        References to accesses outside the cloned region will not be
+  ///        remapped.
+  /// \param CloneWasSimplified If false, the clone was exact. Otherwise,
+  ///        assume that the clone involved simplifications that may have:
+  ///        (1) turned a MemoryUse into an instruction that MemorySSA has no
+  ///        representation for, or (2) turned a MemoryDef into a MemoryUse or
+  ///        an instruction that MemorySSA has no representation for. No other
+  ///        cases are supported.
   void cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB,
                         const ValueToValueMapTy &VMap, PhiToDefMap &MPhiMap,
+                        function_ref<bool(BasicBlock *)> IsInClonedRegion,
                         bool CloneWasSimplified = false);
+
   template <typename Iter>
   void privateUpdateExitBlocksForClonedLoop(ArrayRef<BasicBlock *> ExitBlocks,
                                             Iter ValuesBegin, Iter ValuesEnd,
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 885c5985f9d23..de74524c4b6fe 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -1780,6 +1780,10 @@ class ScalarEvolution {
   /// V.
   const SCEV *getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops);
 
+  /// Returns SCEV for the first operand of a phi if all phi operands have
+  /// identical opcodes and operands.
+  const SCEV *createNodeForPHIWithIdenticalOperands(PHINode *PN);
+
   /// Provide the special handling we need to analyze PHI SCEVs.
   const SCEV *createNodeForPHI(PHINode *PN);
 
@@ -2187,6 +2191,9 @@ class ScalarEvolution {
   bool isGuaranteedToTransferExecutionTo(const Instruction *A,
                                          const Instruction *B);
 
+  /// Returns true if \p Op is guaranteed not to cause immediate UB.
+  bool isGuaranteedNotToCauseUB(const SCEV *Op);
+
   /// Returns true if \p Op is guaranteed to not be poison.
   static bool isGuaranteedNotToBePoison(const SCEV *Op);
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 985ca1532e014..c4d5459d25092 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -335,12 +335,10 @@ class TargetTransformInfo {
   /// chain of loads or stores within same block) operations set when lowered.
   /// \p AccessTy is the type of the loads/stores that will ultimately use the
   /// \p Ptrs.
-  InstructionCost
-  getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
-                       const PointersChainInfo &Info, Type *AccessTy,
-                       TargetCostKind CostKind = TTI::TCK_RecipThroughput
-
-  ) const;
+  InstructionCost getPointersChainCost(
+      ArrayRef<const Value *> Ptrs, const Value *Base,
+      const PointersChainInfo &Info, Type *AccessTy,
+      TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
 
   /// \returns A value by which our inlining threshold should be multiplied.
   /// This is primarily used to bump up the inlining threshold wholesale on
@@ -615,6 +613,9 @@ class TargetTransformInfo {
     unsigned MaxIterationsCountToAnalyze;
     /// Don't disable runtime unroll for the loops which were vectorized.
     bool UnrollVectorizedLoop = false;
+    /// Don't allow runtime unrolling if expanding the trip count takes more
+    /// than SCEVExpansionBudget.
+    unsigned SCEVExpansionBudget;
   };
 
   /// Get target-customized preferences for the generic loop unrolling
@@ -909,11 +910,13 @@ class TargetTransformInfo {
 
   /// Estimate the overhead of scalarizing an instruction. Insert and Extract
   /// are set if the demanded result elements need to be inserted and/or
-  /// extracted from vectors.
+  /// extracted from vectors.  The involved values may be passed in VL if
+  /// Insert is true.
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind) const;
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {}) const;
 
   /// Estimate the overhead of scalarizing an instructions unique
   /// non-constant operands. The (potentially vector) types to use for each of
@@ -2001,10 +2004,10 @@ class TargetTransformInfo::Concept {
                                                   unsigned ScalarOpdIdx) = 0;
   virtual bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
                                                       int ScalarOpdIdx) = 0;
-  virtual InstructionCost getScalarizationOverhead(VectorType *Ty,
-                                                   const APInt &DemandedElts,
-                                                   bool Insert, bool Extract,
-                                                   TargetCostKind CostKind) = 0;
+  virtual InstructionCost
+  getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+                           bool Insert, bool Extract, TargetCostKind CostKind,
+                           ArrayRef<Value *> VL = {}) = 0;
   virtual InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                    ArrayRef<Type *> Tys,
@@ -2585,9 +2588,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TargetCostKind CostKind) override {
+                                           TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {}) override {
     return Impl.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
-                                         CostKind);
+                                         CostKind, VL);
   }
   InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 38aba183f6a17..48ebffff8cbfc 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -404,7 +404,8 @@ class TargetTransformInfoImplBase {
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind) const {
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {}) const {
     return 0;
   }
 
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 2b0377903ac8e..8aa024a72afc8 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -159,7 +159,7 @@ bool isKnownPositive(const Value *V, const SimplifyQuery &SQ,
 
 /// Returns true if the given value is known be negative (i.e. non-positive
 /// and non-zero).
-bool isKnownNegative(const Value *V, const SimplifyQuery &DL,
+bool isKnownNegative(const Value *V, const SimplifyQuery &SQ,
                      unsigned Depth = 0);
 
 /// Return true if the given values are known to be non-equal when defined.
@@ -180,7 +180,7 @@ bool isKnownNonEqual(const Value *V1, const Value *V2, const DataLayout &DL,
 /// same width as the vector element, and the bit is set only if it is true
 /// for all of the elements in the vector.
 bool MaskedValueIsZero(const Value *V, const APInt &Mask,
-                       const SimplifyQuery &DL, unsigned Depth = 0);
+                       const SimplifyQuery &SQ, unsigned Depth = 0);
 
 /// Return the number of times the sign bit of the register is replicated into
 /// the other bits. We know that at least 1 bit is always equal to the sign
@@ -1178,10 +1178,20 @@ SelectPatternResult matchDecomposedSelectPattern(
     CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
     Instruction::CastOps *CastOp = nullptr, unsigned Depth = 0);
 
+/// Determine the pattern for predicate `X Pred Y ? X : Y`.
+SelectPatternResult
+getSelectPattern(CmpInst::Predicate Pred,
+                 SelectPatternNaNBehavior NaNBehavior = SPNB_NA,
+                 bool Ordered = false);
+
 /// Return the canonical comparison predicate for the specified
 /// minimum/maximum flavor.
 CmpInst::Predicate getMinMaxPred(SelectPatternFlavor SPF, bool Ordered = false);
 
+/// Convert given `SPF` to equivalent min/max intrinsic.
+/// Caller must ensure `SPF` is an integer min or max pattern.
+Intrinsic::ID getMinMaxIntrinsic(SelectPatternFlavor SPF);
+
 /// Return the inverse minimum/maximum flavor of the specified flavor.
 /// For example, signed minimum is the inverse of signed maximum.
 SelectPatternFlavor getInverseMinMaxFlavor(SelectPatternFlavor SPF);
@@ -1245,8 +1255,7 @@ std::optional<bool> isImpliedCondition(const Value *LHS, const Value *RHS,
                                        const DataLayout &DL,
                                        bool LHSIsTrue = true,
                                        unsigned Depth = 0);
-std::optional<bool> isImpliedCondition(const Value *LHS,
-                                       CmpInst::Predicate RHSPred,
+std::optional<bool> isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
                                        const Value *RHSOp0, const Value *RHSOp1,
                                        const DataLayout &DL,
                                        bool LHSIsTrue = true,
@@ -1257,8 +1266,8 @@ std::optional<bool> isImpliedCondition(const Value *LHS,
 std::optional<bool> isImpliedByDomCondition(const Value *Cond,
                                             const Instruction *ContextI,
                                             const DataLayout &DL);
-std::optional<bool> isImpliedByDomCondition(CmpInst::Predicate Pred,
-                                            const Value *LHS, const Value *RHS,
+std::optional<bool> isImpliedByDomCondition(CmpPredicate Pred, const Value *LHS,
+                                            const Value *RHS,
                                             const Instruction *ContextI,
                                             const DataLayout &DL);
 
diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
index f28a4646cc6da..05b79eae573f7 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
@@ -143,27 +143,27 @@ ELF_RELOC(R_AARCH64_IRELATIVE,                       0x408)
 // PAuthABI static and dynamic relocations: defined in pauthabielf64,
 // https://github.com/ARM-software/abi-aa
 ELF_RELOC(R_AARCH64_AUTH_ABS64,                      0x244)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G0,             0x245)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G0_NC,          0x246)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G1,             0x247)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G1_NC,          0x248)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G2,             0x249)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G2_NC,          0x24a)
+ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G3,             0x24b)
+ELF_RELOC(R_AARCH64_AUTH_GOT_LD_PREL19,              0x24c)
+ELF_RELOC(R_AARCH64_AUTH_LD64_GOTOFF_LO15,           0x24d)
+ELF_RELOC(R_AARCH64_AUTH_ADR_GOT_PAGE,               0x24e)
+ELF_RELOC(R_AARCH64_AUTH_LD64_GOT_LO12_NC,           0x24f)
+ELF_RELOC(R_AARCH64_AUTH_LD64_GOTPAGE_LO15,          0x250)
+ELF_RELOC(R_AARCH64_AUTH_GOT_ADD_LO12_NC,            0x251)
+ELF_RELOC(R_AARCH64_AUTH_GOT_ADR_PREL_LO21,          0x252)
+ELF_RELOC(R_AARCH64_AUTH_TLSDESC_ADR_PAGE21,         0x253)
+ELF_RELOC(R_AARCH64_AUTH_TLSDESC_LD64_LO12,          0x254)
+ELF_RELOC(R_AARCH64_AUTH_TLSDESC_ADD_LO12,           0x255)
 ELF_RELOC(R_AARCH64_AUTH_RELATIVE,                   0x411)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G0,             0x8110)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G0_NC,          0x8111)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G1,             0x8112)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G1_NC,          0x8113)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G2,             0x8114)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G2_NC,          0x8115)
-ELF_RELOC(R_AARCH64_AUTH_MOVW_GOTOFF_G3,             0x8116)
-ELF_RELOC(R_AARCH64_AUTH_GOT_LD_PREL19,              0x8117)
-ELF_RELOC(R_AARCH64_AUTH_LD64_GOTOFF_LO15,           0x8118)
-ELF_RELOC(R_AARCH64_AUTH_ADR_GOT_PAGE,               0x8119)
-ELF_RELOC(R_AARCH64_AUTH_LD64_GOT_LO12_NC,           0x811a)
-ELF_RELOC(R_AARCH64_AUTH_LD64_GOTPAGE_LO15,          0x811b)
-ELF_RELOC(R_AARCH64_AUTH_GOT_ADD_LO12_NC,            0x811c)
-ELF_RELOC(R_AARCH64_AUTH_GOT_ADR_PREL_LO21,          0x811d)
-ELF_RELOC(R_AARCH64_AUTH_TLSDESC_ADR_PAGE21,         0x811e)
-ELF_RELOC(R_AARCH64_AUTH_TLSDESC_LD64_LO12,          0x811f)
-ELF_RELOC(R_AARCH64_AUTH_TLSDESC_ADD_LO12,           0x8120)
-ELF_RELOC(R_AARCH64_AUTH_GLOB_DAT,                   0xe201)
-ELF_RELOC(R_AARCH64_AUTH_TLSDESC,                    0xe202)
-ELF_RELOC(R_AARCH64_AUTH_IRELATIVE,                  0xe203)
+ELF_RELOC(R_AARCH64_AUTH_GLOB_DAT,                   0x412)
+ELF_RELOC(R_AARCH64_AUTH_TLSDESC,                    0x413)
+ELF_RELOC(R_AARCH64_AUTH_IRELATIVE,                  0x414)
 
 // ELF32
 // ELF_RELOC(R_AARCH64_P32_NONE,                         0)
diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def
index d4be34e3b37e5..ac9a089e853a6 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def
@@ -45,7 +45,6 @@ ELF_RELOC(R_RISCV_GOT32_PCREL,       41)
 ELF_RELOC(R_RISCV_ALIGN,             43)
 ELF_RELOC(R_RISCV_RVC_BRANCH,        44)
 ELF_RELOC(R_RISCV_RVC_JUMP,          45)
-ELF_RELOC(R_RISCV_RVC_LUI,           46)
 ELF_RELOC(R_RISCV_RELAX,             51)
 ELF_RELOC(R_RISCV_SUB6,              52)
 ELF_RELOC(R_RISCV_SET6,              53)
@@ -61,3 +60,68 @@ ELF_RELOC(R_RISCV_TLSDESC_HI20,      62)
 ELF_RELOC(R_RISCV_TLSDESC_LOAD_LO12, 63)
 ELF_RELOC(R_RISCV_TLSDESC_ADD_LO12,  64)
 ELF_RELOC(R_RISCV_TLSDESC_CALL,      65)
+ELF_RELOC(R_RISCV_VENDOR,           191)
+ELF_RELOC(R_RISCV_CUSTOM192,        192)
+ELF_RELOC(R_RISCV_CUSTOM193,        193)
+ELF_RELOC(R_RISCV_CUSTOM194,        194)
+ELF_RELOC(R_RISCV_CUSTOM195,        195)
+ELF_RELOC(R_RISCV_CUSTOM196,        196)
+ELF_RELOC(R_RISCV_CUSTOM197,        197)
+ELF_RELOC(R_RISCV_CUSTOM198,        198)
+ELF_RELOC(R_RISCV_CUSTOM199,        199)
+ELF_RELOC(R_RISCV_CUSTOM200,        200)
+ELF_RELOC(R_RISCV_CUSTOM201,        201)
+ELF_RELOC(R_RISCV_CUSTOM202,        202)
+ELF_RELOC(R_RISCV_CUSTOM203,        203)
+ELF_RELOC(R_RISCV_CUSTOM204,        204)
+ELF_RELOC(R_RISCV_CUSTOM205,        205)
+ELF_RELOC(R_RISCV_CUSTOM206,        206)
+ELF_RELOC(R_RISCV_CUSTOM207,        207)
+ELF_RELOC(R_RISCV_CUSTOM208,        208)
+ELF_RELOC(R_RISCV_CUSTOM209,        209)
+ELF_RELOC(R_RISCV_CUSTOM210,        210)
+ELF_RELOC(R_RISCV_CUSTOM211,        211)
+ELF_RELOC(R_RISCV_CUSTOM212,        212)
+ELF_RELOC(R_RISCV_CUSTOM213,        213)
+ELF_RELOC(R_RISCV_CUSTOM214,        214)
+ELF_RELOC(R_RISCV_CUSTOM215,        215)
+ELF_RELOC(R_RISCV_CUSTOM216,        216)
+ELF_RELOC(R_RISCV_CUSTOM217,        217)
+ELF_RELOC(R_RISCV_CUSTOM218,        218)
+ELF_RELOC(R_RISCV_CUSTOM219,        219)
+ELF_RELOC(R_RISCV_CUSTOM220,        220)
+ELF_RELOC(R_RISCV_CUSTOM221,        221)
+ELF_RELOC(R_RISCV_CUSTOM222,        222)
+ELF_RELOC(R_RISCV_CUSTOM223,        223)
+ELF_RELOC(R_RISCV_CUSTOM224,        224)
+ELF_RELOC(R_RISCV_CUSTOM225,        225)
+ELF_RELOC(R_RISCV_CUSTOM226,        226)
+ELF_RELOC(R_RISCV_CUSTOM227,        227)
+ELF_RELOC(R_RISCV_CUSTOM228,        228)
+ELF_RELOC(R_RISCV_CUSTOM229,        229)
+ELF_RELOC(R_RISCV_CUSTOM230,        230)
+ELF_RELOC(R_RISCV_CUSTOM231,        231)
+ELF_RELOC(R_RISCV_CUSTOM232,        232)
+ELF_RELOC(R_RISCV_CUSTOM233,        233)
+ELF_RELOC(R_RISCV_CUSTOM234,        234)
+ELF_RELOC(R_RISCV_CUSTOM235,        235)
+ELF_RELOC(R_RISCV_CUSTOM236,        236)
+ELF_RELOC(R_RISCV_CUSTOM237,        237)
+ELF_RELOC(R_RISCV_CUSTOM238,        238)
+ELF_RELOC(R_RISCV_CUSTOM239,        239)
+ELF_RELOC(R_RISCV_CUSTOM240,        240)
+ELF_RELOC(R_RISCV_CUSTOM241,        241)
+ELF_RELOC(R_RISCV_CUSTOM242,        242)
+ELF_RELOC(R_RISCV_CUSTOM243,        243)
+ELF_RELOC(R_RISCV_CUSTOM244,        244)
+ELF_RELOC(R_RISCV_CUSTOM245,        245)
+ELF_RELOC(R_RISCV_CUSTOM246,        246)
+ELF_RELOC(R_RISCV_CUSTOM247,        247)
+ELF_RELOC(R_RISCV_CUSTOM248,        248)
+ELF_RELOC(R_RISCV_CUSTOM249,        249)
+ELF_RELOC(R_RISCV_CUSTOM250,        250)
+ELF_RELOC(R_RISCV_CUSTOM251,        251)
+ELF_RELOC(R_RISCV_CUSTOM252,        252)
+ELF_RELOC(R_RISCV_CUSTOM253,        253)
+ELF_RELOC(R_RISCV_CUSTOM254,        254)
+ELF_RELOC(R_RISCV_CUSTOM255,        255)
diff --git a/llvm/include/llvm/BinaryFormat/XCOFF.h b/llvm/include/llvm/BinaryFormat/XCOFF.h
index bbcd8a4f29ae9..b48976769c0c4 100644
--- a/llvm/include/llvm/BinaryFormat/XCOFF.h
+++ b/llvm/include/llvm/BinaryFormat/XCOFF.h
@@ -333,10 +333,33 @@ enum CFileLangId : uint8_t {
   TB_CPLUSPLUS = 9 ///< C++ language.
 };
 
+// XCOFF specific CPU IDs, defined in AIX OS header: `/usr/include/aouthdr.h`.
 enum CFileCpuId : uint8_t {
-  TCPU_PPC64 = 2, ///< PowerPC common architecture 64-bit mode.
-  TCPU_COM = 3,   ///< POWER and PowerPC architecture common.
-  TCPU_970 = 19   ///< PPC970 - PowerPC 64-bit architecture.
+  TCPU_INVALID = 0, ///< Invalid id - assumes POWER for old objects.
+  TCPU_PPC = 1,     ///< PowerPC common architecture 32 bit mode.
+  TCPU_PPC64 = 2,   ///< PowerPC common architecture 64-bit mode.
+  TCPU_COM = 3,     ///< POWER and PowerPC architecture common.
+  TCPU_PWR = 4,     ///< POWER common architecture objects.
+  TCPU_ANY = 5,     ///< Mixture of any incompatable POWER
+                    ///< and PowerPC architecture implementations.
+  TCPU_601 = 6,     ///< 601 implementation of PowerPC architecture.
+  TCPU_603 = 7,     ///< 603 implementation of PowerPC architecture.
+  TCPU_604 = 8,     ///< 604 implementation of PowerPC architecture.
+
+  // The following are PowerPC 64-bit architectures.
+  TCPU_620 = 16,
+  TCPU_A35 = 17,
+  TCPU_PWR5 = 18,
+  TCPU_970 = 19,
+  TCPU_PWR6 = 20,
+  TCPU_PWR5X = 22,
+  TCPU_PWR6E = 23,
+  TCPU_PWR7 = 24,
+  TCPU_PWR8 = 25,
+  TCPU_PWR9 = 26,
+  TCPU_PWR10 = 27,
+
+  TCPU_PWRX = 224 ///< RS2 implementation of POWER architecture.
 };
 
 enum SymbolAuxType : uint8_t {
@@ -350,6 +373,7 @@ enum SymbolAuxType : uint8_t {
 
 StringRef getMappingClassString(XCOFF::StorageMappingClass SMC);
 StringRef getRelocationTypeString(XCOFF::RelocationType Type);
+StringRef getTCPUString(XCOFF::CFileCpuId TCPU);
 Expected<SmallString<32>> parseParmsType(uint32_t Value, unsigned FixedParmsNum,
                                          unsigned FloatingParmsNum);
 Expected<SmallString<32>> parseParmsTypeWithVecInfo(uint32_t Value,
@@ -468,6 +492,7 @@ enum ExtendedTBTableFlag : uint8_t {
 
 StringRef getNameForTracebackTableLanguageId(TracebackTable::LanguageID LangId);
 SmallString<32> getExtendedTBTableFlagString(uint8_t Flag);
+XCOFF::CFileCpuId getCpuID(StringRef CPU);
 
 struct CsectProperties {
   CsectProperties(StorageMappingClass SMC, SymbolType ST)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 98cbb4886642b..8eef8ea665a26 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -780,7 +780,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getScalarizationOverhead(VectorType *InTy,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {}) {
     /// FIXME: a bitfield is not a reasonable abstraction for talking about
     /// which elements are needed from a scalable vector
     if (isa<ScalableVectorType>(InTy))
@@ -788,6 +789,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     auto *Ty = cast<FixedVectorType>(InTy);
 
     assert(DemandedElts.getBitWidth() == Ty->getNumElements() &&
+           (VL.empty() || VL.size() == Ty->getNumElements()) &&
            "Vector size mismatch");
 
     InstructionCost Cost = 0;
@@ -795,9 +797,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     for (int i = 0, e = Ty->getNumElements(); i < e; ++i) {
       if (!DemandedElts[i])
         continue;
-      if (Insert)
+      if (Insert) {
+        Value *InsertedVal = VL.empty() ? nullptr : VL[i];
         Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
-                                            CostKind, i, nullptr, nullptr);
+                                            CostKind, i, nullptr, InsertedVal);
+      }
       if (Extract)
         Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
                                             CostKind, i, nullptr, nullptr);
@@ -1931,6 +1935,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       return Cost;
     }
+    case Intrinsic::experimental_vector_match:
+      return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
     }
 
     // Assume that we need to scalarize this intrinsic.)
@@ -2186,6 +2192,35 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::vector_reduce_fminimum:
       return thisT()->getMinMaxReductionCost(getMinMaxReductionIntrinsicOp(IID),
                                              VecOpTy, ICA.getFlags(), CostKind);
+    case Intrinsic::experimental_vector_match: {
+      auto *SearchTy = cast<VectorType>(ICA.getArgTypes()[0]);
+      auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
+      unsigned SearchSize = NeedleTy->getNumElements();
+
+      // If we're not expanding the intrinsic then we assume this is cheap to
+      // implement.
+      EVT SearchVT = getTLI()->getValueType(DL, SearchTy);
+      if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize))
+        return getTypeLegalizationCost(RetTy).first;
+
+      // Approximate the cost based on the expansion code in
+      // SelectionDAGBuilder.
+      InstructionCost Cost = 0;
+      Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, NeedleTy,
+                                          CostKind, 1, nullptr, nullptr);
+      Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, SearchTy,
+                                          CostKind, 0, nullptr, nullptr);
+      Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, SearchTy, std::nullopt,
+                                      CostKind, 0, nullptr);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SearchTy, RetTy,
+                                          CmpInst::ICMP_EQ, CostKind);
+      Cost +=
+          thisT()->getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
+      Cost *= SearchSize;
+      Cost +=
+          thisT()->getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind);
+      return Cost;
+    }
     case Intrinsic::abs:
       ISD = ISD::ABS;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index b681a0708db4b..5d7e03bbaeb7d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -778,6 +778,11 @@ class LegalizeRuleSet {
   LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
     return actionFor(LegalizeAction::Libcall, Types);
   }
+  LegalizeRuleSet &libcallFor(bool Pred, std::initializer_list<LLT> Types) {
+    if (!Pred)
+      return *this;
+    return actionFor(LegalizeAction::Libcall, Types);
+  }
   LegalizeRuleSet &
   libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
     return actionFor(LegalizeAction::Libcall, Types);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 3516065f9b6cb..fac7fa6417265 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1767,6 +1767,34 @@ class MachineIRBuilder {
     return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags);
   }
 
+  /// Build and insert \p Res = G_ABDS \p Op0, \p Op1
+  ///
+  /// G_ABDS return the signed absolute difference of \p Op0 and \p Op1.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers
+  ///      with the same (scalar or vector) type).
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildAbds(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1) {
+    return buildInstr(TargetOpcode::G_ABDS, {Dst}, {Src0, Src1});
+  }
+
+  /// Build and insert \p Res = G_ABDU \p Op0, \p Op1
+  ///
+  /// G_ABDU return the unsigned absolute difference of \p Op0 and \p Op1.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers
+  ///      with the same (scalar or vector) type).
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildAbdu(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1) {
+    return buildInstr(TargetOpcode::G_ABDU, {Dst}, {Src0, Src1});
+  }
+
   MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0,
                                  const SrcOp &Src1,
                                  std::optional<unsigned> Flags = std::nullopt) {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
index e4d9ff522b5a9..46173ffd76b72 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
@@ -617,7 +617,7 @@ class RegBankSelect : public MachineFunctionPass {
 
 public:
   /// Create a RegBankSelect pass with the specified \p RunningMode.
-  RegBankSelect(char &PassID = ID, Mode RunningMode = Fast);
+  RegBankSelect(Mode RunningMode = Fast);
 
   StringRef getPassName() const override { return "RegBankSelect"; }
 
diff --git a/llvm/include/llvm/CodeGen/LiveDebugVariables.h b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
index a4b5a87fd3887..2cb95f2c71ccd 100644
--- a/llvm/include/llvm/CodeGen/LiveDebugVariables.h
+++ b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
@@ -21,7 +21,10 @@
 #define LLVM_CODEGEN_LIVEDEBUGVARIABLES_H
 
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
 
 namespace llvm {
 
@@ -29,15 +32,15 @@ template <typename T> class ArrayRef;
 class LiveIntervals;
 class VirtRegMap;
 
-class LiveDebugVariables : public MachineFunctionPass {
-  void *pImpl = nullptr;
+class LiveDebugVariables {
 
 public:
-  static char ID; // Pass identification, replacement for typeid
-
+  class LDVImpl;
   LiveDebugVariables();
-  ~LiveDebugVariables() override;
+  ~LiveDebugVariables();
+  LiveDebugVariables(LiveDebugVariables &&);
 
+  void analyze(MachineFunction &MF, LiveIntervals *LIS);
   /// splitRegister - Move any user variables in OldReg to the live ranges in
   /// NewRegs where they are live. Mark the values as unavailable where no new
   /// register is live.
@@ -49,12 +52,39 @@ class LiveDebugVariables : public MachineFunctionPass {
   /// @param VRM Rename virtual registers according to map.
   void emitDebugValues(VirtRegMap *VRM);
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// dump - Print data structures to dbgs().
   void dump() const;
+#endif
+
+  void print(raw_ostream &OS) const;
+
+  void releaseMemory();
+
+  bool invalidate(MachineFunction &MF, const PreservedAnalyses &PA,
+                  MachineFunctionAnalysisManager::Invalidator &Inv);
 
 private:
+  std::unique_ptr<LDVImpl> PImpl;
+};
+
+class LiveDebugVariablesWrapperLegacy : public MachineFunctionPass {
+  std::unique_ptr<LiveDebugVariables> Impl;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+
+  LiveDebugVariablesWrapperLegacy();
+
   bool runOnMachineFunction(MachineFunction &) override;
-  void releaseMemory() override;
+
+  LiveDebugVariables &getLDV() { return *Impl; }
+  const LiveDebugVariables &getLDV() const { return *Impl; }
+
+  void releaseMemory() override {
+    if (Impl)
+      Impl->releaseMemory();
+  }
   void getAnalysisUsage(AnalysisUsage &) const override;
 
   MachineFunctionProperties getSetProperties() const override {
@@ -63,6 +93,32 @@ class LiveDebugVariables : public MachineFunctionPass {
   }
 };
 
+class LiveDebugVariablesAnalysis
+    : public AnalysisInfoMixin<LiveDebugVariablesAnalysis> {
+  friend AnalysisInfoMixin<LiveDebugVariablesAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  using Result = LiveDebugVariables;
+
+  MachineFunctionProperties getSetProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::TracksDebugUserValues);
+  }
+
+  Result run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM);
+};
+
+class LiveDebugVariablesPrinterPass
+    : public PassInfoMixin<LiveDebugVariablesPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  LiveDebugVariablesPrinterPass(raw_ostream &OS) : OS(OS) {}
+
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_LIVEDEBUGVARIABLES_H
diff --git a/llvm/include/llvm/CodeGen/LiveStacks.h b/llvm/include/llvm/CodeGen/LiveStacks.h
index 2edc2985f0ee6..02c640bfc4a93 100644
--- a/llvm/include/llvm/CodeGen/LiveStacks.h
+++ b/llvm/include/llvm/CodeGen/LiveStacks.h
@@ -17,6 +17,7 @@
 
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/PassRegistry.h"
 #include <cassert>
@@ -32,7 +33,7 @@ class raw_ostream;
 class TargetRegisterClass;
 class TargetRegisterInfo;
 
-class LiveStacks : public MachineFunctionPass {
+class LiveStacks {
   const TargetRegisterInfo *TRI = nullptr;
 
   /// Special pool allocator for VNInfo's (LiveInterval val#).
@@ -47,12 +48,6 @@ class LiveStacks : public MachineFunctionPass {
   std::map<int, const TargetRegisterClass *> S2RCMap;
 
 public:
-  static char ID; // Pass identification, replacement for typeid
-
-  LiveStacks() : MachineFunctionPass(ID) {
-    initializeLiveStacksPass(*PassRegistry::getPassRegistry());
-  }
-
   using iterator = SS2IntervalMap::iterator;
   using const_iterator = SS2IntervalMap::const_iterator;
 
@@ -92,6 +87,25 @@ class LiveStacks : public MachineFunctionPass {
 
   VNInfo::Allocator &getVNInfoAllocator() { return VNInfoAllocator; }
 
+  void releaseMemory();
+  /// init - analysis entry point
+  void init(MachineFunction &MF);
+  void print(raw_ostream &O, const Module *M = nullptr) const;
+};
+
+class LiveStacksWrapperLegacy : public MachineFunctionPass {
+  LiveStacks Impl;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+
+  LiveStacksWrapperLegacy() : MachineFunctionPass(ID) {
+    initializeLiveStacksWrapperLegacyPass(*PassRegistry::getPassRegistry());
+  }
+
+  LiveStacks &getLS() { return Impl; }
+  const LiveStacks &getLS() const { return Impl; }
+
   void getAnalysisUsage(AnalysisUsage &AU) const override;
   void releaseMemory() override;
 
@@ -102,6 +116,24 @@ class LiveStacks : public MachineFunctionPass {
   void print(raw_ostream &O, const Module * = nullptr) const override;
 };
 
+class LiveStacksAnalysis : public AnalysisInfoMixin<LiveStacksAnalysis> {
+  static AnalysisKey Key;
+  friend AnalysisInfoMixin<LiveStacksAnalysis>;
+
+public:
+  using Result = LiveStacks;
+
+  LiveStacks run(MachineFunction &MF, MachineFunctionAnalysisManager &);
+};
+
+class LiveStacksPrinterPass : public PassInfoMixin<LiveStacksPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  LiveStacksPrinterPass(raw_ostream &OS) : OS(OS) {}
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &AM);
+};
 } // end namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 2e3507386df30..e97e01839f73b 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -44,6 +44,7 @@
 #include <cstdint>
 #include <functional>
 #include <map>
+#include <set>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -247,6 +248,9 @@ class SelectionDAG {
   BlockFrequencyInfo *BFI = nullptr;
   MachineModuleInfo *MMI = nullptr;
 
+  /// Extended EVTs used for single value VTLists.
+  std::set<EVT, EVT::compareRawBits> EVTs;
+
   /// List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index f99ec4651009a..43ba8f4c44cf9 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -315,7 +315,7 @@ class SelectionDAGISel {
     OPC_MorphNodeTo1GlueOutput,
     OPC_MorphNodeTo2GlueOutput,
     OPC_CompleteMatch,
-    // Contains offset in table for pattern being selected
+    // Contains 32-bit offset in table for pattern being selected
     OPC_Coverage
   };
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 677b59e0c8fbe..61f3c6329efce 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -664,7 +664,7 @@ END_TWO_BYTE_PACK()
   DebugLoc debugLoc;
 
   /// Return a pointer to the specified value type.
-  static const EVT *getValueTypeList(EVT VT);
+  static const EVT *getValueTypeList(MVT VT);
 
   /// Index in worklist of DAGCombiner, or negative if the node is not in the
   /// worklist. -1 = not in worklist; -2 = not in worklist, but has already been
@@ -1124,7 +1124,7 @@ END_TWO_BYTE_PACK()
   void addUse(SDUse &U) { U.addToList(&UseList); }
 
 protected:
-  static SDVTList getSDVTList(EVT VT) {
+  static SDVTList getSDVTList(MVT VT) {
     SDVTList Ret = { getValueTypeList(VT), 1 };
     return Ret;
   }
diff --git a/llvm/lib/CodeGen/SpillPlacement.h b/llvm/include/llvm/CodeGen/SpillPlacement.h
similarity index 83%
rename from llvm/lib/CodeGen/SpillPlacement.h
rename to llvm/include/llvm/CodeGen/SpillPlacement.h
index 5fd9b085259d1..1ef37f2718a65 100644
--- a/llvm/lib/CodeGen/SpillPlacement.h
+++ b/llvm/include/llvm/CodeGen/SpillPlacement.h
@@ -29,6 +29,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseSet.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Support/BlockFrequency.h"
 
@@ -38,13 +39,20 @@ class BitVector;
 class EdgeBundles;
 class MachineBlockFrequencyInfo;
 class MachineFunction;
+class SpillPlacementWrapperLegacy;
+class SpillPlacementAnalysis;
+
+class SpillPlacement {
+  friend class SpillPlacementWrapperLegacy;
+  friend class SpillPlacementAnalysis;
 
-class SpillPlacement : public MachineFunctionPass {
   struct Node;
+
   const MachineFunction *MF = nullptr;
   const EdgeBundles *bundles = nullptr;
   const MachineBlockFrequencyInfo *MBFI = nullptr;
-  Node *nodes = nullptr;
+
+  std::unique_ptr<Node[]> nodes;
 
   // Nodes that are active in the current computation. Owned by the prepare()
   // caller.
@@ -68,11 +76,6 @@ class SpillPlacement : public MachineFunctionPass {
   SparseSet<unsigned> TodoList;
 
 public:
-  static char ID; // Pass identification, replacement for typeid.
-
-  SpillPlacement() : MachineFunctionPass(ID) {}
-  ~SpillPlacement() override { releaseMemory(); }
-
   /// BorderConstraint - A basic block has separate constraints for entry and
   /// exit.
   enum BorderConstraint {
@@ -154,17 +157,50 @@ class SpillPlacement : public MachineFunctionPass {
     return BlockFrequencies[Number];
   }
 
+  bool invalidate(MachineFunction &MF, const PreservedAnalyses &PA,
+                  MachineFunctionAnalysisManager::Invalidator &Inv);
+
+  SpillPlacement(SpillPlacement &&);
+  ~SpillPlacement();
+
 private:
-  bool runOnMachineFunction(MachineFunction &mf) override;
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  void releaseMemory() override;
+  SpillPlacement();
 
+  void releaseMemory();
+
+  void run(MachineFunction &MF, EdgeBundles *Bundles,
+           MachineBlockFrequencyInfo *MBFI);
   void activate(unsigned n);
   void setThreshold(BlockFrequency Entry);
 
   bool update(unsigned n);
 };
 
+class SpillPlacementWrapperLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+  SpillPlacementWrapperLegacy() : MachineFunctionPass(ID) {}
+
+  SpillPlacement &getResult() { return Impl; }
+  const SpillPlacement &getResult() const { return Impl; }
+
+private:
+  SpillPlacement Impl;
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void releaseMemory() override { Impl.releaseMemory(); }
+};
+
+class SpillPlacementAnalysis
+    : public AnalysisInfoMixin<SpillPlacementAnalysis> {
+  friend AnalysisInfoMixin<SpillPlacementAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  using Result = SpillPlacement;
+  SpillPlacement run(MachineFunction &, MachineFunctionAnalysisManager &);
+};
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_CODEGEN_SPILLPLACEMENT_H
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 6a41094ff933b..aaab209bfa75d 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1720,10 +1720,9 @@ class TargetLoweringBase {
     return getValueType(DL, Ty, AllowUnknown).getSimpleVT();
   }
 
-  /// Return the desired alignment for ByVal or InAlloca aggregate function
-  /// arguments in the caller parameter area.  This is the actual alignment, not
-  /// its logarithm.
-  virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
+  /// Returns the desired alignment for ByVal or InAlloca aggregate function
+  /// arguments in the caller parameter area.
+  virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
 
   /// Return the type of registers that this ValueType will eventually require.
   MVT getRegisterType(MVT VT) const {
@@ -2292,7 +2291,7 @@ class TargetLoweringBase {
   virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const {}
 
   /// Returns true if arguments should be sign-extended in lib calls.
-  virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
+  virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const {
     return IsSigned;
   }
 
@@ -4305,6 +4304,12 @@ class TargetLowering : public TargetLoweringBase {
   /// @param Level the current DAGCombine legalization level.
   virtual bool isDesirableToCommuteWithShift(const SDNode *N,
                                              CombineLevel Level) const {
+    SDValue ShiftLHS = N->getOperand(0);
+    if (!ShiftLHS->hasOneUse())
+      return false;
+    if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
+        !ShiftLHS.getOperand(0)->hasOneUse())
+      return false;
     return true;
   }
 
@@ -4713,18 +4718,18 @@ class TargetLowering : public TargetLoweringBase {
     // shouldExtendTypeInLibCall can get the original type before soften.
     ArrayRef<EVT> OpsVTBeforeSoften;
     EVT RetVTBeforeSoften;
-    bool IsSExt : 1;
+    bool IsSigned : 1;
     bool DoesNotReturn : 1;
     bool IsReturnValueUsed : 1;
     bool IsPostTypeLegalization : 1;
     bool IsSoften : 1;
 
     MakeLibCallOptions()
-        : IsSExt(false), DoesNotReturn(false), IsReturnValueUsed(true),
+        : IsSigned(false), DoesNotReturn(false), IsReturnValueUsed(true),
           IsPostTypeLegalization(false), IsSoften(false) {}
 
-    MakeLibCallOptions &setSExt(bool Value = true) {
-      IsSExt = Value;
+    MakeLibCallOptions &setIsSigned(bool Value = true) {
+      IsSigned = Value;
       return *this;
     }
 
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index 23d86248ff87a..a94ebf55f6c1e 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -63,7 +63,8 @@ class Triple;
 class TargetSubtargetInfo : public MCSubtargetInfo {
 protected: // Can only create subclasses...
   TargetSubtargetInfo(const Triple &TT, StringRef CPU, StringRef TuneCPU,
-                      StringRef FS, ArrayRef<SubtargetFeatureKV> PF,
+                      StringRef FS, ArrayRef<StringRef> PN,
+                      ArrayRef<SubtargetFeatureKV> PF,
                       ArrayRef<SubtargetSubTypeKV> PD,
                       const MCWriteProcResEntry *WPR,
                       const MCWriteLatencyEntry *WL,
@@ -349,6 +350,8 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
     // Conservatively assume such instructions exist by default.
     return true;
   }
+
+  virtual bool isRegisterReservedByUser(Register R) const { return false; }
 };
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Config/abi-breaking.h.cmake b/llvm/include/llvm/Config/abi-breaking.h.cmake
index 81495f0569752..2d27e02b1d545 100644
--- a/llvm/include/llvm/Config/abi-breaking.h.cmake
+++ b/llvm/include/llvm/Config/abi-breaking.h.cmake
@@ -12,8 +12,6 @@
 #ifndef LLVM_ABI_BREAKING_CHECKS_H
 #define LLVM_ABI_BREAKING_CHECKS_H
 
-#include "llvm/Support/Compiler.h"
-
 /* Define to enable checks that alter the LLVM C++ ABI */
 #cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS
 
@@ -45,12 +43,12 @@
 #endif
 namespace llvm {
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
-LLVM_ABI extern int EnableABIBreakingChecks;
+extern int EnableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyEnableABIBreakingChecks =
     &EnableABIBreakingChecks;
 #else
-LLVM_ABI extern int DisableABIBreakingChecks;
+extern int DisableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyDisableABIBreakingChecks =
     &DisableABIBreakingChecks;
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
index 69c91835a4d9a..2e98a4a397147 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
@@ -226,6 +226,8 @@ class DWARFDie {
 
   bool addressRangeContainsAddress(const uint64_t Address) const;
 
+  std::optional<uint64_t> getLanguage() const;
+
   Expected<DWARFLocationExpressionsVector>
   getLocations(dwarf::Attribute Attr) const;
 
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
index 97ff7e8407024..3c936b9386504 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/Support/Error.h"
 
 #include <string>
 
@@ -108,13 +109,11 @@ void DWARFTypePrinter<DieType>::appendArrayType(const DieType &D) {
     if (std::optional<typename DieType::DWARFFormValue> UpperV =
             C.find(dwarf::DW_AT_upper_bound))
       UB = UpperV->getAsUnsignedConstant();
-    if (std::optional<typename DieType::DWARFFormValue> LV =
-            D.getDwarfUnit()->getUnitDIE().find(dwarf::DW_AT_language))
-      if (std::optional<uint64_t> LC = LV->getAsUnsignedConstant())
-        if ((DefaultLB =
-                 LanguageLowerBound(static_cast<dwarf::SourceLanguage>(*LC))))
-          if (LB && *LB == *DefaultLB)
-            LB = std::nullopt;
+    if (std::optional<uint64_t> LV = D.getLanguage())
+      if ((DefaultLB =
+               LanguageLowerBound(static_cast<dwarf::SourceLanguage>(*LV))))
+        if (LB && *LB == *DefaultLB)
+          LB = std::nullopt;
     if (!LB && !Count && !UB)
       OS << "[]";
     else if (!LB && (Count || UB) && DefaultLB)
@@ -151,6 +150,16 @@ template <typename DieType>
 DieType resolveReferencedType(DieType D, typename DieType::DWARFFormValue F) {
   return D.resolveReferencedType(F);
 }
+template <typename DWARFFormValueType>
+const char *toString(std::optional<DWARFFormValueType> F) {
+  if (F) {
+    llvm::Expected<const char *> E = F->getAsCString();
+    if (E)
+      return *E;
+    llvm::consumeError(E.takeError());
+  }
+  return nullptr;
+}
 } // namespace detail
 
 template <typename DieType>
@@ -240,7 +249,7 @@ DieType DWARFTypePrinter<DieType>::appendUnqualifiedNameBefore(
     appendConstVolatileQualifierBefore(D);
     break;
   case dwarf::DW_TAG_namespace: {
-    if (const char *Name = dwarf::toString(D.find(dwarf::DW_AT_name), nullptr))
+    if (const char *Name = detail::toString(D.find(dwarf::DW_AT_name)))
       OS << Name;
     else
       OS << "(anonymous namespace)";
@@ -262,7 +271,7 @@ DieType DWARFTypePrinter<DieType>::appendUnqualifiedNameBefore(
   case DW_TAG_base_type:
   */
   default: {
-    const char *NamePtr = dwarf::toString(D.find(dwarf::DW_AT_name), nullptr);
+    const char *NamePtr = detail::toString(D.find(dwarf::DW_AT_name));
     if (!NamePtr) {
       appendTypeTagName(D.getTag());
       return DieType();
@@ -441,7 +450,7 @@ bool DWARFTypePrinter<DieType>::appendTemplateParameters(DieType D,
       if (T.getTag() == dwarf::DW_TAG_pointer_type ||
           T.getTag() == dwarf::DW_TAG_reference_type)
         continue;
-      const char *RawName = dwarf::toString(T.find(dwarf::DW_AT_name), nullptr);
+      const char *RawName = detail::toString(T.find(dwarf::DW_AT_name));
       assert(RawName);
       StringRef Name = RawName;
       auto V = C.find(dwarf::DW_AT_const_value);
@@ -534,7 +543,7 @@ bool DWARFTypePrinter<DieType>::appendTemplateParameters(DieType D,
     }
     if (C.getTag() == dwarf::DW_TAG_GNU_template_template_param) {
       const char *RawName =
-          dwarf::toString(C.find(dwarf::DW_AT_GNU_template_name), nullptr);
+          detail::toString(C.find(dwarf::DW_AT_GNU_template_name));
       assert(RawName);
       StringRef Name = RawName;
       Sep();
@@ -594,7 +603,7 @@ void DWARFTypePrinter<DieType>::appendConstVolatileQualifierAfter(DieType N) {
   decomposeConstVolatile(N, T, C, V);
   if (T && T.getTag() == dwarf::DW_TAG_subroutine_type)
     appendSubroutineNameAfter(T, detail::resolveReferencedType(T), false,
-                              C.isValid(), V.isValid());
+                              static_cast<bool>(C), static_cast<bool>(V));
   else
     appendUnqualifiedNameAfter(T, detail::resolveReferencedType(T));
 }
diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
index df2e806259b36..5747ad99d0f13 100644
--- a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
+++ b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
@@ -120,8 +120,7 @@ class LLVMSymbolizer {
   void pruneCache();
 
   static std::string
-  DemangleName(const std::string &Name,
-               const SymbolizableModule *DbiModuleDescriptor);
+  DemangleName(StringRef Name, const SymbolizableModule *DbiModuleDescriptor);
 
   void setBuildIDFetcher(std::unique_ptr<BuildIDFetcher> Fetcher) {
     BIDFetcher = std::move(Fetcher);
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index 6022a8d276de4..11376955e3f39 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -2087,17 +2087,23 @@ class SizeofParamPackExpr : public Node {
 class CallExpr : public Node {
   const Node *Callee;
   NodeArray Args;
+  bool IsParen; // (func)(args ...) ?
 
 public:
-  CallExpr(const Node *Callee_, NodeArray Args_, Prec Prec_)
-      : Node(KCallExpr, Prec_), Callee(Callee_), Args(Args_) {}
+  CallExpr(const Node *Callee_, NodeArray Args_, bool IsParen_, Prec Prec_)
+      : Node(KCallExpr, Prec_), Callee(Callee_), Args(Args_),
+        IsParen(IsParen_) {}
 
   template <typename Fn> void match(Fn F) const {
-    F(Callee, Args, getPrecedence());
+    F(Callee, Args, IsParen, getPrecedence());
   }
 
   void printLeft(OutputBuffer &OB) const override {
+    if (IsParen)
+      OB.printOpen();
     Callee->print(OB);
+    if (IsParen)
+      OB.printClose();
     OB.printOpen();
     Args.printWithComma(OB);
     OB.printClose();
@@ -3373,9 +3379,12 @@ const typename AbstractManglingParser<
      "operator co_await"},
     {"az", OperatorInfo::OfIdOp, /*Type*/ false, Node::Prec::Unary, "alignof "},
     {"cc", OperatorInfo::NamedCast, false, Node::Prec::Postfix, "const_cast"},
-    {"cl", OperatorInfo::Call, false, Node::Prec::Postfix, "operator()"},
+    {"cl", OperatorInfo::Call, /*Paren*/ false, Node::Prec::Postfix,
+     "operator()"},
     {"cm", OperatorInfo::Binary, false, Node::Prec::Comma, "operator,"},
     {"co", OperatorInfo::Prefix, false, Node::Prec::Unary, "operator~"},
+    {"cp", OperatorInfo::Call, /*Paren*/ true, Node::Prec::Postfix,
+     "operator()"},
     {"cv", OperatorInfo::CCast, false, Node::Prec::Cast, "operator"}, // C Cast
     {"dV", OperatorInfo::Binary, false, Node::Prec::Assign, "operator/="},
     {"da", OperatorInfo::Del, /*Ary*/ true, Node::Prec::Unary,
@@ -5118,6 +5127,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseRequiresExpr() {
 //              ::= <binary operator-name> <expression> <expression>
 //              ::= <ternary operator-name> <expression> <expression> <expression>
 //              ::= cl <expression>+ E                                   # call
+//              ::= cp <base-unresolved-name> <expression>* E            # (name) (expr-list), call that would use argument-dependent lookup but for the parentheses
 //              ::= cv <type> <expression>                               # conversion with one argument
 //              ::= cv <type> _ <expression>* E                          # conversion with a different number of arguments
 //              ::= [gs] nw <expression>* _ <type> E                     # new (expr-list) type
@@ -5253,7 +5263,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() {
         Names.push_back(E);
       }
       return make<CallExpr>(Callee, popTrailingNodeArray(ExprsBegin),
-                            Op->getPrecedence());
+                            /*IsParen=*/Op->getFlag(), Op->getPrecedence());
     }
     case OperatorInfo::CCast: {
       // C Cast: (type)expr
@@ -5440,7 +5450,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() {
       }
     }
     return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin),
-                          Node::Prec::Postfix);
+                          /*IsParen=*/false, Node::Prec::Postfix);
   }
 
   // Only unresolved names remain.
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/COFF.h b/llvm/include/llvm/ExecutionEngine/JITLink/COFF.h
index 87d3648d37e8b..33b661933ace8 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/COFF.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/COFF.h
@@ -24,7 +24,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromCOFFObject(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromCOFFObject(MemoryBufferRef ObjectBuffer,
+                              std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// Link the given graph.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/COFF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/COFF_x86_64.h
index fff32d6d96093..2072ae9dfdbe7 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/COFF_x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/COFF_x86_64.h
@@ -23,8 +23,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromCOFFObject_x86_64(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromCOFFObject_x86_64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a COFF x86-64 object file.
 void link_COFF_x86_64(std::unique_ptr<LinkGraph> G,
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h
index 038591f9add05..3decba65f380c 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h
@@ -24,7 +24,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer,
+                             std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// Link the given graph.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch32.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch32.h
index 25d1c3aac2c26..b865414e520c2 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch32.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch32.h
@@ -24,8 +24,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_aarch32(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be an ELF arm/thumb object
 /// file.
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h
index 50eb598139ea0..45a7a0100593f 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h
@@ -25,8 +25,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_aarch64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a ELF aarch64 relocatable
 /// object file.
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_i386.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_i386.h
index 44ebd96994611..0752f214d9d58 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_i386.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_i386.h
@@ -26,7 +26,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer,
+                                  std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a ELF i386 relocatable
 /// object file.
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
index 4d7655c4b988b..7e5d0f1f91852 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
@@ -25,8 +25,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_loongarch(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be an ELF loongarch object
 /// file.
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_ppc64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_ppc64.h
index 8db986a4a9fa1..c5049a54cdf1d 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_ppc64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_ppc64.h
@@ -25,15 +25,16 @@ namespace llvm::jitlink {
 ///
 /// WARNING: The big-endian backend has not been tested yet.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer,
+                                   std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// Create a LinkGraph from an ELF/ppc64le relocatable object.
 ///
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_ppc64le(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_ppc64le(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a ELF ppc64le object file.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h
index a0e573baca066..d00b5c2868baf 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h
@@ -26,7 +26,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer,
+                                   std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a ELF riscv object file.
 void link_ELF_riscv(std::unique_ptr<LinkGraph> G,
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
index fbe5765438d24..c4f2c532de745 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
@@ -24,7 +24,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer,
+                                    std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a ELF x86-64 object file.
 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 9844214c537a0..df347049e85d8 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -23,6 +23,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamWriter.h"
@@ -393,12 +394,10 @@ const char *getLinkageName(Linkage L);
 /// Defines the scope in which this symbol should be visible:
 ///   Default -- Visible in the public interface of the linkage unit.
 ///   Hidden -- Visible within the linkage unit, but not exported from it.
+///   SideEffectsOnly -- Like hidden, but symbol can only be looked up once
+///                      to trigger materialization of the containing graph.
 ///   Local -- Visible only within the LinkGraph.
-enum class Scope : uint8_t {
-  Default,
-  Hidden,
-  Local
-};
+enum class Scope : uint8_t { Default, Hidden, SideEffectsOnly, Local };
 
 /// For debugging output.
 const char *getScopeName(Scope S);
@@ -422,10 +421,11 @@ class Symbol {
   friend class LinkGraph;
 
 private:
-  Symbol(Addressable &Base, orc::ExecutorAddrDiff Offset, StringRef Name,
-         orc::ExecutorAddrDiff Size, Linkage L, Scope S, bool IsLive,
-         bool IsCallable)
-      : Name(Name), Base(&Base), Offset(Offset), WeakRef(0), Size(Size) {
+  Symbol(Addressable &Base, orc::ExecutorAddrDiff Offset,
+         orc::SymbolStringPtr &&Name, orc::ExecutorAddrDiff Size, Linkage L,
+         Scope S, bool IsLive, bool IsCallable)
+      : Name(std::move(Name)), Base(&Base), Offset(Offset), WeakRef(0),
+        Size(Size) {
     assert(Offset <= MaxOffset && "Offset out of range");
     setLinkage(L);
     setScope(S);
@@ -435,26 +435,29 @@ class Symbol {
   }
 
   static Symbol &constructExternal(BumpPtrAllocator &Allocator,
-                                   Addressable &Base, StringRef Name,
+                                   Addressable &Base,
+                                   orc::SymbolStringPtr &&Name,
                                    orc::ExecutorAddrDiff Size, Linkage L,
                                    bool WeaklyReferenced) {
     assert(!Base.isDefined() &&
            "Cannot create external symbol from defined block");
-    assert(!Name.empty() && "External symbol name cannot be empty");
+    assert(Name && "External symbol name cannot be empty");
     auto *Sym = Allocator.Allocate<Symbol>();
-    new (Sym) Symbol(Base, 0, Name, Size, L, Scope::Default, false, false);
+    new (Sym)
+        Symbol(Base, 0, std::move(Name), Size, L, Scope::Default, false, false);
     Sym->setWeaklyReferenced(WeaklyReferenced);
     return *Sym;
   }
 
   static Symbol &constructAbsolute(BumpPtrAllocator &Allocator,
-                                   Addressable &Base, StringRef Name,
+                                   Addressable &Base,
+                                   orc::SymbolStringPtr &&Name,
                                    orc::ExecutorAddrDiff Size, Linkage L,
                                    Scope S, bool IsLive) {
     assert(!Base.isDefined() &&
            "Cannot create absolute symbol from a defined block");
     auto *Sym = Allocator.Allocate<Symbol>();
-    new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false);
+    new (Sym) Symbol(Base, 0, std::move(Name), Size, L, S, IsLive, false);
     return *Sym;
   }
 
@@ -465,20 +468,22 @@ class Symbol {
     assert((Offset + Size) <= Base.getSize() &&
            "Symbol extends past end of block");
     auto *Sym = Allocator.Allocate<Symbol>();
-    new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong,
-                     Scope::Local, IsLive, IsCallable);
+    new (Sym) Symbol(Base, Offset, nullptr, Size, Linkage::Strong, Scope::Local,
+                     IsLive, IsCallable);
     return *Sym;
   }
 
   static Symbol &constructNamedDef(BumpPtrAllocator &Allocator, Block &Base,
-                                   orc::ExecutorAddrDiff Offset, StringRef Name,
+                                   orc::ExecutorAddrDiff Offset,
+                                   orc::SymbolStringPtr Name,
                                    orc::ExecutorAddrDiff Size, Linkage L,
                                    Scope S, bool IsLive, bool IsCallable) {
     assert((Offset + Size) <= Base.getSize() &&
            "Symbol extends past end of block");
-    assert(!Name.empty() && "Name cannot be empty");
+    assert(Name && "Name cannot be empty");
     auto *Sym = Allocator.Allocate<Symbol>();
-    new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable);
+    new (Sym)
+        Symbol(Base, Offset, std::move(Name), Size, L, S, IsLive, IsCallable);
     return *Sym;
   }
 
@@ -495,18 +500,19 @@ class Symbol {
   Symbol &operator=(Symbol &&) = delete;
 
   /// Returns true if this symbol has a name.
-  bool hasName() const { return !Name.empty(); }
+  bool hasName() const { return Name != nullptr; }
 
   /// Returns the name of this symbol (empty if the symbol is anonymous).
-  StringRef getName() const {
-    assert((!Name.empty() || getScope() == Scope::Local) &&
+  const orc::SymbolStringPtr &getName() const {
+    assert((hasName() || getScope() == Scope::Local) &&
            "Anonymous symbol has non-local scope");
+
     return Name;
   }
 
   /// Rename this symbol. The client is responsible for updating scope and
   /// linkage if this name-change requires it.
-  void setName(StringRef Name) { this->Name = Name; }
+  void setName(const orc::SymbolStringPtr Name) { this->Name = Name; }
 
   /// Returns true if this Symbol has content (potentially) defined within this
   /// object file (i.e. is anything but an external or absolute symbol).
@@ -613,7 +619,7 @@ class Symbol {
 
   /// Set the linkage for this Symbol.
   void setLinkage(Linkage L) {
-    assert((L == Linkage::Strong || (!Base->isAbsolute() && !Name.empty())) &&
+    assert((L == Linkage::Strong || (!Base->isAbsolute() && Name)) &&
            "Linkage can only be applied to defined named symbols");
     this->L = static_cast<uint8_t>(L);
   }
@@ -623,7 +629,7 @@ class Symbol {
 
   /// Set the visibility for this Symbol.
   void setScope(Scope S) {
-    assert((!Name.empty() || S == Scope::Local) &&
+    assert((hasName() || S == Scope::Local) &&
            "Can not set anonymous symbol to non-local scope");
     assert((S != Scope::Local || Base->isDefined() || Base->isAbsolute()) &&
            "Invalid visibility for symbol type");
@@ -675,8 +681,7 @@ class Symbol {
 
   static constexpr uint64_t MaxOffset = (1ULL << 59) - 1;
 
-  // FIXME: A char* or SymbolStringPtr may pack better.
-  StringRef Name;
+  orc::SymbolStringPtr Name = nullptr;
   Addressable *Base = nullptr;
   uint64_t Offset : 57;
   uint64_t L : 1;
@@ -1001,22 +1006,23 @@ class LinkGraph {
 
   using GetEdgeKindNameFunction = const char *(*)(Edge::Kind);
 
-  LinkGraph(std::string Name, const Triple &TT, SubtargetFeatures Features,
-            unsigned PointerSize, llvm::endianness Endianness,
-            GetEdgeKindNameFunction GetEdgeKindName)
-      : Name(std::move(Name)), TT(TT), Features(std::move(Features)),
-        PointerSize(PointerSize), Endianness(Endianness),
-        GetEdgeKindName(std::move(GetEdgeKindName)) {}
-
-  LinkGraph(std::string Name, const Triple &TT, unsigned PointerSize,
+  LinkGraph(std::string Name, std::shared_ptr<orc::SymbolStringPool> SSP,
+            const Triple &TT, SubtargetFeatures Features, unsigned PointerSize,
             llvm::endianness Endianness,
             GetEdgeKindNameFunction GetEdgeKindName)
-      : LinkGraph(std::move(Name), TT, SubtargetFeatures(), PointerSize,
-                  Endianness, GetEdgeKindName) {}
+      : Name(std::move(Name)), SSP(std::move(SSP)), TT(TT),
+        Features(std::move(Features)), PointerSize(PointerSize),
+        Endianness(Endianness), GetEdgeKindName(std::move(GetEdgeKindName)) {}
 
-  LinkGraph(std::string Name, const Triple &TT,
+  LinkGraph(std::string Name, std::shared_ptr<orc::SymbolStringPool> SSP,
+            const Triple &TT, unsigned PointerSize, llvm::endianness Endianness,
             GetEdgeKindNameFunction GetEdgeKindName)
-      : LinkGraph(std::move(Name), TT, SubtargetFeatures(),
+      : LinkGraph(std::move(Name), std::move(SSP), TT, SubtargetFeatures(),
+                  PointerSize, Endianness, GetEdgeKindName) {}
+
+  LinkGraph(std::string Name, std::shared_ptr<orc::SymbolStringPool> SSP,
+            const Triple &TT, GetEdgeKindNameFunction GetEdgeKindName)
+      : LinkGraph(std::move(Name), std::move(SSP), TT, SubtargetFeatures(),
                   Triple::getArchPointerBitWidth(TT.getArch()) / 8,
                   TT.isLittleEndian() ? endianness::little : endianness::big,
                   GetEdgeKindName) {
@@ -1028,6 +1034,7 @@ class LinkGraph {
   LinkGraph &operator=(const LinkGraph &) = delete;
   LinkGraph(LinkGraph &&) = delete;
   LinkGraph &operator=(LinkGraph &&) = delete;
+  ~LinkGraph();
 
   /// Returns the name of this graph (usually the name of the original
   /// underlying MemoryBuffer).
@@ -1047,6 +1054,8 @@ class LinkGraph {
 
   const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); }
 
+  std::shared_ptr<orc::SymbolStringPool> getSymbolStringPool() { return SSP; }
+
   /// Allocate a mutable buffer of the given size using the LinkGraph's
   /// allocator.
   MutableArrayRef<char> allocateBuffer(size_t Size) {
@@ -1260,6 +1269,10 @@ class LinkGraph {
     return splitBlockImpl(std::move(Blocks), Cache);
   }
 
+  //
+  orc::SymbolStringPtr intern(StringRef SymbolName) {
+    return SSP->intern(SymbolName);
+  }
   /// Add an external symbol.
   /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose
   /// size is not known, you should substitute '0'.
@@ -1268,18 +1281,25 @@ class LinkGraph {
   /// found or an error will be emitted. Externals that are weakly referenced
   /// are permitted to be undefined, in which case they are assigned an address
   /// of 0.
-  Symbol &addExternalSymbol(StringRef Name, orc::ExecutorAddrDiff Size,
+  Symbol &addExternalSymbol(orc::SymbolStringPtr Name,
+                            orc::ExecutorAddrDiff Size,
                             bool IsWeaklyReferenced) {
-    assert(!ExternalSymbols.contains(Name) && "Duplicate external symbol");
+    assert(!ExternalSymbols.contains(*Name) && "Duplicate external symbol");
     auto &Sym = Symbol::constructExternal(
-        Allocator, createAddressable(orc::ExecutorAddr(), false), Name, Size,
-        Linkage::Strong, IsWeaklyReferenced);
-    ExternalSymbols.insert({Sym.getName(), &Sym});
+        Allocator, createAddressable(orc::ExecutorAddr(), false),
+        std::move(Name), Size, Linkage::Strong, IsWeaklyReferenced);
+    ExternalSymbols.insert({*Sym.getName(), &Sym});
     return Sym;
   }
 
+  Symbol &addExternalSymbol(StringRef Name, orc::ExecutorAddrDiff Size,
+                            bool IsWeaklyReferenced) {
+    return addExternalSymbol(SSP->intern(Name), Size, IsWeaklyReferenced);
+  }
+
   /// Add an absolute symbol.
-  Symbol &addAbsoluteSymbol(StringRef Name, orc::ExecutorAddr Address,
+  Symbol &addAbsoluteSymbol(orc::SymbolStringPtr Name,
+                            orc::ExecutorAddr Address,
                             orc::ExecutorAddrDiff Size, Linkage L, Scope S,
                             bool IsLive) {
     assert((S == Scope::Local || llvm::count_if(AbsoluteSymbols,
@@ -1288,11 +1308,18 @@ class LinkGraph {
                                                }) == 0) &&
                                     "Duplicate absolute symbol");
     auto &Sym = Symbol::constructAbsolute(Allocator, createAddressable(Address),
-                                          Name, Size, L, S, IsLive);
+                                          std::move(Name), Size, L, S, IsLive);
     AbsoluteSymbols.insert(&Sym);
     return Sym;
   }
 
+  Symbol &addAbsoluteSymbol(StringRef Name, orc::ExecutorAddr Address,
+                            orc::ExecutorAddrDiff Size, Linkage L, Scope S,
+                            bool IsLive) {
+
+    return addAbsoluteSymbol(SSP->intern(Name), Address, Size, L, S, IsLive);
+  }
+
   /// Add an anonymous symbol.
   Symbol &addAnonymousSymbol(Block &Content, orc::ExecutorAddrDiff Offset,
                              orc::ExecutorAddrDiff Size, bool IsCallable,
@@ -1307,13 +1334,22 @@ class LinkGraph {
   Symbol &addDefinedSymbol(Block &Content, orc::ExecutorAddrDiff Offset,
                            StringRef Name, orc::ExecutorAddrDiff Size,
                            Linkage L, Scope S, bool IsCallable, bool IsLive) {
+    return addDefinedSymbol(Content, Offset, SSP->intern(Name), Size, L, S,
+                            IsCallable, IsLive);
+  }
+
+  Symbol &addDefinedSymbol(Block &Content, orc::ExecutorAddrDiff Offset,
+                           orc::SymbolStringPtr Name,
+                           orc::ExecutorAddrDiff Size, Linkage L, Scope S,
+                           bool IsCallable, bool IsLive) {
     assert((S == Scope::Local || llvm::count_if(defined_symbols(),
                                                 [&](const Symbol *Sym) {
                                                   return Sym->getName() == Name;
                                                 }) == 0) &&
            "Duplicate defined symbol");
-    auto &Sym = Symbol::constructNamedDef(Allocator, Content, Offset, Name,
-                                          Size, L, S, IsLive, IsCallable);
+    auto &Sym =
+        Symbol::constructNamedDef(Allocator, Content, Offset, std::move(Name),
+                                  Size, L, S, IsLive, IsCallable);
     Content.getSection().addSymbol(Sym);
     return Sym;
   }
@@ -1398,7 +1434,7 @@ class LinkGraph {
       Sec.removeSymbol(Sym);
       Sym.makeExternal(createAddressable(orc::ExecutorAddr(), false));
     }
-    ExternalSymbols.insert({Sym.getName(), &Sym});
+    ExternalSymbols.insert({*Sym.getName(), &Sym});
   }
 
   /// Make the given symbol an absolute with the given address (must not already
@@ -1412,10 +1448,10 @@ class LinkGraph {
   void makeAbsolute(Symbol &Sym, orc::ExecutorAddr Address) {
     assert(!Sym.isAbsolute() && "Symbol is already absolute");
     if (Sym.isExternal()) {
-      assert(ExternalSymbols.contains(Sym.getName()) &&
+      assert(ExternalSymbols.contains(*Sym.getName()) &&
              "Sym is not in the absolute symbols set");
       assert(Sym.getOffset() == 0 && "External is not at offset 0");
-      ExternalSymbols.erase(Sym.getName());
+      ExternalSymbols.erase(*Sym.getName());
       auto &A = Sym.getAddressable();
       A.setAbsolute(true);
       A.setAddress(Address);
@@ -1440,9 +1476,9 @@ class LinkGraph {
              "Symbol is not in the absolutes set");
       AbsoluteSymbols.erase(&Sym);
     } else {
-      assert(ExternalSymbols.contains(Sym.getName()) &&
+      assert(ExternalSymbols.contains(*Sym.getName()) &&
              "Symbol is not in the externals set");
-      ExternalSymbols.erase(Sym.getName());
+      ExternalSymbols.erase(*Sym.getName());
     }
     Addressable &OldBase = *Sym.Base;
     Sym.setBlock(Content);
@@ -1527,9 +1563,9 @@ class LinkGraph {
   void removeExternalSymbol(Symbol &Sym) {
     assert(!Sym.isDefined() && !Sym.isAbsolute() &&
            "Sym is not an external symbol");
-    assert(ExternalSymbols.contains(Sym.getName()) &&
+    assert(ExternalSymbols.contains(*Sym.getName()) &&
            "Symbol is not in the externals set");
-    ExternalSymbols.erase(Sym.getName());
+    ExternalSymbols.erase(*Sym.getName());
     Addressable &Base = *Sym.Base;
     assert(llvm::none_of(external_symbols(),
                          [&](Symbol *AS) { return AS->Base == &Base; }) &&
@@ -1600,12 +1636,14 @@ class LinkGraph {
   BumpPtrAllocator Allocator;
 
   std::string Name;
+  std::shared_ptr<orc::SymbolStringPool> SSP;
   Triple TT;
   SubtargetFeatures Features;
   unsigned PointerSize;
   llvm::endianness Endianness;
   GetEdgeKindNameFunction GetEdgeKindName = nullptr;
   DenseMap<StringRef, std::unique_ptr<Section>> Sections;
+  // FIXME(jared): these should become dense maps
   ExternalSymbolMap ExternalSymbols;
   AbsoluteSymbolSet AbsoluteSymbols;
   orc::shared::AllocActions AAs;
@@ -1828,7 +1866,8 @@ enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol };
 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF);
 
 /// A map of symbol names to resolved addresses.
-using AsyncLookupResult = DenseMap<StringRef, orc::ExecutorSymbolDef>;
+using AsyncLookupResult =
+    DenseMap<orc::SymbolStringPtr, orc::ExecutorSymbolDef>;
 
 /// A function object to call with a resolved symbol map (See AsyncLookupResult)
 /// or an error if resolution failed.
@@ -1861,7 +1900,7 @@ createLookupContinuation(Continuation Cont) {
 /// Holds context for a single jitLink invocation.
 class JITLinkContext {
 public:
-  using LookupMap = DenseMap<StringRef, SymbolLookupFlags>;
+  using LookupMap = DenseMap<orc::SymbolStringPtr, SymbolLookupFlags>;
 
   /// Create a JITLinkContext.
   JITLinkContext(const JITLinkDylib *JD) : JD(JD) {}
@@ -1994,11 +2033,14 @@ void visitExistingEdges(LinkGraph &G, VisitorTs &&...Vs) {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromObject(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
+                          std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// Create a \c LinkGraph defining the given absolute symbols.
-std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
-                                                    orc::SymbolMap Symbols);
+std::unique_ptr<LinkGraph>
+absoluteSymbolsLinkGraph(const Triple &TT,
+                         std::shared_ptr<orc::SymbolStringPool> SSP,
+                         orc::SymbolMap Symbols);
 
 /// Link the given graph.
 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx);
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
index 75f9f4bbe6145..1f8bab24c4827 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
@@ -320,12 +321,15 @@ class SimpleSegmentAlloc {
   using OnFinalizedFunction =
       JITLinkMemoryManager::InFlightAlloc::OnFinalizedFunction;
 
-  static void Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
-                     SegmentMap Segments, OnCreatedFunction OnCreated);
+  static void Create(JITLinkMemoryManager &MemMgr,
+                     std::shared_ptr<orc::SymbolStringPool> SSP,
+                     const JITLinkDylib *JD, SegmentMap Segments,
+                     OnCreatedFunction OnCreated);
 
-  static Expected<SimpleSegmentAlloc> Create(JITLinkMemoryManager &MemMgr,
-                                             const JITLinkDylib *JD,
-                                             SegmentMap Segments);
+  static Expected<SimpleSegmentAlloc>
+  Create(JITLinkMemoryManager &MemMgr,
+         std::shared_ptr<orc::SymbolStringPool> SSP, const JITLinkDylib *JD,
+         SegmentMap Segments);
 
   SimpleSegmentAlloc(SimpleSegmentAlloc &&);
   SimpleSegmentAlloc &operator=(SimpleSegmentAlloc &&);
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
index bb8da0ab9db27..2010b32cdf766 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
@@ -25,7 +25,8 @@ namespace jitlink {
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer);
+createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer,
+                               std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given ObjBuffer, which must be a MachO object file.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
index 31721bf999ec1..31770c2a601e5 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
@@ -23,8 +23,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_arm64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a MachO arm64 object file.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
index 6aee8c354f91e..72e9883ba18b8 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
@@ -23,8 +23,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer);
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given LinkGraph.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/TableManager.h b/llvm/include/llvm/ExecutionEngine/JITLink/TableManager.h
index 7ab8ae3e53cec..3d068d97202d6 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/TableManager.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/TableManager.h
@@ -69,7 +69,7 @@ template <typename TableManagerImplT> class TableManager {
 
 private:
   TableManagerImplT &impl() { return static_cast<TableManagerImplT &>(*this); }
-  DenseMap<StringRef, Symbol *> Entries;
+  DenseMap<orc::SymbolStringPtr, Symbol *> Entries;
 };
 
 } // namespace jitlink
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index db440c378d24f..1fa957178a122 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -755,6 +755,32 @@ inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
       sizeof(PointerJumpStubContent), true, false);
 }
 
+/// AArch64 reentry trampoline.
+///
+/// Contains the instruction sequence for a trampoline that stores its return
+/// address on the stack and passes its own address in x0:
+///   STP  x29, x30, [sp, #-16]!
+///   BL   <reentry-symbol>
+extern const char ReentryTrampolineContent[8];
+
+/// Create a block of N reentry trampolines.
+inline Block &createReentryTrampolineBlock(LinkGraph &G,
+                                           Section &TrampolineSection,
+                                           Symbol &ReentrySymbol) {
+  auto &B = G.createContentBlock(TrampolineSection, ReentryTrampolineContent,
+                                 orc::ExecutorAddr(~uint64_t(7)), 4, 0);
+  B.addEdge(Branch26PCRel, 4, ReentrySymbol, 0);
+  return B;
+}
+
+inline Symbol &createAnonymousReentryTrampoline(LinkGraph &G,
+                                                Section &TrampolineSection,
+                                                Symbol &ReentrySymbol) {
+  return G.addAnonymousSymbol(
+      createReentryTrampolineBlock(G, TrampolineSection, ReentrySymbol), 0,
+      sizeof(ReentryTrampolineContent), true, false);
+}
+
 /// Global Offset Table Builder.
 class GOTTableManager : public TableManager<GOTTableManager> {
 public:
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index e892005c53d8e..9e3e46285c408 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -51,6 +51,26 @@ enum class SymbolState : uint8_t;
 using ResourceTrackerSP = IntrusiveRefCntPtr<ResourceTracker>;
 using JITDylibSP = IntrusiveRefCntPtr<JITDylib>;
 
+/// A definition of a Symbol within a JITDylib.
+class SymbolInstance {
+public:
+  using LookupAsyncOnCompleteFn =
+      unique_function<void(Expected<ExecutorSymbolDef>)>;
+
+  SymbolInstance(JITDylibSP JD, SymbolStringPtr Name)
+      : JD(std::move(JD)), Name(std::move(Name)) {}
+
+  const JITDylib &getJITDylib() const { return *JD; }
+  const SymbolStringPtr &getName() const { return Name; }
+
+  Expected<ExecutorSymbolDef> lookup() const;
+  void lookupAsync(LookupAsyncOnCompleteFn OnComplete) const;
+
+private:
+  JITDylibSP JD;
+  SymbolStringPtr Name;
+};
+
 using ResourceKey = uintptr_t;
 
 /// API to remove / transfer ownership of JIT resources.
@@ -550,6 +570,9 @@ class MaterializationResponsibility {
   ///        emitted or notified of an error.
   ~MaterializationResponsibility();
 
+  /// Return the ResourceTracker associated with this instance.
+  const ResourceTrackerSP &getResourceTracker() const { return RT; }
+
   /// Runs the given callback under the session lock, passing in the associated
   /// ResourceKey. This is the safe way to associate resources with trackers.
   template <typename Func> Error withResourceKeyDo(Func &&F) const {
@@ -1748,6 +1771,10 @@ class ExecutionSession {
       JITDispatchHandlers;
 };
 
+inline Expected<ExecutorSymbolDef> SymbolInstance::lookup() const {
+  return JD->getExecutionSession().lookup({JD.get()}, Name);
+}
+
 template <typename Func> Error ResourceTracker::withResourceKeyDo(Func &&F) {
   return getJITDylib().getExecutionSession().runSessionLocked([&]() -> Error {
     if (isDefunct())
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
index 035139578e08f..fb66bf812f610 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
@@ -29,12 +29,6 @@ namespace orc {
 
 // --raw_ostream operators for ORC types--
 
-/// Render a SymbolStringPtr.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym);
-
-/// Render a NonOwningSymbolStringPtr.
-raw_ostream &operator<<(raw_ostream &OS, NonOwningSymbolStringPtr Sym);
-
 /// Render a SymbolNameSet.
 raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
index 54442c91096b3..3da5e90a0ec5b 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -138,6 +138,11 @@ class ELFNixPlatform : public Platform {
   static ArrayRef<std::pair<const char *, const char *>>
   standardRuntimeUtilityAliases();
 
+  /// Returns a list of aliases required to enable lazy compilation via the
+  /// ORC runtime.
+  static ArrayRef<std::pair<const char *, const char *>>
+  standardLazyCompilationAliases();
+
 private:
   // Data needed for bootstrap only.
   struct BootstrapInfo {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
index 5d5326c4a469e..dcf5592f1717c 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
@@ -181,7 +181,7 @@ class ExecutorProcessControl {
 
   ExecutorProcessControl(std::shared_ptr<SymbolStringPool> SSP,
                          std::unique_ptr<TaskDispatcher> D)
-    : SSP(std::move(SSP)), D(std::move(D)) {}
+      : SSP(std::move(SSP)), D(std::move(D)) {}
 
   virtual ~ExecutorProcessControl();
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index 14e9c0610bf6a..a8129d70572c0 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -135,7 +135,7 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
   LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
       : ResolveLanding(std::move(ResolveLanding)) {
 
-    ErrorAsOutParameter _(&Err);
+    ErrorAsOutParameter _(Err);
 
     /// Try to set up the resolver block.
     std::error_code EC;
@@ -262,7 +262,7 @@ class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
     using NotifyLandingResolvedFunction =
         TrampolinePool::NotifyLandingResolvedFunction;
 
-    ErrorAsOutParameter _(&Err);
+    ErrorAsOutParameter _(Err);
     auto TP = LocalTrampolinePool<ORCABI>::Create(
         [this](ExecutorAddr TrampolineAddr,
                NotifyLandingResolvedFunction NotifyLandingResolved) {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h
new file mode 100644
index 0000000000000..19075c76a6073
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h
@@ -0,0 +1,26 @@
+//===- JITLinkLazyCallThroughManager.h - JITLink based laziness -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redirectable Symbol Manager implementation using JITLink
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
+
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/Support/StringSaver.h"
+
+#include <atomic>
+
+namespace llvm {
+namespace orc {} // namespace orc
+} // namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
index 81d1d154d5600..83339e56cfa5f 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -39,12 +39,6 @@ class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager {
             ObjLinkingLayer, AnonymousPtrCreator, PtrJumpStubCreator));
   }
 
-  void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
-                               SymbolMap InitialDests) override;
-
-  Error redirect(JITDylib &JD, const SymbolMap &NewDests) override;
-
-private:
   JITLinkRedirectableSymbolManager(
       ObjectLinkingLayer &ObjLinkingLayer,
       jitlink::AnonymousPointerCreator &AnonymousPtrCreator,
@@ -53,6 +47,14 @@ class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager {
         AnonymousPtrCreator(std::move(AnonymousPtrCreator)),
         PtrJumpStubCreator(std::move(PtrJumpStubCreator)) {}
 
+  ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; }
+
+  void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
+                               SymbolMap InitialDests) override;
+
+  Error redirect(JITDylib &JD, const SymbolMap &NewDests) override;
+
+private:
   ObjectLinkingLayer &ObjLinkingLayer;
   jitlink::AnonymousPointerCreator AnonymousPtrCreator;
   jitlink::PointerJumpStubCreator PtrJumpStubCreator;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
new file mode 100644
index 0000000000000..673019b748b3b
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
@@ -0,0 +1,72 @@
+//===- JITLinkReentryTrampolines.h -- JITLink-based trampolines -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Emit reentry trampolines via JITLink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm::jitlink {
+class Block;
+class LinkGraph;
+class Section;
+class Symbol;
+} // namespace llvm::jitlink
+
+namespace llvm::orc {
+
+class ObjectLinkingLayer;
+class RedirectableSymbolManager;
+
+/// Produces trampolines on request using JITLink.
+class JITLinkReentryTrampolines {
+public:
+  using EmitTrampolineFn = unique_function<jitlink::Symbol &(
+      jitlink::LinkGraph &G, jitlink::Section &Sec,
+      jitlink::Symbol &ReentrySym)>;
+  using OnTrampolinesReadyFn = unique_function<void(
+      Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;
+
+  /// Create trampolines using the default reentry trampoline function for
+  /// the session triple.
+  static Expected<std::unique_ptr<JITLinkReentryTrampolines>>
+  Create(ObjectLinkingLayer &ObjLinkingLayer);
+
+  JITLinkReentryTrampolines(ObjectLinkingLayer &ObjLinkingLayer,
+                            EmitTrampolineFn EmitTrampoline);
+  JITLinkReentryTrampolines(JITLinkReentryTrampolines &&) = delete;
+  JITLinkReentryTrampolines &operator=(JITLinkReentryTrampolines &&) = delete;
+
+  void emit(ResourceTrackerSP RT, size_t NumTrampolines,
+            OnTrampolinesReadyFn OnTrampolinesReady);
+
+private:
+  class TrampolineAddrScraperPlugin;
+
+  ObjectLinkingLayer &ObjLinkingLayer;
+  TrampolineAddrScraperPlugin *TrampolineAddrScraper = nullptr;
+  EmitTrampolineFn EmitTrampoline;
+  std::atomic<size_t> ReentryGraphIdx{0};
+};
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
+                                  RedirectableSymbolManager &RSMgr,
+                                  JITDylib &PlatformJD);
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
index 800f6773f16f5..96223d71e4f6a 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
@@ -18,14 +18,13 @@
 namespace llvm::orc {
 
 class ObjectLinkingLayer;
-class LazyCallThroughManager;
+class LazyReexportsManager;
 class RedirectableSymbolManager;
 
 class LazyObjectLinkingLayer : public ObjectLayer {
 public:
   LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
-                         LazyCallThroughManager &LCTMgr,
-                         RedirectableSymbolManager &RSMgr);
+                         LazyReexportsManager &LRMgr);
 
   llvm::Error add(llvm::orc::ResourceTrackerSP RT,
                   std::unique_ptr<llvm::MemoryBuffer> O,
@@ -38,8 +37,7 @@ class LazyObjectLinkingLayer : public ObjectLayer {
   class RenamerPlugin;
 
   ObjectLinkingLayer &BaseLayer;
-  LazyCallThroughManager &LCTMgr;
-  RedirectableSymbolManager &RSMgr;
+  LazyReexportsManager &LRMgr;
 };
 
 } // namespace llvm::orc
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index 6a43cb6fb6ca9..0dcf646b12dd8 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -173,6 +173,70 @@ lazyReexports(LazyCallThroughManager &LCTManager,
       LCTManager, RSManager, SourceJD, std::move(CallableAliases), SrcJDLoc);
 }
 
+class LazyReexportsManager {
+
+  friend std::unique_ptr<MaterializationUnit>
+  lazyReexports(LazyReexportsManager &, SymbolAliasMap);
+
+public:
+  using OnTrampolinesReadyFn = unique_function<void(
+      Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;
+  using EmitTrampolinesFn =
+      unique_function<void(ResourceTrackerSP RT, size_t NumTrampolines,
+                           OnTrampolinesReadyFn OnTrampolinesReady)>;
+
+  /// Create a LazyReexportsManager that uses the ORC runtime for reentry.
+  /// This will work both in-process and out-of-process.
+  static Expected<std::unique_ptr<LazyReexportsManager>>
+  Create(EmitTrampolinesFn EmitTrampolines, RedirectableSymbolManager &RSMgr,
+         JITDylib &PlatformJD);
+
+  LazyReexportsManager(LazyReexportsManager &&) = delete;
+  LazyReexportsManager &operator=(LazyReexportsManager &&) = delete;
+
+private:
+  struct CallThroughInfo {
+    SymbolStringPtr Name;
+    SymbolStringPtr BodyName;
+    JITDylibSP JD;
+  };
+
+  class MU;
+  class Plugin;
+
+  using ResolveSendResultFn =
+      unique_function<void(Expected<ExecutorSymbolDef>)>;
+
+  LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
+                       RedirectableSymbolManager &RSMgr, JITDylib &PlatformJD,
+                       Error &Err);
+
+  std::unique_ptr<MaterializationUnit>
+  createLazyReexports(SymbolAliasMap Reexports);
+
+  void emitReentryTrampolines(std::unique_ptr<MaterializationResponsibility> MR,
+                              SymbolAliasMap Reexports);
+  void emitRedirectableSymbols(
+      std::unique_ptr<MaterializationResponsibility> MR,
+      SymbolAliasMap Reexports,
+      Expected<std::vector<ExecutorSymbolDef>> ReentryPoints);
+  void resolve(ResolveSendResultFn SendResult, ExecutorAddr ReentryStubAddr);
+
+  EmitTrampolinesFn EmitTrampolines;
+  RedirectableSymbolManager &RSMgr;
+
+  std::mutex M;
+  DenseMap<ExecutorAddr, CallThroughInfo> CallThroughs;
+};
+
+/// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export
+/// is a callable symbol that will look up and dispatch to the given aliasee on
+/// first call. All subsequent calls will go directly to the aliasee.
+inline std::unique_ptr<MaterializationUnit>
+lazyReexports(LazyReexportsManager &LRM, SymbolAliasMap Reexports) {
+  return LRM.createLazyReexports(std::move(Reexports));
+}
+
 } // End namespace orc
 } // End namespace llvm
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
index 19f935d665823..1f11d9f61f6a1 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
@@ -176,6 +176,11 @@ class MachOPlatform : public Platform {
   static ArrayRef<std::pair<const char *, const char *>>
   standardRuntimeUtilityAliases();
 
+  /// Returns a list of aliases required to enable lazy compilation via the
+  /// ORC runtime.
+  static ArrayRef<std::pair<const char *, const char *>>
+  standardLazyCompilationAliases();
+
 private:
   using SymbolTableVector = SmallVector<
       std::tuple<ExecutorAddr, ExecutorAddr, MachOExecutorSymbolFlags>>;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
index 868689f38d89f..f3d4c76938778 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
@@ -23,6 +23,7 @@ namespace orc {
 class RedirectionManager {
 public:
   virtual ~RedirectionManager() = default;
+
   /// Change the redirection destination of given symbols to new destination
   /// symbols.
   virtual Error redirect(JITDylib &JD, const SymbolMap &NewDests) = 0;
@@ -31,7 +32,7 @@ class RedirectionManager {
   /// symbol.
   Error redirect(JITDylib &JD, SymbolStringPtr Symbol,
                  ExecutorSymbolDef NewDest) {
-    return redirect(JD, {{Symbol, NewDest}});
+    return redirect(JD, {{std::move(Symbol), NewDest}});
   }
 
 private:
@@ -50,7 +51,7 @@ class RedirectableSymbolManager : public RedirectionManager {
   /// desitnation symbol address.
   Error createRedirectableSymbol(ResourceTrackerSP RT, SymbolStringPtr Symbol,
                                  ExecutorSymbolDef InitialDest) {
-    return createRedirectableSymbols(RT, {{Symbol, InitialDest}});
+    return createRedirectableSymbols(RT, {{std::move(Symbol), InitialDest}});
   }
 
   /// Emit redirectable symbol
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 0c549bcbf0130..aed43f6308cba 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -42,6 +42,7 @@ extern const char *MemoryWriteUInt16sWrapperName;
 extern const char *MemoryWriteUInt32sWrapperName;
 extern const char *MemoryWriteUInt64sWrapperName;
 extern const char *MemoryWriteBuffersWrapperName;
+extern const char *MemoryWritePointersWrapperName;
 
 extern const char *RegisterEHFrameSectionWrapperName;
 extern const char *DeregisterEHFrameSectionWrapperName;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
index f47956a65f2e7..85f08b53f74a3 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
@@ -92,6 +92,9 @@ class SymbolStringPtrBase {
     return LHS.S < RHS.S;
   }
 
+  friend raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolStringPtrBase &Sym);
+
 #ifndef NDEBUG
   // Returns true if the pool entry's ref count is above zero (or if the entry
   // is an empty or tombstone value). Useful for debugging and testing -- this
diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
index 034c134a13731..92fdd376abd1d 100644
--- a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
+++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
@@ -10,6 +10,7 @@
 #define LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index ce09bb5f730a7..67632fb79f8aa 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -242,6 +242,7 @@ ENUM(MotionExpectation, Present);
 // V5.2: [15.9.1] `task-dependence-type` modifier
 ENUM(DependenceType, Depobj, In, Inout, Inoutset, Mutexinoutset, Out, Sink,
      Source);
+ENUM(Prescriptiveness, Strict);
 
 template <typename I, typename E> //
 struct LoopIterationT {
@@ -384,15 +385,13 @@ struct AllocatorT;
 // V5.2: [6.6] `allocate` clause
 template <typename T, typename I, typename E> //
 struct AllocateT {
-  using AllocatorSimpleModifier = E;
+  // AllocatorSimpleModifier is same as AllocatorComplexModifier.
   using AllocatorComplexModifier = AllocatorT<T, I, E>;
   using AlignModifier = AlignT<T, I, E>;
   using List = ObjectListT<I, E>;
 
   using TupleTrait = std::true_type;
-  std::tuple<OPT(AllocatorSimpleModifier), OPT(AllocatorComplexModifier),
-             OPT(AlignModifier), List>
-      t;
+  std::tuple<OPT(AllocatorComplexModifier), OPT(AlignModifier), List> t;
 };
 
 // V5.2: [6.4] `allocator` clause
@@ -645,7 +644,7 @@ struct FullT {
 // V5.2: [12.6.1] `grainsize` clause
 template <typename T, typename I, typename E> //
 struct GrainsizeT {
-  ENUM(Prescriptiveness, Strict);
+  using Prescriptiveness = type::Prescriptiveness;
   using GrainSize = E;
   using TupleTrait = std::true_type;
   std::tuple<OPT(Prescriptiveness), GrainSize> t;
@@ -878,8 +877,8 @@ struct NowaitT {
 // V5.2: [12.6.2] `num_tasks` clause
 template <typename T, typename I, typename E> //
 struct NumTasksT {
+  using Prescriptiveness = type::Prescriptiveness;
   using NumTasks = E;
-  ENUM(Prescriptiveness, Strict);
   using TupleTrait = std::true_type;
   std::tuple<OPT(Prescriptiveness), NumTasks> t;
 };
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index a97deafa3683c..1f0b129f867ae 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1262,12 +1262,15 @@ class OpenMPIRBuilder {
   ///                    cannot be resumed until execution of the structured
   ///                    block that is associated with the generated task is
   ///                    completed.
+  /// \param EventHandle If present, signifies the event handle as part of
+  ///			 the detach clause
   /// \param Mergeable	 If the given task is `mergeable`
   InsertPointOrErrorTy
   createTask(const LocationDescription &Loc, InsertPointTy AllocaIP,
              BodyGenCallbackTy BodyGenCB, bool Tied = true,
              Value *Final = nullptr, Value *IfCondition = nullptr,
-             SmallVector<DependData> Dependencies = {}, bool Mergeable = false);
+             SmallVector<DependData> Dependencies = {}, bool Mergeable = false,
+             Value *EventHandle = nullptr);
 
   /// Generator for the taskgroup construct
   ///
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 6f26f853eca03..928a03148c416 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -365,6 +365,9 @@ __OMP_RTL(__kmpc_omp_task_with_deps, false, Int32, IdentPtr, Int32,
 __OMP_RTL(__kmpc_taskloop, false, Void, IdentPtr, /* Int */ Int32, VoidPtr,
           /* Int */ Int32, Int64Ptr, Int64Ptr, Int64, /* Int */ Int32,
           /* Int */ Int32, Int64, VoidPtr)
+__OMP_RTL(__kmpc_taskloop_5, false, Void, IdentPtr, /* Int */ Int32, VoidPtr,
+          /* Int */ Int32, Int64Ptr, Int64Ptr, Int64, /* Int */ Int32,
+          /* Int */ Int32, Int64, Int32, VoidPtr)
 __OMP_RTL(__kmpc_omp_target_task_alloc, false, /* kmp_task_t */ VoidPtr,
           IdentPtr, Int32, Int32, SizeTy, SizeTy, TaskRoutineEntryPtr, Int64)
 __OMP_RTL(__kmpc_taskred_modifier_init, false, /* kmp_taskgroup */ VoidPtr,
diff --git a/llvm/include/llvm/IR/CmpPredicate.h b/llvm/include/llvm/IR/CmpPredicate.h
new file mode 100644
index 0000000000000..4b1be7beb2b66
--- /dev/null
+++ b/llvm/include/llvm/IR/CmpPredicate.h
@@ -0,0 +1,62 @@
+//===- CmpPredicate.h - CmpInst Predicate with samesign information -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A CmpInst::Predicate with any samesign information (applicable to ICmpInst).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_CMPPREDICATE_H
+#define LLVM_IR_CMPPREDICATE_H
+
+#include "llvm/IR/InstrTypes.h"
+
+namespace llvm {
+/// An abstraction over a floating-point predicate, and a pack of an integer
+/// predicate with samesign information. Some functions in ICmpInst construct
+/// and return this type in place of a Predicate.
+class CmpPredicate {
+  CmpInst::Predicate Pred;
+  bool HasSameSign;
+
+public:
+  /// Constructed implictly with a either Predicate and samesign information, or
+  /// just a Predicate, dropping samesign information.
+  CmpPredicate(CmpInst::Predicate Pred, bool HasSameSign = false)
+      : Pred(Pred), HasSameSign(HasSameSign) {
+    assert(!HasSameSign || CmpInst::isIntPredicate(Pred));
+  }
+
+  /// Implictly converts to the underlying Predicate, dropping samesign
+  /// information.
+  operator CmpInst::Predicate() const { return Pred; }
+
+  /// Query samesign information, for optimizations.
+  bool hasSameSign() const { return HasSameSign; }
+
+  /// Compares two CmpPredicates taking samesign into account and returns the
+  /// canonicalized CmpPredicate if they match. An alternative to operator==.
+  ///
+  /// For example,
+  ///   samesign ult + samesign ult -> samesign ult
+  ///   samesign ult + ult -> ult
+  ///   samesign ult + slt -> slt
+  ///   ult + ult -> ult
+  ///   ult + slt -> std::nullopt
+  static std::optional<CmpPredicate> getMatching(CmpPredicate A,
+                                                 CmpPredicate B);
+
+  /// An operator== on the underlying Predicate.
+  bool operator==(CmpInst::Predicate P) const { return Pred == P; }
+
+  /// There is no operator== defined on CmpPredicate. Use getMatching instead to
+  /// get the canonicalized matching CmpPredicate.
+  bool operator==(CmpPredicate) const = delete;
+};
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 93bd519f5727d..5e6a6198a9473 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -330,9 +330,6 @@ class DataLayout {
   /// the backends/clients are updated.
   unsigned getPointerSize(unsigned AS = 0) const;
 
-  /// Returns the maximum index size over all address spaces.
-  unsigned getMaxIndexSize() const;
-
   // Index size in bytes used for address calculation,
   /// rounded up to a whole number of bytes.
   unsigned getIndexSize(unsigned AS) const;
@@ -369,9 +366,7 @@ class DataLayout {
   }
 
   /// Returns the maximum index size over all address spaces.
-  unsigned getMaxIndexSizeInBits() const {
-    return getMaxIndexSize() * 8;
-  }
+  unsigned getMaxIndexSizeInBits() const;
 
   /// Size in bits of index used for address calculation in getelementptr.
   unsigned getIndexSizeInBits(unsigned AS) const {
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index faece276f3490..853b6adb96b77 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2131,20 +2131,21 @@ class IRBuilderBase {
     return CreateCast(Instruction::SIToFP, V, DestTy, Name);
   }
 
-  Value *CreateFPTrunc(Value *V, Type *DestTy,
-                       const Twine &Name = "") {
+  Value *CreateFPTrunc(Value *V, Type *DestTy, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
     if (IsFPConstrained)
       return CreateConstrainedFPCast(
-          Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
-          Name);
-    return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
+          Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr, Name,
+          FPMathTag);
+    return CreateCast(Instruction::FPTrunc, V, DestTy, Name, FPMathTag);
   }
 
-  Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
+  Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "",
+                     MDNode *FPMathTag = nullptr) {
     if (IsFPConstrained)
       return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
-                                     V, DestTy, nullptr, Name);
-    return CreateCast(Instruction::FPExt, V, DestTy, Name);
+                                     V, DestTy, nullptr, Name, FPMathTag);
+    return CreateCast(Instruction::FPExt, V, DestTy, Name, FPMathTag);
   }
 
   Value *CreatePtrToInt(Value *V, Type *DestTy,
@@ -2192,12 +2193,15 @@ class IRBuilderBase {
   }
 
   Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
-                    const Twine &Name = "") {
+                    const Twine &Name = "", MDNode *FPMathTag = nullptr) {
     if (V->getType() == DestTy)
       return V;
     if (Value *Folded = Folder.FoldCast(Op, V, DestTy))
       return Folded;
-    return Insert(CastInst::Create(Op, V, DestTy), Name);
+    Instruction *Cast = CastInst::Create(Op, V, DestTy);
+    if (isa<FPMathOperator>(Cast))
+      setFPAttrs(Cast, FPMathTag, FMF);
+    return Insert(Cast, Name);
   }
 
   Value *CreatePointerCast(Value *V, Type *DestTy,
@@ -2247,12 +2251,13 @@ class IRBuilderBase {
     return CreateBitCast(V, DestTy, Name);
   }
 
-  Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
+  Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "",
+                      MDNode *FPMathTag = nullptr) {
     Instruction::CastOps CastOp =
         V->getType()->getScalarSizeInBits() > DestTy->getScalarSizeInBits()
             ? Instruction::FPTrunc
             : Instruction::FPExt;
-    return CreateCast(CastOp, V, DestTy, Name);
+    return CreateCast(CastOp, V, DestTy, Name, FPMathTag);
   }
 
   CallInst *CreateConstrainedFPCast(
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 605964af5d676..a42bf6bca1b9f 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -24,6 +24,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/CFG.h"
+#include "llvm/IR/CmpPredicate.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GEPNoWrapFlags.h"
@@ -1203,6 +1204,33 @@ class ICmpInst: public CmpInst {
 #endif
   }
 
+  /// @returns the predicate along with samesign information.
+  CmpPredicate getCmpPredicate() const {
+    return {getPredicate(), hasSameSign()};
+  }
+
+  /// @returns the inverse predicate along with samesign information: static
+  /// variant.
+  static CmpPredicate getInverseCmpPredicate(CmpPredicate Pred) {
+    return {getInversePredicate(Pred), Pred.hasSameSign()};
+  }
+
+  /// @returns the inverse predicate along with samesign information.
+  CmpPredicate getInverseCmpPredicate() const {
+    return getInverseCmpPredicate(getCmpPredicate());
+  }
+
+  /// @returns the swapped predicate along with samesign information: static
+  /// variant.
+  static CmpPredicate getSwappedCmpPredicate(CmpPredicate Pred) {
+    return {getSwappedPredicate(Pred), Pred.hasSameSign()};
+  }
+
+  /// @returns the swapped predicate.
+  Predicate getSwappedCmpPredicate() const {
+    return getSwappedPredicate(getCmpPredicate());
+  }
+
   /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
   /// @returns the predicate that would be the result if the operand were
   /// regarded as signed.
@@ -1212,7 +1240,7 @@ class ICmpInst: public CmpInst {
   }
 
   /// Return the signed version of the predicate: static variant.
-  static Predicate getSignedPredicate(Predicate pred);
+  static Predicate getSignedPredicate(Predicate Pred);
 
   /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
   /// @returns the predicate that would be the result if the operand were
@@ -1223,14 +1251,15 @@ class ICmpInst: public CmpInst {
   }
 
   /// Return the unsigned version of the predicate: static variant.
-  static Predicate getUnsignedPredicate(Predicate pred);
+  static Predicate getUnsignedPredicate(Predicate Pred);
 
-  /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
+  /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ
   /// @returns the unsigned version of the signed predicate pred or
   ///          the signed version of the signed predicate pred.
-  static Predicate getFlippedSignednessPredicate(Predicate pred);
+  /// Static variant.
+  static Predicate getFlippedSignednessPredicate(Predicate Pred);
 
-  /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert
+  /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ
   /// @returns the unsigned version of the signed predicate pred or
   ///          the signed version of the signed predicate pred.
   Predicate getFlippedSignednessPredicate() const {
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index ecf3f716265a5..2a6c922c36b3e 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -112,13 +112,6 @@ namespace Intrinsic {
   Function *getDeclarationIfExists(Module *M, ID id, ArrayRef<Type *> Tys,
                                    FunctionType *FT = nullptr);
 
-  /// Looks up Name in NameTable via binary search. NameTable must be sorted
-  /// and all entries must start with "llvm.".  If NameTable contains an exact
-  /// match for Name or a prefix of Name followed by a dot, its index in
-  /// NameTable is returned. Otherwise, -1 is returned.
-  int lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
-                                StringRef Name, StringRef Target = "");
-
   /// Map a Clang builtin name to an intrinsic ID.
   ID getIntrinsicForClangBuiltin(StringRef TargetPrefix, StringRef BuiltinName);
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 6a09a8647096f..8d30b9bd49ff9 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3812,7 +3812,6 @@ let TargetPrefix = "aarch64" in {
                             [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
                              LLVMVectorOfBitcastsToInt<0>, LLVMVectorOfBitcastsToInt<0>, LLVMVectorOfBitcastsToInt<0>, LLVMVectorOfBitcastsToInt<0>],
                             [IntrNoMem]>;
-
 }
 
 // SVE2.1 - ZIPQ1, ZIPQ2, UZPQ1, UZPQ2
@@ -3855,3 +3854,47 @@ def int_aarch64_sve_famin_u : AdvSIMD_Pred2VectorArg_Intrinsic;
 // Neon absolute maximum and minimum
 def int_aarch64_neon_famax :  AdvSIMD_2VectorArg_Intrinsic;
 def int_aarch64_neon_famin :  AdvSIMD_2VectorArg_Intrinsic;
+
+//
+// FP8 Intrinsics
+//
+let TargetPrefix = "aarch64" in {
+
+  // SVE Widening Conversions
+  class SVE2_FP8_Cvt
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                            [llvm_nxv16i8_ty],
+                            [IntrReadMem, IntrInaccessibleMemOnly]>;
+
+  def int_aarch64_sve_fp8_cvt1   : SVE2_FP8_Cvt;
+  def int_aarch64_sve_fp8_cvt2   : SVE2_FP8_Cvt;
+  def int_aarch64_sve_fp8_cvtlt1 : SVE2_FP8_Cvt;
+  def int_aarch64_sve_fp8_cvtlt2 : SVE2_FP8_Cvt;
+
+  class SME2_FP8_CVT_X2_Single_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
+                            [llvm_nxv16i8_ty],
+                            [IntrReadMem, IntrInaccessibleMemOnly]>;
+
+  class SME_FP8_OuterProduct_Intrinsic
+      : DefaultAttrsIntrinsic<[],
+          [llvm_i32_ty,
+          llvm_nxv16i1_ty, llvm_nxv16i1_ty,
+          llvm_nxv16i8_ty, llvm_nxv16i8_ty],
+          [ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly, IntrHasSideEffects]>;
+  //
+  // CVT from FP8 to half-precision/BFloat16 multi-vector
+  //
+  def int_aarch64_sve_fp8_cvt1_x2 : SME2_FP8_CVT_X2_Single_Intrinsic;
+  def int_aarch64_sve_fp8_cvt2_x2 : SME2_FP8_CVT_X2_Single_Intrinsic;
+
+  //
+  // CVT from FP8 to deinterleaved half-precision/BFloat16 multi-vector
+  //
+  def int_aarch64_sve_fp8_cvtl1_x2 : SME2_FP8_CVT_X2_Single_Intrinsic;
+  def int_aarch64_sve_fp8_cvtl2_x2 : SME2_FP8_CVT_X2_Single_Intrinsic;
+
+  // FP8 outer product
+  def int_aarch64_sme_fp8_fmopa_za16 : SME_FP8_OuterProduct_Intrinsic;
+  def int_aarch64_sme_fp8_fmopa_za32 : SME_FP8_OuterProduct_Intrinsic;
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 26d7d6d251867..444bbe8a1ceb0 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -774,7 +774,7 @@ def int_amdgcn_prng_b32 : DefaultAttrsIntrinsic<
 
 def int_amdgcn_bitop3 :
   DefaultAttrsIntrinsic<[llvm_anyint_ty],
-                        [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i8_ty],
+                        [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
                         [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<3>>]>;
 
 } // TargetPrefix = "amdgcn"
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index bf49ec6f6c649..5696345a617fe 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -27,6 +27,9 @@ def int_dx_handle_fromBinding
           [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty],
           [IntrNoMem]>;
 
+def int_dx_resource_getpointer
+    : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_any_ty, llvm_i32_ty],
+                            [IntrNoMem]>;
 def int_dx_typedBufferLoad
     : DefaultAttrsIntrinsic<[llvm_any_ty], [llvm_any_ty, llvm_i32_ty],
                             [IntrReadMem]>;
@@ -107,4 +110,6 @@ def int_dx_radians : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>
 def int_dx_discard : DefaultAttrsIntrinsic<[], [llvm_i1_ty], []>;
 def int_dx_firstbituhigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
 def int_dx_firstbitshigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
+
+def int_dx_group_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], []>;
 }
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 17b70062e58fa..fd0c3b2a59e1d 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -36,8 +36,8 @@ let TargetPrefix = "spv" in {
   def int_spv_selection_merge : Intrinsic<[], [llvm_vararg_ty]>;
   def int_spv_cmpxchg : Intrinsic<[llvm_i32_ty], [llvm_any_ty, llvm_vararg_ty]>;
   def int_spv_unreachable : Intrinsic<[], []>;
-  def int_spv_alloca : Intrinsic<[llvm_any_ty], []>;
-  def int_spv_alloca_array : Intrinsic<[llvm_any_ty], [llvm_anyint_ty]>;
+  def int_spv_alloca : Intrinsic<[llvm_any_ty], [llvm_i8_ty], [ImmArg<ArgIndex<0>>]>;
+  def int_spv_alloca_array : Intrinsic<[llvm_any_ty], [llvm_anyint_ty, llvm_i8_ty], [ImmArg<ArgIndex<1>>]>;
   def int_spv_undef : Intrinsic<[llvm_i32_ty], []>;
   def int_spv_inline_asm : Intrinsic<[], [llvm_metadata_ty, llvm_metadata_ty, llvm_vararg_ty]>;
 
@@ -59,6 +59,7 @@ let TargetPrefix = "spv" in {
 
   // The following intrinsic(s) are mirrored from IntrinsicsDirectX.td for HLSL support.
   def int_spv_thread_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>;
+  def int_spv_thread_id_in_group : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>;
   def int_spv_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>;
   def int_spv_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>;
   def int_spv_cross : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index a4eb75ceb6930..2a05c2ac0758c 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -1913,6 +1913,11 @@ class ModuleSummaryIndex {
 
   /// Checks if we can import global variable from another module.
   bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const;
+
+  /// Same as above but checks whether the global var is importable as a
+  /// declaration.
+  bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs,
+                          bool &CanImportDecl) const;
 };
 
 /// GraphTraits definition to build SCC for the index
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 93bdcc5bb0291..4d77860be994e 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -365,6 +365,8 @@ class FPMathOperator : public Operator {
     case Instruction::FMul:
     case Instruction::FDiv:
     case Instruction::FRem:
+    case Instruction::FPTrunc:
+    case Instruction::FPExt:
     // FIXME: To clean up and correct the semantics of fast-math-flags, FCmp
     //        should not be treated as a math op, but the other opcodes should.
     //        This would make things consistent with Select/PHI (FP value type
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 13a27b58b9cd7..8153845b52c7a 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -307,22 +307,22 @@ HANDLE_LIBCALL(FMAX_PPCF128, "fmaxl")
 HANDLE_LIBCALL(FMINIMUM_F32, "fminimumf")
 HANDLE_LIBCALL(FMINIMUM_F64, "fminimum")
 HANDLE_LIBCALL(FMINIMUM_F80, "fminimuml")
-HANDLE_LIBCALL(FMINIMUM_F128, "fminmuml")
+HANDLE_LIBCALL(FMINIMUM_F128, "fminimuml")
 HANDLE_LIBCALL(FMINIMUM_PPCF128, "fminimuml")
 HANDLE_LIBCALL(FMAXIMUM_F32, "fmaximumf")
 HANDLE_LIBCALL(FMAXIMUM_F64, "fmaximum")
 HANDLE_LIBCALL(FMAXIMUM_F80, "fmaximuml")
-HANDLE_LIBCALL(FMAXIMUM_F128, "fmaxmuml")
+HANDLE_LIBCALL(FMAXIMUM_F128, "fmaximuml")
 HANDLE_LIBCALL(FMAXIMUM_PPCF128, "fmaximum_numl")
 HANDLE_LIBCALL(FMINIMUMNUM_F32, "fminimum_numf")
 HANDLE_LIBCALL(FMINIMUMNUM_F64, "fminimum_num")
 HANDLE_LIBCALL(FMINIMUMNUM_F80, "fminimum_numl")
-HANDLE_LIBCALL(FMINIMUMNUM_F128, "fminmum_numl")
+HANDLE_LIBCALL(FMINIMUMNUM_F128, "fminimum_numl")
 HANDLE_LIBCALL(FMINIMUMNUM_PPCF128, "fminimum_numl")
 HANDLE_LIBCALL(FMAXIMUMNUM_F32, "fmaximum_numf")
 HANDLE_LIBCALL(FMAXIMUMNUM_F64, "fmaximum_num")
 HANDLE_LIBCALL(FMAXIMUMNUM_F80, "fmaximum_numl")
-HANDLE_LIBCALL(FMAXIMUMNUM_F128, "fmaxmum_numl")
+HANDLE_LIBCALL(FMAXIMUMNUM_F128, "fmaximum_numl")
 HANDLE_LIBCALL(FMAXIMUMNUM_PPCF128, "fmaximum_numl")
 HANDLE_LIBCALL(LROUND_F32, "lroundf")
 HANDLE_LIBCALL(LROUND_F64, "lround")
diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h
index 071575137ff57..514dd6f174b90 100644
--- a/llvm/include/llvm/IR/StructuralHash.h
+++ b/llvm/include/llvm/IR/StructuralHash.h
@@ -31,6 +31,9 @@ class Module;
 /// to true includes instruction and operand type information.
 stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
 
+/// Returns a hash of the global variable \p G.
+stable_hash StructuralHash(const GlobalVariable &G);
+
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h
index 945081b77e953..d444a768a6543 100644
--- a/llvm/include/llvm/IR/Value.h
+++ b/llvm/include/llvm/IR/Value.h
@@ -723,12 +723,16 @@ class Value {
       bool AllowInvariantGroup = false,
       function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis =
           nullptr) const;
-  Value *stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset,
-                                           bool AllowNonInbounds,
-                                           bool AllowInvariantGroup = false) {
+
+  Value *stripAndAccumulateConstantOffsets(
+      const DataLayout &DL, APInt &Offset, bool AllowNonInbounds,
+      bool AllowInvariantGroup = false,
+      function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis =
+          nullptr) {
     return const_cast<Value *>(
         static_cast<const Value *>(this)->stripAndAccumulateConstantOffsets(
-            DL, Offset, AllowNonInbounds, AllowInvariantGroup));
+            DL, Offset, AllowNonInbounds, AllowInvariantGroup,
+            ExternalAnalysis));
   }
 
   /// This is a wrapper around stripAndAccumulateConstantOffsets with the
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 757cef7f4d979..8c82b9291aef9 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -156,11 +156,11 @@ void initializeLegalizerPass(PassRegistry &);
 void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &);
 void initializeGISelKnownBitsAnalysisPass(PassRegistry &);
 void initializeLiveDebugValuesPass(PassRegistry &);
-void initializeLiveDebugVariablesPass(PassRegistry &);
+void initializeLiveDebugVariablesWrapperLegacyPass(PassRegistry &);
 void initializeLiveIntervalsWrapperPassPass(PassRegistry &);
 void initializeLiveRangeShrinkPass(PassRegistry &);
 void initializeLiveRegMatrixWrapperLegacyPass(PassRegistry &);
-void initializeLiveStacksPass(PassRegistry &);
+void initializeLiveStacksWrapperLegacyPass(PassRegistry &);
 void initializeLiveVariablesWrapperPassPass(PassRegistry &);
 void initializeLoadStoreOptPass(PassRegistry &);
 void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry &);
@@ -291,7 +291,7 @@ void initializeSinkingLegacyPassPass(PassRegistry &);
 void initializeSjLjEHPreparePass(PassRegistry &);
 void initializeSlotIndexesWrapperPassPass(PassRegistry &);
 void initializeSpeculativeExecutionLegacyPassPass(PassRegistry &);
-void initializeSpillPlacementPass(PassRegistry &);
+void initializeSpillPlacementWrapperLegacyPass(PassRegistry &);
 void initializeStackColoringLegacyPass(PassRegistry &);
 void initializeStackFrameLayoutAnalysisPassPass(PassRegistry &);
 void initializeStackMapLivenessPass(PassRegistry &);
diff --git a/llvm/include/llvm/LineEditor/LineEditor.h b/llvm/include/llvm/LineEditor/LineEditor.h
index 29fdf9f85c6fa..ca0312bf6297f 100644
--- a/llvm/include/llvm/LineEditor/LineEditor.h
+++ b/llvm/include/llvm/LineEditor/LineEditor.h
@@ -41,6 +41,7 @@ class LineEditor {
 
   void saveHistory();
   void loadHistory();
+  void setHistorySize(int size);
 
   static std::string getDefaultHistoryPath(StringRef ProgName);
 
diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h
index 6db42878d2e52..164ef1ef44bbb 100644
--- a/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -129,6 +129,9 @@ struct MCRegisterDesc {
 
   // Is true for constant registers.
   bool IsConstant;
+
+  // Is true for artificial registers.
+  bool IsArtificial;
 };
 
 /// MCRegisterInfo base class - We assume that the target defines a static
@@ -396,6 +399,16 @@ class MCRegisterInfo {
   /// Returns true if the given register is constant.
   bool isConstant(MCRegister RegNo) const { return get(RegNo).IsConstant; }
 
+  /// Returns true if the given register is artificial, which means it
+  /// represents a regunit that is not separately addressable but still needs to
+  /// be modelled, such as the top 16-bits of a 32-bit GPR.
+  bool isArtificial(MCRegister RegNo) const { return get(RegNo).IsArtificial; }
+
+  /// Returns true when the given register unit is considered artificial.
+  /// Register units are considered artificial when at least one of the
+  /// root registers is artificial.
+  bool isArtificialRegUnit(MCRegUnit Unit) const;
+
   /// Return the number of registers this target has (useful for
   /// sizing arrays holding per register information)
   unsigned getNumRegs() const {
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 9891f1d127f1c..535bcfe2fb6d7 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -77,6 +77,7 @@ class MCSubtargetInfo {
   Triple TargetTriple;
   std::string CPU; // CPU being targeted.
   std::string TuneCPU; // CPU being tuned for.
+  ArrayRef<StringRef> ProcNames; // Processor list, including aliases
   ArrayRef<SubtargetFeatureKV> ProcFeatures;  // Processor feature list
   ArrayRef<SubtargetSubTypeKV> ProcDesc;  // Processor descriptions
 
@@ -95,7 +96,8 @@ class MCSubtargetInfo {
 public:
   MCSubtargetInfo(const MCSubtargetInfo &) = default;
   MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef TuneCPU,
-                  StringRef FS, ArrayRef<SubtargetFeatureKV> PF,
+                  StringRef FS, ArrayRef<StringRef> PN,
+                  ArrayRef<SubtargetFeatureKV> PF,
                   ArrayRef<SubtargetSubTypeKV> PD,
                   const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL,
                   const MCReadAdvanceEntry *RA, const InstrStage *IS,
diff --git a/llvm/include/llvm/MC/MCXCOFFObjectWriter.h b/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
index 968d938a65498..04172286ca770 100644
--- a/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCXCOFFObjectWriter.h
@@ -40,11 +40,16 @@ class MCXCOFFObjectTargetWriter : public MCObjectTargetWriter {
 };
 
 class XCOFFObjectWriter : public MCObjectWriter {
+  // AIX specific CPU type.
+  std::string CPUType;
+
 public:
   virtual void addExceptionEntry(const MCSymbol *Symbol, const MCSymbol *Trap,
                                  unsigned LanguageCode, unsigned ReasonCode,
                                  unsigned FunctionSize, bool hasDebug) = 0;
   virtual void addCInfoSymEntry(StringRef Name, StringRef Metadata) = 0;
+  StringRef getCPUType() const { return CPUType; }
+  void setCPU(StringRef TargetCPU) { CPUType = TargetCPU; }
 };
 
 std::unique_ptr<MCObjectWriter>
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index 6aff1e62276b0..3aa1d7864fcb7 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -401,7 +401,7 @@ class ELFFile {
   ///  be checked after iteration ends.
   Elf_Note_Iterator notes_begin(const Elf_Phdr &Phdr, Error &Err) const {
     assert(Phdr.p_type == ELF::PT_NOTE && "Phdr is not of type PT_NOTE");
-    ErrorAsOutParameter ErrAsOutParam(&Err);
+    ErrorAsOutParameter ErrAsOutParam(Err);
     if (Phdr.p_offset + Phdr.p_filesz > getBufSize()) {
       Err =
           createError("invalid offset (0x" + Twine::utohexstr(Phdr.p_offset) +
@@ -429,7 +429,7 @@ class ELFFile {
   ///  be checked after iteration ends.
   Elf_Note_Iterator notes_begin(const Elf_Shdr &Shdr, Error &Err) const {
     assert(Shdr.sh_type == ELF::SHT_NOTE && "Shdr is not of type SHT_NOTE");
-    ErrorAsOutParameter ErrAsOutParam(&Err);
+    ErrorAsOutParameter ErrAsOutParam(Err);
     if (Shdr.sh_offset + Shdr.sh_size > getBufSize()) {
       Err =
           createError("invalid offset (0x" + Twine::utohexstr(Shdr.sh_offset) +
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 71345a374e86f..8e95496570e00 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -28,7 +28,7 @@ MODULE_PASS("global-merge", GlobalMergePass(TM, GlobalMergeOptions()))
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
 MODULE_PASS("lower-emutls", LowerEmuTLSPass())
 MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass())
-MODULE_PASS("print<regusage>", PhysicalRegisterUsageInfoPrinterPass(dbgs()))
+MODULE_PASS("print<regusage>", PhysicalRegisterUsageInfoPrinterPass(errs()))
 MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass())
 MODULE_PASS("global-merge-func", GlobalMergeFuncPass())
 #undef MODULE_PASS
@@ -99,8 +99,10 @@ LOOP_PASS("loop-term-fold", LoopTermFoldPass())
 // computed. (We still either need to regenerate kill flags after regalloc, or
 // preferably fix the scavenger to not depend on them).
 MACHINE_FUNCTION_ANALYSIS("edge-bundles", EdgeBundlesAnalysis())
+MACHINE_FUNCTION_ANALYSIS("livedebugvars", LiveDebugVariablesAnalysis())
 MACHINE_FUNCTION_ANALYSIS("live-intervals", LiveIntervalsAnalysis())
 MACHINE_FUNCTION_ANALYSIS("live-reg-matrix", LiveRegMatrixAnalysis())
+MACHINE_FUNCTION_ANALYSIS("live-stacks", LiveStacksAnalysis())
 MACHINE_FUNCTION_ANALYSIS("live-vars", LiveVariablesAnalysis())
 MACHINE_FUNCTION_ANALYSIS("machine-block-freq", MachineBlockFrequencyAnalysis())
 MACHINE_FUNCTION_ANALYSIS("machine-branch-prob",
@@ -114,8 +116,8 @@ MACHINE_FUNCTION_ANALYSIS("machine-post-dom-tree",
 MACHINE_FUNCTION_ANALYSIS("machine-trace-metrics", MachineTraceMetricsAnalysis())
 MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
 MACHINE_FUNCTION_ANALYSIS("slot-indexes", SlotIndexesAnalysis())
+MACHINE_FUNCTION_ANALYSIS("spill-code-placement", SpillPlacementAnalysis())
 MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
-// MACHINE_FUNCTION_ANALYSIS("live-stacks", LiveStacksPass())
 // MACHINE_FUNCTION_ANALYSIS("lazy-machine-bfi",
 // LazyMachineBlockFrequencyInfoAnalysis())
 // MACHINE_FUNCTION_ANALYSIS("machine-loops", MachineLoopInfoAnalysis())
@@ -146,19 +148,21 @@ MACHINE_FUNCTION_PASS("opt-phis", OptimizePHIsPass())
 MACHINE_FUNCTION_PASS("peephole-opt", PeepholeOptimizerPass())
 MACHINE_FUNCTION_PASS("phi-node-elimination", PHIEliminationPass())
 MACHINE_FUNCTION_PASS("print", PrintMIRPass())
-MACHINE_FUNCTION_PASS("print<live-intervals>", LiveIntervalsPrinterPass(dbgs()))
-MACHINE_FUNCTION_PASS("print<live-vars>", LiveVariablesPrinterPass(dbgs()))
+MACHINE_FUNCTION_PASS("print<livedebugvars>", LiveDebugVariablesPrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<live-intervals>", LiveIntervalsPrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<live-stacks>", LiveStacksPrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<live-vars>", LiveVariablesPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<machine-block-freq>",
-                      MachineBlockFrequencyPrinterPass(dbgs()))
+                      MachineBlockFrequencyPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<machine-branch-prob>",
-                      MachineBranchProbabilityPrinterPass(dbgs()))
+                      MachineBranchProbabilityPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<machine-dom-tree>",
-                      MachineDominatorTreePrinterPass(dbgs()))
-MACHINE_FUNCTION_PASS("print<machine-loops>", MachineLoopPrinterPass(dbgs()))
+                      MachineDominatorTreePrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<machine-loops>", MachineLoopPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("print<machine-post-dom-tree>",
-                      MachinePostDominatorTreePrinterPass(dbgs()))
-MACHINE_FUNCTION_PASS("print<slot-indexes>", SlotIndexesPrinterPass(dbgs()))
-MACHINE_FUNCTION_PASS("print<virtregmap>", VirtRegMapPrinterPass(dbgs()))
+                      MachinePostDominatorTreePrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<slot-indexes>", SlotIndexesPrinterPass(errs()))
+MACHINE_FUNCTION_PASS("print<virtregmap>", VirtRegMapPrinterPass(errs()))
 MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass())
 MACHINE_FUNCTION_PASS("reg-usage-propagation", RegUsageInfoPropagationPass())
 MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index e631e3899fd4d..590185d42e72a 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -461,7 +461,7 @@ struct MCDCRecord {
         Folded(std::move(Folded)), PosToID(std::move(PosToID)),
         CondLoc(std::move(CondLoc)){};
 
-  CounterMappingRegion getDecisionRegion() const { return Region; }
+  const CounterMappingRegion &getDecisionRegion() const { return Region; }
   unsigned getNumConditions() const {
     return Region.getDecisionParams().NumConditions;
   }
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index c5f7800097807..7133c0c6a302c 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -344,7 +344,9 @@ enum class InstrProfKind {
   MemProf = 0x40,
   // A temporal profile.
   TemporalProfile = 0x80,
-  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/TemporalProfile)
+  // A profile with loop entry basic blocks instrumentation.
+  LoopEntriesInstrumentation = 0x100,
+  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/LoopEntriesInstrumentation)
 };
 
 const std::error_category &instrprof_category();
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index c66b0465a0b54..39613da81ecb4 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -730,10 +730,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_COVMAP_VERSION 6
 
 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
- * version for other variants of profile. We set the 8th most significant bit 
+ * version for other variants of profile. We set the 8th most significant bit
  * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
  * generated profile, and 0 if this is a Clang FE generated profile.
  * 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 54th bit indicates whether to always instrument loop entry blocks.
+ * The 58th bit indicates whether to always instrument function entry blocks.
  * The 59th bit indicates whether to use debug info to correlate profiles.
  * The 60th bit indicates single byte coverage instrumentation.
  * The 61st bit indicates function entry instrumentation only.
@@ -742,6 +744,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
  */
 #define VARIANT_MASKS_ALL 0xffffffff00000000ULL
 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
+#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 1fad2343e2c96..25e13f575296d 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -123,6 +123,9 @@ class InstrProfReader {
 
   virtual bool instrEntryBBEnabled() const = 0;
 
+  /// Return true if the profile instruments all loop entries.
+  virtual bool instrLoopEntriesEnabled() const = 0;
+
   /// Return true if the profile has single byte counters representing coverage.
   virtual bool hasSingleByteCoverage() const = 0;
 
@@ -274,6 +277,11 @@ class TextInstrProfReader : public InstrProfReader {
                              InstrProfKind::FunctionEntryInstrumentation);
   }
 
+  bool instrLoopEntriesEnabled() const override {
+    return static_cast<bool>(ProfileKind &
+                             InstrProfKind::LoopEntriesInstrumentation);
+  }
+
   bool hasSingleByteCoverage() const override {
     return static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage);
   }
@@ -398,6 +406,10 @@ class RawInstrProfReader : public InstrProfReader {
     return (Version & VARIANT_MASK_INSTR_ENTRY) != 0;
   }
 
+  bool instrLoopEntriesEnabled() const override {
+    return (Version & VARIANT_MASK_INSTR_LOOP_ENTRIES) != 0;
+  }
+
   bool hasSingleByteCoverage() const override {
     return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0;
   }
@@ -564,6 +576,7 @@ struct InstrProfReaderIndexBase {
   virtual bool isIRLevelProfile() const = 0;
   virtual bool hasCSIRLevelProfile() const = 0;
   virtual bool instrEntryBBEnabled() const = 0;
+  virtual bool instrLoopEntriesEnabled() const = 0;
   virtual bool hasSingleByteCoverage() const = 0;
   virtual bool functionEntryOnly() const = 0;
   virtual bool hasMemoryProfile() const = 0;
@@ -628,6 +641,10 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
     return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0;
   }
 
+  bool instrLoopEntriesEnabled() const override {
+    return (FormatVersion & VARIANT_MASK_INSTR_LOOP_ENTRIES) != 0;
+  }
+
   bool hasSingleByteCoverage() const override {
     return (FormatVersion & VARIANT_MASK_BYTE_COVERAGE) != 0;
   }
@@ -699,6 +716,9 @@ class IndexedMemProfReader {
 
   DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>>
   getMemProfCallerCalleePairs() const;
+
+  // Return the entire MemProf profile.
+  memprof::AllMemProfData getAllMemProfData() const;
 };
 
 /// Reader for the indexed binary instrprof format.
@@ -753,6 +773,10 @@ class IndexedInstrProfReader : public InstrProfReader {
     return Index->instrEntryBBEnabled();
   }
 
+  bool instrLoopEntriesEnabled() const override {
+    return Index->instrLoopEntriesEnabled();
+  }
+
   bool hasSingleByteCoverage() const override {
     return Index->hasSingleByteCoverage();
   }
@@ -802,6 +826,10 @@ class IndexedInstrProfReader : public InstrProfReader {
     return MemProfReader.getMemProfCallerCalleePairs();
   }
 
+  memprof::AllMemProfData getAllMemProfData() const {
+    return MemProfReader.getAllMemProfData();
+  }
+
   /// Fill Counts with the profile data for the given function name.
   Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash,
                           std::vector<uint64_t> &Counts);
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index fa30926c66258..117fddb5729e4 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -190,7 +190,9 @@ class InstrProfWriter {
       return make_error<InstrProfError>(instrprof_error::unsupported_version);
     }
     if (testIncompatible(InstrProfKind::FunctionEntryOnly,
-                         InstrProfKind::FunctionEntryInstrumentation)) {
+                         InstrProfKind::FunctionEntryInstrumentation) ||
+        testIncompatible(InstrProfKind::FunctionEntryOnly,
+                         InstrProfKind::LoopEntriesInstrumentation)) {
       return make_error<InstrProfError>(
           instrprof_error::unsupported_version,
           "cannot merge FunctionEntryOnly profiles and BB profiles together");
@@ -235,6 +237,9 @@ class InstrProfWriter {
   uint64_t writeHeader(const IndexedInstrProf::Header &header,
                        const bool WritePrevVersion, ProfOStream &OS);
 
+  // Writes binary IDs.
+  Error writeBinaryIds(ProfOStream &OS);
+
   // Writes compressed vtable names to profiles.
   Error writeVTableNames(ProfOStream &OS);
 };
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index d64addcfe3ed0..ace3c044d8c27 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -11,7 +11,9 @@
 #include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/HashBuilder.h"
+#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <bitset>
@@ -19,6 +21,10 @@
 #include <optional>
 
 namespace llvm {
+namespace yaml {
+template <typename T> struct CustomMappingTraits;
+} // namespace yaml
+
 namespace memprof {
 
 struct MemProfRecord;
@@ -193,6 +199,9 @@ struct PortableMemInfoBlock {
     return Result;
   }
 
+  // Give YAML access to the individual MIB fields.
+  friend struct yaml::CustomMappingTraits<memprof::PortableMemInfoBlock>;
+
 private:
   // The set of available fields, indexed by Meta::Name.
   std::bitset<llvm::to_underlying(Meta::Size)> Schema;
@@ -344,24 +353,16 @@ using LinearCallStackId = uint32_t;
 struct IndexedAllocationInfo {
   // The dynamic calling context for the allocation in bottom-up (leaf-to-root)
   // order. Frame contents are stored out-of-line.
-  // TODO: Remove once we fully transition to CSId.
-  llvm::SmallVector<FrameId> CallStack;
-  // Conceptually the same as above.  We are going to keep both CallStack and
-  // CallStackId while we are transitioning from CallStack to CallStackId.
   CallStackId CSId = 0;
   // The statistics obtained from the runtime for the allocation.
   PortableMemInfoBlock Info;
 
   IndexedAllocationInfo() = default;
-  // This constructor is soft deprecated.  It will be removed once we remove all
-  // users of the CallStack field.
-  IndexedAllocationInfo(ArrayRef<FrameId> CS, CallStackId CSId,
-                        const MemInfoBlock &MB,
-                        const MemProfSchema &Schema = getFullSchema())
-      : CallStack(CS), CSId(CSId), Info(MB, Schema) {}
   IndexedAllocationInfo(CallStackId CSId, const MemInfoBlock &MB,
                         const MemProfSchema &Schema = getFullSchema())
       : CSId(CSId), Info(MB, Schema) {}
+  IndexedAllocationInfo(CallStackId CSId, const PortableMemInfoBlock &MB)
+      : CSId(CSId), Info(MB) {}
 
   // Returns the size in bytes when this allocation info struct is serialized.
   size_t serializedSize(const MemProfSchema &Schema,
@@ -415,21 +416,14 @@ struct IndexedMemProfRecord {
   // list of inline locations in bottom-up order i.e. from leaf to root. The
   // inline location list may include additional entries, users should pick
   // the last entry in the list with the same function GUID.
-  llvm::SmallVector<llvm::SmallVector<FrameId>> CallSites;
-  // Conceptually the same as above.  We are going to keep both CallSites and
-  // CallSiteIds while we are transitioning from CallSites to CallSiteIds.
   llvm::SmallVector<CallStackId> CallSiteIds;
 
-  void clear() {
-    AllocSites.clear();
-    CallSites.clear();
-  }
+  void clear() { *this = IndexedMemProfRecord(); }
 
   void merge(const IndexedMemProfRecord &Other) {
     // TODO: Filter out duplicates which may occur if multiple memprof
     // profiles are merged together using llvm-profdata.
     AllocSites.append(Other.AllocSites);
-    CallSites.append(Other.CallSites);
   }
 
   size_t serializedSize(const MemProfSchema &Schema,
@@ -498,6 +492,23 @@ struct MemProfRecord {
   }
 };
 
+// A "typedef" for GUID.  See ScalarTraits<memprof::GUIDHex64> for how a GUID is
+// serialized and deserialized in YAML.
+LLVM_YAML_STRONG_TYPEDEF(uint64_t, GUIDHex64)
+
+// Helper struct for AllMemProfData.  In YAML, we treat the GUID and the fields
+// within MemProfRecord at the same level as if the GUID were part of
+// MemProfRecord.
+struct GUIDMemProfRecordPair {
+  GUIDHex64 GUID;
+  MemProfRecord Record;
+};
+
+// The top-level data structure, only used with YAML for now.
+struct AllMemProfData {
+  std::vector<GUIDMemProfRecordPair> HeapProfileRecords;
+};
+
 // Reads a memprof schema from a buffer. All entries in the buffer are
 // interpreted as uint64_t. The first entry in the buffer denotes the number of
 // ids in the schema. Subsequent entries are integers which map to memprof::Meta
@@ -1017,6 +1028,24 @@ struct IndexedMemProfData {
 
   // A map to hold call stack id to call stacks.
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> CallStacks;
+
+  FrameId addFrame(const Frame &F) {
+    const FrameId Id = F.hash();
+    Frames.try_emplace(Id, F);
+    return Id;
+  }
+
+  CallStackId addCallStack(ArrayRef<FrameId> CS) {
+    CallStackId CSId = hashCallStack(CS);
+    CallStacks.try_emplace(CSId, CS);
+    return CSId;
+  }
+
+  CallStackId addCallStack(SmallVector<FrameId> &&CS) {
+    CallStackId CSId = hashCallStack(CS);
+    CallStacks.try_emplace(CSId, std::move(CS));
+    return CSId;
+  }
 };
 
 struct FrameStat {
@@ -1140,6 +1169,131 @@ template <typename FrameIdTy> class CallStackRadixTreeBuilder {
   }
 };
 } // namespace memprof
+
+namespace yaml {
+template <> struct ScalarTraits<memprof::GUIDHex64> {
+  static void output(const memprof::GUIDHex64 &Val, void *, raw_ostream &Out) {
+    // Print GUID as a 16-digit hexadecimal number.
+    Out << format("0x%016" PRIx64, (uint64_t)Val);
+  }
+  static StringRef input(StringRef Scalar, void *, memprof::GUIDHex64 &Val) {
+    // Reject decimal GUIDs.
+    if (all_of(Scalar, [](char C) { return std::isdigit(C); }))
+      return "use a hexadecimal GUID or a function instead";
+
+    uint64_t Num;
+    if (Scalar.starts_with_insensitive("0x")) {
+      // Accept hexadecimal numbers starting with 0x or 0X.
+      if (Scalar.getAsInteger(0, Num))
+        return "invalid hex64 number";
+      Val = Num;
+    } else {
+      // Otherwise, treat the input as a string containing a function name.
+      Val = memprof::IndexedMemProfRecord::getGUID(Scalar);
+    }
+    return StringRef();
+  }
+  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
+};
+
+template <> struct MappingTraits<memprof::Frame> {
+  static void mapping(IO &Io, memprof::Frame &F) {
+    Io.mapRequired("Function", F.Function);
+    Io.mapRequired("LineOffset", F.LineOffset);
+    Io.mapRequired("Column", F.Column);
+    Io.mapRequired("IsInlineFrame", F.IsInlineFrame);
+
+    // Assert that the definition of Frame matches what we expect.  The
+    // structured bindings below detect changes to the number of fields.
+    // static_assert checks the type of each field.
+    const auto &[Function, SymbolName, LineOffset, Column, IsInlineFrame] = F;
+    static_assert(
+        std::is_same_v<remove_cvref_t<decltype(Function)>, GlobalValue::GUID>);
+    static_assert(std::is_same_v<remove_cvref_t<decltype(SymbolName)>,
+                                 std::unique_ptr<std::string>>);
+    static_assert(
+        std::is_same_v<remove_cvref_t<decltype(LineOffset)>, uint32_t>);
+    static_assert(std::is_same_v<remove_cvref_t<decltype(Column)>, uint32_t>);
+    static_assert(
+        std::is_same_v<remove_cvref_t<decltype(IsInlineFrame)>, bool>);
+
+    // MSVC issues unused variable warnings despite the uses in static_assert
+    // above.
+    (void)Function;
+    (void)SymbolName;
+    (void)LineOffset;
+    (void)Column;
+    (void)IsInlineFrame;
+  }
+
+  // Request the inline notation for brevity:
+  //   { Function: 123, LineOffset: 11, Column: 10; IsInlineFrame: true }
+  static const bool flow = true;
+};
+
+template <> struct CustomMappingTraits<memprof::PortableMemInfoBlock> {
+  static void inputOne(IO &Io, StringRef KeyStr,
+                       memprof::PortableMemInfoBlock &MIB) {
+    // PortableMemInfoBlock keeps track of the set of fields that actually have
+    // values.  We update the set here as we receive a key-value pair from the
+    // YAML document.
+    //
+    // We set MIB.Name via a temporary variable because ScalarTraits<uintptr_t>
+    // isn't available on macOS.
+#define MIBEntryDef(NameTag, Name, Type)                                       \
+  if (KeyStr == #Name) {                                                       \
+    uint64_t Value;                                                            \
+    Io.mapRequired(KeyStr.str().c_str(), Value);                               \
+    MIB.Name = static_cast<Type>(Value);                                       \
+    MIB.Schema.set(llvm::to_underlying(memprof::Meta::Name));                  \
+    return;                                                                    \
+  }
+#include "llvm/ProfileData/MIBEntryDef.inc"
+#undef MIBEntryDef
+    Io.setError("Key is not a valid validation event");
+  }
+
+  static void output(IO &Io, memprof::PortableMemInfoBlock &MIB) {
+    auto Schema = MIB.getSchema();
+#define MIBEntryDef(NameTag, Name, Type)                                       \
+  if (Schema.test(llvm::to_underlying(memprof::Meta::Name))) {                 \
+    uint64_t Value = MIB.Name;                                                 \
+    Io.mapRequired(#Name, Value);                                              \
+  }
+#include "llvm/ProfileData/MIBEntryDef.inc"
+#undef MIBEntryDef
+  }
+};
+
+template <> struct MappingTraits<memprof::AllocationInfo> {
+  static void mapping(IO &Io, memprof::AllocationInfo &AI) {
+    Io.mapRequired("Callstack", AI.CallStack);
+    Io.mapRequired("MemInfoBlock", AI.Info);
+  }
+};
+
+// In YAML, we use GUIDMemProfRecordPair instead of MemProfRecord so that we can
+// treat the GUID and the fields within MemProfRecord at the same level as if
+// the GUID were part of MemProfRecord.
+template <> struct MappingTraits<memprof::GUIDMemProfRecordPair> {
+  static void mapping(IO &Io, memprof::GUIDMemProfRecordPair &Pair) {
+    Io.mapRequired("GUID", Pair.GUID);
+    Io.mapRequired("AllocSites", Pair.Record.AllocSites);
+    Io.mapRequired("CallSites", Pair.Record.CallSites);
+  }
+};
+
+template <> struct MappingTraits<memprof::AllMemProfData> {
+  static void mapping(IO &Io, memprof::AllMemProfData &Data) {
+    Io.mapRequired("HeapProfileRecords", Data.HeapProfileRecords);
+  }
+};
+} // namespace yaml
 } // namespace llvm
 
+LLVM_YAML_IS_SEQUENCE_VECTOR(memprof::Frame)
+LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<memprof::Frame>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(memprof::AllocationInfo)
+LLVM_YAML_IS_SEQUENCE_VECTOR(memprof::GUIDMemProfRecordPair)
+
 #endif // LLVM_PROFILEDATA_MEMPROF_H_
diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h
index bf2f91548fac0..29d9e57cae3e3 100644
--- a/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -209,6 +209,26 @@ class RawMemProfReader final : public MemProfReader {
   // A mapping of the hash to symbol name, only used if KeepSymbolName is true.
   llvm::DenseMap<uint64_t, std::string> GuidToSymbolName;
 };
+
+class YAMLMemProfReader final : public MemProfReader {
+public:
+  YAMLMemProfReader() = default;
+
+  // Return true if the \p DataBuffer starts with "---" indicating it is a YAML
+  // file.
+  static bool hasFormat(const MemoryBuffer &DataBuffer);
+  // Wrapper around hasFormat above, reading the file instead of the memory
+  // buffer.
+  static bool hasFormat(const StringRef Path);
+
+  // Create a YAMLMemProfReader after sanity checking the contents of the file
+  // at \p Path or the \p Buffer.
+  static Expected<std::unique_ptr<YAMLMemProfReader>> create(const Twine &Path);
+  static Expected<std::unique_ptr<YAMLMemProfReader>>
+  create(std::unique_ptr<MemoryBuffer> Buffer);
+
+  void parse(StringRef YAMLData);
+};
 } // namespace memprof
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h
index 3e2f0d9b43fc0..23c54ac9bb4b1 100644
--- a/llvm/include/llvm/Support/Debug.h
+++ b/llvm/include/llvm/Support/Debug.h
@@ -61,15 +61,20 @@ void setCurrentDebugTypes(const char **Types, unsigned Count);
 ///
 /// This will emit the debug information if -debug is present, and -debug-only
 /// is not specified, or is specified as "bitset".
-#define DEBUG_WITH_TYPE(TYPE, X)                                        \
-  do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE)) { X; } \
+#define DEBUG_WITH_TYPE(TYPE, ...)                                             \
+  do {                                                                         \
+    if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE)) {               \
+      __VA_ARGS__;                                                             \
+    }                                                                          \
   } while (false)
 
 #else
 #define isCurrentDebugType(X) (false)
 #define setCurrentDebugType(X) do { (void)(X); } while (false)
 #define setCurrentDebugTypes(X, N) do { (void)(X); (void)(N); } while (false)
-#define DEBUG_WITH_TYPE(TYPE, X) do { } while (false)
+#define DEBUG_WITH_TYPE(TYPE, ...)                                             \
+  do {                                                                         \
+  } while (false)
 #endif
 
 /// This boolean is set to true if the '-debug' command line option
@@ -98,7 +103,7 @@ raw_ostream &dbgs();
 //
 // LLVM_DEBUG(dbgs() << "Bitset contains: " << Bitset << "\n");
 //
-#define LLVM_DEBUG(X) DEBUG_WITH_TYPE(DEBUG_TYPE, X)
+#define LLVM_DEBUG(...) DEBUG_WITH_TYPE(DEBUG_TYPE, __VA_ARGS__)
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index cb06ac19f0bb7..90120156ec2ea 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -1129,12 +1129,17 @@ inline bool errorToBool(Error Err) {
 /// function.
 class ErrorAsOutParameter {
 public:
+
   ErrorAsOutParameter(Error *Err) : Err(Err) {
     // Raise the checked bit if Err is success.
     if (Err)
       (void)!!*Err;
   }
 
+  ErrorAsOutParameter(Error &Err) : Err(&Err) {
+    (void)!!Err;
+  }
+
   ~ErrorAsOutParameter() {
     // Clear the checked bit.
     if (Err && !*Err)
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
index ada0b8962881d..b5af0e0401ef2 100644
--- a/llvm/include/llvm/Support/InstructionCost.h
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -198,10 +198,8 @@ class InstructionCost {
     return Value < RHS.Value;
   }
 
-  // Implement in terms of operator< to ensure that the two comparisons stay in
-  // sync
   bool operator==(const InstructionCost &RHS) const {
-    return !(*this < RHS) && !(RHS < *this);
+    return State == RHS.State && Value == RHS.Value;
   }
 
   bool operator!=(const InstructionCost &RHS) const { return !(*this == RHS); }
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 17987935ed3cf..5ef3707b81fe9 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -289,6 +289,12 @@ HANDLE_TARGET_OPCODE(G_OR)
 /// Generic bitwise exclusive-or instruction.
 HANDLE_TARGET_OPCODE(G_XOR)
 
+/// Generic absolute difference signed instruction.
+HANDLE_TARGET_OPCODE(G_ABDS)
+
+/// Generic absolute difference unsigned instruction.
+HANDLE_TARGET_OPCODE(G_ABDU)
+
 
 HANDLE_TARGET_OPCODE(G_IMPLICIT_DEF)
 
diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h
index 8c0ecf9b7c911..679e157f3dfd8 100644
--- a/llvm/include/llvm/Support/TimeProfiler.h
+++ b/llvm/include/llvm/Support/TimeProfiler.h
@@ -185,25 +185,17 @@ class TimeTraceScope {
   TimeTraceScope(TimeTraceScope &&) = delete;
   TimeTraceScope &operator=(TimeTraceScope &&) = delete;
 
-  TimeTraceScope(StringRef Name) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, StringRef(""));
-  }
-  TimeTraceScope(StringRef Name, StringRef Detail) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, Detail);
-  }
-  TimeTraceScope(StringRef Name, llvm::function_ref<std::string()> Detail) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, Detail);
-  }
+  TimeTraceScope(StringRef Name)
+      : Entry(timeTraceProfilerBegin(Name, StringRef())) {}
+  TimeTraceScope(StringRef Name, StringRef Detail)
+      : Entry(timeTraceProfilerBegin(Name, Detail)) {}
+  TimeTraceScope(StringRef Name, llvm::function_ref<std::string()> Detail)
+      : Entry(timeTraceProfilerBegin(Name, Detail)) {}
   TimeTraceScope(StringRef Name,
-                 llvm::function_ref<TimeTraceMetadata()> Metadata) {
-    if (getTimeTraceProfilerInstance() != nullptr)
-      Entry = timeTraceProfilerBegin(Name, Metadata);
-  }
+                 llvm::function_ref<TimeTraceMetadata()> Metadata)
+      : Entry(timeTraceProfilerBegin(Name, Metadata)) {}
   ~TimeTraceScope() {
-    if (getTimeTraceProfilerInstance() != nullptr)
+    if (Entry)
       timeTraceProfilerEnd(Entry);
   }
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 60606db078b37..c8f91cd0de597 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -386,6 +386,22 @@ def G_ASHR : GenericInstruction {
   let hasSideEffects = false;
 }
 
+// Generic absolute difference signed.
+def G_ABDS : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type0:$src2);
+  let hasSideEffects = false;
+  let isCommutable = true;
+}
+
+// Generic absolute difference unsigned.
+def G_ABDU : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type0:$src2);
+  let hasSideEffects = false;
+  let isCommutable = true;
+}
+
 /// Funnel 'double' shifts take 3 operands, 2 inputs and the shift amount.
 /// fshl(X,Y,Z): (X << (Z % bitwidth)) | (Y >> (bitwidth - (Z % bitwidth)))
 def G_FSHL : GenericInstruction {
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index b0c63fc7c7b80..013c3a6ed83d8 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -424,11 +424,22 @@ def unary_undef_to_zero: GICombineRule<
          [{ return Helper.matchOperandIsUndef(*${root}, 1); }]),
   (apply [{ Helper.replaceInstWithConstant(*${root}, 0); }])>;
 
+def unary_undef_to_undef_frags : GICombinePatFrag<
+  (outs root:$dst), (ins),
+  !foreach(op,
+           [G_TRUNC, G_BITCAST, G_ANYEXT, G_PTRTOINT, G_INTTOPTR, G_FPTOSI,
+            G_FPTOUI],
+           (pattern (op $dst, $x), (G_IMPLICIT_DEF $x)))>;
+def unary_undef_to_undef : GICombineRule<
+  (defs root:$dst),
+  (match (unary_undef_to_undef_frags $dst)),
+  (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
+
 // Instructions where if any source operand is undef, the instruction can be
 // replaced with undef.
 def propagate_undef_any_op: GICombineRule<
   (defs root:$root),
-  (match (wip_match_opcode G_ADD, G_FPTOSI, G_FPTOUI, G_SUB, G_XOR, G_TRUNC, G_BITCAST, G_ANYEXT):$root,
+  (match (wip_match_opcode G_ADD, G_SUB, G_XOR):$root,
          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
 
@@ -1919,6 +1930,7 @@ def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      binop_left_undef_to_zero,
                                      binop_right_undef_to_undef,
                                      unary_undef_to_zero,
+                                     unary_undef_to_undef,
                                      propagate_undef_any_op,
                                      propagate_undef_all_ops,
                                      propagate_undef_shuffle_mask,
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index d7b1ba511f95d..1311329821828 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -268,6 +268,9 @@ void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
 
 bool isX18ReservedByDefault(const Triple &TT);
 
+// Return the priority for a given set of FMV features.
+unsigned getFMVPriority(ArrayRef<StringRef> Features);
+
 // For given feature names, return a bitmask corresponding to the entries of
 // AArch64::CPUFeatures. The values in CPUFeatures are not bitmasks
 // themselves, they are sequential (0, 1, 2, 3, ...).
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index c5cd1b1bc6376..6b96c3e83c8c4 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -374,6 +374,9 @@ ARM_CPU_NAME("cortex-a78ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_RAS | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a78c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              ARM::AEK_FP16 | ARM::AEK_DOTPROD)
+ARM_CPU_NAME("cortex-a510", ARMV9A, FK_NEON_FP_ARMV8, false,
+             (ARM::AEK_DOTPROD | ARM::AEK_FP16FML | ARM::AEK_BF16 |
+              ARM::AEK_SB | ARM::AEK_I8MM))
 ARM_CPU_NAME("cortex-a710", ARMV9A, FK_NEON_FP_ARMV8, false,
              (ARM::AEK_DOTPROD | ARM::AEK_FP16FML | ARM::AEK_BF16 |
               ARM::AEK_SB | ARM::AEK_I8MM))
diff --git a/llvm/include/llvm/TargetParser/LoongArchTargetParser.def b/llvm/include/llvm/TargetParser/LoongArchTargetParser.def
index e3285f89ef9ea..6731a2c975cd5 100644
--- a/llvm/include/llvm/TargetParser/LoongArchTargetParser.def
+++ b/llvm/include/llvm/TargetParser/LoongArchTargetParser.def
@@ -12,6 +12,7 @@ LOONGARCH_FEATURE("+lvz", FK_LVZ)
 LOONGARCH_FEATURE("+ual", FK_UAL)
 LOONGARCH_FEATURE("+frecipe", FK_FRECIPE)
 LOONGARCH_FEATURE("+lam-bh", FK_LAM_BH)
+LOONGARCH_FEATURE("+lamcas", FK_LAMCAS)
 LOONGARCH_FEATURE("+ld-seq-sa", FK_LD_SEQ_SA)
 LOONGARCH_FEATURE("+div32", FK_DIV32)
 
@@ -23,6 +24,6 @@ LOONGARCH_FEATURE("+div32", FK_DIV32)
 
 LOONGARCH_ARCH("loongarch64", AK_LOONGARCH64, FK_64BIT | FK_FP32 | FK_FP64 | FK_UAL)
 LOONGARCH_ARCH("la464", AK_LA464, FK_64BIT | FK_FP32 | FK_FP64 | FK_LSX | FK_LASX | FK_UAL)
-LOONGARCH_ARCH("la664", AK_LA664, FK_64BIT | FK_FP32 | FK_FP64 | FK_LSX | FK_LASX | FK_UAL | FK_FRECIPE | FK_LAM_BH | FK_LD_SEQ_SA | FK_DIV32)
+LOONGARCH_ARCH("la664", AK_LA664, FK_64BIT | FK_FP32 | FK_FP64 | FK_LSX | FK_LASX | FK_UAL | FK_FRECIPE | FK_LAM_BH | FK_LAMCAS | FK_LD_SEQ_SA | FK_DIV32)
 
 #undef LOONGARCH_ARCH
diff --git a/llvm/include/llvm/TargetParser/LoongArchTargetParser.h b/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
index 5862becc92d77..52cd51f43ad64 100644
--- a/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
+++ b/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
@@ -54,6 +54,10 @@ enum FeatureKind : uint32_t {
   // available.
   FK_LAM_BH = 1 << 10,
 
+  // Atomic memory compare and swap instructions for byte, half word, word and
+  // double word are available.
+  FK_LAMCAS = 1 << 11,
+
   // Do not generate load-load barrier instructions (dbar 0x700).
   FK_LD_SEQ_SA = 1 << 12,
 
diff --git a/llvm/include/llvm/Transforms/IPO/MergeFunctions.h b/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
index 822f0fd99188d..71f175c6472b4 100644
--- a/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
+++ b/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
@@ -20,11 +20,16 @@
 namespace llvm {
 
 class Module;
+class Function;
 
 /// Merge identical functions.
 class MergeFunctionsPass : public PassInfoMixin<MergeFunctionsPass> {
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool runOnModule(Module &M);
+  static DenseMap<Function *, Function *>
+  runOnFunctions(ArrayRef<Function *> F);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index 3075b7ebae59e..71592058e3456 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -157,7 +157,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
   /// conditional branch or select to create a compare with a canonical
   /// (inverted) predicate which is then more likely to be matched with other
   /// values.
-  static bool isCanonicalPredicate(CmpInst::Predicate Pred) {
+  static bool isCanonicalPredicate(CmpPredicate Pred) {
     switch (Pred) {
     case CmpInst::ICMP_NE:
     case CmpInst::ICMP_ULE:
@@ -185,10 +185,9 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
   }
 
   std::optional<std::pair<
-      CmpInst::Predicate,
-      Constant *>> static getFlippedStrictnessPredicateAndConstant(CmpInst::
-                                                                       Predicate
-                                                                           Pred,
+      CmpPredicate,
+      Constant *>> static getFlippedStrictnessPredicateAndConstant(CmpPredicate
+                                                                       Pred,
                                                                    Constant *C);
 
   static bool shouldAvoidAbsorbingNotIntoSelect(const SelectInst &SI) {
diff --git a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
index 35b3d615e3844..f6bf045f7de2c 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/BranchProbability.h"
@@ -52,10 +53,14 @@ template <class Edge, class BBInfo> class CFGMST {
 
   BranchProbabilityInfo *const BPI;
   BlockFrequencyInfo *const BFI;
+  LoopInfo *const LI;
 
   // If function entry will be always instrumented.
   const bool InstrumentFuncEntry;
 
+  // If true loop entries will be always instrumented.
+  const bool InstrumentLoopEntries;
+
   // Find the root group of the G and compress the path from G to the root.
   BBInfo *findAndCompressGroup(BBInfo *G) {
     if (G->Group != G)
@@ -154,6 +159,16 @@ template <class Edge, class BBInfo> class CFGMST {
           }
           if (BPI != nullptr)
             Weight = BPI->getEdgeProbability(&BB, TargetBB).scale(scaleFactor);
+          // If InstrumentLoopEntries is on and the current edge leads to a loop
+          // (i.e., TargetBB is a loop head and BB is outside its loop), set
+          // Weight to be minimal, so that the edge won't be chosen for the MST
+          // and will be instrumented.
+          if (InstrumentLoopEntries && LI->isLoopHeader(TargetBB)) {
+            Loop *TargetLoop = LI->getLoopFor(TargetBB);
+            assert(TargetLoop);
+            if (!TargetLoop->contains(&BB))
+              Weight = 0;
+          }
           if (Weight == 0)
             Weight++;
           auto *E = &addEdge(&BB, TargetBB, Weight);
@@ -252,6 +267,19 @@ template <class Edge, class BBInfo> class CFGMST {
     }
   }
 
+  [[maybe_unused]] bool validateLoopEntryInstrumentation() {
+    if (!InstrumentLoopEntries)
+      return true;
+    for (auto &Ei : AllEdges) {
+      if (Ei->Removed)
+        continue;
+      if (Ei->DestBB && LI->isLoopHeader(Ei->DestBB) &&
+          !LI->getLoopFor(Ei->DestBB)->contains(Ei->SrcBB) && Ei->InMST)
+        return false;
+    }
+    return true;
+  }
+
 public:
   // Dump the Debug information about the instrumentation.
   void dumpEdges(raw_ostream &OS, const Twine &Message) const {
@@ -291,13 +319,20 @@ template <class Edge, class BBInfo> class CFGMST {
     return *AllEdges.back();
   }
 
-  CFGMST(Function &Func, bool InstrumentFuncEntry,
+  CFGMST(Function &Func, bool InstrumentFuncEntry, bool InstrumentLoopEntries,
          BranchProbabilityInfo *BPI = nullptr,
-         BlockFrequencyInfo *BFI = nullptr)
-      : F(Func), BPI(BPI), BFI(BFI), InstrumentFuncEntry(InstrumentFuncEntry) {
+         BlockFrequencyInfo *BFI = nullptr, LoopInfo *LI = nullptr)
+      : F(Func), BPI(BPI), BFI(BFI), LI(LI),
+        InstrumentFuncEntry(InstrumentFuncEntry),
+        InstrumentLoopEntries(InstrumentLoopEntries) {
+    assert(!(InstrumentLoopEntries && !LI) &&
+           "expected a LoopInfo to instrumenting loop entries");
     buildEdges();
     sortEdgesByWeight();
     computeMinimumSpanningTree();
+    assert(validateLoopEntryInstrumentation() &&
+           "Loop entries should not be in MST when "
+           "InstrumentLoopEntries is on");
     if (AllEdges.size() > 1 && InstrumentFuncEntry)
       std::iter_swap(std::move(AllEdges.begin()),
                      std::move(AllEdges.begin() + AllEdges.size() - 1));
diff --git a/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
index f2ce1636551ce..151e70e87b294 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
@@ -33,6 +33,13 @@ class RealtimeSanitizerPass : public PassInfoMixin<RealtimeSanitizerPass> {
   static bool isRequired() { return true; }
 };
 
+/// Create ctor and init functions.
+struct ModuleRealtimeSanitizerPass
+    : public PassInfoMixin<ModuleRealtimeSanitizerPass> {
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
+};
+
 } // namespace llvm
 
 #endif // LLVM_TRANSFORMS_INSTRUMENTATION_REALTIMESANITIZER_H
diff --git a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
index 023c9de28209c..496d2958fc2d0 100644
--- a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
+++ b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
@@ -68,7 +68,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
                           BasicBlock::iterator &BBI);
   bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
   bool processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI);
-  bool processMemMove(MemMoveInst *M);
+  bool processMemMove(MemMoveInst *M, BasicBlock::iterator &BBI);
   bool performCallSlotOptzn(Instruction *cpyLoad, Instruction *cpyStore,
                             Value *cpyDst, Value *cpySrc, TypeSize cpyLen,
                             Align cpyAlign, BatchAAResults &BAA,
@@ -87,6 +87,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
   bool performStackMoveOptzn(Instruction *Load, Instruction *Store,
                              AllocaInst *DestAlloca, AllocaInst *SrcAlloca,
                              TypeSize Size, BatchAAResults &BAA);
+  bool isMemMoveMemSetDependency(MemMoveInst *M);
 
   void eraseInstruction(Instruction *I);
   bool iterateOnFunction(Function &F);
diff --git a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
index a40b0a2a9ab72..0cc49e4755257 100644
--- a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
+++ b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
@@ -13,6 +13,7 @@
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include "llvm/Transforms/Utils/ExtraPassManager.h"
 
 namespace llvm {
 
@@ -21,40 +22,6 @@ class Loop;
 class StringRef;
 class raw_ostream;
 
-struct ShouldRunExtraSimpleLoopUnswitch
-    : public AnalysisInfoMixin<ShouldRunExtraSimpleLoopUnswitch> {
-  static AnalysisKey Key;
-  struct Result {
-    bool invalidate(Loop &L, const PreservedAnalyses &PA,
-                    LoopAnalysisManager::Invalidator &) {
-      // Check whether the analysis has been explicitly invalidated. Otherwise,
-      // it remains preserved.
-      auto PAC = PA.getChecker<ShouldRunExtraSimpleLoopUnswitch>();
-      return !PAC.preservedWhenStateless();
-    }
-  };
-
-  Result run(Loop &L, LoopAnalysisManager &AM,
-             LoopStandardAnalysisResults &AR) {
-    return Result();
-  }
-
-  static bool isRequired() { return true; }
-};
-
-struct ExtraSimpleLoopUnswitchPassManager : public LoopPassManager {
-  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
-                        LoopStandardAnalysisResults &AR, LPMUpdater &U) {
-    auto PA = PreservedAnalyses::all();
-    if (AM.getCachedResult<ShouldRunExtraSimpleLoopUnswitch>(L))
-      PA.intersect(LoopPassManager::run(L, AM, AR, U));
-    PA.abandon<ShouldRunExtraSimpleLoopUnswitch>();
-    return PA;
-  }
-
-  static bool isRequired() { return true; }
-};
-
 /// This pass transforms loops that contain branches or switches on loop-
 /// invariant conditions to have multiple loops. For example, it turns the left
 /// into the right code:
@@ -113,6 +80,14 @@ class SimpleLoopUnswitchPass : public PassInfoMixin<SimpleLoopUnswitchPass> {
                      function_ref<StringRef(StringRef)> MapClassName2PassName);
 };
 
+/// A marker analysis to determine if SimpleLoopUnswitch should run again on a
+/// given loop.
+struct ShouldRunExtraSimpleLoopUnswitch
+    : public ShouldRunExtraPasses<ShouldRunExtraSimpleLoopUnswitch>,
+      public AnalysisInfoMixin<ShouldRunExtraSimpleLoopUnswitch> {
+  static AnalysisKey Key;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_SCALAR_SIMPLELOOPUNSWITCH_H
diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index a8fb38e726004..0508f92dfd2ff 100644
--- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -246,32 +246,32 @@ namespace llvm {
                     const DataLayout &DL, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the malloc function.
-  Value *emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL,
-                    const TargetLibraryInfo *TLI);
+  Value *emitMalloc(Type *RetTy, Value *Num, IRBuilderBase &B,
+                    const DataLayout &DL, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the calloc function.
-  Value *emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
-                    const TargetLibraryInfo &TLI, unsigned AddrSpace);
+  Value *emitCalloc(Type *RetTy, Value *Num, Value *Size, IRBuilderBase &B,
+                    const TargetLibraryInfo &TLI);
 
   /// Emit a call to the hot/cold operator new function.
-  Value *emitHotColdNew(Value *Num, IRBuilderBase &B,
+  Value *emitHotColdNew(Type *RetTy, Value *Num, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI, LibFunc NewFunc,
                         uint8_t HotCold);
-  Value *emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
-                               const TargetLibraryInfo *TLI, LibFunc NewFunc,
-                               uint8_t HotCold);
-  Value *emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
-                               const TargetLibraryInfo *TLI, LibFunc NewFunc,
-                               uint8_t HotCold);
-  Value *emitHotColdNewAlignedNoThrow(Value *Num, Value *Align, Value *NoThrow,
-                                      IRBuilderBase &B,
+  Value *emitHotColdNewNoThrow(Type *RetTy, Value *Num, Value *NoThrow,
+                               IRBuilderBase &B, const TargetLibraryInfo *TLI,
+                               LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdNewAligned(Type *RetTy, Value *Num, Value *Align,
+                               IRBuilderBase &B, const TargetLibraryInfo *TLI,
+                               LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdNewAlignedNoThrow(Type *RetTy, Value *Num, Value *Align,
+                                      Value *NoThrow, IRBuilderBase &B,
                                       const TargetLibraryInfo *TLI,
                                       LibFunc NewFunc, uint8_t HotCold);
-  Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+  Value *emitHotColdSizeReturningNew(Type *RetTy, Value *Num, IRBuilderBase &B,
                                      const TargetLibraryInfo *TLI,
                                      LibFunc NewFunc, uint8_t HotCold);
-  Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
-                                            IRBuilderBase &B,
+  Value *emitHotColdSizeReturningNewAligned(Type *RetTy, Value *Num,
+                                            Value *Align, IRBuilderBase &B,
                                             const TargetLibraryInfo *TLI,
                                             LibFunc NewFunc, uint8_t HotCold);
 }
diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h
index 049d68b8a3068..7858c9d9def0d 100644
--- a/llvm/include/llvm/Transforms/Utils/Cloning.h
+++ b/llvm/include/llvm/Transforms/Utils/Cloning.h
@@ -119,8 +119,7 @@ struct ClonedCodeInfo {
 /// parameter.
 BasicBlock *CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
                             const Twine &NameSuffix = "", Function *F = nullptr,
-                            ClonedCodeInfo *CodeInfo = nullptr,
-                            DebugInfoFinder *DIFinder = nullptr);
+                            ClonedCodeInfo *CodeInfo = nullptr);
 
 /// Return a copy of the specified function and add it to that
 /// function's module.  Also, any references specified in the VMap are changed
@@ -221,6 +220,14 @@ DISubprogram *CollectDebugInfoForCloning(const Function &F,
                                          CloneFunctionChangeType Changes,
                                          DebugInfoFinder &DIFinder);
 
+/// Build a map of debug info to use during Metadata cloning.
+/// Returns true if cloning would need module level changes and false if there
+/// would only be local changes.
+bool BuildDebugInfoMDMap(DenseMap<const Metadata *, TrackingMDRef> &MD,
+                         CloneFunctionChangeType Changes,
+                         DebugInfoFinder &DIFinder,
+                         DISubprogram *SPClonedWithinModule);
+
 /// This class captures the data input to the InlineFunction call, and records
 /// the auxiliary results produced by it.
 class InlineFunctionInfo {
diff --git a/llvm/include/llvm/Transforms/Utils/ExtraPassManager.h b/llvm/include/llvm/Transforms/Utils/ExtraPassManager.h
new file mode 100644
index 0000000000000..7ea50a5584dde
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/ExtraPassManager.h
@@ -0,0 +1,85 @@
+//===- ExtraFunctionPassManager.h - Run Optimizations on Demand -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides a pass manager that only runs its passes if the
+/// provided marker analysis has been preserved, together with a class to
+/// define such a marker analysis.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_EXTRAPASSMANAGER_H
+#define LLVM_TRANSFORMS_UTILS_EXTRAPASSMANAGER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+
+namespace llvm {
+
+/// A marker analysis to determine if extra passes should be run on demand.
+/// Passes requesting extra transformations to run need to request and preserve
+/// this analysis.
+template <typename MarkerTy> struct ShouldRunExtraPasses {
+  struct Result {
+    bool invalidate(Function &F, const PreservedAnalyses &PA,
+                    FunctionAnalysisManager::Invalidator &) {
+      // Check whether the analysis has been explicitly invalidated. Otherwise,
+      // it remains preserved.
+      auto PAC = PA.getChecker<MarkerTy>();
+      return !PAC.preservedWhenStateless();
+    }
+
+    bool invalidate(Loop &L, const PreservedAnalyses &PA,
+                    LoopAnalysisManager::Invalidator &) {
+      // Check whether the analysis has been explicitly invalidated. Otherwise,
+      // it remains preserved.
+      auto PAC = PA.getChecker<MarkerTy>();
+      return !PAC.preservedWhenStateless();
+    }
+  };
+
+  Result run(Function &F, FunctionAnalysisManager &FAM) { return Result(); }
+
+  Result run(Loop &L, LoopAnalysisManager &AM,
+             LoopStandardAnalysisResults &AR) {
+    return Result();
+  }
+};
+
+/// A pass manager to run a set of extra function passes if the
+/// ShouldRunExtraPasses marker analysis is present. This allows passes to
+/// request additional transformations on demand. An example is extra
+/// simplifications after loop-vectorization, if runtime checks have been added.
+template <typename MarkerTy>
+struct ExtraFunctionPassManager : public FunctionPassManager {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {
+    auto PA = PreservedAnalyses::all();
+    if (AM.getCachedResult<MarkerTy>(F))
+      PA.intersect(FunctionPassManager::run(F, AM));
+    PA.abandon<MarkerTy>();
+    return PA;
+  }
+};
+
+/// A pass manager to run a set of extra loop passes if the MarkerTy analysis is
+/// present. This allows passes to request additional transformations on demand.
+/// An example is doing additional runs of SimpleLoopUnswitch.
+template <typename MarkerTy>
+struct ExtraLoopPassManager : public LoopPassManager {
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U) {
+    auto PA = PreservedAnalyses::all();
+    if (AM.getCachedResult<MarkerTy>(L))
+      PA.intersect(LoopPassManager::run(L, AM, AR, U));
+    PA.abandon<MarkerTy>();
+    return PA;
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_EXTRAPASSMANAGER_H
diff --git a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
index c28f868039a1f..19c5f7449f23e 100644
--- a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
+++ b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
@@ -317,6 +317,7 @@ class FunctionComparator {
   int cmpNumbers(uint64_t L, uint64_t R) const;
   int cmpAligns(Align L, Align R) const;
   int cmpAPInts(const APInt &L, const APInt &R) const;
+  int cmpConstantRanges(const ConstantRange &L, const ConstantRange &R) const;
   int cmpAPFloats(const APFloat &L, const APFloat &R) const;
   int cmpMem(StringRef L, StringRef R) const;
 
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 797c082333a76..8cf17ced458c8 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -75,6 +75,7 @@ struct UnrollLoopOptions {
   bool UnrollRemainder;
   bool ForgetAllSCEV;
   const Instruction *Heart = nullptr;
+  unsigned SCEVExpansionBudget;
 };
 
 LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -90,7 +91,7 @@ bool UnrollRuntimeLoopRemainder(
     bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
     LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
     const TargetTransformInfo *TTI, bool PreserveLCSSA,
-    Loop **ResultLoop = nullptr);
+    unsigned SCEVExpansionBudget, Loop **ResultLoop = nullptr);
 
 LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
                                   unsigned TripMultiple, bool UnrollRemainder,
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 24b6354662955..02935d0943f76 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -58,6 +58,7 @@
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/ExtraPassManager.h"
 #include <functional>
 
 namespace llvm {
@@ -80,38 +81,6 @@ class TargetTransformInfo;
 extern cl::opt<bool> EnableLoopInterleaving;
 extern cl::opt<bool> EnableLoopVectorization;
 
-/// A marker to determine if extra passes after loop vectorization should be
-/// run.
-struct ShouldRunExtraVectorPasses
-    : public AnalysisInfoMixin<ShouldRunExtraVectorPasses> {
-  static AnalysisKey Key;
-  struct Result {
-    bool invalidate(Function &F, const PreservedAnalyses &PA,
-                    FunctionAnalysisManager::Invalidator &) {
-      // Check whether the analysis has been explicitly invalidated. Otherwise,
-      // it remains preserved.
-      auto PAC = PA.getChecker<ShouldRunExtraVectorPasses>();
-      return !PAC.preservedWhenStateless();
-    }
-  };
-
-  Result run(Function &F, FunctionAnalysisManager &FAM) { return Result(); }
-};
-
-/// A pass manager to run a set of extra function simplification passes after
-/// vectorization, if requested. LoopVectorize caches the
-/// ShouldRunExtraVectorPasses analysis to request extra simplifications, if
-/// they could be beneficial.
-struct ExtraVectorPassManager : public FunctionPassManager {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {
-    auto PA = PreservedAnalyses::all();
-    if (AM.getCachedResult<ShouldRunExtraVectorPasses>(F))
-      PA.intersect(FunctionPassManager::run(F, AM));
-    PA.abandon<ShouldRunExtraVectorPasses>();
-    return PA;
-  }
-};
-
 struct LoopVectorizeOptions {
   /// If false, consider all loops for interleaving.
   /// If true, only loops that explicitly request interleaving are considered.
@@ -201,6 +170,13 @@ void reportVectorizationFailure(const StringRef DebugMsg,
     const StringRef OREMsg, const StringRef ORETag,
     OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I = nullptr);
 
+/// A marker analysis to determine if extra passes should be run after loop
+/// vectorization.
+struct ShouldRunExtraVectorPasses
+    : public ShouldRunExtraPasses<ShouldRunExtraVectorPasses>,
+      public AnalysisInfoMixin<ShouldRunExtraVectorPasses> {
+  static AnalysisKey Key;
+};
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index 68a2daca1403d..b1cad2421bc0d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -329,18 +329,20 @@ class DependencyGraph {
   /// chain.
   void createNewNodes(const Interval<Instruction> &NewInterval);
 
+  /// Helper for `notify*Instr()`. \Returns the first MemDGNode that comes
+  /// before \p N, including or excluding \p N based on \p IncludingN, or
+  /// nullptr if not found.
+  MemDGNode *getMemDGNodeBefore(DGNode *N, bool IncludingN) const;
+  /// Helper for `notifyMoveInstr()`. \Returns the first MemDGNode that comes
+  /// after \p N, including or excluding \p N based on \p IncludingN, or nullptr
+  /// if not found.
+  MemDGNode *getMemDGNodeAfter(DGNode *N, bool IncludingN) const;
+
   /// Called by the callbacks when a new instruction \p I has been created.
-  void notifyCreateInstr(Instruction *I) {
-    getOrCreateNode(I);
-    // TODO: Update the dependencies for the new node.
-    // TODO: Update the MemDGNode chain to include the new node if needed.
-  }
-  /// Called by the callbacks when instruction \p I is about to get deleted.
-  void notifyEraseInstr(Instruction *I) {
-    InstrToNodeMap.erase(I);
-    // TODO: Update the dependencies.
-    // TODO: Update the MemDGNode chain to remove the node if needed.
-  }
+  void notifyCreateInstr(Instruction *I);
+  /// Called by the callbacks when instruction \p I is about to get
+  /// deleted.
+  void notifyEraseInstr(Instruction *I);
 
 public:
   /// This constructor also registers callbacks.
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index 02f2a3bcdb8bf..62cdce56fe388 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -828,6 +828,14 @@ bool llvm::isIdentifiedFunctionLocal(const Value *V) {
   return isa<AllocaInst>(V) || isNoAliasCall(V) || isNoAliasOrByValArgument(V);
 }
 
+bool llvm::isBaseOfObject(const Value *V) {
+  // TODO: We can handle other cases here
+  // 1) For GC languages, arguments to functions are often required to be
+  //    base pointers.
+  // 2) Result of allocation routines are often base pointers.  Leverage TLI.
+  return (isa<AllocaInst>(V) || isa<GlobalVariable>(V));
+}
+
 bool llvm::isEscapeSource(const Value *V) {
   if (auto *CB = dyn_cast<CallBase>(V))
     return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(CB,
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 178ad863eb06a..dc54d566e55b1 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -947,8 +947,14 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
   //
   // Make sure the object has not escaped here, and then check that none of the
   // call arguments alias the object below.
+  //
+  // We model calls that can return twice (setjmp) as clobbering non-escaping
+  // objects, to model any accesses that may occur prior to the second return.
+  // As an exception, ignore allocas, as setjmp is not required to preserve
+  // non-volatile stores for them.
   if (!isa<Constant>(Object) && Call != Object &&
-      AAQI.CA->isNotCapturedBefore(Object, Call, /*OrAt*/ false)) {
+      AAQI.CA->isNotCapturedBefore(Object, Call, /*OrAt*/ false) &&
+      (isa<AllocaInst>(Object) || !Call->hasFnAttr(Attribute::ReturnsTwice))) {
 
     // Optimistically assume that call doesn't touch Object and check this
     // assumption in the following loop.
@@ -1068,19 +1074,6 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1,
   return ModRefInfo::ModRef;
 }
 
-/// Return true if we know V to the base address of the corresponding memory
-/// object.  This implies that any address less than V must be out of bounds
-/// for the underlying object.  Note that just being isIdentifiedObject() is
-/// not enough - For example, a negative offset from a noalias argument or call
-/// can be inbounds w.r.t the actual underlying object.
-static bool isBaseOfObject(const Value *V) {
-  // TODO: We can handle other cases here
-  // 1) For GC languages, arguments to functions are often required to be
-  //    base pointers.
-  // 2) Result of allocation routines are often base pointers.  Leverage TLI.
-  return (isa<AllocaInst>(V) || isa<GlobalVariable>(V));
-}
-
 /// Provides a bunch of ad-hoc rules to disambiguate a GEP instruction against
 /// another pointer.
 ///
@@ -1449,9 +1442,10 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize,
     return AliasResult::NoAlias;
   // If the values are PHIs in the same block, we can do a more precise
   // as well as efficient check: just check for aliases between the values
-  // on corresponding edges.
+  // on corresponding edges. Don't do this if we are analyzing across
+  // iterations, as we may pick a different phi entry in different iterations.
   if (const PHINode *PN2 = dyn_cast<PHINode>(V2))
-    if (PN2->getParent() == PN->getParent()) {
+    if (PN2->getParent() == PN->getParent() && !AAQI.MayBeCrossIteration) {
       std::optional<AliasResult> Alias;
       for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
         AliasResult ThisAlias = AAQI.AAR.alias(
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 1971c28fc4c4d..3d5022e5502e2 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -151,8 +151,14 @@ Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
     return FoldBitCast(ConstantVector::get(Ops), DestTy, DL);
   }
 
+  // Some of what follows may extend to cover scalable vectors but the current
+  // implementation is fixed length specific.
+  if (!isa<FixedVectorType>(C->getType()))
+    return ConstantExpr::getBitCast(C, DestTy);
+
   // If this is a bitcast from constant vector -> vector, fold it.
-  if (!isa<ConstantDataVector>(C) && !isa<ConstantVector>(C))
+  if (!isa<ConstantDataVector>(C) && !isa<ConstantVector>(C) &&
+      !isa<ConstantInt>(C) && !isa<ConstantFP>(C))
     return ConstantExpr::getBitCast(C, DestTy);
 
   // If the element types match, IR can fold it.
@@ -194,10 +200,9 @@ Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
         IntegerType::get(C->getContext(), FPWidth), NumSrcElt);
     // Ask IR to do the conversion now that #elts line up.
     C = ConstantExpr::getBitCast(C, SrcIVTy);
-    // If IR wasn't able to fold it, bail out.
-    if (!isa<ConstantVector>(C) &&  // FIXME: Remove ConstantVector.
-        !isa<ConstantDataVector>(C))
-      return C;
+    assert((isa<ConstantVector>(C) || // FIXME: Remove ConstantVector.
+            isa<ConstantDataVector>(C) || isa<ConstantInt>(C)) &&
+           "Constant folding cannot fail for plain fp->int bitcast!");
   }
 
   // Now we know that the input and output vectors are both integer vectors
@@ -881,7 +886,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
   Type *IntIdxTy = DL.getIndexType(Ptr->getType());
 
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
-    if (!isa<ConstantInt>(Ops[i]))
+    if (!isa<ConstantInt>(Ops[i]) || !Ops[i]->getType()->isIntegerTy())
       return nullptr;
 
   unsigned BitWidth = DL.getTypeSizeInBits(IntIdxTy);
@@ -952,7 +957,6 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
   }
 
   // Try to infer inbounds for GEPs of globals.
-  // TODO(gep_nowrap): Also infer nuw flag.
   if (!NW.isInBounds() && Offset.isNonNegative()) {
     bool CanBeNull, CanBeFreed;
     uint64_t DerefBytes =
@@ -961,6 +965,10 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
       NW |= GEPNoWrapFlags::inBounds();
   }
 
+  // nusw + nneg -> nuw
+  if (NW.hasNoUnsignedSignedWrap() && Offset.isNonNegative())
+    NW |= GEPNoWrapFlags::noUnsignedWrap();
+
   // Otherwise canonicalize this to a single ptradd.
   LLVMContext &Ctx = Ptr->getContext();
   return ConstantExpr::getGetElementPtr(Type::getInt8Ty(Ctx), Ptr,
@@ -1632,6 +1640,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::sqrt:
   case Intrinsic::sin:
   case Intrinsic::cos:
+  case Intrinsic::sincos:
   case Intrinsic::pow:
   case Intrinsic::powi:
   case Intrinsic::ldexp:
@@ -3018,9 +3027,9 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
       assert(C1 && "Must be constant int");
       assert((C1->isOne() || C1->isZero()) && "Must be 0 or 1");
 
-      // Undef or minimum val operand with poison min --> undef
+      // Undef or minimum val operand with poison min --> poison
       if (C1->isOne() && (!C0 || C0->isMinSignedValue()))
-        return UndefValue::get(Ty);
+        return PoisonValue::get(Ty);
 
       // Undef operand with no poison min --> 0 (sign bit must be clear)
       if (!C0)
@@ -3534,6 +3543,40 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
       return nullptr;
     return ConstantStruct::get(StTy, Result0, Result1);
   }
+  case Intrinsic::sincos: {
+    Type *Ty = StTy->getContainedType(0);
+    Type *TyScalar = Ty->getScalarType();
+
+    auto ConstantFoldScalarSincosCall =
+        [&](Constant *Op) -> std::pair<Constant *, Constant *> {
+      Constant *SinResult =
+          ConstantFoldScalarCall(Name, Intrinsic::sin, TyScalar, Op, TLI, Call);
+      Constant *CosResult =
+          ConstantFoldScalarCall(Name, Intrinsic::cos, TyScalar, Op, TLI, Call);
+      return std::make_pair(SinResult, CosResult);
+    };
+
+    if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
+      SmallVector<Constant *> SinResults(FVTy->getNumElements());
+      SmallVector<Constant *> CosResults(FVTy->getNumElements());
+
+      for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
+        Constant *Lane = Operands[0]->getAggregateElement(I);
+        std::tie(SinResults[I], CosResults[I]) =
+            ConstantFoldScalarSincosCall(Lane);
+        if (!SinResults[I] || !CosResults[I])
+          return nullptr;
+      }
+
+      return ConstantStruct::get(StTy, ConstantVector::get(SinResults),
+                                 ConstantVector::get(CosResults));
+    }
+
+    auto [SinResult, CosResult] = ConstantFoldScalarSincosCall(Operands[0]);
+    if (!SinResult || !CosResult)
+      return nullptr;
+    return ConstantStruct::get(StTy, SinResult, CosResult);
+  }
   default:
     // TODO: Constant folding of vector intrinsics that fall through here does
     // not work (e.g. overflow intrinsics)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 23e11bdbeab4c..e1eb219cf977e 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1063,7 +1063,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
 SmallVector<Instruction *, 4>
 RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
   SmallVector<Instruction *, 4> ReductionOperations;
-  unsigned RedOp = getOpcode(Kind);
+  unsigned RedOp = getOpcode();
 
   // Search down from the Phi to the LoopExitInstr, looking for instructions
   // with a single user of the correct type for the reduction.
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 22bb406c01a4e..85287a39f2caa 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -249,6 +249,9 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   /// Getter for BlockFrequencyInfo
   function_ref<BlockFrequencyInfo &(Function &)> GetBFI;
 
+  /// Getter for TargetLibraryInfo
+  function_ref<const TargetLibraryInfo &(Function &)> GetTLI;
+
   /// Profile summary information.
   ProfileSummaryInfo *PSI;
 
@@ -419,6 +422,14 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
     return It->second;
   }
 
+  /// Use a value in its given form directly if possible, otherwise try looking
+  /// for it in SimplifiedValues.
+  template <typename T> T *getDirectOrSimplifiedValue(Value *V) const {
+    if (auto *Direct = dyn_cast<T>(V))
+      return Direct;
+    return dyn_cast_if_present<T>(SimplifiedValues.lookup(V));
+  }
+
   // Custom simplification helper routines.
   bool isAllocaDerivedArg(Value *V);
   void disableSROAForArg(AllocaInst *SROAArg);
@@ -433,6 +444,7 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool simplifyIntrinsicCallIsConstant(CallBase &CB);
   bool simplifyIntrinsicCallObjectSize(CallBase &CB);
   ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V);
+  bool isLoweredToCall(Function *F, CallBase &Call);
 
   /// Return true if the given argument to the function being considered for
   /// inlining has the given attribute set either at the call site or the
@@ -492,13 +504,15 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool visitUnreachableInst(UnreachableInst &I);
 
 public:
-  CallAnalyzer(Function &Callee, CallBase &Call, const TargetTransformInfo &TTI,
-               function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
-               function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
-               ProfileSummaryInfo *PSI = nullptr,
-               OptimizationRemarkEmitter *ORE = nullptr)
+  CallAnalyzer(
+      Function &Callee, CallBase &Call, const TargetTransformInfo &TTI,
+      function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
+      function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
+      function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
+      ProfileSummaryInfo *PSI = nullptr,
+      OptimizationRemarkEmitter *ORE = nullptr)
       : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI),
-        PSI(PSI), F(Callee), DL(F.getDataLayout()), ORE(ORE),
+        GetTLI(GetTLI), PSI(PSI), F(Callee), DL(F.getDataLayout()), ORE(ORE),
         CandidateCall(Call) {}
 
   InlineResult analyze();
@@ -688,7 +702,8 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
       /// FIXME: if InlineCostCallAnalyzer is derived from, this may need
       /// to instantiate the derived class.
       InlineCostCallAnalyzer CA(*F, Call, IndirectCallParams, TTI,
-                                GetAssumptionCache, GetBFI, PSI, ORE, false);
+                                GetAssumptionCache, GetBFI, GetTLI, PSI, ORE,
+                                false);
       if (CA.analyze().isSuccess()) {
         // We were able to inline the indirect call! Subtract the cost from the
         // threshold to get the bonus we want to apply, but don't go below zero.
@@ -1106,10 +1121,12 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
       const TargetTransformInfo &TTI,
       function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
       function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
+      function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
       ProfileSummaryInfo *PSI = nullptr,
       OptimizationRemarkEmitter *ORE = nullptr, bool BoostIndirect = true,
       bool IgnoreThreshold = false)
-      : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, PSI, ORE),
+      : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, GetTLI, PSI,
+                     ORE),
         ComputeFullInlineCost(OptComputeFullInlineCost ||
                               Params.ComputeFullInlineCost || ORE ||
                               isCostBenefitAnalysisEnabled()),
@@ -1228,8 +1245,8 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
           InlineConstants::IndirectCallThreshold;
 
       InlineCostCallAnalyzer CA(*F, Call, IndirectCallParams, TTI,
-                                GetAssumptionCache, GetBFI, PSI, ORE, false,
-                                true);
+                                GetAssumptionCache, GetBFI, GetTLI, PSI, ORE,
+                                false, true);
       if (CA.analyze().isSuccess()) {
         increment(InlineCostFeatureIndex::nested_inline_cost_estimate,
                   CA.getCost());
@@ -1355,9 +1372,11 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
       const TargetTransformInfo &TTI,
       function_ref<AssumptionCache &(Function &)> &GetAssumptionCache,
       function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
+      function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
       ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE, Function &Callee,
       CallBase &Call)
-      : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, PSI) {}
+      : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, GetTLI,
+                     PSI) {}
 
   const InlineCostFeatures &features() const { return Cost; }
 };
@@ -1424,10 +1443,8 @@ bool CallAnalyzer::accumulateGEPOffset(GEPOperator &GEP, APInt &Offset) {
 
   for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
        GTI != GTE; ++GTI) {
-    ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
-    if (!OpC)
-      if (Constant *SimpleOp = SimplifiedValues.lookup(GTI.getOperand()))
-        OpC = dyn_cast<ConstantInt>(SimpleOp);
+    ConstantInt *OpC =
+        getDirectOrSimplifiedValue<ConstantInt>(GTI.getOperand());
     if (!OpC)
       return false;
     if (OpC->isZero())
@@ -1541,9 +1558,7 @@ bool CallAnalyzer::visitPHI(PHINode &I) {
     if (&I == V)
       continue;
 
-    Constant *C = dyn_cast<Constant>(V);
-    if (!C)
-      C = SimplifiedValues.lookup(V);
+    Constant *C = getDirectOrSimplifiedValue<Constant>(V);
 
     std::pair<Value *, APInt> BaseAndOffset = {nullptr, ZeroOffset};
     if (!C && CheckSROA)
@@ -1628,7 +1643,7 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) {
   // Lambda to check whether a GEP's indices are all constant.
   auto IsGEPOffsetConstant = [&](GetElementPtrInst &GEP) {
     for (const Use &Op : GEP.indices())
-      if (!isa<Constant>(Op) && !SimplifiedValues.lookup(Op))
+      if (!getDirectOrSimplifiedValue<Constant>(Op))
         return false;
     return true;
   };
@@ -1655,9 +1670,7 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) {
 bool CallAnalyzer::simplifyInstruction(Instruction &I) {
   SmallVector<Constant *> COps;
   for (Value *Op : I.operands()) {
-    Constant *COp = dyn_cast<Constant>(Op);
-    if (!COp)
-      COp = SimplifiedValues.lookup(Op);
+    Constant *COp = getDirectOrSimplifiedValue<Constant>(Op);
     if (!COp)
       return false;
     COps.push_back(COp);
@@ -1680,10 +1693,7 @@ bool CallAnalyzer::simplifyInstruction(Instruction &I) {
 /// llvm.is.constant would evaluate.
 bool CallAnalyzer::simplifyIntrinsicCallIsConstant(CallBase &CB) {
   Value *Arg = CB.getArgOperand(0);
-  auto *C = dyn_cast<Constant>(Arg);
-
-  if (!C)
-    C = dyn_cast_or_null<Constant>(SimplifiedValues.lookup(Arg));
+  auto *C = getDirectOrSimplifiedValue<Constant>(Arg);
 
   Type *RT = CB.getFunctionType()->getReturnType();
   SimplifiedValues[&CB] = ConstantInt::get(RT, C ? 1 : 0);
@@ -2115,12 +2125,8 @@ bool CallAnalyzer::visitSub(BinaryOperator &I) {
 
 bool CallAnalyzer::visitBinaryOperator(BinaryOperator &I) {
   Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
-  Constant *CLHS = dyn_cast<Constant>(LHS);
-  if (!CLHS)
-    CLHS = SimplifiedValues.lookup(LHS);
-  Constant *CRHS = dyn_cast<Constant>(RHS);
-  if (!CRHS)
-    CRHS = SimplifiedValues.lookup(RHS);
+  Constant *CLHS = getDirectOrSimplifiedValue<Constant>(LHS);
+  Constant *CRHS = getDirectOrSimplifiedValue<Constant>(RHS);
 
   Value *SimpleV = nullptr;
   if (auto FI = dyn_cast<FPMathOperator>(&I))
@@ -2154,9 +2160,7 @@ bool CallAnalyzer::visitBinaryOperator(BinaryOperator &I) {
 
 bool CallAnalyzer::visitFNeg(UnaryOperator &I) {
   Value *Op = I.getOperand(0);
-  Constant *COp = dyn_cast<Constant>(Op);
-  if (!COp)
-    COp = SimplifiedValues.lookup(Op);
+  Constant *COp = getDirectOrSimplifiedValue<Constant>(Op);
 
   Value *SimpleV = simplifyFNegInst(
       COp ? COp : Op, cast<FPMathOperator>(I).getFastMathFlags(), DL);
@@ -2244,9 +2248,7 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallBase &Call) {
   SmallVector<Constant *, 4> ConstantArgs;
   ConstantArgs.reserve(Call.arg_size());
   for (Value *I : Call.args()) {
-    Constant *C = dyn_cast<Constant>(I);
-    if (!C)
-      C = dyn_cast_or_null<Constant>(SimplifiedValues.lookup(I));
+    Constant *C = getDirectOrSimplifiedValue<Constant>(I);
     if (!C)
       return false; // This argument doesn't map to a constant.
 
@@ -2260,6 +2262,39 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallBase &Call) {
   return false;
 }
 
+bool CallAnalyzer::isLoweredToCall(Function *F, CallBase &Call) {
+  const TargetLibraryInfo *TLI = GetTLI ? &GetTLI(*F) : nullptr;
+  LibFunc LF;
+  if (!TLI || !TLI->getLibFunc(*F, LF) || !TLI->has(LF))
+    return TTI.isLoweredToCall(F);
+
+  switch (LF) {
+  case LibFunc_memcpy_chk:
+  case LibFunc_memmove_chk:
+  case LibFunc_mempcpy_chk:
+  case LibFunc_memset_chk: {
+    // Calls to  __memcpy_chk whose length is known to fit within the object
+    // size will eventually be replaced by inline stores. Therefore, these
+    // should not incur a call penalty. This is only really relevant on
+    // platforms whose headers redirect memcpy to __memcpy_chk (e.g. Darwin), as
+    // other platforms use memcpy intrinsics, which are already exempt from the
+    // call penalty.
+    auto *LenOp = getDirectOrSimplifiedValue<ConstantInt>(Call.getOperand(2));
+    auto *ObjSizeOp =
+        getDirectOrSimplifiedValue<ConstantInt>(Call.getOperand(3));
+    if (LenOp && ObjSizeOp &&
+        LenOp->getLimitedValue() <= ObjSizeOp->getLimitedValue()) {
+      return false;
+    }
+    break;
+  }
+  default:
+    break;
+  }
+
+  return TTI.isLoweredToCall(F);
+}
+
 bool CallAnalyzer::visitCallBase(CallBase &Call) {
   if (!onCallBaseVisitStart(Call))
     return true;
@@ -2341,7 +2376,7 @@ bool CallAnalyzer::visitCallBase(CallBase &Call) {
       return false;
   }
 
-  if (TTI.isLoweredToCall(F)) {
+  if (isLoweredToCall(F, Call)) {
     onLoweredCall(F, Call, IsIndirectCall);
   }
 
@@ -2362,10 +2397,9 @@ bool CallAnalyzer::visitBranchInst(BranchInst &BI) {
   // shouldn't exist at all, but handling them makes the behavior of the
   // inliner more regular and predictable. Interestingly, conditional branches
   // which will fold away are also free.
-  return BI.isUnconditional() || isa<ConstantInt>(BI.getCondition()) ||
-         BI.getMetadata(LLVMContext::MD_make_implicit) ||
-         isa_and_nonnull<ConstantInt>(
-             SimplifiedValues.lookup(BI.getCondition()));
+  return BI.isUnconditional() ||
+         getDirectOrSimplifiedValue<ConstantInt>(BI.getCondition()) ||
+         BI.getMetadata(LLVMContext::MD_make_implicit);
 }
 
 bool CallAnalyzer::visitSelectInst(SelectInst &SI) {
@@ -2373,12 +2407,8 @@ bool CallAnalyzer::visitSelectInst(SelectInst &SI) {
   Value *TrueVal = SI.getTrueValue();
   Value *FalseVal = SI.getFalseValue();
 
-  Constant *TrueC = dyn_cast<Constant>(TrueVal);
-  if (!TrueC)
-    TrueC = SimplifiedValues.lookup(TrueVal);
-  Constant *FalseC = dyn_cast<Constant>(FalseVal);
-  if (!FalseC)
-    FalseC = SimplifiedValues.lookup(FalseVal);
+  Constant *TrueC = getDirectOrSimplifiedValue<Constant>(TrueVal);
+  Constant *FalseC = getDirectOrSimplifiedValue<Constant>(FalseVal);
   Constant *CondC =
       dyn_cast_or_null<Constant>(SimplifiedValues.lookup(SI.getCondition()));
 
@@ -2448,11 +2478,8 @@ bool CallAnalyzer::visitSelectInst(SelectInst &SI) {
 bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) {
   // We model unconditional switches as free, see the comments on handling
   // branches.
-  if (isa<ConstantInt>(SI.getCondition()))
+  if (getDirectOrSimplifiedValue<ConstantInt>(SI.getCondition()))
     return true;
-  if (Value *V = SimplifiedValues.lookup(SI.getCondition()))
-    if (isa<ConstantInt>(V))
-      return true;
 
   // Assume the most general case where the switch is lowered into
   // either a jump table, bit test, or a balanced binary tree consisting of
@@ -2945,6 +2972,7 @@ std::optional<int> llvm::getInliningCostEstimate(
     CallBase &Call, TargetTransformInfo &CalleeTTI,
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
+    function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
     ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) {
   const InlineParams Params = {/* DefaultThreshold*/ 0,
                                /*HintThreshold*/ {},
@@ -2958,7 +2986,7 @@ std::optional<int> llvm::getInliningCostEstimate(
                                /*EnableDeferral*/ true};
 
   InlineCostCallAnalyzer CA(*Call.getCalledFunction(), Call, Params, CalleeTTI,
-                            GetAssumptionCache, GetBFI, PSI, ORE, true,
+                            GetAssumptionCache, GetBFI, GetTLI, PSI, ORE, true,
                             /*IgnoreThreshold*/ true);
   auto R = CA.analyze();
   if (!R.isSuccess())
@@ -2970,9 +2998,10 @@ std::optional<InlineCostFeatures> llvm::getInliningCostFeatures(
     CallBase &Call, TargetTransformInfo &CalleeTTI,
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
+    function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
     ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) {
-  InlineCostFeaturesAnalyzer CFA(CalleeTTI, GetAssumptionCache, GetBFI, PSI,
-                                 ORE, *Call.getCalledFunction(), Call);
+  InlineCostFeaturesAnalyzer CFA(CalleeTTI, GetAssumptionCache, GetBFI, GetTLI,
+                                 PSI, ORE, *Call.getCalledFunction(), Call);
   auto R = CFA.analyze();
   if (!R.isSuccess())
     return std::nullopt;
@@ -3072,7 +3101,7 @@ InlineCost llvm::getInlineCost(
                           << ")\n");
 
   InlineCostCallAnalyzer CA(*Callee, Call, Params, CalleeTTI,
-                            GetAssumptionCache, GetBFI, PSI, ORE);
+                            GetAssumptionCache, GetBFI, GetTLI, PSI, ORE);
   InlineResult ShouldInline = CA.analyze();
 
   LLVM_DEBUG(CA.dump());
@@ -3263,7 +3292,8 @@ InlineCostAnnotationPrinterPass::run(Function &F,
           continue;
         OptimizationRemarkEmitter ORE(CalledFunction);
         InlineCostCallAnalyzer ICCA(*CalledFunction, *CB, Params, TTI,
-                                    GetAssumptionCache, nullptr, &PSI, &ORE);
+                                    GetAssumptionCache, nullptr, nullptr, &PSI,
+                                    &ORE);
         ICCA.analyze();
         OS << "      Analyzing call of " << CalledFunction->getName()
            << "... (caller:" << CB->getCaller()->getName() << ")\n";
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 01b0a089aab71..62edea38745b1 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -63,9 +63,9 @@ static Value *simplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &,
                             unsigned);
 static Value *simplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &,
                             const SimplifyQuery &, unsigned);
-static Value *simplifyCmpInst(unsigned, Value *, Value *, const SimplifyQuery &,
-                              unsigned);
-static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+static Value *simplifyCmpInst(CmpPredicate, Value *, Value *,
+                              const SimplifyQuery &, unsigned);
+static Value *simplifyICmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                                const SimplifyQuery &Q, unsigned MaxRecurse);
 static Value *simplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned);
 static Value *simplifyXorInst(Value *, Value *, const SimplifyQuery &,
@@ -81,48 +81,6 @@ static Value *simplifyInstructionWithOperands(Instruction *I,
                                               const SimplifyQuery &SQ,
                                               unsigned MaxRecurse);
 
-static Value *foldSelectWithBinaryOp(Value *Cond, Value *TrueVal,
-                                     Value *FalseVal) {
-  BinaryOperator::BinaryOps BinOpCode;
-  if (auto *BO = dyn_cast<BinaryOperator>(Cond))
-    BinOpCode = BO->getOpcode();
-  else
-    return nullptr;
-
-  CmpInst::Predicate ExpectedPred;
-  if (BinOpCode == BinaryOperator::Or) {
-    ExpectedPred = ICmpInst::ICMP_NE;
-  } else if (BinOpCode == BinaryOperator::And) {
-    ExpectedPred = ICmpInst::ICMP_EQ;
-  } else
-    return nullptr;
-
-  // %A = icmp eq %TV, %FV
-  // %B = icmp eq %X, %Y (and one of these is a select operand)
-  // %C = and %A, %B
-  // %D = select %C, %TV, %FV
-  // -->
-  // %FV
-
-  // %A = icmp ne %TV, %FV
-  // %B = icmp ne %X, %Y (and one of these is a select operand)
-  // %C = or %A, %B
-  // %D = select %C, %TV, %FV
-  // -->
-  // %TV
-  Value *X, *Y;
-  if (!match(Cond,
-             m_c_BinOp(m_c_SpecificICmp(ExpectedPred, m_Specific(TrueVal),
-                                        m_Specific(FalseVal)),
-                       m_SpecificICmp(ExpectedPred, m_Value(X), m_Value(Y)))))
-    return nullptr;
-
-  if (X == TrueVal || X == FalseVal || Y == TrueVal || Y == FalseVal)
-    return BinOpCode == BinaryOperator::Or ? TrueVal : FalseVal;
-
-  return nullptr;
-}
-
 /// For a boolean type or a vector of boolean type, return false or a vector
 /// with every element false.
 static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); }
@@ -132,8 +90,7 @@ static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); }
 static Constant *getTrue(Type *Ty) { return ConstantInt::getTrue(Ty); }
 
 /// isSameCompare - Is V equivalent to the comparison "LHS Pred RHS"?
-static bool isSameCompare(Value *V, CmpInst::Predicate Pred, Value *LHS,
-                          Value *RHS) {
+static bool isSameCompare(Value *V, CmpPredicate Pred, Value *LHS, Value *RHS) {
   CmpInst *Cmp = dyn_cast<CmpInst>(V);
   if (!Cmp)
     return false;
@@ -150,10 +107,9 @@ static bool isSameCompare(Value *V, CmpInst::Predicate Pred, Value *LHS,
 ///  %cmp = icmp sle i32 %sel, %rhs
 /// Compose new comparison by substituting %sel with either %tv or %fv
 /// and see if it simplifies.
-static Value *simplifyCmpSelCase(CmpInst::Predicate Pred, Value *LHS,
-                                 Value *RHS, Value *Cond,
-                                 const SimplifyQuery &Q, unsigned MaxRecurse,
-                                 Constant *TrueOrFalse) {
+static Value *simplifyCmpSelCase(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                 Value *Cond, const SimplifyQuery &Q,
+                                 unsigned MaxRecurse, Constant *TrueOrFalse) {
   Value *SimplifiedCmp = simplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse);
   if (SimplifiedCmp == Cond) {
     // %cmp simplified to the select condition (%cond).
@@ -167,18 +123,16 @@ static Value *simplifyCmpSelCase(CmpInst::Predicate Pred, Value *LHS,
 }
 
 /// Simplify comparison with true branch of select
-static Value *simplifyCmpSelTrueCase(CmpInst::Predicate Pred, Value *LHS,
-                                     Value *RHS, Value *Cond,
-                                     const SimplifyQuery &Q,
+static Value *simplifyCmpSelTrueCase(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                     Value *Cond, const SimplifyQuery &Q,
                                      unsigned MaxRecurse) {
   return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,
                             getTrue(Cond->getType()));
 }
 
 /// Simplify comparison with false branch of select
-static Value *simplifyCmpSelFalseCase(CmpInst::Predicate Pred, Value *LHS,
-                                      Value *RHS, Value *Cond,
-                                      const SimplifyQuery &Q,
+static Value *simplifyCmpSelFalseCase(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                      Value *Cond, const SimplifyQuery &Q,
                                       unsigned MaxRecurse) {
   return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,
                             getFalse(Cond->getType()));
@@ -471,9 +425,8 @@ static Value *threadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS,
 /// We can simplify %cmp1 to true, because both branches of select are
 /// less than 3. We compose new comparison by substituting %tmp with both
 /// branches of select and see if it can be simplified.
-static Value *threadCmpOverSelect(CmpInst::Predicate Pred, Value *LHS,
-                                  Value *RHS, const SimplifyQuery &Q,
-                                  unsigned MaxRecurse) {
+static Value *threadCmpOverSelect(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                  const SimplifyQuery &Q, unsigned MaxRecurse) {
   // Recursion is always used, so bail out at once if we already hit the limit.
   if (!MaxRecurse--)
     return nullptr;
@@ -564,7 +517,7 @@ static Value *threadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS,
 /// comparison by seeing whether comparing with all of the incoming phi values
 /// yields the same result every time. If so returns the common result,
 /// otherwise returns null.
-static Value *threadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
+static Value *threadCmpOverPHI(CmpPredicate Pred, Value *LHS, Value *RHS,
                                const SimplifyQuery &Q, unsigned MaxRecurse) {
   // Recursion is always used, so bail out at once if we already hit the limit.
   if (!MaxRecurse--)
@@ -1001,7 +954,7 @@ Value *llvm::simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
 /// Given a predicate and two operands, return true if the comparison is true.
 /// This is a helper for div/rem simplification where we return some other value
 /// when we can prove a relationship between the operands.
-static bool isICmpTrue(ICmpInst::Predicate Pred, Value *LHS, Value *RHS,
+static bool isICmpTrue(CmpPredicate Pred, Value *LHS, Value *RHS,
                        const SimplifyQuery &Q, unsigned MaxRecurse) {
   Value *V = simplifyICmpInst(Pred, LHS, RHS, Q, MaxRecurse);
   Constant *C = dyn_cast_or_null<Constant>(V);
@@ -2601,7 +2554,7 @@ static Type *getCompareTy(Value *Op) {
 /// Rummage around inside V looking for something equivalent to the comparison
 /// "LHS Pred RHS". Return such a value if found, otherwise return null.
 /// Helper function for analyzing max/min idioms.
-static Value *extractEquivalentCondition(Value *V, CmpInst::Predicate Pred,
+static Value *extractEquivalentCondition(Value *V, CmpPredicate Pred,
                                          Value *LHS, Value *RHS) {
   SelectInst *SI = dyn_cast<SelectInst>(V);
   if (!SI)
@@ -2710,8 +2663,8 @@ static bool haveNonOverlappingStorage(const Value *V1, const Value *V2) {
 // If the C and C++ standards are ever made sufficiently restrictive in this
 // area, it may be possible to update LLVM's semantics accordingly and reinstate
 // this optimization.
-static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
-                                    Value *RHS, const SimplifyQuery &Q) {
+static Constant *computePointerICmp(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                    const SimplifyQuery &Q) {
   assert(LHS->getType() == RHS->getType() && "Must have same types");
   const DataLayout &DL = Q.DL;
   const TargetLibraryInfo *TLI = Q.TLI;
@@ -2859,8 +2812,8 @@ static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
 }
 
 /// Fold an icmp when its operands have i1 scalar type.
-static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
-                                  Value *RHS, const SimplifyQuery &Q) {
+static Value *simplifyICmpOfBools(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                  const SimplifyQuery &Q) {
   Type *ITy = getCompareTy(LHS); // The return type.
   Type *OpTy = LHS->getType();   // The operand type.
   if (!OpTy->isIntOrIntVectorTy(1))
@@ -2962,8 +2915,8 @@ static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
 }
 
 /// Try hard to fold icmp with zero RHS because this is a common case.
-static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
-                                   Value *RHS, const SimplifyQuery &Q) {
+static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                   const SimplifyQuery &Q) {
   if (!match(RHS, m_Zero()))
     return nullptr;
 
@@ -3022,7 +2975,7 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
-static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS,
+static Value *simplifyICmpWithConstant(CmpPredicate Pred, Value *LHS,
                                        Value *RHS, const InstrInfoQuery &IIQ) {
   Type *ITy = getCompareTy(RHS); // The return type.
 
@@ -3070,20 +3023,77 @@ static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
-static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
-                                         BinaryOperator *LBO, Value *RHS,
-                                         const SimplifyQuery &Q,
+enum class MonotonicType { GreaterEq, LowerEq };
+
+/// Get values V_i such that V uge V_i (GreaterEq) or V ule V_i (LowerEq).
+static void getUnsignedMonotonicValues(SmallPtrSetImpl<Value *> &Res, Value *V,
+                                       MonotonicType Type, unsigned Depth = 0) {
+  if (!Res.insert(V).second)
+    return;
+
+  // Can be increased if useful.
+  if (++Depth > 1)
+    return;
+
+  auto *I = dyn_cast<Instruction>(V);
+  if (!I)
+    return;
+
+  Value *X, *Y;
+  if (Type == MonotonicType::GreaterEq) {
+    if (match(I, m_Or(m_Value(X), m_Value(Y))) ||
+        match(I, m_Intrinsic<Intrinsic::uadd_sat>(m_Value(X), m_Value(Y)))) {
+      getUnsignedMonotonicValues(Res, X, Type, Depth);
+      getUnsignedMonotonicValues(Res, Y, Type, Depth);
+    }
+  } else {
+    assert(Type == MonotonicType::LowerEq);
+    switch (I->getOpcode()) {
+    case Instruction::And:
+      getUnsignedMonotonicValues(Res, I->getOperand(0), Type, Depth);
+      getUnsignedMonotonicValues(Res, I->getOperand(1), Type, Depth);
+      break;
+    case Instruction::URem:
+    case Instruction::UDiv:
+    case Instruction::LShr:
+      getUnsignedMonotonicValues(Res, I->getOperand(0), Type, Depth);
+      break;
+    case Instruction::Call:
+      if (match(I, m_Intrinsic<Intrinsic::usub_sat>(m_Value(X))))
+        getUnsignedMonotonicValues(Res, X, Type, Depth);
+      break;
+    default:
+      break;
+    }
+  }
+}
+
+static Value *simplifyICmpUsingMonotonicValues(CmpPredicate Pred, Value *LHS,
+                                               Value *RHS) {
+  if (Pred != ICmpInst::ICMP_UGE && Pred != ICmpInst::ICMP_ULT)
+    return nullptr;
+
+  // We have LHS uge GreaterValues and LowerValues uge RHS. If any of the
+  // GreaterValues and LowerValues are the same, it follows that LHS uge RHS.
+  SmallPtrSet<Value *, 4> GreaterValues;
+  SmallPtrSet<Value *, 4> LowerValues;
+  getUnsignedMonotonicValues(GreaterValues, LHS, MonotonicType::GreaterEq);
+  getUnsignedMonotonicValues(LowerValues, RHS, MonotonicType::LowerEq);
+  for (Value *GV : GreaterValues)
+    if (LowerValues.contains(GV))
+      return ConstantInt::getBool(getCompareTy(LHS),
+                                  Pred == ICmpInst::ICMP_UGE);
+  return nullptr;
+}
+
+static Value *simplifyICmpWithBinOpOnLHS(CmpPredicate Pred, BinaryOperator *LBO,
+                                         Value *RHS, const SimplifyQuery &Q,
                                          unsigned MaxRecurse) {
   Type *ITy = getCompareTy(RHS); // The return type.
 
   Value *Y = nullptr;
   // icmp pred (or X, Y), X
   if (match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) {
-    if (Pred == ICmpInst::ICMP_ULT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_UGE)
-      return getTrue(ITy);
-
     if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
       KnownBits RHSKnown = computeKnownBits(RHS, /* Depth */ 0, Q);
       KnownBits YKnown = computeKnownBits(Y, /* Depth */ 0, Q);
@@ -3094,14 +3104,6 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
     }
   }
 
-  // icmp pred (and X, Y), X
-  if (match(LBO, m_c_And(m_Value(), m_Specific(RHS)))) {
-    if (Pred == ICmpInst::ICMP_UGT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_ULE)
-      return getTrue(ITy);
-  }
-
   // icmp pred (urem X, Y), Y
   if (match(LBO, m_URem(m_Value(), m_Specific(RHS)))) {
     switch (Pred) {
@@ -3132,27 +3134,6 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
     }
   }
 
-  // icmp pred (urem X, Y), X
-  if (match(LBO, m_URem(m_Specific(RHS), m_Value()))) {
-    if (Pred == ICmpInst::ICMP_ULE)
-      return getTrue(ITy);
-    if (Pred == ICmpInst::ICMP_UGT)
-      return getFalse(ITy);
-  }
-
-  // x >>u y <=u x --> true.
-  // x >>u y >u  x --> false.
-  // x udiv y <=u x --> true.
-  // x udiv y >u  x --> false.
-  if (match(LBO, m_LShr(m_Specific(RHS), m_Value())) ||
-      match(LBO, m_UDiv(m_Specific(RHS), m_Value()))) {
-    // icmp pred (X op Y), X
-    if (Pred == ICmpInst::ICMP_UGT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_ULE)
-      return getTrue(ITy);
-  }
-
   // If x is nonzero:
   // x >>u C <u  x --> true  for C != 0.
   // x >>u C !=  x --> true  for C != 0.
@@ -3172,14 +3153,12 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
         break;
       case ICmpInst::ICMP_EQ:
       case ICmpInst::ICMP_UGE:
+      case ICmpInst::ICMP_UGT:
         return getFalse(ITy);
       case ICmpInst::ICMP_NE:
       case ICmpInst::ICMP_ULT:
-        return getTrue(ITy);
-      case ICmpInst::ICMP_UGT:
       case ICmpInst::ICMP_ULE:
-        // UGT/ULE are handled by the more general case just above
-        llvm_unreachable("Unexpected UGT/ULE, should have been handled");
+        return getTrue(ITy);
       }
     }
   }
@@ -3227,8 +3206,8 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
 // *) C1 < C2 && C1 >= 0, or
 // *) C2 < C1 && C1 <= 0.
 //
-static bool trySimplifyICmpWithAdds(CmpInst::Predicate Pred, Value *LHS,
-                                    Value *RHS, const InstrInfoQuery &IIQ) {
+static bool trySimplifyICmpWithAdds(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                    const InstrInfoQuery &IIQ) {
   // TODO: only support icmp slt for now.
   if (Pred != CmpInst::ICMP_SLT || !IIQ.UseInstrInfo)
     return false;
@@ -3252,8 +3231,8 @@ static bool trySimplifyICmpWithAdds(CmpInst::Predicate Pred, Value *LHS,
 /// TODO: A large part of this logic is duplicated in InstCombine's
 /// foldICmpBinOp(). We should be able to share that and avoid the code
 /// duplication.
-static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
-                                    Value *RHS, const SimplifyQuery &Q,
+static Value *simplifyICmpWithBinOp(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                    const SimplifyQuery &Q,
                                     unsigned MaxRecurse) {
   BinaryOperator *LBO = dyn_cast<BinaryOperator>(LHS);
   BinaryOperator *RBO = dyn_cast<BinaryOperator>(RHS);
@@ -3486,8 +3465,8 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
 
 /// simplify integer comparisons where at least one operand of the compare
 /// matches an integer min/max idiom.
-static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS,
-                                     Value *RHS, const SimplifyQuery &Q,
+static Value *simplifyICmpWithMinMax(CmpPredicate Pred, Value *LHS, Value *RHS,
+                                     const SimplifyQuery &Q,
                                      unsigned MaxRecurse) {
   Type *ITy = getCompareTy(LHS); // The return type.
   Value *A, *B;
@@ -3671,7 +3650,7 @@ static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
-static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate,
+static Value *simplifyICmpWithDominatingAssume(CmpPredicate Predicate,
                                                Value *LHS, Value *RHS,
                                                const SimplifyQuery &Q) {
   // Gracefully handle instructions that have not been inserted yet.
@@ -3694,21 +3673,14 @@ static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate,
   return nullptr;
 }
 
-static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
-                                             Value *LHS, Value *RHS) {
+static Value *simplifyICmpWithIntrinsicOnLHS(CmpPredicate Pred, Value *LHS,
+                                             Value *RHS) {
   auto *II = dyn_cast<IntrinsicInst>(LHS);
   if (!II)
     return nullptr;
 
   switch (II->getIntrinsicID()) {
   case Intrinsic::uadd_sat:
-    // uadd.sat(X, Y) uge X, uadd.sat(X, Y) uge Y
-    if (II->getArgOperand(0) == RHS || II->getArgOperand(1) == RHS) {
-      if (Pred == ICmpInst::ICMP_UGE)
-        return ConstantInt::getTrue(getCompareTy(II));
-      if (Pred == ICmpInst::ICMP_ULT)
-        return ConstantInt::getFalse(getCompareTy(II));
-    }
     // uadd.sat(X, Y) uge X + Y
     if (match(RHS, m_c_Add(m_Specific(II->getArgOperand(0)),
                            m_Specific(II->getArgOperand(1))))) {
@@ -3719,13 +3691,6 @@ static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
     }
     return nullptr;
   case Intrinsic::usub_sat:
-    // usub.sat(X, Y) ule X
-    if (II->getArgOperand(0) == RHS) {
-      if (Pred == ICmpInst::ICMP_ULE)
-        return ConstantInt::getTrue(getCompareTy(II));
-      if (Pred == ICmpInst::ICMP_UGT)
-        return ConstantInt::getFalse(getCompareTy(II));
-    }
     // usub.sat(X, Y) ule X - Y
     if (match(RHS, m_Sub(m_Specific(II->getArgOperand(0)),
                          m_Specific(II->getArgOperand(1))))) {
@@ -3757,9 +3722,8 @@ static std::optional<ConstantRange> getRange(Value *V,
 
 /// Given operands for an ICmpInst, see if we can fold the result.
 /// If not, this returns null.
-static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+static Value *simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
                                const SimplifyQuery &Q, unsigned MaxRecurse) {
-  CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate;
   assert(CmpInst::isIntPredicate(Pred) && "Not an integer compare!");
 
   if (Constant *CLHS = dyn_cast<Constant>(LHS)) {
@@ -4030,6 +3994,12 @@ static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
           ICmpInst::getSwappedPredicate(Pred), RHS, LHS))
     return V;
 
+  if (Value *V = simplifyICmpUsingMonotonicValues(Pred, LHS, RHS))
+    return V;
+  if (Value *V = simplifyICmpUsingMonotonicValues(
+          ICmpInst::getSwappedPredicate(Pred), RHS, LHS))
+    return V;
+
   if (Value *V = simplifyICmpWithDominatingAssume(Pred, LHS, RHS, Q))
     return V;
 
@@ -4066,17 +4036,16 @@ static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   return nullptr;
 }
 
-Value *llvm::simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *llvm::simplifyICmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                               const SimplifyQuery &Q) {
   return ::simplifyICmpInst(Predicate, LHS, RHS, Q, RecursionLimit);
 }
 
 /// Given operands for an FCmpInst, see if we can fold the result.
 /// If not, this returns null.
-static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+static Value *simplifyFCmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
                                FastMathFlags FMF, const SimplifyQuery &Q,
                                unsigned MaxRecurse) {
-  CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate;
   assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!");
 
   if (Constant *CLHS = dyn_cast<Constant>(LHS)) {
@@ -4301,7 +4270,7 @@ static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   return nullptr;
 }
 
-Value *llvm::simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *llvm::simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                               FastMathFlags FMF, const SimplifyQuery &Q) {
   return ::simplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, RecursionLimit);
 }
@@ -4538,7 +4507,7 @@ static Value *simplifySelectBitTest(Value *TrueVal, Value *FalseVal, Value *X,
 }
 
 static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS,
-                                     ICmpInst::Predicate Pred, Value *TVal,
+                                     CmpPredicate Pred, Value *TVal,
                                      Value *FVal) {
   // Canonicalize common cmp+sel operand as CmpLHS.
   if (CmpRHS == TVal || CmpRHS == FVal) {
@@ -4612,8 +4581,8 @@ static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS,
 /// An alternative way to test if a bit is set or not uses sgt/slt instead of
 /// eq/ne.
 static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *CmpRHS,
-                                           ICmpInst::Predicate Pred,
-                                           Value *TrueVal, Value *FalseVal) {
+                                           CmpPredicate Pred, Value *TrueVal,
+                                           Value *FalseVal) {
   if (auto Res = decomposeBitTestICmp(CmpLHS, CmpRHS, Pred))
     return simplifySelectBitTest(TrueVal, FalseVal, Res->X, &Res->Mask,
                                  Res->Pred == ICmpInst::ICMP_EQ);
@@ -4983,9 +4952,6 @@ static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
   if (Value *V = simplifySelectWithFCmp(Cond, TrueVal, FalseVal, Q, MaxRecurse))
     return V;
 
-  if (Value *V = foldSelectWithBinaryOp(Cond, TrueVal, FalseVal))
-    return V;
-
   std::optional<bool> Imp = isImpliedByDomCondition(Cond, Q.CxtI, Q.DL);
   if (Imp)
     return *Imp ? TrueVal : FalseVal;
@@ -6123,14 +6089,14 @@ Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
 }
 
 /// Given operands for a CmpInst, see if we can fold the result.
-static Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+static Value *simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                               const SimplifyQuery &Q, unsigned MaxRecurse) {
-  if (CmpInst::isIntPredicate((CmpInst::Predicate)Predicate))
+  if (CmpInst::isIntPredicate(Predicate))
     return simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse);
   return simplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse);
 }
 
-Value *llvm::simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
+Value *llvm::simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,
                              const SimplifyQuery &Q) {
   return ::simplifyCmpInst(Predicate, LHS, RHS, Q, RecursionLimit);
 }
@@ -7168,7 +7134,7 @@ static Value *simplifyInstructionWithOperands(Instruction *I,
   case Instruction::Xor:
     return simplifyXorInst(NewOps[0], NewOps[1], Q, MaxRecurse);
   case Instruction::ICmp:
-    return simplifyICmpInst(cast<ICmpInst>(I)->getPredicate(), NewOps[0],
+    return simplifyICmpInst(cast<ICmpInst>(I)->getCmpPredicate(), NewOps[0],
                             NewOps[1], Q, MaxRecurse);
   case Instruction::FCmp:
     return simplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), NewOps[0],
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 42b04046ce10d..349a0a1a2d3c4 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -924,18 +924,65 @@ LazyValueInfoImpl::solveBlockValueBinaryOpImpl(
     Instruction *I, BasicBlock *BB,
     std::function<ConstantRange(const ConstantRange &, const ConstantRange &)>
         OpFn) {
+  Value *LHS = I->getOperand(0);
+  Value *RHS = I->getOperand(1);
+
+  auto ThreadBinOpOverSelect =
+      [&](Value *X, const ConstantRange &CRX, SelectInst *Y,
+          bool XIsLHS) -> std::optional<ValueLatticeElement> {
+    Value *Cond = Y->getCondition();
+    // Only handle selects with constant values.
+    Constant *TrueC = dyn_cast<Constant>(Y->getTrueValue());
+    if (!TrueC)
+      return std::nullopt;
+    Constant *FalseC = dyn_cast<Constant>(Y->getFalseValue());
+    if (!FalseC)
+      return std::nullopt;
+    if (!isGuaranteedNotToBeUndef(Cond, AC))
+      return std::nullopt;
+
+    ConstantRange TrueX =
+        CRX.intersectWith(getValueFromCondition(X, Cond, /*CondIsTrue=*/true,
+                                                /*UseBlockValue=*/false)
+                              ->asConstantRange(X->getType()));
+    ConstantRange FalseX =
+        CRX.intersectWith(getValueFromCondition(X, Cond, /*CondIsTrue=*/false,
+                                                /*UseBlockValue=*/false)
+                              ->asConstantRange(X->getType()));
+    ConstantRange TrueY = TrueC->toConstantRange();
+    ConstantRange FalseY = FalseC->toConstantRange();
+
+    if (XIsLHS)
+      return ValueLatticeElement::getRange(
+          OpFn(TrueX, TrueY).unionWith(OpFn(FalseX, FalseY)));
+    return ValueLatticeElement::getRange(
+        OpFn(TrueY, TrueX).unionWith(OpFn(FalseY, FalseX)));
+  };
+
   // Figure out the ranges of the operands.  If that fails, use a
   // conservative range, but apply the transfer rule anyways.  This
   // lets us pick up facts from expressions like "and i32 (call i32
   // @foo()), 32"
-  std::optional<ConstantRange> LHSRes = getRangeFor(I->getOperand(0), I, BB);
+  std::optional<ConstantRange> LHSRes = getRangeFor(LHS, I, BB);
   if (!LHSRes)
     return std::nullopt;
 
-  std::optional<ConstantRange> RHSRes = getRangeFor(I->getOperand(1), I, BB);
+  // Try to thread binop over rhs select
+  if (auto *SI = dyn_cast<SelectInst>(RHS)) {
+    if (auto Res = ThreadBinOpOverSelect(LHS, *LHSRes, SI, /*XIsLHS=*/true))
+      return *Res;
+  }
+
+  std::optional<ConstantRange> RHSRes = getRangeFor(RHS, I, BB);
   if (!RHSRes)
     return std::nullopt;
 
+  // Try to thread binop over lhs select
+  if (auto *SI = dyn_cast<SelectInst>(LHS)) {
+    if (auto Res = ThreadBinOpOverSelect(RHS, *RHSRes, SI, /*XIsLHS=*/false))
+      return *Res;
+  }
+
   const ConstantRange &LHSRange = *LHSRes;
   const ConstantRange &RHSRange = *RHSRes;
   return ValueLatticeElement::getRange(OpFn(LHSRange, RHSRange));
@@ -1005,6 +1052,13 @@ LazyValueInfoImpl::solveBlockValueInsertElement(InsertElementInst *IEI,
   if (!OptVecVal)
     return std::nullopt;
 
+  // Bail out if the inserted element is a constant expression. Unlike other
+  // ValueLattice types, these are not considered an implicit splat when a
+  // vector type is used.
+  // We could call ConstantFoldInsertElementInstruction here to handle these.
+  if (OptEltVal->isConstant())
+    return ValueLatticeElement::getOverdefined();
+
   Res.mergeIn(*OptVecVal);
   return Res;
 }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 907bb7875dc80..2c75d5625cb66 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1566,10 +1566,10 @@ std::optional<int> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
   unsigned IdxWidth = DL.getIndexSizeInBits(ASA);
 
   APInt OffsetA(IdxWidth, 0), OffsetB(IdxWidth, 0);
-  const Value *PtrA1 =
-      PtrA->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetA);
-  const Value *PtrB1 =
-      PtrB->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetB);
+  const Value *PtrA1 = PtrA->stripAndAccumulateConstantOffsets(
+      DL, OffsetA, /*AllowNonInbounds=*/true);
+  const Value *PtrB1 = PtrB->stripAndAccumulateConstantOffsets(
+      DL, OffsetB, /*AllowNonInbounds=*/true);
 
   int Val;
   if (PtrA1 == PtrB1) {
@@ -1978,13 +1978,12 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
                     << " Sink induction step: " << StrideBPtrInt << "\n");
   // At least Src or Sink are loop invariant and the other is strided or
   // invariant. We can generate a runtime check to disambiguate the accesses.
-  if (StrideAPtrInt == 0 || StrideBPtrInt == 0)
+  if (!StrideAPtrInt || !StrideBPtrInt)
     return MemoryDepChecker::Dependence::Unknown;
 
   // Both Src and Sink have a constant stride, check if they are in the same
   // direction.
-  if ((StrideAPtrInt > 0 && StrideBPtrInt < 0) ||
-      (StrideAPtrInt < 0 && StrideBPtrInt > 0)) {
+  if ((StrideAPtrInt > 0) != (StrideBPtrInt > 0)) {
     LLVM_DEBUG(
         dbgs() << "Pointer access with strides in different directions\n");
     return MemoryDepChecker::Dependence::Unknown;
@@ -2040,19 +2039,16 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
                          *Dist, MaxStride, TypeByteSize))
     return Dependence::NoDep;
 
-  const SCEVConstant *C = dyn_cast<SCEVConstant>(Dist);
+  const SCEVConstant *ConstDist = dyn_cast<SCEVConstant>(Dist);
 
   // Attempt to prove strided accesses independent.
-  if (C) {
-    const APInt &Val = C->getAPInt();
-    int64_t Distance = Val.getSExtValue();
+  if (ConstDist) {
+    uint64_t Distance = ConstDist->getAPInt().abs().getZExtValue();
 
     // If the distance between accesses and their strides are known constants,
     // check whether the accesses interlace each other.
-    if (std::abs(Distance) > 0 && CommonStride && *CommonStride > 1 &&
-        HasSameSize &&
-        areStridedAccessesIndependent(std::abs(Distance), *CommonStride,
-                                      TypeByteSize)) {
+    if (Distance > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
+        areStridedAccessesIndependent(Distance, *CommonStride, TypeByteSize)) {
       LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
       return Dependence::NoDep;
     }
@@ -2085,7 +2081,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     // forward dependency will allow vectorization using any width.
 
     if (IsTrueDataDependence && EnableForwardingConflictDetection) {
-      if (!C) {
+      if (!ConstDist) {
         // TODO: FoundNonConstantDistanceDependence is used as a necessary
         // condition to consider retrying with runtime checks. Historically, we
         // did not set it when strides were different but there is no inherent
@@ -2094,8 +2090,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
         return Dependence::Unknown;
       }
       if (!HasSameSize ||
-          couldPreventStoreLoadForward(C->getAPInt().abs().getZExtValue(),
-                                       TypeByteSize)) {
+          couldPreventStoreLoadForward(
+              ConstDist->getAPInt().abs().getZExtValue(), TypeByteSize)) {
         LLVM_DEBUG(
             dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
         return Dependence::ForwardButPreventsForwarding;
@@ -2113,7 +2109,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     return Dependence::Unknown;
   }
 
-  if (!isa<SCEVConstant>(Dist)) {
+  if (!ConstDist) {
     // Previously this case would be treated as Unknown, possibly setting
     // FoundNonConstantDistanceDependence to force re-trying with runtime
     // checks. Until the TODO below is addressed, set it here to preserve
@@ -2175,7 +2171,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   uint64_t MinDistanceNeeded =
       TypeByteSize * *CommonStride * (MinNumIter - 1) + TypeByteSize;
   if (MinDistanceNeeded > static_cast<uint64_t>(MinDistance)) {
-    if (!isa<SCEVConstant>(Dist)) {
+    if (!ConstDist) {
       // For non-constant distances, we checked the lower bound of the
       // dependence distance and the distance may be larger at runtime (and safe
       // for vectorization). Classify it as Unknown, so we re-try with runtime
@@ -2216,8 +2212,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
 
   bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
   uint64_t MinDepDistBytesOld = MinDepDistBytes;
-  if (IsTrueDataDependence && EnableForwardingConflictDetection &&
-      isa<SCEVConstant>(Dist) &&
+  if (IsTrueDataDependence && EnableForwardingConflictDetection && ConstDist &&
       couldPreventStoreLoadForward(MinDistance, TypeByteSize)) {
     // Sanity check that we didn't update MinDepDistBytes when calling
     // couldPreventStoreLoadForward
@@ -2235,7 +2230,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
                     << " with max VF = " << MaxVF << '\n');
 
   uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
-  if (!isa<SCEVConstant>(Dist) && MaxVFInBits < MaxTargetVectorWidthInBits) {
+  if (!ConstDist && MaxVFInBits < MaxTargetVectorWidthInBits) {
     // For non-constant distances, we checked the lower bound of the dependence
     // distance and the distance may be larger at runtime (and safe for
     // vectorization). Classify it as Unknown, so we re-try with runtime checks.
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 4028d5f4e2e1b..57b97999b0886 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -670,6 +670,65 @@ STATISTIC(ObjectVisitorArgument,
 STATISTIC(ObjectVisitorLoad,
           "Number of load instructions with unsolved size and offset");
 
+static std::optional<APInt>
+combinePossibleConstantValues(std::optional<APInt> LHS,
+                              std::optional<APInt> RHS,
+                              ObjectSizeOpts::Mode EvalMode) {
+  if (!LHS || !RHS)
+    return std::nullopt;
+  if (EvalMode == ObjectSizeOpts::Mode::Max)
+    return LHS->sge(*RHS) ? *LHS : *RHS;
+  else
+    return LHS->sle(*RHS) ? *LHS : *RHS;
+}
+
+static std::optional<APInt> aggregatePossibleConstantValuesImpl(
+    const Value *V, ObjectSizeOpts::Mode EvalMode, unsigned recursionDepth) {
+  constexpr unsigned maxRecursionDepth = 4;
+  if (recursionDepth == maxRecursionDepth)
+    return std::nullopt;
+
+  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
+    return CI->getValue();
+  } else if (const auto *SI = dyn_cast<SelectInst>(V)) {
+    return combinePossibleConstantValues(
+        aggregatePossibleConstantValuesImpl(SI->getTrueValue(), EvalMode,
+                                            recursionDepth + 1),
+        aggregatePossibleConstantValuesImpl(SI->getFalseValue(), EvalMode,
+                                            recursionDepth + 1),
+        EvalMode);
+  } else if (const auto *PN = dyn_cast<PHINode>(V)) {
+    unsigned Count = PN->getNumIncomingValues();
+    if (Count == 0)
+      return std::nullopt;
+    auto Acc = aggregatePossibleConstantValuesImpl(
+        PN->getIncomingValue(0), EvalMode, recursionDepth + 1);
+    for (unsigned I = 1; Acc && I < Count; ++I) {
+      auto Tmp = aggregatePossibleConstantValuesImpl(
+          PN->getIncomingValue(I), EvalMode, recursionDepth + 1);
+      Acc = combinePossibleConstantValues(Acc, Tmp, EvalMode);
+    }
+    return Acc;
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<APInt>
+aggregatePossibleConstantValues(const Value *V, ObjectSizeOpts::Mode EvalMode) {
+  if (auto *CI = dyn_cast<ConstantInt>(V))
+    return CI->getValue();
+
+  if (EvalMode != ObjectSizeOpts::Mode::Min &&
+      EvalMode != ObjectSizeOpts::Mode::Max)
+    return std::nullopt;
+
+  // Not using computeConstantRange here because we cannot guarantee it's not
+  // doing optimization based on UB which we want to avoid when expanding
+  // __builtin_object_size.
+  return aggregatePossibleConstantValuesImpl(V, EvalMode, 0u);
+}
+
 /// Align \p Size according to \p Alignment. If \p Size is greater than
 /// getSignedMaxValue(), set it as unknown as we can only represent signed value
 /// in OffsetSpan.
@@ -717,11 +776,36 @@ OffsetSpan ObjectSizeOffsetVisitor::computeImpl(Value *V) {
   V = V->stripAndAccumulateConstantOffsets(
       DL, Offset, /* AllowNonInbounds */ true, /* AllowInvariantGroup */ true);
 
+  // Give it another try with approximated analysis. We don't start with this
+  // one because stripAndAccumulateConstantOffsets behaves differently wrt.
+  // overflows if we provide an external Analysis.
+  if ((Options.EvalMode == ObjectSizeOpts::Mode::Min ||
+       Options.EvalMode == ObjectSizeOpts::Mode::Max) &&
+      isa<GEPOperator>(V)) {
+    // External Analysis used to compute the Min/Max value of individual Offsets
+    // within a GEP.
+    ObjectSizeOpts::Mode EvalMode =
+        Options.EvalMode == ObjectSizeOpts::Mode::Min
+            ? ObjectSizeOpts::Mode::Max
+            : ObjectSizeOpts::Mode::Min;
+    auto OffsetRangeAnalysis = [EvalMode](Value &VOffset, APInt &Offset) {
+      if (auto PossibleOffset =
+              aggregatePossibleConstantValues(&VOffset, EvalMode)) {
+        Offset = *PossibleOffset;
+        return true;
+      }
+      return false;
+    };
+
+    V = V->stripAndAccumulateConstantOffsets(
+        DL, Offset, /* AllowNonInbounds */ true, /* AllowInvariantGroup */ true,
+        /*ExternalAnalysis=*/OffsetRangeAnalysis);
+  }
+
   // Later we use the index type size and zero but it will match the type of the
   // value that is passed to computeImpl.
   IntTyBits = DL.getIndexTypeSizeInBits(V->getType());
   Zero = APInt::getZero(IntTyBits);
-
   OffsetSpan ORT = computeValue(V);
 
   bool IndexTypeSizeChanged = InitialIntTyBits != IntTyBits;
@@ -813,8 +897,9 @@ OffsetSpan ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) {
     return OffsetSpan(Zero, align(Size, I.getAlign()));
 
   Value *ArraySize = I.getArraySize();
-  if (const ConstantInt *C = dyn_cast<ConstantInt>(ArraySize)) {
-    APInt NumElems = C->getValue();
+  if (auto PossibleSize =
+          aggregatePossibleConstantValues(ArraySize, Options.EvalMode)) {
+    APInt NumElems = *PossibleSize;
     if (!CheckedZextOrTrunc(NumElems))
       return ObjectSizeOffsetVisitor::unknown();
 
@@ -840,7 +925,18 @@ OffsetSpan ObjectSizeOffsetVisitor::visitArgument(Argument &A) {
 }
 
 OffsetSpan ObjectSizeOffsetVisitor::visitCallBase(CallBase &CB) {
-  if (std::optional<APInt> Size = getAllocSize(&CB, TLI)) {
+  auto Mapper = [this](const Value *V) -> const Value * {
+    if (!V->getType()->isIntegerTy())
+      return V;
+
+    if (auto PossibleBound =
+            aggregatePossibleConstantValues(V, Options.EvalMode))
+      return ConstantInt::get(V->getType(), *PossibleBound);
+
+    return V;
+  };
+
+  if (std::optional<APInt> Size = getAllocSize(&CB, TLI, Mapper)) {
     // Very large unsigned value cannot be represented as OffsetSpan.
     if (Size->isNegative())
       return ObjectSizeOffsetVisitor::unknown();
diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp
index f672bd0e1e133..050b827388d3c 100644
--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp
+++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -565,24 +565,26 @@ static MemoryAccess *onlySingleValue(MemoryPhi *MP) {
   return MA;
 }
 
-static MemoryAccess *getNewDefiningAccessForClone(MemoryAccess *MA,
-                                                  const ValueToValueMapTy &VMap,
-                                                  PhiToDefMap &MPhiMap,
-                                                  MemorySSA *MSSA) {
+static MemoryAccess *getNewDefiningAccessForClone(
+    MemoryAccess *MA, const ValueToValueMapTy &VMap, PhiToDefMap &MPhiMap,
+    MemorySSA *MSSA, function_ref<bool(BasicBlock *BB)> IsInClonedRegion) {
   MemoryAccess *InsnDefining = MA;
   if (MemoryDef *DefMUD = dyn_cast<MemoryDef>(InsnDefining)) {
-    if (!MSSA->isLiveOnEntryDef(DefMUD)) {
-      Instruction *DefMUDI = DefMUD->getMemoryInst();
-      assert(DefMUDI && "Found MemoryUseOrDef with no Instruction.");
-      if (Instruction *NewDefMUDI =
-              cast_or_null<Instruction>(VMap.lookup(DefMUDI))) {
-        InsnDefining = MSSA->getMemoryAccess(NewDefMUDI);
-        if (!InsnDefining || isa<MemoryUse>(InsnDefining)) {
-          // The clone was simplified, it's no longer a MemoryDef, look up.
-          InsnDefining = getNewDefiningAccessForClone(
-              DefMUD->getDefiningAccess(), VMap, MPhiMap, MSSA);
-        }
-      }
+    if (MSSA->isLiveOnEntryDef(DefMUD))
+      return DefMUD;
+
+    // If the MemoryDef is not part of the cloned region, leave it alone.
+    Instruction *DefMUDI = DefMUD->getMemoryInst();
+    assert(DefMUDI && "Found MemoryUseOrDef with no Instruction.");
+    if (!IsInClonedRegion(DefMUDI->getParent()))
+      return DefMUD;
+
+    auto *NewDefMUDI = cast_or_null<Instruction>(VMap.lookup(DefMUDI));
+    InsnDefining = NewDefMUDI ? MSSA->getMemoryAccess(NewDefMUDI) : nullptr;
+    if (!InsnDefining || isa<MemoryUse>(InsnDefining)) {
+      // The clone was simplified, it's no longer a MemoryDef, look up.
+      InsnDefining = getNewDefiningAccessForClone(
+          DefMUD->getDefiningAccess(), VMap, MPhiMap, MSSA, IsInClonedRegion);
     }
   } else {
     MemoryPhi *DefPhi = cast<MemoryPhi>(InsnDefining);
@@ -593,10 +595,10 @@ static MemoryAccess *getNewDefiningAccessForClone(MemoryAccess *MA,
   return InsnDefining;
 }
 
-void MemorySSAUpdater::cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB,
-                                        const ValueToValueMapTy &VMap,
-                                        PhiToDefMap &MPhiMap,
-                                        bool CloneWasSimplified) {
+void MemorySSAUpdater::cloneUsesAndDefs(
+    BasicBlock *BB, BasicBlock *NewBB, const ValueToValueMapTy &VMap,
+    PhiToDefMap &MPhiMap, function_ref<bool(BasicBlock *)> IsInClonedRegion,
+    bool CloneWasSimplified) {
   const MemorySSA::AccessList *Acc = MSSA->getBlockAccesses(BB);
   if (!Acc)
     return;
@@ -615,7 +617,7 @@ void MemorySSAUpdater::cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB,
         MemoryAccess *NewUseOrDef = MSSA->createDefinedAccess(
             NewInsn,
             getNewDefiningAccessForClone(MUD->getDefiningAccess(), VMap,
-                                         MPhiMap, MSSA),
+                                         MPhiMap, MSSA, IsInClonedRegion),
             /*Template=*/CloneWasSimplified ? nullptr : MUD,
             /*CreationMustSucceed=*/false);
         if (NewUseOrDef)
@@ -668,8 +670,13 @@ void MemorySSAUpdater::updateForClonedLoop(const LoopBlocksRPO &LoopBlocks,
                                            ArrayRef<BasicBlock *> ExitBlocks,
                                            const ValueToValueMapTy &VMap,
                                            bool IgnoreIncomingWithNoClones) {
-  PhiToDefMap MPhiMap;
+  SmallSetVector<BasicBlock *, 16> Blocks;
+  for (BasicBlock *BB : concat<BasicBlock *const>(LoopBlocks, ExitBlocks))
+    Blocks.insert(BB);
 
+  auto IsInClonedRegion = [&](BasicBlock *BB) { return Blocks.contains(BB); };
+
+  PhiToDefMap MPhiMap;
   auto FixPhiIncomingValues = [&](MemoryPhi *Phi, MemoryPhi *NewPhi) {
     assert(Phi && NewPhi && "Invalid Phi nodes.");
     BasicBlock *NewPhiBB = NewPhi->getBlock();
@@ -692,9 +699,10 @@ void MemorySSAUpdater::updateForClonedLoop(const LoopBlocksRPO &LoopBlocks,
         continue;
 
       // Determine incoming value and add it as incoming from IncBB.
-      NewPhi->addIncoming(
-          getNewDefiningAccessForClone(IncomingAccess, VMap, MPhiMap, MSSA),
-          IncBB);
+      NewPhi->addIncoming(getNewDefiningAccessForClone(IncomingAccess, VMap,
+                                                       MPhiMap, MSSA,
+                                                       IsInClonedRegion),
+                          IncBB);
     }
     if (auto *SingleAccess = onlySingleValue(NewPhi)) {
       MPhiMap[Phi] = SingleAccess;
@@ -716,13 +724,13 @@ void MemorySSAUpdater::updateForClonedLoop(const LoopBlocksRPO &LoopBlocks,
       MPhiMap[MPhi] = NewPhi;
     }
     // Update Uses and Defs.
-    cloneUsesAndDefs(BB, NewBlock, VMap, MPhiMap);
+    cloneUsesAndDefs(BB, NewBlock, VMap, MPhiMap, IsInClonedRegion);
   };
 
-  for (auto *BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks))
+  for (auto *BB : Blocks)
     ProcessBlock(BB);
 
-  for (auto *BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks))
+  for (auto *BB : Blocks)
     if (MemoryPhi *MPhi = MSSA->getMemoryAccess(BB))
       if (MemoryAccess *NewPhi = MPhiMap.lookup(MPhi))
         FixPhiIncomingValues(MPhi, cast<MemoryPhi>(NewPhi));
@@ -741,7 +749,9 @@ void MemorySSAUpdater::updateForClonedBlockIntoPred(
   PhiToDefMap MPhiMap;
   if (MemoryPhi *MPhi = MSSA->getMemoryAccess(BB))
     MPhiMap[MPhi] = MPhi->getIncomingValueForBlock(P1);
-  cloneUsesAndDefs(BB, P1, VM, MPhiMap, /*CloneWasSimplified=*/true);
+  cloneUsesAndDefs(
+      BB, P1, VM, MPhiMap, [&](BasicBlock *CheckBB) { return BB == CheckBB; },
+      /*CloneWasSimplified=*/true);
 }
 
 template <typename Iter>
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index c3f296b9ff334..cad10486cbf3f 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4307,15 +4307,7 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
   }
 
   for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
-    bool MayBeUB = SCEVExprContains(Ops[i], [this](const SCEV *S) {
-      auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
-      // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
-      // is a non-zero constant, we have to assume the UDiv may be UB.
-      return UDiv && (!isKnownNonZero(UDiv->getOperand(1)) ||
-                      !isGuaranteedNotToBePoison(UDiv->getOperand(1)));
-    });
-
-    if (MayBeUB)
+    if (!isGuaranteedNotToCauseUB(Ops[i]))
       continue;
     // We can replace %x umin_seq %y with %x umin %y if either:
     //  * %y being poison implies %x is also poison.
@@ -5933,18 +5925,22 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
     // We can generalize this saying that i is the shifted value of BEValue
     // by one iteration:
     //   PHI(f(0), f({1,+,1})) --> f({0,+,1})
-    const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
-    const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
-    if (Shifted != getCouldNotCompute() &&
-        Start != getCouldNotCompute()) {
-      const SCEV *StartVal = getSCEV(StartValueV);
-      if (Start == StartVal) {
-        // Okay, for the entire analysis of this edge we assumed the PHI
-        // to be symbolic.  We now need to go back and purge all of the
-        // entries for the scalars that use the symbolic expression.
-        forgetMemoizedResults(SymbolicName);
-        insertValueToMap(PN, Shifted);
-        return Shifted;
+
+    // Do not allow refinement in rewriting of BEValue.
+    if (isGuaranteedNotToCauseUB(BEValue)) {
+      const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
+      const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
+      if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
+          ::impliesPoison(BEValue, Start)) {
+        const SCEV *StartVal = getSCEV(StartValueV);
+        if (Start == StartVal) {
+          // Okay, for the entire analysis of this edge we assumed the PHI
+          // to be symbolic.  We now need to go back and purge all of the
+          // entries for the scalars that use the symbolic expression.
+          forgetMemoizedResults(SymbolicName);
+          insertValueToMap(PN, Shifted);
+          return Shifted;
+        }
       }
     }
   }
@@ -6023,6 +6019,42 @@ const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
   return nullptr;
 }
 
+/// Returns SCEV for the first operand of a phi if all phi operands have
+/// identical opcodes and operands
+/// eg.
+/// a: %add = %a + %b
+///    br %c
+/// b: %add1 = %a + %b
+///    br %c
+/// c: %phi = phi [%add, a], [%add1, b]
+/// scev(%phi) => scev(%add)
+const SCEV *
+ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
+  BinaryOperator *CommonInst = nullptr;
+  // Check if instructions are identical.
+  for (Value *Incoming : PN->incoming_values()) {
+    auto *IncomingInst = dyn_cast<BinaryOperator>(Incoming);
+    if (!IncomingInst)
+      return nullptr;
+    if (CommonInst) {
+      if (!CommonInst->isIdenticalToWhenDefined(IncomingInst))
+        return nullptr; // Not identical, give up
+    } else {
+      // Remember binary operator
+      CommonInst = IncomingInst;
+    }
+  }
+  if (!CommonInst)
+    return nullptr;
+
+  // Check if SCEV exprs for instructions are identical.
+  const SCEV *CommonSCEV = getSCEV(CommonInst);
+  bool SCEVExprsIdentical =
+      all_of(drop_begin(PN->incoming_values()),
+             [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
+  return SCEVExprsIdentical ? CommonSCEV : nullptr;
+}
+
 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
   if (const SCEV *S = createAddRecFromPHI(PN))
     return S;
@@ -6034,6 +6066,9 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
                /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
     return getSCEV(V);
 
+  if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
+    return S;
+
   if (const SCEV *S = createNodeFromSelectLikePHI(PN))
     return S;
 
@@ -7324,6 +7359,16 @@ bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
   return PC.MaybePoison.empty();
 }
 
+bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
+  return !SCEVExprContains(Op, [this](const SCEV *S) {
+    auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
+    // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
+    // is a non-zero constant, we have to assume the UDiv may be UB.
+    return UDiv && (!isKnownNonZero(UDiv->getOperand(1)) ||
+                    !isGuaranteedNotToBePoison(UDiv->getOperand(1)));
+  });
+}
+
 bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
   // Only proceed if we can prove that I does not yield poison.
   if (!programUndefinedIfPoison(I))
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 1fb2b9836de0c..d4b6c08c5a32b 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -622,9 +622,9 @@ bool TargetTransformInfo::isVectorIntrinsicWithOverloadTypeAtArg(
 
 InstructionCost TargetTransformInfo::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind) const {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
   return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
-                                           CostKind);
+                                           CostKind, VL);
 }
 
 InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index c48068afc0481..f2c6949e535d2 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4882,9 +4882,19 @@ llvm::fcmpImpliesClass(CmpInst::Predicate Pred, const Function &F, Value *LHS,
 }
 
 static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
-                                        bool CondIsTrue,
+                                        unsigned Depth, bool CondIsTrue,
                                         const Instruction *CxtI,
                                         KnownFPClass &KnownFromContext) {
+  Value *A, *B;
+  if (Depth < MaxAnalysisRecursionDepth &&
+      (CondIsTrue ? match(Cond, m_LogicalAnd(m_Value(A), m_Value(B)))
+                  : match(Cond, m_LogicalOr(m_Value(A), m_Value(B))))) {
+    computeKnownFPClassFromCond(V, A, Depth + 1, CondIsTrue, CxtI,
+                                KnownFromContext);
+    computeKnownFPClassFromCond(V, B, Depth + 1, CondIsTrue, CxtI,
+                                KnownFromContext);
+    return;
+  }
   CmpInst::Predicate Pred;
   Value *LHS;
   uint64_t ClassVal = 0;
@@ -4925,13 +4935,13 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
 
       BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
       if (Q.DT->dominates(Edge0, Q.CxtI->getParent()))
-        computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, Q.CxtI,
-                                    KnownFromContext);
+        computeKnownFPClassFromCond(V, Cond, /*Depth=*/0, /*CondIsTrue=*/true,
+                                    Q.CxtI, KnownFromContext);
 
       BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
       if (Q.DT->dominates(Edge1, Q.CxtI->getParent()))
-        computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, Q.CxtI,
-                                    KnownFromContext);
+        computeKnownFPClassFromCond(V, Cond, /*Depth=*/0, /*CondIsTrue=*/false,
+                                    Q.CxtI, KnownFromContext);
     }
   }
 
@@ -4953,8 +4963,8 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
     if (!isValidAssumeForContext(I, Q.CxtI, Q.DT))
       continue;
 
-    computeKnownFPClassFromCond(V, I->getArgOperand(0), /*CondIsTrue=*/true,
-                                Q.CxtI, KnownFromContext);
+    computeKnownFPClassFromCond(V, I->getArgOperand(0), /*Depth=*/0,
+                                /*CondIsTrue=*/true, Q.CxtI, KnownFromContext);
   }
 
   return KnownFromContext;
@@ -8589,6 +8599,37 @@ bool llvm::isKnownInversion(const Value *X, const Value *Y) {
   return CR1.inverse() == CR2;
 }
 
+SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
+                                           SelectPatternNaNBehavior NaNBehavior,
+                                           bool Ordered) {
+  switch (Pred) {
+  default:
+    return {SPF_UNKNOWN, SPNB_NA, false}; // Equality.
+  case ICmpInst::ICMP_UGT:
+  case ICmpInst::ICMP_UGE:
+    return {SPF_UMAX, SPNB_NA, false};
+  case ICmpInst::ICMP_SGT:
+  case ICmpInst::ICMP_SGE:
+    return {SPF_SMAX, SPNB_NA, false};
+  case ICmpInst::ICMP_ULT:
+  case ICmpInst::ICMP_ULE:
+    return {SPF_UMIN, SPNB_NA, false};
+  case ICmpInst::ICMP_SLT:
+  case ICmpInst::ICMP_SLE:
+    return {SPF_SMIN, SPNB_NA, false};
+  case FCmpInst::FCMP_UGT:
+  case FCmpInst::FCMP_UGE:
+  case FCmpInst::FCMP_OGT:
+  case FCmpInst::FCMP_OGE:
+    return {SPF_FMAXNUM, NaNBehavior, Ordered};
+  case FCmpInst::FCMP_ULT:
+  case FCmpInst::FCMP_ULE:
+  case FCmpInst::FCMP_OLT:
+  case FCmpInst::FCMP_OLE:
+    return {SPF_FMINNUM, NaNBehavior, Ordered};
+  }
+}
+
 static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
                                               FastMathFlags FMF,
                                               Value *CmpLHS, Value *CmpRHS,
@@ -8696,27 +8737,8 @@ static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
   }
 
   // ([if]cmp X, Y) ? X : Y
-  if (TrueVal == CmpLHS && FalseVal == CmpRHS) {
-    switch (Pred) {
-    default: return {SPF_UNKNOWN, SPNB_NA, false}; // Equality.
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE: return {SPF_UMAX, SPNB_NA, false};
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: return {SPF_SMAX, SPNB_NA, false};
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE: return {SPF_UMIN, SPNB_NA, false};
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: return {SPF_SMIN, SPNB_NA, false};
-    case FCmpInst::FCMP_UGT:
-    case FCmpInst::FCMP_UGE:
-    case FCmpInst::FCMP_OGT:
-    case FCmpInst::FCMP_OGE: return {SPF_FMAXNUM, NaNBehavior, Ordered};
-    case FCmpInst::FCMP_ULT:
-    case FCmpInst::FCMP_ULE:
-    case FCmpInst::FCMP_OLT:
-    case FCmpInst::FCMP_OLE: return {SPF_FMINNUM, NaNBehavior, Ordered};
-    }
-  }
+  if (TrueVal == CmpLHS && FalseVal == CmpRHS)
+    return getSelectPattern(Pred, NaNBehavior, Ordered);
 
   if (isKnownNegation(TrueVal, FalseVal)) {
     // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
@@ -8960,6 +8982,21 @@ CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
   llvm_unreachable("unhandled!");
 }
 
+Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
+  switch (SPF) {
+  case SelectPatternFlavor::SPF_UMIN:
+    return Intrinsic::umin;
+  case SelectPatternFlavor::SPF_UMAX:
+    return Intrinsic::umax;
+  case SelectPatternFlavor::SPF_SMIN:
+    return Intrinsic::smin;
+  case SelectPatternFlavor::SPF_SMAX:
+    return Intrinsic::smax;
+  default:
+    llvm_unreachable("Unexpected SPF");
+  }
+}
+
 SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
   if (SPF == SPF_SMIN) return SPF_SMAX;
   if (SPF == SPF_UMIN) return SPF_UMAX;
@@ -9342,7 +9379,7 @@ static std::optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
       (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
       (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
       match(L0, m_c_Add(m_Specific(L1), m_Specific(R1))))
-    return LPred == RPred;
+    return CmpPredicate::getMatching(LPred, RPred).has_value();
 
   if (LPred == RPred)
     return isImpliedCondOperands(LPred, L0, L1, R0, R1);
@@ -9355,7 +9392,7 @@ static std::optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
 /// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
 /// instruction.
 static std::optional<bool>
-isImpliedCondAndOr(const Instruction *LHS, CmpInst::Predicate RHSPred,
+isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
                    const Value *RHSOp0, const Value *RHSOp1,
                    const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
   // The LHS must be an 'or', 'and', or a 'select' instruction.
@@ -9385,7 +9422,7 @@ isImpliedCondAndOr(const Instruction *LHS, CmpInst::Predicate RHSPred,
 }
 
 std::optional<bool>
-llvm::isImpliedCondition(const Value *LHS, CmpInst::Predicate RHSPred,
+llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
                          const Value *RHSOp0, const Value *RHSOp1,
                          const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
   // Bail out when we hit the limit.
@@ -9439,7 +9476,7 @@ std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
 
   if (const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(RHS)) {
     if (auto Implied = isImpliedCondition(
-            LHS, RHSCmp->getPredicate(), RHSCmp->getOperand(0),
+            LHS, RHSCmp->getCmpPredicate(), RHSCmp->getOperand(0),
             RHSCmp->getOperand(1), DL, LHSIsTrue, Depth))
       return InvertRHS ? !*Implied : *Implied;
     return std::nullopt;
@@ -9516,7 +9553,7 @@ std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
   return std::nullopt;
 }
 
-std::optional<bool> llvm::isImpliedByDomCondition(CmpInst::Predicate Pred,
+std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
                                                   const Value *LHS,
                                                   const Value *RHS,
                                                   const Instruction *ContextI,
@@ -10090,7 +10127,7 @@ void llvm::findValuesAffectedByCondition(
 
       if (HasRHSC && match(A, m_Intrinsic<Intrinsic::ctpop>(m_Value(X))))
         AddAffected(X);
-    } else if (match(Cond, m_FCmp(Pred, m_Value(A), m_Value(B)))) {
+    } else if (match(V, m_FCmp(Pred, m_Value(A), m_Value(B)))) {
       AddCmpOperands(A, B);
 
       // fcmp fneg(x), y
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 1789671276ffa..19306679cf6db 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -118,9 +118,13 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
                                               unsigned ScalarOpdIdx) {
   switch (ID) {
   case Intrinsic::abs:
+  case Intrinsic::vp_abs:
   case Intrinsic::ctlz:
+  case Intrinsic::vp_ctlz:
   case Intrinsic::cttz:
+  case Intrinsic::vp_cttz:
   case Intrinsic::is_fpclass:
+  case Intrinsic::vp_is_fpclass:
   case Intrinsic::powi:
     return (ScalarOpdIdx == 1);
   case Intrinsic::smul_fix:
@@ -140,15 +144,21 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
   if (TTI && Intrinsic::isTargetIntrinsic(ID))
     return TTI->isVectorIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);
 
+  if (VPCastIntrinsic::isVPCast(ID))
+    return OpdIdx == -1 || OpdIdx == 0;
+
   switch (ID) {
   case Intrinsic::fptosi_sat:
   case Intrinsic::fptoui_sat:
   case Intrinsic::lrint:
   case Intrinsic::llrint:
+  case Intrinsic::vp_lrint:
+  case Intrinsic::vp_llrint:
   case Intrinsic::ucmp:
   case Intrinsic::scmp:
     return OpdIdx == -1 || OpdIdx == 0;
   case Intrinsic::is_fpclass:
+  case Intrinsic::vp_is_fpclass:
     return OpdIdx == 0;
   case Intrinsic::powi:
     return OpdIdx == -1 || OpdIdx == 1;
@@ -497,25 +507,26 @@ void llvm::processShuffleMasks(
   unsigned SzSrc = Sz / NumOfSrcRegs;
   for (unsigned I = 0; I < NumOfDestRegs; ++I) {
     auto &RegMasks = Res[I];
-    RegMasks.assign(NumOfSrcRegs, {});
+    RegMasks.assign(2 * NumOfSrcRegs, {});
     // Check that the values in dest registers are in the one src
     // register.
     for (unsigned K = 0; K < SzDest; ++K) {
       int Idx = I * SzDest + K;
       if (Idx == Sz)
         break;
-      if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem)
+      if (Mask[Idx] >= 2 * Sz || Mask[Idx] == PoisonMaskElem)
         continue;
-      int SrcRegIdx = Mask[Idx] / SzSrc;
+      int MaskIdx = Mask[Idx] % Sz;
+      int SrcRegIdx = MaskIdx / SzSrc + (Mask[Idx] >= Sz ? NumOfSrcRegs : 0);
       // Add a cost of PermuteTwoSrc for each new source register permute,
       // if we have more than one source registers.
       if (RegMasks[SrcRegIdx].empty())
         RegMasks[SrcRegIdx].assign(SzDest, PoisonMaskElem);
-      RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc;
+      RegMasks[SrcRegIdx][K] = MaskIdx % SzSrc;
     }
   }
   // Process split mask.
-  for (unsigned I = 0; I < NumOfUsedRegs; ++I) {
+  for (unsigned I : seq<unsigned>(NumOfUsedRegs)) {
     auto &Dest = Res[I];
     int NumSrcRegs =
         count_if(Dest, [](ArrayRef<int> Mask) { return !Mask.empty(); });
@@ -560,7 +571,7 @@ void llvm::processShuffleMasks(
         int FirstIdx = -1;
         SecondIdx = -1;
         MutableArrayRef<int> FirstMask, SecondMask;
-        for (unsigned I = 0; I < NumOfDestRegs; ++I) {
+        for (unsigned I : seq<unsigned>(2 * NumOfSrcRegs)) {
           SmallVectorImpl<int> &RegMask = Dest[I];
           if (RegMask.empty())
             continue;
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index b8a8df71d4de2..34311499367b4 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -6261,6 +6261,8 @@ bool LLParser::parseConstantValue(Type *Ty, Constant *&C) {
   case ValID::t_APSInt:
   case ValID::t_APFloat:
   case ValID::t_Undef:
+  case ValID::t_Poison:
+  case ValID::t_Zero:
   case ValID::t_Constant:
   case ValID::t_ConstantSplat:
   case ValID::t_ConstantStruct:
@@ -7002,8 +7004,6 @@ int LLParser::parseInstruction(Instruction *&Inst, BasicBlock *BB,
     return false;
   }
   case lltok::kw_sext:
-  case lltok::kw_fptrunc:
-  case lltok::kw_fpext:
   case lltok::kw_bitcast:
   case lltok::kw_addrspacecast:
   case lltok::kw_sitofp:
@@ -7012,6 +7012,16 @@ int LLParser::parseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_inttoptr:
   case lltok::kw_ptrtoint:
     return parseCast(Inst, PFS, KeywordVal);
+  case lltok::kw_fptrunc:
+  case lltok::kw_fpext: {
+    FastMathFlags FMF = EatFastMathFlagsIfPresent();
+    if (parseCast(Inst, PFS, KeywordVal))
+      return true;
+    if (FMF.any())
+      Inst->setFastMathFlags(FMF);
+    return false;
+  }
+
   // Other.
   case lltok::kw_select: {
     FastMathFlags FMF = EatFastMathFlagsIfPresent();
diff --git a/llvm/lib/BinaryFormat/XCOFF.cpp b/llvm/lib/BinaryFormat/XCOFF.cpp
index 6b11ab2ff96bc..e0a4471bdb9c3 100644
--- a/llvm/lib/BinaryFormat/XCOFF.cpp
+++ b/llvm/lib/BinaryFormat/XCOFF.cpp
@@ -9,8 +9,10 @@
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
+#include "llvm/TargetParser/PPCTargetParser.h"
 
 using namespace llvm;
 
@@ -107,6 +109,63 @@ StringRef XCOFF::getNameForTracebackTableLanguageId(
 }
 #undef LANG_CASE
 
+XCOFF::CFileCpuId XCOFF::getCpuID(StringRef CPUName) {
+  StringRef CPU = PPC::normalizeCPUName(CPUName);
+  return StringSwitch<XCOFF::CFileCpuId>(CPU)
+      .Cases("generic", "COM", XCOFF::TCPU_COM)
+      .Case("601", XCOFF::TCPU_601)
+      .Cases("602", "603", "603e", "603ev", XCOFF::TCPU_603)
+      .Cases("604", "604e", XCOFF::TCPU_604)
+      .Case("620", XCOFF::TCPU_620)
+      .Case("970", XCOFF::TCPU_970)
+      .Cases("a2", "g3", "g4", "g5", "e500", XCOFF::TCPU_COM)
+      .Cases("pwr3", "pwr4", XCOFF::TCPU_COM)
+      .Cases("pwr5", "PWR5", XCOFF::TCPU_PWR5)
+      .Cases("pwr5x", "PWR5X", XCOFF::TCPU_PWR5X)
+      .Cases("pwr6", "PWR6", XCOFF::TCPU_PWR6)
+      .Cases("pwr6x", "PWR6E", XCOFF::TCPU_PWR6E)
+      .Cases("pwr7", "PWR7", XCOFF::TCPU_PWR7)
+      .Cases("pwr8", "PWR8", XCOFF::TCPU_PWR8)
+      .Cases("pwr9", "PWR9", XCOFF::TCPU_PWR9)
+      .Cases("pwr10", "PWR10", XCOFF::TCPU_PWR10)
+      .Cases("ppc", "PPC", "ppc32", "ppc64", XCOFF::TCPU_COM)
+      .Case("ppc64le", XCOFF::TCPU_PWR8)
+      .Case("future", XCOFF::TCPU_PWR10)
+      .Cases("any", "ANY", XCOFF::TCPU_ANY)
+      .Default(XCOFF::TCPU_INVALID);
+}
+
+#define TCPU_CASE(A)                                                           \
+  case XCOFF::TCPU_##A:                                                        \
+    return #A;
+StringRef XCOFF::getTCPUString(XCOFF::CFileCpuId TCPU) {
+  switch (TCPU) {
+    TCPU_CASE(INVALID)
+    TCPU_CASE(PPC)
+    TCPU_CASE(PPC64)
+    TCPU_CASE(COM)
+    TCPU_CASE(PWR)
+    TCPU_CASE(ANY)
+    TCPU_CASE(601)
+    TCPU_CASE(603)
+    TCPU_CASE(604)
+    TCPU_CASE(620)
+    TCPU_CASE(A35)
+    TCPU_CASE(PWR5)
+    TCPU_CASE(970)
+    TCPU_CASE(PWR6)
+    TCPU_CASE(PWR5X)
+    TCPU_CASE(PWR6E)
+    TCPU_CASE(PWR7)
+    TCPU_CASE(PWR8)
+    TCPU_CASE(PWR9)
+    TCPU_CASE(PWR10)
+    TCPU_CASE(PWRX)
+  }
+  return "INVALID";
+}
+#undef TCPU_CASE
+
 Expected<SmallString<32>> XCOFF::parseParmsType(uint32_t Value,
                                                 unsigned FixedParmsNum,
                                                 unsigned FloatingParmsNum) {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 11fbe6e6158ee..85c6fadeda6cc 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1663,15 +1663,42 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
           C = BlockAddress::get(Fn, BB);
           break;
         }
-        case BitcodeConstant::ConstantStructOpcode:
-          C = ConstantStruct::get(cast<StructType>(BC->getType()), ConstOps);
+        case BitcodeConstant::ConstantStructOpcode: {
+          auto *ST = cast<StructType>(BC->getType());
+          if (ST->getNumElements() != ConstOps.size())
+            return error("Invalid number of elements in struct initializer");
+
+          for (const auto [Ty, Op] : zip(ST->elements(), ConstOps))
+            if (Op->getType() != Ty)
+              return error("Incorrect type in struct initializer");
+
+          C = ConstantStruct::get(ST, ConstOps);
           break;
-        case BitcodeConstant::ConstantArrayOpcode:
-          C = ConstantArray::get(cast<ArrayType>(BC->getType()), ConstOps);
+        }
+        case BitcodeConstant::ConstantArrayOpcode: {
+          auto *AT = cast<ArrayType>(BC->getType());
+          if (AT->getNumElements() != ConstOps.size())
+            return error("Invalid number of elements in array initializer");
+
+          for (Constant *Op : ConstOps)
+            if (Op->getType() != AT->getElementType())
+              return error("Incorrect type in array initializer");
+
+          C = ConstantArray::get(AT, ConstOps);
           break;
-        case BitcodeConstant::ConstantVectorOpcode:
+        }
+        case BitcodeConstant::ConstantVectorOpcode: {
+          auto *VT = cast<FixedVectorType>(BC->getType());
+          if (VT->getNumElements() != ConstOps.size())
+            return error("Invalid number of elements in vector initializer");
+
+          for (Constant *Op : ConstOps)
+            if (Op->getType() != VT->getElementType())
+              return error("Incorrect type in vector initializer");
+
           C = ConstantVector::get(ConstOps);
           break;
+        }
         case Instruction::GetElementPtr:
           C = ConstantExpr::getGetElementPtr(
               BC->SrcElemTy, ConstOps[0], ArrayRef(ConstOps).drop_front(),
@@ -5170,6 +5197,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
           if (Record[OpNum] & (1 << bitc::TIO_NO_SIGNED_WRAP))
             cast<TruncInst>(I)->setHasNoSignedWrap(true);
         }
+        if (isa<FPMathOperator>(I)) {
+          FastMathFlags FMF = getDecodedFastMathFlags(Record[OpNum]);
+          if (FMF.any())
+            I->setFastMathFlags(FMF);
+        }
       }
 
       InstructionList.push_back(I);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 63f4e34074e06..0444cb9e1bce5 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -4900,7 +4900,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
       NameVals.push_back(*ValueId);
       assert(ModuleIdMap.count(VS->modulePath()));
       NameVals.push_back(ModuleIdMap[VS->modulePath()]);
-      NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
+      NameVals.push_back(
+          getEncodedGVSummaryFlags(VS->flags(), shouldImportValueAsDecl(VS)));
       NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
       for (auto &RI : VS->refs()) {
         auto RefValueId = getValueId(RI.getGUID());
diff --git a/llvm/lib/Bitstream/Reader/BitstreamReader.cpp b/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
index 5b2c76350029b..fed9994db2ae8 100644
--- a/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
+++ b/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
@@ -334,7 +334,8 @@ Expected<unsigned> BitstreamCursor::readRecord(unsigned AbbrevID,
 
     // Figure out where the end of this blob will be including tail padding.
     size_t CurBitPos = GetCurrentBitNo();
-    const size_t NewEnd = CurBitPos + alignTo(NumElts, 4) * 8;
+    const size_t NewEnd =
+        CurBitPos + static_cast<uint64_t>(alignTo(NumElts, 4)) * 8;
 
     // Make sure the bitstream is large enough to contain the blob.
     if (!canSkipToPos(NewEnd/8))
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 0dbf2ffb6d80f..5cf1b2d038fb9 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -60,10 +60,10 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeInterleavedAccessPass(Registry);
   initializeJMCInstrumenterPass(Registry);
   initializeLiveDebugValuesPass(Registry);
-  initializeLiveDebugVariablesPass(Registry);
+  initializeLiveDebugVariablesWrapperLegacyPass(Registry);
   initializeLiveIntervalsWrapperPassPass(Registry);
   initializeLiveRangeShrinkPass(Registry);
-  initializeLiveStacksPass(Registry);
+  initializeLiveStacksWrapperLegacyPass(Registry);
   initializeLiveVariablesWrapperPassPass(Registry);
   initializeLocalStackSlotPassPass(Registry);
   initializeLowerGlobalDtorsLegacyPassPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index ead8aa6d22097..83c6ecd401039 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7138,6 +7138,7 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
   SmallVector<Instruction *, 8> WorkList;
   SmallPtrSet<Instruction *, 16> Visited;
   SmallVector<Instruction *, 8> AndsToMaybeRemove;
+  SmallVector<Instruction *, 8> DropFlags;
   for (auto *U : Load->users())
     WorkList.push_back(cast<Instruction>(U));
 
@@ -7185,6 +7186,7 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
         return false;
       uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);
       DemandBits.setLowBits(BitWidth - ShiftAmt);
+      DropFlags.push_back(I);
       break;
     }
 
@@ -7192,6 +7194,7 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
       EVT TruncVT = TLI->getValueType(*DL, I->getType());
       unsigned TruncBitWidth = TruncVT.getSizeInBits();
       DemandBits.setLowBits(TruncBitWidth);
+      DropFlags.push_back(I);
       break;
     }
 
@@ -7249,6 +7252,10 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
       ++NumAndUses;
     }
 
+  // NSW flags may not longer hold.
+  for (auto *Inst : DropFlags)
+    Inst->setHasNoSignedWrap(false);
+
   ++NumAndsAdded;
   return true;
 }
diff --git a/llvm/lib/CodeGen/GCMetadata.cpp b/llvm/lib/CodeGen/GCMetadata.cpp
index 6d1cc1a58e27d..f33008c9e0f23 100644
--- a/llvm/lib/CodeGen/GCMetadata.cpp
+++ b/llvm/lib/CodeGen/GCMetadata.cpp
@@ -66,7 +66,7 @@ GCFunctionAnalysis::run(Function &F, FunctionAnalysisManager &FAM) {
 }
 
 INITIALIZE_PASS(GCModuleInfo, "collector-metadata",
-                "Create Garbage Collector Module Metadata", false, false)
+                "Create Garbage Collector Module Metadata", false, true)
 
 // -----------------------------------------------------------------------------
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 5126aba3658c1..d17b20d977ce9 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -258,7 +258,7 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
       MemAlign = *ParamAlign;
     else
-      MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
+      MemAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
   } else if (OpIdx >= AttributeList::FirstArgIndex) {
     if (auto ParamAlign =
             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index d95fc8cfbcf55..a273799544652 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -7717,9 +7717,9 @@ bool CombinerHelper::matchShuffleUndefRHS(MachineInstr &MI,
   if (!Changed)
     return false;
 
-  MatchInfo = [&, NewMask](MachineIRBuilder &B) {
+  MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
     B.buildShuffleVector(MI.getOperand(0), MI.getOperand(1), MI.getOperand(2),
-                         NewMask);
+                         std::move(NewMask));
   };
 
   return true;
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 30cd3ce3baa50..6c15ed3423d3b 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -253,6 +253,7 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
         // For COPYs we don't do anything, don't increase the depth.
         computeKnownBitsImpl(SrcReg, Known2, DemandedElts,
                              Depth + (Opcode != TargetOpcode::COPY));
+        Known2 = Known2.anyextOrTrunc(BitWidth);
         Known = Known.intersectWith(Known2);
         // If we reach a point where we don't know anything
         // just stop looking through the operands.
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 675f55d8086bc..f668e41094bbc 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2431,6 +2431,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
   case Intrinsic::invariant_end:
     return true;
   case Intrinsic::expect:
+  case Intrinsic::expect_with_probability:
   case Intrinsic::annotation:
   case Intrinsic::ptr_annotation:
   case Intrinsic::launder_invariant_group:
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 709eb2525a657..f56be39838ba7 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -182,9 +182,8 @@ void LegalizerHelper::insertParts(Register DstReg,
   // Merge sub-vectors with different number of elements and insert into DstReg.
   if (ResultTy.isVector()) {
     assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
-    SmallVector<Register, 8> AllRegs;
-    for (auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
-      AllRegs.push_back(Reg);
+    SmallVector<Register, 8> AllRegs(PartRegs.begin(), PartRegs.end());
+    AllRegs.append(LeftoverRegs.begin(), LeftoverRegs.end());
     return mergeMixedSubvectors(DstReg, AllRegs);
   }
 
@@ -882,11 +881,20 @@ static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
 
 static LegalizerHelper::LegalizeResult
 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
-                  Type *FromType, LostDebugLocObserver &LocObserver) {
+                  Type *FromType, LostDebugLocObserver &LocObserver,
+                  const TargetLowering &TLI, bool IsSigned = false) {
+  CallLowering::ArgInfo Arg = {MI.getOperand(1).getReg(), FromType, 0};
+  if (FromType->isIntegerTy()) {
+    if (TLI.shouldSignExtendTypeInLibCall(FromType, IsSigned))
+      Arg.Flags[0].setSExt();
+    else
+      Arg.Flags[0].setZExt();
+  }
+
   RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
-  return createLibcall(
-      MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType, 0},
-      {{MI.getOperand(1).getReg(), FromType, 0}}, LocObserver, &MI);
+  return createLibcall(MIRBuilder, Libcall,
+                       {MI.getOperand(0).getReg(), ToType, 0}, Arg, LocObserver,
+                       &MI);
 }
 
 static RTLIB::Libcall
@@ -1233,7 +1241,6 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
   case TargetOpcode::G_FLOG10:
   case TargetOpcode::G_FLOG:
   case TargetOpcode::G_FLOG2:
-  case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_FEXP:
   case TargetOpcode::G_FEXP2:
   case TargetOpcode::G_FEXP10:
@@ -1279,7 +1286,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     MI.eraseFromParent();
     return Legalized;
   }
-  case TargetOpcode::G_FPOWI: {
+  case TargetOpcode::G_FPOWI:
+  case TargetOpcode::G_FLDEXP: {
     LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
     unsigned Size = LLTy.getSizeInBits();
     Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
@@ -1290,9 +1298,10 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
       return UnableToLegalize;
     }
     auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
-    std::initializer_list<CallLowering::ArgInfo> Args = {
+    SmallVector<CallLowering::ArgInfo, 2> Args = {
         {MI.getOperand(1).getReg(), HLTy, 0},
         {MI.getOperand(2).getReg(), ITy, 1}};
+    Args[1].Flags[0].setSExt();
     LegalizeResult Status =
         createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), HLTy, 0},
                       Args, LocObserver, &MI);
@@ -1307,7 +1316,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     if (!FromTy || !ToTy)
       return UnableToLegalize;
     LegalizeResult Status =
-        conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver);
+        conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver, TLI);
     if (Status != Legalized)
       return Status;
     break;
@@ -1328,7 +1337,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
       return UnableToLegalize;
     LegalizeResult Status = conversionLibcall(
-        MI, MIRBuilder, Type::getIntNTy(Ctx, ToSize), FromTy, LocObserver);
+        MI, MIRBuilder, Type::getIntNTy(Ctx, ToSize), FromTy, LocObserver, TLI);
     if (Status != Legalized)
       return Status;
     break;
@@ -1340,8 +1349,10 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
         getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
     if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
       return UnableToLegalize;
-    LegalizeResult Status = conversionLibcall(
-        MI, MIRBuilder, ToTy, Type::getIntNTy(Ctx, FromSize), LocObserver);
+    bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
+    LegalizeResult Status =
+        conversionLibcall(MI, MIRBuilder, ToTy, Type::getIntNTy(Ctx, FromSize),
+                          LocObserver, TLI, IsSigned);
     if (Status != Legalized)
       return Status;
     break;
@@ -6161,9 +6172,8 @@ LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
   if (MaskNumElts < SrcNumElts) {
     // Extend mask to match new destination vector size with
     // undef values.
-    SmallVector<int, 16> NewMask(Mask);
-    for (unsigned I = MaskNumElts; I < SrcNumElts; ++I)
-      NewMask.push_back(-1);
+    SmallVector<int, 16> NewMask(SrcNumElts, -1);
+    llvm::copy(Mask, NewMask.begin());
 
     moreElementsVectorDst(MI, SrcTy, 0);
     MIRBuilder.setInstrAndDebugLoc(MI);
@@ -6243,16 +6253,14 @@ LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
   moreElementsVectorSrc(MI, MoreTy, 2);
 
   // Adjust mask based on new input vector length.
-  SmallVector<int, 16> NewMask;
+  SmallVector<int, 16> NewMask(WidenNumElts, -1);
   for (unsigned I = 0; I != NumElts; ++I) {
     int Idx = Mask[I];
     if (Idx < static_cast<int>(NumElts))
-      NewMask.push_back(Idx);
+      NewMask[I] = Idx;
     else
-      NewMask.push_back(Idx - NumElts + WidenNumElts);
+      NewMask[I] = Idx - NumElts + WidenNumElts;
   }
-  for (unsigned I = NumElts; I != WidenNumElts; ++I)
-    NewMask.push_back(-1);
   moreElementsVectorDst(MI, MoreTy, 0);
   MIRBuilder.setInstrAndDebugLoc(MI);
   MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(),
@@ -6385,19 +6393,19 @@ LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
                MRI);
 
   int NarrowParts = Src1Regs.size();
-  for (int I = 0, E = Src1Left.size(); I != E; ++I) {
-    Src1Regs.push_back(Src1Left[I]);
-    Src2Regs.push_back(Src2Left[I]);
-  }
+  Src1Regs.append(Src1Left);
+  Src2Regs.append(Src2Left);
   DstRegs.reserve(Src1Regs.size());
 
   for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
     Register DstReg =
         MRI.createGenericVirtualRegister(MRI.getType(Src1Regs[i]));
-    Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
+    Register CarryOut;
     // Forward the final carry-out to the destination register
     if (i == e - 1 && CarryDst)
       CarryOut = CarryDst;
+    else
+      CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
 
     if (!CarryIn) {
       MIRBuilder.buildInstr(OpO, {DstReg, CarryOut},
@@ -6558,8 +6566,7 @@ LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
   extractParts(MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs,
                LeftoverRegs, MIRBuilder, MRI);
 
-  for (Register Reg : LeftoverRegs)
-    SrcRegs.push_back(Reg);
+  SrcRegs.append(LeftoverRegs);
 
   uint64_t NarrowSize = NarrowTy.getSizeInBits();
   Register OpReg = MI.getOperand(2).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index e386647daa653..f1fec547ebd60 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -69,8 +69,8 @@ INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,
                     "Assign register bank of generic virtual registers", false,
                     false)
 
-RegBankSelect::RegBankSelect(char &PassID, Mode RunningMode)
-    : MachineFunctionPass(PassID), OptMode(RunningMode) {
+RegBankSelect::RegBankSelect(Mode RunningMode)
+    : MachineFunctionPass(ID), OptMode(RunningMode) {
   if (RegBankSelectMode.getNumOccurrences() != 0) {
     OptMode = RegBankSelectMode;
     if (RegBankSelectMode != RunningMode)
diff --git a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
index 470582885fab0..1187ad06b7372 100644
--- a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
+++ b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
@@ -154,7 +154,7 @@ static Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) {
     auto *DestAT = dyn_cast<ArrayType>(DestTy);
     assert(DestAT);
     assert(SrcAT->getNumElements() == DestAT->getNumElements());
-    Value *Result = UndefValue::get(DestTy);
+    Value *Result = PoisonValue::get(DestTy);
     for (unsigned int I = 0, E = SrcAT->getNumElements(); I < E; ++I) {
       Value *Element =
           createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)),
diff --git a/llvm/lib/CodeGen/InitUndef.cpp b/llvm/lib/CodeGen/InitUndef.cpp
index d4ac131a32a95..d8b3190f31003 100644
--- a/llvm/lib/CodeGen/InitUndef.cpp
+++ b/llvm/lib/CodeGen/InitUndef.cpp
@@ -164,6 +164,14 @@ bool InitUndef::handleSubReg(MachineFunction &MF, MachineInstr &MI,
     TRI->getCoveringSubRegIndexes(*MRI, TargetRegClass, NeedDef,
                                   SubRegIndexNeedInsert);
 
+    // It's not possible to create the INIT_UNDEF when there is no register
+    // class associated for the subreg. This may happen for artificial subregs
+    // that are not directly addressable.
+    if (any_of(SubRegIndexNeedInsert, [&](unsigned Ind) -> bool {
+          return !TRI->getSubRegisterClass(TargetRegClass, Ind);
+        }))
+      continue;
+
     Register LatestReg = Reg;
     for (auto ind : SubRegIndexNeedInsert) {
       Changed = true;
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index e8f7c6850a507..64f290f5930a1 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -131,7 +131,7 @@ class HoistSpillHelper : private LiveRangeEdit::Delegate {
   HoistSpillHelper(MachineFunctionPass &pass, MachineFunction &mf,
                    VirtRegMap &vrm)
       : MF(mf), LIS(pass.getAnalysis<LiveIntervalsWrapperPass>().getLIS()),
-        LSS(pass.getAnalysis<LiveStacks>()),
+        LSS(pass.getAnalysis<LiveStacksWrapperLegacy>().getLS()),
         MDT(pass.getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree()),
         VRM(vrm), MRI(mf.getRegInfo()), TII(*mf.getSubtarget().getInstrInfo()),
         TRI(*mf.getSubtarget().getRegisterInfo()),
@@ -193,7 +193,7 @@ class InlineSpiller : public Spiller {
   InlineSpiller(MachineFunctionPass &Pass, MachineFunction &MF, VirtRegMap &VRM,
                 VirtRegAuxInfo &VRAI)
       : MF(MF), LIS(Pass.getAnalysis<LiveIntervalsWrapperPass>().getLIS()),
-        LSS(Pass.getAnalysis<LiveStacks>()),
+        LSS(Pass.getAnalysis<LiveStacksWrapperLegacy>().getLS()),
         MDT(Pass.getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree()),
         VRM(VRM), MRI(MF.getRegInfo()), TII(*MF.getSubtarget().getInstrInfo()),
         TRI(*MF.getSubtarget().getRegisterInfo()),
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 2ff346d3fd022..79085e587ebc4 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -38,6 +38,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -74,24 +75,27 @@ EnableLDV("live-debug-variables", cl::init(true),
 STATISTIC(NumInsertedDebugValues, "Number of DBG_VALUEs inserted");
 STATISTIC(NumInsertedDebugLabels, "Number of DBG_LABELs inserted");
 
-char LiveDebugVariables::ID = 0;
+char LiveDebugVariablesWrapperLegacy::ID = 0;
 
-INITIALIZE_PASS_BEGIN(LiveDebugVariables, DEBUG_TYPE,
-                "Debug Variable Analysis", false, false)
+INITIALIZE_PASS_BEGIN(LiveDebugVariablesWrapperLegacy, DEBUG_TYPE,
+                      "Debug Variable Analysis", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_END(LiveDebugVariables, DEBUG_TYPE,
-                "Debug Variable Analysis", false, false)
+INITIALIZE_PASS_END(LiveDebugVariablesWrapperLegacy, DEBUG_TYPE,
+                    "Debug Variable Analysis", false, true)
 
-void LiveDebugVariables::getAnalysisUsage(AnalysisUsage &AU) const {
+void LiveDebugVariablesWrapperLegacy::getAnalysisUsage(
+    AnalysisUsage &AU) const {
   AU.addRequired<MachineDominatorTreeWrapperPass>();
   AU.addRequiredTransitive<LiveIntervalsWrapperPass>();
   AU.setPreservesAll();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-LiveDebugVariables::LiveDebugVariables() : MachineFunctionPass(ID) {
-  initializeLiveDebugVariablesPass(*PassRegistry::getPassRegistry());
+LiveDebugVariablesWrapperLegacy::LiveDebugVariablesWrapperLegacy()
+    : MachineFunctionPass(ID) {
+  initializeLiveDebugVariablesWrapperLegacyPass(
+      *PassRegistry::getPassRegistry());
 }
 
 enum : unsigned { UndefLocNo = ~0U };
@@ -274,8 +278,6 @@ using BlockSkipInstsMap =
 
 namespace {
 
-class LDVImpl;
-
 /// A user value is a part of a debug info user variable.
 ///
 /// A DBG_VALUE instruction notes that (a sub-register of) a virtual register
@@ -285,6 +287,8 @@ class LDVImpl;
 /// user values are related if they are held by the same virtual register. The
 /// equivalence class is the transitive closure of that relation.
 class UserValue {
+  using LDVImpl = LiveDebugVariables::LDVImpl;
+
   const DILocalVariable *Variable; ///< The debug info variable we are part of.
   /// The part of the variable we describe.
   const std::optional<DIExpression::FragmentInfo> Fragment;
@@ -528,9 +532,17 @@ class UserLabel {
   void print(raw_ostream &, const TargetRegisterInfo *);
 };
 
+} // end anonymous namespace
+
+namespace llvm {
+
 /// Implementation of the LiveDebugVariables pass.
-class LDVImpl {
-  LiveDebugVariables &pass;
+
+LiveDebugVariables::LiveDebugVariables() = default;
+LiveDebugVariables::~LiveDebugVariables() = default;
+LiveDebugVariables::LiveDebugVariables(LiveDebugVariables &&) = default;
+
+class LiveDebugVariables::LDVImpl {
   LocMap::Allocator allocator;
   MachineFunction *MF = nullptr;
   LiveIntervals *LIS;
@@ -634,7 +646,7 @@ class LDVImpl {
   void computeIntervals();
 
 public:
-  LDVImpl(LiveDebugVariables *ps) : pass(*ps) {}
+  LDVImpl(LiveIntervals *LIS) : LIS(LIS) {}
 
   bool runOnMachineFunction(MachineFunction &mf, bool InstrRef);
 
@@ -671,9 +683,8 @@ class LDVImpl {
   void print(raw_ostream&);
 };
 
-} // end anonymous namespace
+} // namespace llvm
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 static void printDebugLoc(const DebugLoc &DL, raw_ostream &CommentOS,
                           const LLVMContext &Ctx) {
   if (!DL)
@@ -753,7 +764,7 @@ void UserLabel::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
   OS << '\n';
 }
 
-void LDVImpl::print(raw_ostream &OS) {
+void LiveDebugVariables::LDVImpl::print(raw_ostream &OS) {
   OS << "********** DEBUG VARIABLES **********\n";
   for (auto &userValue : userValues)
     userValue->print(OS, TRI);
@@ -761,18 +772,16 @@ void LDVImpl::print(raw_ostream &OS) {
   for (auto &userLabel : userLabels)
     userLabel->print(OS, TRI);
 }
-#endif
 
-void UserValue::mapVirtRegs(LDVImpl *LDV) {
+void UserValue::mapVirtRegs(LiveDebugVariables::LDVImpl *LDV) {
   for (const MachineOperand &MO : locations)
     if (MO.isReg() && MO.getReg().isVirtual())
       LDV->mapVirtReg(MO.getReg(), this);
 }
 
-UserValue *
-LDVImpl::getUserValue(const DILocalVariable *Var,
-                      std::optional<DIExpression::FragmentInfo> Fragment,
-                      const DebugLoc &DL) {
+UserValue *LiveDebugVariables::LDVImpl::getUserValue(
+    const DILocalVariable *Var,
+    std::optional<DIExpression::FragmentInfo> Fragment, const DebugLoc &DL) {
   // FIXME: Handle partially overlapping fragments. See
   // https://reviews.llvm.org/D70121#1849741.
   DebugVariable ID(Var, Fragment, DL->getInlinedAt());
@@ -785,19 +794,20 @@ LDVImpl::getUserValue(const DILocalVariable *Var,
   return UV;
 }
 
-void LDVImpl::mapVirtReg(Register VirtReg, UserValue *EC) {
+void LiveDebugVariables::LDVImpl::mapVirtReg(Register VirtReg, UserValue *EC) {
   assert(VirtReg.isVirtual() && "Only map VirtRegs");
   UserValue *&Leader = virtRegToEqClass[VirtReg];
   Leader = UserValue::merge(Leader, EC);
 }
 
-UserValue *LDVImpl::lookupVirtReg(Register VirtReg) {
+UserValue *LiveDebugVariables::LDVImpl::lookupVirtReg(Register VirtReg) {
   if (UserValue *UV = virtRegToEqClass.lookup(VirtReg))
     return UV->getLeader();
   return nullptr;
 }
 
-bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
+bool LiveDebugVariables::LDVImpl::handleDebugValue(MachineInstr &MI,
+                                                   SlotIndex Idx) {
   // DBG_VALUE loc, offset, variable, expr
   // DBG_VALUE_LIST variable, expr, locs...
   if (!MI.isDebugValue()) {
@@ -873,8 +883,8 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   return true;
 }
 
-MachineBasicBlock::iterator LDVImpl::handleDebugInstr(MachineInstr &MI,
-                                                      SlotIndex Idx) {
+MachineBasicBlock::iterator
+LiveDebugVariables::LDVImpl::handleDebugInstr(MachineInstr &MI, SlotIndex Idx) {
   assert(MI.isDebugValueLike() || MI.isDebugPHI());
 
   // In instruction referencing mode, there should be no DBG_VALUE instructions
@@ -894,7 +904,8 @@ MachineBasicBlock::iterator LDVImpl::handleDebugInstr(MachineInstr &MI,
   return NextInst;
 }
 
-bool LDVImpl::handleDebugLabel(MachineInstr &MI, SlotIndex Idx) {
+bool LiveDebugVariables::LDVImpl::handleDebugLabel(MachineInstr &MI,
+                                                   SlotIndex Idx) {
   // DBG_LABEL label
   if (MI.getNumOperands() != 1 || !MI.getOperand(0).isMetadata()) {
     LLVM_DEBUG(dbgs() << "Can't handle " << MI);
@@ -917,7 +928,8 @@ bool LDVImpl::handleDebugLabel(MachineInstr &MI, SlotIndex Idx) {
   return true;
 }
 
-bool LDVImpl::collectDebugValues(MachineFunction &mf, bool InstrRef) {
+bool LiveDebugVariables::LDVImpl::collectDebugValues(MachineFunction &mf,
+                                                     bool InstrRef) {
   bool Changed = false;
   for (MachineBasicBlock &MBB : mf) {
     for (MachineBasicBlock::iterator MBBI = MBB.begin(), MBBE = MBB.end();
@@ -1250,7 +1262,7 @@ void UserValue::computeIntervals(MachineRegisterInfo &MRI,
     I.setStopUnchecked(PrevEnd);
 }
 
-void LDVImpl::computeIntervals() {
+void LiveDebugVariables::LDVImpl::computeIntervals() {
   LexicalScopes LS;
   LS.initialize(*MF);
 
@@ -1260,10 +1272,10 @@ void LDVImpl::computeIntervals() {
   }
 }
 
-bool LDVImpl::runOnMachineFunction(MachineFunction &mf, bool InstrRef) {
+bool LiveDebugVariables::LDVImpl::runOnMachineFunction(MachineFunction &mf,
+                                                       bool InstrRef) {
   clear();
   MF = &mf;
-  LIS = &pass.getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   TRI = mf.getSubtarget().getRegisterInfo();
   LLVM_DEBUG(dbgs() << "********** COMPUTING LIVE DEBUG VARIABLES: "
                     << mf.getName() << " **********\n");
@@ -1298,31 +1310,65 @@ static void removeDebugInstrs(MachineFunction &mf) {
   }
 }
 
-bool LiveDebugVariables::runOnMachineFunction(MachineFunction &mf) {
-  if (!EnableLDV)
-    return false;
-  if (!mf.getFunction().getSubprogram()) {
-    removeDebugInstrs(mf);
-    return false;
-  }
+bool LiveDebugVariablesWrapperLegacy::runOnMachineFunction(
+    MachineFunction &mf) {
+  auto *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
 
-  // Have we been asked to track variable locations using instruction
-  // referencing?
-  bool InstrRef = mf.useDebugInstrRef();
+  Impl = std::make_unique<LiveDebugVariables>();
+  Impl->analyze(mf, LIS);
+  return false;
+}
 
-  if (!pImpl)
-    pImpl = new LDVImpl(this);
-  return static_cast<LDVImpl *>(pImpl)->runOnMachineFunction(mf, InstrRef);
+AnalysisKey LiveDebugVariablesAnalysis::Key;
+
+LiveDebugVariables
+LiveDebugVariablesAnalysis::run(MachineFunction &MF,
+                                MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+
+  auto *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  LiveDebugVariables LDV;
+  LDV.analyze(MF, LIS);
+  return LDV;
+}
+
+PreservedAnalyses
+LiveDebugVariablesPrinterPass::run(MachineFunction &MF,
+                                   MachineFunctionAnalysisManager &MFAM) {
+  auto &LDV = MFAM.getResult<LiveDebugVariablesAnalysis>(MF);
+  LDV.print(OS);
+  return PreservedAnalyses::all();
 }
 
 void LiveDebugVariables::releaseMemory() {
-  if (pImpl)
-    static_cast<LDVImpl*>(pImpl)->clear();
+  if (PImpl)
+    PImpl->clear();
 }
 
-LiveDebugVariables::~LiveDebugVariables() {
-  if (pImpl)
-    delete static_cast<LDVImpl*>(pImpl);
+bool LiveDebugVariables::invalidate(
+    MachineFunction &, const PreservedAnalyses &PA,
+    MachineFunctionAnalysisManager::Invalidator &) {
+  auto PAC = PA.getChecker<LiveDebugVariablesAnalysis>();
+  // Some architectures split the register allocation into multiple phases based
+  // on register classes. This requires preserving analyses between the phases
+  // by default.
+  return !PAC.preservedWhenStateless();
+}
+
+void LiveDebugVariables::analyze(MachineFunction &MF, LiveIntervals *LIS) {
+  if (!EnableLDV)
+    return;
+  if (!MF.getFunction().getSubprogram()) {
+    removeDebugInstrs(MF);
+    return;
+  }
+
+  PImpl.reset(new LDVImpl(LIS));
+
+  // Have we been asked to track variable locations using instruction
+  // referencing?
+  bool InstrRef = MF.useDebugInstrRef();
+  PImpl->runOnMachineFunction(MF, InstrRef);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1445,7 +1491,8 @@ UserValue::splitRegister(Register OldReg, ArrayRef<Register> NewRegs,
   return DidChange;
 }
 
-void LDVImpl::splitPHIRegister(Register OldReg, ArrayRef<Register> NewRegs) {
+void LiveDebugVariables::LDVImpl::splitPHIRegister(Register OldReg,
+                                                   ArrayRef<Register> NewRegs) {
   auto RegIt = RegToPHIIdx.find(OldReg);
   if (RegIt == RegToPHIIdx.end())
     return;
@@ -1483,7 +1530,8 @@ void LDVImpl::splitPHIRegister(Register OldReg, ArrayRef<Register> NewRegs) {
     RegToPHIIdx[RegAndInstr.first].push_back(RegAndInstr.second);
 }
 
-void LDVImpl::splitRegister(Register OldReg, ArrayRef<Register> NewRegs) {
+void LiveDebugVariables::LDVImpl::splitRegister(Register OldReg,
+                                                ArrayRef<Register> NewRegs) {
   // Consider whether this split range affects any PHI locations.
   splitPHIRegister(OldReg, NewRegs);
 
@@ -1504,8 +1552,8 @@ void LDVImpl::splitRegister(Register OldReg, ArrayRef<Register> NewRegs) {
 
 void LiveDebugVariables::
 splitRegister(Register OldReg, ArrayRef<Register> NewRegs, LiveIntervals &LIS) {
-  if (pImpl)
-    static_cast<LDVImpl*>(pImpl)->splitRegister(OldReg, NewRegs);
+  if (PImpl)
+    PImpl->splitRegister(OldReg, NewRegs);
 }
 
 void UserValue::rewriteLocations(VirtRegMap &VRM, const MachineFunction &MF,
@@ -1807,7 +1855,7 @@ void UserLabel::emitDebugLabel(LiveIntervals &LIS, const TargetInstrInfo &TII,
   LLVM_DEBUG(dbgs() << '\n');
 }
 
-void LDVImpl::emitDebugValues(VirtRegMap *VRM) {
+void LiveDebugVariables::LDVImpl::emitDebugValues(VirtRegMap *VRM) {
   LLVM_DEBUG(dbgs() << "********** EMITTING LIVE DEBUG VARIABLES **********\n");
   if (!MF)
     return;
@@ -1956,13 +2004,15 @@ void LDVImpl::emitDebugValues(VirtRegMap *VRM) {
 }
 
 void LiveDebugVariables::emitDebugValues(VirtRegMap *VRM) {
-  if (pImpl)
-    static_cast<LDVImpl*>(pImpl)->emitDebugValues(VRM);
+  if (PImpl)
+    PImpl->emitDebugValues(VRM);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void LiveDebugVariables::dump() const {
-  if (pImpl)
-    static_cast<LDVImpl*>(pImpl)->print(dbgs());
-}
+LLVM_DUMP_METHOD void LiveDebugVariables::dump() const { print(dbgs()); }
 #endif
+
+void LiveDebugVariables::print(raw_ostream &OS) const {
+  if (PImpl)
+    PImpl->print(OS);
+}
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index a0b6bf445fa8a..f38527a3ce6a3 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -83,7 +83,7 @@ INITIALIZE_PASS_BEGIN(LiveIntervalsWrapperPass, "liveintervals",
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_END(LiveIntervalsWrapperPass, "liveintervals",
-                    "Live Interval Analysis", false, false)
+                    "Live Interval Analysis", false, true)
 
 bool LiveIntervalsWrapperPass::runOnMachineFunction(MachineFunction &MF) {
   LIS.Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
@@ -730,7 +730,12 @@ void LiveIntervals::addKillFlags(const VirtRegMap *VRM) {
     // Find the regunit intervals for the assigned register. They may overlap
     // the virtual register live range, cancelling any kills.
     RU.clear();
-    for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
+    LaneBitmask ArtificialLanes;
+    for (MCRegUnitMaskIterator UI(PhysReg, TRI); UI.isValid(); ++UI) {
+      auto [Unit, Bitmask] = *UI;
+      // Record lane mask for all artificial RegUnits for this physreg.
+      if (TRI->isArtificialRegUnit(Unit))
+        ArtificialLanes |= Bitmask;
       const LiveRange &RURange = getRegUnit(Unit);
       if (RURange.empty())
         continue;
@@ -782,7 +787,11 @@ void LiveIntervals::addKillFlags(const VirtRegMap *VRM) {
         LaneBitmask DefinedLanesMask;
         if (LI.hasSubRanges()) {
           // Compute a mask of lanes that are defined.
-          DefinedLanesMask = LaneBitmask::getNone();
+          // Artificial regunits are not independently allocatable so the
+          // register allocator cannot have used them to represent any other
+          // values. That's why we mark them as 'defined' here, as this
+          // otherwise prevents kill flags from being added.
+          DefinedLanesMask = ArtificialLanes;
           for (const LiveInterval::SubRange &SR : LI.subranges())
             for (const LiveRange::Segment &Segment : SR.segments) {
               if (Segment.start >= RI->end)
diff --git a/llvm/lib/CodeGen/LiveRegMatrix.cpp b/llvm/lib/CodeGen/LiveRegMatrix.cpp
index bc8c59381a40e..9744c47d5a851 100644
--- a/llvm/lib/CodeGen/LiveRegMatrix.cpp
+++ b/llvm/lib/CodeGen/LiveRegMatrix.cpp
@@ -41,7 +41,7 @@ INITIALIZE_PASS_BEGIN(LiveRegMatrixWrapperLegacy, "liveregmatrix",
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
 INITIALIZE_PASS_END(LiveRegMatrixWrapperLegacy, "liveregmatrix",
-                    "Live Register Matrix", false, false)
+                    "Live Register Matrix", false, true)
 
 void LiveRegMatrixWrapperLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
diff --git a/llvm/lib/CodeGen/LiveStacks.cpp b/llvm/lib/CodeGen/LiveStacks.cpp
index 6228a4dd2ad3b..d615caf48c0ad 100644
--- a/llvm/lib/CodeGen/LiveStacks.cpp
+++ b/llvm/lib/CodeGen/LiveStacks.cpp
@@ -15,20 +15,21 @@
 #include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/Function.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "livestacks"
 
-char LiveStacks::ID = 0;
-INITIALIZE_PASS_BEGIN(LiveStacks, DEBUG_TYPE,
-                "Live Stack Slot Analysis", false, false)
+char LiveStacksWrapperLegacy::ID = 0;
+INITIALIZE_PASS_BEGIN(LiveStacksWrapperLegacy, DEBUG_TYPE,
+                      "Live Stack Slot Analysis", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
-INITIALIZE_PASS_END(LiveStacks, DEBUG_TYPE,
-                "Live Stack Slot Analysis", false, false)
+INITIALIZE_PASS_END(LiveStacksWrapperLegacy, DEBUG_TYPE,
+                    "Live Stack Slot Analysis", false, true)
 
-char &llvm::LiveStacksID = LiveStacks::ID;
+char &llvm::LiveStacksID = LiveStacksWrapperLegacy::ID;
 
-void LiveStacks::getAnalysisUsage(AnalysisUsage &AU) const {
+void LiveStacksWrapperLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
   AU.addPreserved<SlotIndexesWrapperPass>();
   AU.addRequiredTransitive<SlotIndexesWrapperPass>();
@@ -42,11 +43,10 @@ void LiveStacks::releaseMemory() {
   S2RCMap.clear();
 }
 
-bool LiveStacks::runOnMachineFunction(MachineFunction &MF) {
+void LiveStacks::init(MachineFunction &MF) {
   TRI = MF.getSubtarget().getRegisterInfo();
   // FIXME: No analysis is being done right now. We are relying on the
   // register allocators to provide the information.
-  return false;
 }
 
 LiveInterval &
@@ -68,6 +68,33 @@ LiveStacks::getOrCreateInterval(int Slot, const TargetRegisterClass *RC) {
   return I->second;
 }
 
+AnalysisKey LiveStacksAnalysis::Key;
+
+LiveStacks LiveStacksAnalysis::run(MachineFunction &MF,
+                                   MachineFunctionAnalysisManager &) {
+  LiveStacks Impl;
+  Impl.init(MF);
+  return Impl;
+}
+PreservedAnalyses
+LiveStacksPrinterPass::run(MachineFunction &MF,
+                           MachineFunctionAnalysisManager &AM) {
+  AM.getResult<LiveStacksAnalysis>(MF).print(OS, MF.getFunction().getParent());
+  return PreservedAnalyses::all();
+}
+
+bool LiveStacksWrapperLegacy::runOnMachineFunction(MachineFunction &MF) {
+  Impl = LiveStacks();
+  Impl.init(MF);
+  return false;
+}
+
+void LiveStacksWrapperLegacy::releaseMemory() { Impl = LiveStacks(); }
+
+void LiveStacksWrapperLegacy::print(raw_ostream &OS, const Module *) const {
+  Impl.print(OS);
+}
+
 /// print - Implement the dump method.
 void LiveStacks::print(raw_ostream &OS, const Module*) const {
 
diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
index d099544c2a491..03f015f8c9e32 100644
--- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
@@ -559,14 +559,13 @@ class DevelopmentModeEvictionAdvisorAnalysis final
 
 float MLEvictAdvisor::getInitialQueueSize(const MachineFunction &MF) {
   auto &MRI = MF.getRegInfo();
-  float Ret = 0.0;
+  unsigned NumUsedRegs = 0;
   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
     Register Reg = Register::index2VirtReg(I);
-    if (MRI.reg_nodbg_empty(Reg))
-      continue;
-    ++Ret;
+    if (!MRI.reg_nodbg_empty(Reg))
+      ++NumUsedRegs;
   }
-  return Ret;
+  return static_cast<float>(NumUsedRegs);
 }
 
 MLEvictAdvisor::MLEvictAdvisor(const MachineFunction &MF, const RAGreedy &RA,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index d9e5e9d9d1e41..3a9bdde28a2e7 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -909,7 +909,11 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << printJumpTableEntryReference(getIndex());
     break;
   case MachineOperand::MO_GlobalAddress:
-    getGlobal()->printAsOperand(OS, /*PrintType=*/false, MST);
+    if (auto *GV = getGlobal())
+      GV->printAsOperand(OS, /*PrintType=*/false, MST);
+    else // Invalid, but may appear in debugging scenarios.
+      OS << "globaladdress(null)";
+
     printOperandOffset(OS, getOffset());
     break;
   case MachineOperand::MO_ExternalSymbol: {
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index fcedb302d228c..6f636a161f500 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -635,7 +635,13 @@ const MCPhysReg *MachineRegisterInfo::getCalleeSavedRegs() const {
   if (IsUpdatedCSRsInitialized)
     return UpdatedCSRs.data();
 
-  return getTargetRegisterInfo()->getCalleeSavedRegs(MF);
+  const MCPhysReg *Regs = getTargetRegisterInfo()->getCalleeSavedRegs(MF);
+
+  for (unsigned I = 0; Regs[I]; ++I)
+    if (MF->getSubtarget().isRegisterReservedByUser(Regs[I]))
+      MF->getRegInfo().disableCalleeSavedRegister(Regs[I]);
+
+  return Regs;
 }
 
 void MachineRegisterInfo::setCalleeSavedRegs(ArrayRef<MCPhysReg> CSRs) {
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index d65db91ac1ebd..1722bdda99e4a 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1947,6 +1947,9 @@ void BaseMemOpClusterMutation::collectMemOpRecords(
     LocationSize Width = 0;
     if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
                                            OffsetIsScalable, Width, TRI)) {
+      if (!Width.hasValue())
+        continue;
+
       MemOpRecords.push_back(
           MemOpInfo(&SU, BaseOps, Offset, OffsetIsScalable, Width));
 
@@ -3955,9 +3958,12 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand,
     return TryCand.Reason != NoCand;
 
   // Keep clustered nodes together.
-  if (tryGreater(TryCand.SU == DAG->getNextClusterSucc(),
-                 Cand.SU == DAG->getNextClusterSucc(),
-                 TryCand, Cand, Cluster))
+  const SUnit *CandNextClusterSU =
+      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  const SUnit *TryCandNextClusterSU =
+      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
+                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
     return TryCand.Reason != NoCand;
 
   // Avoid critical resource consumption and balance the schedule.
@@ -3969,9 +3975,13 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand,
                  TryCand, Cand, ResourceDemand))
     return TryCand.Reason != NoCand;
 
-  // Avoid serializing long latency dependence chains.
-  if (Cand.Policy.ReduceLatency && tryLatency(TryCand, Cand, Top)) {
-    return TryCand.Reason != NoCand;
+  // We only compare a subset of features when comparing nodes between
+  // Top and Bottom boundary.
+  if (Cand.AtTop == TryCand.AtTop) {
+    // Avoid serializing long latency dependence chains.
+    if (Cand.Policy.ReduceLatency &&
+        tryLatency(TryCand, Cand, Cand.AtTop ? Top : Bot))
+      return TryCand.Reason != NoCand;
   }
 
   // Fall through to original instruction order.
diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp
index facda7a59e2f8..5ab589acee413 100644
--- a/llvm/lib/CodeGen/MachineStableHash.cpp
+++ b/llvm/lib/CodeGen/MachineStableHash.cpp
@@ -27,6 +27,8 @@
 #include "llvm/CodeGen/Register.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/StructuralHash.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -93,13 +95,19 @@ stable_hash llvm::stableHashValue(const MachineOperand &MO) {
     return 0;
   case MachineOperand::MO_GlobalAddress: {
     const GlobalValue *GV = MO.getGlobal();
-    if (!GV->hasName()) {
-      ++StableHashBailingGlobalAddress;
-      return 0;
+    stable_hash GVHash = 0;
+    if (auto *GVar = dyn_cast<GlobalVariable>(GV))
+      GVHash = StructuralHash(*GVar);
+    if (!GVHash) {
+      if (!GV->hasName()) {
+        ++StableHashBailingGlobalAddress;
+        return 0;
+      }
+      GVHash = stable_hash_name(GV->getName());
     }
-    auto Name = GV->getName();
-    return stable_hash_combine(MO.getType(), MO.getTargetFlags(),
-                               stable_hash_name(Name), MO.getOffset());
+
+    return stable_hash_combine(MO.getType(), MO.getTargetFlags(), GVHash,
+                               MO.getOffset());
   }
 
   case MachineOperand::MO_TargetIndex: {
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index b08a93ae9a6d5..fb4d96fdad0a5 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -369,7 +369,7 @@ struct MachineVerifierLegacyPass : public MachineFunctionPass {
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addUsedIfAvailable<LiveStacks>();
+    AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
     AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
     AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
     AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
@@ -491,7 +491,8 @@ bool MachineVerifier::verify(const MachineFunction &MF) {
     auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>();
     if (!LiveInts)
       LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr;
-    LiveStks = PASS->getAnalysisIfAvailable<LiveStacks>();
+    auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
+    LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
     auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
     Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
   }
@@ -1585,6 +1586,31 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
 
     break;
   }
+  case TargetOpcode::G_ABDS:
+  case TargetOpcode::G_ABDU: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
+    LLT SrcTy2 = MRI->getType(MI->getOperand(2).getReg());
+
+    if ((DstTy.isVector() != SrcTy.isVector()) ||
+        (DstTy.isVector() &&
+         DstTy.getElementCount() != SrcTy.getElementCount())) {
+      report("Generic vector abds/abdu must preserve number of lanes", MI);
+      break;
+    }
+
+    if (SrcTy != SrcTy2) {
+      report("Generic abds/abdu must have same input types", MI);
+      break;
+    }
+
+    if (DstTy != SrcTy) {
+      report("Generic abds/abdu must have same input and output types", MI);
+      break;
+    }
+
+    break;
+  }
   case TargetOpcode::G_SCMP:
   case TargetOpcode::G_UCMP: {
     LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index 0e8220ec6251c..79b0fa672cc69 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -16,7 +16,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "reaching-deps-analysis"
+#define DEBUG_TYPE "reaching-defs-analysis"
 
 char ReachingDefAnalysis::ID = 0;
 INITIALIZE_PASS(ReachingDefAnalysis, DEBUG_TYPE, "ReachingDefAnalysis", false,
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index 55d806e768b91..c05aa1e40e477 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -130,12 +130,12 @@ char &llvm::RABasicID = RABasic::ID;
 
 INITIALIZE_PASS_BEGIN(RABasic, "regallocbasic", "Basic Register Allocator",
                       false, false)
-INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
+INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
 INITIALIZE_PASS_DEPENDENCY(MachineScheduler)
-INITIALIZE_PASS_DEPENDENCY(LiveStacks)
+INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
@@ -180,10 +180,10 @@ void RABasic::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<LiveIntervalsWrapperPass>();
   AU.addPreserved<LiveIntervalsWrapperPass>();
   AU.addPreserved<SlotIndexesWrapperPass>();
-  AU.addRequired<LiveDebugVariables>();
-  AU.addPreserved<LiveDebugVariables>();
-  AU.addRequired<LiveStacks>();
-  AU.addPreserved<LiveStacks>();
+  AU.addRequired<LiveDebugVariablesWrapperLegacy>();
+  AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+  AU.addRequired<LiveStacksWrapperLegacy>();
+  AU.addPreserved<LiveStacksWrapperLegacy>();
   AU.addRequired<ProfileSummaryInfoWrapperPass>();
   AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
   AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 3542bfe18af46..cb29218e966e0 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -17,7 +17,6 @@
 #include "RegAllocBase.h"
 #include "RegAllocEvictionAdvisor.h"
 #include "RegAllocPriorityAdvisor.h"
-#include "SpillPlacement.h"
 #include "SplitKit.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
@@ -50,6 +49,7 @@
 #include "llvm/CodeGen/RegAllocRegistry.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/SpillPlacement.h"
 #include "llvm/CodeGen/Spiller.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -151,18 +151,18 @@ char &llvm::RAGreedyID = RAGreedy::ID;
 
 INITIALIZE_PASS_BEGIN(RAGreedy, "greedy",
                 "Greedy Register Allocator", false, false)
-INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
+INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
 INITIALIZE_PASS_DEPENDENCY(MachineScheduler)
-INITIALIZE_PASS_DEPENDENCY(LiveStacks)
+INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(EdgeBundlesWrapperLegacy)
-INITIALIZE_PASS_DEPENDENCY(SpillPlacement)
+INITIALIZE_PASS_DEPENDENCY(SpillPlacementWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
 INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysis)
 INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysis)
@@ -204,10 +204,10 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<LiveIntervalsWrapperPass>();
   AU.addRequired<SlotIndexesWrapperPass>();
   AU.addPreserved<SlotIndexesWrapperPass>();
-  AU.addRequired<LiveDebugVariables>();
-  AU.addPreserved<LiveDebugVariables>();
-  AU.addRequired<LiveStacks>();
-  AU.addPreserved<LiveStacks>();
+  AU.addRequired<LiveDebugVariablesWrapperLegacy>();
+  AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+  AU.addRequired<LiveStacksWrapperLegacy>();
+  AU.addPreserved<LiveStacksWrapperLegacy>();
   AU.addRequired<MachineDominatorTreeWrapperPass>();
   AU.addPreserved<MachineDominatorTreeWrapperPass>();
   AU.addRequired<MachineLoopInfoWrapperPass>();
@@ -217,7 +217,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<LiveRegMatrixWrapperLegacy>();
   AU.addPreserved<LiveRegMatrixWrapperLegacy>();
   AU.addRequired<EdgeBundlesWrapperLegacy>();
-  AU.addRequired<SpillPlacement>();
+  AU.addRequired<SpillPlacementWrapperLegacy>();
   AU.addRequired<MachineOptimizationRemarkEmitterPass>();
   AU.addRequired<RegAllocEvictionAdvisorAnalysis>();
   AU.addRequired<RegAllocPriorityAdvisorAnalysis>();
@@ -2731,8 +2731,8 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
   ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
   Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
   Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
-  SpillPlacer = &getAnalysis<SpillPlacement>();
-  DebugVars = &getAnalysis<LiveDebugVariables>();
+  SpillPlacer = &getAnalysis<SpillPlacementWrapperLegacy>().getResult();
+  DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
 
   initializeCSRCost();
 
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 2e7608a53e9ce..594c481826cf0 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -16,7 +16,6 @@
 #include "RegAllocBase.h"
 #include "RegAllocEvictionAdvisor.h"
 #include "RegAllocPriorityAdvisor.h"
-#include "SpillPlacement.h"
 #include "SplitKit.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
@@ -25,11 +24,13 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveRangeEdit.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/SpillPlacement.h"
 #include "llvm/CodeGen/Spiller.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include <algorithm>
@@ -42,7 +43,7 @@ namespace llvm {
 class AllocationOrder;
 class AnalysisUsage;
 class EdgeBundles;
-class LiveDebugVariables;
+class LiveDebugVariablesWrapperLegacy;
 class LiveIntervals;
 class LiveRegMatrix;
 class MachineBasicBlock;
diff --git a/llvm/lib/CodeGen/RegAllocPBQP.cpp b/llvm/lib/CodeGen/RegAllocPBQP.cpp
index 261e93add7d88..696c312e4ba00 100644
--- a/llvm/lib/CodeGen/RegAllocPBQP.cpp
+++ b/llvm/lib/CodeGen/RegAllocPBQP.cpp
@@ -121,7 +121,7 @@ class RegAllocPBQP : public MachineFunctionPass {
       : MachineFunctionPass(ID), customPassID(cPassID) {
     initializeSlotIndexesWrapperPassPass(*PassRegistry::getPassRegistry());
     initializeLiveIntervalsWrapperPassPass(*PassRegistry::getPassRegistry());
-    initializeLiveStacksPass(*PassRegistry::getPassRegistry());
+    initializeLiveStacksWrapperLegacyPass(*PassRegistry::getPassRegistry());
     initializeVirtRegMapWrapperLegacyPass(*PassRegistry::getPassRegistry());
   }
 
@@ -550,8 +550,8 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const {
   //au.addRequiredID(SplitCriticalEdgesID);
   if (customPassID)
     au.addRequiredID(*customPassID);
-  au.addRequired<LiveStacks>();
-  au.addPreserved<LiveStacks>();
+  au.addRequired<LiveStacksWrapperLegacy>();
+  au.addPreserved<LiveStacksWrapperLegacy>();
   au.addRequired<MachineBlockFrequencyInfoWrapperPass>();
   au.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
   au.addRequired<MachineLoopInfoWrapperPass>();
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 2e1f498c090d1..20ad6445344d8 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -1374,6 +1374,27 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
   MachineInstr &NewMI = *std::prev(MII);
   NewMI.setDebugLoc(DL);
 
+  // In a situation like the following:
+  //
+  //    undef %2.subreg:reg = INST %1:reg         ; DefMI (rematerializable),
+  //                                              ; DefSubIdx = subreg
+  //    %3:reg = COPY %2                          ; SrcIdx = DstIdx = 0
+  //    .... = SOMEINSTR %3:reg
+  //
+  // there are no subranges for %3 so after rematerialization we need
+  // to explicitly create them. Undefined subranges are removed later on.
+  if (DstReg.isVirtual() && DefSubIdx && !CP.getSrcIdx() && !CP.getDstIdx() &&
+      MRI->shouldTrackSubRegLiveness(DstReg)) {
+    LiveInterval &DstInt = LIS->getInterval(DstReg);
+    if (!DstInt.hasSubRanges()) {
+      LaneBitmask FullMask = MRI->getMaxLaneMaskForVReg(DstReg);
+      LaneBitmask UsedLanes = TRI->getSubRegIndexLaneMask(DefSubIdx);
+      LaneBitmask UnusedLanes = FullMask & ~UsedLanes;
+      DstInt.createSubRangeFrom(LIS->getVNInfoAllocator(), UsedLanes, DstInt);
+      DstInt.createSubRangeFrom(LIS->getVNInfoAllocator(), UnusedLanes, DstInt);
+    }
+  }
+
   // In a situation like the following:
   //     %0:subreg = instr              ; DefMI, subreg = DstIdx
   //     %1        = copy %0:subreg ; CopyMI, SrcIdx = 0
@@ -1486,6 +1507,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
         NewRC = TRI->getCommonSubClass(NewRC, DefRC);
       assert(NewRC && "subreg chosen for remat incompatible with instruction");
     }
+
     // Remap subranges to new lanemask and change register class.
     LiveInterval &DstInt = LIS->getInterval(DstReg);
     for (LiveInterval::SubRange &SR : DstInt.subranges()) {
@@ -1820,9 +1842,12 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg,
 
   if (DstInt && DstInt->hasSubRanges() && DstReg != SrcReg) {
     for (MachineOperand &MO : MRI->reg_operands(DstReg)) {
+      if (MO.isUndef())
+        continue;
       unsigned SubReg = MO.getSubReg();
-      if (SubReg == 0 || MO.isUndef())
+      if (SubReg == 0 && MO.isDef())
         continue;
+
       MachineInstr &MI = *MO.getParent();
       if (MI.isDebugInstr())
         continue;
@@ -1866,7 +1891,7 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg,
 
       // A subreg use of a partially undef (super) register may be a complete
       // undef use now and then has to be marked that way.
-      if (MO.isUse() && !DstIsPhys) {
+      if (MO.isUse() && !MO.isUndef() && !DstIsPhys) {
         unsigned SubUseIdx = TRI->composeSubRegIndices(SubIdx, MO.getSubReg());
         if (SubUseIdx != 0 && MRI->shouldTrackSubRegLiveness(DstReg)) {
           if (!DstInt->hasSubRanges()) {
diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp
index 81796fcf2842a..2ec5af2d864f7 100644
--- a/llvm/lib/CodeGen/SelectOptimize.cpp
+++ b/llvm/lib/CodeGen/SelectOptimize.cpp
@@ -127,77 +127,26 @@ class SelectOptimizeImpl {
   /// act like selects. For example Or(Zext(icmp), X) can be treated like
   /// select(icmp, X|1, X).
   class SelectLike {
-    SelectLike(Instruction *I) : I(I) {}
-
     /// The select (/or) instruction.
     Instruction *I;
     /// Whether this select is inverted, "not(cond), FalseVal, TrueVal", as
     /// opposed to the original condition.
     bool Inverted = false;
 
-  public:
-    /// Match a select or select-like instruction, returning a SelectLike.
-    static SelectLike match(Instruction *I) {
-      // Select instruction are what we are usually looking for.
-      if (isa<SelectInst>(I))
-        return SelectLike(I);
-
-      // An Or(zext(i1 X), Y) can also be treated like a select, with condition
-      // C and values Y|1 and Y.
-      Value *X;
-      if (PatternMatch::match(
-              I, m_c_Or(m_OneUse(m_ZExt(m_Value(X))), m_Value())) &&
-          X->getType()->isIntegerTy(1))
-        return SelectLike(I);
-
-      return SelectLike(nullptr);
-    }
-
-    bool isValid() { return I; }
-    operator bool() { return isValid(); }
+    /// The index of the operand that depends on condition. Only for select-like
+    /// instruction such as Or/Add.
+    unsigned CondIdx;
 
-    /// Invert the select by inverting the condition and switching the operands.
-    void setInverted() {
-      assert(!Inverted && "Trying to invert an inverted SelectLike");
-      assert(isa<Instruction>(getCondition()) &&
-             cast<Instruction>(getCondition())->getOpcode() ==
-                 Instruction::Xor);
-      Inverted = true;
-    }
-    bool isInverted() const { return Inverted; }
+  public:
+    SelectLike(Instruction *I, bool Inverted = false, unsigned CondIdx = 0)
+        : I(I), Inverted(Inverted), CondIdx(CondIdx) {}
 
     Instruction *getI() { return I; }
     const Instruction *getI() const { return I; }
 
     Type *getType() const { return I->getType(); }
 
-    Value *getNonInvertedCondition() const {
-      if (auto *Sel = dyn_cast<SelectInst>(I))
-        return Sel->getCondition();
-      // Or(zext) case
-      if (auto *BO = dyn_cast<BinaryOperator>(I)) {
-        Value *X;
-        if (PatternMatch::match(BO->getOperand(0),
-                                m_OneUse(m_ZExt(m_Value(X)))))
-          return X;
-        if (PatternMatch::match(BO->getOperand(1),
-                                m_OneUse(m_ZExt(m_Value(X)))))
-          return X;
-      }
-
-      llvm_unreachable("Unhandled case in getCondition");
-    }
-
-    /// Return the condition for the SelectLike instruction. For example the
-    /// condition of a select or c in `or(zext(c), x)`
-    Value *getCondition() const {
-      Value *CC = getNonInvertedCondition();
-      // For inverted conditions the CC is checked when created to be a not
-      // (xor) instruction.
-      if (Inverted)
-        return cast<Instruction>(CC)->getOperand(0);
-      return CC;
-    }
+    unsigned getConditionOpIndex() { return CondIdx; };
 
     /// Return the true value for the SelectLike instruction. Note this may not
     /// exist for all SelectLike instructions. For example, for `or(zext(c), x)`
@@ -224,74 +173,56 @@ class SelectOptimizeImpl {
         return getTrueValue(/*HonorInverts=*/false);
       if (auto *Sel = dyn_cast<SelectInst>(I))
         return Sel->getFalseValue();
-      // Or(zext) case - return the operand which is not the zext.
-      if (auto *BO = dyn_cast<BinaryOperator>(I)) {
-        Value *X;
-        if (PatternMatch::match(BO->getOperand(0),
-                                m_OneUse(m_ZExt(m_Value(X)))))
-          return BO->getOperand(1);
-        if (PatternMatch::match(BO->getOperand(1),
-                                m_OneUse(m_ZExt(m_Value(X)))))
-          return BO->getOperand(0);
-      }
+      // We are on the branch where the condition is zero, which means BinOp
+      // does not perform any computation, and we can simply return the operand
+      // that is not related to the condition
+      if (auto *BO = dyn_cast<BinaryOperator>(I))
+        return BO->getOperand(1 - CondIdx);
 
       llvm_unreachable("Unhandled case in getFalseValue");
     }
 
-    /// Return the NonPredCost cost of the true op, given the costs in
-    /// InstCostMap. This may need to be generated for select-like instructions.
-    Scaled64 getTrueOpCost(DenseMap<const Instruction *, CostInfo> &InstCostMap,
-                           const TargetTransformInfo *TTI) {
-      if (isa<SelectInst>(I))
-        if (auto *I = dyn_cast<Instruction>(getTrueValue())) {
-          auto It = InstCostMap.find(I);
-          return It != InstCostMap.end() ? It->second.NonPredCost
-                                         : Scaled64::getZero();
-        }
-
-      // Or case - add the cost of an extra Or to the cost of the False case.
-      if (isa<BinaryOperator>(I))
-        if (auto I = dyn_cast<Instruction>(getFalseValue())) {
-          auto It = InstCostMap.find(I);
-          if (It != InstCostMap.end()) {
-            InstructionCost OrCost = TTI->getArithmeticInstrCost(
-                Instruction::Or, I->getType(), TargetTransformInfo::TCK_Latency,
-                {TargetTransformInfo::OK_AnyValue,
-                 TargetTransformInfo::OP_None},
-                {TTI::OK_UniformConstantValue, TTI::OP_PowerOf2});
-            return It->second.NonPredCost + Scaled64::get(*OrCost.getValue());
-          }
-        }
-
-      return Scaled64::getZero();
-    }
-
-    /// Return the NonPredCost cost of the false op, given the costs in
-    /// InstCostMap. This may need to be generated for select-like instructions.
-    Scaled64
-    getFalseOpCost(DenseMap<const Instruction *, CostInfo> &InstCostMap,
-                   const TargetTransformInfo *TTI) {
-      if (isa<SelectInst>(I))
-        if (auto *I = dyn_cast<Instruction>(getFalseValue())) {
-          auto It = InstCostMap.find(I);
+    /// Return the NonPredCost cost of the op on \p isTrue branch, given the
+    /// costs in \p InstCostMap. This may need to be generated for select-like
+    /// instructions.
+    Scaled64 getOpCostOnBranch(
+        bool IsTrue, const DenseMap<const Instruction *, CostInfo> &InstCostMap,
+        const TargetTransformInfo *TTI) {
+      auto *V = IsTrue ? getTrueValue() : getFalseValue();
+      if (V) {
+        if (auto *IV = dyn_cast<Instruction>(V)) {
+          auto It = InstCostMap.find(IV);
           return It != InstCostMap.end() ? It->second.NonPredCost
                                          : Scaled64::getZero();
         }
-
-      // Or case - return the cost of the false case
-      if (isa<BinaryOperator>(I))
-        if (auto I = dyn_cast<Instruction>(getFalseValue()))
-          if (auto It = InstCostMap.find(I); It != InstCostMap.end())
-            return It->second.NonPredCost;
-
-      return Scaled64::getZero();
+        return Scaled64::getZero();
+      }
+      // If getTrue(False)Value() return nullptr, it means we are dealing with
+      // select-like instructions on the branch where the actual computation is
+      // happening. In that case the cost is equal to the cost of computation +
+      // cost of non-dependant on condition operand
+      InstructionCost Cost = TTI->getArithmeticInstrCost(
+          getI()->getOpcode(), I->getType(), TargetTransformInfo::TCK_Latency,
+          {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+          {TTI::OK_UniformConstantValue, TTI::OP_PowerOf2});
+      auto TotalCost = Scaled64::get(*Cost.getValue());
+      if (auto *OpI = dyn_cast<Instruction>(I->getOperand(1 - CondIdx))) {
+        auto It = InstCostMap.find(OpI);
+        if (It != InstCostMap.end())
+          TotalCost += It->second.NonPredCost;
+      }
+      return TotalCost;
     }
   };
 
 private:
-  // Select groups consist of consecutive select instructions with the same
-  // condition.
-  using SelectGroup = SmallVector<SelectLike, 2>;
+  // Select groups consist of consecutive select-like instructions with the same
+  // condition. Between select-likes could be any number of auxiliary
+  // instructions related to the condition like not, zext
+  struct SelectGroup {
+    Value *Condition;
+    SmallVector<SelectLike, 2> Selects;
+  };
   using SelectGroups = SmallVector<SelectGroup, 2>;
 
   // Converts select instructions of a function to conditional jumps when deemed
@@ -351,6 +282,11 @@ class SelectOptimizeImpl {
   SmallDenseMap<const Instruction *, SelectLike, 2>
   getSImap(const SelectGroups &SIGroups);
 
+  // Returns a map from select-like instructions to the corresponding select
+  // group.
+  SmallDenseMap<const Instruction *, const SelectGroup *, 2>
+  getSGmap(const SelectGroups &SIGroups);
+
   // Returns the latency cost of a given instruction.
   std::optional<uint64_t> computeInstCost(const Instruction *I);
 
@@ -529,34 +465,47 @@ void SelectOptimizeImpl::optimizeSelectsInnerLoops(Function &F,
   }
 }
 
-/// If \p isTrue is true, return the true value of \p SI, otherwise return
-/// false value of \p SI. If the true/false value of \p SI is defined by any
-/// select instructions in \p Selects, look through the defining select
-/// instruction until the true/false value is not defined in \p Selects.
-static Value *
-getTrueOrFalseValue(SelectOptimizeImpl::SelectLike SI, bool isTrue,
-                    const SmallPtrSet<const Instruction *, 2> &Selects,
-                    IRBuilder<> &IB) {
-  Value *V = nullptr;
-  for (SelectInst *DefSI = dyn_cast<SelectInst>(SI.getI());
-       DefSI != nullptr && Selects.count(DefSI);
-       DefSI = dyn_cast<SelectInst>(V)) {
-    if (DefSI->getCondition() == SI.getCondition())
-      V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
-    else // Handle inverted SI
-      V = (!isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
+/// Returns optimised value on \p IsTrue branch. For SelectInst that would be
+/// either True or False value. For (BinaryOperator) instructions, where the
+/// condition may be skipped, the operation will use a non-conditional operand.
+/// For example, for `or(V,zext(cond))` this function would return V.
+/// However, if the conditional operand on \p IsTrue branch matters, we create a
+/// clone of instruction at the end of that branch \p B and replace the
+/// condition operand with a constant.
+///
+/// Also /p OptSelects contains previously optimised select-like instructions.
+/// If the current value uses one of the optimised values, we can optimise it
+/// further by replacing it with the corresponding value on the given branch
+static Value *getTrueOrFalseValue(
+    SelectOptimizeImpl::SelectLike &SI, bool isTrue,
+    SmallDenseMap<Instruction *, std::pair<Value *, Value *>, 2> &OptSelects,
+    BasicBlock *B) {
+  Value *V = isTrue ? SI.getTrueValue() : SI.getFalseValue();
+  if (V) {
+    auto *IV = dyn_cast<Instruction>(V);
+    if (IV && OptSelects.count(IV))
+      return isTrue ? OptSelects[IV].first : OptSelects[IV].second;
+    return V;
   }
 
-  if (isa<BinaryOperator>(SI.getI())) {
-    assert(SI.getI()->getOpcode() == Instruction::Or &&
-           "Only currently handling Or instructions.");
-    V = SI.getFalseValue();
-    if (isTrue)
-      V = IB.CreateOr(V, ConstantInt::get(V->getType(), 1));
+  auto *BO = cast<BinaryOperator>(SI.getI());
+  assert((BO->getOpcode() == Instruction::Add ||
+          BO->getOpcode() == Instruction::Or ||
+          BO->getOpcode() == Instruction::Sub) &&
+         "Only currently handling Add, Or and Sub binary operators.");
+
+  auto *CBO = BO->clone();
+  auto CondIdx = SI.getConditionOpIndex();
+  CBO->setOperand(CondIdx, ConstantInt::get(CBO->getType(), 1));
+
+  unsigned OtherIdx = 1 - CondIdx;
+  if (auto *IV = dyn_cast<Instruction>(CBO->getOperand(OtherIdx))) {
+    if (OptSelects.count(IV))
+      CBO->setOperand(OtherIdx,
+                      isTrue ? OptSelects[IV].first : OptSelects[IV].second);
   }
-
-  assert(V && "Failed to get select true/false value");
-  return V;
+  CBO->insertBefore(B->getTerminator());
+  return CBO;
 }
 
 void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
@@ -602,7 +551,9 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
     SmallVector<std::stack<Instruction *>, 2> TrueSlices, FalseSlices;
     typedef std::stack<Instruction *>::size_type StackSizeType;
     StackSizeType maxTrueSliceLen = 0, maxFalseSliceLen = 0;
-    for (SelectLike SI : ASI) {
+    for (SelectLike &SI : ASI.Selects) {
+      if (!isa<SelectInst>(SI.getI()))
+        continue;
       // For each select, compute the sinkable dependence chains of the true and
       // false operands.
       if (auto *TI = dyn_cast_or_null<Instruction>(SI.getTrueValue())) {
@@ -649,8 +600,8 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
     }
 
     // We split the block containing the select(s) into two blocks.
-    SelectLike SI = ASI.front();
-    SelectLike LastSI = ASI.back();
+    SelectLike &SI = ASI.Selects.front();
+    SelectLike &LastSI = ASI.Selects.back();
     BasicBlock *StartBlock = SI.getI()->getParent();
     BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(LastSI.getI()));
     // With RemoveDIs turned off, SplitPt can be a dbg.* intrinsic. With
@@ -664,19 +615,21 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
     // Delete the unconditional branch that was just created by the split.
     StartBlock->getTerminator()->eraseFromParent();
 
-    // Move any debug/pseudo instructions and not's that were in-between the
+    // Move any debug/pseudo and auxiliary instructions that were in-between the
     // select group to the newly-created end block.
     SmallVector<Instruction *, 2> SinkInstrs;
     auto DIt = SI.getI()->getIterator();
+    auto NIt = ASI.Selects.begin();
     while (&*DIt != LastSI.getI()) {
-      if (DIt->isDebugOrPseudoInst())
-        SinkInstrs.push_back(&*DIt);
-      if (match(&*DIt, m_Not(m_Specific(SI.getCondition()))))
+      if (NIt != ASI.Selects.end() && &*DIt == NIt->getI())
+        ++NIt;
+      else
         SinkInstrs.push_back(&*DIt);
       DIt++;
     }
+    auto InsertionPoint = EndBlock->getFirstInsertionPt();
     for (auto *DI : SinkInstrs)
-      DI->moveBeforePreserving(&*EndBlock->getFirstInsertionPt());
+      DI->moveBeforePreserving(&*InsertionPoint);
 
     // Duplicate implementation for DbgRecords, the non-instruction debug-info
     // format. Helper lambda for moving DbgRecords to the end block.
@@ -700,7 +653,15 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
     // At least one will become an actual new basic block.
     BasicBlock *TrueBlock = nullptr, *FalseBlock = nullptr;
     BranchInst *TrueBranch = nullptr, *FalseBranch = nullptr;
-    if (!TrueSlicesInterleaved.empty()) {
+    // Checks if select-like instruction would materialise on the given branch
+    auto HasSelectLike = [](SelectGroup &SG, bool IsTrue) {
+      for (auto &SL : SG.Selects) {
+        if ((IsTrue ? SL.getTrueValue() : SL.getFalseValue()) == nullptr)
+          return true;
+      }
+      return false;
+    };
+    if (!TrueSlicesInterleaved.empty() || HasSelectLike(ASI, true)) {
       TrueBlock = BasicBlock::Create(EndBlock->getContext(), "select.true.sink",
                                      EndBlock->getParent(), EndBlock);
       TrueBranch = BranchInst::Create(EndBlock, TrueBlock);
@@ -708,7 +669,7 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
       for (Instruction *TrueInst : TrueSlicesInterleaved)
         TrueInst->moveBefore(TrueBranch);
     }
-    if (!FalseSlicesInterleaved.empty()) {
+    if (!FalseSlicesInterleaved.empty() || HasSelectLike(ASI, false)) {
       FalseBlock =
           BasicBlock::Create(EndBlock->getContext(), "select.false.sink",
                              EndBlock->getParent(), EndBlock);
@@ -748,93 +709,187 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
       FT = FalseBlock;
     }
     IRBuilder<> IB(SI.getI());
-    auto *CondFr = IB.CreateFreeze(SI.getCondition(),
-                                   SI.getCondition()->getName() + ".frozen");
+    auto *CondFr =
+        IB.CreateFreeze(ASI.Condition, ASI.Condition->getName() + ".frozen");
 
-    SmallPtrSet<const Instruction *, 2> INS;
-    for (auto SI : ASI)
-      INS.insert(SI.getI());
+    SmallDenseMap<Instruction *, std::pair<Value *, Value *>, 2> INS;
 
     // Use reverse iterator because later select may use the value of the
     // earlier select, and we need to propagate value through earlier select
     // to get the PHI operand.
-    for (auto It = ASI.rbegin(); It != ASI.rend(); ++It) {
-      SelectLike SI = *It;
+    InsertionPoint = EndBlock->begin();
+    for (SelectLike &SI : ASI.Selects) {
       // The select itself is replaced with a PHI Node.
       PHINode *PN = PHINode::Create(SI.getType(), 2, "");
-      PN->insertBefore(EndBlock->begin());
+      PN->insertBefore(InsertionPoint);
       PN->takeName(SI.getI());
-      PN->addIncoming(getTrueOrFalseValue(SI, true, INS, IB), TrueBlock);
-      PN->addIncoming(getTrueOrFalseValue(SI, false, INS, IB), FalseBlock);
-      PN->setDebugLoc(SI.getI()->getDebugLoc());
+      // Current instruction might be a condition of some other group, so we
+      // need to replace it there to avoid dangling pointer
+      if (PN->getType()->isIntegerTy(1)) {
+        for (auto &SG : ProfSIGroups) {
+          if (SG.Condition == SI.getI())
+            SG.Condition = PN;
+        }
+      }
       SI.getI()->replaceAllUsesWith(PN);
-      INS.erase(SI.getI());
+      auto *TV = getTrueOrFalseValue(SI, true, INS, TrueBlock);
+      auto *FV = getTrueOrFalseValue(SI, false, INS, FalseBlock);
+      INS[PN] = {TV, FV};
+      PN->addIncoming(TV, TrueBlock);
+      PN->addIncoming(FV, FalseBlock);
+      PN->setDebugLoc(SI.getI()->getDebugLoc());
       ++NumSelectsConverted;
     }
     IB.CreateCondBr(CondFr, TT, FT, SI.getI());
 
     // Remove the old select instructions, now that they are not longer used.
-    for (auto SI : ASI)
+    for (SelectLike &SI : ASI.Selects)
       SI.getI()->eraseFromParent();
   }
 }
 
 void SelectOptimizeImpl::collectSelectGroups(BasicBlock &BB,
                                              SelectGroups &SIGroups) {
+  // Represents something that can be considered as select instruction.
+  // Auxiliary instruction are instructions that depends on a condition and have
+  // zero or some constant value on True/False branch, such as:
+  // * ZExt(1bit)
+  // * Not(1bit)
+  struct SelectLikeInfo {
+    Value *Cond;
+    bool IsAuxiliary;
+    bool IsInverted;
+    unsigned ConditionIdx;
+  };
+
+  DenseMap<Value *, SelectLikeInfo> SelectInfo;
+
+  // Check if the instruction is SelectLike or might be part of SelectLike
+  // expression, put information into SelectInfo and return the iterator to the
+  // inserted position.
+  auto ProcessSelectInfo = [&SelectInfo](Instruction *I) {
+    Value *Cond;
+    if (match(I, m_OneUse(m_ZExt(m_Value(Cond)))) &&
+        Cond->getType()->isIntegerTy(1)) {
+      bool Inverted = match(Cond, m_Not(m_Value(Cond)));
+      return SelectInfo.insert({I, {Cond, true, Inverted, 0}}).first;
+    }
+
+    if (match(I, m_Not(m_Value(Cond)))) {
+      return SelectInfo.insert({I, {Cond, true, true, 0}}).first;
+    }
+
+    // Select instruction are what we are usually looking for.
+    if (match(I, m_Select(m_Value(Cond), m_Value(), m_Value()))) {
+      bool Inverted = match(Cond, m_Not(m_Value(Cond)));
+      return SelectInfo.insert({I, {Cond, false, Inverted, 0}}).first;
+    }
+
+    // An Or(zext(i1 X), Y) can also be treated like a select, with condition X
+    // and values Y|1 and Y.
+    if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+      switch (I->getOpcode()) {
+      case Instruction::Add:
+      case Instruction::Sub: {
+        Value *X;
+        if (!((PatternMatch::match(I->getOperand(0),
+                                   m_OneUse(m_ZExt(m_Value(X)))) ||
+               PatternMatch::match(I->getOperand(1),
+                                   m_OneUse(m_ZExt(m_Value(X))))) &&
+              X->getType()->isIntegerTy(1)))
+          return SelectInfo.end();
+        break;
+      }
+      case Instruction::Or:
+        if (BO->getType()->isIntegerTy(1) || BO->getOpcode() != Instruction::Or)
+          return SelectInfo.end();
+        break;
+      }
+
+      // Iterate through operands and find dependant on recognised sign
+      // extending auxiliary select-like instructions. The operand index does
+      // not matter for Add and Or. However, for Sub, we can only safely
+      // transform when the operand is second.
+      unsigned Idx = BO->getOpcode() == Instruction::Sub ? 1 : 0;
+      for (; Idx < 2; Idx++) {
+        auto *Op = BO->getOperand(Idx);
+        auto It = SelectInfo.find(Op);
+        if (It != SelectInfo.end() && It->second.IsAuxiliary) {
+          Cond = It->second.Cond;
+          bool Inverted = It->second.IsInverted;
+          return SelectInfo.insert({I, {Cond, false, Inverted, Idx}}).first;
+        }
+      }
+    }
+    return SelectInfo.end();
+  };
+
+  bool AlreadyProcessed = false;
   BasicBlock::iterator BBIt = BB.begin();
+  DenseMap<Value *, SelectLikeInfo>::iterator It;
   while (BBIt != BB.end()) {
     Instruction *I = &*BBIt++;
-    if (SelectLike SI = SelectLike::match(I)) {
-      if (!TTI->shouldTreatInstructionLikeSelect(I))
-        continue;
+    if (I->isDebugOrPseudoInst())
+      continue;
 
-      SelectGroup SIGroup;
-      SIGroup.push_back(SI);
-      while (BBIt != BB.end()) {
-        Instruction *NI = &*BBIt;
-        // Debug/pseudo instructions should be skipped and not prevent the
-        // formation of a select group.
-        if (NI->isDebugOrPseudoInst()) {
-          ++BBIt;
-          continue;
-        }
+    if (!AlreadyProcessed)
+      It = ProcessSelectInfo(I);
+    else
+      AlreadyProcessed = false;
 
-        // Skip not(select(..)), if the not is part of the same select group
-        if (match(NI, m_Not(m_Specific(SI.getCondition())))) {
-          ++BBIt;
-          continue;
-        }
+    if (It == SelectInfo.end() || It->second.IsAuxiliary)
+      continue;
+
+    if (!TTI->shouldTreatInstructionLikeSelect(I))
+      continue;
+
+    Value *Cond = It->second.Cond;
+    // Vector conditions are not supported.
+    if (!Cond->getType()->isIntegerTy(1))
+      continue;
+
+    SelectGroup SIGroup = {Cond, {}};
+    SIGroup.Selects.emplace_back(I, It->second.IsInverted,
+                                 It->second.ConditionIdx);
+
+    // If the select type is not supported, no point optimizing it.
+    // Instruction selection will take care of it.
+    if (!isSelectKindSupported(SIGroup.Selects.front()))
+      continue;
 
-        // We only allow selects in the same group, not other select-like
-        // instructions.
-        if (!isa<SelectInst>(NI))
-          break;
-
-        SelectLike NSI = SelectLike::match(NI);
-        if (NSI && SI.getCondition() == NSI.getCondition()) {
-          SIGroup.push_back(NSI);
-        } else if (NSI && match(NSI.getCondition(),
-                                m_Not(m_Specific(SI.getCondition())))) {
-          NSI.setInverted();
-          SIGroup.push_back(NSI);
-        } else
-          break;
+    while (BBIt != BB.end()) {
+      Instruction *NI = &*BBIt;
+      // Debug/pseudo instructions should be skipped and not prevent the
+      // formation of a select group.
+      if (NI->isDebugOrPseudoInst()) {
         ++BBIt;
+        continue;
       }
 
-      // If the select type is not supported, no point optimizing it.
-      // Instruction selection will take care of it.
-      if (!isSelectKindSupported(SI))
-        continue;
+      It = ProcessSelectInfo(NI);
+      if (It == SelectInfo.end()) {
+        AlreadyProcessed = true;
+        break;
+      }
 
-      LLVM_DEBUG({
-        dbgs() << "New Select group with\n";
-        for (auto SI : SIGroup)
-          dbgs() << "  " << *SI.getI() << "\n";
-      });
+      // Auxiliary with same condition
+      auto [CurrCond, IsAux, IsRev, CondIdx] = It->second;
+      if (Cond != CurrCond) {
+        AlreadyProcessed = true;
+        break;
+      }
 
-      SIGroups.push_back(SIGroup);
+      if (!IsAux)
+        SIGroup.Selects.emplace_back(NI, IsRev, CondIdx);
+      ++BBIt;
     }
+    LLVM_DEBUG({
+      dbgs() << "New Select group (" << SIGroup.Selects.size() << ") with\n";
+      for (auto &SI : SIGroup.Selects)
+        dbgs() << "  " << *SI.getI() << "\n";
+    });
+
+    SIGroups.push_back(SIGroup);
   }
 }
 
@@ -878,12 +933,13 @@ void SelectOptimizeImpl::findProfitableSIGroupsInnerLoops(
     // Assuming infinite resources, the cost of a group of instructions is the
     // cost of the most expensive instruction of the group.
     Scaled64 SelectCost = Scaled64::getZero(), BranchCost = Scaled64::getZero();
-    for (SelectLike SI : ASI) {
+    for (SelectLike &SI : ASI.Selects) {
       SelectCost = std::max(SelectCost, InstCostMap[SI.getI()].PredCost);
       BranchCost = std::max(BranchCost, InstCostMap[SI.getI()].NonPredCost);
     }
     if (BranchCost < SelectCost) {
-      OptimizationRemark OR(DEBUG_TYPE, "SelectOpti", ASI.front().getI());
+      OptimizationRemark OR(DEBUG_TYPE, "SelectOpti",
+                            ASI.Selects.front().getI());
       OR << "Profitable to convert to branch (loop analysis). BranchCost="
          << BranchCost.toString() << ", SelectCost=" << SelectCost.toString()
          << ". ";
@@ -892,7 +948,7 @@ void SelectOptimizeImpl::findProfitableSIGroupsInnerLoops(
       ProfSIGroups.push_back(ASI);
     } else {
       OptimizationRemarkMissed ORmiss(DEBUG_TYPE, "SelectOpti",
-                                      ASI.front().getI());
+                                      ASI.Selects.front().getI());
       ORmiss << "Select is more profitable (loop analysis). BranchCost="
              << BranchCost.toString()
              << ", SelectCost=" << SelectCost.toString() << ". ";
@@ -903,7 +959,7 @@ void SelectOptimizeImpl::findProfitableSIGroupsInnerLoops(
 
 bool SelectOptimizeImpl::isConvertToBranchProfitableBase(
     const SelectGroup &ASI) {
-  SelectLike SI = ASI.front();
+  const SelectLike &SI = ASI.Selects.front();
   LLVM_DEBUG(dbgs() << "Analyzing select group containing " << *SI.getI()
                     << "\n");
   OptimizationRemark OR(DEBUG_TYPE, "SelectOpti", SI.getI());
@@ -963,14 +1019,14 @@ static bool extractBranchWeights(const SelectOptimizeImpl::SelectLike SI,
 bool SelectOptimizeImpl::hasExpensiveColdOperand(const SelectGroup &ASI) {
   bool ColdOperand = false;
   uint64_t TrueWeight, FalseWeight, TotalWeight;
-  if (extractBranchWeights(ASI.front(), TrueWeight, FalseWeight)) {
+  if (extractBranchWeights(ASI.Selects.front(), TrueWeight, FalseWeight)) {
     uint64_t MinWeight = std::min(TrueWeight, FalseWeight);
     TotalWeight = TrueWeight + FalseWeight;
     // Is there a path with frequency <ColdOperandThreshold% (default:20%) ?
     ColdOperand = TotalWeight * ColdOperandThreshold > 100 * MinWeight;
   } else if (PSI->hasProfileSummary()) {
     OptimizationRemarkMissed ORmiss(DEBUG_TYPE, "SelectOpti",
-                                    ASI.front().getI());
+                                    ASI.Selects.front().getI());
     ORmiss << "Profile data available but missing branch-weights metadata for "
               "select instruction. ";
     EmitAndPrintRemark(ORE, ORmiss);
@@ -979,7 +1035,7 @@ bool SelectOptimizeImpl::hasExpensiveColdOperand(const SelectGroup &ASI) {
     return false;
   // Check if the cold path's dependence slice is expensive for any of the
   // selects of the group.
-  for (SelectLike SI : ASI) {
+  for (SelectLike SI : ASI.Selects) {
     Instruction *ColdI = nullptr;
     uint64_t HotWeight;
     if (TrueWeight < FalseWeight) {
@@ -1169,7 +1225,8 @@ bool SelectOptimizeImpl::computeLoopCosts(
     DenseMap<const Instruction *, CostInfo> &InstCostMap, CostInfo *LoopCost) {
   LLVM_DEBUG(dbgs() << "Calculating Latency / IPredCost / INonPredCost of loop "
                     << L->getHeader()->getName() << "\n");
-  const auto &SImap = getSImap(SIGroups);
+  const auto SImap = getSImap(SIGroups);
+  const auto SGmap = getSGmap(SIGroups);
   // Compute instruction and loop-critical-path costs across two iterations for
   // both predicated and non-predicated version.
   const unsigned Iterations = 2;
@@ -1216,13 +1273,14 @@ bool SelectOptimizeImpl::computeLoopCosts(
         // MispredictCost = max(MispredictPenalty, CondCost) * MispredictRate
         if (SImap.contains(&I)) {
           auto SI = SImap.at(&I);
-          Scaled64 TrueOpCost = SI.getTrueOpCost(InstCostMap, TTI);
-          Scaled64 FalseOpCost = SI.getFalseOpCost(InstCostMap, TTI);
+          const auto *SG = SGmap.at(&I);
+          Scaled64 TrueOpCost = SI.getOpCostOnBranch(true, InstCostMap, TTI);
+          Scaled64 FalseOpCost = SI.getOpCostOnBranch(false, InstCostMap, TTI);
           Scaled64 PredictedPathCost =
               getPredictedPathCost(TrueOpCost, FalseOpCost, SI);
 
           Scaled64 CondCost = Scaled64::getZero();
-          if (auto *CI = dyn_cast<Instruction>(SI.getCondition()))
+          if (auto *CI = dyn_cast<Instruction>(SG->Condition))
             if (InstCostMap.count(CI))
               CondCost = InstCostMap[CI].NonPredCost;
           Scaled64 MispredictCost = getMispredictionCost(SI, CondCost);
@@ -1248,11 +1306,20 @@ SmallDenseMap<const Instruction *, SelectOptimizeImpl::SelectLike, 2>
 SelectOptimizeImpl::getSImap(const SelectGroups &SIGroups) {
   SmallDenseMap<const Instruction *, SelectLike, 2> SImap;
   for (const SelectGroup &ASI : SIGroups)
-    for (SelectLike SI : ASI)
+    for (const SelectLike &SI : ASI.Selects)
       SImap.try_emplace(SI.getI(), SI);
   return SImap;
 }
 
+SmallDenseMap<const Instruction *, const SelectOptimizeImpl::SelectGroup *, 2>
+SelectOptimizeImpl::getSGmap(const SelectGroups &SIGroups) {
+  SmallDenseMap<const Instruction *, const SelectGroup *, 2> SImap;
+  for (const SelectGroup &ASI : SIGroups)
+    for (const SelectLike &SI : ASI.Selects)
+      SImap.try_emplace(SI.getI(), &ASI);
+  return SImap;
+}
+
 std::optional<uint64_t>
 SelectOptimizeImpl::computeInstCost(const Instruction *I) {
   InstructionCost ICost =
@@ -1311,9 +1378,6 @@ SelectOptimizeImpl::getPredictedPathCost(Scaled64 TrueCost, Scaled64 FalseCost,
 }
 
 bool SelectOptimizeImpl::isSelectKindSupported(const SelectLike SI) {
-  bool VectorCond = !SI.getCondition()->getType()->isIntegerTy(1);
-  if (VectorCond)
-    return false;
   TargetLowering::SelectSupportKind SelectKind;
   if (SI.getType()->isVectorTy())
     SelectKind = TargetLowering::ScalarCondVectorVal;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 521829675ae7c..605937503407a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1220,8 +1220,11 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
 
     if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
       // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
-      if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1}))
+      if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1})) {
+        NewFlags.setDisjoint(Flags.hasDisjoint() &&
+                             N0->getFlags().hasDisjoint());
         return DAG.getNode(Opc, DL, VT, N00, OpNode, NewFlags);
+      }
       return SDValue();
     }
     if (TLI.isReassocProfitable(DAG, N0, N1)) {
@@ -10230,7 +10233,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   // Variant of version done on multiply, except mul by a power of 2 is turned
   // into a shift.
   if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
-      N0->hasOneUse() && TLI.isDesirableToCommuteWithShift(N, Level)) {
+      TLI.isDesirableToCommuteWithShift(N, Level)) {
     SDValue N01 = N0.getOperand(1);
     if (SDValue Shl1 =
             DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) {
@@ -10249,8 +10252,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   // TODO: Should we limit this with isLegalAddImmediate?
   if (N0.getOpcode() == ISD::SIGN_EXTEND &&
       N0.getOperand(0).getOpcode() == ISD::ADD &&
-      N0.getOperand(0)->getFlags().hasNoSignedWrap() && N0->hasOneUse() &&
-      N0.getOperand(0)->hasOneUse() &&
+      N0.getOperand(0)->getFlags().hasNoSignedWrap() &&
       TLI.isDesirableToCommuteWithShift(N, Level)) {
     SDValue Add = N0.getOperand(0);
     SDLoc DL(N0);
@@ -15495,12 +15497,14 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
       unsigned BuildVecNumElts =  BuildVect.getNumOperands();
       unsigned TruncVecNumElts = VT.getVectorNumElements();
       unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
+      unsigned FirstElt = isLE ? 0 : (TruncEltOffset - 1);
 
       assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
              "Invalid number of elements");
 
       SmallVector<SDValue, 8> Opnds;
-      for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
+      for (unsigned i = FirstElt, e = BuildVecNumElts; i < e;
+           i += TruncEltOffset)
         Opnds.push_back(BuildVect.getOperand(i));
 
       return DAG.getBuildVector(VT, DL, Opnds);
@@ -18758,9 +18762,13 @@ SDValue DAGCombiner::rebuildSetCC(SDValue N) {
       EVT SetCCVT = N.getValueType();
       if (LegalTypes)
         SetCCVT = getSetCCResultType(SetCCVT);
-      // Replace the uses of XOR with SETCC
-      return DAG.getSetCC(SDLoc(N), SetCCVT, Op0, Op1,
-                          Equal ? ISD::SETEQ : ISD::SETNE);
+      // Replace the uses of XOR with SETCC. Note, avoid this transformation if
+      // it would introduce illegal operations post-legalization as this can
+      // result in infinite looping between converting xor->setcc here, and
+      // expanding setcc->xor in LegalizeSetCCCondCode if requested.
+      const ISD::CondCode CC = Equal ? ISD::SETEQ : ISD::SETNE;
+      if (!LegalOperations || TLI.isCondCodeLegal(CC, Op0.getSimpleValueType()))
+        return DAG.getSetCC(SDLoc(N), SetCCVT, Op0, Op1, CC);
     }
   }
 
@@ -22746,16 +22754,22 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
 
 /// Transform a vector binary operation into a scalar binary operation by moving
 /// the math/logic after an extract element of a vector.
-static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
-                                       const SDLoc &DL, bool LegalOperations) {
+static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
+                                       const SDLoc &DL, bool LegalTypes) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Vec = ExtElt->getOperand(0);
   SDValue Index = ExtElt->getOperand(1);
   auto *IndexC = dyn_cast<ConstantSDNode>(Index);
-  if (!IndexC || !TLI.isBinOp(Vec.getOpcode()) || !Vec.hasOneUse() ||
+  unsigned Opc = Vec.getOpcode();
+  if (!IndexC || !Vec.hasOneUse() || (!TLI.isBinOp(Opc) && Opc != ISD::SETCC) ||
       Vec->getNumValues() != 1)
     return SDValue();
 
+  EVT ResVT = ExtElt->getValueType(0);
+  if (Opc == ISD::SETCC &&
+      (ResVT != Vec.getValueType().getVectorElementType() || LegalTypes))
+    return SDValue();
+
   // Targets may want to avoid this to prevent an expensive register transfer.
   if (!TLI.shouldScalarizeBinop(Vec))
     return SDValue();
@@ -22766,19 +22780,24 @@ static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
   SDValue Op0 = Vec.getOperand(0);
   SDValue Op1 = Vec.getOperand(1);
   APInt SplatVal;
-  if (isAnyConstantBuildVector(Op0, true) ||
-      ISD::isConstantSplatVector(Op0.getNode(), SplatVal) ||
-      isAnyConstantBuildVector(Op1, true) ||
-      ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) {
-    // extractelt (binop X, C), IndexC --> binop (extractelt X, IndexC), C'
-    // extractelt (binop C, X), IndexC --> binop C', (extractelt X, IndexC)
-    EVT VT = ExtElt->getValueType(0);
-    SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
-    SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
-    return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1);
-  }
+  if (!isAnyConstantBuildVector(Op0, true) &&
+      !ISD::isConstantSplatVector(Op0.getNode(), SplatVal) &&
+      !isAnyConstantBuildVector(Op1, true) &&
+      !ISD::isConstantSplatVector(Op1.getNode(), SplatVal))
+    return SDValue();
 
-  return SDValue();
+  // extractelt (op X, C), IndexC --> op (extractelt X, IndexC), C'
+  // extractelt (op C, X), IndexC --> op C', (extractelt X, IndexC)
+  if (Opc == ISD::SETCC) {
+    EVT OpVT = Op0.getValueType().getVectorElementType();
+    Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op0, Index);
+    Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op1, Index);
+    return DAG.getSetCC(DL, ResVT, Op0, Op1,
+                        cast<CondCodeSDNode>(Vec->getOperand(2))->get());
+  }
+  Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op0, Index);
+  Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op1, Index);
+  return DAG.getNode(Opc, DL, ResVT, Op0, Op1);
 }
 
 // Given a ISD::EXTRACT_VECTOR_ELT, which is a glorified bit sequence extract,
@@ -23011,7 +23030,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     }
   }
 
-  if (SDValue BO = scalarizeExtractedBinop(N, DAG, DL, LegalOperations))
+  if (SDValue BO = scalarizeExtractedBinOp(N, DAG, DL, LegalTypes))
     return BO;
 
   if (VecVT.isScalableVector())
@@ -23055,18 +23074,29 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
       // ext_elt (bitcast (scalar_to_vec i64 X to v2i64) to v4i32), TruncElt -->
       // trunc i64 X to i32
       SDValue X = BCSrc.getOperand(0);
-      assert(X.getValueType().isScalarInteger() && ScalarVT.isScalarInteger() &&
+      EVT XVT = X.getValueType();
+      assert(XVT.isScalarInteger() && ScalarVT.isScalarInteger() &&
              "Extract element and scalar to vector can't change element type "
              "from FP to integer.");
       unsigned XBitWidth = X.getValueSizeInBits();
-      BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
+      unsigned Scale = XBitWidth / VecEltBitWidth;
+      BCTruncElt = IsLE ? 0 : Scale - 1;
 
       // An extract element return value type can be wider than its vector
       // operand element type. In that case, the high bits are undefined, so
       // it's possible that we may need to extend rather than truncate.
-      if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
+      if (ExtractIndex < Scale && XBitWidth > VecEltBitWidth) {
         assert(XBitWidth % VecEltBitWidth == 0 &&
                "Scalar bitwidth must be a multiple of vector element bitwidth");
+
+        if (ExtractIndex != BCTruncElt) {
+          unsigned ShiftIndex =
+              IsLE ? ExtractIndex : (Scale - 1) - ExtractIndex;
+          X = DAG.getNode(
+              ISD::SRL, DL, XVT, X,
+              DAG.getShiftAmountConstant(ShiftIndex * VecEltBitWidth, XVT, DL));
+        }
+
         return DAG.getAnyExtOrTrunc(X, DL, ScalarVT);
       }
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index eede879e7e80d..2480a94546c59 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1078,7 +1078,7 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
       // For ByVal, alignment should come from FE. BE will guess if this info
       // is not there, but there are cases it cannot get right.
       if (!MemAlign)
-        MemAlign = Align(TLI.getByValTypeAlignment(Arg.IndirectType, DL));
+        MemAlign = TLI.getByValTypeAlignment(Arg.IndirectType, DL);
       Flags.setByValSize(FrameSize);
     } else if (!MemAlign) {
       MemAlign = DL.getABITypeAlign(Arg.Ty);
@@ -1851,11 +1851,19 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) {
     return false;
   }
 
-  case Instruction::Unreachable:
-    if (TM.Options.TrapUnreachable)
+  case Instruction::Unreachable: {
+    if (TM.Options.TrapUnreachable) {
+      if (TM.Options.NoTrapAfterNoreturn) {
+        const auto *Call =
+            dyn_cast_or_null<CallInst>(cast<Instruction>(I)->getPrevNode());
+        if (Call && Call->doesNotReturn())
+          return true;
+      }
+
       return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
-    else
-      return true;
+    }
+    return true;
+  }
 
   case Instruction::Alloca:
     // FunctionLowering has the static-sized case covered.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 63536336e9622..ca87168929f96 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2104,7 +2104,7 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall L
     InChain = TCChain;
 
   TargetLowering::CallLoweringInfo CLI(DAG);
-  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
+  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetTy, isSigned);
   CLI.setDebugLoc(SDLoc(Node))
       .setChain(InChain)
       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
@@ -2135,7 +2135,7 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall L
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = Op;
     Entry.Ty = ArgTy;
-    Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
+    Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, isSigned);
     Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
   }
@@ -4794,7 +4794,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
                              NVT, Node->getOperand(IsStrict ? 1 : 0));
     TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setSExt(Signed);
+    CallOptions.setIsSigned(Signed);
     std::pair<SDValue, SDValue> Tmp =
         TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
     Results.push_back(Tmp.first);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 3f8d117400efd..71f100bfa0343 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1044,7 +1044,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
   SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
                            NVT, N->getOperand(IsStrict ? 1 : 0));
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(Signed);
+  CallOptions.setIsSigned(Signed);
   CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
   std::pair<SDValue, SDValue> Tmp =
       TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
@@ -2099,7 +2099,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
 
     TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setSExt(true);
+    CallOptions.setIsSigned(true);
     std::pair<SDValue, SDValue> Tmp =
         TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
     if (Strict)
@@ -3415,6 +3415,23 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
   SDValue Op = N->getOperand(IsStrict ? 1 : 0);
   EVT SVT = Op.getValueType();
 
+  // If the input type needs to be softened, do that now so that call lowering
+  // will see the f16 type.
+  if (getTypeAction(SVT) == TargetLowering::TypeSoftenFloat) {
+    RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, RVT);
+    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
+
+    SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
+    Op = GetSoftenedFloat(Op);
+    TargetLowering::MakeLibCallOptions CallOptions;
+    CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
+    std::pair<SDValue, SDValue> Tmp =
+        TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N), Chain);
+    if (IsStrict)
+      ReplaceValueWith(SDValue(N, 1), Tmp.second);
+    return DAG.getNode(ISD::BITCAST, SDLoc(N), MVT::i16, Tmp.first);
+  }
+
   if (IsStrict) {
     SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
                               {MVT::i16, MVT::Other}, {N->getOperand(0), Op});
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 7b9f544a5f9a4..69f25ebc88004 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2601,7 +2601,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
              N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
          "POWI exponent should match with sizeof(int) when doing the libcall.");
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
       DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
@@ -2835,6 +2835,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
   case ISD::FREEZE:       SplitRes_FREEZE(N, Lo, Hi); break;
+  case ISD::SETCC:        ExpandIntRes_SETCC(N, Lo, Hi); break;
 
   case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
   case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
@@ -3316,6 +3317,20 @@ static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
   }
 }
 
+void DAGTypeLegalizer::ExpandIntRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
+  SDLoc DL(N);
+
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+  EVT NewVT = getSetCCResultType(LHS.getValueType());
+
+  // Taking the same approach as ScalarizeVecRes_SETCC
+  SDValue Res = DAG.getNode(ISD::SETCC, DL, NewVT, LHS, RHS, N->getOperand(2));
+
+  Res = DAG.getBoolExtOrTrunc(Res, DL, N->getValueType(0), NewVT);
+  SplitInteger(Res, Lo, Hi);
+}
+
 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
                                            SDValue &Lo, SDValue &Hi) {
   SDLoc DL(N);
@@ -3995,11 +4010,18 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
     Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
   }
 
-  RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(Op.getValueType(), VT)
-                               : RTLIB::getFPTOUINT(Op.getValueType(), VT);
+  // NOTE: We need a variable that lives across makeLibCall so
+  // CallOptions.setTypeListBeforeSoften can save a reference to it.
+  EVT OpVT = Op.getValueType();
+
+  RTLIB::Libcall LC =
+      IsSigned ? RTLIB::getFPTOSINT(OpVT, VT) : RTLIB::getFPTOUINT(OpVT, VT);
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftenFloat)
+    CallOptions.setTypeListBeforeSoften(OpVT, VT);
+  else
+    CallOptions.setIsSigned(true); // FIXME: Is this needed?
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
                                                     CallOptions, dl, Chain);
   SplitInteger(Tmp.first, Lo, Hi);
@@ -4091,7 +4113,7 @@ void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
   EVT RetVT = N->getValueType(0);
 
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
                                                     Op, CallOptions, dl,
                                                     Chain);
@@ -4262,7 +4284,7 @@ void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
   // upper half of the result if it exceeds VT.
   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
                Lo, Hi);
 }
@@ -4633,7 +4655,7 @@ void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
 
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
 }
 
@@ -4873,7 +4895,7 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
     SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
     SDValue Ops[2] = {N->getOperand(0), ShAmt};
     TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setSExt(isSigned);
+    CallOptions.setIsSigned(isSigned);
     SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
     return;
   }
@@ -4963,7 +4985,7 @@ void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
 
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
 }
 
@@ -5652,7 +5674,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
          "Don't know how to expand this XINT_TO_FP!");
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setSExt(true);
+  CallOptions.setIsSigned(true);
   std::pair<SDValue, SDValue> Tmp =
       TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 1703149aca746..571a710cc92a3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -487,6 +487,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void ExpandIntRes_MINMAX            (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_CMP               (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_SETCC             (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_SADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_UADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 2655e8428309d..113a3bc0bbea6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -265,7 +265,7 @@ void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
 
   // Increment the pointer to the other half.
   unsigned IncrementSize = NVT.getSizeInBits() / 8;
-  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
+  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
   Hi = DAG.getLoad(
       NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize),
       LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 465128099f444..107454a92e356 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -6421,16 +6421,14 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
 
   // Adjust mask based on new input vector length.
-  SmallVector<int, 16> NewMask;
+  SmallVector<int, 16> NewMask(WidenNumElts, -1);
   for (unsigned i = 0; i != NumElts; ++i) {
     int Idx = N->getMaskElt(i);
     if (Idx < (int)NumElts)
-      NewMask.push_back(Idx);
+      NewMask[i] = Idx;
     else
-      NewMask.push_back(Idx - NumElts + WidenNumElts);
+      NewMask[i] = Idx - NumElts + WidenNumElts;
   }
-  for (unsigned i = NumElts; i != WidenNumElts; ++i)
-    NewMask.push_back(-1);
   return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
 }
 
@@ -6478,12 +6476,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
 
   // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
   // fixed-vectors.
-  SmallVector<int, 16> Mask;
-  for (unsigned i = 0; i != VTNumElts; ++i) {
-    Mask.push_back(IdxVal + i);
-  }
-  for (unsigned i = VTNumElts; i != WidenNumElts; ++i)
-    Mask.push_back(-1);
+  SmallVector<int, 16> Mask(WidenNumElts, -1);
+  std::iota(Mask.begin(), Mask.begin() + VTNumElts, IdxVal);
 
   return DAG.getVectorShuffle(WidenVT, dl, ReverseVal, DAG.getUNDEF(WidenVT),
                               Mask);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 182529123ec6d..34214550f3a12 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6823,7 +6823,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
             ScalarOps.push_back(getUNDEF(OpVT));
             continue;
           }
-          APInt Val = cast<ConstantSDNode>(Op)->getAPIntValue();
+          const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
           ScalarOps.push_back(SignExtendInReg(Val, OpVT));
         }
         return getBuildVector(VT, DL, ScalarOps);
@@ -10623,7 +10623,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
 }
 
 SDVTList SelectionDAG::getVTList(EVT VT) {
-  return makeVTList(SDNode::getValueTypeList(VT), 1);
+  if (!VT.isExtended())
+    return makeVTList(SDNode::getValueTypeList(VT.getSimpleVT()), 1);
+
+  return makeVTList(&(*EVTs.insert(VT).first), 1);
 }
 
 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
@@ -12383,17 +12386,11 @@ namespace {
 
 /// getValueTypeList - Return a pointer to the specified value type.
 ///
-const EVT *SDNode::getValueTypeList(EVT VT) {
-  static std::set<EVT, EVT::compareRawBits> EVTs;
+const EVT *SDNode::getValueTypeList(MVT VT) {
   static EVTArray SimpleVTArray;
-  static sys::SmartMutex<true> VTMutex;
 
-  if (VT.isExtended()) {
-    sys::SmartScopedLock<true> Lock(VTMutex);
-    return &(*EVTs.insert(VT).first);
-  }
-  assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
-  return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
+  assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
+  return &SimpleVTArray.VTs[VT.SimpleTy];
 }
 
 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a38a3e9b91052..519e828cc35bc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1896,6 +1896,18 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
                          DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
     }
 
+    if (VT.isRISCVVectorTuple()) {
+      assert(C->isNullValue() && "Can only zero this target type!");
+      return NodeMap[V] = DAG.getNode(
+                 ISD::BITCAST, getCurSDLoc(), VT,
+                 DAG.getNode(
+                     ISD::SPLAT_VECTOR, getCurSDLoc(),
+                     EVT::getVectorVT(*DAG.getContext(), MVT::i8,
+                                      VT.getSizeInBits().getKnownMinValue() / 8,
+                                      true),
+                     DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));
+    }
+
     VectorType *VecTy = cast<VectorType>(V->getType());
 
     // Now that we know the number and type of the elements, get that number of
@@ -11185,7 +11197,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         if (auto MA = Args[i].Alignment)
           MemAlign = *MA;
         else
-          MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
+          MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
       } else if (auto MA = Args[i].Alignment) {
         MemAlign = *MA;
       } else {
@@ -11742,7 +11754,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
         else if ((ParamAlign = Arg.getParamAlign()))
           MemAlign = *ParamAlign;
         else
-          MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
+          MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
         if (Flags.isByRef())
           Flags.setByRefSize(MemSize);
         else
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 0d99ae9cdebd5..3000dfda1bea0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -4045,6 +4045,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       // So it should be safe to assume that this node has been selected
       unsigned index = MatcherTable[MatcherIndex++];
       index |= (MatcherTable[MatcherIndex++] << 8);
+      index |= (MatcherTable[MatcherIndex++] << 16);
+      index |= (MatcherTable[MatcherIndex++] << 24);
       dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
       dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
       continue;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index bd4bcadb57d7a..e87d809f88eb9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -159,8 +159,8 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
     SDValue NewOp = Ops[i];
     Entry.Node = NewOp;
     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
-    Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
-                                                 CallOptions.IsSExt);
+    Entry.IsSExt =
+        shouldSignExtendTypeInLibCall(Entry.Ty, CallOptions.IsSigned);
     Entry.IsZExt = !Entry.IsSExt;
 
     if (CallOptions.IsSoften &&
@@ -177,7 +177,7 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   TargetLowering::CallLoweringInfo CLI(DAG);
-  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
+  bool signExtend = shouldSignExtendTypeInLibCall(RetTy, CallOptions.IsSigned);
   bool zeroExtend = !signExtend;
 
   if (CallOptions.IsSoften &&
@@ -3736,6 +3736,15 @@ bool TargetLowering::SimplifyDemandedVectorElts(
       KnownUndef.clearAllBits();
     }
     break;
+  case ISD::SINT_TO_FP:
+  case ISD::UINT_TO_FP:
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
+                                   KnownZero, TLO, Depth + 1))
+      return true;
+    // Don't fall through to generic undef -> undef handling.
+    return false;
   default: {
     if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
       if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
@@ -10876,7 +10885,7 @@ void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
     // Attempt a libcall.
     SDValue Ret;
     TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setSExt(Signed);
+    CallOptions.setIsSigned(Signed);
     CallOptions.setIsPostTypeLegalization(true);
     if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
       // Halves of WideVT are packed into registers in different order
@@ -11880,6 +11889,47 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
       return true;
     }
 
+    // Special case: expand i1 comparisons using logical operations.
+    if (OpVT == MVT::i1) {
+      SDValue Ret;
+      switch (CCCode) {
+      default:
+        llvm_unreachable("Unknown integer setcc!");
+      case ISD::SETEQ: // X == Y  -->  ~(X ^ Y)
+        Ret = DAG.getNOT(dl, DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS),
+                         MVT::i1);
+        break;
+      case ISD::SETNE: // X != Y  -->  (X ^ Y)
+        Ret = DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS);
+        break;
+      case ISD::SETGT:  // X >s Y  -->  X == 0 & Y == 1  -->  ~X & Y
+      case ISD::SETULT: // X <u Y  -->  X == 0 & Y == 1  -->  ~X & Y
+        Ret = DAG.getNode(ISD::AND, dl, MVT::i1, RHS,
+                          DAG.getNOT(dl, LHS, MVT::i1));
+        break;
+      case ISD::SETLT:  // X <s Y  -->  X == 1 & Y == 0  -->  ~Y & X
+      case ISD::SETUGT: // X >u Y  -->  X == 1 & Y == 0  -->  ~Y & X
+        Ret = DAG.getNode(ISD::AND, dl, MVT::i1, LHS,
+                          DAG.getNOT(dl, RHS, MVT::i1));
+        break;
+      case ISD::SETULE: // X <=u Y  -->  X == 0 | Y == 1  -->  ~X | Y
+      case ISD::SETGE:  // X >=s Y  -->  X == 0 | Y == 1  -->  ~X | Y
+        Ret = DAG.getNode(ISD::OR, dl, MVT::i1, RHS,
+                          DAG.getNOT(dl, LHS, MVT::i1));
+        break;
+      case ISD::SETUGE: // X >=u Y  -->  X == 1 | Y == 0  -->  ~Y | X
+      case ISD::SETLE:  // X <=s Y  -->  X == 1 | Y == 0  -->  ~Y | X
+        Ret = DAG.getNode(ISD::OR, dl, MVT::i1, LHS,
+                          DAG.getNOT(dl, RHS, MVT::i1));
+        break;
+      }
+
+      LHS = DAG.getZExtOrTrunc(Ret, dl, VT);
+      RHS = SDValue();
+      CC = SDValue();
+      return true;
+    }
+
     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
     unsigned Opc = 0;
     switch (CCCode) {
diff --git a/llvm/lib/CodeGen/SpillPlacement.cpp b/llvm/lib/CodeGen/SpillPlacement.cpp
index 318e2b19322bb..55a96a22a00ec 100644
--- a/llvm/lib/CodeGen/SpillPlacement.cpp
+++ b/llvm/lib/CodeGen/SpillPlacement.cpp
@@ -26,7 +26,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "SpillPlacement.h"
+#include "llvm/CodeGen/SpillPlacement.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/CodeGen/EdgeBundles.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -44,17 +44,17 @@ using namespace llvm;
 
 #define DEBUG_TYPE "spill-code-placement"
 
-char SpillPlacement::ID = 0;
+char SpillPlacementWrapperLegacy::ID = 0;
 
-char &llvm::SpillPlacementID = SpillPlacement::ID;
+char &llvm::SpillPlacementID = SpillPlacementWrapperLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(SpillPlacement, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(SpillPlacementWrapperLegacy, DEBUG_TYPE,
                       "Spill Code Placement Analysis", true, true)
 INITIALIZE_PASS_DEPENDENCY(EdgeBundlesWrapperLegacy)
-INITIALIZE_PASS_END(SpillPlacement, DEBUG_TYPE,
+INITIALIZE_PASS_END(SpillPlacementWrapperLegacy, DEBUG_TYPE,
                     "Spill Code Placement Analysis", true, true)
 
-void SpillPlacement::getAnalysisUsage(AnalysisUsage &AU) const {
+void SpillPlacementWrapperLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
   AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
   AU.addRequiredTransitive<EdgeBundlesWrapperLegacy>();
@@ -189,32 +189,64 @@ struct SpillPlacement::Node {
   }
 };
 
-bool SpillPlacement::runOnMachineFunction(MachineFunction &mf) {
+bool SpillPlacementWrapperLegacy::runOnMachineFunction(MachineFunction &MF) {
+  auto *Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
+  auto *MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
+
+  Impl.run(MF, Bundles, MBFI);
+  return false;
+}
+
+AnalysisKey SpillPlacementAnalysis::Key;
+
+SpillPlacement
+SpillPlacementAnalysis::run(MachineFunction &MF,
+                            MachineFunctionAnalysisManager &MFAM) {
+  auto *Bundles = &MFAM.getResult<EdgeBundlesAnalysis>(MF);
+  auto *MBFI = &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
+  SpillPlacement Impl;
+  Impl.run(MF, Bundles, MBFI);
+  return Impl;
+}
+
+bool SpillPlacementAnalysis::Result::invalidate(
+    MachineFunction &MF, const PreservedAnalyses &PA,
+    MachineFunctionAnalysisManager::Invalidator &Inv) {
+  auto PAC = PA.getChecker<SpillPlacementAnalysis>();
+  if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<MachineFunction>>())
+    return true;
+  // Check dependencies.
+  return Inv.invalidate<EdgeBundlesAnalysis>(MF, PA) ||
+         Inv.invalidate<MachineBlockFrequencyAnalysis>(MF, PA);
+}
+
+SpillPlacement::SpillPlacement() = default;
+SpillPlacement::~SpillPlacement() = default;
+SpillPlacement::SpillPlacement(SpillPlacement &&) = default;
+
+void SpillPlacement::releaseMemory() {
+  nodes.reset();
+  TodoList.clear();
+}
+
+void SpillPlacement::run(MachineFunction &mf, EdgeBundles *Bundles,
+                         MachineBlockFrequencyInfo *MBFI) {
   MF = &mf;
-  bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
+  this->bundles = Bundles;
+  this->MBFI = MBFI;
 
   assert(!nodes && "Leaking node array");
-  nodes = new Node[bundles->getNumBundles()];
+  nodes.reset(new Node[bundles->getNumBundles()]);
   TodoList.clear();
   TodoList.setUniverse(bundles->getNumBundles());
 
   // Compute total ingoing and outgoing block frequencies for all bundles.
   BlockFrequencies.resize(mf.getNumBlockIDs());
-  MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
   setThreshold(MBFI->getEntryFreq());
   for (auto &I : mf) {
     unsigned Num = I.getNumber();
     BlockFrequencies[Num] = MBFI->getBlockFreq(&I);
   }
-
-  // We never change the function.
-  return false;
-}
-
-void SpillPlacement::releaseMemory() {
-  delete[] nodes;
-  nodes = nullptr;
-  TodoList.clear();
 }
 
 /// activate - mark node n as active if it wasn't already.
@@ -323,9 +355,9 @@ bool SpillPlacement::scanActiveBundles() {
 }
 
 bool SpillPlacement::update(unsigned n) {
-  if (!nodes[n].update(nodes, Threshold))
+  if (!nodes[n].update(nodes.get(), Threshold))
     return false;
-  nodes[n].getDissentingNeighbors(TodoList, nodes);
+  nodes[n].getDissentingNeighbors(TodoList, nodes.get());
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/StackSlotColoring.cpp b/llvm/lib/CodeGen/StackSlotColoring.cpp
index a1fa266354a52..4dc5dc87ba3fc 100644
--- a/llvm/lib/CodeGen/StackSlotColoring.cpp
+++ b/llvm/lib/CodeGen/StackSlotColoring.cpp
@@ -149,7 +149,7 @@ namespace {
       AU.setPreservesCFG();
       AU.addRequired<SlotIndexesWrapperPass>();
       AU.addPreserved<SlotIndexesWrapperPass>();
-      AU.addRequired<LiveStacks>();
+      AU.addRequired<LiveStacksWrapperLegacy>();
       AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
       AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
       AU.addPreservedID(MachineDominatorsID);
@@ -159,7 +159,7 @@ namespace {
       // may be invoked multiple times requiring it to save these analyses to be
       // used by RA later.
       AU.addPreserved<LiveIntervalsWrapperPass>();
-      AU.addPreserved<LiveDebugVariables>();
+      AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
 
       MachineFunctionPass::getAnalysisUsage(AU);
     }
@@ -185,7 +185,7 @@ char &llvm::StackSlotColoringID = StackSlotColoring::ID;
 INITIALIZE_PASS_BEGIN(StackSlotColoring, DEBUG_TYPE,
                 "Stack Slot Coloring", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LiveStacks)
+INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
 INITIALIZE_PASS_END(StackSlotColoring, DEBUG_TYPE,
                 "Stack Slot Coloring", false, false)
@@ -522,7 +522,7 @@ bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) {
 
   MFI = &MF.getFrameInfo();
   TII = MF.getSubtarget().getInstrInfo();
-  LS = &getAnalysis<LiveStacks>();
+  LS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
   MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
   Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
 
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 392cfbdd21273..9dc2701654b8b 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1697,12 +1697,9 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
   }
 }
 
-/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
-/// function arguments in the caller parameter area.  This is the actual
-/// alignment, not its logarithm.
-uint64_t TargetLoweringBase::getByValTypeAlignment(Type *Ty,
-                                                   const DataLayout &DL) const {
-  return DL.getABITypeAlign(Ty).value();
+Align TargetLoweringBase::getByValTypeAlignment(Type *Ty,
+                                                const DataLayout &DL) const {
+  return DL.getABITypeAlign(Ty);
 }
 
 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
diff --git a/llvm/lib/CodeGen/TargetSubtargetInfo.cpp b/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
index 6c97bc0568bde..cd396e6a619a8 100644
--- a/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
+++ b/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
@@ -16,11 +16,12 @@ using namespace llvm;
 
 TargetSubtargetInfo::TargetSubtargetInfo(
     const Triple &TT, StringRef CPU, StringRef TuneCPU, StringRef FS,
-    ArrayRef<SubtargetFeatureKV> PF, ArrayRef<SubtargetSubTypeKV> PD,
-    const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL,
-    const MCReadAdvanceEntry *RA, const InstrStage *IS, const unsigned *OC,
-    const unsigned *FP)
-    : MCSubtargetInfo(TT, CPU, TuneCPU, FS, PF, PD, WPR, WL, RA, IS, OC, FP) {}
+    ArrayRef<StringRef> PN, ArrayRef<SubtargetFeatureKV> PF,
+    ArrayRef<SubtargetSubTypeKV> PD, const MCWriteProcResEntry *WPR,
+    const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA,
+    const InstrStage *IS, const unsigned *OC, const unsigned *FP)
+    : MCSubtargetInfo(TT, CPU, TuneCPU, FS, PN, PF, PD, WPR, WL, RA, IS, OC,
+                      FP) {}
 
 TargetSubtargetInfo::~TargetSubtargetInfo() = default;
 
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 26a12512c87be..b28c74600e7a2 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -60,7 +60,7 @@ STATISTIC(NumIdCopies,   "Number of identity moves eliminated after rewriting");
 char VirtRegMapWrapperLegacy::ID = 0;
 
 INITIALIZE_PASS(VirtRegMapWrapperLegacy, "virtregmap", "Virtual Register Map",
-                false, false)
+                false, true)
 
 void VirtRegMap::init(MachineFunction &mf) {
   MRI = &mf.getRegInfo();
@@ -251,9 +251,9 @@ INITIALIZE_PASS_BEGIN(VirtRegRewriter, "virtregrewriter",
                       "Virtual Register Rewriter", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
+INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
-INITIALIZE_PASS_DEPENDENCY(LiveStacks)
+INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
 INITIALIZE_PASS_END(VirtRegRewriter, "virtregrewriter",
                     "Virtual Register Rewriter", false, false)
@@ -264,14 +264,14 @@ void VirtRegRewriter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<LiveIntervalsWrapperPass>();
   AU.addRequired<SlotIndexesWrapperPass>();
   AU.addPreserved<SlotIndexesWrapperPass>();
-  AU.addRequired<LiveDebugVariables>();
-  AU.addRequired<LiveStacks>();
-  AU.addPreserved<LiveStacks>();
+  AU.addRequired<LiveDebugVariablesWrapperLegacy>();
+  AU.addRequired<LiveStacksWrapperLegacy>();
+  AU.addPreserved<LiveStacksWrapperLegacy>();
   AU.addRequired<VirtRegMapWrapperLegacy>();
   AU.addRequired<LiveRegMatrixWrapperLegacy>();
 
   if (!ClearVirtRegs)
-    AU.addPreserved<LiveDebugVariables>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
 
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -285,7 +285,7 @@ bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
   LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   LRM = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
   VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
-  DebugVars = &getAnalysis<LiveDebugVariables>();
+  DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
   LLVM_DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n"
                     << "********** Function: " << MF->getName() << '\n');
   LLVM_DEBUG(VRM->dump());
diff --git a/llvm/lib/CodeGen/XRayInstrumentation.cpp b/llvm/lib/CodeGen/XRayInstrumentation.cpp
index 8f718d884cd06..8af16fa6249f4 100644
--- a/llvm/lib/CodeGen/XRayInstrumentation.cpp
+++ b/llvm/lib/CodeGen/XRayInstrumentation.cpp
@@ -233,10 +233,13 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
     case Triple::ArchType::mips:
     case Triple::ArchType::mipsel:
     case Triple::ArchType::mips64:
-    case Triple::ArchType::mips64el: {
+    case Triple::ArchType::mips64el:
+    case Triple::ArchType::riscv32:
+    case Triple::ArchType::riscv64: {
       // For the architectures which don't have a single return instruction
       InstrumentationOptions op;
-      op.HandleTailcall = false;
+      // RISC-V supports patching tail calls.
+      op.HandleTailcall = MF.getTarget().getTargetTriple().isRISCV();
       op.HandleAllReturns = true;
       prependRetWithPatchableExit(MF, TII, op);
       break;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
index dcce484a7a37e..a0ce7810f91b0 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
@@ -413,6 +413,15 @@ bool DWARFDie::addressRangeContainsAddress(const uint64_t Address) const {
   return false;
 }
 
+std::optional<uint64_t> DWARFDie::getLanguage() const {
+  if (isValid()) {
+    if (std::optional<DWARFFormValue> LV =
+            U->getUnitDIE().find(dwarf::DW_AT_language))
+      return LV->getAsUnsignedConstant();
+  }
+  return std::nullopt;
+}
+
 Expected<DWARFLocationExpressionsVector>
 DWARFDie::getLocations(dwarf::Attribute Attr) const {
   std::optional<DWARFFormValue> Location = find(Attr);
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 95d435b95ff99..092495e87443c 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -253,15 +253,18 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
   // We are processing .debug_info section, implicit_const attribute
   // values are not really stored here, but in .debug_abbrev section.
   auto GetAsUnsignedConstant = [&]() -> int64_t {
-    return AttrSpec.isImplicitConst() ? AttrSpec.getImplicitConstValue()
-                                      : *FormValue.getAsUnsignedConstant();
+    if (AttrSpec.isImplicitConst())
+      return AttrSpec.getImplicitConstValue();
+    if (std::optional<uint64_t> Val = FormValue.getAsUnsignedConstant())
+      return *Val;
+    return 0;
   };
 
   auto GetFlag = [](const DWARFFormValue &FormValue) -> bool {
     return FormValue.isFormClass(DWARFFormValue::FC_Flag);
   };
 
-  auto GetBoundValue = [](const DWARFFormValue &FormValue) -> int64_t {
+  auto GetBoundValue = [&AttrSpec](const DWARFFormValue &FormValue) -> int64_t {
     switch (FormValue.getForm()) {
     case dwarf::DW_FORM_ref_addr:
     case dwarf::DW_FORM_ref1:
@@ -282,6 +285,8 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
       return *FormValue.getAsUnsignedConstant();
     case dwarf::DW_FORM_sdata:
       return *FormValue.getAsSignedConstant();
+    case dwarf::DW_FORM_implicit_const:
+      return AttrSpec.getImplicitConstValue();
     default:
       return 0;
     }
@@ -294,13 +299,13 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
 
   switch (AttrSpec.Attr) {
   case dwarf::DW_AT_accessibility:
-    CurrentElement->setAccessibilityCode(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setAccessibilityCode(GetAsUnsignedConstant());
     break;
   case dwarf::DW_AT_artificial:
     CurrentElement->setIsArtificial();
     break;
   case dwarf::DW_AT_bit_size:
-    CurrentElement->setBitSize(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setBitSize(GetAsUnsignedConstant());
     break;
   case dwarf::DW_AT_call_file:
     CurrentElement->setCallFilenameIndex(IncrementFileIndex
@@ -332,13 +337,12 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
         Stream << hexString(Value, 2);
         CurrentElement->setValue(Stream.str());
       } else
-        CurrentElement->setValue(
-            hexString(*FormValue.getAsUnsignedConstant(), 2));
+        CurrentElement->setValue(hexString(GetAsUnsignedConstant(), 2));
     } else
       CurrentElement->setValue(dwarf::toStringRef(FormValue));
     break;
   case dwarf::DW_AT_count:
-    CurrentElement->setCount(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setCount(GetAsUnsignedConstant());
     break;
   case dwarf::DW_AT_decl_line:
     CurrentElement->setLineNumber(GetAsUnsignedConstant());
@@ -357,10 +361,10 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
       CurrentElement->setIsExternal();
     break;
   case dwarf::DW_AT_GNU_discriminator:
-    CurrentElement->setDiscriminator(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setDiscriminator(GetAsUnsignedConstant());
     break;
   case dwarf::DW_AT_inline:
-    CurrentElement->setInlineCode(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setInlineCode(GetAsUnsignedConstant());
     break;
   case dwarf::DW_AT_lower_bound:
     CurrentElement->setLowerBound(GetBoundValue(FormValue));
@@ -368,6 +372,9 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
   case dwarf::DW_AT_name:
     CurrentElement->setName(dwarf::toStringRef(FormValue));
     break;
+  case dwarf::DW_AT_GNU_template_name:
+    CurrentElement->setValue(dwarf::toStringRef(FormValue));
+    break;
   case dwarf::DW_AT_linkage_name:
   case dwarf::DW_AT_MIPS_linkage_name:
     CurrentElement->setLinkageName(dwarf::toStringRef(FormValue));
@@ -380,7 +387,7 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
     CurrentElement->setUpperBound(GetBoundValue(FormValue));
     break;
   case dwarf::DW_AT_virtuality:
-    CurrentElement->setVirtualityCode(*FormValue.getAsUnsignedConstant());
+    CurrentElement->setVirtualityCode(GetAsUnsignedConstant());
     break;
 
   case dwarf::DW_AT_abstract_origin:
diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index 01ccbe886a2bc..b96f97a159243 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -747,7 +747,7 @@ StringRef demanglePE32ExternCFunc(StringRef SymbolName) {
 } // end anonymous namespace
 
 std::string
-LLVMSymbolizer::DemangleName(const std::string &Name,
+LLVMSymbolizer::DemangleName(StringRef Name,
                              const SymbolizableModule *DbiModuleDescriptor) {
   std::string Result;
   if (nonMicrosoftDemangle(Name, Result))
@@ -761,7 +761,7 @@ LLVMSymbolizer::DemangleName(const std::string &Name,
         MSDemangleFlags(MSDF_NoAccessSpecifier | MSDF_NoCallingConvention |
                         MSDF_NoMemberType | MSDF_NoReturnType));
     if (status != 0)
-      return Name;
+      return std::string{Name};
     Result = DemangledName;
     free(DemangledName);
     return Result;
@@ -775,7 +775,7 @@ LLVMSymbolizer::DemangleName(const std::string &Name,
       return Result;
     return DemangledCName;
   }
-  return Name;
+  return std::string{Name};
 }
 
 void LLVMSymbolizer::recordAccess(CachedBinary &Bin) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
index 4503642f52a9d..d370e99169b14 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
@@ -40,7 +40,8 @@ static StringRef getMachineName(uint16_t Machine) {
 }
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromCOFFObject(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromCOFFObject(MemoryBufferRef ObjectBuffer,
+                              std::shared_ptr<orc::SymbolStringPool> SSP) {
   StringRef Data = ObjectBuffer.getBuffer();
 
   // Check magic
@@ -108,7 +109,7 @@ createLinkGraphFromCOFFObject(MemoryBufferRef ObjectBuffer) {
 
   switch (Machine) {
   case COFF::IMAGE_FILE_MACHINE_AMD64:
-    return createLinkGraphFromCOFFObject_x86_64(ObjectBuffer);
+    return createLinkGraphFromCOFFObject_x86_64(ObjectBuffer, std::move(SSP));
   default:
     return make_error<JITLinkError>(
         "Unsupported target machine architecture in COFF object " +
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
index 92630650ec937..e8c8483ba7dfb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
@@ -11,6 +11,8 @@
 //===----------------------------------------------------------------------===//
 #include "COFFLinkGraphBuilder.h"
 
+#include <memory>
+
 #define DEBUG_TYPE "jitlink"
 
 static const char *CommonSectionName = "__common";
@@ -24,12 +26,16 @@ static Triple createTripleWithCOFFFormat(Triple T) {
 }
 
 COFFLinkGraphBuilder::COFFLinkGraphBuilder(
-    const object::COFFObjectFile &Obj, Triple TT, SubtargetFeatures Features,
+    const object::COFFObjectFile &Obj,
+    std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
+    SubtargetFeatures Features,
     LinkGraph::GetEdgeKindNameFunction GetEdgeKindName)
-    : Obj(Obj), G(std::make_unique<LinkGraph>(
-                    Obj.getFileName().str(), createTripleWithCOFFFormat(TT),
-                    std::move(Features), getPointerSize(Obj),
-                    getEndianness(Obj), std::move(GetEdgeKindName))) {
+    : Obj(Obj),
+      G(std::make_unique<LinkGraph>(Obj.getFileName().str(), std::move(SSP),
+                                    createTripleWithCOFFFormat(TT),
+                                    std::move(Features), getPointerSize(Obj),
+                                    getEndianness(Obj),
+                                    std::move(GetEdgeKindName))) {
   LLVM_DEBUG({
     dbgs() << "Created COFFLinkGraphBuilder for \"" << Obj.getFileName()
            << "\"\n";
@@ -226,18 +232,19 @@ Error COFFLinkGraphBuilder::graphifySymbols() {
             " (" + toString(SecOrErr.takeError()) + ")");
       Sec = *SecOrErr;
     }
+    auto InternedSymbolName = G->intern(std::move(SymbolName));
 
     // Create jitlink symbol
     jitlink::Symbol *GSym = nullptr;
     if (Sym->isFileRecord())
       LLVM_DEBUG({
         dbgs() << "    " << SymIndex << ": Skipping FileRecord symbol \""
-               << SymbolName << "\" in "
+               << InternedSymbolName << "\" in "
                << getCOFFSectionName(SectionIndex, Sec, *Sym)
                << " (index: " << SectionIndex << ") \n";
       });
     else if (Sym->isUndefined()) {
-      GSym = createExternalSymbol(SymIndex, SymbolName, *Sym, Sec);
+      GSym = createExternalSymbol(SymIndex, InternedSymbolName, *Sym, Sec);
     } else if (Sym->isWeakExternal()) {
       auto *WeakExternal = Sym->getAux<object::coff_aux_weak_external>();
       COFFSymbolIndex TagIndex = WeakExternal->TagIndex;
@@ -246,7 +253,7 @@ Error COFFLinkGraphBuilder::graphifySymbols() {
           {SymIndex, TagIndex, Characteristics, SymbolName});
     } else {
       Expected<jitlink::Symbol *> NewGSym =
-          createDefinedSymbol(SymIndex, SymbolName, *Sym, Sec);
+          createDefinedSymbol(SymIndex, InternedSymbolName, *Sym, Sec);
       if (!NewGSym)
         return NewGSym.takeError();
       GSym = *NewGSym;
@@ -254,7 +261,7 @@ Error COFFLinkGraphBuilder::graphifySymbols() {
         LLVM_DEBUG({
           dbgs() << "    " << SymIndex
                  << ": Creating defined graph symbol for COFF symbol \""
-                 << SymbolName << "\" in "
+                 << InternedSymbolName << "\" in "
                  << getCOFFSectionName(SectionIndex, Sec, *Sym)
                  << " (index: " << SectionIndex << ") \n";
           dbgs() << "      " << *GSym << "\n";
@@ -293,14 +300,13 @@ Error COFFLinkGraphBuilder::handleDirectiveSection(StringRef Str) {
       if (From.empty() || To.empty())
         return make_error<JITLinkError>(
             "Invalid COFF /alternatename directive");
-      AlternateNames[From] = To;
+      AlternateNames[G->intern(From)] = G->intern(To);
       break;
     }
     case COFF_OPT_incl: {
-      auto DataCopy = G->allocateContent(S);
-      StringRef StrCopy(DataCopy.data(), DataCopy.size());
-      ExternalSymbols[StrCopy] = &G->addExternalSymbol(StrCopy, 0, false);
-      ExternalSymbols[StrCopy]->setLive(true);
+      auto Symbol = &G->addExternalSymbol(S, 0, false);
+      Symbol->setLive(true);
+      ExternalSymbols[Symbol->getName()] = Symbol;
       break;
     }
     case COFF_OPT_export:
@@ -332,8 +338,8 @@ Error COFFLinkGraphBuilder::flushWeakAliasRequests() {
               ? Scope::Default
               : Scope::Local;
 
-      auto NewSymbol =
-          createAliasSymbol(WeakExternal.SymbolName, Linkage::Weak, S, *Target);
+      auto NewSymbol = createAliasSymbol(G->intern(WeakExternal.SymbolName),
+                                         Linkage::Weak, S, *Target);
       if (!NewSymbol)
         return NewSymbol.takeError();
       setGraphSymbol(AliasSymbol->getSectionNumber(), WeakExternal.Alias,
@@ -354,37 +360,44 @@ Error COFFLinkGraphBuilder::flushWeakAliasRequests() {
 }
 
 Error COFFLinkGraphBuilder::handleAlternateNames() {
-  for (auto &KeyValue : AlternateNames)
-    if (DefinedSymbols.count(KeyValue.second) &&
-        ExternalSymbols.count(KeyValue.first)) {
-      auto *Target = DefinedSymbols[KeyValue.second];
-      auto *Alias = ExternalSymbols[KeyValue.first];
+  for (auto &KeyValue : AlternateNames) {
+    auto DefinedSymbolName = KeyValue.second;
+    auto ExternalSymbolsName = KeyValue.first;
+    if (DefinedSymbols.count(DefinedSymbolName) &&
+        ExternalSymbols.count(ExternalSymbolsName)) {
+      auto *Target = DefinedSymbols[DefinedSymbolName];
+      auto *Alias = ExternalSymbols[ExternalSymbolsName];
       G->makeDefined(*Alias, Target->getBlock(), Target->getOffset(),
                      Target->getSize(), Linkage::Weak, Scope::Local, false);
     }
+  }
   return Error::success();
 }
 
 Symbol *COFFLinkGraphBuilder::createExternalSymbol(
-    COFFSymbolIndex SymIndex, StringRef SymbolName,
+    COFFSymbolIndex SymIndex, orc::SymbolStringPtr SymbolName,
     object::COFFSymbolRef Symbol, const object::coff_section *Section) {
-  if (!ExternalSymbols.count(SymbolName))
-    ExternalSymbols[SymbolName] =
-        &G->addExternalSymbol(SymbolName, Symbol.getValue(), false);
+  llvm::jitlink::Symbol *Sym = nullptr;
+  if (!ExternalSymbols.count(SymbolName)) {
+    Sym = &G->addExternalSymbol(*SymbolName, Symbol.getValue(), false);
+    ExternalSymbols[Sym->getName()] = Sym;
+  } else {
+    Sym = ExternalSymbols[SymbolName];
+  }
 
   LLVM_DEBUG({
     dbgs() << "    " << SymIndex
            << ": Creating external graph symbol for COFF symbol \""
-           << SymbolName << "\" in "
+           << Sym->getName() << "\" in "
            << getCOFFSectionName(Symbol.getSectionNumber(), Section, Symbol)
            << " (index: " << Symbol.getSectionNumber() << ") \n";
   });
-  return ExternalSymbols[SymbolName];
+  return Sym;
 }
 
-Expected<Symbol *> COFFLinkGraphBuilder::createAliasSymbol(StringRef SymbolName,
-                                                           Linkage L, Scope S,
-                                                           Symbol &Target) {
+Expected<Symbol *>
+COFFLinkGraphBuilder::createAliasSymbol(orc::SymbolStringPtr SymbolName,
+                                        Linkage L, Scope S, Symbol &Target) {
   if (!Target.isDefined()) {
     // FIXME: Support this when there's a way to handle this.
     return make_error<JITLinkError>("Weak external symbol with external "
@@ -460,8 +473,9 @@ Error COFFLinkGraphBuilder::calculateImplicitSizeOfSymbols() {
 }
 
 Expected<Symbol *> COFFLinkGraphBuilder::createDefinedSymbol(
-    COFFSymbolIndex SymIndex, StringRef SymbolName,
+    COFFSymbolIndex SymIndex, orc::SymbolStringPtr SymbolName,
     object::COFFSymbolRef Symbol, const object::coff_section *Section) {
+
   if (Symbol.isCommon()) {
     // FIXME: correct alignment
     return &G->addDefinedSymbol(
@@ -470,6 +484,7 @@ Expected<Symbol *> COFFLinkGraphBuilder::createDefinedSymbol(
         0, SymbolName, Symbol.getValue(), Linkage::Weak, Scope::Default,
         false, false);
   }
+
   if (Symbol.isAbsolute())
     return &G->addAbsoluteSymbol(SymbolName,
                                  orc::ExecutorAddr(Symbol.getValue()), 0,
@@ -603,7 +618,7 @@ Expected<Symbol *> COFFLinkGraphBuilder::createCOMDATExportRequest(
 // Process the second symbol of COMDAT sequence.
 Expected<Symbol *>
 COFFLinkGraphBuilder::exportCOMDATSymbol(COFFSymbolIndex SymIndex,
-                                         StringRef SymbolName,
+                                         orc::SymbolStringPtr SymbolName,
                                          object::COFFSymbolRef Symbol) {
   Block *B = getGraphBlock(Symbol.getSectionNumber());
   auto &PendingComdatExport = PendingComdatExports[Symbol.getSectionNumber()];
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
index e5f3ce8c53f5e..def0d94dda468 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
@@ -37,7 +37,8 @@ class COFFLinkGraphBuilder {
   using COFFSectionIndex = int32_t;
   using COFFSymbolIndex = int32_t;
 
-  COFFLinkGraphBuilder(const object::COFFObjectFile &Obj, Triple TT,
+  COFFLinkGraphBuilder(const object::COFFObjectFile &Obj,
+                       std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
                        SubtargetFeatures Features,
                        LinkGraph::GetEdgeKindNameFunction GetEdgeKindName);
 
@@ -134,20 +135,21 @@ class COFFLinkGraphBuilder {
 
   Section &getCommonSection();
 
-  Symbol *createExternalSymbol(COFFSymbolIndex SymIndex, StringRef SymbolName,
+  Symbol *createExternalSymbol(COFFSymbolIndex SymIndex,
+                               orc::SymbolStringPtr SymbolName,
                                object::COFFSymbolRef Symbol,
                                const object::coff_section *Section);
-  Expected<Symbol *> createAliasSymbol(StringRef SymbolName, Linkage L, Scope S,
-                                       Symbol &Target);
+  Expected<Symbol *> createAliasSymbol(orc::SymbolStringPtr SymbolName,
+                                       Linkage L, Scope S, Symbol &Target);
   Expected<Symbol *> createDefinedSymbol(COFFSymbolIndex SymIndex,
-                                         StringRef SymbolName,
+                                         orc::SymbolStringPtr SymbolName,
                                          object::COFFSymbolRef Symbol,
                                          const object::coff_section *Section);
   Expected<Symbol *> createCOMDATExportRequest(
       COFFSymbolIndex SymIndex, object::COFFSymbolRef Symbol,
       const object::coff_aux_section_definition *Definition);
   Expected<Symbol *> exportCOMDATSymbol(COFFSymbolIndex SymIndex,
-                                        StringRef SymbolName,
+                                        orc::SymbolStringPtr SymbolName,
                                         object::COFFSymbolRef Symbol);
 
   Error handleDirectiveSection(StringRef Str);
@@ -176,9 +178,9 @@ class COFFLinkGraphBuilder {
   std::vector<Block *> GraphBlocks;
   std::vector<Symbol *> GraphSymbols;
 
-  DenseMap<StringRef, StringRef> AlternateNames;
-  DenseMap<StringRef, Symbol *> ExternalSymbols;
-  DenseMap<StringRef, Symbol *> DefinedSymbols;
+  DenseMap<orc::SymbolStringPtr, orc::SymbolStringPtr> AlternateNames;
+  DenseMap<orc::SymbolStringPtr, Symbol *> ExternalSymbols;
+  DenseMap<orc::SymbolStringPtr, Symbol *> DefinedSymbols;
 };
 
 template <typename RelocHandlerFunction>
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
index 3257a2ae94f66..008f920c1ef78 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp
@@ -148,6 +148,7 @@ class COFFLinkGraphBuilder_x86_64 : public COFFLinkGraphBuilder {
         SectionIdx = getObject().getNumberOfSections() + 1;
       else
         SectionIdx = COFFSymbol.getSectionNumber();
+
       auto *AbsSym = &getGraph().addAbsoluteSymbol(
           "secidx", orc::ExecutorAddr(SectionIdx), 2, Linkage::Strong,
           Scope::Local, false);
@@ -181,14 +182,20 @@ class COFFLinkGraphBuilder_x86_64 : public COFFLinkGraphBuilder {
   }
 
 public:
-  COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj, const Triple T,
-                              const SubtargetFeatures Features)
-      : COFFLinkGraphBuilder(Obj, std::move(T), std::move(Features),
+  COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj,
+                              std::shared_ptr<orc::SymbolStringPool> SSP,
+                              const Triple T, const SubtargetFeatures Features)
+      : COFFLinkGraphBuilder(Obj, std::move(SSP), std::move(T),
+                             std::move(Features),
                              getCOFFX86RelocationKindName) {}
 };
 
 class COFFLinkGraphLowering_x86_64 {
 public:
+  COFFLinkGraphLowering_x86_64(std::shared_ptr<orc::SymbolStringPool> SSP)
+      : SSP(std::move(SSP)) {
+    ImageBaseName = this->SSP->intern("__ImageBase");
+  }
   // Lowers COFF x86_64 specific edges to generic x86_64 edges.
   Error lowerCOFFRelocationEdges(LinkGraph &G, JITLinkContext &Ctx) {
     for (auto *B : G.blocks()) {
@@ -230,7 +237,9 @@ class COFFLinkGraphLowering_x86_64 {
   }
 
 private:
-  static StringRef getImageBaseSymbolName() { return "__ImageBase"; }
+  const orc::SymbolStringPtr &getImageBaseSymbolName() const {
+    return this->ImageBaseName;
+  }
 
   orc::ExecutorAddr getSectionStart(Section &Sec) {
     if (!SectionStartCache.count(&Sec)) {
@@ -256,7 +265,7 @@ class COFFLinkGraphLowering_x86_64 {
     Error Err = Error::success();
     Ctx.lookup(Symbols,
                createLookupContinuation([&](Expected<AsyncLookupResult> LR) {
-                 ErrorAsOutParameter EAO(&Err);
+                 ErrorAsOutParameter _(Err);
                  if (!LR) {
                    Err = LR.takeError();
                    return;
@@ -271,12 +280,13 @@ class COFFLinkGraphLowering_x86_64 {
 
   DenseMap<Section *, orc::ExecutorAddr> SectionStartCache;
   orc::ExecutorAddr ImageBase;
+  std::shared_ptr<orc::SymbolStringPool> SSP;
+  orc::SymbolStringPtr ImageBaseName = nullptr;
 };
 
 Error lowerEdges_COFF_x86_64(LinkGraph &G, JITLinkContext *Ctx) {
   LLVM_DEBUG(dbgs() << "Lowering COFF x86_64 edges:\n");
-  COFFLinkGraphLowering_x86_64 GraphLowering;
-
+  COFFLinkGraphLowering_x86_64 GraphLowering(G.getSymbolStringPool());
   if (auto Err = GraphLowering.lowerCOFFRelocationEdges(G, *Ctx))
     return Err;
 
@@ -305,8 +315,8 @@ const char *getCOFFX86RelocationKindName(Edge::Kind R) {
   }
 }
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromCOFFObject_x86_64(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromCOFFObject_x86_64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -320,7 +330,8 @@ createLinkGraphFromCOFFObject_x86_64(MemoryBufferRef ObjectBuffer) {
   if (!Features)
     return Features.takeError();
 
-  return COFFLinkGraphBuilder_x86_64(**COFFObj, (*COFFObj)->makeTriple(),
+  return COFFLinkGraphBuilder_x86_64(**COFFObj, std::move(SSP),
+                                     (*COFFObj)->makeTriple(),
                                      std::move(*Features))
       .buildGraph();
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
index 49cfc5dec25bb..c7a2f071c29f3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
+++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
@@ -112,13 +112,13 @@ identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
   constexpr StringRef EndSymbolPrefix = "__stop_";
 
   auto SymName = Sym.getName();
-  if (SymName.starts_with(StartSymbolPrefix)) {
-    if (auto *Sec =
-            G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
+  if ((*SymName).starts_with(StartSymbolPrefix)) {
+    if (auto *Sec = G.findSectionByName(
+            (*SymName).drop_front(StartSymbolPrefix.size())))
       return {*Sec, true};
-  } else if (SymName.starts_with(EndSymbolPrefix)) {
+  } else if ((*SymName).starts_with(EndSymbolPrefix)) {
     if (auto *Sec =
-            G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
+            G.findSectionByName((*SymName).drop_front(EndSymbolPrefix.size())))
       return {*Sec, false};
   }
   return {};
@@ -131,15 +131,15 @@ identifyMachOSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
   constexpr StringRef EndSymbolPrefix = "section$end$";
 
   auto SymName = Sym.getName();
-  if (SymName.starts_with(StartSymbolPrefix)) {
+  if ((*SymName).starts_with(StartSymbolPrefix)) {
     auto [SegName, SecName] =
-      SymName.drop_front(StartSymbolPrefix.size()).split('$');
+        (*SymName).drop_front(StartSymbolPrefix.size()).split('$');
     std::string SectionName = (SegName + "," + SecName).str();
     if (auto *Sec = G.findSectionByName(SectionName))
       return {*Sec, true};
-  } else if (SymName.starts_with(EndSymbolPrefix)) {
+  } else if ((*SymName).starts_with(EndSymbolPrefix)) {
     auto [SegName, SecName] =
-      SymName.drop_front(EndSymbolPrefix.size()).split('$');
+        (*SymName).drop_front(EndSymbolPrefix.size()).split('$');
     std::string SectionName = (SegName + "," + SecName).str();
     if (auto *Sec = G.findSectionByName(SectionName))
       return {*Sec, false};
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
index d92feeccf5e92..663a883a4bcce 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
@@ -69,7 +69,8 @@ Expected<uint16_t> readTargetMachineArch(StringRef Buffer) {
 }
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer,
+                             std::shared_ptr<orc::SymbolStringPool> SSP) {
   StringRef Buffer = ObjectBuffer.getBuffer();
   if (Buffer.size() < ELF::EI_NIDENT)
     return make_error<JITLinkError>("Truncated ELF buffer");
@@ -84,23 +85,23 @@ createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) {
 
   switch (*TargetMachineArch) {
   case ELF::EM_AARCH64:
-    return createLinkGraphFromELFObject_aarch64(ObjectBuffer);
+    return createLinkGraphFromELFObject_aarch64(ObjectBuffer, std::move(SSP));
   case ELF::EM_ARM:
-    return createLinkGraphFromELFObject_aarch32(ObjectBuffer);
-  case ELF::EM_LOONGARCH:
-    return createLinkGraphFromELFObject_loongarch(ObjectBuffer);
+    return createLinkGraphFromELFObject_aarch32(ObjectBuffer, std::move(SSP));
   case ELF::EM_PPC64: {
     if (DataEncoding == ELF::ELFDATA2LSB)
-      return createLinkGraphFromELFObject_ppc64le(ObjectBuffer);
+      return createLinkGraphFromELFObject_ppc64le(ObjectBuffer, std::move(SSP));
     else
-      return createLinkGraphFromELFObject_ppc64(ObjectBuffer);
+      return createLinkGraphFromELFObject_ppc64(ObjectBuffer, std::move(SSP));
   }
+  case ELF::EM_LOONGARCH:
+    return createLinkGraphFromELFObject_loongarch(ObjectBuffer, std::move(SSP));
   case ELF::EM_RISCV:
-    return createLinkGraphFromELFObject_riscv(ObjectBuffer);
+    return createLinkGraphFromELFObject_riscv(ObjectBuffer, std::move(SSP));
   case ELF::EM_X86_64:
-    return createLinkGraphFromELFObject_x86_64(ObjectBuffer);
+    return createLinkGraphFromELFObject_x86_64(ObjectBuffer, std::move(SSP));
   case ELF::EM_386:
-    return createLinkGraphFromELFObject_i386(ObjectBuffer);
+    return createLinkGraphFromELFObject_i386(ObjectBuffer, std::move(SSP));
   default:
     return make_error<JITLinkError>(
         "Unsupported target machine architecture in ELF object " +
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
index c05961c64837c..fbe40fe5f7571 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
@@ -58,7 +58,8 @@ class ELFLinkGraphBuilder : public ELFLinkGraphBuilderBase {
   using ELFFile = object::ELFFile<ELFT>;
 
 public:
-  ELFLinkGraphBuilder(const object::ELFFile<ELFT> &Obj, Triple TT,
+  ELFLinkGraphBuilder(const object::ELFFile<ELFT> &Obj,
+                      std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
                       SubtargetFeatures Features, StringRef FileName,
                       LinkGraph::GetEdgeKindNameFunction GetEdgeKindName);
 
@@ -189,12 +190,13 @@ class ELFLinkGraphBuilder : public ELFLinkGraphBuilderBase {
 
 template <typename ELFT>
 ELFLinkGraphBuilder<ELFT>::ELFLinkGraphBuilder(
-    const ELFFile &Obj, Triple TT, SubtargetFeatures Features,
-    StringRef FileName, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName)
+    const ELFFile &Obj, std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
+    SubtargetFeatures Features, StringRef FileName,
+    LinkGraph::GetEdgeKindNameFunction GetEdgeKindName)
     : ELFLinkGraphBuilderBase(std::make_unique<LinkGraph>(
-          FileName.str(), Triple(std::move(TT)), std::move(Features),
-          ELFT::Is64Bits ? 8 : 4, llvm::endianness(ELFT::Endianness),
-          std::move(GetEdgeKindName))),
+          FileName.str(), std::move(SSP), Triple(std::move(TT)),
+          std::move(Features), ELFT::Is64Bits ? 8 : 4,
+          llvm::endianness(ELFT::Endianness), std::move(GetEdgeKindName))),
       Obj(Obj) {
   LLVM_DEBUG(
       { dbgs() << "Created ELFLinkGraphBuilder for \"" << FileName << "\""; });
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
index 866de2cb227c3..475de48660130 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
@@ -219,11 +219,13 @@ class ELFLinkGraphBuilder_aarch32
 
 public:
   ELFLinkGraphBuilder_aarch32(StringRef FileName,
-                              const llvm::object::ELFFile<ELFT> &Obj, Triple TT,
-                              SubtargetFeatures Features,
+                              const llvm::object::ELFFile<ELFT> &Obj,
+                              std::shared_ptr<orc::SymbolStringPool> SSP,
+                              Triple TT, SubtargetFeatures Features,
                               aarch32::ArmConfig ArmCfg)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, getELFAArch32EdgeKindName),
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  getELFAArch32EdgeKindName),
         ArmCfg(std::move(ArmCfg)) {}
 };
 
@@ -239,8 +241,8 @@ Error buildTables_ELF_aarch32(LinkGraph &G) {
   return Error::success();
 }
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_aarch32(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -273,16 +275,16 @@ createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) {
   case Triple::thumb: {
     auto &ELFFile = cast<ELFObjectFile<ELF32LE>>(**ELFObj).getELFFile();
     return ELFLinkGraphBuilder_aarch32<llvm::endianness::little>(
-               (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features),
-               ArmCfg)
+               (*ELFObj)->getFileName(), ELFFile, std::move(SSP), TT,
+               std::move(*Features), ArmCfg)
         .buildGraph();
   }
   case Triple::armeb:
   case Triple::thumbeb: {
     auto &ELFFile = cast<ELFObjectFile<ELF32BE>>(**ELFObj).getELFFile();
     return ELFLinkGraphBuilder_aarch32<llvm::endianness::big>(
-               (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features),
-               ArmCfg)
+               (*ELFObj)->getFileName(), ELFFile, std::move(SSP), TT,
+               std::move(*Features), ArmCfg)
         .buildGraph();
   }
   default:
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index 86e5054890799..260a493f0ffd2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -52,10 +52,13 @@ class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
   }
 
   Error getOrCreateGOTSymbol(LinkGraph &G) {
+    auto InteredGOTSymbolName =
+        G.getSymbolStringPool()->intern(ELFGOTSymbolName);
+
     auto DefineExternalGOTSymbolIfPresent =
         createDefineExternalSectionStartAndEndSymbolsPass(
             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
-              if (Sym.getName() == ELFGOTSymbolName)
+              if (*Sym.getName() == ELFGOTSymbolName)
                 if (auto *GOTSection = G.findSectionByName(
                         aarch64::GOTTableManager::getSectionName())) {
                   GOTSymbol = &Sym;
@@ -81,7 +84,7 @@ class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
 
       // Check for an existing defined symbol.
       for (auto *Sym : GOTSection->symbols())
-        if (Sym->getName() == ELFGOTSymbolName) {
+        if (Sym->getName() == InteredGOTSymbolName) {
           GOTSymbol = Sym;
           return Error::success();
         }
@@ -90,11 +93,12 @@ class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
       SectionRange SR(*GOTSection);
       if (SR.empty())
         GOTSymbol =
-            &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
+            // FIXME: we should only do this once
+            &G.addAbsoluteSymbol(InteredGOTSymbolName, orc::ExecutorAddr(), 0,
                                  Linkage::Strong, Scope::Local, true);
       else
         GOTSymbol =
-            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
+            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, InteredGOTSymbolName, 0,
                                 Linkage::Strong, Scope::Local, false, true);
     }
 
@@ -103,7 +107,7 @@ class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
     // we just need to point the GOT symbol at some address in this graph.
     if (!GOTSymbol) {
       for (auto *Sym : G.external_symbols()) {
-        if (Sym->getName() == ELFGOTSymbolName) {
+        if (*Sym->getName() == ELFGOTSymbolName) {
           auto Blocks = G.blocks();
           if (!Blocks.empty()) {
             G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
@@ -525,10 +529,13 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
 
 public:
   ELFLinkGraphBuilder_aarch64(StringRef FileName,
-                              const object::ELFFile<ELFT> &Obj, Triple TT,
-                              SubtargetFeatures Features)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, aarch64::getEdgeKindName) {}
+                              const object::ELFFile<ELFT> &Obj,
+                              std::shared_ptr<orc::SymbolStringPool> SSP,
+                              Triple TT, SubtargetFeatures Features)
+
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  aarch64::getEdgeKindName) {}
 };
 
 // TLS Info Builder.
@@ -665,8 +672,8 @@ Error buildTables_ELF_aarch64(LinkGraph &G) {
 namespace llvm {
 namespace jitlink {
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_aarch64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -685,7 +692,7 @@ createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
 
   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
   return ELFLinkGraphBuilder_aarch64<object::ELF64LE>(
-             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
+             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), std::move(SSP),
              (*ELFObj)->makeTriple(), std::move(*Features))
       .buildGraph();
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp
index 2d5f28cad1cc6..4ce43c1962c84 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp
@@ -56,7 +56,8 @@ class ELFJITLinker_i386 : public JITLinker<ELFJITLinker_i386> {
     auto DefineExternalGOTSymbolIfPresent =
         createDefineExternalSectionStartAndEndSymbolsPass(
             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
-              if (Sym.getName() == ELFGOTSymbolName)
+              if (Sym.getName() != nullptr &&
+                  *Sym.getName() == ELFGOTSymbolName)
                 if (auto *GOTSection = G.findSectionByName(
                         i386::GOTTableManager::getSectionName())) {
                   GOTSymbol = &Sym;
@@ -82,7 +83,7 @@ class ELFJITLinker_i386 : public JITLinker<ELFJITLinker_i386> {
 
       // Check for an existing defined symbol.
       for (auto *Sym : GOTSection->symbols())
-        if (Sym->getName() == ELFGOTSymbolName) {
+        if (Sym->getName() != nullptr && *Sym->getName() == ELFGOTSymbolName) {
           GOTSymbol = Sym;
           return Error::success();
         }
@@ -224,13 +225,16 @@ class ELFLinkGraphBuilder_i386 : public ELFLinkGraphBuilder<ELFT> {
 
 public:
   ELFLinkGraphBuilder_i386(StringRef FileName, const object::ELFFile<ELFT> &Obj,
+                           std::shared_ptr<orc::SymbolStringPool> SSP,
                            Triple TT, SubtargetFeatures Features)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, i386::getEdgeKindName) {}
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  i386::getEdgeKindName) {}
 };
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer,
+                                  std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -248,8 +252,9 @@ createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer) {
          "Only i386 (little endian) is supported for now");
 
   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj);
+
   return ELFLinkGraphBuilder_i386<object::ELF32LE>(
-             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
+             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), std::move(SSP),
              (*ELFObj)->makeTriple(), std::move(*Features))
       .buildGraph();
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
index 42fcfebd12d61..56c32aeecf55a 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
@@ -131,10 +131,12 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
 
 public:
   ELFLinkGraphBuilder_loongarch(StringRef FileName,
-                                const object::ELFFile<ELFT> &Obj, Triple TT,
-                                SubtargetFeatures Features)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, loongarch::getEdgeKindName) {}
+                                const object::ELFFile<ELFT> &Obj,
+                                std::shared_ptr<orc::SymbolStringPool> SSP,
+                                Triple TT, SubtargetFeatures Features)
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  loongarch::getEdgeKindName) {}
 };
 
 Error buildTables_ELF_loongarch(LinkGraph &G) {
@@ -151,8 +153,8 @@ Error buildTables_ELF_loongarch(LinkGraph &G) {
 namespace llvm {
 namespace jitlink {
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_loongarch(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -170,7 +172,7 @@ createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) {
     auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
     return ELFLinkGraphBuilder_loongarch<object::ELF64LE>(
                (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
-               (*ELFObj)->makeTriple(), std::move(*Features))
+               std::move(SSP), (*ELFObj)->makeTriple(), std::move(*Features))
         .buildGraph();
   }
 
@@ -178,7 +180,7 @@ createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) {
          "Invalid triple for LoongArch ELF object file");
   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj);
   return ELFLinkGraphBuilder_loongarch<object::ELF32LE>(
-             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
+             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), std::move(SSP),
              (*ELFObj)->makeTriple(), std::move(*Features))
       .buildGraph();
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
index 3b86250b60a44..83b51eb61e1cc 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
@@ -106,14 +106,14 @@ Symbol &createELFGOTHeader(LinkGraph &G,
   Symbol *TOCSymbol = nullptr;
 
   for (Symbol *Sym : G.defined_symbols())
-    if (LLVM_UNLIKELY(Sym->getName() == ELFTOCSymbolName)) {
+    if (LLVM_UNLIKELY(Sym->hasName() && *Sym->getName() == ELFTOCSymbolName)) {
       TOCSymbol = Sym;
       break;
     }
 
   if (LLVM_LIKELY(TOCSymbol == nullptr)) {
     for (Symbol *Sym : G.external_symbols())
-      if (Sym->getName() == ELFTOCSymbolName) {
+      if (Sym->hasName() && *Sym->getName() == ELFTOCSymbolName) {
         TOCSymbol = Sym;
         break;
       }
@@ -393,10 +393,12 @@ class ELFLinkGraphBuilder_ppc64
 
 public:
   ELFLinkGraphBuilder_ppc64(StringRef FileName,
-                            const object::ELFFile<ELFT> &Obj, Triple TT,
-                            SubtargetFeatures Features)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, ppc64::getEdgeKindName) {}
+                            const object::ELFFile<ELFT> &Obj,
+                            std::shared_ptr<orc::SymbolStringPool> SSP,
+                            Triple TT, SubtargetFeatures Features)
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  ppc64::getEdgeKindName) {}
 };
 
 template <llvm::endianness Endianness>
@@ -417,7 +419,8 @@ class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> {
 
   Error defineTOCBase(LinkGraph &G) {
     for (Symbol *Sym : G.defined_symbols()) {
-      if (LLVM_UNLIKELY(Sym->getName() == ELFTOCSymbolName)) {
+      if (LLVM_UNLIKELY(Sym->hasName() &&
+                        *Sym->getName() == ELFTOCSymbolName)) {
         TOCSymbol = Sym;
         return Error::success();
       }
@@ -427,7 +430,7 @@ class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> {
            "TOCSymbol should not be defined at this point");
 
     for (Symbol *Sym : G.external_symbols()) {
-      if (Sym->getName() == ELFTOCSymbolName) {
+      if (Sym->hasName() && *Sym->getName() == ELFTOCSymbolName) {
         TOCSymbol = Sym;
         break;
       }
@@ -463,7 +466,8 @@ class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> {
 
 template <llvm::endianness Endianness>
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer,
+                                   std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -480,7 +484,7 @@ createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
   using ELFT = object::ELFType<Endianness, true>;
   auto &ELFObjFile = cast<object::ELFObjectFile<ELFT>>(**ELFObj);
   return ELFLinkGraphBuilder_ppc64<Endianness>(
-             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
+             (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), std::move(SSP),
              (*ELFObj)->makeTriple(), std::move(*Features))
       .buildGraph();
 }
@@ -517,15 +521,16 @@ void link_ELF_ppc64(std::unique_ptr<LinkGraph> G,
 }
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer,
+                                   std::shared_ptr<orc::SymbolStringPool> SSP) {
   return createLinkGraphFromELFObject_ppc64<llvm::endianness::big>(
-      std::move(ObjectBuffer));
+      std::move(ObjectBuffer), std::move(SSP));
 }
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_ppc64le(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_ppc64le(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   return createLinkGraphFromELFObject_ppc64<llvm::endianness::little>(
-      std::move(ObjectBuffer));
+      std::move(ObjectBuffer), std::move(SSP));
 }
 
 /// jit-link the given object buffer, which must be a ELF ppc64 object file.
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
index 0cf548ede9389..71a0f14368ac6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp
@@ -927,14 +927,17 @@ class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder<ELFT> {
 
 public:
   ELFLinkGraphBuilder_riscv(StringRef FileName,
-                            const object::ELFFile<ELFT> &Obj, Triple TT,
-                            SubtargetFeatures Features)
-      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features),
-                                  FileName, riscv::getEdgeKindName) {}
+                            const object::ELFFile<ELFT> &Obj,
+                            std::shared_ptr<orc::SymbolStringPool> SSP,
+                            Triple TT, SubtargetFeatures Features)
+      : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),
+                                  std::move(Features), FileName,
+                                  riscv::getEdgeKindName) {}
 };
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer,
+                                   std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -952,7 +955,7 @@ createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) {
     auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
     return ELFLinkGraphBuilder_riscv<object::ELF64LE>(
                (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
-               (*ELFObj)->makeTriple(), std::move(*Features))
+               std::move(SSP), (*ELFObj)->makeTriple(), std::move(*Features))
         .buildGraph();
   } else {
     assert((*ELFObj)->getArch() == Triple::riscv32 &&
@@ -960,7 +963,7 @@ createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) {
     auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj);
     return ELFLinkGraphBuilder_riscv<object::ELF32LE>(
                (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),
-               (*ELFObj)->makeTriple(), std::move(*Features))
+               std::move(SSP), (*ELFObj)->makeTriple(), std::move(*Features))
         .buildGraph();
   }
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 44122726fb5c0..bccda8e90a1fb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -238,9 +238,10 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
 
 public:
   ELFLinkGraphBuilder_x86_64(StringRef FileName,
+                             std::shared_ptr<orc::SymbolStringPool> SSP,
                              const object::ELFFile<object::ELF64LE> &Obj,
                              SubtargetFeatures Features)
-      : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"),
+      : ELFLinkGraphBuilder(Obj, std::move(SSP), Triple("x86_64-unknown-linux"),
                             std::move(Features), FileName,
                             x86_64::getEdgeKindName) {}
 };
@@ -261,12 +262,12 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
 
 private:
   Symbol *GOTSymbol = nullptr;
-
   Error getOrCreateGOTSymbol(LinkGraph &G) {
     auto DefineExternalGOTSymbolIfPresent =
         createDefineExternalSectionStartAndEndSymbolsPass(
             [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
-              if (Sym.getName() == ELFGOTSymbolName)
+              if (Sym.getName() != nullptr &&
+                  *Sym.getName() == ELFGOTSymbolName)
                 if (auto *GOTSection = G.findSectionByName(
                         x86_64::GOTTableManager::getSectionName())) {
                   GOTSymbol = &Sym;
@@ -292,7 +293,7 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
 
       // Check for an existing defined symbol.
       for (auto *Sym : GOTSection->symbols())
-        if (Sym->getName() == ELFGOTSymbolName) {
+        if (Sym->getName() != nullptr && *Sym->getName() == ELFGOTSymbolName) {
           GOTSymbol = Sym;
           return Error::success();
         }
@@ -314,7 +315,7 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
     // we just need to point the GOT symbol at some address in this graph.
     if (!GOTSymbol) {
       for (auto *Sym : G.external_symbols()) {
-        if (Sym->getName() == ELFGOTSymbolName) {
+        if (*Sym->getName() == ELFGOTSymbolName) {
           auto Blocks = G.blocks();
           if (!Blocks.empty()) {
             G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
@@ -333,8 +334,8 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
   }
 };
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_x86_64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   LLVM_DEBUG({
     dbgs() << "Building jitlink graph for new input "
            << ObjectBuffer.getBufferIdentifier() << "...\n";
@@ -349,7 +350,7 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
     return Features.takeError();
 
   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
-  return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
+  return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), std::move(SSP),
                                     ELFObjFile.getELFFile(),
                                     std::move(*Features))
       .buildGraph();
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 9041dc3a52dcf..5edfbdb05f7b3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -85,6 +85,8 @@ const char *getScopeName(Scope S) {
     return "default";
   case Scope::Hidden:
     return "hidden";
+  case Scope::SideEffectsOnly:
+    return "side-effects-only";
   case Scope::Local:
     return "local";
   }
@@ -123,7 +125,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
      << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
      << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
      << (Sym.isLive() ? "live" : "dead") << "  -   "
-     << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>");
+     << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
   return OS;
 }
 
@@ -165,6 +167,16 @@ Section::~Section() {
     B->~Block();
 }
 
+LinkGraph::~LinkGraph() {
+  for (auto *Sym : AbsoluteSymbols) {
+    Sym->~Symbol();
+  }
+  for (auto *Sym : external_symbols()) {
+    Sym->~Symbol();
+  }
+  ExternalSymbols.clear();
+}
+
 std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
                                                SplitBlockCache *Cache) {
   assert(!Blocks.empty() && "Blocks must at least contain the original block");
@@ -479,22 +491,25 @@ PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
 }
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
+                          std::shared_ptr<orc::SymbolStringPool> SSP) {
   auto Magic = identify_magic(ObjectBuffer.getBuffer());
   switch (Magic) {
   case file_magic::macho_object:
-    return createLinkGraphFromMachOObject(ObjectBuffer);
+    return createLinkGraphFromMachOObject(ObjectBuffer, std::move(SSP));
   case file_magic::elf_relocatable:
-    return createLinkGraphFromELFObject(ObjectBuffer);
+    return createLinkGraphFromELFObject(ObjectBuffer, std::move(SSP));
   case file_magic::coff_object:
-    return createLinkGraphFromCOFFObject(ObjectBuffer);
+    return createLinkGraphFromCOFFObject(ObjectBuffer, std::move(SSP));
   default:
     return make_error<JITLinkError>("Unsupported file format");
   };
 }
 
-std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
-                                                    orc::SymbolMap Symbols) {
+std::unique_ptr<LinkGraph>
+absoluteSymbolsLinkGraph(const Triple &TT,
+                         std::shared_ptr<orc::SymbolStringPool> SSP,
+                         orc::SymbolMap Symbols) {
   unsigned PointerSize;
   endianness Endianness =
       TT.isLittleEndian() ? endianness::little : endianness::big;
@@ -516,8 +531,8 @@ std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
   static std::atomic<uint64_t> Counter = {0};
   auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
   auto G = std::make_unique<LinkGraph>(
-      "<Absolute Symbols " + std::to_string(Index) + ">", TT, PointerSize,
-      Endianness, /*GetEdgeKindName=*/nullptr);
+      "<Absolute Symbols " + std::to_string(Index) + ">", std::move(SSP), TT,
+      PointerSize, Endianness, /*GetEdgeKindName=*/nullptr);
   for (auto &[Name, Def] : Symbols) {
     auto &Sym =
         G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 254c04b198612..584b9f0067460 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -211,8 +211,7 @@ JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const {
   for (auto *Sym : G->external_symbols()) {
     assert(!Sym->getAddress() &&
            "External has already been assigned an address");
-    assert(Sym->getName() != StringRef() && Sym->getName() != "" &&
-           "Externals must be named");
+    assert(Sym->hasName() && "Externals must be named");
     SymbolLookupFlags LookupFlags =
         Sym->isWeaklyReferenced() ? SymbolLookupFlags::WeaklyReferencedSymbol
                                   : SymbolLookupFlags::RequiredSymbol;
@@ -253,7 +252,9 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
       }
       switch (Sym->getScope()) {
       case Scope::Local:
-        llvm_unreachable("External symbol should not have local linkage");
+      case Scope::SideEffectsOnly:
+        llvm_unreachable("External symbol should not have local or "
+                         "side-effects-only linkage");
       case Scope::Hidden:
         break;
       case Scope::Default:
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 42a77a52beef0..2e3323af98e94 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -144,6 +144,7 @@ orc::shared::AllocActions &BasicLayout::graphAllocActions() {
 }
 
 void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
+                                std::shared_ptr<orc::SymbolStringPool> SSP,
                                 const JITLinkDylib *JD, SegmentMap Segments,
                                 OnCreatedFunction OnCreated) {
 
@@ -155,7 +156,7 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
       "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",
       "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};
 
-  auto G = std::make_unique<LinkGraph>("", Triple(), 0,
+  auto G = std::make_unique<LinkGraph>("", std::move(SSP), Triple(), 0,
                                        llvm::endianness::native, nullptr);
   orc::AllocGroupSmallMap<Block *> ContentBlocks;
 
@@ -201,11 +202,12 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
 }
 
 Expected<SimpleSegmentAlloc>
-SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
-                           SegmentMap Segments) {
+SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
+                           std::shared_ptr<orc::SymbolStringPool> SSP,
+                           const JITLinkDylib *JD, SegmentMap Segments) {
   std::promise<MSVCPExpected<SimpleSegmentAlloc>> AllocP;
   auto AllocF = AllocP.get_future();
-  Create(MemMgr, JD, std::move(Segments),
+  Create(MemMgr, std::move(SSP), JD, std::move(Segments),
          [&](Expected<SimpleSegmentAlloc> Result) {
            AllocP.set_value(std::move(Result));
          });
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
index f1181b59234e4..b9cfb8fb8fa96 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
@@ -25,7 +25,8 @@ namespace llvm {
 namespace jitlink {
 
 Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) {
+createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer,
+                               std::shared_ptr<orc::SymbolStringPool> SSP) {
   StringRef Data = ObjectBuffer.getBuffer();
   if (Data.size() < 4)
     return make_error<JITLinkError>("Truncated MachO buffer \"" +
@@ -61,9 +62,10 @@ createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) {
 
     switch (CPUType) {
     case MachO::CPU_TYPE_ARM64:
-      return createLinkGraphFromMachOObject_arm64(ObjectBuffer);
+      return createLinkGraphFromMachOObject_arm64(ObjectBuffer, std::move(SSP));
     case MachO::CPU_TYPE_X86_64:
-      return createLinkGraphFromMachOObject_x86_64(ObjectBuffer);
+      return createLinkGraphFromMachOObject_x86_64(ObjectBuffer,
+                                                   std::move(SSP));
     }
     return make_error<JITLinkError>("MachO-64 CPU type not valid");
   } else
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index 15d679bcbb5df..0e7ecf6cc1383 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -49,13 +49,14 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() {
 }
 
 MachOLinkGraphBuilder::MachOLinkGraphBuilder(
-    const object::MachOObjectFile &Obj, Triple TT, SubtargetFeatures Features,
+    const object::MachOObjectFile &Obj,
+    std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
+    SubtargetFeatures Features,
     LinkGraph::GetEdgeKindNameFunction GetEdgeKindName)
-    : Obj(Obj),
-      G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()),
-                                    std::move(TT), std::move(Features),
-                                    getPointerSize(Obj), getEndianness(Obj),
-                                    std::move(GetEdgeKindName))) {
+    : Obj(Obj), G(std::make_unique<LinkGraph>(
+                    std::string(Obj.getFileName()), std::move(SSP),
+                    std::move(TT), std::move(Features), getPointerSize(Obj),
+                    getEndianness(Obj), std::move(GetEdgeKindName))) {
   auto &MachHeader = Obj.getHeader64();
   SubsectionsViaSymbols = MachHeader.flags & MachO::MH_SUBSECTIONS_VIA_SYMBOLS;
 }
@@ -319,9 +320,8 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() {
       }
     }
 
-    IndexToSymbol[SymbolIndex] =
-        &createNormalizedSymbol(*Name, Value, Type, Sect, Desc,
-                                getLinkage(Desc), getScope(*Name, Type));
+    IndexToSymbol[SymbolIndex] = &createNormalizedSymbol(
+        Name, Value, Type, Sect, Desc, getLinkage(Desc), getScope(*Name, Type));
   }
 
   return Error::success();
@@ -587,7 +587,7 @@ Symbol &MachOLinkGraphBuilder::createStandardGraphSymbol(NormalizedSymbol &NSym,
     if (!NSym.Name)
       dbgs() << "<anonymous symbol>";
     else
-      dbgs() << NSym.Name;
+      dbgs() << *NSym.Name;
     if (IsText)
       dbgs() << " [text]";
     if (IsNoDeadStrip)
@@ -726,7 +726,7 @@ Error MachOLinkGraphBuilder::graphifyCStringSection(
         LLVM_DEBUG({
           dbgs() << "      Adding symbol for c-string block " << B.getRange()
                  << ": "
-                 << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>")
+                 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>")
                  << " at offset " << formatv("{0:x}", Sym.getOffset()) << "\n";
         });
 
@@ -826,7 +826,7 @@ Error CompactUnwindSplitter::operator()(LinkGraph &G) {
           LLVM_DEBUG({
             dbgs() << "    Updating compact unwind record at "
                    << CURec->getAddress() << " to point to "
-                   << (E.getTarget().hasName() ? E.getTarget().getName()
+                   << (E.getTarget().hasName() ? *E.getTarget().getName()
                                                : StringRef())
                    << " (at " << E.getTarget().getAddress() << ")\n";
           });
@@ -835,7 +835,7 @@ Error CompactUnwindSplitter::operator()(LinkGraph &G) {
             return make_error<JITLinkError>(
                 "Error adding keep-alive edge for compact unwind record at " +
                 formatv("{0:x}", CURec->getAddress()) + ": target " +
-                E.getTarget().getName() + " is an external symbol");
+                *E.getTarget().getName() + " is an external symbol");
           auto &TgtBlock = E.getTarget().getBlock();
           auto &CURecSym =
               G.addAnonymousSymbol(*CURec, 0, CURecordSize, false, false);
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
index a4ae0ac1ecfc0..6afa01250f62d 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -83,10 +83,10 @@ class MachOLinkGraphBuilder {
 
   using SectionParserFunction = std::function<Error(NormalizedSection &S)>;
 
-  MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT,
+  MachOLinkGraphBuilder(const object::MachOObjectFile &Obj,
+                        std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
                         SubtargetFeatures Features,
                         LinkGraph::GetEdgeKindNameFunction GetEdgeKindName);
-
   LinkGraph &getGraph() const { return *G; }
 
   const object::MachOObjectFile &getObject() const { return Obj; }
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 6a947828a5e61..8b43dc97e49a8 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -27,9 +27,10 @@ namespace {
 class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
 public:
   MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj,
+                              std::shared_ptr<orc::SymbolStringPool> SSP,
                               SubtargetFeatures Features)
-      : MachOLinkGraphBuilder(Obj, getObjectTriple(Obj), std::move(Features),
-                              aarch64::getEdgeKindName),
+      : MachOLinkGraphBuilder(Obj, std::move(SSP), getObjectTriple(Obj),
+                              std::move(Features), aarch64::getEdgeKindName),
         NumSymbols(Obj.getSymtabLoadCommand().nsyms) {}
 
 private:
@@ -580,8 +581,8 @@ class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> {
   uint64_t NullValue = 0;
 };
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_arm64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer);
   if (!MachOObj)
     return MachOObj.takeError();
@@ -590,7 +591,8 @@ createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer) {
   if (!Features)
     return Features.takeError();
 
-  return MachOLinkGraphBuilder_arm64(**MachOObj, std::move(*Features))
+  return MachOLinkGraphBuilder_arm64(**MachOObj, std::move(SSP),
+                                     std::move(*Features))
       .buildGraph();
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 2c69d61316a8a..113b1953e36a6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -27,8 +27,10 @@ namespace {
 class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
 public:
   MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj,
+                               std::shared_ptr<orc::SymbolStringPool> SSP,
                                SubtargetFeatures Features)
-      : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin"),
+      : MachOLinkGraphBuilder(Obj, std::move(SSP),
+                              Triple("x86_64-apple-darwin"),
                               std::move(Features), x86_64::getEdgeKindName) {}
 
 private:
@@ -483,8 +485,8 @@ class MachOJITLinker_x86_64 : public JITLinker<MachOJITLinker_x86_64> {
   }
 };
 
-Expected<std::unique_ptr<LinkGraph>>
-createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer) {
+Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64(
+    MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
   auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer);
   if (!MachOObj)
     return MachOObj.takeError();
@@ -493,7 +495,8 @@ createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer) {
   if (!Features)
     return Features.takeError();
 
-  return MachOLinkGraphBuilder_x86_64(**MachOObj, std::move(*Features))
+  return MachOLinkGraphBuilder_x86_64(**MachOObj, std::move(SSP),
+                                      std::move(*Features))
       .buildGraph();
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h
index 6e325f92bafbe..a073ddcae2949 100644
--- a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h
@@ -114,8 +114,8 @@ class PerGraphGOTAndPLTStubsBuilder {
 private:
   BuilderImplT &impl() { return static_cast<BuilderImplT &>(*this); }
 
-  DenseMap<StringRef, Symbol *> GOTEntries;
-  DenseMap<StringRef, Symbol *> PLTStubs;
+  DenseMap<orc::SymbolStringPtr, Symbol *> GOTEntries;
+  DenseMap<orc::SymbolStringPtr, Symbol *> PLTStubs;
 };
 
 } // end namespace jitlink
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
index d183498f30a68..7b255717f2383 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
@@ -852,7 +852,7 @@ bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
 
   Symbol &Target = E.getTarget();
   assert(Target.hasName() && "Edge cannot point to anonymous target");
-  auto [Slot, NewStub] = getStubMapSlot(Target.getName());
+  auto [Slot, NewStub] = getStubMapSlot(*Target.getName());
 
   if (NewStub) {
     if (!StubsSection)
@@ -896,7 +896,7 @@ bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
 
   Symbol &Target = E.getTarget();
   assert(Target.hasName() && "Edge cannot point to anonymous target");
-  Symbol *&StubSymbol = getStubSymbolSlot(Target.getName(), MakeThumb);
+  Symbol *&StubSymbol = getStubSymbolSlot(*Target.getName(), MakeThumb);
 
   if (!StubSymbol) {
     if (!StubsSection)
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index a79dbd5e4494f..968ed217d8a96 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -29,6 +29,11 @@ const char PointerJumpStubContent[12] = {
     0x00, 0x02, 0x1f, (char)0xd6u  // BR  x16
 };
 
+const char ReentryTrampolineContent[8] = {
+    (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
+    0x00,       0x00, 0x00,       (char)0x94  // BL
+};
+
 const char *getEdgeKindName(Edge::Kind R) {
   switch (R) {
   case Pointer64:
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index ebfbeb990180f..5615ad94006d3 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -29,6 +29,7 @@ add_llvm_component_library(LLVMOrcJIT
   IRTransformLayer.cpp
   IRPartitionLayer.cpp
   JITTargetMachineBuilder.cpp
+  JITLinkReentryTrampolines.cpp
   LazyObjectLinkingLayer.cpp
   LazyReexports.cpp
   Layer.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
index 2176acc5bba80..a435997478151 100644
--- a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
@@ -68,8 +68,8 @@ class COFFHeaderMaterializationUnit : public MaterializationUnit {
     }
 
     auto G = std::make_unique<jitlink::LinkGraph>(
-        "<COFFHeaderMU>", TT, PointerSize, Endianness,
-        jitlink::getGenericEdgeKindName);
+        "<COFFHeaderMU>", CP.getExecutionSession().getSymbolStringPool(), TT,
+        PointerSize, Endianness, jitlink::getGenericEdgeKindName);
     auto &HeaderSection = G->createSection("__header", MemProt::Read);
     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
 
@@ -399,7 +399,7 @@ COFFPlatform::COFFPlatform(
       OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
       StaticVCRuntime(StaticVCRuntime),
       COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
 
   Bootstrapping.store(true);
   ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
@@ -792,7 +792,7 @@ Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
     jitlink::LinkGraph &G, MaterializationResponsibility &MR,
     bool IsBootstraping) {
   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
-    return Sym->getName() == *CP.COFFHeaderStartSymbol;
+    return *Sym->getName() == *CP.COFFHeaderStartSymbol;
   });
   assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
 
@@ -862,9 +862,9 @@ Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
       // to the first block.
       if (!InitSym) {
         auto &B = **InitSection.blocks().begin();
-        InitSym = &G.addDefinedSymbol(B, 0, *InitSymName, B.getSize(),
-                                      jitlink::Linkage::Strong,
-                                      jitlink::Scope::Default, false, true);
+        InitSym = &G.addDefinedSymbol(
+            B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
+            jitlink::Scope::SideEffectsOnly, false, true);
       }
 
       // Add keep-alive edges to anonymous symbols in all other init blocks.
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index 7804199364883..6a9ebb41e79f6 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include <condition_variable>
 #include <future>
@@ -177,6 +178,27 @@ void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
      << ": " << Symbols;
 }
 
+void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
+  JD->getExecutionSession().lookup(
+      LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
+      SymbolLookupSet(Name), SymbolState::Ready,
+      [OnComplete = std::move(OnComplete)
+#ifndef NDEBUG
+           ,
+       Name = this->Name // Captured for the assert below only.
+#endif                   // NDEBUG
+  ](Expected<SymbolMap> Result) mutable {
+        if (Result) {
+          assert(Result->size() == 1 && "Unexpected number of results");
+          assert(Result->count(Name) &&
+                 "Result does not contain expected symbol");
+          OnComplete(Result->begin()->second);
+        } else
+          OnComplete(Result.takeError());
+      },
+      NoDependenciesToRegister);
+}
+
 AsynchronousSymbolQuery::AsynchronousSymbolQuery(
     const SymbolLookupSet &Symbols, SymbolState RequiredState,
     SymbolsResolvedCallback NotifyComplete)
@@ -938,7 +960,6 @@ Error JITDylib::resolve(MaterializationResponsibility &MR,
           auto &MI = MII->second;
           for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
             Q->notifySymbolMetRequiredState(Name, ResolvedSym);
-            Q->removeQueryDependence(*this, Name);
             if (Q->isComplete())
               CompletedQueries.insert(std::move(Q));
           }
@@ -1207,9 +1228,8 @@ void JITDylib::MaterializingInfo::removeQuery(
       PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
         return V.get() == &Q;
       });
-  assert(I != PendingQueries.end() &&
-         "Query is not attached to this MaterializingInfo");
-  PendingQueries.erase(I);
+  if (I != PendingQueries.end())
+    PendingQueries.erase(I);
 }
 
 JITDylib::AsynchronousSymbolQueryList
@@ -1456,10 +1476,9 @@ void JITDylib::installMaterializationUnit(
 void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
                                  const SymbolNameSet &QuerySymbols) {
   for (auto &QuerySymbol : QuerySymbols) {
-    assert(MaterializingInfos.count(QuerySymbol) &&
-           "QuerySymbol does not have MaterializingInfo");
-    auto &MI = MaterializingInfos[QuerySymbol];
-    MI.removeQuery(Q);
+    auto MII = MaterializingInfos.find(QuerySymbol);
+    if (MII != MaterializingInfos.end())
+      MII->second.removeQuery(Q);
   }
 }
 
@@ -1809,7 +1828,7 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
     if (R)
       PromisedResult.set_value(std::move(*R));
     else {
-      ErrorAsOutParameter _(&ResolutionError);
+      ErrorAsOutParameter _(ResolutionError);
       ResolutionError = R.takeError();
       PromisedResult.set_value(SymbolMap());
     }
@@ -1820,7 +1839,7 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
   Error ResolutionError = Error::success();
 
   auto NotifyComplete = [&](Expected<SymbolMap> R) {
-    ErrorAsOutParameter _(&ResolutionError);
+    ErrorAsOutParameter _(ResolutionError);
     if (R)
       Result = std::move(*R);
     else
@@ -1878,31 +1897,39 @@ ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
 Error ExecutionSession::registerJITDispatchHandlers(
     JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
 
-  auto TagAddrs = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}},
-                         SymbolLookupSet::fromMapKeys(
-                             WFs, SymbolLookupFlags::WeaklyReferencedSymbol));
-  if (!TagAddrs)
-    return TagAddrs.takeError();
+  auto TagSyms = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}},
+                        SymbolLookupSet::fromMapKeys(
+                            WFs, SymbolLookupFlags::WeaklyReferencedSymbol));
+  if (!TagSyms)
+    return TagSyms.takeError();
 
   // Associate tag addresses with implementations.
   std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
-  for (auto &KV : *TagAddrs) {
-    auto TagAddr = KV.second.getAddress();
+
+  // Check that no tags are being overwritten.
+  for (auto &[TagName, TagSym] : *TagSyms) {
+    auto TagAddr = TagSym.getAddress();
     if (JITDispatchHandlers.count(TagAddr))
-      return make_error<StringError>("Tag " + formatv("{0:x16}", TagAddr) +
-                                         " (for " + *KV.first +
+      return make_error<StringError>("Tag " + formatv("{0:x}", TagAddr) +
+                                         " (for " + *TagName +
                                          ") already registered",
                                      inconvertibleErrorCode());
-    auto I = WFs.find(KV.first);
+  }
+
+  // At this point we're guaranteed to succeed. Install the handlers.
+  for (auto &[TagName, TagSym] : *TagSyms) {
+    auto TagAddr = TagSym.getAddress();
+    auto I = WFs.find(TagName);
     assert(I != WFs.end() && I->second &&
            "JITDispatchHandler implementation missing");
-    JITDispatchHandlers[KV.second.getAddress()] =
+    JITDispatchHandlers[TagAddr] =
         std::make_shared<JITDispatchHandlerFunction>(std::move(I->second));
     LLVM_DEBUG({
-      dbgs() << "Associated function tag \"" << *KV.first << "\" ("
-             << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
+      dbgs() << "Associated function tag \"" << *TagName << "\" ("
+             << formatv("{0:x}", TagAddr) << ") with handler\n";
     });
   }
+
   return Error::success();
 }
 
@@ -2607,6 +2634,12 @@ void ExecutionSession::OL_completeLookup(
               LLVM_DEBUG(dbgs()
                          << "matched, symbol already in required state\n");
               Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol());
+
+              // If this symbol is in anything other than the Ready state then
+              // we need to track the dependence.
+              if (SymI->second.getState() != SymbolState::Ready)
+                Q->addQueryDependence(JD, Name);
+
               return true;
             }
 
@@ -3157,7 +3190,6 @@ void ExecutionSession::IL_makeEDUEmitted(
       Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
       if (Q->isComplete())
         Queries.insert(Q);
-      Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
     }
   }
 
@@ -3300,7 +3332,7 @@ ExecutionSession::IL_emit(MaterializationResponsibility &MR,
           continue;
         }
 
-        // If we get here thene Dep is Emitted. We need to look up its defining
+        // If we get here then Dep is Emitted. We need to look up its defining
         // EDU and add this EDU to the defining EDU's list of users (this means
         // creating an EDUInfos entry if the defining EDU doesn't have one
         // already).
@@ -3309,8 +3341,6 @@ ExecutionSession::IL_emit(MaterializationResponsibility &MR,
         auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
         assert(DepMI.DefiningEDU &&
                "Emitted symbol does not have a defining EDU");
-        assert(!DepMI.DefiningEDU->Dependencies.empty() &&
-               "Emitted symbol has empty dependencies (should be ready)");
         assert(DepMI.DependantEDUs.empty() &&
                "Already-emitted symbol has dependant EDUs?");
         auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
diff --git a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
index 9859e4a4c6fd1..180905346d561 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
@@ -149,9 +149,9 @@ class DebugObject {
 
   JITLinkMemoryManager &MemMgr;
   const JITLinkDylib *JD = nullptr;
+  ExecutionSession &ES;
 
 private:
-  ExecutionSession &ES;
   DebugObjectFlags Flags;
   FinalizedAlloc Alloc;
 };
@@ -234,7 +234,7 @@ static bool isDwarfSection(StringRef SectionName) {
 
 std::unique_ptr<WritableMemoryBuffer>
 ELFDebugObject::CopyBuffer(MemoryBufferRef Buffer, Error &Err) {
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
   size_t Size = Buffer.getBufferSize();
   StringRef Name = Buffer.getBufferIdentifier();
   if (auto Copy = WritableMemoryBuffer::getNewUninitMemBuffer(Size, Name)) {
@@ -331,8 +331,9 @@ Expected<SimpleSegmentAlloc> ELFDebugObject::finalizeWorkingMemory() {
   size_t Size = Buffer->getBufferSize();
 
   // Allocate working memory for debug object in read-only segment.
-  auto Alloc = SimpleSegmentAlloc::Create(
-      MemMgr, JD, {{MemProt::Read, {Size, Align(PageSize)}}});
+  auto Alloc =
+      SimpleSegmentAlloc::Create(MemMgr, ES.getSymbolStringPool(), JD,
+                                 {{MemProt::Read, {Size, Align(PageSize)}}});
   if (!Alloc)
     return Alloc;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
index de8d003408871..6ced8c76b037c 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
@@ -137,14 +137,6 @@ struct PrintSymbolMapElemsMatchingCLOpts {
 namespace llvm {
 namespace orc {
 
-raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) {
-  return OS << *Sym;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, NonOwningSymbolStringPtr Sym) {
-  return OS << *Sym;
-}
-
 raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) {
   return OS << printSequence(Symbols, '{', '}', PrintAll<SymbolStringPtr>());
 }
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
index e387b06ee9346..c08e52e943c92 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
@@ -214,8 +214,8 @@ class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase {
 
         Builder.addSymbol("", MachO::N_BNSYM, 1, 0, 0);
         StabSymbols.push_back(
-            {*Sym, Builder.addSymbol(Sym->getName(), SymType, 1, 0, 0),
-             Builder.addSymbol(Sym->getName(), SymType, 0, 0, 0)});
+            {*Sym, Builder.addSymbol(*Sym->getName(), SymType, 1, 0, 0),
+             Builder.addSymbol(*Sym->getName(), SymType, 0, 0, 0)});
         Builder.addSymbol("", MachO::N_ENSYM, 1, 0, 0);
       }
     }
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
index 767115f984c1f..7a970f3cccf46 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
@@ -86,7 +86,7 @@ constexpr StringRef RegisterPerfImplSymbolName =
 static PerfJITCodeLoadRecord
 getCodeLoadRecord(const Symbol &Sym, std::atomic<uint64_t> &CodeIndex) {
   PerfJITCodeLoadRecord Record;
-  auto Name = Sym.getName();
+  auto Name = *Sym.getName();
   auto Addr = Sym.getAddress();
   auto Size = Sym.getSize();
   Record.Prefix.Id = PerfJITRecordType::JIT_CODE_LOAD;
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
index de02a20524b9c..9715a503629bf 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
@@ -56,7 +56,7 @@ static VTuneMethodBatch getMethodBatch(LinkGraph &G, bool EmitDebugInfo) {
     Method.ParentMI = 0;
     Method.LoadAddr = Sym->getAddress();
     Method.LoadSize = Sym->getSize();
-    Method.NameSI = GetStringIdx(Sym->getName());
+    Method.NameSI = GetStringIdx(*Sym->getName());
     Method.ClassFileSI = 0;
     Method.SourceFileSI = 0;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index c48c73769955a..5f017d9979c3c 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -65,9 +65,9 @@ std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
     llvm_unreachable("Unrecognized architecture");
   }
 
-  return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
-                                              Endianness,
-                                              jitlink::getGenericEdgeKindName);
+  return std::make_unique<jitlink::LinkGraph>(
+      std::move(Name), MOP.getExecutionSession().getSymbolStringPool(), TT,
+      PointerSize, Endianness, jitlink::getGenericEdgeKindName);
 }
 
 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
@@ -188,8 +188,8 @@ class DSOHandleMaterializationUnit : public MaterializationUnit {
 
     // void *__dso_handle = &__dso_handle;
     auto G = std::make_unique<jitlink::LinkGraph>(
-        "<DSOHandleMU>", TT, PointerSize, Endianness,
-        jitlink::getGenericEdgeKindName);
+        "<DSOHandleMU>", ENP.getExecutionSession().getSymbolStringPool(), TT,
+        PointerSize, Endianness, jitlink::getGenericEdgeKindName);
     auto &DSOHandleSection =
         G->createSection(".data.__dso_handle", MemProt::Read);
     auto &DSOHandleBlock = G->createContentBlock(
@@ -349,6 +349,7 @@ ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
   SymbolAliasMap Aliases;
   addAliases(ES, Aliases, requiredCXXAliases());
   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
+  addAliases(ES, Aliases, standardLazyCompilationAliases());
   return Aliases;
 }
 
@@ -377,6 +378,16 @@ ELFNixPlatform::standardRuntimeUtilityAliases() {
       StandardRuntimeUtilityAliases);
 }
 
+ArrayRef<std::pair<const char *, const char *>>
+ELFNixPlatform::standardLazyCompilationAliases() {
+  static const std::pair<const char *, const char *>
+      StandardLazyCompilationAliases[] = {
+          {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
+
+  return ArrayRef<std::pair<const char *, const char *>>(
+      StandardLazyCompilationAliases);
+}
+
 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
   switch (TT.getArch()) {
   case Triple::x86_64:
@@ -396,7 +407,7 @@ ELFNixPlatform::ELFNixPlatform(
     : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
       ObjLinkingLayer(ObjLinkingLayer),
       DSOHandleSymbol(ES.intern("__dso_handle")) {
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
 
   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
@@ -663,14 +674,14 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::
 
   for (auto *Sym : G.defined_symbols()) {
     for (auto &RTSym : RuntimeSymbols) {
-      if (Sym->hasName() && Sym->getName() == RTSym.first) {
+      if (Sym->hasName() && *Sym->getName() == RTSym.first) {
         if (*RTSym.second)
           return make_error<StringError>(
               "Duplicate " + RTSym.first +
                   " detected during ELFNixPlatform bootstrap",
               inconvertibleErrorCode());
 
-        if (Sym->getName() == *MP.DSOHandleSymbol)
+        if (*Sym->getName() == *MP.DSOHandleSymbol)
           RegisterELFNixHeader = true;
 
         *RTSym.second = Sym->getAddress();
@@ -798,7 +809,7 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
                                             jitlink::LinkGraph &G) -> Error {
     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
-      return Sym->getName() == *MP.DSOHandleSymbol;
+      return Sym->getName() == MP.DSOHandleSymbol;
     });
     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
     {
@@ -894,9 +905,9 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
       // to the first block.
       if (!InitSym) {
         auto &B = **InitSection.blocks().begin();
-        InitSym = &G.addDefinedSymbol(B, 0, *InitSymName, B.getSize(),
-                                      jitlink::Linkage::Strong,
-                                      jitlink::Scope::Default, false, true);
+        InitSym = &G.addDefinedSymbol(
+            B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
+            jitlink::Scope::SideEffectsOnly, false, true);
       }
 
       // Add keep-alive edges to anonymous symbols in all other init blocks.
@@ -997,12 +1008,17 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
 
 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
     jitlink::LinkGraph &G, JITDylib &JD) {
-
+  auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");
+  auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");
   for (auto *Sym : G.external_symbols()) {
-    if (Sym->getName() == "__tls_get_addr") {
-      Sym->setName("___orc_rt_elfnix_tls_get_addr");
-    } else if (Sym->getName() == "__tlsdesc_resolver") {
-      Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
+    if (Sym->getName() == TLSGetAddrSymbolName) {
+      auto TLSGetAddr =
+          MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");
+      Sym->setName(std::move(TLSGetAddr));
+    } else if (Sym->getName() == TLSDescResolveSymbolName) {
+      auto TLSGetAddr =
+          MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");
+      Sym->setName(std::move(TLSGetAddr));
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
index 833be826f8aed..a44da22cb3512 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
@@ -110,7 +110,7 @@ Error EPCTrampolinePool::grow() {
   auto &EPC = EPCIU.getExecutorProcessControl();
   auto PageSize = EPC.getPageSize();
   auto Alloc = SimpleSegmentAlloc::Create(
-      EPC.getMemMgr(), nullptr,
+      EPC.getMemMgr(), EPC.getSymbolStringPool(), nullptr,
       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
   if (!Alloc)
     return Alloc.takeError();
@@ -294,10 +294,10 @@ EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
   assert(ABI && "ABI can not be null");
   auto ResolverSize = ABI->getResolverCodeSize();
 
-  auto Alloc =
-      SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
-                                 {{MemProt::Read | MemProt::Exec,
-                                   {ResolverSize, Align(EPC.getPageSize())}}});
+  auto Alloc = SimpleSegmentAlloc::Create(
+      EPC.getMemMgr(), EPC.getSymbolStringPool(), nullptr,
+      {{MemProt::Read | MemProt::Exec,
+        {ResolverSize, Align(EPC.getPageSize())}}});
 
   if (!Alloc)
     return Alloc.takeError();
@@ -363,7 +363,7 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
     auto PtrProt = MemProt::Read | MemProt::Write;
 
     auto Alloc = SimpleSegmentAlloc::Create(
-        EPC.getMemMgr(), nullptr,
+        EPC.getMemMgr(), EPC.getSymbolStringPool(), nullptr,
         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
 
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index edb49a41ecd12..58cd05cd88ba1 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -13,6 +13,7 @@
 #include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"
 #include "llvm/ExecutionEngine/Orc/MachO.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -467,7 +468,7 @@ StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
     GetObjectFileInterface GetObjFileInterface, Error &Err)
     : L(L), GetObjFileInterface(std::move(GetObjFileInterface)),
       ArchiveBuffer(std::move(ArchiveBuffer)), Archive(std::move(Archive)) {
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
   if (!this->GetObjFileInterface)
     this->GetObjFileInterface = getObjectFileInterface;
   if (!Err)
@@ -559,8 +560,8 @@ DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) {
     return Endianness.takeError();
 
   auto G = std::make_unique<jitlink::LinkGraph>(
-      "<DLLIMPORT_STUBS>", TT, *PointerSize, *Endianness,
-      jitlink::getGenericEdgeKindName);
+      "<DLLIMPORT_STUBS>", ES.getSymbolStringPool(), TT, *PointerSize,
+      *Endianness, jitlink::getGenericEdgeKindName);
   jitlink::Section &Sec =
       G->createSection(getSectionName(), MemProt::Read | MemProt::Exec);
 
@@ -572,9 +573,7 @@ DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) {
     // Create __imp_ symbol
     jitlink::Symbol &Ptr =
         jitlink::x86_64::createAnonymousPointer(*G, Sec, &Target);
-    auto NameCopy = G->allocateContent(Twine(getImpPrefix()) + *KV.first);
-    StringRef NameCopyRef = StringRef(NameCopy.data(), NameCopy.size());
-    Ptr.setName(NameCopyRef);
+    Ptr.setName(G->intern((Twine(getImpPrefix()) + *KV.first).str()));
     Ptr.setLinkage(jitlink::Linkage::Strong);
     Ptr.setScope(jitlink::Scope::Default);
 
diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
index 5a08083497d0d..8dd479a64f1b5 100644
--- a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
@@ -27,8 +27,8 @@ void JITLinkRedirectableSymbolManager::emitRedirectableSymbols(
   Triple TT = ES.getTargetTriple();
 
   auto G = std::make_unique<jitlink::LinkGraph>(
-      ("<indirect stubs graph #" + Twine(++StubGraphIdx) + ">").str(), TT,
-      TT.isArch64Bit() ? 8 : 4,
+      ("<indirect stubs graph #" + Twine(++StubGraphIdx) + ">").str(),
+      ES.getSymbolStringPool(), TT, TT.isArch64Bit() ? 8 : 4,
       TT.isLittleEndian() ? endianness::little : endianness::big,
       jitlink::getGenericEdgeKindName);
   auto &PointerSection =
@@ -46,10 +46,10 @@ void JITLinkRedirectableSymbolManager::emitRedirectableSymbols(
 
     auto PtrName = ES.intern((*Name + StubSuffix).str());
     auto &Ptr = AnonymousPtrCreator(*G, PointerSection, TargetSym, 0);
-    Ptr.setName(*PtrName);
+    Ptr.setName(PtrName);
     Ptr.setScope(jitlink::Scope::Hidden);
     auto &Stub = PtrJumpStubCreator(*G, StubsSection, Ptr);
-    Stub.setName(*Name);
+    Stub.setName(Name);
     Stub.setScope(jitlink::Scope::Default);
     NewSymbols[std::move(PtrName)] = JITSymbolFlags();
   }
diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
new file mode 100644
index 0000000000000..1f6ec783b6c1d
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
@@ -0,0 +1,184 @@
+//===----- JITLinkReentryTrampolines.cpp -- JITLink-based trampoline- -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
+
+#include "llvm/ExecutionEngine/JITLink/aarch64.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+
+#include <memory>
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::jitlink;
+
+namespace {
+constexpr StringRef ReentryFnName = "__orc_rt_reenter";
+constexpr StringRef ReentrySectionName = "__orc_stubs";
+} // namespace
+
+namespace llvm::orc {
+
+class JITLinkReentryTrampolines::TrampolineAddrScraperPlugin
+    : public ObjectLinkingLayer::Plugin {
+public:
+  void modifyPassConfig(MaterializationResponsibility &MR,
+                        jitlink::LinkGraph &G,
+                        jitlink::PassConfiguration &Config) override {
+    Config.PreFixupPasses.push_back(
+        [this](LinkGraph &G) { return recordTrampolineAddrs(G); });
+  }
+
+  Error notifyFailed(MaterializationResponsibility &MR) override {
+    return Error::success();
+  }
+
+  Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+    return Error::success();
+  }
+
+  void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
+                                   ResourceKey SrcKey) override {}
+
+  void registerGraph(LinkGraph &G,
+                     std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs) {
+    std::lock_guard<std::mutex> Lock(M);
+    assert(!PendingAddrs.count(&G) && "Duplicate registration");
+    PendingAddrs[&G] = std::move(Addrs);
+  }
+
+  Error recordTrampolineAddrs(LinkGraph &G) {
+    std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs;
+    {
+      std::lock_guard<std::mutex> Lock(M);
+      auto I = PendingAddrs.find(&G);
+      if (I == PendingAddrs.end())
+        return Error::success();
+      Addrs = std::move(I->second);
+      PendingAddrs.erase(I);
+    }
+
+    auto *Sec = G.findSectionByName(ReentrySectionName);
+    assert(Sec && "Reentry graph missing reentry section");
+    assert(!Sec->empty() && "Reentry graph is empty");
+
+    for (auto *Sym : Sec->symbols())
+      if (!Sym->hasName())
+        Addrs->push_back({Sym->getAddress(), JITSymbolFlags()});
+
+    return Error::success();
+  }
+
+private:
+  std::mutex M;
+  DenseMap<LinkGraph *, std::shared_ptr<std::vector<ExecutorSymbolDef>>>
+      PendingAddrs;
+};
+
+Expected<std::unique_ptr<JITLinkReentryTrampolines>>
+JITLinkReentryTrampolines::Create(ObjectLinkingLayer &ObjLinkingLayer) {
+
+  EmitTrampolineFn EmitTrampoline;
+
+  switch (ObjLinkingLayer.getExecutionSession().getTargetTriple().getArch()) {
+  case Triple::aarch64:
+    EmitTrampoline = aarch64::createAnonymousReentryTrampoline;
+    break;
+  default:
+    return make_error<StringError>("Architecture not supported",
+                                   inconvertibleErrorCode());
+  }
+
+  return std::make_unique<JITLinkReentryTrampolines>(ObjLinkingLayer,
+                                                     std::move(EmitTrampoline));
+}
+
+JITLinkReentryTrampolines::JITLinkReentryTrampolines(
+    ObjectLinkingLayer &ObjLinkingLayer, EmitTrampolineFn EmitTrampoline)
+    : ObjLinkingLayer(ObjLinkingLayer),
+      EmitTrampoline(std::move(EmitTrampoline)) {
+  auto TAS = std::make_shared<TrampolineAddrScraperPlugin>();
+  TrampolineAddrScraper = TAS.get();
+  ObjLinkingLayer.addPlugin(std::move(TAS));
+}
+
+void JITLinkReentryTrampolines::emit(ResourceTrackerSP RT,
+                                     size_t NumTrampolines,
+                                     OnTrampolinesReadyFn OnTrampolinesReady) {
+
+  if (NumTrampolines == 0)
+    return OnTrampolinesReady(std::vector<ExecutorSymbolDef>());
+
+  JITDylibSP JD(&RT->getJITDylib());
+  auto &ES = ObjLinkingLayer.getExecutionSession();
+  Triple TT = ES.getTargetTriple();
+
+  auto ReentryGraphSym =
+      ES.intern(("__orc_reentry_graph_#" + Twine(++ReentryGraphIdx)).str());
+
+  auto G = std::make_unique<jitlink::LinkGraph>(
+      (*ReentryGraphSym).str(), ES.getSymbolStringPool(), TT,
+      TT.isArch64Bit() ? 8 : 4,
+      TT.isLittleEndian() ? endianness::little : endianness::big,
+      jitlink::getGenericEdgeKindName);
+
+  auto &ReentryFnSym = G->addExternalSymbol(ReentryFnName, 0, false);
+
+  auto &ReentrySection =
+      G->createSection(ReentrySectionName, MemProt::Exec | MemProt::Read);
+
+  for (size_t I = 0; I != NumTrampolines; ++I)
+    EmitTrampoline(*G, ReentrySection, ReentryFnSym).setLive(true);
+
+  auto &FirstBlock = **ReentrySection.blocks().begin();
+  G->addDefinedSymbol(FirstBlock, 0, *ReentryGraphSym, FirstBlock.getSize(),
+                      Linkage::Strong, Scope::SideEffectsOnly, true, true);
+
+  auto TrampolineAddrs = std::make_shared<std::vector<ExecutorSymbolDef>>();
+  TrampolineAddrScraper->registerGraph(*G, TrampolineAddrs);
+
+  // Add Graph via object linking layer.
+  if (auto Err = ObjLinkingLayer.add(std::move(RT), std::move(G)))
+    return OnTrampolinesReady(std::move(Err));
+
+  // Trigger graph emission.
+  ES.lookup(
+      LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
+      SymbolLookupSet(ReentryGraphSym,
+                      SymbolLookupFlags::WeaklyReferencedSymbol),
+      SymbolState::Ready,
+      [OnTrampolinesReady = std::move(OnTrampolinesReady),
+       TrampolineAddrs =
+           std::move(TrampolineAddrs)](Expected<SymbolMap> Result) mutable {
+        if (Result)
+          OnTrampolinesReady(std::move(*TrampolineAddrs));
+        else
+          OnTrampolinesReady(Result.takeError());
+      },
+      NoDependenciesToRegister);
+}
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
+                                  RedirectableSymbolManager &RSMgr,
+                                  JITDylib &PlatformJD) {
+  auto JLT = JITLinkReentryTrampolines::Create(ObjLinkingLayer);
+  if (!JLT)
+    return JLT.takeError();
+
+  return LazyReexportsManager::Create(
+      [JLT = std::move(*JLT)](ResourceTrackerSP RT, size_t NumTrampolines,
+                              LazyReexportsManager::OnTrampolinesReadyFn
+                                  OnTrampolinesReady) mutable {
+        JLT->emit(std::move(RT), NumTrampolines, std::move(OnTrampolinesReady));
+      },
+      RSMgr, PlatformJD);
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index b49c80b9287ed..59bd95e96167f 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -1006,7 +1006,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S,
 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
     : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
 
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
 
   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
index 4dc2f2efd7e47..543337eb60f5d 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
@@ -60,10 +60,10 @@ class LazyObjectLinkingLayer::RenamerPlugin
     for (auto *Sym : G.defined_symbols()) {
       if (!Sym->hasName())
         continue;
-      auto I = SymsToRename.find(Sym->getName());
+      auto I = SymsToRename.find(*Sym->getName());
       if (I == SymsToRename.end())
         continue;
-      Sym->setName(G.allocateName(*I->second));
+      Sym->setName(G.intern(G.allocateName(*I->second)));
     }
 
     return Error::success();
@@ -71,10 +71,9 @@ class LazyObjectLinkingLayer::RenamerPlugin
 };
 
 LazyObjectLinkingLayer::LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
-                                               LazyCallThroughManager &LCTMgr,
-                                               RedirectableSymbolManager &RSMgr)
+                                               LazyReexportsManager &LRMgr)
     : ObjectLayer(BaseLayer.getExecutionSession()), BaseLayer(BaseLayer),
-      LCTMgr(LCTMgr), RSMgr(RSMgr) {
+      LRMgr(LRMgr) {
   BaseLayer.addPlugin(std::make_unique<RenamerPlugin>());
 }
 
@@ -101,8 +100,7 @@ Error LazyObjectLinkingLayer::add(ResourceTrackerSP RT,
     return Err;
 
   auto &JD = RT->getJITDylib();
-  return JD.define(lazyReexports(LCTMgr, RSMgr, JD, std::move(LazySymbols)),
-                   std::move(RT));
+  return JD.define(lazyReexports(LRMgr, std::move(LazySymbols)), std::move(RT));
 }
 
 void LazyObjectLinkingLayer::emit(
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 15c5f79fdbd3c..7a7e5d13ce03f 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -8,7 +8,9 @@
 
 #include "llvm/ExecutionEngine/Orc/LazyReexports.h"
 
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
+#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
 #include "llvm/TargetParser/Triple.h"
 
 #define DEBUG_TYPE "orc"
@@ -229,5 +231,167 @@ LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
   return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
 }
 
+class LazyReexportsManager::MU : public MaterializationUnit {
+public:
+  MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)
+      : MaterializationUnit(getInterface(Reexports)), LRMgr(LRMgr),
+        Reexports(std::move(Reexports)) {}
+
+private:
+  Interface getInterface(const SymbolAliasMap &Reexports) {
+    SymbolFlagsMap SF;
+    for (auto &[Alias, AI] : Reexports)
+      SF[Alias] = AI.AliasFlags;
+    return {std::move(SF), nullptr};
+  }
+
+  StringRef getName() const override { return "LazyReexportsManager::MU"; }
+
+  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+    LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));
+  }
+
+  void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
+    Reexports.erase(Name);
+  }
+
+  LazyReexportsManager &LRMgr;
+  SymbolAliasMap Reexports;
+};
+
+class LazyReexportsManager::Plugin : public ObjectLinkingLayer::Plugin {
+public:
+  void modifyPassConfig(MaterializationResponsibility &MR,
+                        jitlink::LinkGraph &G,
+                        jitlink::PassConfiguration &Config) override {}
+
+  Error notifyFailed(MaterializationResponsibility &MR) override {
+    return Error::success();
+  }
+
+  Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+    return Error::success();
+  }
+
+  void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
+                                   ResourceKey SrcKey) override {}
+
+private:
+  std::mutex M;
+};
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+LazyReexportsManager::Create(EmitTrampolinesFn EmitTrampolines,
+                             RedirectableSymbolManager &RSMgr,
+                             JITDylib &PlatformJD) {
+  Error Err = Error::success();
+  std::unique_ptr<LazyReexportsManager> LRM(new LazyReexportsManager(
+      std::move(EmitTrampolines), RSMgr, PlatformJD, Err));
+  if (Err)
+    return std::move(Err);
+  return std::move(LRM);
+}
+
+LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
+                                           RedirectableSymbolManager &RSMgr,
+                                           JITDylib &PlatformJD, Error &Err)
+    : EmitTrampolines(std::move(EmitTrampolines)), RSMgr(RSMgr) {
+
+  using namespace shared;
+
+  ErrorAsOutParameter _(&Err);
+
+  auto &ES = PlatformJD.getExecutionSession();
+  ExecutionSession::JITDispatchHandlerAssociationMap WFs;
+
+  WFs[ES.intern("__orc_rt_resolve_tag")] =
+      ES.wrapAsyncWithSPS<SPSExpected<SPSExecutorSymbolDef>(SPSExecutorAddr)>(
+          this, &LazyReexportsManager::resolve);
+
+  Err = ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
+}
+
+std::unique_ptr<MaterializationUnit>
+LazyReexportsManager::createLazyReexports(SymbolAliasMap Reexports) {
+  return std::make_unique<MU>(*this, std::move(Reexports));
+}
+
+void LazyReexportsManager::emitReentryTrampolines(
+    std::unique_ptr<MaterializationResponsibility> MR,
+    SymbolAliasMap Reexports) {
+  size_t NumTrampolines = Reexports.size();
+  auto RT = MR->getResourceTracker();
+  EmitTrampolines(
+      std::move(RT), NumTrampolines,
+      [this, MR = std::move(MR), Reexports = std::move(Reexports)](
+          Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) mutable {
+        emitRedirectableSymbols(std::move(MR), std::move(Reexports),
+                                std::move(ReentryPoints));
+      });
+}
+
+void LazyReexportsManager::emitRedirectableSymbols(
+    std::unique_ptr<MaterializationResponsibility> MR, SymbolAliasMap Reexports,
+    Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {
+
+  if (!ReentryPoints) {
+    MR->getExecutionSession().reportError(ReentryPoints.takeError());
+    MR->failMaterialization();
+    return;
+  }
+
+  assert(Reexports.size() == ReentryPoints->size() &&
+         "Number of reentry points doesn't match number of reexports");
+
+  // Bind entry points to names.
+  SymbolMap Redirs;
+  {
+    std::lock_guard<std::mutex> Lock(M);
+    size_t I = 0;
+    for (auto &[Name, AI] : Reexports) {
+      const auto &ReentryPoint = (*ReentryPoints)[I++];
+      Redirs[Name] = ReentryPoint;
+      CallThroughs[ReentryPoint.getAddress()] = {Name, AI.Aliasee,
+                                                 &MR->getTargetJITDylib()};
+    }
+  }
+
+  RSMgr.emitRedirectableSymbols(std::move(MR), std::move(Redirs));
+}
+
+void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,
+                                   ExecutorAddr ReentryStubAddr) {
+
+  CallThroughInfo LandingInfo;
+
+  {
+    std::lock_guard<std::mutex> Lock(M);
+
+    auto I = CallThroughs.find(ReentryStubAddr);
+    if (I == CallThroughs.end())
+      return SendResult(make_error<StringError>(
+          "Reentry address " + formatv("{0:x}", ReentryStubAddr) +
+              " not registered",
+          inconvertibleErrorCode()));
+    LandingInfo = I->second;
+  }
+
+  SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));
+  LandingSym.lookupAsync([this, JD = std::move(LandingInfo.JD),
+                          ReentryName = std::move(LandingInfo.Name),
+                          SendResult = std::move(SendResult)](
+                             Expected<ExecutorSymbolDef> Result) mutable {
+    if (Result) {
+      // FIXME: Make RedirectionManager operations async, then use the async
+      //        APIs here.
+      if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))
+        SendResult(std::move(Err));
+      else
+        SendResult(std::move(Result));
+    } else
+      SendResult(std::move(Result));
+  });
+}
+
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index b4326a17de3b7..0e8349711e6fe 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -108,9 +108,9 @@ std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP,
     llvm_unreachable("Unrecognized architecture");
   }
 
-  return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
-                                              Endianness,
-                                              jitlink::getGenericEdgeKindName);
+  return std::make_unique<jitlink::LinkGraph>(
+      std::move(Name), MOP.getExecutionSession().getSymbolStringPool(), TT,
+      PointerSize, Endianness, jitlink::getGenericEdgeKindName);
 }
 
 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
@@ -409,6 +409,7 @@ SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
   SymbolAliasMap Aliases;
   addAliases(ES, Aliases, requiredCXXAliases());
   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
+  addAliases(ES, Aliases, standardLazyCompilationAliases());
   return Aliases;
 }
 
@@ -436,6 +437,17 @@ MachOPlatform::standardRuntimeUtilityAliases() {
       StandardRuntimeUtilityAliases);
 }
 
+ArrayRef<std::pair<const char *, const char *>>
+MachOPlatform::standardLazyCompilationAliases() {
+  static const std::pair<const char *, const char *>
+      StandardLazyCompilationAliases[] = {
+          {"__orc_rt_reenter", "__orc_rt_sysv_reenter"},
+          {"__orc_rt_resolve_tag", "___orc_rt_resolve_tag"}};
+
+  return ArrayRef<std::pair<const char *, const char *>>(
+      StandardLazyCompilationAliases);
+}
+
 bool MachOPlatform::supportedTarget(const Triple &TT) {
   switch (TT.getArch()) {
   case Triple::aarch64:
@@ -477,7 +489,7 @@ MachOPlatform::MachOPlatform(
     : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
       ObjLinkingLayer(ObjLinkingLayer),
       BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) {
-  ErrorAsOutParameter _(&Err);
+  ErrorAsOutParameter _(Err);
   ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 
@@ -893,14 +905,14 @@ Error MachOPlatform::MachOPlatformPlugin::
 
   for (auto *Sym : G.defined_symbols()) {
     for (auto &RTSym : RuntimeSymbols) {
-      if (Sym->hasName() && Sym->getName() == RTSym.first) {
+      if (Sym->hasName() && *Sym->getName() == RTSym.first) {
         if (*RTSym.second)
           return make_error<StringError>(
               "Duplicate " + RTSym.first +
                   " detected during MachOPlatform bootstrap",
               inconvertibleErrorCode());
 
-        if (Sym->getName() == *MP.MachOHeaderStartSymbol)
+        if (Sym->getName() == MP.MachOHeaderStartSymbol)
           RegisterMachOHeader = true;
 
         *RTSym.second = Sym->getAddress();
@@ -936,7 +948,7 @@ Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd(
 Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
-    return Sym->getName() == *MP.MachOHeaderStartSymbol;
+    return Sym->getName() == MP.MachOHeaderStartSymbol;
   });
   assert(I != G.defined_symbols().end() && "Missing MachO header start symbol");
 
@@ -1001,9 +1013,9 @@ Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
       // to the first block.
       if (!InitSym) {
         auto &B = **InitSection->blocks().begin();
-        InitSym = &G.addDefinedSymbol(B, 0, *InitSymName, B.getSize(),
-                                      jitlink::Linkage::Strong,
-                                      jitlink::Scope::Default, false, true);
+        InitSym = &G.addDefinedSymbol(
+            B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
+            jitlink::Scope::SideEffectsOnly, false, true);
       }
 
       // Add keep-alive edges to anonymous symbols in all other init blocks.
@@ -1175,11 +1187,13 @@ Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags(
 
 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
     jitlink::LinkGraph &G, JITDylib &JD) {
-
+  auto TLVBootStrapSymbolName = G.intern("__tlv_bootstrap");
   // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
   for (auto *Sym : G.external_symbols())
-    if (Sym->getName() == "__tlv_bootstrap") {
-      Sym->setName("___orc_rt_macho_tlv_get_addr");
+    if (Sym->getName() == TLVBootStrapSymbolName) {
+      auto TLSGetADDR =
+          MP.getExecutionSession().intern("___orc_rt_macho_tlv_get_addr");
+      Sym->setName(std::move(TLSGetADDR));
       break;
     }
 
@@ -1499,19 +1513,19 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject(
       // Look for an existing __objc_imageinfo symbol.
       jitlink::Symbol *ObjCImageInfoSym = nullptr;
       for (auto *Sym : G.external_symbols())
-        if (Sym->getName() == ObjCImageInfoSymbolName) {
+        if (Sym->hasName() && *Sym->getName() == ObjCImageInfoSymbolName) {
           ObjCImageInfoSym = Sym;
           break;
         }
       if (!ObjCImageInfoSym)
         for (auto *Sym : G.absolute_symbols())
-          if (Sym->getName() == ObjCImageInfoSymbolName) {
+          if (Sym->hasName() && *Sym->getName() == ObjCImageInfoSymbolName) {
             ObjCImageInfoSym = Sym;
             break;
           }
       if (!ObjCImageInfoSym)
         for (auto *Sym : G.defined_symbols())
-          if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) {
+          if (Sym->hasName() && *Sym->getName() == ObjCImageInfoSymbolName) {
             ObjCImageInfoSym = Sym;
             std::optional<uint32_t> Flags;
             {
@@ -1652,11 +1666,11 @@ Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration(
       if (!Sym->hasName())
         continue;
 
-      auto I = ExistingStrings.find(Sym->getName());
+      auto I = ExistingStrings.find(*Sym->getName());
       if (I == ExistingStrings.end()) {
         auto &NameBlock = G.createMutableContentBlock(
-            *CStringSec, G.allocateCString(Sym->getName()), orc::ExecutorAddr(),
-            1, 0);
+            *CStringSec, G.allocateCString(*Sym->getName()),
+            orc::ExecutorAddr(), 1, 0);
         auto &SymbolNameSym = G.addAnonymousSymbol(
             NameBlock, 0, NameBlock.getSize(), false, true);
         JITSymTabInfo.push_back({Sym, &SymbolNameSym});
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
index 71221c2d3fce5..0b2cafb0bff13 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
@@ -129,7 +129,7 @@ getELFObjectFileSymbolInfo(ExecutionSession &ES,
     if (Sym.getBinding() == ELF::STB_GNU_UNIQUE)
       *SymFlags |= JITSymbolFlags::Weak;
 
-    I.SymbolFlags[ES.intern(*Name)] = std::move(*SymFlags);
+    I.SymbolFlags[ES.intern(std::move(*Name))] = std::move(*SymFlags);
   }
 
   SymbolStringPtr InitSymbol;
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 592941752805a..dd15499d4791f 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -65,6 +65,8 @@ JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) {
 
   if (Sym.getScope() == Scope::Default)
     Flags |= JITSymbolFlags::Exported;
+  else if (Sym.getScope() == Scope::SideEffectsOnly)
+    Flags |= JITSymbolFlags::MaterializationSideEffectsOnly;
 
   if (Sym.isCallable())
     Flags |= JITSymbolFlags::Callable;
@@ -98,8 +100,7 @@ class LinkGraphMaterializationUnit : public MaterializationUnit {
         return;
       assert(Sym->hasName() && "Anonymous non-local symbol?");
 
-      LGI.SymbolFlags[ES.intern(Sym->getName())] =
-          getJITSymbolFlagsForSymbol(*Sym);
+      LGI.SymbolFlags[Sym->getName()] = getJITSymbolFlagsForSymbol(*Sym);
     };
 
     for (auto *Sym : G.defined_symbols())
@@ -127,7 +128,7 @@ class LinkGraphMaterializationUnit : public MaterializationUnit {
 
   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
     for (auto *Sym : G->defined_symbols())
-      if (Sym->getName() == *Name) {
+      if (Sym->getName() == Name) {
         assert(Sym->getLinkage() == Linkage::Weak &&
                "Discarding non-weak definition");
         G->makeExternal(*Sym);
@@ -202,7 +203,7 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
         LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol;
         break;
       }
-      LookupSet.add(ES.intern(KV.first), LookupFlags);
+      LookupSet.add(KV.first, LookupFlags);
     }
 
     // OnResolve -- De-intern the symbols and pass the result to the linker.
@@ -213,7 +214,7 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       else {
         AsyncLookupResult LR;
         for (auto &KV : *Result)
-          LR[*KV.first] = KV.second;
+          LR[KV.first] = KV.second;
         LookupContinuation->run(std::move(LR));
       }
     };
@@ -229,35 +230,32 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
   }
 
   Error notifyResolved(LinkGraph &G) override {
-    auto &ES = Layer.getExecutionSession();
 
     SymbolFlagsMap ExtraSymbolsToClaim;
     bool AutoClaim = Layer.AutoClaimObjectSymbols;
 
     SymbolMap InternedResult;
     for (auto *Sym : G.defined_symbols())
-      if (Sym->hasName() && Sym->getScope() != Scope::Local) {
-        auto InternedName = ES.intern(Sym->getName());
+      if (Sym->getScope() < Scope::SideEffectsOnly) {
         auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
         auto Flags = getJITSymbolFlagsForSymbol(*Sym);
-        InternedResult[InternedName] = {Ptr, Flags};
-        if (AutoClaim && !MR->getSymbols().count(InternedName)) {
-          assert(!ExtraSymbolsToClaim.count(InternedName) &&
+        InternedResult[Sym->getName()] = {Ptr, Flags};
+        if (AutoClaim && !MR->getSymbols().count(Sym->getName())) {
+          assert(!ExtraSymbolsToClaim.count(Sym->getName()) &&
                  "Duplicate symbol to claim?");
-          ExtraSymbolsToClaim[InternedName] = Flags;
+          ExtraSymbolsToClaim[Sym->getName()] = Flags;
         }
       }
 
     for (auto *Sym : G.absolute_symbols())
-      if (Sym->hasName() && Sym->getScope() != Scope::Local) {
-        auto InternedName = ES.intern(Sym->getName());
+      if (Sym->getScope() < Scope::SideEffectsOnly) {
         auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
         auto Flags = getJITSymbolFlagsForSymbol(*Sym);
-        InternedResult[InternedName] = {Ptr, Flags};
-        if (AutoClaim && !MR->getSymbols().count(InternedName)) {
-          assert(!ExtraSymbolsToClaim.count(InternedName) &&
+        InternedResult[Sym->getName()] = {Ptr, Flags};
+        if (AutoClaim && !MR->getSymbols().count(Sym->getName())) {
+          assert(!ExtraSymbolsToClaim.count(Sym->getName()) &&
                  "Duplicate symbol to claim?");
-          ExtraSymbolsToClaim[InternedName] = Flags;
+          ExtraSymbolsToClaim[Sym->getName()] = Flags;
         }
       }
 
@@ -281,11 +279,9 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
         // If this is a materialization-side-effects only symbol then bump
         // the counter and remove in from the result, otherwise make sure that
         // it's defined.
-        if (Flags.hasMaterializationSideEffectsOnly()) {
+        if (Flags.hasMaterializationSideEffectsOnly())
           ++NumMaterializationSideEffectsOnlySymbols;
-          InternedResult.erase(Sym);
-          continue;
-        } else if (I == InternedResult.end())
+        else if (I == InternedResult.end())
           MissingSymbols.push_back(Sym);
         else if (Layer.OverrideObjectFlags)
           I->second.setFlags(Flags);
@@ -379,19 +375,16 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
 
 private:
   Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) {
-    auto &ES = Layer.getExecutionSession();
-
     SymbolFlagsMap NewSymbolsToClaim;
     std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
 
     auto ProcessSymbol = [&](Symbol *Sym) {
       if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak &&
           Sym->getScope() != Scope::Local) {
-        auto Name = ES.intern(Sym->getName());
-        if (!MR->getSymbols().count(ES.intern(Sym->getName()))) {
-          NewSymbolsToClaim[Name] =
+        if (!MR->getSymbols().count(Sym->getName())) {
+          NewSymbolsToClaim[Sym->getName()] =
               getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak;
-          NameToSym.push_back(std::make_pair(std::move(Name), Sym));
+          NameToSym.push_back(std::make_pair(Sym->getName(), Sym));
         }
       }
     };
@@ -421,9 +414,8 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
   }
 
   Error markResponsibilitySymbolsLive(LinkGraph &G) const {
-    auto &ES = Layer.getExecutionSession();
     for (auto *Sym : G.defined_symbols())
-      if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName())))
+      if (Sym->hasName() && MR->getSymbols().count(Sym->getName()))
         Sym->setLive(true);
     return Error::success();
   }
@@ -558,15 +550,6 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
     // SymbolDependenceGroups (in the SymbolDepGroups member), ready for use in
     // the upcoming notifyFinalized call.
     auto &TargetJD = MR->getTargetJITDylib();
-    auto &ES = TargetJD.getExecutionSession();
-
-    DenseMap<Symbol *, SymbolStringPtr> InternedNames;
-    auto GetInternedName = [&](Symbol *S) {
-      auto &Name = InternedNames[S];
-      if (!Name)
-        Name = ES.intern(S->getName());
-      return Name;
-    };
 
     for (auto &[B, BI] : BlockInfos) {
       if (!BI.Defs.empty()) {
@@ -574,10 +557,10 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
         auto &SDG = SymbolDepGroups.back();
 
         for (auto *Def : BI.Defs)
-          SDG.Symbols.insert(GetInternedName(Def));
+          SDG.Symbols.insert(Def->getName());
 
         for (auto *Dep : BI.SymbolDeps) {
-          auto DepName = GetInternedName(Dep);
+          auto DepName = Dep->getName();
           if (Dep->isDefined())
             SDG.Dependencies[&TargetJD].insert(std::move(DepName));
           else {
@@ -643,8 +626,8 @@ void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
 
   auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
       *this, std::move(R), std::move(O));
-
-  if (auto G = createLinkGraphFromObject(ObjBuffer)) {
+  if (auto G = createLinkGraphFromObject(
+          ObjBuffer, getExecutionSession().getSymbolStringPool())) {
     Ctx->notifyMaterializing(**G);
     link(std::move(*G), std::move(Ctx));
   } else {
diff --git a/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp b/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
index 4f28b8bda529b..4ed05ef3573f8 100644
--- a/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
@@ -18,6 +18,7 @@ void SectCreateMaterializationUnit::materialize(
     std::unique_ptr<MaterializationResponsibility> R) {
   auto G = std::make_unique<LinkGraph>(
       "orc_sectcreate_" + SectName,
+      ObjLinkingLayer.getExecutionSession().getSymbolStringPool(),
       ObjLinkingLayer.getExecutionSession().getTargetTriple(),
       getGenericEdgeKindName);
 
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
index f4e4a6b4f53fc..792b0cc8251cc 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_component_library(LLVMOrcShared
   OrcError.cpp
   OrcRTBridge.cpp
   SimpleRemoteEPCUtils.cpp
+  SymbolStringPool.cpp
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
 
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
index ae39b1d1bfaa9..54a25c007c589 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
@@ -49,6 +49,8 @@ const char *MemoryWriteUInt64sWrapperName =
     "__llvm_orc_bootstrap_mem_write_uint64s_wrapper";
 const char *MemoryWriteBuffersWrapperName =
     "__llvm_orc_bootstrap_mem_write_buffers_wrapper";
+const char *MemoryWritePointersWrapperName =
+    "__llvm_orc_bootstrap_mem_write_pointers_wrapper";
 
 const char *RegisterEHFrameSectionWrapperName =
     "llvm_orc_registerEHFrameSectionWrapper";
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/SymbolStringPool.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/SymbolStringPool.cpp
new file mode 100644
index 0000000000000..9ca4e59288ecf
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/SymbolStringPool.cpp
@@ -0,0 +1,18 @@
+//===------- SymbolStringPool.cpp - SymbolStringPool implementation -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm::orc {
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtrBase &Sym) {
+  return OS << Sym.S->first();
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
index 41b97527f38d6..53d5e049798aa 100644
--- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
@@ -227,7 +227,17 @@ SimpleRemoteEPC::createDefaultMemoryManager(SimpleRemoteEPC &SREPC) {
 
 Expected<std::unique_ptr<ExecutorProcessControl::MemoryAccess>>
 SimpleRemoteEPC::createDefaultMemoryAccess(SimpleRemoteEPC &SREPC) {
-  return nullptr;
+  EPCGenericMemoryAccess::FuncAddrs FAs;
+  if (auto Err = SREPC.getBootstrapSymbols(
+          {{FAs.WriteUInt8s, rt::MemoryWriteUInt8sWrapperName},
+           {FAs.WriteUInt16s, rt::MemoryWriteUInt16sWrapperName},
+           {FAs.WriteUInt32s, rt::MemoryWriteUInt32sWrapperName},
+           {FAs.WriteUInt64s, rt::MemoryWriteUInt64sWrapperName},
+           {FAs.WriteBuffers, rt::MemoryWriteBuffersWrapperName},
+           {FAs.WritePointers, rt::MemoryWritePointersWrapperName}}))
+    return std::move(Err);
+
+  return std::make_unique<EPCGenericMemoryAccess>(SREPC, FAs);
 }
 
 Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo,
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
index b38877955282e..d88fbbfc86385 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
@@ -45,6 +45,18 @@ writeBuffersWrapper(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
+static llvm::orc::shared::CWrapperFunctionResult
+writePointersWrapper(const char *ArgData, size_t ArgSize) {
+  return WrapperFunction<void(SPSSequence<SPSMemoryAccessPointerWrite>)>::
+      handle(ArgData, ArgSize,
+             [](std::vector<tpctypes::PointerWrite> Ws) {
+               for (auto &W : Ws)
+                 *W.Addr.template toPtr<void **>() =
+                     W.Value.template toPtr<void *>();
+             })
+          .release();
+}
+
 static llvm::orc::shared::CWrapperFunctionResult
 runAsMainWrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<rt::SPSRunAsMainSignature>::handle(
@@ -92,6 +104,8 @@ void addTo(StringMap<ExecutorAddr> &M) {
                          shared::SPSMemoryAccessUInt64Write>);
   M[rt::MemoryWriteBuffersWrapperName] =
       ExecutorAddr::fromPtr(&writeBuffersWrapper);
+  M[rt::MemoryWritePointersWrapperName] =
+      ExecutorAddr::fromPtr(&writePointersWrapper);
   M[rt::RegisterEHFrameSectionWrapperName] =
       ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
   M[rt::DeregisterEHFrameSectionWrapperName] =
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index c4326200b4dd4..0fc05a47648c1 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -14,6 +14,7 @@
 #include "Targets/RuntimeDyldELFMips.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Endian.h"
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index de7630b9747ea..24b03a058981a 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/ExecutionEngine/RuntimeDyldChecker.h"
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 1fae138b449ed..21004e6a15d49 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -1818,7 +1818,7 @@ static Value *emitTaskDependencies(
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     BodyGenCallbackTy BodyGenCB, bool Tied, Value *Final, Value *IfCondition,
-    SmallVector<DependData> Dependencies, bool Mergeable) {
+    SmallVector<DependData> Dependencies, bool Mergeable, Value *EventHandle) {
 
   if (!updateToLocation(Loc))
     return InsertPointTy();
@@ -1864,7 +1864,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
       Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false));
 
   OI.PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
-                      Mergeable, TaskAllocaBB,
+                      Mergeable, EventHandle, TaskAllocaBB,
                       ToBeDeleted](Function &OutlinedFn) mutable {
     // Replace the Stale CI by appropriate RTL function call.
     assert(OutlinedFn.getNumUses() == 1 &&
@@ -1935,6 +1935,20 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
                       /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
                       /*task_func=*/&OutlinedFn});
 
+    // Emit detach clause initialization.
+    // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
+    // task_descriptor);
+    if (EventHandle) {
+      Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
+          OMPRTL___kmpc_task_allow_completion_event);
+      llvm::Value *EventVal =
+          Builder.CreateCall(TaskDetachFn, {Ident, ThreadID, TaskData});
+      llvm::Value *EventHandleAddr =
+          Builder.CreatePointerBitCastOrAddrSpaceCast(EventHandle,
+                                                      Builder.getPtrTy(0));
+      EventVal = Builder.CreatePtrToInt(EventVal, Builder.getInt64Ty());
+      Builder.CreateStore(EventVal, EventHandleAddr);
+    }
     // Copy the arguments for outlined function
     if (HasShareds) {
       Value *Shareds = StaleCI->getArgOperand(1);
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index f8183774f4056..a37a8901489cf 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -649,8 +649,10 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) {
     OS << "target(\"";
     printEscapedString(Ty->getTargetExtName(), OS);
     OS << "\"";
-    for (Type *Inner : TETy->type_params())
-      OS << ", " << *Inner;
+    for (Type *Inner : TETy->type_params()) {
+      OS << ", ";
+      Inner->print(OS, /*IsForDebug=*/false, /*NoDetails=*/true);
+    }
     for (unsigned IntParam : TETy->int_params())
       OS << ", " << IntParam;
     OS << ")";
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 74b423b71d1e7..54601193b5bf6 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5557,7 +5557,8 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
     return Res;
   }
 
-  if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m"))) {
+  if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
+      T.isWasm()) {
     // Mips64 with o32 ABI did not add "-i128:128".
     // Add "-i128:128"
     std::string I64 = "-i64:64";
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index a4af0ead07cf6..5b3ce2d62efd5 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -740,12 +740,10 @@ unsigned DataLayout::getPointerSize(unsigned AS) const {
   return divideCeil(getPointerSpec(AS).BitWidth, 8);
 }
 
-unsigned DataLayout::getMaxIndexSize() const {
+unsigned DataLayout::getMaxIndexSizeInBits() const {
   unsigned MaxIndexSize = 0;
   for (const PointerSpec &Spec : PointerSpecs)
-    MaxIndexSize =
-        std::max(MaxIndexSize, (unsigned)divideCeil(Spec.BitWidth, 8));
-
+    MaxIndexSize = std::max(MaxIndexSize, Spec.IndexBitWidth);
   return MaxIndexSize;
 }
 
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 065ce3a017283..4f07a4c4dd017 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -3842,9 +3842,8 @@ std::optional<bool> ICmpInst::compare(const KnownBits &LHS,
 }
 
 CmpInst::Predicate ICmpInst::getFlippedSignednessPredicate(Predicate pred) {
-  assert(CmpInst::isRelational(pred) &&
-         "Call only with non-equality predicates!");
-
+  if (CmpInst::isEquality(pred))
+    return pred;
   if (isSigned(pred))
     return getUnsignedPredicate(pred);
   if (isUnsigned(pred))
@@ -3916,6 +3915,23 @@ bool CmpInst::isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2) {
   return isImpliedTrueByMatchingCmp(Pred1, getInversePredicate(Pred2));
 }
 
+//===----------------------------------------------------------------------===//
+//                       CmpPredicate Implementation
+//===----------------------------------------------------------------------===//
+
+std::optional<CmpPredicate> CmpPredicate::getMatching(CmpPredicate A,
+                                                      CmpPredicate B) {
+  if (A.Pred == B.Pred)
+    return A.HasSameSign == B.HasSameSign ? A : CmpPredicate(A.Pred);
+  if (A.HasSameSign &&
+      A.Pred == ICmpInst::getFlippedSignednessPredicate(B.Pred))
+    return B.Pred;
+  if (B.HasSameSign &&
+      B.Pred == ICmpInst::getFlippedSignednessPredicate(A.Pred))
+    return A.Pred;
+  return {};
+}
+
 //===----------------------------------------------------------------------===//
 //                        SwitchInst Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index 3130a0bd2955a..ec1184e8d835d 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -34,16 +34,13 @@
 using namespace llvm;
 
 /// Table of string intrinsic names indexed by enum value.
-static constexpr const char *const IntrinsicNameTable[] = {
-    "not_intrinsic",
 #define GET_INTRINSIC_NAME_TABLE
 #include "llvm/IR/IntrinsicImpl.inc"
 #undef GET_INTRINSIC_NAME_TABLE
-};
 
 StringRef Intrinsic::getBaseName(ID id) {
   assert(id < num_intrinsics && "Invalid intrinsic ID!");
-  return IntrinsicNameTable[id];
+  return IntrinsicNameTable + IntrinsicNameOffsetTable[id];
 }
 
 StringRef Intrinsic::getName(ID id) {
@@ -621,9 +618,12 @@ bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
   return IID > TargetInfos[0].Count;
 }
 
-int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
-                                               StringRef Name,
-                                               StringRef Target) {
+/// Looks up Name in NameTable via binary search. NameTable must be sorted
+/// and all entries must start with "llvm.".  If NameTable contains an exact
+/// match for Name or a prefix of Name followed by a dot, its index in
+/// NameTable is returned. Otherwise, -1 is returned.
+static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
+                                     StringRef Name, StringRef Target = "") {
   assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
   assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
 
@@ -638,15 +638,31 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
   if (!Target.empty())
     CmpEnd += 1 + Target.size(); // skip the .target component.
 
-  const char *const *Low = NameTable.begin();
-  const char *const *High = NameTable.end();
-  const char *const *LastLow = Low;
+  const unsigned *Low = NameOffsetTable.begin();
+  const unsigned *High = NameOffsetTable.end();
+  const unsigned *LastLow = Low;
   while (CmpEnd < Name.size() && High - Low > 0) {
     size_t CmpStart = CmpEnd;
     CmpEnd = Name.find('.', CmpStart + 1);
     CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
-    auto Cmp = [CmpStart, CmpEnd](const char *LHS, const char *RHS) {
-      return strncmp(LHS + CmpStart, RHS + CmpStart, CmpEnd - CmpStart) < 0;
+    auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
+      // `equal_range` requires the comparison to work with either side being an
+      // offset or the value. Detect which kind each side is to set up the
+      // compared strings.
+      const char *LHSStr;
+      if constexpr (std::is_integral_v<decltype(LHS)>) {
+        LHSStr = &IntrinsicNameTable[LHS];
+      } else {
+        LHSStr = LHS;
+      }
+      const char *RHSStr;
+      if constexpr (std::is_integral_v<decltype(RHS)>) {
+        RHSStr = &IntrinsicNameTable[RHS];
+      } else {
+        RHSStr = RHS;
+      }
+      return strncmp(LHSStr + CmpStart, RHSStr + CmpStart, CmpEnd - CmpStart) <
+             0;
     };
     LastLow = Low;
     std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp);
@@ -654,20 +670,21 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
   if (High - Low > 0)
     LastLow = Low;
 
-  if (LastLow == NameTable.end())
+  if (LastLow == NameOffsetTable.end())
     return -1;
-  StringRef NameFound = *LastLow;
+  StringRef NameFound = &IntrinsicNameTable[*LastLow];
   if (Name == NameFound ||
       (Name.starts_with(NameFound) && Name[NameFound.size()] == '.'))
-    return LastLow - NameTable.begin();
+    return LastLow - NameOffsetTable.begin();
   return -1;
 }
 
-/// Find the segment of \c IntrinsicNameTable for intrinsics with the same
+/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
 /// target as \c Name, or the generic table if \c Name is not target specific.
 ///
-/// Returns the relevant slice of \c IntrinsicNameTable and the target name.
-static std::pair<ArrayRef<const char *>, StringRef>
+/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
+/// name.
+static std::pair<ArrayRef<unsigned>, StringRef>
 findTargetSubtable(StringRef Name) {
   assert(Name.starts_with("llvm."));
 
@@ -680,25 +697,26 @@ findTargetSubtable(StringRef Name) {
   // We've either found the target or just fall back to the generic set, which
   // is always first.
   const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
-  return {ArrayRef(&IntrinsicNameTable[1] + TI.Offset, TI.Count), TI.Name};
+  return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
+          TI.Name};
 }
 
 /// This does the actual lookup of an intrinsic ID which matches the given
 /// function name.
 Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
-  auto [NameTable, Target] = findTargetSubtable(Name);
-  int Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name, Target);
+  auto [NameOffsetTable, Target] = findTargetSubtable(Name);
+  int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
   if (Idx == -1)
     return Intrinsic::not_intrinsic;
 
   // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
   // an index into a sub-table.
-  int Adjust = NameTable.data() - IntrinsicNameTable;
+  int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
   Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
 
   // If the intrinsic is not overloaded, require an exact match. If it is
   // overloaded, require either exact or prefix match.
-  const auto MatchSize = strlen(NameTable[Idx]);
+  const auto MatchSize = strlen(&IntrinsicNameTable[NameOffsetTable[Idx]]);
   assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
   bool IsExactMatch = Name.size() == MatchSize;
   return IsExactMatch || Intrinsic::isOverloaded(ID) ? ID
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 12a558b3bc1b1..d9024b0a8673f 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -328,6 +328,13 @@ void ModuleSummaryIndex::propagateAttributes(
 
 bool ModuleSummaryIndex::canImportGlobalVar(const GlobalValueSummary *S,
                                             bool AnalyzeRefs) const {
+  bool CanImportDecl;
+  return canImportGlobalVar(S, AnalyzeRefs, CanImportDecl);
+}
+
+bool ModuleSummaryIndex::canImportGlobalVar(const GlobalValueSummary *S,
+                                            bool AnalyzeRefs,
+                                            bool &CanImportDecl) const {
   auto HasRefsPreventingImport = [this](const GlobalVarSummary *GVS) {
     // We don't analyze GV references during attribute propagation, so
     // GV with non-trivial initializer can be marked either read or
@@ -348,13 +355,20 @@ bool ModuleSummaryIndex::canImportGlobalVar(const GlobalValueSummary *S,
   };
   auto *GVS = cast<GlobalVarSummary>(S->getBaseObject());
 
+  const bool nonInterposable =
+      !GlobalValue::isInterposableLinkage(S->linkage());
+  const bool eligibleToImport = !S->notEligibleToImport();
+
+  // It's correct to import a global variable only when it is not interposable
+  // and eligible to import.
+  CanImportDecl = (nonInterposable && eligibleToImport);
+
   // Global variable with non-trivial initializer can be imported
   // if it's readonly. This gives us extra opportunities for constant
   // folding and converting indirect calls to direct calls. We don't
   // analyze GV references during attribute propagation, because we
   // don't know yet if it is readonly or not.
-  return !GlobalValue::isInterposableLinkage(S->linkage()) &&
-         !S->notEligibleToImport() &&
+  return nonInterposable && eligibleToImport &&
          (!AnalyzeRefs || !HasRefsPreventingImport(GVS));
 }
 
diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp
index e1580cf760969..39e5463cb6fc3 100644
--- a/llvm/lib/IR/Operator.cpp
+++ b/llvm/lib/IR/Operator.cpp
@@ -125,8 +125,9 @@ bool GEPOperator::accumulateConstantOffset(
     Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL,
     APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis) {
   // Fast path for canonical getelementptr i8 form.
-  if (SourceType->isIntegerTy(8) && !ExternalAnalysis) {
-    if (auto *CI = dyn_cast<ConstantInt>(Index.front())) {
+  if (SourceType->isIntegerTy(8) && !Index.empty() && !ExternalAnalysis) {
+    auto *CI = dyn_cast<ConstantInt>(Index.front());
+    if (CI && CI->getType()->isIntegerTy()) {
       Offset += CI->getValue().sextOrTrunc(Offset.getBitWidth());
       return true;
     }
@@ -165,7 +166,8 @@ bool GEPOperator::accumulateConstantOffset(
     Value *V = GTI.getOperand();
     StructType *STy = GTI.getStructTypeOrNull();
     // Handle ConstantInt if possible.
-    if (auto ConstOffset = dyn_cast<ConstantInt>(V)) {
+    auto *ConstOffset = dyn_cast<ConstantInt>(V);
+    if (ConstOffset && ConstOffset->getType()->isIntegerTy()) {
       if (ConstOffset->isZero())
         continue;
       // if the type is scalable and the constant is not zero (vscale * n * 0 =
@@ -226,7 +228,8 @@ bool GEPOperator::collectOffset(
     Value *V = GTI.getOperand();
     StructType *STy = GTI.getStructTypeOrNull();
     // Handle ConstantInt if possible.
-    if (auto ConstOffset = dyn_cast<ConstantInt>(V)) {
+    auto *ConstOffset = dyn_cast<ConstantInt>(V);
+    if (ConstOffset && ConstOffset->getType()->isIntegerTy()) {
       if (ConstOffset->isZero())
         continue;
       // If the type is scalable and the constant is not zero (vscale * n * 0 =
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index ccc534a890419..1c617c100c7dc 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -46,7 +46,7 @@ class StructuralHashImpl {
   /// Assign a unique ID to each Value in the order they are first seen.
   DenseMap<const Value *, int> ValueToId;
 
-  stable_hash hashType(Type *ValueType) {
+  static stable_hash hashType(Type *ValueType) {
     SmallVector<stable_hash> Hashes;
     Hashes.emplace_back(ValueType->getTypeID());
     if (ValueType->isIntegerTy())
@@ -65,7 +65,7 @@ class StructuralHashImpl {
     }
   }
 
-  stable_hash hashAPInt(const APInt &I) {
+  static stable_hash hashAPInt(const APInt &I) {
     SmallVector<stable_hash> Hashes;
     Hashes.emplace_back(I.getBitWidth());
     auto RawVals = ArrayRef<uint64_t>(I.getRawData(), I.getNumWords());
@@ -73,11 +73,39 @@ class StructuralHashImpl {
     return stable_hash_combine(Hashes);
   }
 
-  stable_hash hashAPFloat(const APFloat &F) {
+  static stable_hash hashAPFloat(const APFloat &F) {
     return hashAPInt(F.bitcastToAPInt());
   }
 
-  stable_hash hashGlobalValue(const GlobalValue *GV) {
+  static stable_hash hashGlobalVariable(const GlobalVariable &GVar) {
+    if (!GVar.hasInitializer())
+      return hashGlobalValue(&GVar);
+
+    // Hash the contents of a string.
+    if (GVar.getName().starts_with(".str")) {
+      auto *C = GVar.getInitializer();
+      if (const auto *Seq = dyn_cast<ConstantDataSequential>(C))
+        if (Seq->isString())
+          return stable_hash_name(Seq->getAsString());
+    }
+
+    // Hash structural contents of Objective-C metadata in specific sections.
+    // This can be extended to other metadata if needed.
+    static constexpr const char *SectionNames[] = {
+        "__cfstring",      "__cstring",      "__objc_classrefs",
+        "__objc_methname", "__objc_selrefs",
+    };
+    if (GVar.hasSection()) {
+      StringRef SectionName = GVar.getSection();
+      for (const char *Name : SectionNames)
+        if (SectionName.contains(Name))
+          return hashConstant(GVar.getInitializer());
+    }
+
+    return hashGlobalValue(&GVar);
+  }
+
+  static stable_hash hashGlobalValue(const GlobalValue *GV) {
     if (!GV->hasName())
       return 0;
     return stable_hash_name(GV->getName());
@@ -87,7 +115,7 @@ class StructuralHashImpl {
   // FunctionComparator::cmpConstants() in FunctionComparator.cpp, but here
   // we're interested in computing a hash rather than comparing two Constants.
   // Some of the logic is simplified, e.g, we don't expand GEPOperator.
-  stable_hash hashConstant(Constant *C) {
+  static stable_hash hashConstant(const Constant *C) {
     SmallVector<stable_hash> Hashes;
 
     Type *Ty = C->getType();
@@ -98,14 +126,21 @@ class StructuralHashImpl {
       return stable_hash_combine(Hashes);
     }
 
+    if (auto *GVar = dyn_cast<GlobalVariable>(C)) {
+      Hashes.emplace_back(hashGlobalVariable(*GVar));
+      return stable_hash_combine(Hashes);
+    }
+
     if (auto *G = dyn_cast<GlobalValue>(C)) {
       Hashes.emplace_back(hashGlobalValue(G));
       return stable_hash_combine(Hashes);
     }
 
     if (const auto *Seq = dyn_cast<ConstantDataSequential>(C)) {
-      Hashes.emplace_back(xxh3_64bits(Seq->getRawDataValues()));
-      return stable_hash_combine(Hashes);
+      if (Seq->isString()) {
+        Hashes.emplace_back(stable_hash_name(Seq->getAsString()));
+        return stable_hash_combine(Hashes);
+      }
     }
 
     switch (C->getValueID()) {
@@ -297,6 +332,10 @@ stable_hash llvm::StructuralHash(const Function &F, bool DetailedHash) {
   return H.getHash();
 }
 
+stable_hash llvm::StructuralHash(const GlobalVariable &GVar) {
+  return StructuralHashImpl::hashGlobalVariable(GVar);
+}
+
 stable_hash llvm::StructuralHash(const Module &M, bool DetailedHash) {
   StructuralHashImpl H(DetailedHash);
   H.update(M);
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 48e39d7026041..1060432b1c196 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -989,7 +989,7 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
         Ty->getIntParameter(0);
     return TargetTypeInfo(
         ScalableVectorType::get(Type::getInt8Ty(C), TotalNumElts),
-        TargetExtType::CanBeLocal);
+        TargetExtType::CanBeLocal, TargetExtType::HasZeroInit);
   }
 
   // DirectX resources
diff --git a/llvm/lib/LineEditor/LineEditor.cpp b/llvm/lib/LineEditor/LineEditor.cpp
index d0d138bb1f9d8..c7d5fd7dcf8ac 100644
--- a/llvm/lib/LineEditor/LineEditor.cpp
+++ b/llvm/lib/LineEditor/LineEditor.cpp
@@ -17,6 +17,7 @@
 #include <cstdio>
 #ifdef HAVE_LIBEDIT
 #include <histedit.h>
+constexpr int DefaultHistorySize = 800;
 #endif
 
 using namespace llvm;
@@ -220,8 +221,8 @@ LineEditor::LineEditor(StringRef ProgName, StringRef HistoryPath, FILE *In,
            NULL); // Fix the delete key.
   ::el_set(Data->EL, EL_CLIENTDATA, Data.get());
 
+  setHistorySize(DefaultHistorySize);
   HistEvent HE;
-  ::history(Data->Hist, &HE, H_SETSIZE, 800);
   ::history(Data->Hist, &HE, H_SETUNIQUE, 1);
   loadHistory();
 }
@@ -248,6 +249,11 @@ void LineEditor::loadHistory() {
   }
 }
 
+void LineEditor::setHistorySize(int size) {
+  HistEvent HE;
+  ::history(Data->Hist, &HE, H_SETSIZE, size);
+}
+
 std::optional<std::string> LineEditor::readLine() const {
   // Call el_gets to prompt the user and read the user's input.
   int LineLen = 0;
@@ -291,6 +297,7 @@ LineEditor::~LineEditor() {
 
 void LineEditor::saveHistory() {}
 void LineEditor::loadHistory() {}
+void LineEditor::setHistorySize(int size) {}
 
 std::optional<std::string> LineEditor::readLine() const {
   ::fprintf(Data->Out, "%s", Prompt.c_str());
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 3ce45f7d5d67e..d15a9a8a36c5a 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -6287,7 +6287,7 @@ bool AsmParser::parseMSInlineAsm(
   if (AsmStart != AsmEnd)
     OS << StringRef(AsmStart, AsmEnd - AsmStart);
 
-  AsmString = AsmStringIR;
+  AsmString = std::move(AsmStringIR);
   return false;
 }
 
diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp
index 178b1d21e5200..4a9bacdbc8fc4 100644
--- a/llvm/lib/MC/MCRegisterInfo.cpp
+++ b/llvm/lib/MC/MCRegisterInfo.cpp
@@ -220,3 +220,10 @@ bool MCRegisterInfo::regsOverlap(MCRegister RegA, MCRegister RegB) const {
   } while (*IA < *IB ? ++IA != EA : ++IB != EB);
   return false;
 }
+
+bool MCRegisterInfo::isArtificialRegUnit(unsigned Unit) const {
+  for (MCRegUnitRootIterator Root(Unit, this); Root.isValid(); ++Root)
+    if (isArtificial(*Root))
+      return true;
+  return false;
+}
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index df1b160f9df28..5703cfd03fceb 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -463,7 +463,7 @@ void MCStreamer::emitCFIStartProc(bool IsSimple, SMLoc Loc) {
   }
 
   FrameInfoStack.emplace_back(DwarfFrameInfos.size(), getCurrentSectionOnly());
-  DwarfFrameInfos.push_back(Frame);
+  DwarfFrameInfos.push_back(std::move(Frame));
 }
 
 void MCStreamer::emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) {
@@ -510,7 +510,7 @@ void MCStreamer::emitCFIDefCfa(int64_t Register, int64_t Offset, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
   CurFrame->CurrentCfaRegister = static_cast<unsigned>(Register);
 }
 
@@ -521,7 +521,7 @@ void MCStreamer::emitCFIDefCfaOffset(int64_t Offset, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIAdjustCfaOffset(int64_t Adjustment, SMLoc Loc) {
@@ -531,7 +531,7 @@ void MCStreamer::emitCFIAdjustCfaOffset(int64_t Adjustment, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIDefCfaRegister(int64_t Register, SMLoc Loc) {
@@ -541,7 +541,7 @@ void MCStreamer::emitCFIDefCfaRegister(int64_t Register, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
   CurFrame->CurrentCfaRegister = static_cast<unsigned>(Register);
 }
 
@@ -553,7 +553,7 @@ void MCStreamer::emitCFILLVMDefAspaceCfa(int64_t Register, int64_t Offset,
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
   CurFrame->CurrentCfaRegister = static_cast<unsigned>(Register);
 }
 
@@ -564,7 +564,7 @@ void MCStreamer::emitCFIOffset(int64_t Register, int64_t Offset, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIRelOffset(int64_t Register, int64_t Offset, SMLoc Loc) {
@@ -574,7 +574,7 @@ void MCStreamer::emitCFIRelOffset(int64_t Register, int64_t Offset, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIPersonality(const MCSymbol *Sym,
@@ -601,7 +601,7 @@ void MCStreamer::emitCFIRememberState(SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIRestoreState(SMLoc Loc) {
@@ -612,7 +612,7 @@ void MCStreamer::emitCFIRestoreState(SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFISameValue(int64_t Register, SMLoc Loc) {
@@ -622,7 +622,7 @@ void MCStreamer::emitCFISameValue(int64_t Register, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIRestore(int64_t Register, SMLoc Loc) {
@@ -632,7 +632,7 @@ void MCStreamer::emitCFIRestore(int64_t Register, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIEscape(StringRef Values, SMLoc Loc) {
@@ -642,7 +642,7 @@ void MCStreamer::emitCFIEscape(StringRef Values, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIGnuArgsSize(int64_t Size, SMLoc Loc) {
@@ -652,7 +652,7 @@ void MCStreamer::emitCFIGnuArgsSize(int64_t Size, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFISignalFrame() {
@@ -669,7 +669,7 @@ void MCStreamer::emitCFIUndefined(int64_t Register, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIRegister(int64_t Register1, int64_t Register2,
@@ -680,7 +680,7 @@ void MCStreamer::emitCFIRegister(int64_t Register1, int64_t Register2,
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIWindowSave(SMLoc Loc) {
@@ -689,7 +689,7 @@ void MCStreamer::emitCFIWindowSave(SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFINegateRAState(SMLoc Loc) {
@@ -699,7 +699,7 @@ void MCStreamer::emitCFINegateRAState(SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) {
@@ -709,7 +709,7 @@ void MCStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 void MCStreamer::emitCFIReturnColumn(int64_t Register) {
@@ -733,7 +733,7 @@ void MCStreamer::emitCFIValOffset(int64_t Register, int64_t Offset, SMLoc Loc) {
   MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
   if (!CurFrame)
     return;
-  CurFrame->Instructions.push_back(Instruction);
+  CurFrame->Instructions.push_back(std::move(Instruction));
 }
 
 WinEH::FrameInfo *MCStreamer::EnsureValidWinFrameInfo(SMLoc Loc) {
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index 1de0a9f66669a..f59ec2f7c2602 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -85,16 +85,22 @@ static void ApplyFeatureFlag(FeatureBitset &Bits, StringRef Feature,
 }
 
 /// Return the length of the longest entry in the table.
-template <typename T>
-static size_t getLongestEntryLength(ArrayRef<T> Table) {
+static size_t getLongestEntryLength(ArrayRef<SubtargetFeatureKV> Table) {
   size_t MaxLen = 0;
   for (auto &I : Table)
     MaxLen = std::max(MaxLen, std::strlen(I.Key));
   return MaxLen;
 }
 
+static size_t getLongestEntryLength(ArrayRef<StringRef> Table) {
+  size_t MaxLen = 0;
+  for (StringRef I : Table)
+    MaxLen = std::max(MaxLen, I.size());
+  return MaxLen;
+}
+
 /// Display help for feature and mcpu choices.
-static void Help(ArrayRef<SubtargetSubTypeKV> CPUTable,
+static void Help(ArrayRef<StringRef> CPUNames,
                  ArrayRef<SubtargetFeatureKV> FeatTable) {
   // the static variable ensures that the help information only gets
   // printed once even though a target machine creates multiple subtargets
@@ -104,14 +110,20 @@ static void Help(ArrayRef<SubtargetSubTypeKV> CPUTable,
   }
 
   // Determine the length of the longest CPU and Feature entries.
-  unsigned MaxCPULen  = getLongestEntryLength(CPUTable);
+  unsigned MaxCPULen = getLongestEntryLength(CPUNames);
   unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
 
   // Print the CPU table.
   errs() << "Available CPUs for this target:\n\n";
-  for (auto &CPU : CPUTable)
-    errs() << format("  %-*s - Select the %s processor.\n", MaxCPULen, CPU.Key,
-                     CPU.Key);
+  for (auto &CPUName : CPUNames) {
+    // Skip apple-latest, as that's only meant to be used in
+    // disassemblers/debuggers, and we don't want normal code to be built with
+    // it as an -mcpu=
+    if (CPUName == "apple-latest")
+      continue;
+    errs() << format("  %-*s - Select the %s processor.\n", MaxCPULen,
+                     CPUName.str().c_str(), CPUName.str().c_str());
+  }
   errs() << '\n';
 
   // Print the Feature table.
@@ -127,7 +139,7 @@ static void Help(ArrayRef<SubtargetSubTypeKV> CPUTable,
 }
 
 /// Display help for mcpu choices only
-static void cpuHelp(ArrayRef<SubtargetSubTypeKV> CPUTable) {
+static void cpuHelp(ArrayRef<StringRef> CPUNames) {
   // the static variable ensures that the help information only gets
   // printed once even though a target machine creates multiple subtargets
   static bool PrintOnce = false;
@@ -137,8 +149,14 @@ static void cpuHelp(ArrayRef<SubtargetSubTypeKV> CPUTable) {
 
   // Print the CPU table.
   errs() << "Available CPUs for this target:\n\n";
-  for (auto &CPU : CPUTable)
-    errs() << "\t" << CPU.Key << "\n";
+  for (auto &CPU : CPUNames) {
+    // Skip apple-latest, as that's only meant to be used in
+    // disassemblers/debuggers, and we don't want normal code to be built with
+    // it as an -mcpu=
+    if (CPU == "apple-latest")
+      continue;
+    errs() << "\t" << CPU << "\n";
+  }
   errs() << '\n';
 
   errs() << "Use -mcpu or -mtune to specify the target's processor.\n"
@@ -148,7 +166,9 @@ static void cpuHelp(ArrayRef<SubtargetSubTypeKV> CPUTable) {
   PrintOnce = true;
 }
 
-static FeatureBitset getFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS,
+static FeatureBitset getFeatures(MCSubtargetInfo &STI, StringRef CPU,
+                                 StringRef TuneCPU, StringRef FS,
+                                 ArrayRef<StringRef> ProcNames,
                                  ArrayRef<SubtargetSubTypeKV> ProcDesc,
                                  ArrayRef<SubtargetFeatureKV> ProcFeatures) {
   SubtargetFeatures Features(FS);
@@ -163,7 +183,7 @@ static FeatureBitset getFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS,
 
   // Check if help is needed
   if (CPU == "help")
-    Help(ProcDesc, ProcFeatures);
+    Help(ProcNames, ProcFeatures);
 
   // Find CPU entry if CPU name is specified.
   else if (!CPU.empty()) {
@@ -196,9 +216,9 @@ static FeatureBitset getFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS,
   for (const std::string &Feature : Features.getFeatures()) {
     // Check for help
     if (Feature == "+help")
-      Help(ProcDesc, ProcFeatures);
+      Help(ProcNames, ProcFeatures);
     else if (Feature == "+cpuhelp")
-      cpuHelp(ProcDesc);
+      cpuHelp(ProcNames);
     else
       ApplyFeatureFlag(Bits, Feature, ProcFeatures);
   }
@@ -208,7 +228,8 @@ static FeatureBitset getFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS,
 
 void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU,
                                           StringRef FS) {
-  FeatureBits = getFeatures(CPU, TuneCPU, FS, ProcDesc, ProcFeatures);
+  FeatureBits =
+      getFeatures(*this, CPU, TuneCPU, FS, ProcNames, ProcDesc, ProcFeatures);
   FeatureString = std::string(FS);
 
   if (!TuneCPU.empty())
@@ -219,20 +240,19 @@ void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU,
 
 void MCSubtargetInfo::setDefaultFeatures(StringRef CPU, StringRef TuneCPU,
                                          StringRef FS) {
-  FeatureBits = getFeatures(CPU, TuneCPU, FS, ProcDesc, ProcFeatures);
+  FeatureBits =
+      getFeatures(*this, CPU, TuneCPU, FS, ProcNames, ProcDesc, ProcFeatures);
   FeatureString = std::string(FS);
 }
 
-MCSubtargetInfo::MCSubtargetInfo(const Triple &TT, StringRef C, StringRef TC,
-                                 StringRef FS, ArrayRef<SubtargetFeatureKV> PF,
-                                 ArrayRef<SubtargetSubTypeKV> PD,
-                                 const MCWriteProcResEntry *WPR,
-                                 const MCWriteLatencyEntry *WL,
-                                 const MCReadAdvanceEntry *RA,
-                                 const InstrStage *IS, const unsigned *OC,
-                                 const unsigned *FP)
+MCSubtargetInfo::MCSubtargetInfo(
+    const Triple &TT, StringRef C, StringRef TC, StringRef FS,
+    ArrayRef<StringRef> PN, ArrayRef<SubtargetFeatureKV> PF,
+    ArrayRef<SubtargetSubTypeKV> PD, const MCWriteProcResEntry *WPR,
+    const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA,
+    const InstrStage *IS, const unsigned *OC, const unsigned *FP)
     : TargetTriple(TT), CPU(std::string(C)), TuneCPU(std::string(TC)),
-      ProcFeatures(PF), ProcDesc(PD), WriteProcResTable(WPR),
+      ProcNames(PN), ProcFeatures(PF), ProcDesc(PD), WriteProcResTable(WPR),
       WriteLatencyTable(WL), ReadAdvanceTable(RA), Stages(IS),
       OperandCycles(OC), ForwardingPaths(FP) {
   InitMCProcessorInfo(CPU, TuneCPU, FS);
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index f25dc92fa235a..8526469245676 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -1326,6 +1326,22 @@ static bool isInSymtab(const MCSymbolWasm &Sym) {
   return true;
 }
 
+static bool isSectionReferenced(MCAssembler &Asm, MCSectionWasm &Section) {
+  StringRef SectionName = Section.getName();
+
+  for (const MCSymbol &S : Asm.symbols()) {
+    const auto &WS = static_cast<const MCSymbolWasm &>(S);
+    if (WS.isData() && WS.isInSection()) {
+      auto &RefSection = static_cast<MCSectionWasm &>(WS.getSection());
+      if (RefSection.getName() == SectionName) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
 void WasmObjectWriter::prepareImports(
     SmallVectorImpl<wasm::WasmImport> &Imports, MCAssembler &Asm) {
   // For now, always emit the memory import, since loads and stores are not
@@ -1482,8 +1498,10 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
     LLVM_DEBUG(dbgs() << "Processing Section " << SectionName << "  group "
                       << Section.getGroup() << "\n";);
 
-    // .init_array sections are handled specially elsewhere.
-    if (SectionName.starts_with(".init_array"))
+    // .init_array sections are handled specially elsewhere, include them in
+    // data segments if and only if referenced by a symbol.
+    if (SectionName.starts_with(".init_array") &&
+        !isSectionReferenced(Asm, Section))
       continue;
 
     // Code is handled separately
@@ -1853,49 +1871,54 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
     if (EmptyFrag.getKind() != MCFragment::FT_Data)
       report_fatal_error(".init_array section should be aligned");
 
-    const MCFragment &AlignFrag = *EmptyFrag.getNext();
-    if (AlignFrag.getKind() != MCFragment::FT_Align)
-      report_fatal_error(".init_array section should be aligned");
-    if (cast<MCAlignFragment>(AlignFrag).getAlignment() !=
-        Align(is64Bit() ? 8 : 4))
-      report_fatal_error(".init_array section should be aligned for pointers");
-
-    const MCFragment &Frag = *AlignFrag.getNext();
-    if (Frag.hasInstructions() || Frag.getKind() != MCFragment::FT_Data)
-      report_fatal_error("only data supported in .init_array section");
-
-    uint16_t Priority = UINT16_MAX;
-    unsigned PrefixLength = strlen(".init_array");
-    if (WS.getName().size() > PrefixLength) {
-      if (WS.getName()[PrefixLength] != '.')
+    const MCFragment *nextFrag = EmptyFrag.getNext();
+    while (nextFrag != nullptr) {
+      const MCFragment &AlignFrag = *nextFrag;
+      if (AlignFrag.getKind() != MCFragment::FT_Align)
+        report_fatal_error(".init_array section should be aligned");
+      if (cast<MCAlignFragment>(AlignFrag).getAlignment() !=
+          Align(is64Bit() ? 8 : 4))
         report_fatal_error(
-            ".init_array section priority should start with '.'");
-      if (WS.getName().substr(PrefixLength + 1).getAsInteger(10, Priority))
-        report_fatal_error("invalid .init_array section priority");
-    }
-    const auto &DataFrag = cast<MCDataFragment>(Frag);
-    const SmallVectorImpl<char> &Contents = DataFrag.getContents();
-    for (const uint8_t *
-             P = (const uint8_t *)Contents.data(),
-            *End = (const uint8_t *)Contents.data() + Contents.size();
-         P != End; ++P) {
-      if (*P != 0)
-        report_fatal_error("non-symbolic data in .init_array section");
-    }
-    for (const MCFixup &Fixup : DataFrag.getFixups()) {
-      assert(Fixup.getKind() ==
-             MCFixup::getKindForSize(is64Bit() ? 8 : 4, false));
-      const MCExpr *Expr = Fixup.getValue();
-      auto *SymRef = dyn_cast<MCSymbolRefExpr>(Expr);
-      if (!SymRef)
-        report_fatal_error("fixups in .init_array should be symbol references");
-      const auto &TargetSym = cast<const MCSymbolWasm>(SymRef->getSymbol());
-      if (TargetSym.getIndex() == InvalidIndex)
-        report_fatal_error("symbols in .init_array should exist in symtab");
-      if (!TargetSym.isFunction())
-        report_fatal_error("symbols in .init_array should be for functions");
-      InitFuncs.push_back(
-          std::make_pair(Priority, TargetSym.getIndex()));
+            ".init_array section should be aligned for pointers");
+
+      const MCFragment &Frag = *AlignFrag.getNext();
+      nextFrag = Frag.getNext();
+      if (Frag.hasInstructions() || Frag.getKind() != MCFragment::FT_Data)
+        report_fatal_error("only data supported in .init_array section");
+
+      uint16_t Priority = UINT16_MAX;
+      unsigned PrefixLength = strlen(".init_array");
+      if (WS.getName().size() > PrefixLength) {
+        if (WS.getName()[PrefixLength] != '.')
+          report_fatal_error(
+              ".init_array section priority should start with '.'");
+        if (WS.getName().substr(PrefixLength + 1).getAsInteger(10, Priority))
+          report_fatal_error("invalid .init_array section priority");
+      }
+      const auto &DataFrag = cast<MCDataFragment>(Frag);
+      const SmallVectorImpl<char> &Contents = DataFrag.getContents();
+      for (const uint8_t *
+               P = (const uint8_t *)Contents.data(),
+              *End = (const uint8_t *)Contents.data() + Contents.size();
+           P != End; ++P) {
+        if (*P != 0)
+          report_fatal_error("non-symbolic data in .init_array section");
+      }
+      for (const MCFixup &Fixup : DataFrag.getFixups()) {
+        assert(Fixup.getKind() ==
+               MCFixup::getKindForSize(is64Bit() ? 8 : 4, false));
+        const MCExpr *Expr = Fixup.getValue();
+        auto *SymRef = dyn_cast<MCSymbolRefExpr>(Expr);
+        if (!SymRef)
+          report_fatal_error(
+              "fixups in .init_array should be symbol references");
+        const auto &TargetSym = cast<const MCSymbolWasm>(SymRef->getSymbol());
+        if (TargetSym.getIndex() == InvalidIndex)
+          report_fatal_error("symbols in .init_array should exist in symtab");
+        if (!TargetSym.isFunction())
+          report_fatal_error("symbols in .init_array should be for functions");
+        InitFuncs.push_back(std::make_pair(Priority, TargetSym.getIndex()));
+      }
     }
   }
 
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 5d8f3dbdaadad..04ea8fa3511a7 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -1180,11 +1180,8 @@ void XCOFFWriter::writeSymbolTable(MCAssembler &Asm) {
       LangID = XCOFF::TB_Fortran;
     else
       LangID = XCOFF::TB_CPLUSPLUS;
-    uint8_t CpuID;
-    if (is64Bit())
-      CpuID = XCOFF::TCPU_PPC64;
-    else
-      CpuID = XCOFF::TCPU_COM;
+
+    uint8_t CpuID = XCOFF::getCpuID(getCPUType());
 
     int NumberOfFileAuxEntries = 1;
     if (!Vers.empty())
diff --git a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
index 3c1c35adf8897..cfab40a1c1595 100644
--- a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
+++ b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
@@ -24,6 +24,11 @@ namespace mca {
 
 const unsigned WriteRef::INVALID_IID = std::numeric_limits<unsigned>::max();
 
+static std::function<bool(MCPhysReg)>
+isNonArtificial(const MCRegisterInfo &MRI) {
+  return [&MRI](MCPhysReg R) { return !MRI.isArtificial(R); };
+}
+
 WriteRef::WriteRef(unsigned SourceIndex, WriteState *WS)
     : IID(SourceIndex), WriteBackCycle(), WriteResID(), RegisterID(),
       Write(WS) {}
@@ -282,7 +287,8 @@ void RegisterFile::addRegisterWrite(WriteRef Write,
   MCPhysReg ZeroRegisterID =
       WS.clearsSuperRegisters() ? RegID : WS.getRegisterID();
   ZeroRegisters.setBitVal(ZeroRegisterID, IsWriteZero);
-  for (MCPhysReg I : MRI.subregs(ZeroRegisterID))
+  for (MCPhysReg I :
+       make_filter_range(MRI.subregs(ZeroRegisterID), isNonArtificial(MRI)))
     ZeroRegisters.setBitVal(I, IsWriteZero);
 
   // If this move has been eliminated, then method tryEliminateMoveOrSwap should
@@ -304,7 +310,8 @@ void RegisterFile::addRegisterWrite(WriteRef Write,
     // Update the mapping for register RegID including its sub-registers.
     RegisterMappings[RegID].first = Write;
     RegisterMappings[RegID].second.AliasRegID = 0U;
-    for (MCPhysReg I : MRI.subregs(RegID)) {
+    for (MCPhysReg I :
+         make_filter_range(MRI.subregs(RegID), isNonArtificial(MRI))) {
       RegisterMappings[I].first = Write;
       RegisterMappings[I].second.AliasRegID = 0U;
     }
@@ -472,7 +479,8 @@ bool RegisterFile::tryEliminateMoveOrSwap(MutableArrayRef<WriteState> Writes,
       AliasedReg = RMAlias.AliasRegID;
 
     RegisterMappings[AliasReg].second.AliasRegID = AliasedReg;
-    for (MCPhysReg I : MRI.subregs(AliasReg))
+    for (MCPhysReg I :
+         make_filter_range(MRI.subregs(AliasReg), isNonArtificial(MRI)))
       RegisterMappings[I].second.AliasRegID = AliasedReg;
 
     if (ZeroRegisters[RS.getRegisterID()]) {
diff --git a/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp b/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
index a3d4ba3a94f7a..93bc6631e64c8 100644
--- a/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
@@ -360,13 +360,10 @@ Error MachOLayoutBuilder::layoutTail(uint64_t Offset) {
           MLC.dysymtab_command_data.nextrel != 0)
         return createStringError(llvm::errc::not_supported,
                                  "shared library is not yet supported");
-
-      if (!O.IndirectSymTable.Symbols.empty()) {
-        MLC.dysymtab_command_data.indirectsymoff = StartOfIndirectSymbols;
-        MLC.dysymtab_command_data.nindirectsyms =
-            O.IndirectSymTable.Symbols.size();
-      }
-
+      MLC.dysymtab_command_data.indirectsymoff =
+          O.IndirectSymTable.Symbols.size() ? StartOfIndirectSymbols : 0;
+      MLC.dysymtab_command_data.nindirectsyms =
+          O.IndirectSymTable.Symbols.size();
       updateDySymTab(MLC);
       break;
     }
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index e755e8161d36d..92f31c909efd4 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -707,7 +707,7 @@ void Archive::setFirstRegular(const Child &C) {
 
 Archive::Archive(MemoryBufferRef Source, Error &Err)
     : Binary(Binary::ID_Archive, Source) {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   StringRef Buffer = Data.getBuffer();
   // Check for sufficient magic.
   if (Buffer.starts_with(ThinArchiveMagic)) {
diff --git a/llvm/lib/Object/GOFFObjectFile.cpp b/llvm/lib/Object/GOFFObjectFile.cpp
index 3b8704f28fdbb..e3c4383d27aad 100644
--- a/llvm/lib/Object/GOFFObjectFile.cpp
+++ b/llvm/lib/Object/GOFFObjectFile.cpp
@@ -35,7 +35,7 @@ ObjectFile::createGOFFObjectFile(MemoryBufferRef Object) {
 
 GOFFObjectFile::GOFFObjectFile(MemoryBufferRef Object, Error &Err)
     : ObjectFile(Binary::ID_GOFF, Object) {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   // Object file isn't the right size, bail out early.
   if ((Object.getBufferSize() % GOFF::RecordLength) != 0) {
     Err = createStringError(
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 46ed3b2ff7c58..69d36e6a77db7 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -1270,7 +1270,7 @@ MachOObjectFile::MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian,
                                  size_t MachOFilesetEntryOffset)
     : ObjectFile(getMachOType(IsLittleEndian, Is64bits), Object),
       MachOFilesetEntryOffset(MachOFilesetEntryOffset) {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   uint64_t SizeOfHeaders;
   uint32_t cputype;
   if (is64Bit()) {
diff --git a/llvm/lib/Object/MachOUniversal.cpp b/llvm/lib/Object/MachOUniversal.cpp
index 655875f3396c1..31bb2a619ff33 100644
--- a/llvm/lib/Object/MachOUniversal.cpp
+++ b/llvm/lib/Object/MachOUniversal.cpp
@@ -131,7 +131,7 @@ MachOUniversalBinary::create(MemoryBufferRef Source) {
 MachOUniversalBinary::MachOUniversalBinary(MemoryBufferRef Source, Error &Err)
     : Binary(Binary::ID_MachOUniversalBinary, Source), Magic(0),
       NumberOfObjects(0) {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   if (Data.getBufferSize() < sizeof(MachO::fat_header)) {
     Err = make_error<GenericBinaryError>("File too small to be a Mach-O "
                                          "universal file",
diff --git a/llvm/lib/Object/Minidump.cpp b/llvm/lib/Object/Minidump.cpp
index a96e23893bfba..41a5da73c7783 100644
--- a/llvm/lib/Object/Minidump.cpp
+++ b/llvm/lib/Object/Minidump.cpp
@@ -145,7 +145,7 @@ MinidumpFile::create(MemoryBufferRef Source) {
 
 iterator_range<MinidumpFile::FallibleMemory64Iterator>
 MinidumpFile::getMemory64List(Error &Err) const {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   auto end = FallibleMemory64Iterator::end(Memory64Iterator::end());
   Expected<minidump::Memory64ListHeader> ListHeader = getMemoryList64Header();
   if (!ListHeader) {
diff --git a/llvm/lib/Object/RecordStreamer.h b/llvm/lib/Object/RecordStreamer.h
index 70b41f270720b..d9ef27c790168 100644
--- a/llvm/lib/Object/RecordStreamer.h
+++ b/llvm/lib/Object/RecordStreamer.h
@@ -10,6 +10,7 @@
 #define LLVM_LIB_OBJECT_RECORDSTREAMER_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCStreamer.h"
@@ -32,7 +33,7 @@ class RecordStreamer : public MCStreamer {
   // Map of aliases created by .symver directives, saved so we can update
   // their symbol binding after parsing complete. This maps from each
   // aliasee to its list of aliases.
-  DenseMap<const MCSymbol *, std::vector<StringRef>> SymverAliasMap;
+  MapVector<const MCSymbol *, std::vector<StringRef>> SymverAliasMap;
 
   /// Get the state recorded for the given symbol.
   State getSymbolState(const MCSymbol *Sym);
diff --git a/llvm/lib/Object/TapiUniversal.cpp b/llvm/lib/Object/TapiUniversal.cpp
index 6667a68f5c8c6..74e0c519ddfdb 100644
--- a/llvm/lib/Object/TapiUniversal.cpp
+++ b/llvm/lib/Object/TapiUniversal.cpp
@@ -22,7 +22,7 @@ using namespace object;
 TapiUniversal::TapiUniversal(MemoryBufferRef Source, Error &Err)
     : Binary(ID_TapiUniversal, Source) {
   Expected<std::unique_ptr<InterfaceFile>> Result = TextAPIReader::get(Source);
-  ErrorAsOutParameter ErrAsOuParam(&Err);
+  ErrorAsOutParameter ErrAsOuParam(Err);
   if (!Result) {
     Err = Result.takeError();
     return;
diff --git a/llvm/lib/Object/WasmObjectFile.cpp b/llvm/lib/Object/WasmObjectFile.cpp
index dcd728dfa2d95..2c9b878a4cde9 100644
--- a/llvm/lib/Object/WasmObjectFile.cpp
+++ b/llvm/lib/Object/WasmObjectFile.cpp
@@ -344,7 +344,7 @@ static Error readSection(WasmSection &Section, WasmObjectFile::ReadContext &Ctx,
 
 WasmObjectFile::WasmObjectFile(MemoryBufferRef Buffer, Error &Err)
     : ObjectFile(Binary::ID_Wasm, Buffer) {
-  ErrorAsOutParameter ErrAsOutParam(&Err);
+  ErrorAsOutParameter ErrAsOutParam(Err);
   Header.Magic = getData().substr(0, 4);
   if (Header.Magic != StringRef("\0asm", 4)) {
     Err = make_error<StringError>("invalid magic number",
diff --git a/llvm/lib/Object/XCOFFObjectFile.cpp b/llvm/lib/Object/XCOFFObjectFile.cpp
index cc0da5db52425..df3b2a091aec2 100644
--- a/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -1414,7 +1414,7 @@ TBVectorExt::TBVectorExt(StringRef TBvectorStrRef, Error &Err) {
   unsigned ParmsNum =
       GETVALUEWITHMASKSHIFT(NumberOfVectorParmsMask, NumberOfVectorParmsShift);
 
-  ErrorAsOutParameter EAO(&Err);
+  ErrorAsOutParameter EAO(Err);
   Expected<SmallString<32>> VecParmsTypeOrError =
       parseVectorParmsType(VecParmsTypeValue, ParmsNum);
   if (!VecParmsTypeOrError)
@@ -1458,7 +1458,7 @@ XCOFFTracebackTable::create(const uint8_t *Ptr, uint64_t &Size, bool Is64Bit) {
 XCOFFTracebackTable::XCOFFTracebackTable(const uint8_t *Ptr, uint64_t &Size,
                                          Error &Err, bool Is64Bit)
     : TBPtr(Ptr), Is64BitObj(Is64Bit) {
-  ErrorAsOutParameter EAO(&Err);
+  ErrorAsOutParameter EAO(Err);
   DataExtractor DE(ArrayRef<uint8_t>(Ptr, Size), /*IsLittleEndian=*/false,
                    /*AddressSize=*/0);
   DataExtractor::Cursor Cur(/*Offset=*/0);
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 476334024151a..5daf6c32ec936 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -1799,6 +1799,21 @@ void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
   if (!Section.Notes)
     return;
 
+  unsigned Align;
+  switch (Section.AddressAlign) {
+  case 0:
+  case 4:
+    Align = 4;
+    break;
+  case 8:
+    Align = 8;
+    break;
+  default:
+    reportError(Section.Name + ": invalid alignment for a note section: 0x" +
+                Twine::utohexstr(Section.AddressAlign));
+    return;
+  }
+
   uint64_t Offset = CBA.tell();
   for (const ELFYAML::NoteEntry &NE : *Section.Notes) {
     // Write name size.
@@ -1820,14 +1835,15 @@ void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
     if (!NE.Name.empty()) {
       CBA.write(NE.Name.data(), NE.Name.size());
       CBA.write('\0');
-      CBA.padToAlignment(4);
     }
 
     // Write description and padding.
     if (NE.Desc.binary_size() != 0) {
+      CBA.padToAlignment(Align);
       CBA.writeAsBinary(NE.Desc);
-      CBA.padToAlignment(4);
     }
+
+    CBA.padToAlignment(Align);
   }
 
   SHeader.sh_size = CBA.tell() - Offset;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c615d91cc8ac2..52507a604c635 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -98,8 +98,10 @@
 #include "llvm/CodeGen/InterleavedAccess.h"
 #include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/JMCInstrumenter.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/LocalStackSlotAllocation.h"
 #include "llvm/CodeGen/LowerEmuTLS.h"
@@ -130,6 +132,7 @@
 #include "llvm/CodeGen/ShadowStackGCLowering.h"
 #include "llvm/CodeGen/SjLjEHPrepare.h"
 #include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/SpillPlacement.h"
 #include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/TailDuplication.h"
@@ -327,6 +330,7 @@
 #include "llvm/Transforms/Utils/DXILUpgrade.h"
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/ExtraPassManager.h"
 #include "llvm/Transforms/Utils/FixIrreducible.h"
 #include "llvm/Transforms/Utils/HelloWorld.h"
 #include "llvm/Transforms/Utils/IRNormalizer.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 1428d90477f59..8dc7a20639757 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -134,6 +134,7 @@
 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 #include "llvm/Transforms/Utils/CountVisits.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/ExtraPassManager.h"
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 #include "llvm/Transforms/Utils/Mem2Reg.h"
@@ -677,7 +678,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
     LPM2.addPass(IndVarSimplifyPass());
 
   {
-    ExtraSimpleLoopUnswitchPassManager ExtraPasses;
+    ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;
     ExtraPasses.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
                                                OptimizationLevel::O3));
     LPM2.addPass(std::move(ExtraPasses));
@@ -1330,7 +1331,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
   FPM.addPass(InstCombinePass());
 
   if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
-    ExtraVectorPassManager ExtraPasses;
+    ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
     // At higher optimization levels, try to clean up any runtime overlap and
     // alignment checks inserted by the vectorizer. We want to track correlated
     // runtime checks for two inner loops in the same outer loop, fold any
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 7e9e2d66a06b0..30b4e0bc3af91 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -89,7 +89,7 @@ MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfilingLoweringPass())
 MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass())
-MODULE_PASS("print<ctx-prof-analysis>", CtxProfAnalysisPrinterPass(dbgs()))
+MODULE_PASS("print<ctx-prof-analysis>", CtxProfAnalysisPrinterPass(errs()))
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
@@ -116,21 +116,21 @@ MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen())
 MODULE_PASS("pgo-instr-use", PGOInstrumentationUse())
 MODULE_PASS("poison-checking", PoisonCheckingPass())
 MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass(TM))
-MODULE_PASS("print", PrintModulePass(dbgs()))
-MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs()))
-MODULE_PASS("print-callgraph-sccs", CallGraphSCCsPrinterPass(dbgs()))
-MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs()))
-MODULE_PASS("print-lcg", LazyCallGraphPrinterPass(dbgs()))
-MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(dbgs()))
+MODULE_PASS("print", PrintModulePass(errs()))
+MODULE_PASS("print-callgraph", CallGraphPrinterPass(errs()))
+MODULE_PASS("print-callgraph-sccs", CallGraphSCCsPrinterPass(errs()))
+MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(errs()))
+MODULE_PASS("print-lcg", LazyCallGraphPrinterPass(errs()))
+MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(errs()))
 MODULE_PASS("print-must-be-executed-contexts",
-            MustBeExecutedContextPrinterPass(dbgs()))
-MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs()))
-MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs()))
-MODULE_PASS("print<dxil-metadata>", DXILMetadataAnalysisPrinterPass(dbgs()))
-MODULE_PASS("print<dxil-resource>", DXILResourcePrinterPass(dbgs()))
-MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(dbgs()))
-MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(dbgs()))
-MODULE_PASS("print<reg-usage>", PhysicalRegisterUsageInfoPrinterPass(dbgs()))
+            MustBeExecutedContextPrinterPass(errs()))
+MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(errs()))
+MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(errs()))
+MODULE_PASS("print<dxil-metadata>", DXILMetadataAnalysisPrinterPass(errs()))
+MODULE_PASS("print<dxil-resource>", DXILResourcePrinterPass(errs()))
+MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(errs()))
+MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(errs()))
+MODULE_PASS("print<reg-usage>", PhysicalRegisterUsageInfoPrinterPass(errs()))
 MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
 MODULE_PASS("pseudo-probe-update", PseudoProbeUpdatePass())
 MODULE_PASS("recompute-globalsaa", RecomputeGlobalsAAPass())
@@ -138,6 +138,7 @@ MODULE_PASS("rel-lookup-table-converter", RelLookupTableConverterPass())
 MODULE_PASS("rewrite-statepoints-for-gc", RewriteStatepointsForGC())
 MODULE_PASS("rewrite-symbols", RewriteSymbolPass())
 MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass())
+MODULE_PASS("rtsan-module", ModuleRealtimeSanitizerPass())
 MODULE_PASS("sample-profile", SampleProfileLoaderPass())
 MODULE_PASS("sancov-module", SanitizerCoveragePass())
 MODULE_PASS("sanmd-module", SanitizerBinaryMetadataPass())
@@ -249,7 +250,7 @@ MODULE_PASS_WITH_PARAMS(
 MODULE_PASS_WITH_PARAMS(
     "print<structural-hash>", "StructuralHashPrinterPass",
     [](StructuralHashOptions Options) {
-      return StructuralHashPrinterPass(dbgs(), Options);
+      return StructuralHashPrinterPass(errs(), Options);
     },
     parseStructuralHashPrinterPassOptions, "detailed;call-target-ignored")
 
@@ -448,38 +449,38 @@ FUNCTION_PASS("pa-eval", PAEvalPass())
 FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass())
 FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt())
 FUNCTION_PASS("place-safepoints", PlaceSafepointsPass())
-FUNCTION_PASS("print", PrintFunctionPass(dbgs()))
+FUNCTION_PASS("print", PrintFunctionPass(errs()))
 // TODO: rename to print<foo> after NPM switch
-FUNCTION_PASS("print-alias-sets", AliasSetsPrinterPass(dbgs()))
-FUNCTION_PASS("print-cfg-sccs", CFGSCCPrinterPass(dbgs()))
-FUNCTION_PASS("print-memderefs", MemDerefPrinterPass(dbgs()))
-FUNCTION_PASS("print-mustexecute", MustExecutePrinterPass(dbgs()))
-FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(dbgs()))
-FUNCTION_PASS("print<access-info>", LoopAccessInfoPrinterPass(dbgs()))
-FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(dbgs()))
-FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(dbgs()))
-FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(dbgs()))
-FUNCTION_PASS("print<cost-model>", CostModelPrinterPass(dbgs()))
-FUNCTION_PASS("print<cycles>", CycleInfoPrinterPass(dbgs()))
-FUNCTION_PASS("print<da>", DependenceAnalysisPrinterPass(dbgs()))
-FUNCTION_PASS("print<debug-ata>", DebugAssignmentTrackingPrinterPass(dbgs()))
-FUNCTION_PASS("print<delinearization>", DelinearizationPrinterPass(dbgs()))
-FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(dbgs()))
-FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(dbgs()))
-FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
-FUNCTION_PASS("print<func-properties>", FunctionPropertiesPrinterPass(dbgs()))
-FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(dbgs()))
+FUNCTION_PASS("print-alias-sets", AliasSetsPrinterPass(errs()))
+FUNCTION_PASS("print-cfg-sccs", CFGSCCPrinterPass(errs()))
+FUNCTION_PASS("print-memderefs", MemDerefPrinterPass(errs()))
+FUNCTION_PASS("print-mustexecute", MustExecutePrinterPass(errs()))
+FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(errs()))
+FUNCTION_PASS("print<access-info>", LoopAccessInfoPrinterPass(errs()))
+FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(errs()))
+FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(errs()))
+FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(errs()))
+FUNCTION_PASS("print<cost-model>", CostModelPrinterPass(errs()))
+FUNCTION_PASS("print<cycles>", CycleInfoPrinterPass(errs()))
+FUNCTION_PASS("print<da>", DependenceAnalysisPrinterPass(errs()))
+FUNCTION_PASS("print<debug-ata>", DebugAssignmentTrackingPrinterPass(errs()))
+FUNCTION_PASS("print<delinearization>", DelinearizationPrinterPass(errs()))
+FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(errs()))
+FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(errs()))
+FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(errs()))
+FUNCTION_PASS("print<func-properties>", FunctionPropertiesPrinterPass(errs()))
+FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(errs()))
 FUNCTION_PASS("print<inliner-size-estimator>",
-              InlineSizeEstimatorAnalysisPrinterPass(dbgs()))
-FUNCTION_PASS("print<lazy-value-info>", LazyValueInfoPrinterPass(dbgs()))
-FUNCTION_PASS("print<loops>", LoopPrinterPass(dbgs()))
-FUNCTION_PASS("print<memoryssa-walker>", MemorySSAWalkerPrinterPass(dbgs()))
-FUNCTION_PASS("print<phi-values>", PhiValuesPrinterPass(dbgs()))
-FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(dbgs()))
-FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(dbgs()))
-FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(dbgs()))
-FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(dbgs()))
-FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(dbgs()))
+              InlineSizeEstimatorAnalysisPrinterPass(errs()))
+FUNCTION_PASS("print<lazy-value-info>", LazyValueInfoPrinterPass(errs()))
+FUNCTION_PASS("print<loops>", LoopPrinterPass(errs()))
+FUNCTION_PASS("print<memoryssa-walker>", MemorySSAWalkerPrinterPass(errs()))
+FUNCTION_PASS("print<phi-values>", PhiValuesPrinterPass(errs()))
+FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(errs()))
+FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(errs()))
+FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(errs()))
+FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(errs()))
+FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(errs()))
 FUNCTION_PASS("reassociate", ReassociatePass())
 FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
 FUNCTION_PASS("reg2mem", RegToMemPass())
@@ -588,19 +589,19 @@ FUNCTION_PASS_WITH_PARAMS(
 FUNCTION_PASS_WITH_PARAMS(
     "print<da>", "DependenceAnalysisPrinterPass",
     [](bool NormalizeResults) {
-      return DependenceAnalysisPrinterPass(dbgs(), NormalizeResults);
+      return DependenceAnalysisPrinterPass(errs(), NormalizeResults);
     },
     parseDependenceAnalysisPrinterOptions, "normalized-results")
 FUNCTION_PASS_WITH_PARAMS(
     "print<memoryssa>", "MemorySSAPrinterPass",
     [](bool NoEnsureOptimizedUses) {
-      return MemorySSAPrinterPass(dbgs(), !NoEnsureOptimizedUses);
+      return MemorySSAPrinterPass(errs(), !NoEnsureOptimizedUses);
     },
     parseMemorySSAPrinterPassOptions, "no-ensure-optimized-uses")
 FUNCTION_PASS_WITH_PARAMS(
     "print<stack-lifetime>", "StackLifetimePrinterPass",
     [](StackLifetime::LivenessType Type) {
-      return StackLifetimePrinterPass(dbgs(), Type);
+      return StackLifetimePrinterPass(errs(), Type);
     },
     parseStackLifetimeOptions, "may;must")
 FUNCTION_PASS_WITH_PARAMS(
@@ -690,11 +691,11 @@ LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass())
 LOOP_PASS("loop-unroll-full", LoopFullUnrollPass())
 LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass())
 LOOP_PASS("no-op-loop", NoOpLoopPass())
-LOOP_PASS("print", PrintLoopPass(dbgs()))
-LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(dbgs()))
-LOOP_PASS("print<iv-users>", IVUsersPrinterPass(dbgs()))
-LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(dbgs()))
-LOOP_PASS("print<loopnest>", LoopNestPrinterPass(dbgs()))
+LOOP_PASS("print", PrintLoopPass(errs()))
+LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(errs()))
+LOOP_PASS("print<iv-users>", IVUsersPrinterPass(errs()))
+LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(errs()))
+LOOP_PASS("print<loopnest>", LoopNestPrinterPass(errs()))
 #undef LOOP_PASS
 
 #ifndef LOOP_PASS_WITH_PARAMS
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 7663852236594..cac1760d3ef80 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -52,6 +52,9 @@ static InstrProfKind getProfileKindFromVersion(uint64_t Version) {
   if (Version & VARIANT_MASK_INSTR_ENTRY) {
     ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
   }
+  if (Version & VARIANT_MASK_INSTR_LOOP_ENTRIES) {
+    ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
+  }
   if (Version & VARIANT_MASK_BYTE_COVERAGE) {
     ProfileKind |= InstrProfKind::SingleByteCoverage;
   }
@@ -262,6 +265,8 @@ Error TextInstrProfReader::readHeader() {
       ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
     else if (Str.equals_insensitive("not_entry_first"))
       ProfileKind &= ~InstrProfKind::FunctionEntryInstrumentation;
+    else if (Str.equals_insensitive("instrument_loop_entries"))
+      ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
     else if (Str.equals_insensitive("single_byte_coverage"))
       ProfileKind |= InstrProfKind::SingleByteCoverage;
     else if (Str.equals_insensitive("temporal_prof_traces")) {
@@ -1659,6 +1664,22 @@ IndexedMemProfReader::getMemProfCallerCalleePairs() const {
   return Pairs;
 }
 
+memprof::AllMemProfData IndexedMemProfReader::getAllMemProfData() const {
+  memprof::AllMemProfData AllMemProfData;
+  AllMemProfData.HeapProfileRecords.reserve(
+      MemProfRecordTable->getNumEntries());
+  for (uint64_t Key : MemProfRecordTable->keys()) {
+    auto Record = getMemProfRecord(Key);
+    if (Record.takeError())
+      continue;
+    memprof::GUIDMemProfRecordPair Pair;
+    Pair.GUID = Key;
+    Pair.Record = std::move(*Record);
+    AllMemProfData.HeapProfileRecords.push_back(std::move(Pair));
+  }
+  return AllMemProfData;
+}
+
 Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,
                                                 uint64_t FuncHash,
                                                 std::vector<uint64_t> &Counts) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index d8ab18d213e3d..f112ea2efcaa9 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -810,6 +810,45 @@ uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
   return BackPatchStartOffset;
 }
 
+Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {
+  // BinaryIdSection has two parts:
+  // 1. uint64_t BinaryIdsSectionSize
+  // 2. list of binary ids that consist of:
+  //    a. uint64_t BinaryIdLength
+  //    b. uint8_t  BinaryIdData
+  //    c. uint8_t  Padding (if necessary)
+  // Calculate size of binary section.
+  uint64_t BinaryIdsSectionSize = 0;
+
+  // Remove duplicate binary ids.
+  llvm::sort(BinaryIds);
+  BinaryIds.erase(llvm::unique(BinaryIds), BinaryIds.end());
+
+  for (const auto &BI : BinaryIds) {
+    // Increment by binary id length data type size.
+    BinaryIdsSectionSize += sizeof(uint64_t);
+    // Increment by binary id data length, aligned to 8 bytes.
+    BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
+  }
+  // Write binary ids section size.
+  OS.write(BinaryIdsSectionSize);
+
+  for (const auto &BI : BinaryIds) {
+    uint64_t BILen = BI.size();
+    // Write binary id length.
+    OS.write(BILen);
+    // Write binary id data.
+    for (unsigned K = 0; K < BILen; K++)
+      OS.writeByte(BI[K]);
+    // Write padding if necessary.
+    uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
+    for (unsigned K = 0; K < PaddingSize; K++)
+      OS.writeByte(0);
+  }
+
+  return Error::success();
+}
+
 Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
   std::vector<std::string> VTableNameStrs;
   for (StringRef VTableName : VTableNames.keys())
@@ -877,6 +916,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   if (static_cast<bool>(ProfileKind &
                         InstrProfKind::FunctionEntryInstrumentation))
     Header.Version |= VARIANT_MASK_INSTR_ENTRY;
+  if (static_cast<bool>(ProfileKind &
+                        InstrProfKind::LoopEntriesInstrumentation))
+    Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
   if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
     Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
   if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
@@ -917,41 +959,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
       return E;
   }
 
-  // BinaryIdSection has two parts:
-  // 1. uint64_t BinaryIdsSectionSize
-  // 2. list of binary ids that consist of:
-  //    a. uint64_t BinaryIdLength
-  //    b. uint8_t  BinaryIdData
-  //    c. uint8_t  Padding (if necessary)
   uint64_t BinaryIdSectionStart = OS.tell();
-  // Calculate size of binary section.
-  uint64_t BinaryIdsSectionSize = 0;
-
-  // Remove duplicate binary ids.
-  llvm::sort(BinaryIds);
-  BinaryIds.erase(llvm::unique(BinaryIds), BinaryIds.end());
-
-  for (const auto &BI : BinaryIds) {
-    // Increment by binary id length data type size.
-    BinaryIdsSectionSize += sizeof(uint64_t);
-    // Increment by binary id data length, aligned to 8 bytes.
-    BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
-  }
-  // Write binary ids section size.
-  OS.write(BinaryIdsSectionSize);
-
-  for (const auto &BI : BinaryIds) {
-    uint64_t BILen = BI.size();
-    // Write binary id length.
-    OS.write(BILen);
-    // Write binary id data.
-    for (unsigned K = 0; K < BILen; K++)
-      OS.writeByte(BI[K]);
-    // Write padding if necessary.
-    uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
-    for (unsigned K = 0; K < PaddingSize; K++)
-      OS.writeByte(0);
-  }
+  if (auto E = writeBinaryIds(OS))
+    return E;
 
   uint64_t VTableNamesSectionStart = OS.tell();
 
@@ -1120,6 +1130,10 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
   if (static_cast<bool>(ProfileKind &
                         InstrProfKind::FunctionEntryInstrumentation))
     OS << "# Always instrument the function entry block\n:entry_first\n";
+  if (static_cast<bool>(ProfileKind &
+                        InstrProfKind::LoopEntriesInstrumentation))
+    OS << "# Always instrument the loop entry "
+          "blocks\n:instrument_loop_entries\n";
   if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
     OS << "# Instrument block coverage\n:single_byte_coverage\n";
   InstrProfSymtab Symtab;
diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index 921ff3429fed7..19ae9a79ea519 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -40,11 +40,12 @@
 #include "llvm/Support/Path.h"
 
 #define DEBUG_TYPE "memprof"
+
 namespace llvm {
 namespace memprof {
 namespace {
 template <class T = uint64_t> inline T alignedRead(const char *Ptr) {
-  static_assert(std::is_pod<T>::value, "Not a pod type.");
+  static_assert(std::is_integral_v<T>, "Not an integral type");
   assert(reinterpret_cast<size_t>(Ptr) % sizeof(T) == 0 && "Unaligned Read");
   return *reinterpret_cast<const T *>(Ptr);
 }
@@ -499,15 +500,14 @@ Error RawMemProfReader::mapRawProfileToRecords() {
       Callstack.append(Frames.begin(), Frames.end());
     }
 
-    CallStackId CSId = hashCallStack(Callstack);
-    MemProfData.CallStacks.insert({CSId, Callstack});
+    CallStackId CSId = MemProfData.addCallStack(Callstack);
 
     // We attach the memprof record to each function bottom-up including the
     // first non-inline frame.
     for (size_t I = 0; /*Break out using the condition below*/; I++) {
       const Frame &F = idToFrame(Callstack[I]);
       IndexedMemProfRecord &Record = MemProfData.Records[F.Function];
-      Record.AllocSites.emplace_back(Callstack, CSId, MIB);
+      Record.AllocSites.emplace_back(CSId, MIB);
 
       if (!F.IsInlineFrame)
         break;
@@ -519,12 +519,8 @@ Error RawMemProfReader::mapRawProfileToRecords() {
     // Some functions may have only callsite data and no allocation data. Here
     // we insert a new entry for callsite data if we need to.
     IndexedMemProfRecord &Record = MemProfData.Records[Id];
-    for (LocationPtr Loc : Locs) {
-      CallStackId CSId = hashCallStack(*Loc);
-      MemProfData.CallStacks.insert({CSId, *Loc});
-      Record.CallSites.push_back(*Loc);
-      Record.CallSiteIds.push_back(CSId);
-    }
+    for (LocationPtr Loc : Locs)
+      Record.CallSiteIds.push_back(MemProfData.addCallStack(*Loc));
   }
 
   return Error::success();
@@ -584,9 +580,7 @@ Error RawMemProfReader::symbolizeAndFilterStackFrames(
           GuidToSymbolName.insert({Guid, CanonicalName.str()});
         }
 
-        const FrameId Hash = F.hash();
-        MemProfData.Frames.insert({Hash, F});
-        SymbolizedFrame[VAddr].push_back(Hash);
+        SymbolizedFrame[VAddr].push_back(MemProfData.addFrame(F));
       }
     }
 
@@ -756,5 +750,71 @@ Error RawMemProfReader::readNextRecord(
   };
   return MemProfReader::readNextRecord(GuidRecord, IdToFrameCallback);
 }
+
+Expected<std::unique_ptr<YAMLMemProfReader>>
+YAMLMemProfReader::create(const Twine &Path) {
+  auto BufferOr = MemoryBuffer::getFileOrSTDIN(Path);
+  if (std::error_code EC = BufferOr.getError())
+    return report(errorCodeToError(EC), Path.getSingleStringRef());
+
+  std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
+  return create(std::move(Buffer));
+}
+
+Expected<std::unique_ptr<YAMLMemProfReader>>
+YAMLMemProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
+  auto Reader = std::make_unique<YAMLMemProfReader>();
+  Reader->parse(Buffer->getBuffer());
+  return std::move(Reader);
+}
+
+bool YAMLMemProfReader::hasFormat(const StringRef Path) {
+  auto BufferOr = MemoryBuffer::getFileOrSTDIN(Path);
+  if (!BufferOr)
+    return false;
+
+  std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
+  return hasFormat(*Buffer);
+}
+
+bool YAMLMemProfReader::hasFormat(const MemoryBuffer &Buffer) {
+  return Buffer.getBuffer().starts_with("---");
+}
+
+void YAMLMemProfReader::parse(StringRef YAMLData) {
+  memprof::AllMemProfData Doc;
+  yaml::Input Yin(YAMLData);
+
+  Yin >> Doc;
+  if (Yin.error())
+    return;
+
+  // Add a call stack to MemProfData.CallStacks and return its CallStackId.
+  auto AddCallStack = [&](ArrayRef<Frame> CallStack) -> CallStackId {
+    SmallVector<FrameId> IndexedCallStack;
+    IndexedCallStack.reserve(CallStack.size());
+    for (const Frame &F : CallStack)
+      IndexedCallStack.push_back(MemProfData.addFrame(F));
+    return MemProfData.addCallStack(std::move(IndexedCallStack));
+  };
+
+  for (const auto &[GUID, Record] : Doc.HeapProfileRecords) {
+    IndexedMemProfRecord IndexedRecord;
+
+    // Convert AllocationInfo to IndexedAllocationInfo.
+    for (const AllocationInfo &AI : Record.AllocSites) {
+      CallStackId CSId = AddCallStack(AI.CallStack);
+      IndexedRecord.AllocSites.emplace_back(CSId, AI.Info);
+    }
+
+    // Populate CallSiteIds.
+    for (const auto &CallSite : Record.CallSites) {
+      CallStackId CSId = AddCallStack(CallSite);
+      IndexedRecord.CallSiteIds.push_back(CSId);
+    }
+
+    MemProfData.Records.try_emplace(GUID, std::move(IndexedRecord));
+  }
+}
 } // namespace memprof
 } // namespace llvm
diff --git a/llvm/lib/SYCLLowerIR/CMakeLists.txt b/llvm/lib/SYCLLowerIR/CMakeLists.txt
index bc221b912977c..c9bd07eb7e78e 100644
--- a/llvm/lib/SYCLLowerIR/CMakeLists.txt
+++ b/llvm/lib/SYCLLowerIR/CMakeLists.txt
@@ -6,7 +6,7 @@ if (NOT TARGET LLVMGenXIntrinsics)
 
     # Date: Jul 25, 2024
     # Support MMX removal in LLVM upstream
-    set(LLVMGenXIntrinsics_GIT_TAG dpcpp_staging)
+    set(LLVMGenXIntrinsics_GIT_TAG 4e51b2467104a257c22788e343dafbdde72e28bb)
 
     message(STATUS "vc-intrinsics repo is missing. Will try to download it from ${LLVMGenXIntrinsics_GIT_REPO}")
     include(FetchContent)
diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index 81e297c3ab033..c9adfca8b3b76 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -164,42 +164,7 @@ static constexpr fltSemantics semFloat4E2M1FN = {
     2, 0, 2, 4, fltNonfiniteBehavior::FiniteOnly};
 static constexpr fltSemantics semX87DoubleExtended = {16383, -16382, 64, 80};
 static constexpr fltSemantics semBogus = {0, 0, 0, 0};
-
-/* The IBM double-double semantics. Such a number consists of a pair of IEEE
-   64-bit doubles (Hi, Lo), where |Hi| > |Lo|, and if normal,
-   (double)(Hi + Lo) == Hi. The numeric value it's modeling is Hi + Lo.
-   Therefore it has two 53-bit mantissa parts that aren't necessarily adjacent
-   to each other, and two 11-bit exponents.
-
-   Note: we need to make the value different from semBogus as otherwise
-   an unsafe optimization may collapse both values to a single address,
-   and we heavily rely on them having distinct addresses.             */
 static constexpr fltSemantics semPPCDoubleDouble = {-1, 0, 0, 128};
-
-/* These are legacy semantics for the fallback, inaccrurate implementation of
-   IBM double-double, if the accurate semPPCDoubleDouble doesn't handle the
-   operation. It's equivalent to having an IEEE number with consecutive 106
-   bits of mantissa and 11 bits of exponent.
-
-   It's not equivalent to IBM double-double. For example, a legit IBM
-   double-double, 1 + epsilon:
-
-     1 + epsilon = 1 + (1 >> 1076)
-
-   is not representable by a consecutive 106 bits of mantissa.
-
-   Currently, these semantics are used in the following way:
-
-     semPPCDoubleDouble -> (IEEEdouble, IEEEdouble) ->
-     (64-bit APInt, 64-bit APInt) -> (128-bit APInt) ->
-     semPPCDoubleDoubleLegacy -> IEEE operations
-
-   We use bitcastToAPInt() to get the bit representation (in APInt) of the
-   underlying IEEEdouble, then use the APInt constructor to construct the
-   legacy IEEE float.
-
-   TODO: Implement all operations in semPPCDoubleDouble, and delete these
-   semantics.  */
 static constexpr fltSemantics semPPCDoubleDoubleLegacy = {1023, -1022 + 53,
                                                           53 + 53, 128};
 
@@ -217,6 +182,8 @@ const llvm::fltSemantics &APFloatBase::EnumToSemantics(Semantics S) {
     return IEEEquad();
   case S_PPCDoubleDouble:
     return PPCDoubleDouble();
+  case S_PPCDoubleDoubleLegacy:
+    return PPCDoubleDoubleLegacy();
   case S_Float8E5M2:
     return Float8E5M2();
   case S_Float8E5M2FNUZ:
@@ -261,6 +228,8 @@ APFloatBase::SemanticsToEnum(const llvm::fltSemantics &Sem) {
     return S_IEEEquad;
   else if (&Sem == &llvm::APFloat::PPCDoubleDouble())
     return S_PPCDoubleDouble;
+  else if (&Sem == &llvm::APFloat::PPCDoubleDoubleLegacy())
+    return S_PPCDoubleDoubleLegacy;
   else if (&Sem == &llvm::APFloat::Float8E5M2())
     return S_Float8E5M2;
   else if (&Sem == &llvm::APFloat::Float8E5M2FNUZ())
@@ -299,6 +268,9 @@ const fltSemantics &APFloatBase::IEEEquad() { return semIEEEquad; }
 const fltSemantics &APFloatBase::PPCDoubleDouble() {
   return semPPCDoubleDouble;
 }
+const fltSemantics &APFloatBase::PPCDoubleDoubleLegacy() {
+  return semPPCDoubleDoubleLegacy;
+}
 const fltSemantics &APFloatBase::Float8E5M2() { return semFloat8E5M2; }
 const fltSemantics &APFloatBase::Float8E5M2FNUZ() { return semFloat8E5M2FNUZ; }
 const fltSemantics &APFloatBase::Float8E4M3() { return semFloat8E4M3; }
@@ -3574,7 +3546,7 @@ APInt IEEEFloat::convertF80LongDoubleAPFloatToAPInt() const {
   return APInt(80, words);
 }
 
-APInt IEEEFloat::convertPPCDoubleDoubleAPFloatToAPInt() const {
+APInt IEEEFloat::convertPPCDoubleDoubleLegacyAPFloatToAPInt() const {
   assert(semantics == (const llvm::fltSemantics *)&semPPCDoubleDoubleLegacy);
   assert(partCount()==2);
 
@@ -3796,7 +3768,7 @@ APInt IEEEFloat::bitcastToAPInt() const {
     return convertQuadrupleAPFloatToAPInt();
 
   if (semantics == (const llvm::fltSemantics *)&semPPCDoubleDoubleLegacy)
-    return convertPPCDoubleDoubleAPFloatToAPInt();
+    return convertPPCDoubleDoubleLegacyAPFloatToAPInt();
 
   if (semantics == (const llvm::fltSemantics *)&semFloat8E5M2)
     return convertFloat8E5M2APFloatToAPInt();
@@ -3900,7 +3872,7 @@ void IEEEFloat::initFromF80LongDoubleAPInt(const APInt &api) {
   }
 }
 
-void IEEEFloat::initFromPPCDoubleDoubleAPInt(const APInt &api) {
+void IEEEFloat::initFromPPCDoubleDoubleLegacyAPInt(const APInt &api) {
   uint64_t i1 = api.getRawData()[0];
   uint64_t i2 = api.getRawData()[1];
   opStatus fs;
@@ -4119,7 +4091,7 @@ void IEEEFloat::initFromAPInt(const fltSemantics *Sem, const APInt &api) {
   if (Sem == &semIEEEquad)
     return initFromQuadrupleAPInt(api);
   if (Sem == &semPPCDoubleDoubleLegacy)
-    return initFromPPCDoubleDoubleAPInt(api);
+    return initFromPPCDoubleDoubleLegacyAPInt(api);
   if (Sem == &semFloat8E5M2)
     return initFromFloat8E5M2APInt(api);
   if (Sem == &semFloat8E5M2FNUZ)
@@ -4145,7 +4117,7 @@ void IEEEFloat::initFromAPInt(const fltSemantics *Sem, const APInt &api) {
   if (Sem == &semFloat4E2M1FN)
     return initFromFloat4E2M1FNAPInt(api);
 
-  llvm_unreachable(nullptr);
+  llvm_unreachable("unsupported semantics");
 }
 
 /// Make this number the largest magnitude normal number in the given
diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp
index bc34b23f77cc8..19977c57c08db 100644
--- a/llvm/lib/Support/BalancedPartitioning.cpp
+++ b/llvm/lib/Support/BalancedPartitioning.cpp
@@ -31,7 +31,7 @@ void BalancedPartitioning::BPThreadPool::async(Func &&F) {
 #if LLVM_ENABLE_THREADS
   // This new thread could spawn more threads, so mark it as active
   ++NumActiveThreads;
-  TheThreadPool.async([=]() {
+  TheThreadPool.async([this, F]() {
     // Run the task
     F();
 
@@ -93,7 +93,7 @@ void BalancedPartitioning::run(std::vector<BPFunctionNode> &Nodes) const {
     Nodes[I].InputOrderIndex = I;
 
   auto NodesRange = llvm::make_range(Nodes.begin(), Nodes.end());
-  auto BisectTask = [=, &TP]() {
+  auto BisectTask = [this, NodesRange, &TP]() {
     bisect(NodesRange, /*RecDepth=*/0, /*RootBucket=*/1, /*Offset=*/0, TP);
   };
   if (TP) {
@@ -147,10 +147,11 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes,
   auto LeftNodes = llvm::make_range(Nodes.begin(), NodesMid);
   auto RightNodes = llvm::make_range(NodesMid, Nodes.end());
 
-  auto LeftRecTask = [=, &TP]() {
+  auto LeftRecTask = [this, LeftNodes, RecDepth, LeftBucket, Offset, &TP]() {
     bisect(LeftNodes, RecDepth + 1, LeftBucket, Offset, TP);
   };
-  auto RightRecTask = [=, &TP]() {
+  auto RightRecTask = [this, RightNodes, RecDepth, RightBucket, MidOffset,
+                       &TP]() {
     bisect(RightNodes, RecDepth + 1, RightBucket, MidOffset, TP);
   };
 
diff --git a/llvm/lib/Support/DataExtractor.cpp b/llvm/lib/Support/DataExtractor.cpp
index eac3c32cfd3b7..3da9511bf2669 100644
--- a/llvm/lib/Support/DataExtractor.cpp
+++ b/llvm/lib/Support/DataExtractor.cpp
@@ -227,7 +227,7 @@ int64_t DataExtractor::getSLEB128(uint64_t *offset_ptr, Error *Err) const {
 }
 
 void DataExtractor::skip(Cursor &C, uint64_t Length) const {
-  ErrorAsOutParameter ErrAsOut(&C.Err);
+  ErrorAsOutParameter ErrAsOut(C.Err);
   if (isError(&C.Err))
     return;
 
diff --git a/llvm/lib/Support/FormattedStream.cpp b/llvm/lib/Support/FormattedStream.cpp
index c50530e76efc0..4192893b38a76 100644
--- a/llvm/lib/Support/FormattedStream.cpp
+++ b/llvm/lib/Support/FormattedStream.cpp
@@ -75,6 +75,13 @@ void formatted_raw_ostream::UpdatePosition(const char *Ptr, size_t Size) {
   // Now scan the rest of the buffer.
   unsigned NumBytes;
   for (const char *End = Ptr + Size; Ptr < End; Ptr += NumBytes) {
+    // Fast path for printable ASCII characters without special handling.
+    if (*Ptr >= 0x20 && *Ptr <= 0x7e) {
+      NumBytes = 1;
+      ++Column;
+      continue;
+    }
+
     NumBytes = getNumBytesForUTF8(*Ptr);
 
     // The buffer might end part way through a UTF-8 code unit sequence for a
diff --git a/llvm/lib/Support/SmallPtrSet.cpp b/llvm/lib/Support/SmallPtrSet.cpp
index 1bec911f39b13..83143a7fe80fa 100644
--- a/llvm/lib/Support/SmallPtrSet.cpp
+++ b/llvm/lib/Support/SmallPtrSet.cpp
@@ -134,32 +134,33 @@ void SmallPtrSetImplBase::Grow(unsigned NewSize) {
     free(OldBuckets);
   NumNonEmpty -= NumTombstones;
   NumTombstones = 0;
+  IsSmall = false;
 }
 
 SmallPtrSetImplBase::SmallPtrSetImplBase(const void **SmallStorage,
                                          const SmallPtrSetImplBase &that) {
-  SmallArray = SmallStorage;
-
-  // If we're becoming small, prepare to insert into our stack space
-  if (that.isSmall()) {
-    CurArray = SmallArray;
-  // Otherwise, allocate new heap space (unless we were the same size)
+  IsSmall = that.isSmall();
+  if (IsSmall) {
+    // If we're becoming small, prepare to insert into our stack space
+    CurArray = SmallStorage;
   } else {
+    // Otherwise, allocate new heap space (unless we were the same size)
     CurArray = (const void**)safe_malloc(sizeof(void*) * that.CurArraySize);
   }
 
   // Copy over the that array.
-  CopyHelper(that);
+  copyHelper(that);
 }
 
 SmallPtrSetImplBase::SmallPtrSetImplBase(const void **SmallStorage,
                                          unsigned SmallSize,
+                                         const void **RHSSmallStorage,
                                          SmallPtrSetImplBase &&that) {
-  SmallArray = SmallStorage;
-  MoveHelper(SmallSize, std::move(that));
+  moveHelper(SmallStorage, SmallSize, RHSSmallStorage, std::move(that));
 }
 
-void SmallPtrSetImplBase::CopyFrom(const SmallPtrSetImplBase &RHS) {
+void SmallPtrSetImplBase::copyFrom(const void **SmallStorage,
+                                   const SmallPtrSetImplBase &RHS) {
   assert(&RHS != this && "Self-copy should be handled by the caller.");
 
   if (isSmall() && RHS.isSmall())
@@ -170,8 +171,9 @@ void SmallPtrSetImplBase::CopyFrom(const SmallPtrSetImplBase &RHS) {
   if (RHS.isSmall()) {
     if (!isSmall())
       free(CurArray);
-    CurArray = SmallArray;
-  // Otherwise, allocate new heap space (unless we were the same size)
+    CurArray = SmallStorage;
+    IsSmall = true;
+    // Otherwise, allocate new heap space (unless we were the same size)
   } else if (CurArraySize != RHS.CurArraySize) {
     if (isSmall())
       CurArray = (const void**)safe_malloc(sizeof(void*) * RHS.CurArraySize);
@@ -180,12 +182,13 @@ void SmallPtrSetImplBase::CopyFrom(const SmallPtrSetImplBase &RHS) {
                                              sizeof(void*) * RHS.CurArraySize);
       CurArray = T;
     }
+    IsSmall = false;
   }
 
-  CopyHelper(RHS);
+  copyHelper(RHS);
 }
 
-void SmallPtrSetImplBase::CopyHelper(const SmallPtrSetImplBase &RHS) {
+void SmallPtrSetImplBase::copyHelper(const SmallPtrSetImplBase &RHS) {
   // Copy over the new array size
   CurArraySize = RHS.CurArraySize;
 
@@ -196,39 +199,46 @@ void SmallPtrSetImplBase::CopyHelper(const SmallPtrSetImplBase &RHS) {
   NumTombstones = RHS.NumTombstones;
 }
 
-void SmallPtrSetImplBase::MoveFrom(unsigned SmallSize,
+void SmallPtrSetImplBase::moveFrom(const void **SmallStorage,
+                                   unsigned SmallSize,
+                                   const void **RHSSmallStorage,
                                    SmallPtrSetImplBase &&RHS) {
   if (!isSmall())
     free(CurArray);
-  MoveHelper(SmallSize, std::move(RHS));
+  moveHelper(SmallStorage, SmallSize, RHSSmallStorage, std::move(RHS));
 }
 
-void SmallPtrSetImplBase::MoveHelper(unsigned SmallSize,
+void SmallPtrSetImplBase::moveHelper(const void **SmallStorage,
+                                     unsigned SmallSize,
+                                     const void **RHSSmallStorage,
                                      SmallPtrSetImplBase &&RHS) {
   assert(&RHS != this && "Self-move should be handled by the caller.");
 
   if (RHS.isSmall()) {
     // Copy a small RHS rather than moving.
-    CurArray = SmallArray;
+    CurArray = SmallStorage;
     std::copy(RHS.CurArray, RHS.CurArray + RHS.NumNonEmpty, CurArray);
   } else {
     CurArray = RHS.CurArray;
-    RHS.CurArray = RHS.SmallArray;
+    RHS.CurArray = RHSSmallStorage;
   }
 
   // Copy the rest of the trivial members.
   CurArraySize = RHS.CurArraySize;
   NumNonEmpty = RHS.NumNonEmpty;
   NumTombstones = RHS.NumTombstones;
+  IsSmall = RHS.IsSmall;
 
   // Make the RHS small and empty.
   RHS.CurArraySize = SmallSize;
-  assert(RHS.CurArray == RHS.SmallArray);
   RHS.NumNonEmpty = 0;
   RHS.NumTombstones = 0;
+  RHS.IsSmall = true;
 }
 
-void SmallPtrSetImplBase::swap(SmallPtrSetImplBase &RHS) {
+void SmallPtrSetImplBase::swap(const void **SmallStorage,
+                               const void **RHSSmallStorage,
+                               SmallPtrSetImplBase &RHS) {
   if (this == &RHS) return;
 
   // We can only avoid copying elements if neither set is small.
@@ -245,42 +255,42 @@ void SmallPtrSetImplBase::swap(SmallPtrSetImplBase &RHS) {
   // If only RHS is small, copy the small elements into LHS and move the pointer
   // from LHS to RHS.
   if (!this->isSmall() && RHS.isSmall()) {
-    assert(RHS.CurArray == RHS.SmallArray);
-    std::copy(RHS.CurArray, RHS.CurArray + RHS.NumNonEmpty, this->SmallArray);
+    std::copy(RHS.CurArray, RHS.CurArray + RHS.NumNonEmpty, SmallStorage);
     std::swap(RHS.CurArraySize, this->CurArraySize);
     std::swap(this->NumNonEmpty, RHS.NumNonEmpty);
     std::swap(this->NumTombstones, RHS.NumTombstones);
     RHS.CurArray = this->CurArray;
-    this->CurArray = this->SmallArray;
+    RHS.IsSmall = false;
+    this->CurArray = SmallStorage;
+    this->IsSmall = true;
     return;
   }
 
   // If only LHS is small, copy the small elements into RHS and move the pointer
   // from RHS to LHS.
   if (this->isSmall() && !RHS.isSmall()) {
-    assert(this->CurArray == this->SmallArray);
     std::copy(this->CurArray, this->CurArray + this->NumNonEmpty,
-              RHS.SmallArray);
+              RHSSmallStorage);
     std::swap(RHS.CurArraySize, this->CurArraySize);
     std::swap(RHS.NumNonEmpty, this->NumNonEmpty);
     std::swap(RHS.NumTombstones, this->NumTombstones);
     this->CurArray = RHS.CurArray;
-    RHS.CurArray = RHS.SmallArray;
+    this->IsSmall = false;
+    RHS.CurArray = RHSSmallStorage;
+    RHS.IsSmall = true;
     return;
   }
 
   // Both a small, just swap the small elements.
   assert(this->isSmall() && RHS.isSmall());
   unsigned MinNonEmpty = std::min(this->NumNonEmpty, RHS.NumNonEmpty);
-  std::swap_ranges(this->SmallArray, this->SmallArray + MinNonEmpty,
-                   RHS.SmallArray);
+  std::swap_ranges(this->CurArray, this->CurArray + MinNonEmpty, RHS.CurArray);
   if (this->NumNonEmpty > MinNonEmpty) {
-    std::copy(this->SmallArray + MinNonEmpty,
-              this->SmallArray + this->NumNonEmpty,
-              RHS.SmallArray + MinNonEmpty);
+    std::copy(this->CurArray + MinNonEmpty, this->CurArray + this->NumNonEmpty,
+              RHS.CurArray + MinNonEmpty);
   } else {
-    std::copy(RHS.SmallArray + MinNonEmpty, RHS.SmallArray + RHS.NumNonEmpty,
-              this->SmallArray + MinNonEmpty);
+    std::copy(RHS.CurArray + MinNonEmpty, RHS.CurArray + RHS.NumNonEmpty,
+              this->CurArray + MinNonEmpty);
   }
   assert(this->CurArraySize == RHS.CurArraySize);
   std::swap(this->NumNonEmpty, RHS.NumNonEmpty);
diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp
index d259da65c5cf7..f326422138488 100644
--- a/llvm/lib/Support/YAMLTraits.cpp
+++ b/llvm/lib/Support/YAMLTraits.cpp
@@ -837,26 +837,40 @@ void Output::newLineCheck(bool EmptySequence) {
     return;
 
   unsigned Indent = StateStack.size() - 1;
-  bool OutputDash = false;
-
-  if (StateStack.back() == inSeqFirstElement ||
-      StateStack.back() == inSeqOtherElement) {
-    OutputDash = true;
-  } else if ((StateStack.size() > 1) &&
-             ((StateStack.back() == inMapFirstKey) ||
-              inFlowSeqAnyElement(StateStack.back()) ||
-              (StateStack.back() == inFlowMapFirstKey)) &&
-             inSeqAnyElement(StateStack[StateStack.size() - 2])) {
-    --Indent;
-    OutputDash = true;
+  bool PossiblyNestedSeq = false;
+  auto I = StateStack.rbegin(), E = StateStack.rend();
+
+  if (inSeqAnyElement(*I)) {
+    PossiblyNestedSeq = true; // Not possibly but always.
+    ++Indent;
+  } else if (*I == inMapFirstKey || *I == inFlowMapFirstKey ||
+             inFlowSeqAnyElement(*I)) {
+    PossiblyNestedSeq = true;
+    ++I; // Skip back().
   }
 
-  for (unsigned i = 0; i < Indent; ++i) {
-    output("  ");
+  unsigned OutputDashCount = 0;
+  if (PossiblyNestedSeq) {
+    // Count up consecutive inSeqFirstElement from the end, unless
+    // inSeqFirstElement is the top of nested sequence.
+    while (I != E) {
+      // Don't count the top of nested sequence.
+      if (!inSeqAnyElement(*I))
+        break;
+
+      ++OutputDashCount;
+
+      // Stop counting if consecutive inSeqFirstElement ends.
+      if (*I++ != inSeqFirstElement)
+        break;
+    }
   }
-  if (OutputDash) {
+
+  for (unsigned I = OutputDashCount; I < Indent; ++I)
+    output("  ");
+
+  for (unsigned I = 0; I < OutputDashCount; ++I)
     output("- ");
-  }
 }
 
 void Output::paddedKey(StringRef key) {
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index e01342ffcd3c8..8a8cd2b7356cd 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -3504,7 +3504,7 @@ bool TGParser::ParseBodyItem(Record *CurRec) {
 
   RecordVal *Field = CurRec->getValue(FieldName);
   if (!Field)
-    return TokError("Value '" + FieldName->getValue() + "' unknown!");
+    return Error(IdLoc, "Value '" + FieldName->getValue() + "' unknown!");
 
   const RecTy *Type = Field->getType();
   if (!BitList.empty() && isa<BitsRecTy>(Type)) {
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index af26fc6229237..69d07f27fa8e1 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -162,8 +162,27 @@ class AArch64AsmPrinter : public AsmPrinter {
   // Emit the sequence for AUT or AUTPAC.
   void emitPtrauthAuthResign(const MachineInstr *MI);
 
-  // Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
-  unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc);
+  // Emit the sequence to compute the discriminator.
+  //
+  // ScratchReg should be x16/x17.
+  //
+  // The returned register is either unmodified AddrDisc or x16/x17.
+  //
+  // If the expanded pseudo is allowed to clobber AddrDisc register, setting
+  // MayUseAddrAsScratch may save one MOV instruction, provided the address
+  // is already in x16/x17 (i.e. return x16/x17 which is the *modified* AddrDisc
+  // register at the same time):
+  //
+  //   mov   x17, x16
+  //   movk  x17, #1234, lsl #48
+  //   ; x16 is not used anymore
+  //
+  // can be replaced by
+  //
+  //   movk  x16, #1234, lsl #48
+  Register emitPtrauthDiscriminator(uint16_t Disc, Register AddrDisc,
+                                    Register ScratchReg,
+                                    bool MayUseAddrAsScratch = false);
 
   // Emit the sequence for LOADauthptrstatic
   void LowerLOADauthptrstatic(const MachineInstr &MI);
@@ -1726,8 +1745,11 @@ void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
   }
 }
 
-unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
-                                                     unsigned AddrDisc) {
+Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
+                                                     Register AddrDisc,
+                                                     Register ScratchReg,
+                                                     bool MayUseAddrAsScratch) {
+  assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17);
   // So far we've used NoRegister in pseudos.  Now we need real encodings.
   if (AddrDisc == AArch64::NoRegister)
     AddrDisc = AArch64::XZR;
@@ -1737,16 +1759,24 @@ unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
   if (!Disc)
     return AddrDisc;
 
-  // If there's only a constant discriminator, MOV it into x17.
+  // If there's only a constant discriminator, MOV it into the scratch register.
   if (AddrDisc == AArch64::XZR) {
-    emitMOVZ(AArch64::X17, Disc, 0);
-    return AArch64::X17;
+    emitMOVZ(ScratchReg, Disc, 0);
+    return ScratchReg;
   }
 
-  // If there are both, emit a blend into x17.
-  emitMovXReg(AArch64::X17, AddrDisc);
-  emitMOVK(AArch64::X17, Disc, 48);
-  return AArch64::X17;
+  // If there are both, emit a blend into the scratch register.
+
+  // Check if we can save one MOV instruction.
+  assert(MayUseAddrAsScratch || ScratchReg != AddrDisc);
+  bool AddrDiscIsSafe = AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
+  if (MayUseAddrAsScratch && AddrDiscIsSafe)
+    ScratchReg = AddrDisc;
+  else
+    emitMovXReg(ScratchReg, AddrDisc);
+
+  emitMOVK(ScratchReg, Disc, 48);
+  return ScratchReg;
 }
 
 /// Emits a code sequence to check an authenticated pointer value.
@@ -1963,7 +1993,8 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
 
   // Compute aut discriminator into x17
   assert(isUInt<16>(AUTDisc));
-  unsigned AUTDiscReg = emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc);
+  Register AUTDiscReg =
+      emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc, AArch64::X17);
   bool AUTZero = AUTDiscReg == AArch64::XZR;
   unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero);
 
@@ -2004,7 +2035,8 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
 
   // Compute pac discriminator into x17
   assert(isUInt<16>(PACDisc));
-  unsigned PACDiscReg = emitPtrauthDiscriminator(PACDisc, PACAddrDisc);
+  Register PACDiscReg =
+      emitPtrauthDiscriminator(PACDisc, PACAddrDisc, AArch64::X17);
   bool PACZero = PACDiscReg == AArch64::XZR;
   unsigned PACOpc = getPACOpcodeForKey(PACKey, PACZero);
 
@@ -2036,8 +2068,20 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
 
   unsigned AddrDisc = MI->getOperand(3).getReg();
 
-  // Compute discriminator into x17
-  unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc);
+  // Make sure AddrDisc is solely used to compute the discriminator.
+  // While hardly meaningful, it is still possible to describe an authentication
+  // of a pointer against its own value (instead of storage address) with
+  // intrinsics, so use report_fatal_error instead of assert.
+  if (BrTarget == AddrDisc)
+    report_fatal_error("Branch target is signed with its own value");
+
+  // If we are printing BLRA pseudo instruction, then x16 and x17 are
+  // implicit-def'ed by the MI and AddrDisc is not used as any other input, so
+  // try to save one MOV by setting MayUseAddrAsScratch.
+  // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
+  // declared as clobbering x16/x17.
+  Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17,
+                                              /*MayUseAddrAsScratch=*/IsCall);
   bool IsZeroDisc = DiscReg == AArch64::XZR;
 
   unsigned Opc;
@@ -2331,16 +2375,7 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
     }
   }
 
-  unsigned DiscReg = AddrDisc;
-  if (Disc != 0) {
-    if (AddrDisc != AArch64::XZR) {
-      emitMovXReg(AArch64::X17, AddrDisc);
-      emitMOVK(AArch64::X17, Disc, 48);
-    } else {
-      emitMOVZ(AArch64::X17, Disc, 0);
-    }
-    DiscReg = AArch64::X17;
-  }
+  Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17);
 
   auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
                  .addReg(AArch64::X16)
@@ -2356,28 +2391,39 @@ void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
   const MachineOperand &GAMO = MI.getOperand(1);
   assert(GAMO.getOffset() == 0);
 
-  MachineOperand GAHiOp(GAMO);
-  MachineOperand GALoOp(GAMO);
-  GAHiOp.addTargetFlag(AArch64II::MO_PAGE);
-  GALoOp.addTargetFlag(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+  if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
+    MCOperand GAMC;
+    MCInstLowering.lowerOperand(GAMO, GAMC);
+    EmitToStreamer(
+        MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addOperand(GAMC));
+    EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
+                       .addReg(AuthResultReg)
+                       .addReg(AArch64::X17)
+                       .addImm(0));
+  } else {
+    MachineOperand GAHiOp(GAMO);
+    MachineOperand GALoOp(GAMO);
+    GAHiOp.addTargetFlag(AArch64II::MO_PAGE);
+    GALoOp.addTargetFlag(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
 
-  MCOperand GAMCHi, GAMCLo;
-  MCInstLowering.lowerOperand(GAHiOp, GAMCHi);
-  MCInstLowering.lowerOperand(GALoOp, GAMCLo);
+    MCOperand GAMCHi, GAMCLo;
+    MCInstLowering.lowerOperand(GAHiOp, GAMCHi);
+    MCInstLowering.lowerOperand(GALoOp, GAMCLo);
 
-  EmitToStreamer(
-      MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(GAMCHi));
+    EmitToStreamer(
+        MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(GAMCHi));
 
-  EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
-                     .addReg(AArch64::X17)
-                     .addReg(AArch64::X17)
-                     .addOperand(GAMCLo)
-                     .addImm(0));
+    EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
+                       .addReg(AArch64::X17)
+                       .addReg(AArch64::X17)
+                       .addOperand(GAMCLo)
+                       .addImm(0));
 
-  EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
-                     .addReg(AuthResultReg)
-                     .addReg(AArch64::X17)
-                     .addImm(0));
+    EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
+                       .addReg(AuthResultReg)
+                       .addReg(AArch64::X17)
+                       .addImm(0));
+  }
 
   assert(GAMO.isGlobal());
   MCSymbol *UndefWeakSym;
@@ -2608,6 +2654,7 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
   // instruction here.
   case AArch64::AUTH_TCRETURN:
   case AArch64::AUTH_TCRETURN_BTI: {
+    Register Callee = MI->getOperand(0).getReg();
     const uint64_t Key = MI->getOperand(2).getImm();
     assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
            "Invalid auth key for tail-call return");
@@ -2617,31 +2664,23 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
 
     Register AddrDisc = MI->getOperand(4).getReg();
 
-    Register ScratchReg = MI->getOperand(0).getReg() == AArch64::X16
-                              ? AArch64::X17
-                              : AArch64::X16;
+    Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
 
     emitPtrauthTailCallHardening(MI);
 
-    unsigned DiscReg = AddrDisc;
-    if (Disc) {
-      if (AddrDisc != AArch64::NoRegister) {
-        if (ScratchReg != AddrDisc)
-          emitMovXReg(ScratchReg, AddrDisc);
-        emitMOVK(ScratchReg, Disc, 48);
-      } else {
-        emitMOVZ(ScratchReg, Disc, 0);
-      }
-      DiscReg = ScratchReg;
-    }
+    // See the comments in emitPtrauthBranch.
+    if (Callee == AddrDisc)
+      report_fatal_error("Call target is signed with its own value");
+    Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
+                                                /*MayUseAddrAsScratch=*/true);
 
-    const bool IsZero = DiscReg == AArch64::NoRegister;
+    const bool IsZero = DiscReg == AArch64::XZR;
     const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
                                     {AArch64::BRAB, AArch64::BRABZ}};
 
     MCInst TmpInst;
     TmpInst.setOpcode(Opcodes[Key][IsZero]);
-    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+    TmpInst.addOperand(MCOperand::createReg(Callee));
     if (!IsZero)
       TmpInst.addOperand(MCOperand::createReg(DiscReg));
     EmitToStreamer(*OutStreamer, TmpInst);
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index d6673969aa305..64dfb1e39485f 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -3167,11 +3167,24 @@ static void computeCalleeSaveRegisterPairs(
             (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
            "Offset out of bounds for LDP/STP immediate");
 
+    auto isFrameRecord = [&] {
+      if (RPI.isPaired())
+        return IsWindows ? RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR
+                         : RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP;
+      // Otherwise, look for the frame record as two unpaired registers. This is
+      // needed for -aarch64-stack-hazard-size=<val>, which disables register
+      // pairing (as the padding may be too large for the LDP/STP offset). Note:
+      // On Windows, this check works out as current reg == FP, next reg == LR,
+      // and on other platforms current reg == FP, previous reg == LR. This
+      // works out as the correct pre-increment or post-increment offsets
+      // respectively.
+      return i > 0 && RPI.Reg1 == AArch64::FP &&
+             CSI[i - 1].getReg() == AArch64::LR;
+    };
+
     // Save the offset to frame record so that the FP register can point to the
     // innermost frame record (spilled FP and LR registers).
-    if (NeedsFrameRecord &&
-        ((!IsWindows && RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
-         (IsWindows && RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR)))
+    if (NeedsFrameRecord && isFrameRecord())
       AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
 
     RegPairs.push_back(RPI);
@@ -4355,26 +4368,37 @@ void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
 
   int64_t ExtraBaseRegUpdate =
       FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
+  LLVM_DEBUG(dbgs() << "TagStoreEdit::emitLoop: LoopSize=" << LoopSize
+                    << ", Size=" << Size
+                    << ", ExtraBaseRegUpdate=" << ExtraBaseRegUpdate
+                    << ", FrameRegUpdate=" << FrameRegUpdate
+                    << ", FrameRegOffset.getFixed()="
+                    << FrameRegOffset.getFixed() << "\n");
   if (LoopSize < Size) {
     assert(FrameRegUpdate);
     assert(Size - LoopSize == 16);
     // Tag 16 more bytes at BaseReg and update BaseReg.
+    int64_t STGOffset = ExtraBaseRegUpdate + 16;
+    assert(STGOffset % 16 == 0 && STGOffset >= -4096 && STGOffset <= 4080 &&
+           "STG immediate out of range");
     BuildMI(*MBB, InsertI, DL,
             TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
         .addDef(BaseReg)
         .addReg(BaseReg)
         .addReg(BaseReg)
-        .addImm(1 + ExtraBaseRegUpdate / 16)
+        .addImm(STGOffset / 16)
         .setMemRefs(CombinedMemRefs)
         .setMIFlags(FrameRegUpdateFlags);
   } else if (ExtraBaseRegUpdate) {
     // Update BaseReg.
+    int64_t AddSubOffset = std::abs(ExtraBaseRegUpdate);
+    assert(AddSubOffset <= 4095 && "ADD/SUB immediate out of range");
     BuildMI(
         *MBB, InsertI, DL,
         TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
         .addDef(BaseReg)
         .addReg(BaseReg)
-        .addImm(std::abs(ExtraBaseRegUpdate))
+        .addImm(AddSubOffset)
         .addImm(0)
         .setMIFlags(FrameRegUpdateFlags);
   }
@@ -4393,10 +4417,20 @@ bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
     int64_t Offset = MI.getOperand(2).getImm() << Shift;
     if (MI.getOpcode() == AArch64::SUBXri)
       Offset = -Offset;
-    int64_t AbsPostOffset = std::abs(Offset - Size);
-    const int64_t kMaxOffset =
-        0xFFF; // Max encoding for unshifted ADDXri / SUBXri
-    if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
+    int64_t PostOffset = Offset - Size;
+    // TagStoreEdit::emitLoop might emit either an ADD/SUB after the loop, or
+    // an STGPostIndex which does the last 16 bytes of tag write. Which one is
+    // chosen depends on the alignment of the loop size, but the difference
+    // between the valid ranges for the two instructions is small, so we
+    // conservatively assume that it could be either case here.
+    //
+    // Max offset of STGPostIndex, minus the 16 byte tag write folded into that
+    // instruction.
+    const int64_t kMaxOffset = 4080 - 16;
+    // Max offset of SUBXri.
+    const int64_t kMinOffset = -4095;
+    if (PostOffset <= kMaxOffset && PostOffset >= kMinOffset &&
+        PostOffset % 16 == 0) {
       *TotalOffset = Offset;
       return true;
     }
@@ -4572,7 +4606,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
       break;
 
     // Reject anything that may alias the collected instructions.
-    if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
+    if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() || MI.isCall())
       break;
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 8512fd9ae4017..5df61b3722037 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -383,6 +383,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   void SelectPExtPair(SDNode *N, unsigned Opc);
   void SelectWhilePair(SDNode *N, unsigned Opc);
   void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
+  void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
   void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
   void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
                                  bool IsTupleInput, unsigned Opc);
@@ -1866,6 +1867,27 @@ void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
   CurDAG->RemoveDeadNode(N);
 }
 
+void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
+                                                unsigned Opcode) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
+  Ops.push_back(/*Chain*/ N->getOperand(0));
+
+  SDNode *Instruction =
+      CurDAG->getMachineNode(Opcode, DL, {MVT::Untyped, MVT::Other}, Ops);
+  SDValue SuperReg = SDValue(Instruction, 0);
+
+  for (unsigned i = 0; i < NumVecs; ++i)
+    ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
+                                   AArch64::zsub0 + i, DL, VT, SuperReg));
+
+  // Copy chain
+  unsigned ChainIdx = NumVecs;
+  ReplaceUses(SDValue(N, ChainIdx), SDValue(Instruction, 1));
+  CurDAG->RemoveDeadNode(N);
+}
+
 void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
                                                           unsigned NumVecs,
                                                           bool IsZmMulti,
@@ -5547,6 +5569,30 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
       SelectMultiVectorLuti(Node, 4, AArch64::LUTI4_4ZZT2Z);
       return;
     }
+    case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
+      if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
+              Node->getValueType(0),
+              {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
+        SelectCVTIntrinsicFP8(Node, 2, Opc);
+      return;
+    case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
+      if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
+              Node->getValueType(0),
+              {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
+        SelectCVTIntrinsicFP8(Node, 2, Opc);
+      return;
+    case Intrinsic::aarch64_sve_fp8_cvt1_x2:
+      if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
+              Node->getValueType(0),
+              {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
+        SelectCVTIntrinsicFP8(Node, 2, Opc);
+      return;
+    case Intrinsic::aarch64_sve_fp8_cvt2_x2:
+      if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
+              Node->getValueType(0),
+              {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
+        SelectCVTIntrinsicFP8(Node, 2, Opc);
+      return;
     }
   } break;
   case ISD::INTRINSIC_WO_CHAIN: {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e1be825fcf7bf..424f18ba4d822 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15841,11 +15841,27 @@ static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
       return getVectorBitwiseReduce(Opcode, HalfVec, VT, DL, DAG);
     }
 
-    // Vectors that are less than 64 bits get widened to neatly fit a 64 bit
-    // register, so e.g. <4 x i1> gets lowered to <4 x i16>. Sign extending to
+    // Results of setcc operations get widened to 128 bits for xor reduce if
+    // their input operands are 128 bits wide, otherwise vectors that are less
+    // than 64 bits get widened to neatly fit a 64 bit register, so e.g.
+    // <4 x i1> gets lowered to either <4 x i16> or <4 x i32>. Sign extending to
     // this element size leads to the best codegen, since e.g. setcc results
     // might need to be truncated otherwise.
-    EVT ExtendedVT = MVT::getIntegerVT(std::max(64u / NumElems, 8u));
+    unsigned ExtendedWidth = 64;
+    if (ScalarOpcode == ISD::XOR && Vec.getOpcode() == ISD::SETCC &&
+        Vec.getOperand(0).getValueSizeInBits() >= 128) {
+      ExtendedWidth = 128;
+    }
+    EVT ExtendedVT = MVT::getIntegerVT(std::max(ExtendedWidth / NumElems, 8u));
+
+    // Negate the reduced vector value for reduce and operations that use
+    // fcmp.
+    if (ScalarOpcode == ISD::AND && NumElems < 16) {
+      Vec = DAG.getNode(
+          ISD::XOR, DL, VecVT, Vec,
+          DAG.getSplatVector(
+              VecVT, DL, DAG.getConstant(APInt::getAllOnes(32), DL, MVT::i32)));
+    }
 
     // any_ext doesn't work with umin/umax, so only use it for uadd.
     unsigned ExtendOp =
@@ -15854,10 +15870,36 @@ static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
         ExtendOp, DL, VecVT.changeVectorElementType(ExtendedVT), Vec);
     switch (ScalarOpcode) {
     case ISD::AND:
-      Result = DAG.getNode(ISD::VECREDUCE_UMIN, DL, ExtendedVT, Extended);
+      if (NumElems < 16) {
+        // Check if all lanes of the negated bool vector value are zero by
+        // comparing against 0.0 with ordered and equal predicate. The only
+        // non-zero bit pattern that compares ordered and equal to 0.0 is -0.0,
+        // where only the sign bit is set. However the bool vector is
+        // sign-extended so that each bit in a lane is either zero or one,
+        // meaning that it is impossible to get the bit pattern of -0.0.
+        assert(Extended.getValueSizeInBits() == 64);
+        Extended = DAG.getBitcast(MVT::f64, Extended);
+        Result =
+            DAG.getSetCC(DL, MVT::i32, Extended,
+                         DAG.getConstantFP(0.0, DL, MVT::f64), ISD::SETOEQ);
+      } else {
+        Result = DAG.getNode(ISD::VECREDUCE_UMIN, DL, ExtendedVT, Extended);
+      }
       break;
     case ISD::OR:
-      Result = DAG.getNode(ISD::VECREDUCE_UMAX, DL, ExtendedVT, Extended);
+      if (NumElems < 16) {
+        // Check if any lane of the bool vector is set by comparing against 0.0.
+        // NaN bit patterns are handled by using the 'unordered or not equal'
+        // predicate. Similarly to the reduce and case, -0.0 doesn't have to be
+        // handled here (see explanation above).
+        assert(Extended.getValueSizeInBits() == 64);
+        Extended = DAG.getBitcast(MVT::f64, Extended);
+        Result =
+            DAG.getSetCC(DL, MVT::i32, Extended,
+                         DAG.getConstantFP(0.0, DL, MVT::f64), ISD::SETUNE);
+      } else {
+        Result = DAG.getNode(ISD::VECREDUCE_UMAX, DL, ExtendedVT, Extended);
+      }
       break;
     case ISD::XOR:
       Result = DAG.getNode(ISD::VECREDUCE_ADD, DL, ExtendedVT, Extended);
@@ -17931,6 +17973,13 @@ AArch64TargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
   SDValue ShiftLHS = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
+  if (!ShiftLHS->hasOneUse())
+    return false;
+
+  if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
+      !ShiftLHS.getOperand(0)->hasOneUse())
+    return false;
+
   // If ShiftLHS is unsigned bit extraction: ((x >> C) & mask), then do not
   // combine it with shift 'N' to let it be lowered to UBFX except:
   // ((x >> C) & mask) << C.
@@ -21817,7 +21866,7 @@ SDValue tryLowerPartialReductionToWideAdd(SDNode *N,
              Intrinsic::experimental_vector_partial_reduce_add &&
          "Expected a partial reduction node");
 
-  if (!Subtarget->isSVEorStreamingSVEAvailable())
+  if (!Subtarget->hasSVE2() && !Subtarget->isStreamingSVEAvailable())
     return SDValue();
 
   SDLoc DL(N);
@@ -23236,10 +23285,15 @@ static SDValue performPostLD1Combine(SDNode *N,
   if (!VT.is128BitVector() && !VT.is64BitVector())
     return SDValue();
 
-  unsigned LoadIdx = IsLaneOp ? 1 : 0;
-  SDNode *LD = N->getOperand(LoadIdx).getNode();
   // If it is not LOAD, can not do such combine.
-  if (LD->getOpcode() != ISD::LOAD)
+  unsigned LoadIdx = IsLaneOp ? 1 : 0;
+  LoadSDNode *LD = dyn_cast<LoadSDNode>(N->getOperand(LoadIdx).getNode());
+  if (!LD)
+    return SDValue();
+
+  // If the Generic combiner already helped form a pre- or post-indexed load,
+  // skip forming one here.
+  if (LD->isIndexed())
     return SDValue();
 
   // The vector lane must be a constant in the LD1LANE opcode.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index cb0b9e965277a..d51b36f7e4994 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1348,6 +1348,10 @@ class AArch64TargetLowering : public TargetLowering {
   unsigned getMinimumJumpTableEntries() const override;
 
   bool softPromoteHalfType() const override { return true; }
+
+  bool shouldScalarizeBinop(SDValue VecOp) const override {
+    return VecOp.getOpcode() == ISD::SETCC;
+  }
 };
 
 namespace AArch64 {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index a8ba89f784c8c..56ff7b0d3a280 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -145,8 +145,12 @@ def gi_extract_high_v4i32 :
 
 def extract_high_v8f16 :
     ComplexPattern<v4f16, 1, "SelectExtractHigh", [extract_subvector, bitconvert]>;
+def extract_high_v8bf16 :
+    ComplexPattern<v4bf16, 1, "SelectExtractHigh", [extract_subvector, bitconvert]>;
 def extract_high_v4f32 :
     ComplexPattern<v2f32, 1, "SelectExtractHigh", [extract_subvector, bitconvert]>;
+def extract_high_v2f64 :
+    ComplexPattern<v1f64, 1, "SelectExtractHigh", [extract_subvector, bitconvert]>;
 
 def gi_extract_high_v8f16 :
   GIComplexOperandMatcher<v4s16, "selectExtractHigh">,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index b4b3eccf82427..d015cc15581ad 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -381,6 +381,9 @@ def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
 
 def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
 
+def UseUnaryUndefPseudos
+  : Predicate<"!(Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))">;
+
 def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER",
                                   SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                                        SDTCisInt<1>]>>;
@@ -1841,28 +1844,28 @@ let Predicates = [HasPAuth] in {
   // materialization here), in part because they're handled in a safer way by
   // the kernel, notably on Darwin.
   def BLRA : Pseudo<(outs), (ins GPR64noip:$Rn, i32imm:$Key, i64imm:$Disc,
-                                 GPR64noip:$AddrDisc),
+                                 GPR64:$AddrDisc),
                     [(AArch64authcall GPR64noip:$Rn, timm:$Key, timm:$Disc,
-                                      GPR64noip:$AddrDisc)]>, Sched<[]> {
+                                      GPR64:$AddrDisc)]>, Sched<[]> {
     let isCodeGenOnly = 1;
     let hasSideEffects = 1;
     let mayStore = 0;
     let mayLoad = 0;
     let isCall = 1;
     let Size = 12; // 4 fixed + 8 variable, to compute discriminator.
-    let Defs = [X17,LR];
+    let Defs = [X16,X17,LR];
     let Uses = [SP];
   }
 
   def BLRA_RVMARKER : Pseudo<
         (outs), (ins i64imm:$rvfunc, GPR64noip:$Rn, i32imm:$Key, i64imm:$Disc,
-                     GPR64noip:$AddrDisc),
+                     GPR64:$AddrDisc),
         [(AArch64authcall_rvmarker tglobaladdr:$rvfunc,
                                    GPR64noip:$Rn, timm:$Key, timm:$Disc,
-                                   GPR64noip:$AddrDisc)]>, Sched<[]> {
+                                   GPR64:$AddrDisc)]>, Sched<[]> {
     let isCodeGenOnly = 1;
     let isCall = 1;
-    let Defs = [X17,LR];
+    let Defs = [X16,X17,LR];
     let Uses = [SP];
   }
 
@@ -1972,7 +1975,7 @@ let Predicates = [HasPAuth] in {
   // make sure at least one register is usable as a scratch one - for that
   // purpose, use tcGPRnotx16x17 register class for one of the operands.
   let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Size = 16,
-      Uses = [SP] in {
+      Defs = [X16,X17], Uses = [SP] in {
     def AUTH_TCRETURN
       : Pseudo<(outs), (ins tcGPRnotx16x17:$dst, i32imm:$FPDiff, i32imm:$Key,
                             i64imm:$Disc, tcGPR64:$AddrDisc),
@@ -7349,7 +7352,8 @@ def : Pat<(vector_extract (v8bf16 V128:$Rn), VectorIndexH:$idx),
 // All concat_vectors operations are canonicalised to act on i64 vectors for
 // AArch64. In the general case we need an instruction, which had just as well be
 // INS.
-multiclass ConcatPat<ValueType DstTy, ValueType SrcTy> {
+multiclass ConcatPat<ValueType DstTy, ValueType SrcTy,
+                     ComplexPattern ExtractHigh> {
   def : Pat<(DstTy (concat_vectors (SrcTy V64:$Rd), V64:$Rn)),
             (INSvi64lane (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), 1,
                          (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub), 0)>;
@@ -7362,16 +7366,22 @@ multiclass ConcatPat<ValueType DstTy, ValueType SrcTy> {
   // If the high lanes are undef we can just ignore them:
   def : Pat<(DstTy (concat_vectors (SrcTy V64:$Rn), undef)),
             (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub)>;
-}
 
-defm : ConcatPat<v2i64, v1i64>;
-defm : ConcatPat<v2f64, v1f64>;
-defm : ConcatPat<v4i32, v2i32>;
-defm : ConcatPat<v4f32, v2f32>;
-defm : ConcatPat<v8i16, v4i16>;
-defm : ConcatPat<v8f16, v4f16>;
-defm : ConcatPat<v8bf16, v4bf16>;
-defm : ConcatPat<v16i8, v8i8>;
+  // Concatting the high half of two vectors is the insert of the first
+  // into the low half of the second.
+  def : Pat<(DstTy (concat_vectors (ExtractHigh (DstTy V128:$Rn)),
+                                   (ExtractHigh (DstTy V128:$Rm)))),
+            (INSvi64lane V128:$Rm, (i64 0), V128:$Rn, (i64 1))>;
+}
+
+defm : ConcatPat<v2i64, v1i64, extract_high_v2i64>;
+defm : ConcatPat<v2f64, v1f64, extract_high_v2f64>;
+defm : ConcatPat<v4i32, v2i32, extract_high_v4i32>;
+defm : ConcatPat<v4f32, v2f32, extract_high_v4f32>;
+defm : ConcatPat<v8i16, v4i16, extract_high_v8i16>;
+defm : ConcatPat<v8f16, v4f16, extract_high_v8f16>;
+defm : ConcatPat<v8bf16, v4bf16, extract_high_v8bf16>;
+defm : ConcatPat<v16i8, v8i8, extract_high_v16i8>;
 
 //----------------------------------------------------------------------------
 // AdvSIMD across lanes instructions
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index ab00da51cf4fa..cd976790ebb6f 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -733,7 +733,7 @@ static bool isPromotableLoadFromStore(MachineInstr &MI) {
   }
 }
 
-static bool isMergeableLdStUpdate(MachineInstr &MI) {
+static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI) {
   unsigned Opc = MI.getOpcode();
   switch (Opc) {
   default:
@@ -785,6 +785,15 @@ static bool isMergeableLdStUpdate(MachineInstr &MI) {
     if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
       return false;
 
+    // When using stack tagging, simple sp+imm loads and stores are not
+    // tag-checked, but pre- and post-indexed versions of them are, so we can't
+    // replace the former with the latter. This transformation would be valid
+    // if the load/store accesses an untagged stack slot, but we don't have
+    // that information available after frame indices have been eliminated.
+    if (AFI.isMTETagged() &&
+        AArch64InstrInfo::getLdStBaseOp(MI).getReg() == AArch64::SP)
+      return false;
+
     return true;
   }
 }
@@ -1540,7 +1549,10 @@ static bool canRenameMOP(const MachineOperand &MOP,
     // Note that this relies on the structure of the AArch64 register file. In
     // particular, a subregister cannot be written without overwriting the
     // whole register.
-    if (RegClass->HasDisjunctSubRegs) {
+    if (RegClass->HasDisjunctSubRegs && RegClass->CoveredBySubRegs &&
+        (TRI->getSubRegisterClass(RegClass, AArch64::dsub0) ||
+         TRI->getSubRegisterClass(RegClass, AArch64::qsub0) ||
+         TRI->getSubRegisterClass(RegClass, AArch64::zsub0))) {
       LLVM_DEBUG(
           dbgs()
           << "  Cannot rename operands with multiple disjunct subregisters ("
@@ -2769,6 +2781,7 @@ bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
 
 bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
                                         bool EnableNarrowZeroStOpt) {
+  AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
 
   bool Modified = false;
   // Four tranformations to do here:
@@ -2839,7 +2852,7 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
   //        ldr x0, [x2], #4
   for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
        MBBI != E;) {
-    if (isMergeableLdStUpdate(*MBBI) && tryToMergeLdStUpdate(MBBI))
+    if (isMergeableLdStUpdate(*MBBI, AFI) && tryToMergeLdStUpdate(MBBI))
       Modified = true;
     else
       ++MBBI;
diff --git a/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp b/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
index 1aa55864dbff4..ed531038395c4 100644
--- a/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
@@ -169,9 +169,7 @@ static MachineFunction &createFrameHelperMachineFunction(Module *M,
   F->setLinkage(GlobalValue::LinkOnceODRLinkage);
   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
 
-  // Set no-opt/minsize, so we don't insert padding between outlined
-  // functions.
-  F->addFnAttr(Attribute::OptimizeNone);
+  // Set minsize, so we don't insert padding between outlined functions.
   F->addFnAttr(Attribute::NoInline);
   F->addFnAttr(Attribute::MinSize);
   F->addFnAttr(Attribute::Naked);
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 6886df5392565..2da67126a1753 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -266,6 +266,14 @@ def TuneA64FX : SubtargetFeature<"a64fx", "ARMProcFamily", "A64FX",
                                  FeatureStorePairSuppress,
                                  FeaturePredictableSelectIsExpensive]>;
 
+def TuneMONAKA : SubtargetFeature<"fujitsu-monaka", "ARMProcFamily", "MONAKA",
+                                 "Fujitsu FUJITSU-MONAKA processors", [
+                                 FeaturePredictableSelectIsExpensive,
+                                 FeatureEnableSelectOptimize,
+                                 FeaturePostRAScheduler,
+                                 FeatureArithmeticBccFusion,
+                                 ]>;
+
 def TuneCarmel : SubtargetFeature<"carmel", "ARMProcFamily", "Carmel",
                                   "Nvidia Carmel processors">;
 
@@ -843,6 +851,12 @@ def ProcessorFeatures {
                                      FeatureSHA2, FeaturePerfMon, FeatureFullFP16,
                                      FeatureSVE, FeatureComplxNum,
                                      FeatureAES, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM];
+  list<SubtargetFeature> MONAKA = [HasV9_3aOps, FeaturePerfMon, FeatureCCIDX,
+                                   FeatureFPAC, FeatureFP16FML, FeatureRandGen,
+                                   FeatureSSBS, FeatureLS64, FeatureCLRBHB,
+                                   FeatureSPECRES2, FeatureSVEAES, FeatureSVE2SM4,
+                                   FeatureSVE2SHA3, FeatureSVE2BitPerm, FeatureETE,
+                                   FeatureMEC, FeatureFP8DOT2];
   list<SubtargetFeature> Carmel   = [HasV8_2aOps, FeatureNEON, FeatureSHA2, FeatureAES,
                                      FeatureFullFP16, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM,
                                      FeatureFPARMv8];
@@ -1113,15 +1127,15 @@ def : ProcessorModel<"cortex-r82", CortexA55Model, ProcessorFeatures.R82,
                      [TuneR82]>;
 def : ProcessorModel<"cortex-r82ae", CortexA55Model, ProcessorFeatures.R82AE,
                      [TuneR82AE]>;
-def : ProcessorModel<"cortex-x1", CortexA57Model, ProcessorFeatures.X1,
+def : ProcessorModel<"cortex-x1", NeoverseV1Model, ProcessorFeatures.X1,
                      [TuneX1]>;
-def : ProcessorModel<"cortex-x1c", CortexA57Model, ProcessorFeatures.X1C,
+def : ProcessorModel<"cortex-x1c", NeoverseV1Model, ProcessorFeatures.X1C,
                      [TuneX1]>;
-def : ProcessorModel<"cortex-x2", NeoverseN2Model, ProcessorFeatures.X2,
+def : ProcessorModel<"cortex-x2", NeoverseV2Model, ProcessorFeatures.X2,
                      [TuneX2]>;
-def : ProcessorModel<"cortex-x3", NeoverseN2Model, ProcessorFeatures.X3,
+def : ProcessorModel<"cortex-x3", NeoverseV2Model, ProcessorFeatures.X3,
                      [TuneX3]>;
-def : ProcessorModel<"cortex-x4", NeoverseN2Model, ProcessorFeatures.X4,
+def : ProcessorModel<"cortex-x4", NeoverseV2Model, ProcessorFeatures.X4,
                      [TuneX4]>;
 def : ProcessorModel<"cortex-x925", NeoverseV2Model, ProcessorFeatures.X925,
                      [TuneX925]>;
@@ -1225,6 +1239,10 @@ def : ProcessorAlias<"apple-latest", "apple-m4">;
 def : ProcessorModel<"a64fx", A64FXModel, ProcessorFeatures.A64FX,
                      [TuneA64FX]>;
 
+// Fujitsu FUJITSU-MONAKA
+def : ProcessorModel<"fujitsu-monaka", A64FXModel, ProcessorFeatures.MONAKA,
+                     [TuneMONAKA]>;
+
 // Nvidia Carmel
 def : ProcessorModel<"carmel", NoSchedModel, ProcessorFeatures.Carmel,
                      [TuneCarmel]>;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 380f37df0bc2b..133cc1344b98f 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -490,13 +490,36 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
   }
 
   assert(checkAllSuperRegsMarked(Reserved));
+
+  // Add _HI registers after checkAllSuperRegsMarked as this check otherwise
+  // becomes considerably more expensive.
+  Reserved.set(AArch64::WSP_HI);
+  Reserved.set(AArch64::WZR_HI);
+  static_assert(AArch64::W30_HI - AArch64::W0_HI == 30,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::W0_HI, AArch64::W30_HI);
+  static_assert(AArch64::B31_HI - AArch64::B0_HI == 31,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::B0_HI, AArch64::B31_HI);
+  static_assert(AArch64::H31_HI - AArch64::H0_HI == 31,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::H0_HI, AArch64::H31_HI);
+  static_assert(AArch64::S31_HI - AArch64::S0_HI == 31,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::S0_HI, AArch64::S31_HI);
+  static_assert(AArch64::D31_HI - AArch64::D0_HI == 31,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::D0_HI, AArch64::D31_HI);
+  static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31,
+                "Unexpected order of registers");
+  Reserved.set(AArch64::Q0_HI, AArch64::Q31_HI);
+
   return Reserved;
 }
 
 BitVector
 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
-  BitVector Reserved = getStrictlyReservedRegs(MF);
-
+  BitVector Reserved(getNumRegs());
   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))
       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
@@ -514,6 +537,11 @@ AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   }
 
   assert(checkAllSuperRegsMarked(Reserved));
+
+  // Handle strictlyReservedRegs separately to avoid re-evaluating the assert,
+  // which becomes considerably expensive when considering the _HI registers.
+  Reserved |= getStrictlyReservedRegs(MF);
+
   return Reserved;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index ed16a007b49cf..4fec120391f01 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -20,33 +20,49 @@ class AArch64Reg<bits<16> enc, string n, list<Register> subregs = [],
 
 let Namespace = "AArch64" in {
   // SubRegIndexes for GPR registers
-  def sub_32 : SubRegIndex<32>;
-  def sube64 : SubRegIndex<64>;
-  def subo64 : SubRegIndex<64>;
-  def sube32 : SubRegIndex<32>;
-  def subo32 : SubRegIndex<32>;
+  def sub_32   : SubRegIndex<32>;
+  def sube64   : SubRegIndex<64>;
+  def subo64   : SubRegIndex<64>;
+  def sube32   : SubRegIndex<32>;
+  def subo32   : SubRegIndex<32>;
 
   // SubRegIndexes for FPR/Vector registers
-  def bsub : SubRegIndex<8>;
-  def hsub : SubRegIndex<16>;
-  def ssub : SubRegIndex<32>;
-  def dsub : SubRegIndex<64>;
-  def zsub : SubRegIndex<128>;
+  def bsub    : SubRegIndex<8, 0>;
+  def hsub    : SubRegIndex<16, 0>;
+  def ssub    : SubRegIndex<32, 0>;
+  def dsub    : SubRegIndex<64, 0>;
+  def zsub    : SubRegIndex<128, 0>;
+
+  // The _hi SubRegIndexes describe the high bits of a register which are not
+  // separately addressable. They need to be described so that partially
+  // overlapping registers end up with a different lane mask. This is required
+  // to enable subreg liveness tracking.
+  //
+  // For example: 8-bit B0 is a sub-register of 16-bit H0.
+  // * B0 is described with 'bsub'.
+  // * H0 is described with 'bsub + bsub_hi' == 'hsub'.
+  def bsub_hi   : SubRegIndex<8, 8>;
+  def hsub_hi   : SubRegIndex<16, 16>;
+  def ssub_hi   : SubRegIndex<32, 32>;
+  def dsub_hi   : SubRegIndex<64, 64>;
+  def zsub_hi   : SubRegIndex<-1, 128>;
+  // sub_32_hi describes the top 32 bits in e.g. X0
+  def sub_32_hi : SubRegIndex<32, 32>;
   // Note: Code depends on these having consecutive numbers
-  def zsub0 : SubRegIndex<128, -1>;
-  def zsub1 : SubRegIndex<128, -1>;
-  def zsub2 : SubRegIndex<128, -1>;
-  def zsub3 : SubRegIndex<128, -1>;
-  // Note: Code depends on these having consecutive numbers
-  def dsub0 : SubRegIndex<64>;
-  def dsub1 : SubRegIndex<64>;
-  def dsub2 : SubRegIndex<64>;
-  def dsub3 : SubRegIndex<64>;
+  def zsub0 : SubRegIndex<-1>;
+  def zsub1 : SubRegIndex<-1>;
+  def zsub2 : SubRegIndex<-1>;
+  def zsub3 : SubRegIndex<-1>;
   // Note: Code depends on these having consecutive numbers
   def qsub0 : SubRegIndex<128>;
-  def qsub1 : SubRegIndex<128>;
-  def qsub2 : SubRegIndex<128>;
-  def qsub3 : SubRegIndex<128>;
+  def qsub1 : ComposedSubRegIndex<zsub1, zsub>;
+  def qsub2 : ComposedSubRegIndex<zsub2, zsub>;
+  def qsub3 : ComposedSubRegIndex<zsub3, zsub>;
+  // Note: Code depends on these having consecutive numbers
+  def dsub0 : SubRegIndex<64>;
+  def dsub1 : ComposedSubRegIndex<qsub1, dsub>;
+  def dsub2 : ComposedSubRegIndex<qsub2, dsub>;
+  def dsub3 : ComposedSubRegIndex<qsub3, dsub>;
 
   // SubRegIndexes for SME Matrix tiles
   def zasubb  : SubRegIndex<2048>; // (16 x 16)/1 bytes  = 2048 bits
@@ -60,10 +76,10 @@ let Namespace = "AArch64" in {
   def zasubq1 : SubRegIndex<128>;  // (16 x 16)/16 bytes = 128 bits
 
   // SubRegIndexes for SVE Predicates
-  def psub  : SubRegIndex<16>;
+  def psub  : SubRegIndex<-1>;
   // Note: Code depends on these having consecutive numbers
-  def psub0 : SubRegIndex<16, -1>;
-  def psub1 : SubRegIndex<16, -1>;
+  def psub0 : SubRegIndex<-1>;
+  def psub1 : SubRegIndex<-1>;
 }
 
 let Namespace = "AArch64" in {
@@ -74,6 +90,14 @@ let Namespace = "AArch64" in {
 //===----------------------------------------------------------------------===//
 // Registers
 //===----------------------------------------------------------------------===//
+
+foreach i = 0-30 in {
+  // Define W0_HI, W1_HI, .. W30_HI
+  def W#i#_HI : AArch64Reg<-1,  "w"#i#"_hi"> { let isArtificial = 1; }
+}
+def WSP_HI : AArch64Reg<-1,  "wsp_hi"> { let isArtificial = 1; }
+def WZR_HI : AArch64Reg<-1,  "wzr_hi"> { let isArtificial = 1; }
+
 def W0    : AArch64Reg<0,   "w0" >, DwarfRegNum<[0]>;
 def W1    : AArch64Reg<1,   "w1" >, DwarfRegNum<[1]>;
 def W2    : AArch64Reg<2,   "w2" >, DwarfRegNum<[2]>;
@@ -106,44 +130,42 @@ def W28   : AArch64Reg<28, "w28">, DwarfRegNum<[28]>;
 def W29   : AArch64Reg<29, "w29">, DwarfRegNum<[29]>;
 def W30   : AArch64Reg<30, "w30">, DwarfRegNum<[30]>;
 def WSP   : AArch64Reg<31, "wsp">, DwarfRegNum<[31]>;
-let isConstant = true in
-def WZR   : AArch64Reg<31, "wzr">, DwarfRegAlias<WSP>;
-
-let SubRegIndices = [sub_32] in {
-def X0    : AArch64Reg<0,   "x0",  [W0]>, DwarfRegAlias<W0>;
-def X1    : AArch64Reg<1,   "x1",  [W1]>, DwarfRegAlias<W1>;
-def X2    : AArch64Reg<2,   "x2",  [W2]>, DwarfRegAlias<W2>;
-def X3    : AArch64Reg<3,   "x3",  [W3]>, DwarfRegAlias<W3>;
-def X4    : AArch64Reg<4,   "x4",  [W4]>, DwarfRegAlias<W4>;
-def X5    : AArch64Reg<5,   "x5",  [W5]>, DwarfRegAlias<W5>;
-def X6    : AArch64Reg<6,   "x6",  [W6]>, DwarfRegAlias<W6>;
-def X7    : AArch64Reg<7,   "x7",  [W7]>, DwarfRegAlias<W7>;
-def X8    : AArch64Reg<8,   "x8",  [W8]>, DwarfRegAlias<W8>;
-def X9    : AArch64Reg<9,   "x9",  [W9]>, DwarfRegAlias<W9>;
-def X10   : AArch64Reg<10, "x10", [W10]>, DwarfRegAlias<W10>;
-def X11   : AArch64Reg<11, "x11", [W11]>, DwarfRegAlias<W11>;
-def X12   : AArch64Reg<12, "x12", [W12]>, DwarfRegAlias<W12>;
-def X13   : AArch64Reg<13, "x13", [W13]>, DwarfRegAlias<W13>;
-def X14   : AArch64Reg<14, "x14", [W14]>, DwarfRegAlias<W14>;
-def X15   : AArch64Reg<15, "x15", [W15]>, DwarfRegAlias<W15>;
-def X16   : AArch64Reg<16, "x16", [W16]>, DwarfRegAlias<W16>;
-def X17   : AArch64Reg<17, "x17", [W17]>, DwarfRegAlias<W17>;
-def X18   : AArch64Reg<18, "x18", [W18]>, DwarfRegAlias<W18>;
-def X19   : AArch64Reg<19, "x19", [W19]>, DwarfRegAlias<W19>;
-def X20   : AArch64Reg<20, "x20", [W20]>, DwarfRegAlias<W20>;
-def X21   : AArch64Reg<21, "x21", [W21]>, DwarfRegAlias<W21>;
-def X22   : AArch64Reg<22, "x22", [W22]>, DwarfRegAlias<W22>;
-def X23   : AArch64Reg<23, "x23", [W23]>, DwarfRegAlias<W23>;
-def X24   : AArch64Reg<24, "x24", [W24]>, DwarfRegAlias<W24>;
-def X25   : AArch64Reg<25, "x25", [W25]>, DwarfRegAlias<W25>;
-def X26   : AArch64Reg<26, "x26", [W26]>, DwarfRegAlias<W26>;
-def X27   : AArch64Reg<27, "x27", [W27]>, DwarfRegAlias<W27>;
-def X28   : AArch64Reg<28, "x28", [W28]>, DwarfRegAlias<W28>;
-def FP    : AArch64Reg<29, "x29", [W29]>, DwarfRegAlias<W29>;
-def LR    : AArch64Reg<30, "x30", [W30]>, DwarfRegAlias<W30>;
-def SP    : AArch64Reg<31, "sp",  [WSP]>, DwarfRegAlias<WSP>;
-let isConstant = true in
-def XZR   : AArch64Reg<31, "xzr", [WZR]>, DwarfRegAlias<WSP>;
+def WZR   : AArch64Reg<31, "wzr">, DwarfRegAlias<WSP> { let isConstant = true; }
+
+let SubRegIndices = [sub_32, sub_32_hi], CoveredBySubRegs = 1 in {
+def X0    : AArch64Reg<0,   "x0",  [W0,  W0_HI]>, DwarfRegAlias<W0>;
+def X1    : AArch64Reg<1,   "x1",  [W1,  W1_HI]>, DwarfRegAlias<W1>;
+def X2    : AArch64Reg<2,   "x2",  [W2,  W2_HI]>, DwarfRegAlias<W2>;
+def X3    : AArch64Reg<3,   "x3",  [W3,  W3_HI]>, DwarfRegAlias<W3>;
+def X4    : AArch64Reg<4,   "x4",  [W4,  W4_HI]>, DwarfRegAlias<W4>;
+def X5    : AArch64Reg<5,   "x5",  [W5,  W5_HI]>, DwarfRegAlias<W5>;
+def X6    : AArch64Reg<6,   "x6",  [W6,  W6_HI]>, DwarfRegAlias<W6>;
+def X7    : AArch64Reg<7,   "x7",  [W7,  W7_HI]>, DwarfRegAlias<W7>;
+def X8    : AArch64Reg<8,   "x8",  [W8,  W8_HI]>, DwarfRegAlias<W8>;
+def X9    : AArch64Reg<9,   "x9",  [W9,  W9_HI]>, DwarfRegAlias<W9>;
+def X10   : AArch64Reg<10, "x10", [W10, W10_HI]>, DwarfRegAlias<W10>;
+def X11   : AArch64Reg<11, "x11", [W11, W11_HI]>, DwarfRegAlias<W11>;
+def X12   : AArch64Reg<12, "x12", [W12, W12_HI]>, DwarfRegAlias<W12>;
+def X13   : AArch64Reg<13, "x13", [W13, W13_HI]>, DwarfRegAlias<W13>;
+def X14   : AArch64Reg<14, "x14", [W14, W14_HI]>, DwarfRegAlias<W14>;
+def X15   : AArch64Reg<15, "x15", [W15, W15_HI]>, DwarfRegAlias<W15>;
+def X16   : AArch64Reg<16, "x16", [W16, W16_HI]>, DwarfRegAlias<W16>;
+def X17   : AArch64Reg<17, "x17", [W17, W17_HI]>, DwarfRegAlias<W17>;
+def X18   : AArch64Reg<18, "x18", [W18, W18_HI]>, DwarfRegAlias<W18>;
+def X19   : AArch64Reg<19, "x19", [W19, W19_HI]>, DwarfRegAlias<W19>;
+def X20   : AArch64Reg<20, "x20", [W20, W20_HI]>, DwarfRegAlias<W20>;
+def X21   : AArch64Reg<21, "x21", [W21, W21_HI]>, DwarfRegAlias<W21>;
+def X22   : AArch64Reg<22, "x22", [W22, W22_HI]>, DwarfRegAlias<W22>;
+def X23   : AArch64Reg<23, "x23", [W23, W23_HI]>, DwarfRegAlias<W23>;
+def X24   : AArch64Reg<24, "x24", [W24, W24_HI]>, DwarfRegAlias<W24>;
+def X25   : AArch64Reg<25, "x25", [W25, W25_HI]>, DwarfRegAlias<W25>;
+def X26   : AArch64Reg<26, "x26", [W26, W26_HI]>, DwarfRegAlias<W26>;
+def X27   : AArch64Reg<27, "x27", [W27, W27_HI]>, DwarfRegAlias<W27>;
+def X28   : AArch64Reg<28, "x28", [W28, W28_HI]>, DwarfRegAlias<W28>;
+def FP    : AArch64Reg<29, "x29", [W29, W29_HI]>, DwarfRegAlias<W29>;
+def LR    : AArch64Reg<30, "x30", [W30, W30_HI]>, DwarfRegAlias<W30>;
+def SP    : AArch64Reg<31, "sp",  [WSP, WSP_HI]>, DwarfRegAlias<WSP>;
+def XZR   : AArch64Reg<31, "xzr", [WZR, WZR_HI]>, DwarfRegAlias<WSP> { let isConstant = true; }
 }
 
 // Condition code register.
@@ -294,6 +316,14 @@ def CCR : RegisterClass<"AArch64", [i32], 32, (add NZCV)> {
 // Floating Point Scalar Registers
 //===----------------------------------------------------------------------===//
 
+foreach i = 0-31 in {
+  def B#i#_HI : AArch64Reg<-1,  "b"#i#"_hi"> { let isArtificial = 1; }
+  def H#i#_HI : AArch64Reg<-1,  "h"#i#"_hi"> { let isArtificial = 1; }
+  def S#i#_HI : AArch64Reg<-1,  "s"#i#"_hi"> { let isArtificial = 1; }
+  def D#i#_HI : AArch64Reg<-1,  "d"#i#"_hi"> { let isArtificial = 1; }
+  def Q#i#_HI : AArch64Reg<-1,  "q"#i#"_hi"> { let isArtificial = 1; }
+}
+
 def B0    : AArch64Reg<0,   "b0">, DwarfRegNum<[64]>;
 def B1    : AArch64Reg<1,   "b1">, DwarfRegNum<[65]>;
 def B2    : AArch64Reg<2,   "b2">, DwarfRegNum<[66]>;
@@ -327,144 +357,144 @@ def B29   : AArch64Reg<29, "b29">, DwarfRegNum<[93]>;
 def B30   : AArch64Reg<30, "b30">, DwarfRegNum<[94]>;
 def B31   : AArch64Reg<31, "b31">, DwarfRegNum<[95]>;
 
-let SubRegIndices = [bsub] in {
-def H0    : AArch64Reg<0,   "h0", [B0]>, DwarfRegAlias<B0>;
-def H1    : AArch64Reg<1,   "h1", [B1]>, DwarfRegAlias<B1>;
-def H2    : AArch64Reg<2,   "h2", [B2]>, DwarfRegAlias<B2>;
-def H3    : AArch64Reg<3,   "h3", [B3]>, DwarfRegAlias<B3>;
-def H4    : AArch64Reg<4,   "h4", [B4]>, DwarfRegAlias<B4>;
-def H5    : AArch64Reg<5,   "h5", [B5]>, DwarfRegAlias<B5>;
-def H6    : AArch64Reg<6,   "h6", [B6]>, DwarfRegAlias<B6>;
-def H7    : AArch64Reg<7,   "h7", [B7]>, DwarfRegAlias<B7>;
-def H8    : AArch64Reg<8,   "h8", [B8]>, DwarfRegAlias<B8>;
-def H9    : AArch64Reg<9,   "h9", [B9]>, DwarfRegAlias<B9>;
-def H10   : AArch64Reg<10, "h10", [B10]>, DwarfRegAlias<B10>;
-def H11   : AArch64Reg<11, "h11", [B11]>, DwarfRegAlias<B11>;
-def H12   : AArch64Reg<12, "h12", [B12]>, DwarfRegAlias<B12>;
-def H13   : AArch64Reg<13, "h13", [B13]>, DwarfRegAlias<B13>;
-def H14   : AArch64Reg<14, "h14", [B14]>, DwarfRegAlias<B14>;
-def H15   : AArch64Reg<15, "h15", [B15]>, DwarfRegAlias<B15>;
-def H16   : AArch64Reg<16, "h16", [B16]>, DwarfRegAlias<B16>;
-def H17   : AArch64Reg<17, "h17", [B17]>, DwarfRegAlias<B17>;
-def H18   : AArch64Reg<18, "h18", [B18]>, DwarfRegAlias<B18>;
-def H19   : AArch64Reg<19, "h19", [B19]>, DwarfRegAlias<B19>;
-def H20   : AArch64Reg<20, "h20", [B20]>, DwarfRegAlias<B20>;
-def H21   : AArch64Reg<21, "h21", [B21]>, DwarfRegAlias<B21>;
-def H22   : AArch64Reg<22, "h22", [B22]>, DwarfRegAlias<B22>;
-def H23   : AArch64Reg<23, "h23", [B23]>, DwarfRegAlias<B23>;
-def H24   : AArch64Reg<24, "h24", [B24]>, DwarfRegAlias<B24>;
-def H25   : AArch64Reg<25, "h25", [B25]>, DwarfRegAlias<B25>;
-def H26   : AArch64Reg<26, "h26", [B26]>, DwarfRegAlias<B26>;
-def H27   : AArch64Reg<27, "h27", [B27]>, DwarfRegAlias<B27>;
-def H28   : AArch64Reg<28, "h28", [B28]>, DwarfRegAlias<B28>;
-def H29   : AArch64Reg<29, "h29", [B29]>, DwarfRegAlias<B29>;
-def H30   : AArch64Reg<30, "h30", [B30]>, DwarfRegAlias<B30>;
-def H31   : AArch64Reg<31, "h31", [B31]>, DwarfRegAlias<B31>;
-}
-
-let SubRegIndices = [hsub] in {
-def S0    : AArch64Reg<0,   "s0", [H0]>, DwarfRegAlias<B0>;
-def S1    : AArch64Reg<1,   "s1", [H1]>, DwarfRegAlias<B1>;
-def S2    : AArch64Reg<2,   "s2", [H2]>, DwarfRegAlias<B2>;
-def S3    : AArch64Reg<3,   "s3", [H3]>, DwarfRegAlias<B3>;
-def S4    : AArch64Reg<4,   "s4", [H4]>, DwarfRegAlias<B4>;
-def S5    : AArch64Reg<5,   "s5", [H5]>, DwarfRegAlias<B5>;
-def S6    : AArch64Reg<6,   "s6", [H6]>, DwarfRegAlias<B6>;
-def S7    : AArch64Reg<7,   "s7", [H7]>, DwarfRegAlias<B7>;
-def S8    : AArch64Reg<8,   "s8", [H8]>, DwarfRegAlias<B8>;
-def S9    : AArch64Reg<9,   "s9", [H9]>, DwarfRegAlias<B9>;
-def S10   : AArch64Reg<10, "s10", [H10]>, DwarfRegAlias<B10>;
-def S11   : AArch64Reg<11, "s11", [H11]>, DwarfRegAlias<B11>;
-def S12   : AArch64Reg<12, "s12", [H12]>, DwarfRegAlias<B12>;
-def S13   : AArch64Reg<13, "s13", [H13]>, DwarfRegAlias<B13>;
-def S14   : AArch64Reg<14, "s14", [H14]>, DwarfRegAlias<B14>;
-def S15   : AArch64Reg<15, "s15", [H15]>, DwarfRegAlias<B15>;
-def S16   : AArch64Reg<16, "s16", [H16]>, DwarfRegAlias<B16>;
-def S17   : AArch64Reg<17, "s17", [H17]>, DwarfRegAlias<B17>;
-def S18   : AArch64Reg<18, "s18", [H18]>, DwarfRegAlias<B18>;
-def S19   : AArch64Reg<19, "s19", [H19]>, DwarfRegAlias<B19>;
-def S20   : AArch64Reg<20, "s20", [H20]>, DwarfRegAlias<B20>;
-def S21   : AArch64Reg<21, "s21", [H21]>, DwarfRegAlias<B21>;
-def S22   : AArch64Reg<22, "s22", [H22]>, DwarfRegAlias<B22>;
-def S23   : AArch64Reg<23, "s23", [H23]>, DwarfRegAlias<B23>;
-def S24   : AArch64Reg<24, "s24", [H24]>, DwarfRegAlias<B24>;
-def S25   : AArch64Reg<25, "s25", [H25]>, DwarfRegAlias<B25>;
-def S26   : AArch64Reg<26, "s26", [H26]>, DwarfRegAlias<B26>;
-def S27   : AArch64Reg<27, "s27", [H27]>, DwarfRegAlias<B27>;
-def S28   : AArch64Reg<28, "s28", [H28]>, DwarfRegAlias<B28>;
-def S29   : AArch64Reg<29, "s29", [H29]>, DwarfRegAlias<B29>;
-def S30   : AArch64Reg<30, "s30", [H30]>, DwarfRegAlias<B30>;
-def S31   : AArch64Reg<31, "s31", [H31]>, DwarfRegAlias<B31>;
-}
-
-let SubRegIndices = [ssub], RegAltNameIndices = [vreg, vlist1] in {
-def D0    : AArch64Reg<0,   "d0", [S0], ["v0", ""]>, DwarfRegAlias<B0>;
-def D1    : AArch64Reg<1,   "d1", [S1], ["v1", ""]>, DwarfRegAlias<B1>;
-def D2    : AArch64Reg<2,   "d2", [S2], ["v2", ""]>, DwarfRegAlias<B2>;
-def D3    : AArch64Reg<3,   "d3", [S3], ["v3", ""]>, DwarfRegAlias<B3>;
-def D4    : AArch64Reg<4,   "d4", [S4], ["v4", ""]>, DwarfRegAlias<B4>;
-def D5    : AArch64Reg<5,   "d5", [S5], ["v5", ""]>, DwarfRegAlias<B5>;
-def D6    : AArch64Reg<6,   "d6", [S6], ["v6", ""]>, DwarfRegAlias<B6>;
-def D7    : AArch64Reg<7,   "d7", [S7], ["v7", ""]>, DwarfRegAlias<B7>;
-def D8    : AArch64Reg<8,   "d8", [S8], ["v8", ""]>, DwarfRegAlias<B8>;
-def D9    : AArch64Reg<9,   "d9", [S9], ["v9", ""]>, DwarfRegAlias<B9>;
-def D10   : AArch64Reg<10, "d10", [S10], ["v10", ""]>, DwarfRegAlias<B10>;
-def D11   : AArch64Reg<11, "d11", [S11], ["v11", ""]>, DwarfRegAlias<B11>;
-def D12   : AArch64Reg<12, "d12", [S12], ["v12", ""]>, DwarfRegAlias<B12>;
-def D13   : AArch64Reg<13, "d13", [S13], ["v13", ""]>, DwarfRegAlias<B13>;
-def D14   : AArch64Reg<14, "d14", [S14], ["v14", ""]>, DwarfRegAlias<B14>;
-def D15   : AArch64Reg<15, "d15", [S15], ["v15", ""]>, DwarfRegAlias<B15>;
-def D16   : AArch64Reg<16, "d16", [S16], ["v16", ""]>, DwarfRegAlias<B16>;
-def D17   : AArch64Reg<17, "d17", [S17], ["v17", ""]>, DwarfRegAlias<B17>;
-def D18   : AArch64Reg<18, "d18", [S18], ["v18", ""]>, DwarfRegAlias<B18>;
-def D19   : AArch64Reg<19, "d19", [S19], ["v19", ""]>, DwarfRegAlias<B19>;
-def D20   : AArch64Reg<20, "d20", [S20], ["v20", ""]>, DwarfRegAlias<B20>;
-def D21   : AArch64Reg<21, "d21", [S21], ["v21", ""]>, DwarfRegAlias<B21>;
-def D22   : AArch64Reg<22, "d22", [S22], ["v22", ""]>, DwarfRegAlias<B22>;
-def D23   : AArch64Reg<23, "d23", [S23], ["v23", ""]>, DwarfRegAlias<B23>;
-def D24   : AArch64Reg<24, "d24", [S24], ["v24", ""]>, DwarfRegAlias<B24>;
-def D25   : AArch64Reg<25, "d25", [S25], ["v25", ""]>, DwarfRegAlias<B25>;
-def D26   : AArch64Reg<26, "d26", [S26], ["v26", ""]>, DwarfRegAlias<B26>;
-def D27   : AArch64Reg<27, "d27", [S27], ["v27", ""]>, DwarfRegAlias<B27>;
-def D28   : AArch64Reg<28, "d28", [S28], ["v28", ""]>, DwarfRegAlias<B28>;
-def D29   : AArch64Reg<29, "d29", [S29], ["v29", ""]>, DwarfRegAlias<B29>;
-def D30   : AArch64Reg<30, "d30", [S30], ["v30", ""]>, DwarfRegAlias<B30>;
-def D31   : AArch64Reg<31, "d31", [S31], ["v31", ""]>, DwarfRegAlias<B31>;
-}
-
-let SubRegIndices = [dsub], RegAltNameIndices = [vreg, vlist1] in {
-def Q0    : AArch64Reg<0,   "q0", [D0], ["v0", ""]>, DwarfRegAlias<B0>;
-def Q1    : AArch64Reg<1,   "q1", [D1], ["v1", ""]>, DwarfRegAlias<B1>;
-def Q2    : AArch64Reg<2,   "q2", [D2], ["v2", ""]>, DwarfRegAlias<B2>;
-def Q3    : AArch64Reg<3,   "q3", [D3], ["v3", ""]>, DwarfRegAlias<B3>;
-def Q4    : AArch64Reg<4,   "q4", [D4], ["v4", ""]>, DwarfRegAlias<B4>;
-def Q5    : AArch64Reg<5,   "q5", [D5], ["v5", ""]>, DwarfRegAlias<B5>;
-def Q6    : AArch64Reg<6,   "q6", [D6], ["v6", ""]>, DwarfRegAlias<B6>;
-def Q7    : AArch64Reg<7,   "q7", [D7], ["v7", ""]>, DwarfRegAlias<B7>;
-def Q8    : AArch64Reg<8,   "q8", [D8], ["v8", ""]>, DwarfRegAlias<B8>;
-def Q9    : AArch64Reg<9,   "q9", [D9], ["v9", ""]>, DwarfRegAlias<B9>;
-def Q10   : AArch64Reg<10, "q10", [D10], ["v10", ""]>, DwarfRegAlias<B10>;
-def Q11   : AArch64Reg<11, "q11", [D11], ["v11", ""]>, DwarfRegAlias<B11>;
-def Q12   : AArch64Reg<12, "q12", [D12], ["v12", ""]>, DwarfRegAlias<B12>;
-def Q13   : AArch64Reg<13, "q13", [D13], ["v13", ""]>, DwarfRegAlias<B13>;
-def Q14   : AArch64Reg<14, "q14", [D14], ["v14", ""]>, DwarfRegAlias<B14>;
-def Q15   : AArch64Reg<15, "q15", [D15], ["v15", ""]>, DwarfRegAlias<B15>;
-def Q16   : AArch64Reg<16, "q16", [D16], ["v16", ""]>, DwarfRegAlias<B16>;
-def Q17   : AArch64Reg<17, "q17", [D17], ["v17", ""]>, DwarfRegAlias<B17>;
-def Q18   : AArch64Reg<18, "q18", [D18], ["v18", ""]>, DwarfRegAlias<B18>;
-def Q19   : AArch64Reg<19, "q19", [D19], ["v19", ""]>, DwarfRegAlias<B19>;
-def Q20   : AArch64Reg<20, "q20", [D20], ["v20", ""]>, DwarfRegAlias<B20>;
-def Q21   : AArch64Reg<21, "q21", [D21], ["v21", ""]>, DwarfRegAlias<B21>;
-def Q22   : AArch64Reg<22, "q22", [D22], ["v22", ""]>, DwarfRegAlias<B22>;
-def Q23   : AArch64Reg<23, "q23", [D23], ["v23", ""]>, DwarfRegAlias<B23>;
-def Q24   : AArch64Reg<24, "q24", [D24], ["v24", ""]>, DwarfRegAlias<B24>;
-def Q25   : AArch64Reg<25, "q25", [D25], ["v25", ""]>, DwarfRegAlias<B25>;
-def Q26   : AArch64Reg<26, "q26", [D26], ["v26", ""]>, DwarfRegAlias<B26>;
-def Q27   : AArch64Reg<27, "q27", [D27], ["v27", ""]>, DwarfRegAlias<B27>;
-def Q28   : AArch64Reg<28, "q28", [D28], ["v28", ""]>, DwarfRegAlias<B28>;
-def Q29   : AArch64Reg<29, "q29", [D29], ["v29", ""]>, DwarfRegAlias<B29>;
-def Q30   : AArch64Reg<30, "q30", [D30], ["v30", ""]>, DwarfRegAlias<B30>;
-def Q31   : AArch64Reg<31, "q31", [D31], ["v31", ""]>, DwarfRegAlias<B31>;
+let SubRegIndices = [bsub, bsub_hi] in {
+def H0    : AArch64Reg<0,   "h0", [B0, B0_HI]>, DwarfRegAlias<B0>;
+def H1    : AArch64Reg<1,   "h1", [B1, B1_HI]>, DwarfRegAlias<B1>;
+def H2    : AArch64Reg<2,   "h2", [B2, B2_HI]>, DwarfRegAlias<B2>;
+def H3    : AArch64Reg<3,   "h3", [B3, B3_HI]>, DwarfRegAlias<B3>;
+def H4    : AArch64Reg<4,   "h4", [B4, B4_HI]>, DwarfRegAlias<B4>;
+def H5    : AArch64Reg<5,   "h5", [B5, B5_HI]>, DwarfRegAlias<B5>;
+def H6    : AArch64Reg<6,   "h6", [B6, B6_HI]>, DwarfRegAlias<B6>;
+def H7    : AArch64Reg<7,   "h7", [B7, B7_HI]>, DwarfRegAlias<B7>;
+def H8    : AArch64Reg<8,   "h8", [B8, B8_HI]>, DwarfRegAlias<B8>;
+def H9    : AArch64Reg<9,   "h9", [B9, B9_HI]>, DwarfRegAlias<B9>;
+def H10   : AArch64Reg<10, "h10", [B10, B10_HI]>, DwarfRegAlias<B10>;
+def H11   : AArch64Reg<11, "h11", [B11, B11_HI]>, DwarfRegAlias<B11>;
+def H12   : AArch64Reg<12, "h12", [B12, B12_HI]>, DwarfRegAlias<B12>;
+def H13   : AArch64Reg<13, "h13", [B13, B13_HI]>, DwarfRegAlias<B13>;
+def H14   : AArch64Reg<14, "h14", [B14, B14_HI]>, DwarfRegAlias<B14>;
+def H15   : AArch64Reg<15, "h15", [B15, B15_HI]>, DwarfRegAlias<B15>;
+def H16   : AArch64Reg<16, "h16", [B16, B16_HI]>, DwarfRegAlias<B16>;
+def H17   : AArch64Reg<17, "h17", [B17, B17_HI]>, DwarfRegAlias<B17>;
+def H18   : AArch64Reg<18, "h18", [B18, B18_HI]>, DwarfRegAlias<B18>;
+def H19   : AArch64Reg<19, "h19", [B19, B19_HI]>, DwarfRegAlias<B19>;
+def H20   : AArch64Reg<20, "h20", [B20, B20_HI]>, DwarfRegAlias<B20>;
+def H21   : AArch64Reg<21, "h21", [B21, B21_HI]>, DwarfRegAlias<B21>;
+def H22   : AArch64Reg<22, "h22", [B22, B22_HI]>, DwarfRegAlias<B22>;
+def H23   : AArch64Reg<23, "h23", [B23, B23_HI]>, DwarfRegAlias<B23>;
+def H24   : AArch64Reg<24, "h24", [B24, B24_HI]>, DwarfRegAlias<B24>;
+def H25   : AArch64Reg<25, "h25", [B25, B25_HI]>, DwarfRegAlias<B25>;
+def H26   : AArch64Reg<26, "h26", [B26, B26_HI]>, DwarfRegAlias<B26>;
+def H27   : AArch64Reg<27, "h27", [B27, B27_HI]>, DwarfRegAlias<B27>;
+def H28   : AArch64Reg<28, "h28", [B28, B28_HI]>, DwarfRegAlias<B28>;
+def H29   : AArch64Reg<29, "h29", [B29, B29_HI]>, DwarfRegAlias<B29>;
+def H30   : AArch64Reg<30, "h30", [B30, B30_HI]>, DwarfRegAlias<B30>;
+def H31   : AArch64Reg<31, "h31", [B31, B31_HI]>, DwarfRegAlias<B31>;
+}
+
+let SubRegIndices = [hsub, hsub_hi] in {
+def S0    : AArch64Reg<0,   "s0", [H0, H0_HI]>, DwarfRegAlias<B0>;
+def S1    : AArch64Reg<1,   "s1", [H1, H1_HI]>, DwarfRegAlias<B1>;
+def S2    : AArch64Reg<2,   "s2", [H2, H2_HI]>, DwarfRegAlias<B2>;
+def S3    : AArch64Reg<3,   "s3", [H3, H3_HI]>, DwarfRegAlias<B3>;
+def S4    : AArch64Reg<4,   "s4", [H4, H4_HI]>, DwarfRegAlias<B4>;
+def S5    : AArch64Reg<5,   "s5", [H5, H5_HI]>, DwarfRegAlias<B5>;
+def S6    : AArch64Reg<6,   "s6", [H6, H6_HI]>, DwarfRegAlias<B6>;
+def S7    : AArch64Reg<7,   "s7", [H7, H7_HI]>, DwarfRegAlias<B7>;
+def S8    : AArch64Reg<8,   "s8", [H8, H8_HI]>, DwarfRegAlias<B8>;
+def S9    : AArch64Reg<9,   "s9", [H9, H9_HI]>, DwarfRegAlias<B9>;
+def S10   : AArch64Reg<10, "s10", [H10, H10_HI]>, DwarfRegAlias<B10>;
+def S11   : AArch64Reg<11, "s11", [H11, H11_HI]>, DwarfRegAlias<B11>;
+def S12   : AArch64Reg<12, "s12", [H12, H12_HI]>, DwarfRegAlias<B12>;
+def S13   : AArch64Reg<13, "s13", [H13, H13_HI]>, DwarfRegAlias<B13>;
+def S14   : AArch64Reg<14, "s14", [H14, H14_HI]>, DwarfRegAlias<B14>;
+def S15   : AArch64Reg<15, "s15", [H15, H15_HI]>, DwarfRegAlias<B15>;
+def S16   : AArch64Reg<16, "s16", [H16, H16_HI]>, DwarfRegAlias<B16>;
+def S17   : AArch64Reg<17, "s17", [H17, H17_HI]>, DwarfRegAlias<B17>;
+def S18   : AArch64Reg<18, "s18", [H18, H18_HI]>, DwarfRegAlias<B18>;
+def S19   : AArch64Reg<19, "s19", [H19, H19_HI]>, DwarfRegAlias<B19>;
+def S20   : AArch64Reg<20, "s20", [H20, H20_HI]>, DwarfRegAlias<B20>;
+def S21   : AArch64Reg<21, "s21", [H21, H21_HI]>, DwarfRegAlias<B21>;
+def S22   : AArch64Reg<22, "s22", [H22, H22_HI]>, DwarfRegAlias<B22>;
+def S23   : AArch64Reg<23, "s23", [H23, H23_HI]>, DwarfRegAlias<B23>;
+def S24   : AArch64Reg<24, "s24", [H24, H24_HI]>, DwarfRegAlias<B24>;
+def S25   : AArch64Reg<25, "s25", [H25, H25_HI]>, DwarfRegAlias<B25>;
+def S26   : AArch64Reg<26, "s26", [H26, H26_HI]>, DwarfRegAlias<B26>;
+def S27   : AArch64Reg<27, "s27", [H27, H27_HI]>, DwarfRegAlias<B27>;
+def S28   : AArch64Reg<28, "s28", [H28, H28_HI]>, DwarfRegAlias<B28>;
+def S29   : AArch64Reg<29, "s29", [H29, H29_HI]>, DwarfRegAlias<B29>;
+def S30   : AArch64Reg<30, "s30", [H30, H30_HI]>, DwarfRegAlias<B30>;
+def S31   : AArch64Reg<31, "s31", [H31, H31_HI]>, DwarfRegAlias<B31>;
+}
+
+let SubRegIndices = [ssub, ssub_hi], RegAltNameIndices = [vreg, vlist1] in {
+def D0    : AArch64Reg<0,   "d0", [S0, S0_HI], ["v0", ""]>, DwarfRegAlias<B0>;
+def D1    : AArch64Reg<1,   "d1", [S1, S1_HI], ["v1", ""]>, DwarfRegAlias<B1>;
+def D2    : AArch64Reg<2,   "d2", [S2, S2_HI], ["v2", ""]>, DwarfRegAlias<B2>;
+def D3    : AArch64Reg<3,   "d3", [S3, S3_HI], ["v3", ""]>, DwarfRegAlias<B3>;
+def D4    : AArch64Reg<4,   "d4", [S4, S4_HI], ["v4", ""]>, DwarfRegAlias<B4>;
+def D5    : AArch64Reg<5,   "d5", [S5, S5_HI], ["v5", ""]>, DwarfRegAlias<B5>;
+def D6    : AArch64Reg<6,   "d6", [S6, S6_HI], ["v6", ""]>, DwarfRegAlias<B6>;
+def D7    : AArch64Reg<7,   "d7", [S7, S7_HI], ["v7", ""]>, DwarfRegAlias<B7>;
+def D8    : AArch64Reg<8,   "d8", [S8, S8_HI], ["v8", ""]>, DwarfRegAlias<B8>;
+def D9    : AArch64Reg<9,   "d9", [S9, S9_HI], ["v9", ""]>, DwarfRegAlias<B9>;
+def D10   : AArch64Reg<10, "d10", [S10, S10_HI], ["v10", ""]>, DwarfRegAlias<B10>;
+def D11   : AArch64Reg<11, "d11", [S11, S11_HI], ["v11", ""]>, DwarfRegAlias<B11>;
+def D12   : AArch64Reg<12, "d12", [S12, S12_HI], ["v12", ""]>, DwarfRegAlias<B12>;
+def D13   : AArch64Reg<13, "d13", [S13, S13_HI], ["v13", ""]>, DwarfRegAlias<B13>;
+def D14   : AArch64Reg<14, "d14", [S14, S14_HI], ["v14", ""]>, DwarfRegAlias<B14>;
+def D15   : AArch64Reg<15, "d15", [S15, S15_HI], ["v15", ""]>, DwarfRegAlias<B15>;
+def D16   : AArch64Reg<16, "d16", [S16, S16_HI], ["v16", ""]>, DwarfRegAlias<B16>;
+def D17   : AArch64Reg<17, "d17", [S17, S17_HI], ["v17", ""]>, DwarfRegAlias<B17>;
+def D18   : AArch64Reg<18, "d18", [S18, S18_HI], ["v18", ""]>, DwarfRegAlias<B18>;
+def D19   : AArch64Reg<19, "d19", [S19, S19_HI], ["v19", ""]>, DwarfRegAlias<B19>;
+def D20   : AArch64Reg<20, "d20", [S20, S20_HI], ["v20", ""]>, DwarfRegAlias<B20>;
+def D21   : AArch64Reg<21, "d21", [S21, S21_HI], ["v21", ""]>, DwarfRegAlias<B21>;
+def D22   : AArch64Reg<22, "d22", [S22, S22_HI], ["v22", ""]>, DwarfRegAlias<B22>;
+def D23   : AArch64Reg<23, "d23", [S23, S23_HI], ["v23", ""]>, DwarfRegAlias<B23>;
+def D24   : AArch64Reg<24, "d24", [S24, S24_HI], ["v24", ""]>, DwarfRegAlias<B24>;
+def D25   : AArch64Reg<25, "d25", [S25, S25_HI], ["v25", ""]>, DwarfRegAlias<B25>;
+def D26   : AArch64Reg<26, "d26", [S26, S26_HI], ["v26", ""]>, DwarfRegAlias<B26>;
+def D27   : AArch64Reg<27, "d27", [S27, S27_HI], ["v27", ""]>, DwarfRegAlias<B27>;
+def D28   : AArch64Reg<28, "d28", [S28, S28_HI], ["v28", ""]>, DwarfRegAlias<B28>;
+def D29   : AArch64Reg<29, "d29", [S29, S29_HI], ["v29", ""]>, DwarfRegAlias<B29>;
+def D30   : AArch64Reg<30, "d30", [S30, S30_HI], ["v30", ""]>, DwarfRegAlias<B30>;
+def D31   : AArch64Reg<31, "d31", [S31, S31_HI], ["v31", ""]>, DwarfRegAlias<B31>;
+}
+
+let SubRegIndices = [dsub, dsub_hi], RegAltNameIndices = [vreg, vlist1] in {
+def Q0    : AArch64Reg<0,   "q0", [D0, D0_HI], ["v0", ""]>, DwarfRegAlias<B0>;
+def Q1    : AArch64Reg<1,   "q1", [D1, D1_HI], ["v1", ""]>, DwarfRegAlias<B1>;
+def Q2    : AArch64Reg<2,   "q2", [D2, D2_HI], ["v2", ""]>, DwarfRegAlias<B2>;
+def Q3    : AArch64Reg<3,   "q3", [D3, D3_HI], ["v3", ""]>, DwarfRegAlias<B3>;
+def Q4    : AArch64Reg<4,   "q4", [D4, D4_HI], ["v4", ""]>, DwarfRegAlias<B4>;
+def Q5    : AArch64Reg<5,   "q5", [D5, D5_HI], ["v5", ""]>, DwarfRegAlias<B5>;
+def Q6    : AArch64Reg<6,   "q6", [D6, D6_HI], ["v6", ""]>, DwarfRegAlias<B6>;
+def Q7    : AArch64Reg<7,   "q7", [D7, D7_HI], ["v7", ""]>, DwarfRegAlias<B7>;
+def Q8    : AArch64Reg<8,   "q8", [D8, D8_HI], ["v8", ""]>, DwarfRegAlias<B8>;
+def Q9    : AArch64Reg<9,   "q9", [D9, D9_HI], ["v9", ""]>, DwarfRegAlias<B9>;
+def Q10   : AArch64Reg<10, "q10", [D10, D10_HI], ["v10", ""]>, DwarfRegAlias<B10>;
+def Q11   : AArch64Reg<11, "q11", [D11, D11_HI], ["v11", ""]>, DwarfRegAlias<B11>;
+def Q12   : AArch64Reg<12, "q12", [D12, D12_HI], ["v12", ""]>, DwarfRegAlias<B12>;
+def Q13   : AArch64Reg<13, "q13", [D13, D13_HI], ["v13", ""]>, DwarfRegAlias<B13>;
+def Q14   : AArch64Reg<14, "q14", [D14, D14_HI], ["v14", ""]>, DwarfRegAlias<B14>;
+def Q15   : AArch64Reg<15, "q15", [D15, D15_HI], ["v15", ""]>, DwarfRegAlias<B15>;
+def Q16   : AArch64Reg<16, "q16", [D16, D16_HI], ["v16", ""]>, DwarfRegAlias<B16>;
+def Q17   : AArch64Reg<17, "q17", [D17, D17_HI], ["v17", ""]>, DwarfRegAlias<B17>;
+def Q18   : AArch64Reg<18, "q18", [D18, D18_HI], ["v18", ""]>, DwarfRegAlias<B18>;
+def Q19   : AArch64Reg<19, "q19", [D19, D19_HI], ["v19", ""]>, DwarfRegAlias<B19>;
+def Q20   : AArch64Reg<20, "q20", [D20, D20_HI], ["v20", ""]>, DwarfRegAlias<B20>;
+def Q21   : AArch64Reg<21, "q21", [D21, D21_HI], ["v21", ""]>, DwarfRegAlias<B21>;
+def Q22   : AArch64Reg<22, "q22", [D22, D22_HI], ["v22", ""]>, DwarfRegAlias<B22>;
+def Q23   : AArch64Reg<23, "q23", [D23, D23_HI], ["v23", ""]>, DwarfRegAlias<B23>;
+def Q24   : AArch64Reg<24, "q24", [D24, D24_HI], ["v24", ""]>, DwarfRegAlias<B24>;
+def Q25   : AArch64Reg<25, "q25", [D25, D25_HI], ["v25", ""]>, DwarfRegAlias<B25>;
+def Q26   : AArch64Reg<26, "q26", [D26, D26_HI], ["v26", ""]>, DwarfRegAlias<B26>;
+def Q27   : AArch64Reg<27, "q27", [D27, D27_HI], ["v27", ""]>, DwarfRegAlias<B27>;
+def Q28   : AArch64Reg<28, "q28", [D28, D28_HI], ["v28", ""]>, DwarfRegAlias<B28>;
+def Q29   : AArch64Reg<29, "q29", [D29, D29_HI], ["v29", ""]>, DwarfRegAlias<B29>;
+def Q30   : AArch64Reg<30, "q30", [D30, D30_HI], ["v30", ""]>, DwarfRegAlias<B30>;
+def Q31   : AArch64Reg<31, "q31", [D31, D31_HI], ["v31", ""]>, DwarfRegAlias<B31>;
 }
 
 def FPR8  : RegisterClass<"AArch64", [i8], 8, (sequence "B%u", 0, 31)> {
@@ -872,39 +902,39 @@ let SubRegIndices = [psub] in {
 }
 
 // SVE variable-size vector registers
-let SubRegIndices = [zsub] in {
-def Z0    : AArch64Reg<0,   "z0",  [Q0]>, DwarfRegNum<[96]>;
-def Z1    : AArch64Reg<1,   "z1",  [Q1]>, DwarfRegNum<[97]>;
-def Z2    : AArch64Reg<2,   "z2",  [Q2]>, DwarfRegNum<[98]>;
-def Z3    : AArch64Reg<3,   "z3",  [Q3]>, DwarfRegNum<[99]>;
-def Z4    : AArch64Reg<4,   "z4",  [Q4]>, DwarfRegNum<[100]>;
-def Z5    : AArch64Reg<5,   "z5",  [Q5]>, DwarfRegNum<[101]>;
-def Z6    : AArch64Reg<6,   "z6",  [Q6]>, DwarfRegNum<[102]>;
-def Z7    : AArch64Reg<7,   "z7",  [Q7]>, DwarfRegNum<[103]>;
-def Z8    : AArch64Reg<8,   "z8",  [Q8]>, DwarfRegNum<[104]>;
-def Z9    : AArch64Reg<9,   "z9",  [Q9]>, DwarfRegNum<[105]>;
-def Z10   : AArch64Reg<10, "z10", [Q10]>, DwarfRegNum<[106]>;
-def Z11   : AArch64Reg<11, "z11", [Q11]>, DwarfRegNum<[107]>;
-def Z12   : AArch64Reg<12, "z12", [Q12]>, DwarfRegNum<[108]>;
-def Z13   : AArch64Reg<13, "z13", [Q13]>, DwarfRegNum<[109]>;
-def Z14   : AArch64Reg<14, "z14", [Q14]>, DwarfRegNum<[110]>;
-def Z15   : AArch64Reg<15, "z15", [Q15]>, DwarfRegNum<[111]>;
-def Z16   : AArch64Reg<16, "z16", [Q16]>, DwarfRegNum<[112]>;
-def Z17   : AArch64Reg<17, "z17", [Q17]>, DwarfRegNum<[113]>;
-def Z18   : AArch64Reg<18, "z18", [Q18]>, DwarfRegNum<[114]>;
-def Z19   : AArch64Reg<19, "z19", [Q19]>, DwarfRegNum<[115]>;
-def Z20   : AArch64Reg<20, "z20", [Q20]>, DwarfRegNum<[116]>;
-def Z21   : AArch64Reg<21, "z21", [Q21]>, DwarfRegNum<[117]>;
-def Z22   : AArch64Reg<22, "z22", [Q22]>, DwarfRegNum<[118]>;
-def Z23   : AArch64Reg<23, "z23", [Q23]>, DwarfRegNum<[119]>;
-def Z24   : AArch64Reg<24, "z24", [Q24]>, DwarfRegNum<[120]>;
-def Z25   : AArch64Reg<25, "z25", [Q25]>, DwarfRegNum<[121]>;
-def Z26   : AArch64Reg<26, "z26", [Q26]>, DwarfRegNum<[122]>;
-def Z27   : AArch64Reg<27, "z27", [Q27]>, DwarfRegNum<[123]>;
-def Z28   : AArch64Reg<28, "z28", [Q28]>, DwarfRegNum<[124]>;
-def Z29   : AArch64Reg<29, "z29", [Q29]>, DwarfRegNum<[125]>;
-def Z30   : AArch64Reg<30, "z30", [Q30]>, DwarfRegNum<[126]>;
-def Z31   : AArch64Reg<31, "z31", [Q31]>, DwarfRegNum<[127]>;
+let SubRegIndices = [zsub, zsub_hi] in {
+def Z0    : AArch64Reg<0,   "z0",  [Q0, Q0_HI]>, DwarfRegNum<[96]>;
+def Z1    : AArch64Reg<1,   "z1",  [Q1, Q1_HI]>, DwarfRegNum<[97]>;
+def Z2    : AArch64Reg<2,   "z2",  [Q2, Q2_HI]>, DwarfRegNum<[98]>;
+def Z3    : AArch64Reg<3,   "z3",  [Q3, Q3_HI]>, DwarfRegNum<[99]>;
+def Z4    : AArch64Reg<4,   "z4",  [Q4, Q4_HI]>, DwarfRegNum<[100]>;
+def Z5    : AArch64Reg<5,   "z5",  [Q5, Q5_HI]>, DwarfRegNum<[101]>;
+def Z6    : AArch64Reg<6,   "z6",  [Q6, Q6_HI]>, DwarfRegNum<[102]>;
+def Z7    : AArch64Reg<7,   "z7",  [Q7, Q7_HI]>, DwarfRegNum<[103]>;
+def Z8    : AArch64Reg<8,   "z8",  [Q8, Q8_HI]>, DwarfRegNum<[104]>;
+def Z9    : AArch64Reg<9,   "z9",  [Q9, Q9_HI]>, DwarfRegNum<[105]>;
+def Z10   : AArch64Reg<10, "z10", [Q10, Q10_HI]>, DwarfRegNum<[106]>;
+def Z11   : AArch64Reg<11, "z11", [Q11, Q11_HI]>, DwarfRegNum<[107]>;
+def Z12   : AArch64Reg<12, "z12", [Q12, Q12_HI]>, DwarfRegNum<[108]>;
+def Z13   : AArch64Reg<13, "z13", [Q13, Q13_HI]>, DwarfRegNum<[109]>;
+def Z14   : AArch64Reg<14, "z14", [Q14, Q14_HI]>, DwarfRegNum<[110]>;
+def Z15   : AArch64Reg<15, "z15", [Q15, Q15_HI]>, DwarfRegNum<[111]>;
+def Z16   : AArch64Reg<16, "z16", [Q16, Q16_HI]>, DwarfRegNum<[112]>;
+def Z17   : AArch64Reg<17, "z17", [Q17, Q17_HI]>, DwarfRegNum<[113]>;
+def Z18   : AArch64Reg<18, "z18", [Q18, Q18_HI]>, DwarfRegNum<[114]>;
+def Z19   : AArch64Reg<19, "z19", [Q19, Q19_HI]>, DwarfRegNum<[115]>;
+def Z20   : AArch64Reg<20, "z20", [Q20, Q20_HI]>, DwarfRegNum<[116]>;
+def Z21   : AArch64Reg<21, "z21", [Q21, Q21_HI]>, DwarfRegNum<[117]>;
+def Z22   : AArch64Reg<22, "z22", [Q22, Q22_HI]>, DwarfRegNum<[118]>;
+def Z23   : AArch64Reg<23, "z23", [Q23, Q23_HI]>, DwarfRegNum<[119]>;
+def Z24   : AArch64Reg<24, "z24", [Q24, Q24_HI]>, DwarfRegNum<[120]>;
+def Z25   : AArch64Reg<25, "z25", [Q25, Q25_HI]>, DwarfRegNum<[121]>;
+def Z26   : AArch64Reg<26, "z26", [Q26, Q26_HI]>, DwarfRegNum<[122]>;
+def Z27   : AArch64Reg<27, "z27", [Q27, Q27_HI]>, DwarfRegNum<[123]>;
+def Z28   : AArch64Reg<28, "z28", [Q28, Q28_HI]>, DwarfRegNum<[124]>;
+def Z29   : AArch64Reg<29, "z29", [Q29, Q29_HI]>, DwarfRegNum<[125]>;
+def Z30   : AArch64Reg<30, "z30", [Q30, Q30_HI]>, DwarfRegNum<[126]>;
+def Z31   : AArch64Reg<31, "z31", [Q31, Q31_HI]>, DwarfRegNum<[127]>;
 }
 
 // Enum describing the element size for destructive
@@ -1949,6 +1979,15 @@ def svcr_op : Operand<i32>, TImmLeaf<i32, [{
   }];
 }
 
+let isAllocatable = 0, GeneratePressureSet = 0 in {
+  def W_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (add (sequence "W%u_HI", 0, 30), WZR_HI, WSP_HI)>;
+  def B_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (sequence "B%u_HI", 0, 31)>;
+  def H_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (sequence "H%u_HI", 0, 31)>;
+  def S_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (sequence "S%u_HI", 0, 31)>;
+  def D_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (sequence "D%u_HI", 0, 31)>;
+  def Q_HI_DummyRC : RegisterClass<"AArch64", [untyped], 0, (sequence "Q%u_HI", 0, 31)>;
+}
+
 //===----------------------------------------------------------------------===//
 // Register categories.
 //
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 37ac915d1d880..9c657787d3492 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -990,7 +990,7 @@ defm FDOT_VG2_M2ZZI_BtoH  : sme2p1_multi_vec_array_vg2_index_f8f16<"fdot",  0b11
 defm FDOT_VG4_M4ZZI_BtoH  : sme2p1_multi_vec_array_vg4_index_f8f16<"fdot",    0b100, ZZZZ_b_mul_r, ZPR4b8>;
 defm FDOT_VG2_M2ZZ_BtoH   :  sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0010001, MatrixOp16, ZZ_b, ZPR4b8>;
 defm FDOT_VG4_M4ZZ_BtoH   :  sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0110001, MatrixOp16, ZZZZ_b, ZPR4b8>;
-// TODO: Replace nxv16i8 by nxv16f8
+
 defm FDOT_VG2_M2Z2Z_BtoH  : sme2_dot_mla_add_sub_array_vg2_multi<"fdot",    0b0100100, MatrixOp16, ZZ_b_mul_r, nxv16i8, null_frag>;
 defm FDOT_VG4_M4Z4Z_BtoH  : sme2_dot_mla_add_sub_array_vg4_multi<"fdot",    0b0100100, MatrixOp16, ZZZZ_b_mul_r, nxv16i8, null_frag>;
 
@@ -998,23 +998,22 @@ def  FMLAL_MZZI_BtoH      : sme2_mla_ll_array_index_16b<"fmlal", 0b11, 0b00>;
 defm FMLAL_VG2_M2ZZI_BtoH : sme2_multi_vec_array_vg2_index_16b<"fmlal", 0b10, 0b111>;
 defm FMLAL_VG4_M4ZZI_BtoH : sme2_multi_vec_array_vg4_index_16b<"fmlal", 0b10, 0b110>;
 def  FMLAL_VG2_MZZ_BtoH   : sme2_mla_long_array_single_16b<"fmlal">;
-// TODO: Replace nxv16i8 by nxv16f8
+
 defm FMLAL_VG2_M2ZZ_BtoH  : sme2_fp_mla_long_array_vg2_single<"fmlal",  0b001, MatrixOp16, ZZ_b, ZPR4b8, nxv16i8, null_frag>;
 defm FMLAL_VG4_M4ZZ_BtoH  :  sme2_fp_mla_long_array_vg4_single<"fmlal", 0b001, MatrixOp16, ZZZZ_b, ZPR4b8, nxv16i8, null_frag>;
 defm FMLAL_VG2_M2Z2Z_BtoH : sme2_fp_mla_long_array_vg2_multi<"fmlal",   0b100, MatrixOp16, ZZ_b_mul_r, nxv16i8, null_frag>;
 defm FMLAL_VG4_M4Z4Z_BtoH : sme2_fp_mla_long_array_vg4_multi<"fmlal",   0b100, MatrixOp16, ZZZZ_b_mul_r, nxv16i8, null_frag>;
 
-defm FMOPA_MPPZZ_BtoH     : sme2p1_fmop_tile_f8f16<"fmopa", 0b1, 0b0, 0b01>;
-
+defm FMOPA_MPPZZ_BtoH : sme2_fp8_fmopa_za16<"fmopa", int_aarch64_sme_fp8_fmopa_za16>;
 } //[HasSMEF8F16]
 
 let Predicates = [HasSMEF8F32] in {
-// TODO : Replace nxv16i8 by nxv16f8
+
 defm FDOT_VG2_M2ZZI_BtoS : sme2_multi_vec_array_vg2_index_32b<"fdot", 0b01, 0b0111, ZZ_b_mul_r, ZPR4b8, nxv16i8, null_frag>;
 defm FDOT_VG4_M4ZZI_BtoS : sme2_multi_vec_array_vg4_index_32b<"fdot", 0b0001, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, null_frag>;
 defm FDOT_VG2_M2ZZ_BtoS  : sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0010011, MatrixOp32, ZZ_b, ZPR4b8>;
 defm FDOT_VG4_M4ZZ_BtoS  : sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0110011, MatrixOp32, ZZZZ_b, ZPR4b8>;
-// TODO : Replace nxv16i8 by nxv16f8
+
 defm FDOT_VG2_M2Z2Z_BtoS : sme2_dot_mla_add_sub_array_vg2_multi<"fdot",   0b0100110, MatrixOp32, ZZ_b_mul_r, nxv16i8, null_frag>;
 defm FDOT_VG4_M4Z4Z_BtoS : sme2_dot_mla_add_sub_array_vg4_multi<"fdot",   0b0100110, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, null_frag>;
 
@@ -1024,16 +1023,14 @@ def FVDOTT_VG4_M2ZZI_BtoS : sme2_fp8_multi_vec_array_vg4_index<"fvdott", 0b1>;
 defm FMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"fmlall",     0b01, 0b000, null_frag>;
 defm FMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"fmlall", 0b10, 0b100, null_frag>;
 defm FMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"fmlall", 0b00, 0b1000, null_frag>;
-// TODO: Replace nxv16i8 by nxv16f8
+
 defm FMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"fmlall",      0b01000, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, null_frag>;
 defm FMLALL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg24_single<"fmlall", 0b000001, MatrixOp32, ZZ_b, ZPR4b8>;
 defm FMLALL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg24_single<"fmlall", 0b010001, MatrixOp32, ZZZZ_b, ZPR4b8>;
 defm FMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"fmlall",   0b01000, MatrixOp32, ZZ_b_mul_r, nxv16i8, null_frag>;
 defm FMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"fmlall",   0b01000, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, null_frag>;
 
-
-defm FMOPA_MPPZZ_BtoS : sme_outer_product_fp32<0b0, 0b01, ZPR8, "fmopa", null_frag>;
-
+defm FMOPA_MPPZZ_BtoS : sme2_fp8_fmopa_za32<"fmopa", int_aarch64_sme_fp8_fmopa_za32>;
 } //[HasSMEF8F32]
 
 let Predicates = [HasSME2, HasSVEBFSCALE] in {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 564fb33758ad5..da585dd3a21c8 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -675,6 +675,14 @@ let Predicates = [HasSVEorSME] in {
   defm FABS_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b100, "fabs", AArch64fabs_mt>;
   defm FNEG_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b101, "fneg", AArch64fneg_mt>;
 
+  let Predicates = [HasSVEorSME, UseUnaryUndefPseudos] in {
+    defm FABS_ZPmZ : sve_fp_un_pred_arit_hsd<AArch64fabs_mt>;
+    defm FNEG_ZPmZ : sve_fp_un_pred_arit_hsd<AArch64fneg_mt>;
+
+    defm ABS_ZPmZ : sve_int_un_pred_arit_bhsd<AArch64abs_mt>;
+    defm NEG_ZPmZ : sve_int_un_pred_arit_bhsd<AArch64neg_mt>;
+  }
+
   foreach VT = [nxv2bf16, nxv4bf16, nxv8bf16] in {
     // No dedicated instruction, so just clear the sign bit.
     def : Pat<(VT (fabs VT:$op)),
@@ -4257,25 +4265,25 @@ defm TBLQ_ZZZ  : sve2p1_tblq<"tblq", int_aarch64_sve_tblq>;
 //===----------------------------------------------------------------------===//
 let Predicates = [HasSVE2p2orSME2p2] in {
   // SVE Floating-point convert precision, zeroing predicate
-  defm FCVT_ZPzZ : sve_fp_z2op_p_zd_b_0<"fcvt">;
+  defm FCVT_ZPzZ : sve_fp_z2op_p_zd_b_0<"fcvt", "int_aarch64_sve_fcvt">;
 
   // SVE2p2 floating-point convert precision down (placing odd), zeroing predicate
   defm FCVTNT_ZPzZ      : sve_fp_fcvtntz<"fcvtnt">;
   def FCVTXNT_ZPzZ_DtoS : sve_fp_fcvt2z<0b0010, "fcvtxnt", ZPR32, ZPR64>;
   // Placing even
-  def FCVTX_ZPzZ_DtoS   : sve_fp_z2op_p_zd<0b0001010, "fcvtx", ZPR64, ZPR32>;
+  defm FCVTX_ZPzZ       : sve_fp_z2op_p_zd<"fcvtx", int_aarch64_sve_fcvtx_f32f64>;
 
   // SVE2p2 floating-point convert precision up, zeroing predicate
-  defm FCVTLT_ZPzZ      : sve_fp_fcvtltz<"fcvtlt">;
+  defm FCVTLT_ZPzZ      : sve_fp_fcvtltz<"fcvtlt", "int_aarch64_sve_fcvtlt">;
 
   // SVE2p2 floating-point convert single-to-bf (placing odd), zeroing predicate
   def BFCVTNT_ZPzZ      : sve_fp_fcvt2z<0b1010, "bfcvtnt", ZPR16, ZPR32>;
   // Placing corresponding
-  def BFCVT_ZPzZ_StoH   : sve_fp_z2op_p_zd<0b1001010, "bfcvt", ZPR32, ZPR16>;
+  defm BFCVT_ZPzZ_StoH  : sve_fp_z2op_p_zd_bfcvt<"bfcvt", int_aarch64_sve_fcvt_bf16f32_v2>;
 
   // Floating-point convert to integer, zeroing predicate
-  defm FCVTZS_ZPzZ : sve_fp_z2op_p_zd_d<0b0, "fcvtzs">;
-  defm FCVTZU_ZPzZ : sve_fp_z2op_p_zd_d<0b1, "fcvtzu">;
+  defm FCVTZS_ZPzZ : sve_fp_z2op_p_zd_d<0b0, "fcvtzs", "int_aarch64_sve_fcvtzs", AArch64fcvtzs_mt>;
+  defm FCVTZU_ZPzZ : sve_fp_z2op_p_zd_d<0b1, "fcvtzu", "int_aarch64_sve_fcvtzu", AArch64fcvtzu_mt>;
   // Integer convert to floating-point, zeroing predicate
   defm SCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b0, "scvtf">;
   defm UCVTF_ZPzZ  : sve_fp_z2op_p_zd_c<0b1, "ucvtf">;
@@ -4321,16 +4329,16 @@ let Predicates = [HasSVE2p2orSME2p2] in {
   defm NOT_ZPzZ  : sve_int_un_pred_arit_bitwise_z<0b110, "not">;
 
   // floating point
-  defm FABS_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b100, "fabs">;
-  defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg">;
+  defm FABS_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b100, "fabs", AArch64fabs_mt>;
+  defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg", AArch64fneg_mt>;
 
   // SVE2p2 integer unary arithmetic, zeroing predicate
   defm SXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b000, "sxtb">;
   defm UXTB_ZPzZ  : sve_int_un_pred_arit_h_z<0b001, "uxtb">;
   defm SXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b010, "sxth">;
   defm UXTH_ZPzZ  : sve_int_un_pred_arit_w_z<0b011, "uxth">;
-  defm ABS_ZPzZ   : sve_int_un_pred_arit_z<  0b110, "abs">;
-  defm NEG_ZPzZ   : sve_int_un_pred_arit_z<  0b111, "neg">;
+  defm ABS_ZPzZ   : sve_int_un_pred_arit_z<  0b110, "abs", AArch64abs_mt>;
+  defm NEG_ZPzZ   : sve_int_un_pred_arit_z<  0b111, "neg", AArch64neg_mt>;
   def SXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1000, "sxtw", ZPR64>;
   def UXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1010, "uxtw", ZPR64>;
 
@@ -4361,14 +4369,14 @@ let Predicates = [HasNonStreamingSVE2p2orSME2p2] in {
 //===----------------------------------------------------------------------===//
 let Predicates = [HasSVE2orSME2, HasFP8] in {
 // FP8 upconvert
-defm F1CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b00, "f1cvt">;
-defm F2CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b01, "f2cvt">;
-defm BF1CVT_ZZ    : sve2_fp8_cvt_single<0b0, 0b10, "bf1cvt">;
-defm BF2CVT_ZZ    : sve2_fp8_cvt_single<0b0, 0b11, "bf2cvt">;
-defm F1CVTLT_ZZ   : sve2_fp8_cvt_single<0b1, 0b00, "f1cvtlt">;
-defm F2CVTLT_ZZ   : sve2_fp8_cvt_single<0b1, 0b01, "f2cvtlt">;
-defm BF1CVTLT_ZZ  : sve2_fp8_cvt_single<0b1, 0b10, "bf1cvtlt">;
-defm BF2CVTLT_ZZ  : sve2_fp8_cvt_single<0b1, 0b11, "bf2cvtlt">;
+defm F1CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b00, "f1cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt1>;
+defm F2CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b01, "f2cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt2>;
+defm BF1CVT_ZZ    : sve2_fp8_cvt_single<0b0, 0b10, "bf1cvt",   nxv8bf16, int_aarch64_sve_fp8_cvt1>;
+defm BF2CVT_ZZ    : sve2_fp8_cvt_single<0b0, 0b11, "bf2cvt",   nxv8bf16, int_aarch64_sve_fp8_cvt2>;
+defm F1CVTLT_ZZ   : sve2_fp8_cvt_single<0b1, 0b00, "f1cvtlt",  nxv8f16,  int_aarch64_sve_fp8_cvtlt1>;
+defm F2CVTLT_ZZ   : sve2_fp8_cvt_single<0b1, 0b01, "f2cvtlt",  nxv8f16,  int_aarch64_sve_fp8_cvtlt2>;
+defm BF1CVTLT_ZZ  : sve2_fp8_cvt_single<0b1, 0b10, "bf1cvtlt", nxv8bf16, int_aarch64_sve_fp8_cvtlt1>;
+defm BF2CVTLT_ZZ  : sve2_fp8_cvt_single<0b1, 0b11, "bf2cvtlt", nxv8bf16, int_aarch64_sve_fp8_cvtlt2>;
 
 // FP8 downconvert
 defm FCVTN_Z2Z_HtoB  : sve2_fp8_down_cvt_single<0b00, "fcvtn", ZZ_h_mul_r>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index 3fd9dd3f2fff9..368665467859f 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -55,10 +55,10 @@ def V1UnitV0  : ProcResource<1>;  // FP/ASIMD 0
 def V1UnitV1  : ProcResource<1>;  // FP/ASIMD 1
 def V1UnitV2  : ProcResource<1>;  // FP/ASIMD 2
 def V1UnitV3  : ProcResource<1>;  // FP/ASIMD 3
+def V1UnitFlg : ProcResource<3>;  // Flags
 
 def V1UnitI   : ProcResGroup<[V1UnitS,
                               V1UnitM0, V1UnitM1]>;   // Integer units
-def V1UnitJ   : ProcResGroup<[V1UnitS, V1UnitM0]>;    // Integer 0-2 units
 def V1UnitM   : ProcResGroup<[V1UnitM0, V1UnitM1]>;   // Integer multicycle units
 def V1UnitL   : ProcResGroup<[V1UnitL01, V1UnitL2]>;  // Load units
 def V1UnitV   : ProcResGroup<[V1UnitV0, V1UnitV1,
@@ -98,13 +98,14 @@ def V1Write_0c_0Z : SchedWriteRes<[]>;
 
 def V1Write_1c_1B      : SchedWriteRes<[V1UnitB]>   { let Latency = 1; }
 def V1Write_1c_1I      : SchedWriteRes<[V1UnitI]>   { let Latency = 1; }
-def V1Write_1c_1J      : SchedWriteRes<[V1UnitJ]>   { let Latency = 1; }
+def V1Write_1c_1I_1Flg : SchedWriteRes<[V1UnitI, V1UnitFlg]>   { let Latency = 1; }
 def V1Write_4c_1L      : SchedWriteRes<[V1UnitL]>   { let Latency = 4; }
 def V1Write_6c_1L      : SchedWriteRes<[V1UnitL]>   { let Latency = 6; }
 def V1Write_1c_1L01    : SchedWriteRes<[V1UnitL01]> { let Latency = 1; }
 def V1Write_4c_1L01    : SchedWriteRes<[V1UnitL01]> { let Latency = 4; }
 def V1Write_6c_1L01    : SchedWriteRes<[V1UnitL01]> { let Latency = 6; }
 def V1Write_2c_1M      : SchedWriteRes<[V1UnitM]>   { let Latency = 2; }
+def V1Write_2c_1M_1Flg : SchedWriteRes<[V1UnitM, V1UnitFlg]>   { let Latency = 2; }
 def V1Write_3c_1M      : SchedWriteRes<[V1UnitM]>   { let Latency = 3; }
 def V1Write_4c_1M      : SchedWriteRes<[V1UnitM]>   { let Latency = 4; }
 def V1Write_1c_1M0     : SchedWriteRes<[V1UnitM0]>  { let Latency = 1; }
@@ -595,7 +596,7 @@ def : InstRW<[V1Write_1c_1B], (instregex "^[CT]BN?Z[XW]$")>;
 def : SchedAlias<WriteI, V1Write_1c_1I>;
 
 // ALU, basic, flagset
-def : InstRW<[V1Write_1c_1J],
+def : InstRW<[V1Write_1c_1I_1Flg],
              (instregex "^(ADD|SUB)S[WX]r[ir]$",
                         "^(ADC|SBC)S[WX]r$",
                         "^ANDS[WX]ri$",
@@ -614,8 +615,8 @@ def              : SchedAlias<WriteISReg, V1WriteISReg>;
 // Arithmetic, flagset, LSL shift, shift <= 4
 // Arithmetic, flagset, LSR/ASR/ROR shift or LSL shift > 4
 def V1WriteISRegS : SchedWriteVariant<
-                      [SchedVar<IsCheapLSL,  [V1Write_1c_1J]>,
-                       SchedVar<NoSchedPred, [V1Write_2c_1M]>]>;
+                      [SchedVar<IsCheapLSL,  [V1Write_1c_1I_1Flg]>,
+                       SchedVar<NoSchedPred, [V1Write_2c_1M_1Flg]>]>;
 def               : InstRW<[V1WriteISRegS],
                            (instregex "^(ADD|SUB)S(([WX]r[sx])|Xrx64)$")>;
 
@@ -623,10 +624,10 @@ def               : InstRW<[V1WriteISRegS],
 def : InstRW<[V1Write_1c_1I], (instregex "^(AND|BIC|EON|EOR|ORN|ORR)[WX]rs$")>;
 
 // Logical, shift, flagset
-def : InstRW<[V1Write_2c_1M], (instregex "^(AND|BIC)S[WX]rs$")>;
+def : InstRW<[V1Write_2c_1M_1Flg], (instregex "^(AND|BIC)S[WX]rs$")>;
 
 // Flag manipulation instructions
-def : InstRW<[V1Write_1c_1J], (instrs SETF8, SETF16, RMIF, CFINV)>;
+def : InstRW<[V1Write_1c_1I_1Flg], (instrs SETF8, SETF16, RMIF, CFINV)>;
 
 
 // Divide and multiply instructions
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
index 2de5f59834d78..39f7077ae4514 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
@@ -52,6 +52,7 @@ def V2UnitV3  : ProcResource<1>;  // FP/ASIMD 3
 def V2UnitL01 : ProcResource<2>;  // Load/Store 0/1
 def V2UnitL2  : ProcResource<1>;  // Load 2
 def V2UnitD   : ProcResource<2>;  // Store data 0/1
+def V2UnitFlg : ProcResource<3>;  // Flags
 
 def V2UnitR   : ProcResGroup<[V2UnitS0, V2UnitS1]>;  // Integer single-cycle 0/1
 def V2UnitS   : ProcResGroup<[V2UnitS0, V2UnitS1, V2UnitS2, V2UnitS3]>;  // Integer single-cycle 0/1/2/3
@@ -97,11 +98,13 @@ def V2Write_0c : SchedWriteRes<[]> { let Latency = 0; }
 
 def V2Write_1c_1B    : SchedWriteRes<[V2UnitB]>   { let Latency = 1; }
 def V2Write_1c_1F    : SchedWriteRes<[V2UnitF]>   { let Latency = 1; }
+def V2Write_1c_1F_1Flg : SchedWriteRes<[V2UnitF, V2UnitFlg]>   { let Latency = 1; }
 def V2Write_1c_1I    : SchedWriteRes<[V2UnitI]>   { let Latency = 1; }
 def V2Write_1c_1M    : SchedWriteRes<[V2UnitM]>   { let Latency = 1; }
 def V2Write_1c_1M0   : SchedWriteRes<[V2UnitM0]>  { let Latency = 1; }
 def V2Write_1c_1L01  : SchedWriteRes<[V2UnitL01]> { let Latency = 1; }
 def V2Write_2c_1M    : SchedWriteRes<[V2UnitM]>   { let Latency = 2; }
+def V2Write_2c_1M_1Flg : SchedWriteRes<[V2UnitM, V2UnitFlg]>   { let Latency = 2; }
 def V2Write_3c_1M    : SchedWriteRes<[V2UnitM]>   { let Latency = 3; }
 def V2Write_2c_1M0   : SchedWriteRes<[V2UnitM0]>  { let Latency = 2; }
 def V2Write_3c_1M0   : SchedWriteRes<[V2UnitM0]>  { let Latency = 3; }
@@ -886,12 +889,12 @@ def V2Write_ArithI : SchedWriteVariant<[
                        SchedVar<NoSchedPred, [V2Write_2c_1M]>]>;
 
 def V2Write_ArithF : SchedWriteVariant<[
-                       SchedVar<IsCheapLSL,  [V2Write_1c_1F]>,
-                       SchedVar<NoSchedPred, [V2Write_2c_1M]>]>;
+                       SchedVar<IsCheapLSL,  [V2Write_1c_1F_1Flg]>,
+                       SchedVar<NoSchedPred, [V2Write_2c_1M_1Flg]>]>;
 
 def V2Write_Logical : SchedWriteVariant<[
-                        SchedVar<NeoverseNoLSL, [V2Write_1c_1F]>,
-                        SchedVar<NoSchedPred,   [V2Write_2c_1M]>]>;
+                        SchedVar<NeoverseNoLSL, [V2Write_1c_1F_1Flg]>,
+                        SchedVar<NoSchedPred,   [V2Write_2c_1M_1Flg]>]>;
 
 def V2Write_Extr : SchedWriteVariant<[
                      SchedVar<IsRORImmIdiomPred, [V2Write_1c_1I]>,
@@ -1106,19 +1109,19 @@ def : InstRW<[V2Write_1c_1B_1R], (instrs BL, BLR)>;
 // -----------------------------------------------------------------------------
 
 // ALU, basic
-// ALU, basic, flagset
 def : SchedAlias<WriteI, V2Write_1c_1I>;
-def : InstRW<[V2Write_1c_1F], (instregex "^(ADD|SUB)S[WX]r[ir]$",
+
+// ALU, basic, flagset
+def : InstRW<[V2Write_1c_1F_1Flg],
+             (instregex "^(ADD|SUB)S[WX]r[ir]$",
                         "^(ADC|SBC)S[WX]r$",
-                        "^ANDS[WX]ri$")>;
+                        "^ANDS[WX]ri$",
+                        "^(AND|BIC)S[WX]rr$")>;
 def : InstRW<[V2Write_0or1c_1I], (instregex "^MOVZ[WX]i$")>;
 
 // ALU, extend and shift
 def : SchedAlias<WriteIEReg, V2Write_2c_1M>;
 
-// Conditional compare
-def : InstRW<[V2Write_1c_1F], (instregex "^CCM[NP][WX][ir]")>;
-
 // Arithmetic, LSL shift, shift <= 4
 // Arithmetic, flagset, LSL shift, shift <= 4
 // Arithmetic, LSR/ASR/ROR shift or LSL shift > 4
@@ -1129,6 +1132,9 @@ def : InstRW<[V2Write_ArithF],
 // Arithmetic, immediate to logical address tag
 def : InstRW<[V2Write_2c_1M], (instrs ADDG, SUBG)>;
 
+// Conditional compare
+def : InstRW<[V2Write_1c_1F_1Flg], (instregex "^CCM[NP][WX][ir]")>;
+
 // Convert floating-point condition flags
 // Flag manipulation instructions
 def : WriteRes<WriteSys, []> { let Latency = 1; }
@@ -1138,8 +1144,10 @@ def : InstRW<[V2Write_2c_1M], (instrs IRG, IRGstack)>;
 
 // Insert Tag Mask
 // Subtract Pointer
+def : InstRW<[V2Write_1c_1I], (instrs GMI, SUBP)>;
+
 // Subtract Pointer, flagset
-def : InstRW<[V2Write_1c_1I], (instrs GMI, SUBP, SUBPS)>;
+def : InstRW<[V2Write_1c_1F_1Flg], (instrs SUBPS)>;
 
 // Logical, shift, no flagset
 def : InstRW<[V2Write_1c_1I],    (instregex "^(AND|BIC|EON|EOR|ORN)[WX]rs$")>;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index e37e2cacc7852..3767b34bd5b0c 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -195,6 +195,9 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
     MaxPrefetchIterationsAhead = 4;
     VScaleForTuning = 4;
     break;
+  case MONAKA:
+    VScaleForTuning = 2;
+    break;
   case AppleA7:
   case AppleA10:
   case AppleA11:
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index d860c29e2291a..7b1f316d048e5 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -88,7 +88,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   unsigned StreamingHazardSize;
   unsigned MinSVEVectorSizeInBits;
   unsigned MaxSVEVectorSizeInBits;
-  unsigned VScaleForTuning = 2;
+  unsigned VScaleForTuning = 1;
   TailFoldingOpts DefaultSVETFOpts = TailFoldingOpts::Disabled;
 
   bool EnableSubregLiveness;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7a1e401bca18c..850a1c3bfca43 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -905,6 +905,23 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     }
     break;
   }
+  case Intrinsic::experimental_vector_match: {
+    auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
+    EVT SearchVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
+    unsigned SearchSize = NeedleTy->getNumElements();
+    if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize)) {
+      // Base cost for MATCH instructions. At least on the Neoverse V2 and
+      // Neoverse V3, these are cheap operations with the same latency as a
+      // vector ADD. In most cases, however, we also need to do an extra DUP.
+      // For fixed-length vectors we currently need an extra five--six
+      // instructions besides the MATCH.
+      InstructionCost Cost = 4;
+      if (isa<FixedVectorType>(RetTy))
+        Cost += 10;
+      return Cost;
+    }
+    break;
+  }
   default:
     break;
   }
@@ -2742,275 +2759,299 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     return AdjustCost(
         BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 
-  static const TypeConversionCostTblEntry
-  ConversionTbl[] = {
-    { ISD::TRUNCATE, MVT::v2i8,   MVT::v2i64,  1},  // xtn
-    { ISD::TRUNCATE, MVT::v2i16,  MVT::v2i64,  1},  // xtn
-    { ISD::TRUNCATE, MVT::v2i32,  MVT::v2i64,  1},  // xtn
-    { ISD::TRUNCATE, MVT::v4i8,   MVT::v4i32,  1},  // xtn
-    { ISD::TRUNCATE, MVT::v4i8,   MVT::v4i64,  3},  // 2 xtn + 1 uzp1
-    { ISD::TRUNCATE, MVT::v4i16,  MVT::v4i32,  1},  // xtn
-    { ISD::TRUNCATE, MVT::v4i16,  MVT::v4i64,  2},  // 1 uzp1 + 1 xtn
-    { ISD::TRUNCATE, MVT::v4i32,  MVT::v4i64,  1},  // 1 uzp1
-    { ISD::TRUNCATE, MVT::v8i8,   MVT::v8i16,  1},  // 1 xtn
-    { ISD::TRUNCATE, MVT::v8i8,   MVT::v8i32,  2},  // 1 uzp1 + 1 xtn
-    { ISD::TRUNCATE, MVT::v8i8,   MVT::v8i64,  4},  // 3 x uzp1 + xtn
-    { ISD::TRUNCATE, MVT::v8i16,  MVT::v8i32,  1},  // 1 uzp1
-    { ISD::TRUNCATE, MVT::v8i16,  MVT::v8i64,  3},  // 3 x uzp1
-    { ISD::TRUNCATE, MVT::v8i32,  MVT::v8i64,  2},  // 2 x uzp1
-    { ISD::TRUNCATE, MVT::v16i8,  MVT::v16i16, 1},  // uzp1
-    { ISD::TRUNCATE, MVT::v16i8,  MVT::v16i32, 3},  // (2 + 1) x uzp1
-    { ISD::TRUNCATE, MVT::v16i8,  MVT::v16i64, 7},  // (4 + 2 + 1) x uzp1
-    { ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, 2},  // 2 x uzp1
-    { ISD::TRUNCATE, MVT::v16i16, MVT::v16i64, 6},  // (4 + 2) x uzp1
-    { ISD::TRUNCATE, MVT::v16i32, MVT::v16i64, 4},  // 4 x uzp1
-
-    // Truncations on nxvmiN
-    { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i16, 1 },
-    { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i32, 1 },
-    { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i64, 1 },
-    { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i16, 1 },
-    { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i32, 1 },
-    { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i64, 2 },
-    { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i16, 1 },
-    { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i32, 3 },
-    { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i64, 5 },
-    { ISD::TRUNCATE, MVT::nxv16i1, MVT::nxv16i8, 1 },
-    { ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i32, 1 },
-    { ISD::TRUNCATE, MVT::nxv2i32, MVT::nxv2i64, 1 },
-    { ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i32, 1 },
-    { ISD::TRUNCATE, MVT::nxv4i32, MVT::nxv4i64, 2 },
-    { ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i32, 3 },
-    { ISD::TRUNCATE, MVT::nxv8i32, MVT::nxv8i64, 6 },
-
-    // The number of shll instructions for the extension.
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16, 3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i16, 3 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32, 2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32, 2 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,  3 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i8,  3 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16, 2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16, 2 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i8,  7 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i8,  7 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i16, 6 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i16, 6 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2 },
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
-
-    // LowerVectorINT_TO_FP:
-    { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 },
-    { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 },
-
-    // Complex: to v2f32
-    { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8,  3 },
-    { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 },
-    { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 },
-    { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8,  3 },
-    { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 },
-    { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 },
-
-    // Complex: to v4f32
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8,  4 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8,  3 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
-
-    // Complex: to v8f32
-    { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8,  10 },
-    { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
-    { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8,  10 },
-    { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
-
-    // Complex: to v16f32
-    { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 },
-    { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 },
-
-    // Complex: to v2f64
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8,  4 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8,  4 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
-
-    // Complex: to v4f64
-    { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32,  4 },
-    { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32,  4 },
-
-    // LowerVectorFP_TO_INT
-    { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1 },
-    { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1 },
-    { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, 1 },
-    { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1 },
-    { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, 1 },
-
-    // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
-    { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f32, 2 },
-    { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 1 },
-    { ISD::FP_TO_SINT, MVT::v2i8,  MVT::v2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f32, 2 },
-    { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::v2i8,  MVT::v2f32, 1 },
-
-    // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
-    { ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2 },
-    { ISD::FP_TO_SINT, MVT::v4i8,  MVT::v4f32, 2 },
-    { ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2 },
-    { ISD::FP_TO_UINT, MVT::v4i8,  MVT::v4f32, 2 },
-
-    // Complex, from nxv2f32.
-    { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i8,  MVT::nxv2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i8,  MVT::nxv2f32, 1 },
-
-    // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
-    { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2 },
-    { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f64, 2 },
-    { ISD::FP_TO_SINT, MVT::v2i8,  MVT::v2f64, 2 },
-    { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2 },
-    { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f64, 2 },
-    { ISD::FP_TO_UINT, MVT::v2i8,  MVT::v2f64, 2 },
-
-    // Complex, from nxv2f64.
-    { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i8,  MVT::nxv2f64, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i8,  MVT::nxv2f64, 1 },
-
-    // Complex, from nxv4f32.
-    { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4 },
-    { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv4i8,  MVT::nxv4f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4 },
-    { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i8,  MVT::nxv4f32, 1 },
-
-    // Complex, from nxv8f64. Illegal -> illegal conversions not required.
-    { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7 },
-    { ISD::FP_TO_SINT, MVT::nxv8i8,  MVT::nxv8f64, 7 },
-    { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f64, 7 },
-    { ISD::FP_TO_UINT, MVT::nxv8i8,  MVT::nxv8f64, 7 },
-
-    // Complex, from nxv4f64. Illegal -> illegal conversions not required.
-    { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f64, 3 },
-    { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f64, 3 },
-    { ISD::FP_TO_SINT, MVT::nxv4i8,  MVT::nxv4f64, 3 },
-    { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f64, 3 },
-    { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f64, 3 },
-    { ISD::FP_TO_UINT, MVT::nxv4i8,  MVT::nxv4f64, 3 },
-
-    // Complex, from nxv8f32. Illegal -> illegal conversions not required.
-    { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f32, 3 },
-    { ISD::FP_TO_SINT, MVT::nxv8i8,  MVT::nxv8f32, 3 },
-    { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f32, 3 },
-    { ISD::FP_TO_UINT, MVT::nxv8i8,  MVT::nxv8f32, 3 },
-
-    // Complex, from nxv8f16.
-    { ISD::FP_TO_SINT, MVT::nxv8i64, MVT::nxv8f16, 10 },
-    { ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4 },
-    { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv8i8,  MVT::nxv8f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10 },
-    { ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4 },
-    { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv8i8,  MVT::nxv8f16, 1 },
-
-    // Complex, from nxv4f16.
-    { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4 },
-    { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv4i8,  MVT::nxv4f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f16, 4 },
-    { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv4i8,  MVT::nxv4f16, 1 },
-
-    // Complex, from nxv2f16.
-    { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_SINT, MVT::nxv2i8,  MVT::nxv2f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f16, 1 },
-    { ISD::FP_TO_UINT, MVT::nxv2i8,  MVT::nxv2f16, 1 },
-
-    // Truncate from nxvmf32 to nxvmf16.
-    { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f32, 1 },
-    { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f32, 1 },
-    { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f32, 3 },
-
-    // Truncate from nxvmf64 to nxvmf16.
-    { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f64, 1 },
-    { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f64, 3 },
-    { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f64, 7 },
-
-    // Truncate from nxvmf64 to nxvmf32.
-    { ISD::FP_ROUND, MVT::nxv2f32, MVT::nxv2f64, 1 },
-    { ISD::FP_ROUND, MVT::nxv4f32, MVT::nxv4f64, 3 },
-    { ISD::FP_ROUND, MVT::nxv8f32, MVT::nxv8f64, 6 },
-
-    // Extend from nxvmf16 to nxvmf32.
-    { ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2f16, 1},
-    { ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4f16, 1},
-    { ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8f16, 2},
-
-    // Extend from nxvmf16 to nxvmf64.
-    { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f16, 1},
-    { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f16, 2},
-    { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f16, 4},
-
-    // Extend from nxvmf32 to nxvmf64.
-    { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f32, 1},
-    { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2},
-    { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6},
-
-    // Bitcasts from float to integer
-    { ISD::BITCAST, MVT::nxv2f16, MVT::nxv2i16, 0 },
-    { ISD::BITCAST, MVT::nxv4f16, MVT::nxv4i16, 0 },
-    { ISD::BITCAST, MVT::nxv2f32, MVT::nxv2i32, 0 },
-
-    // Bitcasts from integer to float
-    { ISD::BITCAST, MVT::nxv2i16, MVT::nxv2f16, 0 },
-    { ISD::BITCAST, MVT::nxv4i16, MVT::nxv4f16, 0 },
-    { ISD::BITCAST, MVT::nxv2i32, MVT::nxv2f32, 0 },
-
-    // Add cost for extending to illegal -too wide- scalable vectors.
-    // zero/sign extend are implemented by multiple unpack operations,
-    // where each operation has a cost of 1.
-    { ISD::ZERO_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
-    { ISD::ZERO_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
-    { ISD::ZERO_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
-    { ISD::ZERO_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
-    { ISD::ZERO_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
-    { ISD::ZERO_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
-
-    { ISD::SIGN_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
-    { ISD::SIGN_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
-    { ISD::SIGN_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
-    { ISD::SIGN_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
-    { ISD::SIGN_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
-    { ISD::SIGN_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
+  static const TypeConversionCostTblEntry ConversionTbl[] = {
+      {ISD::TRUNCATE, MVT::v2i8, MVT::v2i64, 1},    // xtn
+      {ISD::TRUNCATE, MVT::v2i16, MVT::v2i64, 1},   // xtn
+      {ISD::TRUNCATE, MVT::v2i32, MVT::v2i64, 1},   // xtn
+      {ISD::TRUNCATE, MVT::v4i8, MVT::v4i32, 1},    // xtn
+      {ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, 3},    // 2 xtn + 1 uzp1
+      {ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1},   // xtn
+      {ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, 2},   // 1 uzp1 + 1 xtn
+      {ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 1},   // 1 uzp1
+      {ISD::TRUNCATE, MVT::v8i8, MVT::v8i16, 1},    // 1 xtn
+      {ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 2},    // 1 uzp1 + 1 xtn
+      {ISD::TRUNCATE, MVT::v8i8, MVT::v8i64, 4},    // 3 x uzp1 + xtn
+      {ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 1},   // 1 uzp1
+      {ISD::TRUNCATE, MVT::v8i16, MVT::v8i64, 3},   // 3 x uzp1
+      {ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, 2},   // 2 x uzp1
+      {ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 1},  // uzp1
+      {ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 3},  // (2 + 1) x uzp1
+      {ISD::TRUNCATE, MVT::v16i8, MVT::v16i64, 7},  // (4 + 2 + 1) x uzp1
+      {ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, 2}, // 2 x uzp1
+      {ISD::TRUNCATE, MVT::v16i16, MVT::v16i64, 6}, // (4 + 2) x uzp1
+      {ISD::TRUNCATE, MVT::v16i32, MVT::v16i64, 4}, // 4 x uzp1
+
+      // Truncations on nxvmiN
+      {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i16, 1},
+      {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i32, 1},
+      {ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i64, 1},
+      {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i16, 1},
+      {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i32, 1},
+      {ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i64, 2},
+      {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i16, 1},
+      {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i32, 3},
+      {ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i64, 5},
+      {ISD::TRUNCATE, MVT::nxv16i1, MVT::nxv16i8, 1},
+      {ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i32, 1},
+      {ISD::TRUNCATE, MVT::nxv2i32, MVT::nxv2i64, 1},
+      {ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i32, 1},
+      {ISD::TRUNCATE, MVT::nxv4i32, MVT::nxv4i64, 2},
+      {ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i32, 3},
+      {ISD::TRUNCATE, MVT::nxv8i32, MVT::nxv8i64, 6},
+
+      // The number of shll instructions for the extension.
+      {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3},
+      {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3},
+      {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2},
+      {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2},
+      {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3},
+      {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3},
+      {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2},
+      {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2},
+      {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7},
+      {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7},
+      {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6},
+      {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6},
+      {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2},
+      {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2},
+      {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6},
+      {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6},
+
+      // FP Ext and trunc
+      {ISD::FP_EXTEND, MVT::f64, MVT::f32, 1},     // fcvt
+      {ISD::FP_EXTEND, MVT::v2f64, MVT::v2f32, 1}, // fcvtl
+      {ISD::FP_EXTEND, MVT::v4f64, MVT::v4f32, 2}, // fcvtl+fcvtl2
+      //   FP16
+      {ISD::FP_EXTEND, MVT::f32, MVT::f16, 1},     // fcvt
+      {ISD::FP_EXTEND, MVT::f64, MVT::f16, 1},     // fcvt
+      {ISD::FP_EXTEND, MVT::v4f32, MVT::v4f16, 1}, // fcvtl
+      {ISD::FP_EXTEND, MVT::v8f32, MVT::v8f16, 2}, // fcvtl+fcvtl2
+      {ISD::FP_EXTEND, MVT::v2f64, MVT::v2f16, 2}, // fcvtl+fcvtl
+      {ISD::FP_EXTEND, MVT::v4f64, MVT::v4f16, 3}, // fcvtl+fcvtl2+fcvtl
+      {ISD::FP_EXTEND, MVT::v8f64, MVT::v8f16, 6}, // 2 * fcvtl+fcvtl2+fcvtl
+      // FP Ext and trunc
+      {ISD::FP_ROUND, MVT::f32, MVT::f64, 1},     // fcvt
+      {ISD::FP_ROUND, MVT::v2f32, MVT::v2f64, 1}, // fcvtn
+      {ISD::FP_ROUND, MVT::v4f32, MVT::v4f64, 2}, // fcvtn+fcvtn2
+      //   FP16
+      {ISD::FP_ROUND, MVT::f16, MVT::f32, 1},     // fcvt
+      {ISD::FP_ROUND, MVT::f16, MVT::f64, 1},     // fcvt
+      {ISD::FP_ROUND, MVT::v4f16, MVT::v4f32, 1}, // fcvtn
+      {ISD::FP_ROUND, MVT::v8f16, MVT::v8f32, 2}, // fcvtn+fcvtn2
+      {ISD::FP_ROUND, MVT::v2f16, MVT::v2f64, 2}, // fcvtn+fcvtn
+      {ISD::FP_ROUND, MVT::v4f16, MVT::v4f64, 3}, // fcvtn+fcvtn2+fcvtn
+      {ISD::FP_ROUND, MVT::v8f16, MVT::v8f64, 6}, // 2 * fcvtn+fcvtn2+fcvtn
+
+      // LowerVectorINT_TO_FP:
+      {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
+      {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
+      {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 1},
+      {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
+      {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
+      {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1},
+
+      // Complex: to v2f32
+      {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
+      {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3},
+      {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, 2},
+      {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
+      {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3},
+      {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, 2},
+
+      // Complex: to v4f32
+      {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 4},
+      {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
+      {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3},
+      {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
+
+      // Complex: to v8f32
+      {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
+      {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4},
+      {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
+      {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4},
+
+      // Complex: to v16f32
+      {ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21},
+      {ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21},
+
+      // Complex: to v2f64
+      {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4},
+      {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 4},
+      {ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2},
+      {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4},
+      {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4},
+      {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2},
+
+      // Complex: to v4f64
+      {ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, 4},
+      {ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, 4},
+
+      // LowerVectorFP_TO_INT
+      {ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1},
+      {ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1},
+      {ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, 1},
+      {ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f32, 1},
+      {ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1},
+      {ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, 1},
+
+      // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
+      {ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f32, 2},
+      {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 1},
+      {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 1},
+      {ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f32, 2},
+      {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 1},
+      {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 1},
+
+      // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
+      {ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2},
+      {ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 2},
+      {ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2},
+      {ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 2},
+
+      // Complex, from nxv2f32.
+      {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f32, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f32, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f32, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f32, 1},
+
+      // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
+      {ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2},
+      {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f64, 2},
+      {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f64, 2},
+      {ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2},
+      {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f64, 2},
+      {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f64, 2},
+
+      // Complex, from nxv2f64.
+      {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f64, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f64, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f64, 1},
+
+      // Complex, from nxv4f32.
+      {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4},
+      {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1},
+      {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1},
+      {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4},
+      {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f32, 1},
+
+      // Complex, from nxv8f64. Illegal -> illegal conversions not required.
+      {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7},
+      {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f64, 7},
+      {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f64, 7},
+      {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f64, 7},
+
+      // Complex, from nxv4f64. Illegal -> illegal conversions not required.
+      {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f64, 3},
+      {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f64, 3},
+      {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f64, 3},
+      {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f64, 3},
+      {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f64, 3},
+      {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f64, 3},
+
+      // Complex, from nxv8f32. Illegal -> illegal conversions not required.
+      {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f32, 3},
+      {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f32, 3},
+      {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f32, 3},
+      {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f32, 3},
+
+      // Complex, from nxv8f16.
+      {ISD::FP_TO_SINT, MVT::nxv8i64, MVT::nxv8f16, 10},
+      {ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4},
+      {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10},
+      {ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4},
+      {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f16, 1},
+
+      // Complex, from nxv4f16.
+      {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4},
+      {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f16, 4},
+      {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f16, 1},
+
+      // Complex, from nxv2f16.
+      {ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f16, 1},
+      {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f16, 1},
+      {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f16, 1},
+
+      // Truncate from nxvmf32 to nxvmf16.
+      {ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f32, 1},
+      {ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f32, 1},
+      {ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f32, 3},
+
+      // Truncate from nxvmf64 to nxvmf16.
+      {ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f64, 1},
+      {ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f64, 3},
+      {ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f64, 7},
+
+      // Truncate from nxvmf64 to nxvmf32.
+      {ISD::FP_ROUND, MVT::nxv2f32, MVT::nxv2f64, 1},
+      {ISD::FP_ROUND, MVT::nxv4f32, MVT::nxv4f64, 3},
+      {ISD::FP_ROUND, MVT::nxv8f32, MVT::nxv8f64, 6},
+
+      // Extend from nxvmf16 to nxvmf32.
+      {ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2f16, 1},
+      {ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4f16, 1},
+      {ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8f16, 2},
+
+      // Extend from nxvmf16 to nxvmf64.
+      {ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f16, 1},
+      {ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f16, 2},
+      {ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f16, 4},
+
+      // Extend from nxvmf32 to nxvmf64.
+      {ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f32, 1},
+      {ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2},
+      {ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6},
+
+      // Bitcasts from float to integer
+      {ISD::BITCAST, MVT::nxv2f16, MVT::nxv2i16, 0},
+      {ISD::BITCAST, MVT::nxv4f16, MVT::nxv4i16, 0},
+      {ISD::BITCAST, MVT::nxv2f32, MVT::nxv2i32, 0},
+
+      // Bitcasts from integer to float
+      {ISD::BITCAST, MVT::nxv2i16, MVT::nxv2f16, 0},
+      {ISD::BITCAST, MVT::nxv4i16, MVT::nxv4f16, 0},
+      {ISD::BITCAST, MVT::nxv2i32, MVT::nxv2f32, 0},
+
+      // Add cost for extending to illegal -too wide- scalable vectors.
+      // zero/sign extend are implemented by multiple unpack operations,
+      // where each operation has a cost of 1.
+      {ISD::ZERO_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
+      {ISD::ZERO_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
+      {ISD::ZERO_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
+      {ISD::ZERO_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
+      {ISD::ZERO_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
+      {ISD::ZERO_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
+
+      {ISD::SIGN_EXTEND, MVT::nxv16i16, MVT::nxv16i8, 2},
+      {ISD::SIGN_EXTEND, MVT::nxv16i32, MVT::nxv16i8, 6},
+      {ISD::SIGN_EXTEND, MVT::nxv16i64, MVT::nxv16i8, 14},
+      {ISD::SIGN_EXTEND, MVT::nxv8i32, MVT::nxv8i16, 2},
+      {ISD::SIGN_EXTEND, MVT::nxv8i64, MVT::nxv8i16, 6},
+      {ISD::SIGN_EXTEND, MVT::nxv4i64, MVT::nxv4i32, 2},
   };
 
   // We have to estimate a cost of fixed length operation upon
@@ -3022,8 +3063,8 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
       ST->useSVEForFixedLengthVectors(WiderTy)) {
     std::pair<InstructionCost, MVT> LT =
         getTypeLegalizationCost(WiderTy.getTypeForEVT(Dst->getContext()));
-    unsigned NumElements = AArch64::SVEBitsPerBlock /
-                           LT.second.getScalarSizeInBits();
+    unsigned NumElements =
+        AArch64::SVEBitsPerBlock / LT.second.getScalarSizeInBits();
     return AdjustCost(
         LT.first *
         getCastInstrCost(
@@ -3032,9 +3073,8 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
             CostKind, I));
   }
 
-  if (const auto *Entry = ConvertCostTableLookup(ConversionTbl, ISD,
-                                                 DstTy.getSimpleVT(),
-                                                 SrcTy.getSimpleVT()))
+  if (const auto *Entry = ConvertCostTableLookup(
+          ConversionTbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
     return AdjustCost(Entry->Cost);
 
   static const TypeConversionCostTblEntry FP16Tbl[] = {
@@ -3248,19 +3288,18 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
     // Check if the extractelement user is scalar fmul.
     auto IsUserFMulScalarTy = [](const Value *EEUser) {
       // Check if the user is scalar fmul.
-      const auto *BO = dyn_cast_if_present<BinaryOperator>(EEUser);
+      const auto *BO = dyn_cast<BinaryOperator>(EEUser);
       return BO && BO->getOpcode() == BinaryOperator::FMul &&
              !BO->getType()->isVectorTy();
     };
 
     // Check if the extract index is from lane 0 or lane equivalent to 0 for a
     // certain scalar type and a certain vector register width.
-    auto IsExtractLaneEquivalentToZero = [&](const unsigned &Idx,
-                                             const unsigned &EltSz) {
+    auto IsExtractLaneEquivalentToZero = [&](unsigned Idx, unsigned EltSz) {
       auto RegWidth =
           getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
               .getFixedValue();
-      return (Idx == 0 || (Idx * EltSz) % RegWidth == 0);
+      return Idx == 0 || (RegWidth != 0 && (Idx * EltSz) % RegWidth == 0);
     };
 
     // Check if the type constraints on input vector type and result scalar type
@@ -3277,13 +3316,15 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
         // important.
         UserToExtractIdx[U];
       }
+      if (UserToExtractIdx.empty())
+        return false;
       for (auto &[S, U, L] : ScalarUserAndIdx) {
         for (auto *U : S->users()) {
           if (UserToExtractIdx.find(U) != UserToExtractIdx.end()) {
             auto *FMul = cast<BinaryOperator>(U);
             auto *Op0 = FMul->getOperand(0);
             auto *Op1 = FMul->getOperand(1);
-            if ((Op0 == S && Op1 == S) || (Op0 != S) || (Op1 != S)) {
+            if ((Op0 == S && Op1 == S) || Op0 != S || Op1 != S) {
               UserToExtractIdx[U] = L;
               break;
             }
@@ -3362,7 +3403,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
 
 InstructionCost AArch64TTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
   if (isa<ScalableVectorType>(Ty))
     return InstructionCost::getInvalid();
   if (Ty->getElementType()->isFloatingPointTy())
@@ -3988,6 +4029,92 @@ getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   }
 }
 
+/// For Apple CPUs, we want to runtime-unroll loops to make better use if the
+/// OOO engine's wide instruction window and various predictors.
+static void
+getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
+                                 TargetTransformInfo::UnrollingPreferences &UP,
+                                 AArch64TTIImpl &TTI) {
+  // Limit loops with structure that is highly likely to benefit from runtime
+  // unrolling; that is we exclude outer loops, loops with multiple exits and
+  // many blocks (i.e. likely with complex control flow). Note that the
+  // heuristics here may be overly conservative and we err on the side of
+  // avoiding runtime unrolling rather than unroll excessively. They are all
+  // subject to further refinement.
+  if (!L->isInnermost() || !L->getExitBlock() || L->getNumBlocks() > 8)
+    return;
+
+  const SCEV *BTC = SE.getBackedgeTakenCount(L);
+  if (isa<SCEVConstant>(BTC) || isa<SCEVCouldNotCompute>(BTC) ||
+      (SE.getSmallConstantMaxTripCount(L) > 0 &&
+       SE.getSmallConstantMaxTripCount(L) <= 32))
+    return;
+  if (findStringMetadataForLoop(L, "llvm.loop.isvectorized"))
+    return;
+
+  int64_t Size = 0;
+  for (auto *BB : L->getBlocks()) {
+    for (auto &I : *BB) {
+      if (!isa<IntrinsicInst>(&I) && isa<CallBase>(&I))
+        return;
+      SmallVector<const Value *, 4> Operands(I.operand_values());
+      Size +=
+          *TTI.getInstructionCost(&I, Operands, TTI::TCK_CodeSize).getValue();
+    }
+  }
+
+  // Limit to loops with trip counts that are cheap to expand.
+  UP.SCEVExpansionBudget = 1;
+
+  // Try to unroll small, single block loops, if they have load/store
+  // dependencies, to expose more parallel memory access streams.
+  if (L->getHeader() != L->getLoopLatch() || Size > 8)
+    return;
+
+  SmallPtrSet<Value *, 8> LoadedValues;
+  SmallVector<StoreInst *> Stores;
+  for (auto *BB : L->blocks()) {
+    for (auto &I : *BB) {
+      Value *Ptr = getLoadStorePointerOperand(&I);
+      if (!Ptr)
+        continue;
+      const SCEV *PtrSCEV = SE.getSCEV(Ptr);
+      if (SE.isLoopInvariant(PtrSCEV, L))
+        continue;
+      if (isa<LoadInst>(&I))
+        LoadedValues.insert(&I);
+      else
+        Stores.push_back(cast<StoreInst>(&I));
+    }
+  }
+
+  // Try to find an unroll count that maximizes the use of the instruction
+  // window, i.e. trying to fetch as many instructions per cycle as possible.
+  unsigned MaxInstsPerLine = 16;
+  unsigned UC = 1;
+  unsigned BestUC = 1;
+  unsigned SizeWithBestUC = BestUC * Size;
+  while (UC <= 8) {
+    unsigned SizeWithUC = UC * Size;
+    if (SizeWithUC > 48)
+      break;
+    if ((SizeWithUC % MaxInstsPerLine) == 0 ||
+        (SizeWithBestUC % MaxInstsPerLine) < (SizeWithUC % MaxInstsPerLine)) {
+      BestUC = UC;
+      SizeWithBestUC = BestUC * Size;
+    }
+    UC++;
+  }
+
+  if (BestUC == 1 || none_of(Stores, [&LoadedValues](StoreInst *SI) {
+        return LoadedValues.contains(SI->getOperand(0));
+      }))
+    return;
+
+  UP.Runtime = true;
+  UP.DefaultUnrollRuntimeCount = BestUC;
+}
+
 void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                              TTI::UnrollingPreferences &UP,
                                              OptimizationRemarkEmitter *ORE) {
@@ -4005,9 +4132,21 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   // Disable partial & runtime unrolling on -Os.
   UP.PartialOptSizeThreshold = 0;
 
-  if (ST->getProcFamily() == AArch64Subtarget::Falkor &&
-      EnableFalkorHWPFUnrollFix)
-    getFalkorUnrollingPreferences(L, SE, UP);
+  // Apply subtarget-specific unrolling preferences.
+  switch (ST->getProcFamily()) {
+  case AArch64Subtarget::AppleA14:
+  case AArch64Subtarget::AppleA15:
+  case AArch64Subtarget::AppleA16:
+  case AArch64Subtarget::AppleM4:
+    getAppleRuntimeUnrollPreferences(L, SE, UP, *this);
+    break;
+  case AArch64Subtarget::Falkor:
+    if (EnableFalkorHWPFUnrollFix)
+      getFalkorUnrollingPreferences(L, SE, UP);
+    break;
+  default:
+    break;
+  }
 
   // Scan the loop: don't unroll loops with calls as this could prevent
   // inlining. Don't unroll vector loops either, as they don't benefit much from
@@ -4796,14 +4935,20 @@ AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
 }
 
 bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) {
-  // For the binary operators (e.g. or) we need to be more careful than
-  // selects, here we only transform them if they are already at a natural
-  // break point in the code - the end of a block with an unconditional
-  // terminator.
-  if (EnableOrLikeSelectOpt && I->getOpcode() == Instruction::Or &&
-      isa<BranchInst>(I->getNextNode()) &&
-      cast<BranchInst>(I->getNextNode())->isUnconditional())
-    return true;
+  if (EnableOrLikeSelectOpt) {
+    // For the binary operators (e.g. or) we need to be more careful than
+    // selects, here we only transform them if they are already at a natural
+    // break point in the code - the end of a block with an unconditional
+    // terminator.
+    if (I->getOpcode() == Instruction::Or &&
+        isa<BranchInst>(I->getNextNode()) &&
+        cast<BranchInst>(I->getNextNode())->isUnconditional())
+      return true;
+
+    if (I->getOpcode() == Instruction::Add ||
+        I->getOpcode() == Instruction::Sub)
+      return true;
+  }
   return BaseT::shouldTreatInstructionLikeSelect(I);
 }
 
@@ -5163,26 +5308,45 @@ bool AArch64TTIImpl::isProfitableToSinkOperands(
     return false;
   }
   case Instruction::Mul: {
+    auto ShouldSinkSplatForIndexedVariant = [](Value *V) {
+      auto *Ty = cast<VectorType>(V->getType());
+      // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
+      if (Ty->isScalableTy())
+        return false;
+
+      // Indexed variants of Mul exist for i16 and i32 element types only.
+      return Ty->getScalarSizeInBits() == 16 || Ty->getScalarSizeInBits() == 32;
+    };
+
     int NumZExts = 0, NumSExts = 0;
     for (auto &Op : I->operands()) {
       // Make sure we are not already sinking this operand
       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
         continue;
 
-      if (match(&Op, m_SExt(m_Value()))) {
-        NumSExts++;
-        continue;
-      } else if (match(&Op, m_ZExt(m_Value()))) {
-        NumZExts++;
+      if (match(&Op, m_ZExtOrSExt(m_Value()))) {
+        auto *Ext = cast<Instruction>(Op);
+        auto *ExtOp = Ext->getOperand(0);
+        if (isSplatShuffle(ExtOp) && ShouldSinkSplatForIndexedVariant(ExtOp))
+          Ops.push_back(&Ext->getOperandUse(0));
+        Ops.push_back(&Op);
+
+        if (isa<SExtInst>(Ext))
+          NumSExts++;
+        else
+          NumZExts++;
+
         continue;
       }
 
       ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Op);
+      if (!Shuffle)
+        continue;
 
       // If the Shuffle is a splat and the operand is a zext/sext, sinking the
       // operand and the s/zext can help create indexed s/umull. This is
       // especially useful to prevent i64 mul being scalarized.
-      if (Shuffle && isSplatShuffle(Shuffle) &&
+      if (isSplatShuffle(Shuffle) &&
           match(Shuffle->getOperand(0), m_ZExtOrSExt(m_Value()))) {
         Ops.push_back(&Shuffle->getOperandUse(0));
         Ops.push_back(&Op);
@@ -5193,9 +5357,6 @@ bool AArch64TTIImpl::isProfitableToSinkOperands(
         continue;
       }
 
-      if (!Shuffle)
-        continue;
-
       Value *ShuffleOperand = Shuffle->getOperand(0);
       InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
       if (!Insert)
@@ -5227,12 +5388,26 @@ bool AArch64TTIImpl::isProfitableToSinkOperands(
         NumZExts++;
       }
 
+      Ops.push_back(&Insert->getOperandUse(1));
       Ops.push_back(&Shuffle->getOperandUse(0));
       Ops.push_back(&Op);
     }
 
-    // Is it profitable to sink if we found two of the same type of extends.
-    return !Ops.empty() && (NumSExts == 2 || NumZExts == 2);
+    // It is profitable to sink if we found two of the same type of extends.
+    if (!Ops.empty() && (NumSExts == 2 || NumZExts == 2))
+      return true;
+
+    // Otherwise, see if we should sink splats for indexed variants.
+    if (!ShouldSinkSplatForIndexedVariant(I))
+      return false;
+
+    Ops.clear();
+    if (isSplatShuffle(I->getOperand(0)))
+      Ops.push_back(&I->getOperandUse(0));
+    if (isSplatShuffle(I->getOperand(1)))
+      Ops.push_back(&I->getOperandUse(1));
+
+    return !Ops.empty();
   }
   case Instruction::FMul: {
     // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 201bc831b816b..83b86e31565e4 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -423,7 +423,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {});
 
   /// Return the cost of the scaling factor used in the addressing
   /// mode represented by AM for this target, for a load/store
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index d8bd14f97980a..93c85ba62f90e 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3025,7 +3025,7 @@ unsigned AArch64AsmParser::matchRegisterNameAlias(StringRef Name,
     return Kind == RegKind::Matrix ? RegNum : 0;
 
  if (Name.equals_insensitive("zt0"))
-    return Kind == RegKind::LookupTable ? AArch64::ZT0 : 0;
+    return Kind == RegKind::LookupTable ? unsigned(AArch64::ZT0) : 0;
 
   // The parsed register must be of RegKind Scalar
   if ((RegNum = MatchRegisterName(Name)))
@@ -3354,7 +3354,13 @@ ParseStatus AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADR relocation (unfortunately).
       Expr = AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, getContext());
-    } else {
+    } else if (ELFRefKind != AArch64MCExpr::VK_GOT_AUTH_PAGE) {
+      // For tiny code model, we use :got_auth: operator to fill 21-bit imm of
+      // adr. It's not actually GOT entry page address but the GOT address
+      // itself - we just share the same variant kind with :got_auth: operator
+      // applied for adrp.
+      // TODO: can we somehow get current TargetMachine object to call
+      // getCodeModel() on it to ensure we are using tiny code model?
       return Error(S, "unexpected adr label");
     }
   }
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index ad31f29c04599..619a041c273cd 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1727,9 +1727,9 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   case Intrinsic::aarch64_neon_fminnm:
     return LowerBinOp(TargetOpcode::G_FMINNUM);
   case Intrinsic::aarch64_neon_smull:
-    return LowerBinOp(AArch64::G_UMULL);
-  case Intrinsic::aarch64_neon_umull:
     return LowerBinOp(AArch64::G_SMULL);
+  case Intrinsic::aarch64_neon_umull:
+    return LowerBinOp(AArch64::G_UMULL);
   case Intrinsic::aarch64_neon_abs: {
     // Lower the intrinsic to G_ABS.
     MachineIRBuilder MIB(MI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index b5f5a58d96288..04f90e21bb3b1 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -147,6 +147,15 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
       }
       return ELF::R_AARCH64_PREL64;
     case AArch64::fixup_aarch64_pcrel_adr_imm21:
+      if (SymLoc == AArch64MCExpr::VK_GOT_AUTH) {
+        if (IsILP32) {
+          Ctx.reportError(Fixup.getLoc(),
+                          "ILP32 ADR AUTH relocation not supported "
+                          "(LP64 eqv: AUTH_GOT_ADR_PREL_LO21)");
+          return ELF::R_AARCH64_NONE;
+        }
+        return ELF::R_AARCH64_AUTH_GOT_ADR_PREL_LO21;
+      }
       if (SymLoc != AArch64MCExpr::VK_ABS)
         Ctx.reportError(Fixup.getLoc(),
                         "invalid symbol kind for ADR relocation");
@@ -190,6 +199,15 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
         return R_CLS(TLSIE_LD_GOTTPREL_PREL19);
       if (SymLoc == AArch64MCExpr::VK_GOT)
         return R_CLS(GOT_LD_PREL19);
+      if (SymLoc == AArch64MCExpr::VK_GOT_AUTH) {
+        if (IsILP32) {
+          Ctx.reportError(Fixup.getLoc(),
+                          "ILP32 LDR AUTH relocation not supported "
+                          "(LP64 eqv: AUTH_GOT_LD_PREL19)");
+          return ELF::R_AARCH64_NONE;
+        }
+        return ELF::R_AARCH64_AUTH_GOT_LD_PREL19;
+      }
       return R_CLS(LD_PREL_LO19);
     case AArch64::fixup_aarch64_pcrel_branch14:
       return R_CLS(TSTBR14);
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 8c256b5818ee8..e6535f957e202 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -305,6 +305,21 @@ multiclass sme_outer_product_fp32<bit S, bits<2> sz, ZPRRegOp zpr_ty, string mne
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv4i1, nxv4f32>;
 }
 
+multiclass sme2_fp8_fmopa_za32<string mnemonic, SDPatternOperator intrinsic> {
+    def NAME : sme_fp_outer_product_inst<0, 0b01, 0b00, TileOp32, ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+      bits<2> ZAda;
+      let Inst{1-0} = ZAda;
+      let Inst{2}   = 0b0;
+
+      let Uses = [FPMR, FPCR];
+    }
+
+    let mayStore = 1, mayLoad = 1 in
+    def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
+
+    def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv16i1, nxv16i8>;
+}
+
 multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> {
   def NAME : sme_fp_outer_product_inst<S, 0b10, 0b00, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {
     bits<3> ZAda;
@@ -316,12 +331,19 @@ multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op>
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>;
 }
 
-multiclass sme2p1_fmop_tile_f8f16<string mnemonic, bit bf, bit s, bits<2> op> {
-  def NAME : sme_fp_outer_product_inst<s, {0,bf}, op, TileOp16, ZPR8, mnemonic> {
+multiclass sme2_fp8_fmopa_za16<string mnemonic, SDPatternOperator intrinsic> {
+  def NAME : sme_fp_outer_product_inst<0, {0, 0b1}, 0b01, TileOp16, ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {
     bits<1> ZAda;
     let Inst{2-1} = 0b00;
     let Inst{0}   = ZAda;
+
+    let Uses = [FPMR, FPCR];
   }
+
+  let mayStore = 1, mayLoad = 1 in
+  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
+
+  def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_1, nxv16i1, nxv16i8>;
 }
 
 multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, ValueType vt, SDPatternOperator intrinsic = null_frag> {
@@ -2412,7 +2434,7 @@ multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> {
 
 // SME2 multi-vec FP8 up convert two registers
 multiclass sme2p1_fp8_cvt_vector_vg2_single<string mnemonic, bits<2> opc, bit L> {
-  def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>{
+  def NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>{
     let Uses = [FPMR, FPCR];
   }
 }
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 4bdf327e0d3fc..9ae66518dfb4e 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -482,6 +482,8 @@ let Predicates = [HasSVEorSME] in {
 //===----------------------------------------------------------------------===//
 // SVE pattern match helpers.
 //===----------------------------------------------------------------------===//
+def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
+def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;
 
 class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    Instruction inst>
@@ -502,6 +504,11 @@ multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, Value
             (inst $Op3, $Op1, $Op2)>;
 }
 
+class SVE_1_Op_PassthruUndefZero_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
+                   ValueType vts, Instruction inst>
+  : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd (SVEDup0Undef)))),
+        (inst $Op1, $Op2)>;
+
 // Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the
 // type of rounding. This is matched by timm0_1 in pattern below and ignored.
 class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
@@ -517,8 +524,6 @@ multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op,
             (inst $Op3, $Op1, $Op2)>;
 }
 
-def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
-
 class SVE_1_Op_PassthruZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    ValueType vt2, Instruction inst>
    : Pat<(vtd (op (vtd (SVEDup0)), vt1:$Op1, vt2:$Op2)),
@@ -571,6 +576,11 @@ multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1
             (inst $Op1, $Op2, $Op3)>;
 }
 
+class SVE_3_Op_UndefZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
+                             ValueType vt2, ValueType vt3, Instruction inst>
+  : Pat<(vtd (op (vt1 (SVEDup0Undef)), vt2:$Op1, vt3:$Op2)),
+        (inst $Op1, $Op2)>;
+
 class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    ValueType vt2, ValueType vt3, ValueType vt4,
                    Instruction inst>
@@ -606,8 +616,6 @@ class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
 : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),
       (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;
 
-def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;
-
 let AddedComplexity = 1 in {
 class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    ValueType vt2, ValueType vt3, Instruction inst>
@@ -2850,9 +2858,12 @@ multiclass sve_fp_fcvtntz<string asm> {
   def _DtoS : sve_fp_fcvt2z<0b1110, asm,  ZPR32, ZPR64>;
 }
 
-multiclass sve_fp_fcvtltz<string asm> {
+multiclass sve_fp_fcvtltz<string asm, string op> {
   def _HtoS  : sve_fp_fcvt2z<0b1001, asm,  ZPR32, ZPR16>;
   def _StoD  : sve_fp_fcvt2z<0b1111, asm,  ZPR64, ZPR32>;
+
+  def : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -3259,6 +3270,12 @@ class sve_fp_z2op_p_zd<bits<7> opc,string asm, RegisterOperand i_zprtype,
   let mayRaiseFPException = 1;
 }
 
+multiclass sve_fp_z2op_p_zd<string asm, SDPatternOperator op> {
+  def _DtoS : sve_fp_z2op_p_zd<0b0001010, asm, ZPR64, ZPR32>;
+
+  def : SVE_3_Op_UndefZero_Pat<nxv4f32, op, nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
+}
+
 multiclass sve_fp_z2op_p_zd_hsd<bits<5> opc, string asm> {
   def _H : sve_fp_z2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16>;
   def _S : sve_fp_z2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32>;
@@ -3270,7 +3287,13 @@ multiclass sve_fp_z2op_p_zd_frint<bits<2> opc, string asm> {
   def _D : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64>;
 }
 
-multiclass sve_fp_z2op_p_zd_d<bit U, string asm> {
+multiclass sve_fp_z2op_p_zd_bfcvt<string asm, SDPatternOperator op> {
+  def NAME : sve_fp_z2op_p_zd<0b1001010, asm, ZPR32, ZPR16>;
+
+  def : SVE_3_Op_UndefZero_Pat<nxv8bf16, op, nxv8bf16, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;
+}
+
+multiclass sve_fp_z2op_p_zd_d<bit U, string asm, string int_op, SDPatternOperator ir_op> {
   def _HtoH : sve_fp_z2op_p_zd<{ 0b011101, U }, asm, ZPR16, ZPR16>;
   def _HtoS : sve_fp_z2op_p_zd<{ 0b011110, U }, asm, ZPR16, ZPR32>;
   def _HtoD : sve_fp_z2op_p_zd<{ 0b011111, U }, asm, ZPR16, ZPR64>;
@@ -3278,6 +3301,15 @@ multiclass sve_fp_z2op_p_zd_d<bit U, string asm> {
   def _StoD : sve_fp_z2op_p_zd<{ 0b111110, U }, asm, ZPR32, ZPR64>;
   def _DtoS : sve_fp_z2op_p_zd<{ 0b111100, U }, asm, ZPR64, ZPR32>;
   def _DtoD : sve_fp_z2op_p_zd<{ 0b111111, U }, asm, ZPR64, ZPR64>;
+
+  def : SVE_3_Op_UndefZero_Pat<nxv4i32, !cast<SDPatternOperator>(int_op # _i32f64), nxv4i32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv2i64, !cast<SDPatternOperator>(int_op # _i64f32), nxv2i64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv4i32, !cast<SDPatternOperator>(int_op # _i32f16), nxv4i32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv2i64, !cast<SDPatternOperator>(int_op # _i64f16), nxv2i64, nxv2i1, nxv8f16, !cast<Instruction>(NAME # _HtoD)>;
+
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, ir_op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _HtoH)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, ir_op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoS)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, ir_op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoD)>;
 }
 
 multiclass sve_fp_z2op_p_zd_c<bit U, string asm> {
@@ -3296,13 +3328,20 @@ multiclass sve_fp_z2op_p_zd_d_flogb<string asm> {
   def _D : sve_fp_z2op_p_zd<0b0011011, asm, ZPR64, ZPR64>;
 }
 
-multiclass sve_fp_z2op_p_zd_b_0<string asm> {
+multiclass sve_fp_z2op_p_zd_b_0<string asm, string op> {
   def _StoH : sve_fp_z2op_p_zd<0b1001000, asm, ZPR32, ZPR16>;
   def _HtoS : sve_fp_z2op_p_zd<0b1001001, asm, ZPR16, ZPR32>;
   def _DtoH : sve_fp_z2op_p_zd<0b1101000, asm, ZPR64, ZPR16>;
   def _HtoD : sve_fp_z2op_p_zd<0b1101001, asm, ZPR16, ZPR64>;
   def _DtoS : sve_fp_z2op_p_zd<0b1101010, asm, ZPR64, ZPR32>;
   def _StoD : sve_fp_z2op_p_zd<0b1101011, asm, ZPR32, ZPR64>;
+
+  def : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f64), nxv8f16, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoH)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f16), nxv2f64, nxv2i1, nxv8f16, !cast<Instruction>(NAME # _HtoD)>;
+  def : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -4820,23 +4859,18 @@ multiclass sve_int_un_pred_arit<bits<3> opc, string asm,
   def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
   def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
-
-  def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;
-  def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
-  def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
-  def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
-
-  defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
-  defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
-  defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
-  defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm> {
+multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b0 }, asm, ZPR8>;
   def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;
   def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
+
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,
@@ -4950,7 +4984,22 @@ multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
   def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
+}
+
+multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm, SDPatternOperator op> {
+  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
+  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
+  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
 
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
+}
+
+multiclass sve_fp_un_pred_arit_hsd<SDPatternOperator op> {
   def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
   def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
   def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
@@ -4963,10 +5012,16 @@ multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
   defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
-multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm> {
-  def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
-  def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
-  def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
+multiclass sve_int_un_pred_arit_bhsd<SDPatternOperator op> {
+  def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;
+  def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
+  def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
+  def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
+
+  defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
+  defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
+  defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
+  defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -10714,10 +10769,15 @@ class sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic,
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
   let Uses = [FPMR, FPCR];
+
+  let mayLoad  = 1;
+  let mayStore = 0;
 }
 
-multiclass sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic> {
+multiclass sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic, ValueType vtd, SDPatternOperator op> {
   def _BtoH : sve2_fp8_cvt_single<L, opc, mnemonic, ZPR16, ZPR8>;
+  
+  def : SVE_1_Op_Pat<vtd, op, nxv16i8, !cast<Instruction>(NAME # _BtoH)>;
 }
 
 // FP8 downconvert
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index a5adbb5c23394..ec8d69853dd34 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -29,11 +29,13 @@ void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &);
 FunctionPass *createAMDGPUPostLegalizeCombiner(bool IsOptNone);
 FunctionPass *createAMDGPURegBankCombiner(bool IsOptNone);
 void initializeAMDGPURegBankCombinerPass(PassRegistry &);
+FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();
+FunctionPass *createAMDGPURegBankSelectPass();
+FunctionPass *createAMDGPURegBankLegalizePass();
 
 void initializeGlobalOffsetPass(PassRegistry &);
 void initializeLocalAccessorToSharedMemoryPass(PassRegistry &);
 
-void initializeAMDGPURegBankSelectPass(PassRegistry &);
 
 // SI Passes
 FunctionPass *createGCNDPPCombinePass();
@@ -41,13 +43,12 @@ FunctionPass *createSIAnnotateControlFlowLegacyPass();
 FunctionPass *createSIFoldOperandsLegacyPass();
 FunctionPass *createSIPeepholeSDWALegacyPass();
 FunctionPass *createSILowerI1CopiesLegacyPass();
-FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();
 FunctionPass *createSIShrinkInstructionsLegacyPass();
 FunctionPass *createSILoadStoreOptimizerLegacyPass();
 FunctionPass *createSIWholeQuadModePass();
 FunctionPass *createSIFixControlFlowLiveIntervalsPass();
 FunctionPass *createSIOptimizeExecMaskingPreRAPass();
-FunctionPass *createSIOptimizeVGPRLiveRangePass();
+FunctionPass *createSIOptimizeVGPRLiveRangeLegacyPass();
 FunctionPass *createSIFixSGPRCopiesLegacyPass();
 FunctionPass *createLowerWWMCopiesPass();
 FunctionPass *createSIMemoryLegalizerPass();
@@ -197,6 +198,12 @@ extern char &SILowerI1CopiesLegacyID;
 void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &);
 extern char &AMDGPUGlobalISelDivergenceLoweringID;
 
+void initializeAMDGPURegBankSelectPass(PassRegistry &);
+extern char &AMDGPURegBankSelectID;
+
+void initializeAMDGPURegBankLegalizePass(PassRegistry &);
+extern char &AMDGPURegBankLegalizeID;
+
 void initializeAMDGPUMarkLastScratchLoadPass(PassRegistry &);
 extern char &AMDGPUMarkLastScratchLoadID;
 
@@ -370,8 +377,8 @@ struct AMDGPUUnifyMetadataPass : PassInfoMixin<AMDGPUUnifyMetadataPass> {
 void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry&);
 extern char &SIOptimizeExecMaskingPreRAID;
 
-void initializeSIOptimizeVGPRLiveRangePass(PassRegistry &);
-extern char &SIOptimizeVGPRLiveRangeID;
+void initializeSIOptimizeVGPRLiveRangeLegacyPass(PassRegistry &);
+extern char &SIOptimizeVGPRLiveRangeLegacyID;
 
 void initializeAMDGPUAnnotateUniformValuesLegacyPass(PassRegistry &);
 extern char &AMDGPUAnnotateUniformValuesLegacyPassID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 5978f5b0bbae5..ebe6f0965deb9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -432,6 +432,12 @@ def FeatureAshrPkInsts : SubtargetFeature<"ashr-pk-insts",
   "Has Arithmetic Shift Pack instructions"
 >;
 
+def FeatureCvtPkF16F32Inst : SubtargetFeature<"cvt-pk-f16-f32-inst",
+  "HasCvtPkF16F32Inst",
+  "true",
+  "Has cvt_pk_f16_f32 instruction"
+>;
+
 def FeatureGFX950Insts : SubtargetFeature<"gfx950-insts",
   "GFX950Insts",
   "true",
@@ -445,8 +451,9 @@ def FeatureGFX950Insts : SubtargetFeature<"gfx950-insts",
    FeatureFP6BF6ConversionScaleInsts,
    FeatureF16BF16ToFP6BF6ConversionScaleInsts,
    FeatureF32ToF16BF16ConversionSRInsts,
+   FeatureCvtPkF16F32Inst,
    FeatureMinimum3Maximum3F32,
-   FeatureMinimum3Maximum3PKF16
+   FeatureMinimum3Maximum3PKF16,
    ]
 >;
 
@@ -2510,6 +2517,9 @@ def HasFP6BF6ConversionScaleInsts : Predicate<"Subtarget->hasFP6BF6ConversionSca
 def HasF16BF16ToFP6BF6ConversionScaleInsts : Predicate<"Subtarget->hasF16BF16ToFP6BF6ConversionScaleInsts()">,
   AssemblerPredicate<(all_of FeatureF16BF16ToFP6BF6ConversionScaleInsts)>;
 
+def HasCvtPkF16F32Inst : Predicate<"Subtarget->hasCvtPkF16F32Inst()">,
+  AssemblerPredicate<(all_of FeatureCvtPkF16F32Inst)>;
+
 def HasF32ToF16BF16ConversionSRInsts : Predicate<"Subtarget->hasF32ToF16BF16ConversionSRInsts()">,
   AssemblerPredicate<(all_of FeatureF32ToF16BF16ConversionSRInsts)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
index 5feb35a2b553a..9da0d93341ddf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
@@ -92,10 +92,10 @@ AMDGPUAnnotateUniformValuesPass::run(Function &F,
   AMDGPUAnnotateUniformValues Impl(UI, MSSA, AA, F);
   Impl.visit(F);
 
-  PreservedAnalyses PA = PreservedAnalyses::none();
   if (!Impl.changed())
-    return PA;
+    return PreservedAnalyses::all();
 
+  PreservedAnalyses PA = PreservedAnalyses::none();
   // TODO: Should preserve nearly everything
   PA.preserveSet<CFGAnalyses>();
   return PA;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index e4ca1ae0499b9..c09c71c83fead 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -66,11 +66,12 @@ class AMDGPUAtomicOptimizer : public FunctionPass {
 class AMDGPUAtomicOptimizerImpl
     : public InstVisitor<AMDGPUAtomicOptimizerImpl> {
 private:
+  Function &F;
   SmallVector<ReplacementInfo, 8> ToReplace;
-  const UniformityInfo *UA;
-  const DataLayout *DL;
+  const UniformityInfo &UA;
+  const DataLayout &DL;
   DomTreeUpdater &DTU;
-  const GCNSubtarget *ST;
+  const GCNSubtarget &ST;
   bool IsPixelShader;
   ScanOptions ScanImpl;
 
@@ -91,13 +92,14 @@ class AMDGPUAtomicOptimizerImpl
 public:
   AMDGPUAtomicOptimizerImpl() = delete;
 
-  AMDGPUAtomicOptimizerImpl(const UniformityInfo *UA, const DataLayout *DL,
-                            DomTreeUpdater &DTU, const GCNSubtarget *ST,
-                            bool IsPixelShader, ScanOptions ScanImpl)
-      : UA(UA), DL(DL), DTU(DTU), ST(ST), IsPixelShader(IsPixelShader),
+  AMDGPUAtomicOptimizerImpl(Function &F, const UniformityInfo &UA,
+                            DomTreeUpdater &DTU, const GCNSubtarget &ST,
+                            ScanOptions ScanImpl)
+      : F(F), UA(UA), DL(F.getDataLayout()), DTU(DTU), ST(ST),
+        IsPixelShader(F.getCallingConv() == CallingConv::AMDGPU_PS),
         ScanImpl(ScanImpl) {}
 
-  bool run(Function &F);
+  bool run();
 
   void visitAtomicRMWInst(AtomicRMWInst &I);
   void visitIntrinsicInst(IntrinsicInst &I);
@@ -114,40 +116,30 @@ bool AMDGPUAtomicOptimizer::runOnFunction(Function &F) {
     return false;
   }
 
-  const UniformityInfo *UA =
-      &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
-  const DataLayout *DL = &F.getDataLayout();
+  const UniformityInfo &UA =
+      getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
 
-  DominatorTreeWrapperPass *const DTW =
+  DominatorTreeWrapperPass *DTW =
       getAnalysisIfAvailable<DominatorTreeWrapperPass>();
   DomTreeUpdater DTU(DTW ? &DTW->getDomTree() : nullptr,
                      DomTreeUpdater::UpdateStrategy::Lazy);
 
   const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
   const TargetMachine &TM = TPC.getTM<TargetMachine>();
-  const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
+  const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
 
-  bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
-
-  return AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader, ScanImpl)
-      .run(F);
+  return AMDGPUAtomicOptimizerImpl(F, UA, DTU, ST, ScanImpl).run();
 }
 
 PreservedAnalyses AMDGPUAtomicOptimizerPass::run(Function &F,
                                                  FunctionAnalysisManager &AM) {
-
-  const auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
-  const DataLayout *DL = &F.getDataLayout();
+  const auto &UA = AM.getResult<UniformityInfoAnalysis>(F);
 
   DomTreeUpdater DTU(&AM.getResult<DominatorTreeAnalysis>(F),
                      DomTreeUpdater::UpdateStrategy::Lazy);
-  const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
-
-  bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
+  const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
 
-  bool IsChanged =
-      AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader, ScanImpl)
-          .run(F);
+  bool IsChanged = AMDGPUAtomicOptimizerImpl(F, UA, DTU, ST, ScanImpl).run();
 
   if (!IsChanged) {
     return PreservedAnalyses::all();
@@ -158,7 +150,7 @@ PreservedAnalyses AMDGPUAtomicOptimizerPass::run(Function &F,
   return PA;
 }
 
-bool AMDGPUAtomicOptimizerImpl::run(Function &F) {
+bool AMDGPUAtomicOptimizerImpl::run() {
 
   // Scan option None disables the Pass
   if (ScanImpl == ScanOptions::None) {
@@ -234,18 +226,18 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
 
   // If the pointer operand is divergent, then each lane is doing an atomic
   // operation on a different address, and we cannot optimize that.
-  if (UA->isDivergentUse(I.getOperandUse(PtrIdx))) {
+  if (UA.isDivergentUse(I.getOperandUse(PtrIdx))) {
     return;
   }
 
-  bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
+  bool ValDivergent = UA.isDivergentUse(I.getOperandUse(ValIdx));
 
   // If the value operand is divergent, each lane is contributing a different
   // value to the atomic calculation. We can only optimize divergent values if
   // we have DPP available on our subtarget (for DPP strategy), and the atomic
   // operation is 32 or 64 bits.
   if (ValDivergent) {
-    if (ScanImpl == ScanOptions::DPP && !ST->hasDPP())
+    if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
       return;
 
     if (!isLegalCrossLaneType(I.getType()))
@@ -324,14 +316,14 @@ void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
 
   const unsigned ValIdx = 0;
 
-  const bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
+  const bool ValDivergent = UA.isDivergentUse(I.getOperandUse(ValIdx));
 
   // If the value operand is divergent, each lane is contributing a different
   // value to the atomic calculation. We can only optimize divergent values if
   // we have DPP available on our subtarget (for DPP strategy), and the atomic
   // operation is 32 or 64 bits.
   if (ValDivergent) {
-    if (ScanImpl == ScanOptions::DPP && !ST->hasDPP())
+    if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
       return;
 
     if (!isLegalCrossLaneType(I.getType()))
@@ -341,7 +333,7 @@ void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
   // If any of the other arguments to the intrinsic are divergent, we can't
   // optimize the operation.
   for (unsigned Idx = 1; Idx < I.getNumOperands(); Idx++) {
-    if (UA->isDivergentUse(I.getOperandUse(Idx))) {
+    if (UA.isDivergentUse(I.getOperandUse(Idx))) {
       return;
     }
   }
@@ -418,17 +410,17 @@ Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
   }
 
   // Reduce within each pair of rows (i.e. 32 lanes).
-  assert(ST->hasPermLaneX16());
+  assert(ST.hasPermLaneX16());
   Value *Permlanex16Call =
       B.CreateIntrinsic(AtomicTy, Intrinsic::amdgcn_permlanex16,
                         {PoisonValue::get(AtomicTy), V, B.getInt32(0),
                          B.getInt32(0), B.getFalse(), B.getFalse()});
   V = buildNonAtomicBinOp(B, Op, V, Permlanex16Call);
-  if (ST->isWave32()) {
+  if (ST.isWave32()) {
     return V;
   }
 
-  if (ST->hasPermLane64()) {
+  if (ST.hasPermLane64()) {
     // Reduce across the upper and lower 32 lanes.
     Value *Permlane64Call =
         B.CreateIntrinsic(AtomicTy, Intrinsic::amdgcn_permlane64, V);
@@ -461,7 +453,7 @@ Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
                      {Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx),
                       B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
   }
-  if (ST->hasDPPBroadcasts()) {
+  if (ST.hasDPPBroadcasts()) {
     // GFX9 has DPP row broadcast operations.
     V = buildNonAtomicBinOp(
         B, Op, V,
@@ -479,7 +471,7 @@ Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
 
     // Combine lane 15 into lanes 16..31 (and, for wave 64, lane 47 into lanes
     // 48..63).
-    assert(ST->hasPermLaneX16());
+    assert(ST.hasPermLaneX16());
     Value *PermX =
         B.CreateIntrinsic(AtomicTy, Intrinsic::amdgcn_permlanex16,
                           {PoisonValue::get(AtomicTy), V, B.getInt32(-1),
@@ -490,7 +482,7 @@ Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
                     B.getInt32(0xa), B.getInt32(0xf), B.getFalse()});
     V = buildNonAtomicBinOp(B, Op, V, UpdateDPPCall);
 
-    if (!ST->isWave32()) {
+    if (!ST.isWave32()) {
       // Combine lane 31 into lanes 32..63.
       Value *const Lane31 = B.CreateIntrinsic(
           AtomicTy, Intrinsic::amdgcn_readlane, {V, B.getInt32(31)});
@@ -513,7 +505,7 @@ Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
   Module *M = B.GetInsertBlock()->getModule();
   Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::amdgcn_update_dpp, AtomicTy);
-  if (ST->hasDPPWavefrontShifts()) {
+  if (ST.hasDPPWavefrontShifts()) {
     // GFX9 has DPP wavefront shift operations.
     V = B.CreateCall(UpdateDPP,
                      {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf),
@@ -535,7 +527,7 @@ Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
     V = B.CreateCall(WriteLane, {B.CreateCall(ReadLane, {Old, B.getInt32(15)}),
                                  B.getInt32(16), V});
 
-    if (!ST->isWave32()) {
+    if (!ST.isWave32()) {
       // Copy the old lane 31 to the new lane 32.
       V = B.CreateCall(
           WriteLane,
@@ -560,7 +552,7 @@ std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
     IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *const Identity, Value *V,
     Instruction &I, BasicBlock *ComputeLoop, BasicBlock *ComputeEnd) const {
   auto *Ty = I.getType();
-  auto *WaveTy = B.getIntNTy(ST->getWavefrontSize());
+  auto *WaveTy = B.getIntNTy(ST.getWavefrontSize());
   auto *EntryBB = I.getParent();
   auto NeedResult = !I.use_empty();
 
@@ -698,7 +690,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
   Type *const Ty = I.getType();
   Type *Int32Ty = B.getInt32Ty();
   bool isAtomicFloatingPointTy = Ty->isFloatingPointTy();
-  [[maybe_unused]] const unsigned TyBitWidth = DL->getTypeSizeInBits(Ty);
+  [[maybe_unused]] const unsigned TyBitWidth = DL.getTypeSizeInBits(Ty);
 
   // This is the value in the atomic operation we need to combine in order to
   // reduce the number of atomic operations.
@@ -706,7 +698,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
 
   // We need to know how many lanes are active within the wavefront, and we do
   // this by doing a ballot of active lanes.
-  Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize());
+  Type *const WaveTy = B.getIntNTy(ST.getWavefrontSize());
   CallInst *const Ballot =
       B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
 
@@ -715,7 +707,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
   // below us only if its associated index was less than ours. We do this by
   // using the mbcnt intrinsic.
   Value *Mbcnt;
-  if (ST->isWave32()) {
+  if (ST.isWave32()) {
     Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {},
                               {Ballot, B.getInt32(0)});
   } else {
@@ -755,7 +747,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
       // that they can correctly contribute to the final result.
       NewV =
           B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
-      if (!NeedResult && ST->hasPermLaneX16()) {
+      if (!NeedResult && ST.hasPermLaneX16()) {
         // On GFX10 the permlanex16 instruction helps us build a reduction
         // without too many readlanes and writelanes, which are generally bad
         // for performance.
@@ -767,7 +759,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
         // Read the value from the last lane, which has accumulated the values
         // of each active lane in the wavefront. This will be our new value
         // which we will provide to the atomic operation.
-        Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
+        Value *const LastLaneIdx = B.getInt32(ST.getWavefrontSize() - 1);
         NewV = B.CreateIntrinsic(Ty, Intrinsic::amdgcn_readlane,
                                  {NewV, LastLaneIdx});
       }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributes.def b/llvm/lib/Target/AMDGPU/AMDGPUAttributes.def
index bacc8e4e821e5..8c1c8219690ba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributes.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributes.def
@@ -30,5 +30,6 @@ AMDGPU_ATTRIBUTE(WORKITEM_ID_Z, "amdgpu-no-workitem-id-z")
 AMDGPU_ATTRIBUTE(LDS_KERNEL_ID, "amdgpu-no-lds-kernel-id")
 AMDGPU_ATTRIBUTE(DEFAULT_QUEUE, "amdgpu-no-default-queue")
 AMDGPU_ATTRIBUTE(COMPLETION_ACTION, "amdgpu-no-completion-action")
+AMDGPU_ATTRIBUTE(FLAT_SCRATCH_INIT, "amdgpu-no-flat-scratch-init")
 
 #undef AMDGPU_ATTRIBUTE
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 2ae34636005ea..06e852fe4752e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -179,6 +179,11 @@ class AMDGPUInformationCache : public InformationCache {
     return {ST.getMinFlatWorkGroupSize(), ST.getMaxFlatWorkGroupSize()};
   }
 
+  SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
+    const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
+    return ST.getMaxNumWorkGroups(F);
+  }
+
   /// Get code object version.
   unsigned getCodeObjectVersion() const { return CodeObjectVersion; }
 
@@ -260,6 +265,18 @@ class AMDGPUInformationCache : public InformationCache {
     return !HasAperture && (Access & ADDR_SPACE_CAST);
   }
 
+  bool checkConstForAddrSpaceCastFromPrivate(const Constant *C) {
+    SmallPtrSet<const Constant *, 8> Visited;
+    uint8_t Access = getConstantAccess(C, Visited);
+
+    if (Access & ADDR_SPACE_CAST)
+      if (const auto *CE = dyn_cast<ConstantExpr>(C))
+        if (CE->getOperand(0)->getType()->getPointerAddressSpace() ==
+            AMDGPUAS::PRIVATE_ADDRESS)
+          return true;
+    return false;
+  }
+
 private:
   /// Used to determine if the Constant needs the queue pointer.
   DenseMap<const Constant *, uint8_t> ConstantStatus;
@@ -524,6 +541,9 @@ struct AAAMDAttributesFunction : public AAAMDAttributes {
     if (isAssumed(COMPLETION_ACTION) && funcRetrievesCompletionAction(A, COV))
       removeAssumedBits(COMPLETION_ACTION);
 
+    if (isAssumed(FLAT_SCRATCH_INIT) && needFlatScratchInit(A))
+      removeAssumedBits(FLAT_SCRATCH_INIT);
+
     return getAssumed() != OrigAssumed ? ChangeStatus::CHANGED
                                        : ChangeStatus::UNCHANGED;
   }
@@ -682,6 +702,65 @@ struct AAAMDAttributesFunction : public AAAMDAttributes {
     return !A.checkForAllCallLikeInstructions(DoesNotRetrieve, *this,
                                               UsedAssumedInformation);
   }
+
+  // Returns true if FlatScratchInit is needed, i.e., no-flat-scratch-init is
+  // not to be set.
+  bool needFlatScratchInit(Attributor &A) {
+    assert(isAssumed(FLAT_SCRATCH_INIT)); // only called if the bit is still set
+
+    // Check all AddrSpaceCast instructions. FlatScratchInit is needed if
+    // there is a cast from PRIVATE_ADDRESS.
+    auto AddrSpaceCastNotFromPrivate = [](Instruction &I) {
+      return cast<AddrSpaceCastInst>(I).getSrcAddressSpace() !=
+             AMDGPUAS::PRIVATE_ADDRESS;
+    };
+
+    bool UsedAssumedInformation = false;
+    if (!A.checkForAllInstructions(AddrSpaceCastNotFromPrivate, *this,
+                                   {Instruction::AddrSpaceCast},
+                                   UsedAssumedInformation))
+      return true;
+
+    // Check for addrSpaceCast from PRIVATE_ADDRESS in constant expressions
+    auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
+
+    Function *F = getAssociatedFunction();
+    for (Instruction &I : instructions(F)) {
+      for (const Use &U : I.operands()) {
+        if (const auto *C = dyn_cast<Constant>(U)) {
+          if (InfoCache.checkConstForAddrSpaceCastFromPrivate(C))
+            return true;
+        }
+      }
+    }
+
+    // Finally check callees.
+
+    // This is called on each callee; false means callee shouldn't have
+    // no-flat-scratch-init.
+    auto CheckForNoFlatScratchInit = [&](Instruction &I) {
+      const auto &CB = cast<CallBase>(I);
+      const Function *Callee = CB.getCalledFunction();
+
+      // Callee == 0 for inline asm or indirect call with known callees.
+      // In the latter case, updateImpl() already checked the callees and we
+      // know their FLAT_SCRATCH_INIT bit is set.
+      // If function has indirect call with unknown callees, the bit is
+      // already removed in updateImpl() and execution won't reach here.
+      if (!Callee)
+        return true;
+
+      return Callee->getIntrinsicID() !=
+             Intrinsic::amdgcn_addrspacecast_nonnull;
+    };
+
+    UsedAssumedInformation = false;
+    // If any callee is false (i.e. need FlatScratchInit),
+    // checkForAllCallLikeInstructions returns false, in which case this
+    // function returns true.
+    return !A.checkForAllCallLikeInstructions(CheckForNoFlatScratchInit, *this,
+                                              UsedAssumedInformation);
+  }
 };
 
 AAAMDAttributes &AAAMDAttributes::createForPosition(const IRPosition &IRP,
@@ -821,6 +900,145 @@ AAAMDFlatWorkGroupSize::createForPosition(const IRPosition &IRP,
       "AAAMDFlatWorkGroupSize is only valid for function position");
 }
 
+struct TupleDecIntegerRangeState : public AbstractState {
+  DecIntegerState<uint32_t> X, Y, Z;
+
+  bool isValidState() const override {
+    return X.isValidState() && Y.isValidState() && Z.isValidState();
+  }
+
+  bool isAtFixpoint() const override {
+    return X.isAtFixpoint() && Y.isAtFixpoint() && Z.isAtFixpoint();
+  }
+
+  ChangeStatus indicateOptimisticFixpoint() override {
+    return X.indicateOptimisticFixpoint() | Y.indicateOptimisticFixpoint() |
+           Z.indicateOptimisticFixpoint();
+  }
+
+  ChangeStatus indicatePessimisticFixpoint() override {
+    return X.indicatePessimisticFixpoint() | Y.indicatePessimisticFixpoint() |
+           Z.indicatePessimisticFixpoint();
+  }
+
+  TupleDecIntegerRangeState operator^=(const TupleDecIntegerRangeState &Other) {
+    X ^= Other.X;
+    Y ^= Other.Y;
+    Z ^= Other.Z;
+    return *this;
+  }
+
+  bool operator==(const TupleDecIntegerRangeState &Other) const {
+    return X == Other.X && Y == Other.Y && Z == Other.Z;
+  }
+
+  TupleDecIntegerRangeState &getAssumed() { return *this; }
+  const TupleDecIntegerRangeState &getAssumed() const { return *this; }
+};
+
+using AAAMDMaxNumWorkgroupsState =
+    StateWrapper<TupleDecIntegerRangeState, AbstractAttribute, uint32_t>;
+
+/// Propagate amdgpu-max-num-workgroups attribute.
+struct AAAMDMaxNumWorkgroups
+    : public StateWrapper<TupleDecIntegerRangeState, AbstractAttribute> {
+  using Base = StateWrapper<TupleDecIntegerRangeState, AbstractAttribute>;
+
+  AAAMDMaxNumWorkgroups(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+  void initialize(Attributor &A) override {
+    Function *F = getAssociatedFunction();
+    auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
+
+    SmallVector<unsigned> MaxNumWorkgroups = InfoCache.getMaxNumWorkGroups(*F);
+
+    X.takeKnownMinimum(MaxNumWorkgroups[0]);
+    Y.takeKnownMinimum(MaxNumWorkgroups[1]);
+    Z.takeKnownMinimum(MaxNumWorkgroups[2]);
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+      indicatePessimisticFixpoint();
+  }
+
+  ChangeStatus updateImpl(Attributor &A) override {
+    ChangeStatus Change = ChangeStatus::UNCHANGED;
+
+    auto CheckCallSite = [&](AbstractCallSite CS) {
+      Function *Caller = CS.getInstruction()->getFunction();
+      LLVM_DEBUG(dbgs() << "[AAAMDMaxNumWorkgroups] Call " << Caller->getName()
+                        << "->" << getAssociatedFunction()->getName() << '\n');
+
+      const auto *CallerInfo = A.getAAFor<AAAMDMaxNumWorkgroups>(
+          *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
+      if (!CallerInfo || !CallerInfo->isValidState())
+        return false;
+
+      Change |=
+          clampStateAndIndicateChange(this->getState(), CallerInfo->getState());
+      return true;
+    };
+
+    bool AllCallSitesKnown = true;
+    if (!A.checkForAllCallSites(CheckCallSite, *this,
+                                /*RequireAllCallSites=*/true,
+                                AllCallSitesKnown))
+      return indicatePessimisticFixpoint();
+
+    return Change;
+  }
+
+  /// Create an abstract attribute view for the position \p IRP.
+  static AAAMDMaxNumWorkgroups &createForPosition(const IRPosition &IRP,
+                                                  Attributor &A);
+
+  ChangeStatus manifest(Attributor &A) override {
+    Function *F = getAssociatedFunction();
+    LLVMContext &Ctx = F->getContext();
+    SmallString<32> Buffer;
+    raw_svector_ostream OS(Buffer);
+    OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed();
+
+    // TODO: Should annotate loads of the group size for this to do anything
+    // useful.
+    return A.manifestAttrs(
+        getIRPosition(),
+        {Attribute::get(Ctx, "amdgpu-max-num-workgroups", OS.str())},
+        /* ForceReplace= */ true);
+  }
+
+  const std::string getName() const override { return "AAAMDMaxNumWorkgroups"; }
+
+  const std::string getAsStr(Attributor *) const override {
+    std::string Buffer = "AAAMDMaxNumWorkgroupsState[";
+    raw_string_ostream OS(Buffer);
+    OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed()
+       << ']';
+    return OS.str();
+  }
+
+  const char *getIdAddr() const override { return &ID; }
+
+  /// This function should return true if the type of the \p AA is
+  /// AAAMDMaxNumWorkgroups
+  static bool classof(const AbstractAttribute *AA) {
+    return (AA->getIdAddr() == &ID);
+  }
+
+  void trackStatistics() const override {}
+
+  /// Unique ID (due to the unique address)
+  static const char ID;
+};
+
+const char AAAMDMaxNumWorkgroups::ID = 0;
+
+AAAMDMaxNumWorkgroups &
+AAAMDMaxNumWorkgroups::createForPosition(const IRPosition &IRP, Attributor &A) {
+  if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
+    return *new (A.Allocator) AAAMDMaxNumWorkgroups(IRP, A);
+  llvm_unreachable("AAAMDMaxNumWorkgroups is only valid for function position");
+}
+
 /// Propagate amdgpu-waves-per-eu attribute.
 struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
   AAAMDWavesPerEU(const IRPosition &IRP, Attributor &A)
@@ -1046,8 +1264,8 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM,
   DenseSet<const char *> Allowed(
       {&AAAMDAttributes::ID, &AAUniformWorkGroupSize::ID,
        &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID,
-       &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID, &AACallEdges::ID,
-       &AAPointerInfo::ID, &AAPotentialConstantValues::ID,
+       &AAAMDMaxNumWorkgroups::ID, &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID,
+       &AACallEdges::ID, &AAPointerInfo::ID, &AAPotentialConstantValues::ID,
        &AAUnderlyingObjects::ID, &AAAddressSpace::ID, &AAIndirectCallInfo::ID,
        &AAInstanceInfo::ID});
 
@@ -1068,9 +1286,14 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM,
 
   Attributor A(Functions, InfoCache, AC);
 
+  LLVM_DEBUG(dbgs() << "[AMDGPUAttributor] Module " << M.getName() << " is "
+                    << (AC.IsClosedWorldModule ? "" : "not ")
+                    << "assumed to be a closed world.\n");
+
   for (auto *F : Functions) {
     A.getOrCreateAAFor<AAAMDAttributes>(IRPosition::function(*F));
     A.getOrCreateAAFor<AAUniformWorkGroupSize>(IRPosition::function(*F));
+    A.getOrCreateAAFor<AAAMDMaxNumWorkgroups>(IRPosition::function(*F));
     A.getOrCreateAAFor<AAAMDGPUNoAGPR>(IRPosition::function(*F));
     CallingConv::ID CC = F->getCallingConv();
     if (!AMDGPU::isEntryFunctionCC(CC)) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 3e24f7875ac89..bb00442342d84 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -520,6 +520,12 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
 
   // TODO: Align down to dword alignment and extract bits for extending loads.
   for (auto &Arg : F.args()) {
+    // TODO: Add support for kernarg preload.
+    if (Arg.hasAttribute("amdgpu-hidden-argument")) {
+      LLVM_DEBUG(dbgs() << "Preloading hidden arguments is not supported\n");
+      return false;
+    }
+
     const bool IsByRef = Arg.hasByRefAttr();
     Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
     unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index c49aab823b44a..75e20c7930168 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -95,32 +95,45 @@ static cl::opt<bool> DisableFDivExpand(
   cl::ReallyHidden,
   cl::init(false));
 
+static bool hasUnsafeFPMath(const Function &F) {
+  return F.getFnAttribute("unsafe-fp-math").getValueAsBool();
+}
+
 class AMDGPUCodeGenPrepareImpl
     : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
 public:
-  const GCNSubtarget *ST = nullptr;
-  const AMDGPUTargetMachine *TM = nullptr;
-  const TargetLibraryInfo *TLInfo = nullptr;
-  AssumptionCache *AC = nullptr;
-  DominatorTree *DT = nullptr;
-  UniformityInfo *UA = nullptr;
-  Module *Mod = nullptr;
-  const DataLayout *DL = nullptr;
-  bool HasUnsafeFPMath = false;
-  bool HasFP32DenormalFlush = false;
+  Function &F;
+  const GCNSubtarget &ST;
+  const AMDGPUTargetMachine &TM;
+  const TargetLibraryInfo *TLI;
+  AssumptionCache *AC;
+  const DominatorTree *DT;
+  const UniformityInfo &UA;
+  const DataLayout &DL;
+  const bool HasUnsafeFPMath;
+  const bool HasFP32DenormalFlush;
   bool FlowChanged = false;
   mutable Function *SqrtF32 = nullptr;
   mutable Function *LdexpF32 = nullptr;
 
   DenseMap<const PHINode *, bool> BreakPhiNodesCache;
 
+  AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
+                           const TargetLibraryInfo *TLI, AssumptionCache *AC,
+                           const DominatorTree *DT, const UniformityInfo &UA)
+      : F(F), ST(TM.getSubtarget<GCNSubtarget>(F)), TM(TM), TLI(TLI), AC(AC),
+        DT(DT), UA(UA), DL(F.getDataLayout()),
+        HasUnsafeFPMath(hasUnsafeFPMath(F)),
+        HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
+                             DenormalMode::getPreserveSign()) {}
+
   Function *getSqrtF32() const {
     if (SqrtF32)
       return SqrtF32;
 
-    LLVMContext &Ctx = Mod->getContext();
-    SqrtF32 = Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::amdgcn_sqrt,
-                                                {Type::getFloatTy(Ctx)});
+    LLVMContext &Ctx = F.getContext();
+    SqrtF32 = Intrinsic::getOrInsertDeclaration(
+        F.getParent(), Intrinsic::amdgcn_sqrt, {Type::getFloatTy(Ctx)});
     return SqrtF32;
   }
 
@@ -128,9 +141,10 @@ class AMDGPUCodeGenPrepareImpl
     if (LdexpF32)
       return LdexpF32;
 
-    LLVMContext &Ctx = Mod->getContext();
+    LLVMContext &Ctx = F.getContext();
     LdexpF32 = Intrinsic::getOrInsertDeclaration(
-        Mod, Intrinsic::ldexp, {Type::getFloatTy(Ctx), Type::getInt32Ty(Ctx)});
+        F.getParent(), Intrinsic::ldexp,
+        {Type::getFloatTy(Ctx), Type::getInt32Ty(Ctx)});
     return LdexpF32;
   }
 
@@ -166,8 +180,7 @@ class AMDGPUCodeGenPrepareImpl
   /// Wrapper to pass all the arguments to computeKnownFPClass
   KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
                                    const Instruction *CtxI) const {
-    return llvm::computeKnownFPClass(V, *DL, Interested, 0, TLInfo, AC, CtxI,
-                                     DT);
+    return llvm::computeKnownFPClass(V, DL, Interested, 0, TLI, AC, CtxI, DT);
   }
 
   bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
@@ -317,13 +330,10 @@ class AMDGPUCodeGenPrepareImpl
   bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
   bool visitMinNum(IntrinsicInst &I);
   bool visitSqrt(IntrinsicInst &I);
-  bool run(Function &F);
+  bool run();
 };
 
 class AMDGPUCodeGenPrepare : public FunctionPass {
-private:
-  AMDGPUCodeGenPrepareImpl Impl;
-
 public:
   static char ID;
   AMDGPUCodeGenPrepare() : FunctionPass(ID) {
@@ -339,13 +349,12 @@ class AMDGPUCodeGenPrepare : public FunctionPass {
       AU.setPreservesAll();
   }
   bool runOnFunction(Function &F) override;
-  bool doInitialization(Module &M) override;
   StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
 };
 
 } // end anonymous namespace
 
-bool AMDGPUCodeGenPrepareImpl::run(Function &F) {
+bool AMDGPUCodeGenPrepareImpl::run() {
   BreakPhiNodesCache.clear();
   bool MadeChange = false;
 
@@ -411,7 +420,7 @@ bool AMDGPUCodeGenPrepareImpl::needsPromotionToI32(const Type *T) const {
   if (const VectorType *VT = dyn_cast<VectorType>(T)) {
     // TODO: The set of packed operations is more limited, so may want to
     // promote some anyway.
-    if (ST->hasVOP3PInsts())
+    if (ST.hasVOP3PInsts())
       return false;
 
     return needsPromotionToI32(VT->getElementType());
@@ -422,7 +431,7 @@ bool AMDGPUCodeGenPrepareImpl::needsPromotionToI32(const Type *T) const {
 
 bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
   return Ty->isFloatTy() || Ty->isDoubleTy() ||
-         (Ty->isHalfTy() && ST->has16BitInsts());
+         (Ty->isHalfTy() && ST.has16BitInsts());
 }
 
 // Return true if the op promoted to i32 should have nsw set.
@@ -455,11 +464,10 @@ static bool promotedOpIsNUW(const Instruction &I) {
 
 bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
   Type *Ty = I.getType();
-  const DataLayout &DL = Mod->getDataLayout();
   int TySize = DL.getTypeSizeInBits(Ty);
   Align Alignment = DL.getValueOrABITypeAlignment(I.getAlign(), Ty);
 
-  return I.isSimple() && TySize < 32 && Alignment >= 4 && UA->isUniform(&I);
+  return I.isSimple() && TySize < 32 && Alignment >= 4 && UA.isUniform(&I);
 }
 
 bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(BinaryOperator &I) const {
@@ -591,11 +599,11 @@ bool AMDGPUCodeGenPrepareImpl::promoteUniformBitreverseToI32(
 }
 
 unsigned AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op) const {
-  return computeKnownBits(Op, *DL, 0, AC).countMaxActiveBits();
+  return computeKnownBits(Op, DL, 0, AC).countMaxActiveBits();
 }
 
 unsigned AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op) const {
-  return ComputeMaxSignificantBits(Op, *DL, 0, AC);
+  return ComputeMaxSignificantBits(Op, DL, 0, AC);
 }
 
 static void extractValues(IRBuilder<> &Builder,
@@ -631,11 +639,11 @@ bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
 
   Type *Ty = I.getType();
   unsigned Size = Ty->getScalarSizeInBits();
-  if (Size <= 16 && ST->has16BitInsts())
+  if (Size <= 16 && ST.has16BitInsts())
     return false;
 
   // Prefer scalar if this could be s_mul_i32
-  if (UA->isUniform(&I))
+  if (UA.isUniform(&I))
     return false;
 
   Value *LHS = I.getOperand(0);
@@ -646,11 +654,11 @@ bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
   unsigned LHSBits = 0, RHSBits = 0;
   bool IsSigned = false;
 
-  if (ST->hasMulU24() && (LHSBits = numBitsUnsigned(LHS)) <= 24 &&
+  if (ST.hasMulU24() && (LHSBits = numBitsUnsigned(LHS)) <= 24 &&
       (RHSBits = numBitsUnsigned(RHS)) <= 24) {
     IsSigned = false;
 
-  } else if (ST->hasMulI24() && (LHSBits = numBitsSigned(LHS)) <= 24 &&
+  } else if (ST.hasMulI24() && (LHSBits = numBitsSigned(LHS)) <= 24 &&
              (RHSBits = numBitsSigned(RHS)) <= 24) {
     IsSigned = true;
 
@@ -730,21 +738,21 @@ bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
   if (CastOp) {
     if (!CastOp->hasOneUse())
       return false;
-    CT = ConstantFoldCastOperand(CastOp->getOpcode(), CT, BO.getType(), *DL);
-    CF = ConstantFoldCastOperand(CastOp->getOpcode(), CF, BO.getType(), *DL);
+    CT = ConstantFoldCastOperand(CastOp->getOpcode(), CT, BO.getType(), DL);
+    CF = ConstantFoldCastOperand(CastOp->getOpcode(), CF, BO.getType(), DL);
   }
 
   // TODO: Handle special 0/-1 cases DAG combine does, although we only really
   // need to handle divisions here.
-  Constant *FoldedT = SelOpNo ?
-    ConstantFoldBinaryOpOperands(BO.getOpcode(), CBO, CT, *DL) :
-    ConstantFoldBinaryOpOperands(BO.getOpcode(), CT, CBO, *DL);
+  Constant *FoldedT =
+      SelOpNo ? ConstantFoldBinaryOpOperands(BO.getOpcode(), CBO, CT, DL)
+              : ConstantFoldBinaryOpOperands(BO.getOpcode(), CT, CBO, DL);
   if (!FoldedT || isa<ConstantExpr>(FoldedT))
     return false;
 
-  Constant *FoldedF = SelOpNo ?
-    ConstantFoldBinaryOpOperands(BO.getOpcode(), CBO, CF, *DL) :
-    ConstantFoldBinaryOpOperands(BO.getOpcode(), CF, CBO, *DL);
+  Constant *FoldedF =
+      SelOpNo ? ConstantFoldBinaryOpOperands(BO.getOpcode(), CBO, CF, DL)
+              : ConstantFoldBinaryOpOperands(BO.getOpcode(), CF, CBO, DL);
   if (!FoldedF || isa<ConstantExpr>(FoldedF))
     return false;
 
@@ -777,7 +785,7 @@ AMDGPUCodeGenPrepareImpl::getFrexpResults(IRBuilder<> &Builder,
   // result? It's unspecified by the spec.
 
   Value *FrexpExp =
-      ST->hasFractBug()
+      ST.hasFractBug()
           ? Builder.CreateIntrinsic(Intrinsic::amdgcn_frexp_exp,
                                     {Builder.getInt32Ty(), Ty}, Src)
           : Builder.CreateExtractValue(Frexp, {1});
@@ -815,7 +823,7 @@ Value *AMDGPUCodeGenPrepareImpl::emitFrexpDiv(IRBuilder<> &Builder, Value *LHS,
   // If we have have to work around the fract/frexp bug, we're worse off than
   // using the fdiv.fast expansion. The full safe expansion is faster if we have
   // fast FMA.
-  if (HasFP32DenormalFlush && ST->hasFractBug() && !ST->hasFastFMAF32() &&
+  if (HasFP32DenormalFlush && ST.hasFractBug() && !ST.hasFastFMAF32() &&
       (!FMF.noNaNs() || !FMF.noInfs()))
     return nullptr;
 
@@ -1157,17 +1165,12 @@ bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
   if (NewVal) {
     FDiv.replaceAllUsesWith(NewVal);
     NewVal->takeName(&FDiv);
-    RecursivelyDeleteTriviallyDeadInstructions(&FDiv, TLInfo);
+    RecursivelyDeleteTriviallyDeadInstructions(&FDiv, TLI);
   }
 
   return true;
 }
 
-static bool hasUnsafeFPMath(const Function &F) {
-  Attribute Attr = F.getFnAttribute("unsafe-fp-math");
-  return Attr.getValueAsBool();
-}
-
 static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
                                           Value *LHS, Value *RHS) {
   Type *I32Ty = Builder.getInt32Ty();
@@ -1192,7 +1195,6 @@ static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
 int AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
                                             Value *Den, unsigned AtLeast,
                                             bool IsSigned) const {
-  const DataLayout &DL = Mod->getDataLayout();
   unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
   if (LHSSignBits < AtLeast)
     return -1;
@@ -1271,7 +1273,7 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl(
   Value *FQNeg = Builder.CreateFNeg(FQ);
 
   // float fr = mad(fqneg, fb, fa);
-  auto FMAD = !ST->hasMadMacF32Insts()
+  auto FMAD = !ST.hasMadMacF32Insts()
                   ? Intrinsic::fma
                   : (Intrinsic::ID)Intrinsic::amdgcn_fmad_ftz;
   Value *FR = Builder.CreateIntrinsic(FMAD,
@@ -1338,7 +1340,7 @@ bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
     // If there's no wider mulhi, there's only a better expansion for powers of
     // two.
     // TODO: Should really know for each vector element.
-    if (isKnownToBeAPowerOfTwo(C, *DL, true, 0, AC, &I, DT))
+    if (isKnownToBeAPowerOfTwo(C, DL, true, 0, AC, &I, DT))
       return true;
 
     return false;
@@ -1348,8 +1350,8 @@ bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
     // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
     if (BinOpDen->getOpcode() == Instruction::Shl &&
         isa<Constant>(BinOpDen->getOperand(0)) &&
-        isKnownToBeAPowerOfTwo(BinOpDen->getOperand(0), *DL, true,
-                               0, AC, &I, DT)) {
+        isKnownToBeAPowerOfTwo(BinOpDen->getOperand(0), DL, true, 0, AC, &I,
+                               DT)) {
       return true;
     }
   }
@@ -1357,9 +1359,9 @@ bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
   return false;
 }
 
-static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout *DL) {
+static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout DL) {
   // Check whether the sign can be determined statically.
-  KnownBits Known = computeKnownBits(V, *DL);
+  KnownBits Known = computeKnownBits(V, DL);
   if (Known.isNegative())
     return Constant::getAllOnesValue(V->getType());
   if (Known.isNonNegative())
@@ -1542,8 +1544,8 @@ bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
   if (foldBinOpIntoSelect(I))
     return true;
 
-  if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
-      UA->isUniform(&I) && promoteUniformOpToI32(I))
+  if (ST.has16BitInsts() && needsPromotionToI32(I.getType()) &&
+      UA.isUniform(&I) && promoteUniformOpToI32(I))
     return true;
 
   if (UseMul24Intrin && replaceMulWithMul24(I))
@@ -1655,11 +1657,11 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
         };
 
         WidenLoad->setMetadata(LLVMContext::MD_range,
-                               MDNode::get(Mod->getContext(), LowAndHigh));
+                               MDNode::get(F.getContext(), LowAndHigh));
       }
     }
 
-    int TySize = Mod->getDataLayout().getTypeSizeInBits(I.getType());
+    int TySize = DL.getTypeSizeInBits(I.getType());
     Type *IntNTy = Builder.getIntNTy(TySize);
     Value *ValTrunc = Builder.CreateTrunc(WidenLoad, IntNTy);
     Value *ValOrig = Builder.CreateBitCast(ValTrunc, I.getType());
@@ -1674,8 +1676,8 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
 bool AMDGPUCodeGenPrepareImpl::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
 
-  if (ST->has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) &&
-      UA->isUniform(&I))
+  if (ST.has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) &&
+      UA.isUniform(&I))
     Changed |= promoteUniformOpToI32(I);
 
   return Changed;
@@ -1688,8 +1690,8 @@ bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
   Value *CmpVal;
   FCmpInst::Predicate Pred;
 
-  if (ST->has16BitInsts() && needsPromotionToI32(I.getType())) {
-    if (UA->isUniform(&I))
+  if (ST.has16BitInsts() && needsPromotionToI32(I.getType())) {
+    if (UA.isUniform(&I))
       return promoteUniformOpToI32(I);
     return false;
   }
@@ -1722,7 +1724,7 @@ bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
 
   Fract->takeName(&I);
   I.replaceAllUsesWith(Fract);
-  RecursivelyDeleteTriviallyDeadInstructions(&I, TLInfo);
+  RecursivelyDeleteTriviallyDeadInstructions(&I, TLI);
   return true;
 }
 
@@ -1947,7 +1949,7 @@ bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
 
   FixedVectorType *FVT = dyn_cast<FixedVectorType>(I.getType());
   if (!FVT || FVT->getNumElements() == 1 ||
-      DL->getTypeSizeInBits(FVT) <= BreakLargePHIsThreshold)
+      DL.getTypeSizeInBits(FVT) <= BreakLargePHIsThreshold)
     return false;
 
   if (!ForceBreakLargePHIs && !canBreakPHINode(I))
@@ -1960,7 +1962,7 @@ bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
     unsigned Idx = 0;
     // For 8/16 bits type, don't scalarize fully but break it up into as many
     // 32-bit slices as we can, and scalarize the tail.
-    const unsigned EltSize = DL->getTypeSizeInBits(EltTy);
+    const unsigned EltSize = DL.getTypeSizeInBits(EltTy);
     const unsigned NumElts = FVT->getNumElements();
     if (EltSize == 8 || EltSize == 16) {
       const unsigned SubVecSize = (32 / EltSize);
@@ -2079,7 +2081,7 @@ bool AMDGPUCodeGenPrepareImpl::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
   SmallVector<const Value *, 4> WorkList;
   getUnderlyingObjects(I.getOperand(0), WorkList);
   if (!all_of(WorkList, [&](const Value *V) {
-        return isPtrKnownNeverNull(V, *DL, *TM, SrcAS);
+        return isPtrKnownNeverNull(V, DL, TM, SrcAS);
       }))
     return false;
 
@@ -2107,8 +2109,8 @@ bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
 bool AMDGPUCodeGenPrepareImpl::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
   bool Changed = false;
 
-  if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
-      UA->isUniform(&I))
+  if (ST.has16BitInsts() && needsPromotionToI32(I.getType()) &&
+      UA.isUniform(&I))
     Changed |= promoteUniformBitreverseToI32(I);
 
   return Changed;
@@ -2120,7 +2122,7 @@ bool AMDGPUCodeGenPrepareImpl::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
 /// If fract is a useful instruction for the subtarget. Does not account for the
 /// nan handling; the instruction has a nan check on the input value.
 Value *AMDGPUCodeGenPrepareImpl::matchFractPat(IntrinsicInst &I) {
-  if (ST->hasFractBug())
+  if (ST.hasFractBug())
     return nullptr;
 
   if (I.getIntrinsicID() != Intrinsic::minnum)
@@ -2177,7 +2179,7 @@ bool AMDGPUCodeGenPrepareImpl::visitMinNum(IntrinsicInst &I) {
   // Match pattern for fract intrinsic in contexts where the nan check has been
   // optimized out (and hope the knowledge the source can't be nan wasn't lost).
   if (!I.hasNoNaNs() &&
-      !isKnownNeverNaN(FractArg, /*Depth=*/0, SimplifyQuery(*DL, TLInfo)))
+      !isKnownNeverNaN(FractArg, /*Depth=*/0, SimplifyQuery(DL, TLI)))
     return false;
 
   IRBuilder<> Builder(&I);
@@ -2189,7 +2191,7 @@ bool AMDGPUCodeGenPrepareImpl::visitMinNum(IntrinsicInst &I) {
   Fract->takeName(&I);
   I.replaceAllUsesWith(Fract);
 
-  RecursivelyDeleteTriviallyDeadInstructions(&I, TLInfo);
+  RecursivelyDeleteTriviallyDeadInstructions(&I, TLI);
   return true;
 }
 
@@ -2201,7 +2203,7 @@ static bool isOneOrNegOne(const Value *Val) {
 // Expand llvm.sqrt.f32 calls with !fpmath metadata in a semi-fast way.
 bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
   Type *Ty = Sqrt.getType()->getScalarType();
-  if (!Ty->isFloatTy() && (!Ty->isHalfTy() || ST->has16BitInsts()))
+  if (!Ty->isFloatTy() && (!Ty->isHalfTy() || ST.has16BitInsts()))
     return false;
 
   const FPMathOperator *FPOp = cast<const FPMathOperator>(&Sqrt);
@@ -2257,14 +2259,6 @@ bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
-  Impl.Mod = &M;
-  Impl.DL = &Impl.Mod->getDataLayout();
-  Impl.SqrtF32 = nullptr;
-  Impl.LdexpF32 = nullptr;
-  return false;
-}
-
 bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
@@ -2274,39 +2268,31 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
     return false;
 
   const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
-  Impl.TM = &TM;
-  Impl.TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-  Impl.ST = &TM.getSubtarget<GCNSubtarget>(F);
-  Impl.AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
-  Impl.UA = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
+  const TargetLibraryInfo *TLI =
+      &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+  AssumptionCache *AC =
+      &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
-  Impl.DT = DTWP ? &DTWP->getDomTree() : nullptr;
-  Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
-  SIModeRegisterDefaults Mode(F, *Impl.ST);
-  Impl.HasFP32DenormalFlush =
-      Mode.FP32Denormals == DenormalMode::getPreserveSign();
-  return Impl.run(F);
+  const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
+  const UniformityInfo &UA =
+      getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
+  return AMDGPUCodeGenPrepareImpl(F, TM, TLI, AC, DT, UA).run();
 }
 
 PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
                                                 FunctionAnalysisManager &FAM) {
-  AMDGPUCodeGenPrepareImpl Impl;
-  Impl.Mod = F.getParent();
-  Impl.DL = &Impl.Mod->getDataLayout();
-  Impl.TM = static_cast<const AMDGPUTargetMachine *>(&TM);
-  Impl.TLInfo = &FAM.getResult<TargetLibraryAnalysis>(F);
-  Impl.ST = &TM.getSubtarget<GCNSubtarget>(F);
-  Impl.AC = &FAM.getResult<AssumptionAnalysis>(F);
-  Impl.UA = &FAM.getResult<UniformityInfoAnalysis>(F);
-  Impl.DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
-  Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
-  SIModeRegisterDefaults Mode(F, *Impl.ST);
-  Impl.HasFP32DenormalFlush =
-      Mode.FP32Denormals == DenormalMode::getPreserveSign();
+  const AMDGPUTargetMachine &ATM = static_cast<const AMDGPUTargetMachine &>(TM);
+  const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(F);
+  AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(F);
+  const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+  const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(F);
+  AMDGPUCodeGenPrepareImpl Impl(F, ATM, TLI, AC, DT, UA);
+  if (!Impl.run())
+    return PreservedAnalyses::all();
   PreservedAnalyses PA = PreservedAnalyses::none();
   if (!Impl.FlowChanged)
     PA.preserveSet<CFGAnalyses>();
-  return Impl.run(F) ? PA : PreservedAnalyses::all();
+  return PA;
 }
 
 INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 1e7d4fe4a869f..ac01562e457f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -832,7 +832,8 @@ void PipelineSolver::solve() {
 enum IGLPStrategyID : int {
   MFMASmallGemmOptID = 0,
   MFMASmallGemmSingleWaveOptID = 1,
-  MFMAExpInterleave = 2
+  MFMAExpInterleaveID = 2,
+  MFMAExpSimpleInterleaveID = 3
 };
 
 // Implement a IGLP scheduling strategy.
@@ -1845,6 +1846,48 @@ bool MFMAExpInterleaveOpt::applyIGLPStrategy(
   return true;
 }
 
+class MFMAExpSimpleInterleaveOpt final : public IGLPStrategy {
+public:
+  bool applyIGLPStrategy(
+      DenseMap<int, SUnitsToCandidateSGsMap> &SyncedInstrs,
+      DenseMap<int, SmallVector<SchedGroup, 4>> &SyncedSchedGroups,
+      AMDGPU::SchedulingPhase Phase) override;
+
+  bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
+                           AMDGPU::SchedulingPhase Phase) override {
+    return true;
+  }
+
+  MFMAExpSimpleInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
+      : IGLPStrategy(DAG, TII) {
+    IsBottomUp = true;
+  }
+};
+
+bool MFMAExpSimpleInterleaveOpt::applyIGLPStrategy(
+    DenseMap<int, SUnitsToCandidateSGsMap> &SyncedInstrs,
+    DenseMap<int, SmallVector<SchedGroup, 4>> &SyncedSchedGroups,
+    AMDGPU::SchedulingPhase Phase) {
+  // Count the number of MFMA instructions.
+  unsigned MFMACount = 0;
+  for (const MachineInstr &I : *DAG)
+    if (TII->isMFMAorWMMA(I))
+      ++MFMACount;
+
+  const unsigned PipelineSyncID = 0;
+  for (unsigned I = 0; I < MFMACount * 3; ++I) {
+    SchedGroup *SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
+        SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
+    SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
+
+    SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
+        SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
+    SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
+  }
+
+  return true;
+}
+
 class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
 private:
   // Whether the DS_READ is a predecessor of first four MFMA in region
@@ -2308,8 +2351,10 @@ createIGLPStrategy(IGLPStrategyID ID, ScheduleDAGInstrs *DAG,
     return std::make_unique<MFMASmallGemmOpt>(DAG, TII);
   case MFMASmallGemmSingleWaveOptID:
     return std::make_unique<MFMASmallGemmSingleWaveOpt>(DAG, TII);
-  case MFMAExpInterleave:
+  case MFMAExpInterleaveID:
     return std::make_unique<MFMAExpInterleaveOpt>(DAG, TII);
+  case MFMAExpSimpleInterleaveID:
+    return std::make_unique<MFMAExpSimpleInterleaveOpt>(DAG, TII);
   }
 
   llvm_unreachable("Unknown IGLPStrategyID");
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index d77508227b076..48e9af9fe507f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1072,6 +1072,15 @@ bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
   assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
           N->getOpcode() == ISD::SRL) &&
          "Expected shift op");
+
+  SDValue ShiftLHS = N->getOperand(0);
+  if (!ShiftLHS->hasOneUse())
+    return false;
+
+  if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
+      !ShiftLHS.getOperand(0)->hasOneUse())
+    return false;
+
   // Always commute pre-type legalization and right shifts.
   // We're looking for shl(or(x,y),z) patterns.
   if (Level < CombineLevel::AfterLegalizeTypes ||
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 18a09c39a0638..41b33ac8a7eb4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -450,6 +450,37 @@ static bool isTriviallyUniform(const Use &U) {
   return false;
 }
 
+/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
+///
+/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
+bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
+                                             IntrinsicInst &II,
+                                             unsigned LaneArgIdx) const {
+  unsigned MaskBits = ST->getWavefrontSizeLog2();
+  APInt DemandedMask(32, maskTrailingOnes<unsigned>(MaskBits));
+
+  KnownBits Known(32);
+  if (IC.SimplifyDemandedBits(&II, LaneArgIdx, DemandedMask, Known))
+    return true;
+
+  if (!Known.isConstant())
+    return false;
+
+  // Out of bounds indexes may appear in wave64 code compiled for wave32.
+  // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
+  // manually fix it up.
+
+  Value *LaneArg = II.getArgOperand(LaneArgIdx);
+  Constant *MaskedConst =
+      ConstantInt::get(LaneArg->getType(), Known.getConstant() & DemandedMask);
+  if (MaskedConst != LaneArg) {
+    II.getOperandUse(LaneArgIdx).set(MaskedConst);
+    return true;
+  }
+
+  return false;
+}
+
 std::optional<Instruction *>
 GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
   Intrinsic::ID IID = II.getIntrinsicID();
@@ -1092,7 +1123,17 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     const Use &Src = II.getArgOperandUse(0);
     if (isTriviallyUniform(Src))
       return IC.replaceInstUsesWith(II, Src.get());
-    break;
+
+    if (IID == Intrinsic::amdgcn_readlane &&
+        simplifyDemandedLaneMaskArg(IC, II, 1))
+      return &II;
+
+    return std::nullopt;
+  }
+  case Intrinsic::amdgcn_writelane: {
+    if (simplifyDemandedLaneMaskArg(IC, II, 1))
+      return &II;
+    return std::nullopt;
   }
   case Intrinsic::amdgcn_trig_preop: {
     // The intrinsic is declared with name mangling, but currently the
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 71d23f9fe30c4..3be865f03df1f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -3764,10 +3764,15 @@ bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
   if (!Subtarget->hasBitOp3Insts())
     return false;
 
+  Register DstReg = MI.getOperand(0).getReg();
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
+  const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
+  if (!IsVALU)
+    return false;
+
   SmallVector<Register, 3> Src;
   uint8_t TTbl;
   unsigned NumOpcodes;
-  Register DstReg = MI.getOperand(0).getReg();
 
   std::tie(NumOpcodes, TTbl) = BitOp3_Op(DstReg, Src, *MRI);
 
@@ -3776,13 +3781,10 @@ bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
   if (NumOpcodes < 2 || Src.empty())
     return false;
 
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
-  const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
-
   // For a uniform case threshold should be higher to account for moves between
   // VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be in SGPRs
   // and a readtfirstlane after.
-  if (NumOpcodes < 4 && !IsVALU)
+  if (NumOpcodes < 4)
     return false;
 
   bool IsB32 = MRI->getType(DstReg) == LLT::scalar(32);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 30ef390faab8b..830b50307f837 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -42,25 +42,25 @@ namespace {
 
 class AMDGPULateCodeGenPrepare
     : public InstVisitor<AMDGPULateCodeGenPrepare, bool> {
-  Module *Mod = nullptr;
-  const DataLayout *DL = nullptr;
+  Function &F;
+  const DataLayout &DL;
   const GCNSubtarget &ST;
 
-  AssumptionCache *AC = nullptr;
-  UniformityInfo *UA = nullptr;
+  AssumptionCache *const AC;
+  UniformityInfo &UA;
 
   SmallVector<WeakTrackingVH, 8> DeadInsts;
 
 public:
-  AMDGPULateCodeGenPrepare(Module &M, const GCNSubtarget &ST,
-                           AssumptionCache *AC, UniformityInfo *UA)
-      : Mod(&M), DL(&M.getDataLayout()), ST(ST), AC(AC), UA(UA) {}
-  bool run(Function &F);
+  AMDGPULateCodeGenPrepare(Function &F, const GCNSubtarget &ST,
+                           AssumptionCache *AC, UniformityInfo &UA)
+      : F(F), DL(F.getDataLayout()), ST(ST), AC(AC), UA(UA) {}
+  bool run();
   bool visitInstruction(Instruction &) { return false; }
 
   // Check if the specified value is at least DWORD aligned.
   bool isDWORDAligned(const Value *V) const {
-    KnownBits Known = computeKnownBits(V, *DL, 0, AC);
+    KnownBits Known = computeKnownBits(V, DL, 0, AC);
     return Known.countMinTrailingZeros() >= 2;
   }
 
@@ -72,11 +72,11 @@ using ValueToValueMap = DenseMap<const Value *, Value *>;
 
 class LiveRegOptimizer {
 private:
-  Module *Mod = nullptr;
-  const DataLayout *DL = nullptr;
-  const GCNSubtarget *ST;
+  Module &Mod;
+  const DataLayout &DL;
+  const GCNSubtarget &ST;
   /// The scalar type to convert to
-  Type *ConvertToScalar;
+  Type *const ConvertToScalar;
   /// The set of visited Instructions
   SmallPtrSet<Instruction *, 4> Visited;
   /// Map of Value -> Converted Value
@@ -110,7 +110,7 @@ class LiveRegOptimizer {
     if (!VTy)
       return false;
 
-    const auto *TLI = ST->getTargetLowering();
+    const auto *TLI = ST.getTargetLowering();
 
     Type *EltTy = VTy->getElementType();
     // If the element size is not less than the convert to scalar size, then we
@@ -125,15 +125,14 @@ class LiveRegOptimizer {
     return LK.first != TargetLoweringBase::TypeLegal;
   }
 
-  LiveRegOptimizer(Module *Mod, const GCNSubtarget *ST) : Mod(Mod), ST(ST) {
-    DL = &Mod->getDataLayout();
-    ConvertToScalar = Type::getInt32Ty(Mod->getContext());
-  }
+  LiveRegOptimizer(Module &Mod, const GCNSubtarget &ST)
+      : Mod(Mod), DL(Mod.getDataLayout()), ST(ST),
+        ConvertToScalar(Type::getInt32Ty(Mod.getContext())) {}
 };
 
 } // end anonymous namespace
 
-bool AMDGPULateCodeGenPrepare::run(Function &F) {
+bool AMDGPULateCodeGenPrepare::run() {
   // "Optimize" the virtual regs that cross basic block boundaries. When
   // building the SelectionDAG, vectors of illegal types that cross basic blocks
   // will be scalarized and widened, with each scalar living in its
@@ -141,7 +140,7 @@ bool AMDGPULateCodeGenPrepare::run(Function &F) {
   // vectors to equivalent vectors of legal type (which are converted back
   // before uses in subsequent blocks), to pack the bits into fewer physical
   // registers (used in CopyToReg/CopyFromReg pairs).
-  LiveRegOptimizer LRO(Mod, &ST);
+  LiveRegOptimizer LRO(*F.getParent(), ST);
 
   bool Changed = false;
 
@@ -163,15 +162,15 @@ Type *LiveRegOptimizer::calculateConvertType(Type *OriginalType) {
 
   FixedVectorType *VTy = cast<FixedVectorType>(OriginalType);
 
-  TypeSize OriginalSize = DL->getTypeSizeInBits(VTy);
-  TypeSize ConvertScalarSize = DL->getTypeSizeInBits(ConvertToScalar);
+  TypeSize OriginalSize = DL.getTypeSizeInBits(VTy);
+  TypeSize ConvertScalarSize = DL.getTypeSizeInBits(ConvertToScalar);
   unsigned ConvertEltCount =
       (OriginalSize + ConvertScalarSize - 1) / ConvertScalarSize;
 
   if (OriginalSize <= ConvertScalarSize)
-    return IntegerType::get(Mod->getContext(), ConvertScalarSize);
+    return IntegerType::get(Mod.getContext(), ConvertScalarSize);
 
-  return VectorType::get(Type::getIntNTy(Mod->getContext(), ConvertScalarSize),
+  return VectorType::get(Type::getIntNTy(Mod.getContext(), ConvertScalarSize),
                          ConvertEltCount, false);
 }
 
@@ -180,8 +179,8 @@ Value *LiveRegOptimizer::convertToOptType(Instruction *V,
   FixedVectorType *VTy = cast<FixedVectorType>(V->getType());
   Type *NewTy = calculateConvertType(V->getType());
 
-  TypeSize OriginalSize = DL->getTypeSizeInBits(VTy);
-  TypeSize NewSize = DL->getTypeSizeInBits(NewTy);
+  TypeSize OriginalSize = DL.getTypeSizeInBits(VTy);
+  TypeSize NewSize = DL.getTypeSizeInBits(NewTy);
 
   IRBuilder<> Builder(V->getParent(), InsertPt);
   // If there is a bitsize match, we can fit the old vector into a new vector of
@@ -210,8 +209,8 @@ Value *LiveRegOptimizer::convertFromOptType(Type *ConvertType, Instruction *V,
                                             BasicBlock *InsertBB) {
   FixedVectorType *NewVTy = cast<FixedVectorType>(ConvertType);
 
-  TypeSize OriginalSize = DL->getTypeSizeInBits(V->getType());
-  TypeSize NewSize = DL->getTypeSizeInBits(NewVTy);
+  TypeSize OriginalSize = DL.getTypeSizeInBits(V->getType());
+  TypeSize NewSize = DL.getTypeSizeInBits(NewVTy);
 
   IRBuilder<> Builder(InsertBB, InsertPt);
   // If there is a bitsize match, we simply convert back to the original type.
@@ -224,14 +223,14 @@ Value *LiveRegOptimizer::convertFromOptType(Type *ConvertType, Instruction *V,
   // For wide scalars, we can just truncate the value.
   if (!V->getType()->isVectorTy()) {
     Instruction *Trunc = cast<Instruction>(
-        Builder.CreateTrunc(V, IntegerType::get(Mod->getContext(), NewSize)));
+        Builder.CreateTrunc(V, IntegerType::get(Mod.getContext(), NewSize)));
     return cast<Instruction>(Builder.CreateBitCast(Trunc, NewVTy));
   }
 
   // For wider vectors, we must strip the MSBs to convert back to the original
   // type.
   VectorType *ExpandedVT = VectorType::get(
-      Type::getIntNTy(Mod->getContext(), NewVTy->getScalarSizeInBits()),
+      Type::getIntNTy(Mod.getContext(), NewVTy->getScalarSizeInBits()),
       (OriginalSize / NewVTy->getScalarSizeInBits()), false);
   Instruction *Converted =
       cast<Instruction>(Builder.CreateBitCast(V, ExpandedVT));
@@ -410,15 +409,15 @@ bool AMDGPULateCodeGenPrepare::canWidenScalarExtLoad(LoadInst &LI) const {
   // Skip aggregate types.
   if (Ty->isAggregateType())
     return false;
-  unsigned TySize = DL->getTypeStoreSize(Ty);
+  unsigned TySize = DL.getTypeStoreSize(Ty);
   // Only handle sub-DWORD loads.
   if (TySize >= 4)
     return false;
   // That load must be at least naturally aligned.
-  if (LI.getAlign() < DL->getABITypeAlign(Ty))
+  if (LI.getAlign() < DL.getABITypeAlign(Ty))
     return false;
   // It should be uniform, i.e. a scalar load.
-  return UA->isUniform(&LI);
+  return UA.isUniform(&LI);
 }
 
 bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
@@ -435,7 +434,7 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
 
   int64_t Offset = 0;
   auto *Base =
-      GetPointerBaseWithConstantOffset(LI.getPointerOperand(), Offset, *DL);
+      GetPointerBaseWithConstantOffset(LI.getPointerOperand(), Offset, DL);
   // If that base is not DWORD aligned, it's not safe to perform the following
   // transforms.
   if (!isDWORDAligned(Base))
@@ -452,7 +451,7 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
   IRBuilder<> IRB(&LI);
   IRB.SetCurrentDebugLocation(LI.getDebugLoc());
 
-  unsigned LdBits = DL->getTypeStoreSizeInBits(LI.getType());
+  unsigned LdBits = DL.getTypeStoreSizeInBits(LI.getType());
   auto *IntNTy = Type::getIntNTy(LI.getContext(), LdBits);
 
   auto *NewPtr = IRB.CreateConstGEP1_64(
@@ -480,13 +479,11 @@ AMDGPULateCodeGenPreparePass::run(Function &F, FunctionAnalysisManager &FAM) {
   AssumptionCache &AC = FAM.getResult<AssumptionAnalysis>(F);
   UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(F);
 
-  AMDGPULateCodeGenPrepare Impl(*F.getParent(), ST, &AC, &UI);
-
-  bool Changed = Impl.run(F);
+  bool Changed = AMDGPULateCodeGenPrepare(F, ST, &AC, UI).run();
 
-  PreservedAnalyses PA = PreservedAnalyses::none();
   if (!Changed)
-    return PA;
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA = PreservedAnalyses::none();
   PA.preserveSet<CFGAnalyses>();
   return PA;
 }
@@ -524,9 +521,7 @@ bool AMDGPULateCodeGenPrepareLegacy::runOnFunction(Function &F) {
   UniformityInfo &UI =
       getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
 
-  AMDGPULateCodeGenPrepare Impl(*F.getParent(), ST, &AC, &UI);
-
-  return Impl.run(F);
+  return AMDGPULateCodeGenPrepare(F, ST, &AC, UI).run();
 }
 
 INITIALIZE_PASS_BEGIN(AMDGPULateCodeGenPrepareLegacy, DEBUG_TYPE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 9bf1f281c32a0..2e66f7525b9cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1040,10 +1040,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .lower();
   }
 
-  getActionDefinitionsBuilder(G_FPTRUNC)
-    .legalFor({{S32, S64}, {S16, S32}})
-    .scalarize(0)
-    .lower();
+  auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
+  if (ST.hasCvtPkF16F32Inst())
+    FPTruncActions.legalFor(
+        {{S32, S64}, {S16, S32}, {V2S16, V2S32}, {V2S16, V2S64}});
+  else
+    FPTruncActions.legalFor({{S32, S64}, {S16, S32}});
+  FPTruncActions.scalarize(0).lower();
 
   getActionDefinitionsBuilder(G_FPEXT)
     .legalFor({{S64, S32}, {S32, S16}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 380633f22a178..e9d009baa20af 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -15,6 +15,7 @@
 #include "GCNSubtarget.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Attributes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/MDBuilder.h"
@@ -143,20 +144,20 @@ class PreloadKernelArgInfo {
   // Returns the maximum number of user SGPRs that we have available to preload
   // arguments.
   void setInitialFreeUserSGPRsCount() {
-    const unsigned MaxUserSGPRs = ST.getMaxNumUserSGPRs();
     GCNUserSGPRUsageInfo UserSGPRInfo(F, ST);
-
-    NumFreeUserSGPRs = MaxUserSGPRs - UserSGPRInfo.getNumUsedUserSGPRs();
+    NumFreeUserSGPRs = UserSGPRInfo.getNumFreeUserSGPRs();
   }
 
   bool tryAllocPreloadSGPRs(unsigned AllocSize, uint64_t ArgOffset,
                             uint64_t LastExplicitArgOffset) {
     //  Check if this argument may be loaded into the same register as the
     //  previous argument.
-    if (!isAligned(Align(4), ArgOffset) && AllocSize < 4)
+    if (ArgOffset - LastExplicitArgOffset < 4 &&
+        !isAligned(Align(4), ArgOffset))
       return true;
 
     // Pad SGPRs for kernarg alignment.
+    ArgOffset = alignDown(ArgOffset, 4);
     unsigned Padding = ArgOffset - LastExplicitArgOffset;
     unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
     unsigned NumPreloadSGPRs = alignTo(AllocSize, 4) / 4;
@@ -169,6 +170,7 @@ class PreloadKernelArgInfo {
 
   // Try to allocate SGPRs to preload implicit kernel arguments.
   void tryAllocImplicitArgPreloadSGPRs(uint64_t ImplicitArgsBaseOffset,
+                                       uint64_t LastExplicitArgOffset,
                                        IRBuilder<> &Builder) {
     Function *ImplicitArgPtr = Intrinsic::getDeclarationIfExists(
         F.getParent(), Intrinsic::amdgcn_implicitarg_ptr);
@@ -214,7 +216,6 @@ class PreloadKernelArgInfo {
     // argument can actually be preloaded.
     std::sort(ImplicitArgLoads.begin(), ImplicitArgLoads.end(), less_second());
 
-    uint64_t LastExplicitArgOffset = ImplicitArgsBaseOffset;
     // If we fail to preload any implicit argument we know we don't have SGPRs
     // to preload any subsequent ones with larger offsets. Find the first
     // argument that we cannot preload.
@@ -228,7 +229,8 @@ class PreloadKernelArgInfo {
                                     LastExplicitArgOffset))
             return true;
 
-          LastExplicitArgOffset = LoadOffset + LoadSize;
+          LastExplicitArgOffset =
+              ImplicitArgsBaseOffset + LoadOffset + LoadSize;
           return false;
         });
 
@@ -416,6 +418,16 @@ static bool lowerKernelArguments(Function &F, const TargetMachine &TM) {
 
     MDBuilder MDB(Ctx);
 
+    if (Arg.hasAttribute(Attribute::NoUndef))
+      Load->setMetadata(LLVMContext::MD_noundef, MDNode::get(Ctx, {}));
+
+    if (Arg.hasAttribute(Attribute::Range)) {
+      const ConstantRange &Range =
+          Arg.getAttribute(Attribute::Range).getValueAsConstantRange();
+      Load->setMetadata(LLVMContext::MD_range,
+                        MDB.createRange(Range.getLower(), Range.getUpper()));
+    }
+
     if (isa<PointerType>(ArgTy)) {
       if (Arg.hasNonNullAttr())
         Load->setMetadata(LLVMContext::MD_nonnull, MDNode::get(Ctx, {}));
@@ -475,7 +487,7 @@ static bool lowerKernelArguments(Function &F, const TargetMachine &TM) {
         alignTo(ExplicitArgOffset, ST.getAlignmentForImplicitArgPtr()) +
         BaseOffset;
     PreloadInfo.tryAllocImplicitArgPreloadSGPRs(ImplicitArgsBaseOffset,
-                                                Builder);
+                                                ExplicitArgOffset, Builder);
   }
 
   return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
index 924b4034c2b95..6c8116abd8dcd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Pass.h"
 
@@ -82,6 +83,20 @@ Function *getBasePtrIntrinsic(Module &M, bool IsV5OrAbove) {
 
 } // end anonymous namespace
 
+static void annotateGridSizeLoadWithRangeMD(LoadInst *Load,
+                                            uint32_t MaxNumGroups) {
+  if (MaxNumGroups == 0 || MaxNumGroups == std::numeric_limits<uint32_t>::max())
+    return;
+
+  if (!Load->getType()->isIntegerTy(32))
+    return;
+
+  // TODO: If there is existing range metadata, preserve it if it is stricter.
+  MDBuilder MDB(Load->getContext());
+  MDNode *Range = MDB.createRange(APInt(32, 1), APInt(32, MaxNumGroups + 1));
+  Load->setMetadata(LLVMContext::MD_range, Range);
+}
+
 static bool processUse(CallInst *CI, bool IsV5OrAbove) {
   Function *F = CI->getParent()->getParent();
 
@@ -91,7 +106,11 @@ static bool processUse(CallInst *CI, bool IsV5OrAbove) {
   const bool HasUniformWorkGroupSize =
     F->getFnAttribute("uniform-work-group-size").getValueAsBool();
 
-  if (!HasReqdWorkGroupSize && !HasUniformWorkGroupSize)
+  SmallVector<unsigned> MaxNumWorkgroups =
+      AMDGPU::getIntegerVecAttribute(*F, "amdgpu-max-num-workgroups", 3);
+
+  if (!HasReqdWorkGroupSize && !HasUniformWorkGroupSize &&
+      none_of(MaxNumWorkgroups, [](unsigned X) { return X != 0; }))
     return false;
 
   Value *BlockCounts[3] = {nullptr, nullptr, nullptr};
@@ -132,16 +151,22 @@ static bool processUse(CallInst *CI, bool IsV5OrAbove) {
     if (IsV5OrAbove) { // Base is ImplicitArgPtr.
       switch (Offset) {
       case HIDDEN_BLOCK_COUNT_X:
-        if (LoadSize == 4)
+        if (LoadSize == 4) {
           BlockCounts[0] = Load;
+          annotateGridSizeLoadWithRangeMD(Load, MaxNumWorkgroups[0]);
+        }
         break;
       case HIDDEN_BLOCK_COUNT_Y:
-        if (LoadSize == 4)
+        if (LoadSize == 4) {
           BlockCounts[1] = Load;
+          annotateGridSizeLoadWithRangeMD(Load, MaxNumWorkgroups[1]);
+        }
         break;
       case HIDDEN_BLOCK_COUNT_Z:
-        if (LoadSize == 4)
+        if (LoadSize == 4) {
           BlockCounts[2] = Load;
+          annotateGridSizeLoadWithRangeMD(Load, MaxNumWorkgroups[2]);
+        }
         break;
       case HIDDEN_GROUP_SIZE_X:
         if (LoadSize == 2)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
index 359cd7ad6a714..8eef0c5892109 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp
@@ -44,7 +44,7 @@ class AMDGPUMarkLastScratchLoad : public MachineFunctionPass {
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<SlotIndexesWrapperPass>();
     AU.addRequired<LiveIntervalsWrapperPass>();
-    AU.addRequired<LiveStacks>();
+    AU.addRequired<LiveStacksWrapperLegacy>();
     AU.setPreservesAll();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
@@ -64,7 +64,7 @@ bool AMDGPUMarkLastScratchLoad::runOnMachineFunction(MachineFunction &MF) {
   if (ST.getGeneration() < AMDGPUSubtarget::GFX12)
     return false;
 
-  LS = &getAnalysis<LiveStacks>();
+  LS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
   LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
   SI = &getAnalysis<SlotIndexesWrapperPass>().getSI();
   SII = ST.getInstrInfo();
@@ -137,6 +137,6 @@ char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoad::ID;
 INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoad, DEBUG_TYPE,
                       "AMDGPU Mark last scratch load", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LiveStacks)
+INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
 INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoad, DEBUG_TYPE,
                     "AMDGPU Mark last scratch load", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index c1425292c14e8..cca2a6ec1ff18 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -102,6 +102,7 @@ MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
 MACHINE_FUNCTION_PASS("gcn-dpp-combine", GCNDPPCombinePass())
 MACHINE_FUNCTION_PASS("si-load-store-opt", SILoadStoreOptimizerPass())
 MACHINE_FUNCTION_PASS("si-lower-sgpr-spills", SILowerSGPRSpillsPass())
+MACHINE_FUNCTION_PASS("si-opt-vgpr-liverange", SIOptimizeVGPRLiveRangePass())
 MACHINE_FUNCTION_PASS("si-peephole-sdwa", SIPeepholeSDWAPass())
 MACHINE_FUNCTION_PASS("si-pre-allocate-wwm-regs", SIPreAllocateWWMRegsPass())
 MACHINE_FUNCTION_PASS("si-shrink-instructions", SIShrinkInstructionsPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
new file mode 100644
index 0000000000000..283173deaeedc
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
@@ -0,0 +1,79 @@
+//===-- AMDGPURegBankLegalize.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Lower G_ instructions that can't be inst-selected with register bank
+/// assignment from AMDGPURegBankSelect based on machine uniformity info.
+/// Given types on all operands, some register bank assignments require lowering
+/// while others do not.
+/// Note: cases where all register bank assignments would require lowering are
+/// lowered in legalizer.
+/// For example vgpr S64 G_AND requires lowering to S32 while sgpr S64 does not.
+/// Eliminate sgpr S1 by lowering to sgpr S32.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+#define DEBUG_TYPE "amdgpu-regbanklegalize"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPURegBankLegalize : public MachineFunctionPass {
+public:
+  static char ID;
+
+public:
+  AMDGPURegBankLegalize() : MachineFunctionPass(ID) {
+    initializeAMDGPURegBankLegalizePass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return "AMDGPU Register Bank Legalize";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  // If there were no phis and we do waterfall expansion machine verifier would
+  // fail.
+  MachineFunctionProperties getClearedProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPURegBankLegalize, DEBUG_TYPE,
+                      "AMDGPU Register Bank Legalize", false, false)
+INITIALIZE_PASS_END(AMDGPURegBankLegalize, DEBUG_TYPE,
+                    "AMDGPU Register Bank Legalize", false, false)
+
+char AMDGPURegBankLegalize::ID = 0;
+
+char &llvm::AMDGPURegBankLegalizeID = AMDGPURegBankLegalize::ID;
+
+FunctionPass *llvm::createAMDGPURegBankLegalizePass() {
+  return new AMDGPURegBankLegalize();
+}
+
+using namespace AMDGPU;
+
+bool AMDGPURegBankLegalize::runOnMachineFunction(MachineFunction &MF) {
+  if (MF.getProperties().hasProperty(
+          MachineFunctionProperties::Property::FailedISel))
+    return false;
+  return true;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
index d1985f46b1c44..4c499cb4dfe20 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
@@ -1,4 +1,4 @@
-//===- AMDGPURegBankSelect.cpp -----------------------------------*- C++ -*-==//
+//===-- AMDGPURegBankSelect.cpp -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,73 +6,69 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Use MachineUniformityAnalysis as the primary basis for making SGPR vs. VGPR
-// register bank selection. Use/def analysis as in the default RegBankSelect can
-// be useful in narrower circumstances (e.g. choosing AGPR vs. VGPR for gfx908).
-//
+/// Assign register banks to all register operands of G_ instructions using
+/// machine uniformity analysis.
+/// Sgpr - uniform values and some lane masks
+/// Vgpr - divergent, non S1, values
+/// Vcc  - divergent S1 values(lane masks)
+/// However in some cases G_ instructions with this register bank assignment
+/// can't be inst-selected. This is solved in AMDGPURegBankLegalize.
 //===----------------------------------------------------------------------===//
 
-#include "AMDGPURegBankSelect.h"
 #include "AMDGPU.h"
-#include "GCNSubtarget.h"
-#include "llvm/CodeGen/MachineUniformityAnalysis.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/InitializePasses.h"
 
-#define DEBUG_TYPE "regbankselect"
+#define DEBUG_TYPE "amdgpu-regbankselect"
 
 using namespace llvm;
 
-AMDGPURegBankSelect::AMDGPURegBankSelect(Mode RunningMode)
-    : RegBankSelect(AMDGPURegBankSelect::ID, RunningMode) {}
+namespace {
 
-char AMDGPURegBankSelect::ID = 0;
+class AMDGPURegBankSelect : public MachineFunctionPass {
+public:
+  static char ID;
 
-StringRef AMDGPURegBankSelect::getPassName() const {
-  return "AMDGPURegBankSelect";
-}
+  AMDGPURegBankSelect() : MachineFunctionPass(ID) {
+    initializeAMDGPURegBankSelectPass(*PassRegistry::getPassRegistry());
+  }
 
-void AMDGPURegBankSelect::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<MachineCycleInfoWrapperPass>();
-  AU.addRequired<MachineDominatorTreeWrapperPass>();
-  // TODO: Preserve DomTree
-  RegBankSelect::getAnalysisUsage(AU);
-}
+  bool runOnMachineFunction(MachineFunction &MF) override;
 
-INITIALIZE_PASS_BEGIN(AMDGPURegBankSelect, "amdgpu-" DEBUG_TYPE,
-                      "AMDGPU Register Bank Select", false, false)
-INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
-INITIALIZE_PASS_END(AMDGPURegBankSelect, "amdgpu-" DEBUG_TYPE,
-                    "AMDGPU Register Bank Select", false, false)
+  StringRef getPassName() const override {
+    return "AMDGPU Register Bank Select";
+  }
 
-bool AMDGPURegBankSelect::runOnMachineFunction(MachineFunction &MF) {
-  // If the ISel pipeline failed, do not bother running that pass.
-  if (MF.getProperties().hasProperty(
-          MachineFunctionProperties::Property::FailedISel))
-    return false;
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
 
-  LLVM_DEBUG(dbgs() << "Assign register banks for: " << MF.getName() << '\n');
-  const Function &F = MF.getFunction();
-  Mode SaveOptMode = OptMode;
-  if (F.hasOptNone())
-    OptMode = Mode::Fast;
-  init(MF);
+  // This pass assigns register banks to all virtual registers, and we maintain
+  // this property in subsequent passes
+  MachineFunctionProperties getSetProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::RegBankSelected);
+  }
+};
 
-  assert(checkFunctionIsLegal(MF));
+} // End anonymous namespace.
 
-  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  MachineCycleInfo &CycleInfo =
-      getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
-  MachineDominatorTree &DomTree =
-      getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+INITIALIZE_PASS_BEGIN(AMDGPURegBankSelect, DEBUG_TYPE,
+                      "AMDGPU Register Bank Select", false, false)
+INITIALIZE_PASS_END(AMDGPURegBankSelect, DEBUG_TYPE,
+                    "AMDGPU Register Bank Select", false, false)
 
-  MachineUniformityInfo Uniformity =
-      computeMachineUniformityInfo(MF, CycleInfo, DomTree.getBase(),
-                                   !ST.isSingleLaneExecution(F));
-  (void)Uniformity; // TODO: Use this
+char AMDGPURegBankSelect::ID = 0;
 
-  assignRegisterBanks(MF);
+char &llvm::AMDGPURegBankSelectID = AMDGPURegBankSelect::ID;
+
+FunctionPass *llvm::createAMDGPURegBankSelectPass() {
+  return new AMDGPURegBankSelect();
+}
 
-  OptMode = SaveOptMode;
-  return false;
+bool AMDGPURegBankSelect::runOnMachineFunction(MachineFunction &MF) {
+  if (MF.getProperties().hasProperty(
+          MachineFunctionProperties::Property::FailedISel))
+    return false;
+  return true;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.h
deleted file mode 100644
index 83e4a6b41da1f..0000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===- AMDGPURegBankSelect.h -------------------------------------*- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKSELECT_H
-#define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKSELECT_H
-
-#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
-
-namespace llvm {
-
-class AMDGPURegBankSelect final : public RegBankSelect {
-public:
-  static char ID;
-
-  AMDGPURegBankSelect(Mode RunningMode = Fast);
-
-  StringRef getPassName() const override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  bool runOnMachineFunction(MachineFunction &MF) override;
-};
-
-} // namespace llvm
-#endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
index 3b6c2cbe390ed..65aee2f70104b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -158,13 +158,12 @@ static auto formatRatioOf(CostType Num, CostType Dem) {
 /// Non-copyable functions cannot be cloned into multiple partitions, and only
 /// one copy of the function can be present across all partitions.
 ///
-/// External functions fall into this category. If we were to clone them, we
-/// would end up with multiple symbol definitions and a very unhappy linker.
+/// Kernel functions and external functions fall into this category. If we were
+/// to clone them, we would end up with multiple symbol definitions and a very
+/// unhappy linker.
 static bool isNonCopyable(const Function &F) {
-  assert(AMDGPU::isEntryFunctionCC(F.getCallingConv())
-             ? F.hasExternalLinkage()
-             : true && "Kernel w/o external linkage?");
-  return F.hasExternalLinkage() || !F.isDefinitionExact();
+  return F.hasExternalLinkage() || !F.isDefinitionExact() ||
+         AMDGPU::isEntryFunctionCC(F.getCallingConv());
 }
 
 /// If \p GV has local linkage, make it external + hidden.
@@ -1101,10 +1100,10 @@ void RecursiveSearchSplitting::pickPartition(unsigned Depth, unsigned Idx,
       if (Entry.CostExcludingGraphEntryPoints > LargeClusterThreshold) {
         // Check if the amount of code in common makes it worth it.
         assert(SimilarDepsCost && Entry.CostExcludingGraphEntryPoints);
-        const double Ratio =
-            SimilarDepsCost / Entry.CostExcludingGraphEntryPoints;
+        const double Ratio = static_cast<double>(SimilarDepsCost) /
+                             Entry.CostExcludingGraphEntryPoints;
         assert(Ratio >= 0.0 && Ratio <= 1.0);
-        if (LargeFnOverlapForMerge > Ratio) {
+        if (Ratio > LargeFnOverlapForMerge) {
           // For debug, just print "L", so we'll see "L3=P3" for instance, which
           // will mean we reached max depth and chose P3 based on this
           // heuristic.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 095184d96720a..ae563df2a7a12 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -366,6 +366,7 @@ const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Funct
       TM.getSubtarget<R600Subtarget>(F));
 }
 
+// FIXME: This has no reason to be in subtarget
 SmallVector<unsigned>
 AMDGPUSubtarget::getMaxNumWorkGroups(const Function &F) const {
   return AMDGPU::getIntegerVecAttribute(F, "amdgpu-max-num-workgroups", 3,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index c5c951b58b8d6..7701fef536584 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -55,6 +55,7 @@ class AMDGPUSubtarget {
   bool HasFP4ConversionScaleInsts = false;
   bool HasFP6BF6ConversionScaleInsts = false;
   bool HasF16BF16ToFP6BF6ConversionScaleInsts = false;
+  bool HasCvtPkF16F32Inst = false;
   bool HasF32ToF16BF16ConversionSRInsts = false;
   bool EnableRealTrue16Insts = false;
   bool HasBF16ConversionInsts = false;
@@ -191,6 +192,8 @@ class AMDGPUSubtarget {
 
   bool hasF16BF16ToFP6BF6ConversionScaleInsts() const { return HasF16BF16ToFP6BF6ConversionScaleInsts; }
 
+  bool hasCvtPkF16F32Inst() const { return HasCvtPkF16F32Inst; }
+
   bool hasF32ToF16BF16ConversionSRInsts() const {
     return HasF32ToF16BF16ConversionSRInsts;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index beb4713f770c0..beee2b535b1c4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -23,7 +23,6 @@
 #include "AMDGPUISelDAGToDAG.h"
 #include "AMDGPUMacroFusion.h"
 #include "AMDGPUPerfHintAnalysis.h"
-#include "AMDGPURegBankSelect.h"
 #include "AMDGPUSplitModule.h"
 #include "AMDGPUTargetObjectFile.h"
 #include "AMDGPUTargetTransformInfo.h"
@@ -40,6 +39,7 @@
 #include "SILowerSGPRSpills.h"
 #include "SIMachineFunctionInfo.h"
 #include "SIMachineScheduler.h"
+#include "SIOptimizeVGPRLiveRange.h"
 #include "SIPeepholeSDWA.h"
 #include "SIPreAllocateWWMRegs.h"
 #include "SIShrinkInstructions.h"
@@ -430,10 +430,10 @@ static cl::opt<bool>
                        cl::desc("Enable loop data prefetch on AMDGPU"),
                        cl::Hidden, cl::init(false));
 
-static cl::opt<bool> EnableMaxIlpSchedStrategy(
-    "amdgpu-enable-max-ilp-scheduling-strategy",
-    cl::desc("Enable scheduling strategy to maximize ILP for a single wave."),
-    cl::Hidden, cl::init(false));
+static cl::opt<std::string>
+    AMDGPUSchedStrategy("amdgpu-sched-strategy",
+                        cl::desc("Select custom AMDGPU scheduling strategy."),
+                        cl::Hidden, cl::init(""));
 
 static cl::opt<bool> EnableRewritePartialRegUses(
     "amdgpu-enable-rewrite-partial-reg-uses",
@@ -450,6 +450,17 @@ static cl::opt<bool>
                            cl::desc("Enable AMDGPUAttributorPass"),
                            cl::init(true), cl::Hidden);
 
+static cl::opt<bool> NewRegBankSelect(
+    "new-reg-bank-select",
+    cl::desc("Run amdgpu-regbankselect and amdgpu-regbanklegalize instead of "
+             "regbankselect"),
+    cl::init(false), cl::Hidden);
+
+static cl::opt<bool> HasClosedWorldAssumption(
+    "amdgpu-link-time-closed-world",
+    cl::desc("Whether has closed-world assumption at link time"),
+    cl::init(false), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -466,6 +477,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeGCNDPPCombineLegacyPass(*PR);
   initializeSILowerI1CopiesLegacyPass(*PR);
   initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
+  initializeAMDGPURegBankSelectPass(*PR);
+  initializeAMDGPURegBankLegalizePass(*PR);
   initializeSILowerWWMCopiesPass(*PR);
   initializeAMDGPUMarkLastScratchLoadPass(*PR);
   initializeSILowerSGPRSpillsLegacyPass(*PR);
@@ -475,7 +488,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeSIPeepholeSDWALegacyPass(*PR);
   initializeSIShrinkInstructionsLegacyPass(*PR);
   initializeSIOptimizeExecMaskingPreRAPass(*PR);
-  initializeSIOptimizeVGPRLiveRangePass(*PR);
+  initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
   initializeSILoadStoreOptimizerLegacyPass(*PR);
   initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
   initializeAMDGPUAlwaysInlinePass(*PR);
@@ -492,7 +505,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUPostLegalizerCombinerPass(*PR);
   initializeAMDGPUPreLegalizerCombinerPass(*PR);
   initializeAMDGPURegBankCombinerPass(*PR);
-  initializeAMDGPURegBankSelectPass(*PR);
   initializeAMDGPUPromoteAllocaPass(*PR);
   initializeAMDGPUPromoteAllocaToVectorPass(*PR);
   initializeAMDGPUCodeGenPreparePass(*PR);
@@ -566,6 +578,18 @@ createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
   return DAG;
 }
 
+static ScheduleDAGInstrs *
+createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
+  ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
+      C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(C));
+  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+  DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
+  return DAG;
+}
+
 static ScheduleDAGInstrs *
 createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
@@ -606,6 +630,10 @@ static MachineSchedRegistry
     GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
                            createGCNMaxILPMachineScheduler);
 
+static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
+    "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
+    createGCNMaxMemoryClauseMachineScheduler);
+
 static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
     "gcn-iterative-max-occupancy-experimental",
     "Run GCN scheduler to maximize occupancy (experimental)",
@@ -848,8 +876,12 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
             PM.addPass(InternalizePass(mustPreserveGV));
             PM.addPass(GlobalDCEPass());
           }
-          if (EnableAMDGPUAttributor)
-            PM.addPass(AMDGPUAttributorPass(*this));
+          if (EnableAMDGPUAttributor) {
+            AMDGPUAttributorOptions Opt;
+            if (HasClosedWorldAssumption)
+              Opt.IsClosedWorld = true;
+            PM.addPass(AMDGPUAttributorPass(*this, Opt));
+          }
         }
       });
 
@@ -1305,9 +1337,18 @@ ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
   if (ST.enableSIScheduler())
     return createSIMachineScheduler(C);
 
-  if (EnableMaxIlpSchedStrategy)
+  Attribute SchedStrategyAttr =
+      C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy");
+  StringRef SchedStrategy = SchedStrategyAttr.isValid()
+                                ? SchedStrategyAttr.getValueAsString()
+                                : AMDGPUSchedStrategy;
+
+  if (SchedStrategy == "max-ilp")
     return createGCNMaxILPMachineScheduler(C);
 
+  if (SchedStrategy == "max-memory-clause")
+    return createGCNMaxMemoryClauseMachineScheduler(C);
+
   return createGCNMaxOccupancyMachineScheduler(C);
 }
 
@@ -1404,7 +1445,12 @@ void GCNPassConfig::addPreRegBankSelect() {
 }
 
 bool GCNPassConfig::addRegBankSelect() {
-  addPass(new AMDGPURegBankSelect());
+  if (NewRegBankSelect) {
+    addPass(createAMDGPURegBankSelectPass());
+    addPass(createAMDGPURegBankLegalizePass());
+  } else {
+    addPass(new RegBankSelect());
+  }
   return false;
 }
 
@@ -1441,7 +1487,7 @@ void GCNPassConfig::addOptimizedRegAlloc() {
   // the register in LiveVariables, this would trigger a failure in verifier,
   // we should fix it and enable the verifier.
   if (OptVGPRLiveRange)
-    insertPass(&LiveVariablesID, &SIOptimizeVGPRLiveRangeID);
+    insertPass(&LiveVariablesID, &SIOptimizeVGPRLiveRangeLegacyID);
 
   // This must be run immediately after phi elimination and before
   // TwoAddressInstructions, otherwise the processing of the tied operand of
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 10956861650ab..585f38fc02c29 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -220,6 +220,10 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0,
                                  const Value *Op1, InstCombiner &IC) const;
+
+  bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II,
+                                   unsigned LaneAgIdx) const;
+
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                     IntrinsicInst &II) const;
   std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 37a8c7fef7d0a..ed956a1f755c0 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1920,9 +1920,6 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   ParseStatus parseEndpgm(OperandVector &Operands);
 
   ParseStatus parseVOPD(OperandVector &Operands);
-
-  ParseStatus parseBitOp3(OperandVector &Operands);
-  AMDGPUOperand::Ptr defaultBitOp3() const;
 };
 
 } // end anonymous namespace
@@ -9796,20 +9793,6 @@ ParseStatus AMDGPUAsmParser::parseEndpgm(OperandVector &Operands) {
 
 bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmTyEndpgm); }
 
-//===----------------------------------------------------------------------===//
-// BITOP3
-//===----------------------------------------------------------------------===//
-
-ParseStatus AMDGPUAsmParser::parseBitOp3(OperandVector &Operands) {
-  ParseStatus Res =
-      parseIntWithPrefix("bitop3", Operands, AMDGPUOperand::ImmTyBitOp3);
-  return Res;
-}
-
-AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBitOp3() const {
-  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyBitOp3);
-}
-
 //===----------------------------------------------------------------------===//
 // Split Barrier
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b1d0bb1bc5ded..fbec3f89a33d6 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -93,6 +93,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUPromoteAlloca.cpp
   AMDGPUPromoteKernelArguments.cpp
   AMDGPURegBankCombiner.cpp
+  AMDGPURegBankLegalize.cpp
   AMDGPURegBankSelect.cpp
   AMDGPURegisterBankInfo.cpp
   AMDGPURemoveIncompatibleFunctions.cpp
diff --git a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
index 0541f0f656327..383e3371993d6 100644
--- a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
@@ -193,3 +193,8 @@ multiclass DSDIR_Real_gfx12<bits<2> op> {
 
 defm DS_PARAM_LOAD : DSDIR_Real_gfx12<0x0>;
 defm DS_DIRECT_LOAD : DSDIR_Real_gfx12<0x1>;
+
+let SubtargetPredicate = isGFX12Plus in {
+  def : AMDGPUMnemonicAlias<"lds_param_load", "ds_param_load">;
+  def : AMDGPUMnemonicAlias<"lds_direct_load", "ds_direct_load">;
+}
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 7cbd6d2dc6209..ef618727258cf 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1202,6 +1202,8 @@ let SubtargetPredicate = HasGFX950Insts in {
   def : DSLoadTrPat <DS_READ_B64_TR_B8,  v2i32, int_amdgcn_ds_read_tr8_b64>;
   def : DSLoadTrPat <DS_READ_B96_TR_B6,  v3i32, int_amdgcn_ds_read_tr6_b96>;
   def : DSLoadTrPat <DS_READ_B64_TR_B16, v4i16, int_amdgcn_ds_read_tr16_b64>;
+  def : DSLoadTrPat <DS_READ_B64_TR_B16, v4f16, int_amdgcn_ds_read_tr16_b64>;
+  def : DSLoadTrPat <DS_READ_B64_TR_B16, v4bf16, int_amdgcn_ds_read_tr16_b64>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 72b13fd5f3695..9244b4a664986 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -207,14 +207,13 @@ class FLAT_Load_Pseudo <string opName, RegisterClass regClass,
   opName,
   (outs vdata_op:$vdst),
   !con(
-    !con(
-      !if(EnableSaddr,
-        (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
-        (ins VReg_64:$vaddr)),
-        (ins flat_offset:$offset)),
-        // FIXME: Operands with default values do not work with following non-optional operands.
-        !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
-                           (ins CPol_0:$cpol))),
+    !if(EnableSaddr,
+      (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
+      (ins VReg_64:$vaddr)),
+      (ins flat_offset:$offset),
+      // FIXME: Operands with default values do not work with following non-optional operands.
+      !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
+                         (ins CPol_0:$cpol))),
   " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> {
   let has_data = 0;
   let mayLoad = 1;
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 4c37ef8855a5b..ecf03b14143ee 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -909,8 +909,9 @@ getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
 
   // There are three different types of instructions
   // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
-  // which write hi bits (e.g. op_sel[3] == 1), and 3. CVR_SR_FP8_F32 and
-  // CVT_SR_BF8_F32 with op_sel[3:2]
+  // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
+  // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
+  // op_sel[3:2]
   // != 0
   if (SIInstrInfo::isSDWA(MI)) {
     // Type 1: SDWA with dst_sel != DWORD
@@ -918,8 +919,8 @@ getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
       if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
         return nullptr;
   } else {
-    // Type 2 && Type 3: (VOP3 which write the hi bits) || (CVT_SR_FP8_F32 and
-    // CVT_SR_BF8_F32 with op_sel[3:2] != 0)
+    // Type 2 && Type 3: (VOP3 which write the hi bits) || (FP8DstSelInst
+    // with op_sel[3:2] != 0)
     if (!AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel) ||
         !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
               SISrcMods::DST_OP_SEL ||
@@ -983,7 +984,7 @@ int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
   }
 
-  if (ST.hasDstSelForwardingHazard()) {
+  if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
     const int Shift16DefWaitstates = 1;
 
     auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
@@ -1094,7 +1095,8 @@ int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
   // problematic thus far.
 
   // see checkVALUHazards()
-  if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard())
+  if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
+      !ST.hasCvtScaleForwardingHazard())
     return 0;
 
   const MachineRegisterInfo &MRI = MF.getRegInfo();
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 57f517bfba0eb..1c23b237eaf4b 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -615,6 +615,138 @@ bool GCNMaxILPSchedStrategy::tryCandidate(SchedCandidate &Cand,
   return false;
 }
 
+GCNMaxMemoryClauseSchedStrategy::GCNMaxMemoryClauseSchedStrategy(
+    const MachineSchedContext *C)
+    : GCNSchedStrategy(C) {
+  SchedStages.push_back(GCNSchedStageID::MemoryClauseInitialSchedule);
+}
+
+/// GCNMaxMemoryClauseSchedStrategy tries best to clause memory instructions as
+/// much as possible. This is achieved by:
+//  1. Prioritize clustered operations before stall latency heuristic.
+//  2. Prioritize long-latency-load before stall latency heuristic.
+///
+/// \param Cand provides the policy and current best candidate.
+/// \param TryCand refers to the next SUnit candidate, otherwise uninitialized.
+/// \param Zone describes the scheduled zone that we are extending, or nullptr
+///             if Cand is from a different zone than TryCand.
+/// \return \c true if TryCand is better than Cand (Reason is NOT NoCand)
+bool GCNMaxMemoryClauseSchedStrategy::tryCandidate(SchedCandidate &Cand,
+                                                   SchedCandidate &TryCand,
+                                                   SchedBoundary *Zone) const {
+  // Initialize the candidate if needed.
+  if (!Cand.isValid()) {
+    TryCand.Reason = NodeOrder;
+    return true;
+  }
+
+  // Bias PhysReg Defs and copies to their uses and defined respectively.
+  if (tryGreater(biasPhysReg(TryCand.SU, TryCand.AtTop),
+                 biasPhysReg(Cand.SU, Cand.AtTop), TryCand, Cand, PhysReg))
+    return TryCand.Reason != NoCand;
+
+  if (DAG->isTrackingPressure()) {
+    // Avoid exceeding the target's limit.
+    if (tryPressure(TryCand.RPDelta.Excess, Cand.RPDelta.Excess, TryCand, Cand,
+                    RegExcess, TRI, DAG->MF))
+      return TryCand.Reason != NoCand;
+
+    // Avoid increasing the max critical pressure in the scheduled region.
+    if (tryPressure(TryCand.RPDelta.CriticalMax, Cand.RPDelta.CriticalMax,
+                    TryCand, Cand, RegCritical, TRI, DAG->MF))
+      return TryCand.Reason != NoCand;
+  }
+
+  // MaxMemoryClause-specific: We prioritize clustered instructions as we would
+  // get more benefit from clausing these memory instructions.
+  const SUnit *CandNextClusterSU =
+      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  const SUnit *TryCandNextClusterSU =
+      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
+                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
+    return TryCand.Reason != NoCand;
+
+  // We only compare a subset of features when comparing nodes between
+  // Top and Bottom boundary. Some properties are simply incomparable, in many
+  // other instances we should only override the other boundary if something
+  // is a clear good pick on one boundary. Skip heuristics that are more
+  // "tie-breaking" in nature.
+  bool SameBoundary = Zone != nullptr;
+  if (SameBoundary) {
+    // For loops that are acyclic path limited, aggressively schedule for
+    // latency. Within an single cycle, whenever CurrMOps > 0, allow normal
+    // heuristics to take precedence.
+    if (Rem.IsAcyclicLatencyLimited && !Zone->getCurrMOps() &&
+        tryLatency(TryCand, Cand, *Zone))
+      return TryCand.Reason != NoCand;
+
+    // MaxMemoryClause-specific: Prioritize long latency memory load
+    // instructions in top-bottom order to hide more latency. The mayLoad check
+    // is used to exclude store-like instructions, which we do not want to
+    // scheduler them too early.
+    bool TryMayLoad =
+        TryCand.SU->isInstr() && TryCand.SU->getInstr()->mayLoad();
+    bool CandMayLoad = Cand.SU->isInstr() && Cand.SU->getInstr()->mayLoad();
+
+    if (TryMayLoad || CandMayLoad) {
+      bool TryLongLatency =
+          TryCand.SU->Latency > 10 * Cand.SU->Latency && TryMayLoad;
+      bool CandLongLatency =
+          10 * TryCand.SU->Latency < Cand.SU->Latency && CandMayLoad;
+
+      if (tryGreater(Zone->isTop() ? TryLongLatency : CandLongLatency,
+                     Zone->isTop() ? CandLongLatency : TryLongLatency, TryCand,
+                     Cand, Stall))
+        return TryCand.Reason != NoCand;
+    }
+    // Prioritize instructions that read unbuffered resources by stall cycles.
+    if (tryLess(Zone->getLatencyStallCycles(TryCand.SU),
+                Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
+      return TryCand.Reason != NoCand;
+  }
+
+  if (SameBoundary) {
+    // Weak edges are for clustering and other constraints.
+    if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop),
+                getWeakLeft(Cand.SU, Cand.AtTop), TryCand, Cand, Weak))
+      return TryCand.Reason != NoCand;
+  }
+
+  // Avoid increasing the max pressure of the entire region.
+  if (DAG->isTrackingPressure() &&
+      tryPressure(TryCand.RPDelta.CurrentMax, Cand.RPDelta.CurrentMax, TryCand,
+                  Cand, RegMax, TRI, DAG->MF))
+    return TryCand.Reason != NoCand;
+
+  if (SameBoundary) {
+    // Avoid critical resource consumption and balance the schedule.
+    TryCand.initResourceDelta(DAG, SchedModel);
+    if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
+                TryCand, Cand, ResourceReduce))
+      return TryCand.Reason != NoCand;
+    if (tryGreater(TryCand.ResDelta.DemandedResources,
+                   Cand.ResDelta.DemandedResources, TryCand, Cand,
+                   ResourceDemand))
+      return TryCand.Reason != NoCand;
+
+    // Avoid serializing long latency dependence chains.
+    // For acyclic path limited loops, latency was already checked above.
+    if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
+        !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, *Zone))
+      return TryCand.Reason != NoCand;
+
+    // Fall through to original instruction order.
+    if (Zone->isTop() == (TryCand.SU->NodeNum < Cand.SU->NodeNum)) {
+      assert(TryCand.SU->NodeNum != Cand.SU->NodeNum);
+      TryCand.Reason = NodeOrder;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 GCNScheduleDAGMILive::GCNScheduleDAGMILive(
     MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S)
     : ScheduleDAGMILive(C, std::move(S)), ST(MF.getSubtarget<GCNSubtarget>()),
@@ -644,6 +776,9 @@ GCNScheduleDAGMILive::createSchedStage(GCNSchedStageID SchedStageID) {
     return std::make_unique<PreRARematStage>(SchedStageID, *this);
   case GCNSchedStageID::ILPInitialSchedule:
     return std::make_unique<ILPInitialScheduleStage>(SchedStageID, *this);
+  case GCNSchedStageID::MemoryClauseInitialSchedule:
+    return std::make_unique<MemoryClauseInitialScheduleStage>(SchedStageID,
+                                                              *this);
   }
 
   llvm_unreachable("Unknown SchedStageID.");
@@ -869,6 +1004,9 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const GCNSchedStageID &StageID) {
   case GCNSchedStageID::ILPInitialSchedule:
     OS << "Max ILP Initial Schedule";
     break;
+  case GCNSchedStageID::MemoryClauseInitialSchedule:
+    OS << "Max memory clause Initial Schedule";
+    break;
   }
 
   return OS;
@@ -1088,7 +1226,8 @@ void GCNSchedStage::setupNewBlock() {
   // Get real RP for the region if it hasn't be calculated before. After the
   // initial schedule stage real RP will be collected after scheduling.
   if (StageID == GCNSchedStageID::OccInitialSchedule ||
-      StageID == GCNSchedStageID::ILPInitialSchedule)
+      StageID == GCNSchedStageID::ILPInitialSchedule ||
+      StageID == GCNSchedStageID::MemoryClauseInitialSchedule)
     DAG.computeBlockPressure(RegionIdx, CurrentMBB);
 }
 
@@ -1389,6 +1528,11 @@ bool ILPInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
   return false;
 }
 
+bool MemoryClauseInitialScheduleStage::shouldRevertScheduling(
+    unsigned WavesAfter) {
+  return mayCauseSpilling(WavesAfter);
+}
+
 bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
   if (WavesAfter <= MFI.getMinWavesPerEU() && isRegionWithExcessRP() &&
       !PressureAfter.less(MF, PressureBefore)) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 64d517038f90e..44db834a41f82 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -29,7 +29,8 @@ enum class GCNSchedStageID : unsigned {
   UnclusteredHighRPReschedule = 1,
   ClusteredLowOccupancyReschedule = 2,
   PreRARematerialize = 3,
-  ILPInitialSchedule = 4
+  ILPInitialSchedule = 4,
+  MemoryClauseInitialSchedule = 5
 };
 
 #ifndef NDEBUG
@@ -149,6 +150,17 @@ class GCNMaxILPSchedStrategy final : public GCNSchedStrategy {
   GCNMaxILPSchedStrategy(const MachineSchedContext *C);
 };
 
+/// The goal of this scheduling strategy is to maximize memory clause for a
+/// single wave.
+class GCNMaxMemoryClauseSchedStrategy final : public GCNSchedStrategy {
+protected:
+  bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
+                    SchedBoundary *Zone) const override;
+
+public:
+  GCNMaxMemoryClauseSchedStrategy(const MachineSchedContext *C);
+};
+
 class ScheduleMetrics {
   unsigned ScheduleLength;
   unsigned BubbleCycles;
@@ -463,6 +475,15 @@ class ILPInitialScheduleStage : public GCNSchedStage {
       : GCNSchedStage(StageID, DAG) {}
 };
 
+class MemoryClauseInitialScheduleStage : public GCNSchedStage {
+public:
+  bool shouldRevertScheduling(unsigned WavesAfter) override;
+
+  MemoryClauseInitialScheduleStage(GCNSchedStageID StageID,
+                                   GCNScheduleDAGMILive &DAG)
+      : GCNSchedStage(StageID, DAG) {}
+};
+
 class GCNPostScheduleDAGMILive final : public ScheduleDAGMI {
 private:
   std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index ea5e159fdd836..5cecaf6349c88 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1264,6 +1264,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool hasVALUTransUseHazard() const { return HasVALUTransUseHazard; }
 
+  bool hasCvtScaleForwardingHazard() const { return GFX950Insts; }
+
   bool hasForceStoreSC0SC1() const { return HasForceStoreSC0SC1; }
 
   bool requiresCodeObjectV6() const { return RequiresCOV6; }
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 1005f21808e8b..8d115268d505b 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -1793,6 +1793,11 @@ defm IMAGE_BVH64_INTERSECT_RAY     : MIMG_IntersectRay<mimgopc<0x1a, 0x1a, 0xe7>
 defm IMAGE_BVH64_INTERSECT_RAY_a16 : MIMG_IntersectRay<mimgopc<0x1a, 0x1a, 0xe7>, "image_bvh64_intersect_ray", 1, 1>;
 } // End OtherPredicates = [HasImageInsts, HasGFX10_AEncoding]
 
+let SubtargetPredicate = isGFX12Plus in {
+  def : AMDGPUMnemonicAlias<"bvh_intersect_ray", "image_bvh_intersect_ray">;
+  def : AMDGPUMnemonicAlias<"bvh64_intersect_ray", "image_bvh64_intersect_ray">;
+}
+
 } // End let OtherPredicates = [HasImageInsts]
 
 /********** ========================================= **********/
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index ea653490f1bf3..4ff6fc32b642d 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -38,6 +38,7 @@ using StackVector = SmallVector<StackEntry, 16>;
 
 class SIAnnotateControlFlow {
 private:
+  Function *F;
   UniformityInfo *UA;
 
   Type *Boolean;
@@ -50,18 +51,18 @@ class SIAnnotateControlFlow {
   UndefValue *BoolUndef;
   Constant *IntMaskZero;
 
-  Function *If;
-  Function *Else;
-  Function *IfBreak;
-  Function *Loop;
-  Function *EndCf;
+  Function *If = nullptr;
+  Function *Else = nullptr;
+  Function *IfBreak = nullptr;
+  Function *Loop = nullptr;
+  Function *EndCf = nullptr;
 
   DominatorTree *DT;
   StackVector Stack;
 
   LoopInfo *LI;
 
-  void initialize(Module &M, const GCNSubtarget &ST);
+  void initialize(const GCNSubtarget &ST);
 
   bool isUniform(BranchInst *T);
 
@@ -89,21 +90,27 @@ class SIAnnotateControlFlow {
 
   bool closeControlFlow(BasicBlock *BB);
 
+  Function *getDecl(Function *&Cache, Intrinsic::ID ID, ArrayRef<Type *> Tys) {
+    if (!Cache)
+      Cache = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
+    return Cache;
+  }
+
 public:
-  SIAnnotateControlFlow(Module &M, const GCNSubtarget &ST, DominatorTree &DT,
+  SIAnnotateControlFlow(Function &F, const GCNSubtarget &ST, DominatorTree &DT,
                         LoopInfo &LI, UniformityInfo &UA)
-      : UA(&UA), DT(&DT), LI(&LI) {
-    initialize(M, ST);
+      : F(&F), UA(&UA), DT(&DT), LI(&LI) {
+    initialize(ST);
   }
 
-  bool run(Function &F);
+  bool run();
 };
 
 } // end anonymous namespace
 
 /// Initialize all the types and constants used in the pass
-void SIAnnotateControlFlow::initialize(Module &M, const GCNSubtarget &ST) {
-  LLVMContext &Context = M.getContext();
+void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) {
+  LLVMContext &Context = F->getContext();
 
   Void = Type::getVoidTy(Context);
   Boolean = Type::getInt1Ty(Context);
@@ -115,16 +122,6 @@ void SIAnnotateControlFlow::initialize(Module &M, const GCNSubtarget &ST) {
   BoolFalse = ConstantInt::getFalse(Context);
   BoolUndef = PoisonValue::get(Boolean);
   IntMaskZero = ConstantInt::get(IntMask, 0);
-
-  If = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_if, {IntMask});
-  Else = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_else,
-                                           {IntMask, IntMask});
-  IfBreak = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_if_break,
-                                              {IntMask});
-  Loop =
-      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_loop, {IntMask});
-  EndCf = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_end_cf,
-                                            {IntMask});
 }
 
 /// Is the branch condition uniform or did the StructurizeCFG pass
@@ -190,7 +187,8 @@ bool SIAnnotateControlFlow::openIf(BranchInst *Term) {
     return false;
 
   IRBuilder<> IRB(Term);
-  Value *IfCall = IRB.CreateCall(If, {Term->getCondition()});
+  Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, IntMask),
+                                 {Term->getCondition()});
   Value *Cond = IRB.CreateExtractValue(IfCall, {0});
   Value *Mask = IRB.CreateExtractValue(IfCall, {1});
   Term->setCondition(Cond);
@@ -205,7 +203,8 @@ bool SIAnnotateControlFlow::insertElse(BranchInst *Term) {
   }
 
   IRBuilder<> IRB(Term);
-  Value *ElseCall = IRB.CreateCall(Else, {popSaved()});
+  Value *ElseCall = IRB.CreateCall(
+      getDecl(Else, Intrinsic::amdgcn_else, {IntMask, IntMask}), {popSaved()});
   Value *Cond = IRB.CreateExtractValue(ElseCall, {0});
   Value *Mask = IRB.CreateExtractValue(ElseCall, {1});
   Term->setCondition(Cond);
@@ -218,14 +217,20 @@ Value *SIAnnotateControlFlow::handleLoopCondition(
     Value *Cond, PHINode *Broken, llvm::Loop *L, BranchInst *Term) {
 
   auto CreateBreak = [this, Cond, Broken](Instruction *I) -> CallInst * {
-    return IRBuilder<>(I).CreateCall(IfBreak, {Cond, Broken});
+    return IRBuilder<>(I).CreateCall(
+        getDecl(IfBreak, Intrinsic::amdgcn_if_break, IntMask), {Cond, Broken});
   };
 
   if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
     BasicBlock *Parent = Inst->getParent();
     Instruction *Insert;
-    if (L->contains(Inst)) {
+    if (LI->getLoopFor(Parent) == L) {
+      // Insert IfBreak in the same BB as Cond, which can help
+      // SILowerControlFlow to know that it does not have to insert an
+      // AND with EXEC.
       Insert = Parent->getTerminator();
+    } else if (L->contains(Inst)) {
+      Insert = Term;
     } else {
       Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
     }
@@ -279,7 +284,8 @@ bool SIAnnotateControlFlow::handleLoop(BranchInst *Term) {
     Broken->addIncoming(PHIValue, Pred);
   }
 
-  CallInst *LoopCall = IRBuilder<>(Term).CreateCall(Loop, {Arg});
+  CallInst *LoopCall = IRBuilder<>(Term).CreateCall(
+      getDecl(Loop, Intrinsic::amdgcn_loop, IntMask), {Arg});
   Term->setCondition(LoopCall);
 
   push(Term->getSuccessor(0), Arg);
@@ -324,7 +330,7 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
     // condition, for now just avoid copying these DebugLocs so that stepping
     // out of the then/else block in a debugger doesn't step to the condition.
     IRB.SetCurrentDebugLocation(DebugLoc());
-    IRB.CreateCall(EndCf, {Exec});
+    IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, IntMask), {Exec});
   }
 
   return true;
@@ -332,11 +338,12 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
 
 /// Annotate the control flow with intrinsics so the backend can
 /// recognize if/then/else and loops.
-bool SIAnnotateControlFlow::run(Function &F) {
+bool SIAnnotateControlFlow::run() {
   bool Changed = false;
 
-  for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()),
-       E = df_end(&F.getEntryBlock()); I != E; ++I) {
+  for (df_iterator<BasicBlock *> I = df_begin(&F->getEntryBlock()),
+                                 E = df_end(&F->getEntryBlock());
+       I != E; ++I) {
     BasicBlock *BB = *I;
     BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
 
@@ -386,12 +393,11 @@ PreservedAnalyses SIAnnotateControlFlowPass::run(Function &F,
   UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(F);
   LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
 
-  SIAnnotateControlFlow Impl(*F.getParent(), ST, DT, LI, UI);
+  SIAnnotateControlFlow Impl(F, ST, DT, LI, UI);
 
-  // FIXME: We introduce dead declarations of intrinsics even if never used.
-  bool Changed = Impl.run(F);
+  bool Changed = Impl.run();
   if (!Changed)
-    return PreservedAnalyses::none();
+    return PreservedAnalyses::all();
 
   // TODO: Is LoopInfo preserved?
   PreservedAnalyses PA = PreservedAnalyses::none();
@@ -426,8 +432,8 @@ class SIAnnotateControlFlowLegacy : public FunctionPass {
     const TargetMachine &TM = TPC.getTM<TargetMachine>();
     const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
 
-    SIAnnotateControlFlow Impl(*F.getParent(), ST, DT, LI, UI);
-    return Impl.run(F);
+    SIAnnotateControlFlow Impl(F, ST, DT, LI, UI);
+    return Impl.run();
   }
 };
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a212a9218ca0d..8dfebd36a962e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -902,6 +902,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
   }
 
+  if (Subtarget->hasCvtPkF16F32Inst()) {
+    setOperationAction(ISD::FP_ROUND, MVT::v2f16, Legal);
+  }
+
   setTargetDAGCombine({ISD::ADD,
                        ISD::UADDO_CARRY,
                        ISD::SUB,
@@ -910,6 +914,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
                        ISD::FADD,
                        ISD::FSUB,
                        ISD::FDIV,
+                       ISD::FMUL,
                        ISD::FMINNUM,
                        ISD::FMAXNUM,
                        ISD::FMINNUM_IEEE,
@@ -2533,6 +2538,7 @@ void SITargetLowering::allocatePreloadKernArgSGPRs(
 
       // Arg is preloaded into the previous SGPR.
       if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
+        assert(InIdx >= 1 && "No previous SGPR");
         Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
             Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
         continue;
@@ -2541,8 +2547,7 @@ void SITargetLowering::allocatePreloadKernArgSGPRs(
       unsigned Padding = ArgOffset - LastExplicitArgOffset;
       unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
       // Check for free user SGPRs for preloading.
-      if (PaddingSGPRs + NumAllocSGPRs + 1 /*Synthetic SGPRs*/ >
-          SGPRInfo.getNumFreeUserSGPRs()) {
+      if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
         InPreloadSequence = false;
         break;
       }
@@ -3020,6 +3025,20 @@ SDValue SITargetLowering::LowerFormalArguments(
           NewArg = DAG.getMergeValues({NewArg, Chain}, DL);
         }
       } else {
+        // Hidden arguments that are in the kernel signature must be preloaded
+        // to user SGPRs. Print a diagnostic error if a hidden argument is in
+        // the argument list and is not preloaded.
+        if (Arg.isOrigArg()) {
+          Argument *OrigArg = Fn.getArg(Arg.getOrigArgIndex());
+          if (OrigArg->hasAttribute("amdgpu-hidden-argument")) {
+            DiagnosticInfoUnsupported NonPreloadHiddenArg(
+                *OrigArg->getParent(),
+                "hidden argument in kernel signature was not preloaded",
+                DL.getDebugLoc());
+            DAG.getContext()->diagnose(NonPreloadHiddenArg);
+          }
+        }
+
         NewArg =
             lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset,
                                      Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
@@ -14611,13 +14630,73 @@ SDValue SITargetLowering::performFDivCombine(SDNode *N,
   return SDValue();
 }
 
+SDValue SITargetLowering::performFMulCombine(SDNode *N,
+                                             DAGCombinerInfo &DCI) const {
+  SelectionDAG &DAG = DCI.DAG;
+  EVT VT = N->getValueType(0);
+  EVT ScalarVT = VT.getScalarType();
+  EVT IntVT = VT.changeElementType(MVT::i32);
+
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+
+  // It is cheaper to realize i32 inline constants as compared against
+  // materializing f16 or f64 (or even non-inline f32) values,
+  // possible via ldexp usage, as shown below :
+  //
+  // Given : A = 2^a  &  B = 2^b ; where a and b are integers.
+  // fmul x, (select y, A, B)     -> ldexp( x, (select i32 y, a, b) )
+  // fmul x, (select y, -A, -B)   -> ldexp( (fneg x), (select i32 y, a, b) )
+  if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
+      (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
+    const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(RHS.getOperand(1));
+    if (!TrueNode)
+      return SDValue();
+    const ConstantFPSDNode *FalseNode =
+        isConstOrConstSplatFP(RHS.getOperand(2));
+    if (!FalseNode)
+      return SDValue();
+
+    if (TrueNode->isNegative() != FalseNode->isNegative())
+      return SDValue();
+
+    // For f32, only non-inline constants should be transformed.
+    const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
+    if (ScalarVT == MVT::f32 &&
+        TII->isInlineConstant(TrueNode->getValueAPF()) &&
+        TII->isInlineConstant(FalseNode->getValueAPF()))
+      return SDValue();
+
+    int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
+    if (TrueNodeExpVal == INT_MIN)
+      return SDValue();
+    int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
+    if (FalseNodeExpVal == INT_MIN)
+      return SDValue();
+
+    SDLoc SL(N);
+    SDValue SelectNode =
+        DAG.getNode(ISD::SELECT, SL, IntVT, RHS.getOperand(0),
+                    DAG.getSignedConstant(TrueNodeExpVal, SL, IntVT),
+                    DAG.getSignedConstant(FalseNodeExpVal, SL, IntVT));
+
+    LHS = TrueNode->isNegative()
+              ? DAG.getNode(ISD::FNEG, SL, VT, LHS, LHS->getFlags())
+              : LHS;
+
+    return DAG.getNode(ISD::FLDEXP, SL, VT, LHS, SelectNode, N->getFlags());
+  }
+
+  return SDValue();
+}
+
 SDValue SITargetLowering::performFMACombine(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
   SDLoc SL(N);
 
-  if (!Subtarget->hasDot7Insts() || VT != MVT::f32)
+  if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
     return SDValue();
 
   // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
@@ -14897,6 +14976,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
     return performFSubCombine(N, DCI);
   case ISD::FDIV:
     return performFDivCombine(N, DCI);
+  case ISD::FMUL:
+    return performFMulCombine(N, DCI);
   case ISD::SETCC:
     return performSetCCCombine(N, DCI);
   case ISD::FMAXNUM:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 32e110fdfa84d..631f26542bbe6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -218,6 +218,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFDivCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performFMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFMACombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index e55418326a4bd..960fbb7ea15ce 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1568,6 +1568,11 @@ namespace AMDGPU {
   LLVM_READONLY
   int getMFMAEarlyClobberOp(uint16_t Opcode);
 
+  /// \returns Version of an MFMA instruction which uses AGPRs for srcC and
+  /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.
+  LLVM_READONLY
+  int getMFMASrcCVDstAGPROp(uint16_t Opcode);
+
   /// \returns v_cmpx version of a v_cmp instruction.
   LLVM_READONLY
   int getVCMPXOpFromVCMP(uint16_t Opcode);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index cddb87655ee8c..7bc6db4cec106 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1271,7 +1271,8 @@ def ByteSel : NamedIntOperand<"byte_sel"> {
   let Validator = "isUInt<2>";
 }
 
-def BitOp3 : CustomOperand<i8, 1, "BitOp3">;
+let PrintMethod = "printBitOp3" in
+def BitOp3 : NamedIntOperand<"bitop3">;
 def bitop3_0 : DefaultOperand<BitOp3, 0>;
 
 class KImmFPOperand<ValueType vt> : ImmOperand<vt> {
@@ -3211,6 +3212,15 @@ def getMFMAEarlyClobberOp : InstrMapping {
   let ValueCols = [["0"]];
 }
 
+// Map from an mfma using VGPRs to one using AGPRs.
+def getMFMASrcCVDstAGPROp : InstrMapping {
+  let FilterClass = "MFMATable";
+  let RowFields = ["AGPROp"];
+  let ColFields = ["MFMAKind"];
+  let KeyCol = ["VGPR"];
+  let ValueCols = [["AGPR"]];
+}
+
 // Maps an v_cmp instruction to its v_cmpx equivalent.
 def getVCMPXOpFromVCMP : InstrMapping {
   let FilterClass = "VCMPVCMPXTable";
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
index e173d365ca503..ff0b9b4a7574b 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
@@ -71,6 +71,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "SIOptimizeVGPRLiveRange.h"
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -86,7 +87,7 @@ using namespace llvm;
 
 namespace {
 
-class SIOptimizeVGPRLiveRange : public MachineFunctionPass {
+class SIOptimizeVGPRLiveRange {
 private:
   const SIRegisterInfo *TRI = nullptr;
   const SIInstrInfo *TII = nullptr;
@@ -96,7 +97,10 @@ class SIOptimizeVGPRLiveRange : public MachineFunctionPass {
   MachineRegisterInfo *MRI = nullptr;
 
 public:
-  static char ID;
+  SIOptimizeVGPRLiveRange(LiveVariables *LV, MachineDominatorTree *MDT,
+                          MachineLoopInfo *Loops)
+      : LV(LV), MDT(MDT), Loops(Loops) {}
+  bool run(MachineFunction &MF);
 
   MachineBasicBlock *getElseTarget(MachineBasicBlock *MBB) const;
 
@@ -136,8 +140,13 @@ class SIOptimizeVGPRLiveRange : public MachineFunctionPass {
       Register Reg, MachineBasicBlock *LoopHeader,
       SmallSetVector<MachineBasicBlock *, 2> &LoopBlocks,
       SmallVectorImpl<MachineInstr *> &Instructions) const;
+};
 
-  SIOptimizeVGPRLiveRange() : MachineFunctionPass(ID) {}
+class SIOptimizeVGPRLiveRangeLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  SIOptimizeVGPRLiveRangeLegacy() : MachineFunctionPass(ID) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
@@ -146,6 +155,7 @@ class SIOptimizeVGPRLiveRange : public MachineFunctionPass {
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
     AU.addRequired<LiveVariablesWrapperPass>();
     AU.addRequired<MachineDominatorTreeWrapperPass>();
     AU.addRequired<MachineLoopInfoWrapperPass>();
@@ -611,35 +621,59 @@ void SIOptimizeVGPRLiveRange::optimizeWaterfallLiveRange(
   }
 }
 
-char SIOptimizeVGPRLiveRange::ID = 0;
+char SIOptimizeVGPRLiveRangeLegacy::ID = 0;
 
-INITIALIZE_PASS_BEGIN(SIOptimizeVGPRLiveRange, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(SIOptimizeVGPRLiveRangeLegacy, DEBUG_TYPE,
                       "SI Optimize VGPR LiveRange", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LiveVariablesWrapperPass)
-INITIALIZE_PASS_END(SIOptimizeVGPRLiveRange, DEBUG_TYPE,
+INITIALIZE_PASS_END(SIOptimizeVGPRLiveRangeLegacy, DEBUG_TYPE,
                     "SI Optimize VGPR LiveRange", false, false)
 
-char &llvm::SIOptimizeVGPRLiveRangeID = SIOptimizeVGPRLiveRange::ID;
+char &llvm::SIOptimizeVGPRLiveRangeLegacyID = SIOptimizeVGPRLiveRangeLegacy::ID;
+
+FunctionPass *llvm::createSIOptimizeVGPRLiveRangeLegacyPass() {
+  return new SIOptimizeVGPRLiveRangeLegacy();
+}
+
+bool SIOptimizeVGPRLiveRangeLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
 
-FunctionPass *llvm::createSIOptimizeVGPRLiveRangePass() {
-  return new SIOptimizeVGPRLiveRange();
+  LiveVariables *LV = &getAnalysis<LiveVariablesWrapperPass>().getLV();
+  MachineDominatorTree *MDT =
+      &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+  MachineLoopInfo *Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+  return SIOptimizeVGPRLiveRange(LV, MDT, Loops).run(MF);
 }
 
-bool SIOptimizeVGPRLiveRange::runOnMachineFunction(MachineFunction &MF) {
+PreservedAnalyses
+SIOptimizeVGPRLiveRangePass::run(MachineFunction &MF,
+                                 MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+  LiveVariables *LV = &MFAM.getResult<LiveVariablesAnalysis>(MF);
+  MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
+  MachineLoopInfo *Loops = &MFAM.getResult<MachineLoopAnalysis>(MF);
+
+  bool Changed = SIOptimizeVGPRLiveRange(LV, MDT, Loops).run(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserve<LiveVariablesAnalysis>();
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<MachineLoopAnalysis>();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
 
+bool SIOptimizeVGPRLiveRange::run(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
   TRI = &TII->getRegisterInfo();
-  MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
-  Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
-  LV = &getAnalysis<LiveVariablesWrapperPass>().getLV();
   MRI = &MF.getRegInfo();
 
-  if (skipFunction(MF.getFunction()))
-    return false;
-
   bool MadeChange = false;
 
   // TODO: we need to think about the order of visiting the blocks to get
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.h b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.h
new file mode 100644
index 0000000000000..1139a9c18581e
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.h
@@ -0,0 +1,33 @@
+//===- SIOptimizeVGPRLiveRange.h --------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIOPTIMIZEVGPRLIVERANGE_H
+#define LLVM_LIB_TARGET_AMDGPU_SIOPTIMIZEVGPRLIVERANGE_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class SIOptimizeVGPRLiveRangePass
+    : public PassInfoMixin<SIOptimizeVGPRLiveRangePass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getRequiredProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+
+  MachineFunctionProperties getClearedProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SIOPTIMIZEVGPRLIVERANGE_H
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index ab5f0694c07f9..70deaf82dfaf3 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -378,13 +378,19 @@ struct VOPTrue16Info {
   bool IsTrue16;
 };
 
+#define GET_FP8DstByteSelTable_DECL
+#define GET_FP8DstByteSelTable_IMPL
+
+struct DPMACCInstructionInfo {
+  uint16_t Opcode;
+  bool IsDPMACCInstruction;
+};
+
 struct FP8DstByteSelInfo {
   uint16_t Opcode;
   bool HasFP8DstByteSel;
 };
 
-#define GET_FP8DstByteSelTable_DECL
-#define GET_FP8DstByteSelTable_IMPL
 #define GET_MTBUFInfoTable_DECL
 #define GET_MTBUFInfoTable_IMPL
 #define GET_MUBUFInfoTable_DECL
@@ -418,6 +424,8 @@ struct FP8DstByteSelInfo {
 #define GET_getMFMA_F8F6F4_WithSize_DECL
 #define GET_getMFMA_F8F6F4_WithSize_IMPL
 #define GET_isMFMA_F8F6F4Table_IMPL
+#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
+
 #include "AMDGPUGenSearchableTables.inc"
 
 int getMTBUFBaseOpcode(unsigned Opc) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 9f7fbec6a542f..ea497d7b239d7 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -103,6 +103,10 @@ struct MFMA_F8F6F4_Info {
   uint8_t NumRegsSrcB;
 };
 
+struct CvtScaleF32_F32F16ToF8F4_Info {
+  unsigned Opcode;
+};
+
 #define GET_MIMGBaseOpcode_DECL
 #define GET_MIMGDim_DECL
 #define GET_MIMGEncoding_DECL
@@ -112,6 +116,7 @@ struct MFMA_F8F6F4_Info {
 #define GET_MAIInstInfoTable_DECL
 #define GET_MAIInstInfoTable_DECL
 #define GET_isMFMA_F8F6F4Table_DECL
+#define GET_isCvtScaleF32_F32F16ToF8F4Table_DECL
 #include "AMDGPUGenSearchableTables.inc"
 
 namespace IsaInfo {
diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
index fa06d96085820..5e89e34ca56e9 100644
--- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
@@ -181,9 +181,43 @@ multiclass VInterpF16Pat <SDPatternOperator op, Instruction inst,
   def : VInterpF16Pat<op, inst, dst_type, 1, high_pat>;
 }
 
+class VInterpF16Pat_t16 <SDPatternOperator op, Instruction inst,
+                     ValueType dstVT, bit high, bit isP2> : GCNPat <
+   (dstVT (op
+      (VINTERPMods f32:$src0, i32:$src0_modifiers),
+      (VINTERPMods f32:$src1, i32:$src1_modifiers),
+      (VINTERPMods f32:$src2, i32:$src2_modifiers),
+      !if(high, (i1 -1), (i1 0)))),
+    (inst $src0_modifiers,
+          (f16 (EXTRACT_SUBREG VGPR_32:$src0, !if(high, hi16, lo16))),
+          $src1_modifiers, VGPR_32:$src1,
+          $src2_modifiers,
+          !if(isP2, (f32 VGPR_32:$src2),
+                    (f16 (EXTRACT_SUBREG VGPR_32:$src2, !if(high, hi16, lo16)))),
+          0, /* clamp */
+          7) /* wait_exp */
+>;
+
+multiclass VInterpF16Pat_t16 <SDPatternOperator op, Instruction inst,
+                          ValueType dstVT, bit isP2> {
+  def : VInterpF16Pat_t16<op, inst, dstVT, 0, isP2>;
+  def : VInterpF16Pat_t16<op, inst, dstVT, 1, isP2>;
+}
+
 def : VInterpF32Pat<int_amdgcn_interp_inreg_p10, V_INTERP_P10_F32_inreg>;
 def : VInterpF32Pat<int_amdgcn_interp_inreg_p2, V_INTERP_P2_F32_inreg>;
 
+let True16Predicate = UseRealTrue16Insts in {
+defm : VInterpF16Pat_t16<int_amdgcn_interp_inreg_p10_f16,
+                     V_INTERP_P10_F16_F32_inreg_t16, f32, 0>;
+defm : VInterpF16Pat_t16<int_amdgcn_interp_inreg_p2_f16,
+                     V_INTERP_P2_F16_F32_inreg_t16, f16, 1>;
+defm : VInterpF16Pat_t16<int_amdgcn_interp_p10_rtz_f16,
+                     V_INTERP_P10_RTZ_F16_F32_inreg_t16, f32, 0>;
+defm : VInterpF16Pat_t16<int_amdgcn_interp_p2_rtz_f16,
+                     V_INTERP_P2_RTZ_F16_F32_inreg_t16, f16, 1>;
+}
+
 let True16Predicate = UseFakeTrue16Insts in {
 defm : VInterpF16Pat<int_amdgcn_interp_inreg_p10_f16,
                      V_INTERP_P10_F16_F32_inreg_fake16, f32,
@@ -238,3 +272,6 @@ defm V_INTERP_P10_F16_F32_inreg : VINTERP_Real_t16_and_fake16_gfx11_gfx12<0x002,
 defm V_INTERP_P2_F16_F32_inreg : VINTERP_Real_t16_and_fake16_gfx11_gfx12<0x003, "v_interp_p2_f16_f32">;
 defm V_INTERP_P10_RTZ_F16_F32_inreg : VINTERP_Real_t16_and_fake16_gfx11_gfx12<0x004, "v_interp_p10_rtz_f16_f32">;
 defm V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Real_t16_and_fake16_gfx11_gfx12<0x005, "v_interp_p2_rtz_f16_f32">;
+
+let AssemblerPredicate = isGFX11Plus in
+def : AMDGPUMnemonicAlias<"v_interp_p2_new_f32", "v_interp_p2_f32">;
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 128c775619118..384fec0079a5d 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1190,7 +1190,7 @@ let Constraints = "$vdst = $src2",
     defm V_DOT8C_I32_I4  : VOP2Inst<"v_dot8c_i32_i4",  VOP_DOT_ACC_I32_I32>;
 
   let SubtargetPredicate = HasDot13Insts in
-    defm V_DOT2C_F32_BF16 : VOP2Inst<"v_dot2c_f32_bf16", VOP_DOT_ACC_F32_V2BF16>;
+    defm V_DOT2C_F32_BF16 : VOP2Inst_VOPD<"v_dot2c_f32_bf16", VOP_DOT_ACC_F32_V2BF16, 0xd, "v_dot2acc_f32_bf16">;
 }
 
 let AddedComplexity = 30 in {
@@ -1818,10 +1818,8 @@ defm V_CVT_PK_RTZ_F16_F32  : VOP2_Real_FULL_with_name_gfx11_gfx12<0x02f,
   "V_CVT_PKRTZ_F16_F32", "v_cvt_pk_rtz_f16_f32">;
 defm V_PK_FMAC_F16     : VOP2_Real_e32_gfx11_gfx12<0x03c>;
 
-defm V_ADD_F16_t16         : VOP2_Real_FULL_t16_gfx11_gfx12<0x032, "v_add_f16">;
-defm V_ADD_F16_fake16      : VOP2_Real_FULL_t16_gfx11_gfx12<0x032, "v_add_f16">;
-defm V_SUB_F16_t16         : VOP2_Real_FULL_t16_gfx11_gfx12<0x033, "v_sub_f16">;
-defm V_SUB_F16_fake16      : VOP2_Real_FULL_t16_gfx11_gfx12<0x033, "v_sub_f16">;
+defm V_ADD_F16             : VOP2_Real_FULL_t16_and_fake16_gfx11_gfx12<0x032, "v_add_f16">;
+defm V_SUB_F16             : VOP2_Real_FULL_t16_and_fake16_gfx11_gfx12<0x033, "v_sub_f16">;
 defm V_SUBREV_F16_t16      : VOP2_Real_FULL_t16_gfx11_gfx12<0x034, "v_subrev_f16">;
 defm V_SUBREV_F16_fake16   : VOP2_Real_FULL_t16_gfx11_gfx12<0x034, "v_subrev_f16">;
 defm V_MUL_F16_t16         : VOP2_Real_FULL_t16_gfx11_gfx12<0x035, "v_mul_f16">;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index c8c36714909ad..8a9f8aa3d16d3 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -763,6 +763,10 @@ let SubtargetPredicate = isGFX9Plus, True16Predicate = NotHasTrue16BitInsts in {
   def : OpSelBinOpClampPat<saddsat, V_ADD_I16_e64>;
   def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_e64>;
 } // End SubtargetPredicate = isGFX9Plus, True16Predicate = NotHasTrue16BitInsts
+let True16Predicate = UseRealTrue16Insts in {
+  def : OpSelBinOpClampPat<saddsat, V_ADD_I16_t16_e64>;
+  def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_t16_e64>;
+} // End True16Predicate = UseRealTrue16Insts
 let True16Predicate = UseFakeTrue16Insts in {
   def : OpSelBinOpClampPat<saddsat, V_ADD_I16_fake16_e64>;
   def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_fake16_e64>;
@@ -970,11 +974,16 @@ class VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOPProfile P> : VOP3_Profil
   let HasOMod = 0;
 }
 
+class VOP3_CVT_SCALE_FP4_F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {
+  let HasFP8DstByteSel = 1;
+}
+
 class VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {
   let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,
                           Int32InputMods:$src1_modifiers, Src1RC64:$src1,
                           FP32InputMods:$src2_modifiers, Src2RC64:$src2,
                           VGPR_32:$vdst_in, op_sel0:$op_sel);
+  let HasFP8DstByteSel = 1;
 }
 
 
@@ -992,6 +1001,7 @@ class VOP3_CVT_SCALE_FP4_F16BF16_TiedInput_Profile<VOPProfile P> : VOP3_Profile<
                                             HasSrc0FloatMods, HasSrc1FloatMods,
                                             HasSrc2FloatMods>.ret);
   let HasExtVOP3DPP = 0;
+  let HasFP8DstByteSel = 1;
 }
 
 class VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<ValueType Src0Ty> :
@@ -1004,6 +1014,7 @@ class VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<ValueType Src0Ty> :
   let HasExtVOP3DPP = 0;
   let HasOpSel = 1;
   let HasOMod = 0;
+  let HasFP8DstByteSel = 1;
 }
 
 def VOP3_CVT_SCALE_SR_PK_F4_F32_TiedInput_Profile : VOP3_Profile<VOPProfile<[i32, v2f32, i32, f32]>, VOP3_OPSEL> {
@@ -1015,6 +1026,7 @@ def VOP3_CVT_SCALE_SR_PK_F4_F32_TiedInput_Profile : VOP3_Profile<VOPProfile<[i32
   let HasExtVOP3DPP = 0;
   let HasOpSel = 1;
   let HasOMod = 0;
+  let HasFP8DstByteSel = 1;
 }
 
 class VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<ValueType DstTy> : VOP3_Profile<VOPProfile<[DstTy, i32, f32, untyped]>,
@@ -1090,7 +1102,7 @@ let SubtargetPredicate = HasBF8ConversionScaleInsts, mayRaiseFPException = 0 in
 let SubtargetPredicate = HasFP4ConversionScaleInsts, mayRaiseFPException = 0 in {
   defm V_CVT_SCALEF32_PK_F32_FP4 : VOP3Inst<"v_cvt_scalef32_pk_f32_fp4", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f32>>;
   let Constraints = "$vdst = $vdst_in", DisableEncoding="$vdst_in" in {
-    defm V_CVT_SCALEF32_PK_FP4_F32 : VOP3Inst<"v_cvt_scalef32_pk_fp4_f32", VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOP_I32_F32_F32_F32>>;
+    defm V_CVT_SCALEF32_PK_FP4_F32 : VOP3Inst<"v_cvt_scalef32_pk_fp4_f32", VOP3_CVT_SCALE_FP4_F32_TiedInput_Profile<VOP_I32_F32_F32_F32>>;
     let Constraints = "@earlyclobber $vdst" in {
       defm V_CVT_SCALEF32_SR_PK_FP4_F16:  VOP3Inst<"v_cvt_scalef32_sr_pk_fp4_f16", VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<v2f16>>;
       defm V_CVT_SCALEF32_SR_PK_FP4_BF16: VOP3Inst<"v_cvt_scalef32_sr_pk_fp4_bf16", VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<v2bf16>>;
@@ -1135,6 +1147,21 @@ let SubtargetPredicate = HasGFX950Insts, mayRaiseFPException = 0 in {
   defm V_CVT_SCALEF32_2XPK16_BF6_F32 : VOP3Inst<"v_cvt_scalef32_2xpk16_bf6_f32",  VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V16F32_V16F32_F32>,  int_amdgcn_cvt_scalef32_2xpk16_bf6_f32>;
 }
 
+let SubtargetPredicate = HasCvtPkF16F32Inst in {
+  let ReadsModeReg = 0 in {
+    defm V_CVT_PK_F16_F32 : VOP3Inst<"v_cvt_pk_f16_f32", VOP3_Profile<VOP_V2F16_F32_F32>>;
+  }
+
+  def : GCNPat<(v2f16 (fpround v2f32:$src)),
+               (V_CVT_PK_F16_F32_e64 0, (EXTRACT_SUBREG VReg_64:$src, sub0), 0, (EXTRACT_SUBREG VReg_64:$src, sub1))>;
+  def : GCNPat<(v2f16 (fpround v2f64:$src)),
+               (V_CVT_PK_F16_F32_e64 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub0_sub1)),
+                                     0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub2_sub3)))>;
+  def : GCNPat<(v2f16 (build_vector (f16 (fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),
+                                    (f16 (fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))),
+               (V_CVT_PK_F16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>;
+}
+
 class Cvt_Scale_FP4FP8BF8ToF16F32_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType DstTy> : GCNPat<
     (DstTy (node i32:$src0, f32:$src1, timm:$index)),
     (inst (SrcAndDstSelToOpSelXForm_0_0 $index), $src0, (SrcAndDstSelToOpSelXForm_1_0 $index), $src1)
@@ -1241,6 +1268,11 @@ let True16Predicate = NotHasTrue16BitInsts, SubtargetPredicate = isGFX10Plus in
    >;
 } // End True16Predicate = NotHasTrue16BitInsts, SubtargetPredicate = isGFX10Plus
 
+let True16Predicate = UseRealTrue16Insts in {
+  def : OpSelBinOpClampPat<uaddsat, V_ADD_NC_U16_t16_e64>;
+  def : OpSelBinOpClampPat<usubsat, V_SUB_NC_U16_t16_e64>;
+} // End OtherPredicates = [UseRealTrue16Insts]
+
 let True16Predicate = UseFakeTrue16Insts in {
    def : OpSelBinOpClampPat<uaddsat, V_ADD_NC_U16_fake16_e64>;
    def : OpSelBinOpClampPat<usubsat, V_SUB_NC_U16_fake16_e64>;
@@ -1264,28 +1296,28 @@ let SubtargetPredicate = isGFX12Plus in {
 let SubtargetPredicate = HasBitOp3Insts  in {
   let isReMaterializable = 1 in {
     defm V_BITOP3_B16 : VOP3Inst <"v_bitop3_b16",
-                                  VOP3_BITOP3_Profile<VOPProfile_True16<VOPProfile <[i16, i16, i16, i16, i8]>>,
+                                  VOP3_BITOP3_Profile<VOPProfile_True16<VOPProfile <[i16, i16, i16, i16, i32]>>,
                                                       VOP3_OPSEL>>;
     defm V_BITOP3_B32 : VOP3Inst <"v_bitop3_b32",
-                                  VOP3_BITOP3_Profile<VOPProfile <[i32, i32, i32, i32, i8]>, VOP3_REGULAR>>;
+                                  VOP3_BITOP3_Profile<VOPProfile <[i32, i32, i32, i32, i32]>, VOP3_REGULAR>>;
   }
   def : GCNPat<
-    (i32 (int_amdgcn_bitop3 i32:$src0, i32:$src1, i32:$src2, i8:$bitop3)),
+    (i32 (int_amdgcn_bitop3 i32:$src0, i32:$src1, i32:$src2, i32:$bitop3)),
     (i32 (V_BITOP3_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2, timm:$bitop3))
   >;
 
   def : GCNPat<
-    (i16 (int_amdgcn_bitop3 i16:$src0, i16:$src1, i16:$src2, i8:$bitop3)),
+    (i16 (int_amdgcn_bitop3 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),
     (i16 (V_BITOP3_B16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))
   >;
 
   def : GCNPat<
-    (i32 (BITOP3_32 i32:$src0, i32:$src1, i32:$src2, i8:$bitop3)),
+    (i32 (BITOP3_32 i32:$src0, i32:$src1, i32:$src2, i32:$bitop3)),
     (i32 (V_BITOP3_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2, timm:$bitop3))
   >;
 
   def : GCNPat<
-    (i16 (BITOP3_16 i16:$src0, i16:$src1, i16:$src2, i8:$bitop3)),
+    (i16 (BITOP3_16 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),
     (i16 (V_BITOP3_B16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))
   >;
 } // End SubtargetPredicate = HasBitOp3Insts
@@ -1313,6 +1345,30 @@ class VOP3_DOT_Profile<VOPProfile P> : VOP3_Profile<P, VOP3_OPSEL> {
   let HasOMod = 0;
 }
 
+class VOP3_DOT_Profile_t16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_True16<P, Features> {
+  let HasClamp = 0;
+  let HasOMod = 0;
+  // Override modifiers for bf16(i16) (same as float modifiers).
+  let HasSrc0Mods = 1;
+  let HasSrc1Mods = 1;
+  let HasSrc2Mods = 1;
+  let Src0ModVOP3DPP = FPVRegInputMods;
+  let Src1ModVOP3DPP = FP32VCSrcInputMods;
+  let Src2ModVOP3DPP = FPT16VCSrcInputMods</*IsFake16*/0>;
+}
+
+class VOP3_DOT_Profile_fake16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_Fake16<P, Features> {
+  let HasClamp = 0;
+  let HasOMod = 0;
+  // Override modifiers for bf16(i16) (same as float modifiers).
+  let HasSrc0Mods = 1;
+  let HasSrc1Mods = 1;
+  let HasSrc2Mods = 1;
+  let AsmVOP3Base = getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,
+   HasOpSel, HasOMod, IsVOP3P, HasModifiers, 1/*HasSrc0Mods*/, 1/*HasSrc1Mods*/,
+   1/*HasSrc2Mods*/, DstVT>.ret;
+}
+
 let SubtargetPredicate = isGFX11Plus in {
   defm V_MAXMIN_F32     : VOP3Inst<"v_maxmin_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
   defm V_MINMAX_F32     : VOP3Inst<"v_minmax_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
@@ -1377,9 +1433,15 @@ let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
   defm V_MINIMUMMAXIMUM_F16 : VOP3Inst<"v_minimummaximum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
 } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
 
-let OtherPredicates = [HasDot9Insts], IsDOT=1 in {
-  defm V_DOT2_F16_F16 :   VOP3Inst<"v_dot2_f16_f16",   VOP3_DOT_Profile<VOP_F16_V2F16_V2F16_F16>, int_amdgcn_fdot2_f16_f16>;
-  defm V_DOT2_BF16_BF16 : VOP3Inst<"v_dot2_bf16_bf16", VOP3_DOT_Profile<VOP_BF16_V2BF16_V2BF16_BF16>, int_amdgcn_fdot2_bf16_bf16>;
+let SubtargetPredicate = HasDot9Insts, IsDOT=1 in {
+  defm V_DOT2_F16_F16 :   VOP3Inst_t16_with_profiles<"v_dot2_f16_f16", VOP3_DOT_Profile<VOP_F16_V2F16_V2F16_F16>,
+                                                      VOP3_DOT_Profile_t16<VOP_F16_V2F16_V2F16_F16>,
+                                                      VOP3_DOT_Profile_fake16<VOP_F16_V2F16_V2F16_F16>,
+                                                      int_amdgcn_fdot2_f16_f16>;
+  defm V_DOT2_BF16_BF16 : VOP3Inst_t16_with_profiles<"v_dot2_bf16_bf16", VOP3_DOT_Profile<VOP_BF16_V2BF16_V2BF16_BF16>,
+                                                      VOP3_DOT_Profile_t16<VOP_BF16_V2BF16_V2BF16_BF16>,
+                                                      VOP3_DOT_Profile_fake16<VOP_BF16_V2BF16_V2BF16_BF16>,
+                                                      int_amdgcn_fdot2_bf16_bf16>;
 }
 
 class VOP_Pseudo_Scalar<RegisterClass Dst, RegisterOperand SrcOp,
@@ -1430,6 +1492,23 @@ let SubtargetPredicate = HasAshrPkInsts, isReMaterializable = 1 in {
   defm V_ASHR_PK_U8_I32 : VOP3Inst<"v_ashr_pk_u8_i32", VOP3_Profile<VOP_I16_I32_I32_I32, VOP3_OPSEL_ONLY>, int_amdgcn_ashr_pk_u8_i32>;
 } // End SubtargetPredicate = HasAshrPkInsts, isReMaterializable = 1
 
+class AshrPkI8Pat<VOP3_Pseudo inst, int lo, int hi>: GCNPat<
+    (i16 (or (i16 (shl (i16 (trunc (i32 (AMDGPUsmed3 (i32 (sra i32:$src1, i32:$src2)), (i32 lo), (i32 hi))))), (i16 8))),
+             (i16 (and (i16 (trunc (i32 (AMDGPUsmed3 (i32 (sra i32:$src0, i32:$src2)), (i32 lo), (i32 hi))))), (i16 255))))),
+    (inst 0, VSrc_b32:$src0, 0,  VSrc_b32:$src1, 0, VSrc_b32:$src2, 0 )
+>;
+
+class AshrPkU8Pat<VOP3_Pseudo inst, int lo, int hi>: GCNPat<
+    (i16 (or (i16 (shl (i16 (trunc (i32 (AMDGPUsmed3 (i32 (sra i32:$src1, i32:$src2)), (i32 lo), (i32 hi))))), (i16 8))),
+             (i16 (trunc (i32 (AMDGPUsmed3 (i32 (sra i32:$src0, i32:$src2)), (i32 lo), (i32 hi))))))),
+    (inst 0, VSrc_b32:$src0, 0,  VSrc_b32:$src1, 0, VSrc_b32:$src2, 0 )
+>;
+
+let SubtargetPredicate = HasAshrPkInsts in {
+  def : AshrPkI8Pat<V_ASHR_PK_I8_I32_e64, -128, 127>;
+  def : AshrPkU8Pat<V_ASHR_PK_U8_I32_e64, 0, 255>;
+}
+
 //===----------------------------------------------------------------------===//
 // Integer Clamp Patterns
 //===----------------------------------------------------------------------===//
@@ -1560,8 +1639,10 @@ multiclass VOP3_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
   VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName>,
   VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName>;
 
-multiclass VOP3Dot_Realtriple_gfx11_gfx12<bits<10> op> :
-  VOP3Dot_Realtriple<GFX11Gen, op>, VOP3Dot_Realtriple<GFX12Gen, op>;
+multiclass VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME> {
+  defm _t16: VOP3Dot_Realtriple_gfx11_gfx12<op, asmName, 0, opName#"_t16">;
+  defm _fake16: VOP3Dot_Realtriple_gfx11_gfx12<op, asmName, 0, opName#"_fake16">;
+}
 
 multiclass VOP3_Realtriple_t16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME,
                                            string pseudo_mnemonic = "", bit isSingle = 0> :
@@ -1653,8 +1734,8 @@ defm V_MAXMIN_U32          : VOP3_Realtriple_gfx11_gfx12<0x262>;
 defm V_MINMAX_U32          : VOP3_Realtriple_gfx11_gfx12<0x263>;
 defm V_MAXMIN_I32          : VOP3_Realtriple_gfx11_gfx12<0x264>;
 defm V_MINMAX_I32          : VOP3_Realtriple_gfx11_gfx12<0x265>;
-defm V_DOT2_F16_F16        : VOP3Dot_Realtriple_gfx11_gfx12<0x266>;
-defm V_DOT2_BF16_BF16      : VOP3Dot_Realtriple_gfx11_gfx12<0x267>;
+defm V_DOT2_F16_F16        : VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<0x266, "v_dot2_f16_f16">;
+defm V_DOT2_BF16_BF16      : VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<0x267, "v_dot2_bf16_bf16">;
 defm V_DIV_SCALE_F32       : VOP3be_Real_gfx11_gfx12<0x2fc, "V_DIV_SCALE_F32", "v_div_scale_f32">;
 defm V_DIV_SCALE_F64       : VOP3be_Real_gfx11_gfx12<0x2fd, "V_DIV_SCALE_F64", "v_div_scale_f64">;
 defm V_MAD_U64_U32_gfx11   : VOP3be_Real_gfx11<0x2fe, "V_MAD_U64_U32_gfx11", "v_mad_u64_u32">;
@@ -1701,6 +1782,11 @@ defm V_OR_B16_fake16       : VOP3Only_Realtriple_t16_gfx11_gfx12<0x363, "v_or_b1
 defm V_XOR_B16_t16         : VOP3Only_Realtriple_t16_gfx11_gfx12<0x364, "v_xor_b16">;
 defm V_XOR_B16_fake16      : VOP3Only_Realtriple_t16_gfx11_gfx12<0x364, "v_xor_b16">;
 
+let AssemblerPredicate = isGFX11Plus in {
+  def : AMDGPUMnemonicAlias<"v_add3_nc_u32", "v_add3_u32">;
+  def : AMDGPUMnemonicAlias<"v_xor_add_u32", "v_xad_u32">;
+}
+
 //===----------------------------------------------------------------------===//
 // GFX10.
 //===----------------------------------------------------------------------===//
@@ -2047,6 +2133,7 @@ multiclass VOP3_Real_BITOP3_gfx9<bits<10> op, string AsmName, bit isSingle = 0>
     }
   }
 }
+
 } // End AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9"
 
 defm V_MAD_U64_U32      : VOP3be_Real_vi <0x1E8>;
@@ -2261,6 +2348,9 @@ defm V_CVT_SR_BF16_F32: VOP3OpSel_Real_gfx9 <0x2a7>;
 
 defm V_ASHR_PK_I8_I32 : VOP3OpSel_Real_gfx9 <0x265>;
 defm V_ASHR_PK_U8_I32 : VOP3OpSel_Real_gfx9 <0x266>;
+let OtherPredicates = [HasCvtPkF16F32Inst] in {
+defm V_CVT_PK_F16_F32 : VOP3_Real_gfx9<0x267, "v_cvt_pk_f16_f32">;
+}
 
 defm V_CVT_SCALEF32_2XPK16_FP6_F32 : VOP3_Real_gfx9<0x252, "v_cvt_scalef32_2xpk16_fp6_f32">;
 defm V_CVT_SCALEF32_2XPK16_BF6_F32 : VOP3_Real_gfx9<0x253, "v_cvt_scalef32_2xpk16_bf6_f32">;
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 7d202de6643bc..7638879afcb56 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -758,9 +758,14 @@ def VOPProfileMAI_I32_V4I32_X16     : VOPProfileMAI<VOP_V16I32_V4I32_V4I32_V16I3
 def VOPProfileMAI_I32_V4I32_X16_VCD : VOPProfileMAI<VOP_V16I32_V4I32_V4I32_V16I32,    VISrc_512_b32,  VDst_512, AVSrc_128>;
 
 
-class MFMATable <bit is_mac, string Name> {
+class MFMATable <bit is_mac, string Kind, string Name,
+                 string AGPROpName = NAME> {
   bit IsMac = is_mac;
   string FMAOp = Name;
+  string AGPROp = AGPROpName;
+
+  // Does this MFMA use "AGPR" or "VGPR" for srcC/vdst
+  string MFMAKind = Kind;
 }
 
 class MFMA_F8F6F4_WithSizeTable<int A, int B, Instruction ThisVariant, Instruction F8F8Variant> {
@@ -866,12 +871,12 @@ multiclass MAIInst<string OpName, string P, SDPatternOperator node = null_frag,
     let Constraints = !if(NoDstOverlap, "@earlyclobber $vdst", "") in {
       def _e64 : MAIInst<OpName, !cast<VOPProfileMAI>("VOPProfileMAI_" # P),
                          !if(!or(NoDstOverlap, !eq(node, null_frag)), null_frag, AgprMAIFrag<node, HasAbid, Scaled>), Scaled>,
-                 MFMATable<0, NAME # "_e64">;
+                 MFMATable<0, "AGPR", NAME # "_e64">;
 
       let OtherPredicates = [isGFX90APlus], Mnemonic = OpName in
       def _vgprcd_e64 : MAIInst<OpName # "_vgprcd", !cast<VOPProfileMAI>("VOPProfileMAI_" # P # "_VCD"),
                                 !if(!or(NoDstOverlap, !eq(node, null_frag)), null_frag, VgprMAIFrag<node, HasAbid, Scaled>), Scaled>,
-                        MFMATable<0, NAME # "_vgprcd_e64">;
+                        MFMATable<0, "VGPR", NAME # "_vgprcd_e64", NAME # "_e64">;
     }
 
     if NoDstOverlap then {
@@ -880,12 +885,12 @@ multiclass MAIInst<string OpName, string P, SDPatternOperator node = null_frag,
           Mnemonic = OpName in {
         def "_mac_e64" : MAIInst<OpName # "_mac", !cast<VOPProfileMAI>("VOPProfileMAI_" # P),
                                  !if(!eq(node, null_frag), null_frag, AgprMAIFrag<node, HasAbid, Scaled>), Scaled>,
-                         MFMATable<1, NAME # "_e64">;
+                         MFMATable<1, "AGPR", NAME # "_e64", NAME # "_mac_e64">;
 
         let OtherPredicates = [isGFX90APlus] in
         def _mac_vgprcd_e64 : MAIInst<OpName # "_mac_vgprcd", !cast<VOPProfileMAI>("VOPProfileMAI_" # P # "_VCD"),
                                       !if(!eq(node, null_frag), null_frag, VgprMAIFrag<node, HasAbid, Scaled>), Scaled>,
-                              MFMATable<1, NAME # "_vgprcd_e64">;
+                              MFMATable<1, "VGPR", NAME # "_vgprcd_e64">;
       }
     }
   } // End isConvergent = 1, mayRaiseFPException = 0, ReadsModeReg = 1
@@ -902,11 +907,11 @@ multiclass ScaledMAIInst_mc<string OpName, string UnscaledOpName_, SDPatternOper
 
   def _e64 : ScaledMAIInst<OpName,
         !cast<MAIInst>(UnscaledOpName#"_e64"), !if(NoDstOverlap, null_frag, AgprMAIFrag<node, HasAbid, true>)>,
-      MFMATable<0, NAME # "_e64">;
+      MFMATable<0, "AGPR", NAME # "_e64">;
 
   def _vgprcd_e64 : ScaledMAIInst<OpName # "_vgprcd",
           !cast<MAIInst>(UnscaledOpName#"_vgprcd_e64"), !if(NoDstOverlap, null_frag, VgprMAIFrag<node, HasAbid, true>)>,
-        MFMATable<0, NAME # "_vgprcd_e64">;
+        MFMATable<0, "VGPR", NAME # "_vgprcd_e64", NAME # "_e64">;
 
  if NoDstOverlap then {
    let Constraints = !if(NoDstOverlap, "$vdst = $src2", ""),
@@ -914,11 +919,11 @@ multiclass ScaledMAIInst_mc<string OpName, string UnscaledOpName_, SDPatternOper
        Mnemonic = UnscaledOpName_ in {
      def _mac_e64 : ScaledMAIInst<OpName # "_mac",
           !cast<MAIInst>(UnscaledOpName # "_mac_e64"), AgprMAIFrag<node, HasAbid, true>>,
-        MFMATable<1, NAME # "_e64">;
+        MFMATable<1, "AGPR", NAME # "_e64">;
 
      def _mac_vgprcd_e64 : ScaledMAIInst<OpName # " _mac_vgprcd",
           !cast<MAIInst>(UnscaledOpName # "_mac_vgprcd_e64"), VgprMAIFrag<node, HasAbid, true>>,
-        MFMATable<1, NAME # "_vgprcd_e64">;
+        MFMATable<1, "VGPR", NAME # "_vgprcd_e64", NAME # "_mac_e64">;
     }
   }
 }
@@ -1840,6 +1845,11 @@ defm V_DOT4_I32_IU8  : VOP3P_Real_gfx11_gfx12<0x16>;
 defm V_DOT8_I32_IU4  : VOP3P_Real_gfx11_gfx12<0x18>;
 defm V_DOT2_F32_BF16 : VOP3P_Real_gfx11_gfx12<0x1a>;
 
+let AssemblerPredicate = isGFX11Plus in {
+  def : AMDGPUMnemonicAlias<"v_dot4_i32_i8", "v_dot4_i32_iu8">;
+  def : AMDGPUMnemonicAlias<"v_dot8_i32_i4", "v_dot8_i32_iu4">;
+}
+
 multiclass VOP3P_Real_WMMA <bits<7> op> {
   let WaveSizePredicate = isWave32, DecoderNamespace = "GFX11" in {
     defm _twoaddr_w32 : VOP3P_Real_Base <GFX11Gen, op>;
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index 0a4b51c4ac631..9bf043ea334fe 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -1956,6 +1956,17 @@ defm V_CMPX_CLASS_F16_fake16 : VOPCX_Real_t16_gfx11_gfx12<0x0fd, "v_cmpx_class_f
 defm V_CMPX_CLASS_F32     : VOPCX_Real_gfx11_gfx12<0x0fe>;
 defm V_CMPX_CLASS_F64     : VOPCX_Real_gfx11_gfx12<0x0ff>;
 
+let AssemblerPredicate = isGFX11Only in {
+  def : AMDGPUMnemonicAlias<"v_cmp_tru_i32", "v_cmp_t_i32">;
+  def : AMDGPUMnemonicAlias<"v_cmp_tru_u32", "v_cmp_t_u32">;
+  def : AMDGPUMnemonicAlias<"v_cmp_tru_i64", "v_cmp_t_i64">;
+  def : AMDGPUMnemonicAlias<"v_cmp_tru_u64", "v_cmp_t_u64">;
+  def : AMDGPUMnemonicAlias<"v_cmpx_tru_i32", "v_cmpx_t_i32">;
+  def : AMDGPUMnemonicAlias<"v_cmpx_tru_u32", "v_cmpx_t_u32">;
+  def : AMDGPUMnemonicAlias<"v_cmpx_tru_i64", "v_cmpx_t_i64">;
+  def : AMDGPUMnemonicAlias<"v_cmpx_tru_u64", "v_cmpx_t_u64">;
+}
+
 //===----------------------------------------------------------------------===//
 // GFX10.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOPDInstructions.td b/llvm/lib/Target/AMDGPU/VOPDInstructions.td
index c6af3d67c5600..47aa2e8f3cc08 100644
--- a/llvm/lib/Target/AMDGPU/VOPDInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPDInstructions.td
@@ -87,12 +87,12 @@ class VOPD_Base<dag outs, dag ins, string asm, VOP_Pseudo VDX, VOP_Pseudo VDY,
   let ReadsModeReg = !or(VDX.ReadsModeReg, VDY.ReadsModeReg);
   let mayRaiseFPException = ReadsModeReg;
 
-  // V_DUAL_FMAC and V_DUAL_DOT2ACC_F32_F16 need a dummy src2 tied to dst for
-  // passes to track its uses. Its presence does not affect VOPD formation rules
-  // because the rules for src2 and dst are the same. src2X and src2Y should not
-  // be encoded.
-  bit hasSrc2AccX = !or(!eq(VDX.Mnemonic, "v_fmac_f32"), !eq(VDX.Mnemonic, "v_dot2c_f32_f16"));
-  bit hasSrc2AccY = !or(!eq(VDY.Mnemonic, "v_fmac_f32"), !eq(VDY.Mnemonic, "v_dot2c_f32_f16"));
+  // V_DUAL_FMAC and V_DUAL_DOT2ACC_F32_F16 and V_DUAL_DOT2ACC_F32_BF16 need a
+  // dummy src2 tied to dst for passes to track its uses. Its presence does not
+  // affect VOPD formation rules because the rules for src2 and dst are the
+  // same. src2X and src2Y should not be encoded.
+  bit hasSrc2AccX = !or(!eq(VDX.Mnemonic, "v_fmac_f32"), !eq(VDX.Mnemonic, "v_dot2c_f32_f16"), !eq(VDX.Mnemonic, "v_dot2c_f32_bf16"));
+  bit hasSrc2AccY = !or(!eq(VDY.Mnemonic, "v_fmac_f32"), !eq(VDY.Mnemonic, "v_dot2c_f32_f16"), !eq(VDY.Mnemonic, "v_dot2c_f32_bf16"));
   string ConstraintsX = !if(hasSrc2AccX, "$src2X = $vdstX", "");
   string ConstraintsY = !if(hasSrc2AccY, "$src2Y = $vdstY", "");
   let Constraints =
@@ -125,20 +125,16 @@ class VOPD_MADK<dag outs, dag ins, string asm, VOP_Pseudo VDX, VOP_Pseudo VDY,
   let FixedSize = 1;
 }
 
-// V_DUAL_DOT2ACC_F32_BF16 is a legal instruction, but V_DOT2ACC_F32_BF16 is
-// not. V_DUAL_DOT2C_F32_BF16 is a legal instruction on GFX12, but
-// V_DOT2C_F32_F16_e32 is not. Since we generate the DUAL form by converting
-// from the normal form we will never generate them.
 defvar VOPDPseudosCommon = [
   "V_FMAC_F32_e32", "V_FMAAK_F32", "V_FMAMK_F32", "V_MUL_F32_e32",
   "V_ADD_F32_e32", "V_SUB_F32_e32", "V_SUBREV_F32_e32", "V_MUL_LEGACY_F32_e32",
-  "V_MOV_B32_e32", "V_CNDMASK_B32_e32", "V_MAX_F32_e32", "V_MIN_F32_e32"
+  "V_MOV_B32_e32", "V_CNDMASK_B32_e32", "V_MAX_F32_e32", "V_MIN_F32_e32",
+  "V_DOT2C_F32_F16_e32", "V_DOT2C_F32_BF16_e32"
 ];
-defvar VOPDPseudosGFX11 = ["V_DOT2C_F32_F16_e32"];
 defvar VOPDYOnlyPseudosCommon = ["V_ADD_U32_e32", "V_LSHLREV_B32_e32",
                                  "V_AND_B32_e32"];
 
-defvar VOPDXPseudosGFX11 = !listconcat(VOPDPseudosCommon, VOPDPseudosGFX11);
+defvar VOPDXPseudosGFX11 = VOPDPseudosCommon;
 defvar VOPDXPseudosGFX12 = VOPDPseudosCommon;
 defvar VOPDYPseudosGFX11 = !listconcat(VOPDXPseudosGFX11, VOPDYOnlyPseudosCommon);
 defvar VOPDYPseudosGFX12 = !listconcat(VOPDXPseudosGFX12, VOPDYOnlyPseudosCommon);
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 0e19696a32f86..5c4d96d3688b8 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -359,9 +359,12 @@ class VOP3FP8OpSel_src_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gf
    let Inst{14-13} = byte_sel;  // op_sel2/3
  }
 
-class VOP3DotOpSel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3OpSel_gfx11_gfx12<op, p>{
+class VOP3DotOpSel_gfx11_gfx12<bits<10> op, VOPProfile p> :
+    VOP3e_t16_gfx11_gfx12<op, p>{
   let Inst{11} = ?;
   let Inst{12} = ?;
+  let Inst{13} = !if(p.HasSrc2Mods, src2_modifiers{2}, 0);
+  let Inst{14} = !if(!and(p.HasDst, p.HasSrc0Mods),  src0_modifiers{3}, 0);
 }
 
 // NB: For V_INTERP* opcodes, src0 is encoded as src1 and vice versa
@@ -1706,10 +1709,12 @@ multiclass VOP3_Real_Base<GFXGen Gen, bits<10> op, string opName = NAME,
   }
 }
 
-multiclass VOP3Dot_Real_Base<GFXGen Gen, bits<10> op, string opName = NAME,
+multiclass VOP3Dot_Real_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME,
                              bit isSingle = 0> {
   defvar ps = !cast<VOP_Pseudo>(opName#"_e64");
-  let IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
+  let AsmString = asmName # ps.AsmOperands,
+      DecoderNamespace = Gen.DecoderNamespace # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"),
+      IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
     def _e64#Gen.Suffix :
       VOP3_Real_Gen<ps, Gen>,
       VOP3DotOpSel_gfx11_gfx12<op, ps.Pfl>;
@@ -1773,9 +1778,13 @@ multiclass VOP3_Real_dpp_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
       VOP3_DPP16_Gen<op, ps, Gen>;
 }
 
-multiclass VOP3Dot_Real_dpp_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
+multiclass VOP3Dot_Real_dpp_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME> {
+  defvar ps = !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp");
   def _e64_dpp#Gen.Suffix :
-    VOP3_DPP16_Gen<op, !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp"), Gen> {
+    VOP3_DPP16_Gen_t16<op, ps, Gen> {
+      let AsmString = asmName # ps.Pfl.AsmVOP3DPP16;
+      let DecoderNamespace = Gen.DecoderNamespace
+                             # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
       let Inst{11} = ?;
       let Inst{12} = ?;
     }
@@ -1797,12 +1806,14 @@ multiclass VOP3_Real_dpp8_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
   }
 }
 
-multiclass VOP3Dot_Real_dpp8_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
+multiclass VOP3Dot_Real_dpp8_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME> {
   defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
-  def _e64_dpp8#Gen.Suffix : Base_VOP3_DPP8<op, ps> {
+  def _e64_dpp8#Gen.Suffix : Base_VOP3_DPP8_t16<op, ps> {
     let Inst{11} = ?;
     let Inst{12} = ?;
-    let DecoderNamespace = Gen.DecoderNamespace;
+    let AsmString = asmName # ps.Pfl.AsmVOP3DPP8;
+    let DecoderNamespace = Gen.DecoderNamespace
+                           # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
     let AssemblerPredicate = Gen.AssemblerPredicate;
   }
 }
@@ -1855,11 +1866,11 @@ multiclass VOP3_Realtriple<GFXGen Gen, bits<10> op, bit isSingle = 0,
   VOP3_Real_dpp_Base<Gen, op, opName>,
   VOP3_Real_dpp8_Base<Gen, op, opName>;
 
-multiclass VOP3Dot_Realtriple<GFXGen Gen, bits<10> op, bit isSingle = 0,
+multiclass VOP3Dot_Realtriple<GFXGen Gen, bits<10> op, string asmName, bit isSingle = 0,
                               string opName = NAME> :
-  VOP3Dot_Real_Base<Gen, op, opName, isSingle>,
-  VOP3Dot_Real_dpp_Base<Gen, op, opName>,
-  VOP3Dot_Real_dpp8_Base<Gen, op, opName>;
+  VOP3Dot_Real_Base<Gen, op, asmName, opName, isSingle>,
+  VOP3Dot_Real_dpp_Base<Gen, op, asmName, opName>,
+  VOP3Dot_Real_dpp8_Base<Gen, op, asmName, opName>;
 
 multiclass VOP3Only_Realtriple<GFXGen Gen, bits<10> op> :
   VOP3_Realtriple<Gen, op, 1>;
@@ -1957,6 +1968,12 @@ multiclass VOP3Only_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
   VOP3Only_Realtriple_with_name<GFX11Gen, op, opName, asmName>,
   VOP3Only_Realtriple_with_name<GFX12Gen, op, opName, asmName>;
 
+multiclass VOP3Dot_Realtriple_gfx11_gfx12<bits<10> op, string asmName, bit isSingle = 0,
+                                          string opName = NAME> :
+  VOP3Dot_Realtriple<GFX11Gen, op, asmName, isSingle, opName>,
+  VOP3Dot_Realtriple<GFX12Gen, op, asmName, isSingle, opName>;
+
+
 //===----------------------------------------------------------------------===//
 
 include "VOPCInstructions.td"
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index a119db472fc03..af47d0e176f36 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -1309,6 +1309,10 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
       default:
         MI->print(errs());
         llvm_unreachable("Unsupported opcode for unwinding information");
+      case ARM::tLDRspi:
+        // Used to restore LR in a prologue which uses it as a temporary, has
+        // no effect on unwind tables.
+        return;
       case ARM::MOVr:
       case ARM::tMOVr:
         Offset = 0;
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 73ee8cf81adcd..b03221a440039 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -111,13 +111,6 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
   bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
 
-  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
-    const ConstantSDNode *CN = cast<ConstantSDNode>(N);
-    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
-    Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
-    return true;
-  }
-
   bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
                              SDValue &Offset, SDValue &Opc);
   bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
@@ -3596,7 +3589,11 @@ void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
     ReplaceNode(And.getNode(), NewN);
   } else if (Range->first == Range->second) {
     //  3. Only one bit is set. We can shift this into the sign bit and use a
-    //     PL/MI comparison.
+    //     PL/MI comparison. This is not safe if CMPZ has multiple uses because
+    //     only one of them (the one currently being selected) will be switched
+    //     to use the new condition code.
+    if (!N->hasOneUse())
+      return;
     NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
     ReplaceNode(And.getNode(), NewN);
 
@@ -4123,17 +4120,15 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
     SDValue Chain = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
     SDValue N2 = N->getOperand(2);
-    SDValue N3 = N->getOperand(3);
-    SDValue InGlue = N->getOperand(4);
+    SDValue Flags = N->getOperand(3);
     assert(N1.getOpcode() == ISD::BasicBlock);
     assert(N2.getOpcode() == ISD::Constant);
-    assert(N3.getOpcode() == ISD::Register);
 
     unsigned CC = (unsigned)N2->getAsZExtVal();
 
-    if (InGlue.getOpcode() == ARMISD::CMPZ) {
-      if (InGlue.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN) {
-        SDValue Int = InGlue.getOperand(0);
+    if (Flags.getOpcode() == ARMISD::CMPZ) {
+      if (Flags.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN) {
+        SDValue Int = Flags.getOperand(0);
         uint64_t ID = Int->getConstantOperandVal(1);
 
         // Handle low-overhead loops.
@@ -4155,15 +4150,15 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
 
           ReplaceUses(N, LoopEnd);
           CurDAG->RemoveDeadNode(N);
-          CurDAG->RemoveDeadNode(InGlue.getNode());
+          CurDAG->RemoveDeadNode(Flags.getNode());
           CurDAG->RemoveDeadNode(Int.getNode());
           return;
         }
       }
 
       bool SwitchEQNEToPLMI;
-      SelectCMPZ(InGlue.getNode(), SwitchEQNEToPLMI);
-      InGlue = N->getOperand(4);
+      SelectCMPZ(Flags.getNode(), SwitchEQNEToPLMI);
+      Flags = N->getOperand(3);
 
       if (SwitchEQNEToPLMI) {
         switch ((ARMCC::CondCodes)CC) {
@@ -4179,25 +4174,18 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
     }
 
     SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
-    SDValue Ops[] = { N1, Tmp2, N3, Chain, InGlue };
-    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
-                                             MVT::Glue, Ops);
-    Chain = SDValue(ResNode, 0);
-    if (N->getNumValues() == 2) {
-      InGlue = SDValue(ResNode, 1);
-      ReplaceUses(SDValue(N, 1), InGlue);
-    }
-    ReplaceUses(SDValue(N, 0),
-                SDValue(Chain.getNode(), Chain.getResNo()));
-    CurDAG->RemoveDeadNode(N);
+    Chain = CurDAG->getCopyToReg(Chain, dl, ARM::CPSR, Flags, SDValue());
+    SDValue Ops[] = {N1, Tmp2, CurDAG->getRegister(ARM::CPSR, MVT::i32), Chain,
+                     Chain.getValue(1)};
+    CurDAG->SelectNodeTo(N, Opc, MVT::Other, Ops);
     return;
   }
 
   case ARMISD::CMPZ: {
     // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
     //   This allows us to avoid materializing the expensive negative constant.
-    //   The CMPZ #0 is useless and will be peepholed away but we need to keep it
-    //   for its glue output.
+    //   The CMPZ #0 is useless and will be peepholed away but we need to keep
+    //   it for its flags output.
     SDValue X = N->getOperand(0);
     auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
     if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
@@ -4224,7 +4212,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
       }
       if (Add) {
         SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
-        CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
+        CurDAG->MorphNodeTo(N, ARMISD::CMPZ, N->getVTList(), Ops2);
       }
     }
     // Other cases are autogenerated.
@@ -4232,11 +4220,11 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
   }
 
   case ARMISD::CMOV: {
-    SDValue InGlue = N->getOperand(4);
+    SDValue Flags = N->getOperand(3);
 
-    if (InGlue.getOpcode() == ARMISD::CMPZ) {
+    if (Flags.getOpcode() == ARMISD::CMPZ) {
       bool SwitchEQNEToPLMI;
-      SelectCMPZ(InGlue.getNode(), SwitchEQNEToPLMI);
+      SelectCMPZ(Flags.getNode(), SwitchEQNEToPLMI);
 
       if (SwitchEQNEToPLMI) {
         SDValue ARMcc = N->getOperand(2);
@@ -4253,10 +4241,9 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
         }
         SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
         SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
-                         N->getOperand(3), N->getOperand(4)};
+                         N->getOperand(3)};
         CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
       }
-
     }
     // Other cases are autogenerated.
     break;
@@ -5229,7 +5216,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
       return;
     case Intrinsic::arm_mve_vsbc:
     case Intrinsic::arm_mve_vsbc_predicated:
-      SelectMVE_VADCSBC(N, ARM::MVE_VSBC, ARM::MVE_VSBCI, true,
+      SelectMVE_VADCSBC(N, ARM::MVE_VSBC, ARM::MVE_VSBCI, false,
                         IntNo == Intrinsic::arm_mve_vsbc_predicated);
       return;
     case Intrinsic::arm_mve_vshlc:
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 6b290135c5bcb..595bb6e73f453 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -4924,14 +4924,11 @@ SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) &&
       LHS.getConstantOperandVal(1) < 31) {
     unsigned ShiftAmt = LHS.getConstantOperandVal(1) + 1;
-    SDValue Shift = DAG.getNode(ARMISD::LSLS, dl,
-                                DAG.getVTList(MVT::i32, MVT::i32),
-                                LHS.getOperand(0),
-                                DAG.getConstant(ShiftAmt, dl, MVT::i32));
-    SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
-                                     Shift.getValue(1), SDValue());
+    SDValue Shift =
+        DAG.getNode(ARMISD::LSLS, dl, DAG.getVTList(MVT::i32, FlagsVT),
+                    LHS.getOperand(0), DAG.getConstant(ShiftAmt, dl, MVT::i32));
     ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
-    return Chain.getValue(1);
+    return Shift.getValue(1);
   }
 
   ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
@@ -4963,7 +4960,7 @@ SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     break;
   }
   ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
-  return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
+  return DAG.getNode(CompareType, dl, FlagsVT, LHS, RHS);
 }
 
 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
@@ -4978,24 +4975,7 @@ SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
   else
     Flags = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, dl,
                         FlagsVT, LHS);
-  return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Flags);
-}
-
-/// duplicateCmp - Glue values can have only one use, so this function
-/// duplicates a comparison node.
-SDValue
-ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
-  unsigned Opc = Cmp.getOpcode();
-  SDLoc DL(Cmp);
-  if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ)
-    return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
-
-  assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation");
-  SDValue Flags = Cmp.getOperand(0);
-  assert((Flags.getOpcode() == ARMISD::CMPFP ||
-          Flags.getOpcode() == ARMISD::CMPFPw0) &&
-         "unexpected operand of FMSTAT");
-  return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Flags);
+  return DAG.getNode(ARMISD::FMSTAT, dl, FlagsVT, Flags);
 }
 
 // This function returns three things: the arithmetic computation itself
@@ -5023,7 +5003,7 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
   case ISD::SADDO:
     ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
     Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
     break;
   case ISD::UADDO:
     ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
@@ -5032,17 +5012,17 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
     Value = DAG.getNode(ARMISD::ADDC, dl,
                         DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
                 .getValue(0);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
     break;
   case ISD::SSUBO:
     ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
     Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
     break;
   case ISD::USUBO:
     ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
     Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
     break;
   case ISD::UMULO:
     // We generate a UMUL_LOHI and then check if the high word is 0.
@@ -5050,7 +5030,7 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
     Value = DAG.getNode(ISD::UMUL_LOHI, dl,
                         DAG.getVTList(Op.getValueType(), Op.getValueType()),
                         LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value.getValue(1),
                               DAG.getConstant(0, dl, MVT::i32));
     Value = Value.getValue(0); // We only want the low 32 bits for the result.
     break;
@@ -5061,7 +5041,7 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
     Value = DAG.getNode(ISD::SMUL_LOHI, dl,
                         DAG.getVTList(Op.getValueType(), Op.getValueType()),
                         LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value.getValue(1),
                               DAG.getNode(ISD::SRA, dl, Op.getValueType(),
                                           Value.getValue(0),
                                           DAG.getConstant(31, dl, MVT::i32)));
@@ -5081,15 +5061,14 @@ ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
   SDValue Value, OverflowCmp;
   SDValue ARMcc;
   std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
   SDLoc dl(Op);
   // We use 0 and 1 as false and true values.
   SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
   SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
   EVT VT = Op.getValueType();
 
-  SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
-                                 ARMcc, CCR, OverflowCmp);
+  SDValue Overflow =
+      DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, ARMcc, OverflowCmp);
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
   return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
@@ -5226,11 +5205,9 @@ SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     SDValue Value, OverflowCmp;
     SDValue ARMcc;
     std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
-    SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
     EVT VT = Op.getValueType();
 
-    return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
-                   OverflowCmp, DAG);
+    return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
   }
 
   // Convert:
@@ -5258,14 +5235,9 @@ SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
         False = SelectTrue;
       }
 
-      if (True.getNode() && False.getNode()) {
-        EVT VT = Op.getValueType();
-        SDValue ARMcc = Cond.getOperand(2);
-        SDValue CCR = Cond.getOperand(3);
-        SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
-        assert(True.getValueType() == VT);
-        return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
-      }
+      if (True.getNode() && False.getNode())
+        return getCMOV(dl, Op.getValueType(), True, False, Cond.getOperand(2),
+                       Cond.getOperand(3), DAG);
     }
   }
 
@@ -5330,8 +5302,8 @@ static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
 }
 
 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
-                                   SDValue TrueVal, SDValue ARMcc, SDValue CCR,
-                                   SDValue Cmp, SelectionDAG &DAG) const {
+                                   SDValue TrueVal, SDValue ARMcc,
+                                   SDValue Flags, SelectionDAG &DAG) const {
   if (!Subtarget->hasFP64() && VT == MVT::f64) {
     FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
                            DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
@@ -5344,15 +5316,13 @@ SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
     SDValue FalseHigh = FalseVal.getValue(1);
 
     SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
-                              ARMcc, CCR, Cmp);
+                              ARMcc, Flags);
     SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
-                               ARMcc, CCR, duplicateCmp(Cmp, DAG));
+                               ARMcc, Flags);
 
     return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
-  } else {
-    return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,
-                       Cmp);
   }
+  return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, Flags);
 }
 
 static bool isGTorGE(ISD::CondCode CC) {
@@ -5625,12 +5595,11 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
     }
 
     SDValue ARMcc;
-    SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
     SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
     // Choose GE over PL, which vsel does now support
     if (ARMcc->getAsZExtVal() == ARMCC::PL)
       ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
-    return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
+    return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
   }
 
   ARMCC::CondCodes CondCode, CondCode2;
@@ -5660,13 +5629,10 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-  SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
+  SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
   if (CondCode2 != ARMCC::AL) {
     SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
-    // FIXME: Needs another CMP because flag can have but one use.
-    SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
-    Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
+    Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
   }
   return Result;
 }
@@ -5767,9 +5733,8 @@ ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
       RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
                         bitcastf32Toi32(RHS, DAG), Mask);
       SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
-      SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-      return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
-                         Chain, Dest, ARMcc, CCR, Cmp);
+      return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
+                         Cmp);
     }
 
     SDValue LHS1, LHS2;
@@ -5780,9 +5745,8 @@ ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
     RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
     ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
     ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
-    SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
-    return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
+    return DAG.getNode(ARMISD::BCC_i64, dl, MVT::Other, Ops);
   }
 
   return SDValue();
@@ -5816,9 +5780,8 @@ SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
         (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
     CondCode = ARMCC::getOppositeCondition(CondCode);
     ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
-    SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
 
-    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
+    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
                        OverflowCmp);
   }
 
@@ -5870,18 +5833,15 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
       CondCode = ARMCC::getOppositeCondition(CondCode);
       ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
     }
-    SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
 
-    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
+    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
                        OverflowCmp);
   }
 
   if (LHS.getValueType() == MVT::i32) {
     SDValue ARMcc;
     SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
-    SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
-                       Chain, Dest, ARMcc, CCR, Cmp);
+    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, Cmp);
   }
 
   if (getTargetMachine().Options.UnsafeFPMath &&
@@ -5896,14 +5856,12 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-  SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
-  SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
-  SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
+  SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
+  SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
   if (CondCode2 != ARMCC::AL) {
     ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
-    SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
-    Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
+    SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
+    Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
   }
   return Res;
 }
@@ -6408,7 +6366,6 @@ SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
   SDValue ShOpHi = Op.getOperand(1);
   SDValue ShAmt  = Op.getOperand(2);
   SDValue ARMcc;
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
   unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
 
   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
@@ -6423,8 +6380,8 @@ SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
   SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
   SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                             ISD::SETGE, ARMcc, DAG, dl);
-  SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift,
-                           ARMcc, CCR, CmpLo);
+  SDValue Lo =
+      DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, ARMcc, CmpLo);
 
   SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
   SDValue HiBigShift = Opc == ISD::SRA
@@ -6433,8 +6390,8 @@ SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
                            : DAG.getConstant(0, dl, VT);
   SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                             ISD::SETGE, ARMcc, DAG, dl);
-  SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
-                           ARMcc, CCR, CmpHi);
+  SDValue Hi =
+      DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
 
   SDValue Ops[2] = { Lo, Hi };
   return DAG.getMergeValues(Ops, dl);
@@ -6452,7 +6409,6 @@ SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
   SDValue ShOpHi = Op.getOperand(1);
   SDValue ShAmt  = Op.getOperand(2);
   SDValue ARMcc;
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
 
   assert(Op.getOpcode() == ISD::SHL_PARTS);
   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
@@ -6466,14 +6422,14 @@ SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
   SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
   SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                             ISD::SETGE, ARMcc, DAG, dl);
-  SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
-                           ARMcc, CCR, CmpHi);
+  SDValue Hi =
+      DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
 
   SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                           ISD::SETGE, ARMcc, DAG, dl);
   SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
   SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
-                           DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo);
+                           DAG.getConstant(0, dl, VT), ARMcc, CmpLo);
 
   SDValue Ops[2] = { Lo, Hi };
   return DAG.getMergeValues(Ops, dl);
@@ -7060,11 +7016,8 @@ static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
   SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
   SDValue ARMcc = DAG.getConstant(
       IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-  SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
-                                   Cmp.getValue(1), SDValue());
   return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
-                     CCR, Chain.getValue(1));
+                     Cmp.getValue(1));
 }
 
 /// isVMOVModifiedImm - Check if the specified splat value corresponds to a
@@ -10613,21 +10566,14 @@ SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
   ARMCC::CondCodes CondCode, CondCode2;
   FPCCToARMCC(CC, CondCode, CondCode2);
 
-  // FIXME: Chain is not handled correctly here. Currently the FPSCR is implicit
-  // in CMPFP and CMPFPE, but instead it should be made explicit by these
-  // instructions using a chain instead of glue. This would also fix the problem
-  // here (and also in LowerSELECT_CC) where we generate two comparisons when
-  // CondCode2 != AL.
   SDValue True = DAG.getConstant(1, dl, VT);
   SDValue False =  DAG.getConstant(0, dl, VT);
   SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
-  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
-  SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
+  SDValue Result = getCMOV(dl, VT, False, True, ARMcc, Cmp, DAG);
   if (CondCode2 != ARMCC::AL) {
     ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
-    Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
-    Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
+    Result = getCMOV(dl, VT, Result, True, ARMcc, Cmp, DAG);
   }
   return DAG.getMergeValues({Result, Chain}, dl);
 }
@@ -13880,6 +13826,14 @@ ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
           N->getOpcode() == ISD::SRL) &&
          "Expected shift op");
 
+  SDValue ShiftLHS = N->getOperand(0);
+  if (!ShiftLHS->hasOneUse())
+    return false;
+
+  if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
+      !ShiftLHS.getOperand(0)->hasOneUse())
+    return false;
+
   if (Level == BeforeLegalizeTypes)
     return true;
 
@@ -15057,7 +15011,7 @@ static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
 }
 
 // Check that N is CMPZ(CSINC(0, 0, CC, X)),
-//              or CMPZ(CMOV(1, 0, CC, $cpsr, X))
+//              or CMPZ(CMOV(1, 0, CC, X))
 // return X if valid.
 static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
   if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1)))
@@ -15081,22 +15035,22 @@ static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
   if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(0)) &&
       isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) {
     CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(2);
-    return CSInc.getOperand(4);
+    return CSInc.getOperand(3);
   }
   if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(CSInc.getOperand(1)) &&
       isNullConstant(CSInc.getOperand(0)) && CSInc->hasOneUse()) {
     CC = ARMCC::getOppositeCondition(
         (ARMCC::CondCodes)CSInc.getConstantOperandVal(2));
-    return CSInc.getOperand(4);
+    return CSInc.getOperand(3);
   }
   return SDValue();
 }
 
 static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
   // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
-  //       t92: glue = ARMISD::CMPZ t74, 0
+  //       t92: flags = ARMISD::CMPZ t74, 0
   //     t93: i32 = ARMISD::CSINC 0, 0, 1, t92
-  //   t96: glue = ARMISD::CMPZ t93, 0
+  //   t96: flags = ARMISD::CMPZ t93, 0
   // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
   ARMCC::CondCodes Cond;
   if (SDValue C = IsCMPZCSINC(N, Cond))
@@ -18187,7 +18141,7 @@ SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &D
   SDValue Op0 = CMOV->getOperand(0);
   SDValue Op1 = CMOV->getOperand(1);
   auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
-  SDValue CmpZ = CMOV->getOperand(4);
+  SDValue CmpZ = CMOV->getOperand(3);
 
   // The compare must be against zero.
   if (!isNullConstant(CmpZ->getOperand(1)))
@@ -18431,12 +18385,11 @@ static SDValue PerformHWLoopCombine(SDNode *N,
 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
 SDValue
 ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
-  SDValue Cmp = N->getOperand(4);
+  SDValue Cmp = N->getOperand(3);
   if (Cmp.getOpcode() != ARMISD::CMPZ)
     // Only looking at NE cases.
     return SDValue();
 
-  EVT VT = N->getValueType(0);
   SDLoc dl(N);
   SDValue LHS = Cmp.getOperand(0);
   SDValue RHS = Cmp.getOperand(1);
@@ -18445,17 +18398,17 @@ ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
   SDValue ARMcc = N->getOperand(2);
   ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
 
-  // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0))
-  // -> (brcond Chain BB CC CPSR Cmp)
+  // (brcond Chain BB ne (cmpz (and (cmov 0 1 CC Flags) 1) 0))
+  // -> (brcond Chain BB CC Flags)
   if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
       LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
       LHS->getOperand(0)->hasOneUse() &&
       isNullConstant(LHS->getOperand(0)->getOperand(0)) &&
       isOneConstant(LHS->getOperand(0)->getOperand(1)) &&
       isOneConstant(LHS->getOperand(1)) && isNullConstant(RHS)) {
-    return DAG.getNode(
-        ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2),
-        LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4));
+    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, BB,
+                       LHS->getOperand(0)->getOperand(2),
+                       LHS->getOperand(0)->getOperand(3));
   }
 
   return SDValue();
@@ -18464,7 +18417,7 @@ ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
 SDValue
 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
-  SDValue Cmp = N->getOperand(4);
+  SDValue Cmp = N->getOperand(3);
   if (Cmp.getOpcode() != ARMISD::CMPZ)
     // Only looking at EQ and NE cases.
     return SDValue();
@@ -18504,42 +18457,38 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   /// FIXME: Turn this into a target neutral optimization?
   SDValue Res;
   if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
-    Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
-                      N->getOperand(3), Cmp);
+    Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, Cmp);
   } else if (CC == ARMCC::EQ && TrueVal == RHS) {
     SDValue ARMcc;
     SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
-    Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
-                      N->getOperand(3), NewCmp);
+    Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, NewCmp);
   }
 
-  // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0))
-  // -> (cmov F T CC CPSR Cmp)
+  // (cmov F T ne (cmpz (cmov 0 1 CC Flags) 0))
+  // -> (cmov F T CC Flags)
   if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
       isNullConstant(LHS->getOperand(0)) && isOneConstant(LHS->getOperand(1)) &&
       isNullConstant(RHS)) {
     return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
-                       LHS->getOperand(2), LHS->getOperand(3),
-                       LHS->getOperand(4));
+                       LHS->getOperand(2), LHS->getOperand(3));
   }
 
   if (!VT.isInteger())
       return SDValue();
 
   // Fold away an unneccessary CMPZ/CMOV
-  // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) ->
-  // if C1==EQ -> CMOV A, B, C2, $cpsr, D
-  // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D
+  // CMOV A, B, C1, (CMPZ (CMOV 1, 0, C2, D), 0) ->
+  // if C1==EQ -> CMOV A, B, C2, D
+  // if C1==NE -> CMOV A, B, NOT(C2), D
   if (N->getConstantOperandVal(2) == ARMCC::EQ ||
       N->getConstantOperandVal(2) == ARMCC::NE) {
     ARMCC::CondCodes Cond;
-    if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) {
+    if (SDValue C = IsCMPZCSINC(N->getOperand(3).getNode(), Cond)) {
       if (N->getConstantOperandVal(2) == ARMCC::NE)
         Cond = ARMCC::getOppositeCondition(Cond);
       return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
                          N->getOperand(1),
-                         DAG.getTargetConstant(Cond, SDLoc(N), MVT::i32),
-                         N->getOperand(3), C);
+                         DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
     }
   }
 
@@ -18579,10 +18528,8 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
       // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
       SDValue Sub =
           DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
-      SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
-                                          Sub.getValue(1), SDValue());
       Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc,
-                        N->getOperand(3), CPSRGlue.getValue(1));
+                        Sub.getValue(1));
       FalseVal = Sub;
     }
   } else if (isNullConstant(TrueVal)) {
@@ -18593,11 +18540,9 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
       // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
       SDValue Sub =
           DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
-      SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
-                                          Sub.getValue(1), SDValue());
       Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal,
                         DAG.getConstant(ARMCC::NE, dl, MVT::i32),
-                        N->getOperand(3), CPSRGlue.getValue(1));
+                        Sub.getValue(1));
       FalseVal = Sub;
     }
   }
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 344a0ad91e517..4fa600e0cfcc4 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -978,13 +978,11 @@ class VectorType;
     bool isUnsupportedFloatingType(EVT VT) const;
 
     SDValue getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, SDValue TrueVal,
-                    SDValue ARMcc, SDValue CCR, SDValue Cmp,
-                    SelectionDAG &DAG) const;
+                    SDValue ARMcc, SDValue Flags, SelectionDAG &DAG) const;
     SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                       SDValue &ARMcc, SelectionDAG &DAG, const SDLoc &dl) const;
     SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
                       const SDLoc &dl, bool Signaling = false) const;
-    SDValue duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const;
 
     SDValue OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const;
 
diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td
index d0678f378da1e..041601748b1f7 100644
--- a/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -167,16 +167,6 @@ def pred : PredicateOperand<OtherVT, (ops i32imm, i32imm),
   let DecoderMethod = "DecodePredicateOperand";
 }
 
-// Selectable predicate operand for CMOV instructions. We can't use a normal
-// predicate because the default values interfere with instruction selection. In
-// all other respects it is identical though: pseudo-instruction expansion
-// relies on the MachineOperands being compatible.
-def cmovpred : Operand<i32>, PredicateOp,
-               ComplexPattern<i32, 2, "SelectCMOVPred"> {
-  let MIOperandInfo = (ops i32imm, i32imm);
-  let PrintMethod = "printPredicateOperand";
-}
-
 // Conditional code result for instructions whose 's' bit is set, e.g. subs.
 def CCOutOperand : AsmOperandClass {
   let Name = "CCOut";
@@ -1134,6 +1124,9 @@ class ARMV5MOPat<dag pattern, dag result> : Pat<pattern, result> {
 class ARMV6Pat<dag pattern, dag result> : Pat<pattern, result> {
   list<Predicate> Predicates = [IsARM, HasV6];
 }
+class ARMV6T2Pat<dag pattern, dag result> : Pat<pattern, result> {
+  list<Predicate> Predicates = [IsARM, HasV6T2];
+}
 class VFPPat<dag pattern, dag result> : Pat<pattern, result> {
   list<Predicate> Predicates = [HasVFP2];
 }
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index db38b43279b86..718cb964ab7c3 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -14,6 +14,9 @@
 // ARM specific DAG Nodes.
 //
 
+/// Value type used for "condition code" operands.
+defvar CondCodeVT = i32;
+
 /// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
 defvar FlagsVT = i32;
 
@@ -29,12 +32,19 @@ def SDT_ARMSaveCallPC : SDTypeProfile<0, 1, []>;
 
 def SDT_ARMcall    : SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>;
 
-def SDT_ARMCMov    : SDTypeProfile<1, 3,
-                                   [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
-                                    SDTCisVT<3, i32>]>;
+def SDT_ARMCMov : SDTypeProfile<1, 4, [
+  /* any */                // result
+  SDTCisSameAs<1, 0>,      // value on false
+  SDTCisSameAs<2, 0>,      // value on true
+  SDTCisVT<3, CondCodeVT>, // condition code
+  SDTCisVT<4, FlagsVT>,    // in flags
+]>;
 
-def SDT_ARMBrcond  : SDTypeProfile<0, 2,
-                                   [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>]>;
+def SDT_ARMBrcond : SDTypeProfile<0, 2, [
+  SDTCisVT<0, OtherVT>,    // target basic block
+  SDTCisVT<1, CondCodeVT>, // condition code
+  SDTCisVT<2, FlagsVT>,    // in flags
+]>;
 
 def SDT_ARMBrJT    : SDTypeProfile<0, 2,
                                   [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
@@ -53,7 +63,11 @@ def SDT_ARMAnd     : SDTypeProfile<1, 2,
                                    [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                     SDTCisVT<2, i32>]>;
 
-def SDT_ARMCmp     : SDTypeProfile<0, 2, [SDTCisSameAs<0, 1>]>;
+def SDT_ARMCmp : SDTypeProfile<1, 2, [
+  SDTCisVT<0, FlagsVT>, // out flags
+  SDTCisInt<1>,         // lhs
+  SDTCisSameAs<2, 1>    // rhs
+]>;
 
 def SDT_ARMPICAdd  : SDTypeProfile<1, 2, [SDTCisSameAs<0, 1>,
                                           SDTCisPtrTy<1>, SDTCisVT<2, i32>]>;
@@ -124,15 +138,17 @@ def ARMSmlaldx       : SDNode<"ARMISD::SMLALDX", SDT_LongMac>;
 def ARMSmlsld        : SDNode<"ARMISD::SMLSLD", SDT_LongMac>;
 def ARMSmlsldx       : SDNode<"ARMISD::SMLSLDX", SDT_LongMac>;
 
-def SDT_ARMCSel      : SDTypeProfile<1, 3,
-                                   [SDTCisSameAs<0, 1>,
-                                    SDTCisSameAs<0, 2>,
-                                    SDTCisInt<3>,
-                                    SDTCisVT<3, i32>]>;
+def SDT_ARMCSel : SDTypeProfile<1, 4, [
+  /* any */                // result
+  SDTCisSameAs<1, 0>,      // lhs
+  SDTCisSameAs<2, 0>,      // rhs
+  SDTCisVT<3, CondCodeVT>, // condition code
+  SDTCisVT<3, FlagsVT>     // in flags
+]>;
 
-def ARMcsinv         : SDNode<"ARMISD::CSINV", SDT_ARMCSel, [SDNPOptInGlue]>;
-def ARMcsneg         : SDNode<"ARMISD::CSNEG", SDT_ARMCSel, [SDNPOptInGlue]>;
-def ARMcsinc         : SDNode<"ARMISD::CSINC", SDT_ARMCSel, [SDNPOptInGlue]>;
+def ARMcsinv : SDNode<"ARMISD::CSINV", SDT_ARMCSel>;
+def ARMcsneg : SDNode<"ARMISD::CSNEG", SDT_ARMCSel>;
+def ARMcsinc : SDNode<"ARMISD::CSINC", SDT_ARMCSel>;
 
 def SDT_MulHSR       : SDTypeProfile<1, 3, [SDTCisVT<0,i32>,
                                             SDTCisSameAs<0, 1>,
@@ -173,15 +189,13 @@ def ARMseretglue     : SDNode<"ARMISD::SERET_GLUE", SDTNone,
                               [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
 def ARMintretglue    : SDNode<"ARMISD::INTRET_GLUE", SDT_ARMcall,
                               [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
-def ARMcmov          : SDNode<"ARMISD::CMOV", SDT_ARMCMov,
-                              [SDNPInGlue]>;
+def ARMcmov          : SDNode<"ARMISD::CMOV", SDT_ARMCMov>;
 
 def ARMssat   : SDNode<"ARMISD::SSAT", SDTIntSatNoShOp, []>;
 
 def ARMusat   : SDNode<"ARMISD::USAT", SDTIntSatNoShOp, []>;
 
-def ARMbrcond        : SDNode<"ARMISD::BRCOND", SDT_ARMBrcond,
-                              [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
+def ARMbrcond        : SDNode<"ARMISD::BRCOND", SDT_ARMBrcond, [SDNPHasChain]>;
 
 def ARMbrjt          : SDNode<"ARMISD::BR_JT", SDT_ARMBrJT,
                               [SDNPHasChain]>;
@@ -191,14 +205,11 @@ def ARMbr2jt         : SDNode<"ARMISD::BR2_JT", SDT_ARMBr2JT,
 def ARMBcci64        : SDNode<"ARMISD::BCC_i64", SDT_ARMBCC_i64,
                               [SDNPHasChain]>;
 
-def ARMcmp           : SDNode<"ARMISD::CMP", SDT_ARMCmp,
-                              [SDNPOutGlue]>;
+def ARMcmp           : SDNode<"ARMISD::CMP", SDT_ARMCmp>;
 
-def ARMcmn           : SDNode<"ARMISD::CMN", SDT_ARMCmp,
-                              [SDNPOutGlue]>;
+def ARMcmn           : SDNode<"ARMISD::CMN", SDT_ARMCmp>;
 
-def ARMcmpZ          : SDNode<"ARMISD::CMPZ", SDT_ARMCmp,
-                              [SDNPOutGlue, SDNPCommutative]>;
+def ARMcmpZ          : SDNode<"ARMISD::CMPZ", SDT_ARMCmp, [SDNPCommutative]>;
 
 def ARMpic_add       : SDNode<"ARMISD::PIC_ADD", SDT_ARMPICAdd>;
 
@@ -1776,7 +1787,7 @@ multiclass AI1_cmp_irs<bits<4> opcod, string opc,
                      string rrDecoderMethod = ""> {
   def ri : AI1<opcod, (outs), (ins GPR:$Rn, mod_imm:$imm), DPFrm, iii,
                opc, "\t$Rn, $imm",
-               [(opnode GPR:$Rn, mod_imm:$imm)]>,
+               [(set CPSR, (opnode GPR:$Rn, mod_imm:$imm))]>,
            Sched<[WriteCMP, ReadALU]> {
     bits<4> Rn;
     bits<12> imm;
@@ -1790,7 +1801,7 @@ multiclass AI1_cmp_irs<bits<4> opcod, string opc,
   }
   def rr : AI1<opcod, (outs), (ins GPR:$Rn, GPR:$Rm), DPFrm, iir,
                opc, "\t$Rn, $Rm",
-               [(opnode GPR:$Rn, GPR:$Rm)]>,
+               [(set CPSR, (opnode GPR:$Rn, GPR:$Rm))]>,
            Sched<[WriteCMP, ReadALU, ReadALU]> {
     bits<4> Rn;
     bits<4> Rm;
@@ -1808,7 +1819,7 @@ multiclass AI1_cmp_irs<bits<4> opcod, string opc,
   def rsi : AI1<opcod, (outs),
                (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm, iis,
                opc, "\t$Rn, $shift",
-               [(opnode GPR:$Rn, so_reg_imm:$shift)]>,
+               [(set CPSR, (opnode GPR:$Rn, so_reg_imm:$shift))]>,
             Sched<[WriteCMPsi, ReadALU]> {
     bits<4> Rn;
     bits<12> shift;
@@ -1825,7 +1836,7 @@ multiclass AI1_cmp_irs<bits<4> opcod, string opc,
   def rsr : AI1<opcod, (outs),
                (ins GPRnopc:$Rn, so_reg_reg:$shift), DPSoRegRegFrm, iis,
                opc, "\t$Rn, $shift",
-               [(opnode GPRnopc:$Rn, so_reg_reg:$shift)]>,
+               [(set CPSR, (opnode GPRnopc:$Rn, so_reg_reg:$shift))]>,
             Sched<[WriteCMPsr, ReadALU]> {
     bits<4> Rn;
     bits<12> shift;
@@ -4943,7 +4954,7 @@ def : ARMPat<(ARMcmpZ so_reg_reg:$rhs, 0),
 let isCompare = 1, Defs = [CPSR] in {
 def CMNri : AI1<0b1011, (outs), (ins GPR:$Rn, mod_imm:$imm), DPFrm, IIC_iCMPi,
                 "cmn", "\t$Rn, $imm",
-                [(ARMcmn GPR:$Rn, mod_imm:$imm)]>,
+                [(set CPSR, (ARMcmn GPR:$Rn, mod_imm:$imm))]>,
                 Sched<[WriteCMP, ReadALU]> {
   bits<4> Rn;
   bits<12> imm;
@@ -4959,8 +4970,8 @@ def CMNri : AI1<0b1011, (outs), (ins GPR:$Rn, mod_imm:$imm), DPFrm, IIC_iCMPi,
 // CMN register-register/shift
 def CMNzrr : AI1<0b1011, (outs), (ins GPR:$Rn, GPR:$Rm), DPFrm, IIC_iCMPr,
                  "cmn", "\t$Rn, $Rm",
-                 [(BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
-                   GPR:$Rn, GPR:$Rm)]>, Sched<[WriteCMP, ReadALU, ReadALU]> {
+                 [(set CPSR, (BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
+                   GPR:$Rn, GPR:$Rm))]>, Sched<[WriteCMP, ReadALU, ReadALU]> {
   bits<4> Rn;
   bits<4> Rm;
   let isCommutable = 1;
@@ -4977,8 +4988,8 @@ def CMNzrr : AI1<0b1011, (outs), (ins GPR:$Rn, GPR:$Rm), DPFrm, IIC_iCMPr,
 def CMNzrsi : AI1<0b1011, (outs),
                   (ins GPR:$Rn, so_reg_imm:$shift), DPSoRegImmFrm, IIC_iCMPsr,
                   "cmn", "\t$Rn, $shift",
-                  [(BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
-                    GPR:$Rn, so_reg_imm:$shift)]>,
+                  [(set CPSR, (BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
+                    GPR:$Rn, so_reg_imm:$shift))]>,
                     Sched<[WriteCMPsi, ReadALU]> {
   bits<4> Rn;
   bits<12> shift;
@@ -4996,8 +5007,8 @@ def CMNzrsi : AI1<0b1011, (outs),
 def CMNzrsr : AI1<0b1011, (outs),
                   (ins GPRnopc:$Rn, so_reg_reg:$shift), DPSoRegRegFrm, IIC_iCMPsr,
                   "cmn", "\t$Rn, $shift",
-                  [(BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
-                    GPRnopc:$Rn, so_reg_reg:$shift)]>,
+                  [(set CPSR, (BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
+                    GPRnopc:$Rn, so_reg_reg:$shift))]>,
                     Sched<[WriteCMPsr, ReadALU]> {
   bits<4> Rn;
   bits<12> shift;
@@ -5052,65 +5063,74 @@ let hasSideEffects = 0 in {
 
 let isCommutable = 1, isSelect = 1 in
 def MOVCCr : ARMPseudoInst<(outs GPR:$Rd),
-                           (ins GPR:$false, GPR:$Rm, cmovpred:$p),
-                           4, IIC_iCMOVr,
-                           [(set GPR:$Rd, (ARMcmov GPR:$false, GPR:$Rm,
-                                                   cmovpred:$p))]>,
+                           (ins GPR:$false, GPR:$Rm, pred:$p),
+                           4, IIC_iCMOVr, []>,
              RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 def MOVCCsi : ARMPseudoInst<(outs GPR:$Rd),
-                            (ins GPR:$false, so_reg_imm:$shift, cmovpred:$p),
-                            4, IIC_iCMOVsr,
-                            [(set GPR:$Rd,
-                                  (ARMcmov GPR:$false, so_reg_imm:$shift,
-                                           cmovpred:$p))]>,
+                            (ins GPR:$false, so_reg_imm:$shift, pred:$p),
+                            4, IIC_iCMOVsr, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 def MOVCCsr : ARMPseudoInst<(outs GPR:$Rd),
-                            (ins GPR:$false, so_reg_reg:$shift, cmovpred:$p),
-                           4, IIC_iCMOVsr,
-  [(set GPR:$Rd, (ARMcmov GPR:$false, so_reg_reg:$shift,
-                            cmovpred:$p))]>,
+                            (ins GPR:$false, so_reg_reg:$shift, pred:$p),
+                           4, IIC_iCMOVsr, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 
 let isMoveImm = 1 in
 def MOVCCi16
     : ARMPseudoInst<(outs GPR:$Rd),
-                    (ins GPR:$false, imm0_65535_expr:$imm, cmovpred:$p),
-                    4, IIC_iMOVi,
-                    [(set GPR:$Rd, (ARMcmov GPR:$false, imm0_65535:$imm,
-                                            cmovpred:$p))]>,
+                    (ins GPR:$false, imm0_65535_expr:$imm, pred:$p),
+                    4, IIC_iMOVi, []>,
       RegConstraint<"$false = $Rd">, Requires<[IsARM, HasV6T2]>,
       Sched<[WriteALU]>;
 
 let isMoveImm = 1 in
 def MOVCCi : ARMPseudoInst<(outs GPR:$Rd),
-                           (ins GPR:$false, mod_imm:$imm, cmovpred:$p),
-                           4, IIC_iCMOVi,
-                           [(set GPR:$Rd, (ARMcmov GPR:$false, mod_imm:$imm,
-                                                   cmovpred:$p))]>,
+                           (ins GPR:$false, mod_imm:$imm, pred:$p),
+                           4, IIC_iCMOVi, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 // Two instruction predicate mov immediate.
 let isMoveImm = 1 in
 def MOVCCi32imm
     : ARMPseudoInst<(outs GPR:$Rd),
-                    (ins GPR:$false, i32imm:$src, cmovpred:$p),
-                    8, IIC_iCMOVix2,
-                    [(set GPR:$Rd, (ARMcmov GPR:$false, imm:$src,
-                                            cmovpred:$p))]>,
+                    (ins GPR:$false, i32imm:$src, pred:$p),
+                    8, IIC_iCMOVix2, []>,
       RegConstraint<"$false = $Rd">, Requires<[IsARM, HasV6T2]>;
 
 let isMoveImm = 1 in
 def MVNCCi : ARMPseudoInst<(outs GPR:$Rd),
-                           (ins GPR:$false, mod_imm:$imm, cmovpred:$p),
-                           4, IIC_iCMOVi,
-                           [(set GPR:$Rd, (ARMcmov GPR:$false, mod_imm_not:$imm,
-                                                   cmovpred:$p))]>,
+                           (ins GPR:$false, mod_imm:$imm, pred:$p),
+                           4, IIC_iCMOVi, []>,
                 RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 } // hasSideEffects
 
+// The following patterns have to be defined out-of-line because the number
+// of instruction operands does not match the number of SDNode operands
+// (`pred` counts as one operand).
+
+def : ARMPat<(ARMcmov i32:$false, i32:$Rm, imm:$cc, CPSR),
+             (MOVCCr $false, $Rm, imm:$cc, CPSR)>;
+
+def : ARMPat<(ARMcmov i32:$false, so_reg_imm:$shift, imm:$cc, CPSR),
+             (MOVCCsi $false, so_reg_imm:$shift, imm:$cc, CPSR)>;
+
+def : ARMPat<(ARMcmov i32:$false, so_reg_reg:$shift, imm:$cc, CPSR),
+             (MOVCCsr $false, so_reg_reg:$shift, imm:$cc, CPSR)>;
+
+def : ARMV6T2Pat<(ARMcmov i32:$false, imm0_65535:$imm, imm:$cc, CPSR),
+                 (MOVCCi16 $false, imm0_65535:$imm, imm:$cc, CPSR)>;
+
+def : ARMPat<(ARMcmov i32:$false, mod_imm:$imm, imm:$cc, CPSR),
+             (MOVCCi $false, mod_imm:$imm, imm:$cc, CPSR)>;
+
+def : ARMPat<(ARMcmov i32:$false, mod_imm_not:$imm, imm:$cc, CPSR),
+             (MVNCCi $false, mod_imm_not:$imm, imm:$cc, CPSR)>;
+
+def : ARMV6T2Pat<(ARMcmov i32:$false, imm:$src, imm:$cc, CPSR),
+                 (MOVCCi32imm $false, imm:$src, imm:$cc, CPSR)>;
 
 //===----------------------------------------------------------------------===//
 // Atomic operations intrinsics
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td
index b92f42874bbdd..cc7fc743fe4f9 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb.td
@@ -23,8 +23,7 @@ def imm_sr_XFORM: SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant((Imm == 32 ? 0 : Imm), SDLoc(N), MVT::i32);
 }]>;
 def ThumbSRImmAsmOperand: ImmAsmOperand<1,32> { let Name = "ImmThumbSR"; }
-def imm_sr : Operand<i32>, PatLeaf<(imm), [{
-  uint64_t Imm = N->getZExtValue();
+def imm_sr : Operand<i32>, ImmLeaf<i32, [{
   return Imm > 0 && Imm <= 32;
 }], imm_sr_XFORM> {
   let PrintMethod = "printThumbSRImm";
@@ -1108,13 +1107,14 @@ let isCompare = 1, Defs = [CPSR] in {
 //  T1pIDPEncode<0b1011, (outs), (ins tGPR:$lhs, tGPR:$rhs),
 //               IIC_iCMPr,
 //               "cmn", "\t$lhs, $rhs",
-//               [(ARMcmp tGPR:$lhs, (ineg tGPR:$rhs))]>;
+//               [(set CPSR, (ARMcmp tGPR:$lhs, (ineg tGPR:$rhs)))]>;
 
 def tCMNz :                     // A8.6.33
   T1pIDPEncode<0b1011, (outs), (ins tGPR:$Rn, tGPR:$Rm),
                IIC_iCMPr,
                "cmn", "\t$Rn, $Rm",
-               [(ARMcmpZ tGPR:$Rn, (ineg tGPR:$Rm))]>, Sched<[WriteCMP]>;
+               [(set CPSR, (ARMcmpZ tGPR:$Rn, (ineg tGPR:$Rm)))]>,
+  Sched<[WriteCMP]>;
 
 } // isCompare = 1, Defs = [CPSR]
 
@@ -1122,7 +1122,7 @@ def tCMNz :                     // A8.6.33
 let isCompare = 1, Defs = [CPSR] in {
 def tCMPi8 : T1pI<(outs), (ins tGPR:$Rn, imm0_255:$imm8), IIC_iCMPi,
                   "cmp", "\t$Rn, $imm8",
-                  [(ARMcmp tGPR:$Rn, imm0_255:$imm8)]>,
+                  [(set CPSR, (ARMcmp tGPR:$Rn, imm0_255:$imm8))]>,
              T1General<{1,0,1,?,?}>, Sched<[WriteCMP]> {
   // A8.6.35
   bits<3> Rn;
@@ -1136,7 +1136,7 @@ def tCMPr :                     // A8.6.36 T1
   T1pIDPEncode<0b1010, (outs), (ins tGPR:$Rn, tGPR:$Rm),
                IIC_iCMPr,
                "cmp", "\t$Rn, $Rm",
-               [(ARMcmp tGPR:$Rn, tGPR:$Rm)]>, Sched<[WriteCMP]>;
+               [(set CPSR, (ARMcmp tGPR:$Rn, tGPR:$Rm))]>, Sched<[WriteCMP]>;
 
 def tCMPhir : T1pI<(outs), (ins GPR:$Rn, GPR:$Rm), IIC_iCMPr,
                    "cmp", "\t$Rn, $Rm", []>,
@@ -1423,7 +1423,7 @@ let isCompare = 1, isCommutable = 1, Defs = [CPSR] in
 def tTST :                      // A8.6.230
   T1pIDPEncode<0b1000, (outs), (ins tGPR:$Rn, tGPR:$Rm), IIC_iTSTr,
                "tst", "\t$Rn, $Rm",
-               [(ARMcmpZ (and_su tGPR:$Rn, tGPR:$Rm), 0)]>,
+               [(set CPSR, (ARMcmpZ (and_su tGPR:$Rn, tGPR:$Rm), 0))]>,
                Sched<[WriteALU]>;
 
 // A8.8.247  UDF - Undefined (Encoding T1)
@@ -1466,9 +1466,11 @@ def tUXTH :                     // A8.6.264
 // Expanded after instruction selection into a branch sequence.
 let usesCustomInserter = 1 in  // Expanded after instruction selection.
   def tMOVCCr_pseudo :
-  PseudoInst<(outs tGPR:$dst), (ins tGPR:$false, tGPR:$true, cmovpred:$p),
-             NoItinerary,
-             [(set tGPR:$dst, (ARMcmov tGPR:$false, tGPR:$true, cmovpred:$p))]>;
+  PseudoInst<(outs tGPR:$dst), (ins tGPR:$false, tGPR:$true, pred:$p),
+             NoItinerary, []>;
+
+def : Pat<(ARMcmov tGPR:$false, tGPR:$true, imm:$cc, CPSR),
+          (tMOVCCr_pseudo $false, $true, imm:$cc, CPSR)>;
 
 // tLEApcrel - Load a pc-relative address into a register without offending the
 // assembler.
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index aa5c0a5889768..99617e53d657a 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -1144,7 +1144,8 @@ let isCompare = 1, Defs = [CPSR] in {
    def ri : T2OneRegCmpImm<
                 (outs), (ins LHSGPR:$Rn, t2_so_imm:$imm), iii,
                 opc, ".w\t$Rn, $imm",
-                [(opnode LHSGPR:$Rn, t2_so_imm:$imm)]>, Sched<[WriteCMP]> {
+                [(set CPSR, (opnode LHSGPR:$Rn, t2_so_imm:$imm))]>,
+            Sched<[WriteCMP]> {
      let Inst{31-27} = 0b11110;
      let Inst{25} = 0;
      let Inst{24-21} = opcod;
@@ -1156,7 +1157,8 @@ let isCompare = 1, Defs = [CPSR] in {
    def rr : T2TwoRegCmp<
                 (outs), (ins LHSGPR:$Rn, rGPR:$Rm), iir,
                 opc, ".w\t$Rn, $Rm",
-                [(opnode LHSGPR:$Rn, rGPR:$Rm)]>, Sched<[WriteCMP]> {
+                [(set CPSR, (opnode LHSGPR:$Rn, rGPR:$Rm))]>,
+            Sched<[WriteCMP]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
      let Inst{24-21} = opcod;
@@ -1170,7 +1172,7 @@ let isCompare = 1, Defs = [CPSR] in {
    def rs : T2OneRegCmpShiftedReg<
                 (outs), (ins LHSGPR:$Rn, t2_so_reg:$ShiftedRm), iis,
                 opc, ".w\t$Rn, $ShiftedRm",
-                [(opnode LHSGPR:$Rn, t2_so_reg:$ShiftedRm)]>,
+                [(set CPSR, (opnode LHSGPR:$Rn, t2_so_reg:$ShiftedRm))]>,
                 Sched<[WriteCMPsi]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
@@ -3477,7 +3479,7 @@ let isCompare = 1, Defs = [CPSR] in {
    def t2CMNri : T2OneRegCmpImm<
                 (outs), (ins GPRnopc:$Rn, t2_so_imm:$imm), IIC_iCMPi,
                 "cmn", ".w\t$Rn, $imm",
-                [(ARMcmn GPRnopc:$Rn, (ineg t2_so_imm:$imm))]>,
+                [(set CPSR, (ARMcmn GPRnopc:$Rn, (ineg t2_so_imm:$imm)))]>,
                 Sched<[WriteCMP, ReadALU]> {
      let Inst{31-27} = 0b11110;
      let Inst{25} = 0;
@@ -3490,8 +3492,9 @@ let isCompare = 1, Defs = [CPSR] in {
    def t2CMNzrr : T2TwoRegCmp<
                 (outs), (ins GPRnopc:$Rn, rGPR:$Rm), IIC_iCMPr,
                 "cmn", ".w\t$Rn, $Rm",
-                [(BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
-                  GPRnopc:$Rn, rGPR:$Rm)]>, Sched<[WriteCMP, ReadALU, ReadALU]> {
+                [(set CPSR, (BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
+                  GPRnopc:$Rn, rGPR:$Rm))]>,
+                  Sched<[WriteCMP, ReadALU, ReadALU]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
      let Inst{24-21} = 0b1000;
@@ -3505,8 +3508,8 @@ let isCompare = 1, Defs = [CPSR] in {
    def t2CMNzrs : T2OneRegCmpShiftedReg<
                 (outs), (ins GPRnopc:$Rn, t2_so_reg:$ShiftedRm), IIC_iCMPsi,
                 "cmn", ".w\t$Rn, $ShiftedRm",
-                [(BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
-                  GPRnopc:$Rn, t2_so_reg:$ShiftedRm)]>,
+                [(set CPSR, (BinOpFrag<(ARMcmpZ node:$LHS,(ineg node:$RHS))>
+                  GPRnopc:$Rn, t2_so_reg:$ShiftedRm))]>,
                   Sched<[WriteCMPsi, ReadALU, ReadALU]> {
      let Inst{31-27} = 0b11101;
      let Inst{26-25} = 0b01;
@@ -3542,67 +3545,84 @@ let hasSideEffects = 0 in {
 
 let isCommutable = 1, isSelect = 1 in
 def t2MOVCCr : t2PseudoInst<(outs rGPR:$Rd),
-                            (ins rGPR:$false, rGPR:$Rm, cmovpred:$p),
-                            4, IIC_iCMOVr,
-                            [(set rGPR:$Rd, (ARMcmov rGPR:$false, rGPR:$Rm,
-                                                     cmovpred:$p))]>,
+                            (ins rGPR:$false, rGPR:$Rm, pred:$p),
+                            4, IIC_iCMOVr, []>,
                RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 let isMoveImm = 1 in
 def t2MOVCCi
     : t2PseudoInst<(outs rGPR:$Rd),
-                   (ins rGPR:$false, t2_so_imm:$imm, cmovpred:$p),
-                   4, IIC_iCMOVi,
-                   [(set rGPR:$Rd, (ARMcmov rGPR:$false,t2_so_imm:$imm,
-                                            cmovpred:$p))]>,
+                   (ins rGPR:$false, t2_so_imm:$imm, pred:$p),
+                   4, IIC_iCMOVi, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 let isCodeGenOnly = 1 in {
 let isMoveImm = 1 in
 def t2MOVCCi16
     : t2PseudoInst<(outs rGPR:$Rd),
-                   (ins  rGPR:$false, imm0_65535_expr:$imm, cmovpred:$p),
-                   4, IIC_iCMOVi,
-                   [(set rGPR:$Rd, (ARMcmov rGPR:$false, imm0_65535:$imm,
-                                            cmovpred:$p))]>,
+                   (ins rGPR:$false, imm0_65535_expr:$imm, pred:$p),
+                   4, IIC_iCMOVi, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
 let isMoveImm = 1 in
 def t2MVNCCi
     : t2PseudoInst<(outs rGPR:$Rd),
-                   (ins rGPR:$false, t2_so_imm:$imm, cmovpred:$p),
-                   4, IIC_iCMOVi,
-                   [(set rGPR:$Rd,
-                         (ARMcmov rGPR:$false, t2_so_imm_not:$imm,
-                                  cmovpred:$p))]>,
+                   (ins rGPR:$false, t2_so_imm:$imm, pred:$p),
+                   4, IIC_iCMOVi, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
-class MOVCCShPseudo<SDPatternOperator opnode, Operand ty>
+class MOVCCShPseudo
     : t2PseudoInst<(outs rGPR:$Rd),
-                   (ins rGPR:$false, rGPR:$Rm, i32imm:$imm, cmovpred:$p),
-                   4, IIC_iCMOVsi,
-                   [(set rGPR:$Rd, (ARMcmov rGPR:$false,
-                                            (opnode rGPR:$Rm, (i32 ty:$imm)),
-                                            cmovpred:$p))]>,
+                   (ins rGPR:$false, rGPR:$Rm, i32imm:$imm, pred:$p),
+                   4, IIC_iCMOVsi, []>,
       RegConstraint<"$false = $Rd">, Sched<[WriteALU]>;
 
-def t2MOVCClsl : MOVCCShPseudo<shl,  imm0_31>;
-def t2MOVCClsr : MOVCCShPseudo<srl,  imm_sr>;
-def t2MOVCCasr : MOVCCShPseudo<sra,  imm_sr>;
-def t2MOVCCror : MOVCCShPseudo<rotr, imm0_31>;
+def t2MOVCClsl : MOVCCShPseudo;
+def t2MOVCClsr : MOVCCShPseudo;
+def t2MOVCCasr : MOVCCShPseudo;
+def t2MOVCCror : MOVCCShPseudo;
 
 let isMoveImm = 1 in
 def t2MOVCCi32imm
     : t2PseudoInst<(outs rGPR:$dst),
-                   (ins rGPR:$false, i32imm:$src, cmovpred:$p),
-                   8, IIC_iCMOVix2,
-                   [(set rGPR:$dst, (ARMcmov rGPR:$false, imm:$src,
-                                             cmovpred:$p))]>,
+                   (ins rGPR:$false, i32imm:$src, pred:$p),
+                   8, IIC_iCMOVix2, []>,
       RegConstraint<"$false = $dst">;
 } // isCodeGenOnly = 1
 
 } // hasSideEffects
 
+// The following patterns have to be defined out-of-line because the number
+// of instruction operands does not match the number of SDNode operands
+// (`pred` counts as one operand).
+
+def : T2Pat<(ARMcmov i32:$false, i32:$Rm, imm:$cc, CPSR),
+            (t2MOVCCr $false, $Rm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, t2_so_imm:$imm, imm:$cc, CPSR),
+            (t2MOVCCi $false, t2_so_imm:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, imm0_65535:$imm, imm:$cc, CPSR),
+            (t2MOVCCi16 $false, imm0_65535:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, t2_so_imm_not:$imm, imm:$cc, CPSR),
+            (t2MVNCCi $false, t2_so_imm_not:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, (shl i32:$Rm, imm0_31:$imm), imm:$cc, CPSR),
+            (t2MOVCClsl $false, $Rm, imm0_31:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, (srl i32:$Rm, imm_sr:$imm), imm:$cc, CPSR),
+            (t2MOVCClsr $false, $Rm, imm_sr:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, (sra i32:$Rm, imm_sr:$imm), imm:$cc, CPSR),
+            (t2MOVCCasr $false, $Rm, imm_sr:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, (rotr i32:$Rm, imm0_31:$imm), imm:$cc, CPSR),
+            (t2MOVCCror $false, $Rm, imm0_31:$imm, imm:$cc, CPSR)>;
+
+def : T2Pat<(ARMcmov i32:$false, imm:$src, imm:$cc, CPSR),
+            (t2MOVCCi32imm $false, imm:$src, imm:$cc, CPSR)>;
+
 //===----------------------------------------------------------------------===//
 // Atomic operations intrinsics
 //
@@ -5706,51 +5726,53 @@ def t2CSINC : CS<"csinc", 0b1001>;
 def t2CSINV : CS<"csinv", 0b1010>;
 def t2CSNEG : CS<"csneg", 0b1011>;
 
-def ARMcsinc_su : PatFrag<(ops node:$lhs, node:$rhs, node:$cond),
-                          (ARMcsinc node:$lhs, node:$rhs, node:$cond), [{
+def ARMcsinc_su
+    : PatFrag<(ops node:$lhs, node:$rhs, node:$cc, node:$flags),
+              (ARMcsinc node:$lhs, node:$rhs, node:$cc, node:$flags), [{
   return N->hasOneUse();
 }]>;
 
 let Predicates = [HasV8_1MMainline] in {
   multiclass CSPats<SDNode Node, Instruction Insn> {
-    def : T2Pat<(Node GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
-                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
-    def : T2Pat<(Node (i32 0), GPRwithZR:$fval, imm0_31:$imm),
-                (Insn ZR, GPRwithZR:$fval, imm0_31:$imm)>;
-    def : T2Pat<(Node GPRwithZR:$tval, (i32 0), imm0_31:$imm),
-                (Insn GPRwithZR:$tval, ZR, imm0_31:$imm)>;
-    def : T2Pat<(Node (i32 0), (i32 0), imm0_31:$imm),
-                (Insn ZR, ZR, imm0_31:$imm)>;
+    def : T2Pat<(Node GPRwithZR:$tval, GPRwithZR:$fval, imm:$cc, CPSR),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm:$cc)>;
+    def : T2Pat<(Node (i32 0), GPRwithZR:$fval, imm:$cc, CPSR),
+                (Insn ZR, GPRwithZR:$fval, imm:$cc)>;
+    def : T2Pat<(Node GPRwithZR:$tval, (i32 0), imm:$cc, CPSR),
+                (Insn GPRwithZR:$tval, ZR, imm:$cc)>;
+    def : T2Pat<(Node (i32 0), (i32 0), imm:$cc, CPSR),
+                (Insn ZR, ZR, imm:$cc)>;
   }
 
   defm : CSPats<ARMcsinc, t2CSINC>;
   defm : CSPats<ARMcsinv, t2CSINV>;
   defm : CSPats<ARMcsneg, t2CSNEG>;
 
-  def : T2Pat<(ARMcmov (i32 1), (i32 0), cmovpred:$imm),
-              (t2CSINC ZR, ZR, imm0_31:$imm)>;
-  def : T2Pat<(ARMcmov (i32 -1), (i32 0), cmovpred:$imm),
-              (t2CSINV ZR, ZR, imm0_31:$imm)>;
-  def : T2Pat<(ARMcmov (i32 0), (i32 1), cmovpred:$imm),
-              (t2CSINC ZR, ZR, (inv_cond_XFORM imm:$imm))>;
-  def : T2Pat<(ARMcmov (i32 0), (i32 -1), cmovpred:$imm),
-              (t2CSINV ZR, ZR, (inv_cond_XFORM imm:$imm))>;
+  def : T2Pat<(ARMcmov (i32 1), (i32 0), imm:$cc, CPSR),
+              (t2CSINC ZR, ZR, imm:$cc)>;
+  def : T2Pat<(ARMcmov (i32 -1), (i32 0), imm:$cc, CPSR),
+              (t2CSINV ZR, ZR, imm:$cc)>;
+  def : T2Pat<(ARMcmov (i32 0), (i32 1), imm:$cc, CPSR),
+              (t2CSINC ZR, ZR, (inv_cond_XFORM imm:$cc))>;
+  def : T2Pat<(ARMcmov (i32 0), (i32 -1), imm:$cc, CPSR),
+              (t2CSINV ZR, ZR, (inv_cond_XFORM imm:$cc))>;
 
   multiclass ModifiedV8_1CSEL<Instruction Insn, dag modvalue> {
-    def : T2Pat<(ARMcmov modvalue, GPRwithZR:$tval, cmovpred:$imm),
-                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
-    def : T2Pat<(ARMcmov GPRwithZR:$tval, modvalue, cmovpred:$imm),
+    def : T2Pat<(ARMcmov modvalue, GPRwithZR:$tval, imm:$cc, CPSR),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm:$cc)>;
+    def : T2Pat<(ARMcmov GPRwithZR:$tval, modvalue, imm:$cc, CPSR),
                 (Insn GPRwithZR:$tval, GPRwithZR:$fval,
-                         (i32 (inv_cond_XFORM imm:$imm)))>;
+                         (i32 (inv_cond_XFORM imm:$cc)))>;
   }
   defm : ModifiedV8_1CSEL<t2CSINC, (add rGPR:$fval, 1)>;
   defm : ModifiedV8_1CSEL<t2CSINV, (xor rGPR:$fval, -1)>;
   defm : ModifiedV8_1CSEL<t2CSNEG, (sub 0, rGPR:$fval)>;
 
-  def : T2Pat<(ARMcmov (topbitsallzero32:$Rn), (i32 1), cmovpred:$imm),
-              (t2CSINC $Rn, ZR, (inv_cond_XFORM imm:$imm))>;
-  def : T2Pat<(and (topbitsallzero32:$Rn), (ARMcsinc_su (i32 0), (i32 0), cmovpred:$imm)),
-              (t2CSEL ZR, $Rn, $imm)>;
+  def : T2Pat<(ARMcmov (topbitsallzero32:$Rn), (i32 1), imm:$cc, CPSR),
+              (t2CSINC $Rn, ZR, (inv_cond_XFORM imm:$cc))>;
+  def : T2Pat<(and (topbitsallzero32:$Rn),
+                   (ARMcsinc_su (i32 0), (i32 0), imm:$cc, CPSR)),
+              (t2CSEL ZR, $Rn, imm:$cc)>;
 }
 
 // CS aliases.
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index a29753909ea99..754517f3bc4d5 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -34,10 +34,10 @@ def arm_cmpfpe  : SDNode<"ARMISD::CMPFPE",   SDT_CMPFP>;
 def arm_cmpfpe0 : SDNode<"ARMISD::CMPFPEw0", SDT_CMPFP0>;
 
 def arm_fmstat : SDNode<"ARMISD::FMSTAT",
-  SDTypeProfile<0, 1, [
-    SDTCisVT<0, FlagsVT> // in flags
-  ]>,
-  [SDNPOutGlue] // TODO: Change Glue to a normal result.
+  SDTypeProfile<1, 1, [
+    SDTCisVT<0, FlagsVT>, // out flags
+    SDTCisVT<1, FlagsVT>  // in flags
+  ]>
 >;
 
 def arm_fmdrr  : SDNode<"ARMISD::VMOVDRR", SDT_VMOVDRR>;
@@ -562,19 +562,21 @@ multiclass vsel_inst<string op, bits<2> opc, int CC> {
     def H : AHbInp<0b11100, opc, 0,
                    (outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm),
                    NoItinerary, !strconcat("vsel", op, ".f16\t$Sd, $Sn, $Sm"),
-                   [(set (f16 HPR:$Sd), (ARMcmov (f16 HPR:$Sm), (f16 HPR:$Sn), CC))]>,
+                   [(set (f16 HPR:$Sd),
+                         (ARMcmov (f16 HPR:$Sm), (f16 HPR:$Sn), CC, CPSR))]>,
                    Requires<[HasFullFP16]>;
 
     def S : ASbInp<0b11100, opc, 0,
                    (outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm),
                    NoItinerary, !strconcat("vsel", op, ".f32\t$Sd, $Sn, $Sm"),
-                   [(set SPR:$Sd, (ARMcmov SPR:$Sm, SPR:$Sn, CC))]>,
+                   [(set SPR:$Sd, (ARMcmov SPR:$Sm, SPR:$Sn, CC, CPSR))]>,
                    Requires<[HasFPARMv8]>;
 
     def D : ADbInp<0b11100, opc, 0,
                    (outs DPR:$Dd), (ins DPR:$Dn, DPR:$Dm),
                    NoItinerary, !strconcat("vsel", op, ".f64\t$Dd, $Dn, $Dm"),
-                   [(set DPR:$Dd, (ARMcmov (f64 DPR:$Dm), (f64 DPR:$Dn), CC))]>,
+                   [(set DPR:$Dd,
+                         (ARMcmov (f64 DPR:$Dm), (f64 DPR:$Dn), CC, CPSR))]>,
                    Requires<[HasFPARMv8, HasDPVFP]>;
   }
 }
@@ -2461,25 +2463,35 @@ def : Pat<(fneg (f16 (fma (fneg (f16 HPR:$Sn)), (f16 HPR:$Sm), (f16 HPR:$Sdin)))
 //
 
 let hasSideEffects = 0 in {
-def VMOVDcc  : PseudoInst<(outs DPR:$Dd), (ins DPR:$Dn, DPR:$Dm, cmovpred:$p),
-                    IIC_fpUNA64,
-                    [(set (f64 DPR:$Dd),
-                          (ARMcmov DPR:$Dn, DPR:$Dm, cmovpred:$p))]>,
+def VMOVDcc  : PseudoInst<(outs DPR:$Dd), (ins DPR:$Dn, DPR:$Dm, pred:$p),
+                          IIC_fpUNA64, []>,
                RegConstraint<"$Dn = $Dd">, Requires<[HasFPRegs64]>;
 
-def VMOVScc  : PseudoInst<(outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm, cmovpred:$p),
-                    IIC_fpUNA32,
-                    [(set (f32 SPR:$Sd),
-                          (ARMcmov SPR:$Sn, SPR:$Sm, cmovpred:$p))]>,
+def VMOVScc  : PseudoInst<(outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm, pred:$p),
+                          IIC_fpUNA32, []>,
                RegConstraint<"$Sn = $Sd">, Requires<[HasFPRegs]>;
 
-def VMOVHcc  : PseudoInst<(outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm, cmovpred:$p),
-                    IIC_fpUNA16,
-                    [(set (f16 HPR:$Sd),
-                          (ARMcmov (f16 HPR:$Sn), (f16 HPR:$Sm), cmovpred:$p))]>,
+def VMOVHcc  : PseudoInst<(outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm, pred:$p),
+                          IIC_fpUNA16, []>,
                RegConstraint<"$Sd = $Sn">, Requires<[HasFPRegs]>;
 } // hasSideEffects
 
+// The following patterns have to be defined out-of-line because the number
+// of instruction operands does not match the number of SDNode operands
+// (`pred` counts as one operand).
+
+def : Pat<(ARMcmov f64:$Dn, f64:$Dm, imm:$cc, CPSR),
+          (VMOVDcc $Dn, $Dm, imm:$cc, CPSR)>,
+      Requires<[HasFPRegs64]>;
+
+def : Pat<(ARMcmov f32:$Sn, f32:$Sm, imm:$cc, CPSR),
+          (VMOVScc $Sn, $Sm, imm:$cc, CPSR)>,
+      Requires<[HasFPRegs]>;
+
+def : Pat<(ARMcmov f16:$Sn, f16:$Sm, imm:$cc, CPSR),
+          (VMOVHcc $Sn, $Sm, imm:$cc, CPSR)>,
+      Requires<[HasFPRegs]>; // FIXME: Shouldn't this be HasFPRegs16?
+
 //===----------------------------------------------------------------------===//
 // Move from VFP System Register to ARM core register.
 //
@@ -2510,7 +2522,7 @@ let DecoderMethod = "DecodeForVMRSandVMSR" in {
      Rt = 0b1111 /* apsr_nzcv */ in
  def FMSTAT : MovFromVFP<0b0001 /* fpscr */, (outs), (ins),
                          "vmrs", "\tAPSR_nzcv, fpscr",
-                         [(arm_fmstat FPSCR_NZCV)]>;
+                         [(set CPSR, (arm_fmstat FPSCR_NZCV))]>;
 
  // Application level FPSCR -> GPR
  let hasSideEffects = 1, Uses = [FPSCR], Predicates = [HasFPRegs] in
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index 22e7e6893c1a8..213b104bcfdb2 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -45,6 +45,8 @@ def ProcA78AE   : SubtargetFeature<"cortex-a78ae", "ARMProcFamily", "CortexA78AE
                                    "Cortex-A78AE ARM processors", []>;
 def ProcA78C    : SubtargetFeature<"a78c", "ARMProcFamily", "CortexA78C",
                                    "Cortex-A78C ARM processors", []>;
+def ProcA510    : SubtargetFeature<"cortex-a510", "ARMProcFamily",
+                                   "CortexA510", "Cortex-A510 ARM processors", []>;
 def ProcA710    : SubtargetFeature<"cortex-a710", "ARMProcFamily",
                                    "CortexA710", "Cortex-A710 ARM processors", []>;
 def ProcX1      : SubtargetFeature<"cortex-x1", "ARMProcFamily", "CortexX1",
@@ -518,6 +520,14 @@ def : ProcNoItin<"cortex-a78c",                         [ARMv82a, ProcA78C,
                                                          FeatureDotProd,
                                                          FeatureFullFP16]>;
 
+def : ProcNoItin<"cortex-a510",                         [ARMv9a, ProcA710,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureFP16FML,
+                                                         FeatureBF16,
+                                                         FeatureMatMulInt8,
+                                                         FeatureSB]>;
+
 def : ProcNoItin<"cortex-a710",                         [ARMv9a, ProcA710,
                                                          FeatureHWDivThumb,
                                                          FeatureHWDivARM,
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 61c7fadb20aeb..07207e29bf7d3 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -287,6 +287,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   case CortexA78:
   case CortexA78AE:
   case CortexA78C:
+  case CortexA510:
   case CortexA710:
   case CortexR4:
   case CortexR5:
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index bf62849fba0c3..024a64aceedbd 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -5129,7 +5129,8 @@ ParseStatus ARMAsmParser::parseMemBarrierOptOperand(OperandVector &Operands) {
 
     Opt = ARM_MB::RESERVED_0 + Val;
   } else
-    return ParseStatus::Failure;
+    return Error(Parser.getTok().getLoc(),
+                 "expected an immediate or barrier type");
 
   Operands.push_back(
       ARMOperand::CreateMemBarrierOpt((ARM_MB::MemBOpt)Opt, S, *this));
@@ -5193,7 +5194,8 @@ ARMAsmParser::parseInstSyncBarrierOptOperand(OperandVector &Operands) {
 
     Opt = ARM_ISB::RESERVED_0 + Val;
   } else
-    return ParseStatus::Failure;
+    return Error(Parser.getTok().getLoc(),
+                 "expected an immediate or barrier type");
 
   Operands.push_back(ARMOperand::CreateInstSyncBarrierOpt(
       (ARM_ISB::InstSyncBOpt)Opt, S, *this));
diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index 890c2344c2431..faa0352507fba 100644
--- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -43,6 +43,8 @@
 #include <iterator>
 #include <vector>
 
+#define DEBUG_TYPE "arm-frame-lowering"
+
 using namespace llvm;
 
 Thumb1FrameLowering::Thumb1FrameLowering(const ARMSubtarget &sti)
@@ -277,6 +279,20 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
     }
   }
 
+  // Skip past this code sequence, which is emitted to restore the LR if it is
+  // live-in and clobbered by the frame record setup code:
+  //   ldr rX, [sp, #Y]
+  //   mov lr, rX
+  if (MBBI != MBB.end() && MBBI->getOpcode() == ARM::tLDRspi &&
+      MBBI->getFlag(MachineInstr::FrameSetup)) {
+    ++MBBI;
+    if (MBBI != MBB.end() && MBBI->getOpcode() == ARM::tMOVr &&
+        MBBI->getOperand(0).getReg() == ARM::LR &&
+        MBBI->getFlag(MachineInstr::FrameSetup)) {
+      ++MBBI;
+    }
+  }
+
   // Determine starting offsets of spill areas.
   unsigned DPRCSOffset = NumBytes - ArgRegsSaveSize -
                          (FRSize + GPRCS1Size + GPRCS2Size + DPRCSSize);
@@ -857,7 +873,8 @@ static void pushRegsToStack(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MI,
                             const TargetInstrInfo &TII,
                             const std::set<Register> &RegsToSave,
-                            const std::set<Register> &CopyRegs) {
+                            const std::set<Register> &CopyRegs,
+                            bool &UsedLRAsTemp) {
   MachineFunction &MF = *MBB.getParent();
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL;
@@ -914,6 +931,8 @@ static void pushRegsToStack(MachineBasicBlock &MBB,
         bool isKill = !MRI.isLiveIn(*HiRegToSave);
         if (isKill && !MRI.isReserved(*HiRegToSave))
           MBB.addLiveIn(*HiRegToSave);
+        if (*CopyRegIt == ARM::LR)
+          UsedLRAsTemp = true;
 
         // Emit a MOV from the high reg to the low reg.
         BuildMI(MBB, MI, DL, TII.get(ARM::tMOVr))
@@ -1093,6 +1112,8 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters(
   // In case FP is a high reg, we need a separate push sequence to generate
   // a correct Frame Record
   bool NeedsFrameRecordPush = hasFP(MF) && ARM::hGPRRegClass.contains(FPReg);
+  bool LRLiveIn = MF.getRegInfo().isLiveIn(ARM::LR);
+  bool UsedLRAsTemp = false;
 
   std::set<Register> FrameRecord;
   std::set<Register> SpilledGPRs;
@@ -1104,7 +1125,22 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters(
       SpilledGPRs.insert(Reg);
   }
 
-  pushRegsToStack(MBB, MI, TII, FrameRecord, {ARM::LR});
+  // Determine intermediate registers which can be used for pushing the frame
+  // record:
+  // - Unused argument registers
+  // - LR: This is possible because the first PUSH will save it on the stack,
+  //       so it is free to be used as a temporary for the second. However, it
+  //       is possible for LR to be live-in to the function, in which case we
+  //       will need to restore it later in the prologue, so we only use this
+  //       if there are no free argument registers.
+  std::set<Register> FrameRecordCopyRegs;
+  for (unsigned ArgReg : {ARM::R0, ARM::R1, ARM::R2, ARM::R3})
+    if (!MF.getRegInfo().isLiveIn(ArgReg))
+      FrameRecordCopyRegs.insert(ArgReg);
+  if (FrameRecordCopyRegs.empty())
+    FrameRecordCopyRegs.insert(ARM::LR);
+
+  pushRegsToStack(MBB, MI, TII, FrameRecord, FrameRecordCopyRegs, UsedLRAsTemp);
 
   // Determine intermediate registers which can be used for pushing high regs:
   // - Spilled low regs
@@ -1118,7 +1154,33 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters(
     if (!MF.getRegInfo().isLiveIn(ArgReg))
       CopyRegs.insert(ArgReg);
 
-  pushRegsToStack(MBB, MI, TII, SpilledGPRs, CopyRegs);
+  pushRegsToStack(MBB, MI, TII, SpilledGPRs, CopyRegs, UsedLRAsTemp);
+
+  // If the push sequence used LR as a temporary, and LR is live-in (for
+  // example because it is used by the llvm.returnaddress intrinsic), then we
+  // need to reload it from the stack. Thumb1 does not have a load instruction
+  // which can use LR, so we need to load into a temporary low register and
+  // copy to LR.
+  if (LRLiveIn && UsedLRAsTemp) {
+    auto CopyRegIt = getNextOrderedReg(OrderedCopyRegs.rbegin(),
+                                       OrderedCopyRegs.rend(), CopyRegs);
+    assert(CopyRegIt != OrderedCopyRegs.rend());
+    unsigned NumRegsPushed = FrameRecord.size() + SpilledGPRs.size();
+    LLVM_DEBUG(
+        dbgs() << "LR is live-in but clobbered in prologue, restoring via "
+               << RegInfo->getName(*CopyRegIt) << "\n");
+
+    BuildMI(MBB, MI, DebugLoc(), TII.get(ARM::tLDRspi), *CopyRegIt)
+        .addReg(ARM::SP)
+        .addImm(NumRegsPushed - 1)
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameSetup);
+
+    BuildMI(MBB, MI, DebugLoc(), TII.get(ARM::tMOVr), ARM::LR)
+        .addReg(*CopyRegIt)
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
 
   return true;
 }
diff --git a/llvm/lib/Target/AVR/AVRInstrFormats.td b/llvm/lib/Target/AVR/AVRInstrFormats.td
index 06d14a2fb47fe..407e14a079f3e 100644
--- a/llvm/lib/Target/AVR/AVRInstrFormats.td
+++ b/llvm/lib/Target/AVR/AVRInstrFormats.td
@@ -173,7 +173,6 @@ class FSTDLDD<bit type, dag outs, dag ins, string asmstr, list<dag> pattern>
 //===---------------------------------------------------------------------===//
 class FSTLD<bit type, bits<2> mode, dag outs, dag ins, string asmstr,
             list<dag> pattern> : AVRInst16<outs, ins, asmstr, pattern> {
-  bits<2> ptrreg;
   bits<5> reg;
 
   let Inst{15 - 13} = 0b100;
@@ -187,7 +186,6 @@ class FSTLD<bit type, bits<2> mode, dag outs, dag ins, string asmstr,
 
   let Inst{7 - 4} = reg{3 - 0};
 
-  let Inst{3 - 2} = ptrreg{1 - 0};
   let Inst{1 - 0} = mode{1 - 0};
 
   let DecoderMethod = "decodeLoadStore";
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td
index 3973cd30de1ec..61c17090e3fbe 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.td
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.td
@@ -234,31 +234,6 @@ def imm_port6 : Operand<i8> {
 // Addressing mode pattern reg+imm6
 def addr : ComplexPattern<iPTR, 2, "SelectAddr", [], [SDNPWantRoot]>;
 
-// AsmOperand class for a pointer register.
-// Used with the LD/ST family of instructions.
-// See FSTLD in AVRInstrFormats.td
-def PtrRegAsmOperand : AsmOperandClass { let Name = "Reg"; }
-
-// A special operand type for the LD/ST instructions.
-// It converts the pointer register number into a two-bit field used in the
-// instruction.
-def LDSTPtrReg : Operand<i16> {
-  let MIOperandInfo = (ops PTRREGS);
-  let EncoderMethod = "encodeLDSTPtrReg";
-
-  let ParserMatchClass = PtrRegAsmOperand;
-}
-
-// A special operand type for the LDD/STD instructions.
-// It behaves identically to the LD/ST version, except restricts
-// the pointer registers to Y and Z.
-def LDDSTDPtrReg : Operand<i16> {
-  let MIOperandInfo = (ops PTRDISPREGS);
-  let EncoderMethod = "encodeLDSTPtrReg";
-
-  let ParserMatchClass = PtrRegAsmOperand;
-}
-
 //===----------------------------------------------------------------------===//
 // AVR predicates for subtarget features
 //===----------------------------------------------------------------------===//
@@ -896,7 +871,7 @@ let canFoldAsLoad = 1, isReMaterializable = 1 in {
 
 // Indirect loads.
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
-  def LDRdPtr : FSTLD<0, 0b00, (outs GPR8:$reg), (ins LDSTPtrReg:$ptrreg),
+  def LDRdPtr : FSTLD<0, 0b00, (outs GPR8:$reg), (ins PTRREGS:$ptrreg),
                       "ld\t$reg, $ptrreg",
                       [(set GPR8:$reg, (load i16:$ptrreg))]>,
                 Requires<[HasSRAM]>;
@@ -919,13 +894,8 @@ let canFoldAsLoad = 1, isReMaterializable = 1 in {
 // Indirect loads (with postincrement or predecrement).
 let mayLoad = 1, hasSideEffects = 0,
     Constraints = "$ptrreg = $base_wb,@earlyclobber $reg" in {
-  def LDRdPtrPi : FSTLD<0, 0b01,
-                        (outs GPR8
-                         : $reg, PTRREGS
-                         : $base_wb),
-                        (ins LDSTPtrReg
-                         : $ptrreg),
-                        "ld\t$reg, $ptrreg+", []>,
+  def LDRdPtrPi : FSTLD<0, 0b01, (outs GPR8:$reg, PTRREGS:$base_wb),
+                        (ins PTRREGS:$ptrreg), "ld\t$reg, $ptrreg+", []>,
                   Requires<[HasSRAM]>;
 
   // LDW Rd+1:Rd, P+
@@ -937,7 +907,7 @@ let mayLoad = 1, hasSideEffects = 0,
                    Requires<[HasSRAM]>;
 
   def LDRdPtrPd : FSTLD<0, 0b10, (outs GPR8:$reg, PTRREGS:$base_wb),
-                        (ins LDSTPtrReg:$ptrreg), "ld\t$reg, -$ptrreg", []>,
+                        (ins PTRREGS:$ptrreg), "ld\t$reg, -$ptrreg", []>,
                   Requires<[HasSRAM]>;
 
   // LDW Rd+1:Rd, -P
@@ -1063,7 +1033,7 @@ def STSWKRr : Pseudo<(outs), (ins i16imm:$dst, DREGS:$src),
 // Indirect stores.
 // ST P, Rr
 // Stores the value of Rr into the location addressed by pointer P.
-def STPtrRr : FSTLD<1, 0b00, (outs), (ins LDSTPtrReg:$ptrreg, GPR8:$reg),
+def STPtrRr : FSTLD<1, 0b00, (outs), (ins PTRREGS:$ptrreg, GPR8:$reg),
                     "st\t$ptrreg, $reg", [(store GPR8:$reg, i16:$ptrreg)]>,
               Requires<[HasSRAM]>;
 
@@ -1087,8 +1057,8 @@ let Constraints = "$ptrreg = $base_wb,@earlyclobber $base_wb" in {
   // ST P+, Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Post increments P.
-  def STPtrPiRr : FSTLD<1, 0b01, (outs LDSTPtrReg:$base_wb),
-                        (ins LDSTPtrReg:$ptrreg, GPR8:$reg, i8imm:$offs),
+  def STPtrPiRr : FSTLD<1, 0b01, (outs PTRREGS:$base_wb),
+                        (ins PTRREGS:$ptrreg, GPR8:$reg, i8imm:$offs),
                         "st\t$ptrreg+, $reg",
                         [(set i16:$base_wb, (post_store GPR8:$reg, i16:$ptrreg,
                          imm:$offs))]>,
@@ -1112,8 +1082,8 @@ let Constraints = "$ptrreg = $base_wb,@earlyclobber $base_wb" in {
   // ST -P, Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Pre decrements P.
-  def STPtrPdRr : FSTLD<1, 0b10, (outs LDSTPtrReg:$base_wb),
-                        (ins LDSTPtrReg:$ptrreg, GPR8:$reg, i8imm:$offs),
+  def STPtrPdRr : FSTLD<1, 0b10, (outs PTRREGS:$base_wb),
+                        (ins PTRREGS:$ptrreg, GPR8:$reg, i8imm:$offs),
                         "st\t-$ptrreg, $reg",
                         [(set i16: $base_wb,
                          (pre_store GPR8:$reg, i16:$ptrreg, imm:$offs))]>,
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index 5a0ad8119ddd5..98ddacb31a348 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -72,16 +72,31 @@ AVRMCCodeEmitter::loadStorePostEncoder(const MCInst &MI, unsigned EncodedValue,
 
   unsigned Opcode = MI.getOpcode();
 
-  // check whether either of the registers are the X pointer register.
-  bool IsRegX = MI.getOperand(0).getReg() == AVR::R27R26 ||
-                MI.getOperand(1).getReg() == AVR::R27R26;
+  // Get the index of the pointer register operand.
+  unsigned Idx = 0;
+  if (Opcode == AVR::LDRdPtrPd || Opcode == AVR::LDRdPtrPi ||
+      Opcode == AVR::LDRdPtr)
+    Idx = 1;
 
+  // Check if we need to set the inconsistent bit
   bool IsPredec = Opcode == AVR::LDRdPtrPd || Opcode == AVR::STPtrPdRr;
   bool IsPostinc = Opcode == AVR::LDRdPtrPi || Opcode == AVR::STPtrPiRr;
-
-  // Check if we need to set the inconsistent bit
-  if (IsRegX || IsPredec || IsPostinc) {
+  if (MI.getOperand(Idx).getReg() == AVR::R27R26 || IsPredec || IsPostinc)
     EncodedValue |= (1 << 12);
+
+  // Encode the pointer register.
+  switch (MI.getOperand(Idx).getReg()) {
+  case AVR::R27R26:
+    EncodedValue |= 0xc;
+    break;
+  case AVR::R29R28:
+    EncodedValue |= 0x8;
+    break;
+  case AVR::R31R30:
+    break;
+  default:
+    llvm_unreachable("invalid pointer register");
+    break;
   }
 
   return EncodedValue;
@@ -109,26 +124,6 @@ AVRMCCodeEmitter::encodeRelCondBrTarget(const MCInst &MI, unsigned OpNo,
   return target;
 }
 
-unsigned AVRMCCodeEmitter::encodeLDSTPtrReg(const MCInst &MI, unsigned OpNo,
-                                            SmallVectorImpl<MCFixup> &Fixups,
-                                            const MCSubtargetInfo &STI) const {
-  auto MO = MI.getOperand(OpNo);
-
-  // The operand should be a pointer register.
-  assert(MO.isReg());
-
-  switch (MO.getReg().id()) {
-  case AVR::R27R26:
-    return 0x03; // X: 0b11
-  case AVR::R29R28:
-    return 0x02; // Y: 0b10
-  case AVR::R31R30:
-    return 0x00; // Z: 0b00
-  default:
-    llvm_unreachable("invalid pointer register");
-  }
-}
-
 /// Encodes a `memri` operand.
 /// The operand is 7-bits.
 /// * The lower 6 bits is the immediate
@@ -288,8 +283,7 @@ void AVRMCCodeEmitter::encodeInstruction(const MCInst &MI,
   }
 }
 
-MCCodeEmitter *createAVRMCCodeEmitter(const MCInstrInfo &MCII,
-                                      MCContext &Ctx) {
+MCCodeEmitter *createAVRMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx) {
   return new AVRMCCodeEmitter(MCII, Ctx);
 }
 
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
index a00bbb9ae498b..5b301dfb03ee9 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
@@ -52,11 +52,6 @@ class AVRMCCodeEmitter : public MCCodeEmitter {
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const;
 
-  /// Encodes the `PTRREGS` operand to a load or store instruction.
-  unsigned encodeLDSTPtrReg(const MCInst &MI, unsigned OpNo,
-                            SmallVectorImpl<MCFixup> &Fixups,
-                            const MCSubtargetInfo &STI) const;
-
   /// Encodes a `register+immediate` operand for `LDD`/`STD`.
   unsigned encodeMemri(const MCInst &MI, unsigned OpNo,
                        SmallVectorImpl<MCFixup> &Fixups,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 7cc08b2fe7cc4..cff6cdce813de 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -294,6 +294,60 @@ class Attributes<Version ver = DXIL1_0, list<DXILAttribute> attrs> {
   list<DXILAttribute> op_attrs = attrs;
 }
 
+defvar BarrierMode_DeviceMemoryBarrier              = 2;
+defvar BarrierMode_DeviceMemoryBarrierWithGroupSync = 3;
+defvar BarrierMode_GroupMemoryBarrier               = 8;
+defvar BarrierMode_GroupMemoryBarrierWithGroupSync  = 9;
+defvar BarrierMode_AllMemoryBarrier                 = 10;
+defvar BarrierMode_AllMemoryBarrierWithGroupSync    = 11;
+
+// Intrinsic arg selection
+class IntrinArgSelectType;
+def IntrinArgSelect_Index : IntrinArgSelectType;
+def IntrinArgSelect_I8    : IntrinArgSelectType;
+def IntrinArgSelect_I32   : IntrinArgSelectType;
+
+class IntrinArgSelect<IntrinArgSelectType type_, int value_> {
+  IntrinArgSelectType type = type_;
+  int value = value_;
+}
+class IntrinArgIndex<int index> : IntrinArgSelect<IntrinArgSelect_Index, index>;
+class IntrinArgI8   <int value> : IntrinArgSelect<IntrinArgSelect_I8,    value>;
+class IntrinArgI32  <int value> : IntrinArgSelect<IntrinArgSelect_I32,   value>;
+
+// Select which intrinsic to lower from for a DXILOp.
+// If the intrinsic is the only argument given to IntrinSelect, then the
+// arguments of the intrinsic will be copied in the same order. Example:
+//   let intrinsics = [
+//     IntrinSelect<int_dx_my_intrinsic>,
+//     IntrinSelect<int_dx_my_intrinsic2>,
+//   ]
+//=========================================================================================
+// Using IntrinArgIndex<>, arguments of the intrinsic can be copied to the DXIL
+// OP in specific order:
+//   let intrinsics = [
+//     IntrinSelect<int_dx_my_intrinsic,
+//       [IntrinArgIndex<2>, IntrinArgIndex<1>, IntrinArgIndex<0>> ]
+//     >,
+//   ]
+//=========================================================================================
+// Using IntrinArgI8<> and IntrinArgI32<>, integer constants can be added
+// directly to the dxil op. This can be used in conjunction with
+// IntrinArgIndex:
+//   let intrinsics = [
+//     IntrinSelect<int_dx_wave_active_usum,
+//       [ IntrinArgIndex<0>, IntrinArgI8<0>, IntrinArgI8<1> ]
+//     >,
+//     IntrinSelect<int_dx_wave_active_sum,
+//       [ IntrinArgIndex<0>, IntrinArgI8<0>, IntrinArgI8<0> ]
+//     >,
+//   ]
+//
+class IntrinSelect<Intrinsic intrinsic_, list<IntrinArgSelect> arg_selects_=[]> {
+  Intrinsic intrinsic = intrinsic_;
+  list<IntrinArgSelect> arg_selects = arg_selects_;
+}
+
 // Abstraction DXIL Operation
 class DXILOp<int opcode, DXILOpClass opclass> {
   // A short description of the operation
@@ -305,8 +359,8 @@ class DXILOp<int opcode, DXILOpClass opclass> {
   // Class of DXIL Operation.
   DXILOpClass OpClass = opclass;
 
-  // LLVM Intrinsic DXIL Operation maps to
-  Intrinsic LLVMIntrinsic = ?;
+  // LLVM Intrinsics DXIL Operation maps from
+  list<IntrinSelect> intrinsics = [];
 
   // Result type of the op
   DXILOpParamType result;
@@ -328,7 +382,7 @@ class DXILOp<int opcode, DXILOpClass opclass> {
 
 def Abs :  DXILOp<6, unary> {
   let Doc = "Returns the absolute value of the input.";
-  let LLVMIntrinsic = int_fabs;
+  let intrinsics = [ IntrinSelect<int_fabs> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
@@ -338,7 +392,7 @@ def Abs :  DXILOp<6, unary> {
 
 def Saturate :  DXILOp<7, unary> {
   let Doc = "Clamps a single or double precision floating point value to [0.0f...1.0f].";
-  let LLVMIntrinsic = int_dx_saturate;
+  let intrinsics = [ IntrinSelect<int_dx_saturate> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
@@ -348,7 +402,7 @@ def Saturate :  DXILOp<7, unary> {
 
 def IsInf :  DXILOp<9, isSpecialFloat> {
   let Doc = "Determines if the specified value is infinite.";
-  let LLVMIntrinsic = int_dx_isinf;
+  let intrinsics = [ IntrinSelect<int_dx_isinf> ];
   let arguments = [OverloadTy];
   let result = Int1Ty;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -358,7 +412,7 @@ def IsInf :  DXILOp<9, isSpecialFloat> {
 
 def Cos :  DXILOp<12, unary> {
   let Doc = "Returns cosine(theta) for theta in radians.";
-  let LLVMIntrinsic = int_cos;
+  let intrinsics = [ IntrinSelect<int_cos> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -368,7 +422,7 @@ def Cos :  DXILOp<12, unary> {
 
 def Sin :  DXILOp<13, unary> {
   let Doc = "Returns sine(theta) for theta in radians.";
-  let LLVMIntrinsic = int_sin;
+  let intrinsics = [ IntrinSelect<int_sin> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -378,7 +432,7 @@ def Sin :  DXILOp<13, unary> {
 
 def Tan :  DXILOp<14, unary> {
   let Doc = "Returns tangent(theta) for theta in radians.";
-  let LLVMIntrinsic = int_tan;
+  let intrinsics = [ IntrinSelect<int_tan> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -388,7 +442,7 @@ def Tan :  DXILOp<14, unary> {
 
 def ACos :  DXILOp<15, unary> {
   let Doc = "Returns the arccosine of the specified value.";
-  let LLVMIntrinsic = int_acos;
+  let intrinsics = [ IntrinSelect<int_acos> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -398,7 +452,7 @@ def ACos :  DXILOp<15, unary> {
 
 def ASin :  DXILOp<16, unary> {
   let Doc = "Returns the arcsine of the specified value.";
-  let LLVMIntrinsic = int_asin;
+  let intrinsics = [ IntrinSelect<int_asin> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -408,7 +462,7 @@ def ASin :  DXILOp<16, unary> {
 
 def ATan :  DXILOp<17, unary> {
   let Doc = "Returns the arctangent of the specified value.";
-  let LLVMIntrinsic = int_atan;
+  let intrinsics = [ IntrinSelect<int_atan> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -418,7 +472,7 @@ def ATan :  DXILOp<17, unary> {
 
 def HCos :  DXILOp<18, unary> {
   let Doc = "Returns the hyperbolic cosine of the specified value.";
-  let LLVMIntrinsic = int_cosh;
+  let intrinsics = [ IntrinSelect<int_cosh> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -428,7 +482,7 @@ def HCos :  DXILOp<18, unary> {
 
 def HSin :  DXILOp<19, unary> {
   let Doc = "Returns the hyperbolic sine of the specified value.";
-  let LLVMIntrinsic = int_sinh;
+  let intrinsics = [ IntrinSelect<int_sinh> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -438,7 +492,7 @@ def HSin :  DXILOp<19, unary> {
 
 def HTan :  DXILOp<20, unary> {
   let Doc = "Returns the hyperbolic tan of the specified value.";
-  let LLVMIntrinsic = int_tanh;
+  let intrinsics = [ IntrinSelect<int_tanh> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -449,7 +503,7 @@ def HTan :  DXILOp<20, unary> {
 def Exp2 :  DXILOp<21, unary> {
   let Doc = "Returns the base 2 exponential, or 2**x, of the specified value. "
             "exp2(x) = 2**x.";
-  let LLVMIntrinsic = int_exp2;
+  let intrinsics = [ IntrinSelect<int_exp2> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -460,7 +514,7 @@ def Exp2 :  DXILOp<21, unary> {
 def Frac :  DXILOp<22, unary> {
   let Doc = "Returns a fraction from 0 to 1 that represents the decimal part "
             "of the input.";
-  let LLVMIntrinsic = int_dx_frac;
+  let intrinsics = [ IntrinSelect<int_dx_frac> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -470,7 +524,7 @@ def Frac :  DXILOp<22, unary> {
 
 def Log2 :  DXILOp<23, unary> {
   let Doc = "Returns the base-2 logarithm of the specified value.";
-  let LLVMIntrinsic = int_log2;
+  let intrinsics = [ IntrinSelect<int_log2> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -481,7 +535,7 @@ def Log2 :  DXILOp<23, unary> {
 def Sqrt :  DXILOp<24, unary> {
   let Doc = "Returns the square root of the specified floating-point value, "
             "per component.";
-  let LLVMIntrinsic = int_sqrt;
+  let intrinsics = [ IntrinSelect<int_sqrt> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -492,7 +546,7 @@ def Sqrt :  DXILOp<24, unary> {
 def RSqrt :  DXILOp<25, unary> {
   let Doc = "Returns the reciprocal of the square root of the specified value. "
             "rsqrt(x) = 1 / sqrt(x).";
-  let LLVMIntrinsic = int_dx_rsqrt;
+  let intrinsics = [ IntrinSelect<int_dx_rsqrt> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -503,7 +557,7 @@ def RSqrt :  DXILOp<25, unary> {
 def Round :  DXILOp<26, unary> {
   let Doc = "Returns the input rounded to the nearest integer within a "
             "floating-point type.";
-  let LLVMIntrinsic = int_roundeven;
+  let intrinsics = [ IntrinSelect<int_roundeven> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -514,7 +568,7 @@ def Round :  DXILOp<26, unary> {
 def Floor :  DXILOp<27, unary> {
   let Doc =
       "Returns the largest integer that is less than or equal to the input.";
-  let LLVMIntrinsic = int_floor;
+  let intrinsics = [ IntrinSelect<int_floor> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -525,7 +579,7 @@ def Floor :  DXILOp<27, unary> {
 def Ceil :  DXILOp<28, unary> {
   let Doc = "Returns the smallest integer that is greater than or equal to the "
             "input.";
-  let LLVMIntrinsic = int_ceil;
+  let intrinsics = [ IntrinSelect<int_ceil> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -535,7 +589,7 @@ def Ceil :  DXILOp<28, unary> {
 
 def Trunc :  DXILOp<29, unary> {
   let Doc = "Returns the specified value truncated to the integer component.";
-  let LLVMIntrinsic = int_trunc;
+  let intrinsics = [ IntrinSelect<int_trunc> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -545,7 +599,7 @@ def Trunc :  DXILOp<29, unary> {
 
 def Rbits :  DXILOp<30, unary> {
   let Doc = "Returns the specified value with its bits reversed.";
-  let LLVMIntrinsic = int_bitreverse;
+  let intrinsics = [ IntrinSelect<int_bitreverse> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -567,7 +621,7 @@ def CountBits :  DXILOp<31, unaryBits> {
 def FirstbitHi :  DXILOp<33, unaryBits> {
   let Doc = "Returns the location of the first set bit starting from "
             "the highest order bit and working downward.";
-  let LLVMIntrinsic = int_dx_firstbituhigh;
+  let intrinsics = [ IntrinSelect<int_dx_firstbituhigh> ];
   let arguments = [OverloadTy];
   let result = Int32Ty;
   let overloads =
@@ -579,7 +633,7 @@ def FirstbitHi :  DXILOp<33, unaryBits> {
 def FirstbitSHi :  DXILOp<34, unaryBits> {
   let Doc = "Returns the location of the first set bit from "
             "the highest order bit based on the sign.";
-  let LLVMIntrinsic = int_dx_firstbitshigh;
+  let intrinsics = [ IntrinSelect<int_dx_firstbitshigh> ];
   let arguments = [OverloadTy];
   let result = Int32Ty;
   let overloads =
@@ -590,7 +644,7 @@ def FirstbitSHi :  DXILOp<34, unaryBits> {
 
 def FMax :  DXILOp<35, binary> {
   let Doc = "Float maximum. FMax(a,b) = a > b ? a : b";
-  let LLVMIntrinsic = int_maxnum;
+  let intrinsics = [ IntrinSelect<int_maxnum> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -601,7 +655,7 @@ def FMax :  DXILOp<35, binary> {
 
 def FMin :  DXILOp<36, binary> {
   let Doc = "Float minimum. FMin(a,b) = a < b ? a : b";
-  let LLVMIntrinsic = int_minnum;
+  let intrinsics = [ IntrinSelect<int_minnum> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -612,7 +666,7 @@ def FMin :  DXILOp<36, binary> {
 
 def SMax :  DXILOp<37, binary> {
   let Doc = "Signed integer maximum. SMax(a,b) = a > b ? a : b";
-  let LLVMIntrinsic = int_smax;
+  let intrinsics = [ IntrinSelect<int_smax> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -623,7 +677,7 @@ def SMax :  DXILOp<37, binary> {
 
 def SMin :  DXILOp<38, binary> {
   let Doc = "Signed integer minimum. SMin(a,b) = a < b ? a : b";
-  let LLVMIntrinsic = int_smin;
+  let intrinsics = [ IntrinSelect<int_smin> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -634,7 +688,7 @@ def SMin :  DXILOp<38, binary> {
 
 def UMax :  DXILOp<39, binary> {
   let Doc = "Unsigned integer maximum. UMax(a,b) = a > b ? a : b";
-  let LLVMIntrinsic = int_umax;
+  let intrinsics = [ IntrinSelect<int_umax> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -645,7 +699,7 @@ def UMax :  DXILOp<39, binary> {
 
 def UMin :  DXILOp<40, binary> {
   let Doc = "Unsigned integer minimum. UMin(a,b) = a < b ? a : b";
-  let LLVMIntrinsic = int_umin;
+  let intrinsics = [ IntrinSelect<int_umin> ];
   let arguments = [OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -657,7 +711,7 @@ def UMin :  DXILOp<40, binary> {
 def FMad :  DXILOp<46, tertiary> {
   let Doc = "Floating point arithmetic multiply/add operation. fmad(m,a,b) = m "
             "* a + b.";
-  let LLVMIntrinsic = int_fmuladd;
+  let intrinsics = [ IntrinSelect<int_fmuladd> ];
   let arguments = [OverloadTy, OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -669,7 +723,7 @@ def FMad :  DXILOp<46, tertiary> {
 def IMad :  DXILOp<48, tertiary> {
   let Doc = "Signed integer arithmetic multiply/add operation. imad(m,a,b) = m "
             "* a + b.";
-  let LLVMIntrinsic = int_dx_imad;
+  let intrinsics = [ IntrinSelect<int_dx_imad> ];
   let arguments = [OverloadTy, OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -681,7 +735,7 @@ def IMad :  DXILOp<48, tertiary> {
 def UMad :  DXILOp<49, tertiary> {
   let Doc = "Unsigned integer arithmetic multiply/add operation. umad(m,a, = m "
             "* a + b.";
-  let LLVMIntrinsic = int_dx_umad;
+  let intrinsics = [ IntrinSelect<int_dx_umad> ];
   let arguments = [OverloadTy, OverloadTy, OverloadTy];
   let result = OverloadTy;
   let overloads =
@@ -693,7 +747,7 @@ def UMad :  DXILOp<49, tertiary> {
 def Dot2 :  DXILOp<54, dot2> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is 0 to 1 inclusive";
-  let LLVMIntrinsic = int_dx_dot2;
+  let intrinsics = [ IntrinSelect<int_dx_dot2> ];
   let arguments = !listsplat(OverloadTy, 4);
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -704,7 +758,7 @@ def Dot2 :  DXILOp<54, dot2> {
 def Dot3 :  DXILOp<55, dot3> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is 0 to 2 inclusive";
-  let LLVMIntrinsic = int_dx_dot3;
+  let intrinsics = [ IntrinSelect<int_dx_dot3> ];
   let arguments = !listsplat(OverloadTy, 6);
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -715,7 +769,7 @@ def Dot3 :  DXILOp<55, dot3> {
 def Dot4 :  DXILOp<56, dot4> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is 0 to 3 inclusive";
-  let LLVMIntrinsic = int_dx_dot4;
+  let intrinsics = [ IntrinSelect<int_dx_dot4> ];
   let arguments = !listsplat(OverloadTy, 8);
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
@@ -772,7 +826,7 @@ def CheckAccessFullyMapped : DXILOp<71, checkAccessFullyMapped> {
 
 def Discard : DXILOp<82, discard> {
   let Doc = "discard the current pixel";
-  let LLVMIntrinsic = int_dx_discard;
+  let intrinsics = [ IntrinSelect<int_dx_discard> ];
   let arguments = [Int1Ty];
   let result = VoidTy;
   let stages = [Stages<DXIL1_0, [pixel]>];
@@ -780,7 +834,7 @@ def Discard : DXILOp<82, discard> {
 
 def ThreadId :  DXILOp<93, threadId> {
   let Doc = "Reads the thread ID";
-  let LLVMIntrinsic = int_dx_thread_id;
+  let intrinsics = [ IntrinSelect<int_dx_thread_id> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
@@ -790,7 +844,7 @@ def ThreadId :  DXILOp<93, threadId> {
 
 def GroupId :  DXILOp<94, groupId> {
   let Doc = "Reads the group ID (SV_GroupID)";
-  let LLVMIntrinsic = int_dx_group_id;
+  let intrinsics = [ IntrinSelect<int_dx_group_id> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
@@ -800,7 +854,7 @@ def GroupId :  DXILOp<94, groupId> {
 
 def ThreadIdInGroup :  DXILOp<95, threadIdInGroup> {
   let Doc = "Reads the thread ID within the group  (SV_GroupThreadID)";
-  let LLVMIntrinsic = int_dx_thread_id_in_group;
+  let intrinsics = [ IntrinSelect<int_dx_thread_id_in_group> ];
   let arguments = [OverloadTy];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
@@ -811,7 +865,7 @@ def ThreadIdInGroup :  DXILOp<95, threadIdInGroup> {
 def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
   let Doc = "Provides a flattened index for a given thread within a given "
             "group (SV_GroupIndex)";
-  let LLVMIntrinsic = int_dx_flattened_thread_id_in_group;
+  let intrinsics = [ IntrinSelect<int_dx_flattened_thread_id_in_group> ];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
@@ -820,7 +874,7 @@ def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
 
 def MakeDouble :  DXILOp<101, makeDouble> {
   let Doc = "creates a double value";
-  let LLVMIntrinsic = int_dx_asdouble;
+  let intrinsics = [ IntrinSelect<int_dx_asdouble> ];
   let arguments = [Int32Ty, Int32Ty];
   let result = DoubleTy;
   let stages = [Stages<DXIL1_0, [all_stages]>];
@@ -839,7 +893,7 @@ def SplitDouble :  DXILOp<102, splitDouble> {
 def Dot4AddI8Packed : DXILOp<163, dot4AddPacked> {
   let Doc = "signed dot product of 4 x i8 vectors packed into i32, with "
             "accumulate to i32";
-  let LLVMIntrinsic = int_dx_dot4add_i8packed;
+  let intrinsics = [ IntrinSelect<int_dx_dot4add_i8packed> ];
   let arguments = [Int32Ty, Int32Ty, Int32Ty];
   let result = Int32Ty;
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
@@ -849,7 +903,7 @@ def Dot4AddI8Packed : DXILOp<163, dot4AddPacked> {
 def Dot4AddU8Packed : DXILOp<164, dot4AddPacked> {
   let Doc = "unsigned dot product of 4 x i8 vectors packed into i32, with "
             "accumulate to i32";
-  let LLVMIntrinsic = int_dx_dot4add_u8packed;
+  let intrinsics = [ IntrinSelect<int_dx_dot4add_u8packed> ];
   let arguments = [Int32Ty, Int32Ty, Int32Ty];
   let result = Int32Ty;
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
@@ -872,7 +926,7 @@ def CreateHandleFromBinding : DXILOp<217, createHandleFromBinding> {
 
 def WaveActiveAnyTrue : DXILOp<113, waveAnyTrue> {
   let Doc = "returns true if the expression is true in any of the active lanes in the current wave";
-  let LLVMIntrinsic = int_dx_wave_any;
+  let intrinsics = [ IntrinSelect<int_dx_wave_any> ];
   let arguments = [Int1Ty];
   let result = Int1Ty;
   let stages = [Stages<DXIL1_0, [all_stages]>];
@@ -880,7 +934,7 @@ def WaveActiveAnyTrue : DXILOp<113, waveAnyTrue> {
 
 def WaveIsFirstLane :  DXILOp<110, waveIsFirstLane> {
   let Doc = "returns 1 for the first lane in the wave";
-  let LLVMIntrinsic = int_dx_wave_is_first_lane;
+  let intrinsics = [ IntrinSelect<int_dx_wave_is_first_lane> ];
   let arguments = [];
   let result = Int1Ty;
   let stages = [Stages<DXIL1_0, [all_stages]>];
@@ -889,7 +943,7 @@ def WaveIsFirstLane :  DXILOp<110, waveIsFirstLane> {
 
 def WaveReadLaneAt:  DXILOp<117, waveReadLaneAt> {
   let Doc = "returns the value from the specified lane";
-  let LLVMIntrinsic = int_dx_wave_readlane;
+  let intrinsics = [ IntrinSelect<int_dx_wave_readlane> ];
   let arguments = [OverloadTy, Int32Ty];
   let result = OverloadTy;
   let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy, Int1Ty, Int16Ty, Int32Ty, Int64Ty]>];
@@ -899,7 +953,7 @@ def WaveReadLaneAt:  DXILOp<117, waveReadLaneAt> {
 
 def WaveGetLaneIndex : DXILOp<111, waveGetLaneIndex> {
   let Doc = "returns the index of the current lane in the wave";
-  let LLVMIntrinsic = int_dx_wave_getlaneindex;
+  let intrinsics = [ IntrinSelect<int_dx_wave_getlaneindex> ];
   let arguments = [];
   let result = Int32Ty;
   let stages = [Stages<DXIL1_0, [all_stages]>];
@@ -908,9 +962,23 @@ def WaveGetLaneIndex : DXILOp<111, waveGetLaneIndex> {
 
 def WaveAllBitCount : DXILOp<135, waveAllOp> {
   let Doc = "returns the count of bits set to 1 across the wave";
-  let LLVMIntrinsic = int_dx_wave_active_countbits;
+  let intrinsics = [ IntrinSelect<int_dx_wave_active_countbits> ];
   let arguments = [Int1Ty];
   let result = Int32Ty;
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
+
+def Barrier : DXILOp<80, barrier> {
+  let Doc = "inserts a memory barrier in the shader";
+  let intrinsics = [
+    IntrinSelect<
+        int_dx_group_memory_barrier_with_group_sync,
+        [ IntrinArgI32<BarrierMode_GroupMemoryBarrierWithGroupSync> ]>,
+  ];
+
+  let arguments = [Int32Ty];
+  let result = VoidTy;
+  let stages = [Stages<DXIL1_0, [compute, library]>];
+  let attributes = [Attributes<DXIL1_0, []>];
+}
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
index aa1f55c572df2..79ebbe0925e5c 100644
--- a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
@@ -23,6 +23,8 @@ static bool finalizeLinkage(Module &M) {
 
   // Collect non-entry and non-exported functions to set to internal linkage.
   for (Function &EF : M.functions()) {
+    if (EF.isIntrinsic())
+      continue;
     if (EF.hasFnAttribute("hlsl.shader") || EF.hasFnAttribute("hlsl.export"))
       continue;
     Funcs.insert(&EF);
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index b5cc209493ed1..d9e70da6ed653 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -106,17 +106,43 @@ class OpLowerer {
     return false;
   }
 
-  [[nodiscard]]
-  bool replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp) {
+  struct IntrinArgSelect {
+    enum class Type {
+#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,
+#include "DXILOperation.inc"
+    };
+    Type Type;
+    int Value;
+  };
+
+  [[nodiscard]] bool
+  replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
+                        ArrayRef<IntrinArgSelect> ArgSelects) {
     bool IsVectorArgExpansion = isVectorArgExpansion(F);
+    assert(!(IsVectorArgExpansion && ArgSelects.size()) &&
+           "Cann't do vector arg expansion when using arg selects.");
     return replaceFunction(F, [&](CallInst *CI) -> Error {
-      SmallVector<Value *> Args;
       OpBuilder.getIRB().SetInsertPoint(CI);
-      if (IsVectorArgExpansion) {
-        SmallVector<Value *> NewArgs = argVectorFlatten(CI, OpBuilder.getIRB());
-        Args.append(NewArgs.begin(), NewArgs.end());
-      } else
+      SmallVector<Value *> Args;
+      if (ArgSelects.size()) {
+        for (const IntrinArgSelect &A : ArgSelects) {
+          switch (A.Type) {
+          case IntrinArgSelect::Type::Index:
+            Args.push_back(CI->getArgOperand(A.Value));
+            break;
+          case IntrinArgSelect::Type::I8:
+            Args.push_back(OpBuilder.getIRB().getInt8((uint8_t)A.Value));
+            break;
+          case IntrinArgSelect::Type::I32:
+            Args.push_back(OpBuilder.getIRB().getInt32(A.Value));
+            break;
+          }
+        }
+      } else if (IsVectorArgExpansion) {
+        Args = argVectorFlatten(CI, OpBuilder.getIRB());
+      } else {
         Args.append(CI->arg_begin(), CI->arg_end());
+      }
 
       Expected<CallInst *> OpCall =
           OpBuilder.tryCreateOp(DXILOp, Args, CI->getName(), F.getReturnType());
@@ -542,23 +568,47 @@ class OpLowerer {
         return make_error<StringError>(
             "typedBufferStore data must be a vector of 4 elements",
             inconvertibleErrorCode());
-      Value *Data0 =
-          IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, 0));
-      Value *Data1 =
-          IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, 1));
-      Value *Data2 =
-          IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, 2));
-      Value *Data3 =
-          IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, 3));
-
-      std::array<Value *, 8> Args{Handle, Index0, Index1, Data0,
-                                  Data1,  Data2,  Data3,  Mask};
+
+      // Since we're post-scalarizer, we likely have a vector that's constructed
+      // solely for the argument of the store. If so, just use the scalar values
+      // from before they're inserted into the temporary.
+      std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};
+      auto *IEI = dyn_cast<InsertElementInst>(Data);
+      while (IEI) {
+        auto *IndexOp = dyn_cast<ConstantInt>(IEI->getOperand(2));
+        if (!IndexOp)
+          break;
+        size_t IndexVal = IndexOp->getZExtValue();
+        assert(IndexVal < 4 && "Too many elements for buffer store");
+        DataElements[IndexVal] = IEI->getOperand(1);
+        IEI = dyn_cast<InsertElementInst>(IEI->getOperand(0));
+      }
+
+      // If for some reason we weren't able to forward the arguments from the
+      // scalarizer artifact, then we need to actually extract elements from the
+      // vector.
+      for (int I = 0, E = 4; I != E; ++I)
+        if (DataElements[I] == nullptr)
+          DataElements[I] =
+              IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, I));
+
+      std::array<Value *, 8> Args{
+          Handle,          Index0,          Index1,          DataElements[0],
+          DataElements[1], DataElements[2], DataElements[3], Mask};
       Expected<CallInst *> OpCall =
           OpBuilder.tryCreateOp(OpCode::BufferStore, Args, CI->getName());
       if (Error E = OpCall.takeError())
         return E;
 
       CI->eraseFromParent();
+      // Clean up any leftover `insertelement`s
+      IEI = dyn_cast<InsertElementInst>(Data);
+      while (IEI && IEI->use_empty()) {
+        InsertElementInst *Tmp = IEI;
+        IEI = dyn_cast<InsertElementInst>(IEI->getOperand(0));
+        Tmp->eraseFromParent();
+      }
+
       return Error::success();
     });
   }
@@ -641,9 +691,10 @@ class OpLowerer {
       switch (ID) {
       default:
         continue;
-#define DXIL_OP_INTRINSIC(OpCode, Intrin)                                      \
+#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...)                                 \
   case Intrin:                                                                 \
-    HasErrors |= replaceFunctionWithOp(F, OpCode);                             \
+    HasErrors |= replaceFunctionWithOp(                                        \
+        F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__});                    \
     break;
 #include "DXILOperation.inc"
       case Intrinsic::dx_handle_fromBinding:
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 655427a3e8020..de14c8d9f13e8 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -89,6 +89,7 @@ class DirectXPassConfig : public TargetPassConfig {
 
   FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
   void addCodeGenPrepare() override {
+    addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createDXILIntrinsicExpansionLegacyPass());
     addPass(createDXILDataScalarizationLegacyPass());
     ScalarizerPassOptions DxilScalarOptions;
@@ -96,7 +97,6 @@ class DirectXPassConfig : public TargetPassConfig {
     addPass(createDXILFlattenArraysLegacyPass());
     addPass(createScalarizerPass(DxilScalarOptions));
     addPass(createDXILOpLoweringLegacyPass());
-    addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createDXILTranslateMetadataLegacyPass());
     addPass(createDXILPrepareModulePass());
   }
diff --git a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
index 0351217f441df..7935c4b86af53 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
@@ -40411,6 +40411,7 @@ def Y2_break : HInst<
 tc_55255f2b, TypeCR>, Enc_e3b0c4 {
 let Inst{13-0} = 0b00000000000000;
 let Inst{31-16} = 0b0110110000100000;
+let hasSideEffects = 1;
 let isSolo = 1;
 }
 def Y2_ciad : HInst<
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 2c8d141aa2108..9fbf4cb684a52 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1498,7 +1498,7 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   // All operations default to "legal", except:
   // - indexed loads and stores (pre-/post-incremented),
   // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
-  //   ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
+  //   ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
   //   FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
   //   FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
   // which default to "expand" for at least one type.
@@ -1507,6 +1507,7 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::ConstantFP,           MVT::f32,   Legal);
   setOperationAction(ISD::ConstantFP,           MVT::f64,   Legal);
   setOperationAction(ISD::TRAP,                 MVT::Other, Legal);
+  setOperationAction(ISD::DEBUGTRAP,            MVT::Other, Legal);
   setOperationAction(ISD::ConstantPool,         MVT::i32,   Custom);
   setOperationAction(ISD::JumpTable,            MVT::i32,   Custom);
   setOperationAction(ISD::BUILD_PAIR,           MVT::i64,   Expand);
diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td
index 52fa73f3798c3..9bd45c72b7d4d 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatterns.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td
@@ -2675,7 +2675,7 @@ def s30_2ProperPred  : PatLeaf<(i32 imm), [{
 }]>;
 def RoundTo8 : SDNodeXForm<imm, [{
   int32_t Imm = N->getSExtValue();
-  return CurDAG->getTargetConstant(Imm & -8, SDLoc(N), MVT::i32);
+  return CurDAG->getSignedTargetConstant(Imm & -8, SDLoc(N), MVT::i32);
 }]>;
 
 let AddedComplexity = 150 in
@@ -3384,6 +3384,7 @@ def HexagonBARRIER: SDNode<"HexagonISD::BARRIER", SDTNone, [SDNPHasChain]>;
 def: Pat<(HexagonBARRIER), (Y2_barrier)>;
 
 def: Pat<(trap), (PS_crash)>;
+def: Pat<(debugtrap), (Y2_break)>;
 
 // Read cycle counter.
 def SDTInt64Leaf: SDTypeProfile<1, 0, [SDTCisVT<0, i64>]>;
diff --git a/llvm/lib/Target/LoongArch/LoongArch.td b/llvm/lib/Target/LoongArch/LoongArch.td
index 463786e72bdf8..596c8c90c0a1f 100644
--- a/llvm/lib/Target/LoongArch/LoongArch.td
+++ b/llvm/lib/Target/LoongArch/LoongArch.td
@@ -118,6 +118,12 @@ def FeatureLAM_BH
                         "Support amswap[_db].{b/h} and amadd[_db].{b/h} instructions.">;
 def HasLAM_BH : Predicate<"Subtarget->hasLAM_BH()">;
 
+// Atomic memory compare and swap instructions for byte, half word, word and double word
+def FeatureLAMCAS
+    : SubtargetFeature<"lamcas", "HasLAMCAS", "true",
+                        "Support amcas[_db].{b/h/w/d}.">;
+def HasLAMCAS : Predicate<"Subtarget->hasLAMCAS()">;
+
 def FeatureLD_SEQ_SA
     : SubtargetFeature<"ld-seq-sa", "HasLD_SEQ_SA", "true",
                         "Don't use load-load barrier (dbar 0x700).">;
@@ -172,6 +178,8 @@ def : ProcessorModel<"la664", NoSchedModel, [Feature64Bit,
                                              FeatureExtLBT,
                                              FeatureFrecipe,
                                              FeatureLAM_BH,
+                                             FeatureLAMCAS,
+                                             FeatureLD_SEQ_SA,
                                              FeatureDiv32]>;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp
index f0e6837d49a97..069b181791ac7 100644
--- a/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp
@@ -37,8 +37,8 @@ class LoongArchDeadRegisterDefinitions : public MachineFunctionPass {
     AU.addPreserved<LiveIntervalsWrapperPass>();
     AU.addRequired<LiveIntervalsWrapperPass>();
     AU.addPreserved<SlotIndexesWrapperPass>();
-    AU.addPreserved<LiveDebugVariables>();
-    AU.addPreserved<LiveStacks>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+    AU.addPreserved<LiveStacksWrapperLegacy>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index c84619622270d..104e601de044b 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -270,6 +270,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
           {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
           Expand);
     }
+    for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
+      setOperationAction(ISD::BITREVERSE, VT, Custom);
     for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
       setOperationAction(ISD::BSWAP, VT, Legal);
     for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
@@ -324,6 +326,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
           {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
           Expand);
     }
+    for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
+      setOperationAction(ISD::BITREVERSE, VT, Custom);
     for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
       setOperationAction(ISD::BSWAP, VT, Legal);
     for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
@@ -372,6 +376,10 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
   setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
   setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
+
+  // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
+  if (Subtarget.hasLAMCAS())
+    setMinCmpXchgSizeInBits(8);
 }
 
 bool LoongArchTargetLowering::isOffsetFoldingLegal(
@@ -436,10 +444,56 @@ SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
     return lowerBUILD_VECTOR(Op, DAG);
   case ISD::VECTOR_SHUFFLE:
     return lowerVECTOR_SHUFFLE(Op, DAG);
+  case ISD::BITREVERSE:
+    return lowerBITREVERSE(Op, DAG);
   }
   return SDValue();
 }
 
+SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  EVT ResTy = Op->getValueType(0);
+  SDValue Src = Op->getOperand(0);
+  SDLoc DL(Op);
+
+  EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
+  unsigned int OrigEltNum = ResTy.getVectorNumElements();
+  unsigned int NewEltNum = NewVT.getVectorNumElements();
+
+  SDValue NewSrc = DAG.getNode(ISD::BITCAST, DL, NewVT, Src);
+
+  SmallVector<SDValue, 8> Ops;
+  for (unsigned int i = 0; i < NewEltNum; i++) {
+    SDValue Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, NewSrc,
+                             DAG.getConstant(i, DL, MVT::i64));
+    SDValue RevOp = DAG.getNode((ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
+                                    ? LoongArchISD::BITREV_8B
+                                    : ISD::BITREVERSE,
+                                DL, MVT::i64, Op);
+    Ops.push_back(RevOp);
+  }
+  SDValue Res =
+      DAG.getNode(ISD::BITCAST, DL, ResTy, DAG.getBuildVector(NewVT, DL, Ops));
+
+  switch (ResTy.getSimpleVT().SimpleTy) {
+  default:
+    return SDValue();
+  case MVT::v16i8:
+  case MVT::v32i8:
+    return Res;
+  case MVT::v8i16:
+  case MVT::v16i16:
+  case MVT::v4i32:
+  case MVT::v8i32: {
+    SmallVector<int, 32> Mask;
+    for (unsigned int i = 0; i < NewEltNum; i++)
+      for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
+        Mask.push_back(j + (OrigEltNum / NewEltNum) * i);
+    return DAG.getVectorShuffle(ResTy, DL, Res, DAG.getUNDEF(ResTy), Mask);
+  }
+  }
+}
+
 /// Determine whether a range fits a regular pattern of values.
 /// This function accounts for the possibility of jumping over the End iterator.
 template <typename ValType>
@@ -1665,16 +1719,19 @@ SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
                                                  SelectionDAG &DAG) const {
 
   SDLoc DL(Op);
+  SDValue Op0 = Op.getOperand(0);
+
+  if (Op0.getValueType() == MVT::f16)
+    Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
 
   if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
       !Subtarget.hasBasicD()) {
-    SDValue Dst =
-        DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Op.getOperand(0));
+    SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Op0);
     return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Dst);
   }
 
   EVT FPTy = EVT::getFloatingPointVT(Op.getValueSizeInBits());
-  SDValue Trunc = DAG.getNode(LoongArchISD::FTINT, DL, FPTy, Op.getOperand(0));
+  SDValue Trunc = DAG.getNode(LoongArchISD::FTINT, DL, FPTy, Op0);
   return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Trunc);
 }
 
@@ -2868,6 +2925,8 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0));
     if (getTypeAction(*DAG.getContext(), Src.getValueType()) !=
         TargetLowering::TypeSoftenFloat) {
+      if (Src.getValueType() == MVT::f16)
+        Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
       SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, FVT, Src);
       Results.push_back(DAG.getNode(ISD::BITCAST, DL, VT, Dst));
       return;
@@ -4676,6 +4735,7 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
     NODE_NAME_CASE(REVB_2H)
     NODE_NAME_CASE(REVB_2W)
     NODE_NAME_CASE(BITREV_4B)
+    NODE_NAME_CASE(BITREV_8B)
     NODE_NAME_CASE(BITREV_W)
     NODE_NAME_CASE(ROTR_W)
     NODE_NAME_CASE(ROTL_W)
@@ -5756,6 +5816,62 @@ bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   }
 }
 
+// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
+// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
+// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
+// regression, we need to implement it manually.
+void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
+  AtomicRMWInst::BinOp Op = AI->getOperation();
+
+  assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
+          Op == AtomicRMWInst::And) &&
+         "Unable to expand");
+  unsigned MinWordSize = 4;
+
+  IRBuilder<> Builder(AI);
+  LLVMContext &Ctx = Builder.getContext();
+  const DataLayout &DL = AI->getDataLayout();
+  Type *ValueType = AI->getType();
+  Type *WordType = Type::getIntNTy(Ctx, MinWordSize * 8);
+
+  Value *Addr = AI->getPointerOperand();
+  PointerType *PtrTy = cast<PointerType>(Addr->getType());
+  IntegerType *IntTy = DL.getIndexType(Ctx, PtrTy->getAddressSpace());
+
+  Value *AlignedAddr = Builder.CreateIntrinsic(
+      Intrinsic::ptrmask, {PtrTy, IntTy},
+      {Addr, ConstantInt::get(IntTy, ~(uint64_t)(MinWordSize - 1))}, nullptr,
+      "AlignedAddr");
+
+  Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
+  Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
+  Value *ShiftAmt = Builder.CreateShl(PtrLSB, 3);
+  ShiftAmt = Builder.CreateTrunc(ShiftAmt, WordType, "ShiftAmt");
+  Value *Mask = Builder.CreateShl(
+      ConstantInt::get(WordType,
+                       (1 << (DL.getTypeStoreSize(ValueType) * 8)) - 1),
+      ShiftAmt, "Mask");
+  Value *Inv_Mask = Builder.CreateNot(Mask, "Inv_Mask");
+  Value *ValOperand_Shifted =
+      Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), WordType),
+                        ShiftAmt, "ValOperand_Shifted");
+  Value *NewOperand;
+  if (Op == AtomicRMWInst::And)
+    NewOperand = Builder.CreateOr(ValOperand_Shifted, Inv_Mask, "AndOperand");
+  else
+    NewOperand = ValOperand_Shifted;
+
+  AtomicRMWInst *NewAI =
+      Builder.CreateAtomicRMW(Op, AlignedAddr, NewOperand, Align(MinWordSize),
+                              AI->getOrdering(), AI->getSyncScopeID());
+
+  Value *Shift = Builder.CreateLShr(NewAI, ShiftAmt, "shifted");
+  Value *Trunc = Builder.CreateTrunc(Shift, ValueType, "extracted");
+  Value *FinalOldResult = Builder.CreateBitCast(Trunc, ValueType);
+  AI->replaceAllUsesWith(FinalOldResult);
+  AI->eraseFromParent();
+}
+
 TargetLowering::AtomicExpansionKind
 LoongArchTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // TODO: Add more AtomicRMWInst that needs to be extended.
@@ -5777,6 +5893,15 @@ LoongArchTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   }
 
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
+  if (Subtarget.hasLAMCAS()) {
+    if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
+                      AI->getOperation() == AtomicRMWInst::Or ||
+                      AI->getOperation() == AtomicRMWInst::Xor))
+      return AtomicExpansionKind::Expand;
+    if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
+      return AtomicExpansionKind::CmpXChg;
+  }
+
   if (Size == 8 || Size == 16)
     return AtomicExpansionKind::MaskedIntrinsic;
   return AtomicExpansionKind::None;
@@ -5831,6 +5956,10 @@ getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
 TargetLowering::AtomicExpansionKind
 LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
     AtomicCmpXchgInst *CI) const {
+
+  if (Subtarget.hasLAMCAS())
+    return AtomicExpansionKind::None;
+
   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
   if (Size == 8 || Size == 16)
     return AtomicExpansionKind::MaskedIntrinsic;
@@ -6326,13 +6455,13 @@ bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
 }
 
 ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
-  // TODO: LAMCAS will use amcas{_DB,}.[bhwd] which does not require extension.
-  return ISD::SIGN_EXTEND;
+  // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
+  return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
 }
 
 bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
-    EVT Type, bool IsSigned) const {
-  if (Subtarget.is64Bit() && Type == MVT::i32)
+    Type *Ty, bool IsSigned) const {
+  if (Subtarget.is64Bit() && Ty->isIntegerTy(32))
     return true;
 
   return IsSigned;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 5a47dfb257175..e619cb69f3332 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -68,6 +68,7 @@ enum NodeType : unsigned {
   REVB_2H,
   REVB_2W,
   BITREV_4B,
+  BITREV_8B,
   BITREV_W,
 
   // Intrinsic operations start ============================================
@@ -194,6 +195,7 @@ class LoongArchTargetLowering : public TargetLowering {
   bool hasAndNot(SDValue Y) const override;
   TargetLowering::AtomicExpansionKind
   shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
+  void emitExpandAtomicRMW(AtomicRMWInst *AI) const override;
 
   Value *emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI,
                                       Value *AlignedAddr, Value *Incr,
@@ -272,7 +274,7 @@ class LoongArchTargetLowering : public TargetLowering {
     return false;
   }
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
-  bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
+  bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override;
   bool shouldExtendTypeInLibCall(EVT Type) const override;
 
   bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
@@ -333,6 +335,7 @@ class LoongArchTargetLowering : public TargetLowering {
   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerBITREVERSE(SDValue Op, SelectionDAG &DAG) const;
 
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 0d290e10651c6..2101aa058305f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -112,6 +112,7 @@ def loongarch_bstrpick
 def loongarch_revb_2h : SDNode<"LoongArchISD::REVB_2H", SDTUnaryOp>;
 def loongarch_revb_2w : SDNode<"LoongArchISD::REVB_2W", SDTUnaryOp>;
 def loongarch_bitrev_4b : SDNode<"LoongArchISD::BITREV_4B", SDTUnaryOp>;
+def loongarch_bitrev_8b : SDNode<"LoongArchISD::BITREV_8B", SDTUnaryOp>;
 def loongarch_bitrev_w : SDNode<"LoongArchISD::BITREV_W", SDTUnaryOp>;
 def loongarch_clzw : SDNode<"LoongArchISD::CLZ_W", SDTIntBitCountUnaryOp>;
 def loongarch_ctzw : SDNode<"LoongArchISD::CTZ_W", SDTIntBitCountUnaryOp>;
@@ -721,9 +722,9 @@ class AM_3R<bits<32> op>
 }
 
 class AMCAS_3R<bits<32> op>
-    : Fmt3R<op, (outs GPR:$rd_wb), (ins GPR:$rd, GPR:$rk, GPRMemAtomic:$rj),
+    : Fmt3R<op, (outs GPR:$dst), (ins GPR:$rd, GPR:$rk, GPRMemAtomic:$rj),
             "$rd, $rk, $rj"> {
-  let Constraints = "@earlyclobber $rd_wb, $rd_wb = $rd";
+  let Constraints = "@earlyclobber $dst, $dst = $rd";
   let IsAMCAS = 1;
 }
 } // hasSideEffects = 0, mayLoad = 1, mayStore = 1,
@@ -1765,6 +1766,7 @@ def : Pat<(bitreverse (bswap GPR:$rj)), (BITREV_4B GPR:$rj)>;
 let Predicates = [IsLA64] in {
 def : Pat<(loongarch_revb_2w GPR:$rj), (REVB_2W GPR:$rj)>;
 def : Pat<(bswap GPR:$rj), (REVB_D GPR:$rj)>;
+def : Pat<(loongarch_bitrev_8b GPR:$rj), (BITREV_8B GPR:$rj)>;
 def : Pat<(loongarch_bitrev_w GPR:$rj), (BITREV_W GPR:$rj)>;
 def : Pat<(bitreverse GPR:$rj), (BITREV_D GPR:$rj)>;
 def : Pat<(bswap (bitreverse GPR:$rj)), (BITREV_8B GPR:$rj)>;
@@ -2109,7 +2111,6 @@ multiclass binary_atomic_op_bh<string inst, string op> {
 }
 
 let Predicates = [ HasLAM_BH, IsLA64 ] in {
-  
 defm : binary_atomic_op_bh<"AMSWAP", "atomic_swap">;
 defm : binary_atomic_op_bh<"AMADD", "atomic_load_add">;
 def : Pat<(atomic_load_sub_i8_monotonic GPR:$rj, GPR:$rk),
@@ -2123,9 +2124,27 @@ def : Pat<(atomic_load_sub_i16 GPR:$rj, GPR:$rk),
           (AMADD__DB_H (SUB_W R0, GPR:$rk), GPR:$rj)>;
 } // Predicates = [ IsLA64, HasLAM_BH ]
 
+let Predicates = [ HasLAMCAS, IsLA64 ] in {
+def : Pat<(atomic_cmp_swap_i8_monotonic GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS_B GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i16_monotonic GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS_H GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i32_monotonic GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS_W GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i64_monotonic GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS_D GPR:$cmp, GPR:$new, GPR:$addr)>;
+
+def : Pat<(atomic_cmp_swap_i8 GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS__DB_B GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i16 GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS__DB_H GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i32 GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS__DB_W GPR:$cmp, GPR:$new, GPR:$addr)>;
+def : Pat<(atomic_cmp_swap_i64 GPR:$addr, GPR:$cmp, GPR:$new),
+          (AMCAS__DB_D GPR:$cmp, GPR:$new, GPR:$addr)>;
+}
 
 let Predicates = [IsLA64] in {
-
 defm : binary_atomic_op_wd<"AMSWAP", "atomic_swap">;
 defm : binary_atomic_op_wd<"AMADD", "atomic_load_add">;
 defm : binary_atomic_op_wd<"AMAND", "atomic_load_and">;
diff --git a/llvm/lib/Target/LoongArch/LoongArchOptWInstrs.cpp b/llvm/lib/Target/LoongArch/LoongArchOptWInstrs.cpp
index ab90409fdf47d..51e5e288a2512 100644
--- a/llvm/lib/Target/LoongArch/LoongArchOptWInstrs.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchOptWInstrs.cpp
@@ -139,11 +139,6 @@ static bool hasAllNBitUsers(const MachineInstr &OrigMI,
       case LoongArch::MULH_WU:
       case LoongArch::MULW_D_W:
       case LoongArch::MULW_D_WU:
-      // TODO: {DIV,MOD}.{W,WU} consumes the upper 32 bits before LA664+.
-      // case LoongArch::DIV_W:
-      // case LoongArch::DIV_WU:
-      // case LoongArch::MOD_W:
-      // case LoongArch::MOD_WU:
       case LoongArch::SLL_W:
       case LoongArch::SLLI_W:
       case LoongArch::SRL_W:
@@ -170,6 +165,15 @@ static bool hasAllNBitUsers(const MachineInstr &OrigMI,
         if (Bits >= 32)
           break;
         return false;
+      // {DIV,MOD}.W{U} consumes the upper 32 bits if the div32
+      // feature is not enabled.
+      case LoongArch::DIV_W:
+      case LoongArch::DIV_WU:
+      case LoongArch::MOD_W:
+      case LoongArch::MOD_WU:
+        if (Bits >= 32 && ST.hasDiv32())
+          break;
+        return false;
       case LoongArch::MOVGR2CF:
         if (Bits >= 1)
           break;
diff --git a/llvm/lib/Target/M68k/M68kInstrData.td b/llvm/lib/Target/M68k/M68kInstrData.td
index dc777a933e278..adaa8c85eefb9 100644
--- a/llvm/lib/Target/M68k/M68kInstrData.td
+++ b/llvm/lib/Target/M68k/M68kInstrData.td
@@ -419,6 +419,8 @@ class MxMoveFromCCR_M<MxOperand MEMOp, MxEncMemOp DST_ENC>
 
 class MxMoveFromCCRPseudo<MxOperand MEMOp>
     : MxPseudo<(outs), (ins MEMOp:$dst, CCRC:$src)>;
+class MxMoveFromCCR_RPseudo<MxOperand MEMOp>
+    : MxPseudo<(outs MEMOp:$dst), (ins CCRC:$src)>;
 } // let Uses = [CCR]
 
 let mayStore = 1 in
@@ -432,7 +434,7 @@ foreach AM = MxMoveSupportedAMs in {
 
 // Only data register is allowed.
 def MOV16dc : MxMoveFromCCR_R;
-def MOV8dc  : MxMoveFromCCRPseudo<MxOp8AddrMode_d.Op>;
+def MOV8dc  : MxMoveFromCCR_RPseudo<MxOp8AddrMode_d.Op>;
 
 //===----------------------------------------------------------------------===//
 // LEA
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 2d9285f89b74a..6340464520950 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -23,7 +23,9 @@
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGenTypes/MachineValueType.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Regex.h"
@@ -571,6 +573,12 @@ bool M68kInstrInfo::ExpandPUSH_POP(MachineInstrBuilder &MIB,
 }
 
 bool M68kInstrInfo::ExpandCCR(MachineInstrBuilder &MIB, bool IsToCCR) const {
+  if (MIB->getOpcode() == M68k::MOV8cd) {
+    // Promote used register to the next class
+    MachineOperand &Opd = MIB->getOperand(1);
+    Opd.setReg(getRegisterInfo().getMatchingSuperReg(
+        Opd.getReg(), M68k::MxSubRegIndex8Lo, &M68k::DR16RegClass));
+  }
 
   // Replace the pseudo instruction with the real one
   if (IsToCCR)
@@ -579,11 +587,6 @@ bool M68kInstrInfo::ExpandCCR(MachineInstrBuilder &MIB, bool IsToCCR) const {
     // FIXME M68010 or later is required
     MIB->setDesc(get(M68k::MOV16dc));
 
-  // Promote used register to the next class
-  auto &Opd = MIB->getOperand(1);
-  Opd.setReg(getRegisterInfo().getMatchingSuperReg(
-      Opd.getReg(), M68k::MxSubRegIndex8Lo, &M68k::DR16RegClass));
-
   return true;
 }
 
@@ -757,13 +760,27 @@ void M68kInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   bool ToSR = DstReg == M68k::SR;
 
   if (FromCCR) {
-    assert(M68k::DR8RegClass.contains(DstReg) &&
-           "Need DR8 register to copy CCR");
-    Opc = M68k::MOV8dc;
+    if (M68k::DR8RegClass.contains(DstReg)) {
+      Opc = M68k::MOV8dc;
+    } else if (M68k::DR16RegClass.contains(DstReg)) {
+      Opc = M68k::MOV16dc;
+    } else if (M68k::DR32RegClass.contains(DstReg)) {
+      Opc = M68k::MOV16dc;
+    } else {
+      LLVM_DEBUG(dbgs() << "Cannot copy CCR to " << RI.getName(DstReg) << '\n');
+      llvm_unreachable("Invalid register for MOVE from CCR");
+    }
   } else if (ToCCR) {
-    assert(M68k::DR8RegClass.contains(SrcReg) &&
-           "Need DR8 register to copy CCR");
-    Opc = M68k::MOV8cd;
+    if (M68k::DR8RegClass.contains(SrcReg)) {
+      Opc = M68k::MOV8cd;
+    } else if (M68k::DR16RegClass.contains(SrcReg)) {
+      Opc = M68k::MOV16cd;
+    } else if (M68k::DR32RegClass.contains(SrcReg)) {
+      Opc = M68k::MOV16cd;
+    } else {
+      LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to CCR\n");
+      llvm_unreachable("Invalid register for MOVE to CCR");
+    }
   } else if (FromSR || ToSR)
     llvm_unreachable("Cannot emit SR copy instruction");
 
diff --git a/llvm/lib/Target/M68k/M68kRegisterInfo.h b/llvm/lib/Target/M68k/M68kRegisterInfo.h
index 0cfab150d0c84..5131768345366 100644
--- a/llvm/lib/Target/M68k/M68kRegisterInfo.h
+++ b/llvm/lib/Target/M68k/M68kRegisterInfo.h
@@ -101,7 +101,7 @@ class M68kRegisterInfo : public M68kGenRegisterInfo {
   const TargetRegisterClass *
   getCrossCopyRegClass(const TargetRegisterClass *RC) const override {
     if (RC == &M68k::CCRCRegClass)
-      return &M68k::DR32RegClass;
+      return &M68k::DR16RegClass;
     return RC;
   }
 
diff --git a/llvm/lib/Target/M68k/M68kSubtarget.h b/llvm/lib/Target/M68k/M68kSubtarget.h
index 3fbec2f72fb86..c08a9786fb27b 100644
--- a/llvm/lib/Target/M68k/M68kSubtarget.h
+++ b/llvm/lib/Target/M68k/M68kSubtarget.h
@@ -107,7 +107,7 @@ class M68kSubtarget : public M68kGenSubtargetInfo {
 
   bool isPositionIndependent() const;
 
-  bool isRegisterReservedByUser(Register R) const {
+  bool isRegisterReservedByUser(Register R) const override {
     assert(R < M68k::NUM_TARGET_REGS && "Register out of range");
     return UserReservedRegister[R];
   }
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index d90348153fd3e..d5f38c414e703 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -1102,8 +1102,7 @@ static SDValue performMADD_MSUBCombine(SDNode *ROOTNode, SelectionDAG &CurDAG,
   SDValue MAddOps[3] = {
       CurDAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mult->getOperand(0)),
       CurDAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mult->getOperand(1)), ACCIn};
-  EVT VTs[2] = {MVT::i32, MVT::i32};
-  SDValue MAdd = CurDAG.getNode(Opcode, DL, VTs, MAddOps);
+  SDValue MAdd = CurDAG.getNode(Opcode, DL, MVT::Untyped, MAddOps);
 
   SDValue ResLo = CurDAG.getNode(MipsISD::MFLO, DL, MVT::i32, MAdd);
   SDValue ResHi = CurDAG.getNode(MipsISD::MFHI, DL, MVT::i32, MAdd);
@@ -3871,10 +3870,10 @@ MipsTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
   return CCInfo.CheckReturn(Outs, RetCC_Mips);
 }
 
-bool MipsTargetLowering::shouldSignExtendTypeInLibCall(EVT Type,
+bool MipsTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
                                                        bool IsSigned) const {
-  if ((ABI.IsN32() || ABI.IsN64()) && Type == MVT::i32)
-      return true;
+  if ((ABI.IsN32() || ABI.IsN64()) && Ty->isIntegerTy(32))
+    return true;
 
   return IsSigned;
 }
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h
index 8033898091c75..e245c056de649 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -623,7 +623,7 @@ class TargetRegisterClass;
     SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
                                  const SDLoc &DL, SelectionDAG &DAG) const;
 
-    bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
+    bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override;
 
     // Inline asm support
     ConstraintType getConstraintType(StringRef Constraint) const override;
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index 0f38f644e89af..65e1893d3f3bd 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -450,4 +450,6 @@ void NVPTXInstPrinter::printTmaReductionMode(const MCInst *MI, int OpNum,
     O << ".xor";
     return;
   }
+  llvm_unreachable(
+      "Invalid Reduction Op in printCpAsyncBulkTensorReductionMode");
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 62647b3128518..ce94dded815b8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -667,6 +667,11 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
     setTruncStoreAction(VT, MVT::i1, Expand);
   }
 
+  setCondCodeAction({ISD::SETNE, ISD::SETEQ, ISD::SETUGE, ISD::SETULE,
+                     ISD::SETUGT, ISD::SETULT, ISD::SETGT, ISD::SETLT,
+                     ISD::SETGE, ISD::SETLE},
+                    MVT::i1, Expand);
+
   // expand extload of vector of integers.
   setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::v2i16,
                    MVT::v2i8, Expand);
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index f864bd0f35ef1..4f33d80109a5b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -728,6 +728,20 @@ let hasSideEffects = false in {
   def CVT_f16x2_e5m2x2 : CVT_f16x2_fp8<"e5m2">;
 }
 
+def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{
+  return N->hasOneUse();
+}]>;
+
+def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$lo)),
+                                (bf16 (fpround_oneuse Float32Regs:$hi)))),
+          (CVT_bf16x2_f32 Float32Regs:$hi, Float32Regs:$lo, CvtRN)>,
+      Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>;
+
+def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$lo)),
+                               (f16 (fpround_oneuse Float32Regs:$hi)))),
+          (CVT_f16x2_f32 Float32Regs:$hi, Float32Regs:$lo, CvtRN)>,
+      Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>;
+
 //-----------------------------------
 // Selection instructions (selp)
 //-----------------------------------
@@ -2121,17 +2135,6 @@ defm : ISET_FORMAT_UNSIGNED<setule, CmpLE>;
 defm : ISET_FORMAT_UNSIGNED<setueq, CmpEQ>;
 defm : ISET_FORMAT_UNSIGNED<setune, CmpNE>;
 
-// i1 compares
-def : Pat<(setne Int1Regs:$a, Int1Regs:$b),
-          (XORb1rr Int1Regs:$a, Int1Regs:$b)>;
-def : Pat<(setune Int1Regs:$a, Int1Regs:$b),
-          (XORb1rr Int1Regs:$a, Int1Regs:$b)>;
-
-def : Pat<(seteq Int1Regs:$a, Int1Regs:$b),
-          (NOT1 (XORb1rr Int1Regs:$a, Int1Regs:$b))>;
-def : Pat<(setueq Int1Regs:$a, Int1Regs:$b),
-          (NOT1 (XORb1rr Int1Regs:$a, Int1Regs:$b))>;
-
 // comparisons of i8 extracted with BFE as i32
 // It's faster to do comparison directly on i32 extracted by BFE,
 // instead of the long conversion and sign extending.
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 0f75c10383850..da74add440ca8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -536,22 +536,22 @@ multiclass CP_ASYNC_BULK_TENSOR_G2S_INTR<int dim, bit is_shared32, string mode>
   defvar asm_str = !if(!eq(mode, "im2col"),
     !strconcat(asm_str_default, im2col_asm_str), asm_str_default);
 
-  def "": NVPTXInst<(outs),
-          !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag),
-          !strconcat(G2S_STRINGS<dim, mode, 0, 0>.inst_name, asm_str, ";"), []>,
-          Requires<[hasPTX<80>, hasSM<90>]>;
-  def _MC: NVPTXInst<(outs),
-           !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc)),
-           !strconcat(G2S_STRINGS<dim, mode, 1, 0>.inst_name, asm_str, ", $mc;"), []>,
-           Requires<[hasPTX<80>, hasSM<90>]>;
-  def _CH: NVPTXInst<(outs),
-           !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
-           !strconcat(G2S_STRINGS<dim, mode, 0, 1>.inst_name, asm_str, ", $ch;"), []>,
-           Requires<[hasPTX<80>, hasSM<90>]>;
-  def _MC_CH: NVPTXInst<(outs),
-              !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc, Int64Regs:$ch)),
-              !strconcat(G2S_STRINGS<dim, mode, 1, 1>.inst_name, asm_str, ", $mc, $ch;"), []>,
-              Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME: NVPTXInst<(outs),
+            !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag),
+            !strconcat(G2S_STRINGS<dim, mode, 0, 0>.inst_name, asm_str, ";"), []>,
+            Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME # _MC: NVPTXInst<(outs),
+                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc)),
+                  !strconcat(G2S_STRINGS<dim, mode, 1, 0>.inst_name, asm_str, ", $mc;"), []>,
+                  Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME # _CH: NVPTXInst<(outs),
+                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
+                  !strconcat(G2S_STRINGS<dim, mode, 0, 1>.inst_name, asm_str, ", $ch;"), []>,
+                  Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME # _MC_CH: NVPTXInst<(outs),
+                     !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc, Int64Regs:$ch)),
+                     !strconcat(G2S_STRINGS<dim, mode, 1, 1>.inst_name, asm_str, ", $mc, $ch;"), []>,
+                     Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
 foreach dim = [1, 2, 3, 4, 5] in {
@@ -588,14 +588,14 @@ multiclass CP_ASYNC_BULK_TENSOR_S2G_INTR<int dim, bit shared32, string mode> {
   defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
   defvar rc = !if(shared32, Int32Regs, Int64Regs);
 
-  def "": NVPTXInst<(outs),
-          !con((ins rc:$src, Int64Regs:$tmap), dims_dag),
-          !strconcat(S2G_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
-          Requires<[hasPTX<80>, hasSM<90>]>;
-  def _CH: NVPTXInst<(outs),
-           !con((ins rc:$src, Int64Regs:$tmap), dims_dag, (ins Int64Regs:$ch)),
-           !strconcat(S2G_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
-           Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME: NVPTXInst<(outs),
+            !con((ins rc:$src, Int64Regs:$tmap), dims_dag),
+            !strconcat(S2G_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
+            Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME # _CH: NVPTXInst<(outs),
+                  !con((ins rc:$src, Int64Regs:$tmap), dims_dag, (ins Int64Regs:$ch)),
+                  !strconcat(S2G_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
+                  Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
 def TMAReductionFlags : Operand<i32> {
@@ -662,14 +662,14 @@ multiclass CP_ASYNC_BULK_TENSOR_PREFETCH_INTR<int dim, string mode> {
   defvar asm_str = !if(!eq(mode, "im2col"),
     !strconcat(asm_str_default, im2col_asm_str), asm_str_default);
 
-  def "": NVPTXInst<(outs),
-          !con((ins Int64Regs:$tmap), dims_dag, im2col_dag),
-          !strconcat(PREFETCH_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
-          Requires<[hasPTX<80>, hasSM<90>]>;
-  def _CH: NVPTXInst<(outs),
-           !con((ins Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
-           !strconcat(PREFETCH_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
-           Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME: NVPTXInst<(outs),
+            !con((ins Int64Regs:$tmap), dims_dag, im2col_dag),
+            !strconcat(PREFETCH_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
+            Requires<[hasPTX<80>, hasSM<90>]>;
+  def NAME # _CH: NVPTXInst<(outs),
+                  !con((ins Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
+                  !strconcat(PREFETCH_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
+                  Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
 foreach dim = [1, 2, 3, 4, 5] in {
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index c9fb76dff0509..61be6abaacd2f 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -37,6 +37,7 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCSymbolXCOFF.h"
+#include "llvm/MC/MCXCOFFObjectWriter.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -258,7 +259,11 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
   }
 
   void emitMachine(StringRef CPU) override {
-    OS << "\t.machine " << CPU << '\n';
+    const Triple &TT = Streamer.getContext().getTargetTriple();
+    if (TT.isOSBinFormatXCOFF())
+      OS << "\t.machine\t" << '\"' << CPU << '\"' << '\n';
+    else
+      OS << "\t.machine " << CPU << '\n';
   }
 
   void emitAbiVersion(int AbiVersion) override {
@@ -422,7 +427,8 @@ class PPCTargetXCOFFStreamer : public PPCTargetStreamer {
   }
 
   void emitMachine(StringRef CPU) override {
-    llvm_unreachable("Machine pseudo-ops are invalid for XCOFF.");
+    static_cast<XCOFFObjectWriter &>(Streamer.getAssemblerPtr()->getWriter())
+        .setCPU(CPU);
   }
 
   void emitAbiVersion(int AbiVersion) override {
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 244328d9cd53f..073857e4c2dc0 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -71,6 +71,7 @@
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/TargetParser/PPCTargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <cassert>
@@ -3046,6 +3047,30 @@ void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
 bool PPCAIXAsmPrinter::doInitialization(Module &M) {
   const bool Result = PPCAsmPrinter::doInitialization(M);
 
+  // Emit the .machine directive on AIX.
+  const Triple &Target = TM.getTargetTriple();
+  XCOFF::CFileCpuId TargetCpuId = XCOFF::TCPU_INVALID;
+  // Walk through the "target-cpu" attribute of functions and use the newest
+  // level as the CPU of the module.
+  for (auto &F : M) {
+    XCOFF::CFileCpuId FunCpuId =
+        XCOFF::getCpuID(TM.getSubtargetImpl(F)->getCPU());
+    if (FunCpuId > TargetCpuId)
+      TargetCpuId = FunCpuId;
+  }
+  // If there is no "target-cpu" attribute within the functions, take the
+  // "-mcpu" value. If both are omitted, use getNormalizedPPCTargetCPU() to
+  // determine the default CPU.
+  if (!TargetCpuId) {
+    StringRef TargetCPU = TM.getTargetCPU();
+    TargetCpuId = XCOFF::getCpuID(
+        TargetCPU.empty() ? PPC::getNormalizedPPCTargetCPU(Target) : TargetCPU);
+  }
+
+  PPCTargetStreamer *TS =
+      static_cast<PPCTargetStreamer *>(OutStreamer->getTargetStreamer());
+  TS->emitMachine(XCOFF::getTCPUString(TargetCpuId));
+
   auto setCsectAlignment = [this](const GlobalObject *GO) {
     // Declarations have 0 alignment which is set by default.
     if (GO->isDeclarationForLinker())
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 4706051e60156..2475b8ad11f10 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -193,8 +193,8 @@ namespace {
     }
 
     /// getSmallIPtrImm - Return a target constant of pointer type.
-    inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) {
-      return CurDAG->getTargetConstant(
+    inline SDValue getSmallIPtrImm(int64_t Imm, const SDLoc &dl) {
+      return CurDAG->getSignedTargetConstant(
           Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
     }
 
@@ -207,7 +207,7 @@ namespace {
     /// base register.  Return the virtual register that holds this value.
     SDNode *getGlobalBaseReg();
 
-    void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0);
+    void selectFrameIndex(SDNode *SN, SDNode *N, int64_t Offset = 0);
 
     // Select - Convert the specified operand from a target-independent to a
     // target-specific node if it hasn't already been changed.
@@ -639,7 +639,7 @@ static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
          && isInt32Immediate(N->getOperand(1).getNode(), Imm);
 }
 
-void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) {
+void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, int64_t Offset) {
   SDLoc dl(SN);
   int FI = cast<FrameIndexSDNode>(N)->getIndex();
   SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 5451035b50084..2af983fc7b04e 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -198,6 +198,14 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
   setOperationAction(ISD::UADDO, RegVT, Custom);
 
+  // On P10, the default lowering generates better code using the
+  // setbc instruction.
+  if (!Subtarget.hasP10Vector()) {
+    setOperationAction(ISD::SSUBO, MVT::i32, Custom);
+    if (isPPC64)
+      setOperationAction(ISD::SSUBO, MVT::i64, Custom);
+  }
+
   // Match BITREVERSE to customized fast code sequence in the td file.
   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
@@ -1630,14 +1638,14 @@ static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
 
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
 /// function arguments in the caller parameter area.
-uint64_t PPCTargetLowering::getByValTypeAlignment(Type *Ty,
-                                                  const DataLayout &DL) const {
+Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
+                                               const DataLayout &DL) const {
   // 16byte and wider vectors are passed on 16byte boundary.
   // The rest is 8 on PPC64 and 4 on PPC32 boundary.
   Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
   if (Subtarget.hasAltivec())
     getMaxByValAlign(Ty, Alignment, Align(16));
-  return Alignment.value();
+  return Alignment;
 }
 
 bool PPCTargetLowering::useSoftFloat() const {
@@ -1842,6 +1850,10 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::LXVRZX:          return "PPCISD::LXVRZX";
   case PPCISD::STORE_COND:
     return "PPCISD::STORE_COND";
+  case PPCISD::SETBC:
+    return "PPCISD::SETBC";
+  case PPCISD::SETBCR:
+    return "PPCISD::SETBCR";
   }
   return nullptr;
 }
@@ -2694,7 +2706,7 @@ bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
   if (!isa<ConstantSDNode>(N))
     return false;
 
-  Imm = (int64_t)N->getAsZExtVal();
+  Imm = (int64_t)cast<ConstantSDNode>(N)->getSExtValue();
   return isInt<34>(Imm);
 }
 bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
@@ -2916,7 +2928,7 @@ bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
   if (N.getOpcode() == ISD::ADD) {
     if (!isIntS34Immediate(N.getOperand(1), Imm))
       return false;
-    Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
+    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0)))
       Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
     else
@@ -2937,12 +2949,12 @@ bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
       Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
     else
       Base = N.getOperand(0);
-    Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
+    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
     return true;
   }
 
   if (isIntS34Immediate(N, Imm)) { // If the address is a 34-bit const.
-    Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
+    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
     Base = DAG.getRegister(PPC::ZERO8, N.getValueType());
     return true;
   }
@@ -11256,31 +11268,55 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue };
   SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops);
 
-  // Now that we have the comparison, emit a copy from the CR to a GPR.
-  // This is flagged to the above dot comparison.
-  SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32,
-                                DAG.getRegister(PPC::CR6, MVT::i32),
-                                CompNode.getValue(1));
-
   // Unpack the result based on how the target uses it.
-  unsigned BitNo;   // Bit # of CR6.
-  bool InvertBit;   // Invert result?
+  unsigned BitNo; // Bit # of CR6.
+  bool InvertBit; // Invert result?
+  unsigned Bitx;
+  unsigned SetOp;
   switch (Op.getConstantOperandVal(1)) {
-  default:  // Can't happen, don't crash on invalid number though.
-  case 0:   // Return the value of the EQ bit of CR6.
-    BitNo = 0; InvertBit = false;
+  default: // Can't happen, don't crash on invalid number though.
+  case 0:  // Return the value of the EQ bit of CR6.
+    BitNo = 0;
+    InvertBit = false;
+    Bitx = PPC::sub_eq;
+    SetOp = PPCISD::SETBC;
     break;
-  case 1:   // Return the inverted value of the EQ bit of CR6.
-    BitNo = 0; InvertBit = true;
+  case 1: // Return the inverted value of the EQ bit of CR6.
+    BitNo = 0;
+    InvertBit = true;
+    Bitx = PPC::sub_eq;
+    SetOp = PPCISD::SETBCR;
     break;
-  case 2:   // Return the value of the LT bit of CR6.
-    BitNo = 2; InvertBit = false;
+  case 2: // Return the value of the LT bit of CR6.
+    BitNo = 2;
+    InvertBit = false;
+    Bitx = PPC::sub_lt;
+    SetOp = PPCISD::SETBC;
     break;
-  case 3:   // Return the inverted value of the LT bit of CR6.
-    BitNo = 2; InvertBit = true;
+  case 3: // Return the inverted value of the LT bit of CR6.
+    BitNo = 2;
+    InvertBit = true;
+    Bitx = PPC::sub_lt;
+    SetOp = PPCISD::SETBCR;
     break;
   }
 
+  SDValue GlueOp = CompNode.getValue(1);
+  if (Subtarget.isISA3_1()) {
+    SDValue SubRegIdx = DAG.getTargetConstant(Bitx, dl, MVT::i32);
+    SDValue CR6Reg = DAG.getRegister(PPC::CR6, MVT::i32);
+    SDValue CRBit =
+        SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1,
+                                   CR6Reg, SubRegIdx, GlueOp),
+                0);
+    return DAG.getNode(SetOp, dl, MVT::i32, CRBit);
+  }
+
+  // Now that we have the comparison, emit a copy from the CR to a GPR.
+  // This is flagged to the above dot comparison.
+  SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32,
+                              DAG.getRegister(PPC::CR6, MVT::i32), GlueOp);
+
   // Shift the bit into the low position.
   Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags,
                       DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32));
@@ -12013,6 +12049,30 @@ SDValue PPCTargetLowering::LowerUaddo(SDValue Op, SelectionDAG &DAG) const {
   return Res;
 }
 
+SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
+
+  SDLoc dl(Op);
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+  EVT VT = Op.getNode()->getValueType(0);
+
+  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
+
+  SDValue Xor1 = DAG.getNode(ISD::XOR, dl, VT, RHS, LHS);
+  SDValue Xor2 = DAG.getNode(ISD::XOR, dl, VT, Sub, LHS);
+
+  SDValue And = DAG.getNode(ISD::AND, dl, VT, Xor1, Xor2);
+
+  SDValue Overflow =
+      DAG.getNode(ISD::SRL, dl, VT, And,
+                  DAG.getConstant(VT.getSizeInBits() - 1, dl, MVT::i32));
+
+  SDValue OverflowTrunc =
+      DAG.getNode(ISD::TRUNCATE, dl, Op.getNode()->getValueType(1), Overflow);
+
+  return DAG.getMergeValues({Sub, OverflowTrunc}, dl);
+}
+
 /// LowerOperation - Provide custom lowering hooks for some operations.
 ///
 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
@@ -12035,6 +12095,8 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SETCC:              return LowerSETCC(Op, DAG);
   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
+  case ISD::SSUBO:
+    return LowerSSUBO(Op, DAG);
 
   case ISD::INLINEASM:
   case ISD::INLINEASM_BR:       return LowerINLINEASM(Op, DAG);
@@ -18772,7 +18834,7 @@ SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   SDValue Callee =
       DAG.getExternalSymbol(LibCallName, TLI.getPointerTy(DAG.getDataLayout()));
-  bool SignExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, false);
+  bool SignExtend = TLI.shouldSignExtendTypeInLibCall(RetTy, false);
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   for (const SDValue &N : Op->op_values()) {
@@ -18780,7 +18842,7 @@ SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
     Entry.Node = N;
     Entry.Ty = ArgTy;
-    Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, SignExtend);
+    Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, SignExtend);
     Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
   }
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 0adbad8684597..8f41fc107a691 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -501,6 +501,12 @@ namespace llvm {
     /// Constrained floating point add in round-to-zero mode.
     STRICT_FADDRTZ,
 
+    /// SETBC - The ISA 3.1 (P10) SETBC instruction.
+    SETBC,
+
+    /// SETBCR - The ISA 3.1 (P10) SETBCR instruction.
+    SETBCR,
+
     // NOTE: The nodes below may require PC-Rel specific patterns if the
     // address could be PC-Relative. When adding new nodes below, consider
     // whether or not the address can be PC-Relative and add the corresponding
@@ -979,10 +985,8 @@ namespace llvm {
                                  StringRef Constraint, MVT VT) const override;
 
     /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
-    /// function arguments in the caller parameter area.  This is the actual
-    /// alignment, not its logarithm.
-    uint64_t getByValTypeAlignment(Type *Ty,
-                                   const DataLayout &DL) const override;
+    /// function arguments in the caller parameter area.
+    Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override;
 
     /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
     /// vector.  If it is invalid, don't add anything to Ops.
@@ -1279,6 +1283,7 @@ namespace llvm {
     SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerUaddo(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerSSUBO(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td
index cea44e34f0308..19247c1f3fe6d 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrP10.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td
@@ -79,6 +79,10 @@ def SDT_PPCxxmfacc : SDTypeProfile<1, 1, [
   SDTCisVT<0, v512i1>, SDTCisVT<1, v512i1>
 ]>;
 
+def SDT_PPCsetbc : SDTypeProfile<1, 1, [
+  SDTCisInt<0>, SDTCisInt<1>
+]>;
+
 //===----------------------------------------------------------------------===//
 // ISA 3.1 specific PPCISD nodes.
 //
@@ -91,6 +95,8 @@ def PPCAccExtractVsx : SDNode<"PPCISD::EXTRACT_VSX_REG", SDT_PPCAccExtractVsx,
 def PPCPairExtractVsx : SDNode<"PPCISD::EXTRACT_VSX_REG", SDT_PPCPairExtractVsx,
                         []>;
 def PPCxxmfacc : SDNode<"PPCISD::XXMFACC", SDT_PPCxxmfacc, []>;
+def PPCsetbc  : SDNode<"PPCISD::SETBC",   SDT_PPCsetbc, []>;
+def PPCsetbcr : SDNode<"PPCISD::SETBCR",  SDT_PPCsetbc, []>;
 
 //===----------------------------------------------------------------------===//
 
@@ -1401,10 +1407,12 @@ let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1, Predicates = [P
 
 let Predicates = [IsISA3_1] in {
   def SETBC : XForm_XT5_BI5<31, 384, (outs gprc:$RST), (ins crbitrc:$BI),
-                            "setbc $RST, $BI", IIC_IntCompare, []>,
+                            "setbc $RST, $BI", IIC_IntCompare,
+                            [(set i32:$RST, (PPCsetbc i1:$BI))]>,
                             SExt32To64, ZExt32To64;
   def SETBCR : XForm_XT5_BI5<31, 416, (outs gprc:$RST), (ins crbitrc:$BI),
-                             "setbcr $RST, $BI", IIC_IntCompare, []>,
+                             "setbcr $RST, $BI", IIC_IntCompare,
+                             [(set i32:$RST, (PPCsetbcr i1:$BI))]>,
                              SExt32To64, ZExt32To64;
   def SETNBC : XForm_XT5_BI5<31, 448, (outs gprc:$RST), (ins crbitrc:$BI),
                              "setnbc $RST, $BI", IIC_IntCompare, []>,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 2b64ab9aa6973..b1ad041bde790 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -99,11 +99,6 @@ static cl::opt<bool>
                   cl::desc("Expand eligible cr-logical binary ops to branches"),
                   cl::init(true), cl::Hidden);
 
-static cl::opt<bool> MergeStringPool(
-    "ppc-merge-string-pool",
-    cl::desc("Merge all of the strings in a module into one pool"),
-    cl::init(true), cl::Hidden);
-
 static cl::opt<bool> EnablePPCGenScalarMASSEntries(
     "enable-ppc-gen-scalar-mass", cl::init(false),
     cl::desc("Enable lowering math functions to their corresponding MASS "
@@ -190,7 +185,7 @@ static std::string getDataLayoutString(const Triple &T) {
 
   // PPC64 has 32 and 64 bit registers, PPC32 has only 32 bit ones.
   if (is64Bit)
-    Ret += "-n32:64";
+    Ret += "-i128:128-n32:64";
   else
     Ret += "-n32";
 
diff --git a/llvm/lib/Target/PowerPC/PPCVSXFMAMutate.cpp b/llvm/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
index 174d78e6fc963..e620637288eb6 100644
--- a/llvm/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
+++ b/llvm/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
@@ -288,22 +288,13 @@ namespace {
           UseMO.substVirtReg(KilledProdReg, KilledProdSubReg, *TRI);
         }
 
-        // Extend the live intervals of the killed product operand to hold the
-        // fma result.
+        // Recalculate the live intervals of the killed product operand.
+        LIS->removeInterval(KilledProdReg);
+        LiveInterval &NewFMAInt =
+            LIS->createAndComputeVirtRegInterval(KilledProdReg);
 
-        LiveInterval &NewFMAInt = LIS->getInterval(KilledProdReg);
-        for (auto &AI : FMAInt) {
-          // Don't add the segment that corresponds to the original copy.
-          if (AI.valno == AddendValNo)
-            continue;
-
-          VNInfo *NewFMAValNo =
-              NewFMAInt.getNextValue(AI.start, LIS->getVNInfoAllocator());
-
-          NewFMAInt.addSegment(
-              LiveInterval::Segment(AI.start, AI.end, NewFMAValNo));
-        }
         LLVM_DEBUG(dbgs() << "  extended: " << NewFMAInt << '\n');
+        (void)NewFMAInt;
 
         // Extend the live interval of the addend source (it might end at the
         // copy to be removed, or somewhere in between there and here). This
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index b843bb5ae4310..7c91dc07bbd3e 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -717,6 +717,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
   bool isUImm6() const { return IsUImm<6>(); }
   bool isUImm7() const { return IsUImm<7>(); }
   bool isUImm8() const { return IsUImm<8>(); }
+  bool isUImm11() const { return IsUImm<11>(); }
   bool isUImm16() const { return IsUImm<16>(); }
   bool isUImm20() const { return IsUImm<20>(); }
   bool isUImm32() const { return IsUImm<32>(); }
@@ -1563,6 +1564,8 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, -(1 << 9), (1 << 9) - 16,
         "immediate must be a multiple of 16 bytes and non-zero in the range");
+  case Match_InvalidUImm11:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 11) - 1);
   case Match_InvalidSImm12:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, -(1 << 11), (1 << 11) - 1,
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index fd049d1a57860..b95ad9dd428cc 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -58,6 +58,7 @@ add_llvm_target(RISCVCodeGen
   RISCVTargetMachine.cpp
   RISCVTargetObjectFile.cpp
   RISCVTargetTransformInfo.cpp
+  RISCVVectorMaskDAGMutation.cpp
   RISCVVectorPeephole.cpp
   RISCVVLOptimizer.cpp
   RISCVZacasABIFix.cpp
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index cf8e337810d83..4d563046923a5 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -682,6 +682,12 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
                         "CORE-V SIMD extensions custom opcode table");
   TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbi, DecoderTableXCVbi32,
                         "CORE-V Immediate Branching custom opcode table");
+  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXqcicsr, DecoderTableXqcicsr32,
+                        "Qualcomm uC CSR custom opcode table");
+  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXqcisls, DecoderTableXqcisls32,
+                        "Qualcomm uC Scaled Load Store custom opcode table");
+  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXqcia, DecoderTableXqcia32,
+                        "Qualcomm uC Arithmetic custom opcode table");
   TRY_TO_DECODE(true, DecoderTable32, "RISCV32 table");
 
   return MCDisassembler::Fail;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index f50611aac5dd1..e451f535809c2 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -350,10 +350,13 @@ static bool isSupportedArgumentType(Type *T, const RISCVSubtarget &Subtarget,
                                     bool IsLowerArgs = false) {
   if (T->isIntegerTy())
     return true;
-  if (T->isHalfTy() || T->isFloatTy() || T->isDoubleTy())
+  if (T->isHalfTy() || T->isFloatTy() || T->isDoubleTy() || T->isFP128Ty())
     return true;
   if (T->isPointerTy())
     return true;
+  if (T->isArrayTy())
+    return isSupportedArgumentType(T->getArrayElementType(), Subtarget,
+                                   IsLowerArgs);
   // TODO: Support fixed vector types.
   if (IsLowerArgs && T->isVectorTy() && Subtarget.hasVInstructions() &&
       T->isScalableTy() &&
@@ -593,6 +596,8 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   for (auto &AInfo : Info.OrigArgs) {
     if (!isSupportedArgumentType(AInfo.Ty, Subtarget))
       return false;
+    if (AInfo.Flags[0].isByVal())
+      return false;
   }
 
   if (!Info.OrigRet.Ty->isVoidTy() &&
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 6ddc447da1a56..985264c591e10 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -83,7 +83,14 @@ class RISCVInstructionSelector : public InstructionSelector {
   bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
 
-  ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
+  ComplexRendererFns selectShiftMask(MachineOperand &Root,
+                                     unsigned ShiftWidth) const;
+  ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
+    return selectShiftMask(Root, STI.getXLen());
+  }
+  ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
+    return selectShiftMask(Root, 32);
+  }
   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
 
   ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
@@ -127,6 +134,8 @@ class RISCVInstructionSelector : public InstructionSelector {
 
   void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
                            int OpIdx) const;
+  void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
+                                 const MachineInstr &MI, int OpIdx) const;
 
   const RISCVSubtarget &STI;
   const RISCVInstrInfo &TII;
@@ -172,22 +181,18 @@ RISCVInstructionSelector::RISCVInstructionSelector(
 }
 
 InstructionSelector::ComplexRendererFns
-RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
+RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
+                                          unsigned ShiftWidth) const {
   if (!Root.isReg())
     return std::nullopt;
 
   using namespace llvm::MIPatternMatch;
 
-  Register RootReg = Root.getReg();
-  Register ShAmtReg = RootReg;
-  const LLT ShiftLLT = MRI->getType(RootReg);
-  unsigned ShiftWidth = ShiftLLT.getSizeInBits();
-  assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
+  Register ShAmtReg = Root.getReg();
   // Peek through zext.
   Register ZExtSrcReg;
-  if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
+  if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
     ShAmtReg = ZExtSrcReg;
-  }
 
   APInt AndMask;
   Register AndSrcReg;
@@ -858,6 +863,14 @@ void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
   MIB.addImm(llvm::countr_zero(C));
 }
 
+void RISCVInstructionSelector::renderXLenSubTrailingOnes(
+    MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
+  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
+         "Expected G_CONSTANT");
+  uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
+  MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
+}
+
 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
     LLT Ty, const RegisterBank &RB) const {
   if (RB.getID() == RISCV::GPRBRegBankID) {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index e7166fdab8c26..21bfb444a569d 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -412,9 +412,9 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});
 
   getActionDefinitionsBuilder(G_PHI)
-      .legalFor({p0, sXLen})
+      .legalFor({p0, s32, sXLen})
       .widenScalarToNextPow2(0)
-      .clampScalar(0, sXLen, sXLen);
+      .clampScalar(0, s32, sXLen);
 
   getActionDefinitionsBuilder({G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
       .legalFor({p0});
@@ -482,6 +482,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .minScalar(ST.hasStdExtZbb(), 0, sXLen)
       .lower();
 
+  getActionDefinitionsBuilder({G_SCMP, G_UCMP}).lower();
+
   getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
 
   getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
@@ -491,21 +493,24 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
 
   // FP Operations
 
+  // FIXME: Support s128 for rv32 when libcall handling is able to use sret.
   getActionDefinitionsBuilder(
       {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM})
       .legalFor(ST.hasStdExtF(), {s32})
       .legalFor(ST.hasStdExtD(), {s64})
       .legalFor(ST.hasStdExtZfh(), {s16})
-      .libcallFor({s32, s64});
+      .libcallFor({s32, s64})
+      .libcallFor(ST.is64Bit(), {s128});
 
   getActionDefinitionsBuilder({G_FNEG, G_FABS})
       .legalFor(ST.hasStdExtF(), {s32})
       .legalFor(ST.hasStdExtD(), {s64})
       .legalFor(ST.hasStdExtZfh(), {s16})
-      .lowerFor({s32, s64});
+      .lowerFor({s32, s64, s128});
 
   getActionDefinitionsBuilder(G_FREM)
       .libcallFor({s32, s64})
+      .libcallFor(ST.is64Bit(), {s128})
       .minScalar(0, s32)
       .scalarize(0);
 
@@ -520,18 +525,23 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   getActionDefinitionsBuilder(G_FPTRUNC)
       .legalFor(ST.hasStdExtD(), {{s32, s64}})
       .legalFor(ST.hasStdExtZfh(), {{s16, s32}})
-      .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s16, s64}});
+      .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s16, s64}})
+      .libcallFor({{s32, s64}})
+      .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}});
   getActionDefinitionsBuilder(G_FPEXT)
       .legalFor(ST.hasStdExtD(), {{s64, s32}})
       .legalFor(ST.hasStdExtZfh(), {{s32, s16}})
-      .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s64, s16}});
+      .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s64, s16}})
+      .libcallFor({{s64, s32}})
+      .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}});
 
   getActionDefinitionsBuilder(G_FCMP)
       .legalFor(ST.hasStdExtF(), {{sXLen, s32}})
       .legalFor(ST.hasStdExtD(), {{sXLen, s64}})
       .legalFor(ST.hasStdExtZfh(), {{sXLen, s16}})
       .clampScalar(0, sXLen, sXLen)
-      .libcallFor({{sXLen, s32}, {sXLen, s64}});
+      .libcallFor({{sXLen, s32}, {sXLen, s64}})
+      .libcallFor(ST.is64Bit(), {{sXLen, s128}});
 
   // TODO: Support vector version of G_IS_FPCLASS.
   getActionDefinitionsBuilder(G_IS_FPCLASS)
@@ -544,7 +554,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .legalFor(ST.hasStdExtF(), {s32})
       .legalFor(ST.hasStdExtD(), {s64})
       .legalFor(ST.hasStdExtZfh(), {s16})
-      .lowerFor({s32, s64});
+      .lowerFor({s32, s64, s128});
 
   getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
       .legalFor(ST.hasStdExtF(), {{sXLen, s32}})
@@ -556,23 +566,52 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .widenScalarToNextPow2(0)
       .minScalar(0, s32)
       .libcallFor({{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
-      .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}});
+      .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}}) // FIXME RV32.
+      .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}, {s128, s128}});
 
   getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
       .legalFor(ST.hasStdExtF(), {{s32, sXLen}})
       .legalFor(ST.hasStdExtD(), {{s64, sXLen}})
       .legalFor(ST.hasStdExtZfh(), {{s16, sXLen}})
       .widenScalarToNextPow2(1)
-      .minScalar(1, sXLen)
+      // Promote to XLen if the operation is legal.
+      .widenScalarIf(
+          [=, &ST](const LegalityQuery &Query) {
+            return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
+                   (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
+                   ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
+                    (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
+                    (ST.hasStdExtZfh() &&
+                     Query.Types[0].getSizeInBits() == 16));
+          },
+          LegalizeMutations::changeTo(1, sXLen))
+      // Otherwise only promote to s32 since we have si libcalls.
+      .minScalar(1, s32)
       .libcallFor({{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
-      .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}});
+      .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}}) // FIXME RV32.
+      .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}, {s128, s128}});
 
   // FIXME: We can do custom inline expansion like SelectionDAG.
-  // FIXME: Legal with Zfa.
   getActionDefinitionsBuilder({G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
                                G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
                                G_INTRINSIC_ROUNDEVEN})
+      .legalFor(ST.hasStdExtZfa(), {s32})
+      .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {s64})
+      .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {s16})
+      .libcallFor({s32, s64});
+
+  getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})
+      .legalFor(ST.hasStdExtZfa(), {s32})
+      .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {s64})
+      .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {s16});
+
+  getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
+                               G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
+                               G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
+                               G_FTANH})
       .libcallFor({s32, s64});
+  getActionDefinitionsBuilder({G_FPOWI, G_FLDEXP})
+      .libcallFor({{s32, s32}, {s64, s32}});
 
   getActionDefinitionsBuilder(G_VASTART).customFor({p0});
 
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index 829c0ac92c52a..a082b18867666 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -151,6 +151,7 @@ bool RISCVRegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
   switch (MI.getOpcode()) {
   case RISCV::G_FCVT_W_RV64:
   case RISCV::G_FCVT_WU_RV64:
+  case RISCV::G_FCLASS:
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
   case TargetOpcode::G_FCMP:
@@ -326,19 +327,21 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     // Use FPR64 for s64 loads on rv32.
     if (GPRSize == 32 && Size.getFixedValue() == 64) {
       assert(MF.getSubtarget<RISCVSubtarget>().hasStdExtD());
-      OpdsMapping[0] = getFPValueMapping(Ty.getSizeInBits());
+      OpdsMapping[0] = getFPValueMapping(Size);
       break;
     }
 
     // Check if that load feeds fp instructions.
     // In that case, we want the default mapping to be on FPR
     // instead of blind map every scalar to GPR.
-    if (anyUseOnlyUseFP(MI.getOperand(0).getReg(), MRI, TRI))
+    if (anyUseOnlyUseFP(MI.getOperand(0).getReg(), MRI, TRI)) {
       // If we have at least one direct use in a FP instruction,
       // assume this was a floating point load in the IR. If it was
       // not, we would have had a bitcast before reaching that
       // instruction.
-      OpdsMapping[0] = getFPValueMapping(Ty.getSizeInBits());
+      OpdsMapping[0] = getFPValueMapping(Size);
+      break;
+    }
 
     break;
   }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index ca2f868cd4e76..9e36d62352ae5 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -312,6 +312,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM8_GE32,
   OPERAND_UIMM9_LSB000,
   OPERAND_UIMM10_LSB00_NONZERO,
+  OPERAND_UIMM11,
   OPERAND_UIMM12,
   OPERAND_UIMM16,
   OPERAND_UIMM32,
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 0d818bc837fb7..b1990409754b0 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -113,6 +113,12 @@ class RISCVAsmPrinter : public AsmPrinter {
 
   void emitNTLHint(const MachineInstr *MI);
 
+  // XRay Support
+  void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI);
+  void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI);
+  void LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI);
+  void emitSled(const MachineInstr *MI, SledKind Kind);
+
   bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
 };
 }
@@ -316,6 +322,22 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
   case TargetOpcode::STATEPOINT:
     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
+  case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
+    // patchable-function-entry is handled in lowerToMCInst
+    // Therefore, we break out of the switch statement if we encounter it here.
+    const Function &F = MI->getParent()->getParent()->getFunction();
+    if (F.hasFnAttribute("patchable-function-entry"))
+      break;
+
+    LowerPATCHABLE_FUNCTION_ENTER(MI);
+    return;
+  }
+  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
+    LowerPATCHABLE_FUNCTION_EXIT(MI);
+    return;
+  case TargetOpcode::PATCHABLE_TAIL_CALL:
+    LowerPATCHABLE_TAIL_CALL(MI);
+    return;
   }
 
   MCInst OutInst;
@@ -453,11 +475,71 @@ bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   SetupMachineFunction(MF);
   emitFunctionBody();
 
+  // Emit the XRay table
+  emitXRayTable();
+
   if (EmittedOptionArch)
     RTS.emitDirectiveOptionPop();
   return false;
 }
 
+void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI) {
+  emitSled(MI, SledKind::FUNCTION_ENTER);
+}
+
+void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI) {
+  emitSled(MI, SledKind::FUNCTION_EXIT);
+}
+
+void RISCVAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI) {
+  emitSled(MI, SledKind::TAIL_CALL);
+}
+
+void RISCVAsmPrinter::emitSled(const MachineInstr *MI, SledKind Kind) {
+  // We want to emit the jump instruction and the nops constituting the sled.
+  // The format is as follows:
+  // .Lxray_sled_N
+  //   ALIGN
+  //   J .tmpN
+  //   21 or 33 C.NOP instructions
+  // .tmpN
+
+  // The following variable holds the count of the number of NOPs to be patched
+  // in for XRay instrumentation during compilation.
+  // Note that RV64 and RV32 each has a sled of 68 and 44 bytes, respectively.
+  // Assuming we're using JAL to jump to .tmpN, then we only need
+  // (68 - 4)/2 = 32 NOPs for RV64 and (44 - 4)/2 = 20 for RV32. However, there
+  // is a chance that we'll use C.JAL instead, so an additional NOP is needed.
+  const uint8_t NoopsInSledCount =
+      MI->getParent()->getParent()->getSubtarget<RISCVSubtarget>().is64Bit()
+          ? 33
+          : 21;
+
+  OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
+  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+  OutStreamer->emitLabel(CurSled);
+  auto Target = OutContext.createTempSymbol();
+
+  const MCExpr *TargetExpr = MCSymbolRefExpr::create(
+      Target, MCSymbolRefExpr::VariantKind::VK_None, OutContext);
+
+  // Emit "J bytes" instruction, which jumps over the nop sled to the actual
+  // start of function.
+  EmitToStreamer(
+      *OutStreamer,
+      MCInstBuilder(RISCV::JAL).addReg(RISCV::X0).addExpr(TargetExpr));
+
+  // Emit NOP instructions
+  for (int8_t I = 0; I < NoopsInSledCount; ++I)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
+                                     .addReg(RISCV::X0)
+                                     .addReg(RISCV::X0)
+                                     .addImm(0));
+
+  OutStreamer->emitLabel(Target);
+  recordSled(CurSled, *MI, Kind, 2);
+}
+
 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
   RISCVTargetStreamer &RTS =
       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
index 4b35f3bb0a524..47510ec6b0ea8 100644
--- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
+++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
@@ -37,8 +37,8 @@ class RISCVDeadRegisterDefinitions : public MachineFunctionPass {
     AU.addPreserved<LiveIntervalsWrapperPass>();
     AU.addRequired<LiveIntervalsWrapperPass>();
     AU.addPreserved<SlotIndexesWrapperPass>();
-    AU.addPreserved<LiveDebugVariables>();
-    AU.addPreserved<LiveStacks>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+    AU.addPreserved<LiveStacksWrapperLegacy>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 26e96cf831af7..52268c3fa62cc 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1341,6 +1341,32 @@ def HasVendorXwchc
       AssemblerPredicate<(all_of FeatureVendorXwchc),
                          "'Xwchc' (WCH/QingKe additional compressed opcodes)">;
 
+// Qualcomm Extension(s)
+
+def FeatureVendorXqcicsr
+    : RISCVExperimentalExtension<"xqcicsr", 0, 2,
+                                 "'Xqcicsr' (Qualcomm uC CSR Extension)">;
+def HasVendorXqcicsr
+    : Predicate<"Subtarget->hasVendorXqcicsr()">,
+      AssemblerPredicate<(all_of FeatureVendorXqcicsr),
+                         "'Xqcicsr' (Qualcomm uC CSR Extension)">;
+
+def FeatureVendorXqcisls
+    : RISCVExperimentalExtension<"xqcisls", 0, 2,
+                                 "'Xqcisls' (Qualcomm uC Scaled Load Store Extension)">;
+def HasVendorXqcisls
+    : Predicate<"Subtarget->hasVendorXqcisls()">,
+      AssemblerPredicate<(all_of FeatureVendorXqcisls),
+                         "'Xqcisls' (Qualcomm uC Scaled Load Store Extension)">;
+
+def FeatureVendorXqcia
+    : RISCVExperimentalExtension<"xqcia", 0, 2,
+                                 "'Xqcia' (Qualcomm uC Arithmetic Extension)">;
+def HasVendorXqcia
+    : Predicate<"Subtarget->hasVendorXqcia()">,
+      AssemblerPredicate<(all_of FeatureVendorXqcia),
+                         "'Xqcia' (Qualcomm uC Arithmetic Extension)">;
+
 //===----------------------------------------------------------------------===//
 // LLVM specific features and extensions
 //===----------------------------------------------------------------------===//
@@ -1388,10 +1414,14 @@ def FeatureUnalignedVectorMem
                       "true", "Has reasonably performant unaligned vector "
                       "loads and stores">;
 
-def FeaturePostRAScheduler : SubtargetFeature<"use-postra-scheduler",
+def TunePostRAScheduler : SubtargetFeature<"use-postra-scheduler",
     "UsePostRAScheduler", "true", "Schedule again after register allocation">;
 
-def FeaturePredictableSelectIsExpensive
+def TuneDisableLatencySchedHeuristic
+    : SubtargetFeature<"disable-latency-sched-heuristic", "DisableLatencySchedHeuristic", "true",
+                       "Disable latency scheduling heuristic">;
+
+def TunePredictableSelectIsExpensive
     : SubtargetFeature<"predictable-select-expensive", "PredictableSelectIsExpensive", "true",
                        "Prefer likely predicted branches over selects">;
 
@@ -1453,6 +1483,9 @@ def TuneSiFive7 : SubtargetFeature<"sifive7", "RISCVProcFamily", "SiFive7",
 def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron",
                                          "Ventana Veyron-Series processors">;
 
+def TuneVXRMPipelineFlush : SubtargetFeature<"vxrm-pipeline-flush", "HasVXRMPipelineFlush",
+                                             "true", "VXRM writes causes pipeline flush">;
+
 // Assume that lock-free native-width atomics are available, even if the target
 // and operating system combination would not usually provide them. The user
 // is responsible for providing any necessary __sync implementations. Code
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 2da32fece061b..1028149bf513f 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -14,6 +14,7 @@
 #include "RISCVMachineFunctionInfo.h"
 #include "RISCVSubtarget.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -151,7 +152,7 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
 
   Register SCSPReg = RISCVABI::getSCSPReg();
 
-  bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
+  bool IsRV64 = STI.is64Bit();
   int64_t SlotSize = STI.getXLen() / 8;
   // Store return address to shadow call stack
   // addi    gp, gp, [4|8]
@@ -215,7 +216,7 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
 
   Register SCSPReg = RISCVABI::getSCSPReg();
 
-  bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
+  bool IsRV64 = STI.is64Bit();
   int64_t SlotSize = STI.getXLen() / 8;
   // Load return address from shadow call stack
   // l[w|d]  ra, -[4|8](gp)
@@ -580,22 +581,124 @@ static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
                                         Comment.str());
 }
 
+// Allocate stack space and probe it if necessary.
 void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MBBI,
-                                       StackOffset Offset, bool EmitCFI,
-                                       unsigned CFIIndex) const {
+                                       MachineFunction &MF, uint64_t Offset,
+                                       uint64_t RealStackSize, bool EmitCFI,
+                                       bool NeedProbe,
+                                       uint64_t ProbeSize) const {
   DebugLoc DL;
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
   const RISCVInstrInfo *TII = STI.getInstrInfo();
 
-  RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, Offset, MachineInstr::FrameSetup,
+  // Simply allocate the stack if it's not big enough to require a probe.
+  if (!NeedProbe || Offset <= ProbeSize) {
+    RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Offset),
+                  MachineInstr::FrameSetup, getStackAlign());
+
+    if (EmitCFI) {
+      // Emit ".cfi_def_cfa_offset RealStackSize"
+      unsigned CFIIndex = MF.addFrameInst(
+          MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
+      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+          .addCFIIndex(CFIIndex)
+          .setMIFlag(MachineInstr::FrameSetup);
+    }
+
+    return;
+  }
+
+  // Unroll the probe loop depending on the number of iterations.
+  if (Offset < ProbeSize * 5) {
+    uint64_t CurrentOffset = 0;
+    bool IsRV64 = STI.is64Bit();
+    while (CurrentOffset + ProbeSize <= Offset) {
+      RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
+                    StackOffset::getFixed(-ProbeSize), MachineInstr::FrameSetup,
+                    getStackAlign());
+      // s[d|w] zero, 0(sp)
+      BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
+          .addReg(RISCV::X0)
+          .addReg(SPReg)
+          .addImm(0)
+          .setMIFlags(MachineInstr::FrameSetup);
+
+      CurrentOffset += ProbeSize;
+      if (EmitCFI) {
+        // Emit ".cfi_def_cfa_offset CurrentOffset"
+        unsigned CFIIndex = MF.addFrameInst(
+            MCCFIInstruction::cfiDefCfaOffset(nullptr, CurrentOffset));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex)
+            .setMIFlag(MachineInstr::FrameSetup);
+      }
+    }
+
+    uint64_t Residual = Offset - CurrentOffset;
+    if (Residual) {
+      RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
+                    StackOffset::getFixed(-Residual), MachineInstr::FrameSetup,
+                    getStackAlign());
+      if (EmitCFI) {
+        // Emit ".cfi_def_cfa_offset Offset"
+        unsigned CFIIndex =
+            MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex)
+            .setMIFlag(MachineInstr::FrameSetup);
+      }
+    }
+
+    return;
+  }
+
+  // Emit a variable-length allocation probing loop.
+  uint64_t RoundedSize = alignDown(Offset, ProbeSize);
+  uint64_t Residual = Offset - RoundedSize;
+
+  Register TargetReg = RISCV::X6;
+  // SUB TargetReg, SP, RoundedSize
+  RI->adjustReg(MBB, MBBI, DL, TargetReg, SPReg,
+                StackOffset::getFixed(-RoundedSize), MachineInstr::FrameSetup,
                 getStackAlign());
 
   if (EmitCFI) {
-    // Emit ".cfi_def_cfa_offset StackSize"
+    // Set the CFA register to TargetReg.
+    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(TargetReg, true);
+    unsigned CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, RoundedSize));
     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
+
+  // It will be expanded to a probe loop in `inlineStackProbe`.
+  BuildMI(MBB, MBBI, DL, TII->get(RISCV::PROBED_STACKALLOC))
+      .addReg(SPReg)
+      .addReg(TargetReg);
+
+  if (EmitCFI) {
+    // Set the CFA register back to SP.
+    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(SPReg, true);
+    unsigned CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
+
+  if (Residual)
+    RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Residual),
+                  MachineInstr::FrameSetup, getStackAlign());
+
+  if (EmitCFI) {
+    // Emit ".cfi_def_cfa_offset Offset"
+    unsigned CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameSetup);
   }
 }
 
@@ -713,13 +816,14 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                                           getPushOrLibCallsSavedInfo(MF, CSI));
   }
 
-  if (StackSize != 0) {
-    // Allocate space on the stack if necessary.
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
-    allocateStack(MBB, MBBI, StackOffset::getFixed(-StackSize),
-		  /*EmitCFI=*/ true, CFIIndex);
-  }
+  // Allocate space on the stack if necessary.
+  auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
+  const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
+  bool NeedProbe = TLI->hasInlineStackProbe(MF);
+  uint64_t ProbeSize = TLI->getStackProbeSize(MF, getStackAlign());
+  if (StackSize != 0)
+    allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
+                  NeedProbe, ProbeSize);
 
   // The frame pointer is callee-saved, and code has been generated for us to
   // save it to the stack. We need to skip over the storing of callee-saved
@@ -760,12 +864,9 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     assert(SecondSPAdjustAmount > 0 &&
            "SecondSPAdjustAmount should be greater than zero");
 
-    // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0",
-    // don't emit an sp-based .cfi_def_cfa_offset
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(
-        nullptr, getStackSizeWithRVVPadding(MF)));
-    allocateStack(MBB, MBBI, StackOffset::getFixed(-SecondSPAdjustAmount),
-                  !hasFP(MF), CFIIndex);
+    allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
+                  getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
+                  ProbeSize);
   }
 
   if (RVVStackSize) {
@@ -1913,3 +2014,71 @@ bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
 TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
   return TargetStackID::ScalableVector;
 }
+
+// Synthesize the probe loop.
+static void emitStackProbeInline(MachineFunction &MF, MachineBasicBlock &MBB,
+                                 MachineBasicBlock::iterator MBBI,
+                                 DebugLoc DL) {
+
+  auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
+  const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
+  bool IsRV64 = Subtarget.is64Bit();
+  Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
+  const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
+  uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
+
+  MachineFunction::iterator MBBInsertPoint = std::next(MBB.getIterator());
+  MachineBasicBlock *LoopTestMBB =
+      MF.CreateMachineBasicBlock(MBB.getBasicBlock());
+  MF.insert(MBBInsertPoint, LoopTestMBB);
+  MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
+  MF.insert(MBBInsertPoint, ExitMBB);
+  MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
+  Register TargetReg = RISCV::X6;
+  Register ScratchReg = RISCV::X7;
+
+  // ScratchReg = ProbeSize
+  TII->movImm(MBB, MBBI, DL, ScratchReg, ProbeSize, Flags);
+
+  // LoopTest:
+  //   SUB SP, SP, ProbeSize
+  BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::SUB), SPReg)
+      .addReg(SPReg)
+      .addReg(ScratchReg)
+      .setMIFlags(Flags);
+
+  //   s[d|w] zero, 0(sp)
+  BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL,
+          TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
+      .addReg(RISCV::X0)
+      .addReg(SPReg)
+      .addImm(0)
+      .setMIFlags(Flags);
+
+  //   BNE SP, TargetReg, LoopTest
+  BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::BNE))
+      .addReg(SPReg)
+      .addReg(TargetReg)
+      .addMBB(LoopTestMBB)
+      .setMIFlags(Flags);
+
+  ExitMBB->splice(ExitMBB->end(), &MBB, std::next(MBBI), MBB.end());
+
+  LoopTestMBB->addSuccessor(ExitMBB);
+  LoopTestMBB->addSuccessor(LoopTestMBB);
+  MBB.addSuccessor(LoopTestMBB);
+  // Update liveins.
+  fullyRecomputeLiveIns({ExitMBB, LoopTestMBB});
+}
+
+void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
+                                          MachineBasicBlock &MBB) const {
+  auto Where = llvm::find_if(MBB, [](MachineInstr &MI) {
+    return MI.getOpcode() == RISCV::PROBED_STACKALLOC;
+  });
+  if (Where != MBB.end()) {
+    DebugLoc DL = MBB.findDebugLoc(Where);
+    emitStackProbeInline(MF, MBB, Where, DL);
+    Where->eraseFromParent();
+  }
+}
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
index 84a8fbd117a22..190c063d9d3b5 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
@@ -79,7 +79,9 @@ class RISCVFrameLowering : public TargetFrameLowering {
   }
 
   void allocateStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                     StackOffset Offset, bool EmitCFI, unsigned CFIIndex) const;
+                     MachineFunction &MF, uint64_t Offset,
+                     uint64_t RealStackSize, bool EmitCFI, bool NeedProbe,
+                     uint64_t ProbeSize) const;
 
 protected:
   const RISCVSubtarget &STI;
@@ -102,6 +104,9 @@ class RISCVFrameLowering : public TargetFrameLowering {
 
   std::pair<int64_t, Align>
   assignRVVStackObjectOffsets(MachineFunction &MF) const;
+  // Replace a StackProbe stub (if any) with the actual probe code inline
+  void inlineStackProbe(MachineFunction &MF,
+                        MachineBasicBlock &PrologueMBB) const override;
 };
 } // namespace llvm
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td
index 9670e4b07068a..7cb419b610e60 100644
--- a/llvm/lib/Target/RISCV/RISCVGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVGISel.td
@@ -76,10 +76,10 @@ def gi_trailing_zero : GICustomOperandRenderer<"renderTrailingZeros">,
 // should add a LowLevelTypeByHwMode, and use that to define our XLenLLT instead
 // here.
 def GIShiftMaskXLen :
-    GIComplexOperandMatcher<s32, "selectShiftMask">,
+    GIComplexOperandMatcher<s32, "selectShiftMaskXLen">,
     GIComplexPatternEquiv<shiftMaskXLen>;
 def GIShiftMask32 :
-    GIComplexOperandMatcher<s32, "selectShiftMask">,
+    GIComplexOperandMatcher<s64, "selectShiftMask32">,
     GIComplexPatternEquiv<shiftMask32>;
 
 def gi_sh1add_op : GIComplexOperandMatcher<s32, "selectSHXADDOp<1>">,
@@ -223,17 +223,23 @@ let Predicates = [IsRV32, NoStdExtZbb, NoStdExtZbkb] in
 def : Pat<(XLenVT (zext (i16 GPR:$src))),
           (SRLI (XLenVT (SLLI GPR:$src, 16)), 16)>;
 
-let Predicates = [IsRV64, NoStdExtZbb, NoStdExtZbkb] in
+let Predicates = [IsRV64, NoStdExtZbb, NoStdExtZbkb] in {
 def : Pat<(i64 (zext (i16 GPR:$src))),
           (SRLI (XLenVT (SLLI GPR:$src, 48)), 48)>;
+def : Pat<(i32 (zext (i16 GPR:$src))),
+          (SRLI (XLenVT (SLLI GPR:$src, 48)), 48)>;
+}
 
 let Predicates = [IsRV32, NoStdExtZbb] in
 def : Pat<(XLenVT (sext (i16 GPR:$src))),
           (SRAI (XLenVT (SLLI GPR:$src, 16)), 16)>;
 
-let Predicates = [IsRV64, NoStdExtZbb] in
+let Predicates = [IsRV64, NoStdExtZbb] in {
 def : Pat<(i64 (sext (i16 GPR:$src))),
           (SRAI (XLenVT (SLLI GPR:$src, 48)), 48)>;
+def : Pat<(i32 (sext (i16 GPR:$src))),
+          (SRAI (XLenVT (SLLI GPR:$src, 48)), 48)>;
+}
 
 //===----------------------------------------------------------------------===//
 // Zb* RV64 patterns not used by SelectionDAG.
@@ -243,15 +249,21 @@ let Predicates = [HasStdExtZba, IsRV64] in {
 def : Pat<(zext (i32 GPR:$src)), (ADD_UW GPR:$src, (XLenVT X0))>;
 }
 
-let Predicates= [HasStdExtZbb] in
-def : Pat<(XLenVT (sext (i16 GPR:$rs))), (SEXT_H GPR:$rs)>;
+let Predicates = [HasStdExtZbb] in
+def : Pat<(i32 (sext (i16 GPR:$rs))), (SEXT_H GPR:$rs)>;
+let Predicates = [HasStdExtZbb, IsRV64] in
+def : Pat<(i64 (sext (i16 GPR:$rs))), (SEXT_H GPR:$rs)>;
 
 let Predicates = [HasStdExtZbb, IsRV32] in
 def : Pat<(i32 (zext (i16 GPR:$rs))), (ZEXT_H_RV32 GPR:$rs)>;
-let Predicates = [HasStdExtZbb, IsRV64] in
+let Predicates = [HasStdExtZbb, IsRV64] in {
 def : Pat<(i64 (zext (i16 GPR:$rs))), (ZEXT_H_RV64 GPR:$rs)>;
+def : Pat<(i32 (zext (i16 GPR:$rs))), (ZEXT_H_RV64 GPR:$rs)>;
+}
 
 let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV32] in
 def : Pat<(i32 (zext (i16 GPR:$rs))), (PACK GPR:$rs, (XLenVT X0))>;
-let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in
+let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in {
 def : Pat<(i64 (zext (i16 GPR:$rs))), (PACKW GPR:$rs, (XLenVT X0))>;
+def : Pat<(i32 (zext (i16 GPR:$rs))), (PACKW GPR:$rs, (XLenVT X0))>;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 329b42d621cee..c683857363720 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1544,6 +1544,20 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   // corresponding branch. This information is used in CGP/SelectOpt to decide
   // when to convert selects into branches.
   PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
+
+  MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
+  MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
+
+  MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
+  MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
+  MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
+
+  MaxStoresPerMemmoveOptSize =
+      Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
+  MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
+
+  MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
+  MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
@@ -2093,7 +2107,7 @@ bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
 
   // Assume target opcodes can't be scalarized.
   // TODO - do we have any exceptions?
-  if (Opc >= ISD::BUILTIN_OP_END)
+  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
     return false;
 
   // If the vector op is not supported, try to convert to scalar.
@@ -3496,21 +3510,30 @@ static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
   if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
     return SDValue();
   SDValue Vec = SplatVal.getOperand(0);
-  // Only perform this optimization on vectors of the same size for simplicity.
-  // Don't perform this optimization for i1 vectors.
+  // Don't perform this optimization for i1 vectors, or if the element types are
+  // different
   // FIXME: Support i1 vectors, maybe by promoting to i8?
-  if (Vec.getValueType() != VT || VT.getVectorElementType() == MVT::i1)
+  MVT EltTy = VT.getVectorElementType();
+  if (EltTy == MVT::i1 ||
+      EltTy != Vec.getSimpleValueType().getVectorElementType())
     return SDValue();
   SDValue Idx = SplatVal.getOperand(1);
   // The index must be a legal type.
   if (Idx.getValueType() != Subtarget.getXLenVT())
     return SDValue();
 
+  // Check that Index lies within VT
+  // TODO: Can we check if the Index is constant and known in-bounds?
+  if (!TypeSize::isKnownLE(Vec.getValueSizeInBits(), VT.getSizeInBits()))
+    return SDValue();
+
   MVT ContainerVT = VT;
-  if (VT.isFixedLengthVector()) {
+  if (VT.isFixedLengthVector())
     ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
-    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
-  }
+
+  Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
+                    DAG.getUNDEF(ContainerVT), Vec,
+                    DAG.getVectorIdxConstant(0, DL));
 
   auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
@@ -3523,7 +3546,6 @@ static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
 }
 
-
 /// Try and optimize BUILD_VECTORs with "dominant values" - these are values
 /// which constitute a large proportion of the elements. In such cases we can
 /// splat a vector with the dominant element and make up the shortfall with
@@ -4440,40 +4462,15 @@ static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
   // a zero extend and the simm5 check in isel would fail.
   // FIXME: Should we ignore the upper bits in isel instead?
   unsigned ExtOpc =
-    isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
+      isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
   Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
-  return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT,
-                     DAG.getUNDEF(VT), Scalar, VL);
-}
-
-// Is this a shuffle extracts either the even or odd elements of a vector?
-// That is, specifically, either (a) or (b) in the options below.
-// Single operand shuffle is easy:
-//   a) t35: v8i8 = vector_shuffle<0,2,4,6,u,u,u,u> t34, undef
-//   b) t35: v8i8 = vector_shuffle<1,3,5,7,u,u,u,u> t34, undef
-// Double operand shuffle:
-//   t34: v8i8 = extract_subvector t11, Constant:i64<0>
-//   t33: v8i8 = extract_subvector t11, Constant:i64<8>
-//   a) t35: v8i8 = vector_shuffle<0,2,4,6,8,10,12,14> t34, t33
-//   b) t35: v8i8 = vector_shuffle<1,3,5,7,9,11,13,15> t34, t33
-static SDValue isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1,
-                                     SDValue V2, ArrayRef<int> Mask,
-                                     const RISCVSubtarget &Subtarget) {
-  // Need to be able to widen the vector.
-  if (VT.getScalarSizeInBits() >= Subtarget.getELen())
-    return SDValue();
-
-  // First index must be the first even or odd element from V1.
-  if (Mask[0] != 0 && Mask[0] != 1)
-    return SDValue();
-
-  // The others must increase by 2 each time.
-  for (unsigned i = 1; i != Mask.size(); ++i)
-    if (Mask[i] != -1 && Mask[i] != Mask[0] + (int)i * 2)
-      return SDValue();
+  return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
+                     VL);
+}
 
-  if (1 == count_if(Mask, [](int Idx) { return Idx != -1; }))
-    return SDValue();
+// Can this shuffle be performed on exactly one (possibly larger) input?
+static SDValue getSingleShuffleSrc(MVT VT, MVT ContainerVT, SDValue V1,
+                                   SDValue V2) {
 
   if (V2.isUndef() &&
       RISCVTargetLowering::getLMUL(ContainerVT) != RISCVII::VLMUL::LMUL_8)
@@ -4490,12 +4487,13 @@ static SDValue isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1,
     return SDValue();
 
   // Src needs to have twice the number of elements.
-  if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
+  unsigned NumElts = VT.getVectorNumElements();
+  if (Src.getValueType().getVectorNumElements() != (NumElts * 2))
     return SDValue();
 
   // The extracts must extract the two halves of the source.
   if (V1.getConstantOperandVal(1) != 0 ||
-      V2.getConstantOperandVal(1) != Mask.size())
+      V2.getConstantOperandVal(1) != NumElts)
     return SDValue();
 
   return Src;
@@ -4612,57 +4610,30 @@ static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef<int> Mask) {
   return Rotation;
 }
 
-// Lower a deinterleave shuffle to vnsrl.
-// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (EvenElts == true)
-//                          -> [p, q, r, s] (EvenElts == false)
-// VT is the type of the vector to return, <[vscale x ]n x ty>
-// Src is the vector to deinterleave of type <[vscale x ]n*2 x ty>
-static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src,
-                                       bool EvenElts,
-                                       const RISCVSubtarget &Subtarget,
-                                       SelectionDAG &DAG) {
-  // The result is a vector of type <m x n x ty>
-  MVT ContainerVT = VT;
-  // Convert fixed vectors to scalable if needed
-  if (ContainerVT.isFixedLengthVector()) {
-    assert(Src.getSimpleValueType().isFixedLengthVector());
-    ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
-
-    // The source is a vector of type <m x n*2 x ty> (For the single source
-    // case, the high half is undefined)
-    MVT SrcContainerVT =
-        MVT::getVectorVT(ContainerVT.getVectorElementType(),
-                         ContainerVT.getVectorElementCount() * 2);
-    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
-  }
-
-  auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-
-  // Bitcast the source vector from <m x n*2 x ty> -> <m x n x ty*2>
-  // This also converts FP to int.
-  unsigned EltBits = ContainerVT.getScalarSizeInBits();
-  MVT WideSrcContainerVT = MVT::getVectorVT(
-      MVT::getIntegerVT(EltBits * 2), ContainerVT.getVectorElementCount());
-  Src = DAG.getBitcast(WideSrcContainerVT, Src);
-
-  // The integer version of the container type.
-  MVT IntContainerVT = ContainerVT.changeVectorElementTypeToInteger();
-
-  // If we want even elements, then the shift amount is 0. Otherwise, shift by
-  // the original element size.
-  unsigned Shift = EvenElts ? 0 : EltBits;
-  SDValue SplatShift = DAG.getNode(
-      RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
-      DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL);
-  SDValue Res =
-      DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
-                  DAG.getUNDEF(IntContainerVT), TrueMask, VL);
-  // Cast back to FP if needed.
-  Res = DAG.getBitcast(ContainerVT, Res);
-
-  if (VT.isFixedLengthVector())
-    Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
-  return Res;
+// Lower a deinterleave shuffle to SRL and TRUNC.  Factor must be
+// 2, 4, 8 and the integer type Factor-times larger than VT's
+// element type must be a legal element type.
+// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
+//                          -> [p, q, r, s] (Factor=2, Index=1)
+static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
+                                            SDValue Src, unsigned Factor,
+                                            unsigned Index, SelectionDAG &DAG) {
+  unsigned EltBits = VT.getScalarSizeInBits();
+  ElementCount SrcEC = Src.getValueType().getVectorElementCount();
+  MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
+                                   SrcEC.divideCoefficientBy(Factor));
+  MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
+                               SrcEC.divideCoefficientBy(Factor));
+  Src = DAG.getBitcast(WideSrcVT, Src);
+
+  unsigned Shift = Index * EltBits;
+  SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
+                            DAG.getConstant(Shift, DL, WideSrcVT));
+  Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
+  MVT IntVT = VT.changeVectorElementTypeToInteger();
+  Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, IntVT, DAG.getUNDEF(IntVT), Res,
+                    DAG.getVectorIdxConstant(0, DL));
+  return DAG.getBitcast(VT, Res);
 }
 
 // Lower the following shuffle to vslidedown.
@@ -4867,12 +4838,76 @@ static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
   return convertFromScalableVector(VT, Vec, DAG, Subtarget);
 }
 
+// Match a mask which "spreads" the leading elements of a vector evenly
+// across the result.  Factor is the spread amount, and Index is the
+// offset applied.  (on success, Index < Factor)  This is the inverse
+// of a deinterleave with the same Factor and Index.  This is analogous
+// to an interleave, except that all but one lane is undef.
+static bool isSpreadMask(ArrayRef<int> Mask, unsigned Factor, unsigned &Index) {
+  SmallVector<bool> LaneIsUndef(Factor, true);
+  for (unsigned i = 0; i < Mask.size(); i++)
+    LaneIsUndef[i % Factor] &= (Mask[i] == -1);
+
+  bool Found = false;
+  for (unsigned i = 0; i < Factor; i++) {
+    if (LaneIsUndef[i])
+      continue;
+    if (Found)
+      return false;
+    Index = i;
+    Found = true;
+  }
+  if (!Found)
+    return false;
+
+  for (unsigned i = 0; i < Mask.size() / Factor; i++) {
+    unsigned j = i * Factor + Index;
+    if (Mask[j] != -1 && (unsigned)Mask[j] != i)
+      return false;
+  }
+  return true;
+}
+
+// Given a vector a, b, c, d return a vector Factor times longer
+// with Factor-1 undef's between elements. Ex:
+//   a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
+//   undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
+static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
+                                 const SDLoc &DL, SelectionDAG &DAG) {
+
+  MVT VT = V.getSimpleValueType();
+  unsigned EltBits = VT.getScalarSizeInBits();
+  ElementCount EC = VT.getVectorElementCount();
+  V = DAG.getBitcast(VT.changeTypeToInteger(), V);
+
+  MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
+
+  SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
+  // TODO: On rv32, the constant becomes a splat_vector_parts which does not
+  // allow the SHL to fold away if Index is 0.
+  if (Index != 0)
+    Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
+                         DAG.getConstant(EltBits * Index, DL, WideVT));
+  // Make sure to use original element type
+  MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
+                                  EC.multiplyCoefficientBy(Factor));
+  return DAG.getBitcast(ResultVT, Result);
+}
+
 // Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
 // to create an interleaved vector of <[vscale x] n*2 x ty>.
 // This requires that the size of ty is less than the subtarget's maximum ELEN.
 static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
                                      const SDLoc &DL, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
+
+  // FIXME: Not only does this optimize the code, it fixes some correctness
+  // issues because MIR does not have freeze.
+  if (EvenV.isUndef())
+    return getWideningSpread(OddV, 2, 1, DL, DAG);
+  if (OddV.isUndef())
+    return getWideningSpread(EvenV, 2, 0, DL, DAG);
+
   MVT VecVT = EvenV.getSimpleValueType();
   MVT VecContainerVT = VecVT; // <vscale x n x ty>
   // Convert fixed vectors to scalable if needed
@@ -4904,29 +4939,14 @@ static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
   SDValue Passthru = DAG.getUNDEF(WideContainerVT);
 
   SDValue Interleaved;
-  if (OddV.isUndef()) {
-    // If OddV is undef, this is a zero extend.
-    // FIXME: Not only does this optimize the code, it fixes some correctness
-    // issues because MIR does not have freeze.
-    Interleaved =
-        DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, EvenV, Mask, VL);
-  } else if (Subtarget.hasStdExtZvbb()) {
+  if (Subtarget.hasStdExtZvbb()) {
     // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
     SDValue OffsetVec =
         DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
     Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
                               OffsetVec, Passthru, Mask, VL);
-    if (!EvenV.isUndef())
-      Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
-                                Interleaved, EvenV, Passthru, Mask, VL);
-  } else if (EvenV.isUndef()) {
-    Interleaved =
-        DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, OddV, Mask, VL);
-
-    SDValue OffsetVec =
-        DAG.getConstant(VecVT.getScalarSizeInBits(), DL, WideContainerVT);
-    Interleaved = DAG.getNode(RISCVISD::SHL_VL, DL, WideContainerVT,
-                              Interleaved, OffsetVec, Passthru, Mask, VL);
+    Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
+                              Interleaved, EvenV, Passthru, Mask, VL);
   } else {
     // FIXME: We should freeze the odd vector here. We already handled the case
     // of provably undef/poison above.
@@ -5155,6 +5175,28 @@ static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
   return convertFromScalableVector(VT, Vec, DAG, Subtarget);
 }
 
+// Matches a subset of compress masks with a contiguous prefix of output
+// elements.  This could be extended to allow gaps by deciding which
+// source elements to spuriously demand.
+static bool isCompressMask(ArrayRef<int> Mask) {
+  int Last = -1;
+  bool SawUndef = false;
+  for (unsigned i = 0; i < Mask.size(); i++) {
+    if (Mask[i] == -1) {
+      SawUndef = true;
+      continue;
+    }
+    if (SawUndef)
+      return false;
+    if (i > (unsigned)Mask[i])
+      return false;
+    if (Mask[i] <= Last)
+      return false;
+    Last = Mask[i];
+  }
+  return true;
+}
+
 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
   SDValue V1 = Op.getOperand(0);
@@ -5330,11 +5372,24 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
   if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef())
     return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
 
-  // If this is a deinterleave and we can widen the vector, then we can use
-  // vnsrl to deinterleave.
-  if (SDValue Src =
-          isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget))
-    return getDeinterleaveViaVNSRL(DL, VT, Src, Mask[0] == 0, Subtarget, DAG);
+  // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
+  // use shift and truncate to perform the shuffle.
+  // TODO: For Factor=6, we can perform the first step of the deinterleave via
+  // shift-and-trunc reducing total cost for everything except an mf8 result.
+  // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
+  // to do the entire operation.
+  if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
+    const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
+    assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
+    for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
+      unsigned Index = 0;
+      if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
+          1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
+        if (SDValue Src = getSingleShuffleSrc(VT, ContainerVT, V1, V2))
+          return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
+      }
+    }
+  }
 
   if (SDValue V =
           lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
@@ -5347,17 +5402,32 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     // Extract the halves of the vectors.
     MVT HalfVT = VT.getHalfNumVectorElementsVT();
 
+    // Recognize if one half is actually undef; the matching above will
+    // otherwise reuse the even stream for the undef one.  This improves
+    // spread(2) shuffles.
+    bool LaneIsUndef[2] = { true, true};
+    for (unsigned i = 0; i < Mask.size(); i++)
+      LaneIsUndef[i % 2] &= (Mask[i] == -1);
+
     int Size = Mask.size();
     SDValue EvenV, OddV;
-    assert(EvenSrc >= 0 && "Undef source?");
-    EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
-    EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
-                        DAG.getVectorIdxConstant(EvenSrc % Size, DL));
+    if (LaneIsUndef[0]) {
+      EvenV = DAG.getUNDEF(HalfVT);
+    } else {
+      assert(EvenSrc >= 0 && "Undef source?");
+      EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
+      EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
+                          DAG.getVectorIdxConstant(EvenSrc % Size, DL));
+    }
 
-    assert(OddSrc >= 0 && "Undef source?");
-    OddV = (OddSrc / Size) == 0 ? V1 : V2;
-    OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
-                       DAG.getVectorIdxConstant(OddSrc % Size, DL));
+    if (LaneIsUndef[1]) {
+      OddV = DAG.getUNDEF(HalfVT);
+    } else {
+      assert(OddSrc >= 0 && "Undef source?");
+      OddV = (OddSrc / Size) == 0 ? V1 : V2;
+      OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
+                         DAG.getVectorIdxConstant(OddSrc % Size, DL));
+    }
 
     return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
   }
@@ -5372,6 +5442,42 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
       return V;
 
+    // Can we generate a vcompress instead of a vrgather?  These scale better
+    // at high LMUL, at the cost of not being able to fold a following select
+    // into them.  The mask constants are also smaller than the index vector
+    // constants, and thus easier to materialize.
+    if (isCompressMask(Mask)) {
+      SmallVector<SDValue> MaskVals(NumElts,
+                                    DAG.getConstant(false, DL, XLenVT));
+      for (auto Idx : Mask) {
+        if (Idx == -1)
+          break;
+        assert(Idx >= 0 && (unsigned)Idx < NumElts);
+        MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
+      }
+      MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
+      SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
+      return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
+                         DAG.getUNDEF(VT));
+    }
+
+    // Match a spread(4,8) which can be done via extend and shift.  Spread(2)
+    // is fully covered in interleave(2) above, so it is ignored here.
+    if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
+      unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
+      assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
+      for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
+        unsigned Index;
+        if (isSpreadMask(Mask, Factor, Index)) {
+          MVT NarrowVT =
+              MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
+          SDValue Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, V1,
+                                    DAG.getVectorIdxConstant(0, DL));
+          return getWideningSpread(Src, Factor, Index, DL, DAG);
+        }
+      }
+    }
+
     if (VT.getScalarSizeInBits() == 8 &&
         any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
       // On such a vector we're unable to use i8 as the index type.
@@ -6217,7 +6323,7 @@ static bool hasPassthruOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
-                    128 &&
+                    127 &&
                 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
                         ISD::FIRST_TARGET_STRICTFP_OPCODE ==
                     21 &&
@@ -6243,7 +6349,7 @@ static bool hasMaskOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
-                    128 &&
+                    127 &&
                 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
                         ISD::FIRST_TARGET_STRICTFP_OPCODE ==
                     21 &&
@@ -10715,34 +10821,37 @@ SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
                                Op.getOperand(0), Op.getOperand(1));
 
-  // We want to operate on all lanes, so get the mask and VL and mask for it
-  auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget);
-  SDValue Passthru = DAG.getUNDEF(ConcatVT);
-
   // We can deinterleave through vnsrl.wi if the element type is smaller than
   // ELEN
   if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
-    SDValue Even =
-        getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG);
-    SDValue Odd =
-        getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG);
+    SDValue Even = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 0, DAG);
+    SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 1, DAG);
     return DAG.getMergeValues({Even, Odd}, DL);
   }
 
   // For the indices, use the same SEW to avoid an extra vsetvli
+  // TODO: If container type is larger than m1, we can consider using a splat
+  // of a constant instead of the following sequence
+
+  // Create a vector of even indices {0, 1, 2, ...}
   MVT IdxVT = ConcatVT.changeVectorElementTypeToInteger();
-  // Create a vector of even indices {0, 2, 4, ...}
-  SDValue EvenIdx =
-      DAG.getStepVector(DL, IdxVT, APInt(IdxVT.getScalarSizeInBits(), 2));
-  // Create a vector of odd indices {1, 3, 5, ... }
-  SDValue OddIdx =
-      DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT));
-
-  // Gather the even and odd elements into two separate vectors
-  SDValue EvenWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
-                                 Concat, EvenIdx, Passthru, Mask, VL);
-  SDValue OddWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
-                                Concat, OddIdx, Passthru, Mask, VL);
+  SDValue StepVec = DAG.getStepVector(DL, IdxVT);
+  // 0, 1, 0, 1, 0, 1
+  SDValue ZeroOnes =
+      DAG.getNode(ISD::AND, DL, IdxVT, StepVec, DAG.getConstant(1, DL, IdxVT));
+  MVT MaskVT = ConcatVT.changeVectorElementType(MVT::i1);
+  SDValue EvenMask =
+      DAG.getSetCC(DL, MaskVT, ZeroOnes, DAG.getConstant(0, DL, IdxVT),
+                   ISD::CondCode::SETEQ);
+  // Have the latter be the not of the former to minimize the live range of
+  // the index vector since that might be large.
+  SDValue OddMask = DAG.getLogicalNOT(DL, EvenMask, MaskVT);
+
+  // vcompress the even and odd elements into two separate vectors
+  SDValue EvenWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
+                                 EvenMask, DAG.getUNDEF(ConcatVT));
+  SDValue OddWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
+                                OddMask, DAG.getUNDEF(ConcatVT));
 
   // Extract the result half of the gather for even and odd
   SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
@@ -18054,6 +18163,20 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     SDValue N0 = N->getOperand(0);
     EVT VT = N->getValueType(0);
     EVT SrcVT = N0.getValueType();
+    if (VT.isRISCVVectorTuple() && N0->getOpcode() == ISD::SPLAT_VECTOR) {
+      unsigned NF = VT.getRISCVVectorTupleNumFields();
+      unsigned NumScalElts = VT.getSizeInBits().getKnownMinValue() / (NF * 8);
+      SDValue EltVal = DAG.getConstant(0, DL, Subtarget.getXLenVT());
+      MVT ScalTy = MVT::getScalableVectorVT(MVT::getIntegerVT(8), NumScalElts);
+
+      SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalTy, EltVal);
+
+      SDValue Result = DAG.getUNDEF(VT);
+      for (unsigned i = 0; i < NF; ++i)
+        Result = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VT, Result, Splat,
+                             DAG.getVectorIdxConstant(i, DL));
+      return Result;
+    }
     // If this is a bitcast between a MVT::v4i1/v2i1/v1i1 and an illegal integer
     // type, widen both sides to avoid a trip through memory.
     if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
@@ -18110,8 +18233,46 @@ bool RISCVTargetLowering::isDesirableToCommuteWithShift(
   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
   SDValue N0 = N->getOperand(0);
   EVT Ty = N0.getValueType();
+
+  // LD/ST will optimize constant Offset extraction, so when AddNode is used by
+  // LD/ST, it can still complete the folding optimization operation performed
+  // above.
+  auto isUsedByLdSt = [&]() {
+    bool CanOptAlways = false;
+    if (N0->getOpcode() == ISD::ADD && !N0->hasOneUse()) {
+      for (SDNode *Use : N0->uses()) {
+        // This use is the one we're on right now. Skip it
+        if (Use == N || Use->getOpcode() == ISD::SELECT)
+          continue;
+        if (!isa<StoreSDNode>(Use) && !isa<LoadSDNode>(Use)) {
+          CanOptAlways = false;
+          break;
+        }
+        CanOptAlways = true;
+      }
+    }
+
+    if (N0->getOpcode() == ISD::SIGN_EXTEND &&
+        !N0->getOperand(0)->hasOneUse()) {
+      for (SDNode *Use : N0->getOperand(0)->uses()) {
+        // This use is the one we're on right now. Skip it
+        if (Use == N0.getNode() || Use->getOpcode() == ISD::SELECT)
+          continue;
+        if (!isa<StoreSDNode>(Use) && !isa<LoadSDNode>(Use)) {
+          CanOptAlways = false;
+          break;
+        }
+        CanOptAlways = true;
+      }
+    }
+    return CanOptAlways;
+  };
+
   if (Ty.isScalarInteger() &&
       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
+    if (N0.getOpcode() == ISD::ADD && !N0->hasOneUse())
+      return isUsedByLdSt();
+
     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
     if (C1 && C2) {
@@ -18146,6 +18307,15 @@ bool RISCVTargetLowering::isDesirableToCommuteWithShift(
         return false;
     }
   }
+
+  if (!N0->hasOneUse())
+    return false;
+
+  if (N0->getOpcode() == ISD::SIGN_EXTEND &&
+      N0->getOperand(0)->getOpcode() == ISD::ADD &&
+      !N0->getOperand(0)->hasOneUse())
+    return isUsedByLdSt();
+
   return true;
 }
 
@@ -19953,8 +20123,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   // reserved, if so report an error. Do the same for the return address if this
   // is not a tailcall.
   validateCCReservedRegs(RegsToPass, MF);
-  if (!IsTailCall &&
-      MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
+  if (!IsTailCall && MF.getSubtarget().isRegisterReservedByUser(RISCV::X1))
     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
         MF.getFunction(),
         "Return address register required, but has been reserved."});
@@ -20453,7 +20622,6 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(VWMACC_VL)
   NODE_NAME_CASE(VWMACCU_VL)
   NODE_NAME_CASE(VWMACCSU_VL)
-  NODE_NAME_CASE(VNSRL_VL)
   NODE_NAME_CASE(SETCC_VL)
   NODE_NAME_CASE(VMERGE_VL)
   NODE_NAME_CASE(VMAND_VL)
@@ -21280,8 +21448,9 @@ bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
   return true;
 }
 
-bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
-  if (Subtarget.is64Bit() && Type == MVT::i32)
+bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
+                                                        bool IsSigned) const {
+  if (Subtarget.is64Bit() && Ty->isIntegerTy(32))
     return true;
 
   return IsSigned;
@@ -21784,6 +21953,7 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
                                                 ShuffleVectorInst *SVI,
                                                 unsigned Factor) const {
   IRBuilder<> Builder(SI);
+  auto Mask = SVI->getShuffleMask();
   auto *ShuffleVTy = cast<FixedVectorType>(SVI->getType());
   // Given SVI : <n*factor x ty>, then VTy : <n x ty>
   auto *VTy = FixedVectorType::get(ShuffleVTy->getElementType(),
@@ -21795,11 +21965,35 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
 
   auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
 
+  unsigned Index;
+  // If the segment store only has one active lane (i.e. the interleave is
+  // just a spread shuffle), we can use a strided store instead.  This will
+  // be equally fast, and create less vector register pressure.
+  if (!Subtarget.hasOptimizedSegmentLoadStore(Factor) &&
+      isSpreadMask(Mask, Factor, Index)) {
+    unsigned ScalarSizeInBytes = ShuffleVTy->getScalarSizeInBits() / 8;
+    Value *Data = SVI->getOperand(0);
+    auto *DataVTy = cast<FixedVectorType>(Data->getType());
+    Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
+    Value *Offset = ConstantInt::get(XLenTy, Index * ScalarSizeInBytes);
+    Value *BasePtr = Builder.CreatePtrAdd(SI->getPointerOperand(), Offset);
+    Value *Mask = Builder.getAllOnesMask(DataVTy->getElementCount());
+    Value *VL = Builder.getInt32(VTy->getNumElements());
+
+    CallInst *CI = Builder.CreateIntrinsic(
+        Intrinsic::experimental_vp_strided_store,
+        {Data->getType(), BasePtr->getType(), Stride->getType()},
+        {Data, BasePtr, Stride, Mask, VL});
+    CI->addParamAttr(
+        1, Attribute::getWithAlignment(CI->getContext(), SI->getAlign()));
+
+    return true;
+  }
+
   Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
       SI->getModule(), FixedVssegIntrIds[Factor - 2],
       {VTy, SI->getPointerOperandType(), XLenTy});
 
-  auto Mask = SVI->getShuffleMask();
   SmallVector<Value *, 10> Ops;
 
   for (unsigned i = 0; i < Factor; i++) {
@@ -22156,3 +22350,25 @@ namespace llvm::RISCVVIntrinsicsTable {
 #include "RISCVGenSearchableTables.inc"
 
 } // namespace llvm::RISCVVIntrinsicsTable
+
+bool RISCVTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
+
+  // If the function specifically requests inline stack probes, emit them.
+  if (MF.getFunction().hasFnAttribute("probe-stack"))
+    return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
+           "inline-asm";
+
+  return false;
+}
+
+unsigned RISCVTargetLowering::getStackProbeSize(const MachineFunction &MF,
+                                                Align StackAlign) const {
+  // The default stack probe size is 4096 if the function has no
+  // stack-probe-size attribute.
+  const Function &Fn = MF.getFunction();
+  unsigned StackProbeSize =
+      Fn.getFnAttributeAsParsedInteger("stack-probe-size", 4096);
+  // Round down to the stack alignment.
+  StackProbeSize = alignDown(StackProbeSize, StackAlign.value());
+  return StackProbeSize ? StackProbeSize : StackAlign.value();
+}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 7ada941563c1f..778e38a1a834e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -369,10 +369,6 @@ enum NodeType : unsigned {
   VWMACCU_VL,
   VWMACCSU_VL,
 
-  // Narrowing logical shift right.
-  // Operands are (source, shift, passthru, mask, vl)
-  VNSRL_VL,
-
   // Vector compare producing a mask. Fourth operand is input mask. Fifth
   // operand is VL.
   SETCC_VL,
@@ -750,7 +746,7 @@ class RISCVTargetLowering : public TargetLowering {
   getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
 
   bool shouldExtendTypeInLibCall(EVT Type) const override;
-  bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
+  bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override;
 
   /// Returns the register with the specified architectural or ABI name. This
   /// method is necessary to lower the llvm.read_register.* and
@@ -923,6 +919,11 @@ class RISCVTargetLowering : public TargetLowering {
                               MachineBasicBlock::instr_iterator &MBBI,
                               const TargetInstrInfo *TII) const override;
 
+  /// True if stack clash protection is enabled for this functions.
+  bool hasInlineStackProbe(const MachineFunction &MF) const override;
+
+  unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const;
+
 private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 052b4a6129822..49c2bc90c085a 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -40,6 +40,12 @@ using namespace llvm;
 STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");
 STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");
 
+static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(
+    DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,
+    cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "
+             "vill is cleared"),
+    cl::init(true));
+
 namespace {
 
 /// Given a virtual register \p Reg, return the corresponding VNInfo for it.
@@ -195,6 +201,14 @@ static bool hasUndefinedPassthru(const MachineInstr &MI) {
   return UseMO.getReg() == RISCV::NoRegister || UseMO.isUndef();
 }
 
+/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
+static bool isVectorCopy(const TargetRegisterInfo *TRI,
+                         const MachineInstr &MI) {
+  return MI.isCopy() && MI.getOperand(0).getReg().isPhysical() &&
+         RISCVRegisterInfo::isRVVRegClass(
+             TRI->getMinimalPhysRegClass(MI.getOperand(0).getReg()));
+}
+
 /// Which subfields of VL or VTYPE have values we need to preserve?
 struct DemandedFields {
   // Some unknown property of VL is used.  If demanded, must preserve entire
@@ -221,10 +235,13 @@ struct DemandedFields {
   bool SEWLMULRatio = false;
   bool TailPolicy = false;
   bool MaskPolicy = false;
+  // If this is true, we demand that VTYPE is set to some legal state, i.e. that
+  // vill is unset.
+  bool VILL = false;
 
   // Return true if any part of VTYPE was used
   bool usedVTYPE() const {
-    return SEW || LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
+    return SEW || LMUL || SEWLMULRatio || TailPolicy || MaskPolicy || VILL;
   }
 
   // Return true if any property of VL was used
@@ -239,6 +256,7 @@ struct DemandedFields {
     SEWLMULRatio = true;
     TailPolicy = true;
     MaskPolicy = true;
+    VILL = true;
   }
 
   // Mark all VL properties as demanded
@@ -263,6 +281,7 @@ struct DemandedFields {
     SEWLMULRatio |= B.SEWLMULRatio;
     TailPolicy |= B.TailPolicy;
     MaskPolicy |= B.MaskPolicy;
+    VILL |= B.VILL;
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -308,7 +327,8 @@ struct DemandedFields {
     OS << ", ";
     OS << "SEWLMULRatio=" << SEWLMULRatio << ", ";
     OS << "TailPolicy=" << TailPolicy << ", ";
-    OS << "MaskPolicy=" << MaskPolicy;
+    OS << "MaskPolicy=" << MaskPolicy << ", ";
+    OS << "VILL=" << VILL;
     OS << "}";
   }
 #endif
@@ -503,6 +523,21 @@ DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
     }
   }
 
+  // In §32.16.6, whole vector register moves have a dependency on SEW. At the
+  // MIR level though we don't encode the element type, and it gives the same
+  // result whatever the SEW may be.
+  //
+  // However it does need valid SEW, i.e. vill must be cleared. The entry to a
+  // function, calls and inline assembly may all set it, so make sure we clear
+  // it for whole register copies. Do this by leaving VILL demanded.
+  if (isVectorCopy(ST->getRegisterInfo(), MI)) {
+    Res.LMUL = DemandedFields::LMULNone;
+    Res.SEW = DemandedFields::SEWNone;
+    Res.SEWLMULRatio = false;
+    Res.TailPolicy = false;
+    Res.MaskPolicy = false;
+  }
+
   return Res;
 }
 
@@ -889,8 +924,8 @@ class RISCVInsertVSETVLI : public MachineFunctionPass {
     AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
     AU.addPreserved<LiveIntervalsWrapperPass>();
     AU.addPreserved<SlotIndexesWrapperPass>();
-    AU.addPreserved<LiveDebugVariables>();
-    AU.addPreserved<LiveStacks>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+    AU.addPreserved<LiveStacksWrapperLegacy>();
 
     MachineFunctionPass::getAnalysisUsage(AU);
   }
@@ -1208,6 +1243,18 @@ static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo,
 // legal for MI, but may not be the state requested by MI.
 void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
                                         const MachineInstr &MI) const {
+  if (isVectorCopy(ST->getRegisterInfo(), MI) &&
+      (Info.isUnknown() || !Info.isValid() || Info.hasSEWLMULRatioOnly())) {
+    // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may
+    // be coalesced into another vsetvli since we won't demand any fields.
+    VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly
+    NewInfo.setAVLImm(1);
+    NewInfo.setVTYPE(RISCVII::VLMUL::LMUL_1, /*sew*/ 8, /*ta*/ true,
+                     /*ma*/ true);
+    Info = NewInfo;
+    return;
+  }
+
   if (!RISCVII::hasSEWOp(MI.getDesc().TSFlags))
     return;
 
@@ -1296,7 +1343,8 @@ bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
   for (const MachineInstr &MI : MBB) {
     transferBefore(Info, MI);
 
-    if (isVectorConfigInstr(MI) || RISCVII::hasSEWOp(MI.getDesc().TSFlags))
+    if (isVectorConfigInstr(MI) || RISCVII::hasSEWOp(MI.getDesc().TSFlags) ||
+        isVectorCopy(ST->getRegisterInfo(), MI))
       HadVectorOp = true;
 
     transferAfter(Info, MI);
@@ -1426,6 +1474,16 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
       PrefixTransparent = false;
     }
 
+    if (EnsureWholeVectorRegisterMoveValidVTYPE &&
+        isVectorCopy(ST->getRegisterInfo(), MI)) {
+      if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {
+        insertVSETVLI(MBB, MI, MI.getDebugLoc(), CurInfo, PrevInfo);
+        PrefixTransparent = false;
+      }
+      MI.addOperand(MachineOperand::CreateReg(RISCV::VTYPE, /*isDef*/ false,
+                                              /*isImp*/ true));
+    }
+
     uint64_t TSFlags = MI.getDesc().TSFlags;
     if (RISCVII::hasSEWOp(TSFlags)) {
       if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {
@@ -1619,14 +1677,15 @@ bool RISCVInsertVSETVLI::canMutatePriorConfig(
     }
 
     auto &AVL = MI.getOperand(1);
-    auto &PrevAVL = PrevMI.getOperand(1);
 
-    // If the AVL is a register, we need to make sure MI's AVL dominates PrevMI.
-    // For now just check that PrevMI uses the same virtual register.
-    if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
-        (!MRI->hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
-         PrevAVL.getReg() != AVL.getReg()))
-      return false;
+    // If the AVL is a register, we need to make sure its definition is the same
+    // at PrevMI as it was at MI.
+    if (AVL.isReg() && AVL.getReg() != RISCV::X0) {
+      VNInfo *VNI = getVNInfoFromReg(AVL.getReg(), MI, LIS);
+      VNInfo *PrevVNI = getVNInfoFromReg(AVL.getReg(), PrevMI, LIS);
+      if (!VNI || !PrevVNI || VNI != PrevVNI)
+        return false;
+    }
   }
 
   assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());
diff --git a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
index f72ba2d5c667b..fe593a3cabad7 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp
@@ -109,6 +109,35 @@ class VXRMInfo {
     return VXRMInfo::getUnknown();
   }
 
+  // Calculate the VXRMInfo visible to a block assuming this and Other
+  // are both predecessors. To allow speculatively running WriteVXRM
+  // we will ignore Unknowns if one of this and Other have valid
+  // WriteVXRM. Rationale: WriteVXRM causes a pipeline flush in some
+  // uarchs and moving it outside loops is very important for some
+  // workloads.
+  VXRMInfo intersectAnticipated(const VXRMInfo &Other) const {
+    // If the new value isn't valid, ignore it.
+    if (!Other.isValid())
+      return *this;
+
+    // If this value isn't valid, this must be the first predecessor, use it.
+    if (!isValid())
+      return Other;
+
+    // If either is unknown, the result is the other one.
+    if (isUnknown())
+      return Other;
+    if (Other.isUnknown())
+      return *this;
+
+    // If we have an exact match, return this.
+    if (*this == Other)
+      return *this;
+
+    // Otherwise the result is unknown.
+    return VXRMInfo::getUnknown();
+  }
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Support for debugging, callable in GDB: V->dump()
   LLVM_DUMP_METHOD void dump() const {
@@ -187,7 +216,7 @@ class RISCVInsertWriteVXRM : public MachineFunctionPass {
 private:
   bool computeVXRMChanges(const MachineBasicBlock &MBB);
   void computeAvailable(const MachineBasicBlock &MBB);
-  void computeAnticipated(const MachineBasicBlock &MBB);
+  void computeAnticipated(const MachineFunction &MF, const MachineBasicBlock &MBB);
   void emitWriteVXRM(MachineBasicBlock &MBB);
 };
 
@@ -279,8 +308,9 @@ void RISCVInsertWriteVXRM::computeAvailable(const MachineBasicBlock &MBB) {
   }
 }
 
-void RISCVInsertWriteVXRM::computeAnticipated(const MachineBasicBlock &MBB) {
+void RISCVInsertWriteVXRM::computeAnticipated(const MachineFunction &MF, const MachineBasicBlock &MBB) {
   BlockData &BBInfo = BlockInfo[MBB.getNumber()];
+  const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
 
   BBInfo.InQueue = false;
 
@@ -289,7 +319,11 @@ void RISCVInsertWriteVXRM::computeAnticipated(const MachineBasicBlock &MBB) {
     Anticipated.setUnknown();
   } else {
     for (const MachineBasicBlock *S : MBB.successors())
-      Anticipated =
+      if (ST.hasVXRMPipelineFlush())
+        Anticipated =
+          Anticipated.intersectAnticipated(BlockInfo[S->getNumber()].AnticipatedIn);
+      else
+        Anticipated =
           Anticipated.intersect(BlockInfo[S->getNumber()].AnticipatedIn);
   }
 
@@ -453,7 +487,7 @@ bool RISCVInsertWriteVXRM::runOnMachineFunction(MachineFunction &MF) {
   while (!WorkList.empty()) {
     const MachineBasicBlock &MBB = *WorkList.front();
     WorkList.pop();
-    computeAnticipated(MBB);
+    computeAnticipated(MF, MBB);
   }
 
   // Phase 4 - Emit VXRM writes at the earliest place possible.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 47273d6bc06d6..6a3a89371b57a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1576,6 +1576,26 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     // No patch bytes means at most a PseudoCall is emitted
     return std::max(NumBytes, 8U);
   }
+  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
+  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
+  case TargetOpcode::PATCHABLE_TAIL_CALL: {
+    const MachineFunction &MF = *MI.getParent()->getParent();
+    const Function &F = MF.getFunction();
+    if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
+        F.hasFnAttribute("patchable-function-entry")) {
+      unsigned Num;
+      if (F.getFnAttribute("patchable-function-entry")
+              .getValueAsString()
+              .getAsInteger(10, Num))
+        return get(Opcode).getSize();
+
+      // Number of C.NOP or NOP
+      return (STI.hasStdExtCOrZca() ? 2 : 4) * Num;
+    }
+    // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
+    // respectively.
+    return STI.is64Bit() ? 68 : 44;
+  }
   default:
     return get(Opcode).getSize();
   }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 5747f05ffafd4..78682f2609dba 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -476,6 +476,8 @@ def XLenSubTrailingOnes : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(XLen - TrailingOnes, SDLoc(N),
                                    N->getValueType(0));
 }]>;
+def GIXLenSubTrailingOnes : GICustomOperandRenderer<"renderXLenSubTrailingOnes">,
+  GISDNodeXFormEquiv<XLenSubTrailingOnes>;
 
 // Checks if this mask is a non-empty sequence of ones starting at the
 // most/least significant bit with the remainder zero and exceeds simm32/simm12.
@@ -489,7 +491,15 @@ def TrailingOnesMask : PatLeaf<(imm), [{
   if (!N->hasOneUse())
     return false;
   return !isInt<12>(N->getSExtValue()) && isMask_64(N->getZExtValue());
-}], XLenSubTrailingOnes>;
+}], XLenSubTrailingOnes> {
+  let GISelPredicateCode = [{
+    if (!MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()))
+      return false;
+    const auto &MO = MI.getOperand(1);
+    return !isInt<12>(MO.getCImm()->getSExtValue()) &&
+           isMask_64(MO.getCImm()->getZExtValue());
+  }];
+}
 
 // Similar to LeadingOnesMask, but only consider leading ones in the lower 32
 // bits.
@@ -1373,6 +1383,17 @@ def PseudoAddTPRel : Pseudo<(outs GPR:$rd),
 def : Pat<(FrameAddrRegImm (iPTR GPR:$rs1), simm12:$imm12),
           (ADDI GPR:$rs1, simm12:$imm12)>;
 
+/// Stack probing
+
+let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1 in {
+// Probed stack allocation of a constant size, used in function prologues when
+// stack-clash protection is enabled.
+def PROBED_STACKALLOC : Pseudo<(outs GPR:$sp),
+                               (ins GPR:$scratch),
+                               []>,
+                               Sched<[]>;
+}
+
 /// HI and ADD_LO address nodes.
 
 // Pseudo for a rematerializable LUI+ADDI sequence for loading an address.
@@ -2064,6 +2085,7 @@ include "RISCVInstrInfoXSf.td"
 include "RISCVInstrInfoSFB.td"
 include "RISCVInstrInfoXCV.td"
 include "RISCVInstrInfoXwch.td"
+include "RISCVInstrInfoXqci.td"
 
 //===----------------------------------------------------------------------===//
 // Global ISel
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 8e0c4826ac00d..6506b6746b151 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -1726,7 +1726,7 @@ foreach n = [1, 2, 4, 8] in {
   def VMV#n#R_V  : RVInstV<0b100111, !add(n, -1), OPIVI, (outs vrc:$vd),
                            (ins vrc:$vs2), "vmv" # n # "r.v", "$vd, $vs2">,
                    VMVRSched<n> {
-    let Uses = [];
+    let Uses = [VTYPE];
     let vm = 1;
   }
 }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 4b938fc734e5c..e48a6f9309294 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -459,17 +459,6 @@ def riscv_vfwmul_vl : SDNode<"RISCVISD::VFWMUL_VL", SDT_RISCVVWFPBinOp_VL, [SDNP
 def riscv_vfwadd_vl : SDNode<"RISCVISD::VFWADD_VL", SDT_RISCVVWFPBinOp_VL, [SDNPCommutative]>;
 def riscv_vfwsub_vl : SDNode<"RISCVISD::VFWSUB_VL", SDT_RISCVVWFPBinOp_VL, []>;
 
-def SDT_RISCVVNIntBinOp_VL : SDTypeProfile<1, 5, [SDTCisVec<0>, SDTCisInt<0>,
-                                                  SDTCisInt<1>,
-                                                  SDTCisSameNumEltsAs<0, 1>,
-                                                  SDTCisOpSmallerThanOp<0, 1>,
-                                                  SDTCisSameAs<0, 2>,
-                                                  SDTCisSameAs<0, 3>,
-                                                  SDTCisSameNumEltsAs<0, 4>,
-                                                  SDTCVecEltisVT<4, i1>,
-                                                  SDTCisVT<5, XLenVT>]>;
-def riscv_vnsrl_vl : SDNode<"RISCVISD::VNSRL_VL", SDT_RISCVVNIntBinOp_VL>;
-
 def SDT_RISCVVWIntBinOpW_VL : SDTypeProfile<1, 5, [SDTCisVec<0>, SDTCisInt<0>,
                                                    SDTCisSameAs<0, 1>,
                                                    SDTCisInt<2>,
@@ -885,29 +874,6 @@ multiclass VPatBinaryWVL_VV_VX_WV_WX<SDPatternOperator vop, SDNode vop_w,
   }
 }
 
-multiclass VPatBinaryNVL_WV_WX_WI<SDPatternOperator vop, string instruction_name> {
-  foreach VtiToWti = AllWidenableIntVectors in {
-    defvar vti = VtiToWti.Vti;
-    defvar wti = VtiToWti.Wti;
-    let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
-                                 GetVTypePredicates<wti>.Predicates) in {
-      def : VPatBinaryVL_V<vop, instruction_name, "WV",
-                           vti.Vector, wti.Vector, vti.Vector, vti.Mask,
-                           vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass,
-                           vti.RegClass>;
-      def : VPatBinaryVL_XI<vop, instruction_name, "WX",
-                            vti.Vector, wti.Vector, vti.Vector, vti.Mask,
-                            vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass,
-                            SplatPat, GPR>;
-      def : VPatBinaryVL_XI<vop, instruction_name, "WI",
-                            vti.Vector, wti.Vector, vti.Vector, vti.Mask,
-                            vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass,
-                            !cast<ComplexPattern>(SplatPat#_#uimm5),
-                            uimm5>;
-    }
-  }
-}
-
 class VPatBinaryVL_VF<SDPatternOperator vop,
                       string instruction_name,
                       ValueType result_type,
@@ -2166,8 +2132,6 @@ defm : VPatNarrowShiftSplatExt_WX<riscv_srl_vl, riscv_zext_vl_oneuse, "PseudoVNS
 defm : VPatNarrowShiftVL_WV<riscv_srl_vl, "PseudoVNSRL">;
 defm : VPatNarrowShiftVL_WV<riscv_sra_vl, "PseudoVNSRA">;
 
-defm : VPatBinaryNVL_WV_WX_WI<riscv_vnsrl_vl, "PseudoVNSRL">;
-
 foreach vtiTowti = AllWidenableIntVectors in {
   defvar vti = vtiTowti.Vti;
   defvar wti = vtiTowti.Wti;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
new file mode 100644
index 0000000000000..29758014f73ed
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -0,0 +1,110 @@
+//===---------------- RISCVInstrInfoXQci.td ----------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the vendor extensions defined by QUALCOMM.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand and SDNode transformation definitions.
+//===----------------------------------------------------------------------===//
+
+def uimm11 : RISCVUImmLeafOp<11>;
+
+//===----------------------------------------------------------------------===//
+// Instruction Formats
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Instruction Class Templates
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
+class QCILoad_ScaleIdx<bits<4> func4, string opcodestr>
+    : RVInstRBase<0b111, OPC_CUSTOM_0,
+                  (outs GPR:$rd), (ins GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt),
+                  opcodestr, "$rd, $rs1, $rs2, $shamt"> {
+  bits<3> shamt;
+  let Inst{31-28} = func4;
+  let Inst{27-25} = shamt;
+}
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in {
+// rd corresponds to the source for the store 'rs3' described in the spec.
+class QCIStore_ScaleIdx<bits<4> func4, string opcodestr>
+    : RVInstRBase<0b110, OPC_CUSTOM_1, (outs),
+                  (ins GPR:$rd, GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt),
+                  opcodestr, "$rd, $rs1, $rs2, $shamt"> {
+  bits<3> shamt;
+  let Inst{31-28} = func4;
+  let Inst{27-25} = shamt;
+}
+}
+
+class QCIRVInstR<bits<4> func4, string opcodestr>
+    : RVInstR<{0b000, func4}, 0b011, OPC_CUSTOM_0, (outs GPRNoX0:$rd),
+              (ins GPRNoX0:$rs1), opcodestr, "$rd, $rs1"> {
+  let rs2 = 0;
+}
+
+class QCIRVInstRR<bits<5> func5, DAGOperand InTyRs1, string opcodestr>
+    : RVInstR<{0b00, func5}, 0b011, OPC_CUSTOM_0, (outs GPRNoX0:$rd),
+              (ins InTyRs1:$rs1, GPRNoX0:$rs2), opcodestr, "$rd, $rs1, $rs2">;
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasVendorXqcicsr, IsRV32], DecoderNamespace = "Xqcicsr" in {
+let hasSideEffects = 1, mayLoad = 0, mayStore = 0 in {
+  def QC_CSRRWR : RVInstR<0b1000110, 0b000, OPC_SYSTEM, (outs GPR:$rd),
+                          (ins GPR:$rs1, GPRNoX0:$rs2), "qc.csrrwr",
+                          "$rd, $rs1, $rs2">;
+
+  def QC_CSRRWRI : RVInstR<0b1000111, 0b000, OPC_SYSTEM, (outs GPR:$rd),
+                           (ins uimm5:$rs1, GPRNoX0:$rs2), "qc.csrrwri",
+                           "$rd, $rs1, $rs2">;
+} // hasSideEffects = 1, mayLoad = 0, mayStore = 0
+} // Predicates = [HasVendorXqcicsr, IsRV32], DecoderNamespace = "Xqcicsr"
+
+let Predicates = [HasVendorXqcisls, IsRV32], DecoderNamespace = "Xqcisls" in {
+  def  QC_LRB  : QCILoad_ScaleIdx<0b1000, "qc.lrb">;
+  def  QC_LRH  : QCILoad_ScaleIdx<0b1001, "qc.lrh">;
+  def  QC_LRW  : QCILoad_ScaleIdx<0b1010, "qc.lrw">;
+  def  QC_LRBU : QCILoad_ScaleIdx<0b1011, "qc.lrbu">;
+  def  QC_LRHU : QCILoad_ScaleIdx<0b1100, "qc.lrhu">;
+
+  def  QC_SRB  : QCIStore_ScaleIdx<0b1101, "qc.srb">;
+  def  QC_SRH  : QCIStore_ScaleIdx<0b1110, "qc.srh">;
+  def  QC_SRW  : QCIStore_ScaleIdx<0b1111, "qc.srw">;
+} // Predicates = [HasVendorXqcisls, IsRV32], DecoderNamespace = "Xqcisls"
+
+let Predicates = [HasVendorXqcia, IsRV32], DecoderNamespace = "Xqcia" in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+  def QC_SLASAT : QCIRVInstRR<0b01010, GPRNoX0, "qc.slasat">;
+  def QC_SLLSAT : QCIRVInstRR<0b01100, GPRNoX0, "qc.sllsat">;
+  def QC_ADDSAT : QCIRVInstRR<0b01110, GPRNoX0, "qc.addsat">;
+  def QC_ADDUSAT : QCIRVInstRR<0b01111, GPRNoX0, "qc.addusat">;
+  def QC_SUBSAT : QCIRVInstRR<0b10000, GPRNoX0, "qc.subsat">;
+  def QC_SUBUSAT : QCIRVInstRR<0b10001, GPRNoX0, "qc.subusat">;
+
+  def QC_WRAP : QCIRVInstRR<0b10010, GPR, "qc.wrap">;
+  def QC_WRAPI : RVInstI<0b000, OPC_CUSTOM_0, (outs GPRNoX0:$rd),
+                         (ins GPRNoX0:$rs1, uimm11:$imm11), "qc.wrapi",
+                         "$rd, $rs1, $imm11"> {
+    bits<11> imm11;
+
+    let imm12 = {0b0, imm11};
+  }
+
+  def QC_NORM : QCIRVInstR<0b0111, "qc.norm">;
+  def QC_NORMU : QCIRVInstR<0b1000, "qc.normu">;
+  def QC_NORMEU : QCIRVInstR<0b1001, "qc.normeu">;
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+} // Predicates = [HasVendorXqcia, IsRV32], DecoderNamespace = "Xqcia"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index c57e7af3c5614..8cefceab43e56 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -188,8 +188,6 @@ class binop_with_non_imm12<SDPatternOperator binop>
   let PredicateCodeUsesOperands = 1;
   let GISelPredicateCode = [{
     const MachineOperand &ImmOp = *Operands[1];
-    const MachineFunction &MF = *MI.getParent()->getParent();
-    const MachineRegisterInfo &MRI = MF.getRegInfo();
 
     if (ImmOp.isReg() && ImmOp.getReg())
       if (auto Val = getIConstantVRegValWithLookThrough(ImmOp.getReg(), MRI)) {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index f13b3e69f84f8..ab54b45f4de93 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -215,6 +215,7 @@ def: Pat<(any_frint FPR32:$rs1), (FROUNDNX_S FPR32:$rs1, FRM_DYN)>;
 def: Pat<(any_fnearbyint FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_DYN)>;
 
 def: Pat<(any_fround FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_RMM)>;
+def: Pat<(any_froundeven FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_RNE)>;
 def: Pat<(any_ffloor FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_RDN)>;
 def: Pat<(any_fceil FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_RUP)>;
 def: Pat<(any_ftrunc FPR32:$rs1), (FROUND_S FPR32:$rs1, FRM_RTZ)>;
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
index d0c363042f511..a0d7931763818 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVMachineFunctionInfo.h"
+#include "llvm/IR/Module.h"
 
 using namespace llvm;
 
@@ -26,6 +27,35 @@ MachineFunctionInfo *RISCVMachineFunctionInfo::clone(
   return DestMF.cloneInfo<RISCVMachineFunctionInfo>(*this);
 }
 
+RISCVMachineFunctionInfo::RISCVMachineFunctionInfo(const Function &F,
+                                                   const RISCVSubtarget *STI) {
+
+  // The default stack probe size is 4096 if the function has no
+  // stack-probe-size attribute. This is a safe default because it is the
+  // smallest possible guard page size.
+  uint64_t ProbeSize = 4096;
+  if (F.hasFnAttribute("stack-probe-size"))
+    ProbeSize = F.getFnAttributeAsParsedInteger("stack-probe-size");
+  else if (const auto *PS = mdconst::extract_or_null<ConstantInt>(
+               F.getParent()->getModuleFlag("stack-probe-size")))
+    ProbeSize = PS->getZExtValue();
+  assert(int64_t(ProbeSize) > 0 && "Invalid stack probe size");
+
+  // Round down to the stack alignment.
+  uint64_t StackAlign =
+      STI->getFrameLowering()->getTransientStackAlign().value();
+  ProbeSize = std::max(StackAlign, alignDown(ProbeSize, StackAlign));
+  StringRef ProbeKind;
+  if (F.hasFnAttribute("probe-stack"))
+    ProbeKind = F.getFnAttribute("probe-stack").getValueAsString();
+  else if (const auto *PS = dyn_cast_or_null<MDString>(
+               F.getParent()->getModuleFlag("probe-stack")))
+    ProbeKind = PS->getString();
+  if (ProbeKind.size()) {
+    StackProbeSize = ProbeSize;
+  }
+}
+
 void yaml::RISCVMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
   MappingTraits<RISCVMachineFunctionInfo>::mapping(YamlIO, *this);
 }
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index 779c652b4d8fc..8909f2f3bd317 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -76,8 +76,10 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
   unsigned RVPushRegs = 0;
   int RVPushRlist = llvm::RISCVZC::RLISTENCODE::INVALID_RLIST;
 
+  int64_t StackProbeSize = 0;
+
 public:
-  RISCVMachineFunctionInfo(const Function &F, const TargetSubtargetInfo *STI) {}
+  RISCVMachineFunctionInfo(const Function &F, const RISCVSubtarget *STI);
 
   MachineFunctionInfo *
   clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
diff --git a/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
index 338e23751b8b2..08e2b835547ca 100644
--- a/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
@@ -120,7 +120,7 @@ bool RISCVPostRAExpandPseudo::expandMovAddr(MachineBasicBlock &MBB,
 
 } // end of anonymous namespace
 
-INITIALIZE_PASS(RISCVPostRAExpandPseudo, "riscv-expand-pseudolisimm32",
+INITIALIZE_PASS(RISCVPostRAExpandPseudo, "riscv-post-ra-expand-pseudo",
                 RISCV_POST_RA_EXPAND_PSEUDO_NAME, false, false)
 namespace llvm {
 
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 03a48ff3c1758..c4e19c515b155 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -24,15 +24,32 @@ class RISCVTuneInfo {
 
   // Tail duplication threshold at -O3.
   bits<32> TailDupAggressiveThreshold = 6;
+
+  bits<32> MaxStoresPerMemsetOptSize = 4;
+  bits<32> MaxStoresPerMemset = 8;
+
+  bits<32> MaxGluedStoresPerMemcpy = 0;
+  bits<32> MaxStoresPerMemcpyOptSize = 4;
+  bits<32> MaxStoresPerMemcpy = 8;
+
+  bits<32> MaxStoresPerMemmoveOptSize = 4;
+  bits<32> MaxStoresPerMemmove = 8;
+
+  bits<32> MaxLoadsPerMemcmpOptSize = 4;
+  bits<32> MaxLoadsPerMemcmp = 8;
 }
 
 def RISCVTuneInfoTable : GenericTable {
   let FilterClass = "RISCVTuneInfo";
   let CppTypeName = "RISCVTuneInfo";
   let Fields = ["Name", "PrefFunctionAlignment", "PrefLoopAlignment",
-                "CacheLineSize", "PrefetchDistance",
-                "MinPrefetchStride", "MaxPrefetchIterationsAhead",
-                "MinimumJumpTableEntries", "TailDupAggressiveThreshold"];
+                "CacheLineSize", "PrefetchDistance", "MinPrefetchStride",
+                "MaxPrefetchIterationsAhead", "MinimumJumpTableEntries",
+                "TailDupAggressiveThreshold", "MaxStoresPerMemsetOptSize",
+                "MaxStoresPerMemset", "MaxGluedStoresPerMemcpy",
+                "MaxStoresPerMemcpyOptSize", "MaxStoresPerMemcpy",
+                "MaxStoresPerMemmoveOptSize", "MaxStoresPerMemmove",
+                "MaxLoadsPerMemcmpOptSize", "MaxLoadsPerMemcmp"];
 }
 
 def getRISCVTuneInfo : SearchIndex {
@@ -95,7 +112,7 @@ def ROCKET : RISCVTuneProcessorModel<"rocket",
 
 defvar SiFive7TuneFeatures = [TuneSiFive7, TuneNoDefaultUnroll,
                               TuneShortForwardBranchOpt,
-                              FeaturePostRAScheduler];
+                              TunePostRAScheduler];
 def SIFIVE_7 : RISCVTuneProcessorModel<"sifive-7-series",
                                        SiFive7Model, SiFive7TuneFeatures>;
 
@@ -251,7 +268,7 @@ defvar SiFiveP400TuneFeatures = [TuneNoDefaultUnroll,
                                  TuneConditionalCompressedMoveFusion,
                                  TuneLUIADDIFusion,
                                  TuneAUIPCADDIFusion,
-                                 FeaturePostRAScheduler];
+                                 TunePostRAScheduler];
 
 def SIFIVE_P450 : RISCVProcessorModel<"sifive-p450", SiFiveP400Model,
                                       !listconcat(RVA22U64Features,
@@ -277,7 +294,8 @@ def SIFIVE_P470 : RISCVProcessorModel<"sifive-p470", SiFiveP400Model,
                                        FeatureUnalignedScalarMem,
                                        FeatureUnalignedVectorMem]),
                                       !listconcat(SiFiveP400TuneFeatures,
-                                                  [TuneNoSinkSplatOperands])>;
+                                                  [TuneNoSinkSplatOperands,
+                                                   TuneVXRMPipelineFlush])>;
 
 
 def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
@@ -298,7 +316,8 @@ def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
                                        TuneLUIADDIFusion,
                                        TuneAUIPCADDIFusion,
                                        TuneNoSinkSplatOperands,
-                                       FeaturePostRAScheduler]>;
+                                       TuneVXRMPipelineFlush,
+                                       TunePostRAScheduler]>;
 
 def SYNTACORE_SCR1_BASE : RISCVProcessorModel<"syntacore-scr1-base",
                                               SyntacoreSCR1Model,
@@ -327,7 +346,7 @@ def SYNTACORE_SCR3_RV32 : RISCVProcessorModel<"syntacore-scr3-rv32",
                                                FeatureStdExtZifencei,
                                                FeatureStdExtM,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR3_RV64 : RISCVProcessorModel<"syntacore-scr3-rv64",
                                               SyntacoreSCR3RV64Model,
@@ -338,7 +357,7 @@ def SYNTACORE_SCR3_RV64 : RISCVProcessorModel<"syntacore-scr3-rv64",
                                                FeatureStdExtM,
                                                FeatureStdExtA,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR4_RV32 : RISCVProcessorModel<"syntacore-scr4-rv32",
                                               SyntacoreSCR4RV32Model,
@@ -350,7 +369,7 @@ def SYNTACORE_SCR4_RV32 : RISCVProcessorModel<"syntacore-scr4-rv32",
                                                FeatureStdExtF,
                                                FeatureStdExtD,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR4_RV64 : RISCVProcessorModel<"syntacore-scr4-rv64",
                                               SyntacoreSCR4RV64Model,
@@ -363,7 +382,7 @@ def SYNTACORE_SCR4_RV64 : RISCVProcessorModel<"syntacore-scr4-rv64",
                                                FeatureStdExtF,
                                                FeatureStdExtD,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR5_RV32 : RISCVProcessorModel<"syntacore-scr5-rv32",
                                               SyntacoreSCR5RV32Model,
@@ -376,7 +395,7 @@ def SYNTACORE_SCR5_RV32 : RISCVProcessorModel<"syntacore-scr5-rv32",
                                                FeatureStdExtF,
                                                FeatureStdExtD,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR5_RV64 : RISCVProcessorModel<"syntacore-scr5-rv64",
                                               SyntacoreSCR5RV64Model,
@@ -389,7 +408,7 @@ def SYNTACORE_SCR5_RV64 : RISCVProcessorModel<"syntacore-scr5-rv64",
                                                FeatureStdExtF,
                                                FeatureStdExtD,
                                                FeatureStdExtC],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def SYNTACORE_SCR7 : RISCVProcessorModel<"syntacore-scr7",
                                               SyntacoreSCR7Model,
@@ -408,7 +427,7 @@ def SYNTACORE_SCR7 : RISCVProcessorModel<"syntacore-scr7",
                                                FeatureStdExtZbc,
                                                FeatureStdExtZbs,
                                                FeatureStdExtZkn],
-                                              [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
+                                              [TuneNoDefaultUnroll, TunePostRAScheduler]>;
 
 def TENSTORRENT_ASCALON_D8 : RISCVProcessorModel<"tt-ascalon-d8",
                                                  NoSchedModel,
@@ -430,7 +449,7 @@ def TENSTORRENT_ASCALON_D8 : RISCVProcessorModel<"tt-ascalon-d8",
                                                   FeatureUnalignedVectorMem]),
                                                  [TuneNoDefaultUnroll,
                                                   TuneOptimizedZeroStrideLoad,
-                                                  FeaturePostRAScheduler]>;
+                                                  TunePostRAScheduler]>;
 
 def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1",
                                             NoSchedModel,
@@ -510,7 +529,8 @@ def SPACEMIT_X60 : RISCVProcessorModel<"spacemit-x60",
                                        [TuneDLenFactor2,
                                         TuneOptimizedNF2SegmentLoadStore,
                                         TuneOptimizedNF3SegmentLoadStore,
-                                        TuneOptimizedNF4SegmentLoadStore]> {
+                                        TuneOptimizedNF4SegmentLoadStore,
+                                        TuneVXRMPipelineFlush]> {
   let MVendorID = 0x710;
   let MArchID = 0x8000000058000001;
   let MImpID = 0x1000000049772200;
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index cfcc3119257f6..b9c70fe60fb50 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -162,7 +162,7 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
 bool RISCVRegisterInfo::isAsmClobberable(const MachineFunction &MF,
                                          MCRegister PhysReg) const {
-  return !MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(PhysReg);
+  return !MF.getSubtarget().isRegisterReservedByUser(PhysReg);
 }
 
 const uint32_t *RISCVRegisterInfo::getNoPreservedMask() const {
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 426d368204904..38443e8646de4 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -203,6 +203,10 @@ void RISCVSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
   Policy.OnlyTopDown = false;
   Policy.OnlyBottomUp = false;
 
+  // Disabling the latency heuristic can reduce the number of spills/reloads but
+  // will cause some regressions on some cores.
+  Policy.DisableLatencyHeuristic = DisableLatencySchedHeuristic;
+
   // Spilling is generally expensive on all RISC-V cores, so always enable
   // register-pressure tracking. This will increase compile time.
   Policy.ShouldTrackPressure = true;
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 043838e13b964..9a1881c2d3983 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -53,6 +53,19 @@ struct RISCVTuneInfo {
 
   // Tail duplication threshold at -O3.
   unsigned TailDupAggressiveThreshold;
+
+  unsigned MaxStoresPerMemsetOptSize;
+  unsigned MaxStoresPerMemset;
+
+  unsigned MaxGluedStoresPerMemcpy;
+  unsigned MaxStoresPerMemcpyOptSize;
+  unsigned MaxStoresPerMemcpy;
+
+  unsigned MaxStoresPerMemmoveOptSize;
+  unsigned MaxStoresPerMemmove;
+
+  unsigned MaxLoadsPerMemcmpOptSize;
+  unsigned MaxLoadsPerMemcmp;
 };
 
 #define GET_RISCVTuneInfoTable_DECL
@@ -218,11 +231,14 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
            TargetABI == RISCVABI::ABI_ILP32 ||
            TargetABI == RISCVABI::ABI_ILP32E;
   }
-  bool isRegisterReservedByUser(Register i) const {
+  bool isRegisterReservedByUser(Register i) const override {
     assert(i < RISCV::NUM_TARGET_REGS && "Register out of range");
     return UserReservedRegister[i];
   }
 
+  // XRay support - require D and C extensions.
+  bool isXRaySupported() const override { return hasStdExtD() && hasStdExtC(); }
+
   // Vector codegen related methods.
   bool hasVInstructions() const { return HasStdExtZve32x; }
   bool hasVInstructionsI64() const { return HasStdExtZve64x; }
@@ -325,6 +341,30 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
     return TuneInfo->TailDupAggressiveThreshold;
   }
 
+  unsigned getMaxStoresPerMemset(bool OptSize) const {
+    return OptSize ? TuneInfo->MaxStoresPerMemsetOptSize
+                   : TuneInfo->MaxStoresPerMemset;
+  }
+
+  unsigned getMaxGluedStoresPerMemcpy() const {
+    return TuneInfo->MaxGluedStoresPerMemcpy;
+  }
+
+  unsigned getMaxStoresPerMemcpy(bool OptSize) const {
+    return OptSize ? TuneInfo->MaxStoresPerMemcpyOptSize
+                   : TuneInfo->MaxStoresPerMemcpy;
+  }
+
+  unsigned getMaxStoresPerMemmove(bool OptSize) const {
+    return OptSize ? TuneInfo->MaxStoresPerMemmoveOptSize
+                   : TuneInfo->MaxStoresPerMemmove;
+  }
+
+  unsigned getMaxLoadsPerMemcmp(bool OptSize) const {
+    return OptSize ? TuneInfo->MaxLoadsPerMemcmpOptSize
+                   : TuneInfo->MaxLoadsPerMemcmp;
+  }
+
   void overrideSchedPolicy(MachineSchedPolicy &Policy,
                            unsigned NumRegionInstrs) const override;
 };
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index fa507653264cc..dcd3598f658f6 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -114,6 +114,11 @@ static cl::opt<bool>
                       cl::desc("Enable the RISC-V VL Optimizer pass"),
                       cl::init(false), cl::Hidden);
 
+static cl::opt<bool> DisableVectorMaskMutation(
+    "riscv-disable-vector-mask-mutation",
+    cl::desc("Disable the vector mask scheduling mutation"), cl::init(false),
+    cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
@@ -266,8 +271,8 @@ RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
     BumpPtrAllocator &Allocator, const Function &F,
     const TargetSubtargetInfo *STI) const {
-  return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
-                                                                    F, STI);
+  return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
+      Allocator, F, static_cast<const RISCVSubtarget *>(STI));
 }
 
 TargetTransformInfo
@@ -367,6 +372,12 @@ class RISCVPassConfig : public TargetPassConfig {
       DAG->addMutation(createStoreClusterDAGMutation(
           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
     }
+
+    const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
+    if (!DisableVectorMaskMutation && ST.hasVInstructions()) {
+      DAG = DAG ? DAG : createGenericSchedLive(C);
+      DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
+    }
     return DAG;
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.h b/llvm/lib/Target/RISCV/RISCVTargetMachine.h
index d73447ef273e4..5506196c3c7e8 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.h
@@ -61,6 +61,10 @@ class RISCVTargetMachine : public CodeGenTargetMachineImpl {
                                 SMRange &SourceRange) const override;
   void registerPassBuilderCallbacks(PassBuilder &PB) override;
 };
+
+std::unique_ptr<ScheduleDAGMutation>
+createRISCVVectorMaskDAGMutation(const TargetRegisterInfo *TRI);
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 8f0ef69258b16..cbf8c57fde44d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -669,7 +669,7 @@ static unsigned isM1OrSmaller(MVT VT) {
 
 InstructionCost RISCVTTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
   if (isa<ScalableVectorType>(Ty))
     return InstructionCost::getInvalid();
 
@@ -1013,20 +1013,65 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   case Intrinsic::sadd_sat:
   case Intrinsic::ssub_sat:
   case Intrinsic::uadd_sat:
-  case Intrinsic::usub_sat:
+  case Intrinsic::usub_sat: {
+    auto LT = getTypeLegalizationCost(RetTy);
+    if (ST->hasVInstructions() && LT.second.isVector()) {
+      unsigned Op;
+      switch (ICA.getID()) {
+      case Intrinsic::sadd_sat:
+        Op = RISCV::VSADD_VV;
+        break;
+      case Intrinsic::ssub_sat:
+        Op = RISCV::VSSUBU_VV;
+        break;
+      case Intrinsic::uadd_sat:
+        Op = RISCV::VSADDU_VV;
+        break;
+      case Intrinsic::usub_sat:
+        Op = RISCV::VSSUBU_VV;
+        break;
+      }
+      return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
+    }
+    break;
+  }
   case Intrinsic::fabs:
   case Intrinsic::sqrt: {
     auto LT = getTypeLegalizationCost(RetTy);
-    if (ST->hasVInstructions() && LT.second.isVector())
-      return LT.first;
+    // TODO: add f16/bf16, bf16 with zvfbfmin && f16 with zvfhmin
+    if (ST->hasVInstructions() && LT.second.isVector()) {
+      unsigned Op;
+      switch (ICA.getID()) {
+      case Intrinsic::fabs:
+        Op = RISCV::VFSGNJX_VV;
+        break;
+      case Intrinsic::sqrt:
+        Op = RISCV::VFSQRT_V;
+        break;
+      }
+      return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
+    }
     break;
   }
   case Intrinsic::cttz:
   case Intrinsic::ctlz:
   case Intrinsic::ctpop: {
     auto LT = getTypeLegalizationCost(RetTy);
-    if (ST->hasVInstructions() && ST->hasStdExtZvbb() && LT.second.isVector())
-      return LT.first;
+    if (ST->hasVInstructions() && ST->hasStdExtZvbb() && LT.second.isVector()) {
+      unsigned Op;
+      switch (ICA.getID()) {
+      case Intrinsic::cttz:
+        Op = RISCV::VCTZ_V;
+        break;
+      case Intrinsic::ctlz:
+        Op = RISCV::VCLZ_V;
+        break;
+      case Intrinsic::ctpop:
+        Op = RISCV::VCPOP_V;
+        break;
+      }
+      return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
+    }
     break;
   }
   case Intrinsic::abs: {
@@ -1144,37 +1189,6 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                                   : RISCV::VMV_V_X,
                                               LT.second, CostKind);
   }
-  case Intrinsic::vp_reduce_add:
-  case Intrinsic::vp_reduce_fadd:
-  case Intrinsic::vp_reduce_mul:
-  case Intrinsic::vp_reduce_fmul:
-  case Intrinsic::vp_reduce_and:
-  case Intrinsic::vp_reduce_or:
-  case Intrinsic::vp_reduce_xor: {
-    std::optional<Intrinsic::ID> RedID =
-        VPIntrinsic::getFunctionalIntrinsicIDForVP(ICA.getID());
-    assert(RedID.has_value());
-    unsigned RedOp = getArithmeticReductionInstruction(*RedID);
-    return getArithmeticReductionCost(RedOp,
-                                      cast<VectorType>(ICA.getArgTypes()[1]),
-                                      ICA.getFlags(), CostKind);
-  }
-  case Intrinsic::vp_reduce_smax:
-  case Intrinsic::vp_reduce_smin:
-  case Intrinsic::vp_reduce_umax:
-  case Intrinsic::vp_reduce_umin:
-  case Intrinsic::vp_reduce_fmax:
-  case Intrinsic::vp_reduce_fmaximum:
-  case Intrinsic::vp_reduce_fmin:
-  case Intrinsic::vp_reduce_fminimum: {
-    std::optional<Intrinsic::ID> RedID =
-        VPIntrinsic::getFunctionalIntrinsicIDForVP(ICA.getID());
-    assert(RedID.has_value());
-    Intrinsic::ID MinMaxID = getMinMaxReductionIntrinsicOp(*RedID);
-    return getMinMaxReductionCost(MinMaxID,
-                                  cast<VectorType>(ICA.getArgTypes()[1]),
-                                  ICA.getFlags(), CostKind);
-  }
   }
 
   if (ST->hasVInstructions() && RetTy->isVectorTy()) {
@@ -1462,7 +1476,7 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
     return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
   }
 
-  // IR Reduction is composed by two vmv and one rvv reduction instruction.
+  // IR Reduction is composed by one rvv reduction instruction and vmv
   unsigned SplitOp;
   SmallVector<unsigned, 3> Opcodes;
   switch (IID) {
@@ -1470,27 +1484,27 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
     llvm_unreachable("Unsupported intrinsic");
   case Intrinsic::smax:
     SplitOp = RISCV::VMAX_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDMAX_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
     break;
   case Intrinsic::smin:
     SplitOp = RISCV::VMIN_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDMIN_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
     break;
   case Intrinsic::umax:
     SplitOp = RISCV::VMAXU_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
     break;
   case Intrinsic::umin:
     SplitOp = RISCV::VMINU_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDMINU_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
     break;
   case Intrinsic::maxnum:
     SplitOp = RISCV::VFMAX_VV;
-    Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
+    Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
     break;
   case Intrinsic::minnum:
     SplitOp = RISCV::VFMIN_VV;
-    Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
+    Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
     break;
   }
   // Add a cost for data larger than LMUL8
@@ -1520,35 +1534,48 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
 
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
-  SmallVector<unsigned, 3> Opcodes;
   Type *ElementTy = Ty->getElementType();
   if (ElementTy->isIntegerTy(1)) {
     if (ISD == ISD::AND) {
       // Example sequences:
       //   vsetvli a0, zero, e8, mf8, ta, ma
+      //   vmand.mm v8, v9, v8 ; needed every time type is split
       //   vmnot.m v8, v0
       //   vcpop.m a0, v8
       //   seqz a0, a0
-      Opcodes = {RISCV::VMNAND_MM, RISCV::VCPOP_M};
-      return (LT.first - 1) +
-             getRISCVInstructionCost(Opcodes, LT.second, CostKind) +
+      return LT.first * getRISCVInstructionCost(RISCV::VMNAND_MM, LT.second,
+                                                CostKind) +
+             getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +
              getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,
                                 CmpInst::ICMP_EQ, CostKind);
+    } else if (ISD == ISD::XOR) {
+      // Example sequences:
+      //   vsetvli a0, zero, e8, mf8, ta, ma
+      //   vmxor.mm v8, v0, v8 ; needed every time type is split
+      //   vcpop.m a0, v8
+      //   andi a0, a0, 1
+      return (LT.first - 1) *
+                 getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind) +
+             getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) + 1;
     } else {
       // Example sequences:
       //   vsetvli a0, zero, e8, mf8, ta, ma
+      //   vmxor.mm v8, v9, v8 ; needed every time type is split
       //   vcpop.m a0, v0
       //   snez a0, a0
-      Opcodes = {RISCV::VCPOP_M};
-      return (LT.first - 1) +
-             getRISCVInstructionCost(Opcodes, LT.second, CostKind) +
+      return (LT.first - 1) *
+                 getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind) +
+             getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +
              getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,
                                 CmpInst::ICMP_NE, CostKind);
     }
   }
 
-  // IR Reduction is composed by two vmv and one rvv reduction instruction.
+  // IR Reduction of or/and is composed by one vmv and one rvv reduction
+  // instruction, and others is composed by two vmv and one rvv reduction
+  // instruction
   unsigned SplitOp;
+  SmallVector<unsigned, 3> Opcodes;
   switch (ISD) {
   case ISD::ADD:
     SplitOp = RISCV::VADD_VV;
@@ -1556,7 +1583,7 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     break;
   case ISD::OR:
     SplitOp = RISCV::VOR_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDOR_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
     break;
   case ISD::XOR:
     SplitOp = RISCV::VXOR_VV;
@@ -1564,7 +1591,7 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     break;
   case ISD::AND:
     SplitOp = RISCV::VAND_VV;
-    Opcodes = {RISCV::VMV_S_X, RISCV::VREDAND_VS, RISCV::VMV_X_S};
+    Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
     break;
   case ISD::FADD:
     // We can't promote f16/bf16 fadd reductions.
@@ -2226,7 +2253,6 @@ void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   UP.Runtime = true;
   UP.UnrollRemainder = true;
   UP.UnrollAndJam = true;
-  UP.UnrollAndJamInnerLoopThreshold = 60;
 
   // Force unrolling small loops can be very useful because of the branch
   // taken cost of the backedge.
@@ -2404,6 +2430,7 @@ bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
   case Intrinsic::vp_ssub_sat:
   case Intrinsic::usub_sat:
   case Intrinsic::vp_usub_sat:
+  case Intrinsic::vp_select:
     return Operand == 1;
     // These intrinsics are commutative.
   case Intrinsic::vp_add:
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 6fd36e90a02dd..bd90bfed6e2c9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -149,7 +149,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {});
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index cbe75929e1f59..dabf36480f1dc 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -403,7 +403,19 @@ static OperandInfo getOperandInfo(const MachineInstr &MI,
   case RISCV::VWMULSU_VV:
   case RISCV::VWMULSU_VX:
   case RISCV::VWMULU_VV:
-  case RISCV::VWMULU_VX: {
+  case RISCV::VWMULU_VX:
+  // Vector Widening Integer Multiply-Add Instructions
+  // Destination EEW=2*SEW and EMUL=2*LMUL. Source EEW=SEW and EMUL=LMUL.
+  // A SEW-bit*SEW-bit multiply of the sources forms a 2*SEW-bit value, which
+  // is then added to the 2*SEW-bit Dest. These instructions never have a
+  // passthru operand.
+  case RISCV::VWMACCU_VV:
+  case RISCV::VWMACCU_VX:
+  case RISCV::VWMACC_VV:
+  case RISCV::VWMACC_VX:
+  case RISCV::VWMACCSU_VV:
+  case RISCV::VWMACCSU_VX:
+  case RISCV::VWMACCUS_VX: {
     unsigned Log2EEW = IsMODef ? MILog2SEW + 1 : MILog2SEW;
     RISCVII::VLMUL EMUL =
         IsMODef ? RISCVVType::twoTimesVLMUL(MIVLMul) : MIVLMul;
@@ -418,18 +430,7 @@ static OperandInfo getOperandInfo(const MachineInstr &MI,
   case RISCV::VWADD_WV:
   case RISCV::VWADD_WX:
   case RISCV::VWSUB_WV:
-  case RISCV::VWSUB_WX:
-  // Vector Widening Integer Multiply-Add Instructions
-  // Destination EEW=2*SEW and EMUL=2*LMUL. Source EEW=SEW and EMUL=LMUL.
-  // Even though the add is a 2*SEW addition, the operands of the add are the
-  // Dest which is 2*SEW and the result of the multiply which is 2*SEW.
-  case RISCV::VWMACCU_VV:
-  case RISCV::VWMACCU_VX:
-  case RISCV::VWMACC_VV:
-  case RISCV::VWMACC_VX:
-  case RISCV::VWMACCSU_VV:
-  case RISCV::VWMACCSU_VX:
-  case RISCV::VWMACCUS_VX: {
+  case RISCV::VWSUB_WX: {
     bool IsOp1 = HasPassthru ? MO.getOperandNo() == 2 : MO.getOperandNo() == 1;
     bool TwoTimes = IsMODef || IsOp1;
     unsigned Log2EEW = TwoTimes ? MILog2SEW + 1 : MILog2SEW;
@@ -498,6 +499,26 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VSUB_VX:
   case RISCV::VRSUB_VI:
   case RISCV::VRSUB_VX:
+  // Vector Bitwise Logical Instructions
+  // Vector Single-Width Shift Instructions
+  case RISCV::VAND_VI:
+  case RISCV::VAND_VV:
+  case RISCV::VAND_VX:
+  case RISCV::VOR_VI:
+  case RISCV::VOR_VV:
+  case RISCV::VOR_VX:
+  case RISCV::VXOR_VI:
+  case RISCV::VXOR_VV:
+  case RISCV::VXOR_VX:
+  case RISCV::VSLL_VI:
+  case RISCV::VSLL_VV:
+  case RISCV::VSLL_VX:
+  case RISCV::VSRL_VI:
+  case RISCV::VSRL_VV:
+  case RISCV::VSRL_VX:
+  case RISCV::VSRA_VI:
+  case RISCV::VSRA_VV:
+  case RISCV::VSRA_VX:
   // Vector Widening Integer Add/Subtract
   case RISCV::VWADDU_VV:
   case RISCV::VWADDU_VX:
@@ -524,11 +545,6 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VSEXT_VF8:
   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
   // FIXME: Add support
-  // Vector Bitwise Logical Instructions
-  // FIXME: Add support
-  // Vector Single-Width Shift Instructions
-  // FIXME: Add support
-  case RISCV::VSLL_VI:
   // Vector Narrowing Integer Right Shift Instructions
   // FIXME: Add support
   case RISCV::VNSRL_WI:
@@ -569,17 +585,29 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VWMULU_VV:
   case RISCV::VWMULU_VX:
   // Vector Single-Width Integer Multiply-Add Instructions
-  // FIXME: Add support
+  case RISCV::VMACC_VV:
+  case RISCV::VMACC_VX:
+  case RISCV::VNMSAC_VV:
+  case RISCV::VNMSAC_VX:
+  case RISCV::VMADD_VV:
+  case RISCV::VMADD_VX:
+  case RISCV::VNMSUB_VV:
+  case RISCV::VNMSUB_VX:
   // Vector Widening Integer Multiply-Add Instructions
-  // FIXME: Add support
-  case RISCV::VWMACC_VX:
+  case RISCV::VWMACCU_VV:
   case RISCV::VWMACCU_VX:
+  case RISCV::VWMACC_VV:
+  case RISCV::VWMACC_VX:
+  case RISCV::VWMACCSU_VV:
+  case RISCV::VWMACCSU_VX:
+  case RISCV::VWMACCUS_VX:
   // Vector Integer Merge Instructions
   // FIXME: Add support
   // Vector Integer Move Instructions
   // FIXME: Add support
   case RISCV::VMV_V_I:
   case RISCV::VMV_V_X:
+  case RISCV::VMV_V_V:
 
   // Vector Crypto
   case RISCV::VWSLL_VI:
@@ -660,7 +688,7 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
   // If we're not using VLMAX, then we need to be careful whether we are using
   // TA/TU when there is a non-undef Passthru. But when we are using VLMAX, it
   // does not matter whether we are using TA/TU with a non-undef Passthru, since
-  // there are no tail elements to be perserved.
+  // there are no tail elements to be preserved.
   unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
   const MachineOperand &VLOp = MI.getOperand(VLOpNum);
   if (VLOp.isReg() || VLOp.getImm() != RISCV::VLMaxSentinel) {
@@ -693,7 +721,7 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
   // lower lanes using data from higher lanes. There may be other complex
   // semantics not mentioned here that make it hard to determine whether
   // the VL can be optimized. As a result, a white-list of supported
-  // instructions is used. Over time, more instructions cam be supported
+  // instructions is used. Over time, more instructions can be supported
   // upon careful examination of their semantics under the logic in this
   // optimization.
   // TODO: Use a better approach than a white-list, such as adding
diff --git a/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp b/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp
new file mode 100644
index 0000000000000..c48a97b12e43f
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVVectorMaskDAGMutation.cpp
@@ -0,0 +1,137 @@
+//===- RISCVVectorMaskDAGMutation.cpp - RISC-V Vector Mask DAGMutation ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A schedule mutation that adds an artificial dependency between masks producer
+// instructions and masked instructions, so that we can reduce the live range
+// overlaps of mask registers.
+//
+// The reason why we need to do this:
+// 1. When tracking register pressure, we don't track physical registers.
+// 2. We have a RegisterClass for mask reigster (which is `VMV0`), but we don't
+//    use it in most RVV pseudos (only used in inline asm constraint and add/sub
+//    with carry instructions). Instead, we use physical register V0 directly
+//    and insert a `$v0 = COPY ...` before the use. And, there is a fundamental
+//    issue in register allocator when handling RegisterClass with only one
+//    physical register, so we can't simply replace V0 with VMV0.
+// 3. For mask producers, we are using VR RegisterClass (we can allocate V0-V31
+//    to it). So if V0 is not available, there are still 31 available registers
+//    out there.
+//
+// This means that the RegPressureTracker can't track the pressure of mask
+// registers correctly.
+//
+// This schedule mutation is a workaround to fix this issue.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/RISCVBaseInfo.h"
+#include "MCTargetDesc/RISCVMCTargetDesc.h"
+#include "RISCVRegisterInfo.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/CodeGen/ScheduleDAGMutation.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
+
+#define DEBUG_TYPE "machine-scheduler"
+
+namespace llvm {
+
+static inline bool isVectorMaskProducer(const MachineInstr *MI) {
+  switch (RISCV::getRVVMCOpcode(MI->getOpcode())) {
+  // Vector Mask Instructions
+  case RISCV::VMAND_MM:
+  case RISCV::VMNAND_MM:
+  case RISCV::VMANDN_MM:
+  case RISCV::VMXOR_MM:
+  case RISCV::VMOR_MM:
+  case RISCV::VMNOR_MM:
+  case RISCV::VMORN_MM:
+  case RISCV::VMXNOR_MM:
+  case RISCV::VMSBF_M:
+  case RISCV::VMSIF_M:
+  case RISCV::VMSOF_M:
+  // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
+  case RISCV::VMADC_VV:
+  case RISCV::VMADC_VX:
+  case RISCV::VMADC_VI:
+  case RISCV::VMADC_VVM:
+  case RISCV::VMADC_VXM:
+  case RISCV::VMADC_VIM:
+  case RISCV::VMSBC_VV:
+  case RISCV::VMSBC_VX:
+  case RISCV::VMSBC_VVM:
+  case RISCV::VMSBC_VXM:
+  // Vector Integer Compare Instructions
+  case RISCV::VMSEQ_VV:
+  case RISCV::VMSEQ_VX:
+  case RISCV::VMSEQ_VI:
+  case RISCV::VMSNE_VV:
+  case RISCV::VMSNE_VX:
+  case RISCV::VMSNE_VI:
+  case RISCV::VMSLT_VV:
+  case RISCV::VMSLT_VX:
+  case RISCV::VMSLTU_VV:
+  case RISCV::VMSLTU_VX:
+  case RISCV::VMSLE_VV:
+  case RISCV::VMSLE_VX:
+  case RISCV::VMSLE_VI:
+  case RISCV::VMSLEU_VV:
+  case RISCV::VMSLEU_VX:
+  case RISCV::VMSLEU_VI:
+  case RISCV::VMSGTU_VX:
+  case RISCV::VMSGTU_VI:
+  case RISCV::VMSGT_VX:
+  case RISCV::VMSGT_VI:
+  // Vector Floating-Point Compare Instructions
+  case RISCV::VMFEQ_VV:
+  case RISCV::VMFEQ_VF:
+  case RISCV::VMFNE_VV:
+  case RISCV::VMFNE_VF:
+  case RISCV::VMFLT_VV:
+  case RISCV::VMFLT_VF:
+  case RISCV::VMFLE_VV:
+  case RISCV::VMFLE_VF:
+  case RISCV::VMFGT_VF:
+  case RISCV::VMFGE_VF:
+    return true;
+  }
+  return false;
+}
+
+class RISCVVectorMaskDAGMutation : public ScheduleDAGMutation {
+private:
+  const TargetRegisterInfo *TRI;
+
+public:
+  RISCVVectorMaskDAGMutation(const TargetRegisterInfo *TRI) : TRI(TRI) {}
+
+  void apply(ScheduleDAGInstrs *DAG) override {
+    SUnit *NearestUseV0SU = nullptr;
+    for (SUnit &SU : DAG->SUnits) {
+      const MachineInstr *MI = SU.getInstr();
+      if (MI->findRegisterUseOperand(RISCV::V0, TRI))
+        NearestUseV0SU = &SU;
+
+      if (NearestUseV0SU && NearestUseV0SU != &SU && isVectorMaskProducer(MI) &&
+          // For LMUL=8 cases, there will be more possibilities to spill.
+          // FIXME: We should use RegPressureTracker to do fine-grained
+          // controls.
+          RISCVII::getLMul(MI->getDesc().TSFlags) != RISCVII::LMUL_8)
+        DAG->addEdge(&SU, SDep(NearestUseV0SU, SDep::Artificial));
+    }
+  }
+};
+
+std::unique_ptr<ScheduleDAGMutation>
+createRISCVVectorMaskDAGMutation(const TargetRegisterInfo *TRI) {
+  return std::make_unique<RISCVVectorMaskDAGMutation>(TRI);
+}
+
+} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
index 0f9a2a69e0739..342456757409a 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
@@ -76,13 +76,15 @@ getSymbolicOperandMnemonic(SPIRV::OperandCategory::OperandCategory Category,
   const SPIRV::SymbolicOperand *EnumValueInCategory =
       SPIRV::lookupSymbolicOperandByCategory(Category);
 
+  auto TableEnd = ArrayRef(SPIRV::SymbolicOperands).end();
   while (EnumValueInCategory && EnumValueInCategory->Category == Category) {
     if ((EnumValueInCategory->Value != 0) &&
         (Value & EnumValueInCategory->Value)) {
       Name += Separator + EnumValueInCategory->Mnemonic.str();
       Separator = "|";
     }
-    ++EnumValueInCategory;
+    if (++EnumValueInCategory == TableEnd)
+      break;
   }
 
   return Name;
@@ -115,15 +117,16 @@ getSymbolicOperandMaxVersion(SPIRV::OperandCategory::OperandCategory Category,
 CapabilityList
 getSymbolicOperandCapabilities(SPIRV::OperandCategory::OperandCategory Category,
                                uint32_t Value) {
+  CapabilityList Capabilities;
   const SPIRV::CapabilityEntry *Capability =
       SPIRV::lookupCapabilityByCategoryAndValue(Category, Value);
-
-  CapabilityList Capabilities;
+  auto TableEnd = ArrayRef(SPIRV::CapabilityEntries).end();
   while (Capability && Capability->Category == Category &&
          Capability->Value == Value) {
     Capabilities.push_back(
         static_cast<SPIRV::Capability::Capability>(Capability->ReqCapability));
-    ++Capability;
+    if (++Capability == TableEnd)
+      break;
   }
 
   return Capabilities;
@@ -136,12 +139,15 @@ getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension) {
           Extension, SPIRV::OperandCategory::CapabilityOperand);
 
   CapabilityList Capabilities;
+  auto TableEnd = ArrayRef(SPIRV::ExtensionEntries).end();
   while (Entry &&
-         Entry->Category == SPIRV::OperandCategory::CapabilityOperand &&
-         Entry->ReqExtension == Extension) {
-    Capabilities.push_back(
-        static_cast<SPIRV::Capability::Capability>(Entry->Value));
-    ++Entry;
+         Entry->Category == SPIRV::OperandCategory::CapabilityOperand) {
+    // Some capabilities' codes might go not in order.
+    if (Entry->ReqExtension == Extension)
+      Capabilities.push_back(
+          static_cast<SPIRV::Capability::Capability>(Entry->Value));
+    if (++Entry == TableEnd)
+      break;
   }
 
   return Capabilities;
@@ -154,11 +160,13 @@ getSymbolicOperandExtensions(SPIRV::OperandCategory::OperandCategory Category,
       SPIRV::lookupExtensionByCategoryAndValue(Category, Value);
 
   ExtensionList Extensions;
+  auto TableEnd = ArrayRef(SPIRV::ExtensionEntries).end();
   while (Extension && Extension->Category == Category &&
          Extension->Value == Value) {
     Extensions.push_back(
         static_cast<SPIRV::Extension::Extension>(Extension->ReqExtension));
-    ++Extension;
+    if (++Extension == TableEnd)
+      break;
   }
 
   return Extensions;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index 44625793e9413..2437fbb820a36 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -207,6 +207,16 @@ namespace Opcode {
 #include "SPIRVGenTables.inc"
 } // namespace Opcode
 
+namespace CooperativeMatrixLayout {
+#define GET_CooperativeMatrixLayout_DECL
+#include "SPIRVGenTables.inc"
+} // namespace CooperativeMatrixLayout
+
+namespace CooperativeMatrixOperands {
+#define GET_CooperativeMatrixOperands_DECL
+#include "SPIRVGenTables.inc"
+} // namespace CooperativeMatrixOperands
+
 struct ExtendedBuiltin {
   StringRef Name;
   InstructionSet::InstructionSet Set;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index ff8759755e517..2ee0c79b8f7c1 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -211,6 +211,34 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
           // are part of the variable value.
           printOpConstantVarOps(MI, NumFixedOps - 1, OS);
           break;
+        case SPIRV::OpCooperativeMatrixMulAddKHR: {
+          const unsigned NumOps = MI->getNumOperands();
+          if (NumFixedOps == NumOps)
+            break;
+
+          OS << ' ';
+          const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm();
+          if (MulAddOp == 0) {
+            printSymbolicOperand<
+                OperandCategory::CooperativeMatrixOperandsOperand>(
+                MI, FirstVariableIndex, OS);
+          } else {
+            std::string Buffer;
+            for (unsigned Mask = 0x1;
+                 Mask != SPIRV::CooperativeMatrixOperands::
+                             MatrixResultBFloat16ComponentsINTEL;
+                 Mask <<= 1) {
+              if (MulAddOp & Mask) {
+                if (!Buffer.empty())
+                  Buffer += '|';
+                Buffer += getSymbolicOperandMnemonic(
+                    OperandCategory::CooperativeMatrixOperandsOperand, Mask);
+              }
+            }
+            OS << Buffer;
+          }
+          break;
+        }
         default:
           printRemainingVariableOps(MI, NumFixedOps, OS);
           break;
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index 384133e7b4bd1..81b5720264425 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -37,6 +37,7 @@ void initializeSPIRVModuleAnalysisPass(PassRegistry &);
 void initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PassRegistry &);
 void initializeSPIRVPreLegalizerPass(PassRegistry &);
 void initializeSPIRVPostLegalizerPass(PassRegistry &);
+void initializeSPIRVStructurizerPass(PassRegistry &);
 void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
 void initializeSPIRVEmitNonSemanticDIPass(PassRegistry &);
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 8210e20ce5b10..4012bd7696c45 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -45,6 +45,7 @@ using namespace llvm;
 namespace {
 class SPIRVAsmPrinter : public AsmPrinter {
   unsigned NLabels = 0;
+  SmallPtrSet<const MachineBasicBlock *, 8> LabeledMBB;
 
 public:
   explicit SPIRVAsmPrinter(TargetMachine &TM,
@@ -152,13 +153,9 @@ void SPIRVAsmPrinter::outputOpFunctionEnd() {
   outputMCInst(FunctionEndInst);
 }
 
-// Emit OpFunctionEnd at the end of MF and clear BBNumToRegMap.
 void SPIRVAsmPrinter::emitFunctionBodyEnd() {
-  // Do not emit anything if it's an internal service function.
-  if (isHidden())
-    return;
-  outputOpFunctionEnd();
-  MAI->BBNumToRegMap.clear();
+  if (!isHidden())
+    outputOpFunctionEnd();
 }
 
 void SPIRVAsmPrinter::emitOpLabel(const MachineBasicBlock &MBB) {
@@ -171,6 +168,7 @@ void SPIRVAsmPrinter::emitOpLabel(const MachineBasicBlock &MBB) {
   LabelInst.addOperand(MCOperand::createReg(MAI->getOrCreateMBBRegister(MBB)));
   outputMCInst(LabelInst);
   ++NLabels;
+  LabeledMBB.insert(&MBB);
 }
 
 void SPIRVAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
@@ -267,7 +265,7 @@ void SPIRVAsmPrinter::emitInstruction(const MachineInstr *MI) {
 
   // Output OpLabel after OpFunction and OpFunctionParameter in the first MBB.
   const MachineInstr *NextMI = MI->getNextNode();
-  if (!MAI->hasMBBRegister(*MI->getParent()) && isFuncOrHeaderInstr(MI, TII) &&
+  if (!LabeledMBB.contains(MI->getParent()) && isFuncOrHeaderInstr(MI, TII) &&
       (!NextMI || !isFuncOrHeaderInstr(NextMI, TII))) {
     assert(MI->getParent()->getNumber() == MF->front().getNumber() &&
            "OpFunction is not in the front MBB of MF");
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 73dce230575d8..f4bfda4932b16 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -91,6 +91,7 @@ struct IntelSubgroupsBuiltin {
   uint32_t Opcode;
   bool IsBlock;
   bool IsWrite;
+  bool IsMedia;
 };
 
 #define GET_IntelSubgroupsBuiltins_DECL
@@ -215,6 +216,8 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) {
   // - "__spirv_ReadClockKHR"
   // - "__spirv_SubgroupBlockReadINTEL"
   // - "__spirv_SubgroupImageBlockReadINTEL"
+  // - "__spirv_SubgroupImageMediaBlockReadINTEL"
+  // - "__spirv_SubgroupImageMediaBlockWriteINTEL"
   // - "__spirv_Convert"
   // - "__spirv_UConvert"
   // - "__spirv_SConvert"
@@ -225,7 +228,9 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) {
   static const std::regex SpvWithR(
       "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageQuerySizeLod|UDotKHR|"
       "SDotKHR|SUDotKHR|SDotAccSatKHR|UDotAccSatKHR|SUDotAccSatKHR|"
-      "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|Convert|"
+      "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|"
+      "SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|"
+      "Convert|"
       "UConvert|SConvert|FConvert|SatConvert).*)_R.*");
   std::smatch Match;
   if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 2)
@@ -469,12 +474,8 @@ static Register buildLoadInst(SPIRVType *BaseType, Register PtrRegister,
                               MachineIRBuilder &MIRBuilder,
                               SPIRVGlobalRegistry *GR, LLT LowLevelType,
                               Register DestinationReg = Register(0)) {
-  MachineRegisterInfo *MRI = MIRBuilder.getMRI();
-  if (!DestinationReg.isValid()) {
-    DestinationReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
-    MRI->setType(DestinationReg, LLT::scalar(64));
-    GR->assignSPIRVTypeToVReg(BaseType, DestinationReg, MIRBuilder.getMF());
-  }
+  if (!DestinationReg.isValid())
+    DestinationReg = createVirtualRegister(BaseType, GR, MIRBuilder);
   // TODO: consider using correct address space and alignment (p0 is canonical
   // type for selection though).
   MachinePointerInfo PtrInfo = MachinePointerInfo();
@@ -1196,19 +1197,28 @@ static bool generateIntelSubgroupsInst(const SPIRV::IncomingCall *Call,
   const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
   MachineFunction &MF = MIRBuilder.getMF();
   const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
-  if (!ST->canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
+  const SPIRV::IntelSubgroupsBuiltin *IntelSubgroups =
+      SPIRV::lookupIntelSubgroupsBuiltin(Builtin->Name);
+
+  if (IntelSubgroups->IsMedia &&
+      !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {
+    std::string DiagMsg = std::string(Builtin->Name) +
+                          ": the builtin requires the following SPIR-V "
+                          "extension: SPV_INTEL_media_block_io";
+    report_fatal_error(DiagMsg.c_str(), false);
+  } else if (!IntelSubgroups->IsMedia &&
+             !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
     std::string DiagMsg = std::string(Builtin->Name) +
                           ": the builtin requires the following SPIR-V "
                           "extension: SPV_INTEL_subgroups";
     report_fatal_error(DiagMsg.c_str(), false);
   }
-  const SPIRV::IntelSubgroupsBuiltin *IntelSubgroups =
-      SPIRV::lookupIntelSubgroupsBuiltin(Builtin->Name);
 
   uint32_t OpCode = IntelSubgroups->Opcode;
   if (Call->isSpirvOp()) {
     bool IsSet = OpCode != SPIRV::OpSubgroupBlockWriteINTEL &&
-                 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL;
+                 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL &&
+                 OpCode != SPIRV::OpSubgroupImageMediaBlockWriteINTEL;
     return buildOpFromWrapper(MIRBuilder, OpCode, Call,
                               IsSet ? GR->getSPIRVTypeID(Call->ReturnType)
                                     : Register(0));
@@ -1623,8 +1633,14 @@ static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call,
     }
 
   MachineRegisterInfo *MRI = MIRBuilder.getMRI();
-  Register ResReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
-  MRI->setRegClass(ResReg, &SPIRV::iIDRegClass);
+  Register ResReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
+  if (const TargetRegisterClass *DstRC =
+          MRI->getRegClassOrNull(Call->Arguments[1])) {
+    MRI->setRegClass(ResReg, DstRC);
+    MRI->setType(ResReg, MRI->getType(Call->Arguments[1]));
+  } else {
+    MRI->setType(ResReg, LLT::scalar(64));
+  }
   GR->assignSPIRVTypeToVReg(RetType, ResReg, MIRBuilder.getMF());
   MIRBuilder.buildInstr(Opcode)
       .addDef(ResReg)
@@ -1959,15 +1975,49 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call,
   const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
   unsigned Opcode =
       SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
-  bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR;
+  bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&
+               Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&
+               Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
   unsigned ArgSz = Call->Arguments.size();
   unsigned LiteralIdx = 0;
-  if (Opcode == SPIRV::OpCooperativeMatrixLoadKHR && ArgSz > 3)
-    LiteralIdx = 3;
-  else if (Opcode == SPIRV::OpCooperativeMatrixStoreKHR && ArgSz > 4)
-    LiteralIdx = 4;
+  switch (Opcode) {
+  // Memory operand is optional and is literal.
+  case SPIRV::OpCooperativeMatrixLoadKHR:
+    LiteralIdx = ArgSz > 3 ? 3 : 0;
+    break;
+  case SPIRV::OpCooperativeMatrixStoreKHR:
+    LiteralIdx = ArgSz > 4 ? 4 : 0;
+    break;
+  case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+    LiteralIdx = ArgSz > 7 ? 7 : 0;
+    break;
+  case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+    LiteralIdx = ArgSz > 8 ? 8 : 0;
+    break;
+  // Cooperative Matrix Operands operand is optional and is literal.
+  case SPIRV::OpCooperativeMatrixMulAddKHR:
+    LiteralIdx = ArgSz > 3 ? 3 : 0;
+    break;
+  };
+
   SmallVector<uint32_t, 1> ImmArgs;
   MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+  if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+    const uint32_t CacheLevel = getConstFromIntrinsic(Call->Arguments[3], MRI);
+    auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)
+                   .addUse(Call->Arguments[0])  // pointer
+                   .addUse(Call->Arguments[1])  // rows
+                   .addUse(Call->Arguments[2])  // columns
+                   .addImm(CacheLevel)          // cache level
+                   .addUse(Call->Arguments[4]); // memory layout
+    if (ArgSz > 5)
+      MIB.addUse(Call->Arguments[5]); // stride
+    if (ArgSz > 6) {
+      const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI);
+      MIB.addImm(MemOp); // memory operand
+    }
+    return true;
+  }
   if (LiteralIdx > 0)
     ImmArgs.push_back(getConstFromIntrinsic(Call->Arguments[LiteralIdx], MRI));
   Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
@@ -2151,7 +2201,7 @@ static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call,
     const SPIRVType *PointerSizeTy = GR->getOrCreateSPIRVPointerType(
         Int32Ty, MIRBuilder, SPIRV::StorageClass::Function);
     for (unsigned I = 0; I < LocalSizeNum; ++I) {
-      Register Reg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
+      Register Reg = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
       MRI->setType(Reg, LLType);
       GR->assignSPIRVTypeToVReg(PointerSizeTy, Reg, MIRBuilder.getMF());
       auto GEPInst = MIRBuilder.buildIntrinsic(
@@ -2539,23 +2589,11 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
                                  SPIRVGlobalRegistry *GR) {
   LLVM_DEBUG(dbgs() << "Lowering builtin call: " << DemangledCall << "\n");
 
-  // SPIR-V type and return register.
-  Register ReturnRegister = OrigRet;
-  SPIRVType *ReturnType = nullptr;
-  if (OrigRetTy && !OrigRetTy->isVoidTy()) {
-    ReturnType = GR->assignTypeToVReg(OrigRetTy, ReturnRegister, MIRBuilder);
-    if (!MIRBuilder.getMRI()->getRegClassOrNull(ReturnRegister))
-      MIRBuilder.getMRI()->setRegClass(ReturnRegister,
-                                       GR->getRegClass(ReturnType));
-  } else if (OrigRetTy && OrigRetTy->isVoidTy()) {
-    ReturnRegister = MIRBuilder.getMRI()->createVirtualRegister(&IDRegClass);
-    MIRBuilder.getMRI()->setType(ReturnRegister, LLT::scalar(64));
-    ReturnType = GR->assignTypeToVReg(OrigRetTy, ReturnRegister, MIRBuilder);
-  }
-
   // Lookup the builtin in the TableGen records.
+  SPIRVType *SpvType = GR->getSPIRVTypeForVReg(OrigRet);
+  assert(SpvType && "Inconsistent return register: expected valid type info");
   std::unique_ptr<const IncomingCall> Call =
-      lookupBuiltin(DemangledCall, Set, ReturnRegister, ReturnType, Args);
+      lookupBuiltin(DemangledCall, Set, OrigRet, SpvType, Args);
 
   if (!Call) {
     LLVM_DEBUG(dbgs() << "Builtin record was not found!\n");
@@ -2632,16 +2670,7 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
   return false;
 }
 
-Type *parseBuiltinCallArgumentBaseType(const StringRef DemangledCall,
-                                       unsigned ArgIdx, LLVMContext &Ctx) {
-  SmallVector<StringRef, 10> BuiltinArgsTypeStrs;
-  StringRef BuiltinArgs =
-      DemangledCall.slice(DemangledCall.find('(') + 1, DemangledCall.find(')'));
-  BuiltinArgs.split(BuiltinArgsTypeStrs, ',', -1, false);
-  if (ArgIdx >= BuiltinArgsTypeStrs.size())
-    return nullptr;
-  StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();
-
+Type *parseBuiltinCallArgumentType(StringRef TypeStr, LLVMContext &Ctx) {
   // Parse strings representing OpenCL builtin types.
   if (hasBuiltinTypePrefix(TypeStr)) {
     // OpenCL builtin types in demangled call strings have the following format:
@@ -2685,6 +2714,29 @@ Type *parseBuiltinCallArgumentBaseType(const StringRef DemangledCall,
   return BaseType;
 }
 
+bool parseBuiltinTypeStr(SmallVector<StringRef, 10> &BuiltinArgsTypeStrs,
+                         const StringRef DemangledCall, LLVMContext &Ctx) {
+  auto Pos1 = DemangledCall.find('(');
+  if (Pos1 == StringRef::npos)
+    return false;
+  auto Pos2 = DemangledCall.find(')');
+  if (Pos2 == StringRef::npos || Pos1 > Pos2)
+    return false;
+  DemangledCall.slice(Pos1 + 1, Pos2)
+      .split(BuiltinArgsTypeStrs, ',', -1, false);
+  return true;
+}
+
+Type *parseBuiltinCallArgumentBaseType(const StringRef DemangledCall,
+                                       unsigned ArgIdx, LLVMContext &Ctx) {
+  SmallVector<StringRef, 10> BuiltinArgsTypeStrs;
+  parseBuiltinTypeStr(BuiltinArgsTypeStrs, DemangledCall, Ctx);
+  if (ArgIdx >= BuiltinArgsTypeStrs.size())
+    return nullptr;
+  StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();
+  return parseBuiltinCallArgumentType(TypeStr, Ctx);
+}
+
 struct BuiltinType {
   StringRef Name;
   uint32_t Opcode;
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
index d07fc7c6ca874..42b452db8b9fb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
@@ -56,6 +56,9 @@ mapBuiltinToOpcode(const StringRef DemangledCall,
 /// \p ArgIdx is the index of the argument to parse.
 Type *parseBuiltinCallArgumentBaseType(const StringRef DemangledCall,
                                        unsigned ArgIdx, LLVMContext &Ctx);
+bool parseBuiltinTypeStr(SmallVector<StringRef, 10> &BuiltinArgsTypeStrs,
+                         const StringRef DemangledCall, LLVMContext &Ctx);
+Type *parseBuiltinCallArgumentType(StringRef TypeStr, LLVMContext &Ctx);
 
 /// Translates a string representing a SPIR-V or OpenCL builtin type to a
 /// TargetExtType that can be further lowered with lowerBuiltinType().
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index e0dfc25723b0c..af3901c0e621e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -690,11 +690,18 @@ defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToLocal", OpenCL
 defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToPrivate", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>;
 
 // Cooperative Matrix builtin records:
-defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadKHR", OpenCL_std, CoopMatr, 2, 0, OpCooperativeMatrixLoadKHR>;
-defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreKHR", OpenCL_std, CoopMatr, 3, 0, OpCooperativeMatrixStoreKHR>;
-defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixMulAddKHR", OpenCL_std, CoopMatr, 3, 0, OpCooperativeMatrixMulAddKHR>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadKHR", OpenCL_std, CoopMatr, 2, 4, OpCooperativeMatrixLoadKHR>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreKHR", OpenCL_std, CoopMatr, 3, 5, OpCooperativeMatrixStoreKHR>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixMulAddKHR", OpenCL_std, CoopMatr, 3, 4, OpCooperativeMatrixMulAddKHR>;
 defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLengthKHR", OpenCL_std, CoopMatr, 1, 1, OpCooperativeMatrixLengthKHR>;
 
+// Cooperative Matrix Intel builtin records:
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixPrefetchINTEL", OpenCL_std, CoopMatr, 5, 7, OpCooperativeMatrixPrefetchINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadCheckedINTEL", OpenCL_std, CoopMatr, 6, 8, OpCooperativeMatrixLoadCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreCheckedINTEL", OpenCL_std, CoopMatr, 7, 9, OpCooperativeMatrixStoreCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixConstructCheckedINTEL", OpenCL_std, CoopMatr, 5, 5, OpCooperativeMatrixConstructCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixGetElementCoordINTEL", OpenCL_std, CoopMatr, 2, 2, OpCooperativeMatrixGetElementCoordINTEL>;
+
 //===----------------------------------------------------------------------===//
 // Class defining a work/sub group builtin that should be translated into a
 // SPIR-V instruction using the defined properties.
@@ -1156,14 +1163,19 @@ class IntelSubgroupsBuiltin<string name, Op operation> {
   string Name = name;
   Op Opcode = operation;
   bit IsBlock = !or(!eq(operation, OpSubgroupBlockReadINTEL),
-                    !eq(operation, OpSubgroupBlockWriteINTEL));
-  bit IsWrite = !eq(operation, OpSubgroupBlockWriteINTEL);
+                    !eq(operation, OpSubgroupBlockWriteINTEL),
+                    !eq(operation, OpSubgroupImageMediaBlockReadINTEL),
+                    !eq(operation, OpSubgroupImageMediaBlockWriteINTEL));
+  bit IsWrite = !or(!eq(operation, OpSubgroupBlockWriteINTEL),
+                    !eq(operation, OpSubgroupImageMediaBlockWriteINTEL));
+  bit IsMedia = !or(!eq(operation, OpSubgroupImageMediaBlockReadINTEL),
+                    !eq(operation, OpSubgroupImageMediaBlockWriteINTEL));
 }
 
 // Table gathering all the Intel sub group builtins.
 def IntelSubgroupsBuiltins : GenericTable {
   let FilterClass = "IntelSubgroupsBuiltin";
-  let Fields = ["Name", "Opcode", "IsBlock", "IsWrite"];
+  let Fields = ["Name", "Opcode", "IsBlock", "IsWrite", "IsMedia"];
 }
 
 // Function to lookup group builtins by their name and set.
@@ -1185,23 +1197,30 @@ defm : DemangledIntelSubgroupsBuiltin<"shuffle_down", 3, 3, OpSubgroupShuffleDow
 defm : DemangledIntelSubgroupsBuiltin<"shuffle_up", 3, 3, OpSubgroupShuffleUpINTEL>;
 defm : DemangledIntelSubgroupsBuiltin<"shuffle_xor", 2, 2, OpSubgroupShuffleXorINTEL>;
 foreach i = ["", "2", "4", "8"] in {
-  // cl_intel_subgroups
+  // cl_intel_subgroups, cl_intel_subgroup_local_block_io
   defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_read",  i), 1, 2, OpSubgroupBlockReadINTEL>;
   defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_write", i), 2, 3, OpSubgroupBlockWriteINTEL>;
-  // cl_intel_subgroups_short
+  // cl_intel_subgroups_short, cl_intel_subgroup_local_block_io
   defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_read_ui",  i), 1, 2, OpSubgroupBlockReadINTEL>;
   defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_write_ui", i), 2, 3, OpSubgroupBlockWriteINTEL>;
+  // cl_intel_media_block_io
+  defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_read",  i), 4, 4, OpSubgroupImageMediaBlockReadINTEL>;
+  defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_read_ui",  i), 4, 4, OpSubgroupImageMediaBlockReadINTEL>;
+  defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_write", i), 5, 5, OpSubgroupImageMediaBlockWriteINTEL>;
+  defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_write_ui", i), 5, 5, OpSubgroupImageMediaBlockWriteINTEL>;
 }
-// cl_intel_subgroups_char, cl_intel_subgroups_short, cl_intel_subgroups_long
+// cl_intel_subgroups_char, cl_intel_subgroups_short, cl_intel_subgroups_long, cl_intel_media_block_io, cl_intel_subgroup_local_block_io
 foreach i = ["", "2", "4", "8", "16"] in {
   foreach j = ["c", "s", "l"] in {
     defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_read_u", j,  i), 1, 2, OpSubgroupBlockReadINTEL>;
     defm : DemangledIntelSubgroupsBuiltin<!strconcat("block_write_u", j, i), 2, 3, OpSubgroupBlockWriteINTEL>;
+    defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_read_u", j, i), 4, 4, OpSubgroupImageMediaBlockReadINTEL>;
+    defm : DemangledIntelSubgroupsBuiltin<!strconcat("media_block_write_u", j, i), 5, 5, OpSubgroupImageMediaBlockWriteINTEL>;
   }
 }
 // OpSubgroupImageBlockReadINTEL and OpSubgroupImageBlockWriteINTEL are to be resolved later on (in code)
 
-// Multiclass used to define builtin wrappers for the SPV_INTEL_subgroups extension.
+// Multiclass used to define builtin wrappers for the SPV_INTEL_subgroups and the SPV_INTEL_media_block_io extensions.
 multiclass DemangledIntelSubgroupsBuiltinWrapper<string name, bits<8> numArgs, Op operation> {
   def : DemangledBuiltin<!strconcat("__spirv_", name), OpenCL_std, IntelSubgroups, numArgs, numArgs>;
   def : IntelSubgroupsBuiltin<!strconcat("__spirv_", name), operation>;
@@ -1215,6 +1234,8 @@ defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupBlockReadINTEL", 1, OpSubg
 defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupBlockWriteINTEL", 2, OpSubgroupBlockWriteINTEL>;
 defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupImageBlockReadINTEL", 2, OpSubgroupImageBlockReadINTEL>;
 defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupImageBlockWriteINTEL", 3, OpSubgroupImageBlockWriteINTEL>;
+defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupImageMediaBlockReadINTEL", 4, OpSubgroupImageMediaBlockReadINTEL>;
+defm : DemangledIntelSubgroupsBuiltinWrapper<"SubgroupImageMediaBlockWriteINTEL", 5, OpSubgroupImageMediaBlockWriteINTEL>;
 
 //===----------------------------------------------------------------------===//
 // Class defining a builtin for group operations within uniform control flow.
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 3c5397319aaf2..fa37313f8247c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -65,7 +65,8 @@ bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
 }
 
 // Based on the LLVM function attributes, get a SPIR-V FunctionControl.
-static uint32_t getFunctionControl(const Function &F) {
+static uint32_t getFunctionControl(const Function &F,
+                                   const SPIRVSubtarget *ST) {
   MemoryEffects MemEffects = F.getMemoryEffects();
 
   uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None);
@@ -80,6 +81,11 @@ static uint32_t getFunctionControl(const Function &F) {
   else if (MemEffects.onlyReadsMemory())
     FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const);
 
+  if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_optnone) ||
+      ST->canUseExtension(SPIRV::Extension::SPV_EXT_optnone))
+    if (F.hasFnAttribute(Attribute::OptimizeNone))
+      FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::OptNoneEXT);
+
   return FuncControl;
 }
 
@@ -310,7 +316,7 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
 
       if (Arg.hasName())
         buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder);
-      if (isPointerTy(Arg.getType())) {
+      if (isPointerTyOrWrapper(Arg.getType())) {
         auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes());
         if (DerefBytes != 0)
           buildOpDecorate(VRegs[i][0], MIRBuilder,
@@ -346,6 +352,12 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
         buildOpDecorate(VRegs[i][0], MIRBuilder,
                         SPIRV::Decoration::FuncParamAttr, {Attr});
       }
+      if (Arg.hasAttribute(Attribute::StructRet)) {
+        auto Attr =
+            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sret);
+        buildOpDecorate(VRegs[i][0], MIRBuilder,
+                        SPIRV::Decoration::FuncParamAttr, {Attr});
+      }
 
       if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
         std::vector<SPIRV::Decoration::Decoration> ArgTypeQualDecs =
@@ -397,7 +409,7 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, RetTy, ArgTypeVRegs);
   SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
       FTy, RetTy, ArgTypeVRegs, MIRBuilder);
-  uint32_t FuncControl = getFunctionControl(F);
+  uint32_t FuncControl = getFunctionControl(F, ST);
 
   // Add OpFunction instruction
   MachineInstrBuilder MB = MIRBuilder.buildInstr(SPIRV::OpFunction)
@@ -427,10 +439,8 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
 
   // Handle entry points and function linkage.
   if (isEntryPoint(F)) {
-    const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
-    auto executionModel = getExecutionModel(STI, F);
     auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint)
-                   .addImm(static_cast<uint32_t>(executionModel))
+                   .addImm(static_cast<uint32_t>(getExecutionModel(*ST, F)))
                    .addUse(FuncVReg);
     addStringImm(F.getName(), MIB);
   } else if (F.getLinkage() != GlobalValue::InternalLinkage &&
@@ -539,6 +549,23 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   if (isFunctionDecl && !DemangledName.empty() &&
       (canUseGLSL || canUseOpenCL)) {
+    if (ResVReg.isValid()) {
+      if (!GR->getSPIRVTypeForVReg(ResVReg)) {
+        const Type *RetTy = OrigRetTy;
+        if (auto *PtrRetTy = dyn_cast<PointerType>(OrigRetTy)) {
+          const Value *OrigValue = Info.OrigRet.OrigValue;
+          if (!OrigValue)
+            OrigValue = Info.CB;
+          if (OrigValue)
+            if (Type *ElemTy = GR->findDeducedElementType(OrigValue))
+              RetTy =
+                  TypedPointerType::get(ElemTy, PtrRetTy->getAddressSpace());
+        }
+        setRegClassType(ResVReg, RetTy, GR, MIRBuilder);
+      }
+    } else {
+      ResVReg = createVirtualRegister(OrigRetTy, GR, MIRBuilder);
+    }
     SmallVector<Register, 8> ArgVRegs;
     for (auto Arg : Info.OrigArgs) {
       assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index 186bccc481a8a..fb05c1fdbd1e3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -42,12 +42,17 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
         {"SPV_INTEL_global_variable_host_access",
          SPIRV::Extension::Extension::SPV_INTEL_global_variable_host_access},
         {"SPV_INTEL_optnone", SPIRV::Extension::Extension::SPV_INTEL_optnone},
+        {"SPV_EXT_optnone", SPIRV::Extension::Extension::SPV_EXT_optnone},
         {"SPV_INTEL_usm_storage_classes",
          SPIRV::Extension::Extension::SPV_INTEL_usm_storage_classes},
         {"SPV_INTEL_split_barrier",
          SPIRV::Extension::Extension::SPV_INTEL_split_barrier},
         {"SPV_INTEL_subgroups",
          SPIRV::Extension::Extension::SPV_INTEL_subgroups},
+        {"SPV_INTEL_media_block_io",
+         SPIRV::Extension::Extension::SPV_INTEL_media_block_io},
+        {"SPV_INTEL_joint_matrix",
+         SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
         {"SPV_KHR_uniform_group_instructions",
          SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions},
         {"SPV_KHR_no_integer_wrap_decoration",
diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
index b82c2538a8136..48df845efd76b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
@@ -69,16 +69,31 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph(
         MachineOperand *RegOp = &VRegDef->getOperand(0);
         if (Reg2Entry.count(RegOp) == 0 &&
             (MI->getOpcode() != SPIRV::OpVariable || i != 3)) {
-          std::string DiagMsg;
-          raw_string_ostream OS(DiagMsg);
-          OS << "Unexpected pattern while building a dependency "
-                "graph.\nInstruction: ";
-          MI->print(OS);
-          OS << "Operand: ";
-          Op.print(OS);
-          OS << "\nOperand definition: ";
-          VRegDef->print(OS);
-          report_fatal_error(DiagMsg.c_str());
+          // try to repair the unexpected code pattern
+          bool IsFixed = false;
+          if (VRegDef->getOpcode() == TargetOpcode::G_CONSTANT &&
+              RegOp->isReg() && MRI.getType(RegOp->getReg()).isScalar()) {
+            const Constant *C = VRegDef->getOperand(1).getCImm();
+            add(C, MI->getParent()->getParent(), RegOp->getReg());
+            auto Iter = CT.Storage.find(C);
+            if (Iter != CT.Storage.end()) {
+              SPIRV::DTSortableEntry &MissedEntry = Iter->second;
+              Reg2Entry[RegOp] = &MissedEntry;
+              IsFixed = true;
+            }
+          }
+          if (!IsFixed) {
+            std::string DiagMsg;
+            raw_string_ostream OS(DiagMsg);
+            OS << "Unexpected pattern while building a dependency "
+                  "graph.\nInstruction: ";
+            MI->print(OS);
+            OS << "Operand: ";
+            Op.print(OS);
+            OS << "\nOperand definition: ";
+            VRegDef->print(OS);
+            report_fatal_error(DiagMsg.c_str());
+          }
         }
         if (Reg2Entry.count(RegOp))
           E->addDep(Reg2Entry[RegOp]);
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 5d37414240316..66ef87162d938 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -17,6 +17,7 @@
 #include "SPIRVSubtarget.h"
 #include "SPIRVTargetMachine.h"
 #include "SPIRVUtils.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstVisitor.h"
@@ -67,7 +68,7 @@ class SPIRVEmitIntrinsics
       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
   SPIRVTargetMachine *TM = nullptr;
   SPIRVGlobalRegistry *GR = nullptr;
-  Function *F = nullptr;
+  Function *CurrF = nullptr;
   bool TrackConstants = true;
   bool HaveFunPtrs = false;
   DenseMap<Instruction *, Constant *> AggrConsts;
@@ -75,9 +76,37 @@ class SPIRVEmitIntrinsics
   DenseSet<Instruction *> AggrStores;
   SPIRV::InstructionSet::InstructionSet InstrSet;
 
+  // map of function declarations to <pointer arg index => element type>
+  DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
+
   // a register of Instructions that don't have a complete type definition
-  DenseMap<Value *, unsigned> UncompleteTypeInfo;
-  SmallVector<Value *> PostprocessWorklist;
+  bool CanTodoType = true;
+  unsigned TodoTypeSz = 0;
+  DenseMap<Value *, bool> TodoType;
+  void insertTodoType(Value *Op) {
+    // TODO: add isa<CallInst>(Op) to no-insert
+    if (CanTodoType && !isa<GetElementPtrInst>(Op)) {
+      auto It = TodoType.try_emplace(Op, true);
+      if (It.second)
+        ++TodoTypeSz;
+    }
+  }
+  void eraseTodoType(Value *Op) {
+    auto It = TodoType.find(Op);
+    if (It != TodoType.end() && It->second) {
+      TodoType[Op] = false;
+      --TodoTypeSz;
+    }
+  }
+  bool isTodoType(Value *Op) {
+    if (isa<GetElementPtrInst>(Op))
+      return false;
+    auto It = TodoType.find(Op);
+    return It != TodoType.end() && It->second;
+  }
+  // a register of Instructions that were visited by deduceOperandElementType()
+  // to validate operand types with an instruction
+  std::unordered_set<Instruction *> TypeValidated;
 
   // well known result types of builtins
   enum WellKnownTypes { Event };
@@ -86,7 +115,8 @@ class SPIRVEmitIntrinsics
   Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
   Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
   Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
-                                bool UnknownElemTypeI8);
+                                bool UnknownElemTypeI8,
+                                bool IgnoreKnownType = false);
   Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
                                      bool UnknownElemTypeI8);
   Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
@@ -105,8 +135,10 @@ class SPIRVEmitIntrinsics
                                bool UnknownElemTypeI8);
 
   // deduce Types of operands of the Instruction if possible
-  void deduceOperandElementType(Instruction *I, Instruction *AskOp = 0,
-                                Type *AskTy = 0, CallInst *AssignCI = 0);
+  void deduceOperandElementType(Instruction *I,
+                                SmallPtrSet<Instruction *, 4> *UncompleteRets,
+                                const SmallPtrSet<Value *, 4> *AskOps = nullptr,
+                                bool IsPostprocessing = false);
 
   void preprocessCompositeConstants(IRBuilder<> &B);
   void preprocessUndefs(IRBuilder<> &B);
@@ -122,6 +154,9 @@ class SPIRVEmitIntrinsics
     return B.CreateIntrinsic(IntrID, {Types}, Args);
   }
 
+  Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
+                        bool IsPostprocessing);
+
   void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg);
   void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg);
   void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType);
@@ -145,13 +180,41 @@ class SPIRVEmitIntrinsics
   Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
   Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
                                   std::unordered_set<Function *> &FVisited);
-  void replaceWithPtrcasted(Instruction *CI, Type *NewElemTy, Type *KnownElemTy,
-                            CallInst *AssignCI);
+
+  bool deduceOperandElementTypeCalledFunction(
+      CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
+      Type *&KnownElemTy);
+  void deduceOperandElementTypeFunctionPointer(
+      CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
+      Type *&KnownElemTy, bool IsPostprocessing);
+  bool deduceOperandElementTypeFunctionRet(
+      Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets,
+      const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
+      Type *&KnownElemTy, Value *Op, Function *F);
+
+  CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
+  void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
+                                   DenseMap<Function *, CallInst *> Ptrcasts);
+  void propagateElemType(Value *Op, Type *ElemTy,
+                         DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
+  void
+  propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
+                       DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
+  void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
+                            DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
+                            std::unordered_set<Value *> &Visited,
+                            DenseMap<Function *, CallInst *> Ptrcasts);
+
   void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
+  void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
+                                  Instruction *Dest, bool DeleteOld = true);
+
+  void applyDemangledPtrArgTypes(IRBuilder<> &B);
 
   bool runOnFunction(Function &F);
-  bool postprocessTypes();
+  bool postprocessTypes(Module &M);
   bool processFunctionPointers(Module &M);
+  void parseFunDeclarations(Module &M);
 
 public:
   static char ID;
@@ -203,10 +266,8 @@ bool expectIgnoredInIRTranslation(const Instruction *I) {
 }
 
 bool allowEmitFakeUse(const Value *Arg) {
-  if (const auto *II = dyn_cast<IntrinsicInst>(Arg))
-    if (Function *F = II->getCalledFunction())
-      if (F->getName().starts_with("llvm.spv."))
-        return false;
+  if (isSpvIntrinsic(Arg))
+    return false;
   if (dyn_cast<AtomicCmpXchgInst>(Arg) || dyn_cast<InsertValueInst>(Arg) ||
       dyn_cast<UndefValue>(Arg))
     return false;
@@ -273,6 +334,17 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
                        false);
 }
 
+static void emitAssignName(Instruction *I, IRBuilder<> &B) {
+  if (!I->hasName() || I->getType()->isAggregateType() ||
+      expectIgnoredInIRTranslation(I))
+    return;
+  reportFatalOnTokenType(I);
+  setInsertPointAfterDef(B, I);
+  std::vector<Value *> Args = {I};
+  addStringImm(I->getName(), B, Args);
+  B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
+}
+
 void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
                                              bool DeleteOld) {
   Src->replaceAllUsesWith(Dest);
@@ -280,17 +352,23 @@ void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
   GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld);
   GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld);
   // Update uncomplete type records if any
-  auto It = UncompleteTypeInfo.find(Src);
-  if (It == UncompleteTypeInfo.end())
-    return;
-  if (DeleteOld) {
-    unsigned Pos = It->second;
-    UncompleteTypeInfo.erase(Src);
-    UncompleteTypeInfo[Dest] = Pos;
-    PostprocessWorklist[Pos] = Dest;
-  } else {
-    UncompleteTypeInfo[Dest] = PostprocessWorklist.size();
-    PostprocessWorklist.push_back(Dest);
+  if (isTodoType(Src)) {
+    if (DeleteOld)
+      eraseTodoType(Src);
+    insertTodoType(Dest);
+  }
+}
+
+void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
+                                                     Instruction *Src,
+                                                     Instruction *Dest,
+                                                     bool DeleteOld) {
+  replaceAllUsesWith(Src, Dest, DeleteOld);
+  std::string Name = Src->hasName() ? Src->getName().str() : "";
+  Src->eraseFromParent();
+  if (!Name.empty()) {
+    Dest->setName(Name);
+    emitAssignName(Dest, B);
   }
 }
 
@@ -314,8 +392,11 @@ static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
 
 // Reconstruct type with nested element types according to deduced type info.
 // Return nullptr if no detailed type info is available.
-static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) {
+Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
+                                           bool IsPostprocessing) {
   Type *Ty = Op->getType();
+  if (auto *OpI = dyn_cast<Instruction>(Op))
+    Ty = restoreMutatedType(GR, OpI, Ty);
   if (!isUntypedPointerTy(Ty))
     return Ty;
   // try to find the pointee type
@@ -323,10 +404,17 @@ static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) {
     return getTypedPointerWrapper(NestedTy, getPointerAddressSpace(Ty));
   // not a pointer according to the type info (e.g., Event object)
   CallInst *CI = GR->findAssignPtrTypeInstr(Op);
-  if (!CI)
-    return nullptr;
-  MetadataAsValue *MD = cast<MetadataAsValue>(CI->getArgOperand(1));
-  return cast<ConstantAsMetadata>(MD->getMetadata())->getType();
+  if (CI) {
+    MetadataAsValue *MD = cast<MetadataAsValue>(CI->getArgOperand(1));
+    return cast<ConstantAsMetadata>(MD->getMetadata())->getType();
+  }
+  if (UnknownElemTypeI8) {
+    if (!IsPostprocessing)
+      insertTodoType(Op);
+    return getTypedPointerWrapper(IntegerType::getInt8Ty(Op->getContext()),
+                                  getPointerAddressSpace(Ty));
+  }
+  return nullptr;
 }
 
 void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
@@ -354,7 +442,7 @@ void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,
   Value *OfType = PoisonValue::get(ElemTy);
   CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg);
   if (AssignPtrTyCI == nullptr ||
-      AssignPtrTyCI->getParent()->getParent() != F) {
+      AssignPtrTyCI->getParent()->getParent() != CurrF) {
     AssignPtrTyCI = buildIntrWithMD(
         Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,
         {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
@@ -379,8 +467,97 @@ void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg,
   GR->addDeducedElementType(Arg, ElemTy);
 }
 
+CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
+                                               Type *ElemTy) {
+  IRBuilder<> B(Op->getContext());
+  if (auto *OpI = dyn_cast<Instruction>(Op)) {
+    // spv_ptrcast's argument Op denotes an instruction that generates
+    // a value, and we may use getInsertionPointAfterDef()
+    setInsertPointAfterDef(B, OpI);
+  } else if (auto *OpA = dyn_cast<Argument>(Op)) {
+    B.SetInsertPointPastAllocas(OpA->getParent());
+    B.SetCurrentDebugLocation(DebugLoc());
+  } else {
+    B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
+  }
+  Type *OpTy = Op->getType();
+  SmallVector<Type *, 2> Types = {OpTy, OpTy};
+  SmallVector<Value *, 2> Args = {Op, buildMD(PoisonValue::get(ElemTy)),
+                                  B.getInt32(getPointerAddressSpace(OpTy))};
+  CallInst *PtrCasted =
+      B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+  buildAssignPtr(B, ElemTy, PtrCasted);
+  return PtrCasted;
+}
+
+void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
+    Value *Op, Type *ElemTy, Instruction *I,
+    DenseMap<Function *, CallInst *> Ptrcasts) {
+  Function *F = I->getParent()->getParent();
+  CallInst *PtrCastedI = nullptr;
+  auto It = Ptrcasts.find(F);
+  if (It == Ptrcasts.end()) {
+    PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
+    Ptrcasts[F] = PtrCastedI;
+  } else {
+    PtrCastedI = It->second;
+  }
+  I->replaceUsesOfWith(Op, PtrCastedI);
+}
+
+void SPIRVEmitIntrinsics::propagateElemType(
+    Value *Op, Type *ElemTy,
+    DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
+  DenseMap<Function *, CallInst *> Ptrcasts;
+  SmallVector<User *> Users(Op->users());
+  for (auto *U : Users) {
+    if (!isa<Instruction>(U) || isSpvIntrinsic(U))
+      continue;
+    if (!VisitedSubst.insert(std::make_pair(U, Op)).second)
+      continue;
+    Instruction *UI = dyn_cast<Instruction>(U);
+    // If the instruction was validated already, we need to keep it valid by
+    // keeping current Op type.
+    if (isa<GetElementPtrInst>(UI) ||
+        TypeValidated.find(UI) != TypeValidated.end())
+      replaceUsesOfWithSpvPtrcast(Op, ElemTy, UI, Ptrcasts);
+  }
+}
+
+void SPIRVEmitIntrinsics::propagateElemTypeRec(
+    Value *Op, Type *PtrElemTy, Type *CastElemTy,
+    DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
+  std::unordered_set<Value *> Visited;
+  DenseMap<Function *, CallInst *> Ptrcasts;
+  propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
+                       Ptrcasts);
+}
+
+void SPIRVEmitIntrinsics::propagateElemTypeRec(
+    Value *Op, Type *PtrElemTy, Type *CastElemTy,
+    DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
+    std::unordered_set<Value *> &Visited,
+    DenseMap<Function *, CallInst *> Ptrcasts) {
+  if (!Visited.insert(Op).second)
+    return;
+  SmallVector<User *> Users(Op->users());
+  for (auto *U : Users) {
+    if (!isa<Instruction>(U) || isSpvIntrinsic(U))
+      continue;
+    if (!VisitedSubst.insert(std::make_pair(U, Op)).second)
+      continue;
+    Instruction *UI = dyn_cast<Instruction>(U);
+    // If the instruction was validated already, we need to keep it valid by
+    // keeping current Op type.
+    if (isa<GetElementPtrInst>(UI) ||
+        TypeValidated.find(UI) != TypeValidated.end())
+      replaceUsesOfWithSpvPtrcast(Op, CastElemTy, UI, Ptrcasts);
+  }
+}
+
 // Set element pointer type to the given value of ValueTy and tries to
 // specify this type further (recursively) by Operand value, if needed.
+
 Type *
 SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
                                                   bool UnknownElemTypeI8) {
@@ -455,23 +632,22 @@ void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
   if (isUntypedPointerTy(RefTy)) {
     if (!UnknownElemTypeI8)
       return;
-    if (auto *I = dyn_cast<Instruction>(Op)) {
-      UncompleteTypeInfo[I] = PostprocessWorklist.size();
-      PostprocessWorklist.push_back(I);
-    }
+    insertTodoType(Op);
   }
   Ty = RefTy;
 }
 
 Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
-    Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
+    Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
+    bool IgnoreKnownType) {
   // allow to pass nullptr as an argument
   if (!I)
     return nullptr;
 
   // maybe already known
-  if (Type *KnownTy = GR->findDeducedElementType(I))
-    return KnownTy;
+  if (!IgnoreKnownType)
+    if (Type *KnownTy = GR->findDeducedElementType(I))
+      return KnownTy;
 
   // maybe a cycle
   if (!Visited.insert(I).second)
@@ -483,7 +659,22 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
   if (auto *Ref = dyn_cast<AllocaInst>(I)) {
     maybeAssignPtrType(Ty, I, Ref->getAllocatedType(), UnknownElemTypeI8);
   } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
-    Ty = Ref->getResultElementType();
+    // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
+    // useful here
+    if (isNestedPointer(Ref->getSourceElementType())) {
+      Ty = Ref->getSourceElementType();
+      for (Use &U : drop_begin(Ref->indices()))
+        Ty = GetElementPtrInst::getTypeAtIndex(Ty, U.get());
+    } else {
+      Ty = Ref->getResultElementType();
+    }
+  } else if (auto *Ref = dyn_cast<LoadInst>(I)) {
+    Value *Op = Ref->getPointerOperand();
+    Type *KnownTy = GR->findDeducedElementType(Op);
+    if (!KnownTy)
+      KnownTy = Op->getType();
+    if (Type *ElemTy = getPointeeType(KnownTy))
+      maybeAssignPtrType(Ty, I, ElemTy, UnknownElemTypeI8);
   } else if (auto *Ref = dyn_cast<GlobalValue>(I)) {
     Ty = deduceElementTypeByValueDeep(
         Ref->getValueType(),
@@ -559,7 +750,7 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
   }
 
   // remember the found relationship
-  if (Ty) {
+  if (Ty && !IgnoreKnownType) {
     // specify nested types if needed, otherwise return unchanged
     GR->addDeducedElementType(I, Ty);
   }
@@ -601,7 +792,7 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
         if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
           if (Type *NestedTy =
                   deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
-            Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
+            Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
         } else {
           Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
                                       UnknownElemTypeI8);
@@ -622,7 +813,7 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
       if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
         if (Type *NestedTy =
                 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
-          Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
+          Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
       } else {
         Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
                                     UnknownElemTypeI8);
@@ -661,10 +852,7 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
     return Ty;
   if (!UnknownElemTypeI8)
     return nullptr;
-  if (auto *Instr = dyn_cast<Instruction>(I)) {
-    UncompleteTypeInfo[Instr] = PostprocessWorklist.size();
-    PostprocessWorklist.push_back(Instr);
-  }
+  insertTodoType(I);
   return IntegerType::getInt8Ty(I->getContext());
 }
 
@@ -683,10 +871,9 @@ static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
 
 // Try to deduce element type for a call base. Returns false if this is an
 // indirect function invocation, and true otherwise.
-static bool deduceOperandElementTypeCalledFunction(
-    SPIRVGlobalRegistry *GR, Instruction *I,
-    SPIRV::InstructionSet::InstructionSet InstrSet, CallInst *CI,
-    SmallVector<std::pair<Value *, unsigned>> &Ops, Type *&KnownElemTy) {
+bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
+    CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
+    Type *&KnownElemTy) {
   Function *CalledF = CI->getCalledFunction();
   if (!CalledF)
     return false;
@@ -726,7 +913,7 @@ static bool deduceOperandElementTypeCalledFunction(
       case SPIRV::OpAtomicUMax:
       case SPIRV::OpAtomicSMin:
       case SPIRV::OpAtomicSMax: {
-        KnownElemTy = getAtomicElemTy(GR, I, Op);
+        KnownElemTy = getAtomicElemTy(GR, CI, Op);
         if (!KnownElemTy)
           return true;
         Ops.push_back(std::make_pair(Op, 0));
@@ -738,51 +925,105 @@ static bool deduceOperandElementTypeCalledFunction(
 }
 
 // Try to deduce element type for a function pointer.
-static void deduceOperandElementTypeFunctionPointer(
-    SPIRVGlobalRegistry *GR, Instruction *I, CallInst *CI,
-    SmallVector<std::pair<Value *, unsigned>> &Ops, Type *&KnownElemTy) {
+void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
+    CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
+    Type *&KnownElemTy, bool IsPostprocessing) {
   Value *Op = CI->getCalledOperand();
   if (!Op || !isPointerTy(Op->getType()))
     return;
   Ops.push_back(std::make_pair(Op, std::numeric_limits<unsigned>::max()));
   FunctionType *FTy = CI->getFunctionType();
-  bool IsNewFTy = false;
+  bool IsNewFTy = false, IsUncomplete = false;
   SmallVector<Type *, 4> ArgTys;
   for (Value *Arg : CI->args()) {
     Type *ArgTy = Arg->getType();
-    if (ArgTy->isPointerTy())
+    if (ArgTy->isPointerTy()) {
       if (Type *ElemTy = GR->findDeducedElementType(Arg)) {
         IsNewFTy = true;
-        ArgTy = TypedPointerType::get(ElemTy, getPointerAddressSpace(ArgTy));
+        ArgTy = getTypedPointerWrapper(ElemTy, getPointerAddressSpace(ArgTy));
+        if (isTodoType(Arg))
+          IsUncomplete = true;
+      } else {
+        IsUncomplete = true;
       }
+    }
     ArgTys.push_back(ArgTy);
   }
   Type *RetTy = FTy->getReturnType();
-  if (I->getType()->isPointerTy())
-    if (Type *ElemTy = GR->findDeducedElementType(I)) {
+  if (CI->getType()->isPointerTy()) {
+    if (Type *ElemTy = GR->findDeducedElementType(CI)) {
       IsNewFTy = true;
       RetTy =
-          TypedPointerType::get(ElemTy, getPointerAddressSpace(I->getType()));
+          getTypedPointerWrapper(ElemTy, getPointerAddressSpace(CI->getType()));
+      if (isTodoType(CI))
+        IsUncomplete = true;
+    } else {
+      IsUncomplete = true;
     }
+  }
+  if (!IsPostprocessing && IsUncomplete)
+    insertTodoType(Op);
   KnownElemTy =
       IsNewFTy ? FunctionType::get(RetTy, ArgTys, FTy->isVarArg()) : FTy;
 }
 
+bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
+    Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets,
+    const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
+    Type *&KnownElemTy, Value *Op, Function *F) {
+  KnownElemTy = GR->findDeducedElementType(F);
+  if (KnownElemTy)
+    return false;
+  if (Type *OpElemTy = GR->findDeducedElementType(Op)) {
+    GR->addDeducedElementType(F, OpElemTy);
+    GR->addReturnType(
+        F, TypedPointerType::get(OpElemTy,
+                                 getPointerAddressSpace(F->getReturnType())));
+    // non-recursive update of types in function uses
+    DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(I, Op)};
+    for (User *U : F->users()) {
+      CallInst *CI = dyn_cast<CallInst>(U);
+      if (!CI || CI->getCalledFunction() != F)
+        continue;
+      if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(CI)) {
+        if (Type *PrevElemTy = GR->findDeducedElementType(CI)) {
+          updateAssignType(AssignCI, CI, PoisonValue::get(OpElemTy));
+          propagateElemType(CI, PrevElemTy, VisitedSubst);
+        }
+      }
+    }
+    // Non-recursive update of types in the function uncomplete returns.
+    // This may happen just once per a function, the latch is a pair of
+    // findDeducedElementType(F) / addDeducedElementType(F, ...).
+    // With or without the latch it is a non-recursive call due to
+    // UncompleteRets set to nullptr in this call.
+    if (UncompleteRets)
+      for (Instruction *UncompleteRetI : *UncompleteRets)
+        deduceOperandElementType(UncompleteRetI, nullptr, AskOps,
+                                 IsPostprocessing);
+  } else if (UncompleteRets) {
+    UncompleteRets->insert(I);
+  }
+  TypeValidated.insert(I);
+  return true;
+}
+
 // If the Instruction has Pointer operands with unresolved types, this function
 // tries to deduce them. If the Instruction has Pointer operands with known
 // types which differ from expected, this function tries to insert a bitcast to
 // resolve the issue.
-void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
-                                                   Instruction *AskOp,
-                                                   Type *AskTy,
-                                                   CallInst *AskCI) {
+void SPIRVEmitIntrinsics::deduceOperandElementType(
+    Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets,
+    const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
   SmallVector<std::pair<Value *, unsigned>> Ops;
   Type *KnownElemTy = nullptr;
+  bool Uncomplete = false;
   // look for known basic patterns of type inference
   if (auto *Ref = dyn_cast<PHINode>(I)) {
     if (!isPointerTy(I->getType()) ||
         !(KnownElemTy = GR->findDeducedElementType(I)))
       return;
+    Uncomplete = isTodoType(I);
     for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
       Value *Op = Ref->getIncomingValue(i);
       if (isPointerTy(Op->getType()))
@@ -792,14 +1033,20 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
     KnownElemTy = GR->findDeducedElementType(I);
     if (!KnownElemTy)
       return;
+    Uncomplete = isTodoType(I);
     Ops.push_back(std::make_pair(Ref->getPointerOperand(), 0));
-  } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
-    KnownElemTy = Ref->getSourceElementType();
-    if (isUntypedPointerTy(KnownElemTy))
+  } else if (auto *Ref = dyn_cast<BitCastInst>(I)) {
+    if (!isPointerTy(I->getType()))
       return;
-    Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
-    if (PointeeTy && !isUntypedPointerTy(PointeeTy))
+    KnownElemTy = GR->findDeducedElementType(I);
+    if (!KnownElemTy)
       return;
+    Uncomplete = isTodoType(I);
+    Ops.push_back(std::make_pair(Ref->getOperand(0), 0));
+  } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
+    if (GR->findDeducedElementType(Ref->getPointerOperand()))
+      return;
+    KnownElemTy = Ref->getSourceElementType();
     Ops.push_back(std::make_pair(Ref->getPointerOperand(),
                                  GetElementPtrInst::getPointerOperandIndex()));
   } else if (auto *Ref = dyn_cast<LoadInst>(I)) {
@@ -812,9 +1059,8 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
     Ops.push_back(std::make_pair(Ref->getPointerOperand(),
                                  LoadInst::getPointerOperandIndex()));
   } else if (auto *Ref = dyn_cast<StoreInst>(I)) {
-    if (IsKernelArgInt8(Ref->getParent()->getParent(), Ref))
-      return;
-    if (!(KnownElemTy = reconstructType(GR, Ref->getValueOperand())))
+    if (!(KnownElemTy =
+              reconstructType(Ref->getValueOperand(), false, IsPostprocessing)))
       return;
     Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
     if (PointeeTy && !isUntypedPointerTy(PointeeTy))
@@ -837,27 +1083,23 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
     if (!isPointerTy(I->getType()) ||
         !(KnownElemTy = GR->findDeducedElementType(I)))
       return;
+    Uncomplete = isTodoType(I);
     for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
       Value *Op = Ref->getOperand(i);
       if (isPointerTy(Op->getType()))
         Ops.push_back(std::make_pair(Op, i));
     }
   } else if (auto *Ref = dyn_cast<ReturnInst>(I)) {
-    Type *RetTy = F->getReturnType();
-    if (!isPointerTy(RetTy))
+    if (!isPointerTy(CurrF->getReturnType()))
       return;
     Value *Op = Ref->getReturnValue();
     if (!Op)
       return;
-    if (!(KnownElemTy = GR->findDeducedElementType(F))) {
-      if (Type *OpElemTy = GR->findDeducedElementType(Op)) {
-        GR->addDeducedElementType(F, OpElemTy);
-        TypedPointerType *DerivedTy =
-            TypedPointerType::get(OpElemTy, getPointerAddressSpace(RetTy));
-        GR->addReturnType(F, DerivedTy);
-      }
+    if (deduceOperandElementTypeFunctionRet(I, UncompleteRets, AskOps,
+                                            IsPostprocessing, KnownElemTy, Op,
+                                            CurrF))
       return;
-    }
+    Uncomplete = isTodoType(CurrF);
     Ops.push_back(std::make_pair(Op, 0));
   } else if (auto *Ref = dyn_cast<ICmpInst>(I)) {
     if (!isPointerTy(Ref->getOperand(0)->getType()))
@@ -868,37 +1110,53 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
     Type *ElemTy1 = GR->findDeducedElementType(Op1);
     if (ElemTy0) {
       KnownElemTy = ElemTy0;
+      Uncomplete = isTodoType(Op0);
       Ops.push_back(std::make_pair(Op1, 1));
     } else if (ElemTy1) {
       KnownElemTy = ElemTy1;
+      Uncomplete = isTodoType(Op1);
       Ops.push_back(std::make_pair(Op0, 0));
     }
   } else if (CallInst *CI = dyn_cast<CallInst>(I)) {
     if (!CI->isIndirectCall())
-      deduceOperandElementTypeCalledFunction(GR, I, InstrSet, CI, Ops,
-                                             KnownElemTy);
+      deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy);
     else if (HaveFunPtrs)
-      deduceOperandElementTypeFunctionPointer(GR, I, CI, Ops, KnownElemTy);
+      deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
+                                              IsPostprocessing);
   }
 
   // There is no enough info to deduce types or all is valid.
   if (!KnownElemTy || Ops.size() == 0)
     return;
 
-  LLVMContext &Ctx = F->getContext();
+  LLVMContext &Ctx = CurrF->getContext();
   IRBuilder<> B(Ctx);
   for (auto &OpIt : Ops) {
     Value *Op = OpIt.first;
-    if (Op->use_empty() || (AskOp && Op != AskOp))
+    if (Op->use_empty())
       continue;
-    Type *Ty = AskOp ? AskTy : GR->findDeducedElementType(Op);
+    if (AskOps && !AskOps->contains(Op))
+      continue;
+    Type *AskTy = nullptr;
+    CallInst *AskCI = nullptr;
+    if (IsPostprocessing && AskOps) {
+      AskTy = GR->findDeducedElementType(Op);
+      AskCI = GR->findAssignPtrTypeInstr(Op);
+      assert(AskTy && AskCI);
+    }
+    Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Op);
     if (Ty == KnownElemTy)
       continue;
     Value *OpTyVal = PoisonValue::get(KnownElemTy);
     Type *OpTy = Op->getType();
-    if (!Ty || AskTy || isUntypedPointerTy(Ty) ||
-        UncompleteTypeInfo.contains(Op)) {
+    if (!Ty || AskTy || isUntypedPointerTy(Ty) || isTodoType(Op)) {
+      Type *PrevElemTy = GR->findDeducedElementType(Op);
       GR->addDeducedElementType(Op, KnownElemTy);
+      // check if KnownElemTy is complete
+      if (!Uncomplete)
+        eraseTodoType(Op);
+      else if (!IsPostprocessing)
+        insertTodoType(Op);
       // check if there is existing Intrinsic::spv_assign_ptr_type instruction
       CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Op);
       if (AssignCI == nullptr) {
@@ -910,31 +1168,21 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
         GR->addAssignPtrTypeInstr(Op, CI);
       } else {
         updateAssignType(AssignCI, Op, OpTyVal);
+        DenseSet<std::pair<Value *, Value *>> VisitedSubst{
+            std::make_pair(I, Op)};
+        propagateElemTypeRec(Op, KnownElemTy, PrevElemTy, VisitedSubst);
       }
     } else {
-      if (auto *OpI = dyn_cast<Instruction>(Op)) {
-        // spv_ptrcast's argument Op denotes an instruction that generates
-        // a value, and we may use getInsertionPointAfterDef()
-        B.SetInsertPoint(*OpI->getInsertionPointAfterDef());
-        B.SetCurrentDebugLocation(OpI->getDebugLoc());
-      } else if (auto *OpA = dyn_cast<Argument>(Op)) {
-        B.SetInsertPointPastAllocas(OpA->getParent());
-        B.SetCurrentDebugLocation(DebugLoc());
-      } else {
-        B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
-      }
-      SmallVector<Type *, 2> Types = {OpTy, OpTy};
-      SmallVector<Value *, 2> Args = {Op, buildMD(OpTyVal),
-                                      B.getInt32(getPointerAddressSpace(OpTy))};
+      eraseTodoType(Op);
       CallInst *PtrCastI =
-          B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+          buildSpvPtrcast(I->getParent()->getParent(), Op, KnownElemTy);
       if (OpIt.second == std::numeric_limits<unsigned>::max())
         dyn_cast<CallInst>(I)->setCalledOperand(PtrCastI);
       else
         I->setOperand(OpIt.second, PtrCastI);
-      buildAssignPtr(B, KnownElemTy, PtrCastI);
     }
   }
+  TypeValidated.insert(I);
 }
 
 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
@@ -961,7 +1209,7 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
 
 void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
   std::queue<Instruction *> Worklist;
-  for (auto &I : instructions(F))
+  for (auto &I : instructions(CurrF))
     Worklist.push(&I);
 
   while (!Worklist.empty()) {
@@ -989,7 +1237,7 @@ void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
 
 void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
   std::queue<Instruction *> Worklist;
-  for (auto &I : instructions(F))
+  for (auto &I : instructions(CurrF))
     Worklist.push(&I);
 
   while (!Worklist.empty()) {
@@ -1048,7 +1296,7 @@ Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
     return &Call;
 
   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
-  LLVMContext &Ctx = F->getContext();
+  LLVMContext &Ctx = CurrF->getContext();
 
   Constant *TyC = UndefValue::get(IA->getFunctionType());
   MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
@@ -1106,8 +1354,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
   for (auto &Op : I.operands())
     Args.push_back(Op);
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1129,10 +1376,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
   SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
   SmallVector<Value *> Args(I.op_begin(), I.op_end());
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
-  std::string InstName = I.hasName() ? I.getName().str() : "";
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
-  NewI->setName(InstName);
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1141,9 +1385,9 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
   Type *VTy = V->getType();
 
   // A couple of sanity checks.
-  assert(isPointerTy(VTy) && "Expect a pointer type!");
-  if (auto PType = dyn_cast<TypedPointerType>(VTy))
-    if (PType->getElementType() != AssignedType)
+  assert((isPointerTy(VTy)) && "Expect a pointer type!");
+  if (Type *ElemTy = getPointeeType(VTy))
+    if (ElemTy != AssignedType)
       report_fatal_error("Unexpected pointer element type!");
 
   CallInst *AssignCI = GR->findAssignPtrTypeInstr(V);
@@ -1174,10 +1418,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
 void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
     Instruction *I, Value *Pointer, Type *ExpectedElementType,
     unsigned OperandToReplace, IRBuilder<> &B) {
-  // If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source
-  // pointer instead. The BitCastInst should be later removed when visited.
-  while (BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
-    Pointer = BC->getOperand(0);
+  TypeValidated.insert(I);
 
   // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
   Type *PointerElemTy = deduceElementTypeHelper(Pointer, false);
@@ -1186,7 +1427,8 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
     return;
 
   setInsertPointSkippingPhis(B, I);
-  MetadataAsValue *VMD = buildMD(PoisonValue::get(ExpectedElementType));
+  Value *ExpectedElementVal = PoisonValue::get(ExpectedElementType);
+  MetadataAsValue *VMD = buildMD(ExpectedElementVal);
   unsigned AddressSpace = getPointerAddressSpace(Pointer->getType());
   bool FirstPtrCastOrAssignPtrType = true;
 
@@ -1223,17 +1465,30 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
     return;
   }
 
-  // // Do not emit spv_ptrcast if it would cast to the default pointer element
-  // // type (i8) of the same address space.
-  // if (ExpectedElementType->isIntegerTy(8))
-  //   return;
-
-  // If this would be the first spv_ptrcast, do not emit spv_ptrcast and emit
-  // spv_assign_ptr_type instead.
-  if (FirstPtrCastOrAssignPtrType &&
-      (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
-    buildAssignPtr(B, ExpectedElementType, Pointer);
-    return;
+  if (isa<Instruction>(Pointer) || isa<Argument>(Pointer)) {
+    if (FirstPtrCastOrAssignPtrType) {
+      // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
+      // emit spv_assign_ptr_type instead.
+      buildAssignPtr(B, ExpectedElementType, Pointer);
+      return;
+    } else if (isTodoType(Pointer)) {
+      eraseTodoType(Pointer);
+      if (!isa<CallInst>(Pointer) && !isa<GetElementPtrInst>(Pointer)) {
+        //  If this wouldn't be the first spv_ptrcast but existing type info is
+        //  uncomplete, update spv_assign_ptr_type arguments.
+        if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Pointer)) {
+          Type *PrevElemTy = GR->findDeducedElementType(Pointer);
+          assert(PrevElemTy);
+          DenseSet<std::pair<Value *, Value *>> VisitedSubst{
+              std::make_pair(I, Pointer)};
+          updateAssignType(AssignCI, Pointer, ExpectedElementVal);
+          propagateElemType(Pointer, PrevElemTy, VisitedSubst);
+        } else {
+          buildAssignPtr(B, ExpectedElementType, Pointer);
+        }
+        return;
+      }
+    }
   }
 
   // Emit spv_ptrcast
@@ -1249,27 +1504,48 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
                                                          IRBuilder<> &B) {
   // Handle basic instructions:
   StoreInst *SI = dyn_cast<StoreInst>(I);
-  if (IsKernelArgInt8(F, SI)) {
-    return replacePointerOperandWithPtrCast(
-        I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0,
-        B);
-  } else if (SI) {
+  if (IsKernelArgInt8(CurrF, SI)) {
+    replacePointerOperandWithPtrCast(
+        I, SI->getValueOperand(), IntegerType::getInt8Ty(CurrF->getContext()),
+        0, B);
+  }
+  if (SI) {
     Value *Op = SI->getValueOperand();
+    Value *Pointer = SI->getPointerOperand();
     Type *OpTy = Op->getType();
     if (auto *OpI = dyn_cast<Instruction>(Op))
       OpTy = restoreMutatedType(GR, OpI, OpTy);
     if (OpTy == Op->getType())
       OpTy = deduceElementTypeByValueDeep(OpTy, Op, false);
-    return replacePointerOperandWithPtrCast(I, SI->getPointerOperand(), OpTy, 1,
-                                            B);
-  } else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-    return replacePointerOperandWithPtrCast(I, LI->getPointerOperand(),
-                                            LI->getType(), 0, B);
-  } else if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
-    return replacePointerOperandWithPtrCast(I, GEPI->getPointerOperand(),
-                                            GEPI->getSourceElementType(), 0, B);
+    replacePointerOperandWithPtrCast(I, Pointer, OpTy, 1, B);
+    return;
+  }
+  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+    Value *Pointer = LI->getPointerOperand();
+    Type *OpTy = LI->getType();
+    if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
+      if (Type *ElemTy = GR->findDeducedElementType(LI)) {
+        OpTy = getTypedPointerWrapper(ElemTy, PtrTy->getAddressSpace());
+      } else {
+        Type *NewOpTy = OpTy;
+        OpTy = deduceElementTypeByValueDeep(OpTy, LI, false);
+        if (OpTy == NewOpTy)
+          insertTodoType(Pointer);
+      }
+    }
+    replacePointerOperandWithPtrCast(I, Pointer, OpTy, 0, B);
+    return;
+  }
+  if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
+    Value *Pointer = GEPI->getPointerOperand();
+    Type *OpTy = GEPI->getSourceElementType();
+    replacePointerOperandWithPtrCast(I, Pointer, OpTy, 0, B);
+    if (isNestedPointer(OpTy))
+      insertTodoType(Pointer);
+    return;
   }
 
+  // TODO: review and merge with existing logics:
   // Handle calls to builtins (non-intrinsics):
   CallInst *CI = dyn_cast<CallInst>(I);
   if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
@@ -1287,8 +1563,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
     Type *ArgType = CalledArg->getType();
     if (!isPointerTy(ArgType)) {
       CalledArgTys.push_back(nullptr);
-    } else if (isTypedPointerTy(ArgType)) {
-      CalledArgTys.push_back(cast<TypedPointerType>(ArgType)->getElementType());
+    } else if (Type *ArgTypeElem = getPointeeType(ArgType)) {
+      CalledArgTys.push_back(ArgTypeElem);
       HaveTypes = true;
     } else {
       Type *ElemTy = GR->findDeducedElementType(CalledArg);
@@ -1338,7 +1614,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
     if (!ExpectedType || ExpectedType->isVoidTy())
       continue;
 
-    if (ExpectedType->isTargetExtTy())
+    if (ExpectedType->isTargetExtTy() &&
+        !isTypedPointerWrapper(cast<TargetExtType>(ExpectedType)))
       insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType),
                                    ArgOperand, B);
     else
@@ -1354,10 +1631,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
   B.SetInsertPoint(&I);
   SmallVector<Value *> Args(I.op_begin(), I.op_end());
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
-  std::string InstName = I.hasName() ? I.getName().str() : "";
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
-  NewI->setName(InstName);
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1369,10 +1643,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
                                   I.getIndexOperand()->getType()};
   SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
-  std::string InstName = I.hasName() ? I.getName().str() : "";
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
-  NewI->setName(InstName);
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1406,8 +1677,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
     Args.push_back(B.getInt32(Op));
   auto *NewI =
       B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1419,7 +1689,7 @@ Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
   TrackConstants = false;
   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
   MachineMemOperand::Flags Flags =
-      TLI->getLoadMemOperandFlags(I, F->getDataLayout());
+      TLI->getLoadMemOperandFlags(I, CurrF->getDataLayout());
   auto *NewI =
       B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
                         {I.getPointerOperand(), B.getInt16(Flags),
@@ -1436,7 +1706,7 @@ Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
   TrackConstants = false;
   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
   MachineMemOperand::Flags Flags =
-      TLI->getStoreMemOperandFlags(I, F->getDataLayout());
+      TLI->getStoreMemOperandFlags(I, CurrF->getDataLayout());
   auto *PtrOp = I.getPointerOperand();
   auto *NewI = B.CreateIntrinsic(
       Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
@@ -1463,13 +1733,13 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
   TrackConstants = false;
   Type *PtrTy = I.getType();
   auto *NewI =
-      ArraySize ? B.CreateIntrinsic(Intrinsic::spv_alloca_array,
-                                    {PtrTy, ArraySize->getType()}, {ArraySize})
-                : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
-  std::string InstName = I.hasName() ? I.getName().str() : "";
-  replaceAllUsesWith(&I, NewI);
-  I.eraseFromParent();
-  NewI->setName(InstName);
+      ArraySize
+          ? B.CreateIntrinsic(Intrinsic::spv_alloca_array,
+                              {PtrTy, ArraySize->getType()},
+                              {ArraySize, B.getInt8(I.getAlign().value())})
+          : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy},
+                              {B.getInt8(I.getAlign().value())});
+  replaceAllUsesWithAndErase(B, &I, NewI);
   return NewI;
 }
 
@@ -1528,8 +1798,7 @@ bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
                                                    IRBuilder<> &B,
                                                    bool UnknownElemTypeI8) {
   reportFatalOnTokenType(I);
-  if (!isPointerTy(I->getType()) || !requireAssignType(I) ||
-      isa<BitCastInst>(I))
+  if (!isPointerTy(I->getType()) || !requireAssignType(I))
     return false;
 
   setInsertPointAfterDef(B, I);
@@ -1602,8 +1871,9 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
         GR->addAssignPtrTypeInstr(Op, AssignCI);
       } else if (!isa<Instruction>(Op)) {
         Type *OpTy = Op->getType();
-        if (auto PType = dyn_cast<TypedPointerType>(OpTy)) {
-          buildAssignPtr(B, PType->getElementType(), Op);
+        Type *OpTyElem = getPointeeType(OpTy);
+        if (OpTyElem) {
+          buildAssignPtr(B, OpTyElem, Op);
         } else if (isPointerTy(OpTy)) {
           Type *ElemTy = GR->findDeducedElementType(Op);
           buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op);
@@ -1629,8 +1899,9 @@ void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
                                                  IRBuilder<> &B) {
   auto *II = dyn_cast<IntrinsicInst>(I);
-  if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
-      TrackConstants) {
+  bool IsConstComposite =
+      II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
+  if (IsConstComposite && TrackConstants) {
     setInsertPointAfterDef(B, I);
     auto t = AggrConsts.find(I);
     assert(t != AggrConsts.end());
@@ -1654,23 +1925,31 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
               : B.SetInsertPoint(I);
         BPrepared = true;
       }
+      Type *OpTy = Op->getType();
       Value *OpTyVal = Op;
-      if (Op->getType()->isTargetExtTy())
-        OpTyVal = PoisonValue::get(Op->getType());
-      auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
-                                    {Op->getType(), OpTyVal->getType()}, Op,
-                                    OpTyVal, {}, B);
+      if (OpTy->isTargetExtTy())
+        OpTyVal = PoisonValue::get(OpTy);
+      CallInst *NewOp =
+          buildIntrWithMD(Intrinsic::spv_track_constant,
+                          {OpTy, OpTyVal->getType()}, Op, OpTyVal, {}, B);
+      Type *OpElemTy = nullptr;
+      if (!IsConstComposite && isPointerTy(OpTy) &&
+          (OpElemTy = GR->findDeducedElementType(Op)) != nullptr &&
+          OpElemTy != IntegerType::getInt8Ty(I->getContext())) {
+        buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp);
+        SmallVector<Type *, 2> Types = {OpTy, OpTy};
+        SmallVector<Value *, 2> Args = {
+            NewOp, buildMD(PoisonValue::get(OpElemTy)),
+            B.getInt32(getPointerAddressSpace(OpTy))};
+        CallInst *PtrCasted =
+            B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+        buildAssignPtr(B, OpElemTy, PtrCasted);
+        NewOp = PtrCasted;
+      }
       I->setOperand(OpNo, NewOp);
     }
   }
-  if (I->hasName() && !I->getType()->isAggregateType() &&
-      !expectIgnoredInIRTranslation(I)) {
-    reportFatalOnTokenType(I);
-    setInsertPointAfterDef(B, I);
-    std::vector<Value *> Args = {I};
-    addStringImm(I->getName(), B, Args);
-    B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
-  }
+  emitAssignName(I, B);
 }
 
 Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
@@ -1742,9 +2021,44 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
     if (!isUntypedPointerTy(Arg->getType()))
       continue;
     Type *ElemTy = GR->findDeducedElementType(Arg);
-    if (!ElemTy && hasPointeeTypeAttr(Arg) &&
-        (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr)
+    if (ElemTy)
+      continue;
+    if (hasPointeeTypeAttr(Arg) &&
+        (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
+      buildAssignPtr(B, ElemTy, Arg);
+      continue;
+    }
+    // search in function's call sites
+    for (User *U : F->users()) {
+      CallInst *CI = dyn_cast<CallInst>(U);
+      if (!CI || OpIdx >= CI->arg_size())
+        continue;
+      Value *OpArg = CI->getArgOperand(OpIdx);
+      if (!isPointerTy(OpArg->getType()))
+        continue;
+      // maybe we already know operand's element type
+      if ((ElemTy = GR->findDeducedElementType(OpArg)) != nullptr)
+        break;
+    }
+    if (ElemTy) {
       buildAssignPtr(B, ElemTy, Arg);
+      continue;
+    }
+    if (HaveFunPtrs) {
+      for (User *U : Arg->users()) {
+        CallInst *CI = dyn_cast<CallInst>(U);
+        if (CI && !isa<IntrinsicInst>(CI) && CI->isIndirectCall() &&
+            CI->getCalledOperand() == Arg &&
+            CI->getParent()->getParent() == CurrF) {
+          SmallVector<std::pair<Value *, unsigned>> Ops;
+          deduceOperandElementTypeFunctionPointer(CI, Ops, ElemTy, false);
+          if (ElemTy) {
+            buildAssignPtr(B, ElemTy, Arg);
+            break;
+          }
+        }
+      }
+    }
   }
 }
 
@@ -1755,8 +2069,16 @@ void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
     if (!isUntypedPointerTy(Arg->getType()))
       continue;
     Type *ElemTy = GR->findDeducedElementType(Arg);
-    if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr)
-      buildAssignPtr(B, ElemTy, Arg);
+    if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
+      if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Arg)) {
+        DenseSet<std::pair<Value *, Value *>> VisitedSubst;
+        updateAssignType(AssignCI, Arg, PoisonValue::get(ElemTy));
+        propagateElemType(Arg, IntegerType::getInt8Ty(F->getContext()),
+                          VisitedSubst);
+      } else {
+        buildAssignPtr(B, ElemTy, Arg);
+      }
+    }
   }
 }
 
@@ -1770,7 +2092,7 @@ static FunctionType *getFunctionPointerElemType(Function *F,
     if (ArgTy->isPointerTy())
       if (Type *ElemTy = GR->findDeducedElementType(&Arg)) {
         IsNewFTy = true;
-        ArgTy = TypedPointerType::get(ElemTy, getPointerAddressSpace(ArgTy));
+        ArgTy = getTypedPointerWrapper(ElemTy, getPointerAddressSpace(ArgTy));
       }
     ArgTys.push_back(ArgTy);
   }
@@ -1836,6 +2158,53 @@ bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
   return true;
 }
 
+// Apply types parsed from demangled function declarations.
+void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
+  for (auto It : FDeclPtrTys) {
+    Function *F = It.first;
+    for (auto *U : F->users()) {
+      CallInst *CI = dyn_cast<CallInst>(U);
+      if (!CI || CI->getCalledFunction() != F)
+        continue;
+      unsigned Sz = CI->arg_size();
+      for (auto [Idx, ElemTy] : It.second) {
+        if (Idx >= Sz)
+          continue;
+        Value *Param = CI->getArgOperand(Idx);
+        if (GR->findDeducedElementType(Param) || isa<GlobalValue>(Param))
+          continue;
+        if (Argument *Arg = dyn_cast<Argument>(Param)) {
+          if (!hasPointeeTypeAttr(Arg)) {
+            B.SetInsertPointPastAllocas(Arg->getParent());
+            B.SetCurrentDebugLocation(DebugLoc());
+            buildAssignPtr(B, ElemTy, Arg);
+          }
+        } else if (isa<Instruction>(Param)) {
+          GR->addDeducedElementType(Param, ElemTy);
+          // insertAssignTypeIntrs() will complete buildAssignPtr()
+        } else {
+          B.SetInsertPoint(CI->getParent()
+                               ->getParent()
+                               ->getEntryBlock()
+                               .getFirstNonPHIOrDbgOrAlloca());
+          buildAssignPtr(B, ElemTy, Param);
+        }
+        CallInst *Ref = dyn_cast<CallInst>(Param);
+        if (!Ref)
+          continue;
+        Function *RefF = Ref->getCalledFunction();
+        if (!RefF || !isPointerTy(RefF->getReturnType()) ||
+            GR->findDeducedElementType(RefF))
+          continue;
+        GR->addDeducedElementType(RefF, ElemTy);
+        GR->addReturnType(
+            RefF, TypedPointerType::get(
+                      ElemTy, getPointerAddressSpace(RefF->getReturnType())));
+      }
+    }
+  }
+}
+
 bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
   if (Func.isDeclaration())
     return false;
@@ -1845,17 +2214,17 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
   InstrSet = ST.isOpenCLEnv() ? SPIRV::InstructionSet::OpenCL_std
                               : SPIRV::InstructionSet::GLSL_std_450;
 
-  if (!F)
+  if (!CurrF)
     HaveFunPtrs =
         ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
 
-  F = &Func;
+  CurrF = &Func;
   IRBuilder<> B(Func.getContext());
   AggrConsts.clear();
   AggrConstTypes.clear();
   AggrStores.clear();
 
-  processParamTypesByFunHeader(F, B);
+  processParamTypesByFunHeader(CurrF, B);
 
   // StoreInst's operand type can be changed during the next transformations,
   // so we need to store it in the set. Also store already transformed types.
@@ -1879,6 +2248,9 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
   for (auto &I : instructions(Func))
     Worklist.push_back(&I);
 
+  applyDemangledPtrArgTypes(B);
+
+  // Pass forward: use operand to deduce instructions result.
   for (auto &I : Worklist) {
     // Don't emit intrinsincs for convergence intrinsics.
     if (isConvergenceIntrinsic(I))
@@ -1895,8 +2267,18 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
       insertAssignPtrTypeIntrs(I, B, true);
   }
 
-  for (auto &I : instructions(Func))
-    deduceOperandElementType(&I);
+  // Pass backward: use instructions results to specify/update/cast operands
+  // where needed.
+  SmallPtrSet<Instruction *, 4> UncompleteRets;
+  for (auto &I : llvm::reverse(instructions(Func)))
+    deduceOperandElementType(&I, &UncompleteRets);
+
+  // Pass forward for PHIs only, their operands are not preceed the instruction
+  // in meaning of `instructions(Func)`.
+  for (BasicBlock &BB : Func)
+    for (PHINode &Phi : BB.phis())
+      if (isPointerTy(Phi.getType()))
+        deduceOperandElementType(&Phi, nullptr);
 
   for (auto *I : Worklist) {
     TrackConstants = true;
@@ -1918,83 +2300,127 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
   return true;
 }
 
-void SPIRVEmitIntrinsics::replaceWithPtrcasted(Instruction *CI, Type *NewElemTy,
-                                               Type *KnownElemTy,
-                                               CallInst *AssignCI) {
-  updateAssignType(AssignCI, CI, PoisonValue::get(NewElemTy));
-  IRBuilder<> B(CI->getContext());
-  B.SetInsertPoint(*CI->getInsertionPointAfterDef());
-  B.SetCurrentDebugLocation(CI->getDebugLoc());
-  Type *OpTy = CI->getType();
-  SmallVector<Type *, 2> Types = {OpTy, OpTy};
-  SmallVector<Value *, 2> Args = {CI, buildMD(PoisonValue::get(KnownElemTy)),
-                                  B.getInt32(getPointerAddressSpace(OpTy))};
-  CallInst *PtrCasted =
-      B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
-  SmallVector<User *> Users(CI->users());
-  for (auto *U : Users)
-    if (U != AssignCI && U != PtrCasted)
-      U->replaceUsesOfWith(CI, PtrCasted);
-  buildAssignPtr(B, KnownElemTy, PtrCasted);
-}
-
 // Try to deduce a better type for pointers to untyped ptr.
-bool SPIRVEmitIntrinsics::postprocessTypes() {
-  bool Changed = false;
-  if (!GR)
-    return Changed;
-  for (auto IB = PostprocessWorklist.rbegin(), IE = PostprocessWorklist.rend();
-       IB != IE; ++IB) {
-    CallInst *AssignCI = GR->findAssignPtrTypeInstr(*IB);
-    Type *KnownTy = GR->findDeducedElementType(*IB);
-    if (!KnownTy || !AssignCI || !isa<Instruction>(AssignCI->getArgOperand(0)))
+bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
+  if (!GR || TodoTypeSz == 0)
+    return false;
+
+  unsigned SzTodo = TodoTypeSz;
+  DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
+  for (auto [Op, Enabled] : TodoType) {
+    // TODO: add isa<CallInst>(Op) to continue
+    if (!Enabled || isa<GetElementPtrInst>(Op))
       continue;
+    CallInst *AssignCI = GR->findAssignPtrTypeInstr(Op);
+    Type *KnownTy = GR->findDeducedElementType(Op);
+    if (!KnownTy || !AssignCI)
+      continue;
+    assert(Op == AssignCI->getArgOperand(0));
     // Try to improve the type deduced after all Functions are processed.
-    if (auto *CI = dyn_cast<CallInst>(*IB)) {
-      if (Function *CalledF = CI->getCalledFunction()) {
-        Type *RetElemTy = GR->findDeducedElementType(CalledF);
-        // Fix inconsistency between known type and function's return type.
-        if (RetElemTy && RetElemTy != KnownTy) {
-          replaceWithPtrcasted(CI, RetElemTy, KnownTy, AssignCI);
-          Changed = true;
+    if (auto *CI = dyn_cast<Instruction>(Op)) {
+      CurrF = CI->getParent()->getParent();
+      std::unordered_set<Value *> Visited;
+      if (Type *ElemTy = deduceElementTypeHelper(Op, Visited, false, true)) {
+        if (ElemTy != KnownTy) {
+          DenseSet<std::pair<Value *, Value *>> VisitedSubst;
+          propagateElemType(CI, ElemTy, VisitedSubst);
+          eraseTodoType(Op);
           continue;
         }
       }
     }
-    Instruction *I = cast<Instruction>(AssignCI->getArgOperand(0));
-    for (User *U : I->users()) {
+    for (User *U : Op->users()) {
       Instruction *Inst = dyn_cast<Instruction>(U);
-      if (!Inst || isa<IntrinsicInst>(Inst))
+      if (Inst && !isa<IntrinsicInst>(Inst))
+        ToProcess[Inst].insert(Op);
+    }
+  }
+  if (TodoTypeSz == 0)
+    return true;
+
+  for (auto &F : M) {
+    CurrF = &F;
+    SmallPtrSet<Instruction *, 4> UncompleteRets;
+    for (auto &I : llvm::reverse(instructions(F))) {
+      auto It = ToProcess.find(&I);
+      if (It == ToProcess.end())
         continue;
-      deduceOperandElementType(Inst, I, KnownTy, AssignCI);
-      if (KnownTy != GR->findDeducedElementType(I)) {
-        Changed = true;
-        break;
-      }
+      It->second.remove_if([this](Value *V) { return !isTodoType(V); });
+      if (It->second.size() == 0)
+        continue;
+      deduceOperandElementType(&I, &UncompleteRets, &It->second, true);
+      if (TodoTypeSz == 0)
+        return true;
+    }
+  }
+
+  return SzTodo > TodoTypeSz;
+}
+
+// Parse and store argument types of function declarations where needed.
+void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
+  for (auto &F : M) {
+    if (!F.isDeclaration() || F.isIntrinsic())
+      continue;
+    // get the demangled name
+    std::string DemangledName = getOclOrSpirvBuiltinDemangledName(F.getName());
+    if (DemangledName.empty())
+      continue;
+    // allow only OpGroupAsyncCopy use case at the moment
+    auto [Grp, Opcode, ExtNo] =
+        SPIRV::mapBuiltinToOpcode(DemangledName, InstrSet);
+    if (Opcode != SPIRV::OpGroupAsyncCopy)
+      continue;
+    // find pointer arguments
+    SmallVector<unsigned> Idxs;
+    for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
+      Argument *Arg = F.getArg(OpIdx);
+      if (isPointerTy(Arg->getType()) && !hasPointeeTypeAttr(Arg))
+        Idxs.push_back(OpIdx);
+    }
+    if (!Idxs.size())
+      continue;
+    // parse function arguments
+    LLVMContext &Ctx = F.getContext();
+    SmallVector<StringRef, 10> TypeStrs;
+    SPIRV::parseBuiltinTypeStr(TypeStrs, DemangledName, Ctx);
+    if (!TypeStrs.size())
+      continue;
+    // find type info for pointer arguments
+    for (unsigned Idx : Idxs) {
+      if (Idx >= TypeStrs.size())
+        continue;
+      if (Type *ElemTy =
+              SPIRV::parseBuiltinCallArgumentType(TypeStrs[Idx].trim(), Ctx))
+        if (TypedPointerType::isValidElementType(ElemTy) &&
+            !ElemTy->isTargetExtTy())
+          FDeclPtrTys[&F].push_back(std::make_pair(Idx, ElemTy));
     }
   }
-  return Changed;
 }
 
 bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
   bool Changed = false;
 
-  UncompleteTypeInfo.clear();
-  PostprocessWorklist.clear();
+  parseFunDeclarations(M);
+
+  TodoType.clear();
   for (auto &F : M)
     Changed |= runOnFunction(F);
 
+  // Specify function parameters after all functions were processed.
   for (auto &F : M) {
     // check if function parameter types are set
+    CurrF = &F;
     if (!F.isDeclaration() && !F.isIntrinsic()) {
-      const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
-      GR = ST.getSPIRVGlobalRegistry();
       IRBuilder<> B(F.getContext());
       processParamTypes(&F, B);
     }
   }
 
-  Changed |= postprocessTypes();
+  CanTodoType = false;
+  Changed |= postprocessTypes(M);
+
   if (HaveFunPtrs)
     Changed |= processFunctionPointers(M);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
index d3e323efaee91..b98cef0a4f07f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
@@ -268,7 +268,7 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
     // We aren't extracting any DebugInfoFlags now so we
     // emitting zero to use as <id>Flags argument for DebugBasicType
     const Register I32ZeroReg =
-        GR->buildConstantInt(0, MIRBuilder, I32Ty, false);
+        GR->buildConstantInt(0, MIRBuilder, I32Ty, false, false);
 
     // We need to store pairs because further instructions reference
     // the DIBasicTypes and size will be always small so there isn't
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 6f222883ee07d..5f72a41ddb864 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -28,6 +28,18 @@
 #include <functional>
 
 using namespace llvm;
+
+inline unsigned typeToAddressSpace(const Type *Ty) {
+  if (auto PType = dyn_cast<TypedPointerType>(Ty))
+    return PType->getAddressSpace();
+  if (auto PType = dyn_cast<PointerType>(Ty))
+    return PType->getAddressSpace();
+  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
+      ExtTy && isTypedPointerWrapper(ExtTy))
+    return ExtTy->getIntParameter(0);
+  report_fatal_error("Unable to convert LLVM type to SPIRVType", true);
+}
+
 SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize)
     : PointerSize(PointerSize), Bound(0) {}
 
@@ -69,7 +81,7 @@ SPIRVType *SPIRVGlobalRegistry::assignTypeToVReg(
 
 void SPIRVGlobalRegistry::assignSPIRVTypeToVReg(SPIRVType *SpirvType,
                                                 Register VReg,
-                                                MachineFunction &MF) {
+                                                const MachineFunction &MF) {
   VRegToTypeMap[&MF][VReg] = SpirvType;
 }
 
@@ -84,7 +96,6 @@ inline Register createTypeVReg(MachineIRBuilder &MIRBuilder) {
 }
 
 SPIRVType *SPIRVGlobalRegistry::getOpTypeBool(MachineIRBuilder &MIRBuilder) {
-
   return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
     return MIRBuilder.buildInstr(SPIRV::OpTypeBool)
         .addDef(createTypeVReg(MIRBuilder));
@@ -154,8 +165,11 @@ SPIRVType *SPIRVGlobalRegistry::createOpType(
 
   auto LastInsertedType = LastInsertedTypeMap.find(CurMF);
   if (LastInsertedType != LastInsertedTypeMap.end()) {
-    MIRBuilder.setInsertPt(*MIRBuilder.getMF().begin(),
-                           LastInsertedType->second->getIterator());
+    auto It = LastInsertedType->second->getIterator();
+    auto NewMBB = MIRBuilder.getMF().begin();
+    MIRBuilder.setInsertPt(*NewMBB, It->getNextNode()
+                                        ? It->getNextNode()->getIterator()
+                                        : NewMBB->end());
   } else {
     MIRBuilder.setInsertPt(*MIRBuilder.getMF().begin(),
                            MIRBuilder.getMF().begin()->begin());
@@ -257,24 +271,27 @@ Register SPIRVGlobalRegistry::getOrCreateConstFP(APFloat Val, MachineInstr &I,
   // machine instruction, a new constant instruction should be created.
   if (!New && (!I.getOperand(0).isReg() || Res != I.getOperand(0).getReg()))
     return Res;
-  MachineInstrBuilder MIB;
-  MachineBasicBlock &BB = *I.getParent();
-  // In OpenCL OpConstantNull - Scalar floating point: +0.0 (all bits 0)
-  if (Val.isPosZero() && ZeroAsNull) {
-    MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
-              .addDef(Res)
-              .addUse(getSPIRVTypeID(SpvType));
-  } else {
-    MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantF))
-              .addDef(Res)
-              .addUse(getSPIRVTypeID(SpvType));
-    addNumImm(
-        APInt(BitWidth, CI->getValueAPF().bitcastToAPInt().getZExtValue()),
-        MIB);
-  }
-  const auto &ST = CurMF->getSubtarget();
-  constrainSelectedInstRegOperands(*MIB, *ST.getInstrInfo(),
-                                   *ST.getRegisterInfo(), *ST.getRegBankInfo());
+  MachineIRBuilder MIRBuilder(I);
+  createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+    MachineInstrBuilder MIB;
+    // In OpenCL OpConstantNull - Scalar floating point: +0.0 (all bits 0)
+    if (Val.isPosZero() && ZeroAsNull) {
+      MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+                .addDef(Res)
+                .addUse(getSPIRVTypeID(SpvType));
+    } else {
+      MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)
+                .addDef(Res)
+                .addUse(getSPIRVTypeID(SpvType));
+      addNumImm(
+          APInt(BitWidth, CI->getValueAPF().bitcastToAPInt().getZExtValue()),
+          MIB);
+    }
+    const auto &ST = CurMF->getSubtarget();
+    constrainSelectedInstRegOperands(
+        *MIB, *ST.getInstrInfo(), *ST.getRegisterInfo(), *ST.getRegBankInfo());
+    return MIB;
+  });
   return Res;
 }
 
@@ -293,28 +310,32 @@ Register SPIRVGlobalRegistry::getOrCreateConstInt(uint64_t Val, MachineInstr &I,
   // machine instruction, a new constant instruction should be created.
   if (!New && (!I.getOperand(0).isReg() || Res != I.getOperand(0).getReg()))
     return Res;
-  MachineInstrBuilder MIB;
-  MachineBasicBlock &BB = *I.getParent();
-  if (Val || !ZeroAsNull) {
-    MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
-              .addDef(Res)
-              .addUse(getSPIRVTypeID(SpvType));
-    addNumImm(APInt(BitWidth, Val), MIB);
-  } else {
-    MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
-              .addDef(Res)
-              .addUse(getSPIRVTypeID(SpvType));
-  }
-  const auto &ST = CurMF->getSubtarget();
-  constrainSelectedInstRegOperands(*MIB, *ST.getInstrInfo(),
-                                   *ST.getRegisterInfo(), *ST.getRegBankInfo());
+
+  MachineIRBuilder MIRBuilder(I);
+  createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+    MachineInstrBuilder MIB;
+    if (Val || !ZeroAsNull) {
+      MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)
+                .addDef(Res)
+                .addUse(getSPIRVTypeID(SpvType));
+      addNumImm(APInt(BitWidth, Val), MIB);
+    } else {
+      MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+                .addDef(Res)
+                .addUse(getSPIRVTypeID(SpvType));
+    }
+    const auto &ST = CurMF->getSubtarget();
+    constrainSelectedInstRegOperands(
+        *MIB, *ST.getInstrInfo(), *ST.getRegisterInfo(), *ST.getRegBankInfo());
+    return MIB;
+  });
   return Res;
 }
 
 Register SPIRVGlobalRegistry::buildConstantInt(uint64_t Val,
                                                MachineIRBuilder &MIRBuilder,
-                                               SPIRVType *SpvType,
-                                               bool EmitIR) {
+                                               SPIRVType *SpvType, bool EmitIR,
+                                               bool ZeroAsNull) {
   assert(SpvType);
   auto &MF = MIRBuilder.getMF();
   const IntegerType *LLVMIntTy =
@@ -335,21 +356,24 @@ Register SPIRVGlobalRegistry::buildConstantInt(uint64_t Val,
       MIRBuilder.buildConstant(Res, *ConstInt);
     } else {
       Register SpvTypeReg = getSPIRVTypeID(SpvType);
-      MachineInstrBuilder MIB;
-      if (Val) {
-        MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)
-                  .addDef(Res)
-                  .addUse(SpvTypeReg);
-        addNumImm(APInt(BitWidth, Val), MIB);
-      } else {
-        MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
-                  .addDef(Res)
-                  .addUse(SpvTypeReg);
-      }
-      const auto &Subtarget = CurMF->getSubtarget();
-      constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),
-                                       *Subtarget.getRegisterInfo(),
-                                       *Subtarget.getRegBankInfo());
+      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+        MachineInstrBuilder MIB;
+        if (Val || !ZeroAsNull) {
+          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)
+                    .addDef(Res)
+                    .addUse(SpvTypeReg);
+          addNumImm(APInt(BitWidth, Val), MIB);
+        } else {
+          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+                    .addDef(Res)
+                    .addUse(SpvTypeReg);
+        }
+        const auto &Subtarget = CurMF->getSubtarget();
+        constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),
+                                         *Subtarget.getRegisterInfo(),
+                                         *Subtarget.getRegBankInfo());
+        return MIB;
+      });
     }
   }
   return Res;
@@ -373,12 +397,14 @@ Register SPIRVGlobalRegistry::buildConstantFP(APFloat Val,
     MF.getRegInfo().setRegClass(Res, &SPIRV::fIDRegClass);
     assignSPIRVTypeToVReg(SpvType, Res, MF);
     DT.add(ConstFP, &MF, Res);
-
-    MachineInstrBuilder MIB;
-    MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)
-              .addDef(Res)
-              .addUse(getSPIRVTypeID(SpvType));
-    addNumImm(ConstFP->getValueAPF().bitcastToAPInt(), MIB);
+    createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+      MachineInstrBuilder MIB;
+      MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)
+                .addDef(Res)
+                .addUse(getSPIRVTypeID(SpvType));
+      addNumImm(ConstFP->getValueAPF().bitcastToAPInt(), MIB);
+      return MIB;
+    });
   }
 
   return Res;
@@ -427,23 +453,26 @@ Register SPIRVGlobalRegistry::getOrCreateCompositeOrNull(
     CurMF->getRegInfo().setRegClass(SpvVecConst, getRegClass(SpvType));
     assignSPIRVTypeToVReg(SpvType, SpvVecConst, *CurMF);
     DT.add(CA, CurMF, SpvVecConst);
-    MachineInstrBuilder MIB;
-    MachineBasicBlock &BB = *I.getParent();
-    if (!IsNull) {
-      MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantComposite))
-                .addDef(SpvVecConst)
-                .addUse(getSPIRVTypeID(SpvType));
-      for (unsigned i = 0; i < ElemCnt; ++i)
-        MIB.addUse(SpvScalConst);
-    } else {
-      MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
-                .addDef(SpvVecConst)
-                .addUse(getSPIRVTypeID(SpvType));
-    }
-    const auto &Subtarget = CurMF->getSubtarget();
-    constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),
-                                     *Subtarget.getRegisterInfo(),
-                                     *Subtarget.getRegBankInfo());
+    MachineIRBuilder MIRBuilder(I);
+    createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+      MachineInstrBuilder MIB;
+      if (!IsNull) {
+        MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)
+                  .addDef(SpvVecConst)
+                  .addUse(getSPIRVTypeID(SpvType));
+        for (unsigned i = 0; i < ElemCnt; ++i)
+          MIB.addUse(SpvScalConst);
+      } else {
+        MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+                  .addDef(SpvVecConst)
+                  .addUse(getSPIRVTypeID(SpvType));
+      }
+      const auto &Subtarget = CurMF->getSubtarget();
+      constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),
+                                       *Subtarget.getRegisterInfo(),
+                                       *Subtarget.getRegBankInfo());
+      return MIB;
+    });
     return SpvVecConst;
   }
   return Res;
@@ -532,17 +561,20 @@ Register SPIRVGlobalRegistry::getOrCreateIntCompositeOrNull(
     if (EmitIR) {
       MIRBuilder.buildSplatBuildVector(SpvVecConst, SpvScalConst);
     } else {
-      if (Val) {
-        auto MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)
-                       .addDef(SpvVecConst)
-                       .addUse(getSPIRVTypeID(SpvType));
-        for (unsigned i = 0; i < ElemCnt; ++i)
-          MIB.addUse(SpvScalConst);
-      } else {
-        MIRBuilder.buildInstr(SPIRV::OpConstantNull)
-            .addDef(SpvVecConst)
-            .addUse(getSPIRVTypeID(SpvType));
-      }
+      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+        if (Val) {
+          auto MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)
+                         .addDef(SpvVecConst)
+                         .addUse(getSPIRVTypeID(SpvType));
+          for (unsigned i = 0; i < ElemCnt; ++i)
+            MIB.addUse(SpvScalConst);
+          return MIB;
+        } else {
+          return MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+              .addDef(SpvVecConst)
+              .addUse(getSPIRVTypeID(SpvType));
+        }
+      });
     }
     return SpvVecConst;
   }
@@ -570,19 +602,21 @@ Register
 SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder,
                                              SPIRVType *SpvType) {
   const Type *LLVMTy = getTypeForSPIRVType(SpvType);
-  const TypedPointerType *LLVMPtrTy = cast<TypedPointerType>(LLVMTy);
+  unsigned AddressSpace = typeToAddressSpace(LLVMTy);
   // Find a constant in DT or build a new one.
-  Constant *CP = ConstantPointerNull::get(PointerType::get(
-      LLVMPtrTy->getElementType(), LLVMPtrTy->getAddressSpace()));
+  Constant *CP = ConstantPointerNull::get(
+      PointerType::get(::getPointeeType(LLVMTy), AddressSpace));
   Register Res = DT.find(CP, CurMF);
   if (!Res.isValid()) {
-    LLT LLTy = LLT::pointer(LLVMPtrTy->getAddressSpace(), PointerSize);
+    LLT LLTy = LLT::pointer(AddressSpace, PointerSize);
     Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
-    CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);
+    CurMF->getRegInfo().setRegClass(Res, &SPIRV::pIDRegClass);
     assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
-    MIRBuilder.buildInstr(SPIRV::OpConstantNull)
-        .addDef(Res)
-        .addUse(getSPIRVTypeID(SpvType));
+    createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+      return MIRBuilder.buildInstr(SPIRV::OpConstantNull)
+          .addDef(Res)
+          .addUse(getSPIRVTypeID(SpvType));
+    });
     DT.add(CP, CurMF, Res);
   }
   return Res;
@@ -602,12 +636,14 @@ Register SPIRVGlobalRegistry::buildConstantSampler(
       ResReg.isValid()
           ? ResReg
           : MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
-  auto Res = MIRBuilder.buildInstr(SPIRV::OpConstantSampler)
-                 .addDef(Sampler)
-                 .addUse(getSPIRVTypeID(SampTy))
-                 .addImm(AddrMode)
-                 .addImm(Param)
-                 .addImm(FilerMode);
+  auto Res = createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+    return MIRBuilder.buildInstr(SPIRV::OpConstantSampler)
+        .addDef(Sampler)
+        .addUse(getSPIRVTypeID(SampTy))
+        .addImm(AddrMode)
+        .addImm(Param)
+        .addImm(FilerMode);
+  });
   assert(Res->getOperand(0).isReg());
   return Res->getOperand(0).getReg();
 }
@@ -978,18 +1014,11 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
     }
     return getOpTypeFunction(RetTy, ParamTypes, MIRBuilder);
   }
-  unsigned AddrSpace = 0xFFFF;
-  if (auto PType = dyn_cast<TypedPointerType>(Ty))
-    AddrSpace = PType->getAddressSpace();
-  else if (auto PType = dyn_cast<PointerType>(Ty))
-    AddrSpace = PType->getAddressSpace();
-  else
-    report_fatal_error("Unable to convert LLVM type to SPIRVType", true);
 
+  unsigned AddrSpace = typeToAddressSpace(Ty);
   SPIRVType *SpvElementType = nullptr;
-  if (auto PType = dyn_cast<TypedPointerType>(Ty))
-    SpvElementType = getOrCreateSPIRVType(PType->getElementType(), MIRBuilder,
-                                          AccQual, EmitIR);
+  if (Type *ElemTy = ::getPointeeType(Ty))
+    SpvElementType = getOrCreateSPIRVType(ElemTy, MIRBuilder, AccQual, EmitIR);
   else
     SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder);
 
@@ -1017,7 +1046,7 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
 SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType(
     const Type *Ty, MachineIRBuilder &MIRBuilder,
     SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
-  if (TypesInProcessing.count(Ty) && !isPointerTy(Ty))
+  if (TypesInProcessing.count(Ty) && !isPointerTyOrWrapper(Ty))
     return nullptr;
   TypesInProcessing.insert(Ty);
   SPIRVType *SpirvType = createSPIRVType(Ty, MIRBuilder, AccessQual, EmitIR);
@@ -1029,7 +1058,11 @@ SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType(
   // will be added later. For special types it is already added to DT.
   if (SpirvType->getOpcode() != SPIRV::OpTypeForwardPointer && !Reg.isValid() &&
       !isSpecialOpaqueType(Ty)) {
-    if (!isPointerTy(Ty))
+    if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
+        ExtTy && isTypedPointerWrapper(ExtTy))
+      DT.add(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0),
+             &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType));
+    else if (!isPointerTy(Ty))
       DT.add(Ty, &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType));
     else if (isTypedPointerTy(Ty))
       DT.add(cast<TypedPointerType>(Ty)->getElementType(),
@@ -1065,7 +1098,11 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVType(
     const Type *Ty, MachineIRBuilder &MIRBuilder,
     SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
   Register Reg;
-  if (!isPointerTy(Ty)) {
+  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
+      ExtTy && isTypedPointerWrapper(ExtTy)) {
+    Reg = DT.find(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0),
+                  &MIRBuilder.getMF());
+  } else if (!isPointerTy(Ty)) {
     Ty = adjustIntTypeByWidth(Ty);
     Reg = DT.find(Ty, &MIRBuilder.getMF());
   } else if (isTypedPointerTy(Ty)) {
@@ -1538,14 +1575,17 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
   if (Reg.isValid())
     return getSPIRVTypeForVReg(Reg);
   // create a new type
-  auto MIB = BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(),
-                     MIRBuilder.getDebugLoc(),
-                     MIRBuilder.getTII().get(SPIRV::OpTypePointer))
-                 .addDef(createTypeVReg(CurMF->getRegInfo()))
-                 .addImm(static_cast<uint32_t>(SC))
-                 .addUse(getSPIRVTypeID(BaseType));
-  DT.add(PointerElementType, AddressSpace, CurMF, getSPIRVTypeID(MIB));
-  return finishCreatingSPIRVType(LLVMTy, MIB);
+  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
+    auto MIB = BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(),
+                       MIRBuilder.getDebugLoc(),
+                       MIRBuilder.getTII().get(SPIRV::OpTypePointer))
+                   .addDef(createTypeVReg(CurMF->getRegInfo()))
+                   .addImm(static_cast<uint32_t>(SC))
+                   .addUse(getSPIRVTypeID(BaseType));
+    DT.add(PointerElementType, AddressSpace, CurMF, getSPIRVTypeID(MIB));
+    finishCreatingSPIRVType(LLVMTy, MIB);
+    return MIB;
+  });
 }
 
 SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 3bb86e8be6950..df92325ed1980 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -330,7 +330,7 @@ class SPIRVGlobalRegistry {
   // In cases where the SPIR-V type is already known, this function can be
   // used to map it to the given VReg via an ASSIGN_TYPE instruction.
   void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg,
-                             MachineFunction &MF);
+                             const MachineFunction &MF);
 
   // Either generate a new OpTypeXXX instruction or return an existing one
   // corresponding to the given LLVM IR type.
@@ -509,7 +509,8 @@ class SPIRVGlobalRegistry {
 
 public:
   Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder,
-                            SPIRVType *SpvType, bool EmitIR = true);
+                            SPIRVType *SpvType, bool EmitIR = true,
+                            bool ZeroAsNull = true);
   Register getOrCreateConstInt(uint64_t Val, MachineInstr &I,
                                SPIRVType *SpvType, const SPIRVInstrInfo &TII,
                                bool ZeroAsNull = true);
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 59a1bf50b771b..d5b81bf46c804 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -111,8 +111,8 @@ static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
                             SPIRVGlobalRegistry &GR, MachineInstr &I,
                             Register OpReg, unsigned OpIdx,
                             SPIRVType *NewPtrType) {
-  Register NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
   MachineIRBuilder MIB(I);
+  Register NewReg = createVirtualRegister(NewPtrType, &GR, MRI, MIB.getMF());
   bool Res = MIB.buildInstr(SPIRV::OpBitcast)
                  .addDef(NewReg)
                  .addUse(GR.getSPIRVTypeID(NewPtrType))
@@ -121,8 +121,6 @@ static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
                                    *STI.getRegBankInfo());
   if (!Res)
     report_fatal_error("insert validation bitcast: cannot constrain all uses");
-  MRI->setRegClass(NewReg, &SPIRV::iIDRegClass);
-  GR.assignSPIRVTypeToVReg(NewPtrType, NewReg, MIB.getMF());
   I.getOperand(OpIdx).setReg(NewReg);
 }
 
@@ -396,6 +394,7 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
       case SPIRV::OpGenericCastToPtr:
         validateAccessChain(STI, MRI, GR, MI);
         break;
+      case SPIRV::OpPtrAccessChain:
       case SPIRV::OpInBoundsPtrAccessChain:
         if (MI.getNumOperands() == 4)
           validateAccessChain(STI, MRI, GR, MI);
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index fde23d9d0ca5f..d95803fea56a5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -850,6 +850,12 @@ def OpSubgroupImageBlockReadINTEL: Op<5577, (outs ID:$res), (ins TYPE:$type, ID:
 def OpSubgroupImageBlockWriteINTEL: Op<5578, (outs), (ins ID:$image, ID:$coordinate, ID:$data),
                   "OpSubgroupImageBlockWriteINTEL $image $coordinate $data">;
 
+// SPV_INTEL_media_block_io
+def OpSubgroupImageMediaBlockReadINTEL: Op<5580, (outs ID:$res), (ins TYPE:$type, ID:$image, ID:$coordinate, ID:$width, ID:$height),
+                  "$res = OpSubgroupImageMediaBlockReadINTEL $type $image $coordinate $width $height">;
+def OpSubgroupImageMediaBlockWriteINTEL: Op<5581, (outs), (ins ID:$image, ID:$coordinate, ID:$width, ID:$height, ID:$data),
+                  "OpSubgroupImageMediaBlockWriteINTEL $image $coordinate $width $height $data">;
+
 // - SPV_KHR_uniform_group_instructions
 def OpGroupIMulKHR: Op<6401, (outs ID:$res), (ins TYPE:$type, ID:$scope, i32imm:$groupOp, ID:$value),
                   "$res = OpGroupIMulKHR $type $scope $groupOp $value">;
@@ -889,6 +895,23 @@ def OpCooperativeMatrixMulAddKHR: Op<4459, (outs ID:$res),
 def OpCooperativeMatrixLengthKHR: Op<4460, (outs ID:$res), (ins TYPE:$type, ID:$coop_matr_type),
                   "$res = OpCooperativeMatrixLengthKHR $type $coop_matr_type">;
 
+// SPV_INTEL_joint_matrix
+def OpCooperativeMatrixLoadCheckedINTEL: Op<6193, (outs ID:$res),
+                  (ins TYPE:$resType, ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+                  "$res = OpCooperativeMatrixLoadCheckedINTEL $resType $pointer $xOffset $yOffset $memory_layout $height $width">;
+def OpCooperativeMatrixStoreCheckedINTEL: Op<6194, (outs),
+                  (ins ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$objectToStore, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+                  "OpCooperativeMatrixStoreCheckedINTEL $pointer $xOffset $yOffset $objectToStore $memory_layout $height $width">;
+def OpCooperativeMatrixConstructCheckedINTEL: Op<6195, (outs ID:$res),
+                  (ins TYPE:$resType, ID:$xOffset, ID:$yOffset, ID:$height, ID:$width, ID:$value),
+                  "$res = OpCooperativeMatrixConstructCheckedINTEL $resType $xOffset $yOffset $height $width $value">;
+def OpCooperativeMatrixGetElementCoordINTEL: Op<6440, (outs ID:$res),
+                  (ins TYPE:$resType, ID:$matrix, ID:$index),
+                  "$res = OpCooperativeMatrixGetElementCoordINTEL $resType $matrix $index">;
+def OpCooperativeMatrixPrefetchINTEL: Op<6449, (outs),
+                  (ins ID:$pointer, ID:$rows, ID:$columns, i32imm:$cacheLevel, ID:$memory_layout, variable_ops),
+                  "OpCooperativeMatrixPrefetchINTEL $pointer $rows $columns $cacheLevel $memory_layout">;
+
 // SPV_EXT_arithmetic_fence
 def OpArithmeticFenceEXT: Op<6145, (outs ID:$res), (ins TYPE:$type, ID:$target),
                   "$res = OpArithmeticFenceEXT $type $target">;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 9905691d412bf..824bd6b721679 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -210,6 +210,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
                    MachineInstr &I) const;
 
+  bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
+                   bool IsSigned) const;
+
   bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
                        const SPIRVType *intTy, const SPIRVType *boolTy) const;
 
@@ -241,8 +244,10 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectPhi(Register ResVReg, const SPIRVType *ResType,
                  MachineInstr &I) const;
 
-  bool selectExtInst(Register ResVReg, const SPIRVType *RestType,
-                     MachineInstr &I, GL::GLSLExtInst GLInst) const;
+  [[maybe_unused]] bool selectExtInst(Register ResVReg,
+                                      const SPIRVType *RestType,
+                                      MachineInstr &I,
+                                      GL::GLSLExtInst GLInst) const;
   bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
                      MachineInstr &I, CL::OpenCLExtInst CLInst) const;
   bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
@@ -257,9 +262,6 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectSaturate(Register ResVReg, const SPIRVType *ResType,
                       MachineInstr &I) const;
 
-  bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType,
-                         MachineInstr &I) const;
-
   bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType,
                         MachineInstr &I, unsigned Opcode) const;
 
@@ -304,6 +306,10 @@ class SPIRVInstructionSelector : public InstructionSelector {
   SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const;
   void extractSubvector(Register &ResVReg, const SPIRVType *ResType,
                         Register &ReadReg, MachineInstr &InsertionPoint) const;
+  bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
+  bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
+                              Register ResVReg, const SPIRVType *ResType,
+                              MachineInstr &I) const;
 };
 
 } // end anonymous namespace
@@ -444,6 +450,19 @@ static bool mayApplyGenericSelection(unsigned Opcode) {
   return isTypeFoldingSupported(Opcode);
 }
 
+bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
+                                         MachineInstr &I) const {
+  const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg);
+  const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
+  if (DstRC != SrcRC && SrcRC)
+    MRI->setRegClass(DestReg, SrcRC);
+  return BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                 TII.get(TargetOpcode::COPY))
+      .addDef(DestReg)
+      .addUse(SrcReg)
+      .constrainAllUses(TII, TRI, RBI);
+}
+
 bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
                                          const SPIRVType *ResType,
                                          MachineInstr &I) const {
@@ -533,6 +552,11 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
   case TargetOpcode::G_UMAX:
     return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
 
+  case TargetOpcode::G_SCMP:
+    return selectSUCmp(ResVReg, ResType, I, true);
+  case TargetOpcode::G_UCMP:
+    return selectSUCmp(ResVReg, ResType, I, false);
+
   case TargetOpcode::G_FMA:
     return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
 
@@ -1064,10 +1088,7 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
     addMemoryOperands(*I.memoperands_begin(), MIB);
   Result &= MIB.constrainAllUses(TII, TRI, RBI);
   if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
-    Result &=
-        BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
-            .addUse(MIB->getOperand(0).getReg())
-            .constrainAllUses(TII, TRI, RBI);
+    Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I);
   return Result;
 }
 
@@ -1404,10 +1425,7 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
   // don't generate a cast for a null that may be represented by OpTypeInt
   if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
       ResType->getOpcode() != SPIRV::OpTypePointer)
-    return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(SrcPtr)
-        .constrainAllUses(TII, TRI, RBI);
+    return BuildCOPY(ResVReg, SrcPtr, I);
 
   SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy);
   SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType);
@@ -1445,10 +1463,13 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
 
   // don't generate a cast between identical storage classes
   if (SrcSC == DstSC)
-    return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(SrcPtr)
-        .constrainAllUses(TII, TRI, RBI);
+    return BuildCOPY(ResVReg, SrcPtr, I);
+
+  if ((SrcSC == SPIRV::StorageClass::Function &&
+       DstSC == SPIRV::StorageClass::Private) ||
+      (DstSC == SPIRV::StorageClass::Function &&
+       SrcSC == SPIRV::StorageClass::Private))
+    return BuildCOPY(ResVReg, SrcPtr, I);
 
   // Casting from an eligible pointer to Generic.
   if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
@@ -1623,11 +1644,7 @@ bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
   bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
   if (IsBoolTy && !IsVectorTy) {
     assert(ResVReg == I.getOperand(0).getReg());
-    return BuildMI(*I.getParent(), I, I.getDebugLoc(),
-                   TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(InputRegister)
-        .constrainAllUses(TII, TRI, RBI);
+    return BuildCOPY(ResVReg, InputRegister, I);
   }
 
   bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
@@ -2142,6 +2159,7 @@ bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
     report_fatal_error(
         "There must be at least two constituent operands in a vector");
 
+  MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(IsConst ? SPIRV::OpConstantComposite
                                      : SPIRV::OpCompositeConstruct))
@@ -2175,6 +2193,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
     report_fatal_error(
         "There must be at least two constituent operands in a vector");
 
+  MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(IsConst ? SPIRV::OpConstantComposite
                                      : SPIRV::OpCompositeConstruct))
@@ -2344,7 +2363,7 @@ bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
   bool IsScalarBool =
       GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
   unsigned Opcode =
-      IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond;
+      IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
       .addDef(ResVReg)
       .addUse(GR.getSPIRVTypeID(ResType))
@@ -2382,22 +2401,74 @@ bool SPIRVInstructionSelector::selectExt(Register ResVReg,
     return selectSelect(ResVReg, ResType, I, IsSigned);
 
   SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg);
-  if (SrcType == ResType) {
-    const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg);
-    const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
-    if (DstRC != SrcRC && SrcRC)
-      MRI->setRegClass(ResVReg, SrcRC);
-    return BuildMI(*I.getParent(), I, I.getDebugLoc(),
-                   TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(SrcReg)
-        .constrainAllUses(TII, TRI, RBI);
-  }
+  if (SrcType == ResType)
+    return BuildCOPY(ResVReg, SrcReg, I);
 
   unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
   return selectUnOp(ResVReg, ResType, I, Opcode);
 }
 
+bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
+                                           const SPIRVType *ResType,
+                                           MachineInstr &I,
+                                           bool IsSigned) const {
+  MachineIRBuilder MIRBuilder(I);
+  MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+  MachineBasicBlock &BB = *I.getParent();
+  // Ensure we have bool.
+  SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
+  unsigned N = GR.getScalarOrVectorComponentCount(ResType);
+  if (N > 1)
+    BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
+  Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
+  // Build less-than-equal and less-than.
+  // TODO: replace with one-liner createVirtualRegister() from
+  // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
+  Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
+  MRI->setType(IsLessEqReg, LLT::scalar(64));
+  GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF());
+  bool Result = BuildMI(BB, I, I.getDebugLoc(),
+                        TII.get(IsSigned ? SPIRV::OpSLessThanEqual
+                                         : SPIRV::OpULessThanEqual))
+                    .addDef(IsLessEqReg)
+                    .addUse(BoolTypeReg)
+                    .addUse(I.getOperand(1).getReg())
+                    .addUse(I.getOperand(2).getReg())
+                    .constrainAllUses(TII, TRI, RBI);
+  Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
+  MRI->setType(IsLessReg, LLT::scalar(64));
+  GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF());
+  Result &= BuildMI(BB, I, I.getDebugLoc(),
+                    TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
+                .addDef(IsLessReg)
+                .addUse(BoolTypeReg)
+                .addUse(I.getOperand(1).getReg())
+                .addUse(I.getOperand(2).getReg())
+                .constrainAllUses(TII, TRI, RBI);
+  // Build selects.
+  Register ResTypeReg = GR.getSPIRVTypeID(ResType);
+  Register NegOneOrZeroReg =
+      MRI->createVirtualRegister(GR.getRegClass(ResType));
+  MRI->setType(NegOneOrZeroReg, LLT::scalar(64));
+  GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF());
+  unsigned SelectOpcode =
+      N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
+  Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
+                .addDef(NegOneOrZeroReg)
+                .addUse(ResTypeReg)
+                .addUse(IsLessReg)
+                .addUse(buildOnesVal(true, ResType, I)) // -1
+                .addUse(buildZerosVal(ResType, I))
+                .constrainAllUses(TII, TRI, RBI);
+  return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
+                      .addDef(ResVReg)
+                      .addUse(ResTypeReg)
+                      .addUse(IsLessEqReg)
+                      .addUse(NegOneOrZeroReg) // -1 or 0
+                      .addUse(buildOnesVal(false, ResType, I))
+                      .constrainAllUses(TII, TRI, RBI);
+}
+
 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
                                                Register ResVReg,
                                                MachineInstr &I,
@@ -2431,17 +2502,8 @@ bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
   const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
   if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
     return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
-  if (ArgType == ResType) {
-    const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg);
-    const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg);
-    if (DstRC != SrcRC && SrcRC)
-      MRI->setRegClass(ResVReg, SrcRC);
-    return BuildMI(*I.getParent(), I, I.getDebugLoc(),
-                   TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(IntReg)
-        .constrainAllUses(TII, TRI, RBI);
-  }
+  if (ArgType == ResType)
+    return BuildCOPY(ResVReg, IntReg, I);
   bool IsSigned = GR.isScalarOrVectorSigned(ResType);
   unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
   return selectUnOp(ResVReg, ResType, I, Opcode);
@@ -2463,12 +2525,7 @@ bool SPIRVInstructionSelector::selectConst(Register ResVReg,
   if (TyOpcode == SPIRV::OpTypeInt) {
     assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
     Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
-    if (Reg == ResVReg)
-      return true;
-    return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(Reg)
-        .constrainAllUses(TII, TRI, RBI);
+    return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
   }
   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
                  .addDef(ResVReg)
@@ -2620,7 +2677,7 @@ bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
       continue;
     }
     // Create a new register for the wrapper
-    WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
+    WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType));
     GR.add(OpDefine, MF, WrapReg);
     CompositeArgs.push_back(WrapReg);
     // Decorate the wrapper register and generate a new instruction
@@ -2685,6 +2742,7 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
       if (!wrapIntoSpecConstantOp(I, CompositeArgs))
         return false;
     }
+    MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
                    .addDef(ResVReg)
                    .addUse(GR.getSPIRVTypeID(ResType));
@@ -2764,13 +2822,24 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
           .addUse(I.getOperand(2).getReg())
           .constrainAllUses(TII, TRI, RBI);
     else
-      return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
-                     ResVReg)
-          .addUse(I.getOperand(2).getReg())
-          .constrainAllUses(TII, TRI, RBI);
+      return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I);
     break;
   case Intrinsic::spv_thread_id:
-    return selectSpvThreadId(ResVReg, ResType, I);
+    // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
+    // intrinsic in LLVM IR for SPIR-V backend.
+    //
+    // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
+    // `GlobalInvocationId` builtin variable
+    return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
+                                  ResType, I);
+  case Intrinsic::spv_thread_id_in_group:
+    // The HLSL SV_GroupThreadId semantic is lowered to
+    // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
+    //
+    // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
+    // translated to a `LocalInvocationId` builtin variable
+    return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
+                                  ResType, I);
   case Intrinsic::spv_fdot:
     return selectFloatDot(ResVReg, ResType, I);
   case Intrinsic::spv_udot:
@@ -3217,12 +3286,17 @@ bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
   // there was an allocation size parameter to the allocation instruction
   // that is not 1
   MachineBasicBlock &BB = *I.getParent();
-  return BuildMI(BB, I, I.getDebugLoc(),
-                 TII.get(SPIRV::OpVariableLengthArrayINTEL))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addUse(I.getOperand(2).getReg())
-      .constrainAllUses(TII, TRI, RBI);
+  bool Res = BuildMI(BB, I, I.getDebugLoc(),
+                     TII.get(SPIRV::OpVariableLengthArrayINTEL))
+                 .addDef(ResVReg)
+                 .addUse(GR.getSPIRVTypeID(ResType))
+                 .addUse(I.getOperand(2).getReg())
+                 .constrainAllUses(TII, TRI, RBI);
+  if (!STI.isVulkanEnv()) {
+    unsigned Alignment = I.getOperand(3).getImm();
+    buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
+  }
+  return Res;
 }
 
 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
@@ -3231,12 +3305,18 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
   // Change order of instructions if needed: all OpVariable instructions in a
   // function must be the first instructions in the first block
   auto It = getOpVariableMBBIt(I);
-  return BuildMI(*It->getParent(), It, It->getDebugLoc(),
-                 TII.get(SPIRV::OpVariable))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
-      .constrainAllUses(TII, TRI, RBI);
+  bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(),
+                     TII.get(SPIRV::OpVariable))
+                 .addDef(ResVReg)
+                 .addUse(GR.getSPIRVTypeID(ResType))
+                 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
+                 .constrainAllUses(TII, TRI, RBI);
+  if (!STI.isVulkanEnv()) {
+    unsigned Alignment = I.getOperand(2).getImm();
+    buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
+                    {Alignment});
+  }
+  return Res;
 }
 
 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
@@ -3296,7 +3376,10 @@ bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
     MIB.addUse(I.getOperand(i + 0).getReg());
     MIB.addMBB(I.getOperand(i + 1).getMBB());
   }
-  return MIB.constrainAllUses(TII, TRI, RBI);
+  bool Res = MIB.constrainAllUses(TII, TRI, RBI);
+  MIB->setDesc(TII.get(TargetOpcode::PHI));
+  MIB->removeOperand(1);
+  return Res;
 }
 
 bool SPIRVInstructionSelector::selectGlobalValue(
@@ -3375,10 +3458,7 @@ bool SPIRVInstructionSelector::selectGlobalValue(
           .constrainAllUses(TII, TRI, RBI);
     }
     assert(NewReg != ResVReg);
-    return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
-        .addDef(ResVReg)
-        .addUse(NewReg)
-        .constrainAllUses(TII, TRI, RBI);
+    return BuildCOPY(ResVReg, NewReg, I);
   }
   auto GlobalVar = cast<GlobalVariable>(GV);
   assert(GlobalVar->getName() != "llvm.global.annotations");
@@ -3390,11 +3470,7 @@ bool SPIRVInstructionSelector::selectGlobalValue(
   if (HasInit && !Init)
     return true;
 
-  unsigned AddrSpace = GV->getAddressSpace();
-  SPIRV::StorageClass::StorageClass Storage =
-      addressSpaceToStorageClass(AddrSpace, STI);
-  bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
-                  Storage != SPIRV::StorageClass::Function;
+  bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage;
   SPIRV::LinkageType::LinkageType LnkType =
       (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
           ? SPIRV::LinkageType::Import
@@ -3403,12 +3479,14 @@ bool SPIRVInstructionSelector::selectGlobalValue(
                  ? SPIRV::LinkageType::LinkOnceODR
                  : SPIRV::LinkageType::Export);
 
-  SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
-      PointerBaseType, I, TII,
-      addressSpaceToStorageClass(GV->getAddressSpace(), STI));
-  Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
-                                        Storage, Init, GlobalVar->isConstant(),
-                                        HasLnkTy, LnkType, MIRBuilder, true);
+  const unsigned AddrSpace = GV->getAddressSpace();
+  SPIRV::StorageClass::StorageClass StorageClass =
+      addressSpaceToStorageClass(AddrSpace, STI);
+  SPIRVType *ResType =
+      GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass);
+  Register Reg = GR.buildGlobalVariable(
+      ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
+      GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true);
   return Reg.isValid();
 }
 
@@ -3461,13 +3539,12 @@ bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
                        .constrainAllUses(TII, TRI, RBI);
 }
 
-bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg,
-                                                 const SPIRVType *ResType,
-                                                 MachineInstr &I) const {
-  // DX intrinsic: @llvm.dx.thread.id(i32)
-  // ID  Name      Description
-  // 93  ThreadId  reads the thread ID
-
+// Generate the instructions to load 3-element vector builtin input
+// IDs/Indices.
+// Like: GlobalInvocationId, LocalInvocationId, etc....
+bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
+    SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
+    const SPIRVType *ResType, MachineInstr &I) const {
   MachineIRBuilder MIRBuilder(I);
   const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
   const SPIRVType *Vec3Ty =
@@ -3475,16 +3552,16 @@ bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg,
   const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
       Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
 
-  // Create new register for GlobalInvocationID builtin variable.
+  // Create new register for the input ID builtin variable.
   Register NewRegister =
       MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
   MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64));
   GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
 
-  // Build GlobalInvocationID global variable with the necessary decorations.
+  // Build global variable with the necessary decorations for the input ID
+  // builtin variable.
   Register Variable = GR.buildGlobalVariable(
-      NewRegister, PtrType,
-      getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr,
+      NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
       SPIRV::StorageClass::Input, nullptr, true, true,
       SPIRV::LinkageType::Import, MIRBuilder, false);
 
@@ -3501,12 +3578,12 @@ bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg,
           .addUse(GR.getSPIRVTypeID(Vec3Ty))
           .addUse(Variable);
 
-  // Get Thread ID index. Expecting operand is a constant immediate value,
+  // Get the input ID index. Expecting operand is a constant immediate value,
   // wrapped in a type assignment.
   assert(I.getOperand(2).isReg());
   const uint32_t ThreadId = foldImm(I.getOperand(2), MRI);
 
-  // Extract the thread ID from the loaded vector value.
+  // Extract the input ID from the loaded vector value.
   MachineBasicBlock &BB = *I.getParent();
   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
                  .addDef(ResVReg)
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 460f0127d4ffc..7230e0e6b9fca 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -112,13 +112,16 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
   const LLT p5 =
       LLT::pointer(5, PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
   const LLT p6 = LLT::pointer(6, PSize); // SPV_INTEL_usm_storage_classes (Host)
+  const LLT p7 = LLT::pointer(7, PSize); // Input
+  const LLT p8 = LLT::pointer(8, PSize); // Output
+  const LLT p10 = LLT::pointer(10, PSize); // Private
 
   // TODO: remove copy-pasting here by using concatenation in some way.
   auto allPtrsScalarsAndVectors = {
-      p0,    p1,    p2,    p3,    p4,     p5,     p6,    s1,   s8,   s16,
-      s32,   s64,   v2s1,  v2s8,  v2s16,  v2s32,  v2s64, v3s1, v3s8, v3s16,
-      v3s32, v3s64, v4s1,  v4s8,  v4s16,  v4s32,  v4s64, v8s1, v8s8, v8s16,
-      v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
+      p0,   p1,   p2,    p3,    p4,    p5,    p6,    p7,     p8,     p10,
+      s1,   s8,   s16,   s32,   s64,   v2s1,  v2s8,  v2s16,  v2s32,  v2s64,
+      v3s1, v3s8, v3s16, v3s32, v3s64, v4s1,  v4s8,  v4s16,  v4s32,  v4s64,
+      v8s1, v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
 
   auto allVectors = {v2s1,  v2s8,   v2s16,  v2s32, v2s64, v3s1,  v3s8,
                      v3s16, v3s32,  v3s64,  v4s1,  v4s8,  v4s16, v4s32,
@@ -145,10 +148,10 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
       s16,   s32,   s64,   v2s16, v2s32, v2s64, v3s16,  v3s32,  v3s64,
       v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
 
-  auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
-                                       p2, p3,  p4,  p5,  p6};
+  auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1, p2,
+                                       p3, p4,  p5,  p6,  p7, p8, p10};
 
-  auto allPtrs = {p0, p1, p2, p3, p4, p5, p6};
+  auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p10};
 
   bool IsExtendedInts =
       ST.canUseExtension(
@@ -212,7 +215,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
 
   getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
                                G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
-                               G_USUBSAT})
+                               G_USUBSAT, G_SCMP, G_UCMP})
       .legalFor(allIntScalarsAndVectors)
       .legalIf(extendedScalarsAndVectors);
 
@@ -357,12 +360,13 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
   verify(*ST.getInstrInfo());
 }
 
-static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVType *SpirvType,
+static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVType *SpvType,
                                 LegalizerHelper &Helper,
                                 MachineRegisterInfo &MRI,
                                 SPIRVGlobalRegistry *GR) {
   Register ConvReg = MRI.createGenericVirtualRegister(ConvTy);
-  GR->assignSPIRVTypeToVReg(SpirvType, ConvReg, Helper.MIRBuilder.getMF());
+  MRI.setRegClass(ConvReg, GR->getRegClass(SpvType));
+  GR->assignSPIRVTypeToVReg(SpvType, ConvReg, Helper.MIRBuilder.getMF());
   Helper.MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)
       .addDef(ConvReg)
       .addUse(Reg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index d9f928eb90640..4dea4056799fe 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -421,6 +421,7 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
       continue;
     MachineFunction *MF = MMI->getMachineFunction(*F);
     assert(MF);
+
     for (MachineBasicBlock &MBB : *MF)
       for (MachineInstr &MI : MBB) {
         if (MAI.getSkipEmission(&MI))
@@ -1320,6 +1321,13 @@ void addInstrRequirements(const MachineInstr &MI,
       Reqs.addCapability(SPIRV::Capability::SubgroupImageBlockIOINTEL);
     }
     break;
+  case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
+  case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
+    if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {
+      Reqs.addExtension(SPIRV::Extension::SPV_INTEL_media_block_io);
+      Reqs.addCapability(SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
+    }
+    break;
   case SPIRV::OpAssumeTrueKHR:
   case SPIRV::OpExpectKHR:
     if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
@@ -1429,6 +1437,138 @@ void addInstrRequirements(const MachineInstr &MI,
       Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
     }
     break;
+  case SPIRV::OpCooperativeMatrixMulAddKHR: {
+    if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+      report_fatal_error("Cooperative matrix instructions require the "
+                         "following SPIR-V extension: "
+                         "SPV_KHR_cooperative_matrix",
+                         false);
+    Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+    Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+    constexpr unsigned MulAddMaxSize = 6;
+    if (MI.getNumOperands() != MulAddMaxSize)
+      break;
+    const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
+    if (CoopOperands &
+        SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
+      if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+        report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
+                           "require the following SPIR-V extension: "
+                           "SPV_INTEL_joint_matrix",
+                           false);
+      Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+      Reqs.addCapability(
+          SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
+    }
+    if (CoopOperands & SPIRV::CooperativeMatrixOperands::
+                           MatrixAAndBBFloat16ComponentsINTEL ||
+        CoopOperands &
+            SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
+        CoopOperands & SPIRV::CooperativeMatrixOperands::
+                           MatrixResultBFloat16ComponentsINTEL) {
+      if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+        report_fatal_error("***BF16ComponentsINTEL type interpretations "
+                           "require the following SPIR-V extension: "
+                           "SPV_INTEL_joint_matrix",
+                           false);
+      Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+      Reqs.addCapability(
+          SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
+    }
+    break;
+  }
+  case SPIRV::OpCooperativeMatrixLoadKHR:
+  case SPIRV::OpCooperativeMatrixStoreKHR:
+  case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+  case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+  case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
+    if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+      report_fatal_error("Cooperative matrix instructions require the "
+                         "following SPIR-V extension: "
+                         "SPV_KHR_cooperative_matrix",
+                         false);
+    Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+    Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+
+    // Check Layout operand in case if it's not a standard one and add the
+    // appropriate capability.
+    std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
+        {SPIRV::OpCooperativeMatrixLoadKHR, 3},
+        {SPIRV::OpCooperativeMatrixStoreKHR, 2},
+        {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
+        {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
+        {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}};
+
+    const auto OpCode = MI.getOpcode();
+    const unsigned LayoutNum = LayoutToInstMap[OpCode];
+    Register RegLayout = MI.getOperand(LayoutNum).getReg();
+    const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+    MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
+    if (MILayout->getOpcode() == SPIRV::OpConstantI) {
+      const unsigned LayoutVal = MILayout->getOperand(2).getImm();
+      if (LayoutVal ==
+          static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
+        if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+          report_fatal_error("PackedINTEL layout require the following SPIR-V "
+                             "extension: SPV_INTEL_joint_matrix",
+                             false);
+        Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+        Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
+      }
+    }
+
+    // Nothing to do.
+    if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||
+        OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
+      break;
+
+    std::string InstName;
+    switch (OpCode) {
+    case SPIRV::OpCooperativeMatrixPrefetchINTEL:
+      InstName = "OpCooperativeMatrixPrefetchINTEL";
+      break;
+    case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+      InstName = "OpCooperativeMatrixLoadCheckedINTEL";
+      break;
+    case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+      InstName = "OpCooperativeMatrixStoreCheckedINTEL";
+      break;
+    }
+
+    if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {
+      const std::string ErrorMsg =
+          InstName + " instruction requires the "
+                     "following SPIR-V extension: SPV_INTEL_joint_matrix";
+      report_fatal_error(ErrorMsg.c_str(), false);
+    }
+    Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+    if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+      Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
+      break;
+    }
+    Reqs.addCapability(
+        SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+    break;
+  }
+  case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
+    if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+      report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL "
+                         "instructions require the following SPIR-V extension: "
+                         "SPV_INTEL_joint_matrix",
+                         false);
+    Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+    Reqs.addCapability(
+        SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+    break;
+  case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
+    if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+      report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "
+                         "following SPIR-V extension: SPV_INTEL_joint_matrix",
+                         false);
+    Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+    Reqs.addCapability(
+        SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
+    break;
   case SPIRV::OpKill: {
     Reqs.addCapability(SPIRV::Capability::Shader);
   } break;
@@ -1541,11 +1681,14 @@ static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
           SPIRV::OperandCategory::ExecutionModeOperand,
           SPIRV::ExecutionMode::VecTypeHint, ST);
 
-    if (F.hasOptNone() &&
-        ST.canUseExtension(SPIRV::Extension::SPV_INTEL_optnone)) {
-      // Output OpCapability OptNoneINTEL.
-      MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_optnone);
-      MAI.Reqs.addCapability(SPIRV::Capability::OptNoneINTEL);
+    if (F.hasOptNone()) {
+      if (ST.canUseExtension(SPIRV::Extension::SPV_EXT_optnone)) {
+        MAI.Reqs.addExtension(SPIRV::Extension::SPV_EXT_optnone);
+        MAI.Reqs.addCapability(SPIRV::Capability::OptNoneEXT);
+      } else if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_optnone)) {
+        MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_optnone);
+        MAI.Reqs.addCapability(SPIRV::Capability::OptNoneINTEL);
+      }
     }
   }
 }
@@ -1606,6 +1749,48 @@ static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
   }
 }
 
+static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
+                        MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
+                        SPIRV::ModuleAnalysisInfo &MAI) {
+  for (auto F = M.begin(), E = M.end(); F != E; ++F) {
+    MachineFunction *MF = MMI->getMachineFunction(*F);
+    if (!MF)
+      continue;
+    MachineRegisterInfo &MRI = MF->getRegInfo();
+    for (auto &MBB : *MF) {
+      if (!MBB.hasName() || MBB.empty())
+        continue;
+      // Emit basic block names.
+      Register Reg = MRI.createGenericVirtualRegister(LLT::scalar(64));
+      MRI.setRegClass(Reg, &SPIRV::IDRegClass);
+      buildOpName(Reg, MBB.getName(), *std::prev(MBB.end()), TII);
+      Register GlobalReg = MAI.getOrCreateMBBRegister(MBB);
+      MAI.setRegisterAlias(MF, Reg, GlobalReg);
+    }
+  }
+}
+
+// patching Instruction::PHI to SPIRV::OpPhi
+static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,
+                      const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {
+  for (auto F = M.begin(), E = M.end(); F != E; ++F) {
+    MachineFunction *MF = MMI->getMachineFunction(*F);
+    if (!MF)
+      continue;
+    for (auto &MBB : *MF) {
+      for (MachineInstr &MI : MBB) {
+        if (MI.getOpcode() != TargetOpcode::PHI)
+          continue;
+        MI.setDesc(TII.get(SPIRV::OpPhi));
+        Register ResTypeReg = GR->getSPIRVTypeID(
+            GR->getSPIRVTypeForVReg(MI.getOperand(0).getReg(), MF));
+        MI.insert(MI.operands_begin() + 1,
+                  {MachineOperand::CreateReg(ResTypeReg, false)});
+      }
+    }
+  }
+}
+
 struct SPIRV::ModuleAnalysisInfo SPIRVModuleAnalysis::MAI;
 
 void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
@@ -1624,6 +1809,9 @@ bool SPIRVModuleAnalysis::runOnModule(Module &M) {
 
   setBaseInfo(M);
 
+  patchPhis(M, GR, *TII, MMI);
+
+  addMBBNames(M, *TII, MMI, *ST, MAI);
   addDecorations(M, *TII, MMI, *ST, MAI);
 
   collectReqs(M, MAI, MMI, *ST);
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 024728c347e8a..ee2aaf156aa89 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -157,7 +157,7 @@ struct ModuleAnalysisInfo {
   // The array contains lists of MIs for each module section.
   InstrList MS[NUM_MODULE_SECTIONS];
   // The table maps MBB number to SPIR-V unique ID register.
-  DenseMap<int, Register> BBNumToRegMap;
+  DenseMap<std::pair<const MachineFunction *, int>, Register> BBNumToRegMap;
 
   Register getFuncReg(const Function *F) {
     assert(F && "Function is null");
@@ -188,15 +188,17 @@ struct ModuleAnalysisInfo {
   }
   unsigned getNextID() { return MaxID++; }
   bool hasMBBRegister(const MachineBasicBlock &MBB) {
-    return BBNumToRegMap.contains(MBB.getNumber());
+    auto Key = std::make_pair(MBB.getParent(), MBB.getNumber());
+    return BBNumToRegMap.contains(Key);
   }
   // Convert MBB's number to corresponding ID register.
   Register getOrCreateMBBRegister(const MachineBasicBlock &MBB) {
-    auto f = BBNumToRegMap.find(MBB.getNumber());
-    if (f != BBNumToRegMap.end())
-      return f->second;
+    auto Key = std::make_pair(MBB.getParent(), MBB.getNumber());
+    auto It = BBNumToRegMap.find(Key);
+    if (It != BBNumToRegMap.end())
+      return It->second;
     Register NewReg = Register::index2VirtReg(getNextID());
-    BBNumToRegMap[MBB.getNumber()] = NewReg;
+    BBNumToRegMap[Key] = NewReg;
     return NewReg;
   }
 };
diff --git a/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def
new file mode 100644
index 0000000000000..e4f6c71d5d791
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def
@@ -0,0 +1,21 @@
+//===- SPIRVPassRegistry.def - Registry of SPIRV passes -----*- C++--*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the
+// SPIRV backend.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+
+#ifndef FUNCTION_PASS
+#define FUNCTION_PASS(NAME, CREATE_PASS)
+#endif
+FUNCTION_PASS("spirv-structurizer", SPIRVStructurizerWrapper())
+#undef FUNCTION_PASS
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index 11b9e4f6f6d17..3373d8e24dab4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -102,10 +102,7 @@ static void processNewInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
           if (!ResType) {
             // There was no "assign type" actions, let's fix this now
             ResType = ScalarType;
-            MRI.setRegClass(ResVReg, &SPIRV::iIDRegClass);
-            MRI.setType(ResVReg,
-                        LLT::scalar(GR->getScalarOrVectorBitWidth(ResType)));
-            GR->assignSPIRVTypeToVReg(ResType, ResVReg, *GR->CurMF);
+            setRegClassType(ResVReg, ResType, GR, &MRI, *GR->CurMF, true);
           }
         }
       } else if (mayBeInserted(Opcode) && I.getNumDefs() == 1 &&
@@ -124,9 +121,7 @@ static void processNewInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
           if (!ResVType)
             continue;
           // Set type & class
-          MRI.setRegClass(ResVReg, GR->getRegClass(ResVType));
-          MRI.setType(ResVReg, GR->getRegType(ResVType));
-          GR->assignSPIRVTypeToVReg(ResVType, ResVReg, *GR->CurMF);
+          setRegClassType(ResVReg, ResVType, GR, &MRI, *GR->CurMF, true);
         }
         // If this is a simple operation that is to be reduced by TableGen
         // definition we must apply some of pre-legalizer rules here
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index d5299043d9ef2..ceccf55d1de4d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -466,8 +466,13 @@ void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
   for (auto &Op : MI.operands()) {
     if (!Op.isReg() || Op.isDef())
       continue;
-    auto IdOpInfo = createNewIdReg(nullptr, Op.getReg(), MRI, *GR);
-    MIB.buildInstr(IdOpInfo.second).addDef(IdOpInfo.first).addUse(Op.getReg());
+    Register OpReg = Op.getReg();
+    SPIRVType *SpvType = GR->getSPIRVTypeForVReg(OpReg);
+    auto IdOpInfo = createNewIdReg(SpvType, OpReg, MRI, *GR);
+    MIB.buildInstr(IdOpInfo.second).addDef(IdOpInfo.first).addUse(OpReg);
+    const TargetRegisterClass *RC = GR->getRegClass(SpvType);
+    if (RC != MRI.getRegClassOrNull(OpReg))
+      MRI.setRegClass(OpReg, RC);
     Op.setReg(IdOpInfo.first);
   }
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 13e05b6792751..336cde4e78224 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -10,6 +10,7 @@
 
 #include "Analysis/SPIRVConvergenceRegionAnalysis.h"
 #include "SPIRV.h"
+#include "SPIRVStructurizerWrapper.h"
 #include "SPIRVSubtarget.h"
 #include "SPIRVTargetMachine.h"
 #include "SPIRVUtils.h"
@@ -17,6 +18,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/IR/Analysis.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
@@ -1211,16 +1213,26 @@ class SPIRVStructurizer : public FunctionPass {
 
 char SPIRVStructurizer::ID = 0;
 
-INITIALIZE_PASS_BEGIN(SPIRVStructurizer, "structurizer", "structurize SPIRV",
-                      false, false)
+INITIALIZE_PASS_BEGIN(SPIRVStructurizer, "spirv-structurizer",
+                      "structurize SPIRV", false, false)
 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(SPIRVConvergenceRegionAnalysisWrapperPass)
 
-INITIALIZE_PASS_END(SPIRVStructurizer, "structurize", "structurize SPIRV",
-                    false, false)
+INITIALIZE_PASS_END(SPIRVStructurizer, "spirv-structurizer",
+                    "structurize SPIRV", false, false)
 
 FunctionPass *llvm::createSPIRVStructurizerPass() {
   return new SPIRVStructurizer();
 }
+
+PreservedAnalyses SPIRVStructurizerWrapper::run(Function &F,
+                                                FunctionAnalysisManager &AF) {
+  FunctionPass *StructurizerPass = createSPIRVStructurizerPass();
+  if (!StructurizerPass->runOnFunction(F))
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizerWrapper.h b/llvm/lib/Target/SPIRV/SPIRVStructurizerWrapper.h
new file mode 100644
index 0000000000000..cc69e20847c19
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizerWrapper.h
@@ -0,0 +1,29 @@
+//===- SPIRVStructurizerWrapper.h - New pass manager wrapper from SPIRV
+// Structurizer -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file New pass manager wrapper from SPIRV Structurizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_DIRECTX_SPIRVSTRUCTURIZER_H
+#define LLVM_LIB_TARGET_DIRECTX_SPIRVSTRUCTURIZER_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class SPIRVStructurizerWrapper
+    : public PassInfoMixin<SPIRVStructurizerWrapper> {
+public:
+  PreservedAnalyses run(Function &M, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_DIRECTX_SPIRVSTRUCTURIZER_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index b88f6f5766a05..745d1e1aec67a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -170,6 +170,8 @@ def GroupOperationOperand : OperandCategory;
 def KernelEnqueueFlagsOperand : OperandCategory;
 def KernelProfilingInfoOperand : OperandCategory;
 def OpcodeOperand : OperandCategory;
+def CooperativeMatrixLayoutOperand : OperandCategory;
+def CooperativeMatrixOperandsOperand : OperandCategory;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Extesions enum values and at the same time
@@ -304,6 +306,8 @@ defm SPV_INTEL_global_variable_host_access : ExtensionOperand<109>;
 defm SPV_INTEL_global_variable_fpga_decorations : ExtensionOperand<110>;
 defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>;
 defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>;
+defm SPV_EXT_optnone : ExtensionOperand<113>;
+defm SPV_INTEL_joint_matrix : ExtensionOperand<114>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Capabilities enum values and at the same time
@@ -446,7 +450,7 @@ defm RayTracingNV : CapabilityOperand<5340, 0, 0, [], [Shader]>;
 defm SubgroupShuffleINTEL : CapabilityOperand<5568, 0, 0, [SPV_INTEL_subgroups], []>;
 defm SubgroupBufferBlockIOINTEL : CapabilityOperand<5569, 0, 0, [SPV_INTEL_subgroups], []>;
 defm SubgroupImageBlockIOINTEL : CapabilityOperand<5570, 0, 0, [SPV_INTEL_subgroups], []>;
-defm SubgroupImageMediaBlockIOINTEL : CapabilityOperand<5579, 0, 0, [], []>;
+defm SubgroupImageMediaBlockIOINTEL : CapabilityOperand<5579, 0, 0, [SPV_INTEL_media_block_io], []>;
 defm SubgroupAvcMotionEstimationINTEL : CapabilityOperand<5696, 0, 0, [], []>;
 defm SubgroupAvcMotionEstimationIntraINTEL : CapabilityOperand<5697, 0, 0, [], []>;
 defm SubgroupAvcMotionEstimationChromaINTEL : CapabilityOperand<5698, 0, 0, [], []>;
@@ -463,6 +467,7 @@ defm PhysicalStorageBufferAddressesEXT : CapabilityOperand<5347, 0, 0, [], [Shad
 defm CooperativeMatrixNV : CapabilityOperand<5357, 0, 0, [], [Shader]>;
 defm ArbitraryPrecisionIntegersINTEL : CapabilityOperand<5844, 0, 0, [SPV_INTEL_arbitrary_precision_integers], [Int8, Int16]>;
 defm OptNoneINTEL : CapabilityOperand<6094, 0, 0, [SPV_INTEL_optnone], []>;
+defm OptNoneEXT : CapabilityOperand<6094, 0, 0, [SPV_EXT_optnone], []>;
 defm BitInstructions : CapabilityOperand<6025, 0, 0, [SPV_KHR_bit_instructions], []>;
 defm ExpectAssumeKHR : CapabilityOperand<5629, 0, 0, [SPV_KHR_expect_assume], []>;
 defm FunctionPointersINTEL : CapabilityOperand<5603, 0, 0, [SPV_INTEL_function_pointers], []>;
@@ -490,6 +495,12 @@ defm CacheControlsINTEL : CapabilityOperand<6441, 0, 0, [SPV_INTEL_cache_control
 defm CooperativeMatrixKHR : CapabilityOperand<6022, 0, 0, [SPV_KHR_cooperative_matrix], []>;
 defm ArithmeticFenceEXT : CapabilityOperand<6144, 0, 0, [SPV_EXT_arithmetic_fence], []>;
 defm SplitBarrierINTEL : CapabilityOperand<6141, 0, 0, [SPV_INTEL_split_barrier], []>;
+defm CooperativeMatrixCheckedInstructionsINTEL : CapabilityOperand<6192, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixPrefetchINTEL : CapabilityOperand<6411, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm PackedCooperativeMatrixINTEL : CapabilityOperand<6434, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixInvocationInstructionsINTEL : CapabilityOperand<6435, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixTF32ComponentTypeINTEL : CapabilityOperand<6436, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixBFloat16ComponentTypeINTEL : CapabilityOperand<6437, 0, 0, [SPV_INTEL_joint_matrix], []>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define SourceLanguage enum values and at the same time
@@ -1433,6 +1444,7 @@ defm Inline : FunctionControlOperand<0x1>;
 defm DontInline : FunctionControlOperand<0x2>;
 defm Pure : FunctionControlOperand<0x4>;
 defm Const : FunctionControlOperand<0x8>;
+defm OptNoneEXT : FunctionControlOperand<0x10000>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define MemorySemantics enum values and at the same time
@@ -1646,3 +1658,62 @@ defm GenericCastToPtr : OpcodeOperand<122>;
 defm Bitcast : OpcodeOperand<124>;
 defm ConvertPtrToU : OpcodeOperand<117>;
 defm ConvertUToPtr : OpcodeOperand<120>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Layout enum values and at the
+// same time SymbolicOperand entries extensions and capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixLayout : GenericEnum, Operand<i32> {
+  let FilterClass = "CooperativeMatrixLayout";
+  let NameField = "Name";
+  let ValueField = "Value";
+}
+
+class CooperativeMatrixLayout<string name, bits<32> value> {
+  string Name = name;
+  bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixLayoutOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+  def : CooperativeMatrixLayout<NAME, value>;
+  defm : SymbolicOperandWithRequirements<CooperativeMatrixLayoutOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm RowMajorKHR : CooperativeMatrixLayoutOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm ColumnMajorKHR : CooperativeMatrixLayoutOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm PackedINTEL : CooperativeMatrixLayoutOperand<0x2, [SPV_INTEL_joint_matrix], [PackedCooperativeMatrixINTEL]>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Operands enum values and at the
+// same time SymbolicOperand entries with string mnemonics, extensions and
+// capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixOperands : GenericEnum, Operand<i32> {
+  let FilterClass = "CooperativeMatrixOperands";
+  let NameField = "Name";
+  let ValueField = "Value";
+  let PrintMethod = !strconcat("printSymbolicOperand<OperandCategory::", FilterClass, "Operand>");
+}
+
+class CooperativeMatrixOperands<string name, bits<32> value> {
+  string Name = name;
+  bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+  def : CooperativeMatrixOperands<NAME, value>;
+  defm : SymbolicOperandWithRequirements<CooperativeMatrixOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm NoneKHR : CooperativeMatrixOperandsOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixASignedComponentsKHR : CooperativeMatrixOperandsOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixBSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x2, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixCSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x4, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixResultSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x8, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm SaturatingAccumulationKHR : CooperativeMatrixOperandsOperand<0x10, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixAAndBTF32ComponentsINTEL : CooperativeMatrixOperandsOperand<0x20, [SPV_INTEL_joint_matrix], [CooperativeMatrixTF32ComponentTypeINTEL]>;
+defm MatrixAAndBBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x40, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixCBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x80, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixResultBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x100, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 8f38d4b8307da..dca67cb6c632b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -15,6 +15,7 @@
 #include "SPIRVCallLowering.h"
 #include "SPIRVGlobalRegistry.h"
 #include "SPIRVLegalizerInfo.h"
+#include "SPIRVStructurizerWrapper.h"
 #include "SPIRVTargetObjectFile.h"
 #include "SPIRVTargetTransformInfo.h"
 #include "TargetInfo/SPIRVTargetInfo.h"
@@ -28,6 +29,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Pass.h"
+#include "llvm/Passes/PassBuilder.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/Scalar/Reg2Mem.h"
 #include "llvm/Transforms/Utils.h"
@@ -45,6 +47,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() {
   initializeGlobalISel(PR);
   initializeSPIRVModuleAnalysisPass(PR);
   initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PR);
+  initializeSPIRVStructurizerPass(PR);
 }
 
 static std::string computeDataLayout(const Triple &TT) {
@@ -92,6 +95,11 @@ SPIRVTargetMachine::SPIRVTargetMachine(const Target &T, const Triple &TT,
   setRequiresStructuredCFG(false);
 }
 
+void SPIRVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
+#define GET_PASS_REGISTRY "SPIRVPassRegistry.def"
+#include "llvm/Passes/TargetPassRegistry.inc"
+}
+
 namespace {
 // SPIR-V Code Generator Pass Configuration Options.
 class SPIRVPassConfig : public TargetPassConfig {
@@ -130,13 +138,8 @@ FunctionPass *SPIRVPassConfig::createTargetRegisterAllocator(bool) {
   return nullptr;
 }
 
-// Disable passes that may break CFG.
+// A place to disable passes that may break CFG.
 void SPIRVPassConfig::addMachineSSAOptimization() {
-  // Some standard passes that optimize machine instructions in SSA form uses
-  // MI.isPHI() that doesn't account for OpPhi in SPIR-V and so are able to
-  // break the CFG (e.g., MachineSink).
-  disablePass(&MachineSinkingID);
-
   TargetPassConfig::addMachineSSAOptimization();
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.h b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.h
index 3ea0f671ff6d8..9c59d021dfc1b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.h
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.h
@@ -43,6 +43,8 @@ class SPIRVTargetMachine : public CodeGenTargetMachineImpl {
   TargetLoweringObjectFile *getObjFileLowering() const override {
     return TLOF.get();
   }
+
+  void registerPassBuilderCallbacks(PassBuilder &PB) override;
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index ad8dfa0e8811b..ce90e335fe404 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -13,6 +13,7 @@
 #include "SPIRVUtils.h"
 #include "MCTargetDesc/SPIRVBaseInfo.h"
 #include "SPIRV.h"
+#include "SPIRVGlobalRegistry.h"
 #include "SPIRVInstrInfo.h"
 #include "SPIRVSubtarget.h"
 #include "llvm/ADT/StringRef.h"
@@ -21,6 +22,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Demangle/Demangle.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsSPIRV.h"
 #include <queue>
 #include <vector>
@@ -106,6 +108,16 @@ void buildOpName(Register Target, const StringRef &Name,
   }
 }
 
+void buildOpName(Register Target, const StringRef &Name, MachineInstr &I,
+                 const SPIRVInstrInfo &TII) {
+  if (!Name.empty()) {
+    auto MIB =
+        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpName))
+            .addUse(Target);
+    addStringImm(Name, MIB);
+  }
+}
+
 static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
                                   const std::vector<uint32_t> &DecArgs,
                                   StringRef StrImm) {
@@ -205,8 +217,12 @@ addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
                : SPIRV::StorageClass::CrossWorkgroup;
   case 7:
     return SPIRV::StorageClass::Input;
+  case 8:
+    return SPIRV::StorageClass::Output;
   case 9:
     return SPIRV::StorageClass::CodeSectionINTEL;
+  case 10:
+    return SPIRV::StorageClass::Private;
   default:
     report_fatal_error("Unknown address space");
   }
@@ -405,8 +421,10 @@ bool hasBuiltinTypePrefix(StringRef Name) {
 }
 
 bool isSpecialOpaqueType(const Type *Ty) {
-  if (const TargetExtType *EType = dyn_cast<TargetExtType>(Ty))
-    return hasBuiltinTypePrefix(EType->getName());
+  if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Ty))
+    return isTypedPointerWrapper(ExtTy)
+               ? false
+               : hasBuiltinTypePrefix(ExtTy->getName());
 
   return false;
 }
@@ -429,25 +447,31 @@ Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
   TypeName.consume_front("atomic_");
   if (TypeName.consume_front("void"))
     return Type::getVoidTy(Ctx);
-  else if (TypeName.consume_front("bool"))
+  else if (TypeName.consume_front("bool") || TypeName.consume_front("_Bool"))
     return Type::getIntNTy(Ctx, 1);
   else if (TypeName.consume_front("char") ||
+           TypeName.consume_front("signed char") ||
            TypeName.consume_front("unsigned char") ||
            TypeName.consume_front("uchar"))
     return Type::getInt8Ty(Ctx);
   else if (TypeName.consume_front("short") ||
+           TypeName.consume_front("signed short") ||
            TypeName.consume_front("unsigned short") ||
            TypeName.consume_front("ushort"))
     return Type::getInt16Ty(Ctx);
   else if (TypeName.consume_front("int") ||
+           TypeName.consume_front("signed int") ||
            TypeName.consume_front("unsigned int") ||
            TypeName.consume_front("uint"))
     return Type::getInt32Ty(Ctx);
   else if (TypeName.consume_front("long") ||
+           TypeName.consume_front("signed long") ||
            TypeName.consume_front("unsigned long") ||
            TypeName.consume_front("ulong"))
     return Type::getInt64Ty(Ctx);
-  else if (TypeName.consume_front("half"))
+  else if (TypeName.consume_front("half") ||
+           TypeName.consume_front("_Float16") ||
+           TypeName.consume_front("__fp16"))
     return Type::getHalfTy(Ctx);
   else if (TypeName.consume_front("float"))
     return Type::getFloatTy(Ctx);
@@ -519,8 +543,11 @@ bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
 }
 
 size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
-  size_t result = 0;
+  auto It = BlockToOrder.find(BB);
+  if (It != BlockToOrder.end())
+    return It->second.Rank;
 
+  size_t result = 0;
   for (BasicBlock *P : predecessors(BB)) {
     // Ignore back-edges.
     if (DT.dominates(BB, P))
@@ -552,15 +579,21 @@ size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
   ToVisit.push(BB);
   Queued.insert(BB);
 
+  size_t QueueIndex = 0;
   while (ToVisit.size() != 0) {
     BasicBlock *BB = ToVisit.front();
     ToVisit.pop();
 
     if (!CanBeVisited(BB)) {
       ToVisit.push(BB);
+      if (QueueIndex >= ToVisit.size())
+        llvm::report_fatal_error(
+            "No valid candidate in the queue. Is the graph reducible?");
+      QueueIndex++;
       continue;
     }
 
+    QueueIndex = 0;
     size_t Rank = GetNodeRank(BB);
     OrderInfo Info = {Rank, BlockToOrder.size()};
     BlockToOrder.emplace(BB, Info);
@@ -635,15 +668,12 @@ bool sortBlocks(Function &F) {
     return false;
 
   bool Modified = false;
-
   std::vector<BasicBlock *> Order;
   Order.reserve(F.size());
 
-  PartialOrderingVisitor Visitor(F);
-  Visitor.partialOrderVisit(*F.begin(), [&Order](BasicBlock *Block) {
-    Order.push_back(Block);
-    return true;
-  });
+  ReversePostOrderTraversal<Function *> RPOT(&F);
+  for (BasicBlock *BB : RPOT)
+    Order.push_back(BB);
 
   assert(&*F.begin() == Order[0]);
   BasicBlock *LastBlock = &*F.begin();
@@ -679,4 +709,77 @@ bool getVacantFunctionName(Module &M, std::string &Name) {
   return false;
 }
 
+// Assign SPIR-V type to the register. If the register has no valid assigned
+// class, set register LLT type and class according to the SPIR-V type.
+void setRegClassType(Register Reg, SPIRVType *SpvType, SPIRVGlobalRegistry *GR,
+                     MachineRegisterInfo *MRI, const MachineFunction &MF,
+                     bool Force) {
+  GR->assignSPIRVTypeToVReg(SpvType, Reg, MF);
+  if (!MRI->getRegClassOrNull(Reg) || Force) {
+    MRI->setRegClass(Reg, GR->getRegClass(SpvType));
+    MRI->setType(Reg, GR->getRegType(SpvType));
+  }
+}
+
+// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
+// no valid assigned class, set register LLT type and class according to the
+// SPIR-V type.
+void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
+                     MachineIRBuilder &MIRBuilder, bool Force) {
+  setRegClassType(Reg, GR->getOrCreateSPIRVType(Ty, MIRBuilder), GR,
+                  MIRBuilder.getMRI(), MIRBuilder.getMF(), Force);
+}
+
+// Create a virtual register and assign SPIR-V type to the register. Set
+// register LLT type and class according to the SPIR-V type.
+Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR,
+                               MachineRegisterInfo *MRI,
+                               const MachineFunction &MF) {
+  Register Reg = MRI->createVirtualRegister(GR->getRegClass(SpvType));
+  MRI->setType(Reg, GR->getRegType(SpvType));
+  GR->assignSPIRVTypeToVReg(SpvType, Reg, MF);
+  return Reg;
+}
+
+// Create a virtual register and assign SPIR-V type to the register. Set
+// register LLT type and class according to the SPIR-V type.
+Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR,
+                               MachineIRBuilder &MIRBuilder) {
+  return createVirtualRegister(SpvType, GR, MIRBuilder.getMRI(),
+                               MIRBuilder.getMF());
+}
+
+// Create a SPIR-V type, virtual register and assign SPIR-V type to the
+// register. Set register LLT type and class according to the SPIR-V type.
+Register createVirtualRegister(const Type *Ty, SPIRVGlobalRegistry *GR,
+                               MachineIRBuilder &MIRBuilder) {
+  return createVirtualRegister(GR->getOrCreateSPIRVType(Ty, MIRBuilder), GR,
+                               MIRBuilder);
+}
+
+// Return true if there is an opaque pointer type nested in the argument.
+bool isNestedPointer(const Type *Ty) {
+  if (Ty->isPtrOrPtrVectorTy())
+    return true;
+  if (const FunctionType *RefTy = dyn_cast<FunctionType>(Ty)) {
+    if (isNestedPointer(RefTy->getReturnType()))
+      return true;
+    for (const Type *ArgTy : RefTy->params())
+      if (isNestedPointer(ArgTy))
+        return true;
+    return false;
+  }
+  if (const ArrayType *RefTy = dyn_cast<ArrayType>(Ty))
+    return isNestedPointer(RefTy->getElementType());
+  return false;
+}
+
+bool isSpvIntrinsic(const Value *Arg) {
+  if (const auto *II = dyn_cast<IntrinsicInst>(Arg))
+    if (Function *F = II->getCalledFunction())
+      if (F->getName().starts_with("llvm.spv."))
+        return true;
+  return false;
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index da0e8769cac1b..cc77e0afa275a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -34,6 +34,7 @@ class Register;
 class StringRef;
 class SPIRVInstrInfo;
 class SPIRVSubtarget;
+class SPIRVGlobalRegistry;
 
 // This class implements a partial ordering visitor, which visits a cyclic graph
 // in natural topological-like ordering. Topological ordering is not defined for
@@ -41,6 +42,8 @@ class SPIRVSubtarget;
 // ignores back-edges. The cycle is visited from the entry in the same
 // topological-like ordering.
 //
+// Note: this visitor REQUIRES a reducible graph.
+//
 // This means once we visit a node, we know all the possible ancestors have been
 // visited.
 //
@@ -84,10 +87,11 @@ class PartialOrderingVisitor {
   // Visits |BB| with the current rank being |Rank|.
   size_t visit(BasicBlock *BB, size_t Rank);
 
-  size_t GetNodeRank(BasicBlock *BB) const;
   bool CanBeVisited(BasicBlock *BB) const;
 
 public:
+  size_t GetNodeRank(BasicBlock *BB) const;
+
   // Build the visitor to operate on the function F.
   PartialOrderingVisitor(Function &F);
 
@@ -125,6 +129,8 @@ void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB);
 // Add an OpName instruction for the given target register.
 void buildOpName(Register Target, const StringRef &Name,
                  MachineIRBuilder &MIRBuilder);
+void buildOpName(Register Target, const StringRef &Name, MachineInstr &I,
+                 const SPIRVInstrInfo &TII);
 
 // Add an OpDecorate instruction for the given Reg.
 void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
@@ -166,8 +172,12 @@ storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) {
     return 6;
   case SPIRV::StorageClass::Input:
     return 7;
+  case SPIRV::StorageClass::Output:
+    return 8;
   case SPIRV::StorageClass::CodeSectionINTEL:
     return 9;
+  case SPIRV::StorageClass::Private:
+    return 10;
   default:
     report_fatal_error("Unable to get address space id");
   }
@@ -195,6 +205,8 @@ uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI);
 
 // Check if MI is a SPIR-V specific intrinsic call.
 bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID);
+// Check if it's a SPIR-V specific intrinsic call.
+bool isSpvIntrinsic(const Value *Arg);
 
 // Get type of i-th operand of the metadata node.
 Type *getMDOperandAsType(const MDNode *N, unsigned I);
@@ -273,12 +285,19 @@ inline Type *getTypedPointerWrapper(Type *ElemTy, unsigned AS) {
                             {ElemTy}, {AS});
 }
 
-inline bool isTypedPointerWrapper(TargetExtType *ExtTy) {
+inline bool isTypedPointerWrapper(const TargetExtType *ExtTy) {
   return ExtTy->getName() == TYPED_PTR_TARGET_EXT_NAME &&
          ExtTy->getNumIntParameters() == 1 &&
          ExtTy->getNumTypeParameters() == 1;
 }
 
+// True if this is an instance of PointerType or TypedPointerType.
+inline bool isPointerTyOrWrapper(const Type *Ty) {
+  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty))
+    return isTypedPointerWrapper(ExtTy);
+  return isPointerTy(Ty);
+}
+
 inline Type *applyWrappers(Type *Ty) {
   if (auto *ExtTy = dyn_cast<TargetExtType>(Ty)) {
     if (isTypedPointerWrapper(ExtTy))
@@ -293,12 +312,14 @@ inline Type *applyWrappers(Type *Ty) {
   return Ty;
 }
 
-inline Type *getPointeeType(Type *Ty) {
-  if (auto PType = dyn_cast<TypedPointerType>(Ty))
-    return PType->getElementType();
-  else if (auto *ExtTy = dyn_cast<TargetExtType>(Ty))
-    if (isTypedPointerWrapper(ExtTy))
-      return applyWrappers(ExtTy->getTypeParameter(0));
+inline Type *getPointeeType(const Type *Ty) {
+  if (Ty) {
+    if (auto PType = dyn_cast<TypedPointerType>(Ty))
+      return PType->getElementType();
+    else if (auto *ExtTy = dyn_cast<TargetExtType>(Ty))
+      if (isTypedPointerWrapper(ExtTy))
+        return ExtTy->getTypeParameter(0);
+  }
   return nullptr;
 }
 
@@ -357,5 +378,23 @@ MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg);
 #define SPIRV_BACKEND_SERVICE_FUN_NAME "__spirv_backend_service_fun"
 bool getVacantFunctionName(Module &M, std::string &Name);
 
+void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
+                     MachineIRBuilder &MIRBuilder, bool Force = false);
+void setRegClassType(Register Reg, const MachineInstr *SpvType,
+                     SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
+                     const MachineFunction &MF, bool Force = false);
+Register createVirtualRegister(const MachineInstr *SpvType,
+                               SPIRVGlobalRegistry *GR,
+                               MachineRegisterInfo *MRI,
+                               const MachineFunction &MF);
+Register createVirtualRegister(const MachineInstr *SpvType,
+                               SPIRVGlobalRegistry *GR,
+                               MachineIRBuilder &MIRBuilder);
+Register createVirtualRegister(const Type *Ty, SPIRVGlobalRegistry *GR,
+                               MachineIRBuilder &MIRBuilder);
+
+// Return true if there is an opaque pointer type nested in the argument.
+bool isNestedPointer(const Type *Ty);
+
 } // namespace llvm
 #endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index e4aefc42d860f..599afed2199fb 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -1148,9 +1148,10 @@ ParseStatus SystemZAsmParser::parseAddress(OperandVector &Operands,
     if (HaveReg1) {
       if (parseAddressRegister(Reg1))
         return ParseStatus::Failure;
-      // If the are two registers, the first one is the index and the
-      // second is the base.
-      if (HaveReg2)
+      // If there are two registers, the first one is the index and the
+      // second is the base.  If there is only a single register, it is
+      // used as base with GAS and as index with HLASM.
+      if (HaveReg2 || isParsingHLASM())
         Index = Reg1.Num == 0 ? 0 : Regs[Reg1.Num];
       else
         Base = Reg1.Num == 0 ? 0 : Regs[Reg1.Num];
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index 59154431877a8..d307c73a87fc9 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -705,6 +705,11 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
   }
 
+  // EH_SjLj_Setup is a dummy terminator instruction of size 0.
+  // It is used to handle the clobber register for builtin setjmp.
+  case SystemZ::EH_SjLj_Setup:
+    return;
+
   default:
     Lower.lower(MI, LoweredMI);
     break;
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 8f505b7e198cf..becc3936eef89 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -751,6 +751,11 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
+  // We're not using SJLJ for exception handling, but they're implemented
+  // solely to support use of __builtin_setjmp / __builtin_longjmp.
+  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
+  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
+
   // We want to use MVC in preference to even a single load/store pair.
   MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
   MaxStoresPerMemcpyOptSize = 0;
@@ -940,6 +945,241 @@ bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
 }
 
+MachineBasicBlock *
+SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
+                                        MachineBasicBlock *MBB) const {
+  DebugLoc DL = MI.getDebugLoc();
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator I = ++MBB->getIterator();
+
+  Register DstReg = MI.getOperand(0).getReg();
+  const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
+  assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
+  (void)TRI;
+  Register mainDstReg = MRI.createVirtualRegister(RC);
+  Register restoreDstReg = MRI.createVirtualRegister(RC);
+
+  MVT PVT = getPointerTy(MF->getDataLayout());
+  assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
+  // For v = setjmp(buf), we generate.
+  // Algorithm:
+  //
+  //                 ---------
+  //                | thisMBB |
+  //                 ---------
+  //                     |
+  //         ------------------------
+  //        |                        |
+  //     ----------           ---------------
+  //    |  mainMBB |         | restoreMBB    |
+  //    |   v = 0  |         |  v = 1        |
+  //     ----------           ---------------
+  //        |                        |
+  //         -------------------------
+  //                     |
+  //           -----------------------------
+  //          |       sinkMBB               |
+  //          | phi(v_mainMBB,v_restoreMBB) |
+  //           -----------------------------
+  // thisMBB:
+  //  buf[FPOffset] = Frame Pointer if hasFP.
+  //  buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
+  //  buf[BCOffset] = Backchain value if building with -mbackchain.
+  //  buf[SPOffset] = Stack Pointer.
+  //  buf[LPOffset] = We never write this slot with  R13, gcc stores R13 always.
+  //  SjLjSetup restoreMBB
+  // mainMBB:
+  //  v_main = 0
+  // sinkMBB:
+  //  v = phi(v_main, v_restore)
+  // restoreMBB:
+  //  v_restore = 1
+
+  MachineBasicBlock *thisMBB = MBB;
+  MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
+
+  MF->insert(I, mainMBB);
+  MF->insert(I, sinkMBB);
+  MF->push_back(restoreMBB);
+  restoreMBB->setMachineBlockAddressTaken();
+
+  MachineInstrBuilder MIB;
+
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), MBB,
+                  std::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+  // thisMBB:
+  const int64_t FPOffset = 0;                         // Slot 1.
+  const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
+  const int64_t BCOffset = 2 * PVT.getStoreSize();    // Slot 3.
+  const int64_t SPOffset = 3 * PVT.getStoreSize();    // Slot 4.
+
+  // Buf address.
+  Register BufReg = MI.getOperand(1).getReg();
+
+  const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
+  unsigned LabelReg = MRI.createVirtualRegister(PtrRC);
+
+  // Prepare IP for longjmp.
+  BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::LARL), LabelReg)
+      .addMBB(restoreMBB);
+  // Store IP for return from jmp, slot 2, offset = 1.
+  BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::STG))
+      .addReg(LabelReg)
+      .addReg(BufReg)
+      .addImm(LabelOffset)
+      .addReg(0);
+
+  auto *SpecialRegs = Subtarget.getSpecialRegisters();
+  bool HasFP = Subtarget.getFrameLowering()->hasFP(*MF);
+  if (HasFP) {
+    BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::STG))
+        .addReg(SpecialRegs->getFramePointerRegister())
+        .addReg(BufReg)
+        .addImm(FPOffset)
+        .addReg(0);
+  }
+
+  // Store SP.
+  BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::STG))
+      .addReg(SpecialRegs->getStackPointerRegister())
+      .addReg(BufReg)
+      .addImm(SPOffset)
+      .addReg(0);
+
+  // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
+  bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
+  if (BackChain) {
+    Register BCReg = MRI.createVirtualRegister(PtrRC);
+    auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
+    MIB = BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::LG), BCReg)
+              .addReg(SpecialRegs->getStackPointerRegister())
+              .addImm(TFL->getBackchainOffset(*MF))
+              .addReg(0);
+
+    BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::STG))
+        .addReg(BCReg)
+        .addReg(BufReg)
+        .addImm(BCOffset)
+        .addReg(0);
+  }
+
+  // Setup.
+  MIB = BuildMI(*thisMBB, MI, DL, TII->get(SystemZ::EH_SjLj_Setup))
+            .addMBB(restoreMBB);
+
+  const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
+  MIB.addRegMask(RegInfo->getNoPreservedMask());
+
+  thisMBB->addSuccessor(mainMBB);
+  thisMBB->addSuccessor(restoreMBB);
+
+  // mainMBB:
+  BuildMI(mainMBB, DL, TII->get(SystemZ::LHI), mainDstReg).addImm(0);
+  mainMBB->addSuccessor(sinkMBB);
+
+  // sinkMBB:
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL, TII->get(SystemZ::PHI), DstReg)
+      .addReg(mainDstReg)
+      .addMBB(mainMBB)
+      .addReg(restoreDstReg)
+      .addMBB(restoreMBB);
+
+  // restoreMBB.
+  BuildMI(restoreMBB, DL, TII->get(SystemZ::LHI), restoreDstReg).addImm(1);
+  BuildMI(restoreMBB, DL, TII->get(SystemZ::J)).addMBB(sinkMBB);
+  restoreMBB->addSuccessor(sinkMBB);
+
+  MI.eraseFromParent();
+
+  return sinkMBB;
+}
+
+MachineBasicBlock *
+SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
+                                         MachineBasicBlock *MBB) const {
+
+  DebugLoc DL = MI.getDebugLoc();
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  MVT PVT = getPointerTy(MF->getDataLayout());
+  assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
+  Register BufReg = MI.getOperand(0).getReg();
+  const TargetRegisterClass *RC = MRI.getRegClass(BufReg);
+  auto *SpecialRegs = Subtarget.getSpecialRegisters();
+
+  Register Tmp = MRI.createVirtualRegister(RC);
+  Register BCReg = MRI.createVirtualRegister(RC);
+
+  MachineInstrBuilder MIB;
+
+  const int64_t FPOffset = 0;
+  const int64_t LabelOffset = 1 * PVT.getStoreSize();
+  const int64_t BCOffset = 2 * PVT.getStoreSize();
+  const int64_t SPOffset = 3 * PVT.getStoreSize();
+  const int64_t LPOffset = 4 * PVT.getStoreSize();
+
+  MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), Tmp)
+            .addReg(BufReg)
+            .addImm(LabelOffset)
+            .addReg(0);
+
+  MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG),
+                SpecialRegs->getFramePointerRegister())
+            .addReg(BufReg)
+            .addImm(FPOffset)
+            .addReg(0);
+
+  // We are restoring R13 even though we never stored in setjmp from llvm,
+  // as gcc always stores R13 in builtin_setjmp. We could have mixed code
+  // gcc setjmp and llvm longjmp.
+  MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), SystemZ::R13D)
+            .addReg(BufReg)
+            .addImm(LPOffset)
+            .addReg(0);
+
+  bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
+  if (BackChain) {
+    MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), BCReg)
+              .addReg(BufReg)
+              .addImm(BCOffset)
+              .addReg(0);
+  }
+
+  MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG),
+                SpecialRegs->getStackPointerRegister())
+            .addReg(BufReg)
+            .addImm(SPOffset)
+            .addReg(0);
+
+  if (BackChain) {
+    auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
+    BuildMI(*MBB, MI, DL, TII->get(SystemZ::STG))
+        .addReg(BCReg)
+        .addReg(SpecialRegs->getStackPointerRegister())
+        .addImm(TFL->getBackchainOffset(*MF))
+        .addReg(0);
+  }
+
+  MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::BR)).addReg(Tmp);
+
+  MI.eraseFromParent();
+  return MBB;
+}
+
 /// Returns true if stack probing through inline assembly is requested.
 bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
   // If the function specifically requests inline stack probes, emit them.
@@ -2147,8 +2387,8 @@ std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
   for (SDValue Op : Ops) {
     Entry.Node = Op;
     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
-    Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), IsSigned);
-    Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), IsSigned);
+    Entry.IsSExt = shouldSignExtendTypeInLibCall(Entry.Ty, IsSigned);
+    Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
   }
 
@@ -2157,7 +2397,7 @@ std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   TargetLowering::CallLoweringInfo CLI(DAG);
-  bool SignExtend = shouldSignExtendTypeInLibCall(RetVT, IsSigned);
+  bool SignExtend = shouldSignExtendTypeInLibCall(RetTy, IsSigned);
   CLI.setDebugLoc(DL)
       .setChain(Chain)
       .setCallee(CallConv, RetTy, Callee, std::move(Args))
@@ -6292,6 +6532,14 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerGET_ROUNDING(Op, DAG);
   case ISD::READCYCLECOUNTER:
     return lowerREADCYCLECOUNTER(Op, DAG);
+  case ISD::EH_SJLJ_SETJMP:
+  case ISD::EH_SJLJ_LONGJMP:
+    // These operations are legal on our platform, but we cannot actually
+    // set the operation action to Legal as common code would treat this
+    // as equivalent to Expand. Instead, we keep the operation action to
+    // Custom and just leave them unchanged here.
+    return Op;
+
   default:
     llvm_unreachable("Unexpected node to lower");
   }
@@ -9733,6 +9981,10 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
 
   case SystemZ::PROBED_ALLOCA:
     return emitProbedAlloca(MI, MBB);
+  case SystemZ::EH_SjLj_SetJmp:
+    return emitEHSjLjSetJmp(MI, MBB);
+  case SystemZ::EH_SjLj_LongJmp:
+    return emitEHSjLjLongJmp(MI, MBB);
 
   case TargetOpcode::STACKMAP:
   case TargetOpcode::PATCHPOINT:
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 3c06c1fdf2b1b..0a899e861c73b 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -476,6 +476,12 @@ class SystemZTargetLowering : public TargetLowering {
     // LD, and having the full constant in memory enables reg/mem opcodes.
     return VT != MVT::f64;
   }
+  MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
+                                      MachineBasicBlock *MBB) const;
+
+  MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
+                                       MachineBasicBlock *MBB) const;
+
   bool hasInlineStackProbe(const MachineFunction &MF) const override;
   AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override;
   AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index 6e136b10aed42..ae8f669e9bab4 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -2328,6 +2328,12 @@ class AsmCondBranchRI<string mnemonic, bits<12> opcode>
   : InstRIc<opcode, (outs), (ins imm32zx4:$M1, brtarget16:$RI2),
             mnemonic#"\t$M1, $RI2", []>;
 
+class NeverCondBranchRI<string mnemonic, bits<12> opcode>
+  : InstRIc<opcode, (outs), (ins brtarget16:$RI2),
+            mnemonic#"\t$RI2", []> {
+  let M1 = 0;
+}
+
 class FixedCondBranchRI<CondVariant V, string mnemonic, bits<12> opcode,
                         SDPatternOperator operator = null_frag>
   : InstRIc<opcode, (outs), (ins brtarget16:$RI2),
@@ -2347,6 +2353,12 @@ class AsmCondBranchRIL<string mnemonic, bits<12> opcode>
   : InstRILc<opcode, (outs), (ins imm32zx4:$M1, brtarget32:$RI2),
              mnemonic#"\t$M1, $RI2", []>;
 
+class NeverCondBranchRIL<string mnemonic, bits<12> opcode>
+  : InstRILc<opcode, (outs), (ins brtarget32:$RI2),
+             mnemonic#"\t$RI2", []> {
+  let M1 = 0;
+}
+
 class FixedCondBranchRIL<CondVariant V, string mnemonic, bits<12> opcode>
   : InstRILc<opcode, (outs), (ins brtarget32:$RI2),
              !subst("#", V.suffix, mnemonic)#"\t$RI2", []> {
@@ -2365,10 +2377,16 @@ class AsmCondBranchRR<string mnemonic, bits<8> opcode>
   : InstRR<opcode, (outs), (ins imm32zx4:$R1, GR64:$R2),
            mnemonic#"\t$R1, $R2", []>;
 
-class NeverCondBranchRR<string mnemonic, bits<8> opcode>
-  : InstRR<opcode, (outs), (ins GR64:$R2),
-           mnemonic#"\t$R2", []> {
-  let R1 = 0;
+multiclass NeverCondBranchRR<string mnemonic, bits<8> opcode> {
+  // For the no-op (always false) branch, the target is optional.
+  def "" : InstRR<opcode, (outs), (ins GR64:$R2),
+                  mnemonic#"\t$R2", []> {
+             let R1 = 0;
+           }
+  def Opt : InstRR<opcode, (outs), (ins), mnemonic, []> {
+              let R1 = 0;
+              let R2 = 0;
+            }
 }
 
 class FixedCondBranchRR<CondVariant V, string mnemonic, bits<8> opcode,
@@ -2392,11 +2410,19 @@ class AsmCondBranchRX<string mnemonic, bits<8> opcode>
             (ins imm32zx4:$M1, (bdxaddr12only $B2, $D2, $X2):$XBD2),
             mnemonic#"\t$M1, $XBD2", []>;
 
-class NeverCondBranchRX<string mnemonic, bits<8> opcode> 
-  : InstRXb<opcode, (outs),
-            (ins (bdxaddr12only $B2, $D2, $X2):$XBD2),
-            mnemonic#"\t$XBD2", []> {
-  let M1 = 0;
+multiclass NeverCondBranchRX<string mnemonic, bits<8> opcode> {
+  // For the no-op (always false) branch, the target is optional.
+  def "" : InstRXb<opcode, (outs),
+                  (ins (bdxaddr12only $B2, $D2, $X2):$XBD2),
+                  mnemonic#"\t$XBD2", []> {
+             let M1 = 0;
+           }
+  def Opt : InstRXb<opcode, (outs), (ins), mnemonic, []> {
+              let M1 = 0;
+              let B2 = 0;
+              let D2 = 0;
+              let X2 = 0;
+            }
 }
 
 class FixedCondBranchRX<CondVariant V, string mnemonic, bits<8> opcode>
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index f3baf896658de..d6cddeb8b6c30 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -109,20 +109,11 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
 }
 
 // NOPs.  These are again variants of the conditional branches, with the
-// condition mask set to "never".  NOP_bare can't be an InstAlias since it
-// would need R0D hard coded which is not part of ADDR64BitRegClass.
-def NOP : NeverCondBranchRX<"nop", 0x47>;
-let isAsmParserOnly = 1, hasNoSchedulingInfo = 1, M1 = 0, X2 = 0, B2 = 0, D2 = 0 in
-  def NOP_bare  : InstRXb<0x47,(outs), (ins), "nop", []>;
-def NOPR : NeverCondBranchRR<"nopr", 0x07>;
-def NOPR_bare : InstAlias<"nopr", (NOPR R0D), 0>;
-
-// An alias of BRC 0, label
-def JNOP : InstAlias<"jnop\t$RI2", (BRCAsm 0, brtarget16:$RI2), 0>;
-
-// An alias of BRCL 0, label
-// jgnop on gnu ; jlnop on hlasm
-def JGNOP : InstAlias<"{jgnop|jlnop}\t$RI2", (BRCLAsm 0, brtarget32:$RI2), 0>;
+// condition mask set to "never".
+defm NOP  : NeverCondBranchRX<"nop", 0x47>;
+defm NOPR : NeverCondBranchRR<"nopr", 0x07>;
+def JNOP  : NeverCondBranchRI<"jnop", 0xA74>;
+def JGNOP : NeverCondBranchRIL<"j{g|l}nop", 0xC04>;
 
 // Fused compare-and-branch instructions.
 //
@@ -1871,6 +1862,23 @@ let mayLoad = 1, mayStore = 1, Defs = [CC] in {
   }
 }
 
+//--------------------------------------------------------------------------
+// Setjmp/Longjmp.
+//--------------------------------------------------------------------------
+let isBarrier = 1,  hasNoSchedulingInfo = 1 in {
+  let hasSideEffects = 1, usesCustomInserter = 1 in {
+    def EH_SjLj_SetJmp : Pseudo<(outs GR32:$dst), (ins ADDR64:$R2), 
+                        [(set GR32:$dst, (z_eh_sjlj_setjmp ADDR64:$R2))]>;
+    let isTerminator = 1 in {
+      def EH_SjLj_LongJmp : Pseudo<(outs), (ins ADDR64:$R2), 
+                           [(z_eh_sjlj_longjmp  ADDR64:$R2)]>;
+    }
+  }
+  let isTerminator = 1, isCodeGenOnly = 1, Size = 0 in {
+    def EH_SjLj_Setup : Pseudo<(outs), (ins brtarget32:$dst), []>;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Message-security assist
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
index 632218cc61eef..36d76235398ed 100644
--- a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
@@ -220,7 +220,10 @@ static unsigned getInstSizeInBytes(const MachineInstr &MI,
           MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::MEMBARRIER ||
           // These have a size that may be zero:
           MI.isInlineAsm() || MI.getOpcode() == SystemZ::STACKMAP ||
-          MI.getOpcode() == SystemZ::PATCHPOINT) &&
+          MI.getOpcode() == SystemZ::PATCHPOINT ||
+          // EH_SjLj_Setup is a dummy terminator instruction of size 0,
+          // It is used to handle the clobber register for builtin setjmp.
+          MI.getOpcode() == SystemZ::EH_SjLj_Setup) &&
          "Missing size value for instruction.");
   return Size;
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZOperands.td b/llvm/lib/Target/SystemZ/SystemZOperands.td
index 64345ca3a1394..e7b45a40a3cc0 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperands.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperands.td
@@ -220,7 +220,7 @@ def NEGLF32 : SDNodeXForm<imm, [{
 
 // Truncate an immediate to a 8-bit signed quantity.
 def SIMM8 : SDNodeXForm<imm, [{
-  return CurDAG->getSignedTargetConstant(int8_t(N->getZExtValue()), SDLoc(N),
+  return CurDAG->getSignedTargetConstant(int8_t(N->getSExtValue()), SDLoc(N),
                                          MVT::i64);
 }]>;
 
@@ -244,13 +244,13 @@ def UIMM12 : SDNodeXForm<imm, [{
 
 // Truncate an immediate to a 16-bit signed quantity.
 def SIMM16 : SDNodeXForm<imm, [{
-  return CurDAG->getSignedTargetConstant(int16_t(N->getZExtValue()), SDLoc(N),
+  return CurDAG->getSignedTargetConstant(int16_t(N->getSExtValue()), SDLoc(N),
                                          MVT::i64);
 }]>;
 
 // Negate and then truncate an immediate to a 16-bit signed quantity.
 def NEGSIMM16 : SDNodeXForm<imm, [{
-  return CurDAG->getSignedTargetConstant(int16_t(-N->getZExtValue()), SDLoc(N),
+  return CurDAG->getSignedTargetConstant(int16_t(-N->getSExtValue()), SDLoc(N),
                                          MVT::i64);
 }]>;
 
@@ -262,13 +262,13 @@ def UIMM16 : SDNodeXForm<imm, [{
 
 // Truncate an immediate to a 32-bit signed quantity.
 def SIMM32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(int32_t(N->getZExtValue()), SDLoc(N),
-                                   MVT::i64);
+  return CurDAG->getSignedTargetConstant(int32_t(N->getSExtValue()), SDLoc(N),
+                                         MVT::i64);
 }]>;
 
 // Negate and then truncate an immediate to a 32-bit unsigned quantity.
 def NEGSIMM32 : SDNodeXForm<imm, [{
-  return CurDAG->getSignedTargetConstant(int32_t(-N->getZExtValue()), SDLoc(N),
+  return CurDAG->getSignedTargetConstant(int32_t(-N->getSExtValue()), SDLoc(N),
                                          MVT::i64);
 }]>;
 
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index 6cb89ccff85e6..90fb4e5f370da 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -238,6 +238,12 @@ def SDT_ZTest               : SDTypeProfile<1, 2,
                                             [SDTCisVT<0, i32>,
                                              SDTCisVT<2, i64>]>;
 
+def SDT_ZSetJmp             : SDTypeProfile<1, 1,
+                                            [SDTCisInt<0>,
+                                             SDTCisPtrTy<1>]>;
+def SDT_ZLongJmp            : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
+
+
 //===----------------------------------------------------------------------===//
 // Node definitions
 //===----------------------------------------------------------------------===//
@@ -314,6 +320,12 @@ def z_stckf            : SDNode<"SystemZISD::STCKF", SDT_ZStoreInherent,
 
 def z_tdc               : SDNode<"SystemZISD::TDC", SDT_ZTest>;
 
+def z_eh_sjlj_setjmp    : SDNode<"ISD::EH_SJLJ_SETJMP", SDT_ZSetJmp,
+                                 [SDNPHasChain, SDNPSideEffect]>;
+def z_eh_sjlj_longjmp   : SDNode<"ISD::EH_SJLJ_LONGJMP", SDT_ZLongJmp,
+                                 [SDNPHasChain, SDNPSideEffect]>;
+
+
 // Defined because the index is an i32 rather than a pointer.
 def z_vector_insert     : SDNode<"ISD::INSERT_VECTOR_ELT",
                                  SDT_ZInsertVectorElt>;
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index d246d3f3c5bd1..0df2a15753cf3 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -254,6 +254,10 @@ SystemZRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
   return Regs->getCallPreservedMask(MF, CC);
 }
 
+const uint32_t *SystemZRegisterInfo::getNoPreservedMask() const {
+  return CSR_SystemZ_NoRegs_RegMask;
+}
+
 BitVector
 SystemZRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
index cbc02c73f1ac7..4f497f8d23d29 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
@@ -161,6 +161,7 @@ struct SystemZRegisterInfo : public SystemZGenRegisterInfo {
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
   const uint32_t *getCallPreservedMask(const MachineFunction &MF,
                                        CallingConv::ID CC) const override;
+  const uint32_t *getNoPreservedMask() const override;
   BitVector getReservedRegs(const MachineFunction &MF) const override;
   bool eliminateFrameIndex(MachineBasicBlock::iterator MI,
                            int SPAdj, unsigned FIOperandNum,
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
index d0fec02777875..094b481b81f83 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
@@ -1557,7 +1557,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?$")>;
-
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, VBU, NormalGr], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
index a6d89ce9443c5..b9376d422ded2 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
@@ -1647,7 +1647,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?$")>;
-
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, VBU, NormalGr], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
index 455354e283ad8..5e28bf935a24b 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
@@ -1694,6 +1694,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, VBU, NormalGr], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
index 92abf0ba4022c..2c01691707cc3 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
@@ -1727,6 +1727,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, VBU, NormalGr], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
index 99d0d674bbbb2..f41a7057bb1f6 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
@@ -1239,6 +1239,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, LSU, EndGroup], (instregex "NOP(R)?$")>;
+def : InstRW<[WLat1, LSU, EndGroup], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, LSU, EndGroup], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td b/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
index 5b334da2bac34..8f0a10d2863a3 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
@@ -1284,6 +1284,7 @@ def : InstRW<[WLat30, MCD], (instregex "SAL$")>;
 // NOPs
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, LSU, NormalGr], (instregex "NOP(R)?$")>;
+def : InstRW<[WLat1, LSU, NormalGr], (instregex "NOP(R)?(Opt)?$")>;
+def : InstRW<[WLat1, VBU, NormalGr], (instregex "J(G)?NOP$")>;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index a586eedd58b66..772efcdf8f9fc 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/MathExtras.h"
 
 using namespace llvm;
@@ -468,6 +469,42 @@ bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
   return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
 }
 
+static bool isFreeEltLoad(Value *Op) {
+  if (isa<LoadInst>(Op) && Op->hasOneUse()) {
+    const Instruction *UserI = cast<Instruction>(*Op->user_begin());
+    return !isa<StoreInst>(UserI); // Prefer MVC
+  }
+  return false;
+}
+
+InstructionCost SystemZTTIImpl::getScalarizationOverhead(
+    VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+  unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
+  InstructionCost Cost = 0;
+
+  if (Insert && Ty->isIntOrIntVectorTy(64)) {
+    // VLVGP will insert two GPRs with one instruction, while VLE will load
+    // an element directly with no extra cost
+    assert((VL.empty() || VL.size() == NumElts) &&
+           "Type does not match the number of values.");
+    InstructionCost CurrVectorCost = 0;
+    for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
+      if (DemandedElts[Idx] && !(VL.size() && isFreeEltLoad(VL[Idx])))
+        ++CurrVectorCost;
+      if (Idx % 2 == 1) {
+        Cost += std::min(InstructionCost(1), CurrVectorCost);
+        CurrVectorCost = 0;
+      }
+    }
+    Insert = false;
+  }
+
+  Cost += BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
+                                          CostKind, VL);
+  return Cost;
+}
+
 // Return the bit size for the scalar type or vector element
 // type. getScalarSizeInBits() returns 0 for a pointer type.
 static unsigned getScalarSizeInBits(Type *Ty) {
@@ -609,7 +646,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
     if (DivRemConst) {
       SmallVector<Type *> Tys(Args.size(), Ty);
       return VF * DivMulSeqCost +
-             getScalarizationOverhead(VTy, Args, Tys, CostKind);
+             BaseT::getScalarizationOverhead(VTy, Args, Tys, CostKind);
     }
     if ((SignedDivRem || UnsignedDivRem) && VF > 4)
       // Temporary hack: disable high vectorization factors with integer
@@ -636,7 +673,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
         SmallVector<Type *> Tys(Args.size(), Ty);
         InstructionCost Cost =
             (VF * ScalarCost) +
-            getScalarizationOverhead(VTy, Args, Tys, CostKind);
+            BaseT::getScalarizationOverhead(VTy, Args, Tys, CostKind);
         // FIXME: VF 2 for these FP operations are currently just as
         // expensive as for VF 4.
         if (VF == 2)
@@ -654,8 +691,9 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
     // There is no native support for FRem.
     if (Opcode == Instruction::FRem) {
       SmallVector<Type *> Tys(Args.size(), Ty);
-      InstructionCost Cost = (VF * LIBCALL_COST) +
-                             getScalarizationOverhead(VTy, Args, Tys, CostKind);
+      InstructionCost Cost =
+          (VF * LIBCALL_COST) +
+          BaseT::getScalarizationOverhead(VTy, Args, Tys, CostKind);
       // FIXME: VF 2 for float is currently just as expensive as for VF 4.
       if (VF == 2 && ScalarBits == 32)
         Cost *= 2;
@@ -975,10 +1013,10 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
           (Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI))
         NeedsExtracts = false;
 
-      TotCost += getScalarizationOverhead(SrcVecTy, /*Insert*/ false,
-                                          NeedsExtracts, CostKind);
-      TotCost += getScalarizationOverhead(DstVecTy, NeedsInserts,
-                                          /*Extract*/ false, CostKind);
+      TotCost += BaseT::getScalarizationOverhead(SrcVecTy, /*Insert*/ false,
+                                                 NeedsExtracts, CostKind);
+      TotCost += BaseT::getScalarizationOverhead(DstVecTy, NeedsInserts,
+                                                 /*Extract*/ false, CostKind);
 
       // FIXME: VF 2 for float<->i32 is currently just as expensive as for VF 4.
       if (VF == 2 && SrcScalarBits == 32 && DstScalarBits == 32)
@@ -990,8 +1028,8 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     if (Opcode == Instruction::FPTrunc) {
       if (SrcScalarBits == 128)  // fp128 -> double/float + inserts of elements.
         return VF /*ldxbr/lexbr*/ +
-               getScalarizationOverhead(DstVecTy, /*Insert*/ true,
-                                        /*Extract*/ false, CostKind);
+               BaseT::getScalarizationOverhead(DstVecTy, /*Insert*/ true,
+                                               /*Extract*/ false, CostKind);
       else // double -> float
         return VF / 2 /*vledb*/ + std::max(1U, VF / 4 /*vperm*/);
     }
@@ -1004,8 +1042,8 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
         return VF * 2;
       }
       // -> fp128.  VF * lxdb/lxeb + extraction of elements.
-      return VF + getScalarizationOverhead(SrcVecTy, /*Insert*/ false,
-                                           /*Extract*/ true, CostKind);
+      return VF + BaseT::getScalarizationOverhead(SrcVecTy, /*Insert*/ false,
+                                                  /*Extract*/ true, CostKind);
     }
   }
 
@@ -1114,10 +1152,17 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                    TTI::TargetCostKind CostKind,
                                                    unsigned Index, Value *Op0,
                                                    Value *Op1) {
-  // vlvgp will insert two grs into a vector register, so only count half the
-  // number of instructions.
-  if (Opcode == Instruction::InsertElement && Val->isIntOrIntVectorTy(64))
-    return ((Index % 2 == 0) ? 1 : 0);
+  if (Opcode == Instruction::InsertElement) {
+    // Vector Element Load.
+    if (Op1 != nullptr && isFreeEltLoad(Op1))
+      return 0;
+
+    // vlvgp will insert two grs into a vector register, so count half the
+    // number of instructions as an estimate when we don't have the full
+    // picture (as in getScalarizationOverhead()).
+    if (Val->isIntOrIntVectorTy(64))
+      return ((Index % 2 == 0) ? 1 : 0);
+  }
 
   if (Opcode == Instruction::ExtractElement) {
     int Cost = ((getScalarSizeInBits(Val) == 1) ? 2 /*+test-under-mask*/ : 1);
@@ -1352,30 +1397,86 @@ InstructionCost SystemZTTIImpl::getInterleavedMemoryOpCost(
   return NumVectorMemOps + NumPermutes;
 }
 
+InstructionCost getIntAddReductionCost(unsigned NumVec, unsigned ScalarBits) {
+  InstructionCost Cost = 0;
+  // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+  Cost += NumVec - 1;
+  // For integer adds, VSUM creates shorter reductions on the final vector.
+  Cost += (ScalarBits < 32) ? 3 : 2;
+  return Cost;
+}
+
+InstructionCost getFastReductionCost(unsigned NumVec, unsigned NumElems,
+                                     unsigned ScalarBits) {
+  unsigned NumEltsPerVecReg = (SystemZ::VectorBits / ScalarBits);
+  InstructionCost Cost = 0;
+  // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+  Cost += NumVec - 1;
+  // For each shuffle / arithmetic layer, we need 2 instructions, and we need
+  // log2(Elements in Last Vector) layers.
+  Cost += 2 * Log2_32_Ceil(std::min(NumElems, NumEltsPerVecReg));
+  return Cost;
+}
+
+inline bool customCostReductions(unsigned Opcode) {
+  return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
+         Opcode == Instruction::Add || Opcode == Instruction::Mul;
+}
+
+InstructionCost
+SystemZTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                                           std::optional<FastMathFlags> FMF,
+                                           TTI::TargetCostKind CostKind) {
+  unsigned ScalarBits = Ty->getScalarSizeInBits();
+  // The following is only for subtargets with vector math, non-ordered
+  // reductions, and reasonable scalar sizes for int and fp add/mul.
+  if (customCostReductions(Opcode) && ST->hasVector() &&
+      !TTI::requiresOrderedReduction(FMF) &&
+      ScalarBits <= SystemZ::VectorBits) {
+    unsigned NumVectors = getNumVectorRegs(Ty);
+    unsigned NumElems = ((FixedVectorType *)Ty)->getNumElements();
+    // Integer Add is using custom code gen, that needs to be accounted for.
+    if (Opcode == Instruction::Add)
+      return getIntAddReductionCost(NumVectors, ScalarBits);
+    // The base cost is the same across all other arithmetic instructions
+    InstructionCost Cost =
+        getFastReductionCost(NumVectors, NumElems, ScalarBits);
+    // But we need to account for the final op involving the scalar operand.
+    if ((Opcode == Instruction::FAdd) || (Opcode == Instruction::FMul))
+      Cost += 1;
+    return Cost;
+  }
+  // otherwise, fall back to the standard implementation
+  return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
+}
+
+InstructionCost
+SystemZTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
+                                       FastMathFlags FMF,
+                                       TTI::TargetCostKind CostKind) {
+  // Return custom costs only on subtargets with vector enhancements.
+  if (ST->hasVectorEnhancements1()) {
+    unsigned NumVectors = getNumVectorRegs(Ty);
+    unsigned NumElems = ((FixedVectorType *)Ty)->getNumElements();
+    unsigned ScalarBits = Ty->getScalarSizeInBits();
+    InstructionCost Cost = 0;
+    // Binary Tree of N/2 + N/4 + ... operations yields N - 1 operations total.
+    Cost += NumVectors - 1;
+    // For the final vector, we need shuffle + min/max operations, and
+    // we need #Elements - 1 of them.
+    Cost += 2 * (std::min(NumElems, SystemZ::VectorBits / ScalarBits) - 1);
+    return Cost;
+  }
+  // For other targets, fall back to the standard implementation
+  return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
+}
+
 static int
 getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
                             const SmallVectorImpl<Type *> &ParamTys) {
   if (RetTy->isVectorTy() && ID == Intrinsic::bswap)
     return getNumVectorRegs(RetTy); // VPERM
 
-  if (ID == Intrinsic::vector_reduce_add) {
-    // Retrieve number and size of elements for the vector op.
-    auto *VTy = cast<FixedVectorType>(ParamTys.front());
-    unsigned ScalarSize = VTy->getScalarSizeInBits();
-    // For scalar sizes >128 bits, we fall back to the generic cost estimate.
-    if (ScalarSize > SystemZ::VectorBits)
-      return -1;
-    // This many vector regs are needed to represent the input elements (V).
-    unsigned VectorRegsNeeded = getNumVectorRegs(VTy);
-    // This many instructions are needed for the final sum of vector elems (S).
-    unsigned LastVectorHandling = (ScalarSize < 32) ? 3 : 2;
-    // We use vector adds to create a sum vector, which takes
-    // V/2 + V/4 + ... = V - 1 operations.
-    // Then, we need S operations to sum up the elements of that sum vector,
-    // for a total of V + S - 1 operations.
-    int Cost = VectorRegsNeeded + LastVectorHandling - 1;
-    return Cost;
-  }
   return -1;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 8cc71a6c528f8..512fcc854d532 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -81,6 +81,11 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   bool hasDivRemOp(Type *DataType, bool IsSigned);
   bool prefersVectorizedAddressing() { return false; }
   bool LSRWithInstrQueries() { return true; }
+  InstructionCost getScalarizationOverhead(VectorType *Ty,
+                                           const APInt &DemandedElts,
+                                           bool Insert, bool Extract,
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {});
   bool supportsEfficientVectorElementLoadStore() { return true; }
   bool enableInterleavedAccessVectorization() { return true; }
 
@@ -125,6 +130,13 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
+  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                                             std::optional<FastMathFlags> FMF,
+                                             TTI::TargetCostKind CostKind);
+  InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
+                                         FastMathFlags FMF,
+                                         TTI::TargetCostKind CostKind);
+
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
 
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 10451600050ca..f693ef3dbf962 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -276,7 +276,18 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       : MCTargetAsmParser(Options, STI, MII), Parser(Parser),
         Lexer(Parser.getLexer()), Is64(STI.getTargetTriple().isArch64Bit()),
         TC(Parser, MII, Is64), SkipTypeCheck(Options.MCNoTypeCheck) {
-    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
+    FeatureBitset FBS = ComputeAvailableFeatures(STI.getFeatureBits());
+
+    // bulk-memory implies bulk-memory-opt
+    if (FBS.test(WebAssembly::FeatureBulkMemory)) {
+      FBS.set(WebAssembly::FeatureBulkMemoryOpt);
+    }
+    // reference-types implies call-indirect-overlong
+    if (FBS.test(WebAssembly::FeatureReferenceTypes)) {
+      FBS.set(WebAssembly::FeatureCallIndirectOverlong);
+    }
+
+    setAvailableFeatures(FBS);
     // Don't type check if this is inline asm, since that is a naked sequence of
     // instructions without a function/locals decl.
     auto &SM = Parser.getSourceManager();
@@ -291,7 +302,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
 
     DefaultFunctionTable = getOrCreateFunctionTableSymbol(
         getContext(), "__indirect_function_table", Is64);
-    if (!STI->checkFeatures("+reference-types"))
+    if (!STI->checkFeatures("+call-indirect-overlong") &&
+        !STI->checkFeatures("+reference-types"))
       DefaultFunctionTable->setOmitFromLinkingSection();
   }
 
@@ -531,11 +543,13 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
   }
 
   bool parseFunctionTableOperand(std::unique_ptr<WebAssemblyOperand> *Op) {
-    if (STI->checkFeatures("+reference-types")) {
-      // If the reference-types feature is enabled, there is an explicit table
-      // operand.  To allow the same assembly to be compiled with or without
-      // reference types, we allow the operand to be omitted, in which case we
-      // default to __indirect_function_table.
+    if (STI->checkFeatures("+call-indirect-overlong") ||
+        STI->checkFeatures("+reference-types")) {
+      // If the call-indirect-overlong feature is enabled, or implied by the
+      // reference-types feature, there is an explicit table operand.  To allow
+      // the same assembly to be compiled with or without
+      // call-indirect-overlong, we allow the operand to be omitted, in which
+      // case we default to __indirect_function_table.
       auto &Tok = Lexer.getTok();
       if (Tok.is(AsmToken::Identifier)) {
         auto *Sym =
diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.td b/llvm/lib/Target/WebAssembly/WebAssembly.td
index 88628f2a79354..13603f8181198 100644
--- a/llvm/lib/Target/WebAssembly/WebAssembly.td
+++ b/llvm/lib/Target/WebAssembly/WebAssembly.td
@@ -29,6 +29,14 @@ def FeatureBulkMemory :
       SubtargetFeature<"bulk-memory", "HasBulkMemory", "true",
                        "Enable bulk memory operations">;
 
+def FeatureBulkMemoryOpt :
+      SubtargetFeature<"bulk-memory-opt", "HasBulkMemoryOpt", "true",
+                       "Enable bulk memory optimization operations">;
+
+def FeatureCallIndirectOverlong :
+      SubtargetFeature<"call-indirect-overlong", "HasCallIndirectOverlong", "true",
+                       "Enable overlong encoding for call_indirect immediates">;
+
 def FeatureExceptionHandling :
       SubtargetFeature<"exception-handling", "HasExceptionHandling", "true",
                        "Enable Wasm exception handling">;
@@ -114,15 +122,23 @@ def : ProcessorModel<"mvp", NoSchedModel, []>;
 // consideration given to available support in relevant engines and tools, and
 // the importance of the features.
 def : ProcessorModel<"generic", NoSchedModel,
-                      [FeatureBulkMemory, FeatureMultivalue,
+                      [FeatureBulkMemory, FeatureBulkMemoryOpt,
+                       FeatureCallIndirectOverlong, FeatureMultivalue,
                        FeatureMutableGlobals, FeatureNontrappingFPToInt,
                        FeatureReferenceTypes, FeatureSignExt]>;
 
+// Lime1: <https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1>
+def : ProcessorModel<"lime1", NoSchedModel,
+                      [FeatureBulkMemoryOpt, FeatureCallIndirectOverlong,
+                       FeatureExtendedConst, FeatureMultivalue,
+                       FeatureMutableGlobals, FeatureNontrappingFPToInt,
+                       FeatureSignExt]>;
+
 // Latest and greatest experimental version of WebAssembly. Bugs included!
 def : ProcessorModel<"bleeding-edge", NoSchedModel,
-                      [FeatureAtomics, FeatureBulkMemory,
-                       FeatureExceptionHandling, FeatureExtendedConst,
-                       FeatureFP16, FeatureMultiMemory,
+                      [FeatureAtomics, FeatureBulkMemory, FeatureBulkMemoryOpt,
+                       FeatureCallIndirectOverlong, FeatureExceptionHandling,
+                       FeatureExtendedConst, FeatureFP16, FeatureMultiMemory,
                        FeatureMultivalue, FeatureMutableGlobals,
                        FeatureNontrappingFPToInt, FeatureRelaxedSIMD,
                        FeatureReferenceTypes, FeatureSIMD128, FeatureSignExt,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index 558aaa38096f7..210a35e1462ac 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -895,7 +895,7 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
     // The table into which this call_indirect indexes.
     MCSymbolWasm *Table = WebAssembly::getOrCreateFunctionTableSymbol(
         MF->getContext(), Subtarget);
-    if (Subtarget->hasReferenceTypes()) {
+    if (Subtarget->hasCallIndirectOverlong()) {
       MIB.addSym(Table);
     } else {
       // Otherwise for the MVP there is at most one table whose number is 0, but
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
index 7c3e8d18ad276..4fe1df0ab3e42 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
@@ -86,9 +86,9 @@ static void findUses(Value *V, Function &F,
 // Create a wrapper function with type Ty that calls F (which may have a
 // different type). Attempt to support common bitcasted function idioms:
 //  - Call with more arguments than needed: arguments are dropped
-//  - Call with fewer arguments than needed: arguments are filled in with undef
+//  - Call with fewer arguments than needed: arguments are filled in with poison
 //  - Return value is not needed: drop it
-//  - Return value needed but not present: supply an undef
+//  - Return value needed but not present: supply a poison value
 //
 // If the all the argument types of trivially castable to one another (i.e.
 // I32 vs pointer type) then we don't create a wrapper at all (return nullptr
@@ -161,7 +161,7 @@ static Function *createWrapper(Function *F, FunctionType *Ty) {
 
   if (WrapperNeeded && !TypeMismatch) {
     for (; PI != PE; ++PI)
-      Args.push_back(UndefValue::get(*PI));
+      Args.push_back(PoisonValue::get(*PI));
     if (F->isVarArg())
       for (; AI != AE; ++AI)
         Args.push_back(&*AI);
@@ -175,7 +175,7 @@ static Function *createWrapper(Function *F, FunctionType *Ty) {
       ReturnInst::Create(M->getContext(), BB);
     } else if (ExpectedRtnType->isVoidTy()) {
       LLVM_DEBUG(dbgs() << "Creating dummy return: " << *RtnType << "\n");
-      ReturnInst::Create(M->getContext(), UndefValue::get(RtnType), BB);
+      ReturnInst::Create(M->getContext(), PoisonValue::get(RtnType), BB);
     } else if (RtnType == ExpectedRtnType) {
       ReturnInst::Create(M->getContext(), Call, BB);
     } else if (CastInst::isBitOrNoopPointerCastable(ExpectedRtnType, RtnType,
@@ -255,8 +255,8 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
       if (shouldFixMainFunction(F.getFunctionType(), MainTy)) {
         LLVM_DEBUG(dbgs() << "Found `main` function with incorrect type: "
                           << *F.getFunctionType() << "\n");
-        Value *Args[] = {UndefValue::get(MainArgTys[0]),
-                         UndefValue::get(MainArgTys[1])};
+        Value *Args[] = {PoisonValue::get(MainArgTys[0]),
+                         PoisonValue::get(MainArgTys[1])};
         CallMain = CallInst::Create(MainTy, Main, Args, "call_main");
         Uses.push_back(std::make_pair(CallMain, &F));
       }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 94b49387b58f9..7712570869ff6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -429,7 +429,7 @@ bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
 
   // Assume target opcodes can't be scalarized.
   // TODO - do we have any exceptions?
-  if (Opc >= ISD::BUILTIN_OP_END)
+  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
     return false;
 
   // If the vector op is not supported, try to convert to scalar.
@@ -768,7 +768,7 @@ LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
                                     MF.getContext(), Subtarget)
                               : WebAssembly::getOrCreateFunctionTableSymbol(
                                     MF.getContext(), Subtarget);
-    if (Subtarget->hasReferenceTypes()) {
+    if (Subtarget->hasCallIndirectOverlong()) {
       MIB.addSym(Table);
     } else {
       // For the MVP there is at most one table whose number is 0, but we can't
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
index 0772afb039f82..79d6f21517e5d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
@@ -11,13 +11,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
-// Instruction requiring HasBulkMemory and the bulk memory prefix byte
+// Instruction requiring HasBulkMemoryOpt and the bulk memory prefix byte
 multiclass BULK_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
                   list<dag> pattern_r, string asmstr_r = "",
                   string asmstr_s = "", bits<32> simdop = -1> {
   defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
               !or(0xfc00, !and(0xff, simdop))>,
-            Requires<[HasBulkMemory]>;
+            Requires<[HasBulkMemoryOpt]>;
 }
 
 // Bespoke types and nodes for bulk memory ops
@@ -89,14 +89,14 @@ defm CPY_A#B : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx,
                    rc:$dst, rc:$src, rc:$len
                  )],
                  "", "", 0>,
-                  Requires<[HasBulkMemory]>;
+                  Requires<[HasBulkMemoryOpt]>;
 
 let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in
 defm SET_A#B : I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size),
                  (outs), (ins i32imm_op:$idx),
                  [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)],
                  "", "", 0>,
-                 Requires<[HasBulkMemory]>;
+                 Requires<[HasBulkMemoryOpt]>;
 
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index b3ea499c4f915..415e802951a94 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -30,6 +30,14 @@ def HasBulkMemory :
     Predicate<"Subtarget->hasBulkMemory()">,
     AssemblerPredicate<(all_of FeatureBulkMemory), "bulk-memory">;
 
+def HasBulkMemoryOpt :
+    Predicate<"Subtarget->hasBulkMemoryOpt()">,
+    AssemblerPredicate<(all_of FeatureBulkMemoryOpt), "bulk-memory-opt">;
+
+def HasCallIndirectOverlong :
+    Predicate<"Subtarget->hasCallIndirectOverlong()">,
+    AssemblerPredicate<(all_of FeatureCallIndirectOverlong), "call-indirect-overlong">;
+
 def HasExceptionHandling :
     Predicate<"Subtarget->hasExceptionHandling()">,
     AssemblerPredicate<(all_of FeatureExceptionHandling), "exception-handling">;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
index d51bfeb6d8592..6f37dab409534 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
@@ -23,7 +23,7 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
     SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   auto &ST = DAG.getMachineFunction().getSubtarget<WebAssemblySubtarget>();
-  if (!ST.hasBulkMemory())
+  if (!ST.hasBulkMemoryOpt())
     return SDValue();
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
@@ -51,7 +51,7 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(
     SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo) const {
   auto &ST = DAG.getMachineFunction().getSubtarget<WebAssemblySubtarget>();
-  if (!ST.hasBulkMemory())
+  if (!ST.hasBulkMemoryOpt())
     return SDValue();
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
index 912f61765579f..40ea48ab3ac48 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
@@ -34,6 +34,24 @@ WebAssemblySubtarget::initializeSubtargetDependencies(StringRef CPU,
     CPU = "generic";
 
   ParseSubtargetFeatures(CPU, /*TuneCPU*/ CPU, FS);
+
+  FeatureBitset Bits = getFeatureBits();
+
+  // bulk-memory implies bulk-memory-opt
+  if (HasBulkMemory) {
+    HasBulkMemoryOpt = true;
+    Bits.set(WebAssembly::FeatureBulkMemoryOpt);
+  }
+
+  // reference-types implies call-indirect-overlong
+  if (HasReferenceTypes) {
+    HasCallIndirectOverlong = true;
+    Bits.set(WebAssembly::FeatureCallIndirectOverlong);
+  }
+
+  // In case we changed any bits, update `MCSubtargetInfo`'s `FeatureBitset`.
+  setFeatureBits(Bits);
+
   return *this;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
index f2bf2902f775b..591ce25611e3e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -41,6 +41,8 @@ class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
 
   bool HasAtomics = false;
   bool HasBulkMemory = false;
+  bool HasBulkMemoryOpt = false;
+  bool HasCallIndirectOverlong = false;
   bool HasExceptionHandling = false;
   bool HasExtendedConst = false;
   bool HasFP16 = false;
@@ -95,6 +97,8 @@ class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
   bool hasAddr64() const { return TargetTriple.isArch64Bit(); }
   bool hasAtomics() const { return HasAtomics; }
   bool hasBulkMemory() const { return HasBulkMemory; }
+  bool hasBulkMemoryOpt() const { return HasBulkMemoryOpt; }
+  bool hasCallIndirectOverlong() const { return HasCallIndirectOverlong; }
   bool hasExceptionHandling() const { return HasExceptionHandling; }
   bool hasExtendedConst() const { return HasExtendedConst; }
   bool hasFP16() const { return HasFP16; }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 8ec72d5c47833..9c95d730480f9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -118,13 +118,13 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
           T,
           TT.isArch64Bit()
               ? (TT.isOSEmscripten() ? "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-"
-                                       "f128:64-n32:64-S128-ni:1:10:20"
+                                       "i128:128-f128:64-n32:64-S128-ni:1:10:20"
                                      : "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-"
-                                       "n32:64-S128-ni:1:10:20")
+                                       "i128:128-n32:64-S128-ni:1:10:20")
               : (TT.isOSEmscripten() ? "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-"
-                                       "f128:64-n32:64-S128-ni:1:10:20"
+                                       "i128:128-f128:64-n32:64-S128-ni:1:10:20"
                                      : "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-"
-                                       "n32:64-S128-ni:1:10:20"),
+                                       "i128:128-n32:64-S128-ni:1:10:20"),
           TT, CPU, FS, Options, getEffectiveRelocModel(RM, TT),
           getEffectiveCodeModel(CM, CodeModel::Large), OL),
       TLOF(new WebAssemblyTargetObjectFile()),
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
index f950a915db96f..6cfc93ef1faae 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
@@ -116,7 +116,7 @@ MCSymbolWasm *WebAssembly::getOrCreateFunctionTableSymbol(
     Sym->setUndefined();
   }
   // MVP object files can't have symtab entries for tables.
-  if (!(Subtarget && Subtarget->hasReferenceTypes()))
+  if (!(Subtarget && Subtarget->hasCallIndirectOverlong()))
     Sym->setOmitFromLinkingSection();
   return Sym;
 }
@@ -141,7 +141,7 @@ MCSymbolWasm *WebAssembly::getOrCreateFuncrefCallTableSymbol(
     Sym->setTableType(TableType);
   }
   // MVP object files can't have symtab entries for tables.
-  if (!(Subtarget && Subtarget->hasReferenceTypes()))
+  if (!(Subtarget && Subtarget->hasCallIndirectOverlong()))
     Sym->setOmitFromLinkingSection();
   return Sym;
 }
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 0641dca07a890..76ef207f7d47d 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -2423,11 +2423,11 @@ SDValue X86DAGToDAGISel::matchIndexRecursively(SDValue N,
       if (CurDAG->isBaseWithConstantOffset(Src)) {
         SDValue AddSrc = Src.getOperand(0);
         auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
-        uint64_t Offset = (uint64_t)AddVal->getSExtValue();
-        if (!foldOffsetIntoAddress(Offset * AM.Scale, AM)) {
+        int64_t Offset = AddVal->getSExtValue();
+        if (!foldOffsetIntoAddress((uint64_t)Offset * AM.Scale, AM)) {
           SDLoc DL(N);
           SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc);
-          SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT);
+          SDValue ExtVal = CurDAG->getSignedConstant(Offset, DL, VT);
           SDValue ExtAdd = CurDAG->getNode(ISD::ADD, DL, VT, ExtSrc, ExtVal);
           insertDAGNode(*CurDAG, N, ExtSrc);
           insertDAGNode(*CurDAG, N, ExtVal);
@@ -4828,7 +4828,9 @@ bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
 
   // The variable operand must be all zeros in the top bits to allow using the
   // new, negative constant as the mask.
-  if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
+  // TODO: Handle constant folding?
+  KnownBits Known0 = CurDAG->computeKnownBits(And0);
+  if (Known0.isConstant() || !HighZeros.isSubsetOf(Known0.Zero))
     return false;
 
   // Check if the mask is -1. In that case, this is an unnecessary instruction
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e4533570f7508..decd9a64d8845 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3306,7 +3306,7 @@ bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
 
   // Assume target opcodes can't be scalarized.
   // TODO - do we have any exceptions?
-  if (Opc >= ISD::BUILTIN_OP_END)
+  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
     return false;
 
   // If the vector op is not supported, try to convert to scalar.
@@ -17318,6 +17318,13 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                               Zeroable, Subtarget, DAG))
     return PSHUFB;
 
+  // Try to simplify this by merging 128-bit lanes to enable a lane-based
+  // shuffle.
+  if (!V2.isUndef())
+    if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
+            DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
+      return Result;
+
   return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);
 }
 
@@ -23559,6 +23566,9 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
 
     SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
     if (isSoftF16(EltVT, Subtarget)) {
+      if (Subtarget.hasAVX512() && !Subtarget.hasVLX())
+        return SDValue();
+
       // Break 256-bit FP vector compare into smaller ones.
       if (OpVT.is256BitVector() && !Subtarget.useAVX512Regs())
         return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl);
@@ -24228,8 +24238,10 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   if (Subtarget.hasAVX10_2()) {
     if (CC == ISD::SETOEQ || CC == ISD::SETUNE) {
       auto NewCC = (CC == ISD::SETOEQ) ? X86::COND_E : (X86::COND_NE);
-      return getSETCC(NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1),
-                      dl, DAG);
+      assert(Op0.getSimpleValueType() != MVT::bf16 && "Unsupported Type");
+      if (Op0.getSimpleValueType() != MVT::f80)
+        return getSETCC(
+            NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1), dl, DAG);
     }
   }
   // Handle floating point.
@@ -26265,6 +26277,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       }
       if (!NewOp)
         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3);
+      if (IntrData->Opc0 == X86ISD::VFMADDCSH ||
+          IntrData->Opc0 == X86ISD::VFCMADDCSH)
+        return getScalarMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
       return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
     }
     case IFMA_OP:
@@ -28412,6 +28427,43 @@ SDValue X86TargetLowering::LowerRESET_FPENV(SDValue Op,
   return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget);
 }
 
+// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
+uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt = 0) {
+  assert((Amt < 8) && "Shift/Rotation amount out of range");
+  switch (Opcode) {
+  case ISD::BITREVERSE:
+    return 0x8040201008040201ULL;
+  case ISD::SHL:
+    return ((0x0102040810204080ULL >> (Amt)) &
+            (0x0101010101010101ULL * (0xFF >> (Amt))));
+  case ISD::SRL:
+    return ((0x0102040810204080ULL << (Amt)) &
+            (0x0101010101010101ULL * ((0xFF << (Amt)) & 0xFF)));
+  case ISD::SRA:
+    return (getGFNICtrlImm(ISD::SRL, Amt) |
+            (0x8080808080808080ULL >> (64 - (8 * Amt))));
+  case ISD::ROTL:
+    return getGFNICtrlImm(ISD::SRL, 8 - Amt) | getGFNICtrlImm(ISD::SHL, Amt);
+  case ISD::ROTR:
+    return getGFNICtrlImm(ISD::SHL, 8 - Amt) | getGFNICtrlImm(ISD::SRL, Amt);
+  }
+  llvm_unreachable("Unsupported GFNI opcode");
+}
+
+// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
+SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL,
+                        MVT VT, unsigned Amt = 0) {
+  assert(VT.getVectorElementType() == MVT::i8 &&
+         (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type");
+  uint64_t Imm = getGFNICtrlImm(Opcode, Amt);
+  SmallVector<SDValue> MaskBits;
+  for (unsigned I = 0, E = VT.getSizeInBits(); I != E; I += 8) {
+    uint64_t Bits = (Imm >> (I % 64)) & 255;
+    MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8));
+  }
+  return DAG.getBuildVector(VT, DL, MaskBits);
+}
+
 /// Lower a vector CTLZ using native supported vector CTLZ instruction.
 //
 // i8/i16 vector implemented using dword LZCNT vector instruction
@@ -29597,43 +29649,6 @@ SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
   return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
 }
 
-// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
-uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt = 0) {
-  assert((Amt < 8) && "Shift/Rotation amount out of range");
-  switch (Opcode) {
-  case ISD::BITREVERSE:
-    return 0x8040201008040201ULL;
-  case ISD::SHL:
-    return ((0x0102040810204080ULL >> (Amt)) &
-            (0x0101010101010101ULL * (0xFF >> (Amt))));
-  case ISD::SRL:
-    return ((0x0102040810204080ULL << (Amt)) &
-            (0x0101010101010101ULL * ((0xFF << (Amt)) & 0xFF)));
-  case ISD::SRA:
-    return (getGFNICtrlImm(ISD::SRL, Amt) |
-            (0x8080808080808080ULL >> (64 - (8 * Amt))));
-  case ISD::ROTL:
-    return getGFNICtrlImm(ISD::SRL, 8 - Amt) | getGFNICtrlImm(ISD::SHL, Amt);
-  case ISD::ROTR:
-    return getGFNICtrlImm(ISD::SHL, 8 - Amt) | getGFNICtrlImm(ISD::SRL, Amt);
-  }
-  llvm_unreachable("Unsupported GFNI opcode");
-}
-
-// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
-SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT,
-                        unsigned Amt = 0) {
-  assert(VT.getVectorElementType() == MVT::i8 &&
-         (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type");
-  uint64_t Imm = getGFNICtrlImm(Opcode, Amt);
-  SmallVector<SDValue> MaskBits;
-  for (unsigned I = 0, E = VT.getSizeInBits(); I != E; I += 8) {
-    uint64_t Bits = (Imm >> (I % 64)) & 255;
-    MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8));
-  }
-  return DAG.getBuildVector(VT, DL, MaskBits);
-}
-
 // Return true if the required (according to Opcode) shift-imm form is natively
 // supported by the Subtarget
 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
@@ -33247,7 +33262,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                                            SmallVectorImpl<SDValue>&Results,
                                            SelectionDAG &DAG) const {
   SDLoc dl(N);
-  switch (N->getOpcode()) {
+  unsigned Opc = N->getOpcode();
+  switch (Opc) {
   default:
 #ifndef NDEBUG
     dbgs() << "ReplaceNodeResults: ";
@@ -33343,7 +33359,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     EVT VT = N->getValueType(0);
     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
            VT == MVT::v2i32 && "Unexpected VT!");
-    bool IsSigned = N->getOpcode() == ISD::SMULO;
+    bool IsSigned = Opc == ISD::SMULO;
     unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
     SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
     SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
@@ -33400,7 +33416,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     Ops[0] = N->getOperand(1);
     SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
 
-    SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
+    SDValue Res = DAG.getNode(Opc, dl, WideVT, InVec0, InVec1);
     Results.push_back(Res);
     return;
   }
@@ -33438,7 +33454,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
         EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
         SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
         SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
-        SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
+        SDValue Res = DAG.getNode(Opc, dl, ResVT, N0, N1);
         Results.push_back(Res);
       }
       return;
@@ -33559,7 +33575,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
         (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
       assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
              "Unexpected type action!");
-      assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode");
+      assert(Opc == ISD::SIGN_EXTEND && "Unexpected opcode");
       // Custom split this so we can extend i8/i16->i32 invec. This is better
       // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
       // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
@@ -33596,7 +33612,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
 
         // Promote the input to 128 bits. Type legalization will turn this into
         // zext_inreg/sext_inreg.
-        In = DAG.getNode(N->getOpcode(), dl, InVT, In);
+        In = DAG.getNode(Opc, dl, InVT, In);
       }
 
       // Perform custom splitting instead of the two stage extend we would get
@@ -33605,7 +33621,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
       assert(isTypeLegal(LoVT) && "Split VT not legal?");
 
-      SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG);
+      SDValue Lo = getEXTEND_VECTOR_INREG(Opc, dl, LoVT, In, DAG);
 
       // We need to shift the input over by half the number of elements.
       unsigned NumElts = InVT.getVectorNumElements();
@@ -33615,7 +33631,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
         ShufMask[i] = i + HalfNumElts;
 
       SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
-      Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG);
+      Hi = getEXTEND_VECTOR_INREG(Opc, dl, HiVT, Hi, DAG);
 
       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
       Results.push_back(Res);
@@ -33627,8 +33643,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::FP_TO_UINT:
   case ISD::STRICT_FP_TO_UINT: {
     bool IsStrict = N->isStrictFPOpcode();
-    bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
-                    N->getOpcode() == ISD::STRICT_FP_TO_SINT;
+    bool IsSigned = Opc == ISD::FP_TO_SINT || Opc == ISD::STRICT_FP_TO_SINT;
     EVT VT = N->getValueType(0);
     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
@@ -33639,13 +33654,13 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
       if (IsStrict) {
         Res =
-            DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
+            DAG.getNode(Opc, dl, {VT, MVT::Other},
                         {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
                                             {NVT, MVT::Other}, {Chain, Src})});
         Chain = Res.getValue(1);
       } else {
-        Res = DAG.getNode(N->getOpcode(), dl, VT,
-                          DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
+        Res =
+            DAG.getNode(Opc, dl, VT, DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
       }
       Results.push_back(Res);
       if (IsStrict)
@@ -33668,13 +33683,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       }
 
       if (IsStrict) {
-        unsigned Opc =
-            IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
+        Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
         Res =
             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
         Chain = Res.getValue(1);
       } else {
-        unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
+        Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         Res = DAG.getNode(Opc, dl, ResVT, Src);
       }
 
@@ -33760,7 +33774,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
           return;
         }
 
-        unsigned Opc;
         if (IsStrict)
           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
         else
@@ -33799,7 +33812,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       if (Src.getValueType() == MVT::v2f32 && IsStrict) {
         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
                           DAG.getConstantFP(0.0, dl, MVT::v2f32));
-        SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
+        SDValue Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
                                   {N->getOperand(0), Src});
         Results.push_back(Res);
         Results.push_back(Res.getValue(1));
@@ -33823,7 +33836,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
           std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits());
       MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);
       MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);
-      unsigned Opc = N->getOpcode();
       if (NumElts != SrcElts) {
         if (IsStrict)
           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
@@ -33877,8 +33889,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::UINT_TO_FP:
   case ISD::STRICT_UINT_TO_FP: {
     bool IsStrict = N->isStrictFPOpcode();
-    bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
-                    N->getOpcode() == ISD::STRICT_SINT_TO_FP;
+    bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
     EVT VT = N->getValueType(0);
     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
     if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() &&
@@ -33972,7 +33983,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       // FIXME: Should generic type legalizer do this?
       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
                         DAG.getConstant(0, dl, MVT::v2i32));
-      SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
+      SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
                                 {N->getOperand(0), Src});
       Results.push_back(Res);
       Results.push_back(Res.getValue(1));
@@ -39832,6 +39843,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   bool AllowBWIVPERMV3 =
       (Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
 
+  // If root was a VPERMV3 node, always allow a variable shuffle.
+  if (Root.getOpcode() == X86ISD::VPERMV3)
+    AllowVariableCrossLaneMask = AllowVariablePerLaneMask = true;
+
   bool MaskContainsZeros = isAnyZero(Mask);
 
   if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
@@ -41401,6 +41416,7 @@ static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
         N->isOnlyUserOf(N.getOperand(0).getNode())) {
       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
       unsigned SrcOpcode = N0.getOpcode();
+      EVT OpVT = N0.getValueType();
       if (TLI.isBinOp(SrcOpcode) && IsSafeToMoveShuffle(N0, SrcOpcode)) {
         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
@@ -41418,13 +41434,23 @@ static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00);
             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01);
           }
-          EVT OpVT = N0.getValueType();
           return DAG.getBitcast(ShuffleVT,
                                 DAG.getNode(SrcOpcode, DL, OpVT,
                                             DAG.getBitcast(OpVT, LHS),
                                             DAG.getBitcast(OpVT, RHS)));
         }
       }
+      if (SrcOpcode == ISD::SINT_TO_FP && IsSafeToMoveShuffle(N0, SrcOpcode) &&
+          OpVT.getScalarSizeInBits() ==
+              N0.getOperand(0).getScalarValueSizeInBits()) {
+        SDValue Op00 = DAG.getBitcast(ShuffleVT, N0.getOperand(0));
+        SDValue Res =
+            N.getNumOperands() == 2
+                ? DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1))
+                : DAG.getNode(Opc, DL, ShuffleVT, Op00);
+        Res = DAG.getBitcast(N0.getOperand(0).getValueType(), Res);
+        return DAG.getBitcast(ShuffleVT, DAG.getNode(SrcOpcode, DL, OpVT, Res));
+      }
     }
     break;
   }
@@ -41500,6 +41526,21 @@ static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
             ShuffleVT,
             DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res)));
       }
+      // TODO: We can generalize this for other shuffles/conversions.
+      if (Opc == X86ISD::UNPCKL && SrcOpcode == X86ISD::CVTPH2PS &&
+          N1.getOpcode() == SrcOpcode &&
+          N0.getValueType() == N1.getValueType() &&
+          N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
+          ShuffleVT.getScalarSizeInBits() == N0.getScalarValueSizeInBits() &&
+          IsSafeToMoveShuffle(N0, SrcOpcode) &&
+          IsSafeToMoveShuffle(N1, SrcOpcode)) {
+        EVT OpSrcVT = N0.getOperand(0).getValueType();
+        EVT OpDstVT = N0.getValueType();
+        SDValue Res =
+            DAG.getNode(Opc, DL, OpSrcVT, N0.getOperand(0), N1.getOperand(0));
+        return DAG.getBitcast(ShuffleVT,
+                              DAG.getNode(SrcOpcode, DL, OpDstVT, Res));
+      }
     }
     break;
   }
@@ -42259,11 +42300,11 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
     return SDValue();
   }
   case X86ISD::VPERMV3: {
-    // Combine VPERMV3 to widened VPERMV if the two source operands are split
-    // from the same vector.
     SDValue V1 = peekThroughBitcasts(N.getOperand(0));
     SDValue V2 = peekThroughBitcasts(N.getOperand(2));
     MVT SVT = V1.getSimpleValueType();
+    // Combine VPERMV3 to widened VPERMV if the two source operands are split
+    // from the same vector.
     if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
         V1.getConstantOperandVal(1) == 0 &&
         V2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
@@ -42284,14 +42325,25 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
     }
     SmallVector<SDValue, 2> Ops;
     SmallVector<int, 32> Mask;
-    if (isShuffleFoldableLoad(N.getOperand(0)) &&
-        !isShuffleFoldableLoad(N.getOperand(2)) &&
-        getTargetShuffleMask(N, /*AllowSentinelZero=*/false, Ops, Mask)) {
-      ShuffleVectorSDNode::commuteMask(Mask);
-      SDValue NewMask = getConstVector(
-          Mask, N.getOperand(1).getSimpleValueType(), DAG, DL, /*IsMask=*/true);
-      return DAG.getNode(X86ISD::VPERMV3, DL, VT, N.getOperand(2), NewMask,
-                         N.getOperand(0));
+    if (getTargetShuffleMask(N, /*AllowSentinelZero=*/false, Ops, Mask)) {
+      MVT MaskVT = N.getOperand(1).getSimpleValueType();
+      // Canonicalize to VPERMV if both sources are the same.
+      if (V1 == V2) {
+        for (int &M : Mask)
+          M = (M < 0 ? M : M & Mask.size() - 1);
+        SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL,
+                                         /*IsMask=*/true);
+        return DAG.getNode(X86ISD::VPERMV, DL, VT, NewMask, N.getOperand(0));
+      }
+      // Commute foldable source to the RHS.
+      if (isShuffleFoldableLoad(N.getOperand(0)) &&
+          !isShuffleFoldableLoad(N.getOperand(2))) {
+        ShuffleVectorSDNode::commuteMask(Mask);
+        SDValue NewMask =
+            getConstVector(Mask, MaskVT, DAG, DL, /*IsMask=*/true);
+        return DAG.getNode(X86ISD::VPERMV3, DL, VT, N.getOperand(2), NewMask,
+                           N.getOperand(0));
+      }
     }
     return SDValue();
   }
@@ -45842,7 +45894,8 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
 /// Extracting a scalar FP value from vector element 0 is free, so extract each
 /// operand first, then perform the math as a scalar op.
 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
-                                 const X86Subtarget &Subtarget) {
+                                 const X86Subtarget &Subtarget,
+                                 TargetLowering::DAGCombinerInfo &DCI) {
   assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract");
   SDValue Vec = ExtElt->getOperand(0);
   SDValue Index = ExtElt->getOperand(1);
@@ -45877,13 +45930,13 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
   // Vector FP selects don't fit the pattern of FP math ops (because the
   // condition has a different type and we have to change the opcode), so deal
   // with those here.
-  // FIXME: This is restricted to pre type legalization by ensuring the setcc
-  // has i1 elements. If we loosen this we need to convert vector bool to a
-  // scalar bool.
-  if (Vec.getOpcode() == ISD::VSELECT &&
+  // FIXME: This is restricted to pre type legalization. If we loosen this we
+  // need to convert vector bool to a scalar bool.
+  if (DCI.isBeforeLegalize() && Vec.getOpcode() == ISD::VSELECT &&
       Vec.getOperand(0).getOpcode() == ISD::SETCC &&
-      Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
       Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
+    assert(Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
+           "Unexpected cond type for combine");
     // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
     SDLoc DL(ExtElt);
     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
@@ -46242,7 +46295,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineArithReduction(N, DAG, Subtarget))
     return V;
 
-  if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget))
+  if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget, DCI))
     return V;
 
   if (CIdx)
@@ -52631,6 +52684,29 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
                         St->getMemOperand()->getFlags());
   }
 
+  // Convert scalar fabs/fneg load-store to integer equivalents.
+  if ((VT == MVT::f16 || VT == MVT::bf16 || VT == MVT::f32 || VT == MVT::f64) &&
+      (StoredVal.getOpcode() == ISD::FABS ||
+       StoredVal.getOpcode() == ISD::FNEG) &&
+      ISD::isNormalLoad(StoredVal.getOperand(0).getNode()) &&
+      StoredVal.hasOneUse() && StoredVal.getOperand(0).hasOneUse()) {
+    MVT IntVT = VT.getSimpleVT().changeTypeToInteger();
+    if (TLI.isTypeLegal(IntVT)) {
+      APInt SignMask = APInt::getSignMask(VT.getScalarSizeInBits());
+      unsigned SignOp = ISD::XOR;
+      if (StoredVal.getOpcode() == ISD::FABS) {
+        SignMask = ~SignMask;
+        SignOp = ISD::AND;
+      }
+      SDValue LogicOp = DAG.getNode(
+          SignOp, dl, IntVT, DAG.getBitcast(IntVT, StoredVal.getOperand(0)),
+          DAG.getConstant(SignMask, dl, IntVT));
+      return DAG.getStore(St->getChain(), dl, LogicOp, St->getBasePtr(),
+                          St->getPointerInfo(), St->getOriginalAlign(),
+                          St->getMemOperand()->getFlags());
+    }
+  }
+
   // If we are saving a 32-byte vector and 32-byte stores are slow, such as on
   // Sandy Bridge, perform two 16-byte stores.
   unsigned Fast;
@@ -52756,6 +52832,44 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Convert store(cmov(load(p), x, CC), p) to cstore(x, p, CC)
+  //         store(cmov(x, load(p), CC), p) to cstore(x, p, InvertCC)
+  if ((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
+      Subtarget.hasCF() && St->isSimple()) {
+    SDValue Cmov;
+    if (StoredVal.getOpcode() == X86ISD::CMOV)
+      Cmov = StoredVal;
+    else if (StoredVal.getOpcode() == ISD::TRUNCATE &&
+             StoredVal.getOperand(0).getOpcode() == X86ISD::CMOV)
+      Cmov = StoredVal.getOperand(0);
+    else
+      return SDValue();
+
+    auto *Ld = dyn_cast<LoadSDNode>(St->getChain());
+    if (!Ld || !Ld->isSimple() || Ld->getBasePtr() != St->getBasePtr())
+      return SDValue();
+
+    bool InvertCC = false;
+    SDValue V = SDValue(Ld, 0);
+    if (V == Cmov.getOperand(1))
+      InvertCC = true;
+    else if (V != Cmov.getOperand(0))
+      return SDValue();
+
+    SDVTList Tys = DAG.getVTList(MVT::Other);
+    SDValue CC = Cmov.getOperand(2);
+    SDValue Src = DAG.getAnyExtOrTrunc(Cmov.getOperand(!InvertCC), dl, VT);
+    if (InvertCC)
+      CC = DAG.getTargetConstant(
+          GetOppositeBranchCondition(
+              (X86::CondCode)Cmov.getConstantOperandVal(2)),
+          dl, MVT::i8);
+    SDValue Ops[] = {St->getChain(), Src, St->getBasePtr(), CC,
+                     Cmov.getOperand(3)};
+    return DAG.getMemIntrinsicNode(X86ISD::CSTORE, dl, Tys, Ops, VT,
+                                   St->getMemOperand());
+  }
+
   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
   // the FP state in cases where an emms may be missing.
   // A preferable solution to the general problem is to figure out the right
@@ -56394,9 +56508,11 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
+static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
                             const SDLoc &DL, EVT VT,
                             const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
+
   // Example of pattern we try to detect:
   // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1))))
   //(add (build_vector (extract_elt t, 0),
@@ -56411,15 +56527,19 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
   if (!Subtarget.hasSSE2())
     return SDValue();
 
-  if (Op0.getOpcode() != ISD::BUILD_VECTOR ||
-      Op1.getOpcode() != ISD::BUILD_VECTOR)
-    return SDValue();
-
   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
       VT.getVectorNumElements() < 4 ||
       !isPowerOf2_32(VT.getVectorNumElements()))
     return SDValue();
 
+  SDValue Op0, Op1, Accum;
+  if (!sd_match(N, m_Add(m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op0)),
+                         m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op1)))) &&
+      !sd_match(N, m_Add(m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op0)),
+                         m_Add(m_Value(Accum), m_AllOf(m_Opc(ISD::BUILD_VECTOR),
+                                                       m_Value(Op1))))))
+    return SDValue();
+
   // Check if one of Op0,Op1 is of the form:
   // (build_vector (extract_elt Mul, 0),
   //               (extract_elt Mul, 2),
@@ -56436,26 +56556,23 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
     SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
             Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1);
     // TODO: Be more tolerant to undefs.
-    if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
-      return SDValue();
-    auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1));
-    auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1));
-    auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1));
-    auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1));
-    if (!Const0L || !Const1L || !Const0H || !Const1H)
+    APInt Idx0L, Idx0H, Idx1L, Idx1H;
+    if (!sd_match(Op0L, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(),
+                                m_ConstInt(Idx0L))) ||
+        !sd_match(Op0H, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(),
+                                m_ConstInt(Idx0H))) ||
+        !sd_match(Op1L, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(),
+                                m_ConstInt(Idx1L))) ||
+        !sd_match(Op1H, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(),
+                                m_ConstInt(Idx1H))))
       return SDValue();
-    unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(),
-             Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue();
     // Commutativity of mul allows factors of a product to reorder.
-    if (Idx0L > Idx1L)
+    if (Idx0L.getZExtValue() > Idx1L.getZExtValue())
       std::swap(Idx0L, Idx1L);
-    if (Idx0H > Idx1H)
+    if (Idx0H.getZExtValue() > Idx1H.getZExtValue())
       std::swap(Idx0H, Idx1H);
     // Commutativity of add allows pairs of factors to reorder.
-    if (Idx0L > Idx0H) {
+    if (Idx0L.getZExtValue() > Idx0H.getZExtValue()) {
       std::swap(Idx0L, Idx0H);
       std::swap(Idx1L, Idx1H);
     }
@@ -56496,19 +56613,21 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
                                  InVT.getVectorNumElements() / 2);
     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
   };
-  return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
+  SDValue R = SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDBuilder);
+  if (Accum)
+    R = DAG.getNode(ISD::ADD, DL, VT, R, Accum);
+  return R;
 }
 
 // Attempt to turn this pattern into PMADDWD.
 // (add (mul (sext (build_vector)), (sext (build_vector))),
 //      (mul (sext (build_vector)), (sext (build_vector)))
-static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
+static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
                               const SDLoc &DL, EVT VT,
                               const X86Subtarget &Subtarget) {
-  if (!Subtarget.hasSSE2())
-    return SDValue();
+  using namespace SDPatternMatch;
 
-  if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
+  if (!Subtarget.hasSSE2())
     return SDValue();
 
   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
@@ -56516,25 +56635,13 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
       !isPowerOf2_32(VT.getVectorNumElements()))
     return SDValue();
 
-  SDValue N00 = N0.getOperand(0);
-  SDValue N01 = N0.getOperand(1);
-  SDValue N10 = N1.getOperand(0);
-  SDValue N11 = N1.getOperand(1);
-
   // All inputs need to be sign extends.
   // TODO: Support ZERO_EXTEND from known positive?
-  if (N00.getOpcode() != ISD::SIGN_EXTEND ||
-      N01.getOpcode() != ISD::SIGN_EXTEND ||
-      N10.getOpcode() != ISD::SIGN_EXTEND ||
-      N11.getOpcode() != ISD::SIGN_EXTEND)
+  SDValue N00, N01, N10, N11;
+  if (!sd_match(N, m_Add(m_Mul(m_SExt(m_Value(N00)), m_SExt(m_Value(N01))),
+                         m_Mul(m_SExt(m_Value(N10)), m_SExt(m_Value(N11))))))
     return SDValue();
 
-  // Peek through the extends.
-  N00 = N00.getOperand(0);
-  N01 = N01.getOperand(0);
-  N10 = N10.getOperand(0);
-  N11 = N11.getOperand(0);
-
   // Must be extending from vXi16.
   EVT InVT = N00.getValueType();
   if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||
@@ -56561,34 +56668,26 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
     SDValue N10Elt = N10.getOperand(i);
     SDValue N11Elt = N11.getOperand(i);
     // TODO: Be more tolerant to undefs.
-    if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-        N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+    SDValue N00In, N01In, N10In, N11In;
+    APInt IdxN00, IdxN01, IdxN10, IdxN11;
+    if (!sd_match(N00Elt, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(N00In),
+                                  m_ConstInt(IdxN00))) ||
+        !sd_match(N01Elt, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(N01In),
+                                  m_ConstInt(IdxN01))) ||
+        !sd_match(N10Elt, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(N10In),
+                                  m_ConstInt(IdxN10))) ||
+        !sd_match(N11Elt, m_BinOp(ISD::EXTRACT_VECTOR_ELT, m_Value(N11In),
+                                  m_ConstInt(IdxN11))))
       return SDValue();
-    auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
-    auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
-    auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
-    auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
-    if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
-      return SDValue();
-    unsigned IdxN00 = ConstN00Elt->getZExtValue();
-    unsigned IdxN01 = ConstN01Elt->getZExtValue();
-    unsigned IdxN10 = ConstN10Elt->getZExtValue();
-    unsigned IdxN11 = ConstN11Elt->getZExtValue();
     // Add is commutative so indices can be reordered.
-    if (IdxN00 > IdxN10) {
+    if (IdxN00.getZExtValue() > IdxN10.getZExtValue()) {
       std::swap(IdxN00, IdxN10);
       std::swap(IdxN01, IdxN11);
     }
     // N0 indices be the even element. N1 indices must be the next odd element.
-    if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
-        IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
+    if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 || IdxN01 != 2 * i ||
+        IdxN11 != 2 * i + 1)
       return SDValue();
-    SDValue N00In = N00Elt.getOperand(0);
-    SDValue N01In = N01Elt.getOperand(0);
-    SDValue N10In = N10Elt.getOperand(0);
-    SDValue N11In = N11Elt.getOperand(0);
 
     // First time we find an input capture it.
     if (!In0) {
@@ -56762,9 +56861,9 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
   if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget))
     return Select;
 
-  if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget))
+  if (SDValue MAdd = matchPMADDWD(DAG, N, DL, VT, Subtarget))
     return MAdd;
-  if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget))
+  if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
     return MAdd;
   if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
     return MAdd;
@@ -56809,6 +56908,23 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Peephole for 512-bit VPDPBSSD on non-VLX targets.
+  // TODO: Should this be part of matchPMADDWD/matchPMADDWD_2?
+  if (Subtarget.hasVNNI() && Subtarget.useAVX512Regs() && VT == MVT::v16i32) {
+    using namespace SDPatternMatch;
+    SDValue Accum, Lo0, Lo1, Hi0, Hi1;
+    if (sd_match(N, m_Add(m_Value(Accum),
+                          m_Node(ISD::CONCAT_VECTORS,
+                                 m_BinOp(X86ISD::VPMADDWD, m_Value(Lo0),
+                                         m_Value(Lo1)),
+                                 m_BinOp(X86ISD::VPMADDWD, m_Value(Hi0),
+                                         m_Value(Hi1)))))) {
+      return DAG.getNode(X86ISD::VPDPWSSD, DL, VT, Accum,
+                         concatSubVectors(Lo0, Hi0, DAG, DL),
+                         concatSubVectors(Lo1, Hi1, DAG, DL));
+    }
+  }
+
   // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W)
   if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() &&
       X86::isZeroNode(Op0.getOperand(1))) {
@@ -58343,10 +58459,16 @@ static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG,
                                   DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32))));
   }
 
-  // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
-  if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
-      Src.getOperand(0).getValueType() == MVT::x86mmx)
-    return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
+  if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST) {
+    SDValue SrcOp = Src.getOperand(0);
+    // Combine (v2i64 (scalar_to_vector (i64 (bitcast (double))))) to MOVQ.
+    if (SrcOp.getValueType() == MVT::f64)
+      return DAG.getBitcast(
+          VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, SrcOp));
+    // Combine (v2i64 (scalar_to_vector (i64 (bitcast (mmx))))) to MOVQ2DQ.
+    if (SrcOp.getValueType() == MVT::x86mmx)
+      return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, SrcOp);
+  }
 
   // See if we're broadcasting the scalar value, in which case just reuse that.
   // Ensure the same SDValue from the SDNode use is being used.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 2db25d6dda061..d154ee9745b97 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1079,8 +1079,7 @@ namespace llvm {
     /// function arguments in the caller parameter area. For X86, aggregates
     /// that contains are placed at 16-byte boundaries while the rest are at
     /// 4-byte boundaries.
-    uint64_t getByValTypeAlignment(Type *Ty,
-                                   const DataLayout &DL) const override;
+    Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override;
 
     EVT getOptimalMemOpType(const MemOp &Op,
                             const AttributeList &FuncAttributes) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 8a764de561413..05a5a36ce5cbe 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -262,20 +262,15 @@ static void getMaxByValAlign(Type *Ty, Align &MaxAlign) {
 /// function arguments in the caller parameter area. For X86, aggregates
 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
 /// are at 4-byte boundaries.
-uint64_t X86TargetLowering::getByValTypeAlignment(Type *Ty,
-                                                  const DataLayout &DL) const {
-  if (Subtarget.is64Bit()) {
-    // Max of 8 and alignment of type.
-    Align TyAlign = DL.getABITypeAlign(Ty);
-    if (TyAlign > 8)
-      return TyAlign.value();
-    return 8;
-  }
+Align X86TargetLowering::getByValTypeAlignment(Type *Ty,
+                                               const DataLayout &DL) const {
+  if (Subtarget.is64Bit())
+    return std::max(DL.getABITypeAlign(Ty), Align::Constant<8>());
 
   Align Alignment(4);
   if (Subtarget.hasSSE1())
     getMaxByValAlign(Ty, Alignment);
-  return Alignment.value();
+  return Alignment;
 }
 
 /// It returns EVT::Other if the type should be determined using generic
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 299a2a74d86fc..83a2e981ffd7a 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -13533,17 +13533,17 @@ let Uses = [MXCSR] in {
 multiclass avx512_cfmaop_sh_common<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd,
                                    bit IsCommutable> {
   let Predicates = [HasFP16], Constraints = "@earlyclobber $dst, $src1 = $dst" in {
-    defm r : AVX512_maskable_3src<opc, MRMSrcReg, v4f32x_info, (outs VR128X:$dst),
+    defm r : AVX512_maskable_3src_scalar<opc, MRMSrcReg, f32x_info, (outs VR128X:$dst),
                         (ins VR128X:$src2, VR128X:$src3), OpcodeStr,
                         "$src3, $src2", "$src2, $src3",
                         (v4f32 (OpNode VR128X:$src2, VR128X:$src3, VR128X:$src1)), IsCommutable>,
                         Sched<[WriteFMAX]>;
-    defm m : AVX512_maskable_3src<opc, MRMSrcMem, v4f32x_info, (outs VR128X:$dst),
+    defm m : AVX512_maskable_3src_scalar<opc, MRMSrcMem, f32x_info, (outs VR128X:$dst),
                         (ins VR128X:$src2, ssmem:$src3), OpcodeStr,
                         "$src3, $src2", "$src2, $src3",
                         (v4f32 (OpNode VR128X:$src2, (sse_load_f32 addr:$src3), VR128X:$src1))>,
                         Sched<[WriteFMAX.Folded, WriteFMAX.ReadAfterFold]>;
-    defm rb : AVX512_maskable_3src<opc,  MRMSrcReg, v4f32x_info, (outs VR128X:$dst),
+    defm rb : AVX512_maskable_3src_scalar<opc,  MRMSrcReg, f32x_info, (outs VR128X:$dst),
                         (ins VR128X:$src2, VR128X:$src3, AVX512RC:$rc), OpcodeStr,
                         "$rc, $src3, $src2", "$src2, $src3, $rc",
                         (v4f32 (OpNodeRnd VR128X:$src2, VR128X:$src3, VR128X:$src1, (i32 timm:$rc)))>,
diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td
index 5b50e1943e3db..4b697c4336341 100644
--- a/llvm/lib/Target/X86/X86SchedBroadwell.td
+++ b/llvm/lib/Target/X86/X86SchedBroadwell.td
@@ -162,9 +162,9 @@ defm : X86WriteRes<WriteXCHG,      [BWPort0156], 2, [3], 3>;
 
 // Integer shifts and rotates.
 defm : BWWriteResPair<WriteShift,    [BWPort06],  1>;
-defm : BWWriteResPair<WriteShiftCL,  [BWPort06,BWPort0156],  3, [2,1], 3>;
+defm : BWWriteResPair<WriteShiftCL,  [BWPort06],  3, [3], 3>;
 defm : BWWriteResPair<WriteRotate,   [BWPort06],  1, [1], 1>;
-defm : BWWriteResPair<WriteRotateCL, [BWPort06,BWPort0156],  3, [2,1], 3>;
+defm : BWWriteResPair<WriteRotateCL, [BWPort06],  3, [3], 3>;
 
 // SHLD/SHRD.
 defm : X86WriteRes<WriteSHDrri, [BWPort1], 3, [1], 1>;
@@ -247,10 +247,10 @@ defm : X86WriteRes<WriteFStoreNT,      [BWPort237,BWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteFStoreNTX,     [BWPort237,BWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteFStoreNTY,     [BWPort237,BWPort4], 1, [1,1], 2>;
 
-defm : X86WriteRes<WriteFMaskedStore32,  [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore32Y, [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore64,  [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore64Y, [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore32,  [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore32Y, [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore64,  [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore64Y, [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
 
 defm : X86WriteRes<WriteFMove,         [BWPort5], 1, [1], 1>;
 defm : X86WriteRes<WriteFMoveX,        [BWPort5], 1, [1], 1>;
@@ -420,10 +420,10 @@ defm : X86WriteRes<WriteVecStoreX,       [BWPort237,BWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreY,       [BWPort237,BWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreNT,      [BWPort237,BWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreNTY,     [BWPort237,BWPort4], 1, [1,1], 2>;
-defm : X86WriteRes<WriteVecMaskedStore32,  [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore32Y, [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore64,  [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore64Y, [BWPort0,BWPort4,BWPort237,BWPort15], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore32,  [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore32Y, [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore64,  [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore64Y, [BWPort0,BWPort4,BWPort23,BWPort1], 5, [1,1,1,1], 4>;
 defm : X86WriteRes<WriteVecMove,         [BWPort015], 1, [1], 1>;
 defm : X86WriteRes<WriteVecMoveX,        [BWPort015], 1, [1], 1>;
 defm : X86WriteRes<WriteVecMoveY,        [BWPort015], 1, [1], 1>;
@@ -615,7 +615,7 @@ def : WriteRes<WriteSystem,     [BWPort0156]> { let Latency = 100; }
 def : WriteRes<WriteMicrocoded, [BWPort0156]> { let Latency = 100; }
 
 // Fence instructions.
-def : WriteRes<WriteFence,  [BWPort23, BWPort4]>;
+def : WriteRes<WriteFence,  [BWPort23, BWPort4]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 
 // Nop, not very useful expect it provides a model for nops!
 def : WriteRes<WriteNop, []>;
@@ -648,13 +648,6 @@ def BWWriteResGroup2 : SchedWriteRes<[BWPort1]> {
 def: InstRW<[BWWriteResGroup2], (instregex "COM(P?)_FST0r",
                                            "UCOM_F(P?)r")>;
 
-def BWWriteResGroup3 : SchedWriteRes<[BWPort5]> {
-  let Latency = 1;
-  let NumMicroOps = 1;
-  let ReleaseAtCycles = [1];
-}
-def: InstRW<[BWWriteResGroup3], (instrs MMX_MOVQ2DQrr)>;
-
 def BWWriteResGroup4 : SchedWriteRes<[BWPort6]> {
   let Latency = 1;
   let NumMicroOps = 1;
@@ -722,7 +715,6 @@ def BWWriteResGroup14 : SchedWriteRes<[BWPort0156]> {
   let ReleaseAtCycles = [2];
 }
 def: InstRW<[BWWriteResGroup14], (instrs LFENCE,
-                                         MFENCE,
                                          WAIT,
                                          XGETBV)>;
 
@@ -740,13 +732,6 @@ def BWWriteResGroup17 : SchedWriteRes<[BWPort01,BWPort015]> {
 }
 def: InstRW<[BWWriteResGroup17], (instrs MMX_MOVDQ2Qrr)>;
 
-def BWWriteResGroup18 : SchedWriteRes<[BWPort237,BWPort0156]> {
-  let Latency = 2;
-  let NumMicroOps = 2;
-  let ReleaseAtCycles = [1,1];
-}
-def: InstRW<[BWWriteResGroup18], (instrs SFENCE)>;
-
 def BWWriteResGroup20 : SchedWriteRes<[BWPort06,BWPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td
index d06e8a9937097..dd1e1c3a6cb53 100644
--- a/llvm/lib/Target/X86/X86SchedHaswell.td
+++ b/llvm/lib/Target/X86/X86SchedHaswell.td
@@ -164,9 +164,9 @@ defm : X86WriteRes<WriteXCHG, [HWPort0156], 2, [3], 3>;
 
 // Integer shifts and rotates.
 defm : HWWriteResPair<WriteShift,    [HWPort06],  1>;
-defm : HWWriteResPair<WriteShiftCL,  [HWPort06, HWPort0156],  3, [2,1], 3>;
+defm : HWWriteResPair<WriteShiftCL,  [HWPort06],  3, [3], 3>;
 defm : HWWriteResPair<WriteRotate,   [HWPort06],  1, [1], 1>;
-defm : HWWriteResPair<WriteRotateCL, [HWPort06, HWPort0156],  3, [2,1], 3>;
+defm : HWWriteResPair<WriteRotateCL, [HWPort06],  3, [3], 3>;
 
 // SHLD/SHRD.
 defm : X86WriteRes<WriteSHDrri, [HWPort1], 3, [1], 1>;
@@ -249,10 +249,10 @@ defm : X86WriteRes<WriteFStoreNT,      [HWPort237,HWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteFStoreNTX,     [HWPort237,HWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteFStoreNTY,     [HWPort237,HWPort4], 1, [1,1], 2>;
 
-defm : X86WriteRes<WriteFMaskedStore32,  [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore32Y, [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore64,  [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteFMaskedStore64Y, [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore32,  [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore32Y, [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore64,  [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteFMaskedStore64Y, [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
 
 defm : X86WriteRes<WriteFMove,         [HWPort5], 1, [1], 1>;
 defm : X86WriteRes<WriteFMoveX,        [HWPort5], 1, [1], 1>;
@@ -420,10 +420,10 @@ defm : X86WriteRes<WriteVecStoreX,       [HWPort237,HWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreY,       [HWPort237,HWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreNT,      [HWPort237,HWPort4], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecStoreNTY,     [HWPort237,HWPort4], 1, [1,1], 2>;
-defm : X86WriteRes<WriteVecMaskedStore32,  [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore32Y, [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore64,  [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
-defm : X86WriteRes<WriteVecMaskedStore64Y, [HWPort0,HWPort4,HWPort237,HWPort15], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore32,  [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore32Y, [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore64,  [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
+defm : X86WriteRes<WriteVecMaskedStore64Y, [HWPort0,HWPort4,HWPort23,HWPort1], 5, [1,1,1,1], 4>;
 defm : X86WriteRes<WriteVecMove,         [HWPort015], 1, [1], 1>;
 defm : X86WriteRes<WriteVecMoveX,        [HWPort015], 1, [1], 1>;
 defm : X86WriteRes<WriteVecMoveY,        [HWPort015], 1, [1], 1>;
@@ -627,7 +627,7 @@ def : WriteRes<WriteSystem,     [HWPort0156]> { let Latency = 100; }
 def : WriteRes<WriteMicrocoded, [HWPort0156]> { let Latency = 100; }
 
 // Fence instructions.
-def : WriteRes<WriteFence,  [HWPort23, HWPort4]>;
+def : WriteRes<WriteFence,  [HWPort23, HWPort4]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 
 // Nop, not very useful expect it provides a model for nops!
 def : WriteRes<WriteNop, []>;
@@ -919,13 +919,6 @@ def HWWriteResGroup3 : SchedWriteRes<[HWPort1]> {
 def: InstRW<[HWWriteResGroup3], (instregex "COM(P?)_FST0r",
                                            "UCOM_F(P?)r")>;
 
-def HWWriteResGroup4 : SchedWriteRes<[HWPort5]> {
-  let Latency = 1;
-  let NumMicroOps = 1;
-  let ReleaseAtCycles = [1];
-}
-def: InstRW<[HWWriteResGroup4], (instrs MMX_MOVQ2DQrr)>;
-
 def HWWriteResGroup5 : SchedWriteRes<[HWPort6]> {
   let Latency = 1;
   let NumMicroOps = 1;
@@ -1054,13 +1047,6 @@ def HWWriteResGroup18 : SchedWriteRes<[HWPort23,HWPort0156]> {
 def: InstRW<[HWWriteResGroup18], (instrs POP16r, POP32r, POP64r)>;
 def: InstRW<[HWWriteResGroup18], (instregex "POP(16|32|64)rmr")>;
 
-def HWWriteResGroup19 : SchedWriteRes<[HWPort237,HWPort0156]> {
-  let Latency = 2;
-  let NumMicroOps = 2;
-  let ReleaseAtCycles = [1,1];
-}
-def: InstRW<[HWWriteResGroup19], (instrs SFENCE)>;
-
 def HWWriteResGroup21 : SchedWriteRes<[HWPort4,HWPort6,HWPort237]> {
   let Latency = 2;
   let NumMicroOps = 3;
@@ -1121,7 +1107,6 @@ def HWWriteResGroup30 : SchedWriteRes<[HWPort0156]> {
   let ReleaseAtCycles = [2];
 }
 def: InstRW<[HWWriteResGroup30], (instrs LFENCE,
-                                         MFENCE,
                                          WAIT,
                                          XGETBV)>;
 
diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 9d6368f19607b..1c57093c89796 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -429,16 +429,14 @@ defm : ICXWriteResPair<WriteVarVecShiftY, [ICXPort01], 1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteVarVecShiftZ, [ICXPort0], 1, [1], 1, 7>;
 
 // Vector insert/extract operations.
-def : WriteRes<WriteVecInsert, [ICXPort5]> {
+def : WriteRes<WriteVecInsert, [ICXPort15,ICXPort5]> {
   let Latency = 2;
   let NumMicroOps = 2;
-  let ReleaseAtCycles = [2];
 }
-def : WriteRes<WriteVecInsertLd, [ICXPort5,ICXPort23]> {
+def : WriteRes<WriteVecInsertLd, [ICXPort15,ICXPort23]> {
   let Latency = 6;
   let NumMicroOps = 2;
 }
-def: InstRW<[WriteVecInsertLd], (instregex "(V?)MOV(H|L)(PD|PS)rm")>;
 
 def : WriteRes<WriteVecExtract, [ICXPort0,ICXPort15]> {
   let Latency = 3;
@@ -607,7 +605,7 @@ defm : ICXWriteResPair<WriteVarShuffle256, [ICXPort5], 3, [1], 1, 7>;  // 256-bi
 def : WriteRes<WriteMicrocoded, [ICXPort0156]> { let Latency = 100; } // def WriteMicrocoded : SchedWrite;
 
 // Fence instructions.
-def : WriteRes<WriteFence,  [ICXPort78, ICXPort49]>;
+def : WriteRes<WriteFence,  [ICXPort78, ICXPort49]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 
 // Load/store MXCSR.
 def : WriteRes<WriteLDMXCSR, [ICXPort0,ICXPort23,ICXPort0156]> { let Latency = 7; let NumMicroOps = 3; let ReleaseAtCycles = [1,1,1]; }
@@ -771,13 +769,6 @@ def ICXWriteResGroup20 : SchedWriteRes<[ICXPort6,ICXPort0156]> {
 }
 def: InstRW<[ICXWriteResGroup20], (instregex "CLFLUSH")>;
 
-def ICXWriteResGroup21 : SchedWriteRes<[ICXPort49,ICXPort78]> {
-  let Latency = 2;
-  let NumMicroOps = 2;
-  let ReleaseAtCycles = [1,1];
-}
-def: InstRW<[ICXWriteResGroup21], (instrs SFENCE)>;
-
 def ICXWriteResGroup23 : SchedWriteRes<[ICXPort06,ICXPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
@@ -916,13 +907,6 @@ def ICXWriteResGroup42 : SchedWriteRes<[ICXPort6,ICXPort0156]> {
 }
 def: InstRW<[ICXWriteResGroup42], (instregex "CLD")>;
 
-def ICXWriteResGroup43 : SchedWriteRes<[ICXPort49,ICXPort78]> {
-  let Latency = 3;
-  let NumMicroOps = 3;
-  let ReleaseAtCycles = [1,2];
-}
-def: InstRW<[ICXWriteResGroup43], (instrs MFENCE)>;
-
 def ICXWriteResGroup44 : SchedWriteRes<[ICXPort06,ICXPort0156]> {
   let Latency = 2;
   let NumMicroOps = 3;
@@ -1311,7 +1295,8 @@ def ICXWriteResGroup92 : SchedWriteRes<[ICXPort5,ICXPort23]> {
   let ReleaseAtCycles = [1,1];
 }
 def: InstRW<[ICXWriteResGroup92], (instregex "VMOV(SD|SS)Zrm(b?)",
-                                              "VPBROADCAST(B|W)(Z128)?rm",
+                                             "VPBROADCAST(B|W)(Z128)?rm",
+                                             "(V?)MOV(H|L)(PD|PS)rm",
                                              "(V?)INSERTPS(Z?)rm",
                                              "(V?)PALIGNR(Z128)?rmi",
                                              "(V?)PERMIL(PD|PS)(Z128)?m(b?)i",
diff --git a/llvm/lib/Target/X86/X86SchedSandyBridge.td b/llvm/lib/Target/X86/X86SchedSandyBridge.td
index 775ad6b1078a5..560a2b12cc221 100644
--- a/llvm/lib/Target/X86/X86SchedSandyBridge.td
+++ b/llvm/lib/Target/X86/X86SchedSandyBridge.td
@@ -588,7 +588,7 @@ def : WriteRes<WriteSTMXCSR, [SBPort0,SBPort4,SBPort5,SBPort23]> { let Latency =
 
 def : WriteRes<WriteSystem,     [SBPort015]> { let Latency = 100; }
 def : WriteRes<WriteMicrocoded, [SBPort015]> { let Latency = 100; }
-def : WriteRes<WriteFence, [SBPort23, SBPort4]>;
+def : WriteRes<WriteFence, [SBPort23, SBPort4]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 def : WriteRes<WriteNop, []>;
 
 // AVX2/FMA is not supported on that architecture, but we should define the basic
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
index e4e833c8f752c..8cd52e2a8ebc9 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
@@ -608,7 +608,7 @@ defm : SKLWriteResPair<WriteVarShuffle256, [SKLPort5], 3, [1], 1, 7>;  // 256-bi
 def : WriteRes<WriteMicrocoded, [SKLPort0156]> { let Latency = 100; } // def WriteMicrocoded : SchedWrite;
 
 // Fence instructions.
-def : WriteRes<WriteFence,  [SKLPort23, SKLPort4]>;
+def : WriteRes<WriteFence,  [SKLPort23, SKLPort4]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 
 // Load/store MXCSR.
 def : WriteRes<WriteLDMXCSR, [SKLPort0,SKLPort23,SKLPort0156]> { let Latency = 7; let NumMicroOps = 3; let ReleaseAtCycles = [1,1,1]; }
@@ -738,13 +738,6 @@ def SKLWriteResGroup20 : SchedWriteRes<[SKLPort6,SKLPort0156]> {
 }
 def: InstRW<[SKLWriteResGroup20], (instregex "CLFLUSH")>;
 
-def SKLWriteResGroup21 : SchedWriteRes<[SKLPort237,SKLPort0156]> {
-  let Latency = 2;
-  let NumMicroOps = 2;
-  let ReleaseAtCycles = [1,1];
-}
-def: InstRW<[SKLWriteResGroup21], (instrs SFENCE)>;
-
 def SKLWriteResGroup23 : SchedWriteRes<[SKLPort06,SKLPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
@@ -834,13 +827,6 @@ def SKLWriteResGroup40 : SchedWriteRes<[SKLPort6,SKLPort0156]> {
 }
 def: InstRW<[SKLWriteResGroup40], (instregex "CLD")>;
 
-def SKLWriteResGroup41 : SchedWriteRes<[SKLPort237,SKLPort0156]> {
-  let Latency = 3;
-  let NumMicroOps = 3;
-  let ReleaseAtCycles = [1,2];
-}
-def: InstRW<[SKLWriteResGroup41], (instrs MFENCE)>;
-
 def SKLWriteResGroup42 : SchedWriteRes<[SKLPort06,SKLPort0156]> {
   let Latency = 2;
   let NumMicroOps = 3;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 62cc4a9ea290c..14a51d1ed9ce6 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -602,7 +602,7 @@ defm : SKXWriteResPair<WriteVarShuffle256, [SKXPort5], 3, [1], 1, 7>;  // 256-bi
 def : WriteRes<WriteMicrocoded, [SKXPort0156]> { let Latency = 100; } // def WriteMicrocoded : SchedWrite;
 
 // Fence instructions.
-def : WriteRes<WriteFence,  [SKXPort23, SKXPort4]>;
+def : WriteRes<WriteFence,  [SKXPort23, SKXPort4]> { let NumMicroOps = 2; let ReleaseAtCycles = [1,1]; }
 
 // Load/store MXCSR.
 def : WriteRes<WriteLDMXCSR, [SKXPort0,SKXPort23,SKXPort0156]> { let Latency = 7; let NumMicroOps = 3; let ReleaseAtCycles = [1,1,1]; }
@@ -757,13 +757,6 @@ def SKXWriteResGroup20 : SchedWriteRes<[SKXPort6,SKXPort0156]> {
 }
 def: InstRW<[SKXWriteResGroup20], (instregex "CLFLUSH")>;
 
-def SKXWriteResGroup21 : SchedWriteRes<[SKXPort237,SKXPort0156]> {
-  let Latency = 2;
-  let NumMicroOps = 2;
-  let ReleaseAtCycles = [1,1];
-}
-def: InstRW<[SKXWriteResGroup21], (instrs SFENCE)>;
-
 def SKXWriteResGroup23 : SchedWriteRes<[SKXPort06,SKXPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
@@ -894,13 +887,6 @@ def SKXWriteResGroup42 : SchedWriteRes<[SKXPort6,SKXPort0156]> {
 }
 def: InstRW<[SKXWriteResGroup42], (instregex "CLD")>;
 
-def SKXWriteResGroup43 : SchedWriteRes<[SKXPort237,SKXPort0156]> {
-  let Latency = 3;
-  let NumMicroOps = 3;
-  let ReleaseAtCycles = [1,2];
-}
-def: InstRW<[SKXWriteResGroup43], (instrs MFENCE)>;
-
 def SKXWriteResGroup44 : SchedWriteRes<[SKXPort06,SKXPort0156]> {
   let Latency = 2;
   let NumMicroOps = 3;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 179e29e40614e..6f5d41e99e2c8 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1584,7 +1584,7 @@ InstructionCost X86TTIImpl::getShuffleCost(
   if (Kind == TTI::SK_ExtractSubvector && LT.second.isVector()) {
     int NumElts = LT.second.getVectorNumElements();
     if ((Index % NumElts) == 0)
-      return 0;
+      return TTI::TCC_Free;
     std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
     if (SubLT.second.isVector()) {
       int NumSubElts = SubLT.second.getVectorNumElements();
@@ -1630,14 +1630,19 @@ InstructionCost X86TTIImpl::getShuffleCost(
 
   // Subvector insertions are cheap if the subvectors are aligned.
   // Note that in general, the insertion starting at the beginning of a vector
-  // isn't free, because we need to preserve the rest of the wide vector.
+  // isn't free, because we need to preserve the rest of the wide vector,
+  // but if the destination vector legalizes to the same width as the subvector
+  // then the insertion will simplify to a (free) register copy.
   if (Kind == TTI::SK_InsertSubvector && LT.second.isVector()) {
     int NumElts = LT.second.getVectorNumElements();
     std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
     if (SubLT.second.isVector()) {
       int NumSubElts = SubLT.second.getVectorNumElements();
+      bool MatchingTypes =
+          NumElts == NumSubElts &&
+          (SubTp->getElementCount().getKnownMinValue() % NumSubElts) == 0;
       if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
-        return SubLT.first;
+        return MatchingTypes ? TTI::TCC_Free : SubLT.first;
     }
 
     // If the insertion isn't aligned, treat it like a 2-op shuffle.
@@ -1775,7 +1780,7 @@ InstructionCost X86TTIImpl::getShuffleCost(
   }
 
   // For 2-input shuffles, we must account for splitting the 2 inputs into many.
-  if (Kind == TTI::SK_PermuteTwoSrc && LT.first != 1) {
+  if (Kind == TTI::SK_PermuteTwoSrc && !IsInLaneShuffle && LT.first != 1) {
     // We assume that source and destination have the same vector type.
     InstructionCost NumOfDests = LT.first;
     InstructionCost NumOfShufflesPerDest = LT.first * 2 - 1;
@@ -4763,7 +4768,7 @@ InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 
     // This type is legalized to a scalar type.
     if (!LT.second.isVector())
-      return 0;
+      return TTI::TCC_Free;
 
     // The type may be split. Normalize the index to the new type.
     unsigned SizeInBits = LT.second.getSizeInBits();
@@ -4854,10 +4859,9 @@ InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
          RegisterFileMoveCost;
 }
 
-InstructionCost
-X86TTIImpl::getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
-                                     bool Insert, bool Extract,
-                                     TTI::TargetCostKind CostKind) {
+InstructionCost X86TTIImpl::getScalarizationOverhead(
+    VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
   assert(DemandedElts.getBitWidth() ==
              cast<FixedVectorType>(Ty)->getNumElements() &&
          "Vector size mismatch");
@@ -5148,7 +5152,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // Type legalization can't handle structs
   if (TLI->getValueType(DL, Src, true) == MVT::Other)
     return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
-                                  CostKind);
+                                  CostKind, OpInfo, I);
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Src);
@@ -5160,7 +5164,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // Add a cost for constant load to vector.
   if (Opcode == Instruction::Store && OpInfo.isConstant())
     Cost += getMemoryOpCost(Instruction::Load, Src, DL.getABITypeAlign(Src),
-                            /*AddressSpace=*/0, CostKind);
+                            /*AddressSpace=*/0, CostKind, OpInfo);
 
   // Handle the simple case of non-vectors.
   // NOTE: this assumes that legalization never creates vector from scalars!
@@ -5190,7 +5194,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   if (XMMBits % EltTyBits != 0)
     // Vector size must be a multiple of the element size. I.e. no padding.
     return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
-                                  CostKind);
+                                  CostKind, OpInfo, I);
   const int NumEltPerXMM = XMMBits / EltTyBits;
 
   auto *XMMVecTy = FixedVectorType::get(EltTy, NumEltPerXMM);
@@ -5201,7 +5205,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
     if ((8 * CurrOpSizeBytes) % EltTyBits != 0)
       // Vector size must be a multiple of the element size. I.e. no padding.
       return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
-                                    CostKind);
+                                    CostKind, OpInfo, I);
     int CurrNumEltPerOp = (8 * CurrOpSizeBytes) / EltTyBits;
 
     assert(CurrOpSizeBytes > 0 && CurrNumEltPerOp > 0 && "How'd we get here?");
@@ -5238,6 +5242,23 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
           CurrOpSizeBytes != 1)
         break; // Try smalled vector size.
 
+      // This isn't exactly right. We're using slow unaligned 32-byte accesses
+      // as a proxy for a double-pumped AVX memory interface such as on
+      // Sandybridge.
+      // Sub-32-bit loads/stores will be slower either with PINSR*/PEXTR* or
+      // will be scalarized.
+      if (CurrOpSizeBytes == 32 && ST->isUnalignedMem32Slow())
+        Cost += 2;
+      else if (CurrOpSizeBytes < 4)
+        Cost += 2;
+      else
+        Cost += 1;
+
+      // If we're loading a uniform value, then we don't need to split the load,
+      // loading just a single (widest) vector can be reused by all splits.
+      if (IsLoad && OpInfo.isUniform())
+        return Cost;
+
       bool Is0thSubVec = (NumEltDone() % LT.second.getVectorNumElements()) == 0;
 
       // If we have fully processed the previous reg, we need to replenish it.
@@ -5266,18 +5287,6 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                          !IsLoad, CostKind);
       }
 
-      // This isn't exactly right. We're using slow unaligned 32-byte accesses
-      // as a proxy for a double-pumped AVX memory interface such as on
-      // Sandybridge.
-      // Sub-32-bit loads/stores will be slower either with PINSR*/PEXTR* or
-      // will be scalarized.
-      if (CurrOpSizeBytes == 32 && ST->isUnalignedMem32Slow())
-        Cost += 2;
-      else if (CurrOpSizeBytes < 4)
-        Cost += 2;
-      else
-        Cost += 1;
-
       SubVecEltsLeft -= CurrNumEltPerOp;
       NumEltRemaining -= CurrNumEltPerOp;
       Alignment = commonAlignment(Alignment.valueOrOne(), CurrOpSizeBytes);
@@ -6026,9 +6035,9 @@ InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
                                            TTI::TargetCostKind CostKind,
                                            const Instruction *I) {
   if (CostKind != TTI::TCK_RecipThroughput)
-    return Opcode == Instruction::PHI ? 0 : 1;
+    return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
   // Branches are assumed to be predicted.
-  return 0;
+  return TTI::TCC_Free;
 }
 
 int X86TTIImpl::getGatherOverhead() const {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 36d00cee0d18b..7786616f89aa6 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -169,7 +169,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {});
   InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
                                             int VF,
                                             const APInt &DemandedDstElts,
diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp
index 75f002f418ba6..c054b26727855 100644
--- a/llvm/lib/Target/X86/X86TileConfig.cpp
+++ b/llvm/lib/Target/X86/X86TileConfig.cpp
@@ -103,7 +103,7 @@ static void collectVirtRegShapes(MachineRegisterInfo *MRI, VirtRegMap &VRM,
     unsigned Index = PhysReg - X86::TMM0;
     if (!Phys2Shapes[Index].isValid()) {
       ShapeT Shape = VRM.getShape(VirtReg);
-      Phys2Shapes[Index] = Shape;
+      Phys2Shapes[Index] = std::move(Shape);
       return;
     }
   }
@@ -118,12 +118,12 @@ static void collectVirtRegShapes(MachineRegisterInfo *MRI, VirtRegMap &VRM,
 
     if (!Phys2Shapes[Index0].isValid()) {
       ShapeT Shape0(Shape.getRow(0), Shape.getCol(0), MRI);
-      Phys2Shapes[Index0] = Shape0;
+      Phys2Shapes[Index0] = std::move(Shape0);
     }
 
     if (!Phys2Shapes[Index1].isValid()) {
       ShapeT Shape1(Shape.getRow(1), Shape.getCol(1), MRI);
-      Phys2Shapes[Index1] = Shape1;
+      Phys2Shapes[Index1] = std::move(Shape1);
     }
   }
 }
diff --git a/llvm/lib/Target/X86/X86WinEHState.cpp b/llvm/lib/Target/X86/X86WinEHState.cpp
index 85485229aaf24..b3e4c70eb57f3 100644
--- a/llvm/lib/Target/X86/X86WinEHState.cpp
+++ b/llvm/lib/Target/X86/X86WinEHState.cpp
@@ -517,11 +517,28 @@ int WinEHStatePass::getBaseStateForBB(
   return BaseState;
 }
 
+static bool isIntrinsic(const CallBase &Call, Intrinsic::ID ID) {
+  const Function *CF = Call.getCalledFunction();
+  return CF && CF->isIntrinsic() && CF->getIntrinsicID() == ID;
+}
+
+static bool isSehScopeEnd(const CallBase &Call) {
+  return isIntrinsic(Call, Intrinsic::seh_scope_end);
+}
+
+static bool isSehScopeBegin(const CallBase &Call) {
+  return isIntrinsic(Call, Intrinsic::seh_scope_begin);
+}
+
 // Calculate the state a call-site is in.
 int WinEHStatePass::getStateForCall(
     DenseMap<BasicBlock *, ColorVector> &BlockColors, WinEHFuncInfo &FuncInfo,
     CallBase &Call) {
   if (auto *II = dyn_cast<InvokeInst>(&Call)) {
+    if (isSehScopeEnd(*II)) {
+      return getBaseStateForBB(BlockColors, FuncInfo, II->getNormalDest());
+    }
+
     // Look up the state number of the EH pad this unwinds to.
     assert(FuncInfo.InvokeStateMap.count(II) && "invoke has no state!");
     return FuncInfo.InvokeStateMap[II];
@@ -610,6 +627,10 @@ static int getSuccState(DenseMap<BasicBlock *, int> &InitialStates, Function &F,
 
 bool WinEHStatePass::isStateStoreNeeded(EHPersonality Personality,
                                         CallBase &Call) {
+  if (isSehScopeBegin(Call) || isSehScopeEnd(Call)) {
+    return true;
+  }
+
   // If the function touches memory, it needs a state store.
   if (isAsynchronousEHPersonality(Personality))
     return !Call.doesNotAccessMemory();
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index 45ecc4f24c2af..fe5ab0fabefa6 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -48,6 +48,19 @@ std::optional<AArch64::ArchInfo> AArch64::ArchInfo::findBySubArch(StringRef SubA
   return {};
 }
 
+unsigned AArch64::getFMVPriority(ArrayRef<StringRef> Features) {
+  constexpr unsigned MaxFMVPriority = 1000;
+  unsigned Priority = 0;
+  unsigned NumFeatures = 0;
+  for (StringRef Feature : Features) {
+    if (auto Ext = parseFMVExtension(Feature)) {
+      Priority = std::max(Priority, Ext->Priority);
+      NumFeatures++;
+    }
+  }
+  return Priority + MaxFMVPriority * NumFeatures;
+}
+
 uint64_t AArch64::getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs) {
   uint64_t FeaturesMask = 0;
   for (const StringRef &FeatureStr : FeatureStrs) {
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 05d97e03a67c9..b5119c8e2ac50 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -280,8 +280,9 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
 
   if (Implementer == "0x46") { // Fujitsu Ltd.
     return StringSwitch<const char *>(Part)
-      .Case("0x001", "a64fx")
-      .Default("generic");
+        .Case("0x001", "a64fx")
+        .Case("0x003", "fujitsu-monaka")
+        .Default("generic");
   }
 
   if (Implementer == "0x4e") { // NVIDIA Corporation
@@ -808,16 +809,31 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
     // Alderlake:
     case 0x97:
     case 0x9a:
+      CPU = "alderlake";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Gracemont
     case 0xbe:
+      CPU = "gracemont";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Raptorlake:
     case 0xb7:
     case 0xba:
     case 0xbf:
+      CPU = "raptorlake";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_ALDERLAKE;
+      break;
+
     // Meteorlake:
     case 0xaa:
     case 0xac:
-      CPU = "alderlake";
+      CPU = "meteorlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ALDERLAKE;
       break;
@@ -833,9 +849,14 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
 
     // Arrowlake S:
     case 0xc6:
+      CPU = "arrowlake-s";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_ARROWLAKE_S;
+      break;
+
     // Lunarlake:
     case 0xbd:
-      CPU = "arrowlake-s";
+      CPU = "lunarlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ARROWLAKE_S;
       break;
@@ -871,6 +892,11 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
 
     // Emerald Rapids:
     case 0xcf:
+      CPU = "emeraldrapids";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_SAPPHIRERAPIDS;
+      break;
+
     // Sapphire Rapids:
     case 0x8f:
       CPU = "sapphirerapids";
@@ -2027,11 +2053,11 @@ const StringMap<bool> sys::getHostCPUFeatures() {
   Features["frecipe"] = cpucfg2 & (1U << 25); // CPUCFG.2.FRECIPE
   Features["div32"] = cpucfg2 & (1U << 26);   // CPUCFG.2.DIV32
   Features["lam-bh"] = cpucfg2 & (1U << 27);  // CPUCFG.2.LAM_BH
+  Features["lamcas"] = cpucfg2 & (1U << 28);  // CPUCFG.2.LAMCAS
 
   Features["ld-seq-sa"] = cpucfg3 & (1U << 23); // CPUCFG.3.LD_SEQ_SA
 
   // TODO: Need to complete.
-  // Features["lamcas"] = cpucfg2 & (1U << 28);      // CPUCFG.2.LAMCAS
   // Features["llacq-screl"] = cpucfg2 & (1U << 29); // CPUCFG.2.LLACQ_SCREL
   // Features["scq"] = cpucfg2 & (1U << 30);         // CPUCFG.2.SCQ
   return Features;
diff --git a/llvm/lib/TargetParser/LoongArchTargetParser.cpp b/llvm/lib/TargetParser/LoongArchTargetParser.cpp
index 8e7681d526cef..c8a07c32247cd 100644
--- a/llvm/lib/TargetParser/LoongArchTargetParser.cpp
+++ b/llvm/lib/TargetParser/LoongArchTargetParser.cpp
@@ -53,6 +53,7 @@ bool LoongArch::getArchFeatures(StringRef Arch,
     if (Arch == "la64v1.1") {
       Features.push_back("+frecipe");
       Features.push_back("+lam-bh");
+      Features.push_back("+lamcas");
       Features.push_back("+ld-seq-sa");
       Features.push_back("+div32");
     }
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index c1bc441fc3f63..d54b81e0d3981 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -741,6 +741,8 @@ Error RISCVISAInfo::checkDependency() {
   bool HasVector = Exts.count("zve32x") != 0;
   bool HasZvl = MinVLen != 0;
   bool HasZcmt = Exts.count("zcmt") != 0;
+  static constexpr StringLiteral XqciExts[] = {
+      {"xqcia"}, {"xqcicsr"}, {"xqcisls"}};
 
   if (HasI && HasE)
     return getIncompatibleError("i", "e");
@@ -771,6 +773,10 @@ Error RISCVISAInfo::checkDependency() {
       return getIncompatibleError("xwchc", "zcb");
   }
 
+  for (auto Ext : XqciExts)
+    if (Exts.count(Ext.str()) && (XLen != 32))
+      return getError("'" + Twine(Ext) + "'" + " is only supported for 'rv32'");
+
   return Error::success();
 }
 
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index b5b561797f75a..45ee2d472a11b 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -803,8 +803,7 @@ static bool foldConsecutiveLoads(Instruction &I, const DataLayout &DL,
     APInt Offset1(DL.getIndexTypeSizeInBits(Load1Ptr->getType()), 0);
     Load1Ptr = Load1Ptr->stripAndAccumulateConstantOffsets(
         DL, Offset1, /* AllowNonInbounds */ true);
-    Load1Ptr = Builder.CreatePtrAdd(Load1Ptr,
-                                    Builder.getInt32(Offset1.getZExtValue()));
+    Load1Ptr = Builder.CreatePtrAdd(Load1Ptr, Builder.getInt(Offset1));
   }
   // Generate wider load.
   NewLoad = Builder.CreateAlignedLoad(WiderType, Load1Ptr, LI1->getAlign(),
diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h
new file mode 100644
index 0000000000000..d1887980fb3bc
--- /dev/null
+++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h
@@ -0,0 +1,155 @@
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Helper class for splitting a coroutine into separate functions. For example
+// the returned-continuation coroutine is split into separate continuation
+// functions.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H
+#define LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H
+
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Transforms/Coroutines/ABI.h"
+#include "llvm/Transforms/Coroutines/CoroInstr.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+
+namespace llvm {
+
+namespace coro {
+
+enum class CloneKind {
+  /// The shared resume function for a switch lowering.
+  SwitchResume,
+
+  /// The shared unwind function for a switch lowering.
+  SwitchUnwind,
+
+  /// The shared cleanup function for a switch lowering.
+  SwitchCleanup,
+
+  /// An individual continuation function.
+  Continuation,
+
+  /// An async resume function.
+  Async,
+};
+
+class BaseCloner {
+protected:
+  Function &OrigF;
+  const Twine &Suffix;
+  coro::Shape &Shape;
+  CloneKind FKind;
+  IRBuilder<> Builder;
+  TargetTransformInfo &TTI;
+
+  ValueToValueMapTy VMap;
+  Function *NewF = nullptr;
+  Value *NewFramePtr = nullptr;
+
+  /// The active suspend instruction; meaningful only for continuation and async
+  /// ABIs.
+  AnyCoroSuspendInst *ActiveSuspend = nullptr;
+
+  /// Create a cloner for a continuation lowering.
+  BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
+             Function *NewF, AnyCoroSuspendInst *ActiveSuspend,
+             TargetTransformInfo &TTI)
+      : OrigF(OrigF), Suffix(Suffix), Shape(Shape),
+        FKind(Shape.ABI == ABI::Async ? CloneKind::Async
+                                      : CloneKind::Continuation),
+        Builder(OrigF.getContext()), TTI(TTI), NewF(NewF),
+        ActiveSuspend(ActiveSuspend) {
+    assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce ||
+           Shape.ABI == ABI::Async);
+    assert(NewF && "need existing function for continuation");
+    assert(ActiveSuspend && "need active suspend point for continuation");
+  }
+
+public:
+  BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
+             CloneKind FKind, TargetTransformInfo &TTI)
+      : OrigF(OrigF), Suffix(Suffix), Shape(Shape), FKind(FKind),
+        Builder(OrigF.getContext()), TTI(TTI) {}
+
+  virtual ~BaseCloner() {}
+
+  /// Create a clone for a continuation lowering.
+  static Function *createClone(Function &OrigF, const Twine &Suffix,
+                               coro::Shape &Shape, Function *NewF,
+                               AnyCoroSuspendInst *ActiveSuspend,
+                               TargetTransformInfo &TTI) {
+    assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce ||
+           Shape.ABI == ABI::Async);
+    TimeTraceScope FunctionScope("BaseCloner");
+
+    BaseCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI);
+    Cloner.create();
+    return Cloner.getFunction();
+  }
+
+  Function *getFunction() const {
+    assert(NewF != nullptr && "declaration not yet set");
+    return NewF;
+  }
+
+  virtual void create();
+
+protected:
+  bool isSwitchDestroyFunction() {
+    switch (FKind) {
+    case CloneKind::Async:
+    case CloneKind::Continuation:
+    case CloneKind::SwitchResume:
+      return false;
+    case CloneKind::SwitchUnwind:
+    case CloneKind::SwitchCleanup:
+      return true;
+    }
+    llvm_unreachable("Unknown ClonerKind enum");
+  }
+
+  void replaceEntryBlock();
+  Value *deriveNewFramePointer();
+  void replaceRetconOrAsyncSuspendUses();
+  void replaceCoroSuspends();
+  void replaceCoroEnds();
+  void replaceSwiftErrorOps();
+  void salvageDebugInfo();
+  void handleFinalSuspend();
+};
+
+class SwitchCloner : public BaseCloner {
+protected:
+  /// Create a cloner for a switch lowering.
+  SwitchCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
+               CloneKind FKind, TargetTransformInfo &TTI)
+      : BaseCloner(OrigF, Suffix, Shape, FKind, TTI) {}
+
+  void create() override;
+
+public:
+  /// Create a clone for a switch lowering.
+  static Function *createClone(Function &OrigF, const Twine &Suffix,
+                               coro::Shape &Shape, CloneKind FKind,
+                               TargetTransformInfo &TTI) {
+    assert(Shape.ABI == ABI::Switch);
+    TimeTraceScope FunctionScope("SwitchCloner");
+
+    SwitchCloner Cloner(OrigF, Suffix, Shape, FKind, TTI);
+    Cloner.create();
+    return Cloner.getFunction();
+  }
+};
+
+} // end namespace coro
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 441a83310c632..d3732fec603f6 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1133,7 +1133,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
           bool AllowUnresolved = false;
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
-          // processed by coro::salvageDebugInfo() by CoroCloner.
+          // processed by coro::salvageDebugInfo() by the Cloner.
           if (UseNewDbgInfoFormat) {
             DbgVariableRecord *NewDVR = new DbgVariableRecord(
                 ValueAsMetadata::get(CurrentReload), DDI->getVariable(),
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 8a5bae9f6f0d4..3808147fc2600 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -19,6 +19,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "CoroCloner.h"
 #include "CoroInternal.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PriorityWorklist.h"
@@ -44,10 +45,8 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Dominators.h"
-#include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
@@ -61,17 +60,13 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Coroutines/ABI.h"
-#include "llvm/Transforms/Coroutines/CoroInstr.h"
 #include "llvm/Transforms/Coroutines/MaterializationUtils.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Transforms/Utils/ValueMapper.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -82,122 +77,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "coro-split"
 
-namespace {
-
-/// A little helper class for building
-class CoroCloner {
-public:
-  enum class Kind {
-    /// The shared resume function for a switch lowering.
-    SwitchResume,
-
-    /// The shared unwind function for a switch lowering.
-    SwitchUnwind,
-
-    /// The shared cleanup function for a switch lowering.
-    SwitchCleanup,
-
-    /// An individual continuation function.
-    Continuation,
-
-    /// An async resume function.
-    Async,
-  };
-
-private:
-  Function &OrigF;
-  Function *NewF;
-  const Twine &Suffix;
-  coro::Shape &Shape;
-  Kind FKind;
-  ValueToValueMapTy VMap;
-  IRBuilder<> Builder;
-  Value *NewFramePtr = nullptr;
-
-  /// The active suspend instruction; meaningful only for continuation and async
-  /// ABIs.
-  AnyCoroSuspendInst *ActiveSuspend = nullptr;
-
-  TargetTransformInfo &TTI;
-
-  /// Create a cloner for a switch lowering.
-  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
-             Kind FKind, TargetTransformInfo &TTI)
-      : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), FKind(FKind),
-        Builder(OrigF.getContext()), TTI(TTI) {
-    assert(Shape.ABI == coro::ABI::Switch);
-  }
-
-  /// Create a cloner for a continuation lowering.
-  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
-             Function *NewF, AnyCoroSuspendInst *ActiveSuspend,
-             TargetTransformInfo &TTI)
-      : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
-        FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
-        Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend), TTI(TTI) {
-    assert(Shape.ABI == coro::ABI::Retcon ||
-           Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
-    assert(NewF && "need existing function for continuation");
-    assert(ActiveSuspend && "need active suspend point for continuation");
-  }
-
-public:
-  /// Create a clone for a switch lowering.
-  static Function *createClone(Function &OrigF, const Twine &Suffix,
-                               coro::Shape &Shape, Kind FKind,
-                               TargetTransformInfo &TTI) {
-    TimeTraceScope FunctionScope("CoroCloner");
-
-    CoroCloner Cloner(OrigF, Suffix, Shape, FKind, TTI);
-    Cloner.create();
-    return Cloner.getFunction();
-  }
-
-  /// Create a clone for a continuation lowering.
-  static Function *createClone(Function &OrigF, const Twine &Suffix,
-                               coro::Shape &Shape, Function *NewF,
-                               AnyCoroSuspendInst *ActiveSuspend,
-                               TargetTransformInfo &TTI) {
-    TimeTraceScope FunctionScope("CoroCloner");
-
-    CoroCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI);
-    Cloner.create();
-    return Cloner.getFunction();
-  }
-
-  Function *getFunction() const {
-    assert(NewF != nullptr && "declaration not yet set");
-    return NewF;
-  }
-
-  void create();
-
-private:
-  bool isSwitchDestroyFunction() {
-    switch (FKind) {
-    case Kind::Async:
-    case Kind::Continuation:
-    case Kind::SwitchResume:
-      return false;
-    case Kind::SwitchUnwind:
-    case Kind::SwitchCleanup:
-      return true;
-    }
-    llvm_unreachable("Unknown CoroCloner::Kind enum");
-  }
-
-  void replaceEntryBlock();
-  Value *deriveNewFramePointer();
-  void replaceRetconOrAsyncSuspendUses();
-  void replaceCoroSuspends();
-  void replaceCoroEnds();
-  void replaceSwiftErrorOps();
-  void salvageDebugInfo();
-  void handleFinalSuspend();
-};
-
-} // end anonymous namespace
-
 // FIXME:
 // Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape
 // and it is known that other transformations, for example, sanitizers
@@ -523,7 +402,7 @@ static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
 // is possible since the coroutine is considered suspended at the final suspend
 // point if promise.unhandled_exception() exits via an exception), we can
 // remove the last case.
-void CoroCloner::handleFinalSuspend() {
+void coro::BaseCloner::handleFinalSuspend() {
   assert(Shape.ABI == coro::ABI::Switch &&
          Shape.SwitchLowering.HasFinalSuspend);
 
@@ -587,7 +466,7 @@ static Function *createCloneDeclaration(Function &OrigF, coro::Shape &Shape,
 /// arguments to the continuation function.
 ///
 /// This assumes that the builder has a meaningful insertion point.
-void CoroCloner::replaceRetconOrAsyncSuspendUses() {
+void coro::BaseCloner::replaceRetconOrAsyncSuspendUses() {
   assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
          Shape.ABI == coro::ABI::Async);
 
@@ -635,7 +514,7 @@ void CoroCloner::replaceRetconOrAsyncSuspendUses() {
   NewS->replaceAllUsesWith(Aggr);
 }
 
-void CoroCloner::replaceCoroSuspends() {
+void coro::BaseCloner::replaceCoroSuspends() {
   Value *SuspendResult;
 
   switch (Shape.ABI) {
@@ -672,7 +551,7 @@ void CoroCloner::replaceCoroSuspends() {
   }
 }
 
-void CoroCloner::replaceCoroEnds() {
+void coro::BaseCloner::replaceCoroEnds() {
   for (AnyCoroEndInst *CE : Shape.CoroEnds) {
     // We use a null call graph because there's no call graph node for
     // the cloned function yet.  We'll just be rebuilding that later.
@@ -751,11 +630,11 @@ collectDbgVariableIntrinsics(Function &F) {
   return {Intrinsics, DbgVariableRecords};
 }
 
-void CoroCloner::replaceSwiftErrorOps() {
+void coro::BaseCloner::replaceSwiftErrorOps() {
   ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
 }
 
-void CoroCloner::salvageDebugInfo() {
+void coro::BaseCloner::salvageDebugInfo() {
   auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
@@ -792,7 +671,7 @@ void CoroCloner::salvageDebugInfo() {
   for_each(DbgVariableRecords, RemoveOne);
 }
 
-void CoroCloner::replaceEntryBlock() {
+void coro::BaseCloner::replaceEntryBlock() {
   // In the original function, the AllocaSpillBlock is a block immediately
   // following the allocation of the frame object which defines GEPs for
   // all the allocas that have been moved into the frame, and it ends by
@@ -860,7 +739,7 @@ void CoroCloner::replaceEntryBlock() {
 }
 
 /// Derive the value of the new frame pointer.
-Value *CoroCloner::deriveNewFramePointer() {
+Value *coro::BaseCloner::deriveNewFramePointer() {
   // Builder should be inserting to the front of the new entry block.
 
   switch (Shape.ABI) {
@@ -984,12 +863,8 @@ static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
 
 /// Clone the body of the original function into a resume function of
 /// some sort.
-void CoroCloner::create() {
-  // Create the new function if we don't already have one.
-  if (!NewF) {
-    NewF = createCloneDeclaration(OrigF, Shape, Suffix,
-                                  OrigF.getParent()->end(), ActiveSuspend);
-  }
+void coro::BaseCloner::create() {
+  assert(NewF);
 
   // Replace all args with dummy instructions. If an argument is the old frame
   // pointer, the dummy will be replaced by the new frame pointer once it is
@@ -1213,12 +1088,20 @@ void CoroCloner::create() {
 
   // Salvage debug info that points into the coroutine frame.
   salvageDebugInfo();
+}
+
+void coro::SwitchCloner::create() {
+  // Create a new function matching the original type
+  NewF = createCloneDeclaration(OrigF, Shape, Suffix, OrigF.getParent()->end(),
+                                ActiveSuspend);
+
+  // Clone the function
+  coro::BaseCloner::create();
 
   // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
   // to suppress deallocation code.
-  if (Shape.ABI == coro::ABI::Switch)
-    coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
-                          /*Elide=*/FKind == CoroCloner::Kind::SwitchCleanup);
+  coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
+                        /*Elide=*/FKind == coro::CloneKind::SwitchCleanup);
 }
 
 static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) {
@@ -1495,12 +1378,12 @@ struct SwitchCoroutineSplitter {
     // setting new entry block and replacing coro.suspend an appropriate value
     // to force resume or cleanup pass for every suspend point.
     createResumeEntryBlock(F, Shape);
-    auto *ResumeClone = CoroCloner::createClone(
-        F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
-    auto *DestroyClone = CoroCloner::createClone(
-        F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
-    auto *CleanupClone = CoroCloner::createClone(
-        F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
+    auto *ResumeClone = coro::SwitchCloner::createClone(
+        F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI);
+    auto *DestroyClone = coro::SwitchCloner::createClone(
+        F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI);
+    auto *CleanupClone = coro::SwitchCloner::createClone(
+        F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI);
 
     postSplitCleanup(*ResumeClone);
     postSplitCleanup(*DestroyClone);
@@ -1889,8 +1772,8 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape,
     auto *Suspend = CS;
     auto *Clone = Clones[Idx];
 
-    CoroCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, Suspend,
-                            TTI);
+    coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone,
+                                  Suspend, TTI);
   }
 }
 
@@ -2020,8 +1903,8 @@ void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape,
     auto Suspend = CS;
     auto Clone = Clones[Idx];
 
-    CoroCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, Suspend,
-                            TTI);
+    coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone,
+                                  Suspend, TTI);
   }
 }
 
diff --git a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
index 1f787c733079e..0baa34d50abf3 100644
--- a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
+++ b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
@@ -34,7 +34,8 @@ bool AlwaysInlineImpl(
     Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<AAResults &(Function &)> GetAAR,
-    function_ref<BlockFrequencyInfo &(Function &)> GetBFI) {
+    function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
+    function_ref<BlockFrequencyInfo *(Function &)> GetCachedBFI) {
   SmallSetVector<CallBase *, 16> Calls;
   bool Changed = false;
   SmallVector<Function *, 16> InlinedComdatFunctions;
@@ -61,9 +62,11 @@ bool AlwaysInlineImpl(
       DebugLoc DLoc = CB->getDebugLoc();
       BasicBlock *Block = CB->getParent();
 
-      InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
-                             GetBFI ? &GetBFI(*Caller) : nullptr,
-                             GetBFI ? &GetBFI(F) : nullptr);
+      // Only update CallerBFI if already available. The CallerBFI update
+      // requires CalleeBFI.
+      BlockFrequencyInfo *CallerBFI = GetCachedBFI(*Caller);
+      InlineFunctionInfo IFI(GetAssumptionCache, &PSI, CallerBFI,
+                             CallerBFI ? &GetBFI(F) : nullptr);
 
       InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
                                         &GetAAR(F), InsertLifetime);
@@ -133,9 +136,12 @@ struct AlwaysInlinerLegacyPass : public ModulePass {
     auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
       return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
     };
+    auto GetCachedBFI = [](Function &) -> BlockFrequencyInfo * {
+      return nullptr;
+    };
 
     return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR,
-                            /*GetBFI*/ nullptr);
+                            /*GetBFI=*/nullptr, GetCachedBFI);
   }
 
   static char ID; // Pass identification, replacement for typeid
@@ -172,13 +178,16 @@ PreservedAnalyses AlwaysInlinerPass::run(Module &M,
   auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
     return FAM.getResult<BlockFrequencyAnalysis>(F);
   };
+  auto GetCachedBFI = [&](Function &F) -> BlockFrequencyInfo * {
+    return FAM.getCachedResult<BlockFrequencyAnalysis>(F);
+  };
   auto GetAAR = [&](Function &F) -> AAResults & {
     return FAM.getResult<AAManager>(F);
   };
   auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
 
   bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache,
-                                  GetAAR, GetBFI);
+                                  GetAAR, GetBFI, GetCachedBFI);
 
   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index fee27f72f208b..fde43bb354e83 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -430,10 +430,18 @@ class GlobalsImporter final {
         // than as part of the logic deciding which functions to import (i.e.
         // based on profile information). Should we decide to handle them here,
         // we can refactor accordingly at that time.
-        if (!GVS || !Index.canImportGlobalVar(GVS, /* AnalyzeRefs */ true) ||
+        bool CanImportDecl = false;
+        if (!GVS ||
             shouldSkipLocalInAnotherModule(GVS, VI.getSummaryList().size(),
-                                           Summary.modulePath()))
+                                           Summary.modulePath()) ||
+            !Index.canImportGlobalVar(GVS, /* AnalyzeRefs */ true,
+                                      CanImportDecl)) {
+          if (ImportDeclaration && CanImportDecl)
+            ImportList.maybeAddDeclaration(RefSummary->modulePath(),
+                                           VI.getGUID());
+
           continue;
+        }
 
         // If there isn't an entry for GUID, insert <GUID, Definition> pair.
         // Otherwise, definition should take precedence over declaration.
@@ -1540,8 +1548,7 @@ void llvm::gatherImportedSummariesForModule(
     auto &SummariesForIndex =
         LookupOrCreate(ModuleToSummariesForIndex, FromModule);
 
-    const auto &DefinedGVSummaries =
-        ModuleToDefinedGVSummaries.lookup(FromModule);
+    const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(FromModule);
     const auto &DS = DefinedGVSummaries.find(GUID);
     assert(DS != DefinedGVSummaries.end() &&
            "Expected a defined summary for imported global value");
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 3381b5f77683b..16a80e9ebbeaa 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -946,11 +946,10 @@ OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI,
 
   // If there is a comparison against null, we will insert a global bool to
   // keep track of whether the global was initialized yet or not.
-  GlobalVariable *InitBool =
-    new GlobalVariable(Type::getInt1Ty(GV->getContext()), false,
-                       GlobalValue::InternalLinkage,
-                       ConstantInt::getFalse(GV->getContext()),
-                       GV->getName()+".init", GV->getThreadLocalMode());
+  GlobalVariable *InitBool = new GlobalVariable(
+      Type::getInt1Ty(GV->getContext()), false, GlobalValue::InternalLinkage,
+      ConstantInt::getFalse(GV->getContext()), GV->getName() + ".init",
+      GV->getThreadLocalMode(), GV->getAddressSpace());
   bool InitBoolUsed = false;
 
   // Loop over all instruction uses of GV, processing them in turn.
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 87d2432803062..e3caefe70311b 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -2090,20 +2090,19 @@ bool LowerTypeTestsModule::lower() {
   };
   MapVector<StringRef, ExportedFunctionInfo> ExportedFunctions;
   if (ExportSummary) {
-    // A set of all functions that are address taken by a live global object.
-    DenseSet<GlobalValue::GUID> AddressTaken;
-    for (auto &I : *ExportSummary)
-      for (auto &GVS : I.second.SummaryList)
-        if (GVS->isLive())
-          for (const auto &Ref : GVS->refs()) {
-            AddressTaken.insert(Ref.getGUID());
-            for (auto &RefGVS : Ref.getSummaryList())
-              if (auto Alias = dyn_cast<AliasSummary>(RefGVS.get()))
-                AddressTaken.insert(Alias->getAliaseeGUID());
-          }
-
     NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions");
     if (CfiFunctionsMD) {
+      // A set of all functions that are address taken by a live global object.
+      DenseSet<GlobalValue::GUID> AddressTaken;
+      for (auto &I : *ExportSummary)
+        for (auto &GVS : I.second.SummaryList)
+          if (GVS->isLive())
+            for (const auto &Ref : GVS->refs()) {
+              AddressTaken.insert(Ref.getGUID());
+              for (auto &RefGVS : Ref.getSummaryList())
+                if (auto Alias = dyn_cast<AliasSummary>(RefGVS.get()))
+                  AddressTaken.insert(Alias->getAliaseeGUID());
+            }
       for (auto *FuncMD : CfiFunctionsMD->operands()) {
         assert(FuncMD->getNumOperands() >= 2);
         StringRef FunctionName =
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index b7d0c3d741deb..e8508416f5427 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -196,7 +196,10 @@ class MergeFunctions {
   MergeFunctions() : FnTree(FunctionNodeCmp(&GlobalNumbers)) {
   }
 
-  bool runOnModule(Module &M);
+  template <typename FuncContainer> bool run(FuncContainer &Functions);
+  DenseMap<Function *, Function *> runOnFunctions(ArrayRef<Function *> F);
+
+  SmallPtrSet<GlobalValue *, 4> &getUsed();
 
 private:
   // The function comparison operator is provided here so that FunctionNodes do
@@ -297,17 +300,36 @@ class MergeFunctions {
   // dangling iterators into FnTree. The invariant that preserves this is that
   // there is exactly one mapping F -> FN for each FunctionNode FN in FnTree.
   DenseMap<AssertingVH<Function>, FnTreeType::iterator> FNodesInTree;
+
+  /// Deleted-New functions mapping
+  DenseMap<Function *, Function *> DelToNewMap;
 };
 } // end anonymous namespace
 
 PreservedAnalyses MergeFunctionsPass::run(Module &M,
                                           ModuleAnalysisManager &AM) {
-  MergeFunctions MF;
-  if (!MF.runOnModule(M))
+  if (!MergeFunctionsPass::runOnModule(M))
     return PreservedAnalyses::all();
   return PreservedAnalyses::none();
 }
 
+SmallPtrSet<GlobalValue *, 4> &MergeFunctions::getUsed() { return Used; }
+
+bool MergeFunctionsPass::runOnModule(Module &M) {
+  MergeFunctions MF;
+  SmallVector<GlobalValue *, 4> UsedV;
+  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/false);
+  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/true);
+  MF.getUsed().insert(UsedV.begin(), UsedV.end());
+  return MF.run(M);
+}
+
+DenseMap<Function *, Function *>
+MergeFunctionsPass::runOnFunctions(ArrayRef<Function *> F) {
+  MergeFunctions MF;
+  return MF.runOnFunctions(F);
+}
+
 #ifndef NDEBUG
 bool MergeFunctions::doFunctionalCheck(std::vector<WeakTrackingVH> &Worklist) {
   if (const unsigned Max = NumFunctionsForVerificationCheck) {
@@ -409,20 +431,19 @@ static bool isEligibleForMerging(Function &F) {
          !hasDistinctMetadataIntrinsic(F);
 }
 
-bool MergeFunctions::runOnModule(Module &M) {
-  bool Changed = false;
+inline Function *asPtr(Function *Fn) { return Fn; }
+inline Function *asPtr(Function &Fn) { return &Fn; }
 
-  SmallVector<GlobalValue *, 4> UsedV;
-  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/false);
-  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/true);
-  Used.insert(UsedV.begin(), UsedV.end());
+template <typename FuncContainer> bool MergeFunctions::run(FuncContainer &M) {
+  bool Changed = false;
 
   // All functions in the module, ordered by hash. Functions with a unique
   // hash value are easily eliminated.
   std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
-  for (Function &Func : M) {
-    if (isEligibleForMerging(Func)) {
-      HashedFuncs.push_back({StructuralHash(Func), &Func});
+  for (auto &Func : M) {
+    Function *FuncPtr = asPtr(Func);
+    if (isEligibleForMerging(*FuncPtr)) {
+      HashedFuncs.push_back({StructuralHash(*FuncPtr), FuncPtr});
     }
   }
 
@@ -433,7 +454,7 @@ bool MergeFunctions::runOnModule(Module &M) {
     // If the hash value matches the previous value or the next one, we must
     // consider merging it. Otherwise it is dropped and never considered again.
     if ((I != S && std::prev(I)->first == I->first) ||
-        (std::next(I) != IE && std::next(I)->first == I->first) ) {
+        (std::next(I) != IE && std::next(I)->first == I->first)) {
       Deferred.push_back(WeakTrackingVH(I->second));
     }
   }
@@ -467,9 +488,16 @@ bool MergeFunctions::runOnModule(Module &M) {
   return Changed;
 }
 
+DenseMap<Function *, Function *>
+MergeFunctions::runOnFunctions(ArrayRef<Function *> F) {
+  [[maybe_unused]] bool MergeResult = this->run(F);
+  assert(MergeResult == !DelToNewMap.empty());
+  return this->DelToNewMap;
+}
+
 // Replace direct callers of Old with New.
 void MergeFunctions::replaceDirectCallers(Function *Old, Function *New) {
-  for (Use &U : llvm::make_early_inc_range(Old->uses())) {
+  for (Use &U : make_early_inc_range(Old->uses())) {
     CallBase *CB = dyn_cast<CallBase>(U.getUser());
     if (CB && CB->isCallee(&U)) {
       // Do not copy attributes from the called function to the call-site.
@@ -768,8 +796,8 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
   ReturnInst *RI = nullptr;
   bool isSwiftTailCall = F->getCallingConv() == CallingConv::SwiftTail &&
                          G->getCallingConv() == CallingConv::SwiftTail;
-  CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail
-                                      : llvm::CallInst::TCK_Tail);
+  CI->setTailCallKind(isSwiftTailCall ? CallInst::TCK_MustTail
+                                      : CallInst::TCK_Tail);
   CI->setCallingConv(F->getCallingConv());
   CI->setAttributes(F->getAttributes());
   if (H->getReturnType()->isVoidTy()) {
@@ -1003,6 +1031,7 @@ bool MergeFunctions::insert(Function *NewFunction) {
 
   Function *DeleteF = NewFunction;
   mergeTwoFunctions(OldF.getFunc(), DeleteF);
+  this->DelToNewMap.insert({DeleteF, OldF.getFunc()});
   return true;
 }
 
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 9e25620710fc8..b40ab357670b8 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1538,26 +1538,12 @@ struct OpenMPOpt {
     // required an RPC server. If its users were all optimized out then we can
     // safely remove it.
     // TODO: This should be somewhere more common in the future.
-    if (GlobalVariable *GV = M.getNamedGlobal("__llvm_libc_rpc_client")) {
-      if (!GV->getType()->isPointerTy())
+    if (GlobalVariable *GV = M.getNamedGlobal("__llvm_rpc_client")) {
+      if (GV->getNumUses() >= 1)
         return false;
 
-      Constant *C = GV->getInitializer();
-      if (!C)
-        return false;
-
-      // Check to see if the only user of the RPC client is the external handle.
-      GlobalVariable *Client = dyn_cast<GlobalVariable>(C->stripPointerCasts());
-      if (!Client || Client->getNumUses() > 1 ||
-          Client->user_back() != GV->getInitializer())
-        return false;
-
-      Client->replaceAllUsesWith(PoisonValue::get(Client->getType()));
-      Client->eraseFromParent();
-
       GV->replaceAllUsesWith(PoisonValue::get(GV->getType()));
       GV->eraseFromParent();
-
       return true;
     }
     return false;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 6fe9693581853..fe0d88fcc6ee4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -994,6 +994,12 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
     }
   }
 
+  // umax(X, C) + -C --> usub.sat(X, C)
+  if (match(Op0, m_OneUse(m_UMax(m_Value(X), m_SpecificInt(-*C)))))
+    return replaceInstUsesWith(
+        Add, Builder.CreateBinaryIntrinsic(
+                 Intrinsic::usub_sat, X, ConstantInt::get(Add.getType(), -*C)));
+
   // Fold (add (zext (add X, -1)), 1) -> (zext X) if X is non-zero.
   // TODO: There's a general form for any constant on the outer add.
   if (C->isOne()) {
@@ -2086,28 +2092,31 @@ Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS,
 
   // To avoid duplicating the offset arithmetic, rewrite the GEP to use the
   // computed offset. This may erase the original GEP, so be sure to cache the
-  // inbounds flag before emitting the offset.
+  // nowrap flags before emitting the offset.
   // TODO: We should probably do this even if there is only one GEP.
   bool RewriteGEPs = GEP2 != nullptr;
 
   // Emit the offset of the GEP and an intptr_t.
-  bool GEP1IsInBounds = GEP1->isInBounds();
+  GEPNoWrapFlags GEP1NW = GEP1->getNoWrapFlags();
   Value *Result = EmitGEPOffset(GEP1, RewriteGEPs);
 
   // If this is a single inbounds GEP and the original sub was nuw,
   // then the final multiplication is also nuw.
   if (auto *I = dyn_cast<Instruction>(Result))
-    if (IsNUW && !GEP2 && !Swapped && GEP1IsInBounds &&
+    if (IsNUW && !GEP2 && !Swapped && GEP1NW.isInBounds() &&
         I->getOpcode() == Instruction::Mul)
       I->setHasNoUnsignedWrap();
 
   // If we have a 2nd GEP of the same base pointer, subtract the offsets.
   // If both GEPs are inbounds, then the subtract does not have signed overflow.
+  // If both GEPs are nuw and the original sub is nuw, the new sub is also nuw.
   if (GEP2) {
-    bool GEP2IsInBounds = GEP2->isInBounds();
+    GEPNoWrapFlags GEP2NW = GEP2->getNoWrapFlags();
     Value *Offset = EmitGEPOffset(GEP2, RewriteGEPs);
-    Result = Builder.CreateSub(Result, Offset, "gepdiff", /* NUW */ false,
-                               GEP1IsInBounds && GEP2IsInBounds);
+    Result = Builder.CreateSub(Result, Offset, "gepdiff",
+                               IsNUW && GEP1NW.hasNoUnsignedWrap() &&
+                                   GEP2NW.hasNoUnsignedWrap(),
+                               GEP1NW.isInBounds() && GEP2NW.isInBounds());
   }
 
   // If we have p - gep(p, ...)  then we have to negate the result.
@@ -2271,6 +2280,16 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
   if (match(Op0, m_OneUse(m_Add(m_Value(X), m_AllOnes()))))
     return BinaryOperator::CreateAdd(Builder.CreateNot(Op1), X);
 
+  // if (C1 & C2) == C2 then (X & C1) - (X & C2) -> X & (C1 ^ C2)
+  Constant *C1, *C2;
+  if (match(Op0, m_And(m_Value(X), m_ImmConstant(C1))) &&
+      match(Op1, m_And(m_Specific(X), m_ImmConstant(C2)))) {
+    Value *AndC = ConstantFoldBinaryInstruction(Instruction::And, C1, C2);
+    if (C2->isElementWiseEqual(AndC))
+      return BinaryOperator::CreateAnd(
+          X, ConstantFoldBinaryInstruction(Instruction::Xor, C1, C2));
+  }
+
   // Reassociate sub/add sequences to create more add instructions and
   // reduce dependency chains:
   // ((X - Y) + Z) - Op1 --> (X + Z) - (Y + Op1)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b4033fc2a418a..314b1f0b43e3b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -695,13 +695,17 @@ Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
                                Cmp1->getPredicate());
 
   Value *Input = Cmp0->getOperand(0);
+  Value *Cmp1Op0 = Cmp1->getOperand(0);
+  Value *Cmp1Op1 = Cmp1->getOperand(1);
   Value *RangeEnd;
-  if (Cmp1->getOperand(0) == Input) {
+  if (match(Cmp1Op0, m_SExtOrSelf(m_Specific(Input)))) {
     // For the upper range compare we have: icmp x, n
-    RangeEnd = Cmp1->getOperand(1);
-  } else if (Cmp1->getOperand(1) == Input) {
+    Input = Cmp1Op0;
+    RangeEnd = Cmp1Op1;
+  } else if (match(Cmp1Op1, m_SExtOrSelf(m_Specific(Input)))) {
     // For the upper range compare we have: icmp n, x
-    RangeEnd = Cmp1->getOperand(0);
+    Input = Cmp1Op1;
+    RangeEnd = Cmp1Op0;
     Pred1 = ICmpInst::getSwappedPredicate(Pred1);
   } else {
     return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index fd59b98c9a8ef..52cb91e66184f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -948,6 +948,11 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
     }
   }
 
+  if (DestWidth == 1 &&
+      (Trunc.hasNoUnsignedWrap() || Trunc.hasNoSignedWrap()) &&
+      isKnownNonZero(Src, SQ.getWithInstruction(&Trunc)))
+    return replaceInstUsesWith(Trunc, ConstantInt::getTrue(DestTy));
+
   bool Changed = false;
   if (!Trunc.hasNoSignedWrap() &&
       ComputeMaxSignificantBits(Src, /*Depth=*/0, &Trunc) <= DestWidth) {
@@ -1861,15 +1866,16 @@ Instruction *InstCombinerImpl::visitFPTrunc(FPTruncInst &FPT) {
   Value *X;
   Instruction *Op = dyn_cast<Instruction>(FPT.getOperand(0));
   if (Op && Op->hasOneUse()) {
-    // FIXME: The FMF should propagate from the fptrunc, not the source op.
     IRBuilder<>::FastMathFlagGuard FMFG(Builder);
-    if (isa<FPMathOperator>(Op))
-      Builder.setFastMathFlags(Op->getFastMathFlags());
+    FastMathFlags FMF = FPT.getFastMathFlags();
+    if (auto *FPMO = dyn_cast<FPMathOperator>(Op))
+      FMF &= FPMO->getFastMathFlags();
+    Builder.setFastMathFlags(FMF);
 
     if (match(Op, m_FNeg(m_Value(X)))) {
       Value *InnerTrunc = Builder.CreateFPTrunc(X, Ty);
-
-      return UnaryOperator::CreateFNegFMF(InnerTrunc, Op);
+      Value *Neg = Builder.CreateFNeg(InnerTrunc);
+      return replaceInstUsesWith(FPT, Neg);
     }
 
     // If we are truncating a select that has an extended operand, we can
@@ -2120,10 +2126,7 @@ Instruction *InstCombinerImpl::visitPtrToInt(PtrToIntInst &CI) {
         Base->getType() == Ty) {
       Value *Offset = EmitGEPOffset(GEP);
       auto *NewOp = BinaryOperator::CreateAdd(Base, Offset);
-      if (GEP->hasNoUnsignedWrap() ||
-          (GEP->hasNoUnsignedSignedWrap() &&
-           isKnownNonNegative(Offset, SQ.getWithInstruction(&CI))))
-        NewOp->setHasNoUnsignedWrap(true);
+      NewOp->setHasNoUnsignedWrap(GEP->hasNoUnsignedWrap());
       return NewOp;
     }
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index fed21db393ed2..54053c4c9e28e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -631,7 +631,7 @@ static Value *rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW,
 /// We can look through PHIs, GEPs and casts in order to determine a common base
 /// between GEPLHS and RHS.
 static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
-                                              ICmpInst::Predicate Cond,
+                                              CmpPredicate Cond,
                                               const DataLayout &DL,
                                               InstCombiner &IC) {
   // FIXME: Support vector of pointers.
@@ -675,8 +675,7 @@ static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
 /// Fold comparisons between a GEP instruction and something else. At this point
 /// we know that the GEP is on the LHS of the comparison.
 Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
-                                           ICmpInst::Predicate Cond,
-                                           Instruction &I) {
+                                           CmpPredicate Cond, Instruction &I) {
   // Don't transform signed compares of GEPs into index compares. Even if the
   // GEP is inbounds, the final add of the base pointer can have signed overflow
   // and would change the result of the icmp.
@@ -690,12 +689,32 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
   if (!isa<GetElementPtrInst>(RHS))
     RHS = RHS->stripPointerCasts();
 
+  auto CanFold = [Cond](GEPNoWrapFlags NW) {
+    if (ICmpInst::isEquality(Cond))
+      return true;
+
+    // Unsigned predicates can be folded if the GEPs have *any* nowrap flags.
+    assert(ICmpInst::isUnsigned(Cond));
+    return NW != GEPNoWrapFlags::none();
+  };
+
+  auto NewICmp = [Cond](GEPNoWrapFlags NW, Value *Op1, Value *Op2) {
+    if (!NW.hasNoUnsignedWrap()) {
+      // Convert signed to unsigned comparison.
+      return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Op1, Op2);
+    }
+
+    auto *I = new ICmpInst(Cond, Op1, Op2);
+    I->setSameSign(NW.hasNoUnsignedSignedWrap());
+    return I;
+  };
+
   Value *PtrBase = GEPLHS->getOperand(0);
-  if (PtrBase == RHS && (GEPLHS->isInBounds() || ICmpInst::isEquality(Cond))) {
+  if (PtrBase == RHS && CanFold(GEPLHS->getNoWrapFlags())) {
     // ((gep Ptr, OFFSET) cmp Ptr)   ---> (OFFSET cmp 0).
     Value *Offset = EmitGEPOffset(GEPLHS);
-    return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset,
-                        Constant::getNullValue(Offset->getType()));
+    return NewICmp(GEPLHS->getNoWrapFlags(), Offset,
+                   Constant::getNullValue(Offset->getType()));
   }
 
   if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
@@ -785,7 +804,7 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
       return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
     }
 
-    bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds();
+    GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags() & GEPRHS->getNoWrapFlags();
     if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
         GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
       // If the GEPs only differ by one index, compare it.
@@ -813,19 +832,18 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
         return replaceInstUsesWith(I, // No comparison is needed here.
           ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
 
-      else if (NumDifferences == 1 && GEPsInBounds) {
+      else if (NumDifferences == 1 && CanFold(NW)) {
         Value *LHSV = GEPLHS->getOperand(DiffOperand);
         Value *RHSV = GEPRHS->getOperand(DiffOperand);
-        // Make sure we do a signed comparison here.
-        return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV);
+        return NewICmp(NW, LHSV, RHSV);
       }
     }
 
-    if (GEPsInBounds || CmpInst::isEquality(Cond)) {
+    if (CanFold(NW)) {
       // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)  --->  (OFFSET1 cmp OFFSET2)
       Value *L = EmitGEPOffset(GEPLHS, /*RewriteGEP=*/true);
       Value *R = EmitGEPOffset(GEPRHS, /*RewriteGEP=*/true);
-      return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R);
+      return NewICmp(NW, L, R);
     }
   }
 
@@ -912,7 +930,7 @@ bool InstCombinerImpl::foldAllocaCmp(AllocaInst *Alloca) {
 
 /// Fold "icmp pred (X+C), X".
 Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C,
-                                                  ICmpInst::Predicate Pred) {
+                                                  CmpPredicate Pred) {
   // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
   // so the values can never be equal.  Similarly for all other "or equals"
   // operators.
@@ -1760,6 +1778,17 @@ Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
   if (!match(And, m_And(m_Value(X), m_APInt(C2))))
     return nullptr;
 
+  // (and X, highmask) s> [0, ~highmask] --> X s> ~highmask
+  if (Cmp.getPredicate() == ICmpInst::ICMP_SGT && C1.ule(~*C2) &&
+      C2->isNegatedPowerOf2())
+    return new ICmpInst(ICmpInst::ICMP_SGT, X,
+                        ConstantInt::get(X->getType(), ~*C2));
+  // (and X, highmask) s< [1, -highmask] --> X s< -highmask
+  if (Cmp.getPredicate() == ICmpInst::ICMP_SLT && !C1.isSignMask() &&
+      (C1 - 1).ule(~*C2) && C2->isNegatedPowerOf2() && !C2->isSignMask())
+    return new ICmpInst(ICmpInst::ICMP_SLT, X,
+                        ConstantInt::get(X->getType(), -*C2));
+
   // Don't perform the following transforms if the AND has multiple uses
   if (!And->hasOneUse())
     return nullptr;
@@ -3593,7 +3622,8 @@ Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
               m_OneUse(m_c_Or(m_CombineAnd(m_Value(Sel),
                                            m_Select(m_Value(Cond), m_Value(TV),
                                                     m_Value(FV))),
-                              m_Value(Other))))) {
+                              m_Value(Other)))) &&
+        Cond->getType() == Cmp.getType()) {
       const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
       // Easy case is if eq/ne matches whether 0 is trueval/falseval.
       if (Pred == ICmpInst::ICMP_EQ
@@ -3949,8 +3979,8 @@ Instruction *InstCombinerImpl::foldICmpBinOpWithConstant(ICmpInst &Cmp,
 }
 
 static Instruction *
-foldICmpUSubSatOrUAddSatWithConstant(ICmpInst::Predicate Pred,
-                                     SaturatingInst *II, const APInt &C,
+foldICmpUSubSatOrUAddSatWithConstant(CmpPredicate Pred, SaturatingInst *II,
+                                     const APInt &C,
                                      InstCombiner::BuilderTy &Builder) {
   // This transform may end up producing more than one instruction for the
   // intrinsic, so limit it to one user of the intrinsic.
@@ -4034,7 +4064,7 @@ foldICmpUSubSatOrUAddSatWithConstant(ICmpInst::Predicate Pred,
 }
 
 static Instruction *
-foldICmpOfCmpIntrinsicWithConstant(ICmpInst::Predicate Pred, IntrinsicInst *I,
+foldICmpOfCmpIntrinsicWithConstant(CmpPredicate Pred, IntrinsicInst *I,
                                    const APInt &C,
                                    InstCombiner::BuilderTy &Builder) {
   std::optional<ICmpInst::Predicate> NewPredicate = std::nullopt;
@@ -4233,9 +4263,8 @@ Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) {
   return nullptr;
 }
 
-Instruction *InstCombinerImpl::foldSelectICmp(ICmpInst::Predicate Pred,
-                                              SelectInst *SI, Value *RHS,
-                                              const ICmpInst &I) {
+Instruction *InstCombinerImpl::foldSelectICmp(CmpPredicate Pred, SelectInst *SI,
+                                              Value *RHS, const ICmpInst &I) {
   // Try to fold the comparison into the select arms, which will cause the
   // select to be converted into a logical and/or.
   auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
@@ -4404,7 +4433,7 @@ static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
 /// The Mask can be a constant, too.
 /// For some predicates, the operands are commutative.
 /// For others, x can only be on a specific side.
-static Value *foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0,
+static Value *foldICmpWithLowBitMaskedVal(CmpPredicate Pred, Value *Op0,
                                           Value *Op1, const SimplifyQuery &Q,
                                           InstCombiner &IC) {
 
@@ -5337,6 +5366,15 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
             return new ICmpInst(Pred, X, Y);
           if (ZKnown.isNegative())
             return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), X, Y);
+          Value *LessThan = simplifyICmpInst(ICmpInst::ICMP_SLT, X, Y,
+                                             SQ.getWithInstruction(&I));
+          if (LessThan && match(LessThan, m_One()))
+            return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Z,
+                                Constant::getNullValue(Z->getType()));
+          Value *GreaterThan = simplifyICmpInst(ICmpInst::ICMP_SGT, X, Y,
+                                                SQ.getWithInstruction(&I));
+          if (GreaterThan && match(GreaterThan, m_One()))
+            return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
         }
       } else {
         bool NonZero;
@@ -5515,8 +5553,7 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
 /// Fold icmp Pred min|max(X, Y), Z.
 Instruction *InstCombinerImpl::foldICmpWithMinMax(Instruction &I,
                                                   MinMaxIntrinsic *MinMax,
-                                                  Value *Z,
-                                                  ICmpInst::Predicate Pred) {
+                                                  Value *Z, CmpPredicate Pred) {
   Value *X = MinMax->getLHS();
   Value *Y = MinMax->getRHS();
   if (ICmpInst::isSigned(Pred) && !MinMax->isSigned())
@@ -6869,8 +6906,8 @@ Instruction *InstCombinerImpl::foldICmpUsingBoolRange(ICmpInst &I) {
   return nullptr;
 }
 
-std::optional<std::pair<CmpInst::Predicate, Constant *>>
-InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
+std::optional<std::pair<CmpPredicate, Constant *>>
+InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred,
                                                        Constant *C) {
   assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
          "Only for relational integer predicates.");
@@ -7276,7 +7313,7 @@ static Instruction *foldReductionIdiom(ICmpInst &I,
 }
 
 // This helper will be called with icmp operands in both orders.
-Instruction *InstCombinerImpl::foldICmpCommutative(ICmpInst::Predicate Pred,
+Instruction *InstCombinerImpl::foldICmpCommutative(CmpPredicate Pred,
                                                    Value *Op0, Value *Op1,
                                                    ICmpInst &CxtI) {
   // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
@@ -7404,7 +7441,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
     Changed = true;
   }
 
-  if (Value *V = simplifyICmpInst(I.getPredicate(), Op0, Op1, Q))
+  if (Value *V = simplifyICmpInst(I.getCmpPredicate(), Op0, Op1, Q))
     return replaceInstUsesWith(I, V);
 
   // Comparing -val or val with non-zero is the same as just comparing val
@@ -7511,10 +7548,10 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   if (Instruction *Res = foldICmpInstWithConstantNotInt(I))
     return Res;
 
-  if (Instruction *Res = foldICmpCommutative(I.getPredicate(), Op0, Op1, I))
+  if (Instruction *Res = foldICmpCommutative(I.getCmpPredicate(), Op0, Op1, I))
     return Res;
   if (Instruction *Res =
-          foldICmpCommutative(I.getSwappedPredicate(), Op1, Op0, I))
+          foldICmpCommutative(I.getSwappedCmpPredicate(), Op1, Op0, I))
     return Res;
 
   if (I.isCommutative()) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 0508ed48fc19c..28474fec8238e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -652,10 +652,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   /// folded operation.
   void PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN);
 
-  Instruction *foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
-                           ICmpInst::Predicate Cond, Instruction &I);
-  Instruction *foldSelectICmp(ICmpInst::Predicate Pred, SelectInst *SI,
-                              Value *RHS, const ICmpInst &I);
+  Instruction *foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond,
+                           Instruction &I);
+  Instruction *foldSelectICmp(CmpPredicate Pred, SelectInst *SI, Value *RHS,
+                              const ICmpInst &I);
   bool foldAllocaCmp(AllocaInst *Alloca);
   Instruction *foldCmpLoadFromIndexedGlobal(LoadInst *LI,
                                             GetElementPtrInst *GEP,
@@ -663,8 +663,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                             ConstantInt *AndCst = nullptr);
   Instruction *foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI,
                                     Constant *RHSC);
-  Instruction *foldICmpAddOpConst(Value *X, const APInt &C,
-                                  ICmpInst::Predicate Pred);
+  Instruction *foldICmpAddOpConst(Value *X, const APInt &C, CmpPredicate Pred);
   Instruction *foldICmpWithCastOp(ICmpInst &ICmp);
   Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp);
 
@@ -678,7 +677,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                                    const APInt &C);
   Instruction *foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ);
   Instruction *foldICmpWithMinMax(Instruction &I, MinMaxIntrinsic *MinMax,
-                                  Value *Z, ICmpInst::Predicate Pred);
+                                  Value *Z, CmpPredicate Pred);
   Instruction *foldICmpEquality(ICmpInst &Cmp);
   Instruction *foldIRemByPowerOfTwoToBitTest(ICmpInst &I);
   Instruction *foldSignBitTest(ICmpInst &I);
@@ -736,8 +735,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                                const APInt &C);
   Instruction *foldICmpBitCast(ICmpInst &Cmp);
   Instruction *foldICmpWithTrunc(ICmpInst &Cmp);
-  Instruction *foldICmpCommutative(ICmpInst::Predicate Pred, Value *Op0,
-                                   Value *Op1, ICmpInst &CxtI);
+  Instruction *foldICmpCommutative(CmpPredicate Pred, Value *Op0, Value *Op1,
+                                   ICmpInst &CxtI);
 
   // Helpers of visitSelectInst().
   Instruction *foldSelectOfBools(SelectInst &SI);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index e5525133e5dbb..c7a0c35d099cc 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1257,23 +1257,7 @@ static Value *canonicalizeSPF(ICmpInst &Cmp, Value *TrueVal, Value *FalseVal,
   }
 
   if (SelectPatternResult::isMinOrMax(SPF)) {
-    Intrinsic::ID IntrinsicID;
-    switch (SPF) {
-    case SelectPatternFlavor::SPF_UMIN:
-      IntrinsicID = Intrinsic::umin;
-      break;
-    case SelectPatternFlavor::SPF_UMAX:
-      IntrinsicID = Intrinsic::umax;
-      break;
-    case SelectPatternFlavor::SPF_SMIN:
-      IntrinsicID = Intrinsic::smin;
-      break;
-    case SelectPatternFlavor::SPF_SMAX:
-      IntrinsicID = Intrinsic::smax;
-      break;
-    default:
-      llvm_unreachable("Unexpected SPF");
-    }
+    Intrinsic::ID IntrinsicID = getMinMaxIntrinsic(SPF);
     return IC.Builder.CreateBinaryIntrinsic(IntrinsicID, LHS, RHS);
   }
 
@@ -1348,7 +1332,7 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
     // If we will be able to evaluate f(Y) to a constant, we can allow undef,
     // otherwise Y cannot be undef as we might pick different values for undef
     // in the cmp and in f(Y).
-    if (TrueVal == OldOp)
+    if (TrueVal == OldOp && (isa<Constant>(OldOp) || !isa<Constant>(NewOp)))
       return nullptr;
 
     if (Value *V = simplifyWithOpReplaced(TrueVal, OldOp, NewOp, SQ,
@@ -1898,6 +1882,63 @@ static Instruction *foldSelectICmpEq(SelectInst &SI, ICmpInst *ICI,
   return nullptr;
 }
 
+/// Fold `X Pred C1 ? X BOp C2 : C1 BOp C2` to `min/max(X, C1) BOp C2`.
+/// This allows for better canonicalization.
+static Value *foldSelectWithConstOpToBinOp(ICmpInst *Cmp, Value *TrueVal,
+                                           Value *FalseVal,
+                                           IRBuilderBase &Builder) {
+  BinaryOperator *BOp;
+  Constant *C1, *C2, *C3;
+  Value *X;
+  ICmpInst::Predicate Predicate;
+
+  if (!match(Cmp, m_ICmp(Predicate, m_Value(X), m_Constant(C1))))
+    return nullptr;
+
+  if (!ICmpInst::isRelational(Predicate))
+    return nullptr;
+
+  if (match(TrueVal, m_Constant())) {
+    std::swap(FalseVal, TrueVal);
+    Predicate = ICmpInst::getInversePredicate(Predicate);
+  }
+
+  if (!match(TrueVal, m_BinOp(BOp)) || !match(FalseVal, m_Constant(C3)))
+    return nullptr;
+
+  unsigned Opcode = BOp->getOpcode();
+
+  // This fold causes some regressions and is primarily intended for
+  // add and sub. So we early exit for div and rem to minimize the
+  // regressions.
+  if (Instruction::isIntDivRem(Opcode))
+    return nullptr;
+
+  if (!match(BOp, m_OneUse(m_BinOp(m_Specific(X), m_Constant(C2)))))
+    return nullptr;
+
+  Value *RHS;
+  SelectPatternFlavor SPF;
+  const DataLayout &DL = BOp->getDataLayout();
+  auto Flipped =
+      InstCombiner::getFlippedStrictnessPredicateAndConstant(Predicate, C1);
+
+  if (C3 == ConstantFoldBinaryOpOperands(Opcode, C1, C2, DL)) {
+    SPF = getSelectPattern(Predicate).Flavor;
+    RHS = C1;
+  } else if (Flipped && C3 == ConstantFoldBinaryOpOperands(
+                                  Opcode, Flipped->second, C2, DL)) {
+    SPF = getSelectPattern(Flipped->first).Flavor;
+    RHS = Flipped->second;
+  } else {
+    return nullptr;
+  }
+
+  Intrinsic::ID IntrinsicID = getMinMaxIntrinsic(SPF);
+  Value *Intrinsic = Builder.CreateBinaryIntrinsic(IntrinsicID, X, RHS);
+  return Builder.CreateBinOp(BOp->getOpcode(), Intrinsic, C2);
+}
+
 /// Visit a SelectInst that has an ICmpInst as its first operand.
 Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
                                                       ICmpInst *ICI) {
@@ -1925,17 +1966,6 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
   ICmpInst::Predicate Pred = ICI->getPredicate();
   Value *CmpLHS = ICI->getOperand(0);
   Value *CmpRHS = ICI->getOperand(1);
-  if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS) && !isa<Constant>(CmpLHS)) {
-    if (CmpLHS == TrueVal && Pred == ICmpInst::ICMP_EQ) {
-      // Transform (X == C) ? X : Y -> (X == C) ? C : Y
-      replaceOperand(SI, 1, CmpRHS);
-      Changed = true;
-    } else if (CmpLHS == FalseVal && Pred == ICmpInst::ICMP_NE) {
-      // Transform (X != C) ? Y : X -> (X != C) ? Y : C
-      replaceOperand(SI, 2, CmpRHS);
-      Changed = true;
-    }
-  }
 
   if (Instruction *NewSel = foldSelectICmpEq(SI, ICI, *this))
     return NewSel;
@@ -1987,6 +2017,9 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
   if (Value *V = foldAbsDiff(ICI, TrueVal, FalseVal, Builder))
     return replaceInstUsesWith(SI, V);
 
+  if (Value *V = foldSelectWithConstOpToBinOp(ICI, TrueVal, FalseVal, Builder))
+    return replaceInstUsesWith(SI, V);
+
   return Changed ? &SI : nullptr;
 }
 
@@ -3864,17 +3897,27 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   if (SIFPOp) {
     // TODO: Try to forward-propagate FMF from select arms to the select.
 
+    auto *FCmp = dyn_cast<FCmpInst>(CondVal);
+
     // Canonicalize select of FP values where NaN and -0.0 are not valid as
     // minnum/maxnum intrinsics.
     if (SIFPOp->hasNoNaNs() && SIFPOp->hasNoSignedZeros()) {
       Value *X, *Y;
-      if (match(&SI, m_OrdOrUnordFMax(m_Value(X), m_Value(Y))))
-        return replaceInstUsesWith(
-            SI, Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, X, Y, &SI));
+      if (match(&SI, m_OrdOrUnordFMax(m_Value(X), m_Value(Y)))) {
+        Value *BinIntr =
+            Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, X, Y, &SI);
+        if (auto *BinIntrInst = dyn_cast<Instruction>(BinIntr))
+          BinIntrInst->setHasNoNaNs(FCmp->hasNoNaNs());
+        return replaceInstUsesWith(SI, BinIntr);
+      }
 
-      if (match(&SI, m_OrdOrUnordFMin(m_Value(X), m_Value(Y))))
-        return replaceInstUsesWith(
-            SI, Builder.CreateBinaryIntrinsic(Intrinsic::minnum, X, Y, &SI));
+      if (match(&SI, m_OrdOrUnordFMin(m_Value(X), m_Value(Y)))) {
+        Value *BinIntr =
+            Builder.CreateBinaryIntrinsic(Intrinsic::minnum, X, Y, &SI);
+        if (auto *BinIntrInst = dyn_cast<Instruction>(BinIntr))
+          BinIntrInst->setHasNoNaNs(FCmp->hasNoNaNs());
+        return replaceInstUsesWith(SI, BinIntr);
+      }
     }
   }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 10c3ccdb2243a..d511e79e3e48a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -427,7 +427,8 @@ Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) {
       if (Instruction *R = FoldOpIntoSelect(I, SI))
         return R;
 
-  if (Constant *CUI = dyn_cast<Constant>(Op1))
+  Constant *CUI;
+  if (match(Op1, m_ImmConstant(CUI)))
     if (Instruction *Res = FoldShiftByConstant(Op0, CUI, I))
       return Res;
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 392c5c78345c2..943598a30f040 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -776,6 +776,15 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
             return InsertNewInstWith(Shl, I->getIterator());
           }
         }
+
+        const APInt *Factor;
+        if (match(I->getOperand(0),
+                  m_OneUse(m_Mul(m_Value(X), m_APInt(Factor)))) &&
+            Factor->countr_zero() >= ShiftAmt) {
+          BinaryOperator *Mul = BinaryOperator::CreateMul(
+              X, ConstantInt::get(X->getType(), Factor->lshr(ShiftAmt)));
+          return InsertNewInstWith(Mul, I->getIterator());
+        }
       }
 
       // Unsigned shift right.
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 763d1c9d58ba0..1bcb88f327c26 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1753,9 +1753,9 @@ static Value *simplifyInstructionWithPHI(Instruction &I, PHINode *PN,
   if (TerminatorBI && TerminatorBI->isConditional() &&
       TerminatorBI->getSuccessor(0) != TerminatorBI->getSuccessor(1) && ICmp) {
     bool LHSIsTrue = TerminatorBI->getSuccessor(0) == PN->getParent();
-    std::optional<bool> ImpliedCond =
-        isImpliedCondition(TerminatorBI->getCondition(), ICmp->getPredicate(),
-                           Ops[0], Ops[1], DL, LHSIsTrue);
+    std::optional<bool> ImpliedCond = isImpliedCondition(
+        TerminatorBI->getCondition(), ICmp->getCmpPredicate(), Ops[0], Ops[1],
+        DL, LHSIsTrue);
     if (ImpliedCond)
       return ConstantInt::getBool(I.getType(), ImpliedCond.value());
   }
@@ -2757,6 +2757,111 @@ static bool shouldCanonicalizeGEPToPtrAdd(GetElementPtrInst &GEP) {
          });
 }
 
+static Instruction *foldGEPOfPhi(GetElementPtrInst &GEP, PHINode *PN,
+                                 IRBuilderBase &Builder) {
+  auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
+  if (!Op1)
+    return nullptr;
+
+  // Don't fold a GEP into itself through a PHI node. This can only happen
+  // through the back-edge of a loop. Folding a GEP into itself means that
+  // the value of the previous iteration needs to be stored in the meantime,
+  // thus requiring an additional register variable to be live, but not
+  // actually achieving anything (the GEP still needs to be executed once per
+  // loop iteration).
+  if (Op1 == &GEP)
+    return nullptr;
+
+  int DI = -1;
+
+  for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
+    auto *Op2 = dyn_cast<GetElementPtrInst>(*I);
+    if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands() ||
+        Op1->getSourceElementType() != Op2->getSourceElementType())
+      return nullptr;
+
+    // As for Op1 above, don't try to fold a GEP into itself.
+    if (Op2 == &GEP)
+      return nullptr;
+
+    // Keep track of the type as we walk the GEP.
+    Type *CurTy = nullptr;
+
+    for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
+      if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
+        return nullptr;
+
+      if (Op1->getOperand(J) != Op2->getOperand(J)) {
+        if (DI == -1) {
+          // We have not seen any differences yet in the GEPs feeding the
+          // PHI yet, so we record this one if it is allowed to be a
+          // variable.
+
+          // The first two arguments can vary for any GEP, the rest have to be
+          // static for struct slots
+          if (J > 1) {
+            assert(CurTy && "No current type?");
+            if (CurTy->isStructTy())
+              return nullptr;
+          }
+
+          DI = J;
+        } else {
+          // The GEP is different by more than one input. While this could be
+          // extended to support GEPs that vary by more than one variable it
+          // doesn't make sense since it greatly increases the complexity and
+          // would result in an R+R+R addressing mode which no backend
+          // directly supports and would need to be broken into several
+          // simpler instructions anyway.
+          return nullptr;
+        }
+      }
+
+      // Sink down a layer of the type for the next iteration.
+      if (J > 0) {
+        if (J == 1) {
+          CurTy = Op1->getSourceElementType();
+        } else {
+          CurTy =
+              GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J));
+        }
+      }
+    }
+  }
+
+  // If not all GEPs are identical we'll have to create a new PHI node.
+  // Check that the old PHI node has only one use so that it will get
+  // removed.
+  if (DI != -1 && !PN->hasOneUse())
+    return nullptr;
+
+  auto *NewGEP = cast<GetElementPtrInst>(Op1->clone());
+  if (DI == -1) {
+    // All the GEPs feeding the PHI are identical. Clone one down into our
+    // BB so that it can be merged with the current GEP.
+  } else {
+    // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
+    // into the current block so it can be merged, and create a new PHI to
+    // set that index.
+    PHINode *NewPN;
+    {
+      IRBuilderBase::InsertPointGuard Guard(Builder);
+      Builder.SetInsertPoint(PN);
+      NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(),
+                                PN->getNumOperands());
+    }
+
+    for (auto &I : PN->operands())
+      NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
+                         PN->getIncomingBlock(I));
+
+    NewGEP->setOperand(DI, NewPN);
+  }
+
+  NewGEP->insertBefore(*GEP.getParent(), GEP.getParent()->getFirstInsertionPt());
+  return NewGEP;
+}
+
 Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   Value *PtrOp = GEP.getOperand(0);
   SmallVector<Value *, 8> Indices(GEP.indices());
@@ -2847,107 +2952,8 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
   // Check to see if the inputs to the PHI node are getelementptr instructions.
   if (auto *PN = dyn_cast<PHINode>(PtrOp)) {
-    auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
-    if (!Op1)
-      return nullptr;
-
-    // Don't fold a GEP into itself through a PHI node. This can only happen
-    // through the back-edge of a loop. Folding a GEP into itself means that
-    // the value of the previous iteration needs to be stored in the meantime,
-    // thus requiring an additional register variable to be live, but not
-    // actually achieving anything (the GEP still needs to be executed once per
-    // loop iteration).
-    if (Op1 == &GEP)
-      return nullptr;
-
-    int DI = -1;
-
-    for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
-      auto *Op2 = dyn_cast<GetElementPtrInst>(*I);
-      if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands() ||
-          Op1->getSourceElementType() != Op2->getSourceElementType())
-        return nullptr;
-
-      // As for Op1 above, don't try to fold a GEP into itself.
-      if (Op2 == &GEP)
-        return nullptr;
-
-      // Keep track of the type as we walk the GEP.
-      Type *CurTy = nullptr;
-
-      for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
-        if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
-          return nullptr;
-
-        if (Op1->getOperand(J) != Op2->getOperand(J)) {
-          if (DI == -1) {
-            // We have not seen any differences yet in the GEPs feeding the
-            // PHI yet, so we record this one if it is allowed to be a
-            // variable.
-
-            // The first two arguments can vary for any GEP, the rest have to be
-            // static for struct slots
-            if (J > 1) {
-              assert(CurTy && "No current type?");
-              if (CurTy->isStructTy())
-                return nullptr;
-            }
-
-            DI = J;
-          } else {
-            // The GEP is different by more than one input. While this could be
-            // extended to support GEPs that vary by more than one variable it
-            // doesn't make sense since it greatly increases the complexity and
-            // would result in an R+R+R addressing mode which no backend
-            // directly supports and would need to be broken into several
-            // simpler instructions anyway.
-            return nullptr;
-          }
-        }
-
-        // Sink down a layer of the type for the next iteration.
-        if (J > 0) {
-          if (J == 1) {
-            CurTy = Op1->getSourceElementType();
-          } else {
-            CurTy =
-                GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J));
-          }
-        }
-      }
-    }
-
-    // If not all GEPs are identical we'll have to create a new PHI node.
-    // Check that the old PHI node has only one use so that it will get
-    // removed.
-    if (DI != -1 && !PN->hasOneUse())
-      return nullptr;
-
-    auto *NewGEP = cast<GetElementPtrInst>(Op1->clone());
-    if (DI == -1) {
-      // All the GEPs feeding the PHI are identical. Clone one down into our
-      // BB so that it can be merged with the current GEP.
-    } else {
-      // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
-      // into the current block so it can be merged, and create a new PHI to
-      // set that index.
-      PHINode *NewPN;
-      {
-        IRBuilderBase::InsertPointGuard Guard(Builder);
-        Builder.SetInsertPoint(PN);
-        NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(),
-                                  PN->getNumOperands());
-      }
-
-      for (auto &I : PN->operands())
-        NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
-                           PN->getIncomingBlock(I));
-
-      NewGEP->setOperand(DI, NewPN);
-    }
-
-    NewGEP->insertBefore(*GEP.getParent(), GEP.getParent()->getFirstInsertionPt());
-    return replaceOperand(GEP, 0, NewGEP);
+    if (Value *NewPtrOp = foldGEPOfPhi(GEP, PN, Builder))
+      return replaceOperand(GEP, 0, NewPtrOp);
   }
 
   if (auto *Src = dyn_cast<GEPOperator>(PtrOp))
@@ -3114,6 +3120,35 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     }
   }
 
+  // The single (non-zero) index of an inbounds GEP of a base object cannot
+  // be negative.
+  auto HasOneNonZeroIndex = [&]() {
+    bool FoundNonZero = false;
+    for (Value *Idx : GEP.indices()) {
+      auto *C = dyn_cast<Constant>(Idx);
+      if (C && C->isNullValue())
+        continue;
+      if (FoundNonZero)
+        return false;
+      FoundNonZero = true;
+    }
+    return true;
+  };
+  if (GEP.isInBounds() && !GEP.hasNoUnsignedWrap() && isBaseOfObject(PtrOp) &&
+      HasOneNonZeroIndex()) {
+    GEP.setNoWrapFlags(GEP.getNoWrapFlags() | GEPNoWrapFlags::noUnsignedWrap());
+    return &GEP;
+  }
+
+  // nusw + nneg -> nuw
+  if (GEP.hasNoUnsignedSignedWrap() && !GEP.hasNoUnsignedWrap() &&
+      all_of(GEP.indices(), [&](Value *Idx) {
+        return isKnownNonNegative(Idx, SQ.getWithInstruction(&GEP));
+      })) {
+    GEP.setNoWrapFlags(GEP.getNoWrapFlags() | GEPNoWrapFlags::noUnsignedWrap());
+    return &GEP;
+  }
+
   if (Instruction *R = foldSelectGEP(GEP, Builder))
     return R;
 
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 2ea89be40a3d4..f9be7f933d31e 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -820,7 +820,8 @@ bool GCOVProfiler::emitProfileNotes(
       SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/false, BPI,
                                    BFI);
 
-      CFGMST<Edge, BBInfo> MST(F, /*InstrumentFuncEntry_=*/false, BPI, BFI);
+      CFGMST<Edge, BBInfo> MST(F, /*InstrumentFuncEntry=*/false,
+                               /*InstrumentLoopEntries=*/false, BPI, BFI);
 
       // getInstrBB can split basic blocks and push elements to AllEdges.
       for (size_t I : llvm::seq<size_t>(0, MST.numEdges())) {
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 33a7a37fa28e6..ab7d3c7002991 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -166,6 +166,11 @@ static cl::opt<bool>
                                      "context in this module's profiles"),
                             cl::Hidden, cl::init(false));
 
+static cl::opt<std::string>
+    MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
+                                 cl::desc("The default memprof options"),
+                                 cl::Hidden, cl::init(""));
+
 extern cl::opt<bool> MemProfReportHintedSizes;
 
 // Instrumentation statistics
@@ -547,6 +552,20 @@ void createMemprofHistogramFlagVar(Module &M) {
   appendToCompilerUsed(M, MemprofHistogramFlag);
 }
 
+void createMemprofDefaultOptionsVar(Module &M) {
+  Constant *OptionsConst = ConstantDataArray::getString(
+      M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
+  GlobalVariable *OptionsVar =
+      new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
+                         GlobalValue::WeakAnyLinkage, OptionsConst,
+                         "__memprof_default_options_str");
+  Triple TT(M.getTargetTriple());
+  if (TT.supportsCOMDAT()) {
+    OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
+    OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
+  }
+}
+
 bool ModuleMemProfiler::instrumentModule(Module &M) {
 
   // Create a module constructor.
@@ -566,6 +585,8 @@ bool ModuleMemProfiler::instrumentModule(Module &M) {
 
   createMemprofHistogramFlagVar(M);
 
+  createMemprofDefaultOptionsVar(M);
+
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index ab31c14c7be6b..8ed60d73209e2 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -4085,6 +4085,20 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  // _mm_round_ps / _mm_round_ps.
+  // Similar to maybeHandleSimpleNomemIntrinsic except
+  // the second argument is guranteed to be a constant integer.
+  void handleRoundPdPsIntrinsic(IntrinsicInst &I) {
+    assert(I.getArgOperand(0)->getType() == I.getType());
+    assert(I.arg_size() == 2);
+    assert(isa<ConstantInt>(I.getArgOperand(1)));
+
+    IRBuilder<> IRB(&I);
+    ShadowAndOriginCombiner SC(this, IRB);
+    SC.Add(I.getArgOperand(0));
+    SC.Done(&I);
+  }
+
   // Instrument abs intrinsic.
   // handleUnknownIntrinsic can't handle it because of the last
   // is_int_min_poison argument which does not match the result type.
@@ -4265,6 +4279,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  // Approximation only
+  void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
+    handleShadowOr(I);
+  }
+
   void visitIntrinsicInst(IntrinsicInst &I) {
     switch (I.getIntrinsicID()) {
     case Intrinsic::uadd_with_overflow:
@@ -4580,10 +4599,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       handlePclmulIntrinsic(I);
       break;
 
+    case Intrinsic::x86_avx_round_pd_256:
+    case Intrinsic::x86_avx_round_ps_256:
+    case Intrinsic::x86_sse41_round_pd:
+    case Intrinsic::x86_sse41_round_ps:
+      handleRoundPdPsIntrinsic(I);
+      break;
+
     case Intrinsic::x86_sse41_round_sd:
     case Intrinsic::x86_sse41_round_ss:
       handleUnarySdSsIntrinsic(I);
       break;
+
     case Intrinsic::x86_sse2_max_sd:
     case Intrinsic::x86_sse_max_ss:
     case Intrinsic::x86_sse2_min_sd:
@@ -4662,6 +4689,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    case Intrinsic::aarch64_neon_fmulx:
+    case Intrinsic::aarch64_neon_pmul:
+    case Intrinsic::aarch64_neon_pmull:
+    case Intrinsic::aarch64_neon_smull:
+    case Intrinsic::aarch64_neon_pmull64:
+    case Intrinsic::aarch64_neon_umull: {
+      handleNEONVectorMultiplyIntrinsic(I);
+      break;
+    }
+
     default:
       if (!handleUnknownIntrinsic(I))
         visitInstruction(I);
@@ -6412,8 +6449,10 @@ struct VarArgGenericHelper : public VarArgHelperBase {
     unsigned VAArgOffset = 0;
     const DataLayout &DL = F.getDataLayout();
     unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
-    for (Value *A :
-         llvm::drop_begin(CB.args(), CB.getFunctionType()->getNumParams())) {
+    for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
+      bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
+      if (IsFixed)
+        continue;
       uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
       if (DL.isBigEndian()) {
         // Adjusting the shadow for argument with size < IntptrSize to match the
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 4d8141431a0c1..471086ce3a751 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -259,6 +259,11 @@ static cl::opt<bool> PGOInstrumentEntry(
     "pgo-instrument-entry", cl::init(false), cl::Hidden,
     cl::desc("Force to instrument function entry basicblock."));
 
+static cl::opt<bool>
+    PGOInstrumentLoopEntries("pgo-instrument-loop-entries", cl::init(false),
+                             cl::Hidden,
+                             cl::desc("Force to instrument loop entries."));
+
 static cl::opt<bool> PGOFunctionEntryCoverage(
     "pgo-function-entry-coverage", cl::Hidden,
     cl::desc(
@@ -359,6 +364,7 @@ class FunctionInstrumenter final {
   std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
   BranchProbabilityInfo *const BPI;
   BlockFrequencyInfo *const BFI;
+  LoopInfo *const LI;
 
   const PGOInstrumentationType InstrumentationType;
 
@@ -376,14 +382,17 @@ class FunctionInstrumenter final {
            InstrumentationType == PGOInstrumentationType::CTXPROF;
   }
 
+  bool shouldInstrumentLoopEntries() const { return PGOInstrumentLoopEntries; }
+
 public:
   FunctionInstrumenter(
       Module &M, Function &F, TargetLibraryInfo &TLI,
       std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
       BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr,
+      LoopInfo *LI = nullptr,
       PGOInstrumentationType InstrumentationType = PGOInstrumentationType::FDO)
       : M(M), F(F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),
-        InstrumentationType(InstrumentationType) {}
+        LI(LI), InstrumentationType(InstrumentationType) {}
 
   void instrument();
 };
@@ -439,6 +448,8 @@ createIRLevelProfileFlagVar(Module &M,
   if (PGOInstrumentEntry ||
       InstrumentationType == PGOInstrumentationType::CTXPROF)
     ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
+  if (PGOInstrumentLoopEntries)
+    ProfileVersion |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
   if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
     ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
   if (PGOFunctionEntryCoverage)
@@ -625,12 +636,13 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
       Function &Func, TargetLibraryInfo &TLI,
       std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
       bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
-      BlockFrequencyInfo *BFI = nullptr, bool IsCS = false,
-      bool InstrumentFuncEntry = true, bool HasSingleByteCoverage = false)
+      BlockFrequencyInfo *BFI = nullptr, LoopInfo *LI = nullptr,
+      bool IsCS = false, bool InstrumentFuncEntry = true,
+      bool InstrumentLoopEntries = false, bool HasSingleByteCoverage = false)
       : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI),
         TLI(TLI), ValueSites(IPVK_Last + 1),
         SIVisitor(Func, HasSingleByteCoverage),
-        MST(F, InstrumentFuncEntry, BPI, BFI),
+        MST(F, InstrumentFuncEntry, InstrumentLoopEntries, BPI, BFI, LI),
         BCI(constructBCI(Func, HasSingleByteCoverage, InstrumentFuncEntry)) {
     if (BCI && PGOViewBlockCoverageGraph)
       BCI->viewBlockCoverageGraph();
@@ -916,9 +928,10 @@ void FunctionInstrumenter::instrument() {
 
   const bool IsCtxProf = InstrumentationType == PGOInstrumentationType::CTXPROF;
   FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
-      F, TLI, ComdatMembers, /*CreateGlobalVar=*/!IsCtxProf, BPI, BFI,
+      F, TLI, ComdatMembers, /*CreateGlobalVar=*/!IsCtxProf, BPI, BFI, LI,
       InstrumentationType == PGOInstrumentationType::CSFDO,
-      shouldInstrumentEntryBB(), PGOBlockCoverage);
+      shouldInstrumentEntryBB(), shouldInstrumentLoopEntries(),
+      PGOBlockCoverage);
 
   auto *const Name = IsCtxProf ? cast<GlobalValue>(&F) : FuncInfo.FuncNameVar;
   auto *const CFGHash =
@@ -1136,11 +1149,13 @@ class PGOUseFunc {
   PGOUseFunc(Function &Func, Module *Modu, TargetLibraryInfo &TLI,
              std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
              BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
-             ProfileSummaryInfo *PSI, bool IsCS, bool InstrumentFuncEntry,
+             LoopInfo *LI, ProfileSummaryInfo *PSI, bool IsCS,
+             bool InstrumentFuncEntry, bool InstrumentLoopEntries,
              bool HasSingleByteCoverage)
       : F(Func), M(Modu), BFI(BFIin), PSI(PSI),
-        FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS,
-                 InstrumentFuncEntry, HasSingleByteCoverage),
+        FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, LI, IsCS,
+                 InstrumentFuncEntry, InstrumentLoopEntries,
+                 HasSingleByteCoverage),
         FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {}
 
   void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);
@@ -1923,6 +1938,7 @@ static bool InstrumentAllFunctions(
     Module &M, function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
     function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
     function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
+    function_ref<LoopInfo *(Function &)> LookupLI,
     PGOInstrumentationType InstrumentationType) {
   // For the context-sensitve instrumentation, we should have a separated pass
   // (before LTO/ThinLTO linking) to create these variables.
@@ -1943,10 +1959,11 @@ static bool InstrumentAllFunctions(
   for (auto &F : M) {
     if (skipPGOGen(F))
       continue;
-    auto &TLI = LookupTLI(F);
-    auto *BPI = LookupBPI(F);
-    auto *BFI = LookupBFI(F);
-    FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI,
+    TargetLibraryInfo &TLI = LookupTLI(F);
+    BranchProbabilityInfo *BPI = LookupBPI(F);
+    BlockFrequencyInfo *BFI = LookupBFI(F);
+    LoopInfo *LI = LookupLI(F);
+    FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI, LI,
                             InstrumentationType);
     FI.instrument();
   }
@@ -1980,8 +1997,11 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
   auto LookupBFI = [&FAM](Function &F) {
     return &FAM.getResult<BlockFrequencyAnalysis>(F);
   };
+  auto LookupLI = [&FAM](Function &F) {
+    return &FAM.getResult<LoopAnalysis>(F);
+  };
 
-  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI,
+  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI, LookupLI,
                               InstrumentationType))
     return PreservedAnalyses::all();
 
@@ -2116,7 +2136,8 @@ static bool annotateAllFunctions(
     function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
     function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
     function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
-    ProfileSummaryInfo *PSI, bool IsCS) {
+    function_ref<LoopInfo *(Function &)> LookupLI, ProfileSummaryInfo *PSI,
+    bool IsCS) {
   LLVM_DEBUG(dbgs() << "Read in profile counters: ");
   auto &Ctx = M.getContext();
   // Read the counter array from file.
@@ -2181,22 +2202,27 @@ static bool annotateAllFunctions(
   bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
   if (PGOInstrumentEntry.getNumOccurrences() > 0)
     InstrumentFuncEntry = PGOInstrumentEntry;
+  bool InstrumentLoopEntries = PGOReader->instrLoopEntriesEnabled();
+  if (PGOInstrumentLoopEntries.getNumOccurrences() > 0)
+    InstrumentLoopEntries = PGOInstrumentLoopEntries;
 
   bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
   for (auto &F : M) {
     if (skipPGOUse(F))
       continue;
-    auto &TLI = LookupTLI(F);
-    auto *BPI = LookupBPI(F);
-    auto *BFI = LookupBFI(F);
+    TargetLibraryInfo &TLI = LookupTLI(F);
+    BranchProbabilityInfo *BPI = LookupBPI(F);
+    BlockFrequencyInfo *BFI = LookupBFI(F);
+    LoopInfo *LI = LookupLI(F);
     if (!HasSingleByteCoverage) {
       // Split indirectbr critical edges here before computing the MST rather
       // than later in getInstrBB() to avoid invalidating it.
       SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/false, BPI,
                                    BFI);
     }
-    PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
-                    InstrumentFuncEntry, HasSingleByteCoverage);
+    PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, LI, PSI, IsCS,
+                    InstrumentFuncEntry, InstrumentLoopEntries,
+                    HasSingleByteCoverage);
     if (HasSingleByteCoverage) {
       Func.populateCoverage(PGOReader.get());
       continue;
@@ -2335,10 +2361,14 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M,
   auto LookupBFI = [&FAM](Function &F) {
     return &FAM.getResult<BlockFrequencyAnalysis>(F);
   };
+  auto LookupLI = [&FAM](Function &F) {
+    return &FAM.getResult<LoopAnalysis>(F);
+  };
 
   auto *PSI = &MAM.getResult<ProfileSummaryAnalysis>(M);
   if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, *FS,
-                            LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
+                            LookupTLI, LookupBPI, LookupBFI, LookupLI, PSI,
+                            IsCS))
     return PreservedAnalyses::all();
 
   return PreservedAnalyses::none();
diff --git a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
index 88cb04695217d..890c79ae96718 100644
--- a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
@@ -17,12 +17,16 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
 
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Transforms/Instrumentation/RealtimeSanitizer.h"
 
 using namespace llvm;
 
+const char kRtsanModuleCtorName[] = "rtsan.module_ctor";
+const char kRtsanInitName[] = "__rtsan_ensure_initialized";
+
 static SmallVector<Type *> getArgTypes(ArrayRef<Value *> FunctionArgs) {
   SmallVector<Type *> Types;
   for (Value *Arg : FunctionArgs)
@@ -89,3 +93,14 @@ PreservedAnalyses RealtimeSanitizerPass::run(Function &Fn,
 
   return PreservedAnalyses::all();
 }
+
+PreservedAnalyses ModuleRealtimeSanitizerPass::run(Module &M,
+                                                   ModuleAnalysisManager &MAM) {
+  getOrCreateSanitizerCtorAndInitFunctions(
+      M, kRtsanModuleCtorName, kRtsanInitName, /*InitArgTypes=*/{},
+      /*InitArgs=*/{},
+      // This callback is invoked when the functions are created the first
+      // time. Hook them into the global ctors list in that case:
+      [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });
+  return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 0deab26a2d747..d0752572cb6c6 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -1036,10 +1036,8 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
         ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
   }
   if (Options.TracePCGuard) {
-    auto GuardPtr = IRB.CreateIntToPtr(
-        IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy),
-                      ConstantInt::get(IntptrTy, Idx * 4)),
-        PtrTy);
+    auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
+        FunctionGuardArray->getValueType(), FunctionGuardArray, 0, Idx);
     if (Options.GatedCallbacks) {
       Instruction *I = &*IP;
       auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 4884c23f16e12..e64fc153cf3d2 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -387,12 +387,12 @@ struct OffsetResult {
   Value *BasePtr;
   APInt ConstantOffset;
   SmallMapVector<Value *, APInt, 4> VariableOffsets;
-  bool AllInbounds;
+  GEPNoWrapFlags NW;
 
   OffsetResult() : BasePtr(nullptr), ConstantOffset(0, uint64_t(0)) {}
 
   OffsetResult(GEPOperator &GEP, const DataLayout &DL)
-      : BasePtr(GEP.getPointerOperand()), AllInbounds(GEP.isInBounds()) {
+      : BasePtr(GEP.getPointerOperand()), NW(GEP.getNoWrapFlags()) {
     ConstantOffset = APInt(DL.getIndexTypeSizeInBits(BasePtr->getType()), 0);
   }
 };
@@ -426,7 +426,7 @@ static OffsetResult collectOffsets(GEPOperator &GEP, const DataLayout &DL) {
     Result.ConstantOffset += ConstantOffset2;
     if (Result.VariableOffsets.size() == 0 && VariableOffsets2.size() == 1)
       Result.VariableOffsets = VariableOffsets2;
-    Result.AllInbounds &= InnerGEP->isInBounds();
+    Result.NW &= InnerGEP->getNoWrapFlags();
   }
   return Result;
 }
@@ -450,9 +450,11 @@ static Decomposition decomposeGEP(GEPOperator &GEP,
 
   assert(!IsSigned && "The logic below only supports decomposition for "
                       "unsigned predicates at the moment.");
-  const auto &[BasePtr, ConstantOffset, VariableOffsets, AllInbounds] =
+  const auto &[BasePtr, ConstantOffset, VariableOffsets, NW] =
       collectOffsets(GEP, DL);
-  if (!BasePtr || !AllInbounds)
+  // We support either plain gep nuw, or gep nusw with non-negative offset,
+  // which implies gep nuw.
+  if (!BasePtr || NW == GEPNoWrapFlags::none())
     return &GEP;
 
   Decomposition Result(ConstantOffset.getSExtValue(), DecompEntry(1, BasePtr));
@@ -461,11 +463,13 @@ static Decomposition decomposeGEP(GEPOperator &GEP,
     IdxResult.mul(Scale.getSExtValue());
     Result.add(IdxResult);
 
-    // If Op0 is signed non-negative, the GEP is increasing monotonically and
-    // can be de-composed.
-    if (!isKnownNonNegative(Index, DL))
-      Preconditions.emplace_back(CmpInst::ICMP_SGE, Index,
-                                 ConstantInt::get(Index->getType(), 0));
+    if (!NW.hasNoUnsignedWrap()) {
+      // Try to prove nuw from nusw and nneg.
+      assert(NW.hasNoUnsignedSignedWrap() && "Must have nusw flag");
+      if (!isKnownNonNegative(Index, DL))
+        Preconditions.emplace_back(CmpInst::ICMP_SGE, Index,
+                                   ConstantInt::get(Index->getType(), 0));
+    }
   }
   return Result;
 }
@@ -517,6 +521,9 @@ static Decomposition decompose(Value *V,
     else if (match(V, m_NNegZExt(m_Value(Op0)))) {
       V = Op0;
       IsKnownNonNegative = true;
+    } else if (match(V, m_NSWTrunc(m_Value(Op0)))) {
+      if (Op0->getType()->getScalarSizeInBits() <= 64)
+        V = Op0;
     }
 
     if (match(V, m_NSWAdd(m_Value(Op0), m_Value(Op1))))
@@ -554,12 +561,19 @@ static Decomposition decompose(Value *V,
   if (match(V, m_ZExt(m_Value(Op0)))) {
     IsKnownNonNegative = true;
     V = Op0;
-  }
-
-  if (match(V, m_SExt(m_Value(Op0)))) {
+  } else if (match(V, m_SExt(m_Value(Op0)))) {
     V = Op0;
     Preconditions.emplace_back(CmpInst::ICMP_SGE, Op0,
                                ConstantInt::get(Op0->getType(), 0));
+  } else if (auto *Trunc = dyn_cast<TruncInst>(V)) {
+    if (Trunc->getSrcTy()->getScalarSizeInBits() <= 64) {
+      if (Trunc->hasNoUnsignedWrap() || Trunc->hasNoSignedWrap()) {
+        V = Trunc->getOperand(0);
+        if (!Trunc->hasNoUnsignedWrap())
+          Preconditions.emplace_back(CmpInst::ICMP_SGE, V,
+                                     ConstantInt::get(V->getType(), 0));
+      }
+    }
   }
 
   Value *Op1;
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 5555b5e29cc74..778f83b792569 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -164,9 +164,9 @@ static cl::opt<bool>
     OptimizeMemorySSA("dse-optimize-memoryssa", cl::init(true), cl::Hidden,
                       cl::desc("Allow DSE to optimize memory accesses."));
 
-// TODO: turn on and remove this flag.
+// TODO: remove this flag.
 static cl::opt<bool> EnableInitializesImprovement(
-    "enable-dse-initializes-attr-improvement", cl::init(false), cl::Hidden,
+    "enable-dse-initializes-attr-improvement", cl::init(true), cl::Hidden,
     cl::desc("Enable the initializes attr improvement in DSE"));
 
 //===----------------------------------------------------------------------===//
@@ -2014,9 +2014,8 @@ struct DSEState {
       return false;
     IRBuilder<> IRB(Malloc);
     Type *SizeTTy = Malloc->getArgOperand(0)->getType();
-    auto *Calloc =
-        emitCalloc(ConstantInt::get(SizeTTy, 1), Malloc->getArgOperand(0), IRB,
-                   TLI, Malloc->getType()->getPointerAddressSpace());
+    auto *Calloc = emitCalloc(Malloc->getType(), ConstantInt::get(SizeTTy, 1),
+                              Malloc->getArgOperand(0), IRB, TLI);
     if (!Calloc)
       return false;
 
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index a49dc7d30a00b..0bc783412595e 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -248,7 +248,7 @@ class InductiveRangeCheckElimination {
 
   // Returns true if it is profitable to do a transform basing on estimation of
   // number of iterations.
-  bool isProfitableToTransform(const Loop &L, LoopStructure &LS);
+  bool isProfitableToTransform(const Loop &L);
 
 public:
   InductiveRangeCheckElimination(ScalarEvolution &SE,
@@ -938,14 +938,12 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
   return getLoopPassPreservedAnalyses();
 }
 
-bool
-InductiveRangeCheckElimination::isProfitableToTransform(const Loop &L,
-                                                        LoopStructure &LS) {
+bool InductiveRangeCheckElimination::isProfitableToTransform(const Loop &L) {
   if (SkipProfitabilityChecks)
     return true;
   if (GetBFI) {
     BlockFrequencyInfo &BFI = (*GetBFI)();
-    uint64_t hFreq = BFI.getBlockFreq(LS.Header).getFrequency();
+    uint64_t hFreq = BFI.getBlockFreq(L.getHeader()).getFrequency();
     uint64_t phFreq = BFI.getBlockFreq(L.getLoopPreheader()).getFrequency();
     if (phFreq != 0 && hFreq != 0 && (hFreq / phFreq < MinRuntimeIterations)) {
       LLVM_DEBUG(dbgs() << "irce: could not prove profitability: "
@@ -958,8 +956,17 @@ InductiveRangeCheckElimination::isProfitableToTransform(const Loop &L,
 
   if (!BPI)
     return true;
+
+  auto *Latch = L.getLoopLatch();
+  if (!Latch)
+    return true;
+  auto *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
+  if (!LatchBr)
+    return true;
+  auto LatchBrExitIdx = LatchBr->getSuccessor(0) == L.getHeader() ? 1 : 0;
+
   BranchProbability ExitProbability =
-      BPI->getEdgeProbability(LS.Latch, LS.LatchBrExitIdx);
+      BPI->getEdgeProbability(Latch, LatchBrExitIdx);
   if (ExitProbability > BranchProbability(1, MinRuntimeIterations)) {
     LLVM_DEBUG(dbgs() << "irce: could not prove profitability: "
                       << "the exit probability is too big " << ExitProbability
@@ -982,6 +989,9 @@ bool InductiveRangeCheckElimination::run(
     return false;
   }
 
+  if (!isProfitableToTransform(*L))
+    return false;
+
   LLVMContext &Context = Preheader->getContext();
   SmallVector<InductiveRangeCheck, 16> RangeChecks;
   bool Changed = false;
@@ -1017,8 +1027,6 @@ bool InductiveRangeCheckElimination::run(
     return Changed;
   }
   LoopStructure LS = *MaybeLoopStructure;
-  if (!isProfitableToTransform(*L, LS))
-    return Changed;
   const SCEVAddRecExpr *IndVar =
       cast<SCEVAddRecExpr>(SE.getMinusSCEV(SE.getSCEV(LS.IndVarBase), SE.getSCEV(LS.IndVarStep)));
 
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 953748f07af02..260cc72c3188d 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -57,6 +57,7 @@
 #include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include "llvm/Transforms/Utils/SizeOpts.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 #include <algorithm>
@@ -218,6 +219,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
   UP.UnrollAndJam = false;
   UP.UnrollAndJamInnerLoopThreshold = 60;
   UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
+  UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
 
   // Override with any target specific settings
   TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1349,6 +1351,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
   ULO.Runtime = UP.Runtime;
   ULO.ForgetAllSCEV = ForgetAllSCEV;
   ULO.Heart = getLoopConvergenceHeart(L);
+  ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
   LoopUnrollResult UnrollResult = UnrollLoop(
       L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
   if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index eaf58ea8dd9d0..796fba67ee257 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -97,19 +97,6 @@ static DISubprogram *getSubprogram(DIScope *Scope) {
   return cast<DILocalScope>(Scope)->getSubprogram();
 }
 
-/// Erase \p V from \p BB and move \II forward to avoid invalidating
-/// iterators.
-static void eraseFromParentAndMove(Value *V, BasicBlock::reverse_iterator &II,
-                                   BasicBlock &BB) {
-  auto *Inst = cast<Instruction>(V);
-  // Still used, don't erase.
-  if (!Inst->use_empty())
-    return;
-  if (II != BB.rend() && Inst == &*II)
-    ++II;
-  Inst->eraseFromParent();
-}
-
 /// Return true if V is a splat of a value (which is used when multiplying a
 /// matrix with a scalar).
 static bool isSplat(Value *V) {
@@ -259,7 +246,7 @@ static bool isUniformShape(Value *V) {
 /// Return the ShapeInfo for the result of \p I, it it can be determined.
 static std::optional<ShapeInfo>
 computeShapeInfoForInst(Instruction *I,
-                        const ValueMap<Value *, ShapeInfo> &ShapeMap) {
+                        const DenseMap<Value *, ShapeInfo> &ShapeMap) {
   Value *M;
   Value *N;
   Value *K;
@@ -323,10 +310,11 @@ class LowerMatrixIntrinsics {
   Function &Func;
   const DataLayout &DL;
   const TargetTransformInfo &TTI;
-  AliasAnalysis *AA;
-  DominatorTree *DT;
-  LoopInfo *LI;
-  OptimizationRemarkEmitter *ORE;
+  FunctionAnalysisManager *AM;
+  AliasAnalysis *AA = nullptr;
+  DominatorTree *DT = nullptr;
+  LoopInfo *LI = nullptr;
+  OptimizationRemarkEmitter *ORE = nullptr;
 
   /// Contains estimates of the number of operations (loads, stores, compute) required to lower a matrix operation.
   struct OpInfoTy {
@@ -492,10 +480,16 @@ class LowerMatrixIntrinsics {
   /// the result value of the instruction, with the only exceptions being store
   /// instructions and the matrix_column_major_store intrinsics. For those, the
   /// shape information indicates that those instructions should be lowered
-  /// using shape information as well.  A ValueMap is used so that when
-  /// sub-passes like optimizeTransposes performs RAUW the map stays
-  /// up-to-date.
-  ValueMap<Value *, ShapeInfo> ShapeMap;
+  /// using shape information as well. Note that extra care is needed when
+  /// erasing or RAUW'ing a value that is present in ShapeMap. If the
+  /// replacement is also a matrix operation, use
+  /// updateShapeAndReplaceAllUsesWith to make sure the replacement is added to
+  /// ShapeMap.  We don't use ValueMap, as there are also cases where we do not
+  /// want to add shape information for a replacement instruction. When directly
+  /// erasing a value with an entry in ShapeMap, use
+  /// eraseFromParentAndRemoveFromShapeMap to make sure ShapeMap is also updated
+  /// accordingly.
+  DenseMap<Value *, ShapeInfo> ShapeMap;
 
   /// List of instructions to remove. While lowering, we are not replacing all
   /// users of a lowered instruction, if shape information is available and
@@ -519,10 +513,8 @@ class LowerMatrixIntrinsics {
 
 public:
   LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI,
-                        AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
-                        OptimizationRemarkEmitter *ORE)
-      : Func(F), DL(F.getDataLayout()), TTI(TTI), AA(AA), DT(DT),
-        LI(LI), ORE(ORE) {}
+                        FunctionAnalysisManager *AM)
+      : Func(F), DL(F.getDataLayout()), TTI(TTI), AM(AM) {}
 
   unsigned getNumOps(Type *VT) {
     assert(isa<VectorType>(VT) && "Expected vector type");
@@ -759,6 +751,30 @@ class LowerMatrixIntrinsics {
     return Operation(T0, Shape0.t(), T1, Shape1.t());
   }
 
+  /// Erase \p Inst from both ShapeMap (if an entry exists) and erase \p Inst
+  /// itself.
+  void eraseFromParentAndRemoveFromShapeMap(Instruction *Inst) {
+    auto Iter = ShapeMap.find(Inst);
+    if (Iter != ShapeMap.end())
+      ShapeMap.erase(Iter);
+    Inst->eraseFromParent();
+  }
+
+  /// Erase \p V from \p BB and move \II forward to avoid invalidating
+  /// iterators.
+  void eraseFromParentAndMove(Value *V, BasicBlock::reverse_iterator &II,
+                              BasicBlock &BB) {
+    auto *Inst = cast<Instruction>(V);
+    // Still used, don't erase.
+    if (!Inst->use_empty())
+      return;
+    if (II != BB.rend() && Inst == &*II)
+      ++II;
+    eraseFromParentAndRemoveFromShapeMap(Inst);
+  }
+
+  /// Add a new entry to ShapeMap for \p New with \p Old's shape info, erase the
+  /// entry for \p Old and replace all uses of \p Old with \p New.
   void updateShapeAndReplaceAllUsesWith(Instruction &Old, Value *New) {
     // We need to remove Old from the ShapeMap otherwise RAUW will replace it
     // with New. We should only add New it it supportsShapeInfo so we insert
@@ -872,13 +888,13 @@ class LowerMatrixIntrinsics {
 
   void liftTranspose(Instruction &I) {
     // Erase dead Instructions after lifting transposes from binops.
-    auto CleanupBinOp = [](Instruction &T, Value *A, Value *B) {
+    auto CleanupBinOp = [this](Instruction &T, Value *A, Value *B) {
       if (T.use_empty())
-        T.eraseFromParent();
+        eraseFromParentAndRemoveFromShapeMap(&T);
       if (A->use_empty())
-        cast<Instruction>(A)->eraseFromParent();
+        eraseFromParentAndRemoveFromShapeMap(cast<Instruction>(A));
       if (A != B && B->use_empty())
-        cast<Instruction>(B)->eraseFromParent();
+        eraseFromParentAndRemoveFromShapeMap(cast<Instruction>(B));
     };
 
     Value *A, *B, *AT, *BT;
@@ -908,8 +924,7 @@ class LowerMatrixIntrinsics {
              match(B, m_Intrinsic<Intrinsic::matrix_transpose>(
                           m_Value(BT), m_ConstantInt(), m_ConstantInt()))) {
       IRBuilder<> Builder(&I);
-      auto *Add = cast<Instruction>(Builder.CreateFAdd(AT, BT, "mfadd"));
-      setShapeInfo(Add, {R, C});
+      auto *Add = Builder.CreateFAdd(AT, BT, "mfadd");
       MatrixBuilder MBuilder(Builder);
       Instruction *NewInst = MBuilder.CreateMatrixTranspose(
           Add, R->getZExtValue(), C->getZExtValue(), "mfadd_t");
@@ -918,9 +933,13 @@ class LowerMatrixIntrinsics {
                  computeShapeInfoForInst(&I, ShapeMap) &&
              "Shape of new instruction doesn't match original shape.");
       CleanupBinOp(I, A, B);
-      assert(computeShapeInfoForInst(Add, ShapeMap).value_or(ShapeMap[Add]) ==
-                 ShapeMap[Add] &&
-             "Shape of updated addition doesn't match cached shape.");
+      if (auto *AddI = dyn_cast<Instruction>(Add)) {
+        setShapeInfo(AddI, {R, C});
+        assert(
+            computeShapeInfoForInst(AddI, ShapeMap).value_or(ShapeMap[AddI]) ==
+                ShapeMap[AddI] &&
+            "Shape of updated addition doesn't match cached shape.");
+      }
     }
   }
 
@@ -974,6 +993,13 @@ class LowerMatrixIntrinsics {
     if (WorkList.empty())
       return false;
 
+    if (AM) {
+      ORE = &AM->getResult<OptimizationRemarkEmitterAnalysis>(Func);
+      AA = &AM->getResult<AAManager>(Func);
+      DT = &AM->getResult<DominatorTreeAnalysis>(Func);
+      LI = &AM->getResult<LoopAnalysis>(Func);
+    }
+
     // Propagate shapes until nothing changes any longer.
     while (!WorkList.empty()) {
       WorkList = propagateShapeForward(WorkList);
@@ -1014,7 +1040,8 @@ class LowerMatrixIntrinsics {
 
     // Third, try to fuse candidates.
     for (CallInst *CI : MaybeFusableInsts)
-      LowerMatrixMultiplyFused(CI, FusedInsts, LifetimeEnds);
+      if (!FusedInsts.contains(CI))
+        LowerMatrixMultiplyFused(CI, FusedInsts, LifetimeEnds);
 
     Changed = !FusedInsts.empty();
 
@@ -1477,7 +1504,7 @@ class LowerMatrixIntrinsics {
                         m_Value(Arg)))) {
         auto *NewLoad = Builder.CreateLoad(Op->getType(), Arg);
         Op->replaceAllUsesWith(NewLoad);
-        cast<Instruction>(Op)->eraseFromParent();
+        eraseFromParentAndRemoveFromShapeMap(cast<Instruction>(Op));
         return;
       } else if (match(Op, m_Intrinsic<Intrinsic::matrix_transpose>(
                                m_Value(Arg)))) {
@@ -1846,15 +1873,15 @@ class LowerMatrixIntrinsics {
     // Mark eliminated instructions as fused and remove them.
     FusedInsts.insert(Store);
     FusedInsts.insert(MatMul);
-    Store->eraseFromParent();
-    MatMul->eraseFromParent();
+    eraseFromParentAndRemoveFromShapeMap(Store);
+    eraseFromParentAndRemoveFromShapeMap(MatMul);
     if (LoadOp0->hasNUses(0)) {
       FusedInsts.insert(LoadOp0);
-      LoadOp0->eraseFromParent();
+      eraseFromParentAndRemoveFromShapeMap(LoadOp0);
     }
     if (LoadOp1 != LoadOp0 && LoadOp1->hasNUses(0)) {
       FusedInsts.insert(LoadOp1);
-      LoadOp1->eraseFromParent();
+      eraseFromParentAndRemoveFromShapeMap(LoadOp1);
     }
   }
 
@@ -2617,19 +2644,8 @@ class LowerMatrixIntrinsics {
 PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
                                                  FunctionAnalysisManager &AM) {
   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
-  OptimizationRemarkEmitter *ORE = nullptr;
-  AAResults *AA = nullptr;
-  DominatorTree *DT = nullptr;
-  LoopInfo *LI = nullptr;
-
-  if (!Minimal) {
-    ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-    AA = &AM.getResult<AAManager>(F);
-    DT = &AM.getResult<DominatorTreeAnalysis>(F);
-    LI = &AM.getResult<LoopAnalysis>(F);
-  }
 
-  LowerMatrixIntrinsics LMT(F, TTI, AA, DT, LI, ORE);
+  LowerMatrixIntrinsics LMT(F, TTI, Minimal ? nullptr : &AM);
   if (LMT.Visit()) {
     PreservedAnalyses PA;
     if (!Minimal) {
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index e9e1071ea210c..bb98b3d1c0725 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -68,6 +68,7 @@ static cl::opt<bool> EnableMemCpyOptWithoutLibcalls(
     cl::desc("Enable memcpyopt even when libcalls are disabled"));
 
 STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
+STATISTIC(NumMemMoveInstr, "Number of memmove instructions deleted");
 STATISTIC(NumMemSetInfer, "Number of memsets inferred");
 STATISTIC(NumMoveToCpy, "Number of memmoves converted to memcpy");
 STATISTIC(NumCpyToSet, "Number of memcpys converted to memset");
@@ -786,43 +787,47 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
   // Ensure that the value being stored is something that can be memset'able a
   // byte at a time like "0" or "-1" or any width, as well as things like
   // 0xA0A0A0A0 and 0.0.
-  auto *V = SI->getOperand(0);
-  if (Value *ByteVal = isBytewiseValue(V, DL)) {
-    if (Instruction *I =
-            tryMergingIntoMemset(SI, SI->getPointerOperand(), ByteVal)) {
-      BBI = I->getIterator(); // Don't invalidate iterator.
-      return true;
-    }
+  Value *V = SI->getOperand(0);
+  Value *ByteVal = isBytewiseValue(V, DL);
+  if (!ByteVal)
+    return false;
 
-    // If we have an aggregate, we try to promote it to memset regardless
-    // of opportunity for merging as it can expose optimization opportunities
-    // in subsequent passes.
-    auto *T = V->getType();
-    if (T->isAggregateType()) {
-      uint64_t Size = DL.getTypeStoreSize(T);
-      IRBuilder<> Builder(SI);
-      auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size,
-                                     SI->getAlign());
-      M->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
+  if (Instruction *I =
+          tryMergingIntoMemset(SI, SI->getPointerOperand(), ByteVal)) {
+    BBI = I->getIterator(); // Don't invalidate iterator.
+    return true;
+  }
 
-      LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
+  // If we have an aggregate, we try to promote it to memset regardless
+  // of opportunity for merging as it can expose optimization opportunities
+  // in subsequent passes.
+  auto *T = V->getType();
+  if (!T->isAggregateType())
+    return false;
 
-      // The newly inserted memset is immediately overwritten by the original
-      // store, so we do not need to rename uses.
-      auto *StoreDef = cast<MemoryDef>(MSSA->getMemoryAccess(SI));
-      auto *NewAccess = MSSAU->createMemoryAccessBefore(M, nullptr, StoreDef);
-      MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/false);
+  TypeSize Size = DL.getTypeStoreSize(T);
+  if (Size.isScalable())
+    return false;
 
-      eraseInstruction(SI);
-      NumMemSetInfer++;
+  IRBuilder<> Builder(SI);
+  auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size,
+                                 SI->getAlign());
+  M->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
 
-      // Make sure we do not invalidate the iterator.
-      BBI = M->getIterator();
-      return true;
-    }
-  }
+  LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
 
-  return false;
+  // The newly inserted memset is immediately overwritten by the original
+  // store, so we do not need to rename uses.
+  auto *StoreDef = cast<MemoryDef>(MSSA->getMemoryAccess(SI));
+  auto *NewAccess = MSSAU->createMemoryAccessBefore(M, nullptr, StoreDef);
+  MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/false);
+
+  eraseInstruction(SI);
+  NumMemSetInfer++;
+
+  // Make sure we do not invalidate the iterator.
+  BBI = M->getIterator();
+  return true;
 }
 
 bool MemCpyOptPass::processMemSet(MemSetInst *MSI, BasicBlock::iterator &BBI) {
@@ -1841,12 +1846,75 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
   return false;
 }
 
+/// Memmove calls with overlapping src/dest buffers that come after a memset may
+/// be removed.
+bool MemCpyOptPass::isMemMoveMemSetDependency(MemMoveInst *M) {
+  const auto &DL = M->getDataLayout();
+  MemoryUseOrDef *MemMoveAccess = MSSA->getMemoryAccess(M);
+  if (!MemMoveAccess)
+    return false;
+
+  // The memmove is of form memmove(x, x + A, B).
+  MemoryLocation SourceLoc = MemoryLocation::getForSource(M);
+  auto *MemMoveSourceOp = M->getSource();
+  auto *Source = dyn_cast<GEPOperator>(MemMoveSourceOp);
+  if (!Source)
+    return false;
+
+  APInt Offset(DL.getIndexTypeSizeInBits(Source->getType()), 0);
+  LocationSize MemMoveLocSize = SourceLoc.Size;
+  if (Source->getPointerOperand() != M->getDest() ||
+      !MemMoveLocSize.hasValue() ||
+      !Source->accumulateConstantOffset(DL, Offset) || Offset.isNegative()) {
+    return false;
+  }
+
+  uint64_t MemMoveSize = MemMoveLocSize.getValue();
+  LocationSize TotalSize =
+      LocationSize::precise(Offset.getZExtValue() + MemMoveSize);
+  MemoryLocation CombinedLoc(M->getDest(), TotalSize);
+
+  // The first dominating clobbering MemoryAccess for the combined location
+  // needs to be a memset.
+  BatchAAResults BAA(*AA);
+  MemoryAccess *FirstDef = MemMoveAccess->getDefiningAccess();
+  auto *DestClobber = dyn_cast<MemoryDef>(
+      MSSA->getWalker()->getClobberingMemoryAccess(FirstDef, CombinedLoc, BAA));
+  if (!DestClobber)
+    return false;
+
+  auto *MS = dyn_cast_or_null<MemSetInst>(DestClobber->getMemoryInst());
+  if (!MS)
+    return false;
+
+  // Memset length must be sufficiently large.
+  auto *MemSetLength = dyn_cast<ConstantInt>(MS->getLength());
+  if (!MemSetLength || MemSetLength->getZExtValue() < MemMoveSize)
+    return false;
+
+  // The destination buffer must have been memset'd.
+  if (!BAA.isMustAlias(MS->getDest(), M->getDest()))
+    return false;
+
+  return true;
+}
+
 /// Transforms memmove calls to memcpy calls when the src/dst are guaranteed
 /// not to alias.
-bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
+bool MemCpyOptPass::processMemMove(MemMoveInst *M, BasicBlock::iterator &BBI) {
   // See if the source could be modified by this memmove potentially.
-  if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(M))))
+  if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(M)))) {
+    // On the off-chance the memmove clobbers src with previously memset'd
+    // bytes, the memmove may be redundant.
+    if (!M->isVolatile() && isMemMoveMemSetDependency(M)) {
+      LLVM_DEBUG(dbgs() << "Removed redundant memmove.\n");
+      ++BBI;
+      eraseInstruction(M);
+      ++NumMemMoveInstr;
+      return true;
+    }
     return false;
+  }
 
   LLVM_DEBUG(dbgs() << "MemCpyOptPass: Optimizing memmove -> memcpy: " << *M
                     << "\n");
@@ -2064,7 +2132,7 @@ bool MemCpyOptPass::iterateOnFunction(Function &F) {
       else if (auto *M = dyn_cast<MemCpyInst>(I))
         RepeatInstruction = processMemCpy(M, BI);
       else if (auto *M = dyn_cast<MemMoveInst>(I))
-        RepeatInstruction = processMemMove(M);
+        RepeatInstruction = processMemMove(M, BI);
       else if (auto *CB = dyn_cast<CallBase>(I)) {
         for (unsigned i = 0, e = CB->arg_size(); i != e; ++i) {
           if (CB->isByValArgument(i))
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index aa3cbc5e4bddc..d8ef450eeb9a1 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -1248,8 +1248,9 @@ static BasicBlock *buildClonedLoopBlocks(
       assert(VMap.lookup(&I) == &ClonedI && "Mismatch in the value map!");
 
       // Forget SCEVs based on exit phis in case SCEV looked through the phi.
-      if (SE && isa<PHINode>(I))
-        SE->forgetValue(&I);
+      if (SE)
+        if (auto *PN = dyn_cast<PHINode>(&I))
+          SE->forgetLcssaPhiWithNewPredecessor(&L, PN);
 
       BasicBlock::iterator InsertPt = MergeBB->getFirstInsertionPt();
 
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 01090b54e5afa..b1f742b838f2a 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -686,8 +686,8 @@ void StructurizeCFG::delPhiValues(BasicBlock *From, BasicBlock *To) {
 /// Add a dummy PHI value as soon as we knew the new predecessor
 void StructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
   for (PHINode &Phi : To->phis()) {
-    Value *Undef = UndefValue::get(Phi.getType());
-    Phi.addIncoming(Undef, From);
+    Value *Poison = PoisonValue::get(Phi.getType());
+    Phi.addIncoming(Poison, From);
   }
   AddedPhis[To].push_back(From);
 }
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index e4f4052e5e481..287538179af8f 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1545,20 +1545,20 @@ static Value *emitLibCall(LibFunc TheLibFunc, Type *ReturnType,
 
 Value *llvm::emitStrLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL,
                         const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Ptr->getType();
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_strlen, SizeTTy, CharPtrTy, Ptr, B, TLI);
 }
 
 Value *llvm::emitStrDup(Value *Ptr, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Ptr->getType();
   return emitLibCall(LibFunc_strdup, CharPtrTy, CharPtrTy, Ptr, B, TLI);
 }
 
 Value *llvm::emitStrChr(Value *Ptr, char C, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Ptr->getType();
   Type *IntTy = getIntTy(B, TLI);
   return emitLibCall(LibFunc_strchr, CharPtrTy, {CharPtrTy, IntTy},
                      {Ptr, ConstantInt::get(IntTy, C)}, B, TLI);
@@ -1566,7 +1566,8 @@ Value *llvm::emitStrChr(Value *Ptr, char C, IRBuilderBase &B,
 
 Value *llvm::emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
                          const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Ptr1->getType();
+  assert(CharPtrTy == Ptr2->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(
@@ -1578,20 +1579,23 @@ Value *llvm::emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
 Value *llvm::emitStrCpy(Value *Dst, Value *Src, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI) {
   Type *CharPtrTy = Dst->getType();
+  assert(CharPtrTy == Src->getType());
   return emitLibCall(LibFunc_strcpy, CharPtrTy, {CharPtrTy, CharPtrTy},
                      {Dst, Src}, B, TLI);
 }
 
 Value *llvm::emitStpCpy(Value *Dst, Value *Src, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dst->getType();
+  assert(CharPtrTy == Src->getType());
   return emitLibCall(LibFunc_stpcpy, CharPtrTy, {CharPtrTy, CharPtrTy},
                      {Dst, Src}, B, TLI);
 }
 
 Value *llvm::emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dst->getType();
+  assert(CharPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_strncpy, CharPtrTy, {CharPtrTy, CharPtrTy, SizeTTy},
                      {Dst, Src, Len}, B, TLI);
@@ -1599,7 +1603,8 @@ Value *llvm::emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitStpNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dst->getType();
+  assert(CharPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_stpncpy, CharPtrTy, {CharPtrTy, CharPtrTy, SizeTTy},
                      {Dst, Src, Len}, B, TLI);
@@ -1615,7 +1620,8 @@ Value *llvm::emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
   AttributeList AS;
   AS = AttributeList::get(M->getContext(), AttributeList::FunctionIndex,
                           Attribute::NoUnwind);
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Dst->getType();
+  assert(VoidPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   FunctionCallee MemCpy = getOrInsertLibFunc(M, *TLI, LibFunc_memcpy_chk,
       AttributeList::get(M->getContext(), AS), VoidPtrTy,
@@ -1629,7 +1635,8 @@ Value *llvm::emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
 
 Value *llvm::emitMemPCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
                          const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Dst->getType();
+  assert(VoidPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_mempcpy, VoidPtrTy,
                      {VoidPtrTy, VoidPtrTy, SizeTTy},
@@ -1638,7 +1645,7 @@ Value *llvm::emitMemPCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
                         const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Ptr->getType();
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_memchr, VoidPtrTy,
@@ -1648,7 +1655,7 @@ Value *llvm::emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
                         const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Ptr->getType();
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_memrchr, VoidPtrTy,
@@ -1658,7 +1665,8 @@ Value *llvm::emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
                         const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Ptr1->getType();
+  assert(VoidPtrTy == Ptr2->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_memcmp, IntTy,
@@ -1668,7 +1676,8 @@ Value *llvm::emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
                       const DataLayout &DL, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Ptr1->getType();
+  assert(VoidPtrTy == Ptr2->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_bcmp, IntTy,
@@ -1678,7 +1687,8 @@ Value *llvm::emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
 
 Value *llvm::emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len,
                          IRBuilderBase &B, const TargetLibraryInfo *TLI) {
-  Type *VoidPtrTy = B.getPtrTy();
+  Type *VoidPtrTy = Ptr1->getType();
+  assert(VoidPtrTy == Ptr2->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_memccpy, VoidPtrTy,
@@ -1689,7 +1699,8 @@ Value *llvm::emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len,
 Value *llvm::emitSNPrintf(Value *Dest, Value *Size, Value *Fmt,
                           ArrayRef<Value *> VariadicArgs, IRBuilderBase &B,
                           const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Fmt->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   SmallVector<Value *, 8> Args{Dest, Size, Fmt};
@@ -1702,7 +1713,7 @@ Value *llvm::emitSNPrintf(Value *Dest, Value *Size, Value *Fmt,
 Value *llvm::emitSPrintf(Value *Dest, Value *Fmt,
                          ArrayRef<Value *> VariadicArgs, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
   Type *IntTy = getIntTy(B, TLI);
   SmallVector<Value *, 8> Args{Dest, Fmt};
   llvm::append_range(Args, VariadicArgs);
@@ -1713,7 +1724,8 @@ Value *llvm::emitSPrintf(Value *Dest, Value *Fmt,
 
 Value *llvm::emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B,
                         const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Src->getType());
   return emitLibCall(LibFunc_strcat, CharPtrTy,
                      {CharPtrTy, CharPtrTy},
                      {Dest, Src}, B, TLI);
@@ -1721,7 +1733,8 @@ Value *llvm::emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B,
 
 Value *llvm::emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_strlcpy, SizeTTy,
                      {CharPtrTy, CharPtrTy, SizeTTy},
@@ -1730,7 +1743,8 @@ Value *llvm::emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
 
 Value *llvm::emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_strlcat, SizeTTy,
                      {CharPtrTy, CharPtrTy, SizeTTy},
@@ -1739,7 +1753,8 @@ Value *llvm::emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
 
 Value *llvm::emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
                          const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Src->getType());
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(LibFunc_strncat, CharPtrTy,
                      {CharPtrTy, CharPtrTy, SizeTTy},
@@ -1748,7 +1763,8 @@ Value *llvm::emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
 
 Value *llvm::emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList,
                            IRBuilderBase &B, const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Fmt->getType());
   Type *IntTy = getIntTy(B, TLI);
   Type *SizeTTy = getSizeTTy(B, TLI);
   return emitLibCall(
@@ -1759,7 +1775,8 @@ Value *llvm::emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList,
 
 Value *llvm::emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList,
                           IRBuilderBase &B, const TargetLibraryInfo *TLI) {
-  Type *CharPtrTy = B.getPtrTy();
+  Type *CharPtrTy = Dest->getType();
+  assert(CharPtrTy == Fmt->getType());
   Type *IntTy = getIntTy(B, TLI);
   return emitLibCall(LibFunc_vsprintf, IntTy,
                      {CharPtrTy, CharPtrTy, VAList->getType()},
@@ -1912,8 +1929,8 @@ Value *llvm::emitPutS(Value *Str, IRBuilderBase &B,
 
   Type *IntTy = getIntTy(B, TLI);
   StringRef PutsName = TLI->getName(LibFunc_puts);
-  FunctionCallee PutS = getOrInsertLibFunc(M, *TLI, LibFunc_puts, IntTy,
-                                           B.getPtrTy());
+  FunctionCallee PutS =
+      getOrInsertLibFunc(M, *TLI, LibFunc_puts, IntTy, Str->getType());
   inferNonMandatoryLibFuncAttrs(M, PutsName, *TLI);
   CallInst *CI = B.CreateCall(PutS, Str, PutsName);
   if (const Function *F =
@@ -1951,7 +1968,7 @@ Value *llvm::emitFPutS(Value *Str, Value *File, IRBuilderBase &B,
   Type *IntTy = getIntTy(B, TLI);
   StringRef FPutsName = TLI->getName(LibFunc_fputs);
   FunctionCallee F = getOrInsertLibFunc(M, *TLI, LibFunc_fputs, IntTy,
-                                        B.getPtrTy(), File->getType());
+                                        Str->getType(), File->getType());
   if (File->getType()->isPointerTy())
     inferNonMandatoryLibFuncAttrs(M, FPutsName, *TLI);
   CallInst *CI = B.CreateCall(F, {Str, File}, FPutsName);
@@ -1970,9 +1987,9 @@ Value *llvm::emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilderBase &B,
 
   Type *SizeTTy = getSizeTTy(B, TLI);
   StringRef FWriteName = TLI->getName(LibFunc_fwrite);
-  FunctionCallee F = getOrInsertLibFunc(M, *TLI, LibFunc_fwrite,
-                                        SizeTTy, B.getPtrTy(), SizeTTy,
-                                        SizeTTy, File->getType());
+  FunctionCallee F =
+      getOrInsertLibFunc(M, *TLI, LibFunc_fwrite, SizeTTy, Ptr->getType(),
+                         SizeTTy, SizeTTy, File->getType());
 
   if (File->getType()->isPointerTy())
     inferNonMandatoryLibFuncAttrs(M, FWriteName, *TLI);
@@ -1986,16 +2003,16 @@ Value *llvm::emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL,
-                        const TargetLibraryInfo *TLI) {
+Value *llvm::emitMalloc(Type *RetTy, Value *Num, IRBuilderBase &B,
+                        const DataLayout &DL, const TargetLibraryInfo *TLI) {
   Module *M = B.GetInsertBlock()->getModule();
   if (!isLibFuncEmittable(M, TLI, LibFunc_malloc))
     return nullptr;
 
   StringRef MallocName = TLI->getName(LibFunc_malloc);
   Type *SizeTTy = getSizeTTy(B, TLI);
-  FunctionCallee Malloc = getOrInsertLibFunc(M, *TLI, LibFunc_malloc,
-                                             B.getPtrTy(), SizeTTy);
+  FunctionCallee Malloc =
+      getOrInsertLibFunc(M, *TLI, LibFunc_malloc, RetTy, SizeTTy);
   inferNonMandatoryLibFuncAttrs(M, MallocName, *TLI);
   CallInst *CI = B.CreateCall(Malloc, Num, MallocName);
 
@@ -2006,16 +2023,16 @@ Value *llvm::emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL,
   return CI;
 }
 
-Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
-                        const TargetLibraryInfo &TLI, unsigned AddrSpace) {
+Value *llvm::emitCalloc(Type *RetTy, Value *Num, Value *Size, IRBuilderBase &B,
+                        const TargetLibraryInfo &TLI) {
   Module *M = B.GetInsertBlock()->getModule();
   if (!isLibFuncEmittable(M, &TLI, LibFunc_calloc))
     return nullptr;
 
   StringRef CallocName = TLI.getName(LibFunc_calloc);
   Type *SizeTTy = getSizeTTy(B, &TLI);
-  FunctionCallee Calloc = getOrInsertLibFunc(
-      M, TLI, LibFunc_calloc, B.getPtrTy(AddrSpace), SizeTTy, SizeTTy);
+  FunctionCallee Calloc =
+      getOrInsertLibFunc(M, TLI, LibFunc_calloc, RetTy, SizeTTy, SizeTTy);
   inferNonMandatoryLibFuncAttrs(M, CallocName, TLI);
   CallInst *CI = B.CreateCall(Calloc, {Num, Size}, CallocName);
 
@@ -2026,7 +2043,8 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+Value *llvm::emitHotColdSizeReturningNew(Type *RetPtrTy, Value *Num,
+                                         IRBuilderBase &B,
                                          const TargetLibraryInfo *TLI,
                                          LibFunc SizeFeedbackNewFunc,
                                          uint8_t HotCold) {
@@ -2038,7 +2056,7 @@ Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
 
   // __sized_ptr_t struct return type { void*, size_t }
   StructType *SizedPtrT =
-      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+      StructType::get(M->getContext(), {RetPtrTy, Num->getType()});
   FunctionCallee Func =
       M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
@@ -2050,8 +2068,8 @@ Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
-                                                IRBuilderBase &B,
+Value *llvm::emitHotColdSizeReturningNewAligned(Type *RetPtrTy, Value *Num,
+                                                Value *Align, IRBuilderBase &B,
                                                 const TargetLibraryInfo *TLI,
                                                 LibFunc SizeFeedbackNewFunc,
                                                 uint8_t HotCold) {
@@ -2063,7 +2081,7 @@ Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
 
   // __sized_ptr_t struct return type { void*, size_t }
   StructType *SizedPtrT =
-      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+      StructType::get(M->getContext(), {RetPtrTy, Num->getType()});
   FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
                                                Align->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
@@ -2076,7 +2094,7 @@ Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
   return CI;
 }
 
-Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
+Value *llvm::emitHotColdNew(Type *RetTy, Value *Num, IRBuilderBase &B,
                             const TargetLibraryInfo *TLI, LibFunc NewFunc,
                             uint8_t HotCold) {
   Module *M = B.GetInsertBlock()->getModule();
@@ -2084,8 +2102,8 @@ Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
     return nullptr;
 
   StringRef Name = TLI->getName(NewFunc);
-  FunctionCallee Func = M->getOrInsertFunction(Name, B.getPtrTy(),
-                                               Num->getType(), B.getInt8Ty());
+  FunctionCallee Func =
+      M->getOrInsertFunction(Name, RetTy, Num->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
   CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, Name);
 
@@ -2096,7 +2114,8 @@ Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
+Value *llvm::emitHotColdNewNoThrow(Type *RetTy, Value *Num, Value *NoThrow,
+                                   IRBuilderBase &B,
                                    const TargetLibraryInfo *TLI,
                                    LibFunc NewFunc, uint8_t HotCold) {
   Module *M = B.GetInsertBlock()->getModule();
@@ -2104,9 +2123,8 @@ Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
     return nullptr;
 
   StringRef Name = TLI->getName(NewFunc);
-  FunctionCallee Func =
-      M->getOrInsertFunction(Name, B.getPtrTy(), Num->getType(),
-                             NoThrow->getType(), B.getInt8Ty());
+  FunctionCallee Func = M->getOrInsertFunction(
+      Name, RetTy, Num->getType(), NoThrow->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
   CallInst *CI = B.CreateCall(Func, {Num, NoThrow, B.getInt8(HotCold)}, Name);
 
@@ -2117,7 +2135,8 @@ Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
+Value *llvm::emitHotColdNewAligned(Type *RetTy, Value *Num, Value *Align,
+                                   IRBuilderBase &B,
                                    const TargetLibraryInfo *TLI,
                                    LibFunc NewFunc, uint8_t HotCold) {
   Module *M = B.GetInsertBlock()->getModule();
@@ -2125,8 +2144,8 @@ Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
     return nullptr;
 
   StringRef Name = TLI->getName(NewFunc);
-  FunctionCallee Func = M->getOrInsertFunction(
-      Name, B.getPtrTy(), Num->getType(), Align->getType(), B.getInt8Ty());
+  FunctionCallee Func = M->getOrInsertFunction(Name, RetTy, Num->getType(),
+                                               Align->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
   CallInst *CI = B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, Name);
 
@@ -2137,7 +2156,7 @@ Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
   return CI;
 }
 
-Value *llvm::emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
+Value *llvm::emitHotColdNewAlignedNoThrow(Type *RetTy, Value *Num, Value *Align,
                                           Value *NoThrow, IRBuilderBase &B,
                                           const TargetLibraryInfo *TLI,
                                           LibFunc NewFunc, uint8_t HotCold) {
@@ -2146,9 +2165,9 @@ Value *llvm::emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
     return nullptr;
 
   StringRef Name = TLI->getName(NewFunc);
-  FunctionCallee Func = M->getOrInsertFunction(
-      Name, B.getPtrTy(), Num->getType(), Align->getType(),
-      NoThrow->getType(), B.getInt8Ty());
+  FunctionCallee Func =
+      M->getOrInsertFunction(Name, RetTy, Num->getType(), Align->getType(),
+                             NoThrow->getType(), B.getInt8Ty());
   inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
   CallInst *CI =
       B.CreateCall(Func, {Num, Align, NoThrow, B.getInt8(HotCold)}, Name);
diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
index 17cba2e642a19..725a0eb97eae1 100644
--- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
@@ -692,14 +692,14 @@ bool llvm::tryPromoteCall(CallBase &CB) {
   if (!VTableEntryLoad)
     return false; // Not a vtable entry load.
   Value *VTableEntryPtr = VTableEntryLoad->getPointerOperand();
-  APInt VTableOffset(DL.getTypeSizeInBits(VTableEntryPtr->getType()), 0);
+  APInt VTableOffset(DL.getIndexTypeSizeInBits(VTableEntryPtr->getType()), 0);
   Value *VTableBasePtr = VTableEntryPtr->stripAndAccumulateConstantOffsets(
       DL, VTableOffset, /* AllowNonInbounds */ true);
   LoadInst *VTablePtrLoad = dyn_cast<LoadInst>(VTableBasePtr);
   if (!VTablePtrLoad)
     return false; // Not a vtable load.
   Value *Object = VTablePtrLoad->getPointerOperand();
-  APInt ObjectOffset(DL.getTypeSizeInBits(Object->getType()), 0);
+  APInt ObjectOffset(DL.getIndexTypeSizeInBits(Object->getType()), 0);
   Value *ObjectBase = Object->stripAndAccumulateConstantOffsets(
       DL, ObjectOffset, /* AllowNonInbounds */ true);
   if (!(isa<AllocaInst>(ObjectBase) && ObjectOffset == 0))
@@ -710,9 +710,9 @@ bool llvm::tryPromoteCall(CallBase &CB) {
   BasicBlock::iterator BBI(VTablePtrLoad);
   Value *VTablePtr = FindAvailableLoadedValue(
       VTablePtrLoad, VTablePtrLoad->getParent(), BBI, 0, nullptr, nullptr);
-  if (!VTablePtr)
+  if (!VTablePtr || !VTablePtr->getType()->isPointerTy())
     return false; // No vtable found.
-  APInt VTableOffsetGVBase(DL.getTypeSizeInBits(VTablePtr->getType()), 0);
+  APInt VTableOffsetGVBase(DL.getIndexTypeSizeInBits(VTablePtr->getType()), 0);
   Value *VTableGVBase = VTablePtr->stripAndAccumulateConstantOffsets(
       DL, VTableOffsetGVBase, /* AllowNonInbounds */ true);
   GlobalVariable *GV = dyn_cast<GlobalVariable>(VTableGVBase);
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index cb6a4e34c226e..6dc5f601b7fca 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -43,21 +43,16 @@ using namespace llvm;
 /// See comments in Cloning.h.
 BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
                                   const Twine &NameSuffix, Function *F,
-                                  ClonedCodeInfo *CodeInfo,
-                                  DebugInfoFinder *DIFinder) {
+                                  ClonedCodeInfo *CodeInfo) {
   BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "", F);
   NewBB->IsNewDbgInfoFormat = BB->IsNewDbgInfoFormat;
   if (BB->hasName())
     NewBB->setName(BB->getName() + NameSuffix);
 
   bool hasCalls = false, hasDynamicAllocas = false, hasMemProfMetadata = false;
-  Module *TheModule = F ? F->getParent() : nullptr;
 
   // Loop over all instructions, and copy them over.
   for (const Instruction &I : *BB) {
-    if (DIFinder && TheModule)
-      DIFinder->processInstruction(*TheModule, I);
-
     Instruction *NewInst = I.clone();
     if (I.hasName())
       NewInst->setName(I.getName() + NameSuffix);
@@ -157,6 +152,54 @@ DISubprogram *llvm::CollectDebugInfoForCloning(const Function &F,
   return SPClonedWithinModule;
 }
 
+bool llvm::BuildDebugInfoMDMap(DenseMap<const Metadata *, TrackingMDRef> &MD,
+                               CloneFunctionChangeType Changes,
+                               DebugInfoFinder &DIFinder,
+                               DISubprogram *SPClonedWithinModule) {
+  bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
+  if (Changes < CloneFunctionChangeType::DifferentModule &&
+      DIFinder.subprogram_count() > 0) {
+    // Turn on module-level changes, since we need to clone (some of) the
+    // debug info metadata.
+    //
+    // FIXME: Metadata effectively owned by a function should be made
+    // local, and only that local metadata should be cloned.
+    ModuleLevelChanges = true;
+
+    auto mapToSelfIfNew = [&MD](MDNode *N) {
+      // Avoid clobbering an existing mapping.
+      (void)MD.try_emplace(N, N);
+    };
+
+    // Avoid cloning types, compile units, and (other) subprograms.
+    SmallPtrSet<const DISubprogram *, 16> MappedToSelfSPs;
+    for (DISubprogram *ISP : DIFinder.subprograms()) {
+      if (ISP != SPClonedWithinModule) {
+        mapToSelfIfNew(ISP);
+        MappedToSelfSPs.insert(ISP);
+      }
+    }
+
+    // If a subprogram isn't going to be cloned skip its lexical blocks as well.
+    for (DIScope *S : DIFinder.scopes()) {
+      auto *LScope = dyn_cast<DILocalScope>(S);
+      if (LScope && MappedToSelfSPs.count(LScope->getSubprogram()))
+        mapToSelfIfNew(S);
+    }
+
+    for (DICompileUnit *CU : DIFinder.compile_units())
+      mapToSelfIfNew(CU);
+
+    for (DIType *Type : DIFinder.types())
+      mapToSelfIfNew(Type);
+  } else {
+    assert(!SPClonedWithinModule &&
+           "Subprogram should be in DIFinder->subprogram_count()...");
+  }
+
+  return ModuleLevelChanges;
+}
+
 // Clone OldFunc into NewFunc, transforming the old arguments into references to
 // VMap values.
 void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
@@ -215,16 +258,27 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   DISubprogram *SPClonedWithinModule =
       CollectDebugInfoForCloning(*OldFunc, Changes, DIFinder);
 
+  ModuleLevelChanges =
+      BuildDebugInfoMDMap(VMap.MD(), Changes, DIFinder, SPClonedWithinModule);
+
+  const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
+  // Duplicate the metadata that is attached to the cloned function.
+  // Subprograms/CUs/types that were already mapped to themselves won't be
+  // duplicated.
+  SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
+  OldFunc->getAllMetadata(MDs);
+  for (auto MD : MDs) {
+    NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag,
+                                                TypeMapper, Materializer));
+  }
+
   // Loop over all of the basic blocks in the function, cloning them as
   // appropriate.  Note that we save BE this way in order to handle cloning of
   // recursive functions into themselves.
   for (const BasicBlock &BB : *OldFunc) {
 
     // Create a new basic block and copy instructions into it!
-    // NOTE: don't pass DIFinder because instructions' debug info was processed
-    // in ProcessSubprogramAttachment. This will be cleaned up further.
-    BasicBlock *CBB =
-        CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo, nullptr);
+    BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo);
 
     // Add basic block mapping.
     VMap[&BB] = CBB;
@@ -246,57 +300,6 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
       Returns.push_back(RI);
   }
 
-  if (Changes < CloneFunctionChangeType::DifferentModule &&
-      DIFinder.subprogram_count() > 0) {
-    // Turn on module-level changes, since we need to clone (some of) the
-    // debug info metadata.
-    //
-    // FIXME: Metadata effectively owned by a function should be made
-    // local, and only that local metadata should be cloned.
-    ModuleLevelChanges = true;
-
-    auto mapToSelfIfNew = [&VMap](MDNode *N) {
-      // Avoid clobbering an existing mapping.
-      (void)VMap.MD().try_emplace(N, N);
-    };
-
-    // Avoid cloning types, compile units, and (other) subprograms.
-    SmallPtrSet<const DISubprogram *, 16> MappedToSelfSPs;
-    for (DISubprogram *ISP : DIFinder.subprograms()) {
-      if (ISP != SPClonedWithinModule) {
-        mapToSelfIfNew(ISP);
-        MappedToSelfSPs.insert(ISP);
-      }
-    }
-
-    // If a subprogram isn't going to be cloned skip its lexical blocks as well.
-    for (DIScope *S : DIFinder.scopes()) {
-      auto *LScope = dyn_cast<DILocalScope>(S);
-      if (LScope && MappedToSelfSPs.count(LScope->getSubprogram()))
-        mapToSelfIfNew(S);
-    }
-
-    for (DICompileUnit *CU : DIFinder.compile_units())
-      mapToSelfIfNew(CU);
-
-    for (DIType *Type : DIFinder.types())
-      mapToSelfIfNew(Type);
-  } else {
-    assert(!SPClonedWithinModule &&
-           "Subprogram should be in DIFinder->subprogram_count()...");
-  }
-
-  const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
-  // Duplicate the metadata that is attached to the cloned function.
-  // Subprograms/CUs/types that were already mapped to themselves won't be
-  // duplicated.
-  SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
-  OldFunc->getAllMetadata(MDs);
-  for (auto MD : MDs) {
-    NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag,
-                                                TypeMapper, Materializer));
-  }
-
   // Loop over all of the instructions in the new function, fixing up operand
   // references as we go. This uses VMap to do all the hard work.
   for (Function::iterator
diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
index 760341a29d8c8..6d4026e8209de 100644
--- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
@@ -83,6 +83,13 @@ int FunctionComparator::cmpAPInts(const APInt &L, const APInt &R) const {
   return 0;
 }
 
+int FunctionComparator::cmpConstantRanges(const ConstantRange &L,
+                                          const ConstantRange &R) const {
+  if (int Res = cmpAPInts(L.getLower(), R.getLower()))
+    return Res;
+  return cmpAPInts(L.getUpper(), R.getUpper());
+}
+
 int FunctionComparator::cmpAPFloats(const APFloat &L, const APFloat &R) const {
   // Floats are ordered first by semantics (i.e. float, double, half, etc.),
   // then by value interpreted as a bitstring (aka APInt).
@@ -147,12 +154,22 @@ int FunctionComparator::cmpAttrs(const AttributeList L,
         if (LA.getKindAsEnum() != RA.getKindAsEnum())
           return cmpNumbers(LA.getKindAsEnum(), RA.getKindAsEnum());
 
-        const ConstantRange &LCR = LA.getRange();
-        const ConstantRange &RCR = RA.getRange();
-        if (int Res = cmpAPInts(LCR.getLower(), RCR.getLower()))
+        if (int Res = cmpConstantRanges(LA.getRange(), RA.getRange()))
           return Res;
-        if (int Res = cmpAPInts(LCR.getUpper(), RCR.getUpper()))
+        continue;
+      } else if (LA.isConstantRangeListAttribute() &&
+                 RA.isConstantRangeListAttribute()) {
+        if (LA.getKindAsEnum() != RA.getKindAsEnum())
+          return cmpNumbers(LA.getKindAsEnum(), RA.getKindAsEnum());
+
+        ArrayRef<ConstantRange> CRL = LA.getValueAsConstantRangeList();
+        ArrayRef<ConstantRange> CRR = RA.getValueAsConstantRangeList();
+        if (int Res = cmpNumbers(CRL.size(), CRR.size()))
           return Res;
+
+        for (const auto &[L, R] : zip(CRL, CRR))
+          if (int Res = cmpConstantRanges(L, R))
+            return Res;
         continue;
       }
       if (LA < RA)
@@ -441,9 +458,7 @@ int FunctionComparator::cmpConstants(const Constant *L,
       if (InRangeL) {
         if (!InRangeR)
           return 1;
-        if (int Res = cmpAPInts(InRangeL->getLower(), InRangeR->getLower()))
-          return Res;
-        if (int Res = cmpAPInts(InRangeL->getUpper(), InRangeR->getUpper()))
+        if (int Res = cmpConstantRanges(*InRangeL, *InRangeR))
           return Res;
       } else if (InRangeR) {
         return -1;
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 04042e71a2b82..fffff295ba926 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -171,14 +171,14 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
       // Users in the OrigPreHeader need to use the value to which the
       // original definitions are mapped and anything else can be handled by
       // the SSAUpdater. To avoid adding PHINodes, check if the value is
-      // available in UserBB, if not substitute undef.
+      // available in UserBB, if not substitute poison.
       Value *NewVal;
       if (UserBB == OrigPreheader)
         NewVal = OrigPreHeaderVal;
       else if (SSA.HasValueForBlock(UserBB))
         NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
       else
-        NewVal = UndefValue::get(OrigHeaderVal->getType());
+        NewVal = PoisonValue::get(OrigHeaderVal->getType());
       DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
     }
 
@@ -194,14 +194,14 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
       // Users in the OrigPreHeader need to use the value to which the
       // original definitions are mapped and anything else can be handled by
       // the SSAUpdater. To avoid adding PHINodes, check if the value is
-      // available in UserBB, if not substitute undef.
+      // available in UserBB, if not substitute poison.
       Value *NewVal;
       if (UserBB == OrigPreheader)
         NewVal = OrigPreHeaderVal;
       else if (SSA.HasValueForBlock(UserBB))
         NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
       else
-        NewVal = UndefValue::get(OrigHeaderVal->getType());
+        NewVal = PoisonValue::get(OrigHeaderVal->getType());
       DVR->replaceVariableLocationOp(OrigHeaderVal, NewVal);
     }
   }
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index 44fdfe530178a..d8298646e18d7 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -777,8 +777,8 @@ INITIALIZE_PASS_BEGIN(LoopSimplify, "loop-simplify",
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_END(LoopSimplify, "loop-simplify",
-                "Canonicalize natural loops", false, false)
+INITIALIZE_PASS_END(LoopSimplify, "loop-simplify", "Canonicalize natural loops",
+                    false, true)
 
 // Publicly exposed interface to pass...
 char &llvm::LoopSimplifyID = LoopSimplify::ID;
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 85d4415970a00..b11d92836a998 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -60,6 +60,7 @@
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include "llvm/Transforms/Utils/SimplifyIndVar.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
@@ -589,10 +590,10 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
                                               : isEpilogProfitable(L);
 
   if (ULO.Runtime &&
-      !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
-                                  EpilogProfitability, ULO.UnrollRemainder,
-                                  ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
-                                  PreserveLCSSA, RemainderLoop)) {
+      !UnrollRuntimeLoopRemainder(
+          L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
+          ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
+          PreserveLCSSA, ULO.SCEVExpansionBudget, RemainderLoop)) {
     if (ULO.Force)
       ULO.Runtime = false;
     else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
index 2c2400d9dd7a8..49209e33f2d1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
@@ -48,6 +48,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <assert.h>
@@ -241,7 +242,8 @@ llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
     if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
                                     /*UseEpilogRemainder*/ true,
                                     UnrollRemainder, /*ForgetAllSCEV*/ false,
-                                    LI, SE, DT, AC, TTI, true, EpilogueLoop)) {
+                                    LI, SE, DT, AC, TTI, true,
+                                    SCEVCheapExpansionBudget, EpilogueLoop)) {
       LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
                            "generated when assuming runtime trip count\n");
       return LoopUnrollResult::Unmodified;
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 0179d350b9e0e..b0bc55cd64c37 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -582,7 +582,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
     Loop *L, unsigned Count, bool AllowExpensiveTripCount,
     bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
     LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
-    const TargetTransformInfo *TTI, bool PreserveLCSSA, Loop **ResultLoop) {
+    const TargetTransformInfo *TTI, bool PreserveLCSSA,
+    unsigned SCEVExpansionBudget, Loop **ResultLoop) {
   LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
   LLVM_DEBUG(L->dump());
   LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -672,8 +673,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
   const DataLayout &DL = Header->getDataLayout();
   SCEVExpander Expander(*SE, DL, "loop-unroll");
   if (!AllowExpensiveTripCount &&
-      Expander.isHighCostExpansion(TripCountSC, L, SCEVCheapExpansionBudget,
-                                   TTI, PreHeaderBR)) {
+      Expander.isHighCostExpansion(TripCountSC, L, SCEVExpansionBudget, TTI,
+                                   PreHeaderBR)) {
     LLVM_DEBUG(dbgs() << "High cost for expanding trip count scev!\n");
     return false;
   }
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index 4225e7e80fda6..81aa7ce1cfe66 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -147,6 +147,16 @@ static bool refineInstruction(SCCPSolver &Solver,
         Changed = true;
       }
     }
+  } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&Inst)) {
+    if (GEP->hasNoUnsignedWrap() || !GEP->hasNoUnsignedSignedWrap())
+      return false;
+
+    if (all_of(GEP->indices(),
+               [&](Value *V) { return GetRange(V).isAllNonNegative(); })) {
+      GEP->setNoWrapFlags(GEP->getNoWrapFlags() |
+                          GEPNoWrapFlags::noUnsignedWrap());
+      Changed = true;
+    }
   }
 
   return Changed;
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index d85e0d9946602..0438ccf36aeaf 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -917,8 +917,9 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
     // Create a bigger, nul-padded array with the same length, SrcLen,
     // as the original string.
     SrcStr.resize(N, '\0');
-    Src = B.CreateGlobalString(SrcStr, "str", /*AddressSpace=*/0,
-                               /*M=*/nullptr, /*AddNull=*/false);
+    Src = B.CreateGlobalString(
+        SrcStr, "str", /*AddressSpace=*/DL.getDefaultGlobalsAddressSpace(),
+        /*M=*/nullptr, /*AddNull=*/false);
   }
 
   Type *PT = Callee->getFunctionType()->getParamType(0);
@@ -1723,7 +1724,8 @@ Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
 
 Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
   if (isa<ConstantPointerNull>(CI->getArgOperand(0)))
-    return copyFlags(*CI, emitMalloc(CI->getArgOperand(1), B, DL, TLI));
+    return copyFlags(*CI, emitMalloc(CI->getArgOperand(0)->getType(),
+                                     CI->getArgOperand(1), B, DL, TLI));
 
   return nullptr;
 }
@@ -1758,122 +1760,124 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
   switch (Func) {
   case LibFunc_Znwm12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
-      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+      return emitHotColdNew(CI->getType(), CI->getArgOperand(0), B, TLI,
                             LibFunc_Znwm12__hot_cold_t, HotCold);
     break;
   case LibFunc_Znwm:
     if (HotCold != NotColdNewHintValue)
-      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+      return emitHotColdNew(CI->getType(), CI->getArgOperand(0), B, TLI,
                             LibFunc_Znwm12__hot_cold_t, HotCold);
     break;
   case LibFunc_Znam12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
-      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+      return emitHotColdNew(CI->getType(), CI->getArgOperand(0), B, TLI,
                             LibFunc_Znam12__hot_cold_t, HotCold);
     break;
   case LibFunc_Znam:
     if (HotCold != NotColdNewHintValue)
-      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+      return emitHotColdNew(CI->getType(), CI->getArgOperand(0), B, TLI,
                             LibFunc_Znam12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmRKSt9nothrow_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamRKSt9nothrow_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmSt11align_val_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamSt11align_val_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
           LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewAlignedNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-          TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
-          HotCold);
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1),
+          CI->getArgOperand(2), B, TLI,
+          LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewAlignedNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-          TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
-          HotCold);
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1),
+          CI->getArgOperand(2), B, TLI,
+          LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
     if (OptimizeExistingHotColdNew)
       return emitHotColdNewAlignedNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-          TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
-          HotCold);
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1),
+          CI->getArgOperand(2), B, TLI,
+          LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdNewAlignedNoThrow(
-          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-          TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
-          HotCold);
+          CI->getType(), CI->getArgOperand(0), CI->getArgOperand(1),
+          CI->getArgOperand(2), B, TLI,
+          LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
     break;
   case LibFunc_size_returning_new:
     if (HotCold != NotColdNewHintValue)
-      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
-                                         LibFunc_size_returning_new_hot_cold,
-                                         HotCold);
+      return emitHotColdSizeReturningNew(
+          CI->getType()->getStructElementType(0), CI->getArgOperand(0), B, TLI,
+          LibFunc_size_returning_new_hot_cold, HotCold);
     break;
   case LibFunc_size_returning_new_hot_cold:
     if (OptimizeExistingHotColdNew)
-      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
-                                         LibFunc_size_returning_new_hot_cold,
-                                         HotCold);
+      return emitHotColdSizeReturningNew(
+          CI->getType()->getStructElementType(0), CI->getArgOperand(0), B, TLI,
+          LibFunc_size_returning_new_hot_cold, HotCold);
     break;
   case LibFunc_size_returning_new_aligned:
     if (HotCold != NotColdNewHintValue)
       return emitHotColdSizeReturningNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType()->getStructElementType(0), CI->getArgOperand(0),
+          CI->getArgOperand(1), B, TLI,
           LibFunc_size_returning_new_aligned_hot_cold, HotCold);
     break;
   case LibFunc_size_returning_new_aligned_hot_cold:
     if (OptimizeExistingHotColdNew)
       return emitHotColdSizeReturningNewAligned(
-          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          CI->getType()->getStructElementType(0), CI->getArgOperand(0),
+          CI->getArgOperand(1), B, TLI,
           LibFunc_size_returning_new_aligned_hot_cold, HotCold);
     break;
   default:
@@ -3345,7 +3349,8 @@ Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
     // printf("%s", str"\n") --> puts(str)
     if (OperandStr.back() == '\n') {
       OperandStr = OperandStr.drop_back();
-      Value *GV = B.CreateGlobalString(OperandStr, "str");
+      Value *GV = B.CreateGlobalString(OperandStr, "str",
+                                       DL.getDefaultGlobalsAddressSpace());
       return copyFlags(*CI, emitPutS(GV, B, TLI));
     }
     return nullptr;
@@ -3357,7 +3362,8 @@ Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
     // Create a string literal with no \n on it.  We expect the constant merge
     // pass to be run after this pass, to merge duplicate strings.
     FormatStr = FormatStr.drop_back();
-    Value *GV = B.CreateGlobalString(FormatStr, "str");
+    Value *GV = B.CreateGlobalString(FormatStr, "str",
+                                     DL.getDefaultGlobalsAddressSpace());
     return copyFlags(*CI, emitPutS(GV, B, TLI));
   }
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d68a26251ac9d..de164ee434d64 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -487,12 +487,11 @@ class InnerLoopVectorizer {
   /// on, while the old loop will be used as the scalar remainder. Control flow
   /// is generated around the vectorized (and scalar epilogue) loops consisting
   /// of various checks and bypasses. Return the pre-header block of the new
-  /// loop and the start value for the canonical induction, if it is != 0. The
-  /// latter is the case when vectorizing the epilogue loop. In the case of
-  /// epilogue vectorization, this function is overriden to handle the more
-  /// complex control flow around the loops.  \p ExpandedSCEVs is used to
-  /// look up SCEV expansions for expressions needed during skeleton creation.
-  virtual std::pair<BasicBlock *, Value *>
+  /// loop. In the case of epilogue vectorization, this function is overriden to
+  /// handle the more complex control flow around the loops. \p ExpandedSCEVs is
+  /// used to look up SCEV expansions for expressions needed during skeleton
+  /// creation.
+  virtual BasicBlock *
   createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs);
 
   /// Fix the vectorized code, taking care of header phi's, and more.
@@ -513,17 +512,21 @@ class InnerLoopVectorizer {
   /// Fix the non-induction PHIs in \p Plan.
   void fixNonInductionPHIs(VPTransformState &State);
 
-  /// Create a new phi node for the induction variable \p OrigPhi to resume
-  /// iteration count in the scalar epilogue, from where the vectorized loop
-  /// left off. \p Step is the SCEV-expanded induction step to use. In cases
-  /// where the loop skeleton is more complicated (i.e., epilogue vectorization)
-  /// and the resume values can come from an additional bypass block, the \p
-  /// AdditionalBypass pair provides information about the bypass block and the
-  /// end value on the edge from bypass to this loop.
-  PHINode *createInductionResumeValue(
-      PHINode *OrigPhi, const InductionDescriptor &ID, Value *Step,
-      ArrayRef<BasicBlock *> BypassBlocks,
-      std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
+  /// Create a ResumePHI VPInstruction for the induction \p InductionPhiIRI to
+  /// resume iteration count in the scalar epilogue from where the vectorized
+  /// loop left off, and add it to the scalar preheader of VPlan. Also creates
+  /// the induction resume value, and the value for the bypass block, if needed.
+  /// \p Step is the SCEV-expanded induction step to use. In cases where the
+  /// loop skeleton is more complicated (i.e., epilogue vectorization) and the
+  /// resume values can come from an additional bypass block,
+  /// \p MainVectorTripCount provides the trip count of the main vector loop,
+  /// used to compute the resume value reaching the scalar loop preheader
+  /// directly from this additional bypass block.
+  void createInductionResumeVPValue(VPIRInstruction *InductionPhiIRI,
+                                    const InductionDescriptor &ID, Value *Step,
+                                    ArrayRef<BasicBlock *> BypassBlocks,
+                                    VPBuilder &ScalarPHBuilder,
+                                    Value *MainVectorTripCount = nullptr);
 
   /// Returns the original loop trip count.
   Value *getTripCount() const { return TripCount; }
@@ -533,6 +536,20 @@ class InnerLoopVectorizer {
   /// count of the original loop for both main loop and epilogue vectorization.
   void setTripCount(Value *TC) { TripCount = TC; }
 
+  // Retrieve the additional bypass value associated with an original
+  /// induction header phi.
+  Value *getInductionAdditionalBypassValue(PHINode *OrigPhi) const {
+    return Induction2AdditionalBypassValue.at(OrigPhi);
+  }
+
+  /// Return the additional bypass block which targets the scalar loop by
+  /// skipping the epilogue loop after completing the main loop.
+  BasicBlock *getAdditionalBypassBlock() const {
+    assert(AdditionalBypassBlock &&
+           "Trying to access AdditionalBypassBlock but it has not been set");
+    return AdditionalBypassBlock;
+  }
+
 protected:
   friend class LoopVectorizationPlanner;
 
@@ -568,13 +585,15 @@ class InnerLoopVectorizer {
 
   /// Create new phi nodes for the induction variables to resume iteration count
   /// in the scalar epilogue, from where the vectorized loop left off.
-  /// In cases where the loop skeleton is more complicated (eg. epilogue
-  /// vectorization) and the resume values can come from an additional bypass
-  /// block, the \p AdditionalBypass pair provides information about the bypass
-  /// block and the end value on the edge from bypass to this loop.
-  void createInductionResumeValues(
-      const SCEV2ValueTy &ExpandedSCEVs,
-      std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
+  /// In cases where the loop skeleton is more complicated (i.e. epilogue
+  /// vectorization), \p MainVectorTripCount provides the trip count of the main
+  /// loop, used to compute these resume values. If \p IVSubset is provided, it
+  /// contains the phi nodes for which resume values are needed, because they
+  /// will generate wide induction phis in the epilogue loop.
+  void
+  createInductionResumeVPValues(const SCEV2ValueTy &ExpandedSCEVs,
+                                Value *MainVectorTripCount = nullptr,
+                                SmallPtrSetImpl<PHINode *> *IVSubset = nullptr);
 
   /// Allow subclasses to override and print debug traces before/after vplan
   /// execution, when trace information is requested.
@@ -664,6 +683,16 @@ class InnerLoopVectorizer {
   /// for cleaning the checks, if vectorization turns out unprofitable.
   GeneratedRTChecks &RTChecks;
 
+  /// Mapping of induction phis to their additional bypass values. They
+  /// need to be added as operands to phi nodes in the scalar loop preheader
+  /// after the epilogue skeleton has been created.
+  DenseMap<PHINode *, Value *> Induction2AdditionalBypassValue;
+
+  /// The additional bypass block which conditionally skips over the epilogue
+  /// loop after executing the main loop. Needed to resume inductions and
+  /// reductions during epilogue vectorization.
+  BasicBlock *AdditionalBypassBlock = nullptr;
+
   VPlan &Plan;
 };
 
@@ -681,10 +710,13 @@ struct EpilogueLoopVectorizationInfo {
   BasicBlock *MemSafetyCheck = nullptr;
   Value *TripCount = nullptr;
   Value *VectorTripCount = nullptr;
+  VPlan &EpiloguePlan;
 
   EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
-                                ElementCount EVF, unsigned EUF)
-      : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF) {
+                                ElementCount EVF, unsigned EUF,
+                                VPlan &EpiloguePlan)
+      : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
+        EpiloguePlan(EpiloguePlan) {
     assert(EUF == 1 &&
            "A high UF for the epilogue loop is likely not beneficial.");
   }
@@ -714,15 +746,15 @@ class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
 
   // Override this function to handle the more complex control flow around the
   // three loops.
-  std::pair<BasicBlock *, Value *> createVectorizedLoopSkeleton(
-      const SCEV2ValueTy &ExpandedSCEVs) final {
+  BasicBlock *
+  createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final {
     return createEpilogueVectorizedLoopSkeleton(ExpandedSCEVs);
   }
 
   /// The interface for creating a vectorized skeleton using one of two
   /// different strategies, each corresponding to one execution of the vplan
   /// as described above.
-  virtual std::pair<BasicBlock *, Value *>
+  virtual BasicBlock *
   createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) = 0;
 
   /// Holds and updates state information required to vectorize the main loop
@@ -751,7 +783,7 @@ class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
                                        EPI, LVL, CM, BFI, PSI, Check, Plan) {}
   /// Implements the interface for creating a vectorized skeleton using the
   /// *main loop* strategy (ie the first pass of vplan execution).
-  std::pair<BasicBlock *, Value *>
+  BasicBlock *
   createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final;
 
 protected:
@@ -786,7 +818,7 @@ class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
   }
   /// Implements the interface for creating a vectorized skeleton using the
   /// *epilogue loop* strategy (ie the second pass of vplan execution).
-  std::pair<BasicBlock *, Value *>
+  BasicBlock *
   createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final;
 
 protected:
@@ -2568,9 +2600,9 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
 void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
   LoopVectorPreHeader = OrigLoop->getLoopPreheader();
   assert(LoopVectorPreHeader && "Invalid loop structure");
-  assert((OrigLoop->getUniqueExitBlock() ||
+  assert((OrigLoop->getUniqueLatchExitBlock() ||
           Cost->requiresScalarEpilogue(VF.isVector())) &&
-         "multiple exit loop without required epilogue?");
+         "loops not exiting via the latch without required epilogue?");
 
   LoopMiddleBlock =
       SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
@@ -2580,20 +2612,22 @@ void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
                  nullptr, Twine(Prefix) + "scalar.ph");
 }
 
-PHINode *InnerLoopVectorizer::createInductionResumeValue(
-    PHINode *OrigPhi, const InductionDescriptor &II, Value *Step,
-    ArrayRef<BasicBlock *> BypassBlocks,
-    std::pair<BasicBlock *, Value *> AdditionalBypass) {
+void InnerLoopVectorizer::createInductionResumeVPValue(
+    VPIRInstruction *InductionPhiRI, const InductionDescriptor &II, Value *Step,
+    ArrayRef<BasicBlock *> BypassBlocks, VPBuilder &ScalarPHBuilder,
+    Value *MainVectorTripCount) {
+  // TODO: Move to LVP or general VPlan construction, once no IR values are
+  // generated.
+  auto *OrigPhi = cast<PHINode>(&InductionPhiRI->getInstruction());
   Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
   assert(VectorTripCount && "Expected valid arguments");
 
   Instruction *OldInduction = Legal->getPrimaryInduction();
-  Value *EndValue = nullptr;
-  Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
-  if (OrigPhi == OldInduction) {
-    // We know what the end value is.
-    EndValue = VectorTripCount;
-  } else {
+  // For the primary induction the end values are known.
+  Value *EndValue = VectorTripCount;
+  Value *EndValueFromAdditionalBypass = MainVectorTripCount;
+  // Otherwise compute them accordingly.
+  if (OrigPhi != OldInduction) {
     IRBuilder<> B(LoopVectorPreHeader->getTerminator());
 
     // Fast-math-flags propagate from the original induction instruction.
@@ -2605,37 +2639,32 @@ PHINode *InnerLoopVectorizer::createInductionResumeValue(
     EndValue->setName("ind.end");
 
     // Compute the end value for the additional bypass (if applicable).
-    if (AdditionalBypass.first) {
-      B.SetInsertPoint(AdditionalBypass.first,
-                       AdditionalBypass.first->getFirstInsertionPt());
+    if (MainVectorTripCount) {
+      B.SetInsertPoint(getAdditionalBypassBlock(),
+                       getAdditionalBypassBlock()->getFirstInsertionPt());
       EndValueFromAdditionalBypass =
-          emitTransformedIndex(B, AdditionalBypass.second, II.getStartValue(),
-                               Step, II.getKind(), II.getInductionBinOp());
+          emitTransformedIndex(B, MainVectorTripCount, II.getStartValue(), Step,
+                               II.getKind(), II.getInductionBinOp());
       EndValueFromAdditionalBypass->setName("ind.end");
     }
   }
 
-  // Create phi nodes to merge from the backedge-taken check block.
-  PHINode *BCResumeVal =
-      PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
-                      LoopScalarPreHeader->getFirstNonPHIIt());
-  // Copy original phi DL over to the new one.
-  BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
-
-  // The new PHI merges the original incoming value, in case of a bypass,
-  // or the value at the end of the vectorized loop.
-  BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
+  auto *ResumePhiRecipe = ScalarPHBuilder.createNaryOp(
+      VPInstruction::ResumePhi,
+      {Plan.getOrAddLiveIn(EndValue), Plan.getOrAddLiveIn(II.getStartValue())},
+      OrigPhi->getDebugLoc(), "bc.resume.val");
+  assert(InductionPhiRI->getNumOperands() == 0 &&
+         "InductionPhiRI should not have any operands");
+  InductionPhiRI->addOperand(ResumePhiRecipe);
 
-  // Fix the scalar body counter (PHI node).
-  // The old induction's phi node in the scalar body needs the truncated
-  // value.
-  for (BasicBlock *BB : BypassBlocks)
-    BCResumeVal->addIncoming(II.getStartValue(), BB);
-
-  if (AdditionalBypass.first)
-    BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
-                                          EndValueFromAdditionalBypass);
-  return BCResumeVal;
+  if (EndValueFromAdditionalBypass) {
+    // Store the bypass value here, as it needs to be added as operand to its
+    // scalar preheader phi node after the epilogue skeleton has been created.
+    // TODO: Directly add as extra operand to the VPResumePHI recipe.
+    assert(!Induction2AdditionalBypassValue.contains(OrigPhi) &&
+           "entry for OrigPhi already exits");
+    Induction2AdditionalBypassValue[OrigPhi] = EndValueFromAdditionalBypass;
+  }
 }
 
 /// Return the expanded step for \p ID using \p ExpandedSCEVs to look up SCEV
@@ -2652,31 +2681,72 @@ static Value *getExpandedStep(const InductionDescriptor &ID,
   return I->second;
 }
 
-void InnerLoopVectorizer::createInductionResumeValues(
-    const SCEV2ValueTy &ExpandedSCEVs,
-    std::pair<BasicBlock *, Value *> AdditionalBypass) {
-  assert(((AdditionalBypass.first && AdditionalBypass.second) ||
-          (!AdditionalBypass.first && !AdditionalBypass.second)) &&
-         "Inconsistent information about additional bypass.");
+/// Knowing that loop \p L executes a single vector iteration, add instructions
+/// that will get simplified and thus should not have any cost to \p
+/// InstsToIgnore.
+static void addFullyUnrolledInstructionsToIgnore(
+    Loop *L, const LoopVectorizationLegality::InductionList &IL,
+    SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
+  auto *Cmp = L->getLatchCmpInst();
+  if (Cmp)
+    InstsToIgnore.insert(Cmp);
+  for (const auto &KV : IL) {
+    // Extract the key by hand so that it can be used in the lambda below.  Note
+    // that captured structured bindings are a C++20 extension.
+    const PHINode *IV = KV.first;
+
+    // Get next iteration value of the induction variable.
+    Instruction *IVInst =
+        cast<Instruction>(IV->getIncomingValueForBlock(L->getLoopLatch()));
+    if (all_of(IVInst->users(),
+               [&](const User *U) { return U == IV || U == Cmp; }))
+      InstsToIgnore.insert(IVInst);
+  }
+}
+
+void InnerLoopVectorizer::createInductionResumeVPValues(
+    const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount,
+    SmallPtrSetImpl<PHINode *> *IVSubset) {
   // We are going to resume the execution of the scalar loop.
-  // Go over all of the induction variables that we found and fix the
-  // PHIs that are left in the scalar version of the loop.
-  // The starting values of PHI nodes depend on the counter of the last
-  // iteration in the vectorized loop.
-  // If we come from a bypass edge then we need to start from the original
-  // start value.
-  for (const auto &InductionEntry : Legal->getInductionVars()) {
-    PHINode *OrigPhi = InductionEntry.first;
-    const InductionDescriptor &II = InductionEntry.second;
-    PHINode *BCResumeVal = createInductionResumeValue(
-        OrigPhi, II, getExpandedStep(II, ExpandedSCEVs), LoopBypassBlocks,
-        AdditionalBypass);
-    OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
-  }
-}
-
-std::pair<BasicBlock *, Value *>
-InnerLoopVectorizer::createVectorizedLoopSkeleton(
+  // Go over all of the induction variable PHIs of the scalar loop header and
+  // fix their starting values, which depend on the counter of the last
+  // iteration of the vectorized loop. If we come from one of the
+  // LoopBypassBlocks then we need to start from the original start value.
+  // Otherwise we provide the trip count from the main vector loop.
+  VPBasicBlock *ScalarPHVPBB = Plan.getScalarPreheader();
+  VPBuilder ScalarPHBuilder(ScalarPHVPBB, ScalarPHVPBB->begin());
+  bool HasCanonical = false;
+  for (VPRecipeBase &R : *Plan.getScalarHeader()) {
+    auto *PhiR = cast<VPIRInstruction>(&R);
+    auto *Phi = dyn_cast<PHINode>(&PhiR->getInstruction());
+    if (!Phi)
+      break;
+    if (!Legal->getInductionVars().contains(Phi) ||
+        (IVSubset && !IVSubset->contains(Phi)))
+      continue;
+    const InductionDescriptor &II = Legal->getInductionVars().find(Phi)->second;
+    createInductionResumeVPValue(PhiR, II, getExpandedStep(II, ExpandedSCEVs),
+                                 LoopBypassBlocks, ScalarPHBuilder,
+                                 MainVectorTripCount);
+    auto *ConstStart = dyn_cast<ConstantInt>(II.getStartValue());
+    auto *ConstStep = II.getConstIntStepValue();
+    if (Phi->getType() == VectorTripCount->getType() && ConstStart &&
+        ConstStart->isZero() && ConstStep && ConstStep->isOne())
+      HasCanonical = true;
+  }
+
+  if (!IVSubset || HasCanonical)
+    return;
+  // When vectorizing the epilogue, create a resume phi for the canonical IV if
+  // no suitable resume phi was already created.
+  ScalarPHBuilder.createNaryOp(
+      VPInstruction::ResumePhi,
+      {Plan.getOrAddLiveIn(VectorTripCount),
+       Plan.getOrAddLiveIn(ConstantInt::get(VectorTripCount->getType(), 0))},
+      {}, "vec.epilog.resume.val");
+}
+
+BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton(
     const SCEV2ValueTy &ExpandedSCEVs) {
   /*
    In this function we generate a new loop. The new loop will contain
@@ -2734,9 +2804,9 @@ InnerLoopVectorizer::createVectorizedLoopSkeleton(
   emitMemRuntimeChecks(LoopScalarPreHeader);
 
   // Emit phis for the new starting index of the scalar loop.
-  createInductionResumeValues(ExpandedSCEVs);
+  createInductionResumeVPValues(ExpandedSCEVs);
 
-  return {LoopVectorPreHeader, nullptr};
+  return LoopVectorPreHeader;
 }
 
 // Fix up external users of the induction variable. At this point, we are
@@ -2753,8 +2823,6 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
   // value (the value that feeds into the phi from the loop latch).
   // We allow both, but they, obviously, have different values.
 
-  assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block");
-
   DenseMap<Value *, Value *> MissingVals;
 
   Value *EndValue = cast<PHINode>(OrigPhi->getIncomingValueForBlock(
@@ -2808,6 +2876,9 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
     }
   }
 
+  assert((MissingVals.empty() || OrigLoop->getUniqueExitBlock()) &&
+         "Expected a single exit block for escaping values");
+
   for (auto &I : MissingVals) {
     PHINode *PHI = cast<PHINode>(I.first);
     // One corner case we have to handle is two IVs "chasing" each-other,
@@ -3153,9 +3224,8 @@ void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
     // If the use of the pointer will be a scalar use, and all users of the
     // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
     // place the pointer in PossibleNonScalarPtrs.
-    if (IsScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
-          return isa<LoadInst>(U) || isa<StoreInst>(U);
-        }))
+    if (IsScalarUse(MemAccess, Ptr) &&
+        all_of(I->users(), IsaPred<LoadInst, StoreInst>))
       ScalarPtrs.insert(I);
     else
       PossibleNonScalarPtrs.insert(I);
@@ -5560,6 +5630,15 @@ InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
 InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
   InstructionCost Cost;
 
+  // If the vector loop gets executed exactly once with the given VF, ignore the
+  // costs of comparison and induction instructions, as they'll get simplified
+  // away.
+  SmallPtrSet<Instruction *, 2> ValuesToIgnoreForVF;
+  auto TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
+  if (VF.isFixed() && TC == VF.getFixedValue() && !foldTailByMasking())
+    addFullyUnrolledInstructionsToIgnore(TheLoop, Legal->getInductionVars(),
+                                         ValuesToIgnoreForVF);
+
   // For each block.
   for (BasicBlock *BB : TheLoop->blocks()) {
     InstructionCost BlockCost;
@@ -5567,7 +5646,7 @@ InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
     // For each instruction in the old loop.
     for (Instruction &I : BB->instructionsWithoutDebug()) {
       // Skip ignored values.
-      if (ValuesToIgnore.count(&I) ||
+      if (ValuesToIgnore.count(&I) || ValuesToIgnoreForVF.count(&I) ||
           (VF.isVector() && VecValuesToIgnore.count(&I)))
         continue;
 
@@ -7249,6 +7328,17 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
         continue;
       IVInsts.push_back(CI);
     }
+
+    // If the vector loop gets executed exactly once with the given VF, ignore
+    // the costs of comparison and induction instructions, as they'll get
+    // simplified away.
+    // TODO: Remove this code after stepping away from the legacy cost model and
+    // adding code to simplify VPlans before calculating their costs.
+    auto TC = PSE.getSE()->getSmallConstantTripCount(OrigLoop);
+    if (VF.isFixed() && TC == VF.getFixedValue() && !CM.foldTailByMasking())
+      addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
+                                           CostCtx.SkipCostComputation);
+
     for (Instruction *IVInst : IVInsts) {
       if (CostCtx.skipCostComputation(IVInst, VF.isVector()))
         continue;
@@ -7303,34 +7393,14 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
 
   // The legacy cost model has special logic to compute the cost of in-loop
   // reductions, which may be smaller than the sum of all instructions involved
-  // in the reduction. For AnyOf reductions, VPlan codegen may remove the select
-  // which the legacy cost model uses to assign cost. Pre-compute their costs
-  // for now.
+  // in the reduction.
   // TODO: Switch to costing based on VPlan once the logic has been ported.
   for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars()) {
     if (ForceTargetInstructionCost.getNumOccurrences())
       continue;
 
-    if (!CM.isInLoopReduction(RedPhi) &&
-        !RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind()))
-      continue;
-
-    // AnyOf reduction codegen may remove the select. To match the legacy cost
-    // model, pre-compute the cost for AnyOf reductions here.
-    if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
-      auto *Select = cast<SelectInst>(*find_if(
-          RedPhi->users(), [](User *U) { return isa<SelectInst>(U); }));
-      assert(!CostCtx.SkipCostComputation.contains(Select) &&
-             "reduction op visited multiple times");
-      CostCtx.SkipCostComputation.insert(Select);
-      auto ReductionCost = CostCtx.getLegacyCost(Select, VF);
-      LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF
-                        << ":\n any-of reduction " << *Select << "\n");
-      Cost += ReductionCost;
+    if (!CM.isInLoopReduction(RedPhi))
       continue;
-    }
 
     const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop);
     SetVector<Instruction *> ChainOpsAndOperands(ChainOps.begin(),
@@ -7605,7 +7675,8 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
 // fix the reduction's scalar PHI node by adding the incoming value from the
 // main vector loop.
 static void fixReductionScalarResumeWhenVectorizingEpilog(
-    VPRecipeBase *R, VPTransformState &State, BasicBlock *LoopMiddleBlock) {
+    VPRecipeBase *R, VPTransformState &State, BasicBlock *LoopMiddleBlock,
+    BasicBlock *BypassBlock) {
   auto *EpiRedResult = dyn_cast<VPInstruction>(R);
   if (!EpiRedResult ||
       EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult)
@@ -7642,21 +7713,8 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   auto *EpiResumePhiVPI =
       cast<VPInstruction>(*find_if(EpiRedResult->users(), IsResumePhi));
   auto *EpiResumePhi = cast<PHINode>(State.get(EpiResumePhiVPI, true));
-  BasicBlock *LoopScalarPreHeader = EpiResumePhi->getParent();
-  bool Updated = false;
-  for (auto *Incoming : predecessors(LoopScalarPreHeader)) {
-    if (is_contained(MainResumePhi->blocks(), Incoming)) {
-      assert(EpiResumePhi->getIncomingValueForBlock(Incoming) ==
-                 RdxDesc.getRecurrenceStartValue() &&
-             "Trying to reset unexpected value");
-      assert(!Updated && "Should update at most 1 incoming value");
-      EpiResumePhi->setIncomingValueForBlock(
-          Incoming, MainResumePhi->getIncomingValueForBlock(Incoming));
-      Updated = true;
-    }
-  }
-  assert(Updated && "Must update EpiResumePhi.");
-  (void)Updated;
+  EpiResumePhi->setIncomingValueForBlock(
+      BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
 }
 
 DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
@@ -7677,15 +7735,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   VPlanTransforms::unrollByUF(BestVPlan, BestUF,
                               OrigLoop->getHeader()->getContext());
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
-
-  LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF
-                    << ", UF=" << BestUF << '\n');
-  BestVPlan.setName("Final VPlan");
-  LLVM_DEBUG(BestVPlan.dump());
+  VPlanTransforms::convertToConcreteRecipes(BestVPlan);
 
   // Perform the actual loop transformation.
   VPTransformState State(&TTI, BestVF, BestUF, LI, DT, ILV.Builder, &ILV,
-                         &BestVPlan);
+                         &BestVPlan, Legal->getWidestInductionType());
 
   // 0. Generate SCEV-dependent code into the preheader, including TripCount,
   // before making any changes to the CFG.
@@ -7702,10 +7756,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
   // 1. Set up the skeleton for vectorization, including vector pre-header and
   // middle block. The vector loop is created during VPlan execution.
-  Value *CanonicalIVStartValue;
-  std::tie(State.CFG.PrevBB, CanonicalIVStartValue) =
-      ILV.createVectorizedLoopSkeleton(ExpandedSCEVs ? *ExpandedSCEVs
-                                                     : State.ExpandedSCEVs);
+  State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton(
+      ExpandedSCEVs ? *ExpandedSCEVs : State.ExpandedSCEVs);
+  if (VectorizingEpilogue)
+    VPlanTransforms::removeDeadRecipes(BestVPlan);
+
 #ifdef EXPENSIVE_CHECKS
   assert(DT->verify(DominatorTree::VerificationLevel::Fast));
 #endif
@@ -7740,18 +7795,26 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
   // 2. Copy and widen instructions from the old loop into the new loop.
   BestVPlan.prepareToExecute(ILV.getTripCount(),
-                             ILV.getOrCreateVectorTripCount(nullptr),
-                             CanonicalIVStartValue, State);
+                             ILV.getOrCreateVectorTripCount(nullptr), State);
 
   BestVPlan.execute(&State);
 
-  // 2.5 Collect reduction resume values.
   auto *ExitVPBB = BestVPlan.getMiddleBlock();
-  if (VectorizingEpilogue)
+  // 2.5 When vectorizing the epilogue, fix reduction and induction resume
+  // values from the additional bypass block.
+  if (VectorizingEpilogue) {
+    BasicBlock *BypassBlock = ILV.getAdditionalBypassBlock();
     for (VPRecipeBase &R : *ExitVPBB) {
       fixReductionScalarResumeWhenVectorizingEpilog(
-          &R, State, State.CFG.VPBB2IRBB[ExitVPBB]);
+          &R, State, State.CFG.VPBB2IRBB[ExitVPBB], BypassBlock);
     }
+    BasicBlock *PH = OrigLoop->getLoopPreheader();
+    for (const auto &[IVPhi, _] : Legal->getInductionVars()) {
+      auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
+      Value *V = ILV.getInductionAdditionalBypassValue(IVPhi);
+      Inc->setIncomingValueForBlock(BypassBlock, V);
+    }
+  }
 
   // 2.6. Maintain Loop Hints
   // Keep all loop hints from the original loop on the vector loop (we'll
@@ -7778,7 +7841,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   }
   TargetTransformInfo::UnrollingPreferences UP;
   TTI.getUnrollingPreferences(L, *PSE.getSE(), UP, ORE);
-  if (!UP.UnrollVectorizedLoop || CanonicalIVStartValue)
+  if (!UP.UnrollVectorizedLoop || VectorizingEpilogue)
     addRuntimeUnrollDisableMetaData(L);
 
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
@@ -7808,8 +7871,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
 /// This function is partially responsible for generating the control flow
 /// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
-std::pair<BasicBlock *, Value *>
-EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(
+BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(
     const SCEV2ValueTy &ExpandedSCEVs) {
   createVectorLoopSkeleton("");
 
@@ -7840,12 +7902,20 @@ EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton(
   // Generate the induction variable.
   EPI.VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
 
-  // Skip induction resume value creation here because they will be created in
-  // the second pass for the scalar loop. The induction resume values for the
-  // inductions in the epilogue loop are created before executing the plan for
-  // the epilogue loop.
+  // Generate VPValues and ResumePhi recipes for wide inductions in the epilogue
+  // plan only. Other inductions only need a resume value for the canonical
+  // induction, which will get created during epilogue skeleton construction.
+  SmallPtrSet<PHINode *, 4> WideIVs;
+  for (VPRecipeBase &H :
+       EPI.EpiloguePlan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
+    if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&H))
+      WideIVs.insert(WideIV->getPHINode());
+    else if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&H))
+      WideIVs.insert(cast<PHINode>(PtrIV->getUnderlyingValue()));
+  }
+  createInductionResumeVPValues(ExpandedSCEVs, nullptr, &WideIVs);
 
-  return {LoopVectorPreHeader, nullptr};
+  return LoopVectorPreHeader;
 }
 
 void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
@@ -7925,7 +7995,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
 
 /// This function is partially responsible for generating the control flow
 /// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
-std::pair<BasicBlock *, Value *>
+BasicBlock *
 EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
     const SCEV2ValueTy &ExpandedSCEVs) {
   createVectorLoopSkeleton("vec.epilog.");
@@ -7938,6 +8008,7 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
                  nullptr, "vec.epilog.iter.check", true);
   emitMinimumVectorEpilogueIterCountCheck(LoopScalarPreHeader,
                                           VecEpilogueIterationCountCheck);
+  AdditionalBypassBlock = VecEpilogueIterationCountCheck;
 
   // Adjust the control flow taking the state info from the main loop
   // vectorization into account.
@@ -8008,27 +8079,15 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
       Phi->removeIncomingValue(EPI.MemSafetyCheck);
   }
 
-  // Generate a resume induction for the vector epilogue and put it in the
-  // vector epilogue preheader
-  Type *IdxTy = Legal->getWidestInductionType();
-  PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val");
-  EPResumeVal->insertBefore(LoopVectorPreHeader->getFirstNonPHIIt());
-  EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck);
-  EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0),
-                           EPI.MainLoopIterationCountCheck);
-
   // Generate induction resume values. These variables save the new starting
   // indexes for the scalar loop. They are used to test if there are any tail
   // iterations left once the vector loop has completed.
   // Note that when the vectorized epilogue is skipped due to iteration count
   // check, then the resume value for the induction variable comes from
-  // the trip count of the main vector loop, hence passing the AdditionalBypass
-  // argument.
-  createInductionResumeValues(ExpandedSCEVs,
-                              {VecEpilogueIterationCountCheck,
-                               EPI.VectorTripCount} /* AdditionalBypass */);
+  // the trip count of the main vector loop, passed as the second argument.
+  createInductionResumeVPValues(ExpandedSCEVs, EPI.VectorTripCount);
 
-  return {LoopVectorPreHeader, EPResumeVal};
+  return LoopVectorPreHeader;
 }
 
 BasicBlock *
@@ -9076,9 +9135,11 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
 
   DebugLoc DL = getDebugLocFromInstOrOperands(Legal->getPrimaryInduction());
   TailFoldingStyle Style = CM.getTailFoldingStyle(IVUpdateMayOverflow);
-  // When not folding the tail, we know that the induction increment will not
-  // overflow.
-  bool HasNUW = Style == TailFoldingStyle::None;
+  // Use NUW for the induction increment if we proved that it won't overflow in
+  // the vector loop or when not folding the tail. In the later case, we know
+  // that the canonical induction increment will not overflow as the vector trip
+  // count is >= increment and a multiple of the increment.
+  bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
   addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);
 
   VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, Legal, CM, PSE, Builder);
@@ -9915,6 +9976,119 @@ LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
       VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
                               !EnableLoopVectorization) {}
 
+/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
+/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
+static void
+preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
+                                 const SCEV2ValueTy &ExpandedSCEVs,
+                                 const EpilogueLoopVectorizationInfo &EPI) {
+  VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
+  VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
+  Header->setName("vec.epilog.vector.body");
+
+  // Re-use the trip count and steps expanded for the main loop, as
+  // skeleton creation needs it as a value that dominates both the scalar
+  // and vector epilogue loops
+  // TODO: This is a workaround needed for epilogue vectorization and it
+  // should be removed once induction resume value creation is done
+  // directly in VPlan.
+  for (auto &R : make_early_inc_range(*Plan.getPreheader())) {
+    auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
+    if (!ExpandR)
+      continue;
+    auto *ExpandedVal =
+        Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
+    ExpandR->replaceAllUsesWith(ExpandedVal);
+    if (Plan.getTripCount() == ExpandR)
+      Plan.resetTripCount(ExpandedVal);
+    ExpandR->eraseFromParent();
+  }
+
+  // Ensure that the start values for all header phi recipes are updated before
+  // vectorizing the epilogue loop.
+  for (VPRecipeBase &R : Header->phis()) {
+    if (auto *IV = dyn_cast<VPCanonicalIVPHIRecipe>(&R)) {
+      // When vectorizing the epilogue loop, the canonical induction start
+      // value needs to be changed from zero to the value after the main
+      // vector loop. Find the resume value created during execution of the main
+      // VPlan.
+      // FIXME: Improve modeling for canonical IV start values in the epilogue
+      // loop.
+      BasicBlock *MainMiddle = find_singleton<BasicBlock>(
+          predecessors(L->getLoopPreheader()),
+          [&EPI](BasicBlock *BB, bool) -> BasicBlock * {
+            if (BB != EPI.MainLoopIterationCountCheck &&
+                BB != EPI.EpilogueIterationCountCheck &&
+                BB != EPI.SCEVSafetyCheck && BB != EPI.MemSafetyCheck)
+              return BB;
+            return nullptr;
+          });
+      using namespace llvm::PatternMatch;
+      Type *IdxTy = IV->getScalarType();
+      PHINode *EPResumeVal = find_singleton<PHINode>(
+          L->getLoopPreheader()->phis(),
+          [&EPI, IdxTy, MainMiddle](PHINode &P, bool) -> PHINode * {
+            if (P.getType() == IdxTy &&
+                P.getIncomingValueForBlock(MainMiddle) == EPI.VectorTripCount &&
+                match(
+                    P.getIncomingValueForBlock(EPI.MainLoopIterationCountCheck),
+                    m_SpecificInt(0)))
+              return &P;
+            return nullptr;
+          });
+      assert(EPResumeVal && "must have a resume value for the canonical IV");
+      VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
+      assert(all_of(IV->users(),
+                    [](const VPUser *U) {
+                      return isa<VPScalarIVStepsRecipe>(U) ||
+                             isa<VPScalarCastRecipe>(U) ||
+                             isa<VPDerivedIVRecipe>(U) ||
+                             cast<VPInstruction>(U)->getOpcode() ==
+                                 Instruction::Add;
+                    }) &&
+             "the canonical IV should only be used by its increment or "
+             "ScalarIVSteps when resetting the start value");
+      IV->setOperand(0, VPV);
+      continue;
+    }
+
+    Value *ResumeV = nullptr;
+    // TODO: Move setting of resume values to prepareToExecute.
+    if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+      ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
+                    ->getIncomingValueForBlock(L->getLoopPreheader());
+      const RecurrenceDescriptor &RdxDesc =
+          ReductionPhi->getRecurrenceDescriptor();
+      RecurKind RK = RdxDesc.getRecurrenceKind();
+      if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+        // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
+        // start value; compare the final value from the main vector loop
+        // to the start value.
+        IRBuilder<> Builder(
+            cast<Instruction>(ResumeV)->getParent()->getFirstNonPHI());
+        ResumeV =
+            Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
+      }
+    } else {
+      // Retrieve the induction resume values for wide inductions from
+      // their original phi nodes in the scalar loop.
+      PHINode *IndPhi = nullptr;
+      if (auto *Ind = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
+        IndPhi = cast<PHINode>(Ind->getUnderlyingValue());
+      } else {
+        auto *WidenInd = cast<VPWidenIntOrFpInductionRecipe>(&R);
+        IndPhi = WidenInd->getPHINode();
+      }
+      // Hook up to the PHINode generated by a ResumePhi recipe of main
+      // loop VPlan, which feeds the scalar loop.
+      ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
+    }
+    assert(ResumeV && "Must have a resume value");
+    VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
+    cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
+  }
+}
+
 bool LoopVectorizePass::processLoop(Loop *L) {
   assert((EnableVPlanNativePath || L->isInnermost()) &&
          "VPlan-native path is not enabled. Only process inner loops.");
@@ -10261,7 +10435,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // The first pass vectorizes the main loop and creates a scalar epilogue
         // to be vectorized by executing the plan (potentially with a different
         // factor) again shortly afterwards.
-        EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1);
+        VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
+        EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
+                                          BestEpiPlan);
         EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE,
                                            EPI, &LVL, &CM, BFI, PSI, Checks,
                                            *BestMainPlan);
@@ -10274,81 +10450,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // edges from the first pass.
         EPI.MainLoopVF = EPI.EpilogueVF;
         EPI.MainLoopUF = EPI.EpilogueUF;
-        VPlan &BestEpiPlan = LVP.getPlanFor(EPI.EpilogueVF);
         EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
                                                  ORE, EPI, &LVL, &CM, BFI, PSI,
                                                  Checks, BestEpiPlan);
-
-        VPRegionBlock *VectorLoop = BestEpiPlan.getVectorLoopRegion();
-        VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
-        Header->setName("vec.epilog.vector.body");
-
-        // Re-use the trip count and steps expanded for the main loop, as
-        // skeleton creation needs it as a value that dominates both the scalar
-        // and vector epilogue loops
-        // TODO: This is a workaround needed for epilogue vectorization and it
-        // should be removed once induction resume value creation is done
-        // directly in VPlan.
         EpilogILV.setTripCount(MainILV.getTripCount());
-        for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
-          auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
-          if (!ExpandR)
-            continue;
-          auto *ExpandedVal = BestEpiPlan.getOrAddLiveIn(
-              ExpandedSCEVs.find(ExpandR->getSCEV())->second);
-          ExpandR->replaceAllUsesWith(ExpandedVal);
-          if (BestEpiPlan.getTripCount() == ExpandR)
-            BestEpiPlan.resetTripCount(ExpandedVal);
-          ExpandR->eraseFromParent();
-        }
-
-        // Ensure that the start values for any VPWidenIntOrFpInductionRecipe,
-        // VPWidenPointerInductionRecipe and VPReductionPHIRecipes are updated
-        // before vectorizing the epilogue loop.
-        for (VPRecipeBase &R : Header->phis()) {
-          if (isa<VPCanonicalIVPHIRecipe>(&R))
-            continue;
-
-          Value *ResumeV = nullptr;
-          // TODO: Move setting of resume values to prepareToExecute.
-          if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
-            ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
-                          ->getIncomingValueForBlock(L->getLoopPreheader());
-            const RecurrenceDescriptor &RdxDesc =
-                ReductionPhi->getRecurrenceDescriptor();
-            RecurKind RK = RdxDesc.getRecurrenceKind();
-            if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
-              // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
-              // start value; compare the final value from the main vector loop
-              // to the start value.
-              IRBuilder<> Builder(
-                  cast<Instruction>(ResumeV)->getParent()->getFirstNonPHI());
-              ResumeV = Builder.CreateICmpNE(ResumeV,
-                                             RdxDesc.getRecurrenceStartValue());
-            }
-          } else {
-            // Create induction resume values for both widened pointer and
-            // integer/fp inductions and update the start value of the induction
-            // recipes to use the resume value.
-            PHINode *IndPhi = nullptr;
-            const InductionDescriptor *ID;
-            if (auto *Ind = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
-              IndPhi = cast<PHINode>(Ind->getUnderlyingValue());
-              ID = &Ind->getInductionDescriptor();
-            } else {
-              auto *WidenInd = cast<VPWidenIntOrFpInductionRecipe>(&R);
-              IndPhi = WidenInd->getPHINode();
-              ID = &WidenInd->getInductionDescriptor();
-            }
-
-            ResumeV = MainILV.createInductionResumeValue(
-                IndPhi, *ID, getExpandedStep(*ID, ExpandedSCEVs),
-                {EPI.MainLoopIterationCountCheck});
-          }
-          assert(ResumeV && "Must have a resume value");
-          VPValue *StartVal = BestEpiPlan.getOrAddLiveIn(ResumeV);
-          cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
-        }
+        preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs, EPI);
 
         assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
                "DT not preserved correctly");
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 48a8520a966fc..a1d7515f031cf 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -807,14 +807,16 @@ static std::optional<unsigned> getExtractIndex(Instruction *E) {
 namespace {
 
 /// Main data required for vectorization of instructions.
-struct InstructionsState {
-  /// The very first instruction in the list with the main opcode.
-  Value *OpValue = nullptr;
-
-  /// The main/alternate instruction.
+class InstructionsState {
+  /// The main/alternate instruction. MainOp is also VL0.
   Instruction *MainOp = nullptr;
   Instruction *AltOp = nullptr;
 
+public:
+  Instruction *getMainOp() const { return MainOp; }
+
+  Instruction *getAltOp() const { return AltOp; }
+
   /// The main/alternate opcodes for the list of instructions.
   unsigned getOpcode() const {
     return MainOp ? MainOp->getOpcode() : 0;
@@ -833,9 +835,9 @@ struct InstructionsState {
   }
 
   InstructionsState() = delete;
-  InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
-      : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
-  static InstructionsState invalid() { return {nullptr, nullptr, nullptr}; }
+  InstructionsState(Instruction *MainOp, Instruction *AltOp)
+      : MainOp(MainOp), AltOp(AltOp) {}
+  static InstructionsState invalid() { return {nullptr, nullptr}; }
 };
 
 } // end anonymous namespace
@@ -1073,7 +1075,7 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
     return InstructionsState::invalid();
   }
 
-  return InstructionsState(V, cast<Instruction>(V),
+  return InstructionsState(cast<Instruction>(V),
                            cast<Instruction>(VL[AltIndex]));
 }
 
@@ -1842,12 +1844,12 @@ class BoUpSLP {
         // Note: Only consider instructions with <= 2 operands to avoid
         // complexity explosion.
         if (S.getOpcode() &&
-            (S.MainOp->getNumOperands() <= 2 || !MainAltOps.empty() ||
+            (S.getMainOp()->getNumOperands() <= 2 || !MainAltOps.empty() ||
              !S.isAltShuffle()) &&
             all_of(Ops, [&S](Value *V) {
               return isa<PoisonValue>(V) ||
                      cast<Instruction>(V)->getNumOperands() ==
-                         S.MainOp->getNumOperands();
+                         S.getMainOp()->getNumOperands();
             }))
           return S.isAltShuffle() ? LookAheadHeuristics::ScoreAltOpcodes
                                   : LookAheadHeuristics::ScoreSameOpcode;
@@ -2017,6 +2019,9 @@ class BoUpSLP {
 
     /// A vector of operand vectors.
     SmallVector<OperandDataVec, 4> OpsVec;
+    /// When VL[0] is IntrinsicInst, ArgSize is CallBase::arg_size. When VL[0]
+    /// is not IntrinsicInst, ArgSize is User::getNumOperands.
+    unsigned ArgSize = 0;
 
     const TargetLibraryInfo &TLI;
     const DataLayout &DL;
@@ -2400,14 +2405,15 @@ class BoUpSLP {
     }
 
     /// Go through the instructions in VL and append their operands.
-    void appendOperandsOfVL(ArrayRef<Value *> VL) {
+    void appendOperandsOfVL(ArrayRef<Value *> VL, Instruction *VL0) {
       assert(!VL.empty() && "Bad VL");
       assert((empty() || VL.size() == getNumLanes()) &&
              "Expected same number of lanes");
+      // IntrinsicInst::isCommutative returns true if swapping the first "two"
+      // arguments to the intrinsic produces the same result.
       constexpr unsigned IntrinsicNumOperands = 2;
-      auto *VL0 = cast<Instruction>(*find_if(VL, IsaPred<Instruction>));
-      unsigned NumOperands = isa<IntrinsicInst>(VL0) ? IntrinsicNumOperands
-                                                     : VL0->getNumOperands();
+      unsigned NumOperands = VL0->getNumOperands();
+      ArgSize = isa<IntrinsicInst>(VL0) ? IntrinsicNumOperands : NumOperands;
       OpsVec.resize(NumOperands);
       unsigned NumLanes = VL.size();
       for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
@@ -2440,7 +2446,7 @@ class BoUpSLP {
     }
 
     /// \returns the number of operands.
-    unsigned getNumOperands() const { return OpsVec.size(); }
+    unsigned getNumOperands() const { return ArgSize; }
 
     /// \returns the number of lanes.
     unsigned getNumLanes() const { return OpsVec[0].size(); }
@@ -2460,6 +2466,8 @@ class BoUpSLP {
     /// the whole vector (it is mixed with constants or loop invariant values).
     /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
     bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
+      assert(Op == getValue(OpIdx, Lane) &&
+             "Op is expected to be getValue(OpIdx, Lane).");
       // Small number of loads - try load matching.
       if (isa<LoadInst>(Op) && getNumLanes() == 2 && getNumOperands() == 2)
         return false;
@@ -2517,6 +2525,8 @@ class BoUpSLP {
     /// Checks if there is at least single compatible operand in lanes other
     /// than \p Lane, compatible with the operand \p Op.
     bool canBeVectorized(Instruction *Op, unsigned OpIdx, unsigned Lane) const {
+      assert(Op == getValue(OpIdx, Lane) &&
+             "Op is expected to be getValue(OpIdx, Lane).");
       bool OpAPO = getData(OpIdx, Lane).APO;
       for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
         if (Ln == Lane)
@@ -2537,13 +2547,11 @@ class BoUpSLP {
 
   public:
     /// Initialize with all the operands of the instruction vector \p RootVL.
-    VLOperands(ArrayRef<Value *> RootVL, const BoUpSLP &R)
+    VLOperands(ArrayRef<Value *> RootVL, Instruction *VL0, const BoUpSLP &R)
         : TLI(*R.TLI), DL(*R.DL), SE(*R.SE), R(R),
-          L(R.LI->getLoopFor(
-              (cast<Instruction>(*find_if(RootVL, IsaPred<Instruction>))
-                   ->getParent()))) {
+          L(R.LI->getLoopFor((VL0->getParent()))) {
       // Append all the operands of RootVL.
-      appendOperandsOfVL(RootVL);
+      appendOperandsOfVL(RootVL, VL0);
     }
 
     /// \Returns a value vector with the operands across all lanes for the
@@ -2617,7 +2625,8 @@ class BoUpSLP {
         ArrayRef<OperandData> Op0 = OpsVec.front();
         for (const OperandData &Data : Op0)
           UniqueValues.insert(Data.V);
-        for (ArrayRef<OperandData> Op : drop_begin(OpsVec, 1)) {
+        for (ArrayRef<OperandData> Op :
+             ArrayRef(OpsVec).slice(1, getNumOperands() - 1)) {
           if (any_of(Op, [&UniqueValues](const OperandData &Data) {
                 return !UniqueValues.contains(Data.V);
               }))
@@ -3110,9 +3119,8 @@ class BoUpSLP {
       SmallVectorImpl<SmallVector<const TreeEntry *>> &Entries,
       unsigned NumParts, bool ForOrder = false);
 
-  /// \returns the scalarization cost for this list of values. Assuming that
-  /// this subtree gets vectorized, we may need to extract the values from the
-  /// roots. This method calculates the cost of extracting the values.
+  /// \returns the cost of gathering (inserting) the values in \p VL into a
+  /// vector.
   /// \param ForPoisonSrc true if initial vector is poison, false otherwise.
   InstructionCost getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
                                 Type *ScalarTy) const;
@@ -3139,13 +3147,6 @@ class BoUpSLP {
                            SmallVector<SmallVector<std::pair<LoadInst *, int>>>,
                            8> &GatheredLoads);
 
-  /// Reorder commutative or alt operands to get better probability of
-  /// generating vectorized code.
-  static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
-                                             SmallVectorImpl<Value *> &Left,
-                                             SmallVectorImpl<Value *> &Right,
-                                             const BoUpSLP &R);
-
   /// Helper for `findExternalStoreUsersReorderIndices()`. It iterates over the
   /// users of \p TE and collects the stores. It returns the map from the store
   /// pointers to the collected stores.
@@ -3340,27 +3341,13 @@ class BoUpSLP {
       copy(OpVL, Operands[OpIdx].begin());
     }
 
-    /// Set the operands of this bundle in their original order.
-    void setOperandsInOrder() {
-      assert(Operands.empty() && "Already initialized?");
-      auto *I0 = cast<Instruction>(*find_if(Scalars, IsaPred<Instruction>));
-      Operands.resize(I0->getNumOperands());
-      unsigned NumLanes = Scalars.size();
-      for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands();
-           OpIdx != NumOperands; ++OpIdx) {
-        Operands[OpIdx].resize(NumLanes);
-        for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
-          if (isa<PoisonValue>(Scalars[Lane])) {
-            Operands[OpIdx][Lane] =
-                PoisonValue::get(I0->getOperand(OpIdx)->getType());
-            continue;
-          }
-          auto *I = cast<Instruction>(Scalars[Lane]);
-          assert(I->getNumOperands() == NumOperands &&
-                 "Expected same number of operands");
-          Operands[OpIdx][Lane] = I->getOperand(OpIdx);
-        }
-      }
+    /// Set this bundle's operand from Scalars.
+    void setOperand(const BoUpSLP &R, bool RequireReorder = false) {
+      VLOperands Ops(Scalars, MainOp, R);
+      if (RequireReorder)
+        Ops.reorder();
+      for (unsigned I : seq<unsigned>(MainOp->getNumOperands()))
+        setOperand(I, Ops.getVL(I));
     }
 
     /// Reorders operands of the node to the given mask \p Mask.
@@ -3411,8 +3398,8 @@ class BoUpSLP {
     }
 
     void setOperations(const InstructionsState &S) {
-      MainOp = S.MainOp;
-      AltOp = S.AltOp;
+      MainOp = S.getMainOp();
+      AltOp = S.getAltOp();
     }
 
     Instruction *getMainOp() const {
@@ -3650,8 +3637,8 @@ class BoUpSLP {
       }
       // Update the scheduler bundle to point to this TreeEntry.
       ScheduleData *BundleMember = *Bundle;
-      assert((BundleMember || isa<PHINode>(S.MainOp) ||
-              isVectorLikeInstWithConstOps(S.MainOp) ||
+      assert((BundleMember || isa<PHINode>(S.getMainOp()) ||
+              isVectorLikeInstWithConstOps(S.getMainOp()) ||
               doesNotNeedToSchedule(VL)) &&
              "Bundle and VL out of sync");
       if (BundleMember) {
@@ -3718,9 +3705,11 @@ class BoUpSLP {
 
   /// Checks if the specified list of the instructions/values can be vectorized
   /// and fills required data before actual scheduling of the instructions.
-  TreeEntry::EntryState getScalarsVectorizationState(
-      InstructionsState &S, ArrayRef<Value *> VL, bool IsScatterVectorizeUserTE,
-      OrdersType &CurrentOrder, SmallVectorImpl<Value *> &PointerOps);
+  TreeEntry::EntryState
+  getScalarsVectorizationState(const InstructionsState &S, ArrayRef<Value *> VL,
+                               bool IsScatterVectorizeUserTE,
+                               OrdersType &CurrentOrder,
+                               SmallVectorImpl<Value *> &PointerOps);
 
   /// Maps a specific scalar to its tree entry.
   SmallDenseMap<Value *, TreeEntry *> ScalarToTreeEntry;
@@ -6074,6 +6063,23 @@ void BoUpSLP::reorderTopToBottom() {
                                      TE->Scalars.size();
                         }) &&
                  "All users must be of VF size.");
+          if (SLPReVec) {
+            assert(SLPReVec && "Only supported by REVEC.");
+            // ShuffleVectorInst does not do reorderOperands (and it should not
+            // because ShuffleVectorInst supports only a limited set of
+            // patterns). Only do reorderNodeWithReuses if all of the users are
+            // not ShuffleVectorInst.
+            if (all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) {
+                  return isa<ShuffleVectorInst>(EI.UserTE->getMainOp());
+                }))
+              continue;
+            assert(none_of(TE->UserTreeIndices,
+                           [&](const EdgeInfo &EI) {
+                             return isa<ShuffleVectorInst>(
+                                 EI.UserTE->getMainOp());
+                           }) &&
+                   "Does not know how to reorder.");
+          }
           // Update ordering of the operands with the smaller VF than the given
           // one.
           reorderNodeWithReuses(*TE, Mask);
@@ -6919,8 +6925,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
               // 2. All users are deleted.
               // 3. The load broadcasts are not allowed or the load is not
               // broadcasted.
-              if (std::distance(LI->user_begin(), LI->user_end()) !=
-                      LI->getNumUses())
+              if (static_cast<unsigned int>(std::distance(
+                      LI->user_begin(), LI->user_end())) != LI->getNumUses())
                 return false;
               if (!IsLegalBroadcastLoad)
                 continue;
@@ -7410,17 +7416,17 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
   unsigned Opcode1 = S.getAltOpcode();
   SmallBitVector OpcodeMask(getAltInstrMask(VL, Opcode0, Opcode1));
   // If this pattern is supported by the target then consider it profitable.
-  if (TTI->isLegalAltInstr(getWidenedType(S.MainOp->getType(), VL.size()),
+  if (TTI->isLegalAltInstr(getWidenedType(S.getMainOp()->getType(), VL.size()),
                            Opcode0, Opcode1, OpcodeMask))
     return true;
   SmallVector<ValueList> Operands;
-  for (unsigned I : seq<unsigned>(0, S.MainOp->getNumOperands())) {
+  for (unsigned I : seq<unsigned>(S.getMainOp()->getNumOperands())) {
     Operands.emplace_back();
     // Prepare the operand vector.
     for (Value *V : VL) {
       if (isa<PoisonValue>(V)) {
         Operands.back().push_back(
-            PoisonValue::get(S.MainOp->getOperand(I)->getType()));
+            PoisonValue::get(S.getMainOp()->getOperand(I)->getType()));
         continue;
       }
       Operands.back().push_back(cast<Instruction>(V)->getOperand(I));
@@ -7470,7 +7476,7 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
       ++ExtraShuffleInsts;
     }
   }
-  const Loop *L = LI->getLoopFor(S.MainOp->getParent());
+  const Loop *L = LI->getLoopFor(S.getMainOp()->getParent());
   // Vectorize node, if:
   // 1. at least single operand is constant or splat.
   // 2. Operands have many loop invariants (the instructions are not loop
@@ -7480,7 +7486,7 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
                  [&](ArrayRef<Value *> Op) {
                    if (allConstant(Op) ||
                        (!isSplat(Op) && allSameBlock(Op) && allSameType(Op) &&
-                        getSameOpcode(Op, *TLI).MainOp))
+                        getSameOpcode(Op, *TLI).getMainOp()))
                      return false;
                    DenseMap<Value *, unsigned> Uniques;
                    for (Value *V : Op) {
@@ -7512,19 +7518,21 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
          // vector operands is number of vector instructions + number of vector
          // instructions for operands (buildvectors). Number of buildvector
          // instructions is just number_of_operands * number_of_scalars.
-         (UndefCnt < (VL.size() - 1) * S.MainOp->getNumOperands() &&
+         (UndefCnt < (VL.size() - 1) * S.getMainOp()->getNumOperands() &&
           (UniqueOpcodes.size() + NonInstCnt + ExtraShuffleInsts +
-           NumAltInsts) < S.MainOp->getNumOperands() * VL.size());
+           NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
 }
 
 BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
-    InstructionsState &S, ArrayRef<Value *> VL, bool IsScatterVectorizeUserTE,
-    OrdersType &CurrentOrder, SmallVectorImpl<Value *> &PointerOps) {
-  assert(S.MainOp && "Expected instructions with same/alternate opcodes only.");
+    const InstructionsState &S, ArrayRef<Value *> VL,
+    bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
+    SmallVectorImpl<Value *> &PointerOps) {
+  assert(S.getMainOp() &&
+         "Expected instructions with same/alternate opcodes only.");
 
   unsigned ShuffleOrOp =
       S.isAltShuffle() ? (unsigned)Instruction::ShuffleVector : S.getOpcode();
-  auto *VL0 = cast<Instruction>(S.OpValue);
+  Instruction *VL0 = S.getMainOp();
   switch (ShuffleOrOp) {
   case Instruction::PHI: {
     // Too many operands - gather, most probably won't be vectorized.
@@ -7696,7 +7704,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
   case Instruction::Or:
   case Instruction::Xor:
   case Instruction::Freeze:
-    if (S.MainOp->getType()->isFloatingPointTy() &&
+    if (S.getMainOp()->getType()->isFloatingPointTy() &&
         TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
           auto *I = dyn_cast<Instruction>(V);
           return I && I->isBinaryOp() && !I->isFast();
@@ -7793,7 +7801,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
     return TreeEntry::NeedToGather;
   }
   case Instruction::Call: {
-    if (S.MainOp->getType()->isFloatingPointTy() &&
+    if (S.getMainOp()->getType()->isFloatingPointTy() &&
         TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
           auto *I = dyn_cast<Instruction>(V);
           return I && !I->isFast();
@@ -8019,7 +8027,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
              return isa<UndefValue>(V) || !isConstant(V);
            }))) {
         if (DoNotFail && UniquePositions.size() > 1 &&
-            NumUniqueScalarValues > 1 && S.MainOp->isSafeToRemove() &&
+            NumUniqueScalarValues > 1 && S.getMainOp()->isSafeToRemove() &&
             all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
           // Find the number of elements, which forms full vectors.
           unsigned PWSz = getFullVectorNumberOfElements(
@@ -8049,8 +8057,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
   // Don't go into catchswitch blocks, which can happen with PHIs.
   // Such blocks can only have PHIs and the catchswitch.  There is no
   // place to insert a shuffle if we need to, so just avoid that issue.
-  if (S.MainOp &&
-      isa<CatchSwitchInst>(S.MainOp->getParent()->getTerminator())) {
+  if (S.getMainOp() &&
+      isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
     LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
     newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
     return;
@@ -8058,10 +8066,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
   // Check if this is a duplicate of another entry.
   if (S.getOpcode()) {
-    if (TreeEntry *E = getTreeEntry(S.OpValue)) {
-      LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
+    if (TreeEntry *E = getTreeEntry(S.getMainOp())) {
+      LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp()
+                        << ".\n");
       if (GatheredLoadsEntriesFirst.has_value() || !E->isSame(VL)) {
-        auto It = MultiNodeScalars.find(S.OpValue);
+        auto It = MultiNodeScalars.find(S.getMainOp());
         if (It != MultiNodeScalars.end()) {
           auto *TEIt = find_if(It->getSecond(),
                                [&](TreeEntry *ME) { return ME->isSame(VL); });
@@ -8074,7 +8083,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         }
       }
       if (!E) {
-        if (!doesNotNeedToBeScheduled(S.OpValue)) {
+        if (!doesNotNeedToBeScheduled(S.getMainOp())) {
           LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
           if (TryToFindDuplicates(S))
             newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
@@ -8082,8 +8091,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           return;
         }
         SmallPtrSet<const TreeEntry *, 4> Nodes;
-        Nodes.insert(getTreeEntry(S.OpValue));
-        for (const TreeEntry *E : MultiNodeScalars.lookup(S.OpValue))
+        Nodes.insert(getTreeEntry(S.getMainOp()));
+        for (const TreeEntry *E : MultiNodeScalars.lookup(S.getMainOp()))
           Nodes.insert(E);
         SmallPtrSet<Value *, 8> Values(VL.begin(), VL.end());
         if (any_of(Nodes, [&](const TreeEntry *E) {
@@ -8106,7 +8115,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         // used to properly draw the graph rather than for the actual
         // vectorization.
         E->UserTreeIndices.push_back(UserTreeIdx);
-        LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue
+        LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
                           << ".\n");
         return;
       }
@@ -8117,13 +8126,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
   // a load), in which case peek through to include it in the tree, without
   // ballooning over-budget.
   if (Depth >= RecursionMaxDepth &&
-      !(S.MainOp && isa<Instruction>(S.MainOp) && S.MainOp == S.AltOp &&
-        VL.size() >= 4 &&
-        (match(S.MainOp, m_Load(m_Value())) || all_of(VL, [&S](const Value *I) {
+      !(S.getMainOp() && !S.isAltShuffle() && VL.size() >= 4 &&
+        (match(S.getMainOp(), m_Load(m_Value())) ||
+         all_of(VL, [&S](const Value *I) {
            return match(I,
                         m_OneUse(m_ZExtOrSExt(m_OneUse(m_Load(m_Value()))))) &&
                   cast<Instruction>(I)->getOpcode() ==
-                      cast<Instruction>(S.MainOp)->getOpcode();
+                      S.getMainOp()->getOpcode();
          })))) {
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
     if (TryToFindDuplicates(S))
@@ -8135,7 +8144,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
   // Don't handle scalable vectors
   if (S.getOpcode() == Instruction::ExtractElement &&
       isa<ScalableVectorType>(
-          cast<ExtractElementInst>(S.OpValue)->getVectorOperandType())) {
+          cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
     if (TryToFindDuplicates(S))
       newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
@@ -8172,7 +8181,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         return isa<Instruction>(Op) || isVectorLikeInstWithConstOps(Op);
       }));
     }
-    bool IsCommutative = isCommutative(S.MainOp) || isCommutative(S.AltOp);
+    bool IsCommutative =
+        isCommutative(S.getMainOp()) || isCommutative(S.getAltOp());
     if ((IsCommutative &&
          std::accumulate(InstsCount.begin(), InstsCount.end(), 0) < 2) ||
         (!IsCommutative &&
@@ -8182,20 +8192,20 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     SmallVector<SmallVector<std::pair<Value *, Value *>>> Candidates;
     auto *I1 = cast<Instruction>(VL.front());
     auto *I2 = cast<Instruction>(VL.back());
-    for (int Op = 0, E = S.MainOp->getNumOperands(); Op < E; ++Op)
+    for (int Op : seq<int>(S.getMainOp()->getNumOperands()))
       Candidates.emplace_back().emplace_back(I1->getOperand(Op),
                                              I2->getOperand(Op));
     if (static_cast<unsigned>(count_if(
             Candidates, [this](ArrayRef<std::pair<Value *, Value *>> Cand) {
               return findBestRootPair(Cand, LookAheadHeuristics::ScoreSplat);
-            })) >= S.MainOp->getNumOperands() / 2)
+            })) >= S.getMainOp()->getNumOperands() / 2)
       return false;
-    if (S.MainOp->getNumOperands() > 2)
+    if (S.getMainOp()->getNumOperands() > 2)
       return true;
     if (IsCommutative) {
       // Check permuted operands.
       Candidates.clear();
-      for (int Op = 0, E = S.MainOp->getNumOperands(); Op < E; ++Op)
+      for (int Op = 0, E = S.getMainOp()->getNumOperands(); Op < E; ++Op)
         Candidates.emplace_back().emplace_back(I1->getOperand(Op),
                                                I2->getOperand((Op + 1) % E));
       if (any_of(
@@ -8230,7 +8240,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
   bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
   if (!AreAllSameInsts || (!S.getOpcode() && allConstant(VL)) || isSplat(VL) ||
       (isa_and_present<InsertElementInst, ExtractValueInst, ExtractElementInst>(
-           S.OpValue) &&
+           S.getMainOp()) &&
        !all_of(VL, isVectorLikeInstWithConstOps)) ||
       NotProfitableForVectorization(VL)) {
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
@@ -8297,10 +8307,10 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
   // Check that all of the users of the scalars that we want to vectorize are
   // schedulable.
-  auto *VL0 = cast<Instruction>(S.OpValue);
+  Instruction *VL0 = S.getMainOp();
   BB = VL0->getParent();
 
-  if (S.MainOp &&
+  if (S.getMainOp() &&
       (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()) ||
        !DT->isReachableFromEntry(BB))) {
     // Don't go into unreachable blocks. They may contain instructions with
@@ -8443,7 +8453,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                                    {}, CurrentOrder);
       LLVM_DEBUG(dbgs() << "SLP: added inserts bundle.\n");
 
-      TE->setOperandsInOrder();
+      TE->setOperand(*this);
       buildTree_rec(TE->getOperand(1), Depth + 1, {TE, 1});
       return;
     }
@@ -8464,27 +8474,26 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
         else
           LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
-        TE->setOperandsInOrder();
         break;
       case TreeEntry::StridedVectorize:
         // Vectorizing non-consecutive loads with `llvm.masked.gather`.
         TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
                           UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
-        TE->setOperandsInOrder();
         LLVM_DEBUG(dbgs() << "SLP: added a vector of strided loads.\n");
         break;
       case TreeEntry::ScatterVectorize:
         // Vectorizing non-consecutive loads with `llvm.masked.gather`.
         TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
                           UserTreeIdx, ReuseShuffleIndices);
-        TE->setOperandsInOrder();
-        buildTree_rec(PointerOps, Depth + 1, {TE, 0});
         LLVM_DEBUG(dbgs() << "SLP: added a vector of non-consecutive loads.\n");
         break;
       case TreeEntry::CombinedVectorize:
       case TreeEntry::NeedToGather:
         llvm_unreachable("Unexpected loads state.");
       }
+      TE->setOperand(*this);
+      if (State == TreeEntry::ScatterVectorize)
+        buildTree_rec(PointerOps, Depth + 1, {TE, 0});
       return;
     }
     case Instruction::ZExt:
@@ -8522,8 +8531,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                                    ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
 
-      TE->setOperandsInOrder();
-      for (unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
+      TE->setOperand(*this);
+      for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
         buildTree_rec(TE->getOperand(I), Depth + 1, {TE, I});
       if (ShuffleOrOp == Instruction::Trunc) {
         ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
@@ -8550,12 +8559,15 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
 
       ValueList Left, Right;
+      VLOperands Ops(VL, VL0, *this);
       if (cast<CmpInst>(VL0)->isCommutative()) {
         // Commutative predicate - collect + sort operands of the instructions
         // so that each side is more likely to have the same opcode.
         assert(P0 == CmpInst::getSwappedPredicate(P0) &&
                "Commutative Predicate mismatch");
-        reorderInputsAccordingToOpcode(VL, Left, Right, *this);
+        Ops.reorder();
+        Left = Ops.getVL(0);
+        Right = Ops.getVL(1);
       } else {
         // Collect operands - commute if it uses the swapped predicate.
         for (Value *V : VL) {
@@ -8616,20 +8628,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                                    ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n");
 
-      // Sort operands of the instructions so that each side is more likely to
-      // have the same opcode.
-      if (isa<BinaryOperator>(VL0) && isCommutative(VL0)) {
-        ValueList Left, Right;
-        reorderInputsAccordingToOpcode(VL, Left, Right, *this);
-        TE->setOperand(0, Left);
-        TE->setOperand(1, Right);
-        buildTree_rec(Left, Depth + 1, {TE, 0});
-        buildTree_rec(Right, Depth + 1, {TE, 1});
-        return;
-      }
-
-      TE->setOperandsInOrder();
-      for (unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
+      TE->setOperand(*this, isa<BinaryOperator>(VL0) && isCommutative(VL0));
+      for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
         buildTree_rec(TE->getOperand(I), Depth + 1, {TE, I});
       return;
     }
@@ -8694,7 +8694,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         fixupOrderingIndices(CurrentOrder);
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
                                    ReuseShuffleIndices, CurrentOrder);
-      TE->setOperandsInOrder();
+      TE->setOperand(*this);
       buildTree_rec(TE->getOperand(0), Depth + 1, {TE, 0});
       if (Consecutive)
         LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
@@ -8710,46 +8710,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
                                    ReuseShuffleIndices);
-      // Sort operands of the instructions so that each side is more likely to
-      // have the same opcode.
-      if (isCommutative(VL0)) {
-        ValueList Left, Right;
-        reorderInputsAccordingToOpcode(VL, Left, Right, *this);
-        TE->setOperand(0, Left);
-        TE->setOperand(1, Right);
-        SmallVector<ValueList> Operands;
-        for (unsigned I : seq<unsigned>(2, CI->arg_size())) {
-          Operands.emplace_back();
-          if (isVectorIntrinsicWithScalarOpAtArg(ID, I))
-            continue;
-          for (Value *V : VL) {
-            auto *CI2 = cast<CallInst>(V);
-            Operands.back().push_back(CI2->getArgOperand(I));
-          }
-          TE->setOperand(I, Operands.back());
-        }
-        buildTree_rec(Left, Depth + 1, {TE, 0});
-        buildTree_rec(Right, Depth + 1, {TE, 1});
-        for (unsigned I : seq<unsigned>(2, CI->arg_size())) {
-          if (Operands[I - 2].empty())
-            continue;
-          buildTree_rec(Operands[I - 2], Depth + 1, {TE, I});
-        }
-        return;
-      }
-      TE->setOperandsInOrder();
-      for (unsigned I : seq<unsigned>(0, CI->arg_size())) {
+      TE->setOperand(*this, isCommutative(VL0));
+      for (unsigned I : seq<unsigned>(CI->arg_size())) {
         // For scalar operands no need to create an entry since no need to
         // vectorize it.
         if (isVectorIntrinsicWithScalarOpAtArg(ID, I))
           continue;
-        ValueList Operands;
-        // Prepare the operand vector.
-        for (Value *V : VL) {
-          auto *CI2 = cast<CallInst>(V);
-          Operands.push_back(CI2->getArgOperand(I));
-        }
-        buildTree_rec(Operands, Depth + 1, {TE, I});
+        buildTree_rec(TE->getOperand(I), Depth + 1, {TE, I});
       }
       return;
     }
@@ -8760,43 +8727,37 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
       // Reorder operands if reordering would enable vectorization.
       auto *CI = dyn_cast<CmpInst>(VL0);
-      if (isa<BinaryOperator>(VL0) || CI) {
+      if (CI && any_of(VL, [](Value *V) {
+            return !isa<PoisonValue>(V) && !cast<CmpInst>(V)->isCommutative();
+          })) {
+        auto *MainCI = cast<CmpInst>(S.getMainOp());
+        auto *AltCI = cast<CmpInst>(S.getAltOp());
+        CmpInst::Predicate MainP = MainCI->getPredicate();
+        CmpInst::Predicate AltP = AltCI->getPredicate();
+        assert(MainP != AltP &&
+               "Expected different main/alternate predicates.");
         ValueList Left, Right;
-        if (!CI || all_of(VL, [](Value *V) {
-              return isa<PoisonValue>(V) || cast<CmpInst>(V)->isCommutative();
-            })) {
-          reorderInputsAccordingToOpcode(VL, Left, Right, *this);
-        } else {
-          auto *MainCI = cast<CmpInst>(S.MainOp);
-          auto *AltCI = cast<CmpInst>(S.AltOp);
-          CmpInst::Predicate MainP = MainCI->getPredicate();
-          CmpInst::Predicate AltP = AltCI->getPredicate();
-          assert(MainP != AltP &&
-                 "Expected different main/alternate predicates.");
-          // Collect operands - commute if it uses the swapped predicate or
-          // alternate operation.
-          for (Value *V : VL) {
-            if (isa<PoisonValue>(V)) {
-              Left.push_back(
-                  PoisonValue::get(MainCI->getOperand(0)->getType()));
-              Right.push_back(
-                  PoisonValue::get(MainCI->getOperand(1)->getType()));
-              continue;
-            }
-            auto *Cmp = cast<CmpInst>(V);
-            Value *LHS = Cmp->getOperand(0);
-            Value *RHS = Cmp->getOperand(1);
+        // Collect operands - commute if it uses the swapped predicate or
+        // alternate operation.
+        for (Value *V : VL) {
+          if (isa<PoisonValue>(V)) {
+            Left.push_back(PoisonValue::get(MainCI->getOperand(0)->getType()));
+            Right.push_back(PoisonValue::get(MainCI->getOperand(1)->getType()));
+            continue;
+          }
+          auto *Cmp = cast<CmpInst>(V);
+          Value *LHS = Cmp->getOperand(0);
+          Value *RHS = Cmp->getOperand(1);
 
-            if (isAlternateInstruction(Cmp, MainCI, AltCI, *TLI)) {
-              if (AltP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
-                std::swap(LHS, RHS);
-            } else {
-              if (MainP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
-                std::swap(LHS, RHS);
-            }
-            Left.push_back(LHS);
-            Right.push_back(RHS);
+          if (isAlternateInstruction(Cmp, MainCI, AltCI, *TLI)) {
+            if (AltP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
+              std::swap(LHS, RHS);
+          } else {
+            if (MainP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
+              std::swap(LHS, RHS);
           }
+          Left.push_back(LHS);
+          Right.push_back(RHS);
         }
         TE->setOperand(0, Left);
         TE->setOperand(1, Right);
@@ -8805,8 +8766,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         return;
       }
 
-      TE->setOperandsInOrder();
-      for (unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
+      TE->setOperand(*this, isa<BinaryOperator>(VL0) || CI);
+      for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
         buildTree_rec(TE->getOperand(I), Depth + 1, {TE, I});
       return;
     }
@@ -9691,7 +9652,7 @@ void BoUpSLP::transformNodes() {
   auto CheckOperandsProfitability = [this](Instruction *I1, Instruction *I2,
                                            const InstructionsState &S) {
     SmallVector<SmallVector<std::pair<Value *, Value *>>> Candidates;
-    for (unsigned Op : seq<unsigned>(S.MainOp->getNumOperands()))
+    for (unsigned Op : seq<unsigned>(S.getMainOp()->getNumOperands()))
       Candidates.emplace_back().emplace_back(I1->getOperand(Op),
                                              I2->getOperand(Op));
     return all_of(
@@ -9762,7 +9723,8 @@ void BoUpSLP::transformNodes() {
                                              Slice.front()->getType(), 2 * VF)),
                                          1U, 2 * VF)) ||
               count(Slice, Slice.front()) ==
-                  (isa<UndefValue>(Slice.front()) ? VF - 1 : 1)) {
+                  static_cast<long>(isa<UndefValue>(Slice.front()) ? VF - 1
+                                                                   : 1)) {
             if (IsSplat)
               continue;
             InstructionsState S = getSameOpcode(Slice, *TLI);
@@ -9775,7 +9737,7 @@ void BoUpSLP::transformNodes() {
               // Try to vectorize reduced values or if all users are vectorized.
               // For expensive instructions extra extracts might be profitable.
               if ((!UserIgnoreList || E.Idx != 0) &&
-                  TTI->getInstructionCost(S.MainOp, CostKind) <
+                  TTI->getInstructionCost(S.getMainOp(), CostKind) <
                       TTI::TCC_Expensive &&
                   !all_of(Slice, [&](Value *V) {
                     if (isa<PoisonValue>(V))
@@ -9802,10 +9764,10 @@ void BoUpSLP::transformNodes() {
                   continue;
                 }
               } else if (S.getOpcode() == Instruction::ExtractElement ||
-                         (TTI->getInstructionCost(S.MainOp, CostKind) <
+                         (TTI->getInstructionCost(S.getMainOp(), CostKind) <
                               TTI::TCC_Expensive &&
                           !CheckOperandsProfitability(
-                              S.MainOp,
+                              S.getMainOp(),
                               cast<Instruction>(*find_if(reverse(Slice),
                                                          IsaPred<Instruction>)),
                               S))) {
@@ -13193,9 +13155,9 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
         copy(CommonMask, Mask.begin());
       }
       // Clear undef scalars.
-      for (int I = 0, Sz = VL.size(); I < Sz; ++I)
+      for (unsigned I : seq<unsigned>(VL.size()))
         if (isa<PoisonValue>(VL[I]))
-          Mask[I] = PoisonMaskElem;
+          Mask[Part * VL.size() + I] = PoisonMaskElem;
       return TargetTransformInfo::SK_PermuteSingleSrc;
     }
     // No perfect match, just shuffle, so choose the first tree node from the
@@ -13498,9 +13460,10 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
               TTI::SK_InsertSubvector, VecTy, std::nullopt, CostKind,
               I * ScalarTyNumElements, cast<FixedVectorType>(ScalarTy));
     } else {
-      Cost = TTI->getScalarizationOverhead(VecTy, ~ShuffledElements,
+      Cost = TTI->getScalarizationOverhead(VecTy,
+                                           /*DemandedElts*/ ~ShuffledElements,
                                            /*Insert*/ true,
-                                           /*Extract*/ false, CostKind);
+                                           /*Extract*/ false, CostKind, VL);
     }
   }
   if (DuplicateNonConst)
@@ -13509,21 +13472,6 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
   return Cost;
 }
 
-// Perform operand reordering on the instructions in VL and return the reordered
-// operands in Left and Right.
-void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
-                                             SmallVectorImpl<Value *> &Left,
-                                             SmallVectorImpl<Value *> &Right,
-                                             const BoUpSLP &R) {
-  if (VL.empty())
-    return;
-  VLOperands Ops(VL, R);
-  // Reorder the operands in place.
-  Ops.reorder();
-  Left = Ops.getVL(0);
-  Right = Ops.getVL(1);
-}
-
 Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
   auto &Res = EntryToLastInstruction.try_emplace(E).first->second;
   if (Res)
@@ -14464,10 +14412,10 @@ BoUpSLP::TreeEntry *BoUpSLP::getMatchedVectorizedOperand(const TreeEntry *E,
                             VE->isSame(TE->Scalars);
                    }));
   };
-  TreeEntry *VE = getTreeEntry(S.OpValue);
+  TreeEntry *VE = getTreeEntry(S.getMainOp());
   if (VE && CheckSameVE(VE))
     return VE;
-  auto It = MultiNodeScalars.find(S.OpValue);
+  auto It = MultiNodeScalars.find(S.getMainOp());
   if (It != MultiNodeScalars.end()) {
     auto *I = find_if(It->getSecond(), [&](const TreeEntry *TE) {
       return TE != VE && CheckSameVE(TE);
@@ -16197,6 +16145,11 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
     }
     Builder.SetCurrentDebugLocation(UserI->getDebugLoc());
     Value *Vec = vectorizeTree(TE, /*PostponedPHIs=*/false);
+    if (auto *VecI = dyn_cast<Instruction>(Vec);
+        VecI && VecI->getParent() == Builder.GetInsertBlock() &&
+        Builder.GetInsertPoint()->comesBefore(VecI))
+      VecI->moveBeforePreserving(*Builder.GetInsertBlock(),
+                                 Builder.GetInsertPoint());
     if (Vec->getType() != PrevVec->getType()) {
       assert(Vec->getType()->isIntOrIntVectorTy() &&
              PrevVec->getType()->isIntOrIntVectorTy() &&
@@ -16949,13 +16902,13 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
                                             const InstructionsState &S) {
   // No need to schedule PHIs, insertelement, extractelement and extractvalue
   // instructions.
-  if (isa<PHINode>(S.OpValue) || isVectorLikeInstWithConstOps(S.OpValue) ||
-      doesNotNeedToSchedule(VL))
+  if (isa<PHINode>(S.getMainOp()) ||
+      isVectorLikeInstWithConstOps(S.getMainOp()) || doesNotNeedToSchedule(VL))
     return nullptr;
 
   // Initialize the instruction bundle.
   Instruction *OldScheduleEnd = ScheduleEnd;
-  LLVM_DEBUG(dbgs() << "SLP:  bundle: " << *S.OpValue << "\n");
+  LLVM_DEBUG(dbgs() << "SLP:  bundle: " << *S.getMainOp() << "\n");
 
   auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](bool ReSchedule,
                                                          ScheduleData *Bundle) {
@@ -17036,7 +16989,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
   auto *Bundle = buildBundle(VL);
   TryScheduleBundleImpl(ReSchedule, Bundle);
   if (!Bundle->isReady()) {
-    cancelScheduling(VL, S.OpValue);
+    cancelScheduling(VL, S.getMainOp());
     return std::nullopt;
   }
   return Bundle;
@@ -17799,6 +17752,8 @@ bool BoUpSLP::collectValuesToDemote(
     // original type and the sign bit of the truncate type are similar.
     auto AShrChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
       return all_of(E.Scalars, [&](Value *V) {
+        if (isa<PoisonValue>(V))
+          return true;
         auto *I = cast<Instruction>(V);
         KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
         unsigned ShiftedBits = OrigBitWidth - BitWidth;
@@ -18413,7 +18368,7 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
         (VectorizeNonPowerOf2 && has_single_bit(ValOps.size() + 1));
     if ((!IsAllowedSize && S.getOpcode() &&
          S.getOpcode() != Instruction::Load &&
-         (!S.MainOp->isSafeToRemove() ||
+         (!S.getMainOp()->isSafeToRemove() ||
           any_of(ValOps.getArrayRef(),
                  [&](Value *V) {
                    return !isa<ExtractElementInst>(V) &&
@@ -18950,7 +18905,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
   if (!S.getOpcode())
     return false;
 
-  Instruction *I0 = cast<Instruction>(S.OpValue);
+  Instruction *I0 = S.getMainOp();
   // Make sure invalid types (including vector type) are rejected before
   // determining vectorization factor for scalar instructions.
   for (Value *V : VL) {
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index 4b0e12c28f07b..25f2665d450d1 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -325,6 +325,67 @@ void DependencyGraph::createNewNodes(const Interval<Instruction> &NewInterval) {
   setDefUseUnscheduledSuccs(NewInterval);
 }
 
+MemDGNode *DependencyGraph::getMemDGNodeBefore(DGNode *N,
+                                               bool IncludingN) const {
+  auto *I = N->getInstruction();
+  for (auto *PrevI = IncludingN ? I : I->getPrevNode(); PrevI != nullptr;
+       PrevI = PrevI->getPrevNode()) {
+    auto *PrevN = getNodeOrNull(PrevI);
+    if (PrevN == nullptr)
+      return nullptr;
+    if (auto *PrevMemN = dyn_cast<MemDGNode>(PrevN))
+      return PrevMemN;
+  }
+  return nullptr;
+}
+
+MemDGNode *DependencyGraph::getMemDGNodeAfter(DGNode *N,
+                                              bool IncludingN) const {
+  auto *I = N->getInstruction();
+  for (auto *NextI = IncludingN ? I : I->getNextNode(); NextI != nullptr;
+       NextI = NextI->getNextNode()) {
+    auto *NextN = getNodeOrNull(NextI);
+    if (NextN == nullptr)
+      return nullptr;
+    if (auto *NextMemN = dyn_cast<MemDGNode>(NextN))
+      return NextMemN;
+  }
+  return nullptr;
+}
+
+void DependencyGraph::notifyCreateInstr(Instruction *I) {
+  auto *MemN = dyn_cast<MemDGNode>(getOrCreateNode(I));
+  // TODO: Update the dependencies for the new node.
+
+  // Update the MemDGNode chain if this is a memory node.
+  if (MemN != nullptr) {
+    if (auto *PrevMemN = getMemDGNodeBefore(MemN, /*IncludingN=*/false)) {
+      PrevMemN->NextMemN = MemN;
+      MemN->PrevMemN = PrevMemN;
+    }
+    if (auto *NextMemN = getMemDGNodeAfter(MemN, /*IncludingN=*/false)) {
+      NextMemN->PrevMemN = MemN;
+      MemN->NextMemN = NextMemN;
+    }
+  }
+}
+
+void DependencyGraph::notifyEraseInstr(Instruction *I) {
+  // Update the MemDGNode chain if this is a memory node.
+  if (auto *MemN = dyn_cast_or_null<MemDGNode>(getNodeOrNull(I))) {
+    auto *PrevMemN = getMemDGNodeBefore(MemN, /*IncludingN=*/false);
+    auto *NextMemN = getMemDGNodeAfter(MemN, /*IncludingN=*/false);
+    if (PrevMemN != nullptr)
+      PrevMemN->NextMemN = NextMemN;
+    if (NextMemN != nullptr)
+      NextMemN->PrevMemN = PrevMemN;
+  }
+
+  InstrToNodeMap.erase(I);
+
+  // TODO: Update the dependencies.
+}
+
 Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
   if (Instrs.empty())
     return {};
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 529108a5aaa97..5122232ffe9b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -222,9 +222,10 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() {
 VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
                                    ElementCount VF, unsigned UF, LoopInfo *LI,
                                    DominatorTree *DT, IRBuilderBase &Builder,
-                                   InnerLoopVectorizer *ILV, VPlan *Plan)
+                                   InnerLoopVectorizer *ILV, VPlan *Plan,
+                                   Type *CanonicalIVTy)
     : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
-      LVer(nullptr), TypeAnalysis(Plan->getCanonicalIV()->getScalarType()) {}
+      LVer(nullptr), TypeAnalysis(CanonicalIVTy) {}
 
 Value *VPTransformState::get(VPValue *Def, const VPLane &Lane) {
   if (Def->isLiveIn())
@@ -927,7 +928,6 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
 }
 
 void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
-                             Value *CanonicalIVStartValue,
                              VPTransformState &State) {
   Type *TCTy = TripCountV->getType();
   // Check if the backedge taken count is needed, and if so build it.
@@ -953,25 +953,6 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
   } else {
     VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF));
   }
-
-  // When vectorizing the epilogue loop, the canonical induction start value
-  // needs to be changed from zero to the value after the main vector loop.
-  // FIXME: Improve modeling for canonical IV start values in the epilogue loop.
-  if (CanonicalIVStartValue) {
-    VPValue *VPV = getOrAddLiveIn(CanonicalIVStartValue);
-    auto *IV = getCanonicalIV();
-    assert(all_of(IV->users(),
-                  [](const VPUser *U) {
-                    return isa<VPScalarIVStepsRecipe>(U) ||
-                           isa<VPScalarCastRecipe>(U) ||
-                           isa<VPDerivedIVRecipe>(U) ||
-                           cast<VPInstruction>(U)->getOpcode() ==
-                               Instruction::Add;
-                  }) &&
-           "the canonical IV should only be used by its increment or "
-           "ScalarIVSteps when resetting the start value");
-    IV->setOperand(0, VPV);
-  }
 }
 
 /// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
@@ -1015,6 +996,11 @@ void VPlan::execute(VPTransformState *State) {
   replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh);
   replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
 
+  LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF
+                    << ", UF=" << getUF() << '\n');
+  setName("Final VPlan");
+  LLVM_DEBUG(dump());
+
   // Disconnect the middle block from its single successor (the scalar loop
   // header) in both the CFG and DT. The branch will be recreated during VPlan
   // execution.
@@ -1028,8 +1014,11 @@ void VPlan::execute(VPTransformState *State) {
   State->CFG.DTU.applyUpdates(
       {{DominatorTree::Delete, ScalarPh, ScalarPh->getSingleSuccessor()}});
 
-  // Generate code in the loop pre-header and body.
-  for (VPBlockBase *Block : vp_depth_first_shallow(Entry))
+  ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+      Entry);
+  // Generate code for the VPlan, in parts of the vector skeleton, loop body and
+  // successor blocks including the middle, exit and scalar preheader blocks.
+  for (VPBlockBase *Block : RPOT)
     Block->execute(State);
 
   VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
@@ -1070,10 +1059,9 @@ void VPlan::execute(VPTransformState *State) {
     }
 
     auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
-    bool NeedsScalar =
-        isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
-        (isa<VPReductionPHIRecipe>(PhiR) &&
-         cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
+    bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
+                       (isa<VPReductionPHIRecipe>(PhiR) &&
+                        cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
     Value *Phi = State->get(PhiR, NeedsScalar);
     Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
     cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
@@ -1140,7 +1128,9 @@ void VPlan::print(raw_ostream &O) const {
     getPreheader()->print(O, "", SlotTracker);
   }
 
-  for (const VPBlockBase *Block : vp_depth_first_shallow(getEntry())) {
+  ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<const VPBlockBase *>>
+      RPOT(getEntry());
+  for (const VPBlockBase *Block : RPOT) {
     O << '\n';
     Block->print(O, "", SlotTracker);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1b1630ebc6c23..8794517b777f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -236,7 +236,7 @@ class VPLane {
 struct VPTransformState {
   VPTransformState(const TargetTransformInfo *TTI, ElementCount VF, unsigned UF,
                    LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder,
-                   InnerLoopVectorizer *ILV, VPlan *Plan);
+                   InnerLoopVectorizer *ILV, VPlan *Plan, Type *CanonicalIVTy);
   /// Target Transform Info.
   const TargetTransformInfo *TTI;
 
@@ -2239,6 +2239,45 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe,
 #endif
 };
 
+/// Recipe to generate a scalar PHI. Used to generate code for recipes that
+/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
+/// VPEVLBasedIVPHIRecipe.
+class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
+  std::string Name;
+
+public:
+  VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
+                    StringRef Name)
+      : VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
+        Name(Name.str()) {
+    addOperand(BackedgeValue);
+  }
+
+  ~VPScalarPHIRecipe() override = default;
+
+  VPScalarPHIRecipe *clone() override {
+    llvm_unreachable("cloning not implemented yet");
+  }
+
+  VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
+
+  /// Generate the phi/select nodes.
+  void execute(VPTransformState &State) override;
+
+  /// Returns true if the recipe only uses the first lane of operand \p Op.
+  bool onlyFirstLaneUsed(const VPValue *Op) const override {
+    assert(is_contained(operands(), Op) &&
+           "Op must be an operand of the recipe");
+    return true;
+  }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print the recipe.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
 /// A recipe for handling phis that are widened in the vector loop.
 /// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
 /// managed in the recipe directly.
@@ -3134,8 +3173,10 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
     return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
   }
 
-  /// Generate the canonical scalar induction phi of the vector loop.
-  void execute(VPTransformState &State) override;
+  void execute(VPTransformState &State) override {
+    llvm_unreachable(
+        "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+  }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
@@ -3231,9 +3272,10 @@ class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
     return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
   }
 
-  /// Generate phi for handling IV based on EVL over iterations correctly.
-  /// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
-  void execute(VPTransformState &State) override;
+  void execute(VPTransformState &State) override {
+    llvm_unreachable(
+        "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+  }
 
   /// Return the cost of this VPEVLBasedIVPHIRecipe.
   InstructionCost computeCost(ElementCount VF,
@@ -3814,7 +3856,7 @@ class VPlan {
 
   /// Prepare the plan for execution, setting up the required live-in values.
   void prepareToExecute(Value *TripCount, Value *VectorTripCount,
-                        Value *CanonicalIVStartValue, VPTransformState &State);
+                        VPTransformState &State);
 
   /// Generate the IR code for this VPlan.
   void execute(VPTransformState *State);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index cb42cfe8159b0..969d07b229e46 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -213,14 +213,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
       TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
           .Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
                 VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
-                VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
-              [this](const auto *R) {
-                // Handle header phi recipes, except VPWidenIntOrFpInduction
-                // which needs special handling due it being possibly truncated.
-                // TODO: consider inferring/caching type of siblings, e.g.,
-                // backedge value, here and in cases below.
-                return inferScalarType(R->getStartValue());
-              })
+                VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
+                VPScalarPHIRecipe>([this](const auto *R) {
+            // Handle header phi recipes, except VPWidenIntOrFpInduction
+            // which needs special handling due it being possibly truncated.
+            // TODO: consider inferring/caching type of siblings, e.g.,
+            // backedge value, here and in cases below.
+            return inferScalarType(R->getStartValue());
+          })
           .Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
               [](const auto *R) { return R->getScalarType(); })
           .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b2ee31c3e240a..e882368544e81 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -550,7 +550,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     }
     // Reduce all of the unrolled parts into a single vector.
     Value *ReducedPartRdx = RdxParts[0];
-    unsigned Op = RecurrenceDescriptor::getOpcode(RK);
+    unsigned Op = RdxDesc.getOpcode();
     if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK))
       Op = Instruction::Or;
 
@@ -629,7 +629,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
         State.CFG
             .VPBB2IRBB[cast<VPBasicBlock>(getParent()->getSinglePredecessor())];
     NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);
-    for (auto *OtherPred : predecessors(Builder.GetInsertBlock())) {
+    // TODO: Predecessors are temporarily reversed to reduce test changes.
+    // Remove it and update remaining tests after functional change landed.
+    auto Predecessors = to_vector(predecessors(Builder.GetInsertBlock()));
+    for (auto *OtherPred : reverse(Predecessors)) {
       assert(OtherPred != VPlanPred &&
              "VPlan predecessors should not be connected yet");
       NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);
@@ -964,7 +967,8 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
   Module *M = State.Builder.GetInsertBlock()->getModule();
   Function *VectorF =
       Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
-  assert(VectorF && "Can't retrieve vector intrinsic.");
+  assert(VectorF &&
+         "Can't retrieve vector intrinsic or vector-predication intrinsics.");
 
   auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
   SmallVector<OperandBundleDef, 1> OpBundles;
@@ -993,6 +997,15 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
   for (const auto &[Idx, Op] : enumerate(operands())) {
     auto *V = Op->getUnderlyingValue();
     if (!V) {
+      // Push all the VP Intrinsic's ops into the Argments even if is nullptr.
+      // Some VP Intrinsic's cost will assert the number of parameters.
+      // Mainly appears in the following two scenarios:
+      // 1. EVL Op is nullptr
+      // 2. The Argmunt of the VP Intrinsic is also the VP Intrinsic
+      if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID)) {
+        Arguments.push_back(V);
+        continue;
+      }
       if (auto *UI = dyn_cast_or_null<CallBase>(getUnderlyingValue())) {
         Arguments.push_back(UI->getArgOperand(Idx));
         continue;
@@ -1693,13 +1706,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
     Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
 
     // Create a vector splat to use in the induction update.
-    //
-    // FIXME: If the step is non-constant, we create the vector splat with
-    //        IRBuilder. IRBuilder can constant-fold the multiply, but it
-    //        doesn't handle a constant vector splat.
-    SplatVF = isa<Constant>(Mul)
-                  ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul))
-                  : Builder.CreateVectorSplat(State.VF, Mul);
+    SplatVF = Builder.CreateVectorSplat(State.VF, Mul);
   }
 
   Builder.restoreIP(CurrIP);
@@ -2127,8 +2134,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
           createOrderedReduction(State.Builder, RdxDesc, NewVecOp, PrevInChain);
     else
       NewRed = State.Builder.CreateBinOp(
-          (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
-          NewVecOp);
+          (Instruction::BinaryOps)RdxDesc.getOpcode(), PrevInChain, NewVecOp);
     PrevInChain = NewRed;
     NextInChain = NewRed;
   } else {
@@ -2139,7 +2145,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
                                    NewRed, PrevInChain);
     else
       NextInChain = State.Builder.CreateBinOp(
-          (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
+          (Instruction::BinaryOps)RdxDesc.getOpcode(), NewRed, PrevInChain);
   }
   State.set(this, NextInChain, /*IsScalar*/ true);
 }
@@ -2176,8 +2182,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
     if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
       NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);
     else
-      NewRed = Builder.CreateBinOp(
-          (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, Prev);
+      NewRed = Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
+                                   NewRed, Prev);
   }
   State.set(this, NewRed, /*IsScalar*/ true);
 }
@@ -3099,17 +3105,6 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
                                            VectorTy, std::nullopt, CostKind, 0);
 }
 
-void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
-  Value *Start = getStartValue()->getLiveInIRValue();
-  PHINode *Phi = PHINode::Create(Start->getType(), 2, "index");
-  Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
-
-  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
-  Phi->addIncoming(Start, VectorPH);
-  Phi->setDebugLoc(getDebugLoc());
-  State.set(this, Phi, /*IsScalar*/ true);
-}
-
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPCanonicalIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                                    VPSlotTracker &SlotTracker) const {
@@ -3151,8 +3146,6 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
   assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
          "Recipe should have been replaced");
 
-  auto *IVR = getParent()->getPlan()->getCanonicalIV();
-  PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
   unsigned CurrentPart = getUnrollPart(*this);
 
   // Build a pointer phi
@@ -3162,6 +3155,12 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
   BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
   PHINode *NewPointerPhi = nullptr;
   if (CurrentPart == 0) {
+    auto *IVR = cast<VPHeaderPHIRecipe>(&getParent()
+                                             ->getPlan()
+                                             ->getVectorLoopRegion()
+                                             ->getEntryBasicBlock()
+                                             ->front());
+    PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
     NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
                                     CanonicalIV->getIterator());
     NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
@@ -3475,20 +3474,30 @@ void VPActiveLaneMaskPHIRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
+                                  VPSlotTracker &SlotTracker) const {
+  O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
+
+  printAsOperand(O, SlotTracker);
+  O << " = phi ";
+  printOperands(O, SlotTracker);
+}
+#endif
+
+void VPScalarPHIRecipe::execute(VPTransformState &State) {
   BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
-  Value *Start = State.get(getOperand(0), VPLane(0));
-  PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv");
+  Value *Start = State.get(getStartValue(), VPLane(0));
+  PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
   Phi->addIncoming(Start, VectorPH);
   Phi->setDebugLoc(getDebugLoc());
   State.set(this, Phi, /*IsScalar=*/true);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
-                                  VPSlotTracker &SlotTracker) const {
-  O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
-
+void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent,
+                              VPSlotTracker &SlotTracker) const {
+  O << Indent << "SCALAR-PHI ";
   printAsOperand(O, SlotTracker);
   O << " = phi ";
   printOperands(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1d1029710c709..922cba7831f4e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1157,9 +1157,8 @@ void VPlanTransforms::truncateToMinimalBitwidths(
             continue;
           auto *UV = dyn_cast_or_null<Instruction>(Op->getUnderlyingValue());
           if (UV && MinBWs.contains(UV) && !ProcessedTruncs.contains(Op) &&
-              all_of(Op->users(), [](VPUser *U) {
-                return !isa<VPWidenRecipe, VPWidenSelectRecipe>(U);
-              })) {
+              none_of(Op->users(),
+                      IsaPred<VPWidenRecipe, VPWidenSelectRecipe>)) {
             // Add an entry to ProcessedTruncs to avoid counting the same
             // operand multiple times.
             ProcessedTruncs[Op] = nullptr;
@@ -1481,6 +1480,46 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                 VPValue *NewMask = GetNewMask(Red->getCondOp());
                 return new VPReductionEVLRecipe(*Red, EVL, NewMask);
               })
+              .Case<VPWidenIntrinsicRecipe>(
+                  [&](VPWidenIntrinsicRecipe *CInst) -> VPRecipeBase * {
+                    auto *CI = cast<CallInst>(CInst->getUnderlyingInstr());
+                    Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
+                        CI->getCalledFunction()->getIntrinsicID());
+                    if (VPID == Intrinsic::not_intrinsic)
+                      return nullptr;
+
+                    SmallVector<VPValue *> Ops(CInst->operands());
+                    assert(VPIntrinsic::getMaskParamPos(VPID) &&
+                           VPIntrinsic::getVectorLengthParamPos(VPID) &&
+                           "Expected VP intrinsic");
+                    VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(
+                        IntegerType::getInt1Ty(CI->getContext())));
+                    Ops.push_back(Mask);
+                    Ops.push_back(&EVL);
+                    return new VPWidenIntrinsicRecipe(
+                        *CI, VPID, Ops, TypeInfo.inferScalarType(CInst),
+                        CInst->getDebugLoc());
+                  })
+              .Case<VPWidenCastRecipe>(
+                  [&](VPWidenCastRecipe *CInst) -> VPRecipeBase * {
+                    auto *CI = dyn_cast<CastInst>(CInst->getUnderlyingInstr());
+                    Intrinsic::ID VPID =
+                        VPIntrinsic::getForOpcode(CI->getOpcode());
+                    assert(VPID != Intrinsic::not_intrinsic &&
+                           "Expected vp.casts Instrinsic");
+
+                    SmallVector<VPValue *> Ops(CInst->operands());
+                    assert(VPIntrinsic::getMaskParamPos(VPID) &&
+                           VPIntrinsic::getVectorLengthParamPos(VPID) &&
+                           "Expected VP intrinsic");
+                    VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(
+                        IntegerType::getInt1Ty(CI->getContext())));
+                    Ops.push_back(Mask);
+                    Ops.push_back(&EVL);
+                    return new VPWidenIntrinsicRecipe(
+                        VPID, Ops, TypeInfo.inferScalarType(CInst),
+                        CInst->getDebugLoc());
+                  })
               .Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
                 SmallVector<VPValue *> Ops(Sel->operands());
                 Ops.push_back(&EVL);
@@ -1573,10 +1612,9 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   // The transform updates all users of inductions to work based on EVL, instead
   // of the VF directly. At the moment, widened inductions cannot be updated, so
   // bail out if the plan contains any.
-  bool ContainsWidenInductions = any_of(Header->phis(), [](VPRecipeBase &Phi) {
-    return isa<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>(
-        &Phi);
-  });
+  bool ContainsWidenInductions = any_of(
+      Header->phis(),
+      IsaPred<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>);
   if (ContainsWidenInductions)
     return false;
 
@@ -1801,3 +1839,22 @@ void VPlanTransforms::createInterleaveGroups(
       }
   }
 }
+
+void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+           vp_depth_first_deep(Plan.getEntry()))) {
+    for (VPRecipeBase &R : make_early_inc_range(VPBB->phis())) {
+      if (!isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R))
+        continue;
+      auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
+      StringRef Name =
+          isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
+      auto *ScalarR =
+          new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
+                                PhiR->getDebugLoc(), Name);
+      ScalarR->insertBefore(PhiR);
+      PhiR->replaceAllUsesWith(ScalarR);
+      PhiR->eraseFromParent();
+    }
+  }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 11e094db6294f..9cf314a6a9f44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -123,6 +123,9 @@ struct VPlanTransforms {
 
   /// Remove dead recipes from \p Plan.
   static void removeDeadRecipes(VPlan &Plan);
+
+  /// Lower abstract recipes to concrete ones, that can be codegen'd.
+  static void convertToConcreteRecipes(VPlan &Plan);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f653269713b30..ff6c9295ee205 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -168,7 +168,7 @@ void UnrollState::unrollWidenInductionByUF(
   auto *ConstStep = ScalarStep->isLiveIn()
                         ? dyn_cast<ConstantInt>(ScalarStep->getLiveInIRValue())
                         : nullptr;
-  if (!ConstStep || ConstStep->getZExtValue() != 1) {
+  if (!ConstStep || ConstStep->getValue() != 1) {
     if (TypeInfo.inferScalarType(ScalarStep) != IVTy) {
       ScalarStep =
           Builder.createWidenCast(Instruction::Trunc, ScalarStep, IVTy);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 691b0d40823cf..957a602091c73 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -359,6 +359,7 @@ class VPDef {
     VPFirstOrderRecurrencePHISC,
     VPWidenIntOrFpInductionSC,
     VPWidenPointerInductionSC,
+    VPScalarPHISC,
     VPReductionPHISC,
     // END: SubclassID for recipes that inherit VPHeaderPHIRecipe
     // END: Phi-like recipes
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index e80951e15f6d6..0f5a95b4b36d2 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -68,9 +68,10 @@ class VectorCombine {
 public:
   VectorCombine(Function &F, const TargetTransformInfo &TTI,
                 const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
-                const DataLayout *DL, bool TryEarlyFoldsOnly)
+                const DataLayout *DL, TTI::TargetCostKind CostKind,
+                bool TryEarlyFoldsOnly)
       : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), DL(DL),
-        TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
+        CostKind(CostKind), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
 
   bool run();
 
@@ -82,6 +83,7 @@ class VectorCombine {
   AAResults &AA;
   AssumptionCache &AC;
   const DataLayout *DL;
+  TTI::TargetCostKind CostKind;
 
   /// If true, only perform beneficial early IR transforms. Do not introduce new
   /// vector operations.
@@ -253,16 +255,15 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   Type *LoadTy = Load->getType();
   unsigned AS = Load->getPointerAddressSpace();
   InstructionCost OldCost =
-      TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment, AS);
+      TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment, AS, CostKind);
   APInt DemandedElts = APInt::getOneBitSet(MinVecNumElts, 0);
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   OldCost +=
       TTI.getScalarizationOverhead(MinVecTy, DemandedElts,
                                    /* Insert */ true, HasExtract, CostKind);
 
   // New pattern: load VecPtr
   InstructionCost NewCost =
-      TTI.getMemoryOpCost(Instruction::Load, MinVecTy, Alignment, AS);
+      TTI.getMemoryOpCost(Instruction::Load, MinVecTy, Alignment, AS, CostKind);
   // Optionally, we are shuffling the loaded vector element(s) into place.
   // For the mask set everything but element 0 to undef to prevent poison from
   // propagating from the extra loaded memory. This will also optionally
@@ -276,7 +277,8 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
   Mask[0] = OffsetEltIndex;
   if (OffsetEltIndex)
-    NewCost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, MinVecTy, Mask);
+    NewCost +=
+        TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, MinVecTy, Mask, CostKind);
 
   // We can aggressively convert to the vector form because the backend can
   // invert this transform if it does not result in a performance win.
@@ -335,11 +337,11 @@ bool VectorCombine::widenSubvectorLoad(Instruction &I) {
   // undef value is 0. We could add that cost if the cost model accurately
   // reflects the real cost of that operation.
   InstructionCost OldCost =
-      TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment, AS);
+      TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment, AS, CostKind);
 
   // New pattern: load PtrOp
   InstructionCost NewCost =
-      TTI.getMemoryOpCost(Instruction::Load, Ty, Alignment, AS);
+      TTI.getMemoryOpCost(Instruction::Load, Ty, Alignment, AS, CostKind);
 
   // We can aggressively convert to the vector form because the backend can
   // invert this transform if it does not result in a performance win.
@@ -372,7 +374,6 @@ ExtractElementInst *VectorCombine::getShuffleExtract(
     return nullptr;
 
   Type *VecTy = Ext0->getVectorOperand()->getType();
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
   InstructionCost Cost0 =
       TTI.getVectorInstrCost(*Ext0, VecTy, CostKind, Index0);
@@ -426,23 +427,22 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
   // Get cost estimates for scalar and vector versions of the operation.
   bool IsBinOp = Instruction::isBinaryOp(Opcode);
   if (IsBinOp) {
-    ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, ScalarTy);
-    VectorOpCost = TTI.getArithmeticInstrCost(Opcode, VecTy);
+    ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, ScalarTy, CostKind);
+    VectorOpCost = TTI.getArithmeticInstrCost(Opcode, VecTy, CostKind);
   } else {
     assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
            "Expected a compare");
     CmpInst::Predicate Pred = cast<CmpInst>(I).getPredicate();
     ScalarOpCost = TTI.getCmpSelInstrCost(
-        Opcode, ScalarTy, CmpInst::makeCmpResultType(ScalarTy), Pred);
+        Opcode, ScalarTy, CmpInst::makeCmpResultType(ScalarTy), Pred, CostKind);
     VectorOpCost = TTI.getCmpSelInstrCost(
-        Opcode, VecTy, CmpInst::makeCmpResultType(VecTy), Pred);
+        Opcode, VecTy, CmpInst::makeCmpResultType(VecTy), Pred, CostKind);
   }
 
   // Get cost estimates for the extract elements. These costs will factor into
   // both sequences.
   unsigned Ext0Index = Ext0IndexC->getZExtValue();
   unsigned Ext1Index = Ext1IndexC->getZExtValue();
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
 
   InstructionCost Extract0Cost =
       TTI.getVectorInstrCost(*Ext0, VecTy, CostKind, Ext0Index);
@@ -689,9 +689,8 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) {
   Mask[Index] = Index + NumElts;
 
   Type *ScalarTy = VecTy->getScalarType();
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost OldCost =
-      TTI.getArithmeticInstrCost(Instruction::FNeg, ScalarTy) +
+      TTI.getArithmeticInstrCost(Instruction::FNeg, ScalarTy, CostKind) +
       TTI.getVectorInstrCost(I, VecTy, CostKind, Index);
 
   // If the extract has one use, it will be eliminated, so count it in the
@@ -701,8 +700,8 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) {
     OldCost += TTI.getVectorInstrCost(*Extract, VecTy, CostKind, Index);
 
   InstructionCost NewCost =
-      TTI.getArithmeticInstrCost(Instruction::FNeg, VecTy) +
-      TTI.getShuffleCost(TargetTransformInfo::SK_Select, VecTy, Mask);
+      TTI.getArithmeticInstrCost(Instruction::FNeg, VecTy, CostKind) +
+      TTI.getShuffleCost(TargetTransformInfo::SK_Select, VecTy, Mask, CostKind);
 
   if (NewCost > OldCost)
     return false;
@@ -778,21 +777,20 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) {
   unsigned NumOps = IsUnary ? 1 : 2;
 
   // The new shuffle must not cost more than the old shuffle.
-  TargetTransformInfo::TargetCostKind CK =
-      TargetTransformInfo::TCK_RecipThroughput;
   TargetTransformInfo::ShuffleKind SK =
       IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
               : TargetTransformInfo::SK_PermuteTwoSrc;
 
   InstructionCost DestCost =
-      TTI.getShuffleCost(SK, NewShuffleTy, NewMask, CK) +
+      TTI.getShuffleCost(SK, NewShuffleTy, NewMask, CostKind) +
       (NumOps * TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy,
                                      TargetTransformInfo::CastContextHint::None,
-                                     CK));
+                                     CostKind));
   InstructionCost SrcCost =
-      TTI.getShuffleCost(SK, SrcTy, Mask, CK) +
+      TTI.getShuffleCost(SK, SrcTy, Mask, CostKind) +
       TTI.getCastInstrCost(Instruction::BitCast, DestTy, OldShuffleTy,
-                           TargetTransformInfo::CastContextHint::None, CK);
+                           TargetTransformInfo::CastContextHint::None,
+                           CostKind);
   if (DestCost > SrcCost || !DestCost.isValid())
     return false;
 
@@ -847,13 +845,13 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
   // Calculate cost of splatting both operands into vectors and the vector
   // intrinsic
   VectorType *VecTy = cast<VectorType>(VPI.getType());
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   SmallVector<int> Mask;
   if (auto *FVTy = dyn_cast<FixedVectorType>(VecTy))
     Mask.resize(FVTy->getNumElements(), 0);
   InstructionCost SplatCost =
       TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0) +
-      TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, Mask);
+      TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, Mask,
+                         CostKind);
 
   // Calculate the cost of the VP Intrinsic
   SmallVector<Type *, 4> Args;
@@ -879,8 +877,8 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
     IntrinsicCostAttributes Attrs(*ScalarIntrID, VecTy->getScalarType(), Args);
     ScalarOpCost = TTI.getIntrinsicInstrCost(Attrs, CostKind);
   } else {
-    ScalarOpCost =
-        TTI.getArithmeticInstrCost(*FunctionalOpcode, VecTy->getScalarType());
+    ScalarOpCost = TTI.getArithmeticInstrCost(*FunctionalOpcode,
+                                              VecTy->getScalarType(), CostKind);
   }
 
   // The existing splats may be kept around if other instructions use them.
@@ -999,17 +997,16 @@ bool VectorCombine::scalarizeBinopOrCmp(Instruction &I) {
   if (IsCmp) {
     CmpInst::Predicate Pred = cast<CmpInst>(I).getPredicate();
     ScalarOpCost = TTI.getCmpSelInstrCost(
-        Opcode, ScalarTy, CmpInst::makeCmpResultType(ScalarTy), Pred);
+        Opcode, ScalarTy, CmpInst::makeCmpResultType(ScalarTy), Pred, CostKind);
     VectorOpCost = TTI.getCmpSelInstrCost(
-        Opcode, VecTy, CmpInst::makeCmpResultType(VecTy), Pred);
+        Opcode, VecTy, CmpInst::makeCmpResultType(VecTy), Pred, CostKind);
   } else {
-    ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, ScalarTy);
-    VectorOpCost = TTI.getArithmeticInstrCost(Opcode, VecTy);
+    ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, ScalarTy, CostKind);
+    VectorOpCost = TTI.getArithmeticInstrCost(Opcode, VecTy, CostKind);
   }
 
   // Get cost estimate for the insert element. This cost will factor into
   // both sequences.
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost InsertCost = TTI.getVectorInstrCost(
       Instruction::InsertElement, VecTy, CostKind, Index);
   InstructionCost OldCost =
@@ -1086,7 +1083,7 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
 
   auto *Ext0 = cast<ExtractElementInst>(I0);
   auto *Ext1 = cast<ExtractElementInst>(I1);
-  ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1);
+  ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1, CostKind);
   if (!ConvertToShuf)
     return false;
   assert((ConvertToShuf == Ext0 || ConvertToShuf == Ext1) &&
@@ -1095,23 +1092,23 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
   // The original scalar pattern is:
   // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
   CmpInst::Predicate Pred = P0;
-  unsigned CmpOpcode = CmpInst::isFPPredicate(Pred) ? Instruction::FCmp
-                                                    : Instruction::ICmp;
+  unsigned CmpOpcode =
+      CmpInst::isFPPredicate(Pred) ? Instruction::FCmp : Instruction::ICmp;
   auto *VecTy = dyn_cast<FixedVectorType>(X->getType());
   if (!VecTy)
     return false;
 
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost Ext0Cost =
-                      TTI.getVectorInstrCost(*Ext0, VecTy, CostKind, Index0),
-                  Ext1Cost =
-                      TTI.getVectorInstrCost(*Ext1, VecTy, CostKind, Index1);
+      TTI.getVectorInstrCost(*Ext0, VecTy, CostKind, Index0);
+  InstructionCost Ext1Cost =
+      TTI.getVectorInstrCost(*Ext1, VecTy, CostKind, Index1);
+  InstructionCost CmpCost = TTI.getCmpSelInstrCost(
+      CmpOpcode, I0->getType(), CmpInst::makeCmpResultType(I0->getType()), Pred,
+      CostKind);
+
   InstructionCost OldCost =
-      Ext0Cost + Ext1Cost +
-      TTI.getCmpSelInstrCost(CmpOpcode, I0->getType(),
-                             CmpInst::makeCmpResultType(I0->getType()), Pred) *
-          2 +
-      TTI.getArithmeticInstrCost(I.getOpcode(), I.getType());
+      Ext0Cost + Ext1Cost + CmpCost * 2 +
+      TTI.getArithmeticInstrCost(I.getOpcode(), I.getType(), CostKind);
 
   // The proposed vector pattern is:
   // vcmp = cmp Pred X, VecC
@@ -1120,12 +1117,13 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
   int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
   auto *CmpTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(X->getType()));
   InstructionCost NewCost = TTI.getCmpSelInstrCost(
-      CmpOpcode, X->getType(), CmpInst::makeCmpResultType(X->getType()), Pred);
+      CmpOpcode, X->getType(), CmpInst::makeCmpResultType(X->getType()), Pred,
+      CostKind);
   SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
   ShufMask[CheapIndex] = ExpensiveIndex;
   NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy,
-                                ShufMask);
-  NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy);
+                                ShufMask, CostKind);
+  NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy, CostKind);
   NewCost += TTI.getVectorInstrCost(*Ext0, CmpTy, CostKind, CheapIndex);
   NewCost += Ext0->hasOneUse() ? 0 : Ext0Cost;
   NewCost += Ext1->hasOneUse() ? 0 : Ext1Cost;
@@ -1349,7 +1347,7 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
 
   InstructionCost OriginalCost =
       TTI.getMemoryOpCost(Instruction::Load, VecTy, LI->getAlign(),
-                          LI->getPointerAddressSpace());
+                          LI->getPointerAddressSpace(), CostKind);
   InstructionCost ScalarizedCost = 0;
 
   Instruction *LastCheckedInst = LI;
@@ -1392,13 +1390,12 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
     }
 
     auto *Index = dyn_cast<ConstantInt>(UI->getOperand(1));
-    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
     OriginalCost +=
         TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind,
                                Index ? Index->getZExtValue() : -1);
     ScalarizedCost +=
         TTI.getMemoryOpCost(Instruction::Load, VecTy->getElementType(),
-                            Align(1), LI->getPointerAddressSpace());
+                            Align(1), LI->getPointerAddressSpace(), CostKind);
     ScalarizedCost += TTI.getAddressComputationCost(VecTy->getElementType());
   }
 
@@ -1486,8 +1483,6 @@ bool VectorCombine::foldPermuteOfBinops(Instruction &I) {
   }
 
   // Try to merge shuffles across the binop if the new shuffles are not costly.
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-
   InstructionCost OldCost =
       TTI.getArithmeticInstrCost(Opcode, BinOpTy, CostKind) +
       TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, BinOpTy,
@@ -1581,8 +1576,6 @@ bool VectorCombine::foldShuffleOfBinops(Instruction &I) {
   }
 
   // Try to replace a binop with a shuffle if the shuffle is not costly.
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-
   InstructionCost OldCost =
       TTI.getArithmeticInstrCost(B0->getOpcode(), BinOpTy, CostKind) +
       TTI.getArithmeticInstrCost(B1->getOpcode(), BinOpTy, CostKind) +
@@ -1678,8 +1671,6 @@ bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
       FixedVectorType::get(CastSrcTy->getScalarType(), NewMask.size());
 
   // Try to replace a castop with a shuffle if the shuffle is not costly.
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-
   InstructionCost CostC0 =
       TTI.getCastInstrCost(C0->getOpcode(), CastDstTy, CastSrcTy,
                            TTI::CastContextHint::None, CostKind);
@@ -1721,21 +1712,30 @@ bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
   return true;
 }
 
-/// Try to convert "shuffle (shuffle x, undef), (shuffle y, undef)"
+/// Try to convert any of:
+/// "shuffle (shuffle x, undef), (shuffle y, undef)"
+/// "shuffle (shuffle x, undef), y"
+/// "shuffle x, (shuffle y, undef)"
 /// into "shuffle x, y".
 bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
-  Value *V0, *V1;
-  UndefValue *U0, *U1;
-  ArrayRef<int> OuterMask, InnerMask0, InnerMask1;
+  ArrayRef<int> OuterMask;
+  Value *OuterV0, *OuterV1;
   if (!match(&I,
-             m_Shuffle(
-                 m_Shuffle(m_Value(V0), m_UndefValue(U0), m_Mask(InnerMask0)),
-                 m_Shuffle(m_Value(V1), m_UndefValue(U1), m_Mask(InnerMask1)),
-                 m_Mask(OuterMask))))
+             m_Shuffle(m_Value(OuterV0), m_Value(OuterV1), m_Mask(OuterMask))))
     return false;
 
-  auto *ShufI0 = dyn_cast<Instruction>(I.getOperand(0));
-  auto *ShufI1 = dyn_cast<Instruction>(I.getOperand(1));
+  ArrayRef<int> InnerMask0, InnerMask1;
+  Value *V0 = nullptr, *V1 = nullptr;
+  UndefValue *U0 = nullptr, *U1 = nullptr;
+  bool Match0 = match(
+      OuterV0, m_Shuffle(m_Value(V0), m_UndefValue(U0), m_Mask(InnerMask0)));
+  bool Match1 = match(
+      OuterV1, m_Shuffle(m_Value(V1), m_UndefValue(U1), m_Mask(InnerMask1)));
+  if (!Match0 && !Match1)
+    return false;
+
+  V0 = Match0 ? V0 : OuterV0;
+  V1 = Match1 ? V1 : OuterV1;
   auto *ShuffleDstTy = dyn_cast<FixedVectorType>(I.getType());
   auto *ShuffleSrcTy = dyn_cast<FixedVectorType>(V0->getType());
   auto *ShuffleImmTy = dyn_cast<FixedVectorType>(I.getOperand(0)->getType());
@@ -1747,9 +1747,9 @@ bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
   unsigned NumImmElts = ShuffleImmTy->getNumElements();
 
   // Bail if either inner masks reference a RHS undef arg.
-  if ((!isa<PoisonValue>(U0) &&
+  if ((Match0 && !isa<PoisonValue>(U0) &&
        any_of(InnerMask0, [&](int M) { return M >= (int)NumSrcElts; })) ||
-      (!isa<PoisonValue>(U1) &&
+      (Match1 && !isa<PoisonValue>(U1) &&
        any_of(InnerMask1, [&](int M) { return M >= (int)NumSrcElts; })))
     return false;
 
@@ -1757,12 +1757,15 @@ bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
   SmallVector<int, 16> NewMask(OuterMask);
   for (int &M : NewMask) {
     if (0 <= M && M < (int)NumImmElts) {
-      M = (InnerMask0[M] >= (int)NumSrcElts) ? PoisonMaskElem : InnerMask0[M];
+      if (Match0)
+        M = (InnerMask0[M] >= (int)NumSrcElts) ? PoisonMaskElem : InnerMask0[M];
     } else if (M >= (int)NumImmElts) {
-      if (InnerMask1[M - NumImmElts] >= (int)NumSrcElts)
-        M = PoisonMaskElem;
-      else
-        M = InnerMask1[M - NumImmElts] + (V0 == V1 ? 0 : NumSrcElts);
+      if (Match1) {
+        if (InnerMask1[M - NumImmElts] >= (int)NumSrcElts)
+          M = PoisonMaskElem;
+        else
+          M = InnerMask1[M - NumImmElts] + (V0 == V1 ? 0 : NumSrcElts);
+      }
     }
   }
 
@@ -1773,25 +1776,30 @@ bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
   }
 
   // Try to merge the shuffles if the new shuffle is not costly.
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-
-  InstructionCost InnerCost0 =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, ShuffleSrcTy,
-                         InnerMask0, CostKind, 0, nullptr, {V0, U0}, ShufI0);
-  InstructionCost InnerCost1 =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, ShuffleSrcTy,
-                         InnerMask1, CostKind, 0, nullptr, {V1, U1}, ShufI1);
-  InstructionCost OuterCost =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, ShuffleImmTy,
-                         OuterMask, CostKind, 0, nullptr, {ShufI0, ShufI1}, &I);
+  InstructionCost InnerCost0 = 0;
+  if (Match0)
+    InnerCost0 = TTI.getShuffleCost(
+        TargetTransformInfo::SK_PermuteSingleSrc, ShuffleSrcTy, InnerMask0,
+        CostKind, 0, nullptr, {V0, U0}, cast<ShuffleVectorInst>(OuterV0));
+
+  InstructionCost InnerCost1 = 0;
+  if (Match1)
+    InnerCost1 = TTI.getShuffleCost(
+        TargetTransformInfo::SK_PermuteSingleSrc, ShuffleSrcTy, InnerMask1,
+        CostKind, 0, nullptr, {V1, U1}, cast<ShuffleVectorInst>(OuterV1));
+
+  InstructionCost OuterCost = TTI.getShuffleCost(
+      TargetTransformInfo::SK_PermuteTwoSrc, ShuffleImmTy, OuterMask, CostKind,
+      0, nullptr, {OuterV0, OuterV1}, &I);
+
   InstructionCost OldCost = InnerCost0 + InnerCost1 + OuterCost;
 
   InstructionCost NewCost =
       TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, ShuffleSrcTy,
                          NewMask, CostKind, 0, nullptr, {V0, V1});
-  if (!ShufI0->hasOneUse())
+  if (!OuterV0->hasOneUse())
     NewCost += InnerCost0;
-  if (!ShufI1->hasOneUse())
+  if (!OuterV1->hasOneUse())
     NewCost += InnerCost1;
 
   LLVM_DEBUG(dbgs() << "Found a shuffle feeding two shuffles: " << I
@@ -1843,12 +1851,10 @@ bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
       return false;
 
   InstructionCost OldCost =
-      TTI.getIntrinsicInstrCost(IntrinsicCostAttributes(IID, *II0),
-                                TTI::TCK_RecipThroughput) +
-      TTI.getIntrinsicInstrCost(IntrinsicCostAttributes(IID, *II1),
-                                TTI::TCK_RecipThroughput) +
+      TTI.getIntrinsicInstrCost(IntrinsicCostAttributes(IID, *II0), CostKind) +
+      TTI.getIntrinsicInstrCost(IntrinsicCostAttributes(IID, *II1), CostKind) +
       TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, II0Ty, OldMask,
-                         TTI::TCK_RecipThroughput, 0, nullptr, {II0, II1}, &I);
+                         CostKind, 0, nullptr, {II0, II1}, &I);
 
   SmallVector<Type *> NewArgsTy;
   InstructionCost NewCost = 0;
@@ -1860,10 +1866,10 @@ bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
       NewArgsTy.push_back(FixedVectorType::get(VecTy->getElementType(),
                                                VecTy->getNumElements() * 2));
       NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
-                                    VecTy, OldMask, TTI::TCK_RecipThroughput);
+                                    VecTy, OldMask, CostKind);
     }
   IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
-  NewCost += TTI.getIntrinsicInstrCost(NewAttr, TTI::TCK_RecipThroughput);
+  NewCost += TTI.getIntrinsicInstrCost(NewAttr, CostKind);
 
   LLVM_DEBUG(dbgs() << "Found a shuffle feeding two intrinsics: " << I
                     << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
@@ -1929,7 +1935,7 @@ generateInstLaneVectorFromOperand(ArrayRef<InstLane> Item, int Op) {
 }
 
 /// Detect concat of multiple values into a vector
-static bool isFreeConcat(ArrayRef<InstLane> Item,
+static bool isFreeConcat(ArrayRef<InstLane> Item, TTI::TargetCostKind CostKind,
                          const TargetTransformInfo &TTI) {
   auto *Ty = cast<FixedVectorType>(Item.front().first->get()->getType());
   unsigned NumElts = Ty->getNumElements();
@@ -1940,8 +1946,7 @@ static bool isFreeConcat(ArrayRef<InstLane> Item,
   // during legalization.
   SmallVector<int, 16> ConcatMask(NumElts * 2);
   std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
-  if (TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, Ty, ConcatMask,
-                         TTI::TCK_RecipThroughput) != 0)
+  if (TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, Ty, ConcatMask, CostKind) != 0)
     return false;
 
   unsigned NumSlices = Item.size() / NumElts;
@@ -2195,7 +2200,7 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
       }
     }
 
-    if (isFreeConcat(Item, TTI)) {
+    if (isFreeConcat(Item, CostKind, TTI)) {
       ConcatLeafs.insert(FrontU);
       continue;
     }
@@ -2312,10 +2317,10 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
       (UsesSecondVec && !IsTruncatingShuffle) ? VecType : ShuffleInputType;
   InstructionCost OldCost = TTI.getShuffleCost(
       UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc,
-      VecTyForCost, Shuffle->getShuffleMask());
+      VecTyForCost, Shuffle->getShuffleMask(), CostKind);
   InstructionCost NewCost = TTI.getShuffleCost(
       UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc,
-      VecTyForCost, ConcatMask);
+      VecTyForCost, ConcatMask, CostKind);
 
   LLVM_DEBUG(dbgs() << "Found a reduction feeding from a shuffle: " << *Shuffle
                     << "\n");
@@ -2373,7 +2378,6 @@ bool VectorCombine::foldCastFromReductions(Instruction &I) {
   auto *ReductionSrcTy = cast<VectorType>(ReductionSrc->getType());
   Type *ResultTy = I.getType();
 
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost OldCost = TTI.getArithmeticReductionCost(
       ReductionOpc, ReductionSrcTy, std::nullopt, CostKind);
   OldCost += TTI.getCastInstrCost(CastOpc, ReductionSrcTy, SrcTy,
@@ -2630,17 +2634,17 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
     return C + TTI.getShuffleCost(isa<UndefValue>(SV->getOperand(1))
                                       ? TTI::SK_PermuteSingleSrc
                                       : TTI::SK_PermuteTwoSrc,
-                                  VT, SV->getShuffleMask());
+                                  VT, SV->getShuffleMask(), CostKind);
   };
   auto AddShuffleMaskCost = [&](InstructionCost C, ArrayRef<int> Mask) {
-    return C + TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, VT, Mask);
+    return C + TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, VT, Mask, CostKind);
   };
 
   // Get the costs of the shuffles + binops before and after with the new
   // shuffle masks.
   InstructionCost CostBefore =
-      TTI.getArithmeticInstrCost(Op0->getOpcode(), VT) +
-      TTI.getArithmeticInstrCost(Op1->getOpcode(), VT);
+      TTI.getArithmeticInstrCost(Op0->getOpcode(), VT, CostKind) +
+      TTI.getArithmeticInstrCost(Op1->getOpcode(), VT, CostKind);
   CostBefore += std::accumulate(Shuffles.begin(), Shuffles.end(),
                                 InstructionCost(0), AddShuffleCost);
   CostBefore += std::accumulate(InputShuffles.begin(), InputShuffles.end(),
@@ -2653,8 +2657,8 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
   FixedVectorType *Op1SmallVT =
       FixedVectorType::get(VT->getScalarType(), V2.size());
   InstructionCost CostAfter =
-      TTI.getArithmeticInstrCost(Op0->getOpcode(), Op0SmallVT) +
-      TTI.getArithmeticInstrCost(Op1->getOpcode(), Op1SmallVT);
+      TTI.getArithmeticInstrCost(Op0->getOpcode(), Op0SmallVT, CostKind) +
+      TTI.getArithmeticInstrCost(Op1->getOpcode(), Op1SmallVT, CostKind);
   CostAfter += std::accumulate(ReconstructMasks.begin(), ReconstructMasks.end(),
                                InstructionCost(0), AddShuffleMaskCost);
   std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
@@ -2723,7 +2727,7 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
 ///     lshr((zext(x),y) -> zext(lshr(x,trunc(y)))
 /// Cost model calculations takes into account if zext(x) has other users and
 /// whether it can be propagated through them too.
-bool VectorCombine::shrinkType(llvm::Instruction &I) {
+bool VectorCombine::shrinkType(Instruction &I) {
   Value *ZExted, *OtherOperand;
   if (!match(&I, m_c_BitwiseLogic(m_ZExt(m_Value(ZExted)),
                                   m_Value(OtherOperand))) &&
@@ -2752,7 +2756,6 @@ bool VectorCombine::shrinkType(llvm::Instruction &I) {
 
   // Calculate costs of leaving current IR as it is and moving ZExt operation
   // later, along with adding truncates if needed
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost ZExtCost = TTI.getCastInstrCost(
       Instruction::ZExt, BigTy, SmallTy,
       TargetTransformInfo::CastContextHint::None, CostKind);
@@ -2839,13 +2842,12 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
   auto *Ins = cast<InsertElementInst>(&I);
   auto *Ext = cast<ExtractElementInst>(I.getOperand(1));
 
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost OldCost =
       TTI.getVectorInstrCost(*Ext, VecTy, CostKind, ExtIdx) +
       TTI.getVectorInstrCost(*Ins, VecTy, CostKind, InsIdx);
 
-  InstructionCost NewCost =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, VecTy, Mask);
+  InstructionCost NewCost = TTI.getShuffleCost(
+      TargetTransformInfo::SK_PermuteTwoSrc, VecTy, Mask, CostKind);
   if (!Ext->hasOneUse())
     NewCost += TTI.getVectorInstrCost(*Ext, VecTy, CostKind, ExtIdx);
 
@@ -2987,7 +2989,8 @@ PreservedAnalyses VectorCombinePass::run(Function &F,
   DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
   AAResults &AA = FAM.getResult<AAManager>(F);
   const DataLayout *DL = &F.getDataLayout();
-  VectorCombine Combiner(F, TTI, DT, AA, AC, DL, TryEarlyFoldsOnly);
+  VectorCombine Combiner(F, TTI, DT, AA, AC, DL, TTI::TCK_RecipThroughput,
+                         TryEarlyFoldsOnly);
   if (!Combiner.run())
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
diff --git a/llvm/lib/XRay/InstrumentationMap.cpp b/llvm/lib/XRay/InstrumentationMap.cpp
index 800f0a0f47e42..0ebdcd5bac752 100644
--- a/llvm/lib/XRay/InstrumentationMap.cpp
+++ b/llvm/lib/XRay/InstrumentationMap.cpp
@@ -63,7 +63,8 @@ loadObj(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile,
         ObjFile.getBinary()->getArch() == Triple::loongarch64 ||
         ObjFile.getBinary()->getArch() == Triple::ppc64le ||
         ObjFile.getBinary()->getArch() == Triple::arm ||
-        ObjFile.getBinary()->getArch() == Triple::aarch64))
+        ObjFile.getBinary()->getArch() == Triple::aarch64 ||
+        ObjFile.getBinary()->getArch() == Triple::riscv64))
     return make_error<StringError>(
         "File format not supported (only does ELF and Mach-O little endian "
         "64-bit).",
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 57a56c6a60415..70e85c123e412 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -277,6 +277,7 @@ function(runtime_default_target)
                            PASSTHROUGH_PREFIXES LLVM_ENABLE_RUNTIMES
                                                 LLVM_USE_LINKER
                                                 CUDA # For runtimes that may look for the CUDA SDK (libc, offload)
+                                                FFI # offload uses libffi
                                                 ${ARG_PREFIXES}
                            EXTRA_TARGETS ${extra_targets}
                                          ${test_targets}
@@ -520,24 +521,6 @@ if(build_runtimes)
       endif()
     endforeach()
   endif()
-  if("libc" IN_LIST LLVM_ENABLE_PROJECTS AND
-      (LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD))
-    if(LIBC_HDRGEN_EXE)
-      set(hdrgen_exe ${LIBC_HDRGEN_EXE})
-    else()
-      if(TARGET ${LIBC_TABLEGEN_EXE})
-        set(hdrgen_exe $<TARGET_FILE:${LIBC_TABLEGEN_EXE}>)
-      else()
-        set(hdrgen_exe ${LIBC_TABLEGEN_EXE})
-      endif()
-      set(hdrgen_deps ${LIBC_TABLEGEN_TARGET})
-    endif()
-    if(NOT hdrgen_exe)
-      message(FATAL_ERROR "libc-hdrgen executable missing")
-    endif()
-    list(APPEND extra_cmake_args "-DLIBC_HDRGEN_EXE=${hdrgen_exe}")
-    list(APPEND extra_deps ${hdrgen_deps})
-  endif()
   if(LLVM_LIBC_GPU_BUILD)
     list(APPEND extra_cmake_args "-DLLVM_LIBC_GPU_BUILD=ON")
     if("libc" IN_LIST RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES)
diff --git a/llvm/test/Analysis/BasicAA/featuretest.ll b/llvm/test/Analysis/BasicAA/featuretest.ll
index 5ef94cd8c7fda..fd5d2eff10f83 100644
--- a/llvm/test/Analysis/BasicAA/featuretest.ll
+++ b/llvm/test/Analysis/BasicAA/featuretest.ll
@@ -59,7 +59,7 @@ define i32 @constant_array_index_test() {
 ; CHECK-LABEL: @constant_array_index_test(
 ; CHECK-NEXT:    [[ARRAY1:%.*]] = alloca [100 x i32], align 4
 ; CHECK-NEXT:    call void @external(ptr nonnull [[ARRAY1]])
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[ARRAY1]], i64 24
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAY1]], i64 24
 ; CHECK-NEXT:    store i32 1, ptr [[P2]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Analysis/BasicAA/smaller-index-size-overflow.ll b/llvm/test/Analysis/BasicAA/smaller-index-size-overflow.ll
new file mode 100644
index 0000000000000..a913a4a9e4b1c
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/smaller-index-size-overflow.ll
@@ -0,0 +1,13 @@
+; RUN: opt -S -passes=aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+
+target datalayout = "p1:32:32"
+
+; FIXME: This is a miscompile.
+; CHECK: NoAlias:	i32 addrspace(1)* %gep1, i32 addrspace(1)* %gep2
+define void @test(ptr addrspace(1) %p) {
+  %gep1 = getelementptr i8, ptr addrspace(1) %p, i32 u0x7fffffff
+  %gep2 = getelementptr i8, ptr addrspace(1) %p, i32 u0x80000001
+  store i32 0, ptr addrspace(1) %gep1
+  load i32, ptr addrspace(1) %gep2
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/cast.ll b/llvm/test/Analysis/CostModel/AArch64/cast.ll
index fa778864ae978..c363b58d939e8 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cast.ll
@@ -1,9 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -force-streaming-compatible %s | FileCheck --check-prefixes=SVE,SVE128-NO-NEON %s
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=256 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-256 %s
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=2048 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-2048 %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -79,429 +76,6 @@ define void @ext() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SVE128-NO-NEON-LABEL: 'ext'
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MIN-256-LABEL: 'ext'
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MIN-2048-LABEL: 'ext'
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MAX-LABEL: 'ext'
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE256-LABEL: 'ext'
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE2048-LABEL: 'ext'
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
   %r0 = sext i1 undef to i8
   %r1 = zext i1 undef to i8
   %r2 = sext i1 undef to i16
@@ -615,225 +189,6 @@ define void @trunc() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SVE128-NO-NEON-LABEL: 'trunc'
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MIN-256-LABEL: 'trunc'
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MIN-2048-LABEL: 'trunc'
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MAX-LABEL: 'trunc'
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE256-LABEL: 'trunc'
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE2048-LABEL: 'trunc'
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
   %r8 = trunc i8 undef to i1
   %r15 = trunc i16 undef to i1
   %r16 = trunc i16 undef to i8
@@ -919,14 +274,34 @@ define i32 @casts_no_users() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncf64f16 = fptrunc double undef to half
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %truncv2f64f16 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %truncv4f64f16 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %truncv8f64f16 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %truncv16f64f16 = fptrunc <16 x double> undef to <16 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncv32f16 = fptrunc float undef to half
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncv2f32f16 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncv4f32f16 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %truncv8f32f16 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %truncv16f32f16 = fptrunc <16 x float> undef to <16 x half>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extf16f32 = fpext half undef to float
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extv2f16f32 = fpext <2 x half> undef to <2 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extv4f16f32 = fpext <4 x half> undef to <4 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %extv8f16f32 = fpext <8 x half> undef to <8 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %extv16f16f32 = fpext <16 x half> undef to <16 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extf16f64 = fpext half undef to double
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %extv2f16f64 = fpext <2 x half> undef to <2 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %extv4f16f64 = fpext <4 x half> undef to <4 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %extv8f16f64 = fpext <8 x half> undef to <8 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extv16f16f64 = fpext <16 x half> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -1089,1281 +464,6 @@ define i32 @casts_no_users() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SVE128-NO-NEON-LABEL: 'casts_no_users'
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MIN-256-LABEL: 'casts_no_users'
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MIN-2048-LABEL: 'casts_no_users'
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MAX-LABEL: 'casts_no_users'
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-; FIXED-OVER-SVE256-LABEL: 'casts_no_users'
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-; FIXED-OVER-SVE2048-LABEL: 'casts_no_users'
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
   %r30 = fptoui float undef to i1
   %r31 = fptosi float undef to i1
   %r32 = fptoui float undef to i8
@@ -2409,11 +509,31 @@ define i32 @casts_no_users() {
   %r82 = fptrunc <4 x double> undef to <4 x float>
   %r83 = fptrunc <8 x double> undef to <8 x float>
   %r84 = fptrunc <16 x double> undef to <16 x float>
+  %truncf64f16 = fptrunc double undef to half
+  %truncv2f64f16 = fptrunc <2 x double> undef to <2 x half>
+  %truncv4f64f16 = fptrunc <4 x double> undef to <4 x half>
+  %truncv8f64f16 = fptrunc <8 x double> undef to <8 x half>
+  %truncv16f64f16 = fptrunc <16 x double> undef to <16 x half>
+  %truncv32f16 = fptrunc float undef to half
+  %truncv2f32f16 = fptrunc <2 x float> undef to <2 x half>
+  %truncv4f32f16 = fptrunc <4 x float> undef to <4 x half>
+  %truncv8f32f16 = fptrunc <8 x float> undef to <8 x half>
+  %truncv16f32f16 = fptrunc <16 x float> undef to <16 x half>
   %r85 = fpext float undef to double
   %r86 = fpext <2 x float> undef to <2 x double>
   %r87 = fpext <4 x float> undef to <4 x double>
   %r88 = fpext <8 x float> undef to <8 x double>
   %r89 = fpext <16 x float> undef to <16 x double>
+  %extf16f32 = fpext half undef to float
+  %extv2f16f32 = fpext <2 x half> undef to <2 x float>
+  %extv4f16f32 = fpext <4 x half> undef to <4 x float>
+  %extv8f16f32 = fpext <8 x half> undef to <8 x float>
+  %extv16f16f32 = fpext <16 x half> undef to <16 x float>
+  %extf16f64 = fpext half undef to double
+  %extv2f16f64 = fpext <2 x half> undef to <2 x double>
+  %extv4f16f64 = fpext <4 x half> undef to <4 x double>
+  %extv8f16f64 = fpext <8 x half> undef to <8 x double>
+  %extv16f16f64 = fpext <16 x half> undef to <16 x double>
   %r90 = fptoui <2 x float> undef to <2 x i1>
   %r91 = fptosi <2 x float> undef to <2 x i1>
   %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -2650,64 +770,6 @@ define i32 @casts_with_users(i8 %a, i16 %b, i32 %c, i64 %d, i1 %e) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r24, ptr undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r25, ptr undef, align 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r12
-;
-; SVE-LABEL: 'casts_with_users'
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i8 %a to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = sext i8 %a to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = sext i8 %a to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = sext i16 %b to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i16 %b to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = sext i32 %c to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r0, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r1, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r2, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r3, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r4, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r5, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r6 = zext i8 %a to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r7 = zext i8 %a to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = zext i8 %a to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %b to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i16 %b to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %c to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r6, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r7, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r8, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r9, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r10, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r11, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = trunc i64 %d to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = trunc i64 %d to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = trunc i64 %d to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i32 %c to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i32 %c to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = trunc i16 %b to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.8 = add i8 %r14, %r16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.8.1 = add i8 %tie.trunc.8, %r17
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.16 = add i16 %r13, %r15
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.16.8 = trunc i16 %r6 to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.32.8 = trunc i32 %r7 to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.32.16 = trunc i32 %r9 to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.8 = trunc i64 %r8 to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.16 = trunc i64 %r10 to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.32 = trunc i64 %r11 to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r18 = sext i1 %e to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r19 = sext i1 %e to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r20 = sext i1 %e to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r21 = sext i1 %e to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r22 = zext i1 %e to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r23 = zext i1 %e to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r24 = zext i1 %e to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i1 %e to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r18, ptr undef, align 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r19, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r20, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r21, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r22, ptr undef, align 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r23, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r24, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r25, ptr undef, align 8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r12
 ;
   %r0 = sext i8 %a to i16
   %r1 = sext i8 %a to i32
@@ -2782,17 +844,6 @@ define i32 @bitcasts() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SVE-LABEL: 'bitcasts'
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %b = bitcast float undef to float
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = bitcast i32 undef to float
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = bitcast float undef to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = bitcast i64 undef to double
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %a = bitcast i32 undef to i32
   %b = bitcast float undef to float
@@ -2805,7 +856,7 @@ define i32 @bitcasts() {
   ret i32 undef
 }
 
-define i32 @load_extends() #0 {
+define i32 @load_extends() {
 ; CHECK-LABEL: 'load_extends'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
@@ -2817,8 +868,6 @@ define i32 @load_extends() #0 {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
@@ -2845,279 +894,8 @@ define i32 @load_extends() #0 {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SVE128-NO-NEON-LABEL: 'load_extends'
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MIN-256-LABEL: 'load_extends'
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MIN-2048-LABEL: 'load_extends'
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; FIXED-MAX-LABEL: 'load_extends'
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-; FIXED-OVER-SVE256-LABEL: 'load_extends'
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-; FIXED-OVER-SVE2048-LABEL: 'load_extends'
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
   %loadi8 = load i8, ptr undef
   %loadi16 = load i16, ptr undef
   %loadi32 = load i32, ptr undef
@@ -3128,8 +906,6 @@ define i32 @load_extends() #0 {
   %loadv4i16 = load <4 x i16>, ptr undef
   %loadv2i32 = load <2 x i32>, ptr undef
   %loadv4i32 = load <4 x i32>, ptr undef
-  %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef
-  %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef
 
   %r0 = sext i8 %loadi8 to i16
   %r1 = zext i8 %loadi8 to i16
@@ -3158,10 +934,6 @@ define i32 @load_extends() #0 {
   %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
   %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
   %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
-  %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-  %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
-  %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
-  %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
 
   ret i32 undef
 }
@@ -3181,21 +953,6 @@ define i32 @store_truncs() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = trunc i16 undef to i8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r5, ptr undef, align 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SVE-LABEL: 'store_truncs'
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = trunc i64 undef to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r0, ptr undef, align 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = trunc i64 undef to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r1, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = trunc i64 undef to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r2, ptr undef, align 4
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = trunc i32 undef to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r3, ptr undef, align 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = trunc i32 undef to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r4, ptr undef, align 2
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = trunc i16 undef to i8
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r5, ptr undef, align 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %r0 = trunc i64 undef to i8
   store i8 %r0, ptr undef
@@ -3231,25 +988,6 @@ define void @extend_extract() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z32_64 = zext i32 %e32 to i64
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @use(i16 %s8_16, i16 %z8_16, i32 %s8_32, i32 %z8_32, i64 %s8_64, i64 %z8_64, i32 %s16_32, i32 %z16_32, i64 %s16_64, i64 %z16_64, i64 %s32_64, i64 %z32_64)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SVE-LABEL: 'extend_extract'
-; SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = extractelement <8 x i8> undef, i32 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e16 = extractelement <8 x i16> undef, i32 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e32 = extractelement <8 x i32> undef, i32 1
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_16 = sext i8 %e8 to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8_16 = zext i8 %e8 to i16
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_32 = sext i8 %e8 to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8_32 = zext i8 %e8 to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_64 = sext i8 %e8 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z8_64 = zext i8 %e8 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16_32 = sext i16 %e16 to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16_32 = zext i16 %e16 to i32
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16_64 = sext i16 %e16 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z16_64 = zext i16 %e16 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s32_64 = sext i32 %e32 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z32_64 = zext i32 %e32 to i64
-; SVE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @use(i16 %s8_16, i16 %z8_16, i32 %s8_32, i32 %z8_32, i64 %s8_64, i64 %z8_64, i32 %s16_32, i32 %z16_32, i64 %s16_64, i64 %z16_64, i64 %s32_64, i64 %z32_64)
-; SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %e8 = extractelement <8 x i8> undef, i32 1
   %e16 = extractelement <8 x i16> undef, i32 1
@@ -3346,79 +1084,6 @@ define void @fp16cast() {
 ; CHECK-NOFP16-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
 ; CHECK-NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SVE128-NO-NEON-LABEL: 'fp16cast'
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
 ; CHECK-FP16-LABEL: 'fp16cast'
 ; CHECK-FP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
 ; CHECK-FP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
@@ -3492,368 +1157,6 @@ define void @fp16cast() {
 ; CHECK-FP16-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
 ; CHECK-FP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; FIXED-MIN-256-LABEL: 'fp16cast'
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MIN-2048-LABEL: 'fp16cast'
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; FIXED-MAX-LABEL: 'fp16cast'
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; FIXED-MAX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE256-LABEL: 'fp16cast'
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; FIXED-OVER-SVE256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-; FIXED-OVER-SVE2048-LABEL: 'fp16cast'
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
-; FIXED-OVER-SVE2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
   %r30 = fptoui half undef to i1
   %r31 = fptosi half undef to i1
   %r32 = fptoui half undef to i8
@@ -3933,4 +1236,49 @@ define void @fp16cast() {
   ret void
 }
 
-attributes #0 = { "target-features"="+sve" }
+define void @bf16cast() {
+; CHECK-LABEL: 'bf16cast'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extf16f32 = fpext bfloat undef to float
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extv2f16f32 = fpext <2 x bfloat> undef to <2 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extv4f16f32 = fpext <4 x bfloat> undef to <4 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %extv8f16f32 = fpext <8 x bfloat> undef to <8 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %extv16f16f32 = fpext <16 x bfloat> undef to <16 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extf16f64 = fpext bfloat undef to double
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extv2f16f64 = fpext <2 x bfloat> undef to <2 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %extv4f16f64 = fpext <4 x bfloat> undef to <4 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %extv8f16f64 = fpext <8 x bfloat> undef to <8 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %extv16f16f64 = fpext <16 x bfloat> undef to <16 x double>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncf16f32 = fptrunc float undef to bfloat
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncv2f16f32 = fptrunc <2 x float> undef to <2 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %truncv4f16f32 = fptrunc <4 x float> undef to <4 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %truncv8f16f32 = fptrunc <8 x float> undef to <8 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %truncv16f16f32 = fptrunc <16 x float> undef to <16 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %truncf16f64 = fptrunc double undef to bfloat
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %truncv2f16f64 = fptrunc <2 x double> undef to <2 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %truncv4f16f64 = fptrunc <4 x double> undef to <4 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %truncv8f16f64 = fptrunc <8 x double> undef to <8 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %truncv16f16f64 = fptrunc <16 x double> undef to <16 x bfloat>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %extf16f32 = fpext bfloat undef to float
+  %extv2f16f32 = fpext <2 x bfloat> undef to <2 x float>
+  %extv4f16f32 = fpext <4 x bfloat> undef to <4 x float>
+  %extv8f16f32 = fpext <8 x bfloat> undef to <8 x float>
+  %extv16f16f32 = fpext <16 x bfloat> undef to <16 x float>
+  %extf16f64 = fpext bfloat undef to double
+  %extv2f16f64 = fpext <2 x bfloat> undef to <2 x double>
+  %extv4f16f64 = fpext <4 x bfloat> undef to <4 x double>
+  %extv8f16f64 = fpext <8 x bfloat> undef to <8 x double>
+  %extv16f16f64 = fpext <16 x bfloat> undef to <16 x double>
+  %truncf16f32 = fptrunc float undef to bfloat
+  %truncv2f16f32 = fptrunc <2 x float> undef to <2 x bfloat>
+  %truncv4f16f32 = fptrunc <4 x float> undef to <4 x bfloat>
+  %truncv8f16f32 = fptrunc <8 x float> undef to <8 x bfloat>
+  %truncv16f16f32 = fptrunc <16 x float> undef to <16 x bfloat>
+  %truncf16f64 = fptrunc double undef to bfloat
+  %truncv2f16f64 = fptrunc <2 x double> undef to <2 x bfloat>
+  %truncv4f16f64 = fptrunc <4 x double> undef to <4 x bfloat>
+  %truncv8f16f64 = fptrunc <8 x double> undef to <8 x bfloat>
+  %truncv16f16f64 = fptrunc <16 x double> undef to <16 x bfloat>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/extract_float_streaming.ll b/llvm/test/Analysis/CostModel/AArch64/extract_float_streaming.ll
new file mode 100644
index 0000000000000..84502abceed3b
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/extract_float_streaming.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-unknown-linux -mattr=+sme | FileCheck %s
+
+define double @extract_case7(<4 x double> %a) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: 'extract_case7'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %0 = extractelement <4 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <4 x double> %a, i32 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
+entry:
+  %1 = extractelement <4 x double> %a, i32 1
+  %2 = extractelement <4 x double> %a, i32 2
+  %res = fmul double %1, %2
+  ret double %res
+}
+
+declare void @foo(double)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
new file mode 100644
index 0000000000000..4b7b1ff7a8b47
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
@@ -0,0 +1,2455 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve %s | FileCheck --check-prefixes=CHECK,CHECK-SVE %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -force-streaming-compatible %s | FileCheck --check-prefixes=CHECK,SVE128-NO-NEON %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=256 %s | FileCheck --check-prefixes=CHECK,FIXED-MIN-256 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=2048 %s | FileCheck --check-prefixes=CHECK,FIXED-MIN-2048 %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define void @ext() {
+; CHECK-SVE-LABEL: 'ext'
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SVE128-NO-NEON-LABEL: 'ext'
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-256-LABEL: 'ext'
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-2048-LABEL: 'ext'
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i1 undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i1 undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i1 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i1 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i1 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i1 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i1 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i1 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = sext i8 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i8 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = sext i8 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = zext i8 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = sext i8 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = zext i8 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = sext i16 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r18 = zext i16 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r19 = sext i16 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r20 = zext i16 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r24 = sext i32 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i32 undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i16 = sext <2 x i8> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i16 = zext <2 x i8> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i32 = sext <2 x i8> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i32 = zext <2 x i8> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = sext <2 x i8> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i8i64 = zext <2 x i8> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = sext <2 x i16> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i32 = zext <2 x i16> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = sext <2 x i16> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i16i64 = zext <2 x i16> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = sext <2 x i32> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z2i32i64 = zext <2 x i32> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i16 = sext <4 x i8> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i16 = zext <4 x i8> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = sext <4 x i8> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i32 = zext <4 x i8> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i64 = sext <4 x i8> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i8i64 = zext <4 x i8> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = sext <4 x i16> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i32 = zext <4 x i16> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i64 = sext <4 x i16> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i16i64 = zext <4 x i16> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = sext <4 x i32> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z4i32i64 = zext <4 x i32> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = sext <8 x i8> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i16 = zext <8 x i8> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i32 = sext <8 x i8> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i32 = zext <8 x i8> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i64 = sext <8 x i8> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i8i64 = zext <8 x i8> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = sext <8 x i16> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i32 = zext <8 x i16> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i64 = sext <8 x i16> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i16i64 = zext <8 x i16> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = sext <8 x i32> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8i32i64 = zext <8 x i32> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = sext <16 x i8> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i16 = zext <16 x i8> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i32 = sext <16 x i8> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i32 = zext <16 x i8> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i64 = sext <16 x i8> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i8i64 = zext <16 x i8> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = sext <16 x i16> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i32 = zext <16 x i16> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i64 = sext <16 x i16> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i16i64 = zext <16 x i16> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = sext <16 x i32> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16i32i64 = zext <16 x i32> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %r0 = sext i1 undef to i8
+  %r1 = zext i1 undef to i8
+  %r2 = sext i1 undef to i16
+  %r3 = zext i1 undef to i16
+  %r4 = sext i1 undef to i32
+  %r5 = zext i1 undef to i32
+  %r6 = sext i1 undef to i64
+  %r7 = zext i1 undef to i64
+  %r9 = sext i8 undef to i16
+  %r10 = zext i8 undef to i16
+  %r11 = sext i8 undef to i32
+  %r12 = zext i8 undef to i32
+  %r13 = sext i8 undef to i64
+  %r14 = zext i8 undef to i64
+  %r17 = sext i16 undef to i32
+  %r18 = zext i16 undef to i32
+  %r19 = sext i16 undef to i64
+  %r20 = zext i16 undef to i64
+  %r24 = sext i32 undef to i64
+  %r25 = zext i32 undef to i64
+
+  %s2i8i16 = sext <2 x i8> undef to <2 x i16>
+  %z2i8i16 = zext <2 x i8> undef to <2 x i16>
+  %s2i8i32 = sext <2 x i8> undef to <2 x i32>
+  %z2i8i32 = zext <2 x i8> undef to <2 x i32>
+  %s2i8i64 = sext <2 x i8> undef to <2 x i64>
+  %z2i8i64 = zext <2 x i8> undef to <2 x i64>
+  %s2i16i32 = sext <2 x i16> undef to <2 x i32>
+  %z2i16i32 = zext <2 x i16> undef to <2 x i32>
+  %s2i16i64 = sext <2 x i16> undef to <2 x i64>
+  %z2i16i64 = zext <2 x i16> undef to <2 x i64>
+  %s2i32i64 = sext <2 x i32> undef to <2 x i64>
+  %z2i32i64 = zext <2 x i32> undef to <2 x i64>
+
+  %s4i8i16 = sext <4 x i8>  undef to <4 x i16>
+  %z4i8i16 = zext <4 x i8>  undef to <4 x i16>
+  %s4i8i32 = sext <4 x i8>  undef to <4 x i32>
+  %z4i8i32 = zext <4 x i8>  undef to <4 x i32>
+  %s4i8i64 = sext <4 x i8>  undef to <4 x i64>
+  %z4i8i64 = zext <4 x i8>  undef to <4 x i64>
+  %s4i16i32 = sext <4 x i16> undef to <4 x i32>
+  %z4i16i32 = zext <4 x i16> undef to <4 x i32>
+  %s4i16i64 = sext <4 x i16> undef to <4 x i64>
+  %z4i16i64 = zext <4 x i16> undef to <4 x i64>
+  %s4i32i64 = sext <4 x i32> undef to <4 x i64>
+  %z4i32i64 = zext <4 x i32> undef to <4 x i64>
+
+  %s8i8i16 = sext <8 x i8>  undef to <8 x i16>
+  %z8i8i16 = zext <8 x i8>  undef to <8 x i16>
+  %s8i8i32 = sext <8 x i8>  undef to <8 x i32>
+  %z8i8i32 = zext <8 x i8>  undef to <8 x i32>
+  %s8i8i64 = sext <8 x i8>  undef to <8 x i64>
+  %z8i8i64 = zext <8 x i8>  undef to <8 x i64>
+  %s8i16i32 = sext <8 x i16> undef to <8 x i32>
+  %z8i16i32 = zext <8 x i16> undef to <8 x i32>
+  %s8i16i64 = sext <8 x i16> undef to <8 x i64>
+  %z8i16i64 = zext <8 x i16> undef to <8 x i64>
+  %s8i32i64 = sext <8 x i32> undef to <8 x i64>
+  %z8i32i64 = zext <8 x i32> undef to <8 x i64>
+
+  %s16i8i16 = sext <16 x i8>  undef to <16 x i16>
+  %z16i8i16 = zext <16 x i8>  undef to <16 x i16>
+  %s16i8i32 = sext <16 x i8>  undef to <16 x i32>
+  %z16i8i32 = zext <16 x i8>  undef to <16 x i32>
+  %s16i8i64 = sext <16 x i8>  undef to <16 x i64>
+  %z16i8i64 = zext <16 x i8>  undef to <16 x i64>
+  %s16i16i32 = sext <16 x i16> undef to <16 x i32>
+  %z16i16i32 = zext <16 x i16> undef to <16 x i32>
+  %s16i16i64 = sext <16 x i16> undef to <16 x i64>
+  %z16i16i64 = zext <16 x i16> undef to <16 x i64>
+  %s16i32i64 = sext <16 x i32> undef to <16 x i64>
+  %z16i32i64 = zext <16 x i32> undef to <16 x i64>
+  ret void
+}
+
+define void @trunc() {
+; CHECK-SVE-LABEL: 'trunc'
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SVE128-NO-NEON-LABEL: 'trunc'
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-256-LABEL: 'trunc'
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-2048-LABEL: 'trunc'
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = trunc i8 undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i16 undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i16 undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r21 = trunc i32 undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r22 = trunc i32 undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r23 = trunc i32 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r26 = trunc i64 undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r27 = trunc i64 undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r28 = trunc i64 undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r29 = trunc i64 undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %r8 = trunc i8 undef to i1
+  %r15 = trunc i16 undef to i1
+  %r16 = trunc i16 undef to i8
+  %r21 = trunc i32 undef to i1
+  %r22 = trunc i32 undef to i8
+  %r23 = trunc i32 undef to i16
+  %r26 = trunc i64 undef to i1
+  %r27 = trunc i64 undef to i8
+  %r28 = trunc i64 undef to i16
+  %r29 = trunc i64 undef to i32
+
+  %s2i8i16 = trunc <2 x i16> undef to <2 x i8>
+  %s2i8i32 = trunc <2 x i32> undef to <2 x i8>
+  %s2i8i64 = trunc <2 x i64> undef to <2 x i8>
+  %s2i16i32 = trunc <2 x i32> undef to <2 x i16>
+  %s2i16i64 = trunc <2 x i64> undef to <2 x i16>
+  %s2i32i64 = trunc <2 x i64> undef to <2 x i32>
+
+  %s4i8i16 = trunc <4 x i16> undef to <4 x i8>
+  %s4i8i32 = trunc <4 x i32> undef to <4 x i8>
+  %s4i8i64 = trunc <4 x i64> undef to <4 x i8>
+  %s4i16i32 = trunc <4 x i32> undef to <4 x i16>
+  %s4i16i64 = trunc <4 x i64> undef to <4 x i16>
+  %s4i32i64 = trunc <4 x i64> undef to <4 x i32>
+
+  %s8i8i16 = trunc <8 x i16> undef to <8 x i8>
+  %s8i8i32 = trunc <8 x i32> undef to <8 x i8>
+  %s8i8i64 = trunc <8 x i64> undef to <8 x i8>
+  %s8i16i32 = trunc <8 x i32> undef to <8 x i16>
+  %s8i16i64 = trunc <8 x i64> undef to <8 x i16>
+  %s8i32i64 = trunc <8 x i64> undef to <8 x i32>
+
+  %s16i8i16 = trunc <16 x i16> undef to <16 x i8>
+  %s16i8i32 = trunc <16 x i32> undef to <16 x i8>
+  %s16i8i64 = trunc <16 x i64> undef to <16 x i8>
+  %s16i16i32 = trunc <16 x i32> undef to <16 x i16>
+  %s16i16i64 = trunc <16 x i64> undef to <16 x i16>
+  %s16i32i64 = trunc <16 x i64> undef to <16 x i32>
+  ret void
+}
+
+define i32 @casts_no_users() {
+; CHECK-SVE-LABEL: 'casts_no_users'
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SVE128-NO-NEON-LABEL: 'casts_no_users'
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; FIXED-MIN-256-LABEL: 'casts_no_users'
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; FIXED-MIN-2048-LABEL: 'casts_no_users'
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui float undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi float undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui float undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi float undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui float undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi float undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui float undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi float undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui float undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi float undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r40 = fptoui double undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r41 = fptosi double undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r42 = fptoui double undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r43 = fptosi double undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r44 = fptoui double undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r45 = fptosi double undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r46 = fptoui double undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r47 = fptosi double undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r48 = fptoui double undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r49 = fptosi double undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r50 = sitofp i1 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r51 = uitofp i1 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r52 = sitofp i1 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r53 = uitofp i1 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r54 = sitofp i8 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r55 = uitofp i8 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r56 = sitofp i8 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r57 = uitofp i8 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r58 = sitofp i16 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r59 = uitofp i16 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r60 = sitofp i16 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r61 = uitofp i16 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r62 = sitofp i32 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r63 = uitofp i32 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r64 = sitofp i32 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r65 = uitofp i32 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r66 = sitofp i64 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r67 = uitofp i64 undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r68 = sitofp i64 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r69 = uitofp i64 undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80 = fptrunc double undef to float
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81 = fptrunc <2 x double> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82 = fptrunc <4 x double> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83 = fptrunc <8 x double> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84 = fptrunc <16 x double> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85 = fpext float undef to double
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86 = fpext <2 x float> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87 = fpext <4 x float> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88 = fpext <8 x float> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89 = fpext <16 x float> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r113 = fptosi <4 x float> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r114 = fptoui <4 x float> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x float> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x float> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x float> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x float> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r173 = sitofp <2 x i8> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r174 = uitofp <2 x i16> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r175 = sitofp <2 x i16> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r176 = uitofp <2 x i32> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r177 = sitofp <2 x i32> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178 = uitofp <2 x i64> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179 = sitofp <2 x i64> undef to <2 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r193 = sitofp <4 x i8> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r194 = uitofp <4 x i16> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r195 = sitofp <4 x i16> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r196 = uitofp <4 x i32> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r197 = sitofp <4 x i32> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r198 = uitofp <4 x i64> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r199 = sitofp <4 x i64> undef to <4 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r213 = sitofp <8 x i8> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r214 = uitofp <8 x i16> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r215 = sitofp <8 x i16> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r216 = uitofp <8 x i32> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r217 = sitofp <8 x i32> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r218 = uitofp <8 x i64> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r219 = sitofp <8 x i64> undef to <8 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r233 = sitofp <16 x i8> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r234 = uitofp <16 x i16> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r235 = sitofp <16 x i16> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r236 = uitofp <16 x i32> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r237 = sitofp <16 x i32> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r238 = uitofp <16 x i64> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r239 = sitofp <16 x i64> undef to <16 x float>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %r30 = fptoui float undef to i1
+  %r31 = fptosi float undef to i1
+  %r32 = fptoui float undef to i8
+  %r33 = fptosi float undef to i8
+  %r34 = fptoui float undef to i16
+  %r35 = fptosi float undef to i16
+  %r36 = fptoui float undef to i32
+  %r37 = fptosi float undef to i32
+  %r38 = fptoui float undef to i64
+  %r39 = fptosi float undef to i64
+  %r40 = fptoui double undef to i1
+  %r41 = fptosi double undef to i1
+  %r42 = fptoui double undef to i8
+  %r43 = fptosi double undef to i8
+  %r44 = fptoui double undef to i16
+  %r45 = fptosi double undef to i16
+  %r46 = fptoui double undef to i32
+  %r47 = fptosi double undef to i32
+  %r48 = fptoui double undef to i64
+  %r49 = fptosi double undef to i64
+  %r50 = sitofp i1 undef to float
+  %r51 = uitofp i1 undef to float
+  %r52 = sitofp i1 undef to double
+  %r53 = uitofp i1 undef to double
+  %r54 = sitofp i8 undef to float
+  %r55 = uitofp i8 undef to float
+  %r56 = sitofp i8 undef to double
+  %r57 = uitofp i8 undef to double
+  %r58 = sitofp i16 undef to float
+  %r59 = uitofp i16 undef to float
+  %r60 = sitofp i16 undef to double
+  %r61 = uitofp i16 undef to double
+  %r62 = sitofp i32 undef to float
+  %r63 = uitofp i32 undef to float
+  %r64 = sitofp i32 undef to double
+  %r65 = uitofp i32 undef to double
+  %r66 = sitofp i64 undef to float
+  %r67 = uitofp i64 undef to float
+  %r68 = sitofp i64 undef to double
+  %r69 = uitofp i64 undef to double
+  %r80 = fptrunc double undef to float
+  %r81 = fptrunc <2 x double> undef to <2 x float>
+  %r82 = fptrunc <4 x double> undef to <4 x float>
+  %r83 = fptrunc <8 x double> undef to <8 x float>
+  %r84 = fptrunc <16 x double> undef to <16 x float>
+  %r85 = fpext float undef to double
+  %r86 = fpext <2 x float> undef to <2 x double>
+  %r87 = fpext <4 x float> undef to <4 x double>
+  %r88 = fpext <8 x float> undef to <8 x double>
+  %r89 = fpext <16 x float> undef to <16 x double>
+  %r90 = fptoui <2 x float> undef to <2 x i1>
+  %r91 = fptosi <2 x float> undef to <2 x i1>
+  %r92 = fptoui <2 x float> undef to <2 x i8>
+  %r93 = fptosi <2 x float> undef to <2 x i8>
+  %r94 = fptoui <2 x float> undef to <2 x i16>
+  %r95 = fptosi <2 x float> undef to <2 x i16>
+  %r96 = fptoui <2 x float> undef to <2 x i32>
+  %r97 = fptosi <2 x float> undef to <2 x i32>
+  %r98 = fptoui <2 x float> undef to <2 x i64>
+  %r99 = fptosi <2 x float> undef to <2 x i64>
+  %r100 = fptoui <2 x double> undef to <2 x i1>
+  %r101 = fptosi <2 x double> undef to <2 x i1>
+  %r102 = fptoui <2 x double> undef to <2 x i8>
+  %r103 = fptosi <2 x double> undef to <2 x i8>
+  %r104 = fptoui <2 x double> undef to <2 x i16>
+  %r105 = fptosi <2 x double> undef to <2 x i16>
+  %r106 = fptoui <2 x double> undef to <2 x i32>
+  %r107 = fptosi <2 x double> undef to <2 x i32>
+  %r108 = fptoui <2 x double> undef to <2 x i64>
+  %r109 = fptosi <2 x double> undef to <2 x i64>
+
+  %r110 = fptoui <4 x float> undef to <4 x i1>
+  %r111 = fptosi <4 x float> undef to <4 x i1>
+  %r112 = fptoui <4 x float> undef to <4 x i8>
+  %r113 = fptosi <4 x float> undef to <4 x i8>
+  %r114 = fptoui <4 x float> undef to <4 x i16>
+  %r115 = fptosi <4 x float> undef to <4 x i16>
+  %r116 = fptoui <4 x float> undef to <4 x i32>
+  %r117 = fptosi <4 x float> undef to <4 x i32>
+  %r118 = fptoui <4 x float> undef to <4 x i64>
+  %r119 = fptosi <4 x float> undef to <4 x i64>
+
+  %r120 = fptoui <4 x double> undef to <4 x i1>
+  %r121 = fptosi <4 x double> undef to <4 x i1>
+  %r122 = fptoui <4 x double> undef to <4 x i8>
+  %r123 = fptosi <4 x double> undef to <4 x i8>
+  %r124 = fptoui <4 x double> undef to <4 x i16>
+  %r125 = fptosi <4 x double> undef to <4 x i16>
+  %r126 = fptoui <4 x double> undef to <4 x i32>
+  %r127 = fptosi <4 x double> undef to <4 x i32>
+  %r128 = fptoui <4 x double> undef to <4 x i64>
+  %r129 = fptosi <4 x double> undef to <4 x i64>
+
+  %r130 = fptoui <8 x float> undef to <8 x i1>
+  %r131 = fptosi <8 x float> undef to <8 x i1>
+  %r132 = fptoui <8 x float> undef to <8 x i8>
+  %r133 = fptosi <8 x float> undef to <8 x i8>
+  %r134 = fptoui <8 x float> undef to <8 x i16>
+  %r135 = fptosi <8 x float> undef to <8 x i16>
+  %r136 = fptoui <8 x float> undef to <8 x i32>
+  %r137 = fptosi <8 x float> undef to <8 x i32>
+  %r138 = fptoui <8 x float> undef to <8 x i64>
+  %r139 = fptosi <8 x float> undef to <8 x i64>
+
+  %r140 = fptoui <8 x double> undef to <8 x i1>
+  %r141 = fptosi <8 x double> undef to <8 x i1>
+  %r142 = fptoui <8 x double> undef to <8 x i8>
+  %r143 = fptosi <8 x double> undef to <8 x i8>
+  %r144 = fptoui <8 x double> undef to <8 x i16>
+  %r145 = fptosi <8 x double> undef to <8 x i16>
+  %r146 = fptoui <8 x double> undef to <8 x i32>
+  %r147 = fptosi <8 x double> undef to <8 x i32>
+  %r148 = fptoui <8 x double> undef to <8 x i64>
+  %r149 = fptosi <8 x double> undef to <8 x i64>
+
+  %r150 = fptoui <16 x float> undef to <16 x i1>
+  %r151 = fptosi <16 x float> undef to <16 x i1>
+  %r152 = fptoui <16 x float> undef to <16 x i8>
+  %r153 = fptosi <16 x float> undef to <16 x i8>
+  %r154 = fptoui <16 x float> undef to <16 x i16>
+  %r155 = fptosi <16 x float> undef to <16 x i16>
+  %r156 = fptoui <16 x float> undef to <16 x i32>
+  %r157 = fptosi <16 x float> undef to <16 x i32>
+  %r158 = fptoui <16 x float> undef to <16 x i64>
+  %r159 = fptosi <16 x float> undef to <16 x i64>
+
+  %r160 = fptoui <16 x double> undef to <16 x i1>
+  %r161 = fptosi <16 x double> undef to <16 x i1>
+  %r162 = fptoui <16 x double> undef to <16 x i8>
+  %r163 = fptosi <16 x double> undef to <16 x i8>
+  %r164 = fptoui <16 x double> undef to <16 x i16>
+  %r165 = fptosi <16 x double> undef to <16 x i16>
+  %r166 = fptoui <16 x double> undef to <16 x i32>
+  %r167 = fptosi <16 x double> undef to <16 x i32>
+  %r168 = fptoui <16 x double> undef to <16 x i64>
+  %r169 = fptosi <16 x double> undef to <16 x i64>
+
+  %r170 = uitofp <2 x i1> undef to <2 x float>
+  %r171 = sitofp <2 x i1> undef to <2 x float>
+  %r172 = uitofp <2 x i8> undef to <2 x float>
+  %r173 = sitofp <2 x i8> undef to <2 x float>
+  %r174 = uitofp <2 x i16> undef to <2 x float>
+  %r175 = sitofp <2 x i16> undef to <2 x float>
+  %r176 = uitofp <2 x i32> undef to <2 x float>
+  %r177 = sitofp <2 x i32> undef to <2 x float>
+  %r178 = uitofp <2 x i64> undef to <2 x float>
+  %r179 = sitofp <2 x i64> undef to <2 x float>
+
+  %r180 = uitofp <2 x i1> undef to <2 x double>
+  %r181 = sitofp <2 x i1> undef to <2 x double>
+  %r182 = uitofp <2 x i8> undef to <2 x double>
+  %r183 = sitofp <2 x i8> undef to <2 x double>
+  %r184 = uitofp <2 x i16> undef to <2 x double>
+  %r185 = sitofp <2 x i16> undef to <2 x double>
+  %r186 = uitofp <2 x i32> undef to <2 x double>
+  %r187 = sitofp <2 x i32> undef to <2 x double>
+  %r188 = uitofp <2 x i64> undef to <2 x double>
+  %r189 = sitofp <2 x i64> undef to <2 x double>
+
+  %r190 = uitofp <4 x i1> undef to <4 x float>
+  %r191 = sitofp <4 x i1> undef to <4 x float>
+  %r192 = uitofp <4 x i8> undef to <4 x float>
+  %r193 = sitofp <4 x i8> undef to <4 x float>
+  %r194 = uitofp <4 x i16> undef to <4 x float>
+  %r195 = sitofp <4 x i16> undef to <4 x float>
+  %r196 = uitofp <4 x i32> undef to <4 x float>
+  %r197 = sitofp <4 x i32> undef to <4 x float>
+  %r198 = uitofp <4 x i64> undef to <4 x float>
+  %r199 = sitofp <4 x i64> undef to <4 x float>
+
+  %r200 = uitofp <4 x i1> undef to <4 x double>
+  %r201 = sitofp <4 x i1> undef to <4 x double>
+  %r202 = uitofp <4 x i8> undef to <4 x double>
+  %r203 = sitofp <4 x i8> undef to <4 x double>
+  %r204 = uitofp <4 x i16> undef to <4 x double>
+  %r205 = sitofp <4 x i16> undef to <4 x double>
+  %r206 = uitofp <4 x i32> undef to <4 x double>
+  %r207 = sitofp <4 x i32> undef to <4 x double>
+  %r208 = uitofp <4 x i64> undef to <4 x double>
+  %r209 = sitofp <4 x i64> undef to <4 x double>
+
+  %r210 = uitofp <8 x i1> undef to <8 x float>
+  %r211 = sitofp <8 x i1> undef to <8 x float>
+  %r212 = uitofp <8 x i8> undef to <8 x float>
+  %r213 = sitofp <8 x i8> undef to <8 x float>
+  %r214 = uitofp <8 x i16> undef to <8 x float>
+  %r215 = sitofp <8 x i16> undef to <8 x float>
+  %r216 = uitofp <8 x i32> undef to <8 x float>
+  %r217 = sitofp <8 x i32> undef to <8 x float>
+  %r218 = uitofp <8 x i64> undef to <8 x float>
+  %r219 = sitofp <8 x i64> undef to <8 x float>
+
+  %r220 = uitofp <8 x i1> undef to <8 x double>
+  %r221 = sitofp <8 x i1> undef to <8 x double>
+  %r222 = uitofp <8 x i8> undef to <8 x double>
+  %r223 = sitofp <8 x i8> undef to <8 x double>
+  %r224 = uitofp <8 x i16> undef to <8 x double>
+  %r225 = sitofp <8 x i16> undef to <8 x double>
+  %r226 = uitofp <8 x i16> undef to <8 x double>
+  %r227 = sitofp <8 x i16> undef to <8 x double>
+  %r228 = uitofp <8 x i64> undef to <8 x double>
+  %r229 = sitofp <8 x i64> undef to <8 x double>
+
+  %r230 = uitofp <16 x i1> undef to <16 x float>
+  %r231 = sitofp <16 x i1> undef to <16 x float>
+  %r232 = uitofp <16 x i8> undef to <16 x float>
+  %r233 = sitofp <16 x i8> undef to <16 x float>
+  %r234 = uitofp <16 x i16> undef to <16 x float>
+  %r235 = sitofp <16 x i16> undef to <16 x float>
+  %r236 = uitofp <16 x i32> undef to <16 x float>
+  %r237 = sitofp <16 x i32> undef to <16 x float>
+  %r238 = uitofp <16 x i64> undef to <16 x float>
+  %r239 = sitofp <16 x i64> undef to <16 x float>
+
+  %r240 = uitofp <16 x i1> undef to <16 x double>
+  %r241 = sitofp <16 x i1> undef to <16 x double>
+  %r242 = uitofp <16 x i8> undef to <16 x double>
+  %r243 = sitofp <16 x i8> undef to <16 x double>
+  %r244 = uitofp <16 x i16> undef to <16 x double>
+  %r245 = sitofp <16 x i16> undef to <16 x double>
+  %r246 = uitofp <16 x i16> undef to <16 x double>
+  %r247 = sitofp <16 x i16> undef to <16 x double>
+  %r248 = uitofp <16 x i64> undef to <16 x double>
+  %r249 = sitofp <16 x i64> undef to <16 x double>
+
+  ret i32 undef
+}
+
+define i32 @casts_with_users(i8 %a, i16 %b, i32 %c, i64 %d, i1 %e) {
+; CHECK-LABEL: 'casts_with_users'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i8 %a to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = sext i8 %a to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = sext i8 %a to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = sext i16 %b to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i16 %b to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = sext i32 %c to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r0, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r1, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r2, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r3, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r4, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r5, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r6 = zext i8 %a to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r7 = zext i8 %a to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = zext i8 %a to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %b to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = zext i16 %b to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %c to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r6, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r7, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r8, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r9, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r10, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r11, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r12 = trunc i64 %d to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r13 = trunc i64 %d to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r14 = trunc i64 %d to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r15 = trunc i32 %c to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r16 = trunc i32 %c to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r17 = trunc i16 %b to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.8 = add i8 %r14, %r16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.8.1 = add i8 %tie.trunc.8, %r17
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tie.trunc.16 = add i16 %r13, %r15
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.16.8 = trunc i16 %r6 to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.32.8 = trunc i32 %r7 to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.32.16 = trunc i32 %r9 to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.8 = trunc i64 %r8 to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.16 = trunc i64 %r10 to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %free.trunc.64.32 = trunc i64 %r11 to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r18 = sext i1 %e to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r19 = sext i1 %e to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r20 = sext i1 %e to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r21 = sext i1 %e to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r22 = zext i1 %e to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r23 = zext i1 %e to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r24 = zext i1 %e to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r25 = zext i1 %e to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r18, ptr undef, align 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r19, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r20, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r21, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r22, ptr undef, align 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r23, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r24, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r25, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r12
+;
+  %r0 = sext i8 %a to i16
+  %r1 = sext i8 %a to i32
+  %r2 = sext i8 %a to i64
+  %r3 = sext i16 %b to i32
+  %r4 = sext i16 %b to i64
+  %r5 = sext i32 %c to i64
+  store i16 %r0, ptr undef
+  store i32 %r1, ptr undef
+  store i64 %r2, ptr undef
+  store i32 %r3, ptr undef
+  store i64 %r4, ptr undef
+  store i64 %r5, ptr undef
+
+  %r6 = zext i8 %a to i16
+  %r7 = zext i8 %a to i32
+  %r8 = zext i8 %a to i64
+  %r9 = zext i16 %b to i32
+  %r10 = zext i16 %b to i64
+  %r11 = zext i32 %c to i64
+  store i16 %r6, ptr undef
+  store i32 %r7, ptr undef
+  store i64 %r8, ptr undef
+  store i32 %r9, ptr undef
+  store i64 %r10, ptr undef
+  store i64 %r11, ptr undef
+
+  %r12 = trunc i64 %d to i32
+  %r13 = trunc i64 %d to i16
+  %r14 = trunc i64 %d to i8
+  %r15 = trunc i32 %c to i16
+  %r16 = trunc i32 %c to i8
+  %r17 = trunc i16 %b to i8
+  %tie.trunc.8 = add i8 %r14, %r16
+  %tie.trunc.8.1 = add i8 %tie.trunc.8, %r17
+  %tie.trunc.16 = add i16 %r13, %r15
+
+  %free.trunc.16.8 = trunc i16 %r6 to i8
+  %free.trunc.32.8 = trunc i32 %r7 to i8
+  %free.trunc.32.16 = trunc i32 %r9 to i16
+  %free.trunc.64.8 = trunc i64 %r8 to i8
+  %free.trunc.64.16 = trunc i64 %r10 to i16
+  %free.trunc.64.32 = trunc i64 %r11 to i32
+
+  %r18 = sext i1 %e to i8
+  %r19 = sext i1 %e to i16
+  %r20 = sext i1 %e to i32
+  %r21 = sext i1 %e to i64
+  %r22 = zext i1 %e to i8
+  %r23 = zext i1 %e to i16
+  %r24 = zext i1 %e to i32
+  %r25 = zext i1 %e to i64
+  store i8 %r18, ptr undef
+  store i16 %r19, ptr undef
+  store i32 %r20, ptr undef
+  store i64 %r21, ptr undef
+  store i8 %r22, ptr undef
+  store i16 %r23, ptr undef
+  store i32 %r24, ptr undef
+  store i64 %r25, ptr undef
+  ret i32 %r12
+}
+
+define i32 @bitcasts() {
+; CHECK-LABEL: 'bitcasts'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %b = bitcast float undef to float
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = bitcast i32 undef to float
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = bitcast float undef to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = bitcast i64 undef to double
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %a = bitcast i32 undef to i32
+  %b = bitcast float undef to float
+  %c = bitcast i32 undef to float
+  %d = bitcast float undef to i32
+  %e = bitcast i64 undef to double
+  %f = bitcast double undef to i64
+  %g = bitcast half undef to i16
+  %h = bitcast i16 undef to half
+  ret i32 undef
+}
+
+define i32 @load_extends() #0 {
+; CHECK-SVE-LABEL: 'load_extends'
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SVE128-NO-NEON-LABEL: 'load_extends'
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; FIXED-MIN-256-LABEL: 'load_extends'
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; FIXED-MIN-2048-LABEL: 'load_extends'
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, ptr undef, align 1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, ptr undef, align 2
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, ptr undef, align 4
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = sext i8 %loadi8 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = zext i8 %loadi8 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r8 = sext i16 %loadi16 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r9 = zext i16 %loadi16 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r10 = sext i32 %loadi32 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r11 = zext i32 %loadi32 to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %loadi8 = load i8, ptr undef
+  %loadi16 = load i16, ptr undef
+  %loadi32 = load i32, ptr undef
+  %loadv2i8 = load <2 x i8>, ptr undef
+  %loadv4i8 = load <4 x i8>, ptr undef
+  %loadv8i8 = load <8 x i8>, ptr undef
+  %loadv2i16 = load <2 x i16>, ptr undef
+  %loadv4i16 = load <4 x i16>, ptr undef
+  %loadv2i32 = load <2 x i32>, ptr undef
+  %loadv4i32 = load <4 x i32>, ptr undef
+  %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef
+  %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef
+
+  %r0 = sext i8 %loadi8 to i16
+  %r1 = zext i8 %loadi8 to i16
+  %r2 = sext i8 %loadi8 to i32
+  %r3 = zext i8 %loadi8 to i32
+  %r4 = sext i8 %loadi8 to i64
+  %r5 = zext i8 %loadi8 to i64
+  %r6 = sext i16 %loadi16 to i32
+  %r7 = zext i16 %loadi16 to i32
+  %r8 = sext i16 %loadi16 to i64
+  %r9 = zext i16 %loadi16 to i64
+  %r10 = sext i32 %loadi32 to i64
+  %r11 = zext i32 %loadi32 to i64
+
+  %v0 = sext <8 x i8> %loadv8i8 to <8 x i16>
+  %v1 = zext <8 x i8> %loadv8i8 to <8 x i16>
+  %v2 = sext <4 x i8> %loadv4i8 to <4 x i32>
+  %v3 = zext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
+  %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
+  %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
+  %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
+  %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
+  %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
+  %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
+  %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
+  %v12 = sext <4 x i32> %loadv4i32 to <4 x i64>
+  %v13 = zext <4 x i32> %loadv4i32 to <4 x i64>
+  %v14 = sext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+  %v15 = zext <vscale x 2 x i32> %loadnxv2i32 to <vscale x 2 x i64>
+  %v16 = sext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+  %v17 = zext <vscale x 4 x i32> %loadnxv4i32 to <vscale x 4 x i64>
+
+  ret i32 undef
+}
+
+define i32 @store_truncs() {
+; CHECK-LABEL: 'store_truncs'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = trunc i64 undef to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r0, ptr undef, align 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = trunc i64 undef to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r1, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = trunc i64 undef to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r2, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = trunc i32 undef to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r3, ptr undef, align 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = trunc i32 undef to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r4, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = trunc i16 undef to i8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r5, ptr undef, align 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %r0 = trunc i64 undef to i8
+  store i8 %r0, ptr undef
+  %r1 = trunc i64 undef to i16
+  store i16 %r1, ptr undef
+  %r2 = trunc i64 undef to i32
+  store i32 %r2, ptr undef
+  %r3 = trunc i32 undef to i8
+  store i8 %r3, ptr undef
+  %r4 = trunc i32 undef to i16
+  store i16 %r4, ptr undef
+  %r5 = trunc i16 undef to i8
+  store i8 %r5, ptr undef
+  ret i32 undef
+}
+
+define void @extend_extract() {
+; CHECK-LABEL: 'extend_extract'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = extractelement <8 x i8> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e16 = extractelement <8 x i16> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e32 = extractelement <8 x i32> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_16 = sext i8 %e8 to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8_16 = zext i8 %e8 to i16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_32 = sext i8 %e8 to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z8_32 = zext i8 %e8 to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s8_64 = sext i8 %e8 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z8_64 = zext i8 %e8 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16_32 = sext i16 %e16 to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %z16_32 = zext i16 %e16 to i32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s16_64 = sext i16 %e16 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z16_64 = zext i16 %e16 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s32_64 = sext i32 %e32 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %z32_64 = zext i32 %e32 to i64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @use(i16 %s8_16, i16 %z8_16, i32 %s8_32, i32 %z8_32, i64 %s8_64, i64 %z8_64, i32 %s16_32, i32 %z16_32, i64 %s16_64, i64 %z16_64, i64 %s32_64, i64 %z32_64)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %e8 = extractelement <8 x i8> undef, i32 1
+  %e16 = extractelement <8 x i16> undef, i32 1
+  %e32 = extractelement <8 x i32> undef, i32 1
+  %s8_16 = sext i8 %e8 to i16
+  %z8_16 = zext i8 %e8 to i16
+  %s8_32 = sext i8 %e8 to i32
+  %z8_32 = zext i8 %e8 to i32
+  %s8_64 = sext i8 %e8 to i64
+  %z8_64 = zext i8 %e8 to i64
+  %s16_32 = sext i16 %e16 to i32
+  %z16_32 = zext i16 %e16 to i32
+  %s16_64 = sext i16 %e16 to i64
+  %z16_64 = zext i16 %e16 to i64
+  %s32_64 = sext i32 %e32 to i64
+  %z32_64 = zext i32 %e32 to i64
+  call void @use(i16 %s8_16, i16 %z8_16, i32 %s8_32, i32 %z8_32, i64 %s8_64, i64 %z8_64, i32 %s16_32, i32 %z16_32, i64 %s16_64, i64 %z16_64, i64 %s32_64, i64 %z32_64)
+  ret void
+}
+
+declare void @use(i16, i16, i32, i32, i64, i64, i32, i32, i64, i64, i64, i64)
+
+define void @fp16cast() {
+; CHECK-SVE-LABEL: 'fp16cast'
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SVE128-NO-NEON-LABEL: 'fp16cast'
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
+; SVE128-NO-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-256-LABEL: 'fp16cast'
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
+; FIXED-MIN-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FIXED-MIN-2048-LABEL: 'fp16cast'
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r30 = fptoui half undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r31 = fptosi half undef to i1
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r32 = fptoui half undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r33 = fptosi half undef to i8
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r34 = fptoui half undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r35 = fptosi half undef to i16
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r36 = fptoui half undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r37 = fptosi half undef to i32
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r38 = fptoui half undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r39 = fptosi half undef to i64
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x half> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x half> undef to <2 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x half> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r93 = fptosi <2 x half> undef to <2 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r94 = fptoui <2 x half> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r95 = fptosi <2 x half> undef to <2 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r96 = fptoui <2 x half> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r97 = fptosi <2 x half> undef to <2 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r98 = fptoui <2 x half> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r99 = fptosi <2 x half> undef to <2 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r110 = fptoui <4 x half> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r111 = fptosi <4 x half> undef to <4 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r112 = fptoui <4 x half> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r113 = fptosi <4 x half> undef to <4 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r114 = fptoui <4 x half> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r115 = fptosi <4 x half> undef to <4 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x half> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x half> undef to <4 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r118 = fptoui <4 x half> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r119 = fptosi <4 x half> undef to <4 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r130 = fptoui <8 x half> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r131 = fptosi <8 x half> undef to <8 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132 = fptoui <8 x half> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r133 = fptosi <8 x half> undef to <8 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r134 = fptoui <8 x half> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r135 = fptosi <8 x half> undef to <8 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r136 = fptoui <8 x half> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r137 = fptosi <8 x half> undef to <8 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r138 = fptoui <8 x half> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r139 = fptosi <8 x half> undef to <8 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r150 = fptoui <16 x half> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r151 = fptosi <16 x half> undef to <16 x i1>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r152 = fptoui <16 x half> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r153 = fptosi <16 x half> undef to <16 x i8>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r154 = fptoui <16 x half> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r155 = fptosi <16 x half> undef to <16 x i16>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r156 = fptoui <16 x half> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r157 = fptosi <16 x half> undef to <16 x i32>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r158 = fptoui <16 x half> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r159 = fptosi <16 x half> undef to <16 x i64>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r250 = uitofp <8 x i1> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r251 = sitofp <8 x i1> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r252 = uitofp <8 x i8> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r253 = sitofp <8 x i8> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r254 = uitofp <8 x i16> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r255 = sitofp <8 x i16> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r256 = uitofp <8 x i32> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r257 = sitofp <8 x i32> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r258 = uitofp <8 x i64> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r259 = sitofp <8 x i64> undef to <8 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r260 = uitofp <16 x i1> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r261 = sitofp <16 x i1> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r262 = uitofp <16 x i8> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r263 = sitofp <16 x i8> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r264 = uitofp <16 x i16> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r265 = sitofp <16 x i16> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r266 = uitofp <16 x i32> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r267 = sitofp <16 x i32> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r268 = uitofp <16 x i64> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r269 = sitofp <16 x i64> undef to <16 x half>
+; FIXED-MIN-2048-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %r30 = fptoui half undef to i1
+  %r31 = fptosi half undef to i1
+  %r32 = fptoui half undef to i8
+  %r33 = fptosi half undef to i8
+  %r34 = fptoui half undef to i16
+  %r35 = fptosi half undef to i16
+  %r36 = fptoui half undef to i32
+  %r37 = fptosi half undef to i32
+  %r38 = fptoui half undef to i64
+  %r39 = fptosi half undef to i64
+
+  %r90 = fptoui <2 x half> undef to <2 x i1>
+  %r91 = fptosi <2 x half> undef to <2 x i1>
+  %r92 = fptoui <2 x half> undef to <2 x i8>
+  %r93 = fptosi <2 x half> undef to <2 x i8>
+  %r94 = fptoui <2 x half> undef to <2 x i16>
+  %r95 = fptosi <2 x half> undef to <2 x i16>
+  %r96 = fptoui <2 x half> undef to <2 x i32>
+  %r97 = fptosi <2 x half> undef to <2 x i32>
+  %r98 = fptoui <2 x half> undef to <2 x i64>
+  %r99 = fptosi <2 x half> undef to <2 x i64>
+
+  %r110 = fptoui <4 x half> undef to <4 x i1>
+  %r111 = fptosi <4 x half> undef to <4 x i1>
+  %r112 = fptoui <4 x half> undef to <4 x i8>
+  %r113 = fptosi <4 x half> undef to <4 x i8>
+  %r114 = fptoui <4 x half> undef to <4 x i16>
+  %r115 = fptosi <4 x half> undef to <4 x i16>
+  %r116 = fptoui <4 x half> undef to <4 x i32>
+  %r117 = fptosi <4 x half> undef to <4 x i32>
+  %r118 = fptoui <4 x half> undef to <4 x i64>
+  %r119 = fptosi <4 x half> undef to <4 x i64>
+
+  %r130 = fptoui <8 x half> undef to <8 x i1>
+  %r131 = fptosi <8 x half> undef to <8 x i1>
+  %r132 = fptoui <8 x half> undef to <8 x i8>
+  %r133 = fptosi <8 x half> undef to <8 x i8>
+  %r134 = fptoui <8 x half> undef to <8 x i16>
+  %r135 = fptosi <8 x half> undef to <8 x i16>
+  %r136 = fptoui <8 x half> undef to <8 x i32>
+  %r137 = fptosi <8 x half> undef to <8 x i32>
+  %r138 = fptoui <8 x half> undef to <8 x i64>
+  %r139 = fptosi <8 x half> undef to <8 x i64>
+
+  %r150 = fptoui <16 x half> undef to <16 x i1>
+  %r151 = fptosi <16 x half> undef to <16 x i1>
+  %r152 = fptoui <16 x half> undef to <16 x i8>
+  %r153 = fptosi <16 x half> undef to <16 x i8>
+  %r154 = fptoui <16 x half> undef to <16 x i16>
+  %r155 = fptosi <16 x half> undef to <16 x i16>
+  %r156 = fptoui <16 x half> undef to <16 x i32>
+  %r157 = fptosi <16 x half> undef to <16 x i32>
+  %r158 = fptoui <16 x half> undef to <16 x i64>
+  %r159 = fptosi <16 x half> undef to <16 x i64>
+
+  %r250 = uitofp <8 x i1> undef to <8 x half>
+  %r251 = sitofp <8 x i1> undef to <8 x half>
+  %r252 = uitofp <8 x i8> undef to <8 x half>
+  %r253 = sitofp <8 x i8> undef to <8 x half>
+  %r254 = uitofp <8 x i16> undef to <8 x half>
+  %r255 = sitofp <8 x i16> undef to <8 x half>
+  %r256 = uitofp <8 x i32> undef to <8 x half>
+  %r257 = sitofp <8 x i32> undef to <8 x half>
+  %r258 = uitofp <8 x i64> undef to <8 x half>
+  %r259 = sitofp <8 x i64> undef to <8 x half>
+
+  %r260 = uitofp <16 x i1> undef to <16 x half>
+  %r261 = sitofp <16 x i1> undef to <16 x half>
+  %r262 = uitofp <16 x i8> undef to <16 x half>
+  %r263 = sitofp <16 x i8> undef to <16 x half>
+  %r264 = uitofp <16 x i16> undef to <16 x half>
+  %r265 = sitofp <16 x i16> undef to <16 x half>
+  %r266 = uitofp <16 x i32> undef to <16 x half>
+  %r267 = sitofp <16 x i32> undef to <16 x half>
+  %r268 = uitofp <16 x i64> undef to <16 x half>
+  %r269 = sitofp <16 x i64> undef to <16 x half>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll b/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
index 3d835779aa882..35b987532e394 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
@@ -1,19 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v1 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-2
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-n2 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=cortex-a510 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v2 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple="aarch64--linux-gnu"
 
 define void @masked_gathers(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nxv8i1mask, <4 x i1> %v4i1mask, <1 x i1> %v1i1mask, <vscale x 1 x i1> %nxv1i1mask) #0 {
-; CHECK-LABEL: 'masked_gathers'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv4i1 = call <vscale x 4 x i1> @llvm.masked.gather.nxv4i1.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i1> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_gathers'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv4i1 = call <vscale x 4 x i1> @llvm.masked.gather.nxv4i1.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i1> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gathers'
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
@@ -21,13 +22,6 @@ define void @masked_gathers(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nx
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv4i1 = call <vscale x 4 x i1> @llvm.masked.gather.nxv4i1.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i1> zeroinitializer)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CHECK-VSCALE-1-LABEL: 'masked_gathers'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv4i1 = call <vscale x 4 x i1> @llvm.masked.gather.nxv4i1.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i1> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
   %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
@@ -37,23 +31,17 @@ define void @masked_gathers(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nx
 }
 
 define void @masked_gathers_tune_generic(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nxv8i1mask, <4 x i1> %v4i1mask, <1 x i1> %v1i1mask, <vscale x 1 x i1> %nxv1i1mask) #1 {
-; CHECK-LABEL: 'masked_gathers_tune_generic'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_gathers_tune_generic'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gathers_tune_generic'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CHECK-VSCALE-1-LABEL: 'masked_gathers_tune_generic'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %res.nxv1i64 = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask, <vscale x 1 x i64> zeroinitializer)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask, <vscale x 4 x i32> zeroinitializer)
   %res.nxv8i32 = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32(<vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask, <vscale x 8 x i32> zeroinitializer)
@@ -62,16 +50,16 @@ define void @masked_gathers_tune_generic(<vscale x 4 x i1> %nxv4i1mask, <vscale
 }
 
 define void @masked_gathers_no_vscale_range() #2 {
-; CHECK-LABEL: 'masked_gathers_no_vscale_range'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4f64 = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv2f64 = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8f32 = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4f32 = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv2f32 = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %res.nxv16i16 = call <vscale x 16 x i16> @llvm.masked.gather.nxv16i16.nxv16p0(<vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef, <vscale x 16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_gathers_no_vscale_range'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4f64 = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res.nxv2f64 = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8f32 = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4f32 = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res.nxv2f32 = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv16i16 = call <vscale x 16 x i16> @llvm.masked.gather.nxv16i16.nxv16p0(<vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef, <vscale x 16 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gathers_no_vscale_range'
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4f64 = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x double> undef)
@@ -83,17 +71,6 @@ define void @masked_gathers_no_vscale_range() #2 {
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x i16> undef)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x i16> undef)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CHECK-VSCALE-1-LABEL: 'masked_gathers_no_vscale_range'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4f64 = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x double> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res.nxv2f64 = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8f32 = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4f32 = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res.nxv2f32 = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res.nxv16i16 = call <vscale x 16 x i16> @llvm.masked.gather.nxv16i16.nxv16p0(<vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef, <vscale x 16 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0(<vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef, <vscale x 8 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.nxv4f64 = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64(<vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef, <vscale x 4 x double> undef)
   %res.nxv2f64 = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
@@ -110,17 +87,13 @@ define void @masked_gathers_no_vscale_range() #2 {
 }
 
 define <2 x i128> @masked_gather_v1i128(<2 x ptr> %ld, <2 x i1> %masks, <2 x i128> %passthru) #3 {
-; CHECK-LABEL: 'masked_gather_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
+; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_v1i128'
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
-;
-; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
 ;
   %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
   ret <2 x i128> %res
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 1ecd02e5c124a..dd3909ade5315 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -1,16 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-1
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -mcpu=neoverse-v1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-2
 ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -type-based-intrinsic-cost -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=TYPE_BASED_ONLY
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @vector_insert_extract(<vscale x 4 x i32> %v0, <vscale x 16 x i32> %v1, <16 x i32> %v2) {
-; CHECK-LABEL: 'vector_insert_extract'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'vector_insert_extract'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'vector_insert_extract'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
@@ -31,17 +39,29 @@ declare <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x
 declare <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32>, <vscale x 4 x i32>, i64)
 
 define void @vector_insert_extract_idxzero_128b() #1 {
-; CHECK-LABEL: 'vector_insert_extract_idxzero_128b'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'vector_insert_extract_idxzero_128b'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'vector_insert_extract_idxzero_128b'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_128b'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
@@ -76,17 +96,29 @@ declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x f
 declare <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float>, i64)
 
 define void @vector_insert_extract_idxzero_256b() #2 {
-; CHECK-LABEL: 'vector_insert_extract_idxzero_256b'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'vector_insert_extract_idxzero_256b'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'vector_insert_extract_idxzero_256b'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_256b'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
@@ -115,44 +147,83 @@ declare <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16>
 declare <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float>, i64)
 
 define void @reductions(<vscale x 4 x i32> %v0, <vscale x 4 x i64> %v1, <vscale x 4 x float> %v2, <vscale x 4 x double> %v3) {
-; CHECK-LABEL: 'reductions'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'reductions'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'reductions'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'reductions'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
@@ -235,16 +306,23 @@ define void @reductions(<vscale x 4 x i32> %v0, <vscale x 4 x i64> %v1, <vscale
 }
 
 define void @strict_fp_reductions(<vscale x 4 x float> %v0, <vscale x 4 x double> %v1) {
-; CHECK-LABEL: 'strict_fp_reductions'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'strict_fp_reductions'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'strict_fp_reductions'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'strict_fp_reductions'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -286,10 +364,15 @@ declare double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double>)
 
 
 define void  @count_zeroes(<vscale x 4 x i32> %A) {
-; CHECK-LABEL: 'count_zeroes'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'count_zeroes'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'count_zeroes'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'count_zeroes'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
@@ -305,35 +388,65 @@ declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>, i1)
 
 
 define void @vector_reverse() #0 {
-; CHECK-LABEL: 'vector_reverse'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'vector_reverse'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'vector_reverse'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_reverse'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
@@ -424,17 +537,29 @@ declare <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1>)
 declare <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1>)
 
 define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
-; CHECK-LABEL: 'unsupported_fp_ops'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'unsupported_fp_ops'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'unsupported_fp_ops'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'unsupported_fp_ops'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
@@ -462,9 +587,13 @@ define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 }
 
 define void @powi(<vscale x 4 x float> %vec) {
-; CHECK-LABEL: 'powi'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'powi'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'powi'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'powi'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
@@ -485,69 +614,133 @@ declare <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float>)
 declare <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float>)
 
 define void @vector_splice() #0 {
-; CHECK-LABEL: 'vector_splice'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'vector_splice'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'vector_splice'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_splice'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
@@ -718,28 +911,51 @@ declare <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double>,
 declare <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double>, <vscale x 4 x double>, i32)
 
 define void @get_lane_mask() #0 {
-; CHECK-LABEL: 'get_lane_mask'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'get_lane_mask'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'get_lane_mask'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'get_lane_mask'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
@@ -794,12 +1010,19 @@ define void @get_lane_mask() #0 {
 }
 
 define void @fshr() #0 {
-; CHECK-LABEL: 'fshr'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'fshr'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'fshr'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'fshr'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
@@ -816,12 +1039,19 @@ define void @fshr() #0 {
 }
 
 define void @fshl() #0 {
-; CHECK-LABEL: 'fshl'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'fshl'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'fshl'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'fshl'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
@@ -838,9 +1068,13 @@ define void @fshl() #0 {
 }
 
 define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru) {
-; CHECK-LABEL: 'masked_gather_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
+; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
+;
+; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
@@ -851,9 +1085,13 @@ define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale
 }
 
 define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru) {
-; CHECK-LABEL: 'masked_gather_nxv8i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
+; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv8i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
+;
+; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv8i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv8i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
@@ -864,9 +1102,13 @@ define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale
 }
 
 define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32> %passthru) {
-; CHECK-LABEL: 'masked_gather_v4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; CHECK-VSCALE-1-LABEL: 'masked_gather_v4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
+; CHECK-VSCALE-2-LABEL: 'masked_gather_v4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
@@ -877,9 +1119,13 @@ define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32>
 }
 
 define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i128> %passthru) {
-; CHECK-LABEL: 'masked_gather_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
+; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
+;
+; CHECK-VSCALE-2-LABEL: 'masked_gather_v1i128'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v1i128'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
@@ -890,9 +1136,13 @@ define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i12
 }
 
 define void @masked_scatter_nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, <vscale x 4 x i1> %masks) {
-; CHECK-LABEL: 'masked_scatter_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatter_nxv4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'masked_scatter_nxv4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
@@ -904,9 +1154,13 @@ define void @masked_scatter_nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr>
 }
 
 define void @masked_scatter_nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %masks) {
-; CHECK-LABEL: 'masked_scatter_nxv8i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatter_nxv8i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'masked_scatter_nxv8i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv8i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
@@ -918,9 +1172,13 @@ define void @masked_scatter_nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr>
 }
 
 define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %masks) {
-; CHECK-LABEL: 'masked_scatter_v4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatter_v4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'masked_scatter_v4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
@@ -932,9 +1190,13 @@ define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %ma
 }
 
 define void @masked_scatter_v1i128(<1 x i128> %data, <1 x ptr> %ptrs, <1 x i1> %masks) {
-; CHECK-LABEL: 'masked_scatter_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatter_v1i128'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'masked_scatter_v1i128'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v1i128'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
@@ -946,9 +1208,13 @@ define void @masked_scatter_v1i128(<1 x i128> %data, <1 x ptr> %ptrs, <1 x i1> %
 }
 
 define void @histogram_nxv2i64(<vscale x 2 x ptr> %buckets, <vscale x 2 x i1> %mask) #3 {
-; CHECK-LABEL: 'histogram_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_nxv2i64'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_nxv2i64'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv2i64'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
@@ -959,9 +1225,13 @@ define void @histogram_nxv2i64(<vscale x 2 x ptr> %buckets, <vscale x 2 x i1> %m
 }
 
 define void @histogram_nxv4i32(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %mask) #3 {
-; CHECK-LABEL: 'histogram_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
@@ -972,9 +1242,13 @@ define void @histogram_nxv4i32(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %m
 }
 
 define void @histogram_nxv8i16(<vscale x 8 x ptr> %buckets, <vscale x 8 x i1> %mask) #3 {
-; CHECK-LABEL: 'histogram_nxv8i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_nxv8i16'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_nxv8i16'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv8i16'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
@@ -985,9 +1259,13 @@ define void @histogram_nxv8i16(<vscale x 8 x ptr> %buckets, <vscale x 8 x i1> %m
 }
 
 define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1> %mask) #3 {
-; CHECK-LABEL: 'histogram_nxv16i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_nxv16i8'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_nxv16i8'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv16i8'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
@@ -998,9 +1276,13 @@ define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1>
 }
 
 define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
-; CHECK-LABEL: 'histogram_v2i64'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_v2i64'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_v2i64'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v2i64'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
@@ -1011,9 +1293,13 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
 }
 
 define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
-; CHECK-LABEL: 'histogram_v4i32'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_v4i32'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_v4i32'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v4i32'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
@@ -1024,9 +1310,13 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
 }
 
 define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
-; CHECK-LABEL: 'histogram_v8i16'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_v8i16'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_v8i16'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v8i16'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
@@ -1037,9 +1327,13 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
 }
 
 define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
-; CHECK-LABEL: 'histogram_v16i8'
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_v16i8'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_v16i8'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v16i8'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
@@ -1050,9 +1344,13 @@ define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
 }
 
 define void @histogram_nxv4i64(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %mask) #3 {
-; CHECK-LABEL: 'histogram_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i64'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i64'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i64'
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
@@ -1062,6 +1360,54 @@ define void @histogram_nxv4i64(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %m
   ret void
 }
 
+define void @match() #3 {
+; CHECK-VSCALE-1-LABEL: 'match'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-2-LABEL: 'match'
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; TYPE_BASED_ONLY-LABEL: 'match'
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+
+  %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+  %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+  %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+  %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+
+  %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+  %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+  %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+  %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+
+  ret void
+}
+
 declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64, i64)
 declare <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64, i64)
 declare <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64, i64)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-scatter.ll b/llvm/test/Analysis/CostModel/AArch64/sve-scatter.ll
index 2a94d63278888..b9defdf8be2c7 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-scatter.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-scatter.ll
@@ -1,19 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v1 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-2
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-n2 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=cortex-a510 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-1
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v2 < %s | FileCheck %s --check-prefix=CHECK-V2
 
 target triple="aarch64--linux-gnu"
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @masked_scatters(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nxv8i1mask, <4 x i1> %v4i1mask, <1 x i1> %v1i1mask, <vscale x 1 x i1> %nxv1i1mask) #0 {
-; CHECK-LABEL: 'masked_scatters'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i1.nxv4p0(<vscale x 4 x i1> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatters'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i1.nxv4p0(<vscale x 4 x i1> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatters'
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
@@ -22,12 +23,12 @@ define void @masked_scatters(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %n
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i1.nxv4p0(<vscale x 4 x i1> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; CHECK-VSCALE-1-LABEL: 'masked_scatters'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i1.nxv4p0(<vscale x 4 x i1> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V2-LABEL: 'masked_scatters'
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
+; CHECK-V2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
+; CHECK-V2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i1.nxv4p0(<vscale x 4 x i1> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
   call void @llvm.masked.scatter.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
@@ -37,23 +38,23 @@ define void @masked_scatters(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %n
 }
 
 define void @masked_scatters_tune_generic(<vscale x 4 x i1> %nxv4i1mask, <vscale x 8 x i1> %nxv8i1mask, <4 x i1> %v4i1mask, <1 x i1> %v1i1mask, <vscale x 1 x i1> %nxv1i1mask) #1 {
-; CHECK-LABEL: 'masked_scatters_tune_generic'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
-; CHECK-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatters_tune_generic'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatters_tune_generic'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; CHECK-VSCALE-1-LABEL: 'masked_scatters_tune_generic'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V2-LABEL: 'masked_scatters_tune_generic'
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
+; CHECK-V2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> undef, <vscale x 1 x ptr> undef, i32 0, <vscale x 1 x i1> %nxv1i1mask)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x ptr> undef, i32 0, <vscale x 4 x i1> %nxv4i1mask)
   call void @llvm.masked.scatter.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x ptr> undef, i32 0, <vscale x 8 x i1> %nxv8i1mask)
@@ -62,16 +63,16 @@ define void @masked_scatters_tune_generic(<vscale x 4 x i1> %nxv4i1mask, <vscale
 }
 
 define void @masked_scatters_no_vscale_range() #2 {
-; CHECK-LABEL: 'masked_scatters_no_vscale_range'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4f64.nxv4p0(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8f32.nxv8p0(<vscale x 8 x float> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.scatter.nxv16i16.nxv16p0(<vscale x 16 x i16> undef, <vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-LABEL: 'masked_scatters_no_vscale_range'
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4f64.nxv4p0(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8f32.nxv8p0(<vscale x 8 x float> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv16i16.nxv16p0(<vscale x 16 x i16> undef, <vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatters_no_vscale_range'
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4f64.nxv4p0(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
@@ -84,16 +85,16 @@ define void @masked_scatters_no_vscale_range() #2 {
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; CHECK-VSCALE-1-LABEL: 'masked_scatters_no_vscale_range'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4f64.nxv4p0(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8f32.nxv8p0(<vscale x 8 x float> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv16i16.nxv16p0(<vscale x 16 x i16> undef, <vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V2-LABEL: 'masked_scatters_no_vscale_range'
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: call void @llvm.masked.scatter.nxv4f64.nxv4p0(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: call void @llvm.masked.scatter.nxv8f32.nxv8p0(<vscale x 8 x float> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: call void @llvm.masked.scatter.nxv16i16.nxv16p0(<vscale x 16 x i16> undef, <vscale x 16 x ptr> undef, i32 1, <vscale x 16 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> undef, <vscale x 8 x ptr> undef, i32 1, <vscale x 8 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
+; CHECK-V2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call void @llvm.masked.scatter.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x ptr> undef, i32 1, <vscale x 4 x i1> undef)
   call void @llvm.masked.scatter.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x ptr> undef, i32 1, <vscale x 2 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/PowerPC/load-to-trunc.ll b/llvm/test/Analysis/CostModel/PowerPC/load-to-trunc.ll
index 57a6e98cfb4ee..b78d121ff4f38 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/load-to-trunc.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/load-to-trunc.ll
@@ -7,7 +7,7 @@
 ; Check that cost is 1 for unusual load to register sized load.
 define i32 @loadUnusualIntegerWithTrunc(ptr %ptr) {
 ; CHECK-LABEL: 'loadUnusualIntegerWithTrunc'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load i128, ptr %ptr, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load i128, ptr %ptr, align 16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %trunc = trunc i128 %out to i32
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %trunc
 ;
@@ -18,7 +18,7 @@ define i32 @loadUnusualIntegerWithTrunc(ptr %ptr) {
 
 define i128 @loadUnusualInteger(ptr %ptr) {
 ; CHECK-LABEL: 'loadUnusualInteger'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load i128, ptr %ptr, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load i128, ptr %ptr, align 16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i128 %out
 ;
   %out = load i128, ptr %ptr
diff --git a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
index fce9f71d37634..6d3dfc3061388 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
@@ -9,28 +9,28 @@ define void @foo_no_vscale_range() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 662 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 335 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 334 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 662 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 661 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 335 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 334 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
@@ -75,28 +75,28 @@ define void @foo_vscale_range_2_16() vscale_range(2,16) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 171 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 171 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 171 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 171 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fp-min-max-abs.ll b/llvm/test/Analysis/CostModel/RISCV/fp-min-max-abs.ll
index 0b2c8da4438da..9eb06a07f135f 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fp-min-max-abs.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fp-min-max-abs.ll
@@ -1,30 +1,48 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfh | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfhmin | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define void @fabs() {
 ; CHECK-LABEL: 'fabs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call float @llvm.fabs.f32(float undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x float> @llvm.fabs.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x float> @llvm.fabs.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x float> @llvm.fabs.nxv16f32(<vscale x 16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call double @llvm.fabs.f64(double undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x double> @llvm.fabs.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x double> @llvm.fabs.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x double> @llvm.fabs.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x double> @llvm.fabs.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call bfloat @llvm.fabs.bf16(bfloat undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.fabs.v8bf16(<8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x bfloat> @llvm.fabs.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.fabs.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call float @llvm.fabs.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 1 x float> @llvm.fabs.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 8 x float> @llvm.fabs.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %19 = call <vscale x 16 x float> @llvm.fabs.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call double @llvm.fabs.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %24 = call <16 x double> @llvm.fabs.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 1 x double> @llvm.fabs.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %26 = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %27 = call <vscale x 4 x double> @llvm.fabs.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %28 = call <vscale x 8 x double> @llvm.fabs.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+  call bfloat @llvm.fabs.bf16(bfloat undef)
+  call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> undef)
+  call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> undef)
+  call <8 x bfloat> @llvm.fabs.v8bf16(<8 x bfloat> undef)
+  call <16 x bfloat> @llvm.fabs.v16f16(<16 x bfloat> undef)
+  call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> undef)
+  call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> undef)
+  call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> undef)
+  call <vscale x 16 x bfloat> @llvm.fabs.nxv16f16(<vscale x 16 x bfloat> undef)
   call float @llvm.fabs.f32(float undef)
   call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
   call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
@@ -53,11 +71,11 @@ define void @fabs_f16() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x half> @llvm.fabs.v2f16(<2 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x half> @llvm.fabs.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call <vscale x 16 x half> @llvm.fabs.nxv16f16(<vscale x 16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call half @llvm.fabs.f16(half undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fp-sqrt-pow.ll b/llvm/test/Analysis/CostModel/RISCV/fp-sqrt-pow.ll
index be9c19dc59a85..446627f6bf3c0 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fp-sqrt-pow.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fp-sqrt-pow.ll
@@ -8,30 +8,30 @@ define void @sqrt() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x bfloat> @llvm.sqrt.v2bf16(<2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x bfloat> @llvm.sqrt.v4bf16(<4 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x bfloat> @llvm.sqrt.v8bf16(<8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x bfloat> @llvm.sqrt.v16bf16(<16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x bfloat> @llvm.sqrt.v16bf16(<16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x bfloat> @llvm.sqrt.nxv2bf16(<vscale x 2 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x bfloat> @llvm.sqrt.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.sqrt.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call <vscale x 16 x bfloat> @llvm.sqrt.nxv16bf16(<vscale x 16 x bfloat> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call float @llvm.sqrt.f32(float undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 1 x float> @llvm.sqrt.nxv1f32(<vscale x 1 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 8 x float> @llvm.sqrt.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 16 x float> @llvm.sqrt.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 8 x float> @llvm.sqrt.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %19 = call <vscale x 16 x float> @llvm.sqrt.nxv16f32(<vscale x 16 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call double @llvm.sqrt.f64(double undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <16 x double> @llvm.sqrt.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %24 = call <16 x double> @llvm.sqrt.v16f64(<16 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 1 x double> @llvm.sqrt.nxv1f64(<vscale x 1 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 4 x double> @llvm.sqrt.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 8 x double> @llvm.sqrt.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %26 = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %27 = call <vscale x 4 x double> @llvm.sqrt.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %28 = call <vscale x 8 x double> @llvm.sqrt.nxv8f64(<vscale x 8 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call bfloat @llvm.sqrt.bf16(bfloat undef)
@@ -71,11 +71,11 @@ define void @sqrt_f16() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x half> @llvm.sqrt.v2f16(<2 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x half> @llvm.sqrt.nxv2f16(<vscale x 2 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x half> @llvm.sqrt.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x half> @llvm.sqrt.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call <vscale x 16 x half> @llvm.sqrt.nxv16f16(<vscale x 16 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call half @llvm.sqrt.f16(half undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
index 55db70ce1e912..5f2728f93d551 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
@@ -209,37 +209,37 @@ define void @ctlz() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.ctlz.nxv2i8(<vscale x 2 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.ctlz.nxv4i8(<vscale x 4 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.ctlz.nxv8i8(<vscale x 8 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 32 x i8> @llvm.ctlz.nxv32i8(<vscale x 32 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <vscale x 64 x i8> @llvm.ctlz.nxv64i8(<vscale x 64 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = call <vscale x 32 x i8> @llvm.ctlz.nxv32i8(<vscale x 32 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call <vscale x 64 x i8> @llvm.ctlz.nxv64i8(<vscale x 64 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i16> @llvm.ctlz.nxv1i16(<vscale x 1 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i16> @llvm.ctlz.nxv2i16(<vscale x 2 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctlz.nxv4i16(<vscale x 4 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctlz.nxv16i16(<vscale x 16 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <vscale x 32 x i16> @llvm.ctlz.nxv32i16(<vscale x 32 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctlz.nxv16i16(<vscale x 16 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call <vscale x 32 x i16> @llvm.ctlz.nxv32i16(<vscale x 32 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x i32> @llvm.ctlz.nxv1i32(<vscale x 1 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x i32> @llvm.ctlz.nxv2i32(<vscale x 2 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctlz.nxv8i32(<vscale x 8 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctlz.nxv16i32(<vscale x 16 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctlz.nxv8i32(<vscale x 8 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctlz.nxv16i32(<vscale x 16 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <16 x i64> @llvm.ctlz.v16i64(<16 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %32 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %33 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %34 = call <16 x i64> @llvm.ctlz.v16i64(<16 x i64> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 1 x i64> @llvm.ctlz.nxv1i64(<vscale x 1 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %36 = call <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %37 = call <vscale x 4 x i64> @llvm.ctlz.nxv4i64(<vscale x 4 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = call <vscale x 8 x i64> @llvm.ctlz.nxv8i64(<vscale x 8 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = call <vscale x 16 x i64> @llvm.ctlz.nxv16i64(<vscale x 16 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %36 = call <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %37 = call <vscale x 4 x i64> @llvm.ctlz.nxv4i64(<vscale x 4 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %38 = call <vscale x 8 x i64> @llvm.ctlz.nxv8i64(<vscale x 8 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %39 = call <vscale x 16 x i64> @llvm.ctlz.nxv16i64(<vscale x 16 x i64> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> undef, i1 false)
@@ -336,37 +336,37 @@ define void @cttz() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.cttz.nxv2i8(<vscale x 2 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.cttz.nxv4i8(<vscale x 4 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.cttz.nxv8i8(<vscale x 8 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 32 x i8> @llvm.cttz.nxv32i8(<vscale x 32 x i8> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <vscale x 64 x i8> @llvm.cttz.nxv64i8(<vscale x 64 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = call <vscale x 32 x i8> @llvm.cttz.nxv32i8(<vscale x 32 x i8> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %11 = call <vscale x 64 x i8> @llvm.cttz.nxv64i8(<vscale x 64 x i8> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i16> @llvm.cttz.nxv1i16(<vscale x 1 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i16> @llvm.cttz.nxv2i16(<vscale x 2 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i16> @llvm.cttz.nxv4i16(<vscale x 4 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x i16> @llvm.cttz.nxv16i16(<vscale x 16 x i16> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <vscale x 32 x i16> @llvm.cttz.nxv32i16(<vscale x 32 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = call <vscale x 16 x i16> @llvm.cttz.nxv16i16(<vscale x 16 x i16> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %21 = call <vscale x 32 x i16> @llvm.cttz.nxv32i16(<vscale x 32 x i16> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x i32> @llvm.cttz.nxv1i32(<vscale x 1 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x i32> @llvm.cttz.nxv2i32(<vscale x 2 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i32> @llvm.cttz.nxv8i32(<vscale x 8 x i32> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <vscale x 16 x i32> @llvm.cttz.nxv16i32(<vscale x 16 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %28 = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %29 = call <vscale x 8 x i32> @llvm.cttz.nxv8i32(<vscale x 8 x i32> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %30 = call <vscale x 16 x i32> @llvm.cttz.nxv16i32(<vscale x 16 x i32> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <16 x i64> @llvm.cttz.v16i64(<16 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %32 = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %33 = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %34 = call <16 x i64> @llvm.cttz.v16i64(<16 x i64> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 1 x i64> @llvm.cttz.nxv1i64(<vscale x 1 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %36 = call <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %37 = call <vscale x 4 x i64> @llvm.cttz.nxv4i64(<vscale x 4 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = call <vscale x 8 x i64> @llvm.cttz.nxv8i64(<vscale x 8 x i64> undef, i1 false)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = call <vscale x 16 x i64> @llvm.cttz.nxv16i64(<vscale x 16 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %36 = call <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %37 = call <vscale x 4 x i64> @llvm.cttz.nxv4i64(<vscale x 4 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %38 = call <vscale x 8 x i64> @llvm.cttz.nxv8i64(<vscale x 8 x i64> undef, i1 false)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %39 = call <vscale x 16 x i64> @llvm.cttz.nxv16i64(<vscale x 16 x i64> undef, i1 false)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call <2 x i8> @llvm.cttz.v2i8(<2 x i8> undef, i1 false)
@@ -465,37 +465,37 @@ define void @ctpop() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i8> @llvm.ctpop.nxv2i8(<vscale x 2 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i16> @llvm.ctpop.nxv1i16(<vscale x 1 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i16> @llvm.ctpop.nxv2i16(<vscale x 2 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 1 x i32> @llvm.ctpop.nxv1i32(<vscale x 1 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 2 x i32> @llvm.ctpop.nxv2i32(<vscale x 2 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <16 x i64> @llvm.ctpop.v16i64(<16 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %35 = call <16 x i64> @llvm.ctpop.v16i64(<16 x i64> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %36 = call <vscale x 1 x i64> @llvm.ctpop.nxv1i64(<vscale x 1 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %37 = call <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = call <vscale x 4 x i64> @llvm.ctpop.nxv4i64(<vscale x 4 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = call <vscale x 16 x i64> @llvm.ctpop.nxv16i64(<vscale x 16 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %37 = call <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %38 = call <vscale x 4 x i64> @llvm.ctpop.nxv4i64(<vscale x 4 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %39 = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %40 = call <vscale x 16 x i64> @llvm.ctpop.nxv16i64(<vscale x 16 x i64> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.ctpop.i8(i8 undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll b/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
index be6b7c57d2252..0758eb204be48 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
@@ -11,33 +11,33 @@ define void @sadd.sat() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.sadd.sat.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.sadd.sat.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.sadd.sat.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 2 x i16> @llvm.sadd.sat.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 4 x i16> @llvm.sadd.sat.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 16 x i16> @llvm.sadd.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 16 x i16> @llvm.sadd.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <vscale x 2 x i32> @llvm.sadd.sat.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 8 x i32> @llvm.sadd.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 16 x i32> @llvm.sadd.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call <vscale x 8 x i32> @llvm.sadd.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %27 = call <vscale x 16 x i32> @llvm.sadd.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <16 x i64> @llvm.sadd.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <vscale x 4 x i64> @llvm.sadd.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 8 x i64> @llvm.sadd.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %30 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %32 = call <16 x i64> @llvm.sadd.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %33 = call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %34 = call <vscale x 4 x i64> @llvm.sadd.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %35 = call <vscale x 8 x i64> @llvm.sadd.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
@@ -88,33 +88,33 @@ define void @uadd.sat() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.uadd.sat.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.uadd.sat.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.uadd.sat.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %10 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 2 x i16> @llvm.uadd.sat.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 4 x i16> @llvm.uadd.sat.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 16 x i16> @llvm.uadd.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 16 x i16> @llvm.uadd.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %19 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <vscale x 2 x i32> @llvm.uadd.sat.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 8 x i32> @llvm.uadd.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 16 x i32> @llvm.uadd.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call <vscale x 8 x i32> @llvm.uadd.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %27 = call <vscale x 16 x i32> @llvm.uadd.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %28 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <16 x i64> @llvm.uadd.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <vscale x 4 x i64> @llvm.uadd.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 8 x i64> @llvm.uadd.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %30 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %32 = call <16 x i64> @llvm.uadd.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %33 = call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %34 = call <vscale x 4 x i64> @llvm.uadd.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %35 = call <vscale x 8 x i64> @llvm.uadd.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
@@ -165,33 +165,33 @@ define void @usub.sat() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.usub.sat.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.usub.sat.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.usub.sat.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %10 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 2 x i16> @llvm.usub.sat.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 4 x i16> @llvm.usub.sat.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 16 x i16> @llvm.usub.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 16 x i16> @llvm.usub.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %19 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <vscale x 2 x i32> @llvm.usub.sat.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 8 x i32> @llvm.usub.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 16 x i32> @llvm.usub.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call <vscale x 8 x i32> @llvm.usub.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %27 = call <vscale x 16 x i32> @llvm.usub.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %28 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <16 x i64> @llvm.usub.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <vscale x 4 x i64> @llvm.usub.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 8 x i64> @llvm.usub.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %30 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %32 = call <16 x i64> @llvm.usub.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %33 = call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %34 = call <vscale x 4 x i64> @llvm.usub.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %35 = call <vscale x 8 x i64> @llvm.usub.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
@@ -242,33 +242,33 @@ define void @ssub.sat() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 2 x i8> @llvm.ssub.sat.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 4 x i8> @llvm.ssub.sat.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 8 x i8> @llvm.ssub.sat.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %10 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <vscale x 2 x i16> @llvm.ssub.sat.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 4 x i16> @llvm.ssub.sat.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 16 x i16> @llvm.ssub.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call <vscale x 16 x i16> @llvm.ssub.sat.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %19 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %23 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <vscale x 2 x i32> @llvm.ssub.sat.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %25 = call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = call <vscale x 8 x i32> @llvm.ssub.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = call <vscale x 16 x i32> @llvm.ssub.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call <vscale x 8 x i32> @llvm.ssub.sat.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %27 = call <vscale x 16 x i32> @llvm.ssub.sat.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %28 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <16 x i64> @llvm.ssub.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <vscale x 4 x i64> @llvm.ssub.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %35 = call <vscale x 8 x i64> @llvm.ssub.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %30 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %32 = call <16 x i64> @llvm.ssub.sat.v16i64(<16 x i64> undef, <16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %33 = call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %34 = call <vscale x 4 x i64> @llvm.ssub.sat.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %35 = call <vscale x 8 x i64> @llvm.ssub.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
index a8eb4e9a280dd..463232f082f40 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
@@ -17,6 +17,17 @@ define i32 @reduce_i1(i32 %arg) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512 = call i1 @llvm.vector.reduce.and.v512i1(<512 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V1024 = call i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call i1 @llvm.vector.reduce.and.nxv1i1(<vscale x 1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call i1 @llvm.vector.reduce.and.nxv4i1(<vscale x 4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call i1 @llvm.vector.reduce.and.nxv8i1(<vscale x 8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV32 = call i1 @llvm.vector.reduce.and.nxv32i1(<vscale x 32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV64 = call i1 @llvm.vector.reduce.and.nxv64i1(<vscale x 64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV128 = call i1 @llvm.vector.reduce.and.nxv128i1(<vscale x 128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV256 = call i1 @llvm.vector.reduce.and.nxv256i1(<vscale x 256 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV512 = call i1 @llvm.vector.reduce.and.nxv512i1(<vscale x 512 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV1024 = call i1 @llvm.vector.reduce.and.nxv1024i1(<vscale x 1024 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i1'
@@ -31,6 +42,17 @@ define i32 @reduce_i1(i32 %arg) {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512 = call i1 @llvm.vector.reduce.and.v512i1(<512 x i1> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V1024 = call i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV1 = call i1 @llvm.vector.reduce.and.nxv1i1(<vscale x 1 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV2 = call i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV4 = call i1 @llvm.vector.reduce.and.nxv4i1(<vscale x 4 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV8 = call i1 @llvm.vector.reduce.and.nxv8i1(<vscale x 8 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV16 = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV32 = call i1 @llvm.vector.reduce.and.nxv32i1(<vscale x 32 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %NXV64 = call i1 @llvm.vector.reduce.and.nxv64i1(<vscale x 64 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV128 = call i1 @llvm.vector.reduce.and.nxv128i1(<vscale x 128 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %NXV256 = call i1 @llvm.vector.reduce.and.nxv256i1(<vscale x 256 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NXV512 = call i1 @llvm.vector.reduce.and.nxv512i1(<vscale x 512 x i1> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NXV1024 = call i1 @llvm.vector.reduce.and.nxv1024i1(<vscale x 1024 x i1> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
@@ -44,30 +66,42 @@ define i32 @reduce_i1(i32 %arg) {
   %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
   %V512 = call i1 @llvm.vector.reduce.and.v512i1(<512 x i1> undef)
   %V1024 = call i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1> undef)
+
+  %NXV1   = call i1 @llvm.vector.reduce.and.nxv1i1(<vscale x 1 x i1> undef)
+  %NXV2   = call i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1> undef)
+  %NXV4   = call i1 @llvm.vector.reduce.and.nxv4i1(<vscale x 4 x i1> undef)
+  %NXV8   = call i1 @llvm.vector.reduce.and.nxv8i1(<vscale x 8 x i1> undef)
+  %NXV16  = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> undef)
+  %NXV32  = call i1 @llvm.vector.reduce.and.nxv32i1(<vscale x 32 x i1> undef)
+  %NXV64  = call i1 @llvm.vector.reduce.and.nxv64i1(<vscale x 64 x i1> undef)
+  %NXV128 = call i1 @llvm.vector.reduce.and.nxv128i1(<vscale x 128 x i1> undef)
+  %NXV256 = call i1 @llvm.vector.reduce.and.nxv256i1(<vscale x 256 x i1> undef)
+  %NXV512 = call i1 @llvm.vector.reduce.and.nxv512i1(<vscale x 512 x i1> undef)
+  %NXV1024 = call i1 @llvm.vector.reduce.and.nxv1024i1(<vscale x 1024 x i1> undef)
   ret i32 undef
 }
 
 define i32 @reduce_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
@@ -83,25 +117,25 @@ define i32 @reduce_i8(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.and.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.and.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.and.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.and.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.and.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.and.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.and.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.and.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.and.v1i16(<1 x i16> undef)
@@ -117,25 +151,25 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.and.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.and.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.and.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.and.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.and.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.and.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.and.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.and.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.and.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.and.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.and.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.and.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.and.v1i32(<1 x i32> undef)
@@ -151,25 +185,25 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.and.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.and.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.and.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.and.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.and.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.and.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.and.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.and.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.and.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.and.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.and.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.and.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
index c5d677e574c13..f11e9f2b5ae83 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
@@ -40,25 +40,25 @@ define i32 @reduce_umin_i1(i32 %arg) {
 
 define i32 @reduce_umax_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umax_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umax_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.umax.v1i8(<1 x i8> undef)
@@ -74,25 +74,25 @@ define i32 @reduce_umax_i8(i32 %arg) {
 
 define i32 @reduce_umax_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umax_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umax_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.umax.v1i16(<1 x i16> undef)
@@ -108,25 +108,25 @@ define i32 @reduce_umax_i16(i32 %arg) {
 
 define i32 @reduce_umax_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umax_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umax_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.umax.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.umax.v1i32(<1 x i32> undef)
@@ -142,25 +142,25 @@ define i32 @reduce_umax_i32(i32 %arg) {
 
 define i32 @reduce_umax_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umax_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umax_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.umax.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.umax.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
@@ -210,25 +210,25 @@ define i32 @reduce_smin_i1(i32 %arg) {
 
 define i32 @reduce_smax_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smax_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smax_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.smax.v1i8(<1 x i8> undef)
@@ -244,25 +244,25 @@ define i32 @reduce_smax_i8(i32 %arg) {
 
 define i32 @reduce_smax_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smax_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smax_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.smax.v1i16(<1 x i16> undef)
@@ -278,25 +278,25 @@ define i32 @reduce_smax_i16(i32 %arg) {
 
 define i32 @reduce_smax_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smax_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smax_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.smax.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.smax.v1i32(<1 x i32> undef)
@@ -312,25 +312,25 @@ define i32 @reduce_smax_i32(i32 %arg) {
 
 define i32 @reduce_smax_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smax_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smax_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.smax.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
index 89bff38138415..457fdbe46f73b 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
@@ -40,25 +40,25 @@ define i32 @reduce_umin_i1(i32 %arg) {
 
 define i32 @reduce_umin_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umin_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umin_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.umin.v1i8(<1 x i8> undef)
@@ -74,25 +74,25 @@ define i32 @reduce_umin_i8(i32 %arg) {
 
 define i32 @reduce_umin_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umin_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umin_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.umin.v1i16(<1 x i16> undef)
@@ -108,25 +108,25 @@ define i32 @reduce_umin_i16(i32 %arg) {
 
 define i32 @reduce_umin_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umin_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umin_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.umin.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.umin.v1i32(<1 x i32> undef)
@@ -142,25 +142,25 @@ define i32 @reduce_umin_i32(i32 %arg) {
 
 define i32 @reduce_umin_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_umin_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_umin_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.umin.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.umin.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
@@ -210,25 +210,25 @@ define i32 @reduce_smin_i1(i32 %arg) {
 
 define i32 @reduce_smin_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smin_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smin_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.smin.v1i8(<1 x i8> undef)
@@ -244,25 +244,25 @@ define i32 @reduce_smin_i8(i32 %arg) {
 
 define i32 @reduce_smin_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smin_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smin_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.smin.v1i16(<1 x i16> undef)
@@ -278,25 +278,25 @@ define i32 @reduce_smin_i16(i32 %arg) {
 
 define i32 @reduce_smin_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smin_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smin_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.smin.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.smin.v1i32(<1 x i32> undef)
@@ -312,25 +312,25 @@ define i32 @reduce_smin_i32(i32 %arg) {
 
 define i32 @reduce_smin_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_smin_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_smin_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.smin.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
index d75a95f3fadd9..b0e90253b2810 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
@@ -49,25 +49,25 @@ define i32 @reduce_i1(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
@@ -83,25 +83,25 @@ define i32 @reduce_i8(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.or.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call i16 @llvm.vector.reduce.or.v128i16(<128 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i16 @llvm.vector.reduce.or.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V128 = call i16 @llvm.vector.reduce.or.v128i16(<128 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i16 @llvm.vector.reduce.or.v1i16(<1 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128 = call i16 @llvm.vector.reduce.or.v128i16(<128 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i16 @llvm.vector.reduce.or.v1i16(<1 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call i16 @llvm.vector.reduce.or.v128i16(<128 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i16 @llvm.vector.reduce.or.v1i16(<1 x i16> undef)
@@ -117,25 +117,25 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.or.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64 = call i32 @llvm.vector.reduce.or.v64i32(<64 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V128 = call i32 @llvm.vector.reduce.or.v128i32(<128 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i32 @llvm.vector.reduce.or.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64 = call i32 @llvm.vector.reduce.or.v64i32(<64 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V128 = call i32 @llvm.vector.reduce.or.v128i32(<128 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i32 @llvm.vector.reduce.or.v1i32(<1 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64 = call i32 @llvm.vector.reduce.or.v64i32(<64 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V128 = call i32 @llvm.vector.reduce.or.v128i32(<128 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i32 @llvm.vector.reduce.or.v1i32(<1 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call i32 @llvm.vector.reduce.or.v64i32(<64 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = call i32 @llvm.vector.reduce.or.v128i32(<128 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i32 @llvm.vector.reduce.or.v1i32(<1 x i32> undef)
@@ -151,25 +151,25 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i64(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i64 @llvm.vector.reduce.or.v32i64(<32 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V64 = call i64 @llvm.vector.reduce.or.v64i64(<64 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V128 = call i64 @llvm.vector.reduce.or.v128i64(<128 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i64 @llvm.vector.reduce.or.v32i64(<32 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call i64 @llvm.vector.reduce.or.v64i64(<64 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V128 = call i64 @llvm.vector.reduce.or.v128i64(<128 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIZE-LABEL: 'reduce_i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i64 @llvm.vector.reduce.or.v32i64(<32 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64 = call i64 @llvm.vector.reduce.or.v64i64(<64 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call i64 @llvm.vector.reduce.or.v128i64(<128 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i64 @llvm.vector.reduce.or.v32i64(<32 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i64 @llvm.vector.reduce.or.v64i64(<64 x i64> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i64 @llvm.vector.reduce.or.v128i64(<128 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
index 4f6e0ba074ed8..8ae387f48ccaa 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
@@ -452,11 +452,11 @@ declare half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half>)
 
 define half @vreduce_fmin_nxv1f16(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
@@ -465,11 +465,11 @@ define half @vreduce_fmin_nxv1f16(<vscale x 1 x half> %v) {
 
 define half @vreduce_fmin_nxv1f16_nonans(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f16_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f16_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call nnan half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
@@ -478,11 +478,11 @@ define half @vreduce_fmin_nxv1f16_nonans(<vscale x 1 x half> %v) {
 
 define half @vreduce_fmin_nxv1f16_nonans_noinfs(<vscale x 1 x half> %v) #1 {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f16_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f16_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call nnan ninf half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
@@ -493,11 +493,11 @@ declare half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half>)
 
 define half @vreduce_fmin_nxv2f16(<vscale x 2 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv2f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
@@ -508,11 +508,11 @@ declare half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half>)
 
 define half @vreduce_fmin_nxv4f16(<vscale x 4 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv4f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
@@ -523,11 +523,11 @@ declare half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half>)
 
 define half @vreduce_fmin_nxv64f16(<vscale x 64 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv64f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv64f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
@@ -538,11 +538,11 @@ declare float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float>)
 
 define float @vreduce_fmin_nxv1f32(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
@@ -551,11 +551,11 @@ define float @vreduce_fmin_nxv1f32(<vscale x 1 x float> %v) {
 
 define float @vreduce_fmin_nxv1f32_nonans(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f32_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f32_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call nnan float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
@@ -564,11 +564,11 @@ define float @vreduce_fmin_nxv1f32_nonans(<vscale x 1 x float> %v) {
 
 define float @vreduce_fmin_nxv1f32_nonans_noinfs(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f32_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f32_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call nnan ninf float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
@@ -579,11 +579,11 @@ declare float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float>)
 
 define float @vreduce_fmin_nxv2f32(<vscale x 2 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv2f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
@@ -594,11 +594,11 @@ declare float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float>)
 
 define float @vreduce_fmin_nxv4f32(<vscale x 4 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv4f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
@@ -609,11 +609,11 @@ declare float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float>)
 
 define float @vreduce_fmin_nxv32f32(<vscale x 32 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv32f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv32f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
@@ -624,11 +624,11 @@ declare double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double>)
 
 define double @vreduce_fmin_nxv1f64(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
@@ -637,11 +637,11 @@ define double @vreduce_fmin_nxv1f64(<vscale x 1 x double> %v) {
 
 define double @vreduce_fmin_nxv1f64_nonans(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f64_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f64_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call nnan double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
@@ -650,11 +650,11 @@ define double @vreduce_fmin_nxv1f64_nonans(<vscale x 1 x double> %v) {
 
 define double @vreduce_fmin_nxv1f64_nonans_noinfs(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f64_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv1f64_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call nnan ninf double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
@@ -665,11 +665,11 @@ declare double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double>)
 
 define double @vreduce_fmin_nxv2f64(<vscale x 2 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv2f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
@@ -680,11 +680,11 @@ declare double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double>)
 
 define double @vreduce_fmin_nxv4f64(<vscale x 4 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv4f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
@@ -695,11 +695,11 @@ declare double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double>)
 
 define double @vreduce_fmin_nxv16f64(<vscale x 16 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv16f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmin_nxv16f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
@@ -710,11 +710,11 @@ declare half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half>)
 
 define half @vreduce_fmax_nxv1f16(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
@@ -723,11 +723,11 @@ define half @vreduce_fmax_nxv1f16(<vscale x 1 x half> %v) {
 
 define half @vreduce_fmax_nxv1f16_nonans(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f16_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f16_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call nnan half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
@@ -736,11 +736,11 @@ define half @vreduce_fmax_nxv1f16_nonans(<vscale x 1 x half> %v) {
 
 define half @vreduce_fmax_nxv1f16_nonans_noinfs(<vscale x 1 x half> %v) #1 {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f16_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f16_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call nnan ninf half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
@@ -751,11 +751,11 @@ declare half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half>)
 
 define half @vreduce_fmax_nxv2f16(<vscale x 2 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv2f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
@@ -766,11 +766,11 @@ declare half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half>)
 
 define half @vreduce_fmax_nxv4f16(<vscale x 4 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv4f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
@@ -781,11 +781,11 @@ declare half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half>)
 
 define half @vreduce_fmax_nxv64f16(<vscale x 64 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv64f16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv64f16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret half %red
 ;
   %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
@@ -796,11 +796,11 @@ declare float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float>)
 
 define float @vreduce_fmax_nxv1f32(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
@@ -809,11 +809,11 @@ define float @vreduce_fmax_nxv1f32(<vscale x 1 x float> %v) {
 
 define float @vreduce_fmax_nxv1f32_nonans(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f32_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f32_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call nnan float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
@@ -822,11 +822,11 @@ define float @vreduce_fmax_nxv1f32_nonans(<vscale x 1 x float> %v) {
 
 define float @vreduce_fmax_nxv1f32_nonans_noinfs(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f32_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f32_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call nnan ninf float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
@@ -837,11 +837,11 @@ declare float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float>)
 
 define float @vreduce_fmax_nxv2f32(<vscale x 2 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv2f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
@@ -852,11 +852,11 @@ declare float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float>)
 
 define float @vreduce_fmax_nxv4f32(<vscale x 4 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv4f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
@@ -867,11 +867,11 @@ declare float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float>)
 
 define float @vreduce_fmax_nxv32f32(<vscale x 32 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv32f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv32f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %red
 ;
   %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
@@ -882,11 +882,11 @@ declare double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double>)
 
 define double @vreduce_fmax_nxv1f64(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
@@ -895,11 +895,11 @@ define double @vreduce_fmax_nxv1f64(<vscale x 1 x double> %v) {
 
 define double @vreduce_fmax_nxv1f64_nonans(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f64_nonans'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f64_nonans'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call nnan double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
@@ -908,11 +908,11 @@ define double @vreduce_fmax_nxv1f64_nonans(<vscale x 1 x double> %v) {
 
 define double @vreduce_fmax_nxv1f64_nonans_noinfs(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f64_nonans_noinfs'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv1f64_nonans_noinfs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call nnan ninf double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
@@ -923,11 +923,11 @@ declare double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double>)
 
 define double @vreduce_fmax_nxv2f64(<vscale x 2 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv2f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
@@ -938,11 +938,11 @@ declare double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double>)
 
 define double @vreduce_fmax_nxv4f64(<vscale x 4 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv4f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
@@ -953,11 +953,11 @@ declare double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double>)
 
 define double @vreduce_fmax_nxv16f64(<vscale x 16 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv16f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %red
 ;
 ; SIZE-LABEL: 'vreduce_fmax_nxv16f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double %red
 ;
   %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
index 2807f7526760f..a1a22b56358a3 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
@@ -23,11 +23,11 @@ declare i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_umax_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
@@ -38,11 +38,11 @@ declare i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_smax_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
@@ -53,11 +53,11 @@ declare i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_umin_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
@@ -68,11 +68,11 @@ declare i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_smin_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
@@ -83,11 +83,11 @@ declare i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_and_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
@@ -98,11 +98,11 @@ declare i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8>)
 
 define signext i8 @vreduce_or_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv1i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
@@ -143,11 +143,11 @@ declare i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_umax_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
@@ -158,11 +158,11 @@ declare i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_smax_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
@@ -173,11 +173,11 @@ declare i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_umin_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
@@ -188,11 +188,11 @@ declare i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_smin_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
@@ -203,11 +203,11 @@ declare i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_and_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
@@ -218,11 +218,11 @@ declare i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8>)
 
 define signext i8 @vreduce_or_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv2i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
@@ -263,11 +263,11 @@ declare i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_umax_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
@@ -278,11 +278,11 @@ declare i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_smax_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
@@ -293,11 +293,11 @@ declare i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_umin_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
@@ -308,11 +308,11 @@ declare i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_smin_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
@@ -323,11 +323,11 @@ declare i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_and_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
@@ -338,11 +338,11 @@ declare i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8>)
 
 define signext i8 @vreduce_or_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv4i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %red
 ;
   %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
@@ -415,11 +415,11 @@ declare i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_umax_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
@@ -430,11 +430,11 @@ declare i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_smax_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
@@ -445,11 +445,11 @@ declare i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_umin_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
@@ -460,11 +460,11 @@ declare i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_smin_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
@@ -475,11 +475,11 @@ declare i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_and_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
@@ -490,11 +490,11 @@ declare i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16>)
 
 define signext i16 @vreduce_or_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv1i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
@@ -567,11 +567,11 @@ declare i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_umax_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
@@ -582,11 +582,11 @@ declare i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_smax_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
@@ -597,11 +597,11 @@ declare i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_umin_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
@@ -612,11 +612,11 @@ declare i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_smin_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
@@ -627,11 +627,11 @@ declare i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_and_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
@@ -642,11 +642,11 @@ declare i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16>)
 
 define signext i16 @vreduce_or_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv2i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
@@ -719,11 +719,11 @@ declare i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_umax_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
@@ -734,11 +734,11 @@ declare i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_smax_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
@@ -749,11 +749,11 @@ declare i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_umin_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
@@ -764,11 +764,11 @@ declare i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_smin_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
@@ -779,11 +779,11 @@ declare i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_and_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
@@ -794,11 +794,11 @@ declare i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16>)
 
 define signext i16 @vreduce_or_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv4i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %red
 ;
   %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
@@ -871,11 +871,11 @@ declare i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_umax_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
@@ -886,11 +886,11 @@ declare i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_smax_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
@@ -901,11 +901,11 @@ declare i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_umin_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
@@ -916,11 +916,11 @@ declare i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_smin_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
@@ -931,11 +931,11 @@ declare i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_and_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
@@ -946,11 +946,11 @@ declare i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32>)
 
 define signext i32 @vreduce_or_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv1i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
@@ -1023,11 +1023,11 @@ declare i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_umax_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
@@ -1038,11 +1038,11 @@ declare i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_smax_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
@@ -1053,11 +1053,11 @@ declare i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_umin_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
@@ -1068,11 +1068,11 @@ declare i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_smin_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
@@ -1083,11 +1083,11 @@ declare i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_and_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
@@ -1098,11 +1098,11 @@ declare i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32>)
 
 define signext i32 @vreduce_or_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv2i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
@@ -1175,11 +1175,11 @@ declare i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_umax_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
@@ -1190,11 +1190,11 @@ declare i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_smax_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
@@ -1205,11 +1205,11 @@ declare i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_umin_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
@@ -1220,11 +1220,11 @@ declare i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_smin_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
@@ -1235,11 +1235,11 @@ declare i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_and_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
@@ -1250,11 +1250,11 @@ declare i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32>)
 
 define signext i32 @vreduce_or_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv4i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %red
 ;
   %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
@@ -1327,11 +1327,11 @@ declare i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_umax_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
@@ -1342,11 +1342,11 @@ declare i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_smax_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
@@ -1357,11 +1357,11 @@ declare i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_umin_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
@@ -1372,11 +1372,11 @@ declare i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_smin_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
@@ -1387,11 +1387,11 @@ declare i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_and_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
@@ -1402,11 +1402,11 @@ declare i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64>)
 
 define i64 @vreduce_or_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv1i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
@@ -1479,11 +1479,11 @@ declare i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_umax_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
@@ -1494,11 +1494,11 @@ declare i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_smax_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
@@ -1509,11 +1509,11 @@ declare i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_umin_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
@@ -1524,11 +1524,11 @@ declare i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_smin_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
@@ -1539,11 +1539,11 @@ declare i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_and_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
@@ -1554,11 +1554,11 @@ declare i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64>)
 
 define i64 @vreduce_or_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv2i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
@@ -1631,11 +1631,11 @@ declare i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_umax_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umax_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
@@ -1646,11 +1646,11 @@ declare i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_smax_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smax_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
@@ -1661,11 +1661,11 @@ declare i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_umin_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_umin_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
@@ -1676,11 +1676,11 @@ declare i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_smin_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_smin_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
@@ -1691,11 +1691,11 @@ declare i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_and_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_and_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
@@ -1706,11 +1706,11 @@ declare i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64>)
 
 define i64 @vreduce_or_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
 ; SIZE-LABEL: 'vreduce_or_nxv4i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %red
 ;
   %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
index c7cd845a0a03f..5126a6a0a3cbc 100644
--- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
@@ -1822,35 +1822,35 @@ define void @reduce_fadd() {
 define void @reduce_other() {
 ; CHECK-LABEL: 'reduce_other'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call i32 @llvm.vp.reduce.xor.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call i32 @llvm.vp.reduce.or.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = call i32 @llvm.vp.reduce.or.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %4 = call i32 @llvm.vp.reduce.mul.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = call i32 @llvm.vp.reduce.smin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = call i32 @llvm.vp.reduce.smax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %7 = call i32 @llvm.vp.reduce.umin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %8 = call i32 @llvm.vp.reduce.umax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %5 = call i32 @llvm.vp.reduce.smin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %6 = call i32 @llvm.vp.reduce.smax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = call i32 @llvm.vp.reduce.umin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = call i32 @llvm.vp.reduce.umax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call i32 @llvm.vp.reduce.mul.nxv4i32(i32 undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call float @llvm.vp.reduce.fmul.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %11 = call float @llvm.vp.reduce.fmin.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call float @llvm.vp.reduce.fmin.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call float @llvm.vp.reduce.fminimum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call float @llvm.vp.reduce.fmaximum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPEBASED-LABEL: 'reduce_other'
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call i32 @llvm.vp.reduce.xor.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call i32 @llvm.vp.reduce.or.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call i32 @llvm.vp.reduce.and.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = call i32 @llvm.vp.reduce.or.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %4 = call i32 @llvm.vp.reduce.mul.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = call i32 @llvm.vp.reduce.smin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = call i32 @llvm.vp.reduce.smax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %7 = call i32 @llvm.vp.reduce.umin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %8 = call i32 @llvm.vp.reduce.umax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %5 = call i32 @llvm.vp.reduce.smin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %6 = call i32 @llvm.vp.reduce.smax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = call i32 @llvm.vp.reduce.umin.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = call i32 @llvm.vp.reduce.umax.v4i32(i32 undef, <4 x i32> undef, <4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %9 = call i32 @llvm.vp.reduce.mul.nxv4i32(i32 undef, <vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Invalid cost for instruction: %10 = call float @llvm.vp.reduce.fmul.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %11 = call float @llvm.vp.reduce.fmin.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call float @llvm.vp.reduce.fmin.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %12 = call float @llvm.vp.reduce.fminimum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = call float @llvm.vp.reduce.fmax.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %14 = call float @llvm.vp.reduce.fmaximum.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
 ; TYPEBASED-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll b/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
deleted file mode 100644
index 90b5b746c914a..0000000000000
--- a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: opt < %s -mtriple=systemz-unknown -mcpu=z13 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
-
-define void @reduce(ptr %src, ptr %dst) {
-; CHECK-LABEL: 'reduce'
-; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %V2_64)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %V4_64)
-; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %R8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %V8_64)
-; CHECK:  Cost Model: Found an estimated cost of 9 for instruction: %R16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %V16_64)
-; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %V2_32)
-; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
-; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
-; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
-; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
-;
-; CHECK:  Cost Model: Found an estimated cost of 10 for instruction: %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
-; CHECK:  Cost Model: Found an estimated cost of 20 for instruction: %R4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> %V4_256)
-
-  ; REDUCEADD64
-
-  %V2_64 = load <2 x i64>, ptr %src, align 8
-  %R2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %V2_64)
-  store volatile i64 %R2_64, ptr %dst, align 4
-
-  %V4_64 = load <4 x i64>, ptr %src, align 8
-  %R4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %V4_64)
-  store volatile i64 %R4_64, ptr %dst, align 4
-
-  %V8_64 = load <8 x i64>, ptr %src, align 8
-  %R8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %V8_64)
-  store volatile i64 %R8_64, ptr %dst, align 4
-
-  %V16_64 = load <16 x i64>, ptr %src, align 8
-  %R16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %V16_64)
-  store volatile i64 %R16_64, ptr %dst, align 4
-
-  ; REDUCEADD32
-
-  %V2_32 = load <2 x i32>, ptr %src, align 8
-  %R2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %V2_32)
-  store volatile i32 %R2_32, ptr %dst, align 4
-
-  %V4_32 = load <4 x i32>, ptr %src, align 8
-  %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
-  store volatile i32 %R4_32, ptr %dst, align 4
-
-  %V8_32 = load <8 x i32>, ptr %src, align 8
-  %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
-  store volatile i32 %R8_32, ptr %dst, align 4
-
-  %V16_32 = load <16 x i32>, ptr %src, align 8
-  %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
-  store volatile i32 %R16_32, ptr %dst, align 4
-
-  ; REDUCEADD16
-
-  %V2_16 = load <2 x i16>, ptr %src, align 8
-  %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
-  store volatile i16 %R2_16, ptr %dst, align 4
-
-  %V4_16 = load <4 x i16>, ptr %src, align 8
-  %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
-  store volatile i16 %R4_16, ptr %dst, align 4
-
-  %V8_16 = load <8 x i16>, ptr %src, align 8
-  %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
-  store volatile i16 %R8_16, ptr %dst, align 4
-
-  %V16_16 = load <16 x i16>, ptr %src, align 8
-  %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
-  store volatile i16 %R16_16, ptr %dst, align 4
-
-  ; REDUCEADD8
-
-  %V2_8 = load <2 x i8>, ptr %src, align 8
-  %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
-  store volatile i8 %R2_8, ptr %dst, align 4
-
-  %V4_8 = load <4 x i8>, ptr %src, align 8
-  %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
-  store volatile i8 %R4_8, ptr %dst, align 4
-
-  %V8_8 = load <8 x i8>, ptr %src, align 8
-  %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
-  store volatile i8 %R8_8, ptr %dst, align 4
-
-  %V16_8 = load <16 x i8>, ptr %src, align 8
-  %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
-  store volatile i8 %R16_8, ptr %dst, align 4
-
-  ; EXTREME VALUES
-
-  %V128_8 = load <128 x i8>, ptr %src, align 8
-  %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
-  store volatile i8 %R128_8, ptr %dst, align 4
-
-  %V4_256 = load <4 x i256>, ptr %src, align 8
-  %R4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> %V4_256)
-  store volatile i256 %R4_256, ptr %dst, align 8
-
-  ret void
-}
-
-declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
-declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
-declare i16 @llvm.vector.reduce.add.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
-declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
-
-declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
-declare i256 @llvm.vector.reduce.add.v4i256(<4 x i256>)
diff --git a/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll b/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll
new file mode 100644
index 0000000000000..0def20215e988
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/SystemZ/vector-reductions.ll
@@ -0,0 +1,376 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes='print<cost-model>' -disable-output -mtriple=s390x-unknown-linux \
+; RUN:   -mcpu=z15 < %s 2>&1 | FileCheck %s --check-prefix=Z15
+
+define void @fadd_reductions() {
+; Z15-LABEL: 'fadd_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.0, <4 x float> undef)
+  %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.0, <8 x float> undef)
+  %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.0, <2 x double> undef)
+  %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.0, <4 x double> undef)
+  %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+  ret void
+}
+
+define void @fast_fadd_reductions(ptr %src, ptr %dst) {
+; Z15-LABEL: 'fast_fadd_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.0, <4 x float> undef)
+  %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.0, <8 x float> undef)
+  %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.0, <2 x double> undef)
+  %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.0, <4 x double> undef)
+  %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+  ret void
+}
+
+define void @fmul_reductions() {
+; Z15-LABEL: 'fmul_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f32 = call float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %fmul_v8f32 = call float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmul_v2f64 = call double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f64 = call double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fmul_v4f128 = call fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %fmul_v4f32 = call float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+  %fmul_v8f32 = call float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+  %fmul_v2f64 = call double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+  %fmul_v4f64 = call double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+  %fmul_v4f128 = call fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+  ret void
+}
+
+define void @fast_fmul_reductions() {
+; Z15-LABEL: 'fast_fmul_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fmul_v4f32 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fmul_v8f32 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fmul_v2f64 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmul_v4f64 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %fmul_v4f32 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+  %fmul_v8f32 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+  %fmul_v2f64 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+  %fmul_v4f64 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+  %fadd_v4f128 = call fast fp128 @llvm.vector.reduce.fmul.v4f128(fp128 undef, <4 x fp128> undef)
+
+  ret void
+}
+
+define void @fmin_reductions() {
+; Z15-LABEL: 'fmin_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4f32 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8f32 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2f64 = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f64 = call double @llvm.vector.reduce.fmin.v4f64(<4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f128 = call fp128 @llvm.vector.reduce.fmin.v4f128(<4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V4f32 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> undef)
+  %V8f32 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> undef)
+  %V2f64 = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> undef)
+  %V4f64 = call double @llvm.vector.reduce.fmin.v4f64(<4 x double> undef)
+  %V4f128 = call fp128 @llvm.vector.reduce.fmin.v4f128(<4 x fp128> undef)
+  ret void
+}
+
+define void @fmax_reductions() {
+; Z15-LABEL: 'fmax_reductions'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4f32 = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8f32 = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2f64 = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f64 = call double @llvm.vector.reduce.fmax.v4f64(<4 x double> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f128 = call fp128 @llvm.vector.reduce.fmax.v4f128(<4 x fp128> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V4f32 = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> undef)
+  %V8f32 = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> undef)
+  %V2f64 = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> undef)
+  %V4f64 = call double @llvm.vector.reduce.fmax.v4f64(<4 x double> undef)
+  %V4f128 = call fp128 @llvm.vector.reduce.fmax.v4f128(<4 x fp128> undef)
+  ret void
+}
+
+define void @reduceumin() {
+; Z15-LABEL: 'reduceumin'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 6 for instruction: %V4_32 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 7 for instruction: %V8_32 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 37 for instruction: %V128_8 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_128 = call i128 @llvm.vector.reduce.umin.v4i128(<4 x i128> undef)
+;
+  %V2_64 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+
+  %V128_8 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+  %V4_128 = call i128 @llvm.vector.reduce.umin.v4i128(<4 x i128> undef)
+
+  ret void
+}
+
+define void @reduceumax() {
+; Z15-LABEL: 'reduceumax'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 6 for instruction: %V4_32 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 7 for instruction: %V8_32 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 37 for instruction: %V128_8 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_128 = call i128 @llvm.vector.reduce.umax.v4i128(<4 x i128> undef)
+;
+  %V2_64 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+
+  %V128_8 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+  %V4_128 = call i128 @llvm.vector.reduce.umax.v4i128(<4 x i128> undef)
+
+  ret void
+}
+
+define void @reducesmin() {
+; Z15-LABEL: 'reducesmin'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 6 for instruction: %V4_32 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 7 for instruction: %V8_32 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 37 for instruction: %V128_8 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_128 = call i128 @llvm.vector.reduce.smin.v4i128(<4 x i128> undef)
+;
+  %V2_64 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+
+  %V128_8 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+  %V4_128 = call i128 @llvm.vector.reduce.smin.v4i128(<4 x i128> undef)
+
+  ret void
+}
+
+define void @reducesmax() {
+; Z15-LABEL: 'reducesmax'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 6 for instruction: %V4_32 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 7 for instruction: %V8_32 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 37 for instruction: %V128_8 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; Z15-NEXT  Cost Model: Found an estimated cost of 3 for instruction: %V4_128 = call i128 @llvm.vector.reduce.smax.v4i128(<4 x i128> undef)
+;
+  %V2_64 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+
+  %V128_8 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+  %V4_128 = call i128 @llvm.vector.reduce.smax.v4i128(<4 x i128> undef)
+
+  ret void
+}
+
+define void @reduceadd() {
+; Z15-LABEL: 'reduceadd'
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
+;
+; Z15-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> undef)
+; Z15-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> undef)
+
+  ; REDUCEADD64
+  %V2_64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+  %V8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+  %V16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
+  ; REDUCEADD32
+  %V2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
+  %V16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
+  ; REDUCEADD16
+  %V2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
+  %V4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
+  %V8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
+  %V16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
+  ; REDUCEADD8
+  %V2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
+  %V4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
+  %V8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
+  %V16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
+  ; EXTREME VALUES
+  %V128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> undef)
+  %V4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> undef)
+
+  ret void
+}
+
+define void @reducemul() {
+; CHECK-LABEL: 'reducemul'
+; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %V2_64 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> undef)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %V4_64 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> undef)
+; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %V8_64 = call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> undef)
+; CHECK:  Cost Model: Found an estimated cost of 9 for instruction: %V16_64 = call i64 @llvm.vector.reduce.mul.v16i64(<16 x i64> undef)
+; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %V2_32 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> undef)
+; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %V4_32 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> undef)
+; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %V8_32 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> undef)
+; CHECK:  Cost Model: Found an estimated cost of 7 for instruction: %V16_32 = call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> undef)
+; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %V2_16 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> undef)
+; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %V4_16 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> undef)
+; CHECK:  Cost Model: Found an estimated cost of 6 for instruction: %V8_16 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> undef)
+; CHECK:  Cost Model: Found an estimated cost of 7 for instruction: %V16_16 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> undef)
+; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %V2_8 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> undef)
+; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %V4_8 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> undef)
+; CHECK:  Cost Model: Found an estimated cost of 6 for instruction: %V8_8 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> undef)
+; CHECK:  Cost Model: Found an estimated cost of 8 for instruction: %V16_8 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> undef)
+;
+; CHECK:  Cost Model: Found an estimated cost of 15 for instruction: %V128_8 = call i8 @llvm.vector.reduce.mul.v128i8(<128 x i8> undef)
+; CHECK:  Cost Model: Found an estimated cost of 28 for instruction: %V4_256 = call i256 @llvm.vector.reduce.mul.v4i256(<4 x i256> undef)
+
+  ; REDUCEADD64
+  %V2_64 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> undef)
+  %V4_64 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> undef)
+  %V8_64 = call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> undef)
+  %V16_64 = call i64 @llvm.vector.reduce.mul.v16i64(<16 x i64> undef)
+  ; REDUCEADD32
+  %V2_32 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> undef)
+  %V4_32 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> undef)
+  %V8_32 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> undef)
+  %V16_32 = call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> undef)
+  ; REDUCEADD16
+  %V2_16 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> undef)
+  %V4_16 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> undef)
+  %V8_16 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> undef)
+  %V16_16 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> undef)
+  ; REDUCEADD8
+  %V2_8 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> undef)
+  %V4_8 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> undef)
+  %V8_8 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> undef)
+  %V16_8 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> undef)
+  ; EXTREME VALUES
+  %V128_8 = call i8 @llvm.vector.reduce.mul.v128i8(<128 x i8> undef)
+  %V4_256 = call i256 @llvm.vector.reduce.mul.v4i256(<4 x i256> undef)
+
+  ret void
+}
+
+declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fadd.v8f32(float, <8 x float>)
+declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)
+declare double @llvm.vector.reduce.fadd.v4f64(double, <4 x double>)
+declare fp128 @llvm.vector.reduce.fadd.v4f128(fp128, <4 x fp128>)
+
+declare float @llvm.vector.reduce.fmul.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fmul.v8f32(float, <8 x float>)
+declare double @llvm.vector.reduce.fmul.v2f64(double, <2 x double>)
+declare double @llvm.vector.reduce.fmul.v4f64(double, <4 x double>)
+declare fp128 @llvm.vector.reduce.fmul.v4f128(fp128, <4 x fp128>)
+
+declare float @llvm.vector.reduce.fmin.v4f32(<4 x float>)
+declare float @llvm.vector.reduce.fmin.v8f32(<8 x float>)
+declare double @llvm.vector.reduce.fmin.v2f64(<2 x double>)
+declare double @llvm.vector.reduce.fmin.v4f64(<4 x double>)
+declare fp128 @llvm.vector.reduce.fmin.v4f128(<4 x fp128>)
+
+declare float @llvm.vector.reduce.fmax.v4f32(<4 x float>)
+declare float @llvm.vector.reduce.fmax.v8f32(<8 x float>)
+declare double @llvm.vector.reduce.fmax.v2f64(<2 x double>)
+declare double @llvm.vector.reduce.fmax.v4f64(<4 x double>)
+declare fp128 @llvm.vector.reduce.fmax.v4f128(<4 x fp128>)
+
+declare i64 @llvm.vector.reduce.umin.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.umin.v4i64(<4 x i64>)
+declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.umin.v8i32(<8 x i32>)
+declare i8 @llvm.vector.reduce.umin.v128i8(<128 x i8>)
+declare i128 @llvm.vector.reduce.umin.v4i128(<4 x i128>)
+
+declare i64 @llvm.vector.reduce.umax.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.umax.v4i64(<4 x i64>)
+declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.umax.v8i32(<8 x i32>)
+declare i8 @llvm.vector.reduce.umax.v128i8(<128 x i8>)
+declare i128 @llvm.vector.reduce.umax.v4i128(<4 x i128>)
+
+declare i64 @llvm.vector.reduce.smin.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.smin.v4i64(<4 x i64>)
+declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.smin.v8i32(<8 x i32>)
+declare i8 @llvm.vector.reduce.smin.v128i8(<128 x i8>)
+declare i128 @llvm.vector.reduce.smin.v4i128(<4 x i128>)
+
+declare i64 @llvm.vector.reduce.smax.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.smax.v4i64(<4 x i64>)
+declare i32 @llvm.vector.reduce.smax.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.smax.v8i32(<8 x i32>)
+declare i8 @llvm.vector.reduce.smax.v128i8(<128 x i8>)
+declare i128 @llvm.vector.reduce.smax.v4i128(<4 x i128>)
+
+declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
+declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
+declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)
+declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
+declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
+declare i16 @llvm.vector.reduce.add.v2i16(<2 x i16>)
+declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
+declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
+declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
+declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
+declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
+
+declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
+declare i256 @llvm.vector.reduce.add.v4i256(<4 x i256>)
+
+declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64>)
+declare i64 @llvm.vector.reduce.mul.v16i64(<16 x i64>)
+declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)
+declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
+declare i32 @llvm.vector.reduce.mul.v16i32(<16 x i32>)
+declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)
+declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)
+declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)
+declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
+declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)
+declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)
+
+declare i8 @llvm.vector.reduce.mul.v128i8(<128 x i8>)
+declare i256 @llvm.vector.reduce.mul.v4i256(<4 x i256>)
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll
index 61d99c20fa966..c78023e24572c 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll
@@ -18,24 +18,24 @@
 
 define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -47,24 +47,24 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5
 
 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -76,31 +76,31 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x
 
 define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) {
 ; SSE-LABEL: 'test_vXf32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -112,31 +112,31 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512
 
 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSE-LABEL: 'test_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -150,36 +150,36 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8
 ; AVX1-LABEL: 'test_vXi16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi16'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -193,36 +193,36 @@ define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x
 ; AVX1-LABEL: 'test_vXi8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi8'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll
index 0f8503cd1c3cc..78aaf6e15cd2d 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll
@@ -18,24 +18,24 @@
 
 define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -47,24 +47,24 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5
 
 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -76,31 +76,31 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x
 
 define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) {
 ; SSE-LABEL: 'test_vXf32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -112,31 +112,31 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512
 
 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSE-LABEL: 'test_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -150,36 +150,36 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8
 ; AVX1-LABEL: 'test_vXi16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi16'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -193,36 +193,36 @@ define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x
 ; AVX1-LABEL: 'test_vXi8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi8'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll
index 8c4f55eb8adcb..4edfa8c1384d4 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll
@@ -18,24 +18,24 @@
 
 define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -47,24 +47,24 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5
 
 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -76,31 +76,31 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x
 
 define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) {
 ; SSE-LABEL: 'test_vXf32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -112,31 +112,31 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512
 
 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSE-LABEL: 'test_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -150,36 +150,36 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8
 ; AVX1-LABEL: 'test_vXi16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi16'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -193,36 +193,36 @@ define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x
 ; AVX1-LABEL: 'test_vXi8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi8'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll
index ffc470d5f3448..3d5c2cfb2143d 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll
@@ -18,24 +18,24 @@
 
 define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -47,24 +47,24 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5
 
 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128  = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -76,31 +76,31 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x
 
 define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) {
 ; SSE-LABEL: 'test_vXf32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -112,31 +112,31 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512
 
 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSE-LABEL: 'test_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -150,36 +150,36 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8
 ; AVX1-LABEL: 'test_vXi16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi16'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -193,36 +193,36 @@ define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x
 ; AVX1-LABEL: 'test_vXi8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512VBMI-LABEL: 'test_vXi8'
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
index 7f9a84f0cb0f3..57f72056f9c8d 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
@@ -22,13 +22,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -42,7 +42,7 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -80,13 +80,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
@@ -100,7 +100,7 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -143,13 +143,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -157,11 +157,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
@@ -174,13 +174,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -188,11 +188,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
@@ -205,13 +205,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -219,11 +219,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
@@ -242,7 +242,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -254,7 +254,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
@@ -273,7 +273,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -285,7 +285,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -364,13 +364,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -378,11 +378,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
@@ -395,13 +395,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -409,11 +409,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
@@ -426,13 +426,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -440,11 +440,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
@@ -463,7 +463,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -475,7 +475,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
@@ -494,7 +494,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -506,7 +506,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -595,19 +595,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -630,19 +630,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
@@ -665,19 +665,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
index 00248958d1862..ae91a70f67aee 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
@@ -22,13 +22,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -42,7 +42,7 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -80,13 +80,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
@@ -100,7 +100,7 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -143,13 +143,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -157,11 +157,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
@@ -174,13 +174,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -188,11 +188,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
@@ -205,13 +205,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -219,11 +219,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
@@ -242,7 +242,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -254,7 +254,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
@@ -273,7 +273,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -285,7 +285,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -364,13 +364,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -378,11 +378,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
@@ -395,13 +395,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -409,11 +409,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
@@ -426,13 +426,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -440,11 +440,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
@@ -463,7 +463,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -475,7 +475,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
@@ -494,7 +494,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -506,7 +506,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -595,19 +595,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -630,19 +630,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
@@ -665,19 +665,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
index 3845d582f8b11..c84b2847cf3e9 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
@@ -22,13 +22,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -42,7 +42,7 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -80,13 +80,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
@@ -100,7 +100,7 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -143,13 +143,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -157,11 +157,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
@@ -174,13 +174,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -188,11 +188,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
@@ -205,13 +205,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -219,11 +219,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
@@ -242,7 +242,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -254,7 +254,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
@@ -273,7 +273,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -285,7 +285,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -364,13 +364,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -378,11 +378,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
@@ -395,13 +395,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -409,11 +409,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
@@ -426,13 +426,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -440,11 +440,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
@@ -463,7 +463,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -475,7 +475,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
@@ -494,7 +494,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -506,7 +506,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -595,19 +595,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -630,19 +630,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
@@ -665,19 +665,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
index 5082eed6fd879..c4cbd6141d1f0 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
@@ -22,13 +22,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -42,7 +42,7 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -80,13 +80,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
@@ -100,7 +100,7 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -143,13 +143,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -157,11 +157,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
@@ -174,13 +174,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -188,11 +188,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
@@ -205,13 +205,13 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -219,11 +219,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
@@ -242,7 +242,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -254,7 +254,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
@@ -273,7 +273,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -285,7 +285,7 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -364,13 +364,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -378,11 +378,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
@@ -395,13 +395,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -409,11 +409,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
@@ -426,13 +426,13 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -440,11 +440,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
@@ -463,7 +463,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -475,7 +475,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
@@ -494,7 +494,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -506,7 +506,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -595,19 +595,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
@@ -630,19 +630,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
@@ -665,19 +665,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat-codesize.ll
new file mode 100644
index 0000000000000..39c935fff6b76
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat-codesize.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse2 | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx | FileCheck %s -check-prefixes=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+;
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+
+;
+; Verify the cost model for splat (broadcast on non-0th element) shuffles.
+;
+
+define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
+; SSE-LABEL: 'test_vXf64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
+; SSE-LABEL: 'test_vXi64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
+; SSE-LABEL: 'test_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf16(<2 x half> %src32, <4 x half> %src64, <8 x half> %src128, <16 x half> %src256, <32 x half> %src512) {
+; SSE2-LABEL: 'test_vXf16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXf16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXf16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXf16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXf16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXf16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat> %src128, <16 x bfloat> %src256, <32 x bfloat> %src512) {
+; SSE-LABEL: 'test_vXbf16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXbf16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXbf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXbf16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256, <32 x i16> %src512) {
+; SSE2-LABEL: 'test_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
+; SSE2-LABEL: 'test_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi8'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V16  = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+  %V32  = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V64  = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1> %src16, <32 x i1> %src32, <64 x i1> %src64) {
+; SSE2-LABEL: 'test_vXi1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 480 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 960 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi1'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V2  = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+  %V4  = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V8  = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+;
+; Tests the cost model for splat shuffles of second operand.
+;
+
+define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
+; SSE-LABEL: 'test_upper_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_upper_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_upper_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_upper_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+  %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+  %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat-latency.ll
new file mode 100644
index 0000000000000..2a89924dc7780
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat-latency.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+avx | FileCheck %s -check-prefixes=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+;
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+
+;
+; Verify the cost model for splat (broadcast on non-0th element) shuffles.
+;
+
+define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
+; SSE-LABEL: 'test_vXf64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
+; SSE-LABEL: 'test_vXi64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
+; SSE-LABEL: 'test_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf16(<2 x half> %src32, <4 x half> %src64, <8 x half> %src128, <16 x half> %src256, <32 x half> %src512) {
+; SSE2-LABEL: 'test_vXf16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXf16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXf16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXf16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXf16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXf16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat> %src128, <16 x bfloat> %src256, <32 x bfloat> %src512) {
+; SSE-LABEL: 'test_vXbf16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXbf16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXbf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXbf16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256, <32 x i16> %src512) {
+; SSE2-LABEL: 'test_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
+; SSE2-LABEL: 'test_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi8'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V16  = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+  %V32  = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V64  = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1> %src16, <32 x i1> %src32, <64 x i1> %src64) {
+; SSE2-LABEL: 'test_vXi1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 480 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 960 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi1'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V2  = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+  %V4  = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V8  = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+;
+; Tests the cost model for splat shuffles of second operand.
+;
+
+define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
+; SSE-LABEL: 'test_upper_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_upper_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_upper_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_upper_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+  %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+  %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat-sizelatency.ll
new file mode 100644
index 0000000000000..848e7b4e611a7
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat-sizelatency.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx | FileCheck %s -check-prefixes=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+;
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+
+;
+; Verify the cost model for splat (broadcast on non-0th element) shuffles.
+;
+
+define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
+; SSE-LABEL: 'test_vXf64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
+; SSE-LABEL: 'test_vXi64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
+; SSE-LABEL: 'test_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf16(<2 x half> %src32, <4 x half> %src64, <8 x half> %src128, <16 x half> %src256, <32 x half> %src512) {
+; SSE2-LABEL: 'test_vXf16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXf16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXf16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXf16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXf16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXf16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat> %src128, <16 x bfloat> %src256, <32 x bfloat> %src512) {
+; SSE-LABEL: 'test_vXbf16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXbf16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXbf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXbf16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256, <32 x i16> %src512) {
+; SSE2-LABEL: 'test_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V32  = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
+; SSE2-LABEL: 'test_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi8'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V16  = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+  %V32  = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V64  = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1> %src16, <32 x i1> %src32, <64 x i1> %src64) {
+; SSE2-LABEL: 'test_vXi1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 480 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 960 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi1'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V2  = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+  %V4  = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V8  = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+;
+; Tests the cost model for splat shuffles of second operand.
+;
+
+define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
+; SSE-LABEL: 'test_upper_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX1-LABEL: 'test_upper_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX2-LABEL: 'test_upper_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; AVX512-LABEL: 'test_upper_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+  %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+  %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
new file mode 100644
index 0000000000000..4c6d1ccd5ca34
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+sse2 | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+avx | FileCheck %s -check-prefixes=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+;
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+
+;
+; Verify the cost model for splat (broadcast on non-0th element) shuffles.
+;
+
+define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
+; SSE-LABEL: 'test_vXf64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V128 = shufflevector <2 x double> %src128, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x double> %src256, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x double> %src512, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
+; SSE-LABEL: 'test_vXi64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V128 = shufflevector <2 x i64> %src128, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  %V256 = shufflevector <4 x i64> %src256, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <8 x i64> %src512, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
+; SSE-LABEL: 'test_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %src64, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x float> %src128, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x float> %src256, <8 x float> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x float> %src512, <16 x float> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V64 = shufflevector <2 x i32> %src64, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %V128 = shufflevector <4 x i32> %src128, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <8 x i32> %src256, <8 x i32> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <16 x i32> %src512, <16 x i32> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXf16(<2 x half> %src32, <4 x half> %src64, <8 x half> %src128, <16 x half> %src256, <32 x half> %src512) {
+; SSE2-LABEL: 'test_vXf16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXf16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXf16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXf16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXf16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXf16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V32  = shufflevector <2 x half> %src32, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x half> %src64, <4 x half> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x half> %src128, <8 x half> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x half> %src256, <16 x half> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x half> %src512, <32 x half> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat> %src128, <16 x bfloat> %src256, <32 x bfloat> %src512) {
+; SSE-LABEL: 'test_vXbf16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXbf16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXbf16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXbf16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V32  = shufflevector <2 x bfloat> %src32, <2 x bfloat> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x bfloat> %src64, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16 x i16> %src256, <32 x i16> %src512) {
+; SSE2-LABEL: 'test_vXi16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi16'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V32  = shufflevector <2 x i16> %src32, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  %V64  = shufflevector <4 x i16> %src64, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <8 x i16> %src128, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <16 x i16> %src256, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <32 x i16> %src512, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
+; SSE2-LABEL: 'test_vXi8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'test_vXi8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'test_vXi8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi8'
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32 = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V16  = shufflevector <2 x i8> %src16, <2 x i8> undef, <2 x i32> <i32 1, i32 1>
+  %V32  = shufflevector <4 x i8> %src32, <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V64  = shufflevector <8 x i8> %src64, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V128 = shufflevector <16 x i8> %src128, <16 x i8> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V256 = shufflevector <32 x i8> %src256, <32 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V512 = shufflevector <64 x i8> %src512, <64 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1> %src16, <32 x i1> %src32, <64 x i1> %src64) {
+; SSE2-LABEL: 'test_vXi1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 480 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 960 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'test_vXi1'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test_vXi1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_vXi1'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_vXi1'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_vXi1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V2  = shufflevector <2 x i1> %src2, <2 x i1> undef, <2 x i32> <i32 1, i32 1>
+  %V4  = shufflevector <4 x i1> %src4, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %V8  = shufflevector <8 x i1> %src8, <8 x i1> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V16 = shufflevector <16 x i1> %src16, <16 x i1> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V32 = shufflevector <32 x i1> %src32, <32 x i1> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %V64 = shufflevector <64 x i1> %src64, <64 x i1> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
+
+;
+; Tests the cost model for splat shuffles of second operand.
+;
+
+define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
+; SSE-LABEL: 'test_upper_vXf32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_upper_vXf32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'test_upper_vXf32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test_upper_vXf32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 3, i32 3>
+  %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11>
+  %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-codesize.ll
index 7783161375916..c8662a41b8e3a 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-codesize.ll
@@ -19,14 +19,14 @@
 define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a256, <4 x double> %b256, <8 x double> %a512, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -44,14 +44,14 @@ define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a2
 define void @test_vXi64(<2 x i64> %a128, <2 x i64> %b128, <4 x i64> %a256, <4 x i64> %b256, <8 x i64> %a512, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -70,15 +70,15 @@ define void @test_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <
 ; SSE-LABEL: 'test_vXf32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -99,15 +99,15 @@ define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i3
 ; SSE-LABEL: 'test_vXi32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -129,24 +129,24 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
@@ -154,7 +154,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
@@ -162,7 +162,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
@@ -203,8 +203,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
@@ -212,8 +212,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
@@ -221,8 +221,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi8'
@@ -231,7 +231,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
@@ -240,7 +240,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-latency.ll
index f96bb0d8e6ed0..a207bc8e20f5f 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-latency.ll
@@ -19,14 +19,14 @@
 define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a256, <4 x double> %b256, <8 x double> %a512, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -44,14 +44,14 @@ define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a2
 define void @test_vXi64(<2 x i64> %a128, <2 x i64> %b128, <4 x i64> %a256, <4 x i64> %b256, <8 x i64> %a512, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -70,15 +70,15 @@ define void @test_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <
 ; SSE-LABEL: 'test_vXf32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -99,15 +99,15 @@ define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i3
 ; SSE-LABEL: 'test_vXi32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -129,24 +129,24 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
@@ -154,7 +154,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
@@ -162,7 +162,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
@@ -203,8 +203,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
@@ -212,8 +212,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
@@ -221,8 +221,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi8'
@@ -231,7 +231,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
@@ -240,7 +240,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-sizelatency.ll
index e9927c61e7741..75130deda1df8 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose-sizelatency.ll
@@ -19,14 +19,14 @@
 define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a256, <4 x double> %b256, <8 x double> %a512, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -44,14 +44,14 @@ define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a2
 define void @test_vXi64(<2 x i64> %a128, <2 x i64> %b128, <4 x i64> %a256, <4 x i64> %b256, <8 x i64> %a512, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -70,15 +70,15 @@ define void @test_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <
 ; SSE-LABEL: 'test_vXf32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -99,15 +99,15 @@ define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i3
 ; SSE-LABEL: 'test_vXi32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -129,24 +129,24 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
@@ -154,7 +154,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
@@ -162,7 +162,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
@@ -203,8 +203,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
@@ -212,8 +212,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
@@ -221,8 +221,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi8'
@@ -231,7 +231,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
@@ -240,7 +240,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
index 4e32b45f4e414..5c50ec1feb058 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-transpose.ll
@@ -19,14 +19,14 @@
 define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a256, <4 x double> %b256, <8 x double> %a512, <8 x double> %b512) {
 ; SSE-LABEL: 'test_vXf64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x double> %a128, <2 x double> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x double> %a256, <4 x double> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x double> %a512, <8 x double> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -44,14 +44,14 @@ define void @test_vXf64(<2 x double> %a128, <2 x double> %b128, <4 x double> %a2
 define void @test_vXi64(<2 x i64> %a128, <2 x i64> %b128, <4 x i64> %a256, <4 x i64> %b256, <8 x i64> %a512, <8 x i64> %b512) {
 ; SSE-LABEL: 'test_vXi64'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
@@ -70,15 +70,15 @@ define void @test_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a128, <
 ; SSE-LABEL: 'test_vXf32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x float> %a64, <2 x float> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
@@ -99,15 +99,15 @@ define void @test_vXi32(<2 x i32> %a64, <2 x i32> %b64, <4 x i32> %a128, <4 x i3
 ; SSE-LABEL: 'test_vXi32'
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <2 x i32> %a64, <2 x i32> %b64, <2 x i32> <i32 0, i32 2>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
@@ -129,24 +129,24 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <2 x i16> %a32, <2 x i16> %b32, <2 x i32> <i32 0, i32 2>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
@@ -154,7 +154,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
@@ -162,7 +162,7 @@ define void @test_vXi16(<2 x i16> %a32, <2 x i16> %b32, <4 x i16> %a64, <4 x i16
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <4 x i16> %a64, <4 x i16> %b64, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
@@ -203,8 +203,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
@@ -212,8 +212,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
@@ -221,8 +221,8 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = shufflevector <4 x i8> %a32, <4 x i8> %b32, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi8'
@@ -231,7 +231,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi8'
@@ -240,7 +240,7 @@ define void @test_vXi8(<2 x i8> %a16, <2 x i8> %b16, <4 x i8> %a32, <4 x i8> %b3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = shufflevector <8 x i8> %a64, <8 x i8> %b64, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <64 x i32> <i32 0, i32 64, i32 2, i32 66, i32 4, i32 68, i32 6, i32 70, i32 8, i32 72, i32 10, i32 74, i32 12, i32 76, i32 14, i32 78, i32 16, i32 80, i32 18, i32 82, i32 20, i32 84, i32 22, i32 86, i32 24, i32 88, i32 26, i32 90, i32 28, i32 92, i32 30, i32 94, i32 32, i32 96, i32 34, i32 98, i32 36, i32 100, i32 38, i32 102, i32 40, i32 104, i32 42, i32 106, i32 44, i32 108, i32 46, i32 110, i32 48, i32 112, i32 50, i32 114, i32 52, i32 116, i32 54, i32 118, i32 56, i32 120, i32 58, i32 122, i32 60, i32 124, i32 62, i32 126>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
diff --git a/llvm/test/Analysis/LazyCallGraph/blockaddress.ll b/llvm/test/Analysis/LazyCallGraph/blockaddress.ll
index 4ce41c500fba8..a0dbe3009606e 100644
--- a/llvm/test/Analysis/LazyCallGraph/blockaddress.ll
+++ b/llvm/test/Analysis/LazyCallGraph/blockaddress.ll
@@ -7,7 +7,7 @@ define i32 @baz(i32 %y, i1 %b) {
 ; CHECK-NEXT:    br i1 [[B:%.*]], label [[LAB:%.*]], label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[P_0:%.*]] = phi ptr [ null, [[FOR_COND]] ], [ blockaddress(@baz, [[LAB]]), [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[P_0]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[P_0]], i64 1
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       lab:
 ; CHECK-NEXT:    ret i32 0
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/nssw-predicate-implied.ll b/llvm/test/Analysis/LoopAccessAnalysis/nssw-predicate-implied.ll
new file mode 100644
index 0000000000000..1a07805c2614f
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/nssw-predicate-implied.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<access-info>' %s -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+
+; FIXME: {0,+,3} implies {0,+,2}.
+define void @wrap_check_iv.3_implies_iv.2(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_implies_iv.2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP1]]:
+; CHECK-NEXT:          (Low: %dst High: (4 + (12 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %dst))
+; CHECK-NEXT:            Member: {%dst,+,12}<%loop>
+; CHECK-NEXT:        Group [[GRP2]]:
+; CHECK-NEXT:          (Low: %src High: (4 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {%src,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {0,+,3}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {0,+,2}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,2}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {%src,+,8}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,3}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {%dst,+,12}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+  %l = load i32, ptr %gep.iv.2, align 4
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+  store i32 %l, ptr %gep.iv.3, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, 3
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; FIXME: {2,+,2} implies {0,+,2}.
+define void @wrap_check_iv.3_implies_iv.2_different_start(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_implies_iv.2_different_start'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP3]]:
+; CHECK-NEXT:          (Low: (12 + %dst) High: (16 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %dst))
+; CHECK-NEXT:            Member: {(12 + %dst),+,8}<%loop>
+; CHECK-NEXT:        Group [[GRP4]]:
+; CHECK-NEXT:          (Low: %src High: (4 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {%src,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {2,+,2}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {0,+,2}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,2}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {%src,+,8}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3:
+; CHECK-NEXT:        (4 + (4 * (sext i32 {2,+,2}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {(12 + %dst),+,8}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 3, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+  %l = load i32, ptr %gep.iv.2, align 4
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+  store i32 %l, ptr %gep.iv.3, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, 2
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; FIXME: {0,+,3} implies {0,+,2}.
+define void @wrap_check_iv.3_implies_iv.2_predicates_added_in_different_order(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_implies_iv.2_predicates_added_in_different_order'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.2
+; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.3
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP5]]:
+; CHECK-NEXT:          (Low: %dst High: (4 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %dst))
+; CHECK-NEXT:            Member: {%dst,+,8}<%loop>
+; CHECK-NEXT:        Group [[GRP6]]:
+; CHECK-NEXT:          (Low: %src High: (4 + (12 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {%src,+,12}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {0,+,2}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {0,+,3}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.3:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,3}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {%src,+,12}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,2}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {%dst,+,8}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.3
+  %l = load i32, ptr %gep.iv.3, align 4
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.2
+  store i32 %l, ptr %gep.iv.2, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, 3
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @wrap_check_iv.3_does_not_implies_iv.2_due_to_start(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_does_not_implies_iv.2_due_to_start'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP7:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+; CHECK-NEXT:        Against group ([[GRP8:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP7]]:
+; CHECK-NEXT:          (Low: %dst High: (4 + (12 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %dst))
+; CHECK-NEXT:            Member: {%dst,+,12}<%loop>
+; CHECK-NEXT:        Group [[GRP8]]:
+; CHECK-NEXT:          (Low: (40 + %src) High: (44 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {(40 + %src),+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {0,+,3}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {10,+,2}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {10,+,2}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {(40 + %src),+,8}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,3}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {%dst,+,12}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 10, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+  %l = load i32, ptr %gep.iv.2, align 4
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+  store i32 %l, ptr %gep.iv.3, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, 3
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @wrap_check_iv.3_does_not_imply_iv.2_due_to_start_negative(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_does_not_imply_iv.2_due_to_start_negative'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP9:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+; CHECK-NEXT:        Against group ([[GRP10:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP9]]:
+; CHECK-NEXT:          (Low: (-4 + %dst) High: ((12 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %dst))
+; CHECK-NEXT:            Member: {(-4 + %dst),+,12}<%loop>
+; CHECK-NEXT:        Group [[GRP10]]:
+; CHECK-NEXT:          (Low: %src High: (4 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {%src,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {-1,+,3}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {0,+,2}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,2}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {%src,+,8}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3:
+; CHECK-NEXT:        ((4 * (sext i32 {-1,+,3}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {(-4 + %dst),+,12}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ -1, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+  %l = load i32, ptr %gep.iv.2, align 4
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+  store i32 %l, ptr %gep.iv.3, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, 3
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @wrap_check_iv.3_does_not_imply_iv.2_due_to_negative_step(i32 noundef %N, ptr %dst, ptr %src) {
+; CHECK-LABEL: 'wrap_check_iv.3_does_not_imply_iv.2_due_to_negative_step'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP11:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+; CHECK-NEXT:        Against group ([[GRP12:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP11]]:
+; CHECK-NEXT:          (Low: ((-4 * (zext i32 (-1 + %N) to i64))<nsw> + %dst) High: (4 + %dst))
+; CHECK-NEXT:            Member: {%dst,+,-4}<%loop>
+; CHECK-NEXT:        Group [[GRP12]]:
+; CHECK-NEXT:          (Low: %src High: (4 + (8 * (zext i32 (-1 + %N) to i64))<nuw><nsw> + %src))
+; CHECK-NEXT:            Member: {%src,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {0,+,-1}<%loop> Added Flags: <nssw>
+; CHECK-NEXT:      {0,+,2}<%loop> Added Flags: <nssw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:      [PSE] %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,2}<%loop> to i64))<nsw> + %src)
+; CHECK-NEXT:        --> {%src,+,8}<%loop>
+; CHECK-NEXT:      [PSE] %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3:
+; CHECK-NEXT:        ((4 * (sext i32 {0,+,-1}<%loop> to i64))<nsw> + %dst)
+; CHECK-NEXT:        --> {%dst,+,-4}<%loop>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
+  %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]
+  %ext.iv.2 = sext i32 %iv.2 to i64
+  %gep.iv.2 = getelementptr inbounds i32, ptr %src, i64 %ext.iv.2
+  %l = load i32, ptr %gep.iv.2, align 4
+  %ext.iv.3 = sext i32 %iv.3 to i64
+  %gep.iv.3 = getelementptr inbounds i32, ptr %dst, i64 %ext.iv.3
+  store i32 %l, ptr %gep.iv.3, align 4
+  %iv.1.next = add nuw nsw i32 %iv.1, 1
+  %iv.2.next = add i32 %iv.2, 2
+  %iv.3.next = add i32 %iv.3, -1
+  %ec = icmp eq i32 %iv.1.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/select-dependence.ll b/llvm/test/Analysis/LoopAccessAnalysis/select-dependence.ll
index 7d8a25f022e7d..5b01efd821c47 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/select-dependence.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/select-dependence.ll
@@ -44,8 +44,13 @@ exit:
 define void @test_phi(ptr noalias %x, ptr noalias %y, ptr noalias %z) {
 ; CHECK-LABEL: 'test_phi'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unsafe indirect dependence.
 ; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        IndirectUnsafe:
+; CHECK-NEXT:            %load = load double, ptr %gep.sel, align 8 ->
+; CHECK-NEXT:            store double %load, ptr %gep.sel2, align 8
+; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
 ; CHECK-EMPTY:
diff --git a/llvm/test/Analysis/MemorySSA/loop-rotate-update.ll b/llvm/test/Analysis/MemorySSA/loop-rotate-update.ll
new file mode 100644
index 0000000000000..4ed4e825af569
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/loop-rotate-update.ll
@@ -0,0 +1,37 @@
+; RUN: opt -disable-output -passes="loop-mssa(loop-rotate),print<memoryssa>" -verify-memoryssa < %s 2>&1 | FileCheck %s
+
+; CHECK: entry:
+; CHECK-NEXT: 3 = MemoryDef(liveOnEntry)
+; CHECK-NEXT: store ptr null, ptr %p, align 8
+; CHECK-NEXT: MemoryUse(3)
+; CHECK-NEXT: %val11 = load ptr, ptr %p, align 8
+
+; CHECK: loop.latch:
+; CHECK-NEXT: 5 = MemoryPhi({loop.latch,1},{loop.latch.lr.ph,3})
+; CHECK-NEXT: MemoryUse(5)
+; CHECK-NEXT: %val2 = load ptr, ptr %p, align 8
+; CHECK-NEXT: 1 = MemoryDef(5)
+; CHECK-NEXT: store ptr null, ptr %p, align 8
+; CHECK-NEXT: MemoryUse(1)
+; CHECK-NEXT: %val1 = load ptr, ptr %p, align 8
+
+; CHECK: exit:
+; CHECK-NEXT: 4 = MemoryPhi({entry,3},{loop.exit_crit_edge,1})
+
+define void @test(ptr %p) {
+entry:
+  br label %loop
+
+loop:
+  store ptr null, ptr %p
+  %val1 = load ptr, ptr %p
+  %cmp = icmp eq ptr %val1, null
+  br i1 %cmp, label %exit, label %loop.latch
+
+loop.latch:
+  %val2 = load ptr, ptr %p
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/loop-unswitch.ll b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
index 107776d2dae30..4615b312bee9d 100644
--- a/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
+++ b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -passes=simple-loop-unswitch -disable-basic-aa -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -passes="loop-mssa(simple-loop-unswitch)" -disable-basic-aa -verify-memoryssa < %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/MemorySSA/pr116227.ll b/llvm/test/Analysis/MemorySSA/pr116227.ll
new file mode 100644
index 0000000000000..cd9686b018aa6
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr116227.ll
@@ -0,0 +1,61 @@
+; RUN: opt -disable-output -passes="loop-mssa(simple-loop-unswitch<nontrivial>),print<memoryssa>" -verify-memoryssa < %s 2>&1 | FileCheck %s
+
+declare ptr @malloc() allockind("alloc,uninitialized")
+
+; CHECK-LABEL: MemorySSA for function: test
+
+; CHECK: for.body.us:
+; CHECK-NEXT: 3 = MemoryPhi({entry.split.us,liveOnEntry},{for.body.us,3})
+
+; CHECK: for.body:
+; CHECK-NEXT: 2 = MemoryPhi({entry.split,liveOnEntry},{for.body,1})
+; CHECK-NEXT: 1 = MemoryDef(2)
+; CHECK-NEXT:  %call.i = call ptr @malloc()
+
+define void @test(i1 %arg) {
+entry:
+  br label %for.body
+
+for.body:
+  %call.i = call ptr @malloc()
+  %cmp.i = icmp ne ptr %call.i, null
+  %or.cond.i = select i1 %cmp.i, i1 %arg, i1 false
+  br i1 %or.cond.i, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: MemorySSA for function: test_extra_defs
+
+; CHECK: entry:
+; CHECK-NEXT: 1 = MemoryDef(liveOnEntry)
+; CHECK-NEXT: store i8 1, ptr %p, align 1
+
+; CHECK: for.body.us:
+; CHECK-NEXT: 5 = MemoryPhi({entry.split.us,1},{for.body.us,6})
+; CHECK-NEXT: 6 = MemoryDef(5)
+; CHECK-NEXT: store i8 2, ptr %p, align 1
+
+; CHECK: for.body:
+; CHECK-NEXT: 4 = MemoryPhi({entry.split,1},{for.body,3})
+; CHECK-NEXT: 2 = MemoryDef(4)
+; CHECK-NEXT: store i8 2, ptr %p
+; CHECK-NEXT: 3 = MemoryDef(2)
+; CHECK-NEXT: %call.i = call ptr @malloc()
+
+define void @test_extra_defs(ptr %p, i1 %arg) {
+entry:
+  store i8 1, ptr %p
+  br label %for.body
+
+for.body:
+  store i8 2, ptr %p
+  %call.i = call ptr @malloc()
+  %cmp.i = icmp ne ptr %call.i, null
+  %or.cond.i = select i1 %cmp.i, i1 %arg, i1 false
+  br i1 %or.cond.i, label %exit, label %for.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr40037.ll b/llvm/test/Analysis/MemorySSA/pr40037.ll
index 656328de3567a..ab22470af9d90 100644
--- a/llvm/test/Analysis/MemorySSA/pr40037.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40037.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -S -passes=simple-loop-unswitch -verify-memoryssa  < %s | FileCheck %s
+; RUN: opt -S -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa  < %s | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
diff --git a/llvm/test/Analysis/MemorySSA/pr40749_2.ll b/llvm/test/Analysis/MemorySSA/pr40749_2.ll
index 0fa45534f9c60..0ebe9eb3eb97b 100644
--- a/llvm/test/Analysis/MemorySSA/pr40749_2.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40749_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -passes=licm,simple-loop-unswitch -verify-memoryssa %s | FileCheck %s
+; RUN: opt -S -passes="loop-mssa(licm,simple-loop-unswitch)" -verify-memoryssa %s | FileCheck %s
 ; REQUIRES: asserts
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
diff --git a/llvm/test/Analysis/MemorySSA/pr43641.ll b/llvm/test/Analysis/MemorySSA/pr43641.ll
index 25bbee007037a..46b9adb51c424 100644
--- a/llvm/test/Analysis/MemorySSA/pr43641.ll
+++ b/llvm/test/Analysis/MemorySSA/pr43641.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=simple-loop-unswitch -enable-nontrivial-unswitch -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -passes="loop-mssa(simple-loop-unswitch)" -enable-nontrivial-unswitch -verify-memoryssa -S < %s | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK-LABEL: @c
diff --git a/llvm/test/Analysis/MemorySSA/pr46574.ll b/llvm/test/Analysis/MemorySSA/pr46574.ll
index 0b5da6ec07b99..320f37a19f2eb 100644
--- a/llvm/test/Analysis/MemorySSA/pr46574.ll
+++ b/llvm/test/Analysis/MemorySSA/pr46574.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='loop-mssa(loop-instsimplify,loop-simplifycfg,licm,simple-loop-unswitch)' -enable-nontrivial-unswitch %s -disable-output
+; RUN: opt -passes='loop-mssa(loop-instsimplify,loop-simplifycfg,licm,simple-loop-unswitch)' -enable-nontrivial-unswitch -verify-memoryssa %s -disable-output
 ; REQUIRES: asserts
 
 %0 = type { i24, i32, i32, i8 }
diff --git a/llvm/test/Analysis/MemorySSA/simple_loop_unswitch_nontrivial.ll b/llvm/test/Analysis/MemorySSA/simple_loop_unswitch_nontrivial.ll
index b87b8e7a44e9f..35dda3711cbd9 100644
--- a/llvm/test/Analysis/MemorySSA/simple_loop_unswitch_nontrivial.ll
+++ b/llvm/test/Analysis/MemorySSA/simple_loop_unswitch_nontrivial.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=simple-loop-unswitch -enable-nontrivial-unswitch -verify-memoryssa -S %s | FileCheck %s
+; RUN: opt -passes="loop-mssa(simple-loop-unswitch)" -enable-nontrivial-unswitch -verify-memoryssa -S %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/ScalarEvolution/pr117133.ll b/llvm/test/Analysis/ScalarEvolution/pr117133.ll
new file mode 100644
index 0000000000000..3aadc18d6157c
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/pr117133.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -disable-output "-passes=print<scalar-evolution>" < %s 2>&1 | FileCheck %s
+
+define i32 @widget() {
+; CHECK-LABEL: 'widget'
+; CHECK-NEXT:  Classifying expressions for: @widget
+; CHECK-NEXT:    %phi = phi i32 [ 0, %b ], [ %udiv6, %b5 ]
+; CHECK-NEXT:    --> %phi U: [0,1) S: [0,1) Exits: <<Unknown>> LoopDispositions: { %b1: Variant }
+; CHECK-NEXT:    %phi2 = phi i32 [ 1, %b ], [ %add, %b5 ]
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%b1> U: [1,2) S: [1,2) Exits: <<Unknown>> LoopDispositions: { %b1: Computable }
+; CHECK-NEXT:    %udiv = udiv i32 10, %phi2
+; CHECK-NEXT:    --> (10 /u {1,+,1}<nuw><nsw><%b1>) U: [10,11) S: [10,11) Exits: <<Unknown>> LoopDispositions: { %b1: Computable }
+; CHECK-NEXT:    %urem = urem i32 %udiv, 10
+; CHECK-NEXT:    --> ((-10 * ((10 /u {1,+,1}<nuw><nsw><%b1>) /u 10))<nuw><nsw> + (10 /u {1,+,1}<nuw><nsw><%b1>)) U: [0,1) S: [0,1) Exits: <<Unknown>> LoopDispositions: { %b1: Computable }
+; CHECK-NEXT:    %udiv6 = udiv i32 %phi2, 0
+; CHECK-NEXT:    --> ({1,+,1}<nuw><nsw><%b1> /u 0) U: empty-set S: empty-set Exits: <<Unknown>> LoopDispositions: { %b1: Computable }
+; CHECK-NEXT:    %add = add i32 %phi2, 1
+; CHECK-NEXT:    --> {2,+,1}<nuw><nsw><%b1> U: [2,3) S: [2,3) Exits: <<Unknown>> LoopDispositions: { %b1: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @widget
+; CHECK-NEXT:  Loop %b1: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for b1: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for b3: i32 0
+; CHECK-NEXT:  Loop %b1: constant max backedge-taken count is i32 0
+; CHECK-NEXT:  Loop %b1: symbolic max backedge-taken count is i32 0
+; CHECK-NEXT:    symbolic max exit count for b1: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for b3: i32 0
+;
+b:
+  br label %b1
+
+b1:                                              ; preds = %b5, %b
+  %phi = phi i32 [ 0, %b ], [ %udiv6, %b5 ]
+  %phi2 = phi i32 [ 1, %b ], [ %add, %b5 ]
+  %icmp = icmp eq i32 %phi, 0
+  br i1 %icmp, label %b3, label %b8
+
+b3:                                              ; preds = %b1
+  %udiv = udiv i32 10, %phi2
+  %urem = urem i32 %udiv, 10
+  %icmp4 = icmp eq i32 %urem, 0
+  br i1 %icmp4, label %b7, label %b5
+
+b5:                                              ; preds = %b3
+  %udiv6 = udiv i32 %phi2, 0
+  %add = add i32 %phi2, 1
+  br label %b1
+
+b7:                                              ; preds = %b3
+  ret i32 5
+
+b8:                                              ; preds = %b1
+  ret i32 7
+}
+
+; Don't fold %indvar2 into (zext {0,+,1}) * %a
+define i64 @test_poisonous(i64 %a, i32 %n) {
+; CHECK-LABEL: 'test_poisonous'
+; CHECK-NEXT:  Classifying expressions for: @test_poisonous
+; CHECK-NEXT:    %indvar1 = phi i32 [ 0, %entry ], [ %indvar1.next, %loop.body ]
+; CHECK-NEXT:    --> {0,+,1}<%loop.body> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop.body: Computable }
+; CHECK-NEXT:    %indvar2 = phi i64 [ 0, %entry ], [ %mul, %loop.body ]
+; CHECK-NEXT:    --> %indvar2 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop.body: Variant }
+; CHECK-NEXT:    %indvar1.next = add i32 %indvar1, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop.body> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop.body: Computable }
+; CHECK-NEXT:    %ext = zext i32 %indvar1.next to i64
+; CHECK-NEXT:    --> (zext i32 {1,+,1}<%loop.body> to i64) U: [0,4294967296) S: [0,4294967296) Exits: <<Unknown>> LoopDispositions: { %loop.body: Computable }
+; CHECK-NEXT:    %mul = mul i64 %ext, %a
+; CHECK-NEXT:    --> ((zext i32 {1,+,1}<%loop.body> to i64) * %a) U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop.body: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @test_poisonous
+; CHECK-NEXT:  Loop %loop.body: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop.body: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop.body: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %loop.body: Predicated backedge-taken count is (-1 + (1 smax (1 + (sext i32 %n to i64))<nsw>))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {1,+,1}<%loop.body> Added Flags: <nssw>
+; CHECK-NEXT:  Loop %loop.body: Predicated constant max backedge-taken count is i64 2147483647
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {1,+,1}<%loop.body> Added Flags: <nssw>
+; CHECK-NEXT:  Loop %loop.body: Predicated symbolic max backedge-taken count is (-1 + (1 smax (1 + (sext i32 %n to i64))<nsw>))<nsw>
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      {1,+,1}<%loop.body> Added Flags: <nssw>
+;
+entry:
+  br label %loop.body
+
+loop.body:
+  %indvar1 = phi i32 [ 0, %entry ], [ %indvar1.next, %loop.body ]
+  %indvar2 = phi i64 [ 0, %entry ], [ %mul, %loop.body ]
+  %indvar1.next = add i32 %indvar1, 1
+  %ext = zext i32 %indvar1.next to i64
+  %mul = mul i64 %ext, %a
+  %exitcond = icmp sgt i32 %indvar1.next, %n
+  br i1 %exitcond, label %loop.exit, label %loop.body
+
+loop.exit:
+  ret i64 %mul
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-phi-increment.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-phi-increment.ll
new file mode 100644
index 0000000000000..db284b263d068
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-phi-increment.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>" 2>&1 | FileCheck %s
+define void @test1(ptr %x, ptr %y) {
+; CHECK-LABEL: 'test1'
+; CHECK-NEXT:  Classifying expressions for: @test1
+; CHECK-NEXT:    %v1.0 = phi i32 [ 0, %entry ], [ %k.0, %if.end ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.cond> U: [0,7) S: [0,7) Exits: 6 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %add = add nsw i32 %v1.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %add6 = add nsw i32 %v1.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %k.0 = phi i32 [ %add, %if.then ], [ %add6, %if.else ]
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @test1
+; CHECK-NEXT:  Loop %for.cond: backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: constant max backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: symbolic max backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: Trip multiple is 7
+;
+entry:
+  br label %for.cond
+
+for.cond:                                                ; preds = %6, %0
+  %v1.0 = phi i32 [ 0, %entry ], [ %k.0, %if.end ]
+  %cmp = icmp slt i32 %v1.0, 6
+  br i1 %cmp, label %for.body, label %exit
+
+for.body:                                                ; preds = %1
+  %cmp3 = icmp slt i32 %v1.0, 2
+  br i1 %cmp3, label %if.then, label %if.else
+
+if.then:                                                ; preds = %2
+  %add = add nsw i32 %v1.0, 1
+  br label %if.end
+
+if.else:                                                ; preds = %2
+  %add6 = add nsw i32 %v1.0, 1
+  br label %if.end
+
+if.end:                                                ; preds = %4, %3
+  %k.0 = phi i32 [ %add, %if.then ], [ %add6, %if.else ]
+  br label %for.cond
+
+exit:                                                ; preds = %5
+  ret void
+}
+
+define void @test2(ptr %x, ptr %y) {
+; CHECK-LABEL: 'test2'
+; CHECK-NEXT:  Classifying expressions for: @test2
+; CHECK-NEXT:    %v1.0 = phi i32 [ 0, %entry ], [ %k.0, %if.end ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.cond> U: [0,7) S: [0,7) Exits: 6 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %add = add nuw i32 %v1.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %add6 = add nsw i32 %v1.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %k.0 = phi i32 [ %add, %if.then ], [ %add6, %if.else ]
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.cond> U: [1,8) S: [1,8) Exits: 7 LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @test2
+; CHECK-NEXT:  Loop %for.cond: backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: constant max backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: symbolic max backedge-taken count is i32 6
+; CHECK-NEXT:  Loop %for.cond: Trip multiple is 7
+;
+entry:
+  br label %for.cond
+
+for.cond:                                                ; preds = %6, %0
+  %v1.0 = phi i32 [ 0, %entry ], [ %k.0, %if.end ]
+  %cmp = icmp slt i32 %v1.0, 6
+  br i1 %cmp, label %for.body, label %exit
+
+for.body:                                                ; preds = %1
+  %cmp3 = icmp slt i32 %v1.0, 2
+  br i1 %cmp3, label %if.then, label %if.else
+
+if.then:                                                ; preds = %2
+  %add = add nuw i32 %v1.0, 1
+  br label %if.end
+
+if.else:                                                ; preds = %2
+  %add6 = add nsw i32 %v1.0, 1
+  br label %if.end
+
+if.end:                                                ; preds = %4, %3
+  %k.0 = phi i32 [ %add, %if.then ], [ %add6, %if.else ]
+  br label %for.cond
+
+exit:                                                ; preds = %5
+  ret void
+}
+
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index fb1420ee34004..aa5208560817f 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -305,6 +305,28 @@ bb:
   ret void
 }
 
+declare <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3))
+
+; CHECK: DIVERGENT: %tmp0 = call <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3) %gep)
+define amdgpu_kernel void @ds_read_b64_tr_b16_v4f16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %tmp0 = call <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3) %gep)
+  store <4 x half> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+declare <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16(ptr addrspace(3))
+
+; CHECK: DIVERGENT: %tmp0 = call <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16(ptr addrspace(3) %gep)
+define amdgpu_kernel void @ds_read_b64_tr_b16_v4bf16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %tmp0 = call <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16(ptr addrspace(3) %gep)
+  store <4 x bfloat> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
 declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
 declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half>, <8 x half>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
 
diff --git a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
index 8691e63a4f3ee..84220832f068f 100644
--- a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
+++ b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
@@ -436,7 +436,7 @@ define i1 @recursiveGEP_withPtrSub1(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -467,7 +467,7 @@ define i1 @recursiveGEP_withPtrSub1_PhiOperandsCommuted(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[VAL1:%.*]], [[ENTRY:%.*]] ], [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -498,7 +498,7 @@ define i1 @recursiveGEP_withPtrSub1_SubOperandsCommuted(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -557,11 +557,11 @@ while.end.i:
 define i1 @recursiveGEP_withPtrSub3(ptr %val1) {
 ; CHECK-LABEL: @recursiveGEP_withPtrSub3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TEST_VAL1:%.*]] = getelementptr inbounds i8, ptr [[VAL1:%.*]], i64 7
+; CHECK-NEXT:    [[TEST_VAL1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAL1:%.*]], i64 7
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST_VAL1]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -594,7 +594,7 @@ define i1 @recursiveGEP_withPtrSub1_notKnownNonEqual1(ptr %val1, i64 %val2) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -629,7 +629,7 @@ define i1 @recursiveGEP_withPtrSub1_notKnownNonEqual2(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST_VAL1]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -659,7 +659,7 @@ while.end.i:
 define i1 @recursiveGEP_withPtrSub1_notKnownNonEqual3(ptr %val1) {
 ; CHECK-LABEL: @recursiveGEP_withPtrSub1_notKnownNonEqual3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TEST_VAL1:%.*]] = getelementptr inbounds i8, ptr [[VAL1:%.*]], i64 5
+; CHECK-NEXT:    [[TEST_VAL1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAL1:%.*]], i64 5
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST_VAL1]], [[ENTRY:%.*]] ]
@@ -697,7 +697,7 @@ define i1 @recursiveGEP_withPtrSub_maybeZero(ptr %val1) {
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_PN_I]], align 2
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
 ; CHECK:       while.end.i:
@@ -792,7 +792,7 @@ while.end.i:
 define i1 @recursiveGEP_withPtrSub_noninboundStepAndB(ptr %val1) {
 ; CHECK-LABEL: @recursiveGEP_withPtrSub_noninboundStepAndB(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TEST:%.*]] = getelementptr inbounds i8, ptr [[VAL1:%.*]], i64 2
+; CHECK-NEXT:    [[TEST:%.*]] = getelementptr inbounds nuw i8, ptr [[VAL1:%.*]], i64 2
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST]], [[ENTRY:%.*]] ]
@@ -867,7 +867,7 @@ define i1 @recursiveGEP_withPtrSub_noninboundSameDefStartAndB(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -903,7 +903,7 @@ define i1 @recursiveGEP_withPtrSub_noninboundSameStartAndB(ptr %val1) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[TEST]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -1004,7 +1004,7 @@ define i1 @recursiveGEP_orcmp(ptr %val1, i64 %val2) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -1036,7 +1036,7 @@ define i1 @recursiveGEP_orcmp_orOperandsCommuted(ptr %val1, i64 %val2) {
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -1069,7 +1069,7 @@ define i1 @recursiveGEP_orcmpMultiUse(ptr %val1, i64 %val2, ptr %dv1, ptr %dv2)
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TEST_0_I]], align 2
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
diff --git a/llvm/test/Assembler/fast-math-flags.ll b/llvm/test/Assembler/fast-math-flags.ll
index 116f9305af8d3..9c08e9da1d19e 100644
--- a/llvm/test/Assembler/fast-math-flags.ll
+++ b/llvm/test/Assembler/fast-math-flags.ll
@@ -17,6 +17,8 @@ entry:
   %select = load i1, ptr @select
 ; CHECK:  %arr = load [3 x float], ptr @arr
   %arr    = load [3 x float], ptr @arr
+; CHECK:  %scalable = load <vscale x 3 x float>, ptr @vec
+  %scalable = load <vscale x 3 x float>, ptr @vec
 
 ; CHECK:  %a = fadd float %x, %y
   %a = fadd float %x, %y
@@ -42,6 +44,18 @@ entry:
   %f = fneg float %x
 ; CHECK:  %f_vec = fneg <3 x float> %vec
   %f_vec = fneg <3 x float> %vec
+; CHECK: %g = fpext float %x to double
+  %g = fpext float %x to double
+; CHECK: %g_vec = fpext <3 x float> %vec to <3 x double>
+  %g_vec = fpext <3 x float> %vec to <3 x double>
+; CHECK: %g_scalable = fpext <vscale x 3 x float> %scalable to <vscale x 3 x double>
+  %g_scalable = fpext <vscale x 3 x float> %scalable to <vscale x 3 x double>
+; CHECK: %h = fptrunc float %x to half
+  %h = fptrunc float %x to half
+; CHECK: %h_vec = fptrunc <3 x float> %vec to <3 x half>
+  %h_vec = fptrunc <3 x float> %vec to <3 x half>
+; CHECK: %h_scalable = fptrunc <vscale x 3 x float> %scalable to <vscale x 3 x half>
+  %h_scalable = fptrunc <vscale x 3 x float> %scalable to <vscale x 3 x half>
 ; CHECK:  ret float %f
   ret  float %f
 }
@@ -55,6 +69,8 @@ entry:
   %select = load i1, ptr @select
 ; CHECK:  %arr = load [3 x float], ptr @arr
   %arr    = load [3 x float], ptr @arr
+; CHECK:  %scalable = load <vscale x 3 x float>, ptr @vec
+  %scalable = load <vscale x 3 x float>, ptr @vec
 
 ; CHECK:  %a = fadd nnan float %x, %y
   %a = fadd nnan float %x, %y
@@ -80,6 +96,18 @@ entry:
   %f = fneg nnan float %x
 ; CHECK:  %f_vec = fneg nnan <3 x float> %vec
   %f_vec = fneg nnan <3 x float> %vec
+; CHECK: %g = fpext nnan float %x to double
+  %g = fpext nnan float %x to double
+; CHECK: %g_vec = fpext nnan <3 x float> %vec to <3 x double>
+  %g_vec = fpext nnan <3 x float> %vec to <3 x double>
+; CHECK: %g_scalable = fpext nnan <vscale x 3 x float> %scalable to <vscale x 3 x double>
+  %g_scalable = fpext nnan <vscale x 3 x float> %scalable to <vscale x 3 x double>
+; CHECK: %h = fptrunc nnan float %x to half
+  %h = fptrunc nnan float %x to half
+; CHECK: %h_vec = fptrunc nnan <3 x float> %vec to <3 x half>
+  %h_vec = fptrunc nnan <3 x float> %vec to <3 x half>
+; CHECK: %h_scalable = fptrunc nnan <vscale x 3 x float> %scalable to <vscale x 3 x half>
+  %h_scalable = fptrunc nnan <vscale x 3 x float> %scalable to <vscale x 3 x half>
 ; CHECK:  ret float %f
   ret float %f
 }
@@ -93,6 +121,10 @@ entry:
   %b = fadd contract float %x, %y
 ; CHECK: %c = fmul contract float %a, %b
   %c = fmul contract float %a, %b
+; CHECK: %d = fpext contract float %x to double
+  %d = fpext contract float %x to double
+; CHECK: %e = fptrunc contract float %x to half
+  %e = fptrunc contract float %x to half
   ret float %c
 }
 
@@ -104,6 +136,10 @@ define float @reassoc(float %x, float %y) {
   %b = fmul reassoc float %x, %y
 ; CHECK: %c = call reassoc float @foo(float %b)
   %c = call reassoc float @foo(float %b)
+; CHECK: %d = fpext reassoc float %x to double
+  %d = fpext reassoc float %x to double
+; CHECK: %e = fptrunc reassoc float %x to half
+  %e = fptrunc reassoc float %x to half
   ret float %c
 }
 
@@ -127,6 +163,8 @@ entry:
   %select = load i1, ptr @select
 ; CHECK:  %arr = load [3 x float], ptr @arr
   %arr    = load [3 x float], ptr @arr
+; CHECK:  %scalable = load <vscale x 3 x float>, ptr @vec
+  %scalable = load <vscale x 3 x float>, ptr @vec
 
 ; CHECK:  %a = fadd nnan ninf float %x, %y
   %a = fadd ninf nnan float %x, %y
@@ -148,6 +186,18 @@ entry:
   %e = frem nnan float %x, %y
 ; CHECK:  %e_vec = frem nnan ninf <3 x float> %vec, %vec
   %e_vec = frem ninf nnan <3 x float> %vec, %vec
+; CHECK: %f = fpext nnan ninf float %x to double
+  %f = fpext ninf nnan float %x to double
+; CHECK: %f_vec = fpext nnan ninf <3 x float> %vec to <3 x double>
+  %f_vec = fpext ninf nnan <3 x float> %vec to <3 x double>
+; CHECK: %f_scalable = fpext nnan ninf <vscale x 3 x float> %scalable to <vscale x 3 x double>
+  %f_scalable = fpext ninf nnan <vscale x 3 x float> %scalable to <vscale x 3 x double>
+; CHECK: %g = fptrunc nnan ninf float %x to half
+  %g = fptrunc ninf nnan float %x to half
+; CHECK: %g_vec = fptrunc nnan ninf <3 x float> %vec to <3 x half>
+  %g_vec = fptrunc ninf nnan <3 x float> %vec to <3 x half>
+; CHECK: %g_scalable = fptrunc nnan ninf <vscale x 3 x float> %scalable to <vscale x 3 x half>
+  %g_scalable = fptrunc ninf nnan <vscale x 3 x float> %scalable to <vscale x 3 x half>
 ; CHECK:  ret float %e
   ret float %e
 }
diff --git a/llvm/test/Bitcode/Inputs/invalid-initializer.bc b/llvm/test/Bitcode/Inputs/invalid-initializer.bc
new file mode 100644
index 0000000000000..df5b6d37bde5e
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/invalid-initializer.bc differ
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index a849789da536a..a28156cdaa279 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1142,6 +1142,48 @@ define void @fastMathFlagsForStructCalls() {
   ret void
 }
 
+; CHECK-LABEL: fastmathflags_fpext(
+define void @fastmathflags_fpext(float %op1) {
+  %f.nnan = fpext nnan float %op1 to double
+  ; CHECK: %f.nnan = fpext nnan float %op1 to double
+  %f.ninf = fpext ninf float %op1 to double
+  ; CHECK: %f.ninf = fpext ninf float %op1 to double
+  %f.nsz = fpext nsz float %op1 to double
+  ; CHECK: %f.nsz = fpext nsz float %op1 to double
+  %f.arcp = fpext arcp float %op1 to double
+  ; CHECK: %f.arcp = fpext arcp float %op1 to double
+  %f.contract = fpext contract float %op1 to double
+  ; CHECK: %f.contract = fpext contract float %op1 to double
+  %f.afn = fpext afn float %op1 to double
+  ; CHECK: %f.afn = fpext afn float %op1 to double
+  %f.reassoc = fpext reassoc float %op1 to double
+  ; CHECK: %f.reassoc = fpext reassoc float %op1 to double
+  %f.fast = fpext fast float %op1 to double
+  ; CHECK: %f.fast = fpext fast float %op1 to double
+  ret void
+}
+
+; CHECK-LABEL: fastmathflags_fptrunc(
+define void @fastmathflags_fptrunc(float %op1) {
+  %f.nnan = fptrunc nnan float %op1 to half
+  ; CHECK: %f.nnan = fptrunc nnan float %op1 to half
+  %f.ninf = fptrunc ninf float %op1 to half
+  ; CHECK: %f.ninf = fptrunc ninf float %op1 to half
+  %f.nsz = fptrunc nsz float %op1 to half
+  ; CHECK: %f.nsz = fptrunc nsz float %op1 to half
+  %f.arcp = fptrunc arcp float %op1 to half
+  ; CHECK: %f.arcp = fptrunc arcp float %op1 to half
+  %f.contract = fptrunc contract float %op1 to half
+  ; CHECK: %f.contract = fptrunc contract float %op1 to half
+  %f.afn = fptrunc afn float %op1 to half
+  ; CHECK: %f.afn = fptrunc afn float %op1 to half
+  %f.reassoc = fptrunc reassoc float %op1 to half
+  ; CHECK: %f.reassoc = fptrunc reassoc float %op1 to half
+  %f.fast = fptrunc fast float %op1 to half
+  ; CHECK: %f.fast = fptrunc fast float %op1 to half
+  ret void
+}
+
 ;; Type System
 %opaquety = type opaque
 define void @typesystem() {
diff --git a/llvm/test/Bitcode/invalid.test b/llvm/test/Bitcode/invalid.test
index cf5dccb469d7d..7bf1c86c035e6 100644
--- a/llvm/test/Bitcode/invalid.test
+++ b/llvm/test/Bitcode/invalid.test
@@ -286,3 +286,8 @@ RUN: not llvm-dis -disable-output %p/Inputs/invalid-forward-declare.bc 2>&1 | \
 RUN:   FileCheck --check-prefix=INVALID-FORWARD-DECLARE %s
 
 INVALID-FORWARD-DECLARE: Assigned value does not match type of forward declaration
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-initializer.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-INITIALIZER %s
+
+INVALID-INITIALIZER: Invalid record
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-cast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-cast.mir
index b045deebc56e0..8eab47171e0d1 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-cast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-cast.mir
@@ -217,3 +217,27 @@ body:             |
     %large:_(<2 x s64>) = G_ANYEXT %bv(<2 x s32>)
     $q0 = COPY %large(<2 x s64>)
     $d0 = COPY %bv(<2 x s32>)
+...
+---
+name:            test_undef_ptrtoint
+legalized: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_undef_ptrtoint
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $q0 = COPY [[DEF]](<2 x s64>)
+    %0:_(<2 x p0>) = G_IMPLICIT_DEF
+    %1:_(<2 x s64>) = G_PTRTOINT %0(<2 x p0>)
+    $q0 = COPY %1(<2 x s64>)
+...
+---
+name:            test_undef_inttoptr
+legalized: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_undef_inttoptr
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x p0>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $q0 = COPY [[DEF]](<2 x p0>)
+    %0:_(<2 x s64>) = G_IMPLICIT_DEF
+    %1:_(<2 x p0>) = G_INTTOPTR %0(<2 x s64>)
+    $q0 = COPY %1(<2 x p0>)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll
index 88828950353a7..4ab347c37f0d7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll
@@ -20,9 +20,8 @@ define i64 @expect_with_probability_i64(i64 %arg0) {
   ; CHECK-NEXT:   liveins: $x0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x0
-  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-  ; CHECK-NEXT:   [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.expect.with.probability), [[COPY]](s64), [[C]](s64), double 5.000000e-01
-  ; CHECK-NEXT:   $x0 = COPY [[INT]](s64)
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+  ; CHECK-NEXT:   $x0 = COPY [[COPY1]](s64)
   ; CHECK-NEXT:   RET_ReallyLR implicit $x0
   %expval = call i64 @llvm.expect.with.probability.i64(i64 %arg0, i64 1, double 0.5)
   ret i64 %expval
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index d35bface7cb48..7c9c958b5a818 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -70,6 +70,14 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 #
+# DEBUG-NEXT: G_ABDS (opcode 65): 1 type index, 0 imm indices
+# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+#
+# DEBUG-NEXT:G_ABDU (opcode 66): 1 type index, 0 imm indices
+# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+#
 # DEBUG-NEXT: G_IMPLICIT_DEF (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: {{[0-9]+}}, OK
 # DEBUG-NEXT: .. the first uncovered imm index: {{[0-9]+}}, OK
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
index f1d1b691fe1aa..216f94b2b51e3 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
@@ -57,11 +57,11 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: inlineasm_virt_reg_output
-    ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2490378 /* regdef:GPR32common */, def %0
+    ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
-    INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2490378 /* regdef:GPR32common */, def %0:gpr32common
+    INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0:gpr32common
     %1:_(s32) = COPY %0
     $w0 = COPY %1(s32)
     RET_ReallyLR implicit $w0
@@ -75,12 +75,12 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: inlineasm_virt_mixed_types
-    ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2490378 /* regdef:GPR32common */, def %0, 3342346 /* regdef:FPR64 */, def %1
+    ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0, 3735562 /* regdef:FPR64 */, def %1
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr(s64) = COPY %1
     ; CHECK-NEXT: $d0 = COPY [[COPY1]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2490378 /* regdef:GPR32common */, def %0:gpr32common, 3342346 /* regdef:FPR64 */, def %1:fpr64
+    INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0:gpr32common, 3735562 /* regdef:FPR64 */, def %1:fpr64
     %3:_(s32) = COPY %0
     %4:_(s64) = COPY %1
     $d0 = COPY %4(s64)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-crash.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-crash.ll
index ef54cc4bbf718..482135b721da4 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-crash.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-crash.ll
@@ -10,14 +10,18 @@ target triple = "aarch64-unknown-linux-gnu"
 define dso_local i32 @dupext_crashtest(i32 %e) local_unnamed_addr {
 ; CHECK-LABEL: dupext_crashtest:
 ; CHECK:       // %bb.0: // %for.body.lr.ph
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    dup v0.2s, w8
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr d1, [x8]
-; CHECK-NEXT:    smull v1.2d, v0.2s, v1.2s
-; CHECK-NEXT:    xtn v1.2s, v1.2d
-; CHECK-NEXT:    str d1, [x8]
+; CHECK-NEXT:    ldr d0, [x8]
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    mul w9, w0, w9
+; CHECK-NEXT:    mul w8, w0, w8
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    str d0, [x8]
 ; CHECK-NEXT:    b .LBB0_1
 for.body.lr.ph:
   %conv314 = zext i32 %e to i64
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
index 0c7a61739695f..fb6575cc0ee83 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
@@ -5,9 +5,8 @@
 define void @matrix_mul_unsigned(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
 ; CHECK-SD-LABEL: matrix_mul_unsigned:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    dup v0.4h, w3
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4h, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
 ; CHECK-SD-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
@@ -91,9 +90,8 @@ for.end12:                                        ; preds = %vector.body
 define void @matrix_mul_signed(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
 ; CHECK-SD-LABEL: matrix_mul_signed:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    sxth w8, w3
+; CHECK-SD-NEXT:    dup v0.4h, w3
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4h, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
 ; CHECK-SD-NEXT:  .LBB1_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
@@ -179,9 +177,8 @@ for.end12:                                        ; preds = %vector.body
 define void @matrix_mul_double_shuffle(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
 ; CHECK-SD-LABEL: matrix_mul_double_shuffle:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    dup v0.4h, w3
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4h, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0 def $x0
 ; CHECK-SD-NEXT:  .LBB2_1: // %vector.body
@@ -261,44 +258,44 @@ define void @larger_smull(ptr nocapture noundef readonly %x, i16 noundef %y, ptr
 ; CHECK-SD-NEXT:    cmp w3, #1
 ; CHECK-SD-NEXT:    b.lt .LBB3_8
 ; CHECK-SD-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-SD-NEXT:    sxth w8, w1
 ; CHECK-SD-NEXT:    cmp w3, #15
-; CHECK-SD-NEXT:    mov w9, w3
+; CHECK-SD-NEXT:    mov w8, w3
 ; CHECK-SD-NEXT:    b.hi .LBB3_3
 ; CHECK-SD-NEXT:  // %bb.2:
-; CHECK-SD-NEXT:    mov x10, xzr
+; CHECK-SD-NEXT:    mov x9, xzr
 ; CHECK-SD-NEXT:    b .LBB3_6
 ; CHECK-SD-NEXT:  .LBB3_3: // %vector.ph
-; CHECK-SD-NEXT:    dup v0.8h, w8
-; CHECK-SD-NEXT:    and x10, x9, #0xfffffff0
-; CHECK-SD-NEXT:    add x11, x2, #32
-; CHECK-SD-NEXT:    add x12, x0, #16
-; CHECK-SD-NEXT:    mov x13, x10
+; CHECK-SD-NEXT:    dup v0.8h, w1
+; CHECK-SD-NEXT:    and x9, x8, #0xfffffff0
+; CHECK-SD-NEXT:    add x10, x2, #32
+; CHECK-SD-NEXT:    add x11, x0, #16
+; CHECK-SD-NEXT:    mov x12, x9
 ; CHECK-SD-NEXT:  .LBB3_4: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-SD-NEXT:    ldp q1, q2, [x12, #-16]
-; CHECK-SD-NEXT:    subs x13, x13, #16
-; CHECK-SD-NEXT:    add x12, x12, #32
+; CHECK-SD-NEXT:    ldp q1, q2, [x11, #-16]
+; CHECK-SD-NEXT:    subs x12, x12, #16
+; CHECK-SD-NEXT:    add x11, x11, #32
 ; CHECK-SD-NEXT:    smull2 v3.4s, v0.8h, v1.8h
 ; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
 ; CHECK-SD-NEXT:    smull2 v4.4s, v0.8h, v2.8h
 ; CHECK-SD-NEXT:    smull v2.4s, v0.4h, v2.4h
-; CHECK-SD-NEXT:    stp q1, q3, [x11, #-32]
-; CHECK-SD-NEXT:    stp q2, q4, [x11], #64
+; CHECK-SD-NEXT:    stp q1, q3, [x10, #-32]
+; CHECK-SD-NEXT:    stp q2, q4, [x10], #64
 ; CHECK-SD-NEXT:    b.ne .LBB3_4
 ; CHECK-SD-NEXT:  // %bb.5: // %middle.block
-; CHECK-SD-NEXT:    cmp x10, x9
+; CHECK-SD-NEXT:    cmp x9, x8
 ; CHECK-SD-NEXT:    b.eq .LBB3_8
 ; CHECK-SD-NEXT:  .LBB3_6: // %for.body.preheader1
-; CHECK-SD-NEXT:    add x11, x2, x10, lsl #2
-; CHECK-SD-NEXT:    add x12, x0, x10, lsl #1
-; CHECK-SD-NEXT:    sub x9, x9, x10
+; CHECK-SD-NEXT:    sxth w10, w1
+; CHECK-SD-NEXT:    add x11, x2, x9, lsl #2
+; CHECK-SD-NEXT:    add x12, x0, x9, lsl #1
+; CHECK-SD-NEXT:    sub x8, x8, x9
 ; CHECK-SD-NEXT:  .LBB3_7: // %for.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-SD-NEXT:    ldrsh w10, [x12], #2
-; CHECK-SD-NEXT:    subs x9, x9, #1
-; CHECK-SD-NEXT:    mul w10, w10, w8
-; CHECK-SD-NEXT:    str w10, [x11], #4
+; CHECK-SD-NEXT:    ldrsh w9, [x12], #2
+; CHECK-SD-NEXT:    subs x8, x8, #1
+; CHECK-SD-NEXT:    mul w9, w9, w10
+; CHECK-SD-NEXT:    str w9, [x11], #4
 ; CHECK-SD-NEXT:    b.ne .LBB3_7
 ; CHECK-SD-NEXT:  .LBB3_8: // %for.cond.cleanup
 ; CHECK-SD-NEXT:    ret
@@ -424,43 +421,43 @@ define void @larger_umull(ptr nocapture noundef readonly %x, i16 noundef %y, ptr
 ; CHECK-SD-NEXT:    b.lt .LBB4_8
 ; CHECK-SD-NEXT:  // %bb.1: // %for.body.preheader
 ; CHECK-SD-NEXT:    cmp w3, #15
-; CHECK-SD-NEXT:    and w8, w1, #0xffff
-; CHECK-SD-NEXT:    mov w9, w3
+; CHECK-SD-NEXT:    mov w8, w3
 ; CHECK-SD-NEXT:    b.hi .LBB4_3
 ; CHECK-SD-NEXT:  // %bb.2:
-; CHECK-SD-NEXT:    mov x10, xzr
+; CHECK-SD-NEXT:    mov x9, xzr
 ; CHECK-SD-NEXT:    b .LBB4_6
 ; CHECK-SD-NEXT:  .LBB4_3: // %vector.ph
-; CHECK-SD-NEXT:    dup v0.8h, w8
-; CHECK-SD-NEXT:    and x10, x9, #0xfffffff0
-; CHECK-SD-NEXT:    add x11, x2, #32
-; CHECK-SD-NEXT:    add x12, x0, #16
-; CHECK-SD-NEXT:    mov x13, x10
+; CHECK-SD-NEXT:    dup v0.8h, w1
+; CHECK-SD-NEXT:    and x9, x8, #0xfffffff0
+; CHECK-SD-NEXT:    add x10, x2, #32
+; CHECK-SD-NEXT:    add x11, x0, #16
+; CHECK-SD-NEXT:    mov x12, x9
 ; CHECK-SD-NEXT:  .LBB4_4: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-SD-NEXT:    ldp q1, q2, [x12, #-16]
-; CHECK-SD-NEXT:    subs x13, x13, #16
-; CHECK-SD-NEXT:    add x12, x12, #32
+; CHECK-SD-NEXT:    ldp q1, q2, [x11, #-16]
+; CHECK-SD-NEXT:    subs x12, x12, #16
+; CHECK-SD-NEXT:    add x11, x11, #32
 ; CHECK-SD-NEXT:    umull2 v3.4s, v0.8h, v1.8h
 ; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
 ; CHECK-SD-NEXT:    umull2 v4.4s, v0.8h, v2.8h
 ; CHECK-SD-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-SD-NEXT:    stp q1, q3, [x11, #-32]
-; CHECK-SD-NEXT:    stp q2, q4, [x11], #64
+; CHECK-SD-NEXT:    stp q1, q3, [x10, #-32]
+; CHECK-SD-NEXT:    stp q2, q4, [x10], #64
 ; CHECK-SD-NEXT:    b.ne .LBB4_4
 ; CHECK-SD-NEXT:  // %bb.5: // %middle.block
-; CHECK-SD-NEXT:    cmp x10, x9
+; CHECK-SD-NEXT:    cmp x9, x8
 ; CHECK-SD-NEXT:    b.eq .LBB4_8
 ; CHECK-SD-NEXT:  .LBB4_6: // %for.body.preheader1
-; CHECK-SD-NEXT:    add x11, x2, x10, lsl #2
-; CHECK-SD-NEXT:    add x12, x0, x10, lsl #1
-; CHECK-SD-NEXT:    sub x9, x9, x10
+; CHECK-SD-NEXT:    add x10, x2, x9, lsl #2
+; CHECK-SD-NEXT:    add x11, x0, x9, lsl #1
+; CHECK-SD-NEXT:    and w12, w1, #0xffff
+; CHECK-SD-NEXT:    sub x8, x8, x9
 ; CHECK-SD-NEXT:  .LBB4_7: // %for.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-SD-NEXT:    ldrh w10, [x12], #2
-; CHECK-SD-NEXT:    subs x9, x9, #1
-; CHECK-SD-NEXT:    mul w10, w10, w8
-; CHECK-SD-NEXT:    str w10, [x11], #4
+; CHECK-SD-NEXT:    ldrh w9, [x11], #2
+; CHECK-SD-NEXT:    subs x8, x8, #1
+; CHECK-SD-NEXT:    mul w9, w9, w12
+; CHECK-SD-NEXT:    str w9, [x10], #4
 ; CHECK-SD-NEXT:    b.ne .LBB4_7
 ; CHECK-SD-NEXT:  .LBB4_8: // %for.cond.cleanup
 ; CHECK-SD-NEXT:    ret
@@ -470,47 +467,48 @@ define void @larger_umull(ptr nocapture noundef readonly %x, i16 noundef %y, ptr
 ; CHECK-GI-NEXT:    cmp w3, #0
 ; CHECK-GI-NEXT:    b.le .LBB4_7
 ; CHECK-GI-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-GI-NEXT:    mov x9, xzr
+; CHECK-GI-NEXT:    mov x8, xzr
 ; CHECK-GI-NEXT:    cmp w3, #16
-; CHECK-GI-NEXT:    and w8, w1, #0xffff
-; CHECK-GI-NEXT:    mov w10, w3
+; CHECK-GI-NEXT:    mov w9, w3
 ; CHECK-GI-NEXT:    b.lo .LBB4_5
 ; CHECK-GI-NEXT:  // %bb.2: // %vector.ph
-; CHECK-GI-NEXT:    dup v0.4s, w8
-; CHECK-GI-NEXT:    and x9, x10, #0xfffffff0
-; CHECK-GI-NEXT:    add x11, x2, #32
-; CHECK-GI-NEXT:    add x12, x0, #16
-; CHECK-GI-NEXT:    mov x13, x9
+; CHECK-GI-NEXT:    and x8, x9, #0xfffffff0
+; CHECK-GI-NEXT:    add x10, x2, #32
+; CHECK-GI-NEXT:    add x11, x0, #16
+; CHECK-GI-NEXT:    mov x12, x8
 ; CHECK-GI-NEXT:  .LBB4_3: // %vector.body
 ; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-GI-NEXT:    ldp q1, q2, [x12, #-16]
-; CHECK-GI-NEXT:    mov x14, x11
-; CHECK-GI-NEXT:    subs x13, x13, #16
-; CHECK-GI-NEXT:    add x12, x12, #32
-; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ldp q0, q1, [x11, #-16]
+; CHECK-GI-NEXT:    and w13, w1, #0xffff
+; CHECK-GI-NEXT:    dup v2.4s, w13
+; CHECK-GI-NEXT:    mov x13, x10
+; CHECK-GI-NEXT:    subs x12, x12, #16
+; CHECK-GI-NEXT:    add x11, x11, #32
+; CHECK-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    ushll v4.4s, v1.4h, #0
 ; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v4.4s, v2.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    mul v3.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    mul v4.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    stp q3, q1, [x14, #-32]!
-; CHECK-GI-NEXT:    stp q4, q2, [x11], #64
+; CHECK-GI-NEXT:    mul v3.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    mul v4.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mul v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    stp q3, q0, [x13, #-32]!
+; CHECK-GI-NEXT:    stp q4, q1, [x10], #64
 ; CHECK-GI-NEXT:    b.ne .LBB4_3
 ; CHECK-GI-NEXT:  // %bb.4: // %middle.block
-; CHECK-GI-NEXT:    cmp x9, x10
+; CHECK-GI-NEXT:    cmp x8, x9
 ; CHECK-GI-NEXT:    b.eq .LBB4_7
 ; CHECK-GI-NEXT:  .LBB4_5: // %for.body.preheader1
-; CHECK-GI-NEXT:    add x11, x2, x9, lsl #2
-; CHECK-GI-NEXT:    add x12, x0, x9, lsl #1
-; CHECK-GI-NEXT:    sub x9, x10, x9
+; CHECK-GI-NEXT:    add x10, x2, x8, lsl #2
+; CHECK-GI-NEXT:    add x11, x0, x8, lsl #1
+; CHECK-GI-NEXT:    and w12, w1, #0xffff
+; CHECK-GI-NEXT:    sub x8, x9, x8
 ; CHECK-GI-NEXT:  .LBB4_6: // %for.body
 ; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-GI-NEXT:    ldrh w10, [x12], #2
-; CHECK-GI-NEXT:    subs x9, x9, #1
-; CHECK-GI-NEXT:    mul w10, w10, w8
-; CHECK-GI-NEXT:    str w10, [x11], #4
+; CHECK-GI-NEXT:    ldrh w9, [x11], #2
+; CHECK-GI-NEXT:    subs x8, x8, #1
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    str w9, [x10], #4
 ; CHECK-GI-NEXT:    b.ne .LBB4_6
 ; CHECK-GI-NEXT:  .LBB4_7: // %for.cond.cleanup
 ; CHECK-GI-NEXT:    ret
@@ -600,7 +598,7 @@ define i16 @red_mla_dup_ext_u8_s8_s16(ptr noalias nocapture noundef readonly %A,
 ; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-SD-NEXT:    and x11, x10, #0xfffffff0
-; CHECK-SD-NEXT:    dup v2.8h, w9
+; CHECK-SD-NEXT:    fmov s2, w9
 ; CHECK-SD-NEXT:    add x8, x0, #8
 ; CHECK-SD-NEXT:    mov x12, x11
 ; CHECK-SD-NEXT:  .LBB5_5: // %vector.body
@@ -610,8 +608,8 @@ define i16 @red_mla_dup_ext_u8_s8_s16(ptr noalias nocapture noundef readonly %A,
 ; CHECK-SD-NEXT:    add x8, x8, #16
 ; CHECK-SD-NEXT:    ushll v3.8h, v3.8b, #0
 ; CHECK-SD-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-SD-NEXT:    mla v0.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    mla v1.8h, v2.8h, v4.8h
+; CHECK-SD-NEXT:    mla v0.8h, v3.8h, v2.h[0]
+; CHECK-SD-NEXT:    mla v1.8h, v4.8h, v2.h[0]
 ; CHECK-SD-NEXT:    b.ne .LBB5_5
 ; CHECK-SD-NEXT:  // %bb.6: // %middle.block
 ; CHECK-SD-NEXT:    add v0.8h, v1.8h, v0.8h
@@ -1025,9 +1023,8 @@ exit:
 define void @matrix_mul_unsigned_and(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
 ; CHECK-SD-LABEL: matrix_mul_unsigned_and:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    dup v0.4h, w3
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4h, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
 ; CHECK-SD-NEXT:  .LBB10_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
@@ -1111,9 +1108,8 @@ for.end12:                                        ; preds = %vector.body
 define void @matrix_mul_unsigned_and_double(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
 ; CHECK-SD-LABEL: matrix_mul_unsigned_and_double:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    dup v0.8h, w3
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.8h, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff0
 ; CHECK-SD-NEXT:  .LBB11_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
@@ -1207,10 +1203,10 @@ for.end12:                                        ; preds = %vector.body
 define void @matrix_mul_signed_and(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
 ; CHECK-SD-LABEL: matrix_mul_signed_and:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    and w9, w3, #0xffff
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4s, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:    fmov s0, w9
 ; CHECK-SD-NEXT:  .LBB12_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
@@ -1220,8 +1216,8 @@ define void @matrix_mul_signed_and(i32 %N, ptr nocapture %C, ptr nocapture reado
 ; CHECK-SD-NEXT:    add w0, w0, #8
 ; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
 ; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    mul v1.4s, v1.4s, v0.s[0]
+; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v0.s[0]
 ; CHECK-SD-NEXT:    stp q1, q2, [x9]
 ; CHECK-SD-NEXT:    b.ne .LBB12_1
 ; CHECK-SD-NEXT:  // %bb.2: // %for.end12
@@ -1295,10 +1291,10 @@ for.end12:                                        ; preds = %vector.body
 define void @matrix_mul_signed_and_double(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
 ; CHECK-SD-LABEL: matrix_mul_signed_and_double:
 ; CHECK-SD:       // %bb.0: // %vector.header
-; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    and w9, w3, #0xffff
 ; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-SD-NEXT:    dup v0.4s, w8
 ; CHECK-SD-NEXT:    and x8, x0, #0xfffffff0
+; CHECK-SD-NEXT:    fmov s0, w9
 ; CHECK-SD-NEXT:  .LBB13_1: // %vector.body
 ; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
@@ -1311,10 +1307,10 @@ define void @matrix_mul_signed_and_double(i32 %N, ptr nocapture %C, ptr nocaptur
 ; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
 ; CHECK-SD-NEXT:    sshll2 v4.4s, v2.8h, #0
 ; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    mul v3.4s, v0.4s, v3.4s
-; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    mul v4.4s, v0.4s, v4.4s
-; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    mul v3.4s, v3.4s, v0.s[0]
+; CHECK-SD-NEXT:    mul v1.4s, v1.4s, v0.s[0]
+; CHECK-SD-NEXT:    mul v4.4s, v4.4s, v0.s[0]
+; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v0.s[0]
 ; CHECK-SD-NEXT:    stp q1, q3, [x9]
 ; CHECK-SD-NEXT:    stp q2, q4, [x9, #32]
 ; CHECK-SD-NEXT:    b.ne .LBB13_1
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index 0fe2bbe2c449f..3b589d3480179 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -2032,7 +2032,7 @@ define void @smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
 ; CHECK-GI-NEXT:    ldr q2, [x1, #16]
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
-; CHECK-GI-NEXT:    umlsl v1.8h, v0.8b, v2.8b
+; CHECK-GI-NEXT:    smlsl v1.8h, v0.8b, v2.8b
 ; CHECK-GI-NEXT:    str q1, [x0]
 ; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
@@ -2067,7 +2067,7 @@ define void @umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {
 ; CHECK-GI-NEXT:    ldr q2, [x1, #16]
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
-; CHECK-GI-NEXT:    smlsl v1.8h, v0.8b, v2.8b
+; CHECK-GI-NEXT:    umlsl v1.8h, v0.8b, v2.8b
 ; CHECK-GI-NEXT:    str q1, [x0]
 ; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
@@ -2102,7 +2102,7 @@ define void @smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {
 ; CHECK-GI-NEXT:    ldr q2, [x1, #16]
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
-; CHECK-GI-NEXT:    umlsl v1.4s, v0.4h, v2.4h
+; CHECK-GI-NEXT:    smlsl v1.4s, v0.4h, v2.4h
 ; CHECK-GI-NEXT:    str q1, [x0]
 ; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
@@ -2137,7 +2137,7 @@ define void @umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {
 ; CHECK-GI-NEXT:    ldr q2, [x1, #16]
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
-; CHECK-GI-NEXT:    smlsl v1.4s, v0.4h, v2.4h
+; CHECK-GI-NEXT:    umlsl v1.4s, v0.4h, v2.4h
 ; CHECK-GI-NEXT:    str q1, [x0]
 ; CHECK-GI-NEXT:    ret
   %5 = getelementptr inbounds i32, ptr %3, i64 4
@@ -2202,8 +2202,8 @@ define void @smlsl_smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3,
 ; CHECK-GI-NEXT:    mov d3, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
 ; CHECK-GI-NEXT:    xtn v4.8b, v4.8h
-; CHECK-GI-NEXT:    umull v2.8h, v3.8b, v2.8b
-; CHECK-GI-NEXT:    umlal v2.8h, v0.8b, v4.8b
+; CHECK-GI-NEXT:    smull v2.8h, v3.8b, v2.8b
+; CHECK-GI-NEXT:    smlal v2.8h, v0.8b, v4.8b
 ; CHECK-GI-NEXT:    sub v0.8h, v1.8h, v2.8h
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -2248,8 +2248,8 @@ define void @umlsl_umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3,
 ; CHECK-GI-NEXT:    mov d3, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.8b, v2.8h
 ; CHECK-GI-NEXT:    xtn v4.8b, v4.8h
-; CHECK-GI-NEXT:    smull v2.8h, v3.8b, v2.8b
-; CHECK-GI-NEXT:    smlal v2.8h, v0.8b, v4.8b
+; CHECK-GI-NEXT:    umull v2.8h, v3.8b, v2.8b
+; CHECK-GI-NEXT:    umlal v2.8h, v0.8b, v4.8b
 ; CHECK-GI-NEXT:    sub v0.8h, v1.8h, v2.8h
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -2294,8 +2294,8 @@ define void @smlsl_smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3,
 ; CHECK-GI-NEXT:    mov d3, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
 ; CHECK-GI-NEXT:    xtn v4.4h, v4.4s
-; CHECK-GI-NEXT:    umull v2.4s, v3.4h, v2.4h
-; CHECK-GI-NEXT:    umlal v2.4s, v0.4h, v4.4h
+; CHECK-GI-NEXT:    smull v2.4s, v3.4h, v2.4h
+; CHECK-GI-NEXT:    smlal v2.4s, v0.4h, v4.4h
 ; CHECK-GI-NEXT:    sub v0.4s, v1.4s, v2.4s
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -2340,8 +2340,8 @@ define void @umlsl_umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3,
 ; CHECK-GI-NEXT:    mov d3, v0.d[1]
 ; CHECK-GI-NEXT:    xtn v2.4h, v2.4s
 ; CHECK-GI-NEXT:    xtn v4.4h, v4.4s
-; CHECK-GI-NEXT:    smull v2.4s, v3.4h, v2.4h
-; CHECK-GI-NEXT:    smlal v2.4s, v0.4h, v4.4h
+; CHECK-GI-NEXT:    umull v2.4s, v3.4h, v2.4h
+; CHECK-GI-NEXT:    umlal v2.4s, v0.4h, v4.4h
 ; CHECK-GI-NEXT:    sub v0.4s, v1.4s, v2.4s
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -2382,7 +2382,7 @@ define <2 x i32> @do_stuff(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-GI-NEXT:    mov d2, v1.d[1]
-; CHECK-GI-NEXT:    umull v0.2d, v2.2s, v0.2s
+; CHECK-GI-NEXT:    smull v0.2d, v2.2s, v0.2s
 ; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
index ff66206228a4a..0e8465e741993 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
@@ -13,7 +13,7 @@ define <vscale x 16 x i8> @test_svadd_i8(<vscale x 16 x i8> %Zn, <vscale x 16 x
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:zpr_3b = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"add $0.b, $1.b, $2.b", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %2, 5767177 /* reguse:ZPR */, [[COPY2]], 6357001 /* reguse:ZPR_3b */, [[COPY3]]
+  ; CHECK-NEXT:   INLINEASM &"add $0.b, $1.b, $2.b", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %2, {{[0-9]+}} /* reguse:ZPR */, [[COPY2]], {{[0-9]+}} /* reguse:ZPR_3b */, [[COPY3]]
   ; CHECK-NEXT:   $z0 = COPY %2
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 16 x i8> asm "add $0.b, $1.b, $2.b", "=w,w,y"(<vscale x 16 x i8> %Zn, <vscale x 16 x i8> %Zm)
@@ -29,7 +29,7 @@ define <vscale x 2 x i64> @test_svsub_i64(<vscale x 2 x i64> %Zn, <vscale x 2 x
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:zpr_4b = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"sub $0.d, $1.d, $2.d", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %2, 5767177 /* reguse:ZPR */, [[COPY2]], 6029321 /* reguse:ZPR_4b */, [[COPY3]]
+  ; CHECK-NEXT:   INLINEASM &"sub $0.d, $1.d, $2.d", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %2, {{[0-9]+}} /* reguse:ZPR */, [[COPY2]], {{[0-9]+}} /* reguse:ZPR_4b */, [[COPY3]]
   ; CHECK-NEXT:   $z0 = COPY %2
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 2 x i64> asm "sub $0.d, $1.d, $2.d", "=w,w,x"(<vscale x 2 x i64> %Zn, <vscale x 2 x i64> %Zm)
@@ -45,7 +45,7 @@ define <vscale x 8 x half> @test_svfmul_f16(<vscale x 8 x half> %Zn, <vscale x 8
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:zpr_3b = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"fmul $0.h, $1.h, $2.h", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %2, 5767177 /* reguse:ZPR */, [[COPY2]], 6357001 /* reguse:ZPR_3b */, [[COPY3]]
+  ; CHECK-NEXT:   INLINEASM &"fmul $0.h, $1.h, $2.h", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %2, {{[0-9]+}} /* reguse:ZPR */, [[COPY2]], {{[0-9]+}} /* reguse:ZPR_3b */, [[COPY3]]
   ; CHECK-NEXT:   $z0 = COPY %2
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 8 x half> asm "fmul $0.h, $1.h, $2.h", "=w,w,y"(<vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
@@ -61,7 +61,7 @@ define <vscale x 4 x float> @test_svfmul_f(<vscale x 4 x float> %Zn, <vscale x 4
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:zpr_4b = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"fmul $0.s, $1.s, $2.s", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %2, 5767177 /* reguse:ZPR */, [[COPY2]], 6029321 /* reguse:ZPR_4b */, [[COPY3]]
+  ; CHECK-NEXT:   INLINEASM &"fmul $0.s, $1.s, $2.s", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %2, {{[0-9]+}} /* reguse:ZPR */, [[COPY2]], {{[0-9]+}} /* reguse:ZPR_4b */, [[COPY3]]
   ; CHECK-NEXT:   $z0 = COPY %2
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 4 x float> asm "fmul $0.s, $1.s, $2.s", "=w,w,x"(<vscale x 4 x float> %Zn, <vscale x 4 x float> %Zm)
@@ -79,7 +79,7 @@ define <vscale x 8 x half> @test_svfadd_f16(<vscale x 16 x i1> %Pg, <vscale x 8
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:ppr_3b = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:zpr = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"fadd $0.h, $1/m, $2.h, $3.h", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %3, 720905 /* reguse:PPR_3b */, [[COPY3]], 5767177 /* reguse:ZPR */, [[COPY4]], 5767177 /* reguse:ZPR */, [[COPY5]]
+  ; CHECK-NEXT:   INLINEASM &"fadd $0.h, $1/m, $2.h, $3.h", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %3, {{[0-9]+}} /* reguse:PPR_3b */, [[COPY3]], {{[0-9]+}} /* reguse:ZPR */, [[COPY4]], {{[0-9]+}} /* reguse:ZPR */, [[COPY5]]
   ; CHECK-NEXT:   $z0 = COPY %3
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 8 x half> asm "fadd $0.h, $1/m, $2.h, $3.h", "=w,@3Upl,w,w"(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
@@ -95,7 +95,7 @@ define <vscale x 4 x i32> @test_incp(<vscale x 16 x i1> %Pg, <vscale x 4 x i32>
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:ppr = COPY $p0
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:ppr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:zpr = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"incp $0.s, $1", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %2, 458761 /* reguse:PPR */, [[COPY2]], 2147483657 /* reguse tiedto:$0 */, [[COPY3]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"incp $0.s, $1", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %2, {{[0-9]+}} /* reguse:PPR */, [[COPY2]], {{[0-9]+}} /* reguse tiedto:$0 */, [[COPY3]](tied-def 3)
   ; CHECK-NEXT:   $z0 = COPY %2
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 4 x i32> asm "incp $0.s, $1", "=w,@3Upa,0"(<vscale x 16 x i1> %Pg, <vscale x 4 x i32> %Zn)
@@ -113,7 +113,7 @@ define <vscale x 8 x half> @test_svfadd_f16_Uph_constraint(<vscale x 16 x i1> %P
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:ppr_p8to15 = COPY [[COPY2]]
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:zpr = COPY [[COPY1]]
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:zpr = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"fadd $0.h, $1/m, $2.h, $3.h", 0 /* attdialect */, 5767178 /* regdef:ZPR */, def %3, 786441 /* reguse:PPR_p8to15 */, [[COPY3]], 5767177 /* reguse:ZPR */, [[COPY4]], 5767177 /* reguse:ZPR */, [[COPY5]]
+  ; CHECK-NEXT:   INLINEASM &"fadd $0.h, $1/m, $2.h, $3.h", 0 /* attdialect */, {{[0-9]+}} /* regdef:ZPR */, def %3, {{[0-9]+}} /* reguse:PPR_p8to15 */, [[COPY3]], {{[0-9]+}} /* reguse:ZPR */, [[COPY4]], {{[0-9]+}} /* reguse:ZPR */, [[COPY5]]
   ; CHECK-NEXT:   $z0 = COPY %3
   ; CHECK-NEXT:   RET_ReallyLR implicit $z0
   %1 = tail call <vscale x 8 x half> asm "fadd $0.h, $1/m, $2.h, $3.h", "=w,@3Uph,w,w"(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
@@ -129,7 +129,7 @@ define void @explicit_p0(ptr %p) {
   ; CHECK-NEXT:   [[PTRUE_B:%[0-9]+]]:ppr = PTRUE_B 31, implicit $vg
   ; CHECK-NEXT:   $p0 = COPY [[PTRUE_B]]
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $p0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR64common */, def %1, 9 /* reguse */, $p0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
   ; CHECK-NEXT:   RET_ReallyLR
   %1 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.b8(i32 31)
   %2 = tail call i64 asm sideeffect "ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", "=r,{p0},0"(<vscale x 16 x i1> %1, ptr %p)
@@ -145,7 +145,7 @@ define void @explicit_p8_invalid(ptr %p) {
   ; CHECK-NEXT:   [[PTRUE_B:%[0-9]+]]:ppr = PTRUE_B 31, implicit $vg
   ; CHECK-NEXT:   $p8 = COPY [[PTRUE_B]]
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $p8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR64common */, def %1, 9 /* reguse */, $p8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
   ; CHECK-NEXT:   RET_ReallyLR
   %1 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.b8(i32 31)
   %2 = tail call i64 asm sideeffect "ld4w { z0.s, z1.s, z2.s, z3.s }, $1/z, [$0]", "=r,{p8},0"(<vscale x 16 x i1> %1, ptr %p)
@@ -161,7 +161,7 @@ define void @explicit_pn8(ptr %p) {
   ; CHECK-NEXT:   [[PTRUE_C_B:%[0-9]+]]:pnr_p8to15 = PTRUE_C_B implicit $vg
   ; CHECK-NEXT:   $pn8 = COPY [[PTRUE_C_B]]
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn8, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
   ; CHECK-NEXT:   RET_ReallyLR
   %1 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
   %2 = tail call i64 asm sideeffect "ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", "=r,{pn8},0"(target("aarch64.svcount") %1, ptr %p)
@@ -177,7 +177,7 @@ define void @explicit_pn0_invalid(ptr %p) {
   ; CHECK-NEXT:   [[PTRUE_C_B:%[0-9]+]]:pnr_p8to15 = PTRUE_C_B implicit $vg
   ; CHECK-NEXT:   $pn0 = COPY [[PTRUE_C_B]]
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]]
-  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, 3538954 /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR64common */, def %1, 9 /* reguse */, $pn0, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3)
   ; CHECK-NEXT:   RET_ReallyLR
   %1 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
   %2 = tail call i64 asm sideeffect "ld1w { z0.s, z4.s, z8.s, z12.s }, $1/z, [$0]", "=r,{pn0},0"(target("aarch64.svcount") %1, ptr %p)
diff --git a/llvm/test/CodeGen/AArch64/apply-disjoint-flag-in-dagcombine.ll b/llvm/test/CodeGen/AArch64/apply-disjoint-flag-in-dagcombine.ll
new file mode 100644
index 0000000000000..5622f2ae20efd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/apply-disjoint-flag-in-dagcombine.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-none-eabi %s -o - | FileCheck %s
+
+define i32 @test(i32 %a) {
+; CHECK-LABEL: test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add w0, w0, #193
+; CHECK-NEXT:    ret
+entry:
+  %add = add i32 %a, 1
+  %or1 = or disjoint i32 %add, 64
+  %or = or disjoint i32 %or1, 128
+  ret i32 %or
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
index bfef61abd8c12..f8695b62619c0 100644
--- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=arm64-eabi < %s | FileCheck %s
+; RUN: llc -aarch64-enable-subreg-liveness-tracking -mtriple=arm64-eabi < %s | FileCheck %s
 ; rdar://10232252
 
 @object = external hidden global i64, section "__DATA, __objc_ivar", align 8
diff --git a/llvm/test/CodeGen/AArch64/arm64-dup.ll b/llvm/test/CodeGen/AArch64/arm64-dup.ll
index a25763e3b1590..4c28ea7592202 100644
--- a/llvm/test/CodeGen/AArch64/arm64-dup.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-dup.ll
@@ -733,5 +733,5 @@ define <4 x i16> @dup_i16_v4i16_constant() {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI50_0
 ; CHECK-GI-NEXT:    ldr d0, [x8, :lo12:.LCPI50_0]
 ; CHECK-GI-NEXT:    ret
-    ret <4 x i16> <i16 9211, i16 9211, i16 9211, i16 9211>
+  ret <4 x i16> <i16 9211, i16 9211, i16 9211, i16 9211>
 }
diff --git a/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-frame-tail.ll b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-frame-tail.ll
index 32cc850dfa734..cd751840fee48 100644
--- a/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-frame-tail.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-frame-tail.ll
@@ -51,8 +51,8 @@ declare i32 @_Z3hoov() nounwind ssp optsize
 ; CHECK-LABEL:  _OUTLINED_FUNCTION_PROLOG_FRAME48_x30x29x19x20d8d9d10d11:
 ; CHECK:      stp     d11, d10, [sp, #-48]!
 ; CHECK-NEXT: stp     d9, d8, [sp, #16]
-; CHECK-NEXT: stp     x20, x19, [sp, #32]
 ; CHECK-NEXT: add     x29, sp, #48
+; CHECK-NEXT: stp     x20, x19, [sp, #32]
 ; CHECK-NEXT: ret
 
 ; CHECK-LABEL: _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20d8d9d10d11:
@@ -69,8 +69,8 @@ declare i32 @_Z3hoov() nounwind ssp optsize
 ; CHECK-LINUX-LABEL:  OUTLINED_FUNCTION_PROLOG_FRAME32_x19x20x30x29d8d9d10d11:
 ; CHECK-LINUX:      stp     d11, d10, [sp, #-32]!
 ; CHECK-LINUX-NEXT: stp     d9, d8, [sp, #16]
-; CHECK-LINUX-NEXT: stp     x20, x19, [sp, #48]
 ; CHECK-LINUX-NEXT: add     x29, sp, #32
+; CHECK-LINUX-NEXT: stp     x20, x19, [sp, #48]
 ; CHECK-LINUX-NEXT: ret
 
 ; CHECK-LINUX-LABEL: OUTLINED_FUNCTION_EPILOG_TAIL_x19x20x30x29d8d9d10d11:
diff --git a/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog.ll b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog.ll
index fed564afd1893..c2541a6d819b8 100644
--- a/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog.ll
@@ -67,8 +67,8 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
 ; CHECK-LINUX-NEXT: ret
 
 ; CHECK-LINUX-LABEL: OUTLINED_FUNCTION_EPILOG_x19x20x21x22x30x29:
-; CHECK-LINUX:      mov     x16, x30
-; CHECK-LINUX-NEXT: ldp     x20, x19, [sp, #32]
+; CHECK-LINUX:      ldp     x20, x19, [sp, #32]
+; CHECK-LINUX-NEXT: mov     x16, x30
 ; CHECK-LINUX-NEXT: ldp     x22, x21, [sp, #16]
 ; CHECK-LINUX-NEXT: ldp     x29, x30, [sp], #48
 ; CHECK-LINUX-NEXT: ret     x16
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
index a44dd67eed3fb..d2ce7e6cf0320 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst-2.ll
@@ -1,11 +1,11 @@
 ; RUN: llc < %s
 
-; This used to assert with "Overran sorted position" in AssignTopologicalOrder
-; due to a cycle created in performPostLD1Combine.
-
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios7.0.0"
 
+; This used to assert with "Overran sorted position" in AssignTopologicalOrder
+; due to a cycle created in performPostLD1Combine.
+
 ; Function Attrs: nounwind ssp
 define void @f(ptr %P1) #0 {
 entry:
@@ -50,6 +50,37 @@ define <4 x i32> @f3(ptr %p, <4 x i1> %m, <4 x i32> %v1, <4 x i32> %v2) {
   ret <4 x i32> %vret
 }
 
+; This test used to crash in performPostLD1Combine when the combine attempted to
+; replace a load that already had index writeback, resulting in an incorrect
+; CombineTo, which would have changed the number of SDValue results of the
+; instruction.
+define i32 @rdar138004275(ptr %arg, i1 %arg1) {
+bb:
+  br label %bb3
+
+bb2:                                              ; preds = %bb3
+  store volatile <8 x half> %shufflevector10, ptr null, align 16
+  ret i32 0
+
+bb3:                                              ; preds = %bb3, %bb
+  %phi = phi ptr [ null, %bb ], [ %getelementptr11, %bb3 ]
+  %load = load <2 x half>, ptr %phi, align 4
+  %shufflevector = shufflevector <2 x half> %load, <2 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %getelementptr = getelementptr i8, ptr %phi, i64 4
+  %load4 = load half, ptr %getelementptr, align 2
+  %insertelement = insertelement <2 x half> zeroinitializer, half %load4, i64 0
+  %shufflevector5 = shufflevector <2 x half> %insertelement, <2 x half> zeroinitializer, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %shufflevector6 = shufflevector <8 x half> %shufflevector, <8 x half> %shufflevector5, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+  store <8 x half> %shufflevector6, ptr %arg, align 16
+  %getelementptr7 = getelementptr i8, ptr %phi, i64 6
+  %load8 = load <2 x half>, ptr %getelementptr7, align 4
+  %shufflevector9 = shufflevector <2 x half> %load8, <2 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %shufflevector10 = shufflevector <8 x half> %shufflevector9, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 poison, i32 poison, i32 14, i32 15>
+  %getelementptr11 = getelementptr i8, ptr %phi, i64 6
+  br i1 %arg1, label %bb2, label %bb3
+}
+
+
 ; Function Attrs: nounwind readnone
 declare i64 @llvm.objectsize.i64.p0(ptr, i1) #1
 
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
new file mode 100644
index 0000000000000..33997614598c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -0,0 +1,2347 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple aarch64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16
+
+define bfloat @test_fadd(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_fadd:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fadd s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fadd s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fadd bfloat %a, %b
+  ret bfloat %r
+}
+
+define bfloat @test_fsub(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_fsub:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fsub s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fsub:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fsub s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fsub bfloat %a, %b
+  ret bfloat %r
+}
+
+define bfloat @test_fmul(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_fmul:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fmul s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fmul:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fmul s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fmul bfloat %a, %b
+  ret bfloat %r
+}
+
+define bfloat @test_fmadd(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-CVT-LABEL: test_fmadd:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w10, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fmul s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    ubfx w9, w8, #16, #1
+; CHECK-CVT-NEXT:    add w8, w8, w10
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    fmov w9, s2
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    ubfx w9, w8, #16, #1
+; CHECK-CVT-NEXT:    add w8, w8, w10
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fmadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fmov w9, s2
+; CHECK-BF16-NEXT:    fmul s0, s1, s0
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %mul = fmul fast bfloat %a, %b
+  %r = fadd fast bfloat %mul, %c
+  ret bfloat %r
+}
+
+define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_fdiv:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fdiv s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fdiv:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fdiv s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fdiv bfloat %a, %b
+  ret bfloat %r
+}
+
+define bfloat @test_frem(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_frem:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    bl fmodf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_frem:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bl fmodf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = frem bfloat %a, %b
+  ret bfloat %r
+}
+
+define void @test_store(bfloat %a, ptr %b) #0 {
+; CHECK-LABEL: test_store:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str h0, [x0]
+; CHECK-NEXT:    ret
+  store bfloat %a, ptr %b
+  ret void
+}
+
+define bfloat @test_load(ptr %a) #0 {
+; CHECK-LABEL: test_load:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    ret
+  %r = load bfloat, ptr %a
+  ret bfloat %r
+}
+
+declare bfloat @test_callee(bfloat %a, bfloat %b) #0
+
+define bfloat @test_call(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_call:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl test_callee
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %r = call bfloat @test_callee(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_call_flipped(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_call_flipped:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fmov s2, s0
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    fmov s1, s2
+; CHECK-NEXT:    bl test_callee
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %r = call bfloat @test_callee(bfloat %b, bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_tailcall_flipped(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_tailcall_flipped:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s2, s0
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    fmov s1, s2
+; CHECK-NEXT:    b test_callee
+  %r = tail call bfloat @test_callee(bfloat %b, bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_select(bfloat %a, bfloat %b, i1 zeroext %c) #0 {
+; CHECK-LABEL: test_select:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    cmp w0, #0
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
+  %r = select i1 %c, bfloat %a, bfloat %b
+  ret bfloat %r
+}
+
+define bfloat @test_select_cc(bfloat %a, bfloat %b, bfloat %c, bfloat %d) #0 {
+; CHECK-LABEL: test_select_cc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h3 killed $h3 def $s3
+; CHECK-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s2
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fcmp s3, s2
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
+  %cc = fcmp une bfloat %c, %d
+  %r = select i1 %cc, bfloat %a, bfloat %b
+  ret bfloat %r
+}
+
+define float @test_select_cc_f32_f16(float %a, float %b, bfloat %c, bfloat %d) #0 {
+; CHECK-LABEL: test_select_cc_f32_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h3 killed $h3 def $s3
+; CHECK-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s2
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fcmp s3, s2
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    ret
+  %cc = fcmp une bfloat %c, %d
+  %r = select i1 %cc, float %a, float %b
+  ret float %r
+}
+
+define bfloat @test_select_cc_f16_f32(bfloat %a, bfloat %b, float %c, float %d) #0 {
+; CHECK-LABEL: test_select_cc_f16_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcmp s2, s3
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
+  %cc = fcmp une float %c, %d
+  %r = select i1 %cc, bfloat %a, bfloat %b
+  ret bfloat %r
+}
+
+define i1 @test_fcmp_une(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_une:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %r = fcmp une bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ueq(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ueq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w8, eq
+; CHECK-NEXT:    csinc w0, w8, wzr, vc
+; CHECK-NEXT:    ret
+  %r = fcmp ueq bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ugt(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %r = fcmp ugt bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_uge(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, pl
+; CHECK-NEXT:    ret
+  %r = fcmp uge bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ult(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
+  %r = fcmp ult bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ule(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
+  %r = fcmp ule bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_uno(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_uno:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, vs
+; CHECK-NEXT:    ret
+  %r = fcmp uno bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_one(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w8, mi
+; CHECK-NEXT:    csinc w0, w8, wzr, le
+; CHECK-NEXT:    ret
+  %r = fcmp one bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_oeq(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_oeq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %r = fcmp oeq bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ogt(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ogt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %r = fcmp ogt bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_oge(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_oge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
+  %r = fcmp oge bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_olt(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_olt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, mi
+; CHECK-NEXT:    ret
+  %r = fcmp olt bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ole(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
+  %r = fcmp ole bfloat %a, %b
+  ret i1 %r
+}
+
+define i1 @test_fcmp_ord(bfloat %a, bfloat %b) #0 {
+; CHECK-LABEL: test_fcmp_ord:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    cset w0, vc
+; CHECK-NEXT:    ret
+  %r = fcmp ord bfloat %a, %b
+  ret i1 %r
+}
+
+define void @test_fccmp(bfloat %in, ptr %out) {
+; CHECK-LABEL: test_fccmp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    movi v1.2s, #69, lsl #24
+; CHECK-NEXT:    movi v3.2s, #72, lsl #24
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    adrp x8, .LCPI29_0
+; CHECK-NEXT:    fcmp s2, s1
+; CHECK-NEXT:    ldr h1, [x8, :lo12:.LCPI29_0]
+; CHECK-NEXT:    fccmp s2, s3, #4, mi
+; CHECK-NEXT:    fcsel s0, s0, s1, gt
+; CHECK-NEXT:    str h0, [x0]
+; CHECK-NEXT:    ret
+  %cmp1 = fcmp ogt bfloat %in, 0xR4800
+  %cmp2 = fcmp olt bfloat %in, 0xR4500
+  %cond = and i1 %cmp1, %cmp2
+  %result = select i1 %cond, bfloat %in, bfloat 0xR4500
+  store bfloat %result, ptr %out
+  ret void
+}
+
+define void @test_br_cc(bfloat %a, bfloat %b, ptr %p1, ptr %p2) #0 {
+; CHECK-LABEL: test_br_cc:
+; CHECK:       // %bb.0: // %common.ret
+; CHECK-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csel x8, x0, x1, pl
+; CHECK-NEXT:    str wzr, [x8]
+; CHECK-NEXT:    ret
+  %c = fcmp uge bfloat %a, %b
+  br i1 %c, label %then, label %else
+then:
+  store i32 0, ptr %p1
+  ret void
+else:
+  store i32 0, ptr %p2
+  ret void
+}
+
+define bfloat @test_phi(ptr %p1) #0 {
+; CHECK-LABEL: test_phi:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    ldr h9, [x0]
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:  .LBB31_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    fmov s8, s9
+; CHECK-NEXT:    ldr h9, [x19]
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    bl test_dummy
+; CHECK-NEXT:    tbnz w0, #0, .LBB31_1
+; CHECK-NEXT:  // %bb.2: // %return
+; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fmov s0, s8
+; CHECK-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p1
+  br label %loop
+loop:
+  %r = phi bfloat [%a, %entry], [%b, %loop]
+  %b = load bfloat, ptr %p1
+  %c = call i1 @test_dummy(ptr %p1)
+  br i1 %c, label %loop, label %return
+return:
+  ret bfloat %r
+}
+
+declare i1 @test_dummy(ptr %p1) #0
+
+define i32 @test_fptosi_i32(bfloat %a) #0 {
+; CHECK-LABEL: test_fptosi_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fcvtzs w0, s0
+; CHECK-NEXT:    ret
+  %r = fptosi bfloat %a to i32
+  ret i32 %r
+}
+
+define i64 @test_fptosi_i64(bfloat %a) #0 {
+; CHECK-LABEL: test_fptosi_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fcvtzs x0, s0
+; CHECK-NEXT:    ret
+  %r = fptosi bfloat %a to i64
+  ret i64 %r
+}
+
+define i32 @test_fptoui_i32(bfloat %a) #0 {
+; CHECK-LABEL: test_fptoui_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fcvtzu w0, s0
+; CHECK-NEXT:    ret
+  %r = fptoui bfloat %a to i32
+  ret i32 %r
+}
+
+define i64 @test_fptoui_i64(bfloat %a) #0 {
+; CHECK-LABEL: test_fptoui_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fcvtzu x0, s0
+; CHECK-NEXT:    ret
+  %r = fptoui bfloat %a to i64
+  ret i64 %r
+}
+
+define bfloat @test_uitofp_i32(i32 %a) #0 {
+; CHECK-CVT-LABEL: test_uitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf d0, w0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    fcvtxn s0, d0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_uitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf d0, w0
+; CHECK-BF16-NEXT:    fcvtxn s0, d0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = uitofp i32 %a to bfloat
+  ret bfloat %r
+}
+
+define bfloat @test_uitofp_i64(i64 %a) #0 {
+; CHECK-CVT-LABEL: test_uitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    lsr x9, x0, #53
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    cmp x9, #0
+; CHECK-CVT-NEXT:    and x9, x0, #0xfffffffffffff000
+; CHECK-CVT-NEXT:    csel x9, x9, x0, ne
+; CHECK-CVT-NEXT:    ucvtf d0, x9
+; CHECK-CVT-NEXT:    cset w9, ne
+; CHECK-CVT-NEXT:    tst x0, #0xfff
+; CHECK-CVT-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-NEXT:    fmov x10, d0
+; CHECK-CVT-NEXT:    orr x9, x10, x9
+; CHECK-CVT-NEXT:    fmov d0, x9
+; CHECK-CVT-NEXT:    fcvtxn s0, d0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_uitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    lsr x8, x0, #53
+; CHECK-BF16-NEXT:    and x9, x0, #0xfffffffffffff000
+; CHECK-BF16-NEXT:    cmp x8, #0
+; CHECK-BF16-NEXT:    csel x8, x9, x0, ne
+; CHECK-BF16-NEXT:    ucvtf d0, x8
+; CHECK-BF16-NEXT:    cset w8, ne
+; CHECK-BF16-NEXT:    tst x0, #0xfff
+; CHECK-BF16-NEXT:    csel w8, wzr, w8, eq
+; CHECK-BF16-NEXT:    fmov x9, d0
+; CHECK-BF16-NEXT:    orr x8, x9, x8
+; CHECK-BF16-NEXT:    fmov d0, x8
+; CHECK-BF16-NEXT:    fcvtxn s0, d0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = uitofp i64 %a to bfloat
+  ret bfloat %r
+}
+
+define bfloat @test_sitofp_i32(i32 %a) #0 {
+; CHECK-CVT-LABEL: test_sitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf d0, w0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    fcvtxn s0, d0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf d0, w0
+; CHECK-BF16-NEXT:    fcvtxn s0, d0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = sitofp i32 %a to bfloat
+  ret bfloat %r
+}
+
+define bfloat @test_sitofp_i64(i64 %a) #0 {
+; CHECK-CVT-LABEL: test_sitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    cmp x0, #0
+; CHECK-CVT-NEXT:    and x11, x0, #0x8000000000000000
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    cneg x9, x0, mi
+; CHECK-CVT-NEXT:    lsr x10, x9, #53
+; CHECK-CVT-NEXT:    cmp x10, #0
+; CHECK-CVT-NEXT:    and x10, x9, #0xfffffffffffff000
+; CHECK-CVT-NEXT:    csel x10, x10, x9, ne
+; CHECK-CVT-NEXT:    scvtf d0, x10
+; CHECK-CVT-NEXT:    cset w10, ne
+; CHECK-CVT-NEXT:    tst x9, #0xfff
+; CHECK-CVT-NEXT:    csel w10, wzr, w10, eq
+; CHECK-CVT-NEXT:    fmov x9, d0
+; CHECK-CVT-NEXT:    orr x9, x9, x11
+; CHECK-CVT-NEXT:    orr x9, x9, x10
+; CHECK-CVT-NEXT:    fmov d0, x9
+; CHECK-CVT-NEXT:    fcvtxn s0, d0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    cmp x0, #0
+; CHECK-BF16-NEXT:    cneg x8, x0, mi
+; CHECK-BF16-NEXT:    lsr x9, x8, #53
+; CHECK-BF16-NEXT:    and x10, x8, #0xfffffffffffff000
+; CHECK-BF16-NEXT:    cmp x9, #0
+; CHECK-BF16-NEXT:    csel x9, x10, x8, ne
+; CHECK-BF16-NEXT:    and x10, x0, #0x8000000000000000
+; CHECK-BF16-NEXT:    cset w11, ne
+; CHECK-BF16-NEXT:    scvtf d0, x9
+; CHECK-BF16-NEXT:    tst x8, #0xfff
+; CHECK-BF16-NEXT:    fmov x9, d0
+; CHECK-BF16-NEXT:    orr x8, x9, x10
+; CHECK-BF16-NEXT:    csel w9, wzr, w11, eq
+; CHECK-BF16-NEXT:    orr x8, x8, x9
+; CHECK-BF16-NEXT:    fmov d0, x8
+; CHECK-BF16-NEXT:    fcvtxn s0, d0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = sitofp i64 %a to bfloat
+  ret bfloat %r
+}
+
+define bfloat @test_uitofp_i32_fadd(i32 %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_uitofp_i32_fadd:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf d1, w0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fcvtxn s1, d1
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w9, w9, w8
+; CHECK-CVT-NEXT:    add w9, w10, w9
+; CHECK-CVT-NEXT:    lsr w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    fmov w10, s1
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_uitofp_i32_fadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf d1, w0
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fcvtxn s1, d1
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bfcvt h1, s1
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %c = uitofp i32 %a to bfloat
+  %r = fadd bfloat %b, %c
+  ret bfloat %r
+}
+
+define bfloat @test_sitofp_i32_fadd(i32 %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_sitofp_i32_fadd:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf d1, w0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fcvtxn s1, d1
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w9, w9, w8
+; CHECK-CVT-NEXT:    add w9, w10, w9
+; CHECK-CVT-NEXT:    lsr w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    fmov w10, s1
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sitofp_i32_fadd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf d1, w0
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fcvtxn s1, d1
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bfcvt h1, s1
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %c = sitofp i32 %a to bfloat
+  %r = fadd bfloat %b, %c
+  ret bfloat %r
+}
+
+define bfloat @test_fptrunc_float(float %a) #0 {
+; CHECK-CVT-LABEL: test_fptrunc_float:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    fcmp s0, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    orr w9, w9, #0x400000
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    csel w8, w9, w8, vs
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fptrunc_float:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fptrunc float %a to bfloat
+  ret bfloat %r
+}
+
+define bfloat @test_fptrunc_double(double %a) #0 {
+; CHECK-CVT-LABEL: test_fptrunc_double:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtxn s0, d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fptrunc_double:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    fcvtxn s0, d0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = fptrunc double %a to bfloat
+  ret bfloat %r
+}
+
+define float @test_fpext_float(bfloat %a) #0 {
+; CHECK-LABEL: test_fpext_float:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ret
+  %r = fpext bfloat %a to float
+  ret float %r
+}
+
+define double @test_fpext_double(bfloat %a) #0 {
+; CHECK-LABEL: test_fpext_double:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fcvt d0, s0
+; CHECK-NEXT:    ret
+  %r = fpext bfloat %a to double
+  ret double %r
+}
+
+define i16 @test_bitcast_bfloattoi16(bfloat %a) #0 {
+; CHECK-LABEL: test_bitcast_bfloattoi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+  %r = bitcast bfloat %a to i16
+  ret i16 %r
+}
+
+define bfloat @test_bitcast_i16tobfloat(i16 %a) #0 {
+; CHECK-LABEL: test_bitcast_i16tobfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
+  %r = bitcast i16 %a to bfloat
+  ret bfloat %r
+}
+
+declare bfloat @llvm.sqrt.f16(bfloat %a) #0
+declare bfloat @llvm.powi.f16.i32(bfloat %a, i32 %b) #0
+declare bfloat @llvm.sin.f16(bfloat %a) #0
+declare bfloat @llvm.cos.f16(bfloat %a) #0
+declare bfloat @llvm.tan.f16(bfloat %a) #0
+declare bfloat @llvm.asin.f16(bfloat %a) #0
+declare bfloat @llvm.acos.f16(bfloat %a) #0
+declare bfloat @llvm.atan.f16(bfloat %a) #0
+declare bfloat @llvm.atan2.f16(bfloat %a, bfloat %b) #0
+declare bfloat @llvm.sinh.f16(bfloat %a) #0
+declare bfloat @llvm.cosh.f16(bfloat %a) #0
+declare bfloat @llvm.tanh.f16(bfloat %a) #0
+declare bfloat @llvm.pow.f16(bfloat %a, bfloat %b) #0
+declare bfloat @llvm.exp.f16(bfloat %a) #0
+declare bfloat @llvm.exp2.f16(bfloat %a) #0
+declare bfloat @llvm.log.f16(bfloat %a) #0
+declare bfloat @llvm.log10.f16(bfloat %a) #0
+declare bfloat @llvm.log2.f16(bfloat %a) #0
+declare bfloat @llvm.fma.f16(bfloat %a, bfloat %b, bfloat %c) #0
+declare bfloat @llvm.fabs.f16(bfloat %a) #0
+declare bfloat @llvm.minnum.f16(bfloat %a, bfloat %b) #0
+declare bfloat @llvm.maxnum.f16(bfloat %a, bfloat %b) #0
+declare bfloat @llvm.copysign.f16(bfloat %a, bfloat %b) #0
+declare bfloat @llvm.floor.f16(bfloat %a) #0
+declare bfloat @llvm.ceil.f16(bfloat %a) #0
+declare bfloat @llvm.trunc.f16(bfloat %a) #0
+declare bfloat @llvm.rint.f16(bfloat %a) #0
+declare bfloat @llvm.nearbyint.f16(bfloat %a) #0
+declare bfloat @llvm.round.f16(bfloat %a) #0
+declare bfloat @llvm.roundeven.f16(bfloat %a) #0
+declare bfloat @llvm.fmuladd.f16(bfloat %a, bfloat %b, bfloat %c) #0
+
+
+define bfloat @test_sqrt(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_sqrt:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fsqrt s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sqrt:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fsqrt s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.sqrt.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_powi(bfloat %a, i32 %b) #0 {
+; CHECK-CVT-LABEL: test_powi:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl __powisf2
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_powi:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl __powisf2
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.powi.f16.i32(bfloat %a, i32 %b)
+  ret bfloat %r
+}
+
+
+define bfloat @test_sin(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_sin:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl sinf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sin:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl sinf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.sin.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_cos(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_cos:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl cosf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_cos:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl cosf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.cos.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_tan(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_tan:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl tanf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_tan:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl tanf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.tan.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_acos(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_acos:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl acosf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_acos:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl acosf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.acos.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_asin(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_asin:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl asinf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_asin:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl asinf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.asin.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_atan(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_atan:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl atanf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_atan:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl atanf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.atan.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_atan2(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_atan2:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    bl atan2f
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_atan2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bl atan2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.atan2.f16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_cosh(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_cosh:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl coshf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_cosh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl coshf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.cosh.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_sinh(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_sinh:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl sinhf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_sinh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl sinhf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.sinh.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_tanh(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_tanh:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl tanhf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_tanh:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl tanhf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.tanh.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_pow(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_pow:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    bl powf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_pow:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bl powf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.pow.f16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_exp(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_exp:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl expf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_exp:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl expf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.exp.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_exp2(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_exp2:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl exp2f
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_exp2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl exp2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.exp2.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_log(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_log:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl logf
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_log:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl logf
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.log.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_log10(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_log10:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl log10f
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_log10:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl log10f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.log10.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_log2(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_log2:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bl log2f
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_log2:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bl log2f
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.log2.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-CVT-LABEL: test_fma:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s2
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fmov s2, w10
+; CHECK-CVT-NEXT:    mov w10, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    fmadd s0, s2, s1, s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    ubfx w9, w8, #16, #1
+; CHECK-CVT-NEXT:    add w8, w8, w10
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fma:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s2
+; CHECK-BF16-NEXT:    fmov w9, s1
+; CHECK-BF16-NEXT:    fmov w10, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    lsl w10, w10, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fmov s2, w10
+; CHECK-BF16-NEXT:    fmadd s0, s2, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.fma.f16(bfloat %a, bfloat %b, bfloat %c)
+  ret bfloat %r
+}
+
+define bfloat @test_fabs(bfloat %a) #0 {
+; CHECK-LABEL: test_fabs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w8, w8, #0x7fff
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
+  %r = call bfloat @llvm.fabs.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_minnum(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_minnum:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fminnm s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_minnum:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fminnm s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.minnum.f16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_maxnum:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s1
+; CHECK-CVT-NEXT:    fmov w10, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    lsl w10, w10, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    fmov s1, w10
+; CHECK-CVT-NEXT:    fmaxnm s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_maxnum:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fmaxnm s0, s1, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.maxnum.f16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_copysign:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v2.16b
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_copysign:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bit v0.16b, v1.16b, v2.16b
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.copysign.f16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_copysign_f32(bfloat %a, float %b) #0 {
+; CHECK-CVT-LABEL: test_copysign_f32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_copysign_f32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-BF16-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %tb = fptrunc float %b to bfloat
+  %r = call bfloat @llvm.copysign.f16(bfloat %a, bfloat %tb)
+  ret bfloat %r
+}
+
+define bfloat @test_copysign_f64(bfloat %a, double %b) #0 {
+; CHECK-CVT-LABEL: test_copysign_f64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    fcvt s1, d1
+; CHECK-CVT-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_copysign_f64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    fcvt s1, d1
+; CHECK-BF16-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %tb = fptrunc double %b to bfloat
+  %r = call bfloat @llvm.copysign.f16(bfloat %a, bfloat %tb)
+  ret bfloat %r
+}
+
+; away the (fpext (fp_round <result>)) here.
+
+define float @test_copysign_extended(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-LABEL: test_copysign_extended:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v2.16b
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_copysign_extended:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bit v0.16b, v1.16b, v2.16b
+; CHECK-BF16-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.copysign.f16(bfloat %a, bfloat %b)
+  %xr = fpext bfloat %r to float
+  ret float %xr
+}
+
+define bfloat @test_floor(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_floor:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_floor:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frintm s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.floor.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_ceil(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_ceil:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_ceil:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frintp s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.ceil.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_trunc(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_trunc:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_trunc:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frintz s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.trunc.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_rint(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_rint:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frintx s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_rint:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frintx s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.rint.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_nearbyint(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_nearbyint:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frinti s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_nearbyint:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frinti s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.nearbyint.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_round(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_round:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_round:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frinta s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.round.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_roundeven(bfloat %a) #0 {
+; CHECK-CVT-LABEL: test_roundeven:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w9
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_roundeven:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    frintn s0, s0
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.roundeven.f16(bfloat %a)
+  ret bfloat %r
+}
+
+define bfloat @test_fmuladd(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-CVT-LABEL: test_fmuladd:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s1
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    mov w10, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fmul s0, s1, s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    ubfx w9, w8, #16, #1
+; CHECK-CVT-NEXT:    add w8, w8, w10
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    fmov w9, s2
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w8, w8, #16
+; CHECK-CVT-NEXT:    lsl w9, w9, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    ubfx w9, w8, #16, #1
+; CHECK-CVT-NEXT:    add w8, w8, w10
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: test_fmuladd:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-BF16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-BF16-NEXT:    fmov w8, s1
+; CHECK-BF16-NEXT:    fmov w9, s0
+; CHECK-BF16-NEXT:    // kill: def $h2 killed $h2 def $s2
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    fmov w9, s2
+; CHECK-BF16-NEXT:    fmul s0, s1, s0
+; CHECK-BF16-NEXT:    lsl w9, w9, #16
+; CHECK-BF16-NEXT:    fmov s1, w9
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    fmov w8, s0
+; CHECK-BF16-NEXT:    lsl w8, w8, #16
+; CHECK-BF16-NEXT:    fmov s0, w8
+; CHECK-BF16-NEXT:    fadd s0, s0, s1
+; CHECK-BF16-NEXT:    bfcvt h0, s0
+; CHECK-BF16-NEXT:    ret
+  %r = call bfloat @llvm.fmuladd.f16(bfloat %a, bfloat %b, bfloat %c)
+  ret bfloat %r
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
new file mode 100644
index 0000000000000..9b6e19eba3f4e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -0,0 +1,711 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 -mattr=-bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16
+
+define <4 x bfloat> @add_h(<4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-CVT-LABEL: add_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: add_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+entry:
+
+  %0 = fadd <4 x bfloat> %a, %b
+  ret <4 x bfloat> %0
+}
+
+
+define <4 x bfloat> @build_h4(<4 x bfloat> %a) {
+; CHECK-LABEL: build_h4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #15565 // =0x3ccd
+; CHECK-NEXT:    dup v0.4h, w8
+; CHECK-NEXT:    ret
+entry:
+  ret <4 x bfloat> <bfloat 0xR3CCD, bfloat 0xR3CCD, bfloat 0xR3CCD, bfloat 0xR3CCD>
+}
+
+
+define <4 x bfloat> @sub_h(<4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-CVT-LABEL: sub_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sub_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+entry:
+
+  %0 = fsub <4 x bfloat> %a, %b
+  ret <4 x bfloat> %0
+}
+
+
+define <4 x bfloat> @mul_h(<4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-CVT-LABEL: mul_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: mul_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+entry:
+
+  %0 = fmul <4 x bfloat> %a, %b
+  ret <4 x bfloat> %0
+}
+
+
+define <4 x bfloat> @div_h(<4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-CVT-LABEL: div_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: div_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    fdiv v0.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+entry:
+
+  %0 = fdiv <4 x bfloat> %a, %b
+  ret <4 x bfloat> %0
+}
+
+
+define <4 x bfloat> @load_h(ptr %a) {
+; CHECK-LABEL: load_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <4 x bfloat>, ptr %a, align 4
+  ret <4 x bfloat> %0
+}
+
+
+define void @store_h(ptr %a, <4 x bfloat> %b) {
+; CHECK-LABEL: store_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  store <4 x bfloat> %b, ptr %a, align 4
+  ret void
+}
+
+define <4 x bfloat> @s_to_h(<4 x float> %a) {
+; CHECK-CVT-LABEL: s_to_h:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: s_to_h:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = fptrunc <4 x float> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <4 x bfloat> @d_to_h(<4 x double> %a) {
+; CHECK-CVT-LABEL: d_to_h:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: d_to_h:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = fptrunc <4 x double> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <4 x float> @h_to_s(<4 x bfloat> %a) {
+; CHECK-LABEL: h_to_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    ret
+  %1 = fpext <4 x bfloat> %a to <4 x float>
+  ret <4 x float> %1
+}
+
+define <4 x double> @h_to_d(<4 x bfloat> %a) {
+; CHECK-LABEL: h_to_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    ret
+  %1 = fpext <4 x bfloat> %a to <4 x double>
+  ret <4 x double> %1
+}
+
+define <4 x bfloat> @bitcast_i_to_h(float, <4 x i16> %a) {
+; CHECK-LABEL: bitcast_i_to_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ret
+  %2 = bitcast <4 x i16> %a to <4 x bfloat>
+  ret <4 x bfloat> %2
+}
+
+define <4 x i16> @bitcast_h_to_i(float, <4 x bfloat> %a) {
+; CHECK-LABEL: bitcast_h_to_i:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ret
+  %2 = bitcast <4 x bfloat> %a to <4 x i16>
+  ret <4 x i16> %2
+}
+
+define <4 x bfloat> @sitofp_i8(<4 x i8> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-BF16-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <4 x i8> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <4 x bfloat> @sitofp_i16(<4 x i16> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i16:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <4 x i16> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+
+define <4 x bfloat> @sitofp_i32(<4 x i32> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <4 x i32> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+
+define <4 x bfloat> @sitofp_i64(<4 x i64> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <4 x i64> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <4 x bfloat> @uitofp_i8(<4 x i8> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <4 x i8> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+
+define <4 x bfloat> @uitofp_i16(<4 x i16> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i16:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <4 x i16> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+
+define <4 x bfloat> @uitofp_i32(<4 x i32> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <4 x i32> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+
+define <4 x bfloat> @uitofp_i64(<4 x i64> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <4 x i64> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define void @test_insert_at_zero(bfloat %a, ptr %b) #0 {
+; CHECK-LABEL: test_insert_at_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %1 = insertelement <4 x bfloat> undef, bfloat %a, i64 0
+  store <4 x bfloat> %1, ptr %b, align 4
+  ret void
+}
+
+define <4 x i8> @fptosi_i8(<4 x bfloat> %a) #0 {
+; CHECK-LABEL: fptosi_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %1 = fptosi<4 x bfloat> %a to <4 x i8>
+  ret <4 x i8> %1
+}
+
+define <4 x i16> @fptosi_i16(<4 x bfloat> %a) #0 {
+; CHECK-LABEL: fptosi_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %1 = fptosi<4 x bfloat> %a to <4 x i16>
+  ret <4 x i16> %1
+}
+
+define <4 x i8> @fptoui_i8(<4 x bfloat> %a) #0 {
+; CHECK-LABEL: fptoui_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+; NOTE: fcvtzs selected here because the xtn shaves the sign bit
+  %1 = fptoui<4 x bfloat> %a to <4 x i8>
+  ret <4 x i8> %1
+}
+
+define <4 x i16> @fptoui_i16(<4 x bfloat> %a) #0 {
+; CHECK-LABEL: fptoui_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %1 = fptoui<4 x bfloat> %a to <4 x i16>
+  ret <4 x i16> %1
+}
+
+define <4 x i1> @test_fcmp_une(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_une:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp une <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ueq(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ueq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ueq <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ugt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ugt <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_uge(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp uge <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ult(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ult <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ule(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ule <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_uno(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_uno:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+
+  %1 = fcmp uno <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_one(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp one <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_oeq(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_oeq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp oeq <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ogt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ogt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ogt <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_oge(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_oge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp oge <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_olt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_olt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp olt <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ole(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ole <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+define <4 x i1> @test_fcmp_ord(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ord:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+
+  %1 = fcmp ord <4 x bfloat> %a, %b
+  ret <4 x i1> %1
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
new file mode 100644
index 0000000000000..c03e2e5321321
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -0,0 +1,2192 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 -mattr=-bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16
+
+define <8 x bfloat> @add_h(<8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-CVT-LABEL: add_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll2 v3.4s, v1.8h, #16
+; CHECK-CVT-NEXT:    shll2 v4.4s, v0.8h, #16
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fadd v3.4s, v4.4s, v3.4s
+; CHECK-CVT-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-NEXT:    movi v1.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v3.4s, v3.4s
+; CHECK-CVT-NEXT:    add v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v3.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v1.4s
+; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT:    mov v2.16b, v5.16b
+; CHECK-CVT-NEXT:    bsl v2.16b, v4.16b, v3.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: add_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v2.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v3.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-BF16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT:    ret
+entry:
+  %0 = fadd <8 x bfloat> %a, %b
+  ret <8 x bfloat> %0
+}
+
+
+define <8 x bfloat> @sub_h(<8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-CVT-LABEL: sub_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll2 v3.4s, v1.8h, #16
+; CHECK-CVT-NEXT:    shll2 v4.4s, v0.8h, #16
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fsub v3.4s, v4.4s, v3.4s
+; CHECK-CVT-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-NEXT:    movi v1.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v3.4s, v3.4s
+; CHECK-CVT-NEXT:    add v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v3.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v1.4s
+; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT:    mov v2.16b, v5.16b
+; CHECK-CVT-NEXT:    bsl v2.16b, v4.16b, v3.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sub_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v2.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v3.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-BF16-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT:    ret
+entry:
+  %0 = fsub <8 x bfloat> %a, %b
+  ret <8 x bfloat> %0
+}
+
+
+define <8 x bfloat> @mul_h(<8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-CVT-LABEL: mul_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll2 v3.4s, v1.8h, #16
+; CHECK-CVT-NEXT:    shll2 v4.4s, v0.8h, #16
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-CVT-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-NEXT:    movi v1.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v3.4s, v3.4s
+; CHECK-CVT-NEXT:    add v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v3.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v1.4s
+; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT:    mov v2.16b, v5.16b
+; CHECK-CVT-NEXT:    bsl v2.16b, v4.16b, v3.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: mul_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v2.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v3.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-BF16-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT:    ret
+entry:
+  %0 = fmul <8 x bfloat> %a, %b
+  ret <8 x bfloat> %0
+}
+
+
+define <8 x bfloat> @div_h(<8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-CVT-LABEL: div_h:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-CVT-NEXT:    shll2 v3.4s, v0.8h, #16
+; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-NEXT:    movi v4.4s, #127, msl #8
+; CHECK-CVT-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    shll v3.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    ushr v0.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    fdiv v1.4s, v3.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v3.4s, #1
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-NEXT:    fcmeq v4.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT:    add v3.4s, v2.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v1.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    bit v2.16b, v3.16b, v4.16b
+; CHECK-CVT-NEXT:    bif v0.16b, v1.16b, v5.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: div_h:
+; CHECK-BF16:       // %bb.0: // %entry
+; CHECK-BF16-NEXT:    shll v2.4s, v1.4h, #16
+; CHECK-BF16-NEXT:    shll v3.4s, v0.4h, #16
+; CHECK-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-BF16-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT:    ret
+entry:
+  %0 = fdiv <8 x bfloat> %a, %b
+  ret <8 x bfloat> %0
+}
+
+
+define <8 x bfloat> @load_h(ptr %a) {
+; CHECK-LABEL: load_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <8 x bfloat>, ptr %a, align 4
+  ret <8 x bfloat> %0
+}
+
+
+define void @store_h(ptr %a, <8 x bfloat> %b) {
+; CHECK-LABEL: store_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  store <8 x bfloat> %b, ptr %a, align 4
+  ret void
+}
+
+define <8 x bfloat> @s_to_h(<8 x float> %a) {
+; CHECK-CVT-LABEL: s_to_h:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    movi v3.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v4.4s, v1.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-NEXT:    add v6.4s, v1.4s, v3.4s
+; CHECK-CVT-NEXT:    and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-NEXT:    add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-CVT-NEXT:    orr v1.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    bit v1.16b, v4.16b, v5.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v2.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: s_to_h:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = fptrunc <8 x float> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <8 x bfloat> @d_to_h(<8 x double> %a) {
+; CHECK-CVT-LABEL: d_to_h:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtxn v2.2s, v2.2d
+; CHECK-CVT-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    fcvtxn2 v2.4s, v3.2d
+; CHECK-CVT-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v3.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v4.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v6.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-NEXT:    and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    bit v2.16b, v4.16b, v5.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: d_to_h:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    fcvtxn v2.2s, v2.2d
+; CHECK-BF16-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtxn2 v2.4s, v3.2d
+; CHECK-BF16-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = fptrunc <8 x double> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <8 x float> @h_to_s(<8 x bfloat> %a) {
+; CHECK-LABEL: h_to_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    ret
+  %1 = fpext <8 x bfloat> %a to <8 x float>
+  ret <8 x float> %1
+}
+
+define <8 x double> @h_to_d(<8 x bfloat> %a) {
+; CHECK-LABEL: h_to_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtl v0.2d, v2.2s
+; CHECK-NEXT:    shll v4.4s, v1.4h, #16
+; CHECK-NEXT:    fcvtl2 v1.2d, v2.4s
+; CHECK-NEXT:    fcvtl2 v3.2d, v4.4s
+; CHECK-NEXT:    fcvtl v2.2d, v4.2s
+; CHECK-NEXT:    ret
+  %1 = fpext <8 x bfloat> %a to <8 x double>
+  ret <8 x double> %1
+}
+
+
+define <8 x bfloat> @bitcast_i_to_h(float, <8 x i16> %a) {
+; CHECK-LABEL: bitcast_i_to_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = bitcast <8 x i16> %a to <8 x bfloat>
+  ret <8 x bfloat> %2
+}
+
+define <8 x i16> @bitcast_h_to_i(float, <8 x bfloat> %a) {
+; CHECK-LABEL: bitcast_h_to_i:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %2 = bitcast <8 x bfloat> %a to <8 x i16>
+  ret <8 x i16> %2
+}
+
+define <4 x bfloat> @sitofp_v4i8(<4 x i8> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_v4i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_v4i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-BF16-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <4 x i8> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <8 x bfloat> @sitofp_v8i8(<8 x i8> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_v8i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v4.4s, #127, msl #8
+; CHECK-CVT-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    scvtf v3.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v0.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_v8i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-BF16-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <8 x i8> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <16 x bfloat> @sitofp_v16i8(<16 x i8> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_v16i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    sshll2 v2.8h, v0.16b, #0
+; CHECK-CVT-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v7.4s, #127, msl #8
+; CHECK-CVT-NEXT:    sshll v3.4s, v2.4h, #0
+; CHECK-CVT-NEXT:    sshll v4.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-CVT-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-CVT-NEXT:    scvtf v4.4s, v4.4s
+; CHECK-CVT-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    scvtf v6.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    ushr v0.4s, v4.4s, #16
+; CHECK-CVT-NEXT:    ushr v16.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v17.4s, v6.4s, #16
+; CHECK-CVT-NEXT:    and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v16.16b, v16.16b, v1.16b
+; CHECK-CVT-NEXT:    and v17.16b, v17.16b, v1.16b
+; CHECK-CVT-NEXT:    add v5.4s, v5.4s, v7.4s
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v7.4s
+; CHECK-CVT-NEXT:    addhn v1.4h, v3.4s, v5.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v4.4s, v0.4s
+; CHECK-CVT-NEXT:    add v3.4s, v16.4s, v7.4s
+; CHECK-CVT-NEXT:    add v4.4s, v17.4s, v7.4s
+; CHECK-CVT-NEXT:    addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_v16i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-BF16-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-BF16-NEXT:    sshll2 v2.4s, v1.8h, #0
+; CHECK-BF16-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-BF16-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-BF16-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v2.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v3.4h, v3.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-BF16-NEXT:    mov v0.d[1], v3.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <16 x i8> %a to <16 x bfloat>
+  ret <16 x bfloat> %1
+}
+
+define <8 x bfloat> @sitofp_i16(<8 x i16> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v4.4s, #127, msl #8
+; CHECK-CVT-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    scvtf v3.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v0.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i16:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <8 x i16> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <8 x bfloat> @sitofp_i32(<8 x i32> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v5.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    ushr v4.4s, v1.4s, #16
+; CHECK-CVT-NEXT:    and v3.16b, v3.16b, v2.16b
+; CHECK-CVT-NEXT:    and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <8 x i32> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+
+define <8 x bfloat> @sitofp_i64(<8 x i64> %a) #0 {
+; CHECK-CVT-LABEL: sitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-CVT-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-CVT-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-CVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-CVT-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v3.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v4.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v6.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-NEXT:    and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    bit v2.16b, v4.16b, v5.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: sitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-BF16-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-BF16-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-BF16-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-BF16-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = sitofp <8 x i64> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <4 x bfloat> @uitofp_v4i8(<4 x i8> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_v4i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
+; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_v4i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <4 x i8> %a to <4 x bfloat>
+  ret <4 x bfloat> %1
+}
+
+define <8 x bfloat> @uitofp_v8i8(<8 x i8> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_v8i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v4.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    ucvtf v3.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v0.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_v8i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-BF16-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <8 x i8> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define <16 x bfloat> @uitofp_v16i8(<16 x i8> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_v16i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-CVT-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v7.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushll v3.4s, v2.4h, #0
+; CHECK-CVT-NEXT:    ushll v4.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-CVT-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-CVT-NEXT:    ucvtf v4.4s, v4.4s
+; CHECK-CVT-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    ucvtf v6.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    ushr v0.4s, v4.4s, #16
+; CHECK-CVT-NEXT:    ushr v16.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v17.4s, v6.4s, #16
+; CHECK-CVT-NEXT:    and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v16.16b, v16.16b, v1.16b
+; CHECK-CVT-NEXT:    and v17.16b, v17.16b, v1.16b
+; CHECK-CVT-NEXT:    add v5.4s, v5.4s, v7.4s
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v7.4s
+; CHECK-CVT-NEXT:    addhn v1.4h, v3.4s, v5.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v4.4s, v0.4s
+; CHECK-CVT-NEXT:    add v3.4s, v16.4s, v7.4s
+; CHECK-CVT-NEXT:    add v4.4s, v17.4s, v7.4s
+; CHECK-CVT-NEXT:    addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_v16i8:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-BF16-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-BF16-NEXT:    ushll2 v2.4s, v1.8h, #0
+; CHECK-BF16-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-BF16-NEXT:    ushll2 v3.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-BF16-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v2.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v3.4h, v3.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-BF16-NEXT:    mov v0.d[1], v3.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <16 x i8> %a to <16 x bfloat>
+  ret <16 x bfloat> %1
+}
+
+
+define <8 x bfloat> @uitofp_i16(<8 x i16> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v4.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-CVT-NEXT:    ucvtf v3.4s, v0.4s
+; CHECK-CVT-NEXT:    ushr v0.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i16:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-BF16-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <8 x i16> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+
+define <8 x bfloat> @uitofp_i32(<8 x i32> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-CVT-NEXT:    movi v2.4s, #1
+; CHECK-CVT-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-CVT-NEXT:    movi v5.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    ushr v4.4s, v1.4s, #16
+; CHECK-CVT-NEXT:    and v3.16b, v3.16b, v2.16b
+; CHECK-CVT-NEXT:    and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-NEXT:    addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i32:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v1.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <8 x i32> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+
+define <8 x bfloat> @uitofp_i64(<8 x i64> %a) #0 {
+; CHECK-CVT-LABEL: uitofp_i64:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-CVT-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-CVT-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-CVT-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-CVT-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-CVT-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    movi v3.4s, #127, msl #8
+; CHECK-CVT-NEXT:    ushr v4.4s, v2.4s, #16
+; CHECK-CVT-NEXT:    ushr v5.4s, v0.4s, #16
+; CHECK-CVT-NEXT:    add v6.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-NEXT:    and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-NEXT:    and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-NEXT:    fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-NEXT:    fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
+; CHECK-CVT-NEXT:    bit v2.16b, v4.16b, v5.16b
+; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-BF16-LABEL: uitofp_i64:
+; CHECK-BF16:       // %bb.0:
+; CHECK-BF16-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-BF16-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-BF16-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-BF16-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-BF16-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-BF16-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-BF16-NEXT:    bfcvtn v1.4h, v2.4s
+; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-BF16-NEXT:    ret
+  %1 = uitofp <8 x i64> %a to <8 x bfloat>
+  ret <8 x bfloat> %1
+}
+
+define void @test_insert_at_zero(bfloat %a, ptr %b) #0 {
+; CHECK-LABEL: test_insert_at_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %1 = insertelement <8 x bfloat> undef, bfloat %a, i64 0
+  store <8 x bfloat> %1, ptr %b, align 4
+  ret void
+}
+
+define <8 x i8> @fptosi_i8(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: fptosi_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %1 = fptosi<8 x bfloat> %a to <8 x i8>
+  ret <8 x i8> %1
+}
+
+define <8 x i16> @fptosi_i16(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: fptosi_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %1 = fptosi<8 x bfloat> %a to <8 x i16>
+  ret <8 x i16> %1
+}
+
+define <8 x i8> @fptoui_i8(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: fptoui_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %1 = fptoui<8 x bfloat> %a to <8 x i8>
+  ret <8 x i8> %1
+}
+
+define <8 x i16> @fptoui_i16(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: fptoui_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %1 = fptoui<8 x bfloat> %a to <8 x i16>
+  ret <8 x i16> %1
+}
+
+define <8 x i1> @test_fcmp_une(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_une:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, ne
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, ne
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, ne
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp une <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ueq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ueq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w11, s0
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    lsl w9, w11, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s7, w9
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    csetm w10, eq
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csinv w10, w10, wzr, vc
+; CHECK-NEXT:    fcmp s7, s6
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w11, s4
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h4, v0.h[4]
+; CHECK-NEXT:    mov h7, v1.h[5]
+; CHECK-NEXT:    csetm w9, eq
+; CHECK-NEXT:    csinv w9, w9, wzr, vc
+; CHECK-NEXT:    fcmp s3, s2
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    lsl w11, w11, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s5, w11
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[1], w10
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    csinv w8, w8, wzr, vc
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov s6, w10
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    fmov w9, s7
+; CHECK-NEXT:    fmov w10, s5
+; CHECK-NEXT:    csinv w8, w8, wzr, vc
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    fmov s6, w10
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    csinv w8, w8, wzr, vc
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    lsl w8, w9, #16
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    csetm w10, eq
+; CHECK-NEXT:    csinv w10, w10, wzr, vc
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    csinv w8, w8, wzr, vc
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    csinv w8, w8, wzr, vc
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ueq <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ugt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, hi
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, hi
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, hi
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ugt <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_uge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, pl
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, pl
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, pl
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp uge <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ult(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, lt
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, lt
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, lt
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ult <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ule(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, le
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, le
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, le
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ule <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_uno(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_uno:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, vs
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, vs
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, vs
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp uno <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_one(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w11, s0
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    lsl w9, w11, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s7, w9
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    csetm w10, mi
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csinv w10, w10, wzr, le
+; CHECK-NEXT:    fcmp s7, s6
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w11, s4
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h4, v0.h[4]
+; CHECK-NEXT:    mov h7, v1.h[5]
+; CHECK-NEXT:    csetm w9, mi
+; CHECK-NEXT:    csinv w9, w9, wzr, le
+; CHECK-NEXT:    fcmp s3, s2
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    lsl w11, w11, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov s5, w11
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[1], w10
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    csinv w8, w8, wzr, le
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov s6, w10
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    fmov w9, s7
+; CHECK-NEXT:    fmov w10, s5
+; CHECK-NEXT:    csinv w8, w8, wzr, le
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    fmov s6, w10
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    csinv w8, w8, wzr, le
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    lsl w8, w9, #16
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    csetm w10, mi
+; CHECK-NEXT:    csinv w10, w10, wzr, le
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    csinv w8, w8, wzr, le
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    csinv w8, w8, wzr, le
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp one <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_oeq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_oeq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, eq
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, eq
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, eq
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp oeq <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ogt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ogt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, gt
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, gt
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, gt
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ogt <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_oge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_oge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, ge
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, ge
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, ge
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp oge <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_olt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_olt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, mi
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, mi
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, mi
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp olt <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ole(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, ls
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, ls
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, ls
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ole <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+define <8 x i1> @test_fcmp_ord(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fcmp_ord:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov h2, v1.h[1]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    fmov w9, s3
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h3, v0.h[2]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    mov h3, v1.h[4]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s5, s4
+; CHECK-NEXT:    fmov s5, w9
+; CHECK-NEXT:    mov h4, v1.h[3]
+; CHECK-NEXT:    lsl w10, w10, #16
+; CHECK-NEXT:    fmov s6, w8
+; CHECK-NEXT:    fmov w8, s2
+; CHECK-NEXT:    csetm w9, vc
+; CHECK-NEXT:    fmov s16, w10
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    mov h5, v0.h[3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    mov h6, v0.h[4]
+; CHECK-NEXT:    mov h4, v1.h[5]
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fmov w8, s5
+; CHECK-NEXT:    mov h5, v0.h[5]
+; CHECK-NEXT:    fcmp s16, s7
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    lsl w9, w10, #16
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    fmov s3, w9
+; CHECK-NEXT:    fmov w9, s6
+; CHECK-NEXT:    fmov s7, w8
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fcmp s7, s3
+; CHECK-NEXT:    mov h3, v1.h[6]
+; CHECK-NEXT:    fmov s4, w8
+; CHECK-NEXT:    mov h1, v1.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    lsl w8, w10, #16
+; CHECK-NEXT:    fcmp s6, s4
+; CHECK-NEXT:    mov h4, v0.h[6]
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s5, w8
+; CHECK-NEXT:    mov h0, v0.h[7]
+; CHECK-NEXT:    fmov s6, w9
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    mov v2.h[4], w8
+; CHECK-NEXT:    fmov w8, s3
+; CHECK-NEXT:    fmov w9, s4
+; CHECK-NEXT:    fcmp s6, s5
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    csetm w10, vc
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmov s4, w9
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mov v2.h[5], w10
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    fcmp s4, s3
+; CHECK-NEXT:    lsl w9, w9, #16
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    mov v2.h[6], w8
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    csetm w8, vc
+; CHECK-NEXT:    mov v2.h[7], w8
+; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    ret
+  %1 = fcmp ord <8 x bfloat> %a, %b
+  ret <8 x i1> %1
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir b/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
index 760ae4794e304..6d4f21fdee950 100644
--- a/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
+++ b/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
@@ -8,7 +8,7 @@
 # The arguments to the call must become implicit arguments, because the branch
 # only expects to get 1 explicit operand which is the branch target.
 
-# CHECK:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit $sp, implicit $x0, implicit $w1 {
+# CHECK:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit $sp, implicit $x0, implicit $w1 {
 # CHECK:      BL @_setjmp, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp
 # CHECK:      HINT 36
 # CHECK:    }
diff --git a/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-nsconst.ll b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-nsconst.ll
new file mode 100644
index 0000000000000..490a778f69e26
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-nsconst.ll
@@ -0,0 +1,32 @@
+; This test verifies that global variables (ns constant) are hashed based on their initial contents,
+; allowing them to be merged even if they appear different due to their names.
+; Now they become identical functions that can be merged without creating a parameter
+
+; RUN: llc -mtriple=arm64-apple-darwin -enable-global-merge-func=true -global-merging-skip-no-params=false < %s | FileCheck %s
+
+; CHECK: _f1.Tgm
+; CHECK: _f2.Tgm
+
+%struct.__NSConstantString_tag = type { ptr, i32, ptr, i64 }
+@__CFConstantStringClassReference = external global [0 x i32]
+@.str.2 = private unnamed_addr constant [9 x i8] c"cfstring\00", section "__TEXT,__cstring,cstring_literals", align 1
+@_unnamed_cfstring_ = private global %struct.__NSConstantString_tag { ptr @__CFConstantStringClassReference, i32 1992, ptr @.str.2, i64 8 }, section "__DATA,__cfstring", align 8
+
+@.str.3 = private unnamed_addr constant [9 x i8] c"cfstring\00", section "__TEXT,__cstring,cstring_literals", align 1
+@_unnamed_cfstring_.2 = private global %struct.__NSConstantString_tag { ptr @__CFConstantStringClassReference, i32 1992, ptr @.str.3, i64 8 }, section "__DATA,__cfstring", align 8
+
+declare i32 @hoo(ptr noundef)
+
+define i32 @f1() {
+entry:
+  %call = tail call i32 @hoo(ptr noundef nonnull @_unnamed_cfstring_)
+  %add = sub nsw i32 %call, 1
+  ret i32 %add
+}
+
+define i32 @f2() {
+entry:
+  %call = tail call i32 @hoo(ptr noundef nonnull @_unnamed_cfstring_.2)
+  %add = sub nsw i32 %call, 1
+  ret i32 %add
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-objc.ll b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-objc.ll
new file mode 100644
index 0000000000000..0073114941501
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-objc.ll
@@ -0,0 +1,38 @@
+; This test verifies that global variables (objc metadata) are hashed based on their initial contents,
+; allowing them to be merged even if they appear different due to their names.
+; Now they become identical functions that can be merged without creating a parameter
+
+; RUN: llc -mtriple=arm64-apple-darwin -enable-global-merge-func=true -global-merging-skip-no-params=false < %s | FileCheck %s
+
+; CHECK: _f1.Tgm
+; CHECK: _f2.Tgm
+
+%struct._class_t = type { ptr, ptr, ptr, ptr, ptr }
+
+@"OBJC_CLASS_$_MyClass" = external global %struct._class_t
+@"OBJC_CLASSLIST_REFERENCES_$_" = internal global ptr @"OBJC_CLASS_$_MyClass", section "__DATA,__objc_classrefs,regular,no_dead_strip", align 8
+@"OBJC_CLASSLIST_REFERENCES_$_.1" = internal global ptr @"OBJC_CLASS_$_MyClass", section "__DATA,__objc_classrefs,regular,no_dead_strip", align 8
+
+@OBJC_METH_VAR_NAME_ = private unnamed_addr constant [6 x i8] c"hello\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@OBJC_METH_VAR_NAME_.1 = private unnamed_addr constant [6 x i8] c"hello\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+
+@OBJC_SELECTOR_REFERENCES_ = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip", align 8
+@OBJC_SELECTOR_REFERENCES_.1 = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_.1, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip", align 8
+
+declare ptr @objc_msgSend(ptr, ptr, ...)
+
+define i32 @f1() {
+entry:
+  %0 = load ptr, ptr @"OBJC_CLASSLIST_REFERENCES_$_", align 8
+  %1 = load ptr, ptr @OBJC_SELECTOR_REFERENCES_, align 8
+  %call = tail call i32 @objc_msgSend(ptr noundef %0, ptr noundef %1)
+  ret i32 %call
+}
+
+define i32 @f2() {
+entry:
+  %0 = load ptr, ptr @"OBJC_CLASSLIST_REFERENCES_$_.1", align 8
+  %1 = load ptr, ptr @OBJC_SELECTOR_REFERENCES_.1, align 8
+  %call = tail call i32 @objc_msgSend(ptr noundef %0, ptr noundef %1)
+  ret i32 %call
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-string.ll b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-string.ll
new file mode 100644
index 0000000000000..1e67425f0b847
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-merge-gvar-string.ll
@@ -0,0 +1,46 @@
+; This test verifies that global variables (string) are hashed based on their initial contents,
+; allowing them to be merged even if they appear different due to their names.
+; Now they become identical functions that can be merged without creating a parameter.
+
+; RUN: llc -mtriple=arm64-apple-darwin -enable-global-merge-func=true -global-merging-skip-no-params=false < %s | FileCheck %s
+
+; CHECK: _f1.Tgm
+; CHECK: _f2.Tgm
+; CHECK-NOT: _f3.Tgm
+; CHECK-NOT: _f4.Tgm
+
+; The initial contents of `.str` and `.str.1` are identical, but not with those of `.str.2` and `.str.3`.
+@.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+@.str.1 = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+@.str.2 = private unnamed_addr constant [6 x i8] c"diff2\00", align 1
+@.str.3 = private unnamed_addr constant [6 x i8] c"diff3\00", align 1
+
+declare i32 @goo(ptr noundef)
+
+define i32 @f1() {
+entry:
+  %call = tail call i32 @goo(ptr noundef nonnull @.str)
+  %add = add nsw i32 %call, 1
+  ret i32 %add
+}
+
+define i32 @f2() {
+entry:
+  %call = tail call i32 @goo(ptr noundef nonnull @.str.1)
+  %add = add nsw i32 %call, 1
+  ret i32 %add
+}
+
+define i32 @f3() {
+entry:
+  %call = tail call noundef i32 @goo(ptr noundef nonnull @.str.2)
+  %add = sub nsw i32 %call, 1
+  ret i32 %add
+}
+
+define i32 @f4() {
+entry:
+  %call = tail call noundef i32 @goo(ptr noundef nonnull @.str.3)
+  %add = sub nsw i32 %call, 1
+  ret i32 %add
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-outline-gvar.ll b/llvm/test/CodeGen/AArch64/cgdata-outline-gvar.ll
new file mode 100644
index 0000000000000..63ba1d491f9c7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-outline-gvar.ll
@@ -0,0 +1,64 @@
+; This test verifies that global variables are hashed based on their initial contents,
+; allowing them to be outlined even if they appear different due to their names.
+
+; RUN: split-file %s %t
+
+; The outlined function is created locally.
+; Note that `.str.3` is commonly used in both `f1()` and `f2()`.
+; RUN: llc -mtriple=arm64-apple-darwin -enable-machine-outliner -codegen-data-generate -aarch64-enable-collect-loh=false  \
+; RUN:   %t/local-two.ll -o -  | FileCheck %s --check-prefix=WRITE
+
+; WRITE-LABEL: _OUTLINED_FUNCTION_{{.*}}:
+; WRITE:      adrp x1, l_.str.3
+; WRITE-NEXT: add x1, x1, l_.str.3
+; WRITE-NEXT: mov w2
+; WRITE-NEXT: mov w3
+; WRITE-NEXT: mov w4
+; WRITE-NEXT: b
+
+; Create an object file and merge it into the cgdata.
+; RUN: llc -mtriple=arm64-apple-darwin -enable-machine-outliner -codegen-data-generate -aarch64-enable-collect-loh=false \
+; RUN:   -filetype=obj %t/local-two.ll -o %t_write_base
+; RUN: llvm-cgdata --merge %t_write_base -o %t_cgdata_base
+
+; Read the cgdata in the machine outliner for optimistically outlining in local-one.ll.
+; Note that the hash of `.str.5` in local-one.ll matches that of `.str.3` in an outlined tree in the cgdata.
+
+; RUN: llc -mtriple=arm64-apple-darwin -enable-machine-outliner -codegen-data-use-path=%t_cgdata_base  -aarch64-enable-collect-loh=false \
+; RUN:   %t/local-one.ll -o -  | FileCheck %s --check-prefix=READ
+
+; READ-LABEL: _OUTLINED_FUNCTION_{{.*}}:
+; READ:      adrp x1, l_.str.5
+; READ-NEXT: add x1, x1, l_.str.5
+; READ-NEXT: mov w2
+; READ-NEXT: mov w3
+; READ-NEXT: mov w4
+; READ-NEXT: b
+
+;--- local-two.ll
+@.str.1 = private unnamed_addr constant [3 x i8] c"f1\00", align 1
+@.str.2 = private unnamed_addr constant [3 x i8] c"f2\00", align 1
+@.str.3 = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+
+declare noundef i32 @goo(ptr noundef, ptr noundef, i32, i32, i32)
+define i32 @f1() minsize {
+entry:
+  %call = tail call noundef i32 @goo(ptr noundef nonnull @.str.1, ptr noundef nonnull @.str.3, i32 1, i32 2, i32 3)
+  ret i32 %call
+}
+define i32 @f2() minsize {
+entry:
+  %call = tail call noundef i32 @goo(ptr noundef nonnull @.str.2, ptr noundef nonnull @.str.3, i32 1, i32 2, i32 3)
+  ret i32 %call
+}
+
+;--- local-one.ll
+@.str.4 = private unnamed_addr constant [3 x i8] c"f3\00", align 1
+@.str.5 = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+
+declare noundef i32 @goo(ptr noundef, ptr noundef, i32, i32, i32)
+define i32 @f1() minsize {
+entry:
+  %call = tail call noundef i32 @goo(ptr noundef nonnull @.str.4, ptr noundef nonnull @.str.5, i32 1, i32 2, i32 3)
+  ret i32 %call
+}
diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll
index d4d89a7c9c22e..0daa6e7f16202 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -334,3 +334,176 @@ define <4 x i16> @concat_undef_first_use_undef(ptr %p1, ptr %p2) {
   %t = shufflevector <8 x i16> %ext1, <8 x i16> %ext2, <4 x i32> <i32 poison, i32 poison, i32 2, i32 3>
   ret <4 x i16> %t
 }
+
+define <8 x i16> @concat_low_low_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
+; CHECK-LABEL: concat_low_low_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x i16> %a_vec, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle.i = shufflevector <8 x i16> %b_vec, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle.i4 = shufflevector <4 x i16> %shuffle.i3, <4 x i16> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle.i4
+}
+
+define <8 x i16> @concat_high_low_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
+; CHECK-SD-LABEL: concat_high_low_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_high_low_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x i16> %a_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i = shufflevector <8 x i16> %b_vec, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle.i4 = shufflevector <4 x i16> %shuffle.i3, <4 x i16> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle.i4
+}
+
+define <8 x i16> @concat_low_high_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
+; CHECK-SD-LABEL: concat_low_high_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_low_high_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v1.d[1]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x i16> %a_vec, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle.i = shufflevector <8 x i16> %b_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i4 = shufflevector <4 x i16> %shuffle.i3, <4 x i16> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle.i4
+}
+
+define <8 x i16> @concat_high_high_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
+; CHECK-LABEL: concat_high_high_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x i16> %a_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i = shufflevector <8 x i16> %b_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i4 = shufflevector <4 x i16> %shuffle.i3, <4 x i16> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %shuffle.i4
+}
+
+define <8 x half> @concat_high_high_v8f16(<8 x half> %a_vec, <8 x half> %b_vec) {
+; CHECK-LABEL: concat_high_high_v8f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x half> %a_vec, <8 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i = shufflevector <8 x half> %b_vec, <8 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i4 = shufflevector <4 x half> %shuffle.i3, <4 x half> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x half> %shuffle.i4
+}
+
+define <8 x bfloat> @concat_high_high_v8bf16(<8 x bfloat> %a_vec, <8 x bfloat> %b_vec) {
+; CHECK-LABEL: concat_high_high_v8bf16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <8 x bfloat> %a_vec, <8 x bfloat> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i = shufflevector <8 x bfloat> %b_vec, <8 x bfloat> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle.i4 = shufflevector <4 x bfloat> %shuffle.i3, <4 x bfloat> %shuffle.i, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x bfloat> %shuffle.i4
+}
+
+define <4 x i32> @concat_high_high_v4i32(<4 x i32> %a_vec, <4 x i32> %b_vec) {
+; CHECK-SD-LABEL: concat_high_high_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_high_high_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <4 x i32> %a_vec, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+  %shuffle.i = shufflevector <4 x i32> %b_vec, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+  %shuffle.i4 = shufflevector <2 x i32> %shuffle.i3, <2 x i32> %shuffle.i, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuffle.i4
+}
+
+define <4 x float> @concat_high_high_v4f32(<4 x float> %a_vec, <4 x float> %b_vec) {
+; CHECK-SD-LABEL: concat_high_high_v4f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_high_high_v4f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <4 x float> %a_vec, <4 x float> poison, <2 x i32> <i32 2, i32 3>
+  %shuffle.i = shufflevector <4 x float> %b_vec, <4 x float> poison, <2 x i32> <i32 2, i32 3>
+  %shuffle.i4 = shufflevector <2 x float> %shuffle.i3, <2 x float> %shuffle.i, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuffle.i4
+}
+
+define <16 x i8> @concat_high_high_v16i8(<16 x i8> %a_vec, <16 x i8> %b_vec) {
+; CHECK-LABEL: concat_high_high_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.d[0], v0.d[1]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <16 x i8> %a_vec, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle.i = shufflevector <16 x i8> %b_vec, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle.i4 = shufflevector <8 x i8> %shuffle.i3, <8 x i8> %shuffle.i, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle.i4
+}
+
+define <2 x i64> @concat_high_high_v2i64(<2 x i64> %a_vec, <2 x i64> %b_vec) {
+; CHECK-SD-LABEL: concat_high_high_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_high_high_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v0.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <2 x i64> %a_vec, <2 x i64> poison, <1 x i32> <i32 1>
+  %shuffle.i = shufflevector <2 x i64> %b_vec, <2 x i64> poison, <1 x i32> <i32 1>
+  %shuffle.i4 = shufflevector <1 x i64> %shuffle.i3, <1 x i64> %shuffle.i, <2 x i32> <i32 0, i32 1>
+  ret <2 x i64> %shuffle.i4
+}
+
+define <2 x double> @concat_high_high_v2f64(<2 x double> %a_vec, <2 x double> %b_vec) {
+; CHECK-SD-LABEL: concat_high_high_v2f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: concat_high_high_v2f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v0.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
+; CHECK-GI-NEXT:    ret
+entry:
+  %shuffle.i3 = shufflevector <2 x double> %a_vec, <2 x double> poison, <1 x i32> <i32 1>
+  %shuffle.i = shufflevector <2 x double> %b_vec, <2 x double> poison, <1 x i32> <i32 1>
+  %shuffle.i4 = shufflevector <1 x double> %shuffle.i3, <1 x double> %shuffle.i, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuffle.i4
+}
diff --git a/llvm/test/CodeGen/AArch64/cpus.ll b/llvm/test/CodeGen/AArch64/cpus.ll
index 3e593a82fdf28..e9722f348f411 100644
--- a/llvm/test/CodeGen/AArch64/cpus.ll
+++ b/llvm/test/CodeGen/AArch64/cpus.ll
@@ -36,6 +36,7 @@
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=tsv110 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=apple-latest 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=a64fx 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=fujitsu-monaka 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=ampere1 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=ampere1a 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=ampere1b 2>&1 | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll
index 5a5dee0b53d43..4cb1d5b2fb345 100644
--- a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll
@@ -5,7 +5,7 @@
 
 declare void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8>, <vscale x 16 x ptr>, i32 immarg, <vscale x 16 x i1>)
 
-define fastcc i8 @allocno_reload_assign() {
+define fastcc i8 @allocno_reload_assign(ptr %p) {
 ; CHECK-LABEL: allocno_reload_assign:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, xzr
@@ -14,8 +14,8 @@ define fastcc i8 @allocno_reload_assign() {
 ; CHECK-NEXT:    cmpeq p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    uzp1 p0.s, p0.s, p0.s
 ; CHECK-NEXT:    uzp1 p0.h, p0.h, p0.h
-; CHECK-NEXT:    uzp1 p0.b, p0.b, p0.b
-; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    uzp1 p8.b, p0.b, p0.b
+; CHECK-NEXT:    mov z0.b, p8/z, #1 // =0x1
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    uunpklo z1.h, z0.b
@@ -30,34 +30,35 @@ define fastcc i8 @allocno_reload_assign() {
 ; CHECK-NEXT:    punpklo p1.h, p0.b
 ; CHECK-NEXT:    punpkhi p0.h, p0.b
 ; CHECK-NEXT:    punpklo p2.h, p1.b
-; CHECK-NEXT:    punpkhi p3.h, p1.b
+; CHECK-NEXT:    punpkhi p4.h, p1.b
 ; CHECK-NEXT:    uunpklo z0.d, z2.s
 ; CHECK-NEXT:    uunpkhi z1.d, z2.s
-; CHECK-NEXT:    punpklo p5.h, p0.b
+; CHECK-NEXT:    punpklo p6.h, p0.b
 ; CHECK-NEXT:    uunpklo z2.d, z3.s
 ; CHECK-NEXT:    uunpkhi z3.d, z3.s
-; CHECK-NEXT:    punpkhi p7.h, p0.b
+; CHECK-NEXT:    punpkhi p0.h, p0.b
 ; CHECK-NEXT:    uunpklo z4.d, z5.s
 ; CHECK-NEXT:    uunpkhi z5.d, z5.s
 ; CHECK-NEXT:    uunpklo z6.d, z7.s
 ; CHECK-NEXT:    uunpkhi z7.d, z7.s
-; CHECK-NEXT:    punpklo p0.h, p2.b
-; CHECK-NEXT:    punpkhi p1.h, p2.b
-; CHECK-NEXT:    punpklo p2.h, p3.b
-; CHECK-NEXT:    punpkhi p3.h, p3.b
-; CHECK-NEXT:    punpklo p4.h, p5.b
-; CHECK-NEXT:    punpkhi p5.h, p5.b
-; CHECK-NEXT:    punpklo p6.h, p7.b
-; CHECK-NEXT:    punpkhi p7.h, p7.b
+; CHECK-NEXT:    punpklo p1.h, p2.b
+; CHECK-NEXT:    punpkhi p2.h, p2.b
+; CHECK-NEXT:    punpklo p3.h, p4.b
+; CHECK-NEXT:    punpkhi p4.h, p4.b
+; CHECK-NEXT:    punpklo p5.h, p6.b
+; CHECK-NEXT:    punpkhi p6.h, p6.b
+; CHECK-NEXT:    punpklo p7.h, p0.b
+; CHECK-NEXT:    punpkhi p0.h, p0.b
 ; CHECK-NEXT:  .LBB0_1: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    st1b { z0.d }, p0, [z16.d]
-; CHECK-NEXT:    st1b { z1.d }, p1, [z16.d]
-; CHECK-NEXT:    st1b { z2.d }, p2, [z16.d]
-; CHECK-NEXT:    st1b { z3.d }, p3, [z16.d]
-; CHECK-NEXT:    st1b { z4.d }, p4, [z16.d]
-; CHECK-NEXT:    st1b { z5.d }, p5, [z16.d]
-; CHECK-NEXT:    st1b { z6.d }, p6, [z16.d]
-; CHECK-NEXT:    st1b { z7.d }, p7, [z16.d]
+; CHECK-NEXT:    st1b { z0.d }, p1, [z16.d]
+; CHECK-NEXT:    st1b { z1.d }, p2, [z16.d]
+; CHECK-NEXT:    st1b { z2.d }, p3, [z16.d]
+; CHECK-NEXT:    st1b { z3.d }, p4, [z16.d]
+; CHECK-NEXT:    st1b { z4.d }, p5, [z16.d]
+; CHECK-NEXT:    st1b { z5.d }, p6, [z16.d]
+; CHECK-NEXT:    st1b { z6.d }, p7, [z16.d]
+; CHECK-NEXT:    st1b { z7.d }, p0, [z16.d]
+; CHECK-NEXT:    str p8, [x0]
 ; CHECK-NEXT:    b .LBB0_1
   br label %1
 
@@ -66,6 +67,7 @@ define fastcc i8 @allocno_reload_assign() {
   %constexpr1 = shufflevector <vscale x 16 x i1> %constexpr, <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer
   %constexpr2 = xor <vscale x 16 x i1> %constexpr1, shufflevector (<vscale x 16 x i1> insertelement (<vscale x 16 x i1> poison, i1 true, i64 0), <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer)
   call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> zeroinitializer, i32 0, <vscale x 16 x i1> %constexpr2)
+  store <vscale x 16 x i1> %constexpr, ptr %p, align 16
   br label %1
 }
 
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
index a48a4e0e723eb..fb366564723db 100644
--- a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
+++ b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
@@ -5,10 +5,8 @@ define i1 @combine_setcc_eq_vecreduce_or_v8i1(<8 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_eq_vecreduce_or_v8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    umaxv b0, v0.8b
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %cmp1 = icmp eq <8 x i8> %a, zeroinitializer
   %cast = bitcast <8 x i1> %cmp1 to i8
@@ -73,9 +71,8 @@ define i1 @combine_setcc_ne_vecreduce_or_v8i1(<8 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_or_v8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    umaxv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <8 x i8> %a, zeroinitializer
   %cast = bitcast <8 x i1> %cmp1 to i8
@@ -132,10 +129,9 @@ define i1 @combine_setcc_ne_vecreduce_or_v64i1(<64 x i8> %a) {
 define i1 @combine_setcc_eq_vecreduce_and_v8i1(<8 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_eq_vecreduce_and_v8i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-NEXT:    uminv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %cmp1 = icmp eq <8 x i8> %a, zeroinitializer
   %cast = bitcast <8 x i1> %cmp1 to i8
@@ -192,11 +188,9 @@ define i1 @combine_setcc_eq_vecreduce_and_v64i1(<64 x i8> %a) {
 define i1 @combine_setcc_ne_vecreduce_and_v8i1(<8 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_and_v8i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    uminv b0, v0.8b
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <8 x i8> %a, zeroinitializer
   %cast = bitcast <8 x i1> %cmp1 to i8
diff --git a/llvm/test/CodeGen/AArch64/dup.ll b/llvm/test/CodeGen/AArch64/dup.ll
new file mode 100644
index 0000000000000..a2ebdd28b16b8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/dup.ll
@@ -0,0 +1,1698 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-none-eabi -verify-machineinstrs -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for dup_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v2i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v2i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v2i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v3i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v3i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v3i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v4i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v4i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v4i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v2fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v2fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v2fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v3fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v3fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v3fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for dup_v4fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v4fp128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v4fp128
+
+define <2 x i8> @dup_v2i8(i8 %a) {
+; CHECK-LABEL: dup_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2s, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
+  ret <2 x i8> %c
+}
+
+define <2 x i8> @duplane0_v2i8(<2 x i8> %b) {
+; CHECK-LABEL: duplane0_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
+  ret <2 x i8> %c
+}
+
+define <2 x i8> @loaddup_v2i8(ptr %p) {
+; CHECK-LABEL: loaddup_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    dup v0.2s, w8
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <2 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
+  ret <2 x i8> %c
+}
+
+define <3 x i8> @dup_v3i8(i8 %a) {
+; CHECK-SD-LABEL: dup_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w1, w0
+; CHECK-SD-NEXT:    mov w2, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    dup v0.8b, w0
+; CHECK-GI-NEXT:    umov w0, v0.b[0]
+; CHECK-GI-NEXT:    umov w1, v0.b[1]
+; CHECK-GI-NEXT:    umov w2, v0.b[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <3 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <3 x i8> %b, <3 x i8> poison, <3 x i32> zeroinitializer
+  ret <3 x i8> %c
+}
+
+define <3 x i8> @duplane0_v3i8(<3 x i8> %b) {
+; CHECK-SD-LABEL: duplane0_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w1, w0
+; CHECK-SD-NEXT:    mov w2, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: duplane0_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    mov v0.b[1], w1
+; CHECK-GI-NEXT:    mov v0.b[2], w2
+; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-GI-NEXT:    umov w0, v0.b[0]
+; CHECK-GI-NEXT:    umov w1, v0.b[1]
+; CHECK-GI-NEXT:    umov w2, v0.b[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = shufflevector <3 x i8> %b, <3 x i8> poison, <3 x i32> zeroinitializer
+  ret <3 x i8> %c
+}
+
+define <3 x i8> @loaddup_v3i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w0, [x0]
+; CHECK-SD-NEXT:    mov w1, w0
+; CHECK-SD-NEXT:    mov w2, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.8b }, [x0]
+; CHECK-GI-NEXT:    umov w0, v0.b[0]
+; CHECK-GI-NEXT:    umov w1, v0.b[1]
+; CHECK-GI-NEXT:    umov w2, v0.b[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <3 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <3 x i8> %b, <3 x i8> poison, <3 x i32> zeroinitializer
+  ret <3 x i8> %c
+}
+
+define <4 x i8> @dup_v4i8(i8 %a) {
+; CHECK-SD-LABEL: dup_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v0.4h, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    dup v0.8b, w0
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <4 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
+  ret <4 x i8> %c
+}
+
+define <4 x i8> @duplane0_v4i8(<4 x i8> %b) {
+; CHECK-SD-LABEL: duplane0_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: duplane0_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
+  ret <4 x i8> %c
+}
+
+define <4 x i8> @loaddup_v4i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.8b }, [x0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <4 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
+  ret <4 x i8> %c
+}
+
+define <8 x i8> @dup_v8i8(i8 %a) {
+; CHECK-LABEL: dup_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8b, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <8 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
+  ret <8 x i8> %c
+}
+
+define <8 x i8> @duplane0_v8i8(<8 x i8> %b) {
+; CHECK-LABEL: duplane0_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
+  ret <8 x i8> %c
+}
+
+define <8 x i8> @loaddup_v8i8(ptr %p) {
+; CHECK-LABEL: loaddup_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.8b }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <8 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
+  ret <8 x i8> %c
+}
+
+define <16 x i8> @dup_v16i8(i8 %a) {
+; CHECK-LABEL: dup_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.16b, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <16 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %c
+}
+
+define <16 x i8> @duplane0_v16i8(<16 x i8> %b) {
+; CHECK-LABEL: duplane0_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.16b, v0.b[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %c
+}
+
+define <16 x i8> @loaddup_v16i8(ptr %p) {
+; CHECK-LABEL: loaddup_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.16b }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <16 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %c
+}
+
+define <32 x i8> @dup_v32i8(i8 %a) {
+; CHECK-LABEL: dup_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.16b, w0
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <32 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %c
+}
+
+define <32 x i8> @duplane0_v32i8(<32 x i8> %b) {
+; CHECK-LABEL: duplane0_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.16b, v0.b[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %c
+}
+
+define <32 x i8> @loaddup_v32i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.16b }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.16b }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.16b }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <32 x i8> poison, i8 %a, i64 0
+  %c = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %c
+}
+
+define <2 x i16> @dup_v2i16(i16 %a) {
+; CHECK-SD-LABEL: dup_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v0.2s, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    dup v0.4h, w0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <2 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
+  ret <2 x i16> %c
+}
+
+define <2 x i16> @duplane0_v2i16(<2 x i16> %b) {
+; CHECK-SD-LABEL: duplane0_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: duplane0_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
+  ret <2 x i16> %c
+}
+
+define <2 x i16> @loaddup_v2i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    dup v0.2s, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <2 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
+  ret <2 x i16> %c
+}
+
+define <3 x i16> @dup_v3i16(i16 %a) {
+; CHECK-LABEL: dup_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4h, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <3 x i16> %b, <3 x i16> poison, <3 x i32> zeroinitializer
+  ret <3 x i16> %c
+}
+
+define <3 x i16> @duplane0_v3i16(<3 x i16> %b) {
+; CHECK-LABEL: duplane0_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x i16> %b, <3 x i16> poison, <3 x i32> zeroinitializer
+  ret <3 x i16> %c
+}
+
+define <3 x i16> @loaddup_v3i16(ptr %p) {
+; CHECK-LABEL: loaddup_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <3 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <3 x i16> %b, <3 x i16> poison, <3 x i32> zeroinitializer
+  ret <3 x i16> %c
+}
+
+define <4 x i16> @dup_v4i16(i16 %a) {
+; CHECK-LABEL: dup_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4h, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
+  ret <4 x i16> %c
+}
+
+define <4 x i16> @duplane0_v4i16(<4 x i16> %b) {
+; CHECK-LABEL: duplane0_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
+  ret <4 x i16> %c
+}
+
+define <4 x i16> @loaddup_v4i16(ptr %p) {
+; CHECK-LABEL: loaddup_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <4 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
+  ret <4 x i16> %c
+}
+
+define <8 x i16> @dup_v8i16(i16 %a) {
+; CHECK-LABEL: dup_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <8 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %c
+}
+
+define <8 x i16> @duplane0_v8i16(<8 x i16> %b) {
+; CHECK-LABEL: duplane0_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %c
+}
+
+define <8 x i16> @loaddup_v8i16(ptr %p) {
+; CHECK-LABEL: loaddup_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <8 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %c
+}
+
+define <16 x i16> @dup_v16i16(i16 %a) {
+; CHECK-LABEL: dup_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, w0
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <16 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %c
+}
+
+define <16 x i16> @duplane0_v16i16(<16 x i16> %b) {
+; CHECK-LABEL: duplane0_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %c
+}
+
+define <16 x i16> @loaddup_v16i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.8h }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <16 x i16> poison, i16 %a, i64 0
+  %c = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %c
+}
+
+define <2 x i32> @dup_v2i32(i32 %a) {
+; CHECK-LABEL: dup_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2s, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @duplane0_v2i32(<2 x i32> %b) {
+; CHECK-LABEL: duplane0_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @loaddup_v2i32(ptr %p) {
+; CHECK-LABEL: loaddup_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.2s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <2 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
+  ret <2 x i32> %c
+}
+
+define <3 x i32> @dup_v3i32(i32 %a) {
+; CHECK-LABEL: dup_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <3 x i32> %b, <3 x i32> poison, <3 x i32> zeroinitializer
+  ret <3 x i32> %c
+}
+
+define <3 x i32> @duplane0_v3i32(<3 x i32> %b) {
+; CHECK-LABEL: duplane0_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x i32> %b, <3 x i32> poison, <3 x i32> zeroinitializer
+  ret <3 x i32> %c
+}
+
+define <3 x i32> @loaddup_v3i32(ptr %p) {
+; CHECK-LABEL: loaddup_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <3 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <3 x i32> %b, <3 x i32> poison, <3 x i32> zeroinitializer
+  ret <3 x i32> %c
+}
+
+define <4 x i32> @dup_v4i32(i32 %a) {
+; CHECK-LABEL: dup_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, w0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @duplane0_v4i32(<4 x i32> %b) {
+; CHECK-LABEL: duplane0_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @loaddup_v4i32(ptr %p) {
+; CHECK-LABEL: loaddup_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <4 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @dup_v8i32(i32 %a) {
+; CHECK-LABEL: dup_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, w0
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <8 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %c
+}
+
+define <8 x i32> @duplane0_v8i32(<8 x i32> %b) {
+; CHECK-LABEL: duplane0_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %c
+}
+
+define <8 x i32> @loaddup_v8i32(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.4s }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <8 x i32> poison, i32 %a, i64 0
+  %c = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %c
+}
+
+define <2 x i64> @dup_v2i64(i64 %a) {
+; CHECK-LABEL: dup_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, x0
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @duplane0_v2i64(<2 x i64> %b) {
+; CHECK-LABEL: duplane0_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @loaddup_v2i64(ptr %p) {
+; CHECK-LABEL: loaddup_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <2 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %c
+}
+
+define <3 x i64> @dup_v3i64(i64 %a) {
+; CHECK-SD-LABEL: dup_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d0, x0
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    dup v0.2d, x0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    fmov d2, d0
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <3 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <3 x i64> %b, <3 x i64> poison, <3 x i32> zeroinitializer
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @duplane0_v3i64(<3 x i64> %b) {
+; CHECK-SD-LABEL: duplane0_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: duplane0_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d2, d0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov v2.d[1], v1.d[0]
+; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = shufflevector <3 x i64> %b, <3 x i64> poison, <3 x i32> zeroinitializer
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @loaddup_v3i64(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-GI-NEXT:    ld1r { v2.2d }, [x0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <3 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <3 x i64> %b, <3 x i64> poison, <3 x i32> zeroinitializer
+  ret <3 x i64> %c
+}
+
+define <4 x i64> @dup_v4i64(i64 %a) {
+; CHECK-LABEL: dup_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, x0
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @duplane0_v4i64(<4 x i64> %b) {
+; CHECK-LABEL: duplane0_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @loaddup_v4i64(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.2d }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <4 x i64> poison, i64 %a, i64 0
+  %c = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %c
+}
+
+define <2 x i128> @dup_v2i128(i128 %a) {
+; CHECK-LABEL: dup_v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <2 x i128> %b, <2 x i128> poison, <2 x i32> zeroinitializer
+  ret <2 x i128> %c
+}
+
+define <2 x i128> @duplane0_v2i128(<2 x i128> %b) {
+; CHECK-LABEL: duplane0_v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i128> %b, <2 x i128> poison, <2 x i32> zeroinitializer
+  ret <2 x i128> %c
+}
+
+define <2 x i128> @loaddup_v2i128(ptr %p) {
+; CHECK-LABEL: loaddup_v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp x2, x1, [x0]
+; CHECK-NEXT:    mov x0, x2
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <2 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <2 x i128> %b, <2 x i128> poison, <2 x i32> zeroinitializer
+  ret <2 x i128> %c
+}
+
+define <3 x i128> @dup_v3i128(i128 %a) {
+; CHECK-LABEL: dup_v3i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x0
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <3 x i128> %b, <3 x i128> poison, <3 x i32> zeroinitializer
+  ret <3 x i128> %c
+}
+
+define <3 x i128> @duplane0_v3i128(<3 x i128> %b) {
+; CHECK-LABEL: duplane0_v3i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x0
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x i128> %b, <3 x i128> poison, <3 x i32> zeroinitializer
+  ret <3 x i128> %c
+}
+
+define <3 x i128> @loaddup_v3i128(ptr %p) {
+; CHECK-LABEL: loaddup_v3i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp x2, x1, [x0]
+; CHECK-NEXT:    mov x0, x2
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x2
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <3 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <3 x i128> %b, <3 x i128> poison, <3 x i32> zeroinitializer
+  ret <3 x i128> %c
+}
+
+define <4 x i128> @dup_v4i128(i128 %a) {
+; CHECK-LABEL: dup_v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x0
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    mov x6, x0
+; CHECK-NEXT:    mov x7, x1
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <4 x i128> %b, <4 x i128> poison, <4 x i32> zeroinitializer
+  ret <4 x i128> %c
+}
+
+define <4 x i128> @duplane0_v4i128(<4 x i128> %b) {
+; CHECK-LABEL: duplane0_v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x2, x0
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x0
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    mov x6, x0
+; CHECK-NEXT:    mov x7, x1
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i128> %b, <4 x i128> poison, <4 x i32> zeroinitializer
+  ret <4 x i128> %c
+}
+
+define <4 x i128> @loaddup_v4i128(ptr %p) {
+; CHECK-LABEL: loaddup_v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp x2, x1, [x0]
+; CHECK-NEXT:    mov x0, x2
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    mov x4, x2
+; CHECK-NEXT:    mov x5, x1
+; CHECK-NEXT:    mov x6, x2
+; CHECK-NEXT:    mov x7, x1
+; CHECK-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <4 x i128> poison, i128 %a, i64 0
+  %c = shufflevector <4 x i128> %b, <4 x i128> poison, <4 x i32> zeroinitializer
+  ret <4 x i128> %c
+}
+
+define <2 x half> @dup_v2half(half %a) {
+; CHECK-LABEL: dup_v2half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x half> poison, half %a, i64 0
+  %c = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> zeroinitializer
+  ret <2 x half> %c
+}
+
+define <2 x half> @duplane0_v2half(<2 x half> %b) {
+; CHECK-LABEL: duplane0_v2half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> zeroinitializer
+  ret <2 x half> %c
+}
+
+define <2 x half> @loaddup_v2half(ptr %p) {
+; CHECK-LABEL: loaddup_v2half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <2 x half> poison, half %a, i64 0
+  %c = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> zeroinitializer
+  ret <2 x half> %c
+}
+
+define <3 x half> @dup_v3half(half %a) {
+; CHECK-LABEL: dup_v3half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x half> poison, half %a, i64 0
+  %c = shufflevector <3 x half> %b, <3 x half> poison, <3 x i32> zeroinitializer
+  ret <3 x half> %c
+}
+
+define <3 x half> @duplane0_v3half(<3 x half> %b) {
+; CHECK-LABEL: duplane0_v3half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x half> %b, <3 x half> poison, <3 x i32> zeroinitializer
+  ret <3 x half> %c
+}
+
+define <3 x half> @loaddup_v3half(ptr %p) {
+; CHECK-LABEL: loaddup_v3half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <3 x half> poison, half %a, i64 0
+  %c = shufflevector <3 x half> %b, <3 x half> poison, <3 x i32> zeroinitializer
+  ret <3 x half> %c
+}
+
+define <4 x half> @dup_v4half(half %a) {
+; CHECK-LABEL: dup_v4half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x half> poison, half %a, i64 0
+  %c = shufflevector <4 x half> %b, <4 x half> poison, <4 x i32> zeroinitializer
+  ret <4 x half> %c
+}
+
+define <4 x half> @duplane0_v4half(<4 x half> %b) {
+; CHECK-LABEL: duplane0_v4half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x half> %b, <4 x half> poison, <4 x i32> zeroinitializer
+  ret <4 x half> %c
+}
+
+define <4 x half> @loaddup_v4half(ptr %p) {
+; CHECK-LABEL: loaddup_v4half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <4 x half> poison, half %a, i64 0
+  %c = shufflevector <4 x half> %b, <4 x half> poison, <4 x i32> zeroinitializer
+  ret <4 x half> %c
+}
+
+define <8 x half> @dup_v8half(half %a) {
+; CHECK-LABEL: dup_v8half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <8 x half> poison, half %a, i64 0
+  %c = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> zeroinitializer
+  ret <8 x half> %c
+}
+
+define <8 x half> @duplane0_v8half(<8 x half> %b) {
+; CHECK-LABEL: duplane0_v8half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> zeroinitializer
+  ret <8 x half> %c
+}
+
+define <8 x half> @loaddup_v8half(ptr %p) {
+; CHECK-LABEL: loaddup_v8half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <8 x half> poison, half %a, i64 0
+  %c = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> zeroinitializer
+  ret <8 x half> %c
+}
+
+define <16 x half> @dup_v16half(half %a) {
+; CHECK-SD-LABEL: dup_v16half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-SD-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v16half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-GI-NEXT:    dup v2.8h, v0.h[0]
+; CHECK-GI-NEXT:    dup v1.8h, v0.h[0]
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <16 x half> poison, half %a, i64 0
+  %c = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> zeroinitializer
+  ret <16 x half> %c
+}
+
+define <16 x half> @duplane0_v16half(<16 x half> %b) {
+; CHECK-LABEL: duplane0_v16half:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> zeroinitializer
+  ret <16 x half> %c
+}
+
+define <16 x half> @loaddup_v16half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v16half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v16half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.8h }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <16 x half> poison, half %a, i64 0
+  %c = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> zeroinitializer
+  ret <16 x half> %c
+}
+
+define <2 x bfloat> @dup_v2bfloat(bfloat %a) {
+; CHECK-LABEL: dup_v2bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> zeroinitializer
+  ret <2 x bfloat> %c
+}
+
+define <2 x bfloat> @duplane0_v2bfloat(<2 x bfloat> %b) {
+; CHECK-LABEL: duplane0_v2bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> zeroinitializer
+  ret <2 x bfloat> %c
+}
+
+define <2 x bfloat> @loaddup_v2bfloat(ptr %p) {
+; CHECK-LABEL: loaddup_v2bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <2 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> zeroinitializer
+  ret <2 x bfloat> %c
+}
+
+define <3 x bfloat> @dup_v3bfloat(bfloat %a) {
+; CHECK-LABEL: dup_v3bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <3 x bfloat> %b, <3 x bfloat> poison, <3 x i32> zeroinitializer
+  ret <3 x bfloat> %c
+}
+
+define <3 x bfloat> @duplane0_v3bfloat(<3 x bfloat> %b) {
+; CHECK-LABEL: duplane0_v3bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x bfloat> %b, <3 x bfloat> poison, <3 x i32> zeroinitializer
+  ret <3 x bfloat> %c
+}
+
+define <3 x bfloat> @loaddup_v3bfloat(ptr %p) {
+; CHECK-LABEL: loaddup_v3bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <3 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <3 x bfloat> %b, <3 x bfloat> poison, <3 x i32> zeroinitializer
+  ret <3 x bfloat> %c
+}
+
+define <4 x bfloat> @dup_v4bfloat(bfloat %a) {
+; CHECK-LABEL: dup_v4bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <4 x bfloat> %b, <4 x bfloat> poison, <4 x i32> zeroinitializer
+  ret <4 x bfloat> %c
+}
+
+define <4 x bfloat> @duplane0_v4bfloat(<4 x bfloat> %b) {
+; CHECK-LABEL: duplane0_v4bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x bfloat> %b, <4 x bfloat> poison, <4 x i32> zeroinitializer
+  ret <4 x bfloat> %c
+}
+
+define <4 x bfloat> @loaddup_v4bfloat(ptr %p) {
+; CHECK-LABEL: loaddup_v4bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <4 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <4 x bfloat> %b, <4 x bfloat> poison, <4 x i32> zeroinitializer
+  ret <4 x bfloat> %c
+}
+
+define <8 x bfloat> @dup_v8bfloat(bfloat %a) {
+; CHECK-LABEL: dup_v8bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <8 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <8 x i32> zeroinitializer
+  ret <8 x bfloat> %c
+}
+
+define <8 x bfloat> @duplane0_v8bfloat(<8 x bfloat> %b) {
+; CHECK-LABEL: duplane0_v8bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <8 x i32> zeroinitializer
+  ret <8 x bfloat> %c
+}
+
+define <8 x bfloat> @loaddup_v8bfloat(ptr %p) {
+; CHECK-LABEL: loaddup_v8bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <8 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <8 x i32> zeroinitializer
+  ret <8 x bfloat> %c
+}
+
+define <16 x bfloat> @dup_v16bfloat(bfloat %a) {
+; CHECK-SD-LABEL: dup_v16bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-SD-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v16bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-GI-NEXT:    dup v2.8h, v0.h[0]
+; CHECK-GI-NEXT:    dup v1.8h, v0.h[0]
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <16 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <16 x i32> zeroinitializer
+  ret <16 x bfloat> %c
+}
+
+define <16 x bfloat> @duplane0_v16bfloat(<16 x bfloat> %b) {
+; CHECK-LABEL: duplane0_v16bfloat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <16 x i32> zeroinitializer
+  ret <16 x bfloat> %c
+}
+
+define <16 x bfloat> @loaddup_v16bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v16bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v16bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.8h }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <16 x bfloat> poison, bfloat %a, i64 0
+  %c = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <16 x i32> zeroinitializer
+  ret <16 x bfloat> %c
+}
+
+define <2 x float> @dup_v2float(float %a) {
+; CHECK-LABEL: dup_v2float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x float> poison, float %a, i64 0
+  %c = shufflevector <2 x float> %b, <2 x float> poison, <2 x i32> zeroinitializer
+  ret <2 x float> %c
+}
+
+define <2 x float> @duplane0_v2float(<2 x float> %b) {
+; CHECK-LABEL: duplane0_v2float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x float> %b, <2 x float> poison, <2 x i32> zeroinitializer
+  ret <2 x float> %c
+}
+
+define <2 x float> @loaddup_v2float(ptr %p) {
+; CHECK-LABEL: loaddup_v2float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.2s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <2 x float> poison, float %a, i64 0
+  %c = shufflevector <2 x float> %b, <2 x float> poison, <2 x i32> zeroinitializer
+  ret <2 x float> %c
+}
+
+define <3 x float> @dup_v3float(float %a) {
+; CHECK-LABEL: dup_v3float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x float> poison, float %a, i64 0
+  %c = shufflevector <3 x float> %b, <3 x float> poison, <3 x i32> zeroinitializer
+  ret <3 x float> %c
+}
+
+define <3 x float> @duplane0_v3float(<3 x float> %b) {
+; CHECK-LABEL: duplane0_v3float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x float> %b, <3 x float> poison, <3 x i32> zeroinitializer
+  ret <3 x float> %c
+}
+
+define <3 x float> @loaddup_v3float(ptr %p) {
+; CHECK-LABEL: loaddup_v3float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <3 x float> poison, float %a, i64 0
+  %c = shufflevector <3 x float> %b, <3 x float> poison, <3 x i32> zeroinitializer
+  ret <3 x float> %c
+}
+
+define <4 x float> @dup_v4float(float %a) {
+; CHECK-LABEL: dup_v4float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x float> poison, float %a, i64 0
+  %c = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %c
+}
+
+define <4 x float> @duplane0_v4float(<4 x float> %b) {
+; CHECK-LABEL: duplane0_v4float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %c
+}
+
+define <4 x float> @loaddup_v4float(ptr %p) {
+; CHECK-LABEL: loaddup_v4float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <4 x float> poison, float %a, i64 0
+  %c = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %c
+}
+
+define <8 x float> @dup_v8float(float %a) {
+; CHECK-SD-LABEL: dup_v8float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v8float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    dup v2.4s, v0.s[0]
+; CHECK-GI-NEXT:    dup v1.4s, v0.s[0]
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <8 x float> poison, float %a, i64 0
+  %c = shufflevector <8 x float> %b, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %c
+}
+
+define <8 x float> @duplane0_v8float(<8 x float> %b) {
+; CHECK-LABEL: duplane0_v8float:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <8 x float> %b, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %c
+}
+
+define <8 x float> @loaddup_v8float(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v8float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v8float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.4s }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <8 x float> poison, float %a, i64 0
+  %c = shufflevector <8 x float> %b, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %c
+}
+
+define <2 x double> @dup_v2double(double %a) {
+; CHECK-LABEL: dup_v2double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x double> poison, double %a, i64 0
+  %c = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %c
+}
+
+define <2 x double> @duplane0_v2double(<2 x double> %b) {
+; CHECK-LABEL: duplane0_v2double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %c
+}
+
+define <2 x double> @loaddup_v2double(ptr %p) {
+; CHECK-LABEL: loaddup_v2double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <2 x double> poison, double %a, i64 0
+  %c = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %c
+}
+
+define <3 x double> @dup_v3double(double %a) {
+; CHECK-SD-LABEL: dup_v3double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v3double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    dup v3.2d, v0.d[0]
+; CHECK-GI-NEXT:    dup v2.2d, v0.d[0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v3.d[1]
+; CHECK-GI-NEXT:    fmov d0, d3
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <3 x double> poison, double %a, i64 0
+  %c = shufflevector <3 x double> %b, <3 x double> poison, <3 x i32> zeroinitializer
+  ret <3 x double> %c
+}
+
+define <3 x double> @duplane0_v3double(<3 x double> %b) {
+; CHECK-SD-LABEL: duplane0_v3double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: duplane0_v3double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d2, d0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov v2.d[1], v1.d[0]
+; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = shufflevector <3 x double> %b, <3 x double> poison, <3 x i32> zeroinitializer
+  ret <3 x double> %c
+}
+
+define <3 x double> @loaddup_v3double(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v3double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v3double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-GI-NEXT:    ld1r { v2.2d }, [x0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <3 x double> poison, double %a, i64 0
+  %c = shufflevector <3 x double> %b, <3 x double> poison, <3 x i32> zeroinitializer
+  ret <3 x double> %c
+}
+
+define <4 x double> @dup_v4double(double %a) {
+; CHECK-SD-LABEL: dup_v4double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_v4double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    dup v2.2d, v0.d[0]
+; CHECK-GI-NEXT:    dup v1.2d, v0.d[0]
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %b = insertelement <4 x double> poison, double %a, i64 0
+  %c = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %c
+}
+
+define <4 x double> @duplane0_v4double(<4 x double> %b) {
+; CHECK-LABEL: duplane0_v4double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %c
+}
+
+define <4 x double> @loaddup_v4double(ptr %p) {
+; CHECK-SD-LABEL: loaddup_v4double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_v4double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-GI-NEXT:    ld1r { v1.2d }, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <4 x double> poison, double %a, i64 0
+  %c = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %c
+}
+
+define <2 x fp128> @dup_v2fp128(fp128 %a) {
+; CHECK-LABEL: dup_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <2 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <2 x fp128> %b, <2 x fp128> poison, <2 x i32> zeroinitializer
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @duplane0_v2fp128(<2 x fp128> %b) {
+; CHECK-LABEL: duplane0_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <2 x fp128> %b, <2 x fp128> poison, <2 x i32> zeroinitializer
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @loaddup_v2fp128(ptr %p) {
+; CHECK-LABEL: loaddup_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <2 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <2 x fp128> %b, <2 x fp128> poison, <2 x i32> zeroinitializer
+  ret <2 x fp128> %c
+}
+
+define <3 x fp128> @dup_v3fp128(fp128 %a) {
+; CHECK-LABEL: dup_v3fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <3 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <3 x fp128> %b, <3 x fp128> poison, <3 x i32> zeroinitializer
+  ret <3 x fp128> %c
+}
+
+define <3 x fp128> @duplane0_v3fp128(<3 x fp128> %b) {
+; CHECK-LABEL: duplane0_v3fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <3 x fp128> %b, <3 x fp128> poison, <3 x i32> zeroinitializer
+  ret <3 x fp128> %c
+}
+
+define <3 x fp128> @loaddup_v3fp128(ptr %p) {
+; CHECK-LABEL: loaddup_v3fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <3 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <3 x fp128> %b, <3 x fp128> poison, <3 x i32> zeroinitializer
+  ret <3 x fp128> %c
+}
+
+define <4 x fp128> @dup_v4fp128(fp128 %a) {
+; CHECK-LABEL: dup_v4fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    mov v3.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %b = insertelement <4 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <4 x fp128> %b, <4 x fp128> poison, <4 x i32> zeroinitializer
+  ret <4 x fp128> %c
+}
+
+define <4 x fp128> @duplane0_v4fp128(<4 x fp128> %b) {
+; CHECK-LABEL: duplane0_v4fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    mov v3.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x fp128> %b, <4 x fp128> poison, <4 x i32> zeroinitializer
+  ret <4 x fp128> %c
+}
+
+define <4 x fp128> @loaddup_v4fp128(ptr %p) {
+; CHECK-LABEL: loaddup_v4fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    mov v3.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <4 x fp128> poison, fp128 %a, i64 0
+  %c = shufflevector <4 x fp128> %b, <4 x fp128> poison, <4 x i32> zeroinitializer
+  ret <4 x fp128> %c
+}
diff --git a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
index ffa7453e48b4f..f9878adfe5e44 100644
--- a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
+++ b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
@@ -91,10 +91,10 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @c
   ; CHECK-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
-  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3342346 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:fpr64 = COPY %2
   ; CHECK-NEXT:   [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
-  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3342346 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
   ; CHECK-NEXT:   [[FNEGDr:%[0-9]+]]:fpr64 = FNEGDr %2
   ; CHECK-NEXT:   nofpexcept FCMPDrr %4, killed [[FNEGDr]], implicit-def $nzcv, implicit $fpcr
   ; CHECK-NEXT:   Bcc 1, %bb.2, implicit $nzcv
@@ -111,10 +111,10 @@ body:             |
 
     %6:gpr64common = LOADgot target-flags(aarch64-got) @c
     %3:fpr64 = LDRDui %6, 0 :: (dereferenceable load (s64) from @c)
-    INLINEASM &"", 1 /* sideeffect attdialect */, 3342346 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, %3(tied-def 3)
+    INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, %3(tied-def 3)
     %0:fpr64 = COPY %2
     %5:fpr64 = LDRDui %6, 0 :: (dereferenceable load (s64) from @c)
-    INLINEASM &"", 1 /* sideeffect attdialect */, 3342346 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, %5(tied-def 3)
+    INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, %5(tied-def 3)
     %7:fpr64 = FNEGDr %2
     nofpexcept FCMPDrr %4, killed %7, implicit-def $nzcv, implicit $fpcr
     Bcc 1, %bb.2, implicit $nzcv
diff --git a/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
index b1e48346c2746..89102a8c3770d 100644
--- a/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
+++ b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
@@ -39,7 +39,7 @@
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $x0, implicit $sp, implicit $xzr, implicit $fp {
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $x0, implicit $sp, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BLR $x0, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -62,7 +62,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $sp, implicit $x0, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $x0, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -82,7 +82,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def $x0, implicit-def $w0, implicit-def $fp, implicit-def $w29, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def $x0, implicit-def $w0, implicit-def $w0_hi, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -102,7 +102,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -123,7 +123,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $x8, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $x8, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BLR $x8, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -145,7 +145,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $sp, implicit undef $x0, implicit $xzr, implicit $fp {
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit undef $x0, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit undef $x0, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @objc_retainAutoreleasedReturnValue, implicit-def $lr, implicit internal $sp
diff --git a/llvm/test/CodeGen/AArch64/extract-insert.ll b/llvm/test/CodeGen/AArch64/extract-insert.ll
index 077e5f3d042df..8c133d76ce317 100644
--- a/llvm/test/CodeGen/AArch64/extract-insert.ll
+++ b/llvm/test/CodeGen/AArch64/extract-insert.ll
@@ -5,9 +5,8 @@
 define i32 @trunc_i64_to_i32_le(i64 %x) {
 ; BE-LABEL: trunc_i64_to_i32_le:
 ; BE:       // %bb.0:
-; BE-NEXT:    fmov d0, x0
-; BE-NEXT:    rev64 v0.4s, v0.4s
-; BE-NEXT:    fmov w0, s0
+; BE-NEXT:    lsr x0, x0, #32
+; BE-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; BE-NEXT:    ret
 ;
 ; LE-LABEL: trunc_i64_to_i32_le:
@@ -28,8 +27,8 @@ define i32 @trunc_i64_to_i32_be(i64 %x) {
 ;
 ; LE-LABEL: trunc_i64_to_i32_be:
 ; LE:       // %bb.0:
-; LE-NEXT:    fmov d0, x0
-; LE-NEXT:    mov w0, v0.s[1]
+; LE-NEXT:    lsr x0, x0, #32
+; LE-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; LE-NEXT:    ret
   %ins = insertelement <2 x i64> undef, i64 %x, i32 0
   %bc = bitcast <2 x i64> %ins to <4 x i32>
@@ -40,9 +39,8 @@ define i32 @trunc_i64_to_i32_be(i64 %x) {
 define i16 @trunc_i64_to_i16_le(i64 %x) {
 ; BE-LABEL: trunc_i64_to_i16_le:
 ; BE:       // %bb.0:
-; BE-NEXT:    fmov d0, x0
-; BE-NEXT:    rev64 v0.8h, v0.8h
-; BE-NEXT:    umov w0, v0.h[0]
+; BE-NEXT:    lsr x0, x0, #48
+; BE-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; BE-NEXT:    ret
 ;
 ; LE-LABEL: trunc_i64_to_i16_le:
@@ -63,8 +61,8 @@ define i16 @trunc_i64_to_i16_be(i64 %x) {
 ;
 ; LE-LABEL: trunc_i64_to_i16_be:
 ; LE:       // %bb.0:
-; LE-NEXT:    fmov d0, x0
-; LE-NEXT:    umov w0, v0.h[3]
+; LE-NEXT:    lsr x0, x0, #48
+; LE-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; LE-NEXT:    ret
   %ins = insertelement <2 x i64> undef, i64 %x, i32 0
   %bc = bitcast <2 x i64> %ins to <8 x i16>
@@ -75,9 +73,7 @@ define i16 @trunc_i64_to_i16_be(i64 %x) {
 define i8 @trunc_i32_to_i8_le(i32 %x) {
 ; BE-LABEL: trunc_i32_to_i8_le:
 ; BE:       // %bb.0:
-; BE-NEXT:    fmov s0, w0
-; BE-NEXT:    rev32 v0.16b, v0.16b
-; BE-NEXT:    umov w0, v0.b[0]
+; BE-NEXT:    lsr w0, w0, #24
 ; BE-NEXT:    ret
 ;
 ; LE-LABEL: trunc_i32_to_i8_le:
@@ -96,8 +92,7 @@ define i8 @trunc_i32_to_i8_be(i32 %x) {
 ;
 ; LE-LABEL: trunc_i32_to_i8_be:
 ; LE:       // %bb.0:
-; LE-NEXT:    fmov s0, w0
-; LE-NEXT:    umov w0, v0.b[3]
+; LE-NEXT:    lsr w0, w0, #24
 ; LE-NEXT:    ret
   %ins = insertelement <4 x i32> undef, i32 %x, i32 0
   %bc = bitcast <4 x i32> %ins to <16 x i8>
@@ -115,8 +110,8 @@ define i8 @trunc_i64_to_i8_be(i64 %x) {
 ;
 ; LE-LABEL: trunc_i64_to_i8_be:
 ; LE:       // %bb.0:
-; LE-NEXT:    fmov d0, x0
-; LE-NEXT:    umov w0, v0.b[7]
+; LE-NEXT:    lsr x0, x0, #56
+; LE-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; LE-NEXT:    ret
   %ins = insertelement <3 x i64> undef, i64 %x, i32 0
   %bc = bitcast <3 x i64> %ins to <24 x i8>
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll b/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll
new file mode 100644
index 0000000000000..12bd2db2297d7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+
+define i1 @extract_icmp_v4i32_const_splat_rhs(<4 x i32> %a) {
+; CHECK-LABEL: extract_icmp_v4i32_const_splat_rhs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    cmp w8, #5
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %icmp = icmp ult <4 x i32> %a, splat (i32 5)
+  %ext = extractelement <4 x i1> %icmp, i32 1
+  ret i1 %ext
+}
+
+define i1 @extract_icmp_v4i32_const_splat_lhs(<4 x i32> %a) {
+; CHECK-LABEL: extract_icmp_v4i32_const_splat_lhs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    cmp w8, #7
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+  %icmp = icmp ult <4 x i32> splat(i32 7), %a
+  %ext = extractelement <4 x i1> %icmp, i32 1
+  ret i1 %ext
+}
+
+define i1 @extract_icmp_v4i32_const_vec_rhs(<4 x i32> %a) {
+; CHECK-LABEL: extract_icmp_v4i32_const_vec_rhs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    cmp w8, #234
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %icmp = icmp ult <4 x i32> %a, <i32 5, i32 234, i32 -1, i32 7>
+  %ext = extractelement <4 x i1> %icmp, i32 1
+  ret i1 %ext
+}
+
+define i1 @extract_fcmp_v4f32_const_splat_rhs(<4 x float> %a) {
+; CHECK-LABEL: extract_fcmp_v4f32_const_splat_rhs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    fmov s1, #4.00000000
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
+  %fcmp = fcmp ult <4 x float> %a, splat(float 4.0e+0)
+  %ext = extractelement <4 x i1> %fcmp, i32 1
+  ret i1 %ext
+}
+
+; Tests the code in ExpandIntRes_SETCC
+define i128 @extract_icmp_v1i128(ptr %p) {
+; CHECK-LABEL: extract_icmp_v1i128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x9, x8, [x0]
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    orr x8, x9, x8
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %load = load <1 x i128>, ptr %p, align 16
+  %cmp = icmp eq <1 x i128> %load, zeroinitializer
+  %sext = sext <1 x i1> %cmp to <1 x i128>
+  %res = extractelement <1 x i128> %sext, i32 0
+  ret i128 %res
+}
+
+define void @vector_loop_with_icmp(ptr nocapture noundef writeonly %dest) {
+; CHECK-LABEL: vector_loop_with_icmp:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.d, #0, #1
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    mov w9, #16 // =0x10
+; CHECK-NEXT:    dup v1.2d, x8
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    mov w10, #1 // =0x1
+; CHECK-NEXT:    b .LBB5_2
+; CHECK-NEXT:  .LBB5_1: // %pred.store.continue6
+; CHECK-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    subs x9, x9, #2
+; CHECK-NEXT:    add x8, x8, #8
+; CHECK-NEXT:    b.eq .LBB5_6
+; CHECK-NEXT:  .LBB5_2: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    cmp x11, #14
+; CHECK-NEXT:    b.hi .LBB5_4
+; CHECK-NEXT:  // %bb.3: // %pred.store.if
+; CHECK-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    stur w10, [x8, #-4]
+; CHECK-NEXT:  .LBB5_4: // %pred.store.continue
+; CHECK-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    mov x11, v0.d[1]
+; CHECK-NEXT:    cmp x11, #14
+; CHECK-NEXT:    b.hi .LBB5_1
+; CHECK-NEXT:  // %bb.5: // %pred.store.if5
+; CHECK-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    str w10, [x8]
+; CHECK-NEXT:    b .LBB5_1
+; CHECK-NEXT:  .LBB5_6: // %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %pred.store.continue6 ]
+  %vec.ind = phi <2 x i64> [ <i64 0, i64 1>, %entry ], [ %vec.ind.next, %pred.store.continue6 ]
+  %vec.cmp = icmp ult <2 x i64> %vec.ind, <i64 15, i64 15>
+  %c0 = extractelement <2 x i1> %vec.cmp, i64 0
+  br i1 %c0, label %pred.store.if, label %pred.store.continue
+
+pred.store.if:
+  %arrayidx = getelementptr inbounds i32, ptr %dest, i64 %index
+  store i32 1, ptr %arrayidx, align 4
+  br label %pred.store.continue
+
+pred.store.continue:
+  %c1 = extractelement <2 x i1> %vec.cmp, i64 1
+  br i1 %c1, label %pred.store.if5, label %pred.store.continue6
+
+pred.store.if5:
+  %indexp1 = or disjoint i64 %index, 1
+  %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %indexp1
+  store i32 1, ptr %arrayidx2, align 4
+  br label %pred.store.continue6
+
+pred.store.continue6:
+  %index.next = add i64 %index, 2
+  %vec.ind.next = add <2 x i64> %vec.ind, <i64 2, i64 2>
+  %index.cmp = icmp eq i64 %index.next, 16
+  br i1 %index.cmp, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
+
+
+; Negative tests
+
+define i1 @extract_icmp_v4i32_splat_rhs(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: extract_icmp_v4i32_splat_rhs:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.4s, w0
+; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %ins = insertelement <4 x i32> poison, i32 %b, i32 0
+  %splat = shufflevector <4 x i32> %ins, <4 x i32> poison, <4 x i32> zeroinitializer
+  %icmp = icmp ult <4 x i32> %a, %splat
+  %ext = extractelement <4 x i1> %icmp, i32 1
+  ret i1 %ext
+}
+
+define i1 @extract_icmp_v4i32_splat_rhs_mul_use(<4 x i32> %a, ptr %p) {
+; CHECK-LABEL: extract_icmp_v4i32_splat_rhs_mul_use:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.4s, #235
+; CHECK-NEXT:    adrp x9, .LCPI7_0
+; CHECK-NEXT:    mov x8, x0
+; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI7_0]
+; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v1.4h, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    umov w9, v1.h[1]
+; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    and w0, w9, #0x1
+; CHECK-NEXT:    strb w10, [x8]
+; CHECK-NEXT:    ret
+  %icmp = icmp ult <4 x i32> %a, splat(i32 235)
+  %ext = extractelement <4 x i1> %icmp, i32 1
+  store <4 x i1> %icmp, ptr %p, align 4
+  ret i1 %ext
+}
+
+define i1 @extract_icmp_v4i32_splat_rhs_unknown_idx(<4 x i32> %a, i32 %c) {
+; CHECK-LABEL: extract_icmp_v4i32_splat_rhs_unknown_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movi v1.4s, #127
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    bfi x8, x0, #1, #2
+; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    str d0, [sp, #8]
+; CHECK-NEXT:    ldrh w8, [x8]
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+  %icmp = icmp ult <4 x i32> %a, splat(i32 127)
+  %ext = extractelement <4 x i1> %icmp, i32 %c
+  ret i1 %ext
+}
+
diff --git a/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
new file mode 100644
index 0000000000000..a1093c28467ab
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
+; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define <vscale x 8 x bfloat> @cvt1_bf16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvt1_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt1.nxv8bf16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @cvt2_bf16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvt2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvt2.nxv8bf16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @cvtlt1_bf16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvtlt1_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvtlt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt1.nxv8bf16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @cvtlt2_bf16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvtlt2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvtlt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fp8.cvtlt2.nxv8bf16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x half> @cvt1_f16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvt1_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt1.nxv8f16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @cvt2_f16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvt2_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvt2.nxv8f16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x half> %r
+}
+
+
+define <vscale x 8 x half> @cvtlt1_f16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvtlt1_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvtlt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt1.nxv8f16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @cvtlt2_f16(<vscale x 16 x i8> %s) {
+; CHECK-LABEL: cvtlt2_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvtlt z0.h, z0.b
+; CHECK-NEXT:    ret
+    %r = call <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt2.nxv8f16(<vscale x 16 x i8> %s)
+    ret <vscale x 8 x half> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
index 767ca91a58bb1..5374d4823034f 100644
--- a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
@@ -9,13 +9,11 @@ define i1 @unordered_floating_point_compare_on_v8f32(<8 x float> %a_vec) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.4s, v1.4s, #0.0
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
-; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
-; CHECK-NEXT:    umaxv b0, v0.8b
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %a_cmp = fcmp ule <8 x float> %a_vec, zeroinitializer
   %cmp_result = bitcast <8 x i1> %a_cmp to i8
diff --git a/llvm/test/CodeGen/AArch64/init-undef.mir b/llvm/test/CodeGen/AArch64/init-undef.mir
index 7935c09d7df5e..c9d23006d3523 100644
--- a/llvm/test/CodeGen/AArch64/init-undef.mir
+++ b/llvm/test/CodeGen/AArch64/init-undef.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=aarch64-- -run-pass=init-undef -o - %s | FileCheck %s
+# RUN: llc -mtriple=aarch64-- -aarch64-enable-subreg-liveness-tracking=false -run-pass=init-undef -o - %s | FileCheck %s
+# RUN: llc -mtriple=aarch64-- -aarch64-enable-subreg-liveness-tracking=true -run-pass=init-undef -o - %s | FileCheck %s
 
 ---
 name:            test_stxp_undef
diff --git a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
index 8e29255189bf5..a10d7588cb442 100644
--- a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
@@ -155,7 +155,7 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $x1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: renamable $q1 = LDRQui renamable $x1, 1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 48, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 48, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: STPQi renamable $q0, renamable $q1, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: RET undef $lr
     renamable $q1 = LDRQui renamable $x1, 1 :: (load (s128))
@@ -246,7 +246,7 @@ body:             |
     ; CHECK-LABEL: name: 9-ldrspre-ldrsui-mod-base-reg-no-merge
     ; CHECK: liveins: $s0, $s1, $x0, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: dead early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: dead early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: renamable $x1 = LDRXui renamable $x0, 1 :: (load (s64))
     ; CHECK-NEXT: renamable $s1 = LDRSui renamable $x1, 1 :: (load (s32))
     ; CHECK-NEXT: STPSi renamable $s0, renamable $s1, renamable $x1, 0 :: (store (s32))
@@ -280,7 +280,7 @@ body:             |
     ; CHECK-LABEL: name: 10-ldrspre-ldrsui-used-base-reg-no-merge
     ; CHECK: liveins: $s0, $s1, $x0, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: renamable $x0 = LDRXui renamable $x1, 1 :: (load (s64))
     ; CHECK-NEXT: STRXui renamable $x0, renamable $x0, 1 :: (store (s64))
     ; CHECK-NEXT: renamable $s1 = LDRSui renamable $x1, 1 :: (load (s32))
@@ -315,12 +315,12 @@ body:             |
     ; CHECK-LABEL: name: 11-ldrqpre-ldrqpre-no-merge
     ; CHECK: liveins: $q0, $q1, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 48, implicit $w1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q1 = LDRQpre renamable $x1, 1, implicit $w1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 16, implicit $w1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q1 = LDRQpre renamable $x1, 12, implicit $w1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 16, implicit $w1 :: (load (s128))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q1 = LDRQpre renamable $x1, 16, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 48, implicit $w1, implicit $w1_hi :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q1 = LDRQpre renamable $x1, 1, implicit $w1, implicit $w1_hi :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q1 = LDRQpre renamable $x1, 12, implicit $w1, implicit $w1_hi :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q1 = LDRQpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: STPQi renamable $q0, renamable $q1, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: RET undef $lr
     early-clobber renamable $x1, renamable $q0 = LDRQpre killed renamable $x1, 48 :: (load (s128))
@@ -352,7 +352,7 @@ body:             |
     ; CHECK-LABEL: name: 12-ldrspre-ldrsui-no-merge
     ; CHECK: liveins: $s0, $s1, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 12, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: renamable $s1 = LDRSui renamable $x1, 2 :: (load (s32))
     ; CHECK-NEXT: STPSi renamable $s0, renamable $s1, renamable $x1, 0 :: (store (s32))
     ; CHECK-NEXT: RET undef $lr
@@ -383,7 +383,7 @@ body:             |
     ; CHECK-LABEL: name: 13-ldrqpre-ldrdui-no-merge
     ; CHECK: liveins: $d1, $q0, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: renamable $d1 = LDRDui renamable $x1, 1 :: (load (s64))
     ; CHECK-NEXT: STRQui renamable $q0, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: STRDui renamable $d1, renamable $x1, 1 :: (store (s64))
@@ -415,7 +415,7 @@ body:             |
     ; CHECK-LABEL: name: 14-ldrqpre-strqui-no-merge
     ; CHECK: liveins: $q0, $q1, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: STRQui renamable $q0, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: RET undef $lr
     early-clobber renamable $x1, renamable $q0 = LDRQpre killed renamable $x1, 32 :: (load (s128))
@@ -473,7 +473,7 @@ body:             |
     ; CHECK-LABEL: name: 16-ldrqpre-ldrqui-diff-base-reg-no-merge
     ; CHECK: liveins: $q0, $q1, $x1, $x2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: renamable $q1 = LDRQui renamable $x2, 1 :: (load (s128))
     ; CHECK-NEXT: STPQi renamable $q0, renamable $q1, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: RET undef $lr
@@ -534,7 +534,7 @@ body:             |
     ; CHECK-LABEL: name: 18-ldrqpre-ldurqi-no-merge
     ; CHECK: liveins: $q0, $q1, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1, implicit $w1_hi :: (load (s128))
     ; CHECK-NEXT: renamable $q1 = LDURQi renamable $x1, 1 :: (load (s128))
     ; CHECK-NEXT: STPQi renamable $q0, renamable $q1, renamable $x1, 0 :: (store (s128))
     ; CHECK-NEXT: RET undef $lr
@@ -587,7 +587,7 @@ body:             |
     ; CHECK-LABEL: name: 20-ldrspre-ldrsui-unaligned-no-merge
     ; CHECK: liveins: $s0, $s1, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 251, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $s0 = LDRSpre renamable $x1, 251, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: renamable $s1 = LDRSui renamable $x1, 1 :: (load (s32))
     ; CHECK-NEXT: STPSi renamable $s0, renamable $s1, renamable $x1, 0 :: (store (s32))
     ; CHECK-NEXT: RET undef $lr
@@ -667,7 +667,7 @@ body:             |
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: renamable $x2 = LDRSWui renamable $x1, 1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
     renamable $x2 = LDRSWui renamable $x1, 1 :: (load (s32))
@@ -694,7 +694,7 @@ body:             |
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: renamable $x2 = LDURSWi renamable $x1, 4 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
     renamable $x2 = LDURSWi renamable $x1, 4 :: (load (s32))
@@ -720,12 +720,12 @@ body:             |
     ; CHECK-LABEL: name: 25-ldrswpre-ldrswpre-no-merge
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x0 = LDRSWpre renamable $x1, 48, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x2 = LDRSWpre renamable $x1, 1, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x0 = LDRSWpre renamable $x1, 16, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x2 = LDRSWpre renamable $x1, 12, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 16, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x2 = LDRSWpre renamable $x1, 16, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x0 = LDRSWpre renamable $x1, 48, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x2 = LDRSWpre renamable $x1, 1, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x0 = LDRSWpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $x2 = LDRSWpre renamable $x1, 12, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x2 = LDRSWpre renamable $x1, 16, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
     early-clobber renamable $x1, renamable $x0 = LDRSWpre killed renamable $x1, 48 :: (load (s32))
@@ -755,8 +755,8 @@ body:             |
     ; CHECK-LABEL: name: 26-ldrswpre-ldrwui-no-merge
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1 :: (load (s32))
-    ; CHECK-NEXT: renamable $w2 = LDRWui renamable $x1, 1, implicit-def $x2 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1, implicit $w1_hi :: (load (s32))
+    ; CHECK-NEXT: renamable $w2 = LDRWui renamable $x1, 1, implicit-def $x2, implicit $w2_hi :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
     early-clobber renamable $x1, renamable $x0 = LDRSWpre killed renamable $x1, 40 :: (load (s32))
@@ -782,7 +782,7 @@ body:             |
     ; CHECK-LABEL: name: 27-ldrwpre-ldrswui-no-merge
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x1, renamable $w0 = LDRWpre renamable $x1, 40, implicit $w1 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x1, renamable $w0 = LDRWpre renamable $x1, 40, implicit $w1, implicit $w1_hi :: (load (s32))
     ; CHECK-NEXT: renamable $x2 = LDRSWui renamable $x1, 1 :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
@@ -808,9 +808,9 @@ body:             |
     ; CHECK-LABEL: name: 28-ldrswpre-ldrwpre-no-merge
     ; CHECK: liveins: $x11, $x13
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $x10 = LDRSWpre renamable $x11, 8, implicit $w11 :: (load (s32), align 8)
+    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $x10 = LDRSWpre renamable $x11, 8, implicit $w11, implicit $w11_hi :: (load (s32), align 8)
     ; CHECK-NEXT: $x14 = EORXrs renamable $x11, renamable $x13, 0
-    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $w12 = LDRWpre renamable $x11, 4, implicit $w11 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $w12 = LDRWpre renamable $x11, 4, implicit $w11, implicit $w11_hi :: (load (s32))
     ; CHECK-NEXT: $x13 = EORXrs renamable $x11, renamable $x13, 0
     ; CHECK-NEXT: STPXi renamable $x13, renamable $x14, renamable $x11, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
@@ -838,9 +838,9 @@ body:             |
     ; CHECK-LABEL: name: 29-ldrwpre-ldrswpre-no-merge
     ; CHECK: liveins: $x11, $x13
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $w12 = LDRWpre renamable $x11, 8, implicit $w11 :: (load (s32))
+    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $w12 = LDRWpre renamable $x11, 8, implicit $w11, implicit $w11_hi :: (load (s32))
     ; CHECK-NEXT: $x14 = EORXrs renamable $x11, renamable $x13, 0
-    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $x10 = LDRSWpre renamable $x11, 4, implicit $w11 :: (load (s32), align 8)
+    ; CHECK-NEXT: early-clobber renamable $x11, dead renamable $x10 = LDRSWpre renamable $x11, 4, implicit $w11, implicit $w11_hi :: (load (s32), align 8)
     ; CHECK-NEXT: $x13 = EORXrs renamable $x11, renamable $x13, 0
     ; CHECK-NEXT: STPXi renamable $x13, renamable $x14, renamable $x11, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir b/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
index 8abd56fa20549..700a5b228122f 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
@@ -57,7 +57,7 @@ body:             |
 
 # CHECK: name:            OUTLINED_FUNCTION_0
 # CHECK: bb.0:
-# CHECK: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $d15, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $lr
+# CHECK: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15, $lr
 # CHECK-DAG: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
 # CHECK-NEXT: early-clobber $sp = STRXpre $lr, $sp, -16
diff --git a/llvm/test/CodeGen/AArch64/memtag-merge-writeback.mir b/llvm/test/CodeGen/AArch64/memtag-merge-writeback.mir
new file mode 100644
index 0000000000000..be828f7f1da05
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/memtag-merge-writeback.mir
@@ -0,0 +1,142 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple aarch64-none-elf -mattr=+mte --run-pass=aarch64-ldst-opt %s -o - | FileCheck %s
+
+## When generating code with sanitize_memtag, we make use of the fact that the
+## sp+imm forms of many load and store instructions are not tag-checked, so we
+## can use SP directly instead of needing a register holding the tagged
+## pointer. However, this isn't true for the writeback versions of the
+## instructions, so we can't fold ADDs and SUBs into them in
+## AArch64LoadStoreOptimizer. This would be possible in cases where the
+## loads/stores only access untagged stack slots, but that information isn't
+## easily available after frame index elimination.
+
+--- |
+  define void @pre_index() {
+  entry:
+    ret void
+  }
+  define void @pre_index_memtag() sanitize_memtag {
+  entry:
+    ret void
+  }
+  define void @pre_index_memtag_not_sp() sanitize_memtag {
+  entry:
+    ret void
+  }
+  define void @post_index() {
+  entry:
+    ret void
+  }
+  define void @post_index_memtag() sanitize_memtag {
+  entry:
+    ret void
+  }
+  define void @post_index_memtag_not_sp() sanitize_memtag {
+  entry:
+    ret void
+  }
+...
+---
+name:            pre_index
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: pre_index
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: early-clobber $sp = STRXpre killed renamable $x0, $sp, 16
+    ; CHECK-NEXT: RET undef $lr
+    $sp = frame-setup SUBXri $sp, 16, 0
+    STRXui killed renamable $x0, $sp, 2
+    $sp = ADDXri $sp, 16, 0
+    RET undef $lr
+...
+---
+name:            pre_index_memtag
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: pre_index_memtag
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: STRXui killed renamable $x0, $sp, 2
+    ; CHECK-NEXT: $sp = ADDXri $sp, 16, 0
+    ; CHECK-NEXT: RET undef $lr
+    $sp = frame-setup SUBXri $sp, 16, 0
+    STRXui killed renamable $x0, $sp, 2
+    $sp = ADDXri $sp, 16, 0
+    RET undef $lr
+...
+---
+name:            pre_index_memtag_not_sp
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: pre_index_memtag_not_sp
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x1 = frame-setup SUBXri $x1, 16, 0
+    ; CHECK-NEXT: early-clobber $x1 = STRXpre killed renamable $x0, $x1, 16
+    ; CHECK-NEXT: RET undef $lr, implicit $x1
+    $x1 = frame-setup SUBXri $x1, 16, 0
+    STRXui killed renamable $x0, $x1, 2
+    $x1 = ADDXri $x1, 16, 0
+    RET undef $lr, implicit $x1
+...
+---
+name:            post_index
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_index
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: early-clobber $sp = STRXpost killed renamable $x0, $sp, 16
+    ; CHECK-NEXT: RET undef $lr
+    $sp = frame-setup SUBXri $sp, 16, 0
+    STRXui killed renamable $x0, $sp, 0
+    $sp = ADDXri $sp, 16, 0
+    RET undef $lr
+...
+---
+name:            post_index_memtag
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_index_memtag
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: STRXui killed renamable $x0, $sp, 0
+    ; CHECK-NEXT: $sp = ADDXri $sp, 16, 0
+    ; CHECK-NEXT: RET undef $lr
+    $sp = frame-setup SUBXri $sp, 16, 0
+    STRXui killed renamable $x0, $sp, 0
+    $sp = ADDXri $sp, 16, 0
+    RET undef $lr
+...
+---
+name:            post_index_memtag_not_sp
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: post_index_memtag_not_sp
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x1 = frame-setup SUBXri $x1, 16, 0
+    ; CHECK-NEXT: early-clobber $x1 = STRXpost killed renamable $x0, $x1, 16
+    ; CHECK-NEXT: RET undef $lr, implicit $x1
+    $x1 = frame-setup SUBXri $x1, 16, 0
+    STRXui killed renamable $x0, $x1, 0
+    $x1 = ADDXri $x1, 16, 0
+    RET undef $lr, implicit $x1
+...
diff --git a/llvm/test/CodeGen/AArch64/misched-bundle.mir b/llvm/test/CodeGen/AArch64/misched-bundle.mir
index 9adcd2904a250..ac6112e8c60ef 100644
--- a/llvm/test/CodeGen/AArch64/misched-bundle.mir
+++ b/llvm/test/CodeGen/AArch64/misched-bundle.mir
@@ -17,13 +17,18 @@
 # CHECK-NEXT:   Single Issue       : false;
 # CHECK-NEXT: SU(1):   renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1)
 # CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 4
+# CHECK-NEXT:   # succs left       : 9
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 3
 # CHECK-NEXT:   Depth              : 0
 # CHECK-NEXT:   Height             : 7
 # CHECK-NEXT:   Successors:
 # CHECK-NEXT:     SU(7): Out  Latency=1
+# CHECK-NEXT:     SU(7): Out  Latency=1
+# CHECK-NEXT:     SU(7): Out  Latency=1
+# CHECK-NEXT:     SU(7): Out  Latency=1
+# CHECK-NEXT:     SU(7): Out  Latency=1
+# CHECK-NEXT:     SU(7): Out  Latency=1
 # CHECK-NEXT:     SU(6): Data Latency=3 Reg=$z1
 # CHECK-NEXT:     SU(9): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(8): Ord  Latency=0 Memory
@@ -78,7 +83,7 @@
 # CHECK-NEXT:   Single Issue       : false;
 # CHECK-NEXT: SU(6):   $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0(tied-def 0), killed renamable $z1, killed renamable $z2
 # CHECK-NEXT:   # preds left       : 4
-# CHECK-NEXT:   # succs left       : 2
+# CHECK-NEXT:   # succs left       : 7
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 4
 # CHECK-NEXT:   Depth              : 3
@@ -91,9 +96,14 @@
 # CHECK-NEXT:   Successors:
 # CHECK-NEXT:     SU(8): Data Latency=4 Reg=$z0
 # CHECK-NEXT:     SU(7): Anti Latency=0
+# CHECK-NEXT:     SU(7): Anti Latency=0
+# CHECK-NEXT:     SU(7): Anti Latency=0
+# CHECK-NEXT:     SU(7): Anti Latency=0
+# CHECK-NEXT:     SU(7): Anti Latency=0
+# CHECK-NEXT:     SU(7): Anti Latency=0
 # CHECK-NEXT:   Single Issue       : false;
 # CHECK-NEXT: SU(7):   BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3
-# CHECK-NEXT:   # preds left       : 5
+# CHECK-NEXT:   # preds left       : 15
 # CHECK-NEXT:   # succs left       : 1
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 1
@@ -101,10 +111,20 @@
 # CHECK-NEXT:   Height             : 4
 # CHECK-NEXT:   Predecessors:
 # CHECK-NEXT:     SU(6): Anti Latency=0
+# CHECK-NEXT:     SU(6): Anti Latency=0
+# CHECK-NEXT:     SU(6): Anti Latency=0
+# CHECK-NEXT:     SU(6): Anti Latency=0
+# CHECK-NEXT:     SU(6): Anti Latency=0
+# CHECK-NEXT:     SU(6): Anti Latency=0
 # CHECK-NEXT:     SU(5): Data Latency=3 Reg=$z5
 # CHECK-NEXT:     SU(4): Data Latency=3 Reg=$z4
 # CHECK-NEXT:     SU(3): Data Latency=3 Reg=$z3
 # CHECK-NEXT:     SU(1): Out  Latency=1
+# CHECK-NEXT:     SU(1): Out  Latency=1
+# CHECK-NEXT:     SU(1): Out  Latency=1
+# CHECK-NEXT:     SU(1): Out  Latency=1
+# CHECK-NEXT:     SU(1): Out  Latency=1
+# CHECK-NEXT:     SU(1): Out  Latency=1
 # CHECK-NEXT:   Successors:
 # CHECK-NEXT:     SU(9): Data Latency=4 Reg=$z1
 # CHECK-NEXT:   Single Issue       : false;
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
index 6cdbbb8c53d69..6fb8ba2dfc839 100644
--- a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
@@ -1583,6 +1583,18 @@ body:             |
 # CHECK-NEXT: B0 [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
 # CHECK-NEXT: B1 [0B,88r:0)[208r,224r:1) 0@0B-phi 1@208r
 # CHECK-NEXT: B2 [0B,96r:0) 0@0B-phi
+# CHECK-NEXT: B0_HI [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: H0_HI [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: S0_HI [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: B1_HI [0B,88r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: H1_HI [0B,88r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: S1_HI [0B,88r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: B2_HI [0B,96r:0) 0@0B-phi
+# CHECK-NEXT: H2_HI [0B,96r:0) 0@0B-phi
+# CHECK-NEXT: S2_HI [0B,96r:0) 0@0B-phi
+# CHECK-NEXT: D0_HI [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: D1_HI [0B,88r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: D2_HI [0B,96r:0) 0@0B-phi
 # CHECK-NEXT: %0 [48r,168r:0) 0@48r  weight:0.000000e+00
 # CHECK-NEXT: %1 [88r,120r:0) 0@88r  weight:0.000000e+00
 # CHECK-NEXT: %2 [96r,128r:0) 0@96r  weight:0.000000e+00
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
index 2cc6301881381..9c9b6e281b15d 100644
--- a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
@@ -466,6 +466,9 @@ body:             |
 # CHECK-NEXT: W0 [0B,16r:0) 0@0B-phi
 # CHECK-NEXT: W1 [0B,32r:0) 0@0B-phi
 # CHECK-NEXT: W2 [0B,48r:0) 0@0B-phi
+# CHECK-NEXT: W0_HI [0B,16r:0) 0@0B-phi
+# CHECK-NEXT: W1_HI [0B,32r:0) 0@0B-phi
+# CHECK-NEXT: W2_HI [0B,48r:0) 0@0B-phi
 # CHECK-NEXT: RegMasks:
 # CHECK-NEXT: ********** MACHINEINSTRS **********
 # CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
@@ -476,4 +479,4 @@ body:             |
 # CHECK-NEXT: 32B	  $x4 = ADDXrr $x1, $x1
 # CHECK-NEXT: 48B	  $x5 = ADDXrr $x2, $x2
 # CHECK-EMPTY: 
-# CHECK-NEXT: # End machine code for function f.
\ No newline at end of file
+# CHECK-NEXT: # End machine code for function f.
diff --git a/llvm/test/CodeGen/AArch64/neon-reverseshuffle.ll b/llvm/test/CodeGen/AArch64/neon-reverseshuffle.ll
index 1b0e8dbe0cb6e..abdfb996fa166 100644
--- a/llvm/test/CodeGen/AArch64/neon-reverseshuffle.ll
+++ b/llvm/test/CodeGen/AArch64/neon-reverseshuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon -global-isel < %s -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define <2 x i64> @v2i64(<2 x i64> %a) {
 ; CHECK-LABEL: v2i64:
@@ -22,11 +23,19 @@ entry:
 }
 
 define <4 x i32> @v4i32(<4 x i32> %a) {
-; CHECK-LABEL: v4i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.4s, v0.4s
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.4s, v0.4s
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI2_0
+; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI2_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i32> %V128
@@ -43,25 +52,42 @@ entry:
 }
 
 define <8 x i16> @v8i16(<8 x i16> %a) {
-; CHECK-LABEL: v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.8h, v0.8h
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.8h, v0.8h
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI4_0
+; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI4_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i16> %V128
 }
 
 define <8 x i16> @v8i16_2(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-LABEL: v8i16_2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adrp x8, .LCPI5_0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI5_0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v8i16_2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    adrp x8, .LCPI5_0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI5_0]
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i16_2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI5_0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI5_0]
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i16> %V128
@@ -78,25 +104,42 @@ entry:
 }
 
 define <16 x i8> @v16i8(<16 x i8> %a) {
-; CHECK-LABEL: v16i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.16b, v0.16b
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v16i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.16b, v0.16b
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v16i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI7_0
+; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI7_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i8> %V128
 }
 
 define <16 x i8> @v16i8_2(<8 x i8> %a, <8 x i8> %b) {
-; CHECK-LABEL: v16i8_2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adrp x8, .LCPI8_0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI8_0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v16i8_2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    adrp x8, .LCPI8_0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI8_0]
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v16i8_2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI8_0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI8_0]
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i8> %V128
@@ -123,11 +166,19 @@ entry:
 }
 
 define <4 x float> @v4f32(<4 x float> %a) {
-; CHECK-LABEL: v4f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.4s, v0.4s
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.4s, v0.4s
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI11_0
+; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI11_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %V128
@@ -144,11 +195,19 @@ entry:
 }
 
 define <8 x half> @v8f16(<8 x half> %a) {
-; CHECK-LABEL: v8f16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.8h, v0.8h
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v8f16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.8h, v0.8h
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI13_0
+; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %V128 = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x half> %V128
diff --git a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
index 3bd8f83d27c2d..ff29c78b5a0ce 100644
--- a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
+++ b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 --run-pass=greedy,virtregrewriter -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -aarch64-enable-subreg-liveness-tracking --run-pass=greedy,virtregrewriter -verify-machineinstrs %s -o - | FileCheck %s
 
 # We should ideally not spill around any of the SUBSWri in the loop exit blocks (if.end and if.end27).
 
@@ -219,7 +219,7 @@ body:             |
   ; CHECK-NEXT:   liveins: $w10, $w11, $x2, $x8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   STRXui renamable $x8, %stack.1, 0 :: (store (s64) into %stack.1)
-  ; CHECK-NEXT:   renamable $w9 = MOVi32imm 36, implicit-def $x9
+  ; CHECK-NEXT:   renamable $w9 = MOVi32imm 36
   ; CHECK-NEXT:   renamable $x8 = MADDXrrr killed renamable $x8, killed renamable $x9, $xzr
   ; CHECK-NEXT:   renamable $x9 = MOVaddr target-flags(aarch64-page) @g, target-flags(aarch64-pageoff, aarch64-nc) @g
   ; CHECK-NEXT:   renamable $w8 = LDRWroX killed renamable $x9, killed renamable $x8, 0, 0 :: (load (s32) from %ir.arrayidx9)
@@ -244,7 +244,7 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.5(0x50000000), %bb.8(0x30000000)
   ; CHECK-NEXT:   liveins: $w10, $w11, $x2, $x12
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $w8 = MOVi32imm 36, implicit-def $x8
+  ; CHECK-NEXT:   renamable $w8 = MOVi32imm 36
   ; CHECK-NEXT:   renamable $x8 = MADDXrrr renamable $x12, killed renamable $x8, $xzr
   ; CHECK-NEXT:   renamable $x9 = MOVaddr target-flags(aarch64-page) @g, target-flags(aarch64-pageoff, aarch64-nc) @g
   ; CHECK-NEXT:   renamable $w8 = LDRWroX killed renamable $x9, killed renamable $x8, 0, 0 :: (load (s32) from %ir.arrayidx14)
diff --git a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
index f8af5b9637017..3174d3c8c1a73 100644
--- a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
+++ b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
@@ -487,7 +487,7 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[DEF]]
-    ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3342346 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
     ; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub
     ; CHECK-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
@@ -505,7 +505,7 @@ body:             |
     %0:gpr64common = COPY $x0
     %2:gpr64all = IMPLICIT_DEF
     %3:gpr64sp = COPY %2
-    INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3342346 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
+    INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
     %4:fpr128 = MOVIv2d_ns 0
     %5:fpr64 = COPY %4.dsub
     %7:fpr128 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/preserve.ll b/llvm/test/CodeGen/AArch64/preserve.ll
index ffd479e1bd739..49fb3685bcfc1 100644
--- a/llvm/test/CodeGen/AArch64/preserve.ll
+++ b/llvm/test/CodeGen/AArch64/preserve.ll
@@ -8,20 +8,19 @@
 target triple = "aarch64-unknown-unknown"
 declare void @bar1()
 define preserve_mostcc void @baz() #0 {
-; CHECK: baz Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: baz Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $wsp_hi $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $b0_hi $b1_hi $b2_hi $b3_hi $b4_hi $b5_hi $b6_hi $b7_hi $b16_hi $b17_hi $b18_hi $b19_hi $b20_hi $b21_hi $b22_hi $b23_hi $b24_hi $b25_hi $b26_hi $b27_hi $b28_hi $b29_hi $b30_hi $b31_hi $d0_hi $d1_hi $d2_hi $d3_hi $d4_hi $d5_hi $d6_hi $d7_hi $d8_hi $d9_hi $d10_hi $d11_hi $d12_hi $d13_hi $d14_hi $d15_hi $d16_hi $d17_hi $d18_hi $d19_hi $d20_hi $d21_hi $d22_hi $d23_hi $d24_hi $d25_hi $d26_hi $d27_hi $d28_hi $d29_hi $d30_hi $d31_hi $h0_hi $h1_hi $h2_hi $h3_hi $h4_hi $h5_hi $h6_hi $h7_hi $h16_hi $h17_hi $h18_hi $h19_hi $h20_hi $h21_hi $h22_hi $h23_hi $h24_hi $h25_hi $h26_hi $h27_hi $h28_hi $h29_hi $h30_hi $h31_hi $q0_hi $q1_hi $q2_hi $q3_hi $q4_hi $q5_hi $q6_hi $q7_hi $q8_hi $q9_hi $q10_hi $q11_hi $q12_hi $q13_hi $q14_hi $q15_hi $q16_hi $q17_hi $q18_hi $q19_hi $q20_hi $q21_hi $q22_hi $q23_hi $q24_hi $q25_hi $q26_hi $q27_hi $q28_hi $q29_hi $q30_hi $q31_hi $s0_hi $s1_hi $s2_hi $s3_hi $s4_hi $s5_hi $s6_hi $s7_hi $s16_hi $s17_hi $s18_hi $s19_hi $s20_hi $s21_hi $s22_hi $s23_hi $s24_hi $s25_hi $s26_hi $s27_hi $s28_hi $s29_hi $s30_hi $s31_hi $w0_hi $w1_hi $w2_hi $w3_hi $w4_hi $w5_hi $w6_hi $w7_hi $w8_hi $w16_hi $w17_hi $w18_hi $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15 
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_allcc void @foo() #0 {
-; CHECK: foo Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: foo Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $wsp_hi $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $b0_hi $b1_hi $b2_hi $b3_hi $b4_hi $b5_hi $b6_hi $b7_hi $d0_hi $d1_hi $d2_hi $d3_hi $d4_hi $d5_hi $d6_hi $d7_hi $h0_hi $h1_hi $h2_hi $h3_hi $h4_hi $h5_hi $h6_hi $h7_hi $q0_hi $q1_hi $q2_hi $q3_hi $q4_hi $q5_hi $q6_hi $q7_hi $q8_hi $q9_hi $q10_hi $q11_hi $q12_hi $q13_hi $q14_hi $q15_hi $q16_hi $q17_hi $q18_hi $q19_hi $q20_hi $q21_hi $q22_hi $q23_hi $q24_hi $q25_hi $q26_hi $q27_hi $q28_hi $q29_hi $q30_hi $q31_hi $s0_hi $s1_hi $s2_hi $s3_hi $s4_hi $s5_hi $s6_hi $s7_hi $w0_hi $w1_hi $w2_hi $w3_hi $w4_hi $w5_hi $w6_hi $w7_hi $w8_hi $w16_hi $w17_hi $w18_hi $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15 
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_nonecc void @qux() #0 {
-;  CHECK: qux Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
-
+; CHECK: qux Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $wsp_hi $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $b0_hi $b1_hi $b2_hi $b3_hi $b4_hi $b5_hi $b6_hi $b7_hi $b16_hi $b17_hi $b18_hi $b19_hi $b20_hi $b21_hi $b22_hi $b23_hi $b24_hi $b25_hi $b26_hi $b27_hi $b28_hi $b29_hi $b30_hi $b31_hi $d0_hi $d1_hi $d2_hi $d3_hi $d4_hi $d5_hi $d6_hi $d7_hi $d8_hi $d9_hi $d10_hi $d11_hi $d12_hi $d13_hi $d14_hi $d15_hi $d16_hi $d17_hi $d18_hi $d19_hi $d20_hi $d21_hi $d22_hi $d23_hi $d24_hi $d25_hi $d26_hi $d27_hi $d28_hi $d29_hi $d30_hi $d31_hi $h0_hi $h1_hi $h2_hi $h3_hi $h4_hi $h5_hi $h6_hi $h7_hi $h16_hi $h17_hi $h18_hi $h19_hi $h20_hi $h21_hi $h22_hi $h23_hi $h24_hi $h25_hi $h26_hi $h27_hi $h28_hi $h29_hi $h30_hi $h31_hi $q0_hi $q1_hi $q2_hi $q3_hi $q4_hi $q5_hi $q6_hi $q7_hi $q8_hi $q9_hi $q10_hi $q11_hi $q12_hi $q13_hi $q14_hi $q15_hi $q16_hi $q17_hi $q18_hi $q19_hi $q20_hi $q21_hi $q22_hi $q23_hi $q24_hi $q25_hi $q26_hi $q27_hi $q28_hi $q29_hi $q30_hi $q31_hi $s0_hi $s1_hi $s2_hi $s3_hi $s4_hi $s5_hi $s6_hi $s7_hi $s16_hi $s17_hi $s18_hi $s19_hi $s20_hi $s21_hi $s22_hi $s23_hi $s24_hi $s25_hi $s26_hi $s27_hi $s28_hi $s29_hi $s30_hi $s31_hi $w0_hi $w1_hi $w2_hi $w3_hi $w4_hi $w5_hi $w6_hi $w7_hi $w8_hi $w9_hi $w10_hi $w11_hi $w12_hi $w13_hi $w14_hi $w15_hi $w16_hi $w17_hi $w18_hi $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
     call void @bar1()
     call void @bar2()
     ret void
diff --git a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
index e227f14542cc1..2a77d4dd33fe5 100644
--- a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
+++ b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_darwin.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-apple-darwin < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-apple-darwin -aarch64-enable-subreg-liveness-tracking < %s | FileCheck %s
 
 define preserve_nonecc i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, ...) nounwind noinline ssp {
 ; CHECK-LABEL: callee:
@@ -27,12 +27,11 @@ define i32 @caller() nounwind ssp {
 ; CHECK-NEXT:    sub sp, sp, #208
 ; CHECK-NEXT:    mov w8, #10 ; =0xa
 ; CHECK-NEXT:    mov w9, #9 ; =0x9
-; CHECK-NEXT:    mov w0, #1 ; =0x1
+; CHECK-NEXT:    mov w10, #8 ; =0x8
 ; CHECK-NEXT:    stp x9, x8, [sp, #24]
-; CHECK-NEXT:    mov w8, #8 ; =0x8
-; CHECK-NEXT:    mov w9, #6 ; =0x6
-; CHECK-NEXT:    str x8, [sp, #16]
 ; CHECK-NEXT:    mov w8, #7 ; =0x7
+; CHECK-NEXT:    mov w9, #6 ; =0x6
+; CHECK-NEXT:    mov w0, #1 ; =0x1
 ; CHECK-NEXT:    mov w1, #2 ; =0x2
 ; CHECK-NEXT:    mov w2, #3 ; =0x3
 ; CHECK-NEXT:    mov w3, #4 ; =0x4
@@ -47,7 +46,8 @@ define i32 @caller() nounwind ssp {
 ; CHECK-NEXT:    stp x22, x21, [sp, #160] ; 16-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #176] ; 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #192] ; 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x8, [sp]
+; CHECK-NEXT:    stp x8, x10, [sp, #8]
+; CHECK-NEXT:    str x9, [sp]
 ; CHECK-NEXT:    bl _callee
 ; CHECK-NEXT:    ldp x29, x30, [sp, #192] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #176] ; 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-call.ll b/llvm/test/CodeGen/AArch64/ptrauth-call.ll
index c46fe79f754fe..bf35cf8fecbdb 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-call.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-call.ll
@@ -188,6 +188,33 @@ define void @test_tailcall_omit_mov_x16_x16(ptr %objptr) #0 {
   ret void
 }
 
+define i32 @test_call_omit_extra_moves(ptr %objptr) #0 {
+; CHECK-LABEL: test_call_omit_extra_moves:
+; DARWIN-NEXT:   stp     x29, x30, [sp, #-16]!
+; ELF-NEXT:      str     x30, [sp, #-16]!
+; CHECK-NEXT:    ldr     x16, [x0]
+; CHECK-NEXT:    mov     x17, x0
+; CHECK-NEXT:    movk    x17, #6503, lsl #48
+; CHECK-NEXT:    autda   x16, x17
+; CHECK-NEXT:    ldr     x8, [x16]
+; CHECK-NEXT:    movk    x16, #34646, lsl #48
+; CHECK-NEXT:    blraa   x8, x16
+; CHECK-NEXT:    mov     w0, #42
+; DARWIN-NEXT:   ldp     x29, x30, [sp], #16
+; ELF-NEXT:      ldr     x30, [sp], #16
+; CHECK-NEXT:    ret
+  %vtable.signed = load ptr, ptr %objptr
+  %objptr.int = ptrtoint ptr %objptr to i64
+  %vtable.discr = tail call i64 @llvm.ptrauth.blend(i64 %objptr.int, i64 6503)
+  %vtable.signed.int = ptrtoint ptr %vtable.signed to i64
+  %vtable.int = tail call i64 @llvm.ptrauth.auth(i64 %vtable.signed.int, i32 2, i64 %vtable.discr)
+  %vtable = inttoptr i64 %vtable.int to ptr
+  %callee.signed = load ptr, ptr %vtable
+  %callee.discr = tail call i64 @llvm.ptrauth.blend(i64 %vtable.int, i64 34646)
+  %call.result = tail call i32 %callee.signed(ptr %objptr) [ "ptrauth"(i32 0, i64 %callee.discr) ]
+  ret i32 42
+}
+
 define i32 @test_call_ia_arg(ptr %arg0, i64 %arg1) #0 {
 ; DARWIN-LABEL: test_call_ia_arg:
 ; DARWIN-NEXT:    stp x29, x30, [sp, #-16]!
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-extern-weak.ll b/llvm/test/CodeGen/AArch64/ptrauth-extern-weak.ll
index 5d0a3f556c4c2..141af917fa3e2 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-extern-weak.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-extern-weak.ll
@@ -13,6 +13,11 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=1 -relocation-model=pic \
 ; RUN:   -mattr=+pauth              -o - %s | FileCheck --check-prefixes=CHECK,TRAP %s
 
+; RUN: llc -mtriple=aarch64-none-linux-gnu -code-model=tiny -mattr=+pauth -mattr=+fpac -o - %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-TINY,NOTRAP-TINY %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -code-model=tiny -mattr=+pauth              -o - %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-TINY,TRAP-TINY %s
+
 ;; Note: for FastISel, we fall back to SelectionDAG
 
 declare extern_weak dso_local i32 @var()
@@ -41,6 +46,24 @@ define ptr @foo() {
 ; TRAP-NEXT:   .Lauth_success_0:
 ; TRAP-NEXT:     mov   x0,  x16
 ; CHECK-NEXT:    ret
+
+; CHECK-TINY-LABEL: foo:
+; CHECK-TINY:         adr   x17, :got_auth:var
+; NOTRAP-TINY-NEXT:   ldr   x0,  [x17]
+; NOTRAP-TINY-NEXT:   cbz   x0,  .Lundef_weak0
+; NOTRAP-TINY-NEXT:   autia x0,  x17
+; TRAP-TINY-NEXT:     ldr   x16, [x17]
+; TRAP-TINY-NEXT:     cbz   x16, .Lundef_weak0
+; TRAP-TINY-NEXT:     autia x16, x17
+; CHECK-TINY-NEXT:  .Lundef_weak0:
+; TRAP-TINY-NEXT:     mov   x17, x16
+; TRAP-TINY-NEXT:     xpaci x17
+; TRAP-TINY-NEXT:     cmp   x16, x17
+; TRAP-TINY-NEXT:     b.eq  .Lauth_success_0
+; TRAP-TINY-NEXT:     brk   #0xc470
+; TRAP-TINY-NEXT:   .Lauth_success_0:
+; TRAP-TINY-NEXT:     mov   x0,  x16
+; CHECK-TINY-NEXT:    ret
 }
 
 @arr_var = extern_weak global [10 x i32]
@@ -68,6 +91,25 @@ define ptr @bar() {
 ; TRAP-NEXT:     mov   x8,  x16
 ; CHECK-NEXT:    add   x0,  x8, #20
 ; CHECK-NEXT:    ret
+
+; CHECK-TINY-LABEL: bar:
+; CHECK-TINY:         adr   x17, :got_auth:arr_var
+; NOTRAP-TINY-NEXT:   ldr   x8,  [x17]
+; NOTRAP-TINY-NEXT:   cbz   x8,  .Lundef_weak1
+; NOTRAP-TINY-NEXT:   autda x8,  x17
+; TRAP-TINY-NEXT:     ldr   x16, [x17]
+; TRAP-TINY-NEXT:     cbz   x16, .Lundef_weak1
+; TRAP-TINY-NEXT:     autda x16, x17
+; CHECK-TINY-NEXT:  .Lundef_weak1:
+; TRAP-TINY-NEXT:     mov   x17, x16
+; TRAP-TINY-NEXT:     xpacd x17
+; TRAP-TINY-NEXT:     cmp   x16, x17
+; TRAP-TINY-NEXT:     b.eq  .Lauth_success_1
+; TRAP-TINY-NEXT:     brk   #0xc472
+; TRAP-TINY-NEXT:   .Lauth_success_1:
+; TRAP-TINY-NEXT:     mov   x8,  x16
+; CHECK-TINY-NEXT:    add   x0,  x8, #20
+; CHECK-TINY-NEXT:    ret
 }
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-pic.ll b/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-pic.ll
new file mode 100644
index 0000000000000..9d13714bbefe3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-pic.ll
@@ -0,0 +1,182 @@
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth -mattr=+fpac -code-model=tiny \
+; RUN:   -relocation-model=pic < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth              -code-model=tiny \
+; RUN:   -relocation-model=pic < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth -mattr=+fpac -code-model=tiny \
+; RUN:   -relocation-model=pic -fast-isel < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth              -code-model=tiny \
+; RUN:   -relocation-model=pic -fast-isel < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth -mattr=+fpac -code-model=tiny \
+; RUN:   -relocation-model=pic -global-isel -global-isel-abort=1 < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -mtriple=aarch64 -mattr=+pauth              -code-model=tiny \
+; RUN:   -relocation-model=pic -global-isel -global-isel-abort=1 < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; Note: fast-isel tests here will fall back to isel
+
+@src = external local_unnamed_addr global [65536 x i8], align 1
+@dst = external global [65536 x i8], align 1
+@ptr = external local_unnamed_addr global ptr, align 8
+
+define dso_preemptable void @foo1() {
+; CHECK-LABEL: foo1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:src
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_0
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_0:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    ldrb  w8,  [x8]
+; CHECK-NEXT:    adr   x17, :got_auth:dst
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_1
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_1:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    strb  w8,  [x9]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = load i8, ptr @src, align 1
+  store i8 %0, ptr @dst, align 1
+  ret void
+}
+
+define dso_preemptable void @foo2() {
+; CHECK-LABEL: foo2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:ptr
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_2
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_2:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    adr   x17, :got_auth:dst
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_3
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_3:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    str   x9,  [x8]
+; CHECK-NEXT:    ret
+
+entry:
+  store ptr @dst, ptr @ptr, align 8
+  ret void
+}
+
+define dso_preemptable void @foo3() {
+; CHECK-LABEL: foo3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:src
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_4
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_4:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    ldrb  w8,  [x8]
+; CHECK-NEXT:    adr   x17, :got_auth:ptr
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_5
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_5:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    ldr   x9,  [x9]
+; CHECK-NEXT:    strb  w8,  [x9]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = load i8, ptr @src, align 1
+  %1 = load ptr, ptr @ptr, align 8
+  store i8 %0, ptr %1, align 1
+  ret void
+}
+
+@lsrc = internal global i8 0, align 4
+@ldst = internal global i8 0, align 4
+@lptr = internal global ptr null, align 8
+
+declare void @func(...)
+
+define dso_preemptable ptr @externfuncaddr() {
+; CHECK-LABEL: externfuncaddr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:func
+; NOTRAP-NEXT:   ldr   x0,  [x17]
+; NOTRAP-NEXT:   autia x0,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autia x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpaci x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_6
+; TRAP-NEXT:     brk   #0xc470
+; TRAP-NEXT:   .Lauth_success_6:
+; TRAP-NEXT:     mov   x0,  x16
+; CHECK-NEXT:    ret
+
+entry:
+  ret ptr @func
+}
+
+define dso_preemptable ptr @localfuncaddr() {
+; CHECK-LABEL: localfuncaddr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:externfuncaddr
+; NOTRAP-NEXT:   ldr   x0,  [x17]
+; NOTRAP-NEXT:   autia x0,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autia x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpaci x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_7
+; TRAP-NEXT:     brk   #0xc470
+; TRAP-NEXT:   .Lauth_success_7:
+; TRAP-NEXT:     mov   x0,  x16
+; CHECK-NEXT:    ret
+
+entry:
+  ret ptr @externfuncaddr
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 8, !"ptrauth-elf-got", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-static.ll b/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-static.ll
new file mode 100644
index 0000000000000..2d098b70acccc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-tiny-model-static.ll
@@ -0,0 +1,157 @@
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth -mattr=+fpac \
+; RUN:   -code-model=tiny < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth \
+; RUN:   -code-model=tiny < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth -mattr=+fpac \
+; RUN:   -code-model=tiny -fast-isel < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth \
+; RUN:   -code-model=tiny -fast-isel < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth -mattr=+fpac \
+; RUN:   -code-model=tiny -global-isel -global-isel-abort=1 < %s | FileCheck --check-prefixes=CHECK,NOTRAP %s
+; RUN: llc -verify-machineinstrs -o - -mtriple=aarch64-none-linux-gnu -mattr=+pauth \
+; RUN:   -code-model=tiny -global-isel -global-isel-abort=1 < %s | FileCheck --check-prefixes=CHECK,TRAP %s
+
+; Note fast-isel tests here will fall back to isel
+
+@src = external local_unnamed_addr global [65536 x i8], align 1
+@dst = external global [65536 x i8], align 1
+@ptr = external local_unnamed_addr global ptr, align 8
+
+define dso_local void @foo1() {
+; CHECK-LABEL: foo1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:src
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_0
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_0:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    ldrb  w8,  [x8]
+; CHECK-NEXT:    adr   x17, :got_auth:dst
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_1
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_1:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    strb  w8,  [x9]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = load i8, ptr @src, align 1
+  store i8 %0, ptr @dst, align 1
+  ret void
+}
+
+define dso_local void @foo2() {
+; CHECK-LABEL: foo2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:ptr
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_2
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_2:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    adr   x17, :got_auth:dst
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_3
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_3:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    str   x9,  [x8]
+; CHECK-NEXT:    ret
+
+entry:
+  store ptr @dst, ptr @ptr, align 8
+  ret void
+}
+
+define dso_local void @foo3() {
+; CHECK-LABEL: foo3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:src
+; NOTRAP-NEXT:   ldr   x8,  [x17]
+; NOTRAP-NEXT:   autda x8,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_4
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_4:
+; TRAP-NEXT:     mov   x8,  x16
+; CHECK-NEXT:    ldrb  w8,  [x8]
+; CHECK-NEXT:    adr   x17, :got_auth:ptr
+; NOTRAP-NEXT:   ldr   x9,  [x17]
+; NOTRAP-NEXT:   autda x9,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autda x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpacd x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_5
+; TRAP-NEXT:     brk   #0xc472
+; TRAP-NEXT:   .Lauth_success_5:
+; TRAP-NEXT:     mov   x9,  x16
+; CHECK-NEXT:    ldr   x9,  [x9]
+; CHECK-NEXT:    strb  w8,  [x9]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = load i8, ptr @src, align 1
+  %1 = load ptr, ptr @ptr, align 8
+  store i8 %0, ptr %1, align 1
+  ret void
+}
+
+declare void @func(...)
+
+define dso_local ptr @externfuncaddr() {
+; CHECK-LABEL: externfuncaddr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adr   x17, :got_auth:func
+; NOTRAP-NEXT:   ldr   x0,  [x17]
+; NOTRAP-NEXT:   autia x0,  x17
+; TRAP-NEXT:     ldr   x16, [x17]
+; TRAP-NEXT:     autia x16, x17
+; TRAP-NEXT:     mov   x17, x16
+; TRAP-NEXT:     xpaci x17
+; TRAP-NEXT:     cmp   x16, x17
+; TRAP-NEXT:     b.eq  .Lauth_success_6
+; TRAP-NEXT:     brk   #0xc470
+; TRAP-NEXT:   .Lauth_success_6:
+; TRAP-NEXT:     mov   x0,  x16
+; CHECK-NEXT:    ret
+
+entry:
+  ret ptr @func
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 8, !"ptrauth-elf-got", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/reduce-and.ll b/llvm/test/CodeGen/AArch64/reduce-and.ll
index 8ca521327c2e3..62f3e8d184d24 100644
--- a/llvm/test/CodeGen/AArch64/reduce-and.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-and.ll
@@ -20,11 +20,11 @@ define i1 @test_redand_v1i1(<1 x i1> %a) {
 define i1 @test_redand_v2i1(<2 x i1> %a) {
 ; CHECK-LABEL: test_redand_v2i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #31
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-NEXT:    uminp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v2i1:
@@ -42,11 +42,11 @@ define i1 @test_redand_v2i1(<2 x i1> %a) {
 define i1 @test_redand_v4i1(<4 x i1> %a) {
 ; CHECK-LABEL: test_redand_v4i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #15
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    uminv h0, v0.4h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v4i1:
@@ -68,11 +68,11 @@ define i1 @test_redand_v4i1(<4 x i1> %a) {
 define i1 @test_redand_v8i1(<8 x i1> %a) {
 ; CHECK-LABEL: test_redand_v8i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    shl v0.8b, v0.8b, #7
 ; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-NEXT:    uminv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v8i1:
diff --git a/llvm/test/CodeGen/AArch64/reduce-or.ll b/llvm/test/CodeGen/AArch64/reduce-or.ll
index aac31ce8b71b7..485cb7c916140 100644
--- a/llvm/test/CodeGen/AArch64/reduce-or.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-or.ll
@@ -22,9 +22,8 @@ define i1 @test_redor_v2i1(<2 x i1> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #31
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-NEXT:    umaxp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v2i1:
@@ -44,9 +43,8 @@ define i1 @test_redor_v4i1(<4 x i1> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #15
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    umaxv h0, v0.4h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v4i1:
@@ -70,9 +68,8 @@ define i1 @test_redor_v8i1(<8 x i1> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.8b, v0.8b, #7
 ; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-NEXT:    umaxv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v8i1:
diff --git a/llvm/test/CodeGen/AArch64/register-coalesce-update-subranges-remat.mir b/llvm/test/CodeGen/AArch64/register-coalesce-update-subranges-remat.mir
new file mode 100644
index 0000000000000..b61fa4be04007
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/register-coalesce-update-subranges-remat.mir
@@ -0,0 +1,38 @@
+# RUN: llc -mtriple=aarch64 -verify-machineinstrs -o - -run-pass=register-coalescer -aarch64-enable-subreg-liveness-tracking %s | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=aarch64 -verify-machineinstrs -o /dev/null -run-pass=register-coalescer -aarch64-enable-subreg-liveness-tracking -debug-only=regalloc %s 2>&1 | FileCheck %s --check-prefix=CHECK-DBG
+# REQUIRES: asserts
+
+# CHECK-DBG: ********** REGISTER COALESCER **********
+# CHECK-DBG: ********** Function: test
+# CHECK-DBG: ********** JOINING INTERVALS ***********
+# CHECK-DBG: ********** INTERVALS **********
+# CHECK-DBG: %0 [16r,32r:0) 0@16r  weight:0.000000e+00
+# CHECK-DBG: %3 [48r,112r:0) 0@48r  L0000000000000040 [48r,112r:0) 0@48r  weight:0.000000e+00
+# CHECK-DBG: %4 [80r,112e:1)[112e,112d:0) 0@112e 1@80r  L0000000000000080 [112e,112d:0) 0@112e  L0000000000000040 [80r,112e:1)[112e,112d:0) 0@112e 1@80r  weight:0.000000e+00
+# CHECK-DBG: %5 [32r,112r:1)[112r,112d:0) 0@112r 1@32r  weight:0.000000e+00
+---
+name:            test
+tracksRegLiveness: true
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 65, alignment: 16,
+      stack-id: default }
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test
+    ; CHECK: [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
+    ; CHECK-NEXT: [[ADDXri1:%[0-9]+]]:gpr64common = nuw ADDXri [[ADDXri]], 64, 0
+    ; CHECK-NEXT: undef [[MOVi32imm:%[0-9]+]].sub_32:gpr64 = MOVi32imm 64
+    ; CHECK-NEXT: undef [[MOVi32imm1:%[0-9]+]].sub_32:gpr64 = MOVi32imm 64
+    ; CHECK-NEXT: dead [[ADDXri1]]:gpr64common, dead early-clobber [[MOVi32imm1]]:gpr64 = MOPSMemorySetPseudo [[ADDXri1]], [[MOVi32imm1]], [[MOVi32imm]], implicit-def dead $nzcv
+    ; CHECK-NEXT: RET_ReallyLR
+    %1:gpr64sp = ADDXri %stack.0, 0, 0
+    %2:gpr64common = nuw ADDXri killed %1, 64, 0
+    %3:gpr32 = MOVi32imm 64
+    %4:gpr64 = SUBREG_TO_REG 0, killed %3, %subreg.sub_32
+    %6:gpr64 = COPY %4
+    %5:gpr64common = COPY killed %2
+    dead %5:gpr64common, dead early-clobber %6:gpr64 = MOPSMemorySetPseudo %5, %6, %4, implicit-def dead $nzcv
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/selectopt-cast.ll b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
index b1804a02ec1c8..4afb6af6e757c 100644
--- a/llvm/test/CodeGen/AArch64/selectopt-cast.ll
+++ b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
@@ -1,22 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -select-optimize -mtriple=arm64-apple-macosx-mcpu=apple-m4 -S %s | FileCheck %s
-; RUN: opt -passes='require<profile-summary>,function(select-optimize)' -mtriple=arm64-apple-macosx-mcpu=apple-m4 -S %s | FileCheck %s
+; RUN: opt -select-optimize -mtriple=arm64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -passes='require<profile-summary>,function(select-optimize)' -mtriple=arm64-apple-macosx -S %s | FileCheck %s
 
 define void @test_add_zext(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
 ; CHECK-LABEL: @test_add_zext(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
 ; CHECK-NEXT:    [[GEP_I:%.*]] = getelementptr inbounds ptr, ptr [[SRC:%.*]], i64 [[I]]
 ; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_I]], align 8
 ; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[SRC]], i64 [[J]]
 ; CHECK-NEXT:    [[L_J:%.*]] = load ptr, ptr [[GEP_J]], align 8
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
 ; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[CMP3]] to i64
-; CHECK-NEXT:    [[J_NEXT]] = add nsw i64 [[J]], [[DEC]]
+; CHECK-NEXT:    [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_FROZEN]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[J]], 1
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[J_NEXT]] = phi i64 [ [[TMP0]], [[SELECT_TRUE_SINK]] ], [ [[J]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -49,14 +55,67 @@ exit:
   ret void
 }
 
+define void @test_add_zext_first_op(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
+; CHECK-LABEL: @test_add_zext_first_op(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[J_START:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[J]]
+; CHECK-NEXT:    [[L_J:%.*]] = load ptr, ptr [[GEP_J]], align 8
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
+; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[CMP3]] to i64
+; CHECK-NEXT:    [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_FROZEN]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 1, [[J]]
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[J_NEXT]] = phi i64 [ [[TMP0]], [[SELECT_TRUE_SINK]] ], [ [[J]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP_DST1:%.*]] = getelementptr inbounds ptr, ptr [[DST1:%.*]], i64 [[IV1]]
+; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST1]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV1]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV1]], [[J_START]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %j = phi i64 [ %j.start, %entry ], [ %j.next, %loop ]
+  %i = phi i64 [ %i.start, %entry ], [ %j.next, %loop ]
+  %gep.i = getelementptr inbounds ptr, ptr %src, i64 %i
+  %l.i = load ptr, ptr %gep.i, align 8
+  %gep.j = getelementptr inbounds ptr, ptr %src, i64 %j
+  %l.j = load ptr, ptr %gep.j, align 8
+  %cmp3 = icmp ult ptr %l.i, %l.j
+  %dec = zext i1 %cmp3 to i64
+  %j.next = add nsw i64 %dec, %j
+  %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+  store i64 %j.next, ptr %gep.dst, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %j.start
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @test_add_zext_not(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
 ; CHECK-LABEL: @test_add_zext_not(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[J_START:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
 ; CHECK-NEXT:    [[GEP_I:%.*]] = getelementptr inbounds ptr, ptr [[SRC:%.*]], i64 [[I]]
 ; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_I]], align 8
 ; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[SRC]], i64 [[J]]
@@ -64,12 +123,18 @@ define void @test_add_zext_not(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
 ; CHECK-NEXT:    [[NOT_CMP3:%.*]] = xor i1 [[CMP3]], true
 ; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[NOT_CMP3]] to i64
-; CHECK-NEXT:    [[J_NEXT]] = add nsw i64 [[J]], [[DEC]]
+; CHECK-NEXT:    [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_FROZEN]], label [[SELECT_END]], label [[SELECT_FALSE_SINK:%.*]]
+; CHECK:       select.false.sink:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[J]], 1
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[J_NEXT]] = phi i64 [ [[J]], [[LOOP]] ], [ [[TMP0]], [[SELECT_FALSE_SINK]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[HIGH]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[J_START]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -97,6 +162,7 @@ loop:
 exit:
   ret void
 }
+
 define void @test_add_sext(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
 ; CHECK-LABEL: @test_add_sext(
 ; CHECK-NEXT:  entry:
@@ -308,16 +374,22 @@ define void @test_sub_zext(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.star
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
 ; CHECK-NEXT:    [[GEP_I:%.*]] = getelementptr inbounds ptr, ptr [[SRC:%.*]], i64 [[I]]
 ; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_I]], align 8
 ; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[SRC]], i64 [[J]]
 ; CHECK-NEXT:    [[L_J:%.*]] = load ptr, ptr [[GEP_J]], align 8
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
 ; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[CMP3]] to i64
-; CHECK-NEXT:    [[J_NEXT]] = sub nsw i64 [[J]], [[DEC]]
+; CHECK-NEXT:    [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_FROZEN]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i64 [[J]], 1
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[J_NEXT]] = phi i64 [ [[TMP0]], [[SELECT_TRUE_SINK]] ], [ [[J]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -350,14 +422,61 @@ exit:
   ret void
 }
 
+define void @test_sub_zext_first_op(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
+; CHECK-LABEL: @test_sub_zext_first_op(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[J_START:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[J]]
+; CHECK-NEXT:    [[L_J:%.*]] = load ptr, ptr [[GEP_J]], align 8
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
+; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[CMP3]] to i64
+; CHECK-NEXT:    [[J_NEXT]] = sub nsw i64 [[DEC]], [[J]]
+; CHECK-NEXT:    [[GEP_DST1:%.*]] = getelementptr inbounds ptr, ptr [[DST1:%.*]], i64 [[IV1]]
+; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST1]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV1]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV1]], [[J_START]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %j = phi i64 [ %j.start, %entry ], [ %j.next, %loop ]
+  %i = phi i64 [ %i.start, %entry ], [ %j.next, %loop ]
+  %gep.i = getelementptr inbounds ptr, ptr %src, i64 %i
+  %l.i = load ptr, ptr %gep.i, align 8
+  %gep.j = getelementptr inbounds ptr, ptr %src, i64 %j
+  %l.j = load ptr, ptr %gep.j, align 8
+  %cmp3 = icmp ult ptr %l.i, %l.j
+  %dec = zext i1 %cmp3 to i64
+  %j.next = sub nsw i64 %dec, %j
+  %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+  store i64 %j.next, ptr %gep.dst, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %j.start
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @test_sub_zext_not(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
 ; CHECK-LABEL: @test_sub_zext_not(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[HIGH:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[J_START:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
 ; CHECK-NEXT:    [[GEP_I:%.*]] = getelementptr inbounds ptr, ptr [[SRC:%.*]], i64 [[I]]
 ; CHECK-NEXT:    [[L_I:%.*]] = load ptr, ptr [[GEP_I]], align 8
 ; CHECK-NEXT:    [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[SRC]], i64 [[J]]
@@ -365,12 +484,18 @@ define void @test_sub_zext_not(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
 ; CHECK-NEXT:    [[NOT_CMP3:%.*]] = xor i1 [[CMP3]], true
 ; CHECK-NEXT:    [[DEC:%.*]] = zext i1 [[NOT_CMP3]] to i64
-; CHECK-NEXT:    [[J_NEXT]] = sub nsw i64 [[J]], [[DEC]]
+; CHECK-NEXT:    [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_FROZEN]], label [[SELECT_END]], label [[SELECT_FALSE_SINK:%.*]]
+; CHECK:       select.false.sink:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i64 [[J]], 1
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[J_NEXT]] = phi i64 [ [[J]], [[LOOP]] ], [ [[TMP0]], [[SELECT_FALSE_SINK]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[J_NEXT]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[HIGH]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[J_START]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -398,6 +523,7 @@ loop:
 exit:
   ret void
 }
+
 define void @test_sub_sext(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
 ; CHECK-LABEL: @test_sub_sext(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/CodeGen/AArch64/selectopt.ll b/llvm/test/CodeGen/AArch64/selectopt.ll
index 876d9e68d538a..54309dca3b834 100644
--- a/llvm/test/CodeGen/AArch64/selectopt.ll
+++ b/llvm/test/CodeGen/AArch64/selectopt.ll
@@ -343,15 +343,15 @@ define void @replace_or(ptr nocapture noundef %newst, ptr noundef %t, ptr nounde
 ; CHECKOO-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]]
 ; CHECKOO-NEXT:    [[ADD:%.*]] = zext i1 [[CMP84]] to i64
 ; CHECKOO-NEXT:    [[CMP84_FROZEN:%.*]] = freeze i1 [[CMP84]]
-; CHECKOO-NEXT:    [[TMP9:%.*]] = or i64 [[MUL]], 1
 ; CHECKOO-NEXT:    br i1 [[CMP84_FROZEN]], label [[SELECT_END:%.*]], label [[SELECT_FALSE:%.*]]
-; CHECKOO:       select.false:
-; CHECKOO-NEXT:    br label [[SELECT_END]]
+; CHECKOO:       select.true.sink:
+; CHECKOO-NEXT:    [[TMP9:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECKOO-NEXT:    br label [[SELECT_FALSE]]
 ; CHECKOO:       select.end:
-; CHECKOO-NEXT:    [[SPEC_SELECT:%.*]] = phi i64 [ [[TMP9]], [[IF_THEN]] ], [ [[MUL]], [[SELECT_FALSE]] ]
+; CHECKOO-NEXT:    [[SPEC_SELECT:%.*]] = phi i64 [ [[TMP9]], [[SELECT_END]] ], [ [[MUL]], [[IF_THEN]] ]
 ; CHECKOO-NEXT:    br label [[IF_END87]]
 ; CHECKOO:       if.end87:
-; CHECKOO-NEXT:    [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[SELECT_END]] ]
+; CHECKOO-NEXT:    [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[SELECT_FALSE]] ]
 ; CHECKOO-NEXT:    [[CMP16_NOT:%.*]] = icmp sgt i64 [[CMP_1]], [[MA]]
 ; CHECKOO-NEXT:    br i1 [[CMP16_NOT]], label [[WHILE_END]], label [[LAND_RHS]]
 ; CHECKOO:       while.end:
@@ -666,20 +666,15 @@ define i32 @or_samegroup(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture
 ; CHECKOO-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
 ; CHECKOO-NEXT:    [[SEL_FROZEN:%.*]] = freeze i1 [[CMP5]]
 ; CHECKOO-NEXT:    br i1 [[SEL_FROZEN]], label [[SELECT_END:%.*]], label [[SELECT_FALSE:%.*]]
-; CHECKOO:       select.false:
-; CHECKOO-NEXT:    br label [[SELECT_END]]
+; CHECKOO:       select.true.sink:
+; CHECKOO-NEXT:    [[TMP2:%.*]] = or i32 1, [[ADD]]
+; CHECKOO-NEXT:    br label [[SELECT_FALSE]]
 ; CHECKOO:       select.end:
-; CHECKOO-NEXT:    [[SEL:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[SELECT_FALSE]] ]
-; CHECKOO-NEXT:    [[CMP5_FROZEN3:%.*]] = freeze i1 [[CMP5]]
-; CHECKOO-NEXT:    [[TMP2:%.*]] = or i32 [[SEL]], 1
-; CHECKOO-NEXT:    br i1 [[CMP5_FROZEN3]], label [[SELECT_END1:%.*]], label [[SELECT_FALSE2:%.*]]
-; CHECKOO:       select.false2:
-; CHECKOO-NEXT:    br label [[SELECT_END1]]
-; CHECKOO:       select.end1:
-; CHECKOO-NEXT:    [[OR:%.*]] = phi i32 [ [[TMP2]], [[SELECT_END]] ], [ [[SEL]], [[SELECT_FALSE2]] ]
+; CHECKOO-NEXT:    [[SEL:%.*]] = phi i32 [ [[ADD]], [[SELECT_END]] ], [ 1, [[IF_THEN]] ]
+; CHECKOO-NEXT:    [[OR:%.*]] = phi i32 [ [[TMP2]], [[SELECT_END]] ], [ 1, [[IF_THEN]] ]
 ; CHECKOO-NEXT:    br label [[IF_END]]
 ; CHECKOO:       if.end:
-; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[SEL]], [[SELECT_END1]] ], [ 0, [[FOR_BODY]] ]
+; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[SEL]], [[SELECT_FALSE]] ], [ 0, [[FOR_BODY]] ]
 ; CHECKOO-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
 ; CHECKOO-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECKOO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
@@ -790,15 +785,15 @@ define i32 @or_oneusevalues(ptr nocapture noundef %x, i32 noundef %n, ptr nocapt
 ; CHECKOO-NEXT:    [[ADD1:%.*]] = add i32 [[ADD]], 1
 ; CHECKOO-NEXT:    [[ADD2:%.*]] = or i32 [[ADD1]], 1
 ; CHECKOO-NEXT:    [[CMP5_FROZEN:%.*]] = freeze i1 [[CMP5]]
-; CHECKOO-NEXT:    [[TMP2:%.*]] = or i32 [[ADD2]], 1
 ; CHECKOO-NEXT:    br i1 [[CMP5_FROZEN]], label [[SELECT_END:%.*]], label [[SELECT_FALSE:%.*]]
-; CHECKOO:       select.false:
-; CHECKOO-NEXT:    br label [[SELECT_END]]
+; CHECKOO:       select.true.sink:
+; CHECKOO-NEXT:    [[TMP2:%.*]] = or i32 1, [[ADD2]]
+; CHECKOO-NEXT:    br label [[SELECT_FALSE]]
 ; CHECKOO:       select.end:
-; CHECKOO-NEXT:    [[OR:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[ADD2]], [[SELECT_FALSE]] ]
+; CHECKOO-NEXT:    [[OR:%.*]] = phi i32 [ [[TMP2]], [[SELECT_END]] ], [ [[ADD2]], [[IF_THEN]] ]
 ; CHECKOO-NEXT:    br label [[IF_END]]
 ; CHECKOO:       if.end:
-; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[SELECT_END]] ], [ 0, [[FOR_BODY]] ]
+; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[SELECT_FALSE]] ], [ 0, [[FOR_BODY]] ]
 ; CHECKOO-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
 ; CHECKOO-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECKOO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
diff --git a/llvm/test/CodeGen/AArch64/sink-mul-exts.ll b/llvm/test/CodeGen/AArch64/sink-mul-exts.ll
new file mode 100644
index 0000000000000..d52ac7847f814
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sink-mul-exts.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -o - %s | FileCheck %s
+
+define <8 x i16> @mul_splat_sext_v8i16(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_splat_sext_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    dup v1.8b, v1.b[3]
+; CHECK-NEXT:  .LBB0_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #4
+; CHECK-NEXT:    cmp w8, #4
+; CHECK-NEXT:    smlal v0.8h, v2.8b, v1.8b
+; CHECK-NEXT:    b.eq .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <8 x i8>, ptr %x
+  %x.ext = sext <8 x i8> %x.val to <8 x i16>
+  %a = shufflevector <8 x i16> %x.ext, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <8 x i16> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i8, ptr %y, i32 %y.idx
+  %y.val = load <8 x i8>, ptr %y.ptr
+  %y.ext = sext <8 x i8> %y.val to <8 x i16>
+  %b = mul <8 x i16> %y.ext, %a
+  %c = add <8 x i16> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <8 x i16> %c
+}
+
+define <4 x i32> @mul_splat_sext_v4i32(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_splat_sext_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB1_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #8
+; CHECK-NEXT:    cmp w8, #8
+; CHECK-NEXT:    smlal v0.4s, v2.4h, v1.h[3]
+; CHECK-NEXT:    b.eq .LBB1_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <4 x i16>, ptr %x
+  %x.ext = sext <4 x i16> %x.val to <4 x i32>
+  %a = shufflevector <4 x i32> %x.ext, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <4 x i32> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i16, ptr %y, i32 %y.idx
+  %y.val = load <4 x i16>, ptr %y.ptr
+  %y.ext = sext <4 x i16> %y.val to <4 x i32>
+  %b = mul <4 x i32> %y.ext, %a
+  %c = add <4 x i32> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <4 x i32> %c
+}
+
+define <2 x i64> @mul_splat_sext_v2i64(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_splat_sext_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB2_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #16
+; CHECK-NEXT:    cmp w8, #16
+; CHECK-NEXT:    smlal v0.2d, v2.2s, v1.s[1]
+; CHECK-NEXT:    b.eq .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <2 x i32>, ptr %x
+  %x.ext = sext <2 x i32> %x.val to <2 x i64>
+  %a = shufflevector <2 x i64> %x.ext, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <2 x i64> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i32, ptr %y, i32 %y.idx
+  %y.val = load <2 x i32>, ptr %y.ptr
+  %y.ext = sext <2 x i32> %y.val to <2 x i64>
+  %b = mul <2 x i64> %y.ext, %a
+  %c = add <2 x i64> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <2 x i64> %c
+}
+
+define <8 x i16> @mul_sext_splat_v8i16(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_sext_splat_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    dup v1.8b, v1.b[3]
+; CHECK-NEXT:  .LBB3_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #4
+; CHECK-NEXT:    cmp w8, #4
+; CHECK-NEXT:    smlal v0.8h, v2.8b, v1.8b
+; CHECK-NEXT:    b.eq .LBB3_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <8 x i8>, ptr %x
+  %x.spt = shufflevector <8 x i8> %x.val, <8 x i8> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %a = sext <8 x i8> %x.spt to <8 x i16>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <8 x i16> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i8, ptr %y, i32 %y.idx
+  %y.val = load <8 x i8>, ptr %y.ptr
+  %y.ext = sext <8 x i8> %y.val to <8 x i16>
+  %b = mul <8 x i16> %y.ext, %a
+  %c = add <8 x i16> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <8 x i16> %c
+}
+
+define <4 x i32> @mul_sext_splat_v4i32(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_sext_splat_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB4_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #8
+; CHECK-NEXT:    cmp w8, #8
+; CHECK-NEXT:    smlal v0.4s, v2.4h, v1.h[3]
+; CHECK-NEXT:    b.eq .LBB4_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <4 x i16>, ptr %x
+  %x.spt = shufflevector <4 x i16> %x.val, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %a = sext <4 x i16> %x.spt to <4 x i32>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <4 x i32> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i16, ptr %y, i32 %y.idx
+  %y.val = load <4 x i16>, ptr %y.ptr
+  %y.ext = sext <4 x i16> %y.val to <4 x i32>
+  %b = mul <4 x i32> %y.ext, %a
+  %c = add <4 x i32> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <4 x i32> %c
+}
+
+define <2 x i64> @mul_sext_splat_v2i64(ptr %x, ptr %y) {
+; CHECK-LABEL: mul_sext_splat_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB5_1: // %l1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x1, x8]
+; CHECK-NEXT:    add x8, x8, #16
+; CHECK-NEXT:    cmp w8, #16
+; CHECK-NEXT:    smlal v0.2d, v2.2s, v1.s[1]
+; CHECK-NEXT:    b.eq .LBB5_1
+; CHECK-NEXT:  // %bb.2: // %l2
+; CHECK-NEXT:    ret
+entry:
+  %x.val = load <2 x i32>, ptr %x
+  %x.spt = shufflevector <2 x i32> %x.val, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %a = sext <2 x i32> %x.spt to <2 x i64>
+  br label %l1
+
+l1:
+  %p = phi i32 [ 0, %entry ], [ %pa, %l1 ]
+  %q = phi <2 x i64> [ zeroinitializer, %entry ], [ %c, %l1 ]
+  %y.idx = mul nuw nsw i32 %p, 4
+  %y.ptr = getelementptr i32, ptr %y, i32 %y.idx
+  %y.val = load <2 x i32>, ptr %y.ptr
+  %y.ext = sext <2 x i32> %y.val to <2 x i64>
+  %b = mul <2 x i64> %y.ext, %a
+  %c = add <2 x i64> %q, %b
+  %pa = add i32 %p, 1
+  %c1 = icmp eq i32 %p, 0
+  br i1 %c1, label %l1, label %l2
+
+l2:
+  ret <2 x i64> %c
+}
diff --git a/llvm/test/CodeGen/AArch64/sinksplat.ll b/llvm/test/CodeGen/AArch64/sinksplat.ll
index cceeb9f3e830a..b77f7b97f797c 100644
--- a/llvm/test/CodeGen/AArch64/sinksplat.ll
+++ b/llvm/test/CodeGen/AArch64/sinksplat.ll
@@ -204,10 +204,9 @@ define <4 x i32> @mlal(<4 x i32> %x, ptr %y) {
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    ldr q2, [x0]
 ; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    dup v1.4s, v1.s[3]
 ; CHECK-NEXT:  .LBB6_1: // %l1
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    mla v0.4s, v2.4s, v1.4s
+; CHECK-NEXT:    mla v0.4s, v2.4s, v1.s[3]
 ; CHECK-NEXT:    subs w8, w8, #1
 ; CHECK-NEXT:    b.eq .LBB6_1
 ; CHECK-NEXT:  // %bb.2: // %l2
diff --git a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
new file mode 100644
index 0000000000000..3d3fcb05f6cf0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+fp8 -verify-machineinstrs -force-streaming < %s | FileCheck %s
+
+; F1CVT / F2CVT
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @f1cvt(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: f1cvt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvt { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8f16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @f2cvt(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: f2cvt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvt { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8f16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+; BF1CVT / BF2CVT
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @bf1cvt(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: bf1cvt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvt { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt1.x2.nxv8bf16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @bf2cvt(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: bf2cvt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvt { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvt2.x2.nxv8bf16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
+; F1CVTL / F2CVTL
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @f1cvtl(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: f1cvtl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f1cvtl { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8f16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @f2cvtl(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: f2cvtl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    f2cvtl { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxvbf16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+; BF1CVTL / BF2CVTL
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @bf1cvtl(<vscale x 16 x i8> %zm) {
+; CHECK-LABEL: bf1cvtl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf1cvtl { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl1.x2.nxv8bf16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @bf2cvtl( <vscale x 16 x i8> %zm) {
+; CHECK-LABEL: bf2cvtl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bf2cvtl { z0.h, z1.h }, z0.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fp8.cvtl2.x2.nxv8bf16(<vscale x 16 x i8> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-fmopa.ll b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-fmopa.ll
new file mode 100644
index 0000000000000..6e88cdf4e7fec
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-fmopa.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme-f8f16,+sme-f8f32 -force-streaming < %s | FileCheck %s
+
+define void @test_fmopa_16(<vscale x 16 x i1> %pn, <vscale x 16 x i1> %pm, <vscale x 16 x i8> %vn, <vscale x 16 x i8> %vm) {
+; CHECK-LABEL: test_fmopa_16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmopa za1.h, p0/m, p1/m, z0.b, z1.b
+; CHECK-NEXT:    ret
+    call void @llvm.aarch64.sme.fp8.fmopa.za16(i32 1, <vscale x 16 x i1> %pn, <vscale x 16 x i1> %pm,
+                                               <vscale x 16 x i8> %vn, <vscale x 16 x i8> %vm)
+    ret void
+}
+
+define void @test_fmopa_32(<vscale x 16 x i1> %pn, <vscale x 16 x i1> %pm, <vscale x 16 x i8> %vn, <vscale x 16 x i8> %vm) #0 {
+; CHECK-LABEL: test_fmopa_32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmopa za3.s, p0/m, p1/m, z0.b, z1.b
+; CHECK-NEXT:    ret
+    call void @llvm.aarch64.sme.fp8.fmopa.za32(i32 3, <vscale x 16 x i1> %pn, <vscale x 16 x i1> %pm,
+                                               <vscale x 16 x i8> %vn, <vscale x 16 x i8> %vm)
+    ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll b/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll
new file mode 100644
index 0000000000000..2a034fe5e5290
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-hazard-windows.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-windows-pc-msvc -aarch64-stack-hazard-size=0 | FileCheck %s --check-prefixes=CHECK0
+; RUN: llc < %s -mtriple=aarch64-windows-pc-msvc -aarch64-stack-hazard-size=64 | FileCheck %s --check-prefixes=CHECK64
+; RUN: llc < %s -mtriple=aarch64-windows-pc-msvc -aarch64-stack-hazard-size=1024 | FileCheck %s --check-prefixes=CHECK1024
+
+define i32 @fpr_csr_stackobj(double %x) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
+; CHECK0-LABEL: fpr_csr_stackobj:
+; CHECK0:       .seh_proc fpr_csr_stackobj
+; CHECK0-NEXT:  // %bb.0: // %entry
+; CHECK0-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
+; CHECK0-NEXT:    .seh_save_reg_x x23, 48
+; CHECK0-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
+; CHECK0-NEXT:    .seh_save_fplr 8
+; CHECK0-NEXT:    stp d9, d10, [sp, #24] // 16-byte Folded Spill
+; CHECK0-NEXT:    .seh_save_fregp d9, 24
+; CHECK0-NEXT:    add x29, sp, #8
+; CHECK0-NEXT:    .seh_add_fp 8
+; CHECK0-NEXT:    .seh_endprologue
+; CHECK0-NEXT:    mov w0, wzr
+; CHECK0-NEXT:    //APP
+; CHECK0-NEXT:    //NO_APP
+; CHECK0-NEXT:    str d0, [x29, #32]
+; CHECK0-NEXT:    .seh_startepilogue
+; CHECK0-NEXT:    ldp d9, d10, [sp, #24] // 16-byte Folded Reload
+; CHECK0-NEXT:    .seh_save_fregp d9, 24
+; CHECK0-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
+; CHECK0-NEXT:    .seh_save_fplr 8
+; CHECK0-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
+; CHECK0-NEXT:    .seh_save_reg_x x23, 48
+; CHECK0-NEXT:    .seh_endepilogue
+; CHECK0-NEXT:    ret
+; CHECK0-NEXT:    .seh_endfunclet
+; CHECK0-NEXT:    .seh_endproc
+;
+; CHECK64-LABEL: fpr_csr_stackobj:
+; CHECK64:       .seh_proc fpr_csr_stackobj
+; CHECK64-NEXT:  // %bb.0: // %entry
+; CHECK64-NEXT:    sub sp, sp, #192
+; CHECK64-NEXT:    .seh_stackalloc 192
+; CHECK64-NEXT:    str x23, [sp, #80] // 8-byte Folded Spill
+; CHECK64-NEXT:    .seh_save_reg x23, 80
+; CHECK64-NEXT:    str x29, [sp, #88] // 8-byte Folded Spill
+; CHECK64-NEXT:    .seh_save_reg x29, 88
+; CHECK64-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
+; CHECK64-NEXT:    .seh_save_reg x30, 96
+; CHECK64-NEXT:    str d9, [sp, #168] // 8-byte Folded Spill
+; CHECK64-NEXT:    .seh_save_freg d9, 168
+; CHECK64-NEXT:    str d10, [sp, #176] // 8-byte Folded Spill
+; CHECK64-NEXT:    .seh_save_freg d10, 176
+; CHECK64-NEXT:    add x29, sp, #88
+; CHECK64-NEXT:    .seh_add_fp 88
+; CHECK64-NEXT:    .seh_endprologue
+; CHECK64-NEXT:    mov w0, wzr
+; CHECK64-NEXT:    //APP
+; CHECK64-NEXT:    //NO_APP
+; CHECK64-NEXT:    stur d0, [x29, #-16]
+; CHECK64-NEXT:    .seh_startepilogue
+; CHECK64-NEXT:    ldr d10, [sp, #176] // 8-byte Folded Reload
+; CHECK64-NEXT:    .seh_save_freg d10, 176
+; CHECK64-NEXT:    ldr d9, [sp, #168] // 8-byte Folded Reload
+; CHECK64-NEXT:    .seh_save_freg d9, 168
+; CHECK64-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK64-NEXT:    .seh_save_reg x30, 96
+; CHECK64-NEXT:    ldr x29, [sp, #88] // 8-byte Folded Reload
+; CHECK64-NEXT:    .seh_save_reg x29, 88
+; CHECK64-NEXT:    ldr x23, [sp, #80] // 8-byte Folded Reload
+; CHECK64-NEXT:    .seh_save_reg x23, 80
+; CHECK64-NEXT:    add sp, sp, #192
+; CHECK64-NEXT:    .seh_stackalloc 192
+; CHECK64-NEXT:    .seh_endepilogue
+; CHECK64-NEXT:    ret
+; CHECK64-NEXT:    .seh_endfunclet
+; CHECK64-NEXT:    .seh_endproc
+;
+; CHECK1024-LABEL: fpr_csr_stackobj:
+; CHECK1024:       .seh_proc fpr_csr_stackobj
+; CHECK1024-NEXT:  // %bb.0: // %entry
+; CHECK1024-NEXT:    sub sp, sp, #1072
+; CHECK1024-NEXT:    str x23, [sp] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .seh_save_reg x29, 8
+; CHECK1024-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .seh_save_reg x30, 16
+; CHECK1024-NEXT:    str d9, [sp, #1048] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .seh_save_freg d9, 1048
+; CHECK1024-NEXT:    str d10, [sp, #1056] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .seh_save_freg d10, 1056
+; CHECK1024-NEXT:    add x29, sp, #8
+; CHECK1024-NEXT:    .seh_add_fp 8
+; CHECK1024-NEXT:    .seh_endprologue
+; CHECK1024-NEXT:    sub sp, sp, #1040
+; CHECK1024-NEXT:    mov w0, wzr
+; CHECK1024-NEXT:    //APP
+; CHECK1024-NEXT:    //NO_APP
+; CHECK1024-NEXT:    stur d0, [x29, #-16]
+; CHECK1024-NEXT:    .seh_startepilogue
+; CHECK1024-NEXT:    add sp, sp, #1040
+; CHECK1024-NEXT:    .seh_stackalloc 1040
+; CHECK1024-NEXT:    ldr d10, [sp, #1056] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .seh_save_freg d10, 1056
+; CHECK1024-NEXT:    ldr d9, [sp, #1048] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .seh_save_freg d9, 1048
+; CHECK1024-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .seh_save_reg x30, 16
+; CHECK1024-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .seh_save_reg x29, 8
+; CHECK1024-NEXT:    ldr x23, [sp] // 8-byte Folded Reload
+; CHECK1024-NEXT:    add sp, sp, #1072
+; CHECK1024-NEXT:    .seh_endepilogue
+; CHECK1024-NEXT:    ret
+; CHECK1024-NEXT:    .seh_endfunclet
+; CHECK1024-NEXT:    .seh_endproc
+entry:
+  %a = alloca double
+  tail call void asm sideeffect "", "~{x23},~{d9},~{d10}"()
+  store double %x, ptr %a
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard.ll b/llvm/test/CodeGen/AArch64/stack-hazard.ll
index 50a2e41f45756..a4c2b30566a95 100644
--- a/llvm/test/CodeGen/AArch64/stack-hazard.ll
+++ b/llvm/test/CodeGen/AArch64/stack-hazard.ll
@@ -337,19 +337,18 @@ define i32 @csr_d8_allocd_framepointer(double %d) "aarch64_pstate_sm_compatible"
 ; CHECK64-LABEL: csr_d8_allocd_framepointer:
 ; CHECK64:       // %bb.0: // %entry
 ; CHECK64-NEXT:    sub sp, sp, #176
-; CHECK64-NEXT:    str d8, [sp, #80] // 8-byte Folded Spill
+; CHECK64-NEXT:    stp d0, d8, [sp, #72] // 8-byte Folded Spill
 ; CHECK64-NEXT:    stp x29, x30, [sp, #152] // 16-byte Folded Spill
-; CHECK64-NEXT:    add x29, sp, #80
-; CHECK64-NEXT:    .cfi_def_cfa w29, 96
+; CHECK64-NEXT:    add x29, sp, #152
+; CHECK64-NEXT:    .cfi_def_cfa w29, 24
 ; CHECK64-NEXT:    .cfi_offset w30, -16
 ; CHECK64-NEXT:    .cfi_offset w29, -24
 ; CHECK64-NEXT:    .cfi_offset b8, -96
 ; CHECK64-NEXT:    //APP
 ; CHECK64-NEXT:    //NO_APP
-; CHECK64-NEXT:    stur d0, [x29, #-8]
 ; CHECK64-NEXT:    ldr x29, [sp, #152] // 8-byte Folded Reload
-; CHECK64-NEXT:    ldr d8, [sp, #80] // 8-byte Folded Reload
 ; CHECK64-NEXT:    mov w0, wzr
+; CHECK64-NEXT:    ldr d8, [sp, #80] // 8-byte Folded Reload
 ; CHECK64-NEXT:    add sp, sp, #176
 ; CHECK64-NEXT:    ret
 ;
@@ -358,17 +357,17 @@ define i32 @csr_d8_allocd_framepointer(double %d) "aarch64_pstate_sm_compatible"
 ; CHECK1024-NEXT:    sub sp, sp, #1056
 ; CHECK1024-NEXT:    str d8, [sp] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x29, [sp, #1032] // 8-byte Folded Spill
-; CHECK1024-NEXT:    mov x29, sp
+; CHECK1024-NEXT:    add x29, sp, #1032
 ; CHECK1024-NEXT:    str x30, [sp, #1040] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    sub sp, sp, #1040
-; CHECK1024-NEXT:    .cfi_def_cfa w29, 1056
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 24
 ; CHECK1024-NEXT:    .cfi_offset w30, -16
 ; CHECK1024-NEXT:    .cfi_offset w29, -24
 ; CHECK1024-NEXT:    .cfi_offset b8, -1056
 ; CHECK1024-NEXT:    mov w0, wzr
 ; CHECK1024-NEXT:    //APP
 ; CHECK1024-NEXT:    //NO_APP
-; CHECK1024-NEXT:    stur d0, [x29, #-8]
+; CHECK1024-NEXT:    str d0, [sp, #1032]
 ; CHECK1024-NEXT:    add sp, sp, #1040
 ; CHECK1024-NEXT:    ldr x30, [sp, #1040] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x29, [sp, #1032] // 8-byte Folded Reload
@@ -2893,8 +2892,8 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    stp x29, x30, [sp, #128] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x9, x20, [sp, #144] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str x19, [sp, #160] // 8-byte Folded Spill
-; CHECK64-NEXT:    mov x29, sp
-; CHECK64-NEXT:    .cfi_def_cfa w29, 176
+; CHECK64-NEXT:    add x29, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa w29, 48
 ; CHECK64-NEXT:    .cfi_offset w19, -16
 ; CHECK64-NEXT:    .cfi_offset w20, -24
 ; CHECK64-NEXT:    .cfi_offset w30, -40
@@ -2913,11 +2912,11 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    mov w20, w0
 ; CHECK64-NEXT:    msub x9, x8, x8, x9
 ; CHECK64-NEXT:    mov sp, x9
-; CHECK64-NEXT:    stur x9, [x29, #-80]
-; CHECK64-NEXT:    sub x9, x29, #80
-; CHECK64-NEXT:    sturh wzr, [x29, #-70]
-; CHECK64-NEXT:    stur wzr, [x29, #-68]
-; CHECK64-NEXT:    sturh w8, [x29, #-72]
+; CHECK64-NEXT:    stur x9, [x29, #-208]
+; CHECK64-NEXT:    sub x9, x29, #208
+; CHECK64-NEXT:    sturh wzr, [x29, #-198]
+; CHECK64-NEXT:    stur wzr, [x29, #-196]
+; CHECK64-NEXT:    sturh w8, [x29, #-200]
 ; CHECK64-NEXT:    msr TPIDR2_EL0, x9
 ; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    smstop sm
@@ -2926,14 +2925,14 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    smstart za
 ; CHECK64-NEXT:    mrs x8, TPIDR2_EL0
-; CHECK64-NEXT:    sub x0, x29, #80
+; CHECK64-NEXT:    sub x0, x29, #208
 ; CHECK64-NEXT:    cbnz x8, .LBB33_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    bl __arm_tpidr2_restore
 ; CHECK64-NEXT:  .LBB33_2: // %entry
 ; CHECK64-NEXT:    mov w0, w20
 ; CHECK64-NEXT:    msr TPIDR2_EL0, xzr
-; CHECK64-NEXT:    mov sp, x29
+; CHECK64-NEXT:    sub sp, x29, #128
 ; CHECK64-NEXT:    .cfi_def_cfa wsp, 176
 ; CHECK64-NEXT:    ldp x20, x19, [sp, #152] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr d14, [sp, #8] // 8-byte Folded Reload
@@ -2972,8 +2971,8 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    str x28, [sp, #1112] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x20, [sp, #1120] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x19, [sp, #1128] // 8-byte Folded Spill
-; CHECK1024-NEXT:    mov x29, sp
-; CHECK1024-NEXT:    .cfi_def_cfa w29, 1136
+; CHECK1024-NEXT:    add x29, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 48
 ; CHECK1024-NEXT:    .cfi_offset w19, -8
 ; CHECK1024-NEXT:    .cfi_offset w20, -16
 ; CHECK1024-NEXT:    .cfi_offset w28, -24
@@ -2993,14 +2992,14 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    mov w20, w0
 ; CHECK1024-NEXT:    msub x9, x8, x8, x9
 ; CHECK1024-NEXT:    mov sp, x9
-; CHECK1024-NEXT:    sub x10, x29, #784
+; CHECK1024-NEXT:    sub x10, x29, #1872
 ; CHECK1024-NEXT:    stur x9, [x10, #-256]
-; CHECK1024-NEXT:    sub x9, x29, #774
-; CHECK1024-NEXT:    sub x10, x29, #772
+; CHECK1024-NEXT:    sub x9, x29, #1862
+; CHECK1024-NEXT:    sub x10, x29, #1860
 ; CHECK1024-NEXT:    sturh wzr, [x9, #-256]
-; CHECK1024-NEXT:    sub x9, x29, #1040
+; CHECK1024-NEXT:    sub x9, x29, #2128
 ; CHECK1024-NEXT:    stur wzr, [x10, #-256]
-; CHECK1024-NEXT:    sub x10, x29, #776
+; CHECK1024-NEXT:    sub x10, x29, #1864
 ; CHECK1024-NEXT:    sturh w8, [x10, #-256]
 ; CHECK1024-NEXT:    msr TPIDR2_EL0, x9
 ; CHECK1024-NEXT:    .cfi_offset vg, -32
@@ -3010,14 +3009,14 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    smstart za
 ; CHECK1024-NEXT:    mrs x8, TPIDR2_EL0
-; CHECK1024-NEXT:    sub x0, x29, #1040
+; CHECK1024-NEXT:    sub x0, x29, #2128
 ; CHECK1024-NEXT:    cbnz x8, .LBB33_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    bl __arm_tpidr2_restore
 ; CHECK1024-NEXT:  .LBB33_2: // %entry
 ; CHECK1024-NEXT:    mov w0, w20
 ; CHECK1024-NEXT:    msr TPIDR2_EL0, xzr
-; CHECK1024-NEXT:    mov sp, x29
+; CHECK1024-NEXT:    sub sp, x29, #1088
 ; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1136
 ; CHECK1024-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x19, [sp, #1128] // 8-byte Folded Reload
@@ -3049,3 +3048,109 @@ entry:
   ret i32 %x
 }
 declare void @other()
+
+declare void @bar(ptr noundef) "aarch64_pstate_sm_compatible"
+
+define i32 @sve_stack_object_and_vla(double %d, i64 %sz) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
+; CHECK0-LABEL: sve_stack_object_and_vla:
+; CHECK0:       // %bb.0: // %entry
+; CHECK0-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x28, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    addvl sp, sp, #-1
+; CHECK0-NEXT:    mov x19, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 32
+; CHECK0-NEXT:    .cfi_offset w19, -8
+; CHECK0-NEXT:    .cfi_offset w28, -16
+; CHECK0-NEXT:    .cfi_offset w30, -24
+; CHECK0-NEXT:    .cfi_offset w29, -32
+; CHECK0-NEXT:    lsl x9, x0, #2
+; CHECK0-NEXT:    mov x8, sp
+; CHECK0-NEXT:    add x9, x9, #15
+; CHECK0-NEXT:    and x9, x9, #0xfffffffffffffff0
+; CHECK0-NEXT:    sub x0, x8, x9
+; CHECK0-NEXT:    mov sp, x0
+; CHECK0-NEXT:    mov z0.s, #0 // =0x0
+; CHECK0-NEXT:    ptrue p0.s
+; CHECK0-NEXT:    st1w { z0.s }, p0, [x29, #-1, mul vl]
+; CHECK0-NEXT:    bl bar
+; CHECK0-NEXT:    mov w0, wzr
+; CHECK0-NEXT:    mov sp, x29
+; CHECK0-NEXT:    ldp x28, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK0-NEXT:    ret
+;
+; CHECK64-LABEL: sve_stack_object_and_vla:
+; CHECK64:       // %bb.0: // %entry
+; CHECK64-NEXT:    sub sp, sp, #96
+; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    stp x28, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK64-NEXT:    sub sp, sp, #64
+; CHECK64-NEXT:    addvl sp, sp, #-1
+; CHECK64-NEXT:    mov x19, sp
+; CHECK64-NEXT:    .cfi_def_cfa w29, 32
+; CHECK64-NEXT:    .cfi_offset w19, -8
+; CHECK64-NEXT:    .cfi_offset w28, -16
+; CHECK64-NEXT:    .cfi_offset w30, -24
+; CHECK64-NEXT:    .cfi_offset w29, -32
+; CHECK64-NEXT:    lsl x9, x0, #2
+; CHECK64-NEXT:    mov x8, sp
+; CHECK64-NEXT:    add x9, x9, #15
+; CHECK64-NEXT:    and x9, x9, #0xfffffffffffffff0
+; CHECK64-NEXT:    sub x0, x8, x9
+; CHECK64-NEXT:    mov sp, x0
+; CHECK64-NEXT:    mov z0.s, #0 // =0x0
+; CHECK64-NEXT:    ptrue p0.s
+; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    st1w { z0.s }, p0, [x8, #-1, mul vl]
+; CHECK64-NEXT:    bl bar
+; CHECK64-NEXT:    mov w0, wzr
+; CHECK64-NEXT:    sub sp, x29, #64
+; CHECK64-NEXT:    ldp x28, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK64-NEXT:    add sp, sp, #96
+; CHECK64-NEXT:    ret
+;
+; CHECK1024-LABEL: sve_stack_object_and_vla:
+; CHECK1024:       // %bb.0: // %entry
+; CHECK1024-NEXT:    sub sp, sp, #1056
+; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x28, [sp, #1040] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1048] // 8-byte Folded Spill
+; CHECK1024-NEXT:    sub sp, sp, #1024
+; CHECK1024-NEXT:    addvl sp, sp, #-1
+; CHECK1024-NEXT:    mov x19, sp
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 32
+; CHECK1024-NEXT:    .cfi_offset w19, -8
+; CHECK1024-NEXT:    .cfi_offset w28, -16
+; CHECK1024-NEXT:    .cfi_offset w30, -24
+; CHECK1024-NEXT:    .cfi_offset w29, -32
+; CHECK1024-NEXT:    lsl x9, x0, #2
+; CHECK1024-NEXT:    mov x8, sp
+; CHECK1024-NEXT:    add x9, x9, #15
+; CHECK1024-NEXT:    and x9, x9, #0xfffffffffffffff0
+; CHECK1024-NEXT:    sub x0, x8, x9
+; CHECK1024-NEXT:    mov sp, x0
+; CHECK1024-NEXT:    mov z0.s, #0 // =0x0
+; CHECK1024-NEXT:    ptrue p0.s
+; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    st1w { z0.s }, p0, [x8, #-1, mul vl]
+; CHECK1024-NEXT:    bl bar
+; CHECK1024-NEXT:    mov w0, wzr
+; CHECK1024-NEXT:    sub sp, x29, #1024
+; CHECK1024-NEXT:    ldr x19, [sp, #1048] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x28, [sp, #1040] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
+; CHECK1024-NEXT:    add sp, sp, #1056
+; CHECK1024-NEXT:    ret
+entry:
+  %a = alloca <vscale x 4 x i32>
+  %b = alloca i32, i64 %sz, align 4
+  store <vscale x 4 x i32> zeroinitializer, ptr %a
+  call void @bar(ptr noundef nonnull %b)
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-epilogue-fold.mir b/llvm/test/CodeGen/AArch64/stack-tagging-epilogue-fold.mir
new file mode 100644
index 0000000000000..bcd716c5c8d6c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-epilogue-fold.mir
@@ -0,0 +1,72 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64 -mattr=+mte -run-pass=prologepilog %s -o - | FileCheck %s
+
+--- |
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+  target triple = "aarch64-arm-none-eabi"
+
+  define void @F76(i32 %0, ...) #0 {
+    ret void
+  }
+
+...
+---
+name:            F76
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 4, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4080, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -4080, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -4112, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 2, name: '', type: default, offset: 0, size: 3888, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -8000, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 3, name: '', type: default, offset: 0, size: 56, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -8056, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 4, name: '', type: default, offset: 0, size: 128, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -8192, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+body:             |
+  bb.0 (%ir-block.1):
+    liveins: $q0, $q1, $q2, $q3, $q4, $q5, $q6, $q7, $x1, $x2, $x3, $x4, $x5, $x6, $x7
+
+    ; CHECK-LABEL: name: F76
+    ; CHECK: liveins: $q0, $q1, $q2, $q3, $q4, $q5, $q6, $q7, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $fp
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $fp, $sp, -16 :: (store (s64) into %stack.5)
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 2, 12
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8208
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
+    ; CHECK-NEXT: renamable $x8 = IRGstack $sp, $xzr
+    ; CHECK-NEXT: renamable $x9 = TAGPstack $x8, 2, renamable $x8, 1
+    ; CHECK-NEXT: renamable $x10 = COPY renamable $x9
+    ; CHECK-NEXT: dead early-clobber renamable $x11, dead early-clobber renamable $x10 = STGloop_wback 4080, killed renamable $x10, implicit-def dead $nzcv
+    ; CHECK-NEXT: ST2Gi renamable $x8, renamable $x8, 0
+    ; CHECK-NEXT: $x9 = ADDXri $sp, 0, 0
+    ; CHECK-NEXT: dead early-clobber $x8, dead early-clobber $x9 = STGloop_wback 4112, $x9, implicit-def $nzcv
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 2, 12
+    ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.5)
+    ; CHECK-NEXT: RET_ReallyLR
+    /* Prologue */
+    renamable $x8 = IRGstack $sp, $xzr
+    renamable $x9 = TAGPstack %stack.0, 0, renamable $x8, 1
+    renamable $x10 = COPY renamable $x9
+    dead early-clobber renamable $x11, dead early-clobber renamable $x10 = STGloop_wback 4080, killed renamable $x10, implicit-def dead $nzcv
+    ST2Gi renamable $x8, renamable $x8, 0
+
+    /* Epilogue */
+    dead early-clobber renamable $x8, dead early-clobber renamable $x9 = STGloop 4080, %stack.0, implicit-def dead $nzcv
+    ST2Gi $sp, %stack.1, 0
+    RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir b/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir
new file mode 100644
index 0000000000000..45f6bfe80ac2b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir
@@ -0,0 +1,103 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64 -mattr=+mte -run-pass=prologepilog %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+  target triple = "aarch64-unknown-none-elf"
+
+  @glob = global [8 x i32] zeroinitializer, align 4
+
+  declare dso_local void @F78(ptr %B)
+
+  define void @F55() sanitize_memtag "target-features"="+mte,+strict-align" {
+  entry:
+    %basetag = call ptr @llvm.aarch64.irg.sp(i64 0)
+    %A = alloca i32, i32 8, align 16
+    %A.tag = call ptr @llvm.aarch64.tagp.p0(ptr %A, ptr %basetag, i64 0)
+    %B = alloca i32, i32 8, align 4
+    %C = alloca i32, i32 8, align 16
+    %C.tag = call ptr @llvm.aarch64.tagp.p0(ptr %C, ptr %basetag, i64 1)
+    call void @llvm.aarch64.settag(ptr %C.tag, i64 32)
+    call void @F56(ptr %C.tag)
+    call void @llvm.lifetime.start.p0(i64 32, ptr %A)
+    call void @llvm.aarch64.settag(ptr %A.tag, i64 32)
+    call void @F56(ptr %A.tag)
+    call void @llvm.aarch64.settag(ptr %A, i64 32)
+    call void @llvm.lifetime.end.p0(i64 32, ptr %A)
+    call void @llvm.lifetime.start.p0(i64 32, ptr %A)
+    call void @llvm.memcpy.p0.p0.i64(ptr align 4 %A, ptr align 4 @glob, i64 32, i1 false)
+    call void @F78(ptr %A)
+    call void @llvm.lifetime.end.p0(i64 32, ptr %A)
+    call void @llvm.aarch64.settag(ptr %C, i64 32)
+    ret void
+  }
+
+  declare void @F56(ptr)
+...
+---
+name:            F55
+frameInfo:
+  adjustsStack:    true
+stack:
+  - { id: 0, name: A, type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -32, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 2, name: C, type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -64, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: F55
+    ; CHECK: liveins: $x19, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: frame-setup EMITMTETAGGED
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 80, 0
+    ; CHECK-NEXT: frame-setup STPXi killed $lr, killed $x19, $sp, 8 :: (store (s64) into %stack.3), (store (s64) into %stack.2)
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 80
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w19, -8
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
+    ; CHECK-NEXT: renamable $x0 = IRGstack $sp, $xzr
+    ; CHECK-NEXT: renamable $x19 = TAGPstack $x0, 2, renamable $x0, 1
+    ; CHECK-NEXT: ST2Gi renamable $x0, renamable $x0, 0 :: (store (s256) into %ir.C.tag, align 16)
+    ; CHECK-NEXT: BL @F56, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ; CHECK-NEXT: ST2Gi renamable $x19, renamable $x19, 0 :: (store (s256) into %ir.A.tag, align 16)
+    ; CHECK-NEXT: $x0 = COPY killed renamable $x19
+    ; CHECK-NEXT: BL @F56, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ; CHECK-NEXT: ST2Gi $sp, $sp, 2 :: (store (s256) into %ir.A, align 16)
+    ; CHECK-NEXT: renamable $x1 = LOADgot target-flags(aarch64-got) @glob
+    ; CHECK-NEXT: $x0 = ADDXri $sp, 32, 0
+    ; CHECK-NEXT: dead $w2 = MOVi32imm 32, implicit-def $x2
+    ; CHECK-NEXT: BL &memcpy, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit-def $sp, implicit-def dead $x0
+    ; CHECK-NEXT: $x0 = ADDXri $sp, 32, 0
+    ; CHECK-NEXT: BL @F78, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ; CHECK-NEXT: ST2Gi $sp, $sp, 0 :: (store (s256) into %ir.C, align 16)
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 64, 0
+    ; CHECK-NEXT: early-clobber $sp, $lr, $x19 = frame-destroy LDPXpost $sp, 2 :: (load (s64) from %stack.3), (load (s64) from %stack.2)
+    ; CHECK-NEXT: RET_ReallyLR
+    renamable $x0 = IRGstack $sp, $xzr
+    renamable $x19 = TAGPstack %stack.0.A, 0, renamable $x0, 1
+    ST2Gi renamable $x0, renamable $x0, 0 :: (store (s256) into %ir.C.tag, align 16)
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    BL @F56, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    ST2Gi renamable $x19, renamable $x19, 0 :: (store (s256) into %ir.A.tag, align 16)
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $x0 = COPY killed renamable $x19
+    BL @F56, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    ST2Gi $sp, %stack.0.A, 0 :: (store (s256) into %ir.A, align 16)
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    renamable $x1 = LOADgot target-flags(aarch64-got) @glob
+    $x0 = ADDXri %stack.0.A, 0, 0
+    dead $w2 = MOVi32imm 32, implicit-def $x2
+    BL &memcpy, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit-def $sp, implicit-def dead $x0
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $x0 = ADDXri %stack.0.A, 0, 0
+    BL @F78, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    ST2Gi $sp, %stack.2.C, 0 :: (store (s256) into %ir.C, align 16)
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/strpre-str-merge.mir b/llvm/test/CodeGen/AArch64/strpre-str-merge.mir
index 722de6bb343e2..5c1937e0d7753 100644
--- a/llvm/test/CodeGen/AArch64/strpre-str-merge.mir
+++ b/llvm/test/CodeGen/AArch64/strpre-str-merge.mir
@@ -156,7 +156,7 @@ body:             |
     ; CHECK-LABEL: name: 6-strqui-strqpre-no-merge
     ; CHECK: liveins: $q0, $q1, $x0
     ; CHECK: STRQui renamable $q1, renamable $x0, 1 :: (store (s128))
-    ; CHECK: early-clobber renamable $x0 = STRQpre renamable $q0, renamable $x0, 48, implicit $w0 :: (store (s128))
+    ; CHECK: early-clobber renamable $x0 = STRQpre renamable $q0, renamable $x0, 48, implicit $w0, implicit $w0_hi :: (store (s128))
     ; CHECK: RET undef $lr, implicit $x0
     STRQui killed renamable $q1, renamable $x0, 1 :: (store (s128))
     early-clobber renamable $x0 = STRQpre killed renamable $q0, killed renamable $x0, 48 :: (store (s128))
@@ -235,7 +235,7 @@ body:             |
     liveins: $s0, $s1, $x0, $x1
     ; CHECK-LABEL: name: 9-strspre-strsui-mod-base-reg-no-merge
     ; CHECK: liveins: $s0, $s1, $x0, $x1
-    ; CHECK: dead early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0 :: (store (s32))
+    ; CHECK: dead early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0, implicit $w0_hi :: (store (s32))
     ; CHECK: renamable $x0 = LDRXui renamable $x1, 1 :: (load (s64))
     ; CHECK: STRSui renamable $s1, renamable $x0, 1 :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
@@ -265,7 +265,7 @@ body:             |
     liveins: $s0, $s1, $x0, $x1
     ; CHECK-LABEL: name: 10-strspre-strsui-used-base-reg-no-merge
     ; CHECK: liveins: $s0, $s1, $x0, $x1
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0 :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0, implicit $w0_hi :: (store (s32))
     ; CHECK: STRXui renamable $x1, renamable $x1, 1 :: (store (s32))
     ; CHECK: STRSui renamable $s1, renamable $x0, 1 :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
@@ -296,12 +296,12 @@ body:             |
     liveins: $s0, $s1, $x0
     ; CHECK-LABEL: name: 11-strspre-strspre-no-merge
     ; CHECK: liveins: $s0, $s1, $x0
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0 :: (store (s32))
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 16, implicit $w0 :: (store (s32))
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 4, implicit $w0 :: (store (s32))
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 12, implicit $w0 :: (store (s32))
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 4, implicit $w0 :: (store (s32))
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 4, implicit $w0 :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0, implicit $w0_hi :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 16, implicit $w0, implicit $w0_hi :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 4, implicit $w0, implicit $w0_hi :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 12, implicit $w0, implicit $w0_hi :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 4, implicit $w0, implicit $w0_hi :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s1, renamable $x0, 4, implicit $w0, implicit $w0_hi :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRSpre renamable $s0, killed renamable $x0, 12 :: (store (s32))
     early-clobber renamable $x0 = STRSpre renamable $s1, killed renamable $x0, 16 :: (store (s32))
@@ -335,7 +335,7 @@ body:             |
     liveins: $s0, $s1, $x0
     ; CHECK-LABEL: name: 12-strspre-strsui-no-merge
     ; CHECK: liveins: $s0, $s1, $x0
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0 :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 12, implicit $w0, implicit $w0_hi :: (store (s32))
     ; CHECK: STRSui renamable $s1, renamable $x0, 2 :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRSpre killed renamable $s0, killed renamable $x0, 12 :: (store (s32))
@@ -390,7 +390,7 @@ body:             |
     liveins: $q0, $q1, $x0
     ; CHECK-LABEL: name: 14-strqpre-sturqi-no-merge
     ; CHECK: liveins: $q0, $q1, $x0
-    ; CHECK: early-clobber renamable $x0 = STRQpre renamable $q0, renamable $x0, 48, implicit $w0 :: (store (s128))
+    ; CHECK: early-clobber renamable $x0 = STRQpre renamable $q0, renamable $x0, 48, implicit $w0, implicit $w0_hi :: (store (s128))
     ; CHECK: STURQi renamable $q1, renamable $x0, 1 :: (store (s128))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRQpre killed renamable $q0, killed renamable $x0, 48 :: (store (s128))
@@ -417,7 +417,7 @@ body:             |
     liveins: $s0, $s1, $x0
     ; CHECK-LABEL: name: 15-strspre-strsui-unaligned-no-merge
     ; CHECK: liveins: $s0, $s1, $x0
-    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 251, implicit $w0 :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRSpre renamable $s0, renamable $x0, 251, implicit $w0, implicit $w0_hi :: (store (s32))
     ; CHECK: STRSui renamable $s1, renamable $x0, 1 :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRSpre killed renamable $s0, killed renamable $x0, 251 :: (store (s32))
@@ -443,7 +443,7 @@ body:             |
     liveins: $x0, $x1, $x2
     ; CHECK-LABEL: name: 16-strxpre-strxui-same-reg-no-merge
     ; CHECK: liveins: $x0, $x1, $x2
-    ; CHECK: early-clobber renamable $x0 = STRXpre renamable $x1, renamable $x0, 24, implicit $w0 :: (store (s64))
+    ; CHECK: early-clobber renamable $x0 = STRXpre renamable $x1, renamable $x0, 24, implicit $w0, implicit $w0_hi :: (store (s64))
     ; CHECK: STRXui renamable $x0, renamable $x0, 1 :: (store (s64))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRXpre killed renamable $x1, killed renamable $x0, 24 :: (store (s64))
@@ -470,7 +470,7 @@ body:             |
     liveins: $x0, $x1, $x2
     ; CHECK-LABEL: name: 17-strwpre-strwui-same-reg-no-merge
     ; CHECK: liveins: $x0, $x1, $x2
-    ; CHECK: early-clobber renamable $x0 = STRWpre renamable $w1, renamable $x0, 24, implicit $w0, implicit-def $w0 :: (store (s32))
+    ; CHECK: early-clobber renamable $x0 = STRWpre renamable $w1, renamable $x0, 24, implicit $w0, implicit $w0_hi, implicit-def $w0 :: (store (s32))
     ; CHECK: STRWui renamable $w0, renamable $x0, 1 :: (store (s32))
     ; CHECK: RET undef $lr, implicit $x0
     early-clobber renamable $x0 = STRWpre killed renamable $w1, killed renamable $x0, 24 :: (store (s32))
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
index 45829b3198224..f85658e9a596a 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-LABEL: name: bic_i16_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit killed $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit killed $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_H $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_H killed renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = BIC_ZPmZ_H killed renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
index 970077fdfea7b..0e2f1c3ff4f69 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-LABEL: name: fmul_float_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_S $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_S renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = FMUL_ZPmZ_S renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
index 308291fb6fe93..548ebc26b83e8 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
@@ -30,7 +30,7 @@ body:             |
     ; CHECK-LABEL: name: fsub_s_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_S $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_S renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = FSUBR_ZPmZ_S renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll b/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
index 1d05649964670..b4b946c68566e 100644
--- a/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
+++ b/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
@@ -1,12 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2
+; RUN: llc -mtriple=aarch64 -mattr=+sve %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
 
 define <vscale x 2 x i64> @signed_wide_add_nxv4i32(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %input){
-; CHECK-LABEL: signed_wide_add_nxv4i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddwb z0.d, z0.d, z1.s
-; CHECK-NEXT:    saddwt z0.d, z0.d, z1.s
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: signed_wide_add_nxv4i32:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    saddwb z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT:    saddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: signed_wide_add_nxv4i32:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    sunpklo z2.d, z1.s
+; CHECK-SVE-NEXT:    sunpkhi z1.d, z1.s
+; CHECK-SVE-NEXT:    add z0.d, z0.d, z2.d
+; CHECK-SVE-NEXT:    add z0.d, z1.d, z0.d
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = sext <vscale x 4 x i32> %input to <vscale x 4 x i64>
     %partial.reduce = tail call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv4i64(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %input.wide)
@@ -14,11 +23,19 @@ entry:
 }
 
 define <vscale x 2 x i64> @unsigned_wide_add_nxv4i32(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %input){
-; CHECK-LABEL: unsigned_wide_add_nxv4i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddwb z0.d, z0.d, z1.s
-; CHECK-NEXT:    uaddwt z0.d, z0.d, z1.s
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: unsigned_wide_add_nxv4i32:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    uaddwb z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT:    uaddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: unsigned_wide_add_nxv4i32:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    uunpklo z2.d, z1.s
+; CHECK-SVE-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-SVE-NEXT:    add z0.d, z0.d, z2.d
+; CHECK-SVE-NEXT:    add z0.d, z1.d, z0.d
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = zext <vscale x 4 x i32> %input to <vscale x 4 x i64>
     %partial.reduce = tail call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv4i64(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %input.wide)
@@ -26,11 +43,19 @@ entry:
 }
 
 define <vscale x 4 x i32> @signed_wide_add_nxv8i16(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %input){
-; CHECK-LABEL: signed_wide_add_nxv8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddwb z0.s, z0.s, z1.h
-; CHECK-NEXT:    saddwt z0.s, z0.s, z1.h
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: signed_wide_add_nxv8i16:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    saddwb z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT:    saddwt z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: signed_wide_add_nxv8i16:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    sunpklo z2.s, z1.h
+; CHECK-SVE-NEXT:    sunpkhi z1.s, z1.h
+; CHECK-SVE-NEXT:    add z0.s, z0.s, z2.s
+; CHECK-SVE-NEXT:    add z0.s, z1.s, z0.s
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = sext <vscale x 8 x i16> %input to <vscale x 8 x i32>
     %partial.reduce = tail call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv8i32(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %input.wide)
@@ -38,11 +63,19 @@ entry:
 }
 
 define <vscale x 4 x i32> @unsigned_wide_add_nxv8i16(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %input){
-; CHECK-LABEL: unsigned_wide_add_nxv8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddwb z0.s, z0.s, z1.h
-; CHECK-NEXT:    uaddwt z0.s, z0.s, z1.h
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: unsigned_wide_add_nxv8i16:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    uaddwb z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT:    uaddwt z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: unsigned_wide_add_nxv8i16:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    uunpklo z2.s, z1.h
+; CHECK-SVE-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-SVE-NEXT:    add z0.s, z0.s, z2.s
+; CHECK-SVE-NEXT:    add z0.s, z1.s, z0.s
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = zext <vscale x 8 x i16> %input to <vscale x 8 x i32>
     %partial.reduce = tail call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv8i32(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %input.wide)
@@ -50,11 +83,19 @@ entry:
 }
 
 define <vscale x 8 x i16> @signed_wide_add_nxv16i8(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %input){
-; CHECK-LABEL: signed_wide_add_nxv16i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddwb z0.h, z0.h, z1.b
-; CHECK-NEXT:    saddwt z0.h, z0.h, z1.b
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: signed_wide_add_nxv16i8:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    saddwb z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT:    saddwt z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: signed_wide_add_nxv16i8:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    sunpklo z2.h, z1.b
+; CHECK-SVE-NEXT:    sunpkhi z1.h, z1.b
+; CHECK-SVE-NEXT:    add z0.h, z0.h, z2.h
+; CHECK-SVE-NEXT:    add z0.h, z1.h, z0.h
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = sext <vscale x 16 x i8> %input to <vscale x 16 x i16>
     %partial.reduce = tail call <vscale x 8 x i16> @llvm.experimental.vector.partial.reduce.add.nxv8i16.nxv16i16(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %input.wide)
@@ -62,11 +103,19 @@ entry:
 }
 
 define <vscale x 8 x i16> @unsigned_wide_add_nxv16i8(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %input){
-; CHECK-LABEL: unsigned_wide_add_nxv16i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddwb z0.h, z0.h, z1.b
-; CHECK-NEXT:    uaddwt z0.h, z0.h, z1.b
-; CHECK-NEXT:    ret
+; CHECK-SVE2-LABEL: unsigned_wide_add_nxv16i8:
+; CHECK-SVE2:       // %bb.0: // %entry
+; CHECK-SVE2-NEXT:    uaddwb z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT:    uaddwt z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT:    ret
+;
+; CHECK-SVE-LABEL: unsigned_wide_add_nxv16i8:
+; CHECK-SVE:       // %bb.0: // %entry
+; CHECK-SVE-NEXT:    uunpklo z2.h, z1.b
+; CHECK-SVE-NEXT:    uunpkhi z1.h, z1.b
+; CHECK-SVE-NEXT:    add z0.h, z0.h, z2.h
+; CHECK-SVE-NEXT:    add z0.h, z1.h, z0.h
+; CHECK-SVE-NEXT:    ret
 entry:
     %input.wide = zext <vscale x 16 x i8> %input to <vscale x 16 x i16>
     %partial.reduce = tail call <vscale x 8 x i16> @llvm.experimental.vector.partial.reduce.add.nxv8i16.nxv16i16(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %input.wide)
diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 66c884e95fa47..b52cbfe08156b 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -358,11 +358,10 @@ define void @store_trunc_from_64bits(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    add x9, x0, #4
 ; CHECK-NEXT:    ld1r.4h { v0 }, [x9]
-; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    lsr w9, w8, #16
 ; CHECK-NEXT:    strb w8, [x1]
-; CHECK-NEXT:    add x8, x1, #1
-; CHECK-NEXT:    st1.b { v1 }[2], [x8]
 ; CHECK-NEXT:    add x8, x1, #2
+; CHECK-NEXT:    strb w9, [x1, #1]
 ; CHECK-NEXT:    st1.b { v0 }[4], [x8]
 ; CHECK-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index 184aa0226fe77..8473f45f6c803 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -4885,8 +4885,7 @@ entry:
 define i32 @extract_hi_hi(<8 x i16> %a) {
 ; CHECK-SD-LABEL: extract_hi_hi:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    mov v0.d[1], v0.d[0]
+; CHECK-SD-NEXT:    mov v0.d[0], v0.d[1]
 ; CHECK-SD-NEXT:    uaddlv s0, v0.8h
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
index 7fa416e0dbcd5..fd81deeb7d913 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
@@ -139,11 +139,11 @@ define i32 @test_v3i32(<3 x i32> %a) nounwind {
 define i1 @test_v4i1(<4 x i1> %a) nounwind {
 ; CHECK-LABEL: test_v4i1:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #15
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    uminv h0, v0.4h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %b = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %a)
   ret i1 %b
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
index 58020d28702b2..10a3ef1658a96 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
@@ -15,8 +15,15 @@ declare i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %a)
 declare i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %a)
 declare i1 @llvm.vector.reduce.or.v32i1(<32 x i1> %a)
 
-define i32 @reduce_and_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v1:
+declare i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> %a)
+
+define i32 @reduce_and_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.b[0]
@@ -29,16 +36,14 @@ define i32 @reduce_and_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_and_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v2:
+define i32 @reduce_and_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-NEXT:    uminp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
 ; CHECK-NEXT:    ret
   %x = icmp slt <2 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
@@ -46,16 +51,14 @@ define i32 @reduce_and_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_and_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v4:
+define i32 @reduce_and_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    uminv h0, v0.4h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
 ; CHECK-NEXT:    ret
   %x = icmp slt <4 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
@@ -63,14 +66,12 @@ define i32 @reduce_and_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_and_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v8:
+define i32 @reduce_and_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-NEXT:    uminv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
 ; CHECK-NEXT:    ret
   %x = icmp slt <8 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> %x)
@@ -78,8 +79,8 @@ define i32 @reduce_and_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_and_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v16:
+define i32 @reduce_and_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    uminv b0, v0.16b
@@ -93,8 +94,8 @@ define i32 @reduce_and_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_and_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v32:
+define i32 @reduce_and_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
@@ -109,8 +110,182 @@ define i32 @reduce_and_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v1:
+define i32 @reduce_and_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[0]
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    uminv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <16 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_and_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
+; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, eq
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.b[0]
@@ -123,15 +298,13 @@ define i32 @reduce_or_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v2:
+define i32 @reduce_or_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-NEXT:    umaxp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
 ; CHECK-NEXT:    csel w0, w0, w1, ne
 ; CHECK-NEXT:    ret
   %x = icmp slt <2 x i8> %a0, zeroinitializer
@@ -140,15 +313,13 @@ define i32 @reduce_or_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v4:
+define i32 @reduce_or_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    umaxv h0, v0.4h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
 ; CHECK-NEXT:    csel w0, w0, w1, ne
 ; CHECK-NEXT:    ret
   %x = icmp slt <4 x i8> %a0, zeroinitializer
@@ -157,13 +328,11 @@ define i32 @reduce_or_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v8:
+define i32 @reduce_or_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-NEXT:    umaxv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    fcmp d0, #0.0
 ; CHECK-NEXT:    csel w0, w0, w1, ne
 ; CHECK-NEXT:    ret
   %x = icmp slt <8 x i8> %a0, zeroinitializer
@@ -172,8 +341,8 @@ define i32 @reduce_or_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v16:
+define i32 @reduce_or_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    umaxv b0, v0.16b
@@ -187,8 +356,8 @@ define i32 @reduce_or_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
-define i32 @reduce_or_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v32:
+define i32 @reduce_or_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
@@ -202,3 +371,457 @@ define i32 @reduce_or_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
   %z = select i1 %y, i32 %a1, i32 %a2
   ret i32 %z
 }
+
+define i32 @reduce_or_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[0]
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    umaxv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <16 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_or_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.2d, v1.2d, #0
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    fcmp d0, #0.0
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.b[0]
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    addv h0, v0.4h
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT:    addv b0, v0.8b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <16 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <32 x i8> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    smov w8, v0.h[0]
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    addv h0, v0.4h
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    addv h0, v0.8h
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <16 x i16> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    addv h0, v0.8h
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <8 x i32> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    ret
+  %x = icmp slt <1 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    addp d0, v0.2d
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <2 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
+
+define i32 @reduce_xor_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v1.2d, v1.2d, #0
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel w0, w0, w1, ne
+; CHECK-NEXT:    ret
+  %x = icmp slt <4 x i64> %a0, zeroinitializer
+  %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+  %z = select i1 %y, i32 %a1, i32 %a2
+  ret i32 %z
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll
index 809a6d6556a7b..2a21cc8d7c611 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll
@@ -202,9 +202,8 @@ define i1 @test_v4i1(<4 x i1> %a) nounwind {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #15
 ; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    umaxv h0, v0.4h
-; CHECK-SD-NEXT:    fmov w8, s0
-; CHECK-SD-NEXT:    and w0, w8, #0x1
+; CHECK-SD-NEXT:    fcmp d0, #0.0
+; CHECK-SD-NEXT:    cset w0, ne
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_v4i1:
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index c0f1720e1cf8b..593c9db090a26 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -62,13 +62,11 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v1.8b, v1.8h
 ; NEON-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; NEON-FIXED-NEXT:    umaxv b1, v1.8b
+; NEON-FIXED-NEXT:    fcmp d1, #0.0
 ; NEON-FIXED-NEXT:    umaxv b2, v2.8b
 ; NEON-FIXED-NEXT:    fmov w8, s2
 ; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
 ; NEON-FIXED-NEXT:    ldrh w8, [x9]
-; NEON-FIXED-NEXT:    fmov w9, s1
-; NEON-FIXED-NEXT:    tst w9, #0x1
 ; NEON-FIXED-NEXT:    csel w0, w8, w0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -83,13 +81,11 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
 ; SVE-FIXED-NEXT:    str q0, [sp]
 ; SVE-FIXED-NEXT:    xtn v1.8b, v1.8h
 ; SVE-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; SVE-FIXED-NEXT:    umaxv b1, v1.8b
+; SVE-FIXED-NEXT:    fcmp d1, #0.0
 ; SVE-FIXED-NEXT:    umaxv b2, v2.8b
 ; SVE-FIXED-NEXT:    fmov w8, s2
 ; SVE-FIXED-NEXT:    bfi x9, x8, #1, #3
 ; SVE-FIXED-NEXT:    ldrh w8, [x9]
-; SVE-FIXED-NEXT:    fmov w9, s1
-; SVE-FIXED-NEXT:    tst w9, #0x1
 ; SVE-FIXED-NEXT:    csel w0, w8, w0, ne
 ; SVE-FIXED-NEXT:    add sp, sp, #16
 ; SVE-FIXED-NEXT:    ret
@@ -110,13 +106,11 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v1.4h, v1.4s
 ; NEON-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; NEON-FIXED-NEXT:    umaxv h1, v1.4h
+; NEON-FIXED-NEXT:    fcmp d1, #0.0
 ; NEON-FIXED-NEXT:    umaxv h2, v2.4h
 ; NEON-FIXED-NEXT:    fmov w8, s2
 ; NEON-FIXED-NEXT:    bfi x9, x8, #2, #2
 ; NEON-FIXED-NEXT:    ldr w8, [x9]
-; NEON-FIXED-NEXT:    fmov w9, s1
-; NEON-FIXED-NEXT:    tst w9, #0x1
 ; NEON-FIXED-NEXT:    csel w0, w8, w0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -131,13 +125,11 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
 ; SVE-FIXED-NEXT:    str q0, [sp]
 ; SVE-FIXED-NEXT:    xtn v1.4h, v1.4s
 ; SVE-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; SVE-FIXED-NEXT:    umaxv h1, v1.4h
+; SVE-FIXED-NEXT:    fcmp d1, #0.0
 ; SVE-FIXED-NEXT:    umaxv h2, v2.4h
 ; SVE-FIXED-NEXT:    fmov w8, s2
 ; SVE-FIXED-NEXT:    bfi x9, x8, #2, #2
 ; SVE-FIXED-NEXT:    ldr w8, [x9]
-; SVE-FIXED-NEXT:    fmov w9, s1
-; SVE-FIXED-NEXT:    tst w9, #0x1
 ; SVE-FIXED-NEXT:    csel w0, w8, w0, ne
 ; SVE-FIXED-NEXT:    add sp, sp, #16
 ; SVE-FIXED-NEXT:    ret
@@ -158,13 +150,11 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v1.2s, v1.2d
 ; NEON-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; NEON-FIXED-NEXT:    umaxp v1.2s, v1.2s, v1.2s
+; NEON-FIXED-NEXT:    fcmp d1, #0.0
 ; NEON-FIXED-NEXT:    umaxp v2.2s, v2.2s, v2.2s
 ; NEON-FIXED-NEXT:    fmov w8, s2
 ; NEON-FIXED-NEXT:    bfi x9, x8, #3, #1
 ; NEON-FIXED-NEXT:    ldr x8, [x9]
-; NEON-FIXED-NEXT:    fmov w9, s1
-; NEON-FIXED-NEXT:    tst w9, #0x1
 ; NEON-FIXED-NEXT:    csel x0, x8, x0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -179,13 +169,11 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
 ; SVE-FIXED-NEXT:    str q0, [sp]
 ; SVE-FIXED-NEXT:    xtn v1.2s, v1.2d
 ; SVE-FIXED-NEXT:    and v2.8b, v1.8b, v2.8b
-; SVE-FIXED-NEXT:    umaxp v1.2s, v1.2s, v1.2s
+; SVE-FIXED-NEXT:    fcmp d1, #0.0
 ; SVE-FIXED-NEXT:    umaxp v2.2s, v2.2s, v2.2s
 ; SVE-FIXED-NEXT:    fmov w8, s2
 ; SVE-FIXED-NEXT:    bfi x9, x8, #3, #1
 ; SVE-FIXED-NEXT:    ldr x8, [x9]
-; SVE-FIXED-NEXT:    fmov w9, s1
-; SVE-FIXED-NEXT:    tst w9, #0x1
 ; SVE-FIXED-NEXT:    csel x0, x8, x0, ne
 ; SVE-FIXED-NEXT:    add sp, sp, #16
 ; SVE-FIXED-NEXT:    ret
@@ -206,13 +194,11 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v1.4h, v1.4s
 ; NEON-FIXED-NEXT:    and v3.8b, v1.8b, v3.8b
-; NEON-FIXED-NEXT:    umaxv h1, v1.4h
+; NEON-FIXED-NEXT:    fcmp d1, #0.0
 ; NEON-FIXED-NEXT:    umaxv h3, v3.4h
 ; NEON-FIXED-NEXT:    fmov w8, s3
 ; NEON-FIXED-NEXT:    bfi x9, x8, #2, #2
-; NEON-FIXED-NEXT:    fmov w8, s1
 ; NEON-FIXED-NEXT:    ldr s0, [x9]
-; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -227,13 +213,11 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
 ; SVE-FIXED-NEXT:    str q0, [sp]
 ; SVE-FIXED-NEXT:    xtn v1.4h, v1.4s
 ; SVE-FIXED-NEXT:    and v3.8b, v1.8b, v3.8b
-; SVE-FIXED-NEXT:    umaxv h1, v1.4h
+; SVE-FIXED-NEXT:    fcmp d1, #0.0
 ; SVE-FIXED-NEXT:    umaxv h3, v3.4h
 ; SVE-FIXED-NEXT:    fmov w8, s3
 ; SVE-FIXED-NEXT:    bfi x9, x8, #2, #2
-; SVE-FIXED-NEXT:    fmov w8, s1
 ; SVE-FIXED-NEXT:    ldr s0, [x9]
-; SVE-FIXED-NEXT:    tst w8, #0x1
 ; SVE-FIXED-NEXT:    fcsel s0, s0, s2, ne
 ; SVE-FIXED-NEXT:    add sp, sp, #16
 ; SVE-FIXED-NEXT:    ret
@@ -254,13 +238,11 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    xtn v1.2s, v1.2d
 ; NEON-FIXED-NEXT:    and v3.8b, v1.8b, v3.8b
-; NEON-FIXED-NEXT:    umaxp v1.2s, v1.2s, v1.2s
+; NEON-FIXED-NEXT:    fcmp d1, #0.0
 ; NEON-FIXED-NEXT:    umaxp v3.2s, v3.2s, v3.2s
 ; NEON-FIXED-NEXT:    fmov w8, s3
 ; NEON-FIXED-NEXT:    bfi x9, x8, #3, #1
-; NEON-FIXED-NEXT:    fmov w8, s1
 ; NEON-FIXED-NEXT:    ldr d0, [x9]
-; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    fcsel d0, d0, d2, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -275,13 +257,11 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
 ; SVE-FIXED-NEXT:    str q0, [sp]
 ; SVE-FIXED-NEXT:    xtn v1.2s, v1.2d
 ; SVE-FIXED-NEXT:    and v3.8b, v1.8b, v3.8b
-; SVE-FIXED-NEXT:    umaxp v1.2s, v1.2s, v1.2s
+; SVE-FIXED-NEXT:    fcmp d1, #0.0
 ; SVE-FIXED-NEXT:    umaxp v3.2s, v3.2s, v3.2s
 ; SVE-FIXED-NEXT:    fmov w8, s3
 ; SVE-FIXED-NEXT:    bfi x9, x8, #3, #1
-; SVE-FIXED-NEXT:    fmov w8, s1
 ; SVE-FIXED-NEXT:    ldr d0, [x9]
-; SVE-FIXED-NEXT:    tst w8, #0x1
 ; SVE-FIXED-NEXT:    fcsel d0, d0, d2, ne
 ; SVE-FIXED-NEXT:    add sp, sp, #16
 ; SVE-FIXED-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll
new file mode 100644
index 0000000000000..1caee994220f0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll
@@ -0,0 +1,666 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve    < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 2 x double> @test_svabs_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svabs_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fabs.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svabs_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svabs_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fabs.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svabs_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svabs_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fabs.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 4 x float> @test_svabs_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svabs_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fabs.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svabs_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svabs_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fabs.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svabs_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svabs_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fabs.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 8 x half> @test_svabs_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svabs_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fabs.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svabs_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svabs_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fabs.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svabs_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svabs_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fabs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fabs.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 16 x i8> @test_svabs_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svabs_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    abs z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.b, p0/z, z0.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svabs_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svabs_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %1 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %1
+}
+
+define <vscale x 16 x i8> @test_svabs_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svabs_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %1 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %1
+}
+
+define <vscale x 8 x i16> @test_svabs_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svabs_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    abs z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svabs_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svabs_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    abs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svabs_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svabs_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    abs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svabs_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svabs_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    abs z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svabs_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svabs_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    abs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svabs_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svabs_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    abs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svabs_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svabs_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    abs z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.abs.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svabs_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svabs_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    abs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.abs.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svabs_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svabs_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    abs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svabs_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    abs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.abs.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x double> @test_svneg_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svneg_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fneg z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fneg.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svneg_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svneg_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fneg.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svneg_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svneg_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fneg.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 4 x float> @test_svneg_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svneg_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fneg z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fneg.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svneg_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svneg_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fneg.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svneg_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svneg_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fneg.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 8 x half> @test_svneg_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svneg_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fneg.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svneg_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svneg_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fneg.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svneg_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svneg_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fneg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fneg.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 16 x i8> @test_svneg_s8_x_1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svneg_s8_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    neg z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s8_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.b, p0/z, z0.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.neg.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %0
+}
+
+define <vscale x 16 x i8> @test_svneg_s8_x_2(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svneg_s8_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    neg z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s8_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %1 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.neg.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %1
+}
+
+define <vscale x 16 x i8> @test_svneg_s8_z(<vscale x 16 x i1> %pg, double %z0, <vscale x 16 x i8> %x) {
+; CHECK-LABEL: test_svneg_s8_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, #0 // =0x0
+; CHECK-NEXT:    neg z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s8_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.b, p0/z, z1.b
+; CHECK-2p2-NEXT:    ret
+entry:
+  %1 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.neg.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %x)
+  ret <vscale x 16 x i8> %1
+}
+
+define <vscale x 8 x i16> @test_svneg_s16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svneg_s16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    neg z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.neg.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svneg_s16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svneg_s16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    neg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.neg.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svneg_s16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x i16> %x) {
+; CHECK-LABEL: test_svneg_s16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    neg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.neg.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svneg_s32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svneg_s32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    neg z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.neg.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svneg_s32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svneg_s32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    neg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.neg.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svneg_s32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x i32> %x) {
+; CHECK-LABEL: test_svneg_s32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    neg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.neg.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svneg_s64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svneg_s64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    neg z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.neg.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svneg_s64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svneg_s64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    neg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.neg.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svneg_s64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: test_svneg_s64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    neg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svneg_s64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    neg z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.neg.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %x)
+  ret <vscale x 2 x i64> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll
new file mode 100644
index 0000000000000..cf9ac49ca7b23
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve,+bf16    < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2,+bf16 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme,+bf16    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2,+bf16 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 8 x half> @test_svcvt_f16_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f16_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.h, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f32(<vscale x 8 x half> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svcvt_f16_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f16_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f32(<vscale x 8 x half> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svcvt_f16_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f16_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f32(<vscale x 8 x half> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x bfloat> @test_svcvt_bf16_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_bf16_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_bf16_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    bfcvt z0.h, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x bfloat> @test_svcvt_bf16_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_bf16_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_bf16_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    bfcvt z0.h, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x bfloat> @test_svcvt_bf16_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_bf16_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_bf16_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    bfcvt z0.h, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 8 x bfloat> %0
+}
+
+define <vscale x 8 x half> @test_svcvt_f16_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f16_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.h, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f64(<vscale x 8 x half> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svcvt_f16_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f16_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f64(<vscale x 8 x half> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 8 x half> @test_svcvt_f16_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f16_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f16_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.h, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.fcvt.f16f64(<vscale x 8 x half> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 8 x half> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f32_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.s, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f64(<vscale x 4 x float> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f32_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f64(<vscale x 4 x float> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_f32_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f64(<vscale x 4 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f16_x_1(<vscale x 4 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f32_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f16(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f32_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f16(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvt_f32_f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f32_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f32_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvt.f32f16(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f16_x_1(<vscale x 2 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f64_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.d, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f16(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f64_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f16(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_f64_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f16(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f32_x_1(<vscale x 2 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f64_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.d, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f32(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f64_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f32(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvt_f64_f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_f64_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_f64_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvt z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvt.f64f32(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll
new file mode 100644
index 0000000000000..60879b1529230
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2   < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme2   -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 4 x float> @test_svcvtlt_f32_f16_x_1(<vscale x 4 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvtlt_f32_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtlt z0.s, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f32_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.s, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtlt.f32f16(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvtlt_f32_f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvtlt_f32_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtlt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f32_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtlt.f32f16(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvtlt_f32_f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvtlt_f32_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtlt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f32_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtlt.f32f16(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 2 x double> @test_svcvtlt_f64_f32_x_1(<vscale x 2 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvtlt_f64_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtlt z0.d, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f64_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.d, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvtlt.f64f32(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvtlt_f64_f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvtlt_f64_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtlt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f64_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvtlt.f64f32(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 2 x double> @test_svcvtlt_f64_f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvtlt_f64_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtlt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtlt_f64_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtlt z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fcvtlt.f64f32(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x double> %0
+}
+
+define <vscale x 4 x float> @test_svcvtx_f32_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvtx_f32_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtx z0.s, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtx_f32_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtx z0.s, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvtx_f32_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvtx_f32_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtx z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtx_f32_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtx z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
+
+define <vscale x 4 x float> @test_svcvtx_f32_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvtx_f32_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtx z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvtx_f32_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtx z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x float> %0
+}
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll
new file mode 100644
index 0000000000000..b8b36d390330a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll
@@ -0,0 +1,659 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve    < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+
+target triple = "aarch64-linux"
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f64(<vscale x 4 x i32> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f64(<vscale x 4 x i32> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f64(<vscale x 4 x i32> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f32_x_1(<vscale x 2 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f32(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f32(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f32(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f16_x_1(<vscale x 4 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f16(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f16(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzs_s32_f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s32_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s32_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.i32f16(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f16_x_1(<vscale x 2 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f16(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f16(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzs_s64_f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzs_s64_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzs_s64_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.i64f16(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f64(<vscale x 4 x i32> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f64(<vscale x 4 x i32> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f64(<vscale x 4 x i32> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f32_x_1(<vscale x 2 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f32(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f32_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f32(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f32_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f32(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f16_x_1(<vscale x 4 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f16(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f16_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f16(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_fcvtzu_u32_f16_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u32_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u32_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.i32f16(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f16_x_1(<vscale x 2 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f16(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f16_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f16(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_fcvtzu_u64_f16_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_fcvtzu_u64_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_fcvtzu_u64_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.i64f16(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+
+define <vscale x 8 x i16> @test_svcvt_s16_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_s16_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s16_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svcvt_s16_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_s16_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s16_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svcvt_s16_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_s16_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s16_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svcvt_u16_f16_x_1(<vscale x 8 x i1> %pg, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_u16_f16_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u16_f16_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.h, p0/z, z0.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svcvt_u16_f16_x_2(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_u16_f16_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzu z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u16_f16_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 8 x i16> @test_svcvt_u16_f16_z(<vscale x 8 x i1> %pg, double %z0, <vscale x 8 x half> %x) {
+; CHECK-LABEL: test_svcvt_u16_f16_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.h, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u16_f16_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.h, p0/z, z1.h
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, <vscale x 8 x half> %x)
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_s32_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_s32_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s32_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_s32_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_s32_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s32_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_s32_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_s32_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s32_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_u32_f32_x_1(<vscale x 4 x i1> %pg, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_u32_f32_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u32_f32_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z0.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_u32_f32_x_2(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_u32_f32_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u32_f32_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 4 x i32> @test_svcvt_u32_f32_z(<vscale x 4 x i1> %pg, double %z0, <vscale x 4 x float> %x) {
+; CHECK-LABEL: test_svcvt_u32_f32_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u32_f32_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.s, p0/z, z1.s
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, <vscale x 4 x float> %x)
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_s64_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_s64_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s64_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_s64_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_s64_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s64_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_s64_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_s64_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_s64_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzs z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_u64_f64_x_1(<vscale x 2 x i1> %pg, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_u64_f64_x_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u64_f64_x_1:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z0.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_u64_f64_x_2(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_u64_f64_x_2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u64_f64_x_2:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 2 x i64> @test_svcvt_u64_f64_z(<vscale x 2 x i1> %pg, double %z0, <vscale x 2 x double> %x) {
+; CHECK-LABEL: test_svcvt_u64_f64_z:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-2p2-LABEL: test_svcvt_u64_f64_z:
+; CHECK-2p2:       // %bb.0: // %entry
+; CHECK-2p2-NEXT:    fcvtzu z0.d, p0/z, z1.d
+; CHECK-2p2-NEXT:    ret
+entry:
+  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, <vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
index de46037e96e80..2215df9cef262 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
@@ -1,25 +1,26 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define amdgpu_ps void @v_interp_f32(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v0, attr0.y wait_vdst:15
-; GCN-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v2, s0
-; GCN-NEXT:    v_mov_b32_e32 v4, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p2_f32 v5, v0, v4, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_f32 v4, v1, v4, v5 wait_exp:7
-; GCN-NEXT:    exp mrt0 v3, v2, v5, v4 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    s_mov_b32 m0, s2
+; GFX11-NEXT:    lds_param_load v0, attr0.y wait_vdst:15
+; GFX11-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    v_mov_b32_e32 v4, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:0
+; GFX11-NEXT:    v_interp_p2_f32 v5, v0, v4, v3 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_interp_p2_f32 v4, v1, v4, v5 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v3, v2, v5, v4 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
@@ -32,30 +33,30 @@ main_body:
 }
 
 define amdgpu_ps void @v_interp_f32_many(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32_many:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v0, attr0.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v2, attr2.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v3, attr3.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
-; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
-; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:0
-; GCN-NEXT:    v_interp_p2_f32 v6, v0, v5, v6 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v7, v1, v5, v7 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v8, v2, v5, v8 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v4, v3, v5, v4 wait_exp:7
-; GCN-NEXT:    exp mrt0 v6, v7, v8, v4 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32_many:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    s_mov_b32 m0, s2
+; GFX11-NEXT:    lds_param_load v0, attr0.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v2, attr2.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v3, attr3.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
+; GFX11-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
+; GFX11-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:0
+; GFX11-NEXT:    v_interp_p2_f32 v6, v0, v5, v6 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v7, v1, v5, v7 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v8, v2, v5, v8 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v4, v3, v5, v4 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v6, v7, v8, v4 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
@@ -74,30 +75,30 @@ main_body:
 }
 
 define amdgpu_ps void @v_interp_f32_many_vm(ptr addrspace(1) %ptr, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32_many_vm:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
-; GCN-NEXT:    s_mov_b32 m0, s0
-; GCN-NEXT:    s_mov_b32 s0, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    lds_param_load v2, attr0.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v3, attr1.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v4, attr2.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v5, attr3.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
-; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
-; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:0
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v6, v2, v1, v6 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v7, v3, v1, v7 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v8, v4, v1, v8 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
-; GCN-NEXT:    exp mrt0 v6, v7, v8, v0 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32_many_vm:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
+; GFX11-NEXT:    s_mov_b32 m0, s0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    lds_param_load v2, attr0.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v3, attr1.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v4, attr2.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v5, attr3.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
+; GFX11-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
+; GFX11-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v6, v2, v1, v6 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v7, v3, v1, v7 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v8, v4, v1, v8 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v6, v7, v8, v0 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %i.ptr = getelementptr float, ptr addrspace(1) %ptr, i32 1
   %i = load float, ptr addrspace(1) %i.ptr, align 4
@@ -120,23 +121,41 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f16:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f16_f32 v3, v1, v0, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p10_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
-; GCN-NEXT:    v_interp_p2_f16_f32 v3, v1, v2, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
-; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-TRUE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-TRUE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v3, v1.l, v0, v1.l wait_exp:0
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v4, v1.h, v0, v1.h wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.l, v1.l, v2, v3 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.h, v1.h, v2, v4 wait_exp:7
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-FAKE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-FAKE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-FAKE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v3, v1, v0, v1 wait_exp:0
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v3, v1, v2, v3 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float %p0, float %i, float %p0, i1 0)
@@ -148,23 +167,41 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_rtz_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_rtz_f16:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
-; GCN-NEXT:    v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
-; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_rtz_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-TRUE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-TRUE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1.l, v0, v1.l wait_exp:0
+; GFX11-TRUE16-NEXT:    v_interp_p10_rtz_f16_f32 v4, v1.h, v0, v1.h wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_rtz_f16_f32 v0.l, v1.l, v2, v3 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_interp_p2_rtz_f16_f32 v0.h, v1.h, v2, v4 wait_exp:7
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_rtz_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-FAKE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-FAKE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-FAKE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0
+; GFX11-FAKE16-NEXT:    v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 0)
@@ -176,17 +213,30 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 {
-; GCN-LABEL: v_interp_f16_imm_params:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v0 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GCN-NEXT:    v_add_f16_e32 v0, v1, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_f16_imm_params:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v1, v0.l, v1, v0.l wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.l, v0.l, v3, v2 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v1
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_f16_imm_params:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v0 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float 0.0, float %i, float 0.0, i1 0)
   %l_p1 = call half @llvm.amdgcn.interp.inreg.p2.f16(float 0.0, float %j, float 0.0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 07c94362d0439..756eb2788607b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx801 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s
 
 define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
@@ -65,11 +66,23 @@ define i16 @v_mul_i16(i16 %num, i16 %den) {
 ; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_mul_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_mul_lo_u16 v0, v0, v1
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_mul_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_mul_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mul_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_mul_i16:
 ; GFX12:       ; %bb.0:
@@ -147,12 +160,26 @@ define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
 ; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_mul_i16_zeroext:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_mul_lo_u16 v0, v0, v1
-; GFX10PLUS-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_mul_i16_zeroext:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_mul_i16_zeroext:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mul_i16_zeroext:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_mul_i16_zeroext:
 ; GFX12:       ; %bb.0:
@@ -236,12 +263,26 @@ define signext i16 @v_mul_i16_signext(i16 signext %num, i16 signext %den) {
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_mul_i16_signext:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_mul_lo_u16 v0, v0, v1
-; GFX10PLUS-NEXT:    v_bfe_i32 v0, v0, 0, 16
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_mul_i16_signext:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_mul_i16_signext:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_mul_i16_signext:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_mul_i16_signext:
 ; GFX12:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
index 51ec8ecfd05f2..7f7f8b0121567 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s
 
 ---
 name: abs_sgpr_s16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
index bb27c118f8d11..54ee69fcb2204 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 ---
 name: add_s16_ss
 legalized: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
index c856366b083cb..45332c2870c02 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 
 ---
 name: add_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
index 419478c6d591c..97018fac13a87 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: add_v2s16_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp-compr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp-compr.mir
index fc0564a8e2f13..41a00533fedf4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp-compr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp-compr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: exp_compr_v2f16_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
index 87ba9259106f0..313b0c5b6707c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 --- |
   define void @exp_s() {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
index 0ef5aaea3b149..3d016c143e706 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: buffer_load_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ballot.i64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ballot.i64.mir
index 4f94b6ec9cb83..aa54b425a4db0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ballot.i64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ballot.i64.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name:            ballot_sgpr_src
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.class.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.class.mir
index 747760bf3fab1..95e63c74a5088 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.class.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.class.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: class_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.cvt.pkrtz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.cvt.pkrtz.mir
index eada0ad9b6d00..a39287f1991e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.cvt.pkrtz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.cvt.pkrtz.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: cvt_pkrtz_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
index 031c5a10161d4..3fd01395c7f21 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s | FileCheck %s
 
 ---
 name: div_fmas_sss_scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.scale.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.scale.mir
index 1465ec791dc7d..ea9ec47e594af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.scale.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.scale.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: div_scale_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.append.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.append.mir
index 68839a0da3673..70ca545024843 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.append.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.append.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: ds_append_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.bpermute.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.bpermute.mir
index 7fa99f75a69e1..9085d39e93611 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.bpermute.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.bpermute.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: ds_bpermute_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.consume.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.consume.mir
index 4f9fadf8bbfbc..14cadc90db749 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.consume.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.consume.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: ds_consume_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.init.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.init.mir
index e27882175da62..facf7717ce5a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.init.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.init.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: ds_gws_init_s_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.sema.v.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.sema.v.mir
index e52fe79bf0849..ad575fe813755 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.sema.v.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.gws.sema.v.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: ds_gws_init_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.add.mir
index f564c92bf493a..516e842d1bf11 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.add.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: ds_ordered_add_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.swap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.swap.mir
index 5b834b243e42d..c8cd2f850c1ad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.swap.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.ordered.swap.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: ds_ordered_swap_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.permute.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.permute.mir
index 354678ec074fd..b65898e53f810 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.permute.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.permute.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: ds_permute_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir
index 87b3219f5da7c..2956a2f1e4f45 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: ds_swizzle_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.32.mir
index a0711e6c779cd..f530d876d4069 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.32.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: else
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.64.mir
index 8a2cbd0eafd6c..97fc30233af7a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.else.64.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 ---
 name: else
 legalized: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
index 660deb90322a1..be59d356af05e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: fcmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fmul.legacy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fmul.legacy.mir
index ecb568b7c2d2c..0b83571560267 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fmul.legacy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fmul.legacy.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fmul_legacy_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.groupstaticsize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.groupstaticsize.mir
index b3ad048d46e5e..c5f066de1db1c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.groupstaticsize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.groupstaticsize.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: groupstaticsize
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
index 88e974b41e5c0..434cc138f3704 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: icmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
index 61263e0efa2ea..c561bd85d070a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
 
 ; Natural mapping
 define amdgpu_ps void @load_1d_vgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32 %s) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
index d6a7ae8d867fe..bd3d5f29f11f9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
 
 ; Natural mapping
 define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.mov.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.mov.mir
index 42db8a340dd66..299e8fc389984 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.mov.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.mov.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: interp_mov_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.f16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.f16.mir
index 3b040da301460..b1fc3181c0afc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.f16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.f16.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: interp_p1_f16_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.mir
index 041db1c403b9f..6567c159a745f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p1.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: interp_p1_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.f16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.f16.mir
index 05dacb9dd79c3..27913ec3e85ec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.f16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.f16.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: interp_p2_f16_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.mir
index b344fed13fcce..353bd7207a162 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.interp.p2.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: interp_p2_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kernarg.segment.ptr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kernarg.segment.ptr.mir
index 4f4ce55dc6b53..d1f3a42f80289 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kernarg.segment.ptr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kernarg.segment.ptr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: kernarg_segment_ptr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
index 2a6d8b1abf67e..ab224f5cd8ee7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s| FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s| FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s| FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s| FileCheck %s
 
 ---
 name: kill_scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.direct.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.direct.load.mir
index fbc77708d1497..f39d13f8b4868 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.direct.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.direct.load.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: lds_direct_load_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.param.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.param.load.mir
index 802a3098d46c5..9b5c79f355da7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.param.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.lds.param.load.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: lds_param_load_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.live.mask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.live.mask.mir
index ca2004aac116c..fba83f8d5a193 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.live.mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.live.mask.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: live_mask
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
index 2d3630aca8399..67ed51a396c8b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
 
 ---
 name: mfma_f32_32x32x4bf16_1k_vva
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx940.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx940.mir
index c445814ce2112..2944bb4ebbd17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx940.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx940.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
+# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
 
 ---
 name: mfma_i32_16x16x32_i8_vva
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
index c45f9d46049a1..e34a62d29a102 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: mfma_f32_32x32x1f32_vva
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ps.live.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ps.live.mir
index 0121287351d9e..50382cf4ea8d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ps.live.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ps.live.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: ps_live
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
index 8c7bdb867d168..b9bde7aecd34b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
index 19793f7020dc8..88fcc593dfaf3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps float @raw_ptr_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
index 37313cc622095..04cdf2e9fce73 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: readfirstlane_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readlane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readlane.mir
index 7fa092ed353b9..c02b8ddb3cdea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readlane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readlane.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: readlane_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
index f426fb8954ed2..61cba3d2af84f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -simplify-mir -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s -check-prefix=GFX7
-; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -simplify-mir -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s -check-prefix=GFX7
-; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -simplify-mir -stop-after=amdgpu-regbankselect -o - %s | FileCheck %s -check-prefix=GFX12
+; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -simplify-mir -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s -check-prefix=GFX7
+; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -simplify-mir -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s -check-prefix=GFX7
+; RUN: llc -amdgpu-global-isel-new-legality -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -simplify-mir -stop-after=regbankselect -o - %s | FileCheck %s -check-prefix=GFX12
 
 ; Natural mapping
 define amdgpu_ps i32 @s_buffer_load_i32(<4 x i32> inreg %rsrc, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.mir
index e5549bbe49810..45a2ab5b774c2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=regbankselect -regbankselect-fast -o - %s | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=regbankselect -regbankselect-greedy -o - %s | FileCheck -check-prefix=GREEDY %s
 
 # We see the offset is a VGPR, but this is due to a constant for some
 # reason ending up in a VGPR. This shouldn't really ever happen, but
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.get.waveid.in.workgroup.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.get.waveid.in.workgroup.mir
index e63ee985608bc..8307fa88da182 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.get.waveid.in.workgroup.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.get.waveid.in.workgroup.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: s_get_waveid_in_workgroup
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
index c906e16a6a591..9650da855ba5a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: getpc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getreg.mir
index a0c385f15298b..b4a02721ad631 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getreg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: getreg
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memrealtime.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memrealtime.mir
index 1c25750ce3fe6..a2ec76a4950d3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memrealtime.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memrealtime.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: memrealtime
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memtime.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memtime.mir
index 32f9327bc3765..4caf9882f7b17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memtime.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.memtime.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
 
 ---
 name: memtime
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsg.mir
index 18ae6ac7ec375..d46a2c76c795f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: sendmsg_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsghalt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsghalt.mir
index 3a167f5997ff7..f32ad5c225a32 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsghalt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.sendmsghalt.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: sendmsghalt_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
index f664e62761ad5..e3b04661b15f8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
index 0f72586ed6c12..a4d8410249d90 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
index b835b3a3e380b..4c67fe88c3c32 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps float @struct_ptr_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
index 0cefc373dd7cd..ccbdc4eaebfcc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ; Natural mapping
 define amdgpu_ps void @struct_ptr_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.update.dpp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.update.dpp.mir
index 48cbfe5e997cb..778b577c77e54 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.update.dpp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.update.dpp.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name: update_dpp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
index 7039ae7ce0344..373caa258c43f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s| FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s| FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s| FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s| FileCheck %s
 
 ---
 name: wqm_demote_scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.mir
index 0ff96eb85d41f..d5a0e03acb361 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: wqm_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.vote.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.vote.mir
index ce6d3e63dc5cd..aef8cda8e73e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.vote.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.vote.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: wqm_vote_scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.writelane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.writelane.mir
index b98e6020a615f..9cd2219faf598 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.writelane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.writelane.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: writelane_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wwm.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wwm.mir
index f957491be57c5..e5590d7b59eec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wwm.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wwm.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: strict_wwm_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
index ad124f0b24109..fa5df0f69bb18 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ffbh_u32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
index 7786145bed9ea..23fc3de6490af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ffbl_b32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-wave-address.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-wave-address.mir
index 8548399c1123e..f372c1f81948f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-wave-address.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-wave-address.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -o - %s | FileCheck %s
 
 # TODO: We could use scalar
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
index a1e9b0f8ceee7..1276752017812 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: and_s1_sgpr_sgpr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
index eeab20751d26a..ae818e036747a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: and_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
index bb6a029f1bea5..22de4d00c2a51 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: anyext_s32_to_s64_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
index ad43e6288fe64..615cfec2b31cf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-fast -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-fast -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ---
 name: ashr_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-align.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-align.mir
index e6d1bc7717cb0..08acea98700ab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-align.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-align.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s
 
 ---
 name:            assert_align_vgpr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-zext.mir
index 296e8ad752b2f..0bce908969c17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-assert-zext.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s
 
 ---
 name:            assert_zext_vgpr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir
index f9886c1a54a94..48ce59ffd3c43 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomic_cmpxchg_global_i32_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir
index 1d669836b852e..bcd676f31c90a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_add_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir
index 8aef5bec9fb36..e4fe9c5c7d9f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_and_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-fadd.mir
index 8cb81aaeb7872..11833cab3c07f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-fadd.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_fadd_local_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir
index 6c72e01619c71..82b1dccc5458a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_max_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir
index b057fa27e09f8..44e44851f7004 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_min_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir
index 68a5078b10c6c..3a16d72cb8ebd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_or_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir
index aea8c94396c07..e7b9c8efff6ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_sub_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir
index c268e41c63d77..35ec36275c3cd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_umax_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir
index d0a4e53ee5d79..1e971bfba160e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_umin_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir
index 72dcf4aa7b149..db01a21d061d3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_xchg_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir
index 087570ccb1abe..903d6f54ba46d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: atomicrmw_xor_global_i32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
index 67c12bd41f4bd..550f042618abf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: bitcast_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitreverse.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitreverse.mir
index 0e6933625d4dd..e82a492dbec20 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitreverse.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitreverse.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: bitreverse_i32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-block-addr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-block-addr.mir
index 4efd1413ee7b0..a50c7fe0748b8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-block-addr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-block-addr.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -o - | FileCheck %s
+# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass=regbankselect %s -o - | FileCheck %s
 
 --- |
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
index bca0fdc6a1467..3b2b141539fc5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: brcond_vcc_cond
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bswap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bswap.mir
index cf16aed237bae..c44dd484920e7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bswap.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bswap.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: bswap_i32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
index 996ddd8660c10..0dc1165843e8f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: build_vector_v2s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-concat-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-concat-vector.mir
index d7165795cceff..135aefa98e77c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-concat-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-concat-vector.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: concat_vectors_v4s16_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-constant.mir
index 96560b1bd8435..233176281f8f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-constant.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: test_constant_s32_vgpr_use
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-copy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-copy.mir
index 30c374ddee573..bf8f2d633c1dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-copy.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck --check-prefix=WAVE32 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck --check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck --check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck --check-prefix=WAVE32 %s
 
 ---
 name: copy_s32_vgpr_to_vgpr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-undef.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-undef.mir
index 6b99552f689ec..733d1342ff186 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-undef.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-undef.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ctlz_zero_undef_s32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
index 25ce329015695..1034c3c180468 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ctpop_s32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-undef.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-undef.mir
index 65d9d35efdb1f..8dfcefbcd32df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-undef.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-undef.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: cttz_zero_undef_s32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
index ec59f40eff107..bd699956500ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -o - | FileCheck %s
+# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass=regbankselect %s -o - | FileCheck %s
 
 # Check the default mappings for various instructions.
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
index 857947e28b898..ed1ca320943de 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE64 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE64 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE32 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE32 %s
 
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
index 65eef44294f4d..de02b426776ef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE64 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE64 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE32 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=WAVE32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=WAVE32 %s
 
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
index a71e6cfbe3a7b..85cb851b6bc3d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: extract_lo32_i64_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fabs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fabs.mir
index a1b2eac23f949..55329b008446e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fabs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fabs.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fabs_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
index 2807f97187200..4fba30325f98b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fadd_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
index a03086fbcc145..fac4fa5580b49 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: fcanonicalize_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir
index afa3fb13336f7..593b1c6a2ebfa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fceil_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
index 5d1f074d4c663..0cb6727f6fa05 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=GCN,GFX803 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck --check-prefixes=GCN,GFX803 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=GCN,GFX1150 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck --check-prefixes=GCN,GFX1150 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=GCN,GFX803 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck --check-prefixes=GCN,GFX803 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=GCN,GFX1150 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck --check-prefixes=GCN,GFX1150 %s
 
 ---
 name: fcmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
index 687b9190a8ba0..6b14849556faf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fexp2_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir
index ea07105d21cfb..65b205f66d5f9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: flog2_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fma.mir
index eb9609446598d..d63fc07ada772 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fma.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fma_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fmul.mir
index fcc3782d8b3f6..5766c05426b2d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fmul.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fmul_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fneg.mir
index 54869fec86e84..214dc50c9950a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fneg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fneg_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir
index 8130b7e0c3fc8..df88b99d2ab9b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fpext_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptosi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptosi.mir
index 720c29455060f..c690f8439098f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptosi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptosi.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fptosi_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptoui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptoui.mir
index 2b370490d2419..17e656340f780 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptoui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptoui.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fptoui_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir
index 44808e3c36496..53147d8435ea7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fptrunc_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
index 76994c5cccf5f..2b60dcd9dcabe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: test_frame_index_p5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
index d87bc1f01bdb8..24bc62c4a7253 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=amdgpu-regbankselect %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=amdgpu-regbankselect %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=regbankselect %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=regbankselect %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -o - | FileCheck %s
 
 ---
 name: test_freeze_s1_vgpr_to_vgpr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir
index 6bd0bae229edb..b1a55fe7bc42f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: fshr_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
index 22fc88b519e7e..17b999adf7e56 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fsqrt_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsub.mir
index 39f249abea06e..33b39180d22ea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsub.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: fsub_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
index 2247a9292cf51..b9786893fbac2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX7 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX7 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
 
 ---
 name: icmp_eq_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
index 0da0dcd2def02..1b6689d95a904 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy  -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8  %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8  %s
 
 ---
 name: icmp_eq_s16_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
index 6c82c4a33e33d..8a26fbeb1d11c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
@@ -1,5 +1,5 @@
-# RUN: not --crash llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect  -regbankselect-fast  %s -o /dev/null 2>&1 | FileCheck %s
-# RUN: not --crash llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect  -regbankselect-greedy  %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgcn -run-pass=regbankselect  -regbankselect-fast  %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgcn -run-pass=regbankselect  -regbankselect-greedy  %s -o /dev/null 2>&1 | FileCheck %s
 
 # Check behavior for illegal copies.
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-implicit-def.mir
index 3b620d849c193..ee8dbf1adb7b4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-implicit-def.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefixes=CHECK,FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefixes=CHECK,GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefixes=CHECK,FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefixes=CHECK,GREEDY %s
 
 ---
 name: test_implicit_def_s32_vgpr_use
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert-vector-elt.mir
index b81e9835d4acf..97ca93ad33418 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert-vector-elt.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: insert_vector_elt_v4i32_s_s_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert.mir
index e157a0956cd3d..55f6fa8d3e0ed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-insert.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: insert_lo32_i64_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir
index eb88c37f94c2b..eca10bf2effbf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: intrinsic_trunc_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
index 4eb8cc7d8d978..053aede615f86 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: inttoptr_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
index 74af51f0c1676..b66560710e37e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s -check-prefixes=GCN,GFX7
-# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s -check-prefixes=GCN,GFX7
-# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - | FileCheck %s -check-prefixes=GCN,GFX12
+# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s -check-prefixes=GCN,GFX7
+# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s -check-prefixes=GCN,GFX7
+# RUN: llc -amdgpu-global-isel-new-legality -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s -check-prefixes=GCN,GFX12
 
 --- |
   define amdgpu_kernel void @load_global_v8i32_non_uniform(ptr addrspace(1) %in) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
index 04ac2440f7a5f..c5024924a4d32 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ---
 name: lshr_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mad_64_32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mad_64_32.mir
index 8a77207153ae1..a8dad974ca9c1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mad_64_32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mad_64_32.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX8 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX9MI %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX10 %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX8 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX9MI %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=CHECK,GFX10 %s
 
 ---
 name: mad_u64_u32_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-merge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-merge-values.mir
index 1f802777e9ab2..e22df46b505a5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-merge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-merge-values.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: merge_s64_s32_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
new file mode 100644
index 0000000000000..e840c3f1b86ab
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
@@ -0,0 +1,858 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=none %s -verify-machineinstrs -o - | FileCheck %s
+
+---
+name: uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: uniform_in_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_FPTOUI %0(s32)
+    %6:_(s32) = G_ADD %5, %1
+    G_STORE %6(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: back_to_back_uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: back_to_back_uniform_in_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD]](s32)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FADD %0, %1
+    %7:_(s32) = G_FPTOUI %6(s32)
+    %8:_(s32) = G_ADD %7, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_uniform
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: buffer_load_uniform
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[C1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $sgpr4
+    %6:_(s32) = COPY $vgpr0
+    %7:_(s32) = COPY $vgpr1
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_divergent
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: buffer_load_divergent
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[C1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr0
+    %6:_(s32) = COPY $vgpr1
+    %7:_(s32) = COPY $vgpr2
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_and_i64
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: vgpr_and_i64
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[MV]], [[MV1]]
+    ; CHECK-NEXT: G_STORE [[AND]](s64), [[MV2]](p1) :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s64) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s64) = G_AND %2, %5
+    G_STORE %9(s64), %8(p1) :: (store (s64), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: abs_sgpr_i16
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: abs_sgpr_i16
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(s16) = G_ABS [[TRUNC]]
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ABS]](s16)
+    ; CHECK-NEXT: G_STORE [[ANYEXT]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s16) = G_TRUNC %0(s32)
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s16) = G_ABS %1
+    %6:_(s32) = G_ANYEXT %5(s16)
+    G_STORE %6(s32), %4(p1) :: (store (s16), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: uniform_i1_phi
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: uniform_i1_phi
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; CHECK-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.2
+  ; CHECK-NEXT:   G_BR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.0, [[ICMP2]](s1), %bb.1
+  ; CHECK-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[PHI]](s1)
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C3]]
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $sgpr0
+    %4:_(s32) = COPY $sgpr1
+    %5:_(s32) = G_CONSTANT i32 6
+    %6:_(s1) = G_ICMP intpred(uge), %3(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(s1) = G_ICMP intpred(ne), %4(s32), %7
+    G_BRCOND %8(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %9:_(s32) = G_CONSTANT i32 1
+    %10:_(s1) = G_ICMP intpred(ult), %3(s32), %9
+
+  bb.2:
+    %11:_(s1) = G_PHI %6(s1), %bb.0, %10(s1), %bb.1
+    %12:_(s32) = G_SEXT %11(s1)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s32) = G_ADD %12, %13
+    G_STORE %14(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vcc_to_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: vcc_to_scc
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[FCMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FCONSTANT float 0.000000e+00
+    %7:_(s1) = G_FCMP floatpred(oeq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: scc_to_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: scc_to_vcc
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:_(s1) = G_ICMP intpred(eq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_to_vcc_trunc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: vgpr_to_vcc_trunc
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[TRUNC]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s1) = G_TRUNC %0(s32)
+    %7:_(s32) = G_SELECT %6(s1), %1, %2
+    G_STORE %7(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: zext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: zext
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK-NEXT: G_STORE [[ZEXT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_ZEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: sext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: sext
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK-NEXT: G_STORE [[SEXT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_SEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: and_i1_vcc
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: and_i1_scc
+    ; CHECK: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_phi_with_uniform_inputs
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: divergent_phi_with_uniform_inputs
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; CHECK-NEXT:   G_STORE [[PHI]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:sreg_32_xm0_xexec(s32) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %7:_(s32) = G_CONSTANT i32 1
+
+  bb.2:
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %6(s32)
+    G_STORE %8(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_because_of_temporal_divergent_use
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: divergent_because_of_temporal_divergent_use
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C2]]
+  ; CHECK-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[ADD]](s32)
+  ; CHECK-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; CHECK-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.1
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:_(s32) = G_MUL [[PHI2]], [[C3]]
+  ; CHECK-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 -1
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %10
+    %11:_(s32) = G_UITOFP %9(s32)
+    %12:_(s1) = G_FCMP floatpred(ogt), %11(s32), %0
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %13:_(s32) = G_PHI %9(s32), %bb.1
+    %14:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
+    %15:_(s32) = G_CONSTANT i32 10
+    %16:_(s32) = G_MUL %13, %15
+    G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_2breaks
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: loop_with_2breaks
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; CHECK-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C5]]
+  ; CHECK-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1)
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+  ; CHECK-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
+  ; CHECK-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY11]](s1), [[PHI1]](s32)
+  ; CHECK-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.6
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C7]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD2]], [[C8]]
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C8]]
+  ; CHECK-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; CHECK-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[C9]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY9]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
+  ; CHECK-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+  ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY13]](s1)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; CHECK-NEXT:   G_BR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6:
+  ; CHECK-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
+    %11:sreg_32(s1) = IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %12:sreg_32(s1) = PHI %11(s1), %bb.0, %13(s1), %bb.3
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %10(s32), %bb.0
+    %16:_(s32) = G_PHI %10(s32), %bb.0, %17(s32), %bb.3
+    %18:sreg_32(s1) = COPY %12(s1)
+    %19:_(s64) = G_SEXT %16(s32)
+    %20:_(s32) = G_CONSTANT i32 2
+    %21:_(s64) = G_SHL %19, %20(s32)
+    %22:_(p1) = G_PTR_ADD %5, %21(s64)
+    %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
+    %24:_(s32) = G_CONSTANT i32 0
+    %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
+    %26:_(s1) = G_CONSTANT i1 true
+    %27:sreg_32(s1) = COPY %26(s1)
+    %28:sreg_32(s1) = S_ANDN2_B32 %18(s1), $exec_lo, implicit-def $scc
+    %29:sreg_32(s1) = S_AND_B32 $exec_lo, %27(s1), implicit-def $scc
+    %30:sreg_32(s1) = S_OR_B32 %28(s1), %29(s1), implicit-def $scc
+    %31:sreg_32(s1) = COPY %30(s1)
+    %32:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+    %33:_(s32) = G_CONSTANT i32 2
+    %34:_(s64) = G_SHL %19, %33(s32)
+    %35:_(p1) = G_PTR_ADD %8, %34(s64)
+    %36:_(s32) = G_LOAD %35(p1) :: (load (s32), addrspace 1)
+    %37:_(s32) = G_CONSTANT i32 0
+    %38:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %36(s32), %37
+    %39:_(s1) = G_CONSTANT i1 true
+    %40:sreg_32(s1) = COPY %39(s1)
+    %41:sreg_32(s1) = COPY %40(s1)
+    %42:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %13:sreg_32(s1) = PHI %30(s1), %bb.1, %43(s1), %bb.5
+    %17:_(s32) = G_PHI %44(s32), %bb.5, %9(s32), %bb.1
+    %45:sreg_32(s1) = COPY %13(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %32(s32)
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %45(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    %46:_(s32) = G_CONSTANT i32 2
+    %47:_(s64) = G_SHL %19, %46(s32)
+    %48:_(p1) = G_PTR_ADD %2, %47(s64)
+    %49:_(s32) = G_LOAD %48(p1) :: (load (s32), addrspace 1)
+    %50:_(s32) = G_CONSTANT i32 1
+    %51:_(s32) = G_ADD %49, %50
+    G_STORE %51(s32), %48(p1) :: (store (s32), addrspace 1)
+    %52:_(s32) = G_ADD %16, %50
+    %53:_(s32) = G_CONSTANT i32 100
+    %54:_(s1) = G_ICMP intpred(ult), %16(s32), %53
+    %55:sreg_32(s1) = COPY %54(s1)
+    %56:sreg_32(s1) = S_ANDN2_B32 %41(s1), $exec_lo, implicit-def $scc
+    %57:sreg_32(s1) = S_AND_B32 $exec_lo, %55(s1), implicit-def $scc
+    %58:sreg_32(s1) = S_OR_B32 %56(s1), %57(s1), implicit-def $scc
+
+  bb.5:
+    successors: %bb.3(0x80000000)
+
+    %59:sreg_32(s1) = PHI %40(s1), %bb.2, %58(s1), %bb.4
+    %44:_(s32) = G_PHI %52(s32), %bb.4, %9(s32), %bb.2
+    %60:sreg_32(s1) = COPY %59(s1)
+    %61:sreg_32(s1) = COPY %60(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %42(s32)
+    %62:sreg_32(s1) = S_ANDN2_B32 %31(s1), $exec_lo, implicit-def $scc
+    %63:sreg_32(s1) = S_AND_B32 $exec_lo, %61(s1), implicit-def $scc
+    %43:sreg_32(s1) = S_OR_B32 %62(s1), %63(s1), implicit-def $scc
+    G_BR %bb.3
+
+  bb.6:
+    %64:_(s32) = G_PHI %15(s32), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %64(s32)
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir
new file mode 100644
index 0000000000000..e840c3f1b86ab
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir
@@ -0,0 +1,858 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=none %s -verify-machineinstrs -o - | FileCheck %s
+
+---
+name: uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: uniform_in_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_FPTOUI %0(s32)
+    %6:_(s32) = G_ADD %5, %1
+    G_STORE %6(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: back_to_back_uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: back_to_back_uniform_in_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD]](s32)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FADD %0, %1
+    %7:_(s32) = G_FPTOUI %6(s32)
+    %8:_(s32) = G_ADD %7, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_uniform
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: buffer_load_uniform
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[C1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $sgpr4
+    %6:_(s32) = COPY $vgpr0
+    %7:_(s32) = COPY $vgpr1
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_divergent
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: buffer_load_divergent
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[C1]]
+    ; CHECK-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr0
+    %6:_(s32) = COPY $vgpr1
+    %7:_(s32) = COPY $vgpr2
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_and_i64
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: vgpr_and_i64
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[MV]], [[MV1]]
+    ; CHECK-NEXT: G_STORE [[AND]](s64), [[MV2]](p1) :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s64) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s64) = G_AND %2, %5
+    G_STORE %9(s64), %8(p1) :: (store (s64), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: abs_sgpr_i16
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: abs_sgpr_i16
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(s16) = G_ABS [[TRUNC]]
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ABS]](s16)
+    ; CHECK-NEXT: G_STORE [[ANYEXT]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s16) = G_TRUNC %0(s32)
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s16) = G_ABS %1
+    %6:_(s32) = G_ANYEXT %5(s16)
+    G_STORE %6(s32), %4(p1) :: (store (s16), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: uniform_i1_phi
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: uniform_i1_phi
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; CHECK-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.2
+  ; CHECK-NEXT:   G_BR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.0, [[ICMP2]](s1), %bb.1
+  ; CHECK-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[PHI]](s1)
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C3]]
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $sgpr0
+    %4:_(s32) = COPY $sgpr1
+    %5:_(s32) = G_CONSTANT i32 6
+    %6:_(s1) = G_ICMP intpred(uge), %3(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(s1) = G_ICMP intpred(ne), %4(s32), %7
+    G_BRCOND %8(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %9:_(s32) = G_CONSTANT i32 1
+    %10:_(s1) = G_ICMP intpred(ult), %3(s32), %9
+
+  bb.2:
+    %11:_(s1) = G_PHI %6(s1), %bb.0, %10(s1), %bb.1
+    %12:_(s32) = G_SEXT %11(s1)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s32) = G_ADD %12, %13
+    G_STORE %14(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vcc_to_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: vcc_to_scc
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[FCMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FCONSTANT float 0.000000e+00
+    %7:_(s1) = G_FCMP floatpred(oeq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: scc_to_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: scc_to_vcc
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:_(s1) = G_ICMP intpred(eq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_to_vcc_trunc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: vgpr_to_vcc_trunc
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[TRUNC]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s1) = G_TRUNC %0(s32)
+    %7:_(s32) = G_SELECT %6(s1), %1, %2
+    G_STORE %7(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: zext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: zext
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK-NEXT: G_STORE [[ZEXT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_ZEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: sext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: sext
+    ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK-NEXT: G_STORE [[SEXT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_SEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: and_i1_vcc
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: and_i1_scc
+    ; CHECK: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_phi_with_uniform_inputs
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: divergent_phi_with_uniform_inputs
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; CHECK-NEXT:   G_STORE [[PHI]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:sreg_32_xm0_xexec(s32) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %7:_(s32) = G_CONSTANT i32 1
+
+  bb.2:
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %6(s32)
+    G_STORE %8(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_because_of_temporal_divergent_use
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: divergent_because_of_temporal_divergent_use
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C2]]
+  ; CHECK-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[ADD]](s32)
+  ; CHECK-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; CHECK-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.1
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:_(s32) = G_MUL [[PHI2]], [[C3]]
+  ; CHECK-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 -1
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %10
+    %11:_(s32) = G_UITOFP %9(s32)
+    %12:_(s1) = G_FCMP floatpred(ogt), %11(s32), %0
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %13:_(s32) = G_PHI %9(s32), %bb.1
+    %14:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
+    %15:_(s32) = G_CONSTANT i32 10
+    %16:_(s32) = G_MUL %13, %15
+    G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_2breaks
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: loop_with_2breaks
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; CHECK-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C5]]
+  ; CHECK-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1)
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+  ; CHECK-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
+  ; CHECK-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY11]](s1), [[PHI1]](s32)
+  ; CHECK-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   G_BR %bb.6
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C7]](s32)
+  ; CHECK-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; CHECK-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD2]], [[C8]]
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C8]]
+  ; CHECK-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; CHECK-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[C9]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY9]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
+  ; CHECK-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+  ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY13]](s1)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; CHECK-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; CHECK-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; CHECK-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; CHECK-NEXT:   G_BR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6:
+  ; CHECK-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
+    %11:sreg_32(s1) = IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %12:sreg_32(s1) = PHI %11(s1), %bb.0, %13(s1), %bb.3
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %10(s32), %bb.0
+    %16:_(s32) = G_PHI %10(s32), %bb.0, %17(s32), %bb.3
+    %18:sreg_32(s1) = COPY %12(s1)
+    %19:_(s64) = G_SEXT %16(s32)
+    %20:_(s32) = G_CONSTANT i32 2
+    %21:_(s64) = G_SHL %19, %20(s32)
+    %22:_(p1) = G_PTR_ADD %5, %21(s64)
+    %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
+    %24:_(s32) = G_CONSTANT i32 0
+    %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
+    %26:_(s1) = G_CONSTANT i1 true
+    %27:sreg_32(s1) = COPY %26(s1)
+    %28:sreg_32(s1) = S_ANDN2_B32 %18(s1), $exec_lo, implicit-def $scc
+    %29:sreg_32(s1) = S_AND_B32 $exec_lo, %27(s1), implicit-def $scc
+    %30:sreg_32(s1) = S_OR_B32 %28(s1), %29(s1), implicit-def $scc
+    %31:sreg_32(s1) = COPY %30(s1)
+    %32:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+    %33:_(s32) = G_CONSTANT i32 2
+    %34:_(s64) = G_SHL %19, %33(s32)
+    %35:_(p1) = G_PTR_ADD %8, %34(s64)
+    %36:_(s32) = G_LOAD %35(p1) :: (load (s32), addrspace 1)
+    %37:_(s32) = G_CONSTANT i32 0
+    %38:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %36(s32), %37
+    %39:_(s1) = G_CONSTANT i1 true
+    %40:sreg_32(s1) = COPY %39(s1)
+    %41:sreg_32(s1) = COPY %40(s1)
+    %42:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %13:sreg_32(s1) = PHI %30(s1), %bb.1, %43(s1), %bb.5
+    %17:_(s32) = G_PHI %44(s32), %bb.5, %9(s32), %bb.1
+    %45:sreg_32(s1) = COPY %13(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %32(s32)
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %45(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    %46:_(s32) = G_CONSTANT i32 2
+    %47:_(s64) = G_SHL %19, %46(s32)
+    %48:_(p1) = G_PTR_ADD %2, %47(s64)
+    %49:_(s32) = G_LOAD %48(p1) :: (load (s32), addrspace 1)
+    %50:_(s32) = G_CONSTANT i32 1
+    %51:_(s32) = G_ADD %49, %50
+    G_STORE %51(s32), %48(p1) :: (store (s32), addrspace 1)
+    %52:_(s32) = G_ADD %16, %50
+    %53:_(s32) = G_CONSTANT i32 100
+    %54:_(s1) = G_ICMP intpred(ult), %16(s32), %53
+    %55:sreg_32(s1) = COPY %54(s1)
+    %56:sreg_32(s1) = S_ANDN2_B32 %41(s1), $exec_lo, implicit-def $scc
+    %57:sreg_32(s1) = S_AND_B32 $exec_lo, %55(s1), implicit-def $scc
+    %58:sreg_32(s1) = S_OR_B32 %56(s1), %57(s1), implicit-def $scc
+
+  bb.5:
+    successors: %bb.3(0x80000000)
+
+    %59:sreg_32(s1) = PHI %40(s1), %bb.2, %58(s1), %bb.4
+    %44:_(s32) = G_PHI %52(s32), %bb.4, %9(s32), %bb.2
+    %60:sreg_32(s1) = COPY %59(s1)
+    %61:sreg_32(s1) = COPY %60(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %42(s32)
+    %62:sreg_32(s1) = S_ANDN2_B32 %31(s1), $exec_lo, implicit-def $scc
+    %63:sreg_32(s1) = S_AND_B32 $exec_lo, %61(s1), implicit-def $scc
+    %43:sreg_32(s1) = S_OR_B32 %62(s1), %63(s1), implicit-def $scc
+    G_BR %bb.3
+
+  bb.6:
+    %64:_(s32) = G_PHI %15(s32), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %64(s32)
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.ll
new file mode 100644
index 0000000000000..0b4eb458b254f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=OLD_RBS_GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=OLD_RBS_GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=NEW_RBS_GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=NEW_RBS_GFX12 %s
+
+define amdgpu_ps void @salu_float(float inreg %a, float inreg %b, i32 inreg %c, ptr addrspace(1) %ptr) {
+; OLD_RBS_GFX10-LABEL: salu_float:
+; OLD_RBS_GFX10:       ; %bb.0:
+; OLD_RBS_GFX10-NEXT:    v_add_f32_e64 v2, s0, s1
+; OLD_RBS_GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; OLD_RBS_GFX10-NEXT:    v_add_nc_u32_e32 v2, s2, v2
+; OLD_RBS_GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS_GFX10-NEXT:    s_endpgm
+;
+; OLD_RBS_GFX12-LABEL: salu_float:
+; OLD_RBS_GFX12:       ; %bb.0:
+; OLD_RBS_GFX12-NEXT:    s_add_f32 s0, s0, s1
+; OLD_RBS_GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; OLD_RBS_GFX12-NEXT:    s_cvt_u32_f32 s0, s0
+; OLD_RBS_GFX12-NEXT:    s_add_co_i32 s0, s0, s2
+; OLD_RBS_GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; OLD_RBS_GFX12-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS_GFX12-NEXT:    global_store_b32 v[0:1], v2, off
+; OLD_RBS_GFX12-NEXT:    s_endpgm
+;
+; NEW_RBS_GFX10-LABEL: salu_float:
+; NEW_RBS_GFX10:       ; %bb.0:
+; NEW_RBS_GFX10-NEXT:    v_add_f32_e64 v2, s0, s1
+; NEW_RBS_GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; NEW_RBS_GFX10-NEXT:    v_add_nc_u32_e32 v2, s2, v2
+; NEW_RBS_GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS_GFX10-NEXT:    s_endpgm
+;
+; NEW_RBS_GFX12-LABEL: salu_float:
+; NEW_RBS_GFX12:       ; %bb.0:
+; NEW_RBS_GFX12-NEXT:    s_add_f32 s0, s0, s1
+; NEW_RBS_GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; NEW_RBS_GFX12-NEXT:    s_cvt_u32_f32 s0, s0
+; NEW_RBS_GFX12-NEXT:    s_add_co_i32 s0, s0, s2
+; NEW_RBS_GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NEW_RBS_GFX12-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS_GFX12-NEXT:    global_store_b32 v[0:1], v2, off
+; NEW_RBS_GFX12-NEXT:    s_endpgm
+  %add = fadd float %a, %b
+  %add.i32 = fptoui float %add to i32
+  %res = add i32 %add.i32, %c
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.mir
new file mode 100644
index 0000000000000..98a8f4f04e49d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-salu-float.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -o - | FileCheck %s -check-prefixes=OLD_RBS_GFX10
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=regbankselect %s -o - | FileCheck %s -check-prefixes=OLD_RBS_GFX12
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -o - | FileCheck %s -check-prefixes=NEW_RBS_GFX10
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=regbankselect %s -o - | FileCheck %s -check-prefixes=NEW_RBS_GFX12
+
+---
+name: salu_float
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; OLD_RBS_GFX10-LABEL: name: salu_float
+    ; OLD_RBS_GFX10: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; OLD_RBS_GFX10-NEXT: {{  $}}
+    ; OLD_RBS_GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS_GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS_GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS_GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS_GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS_GFX10-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS_GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; OLD_RBS_GFX10-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; OLD_RBS_GFX10-NEXT: [[FADD:%[0-9]+]]:vgpr(s32) = G_FADD [[COPY5]], [[COPY6]]
+    ; OLD_RBS_GFX10-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; OLD_RBS_GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; OLD_RBS_GFX10-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY7]]
+    ; OLD_RBS_GFX10-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS_GFX10-NEXT: S_ENDPGM 0
+    ;
+    ; OLD_RBS_GFX12-LABEL: name: salu_float
+    ; OLD_RBS_GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; OLD_RBS_GFX12-NEXT: {{  $}}
+    ; OLD_RBS_GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS_GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS_GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS_GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS_GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS_GFX12-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS_GFX12-NEXT: [[FADD:%[0-9]+]]:sgpr(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; OLD_RBS_GFX12-NEXT: [[FPTOUI:%[0-9]+]]:sgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; OLD_RBS_GFX12-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[FPTOUI]], [[COPY2]]
+    ; OLD_RBS_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+    ; OLD_RBS_GFX12-NEXT: G_STORE [[COPY5]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS_GFX12-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS_GFX10-LABEL: name: salu_float
+    ; NEW_RBS_GFX10: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; NEW_RBS_GFX10-NEXT: {{  $}}
+    ; NEW_RBS_GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS_GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS_GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS_GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS_GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS_GFX10-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS_GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; NEW_RBS_GFX10-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; NEW_RBS_GFX10-NEXT: [[FADD:%[0-9]+]]:vgpr(s32) = G_FADD [[COPY5]], [[COPY6]]
+    ; NEW_RBS_GFX10-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; NEW_RBS_GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; NEW_RBS_GFX10-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY7]]
+    ; NEW_RBS_GFX10-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS_GFX10-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS_GFX12-LABEL: name: salu_float
+    ; NEW_RBS_GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; NEW_RBS_GFX12-NEXT: {{  $}}
+    ; NEW_RBS_GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS_GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS_GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS_GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS_GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS_GFX12-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS_GFX12-NEXT: [[FADD:%[0-9]+]]:sgpr(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; NEW_RBS_GFX12-NEXT: [[FPTOUI:%[0-9]+]]:sgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; NEW_RBS_GFX12-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[FPTOUI]], [[COPY2]]
+    ; NEW_RBS_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+    ; NEW_RBS_GFX12-NEXT: G_STORE [[COPY5]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS_GFX12-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FADD %0, %1
+    %7:_(s32) = G_FPTOUI %6(s32)
+    %8:_(s32) = G_ADD %7, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
new file mode 100644
index 0000000000000..287a8ab0e52f5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
@@ -0,0 +1,635 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=OLD_RBS %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=NEW_RBS %s
+
+; if instruction is uniform and there is available instruction, select SALU instruction
+define amdgpu_ps void @uniform_in_vgpr(float inreg %a, i32 inreg %b, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: uniform_in_vgpr:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_cvt_u32_f32_e32 v2, s0
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v2, s1, v2
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: uniform_in_vgpr:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_cvt_u32_f32_e32 v2, s0
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v2, s1, v2
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %a.i32 = fptoui float %a to i32
+  %res = add i32 %a.i32, %b
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+; copy sgpr to vgpr + readfirstlane vgpr to sgpr combine from rb-legalize
+define amdgpu_ps void @back_to_back_uniform_in_vgpr(float inreg %a, float inreg %b, i32 inreg %c, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: back_to_back_uniform_in_vgpr:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_add_f32_e64 v2, s0, s1
+; OLD_RBS-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v2, s2, v2
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: back_to_back_uniform_in_vgpr:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_add_f32_e64 v2, s0, s1
+; NEW_RBS-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v2, s2, v2
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %add = fadd float %a, %b
+  %add.i32 = fptoui float %add to i32
+  %res = add i32 %add.i32, %c
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+; fast rules for vector instructions
+define amdgpu_cs void @buffer_load_uniform(<4 x i32> inreg %rsrc, i32 inreg %voffset, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: buffer_load_uniform:
+; OLD_RBS:       ; %bb.0: ; %.entry
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s4
+; OLD_RBS-NEXT:    buffer_load_dwordx4 v[2:5], v2, s[0:3], 0 offen
+; OLD_RBS-NEXT:    s_waitcnt vmcnt(0)
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v2, 1, v3
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: buffer_load_uniform:
+; NEW_RBS:       ; %bb.0: ; %.entry
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s4
+; NEW_RBS-NEXT:    buffer_load_dwordx4 v[2:5], v2, s[0:3], 0 offen
+; NEW_RBS-NEXT:    s_waitcnt vmcnt(0)
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v2, 1, v3
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+.entry:
+  %vec = call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
+  %el1 = extractelement <4 x i32> %vec, i64 1
+  %res = add i32 %el1, 1
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_cs void @buffer_load_divergent(<4 x i32> inreg %rsrc, i32 %voffset, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: buffer_load_divergent:
+; OLD_RBS:       ; %bb.0: ; %.entry
+; OLD_RBS-NEXT:    buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; OLD_RBS-NEXT:    s_waitcnt vmcnt(0)
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v0, 1, v4
+; OLD_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: buffer_load_divergent:
+; NEW_RBS:       ; %bb.0: ; %.entry
+; NEW_RBS-NEXT:    buffer_load_dwordx4 v[3:6], v0, s[0:3], 0 offen
+; NEW_RBS-NEXT:    s_waitcnt vmcnt(0)
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v0, 1, v4
+; NEW_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+.entry:
+  %vec = call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
+  %el1 = extractelement <4 x i32> %vec, i64 1
+  %res = add i32 %el1, 1
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+;lowering in rb-legalize (sgpr S64 is legal, vgpr has to be split to S32)
+define amdgpu_ps void @vgpr_and_i64(i64 %a, i64 %b, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: vgpr_and_i64:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_and_b32_e32 v0, v0, v2
+; OLD_RBS-NEXT:    v_and_b32_e32 v1, v1, v3
+; OLD_RBS-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: vgpr_and_i64:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_and_b32_e32 v0, v0, v2
+; NEW_RBS-NEXT:    v_and_b32_e32 v1, v1, v3
+; NEW_RBS-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; NEW_RBS-NEXT:    s_endpgm
+  %res = and i64 %a, %b
+  store i64 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+; It is up to user instruction to deal with potential truncated bits in reg.
+; Here G_ABS needs to sign extend S16 in reg to S32 and then do S32 G_ABS.
+define amdgpu_ps void @abs_sgpr_i16(i16 inreg %arg, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: abs_sgpr_i16:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    s_sext_i32_i16 s0, s0
+; OLD_RBS-NEXT:    s_abs_i32 s0, s0
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS-NEXT:    global_store_short v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: abs_sgpr_i16:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    s_sext_i32_i16 s0, s0
+; NEW_RBS-NEXT:    s_abs_i32 s0, s0
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS-NEXT:    global_store_short v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %res = call i16 @llvm.abs.i16(i16 %arg, i1 false)
+  store i16 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_ps void @uniform_i1_phi(ptr addrspace(1) %out, i32 inreg %tid, i32 inreg %cond) {
+; OLD_RBS-LABEL: uniform_i1_phi:
+; OLD_RBS:       ; %bb.0: ; %A
+; OLD_RBS-NEXT:    s_cmp_ge_u32 s0, 6
+; OLD_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; OLD_RBS-NEXT:    s_cmp_lg_u32 s1, 0
+; OLD_RBS-NEXT:    s_cbranch_scc1 .LBB6_2
+; OLD_RBS-NEXT:  ; %bb.1: ; %B
+; OLD_RBS-NEXT:    s_cmp_lt_u32 s0, 1
+; OLD_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; OLD_RBS-NEXT:  .LBB6_2: ; %exit
+; OLD_RBS-NEXT:    s_bfe_i32 s0, s2, 0x10000
+; OLD_RBS-NEXT:    s_add_i32 s0, s0, 2
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: uniform_i1_phi:
+; NEW_RBS:       ; %bb.0: ; %A
+; NEW_RBS-NEXT:    s_cmp_ge_u32 s0, 6
+; NEW_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; NEW_RBS-NEXT:    s_cmp_lg_u32 s1, 0
+; NEW_RBS-NEXT:    s_cbranch_scc1 .LBB6_2
+; NEW_RBS-NEXT:  ; %bb.1: ; %B
+; NEW_RBS-NEXT:    s_cmp_lt_u32 s0, 1
+; NEW_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; NEW_RBS-NEXT:  .LBB6_2: ; %exit
+; NEW_RBS-NEXT:    s_bfe_i32 s0, s2, 0x10000
+; NEW_RBS-NEXT:    s_add_i32 s0, s0, 2
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+A:
+  %val_A = icmp uge i32 %tid, 6
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %B, label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 1
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %sel = select i1 %phi, i32 1, i32 2
+  store i32 %sel, ptr addrspace(1) %out
+  ret void
+}
+
+; this is kind of i1 readfirstlane
+; uniform i1 result on instruction that is only available on VALU
+define amdgpu_ps void @vcc_to_scc(float inreg %a, i32 inreg %b, i32 inreg %c, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: vcc_to_scc:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s2
+; OLD_RBS-NEXT:    v_cmp_eq_f32_e64 s0, s0, 0
+; OLD_RBS-NEXT:    v_cndmask_b32_e64 v2, v2, s1, s0
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: vcc_to_scc:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s2
+; NEW_RBS-NEXT:    v_cmp_eq_f32_e64 s0, s0, 0
+; NEW_RBS-NEXT:    v_cndmask_b32_e64 v2, v2, s1, s0
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %vcc_to_scc = fcmp oeq float %a, 0.0
+  %select = select i1 %vcc_to_scc, i32 %b, i32 %c
+  store i32 %select, ptr addrspace(1) %ptr
+  ret void
+}
+
+; combiner in rb-legalize recognizes sgpr S1 to vcc copy
+define amdgpu_ps void @scc_to_vcc(i32 inreg %a, i32 %b, i32 %c, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: scc_to_vcc:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    s_cmp_eq_u32 s0, 0
+; OLD_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; OLD_RBS-NEXT:    s_and_b32 s0, 1, s0
+; OLD_RBS-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; OLD_RBS-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; OLD_RBS-NEXT:    global_store_dword v[2:3], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: scc_to_vcc:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    s_cmp_eq_u32 s0, 0
+; NEW_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; NEW_RBS-NEXT:    s_and_b32 s0, 1, s0
+; NEW_RBS-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; NEW_RBS-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; NEW_RBS-NEXT:    global_store_dword v[2:3], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+  %scc_to_vcc = icmp eq i32 %a, 0
+  %select = select i1 %scc_to_vcc, i32 %b, i32 %c
+  store i32 %select, ptr addrspace(1) %ptr
+  ret void
+}
+
+; this is only G_TRUNC that is not no-op in global-isel for AMDGPU
+define amdgpu_ps void @vgpr_to_vcc_trunc(i32 %a, i32 %b, i32 %c, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: vgpr_to_vcc_trunc:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_and_b32_e32 v0, 1, v0
+; OLD_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; OLD_RBS-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; OLD_RBS-NEXT:    global_store_dword v[3:4], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: vgpr_to_vcc_trunc:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_and_b32_e32 v0, 1, v0
+; NEW_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; NEW_RBS-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; NEW_RBS-NEXT:    global_store_dword v[3:4], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+  %vcc = trunc i32 %a to i1
+  %select = select i1 %vcc, i32 %b, i32 %c
+  store i32 %select, ptr addrspace(1) %ptr
+  ret void
+}
+
+; i1 input to zext and sext is something that survived legalizer (not trunc)
+; lower to select
+define amdgpu_ps void @zext(i32 inreg %a, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: zext:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    s_cmp_eq_u32 s0, 10
+; OLD_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: zext:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    s_cmp_eq_u32 s0, 10
+; NEW_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %bool = icmp eq i32 %a, 10
+  %zext = zext i1 %bool to i32
+  store i32 %zext, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_ps void @sext(i32 inreg %a, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: sext:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    s_cmp_eq_u32 s0, 10
+; OLD_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; OLD_RBS-NEXT:    s_bfe_i32 s0, s0, 0x10000
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: sext:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    s_cmp_eq_u32 s0, 10
+; NEW_RBS-NEXT:    s_cselect_b32 s0, 1, 0
+; NEW_RBS-NEXT:    s_bfe_i32 s0, s0, 0x10000
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %bool = icmp eq i32 %a, 10
+  %sext = sext i1 %bool to i32
+  store i32 %sext, ptr addrspace(1) %ptr
+  ret void
+}
+
+; divergent i1 bitwise, i1 vcc.
+; inst selected into s_and_b32 on wave32 or s_and_b64 on wave64.
+define amdgpu_ps void @and_i1_vcc(i32 %a, i32 %b, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: and_i1_vcc:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    v_cmp_le_u32_e32 vcc_lo, 10, v0
+; OLD_RBS-NEXT:    v_cmp_le_u32_e64 s0, 20, v1
+; OLD_RBS-NEXT:    s_and_b32 vcc_lo, vcc_lo, s0
+; OLD_RBS-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; OLD_RBS-NEXT:    global_store_dword v[2:3], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: and_i1_vcc:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    v_cmp_le_u32_e32 vcc_lo, 10, v0
+; NEW_RBS-NEXT:    v_cmp_le_u32_e64 s0, 20, v1
+; NEW_RBS-NEXT:    s_and_b32 vcc_lo, vcc_lo, s0
+; NEW_RBS-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; NEW_RBS-NEXT:    global_store_dword v[2:3], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+  %cmp_a = icmp uge i32 %a, 10
+  %cmp_b = icmp uge i32 %b, 20
+  %cc = and i1 %cmp_a, %cmp_b
+  %res = select i1 %cc, i32 %a, i32 %b
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+; uniform i1 bitwise, i32 sgpr. inst selected into s_and_b32.
+define amdgpu_ps void @and_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptr) {
+; OLD_RBS-LABEL: and_i1_scc:
+; OLD_RBS:       ; %bb.0:
+; OLD_RBS-NEXT:    s_cmp_ge_u32 s0, 10
+; OLD_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; OLD_RBS-NEXT:    s_cmp_ge_u32 s1, 20
+; OLD_RBS-NEXT:    s_cselect_b32 s3, 1, 0
+; OLD_RBS-NEXT:    s_and_b32 s2, s2, s3
+; OLD_RBS-NEXT:    s_and_b32 s2, s2, 1
+; OLD_RBS-NEXT:    s_cmp_lg_u32 s2, 0
+; OLD_RBS-NEXT:    s_cselect_b32 s0, s0, s1
+; OLD_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; OLD_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: and_i1_scc:
+; NEW_RBS:       ; %bb.0:
+; NEW_RBS-NEXT:    s_cmp_ge_u32 s0, 10
+; NEW_RBS-NEXT:    s_cselect_b32 s2, 1, 0
+; NEW_RBS-NEXT:    s_cmp_ge_u32 s1, 20
+; NEW_RBS-NEXT:    s_cselect_b32 s3, 1, 0
+; NEW_RBS-NEXT:    s_and_b32 s2, s2, s3
+; NEW_RBS-NEXT:    s_and_b32 s2, s2, 1
+; NEW_RBS-NEXT:    s_cmp_lg_u32 s2, 0
+; NEW_RBS-NEXT:    s_cselect_b32 s0, s0, s1
+; NEW_RBS-NEXT:    v_mov_b32_e32 v2, s0
+; NEW_RBS-NEXT:    global_store_dword v[0:1], v2, off
+; NEW_RBS-NEXT:    s_endpgm
+  %cmp_a = icmp uge i32 %a, 10
+  %cmp_b = icmp uge i32 %b, 20
+  %cc = and i1 %cmp_a, %cmp_b
+  %res = select i1 %cc, i32 %a, i32 %b
+  store i32 %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+; old RBS selects sgpr phi because it had sgpr inputs.
+define amdgpu_ps void @divergent_phi_with_uniform_inputs(i32 %a, ptr addrspace(1) %out) {
+; OLD_RBS-LABEL: divergent_phi_with_uniform_inputs:
+; OLD_RBS:       ; %bb.0: ; %A
+; OLD_RBS-NEXT:    s_mov_b32 s0, 0
+; OLD_RBS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; OLD_RBS-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; OLD_RBS-NEXT:  ; %bb.1: ; %B
+; OLD_RBS-NEXT:    s_mov_b32 s0, 1
+; OLD_RBS-NEXT:  ; %bb.2: ; %exit
+; OLD_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; OLD_RBS-NEXT:    v_mov_b32_e32 v0, s0
+; OLD_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: divergent_phi_with_uniform_inputs:
+; NEW_RBS:       ; %bb.0: ; %A
+; NEW_RBS-NEXT:    s_mov_b32 s0, 0
+; NEW_RBS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; NEW_RBS-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; NEW_RBS-NEXT:  ; %bb.1: ; %B
+; NEW_RBS-NEXT:    s_mov_b32 s0, 1
+; NEW_RBS-NEXT:  ; %bb.2: ; %exit
+; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; NEW_RBS-NEXT:    v_mov_b32_e32 v0, s0
+; NEW_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+A:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %B, label %exit
+
+B:
+  br label %exit
+
+exit:
+  %phi = phi i32 [ 0, %A ], [ 1, %B ]
+  store i32 %phi, ptr addrspace(1) %out
+  ret void
+}
+
+; old RBS assigned vgpr to uniform phi (because one input had undetermined bank)
+; and it propagated to mul, which was not wrong.
+; new RBS assigns vgpr to destination of mul even though both inputs are sgpr.
+; TODO: implement temporal divergence lowering
+define amdgpu_ps void @divergent_because_of_temporal_divergent_use(float %val, ptr addrspace(1) %addr) {
+; OLD_RBS-LABEL: divergent_because_of_temporal_divergent_use:
+; OLD_RBS:       ; %bb.0: ; %entry
+; OLD_RBS-NEXT:    s_mov_b32 s0, -1
+; OLD_RBS-NEXT:    v_mov_b32_e32 v3, s0
+; OLD_RBS-NEXT:    s_mov_b32 s0, 0
+; OLD_RBS-NEXT:  .LBB15_1: ; %loop
+; OLD_RBS-NEXT:    ; =>This Inner Loop Header: Depth=1
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v3, 1, v3
+; OLD_RBS-NEXT:    v_cvt_f32_u32_e32 v4, v3
+; OLD_RBS-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; OLD_RBS-NEXT:    s_or_b32 s0, vcc_lo, s0
+; OLD_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; OLD_RBS-NEXT:    s_cbranch_execnz .LBB15_1
+; OLD_RBS-NEXT:  ; %bb.2: ; %exit
+; OLD_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; OLD_RBS-NEXT:    v_mul_lo_u32 v0, v3, 10
+; OLD_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: divergent_because_of_temporal_divergent_use:
+; NEW_RBS:       ; %bb.0: ; %entry
+; NEW_RBS-NEXT:    s_mov_b32 s0, -1
+; NEW_RBS-NEXT:    v_mov_b32_e32 v3, s0
+; NEW_RBS-NEXT:    s_mov_b32 s0, 0
+; NEW_RBS-NEXT:  .LBB15_1: ; %loop
+; NEW_RBS-NEXT:    ; =>This Inner Loop Header: Depth=1
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v3, 1, v3
+; NEW_RBS-NEXT:    v_cvt_f32_u32_e32 v4, v3
+; NEW_RBS-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; NEW_RBS-NEXT:    s_or_b32 s0, vcc_lo, s0
+; NEW_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; NEW_RBS-NEXT:    s_cbranch_execnz .LBB15_1
+; NEW_RBS-NEXT:  ; %bb.2: ; %exit
+; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; NEW_RBS-NEXT:    v_mul_lo_u32 v0, v3, 10
+; NEW_RBS-NEXT:    global_store_dword v[1:2], v0, off
+; NEW_RBS-NEXT:    s_endpgm
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ceilx10 = mul i32 %counter, 10
+  store i32 %ceilx10, ptr addrspace(1) %addr
+  ret void
+}
+
+; Variables that hande counter can be allocated to sgprs.
+; Machine uniformity analysis claims some of those registers are divergent while
+; LLVM-IR uniformity analysis claims corresponding values are uniform.
+; TODO: fix this in Machine uniformity analysis.
+define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; OLD_RBS-LABEL: loop_with_2breaks:
+; OLD_RBS:       ; %bb.0: ; %entry
+; OLD_RBS-NEXT:    s_mov_b32 s0, 0
+; OLD_RBS-NEXT:    ; implicit-def: $sgpr1
+; OLD_RBS-NEXT:    v_mov_b32_e32 v6, s0
+; OLD_RBS-NEXT:    s_branch .LBB16_3
+; OLD_RBS-NEXT:  .LBB16_1: ; %Flow3
+; OLD_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; OLD_RBS-NEXT:    s_waitcnt_depctr 0xffe3
+; OLD_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; OLD_RBS-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; OLD_RBS-NEXT:    s_and_b32 s3, exec_lo, s4
+; OLD_RBS-NEXT:    s_or_b32 s1, s1, s3
+; OLD_RBS-NEXT:  .LBB16_2: ; %Flow
+; OLD_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; OLD_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; OLD_RBS-NEXT:    s_and_b32 s2, exec_lo, s1
+; OLD_RBS-NEXT:    s_or_b32 s0, s2, s0
+; OLD_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; OLD_RBS-NEXT:    s_cbranch_execz .LBB16_6
+; OLD_RBS-NEXT:  .LBB16_3: ; %A
+; OLD_RBS-NEXT:    ; =>This Inner Loop Header: Depth=1
+; OLD_RBS-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; OLD_RBS-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; OLD_RBS-NEXT:    s_and_b32 s2, exec_lo, -1
+; OLD_RBS-NEXT:    s_or_b32 s1, s1, s2
+; OLD_RBS-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
+; OLD_RBS-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
+; OLD_RBS-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
+; OLD_RBS-NEXT:    global_load_dword v9, v[9:10], off
+; OLD_RBS-NEXT:    s_waitcnt vmcnt(0)
+; OLD_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; OLD_RBS-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; OLD_RBS-NEXT:    s_cbranch_execz .LBB16_2
+; OLD_RBS-NEXT:  ; %bb.4: ; %B
+; OLD_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; OLD_RBS-NEXT:    v_add_co_u32 v9, vcc_lo, v4, v7
+; OLD_RBS-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo
+; OLD_RBS-NEXT:    s_mov_b32 s4, -1
+; OLD_RBS-NEXT:    global_load_dword v9, v[9:10], off
+; OLD_RBS-NEXT:    s_waitcnt vmcnt(0)
+; OLD_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; OLD_RBS-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; OLD_RBS-NEXT:    s_cbranch_execz .LBB16_1
+; OLD_RBS-NEXT:  ; %bb.5: ; %loop.body
+; OLD_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; OLD_RBS-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
+; OLD_RBS-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v10, 1, v6
+; OLD_RBS-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
+; OLD_RBS-NEXT:    s_andn2_b32 s4, -1, exec_lo
+; OLD_RBS-NEXT:    global_load_dword v9, v[7:8], off
+; OLD_RBS-NEXT:    v_mov_b32_e32 v6, v10
+; OLD_RBS-NEXT:    s_and_b32 s5, exec_lo, vcc_lo
+; OLD_RBS-NEXT:    s_or_b32 s4, s4, s5
+; OLD_RBS-NEXT:    s_waitcnt vmcnt(0)
+; OLD_RBS-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; OLD_RBS-NEXT:    global_store_dword v[7:8], v9, off
+; OLD_RBS-NEXT:    s_branch .LBB16_1
+; OLD_RBS-NEXT:  .LBB16_6: ; %exit
+; OLD_RBS-NEXT:    s_endpgm
+;
+; NEW_RBS-LABEL: loop_with_2breaks:
+; NEW_RBS:       ; %bb.0: ; %entry
+; NEW_RBS-NEXT:    s_mov_b32 s0, 0
+; NEW_RBS-NEXT:    ; implicit-def: $sgpr1
+; NEW_RBS-NEXT:    v_mov_b32_e32 v6, s0
+; NEW_RBS-NEXT:    s_branch .LBB16_3
+; NEW_RBS-NEXT:  .LBB16_1: ; %Flow3
+; NEW_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; NEW_RBS-NEXT:    s_waitcnt_depctr 0xffe3
+; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; NEW_RBS-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; NEW_RBS-NEXT:    s_and_b32 s3, exec_lo, s4
+; NEW_RBS-NEXT:    s_or_b32 s1, s1, s3
+; NEW_RBS-NEXT:  .LBB16_2: ; %Flow
+; NEW_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; NEW_RBS-NEXT:    s_and_b32 s2, exec_lo, s1
+; NEW_RBS-NEXT:    s_or_b32 s0, s2, s0
+; NEW_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; NEW_RBS-NEXT:    s_cbranch_execz .LBB16_6
+; NEW_RBS-NEXT:  .LBB16_3: ; %A
+; NEW_RBS-NEXT:    ; =>This Inner Loop Header: Depth=1
+; NEW_RBS-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; NEW_RBS-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; NEW_RBS-NEXT:    s_and_b32 s2, exec_lo, -1
+; NEW_RBS-NEXT:    s_or_b32 s1, s1, s2
+; NEW_RBS-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
+; NEW_RBS-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
+; NEW_RBS-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
+; NEW_RBS-NEXT:    global_load_dword v9, v[9:10], off
+; NEW_RBS-NEXT:    s_waitcnt vmcnt(0)
+; NEW_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; NEW_RBS-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; NEW_RBS-NEXT:    s_cbranch_execz .LBB16_2
+; NEW_RBS-NEXT:  ; %bb.4: ; %B
+; NEW_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; NEW_RBS-NEXT:    v_add_co_u32 v9, vcc_lo, v4, v7
+; NEW_RBS-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo
+; NEW_RBS-NEXT:    s_mov_b32 s4, -1
+; NEW_RBS-NEXT:    global_load_dword v9, v[9:10], off
+; NEW_RBS-NEXT:    s_waitcnt vmcnt(0)
+; NEW_RBS-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; NEW_RBS-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; NEW_RBS-NEXT:    s_cbranch_execz .LBB16_1
+; NEW_RBS-NEXT:  ; %bb.5: ; %loop.body
+; NEW_RBS-NEXT:    ; in Loop: Header=BB16_3 Depth=1
+; NEW_RBS-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
+; NEW_RBS-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v10, 1, v6
+; NEW_RBS-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
+; NEW_RBS-NEXT:    s_andn2_b32 s4, -1, exec_lo
+; NEW_RBS-NEXT:    global_load_dword v9, v[7:8], off
+; NEW_RBS-NEXT:    v_mov_b32_e32 v6, v10
+; NEW_RBS-NEXT:    s_and_b32 s5, exec_lo, vcc_lo
+; NEW_RBS-NEXT:    s_or_b32 s4, s4, s5
+; NEW_RBS-NEXT:    s_waitcnt vmcnt(0)
+; NEW_RBS-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; NEW_RBS-NEXT:    global_store_dword v[7:8], v9, off
+; NEW_RBS-NEXT:    s_branch .LBB16_1
+; NEW_RBS-NEXT:  .LBB16_6: ; %exit
+; NEW_RBS-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %exit, label %B
+
+B:
+  %b.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %b, i32 %counter
+  %b.val = load i32, ptr addrspace(1) %b.plus.counter
+  %b.cond = icmp eq i32 %b.val, 0
+  br i1 %b.cond, label %exit, label %loop.body
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
+declare i16 @llvm.abs.i16(i16, i1)
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
new file mode 100644
index 0000000000000..ef3a0a3a67594
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
@@ -0,0 +1,1377 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s -check-prefixes=OLD_RBS
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s -check-prefixes=NEW_RBS
+
+---
+name: uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: uniform_in_vgpr
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; OLD_RBS-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY5]]
+    ; OLD_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: uniform_in_vgpr
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; NEW_RBS-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY5]]
+    ; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_FPTOUI %0(s32)
+    %6:_(s32) = G_ADD %5, %1
+    G_STORE %6(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: back_to_back_uniform_in_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: back_to_back_uniform_in_vgpr
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; OLD_RBS-NEXT: [[FADD:%[0-9]+]]:vgpr(s32) = G_FADD [[COPY5]], [[COPY6]]
+    ; OLD_RBS-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY7]]
+    ; OLD_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: back_to_back_uniform_in_vgpr
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; NEW_RBS-NEXT: [[FADD:%[0-9]+]]:vgpr(s32) = G_FADD [[COPY5]], [[COPY6]]
+    ; NEW_RBS-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s32) = G_FPTOUI [[FADD]](s32)
+    ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[FPTOUI]], [[COPY7]]
+    ; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FADD %0, %1
+    %7:_(s32) = G_FPTOUI %6(s32)
+    %8:_(s32) = G_ADD %7, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_uniform
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: buffer_load_uniform
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; OLD_RBS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; OLD_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; OLD_RBS-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[UV1]], [[COPY9]]
+    ; OLD_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: buffer_load_uniform
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; NEW_RBS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; NEW_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; NEW_RBS-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[UV1]], [[COPY9]]
+    ; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $sgpr4
+    %6:_(s32) = COPY $vgpr0
+    %7:_(s32) = COPY $vgpr1
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: buffer_load_divergent
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+
+    ; OLD_RBS-LABEL: name: buffer_load_divergent
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; OLD_RBS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; OLD_RBS-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; OLD_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; OLD_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[UV1]], [[COPY8]]
+    ; OLD_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: buffer_load_divergent
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; NEW_RBS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; NEW_RBS-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; NEW_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
+    ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[UV1]], [[COPY8]]
+    ; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr0
+    %6:_(s32) = COPY $vgpr1
+    %7:_(s32) = COPY $vgpr2
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(<4 x s32>) = G_AMDGPU_BUFFER_LOAD %4(<4 x s32>), %9(s32), %5, %9, 0, 0, 0 :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+    %11:_(s32) = G_CONSTANT i32 1
+    %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32) = G_UNMERGE_VALUES %10(<4 x s32>)
+    %16:_(s32) = G_ADD %13, %11
+    G_STORE %16(s32), %8(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_and_i64
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; OLD_RBS-LABEL: name: vgpr_and_i64
+    ; OLD_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; OLD_RBS-NEXT: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+    ; OLD_RBS-NEXT: [[MV2:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+    ; OLD_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; OLD_RBS-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV1]](s64)
+    ; OLD_RBS-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; OLD_RBS-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; OLD_RBS-NEXT: [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; OLD_RBS-NEXT: G_STORE [[MV3]](s64), [[MV2]](p1) :: (store (s64), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: vgpr_and_i64
+    ; NEW_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; NEW_RBS-NEXT: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+    ; NEW_RBS-NEXT: [[MV2:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+    ; NEW_RBS-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; NEW_RBS-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV1]](s64)
+    ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; NEW_RBS-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; NEW_RBS-NEXT: [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; NEW_RBS-NEXT: G_STORE [[MV3]](s64), [[MV2]](p1) :: (store (s64), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s64) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s64) = G_AND %2, %5
+    G_STORE %9(s64), %8(p1) :: (store (s64), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: abs_sgpr_i16
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: abs_sgpr_i16
+    ; OLD_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; OLD_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
+    ; OLD_RBS-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT]]
+    ; OLD_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](s32)
+    ; OLD_RBS-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ANYEXT]](s32)
+    ; OLD_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: abs_sgpr_i16
+    ; NEW_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; NEW_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
+    ; NEW_RBS-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT]]
+    ; NEW_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](s32)
+    ; NEW_RBS-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ANYEXT]](s32)
+    ; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s16) = G_TRUNC %0(s32)
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s16) = G_ABS %1
+    %6:_(s32) = G_ANYEXT %5(s16)
+    G_STORE %6(s32), %4(p1) :: (store (s16), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: uniform_i1_phi
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; OLD_RBS-LABEL: name: uniform_i1_phi
+  ; OLD_RBS: bb.0:
+  ; OLD_RBS-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; OLD_RBS-NEXT:   liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; OLD_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; OLD_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; OLD_RBS-NEXT:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+  ; OLD_RBS-NEXT:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+  ; OLD_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
+  ; OLD_RBS-NEXT:   [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; OLD_RBS-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+  ; OLD_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT:   [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; OLD_RBS-NEXT:   [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP1]](s32)
+  ; OLD_RBS-NEXT:   [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s1)
+  ; OLD_RBS-NEXT:   [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s1)
+  ; OLD_RBS-NEXT:   G_BRCOND [[ZEXT]](s32), %bb.2
+  ; OLD_RBS-NEXT:   G_BR %bb.1
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.1:
+  ; OLD_RBS-NEXT:   successors: %bb.2(0x80000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT:   [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; OLD_RBS-NEXT:   [[TRUNC2:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP2]](s32)
+  ; OLD_RBS-NEXT:   [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s1)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.2:
+  ; OLD_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[ANYEXT]](s32), %bb.0, [[ANYEXT1]](s32), %bb.1
+  ; OLD_RBS-NEXT:   [[TRUNC3:%[0-9]+]]:sgpr(s1) = G_TRUNC [[PHI]](s32)
+  ; OLD_RBS-NEXT:   [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC3]](s1)
+  ; OLD_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; OLD_RBS-NEXT:   [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SEXT]], [[C3]]
+  ; OLD_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+  ; OLD_RBS-NEXT:   G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   S_ENDPGM 0
+  ;
+  ; NEW_RBS-LABEL: name: uniform_i1_phi
+  ; NEW_RBS: bb.0:
+  ; NEW_RBS-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; NEW_RBS-NEXT:   liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; NEW_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; NEW_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; NEW_RBS-NEXT:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+  ; NEW_RBS-NEXT:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+  ; NEW_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
+  ; NEW_RBS-NEXT:   [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; NEW_RBS-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+  ; NEW_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT:   [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; NEW_RBS-NEXT:   [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP1]](s32)
+  ; NEW_RBS-NEXT:   [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s1)
+  ; NEW_RBS-NEXT:   [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s1)
+  ; NEW_RBS-NEXT:   G_BRCOND [[ZEXT]](s32), %bb.2
+  ; NEW_RBS-NEXT:   G_BR %bb.1
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.1:
+  ; NEW_RBS-NEXT:   successors: %bb.2(0x80000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT:   [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; NEW_RBS-NEXT:   [[TRUNC2:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP2]](s32)
+  ; NEW_RBS-NEXT:   [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s1)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.2:
+  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[ANYEXT]](s32), %bb.0, [[ANYEXT1]](s32), %bb.1
+  ; NEW_RBS-NEXT:   [[TRUNC3:%[0-9]+]]:sgpr(s1) = G_TRUNC [[PHI]](s32)
+  ; NEW_RBS-NEXT:   [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC3]](s1)
+  ; NEW_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; NEW_RBS-NEXT:   [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SEXT]], [[C3]]
+  ; NEW_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+  ; NEW_RBS-NEXT:   G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $sgpr0
+    %4:_(s32) = COPY $sgpr1
+    %5:_(s32) = G_CONSTANT i32 6
+    %6:_(s1) = G_ICMP intpred(uge), %3(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(s1) = G_ICMP intpred(ne), %4(s32), %7
+    G_BRCOND %8(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %9:_(s32) = G_CONSTANT i32 1
+    %10:_(s1) = G_ICMP intpred(ult), %3(s32), %9
+
+  bb.2:
+    %11:_(s1) = G_PHI %6(s1), %bb.0, %10(s1), %bb.1
+    %12:_(s32) = G_SEXT %11(s1)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s32) = G_ADD %12, %13
+    G_STORE %14(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vcc_to_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: vcc_to_scc
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 0.000000e+00
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; OLD_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; OLD_RBS-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(oeq), [[COPY5]](s32), [[COPY6]]
+    ; OLD_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; OLD_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; OLD_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[FCMP]](s1), [[COPY7]], [[COPY8]]
+    ; OLD_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: vcc_to_scc
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 0.000000e+00
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; NEW_RBS-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(oeq), [[COPY5]](s32), [[COPY6]]
+    ; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; NEW_RBS-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[FCMP]](s1), [[COPY7]], [[COPY8]]
+    ; NEW_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $vgpr0
+    %4:_(s32) = COPY $vgpr1
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_FCONSTANT float 0.000000e+00
+    %7:_(s1) = G_FCMP floatpred(oeq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: scc_to_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; OLD_RBS-LABEL: name: scc_to_vcc
+    ; OLD_RBS: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; OLD_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY5]](s1), [[COPY1]], [[COPY2]]
+    ; OLD_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: scc_to_vcc
+    ; NEW_RBS: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY5]](s1), [[COPY1]], [[COPY2]]
+    ; NEW_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:_(s1) = G_ICMP intpred(eq), %0(s32), %6
+    %8:_(s32) = G_SELECT %7(s1), %1, %2
+    G_STORE %8(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: vgpr_to_vcc_trunc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    ; OLD_RBS-LABEL: name: vgpr_to_vcc_trunc
+    ; OLD_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; OLD_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY5]](s1), [[COPY1]], [[COPY2]]
+    ; OLD_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: vgpr_to_vcc_trunc
+    ; NEW_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY5]](s1), [[COPY1]], [[COPY2]]
+    ; NEW_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s1) = G_TRUNC %0(s32)
+    %7:_(s32) = G_SELECT %6(s1), %1, %2
+    G_STORE %7(s32), %5(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: zext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: zext
+    ; OLD_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; OLD_RBS-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ZEXT]](s32)
+    ; OLD_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: zext
+    ; NEW_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; NEW_RBS-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ZEXT]](s32)
+    ; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_ZEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: sext
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: sext
+    ; OLD_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; OLD_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s1)
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SEXT]](s32)
+    ; OLD_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: sext
+    ; NEW_RBS: liveins: $sgpr0, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; NEW_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s1)
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SEXT]](s32)
+    ; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 10
+    %5:_(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:_(s32) = G_SEXT %5(s1)
+    G_STORE %6(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_vcc
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    ; OLD_RBS-LABEL: name: and_i1_vcc
+    ; OLD_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[COPY4]]
+    ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
+    ; OLD_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; OLD_RBS-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[COPY5]]
+    ; OLD_RBS-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; OLD_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; OLD_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: and_i1_vcc
+    ; NEW_RBS: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[COPY4]]
+    ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
+    ; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
+    ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(uge), [[COPY1]](s32), [[COPY5]]
+    ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
+    ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AND]](s1), [[COPY]], [[COPY1]]
+    ; NEW_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: and_i1_scc
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+
+    ; OLD_RBS-LABEL: name: and_i1_scc
+    ; OLD_RBS: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; OLD_RBS-NEXT: {{  $}}
+    ; OLD_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; OLD_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; OLD_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; OLD_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; OLD_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; OLD_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; OLD_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; OLD_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; OLD_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
+    ; OLD_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; OLD_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP1]](s32)
+    ; OLD_RBS-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s1)
+    ; OLD_RBS-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s1)
+    ; OLD_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ANYEXT]], [[ANYEXT1]]
+    ; OLD_RBS-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s1) = G_TRUNC [[AND]](s32)
+    ; OLD_RBS-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC2]](s1)
+    ; OLD_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[ZEXT]](s32), [[COPY]], [[COPY1]]
+    ; OLD_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
+    ; OLD_RBS-NEXT: G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; OLD_RBS-NEXT: S_ENDPGM 0
+    ;
+    ; NEW_RBS-LABEL: name: and_i1_scc
+    ; NEW_RBS: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; NEW_RBS-NEXT: {{  $}}
+    ; NEW_RBS-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+    ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+    ; NEW_RBS-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
+    ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
+    ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+    ; NEW_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP1]](s32)
+    ; NEW_RBS-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s1)
+    ; NEW_RBS-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s1)
+    ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ANYEXT]], [[ANYEXT1]]
+    ; NEW_RBS-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s1) = G_TRUNC [[AND]](s32)
+    ; NEW_RBS-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC2]](s1)
+    ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[ZEXT]](s32), [[COPY]], [[COPY1]]
+    ; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
+    ; NEW_RBS-NEXT: G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+    ; NEW_RBS-NEXT: S_ENDPGM 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 10
+    %6:_(s1) = G_ICMP intpred(uge), %0(s32), %5
+    %7:_(s32) = G_CONSTANT i32 20
+    %8:_(s1) = G_ICMP intpred(uge), %1(s32), %7
+    %9:_(s1) = G_AND %6, %8
+    %10:_(s32) = G_SELECT %9(s1), %0, %1
+    G_STORE %10(s32), %4(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_phi_with_uniform_inputs
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; OLD_RBS-LABEL: name: divergent_phi_with_uniform_inputs
+  ; OLD_RBS: bb.0:
+  ; OLD_RBS-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; OLD_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; OLD_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; OLD_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; OLD_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; OLD_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+  ; OLD_RBS-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY3]]
+  ; OLD_RBS-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; OLD_RBS-NEXT:   G_BR %bb.1
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.1:
+  ; OLD_RBS-NEXT:   successors: %bb.2(0x80000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.2:
+  ; OLD_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; OLD_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[PHI]](s32)
+  ; OLD_RBS-NEXT:   G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   S_ENDPGM 0
+  ;
+  ; NEW_RBS-LABEL: name: divergent_phi_with_uniform_inputs
+  ; NEW_RBS: bb.0:
+  ; NEW_RBS-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; NEW_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; NEW_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; NEW_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; NEW_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; NEW_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+  ; NEW_RBS-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY3]]
+  ; NEW_RBS-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; NEW_RBS-NEXT:   G_BR %bb.1
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.1:
+  ; NEW_RBS-NEXT:   successors: %bb.2(0x80000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.2:
+  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, [[C1]](s32), %bb.1
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; NEW_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[PHI]](s32)
+  ; NEW_RBS-NEXT:   G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %4
+    %6:sreg_32_xm0_xexec(s32) = SI_IF %5(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %7:_(s32) = G_CONSTANT i32 1
+
+  bb.2:
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %6(s32)
+    G_STORE %8(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_because_of_temporal_divergent_use
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; OLD_RBS-LABEL: name: divergent_because_of_temporal_divergent_use
+  ; OLD_RBS: bb.0:
+  ; OLD_RBS-NEXT:   successors: %bb.1(0x80000000)
+  ; OLD_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; OLD_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; OLD_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; OLD_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; OLD_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
+  ; OLD_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.1:
+  ; OLD_RBS-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; OLD_RBS-NEXT:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; OLD_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
+  ; OLD_RBS-NEXT:   [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI1]], [[COPY3]]
+  ; OLD_RBS-NEXT:   [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[ADD]](s32)
+  ; OLD_RBS-NEXT:   [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; OLD_RBS-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; OLD_RBS-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; OLD_RBS-NEXT:   G_BR %bb.2
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.2:
+  ; OLD_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
+  ; OLD_RBS-NEXT:   [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; OLD_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+  ; OLD_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
+  ; OLD_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[COPY4]]
+  ; OLD_RBS-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   S_ENDPGM 0
+  ;
+  ; NEW_RBS-LABEL: name: divergent_because_of_temporal_divergent_use
+  ; NEW_RBS: bb.0:
+  ; NEW_RBS-NEXT:   successors: %bb.1(0x80000000)
+  ; NEW_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; NEW_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; NEW_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; NEW_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; NEW_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
+  ; NEW_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.1:
+  ; NEW_RBS-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; NEW_RBS-NEXT:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; NEW_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
+  ; NEW_RBS-NEXT:   [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI1]], [[COPY3]]
+  ; NEW_RBS-NEXT:   [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[ADD]](s32)
+  ; NEW_RBS-NEXT:   [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; NEW_RBS-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; NEW_RBS-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; NEW_RBS-NEXT:   G_BR %bb.2
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.2:
+  ; NEW_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
+  ; NEW_RBS-NEXT:   [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; NEW_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
+  ; NEW_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
+  ; NEW_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[COPY4]]
+  ; NEW_RBS-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 -1
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %10
+    %11:_(s32) = G_UITOFP %9(s32)
+    %12:_(s1) = G_FCMP floatpred(ogt), %11(s32), %0
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %13:_(s32) = G_PHI %9(s32), %bb.1
+    %14:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
+    %15:_(s32) = G_CONSTANT i32 10
+    %16:_(s32) = G_MUL %13, %15
+    G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_2breaks
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; OLD_RBS-LABEL: name: loop_with_2breaks
+  ; OLD_RBS: bb.0:
+  ; OLD_RBS-NEXT:   successors: %bb.1(0x80000000)
+  ; OLD_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; OLD_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; OLD_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; OLD_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; OLD_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; OLD_RBS-NEXT:   [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; OLD_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; OLD_RBS-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; OLD_RBS-NEXT:   [[MV2:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; OLD_RBS-NEXT:   [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
+  ; OLD_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.1:
+  ; OLD_RBS-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
+  ; OLD_RBS-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
+  ; OLD_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+  ; OLD_RBS-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; OLD_RBS-NEXT:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[PHI2]](s32)
+  ; OLD_RBS-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
+  ; OLD_RBS-NEXT:   [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY7]], [[C1]](s32)
+  ; OLD_RBS-NEXT:   [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY7]](s32), [[ASHR]](s32)
+  ; OLD_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; OLD_RBS-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
+  ; OLD_RBS-NEXT:   [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY8]](s32)
+  ; OLD_RBS-NEXT:   [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; OLD_RBS-NEXT:   [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
+  ; OLD_RBS-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY9]]
+  ; OLD_RBS-NEXT:   [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C4]](s32)
+  ; OLD_RBS-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC]](s1)
+  ; OLD_RBS-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+  ; OLD_RBS-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; OLD_RBS-NEXT:   G_BR %bb.2
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.2:
+  ; OLD_RBS-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; OLD_RBS-NEXT:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C5]](s32)
+  ; OLD_RBS-NEXT:   [[SHL1:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY12]](s32)
+  ; OLD_RBS-NEXT:   [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; OLD_RBS-NEXT:   [[LOAD1:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; OLD_RBS-NEXT:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C6]](s32)
+  ; OLD_RBS-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY13]]
+  ; OLD_RBS-NEXT:   [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT:   [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C7]](s32)
+  ; OLD_RBS-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC1]](s1)
+  ; OLD_RBS-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1)
+  ; OLD_RBS-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; OLD_RBS-NEXT:   G_BR %bb.4
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.3:
+  ; OLD_RBS-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
+  ; OLD_RBS-NEXT:   [[PHI4:%[0-9]+]]:vgpr(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; OLD_RBS-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; OLD_RBS-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY16]](s1), [[PHI1]](s32)
+  ; OLD_RBS-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; OLD_RBS-NEXT:   G_BR %bb.6
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.4:
+  ; OLD_RBS-NEXT:   successors: %bb.5(0x80000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; OLD_RBS-NEXT:   [[COPY17:%[0-9]+]]:vgpr(s32) = COPY [[C8]](s32)
+  ; OLD_RBS-NEXT:   [[SHL2:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY17]](s32)
+  ; OLD_RBS-NEXT:   [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
+  ; OLD_RBS-NEXT:   [[LOAD2:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; OLD_RBS-NEXT:   [[COPY18:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
+  ; OLD_RBS-NEXT:   [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY18]]
+  ; OLD_RBS-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
+  ; OLD_RBS-NEXT:   [[COPY19:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
+  ; OLD_RBS-NEXT:   [[ADD1:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI2]], [[COPY19]]
+  ; OLD_RBS-NEXT:   [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
+  ; OLD_RBS-NEXT:   [[COPY20:%[0-9]+]]:vgpr(s32) = COPY [[C10]](s32)
+  ; OLD_RBS-NEXT:   [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[COPY20]]
+  ; OLD_RBS-NEXT:   [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; OLD_RBS-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY15]](s1), $exec_lo, implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.5:
+  ; OLD_RBS-NEXT:   successors: %bb.3(0x80000000)
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY14]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
+  ; OLD_RBS-NEXT:   [[PHI6:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+  ; OLD_RBS-NEXT:   [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+  ; OLD_RBS-NEXT:   [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; OLD_RBS-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY23]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; OLD_RBS-NEXT:   G_BR %bb.3
+  ; OLD_RBS-NEXT: {{  $}}
+  ; OLD_RBS-NEXT: bb.6:
+  ; OLD_RBS-NEXT:   [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; OLD_RBS-NEXT:   S_ENDPGM 0
+  ;
+  ; NEW_RBS-LABEL: name: loop_with_2breaks
+  ; NEW_RBS: bb.0:
+  ; NEW_RBS-NEXT:   successors: %bb.1(0x80000000)
+  ; NEW_RBS-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; NEW_RBS-NEXT:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; NEW_RBS-NEXT:   [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; NEW_RBS-NEXT:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; NEW_RBS-NEXT:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; NEW_RBS-NEXT:   [[MV1:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; NEW_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; NEW_RBS-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; NEW_RBS-NEXT:   [[MV2:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; NEW_RBS-NEXT:   [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
+  ; NEW_RBS-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.1:
+  ; NEW_RBS-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
+  ; NEW_RBS-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
+  ; NEW_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+  ; NEW_RBS-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; NEW_RBS-NEXT:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[PHI2]](s32)
+  ; NEW_RBS-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
+  ; NEW_RBS-NEXT:   [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY7]], [[C1]](s32)
+  ; NEW_RBS-NEXT:   [[MV3:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY7]](s32), [[ASHR]](s32)
+  ; NEW_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; NEW_RBS-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
+  ; NEW_RBS-NEXT:   [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY8]](s32)
+  ; NEW_RBS-NEXT:   [[PTR_ADD:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; NEW_RBS-NEXT:   [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
+  ; NEW_RBS-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[COPY9]]
+  ; NEW_RBS-NEXT:   [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C4]](s32)
+  ; NEW_RBS-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC]](s1)
+  ; NEW_RBS-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
+  ; NEW_RBS-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; NEW_RBS-NEXT:   G_BR %bb.2
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.2:
+  ; NEW_RBS-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; NEW_RBS-NEXT:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C5]](s32)
+  ; NEW_RBS-NEXT:   [[SHL1:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY12]](s32)
+  ; NEW_RBS-NEXT:   [[PTR_ADD1:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; NEW_RBS-NEXT:   [[LOAD1:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+  ; NEW_RBS-NEXT:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C6]](s32)
+  ; NEW_RBS-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[COPY13]]
+  ; NEW_RBS-NEXT:   [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT:   [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[C7]](s32)
+  ; NEW_RBS-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[TRUNC1]](s1)
+  ; NEW_RBS-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1)
+  ; NEW_RBS-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; NEW_RBS-NEXT:   G_BR %bb.4
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.3:
+  ; NEW_RBS-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %43(s1), %bb.5
+  ; NEW_RBS-NEXT:   [[PHI4:%[0-9]+]]:vgpr(s32) = G_PHI %44(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; NEW_RBS-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; NEW_RBS-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY16]](s1), [[PHI1]](s32)
+  ; NEW_RBS-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; NEW_RBS-NEXT:   G_BR %bb.6
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.4:
+  ; NEW_RBS-NEXT:   successors: %bb.5(0x80000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[C8:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
+  ; NEW_RBS-NEXT:   [[COPY17:%[0-9]+]]:vgpr(s32) = COPY [[C8]](s32)
+  ; NEW_RBS-NEXT:   [[SHL2:%[0-9]+]]:vgpr(s64) = G_SHL [[MV3]], [[COPY17]](s32)
+  ; NEW_RBS-NEXT:   [[PTR_ADD2:%[0-9]+]]:vgpr(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
+  ; NEW_RBS-NEXT:   [[LOAD2:%[0-9]+]]:vgpr(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   [[C9:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; NEW_RBS-NEXT:   [[COPY18:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
+  ; NEW_RBS-NEXT:   [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[LOAD2]], [[COPY18]]
+  ; NEW_RBS-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
+  ; NEW_RBS-NEXT:   [[COPY19:%[0-9]+]]:vgpr(s32) = COPY [[C9]](s32)
+  ; NEW_RBS-NEXT:   [[ADD1:%[0-9]+]]:vgpr(s32) = G_ADD [[PHI2]], [[COPY19]]
+  ; NEW_RBS-NEXT:   [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
+  ; NEW_RBS-NEXT:   [[COPY20:%[0-9]+]]:vgpr(s32) = COPY [[C10]](s32)
+  ; NEW_RBS-NEXT:   [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[COPY20]]
+  ; NEW_RBS-NEXT:   [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; NEW_RBS-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY15]](s1), $exec_lo, implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.5:
+  ; NEW_RBS-NEXT:   successors: %bb.3(0x80000000)
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY14]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
+  ; NEW_RBS-NEXT:   [[PHI6:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+  ; NEW_RBS-NEXT:   [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+  ; NEW_RBS-NEXT:   [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; NEW_RBS-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY23]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; NEW_RBS-NEXT:   G_BR %bb.3
+  ; NEW_RBS-NEXT: {{  $}}
+  ; NEW_RBS-NEXT: bb.6:
+  ; NEW_RBS-NEXT:   [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; NEW_RBS-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
+    %11:sreg_32(s1) = IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %12:sreg_32(s1) = PHI %11(s1), %bb.0, %13(s1), %bb.3
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %10(s32), %bb.0
+    %16:_(s32) = G_PHI %10(s32), %bb.0, %17(s32), %bb.3
+    %18:sreg_32(s1) = COPY %12(s1)
+    %19:_(s64) = G_SEXT %16(s32)
+    %20:_(s32) = G_CONSTANT i32 2
+    %21:_(s64) = G_SHL %19, %20(s32)
+    %22:_(p1) = G_PTR_ADD %5, %21(s64)
+    %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
+    %24:_(s32) = G_CONSTANT i32 0
+    %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
+    %26:_(s1) = G_CONSTANT i1 true
+    %27:sreg_32(s1) = COPY %26(s1)
+    %28:sreg_32(s1) = S_ANDN2_B32 %18(s1), $exec_lo, implicit-def $scc
+    %29:sreg_32(s1) = S_AND_B32 $exec_lo, %27(s1), implicit-def $scc
+    %30:sreg_32(s1) = S_OR_B32 %28(s1), %29(s1), implicit-def $scc
+    %31:sreg_32(s1) = COPY %30(s1)
+    %32:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+    %33:_(s32) = G_CONSTANT i32 2
+    %34:_(s64) = G_SHL %19, %33(s32)
+    %35:_(p1) = G_PTR_ADD %8, %34(s64)
+    %36:_(s32) = G_LOAD %35(p1) :: (load (s32), addrspace 1)
+    %37:_(s32) = G_CONSTANT i32 0
+    %38:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %36(s32), %37
+    %39:_(s1) = G_CONSTANT i1 true
+    %40:sreg_32(s1) = COPY %39(s1)
+    %41:sreg_32(s1) = COPY %40(s1)
+    %42:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %13:sreg_32(s1) = PHI %30(s1), %bb.1, %43(s1), %bb.5
+    %17:_(s32) = G_PHI %44(s32), %bb.5, %9(s32), %bb.1
+    %45:sreg_32(s1) = COPY %13(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %32(s32)
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %45(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    %46:_(s32) = G_CONSTANT i32 2
+    %47:_(s64) = G_SHL %19, %46(s32)
+    %48:_(p1) = G_PTR_ADD %2, %47(s64)
+    %49:_(s32) = G_LOAD %48(p1) :: (load (s32), addrspace 1)
+    %50:_(s32) = G_CONSTANT i32 1
+    %51:_(s32) = G_ADD %49, %50
+    G_STORE %51(s32), %48(p1) :: (store (s32), addrspace 1)
+    %52:_(s32) = G_ADD %16, %50
+    %53:_(s32) = G_CONSTANT i32 100
+    %54:_(s1) = G_ICMP intpred(ult), %16(s32), %53
+    %55:sreg_32(s1) = COPY %54(s1)
+    %56:sreg_32(s1) = S_ANDN2_B32 %41(s1), $exec_lo, implicit-def $scc
+    %57:sreg_32(s1) = S_AND_B32 $exec_lo, %55(s1), implicit-def $scc
+    %58:sreg_32(s1) = S_OR_B32 %56(s1), %57(s1), implicit-def $scc
+
+  bb.5:
+    successors: %bb.3(0x80000000)
+
+    %59:sreg_32(s1) = PHI %40(s1), %bb.2, %58(s1), %bb.4
+    %44:_(s32) = G_PHI %52(s32), %bb.4, %9(s32), %bb.2
+    %60:sreg_32(s1) = COPY %59(s1)
+    %61:sreg_32(s1) = COPY %60(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %42(s32)
+    %62:sreg_32(s1) = S_ANDN2_B32 %31(s1), $exec_lo, implicit-def $scc
+    %63:sreg_32(s1) = S_AND_B32 $exec_lo, %61(s1), implicit-def $scc
+    %43:sreg_32(s1) = S_OR_B32 %62(s1), %63(s1), implicit-def $scc
+    G_BR %bb.3
+
+  bb.6:
+    %64:_(s32) = G_PHI %15(s32), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %64(s32)
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
index 9c8410b84ea0b..6b91707328dba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 
 ---
 name: mul_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
index e92c11144cd45..da3b24c404efe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: or_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi-s1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi-s1.mir
index 0c90292add916..39e421c1b8f94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi-s1.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi-s1.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: phi_s1_scc_scc_sbranch
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi.mir
index 33443380b38df..b5a04cef68807 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-phi.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: phi_s32_ss_sbranch
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
index d2ac717494c7f..55048d51918fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: gep_p1_s_k
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrmask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrmask.mir
index c08e6cf256f7b..1cddf55a3a53e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrmask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrmask.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ptrmask_p1_s_k
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
index 063e6a08da1c2..2db1a5b1d2ba2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: ptrtoint_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-reg-sequence.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-reg-sequence.mir
index cafbf0af2cf0e..cee18c420e6c8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-reg-sequence.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-reg-sequence.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: reg_sequence_ss_vreg
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
index 818f19b0cf7f2..524714df5b974 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: sadde_s32_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
index 029e89622dd4a..97c006a1a7216 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ...
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
index c728fdd193393..810724dab685d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: select_s32_scc_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
index 50c3704fccd89..cf0ca2c9eb634 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: sext_inreg_s_s32_1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
index c758bd0056da1..1e0ba2e79b82e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: sext_s32_to_s64_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sextload.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sextload.mir
index 671172c78a053..b257db4f1e665 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sextload.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sextload.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect  -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect  -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: sextload_constant_i8_to_i32_uniform
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
index 3123ab7bc2e20..b4290ea0a4203 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-fast -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=amdgpu-regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
 
 ---
 name: shl_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
index 1745cf213c4d0..66e0d3db24112 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: sitofp_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
index de16194447d47..eee553e4e872e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
 
 ---
 name: smax_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
index 5130e171890cc..ef60aa81e4923 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
 
 ---
 name: smin_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
index 777c64854218e..872e4477edc30 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
@@ -1,9 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
-# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
 
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: smulh_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
index 949ed7946a6b1..c2dbeafce3df7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -run-pass=amdgpu-regbankselect %s -o - | FileCheck -check-prefix=GFX7 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -run-pass=amdgpu-regbankselect %s -o - | FileCheck -check-prefix=GFX12 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -run-pass=regbankselect %s -o - | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -run-pass=regbankselect %s -o - | FileCheck -check-prefix=GFX12 %s
 
 --- |
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
index d346d809dc01b..eae770ae18dce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: ssube_s32_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
index 232112c90cc70..4fcd0fdf105d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 
 ---
 name: sub_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
index 041621bb193ff..e65bf44b9c774 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: trunc_i64_to_i32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
index 4d9b7d17b69b9..48a060171ce89 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
 ---
 name: uadde_s32_sss
 legalized: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
index ecdab0a986638..e53c67bc058bb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: uaddo_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
index 6730f8b698a21..20d280680a09d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ...
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
index a51a4b40fd51c..554c88a68972b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: uitofp_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
index 85fafa5b4d7c7..36a38aac1ccaa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
 
 ---
 name: umax_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
index fa9da1b54f56d..bb232b5e07651 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-fast -o - %s  | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -regbankselect-greedy -o - %s  | FileCheck %s
 
 ---
 name: umin_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
index e9aea5a679799..a5a3546dbcb23 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
@@ -1,9 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
-# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
 
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: umulh_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
index 01bad2557f7d3..8159f1b982c36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=kaveri -run-pass=amdgpu-regbankselect -mattr=+unaligned-access-mode %s -verify-machineinstrs -o - | FileCheck -check-prefixes=GFX7 %s
-# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -mattr=+unaligned-access-mode %s -verify-machineinstrs -o - | FileCheck -check-prefixes=GFX1010 %s
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=kaveri -run-pass=regbankselect -mattr=+unaligned-access-mode %s -verify-machineinstrs -o - | FileCheck -check-prefixes=GFX7 %s
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -run-pass=regbankselect -mattr=+unaligned-access-mode %s -verify-machineinstrs -o - | FileCheck -check-prefixes=GFX1010 %s
 
 ---
 name: test_uniform_load_without_noclobber
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir
index 4621e15923139..8e2a60502350e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 
 ---
 name: test_unmerge_s64_s32_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
index 5d5f0a7a75810..8c7a029d3693e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
 
 ---
 name: usube_s32_sss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
index 733b7016de65c..1e0129172ebcf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: usubo_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-agpr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-agpr.mir
index 23383f27efce7..f70d650e9b319 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-agpr.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -verify-machineinstrs -run-pass=amdgpu-regbankselect -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -verify-machineinstrs -run-pass=regbankselect -o - %s | FileCheck %s
 
 # Make sure that an arbitrary AGPR is treated as a divergent value
 # that needs to be copied to VGPR, and then waterfalled
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
index 76b43bf989737..f1f8d0b6b9df5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -global-isel -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
-# RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -run-pass=amdgpu-regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX10 %s
+# RUN: llc -global-isel -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -run-pass=regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -run-pass=regbankselect -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX10 %s
 ---
 name: constant_load_i8_align8
 legalized: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
index af7e065d6fccb..874a8a064b481 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: xor_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
index 05bbf4f0a3e42..aa3bc8c658f2c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
 name: zext_s32_to_s64_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zextload.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zextload.mir
index c2f219dd10cab..821b78f8810b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zextload.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zextload.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=amdgpu-regbankselect  -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect  -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect  -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name: zextload_constant_i8_to_i32_uniform
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect.mir
index 1565986516860..a5711418a8000 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global -run-pass=amdgpu-regbankselect %s -verify-machineinstrs -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global -run-pass=regbankselect %s -verify-machineinstrs -o - | FileCheck %s
 
 --- |
   define amdgpu_kernel void @load_constant(ptr addrspace(4) %ptr0) {
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
index cff9ce0506679..96bbcb7ed2149 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
@@ -233,9 +233,9 @@ attributes #1 = { nounwind }
 ; AKF_HSA: attributes #[[ATTR1]] = { nounwind }
 ;.
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
-; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
 ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll
index e5d440b96349f..0e4a9791b6f57 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll
@@ -116,7 +116,7 @@ define amdgpu_kernel void @kernel_calls_extern() {
 define amdgpu_kernel void @kernel_calls_extern_marked_callsite() {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_extern_marked_callsite(
 ; CHECK-SAME: ) #[[ATTR4]] {
-; CHECK-NEXT:    call void @unknown() #[[ATTR9:[0-9]+]]
+; CHECK-NEXT:    call void @unknown() #[[ATTR10:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   call void @unknown() #0
@@ -136,7 +136,7 @@ define amdgpu_kernel void @kernel_calls_indirect(ptr %indirect) {
 define amdgpu_kernel void @kernel_calls_indirect_marked_callsite(ptr %indirect) {
 ; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_indirect_marked_callsite(
 ; CHECK-SAME: ptr [[INDIRECT:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    call void [[INDIRECT]]() #[[ATTR9]]
+; CHECK-NEXT:    call void [[INDIRECT]]() #[[ATTR10]]
 ; CHECK-NEXT:    ret void
 ;
   call void %indirect() #0
@@ -254,14 +254,14 @@ define amdgpu_kernel void @indirect_calls_none_agpr(i1 %cond) {
 
 attributes #0 = { "amdgpu-no-agpr" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR3:[0-9]+]] = { "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR4]] = { "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR6:[0-9]+]] = { convergent nocallback nofree nosync nounwind willreturn memory(none) "target-cpu"="gfx90a" }
-; CHECK: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx90a" }
-; CHECK: attributes #[[ATTR8:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) "target-cpu"="gfx90a" }
-; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" }
+; CHECK: attributes #[[ATTR8:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx90a" }
+; CHECK: attributes #[[ATTR9:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) "target-cpu"="gfx90a" }
+; CHECK: attributes #[[ATTR10]] = { "amdgpu-no-agpr" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-max-num-workgroups-load-annotate.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-max-num-workgroups-load-annotate.ll
new file mode 100644
index 0000000000000..9064292129928
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-max-num-workgroups-load-annotate.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-lower-kernel-attributes %s | FileCheck %s
+
+define i32 @use_grid_size_x_max_num_workgroups() #0 {
+; CHECK-LABEL: define i32 @use_grid_size_x_max_num_workgroups(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load i32, ptr addrspace(4) %implicitarg.ptr, align 4
+  ret i32 %grid.size.x
+}
+
+define i32 @use_grid_size_x_max_num_workgroups_existing_nonzero_range() #0 {
+; CHECK-LABEL: define i32 @use_grid_size_x_max_num_workgroups_existing_nonzero_range(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4, !range [[RNG0]]
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load i32, ptr addrspace(4) %implicitarg.ptr, align 4, !range !0
+  ret i32 %grid.size.x
+}
+
+define i32 @use_grid_size_y_max_num_workgroups() #0 {
+; CHECK-LABEL: define i32 @use_grid_size_y_max_num_workgroups(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GEP_GRID_SIZE_Y:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 4
+; CHECK-NEXT:    [[GRID_SIZE_Y:%.*]] = load i32, ptr addrspace(4) [[GEP_GRID_SIZE_Y]], align 4, !range [[RNG1:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_Y]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %gep.grid.size.y = getelementptr inbounds i8, ptr addrspace(4) %implicitarg.ptr, i64 4
+  %grid.size.y = load i32, ptr addrspace(4) %gep.grid.size.y, align 4
+  ret i32 %grid.size.y
+}
+
+define i32 @use_grid_size_z_max_num_workgroups() #0 {
+; CHECK-LABEL: define i32 @use_grid_size_z_max_num_workgroups(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GEP_GRID_SIZE_Z:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 8
+; CHECK-NEXT:    [[GRID_SIZE_Z:%.*]] = load i32, ptr addrspace(4) [[GEP_GRID_SIZE_Z]], align 4, !range [[RNG2:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_Z]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %gep.grid.size.z = getelementptr inbounds i8, ptr addrspace(4) %implicitarg.ptr, i64 8
+  %grid.size.z = load i32, ptr addrspace(4) %gep.grid.size.z, align 4
+  ret i32 %grid.size.z
+}
+
+define <2 x i16> @use_grid_size_x_max_num_workgroups_load_wrong_type() #0 {
+; CHECK-LABEL: define <2 x i16> @use_grid_size_x_max_num_workgroups_load_wrong_type(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load <2 x i16>, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4
+; CHECK-NEXT:    ret <2 x i16> [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load <2 x i16>, ptr addrspace(4) %implicitarg.ptr, align 4
+  ret <2 x i16> %grid.size.x
+}
+
+define i32 @use_grid_size_x_max_num_workgroups_max_minus_1() #1 {
+; CHECK-LABEL: define i32 @use_grid_size_x_max_num_workgroups_max_minus_1(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4, !range [[RNG3:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load i32, ptr addrspace(4) %implicitarg.ptr, align 4
+  ret i32 %grid.size.x
+}
+
+define i32 @use_grid_size_x_max_num_workgroups_max() #2 {
+; CHECK-LABEL: define i32 @use_grid_size_x_max_num_workgroups_max(
+; CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load i32, ptr addrspace(4) %implicitarg.ptr, align 4
+  ret i32 %grid.size.x
+}
+
+define i32 @use_grid_size_x_max_num_workgroups_zero() #3 {
+; CHECK-LABEL: define i32 @use_grid_size_x_max_num_workgroups_zero(
+; CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT:    [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG_PTR]], align 4
+; CHECK-NEXT:    ret i32 [[GRID_SIZE_X]]
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %grid.size.x = load i32, ptr addrspace(4) %implicitarg.ptr, align 4
+  ret i32 %grid.size.x
+}
+
+declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #3
+
+attributes #0 = { "amdgpu-max-num-workgroups"="36,42,89" }
+attributes #1 = { "amdgpu-max-num-workgroups"="4294967294,42,89" }
+attributes #2 = { "amdgpu-max-num-workgroups"="4294967295,42,89" }
+attributes #3 = { "amdgpu-max-num-workgroups"="0,42,89" }
+
+!0 = !{i32 0, i32 -1}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-max-num-workgroups"="36,42,89" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-max-num-workgroups"="4294967294,42,89" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-max-num-workgroups"="4294967295,42,89" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-max-num-workgroups"="0,42,89" }
+; CHECK: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+;.
+; CHECK: [[RNG0]] = !{i32 1, i32 37}
+; CHECK: [[RNG1]] = !{i32 1, i32 43}
+; CHECK: [[RNG2]] = !{i32 1, i32 90}
+; CHECK: [[RNG3]] = !{i32 1, i32 -1}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
index 9d0d85da9f7fa..25b6b7be1f3b5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
@@ -82,10 +82,10 @@ define float @test_pow_fast_f32__integral_y(float %x, i32 %y.i) {
 ; CHECK-NEXT:    v_cvt_f32_i32_e32 v1, v1
 ; CHECK-NEXT:    s_mov_b32 s4, 0x800000
 ; CHECK-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x4f800000
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; CHECK-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; CHECK-NEXT:    v_mul_f32_e64 v3, |v0|, v3
+; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; CHECK-NEXT:    v_ldexp_f32 v3, |v0|, v3
 ; CHECK-NEXT:    v_log_f32_e32 v3, v3
 ; CHECK-NEXT:    v_cvt_f32_i32_e32 v4, v1
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0x42000000
@@ -98,10 +98,10 @@ define float @test_pow_fast_f32__integral_y(float %x, i32 %y.i) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
 ; CHECK-NEXT:    v_fma_f32 v2, v2, v4, v3
 ; CHECK-NEXT:    v_exp_f32_e32 v2, v2
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x1f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
+; CHECK-NEXT:    v_not_b32_e32 v3, 63
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 31, v1
-; CHECK-NEXT:    v_mul_f32_e32 v2, v2, v3
+; CHECK-NEXT:    v_ldexp_f32 v2, v2, v3
 ; CHECK-NEXT:    v_and_or_b32 v0, v1, v0, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %y = sitofp i32 %y.i to float
@@ -228,9 +228,9 @@ define float @test_powr_fast_f32(float %x, float %y) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s4, 0x800000
 ; CHECK-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x4f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; CHECK-NEXT:    v_mul_f32_e32 v0, v0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; CHECK-NEXT:    v_ldexp_f32 v0, v0, v3
 ; CHECK-NEXT:    v_log_f32_e32 v0, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0x42000000
 ; CHECK-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
@@ -242,9 +242,9 @@ define float @test_powr_fast_f32(float %x, float %y) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v2, 0, v3, vcc
 ; CHECK-NEXT:    v_fma_f32 v0, v1, v0, v2
 ; CHECK-NEXT:    v_exp_f32_e32 v0, v0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; CHECK-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CHECK-NEXT:    v_not_b32_e32 v1, 63
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; CHECK-NEXT:    v_ldexp_f32 v0, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %powr = tail call fast float @_Z4powrff(float %x, float %y)
   ret float %powr
@@ -368,9 +368,9 @@ define float @test_pown_fast_f32(float %x, i32 %y) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s4, 0x800000
 ; CHECK-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x4f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; CHECK-NEXT:    v_mul_f32_e64 v3, |v0|, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; CHECK-NEXT:    v_ldexp_f32 v3, |v0|, v3
 ; CHECK-NEXT:    v_log_f32_e32 v3, v3
 ; CHECK-NEXT:    v_cvt_f32_i32_e32 v4, v1
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0x42000000
@@ -383,10 +383,10 @@ define float @test_pown_fast_f32(float %x, i32 %y) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
 ; CHECK-NEXT:    v_fma_f32 v2, v2, v4, v3
 ; CHECK-NEXT:    v_exp_f32_e32 v2, v2
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x1f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
+; CHECK-NEXT:    v_not_b32_e32 v3, 63
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 31, v1
-; CHECK-NEXT:    v_mul_f32_e32 v2, v2, v3
+; CHECK-NEXT:    v_ldexp_f32 v2, v2, v3
 ; CHECK-NEXT:    v_and_or_b32 v0, v1, v0, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %call = tail call fast float @_Z4pownfi(float %x, i32 %y)
@@ -511,9 +511,9 @@ define float @test_pown_fast_f32_known_even(float %x, i32 %y.arg) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s4, 0x800000
 ; CHECK-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x4f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; CHECK-NEXT:    v_mul_f32_e64 v0, |v0|, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; CHECK-NEXT:    v_ldexp_f32 v0, |v0|, v3
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
 ; CHECK-NEXT:    v_log_f32_e32 v0, v0
 ; CHECK-NEXT:    v_cvt_f32_i32_e32 v1, v1
@@ -527,9 +527,9 @@ define float @test_pown_fast_f32_known_even(float %x, i32 %y.arg) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v2, 0, v3, vcc
 ; CHECK-NEXT:    v_fma_f32 v0, v0, v1, v2
 ; CHECK-NEXT:    v_exp_f32_e32 v0, v0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; CHECK-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CHECK-NEXT:    v_not_b32_e32 v1, 63
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; CHECK-NEXT:    v_ldexp_f32 v0, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %y = shl i32 %y.arg, 1
   %call = tail call fast float @_Z4pownfi(float %x, i32 %y)
@@ -651,9 +651,9 @@ define float @test_pown_fast_f32_known_odd(float %x, i32 %y.arg) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s4, 0x800000
 ; CHECK-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x4f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; CHECK-NEXT:    v_mul_f32_e64 v3, |v0|, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; CHECK-NEXT:    v_ldexp_f32 v3, |v0|, v3
 ; CHECK-NEXT:    v_or_b32_e32 v1, 1, v1
 ; CHECK-NEXT:    v_log_f32_e32 v3, v3
 ; CHECK-NEXT:    v_cvt_f32_i32_e32 v1, v1
@@ -667,10 +667,10 @@ define float @test_pown_fast_f32_known_odd(float %x, i32 %y.arg) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
 ; CHECK-NEXT:    v_fma_f32 v1, v2, v1, v3
 ; CHECK-NEXT:    v_exp_f32_e32 v1, v1
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1f800000
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; CHECK-NEXT:    v_not_b32_e32 v2, 63
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; CHECK-NEXT:    s_brev_b32 s4, -2
-; CHECK-NEXT:    v_mul_f32_e32 v1, v1, v2
+; CHECK-NEXT:    v_ldexp_f32 v1, v1, v2
 ; CHECK-NEXT:    v_bfi_b32 v0, s4, v1, v0
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %y = or i32 %y.arg, 1
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
index 3d4ae84d9c698..ed136c58379cb 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
@@ -956,7 +956,7 @@ define amdgpu_kernel void @kern_call_enqueued_block_def() {
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_call_enqueued_block_def
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR27:[0-9]+]] {
+; ATTRIBUTOR_HSA-SAME: () #[[ATTR30:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @enqueue_block_def()
 ; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
@@ -969,7 +969,7 @@ define void @unused_enqueue_block() {
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@unused_enqueue_block
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR27]] {
+; ATTRIBUTOR_HSA-SAME: () #[[ATTR30]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
   ret void
@@ -980,7 +980,7 @@ define internal void @known_func() {
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@known_func
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR27]] {
+; ATTRIBUTOR_HSA-SAME: () #[[ATTR30]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
   ret void
@@ -994,7 +994,7 @@ define amdgpu_kernel void @kern_callsite_enqueue_block() {
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_callsite_enqueue_block
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR27]] {
+; ATTRIBUTOR_HSA-SAME: () #[[ATTR30]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @known_func() #[[ATTR29:[0-9]+]]
 ; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
@@ -1024,34 +1024,35 @@ attributes #6 = { "enqueued-block" }
 ; AKF_HSA: attributes #[[ATTR7]] = { "enqueued-block" }
 ; AKF_HSA: attributes #[[ATTR8]] = { "amdgpu-calls" }
 ;.
+
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR16]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR18]] = { nounwind "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR23:[0-9]+]] = { nounwind sanitize_address "amdgpu-no-implicitarg-ptr" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR24:[0-9]+]] = { "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR26]] = { "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR27]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR27:[0-9]+]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR28]] = { nounwind }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR29]] = { "enqueued-block" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
index 43cdf85ed3818..239bdfde323cf 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
@@ -442,7 +442,7 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %p
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_private_to_flat_addrspacecast
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR12]] {
+; ATTRIBUTOR_HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
 ; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -478,11 +478,17 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast(ptr %ptr) #1 {
 
 ; No-op addrspacecast should not use queue ptr
 define amdgpu_kernel void @use_global_to_flat_addrspacecast(ptr addrspace(1) %ptr) #1 {
-; HSA-LABEL: define {{[^@]+}}@use_global_to_flat_addrspacecast
-; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
-; HSA-NEXT:    ret void
+; AKF_HSA-LABEL: define {{[^@]+}}@use_global_to_flat_addrspacecast
+; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
+; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; AKF_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; AKF_HSA-NEXT:    ret void
+;
+; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_global_to_flat_addrspacecast
+; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
   %stof = addrspacecast ptr addrspace(1) %ptr to ptr
   store volatile i32 0, ptr %stof
@@ -490,11 +496,17 @@ define amdgpu_kernel void @use_global_to_flat_addrspacecast(ptr addrspace(1) %pt
 }
 
 define amdgpu_kernel void @use_constant_to_flat_addrspacecast(ptr addrspace(4) %ptr) #1 {
-; HSA-LABEL: define {{[^@]+}}@use_constant_to_flat_addrspacecast
-; HSA-SAME: (ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
-; HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
-; HSA-NEXT:    [[LD:%.*]] = load volatile i32, ptr [[STOF]], align 4
-; HSA-NEXT:    ret void
+; AKF_HSA-LABEL: define {{[^@]+}}@use_constant_to_flat_addrspacecast
+; AKF_HSA-SAME: (ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; AKF_HSA-NEXT:    [[LD:%.*]] = load volatile i32, ptr [[STOF]], align 4
+; AKF_HSA-NEXT:    ret void
+;
+; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_constant_to_flat_addrspacecast
+; ATTRIBUTOR_HSA-SAME: (ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; ATTRIBUTOR_HSA-NEXT:    [[LD:%.*]] = load volatile i32, ptr [[STOF]], align 4
+; ATTRIBUTOR_HSA-NEXT:    ret void
 ;
   %stof = addrspacecast ptr addrspace(4) %ptr to ptr
   %ld = load volatile i32, ptr %stof
@@ -621,7 +633,7 @@ define void @use_alloca_func() #1 {
 ; AKF_HSA-NEXT:    ret void
 ;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca_func
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR_HSA-SAME: () #[[ATTR14:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
 ; ATTRIBUTOR_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -643,19 +655,20 @@ attributes #1 = { nounwind }
 ; AKF_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-stack-objects" }
 ;.
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
 ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
index 547ff69592ca0..89fe46d975309 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
@@ -398,13 +398,13 @@ attributes #1 = { nounwind }
 ; AKF_CHECK: attributes #[[ATTR1]] = { nounwind }
 ;.
 ; ATTRIBUTOR_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-max-num-workgroups-propagate.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-max-num-workgroups-propagate.ll
new file mode 100644
index 0000000000000..366432e0fc6cb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-max-num-workgroups-propagate.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s | FileCheck %s
+
+; External call to avoid inferring argument attributes. This makes the
+; final attribute groups easier to read
+declare void @dummy()
+
+define void @extern_callee() {
+; CHECK-LABEL: define void @extern_callee(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define internal void @callee_1_2_3() {
+; CHECK-LABEL: define internal void @callee_1_2_3(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define amdgpu_kernel void @kernel_1_2_3() #0 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_1_2_3(
+; CHECK-SAME: ) #[[ATTR1]] {
+; CHECK-NEXT:    call void @callee_1_2_3()
+; CHECK-NEXT:    call void @extern_callee()
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @callee_1_2_3()
+  call void @extern_callee()
+  call void @dummy()
+  ret void
+}
+
+attributes #0 = {"amdgpu-max-num-workgroups"="1,2,3"}
+
+; -> 100,10,99
+define internal void @callee_merge_100_8_32__16_10_99() {
+; CHECK-LABEL: define internal void @callee_merge_100_8_32__16_10_99(
+; CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define amdgpu_kernel void @kernel_100_8_32() #1 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_100_8_32(
+; CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    call void @callee_merge_100_8_32__16_10_99()
+; CHECK-NEXT:    ret void
+;
+  call void @callee_merge_100_8_32__16_10_99()
+  ret void
+}
+
+define amdgpu_cs void @amdgpu_cs_100_8_32() #1 {
+; CHECK-LABEL: define amdgpu_cs void @amdgpu_cs_100_8_32(
+; CHECK-SAME: ) #[[ATTR3]] {
+; CHECK-NEXT:    call void @callee_merge_100_8_32__16_10_99()
+; CHECK-NEXT:    ret void
+;
+  call void @callee_merge_100_8_32__16_10_99()
+  ret void
+}
+
+attributes #1 = {"amdgpu-max-num-workgroups"="100,8,32"}
+
+define amdgpu_kernel void @kernel_16_10_99() #2 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_16_10_99(
+; CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    call void @callee_merge_100_8_32__16_10_99()
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @callee_merge_100_8_32__16_10_99()
+  call void @dummy()
+  ret void
+}
+
+attributes #2 = {"amdgpu-max-num-workgroups"="16,10,99"}
+
+define internal void @merge_to_worst_case() {
+; CHECK-LABEL: define internal void @merge_to_worst_case(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define internal void @callee_x_worst_case() {
+; CHECK-LABEL: define internal void @callee_x_worst_case(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define amdgpu_kernel void @kernel_x_maximum() #3 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_x_maximum(
+; CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
+; CHECK-NEXT:    call void @merge_to_worst_case()
+; CHECK-NEXT:    call void @callee_x_worst_case()
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @merge_to_worst_case()
+  call void @callee_x_worst_case()
+  call void @dummy()
+  ret void
+}
+
+attributes #3 = {"amdgpu-max-num-workgroups"="4294967295,1,1"}
+
+define amdgpu_kernel void @kernel_y_maximum() #4 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_y_maximum(
+; CHECK-SAME: ) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT:    call void @merge_to_worst_case()
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @merge_to_worst_case()
+  call void @dummy()
+  ret void
+}
+
+attributes #4 = {"amdgpu-max-num-workgroups"="1,4294967295,1"}
+
+define amdgpu_kernel void @kernel_z_maximum() #5 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_z_maximum(
+; CHECK-SAME: ) #[[ATTR7:[0-9]+]] {
+; CHECK-NEXT:    call void @merge_to_worst_case()
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @merge_to_worst_case()
+  call void @dummy()
+  ret void
+}
+
+attributes #5 = {"amdgpu-max-num-workgroups"="1,1,4294967295"}
+
+; Make sure the attribute isn't lost from the callee.
+define internal void @annotated_callee_from_unannotated_kernel() #6 {
+; CHECK-LABEL: define internal void @annotated_callee_from_unannotated_kernel(
+; CHECK-SAME: ) #[[ATTR8:[0-9]+]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+attributes #6 = {"amdgpu-max-num-workgroups"="42,99,123"}
+
+define amdgpu_kernel void @unannotated_kernel_calls_annotated_callee()  {
+; CHECK-LABEL: define amdgpu_kernel void @unannotated_kernel_calls_annotated_callee(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @annotated_callee_from_unannotated_kernel()
+; CHECK-NEXT:    ret void
+;
+  call void @annotated_callee_from_unannotated_kernel()
+  ret void
+}
+
+
+define internal void @annotated_callee_merge_caller() #7 {
+; CHECK-LABEL: define internal void @annotated_callee_merge_caller(
+; CHECK-SAME: ) #[[ATTR9:[0-9]+]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+attributes #7 = {"amdgpu-max-num-workgroups"="512,256,1024"}
+
+define amdgpu_kernel void @call_annotated_callee_merge_caller() #8 {
+; CHECK-LABEL: define amdgpu_kernel void @call_annotated_callee_merge_caller(
+; CHECK-SAME: ) #[[ATTR10:[0-9]+]] {
+; CHECK-NEXT:    call void @annotated_callee_merge_caller()
+; CHECK-NEXT:    ret void
+;
+  call void @annotated_callee_merge_caller()
+  ret void
+}
+
+attributes #8 = {"amdgpu-max-num-workgroups"="256,128,2048"}
+
+define internal void @called_by_explicit_worst_case() {
+; CHECK-LABEL: define internal void @called_by_explicit_worst_case(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    ret void
+;
+  call void @dummy()
+  ret void
+}
+
+define amdgpu_kernel void @kernel_explicit_worst_case() #9 {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_explicit_worst_case(
+; CHECK-SAME: ) #[[ATTR11:[0-9]+]] {
+; CHECK-NEXT:    call void @called_by_explicit_worst_case()
+; CHECK-NEXT:    ret void
+;
+  call void @called_by_explicit_worst_case()
+  ret void
+}
+
+attributes #9 = {"amdgpu-max-num-workgroups"="4294967295,4294967295,4294967295"}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-max-num-workgroups"="1,2,3" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-max-num-workgroups"="100,10,99" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-max-num-workgroups"="100,8,32" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR4]] = { "amdgpu-max-num-workgroups"="16,10,99" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR5]] = { "amdgpu-max-num-workgroups"="4294967295,1,1" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR6]] = { "amdgpu-max-num-workgroups"="1,4294967295,1" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR7]] = { "amdgpu-max-num-workgroups"="1,1,4294967295" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR8]] = { "amdgpu-max-num-workgroups"="42,99,123" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR9]] = { "amdgpu-max-num-workgroups"="256,128,1024" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR10]] = { "amdgpu-max-num-workgroups"="256,128,2048" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR11]] = { "amdgpu-max-num-workgroups"="4294967295,4294967295,4294967295" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
new file mode 100644
index 0000000000000..682a57571d11e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
@@ -0,0 +1,570 @@
+; Test the generation of the attribute amdgpu-no-flat-scratch-init
+; RUN: opt -S -O2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -global-isel -stop-after=irtranslator | FileCheck -check-prefixes=GFX10 %s
+
+;; tests of addrspacecast
+
+define void @without_global_to_flat_addrspacecast(ptr addrspace(1) %ptr) {
+  store volatile i32 0, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_global_to_flat_addrspacecast_cc_kernel(ptr addrspace(1) %ptr) {
+  store volatile i32 0, ptr addrspace(1) %ptr
+  ret void
+}
+
+define void @with_global_to_flat_addrspacecast(ptr addrspace(1) %ptr) {
+  %stof = addrspacecast ptr addrspace(1) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_global_to_flat_addrspacecast_cc_kernel(ptr addrspace(1) %ptr) {
+  %stof = addrspacecast ptr addrspace(1) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_region_to_flat_addrspacecast(ptr addrspace(2) %ptr) {
+  store volatile i32 0, ptr addrspace(2) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_region_to_flat_addrspacecast_cc_kernel(ptr addrspace(2) %ptr) {
+  store volatile i32 0, ptr addrspace(2) %ptr
+  ret void
+}
+
+define void @with_region_to_flat_addrspacecast(ptr addrspace(2) %ptr) {
+  %stof = addrspacecast ptr addrspace(2) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_region_to_flat_addrspacecast_cc_kernel(ptr addrspace(2) %ptr) {
+  %stof = addrspacecast ptr addrspace(2) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) {
+  store volatile i32 0, ptr addrspace(3) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_group_to_flat_addrspacecast_cc_kernel(ptr addrspace(3) %ptr) {
+  store volatile i32 0, ptr addrspace(3) %ptr
+  ret void
+}
+
+define void @with_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) {
+  %stof = addrspacecast ptr addrspace(3) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_group_to_flat_addrspacecast_cc_kernel(ptr addrspace(3) %ptr) {
+  %stof = addrspacecast ptr addrspace(3) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_constant_to_flat_addrspacecast(ptr addrspace(4) %ptr) {
+  store volatile i32 0, ptr addrspace(4) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_constant_to_flat_addrspacecast_cc_kernel(ptr addrspace(4) %ptr) {
+  store volatile i32 0, ptr addrspace(4) %ptr
+  ret void
+}
+
+define void @with_constant_to_flat_addrspacecast(ptr addrspace(4) %ptr) {
+  %stof = addrspacecast ptr addrspace(4) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_constant_to_flat_addrspacecast_cc_kernel(ptr addrspace(4) %ptr) {
+  %stof = addrspacecast ptr addrspace(4) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  store volatile i32 0, ptr addrspace(5) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  store volatile i32 0, ptr addrspace(5) %ptr
+  ret void
+}
+
+define void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @with_cast_call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @with_cast_call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @with_cast_call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @with_cast_call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+;; tests of indirect call, intrinsics
+
+@gv.fptr0 = external hidden unnamed_addr addrspace(4) constant ptr, align 4
+
+define void @with_indirect_call() {
+  %fptr = load ptr, ptr addrspace(4) @gv.fptr0
+  call void %fptr()
+  ret void
+}
+
+define amdgpu_kernel void @with_indirect_call_cc_kernel() {
+  %fptr = load ptr, ptr addrspace(4) @gv.fptr0
+  call void %fptr()
+  ret void
+}
+
+define void @call_with_indirect_call() {
+  call void @with_indirect_call()
+  ret void
+}
+
+define amdgpu_kernel void @call_with_indirect_call_cc_kernel() {
+  call void @with_indirect_call()
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workgroup.id.x()
+
+define void @use_intrinsic_workitem_id_x() {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, ptr addrspace(1) null
+  ret void
+}
+
+define amdgpu_kernel void @use_intrinsic_workitem_id_x_cc_kernel() {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, ptr addrspace(1) null
+  ret void
+}
+
+define void @call_use_intrinsic_workitem_id_x() {
+  call void @use_intrinsic_workitem_id_x()
+  ret void
+}
+
+define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
+  call void @use_intrinsic_workitem_id_x()
+  ret void
+}
+
+; GFX10: name:            without_global_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            without_global_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_global_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_global_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    privateSegmentWaveByteOffset: { reg: '$sgpr7' }
+;
+; GFX10: name:            without_region_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            without_region_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_region_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_region_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            without_group_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            without_group_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_group_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_group_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            without_constant_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            without_constant_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_constant_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_constant_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_with_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_with_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_both_with_and_without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_call_without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_call_without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_call_with_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_call_with_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            call_call_both_with_and_without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_cast_call_without_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_cast_call_without_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_cast_call_with_private_to_flat_addrspacecast
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_cast_call_with_private_to_flat_addrspacecast_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+;
+; GFX10: name:            with_indirect_call
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            with_indirect_call_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr8_sgpr9' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr12_sgpr13' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr14' }
+;
+; GFX10: name:            call_with_indirect_call
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_with_indirect_call_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr8_sgpr9' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr12_sgpr13' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr14' }
+
+;
+; GFX10: name:            use_intrinsic_workitem_id_x
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            use_intrinsic_workitem_id_x_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr4' }
+;
+; GFX10: name:            call_use_intrinsic_workitem_id_x
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    dispatchPtr:     { reg: '$sgpr4_sgpr5' }
+; GFX10-NEXT:    queuePtr:        { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    dispatchID:      { reg: '$sgpr10_sgpr11' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr12' }
+;
+; GFX10: name:            call_use_intrinsic_workitem_id_x_cc_kernel
+; GFX10:       argumentInfo:
+; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr4' }
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll
new file mode 100644
index 0000000000000..bcc960f6a56c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit.ll
@@ -0,0 +1,904 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; Test the generation of the attribute amdgpu-no-flat-scratch-init
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=GFX10 %s
+
+;; tests of addrspacecast
+
+;.
+; GFX9: @gv.fptr0 = external hidden unnamed_addr addrspace(4) constant ptr, align 4
+;.
+; GFX10: @gv.fptr0 = external hidden unnamed_addr addrspace(4) constant ptr, align 4
+;.
+define void @without_global_to_flat_addrspacecast(ptr addrspace(1) %ptr) {
+; GFX9-LABEL: define void @without_global_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(1) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @without_global_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_global_to_flat_addrspacecast_cc_kernel(ptr addrspace(1) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @without_global_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(1) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @without_global_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(1) %ptr
+  ret void
+}
+
+define void @with_global_to_flat_addrspacecast(ptr addrspace(1) %ptr) {
+; GFX9-LABEL: define void @with_global_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_global_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(1) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_global_to_flat_addrspacecast_cc_kernel(ptr addrspace(1) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_global_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_global_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(1) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_region_to_flat_addrspacecast(ptr addrspace(2) %ptr) {
+; GFX9-LABEL: define void @without_region_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(2) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @without_region_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(2) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(2) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_region_to_flat_addrspacecast_cc_kernel(ptr addrspace(2) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @without_region_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(2) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @without_region_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(2) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(2) %ptr
+  ret void
+}
+
+define void @with_region_to_flat_addrspacecast(ptr addrspace(2) %ptr) {
+; GFX9-LABEL: define void @with_region_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_region_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(2) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_region_to_flat_addrspacecast_cc_kernel(ptr addrspace(2) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_region_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_region_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(2) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(2) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define void @without_group_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(3) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @without_group_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(3) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(3) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_group_to_flat_addrspacecast_cc_kernel(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @without_group_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(3) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @without_group_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(3) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(3) %ptr
+  ret void
+}
+
+define void @with_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define void @with_group_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_group_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(3) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_group_to_flat_addrspacecast_cc_kernel(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_group_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_group_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(3) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_constant_to_flat_addrspacecast(ptr addrspace(4) %ptr) {
+; GFX9-LABEL: define void @without_constant_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(4) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @without_constant_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(4) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(4) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_constant_to_flat_addrspacecast_cc_kernel(ptr addrspace(4) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @without_constant_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(4) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @without_constant_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(4) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(4) %ptr
+  ret void
+}
+
+define void @with_constant_to_flat_addrspacecast(ptr addrspace(4) %ptr) {
+; GFX9-LABEL: define void @with_constant_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_constant_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(4) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_constant_to_flat_addrspacecast_cc_kernel(ptr addrspace(4) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_constant_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_constant_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(4) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(4) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(4) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(5) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(5) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(5) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    store volatile i32 0, ptr addrspace(5) [[PTR]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    store volatile i32 0, ptr addrspace(5) [[PTR]], align 4
+; GFX10-NEXT:    ret void
+;
+  store volatile i32 0, ptr addrspace(5) %ptr
+  ret void
+}
+
+define void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @with_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_with_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_with_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_both_with_and_without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_both_with_and_without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_call_without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT:    call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_call_without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX9-NEXT:    call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_call_with_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_call_with_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @call_call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @call_call_both_with_and_without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_call_both_with_and_without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_call_both_with_and_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @call_both_with_and_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @with_cast_call_without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @with_cast_call_without_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_cast_call_without_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @with_cast_call_without_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_cast_call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_cast_call_without_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    call void @without_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @without_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define void @with_cast_call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define void @with_cast_call_with_private_to_flat_addrspacecast(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_cast_call_with_private_to_flat_addrspacecast(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @with_cast_call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @with_cast_call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX9-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX9-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX9-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_cast_call_with_private_to_flat_addrspacecast_cc_kernel(
+; GFX10-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; GFX10-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; GFX10-NEXT:    call void @with_private_to_flat_addrspacecast(ptr addrspace(5) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+;; tests of addrspacecast in a constant
+
+define amdgpu_kernel void @private_constant_expression_use(ptr addrspace(1) nocapture %out) {
+; GFX9-LABEL: define amdgpu_kernel void @private_constant_expression_use(
+; GFX9-SAME: ptr addrspace(1) nocapture [[OUT:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) [[OUT]], align 8
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @private_constant_expression_use(
+; GFX10-SAME: ptr addrspace(1) nocapture [[OUT:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) [[OUT]], align 8
+; GFX10-NEXT:    ret void
+;
+  store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) %out, align 8
+  ret void
+}
+
+;; tests of indirect call, intrinsics, inline asm
+
+@gv.fptr0 = external hidden unnamed_addr addrspace(4) constant ptr, align 4
+
+define void @with_indirect_call() {
+; GFX9-LABEL: define void @with_indirect_call(
+; GFX9-SAME: ) #[[ATTR4:[0-9]+]] {
+; GFX9-NEXT:    [[FPTR:%.*]] = load ptr, ptr addrspace(4) @gv.fptr0, align 8
+; GFX9-NEXT:    call void [[FPTR]]()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @with_indirect_call(
+; GFX10-SAME: ) #[[ATTR4:[0-9]+]] {
+; GFX10-NEXT:    [[FPTR:%.*]] = load ptr, ptr addrspace(4) @gv.fptr0, align 8
+; GFX10-NEXT:    call void [[FPTR]]()
+; GFX10-NEXT:    ret void
+;
+  %fptr = load ptr, ptr addrspace(4) @gv.fptr0
+  call void %fptr()
+  ret void
+}
+
+define amdgpu_kernel void @with_indirect_call_cc_kernel() {
+; GFX9-LABEL: define amdgpu_kernel void @with_indirect_call_cc_kernel(
+; GFX9-SAME: ) #[[ATTR5:[0-9]+]] {
+; GFX9-NEXT:    [[FPTR:%.*]] = load ptr, ptr addrspace(4) @gv.fptr0, align 8
+; GFX9-NEXT:    call void [[FPTR]]()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_indirect_call_cc_kernel(
+; GFX10-SAME: ) #[[ATTR5:[0-9]+]] {
+; GFX10-NEXT:    [[FPTR:%.*]] = load ptr, ptr addrspace(4) @gv.fptr0, align 8
+; GFX10-NEXT:    call void [[FPTR]]()
+; GFX10-NEXT:    ret void
+;
+  %fptr = load ptr, ptr addrspace(4) @gv.fptr0
+  call void %fptr()
+  ret void
+}
+
+define void @call_with_indirect_call() {
+; GFX9-LABEL: define void @call_with_indirect_call(
+; GFX9-SAME: ) #[[ATTR4]] {
+; GFX9-NEXT:    call void @with_indirect_call()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_with_indirect_call(
+; GFX10-SAME: ) #[[ATTR4]] {
+; GFX10-NEXT:    call void @with_indirect_call()
+; GFX10-NEXT:    ret void
+;
+  call void @with_indirect_call()
+  ret void
+}
+
+define amdgpu_kernel void @call_with_indirect_call_cc_kernel() {
+; GFX9-LABEL: define amdgpu_kernel void @call_with_indirect_call_cc_kernel(
+; GFX9-SAME: ) #[[ATTR5]] {
+; GFX9-NEXT:    call void @with_indirect_call()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_with_indirect_call_cc_kernel(
+; GFX10-SAME: ) #[[ATTR5]] {
+; GFX10-NEXT:    call void @with_indirect_call()
+; GFX10-NEXT:    ret void
+;
+  call void @with_indirect_call()
+  ret void
+}
+
+define void @empty() {
+; GFX9-LABEL: define void @empty(
+; GFX9-SAME: ) #[[ATTR0]] {
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @empty(
+; GFX10-SAME: ) #[[ATTR0]] {
+; GFX10-NEXT:    ret void
+;
+  ret void
+}
+
+define void @also_empty() {
+; GFX9-LABEL: define void @also_empty(
+; GFX9-SAME: ) #[[ATTR0]] {
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @also_empty(
+; GFX10-SAME: ) #[[ATTR0]] {
+; GFX10-NEXT:    ret void
+;
+  ret void
+}
+
+define amdgpu_kernel void @indirect_call_known_callees(i1 %cond) {
+; GFX9-LABEL: define amdgpu_kernel void @indirect_call_known_callees(
+; GFX9-SAME: i1 [[COND:%.*]]) #[[ATTR6:[0-9]+]] {
+; GFX9-NEXT:    [[FPTR:%.*]] = select i1 [[COND]], ptr @empty, ptr @also_empty
+; GFX9-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[FPTR]], @also_empty
+; GFX9-NEXT:    br i1 [[TMP1]], label %[[BB2:.*]], label %[[BB3:.*]]
+; GFX9:       [[BB2]]:
+; GFX9-NEXT:    call void @also_empty()
+; GFX9-NEXT:    br label %[[BB6:.*]]
+; GFX9:       [[BB3]]:
+; GFX9-NEXT:    br i1 true, label %[[BB4:.*]], label %[[BB5:.*]]
+; GFX9:       [[BB4]]:
+; GFX9-NEXT:    call void @empty()
+; GFX9-NEXT:    br label %[[BB6]]
+; GFX9:       [[BB5]]:
+; GFX9-NEXT:    unreachable
+; GFX9:       [[BB6]]:
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @indirect_call_known_callees(
+; GFX10-SAME: i1 [[COND:%.*]]) #[[ATTR6:[0-9]+]] {
+; GFX10-NEXT:    [[FPTR:%.*]] = select i1 [[COND]], ptr @empty, ptr @also_empty
+; GFX10-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[FPTR]], @also_empty
+; GFX10-NEXT:    br i1 [[TMP1]], label %[[BB2:.*]], label %[[BB3:.*]]
+; GFX10:       [[BB2]]:
+; GFX10-NEXT:    call void @also_empty()
+; GFX10-NEXT:    br label %[[BB6:.*]]
+; GFX10:       [[BB3]]:
+; GFX10-NEXT:    br i1 true, label %[[BB4:.*]], label %[[BB5:.*]]
+; GFX10:       [[BB4]]:
+; GFX10-NEXT:    call void @empty()
+; GFX10-NEXT:    br label %[[BB6]]
+; GFX10:       [[BB5]]:
+; GFX10-NEXT:    unreachable
+; GFX10:       [[BB6]]:
+; GFX10-NEXT:    ret void
+;
+  %fptr = select i1 %cond, ptr @empty, ptr @also_empty
+  call void %fptr()
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workgroup.id.x()
+
+define void @use_intrinsic_workitem_id_x() {
+; GFX9-LABEL: define void @use_intrinsic_workitem_id_x(
+; GFX9-SAME: ) #[[ATTR8:[0-9]+]] {
+; GFX9-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GFX9-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) null, align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @use_intrinsic_workitem_id_x(
+; GFX10-SAME: ) #[[ATTR8:[0-9]+]] {
+; GFX10-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GFX10-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) null, align 4
+; GFX10-NEXT:    ret void
+;
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, ptr addrspace(1) null
+  ret void
+}
+
+define amdgpu_kernel void @use_intrinsic_workitem_id_x_cc_kernel() {
+; GFX9-LABEL: define amdgpu_kernel void @use_intrinsic_workitem_id_x_cc_kernel(
+; GFX9-SAME: ) #[[ATTR1]] {
+; GFX9-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GFX9-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) null, align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @use_intrinsic_workitem_id_x_cc_kernel(
+; GFX10-SAME: ) #[[ATTR1]] {
+; GFX10-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GFX10-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) null, align 4
+; GFX10-NEXT:    ret void
+;
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, ptr addrspace(1) null
+  ret void
+}
+
+define void @call_use_intrinsic_workitem_id_x() {
+; GFX9-LABEL: define void @call_use_intrinsic_workitem_id_x(
+; GFX9-SAME: ) #[[ATTR8]] {
+; GFX9-NEXT:    call void @use_intrinsic_workitem_id_x()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define void @call_use_intrinsic_workitem_id_x(
+; GFX10-SAME: ) #[[ATTR8]] {
+; GFX10-NEXT:    call void @use_intrinsic_workitem_id_x()
+; GFX10-NEXT:    ret void
+;
+  call void @use_intrinsic_workitem_id_x()
+  ret void
+}
+
+define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
+; GFX9-LABEL: define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel(
+; GFX9-SAME: ) #[[ATTR9:[0-9]+]] {
+; GFX9-NEXT:    call void @use_intrinsic_workitem_id_x()
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel(
+; GFX10-SAME: ) #[[ATTR9:[0-9]+]] {
+; GFX10-NEXT:    call void @use_intrinsic_workitem_id_x()
+; GFX10-NEXT:    ret void
+;
+  call void @use_intrinsic_workitem_id_x()
+  ret void
+}
+
+define amdgpu_kernel void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @calls_intrin_ascast_cc_kernel(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    [[TMP1:%.*]] = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) [[PTR]])
+; GFX9-NEXT:    store volatile i32 7, ptr [[TMP1]], align 4
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @calls_intrin_ascast_cc_kernel(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) [[PTR]])
+; GFX10-NEXT:    store volatile i32 7, ptr [[TMP1]], align 4
+; GFX10-NEXT:    ret void
+;
+  %1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) %ptr)
+  store volatile i32 7, ptr %1, align 4
+  ret void
+}
+
+define amdgpu_kernel void @call_calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) {
+; GFX9-LABEL: define amdgpu_kernel void @call_calls_intrin_ascast_cc_kernel(
+; GFX9-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX9-NEXT:    call void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) [[PTR]])
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @call_calls_intrin_ascast_cc_kernel(
+; GFX10-SAME: ptr addrspace(3) [[PTR:%.*]]) #[[ATTR3]] {
+; GFX10-NEXT:    call void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) [[PTR]])
+; GFX10-NEXT:    ret void
+;
+  call void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @with_inline_asm() {
+; GFX9-LABEL: define amdgpu_kernel void @with_inline_asm(
+; GFX9-SAME: ) #[[ATTR6]] {
+; GFX9-NEXT:    call void asm sideeffect "
+; GFX9-NEXT:    ret void
+;
+; GFX10-LABEL: define amdgpu_kernel void @with_inline_asm(
+; GFX10-SAME: ) #[[ATTR6]] {
+; GFX10-NEXT:    call void asm sideeffect "
+; GFX10-NEXT:    ret void
+;
+  call void asm sideeffect "; use $0", "a"(i32 poison)
+  ret void
+}
+
+;.
+; GFX9: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR4]] = { "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR5]] = { "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR6]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx900" }
+; GFX9: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX9: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+;.
+; GFX10: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,20" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,20" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR4]] = { "amdgpu-waves-per-eu"="8,20" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR5]] = { "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR6]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx1010" }
+; GFX10: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,20" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+; GFX10: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 9a647f04d43da..bc359d6ff3aaa 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -25054,26 +25054,26 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GCN-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GCN-NEXT:    s_mov_b32 s5, 0x7f800000
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x41b17218
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x41b17218
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GCN-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-NEXT:    v_log_f32_e32 v0, v0
-; GCN-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
-; GCN-NEXT:    v_sub_f32_e32 v3, v0, v1
-; GCN-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v1
-; GCN-NEXT:    v_mul_f32_e32 v1, 0x3f317000, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
+; GCN-NEXT:    v_sub_f32_e32 v3, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v2
+; GCN-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
 ; GCN-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v3
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v3
 ; GCN-NEXT:    v_add_f32_e32 v3, v4, v3
 ; GCN-NEXT:    v_add_f32_e32 v3, v5, v3
-; GCN-NEXT:    v_add_f32_e32 v1, v1, v3
+; GCN-NEXT:    v_add_f32_e32 v2, v2, v3
 ; GCN-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s5
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -25084,10 +25084,10 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX7-NEXT:    s_mov_b32 s4, 0x800000
-; GFX7-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX7-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    v_log_f32_e32 v0, v0
 ; GFX7-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX7-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -25109,10 +25109,10 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX8-NEXT:    s_mov_b32 s4, 0x800000
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX8-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX8-NEXT:    v_log_f32_e32 v0, v0
 ; GFX8-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -25143,10 +25143,10 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX9-NEXT:    s_mov_b32 s4, 0x800000
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX9-NEXT:    v_log_f32_e32 v0, v0
 ; GFX9-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX9-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -25174,8 +25174,9 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX10-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX10-NEXT:    v_log_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX10-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
@@ -25199,28 +25200,30 @@ define bfloat @v_log_bf16(bfloat %a) {
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX11-NEXT:    v_log_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX11-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
 ; GFX11-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_add_f32_e32 v1, v1, v2
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_sub_f32_e32 v0, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.log.bf16(bfloat %a)
@@ -25233,14 +25236,14 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GCN-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x42000000
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GCN-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-NEXT:    v_log_f32_e32 v0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -25251,10 +25254,10 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX7-NEXT:    s_mov_b32 s4, 0x800000
-; GFX7-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX7-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    v_log_f32_e32 v0, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -25267,10 +25270,10 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX8-NEXT:    s_mov_b32 s4, 0x800000
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX8-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX8-NEXT:    v_log_f32_e32 v0, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -25290,9 +25293,9 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX9-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX9-NEXT:    v_log_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -25311,9 +25314,10 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX10-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX10-NEXT:    v_log_f32_e32 v0, v0
 ; GFX10-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
@@ -25330,20 +25334,21 @@ define bfloat @v_log2_bf16(bfloat %a) {
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX11-NEXT:    v_log_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.log2.bf16(bfloat %a)
@@ -25356,26 +25361,26 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GCN-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GCN-NEXT:    s_mov_b32 s5, 0x7f800000
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x411a209b
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x411a209b
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GCN-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-NEXT:    v_log_f32_e32 v0, v0
-; GCN-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
-; GCN-NEXT:    v_sub_f32_e32 v3, v0, v1
-; GCN-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v1
-; GCN-NEXT:    v_mul_f32_e32 v1, 0x3e9a2000, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
+; GCN-NEXT:    v_sub_f32_e32 v3, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v2
+; GCN-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
 ; GCN-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v3
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v3
 ; GCN-NEXT:    v_add_f32_e32 v3, v4, v3
 ; GCN-NEXT:    v_add_f32_e32 v3, v5, v3
-; GCN-NEXT:    v_add_f32_e32 v1, v1, v3
+; GCN-NEXT:    v_add_f32_e32 v2, v2, v3
 ; GCN-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s5
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -25386,10 +25391,10 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX7-NEXT:    s_mov_b32 s4, 0x800000
-; GFX7-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX7-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    v_log_f32_e32 v0, v0
 ; GFX7-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX7-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -25411,10 +25416,10 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX8-NEXT:    s_mov_b32 s4, 0x800000
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX8-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX8-NEXT:    v_log_f32_e32 v0, v0
 ; GFX8-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -25445,10 +25450,10 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX9-NEXT:    s_mov_b32 s4, 0x800000
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX9-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX9-NEXT:    v_log_f32_e32 v0, v0
 ; GFX9-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX9-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -25476,8 +25481,9 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX10-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX10-NEXT:    v_log_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX10-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
@@ -25501,28 +25507,30 @@ define bfloat @v_log10_bf16(bfloat %a) {
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX11-NEXT:    v_log_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX11-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
 ; GFX11-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_add_f32_e32 v1, v1, v2
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_sub_f32_e32 v0, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.log10.bf16(bfloat %a)
@@ -25719,14 +25727,14 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
 ; GCN-NEXT:    s_mov_b32 s4, 0xc2fc0000
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0x42800000
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x1f800000
+; GCN-NEXT:    v_not_b32_e32 v2, 63
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GCN-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
+; GCN-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -25741,9 +25749,9 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GFX7-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    v_exp_f32_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_not_b32_e32 v1, 63
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -25757,9 +25765,9 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GFX8-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GFX8-NEXT:    v_exp_f32_e32 v0, v0
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_not_b32_e32 v1, 63
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX8-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
@@ -25779,10 +25787,10 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; GFX9-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX9-NEXT:    v_exp_f32_e32 v0, v0
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
+; GFX9-NEXT:    v_not_b32_e32 v1, 63
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s4
 ; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
@@ -25797,10 +25805,10 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x1f800000, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
 ; GFX10-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX10-NEXT:    v_exp_f32_e32 v0, v0
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -25816,12 +25824,12 @@ define bfloat @v_exp2_bf16(bfloat %a) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x1f800000, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
 ; GFX11-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_exp_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index dd608ef0e5a53..b08ab5a2dc422 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-- -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
 
 ; ========= Single bit functions =========
 
@@ -45,10 +45,17 @@ define amdgpu_ps float @not_and_and_not_and(i32 %a, i32 %b, i32 %c) {
 }
 
 define amdgpu_ps float @not_and_and_and(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: not_and_and_and:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:8
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: not_and_and_and:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:8
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: not_and_and_and:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v0, v0
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %nota = xor i32 %a, -1
   %and1 = and i32 %nota, %c
   %and2 = and i32 %and1, %b
@@ -70,10 +77,17 @@ define amdgpu_ps float @and_not_and_not_and(i32 %a, i32 %b, i32 %c) {
 }
 
 define amdgpu_ps float @and_not_and_and(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: and_not_and_and:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x20
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: and_not_and_and:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x20
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: and_not_and_and:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v1, v1
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %notb = xor i32 %b, -1
   %and1 = and i32 %a, %c
   %and2 = and i32 %and1, %notb
@@ -82,10 +96,17 @@ define amdgpu_ps float @and_not_and_and(i32 %a, i32 %b, i32 %c) {
 }
 
 define amdgpu_ps float @and_and_not_and(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: and_and_not_and:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x40
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: and_and_not_and:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x40
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: and_and_not_and:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v2, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %notc = xor i32 %c, -1
   %and1 = and i32 %a, %notc
   %and2 = and i32 %and1, %b
@@ -94,10 +115,16 @@ define amdgpu_ps float @and_and_not_and(i32 %a, i32 %b, i32 %c) {
 }
 
 define amdgpu_ps float @and_and_and(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: and_and_and:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x80
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: and_and_and:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x80
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: and_and_and:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %and1 = and i32 %a, %c
   %and2 = and i32 %and1, %b
   %ret_cast = bitcast i32 %and2 to float
@@ -107,10 +134,16 @@ define amdgpu_ps float @and_and_and(i32 %a, i32 %b, i32 %c) {
 ; ========= Multi bit functions =========
 
 define amdgpu_ps float @test_12(i32 %a, i32 %b) {
-; GCN-LABEL: test_12:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v0 bitop3:0xc
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_12:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v0 bitop3:0xc
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_12:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v0, v0
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %nota = xor i32 %a, -1
   %and1 = and i32 %nota, %b
   %ret_cast = bitcast i32 %and1 to float
@@ -118,10 +151,17 @@ define amdgpu_ps float @test_12(i32 %a, i32 %b) {
 }
 
 define amdgpu_ps float @test_63(i32 %a, i32 %b) {
-; GCN-LABEL: test_63:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v0 bitop3:0x3f
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_63:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v1, v0 bitop3:0x3f
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_63:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v0, v0
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v1, v1
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %nota = xor i32 %a, -1
   %notb = xor i32 %b, -1
   %or = or i32 %nota, %notb
@@ -143,10 +183,17 @@ define amdgpu_ps float @test_59(i32 %a, i32 %b, i32 %c) {
 }
 
 define amdgpu_ps float @test_126(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: test_126:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v0, v0, v2, v1 bitop3:0x7e
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_126:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v0, v2, v1 bitop3:0x7e
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_126:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_xor_b32_e32 v1, v0, v1
+; GFX950-GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %xor1 = xor i32 %a, %b
   %xor2 = xor i32 %a, %c
   %or = or i32 %xor1, %xor2
@@ -167,9 +214,9 @@ define amdgpu_ps float @test_12_src_overflow(i32 %a, i32 %b, i32 %c) {
 ;
 ; GFX950-GISEL-LABEL: test_12_src_overflow:
 ; GFX950-GISEL:       ; %bb.0:
-; GFX950-GISEL-NEXT:    v_bitop3_b32 v3, v0, v2, v0 bitop3:0xc
-; GFX950-GISEL-NEXT:    v_bitop3_b32 v0, v0, v2, v0 bitop3:3
-; GFX950-GISEL-NEXT:    v_bitop3_b32 v0, v3, v1, v0 bitop3:0xc8
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v0, v0
+; GFX950-GISEL-NEXT:    v_bfi_b32 v0, v2, v0, v0
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %nota = xor i32 %a, -1
   %notc = xor i32 %c, -1
@@ -185,13 +232,27 @@ define amdgpu_ps float @test_12_src_overflow(i32 %a, i32 %b, i32 %c) {
 ; This could be a single LOP3 operation with tbl = 100, but Src vector exhausted during search.
 
 define amdgpu_ps float @test_100_src_overflow(i32 %a, i32 %b, i32 %c) {
-; GCN-LABEL: test_100_src_overflow:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_bitop3_b32 v3, v1, v2, v0 bitop3:0x10
-; GCN-NEXT:    v_bitop3_b32 v4, v0, v2, v1 bitop3:0x40
-; GCN-NEXT:    v_bitop3_b32 v0, v1, v2, v0 bitop3:0x20
-; GCN-NEXT:    v_or3_b32 v0, v3, v4, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_100_src_overflow:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v3, v1, v2, v0 bitop3:0x10
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v4, v0, v2, v1 bitop3:0x40
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, v1, v2, v0 bitop3:0x20
+; GFX950-SDAG-NEXT:    v_or3_b32 v0, v3, v4, v0
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_100_src_overflow:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v3, v2, v0
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v3, v3
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v4, v1
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v3, v1, v3
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v4, v0, v4
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v1, v0
+; GFX950-GISEL-NEXT:    v_not_b32_e32 v1, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v4, v4, v2
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_or3_b32 v0, v3, v4, v0
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %or1 = or i32 %c, %a
   %not1 = xor i32 %or1, -1
   %and1 = and i32 %b, %not1
@@ -260,12 +321,19 @@ define amdgpu_ps float @uniform_3_op(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
 }
 
 define amdgpu_ps float @uniform_4_op(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
-; GCN-LABEL: uniform_4_op:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mov_b32_e32 v0, s1
-; GCN-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-NEXT:    v_bitop3_b32 v0, s0, v0, v1 bitop3:2
-; GCN-NEXT:    ; return to shader part epilog
+; GFX950-SDAG-LABEL: uniform_4_op:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v0, s1
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX950-SDAG-NEXT:    v_bitop3_b32 v0, s0, v0, v1 bitop3:2
+; GFX950-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: uniform_4_op:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_andn2_b32 s0, s2, s0
+; GFX950-GISEL-NEXT:    s_andn2_b32 s0, s0, s1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX950-GISEL-NEXT:    ; return to shader part epilog
   %nota = xor i32 %a, -1
   %notb = xor i32 %b, -1
   %and1 = and i32 %nota, %c
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
index 84ea2beb8d04b..384beae07ce2e 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
@@ -13,122 +13,119 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1
   ; GCN-NEXT:   [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg0, addrspace 6)
   ; GCN-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s64) from %ir.arg0, align 16, addrspace 6)
-  ; GCN-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
-  ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 8
-  ; GCN-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], killed [[S_MOV_B32_1]], implicit-def dead $scc
-  ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE killed [[S_ADD_I32_]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1
-  ; GCN-NEXT:   [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[REG_SEQUENCE1]], 0, 0 :: (dereferenceable invariant load (s64) from %ir.arg0 + 8, basealign 16, addrspace 6)
-  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
-  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
-  ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
+  ; GCN-NEXT:   [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[REG_SEQUENCE]], 8, 0 :: (dereferenceable invariant load (s64) from %ir.arg0 + 8, basealign 16, addrspace 6)
+  ; GCN-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
+  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+  ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1, killed [[COPY3]], %subreg.sub2, killed [[COPY2]], %subreg.sub3
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 64
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 128
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 64
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 128
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_2]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE2]], [[COPY7]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[COPY7]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 72
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 144
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 72
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 144
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_4]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE2]], [[COPY9]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[COPY9]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 80
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 160
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 80
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 160
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY10]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_6]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE2]], [[COPY11]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[COPY11]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 88
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sreg_32 = S_MOV_B32 176
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE3]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 88
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 176
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY12:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[S_MOV_B32_8]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[S_MOV_B32_9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE3]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY13:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY13]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY13]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 96
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 192
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sreg_32 = S_MOV_B32 96
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 192
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY14:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY14]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_10]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_11]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[REG_SEQUENCE1]], [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[REG_SEQUENCE1]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY15:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[REG_SEQUENCE2]], [[COPY15]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[REG_SEQUENCE1]], [[COPY15]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 104
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 208
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 104
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 208
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY16:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY16]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[REG_SEQUENCE2]], [[S_MOV_B32_12]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_13]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[REG_SEQUENCE1]], [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[REG_SEQUENCE1]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY17:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[REG_SEQUENCE2]], [[COPY17]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[REG_SEQUENCE1]], [[COPY17]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 112
+  ; GCN-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 112
   ; GCN-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 224
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 224
   ; GCN-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY22:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
@@ -137,27 +134,27 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY23:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY23]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY23]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY24:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY24]], [[REG_SEQUENCE2]], [[S_MOV_B32_14]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY24]], [[REG_SEQUENCE1]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY25:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY25]], [[REG_SEQUENCE2]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_15]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY25]], [[REG_SEQUENCE1]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY26:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY27:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY26]], [[REG_SEQUENCE2]], [[COPY27]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY26]], [[REG_SEQUENCE1]], [[COPY27]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY28:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 120
+  ; GCN-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 120
   ; GCN-NEXT:   [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 240
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 240
   ; GCN-NEXT:   [[COPY30:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY31:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY32:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
@@ -165,26 +162,26 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY33:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY33]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY33]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY34:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY34]], [[REG_SEQUENCE2]], [[S_MOV_B32_16]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY34]], [[REG_SEQUENCE1]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY35:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY35]], [[REG_SEQUENCE2]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_17]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY35]], [[REG_SEQUENCE1]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY36:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY37:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY36]], [[REG_SEQUENCE2]], [[COPY37]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY36]], [[REG_SEQUENCE1]], [[COPY37]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY38:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY38]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY39:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY39]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 256
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY39]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 256
   ; GCN-NEXT:   [[COPY40:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY40]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY40]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY41:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY42:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY41]], [[S_LOAD_DWORDX4_IMM]], [[COPY42]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
@@ -192,54 +189,54 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY43:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY43]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY43]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY44:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY44]], [[REG_SEQUENCE2]], [[S_MOV_B32_3]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY44]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY45:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY45]], [[REG_SEQUENCE2]], [[S_MOV_B32_18]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_18]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY45]], [[REG_SEQUENCE1]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY46:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY47:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY46]], [[REG_SEQUENCE2]], [[COPY47]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY46]], [[REG_SEQUENCE1]], [[COPY47]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY48:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY48]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 136
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY48]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 136
   ; GCN-NEXT:   [[COPY49:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY49]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 272
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY49]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 272
   ; GCN-NEXT:   [[COPY50:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY50]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY50]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY51:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY52:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY51]], [[S_LOAD_DWORDX4_IMM]], [[COPY52]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY51]], [[S_LOAD_DWORDX4_IMM]], [[COPY52]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY53:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY53]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY53]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY54:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY54]], [[REG_SEQUENCE2]], [[S_MOV_B32_19]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY54]], [[REG_SEQUENCE1]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY55:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY55]], [[REG_SEQUENCE2]], [[S_MOV_B32_20]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_20]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY55]], [[REG_SEQUENCE1]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY56:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY57:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY56]], [[REG_SEQUENCE2]], [[COPY57]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE3]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY56]], [[REG_SEQUENCE1]], [[COPY57]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY58:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[COPY58]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY59:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 288
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 288
   ; GCN-NEXT:   [[COPY60:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY61:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY62:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY61]], [[S_LOAD_DWORDX4_IMM]], [[COPY62]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
@@ -247,44 +244,44 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY63:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[COPY63]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[COPY63]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY64:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY64]], [[REG_SEQUENCE2]], [[S_MOV_B32_5]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY64]], [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY65:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY65]], [[REG_SEQUENCE2]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN1]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_21]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY65]], [[REG_SEQUENCE1]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN1]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY66:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY67:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY66]], [[REG_SEQUENCE2]], [[COPY67]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY66]], [[REG_SEQUENCE1]], [[COPY67]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY68:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[COPY68]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 152
+  ; GCN-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 152
   ; GCN-NEXT:   [[COPY69:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sreg_32 = S_MOV_B32 304
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 304
   ; GCN-NEXT:   [[COPY70:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY71:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY72:%[0-9]+]]:sreg_32 = COPY [[COPY]]
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY71]], [[S_LOAD_DWORDX4_IMM]], [[COPY72]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY73:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[COPY73]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[COPY73]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY74:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY74]], [[REG_SEQUENCE2]], [[S_MOV_B32_22]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY74]], [[REG_SEQUENCE1]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY75:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY75]], [[REG_SEQUENCE2]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1]], [[REG_SEQUENCE4]], [[REG_SEQUENCE2]], [[S_MOV_B32_23]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY75]], [[REG_SEQUENCE1]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY76:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY77:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY76]], [[REG_SEQUENCE2]], [[COPY77]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY]], [[REG_SEQUENCE2]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY76]], [[REG_SEQUENCE1]], [[COPY77]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
   ; GCN-NEXT:   S_ENDPGM 0
 bb.0:
   %tmp0 = load <4 x i32>, ptr addrspace(6) %arg0, align 16, !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-undef-subreg-use-invalid-lanemask.mir b/llvm/test/CodeGen/AMDGPU/coalescer-undef-subreg-use-invalid-lanemask.mir
new file mode 100644
index 0000000000000..70b92831a0ecc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-undef-subreg-use-invalid-lanemask.mir
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+
+# Test that an invalid subreg range is not introduced due to the undef
+# %1.sub0 use. An undef use with a subregister index would end up
+# introducing subranges for the empty and full lanemasks.
+
+---
+name:            merge_with_undef_subreg_use_subrange_lanemask_is_invalid
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: merge_with_undef_subreg_use_subrange_lanemask_is_invalid
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr8_sgpr9
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0 :: (load (s128), addrspace 4)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit undef $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[S_LOAD_DWORDX4_IMM:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM:%[0-9]+]].sub1:sgpr_128 = COPY undef [[S_LOAD_DWORDX4_IMM]].sub0
+  ; CHECK-NEXT:   S_ENDPGM 0, implicit [[S_LOAD_DWORDX4_IMM]]
+  bb.0:
+    liveins: $sgpr8_sgpr9
+
+    %0:sgpr_64 = COPY $sgpr8_sgpr9
+    %1:sgpr_128 = S_LOAD_DWORDX4_IMM %0, 0, 0 :: (load (s128), addrspace 4)
+
+  bb.1:
+    %2:sgpr_128 = COPY %1
+    S_CBRANCH_VCCNZ %bb.3, implicit undef $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    undef %3.sub0:sgpr_128 = S_MOV_B32 0
+    %2:sgpr_128 = COPY killed %3
+
+  bb.3:
+    %4:sgpr_128 = COPY killed %2
+    %4.sub1:sgpr_128 = COPY undef %1.sub0
+    S_ENDPGM 0, implicit %4
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/copysign-simplify-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/copysign-simplify-demanded-bits.ll
index b2e4117096ce5..fdc9704a3784e 100644
--- a/llvm/test/CodeGen/AMDGPU/copysign-simplify-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/copysign-simplify-demanded-bits.ll
@@ -337,9 +337,9 @@ define float @test_copysign_pow_fast_f32__integral_y(float %x, i32 %y.i) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX9-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4f800000
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
-; GFX9-NEXT:    v_mul_f32_e64 v3, |v0|, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX9-NEXT:    v_ldexp_f32 v3, |v0|, v3
 ; GFX9-NEXT:    v_log_f32_e32 v3, v3
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x42000000
@@ -353,10 +353,10 @@ define float @test_copysign_pow_fast_f32__integral_y(float %x, i32 %y.i) {
 ; GFX9-NEXT:    v_fma_f32 v2, v2, v1, v3
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v1, v1
 ; GFX9-NEXT:    v_exp_f32_e32 v2, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x1f800000
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, 1.0, v3, vcc
+; GFX9-NEXT:    v_not_b32_e32 v3, 63
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 31, v1
-; GFX9-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GFX9-NEXT:    v_ldexp_f32 v2, v2, v3
 ; GFX9-NEXT:    v_and_b32_e32 v0, v1, v0
 ; GFX9-NEXT:    s_brev_b32 s4, -2
 ; GFX9-NEXT:    v_bfi_b32 v0, s4, v2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index cce0fb7e003c5..5b72795ba07ea 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -270,40 +270,36 @@ define float @fmul_select_f32_test7_sel_log2val_pos59_pos92(float %x, i32 %bool.
 ; GFX7-LABEL: fmul_select_f32_test7_sel_log2val_pos59_pos92:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0x6d800000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x5d000000
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x5c
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v3, 59, vcc
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f32_test7_sel_log2val_pos59_pos92:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x6d800000
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x5d000000
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x5c
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, 59, vcc
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f32_test7_sel_log2val_pos59_pos92:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x5d000000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x6d800000, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 0x5c, 59, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f32_test7_sel_log2val_pos59_pos92:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x5d000000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x6d800000, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0x5c, 59, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, float 0x43A0000000000000, float 0x45B0000000000000
@@ -440,40 +436,36 @@ define float @fmul_select_f32_test11_sel_log2val_pos78_pos56(float %x, i32 %bool
 ; GFX7-LABEL: fmul_select_f32_test11_sel_log2val_pos78_pos56:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0xdb800000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0xe6800000
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x4e
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 56, v3, vcc
+; GFX7-NEXT:    v_ldexp_f32_e64 v0, -v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f32_test11_sel_log2val_pos78_pos56:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0xdb800000
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xe6800000
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4e
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 56, v3, vcc
+; GFX9-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f32_test11_sel_log2val_pos78_pos56:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0xe6800000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0xdb800000, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 56, 0x4e, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f32_test11_sel_log2val_pos78_pos56:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0xe6800000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0xdb800000, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 56, 0x4e, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, float 0xC4D0000000000000, float 0xC370000000000000
@@ -485,40 +477,40 @@ define float @fmul_select_f32_test12_sel_log2val_neg48_pos68(float %x, i32 %bool
 ; GFX7-LABEL: fmul_select_f32_test12_sel_log2val_neg48_pos68:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0x61800000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x27800000
+; GFX7-NEXT:    v_mov_b32_e32 v3, 0x44
+; GFX7-NEXT:    v_not_b32_e32 v4, 47
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f32_test12_sel_log2val_neg48_pos68:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x61800000
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x27800000
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x44
+; GFX9-NEXT:    v_not_b32_e32 v4, 47
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f32_test12_sel_log2val_neg48_pos68:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x27800000
+; GFX1030-NEXT:    v_not_b32_e32 v3, 47
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x61800000, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x44, v3, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f32_test12_sel_log2val_neg48_pos68:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x27800000
+; GFX1100-NEXT:    v_not_b32_e32 v3, 47
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x61800000, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x44, v3, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, float 0x3CF0000000000000, float 0x4430000000000000
@@ -530,40 +522,34 @@ define double @fmul_select_f64_test1(double %x, i32 %bool.arg1, i32 %bool.arg2)
 ; GFX7-LABEL: fmul_select_f64_test1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v4, 2.0, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v4, 2.0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test1:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e64 v5, 0x3ff00000, 2.0, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test1:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1100-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v5, 0x3ff00000, 2.0, vcc_lo
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double 2.000000e+00, double 1.000000e+00
@@ -575,43 +561,34 @@ define double @fmul_select_f64_test2(double %x, i32 %bool.arg1, i32 %bool.arg2)
 ; GFX7-LABEL: fmul_select_f64_test2:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x3fe00000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x3fe00000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test2:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v5, 0x3fe00000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v5, 0x3ff00000, v5, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test2:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v5, 0x3fe00000 :: v_dual_mov_b32 v4, 0
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v5, 0x3ff00000, v5, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double 5.000000e-01, double 1.000000e+00
@@ -623,57 +600,46 @@ define <2 x double> @fmul_select_v2f64_test3(<2 x double> %x, <2 x i32> %bool.ar
 ; GFX7-LABEL: fmul_select_v2f64_test3:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v11, 2.0, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v9, v8
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v11, 2.0, vcc
-; GFX7-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_v2f64_test3:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v11, 2.0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v11, 2.0, vcc
-; GFX9-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_v2f64_test3:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1030-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1030-NEXT:    v_cndmask_b32_e64 v11, 0x3ff00000, 2.0, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1030-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1030-NEXT:    v_cndmask_b32_e64 v9, 0x3ff00000, 2.0, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1030-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX1030-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v5
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_v2f64_test3:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1100-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v11, 0x3ff00000, 2.0, vcc_lo
+; GFX1100-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1100-NEXT:    v_cndmask_b32_e64 v9, 0x3ff00000, 2.0, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX1100-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v5
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> <double 1.000000e+00, double 1.000000e+00>
@@ -685,59 +651,46 @@ define <2 x double> @fmul_select_v2f64_test4(<2 x double> %x, <2 x i32> %bool.ar
 ; GFX7-LABEL: fmul_select_v2f64_test4:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX7-NEXT:    v_mov_b32_e32 v12, 0x3fe00000
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v9, v8
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc
-; GFX7-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_v2f64_test4:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX9-NEXT:    v_mov_b32_e32 v12, 0x3fe00000
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc
-; GFX9-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_v2f64_test4:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v9, 0x3fe00000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1030-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v11, 0x3ff00000, v9, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc_lo
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1030-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1030-NEXT:    v_cndmask_b32_e32 v9, 0x3ff00000, v9, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1030-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX1030-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v5
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_v2f64_test4:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v9, 0x3fe00000 :: v_dual_mov_b32 v8, 0
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_dual_mov_b32 v10, v8 :: v_dual_cndmask_b32 v11, 0x3ff00000, v9
+; GFX1100-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc_lo
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1100-NEXT:    v_cndmask_b32_e32 v9, 0x3ff00000, v9, vcc_lo
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v4
+; GFX1100-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v5
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x double> <double 5.000000e-01, double 5.000000e-01>, <2 x double> <double 1.000000e+00, double 1.000000e+00>
@@ -749,43 +702,34 @@ define double @fmul_select_f64_test5(double %x, i32 %bool.arg1, i32 %bool.arg2)
 ; GFX7-LABEL: fmul_select_f64_test5:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0xbff00000
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0xbfe00000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test5:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xbff00000
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0xbfe00000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test5:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v5, 0xbfe00000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v5, 0xbff00000, v5, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test5:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v5, 0xbfe00000 :: v_dual_mov_b32 v4, 0
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v5, 0xbff00000, v5, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double -5.000000e-01, double -1.000000e+00
@@ -797,40 +741,34 @@ define double @fmul_select_f64_test6(double %x, i32 %bool.arg1, i32 %bool.arg2)
 ; GFX7-LABEL: fmul_select_f64_test6:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0xbff00000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e64 v3, v4, -2.0, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test6:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xbff00000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v4, -2.0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test6:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e64 v5, 0xbff00000, -2.0, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test6:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1100-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v5, 0xbff00000, -2.0, vcc_lo
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double -2.000000e+00, double -1.000000e+00
@@ -887,43 +825,34 @@ define double @fmul_select_f64_test8(double %x, i32 %bool.arg1, i32 %bool.arg2)
 ; GFX7-LABEL: fmul_select_f64_test8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0xc0400000
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0xc0100000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 5, 2, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc0400000
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0xc0100000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 5, 2, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test8:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v5, 0xc0100000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v5, 0xc0400000, v5, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 5, 2, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test8:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v5, 0xc0100000 :: v_dual_mov_b32 v4, 0
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v5, 0xc0400000, v5, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 5, 2, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double -4.000000e+00, double -3.200000e+01
@@ -935,57 +864,46 @@ define <2 x double> @fmul_select_v2f64_test9(<2 x double> %x, <2 x i32> %bool.ar
 ; GFX7-LABEL: fmul_select_v2f64_test9:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v11, 0xbff00000
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX7-NEXT:    v_cndmask_b32_e64 v10, v11, -2.0, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v9, v8
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v11, -2.0, vcc
-; GFX7-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v4
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[2:3], -v[2:3], v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_v2f64_test9:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0xbff00000
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v11, -2.0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v11, -2.0, vcc
-; GFX9-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[2:3], -v[2:3], v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_v2f64_test9:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1030-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1030-NEXT:    v_cndmask_b32_e64 v11, 0xbff00000, -2.0, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1030-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1030-NEXT:    v_cndmask_b32_e64 v9, 0xbff00000, -2.0, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1030-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v4
+; GFX1030-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[2:3], -v[2:3], v5
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_v2f64_test9:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1100-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v11, 0xbff00000, -2.0, vcc_lo
+; GFX1100-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1100-NEXT:    v_cndmask_b32_e64 v9, 0xbff00000, -2.0, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v4
+; GFX1100-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f64 v[2:3], -v[2:3], v5
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x double> <double -2.000000e+00, double -2.000000e+00>, <2 x double> <double -1.000000e+00, double -1.000000e+00>
@@ -997,61 +915,56 @@ define <2 x double> @fmul_select_v2f64_test10(<2 x double> %x, <2 x i32> %bool.a
 ; GFX7-LABEL: fmul_select_v2f64_test10:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v8, 0
-; GFX7-NEXT:    v_mov_b32_e32 v9, 0xbff00000
-; GFX7-NEXT:    v_mov_b32_e32 v10, 0x3fe00000
+; GFX7-NEXT:    v_mov_b32_e32 v8, 0xbff00000
+; GFX7-NEXT:    v_mov_b32_e32 v9, 0x3fe00000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX7-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v9, v10, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v9, v8
+; GFX7-NEXT:    v_cndmask_b32_e32 v9, v8, v9, vcc
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX7-NEXT:    v_cndmask_b32_e64 v9, v11, 2.0, vcc
-; GFX7-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX7-NEXT:    v_mov_b32_e32 v8, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX7-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_v2f64_test10:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0xbff00000
-; GFX9-NEXT:    v_mov_b32_e32 v10, 0x3fe00000
+; GFX9-NEXT:    v_mov_b32_e32 v8, 0xbff00000
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x3fe00000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v6
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0x3ff00000
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v9, v10, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v9, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, v8, v9, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[9:10]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v11, 2.0, vcc
-; GFX9-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v8, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX9-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_v2f64_test10:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v9, 0x3fe00000
+; GFX1030-NEXT:    v_mov_b32_e32 v8, 0x3fe00000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX1030-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v11, 0xbff00000, v9, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e32 v9, 0xbff00000, v8, vcc_lo
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1030-NEXT:    v_mov_b32_e32 v10, v8
-; GFX1030-NEXT:    v_cndmask_b32_e64 v9, 0x3ff00000, 2.0, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1030-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1030-NEXT:    v_mov_b32_e32 v8, 0
+; GFX1030-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX1030-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_v2f64_test10:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v9, 0x3fe00000 :: v_dual_mov_b32 v8, 0
+; GFX1100-NEXT:    v_mov_b32_e32 v8, 0x3fe00000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v6
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-NEXT:    v_dual_mov_b32 v10, v8 :: v_dual_cndmask_b32 v11, 0xbff00000, v9
+; GFX1100-NEXT:    v_dual_cndmask_b32 v9, 0xbff00000, v8 :: v_dual_mov_b32 v8, 0
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[10:11]
-; GFX1100-NEXT:    v_cndmask_b32_e64 v9, 0x3ff00000, 2.0, vcc_lo
+; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX1100-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_mul_f64 v[2:3], v[2:3], v[8:9]
+; GFX1100-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
   %y = select <2 x i1> %bool, <2 x double> <double 5.000000e-01, double 2.000000e+00>, <2 x double> <double -1.000000e+00, double 1.000000e+00>
@@ -1199,43 +1112,40 @@ define double @fmul_select_f64_test14_sel_log2val_pos92_neg27(double %x, i32 %bo
 ; GFX7-LABEL: fmul_select_f64_test14_sel_log2val_pos92_neg27:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x3e400000
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x45b00000
+; GFX7-NEXT:    v_not_b32_e32 v4, 26
+; GFX7-NEXT:    v_mov_b32_e32 v5, 0x5c
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test14_sel_log2val_pos92_neg27:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3e400000
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x45b00000
+; GFX9-NEXT:    v_not_b32_e32 v4, 26
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x5c
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test14_sel_log2val_pos92_neg27:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v5, 0x45b00000
+; GFX1030-NEXT:    v_mov_b32_e32 v4, 0x5c
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v5, 0x3e400000, v5, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e32 v2, 0xffffffe5, v4, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test14_sel_log2val_pos92_neg27:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v5, 0x45b00000 :: v_dual_mov_b32 v4, 0
+; GFX1100-NEXT:    v_mov_b32_e32 v4, 0x5c
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v5, 0x3e400000, v5, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_cndmask_b32_e32 v2, 0xffffffe5, v4, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double 0x45B0000000000000, double 0x3E40000000000000
@@ -1247,43 +1157,40 @@ define double @fmul_select_f64_test15_sel_log2val_neg42_neg33(double %x, i32 %bo
 ; GFX7-LABEL: fmul_select_f64_test15_sel_log2val_neg42_neg33:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x3de00000
-; GFX7-NEXT:    v_mov_b32_e32 v5, 0x3d500000
+; GFX7-NEXT:    v_not_b32_e32 v4, 32
+; GFX7-NEXT:    v_not_b32_e32 v5, 41
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX7-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f64_test15_sel_log2val_neg42_neg33:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3de00000
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x3d500000
+; GFX9-NEXT:    v_not_b32_e32 v4, 32
+; GFX9-NEXT:    v_not_b32_e32 v5, 41
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f64_test15_sel_log2val_neg42_neg33:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v5, 0x3d500000
+; GFX1030-NEXT:    v_not_b32_e32 v4, 41
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1030-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v5, 0x3de00000, v5, vcc_lo
-; GFX1030-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1030-NEXT:    v_cndmask_b32_e32 v2, 0xffffffdf, v4, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f64_test15_sel_log2val_neg42_neg33:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_dual_mov_b32 v5, 0x3d500000 :: v_dual_mov_b32 v4, 0
+; GFX1100-NEXT:    v_not_b32_e32 v4, 41
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v3
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v5, 0x3de00000, v5, vcc_lo
-; GFX1100-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GFX1100-NEXT:    v_cndmask_b32_e32 v2, 0xffffffdf, v4, vcc_lo
+; GFX1100-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, double 0x3D50000000000000, double 0x3DE0000000000000
@@ -1298,38 +1205,34 @@ define half @fmul_select_f16_test1(half %x, i32 %bool.arg1, i32 %bool.arg2) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3c00
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x4000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test1:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x4000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test1:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x4000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half 2.000000e+00, half 1.000000e+00
@@ -1343,38 +1246,41 @@ define half @fmul_select_f16_test2(half %x, i32 %bool.arg1, i32 %bool.arg2) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3c00
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3800
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX9-NEXT:    s_movk_i32 s4, 0x8000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff
+; GFX9-NEXT:    v_med3_i32 v1, v1, s4, v2
+; GFX9-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test2:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x3800
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    s_movk_i32 s4, 0x8000
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX1030-NEXT:    v_med3_i32 v1, v1, s4, 0x7fff
+; GFX1030-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test2:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x3800
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    s_movk_i32 s0, 0x8000
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_med3_i32 v1, v1, s0, 0x7fff
+; GFX1100-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half 5.000000e-01, half 1.000000e+00
@@ -1507,40 +1413,35 @@ define half @fmul_select_f16_test5(half %x, i32 %bool.arg1, i32 %bool.arg2) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0x41000000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v3, 2.0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test5:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x4800
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x4000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc
+; GFX9-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test5:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x4000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x4800, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test5:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x4000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x4800, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 3, 1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half 2.000000e+00, half 8.000000e+00
@@ -1688,42 +1589,36 @@ define half @fmul_select_f16_test9(half %x, i32 %bool.arg1, i32 %bool.arg2) {
 ; GFX7-LABEL: fmul_select_f16_test9:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0xc2000000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0xc1800000
+; GFX7-NEXT:    v_cvt_f16_f32_e64 v0, -v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 5, 4, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test9:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0xd000
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xcc00
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 5, 4, vcc
+; GFX9-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test9:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0xcc00
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0xd000, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 5, 4, vcc_lo
+; GFX1030-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test9:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0xcc00
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0xd000, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 5, 4, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f16_e64 v0, -v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half -1.600000e+01, half -3.200000e+01
@@ -1736,41 +1631,42 @@ define half @fmul_select_f16_test10_sel_log2val_neg11_pos11(half %x, i32 %bool.a
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0x45000000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x3a000000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 11, -11, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x6800
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x1000
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 11, -11, vcc
+; GFX9-NEXT:    s_movk_i32 s4, 0x8000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff
+; GFX9-NEXT:    v_med3_i32 v1, v1, s4, v2
+; GFX9-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x1000
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x6800, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    s_movk_i32 s4, 0x8000
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, 11, -11, vcc_lo
+; GFX1030-NEXT:    v_med3_i32 v1, v1, s4, 0x7fff
+; GFX1030-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x1000
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x6800, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    s_movk_i32 s0, 0x8000
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 11, -11, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_med3_i32 v1, v1, s0, 0x7fff
+; GFX1100-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half 0xH1000, half 0xH6800
@@ -1783,41 +1679,42 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v3, 0x38800000
-; GFX7-NEXT:    v_mov_b32_e32 v4, 0x43000000
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, -14, 7, vcc
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x400
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x5800
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, -14, 7, vcc
+; GFX9-NEXT:    s_movk_i32 s4, 0x8000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff
+; GFX9-NEXT:    v_med3_i32 v1, v1, s4, v2
+; GFX9-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT:    v_mov_b32_e32 v3, 0x5800
 ; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, 0x400, v3, vcc_lo
-; GFX1030-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1030-NEXT:    s_movk_i32 s4, 0x8000
+; GFX1030-NEXT:    v_cndmask_b32_e64 v1, -14, 7, vcc_lo
+; GFX1030-NEXT:    v_med3_i32 v1, v1, s4, 0x7fff
+; GFX1030-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
 ; GFX1100:       ; %bb.0:
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, 0x5800
 ; GFX1100-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_cndmask_b32_e32 v1, 0x400, v3, vcc_lo
-; GFX1100-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX1100-NEXT:    s_movk_i32 s0, 0x8000
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, -14, 7, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_med3_i32 v1, v1, s0, 0x7fff
+; GFX1100-NEXT:    v_ldexp_f16_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
   %bool = icmp eq i32 %bool.arg1, %bool.arg2
   %y = select i1 %bool, half 0xH5800, half 0xH0400
diff --git a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll
index aa182b720c604..c108b93766bcc 100644
--- a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll
@@ -35,6 +35,6 @@ define amdgpu_kernel void @test_direct_indirect_call() {
   ret void
 }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
index 848019c872925..908746a0c7784 100644
--- a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
+++ b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
@@ -42,6 +42,6 @@ attributes #0 = { "amdgpu-no-dispatch-id" }
 ;.
 ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-no-dispatch-id" "amdgpu-stack-objects" }
 ;.
-; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll b/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
index 2140f50611d71..ebfb5e9ccaa35 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
@@ -1613,14 +1613,12 @@ define float @v_recip_sqrt_f32_ulp25_contract(float %x) {
 ; CODEGEN-IEEE-SDAG:       ; %bb.0:
 ; CODEGEN-IEEE-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; CODEGEN-IEEE-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4b800000
 ; CODEGEN-IEEE-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CODEGEN-IEEE-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; CODEGEN-IEEE-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CODEGEN-IEEE-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; CODEGEN-IEEE-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; CODEGEN-IEEE-SDAG-NEXT:    v_rsq_f32_e32 v0, v0
-; CODEGEN-IEEE-SDAG-NEXT:    v_mov_b32_e32 v1, 0x45800000
-; CODEGEN-IEEE-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; CODEGEN-IEEE-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CODEGEN-IEEE-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; CODEGEN-IEEE-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; CODEGEN-IEEE-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CODEGEN-IEEE-GISEL-LABEL: v_recip_sqrt_f32_ulp25_contract:
diff --git a/llvm/test/CodeGen/AMDGPU/fdot2.ll b/llvm/test/CodeGen/AMDGPU/fdot2.ll
index 695042d44d87b..776816d6aa0e3 100644
--- a/llvm/test/CodeGen/AMDGPU/fdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdot2.ll
@@ -5,6 +5,7 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=preserve-sign -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-CONTRACT
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DENORM-CONTRACT
+; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -mattr="+dot7-insts,-dot10-insts" -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DOT10-DISABLED
 ; (fadd (fmul S1.x, S2.x), (fadd (fmul (S1.y, S2.y), z))) -> (fdot2 S1, S2, z)
 
 ; Tests to make sure fdot2 is not generated when vector elements of dot-product expressions
@@ -21,6 +22,7 @@
 
 ; GFX906-CONTRACT: v_mac_f16_e32
 ; GFX906-DENORM-CONTRACT: v_fma_f16
+; GFX906-DOT10-DISABLED: v_fma_f16
 define amdgpu_kernel void @dotproduct_f16(ptr addrspace(1) %src1,
                                           ptr addrspace(1) %src2,
                                           ptr addrspace(1) nocapture %dst) {
@@ -44,8 +46,11 @@ entry:
 }
 
 
-; We only want to generate fdot2 if vector element of dot product is converted from f16 to f32
-; and the vectors are of type <2 x half>
+; We only want to generate fdot2 if:
+; - vector element of dot product is converted from f16 to f32, and
+; - the vectors are of type <2 x half>, and
+; - "dot10-insts" is enabled
+
 ; GCN-LABEL: {{^}}dotproduct_f16_f32
 ; GFX900: v_mad_mix_f32
 ; GFX900: v_mad_mix_f32
@@ -59,6 +64,7 @@ entry:
 ; GFX906-CONTRACT: v_dot2_f32_f16
 
 ; GFX906-DENORM-CONTRACT: v_dot2_f32_f16
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @dotproduct_f16_f32(ptr addrspace(1) %src1,
                                               ptr addrspace(1) %src2,
                                               ptr addrspace(1) nocapture %dst) {
@@ -85,8 +91,11 @@ entry:
   ret void
 }
 
-; We only want to generate fdot2 if vector element of dot product is converted from f16 to f32
-; and the vectors are of type <2 x half>
+; We only want to generate fdot2 if:
+; - vector element of dot product is converted from f16 to f32, and
+; - the vectors are of type <2 x half>, and
+; - "dot10-insts" is enabled
+
 ; GCN-LABEL: {{^}}dotproduct_diffvecorder
 ; GFX900: v_mad_mix_f32
 ; GFX900: v_mad_mix_f32
@@ -99,6 +108,7 @@ entry:
 
 ; GFX906-CONTRACT: v_dot2_f32_f16
 ; GFX906-DENORM-CONTRACT: v_dot2_f32_f16
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @dotproduct_diffvecorder(ptr addrspace(1) %src1,
                                                    ptr addrspace(1) %src2,
                                                    ptr addrspace(1) nocapture %dst) {
@@ -136,6 +146,7 @@ entry:
 
 ; GFX906-CONTRACT: v_fma_mix_f32
 ; GFX906-DENORM-CONTRACT: v_fma_mix_f32
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @dotproduct_v4f16(ptr addrspace(1) %src1,
                                             ptr addrspace(1) %src2,
                                             ptr addrspace(1) nocapture %dst) {
@@ -173,6 +184,7 @@ entry:
 
 ; GFX906-CONTRACT: v_fma_mix_f32
 ; GFX906-DENORM-CONTRACT: v_fma_mix_f32
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @NotAdotproduct(ptr addrspace(1) %src1,
                                           ptr addrspace(1) %src2,
                                           ptr addrspace(1) nocapture %dst) {
@@ -210,6 +222,7 @@ entry:
 
 ; GFX906-CONTRACT: v_fma_mix_f32
 ; GFX906-DENORM-CONTRACT: v_fma_mix_f32
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @Diff_Idx_NotAdotproduct(ptr addrspace(1) %src1,
                                                    ptr addrspace(1) %src2,
                                                    ptr addrspace(1) nocapture %dst) {
diff --git a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
index 12593e3760fd3..4c68b8d35260f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
@@ -20,7 +20,7 @@ define amdgpu_kernel void @store_flat_i32(ptr addrspace(1) %gptr, i32 %x) #0 {
 }
 
 ; GCN-LABEL: {{^}}store_flat_i64:
-; GCN: flat_store_{{dwordx2|b64}}
+; GCN: flat_store_{{dword|b64}}
 define amdgpu_kernel void @store_flat_i64(ptr addrspace(1) %gptr, i64 %x) #0 {
   %fptr = addrspacecast ptr addrspace(1) %gptr to ptr
   store volatile i64 %x, ptr %fptr, align 8
@@ -28,7 +28,7 @@ define amdgpu_kernel void @store_flat_i64(ptr addrspace(1) %gptr, i64 %x) #0 {
 }
 
 ; GCN-LABEL: {{^}}store_flat_v4i32:
-; GCN: flat_store_{{dwordx4|b128}}
+; GCN: flat_store_{{dword|b128}}
 define amdgpu_kernel void @store_flat_v4i32(ptr addrspace(1) %gptr, <4 x i32> %x) #0 {
   %fptr = addrspacecast ptr addrspace(1) %gptr to ptr
   store volatile <4 x i32> %x, ptr %fptr, align 16
@@ -65,7 +65,7 @@ define amdgpu_kernel void @load_flat_i32(ptr addrspace(1) noalias %out, ptr addr
 }
 
 ; GCN-LABEL: load_flat_i64:
-; GCN: flat_load_{{dwordx2|b64}}
+; GCN: flat_load_{{dword|b64}}
 define amdgpu_kernel void @load_flat_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %gptr) #0 {
   %fptr = addrspacecast ptr addrspace(1) %gptr to ptr
   %fload = load volatile i64, ptr %fptr, align 8
@@ -74,7 +74,7 @@ define amdgpu_kernel void @load_flat_i64(ptr addrspace(1) noalias %out, ptr addr
 }
 
 ; GCN-LABEL: load_flat_v4i32:
-; GCN: flat_load_{{dwordx4|b128}}
+; GCN: flat_load_{{dword|b128}}
 define amdgpu_kernel void @load_flat_v4i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %gptr) #0 {
   %fptr = addrspacecast ptr addrspace(1) %gptr to ptr
   %fload = load volatile <4 x i32>, ptr %fptr, align 32
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
index 87d63f4f9cd90..45223a24e021a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
@@ -96,9 +96,9 @@ entry:
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
-; CI: TotalNumSgprs: 4
-; VI-NOXNACK: TotalNumSgprs: 6
-; VI-XNACK: TotalNumSgprs: 6
+; CI: NumSgprs: 4
+; VI-NOXNACK: NumSgprs: 6
+; VI-XNACK: NumSgprs: 6
 ; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
 ; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr() #0 {
@@ -113,9 +113,9 @@ entry:
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
-; CI: TotalNumSgprs: 4
-; VI-NOXNACK: TotalNumSgprs: 6
-; VI-XNACK: TotalNumSgprs: 6
+; CI: NumSgprs: 4
+; VI-NOXNACK: NumSgprs: 6
+; VI-XNACK: NumSgprs: 6
 ; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
 ; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr_lo() #0 {
@@ -130,9 +130,9 @@ entry:
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
-; CI: TotalNumSgprs: 4
-; VI-NOXNACK: TotalNumSgprs: 6
-; VI-XNACK: TotalNumSgprs: 6
+; CI: NumSgprs: 4
+; VI-NOXNACK: NumSgprs: 6
+; VI-XNACK: NumSgprs: 6
 ; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
 ; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr_hi() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 711a5fff1a063..104e157e9e15a 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -3307,22 +3307,54 @@ define amdgpu_ps i32 @s_mul_32_f16(half inreg %x, half inreg %y) {
 ; --------------------------------------------------------------------
 
 define float @v_mul_f32_select_64_1(i32 %arg, float %x) {
-; GFX9-LABEL: v_mul_f32_select_64_1:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, 1.0, v2, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_f32_select_64_1:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_mul_f32_select_64_1:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1011-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x42800000, vcc_lo
-; GFX1011-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_f32_select_64_1:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v0, 1.0, v2, vcc
+; GFX9-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_f32_select_64_1:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_f32_select_64_1:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x42800000, vcc_lo
+; GFX10-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_mul_f32_select_64_1:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_mul_f32_select_64_1:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x42800000, vcc_lo
+; GFX11-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %cond = icmp eq i32 %arg, 0
   %select.pow2 = select i1 %cond, float 64.0, float 1.0
   %mul = fmul float %x, %select.pow2
@@ -3330,22 +3362,54 @@ define float @v_mul_f32_select_64_1(i32 %arg, float %x) {
 }
 
 define float @v_mul_f32_select_1_64(i32 %arg, float %x) {
-; GFX9-LABEL: v_mul_f32_select_1_64:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 1.0, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_f32_select_1_64:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_mul_f32_select_1_64:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1011-NEXT:    v_cndmask_b32_e64 v0, 0x42800000, 1.0, vcc_lo
-; GFX1011-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_f32_select_1_64:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, 1.0, vcc
+; GFX9-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_f32_select_1_64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_f32_select_1_64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0x42800000, 1.0, vcc_lo
+; GFX10-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_mul_f32_select_1_64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_mul_f32_select_1_64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0x42800000, 1.0, vcc_lo
+; GFX11-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %cond = icmp eq i32 %arg, 0
   %select.pow2 = select i1 %cond, float 1.0, float 64.0
   %mul = fmul float %x, %select.pow2
@@ -3353,22 +3417,54 @@ define float @v_mul_f32_select_1_64(i32 %arg, float %x) {
 }
 
 define float @v_mul_f32_select_n1_n64(i32 %arg, float %x) {
-; GFX9-LABEL: v_mul_f32_select_n1_n64:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xc2800000
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, -1.0, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_f32_select_n1_n64:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_mul_f32_select_n1_n64:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1011-NEXT:    v_cndmask_b32_e64 v0, 0xc2800000, -1.0, vcc_lo
-; GFX1011-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_f32_select_n1_n64:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xc2800000
+; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, -1.0, vcc
+; GFX9-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_f32_select_n1_n64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_f32_select_n1_n64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0xc2800000, -1.0, vcc_lo
+; GFX10-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_mul_f32_select_n1_n64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_mul_f32_select_n1_n64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0xc2800000, -1.0, vcc_lo
+; GFX11-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %cond = icmp eq i32 %arg, 0
   %select.pow2 = select i1 %cond, float -1.0, float -64.0
   %mul = fmul float %x, %select.pow2
@@ -3376,22 +3472,54 @@ define float @v_mul_f32_select_n1_n64(i32 %arg, float %x) {
 }
 
 define float @v_mul_f32_select_n64_n1(i32 %arg, float %x) {
-; GFX9-LABEL: v_mul_f32_select_n64_n1:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0xc2800000
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, -1.0, v2, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_f32_select_n64_n1:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1011-LABEL: v_mul_f32_select_n64_n1:
-; GFX1011:       ; %bb.0:
-; GFX1011-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1011-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1011-NEXT:    v_cndmask_b32_e64 v0, -1.0, 0xc2800000, vcc_lo
-; GFX1011-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX1011-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_f32_select_n64_n1:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0xc2800000
+; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v0, -1.0, v2, vcc
+; GFX9-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_f32_select_n64_n1:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_f32_select_n64_n1:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, -1.0, 0xc2800000, vcc_lo
+; GFX10-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_mul_f32_select_n64_n1:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_mul_f32_select_n64_n1:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, -1.0, 0xc2800000, vcc_lo
+; GFX11-GISEL-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %cond = icmp eq i32 %arg, 0
   %select.pow2 = select i1 %cond, float -64.0, float -1.0
   %mul = fmul float %x, %select.pow2
@@ -3402,11 +3530,9 @@ define float @v_mul_f32_select_128_64(i32 %arg, float %x) {
 ; GFX9-SDAG-LABEL: v_mul_f32_select_128_64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x43000000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f32_select_128_64:
@@ -3422,10 +3548,9 @@ define float @v_mul_f32_select_128_64(i32 %arg, float %x) {
 ; GFX10-SDAG-LABEL: v_mul_f32_select_128_64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0x43000000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x42800000, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f32_select_128_64:
@@ -3440,10 +3565,9 @@ define float @v_mul_f32_select_128_64(i32 %arg, float %x) {
 ; GFX11-SDAG-LABEL: v_mul_f32_select_128_64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0x43000000
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x42800000, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f32_select_128_64:
@@ -3464,11 +3588,9 @@ define float @v_mul_f32_select_n128_n64(i32 %arg, float %x) {
 ; GFX9-SDAG-LABEL: v_mul_f32_select_n128_n64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc2800000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc3000000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f32_select_n128_n64:
@@ -3484,10 +3606,9 @@ define float @v_mul_f32_select_n128_n64(i32 %arg, float %x) {
 ; GFX10-SDAG-LABEL: v_mul_f32_select_n128_n64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc3000000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xc2800000, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f32_select_n128_n64:
@@ -3502,10 +3623,9 @@ define float @v_mul_f32_select_n128_n64(i32 %arg, float %x) {
 ; GFX11-SDAG-LABEL: v_mul_f32_select_n128_n64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc3000000
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xc2800000, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f32_select_n128_n64:
@@ -3526,11 +3646,9 @@ define float @v_mul_f32_select_n128_n16(i32 %arg, float %x) {
 ; GFX9-SDAG-LABEL: v_mul_f32_select_n128_n16:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc1800000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc3000000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f32_select_n128_n16:
@@ -3546,10 +3664,9 @@ define float @v_mul_f32_select_n128_n16(i32 %arg, float %x) {
 ; GFX10-SDAG-LABEL: v_mul_f32_select_n128_n16:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc3000000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xc1800000, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f32_select_n128_n16:
@@ -3564,10 +3681,9 @@ define float @v_mul_f32_select_n128_n16(i32 %arg, float %x) {
 ; GFX11-SDAG-LABEL: v_mul_f32_select_n128_n16:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc3000000
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xc1800000, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f32 v0, -v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f32_select_n128_n16:
@@ -3818,12 +3934,9 @@ define double @v_mul_f64_select_64_1(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_64_1:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3ff00000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x40500000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_64_1:
@@ -3840,11 +3953,9 @@ define double @v_mul_f64_select_64_1(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_64_1:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0x40500000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x3ff00000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_64_1:
@@ -3860,10 +3971,9 @@ define double @v_mul_f64_select_64_1(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_64_1:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0x40500000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x3ff00000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_64_1:
@@ -3884,12 +3994,9 @@ define double @v_mul_f64_select_1_64(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_1_64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x40500000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_1_64:
@@ -3906,11 +4013,9 @@ define double @v_mul_f64_select_1_64(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_1_64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0x3ff00000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x40500000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_1_64:
@@ -3926,10 +4031,9 @@ define double @v_mul_f64_select_1_64(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_1_64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0x3ff00000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x40500000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_1_64:
@@ -3950,12 +4054,9 @@ define double @v_mul_f64_select_n1_n64(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_n1_n64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc0500000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbff00000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_n1_n64:
@@ -3972,11 +4073,9 @@ define double @v_mul_f64_select_n1_n64(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_n1_n64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbff00000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0500000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_n1_n64:
@@ -3992,10 +4091,9 @@ define double @v_mul_f64_select_n1_n64(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_n1_n64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0xbff00000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0500000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_n1_n64:
@@ -4016,12 +4114,9 @@ define double @v_mul_f64_select_128_64(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_128_64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x40500000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0x40600000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_128_64:
@@ -4038,11 +4133,9 @@ define double @v_mul_f64_select_128_64(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_128_64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0x40600000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x40500000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_128_64:
@@ -4058,10 +4151,9 @@ define double @v_mul_f64_select_128_64(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_128_64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0x40600000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0x40500000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_128_64:
@@ -4082,12 +4174,9 @@ define double @v_mul_f64_select_n128_n64(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_n128_n64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc0500000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc0600000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_n128_n64:
@@ -4104,11 +4193,9 @@ define double @v_mul_f64_select_n128_n64(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_n128_n64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc0600000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0500000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_n128_n64:
@@ -4124,10 +4211,9 @@ define double @v_mul_f64_select_n128_n64(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_n128_n64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0xc0600000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0500000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_n128_n64:
@@ -4148,12 +4234,9 @@ define double @v_mul_f64_select_n128_n16(i32 %arg, double %x) {
 ; GFX9-SDAG-LABEL: v_mul_f64_select_n128_n16:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc0300000
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc0600000
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f64_select_n128_n16:
@@ -4170,11 +4253,9 @@ define double @v_mul_f64_select_n128_n16(i32 %arg, double %x) {
 ; GFX10-SDAG-LABEL: v_mul_f64_select_n128_n16:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0xc0600000
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0300000, v4, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f64_select_n128_n16:
@@ -4190,10 +4271,9 @@ define double @v_mul_f64_select_n128_n16(i32 %arg, double %x) {
 ; GFX11-SDAG-LABEL: v_mul_f64_select_n128_n16:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0xc0600000 :: v_dual_mov_b32 v3, 0
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0xc0300000, v4, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f64 v[0:1], v[1:2], v[3:4]
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[1:2], v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f64_select_n128_n16:
@@ -4719,11 +4799,9 @@ define half @v_mul_f16_select_64_1(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_64_1:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0x3c00
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x5400
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_64_1:
@@ -4739,10 +4817,9 @@ define half @v_mul_f16_select_64_1(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_64_1:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5400
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x3c00, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_64_1:
@@ -4757,10 +4834,9 @@ define half @v_mul_f16_select_64_1(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_64_1:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5400
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x3c00, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_64_1:
@@ -4781,11 +4857,9 @@ define half @v_mul_f16_select_1_64(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_1_64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5400
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x3c00
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_1_64:
@@ -4801,10 +4875,9 @@ define half @v_mul_f16_select_1_64(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_1_64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0x3c00
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x5400, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_1_64:
@@ -4819,10 +4892,9 @@ define half @v_mul_f16_select_1_64(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_1_64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0x3c00
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x5400, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_1_64:
@@ -4843,11 +4915,9 @@ define half @v_mul_f16_select_n1_n64(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_n1_n64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd400
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xbc00
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_n1_n64:
@@ -4863,10 +4933,9 @@ define half @v_mul_f16_select_n1_n64(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_n1_n64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0xbc00
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xd400, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_n1_n64:
@@ -4881,10 +4950,9 @@ define half @v_mul_f16_select_n1_n64(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_n1_n64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0xbc00
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xd400, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_n1_n64:
@@ -4905,11 +4973,9 @@ define half @v_mul_f16_select_128_64(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_128_64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5400
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0x5800
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_128_64:
@@ -4925,10 +4991,9 @@ define half @v_mul_f16_select_128_64(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_128_64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5800
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x5400, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_128_64:
@@ -4943,10 +5008,9 @@ define half @v_mul_f16_select_128_64(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_128_64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0x5800
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0x5400, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e32 v0, v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_128_64:
@@ -4967,11 +5031,9 @@ define half @v_mul_f16_select_n128_n64(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_n128_n64:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd400
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xd800
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_n128_n64:
@@ -4987,10 +5049,9 @@ define half @v_mul_f16_select_n128_n64(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_n128_n64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd800
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xd400, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_n128_n64:
@@ -5005,10 +5066,9 @@ define half @v_mul_f16_select_n128_n64(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_n128_n64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd800
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xd400, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 6, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_n128_n64:
@@ -5029,11 +5089,9 @@ define half @v_mul_f16_select_n128_n16(i32 %arg, half %x) {
 ; GFX9-SDAG-LABEL: v_mul_f16_select_n128_n16:
 ; GFX9-SDAG:       ; %bb.0:
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, 0xcc00
-; GFX9-SDAG-NEXT:    v_mov_b32_e32 v3, 0xd800
 ; GFX9-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc
+; GFX9-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: v_mul_f16_select_n128_n16:
@@ -5049,10 +5107,9 @@ define half @v_mul_f16_select_n128_n16(i32 %arg, half %x) {
 ; GFX10-SDAG-LABEL: v_mul_f16_select_n128_n16:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd800
 ; GFX10-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xcc00, v2, vcc_lo
-; GFX10-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX10-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-GISEL-LABEL: v_mul_f16_select_n128_n16:
@@ -5067,10 +5124,9 @@ define half @v_mul_f16_select_n128_n16(i32 %arg, half %x) {
 ; GFX11-SDAG-LABEL: v_mul_f16_select_n128_n16:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0xd800
 ; GFX11-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0xcc00, v2, vcc_lo
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v0, v1, v0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX11-SDAG-NEXT:    v_ldexp_f16_e64 v0, -v1, v0
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-LABEL: v_mul_f16_select_n128_n16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 0005f1179a5d2..188b2ada64686 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -5,6 +5,8 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -global-isel=1 -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 -global-isel=0 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX9-SDAG %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX9-GISEL %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX950-SDAG %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX950-GISEL %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -global-isel=0 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-SDAG %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 -mattr=-flat-for-global -denormal-fp-math=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GISEL %s
 
@@ -99,6 +101,36 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
 ; GFX9-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_f32_to_f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_f32_to_f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_f32_to_f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -234,6 +266,38 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
 ; GFX9-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_f64_to_f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_f64_to_f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v0, s[2:3]
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_f64_to_f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -382,6 +446,37 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
 ; GFX9-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_v2f32_to_v2f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_v2f32_to_v2f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_v2f32_to_v2f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -543,6 +638,42 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
 ; GFX9-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_v2f64_to_v2f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f32_f64_e32 v2, v[2:3]
+; GFX950-SDAG-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v2
+; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_v2f64_to_v2f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v2, v[2:3]
+; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v2
+; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_v2f64_to_v2f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -685,6 +816,36 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
 ; GFX9-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fneg_fptrunc_f32_to_f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e64 v0, -v0
+; GFX950-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fneg_fptrunc_f32_to_f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e64 v0, -s2
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fneg_fptrunc_f32_to_f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -815,6 +976,36 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
 ; GFX9-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fabs_fptrunc_f32_to_f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
+; GFX950-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fabs_fptrunc_f32_to_f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e64 v0, |s2|
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fabs_fptrunc_f32_to_f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -945,6 +1136,36 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
 ; GFX9-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fneg_fabs_fptrunc_f32_to_f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e64 v0, -|v0|
+; GFX950-SDAG-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fneg_fabs_fptrunc_f32_to_f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e64 v0, -|s2|
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fneg_fabs_fptrunc_f32_to_f16:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -1076,6 +1297,36 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
 ; GFX9-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_f32_to_f16_zext_i32:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_f32_to_f16_zext_i32:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_f32_to_f16_zext_i32:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -1211,6 +1462,36 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
 ; GFX9-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e64 v0, |v0|
+; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e64 v0, |s2|
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -1353,6 +1634,38 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
 ; GFX9-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
+; GFX950-SDAG-LABEL: fptrunc_f32_to_f16_sext_i32:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; GFX950-SDAG-NEXT:    s_mov_b32 s6, -1
+; GFX950-SDAG-NEXT:    s_mov_b32 s10, s6
+; GFX950-SDAG-NEXT:    s_mov_b32 s11, s7
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_mov_b32 s8, s2
+; GFX950-SDAG-NEXT:    s_mov_b32 s9, s3
+; GFX950-SDAG-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s4, s0
+; GFX950-SDAG-NEXT:    s_mov_b32 s5, s1
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: fptrunc_f32_to_f16_sext_i32:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX950-GISEL-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: fptrunc_f32_to_f16_sext_i32:
 ; GFX11-SDAG:       ; %bb.0: ; %entry
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
index 8e04a240d0a1c..b3001819e9aaf 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
@@ -2368,14 +2368,12 @@ define float @v_sqrt_f32_ulp2_contractable_rcp(float %x) {
 ; SDAG-IEEE:       ; %bb.0:
 ; SDAG-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-IEEE-NEXT:    s_mov_b32 s4, 0x800000
-; SDAG-IEEE-NEXT:    v_mov_b32_e32 v1, 0x4b800000
 ; SDAG-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SDAG-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
-; SDAG-IEEE-NEXT:    v_mov_b32_e32 v1, 0x45800000
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SDAG-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GISEL-IEEE-LABEL: v_sqrt_f32_ulp2_contractable_rcp:
@@ -2718,20 +2716,18 @@ define <2 x float> @v_sqrt_v2f32_ulp2_contractable_rcp(<2 x float> %x) {
 ; SDAG-IEEE:       ; %bb.0:
 ; SDAG-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SDAG-IEEE-NEXT:    s_mov_b32 s4, 0x800000
-; SDAG-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4b800000
 ; SDAG-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e32 v3, 1.0, v2, vcc
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
 ; SDAG-IEEE-NEXT:    v_cmp_gt_f32_e64 s[4:5], s4, v1
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v3
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 1.0, v2, s[4:5]
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, s[4:5]
 ; SDAG-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v1, v1, v2
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
 ; SDAG-IEEE-NEXT:    v_rsq_f32_e32 v1, v1
-; SDAG-IEEE-NEXT:    v_mov_b32_e32 v3, 0x45800000
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v3, vcc
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 1.0, v3, s[4:5]
-; SDAG-IEEE-NEXT:    v_mul_f32_e32 v1, v1, v2
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SDAG-IEEE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, s[4:5]
+; SDAG-IEEE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
 ; SDAG-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GISEL-IEEE-LABEL: v_sqrt_v2f32_ulp2_contractable_rcp:
diff --git a/llvm/test/CodeGen/AMDGPU/group-image-instructions.ll b/llvm/test/CodeGen/AMDGPU/group-image-instructions.ll
new file mode 100644
index 0000000000000..7f587ac0b8716
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/group-image-instructions.ll
@@ -0,0 +1,455 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
+
+define amdgpu_ps void @group_image_sample(i32 inreg noundef %globalTable, i32 inreg noundef %userdata6, i32 inreg noundef %userdata7, i32 inreg noundef %userdata8, i32 inreg noundef %PrimMask, <2 x float> noundef %PerspInterpSample, <2 x float> noundef %PerspInterpCenter, <2 x float> noundef %PerspInterpCentroid) #2 {
+; GFX11-LABEL: group_image_sample:
+; GFX11:       ; %bb.0: ; %.entry
+; GFX11-NEXT:    s_mov_b32 s24, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    s_mov_b32 m0, s4
+; GFX11-NEXT:    s_getpc_b64 s[4:5]
+; GFX11-NEXT:    s_mov_b32 s0, s1
+; GFX11-NEXT:    s_mov_b32 s6, s3
+; GFX11-NEXT:    s_mov_b32 s1, s5
+; GFX11-NEXT:    s_mov_b32 s3, s5
+; GFX11-NEXT:    s_mov_b32 s7, s5
+; GFX11-NEXT:    s_load_b128 s[12:15], s[0:1], 0x0
+; GFX11-NEXT:    s_load_b128 s[8:11], s[2:3], 0x0
+; GFX11-NEXT:    s_load_b256 s[0:7], s[6:7], 0x0
+; GFX11-NEXT:    s_mov_b32 s16, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    lds_param_load v2, attr0.y wait_vdst:15
+; GFX11-NEXT:    lds_param_load v3, attr0.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b64 s[16:17], s[12:15], 0x10
+; GFX11-NEXT:    s_buffer_load_b64 s[18:19], s[12:15], 0x20
+; GFX11-NEXT:    s_buffer_load_b64 s[20:21], s[12:15], 0x30
+; GFX11-NEXT:    s_buffer_load_b64 s[22:23], s[12:15], 0x40
+; GFX11-NEXT:    v_interp_p10_f32 v4, v2, v0, v2 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v0, v3, v0, v3 wait_exp:0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_interp_p2_f32 v0, v3, v1, v0 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v1, v2, v1, v4 wait_exp:7
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_dual_add_f32 v4, s16, v0 :: v_dual_add_f32 v5, s17, v1
+; GFX11-NEXT:    v_dual_add_f32 v12, s20, v0 :: v_dual_add_f32 v13, s21, v1
+; GFX11-NEXT:    v_dual_add_f32 v8, s18, v0 :: v_dual_add_f32 v9, s19, v1
+; GFX11-NEXT:    v_dual_add_f32 v16, s22, v0 :: v_dual_add_f32 v17, s23, v1
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    image_sample v[4:7], v[4:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[8:11], v[8:9], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[12:15], v[12:13], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[16:19], v[16:17], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b64 s[16:17], s[12:15], 0x50
+; GFX11-NEXT:    s_buffer_load_b64 s[18:19], s[12:15], 0x60
+; GFX11-NEXT:    s_buffer_load_b64 s[20:21], s[12:15], 0x70
+; GFX11-NEXT:    s_buffer_load_b64 s[22:23], s[12:15], 0x80
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_add_f32 v20, s16, v0 :: v_dual_add_f32 v21, s17, v1
+; GFX11-NEXT:    v_dual_add_f32 v28, s20, v0 :: v_dual_add_f32 v29, s21, v1
+; GFX11-NEXT:    v_dual_add_f32 v24, s18, v0 :: v_dual_add_f32 v25, s19, v1
+; GFX11-NEXT:    v_dual_add_f32 v32, s22, v0 :: v_dual_add_f32 v33, s23, v1
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    image_sample v[20:23], v[20:21], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[24:27], v[24:25], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[28:31], v[28:29], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[32:35], v[32:33], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b64 s[16:17], s[12:15], 0x90
+; GFX11-NEXT:    s_buffer_load_b64 s[18:19], s[12:15], 0xa0
+; GFX11-NEXT:    s_buffer_load_b64 s[20:21], s[12:15], 0xb0
+; GFX11-NEXT:    s_buffer_load_b64 s[22:23], s[12:15], 0xc0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_add_f32 v36, s16, v0 :: v_dual_add_f32 v37, s17, v1
+; GFX11-NEXT:    v_dual_add_f32 v44, s20, v0 :: v_dual_add_f32 v45, s21, v1
+; GFX11-NEXT:    v_dual_add_f32 v40, s18, v0 :: v_dual_add_f32 v41, s19, v1
+; GFX11-NEXT:    v_dual_add_f32 v48, s22, v0 :: v_dual_add_f32 v49, s23, v1
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    image_sample v[36:39], v[36:37], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[40:43], v[40:41], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[44:47], v[44:45], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[48:51], v[48:49], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b64 s[16:17], s[12:15], 0xd0
+; GFX11-NEXT:    s_buffer_load_b64 s[18:19], s[12:15], 0xe0
+; GFX11-NEXT:    s_buffer_load_b64 s[20:21], s[12:15], 0xf0
+; GFX11-NEXT:    s_buffer_load_b64 s[12:13], s[12:15], 0x100
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_add_f32 v52, s16, v0 :: v_dual_add_f32 v53, s17, v1
+; GFX11-NEXT:    v_dual_add_f32 v56, s18, v0 :: v_dual_add_f32 v57, s19, v1
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    image_sample v[52:55], v[52:53], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[56:59], v[56:57], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    v_dual_add_f32 v60, s20, v0 :: v_dual_add_f32 v61, s21, v1
+; GFX11-NEXT:    v_dual_add_f32 v0, s12, v0 :: v_dual_add_f32 v1, s13, v1
+; GFX11-NEXT:    s_and_b32 exec_lo, exec_lo, s24
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    image_sample v[60:63], v[60:61], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v[64:67], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_waitcnt vmcnt(14)
+; GFX11-NEXT:    v_dual_add_f32 v0, v8, v4 :: v_dual_add_f32 v1, v9, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, v10, v6 :: v_dual_add_f32 v5, v11, v7
+; GFX11-NEXT:    s_waitcnt vmcnt(13)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v12, v0 :: v_dual_add_f32 v1, v13, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v14, v4 :: v_dual_add_f32 v5, v15, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(12)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v16, v0 :: v_dual_add_f32 v1, v17, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v18, v4 :: v_dual_add_f32 v5, v19, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(11)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v20, v0 :: v_dual_add_f32 v1, v21, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v22, v4 :: v_dual_add_f32 v5, v23, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(10)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v24, v0 :: v_dual_add_f32 v1, v25, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v26, v4 :: v_dual_add_f32 v5, v27, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(9)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v28, v0 :: v_dual_add_f32 v1, v29, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v30, v4 :: v_dual_add_f32 v5, v31, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(8)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v32, v0 :: v_dual_add_f32 v1, v33, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v34, v4 :: v_dual_add_f32 v5, v35, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(7)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v36, v0 :: v_dual_add_f32 v1, v37, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v38, v4 :: v_dual_add_f32 v5, v39, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v40, v0 :: v_dual_add_f32 v1, v41, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v42, v4 :: v_dual_add_f32 v5, v43, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(5)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v44, v0 :: v_dual_add_f32 v1, v45, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v46, v4 :: v_dual_add_f32 v5, v47, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v48, v0 :: v_dual_add_f32 v1, v49, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v50, v4 :: v_dual_add_f32 v5, v51, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v52, v0 :: v_dual_add_f32 v1, v53, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v54, v4 :: v_dual_add_f32 v5, v55, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v56, v0 :: v_dual_add_f32 v1, v57, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v58, v4 :: v_dual_add_f32 v5, v59, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v60, v0 :: v_dual_add_f32 v1, v61, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v62, v4 :: v_dual_add_f32 v5, v63, v5
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_add_f32 v0, v64, v0 :: v_dual_add_f32 v1, v65, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, v66, v4 :: v_dual_add_f32 v5, v67, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cvt_pk_rtz_f16_f32_e32 v0, v0, v1
+; GFX11-NEXT:    v_cvt_pk_rtz_f16_f32_e32 v1, v4, v5
+; GFX11-NEXT:    exp mrt0 v0, v1, off, off done
+; GFX11-NEXT:    s_endpgm
+.entry:
+  %i = call i64 @llvm.amdgcn.s.getpc()
+  %i1 = and i64 %i, -4294967296
+  %i2 = zext i32 %userdata6 to i64
+  %i3 = or disjoint i64 %i1, %i2
+  %i4 = inttoptr i64 %i3 to ptr addrspace(4)
+  %i5 = load <4 x i32>, ptr addrspace(4) %i4, align 16
+  %i6 = zext i32 %userdata7 to i64
+  %i7 = or disjoint i64 %i1, %i6
+  %i8 = inttoptr i64 %i7 to ptr addrspace(4)
+  %i9 = load <4 x i32>, ptr addrspace(4) %i8, align 4, !invariant.load !0
+  %i10 = zext i32 %userdata8 to i64
+  %i11 = or disjoint i64 %i1, %i10
+  %i12 = inttoptr i64 %i11 to ptr addrspace(4)
+  %i13 = load <8 x i32>, ptr addrspace(4) %i12, align 4, !invariant.load !0
+  %i14 = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %PrimMask)
+  %PerspInterpCenter.i1 = extractelement <2 x float> %PerspInterpCenter, i64 1
+  %PerspInterpCenter.i0 = extractelement <2 x float> %PerspInterpCenter, i64 0
+  %i15 = call float @llvm.amdgcn.interp.inreg.p10(float %i14, float %PerspInterpCenter.i0, float %i14)
+  %i16 = call float @llvm.amdgcn.interp.inreg.p2(float %i14, float %PerspInterpCenter.i1, float %i15)
+  %i17 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %PrimMask)
+  %i18 = call float @llvm.amdgcn.interp.inreg.p10(float %i17, float %PerspInterpCenter.i0, float %i17)
+  %i19 = call float @llvm.amdgcn.interp.inreg.p2(float %i17, float %PerspInterpCenter.i1, float %i18)
+  %i20 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 16, i32 0), !invariant.load !0
+  %i21 = shufflevector <2 x i32> %i20, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i22 = bitcast <4 x i32> %i21 to <4 x float>
+  %.i0 = extractelement <4 x float> %i22, i64 0
+  %.i1 = extractelement <4 x float> %i22, i64 1
+  %.i03 = fadd reassoc nnan nsz arcp contract afn float %.i0, %i19
+  %.i14 = fadd reassoc nnan nsz arcp contract afn float %.i1, %i16
+  %i23 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i03, float %.i14, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i010 = extractelement <4 x float> %i23, i64 0
+  %.i113 = extractelement <4 x float> %i23, i64 1
+  %.i215 = extractelement <4 x float> %i23, i64 2
+  %.i317 = extractelement <4 x float> %i23, i64 3
+  %i24 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 32, i32 0), !invariant.load !0
+  %i25 = shufflevector <2 x i32> %i24, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i26 = bitcast <4 x i32> %i25 to <4 x float>
+  %.i05 = extractelement <4 x float> %i26, i64 0
+  %.i16 = extractelement <4 x float> %i26, i64 1
+  %.i07 = fadd reassoc nnan nsz arcp contract afn float %.i05, %i19
+  %.i18 = fadd reassoc nnan nsz arcp contract afn float %.i16, %i16
+  %i27 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i07, float %.i18, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i09 = extractelement <4 x float> %i27, i64 0
+  %.i011 = fadd reassoc nnan nsz arcp contract afn float %.i09, %.i010
+  %.i112 = extractelement <4 x float> %i27, i64 1
+  %.i114 = fadd reassoc nnan nsz arcp contract afn float %.i112, %.i113
+  %.i2 = extractelement <4 x float> %i27, i64 2
+  %.i216 = fadd reassoc nnan nsz arcp contract afn float %.i2, %.i215
+  %.i3 = extractelement <4 x float> %i27, i64 3
+  %.i318 = fadd reassoc nnan nsz arcp contract afn float %.i3, %.i317
+  %i28 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 48, i32 0), !invariant.load !0
+  %i29 = shufflevector <2 x i32> %i28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i30 = bitcast <4 x i32> %i29 to <4 x float>
+  %.i019 = extractelement <4 x float> %i30, i64 0
+  %.i120 = extractelement <4 x float> %i30, i64 1
+  %.i021 = fadd reassoc nnan nsz arcp contract afn float %.i019, %i19
+  %.i122 = fadd reassoc nnan nsz arcp contract afn float %.i120, %i16
+  %i31 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i021, float %.i122, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i023 = extractelement <4 x float> %i31, i64 0
+  %.i024 = fadd reassoc nnan nsz arcp contract afn float %.i023, %.i011
+  %.i125 = extractelement <4 x float> %i31, i64 1
+  %.i126 = fadd reassoc nnan nsz arcp contract afn float %.i125, %.i114
+  %.i227 = extractelement <4 x float> %i31, i64 2
+  %.i228 = fadd reassoc nnan nsz arcp contract afn float %.i227, %.i216
+  %.i329 = extractelement <4 x float> %i31, i64 3
+  %.i330 = fadd reassoc nnan nsz arcp contract afn float %.i329, %.i318
+  %i32 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 64, i32 0), !invariant.load !0
+  %i33 = shufflevector <2 x i32> %i32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i34 = bitcast <4 x i32> %i33 to <4 x float>
+  %.i031 = extractelement <4 x float> %i34, i64 0
+  %.i132 = extractelement <4 x float> %i34, i64 1
+  %.i033 = fadd reassoc nnan nsz arcp contract afn float %.i031, %i19
+  %.i134 = fadd reassoc nnan nsz arcp contract afn float %.i132, %i16
+  %i35 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i033, float %.i134, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i035 = extractelement <4 x float> %i35, i64 0
+  %.i036 = fadd reassoc nnan nsz arcp contract afn float %.i035, %.i024
+  %.i137 = extractelement <4 x float> %i35, i64 1
+  %.i138 = fadd reassoc nnan nsz arcp contract afn float %.i137, %.i126
+  %.i239 = extractelement <4 x float> %i35, i64 2
+  %.i240 = fadd reassoc nnan nsz arcp contract afn float %.i239, %.i228
+  %.i341 = extractelement <4 x float> %i35, i64 3
+  %.i342 = fadd reassoc nnan nsz arcp contract afn float %.i341, %.i330
+  %i36 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 80, i32 0), !invariant.load !0
+  %i37 = shufflevector <2 x i32> %i36, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i38 = bitcast <4 x i32> %i37 to <4 x float>
+  %.i043 = extractelement <4 x float> %i38, i64 0
+  %.i144 = extractelement <4 x float> %i38, i64 1
+  %.i045 = fadd reassoc nnan nsz arcp contract afn float %.i043, %i19
+  %.i146 = fadd reassoc nnan nsz arcp contract afn float %.i144, %i16
+  %i39 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i045, float %.i146, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i047 = extractelement <4 x float> %i39, i64 0
+  %.i048 = fadd reassoc nnan nsz arcp contract afn float %.i047, %.i036
+  %.i149 = extractelement <4 x float> %i39, i64 1
+  %.i150 = fadd reassoc nnan nsz arcp contract afn float %.i149, %.i138
+  %.i251 = extractelement <4 x float> %i39, i64 2
+  %.i252 = fadd reassoc nnan nsz arcp contract afn float %.i251, %.i240
+  %.i353 = extractelement <4 x float> %i39, i64 3
+  %.i354 = fadd reassoc nnan nsz arcp contract afn float %.i353, %.i342
+  %i40 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 96, i32 0), !invariant.load !0
+  %i41 = shufflevector <2 x i32> %i40, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i42 = bitcast <4 x i32> %i41 to <4 x float>
+  %.i055 = extractelement <4 x float> %i42, i64 0
+  %.i156 = extractelement <4 x float> %i42, i64 1
+  %.i057 = fadd reassoc nnan nsz arcp contract afn float %.i055, %i19
+  %.i158 = fadd reassoc nnan nsz arcp contract afn float %.i156, %i16
+  %i43 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i057, float %.i158, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i059 = extractelement <4 x float> %i43, i64 0
+  %.i060 = fadd reassoc nnan nsz arcp contract afn float %.i059, %.i048
+  %.i161 = extractelement <4 x float> %i43, i64 1
+  %.i162 = fadd reassoc nnan nsz arcp contract afn float %.i161, %.i150
+  %.i263 = extractelement <4 x float> %i43, i64 2
+  %.i264 = fadd reassoc nnan nsz arcp contract afn float %.i263, %.i252
+  %.i365 = extractelement <4 x float> %i43, i64 3
+  %.i366 = fadd reassoc nnan nsz arcp contract afn float %.i365, %.i354
+  %i44 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 112, i32 0), !invariant.load !0
+  %i45 = shufflevector <2 x i32> %i44, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i46 = bitcast <4 x i32> %i45 to <4 x float>
+  %.i067 = extractelement <4 x float> %i46, i64 0
+  %.i168 = extractelement <4 x float> %i46, i64 1
+  %.i069 = fadd reassoc nnan nsz arcp contract afn float %.i067, %i19
+  %.i170 = fadd reassoc nnan nsz arcp contract afn float %.i168, %i16
+  %i47 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i069, float %.i170, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i071 = extractelement <4 x float> %i47, i64 0
+  %.i072 = fadd reassoc nnan nsz arcp contract afn float %.i071, %.i060
+  %.i173 = extractelement <4 x float> %i47, i64 1
+  %.i174 = fadd reassoc nnan nsz arcp contract afn float %.i173, %.i162
+  %.i275 = extractelement <4 x float> %i47, i64 2
+  %.i276 = fadd reassoc nnan nsz arcp contract afn float %.i275, %.i264
+  %.i377 = extractelement <4 x float> %i47, i64 3
+  %.i378 = fadd reassoc nnan nsz arcp contract afn float %.i377, %.i366
+  %i48 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 128, i32 0), !invariant.load !0
+  %i49 = shufflevector <2 x i32> %i48, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i50 = bitcast <4 x i32> %i49 to <4 x float>
+  %.i079 = extractelement <4 x float> %i50, i64 0
+  %.i180 = extractelement <4 x float> %i50, i64 1
+  %.i081 = fadd reassoc nnan nsz arcp contract afn float %.i079, %i19
+  %.i182 = fadd reassoc nnan nsz arcp contract afn float %.i180, %i16
+  %i51 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i081, float %.i182, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i083 = extractelement <4 x float> %i51, i64 0
+  %.i084 = fadd reassoc nnan nsz arcp contract afn float %.i083, %.i072
+  %.i185 = extractelement <4 x float> %i51, i64 1
+  %.i186 = fadd reassoc nnan nsz arcp contract afn float %.i185, %.i174
+  %.i287 = extractelement <4 x float> %i51, i64 2
+  %.i288 = fadd reassoc nnan nsz arcp contract afn float %.i287, %.i276
+  %.i389 = extractelement <4 x float> %i51, i64 3
+  %.i390 = fadd reassoc nnan nsz arcp contract afn float %.i389, %.i378
+  %i52 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 144, i32 0), !invariant.load !0
+  %i53 = shufflevector <2 x i32> %i52, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i54 = bitcast <4 x i32> %i53 to <4 x float>
+  %.i091 = extractelement <4 x float> %i54, i64 0
+  %.i192 = extractelement <4 x float> %i54, i64 1
+  %.i093 = fadd reassoc nnan nsz arcp contract afn float %.i091, %i19
+  %.i194 = fadd reassoc nnan nsz arcp contract afn float %.i192, %i16
+  %i55 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i093, float %.i194, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i095 = extractelement <4 x float> %i55, i64 0
+  %.i096 = fadd reassoc nnan nsz arcp contract afn float %.i095, %.i084
+  %.i197 = extractelement <4 x float> %i55, i64 1
+  %.i198 = fadd reassoc nnan nsz arcp contract afn float %.i197, %.i186
+  %.i299 = extractelement <4 x float> %i55, i64 2
+  %.i2100 = fadd reassoc nnan nsz arcp contract afn float %.i299, %.i288
+  %.i3101 = extractelement <4 x float> %i55, i64 3
+  %.i3102 = fadd reassoc nnan nsz arcp contract afn float %.i3101, %.i390
+  %i56 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 160, i32 0), !invariant.load !0
+  %i57 = shufflevector <2 x i32> %i56, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i58 = bitcast <4 x i32> %i57 to <4 x float>
+  %.i0103 = extractelement <4 x float> %i58, i64 0
+  %.i1104 = extractelement <4 x float> %i58, i64 1
+  %.i0105 = fadd reassoc nnan nsz arcp contract afn float %.i0103, %i19
+  %.i1106 = fadd reassoc nnan nsz arcp contract afn float %.i1104, %i16
+  %i59 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0105, float %.i1106, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0107 = extractelement <4 x float> %i59, i64 0
+  %.i0108 = fadd reassoc nnan nsz arcp contract afn float %.i0107, %.i096
+  %.i1109 = extractelement <4 x float> %i59, i64 1
+  %.i1110 = fadd reassoc nnan nsz arcp contract afn float %.i1109, %.i198
+  %.i2111 = extractelement <4 x float> %i59, i64 2
+  %.i2112 = fadd reassoc nnan nsz arcp contract afn float %.i2111, %.i2100
+  %.i3113 = extractelement <4 x float> %i59, i64 3
+  %.i3114 = fadd reassoc nnan nsz arcp contract afn float %.i3113, %.i3102
+  %i60 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 176, i32 0), !invariant.load !0
+  %i61 = shufflevector <2 x i32> %i60, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i62 = bitcast <4 x i32> %i61 to <4 x float>
+  %.i0115 = extractelement <4 x float> %i62, i64 0
+  %.i1116 = extractelement <4 x float> %i62, i64 1
+  %.i0117 = fadd reassoc nnan nsz arcp contract afn float %.i0115, %i19
+  %.i1118 = fadd reassoc nnan nsz arcp contract afn float %.i1116, %i16
+  %i63 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0117, float %.i1118, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0119 = extractelement <4 x float> %i63, i64 0
+  %.i0120 = fadd reassoc nnan nsz arcp contract afn float %.i0119, %.i0108
+  %.i1121 = extractelement <4 x float> %i63, i64 1
+  %.i1122 = fadd reassoc nnan nsz arcp contract afn float %.i1121, %.i1110
+  %.i2123 = extractelement <4 x float> %i63, i64 2
+  %.i2124 = fadd reassoc nnan nsz arcp contract afn float %.i2123, %.i2112
+  %.i3125 = extractelement <4 x float> %i63, i64 3
+  %.i3126 = fadd reassoc nnan nsz arcp contract afn float %.i3125, %.i3114
+  %i64 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 192, i32 0), !invariant.load !0
+  %i65 = shufflevector <2 x i32> %i64, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i66 = bitcast <4 x i32> %i65 to <4 x float>
+  %.i0127 = extractelement <4 x float> %i66, i64 0
+  %.i1128 = extractelement <4 x float> %i66, i64 1
+  %.i0129 = fadd reassoc nnan nsz arcp contract afn float %.i0127, %i19
+  %.i1130 = fadd reassoc nnan nsz arcp contract afn float %.i1128, %i16
+  %i67 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0129, float %.i1130, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0131 = extractelement <4 x float> %i67, i64 0
+  %.i0132 = fadd reassoc nnan nsz arcp contract afn float %.i0131, %.i0120
+  %.i1133 = extractelement <4 x float> %i67, i64 1
+  %.i1134 = fadd reassoc nnan nsz arcp contract afn float %.i1133, %.i1122
+  %.i2135 = extractelement <4 x float> %i67, i64 2
+  %.i2136 = fadd reassoc nnan nsz arcp contract afn float %.i2135, %.i2124
+  %.i3137 = extractelement <4 x float> %i67, i64 3
+  %.i3138 = fadd reassoc nnan nsz arcp contract afn float %.i3137, %.i3126
+  %i68 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 208, i32 0), !invariant.load !0
+  %i69 = shufflevector <2 x i32> %i68, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i70 = bitcast <4 x i32> %i69 to <4 x float>
+  %.i0139 = extractelement <4 x float> %i70, i64 0
+  %.i1140 = extractelement <4 x float> %i70, i64 1
+  %.i0141 = fadd reassoc nnan nsz arcp contract afn float %.i0139, %i19
+  %.i1142 = fadd reassoc nnan nsz arcp contract afn float %.i1140, %i16
+  %i71 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0141, float %.i1142, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0143 = extractelement <4 x float> %i71, i64 0
+  %.i0144 = fadd reassoc nnan nsz arcp contract afn float %.i0143, %.i0132
+  %.i1145 = extractelement <4 x float> %i71, i64 1
+  %.i1146 = fadd reassoc nnan nsz arcp contract afn float %.i1145, %.i1134
+  %.i2147 = extractelement <4 x float> %i71, i64 2
+  %.i2148 = fadd reassoc nnan nsz arcp contract afn float %.i2147, %.i2136
+  %.i3149 = extractelement <4 x float> %i71, i64 3
+  %.i3150 = fadd reassoc nnan nsz arcp contract afn float %.i3149, %.i3138
+  %i72 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 224, i32 0), !invariant.load !0
+  %i73 = shufflevector <2 x i32> %i72, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i74 = bitcast <4 x i32> %i73 to <4 x float>
+  %.i0151 = extractelement <4 x float> %i74, i64 0
+  %.i1152 = extractelement <4 x float> %i74, i64 1
+  %.i0153 = fadd reassoc nnan nsz arcp contract afn float %.i0151, %i19
+  %.i1154 = fadd reassoc nnan nsz arcp contract afn float %.i1152, %i16
+  %i75 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0153, float %.i1154, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0155 = extractelement <4 x float> %i75, i64 0
+  %.i0156 = fadd reassoc nnan nsz arcp contract afn float %.i0155, %.i0144
+  %.i1157 = extractelement <4 x float> %i75, i64 1
+  %.i1158 = fadd reassoc nnan nsz arcp contract afn float %.i1157, %.i1146
+  %.i2159 = extractelement <4 x float> %i75, i64 2
+  %.i2160 = fadd reassoc nnan nsz arcp contract afn float %.i2159, %.i2148
+  %.i3161 = extractelement <4 x float> %i75, i64 3
+  %.i3162 = fadd reassoc nnan nsz arcp contract afn float %.i3161, %.i3150
+  %i76 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 240, i32 0), !invariant.load !0
+  %i77 = shufflevector <2 x i32> %i76, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i78 = bitcast <4 x i32> %i77 to <4 x float>
+  %.i0163 = extractelement <4 x float> %i78, i64 0
+  %.i1164 = extractelement <4 x float> %i78, i64 1
+  %.i0165 = fadd reassoc nnan nsz arcp contract afn float %.i0163, %i19
+  %.i1166 = fadd reassoc nnan nsz arcp contract afn float %.i1164, %i16
+  %i79 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0165, float %.i1166, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0167 = extractelement <4 x float> %i79, i64 0
+  %.i0168 = fadd reassoc nnan nsz arcp contract afn float %.i0167, %.i0156
+  %.i1169 = extractelement <4 x float> %i79, i64 1
+  %.i1170 = fadd reassoc nnan nsz arcp contract afn float %.i1169, %.i1158
+  %.i2171 = extractelement <4 x float> %i79, i64 2
+  %.i2172 = fadd reassoc nnan nsz arcp contract afn float %.i2171, %.i2160
+  %.i3173 = extractelement <4 x float> %i79, i64 3
+  %.i3174 = fadd reassoc nnan nsz arcp contract afn float %.i3173, %.i3162
+  %i80 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %i5, i32 256, i32 0), !invariant.load !0
+  %i81 = shufflevector <2 x i32> %i80, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %i82 = bitcast <4 x i32> %i81 to <4 x float>
+  %.i0175 = extractelement <4 x float> %i82, i64 0
+  %.i1176 = extractelement <4 x float> %i82, i64 1
+  %.i0177 = fadd reassoc nnan nsz arcp contract afn float %.i0175, %i19
+  %.i1178 = fadd reassoc nnan nsz arcp contract afn float %.i1176, %i16
+  %i83 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %.i0177, float %.i1178, <8 x i32> %i13, <4 x i32> %i9, i1 false, i32 0, i32 0)
+  %.i0179 = extractelement <4 x float> %i83, i64 0
+  %.i0180 = fadd reassoc nnan nsz arcp contract afn float %.i0179, %.i0168
+  %.i1181 = extractelement <4 x float> %i83, i64 1
+  %.i1182 = fadd reassoc nnan nsz arcp contract afn float %.i1181, %.i1170
+  %.i2183 = extractelement <4 x float> %i83, i64 2
+  %.i2184 = fadd reassoc nnan nsz arcp contract afn float %.i2183, %.i2172
+  %.i3185 = extractelement <4 x float> %i83, i64 3
+  %.i3186 = fadd reassoc nnan nsz arcp contract afn float %.i3185, %.i3174
+  %i84 = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %.i0180, float %.i1182)
+  %i85 = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %.i2184, float %.i3186)
+  %i86 = bitcast <2 x half> %i84 to float
+  %i87 = bitcast <2 x half> %i85 to float
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 3, float %i86, float %i87, float poison, float poison, i1 true, i1 true)
+  ret void
+}
+
+declare noundef i64 @llvm.amdgcn.s.getpc() #3
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #5
+declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #4
+declare float @llvm.amdgcn.lds.param.load(i32 immarg, i32 immarg, i32) #3
+declare float @llvm.amdgcn.interp.inreg.p10(float, float, float) #3
+declare float @llvm.amdgcn.interp.inreg.p2(float, float, float) #3
+declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32 immarg) #8
+
+attributes #2 = { alwaysinline nounwind memory(readwrite) "amdgpu-sched-strategy"="max-memory-clause"}
+attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #5 = { nocallback nofree nosync nounwind willreturn memory(read) }
+attributes #8 = { nocallback nofree nosync nounwind willreturn memory(none) }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/hazards-gfx950.mir b/llvm/test/CodeGen/AMDGPU/hazards-gfx950.mir
index 7583431675095..1bbad901d16b2 100644
--- a/llvm/test/CodeGen/AMDGPU/hazards-gfx950.mir
+++ b/llvm/test/CodeGen/AMDGPU/hazards-gfx950.mir
@@ -1,121 +1,140 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs -run-pass=post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane16_swap_vop1
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vop3_write_exec_permlane16_swap_vop1
-# GCN:      V_CMPX_EQ_I32_e64
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vop3_write_exec_permlane16_swap_vop1
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vop3_write_exec_permlane16_swap_vop1
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop3
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane16_swap_vop3
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop3
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vop3_write_exec_permlane16_swap_vop3
-# GCN:      V_CMPX_EQ_I32_e64
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vop3_write_exec_permlane16_swap_vop3
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vop3_write_exec_permlane16_swap_vop3
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane32_swap_vop1
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane32_swap_vop1
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane32_swap_vop1
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vop3_write_exec_permlane32_swap_vop1
-# GCN:      V_CMPX_EQ_I32_e64
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vop3_write_exec_permlane32_swap_vop1
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vop3_write_exec_permlane32_swap_vop1
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane32_swap_vop3
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane32_swap_vop3
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane32_swap_vop3
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vop3_write_exec_permlane32_swap_vop3
-# GCN:      V_CMPX_EQ_I32_e64
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vop3_write_exec_permlane32_swap_vop3
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vop3_write_exec_permlane32_swap_vop3
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1__nowait
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane16_swap_vop1__nowait
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1__nowait
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     $vgpr2 = V_MOV_B32_e32 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
@@ -125,17 +144,19 @@ body:             |
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1__wait1
-# GCN:      V_CMPX_EQ_I32_e32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: V_MOV_B32
-# GCN-NEXT: S_NOP 0
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_vopc_write_exec_permlane16_swap_vop1__wait1
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vcmpx_vopc_write_exec_permlane16_swap_vop1__wait1
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     V_CMPX_EQ_I32_e32 $vgpr0, $vgpr1, implicit-def $exec, implicit-def $vcc, implicit $exec
     $vgpr2 = V_MOV_B32_e32 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
@@ -144,114 +165,1058 @@ body:             |
 ...
 
 ---
-# GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_0
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 1
-# GCN-NEXT: V_PERMLANE
 name:            valu_write_vdst_read_permlane16_swap_0
 body:             |
   bb.0:
     liveins: $vgpr1
+    ; GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_0
+    ; GCN: liveins: $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 1
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_1
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 1
-# GCN-NEXT: V_PERMLANE
 name:            valu_write_vdst_read_permlane16_swap_1
 body:             |
   bb.0:
     liveins: $vgpr0
+    ; GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_1
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 1
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: valu_write_vdst_read_permlane32_swap_0
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 1
-# GCN-NEXT: V_PERMLANE
 name:            valu_write_vdst_read_permlane32_swap_0
 body:             |
   bb.0:
     liveins: $vgpr1
+    ; GCN-LABEL: name: valu_write_vdst_read_permlane32_swap_0
+    ; GCN: liveins: $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 1
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: valu_write_vdst_read_permlane32_swap_1
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 1
-# GCN-NEXT: V_PERMLANE
 name:            valu_write_vdst_read_permlane32_swap_1
 body:             |
   bb.0:
     liveins: $vgpr0
+    ; GCN-LABEL: name: valu_write_vdst_read_permlane32_swap_1
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 1
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
 # No hazard, write of other register
-# GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_0_otherreg
-# GCN:      V_MOV_B32
-# GCN-NEXT: V_PERMLANE
 name:            valu_write_vdst_read_permlane16_swap_0_otherreg
 body:             |
   bb.0:
     liveins: $vgpr1
+    ; GCN-LABEL: name: valu_write_vdst_read_permlane16_swap_0_otherreg
+    ; GCN: liveins: $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
     $vgpr2 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE16_SWAP_B32_e64 killed $vgpr0, killed $vgpr1, -1, 1, implicit $exec
 ...
 
 ---
 # Both permlane hazards at once.
-# GCN-LABEL: name: valu_writes_vdst__vcmpx_write_exec__permlane32_swap
-# GCN:      V_MOV_B32
-# GCN:      V_CMPX_EQ_I32
-# GCN-NEXT: S_NOP 3
-# GCN-NEXT: V_PERMLANE
 name:            valu_writes_vdst__vcmpx_write_exec__permlane32_swap
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: valu_writes_vdst__vcmpx_write_exec__permlane32_swap
+    ; GCN: liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
+    ; GCN-NEXT: S_NOP 3
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_write_exec__valu_writes_vdst___permlane32_swap
-# GCN:      V_CMPX_EQ_I32
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 2
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_write_exec__valu_writes_vdst___permlane32_swap
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: vcmpx_write_exec__valu_writes_vdst___permlane32_swap
+    ; GCN: liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 2
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
 
 ---
-# GCN-LABEL: name: vcmpx_write_exec__valu_writes_vdstx2___permlane32_swap
-# GCN:      V_CMPX_EQ_I32
-# GCN:      V_MOV_B32
-# GCN:      V_MOV_B32
-# GCN-NEXT: S_NOP 1
-# GCN-NEXT: V_PERMLANE
 name:            vcmpx_write_exec__valu_writes_vdstx2___permlane32_swap
 body:             |
   bb.0:
     liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: vcmpx_write_exec__valu_writes_vdstx2___permlane32_swap
+    ; GCN: liveins: $vgpr0, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_NOP 1
+    ; GCN-NEXT: renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
     $exec = V_CMPX_EQ_I32_e64 $vgpr2, $vgpr3, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     renamable $vgpr0, renamable $vgpr1 = V_PERMLANE32_SWAP_B32_e32 killed $vgpr0, killed $vgpr1, implicit $exec
 ...
+
+
+---
+name:            test_cvt_scalef32_sr_fp8_bf16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_sr_fp8_bf16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr5, 0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            test_cvt_scalef32_sr_fp8_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_sr_fp8_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr5, 0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            test_cvt_scalef32_sr_fp8_f32_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_sr_fp8_f32_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_F32_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr2 = V_ADD_U32_e32 4, killed $vgpr5, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0
+    renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr5 = V_CVT_SCALEF32_SR_FP8_F32_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr5, 0, implicit $mode, implicit $exec
+    renamable $vgpr2 = V_ADD_U32_e32 4, killed $vgpr5, implicit $exec
+    GLOBAL_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec
+    S_ENDPGM 0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp8_f32_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp8_f32_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP8_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP8_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp8_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp8_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP8_F16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP8_F16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp8_bf16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp8_bf16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_sr_bf8_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_sr_bf8_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_sr_bf8_f32_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_sr_bf8_f32_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F32_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F32_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_bf8_f32_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_bf8_f32_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_bf8_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_bf8_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_F16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_F16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_bf8_bf16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_bf8_bf16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_BF16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_BF8_BF16_e64 8, killed $vgpr1, 0, killed $vgpr2, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_PK_ADD_U16 8, killed $vgpr0, 8, $vgpr0, 0, 0, 0, 0, 0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f32_neg_hazard_opsel0
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f32_neg_hazard_opsel0
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 0, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 0, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f32_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f32_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 4, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 4, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f32_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f32_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 0, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr1 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 0, killed $vgpr1, 0, killed $vgpr2, 0, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr1 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f32_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f32_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 4, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr1 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr1 = V_AND_B32_e32 2147483647, killed $vgpr1, implicit $exec
+    renamable $vgpr2 = V_XOR_B32_e32 -2147483648, killed $vgpr2, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_FP4_F32_e64 8, killed $vgpr1, 0, killed $vgpr2, 4, killed $vgpr3, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr1 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_bf16_neg_opsel0_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_bf16_neg_opsel0_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 0, killed $vgpr2, 0, killed $vgpr3, 0, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 0, killed $vgpr2, 0, killed $vgpr3, 0, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_bf16_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_bf16_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_bf16_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_bf16_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 0, killed $vgpr2, 0, killed $vgpr3, 0, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 0, killed $vgpr2, 0, killed $vgpr3, 0, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_bf16_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_bf16_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_BF16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_f32_neg_opsel0_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_f32_neg_opsel0_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 0, killed $vgpr2_vgpr3, 0, killed $vgpr4, 0, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 0, killed $vgpr2_vgpr3, 0, killed $vgpr4, 0, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_f32_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_f32_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 8, killed $vgpr2_vgpr3, 0, killed $vgpr4, 4, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 8, killed $vgpr2_vgpr3, 0, killed $vgpr4, 4, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_f32_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_f32_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 0, killed $vgpr2_vgpr3, 0, killed $vgpr4, 0, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 0, killed $vgpr2_vgpr3, 0, killed $vgpr4, 0, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_scalef32_sr_pk_fp4_f32_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: test_scalef32_sr_pk_fp4_f32_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 8, killed $vgpr2_vgpr3, 0, killed $vgpr4, 4, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    early-clobber renamable $vgpr1 = V_CVT_SCALEF32_SR_PK_FP4_F32_e64 8, killed $vgpr2_vgpr3, 0, killed $vgpr4, 4, killed $vgpr5, killed $vgpr0, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f16_neg_opsel0_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f16_neg_opsel0_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f16_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f16_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f16_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f16_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_f16_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_f16_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr1, $vgpr1, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_bf16_neg_opsel0_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_bf16_neg_opsel0_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_bf16_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_bf16_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr2, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_bf16_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_bf16_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 0, killed $vgpr0, 0, killed $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_fp4_bf16_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_fp4_bf16_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_BF16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_hazard_skipping_over_meta_instr
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_hazard_skipping_over_meta_instr
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: $vgpr4 = KILL
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    $vgpr4 = KILL
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_f16_to_fp4_to_f16_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_f16_to_fp4_to_f16_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 4, killed $vgpr2, 0, killed $vgpr1, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 4, killed $vgpr2, 0, killed $vgpr1, 0, implicit $mode, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_pk_f16_fp4_opsel0_neg_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_f16_fp4_opsel0_neg_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 0, killed $vgpr2, 0, killed $vgpr1, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 0, killed $vgpr2, 0, killed $vgpr1, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+
+---
+name:            test_cvt_scalef32_pk_f16_fp4_opsel3_neg_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_pk_f16_fp4_opsel3_neg_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 4, killed $vgpr2, 4, killed $vgpr1, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = V_CVT_SCALEF32_PK_F16_FP4_e64 4, killed $vgpr2, 4, killed $vgpr1, 0, implicit $mode, implicit $exec
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_hazard_pseudo
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-LABEL: name: test_cvt_scalef32_hazard_pseudo
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAITCNT 3952
+    ; GCN-NEXT: renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: WAVE_BARRIER
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+    S_WAITCNT 3952
+    renamable $vgpr0 = V_CVT_SCALEF32_SR_BF8_F16_e64 8, killed $vgpr2, 0, killed $vgpr3, 4, killed $vgpr4, killed $vgpr0, 0, implicit $mode, implicit $exec
+    WAVE_BARRIER
+    renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr0, $vgpr0, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_call_consuming_cvt_scalef32_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_call_consuming_cvt_scalef32_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: BUNDLE implicit-def $sgpr0_sgpr1, implicit-def $sgpr0, implicit-def $sgpr0_lo16, implicit-def $sgpr0_hi16, implicit-def $sgpr1, implicit-def $sgpr1_lo16, implicit-def $sgpr1_hi16, implicit-def $scc {
+    ; GCN-NEXT:   $sgpr0_sgpr1 = S_GETPC_B64
+    ; GCN-NEXT:   $sgpr0 = S_ADD_U32 internal $sgpr0, target-flags(amdgpu-gotprel32-lo) @test_cvt_scalef32_hazard_pseudo + 4, implicit-def $scc
+    ; GCN-NEXT:   $sgpr1 = S_ADDC_U32 internal $sgpr1, target-flags(amdgpu-gotprel32-hi) @test_cvt_scalef32_hazard_pseudo + 12, implicit-def $scc, implicit internal $scc
+    ; GCN-NEXT: }
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr0_sgpr1, @test_cvt_scalef32_hazard_pseudo, csr_amdgpu_gfx90ainsts, implicit undef $sgpr4_sgpr5, implicit undef $sgpr6_sgpr7, implicit killed $sgpr8_sgpr9, implicit undef $sgpr10_sgpr11, implicit undef $sgpr12, implicit undef $sgpr13, implicit undef $sgpr14, implicit-def $sgpr15, implicit undef $vgpr31, implicit killed $vgpr2, implicit-def $vgpr2
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
+    BUNDLE implicit-def $sgpr0_sgpr1, implicit-def $sgpr0, implicit-def $sgpr0_lo16, implicit-def $sgpr0_hi16, implicit-def $sgpr1, implicit-def $sgpr1_lo16, implicit-def $sgpr1_hi16, implicit-def $scc {
+      $sgpr0_sgpr1 = S_GETPC_B64
+      $sgpr0 = S_ADD_U32 internal $sgpr0, target-flags(amdgpu-gotprel32-lo) @test_cvt_scalef32_hazard_pseudo + 4, implicit-def $scc
+      $sgpr1 = S_ADDC_U32 internal $sgpr1, target-flags(amdgpu-gotprel32-hi) @test_cvt_scalef32_hazard_pseudo + 12, implicit-def $scc, implicit internal $scc
+    }
+    renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr0_sgpr1, 0, 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr0_sgpr1, @test_cvt_scalef32_hazard_pseudo, csr_amdgpu_gfx90ainsts, implicit undef $sgpr4_sgpr5, implicit undef $sgpr6_sgpr7, implicit killed $sgpr8_sgpr9, implicit undef $sgpr10_sgpr11, implicit undef $sgpr12, implicit undef $sgpr13, implicit undef $sgpr14, implicit-def $sgpr15, implicit undef $vgpr31, implicit killed $vgpr2, implicit-def $vgpr2
+    SI_RETURN_TO_EPILOG killed $vgpr0
+...
+
+---
+name:            test_cvt_scalef32_inlineasm_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: test_cvt_scalef32_inlineasm_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2228233 /* reguse:VGPR_32 */, killed renamable $vgpr2
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2228233 /* reguse:VGPR_32 */, killed renamable $vgpr2
+    S_SETPC_B64_return undef $sgpr30_sgpr31
+...
+
+---
+name:            test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_neg_opsel0_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_neg_opsel0_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr3, 0, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr4, implicit $exec, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 0, killed $vgpr0, 0, killed $vgpr3, 0, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    $vgpr0 = V_MOV_B32_e32 killed $vgpr4, implicit $exec, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel3_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel3_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr4, implicit $exec, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    $vgpr0 = V_MOV_B32_e32 killed $vgpr4, implicit $exec, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel0_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel0_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, $vgpr0, 0, $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 0, $vgpr0, 0, $vgpr1, 0, killed $vgpr2, 0, implicit $mode, implicit $exec
+    early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel3_neg_fp4_as_src_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scale_cvt_scalef32_sr_pk_fp4_f16_opsel3_neg_fp4_as_src_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    early-clobber renamable $vgpr4 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr2, 0, implicit $mode, implicit $exec
+    $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
+
+---
+name:            test_cvt_scale_cvt_scale_waw_hazard
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: test_cvt_scale_cvt_scale_waw_hazard
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT 0
+    ; GCN-NEXT: renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: early-clobber renamable $vgpr2 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr1, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+    S_WAITCNT 0
+    renamable $vgpr2 = V_CVT_SCALEF32_PK_FP4_F16_e64 8, $vgpr0, 0, $vgpr1, 4, killed $vgpr2, 0, implicit $mode, implicit $exec
+    early-clobber renamable $vgpr2 = V_CVT_SCALEF32_SR_PK_FP4_F16_e64 8, killed $vgpr0, 0, killed $vgpr3, 4, killed $vgpr1, killed $vgpr1, 0, implicit $mode, implicit $exec
+    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit killed $vgpr0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
index e37b6ff10ffa9..3563e737f5520 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
@@ -5,7 +5,7 @@
 define amdgpu_kernel void @get_local_size_x(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_local_size_x(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
+; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
 ; GCN-NEXT:    [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 4
 ; GCN-NEXT:    store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
@@ -25,7 +25,7 @@ define amdgpu_kernel void @get_local_size_x(ptr addrspace(1) %out) #0 {
 define amdgpu_kernel void @get_local_size_y(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_local_size_y(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
+; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
 ; GCN-NEXT:    [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 2
 ; GCN-NEXT:    store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
@@ -46,7 +46,7 @@ define amdgpu_kernel void @get_local_size_y(ptr addrspace(1) %out) #0 {
 define amdgpu_kernel void @get_local_size_z(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_local_size_z(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
+; GCN-NEXT:    [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
 ; GCN-NEXT:    [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 4
 ; GCN-NEXT:    store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
@@ -106,7 +106,7 @@ define amdgpu_kernel void @get_remainder_z(ptr addrspace(1) %out) #0 {
 define amdgpu_kernel void @get_work_group_size_x(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_work_group_size_x(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_X:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
+; GCN-NEXT:    [[GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
 ; GCN-NEXT:    [[GROUP_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[GEP_X]], align 4
 ; GCN-NEXT:    store i16 [[GROUP_SIZE_X]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
@@ -122,7 +122,7 @@ define amdgpu_kernel void @get_work_group_size_x(ptr addrspace(1) %out) #0 {
 define amdgpu_kernel void @get_work_group_size_y(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_work_group_size_y(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_Y:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
+; GCN-NEXT:    [[GEP_Y:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
 ; GCN-NEXT:    [[GROUP_SIZE_Y:%.*]] = load i16, ptr addrspace(4) [[GEP_Y]], align 2
 ; GCN-NEXT:    store i16 [[GROUP_SIZE_Y]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
@@ -138,7 +138,7 @@ define amdgpu_kernel void @get_work_group_size_y(ptr addrspace(1) %out) #0 {
 define amdgpu_kernel void @get_work_group_size_z(ptr addrspace(1) %out) #0 {
 ; GCN-LABEL: @get_work_group_size_z(
 ; GCN-NEXT:    [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; GCN-NEXT:    [[GEP_Z:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
+; GCN-NEXT:    [[GEP_Z:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
 ; GCN-NEXT:    [[GROUP_SIZE_Z:%.*]] = load i16, ptr addrspace(4) [[GEP_Z]], align 4
 ; GCN-NEXT:    store i16 [[GROUP_SIZE_Z]], ptr addrspace(1) [[OUT:%.*]], align 2
 ; GCN-NEXT:    ret void
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
index 7f334e0ca21e1..3d27b5fe7f30b 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
@@ -101,6 +101,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX9V5-NEXT:    flat_store_dword v[2:3], v0
 ; GFX9V5-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V5-NEXT:    s_endpgm
+
   %flat.private = addrspacecast ptr addrspace(5) %ptr.private to ptr
   %flat.local = addrspacecast ptr addrspace(3) %ptr.local to ptr
   store volatile i32 1, ptr %flat.private
diff --git a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll
index 4c21f87297455..5979f1bbacdd1 100644
--- a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll
@@ -258,25 +258,25 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 
 ;.
 ; V4: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; V4: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; V4: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V4: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V4: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V4: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V4: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; V4: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V4: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V4: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V4: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
 ;.
 ; V5: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; V5: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; V5: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V5: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V5: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V5: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V5: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; V5: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V5: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V5: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V5: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
 ;.
 ; V6: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; V6: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; V6: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V6: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V6: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
-; V6: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V6: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; V6: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V6: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V6: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
+; V6: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" }
 ;.
 ; V4: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 400}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-set-from-other-function.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-set-from-other-function.ll
index f419d89a7f0a4..b283a8fca8a39 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-set-from-other-function.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-set-from-other-function.ll
@@ -68,6 +68,6 @@ if.end:
   ret void
 }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR1]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/invalid-hidden-kernarg-in-kernel-signature.ll b/llvm/test/CodeGen/AMDGPU/invalid-hidden-kernarg-in-kernel-signature.ll
new file mode 100644
index 0000000000000..2e7b2b72088a2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/invalid-hidden-kernarg-in-kernel-signature.ll
@@ -0,0 +1,21 @@
+; RUN: not llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s 2>&1 | FileCheck -check-prefixes=ERROR,GISEL %s
+; RUN: not llc -global-isel=0 -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+; RUN: not llc -global-isel=1 -global-isel-abort=2 -amdgpu-ir-lower-kernel-arguments=0 -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s 2>&1 | FileCheck -check-prefixes=ERROR,GISEL %s
+; RUN: not llc -global-isel=0 -amdgpu-ir-lower-kernel-arguments=0 -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+
+define amdgpu_kernel void @no_free_sgprs_block_count_x_no_preload_diag(ptr addrspace(1) inreg %out, i512 inreg, i32 inreg "amdgpu-hidden-argument" %_hidden_block_count_x) #0 {
+; GISEL: warning: Instruction selection used fallback path for no_free_sgprs_block_count_x_no_preload_diag
+; ERROR: error: <unknown>:0:0: in function no_free_sgprs_block_count_x_no_preload_diag void (ptr addrspace(1), i512, i32): hidden argument in kernel signature was not preloaded
+  store i32 %_hidden_block_count_x, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @preloadremainder_z_no_preload_diag(ptr addrspace(1) inreg %out, i256 inreg, i32 inreg "amdgpu-hidden-argument" %_hidden_block_count_x, i32 inreg "amdgpu-hidden-argument" %_hidden_block_count_y, i32 inreg "amdgpu-hidden-argument" %_hidden_block_count_z, i16 inreg "amdgpu-hidden-argument" %_hidden_group_size_x, i16 inreg "amdgpu-hidden-argument" %_hidden_group_size_y, i16 inreg "amdgpu-hidden-argument" %_hidden_group_size_z, i16 inreg "amdgpu-hidden-argument" %_hidden_remainder_x, i16 inreg "amdgpu-hidden-argument" %_hidden_remainder_y, i16 inreg "amdgpu-hidden-argument" %_hidden_remainder_z) #0 {
+; GISEL: warning: Instruction selection used fallback path for preloadremainder_z_no_preload_diag
+; ERROR: error: <unknown>:0:0: in function preloadremainder_z_no_preload_diag void (ptr addrspace(1), i256, i32, i32, i32, i16, i16, i16, i16, i16, i16): hidden argument in kernel signature was not preloaded
+  %conv = zext i16 %_hidden_remainder_z to i32
+  store i32 %conv, ptr addrspace(1) %out
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
index 397502711283e..e70dc8f7a6576 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
@@ -100,59 +100,29 @@ define void @func_use_lds_global() {
 
 ; ERR: warning: <unknown>:0:0: in function func_use_lds_global_constexpr_cast void (ptr addrspace(1)): local memory global used by non-kernel function
 define void @func_use_lds_global_constexpr_cast(ptr addrspace(1) %out) {
-; GFX8-SDAG-LABEL: func_use_lds_global_constexpr_cast:
-; GFX8-SDAG:       ; %bb.0:
-; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT:    s_mov_b64 s[4:5], 0xc8
-; GFX8-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT:    s_trap 2
-; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: func_use_lds_global_constexpr_cast:
-; GFX8-GISEL:       ; %bb.0:
-; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT:    s_mov_b64 s[4:5], 0xc8
-; GFX8-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-GISEL-NEXT:    s_trap 2
-; GFX8-GISEL-NEXT:    flat_store_dword v[0:1], v0
-; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: func_use_lds_global_constexpr_cast:
-; GFX9-SDAG:       ; %bb.0:
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT:    s_trap 2
-; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; GFX8-LABEL: func_use_lds_global_constexpr_cast:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0xc8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_trap 2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-GISEL-LABEL: func_use_lds_global_constexpr_cast:
-; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT:    s_trap 2
-; GFX9-GISEL-NEXT:    global_store_dword v[0:1], v0, off
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: func_use_lds_global_constexpr_cast:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_trap 2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; SDAG-LABEL: func_use_lds_global_constexpr_cast:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_cbranch_execnz .LBB1_2
-; SDAG-NEXT:  ; %bb.1:
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-; SDAG-NEXT:  .LBB1_2:
-; SDAG-NEXT:    s_endpgm
-;
-; GISEL-LABEL: func_use_lds_global_constexpr_cast:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_cbranch_execnz .LBB1_2
-; GISEL-NEXT:  ; %bb.1:
-; GISEL-NEXT:    global_store_dword v[0:1], v0, off
-; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-; GISEL-NEXT:  .LBB1_2:
-; GISEL-NEXT:    s_endpgm
+; CHECK-LABEL: func_use_lds_global_constexpr_cast:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_cbranch_execnz .LBB1_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    s_endpgm
   store i32 ptrtoint (ptr addrspace(3) @lds to i32), ptr addrspace(1) %out, align 4
   ret void
 }
@@ -611,7 +581,3 @@ ret:
   ret i32 %phi
 }
 
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
-; GFX8: {{.*}}
-; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index ff2f4db0d7a5f..b6232cbc38496 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -2,15 +2,15 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950-SDAG %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950-GISEL %s
 
-declare i32 @llvm.amdgcn.bitop3.i32(i32, i32, i32, i8)
-declare i16 @llvm.amdgcn.bitop3.i16(i16, i16, i16, i8)
+declare i32 @llvm.amdgcn.bitop3.i32(i32, i32, i32, i32)
+declare i16 @llvm.amdgcn.bitop3.i16(i16, i16, i16, i32)
 
 define amdgpu_ps float @bitop3_b32_vvv(i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: bitop3_b32_vvv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0xf
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i8 15)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i32 15)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -20,7 +20,7 @@ define amdgpu_ps float @bitop3_b32_svv(i32 inreg %a, i32 %b, i32 %c) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_bitop3_b32 v0, s0, v0, v1 bitop3:0x10
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i8 16)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i32 16)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -31,7 +31,7 @@ define amdgpu_ps float @bitop3_b32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_bitop3_b32 v0, s0, v1, v0 bitop3:0x11
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i8 17)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i32 17)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -43,7 +43,7 @@ define amdgpu_ps float @bitop3_b32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_bitop3_b32 v0, s0, v0, v1 bitop3:0x12
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i8 18)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 %c, i32 18)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -60,7 +60,7 @@ define amdgpu_ps float @bitop3_b32_vvi(i32 %a, i32 %b) {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x13
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 1000, i8 19)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 %b, i32 1000, i32 19)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -79,7 +79,7 @@ define amdgpu_ps float @bitop3_b32_vii(i32 %a) {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x14
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 2000, i32 1000, i8 20)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 %a, i32 2000, i32 1000, i32 20)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -102,7 +102,7 @@ define amdgpu_ps float @bitop3_b32_iii() {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b32 v0, v0, v1, v2 bitop3:0x15
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 3000, i32 2000, i32 1000, i8 21)
+  %ret = call i32 @llvm.amdgcn.bitop3.i32(i32 3000, i32 2000, i32 1000, i32 21)
   %ret_cast = bitcast i32 %ret to float
   ret float %ret_cast
 }
@@ -112,7 +112,7 @@ define amdgpu_ps half @bitop3_b16_vvv(i16 %a, i16 %b, i16 %c) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_bitop3_b16 v0, v0, v1, v2 bitop3:0xf
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i8 15)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 15)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -122,7 +122,7 @@ define amdgpu_ps half @bitop3_b16_svv(i16 inreg %a, i16 %b, i16 %c) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_bitop3_b16 v0, s0, v0, v1 bitop3:0x10
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i8 16)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 16)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -133,7 +133,7 @@ define amdgpu_ps half @bitop3_b16_ssv(i16 inreg %a, i16 inreg %b, i16 %c) {
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_bitop3_b16 v0, s0, v1, v0 bitop3:0x11
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i8 17)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 17)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -145,7 +145,7 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_bitop3_b16 v0, s0, v0, v1 bitop3:0x12
 ; GCN-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i8 18)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 18)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -162,7 +162,7 @@ define amdgpu_ps half @bitop3_b16_vvi(i16 %a, i16 %b) {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b16 v0, v0, v1, v2 bitop3:0x13
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 1000, i8 19)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 1000, i32 19)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -181,7 +181,7 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b16 v0, v0, v1, v2 bitop3:0x14
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 2000, i16 1000, i8 20)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 2000, i16 1000, i32 20)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
@@ -203,7 +203,7 @@ define amdgpu_ps half @bitop3_b16_iii() {
 ; GFX950-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e8
 ; GFX950-GISEL-NEXT:    v_bitop3_b16 v0, v0, v1, v2 bitop3:0x15
 ; GFX950-GISEL-NEXT:    ; return to shader part epilog
-  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 3000, i16 2000, i16 1000, i8 21)
+  %ret = call i16 @llvm.amdgcn.bitop3.i16(i16 3000, i16 2000, i16 1000, i32 21)
   %ret_cast = bitcast i16 %ret to half
   ret half %ret_cast
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
index f80f2935856e3..046a72b9307d0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
@@ -1305,6 +1305,7 @@ define i32 @test_cvt_scalef32_fp4_f16_byte1(<2 x half> %src0, float %scale, i32
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_scalef32_pk_fp4_f16 v2, v0, v1 op_sel:[0,0,1,0]
+; GCN-NEXT:    s_nop 0
 ; GCN-NEXT:    v_mov_b32_e32 v0, v2
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.scalef32.pk.fp4.f16(i32 %old, <2 x half> %src0, float %scale, i32 1)
@@ -1351,6 +1352,7 @@ define i32 @test_cvt_scalef32_fp4_bf16_byte1(<2 x bfloat> %src0, float %scale, i
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_scalef32_pk_fp4_bf16 v2, v0, v1 op_sel:[0,0,1,0]
+; GCN-NEXT:    s_nop 0
 ; GCN-NEXT:    v_mov_b32_e32 v0, v2
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.scalef32.pk.fp4.bf16(i32 %old, <2 x bfloat> %src0, float %scale, i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
index 0689af0d56268..8481a3c2ccdb1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
@@ -6,6 +6,8 @@ declare <2 x i32>    @llvm.amdgcn.ds.read.tr4.b64.v2i32.p3(ptr addrspace(3))
 declare <2 x i32>    @llvm.amdgcn.ds.read.tr8.b64.v2i32.p3(ptr addrspace(3))
 declare <3 x i32>    @llvm.amdgcn.ds.read.tr6.b64.v3i32.p3(ptr addrspace(3))
 declare <4 x i16>    @llvm.amdgcn.ds.read.tr16.b64.v4i16.p3(ptr addrspace(3))
+declare <4 x half>    @llvm.amdgcn.ds.read.tr16.b64.v4f16.p3(ptr addrspace(3))
+declare <4 x bfloat>    @llvm.amdgcn.ds.read.tr16.b64.v4bf16.p3(ptr addrspace(3))
 
 define amdgpu_ps void @ds_read_b64_tr_b4(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
 ; GFX950-SDAG-LABEL: ds_read_b64_tr_b4:
@@ -106,3 +108,53 @@ entry:
   store <4 x i16> %val, ptr addrspace(1) %use
   ret void
 }
+
+define amdgpu_ps void @ds_read_b64_tr_b16_v4f16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX950-SDAG-LABEL: ds_read_b64_tr_b16_v4f16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT:    ds_read_b64_tr_b16 v[0:1], v0 offset:32
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: ds_read_b64_tr_b16_v4f16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT:    ds_read_b64_tr_b16 v[0:1], v0 offset:32
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX950-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16.p3(ptr addrspace(3) %gep)
+  store <4 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_read_b64_tr_b16_v4bf16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX950-SDAG-LABEL: ds_read_b64_tr_b16_v4bf16:
+; GFX950-SDAG:       ; %bb.0: ; %entry
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT:    ds_read_b64_tr_b16 v[0:1], v0 offset:32
+; GFX950-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-SDAG-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX950-SDAG-NEXT:    s_endpgm
+;
+; GFX950-GISEL-LABEL: ds_read_b64_tr_b16_v4bf16:
+; GFX950-GISEL:       ; %bb.0: ; %entry
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT:    ds_read_b64_tr_b16 v[0:1], v0 offset:32
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX950-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <4 x bfloat> @llvm.amdgcn.ds.read.tr16.b64.v4bf16.p3(ptr addrspace(3) %gep)
+  store <4 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
index c3562708b15d8..9e6a85dd2810d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
@@ -1146,6 +1146,7 @@
   ; GCN-NEXT:    s_waitcnt vmcnt(8)
   ; GCN-NEXT:    ;;#ASMEND
   ; GCN-NEXT:    s_endpgm
+
   attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
   !0 = !{i64 2862105}
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.simple.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.simple.ll
new file mode 100644
index 0000000000000..80f295b939709
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.simple.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942  -o - %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_kernel void @MFMAExpInterleave(ptr addrspace(1) %out0, ptr addrspace(1) %out1, float %in0, <4 x float> %in1) {
+; GCN-LABEL: MFMAExpInterleave:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x20
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x3fb8aa3b
+; GCN-NEXT:    v_mov_b32_e32 v0, 1.0
+; GCN-NEXT:    s_mov_b32 s7, 0x42b17218
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v2, s6, v1
+; GCN-NEXT:    v_rndne_f32_e32 v3, v2
+; GCN-NEXT:    v_sub_f32_e32 v4, v2, v3
+; GCN-NEXT:    v_fma_f32 v1, s6, v1, -v2
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x32a5705f
+; GCN-NEXT:    v_accvgpr_write_b32 a0, s0
+; GCN-NEXT:    v_fmac_f32_e32 v1, s6, v2
+; GCN-NEXT:    v_accvgpr_write_b32 a1, s1
+; GCN-NEXT:    v_accvgpr_write_b32 a2, s2
+; GCN-NEXT:    v_accvgpr_write_b32 a3, s3
+; GCN-NEXT:    v_add_f32_e32 v1, v4, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v3
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_exp_f32_e32 v1, v1
+; GCN-NEXT:    s_mov_b32 s0, 0x3fb8aa3b
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    ; iglp_opt mask(0x00000003)
+; GCN-NEXT:    v_ldexp_f32 v1, v1, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, 0xc2ce8ed0
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s6, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x42b17218
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x7f800000
+; GCN-NEXT:    s_mov_b32 s6, 0xc2ce8ed0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_ldexp_f32 v3, v3, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_ldexp_f32 v3, v3, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_ldexp_f32 v3, v3, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_ldexp_f32 v3, v3, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_ldexp_f32 v3, v3, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x3fb8aa3b, v1
+; GCN-NEXT:    v_fma_f32 v4, v1, s0, -v3
+; GCN-NEXT:    v_rndne_f32_e32 v5, v3
+; GCN-NEXT:    v_fmac_f32_e32 v4, 0x32a5705f, v1
+; GCN-NEXT:    v_sub_f32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_f32_e32 v3, v3, v4
+; GCN-NEXT:    v_exp_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; GCN-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v0, a[0:3]
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v1
+; GCN-NEXT:    v_ldexp_f32 v0, v3, v4
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v1
+; GCN-NEXT:    s_nop 1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GCN-NEXT:    v_fma_f32 v3, v0, s0, -v1
+; GCN-NEXT:    v_rndne_f32_e32 v4, v1
+; GCN-NEXT:    v_fmac_f32_e32 v3, 0x32a5705f, v0
+; GCN-NEXT:    v_sub_f32_e32 v1, v1, v4
+; GCN-NEXT:    v_add_f32_e32 v1, v1, v3
+; GCN-NEXT:    v_exp_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v4
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GCN-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_ldexp_f32 v1, v1, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_dwordx4 v4, a[0:3], s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT:    global_store_dword v4, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
+  %mai0 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %in1, i32 0, i32 0, i32 0)
+  %mai1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai0, i32 0, i32 0, i32 0)
+  %mai2 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai1, i32 0, i32 0, i32 0)
+  %mai3 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai2, i32 0, i32 0, i32 0)
+  %mai4 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai3, i32 0, i32 0, i32 0)
+  %mai5 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai4, i32 0, i32 0, i32 0)
+  %mai6 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai5, i32 0, i32 0, i32 0)
+  %mai7 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 1.0, <4 x float> %mai6, i32 0, i32 0, i32 0)
+  %exp0 = call float @llvm.exp.f32(float %in0)
+  %exp1 = call float @llvm.exp.f32(float %exp0)
+  %exp2 = call float @llvm.exp.f32(float %exp1)
+  %exp3 = call float @llvm.exp.f32(float %exp2)
+  %exp4 = call float @llvm.exp.f32(float %exp3)
+  %exp5 = call float @llvm.exp.f32(float %exp4)
+  %exp6 = call float @llvm.exp.f32(float %exp5)
+  %exp7 = call float @llvm.exp.f32(float %exp6)
+  store <4 x float> %mai7, ptr addrspace(1) %out0
+  store float %exp7, ptr addrspace(1) %out1
+  tail call void @llvm.amdgcn.iglp.opt(i32 3)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
index 6150cb5cd947e..bc4d35f5a1f9a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
@@ -492,7 +492,6 @@
   attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
 
   !0 = !{i64 2862105}
-
 ...
 
 ---
@@ -899,4 +898,3 @@ body:             |
   S_ENDPGM 0
 ...
 
-
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
index e3dd036ecc308..bf545c82f2d56 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
@@ -1,25 +1,26 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define amdgpu_ps void @v_interp_f32(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v0, attr0.y wait_vdst:15
-; GCN-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v2, s0
-; GCN-NEXT:    v_mov_b32_e32 v4, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p2_f32 v5, v0, v4, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_f32 v4, v1, v4, v5 wait_exp:7
-; GCN-NEXT:    exp mrt0 v3, v2, v5, v4 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    s_mov_b32 m0, s2
+; GFX11-NEXT:    lds_param_load v0, attr0.y wait_vdst:15
+; GFX11-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    v_mov_b32_e32 v4, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:0
+; GFX11-NEXT:    v_interp_p2_f32 v5, v0, v4, v3 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_interp_p2_f32 v4, v1, v4, v5 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v3, v2, v5, v4 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
@@ -32,30 +33,30 @@ main_body:
 }
 
 define amdgpu_ps void @v_interp_f32_many(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32_many:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v0, attr0.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v2, attr2.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v3, attr3.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
-; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
-; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:0
-; GCN-NEXT:    v_interp_p2_f32 v6, v0, v5, v6 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v7, v1, v5, v7 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v8, v2, v5, v8 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v4, v3, v5, v4 wait_exp:7
-; GCN-NEXT:    exp mrt0 v6, v7, v8, v4 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32_many:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    s_mov_b32 m0, s2
+; GFX11-NEXT:    lds_param_load v0, attr0.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v1, attr1.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v2, attr2.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v3, attr3.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
+; GFX11-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
+; GFX11-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:0
+; GFX11-NEXT:    v_interp_p2_f32 v6, v0, v5, v6 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v7, v1, v5, v7 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v8, v2, v5, v8 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v4, v3, v5, v4 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v6, v7, v8, v4 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
@@ -74,30 +75,30 @@ main_body:
 }
 
 define amdgpu_ps void @v_interp_f32_many_vm(ptr addrspace(1) %ptr, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f32_many_vm:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
-; GCN-NEXT:    s_mov_b32 m0, s0
-; GCN-NEXT:    s_mov_b32 s0, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    lds_param_load v2, attr0.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v3, attr1.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v4, attr2.x wait_vdst:15
-; GCN-NEXT:    lds_param_load v5, attr3.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
-; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
-; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
-; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:0
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v6, v2, v1, v6 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v7, v3, v1, v7 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GCN-NEXT:    v_interp_p2_f32 v8, v4, v1, v8 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
-; GCN-NEXT:    exp mrt0 v6, v7, v8, v0 done
-; GCN-NEXT:    s_endpgm
+; GFX11-LABEL: v_interp_f32_many_vm:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
+; GFX11-NEXT:    s_mov_b32 m0, s0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT:    lds_param_load v2, attr0.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v3, attr1.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v4, attr2.x wait_vdst:15
+; GFX11-NEXT:    lds_param_load v5, attr3.x wait_vdst:15
+; GFX11-NEXT:    s_mov_b32 exec_lo, s0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
+; GFX11-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
+; GFX11-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
+; GFX11-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v6, v2, v1, v6 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v7, v3, v1, v7 wait_exp:7
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_interp_p2_f32 v8, v4, v1, v8 wait_exp:7
+; GFX11-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
+; GFX11-NEXT:    exp mrt0 v6, v7, v8, v0 done
+; GFX11-NEXT:    s_endpgm
 main_body:
   %i.ptr = getelementptr float, ptr addrspace(1) %ptr, i32 1
   %i = load float, ptr addrspace(1) %i.ptr, align 4
@@ -120,23 +121,41 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_f16:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f16_f32 v3, v1, v0, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p10_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
-; GCN-NEXT:    v_interp_p2_f16_f32 v3, v1, v2, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
-; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-TRUE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-TRUE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v3, v1.l, v0, v1.l wait_exp:0
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v4, v1.h, v0, v1.h wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.l, v1.l, v2, v3 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.h, v1.h, v2, v4 wait_exp:7
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-FAKE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-FAKE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-FAKE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v3, v1, v0, v1 wait_exp:0
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v3, v1, v2, v3 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float %p0, float %i, float %p0, i1 0)
@@ -148,23 +167,41 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_rtz_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
-; GCN-LABEL: v_interp_rtz_f16:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b32 s3, exec_lo
-; GCN-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GCN-NEXT:    s_mov_b32 m0, s2
-; GCN-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
-; GCN-NEXT:    s_mov_b32 exec_lo, s3
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0
-; GCN-NEXT:    v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
-; GCN-NEXT:    v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
-; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_rtz_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-TRUE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-TRUE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1.l, v0, v1.l wait_exp:0
+; GFX11-TRUE16-NEXT:    v_interp_p10_rtz_f16_f32 v4, v1.h, v0, v1.h wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_rtz_f16_f32 v0.l, v1.l, v2, v3 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_interp_p2_rtz_f16_f32 v0.h, v1.h, v2, v4 wait_exp:7
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_rtz_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-FAKE16-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX11-FAKE16-NEXT:    s_mov_b32 m0, s2
+; GFX11-FAKE16-NEXT:    lds_param_load v1, attr0.x wait_vdst:15
+; GFX11-FAKE16-NEXT:    s_mov_b32 exec_lo, s3
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_rtz_f16_f32 v3, v1, v0, v1 wait_exp:0
+; GFX11-FAKE16-NEXT:    v_interp_p10_rtz_f16_f32 v0, v1, v0, v1 op_sel:[1,0,1,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_rtz_f16_f32 v3, v1, v2, v3 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_interp_p2_rtz_f16_f32 v0, v1, v2, v0 op_sel:[1,0,0,0] wait_exp:7
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.p10.rtz.f16(float %p0, float %i, float %p0, i1 0)
@@ -176,17 +213,30 @@ main_body:
 }
 
 define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 {
-; GCN-LABEL: v_interp_f16_imm_params:
-; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GCN-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v0 wait_exp:7
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GCN-NEXT:    v_add_f16_e32 v0, v1, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX11-TRUE16-LABEL: v_interp_f16_imm_params:
+; GFX11-TRUE16:       ; %bb.0: ; %main_body
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, 0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_interp_p10_f16_f32 v1, v0.l, v1, v0.l wait_exp:7
+; GFX11-TRUE16-NEXT:    v_interp_p2_f16_f32 v0.l, v0.l, v3, v2 wait_exp:7
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v1
+; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: v_interp_f16_imm_params:
+; GFX11-FAKE16:       ; %bb.0: ; %main_body
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 wait_exp:7
+; GFX11-FAKE16-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v0 wait_exp:7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_add_f16_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT:    ; return to shader part epilog
 main_body:
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float 0.0, float %i, float 0.0, i1 0)
   %l_p1 = call half @llvm.amdgcn.interp.inreg.p2.f16(float 0.0, float %j, float 0.0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 30cc060d05bb1..ac515808a0d8a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -15,17 +15,17 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
 ; SI-SDAG-NEXT:    s_load_dword s2, s[4:5], 0xb
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; SI-SDAG-NEXT:    v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
 ; SI-SDAG-NEXT:    s_mov_b32 s2, -1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v0, s4
 ; SI-SDAG-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
@@ -53,15 +53,15 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
 ; VI-SDAG-NEXT:    s_load_dword s2, s[4:5], 0x2c
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v1, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, v1, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, v0, s2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-SDAG-NEXT:    flat_store_dword v[0:1], v2
@@ -91,17 +91,17 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
 ; GFX900-SDAG-NEXT:    s_load_dword s2, s[4:5], 0x2c
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v1, v1
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v1, v0
-; GFX900-SDAG-NEXT:    global_store_dword v2, v0, s[0:1]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, s2
+; GFX900-SDAG-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
 ; GFX900-GISEL-LABEL: s_exp2_f32:
@@ -175,25 +175,26 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, s3, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; SI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_exp_f32_e32 v3, v1
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
 ; SI-SDAG-NEXT:    s_mov_b32 s4, s0
 ; SI-SDAG-NEXT:    s_mov_b32 s5, s1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, v4, v2
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v3, v0
+; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v1, v2, s0
+; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v0, s0
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
@@ -225,22 +226,23 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, s3, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, v4, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v2, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    s_cselect_b32 s3, 0xffffffc0, 0
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v2, s3
+; VI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, s2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
@@ -273,23 +275,24 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; GFX900-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, s3, v4
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, s2, v1
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v2, v1
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, v4, v3
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v2, v0
-; GFX900-SDAG-NEXT:    global_store_dwordx2 v5, v[0:1], s[0:1]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, s3, v3
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v3, v3
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, v3, s4
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, s2
+; GFX900-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
 ; GFX900-GISEL-LABEL: s_exp2_v2f32:
@@ -384,29 +387,31 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
+; SI-SDAG-NEXT:    s_and_b64 s[8:9], vcc, exec
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v1, s1, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
+; SI-SDAG-NEXT:    v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
+; SI-SDAG-NEXT:    s_and_b64 s[8:9], vcc, exec
+; SI-SDAG-NEXT:    v_add_f32_e32 v3, s0, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, s1, v4
+; SI-SDAG-NEXT:    v_exp_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
-; SI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v6, s0, v6
-; SI-SDAG-NEXT:    v_exp_f32_e32 v3, v0
-; SI-SDAG-NEXT:    v_exp_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v1, vcc
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v1, v1, s1
+; SI-SDAG-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v3, s1
+; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, v4, v2
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, v3, v7
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v6, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v2, v2, s0
 ; SI-SDAG-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-SDAG-NEXT:    s_endpgm
@@ -446,31 +451,34 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; VI-SDAG-LABEL: s_exp2_v3f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, s2, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, v2, s4
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, s2, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, s0, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, s1, v6
-; VI-SDAG-NEXT:    v_exp_f32_e32 v3, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, v4, v2
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v3, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, v6, v5
-; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s5
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, s1, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_exp_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
+; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v3, s4
+; VI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s3
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s2
 ; VI-SDAG-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
@@ -510,29 +518,31 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, s2, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, v2, s2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, s2, v4
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v6, s1, v6
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, s1, v4
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, s0, v0
 ; GFX900-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v6, v6
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v3, v1
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v7, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, v4, v2
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, v6, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v3, v0
-; GFX900-SDAG-NEXT:    global_store_dwordx3 v7, v[0:2], s[6:7]
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, v4, s2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, s0
+; GFX900-SDAG-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
 ; GFX900-GISEL-LABEL: s_exp2_v3f32:
@@ -659,35 +669,38 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0xd
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; SI-SDAG-NEXT:    s_and_b64 s[8:9], vcc, exec
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, s7, v2
+; SI-SDAG-NEXT:    v_add_f32_e32 v3, s6, v3
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v4, v3
+; SI-SDAG-NEXT:    s_cselect_b32 s8, 0xffffffc0, 0
+; SI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s6, 0xffffffc0, 0
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v8, 0, v3, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v3, v2, s8
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v2, v4, s6
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; SI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, s7, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v6, s6, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v8, s5, v8
-; SI-SDAG-NEXT:    v_add_f32_e32 v1, s4, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v4, s5, v4
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, s4, v0
 ; SI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_exp_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_exp_f32_e32 v8, v8
-; SI-SDAG-NEXT:    v_exp_f32_e32 v9, v1
-; SI-SDAG-NEXT:    s_mov_b32 s2, -1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, v4, v2
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, v6, v5
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, v8, v7
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v9, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    s_cselect_b32 s6, 0xffffffc0, 0
+; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v1, v4, s6
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v0, s4
 ; SI-SDAG-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
@@ -733,34 +746,37 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, s2, v3
+; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v4, v3
+; VI-SDAG-NEXT:    s_cselect_b32 s6, 0xffffffc0, 0
+; VI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v8, 0, v3, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, v2, s6
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, v4, s2
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; VI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, s3, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, s2, v6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v4, s1, v4
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; VI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_exp_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_add_f32_e32 v8, s1, v8
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, s0, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v8, v8
-; VI-SDAG-NEXT:    v_exp_f32_e32 v9, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, v4, v2
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, v6, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v4, s2
+; VI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, v8, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v9, v0
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, s0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v5, s5
 ; VI-SDAG-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-SDAG-NEXT:    s_endpgm
@@ -807,34 +823,37 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX900-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, s2, v3
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v5, v3
+; GFX900-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v8, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v9, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v3, v2, s4
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, v5, s2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX900-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, s3, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v7, s2, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v9, s1, v9
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, s1, v5
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, s0, v0
 ; GFX900-SDAG-NEXT:    v_exp_f32_e32 v5, v5
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v7, v7
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v9, v9
-; GFX900-SDAG-NEXT:    v_exp_f32_e32 v10, v1
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, v5, v2
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, v7, v6
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, v9, v8
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v10, v0
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX900-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, v5, s2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, s0
 ; GFX900-SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
@@ -973,19 +992,19 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 }
 
 define float @v_exp2_f32(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32:
 ; GCN-GISEL:       ; %bb.0:
@@ -1001,6 +1020,34 @@ define float @v_exp2_f32(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1015,19 +1062,19 @@ define float @v_exp2_f32(float %in) {
 }
 
 define float @v_exp2_fabs_f32(float %in) {
-; GCN-SDAG-LABEL: v_exp2_fabs_f32:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fabs_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_fabs_f32:
 ; GCN-GISEL:       ; %bb.0:
@@ -1043,6 +1090,34 @@ define float @v_exp2_fabs_f32(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_fabs_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_fabs_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_fabs_f32:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1058,19 +1133,19 @@ define float @v_exp2_fabs_f32(float %in) {
 }
 
 define float @v_exp2_fneg_fabs_f32(float %in) {
-; GCN-SDAG-LABEL: v_exp2_fneg_fabs_f32:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_sub_f32_e64 v0, v2, |v0|
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_fabs_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e64 v0, v2, |v0|
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_fneg_fabs_f32:
 ; GCN-GISEL:       ; %bb.0:
@@ -1086,6 +1161,34 @@ define float @v_exp2_fneg_fabs_f32(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_fneg_fabs_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e64 v0, v2, |v0|
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_fneg_fabs_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e64 v0, v2, |v0|
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_fneg_fabs_f32:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1102,19 +1205,19 @@ define float @v_exp2_fneg_fabs_f32(float %in) {
 }
 
 define float @v_exp2_fneg_f32(float %in) {
-; GCN-SDAG-LABEL: v_exp2_fneg_f32:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
-; GCN-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_fneg_f32:
 ; GCN-GISEL:       ; %bb.0:
@@ -1130,6 +1233,34 @@ define float @v_exp2_fneg_f32(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_fneg_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_fneg_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x42fc0000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_fneg_f32:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1145,19 +1276,19 @@ define float @v_exp2_fneg_f32(float %in) {
 }
 
 define float @v_exp2_f32_fast(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_fast:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_fast:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_fast:
 ; GCN-GISEL:       ; %bb.0:
@@ -1173,6 +1304,34 @@ define float @v_exp2_f32_fast(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_fast:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_fast:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_fast:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1187,19 +1346,19 @@ define float @v_exp2_f32_fast(float %in) {
 }
 
 define float @v_exp2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
-; GCN-SDAG-LABEL: v_exp2_f32_unsafe_math_attr:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_unsafe_math_attr:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_unsafe_math_attr:
 ; GCN-GISEL:       ; %bb.0:
@@ -1215,6 +1374,34 @@ define float @v_exp2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_unsafe_math_attr:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_unsafe_math_attr:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_unsafe_math_attr:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1229,19 +1416,19 @@ define float @v_exp2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 }
 
 define float @v_exp2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
-; GCN-SDAG-LABEL: v_exp2_f32_approx_fn_attr:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_approx_fn_attr:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_approx_fn_attr:
 ; GCN-GISEL:       ; %bb.0:
@@ -1257,6 +1444,34 @@ define float @v_exp2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_approx_fn_attr:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_approx_fn_attr:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_approx_fn_attr:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1271,19 +1486,19 @@ define float @v_exp2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 }
 
 define float @v_exp2_f32_ninf(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_ninf:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_ninf:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_ninf:
 ; GCN-GISEL:       ; %bb.0:
@@ -1299,6 +1514,34 @@ define float @v_exp2_f32_ninf(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_ninf:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_ninf:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_ninf:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1313,19 +1556,19 @@ define float @v_exp2_f32_ninf(float %in) {
 }
 
 define float @v_exp2_f32_afn(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_afn:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_afn:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_afn:
 ; GCN-GISEL:       ; %bb.0:
@@ -1341,6 +1584,34 @@ define float @v_exp2_f32_afn(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_afn:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_afn:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_afn:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1375,19 +1646,19 @@ define float @v_exp2_f32_afn_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_afn_dynamic(float %in) #1 {
-; GCN-SDAG-LABEL: v_exp2_f32_afn_dynamic:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_afn_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_afn_dynamic:
 ; GCN-GISEL:       ; %bb.0:
@@ -1403,6 +1674,34 @@ define float @v_exp2_f32_afn_dynamic(float %in) #1 {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_afn_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_afn_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_afn_dynamic:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1417,19 +1716,19 @@ define float @v_exp2_f32_afn_dynamic(float %in) #1 {
 }
 
 define float @v_fabs_exp2_f32_afn(float %in) {
-; GCN-SDAG-LABEL: v_fabs_exp2_f32_afn:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_fabs_exp2_f32_afn:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_fabs_exp2_f32_afn:
 ; GCN-GISEL:       ; %bb.0:
@@ -1445,6 +1744,34 @@ define float @v_fabs_exp2_f32_afn(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_fabs_exp2_f32_afn:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_fabs_exp2_f32_afn:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e64 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_fabs_exp2_f32_afn:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1480,19 +1807,19 @@ define float @v_exp2_f32_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_nnan(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_nnan:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_nnan:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_nnan:
 ; GCN-GISEL:       ; %bb.0:
@@ -1508,6 +1835,34 @@ define float @v_exp2_f32_nnan(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_nnan:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_nnan:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_nnan:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1542,19 +1897,19 @@ define float @v_exp2_f32_nnan_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_nnan_dynamic(float %in) #1 {
-; GCN-SDAG-LABEL: v_exp2_f32_nnan_dynamic:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_nnan_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_nnan_dynamic:
 ; GCN-GISEL:       ; %bb.0:
@@ -1570,6 +1925,34 @@ define float @v_exp2_f32_nnan_dynamic(float %in) #1 {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_nnan_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_nnan_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_nnan_dynamic:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1604,19 +1987,19 @@ define float @v_exp2_f32_ninf_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_ninf_dynamic(float %in) #1 {
-; GCN-SDAG-LABEL: v_exp2_f32_ninf_dynamic:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_ninf_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_ninf_dynamic:
 ; GCN-GISEL:       ; %bb.0:
@@ -1632,6 +2015,34 @@ define float @v_exp2_f32_ninf_dynamic(float %in) #1 {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_ninf_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_ninf_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_ninf_dynamic:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1646,19 +2057,19 @@ define float @v_exp2_f32_ninf_dynamic(float %in) #1 {
 }
 
 define float @v_exp2_f32_nnan_ninf(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_nnan_ninf:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_nnan_ninf:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_nnan_ninf:
 ; GCN-GISEL:       ; %bb.0:
@@ -1674,6 +2085,34 @@ define float @v_exp2_f32_nnan_ninf(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_nnan_ninf:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_nnan_ninf:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_nnan_ninf:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1708,19 +2147,19 @@ define float @v_exp2_f32_nnan_ninf_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_nnan_ninf_dynamic(float %in) #1 {
-; GCN-SDAG-LABEL: v_exp2_f32_nnan_ninf_dynamic:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_nnan_ninf_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_nnan_ninf_dynamic:
 ; GCN-GISEL:       ; %bb.0:
@@ -1736,6 +2175,34 @@ define float @v_exp2_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_nnan_ninf_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_nnan_ninf_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_nnan_ninf_dynamic:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1770,19 +2237,19 @@ define float @v_exp2_f32_fast_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_dynamic_mode(float %in) #1 {
-; GCN-SDAG-LABEL: v_exp2_f32_dynamic_mode:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_dynamic_mode:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_dynamic_mode:
 ; GCN-GISEL:       ; %bb.0:
@@ -1798,6 +2265,34 @@ define float @v_exp2_f32_dynamic_mode(float %in) #1 {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_dynamic_mode:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_dynamic_mode:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_dynamic_mode:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -1912,9 +2407,9 @@ define float @v_exp2_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SI-GISEL-LABEL: v_exp2_f32_from_fpext_math_f16:
@@ -1978,9 +2473,9 @@ define float @v_exp2_f32_from_fpext_bf16(bfloat %src) {
 ; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; SI-NEXT:    v_add_f32_e32 v0, v0, v2
 ; SI-NEXT:    v_exp_f32_e32 v0, v0
-; SI-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; SI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NEXT:    v_not_b32_e32 v1, 63
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_exp2_f32_from_fpext_bf16:
@@ -1993,9 +2488,9 @@ define float @v_exp2_f32_from_fpext_bf16(bfloat %src) {
 ; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
 ; VI-NEXT:    v_add_f32_e32 v0, v0, v1
 ; VI-NEXT:    v_exp_f32_e32 v0, v0
-; VI-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; VI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-NEXT:    v_not_b32_e32 v1, 63
+; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_exp2_f32_from_fpext_bf16:
@@ -2008,9 +2503,9 @@ define float @v_exp2_f32_from_fpext_bf16(bfloat %src) {
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; GFX900-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX900-NEXT:    v_exp_f32_e32 v0, v0
-; GFX900-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; R600-LABEL: v_exp2_f32_from_fpext_bf16:
@@ -2850,19 +3345,19 @@ define <3 x half> @v_exp2_v3f16_afn(<3 x half> %in) {
 }
 
 define float @v_exp2_f32_contract(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_contract:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_contract:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_contract:
 ; GCN-GISEL:       ; %bb.0:
@@ -2878,6 +3373,34 @@ define float @v_exp2_f32_contract(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_contract:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_contract:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_contract:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
@@ -2912,19 +3435,19 @@ define float @v_exp2_f32_contract_daz(float %in) #0 {
 }
 
 define float @v_exp2_f32_contract_nnan_ninf(float %in) {
-; GCN-SDAG-LABEL: v_exp2_f32_contract_nnan_ninf:
-; GCN-SDAG:       ; %bb.0:
-; GCN-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
-; GCN-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
-; GCN-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v1, 0x1f800000
-; GCN-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_contract_nnan_ninf:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: v_exp2_f32_contract_nnan_ninf:
 ; GCN-GISEL:       ; %bb.0:
@@ -2940,6 +3463,34 @@ define float @v_exp2_f32_contract_nnan_ninf(float %in) {
 ; GCN-GISEL-NEXT:    v_mul_f32_e32 v0, v0, v1
 ; GCN-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_exp2_f32_contract_nnan_ninf:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_exp2_f32_contract_nnan_ninf:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0xc2fc0000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x42800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_not_b32_e32 v1, 63
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; R600-LABEL: v_exp2_f32_contract_nnan_ninf:
 ; R600:       ; %bb.0:
 ; R600-NEXT:    CF_END
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index 279ffeab51fb3..218e41faa703d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -17,19 +17,19 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
 ; SI-SDAG-NEXT:    s_load_dword s0, s[4:5], 0xb
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; SI-SDAG-NEXT:    s_mov_b32 s1, 0x3377d1cf
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s0, v0
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s0, 0x3f317217
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s0, -v1
-; SI-SDAG-NEXT:    s_mov_b32 s0, 0x3377d1cf
-; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s0, v2
+; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s1, v2
 ; SI-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
 ; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s0
@@ -73,11 +73,11 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
 ; VI-SDAG-NEXT:    s_load_dword s0, s[4:5], 0x2c
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -134,27 +134,27 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_load_dword s0, s[4:5], 0x2c
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s1, 0x3377d1cf
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s0, 0x3f317217
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v3, v0, s0, -v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v3, v0, s1, v3
-; GFX900-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s0, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s1, v3
 ; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x41b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
-; GFX900-SDAG-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
 ; GFX900-GISEL-LABEL: s_log_f32:
@@ -190,21 +190,23 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s2, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s2
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
 ; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_add_f32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, s2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
@@ -314,38 +316,39 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3377d1cf
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x7f800000
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s3, v2
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s3, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s3, 0x3f317217
 ; SI-SDAG-NEXT:    s_mov_b32 s4, s0
 ; SI-SDAG-NEXT:    s_mov_b32 s5, s1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s3, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s8, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s9
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s3, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s8, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s9
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s2, v0
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41b17218
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v2, v1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v0
-; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s3, -v2
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x41b17218
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v0
+; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s3, -v3
 ; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s8, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s9
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -394,43 +397,44 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s2, 0x7f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s7, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s2
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s7, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s6, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s6, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s6, v0
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41b17218
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v2, v1
-; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v0, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x41b17218
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v0, v3
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v4
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v3
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317000, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s5
@@ -488,36 +492,37 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s2, 0x3f317217
 ; GFX900-SDAG-NEXT:    s_mov_b32 s3, 0x3377d1cf
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, s11, v3
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v3
-; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v3
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v3, s2, -v4
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v3, s3, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v3|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s2, -v3
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s3, v4
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s10, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s10, v0
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0x41b17218
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v1
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s2, -v3
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41b17218
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s2, -v4
 ; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s3, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
 ; GFX900-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX900-SDAG-NEXT:    s_endpgm
@@ -564,31 +569,37 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s4, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s5, 0x800000, s2
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s4, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s4
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s5
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s3, v0 :: v_dual_mul_f32 v1, s2, v1
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s5
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s4
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v2, 0x3f317217, v0 :: v_dual_mul_f32 v3, 0x3f317217, v1
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v3, 0x3f317217, v1 :: v_dual_lshlrev_b32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v5, 0x3f317217, v1, -v3
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v5, 0x3377d1cf, v1
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_dual_add_f32 v3, v3, v5 :: v_dual_mul_f32 v2, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x41b17218, s5
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v4, 0x3f317217, v0, -v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_fma_f32 v5, 0x3f317217, v1, -v3
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v4, 0x3377d1cf, v0 :: v_dual_fmac_f32 v5, 0x3377d1cf, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v2, v2, v4 :: v_dual_add_f32 v3, v3, v5
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v4, 0x3377d1cf, v0
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x41b17218, s4
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x41b17218, s5
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
 ; GFX1100-SDAG-NEXT:    v_dual_cndmask_b32 v2, v1, v3 :: v_dual_mov_b32 v3, 0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v1, v0, v4 :: v_dual_sub_f32 v0, v2, v5
 ; GFX1100-SDAG-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
 ; GFX1100-SDAG-NEXT:    s_endpgm
@@ -742,49 +753,51 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s11, 0x3377d1cf
+; SI-SDAG-NEXT:    s_mov_b32 s12, 0x7f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, s9, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s9, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x3f317217
-; SI-SDAG-NEXT:    s_mov_b32 s12, 0x7f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v1
-; SI-SDAG-NEXT:    v_fma_f32 v4, v1, s9, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v1, s11, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
+; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s9, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s11, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v2, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, s8, v3
-; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
-; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x41b17218
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, s8, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41b17218
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
-; SI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317217, v3
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s10, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v3, s9, -v5
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v3, s11, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v3|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
-; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v2
-; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s9, -v3
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
+; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s9, -v4
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s11, v5
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s10, v0
+; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v0
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s12
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v5
+; SI-SDAG-NEXT:    v_fma_f32 v4, v5, s9, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v4, v5, s11, v4
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v5|, s12
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -847,47 +860,49 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s6, 0x7f800000
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s10, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s10, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s9, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, s9, v3
-; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, s9, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x41b17218
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v5
-; VI-SDAG-NEXT:    v_and_b32_e32 v5, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s8, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v3, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3f317000, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v5
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v3, v1
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3805fdf4, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v5
-; VI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317000, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v3|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v5, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
 ; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
 ; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v0
@@ -978,39 +993,41 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
+; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, s10, v2
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s10, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s10, 0x7f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x41b17218
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s4, -v4
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s5, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s10
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s4, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s5, v4
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s10
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s9, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v4, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, s9, v4
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x41b17218
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, s9, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s8, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v6
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317217, v4
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v4, s4, -v6
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v4
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v4, s4, -v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v4, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v4, s5, v6
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s10
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v4, v6, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
@@ -1079,48 +1096,54 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s3, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s1
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s3
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s3
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s6
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v9, 0, 0x41b17218, s3
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, 0x41b17218, s6
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s2, v0 :: v_dual_mul_f32 v1, s1, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3f317217, v2 :: v_dual_lshlrev_b32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s2, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v8, 0x3f317217, v2, -v5
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v8, 0x3377d1cf, v2
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v5, v5, v8
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v0
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v6, 0x3f317217, v0, -v3
+; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v6, 0x3377d1cf, v0 :: v_dual_lshlrev_b32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s1, v1
+; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v3, v3, v6
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 0x41b17218, s7
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317217, v2
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v1
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v7, 0x3f317217, v1, -v4
-; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v6, 0x3377d1cf, v0
-; GFX1100-SDAG-NEXT:    v_fma_f32 v8, 0x3f317217, v2, -v5
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v7, 0x3377d1cf, v1
-; GFX1100-SDAG-NEXT:    v_add_f32_e32 v3, v3, v6
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 0x41b17218, s7
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v8, 0x3377d1cf, v2 :: v_dual_cndmask_b32 v1, v1, v4
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v5, v5, v8 :: v_dual_mov_b32 v4, 0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v10
+; GFX1100-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_sub_f32 v1, v1, v10
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v5, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v2, v0, v9
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -1343,60 +1366,63 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s12, 0x3377d1cf
+; SI-SDAG-NEXT:    s_mov_b32 s13, 0x7f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s11, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s11, 0x3f317217
-; SI-SDAG-NEXT:    s_mov_b32 s13, 0x7f800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x41b17218
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s11, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s12, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x41b17218
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s11, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s12, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; SI-SDAG-NEXT:    v_log_f32_e32 v4, v3
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, s10, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v4
-; SI-SDAG-NEXT:    v_fma_f32 v6, v4, s11, -v2
-; SI-SDAG-NEXT:    v_fma_f32 v6, v4, s12, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v6
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v6, s9, v6
-; SI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v2
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s11, -v1
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s12, v5
+; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v5, s9, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v6
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; SI-SDAG-NEXT:    v_fma_f32 v7, v6, s11, -v4
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s8, v0
+; SI-SDAG-NEXT:    v_fma_f32 v6, v5, s11, -v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; SI-SDAG-NEXT:    v_fma_f32 v7, v6, s12, v7
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v4, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v5, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s11, -v4
+; SI-SDAG-NEXT:    v_fma_f32 v6, v5, s12, v6
+; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
+; SI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317217, v0
+; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s11, -v5
 ; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s12, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; SI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v4
 ; SI-SDAG-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -1470,78 +1496,81 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s6, 0x7f800000
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; VI-SDAG-NEXT:    v_log_f32_e32 v4, v3
-; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x41b17218
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
-; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v4, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, s10, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x41b17218
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v2, v1
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3805fdf4, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317000, v1
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v5, s9, v5
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[0:1]
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v5, v1
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3f317000, v6
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, s9, v6
-; VI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v6
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v7, v6, v4
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3f317000, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3805fdf4, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v9, v7
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v8, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317000, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v4, s[2:3]
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v5, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v0, v4
+; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317000, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
+; VI-SDAG-NEXT:    v_and_b32_e32 v5, 0xfffff000, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v0, v5
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3f317000, v6
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v5
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317000, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317000, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v5, s5
@@ -1635,60 +1664,63 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
+; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s11, 0x7f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v6, 0x41b17218
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3f317217, v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s4, -v3
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s5, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x41b17218
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s4, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s5, v3
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v5, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, s10, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v5
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v5, s4, -v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v5, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v7
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v7, s9, v7
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v7, v7
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v6, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v2, s4, -v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v2, s5, v6
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 5, v6
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v6, s9, v6
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v6, v6
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317217, v7
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v8, v7, s4, -v5
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v6
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v6, s4, -v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v8, v7, s5, v8
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, v5, v8
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v7|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v7, v5, s[2:3]
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v6, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3f317217, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s4, -v5
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v6, s5, v7
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v7
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v1, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v6
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317217, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s4, -v6
 ; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, v5, v7
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v6, s[0:1]
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v5
 ; GFX900-SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
@@ -1760,56 +1792,65 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s3
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s8, 0x800000, s1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s9, 0x800000, s0
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s3
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s8
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s9
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s8
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0x4f800000, s9
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s6
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x41b17218, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s3, v0 :: v_dual_mul_f32 v1, s2, v1
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v2, s1, v2 :: v_dual_mul_f32 v3, s0, v3
-; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(TRANS32_DEP_3)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v9, 0, 0x41b17218, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v14, 0, 0x41b17218, s8
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v15, 0, 0x41b17218, s9
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3f317217, v0 :: v_dual_mul_f32 v6, 0x3f317217, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s1, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v3, s0, v3
+; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3f317217, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3f317217, v3
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v12, 0x3f317217, v2, -v7
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v13, 0x3f317217, v3, -v8
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v12, 0x3377d1cf, v2 :: v_dual_fmac_f32 v13, 0x3377d1cf, v3
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v7, v7, v12
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v7, 0x3f317217, v2 :: v_dual_mul_f32 v8, 0x3f317217, v3
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3f317217, v0 :: v_dual_add_f32 v8, v8, v13
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317217, v1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v10, 0x3f317217, v0, -v5
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v11, 0x3f317217, v1, -v6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_fma_f32 v12, 0x3f317217, v2, -v7
-; GFX1100-SDAG-NEXT:    v_fma_f32 v13, 0x3f317217, v3, -v8
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v10, 0x3377d1cf, v0 :: v_dual_fmac_f32 v11, 0x3377d1cf, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v12, 0x3377d1cf, v2 :: v_dual_fmac_f32 v13, 0x3377d1cf, v3
 ; GFX1100-SDAG-NEXT:    v_dual_add_f32 v5, v5, v10 :: v_dual_add_f32 v6, v6, v11
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v7, v7, v12 :: v_dual_add_f32 v8, v8, v13
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v5, v2, v7, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_dual_mov_b32 v7, 0 :: v_dual_sub_f32 v2, v1, v9
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v6, v3, v8, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v3, v0, v4
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v1, v5, v14 :: v_dual_sub_f32 v0, v6, v15
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    global_store_b128 v7, v[0:3], s[0:1]
@@ -2062,10 +2103,10 @@ define float @v_log_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2108,10 +2149,10 @@ define float @v_log_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2160,10 +2201,10 @@ define float @v_log_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2206,21 +2247,22 @@ define float @v_log_f32(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2264,10 +2306,10 @@ define float @v_log_fabs_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2310,10 +2352,10 @@ define float @v_log_fabs_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2362,10 +2404,10 @@ define float @v_log_fabs_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2409,20 +2451,22 @@ define float @v_log_fabs_f32(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, s0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2467,10 +2511,10 @@ define float @v_log_fneg_fabs_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -|v0|, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2513,10 +2557,10 @@ define float @v_log_fneg_fabs_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2565,10 +2609,10 @@ define float @v_log_fneg_fabs_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2612,20 +2656,22 @@ define float @v_log_fneg_fabs_f32(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e64 s0, 0x80800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, s0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2671,10 +2717,10 @@ define float @v_log_fneg_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2717,10 +2763,10 @@ define float @v_log_fneg_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2769,10 +2815,10 @@ define float @v_log_fneg_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -2815,21 +2861,22 @@ define float @v_log_fneg_f32(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 0x80800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2875,9 +2922,9 @@ define float @v_log_f32_fast(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2902,9 +2949,9 @@ define float @v_log_f32_fast(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2929,9 +2976,9 @@ define float @v_log_f32_fast(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2955,10 +3002,12 @@ define float @v_log_f32_fast(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -2993,9 +3042,9 @@ define float @v_log_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3020,9 +3069,9 @@ define float @v_log_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3047,9 +3096,9 @@ define float @v_log_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3073,10 +3122,12 @@ define float @v_log_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -3111,9 +3162,9 @@ define float @v_log_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3138,9 +3189,9 @@ define float @v_log_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3165,9 +3216,9 @@ define float @v_log_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3191,10 +3242,12 @@ define float @v_log_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -3228,10 +3281,10 @@ define float @v_log_f32_ninf(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -3274,10 +3327,10 @@ define float @v_log_f32_ninf(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -3326,10 +3379,10 @@ define float @v_log_f32_ninf(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -3372,21 +3425,22 @@ define float @v_log_f32_ninf(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3431,9 +3485,9 @@ define float @v_log_f32_afn(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3458,9 +3512,9 @@ define float @v_log_f32_afn(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3485,9 +3539,9 @@ define float @v_log_f32_afn(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3511,10 +3565,12 @@ define float @v_log_f32_afn(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -3578,9 +3634,9 @@ define float @v_log_f32_afn_dynamic(float %in) #1 {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3605,9 +3661,9 @@ define float @v_log_f32_afn_dynamic(float %in) #1 {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3632,9 +3688,9 @@ define float @v_log_f32_afn_dynamic(float %in) #1 {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3658,10 +3714,12 @@ define float @v_log_f32_afn_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -3696,9 +3754,9 @@ define float @v_fabs_log_f32_afn(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3723,9 +3781,9 @@ define float @v_fabs_log_f32_afn(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3750,9 +3808,9 @@ define float @v_fabs_log_f32_afn(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc1b17218
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3777,10 +3835,11 @@ define float @v_fabs_log_f32_afn(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc1b17218, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3f317218, v1
@@ -3956,10 +4015,10 @@ define float @v_log_f32_nnan(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4002,10 +4061,10 @@ define float @v_log_f32_nnan(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4054,10 +4113,10 @@ define float @v_log_f32_nnan(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4100,21 +4159,22 @@ define float @v_log_f32_nnan(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4298,10 +4358,10 @@ define float @v_log_f32_nnan_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4344,10 +4404,10 @@ define float @v_log_f32_nnan_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4396,10 +4456,10 @@ define float @v_log_f32_nnan_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4442,21 +4502,22 @@ define float @v_log_f32_nnan_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4640,10 +4701,10 @@ define float @v_log_f32_ninf_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4686,10 +4747,10 @@ define float @v_log_f32_ninf_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4738,10 +4799,10 @@ define float @v_log_f32_ninf_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4784,21 +4845,22 @@ define float @v_log_f32_ninf_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4842,10 +4904,10 @@ define float @v_log_f32_nnan_ninf(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4882,10 +4944,10 @@ define float @v_log_f32_nnan_ninf(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -4928,10 +4990,10 @@ define float @v_log_f32_nnan_ninf(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -4968,18 +5030,20 @@ define float @v_log_f32_nnan_ninf(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3377d1cf, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5123,10 +5187,10 @@ define float @v_log_f32_nnan_ninf_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -5163,10 +5227,10 @@ define float @v_log_f32_nnan_ninf_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -5209,10 +5273,10 @@ define float @v_log_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -5249,18 +5313,20 @@ define float @v_log_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3377d1cf, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5330,10 +5396,10 @@ define float @v_log_f32_dynamic_mode(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -5376,10 +5442,10 @@ define float @v_log_f32_dynamic_mode(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -5428,10 +5494,10 @@ define float @v_log_f32_dynamic_mode(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -5474,21 +5540,22 @@ define float @v_log_f32_dynamic_mode(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6007,17 +6074,17 @@ define float @v_log_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
 ; SI-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s4, -v1
-; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3377d1cf
-; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s4, v2
+; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s5, v2
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
@@ -6179,10 +6246,10 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) {
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, 0x800000
-; SI-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-NEXT:    v_log_f32_e32 v0, v0
 ; SI-NEXT:    s_mov_b32 s4, 0x3f317217
 ; SI-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -6203,10 +6270,10 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; VI-NEXT:    s_mov_b32 s4, 0x800000
-; VI-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-NEXT:    v_log_f32_e32 v0, v0
 ; VI-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -6230,10 +6297,10 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) {
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0x3f317217
 ; GFX900-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
@@ -6255,20 +6322,22 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) {
 ; GFX1100-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1100-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1100-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index df880164b196b..fd50d1b60fbd1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -17,19 +17,19 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
 ; SI-SDAG-NEXT:    s_load_dword s0, s[4:5], 0xb
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; SI-SDAG-NEXT:    s_mov_b32 s1, 0x3284fbcf
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s0, v0
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s0, 0x3e9a209a
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s0, -v1
-; SI-SDAG-NEXT:    s_mov_b32 s0, 0x3284fbcf
-; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s0, v2
+; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s1, v2
 ; SI-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
 ; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s0
@@ -73,11 +73,11 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
 ; VI-SDAG-NEXT:    s_load_dword s0, s[4:5], 0x2c
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -134,27 +134,27 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_load_dword s0, s[4:5], 0x2c
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s1, 0x3284fbcf
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s0, 0x3e9a209a
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v3, v0, s0, -v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v3, v0, s1, v3
-; GFX900-SDAG-NEXT:    s_mov_b32 s0, 0x7f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s0, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s1, v3
 ; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x411a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
-; GFX900-SDAG-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
 ; GFX900-GISEL-LABEL: s_log10_f32:
@@ -190,21 +190,23 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s2, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s2
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
 ; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_add_f32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, s2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
@@ -314,38 +316,39 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3284fbcf
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x7f800000
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s3, v2
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s3, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s3, 0x3e9a209a
 ; SI-SDAG-NEXT:    s_mov_b32 s4, s0
 ; SI-SDAG-NEXT:    s_mov_b32 s5, s1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s3, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s8, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s9
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s3, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s8, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s9
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s2, v0
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x411a209b
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v2, v1
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v0
-; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s3, -v2
+; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x411a209b
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v0
+; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s3, -v3
 ; SI-SDAG-NEXT:    v_fma_f32 v4, v0, s8, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s9
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -394,43 +397,44 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s2, 0x7f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s7, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a2000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s2
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s7, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s6, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s6, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s6, v0
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x411a209b
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v2, v1
-; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v0, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x411a209b
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v0, v3
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v4
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v3
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a2000, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s5
@@ -488,36 +492,37 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s2, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    s_mov_b32 s3, 0x3284fbcf
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, s11, v3
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v3
-; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v3
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v3, s2, -v4
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v3, s3, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v3|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s2, -v3
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s3, v4
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s10, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s10, v0
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0x411a209b
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v1
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s2, -v3
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x411a209b
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s2, -v4
 ; GFX900-SDAG-NEXT:    v_fma_f32 v5, v0, s3, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
 ; GFX900-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX900-SDAG-NEXT:    s_endpgm
@@ -564,31 +569,37 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s4, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s5, 0x800000, s2
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s4, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s4
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s5
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s3, v0 :: v_dual_mul_f32 v1, s2, v1
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s5
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s4
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v2, 0x3e9a209a, v0 :: v_dual_mul_f32 v3, 0x3e9a209a, v1
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v3, 0x3e9a209a, v1 :: v_dual_lshlrev_b32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v5, 0x3e9a209a, v1, -v3
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v5, 0x3284fbcf, v1
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_dual_add_f32 v3, v3, v5 :: v_dual_mul_f32 v2, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x411a209b, s5
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v4, 0x3e9a209a, v0, -v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_fma_f32 v5, 0x3e9a209a, v1, -v3
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v4, 0x3284fbcf, v0 :: v_dual_fmac_f32 v5, 0x3284fbcf, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v2, v2, v4 :: v_dual_add_f32 v3, v3, v5
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v4, 0x3284fbcf, v0
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x411a209b, s4
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x411a209b, s5
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
 ; GFX1100-SDAG-NEXT:    v_dual_cndmask_b32 v2, v1, v3 :: v_dual_mov_b32 v3, 0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v1, v0, v4 :: v_dual_sub_f32 v0, v2, v5
 ; GFX1100-SDAG-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
 ; GFX1100-SDAG-NEXT:    s_endpgm
@@ -742,49 +753,51 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s11, 0x3284fbcf
+; SI-SDAG-NEXT:    s_mov_b32 s12, 0x7f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, s9, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s9, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x3e9a209a
-; SI-SDAG-NEXT:    s_mov_b32 s12, 0x7f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v1
-; SI-SDAG-NEXT:    v_fma_f32 v4, v1, s9, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v1, s11, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
+; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s9, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s11, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v2, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, s8, v3
-; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
-; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x411a209b
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, s8, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x411a209b
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
-; SI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a209a, v3
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s10, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v3, s9, -v5
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v3, s11, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v3|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
-; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v2
-; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s9, -v3
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
+; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s9, -v4
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s11, v5
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s12
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s10, v0
+; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v0
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s12
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v3, s[0:1]
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v5
+; SI-SDAG-NEXT:    v_fma_f32 v4, v5, s9, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v4, v5, s11, v4
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v5|, s12
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -847,47 +860,49 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s6, 0x7f800000
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s10, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a2000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s10, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s9, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, s9, v3
-; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, s9, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x411a209b
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v5
-; VI-SDAG-NEXT:    v_and_b32_e32 v5, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s8, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v3, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3e9a2000, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v5
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v3, v1
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x369a84fb, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v5
-; VI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a2000, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v3|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v5, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
 ; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
 ; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v0
@@ -978,39 +993,41 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
+; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, s10, v2
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s10, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s10, 0x7f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x411a209b
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s4, -v4
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s5, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s10
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s4, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v4, v1, s5, v4
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s10
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s9, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v4, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, s9, v4
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x411a209b
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, s9, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s8, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v6
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a209a, v4
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v4, s4, -v6
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v4
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v4, s4, -v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v4, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v4, s5, v6
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s10
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v4, v6, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
@@ -1079,48 +1096,54 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s3, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s1
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s3
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s3
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s6
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v9, 0, 0x411a209b, s3
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, 0x411a209b, s6
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s2, v0 :: v_dual_mul_f32 v1, s1, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3e9a209a, v2 :: v_dual_lshlrev_b32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s2, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v8, 0x3e9a209a, v2, -v5
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v8, 0x3284fbcf, v2
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v5, v5, v8
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v0
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v6, 0x3e9a209a, v0, -v3
+; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v6, 0x3284fbcf, v0 :: v_dual_lshlrev_b32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s1, v1
+; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v3, v3, v6
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 0x411a209b, s7
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a209a, v2
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v1
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v7, 0x3e9a209a, v1, -v4
-; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v6, 0x3284fbcf, v0
-; GFX1100-SDAG-NEXT:    v_fma_f32 v8, 0x3e9a209a, v2, -v5
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmac_f32_e32 v7, 0x3284fbcf, v1
-; GFX1100-SDAG-NEXT:    v_add_f32_e32 v3, v3, v6
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 0x411a209b, s7
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v8, 0x3284fbcf, v2 :: v_dual_cndmask_b32 v1, v1, v4
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v5, v5, v8 :: v_dual_mov_b32 v4, 0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v10
+; GFX1100-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_sub_f32 v1, v1, v10
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v5, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v2, v0, v9
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -1343,60 +1366,63 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s12, 0x3284fbcf
+; SI-SDAG-NEXT:    s_mov_b32 s13, 0x7f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s11, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s11, 0x3e9a209a
-; SI-SDAG-NEXT:    s_mov_b32 s13, 0x7f800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x411a209b
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s11, -v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, v2, s12, v4
-; SI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x411a209b
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s11, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v1, s12, v3
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; SI-SDAG-NEXT:    v_log_f32_e32 v4, v3
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, s10, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v4
-; SI-SDAG-NEXT:    v_fma_f32 v6, v4, s11, -v2
-; SI-SDAG-NEXT:    v_fma_f32 v6, v4, s12, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v6
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v6, s9, v6
-; SI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v2
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s11, -v1
+; SI-SDAG-NEXT:    v_fma_f32 v5, v2, s12, v5
+; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v5, s9, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[0:1]
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v6
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; SI-SDAG-NEXT:    v_fma_f32 v7, v6, s11, -v4
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s8, v0
+; SI-SDAG-NEXT:    v_fma_f32 v6, v5, s11, -v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; SI-SDAG-NEXT:    v_fma_f32 v7, v6, s12, v7
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v4, s[2:3]
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v5, vcc
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
-; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s11, -v4
+; SI-SDAG-NEXT:    v_fma_f32 v6, v5, s12, v6
+; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s13
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
+; SI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a209a, v0
+; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s11, -v5
 ; SI-SDAG-NEXT:    v_fma_f32 v6, v0, s12, v6
-; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; SI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s13
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v4
 ; SI-SDAG-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -1470,78 +1496,81 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    s_mov_b32 s6, 0x7f800000
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff000, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v4, v2, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v3
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v5, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a2000, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v6, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v5, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; VI-SDAG-NEXT:    v_log_f32_e32 v4, v3
-; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x411a209b
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
-; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v4, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v2, s10, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x411a209b
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v2, v1
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x369a84fb, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a2000, v1
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v5, s9, v5
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v4, s[0:1]
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v5, v1
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3e9a2000, v6
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, s9, v6
-; VI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v4|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[2:3]
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v6
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v7, v6, v4
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3e9a2000, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x369a84fb, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v9, v7
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v8, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a2000, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v7
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v4, s[2:3]
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v5, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v0, v4
+; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a2000, v1
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s6
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
+; VI-SDAG-NEXT:    v_and_b32_e32 v5, 0xfffff000, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, v0, v5
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3e9a2000, v6
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v5
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v8, v6
 ; VI-SDAG-NEXT:    v_add_f32_e32 v6, v7, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a2000, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a2000, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v5, v6
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v5, s[0:1]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v5, s5
@@ -1635,60 +1664,63 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
+; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s11, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, s11, v2
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX900-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s11, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    s_mov_b32 s11, 0x7f800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v6, 0x411a209b
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3e9a209a, v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s4, -v3
-; GFX900-SDAG-NEXT:    v_fma_f32 v5, v2, s5, v5
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v3, v3, v5
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v2|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0x411a209b
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s4, -v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v3, v1, s5, v3
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v1|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s10, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v3, s10, v3
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v5, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v3
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v2, s10, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v1, v3
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s9, v0
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v5
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v5, s4, -v2
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v5, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v2, v2, v7
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v7, s9, v7
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v7, v7
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v5|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v6, s[0:1]
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v2
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v2, s4, -v1
+; GFX900-SDAG-NEXT:    v_fma_f32 v6, v2, s5, v6
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v6
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 5, v6
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v6, s9, v6
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v6, v6
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v2|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[0:1], s8, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, v1, s[0:1]
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a209a, v7
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, s8, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v8, v7, s4, -v5
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v2
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v6
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, s8, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v6, s4, -v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v8, v7, s5, v8
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, v5, v8
-; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v7|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v7, v5, s[2:3]
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v6, vcc
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3e9a209a, v0
-; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s4, -v5
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v6, s5, v7
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v1, v1, v7
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], |v6|, s11
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, v6, v1, s[2:3]
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v6
+; GFX900-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a209a, v0
+; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s4, -v6
 ; GFX900-SDAG-NEXT:    v_fma_f32 v7, v0, s5, v7
-; GFX900-SDAG-NEXT:    v_add_f32_e32 v5, v5, v7
+; GFX900-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s11
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v6, s[0:1]
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, v5, s[0:1]
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v5
 ; GFX900-SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
@@ -1760,56 +1792,65 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s3
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s8, 0x800000, s1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s9, 0x800000, s0
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s3
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s8
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s9
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s8
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0x4f800000, s9
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s6
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x411a209b, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v0, s3, v0 :: v_dual_mul_f32 v1, s2, v1
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v2, s1, v2 :: v_dual_mul_f32 v3, s0, v3
-; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(TRANS32_DEP_3)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v9, 0, 0x411a209b, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v14, 0, 0x411a209b, s8
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v15, 0, 0x411a209b, s9
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3e9a209a, v0 :: v_dual_mul_f32 v6, 0x3e9a209a, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s1, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v3, s0, v3
+; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3e9a209a, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3e9a209a, v3
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v12, 0x3e9a209a, v2, -v7
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_fma_f32 v13, 0x3e9a209a, v3, -v8
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v12, 0x3284fbcf, v2 :: v_dual_fmac_f32 v13, 0x3284fbcf, v3
+; GFX1100-SDAG-NEXT:    v_add_f32_e32 v7, v7, v12
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v7, 0x3e9a209a, v2 :: v_dual_mul_f32 v8, 0x3e9a209a, v3
+; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v5, 0x3e9a209a, v0 :: v_dual_add_f32 v8, v8, v13
+; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a209a, v1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v10, 0x3e9a209a, v0, -v5
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v11, 0x3e9a209a, v1, -v6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_fma_f32 v12, 0x3e9a209a, v2, -v7
-; GFX1100-SDAG-NEXT:    v_fma_f32 v13, 0x3e9a209a, v3, -v8
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v10, 0x3284fbcf, v0 :: v_dual_fmac_f32 v11, 0x3284fbcf, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_dual_fmac_f32 v12, 0x3284fbcf, v2 :: v_dual_fmac_f32 v13, 0x3284fbcf, v3
 ; GFX1100-SDAG-NEXT:    v_dual_add_f32 v5, v5, v10 :: v_dual_add_f32 v6, v6, v11
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_dual_add_f32 v7, v7, v12 :: v_dual_add_f32 v8, v8, v13
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v5, v2, v7, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_dual_mov_b32 v7, 0 :: v_dual_sub_f32 v2, v1, v9
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v6, v3, v8, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v3, v0, v4
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v1, v5, v14 :: v_dual_sub_f32 v0, v6, v15
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    global_store_b128 v7, v[0:3], s[0:1]
@@ -2062,10 +2103,10 @@ define float @v_log10_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2108,10 +2149,10 @@ define float @v_log10_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2160,10 +2201,10 @@ define float @v_log10_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2206,21 +2247,22 @@ define float @v_log10_f32(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2264,10 +2306,10 @@ define float @v_log10_fabs_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2310,10 +2352,10 @@ define float @v_log10_fabs_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2362,10 +2404,10 @@ define float @v_log10_fabs_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2409,20 +2451,22 @@ define float @v_log10_fabs_f32(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, s0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2467,10 +2511,10 @@ define float @v_log10_fneg_fabs_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -|v0|, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2513,10 +2557,10 @@ define float @v_log10_fneg_fabs_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2565,10 +2609,10 @@ define float @v_log10_fneg_fabs_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2612,20 +2656,22 @@ define float @v_log10_fneg_fabs_f32(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e64 s0, 0x80800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, s0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2671,10 +2717,10 @@ define float @v_log10_fneg_f32(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2717,10 +2763,10 @@ define float @v_log10_fneg_f32(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -2769,10 +2815,10 @@ define float @v_log10_fneg_f32(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -2815,21 +2861,22 @@ define float @v_log10_fneg_f32(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 0x80800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2875,9 +2922,9 @@ define float @v_log10_f32_fast(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2902,9 +2949,9 @@ define float @v_log10_f32_fast(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2929,9 +2976,9 @@ define float @v_log10_f32_fast(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2955,10 +3002,12 @@ define float @v_log10_f32_fast(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -2993,9 +3042,9 @@ define float @v_log10_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3020,9 +3069,9 @@ define float @v_log10_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3047,9 +3096,9 @@ define float @v_log10_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3073,10 +3122,12 @@ define float @v_log10_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -3111,9 +3162,9 @@ define float @v_log10_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3138,9 +3189,9 @@ define float @v_log10_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3165,9 +3216,9 @@ define float @v_log10_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3191,10 +3242,12 @@ define float @v_log10_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -3228,10 +3281,10 @@ define float @v_log10_f32_ninf(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -3274,10 +3327,10 @@ define float @v_log10_f32_ninf(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -3326,10 +3379,10 @@ define float @v_log10_f32_ninf(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -3372,21 +3425,22 @@ define float @v_log10_f32_ninf(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3431,9 +3485,9 @@ define float @v_log10_f32_afn(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3458,9 +3512,9 @@ define float @v_log10_f32_afn(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3485,9 +3539,9 @@ define float @v_log10_f32_afn(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3511,10 +3565,12 @@ define float @v_log10_f32_afn(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -3578,9 +3634,9 @@ define float @v_log10_f32_afn_dynamic(float %in) #1 {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3605,9 +3661,9 @@ define float @v_log10_f32_afn_dynamic(float %in) #1 {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3632,9 +3688,9 @@ define float @v_log10_f32_afn_dynamic(float %in) #1 {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3658,10 +3714,12 @@ define float @v_log10_f32_afn_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -3696,9 +3754,9 @@ define float @v_fabs_log10_f32_afn(float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v2
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3723,9 +3781,9 @@ define float @v_fabs_log10_f32_afn(float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3750,9 +3808,9 @@ define float @v_fabs_log10_f32_afn(float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0xc11a209b
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -3777,10 +3835,11 @@ define float @v_fabs_log10_f32_afn(float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0xc11a209b, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3e9a209b, v1
@@ -3956,10 +4015,10 @@ define float @v_log10_f32_nnan(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4002,10 +4061,10 @@ define float @v_log10_f32_nnan(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4054,10 +4113,10 @@ define float @v_log10_f32_nnan(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4100,21 +4159,22 @@ define float @v_log10_f32_nnan(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4298,10 +4358,10 @@ define float @v_log10_f32_nnan_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4344,10 +4404,10 @@ define float @v_log10_f32_nnan_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4396,10 +4456,10 @@ define float @v_log10_f32_nnan_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4442,21 +4502,22 @@ define float @v_log10_f32_nnan_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4640,10 +4701,10 @@ define float @v_log10_f32_ninf_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4686,10 +4747,10 @@ define float @v_log10_f32_ninf_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -4738,10 +4799,10 @@ define float @v_log10_f32_ninf_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4784,21 +4845,22 @@ define float @v_log10_f32_ninf_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4842,10 +4904,10 @@ define float @v_log10_f32_nnan_ninf(float %in) {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4882,10 +4944,10 @@ define float @v_log10_f32_nnan_ninf(float %in) {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -4928,10 +4990,10 @@ define float @v_log10_f32_nnan_ninf(float %in) {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -4968,18 +5030,20 @@ define float @v_log10_f32_nnan_ninf(float %in) {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3284fbcf, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5123,10 +5187,10 @@ define float @v_log10_f32_nnan_ninf_dynamic(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -5163,10 +5227,10 @@ define float @v_log10_f32_nnan_ninf_dynamic(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -5209,10 +5273,10 @@ define float @v_log10_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -5249,18 +5313,20 @@ define float @v_log10_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fmamk_f32 v0, v0, 0x3284fbcf, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5330,10 +5396,10 @@ define float @v_log10_f32_dynamic_mode(float %in) #1 {
 ; SI-SDAG:       ; %bb.0:
 ; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -5376,10 +5442,10 @@ define float @v_log10_f32_dynamic_mode(float %in) #1 {
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; VI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-SDAG-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -5428,10 +5494,10 @@ define float @v_log10_f32_dynamic_mode(float %in) #1 {
 ; GFX900-SDAG:       ; %bb.0:
 ; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -5474,21 +5540,22 @@ define float @v_log10_f32_dynamic_mode(float %in) #1 {
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
 ; GFX1100-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6007,17 +6074,17 @@ define float @v_log10_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
 ; SI-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s4, -v1
-; SI-SDAG-NEXT:    s_mov_b32 s4, 0x3284fbcf
-; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s4, v2
+; SI-SDAG-NEXT:    v_fma_f32 v2, v0, s5, v2
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x7f800000
 ; SI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
@@ -6179,10 +6246,10 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) {
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, 0x800000
-; SI-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-NEXT:    v_log_f32_e32 v0, v0
 ; SI-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; SI-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -6203,10 +6270,10 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; VI-NEXT:    s_mov_b32 s4, 0x800000
-; VI-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-NEXT:    v_log_f32_e32 v0, v0
 ; VI-NEXT:    s_mov_b32 s4, 0x7f800000
 ; VI-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
@@ -6230,10 +6297,10 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) {
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0x800000
-; GFX900-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX900-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0x3e9a209a
 ; GFX900-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
@@ -6255,20 +6322,22 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) {
 ; GFX1100-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1100-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f32 v0, v0, v1
 ; GFX1100-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-NEXT:    v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_add_f32_e32 v1, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s0
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX1100-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index c5dea7fd8b4b1..2c5a9f58a199e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -22,9 +22,9 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s2, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v1, v0
@@ -59,9 +59,9 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
 ; VI-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v1, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -98,9 +98,9 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
 ; GFX900-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v1, v0
 ; GFX900-SDAG-NEXT:    global_store_dword v2, v0, s[0:1]
@@ -132,11 +132,12 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s0
 ; GFX1100-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v1, v0
@@ -215,24 +216,25 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, s3, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
-; SI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_log_f32_e32 v3, v1
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s3, v3
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s2, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_log_f32_e32 v4, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s4, s0
 ; SI-SDAG-NEXT:    s_mov_b32 s5, s1
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v4, v2
-; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v3, v0
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v4, v0
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
@@ -265,21 +267,22 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v2, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s3, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
-; VI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_log_f32_e32 v2, v1
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v4, v3
-; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v4, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v4, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
@@ -313,21 +316,22 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v2, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, s3, v4
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, s2, v1
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v2, v1
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v4, v3
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v1
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v4, v0
 ; GFX900-SDAG-NEXT:    global_store_dwordx2 v5, v[0:1], s[0:1]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
@@ -362,13 +366,16 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s4, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s5, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 1.0, 0x4f800000, s4
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0x4f800000, s5
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s4
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s5
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s4
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 0x42000000, s5
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v1, s3, v1
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v3, s2, v3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v1, s3, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v1, v1
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v3, v3
@@ -469,28 +476,30 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
+; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 1.0, v3, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, s1, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, s2, v0
-; SI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v6, s0, v6
-; SI-SDAG-NEXT:    v_log_f32_e32 v3, v0
-; SI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s1, v3
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v5, s0, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v7, v0
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
 ; SI-SDAG-NEXT:    s_mov_b32 s6, -1
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v4, v2
-; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v3, v7
-; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v6, v5
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
+; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v7, v6
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v5, v4
 ; SI-SDAG-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-SDAG-NEXT:    s_endpgm
@@ -533,27 +542,29 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s2, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s0, v1
-; VI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, s1, v6
-; VI-SDAG-NEXT:    v_log_f32_e32 v3, v1
-; VI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v4, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v3, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
+; VI-SDAG-NEXT:    v_ldexp_f32 v5, s1, v5
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v6, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v3, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v5, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v6, v5
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v6, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s5
 ; VI-SDAG-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
 ; VI-SDAG-NEXT:    s_endpgm
@@ -595,27 +606,29 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v4, s2, v4
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v6, s1, v6
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, s0, v1
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v1
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v7, 0
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v4, v2
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v6, v5
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v3, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v5, s1, v5
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v6, v1
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v3, v2
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v5, v4
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v6, v0
 ; GFX900-SDAG-NEXT:    global_store_dwordx3 v7, v[0:2], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
@@ -654,28 +667,35 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
 ; GFX1100-SDAG-NEXT:    s_clause 0x1
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s3, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s1
+; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s3, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s3
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 1.0, 0x4f800000, s7
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s3
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s6
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s3
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, s6
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v2, s2, v2
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v4, s1, v4 :: v_dual_mul_f32 v5, s0, v5
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s3
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 5, v4
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 0x42000000, s7
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v4, s1, v4
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v5, s0, v5
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v5, v5
-; GFX1100-SDAG-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v2, v2, v0 :: v_dual_sub_f32 v1, v4, v1
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v1, v4, v1
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s2, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v0
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v5, v3
 ; GFX1100-SDAG-NEXT:    global_store_b96 v6, v[0:2], s[4:5]
 ; GFX1100-SDAG-NEXT:    s_endpgm
@@ -806,34 +826,37 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0xd
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
 ; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v8, 1.0, v3, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v4, s7, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v6, s6, v6
-; SI-SDAG-NEXT:    v_mul_f32_e32 v8, s5, v8
-; SI-SDAG-NEXT:    v_mul_f32_e32 v1, s4, v1
-; SI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; SI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; SI-SDAG-NEXT:    v_log_f32_e32 v8, v8
-; SI-SDAG-NEXT:    v_log_f32_e32 v9, v1
-; SI-SDAG-NEXT:    s_mov_b32 s2, -1
-; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v4, v2
-; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v6, v5
-; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v8, v7
-; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v9, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v7, 5, v7
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s7, v3
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v5, s6, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v7, s5, v7
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s4, v1
+; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; SI-SDAG-NEXT:    v_log_f32_e32 v7, v7
+; SI-SDAG-NEXT:    v_log_f32_e32 v8, v1
+; SI-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v2
+; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v5, v4
+; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v7, v6
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v8, v0
 ; SI-SDAG-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
@@ -880,33 +903,36 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; VI-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 1.0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 1.0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v8, 1.0, v3, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
 ; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s3, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, s2, v6
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 5, v5
 ; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; VI-SDAG-NEXT:    v_log_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_log_f32_e32 v6, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, s1, v8
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s0, v1
-; VI-SDAG-NEXT:    v_log_f32_e32 v8, v8
-; VI-SDAG-NEXT:    v_log_f32_e32 v9, v1
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v4, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v6, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; VI-SDAG-NEXT:    v_ldexp_f32 v5, s2, v5
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v7, 5, v7
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v7, s1, v7
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v7, v7
+; VI-SDAG-NEXT:    v_log_f32_e32 v8, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v5, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, s4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v8, v7
-; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v9, v0
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v7, v6
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v8, v0
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v5, s5
 ; VI-SDAG-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-SDAG-NEXT:    s_endpgm
@@ -954,33 +980,36 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; GFX900-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v3, 0x4f800000
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX900-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 1.0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v7, 1.0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v8, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v9, 1.0, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v3, vcc
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v5, s3, v5
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v7, s2, v7
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v9, s1, v9
-; GFX900-SDAG-NEXT:    v_mul_f32_e32 v1, s0, v1
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v5, v5
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v7, v7
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v9, v9
-; GFX900-SDAG-NEXT:    v_log_f32_e32 v10, v1
-; GFX900-SDAG-NEXT:    v_mov_b32_e32 v4, 0
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v5, v2
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v7, v6
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v9, v8
-; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v10, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 5, v6
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v8, 5, v8
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v6, s2, v6
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v8, s1, v8
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v1, s0, v1
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v6, v6
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v8, v8
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v9, v1
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v2
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v2, v6, v5
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v1, v8, v7
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v9, v0
 ; GFX900-SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GFX900-SDAG-NEXT:    s_endpgm
 ;
@@ -1025,34 +1054,42 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 ; GFX1100-SDAG-NEXT:    s_clause 0x1
 ; GFX1100-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX1100-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
+; GFX1100-SDAG-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX1100-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s6, 0x800000, s3
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s7, 0x800000, s2
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s8, 0x800000, s1
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s9, 0x800000, s0
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s6
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0x4f800000, s7
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 1.0, 0x4f800000, s8
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v7, 1.0, 0x4f800000, s9
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s6
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v2, s3, v2 :: v_dual_mul_f32 v3, s2, v3
-; GFX1100-SDAG-NEXT:    v_dual_mul_f32 v6, s1, v6 :: v_dual_mul_f32 v7, s0, v7
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s6
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s7
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, s7
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s8
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s9
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 5, v3
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 0x42000000, s6
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x42000000, s8
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v7, 5, v7
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v2, s3, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x42000000, s9
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v7, s0, v7
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 5, v6
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v8, v3
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(TRANS32_DEP_3)
-; GFX1100-SDAG-NEXT:    v_log_f32_e32 v6, v6
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v7, v7
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 0x42000000, s8
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 0x42000000, s9
-; GFX1100-SDAG-NEXT:    v_mov_b32_e32 v9, 0
-; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v3, v2, v0 :: v_dual_sub_f32 v2, v8, v1
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v3, v2, v0
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v6, s1, v6
+; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v2, v8, v1
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v7, v5
+; GFX1100-SDAG-NEXT:    v_log_f32_e32 v6, v6
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
-; GFX1100-SDAG-NEXT:    v_dual_sub_f32 v1, v6, v4 :: v_dual_sub_f32 v0, v7, v5
+; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v1, v6, v4
 ; GFX1100-SDAG-NEXT:    global_store_b128 v9, v[0:3], s[4:5]
 ; GFX1100-SDAG-NEXT:    s_endpgm
 ;
@@ -1192,19 +1229,19 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
 }
 
 define float @v_log2_f32(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1220,14 +1257,44 @@ define float @v_log2_f32(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1260,19 +1327,19 @@ define float @v_log2_f32(float %in) {
 }
 
 define float @v_log2_fabs_f32(float %in) {
-; GFX689-SDAG-LABEL: v_log2_fabs_f32:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fabs_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_fabs_f32:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1288,15 +1355,44 @@ define float @v_log2_fabs_f32(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_fabs_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_fabs_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_fabs_f32:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1331,19 +1427,19 @@ define float @v_log2_fabs_f32(float %in) {
 }
 
 define float @v_log2_fneg_fabs_f32(float %in) {
-; GFX689-SDAG-LABEL: v_log2_fneg_fabs_f32:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fneg_fabs_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -|v0|, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_fneg_fabs_f32:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1359,15 +1455,44 @@ define float @v_log2_fneg_fabs_f32(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_fneg_fabs_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_fneg_fabs_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_fneg_fabs_f32:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e64 s0, 0x80800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -|v0|, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -|v0|, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1403,19 +1528,19 @@ define float @v_log2_fneg_fabs_f32(float %in) {
 }
 
 define float @v_log2_fneg_f32(float %in) {
-; GFX689-SDAG-LABEL: v_log2_fneg_f32:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
-; GFX689-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fneg_f32:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, -v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_fneg_f32:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1431,14 +1556,44 @@ define float @v_log2_fneg_f32(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_fneg_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_fneg_f32:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x80800000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_fneg_f32:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 0x80800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, -v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1473,19 +1628,19 @@ define float @v_log2_fneg_f32(float %in) {
 }
 
 define float @v_log2_f32_fast(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32_fast:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_fast:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_fast:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1501,14 +1656,44 @@ define float @v_log2_f32_fast(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_fast:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_fast:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_fast:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1541,19 +1726,19 @@ define float @v_log2_f32_fast(float %in) {
 }
 
 define float @v_log2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
-; GFX689-SDAG-LABEL: v_log2_f32_unsafe_math_attr:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_unsafe_math_attr:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_unsafe_math_attr:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1569,14 +1754,44 @@ define float @v_log2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_unsafe_math_attr:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_unsafe_math_attr:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_unsafe_math_attr:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1609,19 +1824,19 @@ define float @v_log2_f32_unsafe_math_attr(float %in) "unsafe-fp-math"="true" {
 }
 
 define float @v_log2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true" {
-; GFX689-SDAG-LABEL: v_log2_f32_approx_fn_attr:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_approx_fn_attr:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_approx_fn_attr:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1637,14 +1852,44 @@ define float @v_log2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_approx_fn_attr:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_approx_fn_attr:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_approx_fn_attr:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1677,19 +1922,19 @@ define float @v_log2_f32_approx_fn_attr(float %in) "approx-func-fp-math"="true"
 }
 
 define float @v_log2_f32_ninf(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32_ninf:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_ninf:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_ninf:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1705,14 +1950,44 @@ define float @v_log2_f32_ninf(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_ninf:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_ninf:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_ninf:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1745,19 +2020,19 @@ define float @v_log2_f32_ninf(float %in) {
 }
 
 define float @v_log2_f32_afn(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32_afn:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_afn:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_afn:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1773,14 +2048,44 @@ define float @v_log2_f32_afn(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_afn:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_afn:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_afn:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1839,19 +2144,19 @@ define float @v_log2_f32_afn_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_afn_dynamic(float %in) #1 {
-; GFX689-SDAG-LABEL: v_log2_f32_afn_dynamic:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_afn_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_afn_dynamic:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1867,14 +2172,44 @@ define float @v_log2_f32_afn_dynamic(float %in) #1 {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_afn_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_afn_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_afn_dynamic:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -1907,19 +2242,19 @@ define float @v_log2_f32_afn_dynamic(float %in) #1 {
 }
 
 define float @v_fabs_log2_f32_afn(float %in) {
-; GFX689-SDAG-LABEL: v_fabs_log2_f32_afn:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_fabs_log2_f32_afn:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, |v0|, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_fabs_log2_f32_afn:
 ; GFX689-GISEL:       ; %bb.0:
@@ -1935,15 +2270,44 @@ define float @v_fabs_log2_f32_afn(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_fabs_log2_f32_afn:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_fabs_log2_f32_afn:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_fabs_log2_f32_afn:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e64 s0, 0x800000, |v0|
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, s0
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, s0
-; GFX1100-SDAG-NEXT:    v_mul_f32_e64 v0, |v0|, v2
-; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, |v0|, v2
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2004,19 +2368,19 @@ define float @v_log2_f32_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_nnan(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32_nnan:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_nnan:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_nnan:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2032,14 +2396,44 @@ define float @v_log2_f32_nnan(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_nnan:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_nnan:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_nnan:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2098,19 +2492,19 @@ define float @v_log2_f32_nnan_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_nnan_dynamic(float %in) #1 {
-; GFX689-SDAG-LABEL: v_log2_f32_nnan_dynamic:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_nnan_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_nnan_dynamic:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2126,14 +2520,44 @@ define float @v_log2_f32_nnan_dynamic(float %in) #1 {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_nnan_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_nnan_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_nnan_dynamic:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2192,19 +2616,19 @@ define float @v_log2_f32_ninf_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_ninf_dynamic(float %in) #1 {
-; GFX689-SDAG-LABEL: v_log2_f32_ninf_dynamic:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_ninf_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_ninf_dynamic:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2220,14 +2644,44 @@ define float @v_log2_f32_ninf_dynamic(float %in) #1 {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_ninf_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_ninf_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_ninf_dynamic:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2260,19 +2714,19 @@ define float @v_log2_f32_ninf_dynamic(float %in) #1 {
 }
 
 define float @v_log2_f32_nnan_ninf(float %in) {
-; GFX689-SDAG-LABEL: v_log2_f32_nnan_ninf:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_nnan_ninf:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_nnan_ninf:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2288,14 +2742,44 @@ define float @v_log2_f32_nnan_ninf(float %in) {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_nnan_ninf:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_nnan_ninf:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_nnan_ninf:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2354,19 +2838,19 @@ define float @v_log2_f32_nnan_ninf_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_nnan_ninf_dynamic(float %in) #1 {
-; GFX689-SDAG-LABEL: v_log2_f32_nnan_ninf_dynamic:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_nnan_ninf_dynamic:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_nnan_ninf_dynamic:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2382,14 +2866,44 @@ define float @v_log2_f32_nnan_ninf_dynamic(float %in) #1 {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_nnan_ninf_dynamic:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_nnan_ninf_dynamic:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_nnan_ninf_dynamic:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2448,19 +2962,19 @@ define float @v_log2_f32_fast_daz(float %in) #0 {
 }
 
 define float @v_log2_f32_dynamic_mode(float %in) #1 {
-; GFX689-SDAG-LABEL: v_log2_f32_dynamic_mode:
-; GFX689-SDAG:       ; %bb.0:
-; GFX689-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX689-SDAG-NEXT:    s_mov_b32 s4, 0x800000
-; GFX689-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX689-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX689-SDAG-NEXT:    v_log_f32_e32 v0, v0
-; GFX689-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; GFX689-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX689-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
-; GFX689-SDAG-NEXT:    s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_dynamic_mode:
+; SI-SDAG:       ; %bb.0:
+; SI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX689-GISEL-LABEL: v_log2_f32_dynamic_mode:
 ; GFX689-GISEL:       ; %bb.0:
@@ -2476,14 +2990,44 @@ define float @v_log2_f32_dynamic_mode(float %in) #1 {
 ; GFX689-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
 ; GFX689-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; VI-SDAG-LABEL: v_log2_f32_dynamic_mode:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; VI-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; VI-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX900-SDAG-LABEL: v_log2_f32_dynamic_mode:
+; GFX900-SDAG:       ; %bb.0:
+; GFX900-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x800000
+; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX900-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1100-SDAG-LABEL: v_log2_f32_dynamic_mode:
 ; GFX1100-SDAG:       ; %bb.0:
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
 ; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-SDAG-NEXT:    v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-SDAG-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v1
@@ -2649,10 +3193,10 @@ define float @v_log2_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
 ; SI-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-SDAG-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; SI-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; SI-SDAG-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; SI-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2718,9 +3262,9 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, 0x800000
 ; SI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; SI-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; SI-NEXT:    v_log_f32_e32 v0, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2732,10 +3276,10 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; VI-NEXT:    s_mov_b32 s4, 0x800000
-; VI-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; VI-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; VI-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; VI-NEXT:    v_mul_f32_e32 v0, v0, v1
+; VI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; VI-NEXT:    v_ldexp_f32 v0, v0, v1
 ; VI-NEXT:    v_log_f32_e32 v0, v0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2748,9 +3292,9 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0x800000
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GFX900-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX900-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX900-NEXT:    v_log_f32_e32 v0, v0
 ; GFX900-NEXT:    v_mov_b32_e32 v1, 0x42000000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
@@ -2763,10 +3307,11 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
 ; GFX1100-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1100-NEXT:    v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
+; GFX1100-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT:    v_lshlrev_b32_e32 v2, 5, v2
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT:    v_ldexp_f32 v0, v0, v2
 ; GFX1100-NEXT:    v_log_f32_e32 v0, v0
 ; GFX1100-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-NEXT:    v_sub_f32_e32 v0, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
index 043294f5a1163..d31f944a81f11 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
@@ -4,6 +4,8 @@
 
 target datalayout = "A5"
 
+declare void @llvm.fake.use(...)
+
 define amdgpu_kernel void @kern_noargs() {
 ; GCN-LABEL: @kern_noargs(
 ; GCN-NEXT:    ret void
@@ -255,6 +257,25 @@ define amdgpu_kernel void @kern_i32(i32 %arg0) {
   ret void
 }
 
+define amdgpu_kernel void @kern_range_noundef_i32(i32 noundef range(i32 0, 8) %arg0) {
+; HSA-LABEL: @kern_range_noundef_i32(
+; HSA-NEXT:    [[KERN_RANGE_NOUNDEF_I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_RANGE_NOUNDEF_I32_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 16, !range [[RNG2:![0-9]+]], !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    call void (...) @llvm.fake.use(i32 [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @kern_range_noundef_i32(
+; MESA-NEXT:    [[KERN_RANGE_NOUNDEF_I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(260) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_RANGE_NOUNDEF_I32_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 4, !range [[RNG2:![0-9]+]], !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    call void (...) @llvm.fake.use(i32 [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(i32 %arg0)
+  ret void
+}
+
 define amdgpu_kernel void @kern_f32(float %arg0) {
 ; HSA-LABEL: @kern_f32(
 ; HSA-NEXT:    [[KERN_F32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
@@ -1022,14 +1043,14 @@ define amdgpu_kernel void @kern_global_ptr_dereferencable(ptr addrspace(1) deref
 ; HSA-LABEL: @kern_global_ptr_dereferencable(
 ; HSA-NEXT:    [[KERN_GLOBAL_PTR_DEREFERENCABLE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; HSA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_GLOBAL_PTR_DEREFERENCABLE_KERNARG_SEGMENT]], i64 0
-; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !dereferenceable [[META2:![0-9]+]]
+; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !dereferenceable [[META3:![0-9]+]]
 ; HSA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; HSA-NEXT:    ret void
 ;
 ; MESA-LABEL: @kern_global_ptr_dereferencable(
 ; MESA-NEXT:    [[KERN_GLOBAL_PTR_DEREFERENCABLE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; MESA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_GLOBAL_PTR_DEREFERENCABLE_KERNARG_SEGMENT]], i64 36
-; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !dereferenceable [[META2:![0-9]+]]
+; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !dereferenceable [[META3:![0-9]+]]
 ; MESA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; MESA-NEXT:    ret void
 ;
@@ -1041,14 +1062,14 @@ define amdgpu_kernel void @kern_global_ptr_dereferencable_or_null(ptr addrspace(
 ; HSA-LABEL: @kern_global_ptr_dereferencable_or_null(
 ; HSA-NEXT:    [[KERN_GLOBAL_PTR_DEREFERENCABLE_OR_NULL_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; HSA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_GLOBAL_PTR_DEREFERENCABLE_OR_NULL_KERNARG_SEGMENT]], i64 0
-; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !dereferenceable_or_null [[META3:![0-9]+]]
+; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !dereferenceable_or_null [[META4:![0-9]+]]
 ; HSA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; HSA-NEXT:    ret void
 ;
 ; MESA-LABEL: @kern_global_ptr_dereferencable_or_null(
 ; MESA-NEXT:    [[KERN_GLOBAL_PTR_DEREFERENCABLE_OR_NULL_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; MESA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_GLOBAL_PTR_DEREFERENCABLE_OR_NULL_KERNARG_SEGMENT]], i64 36
-; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !dereferenceable_or_null [[META3:![0-9]+]]
+; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !dereferenceable_or_null [[META4:![0-9]+]]
 ; MESA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; MESA-NEXT:    ret void
 ;
@@ -1079,14 +1100,14 @@ define amdgpu_kernel void @kern_align32_global_ptr(ptr addrspace(1) align 1024 %
 ; HSA-LABEL: @kern_align32_global_ptr(
 ; HSA-NEXT:    [[KERN_ALIGN32_GLOBAL_PTR_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; HSA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_ALIGN32_GLOBAL_PTR_KERNARG_SEGMENT]], i64 0
-; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !align [[META4:![0-9]+]]
+; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !align [[META5:![0-9]+]]
 ; HSA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; HSA-NEXT:    ret void
 ;
 ; MESA-LABEL: @kern_align32_global_ptr(
 ; MESA-NEXT:    [[KERN_ALIGN32_GLOBAL_PTR_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; MESA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_ALIGN32_GLOBAL_PTR_KERNARG_SEGMENT]], i64 36
-; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !align [[META4:![0-9]+]]
+; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !align [[META5:![0-9]+]]
 ; MESA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) undef, align 8
 ; MESA-NEXT:    ret void
 ;
@@ -1120,14 +1141,14 @@ define amdgpu_kernel void @kern_noundef_global_ptr(ptr addrspace(1) noundef %ptr
 ; HSA-LABEL: @kern_noundef_global_ptr(
 ; HSA-NEXT:    [[KERN_NOUNDEF_GLOBAL_PTR_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; HSA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_NOUNDEF_GLOBAL_PTR_KERNARG_SEGMENT]], i64 0
-; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]]
+; HSA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !noundef [[META1]]
 ; HSA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) null, align 8
 ; HSA-NEXT:    ret void
 ;
 ; MESA-LABEL: @kern_noundef_global_ptr(
 ; MESA-NEXT:    [[KERN_NOUNDEF_GLOBAL_PTR_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; MESA-NEXT:    [[PTR_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[KERN_NOUNDEF_GLOBAL_PTR_KERNARG_SEGMENT]], i64 36
-; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]]
+; MESA-NEXT:    [[PTR_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[PTR_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !noundef [[META1]]
 ; MESA-NEXT:    store volatile ptr addrspace(1) [[PTR_LOAD]], ptr addrspace(1) null, align 8
 ; MESA-NEXT:    ret void
 ;
@@ -1709,6 +1730,105 @@ define amdgpu_kernel void @byref_constant_i32_arg_offset0(ptr addrspace(4) byref
   ret void
 }
 
+define amdgpu_kernel void @noundef_f32(float noundef %arg0) {
+; HSA-LABEL: @noundef_f32(
+; HSA-NEXT:    [[NOUNDEF_F32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_F32_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load float, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    call void (...) @llvm.fake.use(float [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @noundef_f32(
+; MESA-NEXT:    [[NOUNDEF_F32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(260) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_F32_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load float, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    call void (...) @llvm.fake.use(float [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(float %arg0)
+  ret void
+}
+
+define amdgpu_kernel void @noundef_f16(half noundef %arg0) {
+; HSA-LABEL: @noundef_f16(
+; HSA-NEXT:    [[NOUNDEF_F16_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_F16_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = bitcast i16 [[TMP2]] to half
+; HSA-NEXT:    call void (...) @llvm.fake.use(half [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @noundef_f16(
+; MESA-NEXT:    [[NOUNDEF_F16_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(260) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_F16_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 4, !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = bitcast i16 [[TMP2]] to half
+; MESA-NEXT:    call void (...) @llvm.fake.use(half [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(half %arg0)
+  ret void
+}
+
+define amdgpu_kernel void @noundef_v2i32(<2 x i32> noundef %arg0) {
+; HSA-LABEL: @noundef_v2i32(
+; HSA-NEXT:    [[NOUNDEF_V2I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_V2I32_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load <2 x i32>, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    call void (...) @llvm.fake.use(<2 x i32> [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @noundef_v2i32(
+; MESA-NEXT:    [[NOUNDEF_V2I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_V2I32_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load <2 x i32>, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    call void (...) @llvm.fake.use(<2 x i32> [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(<2 x i32> %arg0)
+  ret void
+}
+
+define amdgpu_kernel void @noundef_p0(ptr noundef %arg0) {
+; HSA-LABEL: @noundef_p0(
+; HSA-NEXT:    [[NOUNDEF_P0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_P0_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load ptr, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    call void (...) @llvm.fake.use(ptr [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @noundef_p0(
+; MESA-NEXT:    [[NOUNDEF_P0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(264) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_P0_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load ptr, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    call void (...) @llvm.fake.use(ptr [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(ptr %arg0)
+  ret void
+}
+
+define amdgpu_kernel void @noundef_v2p0(<2 x ptr> noundef %arg0) {
+; HSA-LABEL: @noundef_v2p0(
+; HSA-NEXT:    [[NOUNDEF_V2P0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(272) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_V2P0_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load <2 x ptr>, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 16, !invariant.load [[META1]], !noundef [[META1]]
+; HSA-NEXT:    call void (...) @llvm.fake.use(<2 x ptr> [[ARG0_LOAD]])
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @noundef_v2p0(
+; MESA-NEXT:    [[NOUNDEF_V2P0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(272) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[NOUNDEF_V2P0_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load <2 x ptr>, ptr addrspace(4) [[ARG0_KERNARG_OFFSET]], align 4, !invariant.load [[META1]], !noundef [[META1]]
+; MESA-NEXT:    call void (...) @llvm.fake.use(<2 x ptr> [[ARG0_LOAD]])
+; MESA-NEXT:    ret void
+;
+  call void (...) @llvm.fake.use(<2 x ptr> %arg0)
+  ret void
+}
+
 attributes #0 = { nounwind "target-cpu"="kaveri" }
 attributes #1 = { nounwind "target-cpu"="kaveri" "amdgpu-implicitarg-num-bytes"="40" }
 attributes #2 = { nounwind "target-cpu"="tahiti" }
@@ -1717,25 +1837,29 @@ attributes #2 = { nounwind "target-cpu"="tahiti" }
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
-; HSA: attributes #[[ATTR0:[0-9]+]] = { nounwind "target-cpu"="kaveri" }
-; HSA: attributes #[[ATTR1:[0-9]+]] = { nounwind "amdgpu-implicitarg-num-bytes"="40" "target-cpu"="kaveri" }
-; HSA: attributes #[[ATTR2:[0-9]+]] = { nounwind "target-cpu"="tahiti" }
-; HSA: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; HSA: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; HSA: attributes #[[ATTR1:[0-9]+]] = { nounwind "target-cpu"="kaveri" }
+; HSA: attributes #[[ATTR2:[0-9]+]] = { nounwind "amdgpu-implicitarg-num-bytes"="40" "target-cpu"="kaveri" }
+; HSA: attributes #[[ATTR3:[0-9]+]] = { nounwind "target-cpu"="tahiti" }
+; HSA: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
-; MESA: attributes #[[ATTR0:[0-9]+]] = { nounwind "target-cpu"="kaveri" }
-; MESA: attributes #[[ATTR1:[0-9]+]] = { nounwind "amdgpu-implicitarg-num-bytes"="40" "target-cpu"="kaveri" }
-; MESA: attributes #[[ATTR2:[0-9]+]] = { nounwind "target-cpu"="tahiti" }
-; MESA: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; MESA: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; MESA: attributes #[[ATTR1:[0-9]+]] = { nounwind "target-cpu"="kaveri" }
+; MESA: attributes #[[ATTR2:[0-9]+]] = { nounwind "amdgpu-implicitarg-num-bytes"="40" "target-cpu"="kaveri" }
+; MESA: attributes #[[ATTR3:[0-9]+]] = { nounwind "target-cpu"="tahiti" }
+; MESA: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ; HSA: [[META1]] = !{}
-; HSA: [[META2]] = !{i64 42}
-; HSA: [[META3]] = !{i64 128}
-; HSA: [[META4]] = !{i64 1024}
+; HSA: [[RNG2]] = !{i32 0, i32 8}
+; HSA: [[META3]] = !{i64 42}
+; HSA: [[META4]] = !{i64 128}
+; HSA: [[META5]] = !{i64 1024}
 ;.
 ; MESA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ; MESA: [[META1]] = !{}
-; MESA: [[META2]] = !{i64 42}
-; MESA: [[META3]] = !{i64 128}
-; MESA: [[META4]] = !{i64 1024}
+; MESA: [[RNG2]] = !{i32 0, i32 8}
+; MESA: [[META3]] = !{i64 42}
+; MESA: [[META4]] = !{i64 128}
+; MESA: [[META5]] = !{i64 1024}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll
index 13932b39ac1a8..2a7553ae5d92b 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll
@@ -187,6 +187,7 @@ define amdgpu_kernel void @k01() {
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GCN-NEXT:    s_endpgm
+
   call void @f0()
   call void @f1()
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index d4110850f3206..7646197f13175 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefix=GFX7 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX9,GFX940 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+xnack < %s | FileCheck -check-prefixes=GFX940 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10_1 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10_3 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
@@ -2249,5 +2249,3 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; are reserved at the end for xnack + vcc).
 attributes #0 = { nounwind alignstack=64 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
 attributes #1 = { nounwind alignstack=16 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mul.i16.ll b/llvm/test/CodeGen/AMDGPU/mul.i16.ll
index 98c3867b11f10..2ba719c4138c6 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.i16.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89 %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11-FAKE16 %s
 
 ; GCN-LABEL: {{^}}v_mul_i16:
 ; SI: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
@@ -8,6 +10,9 @@
 ; SI: v_mul_u32_u24
 
 ; GFX89: v_mul_lo_u16_e32 v0, v0, v1
+
+; GFX11-TRUE16: v_mul_lo_u16 v0.l, v0.l, v0.h
+; GFX11-FAKE16: v_mul_lo_u16 v0, v0, v1
 define i16 @v_mul_i16(i16 %a, i16 %b) {
   %r.val = mul i16 %a, %b
   ret i16 %r.val
diff --git a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
index 65c44768d3d88..6c62f3f225cd9 100644
--- a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -27,10 +27,10 @@ define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; OPT-NEXT:    [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP5]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP7:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP6]])
-; OPT-NEXT:    [[TMP8]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN2]])
 ; OPT-NEXT:    br i1 [[TMP7]], label [[FLOW1]], label [[LOOP]]
 ; OPT:       Flow1:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
+; OPT-NEXT:    [[TMP8]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN2]])
 ; OPT-NEXT:    [[TMP9:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP8]])
 ; OPT-NEXT:    br i1 [[TMP9]], label [[IF:%.*]], label [[LOOP_OUTER]]
 ; OPT:       IF:
@@ -57,33 +57,37 @@ define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
 ; GCN-NEXT:  .LBB0_2: ; %LOOP.outer
 ; GCN-NEXT:    ; =>This Loop Header: Depth=1
 ; GCN-NEXT:    ; Child Loop BB0_4 Depth 2
-; GCN-NEXT:    ; implicit-def: $sgpr6_sgpr7
 ; GCN-NEXT:    ; implicit-def: $sgpr2_sgpr3
+; GCN-NEXT:    ; implicit-def: $sgpr8_sgpr9
+; GCN-NEXT:    ; implicit-def: $sgpr6_sgpr7
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:    s_branch .LBB0_4
 ; GCN-NEXT:  .LBB0_3: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
-; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT:    s_and_b64 s[8:9], exec, s[6:7]
-; GCN-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; GCN-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GCN-NEXT:    s_and_b64 s[10:11], exec, s[8:9]
+; GCN-NEXT:    s_or_b64 s[4:5], s[10:11], s[4:5]
+; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; GCN-NEXT:    s_and_b64 s[10:11], s[6:7], exec
+; GCN-NEXT:    s_or_b64 s[2:3], s[2:3], s[10:11]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_1
 ; GCN-NEXT:  .LBB0_4: ; %LOOP
 ; GCN-NEXT:    ; Parent Loop BB0_2 Depth=1
 ; GCN-NEXT:    ; => This Inner Loop Header: Depth=2
 ; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v4
-; GCN-NEXT:    s_or_b64 s[2:3], s[2:3], exec
 ; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], exec
-; GCN-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], exec
+; GCN-NEXT:    s_and_saveexec_b64 s[10:11], vcc
 ; GCN-NEXT:    s_cbranch_execz .LBB0_3
 ; GCN-NEXT:  ; %bb.5: ; %ENDIF
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
-; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v0
 ; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
-; GCN-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v0
+; GCN-NEXT:    s_andn2_b64 s[8:9], s[8:9], exec
+; GCN-NEXT:    s_and_b64 s[12:13], vcc, exec
+; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
 ; GCN-NEXT:    s_branch .LBB0_3
 ; GCN-NEXT:  .LBB0_6: ; %IF
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
index a52d9ff526c2a..bd6ef9e088b12 100644
--- a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
+++ b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow %s | FileCheck -check-prefix=IR %s
 ; RUN: llc -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
@@ -46,51 +46,52 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) nocap
 ; GCN-NEXT:    s_cbranch_vccz .LBB0_6
 ; GCN-NEXT:  .LBB0_7: ; %DummyReturnBlock
 ; GCN-NEXT:    s_endpgm
-; IR-LABEL: @reduced_nested_loop_conditions(
-; IR-NEXT:  bb:
+; IR-LABEL: define amdgpu_kernel void @reduced_nested_loop_conditions(
+; IR-SAME: ptr addrspace(3) nocapture [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-NEXT:  [[BB:.*]]:
 ; IR-NEXT:    [[MY_TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR4:[0-9]+]]
-; IR-NEXT:    [[MY_TMP1:%.*]] = getelementptr inbounds i64, ptr addrspace(3) [[ARG:%.*]], i32 [[MY_TMP]]
+; IR-NEXT:    [[MY_TMP1:%.*]] = getelementptr inbounds i64, ptr addrspace(3) [[ARG]], i32 [[MY_TMP]]
 ; IR-NEXT:    [[MY_TMP2:%.*]] = load volatile i64, ptr addrspace(3) [[MY_TMP1]], align 8
-; IR-NEXT:    br label [[BB5:%.*]]
-; IR:       bb3:
-; IR-NEXT:    br i1 true, label [[BB4:%.*]], label [[BB13:%.*]]
-; IR:       bb4:
-; IR-NEXT:    br label [[FLOW:%.*]]
-; IR:       bb5:
-; IR-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], [[BB10:%.*]] ], [ 0, [[BB:%.*]] ]
-; IR-NEXT:    [[MY_TMP6:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP5:%.*]], [[BB10]] ]
+; IR-NEXT:    br label %[[BB5:.*]]
+; IR:       [[BB3:.*]]:
+; IR-NEXT:    br i1 true, label %[[BB4:.*]], label %[[BB13:.*]]
+; IR:       [[BB4]]:
+; IR-NEXT:    br label %[[FLOW:.*]]
+; IR:       [[BB5]]:
+; IR-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], %[[BB10:.*]] ], [ 0, %[[BB]] ]
+; IR-NEXT:    [[MY_TMP6:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP5:%.*]], %[[BB10]] ]
 ; IR-NEXT:    [[MY_TMP7:%.*]] = icmp eq i32 [[MY_TMP6]], 1
 ; IR-NEXT:    [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[MY_TMP7]])
 ; IR-NEXT:    [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
 ; IR-NEXT:    [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
-; IR-NEXT:    br i1 [[TMP1]], label [[BB8:%.*]], label [[FLOW]]
-; IR:       bb8:
-; IR-NEXT:    br label [[BB13]]
-; IR:       bb9:
-; IR-NEXT:    br i1 false, label [[BB3:%.*]], label [[BB9:%.*]]
-; IR:       bb10:
+; IR-NEXT:    br i1 [[TMP1]], label %[[BB8:.*]], label %[[FLOW]]
+; IR:       [[BB8]]:
+; IR-NEXT:    br label %[[BB13]]
+; IR:       [[BB9:.*]]:
+; IR-NEXT:    br i1 false, label %[[BB3]], label %[[BB9]]
+; IR:       [[BB10]]:
 ; IR-NEXT:    [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP6]])
-; IR-NEXT:    br i1 [[TMP3]], label [[BB23:%.*]], label [[BB5]]
-; IR:       Flow:
-; IR-NEXT:    [[TMP4:%.*]] = phi i1 [ [[MY_TMP22:%.*]], [[BB4]] ], [ true, [[BB5]] ]
-; IR-NEXT:    [[TMP5]] = phi i32 [ [[MY_TMP21:%.*]], [[BB4]] ], [ undef, [[BB5]] ]
+; IR-NEXT:    br i1 [[TMP3]], label %[[BB23:.*]], label %[[BB5]]
+; IR:       [[FLOW]]:
+; IR-NEXT:    [[TMP4:%.*]] = phi i1 [ [[MY_TMP22:%.*]], %[[BB4]] ], [ true, %[[BB5]] ]
+; IR-NEXT:    [[TMP5]] = phi i32 [ [[MY_TMP21:%.*]], %[[BB4]] ], [ undef, %[[BB5]] ]
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; IR-NEXT:    [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN]])
-; IR-NEXT:    br label [[BB10]]
-; IR:       bb13:
-; IR-NEXT:    [[MY_TMP14:%.*]] = phi i1 [ [[MY_TMP22]], [[BB3]] ], [ true, [[BB8]] ]
+; IR-NEXT:    br label %[[BB10]]
+; IR:       [[BB13]]:
+; IR-NEXT:    [[MY_TMP14:%.*]] = phi i1 [ [[MY_TMP22]], %[[BB3]] ], [ true, %[[BB8]] ]
 ; IR-NEXT:    [[MY_TMP15:%.*]] = bitcast i64 [[MY_TMP2]] to <2 x i32>
-; IR-NEXT:    br i1 [[MY_TMP14]], label [[BB16:%.*]], label [[BB20:%.*]]
-; IR:       bb16:
+; IR-NEXT:    br i1 [[MY_TMP14]], label %[[BB16:.*]], label %[[BB20:.*]]
+; IR:       [[BB16]]:
 ; IR-NEXT:    [[MY_TMP17:%.*]] = extractelement <2 x i32> [[MY_TMP15]], i64 1
 ; IR-NEXT:    [[MY_TMP18:%.*]] = getelementptr inbounds i32, ptr addrspace(3) undef, i32 [[MY_TMP17]]
 ; IR-NEXT:    [[MY_TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[MY_TMP18]], align 4
-; IR-NEXT:    br label [[BB20]]
-; IR:       bb20:
-; IR-NEXT:    [[MY_TMP21]] = phi i32 [ [[MY_TMP19]], [[BB16]] ], [ 0, [[BB13]] ]
-; IR-NEXT:    [[MY_TMP22]] = phi i1 [ false, [[BB16]] ], [ [[MY_TMP14]], [[BB13]] ]
-; IR-NEXT:    br label [[BB9]]
-; IR:       bb23:
+; IR-NEXT:    br label %[[BB20]]
+; IR:       [[BB20]]:
+; IR-NEXT:    [[MY_TMP21]] = phi i32 [ [[MY_TMP19]], %[[BB16]] ], [ 0, %[[BB13]] ]
+; IR-NEXT:    [[MY_TMP22]] = phi i1 [ false, %[[BB16]] ], [ [[MY_TMP14]], %[[BB13]] ]
+; IR-NEXT:    br label %[[BB9]]
+; IR:       [[BB23]]:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
 ; IR-NEXT:    ret void
 bb:
@@ -188,66 +189,67 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) nocapture %ar
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_endpgm
-; IR-LABEL: @nested_loop_conditions(
-; IR-NEXT:  bb:
+; IR-LABEL: define amdgpu_kernel void @nested_loop_conditions(
+; IR-SAME: ptr addrspace(1) nocapture [[ARG:%.*]]) #[[ATTR0]] {
+; IR-NEXT:  [[BB:.*]]:
 ; IR-NEXT:    [[MY_TMP1134:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    [[MY_TMP1235:%.*]] = icmp slt i32 [[MY_TMP1134]], 9
-; IR-NEXT:    br i1 [[MY_TMP1235]], label [[BB14_LR_PH:%.*]], label [[FLOW:%.*]]
-; IR:       bb14.lr.ph:
+; IR-NEXT:    br i1 [[MY_TMP1235]], label %[[BB14_LR_PH:.*]], label %[[FLOW:.*]]
+; IR:       [[BB14_LR_PH]]:
 ; IR-NEXT:    [[MY_TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR4]]
 ; IR-NEXT:    [[MY_TMP1:%.*]] = zext i32 [[MY_TMP]] to i64
-; IR-NEXT:    [[MY_TMP2:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[ARG:%.*]], i64 [[MY_TMP1]]
+; IR-NEXT:    [[MY_TMP2:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[ARG]], i64 [[MY_TMP1]]
 ; IR-NEXT:    [[MY_TMP3:%.*]] = load i64, ptr addrspace(1) [[MY_TMP2]], align 16
 ; IR-NEXT:    [[MY_TMP932:%.*]] = load <4 x i32>, ptr addrspace(1) undef, align 16
 ; IR-NEXT:    [[MY_TMP1033:%.*]] = extractelement <4 x i32> [[MY_TMP932]], i64 0
-; IR-NEXT:    br label [[BB14:%.*]]
-; IR:       Flow3:
+; IR-NEXT:    br label %[[BB14:.*]]
+; IR:       [[FLOW3:.*]]:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP20:%.*]])
 ; IR-NEXT:    [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP14:%.*]])
 ; IR-NEXT:    [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
 ; IR-NEXT:    [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
-; IR-NEXT:    br i1 [[TMP1]], label [[BB4_BB13_CRIT_EDGE:%.*]], label [[FLOW4:%.*]]
-; IR:       bb4.bb13_crit_edge:
-; IR-NEXT:    br label [[FLOW4]]
-; IR:       Flow4:
-; IR-NEXT:    [[TMP3:%.*]] = phi i1 [ true, [[BB4_BB13_CRIT_EDGE]] ], [ false, [[FLOW3:%.*]] ]
+; IR-NEXT:    br i1 [[TMP1]], label %[[BB4_BB13_CRIT_EDGE:.*]], label %[[FLOW4:.*]]
+; IR:       [[BB4_BB13_CRIT_EDGE]]:
+; IR-NEXT:    br label %[[FLOW4]]
+; IR:       [[FLOW4]]:
+; IR-NEXT:    [[TMP3:%.*]] = phi i1 [ true, %[[BB4_BB13_CRIT_EDGE]] ], [ false, %[[FLOW3]] ]
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
-; IR-NEXT:    br label [[FLOW]]
-; IR:       bb13:
-; IR-NEXT:    br label [[BB31:%.*]]
-; IR:       Flow:
-; IR-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP3]], [[FLOW4]] ], [ true, [[BB:%.*]] ]
+; IR-NEXT:    br label %[[FLOW]]
+; IR:       [[BB13:.*]]:
+; IR-NEXT:    br label %[[BB31:.*]]
+; IR:       [[FLOW]]:
+; IR-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP3]], %[[FLOW4]] ], [ true, %[[BB]] ]
 ; IR-NEXT:    [[TMP5:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP4]])
 ; IR-NEXT:    [[TMP6:%.*]] = extractvalue { i1, i64 } [[TMP5]], 0
 ; IR-NEXT:    [[TMP7:%.*]] = extractvalue { i1, i64 } [[TMP5]], 1
-; IR-NEXT:    br i1 [[TMP6]], label [[BB13:%.*]], label [[BB31]]
-; IR:       bb14:
-; IR-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP16:%.*]], [[FLOW1:%.*]] ], [ 0, [[BB14_LR_PH]] ]
-; IR-NEXT:    [[MY_TMP1037:%.*]] = phi i32 [ [[MY_TMP1033]], [[BB14_LR_PH]] ], [ [[TMP12:%.*]], [[FLOW1]] ]
-; IR-NEXT:    [[MY_TMP936:%.*]] = phi <4 x i32> [ [[MY_TMP932]], [[BB14_LR_PH]] ], [ [[TMP11:%.*]], [[FLOW1]] ]
+; IR-NEXT:    br i1 [[TMP6]], label %[[BB13]], label %[[BB31]]
+; IR:       [[BB14]]:
+; IR-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP16:%.*]], %[[FLOW1:.*]] ], [ 0, %[[BB14_LR_PH]] ]
+; IR-NEXT:    [[MY_TMP1037:%.*]] = phi i32 [ [[MY_TMP1033]], %[[BB14_LR_PH]] ], [ [[TMP12:%.*]], %[[FLOW1]] ]
+; IR-NEXT:    [[MY_TMP936:%.*]] = phi <4 x i32> [ [[MY_TMP932]], %[[BB14_LR_PH]] ], [ [[TMP11:%.*]], %[[FLOW1]] ]
 ; IR-NEXT:    [[MY_TMP15:%.*]] = icmp eq i32 [[MY_TMP1037]], 1
 ; IR-NEXT:    [[TMP8:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[MY_TMP15]])
 ; IR-NEXT:    [[TMP9:%.*]] = extractvalue { i1, i64 } [[TMP8]], 0
 ; IR-NEXT:    [[TMP10:%.*]] = extractvalue { i1, i64 } [[TMP8]], 1
-; IR-NEXT:    br i1 [[TMP9]], label [[BB16:%.*]], label [[FLOW1]]
-; IR:       bb16:
+; IR-NEXT:    br i1 [[TMP9]], label %[[BB16:.*]], label %[[FLOW1]]
+; IR:       [[BB16]]:
 ; IR-NEXT:    [[MY_TMP17:%.*]] = bitcast i64 [[MY_TMP3]] to <2 x i32>
-; IR-NEXT:    br label [[BB18:%.*]]
-; IR:       Flow1:
-; IR-NEXT:    [[TMP11]] = phi <4 x i32> [ [[MY_TMP9:%.*]], [[BB21:%.*]] ], [ undef, [[BB14]] ]
-; IR-NEXT:    [[TMP12]] = phi i32 [ [[MY_TMP10:%.*]], [[BB21]] ], [ undef, [[BB14]] ]
-; IR-NEXT:    [[TMP13:%.*]] = phi i1 [ [[MY_TMP12:%.*]], [[BB21]] ], [ true, [[BB14]] ]
-; IR-NEXT:    [[TMP14]] = phi i1 [ [[MY_TMP12]], [[BB21]] ], [ false, [[BB14]] ]
-; IR-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[BB21]] ], [ true, [[BB14]] ]
+; IR-NEXT:    br label %[[BB18:.*]]
+; IR:       [[FLOW1]]:
+; IR-NEXT:    [[TMP11]] = phi <4 x i32> [ [[MY_TMP9:%.*]], %[[BB21:.*]] ], [ undef, %[[BB14]] ]
+; IR-NEXT:    [[TMP12]] = phi i32 [ [[MY_TMP10:%.*]], %[[BB21]] ], [ undef, %[[BB14]] ]
+; IR-NEXT:    [[TMP13:%.*]] = phi i1 [ [[MY_TMP12:%.*]], %[[BB21]] ], [ true, %[[BB14]] ]
+; IR-NEXT:    [[TMP14]] = phi i1 [ [[MY_TMP12]], %[[BB21]] ], [ false, %[[BB14]] ]
+; IR-NEXT:    [[TMP15:%.*]] = phi i1 [ false, %[[BB21]] ], [ true, %[[BB14]] ]
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP10]])
 ; IR-NEXT:    [[TMP16]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP13]], i64 [[PHI_BROKEN]])
 ; IR-NEXT:    [[TMP17:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP16]])
-; IR-NEXT:    br i1 [[TMP17]], label [[FLOW2:%.*]], label [[BB14]]
-; IR:       bb18:
+; IR-NEXT:    br i1 [[TMP17]], label %[[FLOW2:.*]], label %[[BB14]]
+; IR:       [[BB18]]:
 ; IR-NEXT:    [[MY_TMP19:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    [[MY_TMP20:%.*]] = icmp slt i32 [[MY_TMP19]], 9
-; IR-NEXT:    br i1 [[MY_TMP20]], label [[BB21]], label [[BB18]]
-; IR:       bb21:
+; IR-NEXT:    br i1 [[MY_TMP20]], label %[[BB21]], label %[[BB18]]
+; IR:       [[BB21]]:
 ; IR-NEXT:    [[MY_TMP22:%.*]] = extractelement <2 x i32> [[MY_TMP17]], i64 1
 ; IR-NEXT:    [[MY_TMP23:%.*]] = lshr i32 [[MY_TMP22]], 16
 ; IR-NEXT:    [[MY_TMP24:%.*]] = select i1 undef, i32 undef, i32 [[MY_TMP23]]
@@ -263,16 +265,16 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) nocapture %ar
 ; IR-NEXT:    [[MY_TMP10]] = extractelement <4 x i32> [[MY_TMP9]], i64 0
 ; IR-NEXT:    [[MY_TMP11:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    [[MY_TMP12]] = icmp sge i32 [[MY_TMP11]], 9
-; IR-NEXT:    br label [[FLOW1]]
-; IR:       Flow2:
+; IR-NEXT:    br label %[[FLOW1]]
+; IR:       [[FLOW2]]:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP16]])
 ; IR-NEXT:    [[TMP18:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP15]])
 ; IR-NEXT:    [[TMP19:%.*]] = extractvalue { i1, i64 } [[TMP18]], 0
 ; IR-NEXT:    [[TMP20]] = extractvalue { i1, i64 } [[TMP18]], 1
-; IR-NEXT:    br i1 [[TMP19]], label [[BB31_LOOPEXIT:%.*]], label [[FLOW3]]
-; IR:       bb31.loopexit:
-; IR-NEXT:    br label [[FLOW3]]
-; IR:       bb31:
+; IR-NEXT:    br i1 [[TMP19]], label %[[BB31_LOOPEXIT:.*]], label %[[FLOW3]]
+; IR:       [[BB31_LOOPEXIT]]:
+; IR-NEXT:    br label %[[FLOW3]]
+; IR:       [[BB31]]:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
 ; IR-NEXT:    store volatile i32 0, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    ret void
diff --git a/llvm/test/CodeGen/AMDGPU/opt-vgpr-live-range-verifier-error.mir b/llvm/test/CodeGen/AMDGPU/opt-vgpr-live-range-verifier-error.mir
index 0ef3b4e54cf07..9295562529438 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-vgpr-live-range-verifier-error.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-vgpr-live-range-verifier-error.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs -run-pass=si-opt-vgpr-liverange -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=si-opt-vgpr-liverange -o - %s | FileCheck %s
 #
 # This is a very rare case which comes from llvm-reduce. The SI_IF/SI_ELSE usage is quite different from normal.
 #
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 37d0309caac0a..b40d35dbd8ac6 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -20,8 +20,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
   ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %16:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) undef`, addrspace 1)
-  ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
-  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %18:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) undef`, addrspace 1)
+  ; REGALLOC-GFX908-NEXT:   [[COPY3:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
+  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %18:vreg_64, [[COPY3]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) undef`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   S_ENDPGM 0
   ;
   ; PEI-GFX908-LABEL: name: partial_copy
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 5b8acc31b22cf..0c6d8dce193da 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -599,10 +599,8 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
 ; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
 ; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
 ; GFX940-NEXT:  ; %bb.0:
-; GFX940-NEXT:    s_load_dword s0, s[4:5], 0x1c
+; GFX940-NEXT:    s_lshr_b32 s0, s15, 16
 ; GFX940-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    s_lshr_b32 s0, s0, 16
 ; GFX940-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX940-NEXT:    global_store_dword v0, v1, s[8:9] sc0 sc1
 ; GFX940-NEXT:    s_endpgm
@@ -626,4 +624,74 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
   ret void
 }
 
+; Check for consistency between isel and earlier passes preload SGPR accounting with max preload SGPRs.
+
+define amdgpu_kernel void @preload_block_max_user_sgprs(ptr addrspace(1) inreg %out, i192 inreg %t0, i32 inreg %t1) #0 {
+; GFX940-LABEL: preload_block_max_user_sgprs:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s12
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: preload_block_max_user_sgprs:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dword s0, s[4:5], 0x28
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %load = load i32, ptr addrspace(4) %imp_arg_ptr
+  store i32 %load, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(ptr addrspace(1) inreg %out) #0 {
+; GFX940-LABEL: preload_block_count_z_workgroup_size_z_remainder_z:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX940-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s0
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: preload_block_count_z_workgroup_size_z_remainder_z:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
+; GFX90a-NEXT:    s_and_b32 s1, s12, 0xffff
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s1
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s0
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %gep0 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 8
+  %gep1 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 16
+  %gep2 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 22
+  %load0 = load i32, ptr addrspace(4) %gep0
+  %load1 = load i16, ptr addrspace(4) %gep1
+  %load2 = load i16, ptr addrspace(4) %gep2
+  %conv1 = zext i16 %load1 to i32
+  %conv2 = zext i16 %load2 to i32
+  %ins.0 =  insertelement <3 x i32> poison, i32 %load0, i32 0
+  %ins.1 =  insertelement <3 x i32> %ins.0, i32 %conv1, i32 1
+  %ins.2 =  insertelement <3 x i32> %ins.1, i32 %conv2, i32 2
+  store <3 x i32> %ins.2, ptr addrspace(1) %out
+  ret void
+}
+
 attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll
index d070dc3b770f8..51e0eb6d49794 100644
--- a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll
@@ -153,7 +153,7 @@ define internal void @merge_cycle_1() #3 {
 
 define amdgpu_kernel void @kernel_64_256() #7 {
 ; CHECK-LABEL: define {{[^@]+}}@kernel_64_256
-; CHECK-SAME: () #[[ATTR6]] {
+; CHECK-SAME: () #[[ATTR8:[0-9]+]] {
 ; CHECK-NEXT:    call void @merge_cycle_0()
 ; CHECK-NEXT:    call void @default_captured_address()
 ; CHECK-NEXT:    call void @externally_visible_default()
@@ -203,13 +203,13 @@ attributes #5 = { "amdgpu-flat-work-group-size"="128,512" }
 attributes #6 = { "amdgpu-flat-work-group-size"="512,512" }
 attributes #7 = { "amdgpu-flat-work-group-size"="64,256" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll
index f62f1d57aec8e..eb4cf5c063d10 100644
--- a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll
+++ b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll
@@ -210,7 +210,7 @@ define void @externally_visible_default() {
 ; 1,10 -> 3,8
 define internal i32 @bitcasted_function() {
 ; CHECK-LABEL: define internal i32 @bitcasted_function
-; CHECK-SAME: () #[[ATTR8]] {
+; CHECK-SAME: () #[[ATTR10:[0-9]+]] {
 ; CHECK-NEXT:    ret i32 0
 ;
   ret i32 0
@@ -399,26 +399,26 @@ attributes #17 = { "amdgpu-waves-per-eu"="5,8" }
 attributes #18 = { "amdgpu-waves-per-eu"="9,10" }
 attributes #19 = { "amdgpu-waves-per-eu"="8,9" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index 6b097bd71c9f1..ba428df273db5 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -3,20 +3,32 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
 
 define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
-; GFX12-LABEL: v_s_exp_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
-; GFX12-NEXT:    s_cselect_b32 s1, 0x42800000, 0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-NEXT:    s_add_f32 s0, s0, s1
-; GFX12-NEXT:    s_cselect_b32 s1, 0x1f800000, 1.0
-; GFX12-NEXT:    v_s_exp_f32 s0, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-NEXT:    s_mul_f32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_mov_b32_e32 v0, s0
-; GFX12-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_exp_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42800000, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_add_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
+; GFX12-SDAG-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, s0, s1
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: v_s_exp_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x42800000, 0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    s_add_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x1f800000, 1.0
+; GFX12-GISEL-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %result = call float @llvm.exp2.f32(float %src)
   ret float %result
 }
@@ -55,20 +67,38 @@ define amdgpu_cs half @v_s_amdgcn_exp_f16(half inreg %src) {
 }
 
 define amdgpu_cs float @v_s_log_f32(float inreg %src) {
-; GFX12-LABEL: v_s_log_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_cmp_lt_f32 s0, 0x800000
-; GFX12-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-NEXT:    s_mul_f32 s0, s0, s1
-; GFX12-NEXT:    s_cselect_b32 s1, 0x42000000, 0
-; GFX12-NEXT:    v_s_log_f32 s0, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-NEXT:    s_sub_f32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_mov_b32_e32 v0, s0
-; GFX12-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_log_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s0, 0x800000
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, s0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
+; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
+; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: v_s_log_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_cmp_lt_f32 s0, 0x800000
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x42000000, 0
+; GFX12-GISEL-NEXT:    v_s_log_f32 s0, s0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    s_sub_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %result = call float @llvm.log2.f32(float %src)
   ret float %result
 }
@@ -271,22 +301,41 @@ define amdgpu_cs half @v_amdgcn_sqrt_f16(half inreg %src)  {
 }
 
 define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
-; GFX12-LABEL: srcmods_abs_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_bitset0_b32 s0, 31
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT:    s_cmp_lt_f32 s0, 0x800000
-; GFX12-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
-; GFX12-NEXT:    s_mul_f32 s0, s0, s1
-; GFX12-NEXT:    s_cselect_b32 s1, 0x42000000, 0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX12-NEXT:    v_s_log_f32 s0, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_sub_f32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT:    v_mov_b32_e32 v0, s0
-; GFX12-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: srcmods_abs_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_and_b32 s1, s0, 0x7fffffff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s1, 0x800000
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, |s0|, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
+; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: srcmods_abs_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_bitset0_b32 s0, 31
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT:    s_cmp_lt_f32 s0, 0x800000
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
+; GFX12-GISEL-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x42000000, 0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX12-GISEL-NEXT:    v_s_log_f32 s0, s0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_sub_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %abs = call float @llvm.fabs.f32(float %src)
   %result = call float @llvm.log2.f32(float %abs)
   ret float %result
@@ -295,18 +344,20 @@ define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
 define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: srcmods_neg_f32:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    s_xor_b32 s1, s0, 0x80000000
 ; GFX12-SDAG-NEXT:    s_cmp_gt_f32 s0, 0x80800000
-; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x4f800000, 1.0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT:    s_mul_f32 s0, s1, s0
-; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42000000, 0
-; GFX12-SDAG-NEXT:    v_s_log_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, -s0, v0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT:    s_sub_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_and_b32 s0, s1, exec_lo
+; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
+; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: srcmods_neg_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll
index c1d647c5d3b90..9758afa3b4991 100644
--- a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll
+++ b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll
@@ -19,5 +19,5 @@ define void @hoge()  {
   ret void
 }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/register-coalescer--set-undef-full-reg-use-implicit-def-erase-issue109249.mir b/llvm/test/CodeGen/AMDGPU/register-coalescer--set-undef-full-reg-use-implicit-def-erase-issue109249.mir
new file mode 100644
index 0000000000000..3f5df7a7b868a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/register-coalescer--set-undef-full-reg-use-implicit-def-erase-issue109249.mir
@@ -0,0 +1,43 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+
+# Make sure that the undef flag is set on %0 after the IMPLICIT_DEF is
+# deleted when coalescing %0 with %1
+
+---
+name:            test
+tracksRegLiveness: true
+machineFunctionInfo:
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test
+    ; CHECK: [[S_BUFFER_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM undef %0, 36, 0 :: (dereferenceable invariant load (s64))
+    ; CHECK-NEXT: undef [[S_ADD_U32_:%[0-9]+]].sub1:sgpr_128 = S_ADD_U32 [[S_BUFFER_LOAD_DWORDX2_IMM]].sub0, 32, implicit-def dead $scc
+    ; CHECK-NEXT: SI_RETURN implicit [[S_ADD_U32_]].sub1
+    %0:sgpr_128 = IMPLICIT_DEF
+    %1:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %0, 36, 0 :: (dereferenceable invariant load (s64))
+    %2:sreg_32 = S_ADD_U32 %1.sub0, 32, implicit-def dead $scc
+    %0.sub1:sgpr_128 = COPY killed %2
+    SI_RETURN implicit %0.sub1
+
+...
+
+---
+name:            test_w_undef_dead
+tracksRegLiveness: true
+machineFunctionInfo:
+  stackPtrOffsetReg: '$sgpr32'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_w_undef_dead
+    ; CHECK: dead [[S_BUFFER_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM undef %0, 36, 0 :: (dereferenceable invariant load (s64))
+    ; CHECK-NEXT: undef [[S_ADD_U32_:%[0-9]+]].sub1:sgpr_128 = S_ADD_U32 undef [[S_BUFFER_LOAD_DWORDX2_IMM]].sub0, 32, implicit-def dead $scc
+    ; CHECK-NEXT: SI_RETURN implicit [[S_ADD_U32_]].sub1
+    %0:sgpr_128 = IMPLICIT_DEF
+    dead %1:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %0, 36, 0 :: (dereferenceable invariant load (s64))
+    %2:sreg_32 = S_ADD_U32 undef %1.sub0, 32, implicit-def dead $scc
+    %0.sub1:sgpr_128 = COPY killed %2
+    SI_RETURN implicit %0.sub1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll
index 4420833029d46..8792e60bb0ca1 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll
+++ b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll
@@ -191,11 +191,11 @@ define amdgpu_kernel void @kernel_lds_recursion() {
 !1 = !{i32 1, !"amdhsa_code_object_version", i32 400}
 
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR3]] = { "amdgpu-lds-size"="4" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR5:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) }
 ; CHECK: attributes #[[ATTR6:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll
index 8c584a1890c9d..98b701ab7f9d9 100644
--- a/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll
@@ -339,7 +339,7 @@ define amdgpu_kernel void @all_local_size(ptr addrspace(1) nocapture readnone %o
 ; TODO: Should be able to handle this, but not much reason to.
 ; CHECK-LABEL: @partial_load_group_size_x(
 ; CHECK-NEXT: %dispatch.ptr = tail call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, ptr addrspace(4) %dispatch.ptr, i64 4
+; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds nuw i8, ptr addrspace(4) %dispatch.ptr, i64 4
 ; CHECK-NEXT: %group.size.x.lo = load i8, ptr addrspace(4) %gep.group.size.x, align 4
 ; CHECK-NEXT: store i8 %group.size.x.lo, ptr addrspace(1) %out, align 1
 define amdgpu_kernel void @partial_load_group_size_x(ptr addrspace(1) %out) #0 !reqd_work_group_size !0 {
@@ -352,7 +352,7 @@ define amdgpu_kernel void @partial_load_group_size_x(ptr addrspace(1) %out) #0 !
 
 ; CHECK-LABEL: @partial_load_group_size_x_explicit_callsite_align(
 ; CHECK-NEXT: %dispatch.ptr = tail call align 2 ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, ptr addrspace(4) %dispatch.ptr, i64 4
+; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds nuw i8, ptr addrspace(4) %dispatch.ptr, i64 4
 ; CHECK-NEXT: %group.size.x.lo = load i8, ptr addrspace(4) %gep.group.size.x, align 2
 ; CHECK-NEXT: store i8 %group.size.x.lo, ptr addrspace(1) %out, align 1
 define amdgpu_kernel void @partial_load_group_size_x_explicit_callsite_align(ptr addrspace(1) %out) #0 !reqd_work_group_size !0 {
@@ -394,7 +394,7 @@ define amdgpu_kernel void @load_group_size_x_y_multiple_dispatch_ptr(ptr addrspa
 
 ; CHECK-LABEL: @use_local_size_x_uniform_work_group_size(
 ; CHECK-NEXT: %dispatch.ptr = tail call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, ptr addrspace(4) %dispatch.ptr, i64 4
+; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds nuw i8, ptr addrspace(4) %dispatch.ptr, i64 4
 ; CHECK-NEXT: %group.size.x = load i16, ptr addrspace(4) %gep.group.size.x, align 4
 ; CHECK: %group.size.x.zext = zext i16 %group.size.x to i32
 ; CHECK: store i64 %zext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
index f3c9a5c471aca..f4b947ade8dac 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
@@ -2006,14 +2006,12 @@ define float @v_rsq_f32(float %val) {
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4b800000
 ; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
@@ -2296,10 +2294,9 @@ define float @v_rsq_f32_contractable_user(float %val0, float %val1) {
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x4b800000
 ; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
 ; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
@@ -2331,10 +2328,9 @@ define float @v_rsq_f32_contractable_user_missing_contract0(float %val0, float %
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x4b800000
 ; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
 ; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
@@ -2366,14 +2362,12 @@ define float @v_rsq_f32_contractable_user_missing_contract1(float %val0, float %
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x4b800000
 ; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
 ; GCN-IEEE-SAFE-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
@@ -2416,14 +2410,12 @@ define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4b800000
 ; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v1, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
 ; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
diff --git a/llvm/test/CodeGen/AMDGPU/saddsat.ll b/llvm/test/CodeGen/AMDGPU/saddsat.ll
index 9c5214338c54a..a034cf6941c26 100644
--- a/llvm/test/CodeGen/AMDGPU/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddsat.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
 
 define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX6-LABEL: v_saddsat_i8:
@@ -34,14 +35,32 @@ define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_saddsat_i8:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_lshlrev_b16 v1, 8, v1
-; GFX10PLUS-NEXT:    v_lshlrev_b16 v0, 8, v0
-; GFX10PLUS-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    v_ashrrev_i16 v0, 8, v0
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_saddsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
+; GFX10-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_saddsat_i8:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_i16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_saddsat_i8:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
 }
@@ -76,11 +95,24 @@ define i16 @v_saddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_saddsat_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_saddsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_saddsat_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_i16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_saddsat_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir b/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir
index 4b6e204ecf957..c2cd4653bc9bf 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -amdgpu-enable-max-ilp-scheduling-strategy -verify-machineinstrs -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -amdgpu-sched-strategy=max-ilp -verify-machineinstrs -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
 
 ---
 name:            max-ilp-liveness-tracking
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-ilp.ll b/llvm/test/CodeGen/AMDGPU/schedule-ilp.ll
index 11602b1d353f9..350ff94373a72 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-ilp.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-ilp.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -misched=gcn-iterative-ilp -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -misched=gcn-max-ilp -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-enable-max-ilp-scheduling-strategy -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-sched-strategy=max-ilp -verify-machineinstrs < %s | FileCheck %s
 
 ; CHECK: NumVgprs: {{[0-9][0-9][0-9]$}}
 
diff --git a/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir b/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir
index f234ea24a9fe7..93796b3049b5b 100644
--- a/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass=si-opt-vgpr-liverange -o - %s | FileCheck -check-prefixes=CHECK %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -passes=si-opt-vgpr-liverange -o - %s | FileCheck -check-prefixes=CHECK %s
 
 # Tests a case that used to assert in SIOptimizeVGPRLiveRange when trying to optimize %3 which still appears
 # (though in an undef operand) in the REG_SEQUENCE of the "endif block". This undef pattern was caused by
diff --git a/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.mir b/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.mir
index 3bdcc14936fb9..7c7b930b6c318 100644
--- a/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -run-pass=si-opt-vgpr-liverange  %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes=si-opt-vgpr-liverange  %s -o - | FileCheck -check-prefix=GCN %s
 
 # SIOptimizeVGPRLiveRange shouldn't try to modify use of %5 in DBG_VALUE_LIST
 
diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call-2.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call-2.ll
index 3902c6dd422fd..5326c9e2c12f0 100644
--- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call-2.ll
+++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call-2.ll
@@ -100,14 +100,9 @@ entry:
 }
 
 ;.
-; NO: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; NO: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-;.
-; OW: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; OW: attributes #[[ATTR1]] = { "uniform-work-group-size"="false" }
-;.
-; CW: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CW: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CW: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
 ; NO: [[META0]] = !{ptr @bar1, ptr @bar2}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
index 894ab4fde7da0..fabdfff527294 100644
--- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
@@ -81,7 +81,7 @@ define amdgpu_kernel void @test_simple_indirect_call() {
 ;.
 ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-stack-objects" }
 ;.
-; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
 ; AKF_GCN: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
diff --git a/llvm/test/CodeGen/AMDGPU/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
index 71017f15e3c6d..6ed19bd6d764b 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
 
 define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX6-LABEL: v_ssubsat_i8:
@@ -34,14 +35,32 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_ssubsat_i8:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_lshlrev_b16 v1, 8, v1
-; GFX10PLUS-NEXT:    v_lshlrev_b16 v0, 8, v0
-; GFX10PLUS-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    v_ashrrev_i16 v0, 8, v0
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_ssubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
+; GFX10-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_ssubsat_i8:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_i16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_ssubsat_i8:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
 }
@@ -76,11 +95,24 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_ssubsat_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_ssubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_ssubsat_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_i16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_ssubsat_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
index b714fda6f1d07..2775de29368fb 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
@@ -3,6 +3,8 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX6-LABEL: v_uaddsat_i8:
@@ -36,6 +38,28 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
 ; GFX10-NEXT:    v_min_u16 v0, 0xff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_uaddsat_i8:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_min_u16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_uaddsat_i8:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX11-FAKE16-NEXT:    v_min_u16 v0, 0xff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
 }
@@ -67,6 +91,20 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_uaddsat_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_uaddsat_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_add_nc_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
 }
@@ -97,6 +135,12 @@ define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
 }
@@ -136,6 +180,12 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v2i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
 }
@@ -184,6 +234,13 @@ define <3 x i16> @v_uaddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v3i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
   ret <3 x i16> %result
 }
@@ -238,6 +295,13 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v4i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
   ret <2 x float> %cast
@@ -275,6 +339,13 @@ define <2 x i32> @v_uaddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v3 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v2i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v2 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, v1, v3 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
 }
@@ -317,6 +388,14 @@ define <3 x i32> @v_uaddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v5 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v3i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v3 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, v1, v4 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v2, v2, v5 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
 }
@@ -365,6 +444,15 @@ define <4 x i32> @v_uaddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v7 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v4i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v4 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, v1, v5 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v2, v2, v6 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v3, v3, v7 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
 }
@@ -437,6 +525,19 @@ define <8 x i32> @v_uaddsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v6, v6, v14 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v7, v7, v15 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v8i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v8 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, v1, v9 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v2, v2, v10 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v3, v3, v11 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v4, v4, v12 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v5, v5, v13 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v6, v6, v14 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v7, v7, v15 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %lhs, <8 x i32> %rhs)
   ret <8 x i32> %result
 }
@@ -565,6 +666,29 @@ define <16 x i32> @v_uaddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_nc_u32_e64 v15, v15, v31 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_v16i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-NEXT:    v_add_nc_u32_e64 v0, v0, v16 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, v1, v17 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v2, v2, v18 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v3, v3, v19 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v4, v4, v20 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v5, v5, v21 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v6, v6, v22 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v7, v7, v23 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v8, v8, v24 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v9, v9, v25 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v10, v10, v26 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v11, v11, v27 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v12, v12, v28 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v13, v13, v29 clamp
+; GFX11-NEXT:    v_add_nc_u32_e64 v14, v14, v30 clamp
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e64 v15, v15, v31 clamp
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
 }
@@ -610,6 +734,17 @@ define i64 @v_uaddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uaddsat_i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v0, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v1, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
   ret i64 %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll
index 049db01badacf..2ab48479fa1b0 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll
@@ -31,6 +31,6 @@ define amdgpu_kernel void @kernel1() #1 {
 
 attributes #0 = { "uniform-work-group-size"="true" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll
index c9387f196dff9..cc58d34a8b255 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll
@@ -98,7 +98,7 @@ define amdgpu_kernel void @kernel2() #0 {
 attributes #0 = { "uniform-work-group-size"="true" }
 ;.
 ; CHECK: attributes #[[ATTR0]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR2]] = { "uniform-work-group-size"="true" }
-; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
+; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll
index 7183da2c5efc3..33298bde89b97 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll
@@ -41,6 +41,6 @@ define amdgpu_kernel void @kernel3() #2 {
 
 attributes #2 = { "uniform-work-group-size"="true" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll
index 6ed04cf63d20b..6fe85cd7c9e65 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll
@@ -41,7 +41,7 @@ define amdgpu_kernel void @kernel2() #2 {
 
 attributes #1 = { "uniform-work-group-size"="true" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll
index d5ba2fd617c6e..f6ab402e01232 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll
@@ -52,8 +52,8 @@ attributes #0 = { nounwind }
 attributes #1 = { "uniform-work-group-size"="false" }
 attributes #2 = { "uniform-work-group-size"="true" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
 ; CHECK: attributes #[[ATTR3]] = { "uniform-work-group-size"="true" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll
index 7f0dfeaf75c80..37e3376ef9d50 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll
@@ -101,7 +101,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %m) #1 {
 attributes #0 = { nounwind readnone }
 attributes #1 = { "uniform-work-group-size"="true" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
-; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
+; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
+; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll
index 8616c73ad51c1..7f83686bc756e 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll
@@ -61,6 +61,6 @@ define amdgpu_kernel void @kernel3() #0 {
 
 attributes #0 = { "uniform-work-group-size"="false" }
 ;.
-; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
-; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index 8cc7025d671c4..775602ab80cde 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
 
 define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX6-LABEL: v_usubsat_i8:
@@ -27,13 +28,30 @@ define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_usubsat_i8:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX10PLUS-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX10PLUS-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_usubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_usubsat_i8:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_usubsat_i8:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
 }
@@ -60,11 +78,24 @@ define i16 @v_usubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: v_usubsat_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_usubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_usubsat_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v0.h clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_usubsat_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
 }
@@ -93,11 +124,23 @@ define i16 @usubsat_as_bithack_i16(i16 %x) {
 ; GFX9-NEXT:    v_sub_u16_e64 v0, v0, s4 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: usubsat_as_bithack_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: usubsat_as_bithack_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: usubsat_as_bithack_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, 0x8000 clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: usubsat_as_bithack_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %signsplat = ashr i16 %x, 15
   %flipsign = xor i16 %x, 32768
   %result = and i16 %signsplat, %flipsign
@@ -128,11 +171,23 @@ define i16 @usubsat_as_bithack2_i16(i16 %x) {
 ; GFX9-NEXT:    v_sub_u16_e64 v0, v0, s4 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: usubsat_as_bithack2_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: usubsat_as_bithack2_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: usubsat_as_bithack2_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, 0x8000 clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: usubsat_as_bithack2_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %signsplat = ashr i16 %x, 15
   %flipsign = add i16 %x, 32768
   %result = and i16 %signsplat, %flipsign
@@ -163,11 +218,23 @@ define i16 @usubsat_as_bithack_commute_i16(i16 %x) {
 ; GFX9-NEXT:    v_sub_u16_e64 v0, v0, s4 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: usubsat_as_bithack_commute_i16:
-; GFX10PLUS:       ; %bb.0:
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: usubsat_as_bithack_commute_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: usubsat_as_bithack_commute_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, 0x8000 clamp
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: usubsat_as_bithack_commute_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v0, v0, 0x8000 clamp
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %signsplat = ashr i16 %x, 15
   %flipsign = add i16 %x, 32768
   %result = and i16 %flipsign, %signsplat
diff --git a/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
new file mode 100644
index 0000000000000..b5f43f9f68936
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX950 %s
+define amdgpu_kernel void @v_ashr_pk_i8_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) #0 {
+; GFX950-LABEL: v_ashr_pk_i8_i32:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX950-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-NEXT:    v_mov_b32_e32 v1, 0xffffff80
+; GFX950-NEXT:    v_mov_b32_e32 v2, 0x7f
+; GFX950-NEXT:    v_mov_b32_e32 v0, 0
+; GFX950-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-NEXT:    s_ashr_i32 s1, s1, s2
+; GFX950-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX950-NEXT:    v_med3_i32 v3, s0, v1, v2
+; GFX950-NEXT:    v_med3_i32 v1, s1, v1, v2
+; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX950-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX950-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX950-NEXT:    s_endpgm
+  %insert.0 = insertelement <2 x i32> poison, i32 %src0, i64 0
+  %build_vector = insertelement <2 x i32> %insert.0, i32 %src1, i64 1
+  %src2.clamp = and i32 %src2, 31
+  %insert.1 = insertelement <2 x i32> poison, i32 %src2.clamp, i64 0
+  %src2.broadcast = shufflevector <2 x i32> %insert.1, <2 x i32> poison, <2 x i32> zeroinitializer
+  %ashr = ashr <2 x i32> %build_vector, %src2.broadcast
+  %sat.low = tail call <2 x i32> @llvm.smax.v2i32(<2 x i32> %ashr, <2 x i32> <i32 -128, i32 -128>)
+  %sat.hi = tail call <2 x i32> @llvm.smin.v2i32(<2 x i32> %sat.low, <2 x i32> <i32 127, i32 127>)
+  %trunc = trunc nsw <2 x i32> %sat.hi to <2 x i8>
+  %ret = bitcast <2 x i8> %trunc to i16
+  store i16 %ret, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_ashr_pk_u8_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) #0 {
+; GFX950-LABEL: v_ashr_pk_u8_i32:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX950-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-NEXT:    v_mov_b32_e32 v1, 0xff
+; GFX950-NEXT:    v_mov_b32_e32 v0, 0
+; GFX950-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX950-NEXT:    s_ashr_i32 s1, s1, s2
+; GFX950-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX950-NEXT:    v_med3_i32 v2, s0, 0, v1
+; GFX950-NEXT:    v_med3_i32 v1, s1, 0, v1
+; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX950-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX950-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX950-NEXT:    s_endpgm
+  %insert.0 = insertelement <2 x i32> poison, i32 %src0, i64 0
+  %build_vector = insertelement <2 x i32> %insert.0, i32 %src1, i64 1
+  %src2.clamp = and i32 %src2, 31
+  %insert.1 = insertelement <2 x i32> poison, i32 %src2.clamp, i64 0
+  %src2.broadcast = shufflevector <2 x i32> %insert.1, <2 x i32> poison, <2 x i32> zeroinitializer
+  %ashr = ashr <2 x i32> %build_vector, %src2.broadcast
+  %sat.low = tail call <2 x i32> @llvm.smax.v2i32(<2 x i32> %ashr, <2 x i32> <i32 0, i32 0>)
+  %sat.hi = tail call <2 x i32> @llvm.smin.v2i32(<2 x i32> %sat.low, <2 x i32> <i32 255, i32 255>)
+  %trunc = trunc nsw <2 x i32> %sat.hi to <2 x i8>
+  %ret = bitcast <2 x i8> %trunc to i16
+  store i16 %ret, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
index aec86ec343bdb..0af8c95da8d8b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
@@ -368,11 +368,11 @@ define amdgpu_kernel void @vector_bitcast_from_alloca_array(ptr addrspace(1) %ou
 ; OPT-LABEL: @vector_bitcast_to_array_from_alloca_array(
 ; OPT-NOT:   alloca
 ; OPT-NEXT: store i32 1, ptr addrspace(1) %out, align 4
-; OPT-NEXT: %out.repack1 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4
+; OPT-NEXT: %out.repack1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 4
 ; OPT-NEXT: store i32 2, ptr addrspace(1) %out.repack1, align 4
-; OPT-NEXT: %out.repack2 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 8
+; OPT-NEXT: %out.repack2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 8
 ; OPT-NEXT: store i32 3, ptr addrspace(1) %out.repack2, align 4
-; OPT-NEXT: %out.repack3 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 12
+; OPT-NEXT: %out.repack3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 12
 ; OPT-NEXT: store i32 4, ptr addrspace(1) %out.repack3, align 4
 
 ; GCN-LABEL: {{^}}vector_bitcast_to_array_from_alloca_array:
@@ -394,11 +394,11 @@ define amdgpu_kernel void @vector_bitcast_to_array_from_alloca_array(ptr addrspa
 ; OPT-LABEL: @vector_bitcast_to_struct_from_alloca_array(
 ; OPT-NOT:   alloca
 ; OPT-NEXT: store i32 1, ptr addrspace(1) %out, align 4
-; OPT-NEXT: %out.repack1 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4
+; OPT-NEXT: %out.repack1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 4
 ; OPT-NEXT: store i32 2, ptr addrspace(1) %out.repack1, align 4
-; OPT-NEXT: %out.repack2 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 8
+; OPT-NEXT: %out.repack2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 8
 ; OPT-NEXT: store i32 3, ptr addrspace(1) %out.repack2, align 4
-; OPT-NEXT: %out.repack3 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 12
+; OPT-NEXT: %out.repack3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 12
 ; OPT-NEXT: store i32 4, ptr addrspace(1) %out.repack3, align 4
 
 ; GCN-LABEL: {{^}}vector_bitcast_to_struct_from_alloca_array:
diff --git a/llvm/test/CodeGen/ARM/2012-03-13-DAGCombineBug.ll b/llvm/test/CodeGen/ARM/2012-03-13-DAGCombineBug.ll
index 6e8f1f2208e2b..bf523fe185d24 100644
--- a/llvm/test/CodeGen/ARM/2012-03-13-DAGCombineBug.ll
+++ b/llvm/test/CodeGen/ARM/2012-03-13-DAGCombineBug.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios | FileCheck %s
 ; rdar://11035895
 
@@ -5,8 +6,15 @@
 ; (i16 load $addr+c*sizeof(i16)). It should have issued an extload instead. i.e.
 ; (i32 extload $addr+c*sizeof(i16)
 define void @test_hi_short3(ptr nocapture %srcA, ptr nocapture %dst) nounwind {
+; CHECK-LABEL: test_hi_short3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov.u16 r0, d16[2]
+; CHECK-NEXT:    vmov.32 d16[0], r0
+; CHECK-NEXT:    vuzp.16 d16, d17
+; CHECK-NEXT:    vst1.32 {d16[0]}, [r1:32]
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK: vst1.32
   %0 = load <3 x i16> , ptr %srcA, align 8
   %1 = shufflevector <3 x i16> %0, <3 x i16> undef, <2 x i32> <i32 2, i32 undef>
   store <2 x i16> %1, ptr %dst, align 4
diff --git a/llvm/test/CodeGen/ARM/add-like-or.ll b/llvm/test/CodeGen/ARM/add-like-or.ll
index 5de03a92afeb4..c0ddee8388041 100644
--- a/llvm/test/CodeGen/ARM/add-like-or.ll
+++ b/llvm/test/CodeGen/ARM/add-like-or.ll
@@ -29,8 +29,8 @@ define i32 @test_add_i3(i1 %tst, i32 %a, i32 %b) {
 ; CHECK-T2:       @ %bb.0:
 ; CHECK-T2-NEXT:    .save {r4, lr}
 ; CHECK-T2-NEXT:    push {r4, lr}
-; CHECK-T2-NEXT:    lsls r0, r0, #31
 ; CHECK-T2-NEXT:    bic r4, r2, #3
+; CHECK-T2-NEXT:    lsls r0, r0, #31
 ; CHECK-T2-NEXT:    it ne
 ; CHECK-T2-NEXT:    bicne r4, r1, #6
 ; CHECK-T2-NEXT:    mov r0, r4
@@ -144,12 +144,12 @@ define i32 @test_add_i12(i32 %a, i32 %b, i1 %tst) {
 ;
 ; CHECK-A-LABEL: test_add_i12:
 ; CHECK-A:       @ %bb.0:
-; CHECK-A-NEXT:    bfc r1, #0, #13
 ; CHECK-A-NEXT:    bfc r0, #0, #12
+; CHECK-A-NEXT:    bfc r1, #0, #13
 ; CHECK-A-NEXT:    tst r2, #1
-; CHECK-A-NEXT:    moveq r0, r1
-; CHECK-A-NEXT:    movw r1, #854
-; CHECK-A-NEXT:    orr r0, r0, r1
+; CHECK-A-NEXT:    movne r1, r0
+; CHECK-A-NEXT:    movw r0, #854
+; CHECK-A-NEXT:    orr r0, r1, r0
 ; CHECK-A-NEXT:    bx lr
   %tmp = and i32 %a, -4096
   %tmp1 = and i32 %b, -8192
diff --git a/llvm/test/CodeGen/ARM/and-cmpz-to-shift.ll b/llvm/test/CodeGen/ARM/and-cmpz-to-shift.ll
new file mode 100644
index 0000000000000..c35e42418742c
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/and-cmpz-to-shift.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=thumbv6m %s -o - | FileCheck %s --check-prefix=T1
+; RUN: llc -mtriple=thumbv7m %s -o - | FileCheck %s --check-prefix=T2
+
+define void @test(i32 %x, i32 %a, i32 %b, i32 %c, i32 %d, ptr %p1, ptr %p2) {
+; T1-LABEL: test:
+; T1:       @ %bb.0:
+; T1-NEXT:    push {r4, lr}
+; T1-NEXT:    movs r4, #1
+; T1-NEXT:    lsls r4, r4, #24
+; T1-NEXT:    tst r0, r4
+; T1-NEXT:    beq .LBB0_2
+; T1-NEXT:  @ %bb.1:
+; T1-NEXT:    mov r1, r2
+; T1-NEXT:  .LBB0_2:
+; T1-NEXT:    ldr r0, [sp, #12]
+; T1-NEXT:    str r1, [r0]
+; T1-NEXT:    beq .LBB0_4
+; T1-NEXT:  @ %bb.3:
+; T1-NEXT:    ldr r3, [sp, #8]
+; T1-NEXT:  .LBB0_4:
+; T1-NEXT:    ldr r0, [sp, #16]
+; T1-NEXT:    str r3, [r0]
+; T1-NEXT:    pop {r4, pc}
+;
+; T2-LABEL: test:
+; T2:       @ %bb.0:
+; T2-NEXT:    tst.w r0, #16777216
+; T2-NEXT:    ldr r0, [sp, #4]
+; T2-NEXT:    it ne
+; T2-NEXT:    movne r1, r2
+; T2-NEXT:    str r1, [r0]
+; T2-NEXT:    ldr r1, [sp, #8]
+; T2-NEXT:    ldr r0, [sp]
+; T2-NEXT:    it eq
+; T2-NEXT:    moveq r0, r3
+; T2-NEXT:    str r0, [r1]
+; T2-NEXT:    bx lr
+  %and = and i32 %x, u0x1000000
+  %cmp = icmp eq i32 %and, 0
+  %sel1 = select i1 %cmp, i32 %a, i32 %b
+  %sel2 = select i1 %cmp, i32 %c, i32 %d
+  store i32 %sel1, ptr %p1, align 4
+  store i32 %sel2, ptr %p2, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
index b6adc995091ce..75416475289f3 100644
--- a/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/ARM/arm-shrink-wrapping.ll
@@ -1965,32 +1965,34 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; ARM-ENABLE-NEXT:  @ %bb.1: @ %bb3
 ; ARM-ENABLE-NEXT:    push {r4, r7, lr}
 ; ARM-ENABLE-NEXT:    add r7, sp, #4
-; ARM-ENABLE-NEXT:    sub r4, sp, #16
+; ARM-ENABLE-NEXT:    sub r4, sp, #24
 ; ARM-ENABLE-NEXT:    bfc r4, #0, #4
 ; ARM-ENABLE-NEXT:    mov sp, r4
 ; ARM-ENABLE-NEXT:    ldr r1, [r7, #8]
+; ARM-ENABLE-NEXT:    vmov.f64 d16, #1.000000e+00
 ; ARM-ENABLE-NEXT:    mov r2, r3
 ; ARM-ENABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; ARM-ENABLE-NEXT:    vmov d9, r3, r1
 ; ARM-ENABLE-NEXT:    vmov s16, r0
 ; ARM-ENABLE-NEXT:    mov r0, r3
-; ARM-ENABLE-NEXT:    vmov d9, r3, r1
 ; ARM-ENABLE-NEXT:    mov r3, r1
+; ARM-ENABLE-NEXT:    vstr d10, [r4, #16]
+; ARM-ENABLE-NEXT:    vadd.f64 d10, d9, d16
 ; ARM-ENABLE-NEXT:    bl _pow
 ; ARM-ENABLE-NEXT:    vmov.f32 s0, #1.000000e+00
 ; ARM-ENABLE-NEXT:    mov r4, sp
-; ARM-ENABLE-NEXT:    vmov.f64 d16, #1.000000e+00
-; ARM-ENABLE-NEXT:    vadd.f64 d16, d9, d16
+; ARM-ENABLE-NEXT:    vmov.f64 d17, d9
+; ARM-ENABLE-NEXT:    vmov d16, r0, r1
 ; ARM-ENABLE-NEXT:    vcmp.f32 s16, s0
 ; ARM-ENABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; ARM-ENABLE-NEXT:    vmov d17, r0, r1
-; ARM-ENABLE-NEXT:    vmov.f64 d18, d9
-; ARM-ENABLE-NEXT:    vadd.f64 d17, d17, d17
-; ARM-ENABLE-NEXT:    vmovgt.f64 d18, d16
-; ARM-ENABLE-NEXT:    vcmp.f64 d18, d9
+; ARM-ENABLE-NEXT:    vadd.f64 d16, d16, d16
+; ARM-ENABLE-NEXT:    vmovgt.f64 d17, d10
+; ARM-ENABLE-NEXT:    vcmp.f64 d17, d9
 ; ARM-ENABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; ARM-ENABLE-NEXT:    vmovne.f64 d9, d17
+; ARM-ENABLE-NEXT:    vmovne.f64 d9, d16
 ; ARM-ENABLE-NEXT:    vcvt.f32.f64 s0, d9
 ; ARM-ENABLE-NEXT:    vld1.64 {d8, d9}, [r4:128]
+; ARM-ENABLE-NEXT:    vldr d10, [r4, #16]
 ; ARM-ENABLE-NEXT:    sub sp, r7, #4
 ; ARM-ENABLE-NEXT:    pop {r4, r7, lr}
 ; ARM-ENABLE-NEXT:    vmov r0, s0
@@ -2012,32 +2014,33 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; ARM-DISABLE-NEXT:  @ %bb.0: @ %bb
 ; ARM-DISABLE-NEXT:    push {r4, r7, lr}
 ; ARM-DISABLE-NEXT:    add r7, sp, #4
-; ARM-DISABLE-NEXT:    sub r4, sp, #16
+; ARM-DISABLE-NEXT:    sub r4, sp, #24
 ; ARM-DISABLE-NEXT:    bfc r4, #0, #4
 ; ARM-DISABLE-NEXT:    mov sp, r4
 ; ARM-DISABLE-NEXT:    tst r2, #1
 ; ARM-DISABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; ARM-DISABLE-NEXT:    vstr d10, [r4, #16]
 ; ARM-DISABLE-NEXT:    beq LBB12_2
 ; ARM-DISABLE-NEXT:  @ %bb.1: @ %bb3
 ; ARM-DISABLE-NEXT:    ldr r1, [r7, #8]
-; ARM-DISABLE-NEXT:    vmov s16, r0
-; ARM-DISABLE-NEXT:    mov r0, r3
+; ARM-DISABLE-NEXT:    vmov.f64 d16, #1.000000e+00
 ; ARM-DISABLE-NEXT:    mov r2, r3
 ; ARM-DISABLE-NEXT:    vmov d9, r3, r1
+; ARM-DISABLE-NEXT:    vmov s16, r0
+; ARM-DISABLE-NEXT:    mov r0, r3
 ; ARM-DISABLE-NEXT:    mov r3, r1
+; ARM-DISABLE-NEXT:    vadd.f64 d10, d9, d16
 ; ARM-DISABLE-NEXT:    bl _pow
 ; ARM-DISABLE-NEXT:    vmov.f32 s0, #1.000000e+00
-; ARM-DISABLE-NEXT:    vmov.f64 d16, #1.000000e+00
-; ARM-DISABLE-NEXT:    vadd.f64 d16, d9, d16
+; ARM-DISABLE-NEXT:    vmov.f64 d17, d9
+; ARM-DISABLE-NEXT:    vmov d16, r0, r1
 ; ARM-DISABLE-NEXT:    vcmp.f32 s16, s0
 ; ARM-DISABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; ARM-DISABLE-NEXT:    vmov d17, r0, r1
-; ARM-DISABLE-NEXT:    vmov.f64 d18, d9
-; ARM-DISABLE-NEXT:    vadd.f64 d17, d17, d17
-; ARM-DISABLE-NEXT:    vmovgt.f64 d18, d16
-; ARM-DISABLE-NEXT:    vcmp.f64 d18, d9
+; ARM-DISABLE-NEXT:    vadd.f64 d16, d16, d16
+; ARM-DISABLE-NEXT:    vmovgt.f64 d17, d10
+; ARM-DISABLE-NEXT:    vcmp.f64 d17, d9
 ; ARM-DISABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; ARM-DISABLE-NEXT:    vmovne.f64 d9, d17
+; ARM-DISABLE-NEXT:    vmovne.f64 d9, d16
 ; ARM-DISABLE-NEXT:    vcvt.f32.f64 s0, d9
 ; ARM-DISABLE-NEXT:    b LBB12_3
 ; ARM-DISABLE-NEXT:  LBB12_2:
@@ -2046,6 +2049,7 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; ARM-DISABLE-NEXT:    mov r4, sp
 ; ARM-DISABLE-NEXT:    vld1.64 {d8, d9}, [r4:128]
 ; ARM-DISABLE-NEXT:    vmov r0, s0
+; ARM-DISABLE-NEXT:    vldr d10, [r4, #16]
 ; ARM-DISABLE-NEXT:    sub sp, r7, #4
 ; ARM-DISABLE-NEXT:    pop {r4, r7, pc}
 ; ARM-DISABLE-NEXT:    .p2align 2
@@ -2064,34 +2068,36 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; THUMB-ENABLE-NEXT:  @ %bb.1: @ %bb3
 ; THUMB-ENABLE-NEXT:    push {r4, r7, lr}
 ; THUMB-ENABLE-NEXT:    add r7, sp, #4
-; THUMB-ENABLE-NEXT:    sub.w r4, sp, #16
+; THUMB-ENABLE-NEXT:    sub.w r4, sp, #24
 ; THUMB-ENABLE-NEXT:    bfc r4, #0, #4
 ; THUMB-ENABLE-NEXT:    mov sp, r4
 ; THUMB-ENABLE-NEXT:    ldr r1, [r7, #8]
+; THUMB-ENABLE-NEXT:    vmov.f64 d16, #1.000000e+00
 ; THUMB-ENABLE-NEXT:    mov r2, r3
 ; THUMB-ENABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; THUMB-ENABLE-NEXT:    vmov d9, r3, r1
 ; THUMB-ENABLE-NEXT:    vmov s16, r0
 ; THUMB-ENABLE-NEXT:    mov r0, r3
-; THUMB-ENABLE-NEXT:    vmov d9, r3, r1
 ; THUMB-ENABLE-NEXT:    mov r3, r1
+; THUMB-ENABLE-NEXT:    vstr d10, [r4, #16]
+; THUMB-ENABLE-NEXT:    vadd.f64 d10, d9, d16
 ; THUMB-ENABLE-NEXT:    bl _pow
 ; THUMB-ENABLE-NEXT:    vmov.f32 s0, #1.000000e+00
 ; THUMB-ENABLE-NEXT:    mov r4, sp
-; THUMB-ENABLE-NEXT:    vmov.f64 d16, #1.000000e+00
-; THUMB-ENABLE-NEXT:    vmov.f64 d18, d9
+; THUMB-ENABLE-NEXT:    vmov.f64 d17, d9
+; THUMB-ENABLE-NEXT:    vmov d16, r0, r1
 ; THUMB-ENABLE-NEXT:    vcmp.f32 s16, s0
-; THUMB-ENABLE-NEXT:    vadd.f64 d16, d9, d16
 ; THUMB-ENABLE-NEXT:    vmrs APSR_nzcv, fpscr
 ; THUMB-ENABLE-NEXT:    it gt
-; THUMB-ENABLE-NEXT:    vmovgt.f64 d18, d16
-; THUMB-ENABLE-NEXT:    vcmp.f64 d18, d9
-; THUMB-ENABLE-NEXT:    vmov d17, r0, r1
+; THUMB-ENABLE-NEXT:    vmovgt.f64 d17, d10
+; THUMB-ENABLE-NEXT:    vcmp.f64 d17, d9
+; THUMB-ENABLE-NEXT:    vadd.f64 d16, d16, d16
 ; THUMB-ENABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; THUMB-ENABLE-NEXT:    vadd.f64 d17, d17, d17
 ; THUMB-ENABLE-NEXT:    it ne
-; THUMB-ENABLE-NEXT:    vmovne.f64 d9, d17
+; THUMB-ENABLE-NEXT:    vmovne.f64 d9, d16
 ; THUMB-ENABLE-NEXT:    vcvt.f32.f64 s0, d9
 ; THUMB-ENABLE-NEXT:    vld1.64 {d8, d9}, [r4:128]
+; THUMB-ENABLE-NEXT:    vldr d10, [r4, #16]
 ; THUMB-ENABLE-NEXT:    subs r4, r7, #4
 ; THUMB-ENABLE-NEXT:    mov sp, r4
 ; THUMB-ENABLE-NEXT:    pop.w {r4, r7, lr}
@@ -2114,34 +2120,35 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; THUMB-DISABLE-NEXT:  @ %bb.0: @ %bb
 ; THUMB-DISABLE-NEXT:    push {r4, r7, lr}
 ; THUMB-DISABLE-NEXT:    add r7, sp, #4
-; THUMB-DISABLE-NEXT:    sub.w r4, sp, #16
+; THUMB-DISABLE-NEXT:    sub.w r4, sp, #24
 ; THUMB-DISABLE-NEXT:    bfc r4, #0, #4
 ; THUMB-DISABLE-NEXT:    mov sp, r4
 ; THUMB-DISABLE-NEXT:    lsls r1, r2, #31
 ; THUMB-DISABLE-NEXT:    vst1.64 {d8, d9}, [r4:128]
+; THUMB-DISABLE-NEXT:    vstr d10, [r4, #16]
 ; THUMB-DISABLE-NEXT:    beq LBB12_2
 ; THUMB-DISABLE-NEXT:  @ %bb.1: @ %bb3
 ; THUMB-DISABLE-NEXT:    ldr r1, [r7, #8]
-; THUMB-DISABLE-NEXT:    vmov s16, r0
-; THUMB-DISABLE-NEXT:    mov r0, r3
+; THUMB-DISABLE-NEXT:    vmov.f64 d16, #1.000000e+00
 ; THUMB-DISABLE-NEXT:    mov r2, r3
 ; THUMB-DISABLE-NEXT:    vmov d9, r3, r1
+; THUMB-DISABLE-NEXT:    vmov s16, r0
+; THUMB-DISABLE-NEXT:    mov r0, r3
 ; THUMB-DISABLE-NEXT:    mov r3, r1
+; THUMB-DISABLE-NEXT:    vadd.f64 d10, d9, d16
 ; THUMB-DISABLE-NEXT:    bl _pow
 ; THUMB-DISABLE-NEXT:    vmov.f32 s0, #1.000000e+00
-; THUMB-DISABLE-NEXT:    vmov.f64 d16, #1.000000e+00
-; THUMB-DISABLE-NEXT:    vmov.f64 d18, d9
+; THUMB-DISABLE-NEXT:    vmov.f64 d17, d9
+; THUMB-DISABLE-NEXT:    vmov d16, r0, r1
 ; THUMB-DISABLE-NEXT:    vcmp.f32 s16, s0
-; THUMB-DISABLE-NEXT:    vadd.f64 d16, d9, d16
 ; THUMB-DISABLE-NEXT:    vmrs APSR_nzcv, fpscr
 ; THUMB-DISABLE-NEXT:    it gt
-; THUMB-DISABLE-NEXT:    vmovgt.f64 d18, d16
-; THUMB-DISABLE-NEXT:    vcmp.f64 d18, d9
-; THUMB-DISABLE-NEXT:    vmov d17, r0, r1
+; THUMB-DISABLE-NEXT:    vmovgt.f64 d17, d10
+; THUMB-DISABLE-NEXT:    vcmp.f64 d17, d9
+; THUMB-DISABLE-NEXT:    vadd.f64 d16, d16, d16
 ; THUMB-DISABLE-NEXT:    vmrs APSR_nzcv, fpscr
-; THUMB-DISABLE-NEXT:    vadd.f64 d17, d17, d17
 ; THUMB-DISABLE-NEXT:    it ne
-; THUMB-DISABLE-NEXT:    vmovne.f64 d9, d17
+; THUMB-DISABLE-NEXT:    vmovne.f64 d9, d16
 ; THUMB-DISABLE-NEXT:    vcvt.f32.f64 s0, d9
 ; THUMB-DISABLE-NEXT:    b LBB12_3
 ; THUMB-DISABLE-NEXT:  LBB12_2:
@@ -2149,8 +2156,9 @@ define float @debug_info(float %gamma, float %slopeLimit, i1 %or.cond, double %t
 ; THUMB-DISABLE-NEXT:  LBB12_3: @ %bb13
 ; THUMB-DISABLE-NEXT:    mov r4, sp
 ; THUMB-DISABLE-NEXT:    vld1.64 {d8, d9}, [r4:128]
-; THUMB-DISABLE-NEXT:    subs r4, r7, #4
 ; THUMB-DISABLE-NEXT:    vmov r0, s0
+; THUMB-DISABLE-NEXT:    vldr d10, [r4, #16]
+; THUMB-DISABLE-NEXT:    subs r4, r7, #4
 ; THUMB-DISABLE-NEXT:    mov sp, r4
 ; THUMB-DISABLE-NEXT:    pop {r4, r7, pc}
 ; THUMB-DISABLE-NEXT:    .p2align 2
diff --git a/llvm/test/CodeGen/ARM/atomic-64bit.ll b/llvm/test/CodeGen/ARM/atomic-64bit.ll
index ab9e1dfd1cfb1..ca9939c0f8c55 100644
--- a/llvm/test/CodeGen/ARM/atomic-64bit.ll
+++ b/llvm/test/CodeGen/ARM/atomic-64bit.ll
@@ -278,12 +278,9 @@ define i64 @test10(ptr %ptr, i64 %val) {
 ; CHECK-BE: subs {{[^,]+}}, r2, [[REG2]]
 ; CHECK-LE: sbcs {{[^,]+}}, r2, [[REG2]]
 ; CHECK-BE: sbcs {{[^,]+}}, r1, [[REG1]]
-; CHECK: mov     [[CMP:[a-z0-9]+]], #0
-; CHECK: movwge  [[CMP]], #1
-; CHECK: cmp     [[CMP]], #0
-; CHECK: movne   [[OUT_HI]], [[REG2]]
+; CHECK: movge   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
-; CHECK: movne   [[OUT_LO]], [[REG1]]
+; CHECK: movge   [[OUT_LO]], [[REG1]]
 ; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
@@ -297,12 +294,10 @@ define i64 @test10(ptr %ptr, i64 %val) {
 ; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
-; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
-; CHECK-THUMB: movge.w   [[CMP]], #1
-; CHECK-THUMB: cmp.w     [[CMP]], #0
 ; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
-; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
+; CHECK-THUMB: itt     ge
+; CHECK-THUMB: movge   [[OUT_HI]], [[REG2]]
+; CHECK-THUMB: movge   [[OUT_LO]], [[REG1]]
 ; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
@@ -323,12 +318,9 @@ define i64 @test11(ptr %ptr, i64 %val) {
 ; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
-; CHECK: mov     [[CMP:[a-z0-9]+]], #0
-; CHECK: movwhs  [[CMP]], #1
-; CHECK: cmp     [[CMP]], #0
-; CHECK: movne   [[OUT_HI]], [[REG2]]
+; CHECK: movhs   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
-; CHECK: movne   [[OUT_LO]], [[REG1]]
+; CHECK: movhs   [[OUT_LO]], [[REG1]]
 ; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
@@ -342,12 +334,10 @@ define i64 @test11(ptr %ptr, i64 %val) {
 ; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
-; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
-; CHECK-THUMB: movhs.w   [[CMP]], #1
-; CHECK-THUMB: cmp.w     [[CMP]], #0
 ; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
-; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
+; CHECK-THUMB: itt     hs
+; CHECK-THUMB: movhs   [[OUT_HI]], [[REG2]]
+; CHECK-THUMB: movhs   [[OUT_LO]], [[REG1]]
 ; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
@@ -368,12 +358,9 @@ define i64 @test12(ptr %ptr, i64 %val) {
 ; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
-; CHECK: mov     [[CMP:[a-z0-9]+]], #0
-; CHECK: movwlt  [[CMP]], #1
-; CHECK: cmp     [[CMP]], #0
-; CHECK: movne   [[OUT_HI]], [[REG2]]
+; CHECK: movlt   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
-; CHECK: movne   [[OUT_LO]], [[REG1]]
+; CHECK: movlt   [[OUT_LO]], [[REG1]]
 ; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
@@ -387,12 +374,10 @@ define i64 @test12(ptr %ptr, i64 %val) {
 ; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
-; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
-; CHECK-THUMB: movlt.w   [[CMP]], #1
-; CHECK-THUMB: cmp.w     [[CMP]], #0
 ; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
-; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
+; CHECK-THUMB: itt     lt
+; CHECK-THUMB: movlt   [[OUT_HI]], [[REG2]]
+; CHECK-THUMB: movlt   [[OUT_LO]], [[REG1]]
 ; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
@@ -413,12 +398,9 @@ define i64 @test13(ptr %ptr, i64 %val) {
 ; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
 ; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
-; CHECK: mov     [[CMP:[a-z0-9]+]], #0
-; CHECK: movwlo  [[CMP]], #1
-; CHECK: cmp     [[CMP]], #0
-; CHECK: movne   [[OUT_HI]], [[REG2]]
+; CHECK: movlo   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
-; CHECK: movne   [[OUT_LO]], [[REG1]]
+; CHECK: movlo   [[OUT_LO]], [[REG1]]
 ; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
@@ -432,12 +414,10 @@ define i64 @test13(ptr %ptr, i64 %val) {
 ; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
 ; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
-; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
-; CHECK-THUMB: movlo.w   [[CMP]], #1
-; CHECK-THUMB: cmp.w     [[CMP]], #0
 ; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
-; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
+; CHECK-THUMB: itt     lo
+; CHECK-THUMB: movlo   [[OUT_HI]], [[REG2]]
+; CHECK-THUMB: movlo   [[OUT_LO]], [[REG1]]
 ; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
diff --git a/llvm/test/CodeGen/ARM/atomic-ops-v8.ll b/llvm/test/CodeGen/ARM/atomic-ops-v8.ll
index 0a467c2b70acf..d48b070aa862e 100644
--- a/llvm/test/CodeGen/ARM/atomic-ops-v8.ll
+++ b/llvm/test/CodeGen/ARM/atomic-ops-v8.ll
@@ -672,12 +672,9 @@ define void @test_atomic_load_min_i64(i64 %offset) nounwind {
 ; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
-; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movwge [[CMP:r[0-9]+|lr]], #1
-; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movne [[MINHI]], [[OLD2]]
+; CHECK-ARM: movge [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
-; CHECK-ARM: movne [[MINLO]], [[OLD1]]
+; CHECK-ARM: movge [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
@@ -785,12 +782,9 @@ define void @test_atomic_load_max_i64(i64 %offset) nounwind {
 ; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
-; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movwlt [[CMP:r[0-9]+|lr]], #1
-; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movne [[MINHI]], [[OLD2]]
+; CHECK-ARM: movlt [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
-; CHECK-ARM: movne [[MINLO]], [[OLD1]]
+; CHECK-ARM: movlt [[MINLO]], [[OLD1]]
 ; CHECK-ARM: strexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: strexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
@@ -898,12 +892,9 @@ define void @test_atomic_load_umin_i64(i64 %offset) nounwind {
 ; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
-; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movwhs [[CMP:r[0-9]+|lr]], #1
-; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movne [[MINHI]], [[OLD2]]
+; CHECK-ARM: movhs [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
-; CHECK-ARM: movne [[MINLO]], [[OLD1]]
+; CHECK-ARM: movhs [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
@@ -1011,12 +1002,9 @@ define void @test_atomic_load_umax_i64(i64 %offset) nounwind {
 ; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
 ; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
-; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movwlo [[CMP:r[0-9]+|lr]], #1
-; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
-; CHECK-ARM: movne [[MINHI]], [[OLD2]]
+; CHECK-ARM: movlo [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
-; CHECK-ARM: movne [[MINLO]], [[OLD1]]
+; CHECK-ARM: movlo [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
diff --git a/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
index 62711ee683489..8706728c4b841 100644
--- a/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
@@ -68,27 +68,23 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
 define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
 ; CHECK-LABEL: atomicrmw_usub_cond_i64:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    dmb ish
 ; CHECK-NEXT:  .LBB3_1: @ %atomicrmw.start
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrexd r4, r5, [r0]
-; CHECK-NEXT:    mov r1, #0
-; CHECK-NEXT:    subs r6, r4, r2
-; CHECK-NEXT:    sbcs r7, r5, r3
-; CHECK-NEXT:    movwhs r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    moveq r7, r5
-; CHECK-NEXT:    moveq r6, r4
-; CHECK-NEXT:    strexd r1, r6, r7, [r0]
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    ldrexd r0, r1, [r12]
+; CHECK-NEXT:    subs r4, r0, r2
+; CHECK-NEXT:    sbcs r5, r1, r3
+; CHECK-NEXT:    movlo r5, r1
+; CHECK-NEXT:    movlo r4, r0
+; CHECK-NEXT:    strexd lr, r4, r5, [r12]
+; CHECK-NEXT:    cmp lr, #0
 ; CHECK-NEXT:    bne .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %atomicrmw.end
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    dmb ish
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r11, pc}
   %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
   ret i64 %result
 }
@@ -164,7 +160,7 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
 ; CHECK-NEXT:    subs r6, r4, r2
 ; CHECK-NEXT:    sbcs r7, r5, r3
 ; CHECK-NEXT:    adc r1, r12, #0
-; CHECK-NEXT:    eors r1, r1, #1
+; CHECK-NEXT:    teq r1, #1
 ; CHECK-NEXT:    movwne r7, #0
 ; CHECK-NEXT:    movwne r6, #0
 ; CHECK-NEXT:    strexd r1, r6, r7, [r0]
diff --git a/llvm/test/CodeGen/ARM/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/ARM/atomicrmw-uinc-udec-wrap.ll
index 243ec4deecdb8..433fb325a7349 100644
--- a/llvm/test/CodeGen/ARM/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/ARM/atomicrmw-uinc-udec-wrap.ll
@@ -69,29 +69,25 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-LABEL: atomicrmw_uinc_wrap_i64:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r6, r7, lr}
+; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    dmb ish
 ; CHECK-NEXT:  .LBB3_1: @ %atomicrmw.start
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrexd r4, r5, [r0]
-; CHECK-NEXT:    adds r6, r4, #1
-; CHECK-NEXT:    adc r7, r5, #0
-; CHECK-NEXT:    subs r1, r4, r2
-; CHECK-NEXT:    sbcs r1, r5, r3
-; CHECK-NEXT:    mov r1, #0
-; CHECK-NEXT:    movwhs r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    movwne r7, #0
-; CHECK-NEXT:    movwne r6, #0
-; CHECK-NEXT:    strexd r1, r6, r7, [r0]
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    ldrexd r0, r1, [r12]
+; CHECK-NEXT:    adds r6, r0, #1
+; CHECK-NEXT:    adc r7, r1, #0
+; CHECK-NEXT:    subs r4, r0, r2
+; CHECK-NEXT:    sbcs r4, r1, r3
+; CHECK-NEXT:    movwhs r7, #0
+; CHECK-NEXT:    movwhs r6, #0
+; CHECK-NEXT:    strexd r4, r6, r7, [r12]
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    bne .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %atomicrmw.end
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    dmb ish
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r6, r7, pc}
   %result = atomicrmw uinc_wrap ptr %ptr, i64 %val seq_cst
   ret i64 %result
 }
@@ -102,8 +98,8 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) {
 ; CHECK-NEXT:    dmb ish
 ; CHECK-NEXT:  .LBB4_1: @ %atomicrmw.start
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    uxtb r3, r1
 ; CHECK-NEXT:    ldrexb r12, [r0]
+; CHECK-NEXT:    uxtb r3, r1
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    mov r3, r1
 ; CHECK-NEXT:    subls r3, r12, #1
@@ -126,8 +122,8 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; CHECK-NEXT:    dmb ish
 ; CHECK-NEXT:  .LBB5_1: @ %atomicrmw.start
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    uxth r3, r1
 ; CHECK-NEXT:    ldrexh r12, [r0]
+; CHECK-NEXT:    uxth r3, r1
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    mov r3, r1
 ; CHECK-NEXT:    subls r3, r12, #1
diff --git a/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_ints.ll b/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_ints.ll
index a38ade7cdbf06..f633315822cc3 100644
--- a/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_ints.ll
+++ b/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_ints.ll
@@ -1422,8 +1422,8 @@ define i8 @test_max_i8() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB7_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    sxtb r0, r1
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r0, #1
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movgt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i8
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i8
@@ -1468,8 +1468,8 @@ define i8 @test_max_i8() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB7_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    sxtb r0, r1
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r0, #1
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movgt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI7_0
 ; CHECK-ARM6-NEXT:    uxtb r1, r1
@@ -1518,8 +1518,8 @@ define i8 @test_max_i8() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB7_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    sxtb r0, r1
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r0, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it gt
 ; CHECK-THUMB7-NEXT:    movgt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i8
@@ -1643,8 +1643,8 @@ define i8 @test_min_i8() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB8_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    sxtb r0, r1
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r0, #2
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movlt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i8
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i8
@@ -1689,8 +1689,8 @@ define i8 @test_min_i8() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB8_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    sxtb r0, r1
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r0, #2
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movlt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI8_0
 ; CHECK-ARM6-NEXT:    uxtb r1, r1
@@ -1739,8 +1739,8 @@ define i8 @test_min_i8() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB8_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    sxtb r0, r1
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r0, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lt
 ; CHECK-THUMB7-NEXT:    movlt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i8
@@ -1866,8 +1866,8 @@ define i8 @test_umax_i8() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB9_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    uxtb r1, r12
-; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #1
+; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    movhi lr, r12
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i8
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i8
@@ -1913,8 +1913,8 @@ define i8 @test_umax_i8() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB9_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    uxtb r1, r12
-; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #1
+; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    movhi lr, r12
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI9_0
 ; CHECK-ARM6-NEXT:    uxtb r12, r12
@@ -1964,8 +1964,8 @@ define i8 @test_umax_i8() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB9_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    uxtb r1, r4
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it hi
 ; CHECK-THUMB7-NEXT:    movhi r12, r4
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i8
@@ -2091,8 +2091,8 @@ define i8 @test_umin_i8() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB10_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    uxtb r1, r12
-; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #2
+; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    movlo lr, r12
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i8
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i8
@@ -2138,8 +2138,8 @@ define i8 @test_umin_i8() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB10_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    uxtb r1, r12
-; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #2
+; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    movlo lr, r12
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI10_0
 ; CHECK-ARM6-NEXT:    uxtb r12, r12
@@ -2189,8 +2189,8 @@ define i8 @test_umin_i8() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB10_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    uxtb r1, r4
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lo
 ; CHECK-THUMB7-NEXT:    movlo r12, r4
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i8
@@ -3709,8 +3709,8 @@ define i16 @test_max_i16() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB18_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    sxth r0, r1
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r0, #1
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movgt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i16
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i16
@@ -3755,8 +3755,8 @@ define i16 @test_max_i16() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB18_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    sxth r0, r1
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r0, #1
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movgt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI18_0
 ; CHECK-ARM6-NEXT:    uxth r1, r1
@@ -3805,8 +3805,8 @@ define i16 @test_max_i16() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB18_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    sxth r0, r1
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r0, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it gt
 ; CHECK-THUMB7-NEXT:    movgt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i16
@@ -3930,8 +3930,8 @@ define i16 @test_min_i16() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB19_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    sxth r0, r1
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r0, #2
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movlt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i16
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i16
@@ -3976,8 +3976,8 @@ define i16 @test_min_i16() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB19_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    sxth r0, r1
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r0, #2
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movlt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI19_0
 ; CHECK-ARM6-NEXT:    uxth r1, r1
@@ -4026,8 +4026,8 @@ define i16 @test_min_i16() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB19_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    sxth r0, r1
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r0, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lt
 ; CHECK-THUMB7-NEXT:    movlt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i16
@@ -4153,8 +4153,8 @@ define i16 @test_umax_i16() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB20_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    uxth r1, r12
-; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #1
+; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    movhi lr, r12
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i16
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i16
@@ -4200,8 +4200,8 @@ define i16 @test_umax_i16() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB20_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    uxth r1, r12
-; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #1
+; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    movhi lr, r12
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI20_0
 ; CHECK-ARM6-NEXT:    uxth r12, r12
@@ -4251,8 +4251,8 @@ define i16 @test_umax_i16() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB20_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    uxth r1, r4
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it hi
 ; CHECK-THUMB7-NEXT:    movhi r12, r4
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i16
@@ -4378,8 +4378,8 @@ define i16 @test_umin_i16() {
 ; CHECK-ARM8-NEXT:    @ Child Loop BB21_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM8-NEXT:    uxth r1, r12
-; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #2
+; CHECK-ARM8-NEXT:    mov lr, #1
 ; CHECK-ARM8-NEXT:    movlo lr, r12
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i16
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i16
@@ -4425,8 +4425,8 @@ define i16 @test_umin_i16() {
 ; CHECK-ARM6-NEXT:    @ Child Loop BB21_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
 ; CHECK-ARM6-NEXT:    uxth r1, r12
-; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #2
+; CHECK-ARM6-NEXT:    mov lr, #1
 ; CHECK-ARM6-NEXT:    movlo lr, r12
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI21_0
 ; CHECK-ARM6-NEXT:    uxth r12, r12
@@ -4476,8 +4476,8 @@ define i16 @test_umin_i16() {
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB21_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
 ; CHECK-THUMB7-NEXT:    uxth r1, r4
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lo
 ; CHECK-THUMB7-NEXT:    movlo r12, r4
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i16
@@ -5939,8 +5939,8 @@ define i32 @test_max_i32() {
 ; CHECK-ARM8-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM8-NEXT:    @ Child Loop BB29_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #1
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movgt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i32
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i32
@@ -5982,8 +5982,8 @@ define i32 @test_max_i32() {
 ; CHECK-ARM6-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM6-NEXT:    @ Child Loop BB29_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #1
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movgt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI29_0
 ; CHECK-ARM6-NEXT:  .LBB29_2: @ %atomicrmw.start
@@ -6029,8 +6029,8 @@ define i32 @test_max_i32() {
 ; CHECK-THUMB7-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB29_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it gt
 ; CHECK-THUMB7-NEXT:    movgt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i32
@@ -6148,8 +6148,8 @@ define i32 @test_min_i32() {
 ; CHECK-ARM8-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM8-NEXT:    @ Child Loop BB30_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #2
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movlt r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i32
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i32
@@ -6191,8 +6191,8 @@ define i32 @test_min_i32() {
 ; CHECK-ARM6-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM6-NEXT:    @ Child Loop BB30_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #2
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movlt r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI30_0
 ; CHECK-ARM6-NEXT:  .LBB30_2: @ %atomicrmw.start
@@ -6238,8 +6238,8 @@ define i32 @test_min_i32() {
 ; CHECK-THUMB7-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB30_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lt
 ; CHECK-THUMB7-NEXT:    movlt r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i32
@@ -6357,8 +6357,8 @@ define i32 @test_umax_i32() {
 ; CHECK-ARM8-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM8-NEXT:    @ Child Loop BB31_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #1
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movhi r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i32
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i32
@@ -6400,8 +6400,8 @@ define i32 @test_umax_i32() {
 ; CHECK-ARM6-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM6-NEXT:    @ Child Loop BB31_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #1
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movhi r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI31_0
 ; CHECK-ARM6-NEXT:  .LBB31_2: @ %atomicrmw.start
@@ -6447,8 +6447,8 @@ define i32 @test_umax_i32() {
 ; CHECK-THUMB7-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB31_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #1
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it hi
 ; CHECK-THUMB7-NEXT:    movhi r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i32
@@ -6566,8 +6566,8 @@ define i32 @test_umin_i32() {
 ; CHECK-ARM8-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM8-NEXT:    @ Child Loop BB32_2 Depth 2
 ; CHECK-ARM8-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    cmp r1, #2
+; CHECK-ARM8-NEXT:    mov r12, #1
 ; CHECK-ARM8-NEXT:    movlo r12, r1
 ; CHECK-ARM8-NEXT:    movw r3, :lower16:atomic_i32
 ; CHECK-ARM8-NEXT:    movt r3, :upper16:atomic_i32
@@ -6609,8 +6609,8 @@ define i32 @test_umin_i32() {
 ; CHECK-ARM6-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-ARM6-NEXT:    @ Child Loop BB32_2 Depth 2
 ; CHECK-ARM6-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    cmp r1, #2
+; CHECK-ARM6-NEXT:    mov r12, #1
 ; CHECK-ARM6-NEXT:    movlo r12, r1
 ; CHECK-ARM6-NEXT:    ldr r3, .LCPI32_0
 ; CHECK-ARM6-NEXT:  .LBB32_2: @ %atomicrmw.start
@@ -6656,8 +6656,8 @@ define i32 @test_umin_i32() {
 ; CHECK-THUMB7-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-THUMB7-NEXT:    @ Child Loop BB32_2 Depth 2
 ; CHECK-THUMB7-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    cmp r1, #2
+; CHECK-THUMB7-NEXT:    mov.w r12, #1
 ; CHECK-THUMB7-NEXT:    it lo
 ; CHECK-THUMB7-NEXT:    movlo r12, r1
 ; CHECK-THUMB7-NEXT:    movw r3, :lower16:atomic_i32
@@ -8342,8 +8342,7 @@ define i64 @test_max_i64() {
 ; CHECK-ARM8-NEXT:    mov r0, #0
 ; CHECK-ARM8-NEXT:    movwlt r0, #1
 ; CHECK-ARM8-NEXT:    mov r10, #1
-; CHECK-ARM8-NEXT:    cmp r0, #0
-; CHECK-ARM8-NEXT:    movne r10, r2
+; CHECK-ARM8-NEXT:    movlt r10, r2
 ; CHECK-ARM8-NEXT:    cmp r0, #0
 ; CHECK-ARM8-NEXT:    movne r0, r1
 ; CHECK-ARM8-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -8410,8 +8409,7 @@ define i64 @test_max_i64() {
 ; CHECK-ARM6-NEXT:    mov r0, #0
 ; CHECK-ARM6-NEXT:    movlt r0, #1
 ; CHECK-ARM6-NEXT:    mov r10, #1
-; CHECK-ARM6-NEXT:    cmp r0, #0
-; CHECK-ARM6-NEXT:    movne r10, r2
+; CHECK-ARM6-NEXT:    movlt r10, r2
 ; CHECK-ARM6-NEXT:    cmp r0, #0
 ; CHECK-ARM6-NEXT:    movne r0, r1
 ; CHECK-ARM6-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -8483,9 +8481,8 @@ define i64 @test_max_i64() {
 ; CHECK-THUMB7-NEXT:    mov r8, r2
 ; CHECK-THUMB7-NEXT:    mov r9, r1
 ; CHECK-THUMB7-NEXT:    mov.w r10, #1
-; CHECK-THUMB7-NEXT:    cmp r0, #0
-; CHECK-THUMB7-NEXT:    it ne
-; CHECK-THUMB7-NEXT:    movne r10, r2
+; CHECK-THUMB7-NEXT:    it lt
+; CHECK-THUMB7-NEXT:    movlt r10, r2
 ; CHECK-THUMB7-NEXT:    cmp r0, #0
 ; CHECK-THUMB7-NEXT:    it ne
 ; CHECK-THUMB7-NEXT:    movne r0, r1
@@ -8581,7 +8578,7 @@ define i64 @test_max_i64() {
 ; CHECK-THUMB8BASE-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
 ; CHECK-THUMB8BASE-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-THUMB8BASE-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; CHECK-THUMB8BASE-NEXT:    cbnz r1, .LBB40_5
+; CHECK-THUMB8BASE-NEXT:    blt .LBB40_5
 ; CHECK-THUMB8BASE-NEXT:  @ %bb.4: @ %atomicrmw.start
 ; CHECK-THUMB8BASE-NEXT:    @ in Loop: Header=BB40_1 Depth=1
 ; CHECK-THUMB8BASE-NEXT:    ldr r0, [sp, #48] @ 4-byte Reload
@@ -8658,8 +8655,7 @@ define i64 @test_min_i64() {
 ; CHECK-ARM8-NEXT:    mov r0, #0
 ; CHECK-ARM8-NEXT:    movwlt r0, #1
 ; CHECK-ARM8-NEXT:    mov r10, #1
-; CHECK-ARM8-NEXT:    cmp r0, #0
-; CHECK-ARM8-NEXT:    movne r10, r2
+; CHECK-ARM8-NEXT:    movlt r10, r2
 ; CHECK-ARM8-NEXT:    cmp r0, #0
 ; CHECK-ARM8-NEXT:    movne r0, r1
 ; CHECK-ARM8-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -8726,8 +8722,7 @@ define i64 @test_min_i64() {
 ; CHECK-ARM6-NEXT:    mov r0, #0
 ; CHECK-ARM6-NEXT:    movlt r0, #1
 ; CHECK-ARM6-NEXT:    mov r10, #1
-; CHECK-ARM6-NEXT:    cmp r0, #0
-; CHECK-ARM6-NEXT:    movne r10, r2
+; CHECK-ARM6-NEXT:    movlt r10, r2
 ; CHECK-ARM6-NEXT:    cmp r0, #0
 ; CHECK-ARM6-NEXT:    movne r0, r1
 ; CHECK-ARM6-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -8799,9 +8794,8 @@ define i64 @test_min_i64() {
 ; CHECK-THUMB7-NEXT:    it lt
 ; CHECK-THUMB7-NEXT:    movlt r0, #1
 ; CHECK-THUMB7-NEXT:    mov.w r10, #1
-; CHECK-THUMB7-NEXT:    cmp r0, #0
-; CHECK-THUMB7-NEXT:    it ne
-; CHECK-THUMB7-NEXT:    movne r10, r2
+; CHECK-THUMB7-NEXT:    it lt
+; CHECK-THUMB7-NEXT:    movlt r10, r2
 ; CHECK-THUMB7-NEXT:    cmp r0, #0
 ; CHECK-THUMB7-NEXT:    it ne
 ; CHECK-THUMB7-NEXT:    movne r0, r1
@@ -8897,7 +8891,7 @@ define i64 @test_min_i64() {
 ; CHECK-THUMB8BASE-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
 ; CHECK-THUMB8BASE-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-THUMB8BASE-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; CHECK-THUMB8BASE-NEXT:    cbnz r1, .LBB41_5
+; CHECK-THUMB8BASE-NEXT:    blt .LBB41_5
 ; CHECK-THUMB8BASE-NEXT:  @ %bb.4: @ %atomicrmw.start
 ; CHECK-THUMB8BASE-NEXT:    @ in Loop: Header=BB41_1 Depth=1
 ; CHECK-THUMB8BASE-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
@@ -8974,8 +8968,7 @@ define i64 @test_umax_i64() {
 ; CHECK-ARM8-NEXT:    mov r0, #0
 ; CHECK-ARM8-NEXT:    movwlo r0, #1
 ; CHECK-ARM8-NEXT:    mov r10, #1
-; CHECK-ARM8-NEXT:    cmp r0, #0
-; CHECK-ARM8-NEXT:    movne r10, r2
+; CHECK-ARM8-NEXT:    movlo r10, r2
 ; CHECK-ARM8-NEXT:    cmp r0, #0
 ; CHECK-ARM8-NEXT:    movne r0, r1
 ; CHECK-ARM8-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -9042,8 +9035,7 @@ define i64 @test_umax_i64() {
 ; CHECK-ARM6-NEXT:    mov r0, #0
 ; CHECK-ARM6-NEXT:    movlo r0, #1
 ; CHECK-ARM6-NEXT:    mov r10, #1
-; CHECK-ARM6-NEXT:    cmp r0, #0
-; CHECK-ARM6-NEXT:    movne r10, r2
+; CHECK-ARM6-NEXT:    movlo r10, r2
 ; CHECK-ARM6-NEXT:    cmp r0, #0
 ; CHECK-ARM6-NEXT:    movne r0, r1
 ; CHECK-ARM6-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -9115,9 +9107,8 @@ define i64 @test_umax_i64() {
 ; CHECK-THUMB7-NEXT:    mov r8, r2
 ; CHECK-THUMB7-NEXT:    mov r9, r1
 ; CHECK-THUMB7-NEXT:    mov.w r10, #1
-; CHECK-THUMB7-NEXT:    cmp r0, #0
-; CHECK-THUMB7-NEXT:    it ne
-; CHECK-THUMB7-NEXT:    movne r10, r2
+; CHECK-THUMB7-NEXT:    it lo
+; CHECK-THUMB7-NEXT:    movlo r10, r2
 ; CHECK-THUMB7-NEXT:    cmp r0, #0
 ; CHECK-THUMB7-NEXT:    it ne
 ; CHECK-THUMB7-NEXT:    movne r0, r1
@@ -9213,7 +9204,7 @@ define i64 @test_umax_i64() {
 ; CHECK-THUMB8BASE-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
 ; CHECK-THUMB8BASE-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-THUMB8BASE-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; CHECK-THUMB8BASE-NEXT:    cbnz r1, .LBB42_5
+; CHECK-THUMB8BASE-NEXT:    blo .LBB42_5
 ; CHECK-THUMB8BASE-NEXT:  @ %bb.4: @ %atomicrmw.start
 ; CHECK-THUMB8BASE-NEXT:    @ in Loop: Header=BB42_1 Depth=1
 ; CHECK-THUMB8BASE-NEXT:    ldr r0, [sp, #48] @ 4-byte Reload
@@ -9290,8 +9281,7 @@ define i64 @test_umin_i64() {
 ; CHECK-ARM8-NEXT:    mov r0, #0
 ; CHECK-ARM8-NEXT:    movwlo r0, #1
 ; CHECK-ARM8-NEXT:    mov r10, #1
-; CHECK-ARM8-NEXT:    cmp r0, #0
-; CHECK-ARM8-NEXT:    movne r10, r2
+; CHECK-ARM8-NEXT:    movlo r10, r2
 ; CHECK-ARM8-NEXT:    cmp r0, #0
 ; CHECK-ARM8-NEXT:    movne r0, r1
 ; CHECK-ARM8-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -9358,8 +9348,7 @@ define i64 @test_umin_i64() {
 ; CHECK-ARM6-NEXT:    mov r0, #0
 ; CHECK-ARM6-NEXT:    movlo r0, #1
 ; CHECK-ARM6-NEXT:    mov r10, #1
-; CHECK-ARM6-NEXT:    cmp r0, #0
-; CHECK-ARM6-NEXT:    movne r10, r2
+; CHECK-ARM6-NEXT:    movlo r10, r2
 ; CHECK-ARM6-NEXT:    cmp r0, #0
 ; CHECK-ARM6-NEXT:    movne r0, r1
 ; CHECK-ARM6-NEXT:    @ kill: def $r10 killed $r10 def $r10_r11
@@ -9431,9 +9420,8 @@ define i64 @test_umin_i64() {
 ; CHECK-THUMB7-NEXT:    it lo
 ; CHECK-THUMB7-NEXT:    movlo r0, #1
 ; CHECK-THUMB7-NEXT:    mov.w r10, #1
-; CHECK-THUMB7-NEXT:    cmp r0, #0
-; CHECK-THUMB7-NEXT:    it ne
-; CHECK-THUMB7-NEXT:    movne r10, r2
+; CHECK-THUMB7-NEXT:    it lo
+; CHECK-THUMB7-NEXT:    movlo r10, r2
 ; CHECK-THUMB7-NEXT:    cmp r0, #0
 ; CHECK-THUMB7-NEXT:    it ne
 ; CHECK-THUMB7-NEXT:    movne r0, r1
@@ -9529,7 +9517,7 @@ define i64 @test_umin_i64() {
 ; CHECK-THUMB8BASE-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
 ; CHECK-THUMB8BASE-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-THUMB8BASE-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; CHECK-THUMB8BASE-NEXT:    cbnz r1, .LBB43_5
+; CHECK-THUMB8BASE-NEXT:    blo .LBB43_5
 ; CHECK-THUMB8BASE-NEXT:  @ %bb.4: @ %atomicrmw.start
 ; CHECK-THUMB8BASE-NEXT:    @ in Loop: Header=BB43_1 Depth=1
 ; CHECK-THUMB8BASE-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
diff --git a/llvm/test/CodeGen/ARM/bfi.ll b/llvm/test/CodeGen/ARM/bfi.ll
index 91a74e535a221..5aeb99695a5fe 100644
--- a/llvm/test/CodeGen/ARM/bfi.ll
+++ b/llvm/test/CodeGen/ARM/bfi.ll
@@ -204,10 +204,11 @@ define i32 @f12(i32 %x, i32 %y) {
 define i32 @f13(i32 %x, i32 %y) {
 ; CHECK-LABEL: f13:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    and r2, r0, #4
-; CHECK-NEXT:    bic r0, r1, #255
-; CHECK-NEXT:    cmp r2, #42
-; CHECK-NEXT:    orrne r0, r0, #16
+; CHECK-NEXT:    and r0, r0, #4
+; CHECK-NEXT:    bic r1, r1, #255
+; CHECK-NEXT:    cmp r0, #42
+; CHECK-NEXT:    orrne r1, r1, #16
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bx lr
   %y2 = and i32 %y, 4294967040 ; 0xFFFFFF00
   %and = and i32 %x, 4
diff --git a/llvm/test/CodeGen/ARM/build-attributes.ll b/llvm/test/CodeGen/ARM/build-attributes.ll
index 9220f73d94f8b..68844aed03630 100644
--- a/llvm/test/CodeGen/ARM/build-attributes.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes.ll
@@ -187,6 +187,12 @@
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m5 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=exynos-m5 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 
+; ARMv9a (AArch32)
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a510 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a510 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a710 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a710 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+
 ; ARMv7a
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
diff --git a/llvm/test/CodeGen/ARM/cmov_fp16.ll b/llvm/test/CodeGen/ARM/cmov_fp16.ll
index 805955d3e8306..fb8da3724ede8 100644
--- a/llvm/test/CodeGen/ARM/cmov_fp16.ll
+++ b/llvm/test/CodeGen/ARM/cmov_fp16.ll
@@ -5,12 +5,12 @@
 define i32 @test_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_ne:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -26,12 +26,12 @@ entry:
 define i32 @test_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_eq:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov s0, r1
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -47,12 +47,12 @@ entry:
 define i32 @test_gt(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_gt:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov s0, r1
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vselgt.f16 s0, s0, s2
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -68,12 +68,12 @@ entry:
 define i32 @test_ge(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_ge:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vmov s0, r1
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r1
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -89,12 +89,12 @@ entry:
 define i32 @test_lt(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_lt:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -110,12 +110,12 @@ entry:
 define i32 @test_le(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_le:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    cmp r2, r3
-; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-NEXT:    vselgt.f16 s0, s0, s2
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -131,25 +131,25 @@ entry:
 define i32 @test_hi(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_hi:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vmov s2, r1
 ; CHECK-THUMB-NEXT:    cmp r2, r3
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vmov s0, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-THUMB-NEXT:    it hi
-; CHECK-THUMB-NEXT:    vmovhi.f32 s0, s2
-; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    vmovhi.f32 s2, s0
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s2
 ; CHECK-THUMB-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: test_hi:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vmov s0, r0
 ; CHECK-ARM-NEXT:    cmp r2, r3
-; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    vmov s2, r1
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-ARM-NEXT:    vmovhi.f32 s0, s2
-; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    vmovhi.f32 s2, s0
+; CHECK-ARM-NEXT:    vmov.f16 r0, s2
 ; CHECK-ARM-NEXT:    bx lr
 entry:
   %x.half = uitofp i32 %x to half
@@ -164,25 +164,25 @@ entry:
 define i32 @test_hs(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_hs:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vmov s2, r1
 ; CHECK-THUMB-NEXT:    cmp r2, r3
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vmov s0, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-THUMB-NEXT:    it hs
-; CHECK-THUMB-NEXT:    vmovhs.f32 s0, s2
-; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    vmovhs.f32 s2, s0
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s2
 ; CHECK-THUMB-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: test_hs:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vmov s0, r0
 ; CHECK-ARM-NEXT:    cmp r2, r3
-; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    vmov s2, r1
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-ARM-NEXT:    vmovhs.f32 s0, s2
-; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    vmovhs.f32 s2, s0
+; CHECK-ARM-NEXT:    vmov.f16 r0, s2
 ; CHECK-ARM-NEXT:    bx lr
 entry:
   %x.half = uitofp i32 %x to half
@@ -197,25 +197,25 @@ entry:
 define i32 @test_lo(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_lo:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vmov s2, r1
 ; CHECK-THUMB-NEXT:    cmp r2, r3
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vmov s0, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-THUMB-NEXT:    it lo
-; CHECK-THUMB-NEXT:    vmovlo.f32 s0, s2
-; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    vmovlo.f32 s2, s0
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s2
 ; CHECK-THUMB-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: test_lo:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vmov s0, r0
 ; CHECK-ARM-NEXT:    cmp r2, r3
-; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    vmov s2, r1
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-ARM-NEXT:    vmovlo.f32 s0, s2
-; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    vmovlo.f32 s2, s0
+; CHECK-ARM-NEXT:    vmov.f16 r0, s2
 ; CHECK-ARM-NEXT:    bx lr
 entry:
   %x.half = uitofp i32 %x to half
@@ -230,25 +230,25 @@ entry:
 define i32 @test_ls(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_ls:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s0, r1
+; CHECK-THUMB-NEXT:    vmov s2, r1
 ; CHECK-THUMB-NEXT:    cmp r2, r3
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vmov s0, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-THUMB-NEXT:    it ls
-; CHECK-THUMB-NEXT:    vmovls.f32 s0, s2
-; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
+; CHECK-THUMB-NEXT:    vmovls.f32 s2, s0
+; CHECK-THUMB-NEXT:    vmov.f16 r0, s2
 ; CHECK-THUMB-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: test_ls:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vmov s0, r0
 ; CHECK-ARM-NEXT:    cmp r2, r3
-; CHECK-ARM-NEXT:    vmov s2, r0
+; CHECK-ARM-NEXT:    vmov s2, r1
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
-; CHECK-ARM-NEXT:    vmovls.f32 s0, s2
-; CHECK-ARM-NEXT:    vmov.f16 r0, s0
+; CHECK-ARM-NEXT:    vmovls.f32 s2, s0
+; CHECK-ARM-NEXT:    vmov.f16 r0, s2
 ; CHECK-ARM-NEXT:    bx lr
 entry:
   %x.half = uitofp i32 %x to half
diff --git a/llvm/test/CodeGen/ARM/cse-call.ll b/llvm/test/CodeGen/ARM/cse-call.ll
index 71cfa3b9da930..25fa477e5c2d3 100644
--- a/llvm/test/CodeGen/ARM/cse-call.ll
+++ b/llvm/test/CodeGen/ARM/cse-call.ll
@@ -25,7 +25,9 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 ; CHECK-T1: cmp
 
 ; CHECK: S_trimzeros
-; CHECK: cmp
+; CHECK-T1: S_trimzeros
+; CHECK-NOT: moveq
+; CHECK-T1-NOT: beq
 ; CHECK: strlen
 
 @F_floatmul.man1 = external global [200 x i8], align 1
diff --git a/llvm/test/CodeGen/ARM/cttz.ll b/llvm/test/CodeGen/ARM/cttz.ll
index d9663a1c148fc..76adc61c5971f 100644
--- a/llvm/test/CodeGen/ARM/cttz.ll
+++ b/llvm/test/CodeGen/ARM/cttz.ll
@@ -223,42 +223,39 @@ define i64 @test_i64(i64 %a) {
 ; CHECK-6M:       @ %bb.0:
 ; CHECK-6M-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-6M-NEXT:    push {r4, r5, r7, lr}
+; CHECK-6M-NEXT:    mov r2, r0
 ; CHECK-6M-NEXT:    ldr r5, .LCPI3_0
-; CHECK-6M-NEXT:    adr r4, .LCPI3_1
-; CHECK-6M-NEXT:    movs r3, #32
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    mov r2, r3
-; CHECK-6M-NEXT:    bne .LBB3_5
-; CHECK-6M-NEXT:  @ %bb.1:
+; CHECK-6M-NEXT:    adr r3, .LCPI3_1
+; CHECK-6M-NEXT:    movs r0, #32
 ; CHECK-6M-NEXT:    cmp r1, #0
-; CHECK-6M-NEXT:    bne .LBB3_6
+; CHECK-6M-NEXT:    mov r4, r0
+; CHECK-6M-NEXT:    beq .LBB3_2
+; CHECK-6M-NEXT:  @ %bb.1:
+; CHECK-6M-NEXT:    rsbs r4, r1, #0
+; CHECK-6M-NEXT:    ands r4, r1
+; CHECK-6M-NEXT:    muls r4, r5, r4
+; CHECK-6M-NEXT:    lsrs r1, r4, #27
+; CHECK-6M-NEXT:    ldrb r4, [r3, r1]
 ; CHECK-6M-NEXT:  .LBB3_2:
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    bne .LBB3_4
-; CHECK-6M-NEXT:  .LBB3_3:
-; CHECK-6M-NEXT:    adds r3, #32
-; CHECK-6M-NEXT:    mov r2, r3
+; CHECK-6M-NEXT:    adds r4, #32
+; CHECK-6M-NEXT:    rsbs r1, r2, #0
+; CHECK-6M-NEXT:    ands r1, r2
+; CHECK-6M-NEXT:    muls r5, r1, r5
+; CHECK-6M-NEXT:    lsrs r1, r5, #27
+; CHECK-6M-NEXT:    cmp r2, #0
+; CHECK-6M-NEXT:    bne .LBB3_5
+; CHECK-6M-NEXT:  @ %bb.3:
+; CHECK-6M-NEXT:    beq .LBB3_6
 ; CHECK-6M-NEXT:  .LBB3_4:
 ; CHECK-6M-NEXT:    movs r1, #0
-; CHECK-6M-NEXT:    mov r0, r2
 ; CHECK-6M-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-6M-NEXT:  .LBB3_5:
-; CHECK-6M-NEXT:    rsbs r2, r0, #0
-; CHECK-6M-NEXT:    ands r2, r0
-; CHECK-6M-NEXT:    muls r2, r5, r2
-; CHECK-6M-NEXT:    lsrs r2, r2, #27
-; CHECK-6M-NEXT:    ldrb r2, [r4, r2]
-; CHECK-6M-NEXT:    cmp r1, #0
-; CHECK-6M-NEXT:    beq .LBB3_2
+; CHECK-6M-NEXT:    ldrb r0, [r3, r1]
+; CHECK-6M-NEXT:    bne .LBB3_4
 ; CHECK-6M-NEXT:  .LBB3_6:
-; CHECK-6M-NEXT:    rsbs r3, r1, #0
-; CHECK-6M-NEXT:    ands r3, r1
-; CHECK-6M-NEXT:    muls r5, r3, r5
-; CHECK-6M-NEXT:    lsrs r1, r5, #27
-; CHECK-6M-NEXT:    ldrb r3, [r4, r1]
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    beq .LBB3_3
-; CHECK-6M-NEXT:    b .LBB3_4
+; CHECK-6M-NEXT:    mov r0, r4
+; CHECK-6M-NEXT:    movs r1, #0
+; CHECK-6M-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-6M-NEXT:    .p2align 2
 ; CHECK-6M-NEXT:  @ %bb.7:
 ; CHECK-6M-NEXT:  .LCPI3_0:
@@ -270,40 +267,39 @@ define i64 @test_i64(i64 %a) {
 ; CHECK-8MBASE:       @ %bb.0:
 ; CHECK-8MBASE-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-8MBASE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-8MBASE-NEXT:    mov r2, r0
 ; CHECK-8MBASE-NEXT:    movw r5, #46385
 ; CHECK-8MBASE-NEXT:    movt r5, #1916
-; CHECK-8MBASE-NEXT:    adr r4, .LCPI3_0
-; CHECK-8MBASE-NEXT:    movs r3, #32
-; CHECK-8MBASE-NEXT:    mov r2, r3
-; CHECK-8MBASE-NEXT:    cbnz r0, .LBB3_5
+; CHECK-8MBASE-NEXT:    adr r3, .LCPI3_0
+; CHECK-8MBASE-NEXT:    movs r0, #32
+; CHECK-8MBASE-NEXT:    mov r4, r0
+; CHECK-8MBASE-NEXT:    cbz r1, .LBB3_2
 ; CHECK-8MBASE-NEXT:  @ %bb.1:
-; CHECK-8MBASE-NEXT:    cbnz r1, .LBB3_6
+; CHECK-8MBASE-NEXT:    rsbs r4, r1, #0
+; CHECK-8MBASE-NEXT:    ands r4, r1
+; CHECK-8MBASE-NEXT:    muls r4, r5, r4
+; CHECK-8MBASE-NEXT:    lsrs r1, r4, #27
+; CHECK-8MBASE-NEXT:    ldrb r4, [r3, r1]
 ; CHECK-8MBASE-NEXT:  .LBB3_2:
-; CHECK-8MBASE-NEXT:    cbnz r0, .LBB3_4
-; CHECK-8MBASE-NEXT:  .LBB3_3:
-; CHECK-8MBASE-NEXT:    adds r3, #32
-; CHECK-8MBASE-NEXT:    mov r2, r3
+; CHECK-8MBASE-NEXT:    adds r4, #32
+; CHECK-8MBASE-NEXT:    rsbs r1, r2, #0
+; CHECK-8MBASE-NEXT:    ands r1, r2
+; CHECK-8MBASE-NEXT:    muls r5, r1, r5
+; CHECK-8MBASE-NEXT:    lsrs r1, r5, #27
+; CHECK-8MBASE-NEXT:    cmp r2, #0
+; CHECK-8MBASE-NEXT:    bne .LBB3_5
+; CHECK-8MBASE-NEXT:  @ %bb.3:
+; CHECK-8MBASE-NEXT:    beq .LBB3_6
 ; CHECK-8MBASE-NEXT:  .LBB3_4:
 ; CHECK-8MBASE-NEXT:    movs r1, #0
-; CHECK-8MBASE-NEXT:    mov r0, r2
 ; CHECK-8MBASE-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-8MBASE-NEXT:  .LBB3_5:
-; CHECK-8MBASE-NEXT:    rsbs r2, r0, #0
-; CHECK-8MBASE-NEXT:    ands r2, r0
-; CHECK-8MBASE-NEXT:    muls r2, r5, r2
-; CHECK-8MBASE-NEXT:    lsrs r2, r2, #27
-; CHECK-8MBASE-NEXT:    ldrb r2, [r4, r2]
-; CHECK-8MBASE-NEXT:    cmp r1, #0
-; CHECK-8MBASE-NEXT:    beq .LBB3_2
+; CHECK-8MBASE-NEXT:    ldrb r0, [r3, r1]
+; CHECK-8MBASE-NEXT:    bne .LBB3_4
 ; CHECK-8MBASE-NEXT:  .LBB3_6:
-; CHECK-8MBASE-NEXT:    rsbs r3, r1, #0
-; CHECK-8MBASE-NEXT:    ands r3, r1
-; CHECK-8MBASE-NEXT:    muls r5, r3, r5
-; CHECK-8MBASE-NEXT:    lsrs r1, r5, #27
-; CHECK-8MBASE-NEXT:    ldrb r3, [r4, r1]
-; CHECK-8MBASE-NEXT:    cmp r0, #0
-; CHECK-8MBASE-NEXT:    beq .LBB3_3
-; CHECK-8MBASE-NEXT:    b .LBB3_4
+; CHECK-8MBASE-NEXT:    mov r0, r4
+; CHECK-8MBASE-NEXT:    movs r1, #0
+; CHECK-8MBASE-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-8MBASE-NEXT:    .p2align 2
 ; CHECK-8MBASE-NEXT:  @ %bb.7:
 ; CHECK-8MBASE-NEXT:  .LCPI3_0:
@@ -494,42 +490,39 @@ define i64 @test_i64_zero_undef(i64 %a) {
 ; CHECK-6M:       @ %bb.0:
 ; CHECK-6M-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-6M-NEXT:    push {r4, r5, r7, lr}
+; CHECK-6M-NEXT:    mov r2, r0
 ; CHECK-6M-NEXT:    ldr r5, .LCPI7_0
-; CHECK-6M-NEXT:    adr r4, .LCPI7_1
-; CHECK-6M-NEXT:    movs r3, #32
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    mov r2, r3
-; CHECK-6M-NEXT:    bne .LBB7_5
-; CHECK-6M-NEXT:  @ %bb.1:
+; CHECK-6M-NEXT:    adr r3, .LCPI7_1
+; CHECK-6M-NEXT:    movs r0, #32
 ; CHECK-6M-NEXT:    cmp r1, #0
-; CHECK-6M-NEXT:    bne .LBB7_6
+; CHECK-6M-NEXT:    mov r4, r0
+; CHECK-6M-NEXT:    beq .LBB7_2
+; CHECK-6M-NEXT:  @ %bb.1:
+; CHECK-6M-NEXT:    rsbs r4, r1, #0
+; CHECK-6M-NEXT:    ands r4, r1
+; CHECK-6M-NEXT:    muls r4, r5, r4
+; CHECK-6M-NEXT:    lsrs r1, r4, #27
+; CHECK-6M-NEXT:    ldrb r4, [r3, r1]
 ; CHECK-6M-NEXT:  .LBB7_2:
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    bne .LBB7_4
-; CHECK-6M-NEXT:  .LBB7_3:
-; CHECK-6M-NEXT:    adds r3, #32
-; CHECK-6M-NEXT:    mov r2, r3
+; CHECK-6M-NEXT:    adds r4, #32
+; CHECK-6M-NEXT:    rsbs r1, r2, #0
+; CHECK-6M-NEXT:    ands r1, r2
+; CHECK-6M-NEXT:    muls r5, r1, r5
+; CHECK-6M-NEXT:    lsrs r1, r5, #27
+; CHECK-6M-NEXT:    cmp r2, #0
+; CHECK-6M-NEXT:    bne .LBB7_5
+; CHECK-6M-NEXT:  @ %bb.3:
+; CHECK-6M-NEXT:    beq .LBB7_6
 ; CHECK-6M-NEXT:  .LBB7_4:
 ; CHECK-6M-NEXT:    movs r1, #0
-; CHECK-6M-NEXT:    mov r0, r2
 ; CHECK-6M-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-6M-NEXT:  .LBB7_5:
-; CHECK-6M-NEXT:    rsbs r2, r0, #0
-; CHECK-6M-NEXT:    ands r2, r0
-; CHECK-6M-NEXT:    muls r2, r5, r2
-; CHECK-6M-NEXT:    lsrs r2, r2, #27
-; CHECK-6M-NEXT:    ldrb r2, [r4, r2]
-; CHECK-6M-NEXT:    cmp r1, #0
-; CHECK-6M-NEXT:    beq .LBB7_2
+; CHECK-6M-NEXT:    ldrb r0, [r3, r1]
+; CHECK-6M-NEXT:    bne .LBB7_4
 ; CHECK-6M-NEXT:  .LBB7_6:
-; CHECK-6M-NEXT:    rsbs r3, r1, #0
-; CHECK-6M-NEXT:    ands r3, r1
-; CHECK-6M-NEXT:    muls r5, r3, r5
-; CHECK-6M-NEXT:    lsrs r1, r5, #27
-; CHECK-6M-NEXT:    ldrb r3, [r4, r1]
-; CHECK-6M-NEXT:    cmp r0, #0
-; CHECK-6M-NEXT:    beq .LBB7_3
-; CHECK-6M-NEXT:    b .LBB7_4
+; CHECK-6M-NEXT:    mov r0, r4
+; CHECK-6M-NEXT:    movs r1, #0
+; CHECK-6M-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-6M-NEXT:    .p2align 2
 ; CHECK-6M-NEXT:  @ %bb.7:
 ; CHECK-6M-NEXT:  .LCPI7_0:
@@ -541,40 +534,39 @@ define i64 @test_i64_zero_undef(i64 %a) {
 ; CHECK-8MBASE:       @ %bb.0:
 ; CHECK-8MBASE-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-8MBASE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-8MBASE-NEXT:    mov r2, r0
 ; CHECK-8MBASE-NEXT:    movw r5, #46385
 ; CHECK-8MBASE-NEXT:    movt r5, #1916
-; CHECK-8MBASE-NEXT:    adr r4, .LCPI7_0
-; CHECK-8MBASE-NEXT:    movs r3, #32
-; CHECK-8MBASE-NEXT:    mov r2, r3
-; CHECK-8MBASE-NEXT:    cbnz r0, .LBB7_5
+; CHECK-8MBASE-NEXT:    adr r3, .LCPI7_0
+; CHECK-8MBASE-NEXT:    movs r0, #32
+; CHECK-8MBASE-NEXT:    mov r4, r0
+; CHECK-8MBASE-NEXT:    cbz r1, .LBB7_2
 ; CHECK-8MBASE-NEXT:  @ %bb.1:
-; CHECK-8MBASE-NEXT:    cbnz r1, .LBB7_6
+; CHECK-8MBASE-NEXT:    rsbs r4, r1, #0
+; CHECK-8MBASE-NEXT:    ands r4, r1
+; CHECK-8MBASE-NEXT:    muls r4, r5, r4
+; CHECK-8MBASE-NEXT:    lsrs r1, r4, #27
+; CHECK-8MBASE-NEXT:    ldrb r4, [r3, r1]
 ; CHECK-8MBASE-NEXT:  .LBB7_2:
-; CHECK-8MBASE-NEXT:    cbnz r0, .LBB7_4
-; CHECK-8MBASE-NEXT:  .LBB7_3:
-; CHECK-8MBASE-NEXT:    adds r3, #32
-; CHECK-8MBASE-NEXT:    mov r2, r3
+; CHECK-8MBASE-NEXT:    adds r4, #32
+; CHECK-8MBASE-NEXT:    rsbs r1, r2, #0
+; CHECK-8MBASE-NEXT:    ands r1, r2
+; CHECK-8MBASE-NEXT:    muls r5, r1, r5
+; CHECK-8MBASE-NEXT:    lsrs r1, r5, #27
+; CHECK-8MBASE-NEXT:    cmp r2, #0
+; CHECK-8MBASE-NEXT:    bne .LBB7_5
+; CHECK-8MBASE-NEXT:  @ %bb.3:
+; CHECK-8MBASE-NEXT:    beq .LBB7_6
 ; CHECK-8MBASE-NEXT:  .LBB7_4:
 ; CHECK-8MBASE-NEXT:    movs r1, #0
-; CHECK-8MBASE-NEXT:    mov r0, r2
 ; CHECK-8MBASE-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-8MBASE-NEXT:  .LBB7_5:
-; CHECK-8MBASE-NEXT:    rsbs r2, r0, #0
-; CHECK-8MBASE-NEXT:    ands r2, r0
-; CHECK-8MBASE-NEXT:    muls r2, r5, r2
-; CHECK-8MBASE-NEXT:    lsrs r2, r2, #27
-; CHECK-8MBASE-NEXT:    ldrb r2, [r4, r2]
-; CHECK-8MBASE-NEXT:    cmp r1, #0
-; CHECK-8MBASE-NEXT:    beq .LBB7_2
+; CHECK-8MBASE-NEXT:    ldrb r0, [r3, r1]
+; CHECK-8MBASE-NEXT:    bne .LBB7_4
 ; CHECK-8MBASE-NEXT:  .LBB7_6:
-; CHECK-8MBASE-NEXT:    rsbs r3, r1, #0
-; CHECK-8MBASE-NEXT:    ands r3, r1
-; CHECK-8MBASE-NEXT:    muls r5, r3, r5
-; CHECK-8MBASE-NEXT:    lsrs r1, r5, #27
-; CHECK-8MBASE-NEXT:    ldrb r3, [r4, r1]
-; CHECK-8MBASE-NEXT:    cmp r0, #0
-; CHECK-8MBASE-NEXT:    beq .LBB7_3
-; CHECK-8MBASE-NEXT:    b .LBB7_4
+; CHECK-8MBASE-NEXT:    mov r0, r4
+; CHECK-8MBASE-NEXT:    movs r1, #0
+; CHECK-8MBASE-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-8MBASE-NEXT:    .p2align 2
 ; CHECK-8MBASE-NEXT:  @ %bb.7:
 ; CHECK-8MBASE-NEXT:  .LCPI7_0:
diff --git a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
index 3f2b40460917e..90bb02fdc0cd4 100644
--- a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
+++ b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
@@ -4,12 +4,12 @@
 define float @fadd_select_fneg_fneg_f32(i32 %arg0, float %x, float %y, float %z) {
 ; CHECK-LABEL: fadd_select_fneg_fneg_f32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov s0, r3
+; CHECK-NEXT:    vmov s0, r2
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vmov s2, r2
 ; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    vseleq.f32 s2, s4, s2
-; CHECK-NEXT:    vsub.f32 s0, s0, s2
+; CHECK-NEXT:    vmov s2, r3
+; CHECK-NEXT:    vseleq.f32 s0, s4, s0
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bx lr
   %cmp = icmp eq i32 %arg0, 0
@@ -248,10 +248,10 @@ define half @fadd_select_fsub_select_f16(i32 %arg0, half %x, half %y, half %z) {
 define half @fadd_select_fneg_negk_f16(i32 %arg0, half %x, half %y) {
 ; CHECK-LABEL: fadd_select_fneg_negk_f16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.f16 s0, r1
-; CHECK-NEXT:    vmov.f16 s2, #4.000000e+00
+; CHECK-NEXT:    vmov.f16 s0, #4.000000e+00
+; CHECK-NEXT:    vmov.f16 s2, r1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 s2, r2
 ; CHECK-NEXT:    vsub.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov r0, s0
@@ -266,10 +266,10 @@ define half @fadd_select_fneg_negk_f16(i32 %arg0, half %x, half %y) {
 define half @fadd_select_fneg_posk_f16(i32 %arg0, half %x, half %y) {
 ; CHECK-LABEL: fadd_select_fneg_posk_f16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.f16 s0, r1
-; CHECK-NEXT:    vmov.f16 s2, #-4.000000e+00
+; CHECK-NEXT:    vmov.f16 s0, #-4.000000e+00
+; CHECK-NEXT:    vmov.f16 s2, r1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vseleq.f16 s0, s0, s2
+; CHECK-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov.f16 s2, r2
 ; CHECK-NEXT:    vsub.f16 s0, s2, s0
 ; CHECK-NEXT:    vmov r0, s0
diff --git a/llvm/test/CodeGen/ARM/fcmp-xo.ll b/llvm/test/CodeGen/ARM/fcmp-xo.ll
index 908dbd7a11a6b..ad39cb744620d 100644
--- a/llvm/test/CodeGen/ARM/fcmp-xo.ll
+++ b/llvm/test/CodeGen/ARM/fcmp-xo.ll
@@ -69,12 +69,12 @@ define arm_aapcs_vfpcc float @float128(float %a0) local_unnamed_addr {
 define arm_aapcs_vfpcc double @double1(double %a0) local_unnamed_addr {
 ; CHECK-LABEL: double1:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.f64 d18, #1.000000e+00
-; CHECK-NEXT:    vcmp.f64 d18, d0
+; CHECK-NEXT:    vmov.f64 d16, #1.000000e+00
+; CHECK-NEXT:    vcmp.f64 d16, d0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
-; CHECK-NEXT:    vmov.f64 d17, #-5.000000e-01
-; CHECK-NEXT:    vselgt.f64 d0, d17, d16
+; CHECK-NEXT:    vmov.f64 d17, #5.000000e-01
+; CHECK-NEXT:    vmov.f64 d18, #-5.000000e-01
+; CHECK-NEXT:    vselgt.f64 d0, d18, d17
 ; CHECK-NEXT:    bx lr
   %1 = fcmp nsz olt double %a0, 1.000000e+00
   %2 = select i1 %1, double -5.000000e-01, double 5.000000e-01
@@ -87,12 +87,12 @@ define arm_aapcs_vfpcc double @double128(double %a0) local_unnamed_addr {
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    movt r0, #16480
-; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
-; CHECK-NEXT:    vmov d18, r1, r0
-; CHECK-NEXT:    vcmp.f64 d18, d0
+; CHECK-NEXT:    vmov.f64 d17, #5.000000e-01
+; CHECK-NEXT:    vmov d16, r1, r0
+; CHECK-NEXT:    vcmp.f64 d16, d0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vmov.f64 d17, #-5.000000e-01
-; CHECK-NEXT:    vselgt.f64 d0, d17, d16
+; CHECK-NEXT:    vmov.f64 d18, #-5.000000e-01
+; CHECK-NEXT:    vselgt.f64 d0, d18, d17
 ; CHECK-NEXT:    bx lr
   %1 = fcmp nsz olt double %a0, 128.000000e+00
   %2 = select i1 %1, double -5.000000e-01, double 5.000000e-01
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat.ll b/llvm/test/CodeGen/ARM/fpclamptosat.ll
index 8bd8aa7b34dec..478b98dfac80f 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat.ll
@@ -8,27 +8,24 @@
 define i32 @stest_f64i32(double %x) {
 ; SOFT-LABEL: stest_f64i32:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
-; SOFT-NEXT:    ldr r4, .LCPI0_0
-; SOFT-NEXT:    subs r5, r0, r4
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
-; SOFT-NEXT:    bge .LBB0_7
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    ldr r3, .LCPI0_0
+; SOFT-NEXT:    subs r4, r0, r3
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB0_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB0_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB0_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB0_4
-; SOFT-NEXT:  .LBB0_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB0_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB0_4: @ %entry
-; SOFT-NEXT:    mvns r3, r3
+; SOFT-NEXT:    mvns r3, r2
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    lsls r2, r2, #31
 ; SOFT-NEXT:    subs r4, r2, r0
 ; SOFT-NEXT:    sbcs r3, r1
@@ -36,18 +33,9 @@ define i32 @stest_f64i32(double %x) {
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:  .LBB0_6: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
-; SOFT-NEXT:  .LBB0_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB0_2
-; SOFT-NEXT:  .LBB0_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB0_3
-; SOFT-NEXT:    b .LBB0_4
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:  @ %bb.7:
 ; SOFT-NEXT:  .LCPI0_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -135,45 +123,33 @@ define i32 @ustest_f64i32(double %x) {
 ; SOFT-NEXT:    .save {r4, lr}
 ; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    mvns r3, r2
 ; SOFT-NEXT:    adds r4, r0, #1
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    sbcs r4, r3
-; SOFT-NEXT:    mov r4, r2
-; SOFT-NEXT:    bge .LBB2_7
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB2_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB2_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB2_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB2_4
-; SOFT-NEXT:  .LBB2_3: @ %entry
-; SOFT-NEXT:    mvns r0, r3
+; SOFT-NEXT:    blt .LBB2_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB2_4: @ %entry
-; SOFT-NEXT:    rsbs r4, r0, #0
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    sbcs r4, r1
-; SOFT-NEXT:    bge .LBB2_9
+; SOFT-NEXT:    rsbs r3, r0, #0
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:    sbcs r3, r1
+; SOFT-NEXT:    blt .LBB2_7
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB2_10
+; SOFT-NEXT:    beq .LBB2_8
 ; SOFT-NEXT:  .LBB2_6: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB2_7: @ %entry
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB2_2
-; SOFT-NEXT:  .LBB2_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB2_3
-; SOFT-NEXT:    b .LBB2_4
-; SOFT-NEXT:  .LBB2_9: @ %entry
-; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:  .LBB2_7:
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB2_6
-; SOFT-NEXT:  .LBB2_10: @ %entry
+; SOFT-NEXT:  .LBB2_8: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -220,27 +196,24 @@ entry:
 define i32 @stest_f32i32(float %x) {
 ; SOFT-LABEL: stest_f32i32:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
-; SOFT-NEXT:    ldr r4, .LCPI3_0
-; SOFT-NEXT:    subs r5, r0, r4
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
-; SOFT-NEXT:    bge .LBB3_7
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    ldr r3, .LCPI3_0
+; SOFT-NEXT:    subs r4, r0, r3
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB3_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB3_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB3_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB3_4
-; SOFT-NEXT:  .LBB3_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB3_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB3_4: @ %entry
-; SOFT-NEXT:    mvns r3, r3
+; SOFT-NEXT:    mvns r3, r2
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    lsls r2, r2, #31
 ; SOFT-NEXT:    subs r4, r2, r0
 ; SOFT-NEXT:    sbcs r3, r1
@@ -248,18 +221,9 @@ define i32 @stest_f32i32(float %x) {
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:  .LBB3_6: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
-; SOFT-NEXT:  .LBB3_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB3_2
-; SOFT-NEXT:  .LBB3_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB3_3
-; SOFT-NEXT:    b .LBB3_4
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:  @ %bb.7:
 ; SOFT-NEXT:  .LCPI3_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -312,45 +276,33 @@ define i32 @ustest_f32i32(float %x) {
 ; SOFT-NEXT:    .save {r4, lr}
 ; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    mvns r3, r2
 ; SOFT-NEXT:    adds r4, r0, #1
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    sbcs r4, r3
-; SOFT-NEXT:    mov r4, r2
-; SOFT-NEXT:    bge .LBB5_7
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB5_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB5_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB5_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB5_4
-; SOFT-NEXT:  .LBB5_3: @ %entry
-; SOFT-NEXT:    mvns r0, r3
+; SOFT-NEXT:    blt .LBB5_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB5_4: @ %entry
-; SOFT-NEXT:    rsbs r4, r0, #0
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    sbcs r4, r1
-; SOFT-NEXT:    bge .LBB5_9
+; SOFT-NEXT:    rsbs r3, r0, #0
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:    sbcs r3, r1
+; SOFT-NEXT:    blt .LBB5_7
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB5_10
+; SOFT-NEXT:    beq .LBB5_8
 ; SOFT-NEXT:  .LBB5_6: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB5_7: @ %entry
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB5_2
-; SOFT-NEXT:  .LBB5_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB5_3
-; SOFT-NEXT:    b .LBB5_4
-; SOFT-NEXT:  .LBB5_9: @ %entry
-; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:  .LBB5_7:
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB5_6
-; SOFT-NEXT:  .LBB5_10: @ %entry
+; SOFT-NEXT:  .LBB5_8: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -372,29 +324,26 @@ entry:
 define i32 @stest_f16i32(half %x) {
 ; SOFT-LABEL: stest_f16i32:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
-; SOFT-NEXT:    ldr r4, .LCPI6_0
-; SOFT-NEXT:    subs r5, r0, r4
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
-; SOFT-NEXT:    bge .LBB6_7
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    ldr r3, .LCPI6_0
+; SOFT-NEXT:    subs r4, r0, r3
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB6_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB6_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB6_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB6_4
-; SOFT-NEXT:  .LBB6_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB6_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB6_4: @ %entry
-; SOFT-NEXT:    mvns r3, r3
+; SOFT-NEXT:    mvns r3, r2
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    lsls r2, r2, #31
 ; SOFT-NEXT:    subs r4, r2, r0
 ; SOFT-NEXT:    sbcs r3, r1
@@ -402,18 +351,9 @@ define i32 @stest_f16i32(half %x) {
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:  .LBB6_6: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
-; SOFT-NEXT:  .LBB6_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB6_2
-; SOFT-NEXT:  .LBB6_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB6_3
-; SOFT-NEXT:    b .LBB6_4
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:  @ %bb.7:
 ; SOFT-NEXT:  .LCPI6_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -492,45 +432,33 @@ define i32 @ustest_f16i32(half %x) {
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    movs r3, #0
+; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    mvns r3, r2
 ; SOFT-NEXT:    adds r4, r0, #1
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    sbcs r4, r3
-; SOFT-NEXT:    mov r4, r2
-; SOFT-NEXT:    bge .LBB8_7
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    blt .LBB8_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB8_8
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB8_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB8_4
-; SOFT-NEXT:  .LBB8_3: @ %entry
-; SOFT-NEXT:    mvns r0, r3
+; SOFT-NEXT:    blt .LBB8_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    mov r0, r3
 ; SOFT-NEXT:  .LBB8_4: @ %entry
-; SOFT-NEXT:    rsbs r4, r0, #0
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    sbcs r4, r1
-; SOFT-NEXT:    bge .LBB8_9
+; SOFT-NEXT:    rsbs r3, r0, #0
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:    sbcs r3, r1
+; SOFT-NEXT:    blt .LBB8_7
 ; SOFT-NEXT:  @ %bb.5: @ %entry
 ; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB8_10
+; SOFT-NEXT:    beq .LBB8_8
 ; SOFT-NEXT:  .LBB8_6: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB8_7: @ %entry
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB8_2
-; SOFT-NEXT:  .LBB8_8: @ %entry
-; SOFT-NEXT:    mov r1, r3
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB8_3
-; SOFT-NEXT:    b .LBB8_4
-; SOFT-NEXT:  .LBB8_9: @ %entry
-; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:  .LBB8_7:
+; SOFT-NEXT:    movs r2, #1
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB8_6
-; SOFT-NEXT:  .LBB8_10: @ %entry
+; SOFT-NEXT:  .LBB8_8: @ %entry
 ; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -990,86 +918,62 @@ define i64 @stest_f64i64(double %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    ldr r0, .LCPI18_0
-; SOFT-NEXT:    adds r7, r6, #1
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mvns r5, r4
+; SOFT-NEXT:    ldr r6, .LCPI18_0
+; SOFT-NEXT:    adds r7, r0, #1
 ; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    sbcs r7, r0
+; SOFT-NEXT:    sbcs r7, r6
 ; SOFT-NEXT:    mov r7, r2
-; SOFT-NEXT:    sbcs r7, r5
+; SOFT-NEXT:    sbcs r7, r4
 ; SOFT-NEXT:    mov r7, r3
-; SOFT-NEXT:    sbcs r7, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB18_13
+; SOFT-NEXT:    sbcs r7, r4
+; SOFT-NEXT:    bge .LBB18_8
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB18_14
+; SOFT-NEXT:    bge .LBB18_9
 ; SOFT-NEXT:  .LBB18_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_4
+; SOFT-NEXT:    bge .LBB18_10
 ; SOFT-NEXT:  .LBB18_3: @ %entry
-; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    blt .LBB18_5
 ; SOFT-NEXT:  .LBB18_4: @ %entry
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:  .LBB18_6: @ %entry
-; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_8
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:  .LBB18_8: @ %entry
-; SOFT-NEXT:    lsls r3, r4, #31
-; SOFT-NEXT:    rsbs r7, r6, #0
-; SOFT-NEXT:    mov r7, r3
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB18_5: @ %entry
+; SOFT-NEXT:    movs r6, #1
+; SOFT-NEXT:    lsls r6, r6, #31
+; SOFT-NEXT:    rsbs r7, r0, #0
+; SOFT-NEXT:    mov r7, r6
 ; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    sbcs r0, r2
-; SOFT-NEXT:    bge .LBB18_15
-; SOFT-NEXT:  @ %bb.9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB18_16
+; SOFT-NEXT:    sbcs r5, r3
+; SOFT-NEXT:    bge .LBB18_11
+; SOFT-NEXT:  @ %bb.6: @ %entry
+; SOFT-NEXT:    bge .LBB18_12
+; SOFT-NEXT:  .LBB18_7: @ %entry
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB18_8: @ %entry
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    blt .LBB18_2
+; SOFT-NEXT:  .LBB18_9: @ %entry
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    blt .LBB18_3
 ; SOFT-NEXT:  .LBB18_10: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB18_12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bge .LBB18_4
+; SOFT-NEXT:    b .LBB18_5
 ; SOFT-NEXT:  .LBB18_11: @ %entry
-; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB18_7
 ; SOFT-NEXT:  .LBB18_12: @ %entry
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB18_13: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_2
-; SOFT-NEXT:  .LBB18_14: @ %entry
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB18_3
-; SOFT-NEXT:    b .LBB18_4
-; SOFT-NEXT:  .LBB18_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB18_10
-; SOFT-NEXT:  .LBB18_16: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB18_11
-; SOFT-NEXT:    b .LBB18_12
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.17:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI18_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -1100,13 +1004,9 @@ define i64 @stest_f64i64(double %x) {
 ; VFP2-NEXT:    sbcs.w r5, lr, r1
 ; VFP2-NEXT:    sbcs.w r4, r2, r4
 ; VFP2-NEXT:    sbcs r2, r3
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, lr
+; VFP2-NEXT:    itt ge
+; VFP2-NEXT:    movge r0, r12
+; VFP2-NEXT:    movge r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f64i64:
@@ -1131,11 +1031,9 @@ define i64 @stest_f64i64(double %x) {
 ; FULL-NEXT:    sbcs.w r4, r12, r1
 ; FULL-NEXT:    sbcs.w r2, r3, r2
 ; FULL-NEXT:    sbcs.w r2, r3, r5
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r0, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r0, #0
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi double %x to i128
@@ -1156,24 +1054,15 @@ define i64 @utest_f64i64(double %x) {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    subs r2, r2, #1
 ; SOFT-NEXT:    sbcs r3, r4
-; SOFT-NEXT:    blo .LBB19_2
+; SOFT-NEXT:    bhs .LBB19_3
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB19_3
-; SOFT-NEXT:    b .LBB19_4
-; SOFT-NEXT:  .LBB19_2:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB19_4
+; SOFT-NEXT:    bhs .LBB19_4
+; SOFT-NEXT:  .LBB19_2: @ %entry
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB19_3: @ %entry
 ; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blo .LBB19_2
 ; SOFT-NEXT:  .LBB19_4: @ %entry
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB19_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB19_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -1185,13 +1074,9 @@ define i64 @utest_f64i64(double %x) {
 ; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs r2, r3, #0
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lo
-; VFP2-NEXT:    movlo r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    itt hs
+; VFP2-NEXT:    movhs r0, r12
+; VFP2-NEXT:    movhs r1, r12
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: utest_f64i64:
@@ -1200,12 +1085,10 @@ define i64 @utest_f64i64(double %x) {
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixunsdfti
 ; FULL-NEXT:    subs r2, #1
+; FULL-NEXT:    mov.w r12, #0
 ; FULL-NEXT:    sbcs r2, r3, #0
-; FULL-NEXT:    mov.w r3, #0
-; FULL-NEXT:    cset r2, lo
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r3, ne
-; FULL-NEXT:    csel r1, r1, r3, ne
+; FULL-NEXT:    csel r0, r0, r12, lo
+; FULL-NEXT:    csel r1, r1, r12, lo
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptoui double %x to i128
@@ -1226,23 +1109,16 @@ define i64 @ustest_f64i64(double %x) {
 ; SOFT-NEXT:    subs r6, r2, #1
 ; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    mov r6, r4
-; SOFT-NEXT:    bge .LBB20_10
+; SOFT-NEXT:    bge .LBB20_9
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB20_11
+; SOFT-NEXT:    bge .LBB20_10
 ; SOFT-NEXT:  .LBB20_2: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB20_12
+; SOFT-NEXT:    bge .LBB20_11
 ; SOFT-NEXT:  .LBB20_3: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB20_13
+; SOFT-NEXT:    blt .LBB20_5
 ; SOFT-NEXT:  .LBB20_4: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB20_6
-; SOFT-NEXT:  .LBB20_5: @ %entry
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:  .LBB20_6: @ %entry
+; SOFT-NEXT:  .LBB20_5: @ %entry
 ; SOFT-NEXT:    rsbs r6, r0, #0
 ; SOFT-NEXT:    mov r6, r5
 ; SOFT-NEXT:    sbcs r6, r1
@@ -1250,41 +1126,32 @@ define i64 @ustest_f64i64(double %x) {
 ; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB20_14
-; SOFT-NEXT:  @ %bb.7: @ %entry
+; SOFT-NEXT:    bge .LBB20_12
+; SOFT-NEXT:  @ %bb.6: @ %entry
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB20_15
+; SOFT-NEXT:    beq .LBB20_13
+; SOFT-NEXT:  .LBB20_7: @ %entry
+; SOFT-NEXT:    beq .LBB20_14
 ; SOFT-NEXT:  .LBB20_8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB20_16
-; SOFT-NEXT:  .LBB20_9: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB20_10: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB20_2
-; SOFT-NEXT:  .LBB20_11: @ %entry
+; SOFT-NEXT:  .LBB20_9: @ %entry
 ; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB20_3
-; SOFT-NEXT:  .LBB20_12: @ %entry
+; SOFT-NEXT:    blt .LBB20_2
+; SOFT-NEXT:  .LBB20_10: @ %entry
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB20_4
-; SOFT-NEXT:  .LBB20_13: @ %entry
+; SOFT-NEXT:    blt .LBB20_3
+; SOFT-NEXT:  .LBB20_11: @ %entry
 ; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB20_5
-; SOFT-NEXT:    b .LBB20_6
-; SOFT-NEXT:  .LBB20_14: @ %entry
+; SOFT-NEXT:    bge .LBB20_4
+; SOFT-NEXT:    b .LBB20_5
+; SOFT-NEXT:  .LBB20_12: @ %entry
 ; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB20_8
-; SOFT-NEXT:  .LBB20_15: @ %entry
+; SOFT-NEXT:    bne .LBB20_7
+; SOFT-NEXT:  .LBB20_13: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB20_9
-; SOFT-NEXT:  .LBB20_16: @ %entry
+; SOFT-NEXT:    bne .LBB20_8
+; SOFT-NEXT:  .LBB20_14: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ;
@@ -1296,15 +1163,11 @@ define i64 @ustest_f64i64(double %x) {
 ; VFP2-NEXT:    subs.w lr, r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    mov.w lr, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w lr, #1
-; VFP2-NEXT:    cmp.w lr, #0
-; VFP2-NEXT:    itttt eq
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    moveq r2, #1
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    movge r3, r12
+; VFP2-NEXT:    movge r2, #1
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    movge r0, r12
 ; VFP2-NEXT:    rsbs.w lr, r0, #0
 ; VFP2-NEXT:    sbcs.w lr, r12, r1
 ; VFP2-NEXT:    sbcs.w r2, r12, r2
@@ -1322,20 +1185,18 @@ define i64 @ustest_f64i64(double %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixdfti
-; FULL-NEXT:    subs.w r12, r2, #1
-; FULL-NEXT:    mov.w lr, #0
-; FULL-NEXT:    sbcs r12, r3, #0
-; FULL-NEXT:    cset r12, lt
-; FULL-NEXT:    cmp.w r12, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r2, #1
-; FULL-NEXT:    csel r0, r0, lr, ne
-; FULL-NEXT:    csel r12, r3, lr, ne
-; FULL-NEXT:    csel r1, r1, lr, ne
+; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    mov.w r12, #0
+; FULL-NEXT:    sbcs lr, r3, #0
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    csel r0, r0, r12, lt
+; FULL-NEXT:    csel lr, r3, r12, lt
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, lr, r1
-; FULL-NEXT:    sbcs.w r2, lr, r2
-; FULL-NEXT:    sbcs.w r2, lr, r12
+; FULL-NEXT:    sbcs.w r3, r12, r1
+; FULL-NEXT:    sbcs.w r2, r12, r2
+; FULL-NEXT:    sbcs.w r2, r12, lr
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
 ; FULL-NEXT:    csel r0, r0, r2, ne
@@ -1356,86 +1217,62 @@ define i64 @stest_f32i64(float %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    ldr r0, .LCPI21_0
-; SOFT-NEXT:    adds r7, r6, #1
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mvns r5, r4
+; SOFT-NEXT:    ldr r6, .LCPI21_0
+; SOFT-NEXT:    adds r7, r0, #1
 ; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    sbcs r7, r0
+; SOFT-NEXT:    sbcs r7, r6
 ; SOFT-NEXT:    mov r7, r2
-; SOFT-NEXT:    sbcs r7, r5
+; SOFT-NEXT:    sbcs r7, r4
 ; SOFT-NEXT:    mov r7, r3
-; SOFT-NEXT:    sbcs r7, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB21_13
+; SOFT-NEXT:    sbcs r7, r4
+; SOFT-NEXT:    bge .LBB21_8
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB21_14
+; SOFT-NEXT:    bge .LBB21_9
 ; SOFT-NEXT:  .LBB21_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB21_4
+; SOFT-NEXT:    bge .LBB21_10
 ; SOFT-NEXT:  .LBB21_3: @ %entry
-; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    blt .LBB21_5
 ; SOFT-NEXT:  .LBB21_4: @ %entry
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB21_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:  .LBB21_6: @ %entry
-; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB21_8
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:  .LBB21_8: @ %entry
-; SOFT-NEXT:    lsls r3, r4, #31
-; SOFT-NEXT:    rsbs r7, r6, #0
-; SOFT-NEXT:    mov r7, r3
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB21_5: @ %entry
+; SOFT-NEXT:    movs r6, #1
+; SOFT-NEXT:    lsls r6, r6, #31
+; SOFT-NEXT:    rsbs r7, r0, #0
+; SOFT-NEXT:    mov r7, r6
 ; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    sbcs r0, r2
-; SOFT-NEXT:    bge .LBB21_15
-; SOFT-NEXT:  @ %bb.9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB21_16
+; SOFT-NEXT:    sbcs r5, r3
+; SOFT-NEXT:    bge .LBB21_11
+; SOFT-NEXT:  @ %bb.6: @ %entry
+; SOFT-NEXT:    bge .LBB21_12
+; SOFT-NEXT:  .LBB21_7: @ %entry
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB21_8: @ %entry
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    blt .LBB21_2
+; SOFT-NEXT:  .LBB21_9: @ %entry
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    blt .LBB21_3
 ; SOFT-NEXT:  .LBB21_10: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB21_12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bge .LBB21_4
+; SOFT-NEXT:    b .LBB21_5
 ; SOFT-NEXT:  .LBB21_11: @ %entry
-; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB21_7
 ; SOFT-NEXT:  .LBB21_12: @ %entry
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB21_13: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB21_2
-; SOFT-NEXT:  .LBB21_14: @ %entry
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB21_3
-; SOFT-NEXT:    b .LBB21_4
-; SOFT-NEXT:  .LBB21_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB21_10
-; SOFT-NEXT:  .LBB21_16: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB21_11
-; SOFT-NEXT:    b .LBB21_12
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.17:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI21_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -1466,13 +1303,9 @@ define i64 @stest_f32i64(float %x) {
 ; VFP2-NEXT:    sbcs.w r5, lr, r1
 ; VFP2-NEXT:    sbcs.w r4, r2, r4
 ; VFP2-NEXT:    sbcs r2, r3
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, lr
+; VFP2-NEXT:    itt ge
+; VFP2-NEXT:    movge r0, r12
+; VFP2-NEXT:    movge r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f32i64:
@@ -1497,11 +1330,9 @@ define i64 @stest_f32i64(float %x) {
 ; FULL-NEXT:    sbcs.w r4, r12, r1
 ; FULL-NEXT:    sbcs.w r2, r3, r2
 ; FULL-NEXT:    sbcs.w r2, r3, r5
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r0, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r0, #0
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi float %x to i128
@@ -1522,24 +1353,15 @@ define i64 @utest_f32i64(float %x) {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    subs r2, r2, #1
 ; SOFT-NEXT:    sbcs r3, r4
-; SOFT-NEXT:    blo .LBB22_2
+; SOFT-NEXT:    bhs .LBB22_3
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB22_3
-; SOFT-NEXT:    b .LBB22_4
-; SOFT-NEXT:  .LBB22_2:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB22_4
+; SOFT-NEXT:    bhs .LBB22_4
+; SOFT-NEXT:  .LBB22_2: @ %entry
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB22_3: @ %entry
 ; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blo .LBB22_2
 ; SOFT-NEXT:  .LBB22_4: @ %entry
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB22_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB22_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -1551,13 +1373,9 @@ define i64 @utest_f32i64(float %x) {
 ; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs r2, r3, #0
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lo
-; VFP2-NEXT:    movlo r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    itt hs
+; VFP2-NEXT:    movhs r0, r12
+; VFP2-NEXT:    movhs r1, r12
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: utest_f32i64:
@@ -1566,12 +1384,10 @@ define i64 @utest_f32i64(float %x) {
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixunssfti
 ; FULL-NEXT:    subs r2, #1
+; FULL-NEXT:    mov.w r12, #0
 ; FULL-NEXT:    sbcs r2, r3, #0
-; FULL-NEXT:    mov.w r3, #0
-; FULL-NEXT:    cset r2, lo
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r3, ne
-; FULL-NEXT:    csel r1, r1, r3, ne
+; FULL-NEXT:    csel r0, r0, r12, lo
+; FULL-NEXT:    csel r1, r1, r12, lo
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptoui float %x to i128
@@ -1592,23 +1408,16 @@ define i64 @ustest_f32i64(float %x) {
 ; SOFT-NEXT:    subs r6, r2, #1
 ; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    mov r6, r4
-; SOFT-NEXT:    bge .LBB23_10
+; SOFT-NEXT:    bge .LBB23_9
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB23_11
+; SOFT-NEXT:    bge .LBB23_10
 ; SOFT-NEXT:  .LBB23_2: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB23_12
+; SOFT-NEXT:    bge .LBB23_11
 ; SOFT-NEXT:  .LBB23_3: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB23_13
+; SOFT-NEXT:    blt .LBB23_5
 ; SOFT-NEXT:  .LBB23_4: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB23_6
-; SOFT-NEXT:  .LBB23_5: @ %entry
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:  .LBB23_6: @ %entry
+; SOFT-NEXT:  .LBB23_5: @ %entry
 ; SOFT-NEXT:    rsbs r6, r0, #0
 ; SOFT-NEXT:    mov r6, r5
 ; SOFT-NEXT:    sbcs r6, r1
@@ -1616,41 +1425,32 @@ define i64 @ustest_f32i64(float %x) {
 ; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB23_14
-; SOFT-NEXT:  @ %bb.7: @ %entry
+; SOFT-NEXT:    bge .LBB23_12
+; SOFT-NEXT:  @ %bb.6: @ %entry
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB23_15
+; SOFT-NEXT:    beq .LBB23_13
+; SOFT-NEXT:  .LBB23_7: @ %entry
+; SOFT-NEXT:    beq .LBB23_14
 ; SOFT-NEXT:  .LBB23_8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB23_16
-; SOFT-NEXT:  .LBB23_9: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB23_10: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB23_2
-; SOFT-NEXT:  .LBB23_11: @ %entry
+; SOFT-NEXT:  .LBB23_9: @ %entry
 ; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB23_3
-; SOFT-NEXT:  .LBB23_12: @ %entry
+; SOFT-NEXT:    blt .LBB23_2
+; SOFT-NEXT:  .LBB23_10: @ %entry
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB23_4
-; SOFT-NEXT:  .LBB23_13: @ %entry
+; SOFT-NEXT:    blt .LBB23_3
+; SOFT-NEXT:  .LBB23_11: @ %entry
 ; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB23_5
-; SOFT-NEXT:    b .LBB23_6
-; SOFT-NEXT:  .LBB23_14: @ %entry
+; SOFT-NEXT:    bge .LBB23_4
+; SOFT-NEXT:    b .LBB23_5
+; SOFT-NEXT:  .LBB23_12: @ %entry
 ; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB23_8
-; SOFT-NEXT:  .LBB23_15: @ %entry
+; SOFT-NEXT:    bne .LBB23_7
+; SOFT-NEXT:  .LBB23_13: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB23_9
-; SOFT-NEXT:  .LBB23_16: @ %entry
+; SOFT-NEXT:    bne .LBB23_8
+; SOFT-NEXT:  .LBB23_14: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ;
@@ -1662,15 +1462,11 @@ define i64 @ustest_f32i64(float %x) {
 ; VFP2-NEXT:    subs.w lr, r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    mov.w lr, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w lr, #1
-; VFP2-NEXT:    cmp.w lr, #0
-; VFP2-NEXT:    itttt eq
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    moveq r2, #1
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    movge r3, r12
+; VFP2-NEXT:    movge r2, #1
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    movge r0, r12
 ; VFP2-NEXT:    rsbs.w lr, r0, #0
 ; VFP2-NEXT:    sbcs.w lr, r12, r1
 ; VFP2-NEXT:    sbcs.w r2, r12, r2
@@ -1688,20 +1484,18 @@ define i64 @ustest_f32i64(float %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixsfti
-; FULL-NEXT:    subs.w r12, r2, #1
-; FULL-NEXT:    mov.w lr, #0
-; FULL-NEXT:    sbcs r12, r3, #0
-; FULL-NEXT:    cset r12, lt
-; FULL-NEXT:    cmp.w r12, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r2, #1
-; FULL-NEXT:    csel r0, r0, lr, ne
-; FULL-NEXT:    csel r12, r3, lr, ne
-; FULL-NEXT:    csel r1, r1, lr, ne
+; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    mov.w r12, #0
+; FULL-NEXT:    sbcs lr, r3, #0
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    csel r0, r0, r12, lt
+; FULL-NEXT:    csel lr, r3, r12, lt
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, lr, r1
-; FULL-NEXT:    sbcs.w r2, lr, r2
-; FULL-NEXT:    sbcs.w r2, lr, r12
+; FULL-NEXT:    sbcs.w r3, r12, r1
+; FULL-NEXT:    sbcs.w r2, r12, r2
+; FULL-NEXT:    sbcs.w r2, r12, lr
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
 ; FULL-NEXT:    csel r0, r0, r2, ne
@@ -1722,88 +1516,64 @@ define i64 @stest_f16i64(half %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    ldr r0, .LCPI24_0
-; SOFT-NEXT:    adds r7, r6, #1
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mvns r5, r4
+; SOFT-NEXT:    ldr r6, .LCPI24_0
+; SOFT-NEXT:    adds r7, r0, #1
 ; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    sbcs r7, r0
+; SOFT-NEXT:    sbcs r7, r6
 ; SOFT-NEXT:    mov r7, r2
-; SOFT-NEXT:    sbcs r7, r5
+; SOFT-NEXT:    sbcs r7, r4
 ; SOFT-NEXT:    mov r7, r3
-; SOFT-NEXT:    sbcs r7, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB24_13
+; SOFT-NEXT:    sbcs r7, r4
+; SOFT-NEXT:    bge .LBB24_8
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB24_14
+; SOFT-NEXT:    bge .LBB24_9
 ; SOFT-NEXT:  .LBB24_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB24_4
+; SOFT-NEXT:    bge .LBB24_10
 ; SOFT-NEXT:  .LBB24_3: @ %entry
-; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    blt .LBB24_5
 ; SOFT-NEXT:  .LBB24_4: @ %entry
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB24_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:  .LBB24_6: @ %entry
-; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB24_8
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:  .LBB24_8: @ %entry
-; SOFT-NEXT:    lsls r3, r4, #31
-; SOFT-NEXT:    rsbs r7, r6, #0
-; SOFT-NEXT:    mov r7, r3
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB24_5: @ %entry
+; SOFT-NEXT:    movs r6, #1
+; SOFT-NEXT:    lsls r6, r6, #31
+; SOFT-NEXT:    rsbs r7, r0, #0
+; SOFT-NEXT:    mov r7, r6
 ; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    sbcs r0, r2
-; SOFT-NEXT:    bge .LBB24_15
-; SOFT-NEXT:  @ %bb.9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB24_16
+; SOFT-NEXT:    sbcs r5, r3
+; SOFT-NEXT:    bge .LBB24_11
+; SOFT-NEXT:  @ %bb.6: @ %entry
+; SOFT-NEXT:    bge .LBB24_12
+; SOFT-NEXT:  .LBB24_7: @ %entry
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB24_8: @ %entry
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    blt .LBB24_2
+; SOFT-NEXT:  .LBB24_9: @ %entry
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    blt .LBB24_3
 ; SOFT-NEXT:  .LBB24_10: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB24_12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bge .LBB24_4
+; SOFT-NEXT:    b .LBB24_5
 ; SOFT-NEXT:  .LBB24_11: @ %entry
-; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blt .LBB24_7
 ; SOFT-NEXT:  .LBB24_12: @ %entry
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB24_13: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB24_2
-; SOFT-NEXT:  .LBB24_14: @ %entry
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB24_3
-; SOFT-NEXT:    b .LBB24_4
-; SOFT-NEXT:  .LBB24_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB24_10
-; SOFT-NEXT:  .LBB24_16: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB24_11
-; SOFT-NEXT:    b .LBB24_12
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.17:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI24_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -1837,13 +1607,9 @@ define i64 @stest_f16i64(half %x) {
 ; VFP2-NEXT:    sbcs.w r5, lr, r1
 ; VFP2-NEXT:    sbcs.w r4, r2, r4
 ; VFP2-NEXT:    sbcs r2, r3
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, lr
+; VFP2-NEXT:    itt ge
+; VFP2-NEXT:    movge r0, r12
+; VFP2-NEXT:    movge r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f16i64:
@@ -1870,11 +1636,9 @@ define i64 @stest_f16i64(half %x) {
 ; FULL-NEXT:    sbcs.w r4, r12, r1
 ; FULL-NEXT:    sbcs.w r2, r3, r2
 ; FULL-NEXT:    sbcs.w r2, r3, r5
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r0, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r0, #0
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi half %x to i128
@@ -1897,24 +1661,15 @@ define i64 @utesth_f16i64(half %x) {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    subs r2, r2, #1
 ; SOFT-NEXT:    sbcs r3, r4
-; SOFT-NEXT:    blo .LBB25_2
+; SOFT-NEXT:    bhs .LBB25_3
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB25_3
-; SOFT-NEXT:    b .LBB25_4
-; SOFT-NEXT:  .LBB25_2:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB25_4
+; SOFT-NEXT:    bhs .LBB25_4
+; SOFT-NEXT:  .LBB25_2: @ %entry
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB25_3: @ %entry
 ; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    blo .LBB25_2
 ; SOFT-NEXT:  .LBB25_4: @ %entry
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB25_6
-; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    pop {r4, pc}
-; SOFT-NEXT:  .LBB25_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, pc}
 ;
@@ -1929,13 +1684,9 @@ define i64 @utesth_f16i64(half %x) {
 ; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs r2, r3, #0
-; VFP2-NEXT:    mov.w r2, #0
-; VFP2-NEXT:    it lo
-; VFP2-NEXT:    movlo r2, #1
-; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    itt hs
+; VFP2-NEXT:    movhs r0, r12
+; VFP2-NEXT:    movhs r1, r12
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: utesth_f16i64:
@@ -1946,12 +1697,10 @@ define i64 @utesth_f16i64(half %x) {
 ; FULL-NEXT:    vmov s0, r0
 ; FULL-NEXT:    bl __fixunshfti
 ; FULL-NEXT:    subs r2, #1
+; FULL-NEXT:    mov.w r12, #0
 ; FULL-NEXT:    sbcs r2, r3, #0
-; FULL-NEXT:    mov.w r3, #0
-; FULL-NEXT:    cset r2, lo
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r3, ne
-; FULL-NEXT:    csel r1, r1, r3, ne
+; FULL-NEXT:    csel r0, r0, r12, lo
+; FULL-NEXT:    csel r1, r1, r12, lo
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptoui half %x to i128
@@ -1974,23 +1723,16 @@ define i64 @ustest_f16i64(half %x) {
 ; SOFT-NEXT:    subs r6, r2, #1
 ; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    mov r6, r4
-; SOFT-NEXT:    bge .LBB26_10
+; SOFT-NEXT:    bge .LBB26_9
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB26_11
+; SOFT-NEXT:    bge .LBB26_10
 ; SOFT-NEXT:  .LBB26_2: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB26_12
+; SOFT-NEXT:    bge .LBB26_11
 ; SOFT-NEXT:  .LBB26_3: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB26_13
+; SOFT-NEXT:    blt .LBB26_5
 ; SOFT-NEXT:  .LBB26_4: @ %entry
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB26_6
-; SOFT-NEXT:  .LBB26_5: @ %entry
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:  .LBB26_6: @ %entry
+; SOFT-NEXT:  .LBB26_5: @ %entry
 ; SOFT-NEXT:    rsbs r6, r0, #0
 ; SOFT-NEXT:    mov r6, r5
 ; SOFT-NEXT:    sbcs r6, r1
@@ -1998,41 +1740,32 @@ define i64 @ustest_f16i64(half %x) {
 ; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB26_14
-; SOFT-NEXT:  @ %bb.7: @ %entry
+; SOFT-NEXT:    bge .LBB26_12
+; SOFT-NEXT:  @ %bb.6: @ %entry
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_15
+; SOFT-NEXT:    beq .LBB26_13
+; SOFT-NEXT:  .LBB26_7: @ %entry
+; SOFT-NEXT:    beq .LBB26_14
 ; SOFT-NEXT:  .LBB26_8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_16
-; SOFT-NEXT:  .LBB26_9: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB26_10: @ %entry
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB26_2
-; SOFT-NEXT:  .LBB26_11: @ %entry
+; SOFT-NEXT:  .LBB26_9: @ %entry
 ; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB26_3
-; SOFT-NEXT:  .LBB26_12: @ %entry
+; SOFT-NEXT:    blt .LBB26_2
+; SOFT-NEXT:  .LBB26_10: @ %entry
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB26_4
-; SOFT-NEXT:  .LBB26_13: @ %entry
+; SOFT-NEXT:    blt .LBB26_3
+; SOFT-NEXT:  .LBB26_11: @ %entry
 ; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB26_5
-; SOFT-NEXT:    b .LBB26_6
-; SOFT-NEXT:  .LBB26_14: @ %entry
+; SOFT-NEXT:    bge .LBB26_4
+; SOFT-NEXT:    b .LBB26_5
+; SOFT-NEXT:  .LBB26_12: @ %entry
 ; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_8
-; SOFT-NEXT:  .LBB26_15: @ %entry
+; SOFT-NEXT:    bne .LBB26_7
+; SOFT-NEXT:  .LBB26_13: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_9
-; SOFT-NEXT:  .LBB26_16: @ %entry
+; SOFT-NEXT:    bne .LBB26_8
+; SOFT-NEXT:  .LBB26_14: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ;
@@ -2047,15 +1780,11 @@ define i64 @ustest_f16i64(half %x) {
 ; VFP2-NEXT:    subs.w lr, r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
 ; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    mov.w lr, #0
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w lr, #1
-; VFP2-NEXT:    cmp.w lr, #0
-; VFP2-NEXT:    itttt eq
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    moveq r2, #1
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    movge r3, r12
+; VFP2-NEXT:    movge r2, #1
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    movge r0, r12
 ; VFP2-NEXT:    rsbs.w lr, r0, #0
 ; VFP2-NEXT:    sbcs.w lr, r12, r1
 ; VFP2-NEXT:    sbcs.w r2, r12, r2
@@ -2075,20 +1804,18 @@ define i64 @ustest_f16i64(half %x) {
 ; FULL-NEXT:    vmov.f16 r0, s0
 ; FULL-NEXT:    vmov s0, r0
 ; FULL-NEXT:    bl __fixhfti
-; FULL-NEXT:    subs.w r12, r2, #1
-; FULL-NEXT:    mov.w lr, #0
-; FULL-NEXT:    sbcs r12, r3, #0
-; FULL-NEXT:    cset r12, lt
-; FULL-NEXT:    cmp.w r12, #0
-; FULL-NEXT:    it eq
-; FULL-NEXT:    moveq r2, #1
-; FULL-NEXT:    csel r0, r0, lr, ne
-; FULL-NEXT:    csel r12, r3, lr, ne
-; FULL-NEXT:    csel r1, r1, lr, ne
+; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    mov.w r12, #0
+; FULL-NEXT:    sbcs lr, r3, #0
+; FULL-NEXT:    it ge
+; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    csel r0, r0, r12, lt
+; FULL-NEXT:    csel lr, r3, r12, lt
+; FULL-NEXT:    csel r1, r1, r12, lt
 ; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, lr, r1
-; FULL-NEXT:    sbcs.w r2, lr, r2
-; FULL-NEXT:    sbcs.w r2, lr, r12
+; FULL-NEXT:    sbcs.w r3, r12, r1
+; FULL-NEXT:    sbcs.w r2, r12, r2
+; FULL-NEXT:    sbcs.w r2, r12, lr
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
 ; FULL-NEXT:    csel r0, r0, r2, ne
@@ -2121,16 +1848,15 @@ define i32 @stest_f64i32_mm(double %x) {
 ; SOFT-NEXT:    subs r5, r0, r4
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
 ; SOFT-NEXT:    bge .LBB27_7
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB27_8
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    bge .LBB27_8
 ; SOFT-NEXT:  .LBB27_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB27_4
 ; SOFT-NEXT:  .LBB27_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:  .LBB27_4: @ %entry
 ; SOFT-NEXT:    mvns r3, r3
 ; SOFT-NEXT:    lsls r2, r2, #31
@@ -2142,12 +1868,12 @@ define i32 @stest_f64i32_mm(double %x) {
 ; SOFT-NEXT:  .LBB27_6: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r7, pc}
 ; SOFT-NEXT:  .LBB27_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB27_2
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    blt .LBB27_2
 ; SOFT-NEXT:  .LBB27_8: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB27_3
 ; SOFT-NEXT:    b .LBB27_4
 ; SOFT-NEXT:    .p2align 2
@@ -2161,16 +1887,17 @@ define i32 @stest_f64i32_mm(double %x) {
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    vmov r0, r1, d0
 ; VFP2-NEXT:    bl __aeabi_d2lz
-; VFP2-NEXT:    mvn r12, #-2147483648
-; VFP2-NEXT:    subs.w r3, r0, r12
-; VFP2-NEXT:    mov.w r2, #0
+; VFP2-NEXT:    mvn r2, #-2147483648
+; VFP2-NEXT:    subs r3, r0, r2
 ; VFP2-NEXT:    sbcs r3, r1, #0
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r0, r2
+; VFP2-NEXT:    mov.w r2, #0
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt r2, #1
 ; VFP2-NEXT:    cmp r2, #0
-; VFP2-NEXT:    ite ne
+; VFP2-NEXT:    it ne
 ; VFP2-NEXT:    movne r2, r1
-; VFP2-NEXT:    moveq r0, r12
 ; VFP2-NEXT:    mov.w r1, #-1
 ; VFP2-NEXT:    rsbs.w r3, r0, #-2147483648
 ; VFP2-NEXT:    sbcs r1, r2
@@ -2234,24 +1961,19 @@ define i32 @ustest_f64i32_mm(double %x) {
 ; SOFT-NEXT:    .save {r7, lr}
 ; SOFT-NEXT:    push {r7, lr}
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    asrs r3, r1, #31
-; SOFT-NEXT:    ands r3, r1
-; SOFT-NEXT:    movs r2, #0
+; SOFT-NEXT:    mov r2, r0
+; SOFT-NEXT:    movs r0, #0
 ; SOFT-NEXT:    cmp r1, #1
-; SOFT-NEXT:    bge .LBB29_3
+; SOFT-NEXT:    blt .LBB29_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bpl .LBB29_4
+; SOFT-NEXT:    mvns r2, r0
 ; SOFT-NEXT:  .LBB29_2: @ %entry
+; SOFT-NEXT:    asrs r3, r1, #31
+; SOFT-NEXT:    ands r3, r1
+; SOFT-NEXT:    bmi .LBB29_4
+; SOFT-NEXT:  @ %bb.3: @ %entry
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r7, pc}
-; SOFT-NEXT:  .LBB29_3: @ %entry
-; SOFT-NEXT:    mvns r0, r2
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bmi .LBB29_2
 ; SOFT-NEXT:  .LBB29_4: @ %entry
-; SOFT-NEXT:    mov r2, r0
-; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    pop {r7, pc}
 ;
 ; VFP2-LABEL: ustest_f64i32_mm:
@@ -2293,16 +2015,15 @@ define i32 @stest_f32i32_mm(float %x) {
 ; SOFT-NEXT:    subs r5, r0, r4
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
 ; SOFT-NEXT:    bge .LBB30_7
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB30_8
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    bge .LBB30_8
 ; SOFT-NEXT:  .LBB30_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB30_4
 ; SOFT-NEXT:  .LBB30_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:  .LBB30_4: @ %entry
 ; SOFT-NEXT:    mvns r3, r3
 ; SOFT-NEXT:    lsls r2, r2, #31
@@ -2314,12 +2035,12 @@ define i32 @stest_f32i32_mm(float %x) {
 ; SOFT-NEXT:  .LBB30_6: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r7, pc}
 ; SOFT-NEXT:  .LBB30_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB30_2
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    blt .LBB30_2
 ; SOFT-NEXT:  .LBB30_8: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB30_3
 ; SOFT-NEXT:    b .LBB30_4
 ; SOFT-NEXT:    .p2align 2
@@ -2414,16 +2135,15 @@ define i32 @stest_f16i32_mm(half %x) {
 ; SOFT-NEXT:    subs r5, r0, r4
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    sbcs r5, r3
-; SOFT-NEXT:    mov r5, r2
 ; SOFT-NEXT:    bge .LBB33_7
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB33_8
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    bge .LBB33_8
 ; SOFT-NEXT:  .LBB33_2: @ %entry
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB33_4
 ; SOFT-NEXT:  .LBB33_3: @ %entry
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:  .LBB33_4: @ %entry
 ; SOFT-NEXT:    mvns r3, r3
 ; SOFT-NEXT:    lsls r2, r2, #31
@@ -2435,12 +2155,12 @@ define i32 @stest_f16i32_mm(half %x) {
 ; SOFT-NEXT:  .LBB33_6: @ %entry
 ; SOFT-NEXT:    pop {r4, r5, r7, pc}
 ; SOFT-NEXT:  .LBB33_7: @ %entry
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB33_2
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    blt .LBB33_2
 ; SOFT-NEXT:  .LBB33_8: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB33_3
 ; SOFT-NEXT:    b .LBB33_4
 ; SOFT-NEXT:    .p2align 2
@@ -2973,81 +2693,78 @@ define i64 @stest_f64i64_mm(double %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    movs r4, #1
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    movs r0, #1
 ; SOFT-NEXT:    movs r5, #0
 ; SOFT-NEXT:    ldr r6, .LCPI45_0
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    adds r0, r0, #1
-; SOFT-NEXT:    mov r0, r1
-; SOFT-NEXT:    sbcs r0, r6
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB45_12
+; SOFT-NEXT:    adds r4, r7, #1
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r6
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    blt .LBB45_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB45_13
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB45_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB45_14
-; SOFT-NEXT:  .LBB45_3: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB45_5
+; SOFT-NEXT:    mvns r6, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB45_12
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    beq .LBB45_13
 ; SOFT-NEXT:  .LBB45_4: @ %entry
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB45_14
 ; SOFT-NEXT:  .LBB45_5: @ %entry
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB45_7
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:  .LBB45_7: @ %entry
-; SOFT-NEXT:    lsls r6, r4, #31
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    rsbs r7, r7, #0
+; SOFT-NEXT:  .LBB45_6: @ %entry
 ; SOFT-NEXT:    mov r7, r6
-; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    sbcs r0, r3
+; SOFT-NEXT:  .LBB45_7: @ %entry
+; SOFT-NEXT:    lsls r3, r0, #31
+; SOFT-NEXT:    rsbs r4, r7, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r1
+; SOFT-NEXT:    mov r4, r6
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    bge .LBB45_15
 ; SOFT-NEXT:  @ %bb.8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB45_16
 ; SOFT-NEXT:  .LBB45_9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB45_11
 ; SOFT-NEXT:  .LBB45_10: @ %entry
-; SOFT-NEXT:    str r4, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r1, r3
 ; SOFT-NEXT:  .LBB45_11: @ %entry
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB45_12: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB45_2
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    bne .LBB45_4
 ; SOFT-NEXT:  .LBB45_13: @ %entry
-; SOFT-NEXT:    mov r3, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB45_3
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB45_5
 ; SOFT-NEXT:  .LBB45_14: @ %entry
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB45_4
-; SOFT-NEXT:    b .LBB45_5
+; SOFT-NEXT:    ldr r1, .LCPI45_0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB45_6
+; SOFT-NEXT:    b .LBB45_7
 ; SOFT-NEXT:  .LBB45_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB45_9
 ; SOFT-NEXT:  .LBB45_16: @ %entry
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    beq .LBB45_10
 ; SOFT-NEXT:    b .LBB45_11
 ; SOFT-NEXT:    .p2align 2
@@ -3086,8 +2803,8 @@ define i64 @stest_f64i64_mm(double %x) {
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
 ; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f64i64_mm:
@@ -3114,8 +2831,8 @@ define i64 @stest_f64i64_mm(double %x) {
 ; FULL-NEXT:    sbcs.w r2, r3, r5
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
+; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi double %x to i128
@@ -3139,7 +2856,6 @@ define i64 @utest_f64i64_mm(double %x) {
 ; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB46_5
 ; SOFT-NEXT:  .LBB46_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB46_6
 ; SOFT-NEXT:  .LBB46_3: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
@@ -3149,7 +2865,6 @@ define i64 @utest_f64i64_mm(double %x) {
 ; SOFT-NEXT:    bne .LBB46_2
 ; SOFT-NEXT:  .LBB46_5: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB46_3
 ; SOFT-NEXT:  .LBB46_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
@@ -3193,8 +2908,8 @@ entry:
 define i64 @ustest_f64i64_mm(double %x) {
 ; SOFT-LABEL: ustest_f64i64_mm:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixdfti
 ; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    movs r1, #0
@@ -3203,46 +2918,42 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; SOFT-NEXT:    sbcs r2, r1
 ; SOFT-NEXT:    blt .LBB47_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r2, r1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB47_3
 ; SOFT-NEXT:    b .LBB47_4
 ; SOFT-NEXT:  .LBB47_2:
-; SOFT-NEXT:    movs r5, #1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB47_4
 ; SOFT-NEXT:  .LBB47_3: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r4, r2
 ; SOFT-NEXT:  .LBB47_4: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB47_6
+; SOFT-NEXT:    beq .LBB47_10
 ; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bne .LBB47_7
 ; SOFT-NEXT:  .LBB47_6: @ %entry
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:  .LBB47_7: @ %entry
 ; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    mov r2, r1
-; SOFT-NEXT:    bpl .LBB47_10
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB47_11
-; SOFT-NEXT:  .LBB47_8: @ %entry
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bpl .LBB47_11
+; SOFT-NEXT:  @ %bb.8: @ %entry
 ; SOFT-NEXT:    bpl .LBB47_12
 ; SOFT-NEXT:  .LBB47_9: @ %entry
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB47_10: @ %entry
-; SOFT-NEXT:    mov r2, r0
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB47_8
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    beq .LBB47_6
+; SOFT-NEXT:    b .LBB47_7
 ; SOFT-NEXT:  .LBB47_11: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r0
 ; SOFT-NEXT:    bmi .LBB47_9
 ; SOFT-NEXT:  .LBB47_12: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ;
 ; VFP2-LABEL: ustest_f64i64_mm:
 ; VFP2:       @ %bb.0: @ %entry
@@ -3255,17 +2966,13 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
+; VFP2-NEXT:    itte eq
+; VFP2-NEXT:    moveq r1, r12
 ; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
-; VFP2-NEXT:    movmi r0, #0
+; VFP2-NEXT:    movne r12, r3
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    it eq
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
+; VFP2-NEXT:    itt mi
+; VFP2-NEXT:    movmi r0, #0
 ; VFP2-NEXT:    movmi r1, #0
 ; VFP2-NEXT:    pop {r7, pc}
 ;
@@ -3278,15 +2985,12 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; FULL-NEXT:    sbcs r2, r3, #0
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r3, r3, r2, ne
+; FULL-NEXT:    csel r1, r1, r2, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
-; FULL-NEXT:    movmi r0, #0
+; FULL-NEXT:    csel r2, r3, r2, ne
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
+; FULL-NEXT:    itt mi
+; FULL-NEXT:    movmi r0, #0
 ; FULL-NEXT:    movmi r1, #0
 ; FULL-NEXT:    pop {r7, pc}
 entry:
@@ -3302,81 +3006,78 @@ define i64 @stest_f32i64_mm(float %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    movs r0, #1
 ; SOFT-NEXT:    movs r5, #0
 ; SOFT-NEXT:    ldr r6, .LCPI48_0
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    adds r0, r0, #1
-; SOFT-NEXT:    mov r0, r1
-; SOFT-NEXT:    sbcs r0, r6
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB48_12
+; SOFT-NEXT:    adds r4, r7, #1
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r6
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    blt .LBB48_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB48_13
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB48_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB48_14
-; SOFT-NEXT:  .LBB48_3: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB48_5
+; SOFT-NEXT:    mvns r6, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB48_12
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    beq .LBB48_13
 ; SOFT-NEXT:  .LBB48_4: @ %entry
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB48_14
 ; SOFT-NEXT:  .LBB48_5: @ %entry
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB48_7
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:  .LBB48_7: @ %entry
-; SOFT-NEXT:    lsls r6, r4, #31
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    rsbs r7, r7, #0
+; SOFT-NEXT:  .LBB48_6: @ %entry
 ; SOFT-NEXT:    mov r7, r6
-; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    sbcs r0, r3
+; SOFT-NEXT:  .LBB48_7: @ %entry
+; SOFT-NEXT:    lsls r3, r0, #31
+; SOFT-NEXT:    rsbs r4, r7, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r1
+; SOFT-NEXT:    mov r4, r6
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    bge .LBB48_15
 ; SOFT-NEXT:  @ %bb.8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB48_16
 ; SOFT-NEXT:  .LBB48_9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB48_11
 ; SOFT-NEXT:  .LBB48_10: @ %entry
-; SOFT-NEXT:    str r4, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r1, r3
 ; SOFT-NEXT:  .LBB48_11: @ %entry
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB48_12: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB48_2
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    bne .LBB48_4
 ; SOFT-NEXT:  .LBB48_13: @ %entry
-; SOFT-NEXT:    mov r3, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB48_3
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB48_5
 ; SOFT-NEXT:  .LBB48_14: @ %entry
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB48_4
-; SOFT-NEXT:    b .LBB48_5
+; SOFT-NEXT:    ldr r1, .LCPI48_0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB48_6
+; SOFT-NEXT:    b .LBB48_7
 ; SOFT-NEXT:  .LBB48_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB48_9
 ; SOFT-NEXT:  .LBB48_16: @ %entry
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    beq .LBB48_10
 ; SOFT-NEXT:    b .LBB48_11
 ; SOFT-NEXT:    .p2align 2
@@ -3415,8 +3116,8 @@ define i64 @stest_f32i64_mm(float %x) {
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
 ; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f32i64_mm:
@@ -3443,8 +3144,8 @@ define i64 @stest_f32i64_mm(float %x) {
 ; FULL-NEXT:    sbcs.w r2, r3, r5
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
+; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi float %x to i128
@@ -3468,7 +3169,6 @@ define i64 @utest_f32i64_mm(float %x) {
 ; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB49_5
 ; SOFT-NEXT:  .LBB49_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB49_6
 ; SOFT-NEXT:  .LBB49_3: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
@@ -3478,7 +3178,6 @@ define i64 @utest_f32i64_mm(float %x) {
 ; SOFT-NEXT:    bne .LBB49_2
 ; SOFT-NEXT:  .LBB49_5: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB49_3
 ; SOFT-NEXT:  .LBB49_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
@@ -3522,8 +3221,8 @@ entry:
 define i64 @ustest_f32i64_mm(float %x) {
 ; SOFT-LABEL: ustest_f32i64_mm:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixsfti
 ; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    movs r1, #0
@@ -3532,46 +3231,42 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; SOFT-NEXT:    sbcs r2, r1
 ; SOFT-NEXT:    blt .LBB50_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r2, r1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB50_3
 ; SOFT-NEXT:    b .LBB50_4
 ; SOFT-NEXT:  .LBB50_2:
-; SOFT-NEXT:    movs r5, #1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB50_4
 ; SOFT-NEXT:  .LBB50_3: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r4, r2
 ; SOFT-NEXT:  .LBB50_4: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB50_6
+; SOFT-NEXT:    beq .LBB50_10
 ; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bne .LBB50_7
 ; SOFT-NEXT:  .LBB50_6: @ %entry
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:  .LBB50_7: @ %entry
 ; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    mov r2, r1
-; SOFT-NEXT:    bpl .LBB50_10
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB50_11
-; SOFT-NEXT:  .LBB50_8: @ %entry
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bpl .LBB50_11
+; SOFT-NEXT:  @ %bb.8: @ %entry
 ; SOFT-NEXT:    bpl .LBB50_12
 ; SOFT-NEXT:  .LBB50_9: @ %entry
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB50_10: @ %entry
-; SOFT-NEXT:    mov r2, r0
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB50_8
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    beq .LBB50_6
+; SOFT-NEXT:    b .LBB50_7
 ; SOFT-NEXT:  .LBB50_11: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r0
 ; SOFT-NEXT:    bmi .LBB50_9
 ; SOFT-NEXT:  .LBB50_12: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ;
 ; VFP2-LABEL: ustest_f32i64_mm:
 ; VFP2:       @ %bb.0: @ %entry
@@ -3584,17 +3279,13 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
+; VFP2-NEXT:    itte eq
+; VFP2-NEXT:    moveq r1, r12
 ; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
-; VFP2-NEXT:    movmi r0, #0
+; VFP2-NEXT:    movne r12, r3
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    it eq
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
+; VFP2-NEXT:    itt mi
+; VFP2-NEXT:    movmi r0, #0
 ; VFP2-NEXT:    movmi r1, #0
 ; VFP2-NEXT:    pop {r7, pc}
 ;
@@ -3607,15 +3298,12 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; FULL-NEXT:    sbcs r2, r3, #0
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r3, r3, r2, ne
+; FULL-NEXT:    csel r1, r1, r2, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
-; FULL-NEXT:    movmi r0, #0
+; FULL-NEXT:    csel r2, r3, r2, ne
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
+; FULL-NEXT:    itt mi
+; FULL-NEXT:    movmi r0, #0
 ; FULL-NEXT:    movmi r1, #0
 ; FULL-NEXT:    pop {r7, pc}
 entry:
@@ -3631,83 +3319,80 @@ define i64 @stest_f16i64_mm(half %x) {
 ; SOFT:       @ %bb.0: @ %entry
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    movs r0, #1
 ; SOFT-NEXT:    movs r5, #0
 ; SOFT-NEXT:    ldr r6, .LCPI51_0
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    adds r0, r0, #1
-; SOFT-NEXT:    mov r0, r1
-; SOFT-NEXT:    sbcs r0, r6
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    sbcs r0, r5
-; SOFT-NEXT:    mov r7, r4
-; SOFT-NEXT:    bge .LBB51_12
+; SOFT-NEXT:    adds r4, r7, #1
+; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:    sbcs r4, r6
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r5
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    blt .LBB51_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB51_13
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB51_2: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB51_14
-; SOFT-NEXT:  .LBB51_3: @ %entry
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB51_5
+; SOFT-NEXT:    mvns r6, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB51_12
+; SOFT-NEXT:  @ %bb.3: @ %entry
+; SOFT-NEXT:    beq .LBB51_13
 ; SOFT-NEXT:  .LBB51_4: @ %entry
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB51_14
 ; SOFT-NEXT:  .LBB51_5: @ %entry
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB51_7
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:  .LBB51_7: @ %entry
-; SOFT-NEXT:    lsls r6, r4, #31
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    rsbs r7, r7, #0
+; SOFT-NEXT:  .LBB51_6: @ %entry
 ; SOFT-NEXT:    mov r7, r6
-; SOFT-NEXT:    sbcs r7, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    sbcs r7, r2
-; SOFT-NEXT:    sbcs r0, r3
+; SOFT-NEXT:  .LBB51_7: @ %entry
+; SOFT-NEXT:    lsls r3, r0, #31
+; SOFT-NEXT:    rsbs r4, r7, #0
+; SOFT-NEXT:    mov r4, r3
+; SOFT-NEXT:    sbcs r4, r1
+; SOFT-NEXT:    mov r4, r6
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r4, r2
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    sbcs r6, r2
 ; SOFT-NEXT:    bge .LBB51_15
 ; SOFT-NEXT:  @ %bb.8: @ %entry
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB51_16
 ; SOFT-NEXT:  .LBB51_9: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB51_11
 ; SOFT-NEXT:  .LBB51_10: @ %entry
-; SOFT-NEXT:    str r4, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r1, r3
 ; SOFT-NEXT:  .LBB51_11: @ %entry
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB51_12: @ %entry
-; SOFT-NEXT:    mov r7, r5
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB51_2
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    bne .LBB51_4
 ; SOFT-NEXT:  .LBB51_13: @ %entry
-; SOFT-NEXT:    mov r3, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB51_3
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB51_5
 ; SOFT-NEXT:  .LBB51_14: @ %entry
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB51_4
-; SOFT-NEXT:    b .LBB51_5
+; SOFT-NEXT:    ldr r1, .LCPI51_0
+; SOFT-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB51_6
+; SOFT-NEXT:    b .LBB51_7
 ; SOFT-NEXT:  .LBB51_15: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB51_9
 ; SOFT-NEXT:  .LBB51_16: @ %entry
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    beq .LBB51_10
 ; SOFT-NEXT:    b .LBB51_11
 ; SOFT-NEXT:    .p2align 2
@@ -3749,8 +3434,8 @@ define i64 @stest_f16i64_mm(half %x) {
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
 ; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    moveq r0, r12
+; VFP2-NEXT:    moveq r1, lr
 ; VFP2-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; FULL-LABEL: stest_f16i64_mm:
@@ -3779,8 +3464,8 @@ define i64 @stest_f16i64_mm(half %x) {
 ; FULL-NEXT:    sbcs.w r2, r3, r5
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
+; FULL-NEXT:    csel r1, r1, r12, ne
 ; FULL-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi half %x to i128
@@ -3806,7 +3491,6 @@ define i64 @utesth_f16i64_mm(half %x) {
 ; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB52_5
 ; SOFT-NEXT:  .LBB52_2: @ %entry
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB52_6
 ; SOFT-NEXT:  .LBB52_3: @ %entry
 ; SOFT-NEXT:    pop {r4, pc}
@@ -3816,7 +3500,6 @@ define i64 @utesth_f16i64_mm(half %x) {
 ; SOFT-NEXT:    bne .LBB52_2
 ; SOFT-NEXT:  .LBB52_5: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB52_3
 ; SOFT-NEXT:  .LBB52_6: @ %entry
 ; SOFT-NEXT:    mov r1, r4
@@ -3865,8 +3548,8 @@ entry:
 define i64 @ustest_f16i64_mm(half %x) {
 ; SOFT-LABEL: ustest_f16i64_mm:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r7, lr}
-; SOFT-NEXT:    push {r4, r5, r7, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __fixsfti
@@ -3877,46 +3560,42 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; SOFT-NEXT:    sbcs r2, r1
 ; SOFT-NEXT:    blt .LBB53_2
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r2, r1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB53_3
 ; SOFT-NEXT:    b .LBB53_4
 ; SOFT-NEXT:  .LBB53_2:
-; SOFT-NEXT:    movs r5, #1
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB53_4
 ; SOFT-NEXT:  .LBB53_3: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r4, r2
 ; SOFT-NEXT:  .LBB53_4: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB53_6
+; SOFT-NEXT:    beq .LBB53_10
 ; SOFT-NEXT:  @ %bb.5: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bne .LBB53_7
 ; SOFT-NEXT:  .LBB53_6: @ %entry
+; SOFT-NEXT:    mov r3, r2
+; SOFT-NEXT:  .LBB53_7: @ %entry
 ; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    mov r2, r1
-; SOFT-NEXT:    bpl .LBB53_10
-; SOFT-NEXT:  @ %bb.7: @ %entry
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB53_11
-; SOFT-NEXT:  .LBB53_8: @ %entry
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bpl .LBB53_11
+; SOFT-NEXT:  @ %bb.8: @ %entry
 ; SOFT-NEXT:    bpl .LBB53_12
 ; SOFT-NEXT:  .LBB53_9: @ %entry
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB53_10: @ %entry
-; SOFT-NEXT:    mov r2, r0
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB53_8
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    beq .LBB53_6
+; SOFT-NEXT:    b .LBB53_7
 ; SOFT-NEXT:  .LBB53_11: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r0
 ; SOFT-NEXT:    bmi .LBB53_9
 ; SOFT-NEXT:  .LBB53_12: @ %entry
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    pop {r4, r5, r7, pc}
+; SOFT-NEXT:    pop {r4, pc}
 ;
 ; VFP2-LABEL: ustest_f16i64_mm:
 ; VFP2:       @ %bb.0: @ %entry
@@ -3932,17 +3611,13 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt.w r12, #1
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
+; VFP2-NEXT:    itte eq
+; VFP2-NEXT:    moveq r1, r12
 ; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r3, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
-; VFP2-NEXT:    movmi r0, #0
+; VFP2-NEXT:    movne r12, r3
 ; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    it eq
-; VFP2-NEXT:    moveq r1, r12
-; VFP2-NEXT:    cmp r3, #0
-; VFP2-NEXT:    it mi
+; VFP2-NEXT:    itt mi
+; VFP2-NEXT:    movmi r0, #0
 ; VFP2-NEXT:    movmi r1, #0
 ; VFP2-NEXT:    pop {r7, pc}
 ;
@@ -3957,15 +3632,12 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; FULL-NEXT:    sbcs r2, r3, #0
 ; FULL-NEXT:    cset r2, lt
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r3, r3, r2, ne
+; FULL-NEXT:    csel r1, r1, r2, ne
 ; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
-; FULL-NEXT:    movmi r0, #0
+; FULL-NEXT:    csel r2, r3, r2, ne
 ; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r1, r1, r2, ne
-; FULL-NEXT:    cmp r3, #0
-; FULL-NEXT:    it mi
+; FULL-NEXT:    itt mi
+; FULL-NEXT:    movmi r0, #0
 ; FULL-NEXT:    movmi r1, #0
 ; FULL-NEXT:    pop {r7, pc}
 entry:
@@ -4014,16 +3686,15 @@ define void @unroll_maxmin(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    mov r2, r1
 ; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    bge .LBB54_14
 ; SOFT-NEXT:  @ %bb.3: @ in Loop: Header=BB54_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB54_15
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    bge .LBB54_15
 ; SOFT-NEXT:  .LBB54_4: @ in Loop: Header=BB54_2 Depth=1
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB54_6
 ; SOFT-NEXT:  .LBB54_5: @ in Loop: Header=BB54_2 Depth=1
-; SOFT-NEXT:    ldr r0, .LCPI54_0
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:  .LBB54_6: @ in Loop: Header=BB54_2 Depth=1
 ; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:    subs r2, r2, r0
@@ -4049,7 +3720,6 @@ define void @unroll_maxmin(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB54_17
 ; SOFT-NEXT:  .LBB54_10: @ in Loop: Header=BB54_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB54_12
 ; SOFT-NEXT:  .LBB54_11: @ in Loop: Header=BB54_2 Depth=1
 ; SOFT-NEXT:    ldr r0, .LCPI54_0
@@ -4063,11 +3733,11 @@ define void @unroll_maxmin(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:    b .LBB54_1
 ; SOFT-NEXT:  .LBB54_14: @ in Loop: Header=BB54_2 Depth=1
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB54_4
+; SOFT-NEXT:    ldr r0, .LCPI54_0
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    blt .LBB54_4
 ; SOFT-NEXT:  .LBB54_15: @ in Loop: Header=BB54_2 Depth=1
-; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB54_5
 ; SOFT-NEXT:    b .LBB54_6
@@ -4077,7 +3747,6 @@ define void @unroll_maxmin(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    bne .LBB54_10
 ; SOFT-NEXT:  .LBB54_17: @ in Loop: Header=BB54_2 Depth=1
 ; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB54_11
 ; SOFT-NEXT:    b .LBB54_12
 ; SOFT-NEXT:  .LBB54_18:
@@ -4180,18 +3849,17 @@ define void @unroll_minmax(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r0, #0
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    mvns r0, r0
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #1
 ; SOFT-NEXT:    lsls r1, r0, #31
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    lsls r7, r0, #10
 ; SOFT-NEXT:    b .LBB55_2
 ; SOFT-NEXT:  .LBB55_1: @ in Loop: Header=BB55_2 Depth=1
@@ -4199,7 +3867,7 @@ define void @unroll_minmax(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    adds r4, #8
 ; SOFT-NEXT:    adds r5, #8
 ; SOFT-NEXT:    subs r7, r7, #2
-; SOFT-NEXT:    beq .LBB55_18
+; SOFT-NEXT:    beq .LBB55_14
 ; SOFT-NEXT:  .LBB55_2: @ =>This Inner Loop Header: Depth=1
 ; SOFT-NEXT:    ldr r0, [r4]
 ; SOFT-NEXT:    movs r1, #79
@@ -4207,24 +3875,21 @@ define void @unroll_minmax(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fmul
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    subs r2, r2, r0
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    sbcs r2, r1
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    bge .LBB55_14
+; SOFT-NEXT:    blt .LBB55_4
 ; SOFT-NEXT:  @ %bb.3: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB55_15
+; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB55_4: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB55_6
-; SOFT-NEXT:  .LBB55_5: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    blt .LBB55_6
+; SOFT-NEXT:  @ %bb.5: @ in Loop: Header=BB55_2 Depth=1
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB55_6: @ in Loop: Header=BB55_2 Depth=1
 ; SOFT-NEXT:    ldr r2, .LCPI55_0
 ; SOFT-NEXT:    subs r2, r0, r2
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    sbcs r1, r2
 ; SOFT-NEXT:    blt .LBB55_8
 ; SOFT-NEXT:  @ %bb.7: @ in Loop: Header=BB55_2 Depth=1
@@ -4235,52 +3900,31 @@ define void @unroll_minmax(ptr nocapture %0, ptr nocapture readonly %1, i32 %2)
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fmul
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    subs r2, r2, r0
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    sbcs r2, r1
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    bge .LBB55_16
+; SOFT-NEXT:    blt .LBB55_10
 ; SOFT-NEXT:  @ %bb.9: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB55_17
+; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB55_10: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB55_12
-; SOFT-NEXT:  .LBB55_11: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    blt .LBB55_12
+; SOFT-NEXT:  @ %bb.11: @ in Loop: Header=BB55_2 Depth=1
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB55_12: @ in Loop: Header=BB55_2 Depth=1
 ; SOFT-NEXT:    ldr r2, .LCPI55_0
 ; SOFT-NEXT:    subs r2, r0, r2
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    sbcs r1, r2
 ; SOFT-NEXT:    blt .LBB55_1
 ; SOFT-NEXT:  @ %bb.13: @ in Loop: Header=BB55_2 Depth=1
 ; SOFT-NEXT:    ldr r0, .LCPI55_0
 ; SOFT-NEXT:    b .LBB55_1
-; SOFT-NEXT:  .LBB55_14: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB55_4
-; SOFT-NEXT:  .LBB55_15: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB55_5
-; SOFT-NEXT:    b .LBB55_6
-; SOFT-NEXT:  .LBB55_16: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB55_10
-; SOFT-NEXT:  .LBB55_17: @ in Loop: Header=BB55_2 Depth=1
-; SOFT-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB55_11
-; SOFT-NEXT:    b .LBB55_12
-; SOFT-NEXT:  .LBB55_18:
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:  .LBB55_14:
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.19:
+; SOFT-NEXT:  @ %bb.15:
 ; SOFT-NEXT:  .LCPI55_0:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
index 78090083a0026..4d091c2302658 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
@@ -9,58 +9,58 @@ define <2 x i32> @stest_f64i32(<2 x double> %x) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r11, lr}
 ; CHECK-NEXT:    push {r4, r5, r11, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
-; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vmov r0, r1, d9
-; CHECK-NEXT:    adr r2, .LCPI0_0
-; CHECK-NEXT:    vld1.64 {d8, d9}, [r2:128]
-; CHECK-NEXT:    vmov.32 d10[0], r4
+; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov.32 d9[0], r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vmov.32 d8[0], r0
 ; CHECK-NEXT:    mvn r3, #-2147483648
 ; CHECK-NEXT:    subs r4, r4, r3
-; CHECK-NEXT:    sbcs r4, r5, #0
-; CHECK-NEXT:    vmov.32 d11[0], r0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r4, #1
+; CHECK-NEXT:    adr r2, .LCPI0_0
+; CHECK-NEXT:    vmov.32 d9[1], r5
+; CHECK-NEXT:    sbcs r5, r5, #0
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    mvn r4, #0
+; CHECK-NEXT:    movwlt r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    mvnne r5, #0
 ; CHECK-NEXT:    subs r0, r0, r3
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    vmov.32 d11[1], r1
+; CHECK-NEXT:    vmov.32 d8[1], r1
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    vmov.i32 q10, #0x80000000
 ; CHECK-NEXT:    movwlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vdup.32 d19, r5
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    vmov.32 d10[1], r5
-; CHECK-NEXT:    mvnne r4, #0
-; CHECK-NEXT:    vdup.32 d17, r0
-; CHECK-NEXT:    vdup.32 d16, r4
-; CHECK-NEXT:    mvn r4, #0
-; CHECK-NEXT:    vbsl q8, q5, q4
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r3, r5, d17
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2:128]
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    vdup.32 d18, r0
+; CHECK-NEXT:    vbit q8, q4, q9
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r3, r5, d16
 ; CHECK-NEXT:    rsbs r0, r0, #-2147483648
 ; CHECK-NEXT:    sbcs r0, r4, r1
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
 ; CHECK-NEXT:    rsbs r1, r3, #-2147483648
 ; CHECK-NEXT:    sbcs r1, r4, r5
+; CHECK-NEXT:    vdup.32 d19, r0
 ; CHECK-NEXT:    movwlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vdup.32 d19, r2
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d18, r0
+; CHECK-NEXT:    vdup.32 d18, r2
 ; CHECK-NEXT:    vbif q8, q10, q9
 ; CHECK-NEXT:    vmovn.i64 d0, q8
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r11, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.1:
@@ -95,21 +95,21 @@ define <2 x i32> @utest_f64i32(<2 x double> %x) {
 ; CHECK-NEXT:    vmov.32 d9[0], r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    mvn r3, #0
+; CHECK-NEXT:    subs r4, r4, r3
+; CHECK-NEXT:    sbcs r5, r5, #0
 ; CHECK-NEXT:    vmov.32 d8[0], r0
-; CHECK-NEXT:    subs r0, r0, r3
+; CHECK-NEXT:    mov r5, #0
 ; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movwlo r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    mvnne r5, #0
+; CHECK-NEXT:    subs r0, r0, r3
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movwlo r0, #1
-; CHECK-NEXT:    subs r1, r4, r3
-; CHECK-NEXT:    sbcs r1, r5, #0
+; CHECK-NEXT:    vdup.32 d17, r5
 ; CHECK-NEXT:    movwlo r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vdup.32 d17, r2
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d16, r0
+; CHECK-NEXT:    vdup.32 d16, r2
 ; CHECK-NEXT:    vand q9, q4, q8
 ; CHECK-NEXT:    vorn q8, q9, q8
 ; CHECK-NEXT:    vmovn.i64 d0, q8
@@ -131,49 +131,49 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) {
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
-; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vmov r0, r1, d9
-; CHECK-NEXT:    vmov.32 d8[0], r4
+; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov.32 d9[0], r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vmov.32 d8[0], r0
 ; CHECK-NEXT:    mvn r3, #0
 ; CHECK-NEXT:    subs r4, r4, r3
-; CHECK-NEXT:    sbcs r4, r5, #0
-; CHECK-NEXT:    vmov.32 d9[0], r0
-; CHECK-NEXT:    mov r4, #0
 ; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    movwlt r4, #1
+; CHECK-NEXT:    vmov.32 d9[1], r5
+; CHECK-NEXT:    sbcs r5, r5, #0
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movwlt r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    mvnne r5, #0
 ; CHECK-NEXT:    subs r0, r0, r3
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    vmov.32 d9[1], r1
+; CHECK-NEXT:    vmov.32 d8[1], r1
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
 ; CHECK-NEXT:    movwlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vdup.32 d17, r5
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    vmov.32 d8[1], r5
-; CHECK-NEXT:    mvnne r4, #0
-; CHECK-NEXT:    vdup.32 d17, r0
-; CHECK-NEXT:    vdup.32 d16, r4
+; CHECK-NEXT:    vdup.32 d16, r0
 ; CHECK-NEXT:    vbsl q8, q4, q9
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vmov r3, r5, d17
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r3, r5, d16
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    rscs r0, r1, #0
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
 ; CHECK-NEXT:    rsbs r1, r3, #0
 ; CHECK-NEXT:    rscs r1, r5, #0
+; CHECK-NEXT:    vmov.32 d19[0], r0
 ; CHECK-NEXT:    movwlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vmov.32 d19[0], r2
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vmov.32 d18[0], r0
+; CHECK-NEXT:    vmov.32 d18[0], r2
 ; CHECK-NEXT:    vand q8, q8, q9
 ; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9}
@@ -195,106 +195,103 @@ define <4 x i32> @stest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, sp, #4
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vorr q4, q0, q0
-; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    vmov r6, s17
-; CHECK-NEXT:    vmov r10, s19
-; CHECK-NEXT:    vmov.32 d8[0], r7
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov.32 d10[0], r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov.32 d9[0], r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r11, r1
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    adr r1, .LCPI3_0
+; CHECK-NEXT:    vld1.64 {d10, d11}, [r1:128]
+; CHECK-NEXT:    vmov r5, s17
+; CHECK-NEXT:    mov r4, #0
+; CHECK-NEXT:    mvn r9, #-2147483648
+; CHECK-NEXT:    vmov.32 d13[0], r6
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    subs r3, r7, r6
-; CHECK-NEXT:    sbcs r3, r8, #0
-; CHECK-NEXT:    vmov.32 d11[0], r0
-; CHECK-NEXT:    mov r3, #0
-; CHECK-NEXT:    adr r2, .LCPI3_0
-; CHECK-NEXT:    movwlt r3, #1
-; CHECK-NEXT:    subs r7, r5, r6
-; CHECK-NEXT:    sbcs r7, r4, #0
-; CHECK-NEXT:    vmov.32 d11[1], r1
-; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    movwlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    mvnne r7, #0
-; CHECK-NEXT:    subs r0, r0, r6
+; CHECK-NEXT:    subs r2, r6, r9
+; CHECK-NEXT:    vmov.32 d12[0], r0
+; CHECK-NEXT:    sbcs r2, r7, #0
+; CHECK-NEXT:    vmov r8, s16
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    vmov.32 d13[1], r7
+; CHECK-NEXT:    movwlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    subs r0, r0, r9
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    vld1.64 {d18, d19}, [r2:128]
+; CHECK-NEXT:    vdup.32 d17, r2
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    vmov.32 d12[1], r1
 ; CHECK-NEXT:    movwlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vmov.32 d10[1], r4
-; CHECK-NEXT:    vdup.32 d17, r0
-; CHECK-NEXT:    subs r0, r9, r6
-; CHECK-NEXT:    sbcs r0, r11, #0
-; CHECK-NEXT:    vdup.32 d16, r7
+; CHECK-NEXT:    vdup.32 d16, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    vorr q4, q8, q8
+; CHECK-NEXT:    vbsl q4, q6, q5
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    vmov.32 d13[0], r0
+; CHECK-NEXT:    subs r0, r0, r9
+; CHECK-NEXT:    sbcs r0, r1, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r11, r10, d8
+; CHECK-NEXT:    vmov.32 d13[1], r1
+; CHECK-NEXT:    mvnne r6, #0
+; CHECK-NEXT:    vmov r5, r7, d9
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    vmov.32 d12[0], r0
+; CHECK-NEXT:    subs r0, r0, r9
+; CHECK-NEXT:    sbcs r0, r1, #0
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    vbsl q8, q5, q9
+; CHECK-NEXT:    vdup.32 d17, r6
 ; CHECK-NEXT:    movwlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vmov.32 d9[1], r11
 ; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    vmov.32 d12[1], r1
+; CHECK-NEXT:    rsbs r3, r11, #-2147483648
+; CHECK-NEXT:    vdup.32 d16, r0
+; CHECK-NEXT:    mvn r0, #0
+; CHECK-NEXT:    vbsl q8, q6, q5
+; CHECK-NEXT:    adr r1, .LCPI3_1
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1:128]
+; CHECK-NEXT:    sbcs r3, r0, r10
+; CHECK-NEXT:    mov r3, #0
+; CHECK-NEXT:    vmov r1, r2, d17
+; CHECK-NEXT:    movwlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    mvn r6, #0
-; CHECK-NEXT:    vdup.32 d21, r0
 ; CHECK-NEXT:    mvnne r3, #0
-; CHECK-NEXT:    vmov.32 d8[1], r8
-; CHECK-NEXT:    vmov r0, r1, d16
-; CHECK-NEXT:    vdup.32 d20, r3
-; CHECK-NEXT:    vbit q9, q4, q10
-; CHECK-NEXT:    adr r5, .LCPI3_1
-; CHECK-NEXT:    vld1.64 {d20, d21}, [r5:128]
-; CHECK-NEXT:    vmov r5, r4, d17
-; CHECK-NEXT:    vmov r3, r7, d18
-; CHECK-NEXT:    rsbs r0, r0, #-2147483648
-; CHECK-NEXT:    sbcs r0, r6, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    rsbs r1, r3, #-2147483648
-; CHECK-NEXT:    vmov r1, r3, d19
-; CHECK-NEXT:    sbcs r7, r6, r7
+; CHECK-NEXT:    rsbs r6, r5, #-2147483648
+; CHECK-NEXT:    vmov r6, r5, d16
+; CHECK-NEXT:    sbcs r7, r0, r7
 ; CHECK-NEXT:    mov r7, #0
 ; CHECK-NEXT:    movwlt r7, #1
-; CHECK-NEXT:    rsbs r5, r5, #-2147483648
-; CHECK-NEXT:    sbcs r5, r6, r4
-; CHECK-NEXT:    mov r5, #0
-; CHECK-NEXT:    movwlt r5, #1
-; CHECK-NEXT:    rsbs r1, r1, #-2147483648
-; CHECK-NEXT:    sbcs r1, r6, r3
-; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    mvnne r5, #0
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vdup.32 d25, r5
 ; CHECK-NEXT:    mvnne r7, #0
-; CHECK-NEXT:    vdup.32 d23, r2
-; CHECK-NEXT:    vdup.32 d24, r0
-; CHECK-NEXT:    vbif q8, q10, q12
-; CHECK-NEXT:    vdup.32 d22, r7
-; CHECK-NEXT:    vbif q9, q10, q11
-; CHECK-NEXT:    vmovn.i64 d1, q8
-; CHECK-NEXT:    vmovn.i64 d0, q9
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    vdup.32 d23, r7
+; CHECK-NEXT:    vdup.32 d22, r3
+; CHECK-NEXT:    vbsl q11, q4, q9
+; CHECK-NEXT:    vmovn.i64 d1, q11
+; CHECK-NEXT:    rsbs r1, r1, #-2147483648
+; CHECK-NEXT:    sbcs r1, r0, r2
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    mvnne r1, #0
+; CHECK-NEXT:    rsbs r2, r6, #-2147483648
+; CHECK-NEXT:    sbcs r0, r0, r5
+; CHECK-NEXT:    vdup.32 d21, r1
+; CHECK-NEXT:    movwlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mvnne r4, #0
+; CHECK-NEXT:    vdup.32 d20, r4
+; CHECK-NEXT:    vbif q8, q9, q10
+; CHECK-NEXT:    vmovn.i64 d0, q8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, sp, #4
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 4
@@ -329,52 +326,52 @@ define <4 x i32> @utest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vmov r0, s17
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r10, r0
 ; CHECK-NEXT:    vmov r0, s16
 ; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    vmov r6, s19
+; CHECK-NEXT:    vmov r5, s19
 ; CHECK-NEXT:    vmov r7, s18
-; CHECK-NEXT:    vmov.32 d9[0], r9
+; CHECK-NEXT:    vmov.32 d9[0], r10
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov.32 d8[0], r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r9, r1
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    vmov.32 d11[0], r0
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    mvn r7, #0
-; CHECK-NEXT:    subs r2, r5, r7
-; CHECK-NEXT:    sbcs r2, r4, #0
+; CHECK-NEXT:    mvn r3, #0
 ; CHECK-NEXT:    vmov.32 d10[0], r0
+; CHECK-NEXT:    subs r0, r0, r3
 ; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    mov r3, #0
-; CHECK-NEXT:    movwlo r2, #1
-; CHECK-NEXT:    subs r0, r0, r7
 ; CHECK-NEXT:    sbcs r0, r1, #0
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    subs r1, r6, r7
-; CHECK-NEXT:    sbcs r1, r10, #0
+; CHECK-NEXT:    subs r1, r5, r3
+; CHECK-NEXT:    sbcs r1, r4, #0
 ; CHECK-NEXT:    mov r1, #0
 ; CHECK-NEXT:    movwlo r1, #1
-; CHECK-NEXT:    subs r7, r9, r7
-; CHECK-NEXT:    sbcs r7, r8, #0
-; CHECK-NEXT:    movwlo r3, #1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    mvnne r3, #0
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    mvnne r1, #0
-; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    subs r7, r10, r3
+; CHECK-NEXT:    sbcs r7, r8, #0
 ; CHECK-NEXT:    vdup.32 d19, r1
-; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    vdup.32 d17, r3
+; CHECK-NEXT:    mov r7, #0
 ; CHECK-NEXT:    vdup.32 d18, r0
+; CHECK-NEXT:    movwlo r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mvnne r7, #0
+; CHECK-NEXT:    subs r3, r6, r3
+; CHECK-NEXT:    sbcs r3, r9, #0
+; CHECK-NEXT:    vdup.32 d17, r7
+; CHECK-NEXT:    movwlo r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
 ; CHECK-NEXT:    vand q10, q5, q9
 ; CHECK-NEXT:    vdup.32 d16, r2
 ; CHECK-NEXT:    vand q11, q4, q8
@@ -400,97 +397,96 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vorr q4, q0, q0
-; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    vmov.32 d16[0], r2
-; CHECK-NEXT:    mvn r4, #0
-; CHECK-NEXT:    subs r2, r2, r4
-; CHECK-NEXT:    vmov r8, s19
-; CHECK-NEXT:    sbcs r2, r1, #0
-; CHECK-NEXT:    vmov.32 d17[0], r5
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    vmov r5, s17
+; CHECK-NEXT:    vmov r8, s16
+; CHECK-NEXT:    vmov.32 d9[0], r6
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    mvn r9, #0
+; CHECK-NEXT:    subs r2, r6, r9
+; CHECK-NEXT:    sbcs r2, r7, #0
+; CHECK-NEXT:    vmov.32 d8[0], r0
 ; CHECK-NEXT:    mov r2, #0
 ; CHECK-NEXT:    vmov.i64 q5, #0xffffffff
 ; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    subs r3, r5, r4
-; CHECK-NEXT:    sbcs r3, r6, #0
-; CHECK-NEXT:    vmov.32 d17[1], r6
-; CHECK-NEXT:    mov r3, #0
-; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    movwlt r3, #1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    mvnne r3, #0
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    vdup.32 d19, r3
 ; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    vdup.32 d18, r2
-; CHECK-NEXT:    vmov.32 d16[1], r1
-; CHECK-NEXT:    vorr q4, q9, q9
-; CHECK-NEXT:    vbsl q4, q8, q5
-; CHECK-NEXT:    vmov r10, r9, d8
+; CHECK-NEXT:    subs r0, r0, r9
+; CHECK-NEXT:    sbcs r0, r1, #0
+; CHECK-NEXT:    vmov.32 d9[1], r7
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    mov r4, #0
+; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov.32 d8[1], r1
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    vdup.32 d17, r2
+; CHECK-NEXT:    vdup.32 d16, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    vbif q4, q5, q8
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov.32 d12[0], r0
+; CHECK-NEXT:    vmov.32 d13[0], r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    vmov r7, r10, d8
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    subs r2, r5, r4
-; CHECK-NEXT:    vmov.32 d13[0], r0
+; CHECK-NEXT:    subs r2, r5, r9
+; CHECK-NEXT:    vmov.32 d12[0], r0
 ; CHECK-NEXT:    sbcs r2, r6, #0
 ; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    vmov.32 d13[1], r6
 ; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    subs r0, r0, r4
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    subs r0, r0, r9
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    vmov.32 d13[1], r1
+; CHECK-NEXT:    vdup.32 d17, r2
 ; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    vmov r5, r4, d9
+; CHECK-NEXT:    vmov.32 d12[1], r1
 ; CHECK-NEXT:    movwlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    vmov.32 d12[1], r6
-; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    vdup.32 d17, r0
-; CHECK-NEXT:    rsbs r0, r10, #0
-; CHECK-NEXT:    vdup.32 d16, r2
-; CHECK-NEXT:    rscs r0, r9, #0
+; CHECK-NEXT:    vmov r2, r3, d9
+; CHECK-NEXT:    vdup.32 d16, r0
+; CHECK-NEXT:    rsbs r7, r7, #0
 ; CHECK-NEXT:    vbsl q8, q6, q5
+; CHECK-NEXT:    rscs r7, r10, #0
+; CHECK-NEXT:    mov r7, #0
+; CHECK-NEXT:    movwlt r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    mvnne r7, #0
+; CHECK-NEXT:    vmov r6, r5, d16
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    rscs r0, r1, #0
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    vmov r1, r2, d16
-; CHECK-NEXT:    vmov r3, r6, d17
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    rscs r1, r2, #0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    rscs r1, r3, #0
+; CHECK-NEXT:    vmov.32 d19[0], r0
 ; CHECK-NEXT:    mov r1, #0
 ; CHECK-NEXT:    movwlt r1, #1
-; CHECK-NEXT:    rsbs r2, r3, #0
-; CHECK-NEXT:    rscs r2, r6, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    rsbs r3, r5, #0
-; CHECK-NEXT:    rscs r3, r4, #0
-; CHECK-NEXT:    movwlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    mvnne r7, #0
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    mvnne r2, #0
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    mvnne r1, #0
-; CHECK-NEXT:    vmov.32 d21[0], r2
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vmov.32 d20[0], r1
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vmov.32 d19[0], r7
-; CHECK-NEXT:    vand q8, q8, q10
-; CHECK-NEXT:    vmov.32 d18[0], r0
-; CHECK-NEXT:    vmovn.i64 d1, q8
-; CHECK-NEXT:    vand q9, q4, q9
-; CHECK-NEXT:    vmovn.i64 d0, q9
+; CHECK-NEXT:    rsbs r0, r6, #0
+; CHECK-NEXT:    rscs r0, r5, #0
+; CHECK-NEXT:    vmov.32 d21[0], r1
+; CHECK-NEXT:    movwlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov.32 d20[0], r7
+; CHECK-NEXT:    mvnne r4, #0
+; CHECK-NEXT:    vmov.32 d18[0], r4
+; CHECK-NEXT:    vand q10, q4, q10
+; CHECK-NEXT:    vand q8, q8, q9
+; CHECK-NEXT:    vmovn.i64 d1, q10
+; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
@@ -510,112 +506,108 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEON-NEXT:    .pad #4
 ; CHECK-NEON-NEXT:    sub sp, sp, #4
-; CHECK-NEON-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEON-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEON-NEXT:    vmov r0, s0
+; CHECK-NEON-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEON-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEON-NEXT:    vmov r0, s2
 ; CHECK-NEON-NEXT:    vmov.f32 s16, s3
-; CHECK-NEON-NEXT:    vmov.f32 s18, s2
-; CHECK-NEON-NEXT:    vmov.f32 s20, s1
+; CHECK-NEON-NEXT:    vmov.f32 s18, s1
+; CHECK-NEON-NEXT:    vmov.f32 s20, s0
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r9, r0
-; CHECK-NEON-NEXT:    vmov r0, s18
-; CHECK-NEON-NEXT:    vmov r10, s16
-; CHECK-NEON-NEXT:    mov r8, r1
-; CHECK-NEON-NEXT:    vmov r6, s20
-; CHECK-NEON-NEXT:    vmov.32 d8[0], r9
+; CHECK-NEON-NEXT:    mov r6, r0
+; CHECK-NEON-NEXT:    vmov r0, s16
+; CHECK-NEON-NEXT:    mov r5, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r5, r0
-; CHECK-NEON-NEXT:    vmov.32 d10[0], r0
-; CHECK-NEON-NEXT:    mov r0, r6
-; CHECK-NEON-NEXT:    mov r4, r1
-; CHECK-NEON-NEXT:    bl __aeabi_h2f
-; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r11, r0
-; CHECK-NEON-NEXT:    vmov.32 d9[0], r0
-; CHECK-NEON-NEXT:    mov r0, r10
-; CHECK-NEON-NEXT:    mov r7, r1
-; CHECK-NEON-NEXT:    bl __aeabi_h2f
-; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mvn r6, #-2147483648
-; CHECK-NEON-NEXT:    subs r3, r9, r6
-; CHECK-NEON-NEXT:    sbcs r3, r8, #0
-; CHECK-NEON-NEXT:    vmov.32 d11[0], r0
-; CHECK-NEON-NEXT:    mov r3, #0
-; CHECK-NEON-NEXT:    adr r2, .LCPI6_0
-; CHECK-NEON-NEXT:    movwlt r3, #1
-; CHECK-NEON-NEXT:    subs r5, r5, r6
-; CHECK-NEON-NEXT:    sbcs r5, r4, #0
-; CHECK-NEON-NEXT:    vmov.32 d11[1], r1
-; CHECK-NEON-NEXT:    mov r5, #0
-; CHECK-NEON-NEXT:    movwlt r5, #1
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    mvnne r5, #0
-; CHECK-NEON-NEXT:    subs r0, r0, r6
+; CHECK-NEON-NEXT:    vmov r2, s18
+; CHECK-NEON-NEXT:    adr r3, .LCPI6_0
+; CHECK-NEON-NEXT:    vld1.64 {d8, d9}, [r3:128]
+; CHECK-NEON-NEXT:    mvn r9, #-2147483648
+; CHECK-NEON-NEXT:    subs r3, r6, r9
+; CHECK-NEON-NEXT:    mov r4, #0
+; CHECK-NEON-NEXT:    sbcs r3, r5, #0
+; CHECK-NEON-NEXT:    vmov.32 d15[0], r0
+; CHECK-NEON-NEXT:    movwlt r4, #1
+; CHECK-NEON-NEXT:    cmp r4, #0
+; CHECK-NEON-NEXT:    mvnne r4, #0
+; CHECK-NEON-NEXT:    subs r0, r0, r9
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
-; CHECK-NEON-NEXT:    vld1.64 {d18, d19}, [r2:128]
+; CHECK-NEON-NEXT:    vmov.32 d14[0], r6
 ; CHECK-NEON-NEXT:    mov r0, #0
-; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    vmov r8, s20
 ; CHECK-NEON-NEXT:    movwlt r0, #1
 ; CHECK-NEON-NEXT:    cmp r0, #0
 ; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    vmov.32 d10[1], r4
-; CHECK-NEON-NEXT:    vdup.32 d17, r0
-; CHECK-NEON-NEXT:    subs r0, r11, r6
-; CHECK-NEON-NEXT:    sbcs r0, r7, #0
-; CHECK-NEON-NEXT:    vdup.32 d16, r5
+; CHECK-NEON-NEXT:    vmov.32 d15[1], r1
+; CHECK-NEON-NEXT:    mov r7, #0
+; CHECK-NEON-NEXT:    vdup.32 d11, r0
+; CHECK-NEON-NEXT:    vmov.32 d14[1], r5
+; CHECK-NEON-NEXT:    mov r0, r2
+; CHECK-NEON-NEXT:    bl __aeabi_h2f
+; CHECK-NEON-NEXT:    vdup.32 d10, r4
+; CHECK-NEON-NEXT:    bl __aeabi_f2lz
+; CHECK-NEON-NEXT:    vmov.32 d13[0], r0
+; CHECK-NEON-NEXT:    subs r0, r0, r9
+; CHECK-NEON-NEXT:    vbsl q5, q7, q4
+; CHECK-NEON-NEXT:    sbcs r0, r1, #0
+; CHECK-NEON-NEXT:    mov r6, #0
+; CHECK-NEON-NEXT:    mov r0, r8
+; CHECK-NEON-NEXT:    movwlt r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    vmov r11, r10, d10
+; CHECK-NEON-NEXT:    vmov.32 d13[1], r1
+; CHECK-NEON-NEXT:    mvnne r6, #0
+; CHECK-NEON-NEXT:    vmov r5, r4, d11
+; CHECK-NEON-NEXT:    bl __aeabi_h2f
+; CHECK-NEON-NEXT:    bl __aeabi_f2lz
+; CHECK-NEON-NEXT:    vmov.32 d12[0], r0
+; CHECK-NEON-NEXT:    subs r0, r0, r9
+; CHECK-NEON-NEXT:    sbcs r0, r1, #0
 ; CHECK-NEON-NEXT:    mov r0, #0
-; CHECK-NEON-NEXT:    vbsl q8, q5, q9
+; CHECK-NEON-NEXT:    vdup.32 d17, r6
 ; CHECK-NEON-NEXT:    movwlt r0, #1
 ; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    vmov.32 d9[1], r7
 ; CHECK-NEON-NEXT:    mvnne r0, #0
+; CHECK-NEON-NEXT:    vmov.32 d12[1], r1
+; CHECK-NEON-NEXT:    rsbs r3, r11, #-2147483648
+; CHECK-NEON-NEXT:    vdup.32 d16, r0
+; CHECK-NEON-NEXT:    mvn r0, #0
+; CHECK-NEON-NEXT:    vbsl q8, q6, q4
+; CHECK-NEON-NEXT:    adr r1, .LCPI6_1
+; CHECK-NEON-NEXT:    vld1.64 {d18, d19}, [r1:128]
+; CHECK-NEON-NEXT:    sbcs r3, r0, r10
+; CHECK-NEON-NEXT:    mov r3, #0
+; CHECK-NEON-NEXT:    vmov r1, r2, d17
+; CHECK-NEON-NEXT:    movwlt r3, #1
 ; CHECK-NEON-NEXT:    cmp r3, #0
-; CHECK-NEON-NEXT:    mvn r6, #0
-; CHECK-NEON-NEXT:    vdup.32 d21, r0
 ; CHECK-NEON-NEXT:    mvnne r3, #0
-; CHECK-NEON-NEXT:    vmov.32 d8[1], r8
-; CHECK-NEON-NEXT:    vmov r0, r1, d16
-; CHECK-NEON-NEXT:    vdup.32 d20, r3
-; CHECK-NEON-NEXT:    vbit q9, q4, q10
-; CHECK-NEON-NEXT:    adr r5, .LCPI6_1
-; CHECK-NEON-NEXT:    vld1.64 {d20, d21}, [r5:128]
-; CHECK-NEON-NEXT:    vmov r5, r4, d17
-; CHECK-NEON-NEXT:    vmov r3, r7, d18
-; CHECK-NEON-NEXT:    rsbs r0, r0, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r6, r1
-; CHECK-NEON-NEXT:    mov r0, #0
-; CHECK-NEON-NEXT:    movwlt r0, #1
-; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    rsbs r1, r3, #-2147483648
-; CHECK-NEON-NEXT:    vmov r1, r3, d19
-; CHECK-NEON-NEXT:    sbcs r7, r6, r7
-; CHECK-NEON-NEXT:    mov r7, #0
-; CHECK-NEON-NEXT:    movwlt r7, #1
-; CHECK-NEON-NEXT:    rsbs r5, r5, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r5, r6, r4
-; CHECK-NEON-NEXT:    mov r5, #0
-; CHECK-NEON-NEXT:    movwlt r5, #1
+; CHECK-NEON-NEXT:    rsbs r6, r5, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r6, r0, r4
+; CHECK-NEON-NEXT:    vmov r5, r4, d16
+; CHECK-NEON-NEXT:    mov r6, #0
+; CHECK-NEON-NEXT:    movwlt r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    mvnne r6, #0
+; CHECK-NEON-NEXT:    vdup.32 d23, r6
+; CHECK-NEON-NEXT:    vdup.32 d22, r3
+; CHECK-NEON-NEXT:    vbsl q11, q5, q9
+; CHECK-NEON-NEXT:    vmovn.i64 d1, q11
 ; CHECK-NEON-NEXT:    rsbs r1, r1, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r1, r6, r3
-; CHECK-NEON-NEXT:    movwlt r2, #1
-; CHECK-NEON-NEXT:    cmp r2, #0
-; CHECK-NEON-NEXT:    mvnne r2, #0
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    mvnne r5, #0
+; CHECK-NEON-NEXT:    sbcs r1, r0, r2
+; CHECK-NEON-NEXT:    mov r1, #0
+; CHECK-NEON-NEXT:    movwlt r1, #1
+; CHECK-NEON-NEXT:    cmp r1, #0
+; CHECK-NEON-NEXT:    mvnne r1, #0
+; CHECK-NEON-NEXT:    rsbs r2, r5, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r0, r0, r4
+; CHECK-NEON-NEXT:    vdup.32 d21, r1
+; CHECK-NEON-NEXT:    movwlt r7, #1
 ; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    vdup.32 d25, r5
 ; CHECK-NEON-NEXT:    mvnne r7, #0
-; CHECK-NEON-NEXT:    vdup.32 d23, r2
-; CHECK-NEON-NEXT:    vdup.32 d24, r0
-; CHECK-NEON-NEXT:    vbif q8, q10, q12
-; CHECK-NEON-NEXT:    vdup.32 d22, r7
-; CHECK-NEON-NEXT:    vbif q9, q10, q11
-; CHECK-NEON-NEXT:    vmovn.i64 d1, q8
-; CHECK-NEON-NEXT:    vmovn.i64 d0, q9
-; CHECK-NEON-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEON-NEXT:    vdup.32 d20, r7
+; CHECK-NEON-NEXT:    vbif q8, q9, q10
+; CHECK-NEON-NEXT:    vmovn.i64 d0, q8
+; CHECK-NEON-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEON-NEXT:    add sp, sp, #4
 ; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEON-NEXT:    .p2align 4
@@ -635,111 +627,104 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
 ; CHECK-FP16:       @ %bb.0: @ %entry
 ; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-FP16-NEXT:    .vsave {d10, d11, d12, d13}
-; CHECK-FP16-NEXT:    vpush {d10, d11, d12, d13}
-; CHECK-FP16-NEXT:    .vsave {d8}
-; CHECK-FP16-NEXT:    vpush {d8}
-; CHECK-FP16-NEXT:    vmov.u16 r0, d0[0]
+; CHECK-FP16-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    vmov.u16 r0, d0[3]
 ; CHECK-FP16-NEXT:    vorr d8, d0, d0
-; CHECK-FP16-NEXT:    vmov.u16 r6, d0[1]
+; CHECK-FP16-NEXT:    vmov.u16 r8, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r9, d0[1]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
 ; CHECK-FP16-NEXT:    mov r4, r0
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d8[2]
-; CHECK-FP16-NEXT:    mov r8, r1
-; CHECK-FP16-NEXT:    vmov.32 d10[0], r4
-; CHECK-FP16-NEXT:    vmov s0, r0
-; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    vmov s0, r6
-; CHECK-FP16-NEXT:    mov r5, r0
-; CHECK-FP16-NEXT:    mov r7, r1
-; CHECK-FP16-NEXT:    vmov.32 d12[0], r0
-; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    mov r9, r0
-; CHECK-FP16-NEXT:    vmov.u16 r0, d8[3]
-; CHECK-FP16-NEXT:    mov r10, r1
-; CHECK-FP16-NEXT:    vmov.32 d11[0], r9
+; CHECK-FP16-NEXT:    mov r5, r1
+; CHECK-FP16-NEXT:    vmov.32 d9[0], r4
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    mvn r6, #-2147483648
-; CHECK-FP16-NEXT:    subs r3, r4, r6
-; CHECK-FP16-NEXT:    sbcs r3, r8, #0
-; CHECK-FP16-NEXT:    vmov.32 d13[0], r0
-; CHECK-FP16-NEXT:    mov r3, #0
 ; CHECK-FP16-NEXT:    adr r2, .LCPI6_0
-; CHECK-FP16-NEXT:    movwlt r3, #1
-; CHECK-FP16-NEXT:    subs r5, r5, r6
-; CHECK-FP16-NEXT:    sbcs r5, r7, #0
-; CHECK-FP16-NEXT:    vmov.32 d13[1], r1
-; CHECK-FP16-NEXT:    mov r5, #0
-; CHECK-FP16-NEXT:    movwlt r5, #1
-; CHECK-FP16-NEXT:    cmp r5, #0
-; CHECK-FP16-NEXT:    mvnne r5, #0
-; CHECK-FP16-NEXT:    subs r0, r0, r6
+; CHECK-FP16-NEXT:    mvn r10, #-2147483648
+; CHECK-FP16-NEXT:    vld1.64 {d10, d11}, [r2:128]
+; CHECK-FP16-NEXT:    subs r2, r4, r10
+; CHECK-FP16-NEXT:    sbcs r2, r5, #0
+; CHECK-FP16-NEXT:    vmov s0, r9
+; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    vmov.32 d8[0], r0
+; CHECK-FP16-NEXT:    movwlt r2, #1
+; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    mvnne r2, #0
+; CHECK-FP16-NEXT:    subs r0, r0, r10
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    vld1.64 {d18, d19}, [r2:128]
+; CHECK-FP16-NEXT:    vmov.32 d9[1], r5
 ; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    mov r6, #0
 ; CHECK-FP16-NEXT:    movwlt r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
+; CHECK-FP16-NEXT:    vmov.32 d8[1], r1
 ; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    vmov.32 d12[1], r7
-; CHECK-FP16-NEXT:    vdup.32 d17, r0
-; CHECK-FP16-NEXT:    subs r0, r9, r6
-; CHECK-FP16-NEXT:    sbcs r0, r10, #0
-; CHECK-FP16-NEXT:    vdup.32 d16, r5
+; CHECK-FP16-NEXT:    vdup.32 d17, r2
+; CHECK-FP16-NEXT:    vdup.32 d16, r0
+; CHECK-FP16-NEXT:    vbif q4, q5, q8
+; CHECK-FP16-NEXT:    bl __fixhfdi
+; CHECK-FP16-NEXT:    vmov.32 d13[0], r0
+; CHECK-FP16-NEXT:    subs r0, r0, r10
+; CHECK-FP16-NEXT:    vmov s0, r8
+; CHECK-FP16-NEXT:    sbcs r0, r1, #0
+; CHECK-FP16-NEXT:    mov r7, #0
+; CHECK-FP16-NEXT:    vmov r9, r8, d8
+; CHECK-FP16-NEXT:    movwlt r7, #1
+; CHECK-FP16-NEXT:    cmp r7, #0
+; CHECK-FP16-NEXT:    vmov.32 d13[1], r1
+; CHECK-FP16-NEXT:    vmov r5, r4, d9
+; CHECK-FP16-NEXT:    mvnne r7, #0
+; CHECK-FP16-NEXT:    bl __fixhfdi
+; CHECK-FP16-NEXT:    vmov.32 d12[0], r0
+; CHECK-FP16-NEXT:    subs r0, r0, r10
+; CHECK-FP16-NEXT:    sbcs r0, r1, #0
 ; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    vbsl q8, q6, q9
+; CHECK-FP16-NEXT:    vdup.32 d17, r7
 ; CHECK-FP16-NEXT:    movwlt r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    vmov.32 d11[1], r10
 ; CHECK-FP16-NEXT:    mvnne r0, #0
+; CHECK-FP16-NEXT:    vmov.32 d12[1], r1
+; CHECK-FP16-NEXT:    rsbs r3, r9, #-2147483648
+; CHECK-FP16-NEXT:    vdup.32 d16, r0
+; CHECK-FP16-NEXT:    mvn r0, #0
+; CHECK-FP16-NEXT:    vbsl q8, q6, q5
+; CHECK-FP16-NEXT:    adr r1, .LCPI6_1
+; CHECK-FP16-NEXT:    vld1.64 {d18, d19}, [r1:128]
+; CHECK-FP16-NEXT:    sbcs r3, r0, r8
+; CHECK-FP16-NEXT:    mov r3, #0
+; CHECK-FP16-NEXT:    vmov r1, r2, d17
+; CHECK-FP16-NEXT:    movwlt r3, #1
 ; CHECK-FP16-NEXT:    cmp r3, #0
-; CHECK-FP16-NEXT:    mvn r6, #0
-; CHECK-FP16-NEXT:    vdup.32 d21, r0
 ; CHECK-FP16-NEXT:    mvnne r3, #0
-; CHECK-FP16-NEXT:    vmov.32 d10[1], r8
-; CHECK-FP16-NEXT:    vmov r0, r1, d16
-; CHECK-FP16-NEXT:    vdup.32 d20, r3
-; CHECK-FP16-NEXT:    vbit q9, q5, q10
-; CHECK-FP16-NEXT:    adr r5, .LCPI6_1
-; CHECK-FP16-NEXT:    vld1.64 {d20, d21}, [r5:128]
-; CHECK-FP16-NEXT:    vmov r5, r4, d17
-; CHECK-FP16-NEXT:    vmov r3, r7, d18
-; CHECK-FP16-NEXT:    rsbs r0, r0, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r6, r1
-; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    rsbs r1, r3, #-2147483648
-; CHECK-FP16-NEXT:    vmov r1, r3, d19
-; CHECK-FP16-NEXT:    sbcs r7, r6, r7
+; CHECK-FP16-NEXT:    rsbs r7, r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r7, r0, r4
+; CHECK-FP16-NEXT:    vmov r5, r4, d16
 ; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    movwlt r7, #1
-; CHECK-FP16-NEXT:    rsbs r5, r5, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r5, r6, r4
-; CHECK-FP16-NEXT:    mov r5, #0
-; CHECK-FP16-NEXT:    movwlt r5, #1
-; CHECK-FP16-NEXT:    rsbs r1, r1, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r1, r6, r3
-; CHECK-FP16-NEXT:    movwlt r2, #1
-; CHECK-FP16-NEXT:    cmp r2, #0
-; CHECK-FP16-NEXT:    mvnne r2, #0
-; CHECK-FP16-NEXT:    cmp r5, #0
-; CHECK-FP16-NEXT:    mvnne r5, #0
 ; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    vdup.32 d25, r5
 ; CHECK-FP16-NEXT:    mvnne r7, #0
-; CHECK-FP16-NEXT:    vdup.32 d23, r2
-; CHECK-FP16-NEXT:    vdup.32 d24, r0
-; CHECK-FP16-NEXT:    vbif q8, q10, q12
-; CHECK-FP16-NEXT:    vdup.32 d22, r7
-; CHECK-FP16-NEXT:    vbif q9, q10, q11
-; CHECK-FP16-NEXT:    vmovn.i64 d1, q8
-; CHECK-FP16-NEXT:    vmovn.i64 d0, q9
-; CHECK-FP16-NEXT:    vpop {d8}
-; CHECK-FP16-NEXT:    vpop {d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    vdup.32 d23, r7
+; CHECK-FP16-NEXT:    vdup.32 d22, r3
+; CHECK-FP16-NEXT:    vbsl q11, q4, q9
+; CHECK-FP16-NEXT:    vmovn.i64 d1, q11
+; CHECK-FP16-NEXT:    rsbs r1, r1, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r0, r2
+; CHECK-FP16-NEXT:    mov r1, #0
+; CHECK-FP16-NEXT:    movwlt r1, #1
+; CHECK-FP16-NEXT:    cmp r1, #0
+; CHECK-FP16-NEXT:    mvnne r1, #0
+; CHECK-FP16-NEXT:    rsbs r2, r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r0, r0, r4
+; CHECK-FP16-NEXT:    vdup.32 d21, r1
+; CHECK-FP16-NEXT:    movwlt r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    mvnne r6, #0
+; CHECK-FP16-NEXT:    vdup.32 d20, r6
+; CHECK-FP16-NEXT:    vbif q8, q9, q10
+; CHECK-FP16-NEXT:    vmovn.i64 d0, q8
+; CHECK-FP16-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 ; CHECK-FP16-NEXT:    .p2align 4
 ; CHECK-FP16-NEXT:  @ %bb.1:
@@ -766,8 +751,8 @@ entry:
 define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NEON-LABEL: utesth_f16i32:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
 ; CHECK-NEON-NEXT:    .vsave {d12, d13}
 ; CHECK-NEON-NEXT:    vpush {d12, d13}
 ; CHECK-NEON-NEXT:    .vsave {d8, d9, d10}
@@ -778,7 +763,7 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    vmov.f32 s20, s0
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEON-NEXT:    mov r10, r0
+; CHECK-NEON-NEXT:    mov r4, r0
 ; CHECK-NEON-NEXT:    vmov r0, s18
 ; CHECK-NEON-NEXT:    mov r8, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
@@ -794,37 +779,37 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    vmov r0, s16
 ; CHECK-NEON-NEXT:    mov r7, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
-; CHECK-NEON-NEXT:    vmov.32 d9[0], r10
+; CHECK-NEON-NEXT:    vmov.32 d9[0], r4
 ; CHECK-NEON-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEON-NEXT:    mvn r4, #0
-; CHECK-NEON-NEXT:    subs r2, r5, r4
-; CHECK-NEON-NEXT:    sbcs r2, r7, #0
+; CHECK-NEON-NEXT:    mvn r3, #0
 ; CHECK-NEON-NEXT:    vmov.32 d8[0], r0
+; CHECK-NEON-NEXT:    subs r0, r0, r3
 ; CHECK-NEON-NEXT:    mov r2, #0
-; CHECK-NEON-NEXT:    mov r3, #0
-; CHECK-NEON-NEXT:    movwlo r2, #1
-; CHECK-NEON-NEXT:    subs r0, r0, r4
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
 ; CHECK-NEON-NEXT:    mov r0, #0
 ; CHECK-NEON-NEXT:    movwlo r0, #1
 ; CHECK-NEON-NEXT:    cmp r0, #0
 ; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    subs r1, r10, r4
+; CHECK-NEON-NEXT:    subs r1, r4, r3
 ; CHECK-NEON-NEXT:    sbcs r1, r8, #0
 ; CHECK-NEON-NEXT:    mov r1, #0
 ; CHECK-NEON-NEXT:    movwlo r1, #1
-; CHECK-NEON-NEXT:    subs r7, r6, r4
-; CHECK-NEON-NEXT:    sbcs r7, r9, #0
-; CHECK-NEON-NEXT:    movwlo r3, #1
-; CHECK-NEON-NEXT:    cmp r3, #0
-; CHECK-NEON-NEXT:    mvnne r3, #0
 ; CHECK-NEON-NEXT:    cmp r1, #0
 ; CHECK-NEON-NEXT:    mvnne r1, #0
-; CHECK-NEON-NEXT:    cmp r2, #0
+; CHECK-NEON-NEXT:    subs r6, r6, r3
+; CHECK-NEON-NEXT:    sbcs r6, r9, #0
 ; CHECK-NEON-NEXT:    vdup.32 d19, r1
-; CHECK-NEON-NEXT:    mvnne r2, #0
-; CHECK-NEON-NEXT:    vdup.32 d17, r3
+; CHECK-NEON-NEXT:    mov r6, #0
 ; CHECK-NEON-NEXT:    vdup.32 d18, r0
+; CHECK-NEON-NEXT:    movwlo r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    mvnne r6, #0
+; CHECK-NEON-NEXT:    subs r3, r5, r3
+; CHECK-NEON-NEXT:    sbcs r3, r7, #0
+; CHECK-NEON-NEXT:    vdup.32 d17, r6
+; CHECK-NEON-NEXT:    movwlo r2, #1
+; CHECK-NEON-NEXT:    cmp r2, #0
+; CHECK-NEON-NEXT:    mvnne r2, #0
 ; CHECK-NEON-NEXT:    vand q10, q4, q9
 ; CHECK-NEON-NEXT:    vdup.32 d16, r2
 ; CHECK-NEON-NEXT:    vand q11, q6, q8
@@ -834,12 +819,12 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10}
 ; CHECK-NEON-NEXT:    vpop {d12, d13}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
 ;
 ; CHECK-FP16-LABEL: utesth_f16i32:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
 ; CHECK-FP16-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-FP16-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
@@ -847,52 +832,52 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-FP16-NEXT:    vmov.u16 r5, d0[3]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfdi
-; CHECK-FP16-NEXT:    mov r10, r0
+; CHECK-FP16-NEXT:    mov r4, r0
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d8[0]
 ; CHECK-FP16-NEXT:    mov r8, r1
-; CHECK-FP16-NEXT:    vmov.32 d11[0], r10
+; CHECK-FP16-NEXT:    vmov.32 d11[0], r4
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfdi
 ; CHECK-FP16-NEXT:    vmov s0, r5
 ; CHECK-FP16-NEXT:    mov r6, r0
-; CHECK-FP16-NEXT:    mov r7, r1
+; CHECK-FP16-NEXT:    mov r9, r1
 ; CHECK-FP16-NEXT:    vmov.32 d10[0], r0
 ; CHECK-FP16-NEXT:    bl __fixunshfdi
 ; CHECK-FP16-NEXT:    mov r5, r0
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d8[2]
-; CHECK-FP16-NEXT:    mov r9, r1
+; CHECK-FP16-NEXT:    mov r7, r1
 ; CHECK-FP16-NEXT:    vmov.32 d9[0], r5
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfdi
-; CHECK-FP16-NEXT:    mvn r4, #0
-; CHECK-FP16-NEXT:    subs r2, r6, r4
-; CHECK-FP16-NEXT:    sbcs r2, r7, #0
+; CHECK-FP16-NEXT:    mvn r3, #0
 ; CHECK-FP16-NEXT:    vmov.32 d8[0], r0
+; CHECK-FP16-NEXT:    subs r0, r0, r3
 ; CHECK-FP16-NEXT:    mov r2, #0
-; CHECK-FP16-NEXT:    mov r3, #0
-; CHECK-FP16-NEXT:    movwlo r2, #1
-; CHECK-FP16-NEXT:    subs r0, r0, r4
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
 ; CHECK-FP16-NEXT:    mov r0, #0
 ; CHECK-FP16-NEXT:    movwlo r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
 ; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    subs r1, r5, r4
-; CHECK-FP16-NEXT:    sbcs r1, r9, #0
+; CHECK-FP16-NEXT:    subs r1, r5, r3
+; CHECK-FP16-NEXT:    sbcs r1, r7, #0
 ; CHECK-FP16-NEXT:    mov r1, #0
 ; CHECK-FP16-NEXT:    movwlo r1, #1
-; CHECK-FP16-NEXT:    subs r7, r10, r4
-; CHECK-FP16-NEXT:    sbcs r7, r8, #0
-; CHECK-FP16-NEXT:    movwlo r3, #1
-; CHECK-FP16-NEXT:    cmp r3, #0
-; CHECK-FP16-NEXT:    mvnne r3, #0
 ; CHECK-FP16-NEXT:    cmp r1, #0
 ; CHECK-FP16-NEXT:    mvnne r1, #0
-; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    subs r7, r4, r3
+; CHECK-FP16-NEXT:    sbcs r7, r8, #0
 ; CHECK-FP16-NEXT:    vdup.32 d19, r1
-; CHECK-FP16-NEXT:    mvnne r2, #0
-; CHECK-FP16-NEXT:    vdup.32 d17, r3
+; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    vdup.32 d18, r0
+; CHECK-FP16-NEXT:    movwlo r7, #1
+; CHECK-FP16-NEXT:    cmp r7, #0
+; CHECK-FP16-NEXT:    mvnne r7, #0
+; CHECK-FP16-NEXT:    subs r3, r6, r3
+; CHECK-FP16-NEXT:    sbcs r3, r9, #0
+; CHECK-FP16-NEXT:    vdup.32 d17, r7
+; CHECK-FP16-NEXT:    movwlo r2, #1
+; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    mvnne r2, #0
 ; CHECK-FP16-NEXT:    vand q10, q4, q9
 ; CHECK-FP16-NEXT:    vdup.32 d16, r2
 ; CHECK-FP16-NEXT:    vand q11, q5, q8
@@ -901,7 +886,7 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-FP16-NEXT:    vmovn.i64 d1, q9
 ; CHECK-FP16-NEXT:    vmovn.i64 d0, q8
 ; CHECK-FP16-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
 entry:
   %conv = fptoui <4 x half> %x to <4 x i64>
   %0 = icmp ult <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -917,211 +902,207 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEON-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEON-NEXT:    vmov r0, s1
-; CHECK-NEON-NEXT:    vmov.f32 s16, s3
-; CHECK-NEON-NEXT:    vmov.f32 s18, s2
+; CHECK-NEON-NEXT:    vmov r0, s3
+; CHECK-NEON-NEXT:    vmov.f32 s16, s2
+; CHECK-NEON-NEXT:    vmov.f32 s18, s1
 ; CHECK-NEON-NEXT:    vmov.f32 s20, s0
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r6, r0
-; CHECK-NEON-NEXT:    vmov r0, s20
-; CHECK-NEON-NEXT:    mov r7, r1
-; CHECK-NEON-NEXT:    vmov r5, s18
-; CHECK-NEON-NEXT:    vmov r8, s16
-; CHECK-NEON-NEXT:    vmov.32 d9[0], r6
+; CHECK-NEON-NEXT:    mov r5, r0
+; CHECK-NEON-NEXT:    vmov r0, s16
+; CHECK-NEON-NEXT:    mov r6, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
+; CHECK-NEON-NEXT:    vmov r8, s20
+; CHECK-NEON-NEXT:    vmov.32 d13[0], r5
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    vmov.32 d8[0], r0
+; CHECK-NEON-NEXT:    vmov r2, s18
+; CHECK-NEON-NEXT:    vmov.32 d12[0], r0
 ; CHECK-NEON-NEXT:    mvn r9, #0
 ; CHECK-NEON-NEXT:    subs r0, r0, r9
-; CHECK-NEON-NEXT:    mov r4, #0
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
-; CHECK-NEON-NEXT:    vmov.32 d9[1], r7
+; CHECK-NEON-NEXT:    vmov.32 d13[1], r6
 ; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    mov r7, #0
 ; CHECK-NEON-NEXT:    movwlt r0, #1
 ; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    vmov.32 d8[1], r1
+; CHECK-NEON-NEXT:    vmov.32 d12[1], r1
 ; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    subs r1, r6, r9
-; CHECK-NEON-NEXT:    sbcs r1, r7, #0
+; CHECK-NEON-NEXT:    subs r1, r5, r9
+; CHECK-NEON-NEXT:    sbcs r1, r6, #0
 ; CHECK-NEON-NEXT:    mov r1, #0
 ; CHECK-NEON-NEXT:    movwlt r1, #1
 ; CHECK-NEON-NEXT:    cmp r1, #0
 ; CHECK-NEON-NEXT:    mvnne r1, #0
-; CHECK-NEON-NEXT:    vdup.32 d13, r1
-; CHECK-NEON-NEXT:    vdup.32 d12, r0
-; CHECK-NEON-NEXT:    mov r0, r5
+; CHECK-NEON-NEXT:    vdup.32 d9, r1
+; CHECK-NEON-NEXT:    vdup.32 d8, r0
+; CHECK-NEON-NEXT:    mov r0, r2
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov.i64 q5, #0xffffffff
-; CHECK-NEON-NEXT:    vbif q4, q5, q6
+; CHECK-NEON-NEXT:    vbsl q4, q6, q5
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEON-NEXT:    mov r5, r0
-; CHECK-NEON-NEXT:    vmov.32 d12[0], r0
+; CHECK-NEON-NEXT:    vmov.32 d13[0], r0
 ; CHECK-NEON-NEXT:    mov r0, r8
 ; CHECK-NEON-NEXT:    mov r6, r1
-; CHECK-NEON-NEXT:    vmov r7, r10, d8
+; CHECK-NEON-NEXT:    vmov r4, r10, d8
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEON-NEXT:    subs r2, r5, r9
-; CHECK-NEON-NEXT:    vmov.32 d13[0], r0
+; CHECK-NEON-NEXT:    vmov.32 d12[0], r0
 ; CHECK-NEON-NEXT:    sbcs r2, r6, #0
 ; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    vmov.32 d13[1], r6
 ; CHECK-NEON-NEXT:    movwlt r2, #1
+; CHECK-NEON-NEXT:    cmp r2, #0
+; CHECK-NEON-NEXT:    mvnne r2, #0
 ; CHECK-NEON-NEXT:    subs r0, r0, r9
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
-; CHECK-NEON-NEXT:    vmov.32 d13[1], r1
+; CHECK-NEON-NEXT:    vdup.32 d17, r2
 ; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    vmov.32 d12[1], r1
 ; CHECK-NEON-NEXT:    movwlt r0, #1
 ; CHECK-NEON-NEXT:    cmp r0, #0
 ; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    cmp r2, #0
-; CHECK-NEON-NEXT:    vmov.32 d12[1], r6
-; CHECK-NEON-NEXT:    mvnne r2, #0
-; CHECK-NEON-NEXT:    vdup.32 d17, r0
-; CHECK-NEON-NEXT:    rsbs r0, r7, #0
-; CHECK-NEON-NEXT:    vdup.32 d16, r2
-; CHECK-NEON-NEXT:    vmov r7, r5, d9
+; CHECK-NEON-NEXT:    vmov r2, r3, d9
+; CHECK-NEON-NEXT:    vdup.32 d16, r0
+; CHECK-NEON-NEXT:    rsbs r6, r4, #0
 ; CHECK-NEON-NEXT:    vbsl q8, q6, q5
-; CHECK-NEON-NEXT:    rscs r0, r10, #0
+; CHECK-NEON-NEXT:    rscs r6, r10, #0
+; CHECK-NEON-NEXT:    mov r6, #0
+; CHECK-NEON-NEXT:    movwlt r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    vmov r0, r1, d17
+; CHECK-NEON-NEXT:    mvnne r6, #0
+; CHECK-NEON-NEXT:    vmov r5, r4, d16
+; CHECK-NEON-NEXT:    rsbs r0, r0, #0
+; CHECK-NEON-NEXT:    rscs r0, r1, #0
 ; CHECK-NEON-NEXT:    mov r0, #0
 ; CHECK-NEON-NEXT:    movwlt r0, #1
-; CHECK-NEON-NEXT:    vmov r1, r2, d16
-; CHECK-NEON-NEXT:    vmov r3, r6, d17
-; CHECK-NEON-NEXT:    rsbs r1, r1, #0
-; CHECK-NEON-NEXT:    rscs r1, r2, #0
+; CHECK-NEON-NEXT:    cmp r0, #0
+; CHECK-NEON-NEXT:    mvnne r0, #0
+; CHECK-NEON-NEXT:    rsbs r1, r2, #0
+; CHECK-NEON-NEXT:    rscs r1, r3, #0
+; CHECK-NEON-NEXT:    vmov.32 d19[0], r0
 ; CHECK-NEON-NEXT:    mov r1, #0
 ; CHECK-NEON-NEXT:    movwlt r1, #1
-; CHECK-NEON-NEXT:    rsbs r2, r3, #0
-; CHECK-NEON-NEXT:    rscs r2, r6, #0
-; CHECK-NEON-NEXT:    mov r2, #0
-; CHECK-NEON-NEXT:    movwlt r2, #1
-; CHECK-NEON-NEXT:    rsbs r3, r7, #0
-; CHECK-NEON-NEXT:    rscs r3, r5, #0
-; CHECK-NEON-NEXT:    movwlt r4, #1
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    mvnne r4, #0
-; CHECK-NEON-NEXT:    cmp r2, #0
-; CHECK-NEON-NEXT:    mvnne r2, #0
 ; CHECK-NEON-NEXT:    cmp r1, #0
 ; CHECK-NEON-NEXT:    mvnne r1, #0
-; CHECK-NEON-NEXT:    vmov.32 d21[0], r2
-; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    vmov.32 d20[0], r1
-; CHECK-NEON-NEXT:    mvnne r0, #0
-; CHECK-NEON-NEXT:    vmov.32 d19[0], r4
-; CHECK-NEON-NEXT:    vand q8, q8, q10
-; CHECK-NEON-NEXT:    vmov.32 d18[0], r0
-; CHECK-NEON-NEXT:    vmovn.i64 d1, q8
-; CHECK-NEON-NEXT:    vand q9, q4, q9
-; CHECK-NEON-NEXT:    vmovn.i64 d0, q9
+; CHECK-NEON-NEXT:    rsbs r0, r5, #0
+; CHECK-NEON-NEXT:    rscs r0, r4, #0
+; CHECK-NEON-NEXT:    vmov.32 d21[0], r1
+; CHECK-NEON-NEXT:    movwlt r7, #1
+; CHECK-NEON-NEXT:    cmp r7, #0
+; CHECK-NEON-NEXT:    vmov.32 d20[0], r6
+; CHECK-NEON-NEXT:    mvnne r7, #0
+; CHECK-NEON-NEXT:    vmov.32 d18[0], r7
+; CHECK-NEON-NEXT:    vand q10, q4, q10
+; CHECK-NEON-NEXT:    vand q8, q8, q9
+; CHECK-NEON-NEXT:    vmovn.i64 d1, q10
+; CHECK-NEON-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 ;
 ; CHECK-FP16-LABEL: ustest_f16i32:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-FP16-NEXT:    .vsave {d10, d11, d12, d13, d14, d15}
-; CHECK-FP16-NEXT:    vpush {d10, d11, d12, d13, d14, d15}
-; CHECK-FP16-NEXT:    .vsave {d8}
-; CHECK-FP16-NEXT:    vpush {d8}
-; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-FP16-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    vmov.u16 r0, d0[3]
 ; CHECK-FP16-NEXT:    vorr d8, d0, d0
-; CHECK-FP16-NEXT:    vmov.u16 r8, d0[2]
-; CHECK-FP16-NEXT:    vmov.u16 r9, d0[3]
+; CHECK-FP16-NEXT:    vmov.u16 r8, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r9, d0[1]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
 ; CHECK-FP16-NEXT:    mov r4, r0
-; CHECK-FP16-NEXT:    vmov.u16 r0, d8[0]
+; CHECK-FP16-NEXT:    vmov.u16 r0, d8[2]
 ; CHECK-FP16-NEXT:    mov r5, r1
-; CHECK-FP16-NEXT:    vmov.32 d11[0], r4
+; CHECK-FP16-NEXT:    vmov.32 d9[0], r4
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    vmov.32 d10[0], r0
-; CHECK-FP16-NEXT:    mvn r7, #0
-; CHECK-FP16-NEXT:    subs r0, r0, r7
-; CHECK-FP16-NEXT:    vmov.i64 q6, #0xffffffff
+; CHECK-FP16-NEXT:    mvn r10, #0
+; CHECK-FP16-NEXT:    subs r2, r4, r10
+; CHECK-FP16-NEXT:    sbcs r2, r5, #0
+; CHECK-FP16-NEXT:    vmov.32 d8[0], r0
+; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    vmov s0, r9
+; CHECK-FP16-NEXT:    movwlt r2, #1
+; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    mvnne r2, #0
+; CHECK-FP16-NEXT:    subs r0, r0, r10
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    vmov.32 d11[1], r5
+; CHECK-FP16-NEXT:    vmov.32 d9[1], r5
 ; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    vmov s0, r8
+; CHECK-FP16-NEXT:    vmov.i64 q5, #0xffffffff
 ; CHECK-FP16-NEXT:    movwlt r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    vmov.32 d10[1], r1
+; CHECK-FP16-NEXT:    vmov.32 d8[1], r1
 ; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    subs r1, r4, r7
 ; CHECK-FP16-NEXT:    mov r6, #0
-; CHECK-FP16-NEXT:    sbcs r1, r5, #0
-; CHECK-FP16-NEXT:    vmov s16, r9
-; CHECK-FP16-NEXT:    mov r1, #0
-; CHECK-FP16-NEXT:    movwlt r1, #1
-; CHECK-FP16-NEXT:    cmp r1, #0
-; CHECK-FP16-NEXT:    mvnne r1, #0
-; CHECK-FP16-NEXT:    vdup.32 d17, r1
+; CHECK-FP16-NEXT:    vdup.32 d17, r2
 ; CHECK-FP16-NEXT:    vdup.32 d16, r0
-; CHECK-FP16-NEXT:    vbif q5, q6, q8
-; CHECK-FP16-NEXT:    vmov r9, r8, d10
+; CHECK-FP16-NEXT:    vbif q4, q5, q8
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    vmov.f32 s0, s16
+; CHECK-FP16-NEXT:    vmov s0, r8
 ; CHECK-FP16-NEXT:    mov r4, r0
 ; CHECK-FP16-NEXT:    mov r5, r1
-; CHECK-FP16-NEXT:    vmov.32 d14[0], r0
+; CHECK-FP16-NEXT:    vmov.32 d13[0], r0
+; CHECK-FP16-NEXT:    vmov r7, r8, d8
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    subs r2, r4, r7
-; CHECK-FP16-NEXT:    vmov.32 d15[0], r0
+; CHECK-FP16-NEXT:    subs r2, r4, r10
+; CHECK-FP16-NEXT:    vmov.32 d12[0], r0
 ; CHECK-FP16-NEXT:    sbcs r2, r5, #0
 ; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    vmov.32 d13[1], r5
 ; CHECK-FP16-NEXT:    movwlt r2, #1
-; CHECK-FP16-NEXT:    subs r0, r0, r7
+; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    mvnne r2, #0
+; CHECK-FP16-NEXT:    subs r0, r0, r10
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    vmov.32 d15[1], r1
+; CHECK-FP16-NEXT:    vdup.32 d17, r2
 ; CHECK-FP16-NEXT:    mov r0, #0
+; CHECK-FP16-NEXT:    vmov.32 d12[1], r1
 ; CHECK-FP16-NEXT:    movwlt r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
 ; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    cmp r2, #0
-; CHECK-FP16-NEXT:    vmov.32 d14[1], r5
-; CHECK-FP16-NEXT:    mvnne r2, #0
-; CHECK-FP16-NEXT:    vmov r5, r4, d11
-; CHECK-FP16-NEXT:    vdup.32 d17, r0
-; CHECK-FP16-NEXT:    rsbs r0, r9, #0
-; CHECK-FP16-NEXT:    vdup.32 d16, r2
-; CHECK-FP16-NEXT:    rscs r0, r8, #0
-; CHECK-FP16-NEXT:    vbsl q8, q7, q6
+; CHECK-FP16-NEXT:    vmov r2, r3, d9
+; CHECK-FP16-NEXT:    vdup.32 d16, r0
+; CHECK-FP16-NEXT:    rsbs r7, r7, #0
+; CHECK-FP16-NEXT:    vbsl q8, q6, q5
+; CHECK-FP16-NEXT:    rscs r7, r8, #0
+; CHECK-FP16-NEXT:    mov r7, #0
+; CHECK-FP16-NEXT:    movwlt r7, #1
+; CHECK-FP16-NEXT:    cmp r7, #0
+; CHECK-FP16-NEXT:    vmov r0, r1, d17
+; CHECK-FP16-NEXT:    mvnne r7, #0
+; CHECK-FP16-NEXT:    vmov r5, r4, d16
+; CHECK-FP16-NEXT:    rsbs r0, r0, #0
+; CHECK-FP16-NEXT:    rscs r0, r1, #0
 ; CHECK-FP16-NEXT:    mov r0, #0
 ; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    vmov r1, r2, d16
-; CHECK-FP16-NEXT:    vmov r3, r7, d17
-; CHECK-FP16-NEXT:    rsbs r1, r1, #0
-; CHECK-FP16-NEXT:    rscs r1, r2, #0
+; CHECK-FP16-NEXT:    cmp r0, #0
+; CHECK-FP16-NEXT:    mvnne r0, #0
+; CHECK-FP16-NEXT:    rsbs r1, r2, #0
+; CHECK-FP16-NEXT:    rscs r1, r3, #0
+; CHECK-FP16-NEXT:    vmov.32 d19[0], r0
 ; CHECK-FP16-NEXT:    mov r1, #0
 ; CHECK-FP16-NEXT:    movwlt r1, #1
-; CHECK-FP16-NEXT:    rsbs r2, r3, #0
-; CHECK-FP16-NEXT:    rscs r2, r7, #0
-; CHECK-FP16-NEXT:    mov r2, #0
-; CHECK-FP16-NEXT:    movwlt r2, #1
-; CHECK-FP16-NEXT:    rsbs r3, r5, #0
-; CHECK-FP16-NEXT:    rscs r3, r4, #0
+; CHECK-FP16-NEXT:    cmp r1, #0
+; CHECK-FP16-NEXT:    mvnne r1, #0
+; CHECK-FP16-NEXT:    rsbs r0, r5, #0
+; CHECK-FP16-NEXT:    rscs r0, r4, #0
+; CHECK-FP16-NEXT:    vmov.32 d21[0], r1
 ; CHECK-FP16-NEXT:    movwlt r6, #1
 ; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    vmov.32 d20[0], r7
 ; CHECK-FP16-NEXT:    mvnne r6, #0
-; CHECK-FP16-NEXT:    cmp r2, #0
-; CHECK-FP16-NEXT:    mvnne r2, #0
-; CHECK-FP16-NEXT:    cmp r1, #0
-; CHECK-FP16-NEXT:    mvnne r1, #0
-; CHECK-FP16-NEXT:    vmov.32 d21[0], r2
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    vmov.32 d20[0], r1
-; CHECK-FP16-NEXT:    mvnne r0, #0
-; CHECK-FP16-NEXT:    vmov.32 d19[0], r6
-; CHECK-FP16-NEXT:    vand q8, q8, q10
-; CHECK-FP16-NEXT:    vmov.32 d18[0], r0
-; CHECK-FP16-NEXT:    vmovn.i64 d1, q8
-; CHECK-FP16-NEXT:    vand q9, q5, q9
-; CHECK-FP16-NEXT:    vmovn.i64 d0, q9
-; CHECK-FP16-NEXT:    vpop {d8}
-; CHECK-FP16-NEXT:    vpop {d10, d11, d12, d13, d14, d15}
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; CHECK-FP16-NEXT:    vmov.32 d18[0], r6
+; CHECK-FP16-NEXT:    vand q10, q4, q10
+; CHECK-FP16-NEXT:    vand q8, q8, q9
+; CHECK-FP16-NEXT:    vmovn.i64 d1, q10
+; CHECK-FP16-NEXT:    vmovn.i64 d0, q8
+; CHECK-FP16-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %0 = icmp slt <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -1643,59 +1624,56 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mvn r8, #0
-; CHECK-NEXT:    subs r0, r0, r8
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    sbcs r0, r1, r6
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    sbcs r0, r2, #0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    mvn r9, #0
+; CHECK-NEXT:    subs r1, r0, r9
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r4, r5
 ; CHECK-NEXT:    vorr d0, d8, d8
-; CHECK-NEXT:    sbcs r0, r3, #0
+; CHECK-NEXT:    sbcs r1, r2, #0
 ; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r9, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    moveq r3, r0
-; CHECK-NEXT:    movne r0, r2
-; CHECK-NEXT:    moveq r10, r6
-; CHECK-NEXT:    moveq r5, r8
-; CHECK-NEXT:    rsbs r1, r5, #0
-; CHECK-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-NEXT:    sbcs r0, r8, r0
-; CHECK-NEXT:    sbcs r0, r8, r3
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r8, #-2147483648
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    mov r10, #0
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    movne r1, r2
+; CHECK-NEXT:    moveq r4, r5
+; CHECK-NEXT:    moveq r0, r9
+; CHECK-NEXT:    rsbs r2, r0, #0
+; CHECK-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r1
+; CHECK-NEXT:    sbcs r1, r9, r3
 ; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    movne r7, r0
+; CHECK-NEXT:    moveq r4, r8
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    subs r4, r0, r8
-; CHECK-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEXT:    sbcs r4, r1, r6
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    movne r4, r2
-; CHECK-NEXT:    moveq r0, r8
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-NEXT:    sbcs r1, r8, r4
-; CHECK-NEXT:    sbcs r1, r8, r3
-; CHECK-NEXT:    movwlt r9, #1
-; CHECK-NEXT:    cmp r9, #0
+; CHECK-NEXT:    subs r6, r0, r9
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r1, r5
+; CHECK-NEXT:    sbcs r6, r2, #0
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r3, r6
+; CHECK-NEXT:    movne r6, r2
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    moveq r0, r9
-; CHECK-NEXT:    mov r1, #-2147483648
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    moveq r10, r1
-; CHECK-NEXT:    cmp r9, #0
-; CHECK-NEXT:    vmov.32 d1[1], r10
-; CHECK-NEXT:    moveq r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    rsbs r1, r0, #0
+; CHECK-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r6
+; CHECK-NEXT:    sbcs r1, r9, r3
+; CHECK-NEXT:    movwlt r10, #1
+; CHECK-NEXT:    cmp r10, #0
+; CHECK-NEXT:    movne r10, r0
+; CHECK-NEXT:    moveq r5, r8
+; CHECK-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
@@ -1711,39 +1689,36 @@ entry:
 define <2 x i64> @utest_f64i64(<2 x double> %x) {
 ; CHECK-LABEL: utest_f64i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    vorr d0, d8, d8
-; CHECK-NEXT:    sbcs r0, r3, #0
-; CHECK-NEXT:    mov r7, #0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    movwlo r7, #1
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    subs r1, r2, #1
+; CHECK-NEXT:    vorr d0, d8, d8
+; CHECK-NEXT:    sbcs r1, r3, #0
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    movwlo r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r4, r6
+; CHECK-NEXT:    movne r6, r0
 ; CHECK-NEXT:    bl __fixunsdfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r5
+; CHECK-NEXT:    vmov.32 d1[0], r6
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlo r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    moveq r0, r6
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r4
+; CHECK-NEXT:    movwlo r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    moveq r0, r5
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -1755,26 +1730,26 @@ entry:
 define <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-LABEL: ustest_f64i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r8, r1
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    subs r1, r2, #1
 ; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    movwlt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    mov r9, #1
-; CHECK-NEXT:    moveq r3, r6
-; CHECK-NEXT:    moveq r8, r6
-; CHECK-NEXT:    moveq r2, r9
-; CHECK-NEXT:    movne r6, r0
-; CHECK-NEXT:    rsbs r0, r6, #0
-; CHECK-NEXT:    rscs r0, r8, #0
+; CHECK-NEXT:    mov r8, #1
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    movge r2, r8
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    moveq r4, r1
+; CHECK-NEXT:    movne r1, r0
+; CHECK-NEXT:    rsbs r0, r1, #0
+; CHECK-NEXT:    rscs r0, r4, #0
 ; CHECK-NEXT:    vorr d0, d8, d8
 ; CHECK-NEXT:    rscs r0, r2, #0
 ; CHECK-NEXT:    mov r7, #0
@@ -1782,34 +1757,32 @@ define <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    mov r5, #0
 ; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r6, r7
+; CHECK-NEXT:    moveq r4, r7
+; CHECK-NEXT:    movne r7, r1
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    subs r4, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r6
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    movne r9, r2
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    moveq r1, r4
-; CHECK-NEXT:    movne r4, r0
-; CHECK-NEXT:    rsbs r0, r4, #0
+; CHECK-NEXT:    subs r6, r2, #1
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    movlt r8, r2
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movwlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    moveq r3, r2
+; CHECK-NEXT:    moveq r1, r2
+; CHECK-NEXT:    movne r2, r0
+; CHECK-NEXT:    rsbs r0, r2, #0
 ; CHECK-NEXT:    rscs r0, r1, #0
-; CHECK-NEXT:    rscs r0, r9, #0
+; CHECK-NEXT:    rscs r0, r8, #0
 ; CHECK-NEXT:    rscs r0, r3, #0
 ; CHECK-NEXT:    movwlt r5, #1
 ; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    moveq r4, r5
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r8
-; CHECK-NEXT:    vmov.32 d0[0], r4
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
+; CHECK-NEXT:    moveq r2, r5
 ; CHECK-NEXT:    movne r5, r1
+; CHECK-NEXT:    vmov.32 d0[0], r2
+; CHECK-NEXT:    vmov.32 d1[1], r4
 ; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -1830,59 +1803,56 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NEXT:    vmov.f64 d8, d0
 ; CHECK-NEXT:    vmov.f32 s0, s17
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mvn r8, #0
-; CHECK-NEXT:    subs r0, r0, r8
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    sbcs r0, r1, r6
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    mvn r9, #0
+; CHECK-NEXT:    subs r1, r0, r9
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r4, r5
 ; CHECK-NEXT:    vmov.f32 s0, s16
-; CHECK-NEXT:    sbcs r0, r2, #0
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    sbcs r0, r3, #0
+; CHECK-NEXT:    sbcs r1, r2, #0
 ; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r9, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    moveq r3, r0
-; CHECK-NEXT:    movne r0, r2
-; CHECK-NEXT:    moveq r10, r6
-; CHECK-NEXT:    moveq r5, r8
-; CHECK-NEXT:    rsbs r1, r5, #0
-; CHECK-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-NEXT:    sbcs r0, r8, r0
-; CHECK-NEXT:    sbcs r0, r8, r3
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r8, #-2147483648
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    mov r10, #0
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    movne r1, r2
+; CHECK-NEXT:    moveq r4, r5
+; CHECK-NEXT:    moveq r0, r9
+; CHECK-NEXT:    rsbs r2, r0, #0
+; CHECK-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r1
+; CHECK-NEXT:    sbcs r1, r9, r3
 ; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    movne r7, r0
+; CHECK-NEXT:    moveq r4, r8
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs r4, r0, r8
-; CHECK-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEXT:    sbcs r4, r1, r6
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    movne r4, r2
-; CHECK-NEXT:    moveq r0, r8
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-NEXT:    sbcs r1, r8, r4
-; CHECK-NEXT:    sbcs r1, r8, r3
-; CHECK-NEXT:    movwlt r9, #1
-; CHECK-NEXT:    cmp r9, #0
+; CHECK-NEXT:    subs r6, r0, r9
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r1, r5
+; CHECK-NEXT:    sbcs r6, r2, #0
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r3, r6
+; CHECK-NEXT:    movne r6, r2
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    moveq r0, r9
-; CHECK-NEXT:    mov r1, #-2147483648
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    moveq r10, r1
-; CHECK-NEXT:    cmp r9, #0
-; CHECK-NEXT:    vmov.32 d1[1], r10
-; CHECK-NEXT:    moveq r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    rsbs r1, r0, #0
+; CHECK-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r6
+; CHECK-NEXT:    sbcs r1, r9, r3
+; CHECK-NEXT:    movwlt r10, #1
+; CHECK-NEXT:    cmp r10, #0
+; CHECK-NEXT:    movne r10, r0
+; CHECK-NEXT:    moveq r5, r8
+; CHECK-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
@@ -1898,39 +1868,36 @@ entry:
 define <2 x i64> @utest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: utest_f32i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8}
 ; CHECK-NEXT:    vpush {d8}
 ; CHECK-NEXT:    vmov.f64 d8, d0
 ; CHECK-NEXT:    vmov.f32 s0, s17
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vmov.f32 s0, s16
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    sbcs r0, r3, #0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    movwlo r7, #1
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    subs r1, r2, #1
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    movwlo r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r4, r6
+; CHECK-NEXT:    movne r6, r0
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r5
+; CHECK-NEXT:    vmov.32 d1[0], r6
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlo r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    moveq r0, r6
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r4
+; CHECK-NEXT:    movwlo r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    moveq r0, r5
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -1942,61 +1909,59 @@ entry:
 define <2 x i64> @ustest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: ustest_f32i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8}
 ; CHECK-NEXT:    vpush {d8}
 ; CHECK-NEXT:    vmov.f64 d8, d0
 ; CHECK-NEXT:    vmov.f32 s0, s17
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r8, r1
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    vmov.f32 s0, s16
 ; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    mov r9, #1
-; CHECK-NEXT:    movwlt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    moveq r3, r6
-; CHECK-NEXT:    moveq r8, r6
-; CHECK-NEXT:    moveq r2, r9
-; CHECK-NEXT:    movne r6, r0
-; CHECK-NEXT:    rsbs r0, r6, #0
+; CHECK-NEXT:    mov r8, #1
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    vmov.f32 s0, s16
+; CHECK-NEXT:    movge r2, r8
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    rscs r0, r8, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    moveq r4, r1
+; CHECK-NEXT:    movne r1, r0
+; CHECK-NEXT:    rsbs r0, r1, #0
+; CHECK-NEXT:    rscs r0, r4, #0
 ; CHECK-NEXT:    mov r5, #0
 ; CHECK-NEXT:    rscs r0, r2, #0
 ; CHECK-NEXT:    rscs r0, r3, #0
 ; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r6, r7
+; CHECK-NEXT:    moveq r4, r7
+; CHECK-NEXT:    movne r7, r1
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs r4, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r6
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    movne r9, r2
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    moveq r1, r4
-; CHECK-NEXT:    movne r4, r0
-; CHECK-NEXT:    rsbs r0, r4, #0
+; CHECK-NEXT:    subs r6, r2, #1
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    movlt r8, r2
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movwlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    moveq r3, r2
+; CHECK-NEXT:    moveq r1, r2
+; CHECK-NEXT:    movne r2, r0
+; CHECK-NEXT:    rsbs r0, r2, #0
 ; CHECK-NEXT:    rscs r0, r1, #0
-; CHECK-NEXT:    rscs r0, r9, #0
+; CHECK-NEXT:    rscs r0, r8, #0
 ; CHECK-NEXT:    rscs r0, r3, #0
 ; CHECK-NEXT:    movwlt r5, #1
 ; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    moveq r4, r5
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r8
-; CHECK-NEXT:    vmov.32 d0[0], r4
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
+; CHECK-NEXT:    moveq r2, r5
 ; CHECK-NEXT:    movne r5, r1
+; CHECK-NEXT:    vmov.32 d0[0], r2
+; CHECK-NEXT:    vmov.32 d1[1], r4
 ; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -2010,11 +1975,9 @@ entry:
 define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NEON-LABEL: stest_f16i64:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEON-NEXT:    .pad #4
-; CHECK-NEON-NEXT:    sub sp, sp, #4
-; CHECK-NEON-NEXT:    .vsave {d8}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
 ; CHECK-NEON-NEXT:    vmov r0, s0
 ; CHECK-NEON-NEXT:    vmov.f32 s16, s1
@@ -2024,62 +1987,58 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    mov r5, r0
+; CHECK-NEON-NEXT:    mov r4, r1
 ; CHECK-NEON-NEXT:    mvn r9, #0
-; CHECK-NEON-NEXT:    subs r0, r0, r9
-; CHECK-NEON-NEXT:    mvn r7, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r1, r7
-; CHECK-NEON-NEXT:    mov r11, r1
-; CHECK-NEON-NEXT:    sbcs r0, r2, #0
+; CHECK-NEON-NEXT:    subs r1, r0, r9
+; CHECK-NEON-NEXT:    mvn r6, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r4, r6
 ; CHECK-NEON-NEXT:    vmov s0, r8
-; CHECK-NEON-NEXT:    sbcs r0, r3, #0
-; CHECK-NEON-NEXT:    mov r6, #0
-; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    sbcs r1, r2, #0
+; CHECK-NEON-NEXT:    mov r5, #0
+; CHECK-NEON-NEXT:    sbcs r1, r3, #0
+; CHECK-NEON-NEXT:    mov r8, #-2147483648
+; CHECK-NEON-NEXT:    mov r1, #0
 ; CHECK-NEON-NEXT:    mov r10, #0
-; CHECK-NEON-NEXT:    movwlt r0, #1
-; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    moveq r3, r0
-; CHECK-NEON-NEXT:    movne r0, r2
-; CHECK-NEON-NEXT:    moveq r11, r7
-; CHECK-NEON-NEXT:    moveq r5, r9
-; CHECK-NEON-NEXT:    rsbs r1, r5, #0
-; CHECK-NEON-NEXT:    rscs r1, r11, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r9, r0
-; CHECK-NEON-NEXT:    sbcs r0, r9, r3
-; CHECK-NEON-NEXT:    movwlt r6, #1
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    moveq r5, r6
+; CHECK-NEON-NEXT:    movwlt r1, #1
+; CHECK-NEON-NEXT:    cmp r1, #0
+; CHECK-NEON-NEXT:    moveq r3, r1
+; CHECK-NEON-NEXT:    movne r1, r2
+; CHECK-NEON-NEXT:    moveq r4, r6
+; CHECK-NEON-NEXT:    moveq r0, r9
+; CHECK-NEON-NEXT:    rsbs r2, r0, #0
+; CHECK-NEON-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r9, r1
+; CHECK-NEON-NEXT:    sbcs r1, r9, r3
+; CHECK-NEON-NEXT:    movwlt r5, #1
+; CHECK-NEON-NEXT:    cmp r5, #0
+; CHECK-NEON-NEXT:    movne r5, r0
+; CHECK-NEON-NEXT:    moveq r4, r8
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    subs r4, r0, r9
+; CHECK-NEON-NEXT:    subs r7, r0, r9
 ; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEON-NEXT:    sbcs r4, r1, r7
-; CHECK-NEON-NEXT:    sbcs r4, r2, #0
-; CHECK-NEON-NEXT:    sbcs r4, r3, #0
-; CHECK-NEON-NEXT:    mov r4, #0
-; CHECK-NEON-NEXT:    movwlt r4, #1
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    moveq r3, r4
-; CHECK-NEON-NEXT:    movne r7, r1
-; CHECK-NEON-NEXT:    movne r4, r2
+; CHECK-NEON-NEXT:    sbcs r7, r1, r6
+; CHECK-NEON-NEXT:    sbcs r7, r2, #0
+; CHECK-NEON-NEXT:    sbcs r7, r3, #0
+; CHECK-NEON-NEXT:    mov r7, #0
+; CHECK-NEON-NEXT:    movwlt r7, #1
+; CHECK-NEON-NEXT:    cmp r7, #0
+; CHECK-NEON-NEXT:    moveq r3, r7
+; CHECK-NEON-NEXT:    movne r7, r2
+; CHECK-NEON-NEXT:    movne r6, r1
 ; CHECK-NEON-NEXT:    moveq r0, r9
 ; CHECK-NEON-NEXT:    rsbs r1, r0, #0
-; CHECK-NEON-NEXT:    rscs r1, r7, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r1, r9, r4
+; CHECK-NEON-NEXT:    rscs r1, r6, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r9, r7
 ; CHECK-NEON-NEXT:    sbcs r1, r9, r3
 ; CHECK-NEON-NEXT:    movwlt r10, #1
 ; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    moveq r0, r10
-; CHECK-NEON-NEXT:    mov r1, #-2147483648
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEON-NEXT:    moveq r11, r1
-; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r11
-; CHECK-NEON-NEXT:    moveq r7, r1
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    movne r10, r0
+; CHECK-NEON-NEXT:    moveq r6, r8
+; CHECK-NEON-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r6
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    add sp, sp, #4
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 ;
 ; CHECK-FP16-LABEL: stest_f16i64:
 ; CHECK-FP16:       @ %bb.0: @ %entry
@@ -2089,59 +2048,56 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-FP16-NEXT:    vmov.u16 r7, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    mov r5, r0
-; CHECK-FP16-NEXT:    mvn r8, #0
-; CHECK-FP16-NEXT:    subs r0, r0, r8
-; CHECK-FP16-NEXT:    mvn r6, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r1, r6
-; CHECK-FP16-NEXT:    mov r10, r1
-; CHECK-FP16-NEXT:    sbcs r0, r2, #0
+; CHECK-FP16-NEXT:    mov r4, r1
+; CHECK-FP16-NEXT:    mvn r9, #0
+; CHECK-FP16-NEXT:    subs r1, r0, r9
+; CHECK-FP16-NEXT:    mvn r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r4, r5
 ; CHECK-FP16-NEXT:    vmov s0, r7
-; CHECK-FP16-NEXT:    sbcs r0, r3, #0
+; CHECK-FP16-NEXT:    sbcs r1, r2, #0
 ; CHECK-FP16-NEXT:    mov r7, #0
-; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    mov r9, #0
-; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    moveq r3, r0
-; CHECK-FP16-NEXT:    movne r0, r2
-; CHECK-FP16-NEXT:    moveq r10, r6
-; CHECK-FP16-NEXT:    moveq r5, r8
-; CHECK-FP16-NEXT:    rsbs r1, r5, #0
-; CHECK-FP16-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r8, r0
-; CHECK-FP16-NEXT:    sbcs r0, r8, r3
+; CHECK-FP16-NEXT:    sbcs r1, r3, #0
+; CHECK-FP16-NEXT:    mov r8, #-2147483648
+; CHECK-FP16-NEXT:    mov r1, #0
+; CHECK-FP16-NEXT:    mov r10, #0
+; CHECK-FP16-NEXT:    movwlt r1, #1
+; CHECK-FP16-NEXT:    cmp r1, #0
+; CHECK-FP16-NEXT:    moveq r3, r1
+; CHECK-FP16-NEXT:    movne r1, r2
+; CHECK-FP16-NEXT:    moveq r4, r5
+; CHECK-FP16-NEXT:    moveq r0, r9
+; CHECK-FP16-NEXT:    rsbs r2, r0, #0
+; CHECK-FP16-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r9, r1
+; CHECK-FP16-NEXT:    sbcs r1, r9, r3
 ; CHECK-FP16-NEXT:    movwlt r7, #1
 ; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    moveq r5, r7
+; CHECK-FP16-NEXT:    movne r7, r0
+; CHECK-FP16-NEXT:    moveq r4, r8
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    subs r4, r0, r8
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
-; CHECK-FP16-NEXT:    sbcs r4, r1, r6
-; CHECK-FP16-NEXT:    sbcs r4, r2, #0
-; CHECK-FP16-NEXT:    sbcs r4, r3, #0
-; CHECK-FP16-NEXT:    mov r4, #0
-; CHECK-FP16-NEXT:    movwlt r4, #1
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    moveq r3, r4
-; CHECK-FP16-NEXT:    movne r6, r1
-; CHECK-FP16-NEXT:    movne r4, r2
-; CHECK-FP16-NEXT:    moveq r0, r8
-; CHECK-FP16-NEXT:    rsbs r1, r0, #0
-; CHECK-FP16-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r1, r8, r4
-; CHECK-FP16-NEXT:    sbcs r1, r8, r3
-; CHECK-FP16-NEXT:    movwlt r9, #1
-; CHECK-FP16-NEXT:    cmp r9, #0
+; CHECK-FP16-NEXT:    subs r6, r0, r9
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r7
+; CHECK-FP16-NEXT:    sbcs r6, r1, r5
+; CHECK-FP16-NEXT:    sbcs r6, r2, #0
+; CHECK-FP16-NEXT:    sbcs r6, r3, #0
+; CHECK-FP16-NEXT:    mov r6, #0
+; CHECK-FP16-NEXT:    movwlt r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    moveq r3, r6
+; CHECK-FP16-NEXT:    movne r6, r2
+; CHECK-FP16-NEXT:    movne r5, r1
 ; CHECK-FP16-NEXT:    moveq r0, r9
-; CHECK-FP16-NEXT:    mov r1, #-2147483648
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    vmov.32 d0[0], r0
-; CHECK-FP16-NEXT:    moveq r10, r1
-; CHECK-FP16-NEXT:    cmp r9, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r10
-; CHECK-FP16-NEXT:    moveq r6, r1
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r6
+; CHECK-FP16-NEXT:    rsbs r1, r0, #0
+; CHECK-FP16-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r9, r6
+; CHECK-FP16-NEXT:    sbcs r1, r9, r3
+; CHECK-FP16-NEXT:    movwlt r10, #1
+; CHECK-FP16-NEXT:    cmp r10, #0
+; CHECK-FP16-NEXT:    movne r10, r0
+; CHECK-FP16-NEXT:    moveq r5, r8
+; CHECK-FP16-NEXT:    vmov.32 d0[0], r10
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
 ; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
@@ -2156,8 +2112,8 @@ entry:
 define <2 x i64> @utesth_f16i64(<2 x half> %x) {
 ; CHECK-NEON-LABEL: utesth_f16i64:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
 ; CHECK-NEON-NEXT:    vmov r0, s0
@@ -2168,66 +2124,60 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixunssfti
-; CHECK-NEON-NEXT:    mov r6, r0
-; CHECK-NEON-NEXT:    subs r0, r2, #1
+; CHECK-NEON-NEXT:    mov r4, r1
+; CHECK-NEON-NEXT:    subs r1, r2, #1
 ; CHECK-NEON-NEXT:    vmov s0, r5
-; CHECK-NEON-NEXT:    sbcs r0, r3, #0
+; CHECK-NEON-NEXT:    sbcs r1, r3, #0
 ; CHECK-NEON-NEXT:    mov r5, #0
-; CHECK-NEON-NEXT:    mov r4, r1
+; CHECK-NEON-NEXT:    mov r6, #0
 ; CHECK-NEON-NEXT:    movwlo r5, #1
 ; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    mov r7, #0
-; CHECK-NEON-NEXT:    moveq r6, r5
+; CHECK-NEON-NEXT:    moveq r4, r5
+; CHECK-NEON-NEXT:    movne r5, r0
 ; CHECK-NEON-NEXT:    bl __fixunssfti
 ; CHECK-NEON-NEXT:    subs r2, r2, #1
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
 ; CHECK-NEON-NEXT:    sbcs r2, r3, #0
-; CHECK-NEON-NEXT:    movwlo r7, #1
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    moveq r0, r7
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    movne r5, r4
+; CHECK-NEON-NEXT:    movwlo r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    moveq r0, r6
+; CHECK-NEON-NEXT:    movne r6, r1
 ; CHECK-NEON-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r5
-; CHECK-NEON-NEXT:    movne r7, r1
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r6
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, pc}
 ;
 ; CHECK-FP16-LABEL: utesth_f16i64:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
-; CHECK-FP16-NEXT:    vmov.u16 r7, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r6, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfti
-; CHECK-FP16-NEXT:    mov r5, r0
-; CHECK-FP16-NEXT:    subs r0, r2, #1
-; CHECK-FP16-NEXT:    vmov s0, r7
-; CHECK-FP16-NEXT:    sbcs r0, r3, #0
-; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    mov r4, r1
-; CHECK-FP16-NEXT:    movwlo r7, #1
-; CHECK-FP16-NEXT:    cmp r7, #0
+; CHECK-FP16-NEXT:    subs r1, r2, #1
+; CHECK-FP16-NEXT:    vmov s0, r6
+; CHECK-FP16-NEXT:    sbcs r1, r3, #0
 ; CHECK-FP16-NEXT:    mov r6, #0
-; CHECK-FP16-NEXT:    moveq r5, r7
+; CHECK-FP16-NEXT:    mov r5, #0
+; CHECK-FP16-NEXT:    movwlo r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    moveq r4, r6
+; CHECK-FP16-NEXT:    movne r6, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfti
 ; CHECK-FP16-NEXT:    subs r2, r2, #1
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r6
 ; CHECK-FP16-NEXT:    sbcs r2, r3, #0
-; CHECK-FP16-NEXT:    movwlo r6, #1
-; CHECK-FP16-NEXT:    cmp r6, #0
-; CHECK-FP16-NEXT:    moveq r0, r6
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    movne r7, r4
+; CHECK-FP16-NEXT:    movwlo r5, #1
+; CHECK-FP16-NEXT:    cmp r5, #0
+; CHECK-FP16-NEXT:    moveq r0, r5
+; CHECK-FP16-NEXT:    movne r5, r1
 ; CHECK-FP16-NEXT:    vmov.32 d0[0], r0
-; CHECK-FP16-NEXT:    cmp r6, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r7
-; CHECK-FP16-NEXT:    movne r6, r1
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r6
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x half> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -2239,8 +2189,8 @@ entry:
 define <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-NEON-LABEL: ustest_f16i64:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
 ; CHECK-NEON-NEXT:    vmov r0, s0
@@ -2251,110 +2201,106 @@ define <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    mov r8, r1
+; CHECK-NEON-NEXT:    mov r4, r1
 ; CHECK-NEON-NEXT:    subs r1, r2, #1
-; CHECK-NEON-NEXT:    vmov s0, r5
 ; CHECK-NEON-NEXT:    sbcs r1, r3, #0
-; CHECK-NEON-NEXT:    mov r5, #0
-; CHECK-NEON-NEXT:    mov r9, #1
-; CHECK-NEON-NEXT:    movwlt r5, #1
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    moveq r3, r5
-; CHECK-NEON-NEXT:    moveq r8, r5
-; CHECK-NEON-NEXT:    moveq r2, r9
-; CHECK-NEON-NEXT:    movne r5, r0
-; CHECK-NEON-NEXT:    rsbs r0, r5, #0
-; CHECK-NEON-NEXT:    mov r7, #0
-; CHECK-NEON-NEXT:    rscs r0, r8, #0
-; CHECK-NEON-NEXT:    mov r6, #0
+; CHECK-NEON-NEXT:    mov r8, #1
+; CHECK-NEON-NEXT:    mov r1, #0
+; CHECK-NEON-NEXT:    movge r2, r8
+; CHECK-NEON-NEXT:    movwlt r1, #1
+; CHECK-NEON-NEXT:    cmp r1, #0
+; CHECK-NEON-NEXT:    moveq r3, r1
+; CHECK-NEON-NEXT:    moveq r4, r1
+; CHECK-NEON-NEXT:    movne r1, r0
+; CHECK-NEON-NEXT:    rsbs r0, r1, #0
+; CHECK-NEON-NEXT:    rscs r0, r4, #0
+; CHECK-NEON-NEXT:    vmov s0, r5
 ; CHECK-NEON-NEXT:    rscs r0, r2, #0
+; CHECK-NEON-NEXT:    mov r7, #0
 ; CHECK-NEON-NEXT:    rscs r0, r3, #0
+; CHECK-NEON-NEXT:    mov r5, #0
 ; CHECK-NEON-NEXT:    movwlt r7, #1
 ; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    moveq r5, r7
+; CHECK-NEON-NEXT:    moveq r4, r7
+; CHECK-NEON-NEXT:    movne r7, r1
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    subs r4, r2, #1
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEON-NEXT:    sbcs r4, r3, #0
-; CHECK-NEON-NEXT:    mov r4, #0
-; CHECK-NEON-NEXT:    movwlt r4, #1
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    movne r9, r2
-; CHECK-NEON-NEXT:    moveq r3, r4
-; CHECK-NEON-NEXT:    moveq r1, r4
-; CHECK-NEON-NEXT:    movne r4, r0
-; CHECK-NEON-NEXT:    rsbs r0, r4, #0
+; CHECK-NEON-NEXT:    subs r6, r2, #1
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEON-NEXT:    sbcs r6, r3, #0
+; CHECK-NEON-NEXT:    movlt r8, r2
+; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    movwlt r2, #1
+; CHECK-NEON-NEXT:    cmp r2, #0
+; CHECK-NEON-NEXT:    moveq r3, r2
+; CHECK-NEON-NEXT:    moveq r1, r2
+; CHECK-NEON-NEXT:    movne r2, r0
+; CHECK-NEON-NEXT:    rsbs r0, r2, #0
 ; CHECK-NEON-NEXT:    rscs r0, r1, #0
-; CHECK-NEON-NEXT:    rscs r0, r9, #0
+; CHECK-NEON-NEXT:    rscs r0, r8, #0
 ; CHECK-NEON-NEXT:    rscs r0, r3, #0
-; CHECK-NEON-NEXT:    movwlt r6, #1
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    moveq r4, r6
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    movne r7, r8
-; CHECK-NEON-NEXT:    vmov.32 d0[0], r4
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r7
-; CHECK-NEON-NEXT:    movne r6, r1
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEON-NEXT:    movwlt r5, #1
+; CHECK-NEON-NEXT:    cmp r5, #0
+; CHECK-NEON-NEXT:    moveq r2, r5
+; CHECK-NEON-NEXT:    movne r5, r1
+; CHECK-NEON-NEXT:    vmov.32 d0[0], r2
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 ;
 ; CHECK-FP16-LABEL: ustest_f16i64:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
-; CHECK-FP16-NEXT:    vmov.u16 r4, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r5, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    mov r8, r1
+; CHECK-FP16-NEXT:    mov r4, r1
 ; CHECK-FP16-NEXT:    subs r1, r2, #1
 ; CHECK-FP16-NEXT:    sbcs r1, r3, #0
-; CHECK-FP16-NEXT:    mov r6, #0
-; CHECK-FP16-NEXT:    movwlt r6, #1
-; CHECK-FP16-NEXT:    cmp r6, #0
-; CHECK-FP16-NEXT:    mov r9, #1
-; CHECK-FP16-NEXT:    moveq r3, r6
-; CHECK-FP16-NEXT:    moveq r8, r6
-; CHECK-FP16-NEXT:    moveq r2, r9
-; CHECK-FP16-NEXT:    movne r6, r0
-; CHECK-FP16-NEXT:    rsbs r0, r6, #0
-; CHECK-FP16-NEXT:    rscs r0, r8, #0
-; CHECK-FP16-NEXT:    vmov s0, r4
+; CHECK-FP16-NEXT:    mov r8, #1
+; CHECK-FP16-NEXT:    mov r1, #0
+; CHECK-FP16-NEXT:    movge r2, r8
+; CHECK-FP16-NEXT:    movwlt r1, #1
+; CHECK-FP16-NEXT:    cmp r1, #0
+; CHECK-FP16-NEXT:    moveq r3, r1
+; CHECK-FP16-NEXT:    moveq r4, r1
+; CHECK-FP16-NEXT:    movne r1, r0
+; CHECK-FP16-NEXT:    rsbs r0, r1, #0
+; CHECK-FP16-NEXT:    rscs r0, r4, #0
+; CHECK-FP16-NEXT:    vmov s0, r5
 ; CHECK-FP16-NEXT:    rscs r0, r2, #0
 ; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    rscs r0, r3, #0
 ; CHECK-FP16-NEXT:    mov r5, #0
 ; CHECK-FP16-NEXT:    movwlt r7, #1
 ; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    moveq r6, r7
+; CHECK-FP16-NEXT:    moveq r4, r7
+; CHECK-FP16-NEXT:    movne r7, r1
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    subs r4, r2, #1
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r6
-; CHECK-FP16-NEXT:    sbcs r4, r3, #0
-; CHECK-FP16-NEXT:    mov r4, #0
-; CHECK-FP16-NEXT:    movwlt r4, #1
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    movne r9, r2
-; CHECK-FP16-NEXT:    moveq r3, r4
-; CHECK-FP16-NEXT:    moveq r1, r4
-; CHECK-FP16-NEXT:    movne r4, r0
-; CHECK-FP16-NEXT:    rsbs r0, r4, #0
+; CHECK-FP16-NEXT:    subs r6, r2, #1
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r7
+; CHECK-FP16-NEXT:    sbcs r6, r3, #0
+; CHECK-FP16-NEXT:    movlt r8, r2
+; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    movwlt r2, #1
+; CHECK-FP16-NEXT:    cmp r2, #0
+; CHECK-FP16-NEXT:    moveq r3, r2
+; CHECK-FP16-NEXT:    moveq r1, r2
+; CHECK-FP16-NEXT:    movne r2, r0
+; CHECK-FP16-NEXT:    rsbs r0, r2, #0
 ; CHECK-FP16-NEXT:    rscs r0, r1, #0
-; CHECK-FP16-NEXT:    rscs r0, r9, #0
+; CHECK-FP16-NEXT:    rscs r0, r8, #0
 ; CHECK-FP16-NEXT:    rscs r0, r3, #0
 ; CHECK-FP16-NEXT:    movwlt r5, #1
 ; CHECK-FP16-NEXT:    cmp r5, #0
-; CHECK-FP16-NEXT:    moveq r4, r5
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    movne r7, r8
-; CHECK-FP16-NEXT:    vmov.32 d0[0], r4
-; CHECK-FP16-NEXT:    cmp r5, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r7
+; CHECK-FP16-NEXT:    moveq r2, r5
 ; CHECK-FP16-NEXT:    movne r5, r1
+; CHECK-FP16-NEXT:    vmov.32 d0[0], r2
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
 ; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -2380,34 +2326,34 @@ define <2 x i32> @stest_f64i32_mm(<2 x double> %x) {
 ; CHECK-NEXT:    vmov r0, r1, d8
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, r2, d9
-; CHECK-NEXT:    mvn r5, #-2147483648
-; CHECK-NEXT:    subs r3, r4, r5
-; CHECK-NEXT:    sbcs r3, r1, #0
-; CHECK-NEXT:    mvn r7, #0
-; CHECK-NEXT:    mov r3, #0
-; CHECK-NEXT:    mov r8, #-2147483648
-; CHECK-NEXT:    movwlt r3, #1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    movne r3, r1
-; CHECK-NEXT:    moveq r4, r5
-; CHECK-NEXT:    rsbs r1, r4, #-2147483648
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    sbcs r1, r7, r3
-; CHECK-NEXT:    movge r4, r8
-; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    mov r8, r1
+; CHECK-NEXT:    vmov r0, r1, d9
+; CHECK-NEXT:    mvn r6, #-2147483648
+; CHECK-NEXT:    subs r2, r4, r6
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    sbcs r2, r8, #0
+; CHECK-NEXT:    mov r7, #0
+; CHECK-NEXT:    movge r4, r6
+; CHECK-NEXT:    movwlt r5, #1
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    subs r2, r0, r5
-; CHECK-NEXT:    vmov.32 d0[0], r4
+; CHECK-NEXT:    subs r2, r0, r6
 ; CHECK-NEXT:    sbcs r2, r1, #0
-; CHECK-NEXT:    movwlt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    movne r5, r0
-; CHECK-NEXT:    rsbs r0, r5, #-2147483648
-; CHECK-NEXT:    sbcs r0, r7, r6
-; CHECK-NEXT:    movge r5, r8
-; CHECK-NEXT:    vmov.32 d0[1], r5
+; CHECK-NEXT:    movlt r6, r0
+; CHECK-NEXT:    movwlt r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mov r0, #-2147483648
+; CHECK-NEXT:    movne r7, r1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    movne r5, r8
+; CHECK-NEXT:    rsbs r2, r4, #-2147483648
+; CHECK-NEXT:    mvn r1, #0
+; CHECK-NEXT:    sbcs r2, r1, r5
+; CHECK-NEXT:    movge r4, r0
+; CHECK-NEXT:    rsbs r2, r6, #-2147483648
+; CHECK-NEXT:    vmov.32 d0[0], r4
+; CHECK-NEXT:    sbcs r1, r1, r7
+; CHECK-NEXT:    movge r6, r0
+; CHECK-NEXT:    vmov.32 d0[1], r6
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 entry:
@@ -2460,16 +2406,16 @@ define <2 x i32> @ustest_f64i32_mm(<2 x double> %x) {
 ; CHECK-NEXT:    vmov r0, r1, d8
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    vmov r2, r12, d9
-; CHECK-NEXT:    mvn r5, #0
-; CHECK-NEXT:    subs r3, r0, r5
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    sbcs r3, r1, #0
-; CHECK-NEXT:    mov r4, #0
+; CHECK-NEXT:    mvn r4, #0
+; CHECK-NEXT:    subs r5, r0, r4
 ; CHECK-NEXT:    mov r3, #0
+; CHECK-NEXT:    sbcs r5, r1, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movge r0, r4
 ; CHECK-NEXT:    movwlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    mov r5, #0
 ; CHECK-NEXT:    movne r3, r1
-; CHECK-NEXT:    moveq r0, r5
 ; CHECK-NEXT:    rsbs r1, r0, #0
 ; CHECK-NEXT:    rscs r1, r3, #0
 ; CHECK-NEXT:    movwlt r6, #1
@@ -2478,21 +2424,21 @@ define <2 x i32> @ustest_f64i32_mm(<2 x double> %x) {
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    mov r1, r12
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    subs r2, r0, r5
+; CHECK-NEXT:    subs r2, r0, r4
 ; CHECK-NEXT:    vmov.32 d0[0], r6
 ; CHECK-NEXT:    sbcs r2, r1, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    movne r2, r1
-; CHECK-NEXT:    movne r5, r0
-; CHECK-NEXT:    rsbs r0, r5, #0
-; CHECK-NEXT:    rscs r0, r2, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    movne r4, r5
-; CHECK-NEXT:    vmov.32 d0[1], r4
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    movlt r4, r0
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r0, r1
+; CHECK-NEXT:    rsbs r1, r4, #0
+; CHECK-NEXT:    rscs r0, r0, #0
+; CHECK-NEXT:    movwlt r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    movne r5, r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i64>
@@ -2505,72 +2451,81 @@ entry:
 define <4 x i32> @stest_f32i32_mm(<4 x float> %x) {
 ; CHECK-LABEL: stest_f32i32_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, sp, #8
 ; CHECK-NEXT:    vorr q4, q0, q0
-; CHECK-NEXT:    mov r8, #-2147483648
-; CHECK-NEXT:    mvn r9, #0
-; CHECK-NEXT:    mov r10, #0
 ; CHECK-NEXT:    vmov r0, s19
-; CHECK-NEXT:    vmov r5, s16
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mvn r7, #-2147483648
-; CHECK-NEXT:    subs r0, r0, r7
-; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    movne r0, r1
-; CHECK-NEXT:    moveq r4, r7
-; CHECK-NEXT:    rsbs r1, r4, #-2147483648
-; CHECK-NEXT:    sbcs r0, r9, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    movge r4, r8
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    subs r0, r0, r7
-; CHECK-NEXT:    sbcs r0, r1, #0
+; CHECK-NEXT:    vmov r2, s18
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    mvn r6, #-2147483648
+; CHECK-NEXT:    mov r3, #-2147483648
+; CHECK-NEXT:    mvn r10, #0
+; CHECK-NEXT:    vmov r7, s16
+; CHECK-NEXT:    mov r4, #0
+; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    subs r2, r11, r6
+; CHECK-NEXT:    sbcs r2, r1, #0
 ; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    movge r11, r6
 ; CHECK-NEXT:    movwlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    movne r2, r1
-; CHECK-NEXT:    moveq r5, r7
-; CHECK-NEXT:    rsbs r1, r5, #-2147483648
-; CHECK-NEXT:    sbcs r1, r9, r2
-; CHECK-NEXT:    movge r5, r8
+; CHECK-NEXT:    rsbs r1, r11, #-2147483648
+; CHECK-NEXT:    sbcs r1, r10, r2
+; CHECK-NEXT:    movge r11, r3
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    subs r0, r0, r7
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    subs r0, r0, r6
 ; CHECK-NEXT:    sbcs r0, r1, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    movne r0, r1
-; CHECK-NEXT:    moveq r6, r7
-; CHECK-NEXT:    rsbs r1, r6, #-2147483648
-; CHECK-NEXT:    sbcs r0, r9, r0
-; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    movge r6, r8
+; CHECK-NEXT:    mov r9, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    movge r5, r6
+; CHECK-NEXT:    movwlt r9, #1
+; CHECK-NEXT:    cmp r9, #0
+; CHECK-NEXT:    movne r9, r1
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    subs r2, r0, r7
-; CHECK-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    subs r0, r0, r6
+; CHECK-NEXT:    sbcs r0, r1, #0
+; CHECK-NEXT:    mov r8, #0
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    movge r7, r6
+; CHECK-NEXT:    movwlt r8, #1
+; CHECK-NEXT:    cmp r8, #0
+; CHECK-NEXT:    movne r8, r1
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    subs r2, r0, r6
 ; CHECK-NEXT:    sbcs r2, r1, #0
-; CHECK-NEXT:    movwlt r10, #1
-; CHECK-NEXT:    cmp r10, #0
-; CHECK-NEXT:    movne r10, r1
-; CHECK-NEXT:    movne r7, r0
+; CHECK-NEXT:    movlt r6, r0
+; CHECK-NEXT:    movwlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    movne r4, r1
+; CHECK-NEXT:    rsbs r0, r6, #-2147483648
+; CHECK-NEXT:    sbcs r0, r10, r4
+; CHECK-NEXT:    mov r1, #-2147483648
+; CHECK-NEXT:    movge r6, r1
 ; CHECK-NEXT:    rsbs r0, r7, #-2147483648
-; CHECK-NEXT:    vmov.32 d0[0], r5
-; CHECK-NEXT:    sbcs r0, r9, r10
-; CHECK-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEXT:    movge r7, r8
-; CHECK-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEXT:    sbcs r0, r10, r8
+; CHECK-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEXT:    movge r7, r1
+; CHECK-NEXT:    rsbs r0, r5, #-2147483648
+; CHECK-NEXT:    vmov.32 d0[0], r7
+; CHECK-NEXT:    sbcs r0, r10, r9
+; CHECK-NEXT:    movge r5, r1
+; CHECK-NEXT:    vmov.32 d1[1], r11
+; CHECK-NEXT:    vmov.32 d0[1], r5
+; CHECK-NEXT:    add sp, sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    add sp, sp, #4
+; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %conv = fptosi <4 x float> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
@@ -2636,33 +2591,33 @@ define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    mvn r7, #0
-; CHECK-NEXT:    subs r3, r0, r7
+; CHECK-NEXT:    mvn r6, #0
+; CHECK-NEXT:    subs r3, r0, r6
 ; CHECK-NEXT:    mov r4, #0
 ; CHECK-NEXT:    sbcs r3, r1, #0
-; CHECK-NEXT:    mov r10, #0
+; CHECK-NEXT:    vmov r8, s17
 ; CHECK-NEXT:    mov r3, #0
-; CHECK-NEXT:    vmov r9, s18
+; CHECK-NEXT:    movge r0, r6
 ; CHECK-NEXT:    movwlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    movne r3, r1
-; CHECK-NEXT:    moveq r0, r7
 ; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    vmov r8, s17
 ; CHECK-NEXT:    rscs r1, r3, #0
+; CHECK-NEXT:    vmov r9, s18
 ; CHECK-NEXT:    movwlt r4, #1
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    movne r4, r0
+; CHECK-NEXT:    mov r10, #0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    subs r2, r0, r7
+; CHECK-NEXT:    subs r2, r0, r6
 ; CHECK-NEXT:    mov r5, #0
 ; CHECK-NEXT:    sbcs r2, r1, #0
 ; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movge r0, r6
 ; CHECK-NEXT:    movwlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    movne r2, r1
-; CHECK-NEXT:    moveq r0, r7
 ; CHECK-NEXT:    rsbs r1, r0, #0
 ; CHECK-NEXT:    rscs r1, r2, #0
 ; CHECK-NEXT:    movwlt r5, #1
@@ -2670,36 +2625,36 @@ define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NEXT:    movne r5, r0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    subs r2, r0, r7
-; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    subs r2, r0, r6
+; CHECK-NEXT:    mov r7, #0
 ; CHECK-NEXT:    sbcs r2, r1, #0
 ; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movge r0, r6
 ; CHECK-NEXT:    movwlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    movne r2, r1
-; CHECK-NEXT:    moveq r0, r7
 ; CHECK-NEXT:    rsbs r1, r0, #0
 ; CHECK-NEXT:    rscs r1, r2, #0
-; CHECK-NEXT:    movwlt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    movne r6, r0
+; CHECK-NEXT:    movwlt r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    movne r7, r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    subs r2, r0, r7
-; CHECK-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEXT:    subs r2, r0, r6
+; CHECK-NEXT:    vmov.32 d1[0], r7
 ; CHECK-NEXT:    sbcs r2, r1, #0
-; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r6, r0
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r0, r1
+; CHECK-NEXT:    rsbs r1, r6, #0
+; CHECK-NEXT:    rscs r0, r0, #0
 ; CHECK-NEXT:    vmov.32 d0[0], r5
-; CHECK-NEXT:    movwlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    movne r2, r1
-; CHECK-NEXT:    movne r7, r0
-; CHECK-NEXT:    rsbs r0, r7, #0
-; CHECK-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEXT:    rscs r0, r2, #0
 ; CHECK-NEXT:    movwlt r10, #1
 ; CHECK-NEXT:    cmp r10, #0
-; CHECK-NEXT:    movne r10, r7
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    movne r10, r6
 ; CHECK-NEXT:    vmov.32 d0[1], r10
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
@@ -2714,150 +2669,164 @@ entry:
 define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
 ; CHECK-NEON-LABEL: stest_f16i32_mm:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEON-NEXT:    .pad #4
+; CHECK-NEON-NEXT:    sub sp, sp, #4
 ; CHECK-NEON-NEXT:    .vsave {d8, d9, d10}
 ; CHECK-NEON-NEXT:    vpush {d8, d9, d10}
+; CHECK-NEON-NEXT:    .pad #8
+; CHECK-NEON-NEXT:    sub sp, sp, #8
 ; CHECK-NEON-NEXT:    vmov r0, s3
-; CHECK-NEON-NEXT:    vmov.f32 s18, s2
-; CHECK-NEON-NEXT:    vmov.f32 s16, s1
+; CHECK-NEON-NEXT:    vmov.f32 s16, s2
+; CHECK-NEON-NEXT:    vmov.f32 s18, s1
 ; CHECK-NEON-NEXT:    vmov.f32 s20, s0
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r4, r0
-; CHECK-NEON-NEXT:    vmov r0, s20
-; CHECK-NEON-NEXT:    mvn r7, #-2147483648
-; CHECK-NEON-NEXT:    subs r2, r4, r7
+; CHECK-NEON-NEXT:    vmov r2, s16
+; CHECK-NEON-NEXT:    mov r11, r0
+; CHECK-NEON-NEXT:    vmov r0, s18
+; CHECK-NEON-NEXT:    mvn r6, #-2147483648
+; CHECK-NEON-NEXT:    mov r3, #-2147483648
+; CHECK-NEON-NEXT:    mvn r10, #0
+; CHECK-NEON-NEXT:    vmov r7, s20
+; CHECK-NEON-NEXT:    mov r4, #0
+; CHECK-NEON-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; CHECK-NEON-NEXT:    subs r2, r11, r6
 ; CHECK-NEON-NEXT:    sbcs r2, r1, #0
-; CHECK-NEON-NEXT:    mov r8, #-2147483648
 ; CHECK-NEON-NEXT:    mov r2, #0
-; CHECK-NEON-NEXT:    mvn r9, #0
+; CHECK-NEON-NEXT:    movge r11, r6
 ; CHECK-NEON-NEXT:    movwlt r2, #1
 ; CHECK-NEON-NEXT:    cmp r2, #0
 ; CHECK-NEON-NEXT:    movne r2, r1
-; CHECK-NEON-NEXT:    moveq r4, r7
-; CHECK-NEON-NEXT:    rsbs r1, r4, #-2147483648
-; CHECK-NEON-NEXT:    mov r10, #0
-; CHECK-NEON-NEXT:    sbcs r1, r9, r2
-; CHECK-NEON-NEXT:    movge r4, r8
+; CHECK-NEON-NEXT:    rsbs r1, r11, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r10, r2
+; CHECK-NEON-NEXT:    movge r11, r3
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEON-NEXT:    mov r5, r0
-; CHECK-NEON-NEXT:    subs r0, r0, r7
+; CHECK-NEON-NEXT:    subs r0, r0, r6
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
-; CHECK-NEON-NEXT:    mov r2, #0
-; CHECK-NEON-NEXT:    vmov r0, s18
-; CHECK-NEON-NEXT:    movwlt r2, #1
-; CHECK-NEON-NEXT:    cmp r2, #0
-; CHECK-NEON-NEXT:    movne r2, r1
-; CHECK-NEON-NEXT:    moveq r5, r7
-; CHECK-NEON-NEXT:    rsbs r1, r5, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r1, r9, r2
-; CHECK-NEON-NEXT:    movge r5, r8
+; CHECK-NEON-NEXT:    mov r8, #0
+; CHECK-NEON-NEXT:    mov r0, r7
+; CHECK-NEON-NEXT:    movge r5, r6
+; CHECK-NEON-NEXT:    movwlt r8, #1
+; CHECK-NEON-NEXT:    cmp r8, #0
+; CHECK-NEON-NEXT:    movne r8, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    mov r6, r0
-; CHECK-NEON-NEXT:    subs r0, r0, r7
+; CHECK-NEON-NEXT:    mov r7, r0
+; CHECK-NEON-NEXT:    subs r0, r0, r6
 ; CHECK-NEON-NEXT:    sbcs r0, r1, #0
-; CHECK-NEON-NEXT:    mov r0, #0
-; CHECK-NEON-NEXT:    movwlt r0, #1
-; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    movne r0, r1
-; CHECK-NEON-NEXT:    moveq r6, r7
-; CHECK-NEON-NEXT:    rsbs r1, r6, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r9, r0
-; CHECK-NEON-NEXT:    vmov r0, s16
-; CHECK-NEON-NEXT:    movge r6, r8
+; CHECK-NEON-NEXT:    mov r9, #0
+; CHECK-NEON-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEON-NEXT:    movge r7, r6
+; CHECK-NEON-NEXT:    movwlt r9, #1
+; CHECK-NEON-NEXT:    cmp r9, #0
+; CHECK-NEON-NEXT:    movne r9, r1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    subs r2, r0, r7
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEON-NEXT:    subs r2, r0, r6
 ; CHECK-NEON-NEXT:    sbcs r2, r1, #0
-; CHECK-NEON-NEXT:    movwlt r10, #1
-; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    movne r10, r1
-; CHECK-NEON-NEXT:    movne r7, r0
+; CHECK-NEON-NEXT:    movlt r6, r0
+; CHECK-NEON-NEXT:    movwlt r4, #1
+; CHECK-NEON-NEXT:    cmp r4, #0
+; CHECK-NEON-NEXT:    movne r4, r1
+; CHECK-NEON-NEXT:    rsbs r0, r6, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r0, r10, r4
+; CHECK-NEON-NEXT:    mov r1, #-2147483648
+; CHECK-NEON-NEXT:    movge r6, r1
 ; CHECK-NEON-NEXT:    rsbs r0, r7, #-2147483648
-; CHECK-NEON-NEXT:    vmov.32 d0[0], r5
-; CHECK-NEON-NEXT:    sbcs r0, r9, r10
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEON-NEXT:    movge r7, r8
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    sbcs r0, r10, r9
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEON-NEXT:    movge r7, r1
+; CHECK-NEON-NEXT:    rsbs r0, r5, #-2147483648
+; CHECK-NEON-NEXT:    vmov.32 d0[0], r7
+; CHECK-NEON-NEXT:    sbcs r0, r10, r8
+; CHECK-NEON-NEXT:    movge r5, r1
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r11
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r5
+; CHECK-NEON-NEXT:    add sp, sp, #8
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEON-NEXT:    add sp, sp, #4
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ;
 ; CHECK-FP16-LABEL: stest_f16i32_mm:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-FP16-NEXT:    .pad #4
+; CHECK-FP16-NEXT:    sub sp, sp, #4
 ; CHECK-FP16-NEXT:    .vsave {d8, d9}
 ; CHECK-FP16-NEXT:    vpush {d8, d9}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[3]
-; CHECK-FP16-NEXT:    vorr d8, d0, d0
-; CHECK-FP16-NEXT:    vmov.u16 r5, d0[2]
-; CHECK-FP16-NEXT:    vmov.u16 r6, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r4, d0[2]
+; CHECK-FP16-NEXT:    vmov.u16 r5, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r6, d0[1]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    mov r4, r0
+; CHECK-FP16-NEXT:    mov r10, r0
 ; CHECK-FP16-NEXT:    mvn r7, #-2147483648
 ; CHECK-FP16-NEXT:    subs r0, r0, r7
 ; CHECK-FP16-NEXT:    vmov s0, r6
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    mov r8, #-2147483648
+; CHECK-FP16-NEXT:    mov r2, #-2147483648
 ; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    mvn r9, #0
+; CHECK-FP16-NEXT:    movge r10, r7
 ; CHECK-FP16-NEXT:    movwlt r0, #1
 ; CHECK-FP16-NEXT:    cmp r0, #0
 ; CHECK-FP16-NEXT:    movne r0, r1
-; CHECK-FP16-NEXT:    moveq r4, r7
-; CHECK-FP16-NEXT:    rsbs r1, r4, #-2147483648
-; CHECK-FP16-NEXT:    mov r10, #0
+; CHECK-FP16-NEXT:    rsbs r1, r10, #-2147483648
+; CHECK-FP16-NEXT:    mvn r9, #0
 ; CHECK-FP16-NEXT:    sbcs r0, r9, r0
+; CHECK-FP16-NEXT:    vmov s16, r4
+; CHECK-FP16-NEXT:    mov r11, #0
 ; CHECK-FP16-NEXT:    vmov s18, r5
-; CHECK-FP16-NEXT:    movge r4, r8
+; CHECK-FP16-NEXT:    movge r10, r2
 ; CHECK-FP16-NEXT:    bl __fixhfdi
 ; CHECK-FP16-NEXT:    vmov.f32 s0, s18
 ; CHECK-FP16-NEXT:    mov r5, r0
 ; CHECK-FP16-NEXT:    subs r0, r0, r7
+; CHECK-FP16-NEXT:    mov r4, #0
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    movne r0, r1
-; CHECK-FP16-NEXT:    moveq r5, r7
-; CHECK-FP16-NEXT:    rsbs r1, r5, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r9, r0
-; CHECK-FP16-NEXT:    movge r5, r8
+; CHECK-FP16-NEXT:    movge r5, r7
+; CHECK-FP16-NEXT:    movwlt r4, #1
+; CHECK-FP16-NEXT:    cmp r4, #0
+; CHECK-FP16-NEXT:    movne r4, r1
 ; CHECK-FP16-NEXT:    bl __fixhfdi
+; CHECK-FP16-NEXT:    vmov.f32 s0, s16
 ; CHECK-FP16-NEXT:    mov r6, r0
 ; CHECK-FP16-NEXT:    subs r0, r0, r7
+; CHECK-FP16-NEXT:    mov r8, #0
 ; CHECK-FP16-NEXT:    sbcs r0, r1, #0
-; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    movne r0, r1
-; CHECK-FP16-NEXT:    vmov.u16 r1, d8[1]
-; CHECK-FP16-NEXT:    moveq r6, r7
-; CHECK-FP16-NEXT:    vmov s0, r1
-; CHECK-FP16-NEXT:    rsbs r1, r6, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r9, r0
-; CHECK-FP16-NEXT:    movge r6, r8
+; CHECK-FP16-NEXT:    movge r6, r7
+; CHECK-FP16-NEXT:    movwlt r8, #1
+; CHECK-FP16-NEXT:    cmp r8, #0
+; CHECK-FP16-NEXT:    movne r8, r1
 ; CHECK-FP16-NEXT:    bl __fixhfdi
 ; CHECK-FP16-NEXT:    subs r2, r0, r7
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r6
 ; CHECK-FP16-NEXT:    sbcs r2, r1, #0
-; CHECK-FP16-NEXT:    movwlt r10, #1
-; CHECK-FP16-NEXT:    cmp r10, #0
-; CHECK-FP16-NEXT:    movne r10, r1
-; CHECK-FP16-NEXT:    movne r7, r0
+; CHECK-FP16-NEXT:    movlt r7, r0
+; CHECK-FP16-NEXT:    movwlt r11, #1
+; CHECK-FP16-NEXT:    cmp r11, #0
+; CHECK-FP16-NEXT:    movne r11, r1
 ; CHECK-FP16-NEXT:    rsbs r0, r7, #-2147483648
-; CHECK-FP16-NEXT:    vmov.32 d0[0], r5
-; CHECK-FP16-NEXT:    sbcs r0, r9, r10
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
-; CHECK-FP16-NEXT:    movge r7, r8
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r7
+; CHECK-FP16-NEXT:    sbcs r0, r9, r11
+; CHECK-FP16-NEXT:    mov r1, #-2147483648
+; CHECK-FP16-NEXT:    movge r7, r1
+; CHECK-FP16-NEXT:    rsbs r0, r6, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r0, r9, r8
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r7
+; CHECK-FP16-NEXT:    movge r6, r1
+; CHECK-FP16-NEXT:    rsbs r0, r5, #-2147483648
+; CHECK-FP16-NEXT:    vmov.32 d0[0], r6
+; CHECK-FP16-NEXT:    sbcs r0, r9, r4
+; CHECK-FP16-NEXT:    movge r5, r1
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r10
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
 ; CHECK-FP16-NEXT:    vpop {d8, d9}
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-FP16-NEXT:    add sp, sp, #4
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
@@ -2975,34 +2944,34 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEON-NEXT:    vmov r2, s20
-; CHECK-NEON-NEXT:    mvn r7, #0
-; CHECK-NEON-NEXT:    subs r3, r0, r7
+; CHECK-NEON-NEXT:    mvn r6, #0
+; CHECK-NEON-NEXT:    subs r3, r0, r6
 ; CHECK-NEON-NEXT:    mov r4, #0
 ; CHECK-NEON-NEXT:    sbcs r3, r1, #0
-; CHECK-NEON-NEXT:    mov r10, #0
-; CHECK-NEON-NEXT:    mov r3, #0
 ; CHECK-NEON-NEXT:    vmov r8, s18
+; CHECK-NEON-NEXT:    mov r3, #0
+; CHECK-NEON-NEXT:    movge r0, r6
 ; CHECK-NEON-NEXT:    movwlt r3, #1
 ; CHECK-NEON-NEXT:    cmp r3, #0
 ; CHECK-NEON-NEXT:    movne r3, r1
-; CHECK-NEON-NEXT:    moveq r0, r7
 ; CHECK-NEON-NEXT:    rsbs r1, r0, #0
-; CHECK-NEON-NEXT:    vmov r9, s16
 ; CHECK-NEON-NEXT:    rscs r1, r3, #0
+; CHECK-NEON-NEXT:    vmov r9, s16
 ; CHECK-NEON-NEXT:    movwlt r4, #1
 ; CHECK-NEON-NEXT:    cmp r4, #0
 ; CHECK-NEON-NEXT:    movne r4, r0
+; CHECK-NEON-NEXT:    mov r10, #0
 ; CHECK-NEON-NEXT:    mov r0, r2
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    subs r2, r0, r7
+; CHECK-NEON-NEXT:    subs r2, r0, r6
 ; CHECK-NEON-NEXT:    mov r5, #0
 ; CHECK-NEON-NEXT:    sbcs r2, r1, #0
 ; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    movge r0, r6
 ; CHECK-NEON-NEXT:    movwlt r2, #1
 ; CHECK-NEON-NEXT:    cmp r2, #0
 ; CHECK-NEON-NEXT:    movne r2, r1
-; CHECK-NEON-NEXT:    moveq r0, r7
 ; CHECK-NEON-NEXT:    rsbs r1, r0, #0
 ; CHECK-NEON-NEXT:    rscs r1, r2, #0
 ; CHECK-NEON-NEXT:    movwlt r5, #1
@@ -3011,37 +2980,37 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    mov r0, r9
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    subs r2, r0, r7
-; CHECK-NEON-NEXT:    mov r6, #0
+; CHECK-NEON-NEXT:    subs r2, r0, r6
+; CHECK-NEON-NEXT:    mov r7, #0
 ; CHECK-NEON-NEXT:    sbcs r2, r1, #0
 ; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    movge r0, r6
 ; CHECK-NEON-NEXT:    movwlt r2, #1
 ; CHECK-NEON-NEXT:    cmp r2, #0
 ; CHECK-NEON-NEXT:    movne r2, r1
-; CHECK-NEON-NEXT:    moveq r0, r7
 ; CHECK-NEON-NEXT:    rsbs r1, r0, #0
 ; CHECK-NEON-NEXT:    rscs r1, r2, #0
-; CHECK-NEON-NEXT:    movwlt r6, #1
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    movne r6, r0
+; CHECK-NEON-NEXT:    movwlt r7, #1
+; CHECK-NEON-NEXT:    cmp r7, #0
+; CHECK-NEON-NEXT:    movne r7, r0
 ; CHECK-NEON-NEXT:    mov r0, r8
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    bl __aeabi_f2lz
-; CHECK-NEON-NEXT:    subs r2, r0, r7
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEON-NEXT:    subs r2, r0, r6
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r7
 ; CHECK-NEON-NEXT:    sbcs r2, r1, #0
-; CHECK-NEON-NEXT:    mov r2, #0
+; CHECK-NEON-NEXT:    movlt r6, r0
+; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    movwlt r0, #1
+; CHECK-NEON-NEXT:    cmp r0, #0
+; CHECK-NEON-NEXT:    movne r0, r1
+; CHECK-NEON-NEXT:    rsbs r1, r6, #0
+; CHECK-NEON-NEXT:    rscs r0, r0, #0
 ; CHECK-NEON-NEXT:    vmov.32 d0[0], r5
-; CHECK-NEON-NEXT:    movwlt r2, #1
-; CHECK-NEON-NEXT:    cmp r2, #0
-; CHECK-NEON-NEXT:    movne r2, r1
-; CHECK-NEON-NEXT:    movne r7, r0
-; CHECK-NEON-NEXT:    rsbs r0, r7, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEON-NEXT:    rscs r0, r2, #0
 ; CHECK-NEON-NEXT:    movwlt r10, #1
 ; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    movne r10, r7
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    movne r10, r6
 ; CHECK-NEON-NEXT:    vmov.32 d0[1], r10
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10}
 ; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
@@ -3054,75 +3023,75 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-FP16-NEXT:    vpush {d8, d9}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[3]
 ; CHECK-FP16-NEXT:    vorr d8, d0, d0
-; CHECK-FP16-NEXT:    vmov.u16 r6, d0[0]
-; CHECK-FP16-NEXT:    vmov.u16 r7, d0[2]
+; CHECK-FP16-NEXT:    vmov.u16 r5, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
 ; CHECK-FP16-NEXT:    vmov.u16 r2, d8[1]
-; CHECK-FP16-NEXT:    mvn r5, #0
-; CHECK-FP16-NEXT:    vmov s0, r6
+; CHECK-FP16-NEXT:    mvn r4, #0
+; CHECK-FP16-NEXT:    vmov.u16 r3, d8[2]
+; CHECK-FP16-NEXT:    vmov s0, r5
 ; CHECK-FP16-NEXT:    mov r6, #0
 ; CHECK-FP16-NEXT:    mov r8, #0
-; CHECK-FP16-NEXT:    vmov s18, r7
 ; CHECK-FP16-NEXT:    vmov s16, r2
-; CHECK-FP16-NEXT:    subs r2, r0, r5
+; CHECK-FP16-NEXT:    subs r2, r0, r4
 ; CHECK-FP16-NEXT:    sbcs r2, r1, #0
+; CHECK-FP16-NEXT:    vmov s18, r3
 ; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    movge r0, r4
 ; CHECK-FP16-NEXT:    movwlt r2, #1
 ; CHECK-FP16-NEXT:    cmp r2, #0
 ; CHECK-FP16-NEXT:    movne r2, r1
-; CHECK-FP16-NEXT:    moveq r0, r5
 ; CHECK-FP16-NEXT:    rsbs r1, r0, #0
 ; CHECK-FP16-NEXT:    rscs r1, r2, #0
 ; CHECK-FP16-NEXT:    movwlt r6, #1
 ; CHECK-FP16-NEXT:    cmp r6, #0
 ; CHECK-FP16-NEXT:    movne r6, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    subs r2, r0, r5
+; CHECK-FP16-NEXT:    subs r2, r0, r4
 ; CHECK-FP16-NEXT:    vmov.f32 s0, s18
 ; CHECK-FP16-NEXT:    sbcs r2, r1, #0
 ; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    movge r0, r4
 ; CHECK-FP16-NEXT:    movwlt r2, #1
 ; CHECK-FP16-NEXT:    cmp r2, #0
 ; CHECK-FP16-NEXT:    movne r2, r1
-; CHECK-FP16-NEXT:    moveq r0, r5
 ; CHECK-FP16-NEXT:    rsbs r1, r0, #0
 ; CHECK-FP16-NEXT:    rscs r1, r2, #0
 ; CHECK-FP16-NEXT:    movwlt r7, #1
 ; CHECK-FP16-NEXT:    cmp r7, #0
 ; CHECK-FP16-NEXT:    movne r7, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    subs r2, r0, r5
+; CHECK-FP16-NEXT:    subs r2, r0, r4
 ; CHECK-FP16-NEXT:    vmov.f32 s0, s16
 ; CHECK-FP16-NEXT:    sbcs r2, r1, #0
-; CHECK-FP16-NEXT:    mov r4, #0
+; CHECK-FP16-NEXT:    mov r5, #0
 ; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    movge r0, r4
 ; CHECK-FP16-NEXT:    movwlt r2, #1
 ; CHECK-FP16-NEXT:    cmp r2, #0
 ; CHECK-FP16-NEXT:    movne r2, r1
-; CHECK-FP16-NEXT:    moveq r0, r5
 ; CHECK-FP16-NEXT:    rsbs r1, r0, #0
 ; CHECK-FP16-NEXT:    rscs r1, r2, #0
-; CHECK-FP16-NEXT:    movwlt r4, #1
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    movne r4, r0
+; CHECK-FP16-NEXT:    movwlt r5, #1
+; CHECK-FP16-NEXT:    cmp r5, #0
+; CHECK-FP16-NEXT:    movne r5, r0
 ; CHECK-FP16-NEXT:    bl __fixhfdi
-; CHECK-FP16-NEXT:    subs r2, r0, r5
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r4
+; CHECK-FP16-NEXT:    subs r2, r0, r4
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
 ; CHECK-FP16-NEXT:    sbcs r2, r1, #0
-; CHECK-FP16-NEXT:    mov r2, #0
+; CHECK-FP16-NEXT:    movlt r4, r0
+; CHECK-FP16-NEXT:    mov r0, #0
+; CHECK-FP16-NEXT:    movwlt r0, #1
+; CHECK-FP16-NEXT:    cmp r0, #0
+; CHECK-FP16-NEXT:    movne r0, r1
+; CHECK-FP16-NEXT:    rsbs r1, r4, #0
+; CHECK-FP16-NEXT:    rscs r0, r0, #0
 ; CHECK-FP16-NEXT:    vmov.32 d0[0], r7
-; CHECK-FP16-NEXT:    movwlt r2, #1
-; CHECK-FP16-NEXT:    cmp r2, #0
-; CHECK-FP16-NEXT:    movne r2, r1
-; CHECK-FP16-NEXT:    movne r5, r0
-; CHECK-FP16-NEXT:    rsbs r0, r5, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r6
-; CHECK-FP16-NEXT:    rscs r0, r2, #0
 ; CHECK-FP16-NEXT:    movwlt r8, #1
 ; CHECK-FP16-NEXT:    cmp r8, #0
-; CHECK-FP16-NEXT:    movne r8, r5
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r6
+; CHECK-FP16-NEXT:    movne r8, r4
 ; CHECK-FP16-NEXT:    vmov.32 d0[1], r8
 ; CHECK-FP16-NEXT:    vpop {d8, d9}
 ; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, pc}
@@ -3630,59 +3599,56 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mvn r8, #0
-; CHECK-NEXT:    subs r0, r0, r8
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    sbcs r0, r1, r6
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    sbcs r0, r2, #0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    mvn r9, #0
+; CHECK-NEXT:    subs r1, r0, r9
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r4, r5
 ; CHECK-NEXT:    vorr d0, d8, d8
-; CHECK-NEXT:    sbcs r0, r3, #0
+; CHECK-NEXT:    sbcs r1, r2, #0
 ; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r9, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    moveq r3, r0
-; CHECK-NEXT:    movne r0, r2
-; CHECK-NEXT:    moveq r10, r6
-; CHECK-NEXT:    moveq r5, r8
-; CHECK-NEXT:    rsbs r1, r5, #0
-; CHECK-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-NEXT:    sbcs r0, r8, r0
-; CHECK-NEXT:    sbcs r0, r8, r3
-; CHECK-NEXT:    movwlt r7, #1
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r8, #-2147483648
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    mov r10, #0
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    movne r1, r2
+; CHECK-NEXT:    moveq r4, r5
+; CHECK-NEXT:    moveq r0, r9
+; CHECK-NEXT:    rsbs r2, r0, #0
+; CHECK-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r1
+; CHECK-NEXT:    sbcs r1, r9, r3
+; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    movne r7, r0
+; CHECK-NEXT:    moveq r4, r8
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    subs r4, r0, r8
-; CHECK-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEXT:    sbcs r4, r1, r6
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    movne r4, r2
-; CHECK-NEXT:    moveq r0, r8
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-NEXT:    sbcs r1, r8, r4
-; CHECK-NEXT:    sbcs r1, r8, r3
-; CHECK-NEXT:    movwlt r9, #1
-; CHECK-NEXT:    cmp r9, #0
+; CHECK-NEXT:    subs r6, r0, r9
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r1, r5
+; CHECK-NEXT:    sbcs r6, r2, #0
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r3, r6
+; CHECK-NEXT:    movne r6, r2
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    moveq r0, r9
-; CHECK-NEXT:    mov r1, #-2147483648
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    moveq r10, r1
-; CHECK-NEXT:    cmp r9, #0
-; CHECK-NEXT:    vmov.32 d1[1], r10
-; CHECK-NEXT:    moveq r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    rsbs r1, r0, #0
+; CHECK-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r6
+; CHECK-NEXT:    sbcs r1, r9, r3
+; CHECK-NEXT:    movwlt r10, #1
+; CHECK-NEXT:    cmp r10, #0
+; CHECK-NEXT:    movne r10, r0
+; CHECK-NEXT:    moveq r5, r8
+; CHECK-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
@@ -3696,39 +3662,36 @@ entry:
 define <2 x i64> @utest_f64i64_mm(<2 x double> %x) {
 ; CHECK-LABEL: utest_f64i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
 ; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    vorr d0, d8, d8
-; CHECK-NEXT:    sbcs r0, r3, #0
-; CHECK-NEXT:    mov r7, #0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    movwlo r7, #1
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    subs r1, r2, #1
+; CHECK-NEXT:    vorr d0, d8, d8
+; CHECK-NEXT:    sbcs r1, r3, #0
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    movwlo r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r4, r6
+; CHECK-NEXT:    movne r6, r0
 ; CHECK-NEXT:    bl __fixunsdfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r5
+; CHECK-NEXT:    vmov.32 d1[0], r6
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlo r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    moveq r0, r6
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r4
+; CHECK-NEXT:    movwlo r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    moveq r0, r5
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -3739,49 +3702,45 @@ entry:
 define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-LABEL: ustest_f64i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vorr q4, q0, q0
+; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    vorr d0, d9, d9
 ; CHECK-NEXT:    sbcs r0, r3, #0
-; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    mov r5, r3
-; CHECK-NEXT:    movwlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r6, r7
-; CHECK-NEXT:    moveq r5, r7
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwmi r6, #0
+; CHECK-NEXT:    vorr d0, d8, d8
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    moveq r5, r0
+; CHECK-NEXT:    moveq r4, r0
+; CHECK-NEXT:    movne r0, r3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwmi r4, #0
+; CHECK-NEXT:    movwmi r5, #0
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d0[0], r6
+; CHECK-NEXT:    vmov.32 d1[0], r5
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r0, r4
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r1, r6
+; CHECK-NEXT:    moveq r0, r6
+; CHECK-NEXT:    movne r6, r3
+; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    movwmi r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    movne r4, r1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    vmov.32 d1[0], r0
-; CHECK-NEXT:    movwmi r4, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r8
-; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    movwmi r1, #0
+; CHECK-NEXT:    vmov.32 d0[0], r0
 ; CHECK-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEXT:    movwmi r7, #0
-; CHECK-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEXT:    vmov.32 d0[1], r1
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -3800,59 +3759,56 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NEXT:    vmov.f64 d8, d0
 ; CHECK-NEXT:    vmov.f32 s0, s17
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mvn r8, #0
-; CHECK-NEXT:    subs r0, r0, r8
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    sbcs r0, r1, r6
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    mvn r9, #0
+; CHECK-NEXT:    subs r1, r0, r9
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r4, r5
 ; CHECK-NEXT:    vmov.f32 s0, s16
-; CHECK-NEXT:    sbcs r0, r2, #0
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    sbcs r0, r3, #0
+; CHECK-NEXT:    sbcs r1, r2, #0
 ; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r9, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    moveq r3, r0
-; CHECK-NEXT:    movne r0, r2
-; CHECK-NEXT:    moveq r10, r6
-; CHECK-NEXT:    moveq r5, r8
-; CHECK-NEXT:    rsbs r1, r5, #0
-; CHECK-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-NEXT:    sbcs r0, r8, r0
-; CHECK-NEXT:    sbcs r0, r8, r3
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r8, #-2147483648
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    mov r10, #0
+; CHECK-NEXT:    movwlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r3, r1
+; CHECK-NEXT:    movne r1, r2
+; CHECK-NEXT:    moveq r4, r5
+; CHECK-NEXT:    moveq r0, r9
+; CHECK-NEXT:    rsbs r2, r0, #0
+; CHECK-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r1
+; CHECK-NEXT:    sbcs r1, r9, r3
 ; CHECK-NEXT:    movwlt r7, #1
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    movne r7, r0
+; CHECK-NEXT:    moveq r4, r8
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs r4, r0, r8
-; CHECK-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEXT:    sbcs r4, r1, r6
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    movne r4, r2
-; CHECK-NEXT:    moveq r0, r8
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-NEXT:    sbcs r1, r8, r4
-; CHECK-NEXT:    sbcs r1, r8, r3
-; CHECK-NEXT:    movwlt r9, #1
-; CHECK-NEXT:    cmp r9, #0
+; CHECK-NEXT:    subs r6, r0, r9
+; CHECK-NEXT:    vmov.32 d1[0], r7
+; CHECK-NEXT:    sbcs r6, r1, r5
+; CHECK-NEXT:    sbcs r6, r2, #0
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r3, r6
+; CHECK-NEXT:    movne r6, r2
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    moveq r0, r9
-; CHECK-NEXT:    mov r1, #-2147483648
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    moveq r10, r1
-; CHECK-NEXT:    cmp r9, #0
-; CHECK-NEXT:    vmov.32 d1[1], r10
-; CHECK-NEXT:    moveq r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    rsbs r1, r0, #0
+; CHECK-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-NEXT:    sbcs r1, r9, r6
+; CHECK-NEXT:    sbcs r1, r9, r3
+; CHECK-NEXT:    movwlt r10, #1
+; CHECK-NEXT:    cmp r10, #0
+; CHECK-NEXT:    movne r10, r0
+; CHECK-NEXT:    moveq r5, r8
+; CHECK-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8}
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
@@ -3866,39 +3822,36 @@ entry:
 define <2 x i64> @utest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: utest_f32i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8}
 ; CHECK-NEXT:    vpush {d8}
 ; CHECK-NEXT:    vmov.f64 d8, d0
 ; CHECK-NEXT:    vmov.f32 s0, s17
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vmov.f32 s0, s16
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    mov r7, #0
-; CHECK-NEXT:    sbcs r0, r3, #0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    movwlo r7, #1
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    subs r1, r2, #1
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    moveq r5, r7
+; CHECK-NEXT:    sbcs r1, r3, #0
+; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    movwlo r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r4, r6
+; CHECK-NEXT:    movne r6, r0
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d1[0], r5
+; CHECK-NEXT:    vmov.32 d1[0], r6
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlo r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    moveq r0, r6
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r4
+; CHECK-NEXT:    movwlo r5, #1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    moveq r0, r5
+; CHECK-NEXT:    movne r5, r1
 ; CHECK-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov.32 d1[1], r7
-; CHECK-NEXT:    movne r6, r1
-; CHECK-NEXT:    vmov.32 d0[1], r6
+; CHECK-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEXT:    vmov.32 d0[1], r5
 ; CHECK-NEXT:    vpop {d8}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -3909,49 +3862,45 @@ entry:
 define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: ustest_f32i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8}
 ; CHECK-NEXT:    vpush {d8}
 ; CHECK-NEXT:    vmov.f64 d8, d0
-; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vmov.f32 s0, s17
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    vmov.f32 s0, s16
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    subs r0, r2, #1
-; CHECK-NEXT:    mov r7, #0
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    sbcs r0, r3, #0
-; CHECK-NEXT:    mov r5, r3
-; CHECK-NEXT:    movwlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    moveq r6, r7
-; CHECK-NEXT:    moveq r5, r7
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    movwmi r6, #0
+; CHECK-NEXT:    mov r6, #0
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    movwlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    moveq r5, r0
+; CHECK-NEXT:    moveq r4, r0
+; CHECK-NEXT:    movne r0, r3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movwmi r4, #0
+; CHECK-NEXT:    movwmi r5, #0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    subs r2, r2, #1
-; CHECK-NEXT:    vmov.32 d0[0], r6
+; CHECK-NEXT:    vmov.32 d1[0], r5
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    movwlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    moveq r0, r4
-; CHECK-NEXT:    moveq r3, r4
-; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    movwlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    moveq r1, r6
+; CHECK-NEXT:    moveq r0, r6
+; CHECK-NEXT:    movne r6, r3
+; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    movwmi r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    movne r4, r1
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    vmov.32 d1[0], r0
-; CHECK-NEXT:    movwmi r4, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    movne r7, r8
-; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    movwmi r1, #0
+; CHECK-NEXT:    vmov.32 d0[0], r0
 ; CHECK-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEXT:    movwmi r7, #0
-; CHECK-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEXT:    vmov.32 d0[1], r1
 ; CHECK-NEXT:    vpop {d8}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -3963,10 +3912,8 @@ entry:
 define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEON-LABEL: stest_f16i64_mm:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEON-NEXT:    .pad #4
-; CHECK-NEON-NEXT:    sub sp, sp, #4
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
 ; CHECK-NEON-NEXT:    vmov r0, s0
@@ -3977,62 +3924,58 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    mov r5, r0
+; CHECK-NEON-NEXT:    mov r4, r1
 ; CHECK-NEON-NEXT:    mvn r9, #0
-; CHECK-NEON-NEXT:    subs r0, r0, r9
-; CHECK-NEON-NEXT:    mvn r7, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r1, r7
-; CHECK-NEON-NEXT:    mov r11, r1
-; CHECK-NEON-NEXT:    sbcs r0, r2, #0
+; CHECK-NEON-NEXT:    subs r1, r0, r9
+; CHECK-NEON-NEXT:    mvn r6, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r4, r6
 ; CHECK-NEON-NEXT:    vmov s0, r8
-; CHECK-NEON-NEXT:    sbcs r0, r3, #0
-; CHECK-NEON-NEXT:    mov r6, #0
-; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    sbcs r1, r2, #0
+; CHECK-NEON-NEXT:    mov r5, #0
+; CHECK-NEON-NEXT:    sbcs r1, r3, #0
+; CHECK-NEON-NEXT:    mov r8, #-2147483648
+; CHECK-NEON-NEXT:    mov r1, #0
 ; CHECK-NEON-NEXT:    mov r10, #0
-; CHECK-NEON-NEXT:    movwlt r0, #1
-; CHECK-NEON-NEXT:    cmp r0, #0
-; CHECK-NEON-NEXT:    moveq r3, r0
-; CHECK-NEON-NEXT:    movne r0, r2
-; CHECK-NEON-NEXT:    moveq r11, r7
-; CHECK-NEON-NEXT:    moveq r5, r9
-; CHECK-NEON-NEXT:    rsbs r1, r5, #0
-; CHECK-NEON-NEXT:    rscs r1, r11, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r0, r9, r0
-; CHECK-NEON-NEXT:    sbcs r0, r9, r3
-; CHECK-NEON-NEXT:    movwlt r6, #1
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    moveq r5, r6
+; CHECK-NEON-NEXT:    movwlt r1, #1
+; CHECK-NEON-NEXT:    cmp r1, #0
+; CHECK-NEON-NEXT:    moveq r3, r1
+; CHECK-NEON-NEXT:    movne r1, r2
+; CHECK-NEON-NEXT:    moveq r4, r6
+; CHECK-NEON-NEXT:    moveq r0, r9
+; CHECK-NEON-NEXT:    rsbs r2, r0, #0
+; CHECK-NEON-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r9, r1
+; CHECK-NEON-NEXT:    sbcs r1, r9, r3
+; CHECK-NEON-NEXT:    movwlt r5, #1
+; CHECK-NEON-NEXT:    cmp r5, #0
+; CHECK-NEON-NEXT:    movne r5, r0
+; CHECK-NEON-NEXT:    moveq r4, r8
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    subs r4, r0, r9
+; CHECK-NEON-NEXT:    subs r7, r0, r9
 ; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
-; CHECK-NEON-NEXT:    sbcs r4, r1, r7
-; CHECK-NEON-NEXT:    sbcs r4, r2, #0
-; CHECK-NEON-NEXT:    sbcs r4, r3, #0
-; CHECK-NEON-NEXT:    mov r4, #0
-; CHECK-NEON-NEXT:    movwlt r4, #1
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    moveq r3, r4
-; CHECK-NEON-NEXT:    movne r7, r1
-; CHECK-NEON-NEXT:    movne r4, r2
+; CHECK-NEON-NEXT:    sbcs r7, r1, r6
+; CHECK-NEON-NEXT:    sbcs r7, r2, #0
+; CHECK-NEON-NEXT:    sbcs r7, r3, #0
+; CHECK-NEON-NEXT:    mov r7, #0
+; CHECK-NEON-NEXT:    movwlt r7, #1
+; CHECK-NEON-NEXT:    cmp r7, #0
+; CHECK-NEON-NEXT:    moveq r3, r7
+; CHECK-NEON-NEXT:    movne r7, r2
+; CHECK-NEON-NEXT:    movne r6, r1
 ; CHECK-NEON-NEXT:    moveq r0, r9
 ; CHECK-NEON-NEXT:    rsbs r1, r0, #0
-; CHECK-NEON-NEXT:    rscs r1, r7, #-2147483648
-; CHECK-NEON-NEXT:    sbcs r1, r9, r4
+; CHECK-NEON-NEXT:    rscs r1, r6, #-2147483648
+; CHECK-NEON-NEXT:    sbcs r1, r9, r7
 ; CHECK-NEON-NEXT:    sbcs r1, r9, r3
 ; CHECK-NEON-NEXT:    movwlt r10, #1
 ; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    moveq r0, r10
-; CHECK-NEON-NEXT:    mov r1, #-2147483648
-; CHECK-NEON-NEXT:    cmp r6, #0
-; CHECK-NEON-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEON-NEXT:    moveq r11, r1
-; CHECK-NEON-NEXT:    cmp r10, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r11
-; CHECK-NEON-NEXT:    moveq r7, r1
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    movne r10, r0
+; CHECK-NEON-NEXT:    moveq r6, r8
+; CHECK-NEON-NEXT:    vmov.32 d0[0], r10
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r6
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    add sp, sp, #4
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 ;
 ; CHECK-FP16-LABEL: stest_f16i64_mm:
 ; CHECK-FP16:       @ %bb.0: @ %entry
@@ -4042,59 +3985,56 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-FP16-NEXT:    vmov.u16 r7, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    mov r5, r0
-; CHECK-FP16-NEXT:    mvn r8, #0
-; CHECK-FP16-NEXT:    subs r0, r0, r8
-; CHECK-FP16-NEXT:    mvn r6, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r1, r6
-; CHECK-FP16-NEXT:    mov r10, r1
-; CHECK-FP16-NEXT:    sbcs r0, r2, #0
+; CHECK-FP16-NEXT:    mov r4, r1
+; CHECK-FP16-NEXT:    mvn r9, #0
+; CHECK-FP16-NEXT:    subs r1, r0, r9
+; CHECK-FP16-NEXT:    mvn r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r4, r5
 ; CHECK-FP16-NEXT:    vmov s0, r7
-; CHECK-FP16-NEXT:    sbcs r0, r3, #0
+; CHECK-FP16-NEXT:    sbcs r1, r2, #0
 ; CHECK-FP16-NEXT:    mov r7, #0
-; CHECK-FP16-NEXT:    mov r0, #0
-; CHECK-FP16-NEXT:    mov r9, #0
-; CHECK-FP16-NEXT:    movwlt r0, #1
-; CHECK-FP16-NEXT:    cmp r0, #0
-; CHECK-FP16-NEXT:    moveq r3, r0
-; CHECK-FP16-NEXT:    movne r0, r2
-; CHECK-FP16-NEXT:    moveq r10, r6
-; CHECK-FP16-NEXT:    moveq r5, r8
-; CHECK-FP16-NEXT:    rsbs r1, r5, #0
-; CHECK-FP16-NEXT:    rscs r1, r10, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r0, r8, r0
-; CHECK-FP16-NEXT:    sbcs r0, r8, r3
+; CHECK-FP16-NEXT:    sbcs r1, r3, #0
+; CHECK-FP16-NEXT:    mov r8, #-2147483648
+; CHECK-FP16-NEXT:    mov r1, #0
+; CHECK-FP16-NEXT:    mov r10, #0
+; CHECK-FP16-NEXT:    movwlt r1, #1
+; CHECK-FP16-NEXT:    cmp r1, #0
+; CHECK-FP16-NEXT:    moveq r3, r1
+; CHECK-FP16-NEXT:    movne r1, r2
+; CHECK-FP16-NEXT:    moveq r4, r5
+; CHECK-FP16-NEXT:    moveq r0, r9
+; CHECK-FP16-NEXT:    rsbs r2, r0, #0
+; CHECK-FP16-NEXT:    rscs r2, r4, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r9, r1
+; CHECK-FP16-NEXT:    sbcs r1, r9, r3
 ; CHECK-FP16-NEXT:    movwlt r7, #1
 ; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    moveq r5, r7
+; CHECK-FP16-NEXT:    movne r7, r0
+; CHECK-FP16-NEXT:    moveq r4, r8
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    subs r4, r0, r8
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
-; CHECK-FP16-NEXT:    sbcs r4, r1, r6
-; CHECK-FP16-NEXT:    sbcs r4, r2, #0
-; CHECK-FP16-NEXT:    sbcs r4, r3, #0
-; CHECK-FP16-NEXT:    mov r4, #0
-; CHECK-FP16-NEXT:    movwlt r4, #1
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    moveq r3, r4
-; CHECK-FP16-NEXT:    movne r6, r1
-; CHECK-FP16-NEXT:    movne r4, r2
-; CHECK-FP16-NEXT:    moveq r0, r8
-; CHECK-FP16-NEXT:    rsbs r1, r0, #0
-; CHECK-FP16-NEXT:    rscs r1, r6, #-2147483648
-; CHECK-FP16-NEXT:    sbcs r1, r8, r4
-; CHECK-FP16-NEXT:    sbcs r1, r8, r3
-; CHECK-FP16-NEXT:    movwlt r9, #1
-; CHECK-FP16-NEXT:    cmp r9, #0
+; CHECK-FP16-NEXT:    subs r6, r0, r9
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r7
+; CHECK-FP16-NEXT:    sbcs r6, r1, r5
+; CHECK-FP16-NEXT:    sbcs r6, r2, #0
+; CHECK-FP16-NEXT:    sbcs r6, r3, #0
+; CHECK-FP16-NEXT:    mov r6, #0
+; CHECK-FP16-NEXT:    movwlt r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    moveq r3, r6
+; CHECK-FP16-NEXT:    movne r6, r2
+; CHECK-FP16-NEXT:    movne r5, r1
 ; CHECK-FP16-NEXT:    moveq r0, r9
-; CHECK-FP16-NEXT:    mov r1, #-2147483648
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    vmov.32 d0[0], r0
-; CHECK-FP16-NEXT:    moveq r10, r1
-; CHECK-FP16-NEXT:    cmp r9, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r10
-; CHECK-FP16-NEXT:    moveq r6, r1
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r6
+; CHECK-FP16-NEXT:    rsbs r1, r0, #0
+; CHECK-FP16-NEXT:    rscs r1, r5, #-2147483648
+; CHECK-FP16-NEXT:    sbcs r1, r9, r6
+; CHECK-FP16-NEXT:    sbcs r1, r9, r3
+; CHECK-FP16-NEXT:    movwlt r10, #1
+; CHECK-FP16-NEXT:    cmp r10, #0
+; CHECK-FP16-NEXT:    movne r10, r0
+; CHECK-FP16-NEXT:    moveq r5, r8
+; CHECK-FP16-NEXT:    vmov.32 d0[0], r10
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
 ; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
@@ -4107,8 +4047,8 @@ entry:
 define <2 x i64> @utesth_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEON-LABEL: utesth_f16i64_mm:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
 ; CHECK-NEON-NEXT:    vmov r0, s0
@@ -4119,66 +4059,60 @@ define <2 x i64> @utesth_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixunssfti
-; CHECK-NEON-NEXT:    mov r6, r0
-; CHECK-NEON-NEXT:    subs r0, r2, #1
+; CHECK-NEON-NEXT:    mov r4, r1
+; CHECK-NEON-NEXT:    subs r1, r2, #1
 ; CHECK-NEON-NEXT:    vmov s0, r5
-; CHECK-NEON-NEXT:    sbcs r0, r3, #0
+; CHECK-NEON-NEXT:    sbcs r1, r3, #0
 ; CHECK-NEON-NEXT:    mov r5, #0
-; CHECK-NEON-NEXT:    mov r4, r1
+; CHECK-NEON-NEXT:    mov r6, #0
 ; CHECK-NEON-NEXT:    movwlo r5, #1
 ; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    mov r7, #0
-; CHECK-NEON-NEXT:    moveq r6, r5
+; CHECK-NEON-NEXT:    moveq r4, r5
+; CHECK-NEON-NEXT:    movne r5, r0
 ; CHECK-NEON-NEXT:    bl __fixunssfti
 ; CHECK-NEON-NEXT:    subs r2, r2, #1
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r6
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
 ; CHECK-NEON-NEXT:    sbcs r2, r3, #0
-; CHECK-NEON-NEXT:    movwlo r7, #1
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    moveq r0, r7
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    movne r5, r4
+; CHECK-NEON-NEXT:    movwlo r6, #1
+; CHECK-NEON-NEXT:    cmp r6, #0
+; CHECK-NEON-NEXT:    moveq r0, r6
+; CHECK-NEON-NEXT:    movne r6, r1
 ; CHECK-NEON-NEXT:    vmov.32 d0[0], r0
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[1], r5
-; CHECK-NEON-NEXT:    movne r7, r1
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r6
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, pc}
 ;
 ; CHECK-FP16-LABEL: utesth_f16i64_mm:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
-; CHECK-FP16-NEXT:    vmov.u16 r7, d0[0]
+; CHECK-FP16-NEXT:    vmov.u16 r6, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfti
-; CHECK-FP16-NEXT:    mov r5, r0
-; CHECK-FP16-NEXT:    subs r0, r2, #1
-; CHECK-FP16-NEXT:    vmov s0, r7
-; CHECK-FP16-NEXT:    sbcs r0, r3, #0
-; CHECK-FP16-NEXT:    mov r7, #0
 ; CHECK-FP16-NEXT:    mov r4, r1
-; CHECK-FP16-NEXT:    movwlo r7, #1
-; CHECK-FP16-NEXT:    cmp r7, #0
+; CHECK-FP16-NEXT:    subs r1, r2, #1
+; CHECK-FP16-NEXT:    vmov s0, r6
+; CHECK-FP16-NEXT:    sbcs r1, r3, #0
 ; CHECK-FP16-NEXT:    mov r6, #0
-; CHECK-FP16-NEXT:    moveq r5, r7
+; CHECK-FP16-NEXT:    mov r5, #0
+; CHECK-FP16-NEXT:    movwlo r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    moveq r4, r6
+; CHECK-FP16-NEXT:    movne r6, r0
 ; CHECK-FP16-NEXT:    bl __fixunshfti
 ; CHECK-FP16-NEXT:    subs r2, r2, #1
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r6
 ; CHECK-FP16-NEXT:    sbcs r2, r3, #0
-; CHECK-FP16-NEXT:    movwlo r6, #1
-; CHECK-FP16-NEXT:    cmp r6, #0
-; CHECK-FP16-NEXT:    moveq r0, r6
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    movne r7, r4
+; CHECK-FP16-NEXT:    movwlo r5, #1
+; CHECK-FP16-NEXT:    cmp r5, #0
+; CHECK-FP16-NEXT:    moveq r0, r5
+; CHECK-FP16-NEXT:    movne r5, r1
 ; CHECK-FP16-NEXT:    vmov.32 d0[0], r0
-; CHECK-FP16-NEXT:    cmp r6, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[1], r7
-; CHECK-FP16-NEXT:    movne r6, r1
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r6
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r5
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -4189,100 +4123,90 @@ entry:
 define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEON-LABEL: ustest_f16i64_mm:
 ; CHECK-NEON:       @ %bb.0: @ %entry
-; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; CHECK-NEON-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; CHECK-NEON-NEXT:    push {r4, r5, r6, r7, r11, lr}
 ; CHECK-NEON-NEXT:    .vsave {d8}
 ; CHECK-NEON-NEXT:    vpush {d8}
-; CHECK-NEON-NEXT:    vmov r0, s1
-; CHECK-NEON-NEXT:    vmov.f32 s16, s0
+; CHECK-NEON-NEXT:    vmov r0, s0
+; CHECK-NEON-NEXT:    vmov.f32 s16, s1
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
-; CHECK-NEON-NEXT:    mov r7, r0
+; CHECK-NEON-NEXT:    mov r6, r0
 ; CHECK-NEON-NEXT:    vmov r0, s16
 ; CHECK-NEON-NEXT:    bl __aeabi_h2f
 ; CHECK-NEON-NEXT:    vmov s0, r0
 ; CHECK-NEON-NEXT:    bl __fixsfti
-; CHECK-NEON-NEXT:    mov r6, r0
+; CHECK-NEON-NEXT:    mov r5, r0
 ; CHECK-NEON-NEXT:    subs r0, r2, #1
-; CHECK-NEON-NEXT:    vmov s0, r7
 ; CHECK-NEON-NEXT:    sbcs r0, r3, #0
+; CHECK-NEON-NEXT:    vmov s0, r6
+; CHECK-NEON-NEXT:    mov r0, #0
+; CHECK-NEON-NEXT:    mov r4, r1
+; CHECK-NEON-NEXT:    movwlt r0, #1
+; CHECK-NEON-NEXT:    cmp r0, #0
+; CHECK-NEON-NEXT:    moveq r5, r0
+; CHECK-NEON-NEXT:    moveq r4, r0
+; CHECK-NEON-NEXT:    movne r0, r3
+; CHECK-NEON-NEXT:    cmp r0, #0
 ; CHECK-NEON-NEXT:    mov r7, #0
-; CHECK-NEON-NEXT:    mov r5, r3
-; CHECK-NEON-NEXT:    movwlt r7, #1
-; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    moveq r6, r7
-; CHECK-NEON-NEXT:    moveq r5, r7
-; CHECK-NEON-NEXT:    cmp r5, #0
-; CHECK-NEON-NEXT:    mov r8, r1
-; CHECK-NEON-NEXT:    mov r4, #0
-; CHECK-NEON-NEXT:    movwmi r6, #0
+; CHECK-NEON-NEXT:    movwmi r4, #0
+; CHECK-NEON-NEXT:    movwmi r5, #0
 ; CHECK-NEON-NEXT:    bl __fixsfti
 ; CHECK-NEON-NEXT:    subs r2, r2, #1
-; CHECK-NEON-NEXT:    vmov.32 d0[0], r6
+; CHECK-NEON-NEXT:    vmov.32 d1[0], r5
 ; CHECK-NEON-NEXT:    sbcs r2, r3, #0
-; CHECK-NEON-NEXT:    movwlt r4, #1
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    moveq r0, r4
-; CHECK-NEON-NEXT:    moveq r3, r4
-; CHECK-NEON-NEXT:    cmp r3, #0
-; CHECK-NEON-NEXT:    movwmi r0, #0
-; CHECK-NEON-NEXT:    cmp r4, #0
-; CHECK-NEON-NEXT:    movne r4, r1
-; CHECK-NEON-NEXT:    cmp r3, #0
-; CHECK-NEON-NEXT:    vmov.32 d1[0], r0
-; CHECK-NEON-NEXT:    movwmi r4, #0
+; CHECK-NEON-NEXT:    movwlt r7, #1
 ; CHECK-NEON-NEXT:    cmp r7, #0
-; CHECK-NEON-NEXT:    movne r7, r8
-; CHECK-NEON-NEXT:    cmp r5, #0
+; CHECK-NEON-NEXT:    moveq r1, r7
+; CHECK-NEON-NEXT:    moveq r0, r7
+; CHECK-NEON-NEXT:    movne r7, r3
+; CHECK-NEON-NEXT:    cmp r7, #0
+; CHECK-NEON-NEXT:    movwmi r0, #0
+; CHECK-NEON-NEXT:    movwmi r1, #0
+; CHECK-NEON-NEXT:    vmov.32 d0[0], r0
 ; CHECK-NEON-NEXT:    vmov.32 d1[1], r4
-; CHECK-NEON-NEXT:    movwmi r7, #0
-; CHECK-NEON-NEXT:    vmov.32 d0[1], r7
+; CHECK-NEON-NEXT:    vmov.32 d0[1], r1
 ; CHECK-NEON-NEXT:    vpop {d8}
-; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; CHECK-NEON-NEXT:    pop {r4, r5, r6, r7, r11, pc}
 ;
 ; CHECK-FP16-LABEL: ustest_f16i64_mm:
 ; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r8, lr}
-; CHECK-FP16-NEXT:    vmov.u16 r0, d0[0]
-; CHECK-FP16-NEXT:    vmov.u16 r7, d0[1]
+; CHECK-FP16-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; CHECK-FP16-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-FP16-NEXT:    vmov.u16 r0, d0[1]
+; CHECK-FP16-NEXT:    vmov.u16 r7, d0[0]
 ; CHECK-FP16-NEXT:    vmov s0, r0
 ; CHECK-FP16-NEXT:    bl __fixhfti
-; CHECK-FP16-NEXT:    mov r6, r0
+; CHECK-FP16-NEXT:    mov r5, r0
 ; CHECK-FP16-NEXT:    subs r0, r2, #1
-; CHECK-FP16-NEXT:    vmov s0, r7
 ; CHECK-FP16-NEXT:    sbcs r0, r3, #0
-; CHECK-FP16-NEXT:    mov r7, #0
-; CHECK-FP16-NEXT:    mov r5, r3
-; CHECK-FP16-NEXT:    movwlt r7, #1
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    moveq r6, r7
-; CHECK-FP16-NEXT:    moveq r5, r7
-; CHECK-FP16-NEXT:    cmp r5, #0
-; CHECK-FP16-NEXT:    mov r8, r1
-; CHECK-FP16-NEXT:    mov r4, #0
-; CHECK-FP16-NEXT:    movwmi r6, #0
+; CHECK-FP16-NEXT:    vmov s0, r7
+; CHECK-FP16-NEXT:    mov r0, #0
+; CHECK-FP16-NEXT:    mov r4, r1
+; CHECK-FP16-NEXT:    movwlt r0, #1
+; CHECK-FP16-NEXT:    cmp r0, #0
+; CHECK-FP16-NEXT:    moveq r5, r0
+; CHECK-FP16-NEXT:    moveq r4, r0
+; CHECK-FP16-NEXT:    movne r0, r3
+; CHECK-FP16-NEXT:    cmp r0, #0
+; CHECK-FP16-NEXT:    mov r6, #0
+; CHECK-FP16-NEXT:    movwmi r4, #0
+; CHECK-FP16-NEXT:    movwmi r5, #0
 ; CHECK-FP16-NEXT:    bl __fixhfti
 ; CHECK-FP16-NEXT:    subs r2, r2, #1
-; CHECK-FP16-NEXT:    vmov.32 d0[0], r6
+; CHECK-FP16-NEXT:    vmov.32 d1[0], r5
 ; CHECK-FP16-NEXT:    sbcs r2, r3, #0
-; CHECK-FP16-NEXT:    movwlt r4, #1
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    moveq r0, r4
-; CHECK-FP16-NEXT:    moveq r3, r4
-; CHECK-FP16-NEXT:    cmp r3, #0
+; CHECK-FP16-NEXT:    movwlt r6, #1
+; CHECK-FP16-NEXT:    cmp r6, #0
+; CHECK-FP16-NEXT:    moveq r1, r6
+; CHECK-FP16-NEXT:    moveq r0, r6
+; CHECK-FP16-NEXT:    movne r6, r3
+; CHECK-FP16-NEXT:    cmp r6, #0
 ; CHECK-FP16-NEXT:    movwmi r0, #0
-; CHECK-FP16-NEXT:    cmp r4, #0
-; CHECK-FP16-NEXT:    movne r4, r1
-; CHECK-FP16-NEXT:    cmp r3, #0
-; CHECK-FP16-NEXT:    vmov.32 d1[0], r0
-; CHECK-FP16-NEXT:    movwmi r4, #0
-; CHECK-FP16-NEXT:    cmp r7, #0
-; CHECK-FP16-NEXT:    movne r7, r8
-; CHECK-FP16-NEXT:    cmp r5, #0
+; CHECK-FP16-NEXT:    movwmi r1, #0
+; CHECK-FP16-NEXT:    vmov.32 d0[0], r0
 ; CHECK-FP16-NEXT:    vmov.32 d1[1], r4
-; CHECK-FP16-NEXT:    movwmi r7, #0
-; CHECK-FP16-NEXT:    vmov.32 d0[1], r7
-; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; CHECK-FP16-NEXT:    vmov.32 d0[1], r1
+; CHECK-FP16-NEXT:    pop {r4, r5, r6, r7, r11, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
diff --git a/llvm/test/CodeGen/ARM/fpscr-multi-use.ll b/llvm/test/CodeGen/ARM/fpscr-multi-use.ll
index 3e77ad65df992..c59c9824a902c 100644
--- a/llvm/test/CodeGen/ARM/fpscr-multi-use.ll
+++ b/llvm/test/CodeGen/ARM/fpscr-multi-use.ll
@@ -1,31 +1,51 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=armv7 %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv7 %s -o - | FileCheck --check-prefix=THUMB %s
 
 declare double @fn()
 
 define void @test(ptr %p, ptr %res) nounwind {
 ; CHECK-LABEL: test:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    vpush {d8}
-; CHECK-NEXT:    vldr d8, [r0]
+; CHECK-NEXT:    push {r4, r5, r11, lr}
+; CHECK-NEXT:    vldr d16, [r0]
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    vcmp.f64 d8, #0
+; CHECK-NEXT:    vcmp.f64 d16, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vneg.f64 d16, d8
-; CHECK-NEXT:    vmov.f64 d17, d8
-; CHECK-NEXT:    vmovne.f64 d17, d16
-; CHECK-NEXT:    vstr d17, [r1]
+; CHECK-NEXT:    mrs r5, apsr
+; CHECK-NEXT:    vneg.f64 d17, d16
+; CHECK-NEXT:    vmovne.f64 d16, d17
+; CHECK-NEXT:    vstr d16, [r1]
 ; CHECK-NEXT:    bl fn
-; CHECK-NEXT:    vcmp.f64 d8, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vmov d16, r0, r1
 ; CHECK-NEXT:    eor r1, r1, #-2147483648
+; CHECK-NEXT:    msr APSR_nzcvq, r5
 ; CHECK-NEXT:    vmov d17, r0, r1
 ; CHECK-NEXT:    vmovne.f64 d16, d17
 ; CHECK-NEXT:    vstr d16, [r4]
-; CHECK-NEXT:    vpop {d8}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r4, r5, r11, pc}
+;
+; THUMB-LABEL: test:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    push {r4, r5, r7, lr}
+; THUMB-NEXT:    vldr d16, [r0]
+; THUMB-NEXT:    mov r4, r1
+; THUMB-NEXT:    vcmp.f64 d16, #0
+; THUMB-NEXT:    vmrs APSR_nzcv, fpscr
+; THUMB-NEXT:    mrs r5, apsr
+; THUMB-NEXT:    vneg.f64 d17, d16
+; THUMB-NEXT:    it ne
+; THUMB-NEXT:    vmovne.f64 d16, d17
+; THUMB-NEXT:    vstr d16, [r1]
+; THUMB-NEXT:    bl fn
+; THUMB-NEXT:    vmov d16, r0, r1
+; THUMB-NEXT:    eor r1, r1, #-2147483648
+; THUMB-NEXT:    msr APSR_nzcvq, r5
+; THUMB-NEXT:    vmov d17, r0, r1
+; THUMB-NEXT:    it ne
+; THUMB-NEXT:    vmovne.f64 d16, d17
+; THUMB-NEXT:    vstr d16, [r4]
+; THUMB-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %x = load double, ptr %p
   %cmp = fcmp une double %x, 0.000000e+00
diff --git a/llvm/test/CodeGen/ARM/fptoi-sat-store.ll b/llvm/test/CodeGen/ARM/fptoi-sat-store.ll
index 67edf9855f372..d895fe89a2cdc 100644
--- a/llvm/test/CodeGen/ARM/fptoi-sat-store.ll
+++ b/llvm/test/CodeGen/ARM/fptoi-sat-store.ll
@@ -75,64 +75,62 @@ define void @test_signed_i32_f64(ptr %d, double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    mov r6, r2
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    ldr r2, .LCPI1_0
-; SOFT-NEXT:    ldr r3, .LCPI1_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI1_2
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:    mov r7, r2
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    ldr r3, .LCPI1_0
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB1_2
+; SOFT-NEXT:    beq .LBB1_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r0, r0, #31
-; SOFT-NEXT:  .LBB1_2:
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB1_4
-; SOFT-NEXT:  @ %bb.3:
 ; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    b .LBB1_5
-; SOFT-NEXT:  .LBB1_4:
+; SOFT-NEXT:    b .LBB1_3
+; SOFT-NEXT:  .LBB1_2:
+; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    lsls r4, r0, #31
+; SOFT-NEXT:  .LBB1_3:
+; SOFT-NEXT:    ldr r2, .LCPI1_1
+; SOFT-NEXT:    ldr r3, .LCPI1_2
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB1_5
+; SOFT-NEXT:  @ %bb.4:
 ; SOFT-NEXT:    ldr r4, .LCPI1_3
 ; SOFT-NEXT:  .LBB1_5:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpun
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB1_7
 ; SOFT-NEXT:  @ %bb.6:
-; SOFT-NEXT:    mov r7, r4
+; SOFT-NEXT:    mov r5, r4
 ; SOFT-NEXT:  .LBB1_7:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    str r7, [r0]
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    str r5, [r0]
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI1_0:
-; SOFT-NEXT:    .long 4290772992 @ 0xffc00000
+; SOFT-NEXT:    .long 3252682752 @ 0xc1e00000
 ; SOFT-NEXT:  .LCPI1_1:
-; SOFT-NEXT:    .long 1105199103 @ 0x41dfffff
+; SOFT-NEXT:    .long 4290772992 @ 0xffc00000
 ; SOFT-NEXT:  .LCPI1_2:
-; SOFT-NEXT:    .long 3252682752 @ 0xc1e00000
+; SOFT-NEXT:    .long 1105199103 @ 0x41dfffff
 ; SOFT-NEXT:  .LCPI1_3:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -161,31 +159,31 @@ define void @test_unsigned_i32_f32(ptr %d, float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r6, r1
 ; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r1, .LCPI2_0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r1
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB2_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r4, r7
 ; SOFT-NEXT:  .LBB2_2:
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    ldr r1, .LCPI2_0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB2_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r5
+; SOFT-NEXT:    mvns r4, r5
 ; SOFT-NEXT:  .LBB2_4:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [r1]
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    str r4, [r0]
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -209,41 +207,40 @@ define void @test_unsigned_i32_f64(ptr %d, double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r5, r3
-; SOFT-NEXT:    mov r4, r2
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    ldr r2, .LCPI3_0
-; SOFT-NEXT:    ldr r3, .LCPI3_1
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:    mov r7, r2
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_d2uiz
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB3_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB3_2:
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    ldr r2, .LCPI3_0
+; SOFT-NEXT:    ldr r3, .LCPI3_1
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB3_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r6
+; SOFT-NEXT:    mvns r4, r5
 ; SOFT-NEXT:  .LBB3_4:
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [r1]
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    str r4, [r0]
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
diff --git a/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll b/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
index 84f6ee276ba5f..5179f97624489 100644
--- a/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
@@ -468,10 +468,10 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    movtlt r0, #65532
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
+; VFP2-NEXT:    vcmp.f32 s0, s0
 ; VFP2-NEXT:    itt gt
 ; VFP2-NEXT:    movwgt r0, #65535
 ; VFP2-NEXT:    movtgt r0, #3
-; VFP2-NEXT:    vcmp.f32 s0, s0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
 ; VFP2-NEXT:    it vs
 ; VFP2-NEXT:    movvs r0, #0
@@ -569,71 +569,67 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI6_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #27
 ; SOFT-NEXT:    lsls r1, r0, #27
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB6_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB6_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB6_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r4, r6
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    b .LBB6_5
 ; SOFT-NEXT:  .LBB6_4:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB6_6
-; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    ldr r0, .LCPI6_0
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB6_5:
+; SOFT-NEXT:    ldr r1, .LCPI6_1
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:  .LBB6_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB6_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB6_9
-; SOFT-NEXT:  .LBB6_8:
-; SOFT-NEXT:    ldr r3, .LCPI6_1
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB6_7
+; SOFT-NEXT:  @ %bb.6:
+; SOFT-NEXT:    ldr r0, .LCPI6_2
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB6_7:
+; SOFT-NEXT:    bne .LBB6_9
+; SOFT-NEXT:  @ %bb.8:
+; SOFT-NEXT:    mov r6, r5
 ; SOFT-NEXT:  .LBB6_9:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB6_11
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB6_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r3, .LCPI6_2
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB6_11:
-; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB6_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB6_13:
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.14:
 ; SOFT-NEXT:  .LCPI6_0:
-; SOFT-NEXT:    .long 1476395007 @ 0x57ffffff
-; SOFT-NEXT:  .LCPI6_1:
 ; SOFT-NEXT:    .long 4294836224 @ 0xfffe0000
+; SOFT-NEXT:  .LCPI6_1:
+; SOFT-NEXT:    .long 1476395007 @ 0x57ffffff
 ; SOFT-NEXT:  .LCPI6_2:
 ; SOFT-NEXT:    .long 131071 @ 0x1ffff
 ;
@@ -654,11 +650,12 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; VFP-NEXT:    movlt r0, #0
 ; VFP-NEXT:    vcmp.f32 s2, s4
 ; VFP-NEXT:    vmrs APSR_nzcv, fpscr
-; VFP-NEXT:    ittt gt
-; VFP-NEXT:    movwgt r1, #65535
-; VFP-NEXT:    movtgt r1, #1
+; VFP-NEXT:    it gt
 ; VFP-NEXT:    movgt.w r0, #-1
 ; VFP-NEXT:    vcmp.f32 s2, s2
+; VFP-NEXT:    itt gt
+; VFP-NEXT:    movwgt r1, #65535
+; VFP-NEXT:    movtgt r1, #1
 ; VFP-NEXT:    vmrs APSR_nzcv, fpscr
 ; VFP-NEXT:    itt vs
 ; VFP-NEXT:    movvs r0, #0
@@ -679,68 +676,60 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI7_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #223
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    lsls r7, r2, #31
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB7_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB7_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    beq .LBB7_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r4, r6
+; SOFT-NEXT:    mov r7, r1
 ; SOFT-NEXT:  .LBB7_4:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    ldr r1, .LCPI7_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB7_6
+; SOFT-NEXT:    beq .LBB7_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    ldr r7, .LCPI7_1
 ; SOFT-NEXT:  .LBB7_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB7_8
+; SOFT-NEXT:    bne .LBB7_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB7_9
+; SOFT-NEXT:    ldr r5, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB7_8:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    lsls r3, r2, #31
-; SOFT-NEXT:  .LBB7_9:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB7_11
-; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r3, .LCPI7_1
-; SOFT-NEXT:  .LBB7_11:
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB7_13
-; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:  .LBB7_13:
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB7_10
+; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB7_10:
+; SOFT-NEXT:    bne .LBB7_12
+; SOFT-NEXT:  @ %bb.11:
+; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:  .LBB7_12:
 ; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.14:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI7_0:
 ; SOFT-NEXT:    .long 1593835519 @ 0x5effffff
 ; SOFT-NEXT:  .LCPI7_1:
@@ -788,110 +777,106 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #20
 ; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #241
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB8_2
+; SOFT-NEXT:    movs r5, #7
+; SOFT-NEXT:    str r5, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mvns r7, r5
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB8_17
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB8_18
 ; SOFT-NEXT:  .LBB8_2:
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mvns r1, r5
-; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB8_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:  .LBB8_3:
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB8_4:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r3, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bne .LBB8_6
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB8_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r3
 ; SOFT-NEXT:  .LBB8_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB8_8
+; SOFT-NEXT:    ldr r1, .LCPI8_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB8_19
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB8_20
 ; SOFT-NEXT:  .LBB8_8:
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    bne .LBB8_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    beq .LBB8_21
+; SOFT-NEXT:  .LBB8_9:
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB8_11
 ; SOFT-NEXT:  .LBB8_10:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    beq .LBB8_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB8_19
-; SOFT-NEXT:  .LBB8_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB8_14
+; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:  .LBB8_11:
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    beq .LBB8_22
+; SOFT-NEXT:  @ %bb.12:
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB8_23
 ; SOFT-NEXT:  .LBB8_13:
-; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    beq .LBB8_24
 ; SOFT-NEXT:  .LBB8_14:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB8_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:  .LBB8_15:
+; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB8_16:
-; SOFT-NEXT:    movs r4, #7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB8_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB8_21
+; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB8_17:
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB8_2
 ; SOFT-NEXT:  .LBB8_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB8_12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB8_3
+; SOFT-NEXT:    b .LBB8_4
 ; SOFT-NEXT:  .LBB8_19:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB8_13
-; SOFT-NEXT:    b .LBB8_14
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bne .LBB8_8
 ; SOFT-NEXT:  .LBB8_20:
-; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    bne .LBB8_9
 ; SOFT-NEXT:  .LBB8_21:
-; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB8_23
-; SOFT-NEXT:  @ %bb.22:
-; SOFT-NEXT:    mov r4, r7
+; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB8_10
+; SOFT-NEXT:    b .LBB8_11
+; SOFT-NEXT:  .LBB8_22:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bne .LBB8_13
 ; SOFT-NEXT:  .LBB8_23:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB8_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r5, r4
-; SOFT-NEXT:  .LBB8_25:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bne .LBB8_14
+; SOFT-NEXT:  .LBB8_24:
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB8_15
+; SOFT-NEXT:    b .LBB8_16
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI8_0:
 ; SOFT-NEXT:    .long 1895825407 @ 0x70ffffff
 ;
@@ -943,109 +928,104 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #20
 ; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #255
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB9_2
+; SOFT-NEXT:    movs r5, #1
+; SOFT-NEXT:    lsls r7, r5, #31
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB9_18
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB9_19
 ; SOFT-NEXT:  .LBB9_2:
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mvns r1, r5
-; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB9_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:  .LBB9_3:
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB9_4:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r3, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bne .LBB9_6
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB9_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r3
 ; SOFT-NEXT:  .LBB9_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB9_8
+; SOFT-NEXT:    ldr r1, .LCPI9_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB9_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    ldr r7, .LCPI9_1
 ; SOFT-NEXT:  .LBB9_8:
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    bne .LBB9_10
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB9_20
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    beq .LBB9_21
 ; SOFT-NEXT:  .LBB9_10:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    beq .LBB9_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB9_19
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:  .LBB9_11:
+; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB9_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB9_14
-; SOFT-NEXT:  .LBB9_13:
-; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    beq .LBB9_22
+; SOFT-NEXT:  @ %bb.13:
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB9_23
 ; SOFT-NEXT:  .LBB9_14:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    bne .LBB9_16
-; SOFT-NEXT:  @ %bb.15:
 ; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    beq .LBB9_24
+; SOFT-NEXT:  .LBB9_15:
+; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    bne .LBB9_17
 ; SOFT-NEXT:  .LBB9_16:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB9_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB9_21
-; SOFT-NEXT:    b .LBB9_22
+; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:  .LBB9_17:
+; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB9_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB9_2
 ; SOFT-NEXT:  .LBB9_19:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB9_13
-; SOFT-NEXT:    b .LBB9_14
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB9_3
+; SOFT-NEXT:    b .LBB9_4
 ; SOFT-NEXT:  .LBB9_20:
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    lsls r6, r4, #31
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB9_22
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    bne .LBB9_10
 ; SOFT-NEXT:  .LBB9_21:
-; SOFT-NEXT:    ldr r6, .LCPI9_1
+; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB9_11
+; SOFT-NEXT:    b .LBB9_12
 ; SOFT-NEXT:  .LBB9_22:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB9_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r5, r6
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bne .LBB9_14
+; SOFT-NEXT:  .LBB9_23:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bne .LBB9_15
 ; SOFT-NEXT:  .LBB9_24:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB9_16
+; SOFT-NEXT:    b .LBB9_17
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI9_0:
@@ -1216,12 +1196,6 @@ define i8 @test_signed_i8_f64(double %f) nounwind {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    ldr r3, .LCPI11_0
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI11_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
@@ -1229,36 +1203,45 @@ define i8 @test_signed_i8_f64(double %f) nounwind {
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    movs r1, #127
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB11_2
+; SOFT-NEXT:    beq .LBB11_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mvns r0, r1
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    b .LBB11_3
 ; SOFT-NEXT:  .LBB11_2:
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB11_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:  .LBB11_4:
 ; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mvns r0, r1
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB11_3:
+; SOFT-NEXT:    ldr r3, .LCPI11_1
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB11_5
+; SOFT-NEXT:  @ %bb.4:
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:  .LBB11_5:
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB11_6
-; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    bne .LBB11_7
+; SOFT-NEXT:  @ %bb.6:
 ; SOFT-NEXT:    mov r4, r7
-; SOFT-NEXT:  .LBB11_6:
+; SOFT-NEXT:  .LBB11_7:
 ; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI11_0:
-; SOFT-NEXT:    .long 1080016896 @ 0x405fc000
-; SOFT-NEXT:  .LCPI11_1:
 ; SOFT-NEXT:    .long 3227516928 @ 0xc0600000
+; SOFT-NEXT:  .LCPI11_1:
+; SOFT-NEXT:    .long 1080016896 @ 0x405fc000
 ;
 ; VFP2-LABEL: test_signed_i8_f64:
 ; VFP2:       @ %bb.0:
@@ -1327,29 +1310,27 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    ldr r3, .LCPI12_0
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI12_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB12_2
+; SOFT-NEXT:    beq .LBB12_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    ldr r0, .LCPI12_2
-; SOFT-NEXT:  .LBB12_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB12_4
-; SOFT-NEXT:  @ %bb.3:
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    b .LBB12_5
-; SOFT-NEXT:  .LBB12_4:
+; SOFT-NEXT:    b .LBB12_3
+; SOFT-NEXT:  .LBB12_2:
+; SOFT-NEXT:    ldr r7, .LCPI12_1
+; SOFT-NEXT:  .LBB12_3:
+; SOFT-NEXT:    ldr r3, .LCPI12_2
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB12_5
+; SOFT-NEXT:  @ %bb.4:
 ; SOFT-NEXT:    ldr r7, .LCPI12_3
 ; SOFT-NEXT:  .LBB12_5:
 ; SOFT-NEXT:    mov r0, r6
@@ -1368,11 +1349,11 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI12_0:
-; SOFT-NEXT:    .long 1085275648 @ 0x40affe00
-; SOFT-NEXT:  .LCPI12_1:
 ; SOFT-NEXT:    .long 3232759808 @ 0xc0b00000
-; SOFT-NEXT:  .LCPI12_2:
+; SOFT-NEXT:  .LCPI12_1:
 ; SOFT-NEXT:    .long 4294963200 @ 0xfffff000
+; SOFT-NEXT:  .LCPI12_2:
+; SOFT-NEXT:    .long 1085275648 @ 0x40affe00
 ; SOFT-NEXT:  .LCPI12_3:
 ; SOFT-NEXT:    .long 4095 @ 0xfff
 ;
@@ -1444,29 +1425,27 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    ldr r3, .LCPI13_0
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI13_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB13_2
+; SOFT-NEXT:    beq .LBB13_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    ldr r0, .LCPI13_2
-; SOFT-NEXT:  .LBB13_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB13_4
-; SOFT-NEXT:  @ %bb.3:
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    b .LBB13_5
-; SOFT-NEXT:  .LBB13_4:
+; SOFT-NEXT:    b .LBB13_3
+; SOFT-NEXT:  .LBB13_2:
+; SOFT-NEXT:    ldr r7, .LCPI13_1
+; SOFT-NEXT:  .LBB13_3:
+; SOFT-NEXT:    ldr r3, .LCPI13_2
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB13_5
+; SOFT-NEXT:  @ %bb.4:
 ; SOFT-NEXT:    ldr r7, .LCPI13_3
 ; SOFT-NEXT:  .LBB13_5:
 ; SOFT-NEXT:    mov r0, r6
@@ -1485,11 +1464,11 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI13_0:
-; SOFT-NEXT:    .long 1088421824 @ 0x40dfffc0
-; SOFT-NEXT:  .LCPI13_1:
 ; SOFT-NEXT:    .long 3235905536 @ 0xc0e00000
-; SOFT-NEXT:  .LCPI13_2:
+; SOFT-NEXT:  .LCPI13_1:
 ; SOFT-NEXT:    .long 4294934528 @ 0xffff8000
+; SOFT-NEXT:  .LCPI13_2:
+; SOFT-NEXT:    .long 1088421824 @ 0x40dfffc0
 ; SOFT-NEXT:  .LCPI13_3:
 ; SOFT-NEXT:    .long 32767 @ 0x7fff
 ;
@@ -1561,29 +1540,27 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    ldr r3, .LCPI14_0
 ; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI14_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB14_2
+; SOFT-NEXT:    beq .LBB14_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    ldr r0, .LCPI14_2
-; SOFT-NEXT:  .LBB14_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB14_4
-; SOFT-NEXT:  @ %bb.3:
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    b .LBB14_5
-; SOFT-NEXT:  .LBB14_4:
+; SOFT-NEXT:    b .LBB14_3
+; SOFT-NEXT:  .LBB14_2:
+; SOFT-NEXT:    ldr r7, .LCPI14_1
+; SOFT-NEXT:  .LBB14_3:
+; SOFT-NEXT:    ldr r3, .LCPI14_2
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB14_5
+; SOFT-NEXT:  @ %bb.4:
 ; SOFT-NEXT:    ldr r7, .LCPI14_3
 ; SOFT-NEXT:  .LBB14_5:
 ; SOFT-NEXT:    mov r0, r6
@@ -1602,11 +1579,11 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI14_0:
-; SOFT-NEXT:    .long 1091567608 @ 0x410ffff8
-; SOFT-NEXT:  .LCPI14_1:
 ; SOFT-NEXT:    .long 3239051264 @ 0xc1100000
-; SOFT-NEXT:  .LCPI14_2:
+; SOFT-NEXT:  .LCPI14_1:
 ; SOFT-NEXT:    .long 4294705152 @ 0xfffc0000
+; SOFT-NEXT:  .LCPI14_2:
+; SOFT-NEXT:    .long 1091567608 @ 0x410ffff8
 ; SOFT-NEXT:  .LCPI14_3:
 ; SOFT-NEXT:    .long 262143 @ 0x3ffff
 ;
@@ -1674,58 +1651,56 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    ldr r2, .LCPI15_0
-; SOFT-NEXT:    ldr r3, .LCPI15_1
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    ldr r3, .LCPI15_2
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    ldr r3, .LCPI15_0
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2iz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB15_2
+; SOFT-NEXT:    beq .LBB15_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r0, r0, #31
-; SOFT-NEXT:  .LBB15_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB15_4
-; SOFT-NEXT:  @ %bb.3:
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    b .LBB15_5
-; SOFT-NEXT:  .LBB15_4:
+; SOFT-NEXT:    b .LBB15_3
+; SOFT-NEXT:  .LBB15_2:
+; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    lsls r7, r0, #31
+; SOFT-NEXT:  .LBB15_3:
+; SOFT-NEXT:    ldr r2, .LCPI15_1
+; SOFT-NEXT:    ldr r3, .LCPI15_2
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB15_5
+; SOFT-NEXT:  @ %bb.4:
 ; SOFT-NEXT:    ldr r7, .LCPI15_3
 ; SOFT-NEXT:  .LBB15_5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB15_7
 ; SOFT-NEXT:  @ %bb.6:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r7
 ; SOFT-NEXT:  .LBB15_7:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.8:
 ; SOFT-NEXT:  .LCPI15_0:
-; SOFT-NEXT:    .long 4290772992 @ 0xffc00000
+; SOFT-NEXT:    .long 3252682752 @ 0xc1e00000
 ; SOFT-NEXT:  .LCPI15_1:
-; SOFT-NEXT:    .long 1105199103 @ 0x41dfffff
+; SOFT-NEXT:    .long 4290772992 @ 0xffc00000
 ; SOFT-NEXT:  .LCPI15_2:
-; SOFT-NEXT:    .long 3252682752 @ 0xc1e00000
+; SOFT-NEXT:    .long 1105199103 @ 0x41dfffff
 ; SOFT-NEXT:  .LCPI15_3:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -1753,82 +1728,73 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #12
 ; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r0, #15
-; SOFT-NEXT:    mvns r2, r0
-; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #195
 ; SOFT-NEXT:    lsls r3, r0, #24
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    mov r2, r0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB16_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB16_2:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB16_4
+; SOFT-NEXT:    beq .LBB16_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    b .LBB16_5
 ; SOFT-NEXT:  .LBB16_4:
-; SOFT-NEXT:    mvns r0, r4
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    ldr r0, .LCPI16_0
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB16_5:
+; SOFT-NEXT:    movs r0, #15
+; SOFT-NEXT:    mvns r2, r0
+; SOFT-NEXT:    ldr r3, .LCPI16_1
 ; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r7
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    mvns r7, r4
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB16_7
+; SOFT-NEXT:    beq .LBB16_7
 ; SOFT-NEXT:  @ %bb.6:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    ldr r0, .LCPI16_2
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB16_7:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB16_9
+; SOFT-NEXT:    bne .LBB16_9
 ; SOFT-NEXT:  @ %bb.8:
-; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB16_10
+; SOFT-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB16_9:
-; SOFT-NEXT:    ldr r3, .LCPI16_1
-; SOFT-NEXT:  .LBB16_10:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB16_12
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r3, .LCPI16_2
-; SOFT-NEXT:  .LBB16_12:
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB16_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:  .LBB16_14:
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bne .LBB16_11
+; SOFT-NEXT:  @ %bb.10:
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:  .LBB16_11:
+; SOFT-NEXT:    bne .LBB16_13
+; SOFT-NEXT:  @ %bb.12:
+; SOFT-NEXT:    ldr r4, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:  .LBB16_13:
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.15:
+; SOFT-NEXT:  @ %bb.14:
 ; SOFT-NEXT:  .LCPI16_0:
-; SOFT-NEXT:    .long 1124073471 @ 0x42ffffff
-; SOFT-NEXT:  .LCPI16_1:
 ; SOFT-NEXT:    .long 4294836224 @ 0xfffe0000
+; SOFT-NEXT:  .LCPI16_1:
+; SOFT-NEXT:    .long 1124073471 @ 0x42ffffff
 ; SOFT-NEXT:  .LCPI16_2:
 ; SOFT-NEXT:    .long 131071 @ 0x1ffff
 ;
@@ -1850,11 +1816,12 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    vcmp.f64 d17, d18
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
-; VFP2-NEXT:    ittt gt
-; VFP2-NEXT:    movwgt r1, #65535
-; VFP2-NEXT:    movtgt r1, #1
+; VFP2-NEXT:    it gt
 ; VFP2-NEXT:    movgt.w r0, #-1
 ; VFP2-NEXT:    vcmp.f64 d17, d17
+; VFP2-NEXT:    itt gt
+; VFP2-NEXT:    movwgt r1, #65535
+; VFP2-NEXT:    movtgt r1, #1
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
 ; VFP2-NEXT:    itt vs
 ; VFP2-NEXT:    movvs r0, #0
@@ -1908,76 +1875,69 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #12
 ; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r6, r1
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI17_1
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    lsls r0, r0, #31
 ; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB17_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB17_2:
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB17_4
+; SOFT-NEXT:    beq .LBB17_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r1
 ; SOFT-NEXT:  .LBB17_4:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:    ldr r3, .LCPI17_1
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB17_6
+; SOFT-NEXT:    beq .LBB17_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r0, .LCPI17_2
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB17_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB17_8
+; SOFT-NEXT:    bne .LBB17_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r3, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB17_9
+; SOFT-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB17_8:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    lsls r3, r2, #31
-; SOFT-NEXT:  .LBB17_9:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB17_11
-; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r3, .LCPI17_2
-; SOFT-NEXT:  .LBB17_11:
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB17_13
-; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:  .LBB17_13:
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bne .LBB17_10
+; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:  .LBB17_10:
+; SOFT-NEXT:    bne .LBB17_12
+; SOFT-NEXT:  @ %bb.11:
+; SOFT-NEXT:    ldr r4, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:  .LBB17_12:
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.14:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI17_0:
-; SOFT-NEXT:    .long 1138753535 @ 0x43dfffff
-; SOFT-NEXT:  .LCPI17_1:
 ; SOFT-NEXT:    .long 3286237184 @ 0xc3e00000
+; SOFT-NEXT:  .LCPI17_1:
+; SOFT-NEXT:    .long 1138753535 @ 0x43dfffff
 ; SOFT-NEXT:  .LCPI17_2:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -2059,122 +2019,119 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #28
-; SOFT-NEXT:    sub sp, #28
+; SOFT-NEXT:    .pad #20
+; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI18_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    movs r0, #7
+; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mvns r0, r0
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_2
+; SOFT-NEXT:    beq .LBB18_17
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB18_18
 ; SOFT-NEXT:  .LBB18_2:
-; SOFT-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB18_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    beq .LBB18_19
+; SOFT-NEXT:  .LBB18_3:
+; SOFT-NEXT:    beq .LBB18_5
 ; SOFT-NEXT:  .LBB18_4:
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:  .LBB18_5:
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:    ldr r3, .LCPI18_1
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB18_20
+; SOFT-NEXT:  @ %bb.6:
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB18_21
+; SOFT-NEXT:  .LBB18_7:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    bne .LBB18_9
+; SOFT-NEXT:  .LBB18_8:
+; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:  .LBB18_9:
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB18_11
+; SOFT-NEXT:  @ %bb.10:
+; SOFT-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:  .LBB18_11:
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB18_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:  .LBB18_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB18_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:  .LBB18_8:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB18_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:  .LBB18_10:
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    beq .LBB18_22
+; SOFT-NEXT:  @ %bb.12:
 ; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    beq .LBB18_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB18_19
-; SOFT-NEXT:  .LBB18_12:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB18_14
+; SOFT-NEXT:    beq .LBB18_23
 ; SOFT-NEXT:  .LBB18_13:
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; SOFT-NEXT:  .LBB18_14:
-; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    beq .LBB18_24
+; SOFT-NEXT:  .LBB18_14:
 ; SOFT-NEXT:    bne .LBB18_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
+; SOFT-NEXT:  .LBB18_15:
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB18_16:
-; SOFT-NEXT:    movs r5, #7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB18_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB18_21
-; SOFT-NEXT:    b .LBB18_22
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB18_17:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB18_2
 ; SOFT-NEXT:  .LBB18_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB18_12
+; SOFT-NEXT:    str r7, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB18_3
 ; SOFT-NEXT:  .LBB18_19:
-; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB18_13
-; SOFT-NEXT:    b .LBB18_14
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB18_4
+; SOFT-NEXT:    b .LBB18_5
 ; SOFT-NEXT:  .LBB18_20:
-; SOFT-NEXT:    mvns r7, r5
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB18_22
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB18_7
 ; SOFT-NEXT:  .LBB18_21:
-; SOFT-NEXT:    mov r5, r7
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    beq .LBB18_8
+; SOFT-NEXT:    b .LBB18_9
 ; SOFT-NEXT:  .LBB18_22:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB18_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bne .LBB18_13
+; SOFT-NEXT:  .LBB18_23:
+; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bne .LBB18_14
 ; SOFT-NEXT:  .LBB18_24:
-; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #28
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB18_15
+; SOFT-NEXT:    b .LBB18_16
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI18_0:
-; SOFT-NEXT:    .long 1176502271 @ 0x461fffff
-; SOFT-NEXT:  .LCPI18_1:
 ; SOFT-NEXT:    .long 3323985920 @ 0xc6200000
+; SOFT-NEXT:  .LCPI18_1:
+; SOFT-NEXT:    .long 1176502271 @ 0x461fffff
 ;
 ; VFP2-LABEL: test_signed_i100_f64:
 ; VFP2:       @ %bb.0:
@@ -2266,122 +2223,118 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #28
-; SOFT-NEXT:    sub sp, #28
+; SOFT-NEXT:    .pad #20
+; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; SOFT-NEXT:    ldr r3, .LCPI19_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    lsls r0, r0, #31
+; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB19_2
+; SOFT-NEXT:    beq .LBB19_17
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB19_18
 ; SOFT-NEXT:  .LBB19_2:
-; SOFT-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB19_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    beq .LBB19_19
+; SOFT-NEXT:  .LBB19_3:
+; SOFT-NEXT:    beq .LBB19_5
 ; SOFT-NEXT:  .LBB19_4:
+; SOFT-NEXT:    str r3, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:  .LBB19_5:
+; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:    ldr r3, .LCPI19_1
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB19_20
+; SOFT-NEXT:  @ %bb.6:
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB19_21
+; SOFT-NEXT:  .LBB19_7:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    bne .LBB19_9
+; SOFT-NEXT:  .LBB19_8:
+; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:  .LBB19_9:
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB19_11
+; SOFT-NEXT:  @ %bb.10:
+; SOFT-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:  .LBB19_11:
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB19_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:  .LBB19_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB19_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:  .LBB19_8:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB19_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:  .LBB19_10:
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    beq .LBB19_22
+; SOFT-NEXT:  @ %bb.12:
 ; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    beq .LBB19_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB19_19
-; SOFT-NEXT:  .LBB19_12:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB19_14
+; SOFT-NEXT:    beq .LBB19_23
 ; SOFT-NEXT:  .LBB19_13:
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; SOFT-NEXT:  .LBB19_14:
-; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    beq .LBB19_24
+; SOFT-NEXT:  .LBB19_14:
 ; SOFT-NEXT:    bne .LBB19_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
+; SOFT-NEXT:  .LBB19_15:
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB19_16:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB19_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB19_21
-; SOFT-NEXT:    b .LBB19_22
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB19_17:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB19_2
 ; SOFT-NEXT:  .LBB19_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB19_12
+; SOFT-NEXT:    str r7, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB19_3
 ; SOFT-NEXT:  .LBB19_19:
-; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB19_13
-; SOFT-NEXT:    b .LBB19_14
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB19_4
+; SOFT-NEXT:    b .LBB19_5
 ; SOFT-NEXT:  .LBB19_20:
-; SOFT-NEXT:    movs r5, #1
-; SOFT-NEXT:    lsls r5, r5, #31
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB19_22
+; SOFT-NEXT:    ldr r0, .LCPI19_2
+; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB19_7
 ; SOFT-NEXT:  .LBB19_21:
-; SOFT-NEXT:    ldr r5, .LCPI19_2
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    beq .LBB19_8
+; SOFT-NEXT:    b .LBB19_9
 ; SOFT-NEXT:  .LBB19_22:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB19_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bne .LBB19_13
+; SOFT-NEXT:  .LBB19_23:
+; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    bne .LBB19_14
 ; SOFT-NEXT:  .LBB19_24:
-; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #28
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB19_15
+; SOFT-NEXT:    b .LBB19_16
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI19_0:
-; SOFT-NEXT:    .long 1205862399 @ 0x47dfffff
-; SOFT-NEXT:  .LCPI19_1:
 ; SOFT-NEXT:    .long 3353346048 @ 0xc7e00000
+; SOFT-NEXT:  .LCPI19_1:
+; SOFT-NEXT:    .long 1205862399 @ 0x47dfffff
 ; SOFT-NEXT:  .LCPI19_2:
 ; SOFT-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
@@ -2964,10 +2917,10 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    movtlt r0, #65532
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
+; VFP2-NEXT:    vcmp.f32 s0, s0
 ; VFP2-NEXT:    itt gt
 ; VFP2-NEXT:    movwgt r0, #65535
 ; VFP2-NEXT:    movtgt r0, #3
-; VFP2-NEXT:    vcmp.f32 s0, s0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
 ; VFP2-NEXT:    it vs
 ; VFP2-NEXT:    movvs r0, #0
@@ -3078,73 +3031,69 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI26_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #27
 ; SOFT-NEXT:    lsls r1, r0, #27
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB26_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB26_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB26_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r4, r6
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    b .LBB26_5
 ; SOFT-NEXT:  .LBB26_4:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r1, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB26_6
-; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    ldr r0, .LCPI26_0
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB26_5:
+; SOFT-NEXT:    ldr r1, .LCPI26_1
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:  .LBB26_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB26_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB26_9
-; SOFT-NEXT:  .LBB26_8:
-; SOFT-NEXT:    ldr r3, .LCPI26_1
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB26_7
+; SOFT-NEXT:  @ %bb.6:
+; SOFT-NEXT:    ldr r0, .LCPI26_2
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB26_7:
+; SOFT-NEXT:    bne .LBB26_9
+; SOFT-NEXT:  @ %bb.8:
+; SOFT-NEXT:    mov r6, r5
 ; SOFT-NEXT:  .LBB26_9:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB26_11
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB26_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r3, .LCPI26_2
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB26_11:
-; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB26_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB26_13:
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.14:
 ; SOFT-NEXT:  .LCPI26_0:
-; SOFT-NEXT:    .long 1476395007 @ 0x57ffffff
-; SOFT-NEXT:  .LCPI26_1:
 ; SOFT-NEXT:    .long 4294836224 @ 0xfffe0000
+; SOFT-NEXT:  .LCPI26_1:
+; SOFT-NEXT:    .long 1476395007 @ 0x57ffffff
 ; SOFT-NEXT:  .LCPI26_2:
 ; SOFT-NEXT:    .long 131071 @ 0x1ffff
 ;
@@ -3166,11 +3115,12 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    vcmp.f32 s2, s4
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
-; VFP2-NEXT:    ittt gt
-; VFP2-NEXT:    movwgt r1, #65535
-; VFP2-NEXT:    movtgt r1, #1
+; VFP2-NEXT:    it gt
 ; VFP2-NEXT:    movgt.w r0, #-1
 ; VFP2-NEXT:    vcmp.f32 s2, s2
+; VFP2-NEXT:    itt gt
+; VFP2-NEXT:    movwgt r1, #65535
+; VFP2-NEXT:    movtgt r1, #1
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
 ; VFP2-NEXT:    itt vs
 ; VFP2-NEXT:    movvs r0, #0
@@ -3203,11 +3153,12 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; FP16-NEXT:    movlt r0, #0
 ; FP16-NEXT:    vcmp.f32 s16, s2
 ; FP16-NEXT:    vmrs APSR_nzcv, fpscr
-; FP16-NEXT:    ittt gt
-; FP16-NEXT:    movwgt r1, #65535
-; FP16-NEXT:    movtgt r1, #1
+; FP16-NEXT:    it gt
 ; FP16-NEXT:    movgt.w r0, #-1
 ; FP16-NEXT:    vcmp.f32 s16, s16
+; FP16-NEXT:    itt gt
+; FP16-NEXT:    movwgt r1, #65535
+; FP16-NEXT:    movtgt r1, #1
 ; FP16-NEXT:    vmrs APSR_nzcv, fpscr
 ; FP16-NEXT:    itt vs
 ; FP16-NEXT:    movvs r0, #0
@@ -3229,70 +3180,62 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI27_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #223
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    lsls r7, r2, #31
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB27_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB27_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    beq .LBB27_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r4, r6
+; SOFT-NEXT:    mov r7, r1
 ; SOFT-NEXT:  .LBB27_4:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    ldr r1, .LCPI27_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB27_6
+; SOFT-NEXT:    beq .LBB27_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    ldr r7, .LCPI27_1
 ; SOFT-NEXT:  .LBB27_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB27_8
+; SOFT-NEXT:    bne .LBB27_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB27_9
+; SOFT-NEXT:    ldr r5, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB27_8:
-; SOFT-NEXT:    movs r2, #1
-; SOFT-NEXT:    lsls r3, r2, #31
-; SOFT-NEXT:  .LBB27_9:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB27_11
-; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r3, .LCPI27_1
-; SOFT-NEXT:  .LBB27_11:
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB27_13
-; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:  .LBB27_13:
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB27_10
+; SOFT-NEXT:  @ %bb.9:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB27_10:
+; SOFT-NEXT:    bne .LBB27_12
+; SOFT-NEXT:  @ %bb.11:
+; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:  .LBB27_12:
 ; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.14:
+; SOFT-NEXT:  @ %bb.13:
 ; SOFT-NEXT:  .LCPI27_0:
 ; SOFT-NEXT:    .long 1593835519 @ 0x5effffff
 ; SOFT-NEXT:  .LCPI27_1:
@@ -3379,110 +3322,106 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #241
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB28_2
+; SOFT-NEXT:    movs r5, #7
+; SOFT-NEXT:    str r5, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mvns r7, r5
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB28_17
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB28_18
 ; SOFT-NEXT:  .LBB28_2:
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mvns r1, r5
-; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB28_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:  .LBB28_3:
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB28_4:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r3, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bne .LBB28_6
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB28_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r3
 ; SOFT-NEXT:  .LBB28_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB28_8
+; SOFT-NEXT:    ldr r1, .LCPI28_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB28_19
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB28_20
 ; SOFT-NEXT:  .LBB28_8:
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    bne .LBB28_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    beq .LBB28_21
+; SOFT-NEXT:  .LBB28_9:
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB28_11
 ; SOFT-NEXT:  .LBB28_10:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    beq .LBB28_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB28_19
-; SOFT-NEXT:  .LBB28_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB28_14
+; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:  .LBB28_11:
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    beq .LBB28_22
+; SOFT-NEXT:  @ %bb.12:
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB28_23
 ; SOFT-NEXT:  .LBB28_13:
-; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    beq .LBB28_24
 ; SOFT-NEXT:  .LBB28_14:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB28_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:  .LBB28_15:
+; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB28_16:
-; SOFT-NEXT:    movs r4, #7
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB28_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB28_21
+; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB28_17:
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB28_2
 ; SOFT-NEXT:  .LBB28_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB28_12
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB28_3
+; SOFT-NEXT:    b .LBB28_4
 ; SOFT-NEXT:  .LBB28_19:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB28_13
-; SOFT-NEXT:    b .LBB28_14
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bne .LBB28_8
 ; SOFT-NEXT:  .LBB28_20:
-; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    bne .LBB28_9
 ; SOFT-NEXT:  .LBB28_21:
-; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB28_23
-; SOFT-NEXT:  @ %bb.22:
-; SOFT-NEXT:    mov r4, r7
+; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB28_10
+; SOFT-NEXT:    b .LBB28_11
+; SOFT-NEXT:  .LBB28_22:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bne .LBB28_13
 ; SOFT-NEXT:  .LBB28_23:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB28_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r5, r4
-; SOFT-NEXT:  .LBB28_25:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bne .LBB28_14
+; SOFT-NEXT:  .LBB28_24:
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB28_15
+; SOFT-NEXT:    b .LBB28_16
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI28_0:
 ; SOFT-NEXT:    .long 1895825407 @ 0x70ffffff
 ;
@@ -3579,109 +3518,104 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #255
 ; SOFT-NEXT:    lsls r1, r0, #24
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB29_2
+; SOFT-NEXT:    movs r5, #1
+; SOFT-NEXT:    lsls r7, r5, #31
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB29_18
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    beq .LBB29_19
 ; SOFT-NEXT:  .LBB29_2:
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mvns r1, r5
-; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB29_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:  .LBB29_3:
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB29_4:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    mov r3, r0
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bne .LBB29_6
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB29_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r3
 ; SOFT-NEXT:  .LBB29_6:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB29_8
+; SOFT-NEXT:    ldr r1, .LCPI29_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r5, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB29_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    ldr r7, .LCPI29_1
 ; SOFT-NEXT:  .LBB29_8:
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    bne .LBB29_10
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB29_20
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    beq .LBB29_21
 ; SOFT-NEXT:  .LBB29_10:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    beq .LBB29_18
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB29_19
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:  .LBB29_11:
+; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB29_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB29_14
-; SOFT-NEXT:  .LBB29_13:
-; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    beq .LBB29_22
+; SOFT-NEXT:  @ %bb.13:
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB29_23
 ; SOFT-NEXT:  .LBB29_14:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    bne .LBB29_16
-; SOFT-NEXT:  @ %bb.15:
 ; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    beq .LBB29_24
+; SOFT-NEXT:  .LBB29_15:
+; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    bne .LBB29_17
 ; SOFT-NEXT:  .LBB29_16:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB29_20
-; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB29_21
-; SOFT-NEXT:    b .LBB29_22
+; SOFT-NEXT:    mov r6, r3
+; SOFT-NEXT:  .LBB29_17:
+; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB29_18:
-; SOFT-NEXT:    mov r1, r2
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bne .LBB29_2
 ; SOFT-NEXT:  .LBB29_19:
-; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB29_13
-; SOFT-NEXT:    b .LBB29_14
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    beq .LBB29_3
+; SOFT-NEXT:    b .LBB29_4
 ; SOFT-NEXT:  .LBB29_20:
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    lsls r6, r4, #31
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB29_22
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    bne .LBB29_10
 ; SOFT-NEXT:  .LBB29_21:
-; SOFT-NEXT:    ldr r6, .LCPI29_1
+; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    beq .LBB29_11
+; SOFT-NEXT:    b .LBB29_12
 ; SOFT-NEXT:  .LBB29_22:
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    bne .LBB29_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r5, r6
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bne .LBB29_14
+; SOFT-NEXT:  .LBB29_23:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bne .LBB29_15
 ; SOFT-NEXT:  .LBB29_24:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB29_16
+; SOFT-NEXT:    b .LBB29_17
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI29_0:
diff --git a/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll b/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
index 14eb67104edda..4cc5f943dadff 100644
--- a/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
@@ -24,41 +24,38 @@ define i1 @test_signed_i1_f32(float %f) nounwind {
 ; SOFT-NEXT:    .save {r4, r5, r6, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r0, #127
-; SOFT-NEXT:    lsls r1, r0, #23
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB0_3
+; SOFT-NEXT:    bne .LBB0_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB0_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB0_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB0_3:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB0_2
+; SOFT-NEXT:    movs r0, #127
+; SOFT-NEXT:    lsls r1, r0, #23
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB0_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    movs r4, #1
 ; SOFT-NEXT:  .LBB0_4:
-; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ;
 ; VFP2-LABEL: test_signed_i1_f32:
 ; VFP2:       @ %bb.0:
-; VFP2-NEXT:    vmov s2, r0
-; VFP2-NEXT:    vmov.f32 s0, #1.000000e+00
-; VFP2-NEXT:    vcvt.u32.f32 s4, s2
-; VFP2-NEXT:    vcmp.f32 s2, #0
+; VFP2-NEXT:    vmov s0, r0
+; VFP2-NEXT:    vmov.f32 s4, #1.000000e+00
+; VFP2-NEXT:    vcvt.u32.f32 s2, s0
+; VFP2-NEXT:    vcmp.f32 s0, #0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
-; VFP2-NEXT:    vcmp.f32 s2, s0
-; VFP2-NEXT:    vmov r0, s4
+; VFP2-NEXT:    vcmp.f32 s0, s4
+; VFP2-NEXT:    vmov r0, s2
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
@@ -89,29 +86,27 @@ define i8 @test_signed_i8_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI1_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB1_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB1_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB1_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB1_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB1_3:
+; SOFT-NEXT:    ldr r1, .LCPI1_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB1_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB1_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    movs r4, #255
 ; SOFT-NEXT:  .LBB1_4:
-; SOFT-NEXT:    movs r0, #255
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -163,29 +158,27 @@ define i13 @test_signed_i13_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI2_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB2_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB2_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB2_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB2_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB2_3:
+; SOFT-NEXT:    ldr r1, .LCPI2_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB2_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB2_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI2_1
 ; SOFT-NEXT:  .LBB2_4:
-; SOFT-NEXT:    ldr r0, .LCPI2_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -239,29 +232,27 @@ define i16 @test_signed_i16_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI3_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB3_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB3_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB3_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB3_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB3_3:
+; SOFT-NEXT:    ldr r1, .LCPI3_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB3_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB3_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI3_1
 ; SOFT-NEXT:  .LBB3_4:
-; SOFT-NEXT:    ldr r0, .LCPI3_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -315,29 +306,27 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI4_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB4_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB4_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB4_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB4_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB4_3:
+; SOFT-NEXT:    ldr r1, .LCPI4_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB4_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB4_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI4_1
 ; SOFT-NEXT:  .LBB4_4:
-; SOFT-NEXT:    ldr r0, .LCPI4_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -394,31 +383,28 @@ define i32 @test_signed_i32_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI5_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB5_3
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB5_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB5_4
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB5_2:
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB5_3:
+; SOFT-NEXT:    ldr r1, .LCPI5_0
 ; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB5_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB5_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mvns r5, r4
 ; SOFT-NEXT:  .LBB5_4:
-; SOFT-NEXT:    mvns r0, r4
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -443,45 +429,39 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI6_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB6_5
-; SOFT-NEXT:  @ %bb.1:
+; SOFT-NEXT:    mov r7, r1
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB6_6
+; SOFT-NEXT:    bne .LBB6_2
+; SOFT-NEXT:  @ %bb.1:
+; SOFT-NEXT:    mov r7, r4
 ; SOFT-NEXT:  .LBB6_2:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB6_7
-; SOFT-NEXT:  .LBB6_3:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB6_8
+; SOFT-NEXT:    bne .LBB6_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB6_4:
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB6_5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB6_2
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    ldr r1, .LCPI6_0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mvns r0, r5
+; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    bne .LBB6_6
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB6_6:
-; SOFT-NEXT:    mvns r0, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB6_3
-; SOFT-NEXT:  .LBB6_7:
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB6_4
+; SOFT-NEXT:    beq .LBB6_8
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    ldr r7, .LCPI6_1
 ; SOFT-NEXT:  .LBB6_8:
-; SOFT-NEXT:    ldr r1, .LCPI6_1
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -528,48 +508,43 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI7_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB7_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:  .LBB7_2:
-; SOFT-NEXT:    mvns r2, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    beq .LBB7_7
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB7_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB7_8
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB7_4:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB7_6
-; SOFT-NEXT:  .LBB7_5:
-; SOFT-NEXT:    mov r2, r1
+; SOFT-NEXT:    ldr r1, .LCPI7_0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r1, r4
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r1
+; SOFT-NEXT:    beq .LBB7_7
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    beq .LBB7_8
 ; SOFT-NEXT:  .LBB7_6:
-; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB7_7:
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB7_4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB7_6
 ; SOFT-NEXT:  .LBB7_8:
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB7_5
-; SOFT-NEXT:    b .LBB7_6
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI7_0:
@@ -609,76 +584,71 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB8_2
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB8_11
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB8_12
 ; SOFT-NEXT:  .LBB8_2:
-; SOFT-NEXT:    mvns r2, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r6, r2
 ; SOFT-NEXT:    bne .LBB8_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:  .LBB8_3:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:  .LBB8_4:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB8_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB8_6:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    ldr r1, .LCPI8_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    beq .LBB8_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    beq .LBB8_14
 ; SOFT-NEXT:  .LBB8_8:
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB8_15
 ; SOFT-NEXT:  .LBB8_9:
-; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB8_16
 ; SOFT-NEXT:  .LBB8_10:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB8_12
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB8_11:
-; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    bne .LBB8_2
 ; SOFT-NEXT:  .LBB8_12:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    beq .LBB8_3
+; SOFT-NEXT:    b .LBB8_4
 ; SOFT-NEXT:  .LBB8_13:
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    bne .LBB8_8
 ; SOFT-NEXT:  .LBB8_14:
-; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB8_9
 ; SOFT-NEXT:  .LBB8_15:
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB8_10
 ; SOFT-NEXT:  .LBB8_16:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB8_11
-; SOFT-NEXT:    b .LBB8_12
+; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI8_0:
@@ -722,75 +692,72 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB9_2
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB9_11
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB9_12
 ; SOFT-NEXT:  .LBB9_2:
-; SOFT-NEXT:    mvns r6, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bne .LBB9_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:  .LBB9_3:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:  .LBB9_4:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB9_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB9_6:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bne .LBB9_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:  .LBB9_8:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB9_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:  .LBB9_10:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    ldr r1, .LCPI9_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r3, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    beq .LBB9_13
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    beq .LBB9_14
+; SOFT-NEXT:  .LBB9_8:
+; SOFT-NEXT:    mov r2, r3
 ; SOFT-NEXT:    beq .LBB9_15
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:  .LBB9_9:
 ; SOFT-NEXT:    beq .LBB9_16
+; SOFT-NEXT:  .LBB9_10:
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB9_11:
+; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    bne .LBB9_2
 ; SOFT-NEXT:  .LBB9_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB9_14
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    beq .LBB9_3
+; SOFT-NEXT:    b .LBB9_4
 ; SOFT-NEXT:  .LBB9_13:
-; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    bne .LBB9_8
 ; SOFT-NEXT:  .LBB9_14:
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    bne .LBB9_9
 ; SOFT-NEXT:  .LBB9_15:
-; SOFT-NEXT:    mov r7, r2
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    bne .LBB9_10
 ; SOFT-NEXT:  .LBB9_16:
-; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB9_13
-; SOFT-NEXT:    b .LBB9_14
+; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI9_0:
@@ -852,33 +819,32 @@ define i1 @test_signed_i1_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI10_0
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2uiz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB10_3
+; SOFT-NEXT:    bne .LBB10_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB10_4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB10_2:
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r3, .LCPI10_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB10_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB10_3:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB10_2
 ; SOFT-NEXT:  .LBB10_4:
 ; SOFT-NEXT:    movs r0, #1
 ; SOFT-NEXT:    add sp, #4
@@ -931,33 +897,32 @@ define i8 @test_signed_i8_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI11_0
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2uiz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB11_3
+; SOFT-NEXT:    bne .LBB11_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB11_4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB11_2:
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r3, .LCPI11_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB11_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB11_3:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB11_2
 ; SOFT-NEXT:  .LBB11_4:
 ; SOFT-NEXT:    movs r0, #255
 ; SOFT-NEXT:    add sp, #4
@@ -1018,33 +983,32 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI12_0
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2uiz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB12_3
+; SOFT-NEXT:    bne .LBB12_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB12_4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB12_2:
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r3, .LCPI12_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB12_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB12_3:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB12_2
 ; SOFT-NEXT:  .LBB12_4:
 ; SOFT-NEXT:    ldr r0, .LCPI12_1
 ; SOFT-NEXT:    add sp, #4
@@ -1107,33 +1071,32 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI13_0
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2uiz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB13_3
+; SOFT-NEXT:    bne .LBB13_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB13_4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB13_2:
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r3, .LCPI13_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB13_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB13_3:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB13_2
 ; SOFT-NEXT:  .LBB13_4:
 ; SOFT-NEXT:    ldr r0, .LCPI13_1
 ; SOFT-NEXT:    add sp, #4
@@ -1196,33 +1159,32 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    ldr r3, .LCPI14_0
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    bl __aeabi_dcmpgt
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2uiz
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    beq .LBB14_3
+; SOFT-NEXT:    bne .LBB14_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB14_4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB14_2:
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r3, .LCPI14_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB14_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB14_3:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB14_2
 ; SOFT-NEXT:  .LBB14_4:
 ; SOFT-NEXT:    ldr r0, .LCPI14_1
 ; SOFT-NEXT:    add sp, #4
@@ -1286,32 +1248,34 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r2, .LCPI15_0
-; SOFT-NEXT:    ldr r3, .LCPI15_1
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_d2uiz
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB15_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB15_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    beq .LBB15_4
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    ldr r2, .LCPI15_0
+; SOFT-NEXT:    ldr r3, .LCPI15_1
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    bne .LBB15_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB15_4:
+; SOFT-NEXT:    mvns r0, r4
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -1345,49 +1309,47 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r0, #7
-; SOFT-NEXT:    mvns r2, r0
-; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    mov r6, r1
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_d2ulz
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB16_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:  .LBB16_2:
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB16_6
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB16_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB16_7
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB16_4:
-; SOFT-NEXT:    cmp r2, #0
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r0, #7
+; SOFT-NEXT:    mvns r2, r0
+; SOFT-NEXT:    ldr r3, .LCPI16_0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mvns r0, r5
+; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    beq .LBB16_7
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB16_8
-; SOFT-NEXT:  .LBB16_5:
+; SOFT-NEXT:  .LBB16_6:
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB16_6:
-; SOFT-NEXT:    mvns r0, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB16_4
 ; SOFT-NEXT:  .LBB16_7:
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    beq .LBB16_5
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB16_6
 ; SOFT-NEXT:  .LBB16_8:
 ; SOFT-NEXT:    ldr r1, .LCPI16_1
 ; SOFT-NEXT:    add sp, #4
@@ -1458,54 +1420,46 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r4, r6
-; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    mov r3, r7
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_d2ulz
-; SOFT-NEXT:    mov r2, r0
 ; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB17_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:  .LBB17_2:
-; SOFT-NEXT:    ldr r3, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    beq .LBB17_7
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB17_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB17_8
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB17_4:
-; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mvns r7, r7
+; SOFT-NEXT:    ldr r3, .LCPI17_0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bne .LBB17_6
-; SOFT-NEXT:  .LBB17_5:
-; SOFT-NEXT:    mov r4, r1
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:  .LBB17_6:
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bne .LBB17_8
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:  .LBB17_8:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB17_7:
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB17_4
-; SOFT-NEXT:  .LBB17_8:
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    cmp r3, #0
-; SOFT-NEXT:    beq .LBB17_5
-; SOFT-NEXT:    b .LBB17_6
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI17_0:
@@ -1573,82 +1527,76 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    mov r6, r1
 ; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r2, r6
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __fixunsdfti
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB18_2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB18_12
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r4, r6
+; SOFT-NEXT:    beq .LBB18_13
 ; SOFT-NEXT:  .LBB18_2:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB18_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:  .LBB18_3:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:  .LBB18_4:
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB18_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB18_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r4, [sp] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    beq .LBB18_13
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mvns r5, r5
+; SOFT-NEXT:    ldr r3, .LCPI18_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    beq .LBB18_14
-; SOFT-NEXT:  .LBB18_8:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    beq .LBB18_15
-; SOFT-NEXT:  .LBB18_9:
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:  .LBB18_8:
 ; SOFT-NEXT:    beq .LBB18_16
+; SOFT-NEXT:  .LBB18_9:
+; SOFT-NEXT:    bne .LBB18_11
 ; SOFT-NEXT:  .LBB18_10:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB18_12
+; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB18_11:
-; SOFT-NEXT:    movs r3, #15
-; SOFT-NEXT:  .LBB18_12:
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB18_12:
+; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    bne .LBB18_2
 ; SOFT-NEXT:  .LBB18_13:
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB18_8
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    beq .LBB18_3
+; SOFT-NEXT:    b .LBB18_4
 ; SOFT-NEXT:  .LBB18_14:
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB18_9
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    bne .LBB18_8
 ; SOFT-NEXT:  .LBB18_15:
-; SOFT-NEXT:    mov r4, r2
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB18_10
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    bne .LBB18_9
 ; SOFT-NEXT:  .LBB18_16:
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB18_11
-; SOFT-NEXT:    b .LBB18_12
+; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB18_10
+; SOFT-NEXT:    b .LBB18_11
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI18_0:
@@ -1724,78 +1672,77 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r4, r6
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    mov r6, r1
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __fixunsdfti
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB19_2
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB19_12
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r5, r6
+; SOFT-NEXT:    beq .LBB19_13
 ; SOFT-NEXT:  .LBB19_2:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB19_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:  .LBB19_3:
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:  .LBB19_4:
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB19_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB19_6:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bne .LBB19_8
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mvns r5, r5
+; SOFT-NEXT:    ldr r3, .LCPI19_0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    bl __aeabi_dcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB19_14
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    beq .LBB19_15
 ; SOFT-NEXT:  .LBB19_8:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB19_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    beq .LBB19_16
+; SOFT-NEXT:  .LBB19_9:
+; SOFT-NEXT:    bne .LBB19_11
 ; SOFT-NEXT:  .LBB19_10:
-; SOFT-NEXT:    ldr r5, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r5, r4
-; SOFT-NEXT:    bne .LBB19_12
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r5, r2
+; SOFT-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:  .LBB19_11:
+; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB19_12:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    bne .LBB19_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    bne .LBB19_2
+; SOFT-NEXT:  .LBB19_13:
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    beq .LBB19_3
+; SOFT-NEXT:    b .LBB19_4
 ; SOFT-NEXT:  .LBB19_14:
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    bne .LBB19_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    mov r4, r3
-; SOFT-NEXT:  .LBB19_16:
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    bne .LBB19_8
+; SOFT-NEXT:  .LBB19_15:
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    bne .LBB19_9
+; SOFT-NEXT:  .LBB19_16:
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB19_10
+; SOFT-NEXT:    b .LBB19_11
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI19_0:
@@ -1889,30 +1836,27 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r0, #127
-; SOFT-NEXT:    lsls r1, r0, #23
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB20_3
+; SOFT-NEXT:    bne .LBB20_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB20_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB20_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB20_3:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB20_2
+; SOFT-NEXT:    movs r0, #127
+; SOFT-NEXT:    lsls r1, r0, #23
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB20_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    movs r4, #1
 ; SOFT-NEXT:  .LBB20_4:
-; SOFT-NEXT:    movs r0, #1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ;
 ; VFP2-LABEL: test_signed_i1_f16:
@@ -1920,13 +1864,13 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; VFP2-NEXT:    .save {r7, lr}
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    bl __aeabi_h2f
-; VFP2-NEXT:    vmov s2, r0
-; VFP2-NEXT:    vmov.f32 s0, #1.000000e+00
-; VFP2-NEXT:    vcvt.u32.f32 s4, s2
-; VFP2-NEXT:    vcmp.f32 s2, #0
+; VFP2-NEXT:    vmov s0, r0
+; VFP2-NEXT:    vmov.f32 s4, #1.000000e+00
+; VFP2-NEXT:    vcvt.u32.f32 s2, s0
+; VFP2-NEXT:    vcmp.f32 s0, #0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
-; VFP2-NEXT:    vcmp.f32 s2, s0
-; VFP2-NEXT:    vmov r0, s4
+; VFP2-NEXT:    vcmp.f32 s0, s4
+; VFP2-NEXT:    vmov r0, s2
 ; VFP2-NEXT:    it lt
 ; VFP2-NEXT:    movlt r0, #0
 ; VFP2-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1960,29 +1904,27 @@ define i8 @test_signed_i8_f16(half %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI21_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB21_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB21_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB21_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB21_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB21_3:
+; SOFT-NEXT:    ldr r1, .LCPI21_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB21_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB21_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    movs r4, #255
 ; SOFT-NEXT:  .LBB21_4:
-; SOFT-NEXT:    movs r0, #255
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -2040,29 +1982,27 @@ define i13 @test_signed_i13_f16(half %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI22_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB22_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB22_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB22_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB22_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB22_3:
+; SOFT-NEXT:    ldr r1, .LCPI22_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB22_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB22_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI22_1
 ; SOFT-NEXT:  .LBB22_4:
-; SOFT-NEXT:    ldr r0, .LCPI22_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -2122,29 +2062,27 @@ define i16 @test_signed_i16_f16(half %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI23_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB23_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB23_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB23_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB23_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB23_3:
+; SOFT-NEXT:    ldr r1, .LCPI23_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB23_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB23_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI23_1
 ; SOFT-NEXT:  .LBB23_4:
-; SOFT-NEXT:    ldr r0, .LCPI23_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -2204,29 +2142,27 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    ldr r1, .LCPI24_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r1, #0
-; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB24_3
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB24_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB24_4
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB24_2:
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB24_3:
+; SOFT-NEXT:    ldr r1, .LCPI24_0
 ; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB24_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB24_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    ldr r4, .LCPI24_1
 ; SOFT-NEXT:  .LBB24_4:
-; SOFT-NEXT:    ldr r0, .LCPI24_1
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    pop {r4, r5, r6, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.5:
@@ -2289,31 +2225,28 @@ define i32 @test_signed_i32_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI25_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2uiz
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB25_3
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB25_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB25_4
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB25_2:
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB25_3:
+; SOFT-NEXT:    ldr r1, .LCPI25_0
 ; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB25_2
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    beq .LBB25_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mvns r5, r4
 ; SOFT-NEXT:  .LBB25_4:
-; SOFT-NEXT:    mvns r0, r4
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -2350,45 +2283,39 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI26_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB26_5
-; SOFT-NEXT:  @ %bb.1:
+; SOFT-NEXT:    mov r7, r1
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_6
+; SOFT-NEXT:    bne .LBB26_2
+; SOFT-NEXT:  @ %bb.1:
+; SOFT-NEXT:    mov r7, r4
 ; SOFT-NEXT:  .LBB26_2:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB26_7
-; SOFT-NEXT:  .LBB26_3:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_8
+; SOFT-NEXT:    bne .LBB26_4
+; SOFT-NEXT:  @ %bb.3:
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB26_4:
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
-; SOFT-NEXT:  .LBB26_5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_2
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    ldr r1, .LCPI26_0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mvns r0, r5
+; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    bne .LBB26_6
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB26_6:
-; SOFT-NEXT:    mvns r0, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB26_3
-; SOFT-NEXT:  .LBB26_7:
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_4
+; SOFT-NEXT:    beq .LBB26_8
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    ldr r7, .LCPI26_1
 ; SOFT-NEXT:  .LBB26_8:
-; SOFT-NEXT:    ldr r1, .LCPI26_1
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -2465,48 +2392,43 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI27_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB27_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:  .LBB27_2:
-; SOFT-NEXT:    mvns r2, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r0, r2
-; SOFT-NEXT:    beq .LBB27_7
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    bne .LBB27_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    beq .LBB27_8
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB27_4:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB27_6
-; SOFT-NEXT:  .LBB27_5:
-; SOFT-NEXT:    mov r2, r1
+; SOFT-NEXT:    ldr r1, .LCPI27_0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r1, r4
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r1
+; SOFT-NEXT:    beq .LBB27_7
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    beq .LBB27_8
 ; SOFT-NEXT:  .LBB27_6:
-; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB27_7:
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB27_4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    bne .LBB27_6
 ; SOFT-NEXT:  .LBB27_8:
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB27_5
-; SOFT-NEXT:    b .LBB27_6
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI27_0:
@@ -2573,78 +2495,73 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB28_2
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB28_11
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB28_12
 ; SOFT-NEXT:  .LBB28_2:
-; SOFT-NEXT:    mvns r2, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r6, r2
 ; SOFT-NEXT:    bne .LBB28_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:  .LBB28_3:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:  .LBB28_4:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB28_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB28_6:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    ldr r1, .LCPI28_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r2
 ; SOFT-NEXT:    beq .LBB28_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    beq .LBB28_14
 ; SOFT-NEXT:  .LBB28_8:
-; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    beq .LBB28_15
 ; SOFT-NEXT:  .LBB28_9:
-; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB28_16
 ; SOFT-NEXT:  .LBB28_10:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB28_12
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB28_11:
-; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    bne .LBB28_2
 ; SOFT-NEXT:  .LBB28_12:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    beq .LBB28_3
+; SOFT-NEXT:    b .LBB28_4
 ; SOFT-NEXT:  .LBB28_13:
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    bne .LBB28_8
 ; SOFT-NEXT:  .LBB28_14:
-; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB28_9
 ; SOFT-NEXT:  .LBB28_15:
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB28_10
 ; SOFT-NEXT:  .LBB28_16:
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB28_11
-; SOFT-NEXT:    b .LBB28_12
+; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI28_0:
@@ -2719,77 +2636,74 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #4
-; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    mov r7, r1
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB29_2
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB29_11
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    beq .LBB29_12
 ; SOFT-NEXT:  .LBB29_2:
-; SOFT-NEXT:    mvns r6, r6
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bne .LBB29_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:  .LBB29_3:
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:  .LBB29_4:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    bne .LBB29_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r5, r7
 ; SOFT-NEXT:  .LBB29_6:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bne .LBB29_8
+; SOFT-NEXT:    ldr r1, .LCPI29_0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    bl __aeabi_fcmpgt
+; SOFT-NEXT:    mvns r3, r6
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    beq .LBB29_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    beq .LBB29_14
 ; SOFT-NEXT:  .LBB29_8:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB29_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:  .LBB29_10:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    mov r2, r3
 ; SOFT-NEXT:    beq .LBB29_15
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:  .LBB29_9:
 ; SOFT-NEXT:    beq .LBB29_16
+; SOFT-NEXT:  .LBB29_10:
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB29_11:
+; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    bne .LBB29_2
 ; SOFT-NEXT:  .LBB29_12:
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB29_14
+; SOFT-NEXT:    mov r2, r7
+; SOFT-NEXT:    beq .LBB29_3
+; SOFT-NEXT:    b .LBB29_4
 ; SOFT-NEXT:  .LBB29_13:
-; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r3
+; SOFT-NEXT:    bne .LBB29_8
 ; SOFT-NEXT:  .LBB29_14:
-; SOFT-NEXT:    mov r0, r3
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    add sp, #4
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    bne .LBB29_9
 ; SOFT-NEXT:  .LBB29_15:
-; SOFT-NEXT:    mov r7, r2
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    bne .LBB29_10
 ; SOFT-NEXT:  .LBB29_16:
-; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB29_13
-; SOFT-NEXT:    b .LBB29_14
+; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI29_0:
diff --git a/llvm/test/CodeGen/ARM/funnel-shift-rot.ll b/llvm/test/CodeGen/ARM/funnel-shift-rot.ll
index de5bd2a7040b9..a1b6847d623d0 100644
--- a/llvm/test/CodeGen/ARM/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/ARM/funnel-shift-rot.ll
@@ -71,12 +71,12 @@ define i64 @rotl_i64(i64 %x, i64 %z) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    ands r3, r2, #32
-; CHECK-NEXT:    and r12, r2, #31
+; CHECK-NEXT:    tst r2, #32
 ; CHECK-NEXT:    mov r3, r0
-; CHECK-NEXT:    mov r4, #31
+; CHECK-NEXT:    and r12, r2, #31
 ; CHECK-NEXT:    movne r3, r1
 ; CHECK-NEXT:    movne r1, r0
+; CHECK-NEXT:    mov r4, #31
 ; CHECK-NEXT:    bic r2, r4, r2
 ; CHECK-NEXT:    lsl lr, r3, r12
 ; CHECK-NEXT:    lsr r0, r1, #1
@@ -206,7 +206,7 @@ define i32 @rotr_i32(i32 %x, i32 %z) {
 define i64 @rotr_i64(i64 %x, i64 %z) {
 ; CHECK-LABEL: rotr_i64:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ands r3, r2, #32
+; CHECK-NEXT:    tst r2, #32
 ; CHECK-NEXT:    mov r3, r1
 ; CHECK-NEXT:    moveq r3, r0
 ; CHECK-NEXT:    moveq r0, r1
diff --git a/llvm/test/CodeGen/ARM/funnel-shift.ll b/llvm/test/CodeGen/ARM/funnel-shift.ll
index 5a7c4384428e1..191155ae30f3e 100644
--- a/llvm/test/CodeGen/ARM/funnel-shift.ll
+++ b/llvm/test/CodeGen/ARM/funnel-shift.ll
@@ -47,69 +47,67 @@ declare i37 @llvm.fshl.i37(i37, i37, i37)
 define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) {
 ; SCALAR-LABEL: fshl_i37:
 ; SCALAR:       @ %bb.0:
-; SCALAR-NEXT:    .save {r4, r5, r6, r7, r8, r9, r11, lr}
-; SCALAR-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; SCALAR-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; SCALAR-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; SCALAR-NEXT:    mov r8, r0
-; SCALAR-NEXT:    ldr r0, [sp, #36]
+; SCALAR-NEXT:    ldr r0, [sp, #28]
 ; SCALAR-NEXT:    mov r4, r1
-; SCALAR-NEXT:    mov r6, r3
+; SCALAR-NEXT:    mov r5, r3
 ; SCALAR-NEXT:    and r1, r0, #31
-; SCALAR-NEXT:    ldr r0, [sp, #32]
-; SCALAR-NEXT:    mov r9, r2
+; SCALAR-NEXT:    ldr r0, [sp, #24]
+; SCALAR-NEXT:    mov r6, r2
 ; SCALAR-NEXT:    mov r2, #37
 ; SCALAR-NEXT:    mov r3, #0
 ; SCALAR-NEXT:    bl __aeabi_uldivmod
-; SCALAR-NEXT:    lsl r1, r6, #27
-; SCALAR-NEXT:    ands r0, r2, #32
-; SCALAR-NEXT:    orr r1, r1, r9, lsr #5
-; SCALAR-NEXT:    mov r3, r8
-; SCALAR-NEXT:    and r6, r2, #31
+; SCALAR-NEXT:    lsl r0, r5, #27
+; SCALAR-NEXT:    tst r2, #32
+; SCALAR-NEXT:    orr r0, r0, r6, lsr #5
+; SCALAR-NEXT:    mov r1, r8
+; SCALAR-NEXT:    and r3, r2, #31
 ; SCALAR-NEXT:    mov r7, #31
-; SCALAR-NEXT:    movne r3, r1
-; SCALAR-NEXT:    cmp r0, #0
-; SCALAR-NEXT:    lslne r1, r9, #27
+; SCALAR-NEXT:    movne r1, r0
+; SCALAR-NEXT:    lslne r0, r6, #27
 ; SCALAR-NEXT:    bic r2, r7, r2
+; SCALAR-NEXT:    lsl r5, r1, r3
+; SCALAR-NEXT:    lsr r0, r0, #1
 ; SCALAR-NEXT:    movne r4, r8
-; SCALAR-NEXT:    lsl r5, r3, r6
-; SCALAR-NEXT:    lsr r0, r1, #1
-; SCALAR-NEXT:    lsl r1, r4, r6
-; SCALAR-NEXT:    lsr r3, r3, #1
+; SCALAR-NEXT:    lsr r1, r1, #1
+; SCALAR-NEXT:    lsl r3, r4, r3
 ; SCALAR-NEXT:    orr r0, r5, r0, lsr r2
-; SCALAR-NEXT:    orr r1, r1, r3, lsr r2
-; SCALAR-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+; SCALAR-NEXT:    orr r1, r3, r1, lsr r2
+; SCALAR-NEXT:    pop {r4, r5, r6, r7, r8, pc}
 ;
 ; NEON-LABEL: fshl_i37:
 ; NEON:       @ %bb.0:
-; NEON-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; NEON-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; NEON-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; NEON-NEXT:    push {r4, r5, r6, r7, r8, lr}
 ; NEON-NEXT:    mov r4, r1
 ; NEON-NEXT:    ldr r1, [sp, #28]
-; NEON-NEXT:    mov r6, r0
+; NEON-NEXT:    mov r8, r0
 ; NEON-NEXT:    ldr r0, [sp, #24]
 ; NEON-NEXT:    and r1, r1, #31
 ; NEON-NEXT:    mov r5, r3
-; NEON-NEXT:    mov r7, r2
+; NEON-NEXT:    mov r6, r2
 ; NEON-NEXT:    mov r2, #37
 ; NEON-NEXT:    mov r3, #0
 ; NEON-NEXT:    bl __aeabi_uldivmod
-; NEON-NEXT:    mov r0, #31
-; NEON-NEXT:    bic r1, r0, r2
 ; NEON-NEXT:    lsl r0, r5, #27
-; NEON-NEXT:    ands r12, r2, #32
-; NEON-NEXT:    orr r0, r0, r7, lsr #5
-; NEON-NEXT:    mov r5, r6
-; NEON-NEXT:    and r2, r2, #31
-; NEON-NEXT:    movne r5, r0
-; NEON-NEXT:    lslne r0, r7, #27
-; NEON-NEXT:    cmp r12, #0
-; NEON-NEXT:    lsl r3, r5, r2
+; NEON-NEXT:    tst r2, #32
+; NEON-NEXT:    orr r0, r0, r6, lsr #5
+; NEON-NEXT:    mov r1, r8
+; NEON-NEXT:    and r3, r2, #31
+; NEON-NEXT:    mov r7, #31
+; NEON-NEXT:    movne r1, r0
+; NEON-NEXT:    lslne r0, r6, #27
+; NEON-NEXT:    bic r2, r7, r2
+; NEON-NEXT:    lsl r5, r1, r3
 ; NEON-NEXT:    lsr r0, r0, #1
-; NEON-NEXT:    movne r4, r6
-; NEON-NEXT:    orr r0, r3, r0, lsr r1
-; NEON-NEXT:    lsr r3, r5, #1
-; NEON-NEXT:    lsl r2, r4, r2
-; NEON-NEXT:    orr r1, r2, r3, lsr r1
-; NEON-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; NEON-NEXT:    movne r4, r8
+; NEON-NEXT:    lsr r1, r1, #1
+; NEON-NEXT:    lsl r3, r4, r3
+; NEON-NEXT:    orr r0, r5, r0, lsr r2
+; NEON-NEXT:    orr r1, r3, r1, lsr r2
+; NEON-NEXT:    pop {r4, r5, r6, r7, r8, pc}
   %f = call i37 @llvm.fshl.i37(i37 %x, i37 %y, i37 %z)
   ret i37 %f
 }
@@ -237,71 +235,69 @@ declare i37 @llvm.fshr.i37(i37, i37, i37)
 define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) {
 ; SCALAR-LABEL: fshr_i37:
 ; SCALAR:       @ %bb.0:
-; SCALAR-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; SCALAR-NEXT:    push {r4, r5, r6, r7, r8, lr}
-; SCALAR-NEXT:    mov r8, r0
+; SCALAR-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; SCALAR-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; SCALAR-NEXT:    mov r5, r0
 ; SCALAR-NEXT:    ldr r0, [sp, #28]
 ; SCALAR-NEXT:    mov r4, r1
-; SCALAR-NEXT:    mov r5, r3
+; SCALAR-NEXT:    mov r6, r3
 ; SCALAR-NEXT:    and r1, r0, #31
 ; SCALAR-NEXT:    ldr r0, [sp, #24]
 ; SCALAR-NEXT:    mov r7, r2
 ; SCALAR-NEXT:    mov r2, #37
 ; SCALAR-NEXT:    mov r3, #0
 ; SCALAR-NEXT:    bl __aeabi_uldivmod
-; SCALAR-NEXT:    lsl r3, r5, #27
 ; SCALAR-NEXT:    add r0, r2, #27
-; SCALAR-NEXT:    orr r3, r3, r7, lsr #5
-; SCALAR-NEXT:    ands r2, r0, #32
-; SCALAR-NEXT:    mov r5, r8
+; SCALAR-NEXT:    lsl r2, r6, #27
+; SCALAR-NEXT:    orr r2, r2, r7, lsr #5
 ; SCALAR-NEXT:    mov r1, #31
-; SCALAR-NEXT:    moveq r5, r3
-; SCALAR-NEXT:    lsleq r3, r7, #27
-; SCALAR-NEXT:    cmp r2, #0
+; SCALAR-NEXT:    tst r0, #32
+; SCALAR-NEXT:    mov r3, r5
+; SCALAR-NEXT:    moveq r3, r2
+; SCALAR-NEXT:    lsleq r2, r7, #27
 ; SCALAR-NEXT:    bic r1, r1, r0
-; SCALAR-NEXT:    moveq r4, r8
-; SCALAR-NEXT:    lsl r6, r5, #1
 ; SCALAR-NEXT:    and r7, r0, #31
-; SCALAR-NEXT:    lsl r2, r4, #1
+; SCALAR-NEXT:    lsl r6, r3, #1
+; SCALAR-NEXT:    moveq r4, r5
 ; SCALAR-NEXT:    lsl r6, r6, r1
+; SCALAR-NEXT:    orr r0, r6, r2, lsr r7
+; SCALAR-NEXT:    lsl r2, r4, #1
 ; SCALAR-NEXT:    lsl r1, r2, r1
-; SCALAR-NEXT:    orr r0, r6, r3, lsr r7
-; SCALAR-NEXT:    orr r1, r1, r5, lsr r7
-; SCALAR-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; SCALAR-NEXT:    orr r1, r1, r3, lsr r7
+; SCALAR-NEXT:    pop {r4, r5, r6, r7, r11, pc}
 ;
 ; NEON-LABEL: fshr_i37:
 ; NEON:       @ %bb.0:
-; NEON-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; NEON-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; NEON-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; NEON-NEXT:    push {r4, r5, r6, r7, r11, lr}
 ; NEON-NEXT:    mov r4, r1
 ; NEON-NEXT:    ldr r1, [sp, #28]
-; NEON-NEXT:    mov r8, r0
+; NEON-NEXT:    mov r5, r0
 ; NEON-NEXT:    ldr r0, [sp, #24]
 ; NEON-NEXT:    and r1, r1, #31
-; NEON-NEXT:    mov r5, r3
+; NEON-NEXT:    mov r6, r3
 ; NEON-NEXT:    mov r7, r2
 ; NEON-NEXT:    mov r2, #37
 ; NEON-NEXT:    mov r3, #0
 ; NEON-NEXT:    bl __aeabi_uldivmod
-; NEON-NEXT:    lsl r3, r5, #27
 ; NEON-NEXT:    add r0, r2, #27
-; NEON-NEXT:    orr r3, r3, r7, lsr #5
-; NEON-NEXT:    ands r2, r0, #32
-; NEON-NEXT:    mov r5, r8
+; NEON-NEXT:    lsl r2, r6, #27
+; NEON-NEXT:    orr r2, r2, r7, lsr #5
 ; NEON-NEXT:    mov r1, #31
-; NEON-NEXT:    moveq r5, r3
-; NEON-NEXT:    lsleq r3, r7, #27
-; NEON-NEXT:    cmp r2, #0
+; NEON-NEXT:    tst r0, #32
+; NEON-NEXT:    mov r3, r5
+; NEON-NEXT:    moveq r3, r2
+; NEON-NEXT:    lsleq r2, r7, #27
 ; NEON-NEXT:    bic r1, r1, r0
-; NEON-NEXT:    moveq r4, r8
-; NEON-NEXT:    lsl r6, r5, #1
 ; NEON-NEXT:    and r7, r0, #31
-; NEON-NEXT:    lsl r2, r4, #1
+; NEON-NEXT:    lsl r6, r3, #1
+; NEON-NEXT:    moveq r4, r5
 ; NEON-NEXT:    lsl r6, r6, r1
+; NEON-NEXT:    orr r0, r6, r2, lsr r7
+; NEON-NEXT:    lsl r2, r4, #1
 ; NEON-NEXT:    lsl r1, r2, r1
-; NEON-NEXT:    orr r0, r6, r3, lsr r7
-; NEON-NEXT:    orr r1, r1, r5, lsr r7
-; NEON-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+; NEON-NEXT:    orr r1, r1, r3, lsr r7
+; NEON-NEXT:    pop {r4, r5, r6, r7, r11, pc}
   %f = call i37 @llvm.fshr.i37(i37 %x, i37 %y, i37 %z)
   ret i37 %f
 }
diff --git a/llvm/test/CodeGen/ARM/ifcvt1.ll b/llvm/test/CodeGen/ARM/ifcvt1.ll
index d419cbc48fc48..6d59869bc102e 100644
--- a/llvm/test/CodeGen/ARM/ifcvt1.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt1.ll
@@ -13,10 +13,10 @@ define i32 @t1(i32 %a, i32 %b) {
 ;
 ; SWIFT-LABEL: t1:
 ; SWIFT:       @ %bb.0: @ %common.ret
-; SWIFT-NEXT:    mov r2, #1
 ; SWIFT-NEXT:    cmp r0, #0
-; SWIFT-NEXT:    mvneq r2, #0
-; SWIFT-NEXT:    add r0, r1, r2
+; SWIFT-NEXT:    mov r0, #1
+; SWIFT-NEXT:    mvneq r0, #0
+; SWIFT-NEXT:    add r0, r1, r0
 ; SWIFT-NEXT:    bx lr
 	%tmp2 = icmp eq i32 %a, 0
 	br i1 %tmp2, label %cond_false, label %cond_true
diff --git a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
index 4003af5d44be8..dd33b09fe8300 100644
--- a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
@@ -494,14 +494,14 @@ define <4 x float> @fminnumv432_intrinsic(<4 x float> %x, <4 x float> %y) {
 ; ARMV7-NEXT:    vld1.64 {d0, d1}, [r12]
 ; ARMV7-NEXT:    vmov d3, r2, r3
 ; ARMV7-NEXT:    vmov d2, r0, r1
-; ARMV7-NEXT:    vcmp.f32 s7, s3
-; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vcmp.f32 s6, s2
-; ARMV7-NEXT:    vmovlt.f32 s3, s7
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vcmp.f32 s7, s3
 ; ARMV7-NEXT:    vmovlt.f32 s2, s6
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vmovlt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vcmp.f32 s4, s0
 ; ARMV7-NEXT:    vmovlt.f32 s1, s5
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
@@ -676,14 +676,14 @@ define <4 x float> @fmaxnumv432_intrinsic(<4 x float> %x, <4 x float> %y) {
 ; ARMV7-NEXT:    vld1.64 {d0, d1}, [r12]
 ; ARMV7-NEXT:    vmov d3, r2, r3
 ; ARMV7-NEXT:    vmov d2, r0, r1
-; ARMV7-NEXT:    vcmp.f32 s7, s3
-; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vcmp.f32 s6, s2
-; ARMV7-NEXT:    vmovgt.f32 s3, s7
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vcmp.f32 s7, s3
 ; ARMV7-NEXT:    vmovgt.f32 s2, s6
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vcmp.f32 s4, s0
 ; ARMV7-NEXT:    vmovgt.f32 s1, s5
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
@@ -760,16 +760,16 @@ define <4 x float> @fmaxnumv432_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-NEXT:    vmov d3, r2, r3
 ; ARMV7-NEXT:    vldr s0, .LCPI21_0
 ; ARMV7-NEXT:    vmov d2, r0, r1
-; ARMV7-NEXT:    vcmp.f32 s7, #0
-; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vmov.f32 s3, s0
 ; ARMV7-NEXT:    vcmp.f32 s6, #0
-; ARMV7-NEXT:    vmovgt.f32 s3, s7
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vmov.f32 s2, s0
-; ARMV7-NEXT:    vcmp.f32 s5, #0
+; ARMV7-NEXT:    vcmp.f32 s7, #0
 ; ARMV7-NEXT:    vmovgt.f32 s2, s6
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s3, s0
+; ARMV7-NEXT:    vcmp.f32 s5, #0
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vmov.f32 s1, s0
 ; ARMV7-NEXT:    vcmp.f32 s4, #0
 ; ARMV7-NEXT:    vmovgt.f32 s1, s5
@@ -812,18 +812,18 @@ define <4 x float> @fmaxnumv432_minus_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-NEXT:    vldr s0, .LCPI22_0
 ; ARMV7-NEXT:    vmov d3, r2, r3
 ; ARMV7-NEXT:    vmov d2, r0, r1
-; ARMV7-NEXT:    vcmp.f32 s7, s0
-; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vmov.f32 s3, s0
 ; ARMV7-NEXT:    vcmp.f32 s6, s0
-; ARMV7-NEXT:    vmovgt.f32 s3, s7
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s7, s0
 ; ARMV7-NEXT:    vmov.f32 s2, s0
-; ARMV7-NEXT:    vcmp.f32 s5, s0
 ; ARMV7-NEXT:    vmovgt.f32 s2, s6
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV7-NEXT:    vmov.f32 s1, s0
+; ARMV7-NEXT:    vcmp.f32 s5, s0
+; ARMV7-NEXT:    vmov.f32 s3, s0
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vcmp.f32 s4, s0
+; ARMV7-NEXT:    vmov.f32 s1, s0
 ; ARMV7-NEXT:    vmovgt.f32 s1, s5
 ; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV7-NEXT:    vmovgt.f32 s0, s4
@@ -933,8 +933,8 @@ define <2 x double> @fminnumv264_intrinsic(<2 x double> %x, <2 x double> %y) {
 ; ARMV8M-NEXT:    vselgt.f64 d0, d0, d2
 ; ARMV8M-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV8M-NEXT:    vmov r0, r1, d0
-; ARMV8M-NEXT:    vselgt.f64 d1, d1, d3
-; ARMV8M-NEXT:    vmov r2, r3, d1
+; ARMV8M-NEXT:    vselgt.f64 d0, d1, d3
+; ARMV8M-NEXT:    vmov r2, r3, d0
 ; ARMV8M-NEXT:    bx lr
   %a = call nnan <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> %y)
   ret <2 x double> %a
@@ -981,8 +981,8 @@ define <2 x double> @fminnumv264_nsz_intrinsic(<2 x double> %x, <2 x double> %y)
 ; ARMV8M-NEXT:    vselgt.f64 d0, d0, d2
 ; ARMV8M-NEXT:    vmrs APSR_nzcv, fpscr
 ; ARMV8M-NEXT:    vmov r0, r1, d0
-; ARMV8M-NEXT:    vselgt.f64 d1, d1, d3
-; ARMV8M-NEXT:    vmov r2, r3, d1
+; ARMV8M-NEXT:    vselgt.f64 d0, d1, d3
+; ARMV8M-NEXT:    vmov r2, r3, d0
 ; ARMV8M-NEXT:    bx lr
   %a = call nnan nsz <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> %y)
   ret <2 x double> %a
@@ -1225,26 +1225,26 @@ define <2 x double> @fmaxnumv264_zero_intrinsic(<2 x double> %x) {
 ; ARMV8M-LABEL: fmaxnumv264_zero_intrinsic:
 ; ARMV8M:       @ %bb.0:
 ; ARMV8M-NEXT:    vmov d2, r0, r1
-; ARMV8M-NEXT:    vldr d1, .LCPI30_1
+; ARMV8M-NEXT:    vldr d0, .LCPI30_0
 ; ARMV8M-NEXT:    vcmp.f64 d2, #0
 ; ARMV8M-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV8M-NEXT:    vmov d3, r2, r3
-; ARMV8M-NEXT:    vcmp.f64 d3, d1
-; ARMV8M-NEXT:    vldr d0, .LCPI30_0
-; ARMV8M-NEXT:    vselgt.f64 d0, d2, d0
+; ARMV8M-NEXT:    vmov d1, r2, r3
+; ARMV8M-NEXT:    vcmp.f64 d1, d0
+; ARMV8M-NEXT:    vldr d3, .LCPI30_1
+; ARMV8M-NEXT:    vselgt.f64 d2, d2, d3
 ; ARMV8M-NEXT:    vmrs APSR_nzcv, fpscr
-; ARMV8M-NEXT:    vmov r0, r1, d0
-; ARMV8M-NEXT:    vselgt.f64 d1, d3, d1
-; ARMV8M-NEXT:    vmov r2, r3, d1
+; ARMV8M-NEXT:    vmov r0, r1, d2
+; ARMV8M-NEXT:    vselgt.f64 d0, d1, d0
+; ARMV8M-NEXT:    vmov r2, r3, d0
 ; ARMV8M-NEXT:    bx lr
 ; ARMV8M-NEXT:    .p2align 3
 ; ARMV8M-NEXT:  @ %bb.1:
 ; ARMV8M-NEXT:  .LCPI30_0:
-; ARMV8M-NEXT:    .long 0 @ double 0
-; ARMV8M-NEXT:    .long 0
-; ARMV8M-NEXT:  .LCPI30_1:
 ; ARMV8M-NEXT:    .long 0 @ double -0
 ; ARMV8M-NEXT:    .long 2147483648
+; ARMV8M-NEXT:  .LCPI30_1:
+; ARMV8M-NEXT:    .long 0 @ double 0
+; ARMV8M-NEXT:    .long 0
   %a = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> %x, <2 x double><double 0.0, double -0.0>)
   ret <2 x double> %a
 }
diff --git a/llvm/test/CodeGen/ARM/neon_vabd.ll b/llvm/test/CodeGen/ARM/neon_vabd.ll
index 8a268d46304cf..ffc72b242f829 100644
--- a/llvm/test/CodeGen/ARM/neon_vabd.ll
+++ b/llvm/test/CodeGen/ARM/neon_vabd.ll
@@ -144,25 +144,25 @@ define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    vmov r0, r12, d0
+; CHECK-NEXT:    vmov r0, r1, d1
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    vmov r2, r3, d2
-; CHECK-NEXT:    vmov r1, lr, d1
-; CHECK-NEXT:    vmov r4, r5, d3
+; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    vmov r12, lr, d0
+; CHECK-NEXT:    vmov r4, r5, d2
 ; CHECK-NEXT:    vsub.i64 q8, q0, q1
 ; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs r0, r3, r12
+; CHECK-NEXT:    sbcs r0, r3, r1
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    subs r1, r4, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    subs r1, r4, r12
 ; CHECK-NEXT:    sbcs r1, r5, lr
+; CHECK-NEXT:    vdup.32 d19, r0
 ; CHECK-NEXT:    movwlt r6, #1
 ; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    mvnne r6, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vdup.32 d19, r6
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d18, r0
+; CHECK-NEXT:    vdup.32 d18, r6
 ; CHECK-NEXT:    veor q8, q8, q9
 ; CHECK-NEXT:    vsub.i64 q0, q9, q8
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
@@ -475,25 +475,25 @@ define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    vmov r0, r12, d0
+; CHECK-NEXT:    vmov r0, r1, d1
 ; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    vmov r2, r3, d2
-; CHECK-NEXT:    vmov r1, lr, d1
-; CHECK-NEXT:    vmov r4, r5, d3
+; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    vmov r12, lr, d0
+; CHECK-NEXT:    vmov r4, r5, d2
 ; CHECK-NEXT:    vsub.i64 q8, q0, q1
 ; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs r0, r3, r12
+; CHECK-NEXT:    sbcs r0, r3, r1
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    subs r1, r4, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    subs r1, r4, r12
 ; CHECK-NEXT:    sbcs r1, r5, lr
+; CHECK-NEXT:    vdup.32 d19, r0
 ; CHECK-NEXT:    movwlt r6, #1
 ; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    mvnne r6, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    vdup.32 d19, r6
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d18, r0
+; CHECK-NEXT:    vdup.32 d18, r6
 ; CHECK-NEXT:    veor q8, q8, q9
 ; CHECK-NEXT:    vsub.i64 q0, q9, q8
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
diff --git a/llvm/test/CodeGen/ARM/overflow-intrinsic-optimizations.ll b/llvm/test/CodeGen/ARM/overflow-intrinsic-optimizations.ll
index 198927d1da3a4..2631189979e37 100644
--- a/llvm/test/CodeGen/ARM/overflow-intrinsic-optimizations.ll
+++ b/llvm/test/CodeGen/ARM/overflow-intrinsic-optimizations.ll
@@ -164,9 +164,9 @@ cont2:
 
 define void @extern_loop(i32 %n) local_unnamed_addr #0 {
 ; Do not replace the compare around the clobbering call.
-; CHECK: add {{r[0-9]+}}, {{r[0-9]+}}, #1
-; CHECK-NEXT: bl external_fn
-; CHECK: cmp
+; CHECK: bl external_fn
+; CHECK-NEXT: adds
+; CHECK-NEXT: bvs
 entry:
   %0 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %n, i32 1)
   %1 = extractvalue { i32, i1 } %0, 1
diff --git a/llvm/test/CodeGen/ARM/sadd_sat.ll b/llvm/test/CodeGen/ARM/sadd_sat.ll
index 0060b4458081b..b8f7a2daaeaba 100644
--- a/llvm/test/CodeGen/ARM/sadd_sat.ll
+++ b/llvm/test/CodeGen/ARM/sadd_sat.ll
@@ -72,22 +72,21 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T16-NEXT:    adcs r3, r4
 ; CHECK-T16-NEXT:    eors r4, r3
 ; CHECK-T16-NEXT:    bics r4, r1
-; CHECK-T16-NEXT:    asrs r1, r3, #31
+; CHECK-T16-NEXT:    asrs r0, r3, #31
+; CHECK-T16-NEXT:    movs r1, #1
+; CHECK-T16-NEXT:    lsls r1, r1, #31
+; CHECK-T16-NEXT:    eors r1, r0
 ; CHECK-T16-NEXT:    cmp r4, #0
-; CHECK-T16-NEXT:    mov r0, r1
-; CHECK-T16-NEXT:    bmi .LBB1_2
+; CHECK-T16-NEXT:    bpl .LBB1_3
 ; CHECK-T16-NEXT:  @ %bb.1:
-; CHECK-T16-NEXT:    mov r0, r2
+; CHECK-T16-NEXT:    bpl .LBB1_4
 ; CHECK-T16-NEXT:  .LBB1_2:
-; CHECK-T16-NEXT:    cmp r4, #0
-; CHECK-T16-NEXT:    bmi .LBB1_4
-; CHECK-T16-NEXT:  @ %bb.3:
-; CHECK-T16-NEXT:    mov r1, r3
 ; CHECK-T16-NEXT:    pop {r4, pc}
+; CHECK-T16-NEXT:  .LBB1_3:
+; CHECK-T16-NEXT:    mov r0, r2
+; CHECK-T16-NEXT:    bmi .LBB1_2
 ; CHECK-T16-NEXT:  .LBB1_4:
-; CHECK-T16-NEXT:    movs r2, #1
-; CHECK-T16-NEXT:    lsls r2, r2, #31
-; CHECK-T16-NEXT:    eors r1, r2
+; CHECK-T16-NEXT:    mov r1, r3
 ; CHECK-T16-NEXT:    pop {r4, pc}
 ;
 ; CHECK-T2-LABEL: func2:
@@ -128,23 +127,22 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T15TE-NEXT:    adcs r3, r4
 ; CHECK-T15TE-NEXT:    eors r4, r3
 ; CHECK-T15TE-NEXT:    bics r4, r1
-; CHECK-T15TE-NEXT:    asrs r1, r3, #31
+; CHECK-T15TE-NEXT:    asrs r0, r3, #31
+; CHECK-T15TE-NEXT:    movs r1, #1
+; CHECK-T15TE-NEXT:    lsls r1, r1, #31
+; CHECK-T15TE-NEXT:    eors r1, r0
 ; CHECK-T15TE-NEXT:    cmp r4, #0
-; CHECK-T15TE-NEXT:    mov r12, r1
-; CHECK-T15TE-NEXT:    mov r0, r12
-; CHECK-T15TE-NEXT:    bmi .LBB1_2
+; CHECK-T15TE-NEXT:    bpl .LBB1_3
 ; CHECK-T15TE-NEXT:  @ %bb.1:
-; CHECK-T15TE-NEXT:    movs r0, r2
+; CHECK-T15TE-NEXT:    bpl .LBB1_4
 ; CHECK-T15TE-NEXT:  .LBB1_2:
-; CHECK-T15TE-NEXT:    cmp r4, #0
-; CHECK-T15TE-NEXT:    bmi .LBB1_4
-; CHECK-T15TE-NEXT:  @ %bb.3:
-; CHECK-T15TE-NEXT:    movs r1, r3
 ; CHECK-T15TE-NEXT:    pop {r4, pc}
+; CHECK-T15TE-NEXT:  .LBB1_3:
+; CHECK-T15TE-NEXT:    mov r12, r2
+; CHECK-T15TE-NEXT:    mov r0, r12
+; CHECK-T15TE-NEXT:    bmi .LBB1_2
 ; CHECK-T15TE-NEXT:  .LBB1_4:
-; CHECK-T15TE-NEXT:    movs r2, #1
-; CHECK-T15TE-NEXT:    lsls r2, r2, #31
-; CHECK-T15TE-NEXT:    eors r1, r2
+; CHECK-T15TE-NEXT:    movs r1, r3
 ; CHECK-T15TE-NEXT:    pop {r4, pc}
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/ARM/sadd_sat_plus.ll b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
index 859aedc7a3f01..0ddb64fc3f2d1 100644
--- a/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
@@ -63,22 +63,22 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-NEXT:    adcs r3, r1
 ; CHECK-T1-NEXT:    eors r1, r3
 ; CHECK-T1-NEXT:    bics r1, r2
-; CHECK-T1-NEXT:    asrs r2, r3, #31
+; CHECK-T1-NEXT:    asrs r0, r3, #31
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    eors r2, r0
 ; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    bpl .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    bpl .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r1, r3
+; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    bmi .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    movs r1, #1
-; CHECK-T1-NEXT:    lsls r1, r1, #31
-; CHECK-T1-NEXT:    eors r2, r1
+; CHECK-T1-NEXT:    mov r2, r3
 ; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, pc}
 ;
diff --git a/llvm/test/CodeGen/ARM/select.ll b/llvm/test/CodeGen/ARM/select.ll
index 496a6c0f5acbb..48d6ee925d939 100644
--- a/llvm/test/CodeGen/ARM/select.ll
+++ b/llvm/test/CodeGen/ARM/select.ll
@@ -320,11 +320,11 @@ define float @f10(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-VFP-LABEL: f10:
 ; CHECK-VFP:       @ %bb.0:
-; CHECK-VFP-NEXT:    vmov.f32 s2, #1.000000e+00
-; CHECK-VFP-NEXT:    vldr s0, .LCPI9_0
+; CHECK-VFP-NEXT:    vmov.f32 s0, #1.000000e+00
+; CHECK-VFP-NEXT:    vldr s2, .LCPI9_0
 ; CHECK-VFP-NEXT:    cmp r0, r1
-; CHECK-VFP-NEXT:    vmoveq.f32 s0, s2
-; CHECK-VFP-NEXT:    vmov r0, s0
+; CHECK-VFP-NEXT:    vmoveq.f32 s2, s0
+; CHECK-VFP-NEXT:    vmov r0, s2
 ; CHECK-VFP-NEXT:    bx lr
 ; CHECK-VFP-NEXT:    .p2align 2
 ; CHECK-VFP-NEXT:  @ %bb.1:
@@ -333,12 +333,12 @@ define float @f10(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-NEON-LABEL: f10:
 ; CHECK-NEON:       @ %bb.0:
-; CHECK-NEON-NEXT:    vldr s0, LCPI9_0
-; CHECK-NEON-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-NEON-NEXT:    vldr s2, LCPI9_0
+; CHECK-NEON-NEXT:    vmov.f32 s0, #1.000000e+00
 ; CHECK-NEON-NEXT:    cmp r0, r1
 ; CHECK-NEON-NEXT:    it eq
-; CHECK-NEON-NEXT:    vmoveq.f32 s0, s2
-; CHECK-NEON-NEXT:    vmov r0, s0
+; CHECK-NEON-NEXT:    vmoveq.f32 s2, s0
+; CHECK-NEON-NEXT:    vmov r0, s2
 ; CHECK-NEON-NEXT:    bx lr
 ; CHECK-NEON-NEXT:    .p2align 2
 ; CHECK-NEON-NEXT:  @ %bb.1:
@@ -364,11 +364,11 @@ define float @f11(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-VFP-LABEL: f11:
 ; CHECK-VFP:       @ %bb.0:
-; CHECK-VFP-NEXT:    vmov.f32 s2, #-1.000000e+00
-; CHECK-VFP-NEXT:    vldr s0, .LCPI10_0
+; CHECK-VFP-NEXT:    vmov.f32 s0, #-1.000000e+00
+; CHECK-VFP-NEXT:    vldr s2, .LCPI10_0
 ; CHECK-VFP-NEXT:    cmp r0, r1
-; CHECK-VFP-NEXT:    vmoveq.f32 s0, s2
-; CHECK-VFP-NEXT:    vmov r0, s0
+; CHECK-VFP-NEXT:    vmoveq.f32 s2, s0
+; CHECK-VFP-NEXT:    vmov r0, s2
 ; CHECK-VFP-NEXT:    bx lr
 ; CHECK-VFP-NEXT:    .p2align 2
 ; CHECK-VFP-NEXT:  @ %bb.1:
@@ -377,12 +377,12 @@ define float @f11(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-NEON-LABEL: f11:
 ; CHECK-NEON:       @ %bb.0:
-; CHECK-NEON-NEXT:    vldr s0, LCPI10_0
-; CHECK-NEON-NEXT:    vmov.f32 s2, #-1.000000e+00
+; CHECK-NEON-NEXT:    vldr s2, LCPI10_0
+; CHECK-NEON-NEXT:    vmov.f32 s0, #-1.000000e+00
 ; CHECK-NEON-NEXT:    cmp r0, r1
 ; CHECK-NEON-NEXT:    it eq
-; CHECK-NEON-NEXT:    vmoveq.f32 s0, s2
-; CHECK-NEON-NEXT:    vmov r0, s0
+; CHECK-NEON-NEXT:    vmoveq.f32 s2, s0
+; CHECK-NEON-NEXT:    vmov r0, s2
 ; CHECK-NEON-NEXT:    bx lr
 ; CHECK-NEON-NEXT:    .p2align 2
 ; CHECK-NEON-NEXT:  @ %bb.1:
@@ -406,11 +406,11 @@ define float @f12(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-VFP-LABEL: f12:
 ; CHECK-VFP:       @ %bb.0:
-; CHECK-VFP-NEXT:    vmov.f32 s2, #1.000000e+00
-; CHECK-VFP-NEXT:    vldr s0, .LCPI11_0
+; CHECK-VFP-NEXT:    vmov.f32 s0, #1.000000e+00
+; CHECK-VFP-NEXT:    vldr s2, .LCPI11_0
 ; CHECK-VFP-NEXT:    cmp r0, r1
-; CHECK-VFP-NEXT:    vmoveq.f32 s0, s2
-; CHECK-VFP-NEXT:    vmov r0, s0
+; CHECK-VFP-NEXT:    vmoveq.f32 s2, s0
+; CHECK-VFP-NEXT:    vmov r0, s2
 ; CHECK-VFP-NEXT:    bx lr
 ; CHECK-VFP-NEXT:    .p2align 2
 ; CHECK-VFP-NEXT:  @ %bb.1:
@@ -419,12 +419,12 @@ define float @f12(i32 %a, i32 %b) nounwind uwtable readnone ssp {
 ;
 ; CHECK-NEON-LABEL: f12:
 ; CHECK-NEON:       @ %bb.0:
-; CHECK-NEON-NEXT:    vldr s0, LCPI11_0
-; CHECK-NEON-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-NEON-NEXT:    vldr s2, LCPI11_0
+; CHECK-NEON-NEXT:    vmov.f32 s0, #1.000000e+00
 ; CHECK-NEON-NEXT:    cmp r0, r1
 ; CHECK-NEON-NEXT:    it eq
-; CHECK-NEON-NEXT:    vmoveq.f32 s0, s2
-; CHECK-NEON-NEXT:    vmov r0, s0
+; CHECK-NEON-NEXT:    vmoveq.f32 s2, s0
+; CHECK-NEON-NEXT:    vmov r0, s2
 ; CHECK-NEON-NEXT:    bx lr
 ; CHECK-NEON-NEXT:    .p2align 2
 ; CHECK-NEON-NEXT:  @ %bb.1:
diff --git a/llvm/test/CodeGen/ARM/select_const.ll b/llvm/test/CodeGen/ARM/select_const.ll
index df95af313eac6..180daa12e7c52 100644
--- a/llvm/test/CodeGen/ARM/select_const.ll
+++ b/llvm/test/CodeGen/ARM/select_const.ll
@@ -645,12 +645,13 @@ define i64 @opaque_constant1(i1 %cond, i64 %x) {
 ; THUMB2-NEXT:    push {r7, lr}
 ; THUMB2-NEXT:    ands r12, r0, #1
 ; THUMB2-NEXT:    mov.w lr, #1
-; THUMB2-NEXT:    itt ne
-; THUMB2-NEXT:    movne.w lr, #65536
+; THUMB2-NEXT:    it ne
 ; THUMB2-NEXT:    movne.w r12, #1
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne.w lr, #65536
 ; THUMB2-NEXT:    subs.w r0, lr, #1
-; THUMB2-NEXT:    sbc r1, r12, #0
 ; THUMB2-NEXT:    eor r3, r3, #1
+; THUMB2-NEXT:    sbc r1, r12, #0
 ; THUMB2-NEXT:    eor r2, r2, #65537
 ; THUMB2-NEXT:    orrs r2, r3
 ; THUMB2-NEXT:    itt ne
@@ -688,11 +689,12 @@ define i64 @opaque_constant1(i1 %cond, i64 %x) {
 ; THUMB-NEXT:    ldr r6, .LCPI24_0
 ; THUMB-NEXT:    eors r2, r6
 ; THUMB-NEXT:    orrs r2, r3
+; THUMB-NEXT:    cmp r2, #0
 ; THUMB-NEXT:    beq .LBB24_5
 ; THUMB-NEXT:  @ %bb.4:
-; THUMB-NEXT:    movs r1, r4
+; THUMB-NEXT:    mov r12, r4
+; THUMB-NEXT:    mov r1, r12
 ; THUMB-NEXT:  .LBB24_5:
-; THUMB-NEXT:    cmp r2, #0
 ; THUMB-NEXT:    beq .LBB24_7
 ; THUMB-NEXT:  @ %bb.6:
 ; THUMB-NEXT:    movs r0, r5
diff --git a/llvm/test/CodeGen/ARM/shift-i64.ll b/llvm/test/CodeGen/ARM/shift-i64.ll
index 33e0ba1457e72..c326ac1529b2f 100644
--- a/llvm/test/CodeGen/ARM/shift-i64.ll
+++ b/llvm/test/CodeGen/ARM/shift-i64.ll
@@ -52,14 +52,14 @@ define i64 @test_lshr(i64 %val, i64 %amt) {
 define i64 @test_ashr(i64 %val, i64 %amt) {
 ; CHECK-LABEL: test_ashr:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    asr r3, r1, r2
-; CHECK-NEXT:    subs r12, r2, #32
+; CHECK-NEXT:    rsb r3, r2, #32
 ; CHECK-NEXT:    lsr r0, r0, r2
-; CHECK-NEXT:    rsb r2, r2, #32
-; CHECK-NEXT:    asrpl r3, r1, #31
-; CHECK-NEXT:    orr r0, r0, r1, lsl r2
-; CHECK-NEXT:    asrpl r0, r1, r12
-; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    orr r0, r0, r1, lsl r3
+; CHECK-NEXT:    subs r3, r2, #32
+; CHECK-NEXT:    asr r2, r1, r2
+; CHECK-NEXT:    asrpl r2, r1, #31
+; CHECK-NEXT:    asrpl r0, r1, r3
+; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    mov pc, lr
 ;
 ; EXPAND-LABEL: test_ashr:
diff --git a/llvm/test/CodeGen/ARM/ssub_sat.ll b/llvm/test/CodeGen/ARM/ssub_sat.ll
index 1bafba3b49ed7..0978bfd1f0140 100644
--- a/llvm/test/CodeGen/ARM/ssub_sat.ll
+++ b/llvm/test/CodeGen/ARM/ssub_sat.ll
@@ -71,22 +71,21 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T1-NEXT:    sbcs r2, r3
 ; CHECK-T1-NEXT:    eors r4, r2
 ; CHECK-T1-NEXT:    ands r4, r1
-; CHECK-T1-NEXT:    asrs r1, r2, #31
+; CHECK-T1-NEXT:    asrs r0, r2, #31
+; CHECK-T1-NEXT:    movs r1, #1
+; CHECK-T1-NEXT:    lsls r1, r1, #31
+; CHECK-T1-NEXT:    eors r1, r0
 ; CHECK-T1-NEXT:    cmp r4, #0
-; CHECK-T1-NEXT:    mov r0, r1
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    bpl .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r0, r5
+; CHECK-T1-NEXT:    bpl .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r4, #0
-; CHECK-T1-NEXT:    bmi .LBB1_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r5
+; CHECK-T1-NEXT:    bmi .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    lsls r2, r2, #31
-; CHECK-T1-NEXT:    eors r1, r2
+; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; CHECK-T2-LABEL: func2:
diff --git a/llvm/test/CodeGen/ARM/ssub_sat_plus.ll b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
index 0a2d1f0e7a240..adf6cafc6ccb8 100644
--- a/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
@@ -65,22 +65,22 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-NEXT:    sbcs r3, r2
 ; CHECK-T1-NEXT:    eors r1, r3
 ; CHECK-T1-NEXT:    ands r1, r5
-; CHECK-T1-NEXT:    asrs r2, r3, #31
+; CHECK-T1-NEXT:    asrs r0, r3, #31
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    eors r2, r0
 ; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    bpl .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    bpl .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r1, r3
+; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:  .LBB1_3:
+; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    bmi .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    movs r1, #1
-; CHECK-T1-NEXT:    lsls r1, r1, #31
-; CHECK-T1-NEXT:    eors r2, r1
+; CHECK-T1-NEXT:    mov r2, r3
 ; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
 ;
diff --git a/llvm/test/CodeGen/ARM/sub-cmp-peephole.ll b/llvm/test/CodeGen/ARM/sub-cmp-peephole.ll
index 046bbbde68642..fb966c29f39a2 100644
--- a/llvm/test/CodeGen/ARM/sub-cmp-peephole.ll
+++ b/llvm/test/CodeGen/ARM/sub-cmp-peephole.ll
@@ -136,8 +136,8 @@ define float @float_sel(i32 %a, i32 %b, float %x, float %y) {
 ;
 ; CHECK-V8-LABEL: float_sel:
 ; CHECK-V8:       @ %bb.0: @ %entry
-; CHECK-V8-NEXT:    vmov s0, r3
 ; CHECK-V8-NEXT:    subs r0, r0, r1
+; CHECK-V8-NEXT:    vmov s0, r3
 ; CHECK-V8-NEXT:    vmov s2, r2
 ; CHECK-V8-NEXT:    vseleq.f32 s0, s2, s0
 ; CHECK-V8-NEXT:    vmov r0, s0
diff --git a/llvm/test/CodeGen/ARM/uadd_sat.ll b/llvm/test/CodeGen/ARM/uadd_sat.ll
index 39c79f4104e6e..5549d9c6c29c0 100644
--- a/llvm/test/CodeGen/ARM/uadd_sat.ll
+++ b/llvm/test/CodeGen/ARM/uadd_sat.ll
@@ -45,21 +45,19 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T1-NEXT:    movs r5, #0
 ; CHECK-T1-NEXT:    adds r4, r0, r2
 ; CHECK-T1-NEXT:    adcs r1, r3
-; CHECK-T1-NEXT:    mov r3, r5
-; CHECK-T1-NEXT:    adcs r3, r5
+; CHECK-T1-NEXT:    mov r0, r5
+; CHECK-T1-NEXT:    adcs r0, r5
 ; CHECK-T1-NEXT:    mvns r2, r5
-; CHECK-T1-NEXT:    cmp r3, #0
+; CHECK-T1-NEXT:    cmp r0, #0
 ; CHECK-T1-NEXT:    mov r0, r2
 ; CHECK-T1-NEXT:    beq .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    beq .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
 ; CHECK-T1-NEXT:    mov r1, r2
 ; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-T1-NEXT:  .LBB1_3:
 ; CHECK-T1-NEXT:    mov r0, r4
-; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    bne .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
 ; CHECK-T1-NEXT:    mov r2, r1
diff --git a/llvm/test/CodeGen/ARM/uadd_sat_plus.ll b/llvm/test/CodeGen/ARM/uadd_sat_plus.ll
index 451b32f730424..ffacba8cf0124 100644
--- a/llvm/test/CodeGen/ARM/uadd_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/uadd_sat_plus.ll
@@ -44,31 +44,29 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-LABEL: func64:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-T1-NEXT:    push {r4, r5, r7, lr}
-; CHECK-T1-NEXT:    movs r5, #0
-; CHECK-T1-NEXT:    ldr r2, [sp, #20]
-; CHECK-T1-NEXT:    ldr r3, [sp, #16]
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    ldr r2, [sp, #12]
+; CHECK-T1-NEXT:    ldr r3, [sp, #8]
 ; CHECK-T1-NEXT:    adds r3, r0, r3
 ; CHECK-T1-NEXT:    adcs r2, r1
-; CHECK-T1-NEXT:    mov r4, r5
-; CHECK-T1-NEXT:    adcs r4, r5
-; CHECK-T1-NEXT:    mvns r1, r5
-; CHECK-T1-NEXT:    cmp r4, #0
+; CHECK-T1-NEXT:    mov r0, r4
+; CHECK-T1-NEXT:    adcs r0, r4
+; CHECK-T1-NEXT:    mvns r1, r4
+; CHECK-T1-NEXT:    cmp r0, #0
 ; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:    beq .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    beq .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:  .LBB1_3:
 ; CHECK-T1-NEXT:    mov r0, r3
-; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    bne .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
 ; CHECK-T1-NEXT:    mov r1, r2
-; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, pc}
 ;
 ; CHECK-T2-LABEL: func64:
 ; CHECK-T2:       @ %bb.0:
diff --git a/llvm/test/CodeGen/ARM/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/ARM/umulo-128-legalisation-lowering.ll
index 464808ec8861b..4eb82c80e2bff 100644
--- a/llvm/test/CodeGen/ARM/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/ARM/umulo-128-legalisation-lowering.ll
@@ -7,207 +7,209 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; ARMV6:       @ %bb.0: @ %start
 ; ARMV6-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; ARMV6-NEXT:    sub sp, sp, #28
-; ARMV6-NEXT:    ldr lr, [sp, #72]
-; ARMV6-NEXT:    mov r6, r0
-; ARMV6-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; ARMV6-NEXT:    ldr r4, [sp, #84]
-; ARMV6-NEXT:    umull r1, r0, r2, lr
-; ARMV6-NEXT:    umull r5, r10, r4, r2
-; ARMV6-NEXT:    str r1, [r6]
+; ARMV6-NEXT:    ldr r4, [sp, #72]
+; ARMV6-NEXT:    mov r7, r0
+; ARMV6-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARMV6-NEXT:    ldr r12, [sp, #64]
+; ARMV6-NEXT:    umull r1, r0, r2, r4
+; ARMV6-NEXT:    ldr r5, [sp, #68]
+; ARMV6-NEXT:    str r1, [r7]
+; ARMV6-NEXT:    ldr r1, [sp, #76]
+; ARMV6-NEXT:    umull r7, r6, r1, r12
+; ARMV6-NEXT:    str r6, [sp, #8] @ 4-byte Spill
+; ARMV6-NEXT:    umull r6, r9, r5, r4
+; ARMV6-NEXT:    add r7, r6, r7
+; ARMV6-NEXT:    umull r4, r6, r12, r4
+; ARMV6-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; ARMV6-NEXT:    mov r4, #0
+; ARMV6-NEXT:    adds r8, r6, r7
 ; ARMV6-NEXT:    ldr r6, [sp, #80]
-; ARMV6-NEXT:    umull r1, r7, r3, r6
-; ARMV6-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; ARMV6-NEXT:    add r1, r5, r1
-; ARMV6-NEXT:    umull r7, r5, r6, r2
-; ARMV6-NEXT:    mov r6, lr
-; ARMV6-NEXT:    str r7, [sp, #16] @ 4-byte Spill
-; ARMV6-NEXT:    mov r7, #0
-; ARMV6-NEXT:    adds r1, r5, r1
-; ARMV6-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; ARMV6-NEXT:    adc r1, r7, #0
-; ARMV6-NEXT:    str r1, [sp, #24] @ 4-byte Spill
-; ARMV6-NEXT:    ldr r1, [sp, #64]
-; ARMV6-NEXT:    ldr r7, [sp, #76]
-; ARMV6-NEXT:    ldr r5, [sp, #64]
-; ARMV6-NEXT:    umull r12, r9, r7, r1
-; ARMV6-NEXT:    ldr r1, [sp, #68]
-; ARMV6-NEXT:    umull r11, r8, r1, lr
+; ARMV6-NEXT:    adc r7, r4, #0
+; ARMV6-NEXT:    ldr r4, [sp, #84]
+; ARMV6-NEXT:    str r7, [sp, #24] @ 4-byte Spill
+; ARMV6-NEXT:    umull r12, lr, r3, r6
+; ARMV6-NEXT:    umull r11, r7, r4, r2
 ; ARMV6-NEXT:    add r12, r11, r12
-; ARMV6-NEXT:    umull r11, lr, r5, lr
-; ARMV6-NEXT:    mov r5, r6
-; ARMV6-NEXT:    mov r6, #0
-; ARMV6-NEXT:    adds r12, lr, r12
-; ARMV6-NEXT:    umull r2, lr, r2, r7
-; ARMV6-NEXT:    adc r6, r6, #0
+; ARMV6-NEXT:    umull r11, r10, r6, r2
+; ARMV6-NEXT:    adds r12, r10, r12
+; ARMV6-NEXT:    mov r10, #0
+; ARMV6-NEXT:    adc r6, r10, #0
 ; ARMV6-NEXT:    str r6, [sp, #20] @ 4-byte Spill
 ; ARMV6-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; ARMV6-NEXT:    adds r11, r11, r6
-; ARMV6-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; ARMV6-NEXT:    adc r6, r12, r6
-; ARMV6-NEXT:    mov r12, #0
-; ARMV6-NEXT:    umlal r0, r12, r3, r5
-; ARMV6-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; ARMV6-NEXT:    adds r6, r6, r11
+; ARMV6-NEXT:    str r6, [sp, #12] @ 4-byte Spill
+; ARMV6-NEXT:    adc r6, r8, r12
 ; ARMV6-NEXT:    str r6, [sp, #16] @ 4-byte Spill
-; ARMV6-NEXT:    ldr r6, [sp, #64]
+; ARMV6-NEXT:    ldr r6, [sp, #72]
+; ARMV6-NEXT:    mov r12, #0
+; ARMV6-NEXT:    umull r2, r8, r2, r1
+; ARMV6-NEXT:    umlal r0, r12, r3, r6
 ; ARMV6-NEXT:    adds r0, r2, r0
-; ARMV6-NEXT:    str r0, [r5, #4]
-; ARMV6-NEXT:    adcs r0, r12, lr
-; ARMV6-NEXT:    mov r2, #0
-; ARMV6-NEXT:    adc r2, r2, #0
-; ARMV6-NEXT:    orrs lr, r6, r1
-; ARMV6-NEXT:    ldr r6, [sp, #80]
+; ARMV6-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; ARMV6-NEXT:    adcs r8, r12, r8
+; ARMV6-NEXT:    adc r12, r10, #0
+; ARMV6-NEXT:    cmp lr, #0
+; ARMV6-NEXT:    str r0, [r2, #4]
 ; ARMV6-NEXT:    movne lr, #1
-; ARMV6-NEXT:    umlal r0, r2, r3, r7
-; ARMV6-NEXT:    orrs r12, r6, r4
-; ARMV6-NEXT:    movne r12, #1
+; ARMV6-NEXT:    ldr r11, [sp, #8] @ 4-byte Reload
+; ARMV6-NEXT:    cmp r7, #0
+; ARMV6-NEXT:    movne r7, #1
+; ARMV6-NEXT:    ldr r0, [sp, #64]
+; ARMV6-NEXT:    cmp r11, #0
+; ARMV6-NEXT:    umlal r8, r12, r3, r1
+; ARMV6-NEXT:    movne r11, #1
 ; ARMV6-NEXT:    cmp r9, #0
-; ARMV6-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
 ; ARMV6-NEXT:    movne r9, #1
-; ARMV6-NEXT:    cmp r8, #0
-; ARMV6-NEXT:    movne r8, #1
-; ARMV6-NEXT:    cmp r6, #0
-; ARMV6-NEXT:    movne r6, #1
-; ARMV6-NEXT:    cmp r10, #0
+; ARMV6-NEXT:    orrs r10, r0, r5
+; ARMV6-NEXT:    ldr r0, [sp, #80]
 ; ARMV6-NEXT:    movne r10, #1
-; ARMV6-NEXT:    cmp r1, #0
-; ARMV6-NEXT:    movne r1, #1
-; ARMV6-NEXT:    cmp r7, #0
-; ARMV6-NEXT:    movne r7, #1
+; ARMV6-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; ARMV6-NEXT:    orrs r0, r0, r4
+; ARMV6-NEXT:    movne r0, #1
 ; ARMV6-NEXT:    cmp r4, #0
 ; ARMV6-NEXT:    movne r4, #1
 ; ARMV6-NEXT:    cmp r3, #0
 ; ARMV6-NEXT:    movne r3, #1
-; ARMV6-NEXT:    adds r0, r0, r11
-; ARMV6-NEXT:    str r0, [r5, #8]
-; ARMV6-NEXT:    and r1, r1, r7
-; ARMV6-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; ARMV6-NEXT:    orr r1, r1, r8
+; ARMV6-NEXT:    cmp r5, #0
+; ARMV6-NEXT:    movne r5, #1
+; ARMV6-NEXT:    cmp r1, #0
+; ARMV6-NEXT:    movne r1, #1
+; ARMV6-NEXT:    adds r6, r8, r6
+; ARMV6-NEXT:    str r6, [r2, #8]
+; ARMV6-NEXT:    and r1, r5, r1
+; ARMV6-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; ARMV6-NEXT:    orr r1, r1, r9
-; ARMV6-NEXT:    adcs r0, r2, r0
-; ARMV6-NEXT:    str r0, [r5, #12]
-; ARMV6-NEXT:    and r0, r4, r3
-; ARMV6-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; ARMV6-NEXT:    orr r0, r0, r10
-; ARMV6-NEXT:    orr r0, r0, r6
-; ARMV6-NEXT:    orr r0, r0, r2
-; ARMV6-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; ARMV6-NEXT:    orr r1, r1, r2
-; ARMV6-NEXT:    and r2, lr, r12
-; ARMV6-NEXT:    orr r1, r2, r1
-; ARMV6-NEXT:    orr r0, r1, r0
+; ARMV6-NEXT:    orr r1, r1, r11
+; ARMV6-NEXT:    and r0, r10, r0
+; ARMV6-NEXT:    adcs r6, r12, r6
+; ARMV6-NEXT:    str r6, [r2, #12]
+; ARMV6-NEXT:    ldr r6, [sp, #24] @ 4-byte Reload
+; ARMV6-NEXT:    orr r1, r1, r6
+; ARMV6-NEXT:    orr r0, r0, r1
+; ARMV6-NEXT:    and r1, r4, r3
+; ARMV6-NEXT:    orr r1, r1, r7
+; ARMV6-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; ARMV6-NEXT:    orr r1, r1, lr
+; ARMV6-NEXT:    orr r1, r1, r3
+; ARMV6-NEXT:    orr r0, r0, r1
 ; ARMV6-NEXT:    mov r1, #0
 ; ARMV6-NEXT:    adc r1, r1, #0
 ; ARMV6-NEXT:    orr r0, r0, r1
 ; ARMV6-NEXT:    and r0, r0, #1
-; ARMV6-NEXT:    strb r0, [r5, #16]
+; ARMV6-NEXT:    strb r0, [r2, #16]
 ; ARMV6-NEXT:    add sp, sp, #28
 ; ARMV6-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ;
 ; ARMV7-LABEL: muloti_test:
 ; ARMV7:       @ %bb.0: @ %start
 ; ARMV7-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; ARMV7-NEXT:    sub sp, sp, #36
-; ARMV7-NEXT:    ldr r5, [sp, #84]
-; ARMV7-NEXT:    mov r8, r0
-; ARMV7-NEXT:    ldr r1, [sp, #72]
-; ARMV7-NEXT:    ldr r10, [sp, #80]
-; ARMV7-NEXT:    ldr r9, [sp, #76]
-; ARMV7-NEXT:    umull r4, lr, r5, r1
-; ARMV7-NEXT:    umull r0, r7, r2, r10
-; ARMV7-NEXT:    str r4, [sp, #24] @ 4-byte Spill
-; ARMV7-NEXT:    ldr r4, [sp, #88]
-; ARMV7-NEXT:    umull r1, r6, r1, r10
+; ARMV7-NEXT:    sub sp, sp, #44
+; ARMV7-NEXT:    ldr r8, [sp, #88]
+; ARMV7-NEXT:    mov r9, r0
+; ARMV7-NEXT:    ldr r7, [sp, #96]
+; ARMV7-NEXT:    ldr lr, [sp, #100]
+; ARMV7-NEXT:    umull r0, r5, r2, r8
+; ARMV7-NEXT:    ldr r4, [sp, #80]
 ; ARMV7-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; ARMV7-NEXT:    umull r11, r0, r2, r5
-; ARMV7-NEXT:    str r6, [sp, #20] @ 4-byte Spill
-; ARMV7-NEXT:    str r1, [sp, #28] @ 4-byte Spill
-; ARMV7-NEXT:    umull r6, r12, r3, r4
+; ARMV7-NEXT:    umull r1, r0, r3, r7
+; ARMV7-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARMV7-NEXT:    umull r0, r11, lr, r2
+; ARMV7-NEXT:    str r1, [sp, #20] @ 4-byte Spill
 ; ARMV7-NEXT:    ldr r1, [sp, #92]
-; ARMV7-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; ARMV7-NEXT:    mov r0, #0
-; ARMV7-NEXT:    umlal r7, r0, r3, r10
+; ARMV7-NEXT:    str r0, [sp] @ 4-byte Spill
+; ARMV7-NEXT:    umull r0, r10, r7, r2
+; ARMV7-NEXT:    mov r7, r1
+; ARMV7-NEXT:    umull r6, r12, r1, r4
+; ARMV7-NEXT:    str r0, [sp, #40] @ 4-byte Spill
+; ARMV7-NEXT:    ldr r0, [sp, #84]
+; ARMV7-NEXT:    str r6, [sp, #24] @ 4-byte Spill
+; ARMV7-NEXT:    umull r6, r1, r0, r8
 ; ARMV7-NEXT:    str r6, [sp, #16] @ 4-byte Spill
-; ARMV7-NEXT:    umull r6, r1, r1, r2
-; ARMV7-NEXT:    umull r2, r4, r4, r2
-; ARMV7-NEXT:    str r6, [sp, #4] @ 4-byte Spill
+; ARMV7-NEXT:    umull r6, r2, r2, r7
+; ARMV7-NEXT:    mov r7, r4
+; ARMV7-NEXT:    str r6, [sp, #8] @ 4-byte Spill
 ; ARMV7-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; ARMV7-NEXT:    adds r2, r11, r7
-; ARMV7-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; ARMV7-NEXT:    mov r11, #0
-; ARMV7-NEXT:    str r4, [sp] @ 4-byte Spill
-; ARMV7-NEXT:    umull r6, r4, r9, r10
-; ARMV7-NEXT:    adcs r9, r0, r7
-; ARMV7-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
-; ARMV7-NEXT:    adc r10, r11, #0
-; ARMV7-NEXT:    stm r8, {r0, r2}
-; ARMV7-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; ARMV7-NEXT:    umlal r9, r10, r3, r5
+; ARMV7-NEXT:    umull r2, r6, r4, r8
+; ARMV7-NEXT:    str r2, [sp, #36] @ 4-byte Spill
+; ARMV7-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
+; ARMV7-NEXT:    str r6, [sp, #28] @ 4-byte Spill
+; ARMV7-NEXT:    mov r6, #0
+; ARMV7-NEXT:    str r2, [r9]
+; ARMV7-NEXT:    umlal r5, r6, r3, r8
 ; ARMV7-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; ARMV7-NEXT:    add r0, r6, r0
-; ARMV7-NEXT:    adds r0, r2, r0
-; ARMV7-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; ARMV7-NEXT:    adc r2, r11, #0
+; ARMV7-NEXT:    ldr r4, [sp] @ 4-byte Reload
+; ARMV7-NEXT:    add r4, r4, r2
+; ARMV7-NEXT:    adds r2, r10, r4
+; ARMV7-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; ARMV7-NEXT:    mov r2, #0
+; ARMV7-NEXT:    adc r2, r2, #0
+; ARMV7-NEXT:    cmp r12, #0
 ; ARMV7-NEXT:    str r2, [sp, #32] @ 4-byte Spill
-; ARMV7-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; ARMV7-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
-; ARMV7-NEXT:    add r2, r6, r2
-; ARMV7-NEXT:    ldr r6, [sp] @ 4-byte Reload
-; ARMV7-NEXT:    adds r2, r6, r2
-; ARMV7-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
-; ARMV7-NEXT:    adc r11, r11, #0
-; ARMV7-NEXT:    adds r7, r7, r6
-; ARMV7-NEXT:    ldr r6, [sp, #92]
-; ARMV7-NEXT:    adc r0, r0, r2
-; ARMV7-NEXT:    str r0, [sp, #28] @ 4-byte Spill
-; ARMV7-NEXT:    ldr r0, [sp, #92]
-; ARMV7-NEXT:    cmp r3, #0
-; ARMV7-NEXT:    movwne r3, #1
-; ARMV7-NEXT:    ldr r2, [sp, #76]
-; ARMV7-NEXT:    cmp r0, #0
-; ARMV7-NEXT:    movwne r0, #1
+; ARMV7-NEXT:    movwne r12, #1
 ; ARMV7-NEXT:    cmp r1, #0
+; ARMV7-NEXT:    ldr r2, [sp, #96]
 ; ARMV7-NEXT:    movwne r1, #1
-; ARMV7-NEXT:    cmp r12, #0
-; ARMV7-NEXT:    and r0, r0, r3
-; ARMV7-NEXT:    movwne r12, #1
-; ARMV7-NEXT:    cmp r5, #0
-; ARMV7-NEXT:    orr r0, r0, r1
-; ARMV7-NEXT:    movwne r5, #1
+; ARMV7-NEXT:    orrs r10, r7, r0
+; ARMV7-NEXT:    movwne r10, #1
+; ARMV7-NEXT:    orrs r7, r2, lr
+; ARMV7-NEXT:    ldr r2, [sp, #92]
+; ARMV7-NEXT:    movwne r7, #1
+; ARMV7-NEXT:    cmp r0, #0
+; ARMV7-NEXT:    movwne r0, #1
 ; ARMV7-NEXT:    cmp r2, #0
-; ARMV7-NEXT:    mov r1, r2
-; ARMV7-NEXT:    mov r3, r2
-; ARMV7-NEXT:    movwne r1, #1
+; ARMV7-NEXT:    mov r4, r2
+; ARMV7-NEXT:    mov r8, r2
+; ARMV7-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; ARMV7-NEXT:    movwne r4, #1
+; ARMV7-NEXT:    and r0, r0, r4
+; ARMV7-NEXT:    mov r4, #0
+; ARMV7-NEXT:    adds r5, r2, r5
+; ARMV7-NEXT:    str r5, [r9, #4]
+; ARMV7-NEXT:    orr r0, r0, r1
+; ARMV7-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; ARMV7-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; ARMV7-NEXT:    and r5, r10, r7
+; ARMV7-NEXT:    orr r0, r0, r12
+; ARMV7-NEXT:    mov r12, #0
+; ARMV7-NEXT:    add r1, r2, r1
+; ARMV7-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; ARMV7-NEXT:    adcs r2, r6, r2
+; ARMV7-NEXT:    ldr r6, [sp, #28] @ 4-byte Reload
+; ARMV7-NEXT:    adc r7, r4, #0
+; ARMV7-NEXT:    adds r1, r6, r1
+; ARMV7-NEXT:    umlal r2, r7, r3, r8
+; ARMV7-NEXT:    adc r4, r4, #0
+; ARMV7-NEXT:    orr r0, r0, r4
+; ARMV7-NEXT:    orr r0, r5, r0
+; ARMV7-NEXT:    ldr r4, [sp, #40] @ 4-byte Reload
+; ARMV7-NEXT:    ldr r5, [sp, #36] @ 4-byte Reload
+; ARMV7-NEXT:    adds r5, r5, r4
+; ARMV7-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
+; ARMV7-NEXT:    adc r1, r1, r4
+; ARMV7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
 ; ARMV7-NEXT:    cmp r4, #0
-; ARMV7-NEXT:    ldr r2, [sp, #72]
 ; ARMV7-NEXT:    movwne r4, #1
+; ARMV7-NEXT:    cmp r3, #0
+; ARMV7-NEXT:    movwne r3, #1
 ; ARMV7-NEXT:    cmp lr, #0
-; ARMV7-NEXT:    and r1, r1, r5
 ; ARMV7-NEXT:    movwne lr, #1
-; ARMV7-NEXT:    orrs r2, r2, r3
-; ARMV7-NEXT:    ldr r3, [sp, #88]
-; ARMV7-NEXT:    movwne r2, #1
+; ARMV7-NEXT:    cmp r11, #0
+; ARMV7-NEXT:    movwne r11, #1
+; ARMV7-NEXT:    adds r2, r2, r5
+; ARMV7-NEXT:    and r3, lr, r3
+; ARMV7-NEXT:    str r2, [r9, #8]
+; ARMV7-NEXT:    adcs r1, r7, r1
+; ARMV7-NEXT:    str r1, [r9, #12]
+; ARMV7-NEXT:    orr r1, r3, r11
+; ARMV7-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
 ; ARMV7-NEXT:    orr r1, r1, r4
-; ARMV7-NEXT:    orr r0, r0, r12
-; ARMV7-NEXT:    orrs r3, r3, r6
-; ARMV7-NEXT:    orr r1, r1, lr
-; ARMV7-NEXT:    movwne r3, #1
-; ARMV7-NEXT:    adds r7, r9, r7
-; ARMV7-NEXT:    str r7, [r8, #8]
-; ARMV7-NEXT:    and r2, r2, r3
-; ARMV7-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
-; ARMV7-NEXT:    orr r0, r0, r11
-; ARMV7-NEXT:    adcs r7, r10, r7
-; ARMV7-NEXT:    str r7, [r8, #12]
-; ARMV7-NEXT:    ldr r7, [sp, #32] @ 4-byte Reload
-; ARMV7-NEXT:    orr r1, r1, r7
-; ARMV7-NEXT:    orr r1, r2, r1
-; ARMV7-NEXT:    orr r0, r1, r0
-; ARMV7-NEXT:    mov r1, #0
-; ARMV7-NEXT:    adc r1, r1, #0
+; ARMV7-NEXT:    orr r1, r1, r2
+; ARMV7-NEXT:    orr r0, r0, r1
+; ARMV7-NEXT:    adc r1, r12, #0
 ; ARMV7-NEXT:    orr r0, r0, r1
 ; ARMV7-NEXT:    and r0, r0, #1
-; ARMV7-NEXT:    strb r0, [r8, #16]
-; ARMV7-NEXT:    add sp, sp, #36
+; ARMV7-NEXT:    strb r0, [r9, #16]
+; ARMV7-NEXT:    add sp, sp, #44
 ; ARMV7-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/ARM/umulo-64-legalisation-lowering.ll b/llvm/test/CodeGen/ARM/umulo-64-legalisation-lowering.ll
index ddf033b19b949..64d9831442970 100644
--- a/llvm/test/CodeGen/ARM/umulo-64-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/ARM/umulo-64-legalisation-lowering.ll
@@ -5,50 +5,49 @@
 define { i64, i8 } @mulodi_test(i64 %l, i64 %r) unnamed_addr #0 {
 ; ARMV6-LABEL: mulodi_test:
 ; ARMV6:       @ %bb.0: @ %start
-; ARMV6-NEXT:    push {r4, r5, r6, lr}
-; ARMV6-NEXT:    umull r12, lr, r3, r0
-; ARMV6-NEXT:    mov r6, #0
-; ARMV6-NEXT:    umull r4, r5, r1, r2
-; ARMV6-NEXT:    umull r0, r2, r0, r2
-; ARMV6-NEXT:    add r4, r4, r12
-; ARMV6-NEXT:    adds r12, r2, r4
-; ARMV6-NEXT:    adc r2, r6, #0
+; ARMV6-NEXT:    push {r4, r5, r11, lr}
+; ARMV6-NEXT:    umull r12, lr, r1, r2
+; ARMV6-NEXT:    umull r4, r5, r3, r0
+; ARMV6-NEXT:    cmp lr, #0
+; ARMV6-NEXT:    movne lr, #1
 ; ARMV6-NEXT:    cmp r3, #0
 ; ARMV6-NEXT:    movne r3, #1
 ; ARMV6-NEXT:    cmp r1, #0
+; ARMV6-NEXT:    umull r0, r2, r0, r2
 ; ARMV6-NEXT:    movne r1, #1
-; ARMV6-NEXT:    cmp r5, #0
 ; ARMV6-NEXT:    and r1, r1, r3
-; ARMV6-NEXT:    movne r5, #1
-; ARMV6-NEXT:    cmp lr, #0
-; ARMV6-NEXT:    orr r1, r1, r5
-; ARMV6-NEXT:    movne lr, #1
+; ARMV6-NEXT:    cmp r5, #0
 ; ARMV6-NEXT:    orr r1, r1, lr
-; ARMV6-NEXT:    orr r2, r1, r2
-; ARMV6-NEXT:    mov r1, r12
-; ARMV6-NEXT:    pop {r4, r5, r6, pc}
+; ARMV6-NEXT:    movne r5, #1
+; ARMV6-NEXT:    orr r3, r1, r5
+; ARMV6-NEXT:    add r1, r12, r4
+; ARMV6-NEXT:    adds r1, r2, r1
+; ARMV6-NEXT:    mov r5, #0
+; ARMV6-NEXT:    adc r2, r5, #0
+; ARMV6-NEXT:    orr r2, r3, r2
+; ARMV6-NEXT:    pop {r4, r5, r11, pc}
 ;
 ; ARMV7-LABEL: mulodi_test:
 ; ARMV7:       @ %bb.0: @ %start
 ; ARMV7-NEXT:    push {r4, r5, r11, lr}
-; ARMV7-NEXT:    umull r12, lr, r1, r2
+; ARMV7-NEXT:    umull r12, lr, r3, r0
 ; ARMV7-NEXT:    cmp r3, #0
-; ARMV7-NEXT:    umull r4, r5, r3, r0
 ; ARMV7-NEXT:    movwne r3, #1
 ; ARMV7-NEXT:    cmp r1, #0
+; ARMV7-NEXT:    umull r0, r4, r0, r2
+; ARMV7-NEXT:    umull r2, r5, r1, r2
 ; ARMV7-NEXT:    movwne r1, #1
-; ARMV7-NEXT:    umull r0, r2, r0, r2
-; ARMV7-NEXT:    cmp lr, #0
 ; ARMV7-NEXT:    and r1, r1, r3
-; ARMV7-NEXT:    movwne lr, #1
 ; ARMV7-NEXT:    cmp r5, #0
-; ARMV7-NEXT:    orr r1, r1, lr
 ; ARMV7-NEXT:    movwne r5, #1
-; ARMV7-NEXT:    orr r3, r1, r5
-; ARMV7-NEXT:    add r1, r12, r4
-; ARMV7-NEXT:    mov r5, #0
-; ARMV7-NEXT:    adds r1, r2, r1
-; ARMV7-NEXT:    adc r2, r5, #0
+; ARMV7-NEXT:    cmp lr, #0
+; ARMV7-NEXT:    orr r1, r1, r5
+; ARMV7-NEXT:    movwne lr, #1
+; ARMV7-NEXT:    orr r3, r1, lr
+; ARMV7-NEXT:    add r1, r2, r12
+; ARMV7-NEXT:    mov r2, #0
+; ARMV7-NEXT:    adds r1, r4, r1
+; ARMV7-NEXT:    adc r2, r2, #0
 ; ARMV7-NEXT:    orr r2, r3, r2
 ; ARMV7-NEXT:    pop {r4, r5, r11, pc}
 start:
diff --git a/llvm/test/CodeGen/ARM/usub_sat.ll b/llvm/test/CodeGen/ARM/usub_sat.ll
index 9c2fd3966ea98..73e6dafc08590 100644
--- a/llvm/test/CodeGen/ARM/usub_sat.ll
+++ b/llvm/test/CodeGen/ARM/usub_sat.ll
@@ -49,16 +49,15 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T1-NEXT:    adcs r0, r1
 ; CHECK-T1-NEXT:    movs r3, #1
 ; CHECK-T1-NEXT:    eors r3, r0
+; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:    beq .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    beq .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
 ; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:  .LBB1_3:
 ; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    bne .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
 ; CHECK-T1-NEXT:    mov r1, r4
@@ -70,7 +69,7 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T2-NEXT:    mov.w r12, #0
 ; CHECK-T2-NEXT:    sbcs r1, r3
 ; CHECK-T2-NEXT:    adc r2, r12, #0
-; CHECK-T2-NEXT:    eors r2, r2, #1
+; CHECK-T2-NEXT:    teq.w r2, #1
 ; CHECK-T2-NEXT:    itt ne
 ; CHECK-T2-NEXT:    movne r0, #0
 ; CHECK-T2-NEXT:    movne r1, #0
@@ -82,7 +81,7 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-ARM-NEXT:    mov r12, #0
 ; CHECK-ARM-NEXT:    sbcs r1, r1, r3
 ; CHECK-ARM-NEXT:    adc r2, r12, #0
-; CHECK-ARM-NEXT:    eors r2, r2, #1
+; CHECK-ARM-NEXT:    teq r2, #1
 ; CHECK-ARM-NEXT:    movwne r0, #0
 ; CHECK-ARM-NEXT:    movwne r1, #0
 ; CHECK-ARM-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/ARM/usub_sat_plus.ll b/llvm/test/CodeGen/ARM/usub_sat_plus.ll
index 51ec83c707603..a465a413c6d0e 100644
--- a/llvm/test/CodeGen/ARM/usub_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/usub_sat_plus.ll
@@ -55,16 +55,15 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-NEXT:    adcs r0, r1
 ; CHECK-T1-NEXT:    movs r4, #1
 ; CHECK-T1-NEXT:    eors r4, r0
+; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:    beq .LBB1_3
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    beq .LBB1_4
 ; CHECK-T1-NEXT:  .LBB1_2:
 ; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:  .LBB1_3:
 ; CHECK-T1-NEXT:    mov r0, r3
-; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    bne .LBB1_2
 ; CHECK-T1-NEXT:  .LBB1_4:
 ; CHECK-T1-NEXT:    mov r1, r2
@@ -77,7 +76,7 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T2-NEXT:    subs r0, r0, r2
 ; CHECK-T2-NEXT:    sbcs r1, r3
 ; CHECK-T2-NEXT:    adc r2, r12, #0
-; CHECK-T2-NEXT:    eors r2, r2, #1
+; CHECK-T2-NEXT:    teq.w r2, #1
 ; CHECK-T2-NEXT:    itt ne
 ; CHECK-T2-NEXT:    movne r0, #0
 ; CHECK-T2-NEXT:    movne r1, #0
@@ -91,7 +90,7 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-ARM-NEXT:    subs r0, r0, r2
 ; CHECK-ARM-NEXT:    sbcs r1, r1, r3
 ; CHECK-ARM-NEXT:    adc r2, r12, #0
-; CHECK-ARM-NEXT:    eors r2, r2, #1
+; CHECK-ARM-NEXT:    teq r2, #1
 ; CHECK-ARM-NEXT:    movwne r0, #0
 ; CHECK-ARM-NEXT:    movwne r1, #0
 ; CHECK-ARM-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/ARM/vector-trunc.ll b/llvm/test/CodeGen/ARM/vector-trunc.ll
new file mode 100644
index 0000000000000..9acf463c2be93
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vector-trunc.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=armv7-unknown-none-eabihf -mattr=+neon < %s | FileCheck %s --check-prefix=LE
+; RUN: llc -mtriple=armebv7-unknown-none-eabihf -mattr=+neon < %s | FileCheck %s --check-prefix=BE
+
+define i32 @test(i64 %arg1) {
+; LE-LABEL: test:
+; LE:       @ %bb.0: @ %entry
+; LE-NEXT:    subs r0, r0, #1
+; LE-NEXT:    mov r2, #0
+; LE-NEXT:    sbcs r0, r1, #0
+; LE-NEXT:    vldr s0, .LCPI0_0
+; LE-NEXT:    movwhs r2, #1
+; LE-NEXT:    cmp r2, #0
+; LE-NEXT:    mvnne r2, #0
+; LE-NEXT:    vmov s1, r2
+; LE-NEXT:    vmovn.i32 d16, q0
+; LE-NEXT:    vmovn.i16 d16, q8
+; LE-NEXT:    vmov.u8 r0, d16[0]
+; LE-NEXT:    and r0, r0, #1
+; LE-NEXT:    bx lr
+; LE-NEXT:    .p2align 2
+; LE-NEXT:  @ %bb.1:
+; LE-NEXT:  .LCPI0_0:
+; LE-NEXT:    .long 0xffffffff @ float NaN
+;
+; BE-LABEL: test:
+; BE:       @ %bb.0: @ %entry
+; BE-NEXT:    subs r1, r1, #1
+; BE-NEXT:    mov r2, #0
+; BE-NEXT:    sbcs r0, r0, #0
+; BE-NEXT:    vldr s0, .LCPI0_0
+; BE-NEXT:    movwhs r2, #1
+; BE-NEXT:    cmp r2, #0
+; BE-NEXT:    mvnne r2, #0
+; BE-NEXT:    vmov s1, r2
+; BE-NEXT:    vmovn.i32 d16, q0
+; BE-NEXT:    vmovn.i16 d16, q8
+; BE-NEXT:    vmov.u8 r0, d16[0]
+; BE-NEXT:    and r0, r0, #1
+; BE-NEXT:    bx lr
+; BE-NEXT:    .p2align 2
+; BE-NEXT:  @ %bb.1:
+; BE-NEXT:  .LCPI0_0:
+; BE-NEXT:    .long 0xffffffff @ float NaN
+entry:
+  %insert_zero = insertelement <8 x i64> poison, i64 %arg1, i64 0
+  %splat_zero = shufflevector <8 x i64> %insert_zero, <8 x i64> poison, <8 x i32> zeroinitializer
+  %cmp_vec = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %splat_zero
+  %first_cmp = extractelement <8 x i1> %cmp_vec, i32 0
+  %ext = zext i1 %first_cmp to i32
+  ret i32 %ext
+}
diff --git a/llvm/test/CodeGen/ARM/vselect_imax.ll b/llvm/test/CodeGen/ARM/vselect_imax.ll
index 9f0edb7117bd1..bd5e3061f0d18 100644
--- a/llvm/test/CodeGen/ARM/vselect_imax.ll
+++ b/llvm/test/CodeGen/ARM/vselect_imax.ll
@@ -111,8 +111,8 @@ define void @func_blend15(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 define void @func_blend18(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storeaddr) {
 ; CHECK-LABEL: func_blend18:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    vld1.64 {d16, d17}, [r1:128]!
 ; CHECK-NEXT:    vld1.64 {d22, d23}, [r0:128]!
 ; CHECK-NEXT:    vmov r4, r6, d16
@@ -122,7 +122,6 @@ define void @func_blend18(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    vmov r2, r1, d20
 ; CHECK-NEXT:    subs r2, r2, lr
-; CHECK-NEXT:    vmov r7, lr, d17
 ; CHECK-NEXT:    vmov r2, r5, d22
 ; CHECK-NEXT:    sbcs r1, r1, r12
 ; CHECK-NEXT:    mov r1, #0
@@ -131,33 +130,34 @@ define void @func_blend18(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 ; CHECK-NEXT:    mvnne r1, #0
 ; CHECK-NEXT:    subs r2, r2, r4
 ; CHECK-NEXT:    sbcs r6, r5, r6
-; CHECK-NEXT:    vmov r2, r12, d19
-; CHECK-NEXT:    vmov r5, r4, d21
+; CHECK-NEXT:    vmov r2, r12, d17
+; CHECK-NEXT:    vmov r5, r4, d23
 ; CHECK-NEXT:    mov r6, #0
 ; CHECK-NEXT:    movlt r6, #1
 ; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    mvnne r6, #0
 ; CHECK-NEXT:    subs r2, r5, r2
-; CHECK-NEXT:    sbcs r4, r4, r12
+; CHECK-NEXT:    sbcs r2, r4, r12
+; CHECK-NEXT:    vmov lr, r12, d19
+; CHECK-NEXT:    vmov r4, r5, d21
 ; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    vmov r4, r5, d23
 ; CHECK-NEXT:    movlt r2, #1
-; CHECK-NEXT:    subs r7, r4, r7
-; CHECK-NEXT:    sbcs r7, r5, lr
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    vdup.32 d25, r0
 ; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d25, r2
 ; CHECK-NEXT:    vdup.32 d24, r6
-; CHECK-NEXT:    vdup.32 d27, r2
 ; CHECK-NEXT:    vbit q8, q11, q12
+; CHECK-NEXT:    subs r4, r4, lr
+; CHECK-NEXT:    sbcs r5, r5, r12
+; CHECK-NEXT:    movlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    vdup.32 d27, r0
 ; CHECK-NEXT:    vdup.32 d26, r1
 ; CHECK-NEXT:    vbit q9, q10, q13
 ; CHECK-NEXT:    vst1.64 {d16, d17}, [r3:128]!
 ; CHECK-NEXT:    vst1.64 {d18, d19}, [r3:128]
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT:    pop {r4, r5, r6, lr}
 ; CHECK-NEXT:    mov pc, lr
 ; COST: func_blend18
 ; COST: cost of 0 {{.*}} icmp
@@ -198,12 +198,21 @@ define void @func_blend19(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 ; CHECK-NEXT:    mvnne r12, #0
 ; CHECK-NEXT:    subs r1, r1, r2
 ; CHECK-NEXT:    sbcs r0, r4, r0
-; CHECK-NEXT:    vmov r2, r4, d26
+; CHECK-NEXT:    vmov r2, r4, d24
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
 ; CHECK-NEXT:    vdup.32 d1, r0
+; CHECK-NEXT:    vmov r0, r1, d20
+; CHECK-NEXT:    subs r0, r2, r0
+; CHECK-NEXT:    sbcs r0, r4, r1
+; CHECK-NEXT:    vmov r2, r4, d26
+; CHECK-NEXT:    mov r0, #0
+; CHECK-NEXT:    movlt r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mvnne r0, #0
+; CHECK-NEXT:    vdup.32 d0, r0
 ; CHECK-NEXT:    vmov r0, r1, d22
 ; CHECK-NEXT:    subs r0, r2, r0
 ; CHECK-NEXT:    mov r2, #0
@@ -224,14 +233,15 @@ define void @func_blend19(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 ; CHECK-NEXT:    vmov r0, r1, d28
 ; CHECK-NEXT:    subs r0, r4, r0
 ; CHECK-NEXT:    sbcs r0, r5, r1
-; CHECK-NEXT:    vmov r4, r5, d24
+; CHECK-NEXT:    vmov r4, r5, d27
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
 ; CHECK-NEXT:    vdup.32 d2, r0
-; CHECK-NEXT:    vmov r0, r1, d20
+; CHECK-NEXT:    vmov r0, r1, d23
 ; CHECK-NEXT:    vbit q14, q15, q1
+; CHECK-NEXT:    vbit q10, q12, q0
 ; CHECK-NEXT:    subs r0, r4, r0
 ; CHECK-NEXT:    sbcs r0, r5, r1
 ; CHECK-NEXT:    vmov r1, r4, d17
@@ -240,27 +250,17 @@ define void @func_blend19(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d0, r0
-; CHECK-NEXT:    vbit q10, q12, q0
+; CHECK-NEXT:    vdup.32 d31, r0
+; CHECK-NEXT:    vdup.32 d30, r2
+; CHECK-NEXT:    vbit q11, q13, q15
+; CHECK-NEXT:    vst1.64 {d28, d29}, [r3:128]!
 ; CHECK-NEXT:    subs r1, r5, r1
 ; CHECK-NEXT:    sbcs r1, r6, r4
-; CHECK-NEXT:    vmov r4, r5, d27
-; CHECK-NEXT:    vmov r0, r1, d23
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    movlt r6, #1
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r1
 ; CHECK-NEXT:    movlt lr, #1
 ; CHECK-NEXT:    cmp lr, #0
 ; CHECK-NEXT:    mvnne lr, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vdup.32 d31, lr
-; CHECK-NEXT:    mvnne r6, #0
-; CHECK-NEXT:    vdup.32 d30, r2
-; CHECK-NEXT:    vdup.32 d3, r6
-; CHECK-NEXT:    vbit q11, q13, q15
+; CHECK-NEXT:    vdup.32 d3, lr
 ; CHECK-NEXT:    vdup.32 d2, r12
-; CHECK-NEXT:    vst1.64 {d28, d29}, [r3:128]!
 ; CHECK-NEXT:    vbit q8, q9, q1
 ; CHECK-NEXT:    vst1.64 {d20, d21}, [r3:128]!
 ; CHECK-NEXT:    vst1.64 {d22, d23}, [r3:128]!
@@ -283,198 +283,194 @@ define void @func_blend19(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storea
 define void @func_blend20(ptr %loadaddr, ptr %loadaddr2, ptr %blend, ptr %storeaddr) {
 ; CHECK-LABEL: func_blend20:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    mov lr, r0
-; CHECK-NEXT:    vld1.64 {d16, d17}, [r8:128]!
-; CHECK-NEXT:    add r9, r0, #64
-; CHECK-NEXT:    add r10, r1, #64
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    add r8, r1, #64
+; CHECK-NEXT:    add lr, r0, #64
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1:128]!
 ; CHECK-NEXT:    mov r12, #0
-; CHECK-NEXT:    vld1.64 {d22, d23}, [lr:128]!
-; CHECK-NEXT:    vld1.64 {d18, d19}, [r8:128]!
-; CHECK-NEXT:    vld1.64 {d20, d21}, [lr:128]!
-; CHECK-NEXT:    vmov r6, r4, d19
-; CHECK-NEXT:    vmov r5, r7, d21
-; CHECK-NEXT:    vld1.64 {d4, d5}, [r9:128]!
-; CHECK-NEXT:    vld1.64 {d6, d7}, [r10:128]!
-; CHECK-NEXT:    vld1.64 {d0, d1}, [r10:128]!
-; CHECK-NEXT:    vld1.64 {d2, d3}, [r9:128]!
-; CHECK-NEXT:    subs r6, r5, r6
-; CHECK-NEXT:    sbcs r4, r7, r4
-; CHECK-NEXT:    vmov r5, r6, d18
-; CHECK-NEXT:    vmov r7, r2, d20
+; CHECK-NEXT:    vld1.64 {d24, d25}, [r0:128]!
+; CHECK-NEXT:    vmov r4, r5, d17
+; CHECK-NEXT:    vmov r6, r7, d25
+; CHECK-NEXT:    vld1.64 {d18, d19}, [lr:128]!
+; CHECK-NEXT:    vld1.64 {d20, d21}, [r8:128]!
+; CHECK-NEXT:    vld1.64 {d22, d23}, [r8:128]!
+; CHECK-NEXT:    vld1.64 {d0, d1}, [lr:128]!
+; CHECK-NEXT:    subs r4, r6, r4
+; CHECK-NEXT:    sbcs r4, r7, r5
+; CHECK-NEXT:    vmov r5, r6, d16
+; CHECK-NEXT:    vmov r7, r2, d24
 ; CHECK-NEXT:    mov r4, #0
 ; CHECK-NEXT:    movlt r4, #1
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    mvnne r4, #0
-; CHECK-NEXT:    vdup.32 d31, r4
+; CHECK-NEXT:    vdup.32 d27, r4
 ; CHECK-NEXT:    subs r5, r7, r5
 ; CHECK-NEXT:    sbcs r2, r2, r6
-; CHECK-NEXT:    vmov r4, r5, d3
+; CHECK-NEXT:    vmov r5, r6, d1
 ; CHECK-NEXT:    mov r2, #0
 ; CHECK-NEXT:    movlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
-; CHECK-NEXT:    vdup.32 d30, r2
-; CHECK-NEXT:    vmov r0, r2, d1
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    vmov r4, r5, d2
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d9, r0
-; CHECK-NEXT:    vmov r0, r2, d0
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    vmov r4, r5, d5
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d8, r0
-; CHECK-NEXT:    vmov r0, r2, d7
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    vmov r4, r5, d4
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d11, r0
-; CHECK-NEXT:    vmov r0, r2, d6
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    vmov r4, r5, d23
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d10, r0
-; CHECK-NEXT:    vmov r0, r2, d17
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
+; CHECK-NEXT:    vdup.32 d26, r2
+; CHECK-NEXT:    vmov r2, r4, d23
+; CHECK-NEXT:    vbit q8, q12, q13
+; CHECK-NEXT:    vld1.64 {d24, d25}, [r0:128]!
+; CHECK-NEXT:    vld1.64 {d26, d27}, [r1:128]!
+; CHECK-NEXT:    vld1.64 {d28, d29}, [lr:128]!
+; CHECK-NEXT:    subs r2, r5, r2
+; CHECK-NEXT:    sbcs r2, r6, r4
 ; CHECK-NEXT:    vmov r4, r5, d22
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d25, r0
-; CHECK-NEXT:    vmov r0, r2, d16
-; CHECK-NEXT:    subs r0, r4, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    vdup.32 d24, r0
-; CHECK-NEXT:    vorr q13, q12, q12
-; CHECK-NEXT:    vbsl q13, q11, q8
-; CHECK-NEXT:    vld1.64 {d24, d25}, [r9:128]!
-; CHECK-NEXT:    vorr q8, q5, q5
-; CHECK-NEXT:    vld1.64 {d28, d29}, [r10:128]!
-; CHECK-NEXT:    vbsl q8, q2, q3
-; CHECK-NEXT:    vld1.64 {d6, d7}, [r8:128]!
-; CHECK-NEXT:    vld1.64 {d22, d23}, [r8:128]
-; CHECK-NEXT:    vld1.64 {d4, d5}, [lr:128]!
-; CHECK-NEXT:    vbif q10, q9, q15
-; CHECK-NEXT:    vorr q9, q4, q4
-; CHECK-NEXT:    vmov r0, r2, d22
-; CHECK-NEXT:    vbsl q9, q1, q0
-; CHECK-NEXT:    vld1.64 {d30, d31}, [lr:128]
+; CHECK-NEXT:    vmov r6, r7, d0
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d3, r2
+; CHECK-NEXT:    subs r4, r6, r4
+; CHECK-NEXT:    sbcs r4, r7, r5
+; CHECK-NEXT:    vmov r2, r5, d27
+; CHECK-NEXT:    vmov r6, r7, d25
+; CHECK-NEXT:    mov r4, #0
+; CHECK-NEXT:    movlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mvnne r4, #0
+; CHECK-NEXT:    vdup.32 d2, r4
+; CHECK-NEXT:    subs r2, r6, r2
+; CHECK-NEXT:    sbcs r2, r7, r5
+; CHECK-NEXT:    vmov r6, r7, d24
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d5, r2
+; CHECK-NEXT:    vmov r2, r5, d26
+; CHECK-NEXT:    subs r2, r6, r2
+; CHECK-NEXT:    sbcs r2, r7, r5
+; CHECK-NEXT:    vmov r6, r7, d19
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d4, r2
+; CHECK-NEXT:    vmov r2, r5, d21
+; CHECK-NEXT:    subs r2, r6, r2
+; CHECK-NEXT:    sbcs r2, r7, r5
+; CHECK-NEXT:    vmov r6, r7, d18
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d31, r2
+; CHECK-NEXT:    vmov r2, r5, d20
+; CHECK-NEXT:    subs r2, r6, r2
+; CHECK-NEXT:    sbcs r2, r7, r5
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    vdup.32 d30, r2
+; CHECK-NEXT:    vbif q9, q10, q15
+; CHECK-NEXT:    vld1.64 {d30, d31}, [r8:128]!
+; CHECK-NEXT:    vld1.64 {d20, d21}, [r8:128]
+; CHECK-NEXT:    vbit q13, q12, q2
+; CHECK-NEXT:    vld1.64 {d24, d25}, [lr:128]
+; CHECK-NEXT:    vmov r2, r7, d21
+; CHECK-NEXT:    vbit q11, q0, q1
 ; CHECK-NEXT:    mov lr, #0
-; CHECK-NEXT:    vmov r7, r5, d30
-; CHECK-NEXT:    vld1.64 {d0, d1}, [r9:128]
-; CHECK-NEXT:    vld1.64 {d2, d3}, [r10:128]
-; CHECK-NEXT:    subs r0, r7, r0
-; CHECK-NEXT:    sbcs r0, r5, r2
-; CHECK-NEXT:    vmov r5, r4, d24
-; CHECK-NEXT:    vmov r0, r7, d28
+; CHECK-NEXT:    vmov r6, r5, d25
+; CHECK-NEXT:    vld1.64 {d4, d5}, [r1:128]!
+; CHECK-NEXT:    vld1.64 {d6, d7}, [r0:128]!
+; CHECK-NEXT:    vld1.64 {d0, d1}, [r1:128]
+; CHECK-NEXT:    vld1.64 {d2, d3}, [r0:128]
+; CHECK-NEXT:    subs r1, r6, r2
+; CHECK-NEXT:    vmov r0, r6, d2
+; CHECK-NEXT:    sbcs r1, r5, r7
+; CHECK-NEXT:    vmov r2, r7, d0
 ; CHECK-NEXT:    movlt lr, #1
 ; CHECK-NEXT:    cmp lr, #0
 ; CHECK-NEXT:    mvnne lr, #0
-; CHECK-NEXT:    subs r0, r5, r0
-; CHECK-NEXT:    sbcs r0, r4, r7
-; CHECK-NEXT:    vmov r7, r5, d29
-; CHECK-NEXT:    vmov r4, r6, d25
+; CHECK-NEXT:    subs r0, r0, r2
+; CHECK-NEXT:    sbcs r0, r6, r7
+; CHECK-NEXT:    vmov r2, r7, d30
+; CHECK-NEXT:    vmov r6, r5, d28
 ; CHECK-NEXT:    mov r0, #0
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mvnne r0, #0
-; CHECK-NEXT:    subs r7, r4, r7
+; CHECK-NEXT:    subs r2, r6, r2
+; CHECK-NEXT:    sbcs r2, r5, r7
+; CHECK-NEXT:    vmov r7, r6, d31
+; CHECK-NEXT:    vmov r5, r4, d29
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    mvnne r2, #0
+; CHECK-NEXT:    subs r7, r5, r7
+; CHECK-NEXT:    vmov r5, r1, d7
+; CHECK-NEXT:    sbcs r7, r4, r6
 ; CHECK-NEXT:    mov r4, #0
-; CHECK-NEXT:    sbcs r7, r6, r5
-; CHECK-NEXT:    vmov r5, r1, d31
-; CHECK-NEXT:    vmov r7, r6, d23
+; CHECK-NEXT:    vmov r7, r6, d5
 ; CHECK-NEXT:    movlt r4, #1
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    mvnne r4, #0
-; CHECK-NEXT:    subs r7, r5, r7
-; CHECK-NEXT:    mov r5, #0
+; CHECK-NEXT:    subs r5, r5, r7
 ; CHECK-NEXT:    sbcs r1, r1, r6
-; CHECK-NEXT:    vmov r6, r2, d5
-; CHECK-NEXT:    vmov r1, r7, d7
-; CHECK-NEXT:    movlt r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    mvnne r5, #0
-; CHECK-NEXT:    subs r1, r6, r1
-; CHECK-NEXT:    sbcs r1, r2, r7
-; CHECK-NEXT:    vmov r6, r7, d4
+; CHECK-NEXT:    vmov r6, r7, d6
 ; CHECK-NEXT:    mov r1, #0
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    mvnne r1, #0
 ; CHECK-NEXT:    vdup.32 d9, r1
-; CHECK-NEXT:    vmov r1, r2, d6
+; CHECK-NEXT:    vmov r1, r5, d4
 ; CHECK-NEXT:    subs r1, r6, r1
-; CHECK-NEXT:    sbcs r1, r7, r2
-; CHECK-NEXT:    vmov r6, r7, d0
+; CHECK-NEXT:    sbcs r1, r7, r5
+; CHECK-NEXT:    vmov r6, r7, d3
 ; CHECK-NEXT:    mov r1, #0
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    mvnne r1, #0
 ; CHECK-NEXT:    vdup.32 d8, r1
-; CHECK-NEXT:    vmov r1, r2, d2
-; CHECK-NEXT:    vbif q2, q3, q4
-; CHECK-NEXT:    vdup.32 d7, r5
+; CHECK-NEXT:    vmov r1, r5, d1
+; CHECK-NEXT:    vbit q2, q3, q4
 ; CHECK-NEXT:    vdup.32 d9, r4
-; CHECK-NEXT:    vmov r4, r5, d1
-; CHECK-NEXT:    vdup.32 d8, r0
-; CHECK-NEXT:    mov r0, r3
-; CHECK-NEXT:    vst1.64 {d26, d27}, [r0:128]!
-; CHECK-NEXT:    vbif q12, q14, q4
-; CHECK-NEXT:    vdup.32 d6, lr
-; CHECK-NEXT:    vbit q11, q15, q3
-; CHECK-NEXT:    vst1.64 {d20, d21}, [r0:128]!
+; CHECK-NEXT:    vdup.32 d8, r2
 ; CHECK-NEXT:    subs r1, r6, r1
-; CHECK-NEXT:    mov r6, #0
-; CHECK-NEXT:    sbcs r1, r7, r2
-; CHECK-NEXT:    vmov r1, r2, d3
-; CHECK-NEXT:    movlt r6, #1
-; CHECK-NEXT:    subs r1, r4, r1
-; CHECK-NEXT:    sbcs r1, r5, r2
+; CHECK-NEXT:    sbcs r1, r7, r5
+; CHECK-NEXT:    vmov r5, r6, d24
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    mvnne r1, #0
+; CHECK-NEXT:    vdup.32 d7, r1
+; CHECK-NEXT:    vmov r1, r4, d20
+; CHECK-NEXT:    vdup.32 d6, r0
+; CHECK-NEXT:    subs r1, r5, r1
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    sbcs r0, r6, r4
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r1:128]!
+; CHECK-NEXT:    vorr q8, q4, q4
 ; CHECK-NEXT:    movlt r12, #1
 ; CHECK-NEXT:    cmp r12, #0
+; CHECK-NEXT:    vbsl q8, q14, q15
+; CHECK-NEXT:    vdup.32 d29, lr
+; CHECK-NEXT:    vorr q15, q3, q3
 ; CHECK-NEXT:    mvnne r12, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vdup.32 d27, r12
-; CHECK-NEXT:    mvnne r6, #0
-; CHECK-NEXT:    vdup.32 d26, r6
-; CHECK-NEXT:    vorr q10, q13, q13
-; CHECK-NEXT:    vbsl q10, q0, q1
-; CHECK-NEXT:    vst1.64 {d4, d5}, [r0:128]!
-; CHECK-NEXT:    vst1.64 {d22, d23}, [r0:128]
+; CHECK-NEXT:    vdup.32 d28, r12
 ; CHECK-NEXT:    add r0, r3, #64
-; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; CHECK-NEXT:    vbsl q15, q1, q0
+; CHECK-NEXT:    vst1.64 {d26, d27}, [r1:128]!
+; CHECK-NEXT:    vbit q10, q12, q14
 ; CHECK-NEXT:    vst1.64 {d18, d19}, [r0:128]!
-; CHECK-NEXT:    vst1.64 {d24, d25}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d22, d23}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d4, d5}, [r1:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d30, d31}, [r1:128]
 ; CHECK-NEXT:    vst1.64 {d20, d21}, [r0:128]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    mov pc, lr
 ; COST: func_blend20
 ; COST: cost of 0 {{.*}} icmp
diff --git a/llvm/test/CodeGen/ARM/wide-compares.ll b/llvm/test/CodeGen/ARM/wide-compares.ll
index 09e3592b6d420..9acf8d249ddf1 100644
--- a/llvm/test/CodeGen/ARM/wide-compares.ll
+++ b/llvm/test/CodeGen/ARM/wide-compares.ll
@@ -129,19 +129,16 @@ declare void @g()
 define i64 @test_slt_select(i64 %c, i64 %d, i64 %a, i64 %b) {
 ; CHECK-ARM-LABEL: test_slt_select:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-ARM-NEXT:    ldr r12, [sp, #32]
-; CHECK-ARM-NEXT:    mov r6, #0
-; CHECK-ARM-NEXT:    ldr lr, [sp, #24]
-; CHECK-ARM-NEXT:    ldr r7, [sp, #36]
-; CHECK-ARM-NEXT:    ldr r5, [sp, #28]
+; CHECK-ARM-NEXT:    push {r4, r5, r6, lr}
+; CHECK-ARM-NEXT:    ldr r12, [sp, #24]
+; CHECK-ARM-NEXT:    ldr lr, [sp, #16]
+; CHECK-ARM-NEXT:    ldr r6, [sp, #28]
+; CHECK-ARM-NEXT:    ldr r5, [sp, #20]
 ; CHECK-ARM-NEXT:    subs r4, lr, r12
-; CHECK-ARM-NEXT:    sbcs r7, r5, r7
-; CHECK-ARM-NEXT:    movwlo r6, #1
-; CHECK-ARM-NEXT:    cmp r6, #0
-; CHECK-ARM-NEXT:    moveq r0, r2
-; CHECK-ARM-NEXT:    moveq r1, r3
-; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-ARM-NEXT:    sbcs r6, r5, r6
+; CHECK-ARM-NEXT:    movhs r0, r2
+; CHECK-ARM-NEXT:    movhs r1, r3
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, pc}
 ;
 ; CHECK-THUMB1-NOMOV-LABEL: test_slt_select:
 ; CHECK-THUMB1-NOMOV:       @ %bb.0: @ %entry
@@ -157,22 +154,13 @@ define i64 @test_slt_select(i64 %c, i64 %d, i64 %a, i64 %b) {
 ; CHECK-THUMB1-NOMOV-NEXT:    sbcs r5, r4
 ; CHECK-THUMB1-NOMOV-NEXT:    blo .LBB2_2
 ; CHECK-THUMB1-NOMOV-NEXT:  @ %bb.1: @ %entry
-; CHECK-THUMB1-NOMOV-NEXT:    movs r4, #0
-; CHECK-THUMB1-NOMOV-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NOMOV-NEXT:    beq .LBB2_3
-; CHECK-THUMB1-NOMOV-NEXT:    b .LBB2_4
-; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_2:
-; CHECK-THUMB1-NOMOV-NEXT:    movs r4, #1
-; CHECK-THUMB1-NOMOV-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NOMOV-NEXT:    bne .LBB2_4
-; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_3: @ %entry
-; CHECK-THUMB1-NOMOV-NEXT:    movs r0, r2
-; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_4: @ %entry
-; CHECK-THUMB1-NOMOV-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NOMOV-NEXT:    bne .LBB2_6
-; CHECK-THUMB1-NOMOV-NEXT:  @ %bb.5: @ %entry
+; CHECK-THUMB1-NOMOV-NEXT:    mov r12, r2
+; CHECK-THUMB1-NOMOV-NEXT:    mov r0, r12
+; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_2: @ %entry
+; CHECK-THUMB1-NOMOV-NEXT:    blo .LBB2_4
+; CHECK-THUMB1-NOMOV-NEXT:  @ %bb.3: @ %entry
 ; CHECK-THUMB1-NOMOV-NEXT:    movs r1, r3
-; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_6: @ %entry
+; CHECK-THUMB1-NOMOV-NEXT:  .LBB2_4: @ %entry
 ; CHECK-THUMB1-NOMOV-NEXT:    add sp, #4
 ; CHECK-THUMB1-NOMOV-NEXT:    pop {r4, r5, r6, r7}
 ; CHECK-THUMB1-NOMOV-NEXT:    pop {r2}
@@ -188,46 +176,31 @@ define i64 @test_slt_select(i64 %c, i64 %d, i64 %a, i64 %b) {
 ; CHECK-THUMB1-NEXT:    ldr r7, [sp, #24]
 ; CHECK-THUMB1-NEXT:    subs r6, r7, r6
 ; CHECK-THUMB1-NEXT:    sbcs r5, r4
-; CHECK-THUMB1-NEXT:    blo .LBB2_2
+; CHECK-THUMB1-NEXT:    bhs .LBB2_3
 ; CHECK-THUMB1-NEXT:  @ %bb.1: @ %entry
-; CHECK-THUMB1-NEXT:    movs r4, #0
-; CHECK-THUMB1-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NEXT:    beq .LBB2_3
-; CHECK-THUMB1-NEXT:    b .LBB2_4
-; CHECK-THUMB1-NEXT:  .LBB2_2:
-; CHECK-THUMB1-NEXT:    movs r4, #1
-; CHECK-THUMB1-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NEXT:    bne .LBB2_4
+; CHECK-THUMB1-NEXT:    bhs .LBB2_4
+; CHECK-THUMB1-NEXT:  .LBB2_2: @ %entry
+; CHECK-THUMB1-NEXT:    add sp, #4
+; CHECK-THUMB1-NEXT:    pop {r4, r5, r6, r7, pc}
 ; CHECK-THUMB1-NEXT:  .LBB2_3: @ %entry
 ; CHECK-THUMB1-NEXT:    mov r0, r2
+; CHECK-THUMB1-NEXT:    blo .LBB2_2
 ; CHECK-THUMB1-NEXT:  .LBB2_4: @ %entry
-; CHECK-THUMB1-NEXT:    cmp r4, #0
-; CHECK-THUMB1-NEXT:    beq .LBB2_6
-; CHECK-THUMB1-NEXT:  @ %bb.5: @ %entry
-; CHECK-THUMB1-NEXT:    add sp, #4
-; CHECK-THUMB1-NEXT:    pop {r4, r5, r6, r7, pc}
-; CHECK-THUMB1-NEXT:  .LBB2_6: @ %entry
 ; CHECK-THUMB1-NEXT:    mov r1, r3
 ; CHECK-THUMB1-NEXT:    add sp, #4
 ; CHECK-THUMB1-NEXT:    pop {r4, r5, r6, r7, pc}
 ;
 ; CHECK-THUMB2-LABEL: test_slt_select:
 ; CHECK-THUMB2:       @ %bb.0: @ %entry
-; CHECK-THUMB2-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-THUMB2-NEXT:    sub sp, #4
-; CHECK-THUMB2-NEXT:    ldrd r12, r7, [sp, #32]
-; CHECK-THUMB2-NEXT:    movs r6, #0
-; CHECK-THUMB2-NEXT:    ldrd lr, r5, [sp, #24]
+; CHECK-THUMB2-NEXT:    push {r4, r5, r6, lr}
+; CHECK-THUMB2-NEXT:    ldrd r12, r6, [sp, #24]
+; CHECK-THUMB2-NEXT:    ldrd lr, r5, [sp, #16]
 ; CHECK-THUMB2-NEXT:    subs.w r4, lr, r12
-; CHECK-THUMB2-NEXT:    sbcs.w r7, r5, r7
-; CHECK-THUMB2-NEXT:    it lo
-; CHECK-THUMB2-NEXT:    movlo r6, #1
-; CHECK-THUMB2-NEXT:    cmp r6, #0
-; CHECK-THUMB2-NEXT:    itt eq
-; CHECK-THUMB2-NEXT:    moveq r0, r2
-; CHECK-THUMB2-NEXT:    moveq r1, r3
-; CHECK-THUMB2-NEXT:    add sp, #4
-; CHECK-THUMB2-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-THUMB2-NEXT:    sbcs.w r6, r5, r6
+; CHECK-THUMB2-NEXT:    itt hs
+; CHECK-THUMB2-NEXT:    movhs r0, r2
+; CHECK-THUMB2-NEXT:    movhs r1, r3
+; CHECK-THUMB2-NEXT:    pop {r4, r5, r6, pc}
 entry:
     %cmp = icmp ult i64 %a, %b
     %r1 = select i1 %cmp, i64 %c, i64 %d
diff --git a/llvm/test/CodeGen/AVR/inline-asm/loadstore.ll b/llvm/test/CodeGen/AVR/inline-asm/loadstore.ll
index 524fef28382fb..5d45ece8f31c9 100644
--- a/llvm/test/CodeGen/AVR/inline-asm/loadstore.ll
+++ b/llvm/test/CodeGen/AVR/inline-asm/loadstore.ll
@@ -44,6 +44,19 @@ define i8 @loadz(ptr %0) {
   ret i8 %2
 }
 
+define i8 @load_ptr_imm() {
+; CHECK-LABEL: load_ptr_imm:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldi r26, 210
+; CHECK-NEXT:    ldi r27, 4
+; CHECK-NEXT:    ;APP
+; CHECK-NEXT:    ld r24, X
+; CHECK-NEXT:    ;NO_APP
+; CHECK-NEXT:    ret
+  %1 = tail call i8 asm sideeffect "ld $0, $1", "=r,e"(i16 1234)
+  ret i8 %1
+}
+
 define void @storex(ptr %0, i8 %1) {
 ; CHECK-LABEL: storex:
 ; CHECK:       ; %bb.0:
@@ -86,3 +99,16 @@ define void @storez(ptr %0, i8 %1) {
   tail call void asm sideeffect "st ${0:a}, $1", "z,r"(ptr %0, i8 %1)
   ret void
 }
+
+define void @store_ptr_imm(i8 %0) {
+; CHECK-LABEL: store_ptr_imm:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldi r26, 210
+; CHECK-NEXT:    ldi r27, 4
+; CHECK-NEXT:    ;APP
+; CHECK-NEXT:    st X, r24
+; CHECK-NEXT:    ;NO_APP
+; CHECK-NEXT:    ret
+  tail call void asm sideeffect "st $0, $1", "e,r"(i16 1234, i8 %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/BufferStore-errors.ll b/llvm/test/CodeGen/DirectX/BufferStore-errors.ll
index b00fc38c901f9..ffdcb037b8968 100644
--- a/llvm/test/CodeGen/DirectX/BufferStore-errors.ll
+++ b/llvm/test/CodeGen/DirectX/BufferStore-errors.ll
@@ -6,7 +6,7 @@ target triple = "dxil-pc-shadermodel6.6-compute"
 ; CHECK: error:
 ; CHECK-SAME: in function storetoomany
 ; CHECK-SAME: typedBufferStore data must be a vector of 4 elements
-define void @storetoomany(<5 x float> %data, i32 %index) {
+define void @storetoomany(<5 x float> %data, i32 %index) "hlsl.export" {
   %buffer = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
       @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4f32_1_0_0(
           i32 0, i32 0, i32 1, i32 0, i1 false)
@@ -21,7 +21,7 @@ define void @storetoomany(<5 x float> %data, i32 %index) {
 ; CHECK: error:
 ; CHECK-SAME: in function storetoofew
 ; CHECK-SAME: typedBufferStore data must be a vector of 4 elements
-define void @storetoofew(<3 x i32> %data, i32 %index) {
+define void @storetoofew(<3 x i32> %data, i32 %index) "hlsl.export" {
   %buffer = call target("dx.TypedBuffer", <4 x i32>, 1, 0, 0)
       @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4i32_1_0_0(
           i32 0, i32 0, i32 1, i32 0, i1 false)
diff --git a/llvm/test/CodeGen/DirectX/BufferStore.ll b/llvm/test/CodeGen/DirectX/BufferStore.ll
index 9ea7735be59c8..81cc5fd328e0a 100644
--- a/llvm/test/CodeGen/DirectX/BufferStore.ll
+++ b/llvm/test/CodeGen/DirectX/BufferStore.ll
@@ -90,3 +90,27 @@ define void @storei16(<4 x i16> %data, i32 %index) {
 
   ret void
 }
+
+define void @store_scalarized_floats(float %data0, float %data1, float %data2, float %data3, i32 %index) {
+
+  ; CHECK: [[BIND:%.*]] = call %dx.types.Handle @dx.op.createHandleFromBinding(i32 217,
+  ; CHECK: [[HANDLE:%.*]] = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle [[BIND]]
+  %buffer = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
+      @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4f32_1_0_0(
+          i32 0, i32 0, i32 1, i32 0, i1 false)
+
+  ; We shouldn't end up with any inserts/extracts.
+  ; CHECK-NOT: insertelement
+  ; CHECK-NOT: extractelement
+
+  ; CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle [[HANDLE]], i32 %index, i32 undef, float %data0, float %data1, float %data2, float %data3, i8 15)
+  %vec.upto0 = insertelement <4 x float> poison, float %data0, i64 0
+  %vec.upto1 = insertelement <4 x float> %vec.upto0, float %data1, i64 1
+  %vec.upto2 = insertelement <4 x float> %vec.upto1, float %data2, i64 2
+  %vec = insertelement <4 x float> %vec.upto2, float %data3, i64 3
+  call void @llvm.dx.typedBufferStore(
+      target("dx.TypedBuffer", <4 x float>, 1, 0, 0) %buffer,
+      i32 %index, <4 x float> %vec)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/finalize-linkage-intrinsics.ll b/llvm/test/CodeGen/DirectX/finalize-linkage-intrinsics.ll
new file mode 100644
index 0000000000000..e2ac6094f0eb5
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/finalize-linkage-intrinsics.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-finalize-linkage -verify -mtriple=dxil-unknown-shadermodel6.5-library %s
+
+define float @f(float %f) "hlsl.export" {
+  %x = call float @llvm.atan.f32(float %f)
+  ret float %x
+}
+
+declare float @llvm.atan.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll b/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
new file mode 100644
index 0000000000000..baf93d4e177f0
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/group_memory_barrier_with_group_sync.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s
+
+define void @test_group_memory_barrier_with_group_sync() {
+entry:
+  ; CHECK: call void @dx.op.barrier(i32 80, i32 9)
+  call void @llvm.dx.group.memory.barrier.with.group.sync()
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index f0950df08eff5..40fa30778a153 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -7,6 +7,7 @@
 ; CHECK-NEXT: Target Library Information
 ; CHECK-NEXT: Target Transform Information
 ; CHECK-NEXT: ModulePass Manager
+; CHECK-NEXT:   DXIL Finalize Linkage
 ; CHECK-NEXT:   DXIL Intrinsic Expansion
 ; CHECK-NEXT:   DXIL Data Scalarization
 ; CHECK-NEXT:   DXIL Array Flattener
@@ -15,7 +16,6 @@
 ; CHECK-NEXT:     Scalarize vector operations
 ; CHECK-NEXT:   DXIL Resource analysis
 ; CHECK-NEXT:   DXIL Op Lowering
-; CHECK-NEXT:   DXIL Finalize Linkage
 ; CHECK-NEXT:   DXIL resource Information
 ; CHECK-NEXT:   DXIL Shader Flag Analysis
 ; CHECK-NEXT:   DXIL Module Metadata analysis
diff --git a/llvm/test/CodeGen/Generic/builtin-expect-with-probability.ll b/llvm/test/CodeGen/Generic/builtin-expect-with-probability.ll
deleted file mode 100644
index aef134b636d5a..0000000000000
--- a/llvm/test/CodeGen/Generic/builtin-expect-with-probability.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: llc < %s
-
-declare i32 @llvm.expect.with.probability(i32, i32, double)
-
-define i32 @test1(i32 %val) nounwind {
-    %expected = call i32 @llvm.expect.with.probability(i32 %val, i32 1, double 0.5)
-    ret i32 %expected
-}
diff --git a/llvm/test/CodeGen/Hexagon/addrmode-neg-offset.ll b/llvm/test/CodeGen/Hexagon/addrmode-neg-offset.ll
new file mode 100644
index 0000000000000..3c3c7c05e7710
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/addrmode-neg-offset.ll
@@ -0,0 +1,9 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+; CHECK: r0 = add(r0,#4)
+; CHECK: memd(r0+#-16) =
+define void @neg_gep_store(ptr %ptr) {
+  %gep = getelementptr i8, ptr %ptr, i32 -12
+  store i64 0, ptr %gep, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir
index d291af97d3afd..4abc5dae2d90d 100644
--- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir
+++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir
@@ -1,9 +1,9 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -march=hexagon -enable-subreg-liveness -start-after machine-scheduler -stop-after stack-slot-coloring -o - %s | FileCheck %s
 
 --- |
   target triple = "hexagon"
 
-  ; Function Attrs: nounwind optsize
   define void @main() #0 {
   entry:
     br label %for.body
@@ -142,6 +142,64 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
 body:             |
+  ; CHECK-LABEL: name: main
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r20 = A2_tfrsi 524288
+  ; CHECK-NEXT:   renamable $r17 = A2_tfrsi 0
+  ; CHECK-NEXT:   renamable $d2 = S2_extractup undef renamable $d0, 6, 25
+  ; CHECK-NEXT:   renamable $d0 = A2_tfrpi 2
+  ; CHECK-NEXT:   renamable $d13 = A2_tfrpi -1
+  ; CHECK-NEXT:   renamable $d13 = S2_asl_r_p_acc killed renamable $d13, killed renamable $d0, renamable $r4
+  ; CHECK-NEXT:   S2_storerd_io %stack.0, 0, renamable $d2 :: (store (s64) into %stack.0)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.for.body:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $d8:0x0000000000000001, $d13, $r20
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29
+  ; CHECK-NEXT:   J2_call @lrand48, implicit-def dead $d0, implicit-def dead $d1, implicit-def dead $d2, implicit-def dead $d3, implicit-def dead $d4, implicit-def dead $d5, implicit-def dead $d6, implicit-def dead $d7, implicit-def dead $r28, implicit-def dead $r31, implicit-def dead $p0, implicit-def dead $p1, implicit-def dead $p2, implicit-def dead $p3, implicit-def dead $m0, implicit-def dead $m1, implicit-def dead $lc0, implicit-def dead $lc1, implicit-def dead $sa0, implicit-def dead $sa1, implicit-def dead $usr, implicit-def $usr_ovf, implicit-def dead $cs0, implicit-def dead $cs1, implicit-def dead $w0, implicit-def dead $w1, implicit-def dead $w2, implicit-def dead $w3, implicit-def dead $w4, implicit-def dead $w5, implicit-def dead $w6, implicit-def dead $w7, implicit-def dead $w8, implicit-def dead $w9, implicit-def dead $w10, implicit-def dead $w11, implicit-def dead $w12, implicit-def dead $w13, implicit-def dead $w14, implicit-def dead $w15, implicit-def dead $q0, implicit-def dead $q1, implicit-def dead $q2, implicit-def dead $q3, implicit-def $r0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29
+  ; CHECK-NEXT:   renamable $r18 = COPY $r0
+  ; CHECK-NEXT:   renamable $r19 = S2_asr_i_r renamable $r18, 31
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29
+  ; CHECK-NEXT:   J2_call @lrand48, implicit-def dead $d0, implicit-def dead $d1, implicit-def dead $d2, implicit-def dead $d3, implicit-def dead $d4, implicit-def dead $d5, implicit-def dead $d6, implicit-def dead $d7, implicit-def dead $r28, implicit-def dead $r31, implicit-def dead $p0, implicit-def dead $p1, implicit-def dead $p2, implicit-def dead $p3, implicit-def dead $m0, implicit-def dead $m1, implicit-def dead $lc0, implicit-def dead $lc1, implicit-def dead $sa0, implicit-def dead $sa1, implicit-def dead $usr, implicit-def $usr_ovf, implicit-def dead $cs0, implicit-def dead $cs1, implicit-def dead $w0, implicit-def dead $w1, implicit-def dead $w2, implicit-def dead $w3, implicit-def dead $w4, implicit-def dead $w5, implicit-def dead $w6, implicit-def dead $w7, implicit-def dead $w8, implicit-def dead $w9, implicit-def dead $w10, implicit-def dead $w11, implicit-def dead $w12, implicit-def dead $w13, implicit-def dead $w14, implicit-def dead $w15, implicit-def dead $q0, implicit-def dead $q1, implicit-def dead $q2, implicit-def dead $q3, implicit-def $r0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29
+  ; CHECK-NEXT:   renamable $r16 = COPY $r0
+  ; CHECK-NEXT:   renamable $d12 = S2_extractup renamable $d8, 22, 9
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29
+  ; CHECK-NEXT:   J2_call @lrand48, implicit-def dead $d0, implicit-def dead $d1, implicit-def dead $d2, implicit-def dead $d3, implicit-def dead $d4, implicit-def dead $d5, implicit-def dead $d6, implicit-def dead $d7, implicit-def dead $r28, implicit-def dead $r31, implicit-def dead $p0, implicit-def dead $p1, implicit-def dead $p2, implicit-def dead $p3, implicit-def dead $m0, implicit-def dead $m1, implicit-def dead $lc0, implicit-def dead $lc1, implicit-def dead $sa0, implicit-def dead $sa1, implicit-def dead $usr, implicit-def $usr_ovf, implicit-def dead $cs0, implicit-def dead $cs1, implicit-def dead $w0, implicit-def dead $w1, implicit-def dead $w2, implicit-def dead $w3, implicit-def dead $w4, implicit-def dead $w5, implicit-def dead $w6, implicit-def dead $w7, implicit-def dead $w8, implicit-def dead $w9, implicit-def dead $w10, implicit-def dead $w11, implicit-def dead $w12, implicit-def dead $w13, implicit-def dead $w14, implicit-def dead $w15, implicit-def dead $q0, implicit-def dead $q1, implicit-def dead $q2, implicit-def dead $q3, implicit-def $r0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29
+  ; CHECK-NEXT:   renamable $r1 = COPY renamable $r17
+  ; CHECK-NEXT:   renamable $d0 = S2_extractup killed renamable $d0, 6, 25
+  ; CHECK-NEXT:   renamable $d11 = A2_tfrpi -1
+  ; CHECK-NEXT:   renamable $d1 = A2_tfrpi 2
+  ; CHECK-NEXT:   renamable $d11 = S2_asl_r_p_acc killed renamable $d11, killed renamable $d1, killed renamable $r0
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29
+  ; CHECK-NEXT:   J2_call @lrand48, implicit-def dead $d0, implicit-def dead $d1, implicit-def dead $d2, implicit-def dead $d3, implicit-def dead $d4, implicit-def dead $d5, implicit-def dead $d6, implicit-def dead $d7, implicit-def dead $r28, implicit-def dead $r31, implicit-def dead $p0, implicit-def dead $p1, implicit-def dead $p2, implicit-def dead $p3, implicit-def dead $m0, implicit-def dead $m1, implicit-def dead $lc0, implicit-def dead $lc1, implicit-def dead $sa0, implicit-def dead $sa1, implicit-def dead $usr, implicit-def $usr_ovf, implicit-def dead $cs0, implicit-def dead $cs1, implicit-def dead $w0, implicit-def dead $w1, implicit-def dead $w2, implicit-def dead $w3, implicit-def dead $w4, implicit-def dead $w5, implicit-def dead $w6, implicit-def dead $w7, implicit-def dead $w8, implicit-def dead $w9, implicit-def dead $w10, implicit-def dead $w11, implicit-def dead $w12, implicit-def dead $w13, implicit-def dead $w14, implicit-def dead $w15, implicit-def dead $q0, implicit-def dead $q1, implicit-def dead $q2, implicit-def dead $q3
+  ; CHECK-NEXT:   renamable $d2 = L2_loadrd_io %stack.0, 0 :: (load (s64) from %stack.0)
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29
+  ; CHECK-NEXT:   renamable $d0 = S2_asl_r_p killed renamable $d11, killed renamable $r4
+  ; CHECK-NEXT:   renamable $d1 = COPY renamable $d13
+  ; CHECK-NEXT:   renamable $d1 = S2_lsr_i_p_and killed renamable $d1, killed renamable $d9, 9
+  ; CHECK-NEXT:   renamable $d0 = S2_asl_i_p_and killed renamable $d0, killed renamable $d12, 42
+  ; CHECK-NEXT:   S2_storerd_io undef renamable $r0, 0, killed renamable $d0 :: (store (s64) into `ptr undef`)
+  ; CHECK-NEXT:   renamable $d0 = A2_tfrpi 0
+  ; CHECK-NEXT:   renamable $p0 = C2_cmpeqp killed renamable $d1, killed renamable $d0
+  ; CHECK-NEXT:   J2_jumpt killed renamable $p0, %bb.3, implicit-def dead $pc
+  ; CHECK-NEXT:   J2_jump %bb.2, implicit-def dead $pc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.if.end82:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $d8:0x0000000000000001, $d13, $r20
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r20 = A2_addi killed renamable $r20, -1
+  ; CHECK-NEXT:   renamable $p0 = C2_cmpeqi renamable $r20, 0
+  ; CHECK-NEXT:   J2_jumpf killed renamable $p0, %bb.1, implicit-def dead $pc
+  ; CHECK-NEXT:   J2_jump %bb.3, implicit-def dead $pc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.for.end:
   bb.0.entry:
     successors: %bb.1.for.body
 
@@ -153,11 +211,6 @@ body:             |
     %13 = S2_asl_r_p_acc %13, %47, %8.isub_lo
     %51 = A2_tfrpi 0
 
-    ; CHECK: $d2 = S2_extractup undef renamable $d0, 6, 25
-    ; CHECK: $d0 = A2_tfrpi 2
-    ; CHECK: $d13 = A2_tfrpi -1
-    ; CHECK-NOT: undef $r4
-
   bb.1.for.body:
     successors: %bb.3.for.end, %bb.2.if.end82
 
diff --git a/llvm/test/CodeGen/Hexagon/trap-crash.ll b/llvm/test/CodeGen/Hexagon/trap-crash.ll
index e940d98b620c5..a0afe5fdd3041 100644
--- a/llvm/test/CodeGen/Hexagon/trap-crash.ll
+++ b/llvm/test/CodeGen/Hexagon/trap-crash.ll
@@ -1,20 +1,25 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon --verify-machineinstrs < %s | FileCheck %s
 
 ; Generate code that is guaranteed to crash. At the moment, it's a
 ; misaligned load.
+; CHECK-LABEL: f0
 ; CHECK: memd(##3134984174)
 
 target triple = "hexagon"
 
-; Function Attrs: noreturn nounwind
-define i32 @f0() #0 {
+define i32 @f0() noreturn nounwind  {
 entry:
   tail call void @llvm.trap()
   unreachable
 }
 
-; Function Attrs: cold noreturn nounwind
-declare void @llvm.trap() #1
+; CHECK-LABEL: f1
+; CHECK: brkpt
+define i32 @f1() noreturn nounwind {
+entry:
+  tail call void @llvm.debugtrap()
+  unreachable
+}
 
-attributes #0 = { noreturn nounwind "target-cpu"="hexagonv60" }
-attributes #1 = { cold noreturn nounwind }
+declare void @llvm.trap() nounwind
+declare void @llvm.debugtrap() nounwind
diff --git a/llvm/test/CodeGen/LoongArch/fp16-promote.ll b/llvm/test/CodeGen/LoongArch/fp16-promote.ll
index 03965ac81f376..c5a27a7011278 100644
--- a/llvm/test/CodeGen/LoongArch/fp16-promote.ll
+++ b/llvm/test/CodeGen/LoongArch/fp16-promote.ll
@@ -378,3 +378,84 @@ define half @freeze_half_poison(half %maybe.poison) nounwind {
   %t1 = fadd half %y1, %y1
   ret half %t1
 }
+
+define signext i32 @test_half_to_s32(half %a) nounwind {
+; LA32-LABEL: test_half_to_s32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_half_to_s32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+entry:
+  %conv = fptosi half %a to i32
+  ret i32 %conv
+}
+
+define zeroext i32 @test_half_to_s32_u32(half %a) nounwind {
+; LA32-LABEL: test_half_to_s32_u32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_half_to_s32_u32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+entry:
+  %conv = fptosi half %a to i32
+  ret i32 %conv
+}
+
+define i64 @test_half_to_i64(half %a) nounwind {
+; LA32-LABEL: test_half_to_i64:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA32-NEXT:    bl %plt(__fixsfdi)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_half_to_i64:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    bl %plt(__gnu_h2f_ieee)
+; LA64-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+entry:
+  %conv = fptosi half %a to i64
+  ret i64 %conv
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
index 4ff15f2b7e448..159ffa8c7238a 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc --mtriple=loongarch64 -mattr=+d,-ld-seq-sa < %s | FileCheck %s --check-prefixes=LA64,NO-LD-SEQ-SA
 ; RUN: llc --mtriple=loongarch64 -mattr=+d,+ld-seq-sa < %s | FileCheck %s --check-prefixes=LA64,LD-SEQ-SA
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lamcas < %s | FileCheck %s --check-prefix=LA64-LAMCAS
 
 define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i8_acquire_acquire:
@@ -27,6 +28,11 @@ define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB0_4:
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_acquire_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
   ret void
 }
@@ -57,6 +63,11 @@ define void @cmpxchg_i16_acquire_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind
 ; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB1_4:
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_acquire_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
   ret void
 }
@@ -77,6 +88,11 @@ define void @cmpxchg_i32_acquire_acquire(ptr %ptr, i32 %cmp, i32 %val) nounwind
 ; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB2_4:
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_acquire_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
   ret void
 }
@@ -96,6 +112,11 @@ define void @cmpxchg_i64_acquire_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB3_4:
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_acquire_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.d $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
   ret void
 }
@@ -149,6 +170,11 @@ define void @cmpxchg_i8_acquire_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LD-SEQ-SA-NEXT:  .LBB4_3:
 ; LD-SEQ-SA-NEXT:  .LBB4_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_acquire_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire monotonic
   ret void
 }
@@ -204,6 +230,11 @@ define void @cmpxchg_i16_acquire_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwin
 ; LD-SEQ-SA-NEXT:  .LBB5_3:
 ; LD-SEQ-SA-NEXT:  .LBB5_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_acquire_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire monotonic
   ret void
 }
@@ -239,6 +270,11 @@ define void @cmpxchg_i32_acquire_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwin
 ; LD-SEQ-SA-NEXT:  .LBB6_3:
 ; LD-SEQ-SA-NEXT:  .LBB6_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_acquire_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire monotonic
   ret void
 }
@@ -272,6 +308,11 @@ define void @cmpxchg_i64_acquire_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; LD-SEQ-SA-NEXT:  .LBB7_3:
 ; LD-SEQ-SA-NEXT:  .LBB7_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_acquire_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.d $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire monotonic
   ret void
 }
@@ -302,6 +343,12 @@ define i8 @cmpxchg_i8_acquire_acquire_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind
 ; LA64-NEXT:  .LBB8_4:
 ; LA64-NEXT:    srl.w $a0, $a5, $a3
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_acquire_acquire_reti8:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
   %res = extractvalue { i8, i1 } %tmp, 0
   ret i8 %res
@@ -334,6 +381,12 @@ define i16 @cmpxchg_i16_acquire_acquire_reti16(ptr %ptr, i16 %cmp, i16 %val) nou
 ; LA64-NEXT:  .LBB9_4:
 ; LA64-NEXT:    srl.w $a0, $a5, $a3
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_acquire_acquire_reti16:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
   %res = extractvalue { i16, i1 } %tmp, 0
   ret i16 %res
@@ -356,6 +409,12 @@ define i32 @cmpxchg_i32_acquire_acquire_reti32(ptr %ptr, i32 %cmp, i32 %val) nou
 ; LA64-NEXT:  .LBB10_4:
 ; LA64-NEXT:    move $a0, $a1
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_acquire_acquire_reti32:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
   %res = extractvalue { i32, i1 } %tmp, 0
   ret i32 %res
@@ -377,6 +436,12 @@ define i64 @cmpxchg_i64_acquire_acquire_reti64(ptr %ptr, i64 %cmp, i64 %val) nou
 ; LA64-NEXT:  .LBB11_4:
 ; LA64-NEXT:    move $a0, $a3
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_acquire_acquire_reti64:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas_db.d $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
   %res = extractvalue { i64, i1 } %tmp, 0
   ret i64 %res
@@ -410,6 +475,14 @@ define i1 @cmpxchg_i8_acquire_acquire_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind
 ; LA64-NEXT:    xor $a0, $a1, $a0
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_acquire_acquire_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas_db.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
   %res = extractvalue { i8, i1 } %tmp, 1
   ret i1 %res
@@ -444,6 +517,14 @@ define i1 @cmpxchg_i16_acquire_acquire_reti1(ptr %ptr, i16 %cmp, i16 %val) nounw
 ; LA64-NEXT:    xor $a0, $a1, $a0
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_acquire_acquire_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas_db.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
   %res = extractvalue { i16, i1 } %tmp, 1
   ret i1 %res
@@ -467,6 +548,14 @@ define i1 @cmpxchg_i32_acquire_acquire_reti1(ptr %ptr, i32 %cmp, i32 %val) nounw
 ; LA64-NEXT:    xor $a0, $a3, $a1
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_acquire_acquire_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    addi.w $a3, $a1, 0
+; LA64-LAMCAS-NEXT:    amcas_db.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
   %res = extractvalue { i32, i1 } %tmp, 1
   ret i1 %res
@@ -489,6 +578,14 @@ define i1 @cmpxchg_i64_acquire_acquire_reti1(ptr %ptr, i64 %cmp, i64 %val) nounw
 ; LA64-NEXT:    xor $a0, $a3, $a1
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_acquire_acquire_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas_db.d $a3, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a3, $a1
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
   %res = extractvalue { i64, i1 } %tmp, 1
   ret i1 %res
@@ -543,6 +640,11 @@ define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind
 ; LD-SEQ-SA-NEXT:  .LBB16_3:
 ; LD-SEQ-SA-NEXT:  .LBB16_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_monotonic_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
   ret void
 }
@@ -598,6 +700,11 @@ define void @cmpxchg_i16_monotonic_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounw
 ; LD-SEQ-SA-NEXT:  .LBB17_3:
 ; LD-SEQ-SA-NEXT:  .LBB17_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_monotonic_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
   ret void
 }
@@ -633,6 +740,11 @@ define void @cmpxchg_i32_monotonic_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounw
 ; LD-SEQ-SA-NEXT:  .LBB18_3:
 ; LD-SEQ-SA-NEXT:  .LBB18_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_monotonic_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
   ret void
 }
@@ -666,6 +778,11 @@ define void @cmpxchg_i64_monotonic_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounw
 ; LD-SEQ-SA-NEXT:  .LBB19_3:
 ; LD-SEQ-SA-NEXT:  .LBB19_4:
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_monotonic_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.d $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
   ret void
 }
@@ -721,6 +838,12 @@ define i8 @cmpxchg_i8_monotonic_monotonic_reti8(ptr %ptr, i8 %cmp, i8 %val) noun
 ; LD-SEQ-SA-NEXT:  .LBB20_4:
 ; LD-SEQ-SA-NEXT:    srl.w $a0, $a5, $a3
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_monotonic_monotonic_reti8:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
   %res = extractvalue { i8, i1 } %tmp, 0
   ret i8 %res
@@ -779,6 +902,12 @@ define i16 @cmpxchg_i16_monotonic_monotonic_reti16(ptr %ptr, i16 %cmp, i16 %val)
 ; LD-SEQ-SA-NEXT:  .LBB21_4:
 ; LD-SEQ-SA-NEXT:    srl.w $a0, $a5, $a3
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_monotonic_monotonic_reti16:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
   %res = extractvalue { i16, i1 } %tmp, 0
   ret i16 %res
@@ -817,6 +946,12 @@ define i32 @cmpxchg_i32_monotonic_monotonic_reti32(ptr %ptr, i32 %cmp, i32 %val)
 ; LD-SEQ-SA-NEXT:  .LBB22_4:
 ; LD-SEQ-SA-NEXT:    move $a0, $a1
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_monotonic_monotonic_reti32:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
   %res = extractvalue { i32, i1 } %tmp, 0
   ret i32 %res
@@ -853,6 +988,12 @@ define i64 @cmpxchg_i64_monotonic_monotonic_reti64(ptr %ptr, i64 %cmp, i64 %val)
 ; LD-SEQ-SA-NEXT:  .LBB23_4:
 ; LD-SEQ-SA-NEXT:    move $a0, $a3
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_monotonic_monotonic_reti64:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    amcas.d $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    move $a0, $a1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
   %res = extractvalue { i64, i1 } %tmp, 0
   ret i64 %res
@@ -913,6 +1054,14 @@ define i1 @cmpxchg_i8_monotonic_monotonic_reti1(ptr %ptr, i8 %cmp, i8 %val) noun
 ; LD-SEQ-SA-NEXT:    xor $a0, $a1, $a0
 ; LD-SEQ-SA-NEXT:    sltui $a0, $a0, 1
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i8_monotonic_monotonic_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas.b $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
   %res = extractvalue { i8, i1 } %tmp, 1
   ret i1 %res
@@ -975,6 +1124,14 @@ define i1 @cmpxchg_i16_monotonic_monotonic_reti1(ptr %ptr, i16 %cmp, i16 %val) n
 ; LD-SEQ-SA-NEXT:    xor $a0, $a1, $a0
 ; LD-SEQ-SA-NEXT:    sltui $a0, $a0, 1
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i16_monotonic_monotonic_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas.h $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
   %res = extractvalue { i16, i1 } %tmp, 1
   ret i1 %res
@@ -1015,6 +1172,14 @@ define i1 @cmpxchg_i32_monotonic_monotonic_reti1(ptr %ptr, i32 %cmp, i32 %val) n
 ; LD-SEQ-SA-NEXT:    xor $a0, $a3, $a1
 ; LD-SEQ-SA-NEXT:    sltui $a0, $a0, 1
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i32_monotonic_monotonic_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    addi.w $a3, $a1, 0
+; LA64-LAMCAS-NEXT:    amcas.w $a1, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a1, $a3
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
   %res = extractvalue { i32, i1 } %tmp, 1
   ret i1 %res
@@ -1053,6 +1218,14 @@ define i1 @cmpxchg_i64_monotonic_monotonic_reti1(ptr %ptr, i64 %cmp, i64 %val) n
 ; LD-SEQ-SA-NEXT:    xor $a0, $a3, $a1
 ; LD-SEQ-SA-NEXT:    sltui $a0, $a0, 1
 ; LD-SEQ-SA-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: cmpxchg_i64_monotonic_monotonic_reti1:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a3, $a1
+; LA64-LAMCAS-NEXT:    amcas.d $a3, $a2, $a0
+; LA64-LAMCAS-NEXT:    xor $a0, $a3, $a1
+; LA64-LAMCAS-NEXT:    sltui $a0, $a0, 1
+; LA64-LAMCAS-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
   %res = extractvalue { i64, i1 } %tmp, 1
   ret i1 %res
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-lamcas.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-lamcas.ll
new file mode 100644
index 0000000000000..728dd778bdf24
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-lamcas.ll
@@ -0,0 +1,6975 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch64 -mattr=+d,-lamcas < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lamcas < %s | FileCheck %s --check-prefix=LA64-LAMCAS
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lamcas,+lam-bh < %s | FileCheck %s --check-prefix=LA64-LAMCAS-LAM-BH
+
+; i8/i16 atomicrmw and/or/xor shouldn't use amcas[_db].b/h to expand
+; lam-bh precedes lamcas
+
+define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB0_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.bu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB0_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.b $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB0_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_acquire(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB1_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB1_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 acquire
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_acquire(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB2_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB2_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB3_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.hu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB3_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.h $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB3_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_acquire(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB4_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB4_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 acquire
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_acquire(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB5_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB5_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB6_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB6_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB6_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB7_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB7_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB7_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB8_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB8_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB8_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB9_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB9_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB9_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB10_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB10_3: # in Loop: Header=BB10_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB10_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB10_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB10_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB10_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB10_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB11_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB11_3: # in Loop: Header=BB11_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB11_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB11_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB11_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB11_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB11_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB12_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB12_3: # in Loop: Header=BB12_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB12_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB12_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB12_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB12_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB12_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB13_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB13_3: # in Loop: Header=BB13_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB13_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB13_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB13_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB13_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB13_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB14_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB14_3: # in Loop: Header=BB14_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB14_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB14_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB14_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB14_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB14_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB15_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB15_3: # in Loop: Header=BB15_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB15_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB15_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB15_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB15_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB15_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB16_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB16_3: # in Loop: Header=BB16_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB16_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB16_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB16_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB16_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB16_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB17_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB17_3: # in Loop: Header=BB17_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB17_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB17_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB17_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB17_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB17_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB18_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB18_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB18_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB18_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB18_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB19_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB19_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB19_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB19_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB19_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i32_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB20_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i32_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB20_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB20_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i32_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB20_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB20_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i32 %b acquire
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i64_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB21_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i64_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB21_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB21_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i64_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB21_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB21_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b acquire
+  ret i64 %1
+}
+
+define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i8_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i8_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i16_acquire:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i16_acquire:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB28_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.bu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB28_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.b $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB28_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_release(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB29_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB29_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 release
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_release(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB30_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB30_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB31_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB31_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.hu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB31_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.h $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB31_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_release(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB32_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB32_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 release
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_release(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB33_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB33_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB34_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB34_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB34_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB34_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB35_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB35_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB35_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB36_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB36_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB36_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB37_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB37_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB37_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB38_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB38_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB38_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB38_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB38_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB38_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB39_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB39_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB39_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB39_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB39_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB39_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB40_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB40_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB40_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB40_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB40_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB40_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB41_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB41_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB41_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB41_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB41_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB41_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB42_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB42_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB42_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB42_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB42_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB42_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB43_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB43_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB43_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB43_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB43_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB43_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB44_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB44_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB44_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB44_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB44_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB44_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB45_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB45_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB45_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB45_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB45_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB45_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB46_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB46_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB46_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB46_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB46_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB47_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB47_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB47_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB47_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB47_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i32 @atomicrmw_nand_i32_release(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB48_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i32_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB48_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB48_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i32_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB48_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB48_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+    %1 = atomicrmw nand ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_release(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB49_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i64_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB49_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB49_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i64_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB49_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB49_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i8_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i8_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i16_release:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i16_release:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b release
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB56_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.bu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB56_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.b $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB56_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_acq_rel(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB57_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB57_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 acq_rel
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_acq_rel(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB58_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB58_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB59_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.hu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB59_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.h $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB59_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_acq_rel(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB60_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB60_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 acq_rel
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_acq_rel(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB61_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB61_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB62_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB62_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB62_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB63_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB63_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB63_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB64_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB64_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB64_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB65_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB65_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB65_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB66_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB66_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB66_3: # in Loop: Header=BB66_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB66_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB66_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB66_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB66_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB66_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB67_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB67_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB67_3: # in Loop: Header=BB67_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB67_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB67_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB67_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB67_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB67_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB68_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB68_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB68_3: # in Loop: Header=BB68_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB68_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB68_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB68_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB68_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB68_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB69_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB69_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB69_3: # in Loop: Header=BB69_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB69_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB69_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB69_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB69_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB69_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB70_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB70_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB70_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB70_3: # in Loop: Header=BB70_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB70_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB70_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB70_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB70_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB70_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB71_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB71_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB71_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB71_3: # in Loop: Header=BB71_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB71_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB71_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB71_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB71_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB71_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB72_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB72_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB72_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB72_3: # in Loop: Header=BB72_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB72_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB72_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB72_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB72_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB72_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB73_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB73_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB73_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB73_3: # in Loop: Header=BB73_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB73_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB73_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB73_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB73_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB73_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB74_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB74_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB74_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB74_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB74_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB74_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB75_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB75_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB75_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB75_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB75_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i32 @atomicrmw_nand_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB76_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i32_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB76_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB76_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i32_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB76_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB76_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+    %1 = atomicrmw nand ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB77_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i64_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB77_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB77_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i64_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB77_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB77_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+
+
+define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i8_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i8_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i16_acq_rel:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i16_acq_rel:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b acq_rel
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB84_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB84_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.bu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB84_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.b $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB84_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_seq_cst(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB85_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB85_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 seq_cst
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_seq_cst(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB86_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB86_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB87_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB87_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.hu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB87_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas_db.h $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB87_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_seq_cst(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB88_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB88_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 seq_cst
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_seq_cst(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB89_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB89_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB90_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB90_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB90_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB91_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB91_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB91_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB92_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB92_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB92_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB93_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB93_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB93_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd_db.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB94_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB94_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB94_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB94_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB94_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB94_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB95_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB95_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB95_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB95_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB95_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB95_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB96_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB96_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB96_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB96_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB96_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB96_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB97_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB97_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB97_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB97_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB97_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB97_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB98_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB98_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB98_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB98_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB98_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB98_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB99_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB99_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB99_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB99_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB99_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB99_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB100_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB100_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB100_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB100_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB100_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB100_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB101_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB101_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB101_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB101_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB101_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB101_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB102_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB102_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB102_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB102_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.b $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB102_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB103_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB103_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB103_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB103_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.h $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB103_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i32 @atomicrmw_nand_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB104_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i32_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB104_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB104_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i32_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB104_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.w $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB104_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+    %1 = atomicrmw nand ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB105_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i64_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB105_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB105_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i64_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB105_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas_db.d $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB105_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+
+
+define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i8_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i8_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i16_seq_cst:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i16_seq_cst:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b seq_cst
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB112_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB112_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.bu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB112_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas.b $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB112_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_monotonic(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB113_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB113_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.b $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 monotonic
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_monotonic(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB114_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB114_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xchg_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB115_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB115_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    ld.hu $a2, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB115_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a2
+; LA64-LAMCAS-NEXT:    amcas.h $a2, $a1, $a0
+; LA64-LAMCAS-NEXT:    bne $a2, $a3, .LBB115_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    move $a0, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_monotonic(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_0_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    amand.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_0_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB116_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a2, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $zero, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a2, .LBB116_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_0_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.h $a1, $zero, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 monotonic
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_monotonic(ptr %a) nounwind {
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    amor.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xchg_minus_1_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a1, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB117_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a2, $a1
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB117_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xchg_minus_1_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    addi.w $a2, $zero, -1
+; LA64-LAMCAS-LAM-BH-NEXT:    amswap.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 monotonic
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB118_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB118_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB118_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB118_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd.b $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_add_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB119_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB119_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_add_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB119_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    add.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB119_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_add_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd.h $a2, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB120_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB120_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB120_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB120_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd.b $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_sub_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB121_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB121_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_sub_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB121_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    sub.d $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB121_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_sub_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    sub.w $a2, $zero, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    amadd.h $a1, $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB122_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB122_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB122_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB122_3: # in Loop: Header=BB122_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB122_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB122_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB122_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB122_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB122_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB123_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB123_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB123_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB123_3: # in Loop: Header=BB123_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB123_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umax_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB123_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB123_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umax_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB123_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB123_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB124_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB124_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB124_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB124_3: # in Loop: Header=BB124_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB124_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB124_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB124_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a3, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB124_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a4, $a0, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.b $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB124_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB125_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB125_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB125_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB125_3: # in Loop: Header=BB125_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB125_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_umin_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB125_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a5, .LBB125_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_umin_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a3, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB125_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a4, $a0, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sltu $a4, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a4, $a4, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a5, $a1, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a4, $a0, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a4, $a4, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a5, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.h $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a5, .LBB125_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB126_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB126_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB126_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB126_3: # in Loop: Header=BB126_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB126_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB126_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB126_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB126_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB126_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB127_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB127_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB127_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB127_3: # in Loop: Header=BB127_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB127_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_max_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB127_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB127_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_max_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB127_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB127_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    xori $a3, $a3, 56
+; LA64-NEXT:  .LBB128_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB128_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB128_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB128_3: # in Loop: Header=BB128_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB128_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB128_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB128_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB128_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.b $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB128_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    ori $a5, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a5, $a3
+; LA64-NEXT:  .LBB129_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB129_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB129_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB129_3: # in Loop: Header=BB129_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB129_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_min_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB129_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a5, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB129_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_min_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a3, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB129_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    slt $a5, $a3, $a4
+; LA64-LAMCAS-LAM-BH-NEXT:    xori $a5, $a5, 1
+; LA64-LAMCAS-LAM-BH-NEXT:    masknez $a6, $a1, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    maskeqz $a5, $a0, $a5
+; LA64-LAMCAS-LAM-BH-NEXT:    or $a5, $a5, $a6
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.h $a0, $a5, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB129_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+
+
+define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB130_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB130_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB130_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.b $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB130_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.bu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB130_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.b $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.b $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB130_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:  .LBB131_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB131_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB131_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-NEXT:    amcas.h $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a4, .LBB131_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.hu $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB131_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a3, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a3, $a3, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    ext.w.h $a4, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.h $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a4, .LBB131_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
+
+define i32 @atomicrmw_nand_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i32_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB132_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB132_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i32_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB132_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas.w $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB132_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i32_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.w $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB132_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.w $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB132_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+    %1 = atomicrmw nand ptr %a, i32 %b monotonic
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_nand_i64_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB133_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB133_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_nand_i64_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    move $a2, $a0
+; LA64-LAMCAS-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-NEXT:  .LBB133_1: # %atomicrmw.start
+; LA64-LAMCAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-NEXT:    move $a3, $a0
+; LA64-LAMCAS-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-NEXT:    amcas.d $a0, $a4, $a2
+; LA64-LAMCAS-NEXT:    bne $a0, $a3, .LBB133_1
+; LA64-LAMCAS-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_nand_i64_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a2, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    ld.d $a0, $a0, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    .p2align 4, , 16
+; LA64-LAMCAS-LAM-BH-NEXT:  .LBB133_1: # %atomicrmw.start
+; LA64-LAMCAS-LAM-BH-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64-LAMCAS-LAM-BH-NEXT:    move $a3, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    and $a4, $a0, $a1
+; LA64-LAMCAS-LAM-BH-NEXT:    nor $a4, $a4, $zero
+; LA64-LAMCAS-LAM-BH-NEXT:    amcas.d $a0, $a4, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bne $a0, $a3, .LBB133_1
+; LA64-LAMCAS-LAM-BH-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b monotonic
+  ret i64 %1
+}
+
+
+
+define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $zero, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_and_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    amand.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_and_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-NEXT:    amand.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_and_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    lu12i.w $a3, 15
+; LA64-LAMCAS-LAM-BH-NEXT:    ori $a3, $a3, 4095
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a3, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    orn $a1, $a1, $a3
+; LA64-LAMCAS-LAM-BH-NEXT:    amand.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_or_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amor.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_or_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amor.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_or_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amor.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i8_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i8_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    andi $a1, $a1, 255
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_xor_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+;
+; LA64-LAMCAS-LABEL: atomicrmw_xor_i16_monotonic:
+; LA64-LAMCAS:       # %bb.0:
+; LA64-LAMCAS-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-LAMCAS-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-NEXT:    ret
+;
+; LA64-LAMCAS-LAM-BH-LABEL: atomicrmw_xor_i16_monotonic:
+; LA64-LAMCAS-LAM-BH:       # %bb.0:
+; LA64-LAMCAS-LAM-BH-NEXT:    slli.d $a2, $a0, 3
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-LAMCAS-LAM-BH-NEXT:    sll.w $a1, $a1, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    amxor.w $a3, $a1, $a0
+; LA64-LAMCAS-LAM-BH-NEXT:    srl.w $a0, $a3, $a2
+; LA64-LAMCAS-LAM-BH-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b monotonic
+  ret i16 %1
+
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem-div32.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem-div32.ll
index 1d81a367ffdc5..9276195b6fa0b 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem-div32.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem-div32.ll
@@ -73,3 +73,25 @@ define i32 @modwu(i64 %a, i64 %b) {
   %r = urem i32 %conv1, %conv2
   ret i32 %r
 }
+
+define signext i32 @sextw_rmv(i32 signext %a, i32 signext %b, i32 signext %c) {
+; LA64-LABEL: sextw_rmv:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    mul.w $a0, $a1, $a0
+; LA64-NEXT:    div.w $a1, $a2, $a0
+; LA64-NEXT:    sltu $a0, $a1, $a0
+; LA64-NEXT:    ret
+;
+; LA64-DIV32-LABEL: sextw_rmv:
+; LA64-DIV32:       # %bb.0: # %entry
+; LA64-DIV32-NEXT:    mul.w $a0, $a1, $a0
+; LA64-DIV32-NEXT:    div.w $a1, $a2, $a0
+; LA64-DIV32-NEXT:    sltu $a0, $a1, $a0
+; LA64-DIV32-NEXT:    ret
+entry:
+  %mul = mul nsw i32 %b, %a
+  %div = sdiv i32 %c, %mul
+  %cmp = icmp ult i32 %div, %mul
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/bitreverse.ll b/llvm/test/CodeGen/LoongArch/lasx/bitreverse.ll
new file mode 100644
index 0000000000000..11f1bce55fad6
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/bitreverse.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+lasx --verify-machineinstrs < %s \
+; RUN:   | FileCheck %s
+
+declare <32 x i8> @llvm.bitreverse.v32i8(<32 x i8>)
+
+define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %b = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a)
+  ret <32 x i8> %b
+}
+
+declare <16 x i16> @llvm.bitreverse.v16i16(<16 x i16>)
+
+define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    xvshuf4i.h $xr0, $xr1, 27
+; CHECK-NEXT:    ret
+  %b = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a)
+  ret <16 x i16> %b
+}
+
+declare <8 x i32> @llvm.bitreverse.v8i32(<8 x i32>)
+
+define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    xvshuf4i.w $xr0, $xr1, 177
+; CHECK-NEXT:    ret
+  %b = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a)
+  ret <8 x i32> %b
+}
+
+declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>)
+
+define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %b = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a)
+  ret <4 x i64> %b
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/bitreverse.ll b/llvm/test/CodeGen/LoongArch/lsx/bitreverse.ll
new file mode 100644
index 0000000000000..4c17d3fd8d7b2
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/bitreverse.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+lsx --verify-machineinstrs < %s \
+; RUN:   | FileCheck %s
+
+declare <16 x i8> @llvm.bitreverse.v16i8(<16 x i8>)
+
+define <16 x i8> @test_bitreverse_v16i8(<16 x i8> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    bitrev.8b $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 1
+; CHECK-NEXT:    vori.b $vr0, $vr1, 0
+; CHECK-NEXT:    ret
+  %b = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a)
+  ret <16 x i8> %b
+}
+
+declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
+
+define <8 x i16> @test_bitreverse_v8i16(<8 x i16> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 1
+; CHECK-NEXT:    vshuf4i.h $vr0, $vr1, 27
+; CHECK-NEXT:    ret
+  %b = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a)
+  ret <8 x i16> %b
+}
+
+declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
+
+define <4 x i32> @test_bitreverse_v4i32(<4 x i32> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 1
+; CHECK-NEXT:    vshuf4i.w $vr0, $vr1, 177
+; CHECK-NEXT:    ret
+  %b = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a)
+  ret <4 x i32> %b
+}
+
+declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
+
+define <2 x i64> @test_bitreverse_v2i64(<2 x i64> %a) nounwind {
+; CHECK-LABEL: test_bitreverse_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    bitrev.d $a0, $a0
+; CHECK-NEXT:    vinsgr2vr.d $vr1, $a0, 1
+; CHECK-NEXT:    vori.b $vr0, $vr1, 0
+; CHECK-NEXT:    ret
+  %b = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a)
+  ret <2 x i64> %b
+}
diff --git a/llvm/test/CodeGen/LoongArch/sextw-removal.ll b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
index 96853105049b4..0aeafadb9325b 100644
--- a/llvm/test/CodeGen/LoongArch/sextw-removal.ll
+++ b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
@@ -1314,4 +1314,23 @@ bb7:                                              ; preds = %bb2
   ret i32 %trunc
 }
 
- declare void @side_effect(i64)
+declare void @side_effect(i64)
+
+declare i32 @llvm.loongarch.lsx.vpickve2gr.w(<4 x i32>, i32)
+
+define signext i32 @test20(<4 x i32> %v) {
+; CHECK-LABEL: test20:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 3
+; CHECK-NEXT:    addi.w $a0, $a0, 0
+; CHECK-NEXT:    ret
+;
+; NORMV-LABEL: test20:
+; NORMV:       # %bb.0: # %entry
+; NORMV-NEXT:    vpickve2gr.w $a0, $vr0, 3
+; NORMV-NEXT:    addi.w $a0, $a0, 0
+; NORMV-NEXT:    ret
+entry:
+  %a = call i32 @llvm.loongarch.lsx.vpickve2gr.w(<4 x i32> %v, i32 3)
+  ret i32 %a
+}
diff --git a/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll b/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll
new file mode 100644
index 0000000000000..90d2be017ecdb
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=m68k-linux  -mcpu=M68020 --verify-machineinstrs | FileCheck %s
+
+define internal void @select_i32(i32 %self, ptr nonnull %value) {
+; CHECK-LABEL: select_i32:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #4, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -8
+; CHECK-NEXT:    movem.l %d2, (0,%sp) ; 8-byte Folded Spill
+; CHECK-NEXT:    cmpi.l #0, (8,%sp)
+; CHECK-NEXT:    move.w %ccr, %d2
+; CHECK-NEXT:    sne %d1
+; CHECK-NEXT:    move.l (12,%sp), %d0
+; CHECK-NEXT:    move.w %d2, %ccr
+; CHECK-NEXT:    bne .LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %start
+; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    cmpi.l #0, %d1
+; CHECK-NEXT:    bne .LBB0_3
+; CHECK-NEXT:  .LBB0_2: ; %null
+; CHECK-NEXT:    suba.l %a0, %a0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:  .LBB0_3: ; %exit
+; CHECK-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
+; CHECK-NEXT:    adda.l #4, %sp
+; CHECK-NEXT:    rts
+start:
+  %2 = icmp eq i32 %self, 0
+  %3 = select i1 %2, i32 0, i32 1
+  switch i32 %3, label %exit [
+    i32 0, label %nonnull
+    i32 1, label %null
+  ]
+
+nonnull:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+null:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+exit:                                              ; preds = %nonnull, %null
+  ret void
+}
+
+define internal void @select_i16(i16 %self, ptr nonnull %value) {
+; CHECK-LABEL: select_i16:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    suba.l #4, %sp
+; CHECK-NEXT:    .cfi_def_cfa_offset -8
+; CHECK-NEXT:    movem.l %d2, (0,%sp) ; 8-byte Folded Spill
+; CHECK-NEXT:    cmpi.w #0, (10,%sp)
+; CHECK-NEXT:    move.w %ccr, %d2
+; CHECK-NEXT:    sne %d1
+; CHECK-NEXT:    move.l (12,%sp), %d0
+; CHECK-NEXT:    move.w %d2, %ccr
+; CHECK-NEXT:    bne .LBB1_2
+; CHECK-NEXT:  ; %bb.1: ; %start
+; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    cmpi.w #0, %d1
+; CHECK-NEXT:    bne .LBB1_3
+; CHECK-NEXT:  .LBB1_2: ; %null
+; CHECK-NEXT:    suba.l %a0, %a0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:  .LBB1_3: ; %exit
+; CHECK-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
+; CHECK-NEXT:    adda.l #4, %sp
+; CHECK-NEXT:    rts
+start:
+  %2 = icmp eq i16 %self, 0
+  %3 = select i1 %2, i16 0, i16 1
+  switch i16 %3, label %exit [
+    i16 0, label %nonnull
+    i16 1, label %null
+  ]
+
+nonnull:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+null:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+exit:                                              ; preds = %nonnull, %null
+  ret void
+}
+
+define internal void @select_i8(i8 %self, ptr nonnull %value) {
+; CHECK-LABEL: select_i8:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %start
+; CHECK-NEXT:    move.l (8,%sp), %d0
+; CHECK-NEXT:    cmpi.b #0, (7,%sp)
+; CHECK-NEXT:    sne %d1
+; CHECK-NEXT:    bne .LBB2_2
+; CHECK-NEXT:  ; %bb.1: ; %start
+; CHECK-NEXT:    cmpi.b #0, %d1
+; CHECK-NEXT:    bne .LBB2_3
+; CHECK-NEXT:  .LBB2_2: ; %null
+; CHECK-NEXT:    suba.l %a0, %a0
+; CHECK-NEXT:    move.l %d0, (%a0)
+; CHECK-NEXT:  .LBB2_3: ; %exit
+; CHECK-NEXT:    rts
+start:
+  %2 = icmp eq i8 %self, 0
+  %3 = select i1 %2, i8 0, i8 1
+  switch i8 %3, label %exit [
+    i8 0, label %nonnull
+    i8 1, label %null
+  ]
+
+nonnull:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+null:                                              ; preds = %start
+  store ptr %value, ptr null, align 2
+  br label %exit
+
+exit:                                              ; preds = %nonnull, %null
+  ret void
+}
diff --git a/llvm/test/CodeGen/MIR/Mips/expected-global-value-or-symbol-after-call-entry.mir b/llvm/test/CodeGen/MIR/Mips/expected-global-value-or-symbol-after-call-entry.mir
index 4874410c6a6e9..da0af94b98268 100644
--- a/llvm/test/CodeGen/MIR/Mips/expected-global-value-or-symbol-after-call-entry.mir
+++ b/llvm/test/CodeGen/MIR/Mips/expected-global-value-or-symbol-after-call-entry.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -march=mipsel -mattr=mips16 -relocation-model=pic -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
 --- |
   define i32 @test(i32 %a) {
   entry:
diff --git a/llvm/test/CodeGen/MIR/Mips/memory-operands.mir b/llvm/test/CodeGen/MIR/Mips/memory-operands.mir
index c64b9dacd9a62..f7da11372fa65 100644
--- a/llvm/test/CodeGen/MIR/Mips/memory-operands.mir
+++ b/llvm/test/CodeGen/MIR/Mips/memory-operands.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic -run-pass none -o - %s | FileCheck %s
+# RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic -run-pass none -o - %s | FileCheck %s
 # This test ensures that the MIR parser parses the call entry pseudo source
 # values in memory operands correctly.
 
diff --git a/llvm/test/CodeGen/Mips/2008-06-05-Carry.ll b/llvm/test/CodeGen/Mips/2008-06-05-Carry.ll
index 5e6092fc7848d..d233ef978276f 100644
--- a/llvm/test/CodeGen/Mips/2008-06-05-Carry.ll
+++ b/llvm/test/CodeGen/Mips/2008-06-05-Carry.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 define i64 @add64(i64 %u, i64 %v) nounwind  {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-07-03-SRet.ll b/llvm/test/CodeGen/Mips/2008-07-03-SRet.ll
index 7654ef271fcc9..51d73a319494c 100644
--- a/llvm/test/CodeGen/Mips/2008-07-03-SRet.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-03-SRet.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 %struct.sret0 = type { i32, i32, i32 }
 
diff --git a/llvm/test/CodeGen/Mips/2008-07-07-FPExtend.ll b/llvm/test/CodeGen/Mips/2008-07-07-FPExtend.ll
index 29c8e8446e3d2..dcd3adc3c671e 100644
--- a/llvm/test/CodeGen/Mips/2008-07-07-FPExtend.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-07-FPExtend.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=single-float  < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=single-float  < %s | FileCheck %s
 
 define double @dofloat(float %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-07-07-IntDoubleConvertions.ll b/llvm/test/CodeGen/Mips/2008-07-07-IntDoubleConvertions.ll
index 9a6bbdfb22ff5..a748841afe767 100644
--- a/llvm/test/CodeGen/Mips/2008-07-07-IntDoubleConvertions.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-07-IntDoubleConvertions.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=single-float  < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=single-float  < %s | FileCheck %s
 
 define double @int2fp(i32 %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-07-16-SignExtInReg.ll b/llvm/test/CodeGen/Mips/2008-07-16-SignExtInReg.ll
index 8807d750e4992..e4c1eba9a6ca5 100644
--- a/llvm/test/CodeGen/Mips/2008-07-16-SignExtInReg.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-16-SignExtInReg.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s
 ; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips32r2 -mattr=+mips16 < %s | FileCheck %s
 
 define signext i8 @A(i8 %e.0, i8 signext %sum)  nounwind {
diff --git a/llvm/test/CodeGen/Mips/2008-07-23-fpcmp.ll b/llvm/test/CodeGen/Mips/2008-07-23-fpcmp.ll
index 9c547f15c9a6b..8fe8ac539a90f 100644
--- a/llvm/test/CodeGen/Mips/2008-07-23-fpcmp.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-23-fpcmp.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -o %t
+; RUN: llc < %s -mtriple=mips -o %t
 ; RUN: grep "c\..*\.s" %t | count 3
 ; RUN: grep "bc1[tf]" %t | count 3
 
diff --git a/llvm/test/CodeGen/Mips/2008-07-29-icmp.ll b/llvm/test/CodeGen/Mips/2008-07-29-icmp.ll
index e88e3d3755c4b..e962ec6cfb8b9 100644
--- a/llvm/test/CodeGen/Mips/2008-07-29-icmp.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-29-icmp.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips | grep "b[ne][eq]" | count 1
+; RUN: llc < %s -mtriple=mips | grep "b[ne][eq]" | count 1
 
 ; FIXME: Disabled because branch instructions are generated where
 ; conditional move instructions are expected.
diff --git a/llvm/test/CodeGen/Mips/2008-07-31-fcopysign.ll b/llvm/test/CodeGen/Mips/2008-07-31-fcopysign.ll
index f152acc9a15d2..cfbea9cf6277b 100644
--- a/llvm/test/CodeGen/Mips/2008-07-31-fcopysign.ll
+++ b/llvm/test/CodeGen/Mips/2008-07-31-fcopysign.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -o %t
+; RUN: llc < %s -mtriple=mips -o %t
 ; RUN: grep abs.s  %t | count 1
 ; RUN: grep neg.s %t | count 1
 
diff --git a/llvm/test/CodeGen/Mips/2008-08-01-AsmInline.ll b/llvm/test/CodeGen/Mips/2008-08-01-AsmInline.ll
index 74a3f218379b5..4eaf009500745 100644
--- a/llvm/test/CodeGen/Mips/2008-08-01-AsmInline.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-01-AsmInline.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mcpu=mips32 < %s | FileCheck %s
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
 
 %struct.DWstruct = type { i32, i32 }
 
diff --git a/llvm/test/CodeGen/Mips/2008-08-03-ReturnDouble.ll b/llvm/test/CodeGen/Mips/2008-08-03-ReturnDouble.ll
index 7c19c15ca7bb4..c3931417a758b 100644
--- a/llvm/test/CodeGen/Mips/2008-08-03-ReturnDouble.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-03-ReturnDouble.ll
@@ -1,5 +1,5 @@
 ; Double return in abicall (default)
-; RUN: llc < %s -march=mips
+; RUN: llc < %s -mtriple=mips
 ; PR2615
 
 define double @main(...) {
diff --git a/llvm/test/CodeGen/Mips/2008-08-03-fabs64.ll b/llvm/test/CodeGen/Mips/2008-08-03-fabs64.ll
index 2b1713c39f726..18c01065dec95 100644
--- a/llvm/test/CodeGen/Mips/2008-08-03-fabs64.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-03-fabs64.ll
@@ -1,4 +1,4 @@
-; DISABLED: llc < %s -march=mips -o %t
+; DISABLED: llc < %s -mtriple=mips -o %t
 ; DISABLED: grep {lui.*32767} %t | count 1
 ; DISABLED: grep {ori.*65535} %t | count 1
 ; RUN: false
diff --git a/llvm/test/CodeGen/Mips/2008-08-04-Bitconvert.ll b/llvm/test/CodeGen/Mips/2008-08-04-Bitconvert.ll
index 78a49ffbe4443..8fcda34f889a5 100644
--- a/llvm/test/CodeGen/Mips/2008-08-04-Bitconvert.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-04-Bitconvert.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 define float @A(i32 %u) nounwind  {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-08-06-Alloca.ll b/llvm/test/CodeGen/Mips/2008-08-06-Alloca.ll
index 84d92e3189ea1..e4c648888cce8 100644
--- a/llvm/test/CodeGen/Mips/2008-08-06-Alloca.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-06-Alloca.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 define i32 @twoalloca(i32 %size) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-08-07-CC.ll b/llvm/test/CodeGen/Mips/2008-08-07-CC.ll
index 63c25951423a0..cc7493759d423 100644
--- a/llvm/test/CodeGen/Mips/2008-08-07-CC.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-07-CC.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips
+; RUN: llc < %s -mtriple=mips
 ; Mips must ignore fastcc
 
 target datalayout =
diff --git a/llvm/test/CodeGen/Mips/2008-08-07-FPRound.ll b/llvm/test/CodeGen/Mips/2008-08-07-FPRound.ll
index 4fa43b6833bde..ff15bdf367dfc 100644
--- a/llvm/test/CodeGen/Mips/2008-08-07-FPRound.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-07-FPRound.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=single-float  < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=single-float  < %s | FileCheck %s
 
 define float @round2float(double %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-08-08-ctlz.ll b/llvm/test/CodeGen/Mips/2008-08-08-ctlz.ll
index abd61de5a8d87..e76aef9a13408 100644
--- a/llvm/test/CodeGen/Mips/2008-08-08-ctlz.ll
+++ b/llvm/test/CodeGen/Mips/2008-08-08-ctlz.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 define i32 @A0(i32 %u) nounwind  {
 entry:
diff --git a/llvm/test/CodeGen/Mips/2008-10-13-LegalizerBug.ll b/llvm/test/CodeGen/Mips/2008-10-13-LegalizerBug.ll
index d2e054b17973d..ed3938d2b42f1 100644
--- a/llvm/test/CodeGen/Mips/2008-10-13-LegalizerBug.ll
+++ b/llvm/test/CodeGen/Mips/2008-10-13-LegalizerBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips
+; RUN: llc < %s -mtriple=mips
 ; PR2794
 
 define i32 @main(ptr) nounwind {
diff --git a/llvm/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll b/llvm/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
index d80da658d9283..c2a8eb06fbcec 100644
--- a/llvm/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
+++ b/llvm/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=+soft-float < %s
+; RUN: llc -mtriple=mips -mattr=+soft-float < %s
 ; PR2667
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 	%struct._Bigint = type { ptr, i32, i32, i32, i32, [1 x i32] }
diff --git a/llvm/test/CodeGen/Mips/2010-11-09-CountLeading.ll b/llvm/test/CodeGen/Mips/2010-11-09-CountLeading.ll
index 6174500d3e0b4..05a29ed06bea9 100644
--- a/llvm/test/CodeGen/Mips/2010-11-09-CountLeading.ll
+++ b/llvm/test/CodeGen/Mips/2010-11-09-CountLeading.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 ; CHECK: clz $2, $4
 define i32 @t1(i32 %X) nounwind readnone {
diff --git a/llvm/test/CodeGen/Mips/2010-11-09-Mul.ll b/llvm/test/CodeGen/Mips/2010-11-09-Mul.ll
index dcade3c671db1..52112e83be536 100644
--- a/llvm/test/CodeGen/Mips/2010-11-09-Mul.ll
+++ b/llvm/test/CodeGen/Mips/2010-11-09-Mul.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 ; CHECK: mul $2, $5, $4
 define i32 @mul1(i32 %a, i32 %b) nounwind readnone {
diff --git a/llvm/test/CodeGen/Mips/2012-12-12-ExpandMemcpy.ll b/llvm/test/CodeGen/Mips/2012-12-12-ExpandMemcpy.ll
index 745d57c8e9a2d..9cf5c9a72ea04 100644
--- a/llvm/test/CodeGen/Mips/2012-12-12-ExpandMemcpy.ll
+++ b/llvm/test/CodeGen/Mips/2012-12-12-ExpandMemcpy.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64el -mcpu=mips64r2 < %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 < %s
 
 @.str = private unnamed_addr constant [7 x i8] c"hello\0A\00", align 1
 
diff --git a/llvm/test/CodeGen/Mips/DbgValueOtherTargets.test b/llvm/test/CodeGen/Mips/DbgValueOtherTargets.test
index da20e7ef5224e..4d32a7fb9cbb5 100644
--- a/llvm/test/CodeGen/Mips/DbgValueOtherTargets.test
+++ b/llvm/test/CodeGen/Mips/DbgValueOtherTargets.test
@@ -1 +1 @@
-RUN: llc -O0 -march=mips -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll
+RUN: llc -O0 -mtriple=mips -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
index 2f0f1a04a5588..203202e0bd053 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/br1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @b = global i32 1, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/bricmpi1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/bricmpi1.ll
index b5cf2a2030d29..cb2ff41fa7fb4 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/bricmpi1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/bricmpi1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
 ; RUN:     < %s -verify-machineinstrs | FileCheck %s
 
 define void @testeq(i32, i32) {
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/check-disabled-mcpus.ll b/llvm/test/CodeGen/Mips/Fast-ISel/check-disabled-mcpus.ll
index b8973efda1798..c3b1584a7e9a0 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/check-disabled-mcpus.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/check-disabled-mcpus.ll
@@ -1,39 +1,39 @@
 ; Targets where we should not enable FastISel.
-; RUN: llc -march=mips -mcpu=mips2 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips2 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips3 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips3 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips4 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips4 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
 
-; RUN: llc -march=mips -mcpu=mips32r6 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
 
-; RUN: llc -march=mips -mattr=mips16 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mattr=mips16 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
 
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+micromips -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+micromips -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips32r5 -mattr=+micromips -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r5 -mattr=+micromips -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
 
-; RUN: llc -march=mips -mcpu=mips64 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips64 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips64r2 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips64r2 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips64r3 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips64r3 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips64r5 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips -mcpu=mips64r5 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips32r6 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s
 
 ; Valid targets for FastISel.
-; RUN: llc -march=mips -mcpu=mips32r0 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r0 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s -check-prefix=FISEL
-; RUN: llc -march=mips -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel <%s 2>&1 | FileCheck %s -check-prefix=FISEL
 
 ; The CHECK prefix is being used by those targets that do not support FastISel.
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll b/llvm/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll
index f89802d1a11c0..b1fc6cf52a37e 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -mcpu=mips32 -relocation-model=pic \
+; RUN: llc -mtriple=mipsel -mcpu=mips32 -relocation-model=pic \
 ; RUN:     -fast-isel=true -fast-isel-abort=3 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mcpu=mips32r2 -relocation-model=pic \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -relocation-model=pic \
 ; RUN:     -fast-isel=true -fast-isel-abort=3 < %s | FileCheck %s
 
 @ARR = external global [10 x i32], align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/div1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/div1.ll
index effa047383882..fb35864171aea 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/div1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/div1.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
 ; RUN:      -fast-isel-abort=3 | FileCheck %s
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
 ; RUN:      -fast-isel-abort=3 | FileCheck %s
 
 @sj = global i32 200000, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/double-arg.ll b/llvm/test/CodeGen/Mips/Fast-ISel/double-arg.ll
index 16cba99ad0657..6eccc9f498da1 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/double-arg.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/double-arg.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -march=mipsel -mcpu=mips32r2 -mattr=+fp64 \
+; RUN: not --crash llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+fp64 \
 ; RUN:         -O0 -relocation-model=pic -fast-isel-abort=3 < %s
 
 ; Check that FastISel aborts when we have 64bit FPU registers. FastISel currently
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fast-isel-softfloat-lower-args.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fast-isel-softfloat-lower-args.ll
index b69000fc625ea..baac595082365 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fast-isel-softfloat-lower-args.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fast-isel-softfloat-lower-args.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -march=mipsel -mcpu=mips32r2 -mattr=+soft-float \
+; RUN: not --crash llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+soft-float \
 ; RUN:         -O0 -fast-isel-abort=3 -relocation-model=pic < %s
 
 ; Test that FastISel aborts instead of trying to lower arguments for soft-float.
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
index 73e5f714802f6..6044e6b6dd90a 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s -verify-machineinstrs | FileCheck %s
 
 %struct.x = type { i32 }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll
index 0aec8d506f778..9156cc6da25e1 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
 ; RUN:     -pass-remarks-missed=isel 2>&1 | FileCheck %s
 
 ; CHECK:      FastISel missed call:
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
index 95270ff6b272d..a60b9673ad8b1 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 @f1 = common global float 0.000000e+00, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fpext.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fpext.ll
index fbd5039f20a1a..1d4d8ed6d27c5 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fpext.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fpext.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @f = global float 0x40147E6B80000000, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fpintconv.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
index 67b509938c40a..57226a94a759f 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fptrunc.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
index 0a414316fa1b7..36a8cbfd558d7 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @d = global double 0x40147E6B74DF0446, align 8
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll b/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
index 53783c11785fb..3cd2f6b43a50d 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @c = global i32 4, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/icmpbr1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/icmpbr1.ll
index 37e49c2e8a428..84cdd9456dc82 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/icmpbr1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/icmpbr1.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
 ; RUN:     < %s -verify-machineinstrs | FileCheck %s
 
 
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/icmpi1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/icmpi1.ll
index 66c1f4f30c153..31ad7df23a96d 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/icmpi1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/icmpi1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel=true -mcpu=mips32r2 \
 ; RUN:     < %s -verify-machineinstrs | FileCheck %s
 
 
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll b/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
index 93865a9b5c12f..a5c1cec0ea11f 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
@@ -4,9 +4,9 @@ target triple = "mips--linux-gnu"
 
 @c2 = common global i8 0, align 1
 @c1 = common global i8 0, align 1
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @s2 = common global i16 0, align 2
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll b/llvm/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
index 2a75b808cb7ea..740aa78459a19 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32r2
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32
 
 @b2 = global i8 0, align 1
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll b/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
index 66b615bad4dc7..bc6f2c50a805a 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
index 494dcd19a5b8a..90db1fde96fe0 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 < %s | FileCheck %s
 
 @ub1 = common global i8 0, align 1
 @ub2 = common global i8 0, align 1
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/mul1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/mul1.ll
index 8713e7ef1d966..d84a10b2b111d 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/mul1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/mul1.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32 -O0 -relocation-model=pic
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic
 
 ; The test is just to make sure it is able to allocate
 ; registers for this example. There was an issue with allocating AC0
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/nullvoid.ll b/llvm/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
index 617b9bb07dd9f..cb71cb426e640 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll b/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
index 2ed6117dd7338..8f2b8c5ecc7ab 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @x = common global [128000 x float] zeroinitializer, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/rem1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/rem1.ll
index cc0fc47d1b874..c37278350785c 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/rem1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/rem1.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
 ; RUN:      -fast-isel-abort=3 | FileCheck %s
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
 ; RUN:      -fast-isel-abort=3 | FileCheck %s
 
 @sj = global i32 200, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/retabi.ll b/llvm/test/CodeGen/Mips/Fast-ISel/retabi.ll
index c8023e97d5936..dcfe84e4d2226 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/retabi.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/retabi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
 
 @i = global i32 75, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll b/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
index 923ccaede29d5..b49478998a86c 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 \
 ; RUN:     -fast-isel-abort=3 -mcpu=mips32r2  < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 \
 ; RUN:     -fast-isel-abort=3 -mcpu=mips32 < %s | FileCheck %s
 
 @s1 = global i16 -89, align 2
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
index d029ea4973b41..25df91a6fae92 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @abcd = external global i32
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
index 1957453f55068..d1a0574a19826 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s 
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32r2 
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32
 
 @f = common global float 0.000000e+00, align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
index 2f73d1cb890c0..ee174ddbc25d1 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
+; RUN: llc -mtriple=mipsel -relocation-model=pic -O0 -fast-isel-abort=3 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @ijk = external global i32
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll b/llvm/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
index f0974d580fe70..0541b8de7f476 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -fast-isel -frame-pointer=all -relocation-model=pic < %s
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32 -fast-isel -frame-pointer=all -relocation-model=pic < %s
 
 ; Test that negative array access don't crash constant synthesis when fast isel
 ; generates negative offsets.
diff --git a/llvm/test/CodeGen/Mips/addc.ll b/llvm/test/CodeGen/Mips/addc.ll
index 6877b3b633ce0..13467ac84c813 100644
--- a/llvm/test/CodeGen/Mips/addc.ll
+++ b/llvm/test/CodeGen/Mips/addc.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mipsel | FileCheck %s 
-; RUN: llc  < %s -march=mips   | FileCheck %s
+; RUN: llc  < %s -mtriple=mipsel | FileCheck %s 
+; RUN: llc  < %s -mtriple=mips   | FileCheck %s
 
 define void @f(i64 %l, ptr nocapture %p) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/address-selection.ll b/llvm/test/CodeGen/Mips/address-selection.ll
index d7b3fc66813c6..8b269aeb83b3f 100644
--- a/llvm/test/CodeGen/Mips/address-selection.ll
+++ b/llvm/test/CodeGen/Mips/address-selection.ll
@@ -1,9 +1,9 @@
-; RUN: llc -march=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot < %s \
+; RUN: llc -mtriple=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips -relocation-model=pic -mattr=+xgot < %s \
 ; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MIPS-XGOT
 
-; RUN: llc -march=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot,+micromips < %s \
+; RUN: llc -mtriple=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -relocation-model=pic -mattr=+xgot,+micromips < %s \
 ; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MM-XGOT
 
 ; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/Mips/addressing-mode.ll b/llvm/test/CodeGen/Mips/addressing-mode.ll
index bd8daf45be2c4..74543f6cdb9fd 100644
--- a/llvm/test/CodeGen/Mips/addressing-mode.ll
+++ b/llvm/test/CodeGen/Mips/addressing-mode.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 @g0 = common global i32 0, align 4
 @g1 = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/adjust-callstack-sp.ll b/llvm/test/CodeGen/Mips/adjust-callstack-sp.ll
index c583ff0fdd6cf..821904686a436 100644
--- a/llvm/test/CodeGen/Mips/adjust-callstack-sp.ll
+++ b/llvm/test/CodeGen/Mips/adjust-callstack-sp.ll
@@ -1,10 +1,10 @@
-; RUN: llc < %s -march=mips -mattr=mips16 | FileCheck %s -check-prefix=M16
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefix=GP32
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s -check-prefix=GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s -check-prefix=GP32
-; RUN: llc < %s -march=mips -mcpu=mips3 -target-abi n64 | FileCheck %s -check-prefix=GP64
-; RUN: llc < %s -march=mips -mcpu=mips64 -target-abi n64 | FileCheck %s -check-prefix=GP64
-; RUN: llc < %s -march=mips -mcpu=mips64r6 -target-abi n64 | FileCheck %s -check-prefix=GP64
+; RUN: llc < %s -mtriple=mips -mattr=mips16 | FileCheck %s -check-prefix=M16
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s -check-prefix=GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s -check-prefix=GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s -check-prefix=GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips3 -target-abi n64 | FileCheck %s -check-prefix=GP64
+; RUN: llc < %s -mtriple=mips -mcpu=mips64 -target-abi n64 | FileCheck %s -check-prefix=GP64
+; RUN: llc < %s -mtriple=mips -mcpu=mips64r6 -target-abi n64 | FileCheck %s -check-prefix=GP64
 
 declare void @bar(ptr)
 
diff --git a/llvm/test/CodeGen/Mips/alloca.ll b/llvm/test/CodeGen/Mips/alloca.ll
index 1fd6d859954ad..e4ac0e46044c6 100644
--- a/llvm/test/CodeGen/Mips/alloca.ll
+++ b/llvm/test/CodeGen/Mips/alloca.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 define i32 @twoalloca(i32 %size) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/alloca16.ll b/llvm/test/CodeGen/Mips/alloca16.ll
index b6921d59e94c9..f074b2d56936f 100644
--- a/llvm/test/CodeGen/Mips/alloca16.ll
+++ b/llvm/test/CodeGen/Mips/alloca16.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 25, align 4
 @jjjj = global i32 35, align 4
diff --git a/llvm/test/CodeGen/Mips/and1.ll b/llvm/test/CodeGen/Mips/and1.ll
index 7b5380fab2fdf..e3577343dbe77 100644
--- a/llvm/test/CodeGen/Mips/and1.ll
+++ b/llvm/test/CodeGen/Mips/and1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @x = global i32 65504, align 4
 @y = global i32 60929, align 4
diff --git a/llvm/test/CodeGen/Mips/ase_warnings.ll b/llvm/test/CodeGen/Mips/ase_warnings.ll
index ede830c72ebef..35103b4012a6e 100644
--- a/llvm/test/CodeGen/Mips/ase_warnings.ll
+++ b/llvm/test/CodeGen/Mips/ase_warnings.ll
@@ -1,61 +1,61 @@
 ; Check msa warnings.
-; RUN: llc -march=mips -mattr=+mips32r2 -mattr=+msa -mattr=+fp64 < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r2 -mattr=+msa -mattr=+fp64 < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=MSA_32
-; RUN: llc -march=mips64 -mattr=+mips64r2 -mattr=+msa < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r2 -mattr=+msa < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=MSA_64
-; RUN: llc -march=mips -mattr=+mips32r5 -mattr=+msa -mattr=+fp64 < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r5 -mattr=+msa -mattr=+fp64 < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=MSA_32_NO_WARNING
-; RUN: llc -march=mips64 -mattr=+mips64r5 -mattr=+msa < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r5 -mattr=+msa < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=MSA_64_NO_WARNING
 
 ; Check dspr2 warnings.
-; RUN: llc -march=mips -mattr=+mips32 -mattr=+dspr2 < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32 -mattr=+dspr2 < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSPR2_32
-; RUN: llc -march=mips64 -mattr=+mips64 -mattr=+dspr2 < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64 -mattr=+dspr2 < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSPR2_64
-; RUN: llc -march=mips64 -mattr=+mips64r3 -mattr=+dspr2 < %s  2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r3 -mattr=+dspr2 < %s  2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSPR2_64_NO_WARNING
-; RUN: llc -march=mips -mattr=+mips32r2 -mattr=+dspr2 < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r2 -mattr=+dspr2 < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=DSPR2_32_NO_WARNING
 
 ; Check dsp warnings.
-; RUN: llc -march=mips -mattr=+mips32 -mattr=+dsp < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32 -mattr=+dsp < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSP_32
-; RUN: llc -march=mips64 -mattr=+mips64 -mattr=+dsp < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64 -mattr=+dsp < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=DSP_64
-; RUN: llc -march=mips -mattr=+mips32r5 -mattr=+dsp < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r5 -mattr=+dsp < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSP_32_NO_WARNING
-; RUN: llc -march=mips64 -mattr=+mips64r2 -mattr=+dsp < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r2 -mattr=+dsp < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=DSP_64_NO_WARNING
 
 ; Check virt warnings.
-; RUN: llc -march=mips -mattr=+mips32r2 -mattr=+virt < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r2 -mattr=+virt < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=VIRT_32
-; RUN: llc -march=mips64 -mattr=+mips64r2 -mattr=+virt < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r2 -mattr=+virt < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=VIRT_64
-; RUN: llc -march=mips -mattr=+mips32r5 -mattr=+virt < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r5 -mattr=+virt < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=VIRT_32_NO_WARNING
-; RUN: llc -march=mips64 -mattr=+mips64r5 -mattr=+virt < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r5 -mattr=+virt < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=VIRT_64_NO_WARNING
 
 ; Check crc warnings.
-; RUN: llc -march=mips -mattr=+mips32r2 -mattr=+crc < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r2 -mattr=+crc < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=CRC_32
-; RUN: llc -march=mips64 -mattr=+mips64r2 -mattr=+crc < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r2 -mattr=+crc < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=CRC_64
-; RUN: llc -march=mips -mattr=+mips32r6 -mattr=+crc < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r6 -mattr=+crc < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=CRC_32_NO_WARNING
-; RUN: llc -march=mips64 -mattr=+mips64r6 -mattr=+crc < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r6 -mattr=+crc < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=CRC_64_NO_WARNING
 
 ; Check ginv warnings.
-; RUN: llc -march=mips -mattr=+mips32r2 -mattr=+ginv < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r2 -mattr=+ginv < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=GINV_32
-; RUN: llc -march=mips64 -mattr=+mips64r2 -mattr=+ginv < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r2 -mattr=+ginv < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=GINV_64
-; RUN: llc -march=mips -mattr=+mips32r6 -mattr=+ginv < %s 2>&1 | \
+; RUN: llc -mtriple=mips -mattr=+mips32r6 -mattr=+ginv < %s 2>&1 | \
 ; RUN:   FileCheck %s -check-prefix=GINV_32_NO_WARNING
-; RUN: llc -march=mips64 -mattr=+mips64r6 -mattr=+ginv < %s 2>&1 | \
+; RUN: llc -mtriple=mips64 -mattr=+mips64r6 -mattr=+ginv < %s 2>&1 | \
 ; RUN:   FileCheck %s  -check-prefix=GINV_64_NO_WARNING
 
 ; MSA_32: warning: the 'msa' ASE requires MIPS32 revision 5 or greater
diff --git a/llvm/test/CodeGen/Mips/asm-large-immediate.ll b/llvm/test/CodeGen/Mips/asm-large-immediate.ll
index c75b9e4ad12be..848be30839a8a 100644
--- a/llvm/test/CodeGen/Mips/asm-large-immediate.ll
+++ b/llvm/test/CodeGen/Mips/asm-large-immediate.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -no-integrated-as < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -no-integrated-as < %s | FileCheck %s
 
 define void @test() {
 entry:
diff --git a/llvm/test/CodeGen/Mips/assertzext-trunc.ll b/llvm/test/CodeGen/Mips/assertzext-trunc.ll
index 2295727834eb0..a6b9c0a2066a4 100644
--- a/llvm/test/CodeGen/Mips/assertzext-trunc.ll
+++ b/llvm/test/CodeGen/Mips/assertzext-trunc.ll
@@ -1,16 +1,16 @@
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,PRE-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,R6
 
 ; Check that we don't emit redundant SLLs for sequences of
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max-64.ll b/llvm/test/CodeGen/Mips/atomic-min-max-64.ll
index 5273f499cedec..62af633ea8957 100644
--- a/llvm/test/CodeGen/Mips/atomic-min-max-64.ll
+++ b/llvm/test/CodeGen/Mips/atomic-min-max-64.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
-; RUN: llc -march=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+; RUN: llc -mtriple=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+; RUN: llc -mtriple=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
 
 define i64 @test_max(ptr nocapture %ptr, i64 signext %val) {
 ; MIPS-LABEL: test_max:
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll
index 3d3225509d1ae..a10db052a4ff2 100644
--- a/llvm/test/CodeGen/Mips/atomic-min-max.ll
+++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll
@@ -1,17 +1,17 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
-; RUN: llc -march=mips -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMR6
-; RUN: llc -march=mipsel -O0 -mcpu=mips32 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSEL
-; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSELR6
-; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMEL
-; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMELR6
-; RUN: llc -march=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64
-; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64R6
-; RUN: llc -march=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64EL
-; RUN: llc -march=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64ELR6
+; RUN: llc -mtriple=mips -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
+; RUN: llc -mtriple=mips -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMR6
+; RUN: llc -mtriple=mipsel -O0 -mcpu=mips32 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mipsel -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSEL
+; RUN: llc -mtriple=mipsel -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSELR6
+; RUN: llc -mtriple=mipsel -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMEL
+; RUN: llc -mtriple=mipsel -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMELR6
+; RUN: llc -mtriple=mips64 -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64
+; RUN: llc -mtriple=mips64 -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64R6
+; RUN: llc -mtriple=mips64el -O0 -mcpu=mips64r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64EL
+; RUN: llc -mtriple=mips64el -O0 -mcpu=mips64r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS64ELR6
 
 define i32 @test_max_32(ptr nocapture %ptr, i32 signext %val) {
 ; MIPS-LABEL: test_max_32:
diff --git a/llvm/test/CodeGen/Mips/atomicops.ll b/llvm/test/CodeGen/Mips/atomicops.ll
index 14e401e1f0963..906429b14a181 100644
--- a/llvm/test/CodeGen/Mips/atomicops.ll
+++ b/llvm/test/CodeGen/Mips/atomicops.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @.str = private unnamed_addr constant [8 x i8] c"%d, %d\0A\00", align 1
 
diff --git a/llvm/test/CodeGen/Mips/biggot.ll b/llvm/test/CodeGen/Mips/biggot.ll
index 1080922ade809..50d72b9219c5c 100644
--- a/llvm/test/CodeGen/Mips/biggot.ll
+++ b/llvm/test/CodeGen/Mips/biggot.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel -mattr=+xgot \
+; RUN: llc -mtriple=mipsel -mattr=+xgot \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+xgot \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
-; RUN: llc -march=mipsel -mattr=+xgot -fast-isel \
+; RUN: llc -mtriple=mipsel -mattr=+xgot -fast-isel \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot -fast-isel \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+xgot -fast-isel \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
 
 @v0 = external global i32
diff --git a/llvm/test/CodeGen/Mips/brconeq.ll b/llvm/test/CodeGen/Mips/brconeq.ll
index ba7dc0f8540e6..468456effd477 100644
--- a/llvm/test/CodeGen/Mips/brconeq.ll
+++ b/llvm/test/CodeGen/Mips/brconeq.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/brconeqk.ll b/llvm/test/CodeGen/Mips/brconeqk.ll
index 4ee2f772ff68e..d0c6656cb52e1 100644
--- a/llvm/test/CodeGen/Mips/brconeqk.ll
+++ b/llvm/test/CodeGen/Mips/brconeqk.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @result = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/brconeqz.ll b/llvm/test/CodeGen/Mips/brconeqz.ll
index b8e7d1d12f978..c99c5a2e47d6e 100644
--- a/llvm/test/CodeGen/Mips/brconeqz.ll
+++ b/llvm/test/CodeGen/Mips/brconeqz.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @result = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/brconge.ll b/llvm/test/CodeGen/Mips/brconge.ll
index 38e3a7c3706f5..44d7556b5577b 100644
--- a/llvm/test/CodeGen/Mips/brconge.ll
+++ b/llvm/test/CodeGen/Mips/brconge.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O2 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O2 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/brcongt.ll b/llvm/test/CodeGen/Mips/brcongt.ll
index 3231811588fc1..c332820a83a8c 100644
--- a/llvm/test/CodeGen/Mips/brcongt.ll
+++ b/llvm/test/CodeGen/Mips/brcongt.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/brconle.ll b/llvm/test/CodeGen/Mips/brconle.ll
index e0ade5df23775..e695f4be18f1f 100644
--- a/llvm/test/CodeGen/Mips/brconle.ll
+++ b/llvm/test/CodeGen/Mips/brconle.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O2 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O2 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 -5, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/brconlt.ll b/llvm/test/CodeGen/Mips/brconlt.ll
index f3dbb9607eaff..6ae8c64b4f09c 100644
--- a/llvm/test/CodeGen/Mips/brconlt.ll
+++ b/llvm/test/CodeGen/Mips/brconlt.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mattr=micromips -mcpu=mips32r6 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MM32R6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mattr=micromips -mcpu=mips32r6 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MM32R6
 
 @i = global i32 5, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/brconne.ll b/llvm/test/CodeGen/Mips/brconne.ll
index 5c3a0ef343291..40a15cdefba73 100644
--- a/llvm/test/CodeGen/Mips/brconne.ll
+++ b/llvm/test/CodeGen/Mips/brconne.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 5, align 4
 @j = global i32 5, align 4
diff --git a/llvm/test/CodeGen/Mips/brconnek.ll b/llvm/test/CodeGen/Mips/brconnek.ll
index 30c32825da52e..3b74c777e0c2e 100644
--- a/llvm/test/CodeGen/Mips/brconnek.ll
+++ b/llvm/test/CodeGen/Mips/brconnek.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @j = global i32 5, align 4
 @result = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/brconnez.ll b/llvm/test/CodeGen/Mips/brconnez.ll
index 5f8b54e9cbb50..e153964fab40a 100644
--- a/llvm/test/CodeGen/Mips/brconnez.ll
+++ b/llvm/test/CodeGen/Mips/brconnez.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @j = global i32 0, align 4
 @result = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/brind.ll b/llvm/test/CodeGen/Mips/brind.ll
index 8f2954fd2b25e..1cc7fdd6bef86 100644
--- a/llvm/test/CodeGen/Mips/brind.ll
+++ b/llvm/test/CodeGen/Mips/brind.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @main.L = internal unnamed_addr constant [5 x ptr] [ptr blockaddress(@main, %L1), ptr blockaddress(@main, %L2), ptr blockaddress(@main, %L3), ptr blockaddress(@main, %L4), ptr null], align 4
 @str = private unnamed_addr constant [2 x i8] c"A\00"
diff --git a/llvm/test/CodeGen/Mips/brundef.ll b/llvm/test/CodeGen/Mips/brundef.ll
index d63b587af0395..b353862c6eddc 100644
--- a/llvm/test/CodeGen/Mips/brundef.ll
+++ b/llvm/test/CodeGen/Mips/brundef.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32 -verify-machineinstrs -o /dev/null < %s 
+; RUN: llc -mtriple=mips -mcpu=mips32 -verify-machineinstrs -o /dev/null < %s 
 ; Confirm that MachineInstr branch simplification preserves
 ; register operand flags, such as the <undef> flag.
 
diff --git a/llvm/test/CodeGen/Mips/bswap.ll b/llvm/test/CodeGen/Mips/bswap.ll
index ace6c3d6021d2..fa8cda330ba3a 100644
--- a/llvm/test/CodeGen/Mips/bswap.ll
+++ b/llvm/test/CodeGen/Mips/bswap.ll
@@ -1,10 +1,10 @@
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 \
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 \
 ; RUN:   | FileCheck %s -check-prefix=MIPS32
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   | FileCheck %s -check-prefix=MM
-; RUN: llc  < %s -march=mips64el -mcpu=mips64r2 \
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64r2 \
 ; RUN:   | FileCheck %s -check-prefix=MIPS64
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 -mattr=+mips16 \
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 -mattr=+mips16 \
 ; RUN:   | FileCheck %s -check-prefix=MIPS16
 
 define i32 @bswap32(i32 signext %x) nounwind readnone {
diff --git a/llvm/test/CodeGen/Mips/buildpairextractelementf64.ll b/llvm/test/CodeGen/Mips/buildpairextractelementf64.ll
index 95b9ca8caf029..1c1323b10f0af 100644
--- a/llvm/test/CodeGen/Mips/buildpairextractelementf64.ll
+++ b/llvm/test/CodeGen/Mips/buildpairextractelementf64.ll
@@ -1,9 +1,9 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefixes=NO-MFHC1,ALL
-; RUN: llc -march=mips  < %s | FileCheck %s -check-prefixes=NO-MFHC1,ALL
-; RUN: llc -march=mipsel -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
-; RUN: llc -march=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefixes=NO-MFHC1,ALL
+; RUN: llc -mtriple=mips  < %s | FileCheck %s -check-prefixes=NO-MFHC1,ALL
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck %s -check-prefixes=HAS-MFHC1,ALL
 
 @a = external global i32
 
diff --git a/llvm/test/CodeGen/Mips/call-optimization.ll b/llvm/test/CodeGen/Mips/call-optimization.ll
index 89fb970ed8178..e48c291150198 100644
--- a/llvm/test/CodeGen/Mips/call-optimization.ll
+++ b/llvm/test/CodeGen/Mips/call-optimization.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -disable-mips-delay-filler -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -disable-mips-delay-filler -relocation-model=pic < %s | \
 ; RUN:  FileCheck %s -check-prefix=O32
-; RUN: llc -march=mipsel -mips-load-target-from-got=false \
+; RUN: llc -mtriple=mipsel -mips-load-target-from-got=false \
 ; RUN:  -disable-mips-delay-filler -relocation-model=pic < %s | FileCheck %s -check-prefix=O32-LOADTGT
 
 @gd1 = common global double 0.000000e+00, align 8
diff --git a/llvm/test/CodeGen/Mips/cconv/arguments-small-structures-bigger-than-32bits.ll b/llvm/test/CodeGen/Mips/cconv/arguments-small-structures-bigger-than-32bits.ll
index f2355049adfaa..533f8b6426bcd 100644
--- a/llvm/test/CodeGen/Mips/cconv/arguments-small-structures-bigger-than-32bits.ll
+++ b/llvm/test/CodeGen/Mips/cconv/arguments-small-structures-bigger-than-32bits.ll
@@ -1,7 +1,7 @@
-; RUN: llc < %s -march=mips64   -target-abi n64 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEB
-; RUN: llc < %s -march=mips64el -target-abi n64 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEL
-; RUN: llc < %s -march=mips64   -target-abi n32 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEB
-; RUN: llc < %s -march=mips64el -target-abi n32 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEL
+; RUN: llc < %s -mtriple=mips64   -target-abi n64 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEB
+; RUN: llc < %s -mtriple=mips64el -target-abi n64 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEL
+; RUN: llc < %s -mtriple=mips64   -target-abi n32 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEB
+; RUN: llc < %s -mtriple=mips64el -target-abi n32 -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,MIPSEL
 
 ; #include <stdio.h>
 ; 
diff --git a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-byte.ll b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-byte.ll
index 0619ebe48cfe6..6f94dce7f8a63 100644
--- a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-byte.ll
+++ b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-byte.ll
@@ -1,4 +1,4 @@
-; RUN: llc --march=mips64 -mcpu=mips64r2 < %s | FileCheck %s
+; RUN: llc --mtriple=mips64 -mcpu=mips64r2 < %s | FileCheck %s
 
 ; Generated from the C program:
 ; 
diff --git a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-combinations.ll b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-combinations.ll
index db066c0475142..485aab499189e 100644
--- a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-combinations.ll
+++ b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-combinations.ll
@@ -1,4 +1,4 @@
-; RUN: llc --march=mips64 -mcpu=mips64r2 < %s | FileCheck %s
+; RUN: llc --mtriple=mips64 -mcpu=mips64r2 < %s | FileCheck %s
 
 ; Generated from the C program:
 ;
diff --git a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-multiple-args.ll b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-multiple-args.ll
index 3bddbf9d450ab..0193a90d6e9c1 100644
--- a/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-multiple-args.ll
+++ b/llvm/test/CodeGen/Mips/cconv/arguments-varargs-small-structs-multiple-args.ll
@@ -1,4 +1,4 @@
-; RUN: llc --march=mips64 -mcpu=mips64r2 < %s | FileCheck %s
+; RUN: llc --mtriple=mips64 -mcpu=mips64r2 < %s | FileCheck %s
 
 ; Generated from the C program:
 ;  
diff --git a/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx.ll b/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx.ll
index 1a604e7b5e89e..741d5e3645ca2 100644
--- a/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx.ll
+++ b/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx.ll
@@ -1,12 +1,12 @@
-; RUN: llc -march=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
-; RUN: llc -march=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
-; RUN: llc -march=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV %s
-; RUN: llc -march=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV %s
+; RUN: llc -mtriple=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
+; RUN: llc -mtriple=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV %s
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV %s
 
-; RUN-TODO: llc -march=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
-; RUN-TODO: llc -march=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
-; RUN-TODO: llc -march=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV,O32-FPXX-INV %s
-; RUN-TODO: llc -march=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV,O32-FPXX-INV %s
+; RUN-TODO: llc -mtriple=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
+; RUN-TODO: llc -mtriple=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX %s
+; RUN-TODO: llc -mtriple=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV,O32-FPXX-INV %s
+; RUN-TODO: llc -mtriple=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefixes=O32-FPXX-INV,O32-FPXX-INV %s
 
 define void @fpu_clobber() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx1.ll b/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx1.ll
index 1c2f9e2be3411..e4f5564910c32 100644
--- a/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx1.ll
+++ b/llvm/test/CodeGen/Mips/cconv/callee-saved-fpxx1.ll
@@ -1,13 +1,13 @@
-; RUN: llc -march=mips -mattr=+o32,+fp64,+mips32r2 < %s \
+; RUN: llc -mtriple=mips -mattr=+o32,+fp64,+mips32r2 < %s \
 ; RUN:   | FileCheck --check-prefix=O32-FP64-INV %s
-; RUN: llc -march=mipsel -mattr=+o32,+fp64,+mips32r2 < %s \
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fp64,+mips32r2 < %s \
 ; RUN:   | FileCheck --check-prefix=O32-FP64-INV %s
 
-; RUN: llc -march=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
-; RUN: llc -march=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
+; RUN: llc -mtriple=mips -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
 
-; RUN-TODO: llc -march=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
-; RUN-TODO: llc -march=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
+; RUN-TODO: llc -mtriple=mips64 -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
+; RUN-TODO: llc -mtriple=mips64el -mattr=+o32,+fpxx < %s | FileCheck --check-prefix=O32-FPXX %s
 
 define void @fpu_clobber() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/cconv/callee-saved.ll b/llvm/test/CodeGen/Mips/cconv/callee-saved.ll
index 8cfd1397ed974..93808fbcad03d 100644
--- a/llvm/test/CodeGen/Mips/cconv/callee-saved.ll
+++ b/llvm/test/CodeGen/Mips/cconv/callee-saved.ll
@@ -1,33 +1,33 @@
-; RUN: llc -march=mips   < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN: llc -march=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN: llc -march=mips   < %s | FileCheck --check-prefixes=ALL,O32-INV %s
-; RUN: llc -march=mipsel < %s | FileCheck --check-prefixes=ALL,O32-INV %s
+; RUN: llc -mtriple=mips   < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mips   < %s | FileCheck --check-prefixes=ALL,O32-INV %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck --check-prefixes=ALL,O32-INV %s
 
-; RUN-TODO: llc -march=mips64 -target-abi o32 < %s \
+; RUN-TODO: llc -mtriple=mips64 -target-abi o32 < %s \
 ; RUN-TODO:   | FileCheck --check-prefixes=ALL,O32 %s
-; RUN-TODO: llc -march=mips64el -target-abi o32 < %s \
+; RUN-TODO: llc -mtriple=mips64el -target-abi o32 < %s \
 ; RUN-TODO:   | FileCheck --check-prefixes=ALL,O32 %s
-; RUN-TODO: llc -march=mips64 -target-abi o32 < %s \
+; RUN-TODO: llc -mtriple=mips64 -target-abi o32 < %s \
 ; RUN-TODO:   | FileCheck --check-prefixes=ALL,O32-INV %s
-; RUN-TODO: llc -march=mips64el -target-abi o32 < %s \
+; RUN-TODO: llc -mtriple=mips64el -target-abi o32 < %s \
 ; RUN-TODO:   | FileCheck --check-prefixes=ALL,O32-INV %s
 
-; RUN: llc -march=mips64 -target-abi n32 < %s \
+; RUN: llc -mtriple=mips64 -target-abi n32 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64el -target-abi n32 < %s \
+; RUN: llc -mtriple=mips64el -target-abi n32 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64 -target-abi n32 < %s \
+; RUN: llc -mtriple=mips64 -target-abi n32 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N32-INV %s
-; RUN: llc -march=mips64el -target-abi n32 < %s \
+; RUN: llc -mtriple=mips64el -target-abi n32 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N32-INV %s
 
-; RUN: llc -march=mips64 -target-abi n64 < %s \
+; RUN: llc -mtriple=mips64 -target-abi n64 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64el -target-abi n64 < %s \
+; RUN: llc -mtriple=mips64el -target-abi n64 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64 -target-abi n64 < %s \
+; RUN: llc -mtriple=mips64 -target-abi n64 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N64-INV %s
-; RUN: llc -march=mips64el -target-abi n64 < %s \
+; RUN: llc -mtriple=mips64el -target-abi n64 < %s \
 ; RUN:   | FileCheck --check-prefixes=ALL,N64-INV %s
 
 ; Test the callee-saved registers are callee-saved as specified by section
diff --git a/llvm/test/CodeGen/Mips/cconv/memory-layout.ll b/llvm/test/CodeGen/Mips/cconv/memory-layout.ll
index dae4bfc226090..42f9d1890cc37 100644
--- a/llvm/test/CodeGen/Mips/cconv/memory-layout.ll
+++ b/llvm/test/CodeGen/Mips/cconv/memory-layout.ll
@@ -1,14 +1,14 @@
-; RUN: llc -march=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN: llc -march=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN-TODO: llc -march=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN-TODO: llc -march=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN: llc -march=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
 
-; RUN: llc -march=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
 
 ; Test the memory layout for all ABI's and byte orders as specified by section
 ; 4 of MD00305 (MIPS ABIs Described).
diff --git a/llvm/test/CodeGen/Mips/cconv/pr33883.ll b/llvm/test/CodeGen/Mips/cconv/pr33883.ll
index 54d7286ab8fff..9fecf09ef3c27 100644
--- a/llvm/test/CodeGen/Mips/cconv/pr33883.ll
+++ b/llvm/test/CodeGen/Mips/cconv/pr33883.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32 < %s -o /dev/null
+; RUN: llc -mtriple=mips -mcpu=mips32 < %s -o /dev/null
 
 ; Test that calls to vector intrinsics do not crash SelectionDAGBuilder.
 
diff --git a/llvm/test/CodeGen/Mips/cconv/reserved-space.ll b/llvm/test/CodeGen/Mips/cconv/reserved-space.ll
index 0a922345ddf3c..490476961626e 100644
--- a/llvm/test/CodeGen/Mips/cconv/reserved-space.ll
+++ b/llvm/test/CodeGen/Mips/cconv/reserved-space.ll
@@ -1,14 +1,14 @@
-; RUN: llc -march=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN: llc -march=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN-TODO: llc -march=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN-TODO: llc -march=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN: llc -march=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
 
-; RUN: llc -march=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
 
 ; Test that O32 correctly reserved space for the four arguments, even when
 ; there aren't any as per section 5 of MD00305 (MIPS ABIs Described).
diff --git a/llvm/test/CodeGen/Mips/cconv/roundl-call.ll b/llvm/test/CodeGen/Mips/cconv/roundl-call.ll
index 242b4292e5283..98bda89546e43 100644
--- a/llvm/test/CodeGen/Mips/cconv/roundl-call.ll
+++ b/llvm/test/CodeGen/Mips/cconv/roundl-call.ll
@@ -1,22 +1,22 @@
-; RUN: llc -march=mips64 -mcpu=mips64 -target-abi=n32 -relocation-model=pic < \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -target-abi=n32 -relocation-model=pic < \
 ; RUN:     %s | FileCheck %s -check-prefixes=ALL,N32,HARD-FLOAT
-; RUN: llc -march=mips64el -mcpu=mips64 -target-abi=n32 -relocation-model=pic < \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -target-abi=n32 -relocation-model=pic < \
 ; RUN:     %s | FileCheck %s -check-prefixes=ALL,N32,HARD-FLOAT
 
-; RUN: llc -march=mips64 -mcpu=mips64 -target-abi=n64 -relocation-model=pic < \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -target-abi=n64 -relocation-model=pic < \
 ; RUN:     %s | FileCheck %s -check-prefixes=ALL,N64,HARD-FLOAT
-; RUN: llc -march=mips64el -mcpu=mips64 -target-abi=n64 -relocation-model=pic \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -target-abi=n64 -relocation-model=pic \
 ; RUN:     < %s | FileCheck %s -check-prefixes=ALL,N64,HARD-FLOAT
 
-; RUN: llc -march=mips64 -mcpu=mips64 -mattr=+soft-float -target-abi=n32 \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -mattr=+soft-float -target-abi=n32 \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,N32,SOFT-FLOAT
-; RUN: llc -march=mips64el -mcpu=mips64 -mattr=+soft-float -target-abi=n32 \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -mattr=+soft-float -target-abi=n32 \
 ; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,N32,SOFT-FLOAT
 
-; RUN: llc -march=mips64 -mcpu=mips64 -mattr=+soft-float -target-abi=n64 < %s \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -mattr=+soft-float -target-abi=n64 < %s \
 ; RUN:     -relocation-model=pic | FileCheck %s \
 ; RUN:     -check-prefixes=ALL,N64,SOFT-FLOAT
-; RUN: llc -march=mips64el -mcpu=mips64 -mattr=+soft-float -target-abi=n64 < \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -mattr=+soft-float -target-abi=n64 < \
 ; RUN:     %s -relocation-model=pic | FileCheck %s \
 ; RUN:     -check-prefixes=ALL,N64,SOFT-FLOAT
 
diff --git a/llvm/test/CodeGen/Mips/cconv/stack-alignment.ll b/llvm/test/CodeGen/Mips/cconv/stack-alignment.ll
index c957e311b1b3a..9cdb8428b484a 100644
--- a/llvm/test/CodeGen/Mips/cconv/stack-alignment.ll
+++ b/llvm/test/CodeGen/Mips/cconv/stack-alignment.ll
@@ -1,14 +1,14 @@
-; RUN: llc -march=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN: llc -march=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mips < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN-TODO: llc -march=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
-; RUN-TODO: llc -march=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64 -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
+; RUN-TODO: llc -mtriple=mips64el -target-abi o32 < %s | FileCheck --check-prefixes=ALL,O32 %s
 
-; RUN: llc -march=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64 -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
+; RUN: llc -mtriple=mips64el -target-abi n32 < %s | FileCheck --check-prefixes=ALL,N32 %s
 
-; RUN: llc -march=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64 -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
+; RUN: llc -mtriple=mips64el -target-abi n64 < %s | FileCheck --check-prefixes=ALL,N64 %s
 
 ; Test the stack alignment for all ABI's and byte orders as specified by
 ; section 5 of MD00305 (MIPS ABIs Described).
diff --git a/llvm/test/CodeGen/Mips/cfi_offset.ll b/llvm/test/CodeGen/Mips/cfi_offset.ll
index 217adda59468a..f687212558843 100644
--- a/llvm/test/CodeGen/Mips/cfi_offset.ll
+++ b/llvm/test/CodeGen/Mips/cfi_offset.ll
@@ -1,9 +1,9 @@
-; RUN: llc -march=mips -mattr=+o32 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
-; RUN: llc -march=mipsel -mattr=+o32 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
-; RUN: llc -march=mips -mattr=+o32,+fpxx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
-; RUN: llc -march=mipsel -mattr=+o32,+fpxx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
-; RUN: llc -march=mips -mattr=+o32,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
-; RUN: llc -march=mipsel -mattr=+o32,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
+; RUN: llc -mtriple=mips -mattr=+o32 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
+; RUN: llc -mtriple=mipsel -mattr=+o32 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
+; RUN: llc -mtriple=mips -mattr=+o32,+fpxx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fpxx < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
+; RUN: llc -mtriple=mips -mattr=+o32,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EB
+; RUN: llc -mtriple=mipsel -mattr=+o32,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-EL
 
 @var = global double 0.0
 
diff --git a/llvm/test/CodeGen/Mips/check-adde-redundant-moves.ll b/llvm/test/CodeGen/Mips/check-adde-redundant-moves.ll
index cf0fda66ad626..f22848c1c45cc 100644
--- a/llvm/test/CodeGen/Mips/check-adde-redundant-moves.ll
+++ b/llvm/test/CodeGen/Mips/check-adde-redundant-moves.ll
@@ -1,16 +1,16 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r3 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r5 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | FileCheck %s -check-prefix=ALL
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s -check-prefix=ALL
 
 define i64 @add_i64(i64 %a) {
   ; GP32-LABEL: add_i64
diff --git a/llvm/test/CodeGen/Mips/check-noat.ll b/llvm/test/CodeGen/Mips/check-noat.ll
index cfcd367e87af2..1e53eda4d63fa 100644
--- a/llvm/test/CodeGen/Mips/check-noat.ll
+++ b/llvm/test/CodeGen/Mips/check-noat.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s 
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s 
 
 define void @f() nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/cins.ll b/llvm/test/CodeGen/Mips/cins.ll
index 4fe25564d1c12..39d80e3bb36ab 100644
--- a/llvm/test/CodeGen/Mips/cins.ll
+++ b/llvm/test/CodeGen/Mips/cins.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=octeon -target-abi=n64 < %s -o - | FileCheck %s
+; RUN: llc -mtriple=mips64 -mcpu=octeon -target-abi=n64 < %s -o - | FileCheck %s
 
 define i64 @cins_zext(i32 signext %n) {
 entry:
diff --git a/llvm/test/CodeGen/Mips/cmov.ll b/llvm/test/CodeGen/Mips/cmov.ll
index d19c1b1f14c36..bb3c7c27a122d 100644
--- a/llvm/test/CodeGen/Mips/cmov.ll
+++ b/llvm/test/CodeGen/Mips/cmov.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mips     -mcpu=mips32                 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
-; RUN: llc -march=mips     -mcpu=mips32 -regalloc=basic -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
-; RUN: llc -march=mips     -mcpu=mips32r2               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
-; RUN: llc -march=mips     -mcpu=mips32r6               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMP
-; RUN: llc -march=mips64el -mcpu=mips4                  -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMOV
-; RUN: llc -march=mips64el -mcpu=mips64                 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMOV
-; RUN: llc -march=mips64el -mcpu=mips64r6               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMP
+; RUN: llc -mtriple=mips     -mcpu=mips32                 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
+; RUN: llc -mtriple=mips     -mcpu=mips32 -regalloc=basic -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
+; RUN: llc -mtriple=mips     -mcpu=mips32r2               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMOV
+; RUN: llc -mtriple=mips     -mcpu=mips32r6               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,32-CMP
+; RUN: llc -mtriple=mips64el -mcpu=mips4                  -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMOV
+; RUN: llc -mtriple=mips64el -mcpu=mips64                 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMOV
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6               -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,64-CMP
 
 @i1 = global [3 x i32] [i32 1, i32 2, i32 3], align 4
 @i3 = common global ptr null, align 4
diff --git a/llvm/test/CodeGen/Mips/cmplarge.ll b/llvm/test/CodeGen/Mips/cmplarge.ll
index a64983dfed023..db7f37a88ac22 100644
--- a/llvm/test/CodeGen/Mips/cmplarge.ll
+++ b/llvm/test/CodeGen/Mips/cmplarge.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=cmp16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=cmp16
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
 target triple = "mipsel--linux-gnu"
diff --git a/llvm/test/CodeGen/Mips/coalesce-partial-redundant-reguse-terminator.mir b/llvm/test/CodeGen/Mips/coalesce-partial-redundant-reguse-terminator.mir
index bfdd5cfd8b53d..de7286611f954 100644
--- a/llvm/test/CodeGen/Mips/coalesce-partial-redundant-reguse-terminator.mir
+++ b/llvm/test/CodeGen/Mips/coalesce-partial-redundant-reguse-terminator.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips64 -o - %s -run-pass=register-coalescer -verify-coalescing | FileCheck %s
+# RUN: llc -mtriple=mips64 -o - %s -run-pass=register-coalescer -verify-coalescing | FileCheck %s
 
 ---
 name:            f
diff --git a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll
index 13e30676aa6fb..b0eeadfecf6ed 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll
+++ b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mcpu=mips32r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s
-; RUN: llc -march=mips64 -mcpu=mips64r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s
 
 
 ; beqc/bnec have the constraint that $rs < $rt && $rs != 0 && $rt != 0
diff --git a/llvm/test/CodeGen/Mips/compactbranches/compact-branch-implicit-def.mir b/llvm/test/CodeGen/Mips/compactbranches/compact-branch-implicit-def.mir
index 33e43ad06f124..dfaad3a01bb75 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/compact-branch-implicit-def.mir
+++ b/llvm/test/CodeGen/Mips/compactbranches/compact-branch-implicit-def.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips64 -mcpu=mips64r6 -start-after=block-placement -o - %s | FileCheck %s
+# RUN: llc -mtriple=mips64 -mcpu=mips64r6 -start-after=block-placement -o - %s | FileCheck %s
 
 # Check that MipsHazardSchedule sees through basic blocks with transient instructions.
 # The mir code in this file isn't representative of the llvm-ir.
diff --git a/llvm/test/CodeGen/Mips/compactbranches/compact-branch-policy.ll b/llvm/test/CodeGen/Mips/compactbranches/compact-branch-policy.ll
index c819bf59ace6b..5a206f2d8891d 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/compact-branch-policy.ll
+++ b/llvm/test/CodeGen/Mips/compactbranches/compact-branch-policy.ll
@@ -1,7 +1,7 @@
 ; Check that -mips-compact-branches={never,optimal,always} is accepted and honoured.
-; RUN: llc -march=mips -mcpu=mips32r6 -mips-compact-branches=never < %s | FileCheck %s -check-prefix=NEVER
-; RUN: llc -march=mips -mcpu=mips32r6 -mips-compact-branches=optimal < %s | FileCheck %s -check-prefix=OPTIMAL
-; RUN: llc -march=mips -mcpu=mips32r6 -mips-compact-branches=always < %s | FileCheck %s -check-prefix=ALWAYS
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mips-compact-branches=never < %s | FileCheck %s -check-prefix=NEVER
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mips-compact-branches=optimal < %s | FileCheck %s -check-prefix=OPTIMAL
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mips-compact-branches=always < %s | FileCheck %s -check-prefix=ALWAYS
 
 define i32 @l(i32 signext %a, i32 signext %b) {
 entry:
diff --git a/llvm/test/CodeGen/Mips/compactbranches/compact-branches-64.ll b/llvm/test/CodeGen/Mips/compactbranches/compact-branches-64.ll
index 1ba80bff8d5af..a808bec149eae 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/compact-branches-64.ll
+++ b/llvm/test/CodeGen/Mips/compactbranches/compact-branches-64.ll
@@ -1,4 +1,4 @@
-; RUN: llc -relocation-model=pic -march=mipsel -mcpu=mips64r6 \
+; RUN: llc -relocation-model=pic -mtriple=mipsel -mcpu=mips64r6 \
 ; RUN:     -disable-mips-delay-filler -target-abi=n64 < %s | FileCheck %s
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/Mips/compactbranches/empty-block.mir b/llvm/test/CodeGen/Mips/compactbranches/empty-block.mir
index a2caf5b67aab3..41f59f535a4c9 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/empty-block.mir
+++ b/llvm/test/CodeGen/Mips/compactbranches/empty-block.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mipsel -mcpu=mips32r6 -start-after=block-placement %s -o - | FileCheck %s
+# RUN: llc -mtriple=mipsel -mcpu=mips32r6 -start-after=block-placement %s -o - | FileCheck %s
 
 # Check that empty blocks in the cfg don't cause the mips hazard scheduler to
 # crash and that the nop is inserted correctly.
diff --git a/llvm/test/CodeGen/Mips/compactbranches/unsafe-in-forbidden-slot.ll b/llvm/test/CodeGen/Mips/compactbranches/unsafe-in-forbidden-slot.ll
index de61515cff9bc..99efc5e8e4411 100644
--- a/llvm/test/CodeGen/Mips/compactbranches/unsafe-in-forbidden-slot.ll
+++ b/llvm/test/CodeGen/Mips/compactbranches/unsafe-in-forbidden-slot.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips64el -O0 -mcpu=mips64r6 < %s | FileCheck %s
-; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -O0 -mcpu=mips64r6 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 -O0 -mcpu=mips64r6 < %s | FileCheck %s
 
 @boo = global i32 0, align 4
 
diff --git a/llvm/test/CodeGen/Mips/constantfp0.ll b/llvm/test/CodeGen/Mips/constantfp0.ll
index 191f31d013191..dd0ddca036c31 100644
--- a/llvm/test/CodeGen/Mips/constantfp0.ll
+++ b/llvm/test/CodeGen/Mips/constantfp0.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 define i32 @branch(double %d) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/constraint-c-err.ll b/llvm/test/CodeGen/Mips/constraint-c-err.ll
index 94768f89e9924..43229d1d38d9f 100644
--- a/llvm/test/CodeGen/Mips/constraint-c-err.ll
+++ b/llvm/test/CodeGen/Mips/constraint-c-err.ll
@@ -1,5 +1,5 @@
 ; Check that invalid type for constraint `c` causes an error message.
-; RUN: not llc -march=mips -target-abi o32 < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=mips -target-abi o32 < %s 2>&1 | FileCheck %s
 
 define i32 @main() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/constraint-c.ll b/llvm/test/CodeGen/Mips/constraint-c.ll
index 31b75b01a5e85..85eda164a1a61 100644
--- a/llvm/test/CodeGen/Mips/constraint-c.ll
+++ b/llvm/test/CodeGen/Mips/constraint-c.ll
@@ -1,5 +1,5 @@
 ; Check handling of the constraint `c`.
-; RUN: llc -march=mips -target-abi o32 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -target-abi o32 < %s | FileCheck %s
 
 define i32 @main() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/constraint-empty.ll b/llvm/test/CodeGen/Mips/constraint-empty.ll
index 8541a4fa348e6..ff2b9b8408525 100644
--- a/llvm/test/CodeGen/Mips/constraint-empty.ll
+++ b/llvm/test/CodeGen/Mips/constraint-empty.ll
@@ -1,5 +1,5 @@
 ; Check that `getRegForInlineAsmConstraint` does not crash on empty Constraint.
-; RUN: llc -march=mips64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 < %s | FileCheck %s
 
 define void @foo() {
 entry:
diff --git a/llvm/test/CodeGen/Mips/countleading.ll b/llvm/test/CodeGen/Mips/countleading.ll
index 264d0486e3b87..9c512bd3ad383 100644
--- a/llvm/test/CodeGen/Mips/countleading.ll
+++ b/llvm/test/CodeGen/Mips/countleading.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mipsel -mcpu=mips32   < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R1-R2 %s
-; RUN: llc -march=mipsel -mcpu=mips32r2 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R1-R2 %s
-; RUN: llc -march=mipsel -mcpu=mips32r6 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R6 %s
-; RUN: llc -march=mips64el -mcpu=mips64   < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
-; RUN: llc -march=mips64el -mcpu=mips64r2 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
-; RUN: llc -march=mips64el -mcpu=mips64r6 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
-; RUN: llc -march=mips64el -mcpu=mips4    < %s | FileCheck -check-prefixes=MIPS4 %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32   < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R1-R2 %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R1-R2 %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r6 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS32-R6 %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64   < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 < %s | FileCheck -check-prefixes=MIPS-GT-R1,MIPS64-GT-R1 %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4    < %s | FileCheck -check-prefixes=MIPS4 %s
 
 ; Prefixes:
 ;   ALL      - All
diff --git a/llvm/test/CodeGen/Mips/cprestore.ll b/llvm/test/CodeGen/Mips/cprestore.ll
index bb27c947d998e..2bd7591e38abe 100644
--- a/llvm/test/CodeGen/Mips/cprestore.ll
+++ b/llvm/test/CodeGen/Mips/cprestore.ll
@@ -1,4 +1,4 @@
-; DISABLE: llc -march=mipsel < %s | FileCheck %s
+; DISABLE: llc -mtriple=mipsel < %s | FileCheck %s
 ; RUN: false
 ; XFAIL: *
 
diff --git a/llvm/test/CodeGen/Mips/cstmaterialization/constMaterialization.ll b/llvm/test/CodeGen/Mips/cstmaterialization/constMaterialization.ll
index f34c70efa7a80..6a5f65a96095e 100644
--- a/llvm/test/CodeGen/Mips/cstmaterialization/constMaterialization.ll
+++ b/llvm/test/CodeGen/Mips/cstmaterialization/constMaterialization.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips < %s | FileCheck %s -check-prefixes=ALL,MIPS
-; RUN: llc -march=mips < %s -mattr=+micromips | FileCheck %s -check-prefixes=ALL,MM
+; RUN: llc -mtriple=mips < %s | FileCheck %s -check-prefixes=ALL,MIPS
+; RUN: llc -mtriple=mips < %s -mattr=+micromips | FileCheck %s -check-prefixes=ALL,MM
 
 ; Test the patterns used for constant materialization.
 
diff --git a/llvm/test/CodeGen/Mips/cstmaterialization/isel-materialization.ll b/llvm/test/CodeGen/Mips/cstmaterialization/isel-materialization.ll
index b29115a4f7827..dc8efae68fda1 100644
--- a/llvm/test/CodeGen/Mips/cstmaterialization/isel-materialization.ll
+++ b/llvm/test/CodeGen/Mips/cstmaterialization/isel-materialization.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/CodeGen/Mips/cstmaterialization/stack.ll b/llvm/test/CodeGen/Mips/cstmaterialization/stack.ll
index 78c01d9b7473e..0e9075aa16880 100644
--- a/llvm/test/CodeGen/Mips/cstmaterialization/stack.ll
+++ b/llvm/test/CodeGen/Mips/cstmaterialization/stack.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck %s -check-prefix=CHECK-MIPS32
-; RUN: llc -march=mips64el -mcpu=mips64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck %s -check-prefix=CHECK-MIPS32
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -relocation-model=pic < %s | \
 ; RUN:      FileCheck %s -check-prefix=CHECK-MIPS64
-; RUN: llc -march=mipsel -mcpu=mips64 -target-abi n32 < %s | \
+; RUN: llc -mtriple=mipsel -mcpu=mips64 -target-abi n32 < %s | \
 ; RUN:      FileCheck %s -check-prefix=CHECK-MIPSN32
 
 ; Test that the expansion of ADJCALLSTACKDOWN and ADJCALLSTACKUP generate
diff --git a/llvm/test/CodeGen/Mips/ctlz-v.ll b/llvm/test/CodeGen/Mips/ctlz-v.ll
index 156c640681b75..280054dac082a 100644
--- a/llvm/test/CodeGen/Mips/ctlz-v.ll
+++ b/llvm/test/CodeGen/Mips/ctlz-v.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 | FileCheck %s -check-prefix=MIPS32
-; RUN: llc  < %s -march=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefix=MIPS64
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 | FileCheck %s -check-prefix=MIPS32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefix=MIPS64
 
 declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
 
diff --git a/llvm/test/CodeGen/Mips/cttz-v.ll b/llvm/test/CodeGen/Mips/cttz-v.ll
index dbcde7f5fe5b8..347466086a68a 100644
--- a/llvm/test/CodeGen/Mips/cttz-v.ll
+++ b/llvm/test/CodeGen/Mips/cttz-v.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 | FileCheck %s -check-prefix=MIPS32
-; RUN: llc  < %s -march=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefix=MIPS64
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 | FileCheck %s -check-prefix=MIPS32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefix=MIPS64
 
 declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
 
diff --git a/llvm/test/CodeGen/Mips/dagcombine-store-gep-chain-slow.ll b/llvm/test/CodeGen/Mips/dagcombine-store-gep-chain-slow.ll
index 8c2ac7a7f2ce3..c2376a7b56dfb 100644
--- a/llvm/test/CodeGen/Mips/dagcombine-store-gep-chain-slow.ll
+++ b/llvm/test/CodeGen/Mips/dagcombine-store-gep-chain-slow.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -o /dev/null
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -o /dev/null
 
 ; Test that this file is compiled in a reasonable time period. Without the
 ; optimization level check in findBetterNeighbors, this test demonstrates
diff --git a/llvm/test/CodeGen/Mips/delay-slot-kill.ll b/llvm/test/CodeGen/Mips/delay-slot-kill.ll
index 57b630303c264..9d51d72721e94 100644
--- a/llvm/test/CodeGen/Mips/delay-slot-kill.ll
+++ b/llvm/test/CodeGen/Mips/delay-slot-kill.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s
 
 ; Currently, the following IR assembly generates a KILL instruction between
 ; the bitwise-and instruction and the return instruction. We verify that the
diff --git a/llvm/test/CodeGen/Mips/dext.ll b/llvm/test/CodeGen/Mips/dext.ll
index 1794f16b2cd70..b27d67c7da6a1 100644
--- a/llvm/test/CodeGen/Mips/dext.ll
+++ b/llvm/test/CodeGen/Mips/dext.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips64r2 -target-abi=n64 < %s -o - | FileCheck %s
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -target-abi=n64 < %s -o - | FileCheck %s
 
 define i64 @dext_add_zext(i32 signext %n) {
 entry:
diff --git a/llvm/test/CodeGen/Mips/dins.ll b/llvm/test/CodeGen/Mips/dins.ll
index 4deb7455a8012..aecc06bc7203a 100644
--- a/llvm/test/CodeGen/Mips/dins.ll
+++ b/llvm/test/CodeGen/Mips/dins.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -O2 -verify-machineinstrs -march=mips64 -mcpu=mips64r2 \
+; RUN: llc -O2 -verify-machineinstrs -mtriple=mips64 -mcpu=mips64r2 \
 ; RUN:   -target-abi=n64 < %s -o - | FileCheck %s -check-prefix=MIPS64R2
-; RUN: llc -O2 -verify-machineinstrs -march=mips -mcpu=mips32r2 < %s -o - \
+; RUN: llc -O2 -verify-machineinstrs -mtriple=mips -mcpu=mips32r2 < %s -o - \
 ; RUN:   | FileCheck %s -check-prefix=MIPS32R2
-; RUN: llc -O2 -verify-machineinstrs -march=mips -mattr=mips16 < %s -o - \
+; RUN: llc -O2 -verify-machineinstrs -mtriple=mips -mattr=mips16 < %s -o - \
 ; RUN:   | FileCheck %s -check-prefix=MIPS16
-; RUN: llc -O2 -verify-machineinstrs -march=mips64 -mcpu=mips64r2 \
+; RUN: llc -O2 -verify-machineinstrs -mtriple=mips64 -mcpu=mips64r2 \
 ; RUN:   -target-abi=n32 < %s -o - | FileCheck %s -check-prefix=MIPS64R2N32
 
 ; #include <stdint.h>
diff --git a/llvm/test/CodeGen/Mips/disable-tail-merge.ll b/llvm/test/CodeGen/Mips/disable-tail-merge.ll
index 188b83b68f5d3..8e6174c4b3c48 100644
--- a/llvm/test/CodeGen/Mips/disable-tail-merge.ll
+++ b/llvm/test/CodeGen/Mips/disable-tail-merge.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 @g0 = common global i32 0, align 4
 @g1 = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/div.ll b/llvm/test/CodeGen/Mips/div.ll
index 839b2dee6d09e..0dd3bb47049a1 100644
--- a/llvm/test/CodeGen/Mips/div.ll
+++ b/llvm/test/CodeGen/Mips/div.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 100, align 4
 @jjjj = global i32 -4, align 4
diff --git a/llvm/test/CodeGen/Mips/div_rem.ll b/llvm/test/CodeGen/Mips/div_rem.ll
index c8e22f262fb28..7485ca2df5e85 100644
--- a/llvm/test/CodeGen/Mips/div_rem.ll
+++ b/llvm/test/CodeGen/Mips/div_rem.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 103, align 4
 @jjjj = global i32 -4, align 4
diff --git a/llvm/test/CodeGen/Mips/divrem.ll b/llvm/test/CodeGen/Mips/divrem.ll
index b0dd49a062217..a29f2793c081f 100644
--- a/llvm/test/CodeGen/Mips/divrem.ll
+++ b/llvm/test/CodeGen/Mips/divrem.ll
@@ -1,16 +1,16 @@
-; RUN: llc -march=mips   -mcpu=mips32   -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,ACC32-TRAP
-; RUN: llc -march=mips   -mcpu=mips32r2 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,ACC32-TRAP
-; RUN: llc -march=mips   -mcpu=mips32r6 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR32,GPR32-TRAP
-; RUN: llc -march=mips64 -mcpu=mips64   -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,ACC64-TRAP
-; RUN: llc -march=mips64 -mcpu=mips64r2 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,ACC64-TRAP
-; RUN: llc -march=mips64 -mcpu=mips64r6 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR64,GPR64-TRAP
-
-; RUN: llc -march=mips   -mcpu=mips32   -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,NOCHECK
-; RUN: llc -march=mips   -mcpu=mips32r2 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,NOCHECK
-; RUN: llc -march=mips   -mcpu=mips32r6 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR32,NOCHECK
-; RUN: llc -march=mips64 -mcpu=mips64   -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,NOCHECK
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,NOCHECK
-; RUN: llc -march=mips64 -mcpu=mips64r6 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR64,NOCHECK
+; RUN: llc -mtriple=mips   -mcpu=mips32   -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,ACC32-TRAP
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,ACC32-TRAP
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR32,GPR32-TRAP
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,ACC64-TRAP
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,ACC64-TRAP
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -verify-machineinstrs    -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR64,GPR64-TRAP
+
+; RUN: llc -mtriple=mips   -mcpu=mips32   -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,NOCHECK
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC32,NOCHECK
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR32,NOCHECK
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,NOCHECK
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,ACC64,NOCHECK
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -mno-check-zero-division -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,GPR64,NOCHECK
 
 ; FileCheck Prefixes:
 ;   ALL - All targets
diff --git a/llvm/test/CodeGen/Mips/divu.ll b/llvm/test/CodeGen/Mips/divu.ll
index caa49fd472b37..10348001dd4bc 100644
--- a/llvm/test/CodeGen/Mips/divu.ll
+++ b/llvm/test/CodeGen/Mips/divu.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 100, align 4
 @jjjj = global i32 4, align 4
diff --git a/llvm/test/CodeGen/Mips/divu_remu.ll b/llvm/test/CodeGen/Mips/divu_remu.ll
index 820633be119be..7a435e922f1ab 100644
--- a/llvm/test/CodeGen/Mips/divu_remu.ll
+++ b/llvm/test/CodeGen/Mips/divu_remu.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 103, align 4
 @jjjj = global i32 4, align 4
diff --git a/llvm/test/CodeGen/Mips/double2int.ll b/llvm/test/CodeGen/Mips/double2int.ll
index f0d8ad28100f6..a5ca8b9f46aab 100644
--- a/llvm/test/CodeGen/Mips/double2int.ll
+++ b/llvm/test/CodeGen/Mips/double2int.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mcpu=mips32 < %s | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips32r6 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32r6 < %s | FileCheck %s
 
 define i32 @f1(double %d) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/dsp-patterns-cmp-vselect.ll b/llvm/test/CodeGen/Mips/dsp-patterns-cmp-vselect.ll
index a5fe34c1f684f..9444fdf32ebd2 100644
--- a/llvm/test/CodeGen/Mips/dsp-patterns-cmp-vselect.ll
+++ b/llvm/test/CodeGen/Mips/dsp-patterns-cmp-vselect.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=dsp < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=dsp < %s | FileCheck %s
 
 ; CHECK-LABEL: select_v2q15_eq_:
 ; CHECK: cmp.eq.ph ${{[0-9]+}}, ${{[0-9]+}}
diff --git a/llvm/test/CodeGen/Mips/dsp-patterns.ll b/llvm/test/CodeGen/Mips/dsp-patterns.ll
index ba6c4d972c56d..0f044f968bdb1 100644
--- a/llvm/test/CodeGen/Mips/dsp-patterns.ll
+++ b/llvm/test/CodeGen/Mips/dsp-patterns.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=dsp < %s | FileCheck %s -check-prefix=R1
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=dspr2 < %s | FileCheck %s -check-prefix=R2
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=dsp < %s | FileCheck %s -check-prefix=R1
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=dspr2 < %s | FileCheck %s -check-prefix=R2
 
 ; R1-LABEL: test_lbux:
 ; R1: lbux ${{[0-9]+}}
diff --git a/llvm/test/CodeGen/Mips/dsp-r1.ll b/llvm/test/CodeGen/Mips/dsp-r1.ll
index 0ec23b9d7fd77..2b5a0d25aed7f 100644
--- a/llvm/test/CodeGen/Mips/dsp-r1.ll
+++ b/llvm/test/CodeGen/Mips/dsp-r1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32 -mattr=+dsp -verify-machineinstrs < %s | \
+; RUN: llc -mtriple=mipsel -mcpu=mips32 -mattr=+dsp -verify-machineinstrs < %s | \
 ; RUN:     FileCheck %s
 
 define i32 @test__builtin_mips_extr_w1(i32 %i0, i32, i64 %a0) nounwind {
diff --git a/llvm/test/CodeGen/Mips/dsp-r2.ll b/llvm/test/CodeGen/Mips/dsp-r2.ll
index 631f9e43c23ac..584455cf0c522 100644
--- a/llvm/test/CodeGen/Mips/dsp-r2.ll
+++ b/llvm/test/CodeGen/Mips/dsp-r2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=+dspr2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+dspr2 < %s | FileCheck %s
 
 define i64 @test__builtin_mips_dpa_w_ph1(i32 %i0, i32, i64 %a0, i32 %a1.coerce, i32 %a2.coerce) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/dsp-vec-load-store.ll b/llvm/test/CodeGen/Mips/dsp-vec-load-store.ll
index 2feefcc09879a..b1d1f9703fc49 100644
--- a/llvm/test/CodeGen/Mips/dsp-vec-load-store.ll
+++ b/llvm/test/CodeGen/Mips/dsp-vec-load-store.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=+dsp < %s
+; RUN: llc -mtriple=mipsel -mattr=+dsp < %s
 
 @g1 = common global <2 x i8> zeroinitializer, align 2
 @g0 = common global <2 x i8> zeroinitializer, align 2
diff --git a/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll b/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
index c201c34f96861..daa0f79af4c35 100644
--- a/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
+++ b/llvm/test/CodeGen/Mips/dynamic-stack-realignment.ll
@@ -1,24 +1,24 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP32,GP32-M
-; RUN: llc < %s -march=mips -mcpu=mips32 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP32,GP32-M
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP32,GP32-M
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP32,GP32-MM,GP32-MMR2
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP32,GP32-MM,GP32-MMR6
-; RUN: llc < %s -march=mips64 -mcpu=mips3 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N64
-; RUN: llc < %s -march=mips64 -mcpu=mips64 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N64
-; RUN: llc < %s -march=mips64 -mcpu=mips3 -target-abi n32 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -target-abi n32 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N32
-; RUN: llc < %s -march=mips64 -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N32
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 -target-abi n32 -relocation-model=pic | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -target-abi n32 -relocation-model=pic | FileCheck %s \
 ; RUN:    --check-prefixes=ALL,GP64,N32
 
 ; Check dynamic stack realignment in functions without variable-sized objects.
diff --git a/llvm/test/CodeGen/Mips/eh-dwarf-cfa.ll b/llvm/test/CodeGen/Mips/eh-dwarf-cfa.ll
index 2377707b8f728..ce96605572ce5 100644
--- a/llvm/test/CodeGen/Mips/eh-dwarf-cfa.ll
+++ b/llvm/test/CodeGen/Mips/eh-dwarf-cfa.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck %s
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | \
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | \
 ; RUN:      FileCheck %s -check-prefix=CHECK-MIPS64
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | \
 ; RUN:      FileCheck %s -check-prefix=CHECK-MIPS64
 
 declare ptr @llvm.eh.dwarf.cfa(i32) nounwind
diff --git a/llvm/test/CodeGen/Mips/eh-return32.ll b/llvm/test/CodeGen/Mips/eh-return32.ll
index 983fc6f7788c7..50c8b1dca9901 100644
--- a/llvm/test/CodeGen/Mips/eh-return32.ll
+++ b/llvm/test/CodeGen/Mips/eh-return32.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -mcpu=mips32   -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
-; RUN: llc -march=mipsel -mcpu=mips32r2 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
-; RUN: llc -march=mipsel -mcpu=mips32r6 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,R6
+; RUN: llc -mtriple=mipsel -mcpu=mips32   -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
+; RUN: llc -mtriple=mipsel -mcpu=mips32r6 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,R6
 
 declare void @llvm.eh.return.i32(i32, ptr)
 declare void @foo(...)
diff --git a/llvm/test/CodeGen/Mips/eh-return64.ll b/llvm/test/CodeGen/Mips/eh-return64.ll
index 9ae2f00d46c1e..3a2fb2a4868d8 100644
--- a/llvm/test/CodeGen/Mips/eh-return64.ll
+++ b/llvm/test/CodeGen/Mips/eh-return64.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips64el -mcpu=mips4    -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
-; RUN: llc -march=mips64el -mcpu=mips64   -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
-; RUN: llc -march=mips64el -mcpu=mips64r2 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
-; RUN: llc -march=mips64el -mcpu=mips64r6 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,R6
+; RUN: llc -mtriple=mips64el -mcpu=mips4    -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
+; RUN: llc -mtriple=mips64el -mcpu=mips64   -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,NOT-R6
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -asm-show-inst -relocation-model=pic < %s | FileCheck %s -check-prefixes=CHECK,R6
 
 declare void @llvm.eh.return.i64(i64, ptr)
 declare void @foo(...)
diff --git a/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll b/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
index b1045a08dd3a6..3379322f0bf77 100644
--- a/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
+++ b/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -O0 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -O0 -relocation-model=pic < %s | FileCheck %s
 ; Check that register scavenging spill slot is close to $fp.
 target triple="mipsel--"
 
diff --git a/llvm/test/CodeGen/Mips/emit-big-cst.ll b/llvm/test/CodeGen/Mips/emit-big-cst.ll
index cd0666cfd3fc2..5a8852e38e3df 100644
--- a/llvm/test/CodeGen/Mips/emit-big-cst.ll
+++ b/llvm/test/CodeGen/Mips/emit-big-cst.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips < %s | FileCheck %s --check-prefix=BE
-; RUN: llc -march=mipsel < %s | FileCheck %s --check-prefix=LE
+; RUN: llc -mtriple=mips < %s | FileCheck %s --check-prefix=BE
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s --check-prefix=LE
 ; Check assembly printing of odd constants.
 
 ; BE-LABEL: bigCst:
diff --git a/llvm/test/CodeGen/Mips/ex2.ll b/llvm/test/CodeGen/Mips/ex2.ll
index 79aabfcbbfc43..bdc676713ad7b 100644
--- a/llvm/test/CodeGen/Mips/ex2.ll
+++ b/llvm/test/CodeGen/Mips/ex2.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
 @_ZTIPKc = external constant ptr
diff --git a/llvm/test/CodeGen/Mips/extins.ll b/llvm/test/CodeGen/Mips/extins.ll
index 9989c3c00dc79..e5c66602450e8 100644
--- a/llvm/test/CodeGen/Mips/extins.ll
+++ b/llvm/test/CodeGen/Mips/extins.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mips -mcpu=mips32r2 | FileCheck %s -check-prefix=32R2
-; RUN: llc  < %s -march=mips -mattr=mips16 | FileCheck %s -check-prefix=16
+; RUN: llc  < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s -check-prefix=32R2
+; RUN: llc  < %s -mtriple=mips -mattr=mips16 | FileCheck %s -check-prefix=16
 
 define i32 @ext0_5_9(i32 %s, i32 %pos, i32 %sz) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/fastcc.ll b/llvm/test/CodeGen/Mips/fastcc.ll
index ccdeedce6d235..e6d12664d83f4 100644
--- a/llvm/test/CodeGen/Mips/fastcc.ll
+++ b/llvm/test/CodeGen/Mips/fastcc.ll
@@ -1,8 +1,8 @@
-; RUN: llc < %s -march=mipsel -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=mipsel -relocation-model=pic | FileCheck %s
 ; RUN: llc < %s -mtriple=mipsel-none-nacl-gnu -relocation-model=pic -mips-tail-calls=1\
 ; RUN:  | FileCheck %s -check-prefix=CHECK-NACL
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -mattr=+nooddspreg -relocation-model=pic -mips-tail-calls=1| FileCheck %s -check-prefix=NOODDSPREG
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -mattr=+fp64,+nooddspreg -relocation-model=pic -mips-tail-calls=1 | FileCheck %s -check-prefix=FP64-NOODDSPREG
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32 -mattr=+nooddspreg -relocation-model=pic -mips-tail-calls=1| FileCheck %s -check-prefix=NOODDSPREG
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -mattr=+fp64,+nooddspreg -relocation-model=pic -mips-tail-calls=1 | FileCheck %s -check-prefix=FP64-NOODDSPREG
 
 
 @gi0 = external global i32
diff --git a/llvm/test/CodeGen/Mips/fcmp.ll b/llvm/test/CodeGen/Mips/fcmp.ll
index 34088beb2246d..c0b34454d6206 100644
--- a/llvm/test/CodeGen/Mips/fcmp.ll
+++ b/llvm/test/CodeGen/Mips/fcmp.ll
@@ -1,20 +1,20 @@
-; RUN: llc < %s -march=mips -mcpu=mips32 | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,32-C
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,32-C
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,32-CMP
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,64-C
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,64-C
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,64-C
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,64-CMP
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MM,MM32R3
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MM,MMR6,MM32R6
 
 define i32 @false_f32(float %a, float %b) nounwind {
diff --git a/llvm/test/CodeGen/Mips/fcopysign-f32-f64.ll b/llvm/test/CodeGen/Mips/fcopysign-f32-f64.ll
index 695431a5ab607..ed6b2e08a9b93 100644
--- a/llvm/test/CodeGen/Mips/fcopysign-f32-f64.ll
+++ b/llvm/test/CodeGen/Mips/fcopysign-f32-f64.ll
@@ -1,8 +1,8 @@
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips4 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips4 \
 ; RUN:   -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips64 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips64 \
 ; RUN:   -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips64r2 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips64r2 \
 ; RUN:   -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64R2
 
 declare double @copysign(double, double) nounwind readnone
diff --git a/llvm/test/CodeGen/Mips/fcopysign.ll b/llvm/test/CodeGen/Mips/fcopysign.ll
index 810d0f9580861..7ef9d6376e6dc 100644
--- a/llvm/test/CodeGen/Mips/fcopysign.ll
+++ b/llvm/test/CodeGen/Mips/fcopysign.ll
@@ -1,12 +1,12 @@
-; RUN: llc  < %s -verify-machineinstrs -march=mipsel -mcpu=mips32 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mipsel -mcpu=mips32 \
 ; RUN:   | FileCheck %s -check-prefix=32
-; RUN: llc  < %s -verify-machineinstrs -march=mipsel -mcpu=mips32r2 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mipsel -mcpu=mips32r2 \
 ; RUN:   | FileCheck %s -check-prefix=32R2
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips4 -target-abi=n64 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips4 -target-abi=n64 \
 ; RUN:   | FileCheck %s -check-prefix=64
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips64 -target-abi=n64 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips64 -target-abi=n64 \
 ; RUN:   | FileCheck %s -check-prefix=64
-; RUN: llc  < %s -verify-machineinstrs -march=mips64el -mcpu=mips64r2 -target-abi=n64 \
+; RUN: llc  < %s -verify-machineinstrs -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 \
 ; RUN:   | FileCheck %s -check-prefix=64R2
 
 define double @func0(double %d0, double %d1) nounwind readnone {
diff --git a/llvm/test/CodeGen/Mips/fmadd1.ll b/llvm/test/CodeGen/Mips/fmadd1.ll
index 4704387bf289b..b9380c45ca786 100644
--- a/llvm/test/CodeGen/Mips/fmadd1.ll
+++ b/llvm/test/CodeGen/Mips/fmadd1.ll
@@ -5,29 +5,29 @@
 ; IEEE 754 (1985) and IEEE 754 (2008). These instructions are therefore only
 ; available when -enable-no-nans-fp-math is given.
 
-; RUN: llc < %s -march=mipsel   -mcpu=mips32              -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32-NOMADD
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2            -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32R2,32R2-NONAN
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r6            -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32R6-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64,64-NONAN
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64R2,64R2-NONAN
-; RUN: llc < %s -march=mips64el -mcpu=mips64r6 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
-; RUN: llc < %s -march=mipsel   -mcpu=mips32              | FileCheck %s -check-prefixes=ALL,32-NOMADD
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2            | FileCheck %s -check-prefixes=ALL,32R2,32R2-NAN
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r6            | FileCheck %s -check-prefixes=ALL,32R6-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64   -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64,64-NAN
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64R2,64R2-NAN
-; RUN: llc < %s -march=mips64el -mcpu=mips64r6 -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32              -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2            -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32R2,32R2-NONAN
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r6            -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,32R6-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64,64-NONAN
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64R2,64R2-NONAN
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r6 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32              | FileCheck %s -check-prefixes=ALL,32-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2            | FileCheck %s -check-prefixes=ALL,32R2,32R2-NAN
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r6            | FileCheck %s -check-prefixes=ALL,32R6-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64,64-NAN
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64R2,64R2-NAN
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r6 -target-abi=n64 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
 
 ; Check that madd.[ds], msub.[ds], nmadd.[ds], and nmsub.[ds] are not generated
 ; when +nomadd attribute is specified.
 ; Output for mips32 and mips64r6 reused since aforementioned instructions are
 ; not generated in those cases.
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2            -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,32-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2            -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,32-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64   -target-abi=n64 -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2            -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,32-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2            -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,32-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -mattr=+nomadd4 | FileCheck %s -check-prefixes=ALL,64R6-NOMADD
 
 define float @FOO0float(float %a, float %b, float %c) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/fp-contract.ll b/llvm/test/CodeGen/Mips/fp-contract.ll
index f25e9f5258c77..c43d2a644bb36 100644
--- a/llvm/test/CodeGen/Mips/fp-contract.ll
+++ b/llvm/test/CodeGen/Mips/fp-contract.ll
@@ -1,11 +1,11 @@
 ; Test that the compiled does not fuse fmul and fadd into fmadd when no -fp-contract=fast
 ; option is set (the same applies for fmul, fsub and fmsub).
 
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK-CONTRACT-OFF
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -fp-contract=off < %s \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -fp-contract=off < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK-CONTRACT-OFF
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -fp-contract=fast < %s \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -fp-contract=fast < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK-CONTRACT-FAST
 
 declare <4 x float> @llvm.mips.fmul.w(<4 x float>, <4 x float>)
diff --git a/llvm/test/CodeGen/Mips/fp-indexed-ls.ll b/llvm/test/CodeGen/Mips/fp-indexed-ls.ll
index 48052b6e86143..8e20c8229847c 100644
--- a/llvm/test/CodeGen/Mips/fp-indexed-ls.ll
+++ b/llvm/test/CodeGen/Mips/fp-indexed-ls.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel   -mcpu=mips32   -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R1
-; RUN: llc -march=mipsel   -mcpu=mips32r2 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R2
-; RUN: llc -march=mipsel   -mcpu=mips32r6 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R6
-; RUN: llc -march=mips64el -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
-; RUN: llc -march=mips64el -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
-; RUN: llc -march=mips64el -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS64R6
+; RUN: llc -mtriple=mipsel   -mcpu=mips32   -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R1
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r2 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R2
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r6 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS32R6
+; RUN: llc -mtriple=mips64el -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
+; RUN: llc -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS4
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,MIPS64R6
 
 ; Check that [ls][dwu]xc1 are not emitted for nacl.
 ; RUN: llc -mtriple=mipsel-none-nacl-gnu -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=CHECK-NACL
diff --git a/llvm/test/CodeGen/Mips/fp-spill-reload.ll b/llvm/test/CodeGen/Mips/fp-spill-reload.ll
index 21a7ed040f0fd..8a01ec8732641 100644
--- a/llvm/test/CodeGen/Mips/fp-spill-reload.ll
+++ b/llvm/test/CodeGen/Mips/fp-spill-reload.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 ; check that $fp is not reserved. 
 
 define void @foo0(ptr nocapture %b) nounwind {
diff --git a/llvm/test/CodeGen/Mips/fp64a.ll b/llvm/test/CodeGen/Mips/fp64a.ll
index 317afd7003bba..392a4f6e4a2df 100644
--- a/llvm/test/CodeGen/Mips/fp64a.ll
+++ b/llvm/test/CodeGen/Mips/fp64a.ll
@@ -7,16 +7,16 @@
 ; We don't test MIPS32r1 since support for 64-bit coprocessors (such as a 64-bit
 ; FPU) on a 32-bit architecture was added in MIPS32r2.
 
-; RUN: not llc -march=mips -mcpu=mips32 -mattr=fp64 < %s 2>&1 | FileCheck %s -check-prefix=32R1-FP64
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,32R2-NO-FP64A-BE
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=fp64,nooddspreg < %s | FileCheck %s -check-prefixes=ALL,32R2-FP64A
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,32R2-NO-FP64A-LE
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=fp64,nooddspreg < %s | FileCheck %s -check-prefixes=ALL,32R2-FP64A
-
-; RUN: llc -march=mips64 -mcpu=mips64 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,64-NO-FP64A
-; RUN: not llc -march=mips64 -mcpu=mips64 -mattr=fp64,nooddspreg < %s 2>&1 | FileCheck %s -check-prefix=64-FP64A
-; RUN: llc -march=mips64el -mcpu=mips64 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,64-NO-FP64A
-; RUN: not llc -march=mips64el -mcpu=mips64 -mattr=fp64,nooddspreg < %s 2>&1 | FileCheck %s -check-prefix=64-FP64A
+; RUN: not llc -mtriple=mips -mcpu=mips32 -mattr=fp64 < %s 2>&1 | FileCheck %s -check-prefix=32R1-FP64
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,32R2-NO-FP64A-BE
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=fp64,nooddspreg < %s | FileCheck %s -check-prefixes=ALL,32R2-FP64A
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,32R2-NO-FP64A-LE
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=fp64,nooddspreg < %s | FileCheck %s -check-prefixes=ALL,32R2-FP64A
+
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,64-NO-FP64A
+; RUN: not llc -mtriple=mips64 -mcpu=mips64 -mattr=fp64,nooddspreg < %s 2>&1 | FileCheck %s -check-prefix=64-FP64A
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -mattr=fp64 < %s | FileCheck %s -check-prefixes=ALL,64-NO-FP64A
+; RUN: not llc -mtriple=mips64el -mcpu=mips64 -mattr=fp64,nooddspreg < %s 2>&1 | FileCheck %s -check-prefix=64-FP64A
 
 ; 32R1-FP64: LLVM ERROR: FPU with 64-bit registers is not available on MIPS32 pre revision 2. Use -mcpu=mips32r2 or greater.
 ; 64-FP64A: LLVM ERROR: -mattr=+nooddspreg requires the O32 ABI.
diff --git a/llvm/test/CodeGen/Mips/fpbr.ll b/llvm/test/CodeGen/Mips/fpbr.ll
index 251c5392575b2..9f7baca881fc0 100644
--- a/llvm/test/CodeGen/Mips/fpbr.ll
+++ b/llvm/test/CodeGen/Mips/fpbr.ll
@@ -1,9 +1,9 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32   -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,32-FCC
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,32-FCC
-; RUN: llc < %s -march=mipsel -mcpu=mips32r6 -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,GPR,32-GPR
-; RUN: llc < %s -march=mips64el -mcpu=mips64   | FileCheck %s -check-prefixes=ALL,64-FCC
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,64-FCC
-; RUN: llc < %s -march=mips64el -mcpu=mips64r6 | FileCheck %s -check-prefixes=ALL,GPR,64-GPR
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32   -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,32-FCC
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,32-FCC
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r6 -relocation-model=pic  | FileCheck %s -check-prefixes=ALL,GPR,32-GPR
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   | FileCheck %s -check-prefixes=ALL,64-FCC
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 | FileCheck %s -check-prefixes=ALL,64-FCC
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r6 | FileCheck %s -check-prefixes=ALL,GPR,64-GPR
 
 define void @func0(float %f2, float %f3) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/fpxx.ll b/llvm/test/CodeGen/Mips/fpxx.ll
index 6fdb95efe8ecf..a12aa74063565 100644
--- a/llvm/test/CodeGen/Mips/fpxx.ll
+++ b/llvm/test/CodeGen/Mips/fpxx.ll
@@ -1,20 +1,20 @@
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck %s -check-prefixes=ALL,32-NOFPXX
-; RUN: llc -march=mipsel -mcpu=mips32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,32-FPXX
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck %s -check-prefixes=ALL,32-NOFPXX
+; RUN: llc -mtriple=mipsel -mcpu=mips32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,32-FPXX
 
-; RUN: llc -march=mipsel -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=ALL,32R2-NOFPXX
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,32R2-FPXX
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=ALL,32R2-NOFPXX
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,32R2-FPXX
 
-; RUN: llc -march=mips64 -mcpu=mips4 < %s | FileCheck %s -check-prefixes=ALL,4-NOFPXX
-; RUN: not llc -march=mips64 -mcpu=mips4 -mattr=fpxx < %s 2>&1 | FileCheck %s -check-prefix=4-FPXX
+; RUN: llc -mtriple=mips64 -mcpu=mips4 < %s | FileCheck %s -check-prefixes=ALL,4-NOFPXX
+; RUN: not llc -mtriple=mips64 -mcpu=mips4 -mattr=fpxx < %s 2>&1 | FileCheck %s -check-prefix=4-FPXX
 
-; RUN: llc -march=mips64 -mcpu=mips64 < %s | FileCheck %s -check-prefixes=ALL,64-NOFPXX
-; RUN: not llc -march=mips64 -mcpu=mips64 -mattr=fpxx < %s 2>&1 | FileCheck %s -check-prefix=64-FPXX
+; RUN: llc -mtriple=mips64 -mcpu=mips64 < %s | FileCheck %s -check-prefixes=ALL,64-NOFPXX
+; RUN: not llc -mtriple=mips64 -mcpu=mips64 -mattr=fpxx < %s 2>&1 | FileCheck %s -check-prefix=64-FPXX
 
-; RUN-TODO: llc -march=mips64 -mcpu=mips4 -target-abi o32 < %s | FileCheck %s -check-prefixes=ALL,4-O32-NOFPXX
-; RUN-TODO: llc -march=mips64 -mcpu=mips4 -target-abi o32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,4-O32-FPXX
+; RUN-TODO: llc -mtriple=mips64 -mcpu=mips4 -target-abi o32 < %s | FileCheck %s -check-prefixes=ALL,4-O32-NOFPXX
+; RUN-TODO: llc -mtriple=mips64 -mcpu=mips4 -target-abi o32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,4-O32-FPXX
 
-; RUN-TODO: llc -march=mips64 -mcpu=mips64 -target-abi o32 < %s | FileCheck %s -check-prefixes=ALL,64-O32-NOFPXX
-; RUN-TODO: llc -march=mips64 -mcpu=mips64 -target-abi o32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,64-O32-FPXX
+; RUN-TODO: llc -mtriple=mips64 -mcpu=mips64 -target-abi o32 < %s | FileCheck %s -check-prefixes=ALL,64-O32-NOFPXX
+; RUN-TODO: llc -mtriple=mips64 -mcpu=mips64 -target-abi o32 -mattr=fpxx < %s | FileCheck %s -check-prefixes=ALL,64-O32-FPXX
 
 declare double @dbl();
 
diff --git a/llvm/test/CodeGen/Mips/frame-address-err.ll b/llvm/test/CodeGen/Mips/frame-address-err.ll
index 45f20ac8119c0..9b75d3c66e27e 100644
--- a/llvm/test/CodeGen/Mips/frame-address-err.ll
+++ b/llvm/test/CodeGen/Mips/frame-address-err.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -march=mips < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=mips < %s 2>&1 | FileCheck %s
 
 declare ptr @llvm.frameaddress(i32) nounwind readnone
 
diff --git a/llvm/test/CodeGen/Mips/frame-address.ll b/llvm/test/CodeGen/Mips/frame-address.ll
index 685d1fe1f4651..7e92e3e7de6da 100644
--- a/llvm/test/CodeGen/Mips/frame-address.ll
+++ b/llvm/test/CodeGen/Mips/frame-address.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 declare ptr @llvm.frameaddress(i32) nounwind readnone
 
diff --git a/llvm/test/CodeGen/Mips/frem.ll b/llvm/test/CodeGen/Mips/frem.ll
index be222b2d9172a..556ca6c2e7e7a 100644
--- a/llvm/test/CodeGen/Mips/frem.ll
+++ b/llvm/test/CodeGen/Mips/frem.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel 
+; RUN: llc < %s -mtriple=mipsel 
 
 define float @fmods(float %x, float %y) {
 entry:
diff --git a/llvm/test/CodeGen/Mips/global-pointer-reg.ll b/llvm/test/CodeGen/Mips/global-pointer-reg.ll
index 3cb2aaed47669..a616fa3ec2fa4 100644
--- a/llvm/test/CodeGen/Mips/global-pointer-reg.ll
+++ b/llvm/test/CodeGen/Mips/global-pointer-reg.ll
@@ -1,4 +1,4 @@
-; DISABLED: llc < %s -march=mipsel -mips-fix-global-base-reg=false | FileCheck %s 
+; DISABLED: llc < %s -mtriple=mipsel -mips-fix-global-base-reg=false | FileCheck %s 
 ; RUN: false
 ; XFAIL: *
 
diff --git a/llvm/test/CodeGen/Mips/gpreg-lazy-binding.ll b/llvm/test/CodeGen/Mips/gpreg-lazy-binding.ll
index 35cebdf302b5d..ae6627a8c768c 100644
--- a/llvm/test/CodeGen/Mips/gpreg-lazy-binding.ll
+++ b/llvm/test/CodeGen/Mips/gpreg-lazy-binding.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -disable-mips-delay-filler -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -disable-mips-delay-filler -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
 
 @g = external global i32
 
diff --git a/llvm/test/CodeGen/Mips/hf16_1.ll b/llvm/test/CodeGen/Mips/hf16_1.ll
index 327f04f648598..611e53f4fd2f6 100644
--- a/llvm/test/CodeGen/Mips/hf16_1.ll
+++ b/llvm/test/CodeGen/Mips/hf16_1.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=1
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=2
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=1
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=2
 
 
 @x = common global float 0.000000e+00, align 4
diff --git a/llvm/test/CodeGen/Mips/i64arg.ll b/llvm/test/CodeGen/Mips/i64arg.ll
index ec26cdc6776c3..2af16f7d23f3c 100644
--- a/llvm/test/CodeGen/Mips/i64arg.ll
+++ b/llvm/test/CodeGen/Mips/i64arg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
 
 define void @f1(i64 %ll1, float %f, i64 %ll, i32 %i, float %f2) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/imm.ll b/llvm/test/CodeGen/Mips/imm.ll
index eea391e8707e6..bf62df6992824 100644
--- a/llvm/test/CodeGen/Mips/imm.ll
+++ b/llvm/test/CodeGen/Mips/imm.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 define i32 @foo0() nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/inlineasm-assembler-directives.ll b/llvm/test/CodeGen/Mips/inlineasm-assembler-directives.ll
index 759e84b02007c..cb01e605e9a1f 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-assembler-directives.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-assembler-directives.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 ; Check for the emission of appropriate assembler directives before and
 ; after the inline assembly code.
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-R.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-R.ll
index aa75a1d0a9bc4..f04e64889e001 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-R.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-R.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 @data = global [8193 x i32] zeroinitializer
 
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-1.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-1.ll
index f3431aa4c8251..fb075c2686267 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-1.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -mcpu=mips32r6 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,09BIT
-; RUN: llc -march=mipsel -mattr=+micromips -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,12BIT
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,16BIT
+; RUN: llc -mtriple=mipsel -mcpu=mips32r6 -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,09BIT
+; RUN: llc -mtriple=mipsel -mattr=+micromips -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,12BIT
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefixes=ALL,16BIT
 
 @data = global [8193 x i32] zeroinitializer
 
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-2.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-2.ll
index b05f586b1351b..6071469e82bde 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-2.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-ZC-2.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32r6 < %s | FileCheck %s --check-prefixes=ALL,R6
-; RUN: llc -march=mips -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 < %s | FileCheck %s --check-prefixes=ALL,R6
+; RUN: llc -mtriple=mips -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic \
 ; RUN:     < %s | FileCheck %s --check-prefixes=ALL,R6
-; RUN: llc -march=mips -mcpu=mips32 < %s | FileCheck %s --check-prefixes=ALL,PRER6
-; RUN: llc -march=mips -mcpu=mips64 -target-abi=n64 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32 < %s | FileCheck %s --check-prefixes=ALL,PRER6
+; RUN: llc -mtriple=mips -mcpu=mips64 -target-abi=n64 -relocation-model=pic \
 ; RUN:     < %s | FileCheck %s --check-prefixes=ALL,PRER6
 
 
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-I-1.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-I-1.ll
index c09108dc07447..48b754f3a6ca3 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-I-1.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-I-1.ll
@@ -2,7 +2,7 @@
 ;This is a negative test. The constant value given for the constraint
 ;is greater than 16 bits.
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-J.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-J.ll
index 2b24b0f82c57f..1239263287f38 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-J.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-J.ll
@@ -2,7 +2,7 @@
 ;This is a negative test. The constant value given for the constraint (J)
 ;is non-zero (3).
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-K.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-K.ll
index 3baf437324acb..13a41e8dc04e4 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-K.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-K.ll
@@ -2,7 +2,7 @@
 ;This is a negative test. The constant value given for the constraint (K)
 ;is greater than 16 bits (0x00100000).
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-L.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-L.ll
index 5edb3e24674e0..75b42e8c93a0c 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-L.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-L.ll
@@ -2,7 +2,7 @@
 ;This is a negative test. The constant value given for the constraint (L)
 ;is non-zero in the lower 16 bits (0x00100003).
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-N.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-N.ll
index eaa540acdafab..84121ad8ddd4c 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-N.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-N.ll
@@ -3,7 +3,7 @@
 ;immediate in the range of -65535 to -1 (inclusive).
 ;Our example uses the positive value 3.
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-O.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-O.ll
index 56afbaaa9cd67..e4b760955a650 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-O.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-O.ll
@@ -3,7 +3,7 @@
 ;signed 15 bit immediate (+- 16383).
 ;Our example uses the positive value 16384.
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-P.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-P.ll
index 0a55cb55e5f23..c240e3457e487 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-P.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-bad-P.ll
@@ -3,7 +3,7 @@
 ; A constant in the range of 1 to 655535 inclusive.
 ; Our example uses the positive value 655536.
 ;
-; RUN: not llc -march=mipsel < %s  2> %t
+; RUN: not llc -mtriple=mipsel < %s  2> %t
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-m-1.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-m-1.ll
index a2a863671d40e..27be7815c19f0 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-m-1.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-m-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 @data = global [8193 x i32] zeroinitializer
 
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-m-2.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-m-2.ll
index 295b93b6f1822..9ec7f8196cd26 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-m-2.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-m-2.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mips -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips -relocation-model=pic < %s \
 ; RUN:   | FileCheck --check-prefixes=CHECK,EB %s
-; RUN: llc -march=mipsel -relocation-model=pic < %s \
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s \
 ; RUN:   | FileCheck --check-prefixes=CHECK,EL %s
 
 ; Simple memory
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-o.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-o.ll
index 550446deb5816..356c5f41f5270 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-o.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-o.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 @data = global [8193 x i32] zeroinitializer
 
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg.ll
index 4d3a288875fa5..19dd3d6b39a6e 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg.ll
@@ -1,7 +1,7 @@
 ; Positive test for inline register constraints
 ;
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mcpu=mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 < %s | FileCheck %s
 
 define i32 @main() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
index ae0504980cc2a..661c0cb717f73 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint-reg64.ll
@@ -4,7 +4,7 @@
 ; The target is 64 bit.
 ;
 ;
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
 
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraint.ll b/llvm/test/CodeGen/Mips/inlineasm-constraint.ll
index 164d28f733e4f..fbbe50b133b2c 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraint.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraint.ll
@@ -1,6 +1,6 @@
-; RUN: llc -no-integrated-as -march=mipsel < %s | \
+; RUN: llc -no-integrated-as -mtriple=mipsel < %s | \
 ; RUN:     FileCheck %s -check-prefixes=ALL,GAS
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefixes=ALL,IAS
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefixes=ALL,IAS
 
 define void @constraint_I() nounwind {
 ; First I with short
diff --git a/llvm/test/CodeGen/Mips/inlineasm-constraints-softfloat.ll b/llvm/test/CodeGen/Mips/inlineasm-constraints-softfloat.ll
index 705570f808ce0..6366972e2b495 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-constraints-softfloat.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-constraints-softfloat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -march=mips < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips64 < %s | FileCheck %s --check-prefix=MIPS64
+; RUN: llc -mtriple=mips < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mips64 < %s | FileCheck %s --check-prefix=MIPS64
 
 define dso_local void @read_double(ptr nocapture noundef readonly %0) local_unnamed_addr #0 {
 ; MIPS32-LABEL: read_double:
diff --git a/llvm/test/CodeGen/Mips/inlineasm-opcode-bad-y.ll b/llvm/test/CodeGen/Mips/inlineasm-opcode-bad-y.ll
index e94b608d047e7..3bf1a26641cf6 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-opcode-bad-y.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-opcode-bad-y.ll
@@ -1,7 +1,7 @@
 ; Negative test for the 'm' operand code. This operand code is applicable
 ; for an immediate whic is exact power of 2.
 
-; RUN: not llc -march=mips < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=mips < %s 2>&1 | FileCheck %s
 
 define i32 @foo() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/inlineasm-operand-code.ll b/llvm/test/CodeGen/Mips/inlineasm-operand-code.ll
index 88e56a89cd7bb..db9edb2974f9e 100644
--- a/llvm/test/CodeGen/Mips/inlineasm-operand-code.ll
+++ b/llvm/test/CodeGen/Mips/inlineasm-operand-code.ll
@@ -1,14 +1,14 @@
 ; Positive test for inline register constraints
 ;
-; RUN: llc -no-integrated-as -march=mipsel -relocation-model=pic < %s | \
+; RUN: llc -no-integrated-as -mtriple=mipsel -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefixes=ALL,LE32,GAS %s
-; RUN: llc -no-integrated-as -march=mips -relocation-model=pic < %s | \
+; RUN: llc -no-integrated-as -mtriple=mips -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefixes=ALL,BE32,GAS %s
 
 ; IAS might not print in the same way since it parses the assembly.
-; RUN: llc -march=mipsel -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefixes=ALL,LE32,IAS %s
-; RUN: llc -march=mips -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefixes=ALL,BE32,IAS %s
 
 %union.u_tag = type { i64 }
diff --git a/llvm/test/CodeGen/Mips/ins.ll b/llvm/test/CodeGen/Mips/ins.ll
index 615dc8f3d6ac5..06e493bcd759c 100644
--- a/llvm/test/CodeGen/Mips/ins.ll
+++ b/llvm/test/CodeGen/Mips/ins.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -O3 -mcpu=mips32r2 -mtriple=mipsel-linux-gnu < %s -o - \
 ; RUN:   | FileCheck %s --check-prefixes=MIPS32R2
-; RUN: llc -O3 -mcpu=mips64r2 -march=mips64el  < %s \
+; RUN: llc -O3 -mcpu=mips64r2 -mtriple=mips64el  < %s \
 ; RUN:   | FileCheck %s --check-prefixes=MIPS64R2
 
 define i32 @or_and_shl(i32 %a, i32 %b) {
diff --git a/llvm/test/CodeGen/Mips/instverify/dext-pos.mir b/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
index 02dd9085c31e8..1574eeb11a3d5 100644
--- a/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dext-size.mir b/llvm/test/CodeGen/Mips/instverify/dext-size.mir
index 97cb085eac505..2e750aade31a5 100644
--- a/llvm/test/CodeGen/Mips/instverify/dext-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dext-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
index e76af1be9493f..9a0e9a37282cd 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir b/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
index 2649683824472..e3862d7d242ea 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-size.mir b/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
index fc24a2756c6ab..74522e991d97a 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
index 7001221bb0db1..f69f20fcfb7e4 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir b/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
index b9e3b8c169e42..52e5b6d0f6921 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir b/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
index 6663b96494ab9..75c1c1e44c6c6 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:     -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK-NOT: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-size.mir b/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
index 8407a7a836f76..b001b42c2cf3c 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
index d72837850cfbe..c881d65898ee8 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-pos.mir b/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
index 71d4242c948c1..b074b87821947 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-size.mir b/llvm/test/CodeGen/Mips/instverify/dins-size.mir
index 35848a936e5c0..c9a3410aef673 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
index a00d3cf715a79..0ae16d1d43d6b 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
index 0bbbdd2322459..fe622e12e7fe7 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
index c1f5f044bdeea..2ae4dc7c9f2ba 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
index 9f9953a855658..34c509a016fc6 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
index 12e999d5d488f..7ce2c9509f9e7 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
index f204a3373f76f..9e185cb943f26 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
index c7b16fd50ab0e..da42bac7140c7 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-pos.mir b/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
index ce2abeb3cdcf4..1762d8942aee1 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-size.mir b/llvm/test/CodeGen/Mips/instverify/ext-size.mir
index 57737ea60283d..42158f63872c2 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
index 1e48f1e8a236c..72d6d690429e4 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position + Size is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-pos.mir b/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
index c72e6f5a3be30..c2464d3d94c24 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Position operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-size.mir b/llvm/test/CodeGen/Mips/instverify/ins-size.mir
index 92319bd3ff8a6..050e909933efb 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-size.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -march=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
+# RUN: not --crash llc -mtriple=mips64 -mcpu=mips64r2 -start-after=finalize-isel -stop-after=finalize-isel \
 # RUN:         -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
 
 # CHECK: Size operand is out of range!
diff --git a/llvm/test/CodeGen/Mips/int-to-float-conversion.ll b/llvm/test/CodeGen/Mips/int-to-float-conversion.ll
index 6444d1fe85563..84bc6a253595a 100644
--- a/llvm/test/CodeGen/Mips/int-to-float-conversion.ll
+++ b/llvm/test/CodeGen/Mips/int-to-float-conversion.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
 
 @i1 = global [3 x i32] [i32 1, i32 2, i32 3], align 4
 @i3 = common global ptr null, align 4
diff --git a/llvm/test/CodeGen/Mips/internalfunc.ll b/llvm/test/CodeGen/Mips/internalfunc.ll
index 7db46f54da537..34bf5d03d8f98 100644
--- a/llvm/test/CodeGen/Mips/internalfunc.ll
+++ b/llvm/test/CodeGen/Mips/internalfunc.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=mipsel -relocation-model=pic | FileCheck %s
 
 @caller.sf1 = internal unnamed_addr global ptr null, align 4
 @gf1 = external global ptr
diff --git a/llvm/test/CodeGen/Mips/jumptable_labels.ll b/llvm/test/CodeGen/Mips/jumptable_labels.ll
index 8ae22be9dd23a..fa8180847885a 100644
--- a/llvm/test/CodeGen/Mips/jumptable_labels.ll
+++ b/llvm/test/CodeGen/Mips/jumptable_labels.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mips < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64 -target-abi=n32 < %s | FileCheck %s -check-prefix=N32
-; RUN: llc -march=mips64 < %s | FileCheck %s -check-prefix=N64
+; RUN: llc -mtriple=mips < %s | FileCheck %s -check-prefix=O32
+; RUN: llc -mtriple=mips64 -target-abi=n32 < %s | FileCheck %s -check-prefix=N32
+; RUN: llc -mtriple=mips64 < %s | FileCheck %s -check-prefix=N64
 
 ; We only use the '$' prefix on O32. The others use the ELF convention.
 ; O32: $JTI0_0
diff --git a/llvm/test/CodeGen/Mips/largeimm1.ll b/llvm/test/CodeGen/Mips/largeimm1.ll
index adef511a4aadd..5d19879413637 100644
--- a/llvm/test/CodeGen/Mips/largeimm1.ll
+++ b/llvm/test/CodeGen/Mips/largeimm1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 define void @f() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/largeimmprinting.ll b/llvm/test/CodeGen/Mips/largeimmprinting.ll
index eed9e12eac8b0..efa0aa63abf67 100644
--- a/llvm/test/CodeGen/Mips/largeimmprinting.ll
+++ b/llvm/test/CodeGen/Mips/largeimmprinting.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips4 -target-abi=n64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips4 -target-abi=n64 -relocation-model=pic < %s | \
 ; RUN:     FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 -target-abi=n64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -target-abi=n64 -relocation-model=pic < %s | \
 ; RUN:     FileCheck %s -check-prefix=64
 
 %struct.S1 = type { [65536 x i8] }
diff --git a/llvm/test/CodeGen/Mips/lazy-binding.ll b/llvm/test/CodeGen/Mips/lazy-binding.ll
index 4f5c71effe696..e9860acb91bb4 100644
--- a/llvm/test/CodeGen/Mips/lazy-binding.ll
+++ b/llvm/test/CodeGen/Mips/lazy-binding.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck %s
 
 ; CHECK-LABEL: foo6:
 ; CHECK: %while.body
diff --git a/llvm/test/CodeGen/Mips/lb1.ll b/llvm/test/CodeGen/Mips/lb1.ll
index caff4c7fa33a4..7d93a9076d3c8 100644
--- a/llvm/test/CodeGen/Mips/lb1.ll
+++ b/llvm/test/CodeGen/Mips/lb1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @c = global i8 -1, align 1
 @.str = private unnamed_addr constant [5 x i8] c"%i \0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/lbu1.ll b/llvm/test/CodeGen/Mips/lbu1.ll
index 13fe20af1aef9..c492c0d4784d9 100644
--- a/llvm/test/CodeGen/Mips/lbu1.ll
+++ b/llvm/test/CodeGen/Mips/lbu1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @c = global i8 97, align 1
 @.str = private unnamed_addr constant [5 x i8] c"%c \0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/lh1.ll b/llvm/test/CodeGen/Mips/lh1.ll
index 4ec8d928cd063..5d4facc38ce6b 100644
--- a/llvm/test/CodeGen/Mips/lh1.ll
+++ b/llvm/test/CodeGen/Mips/lh1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @s = global i16 -1, align 2
 @.str = private unnamed_addr constant [5 x i8] c"%i \0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/lhu1.ll b/llvm/test/CodeGen/Mips/lhu1.ll
index 1d438b6093de7..561e8461552fe 100644
--- a/llvm/test/CodeGen/Mips/lhu1.ll
+++ b/llvm/test/CodeGen/Mips/lhu1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 
 @s = global i16 255, align 2
diff --git a/llvm/test/CodeGen/Mips/llcarry.ll b/llvm/test/CodeGen/Mips/llcarry.ll
index 4bba047ba1f0b..45e4cbd2a5970 100644
--- a/llvm/test/CodeGen/Mips/llcarry.ll
+++ b/llvm/test/CodeGen/Mips/llcarry.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i64 4294967295, align 8
 @j = global i64 15, align 8
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/abs.ll b/llvm/test/CodeGen/Mips/llvm-ir/abs.ll
index c0812977e3a11..e4dde185b48b5 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/abs.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/abs.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32                                    -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+abs2008,+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips                -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
+; RUN: llc -mtriple=mips -mcpu=mips32                                    -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+abs2008,+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips                -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
 
 define float @abs_s(float %a) {
 ; MIPS32: {{(ori|ins)}}
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/add.ll b/llvm/test/CodeGen/Mips/llvm-ir/add.ll
index 84c4bf677f945..6a08b29458485 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/add.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/add.ll
@@ -1,32 +1,32 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,NOT-R2-R6,GP32,PRE4
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,NOT-R2-R6,GP32,GP32-CMOV
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP32,GP32-CMOV
-; RUN: llc < %s -march=mips -mcpu=mips32r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP32,GP32-CMOV
-; RUN: llc < %s -march=mips -mcpu=mips32r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP32,GP32-CMOV
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP32
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,NOT-R2-R6,GP64,GP64-NOT-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,NOT-R2-R6,GP64,GP64-NOT-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,NOT-R2-R6,GP64,GP64-NOT-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP64,GP64-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP64,GP64-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP64,GP64-R2-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,R2-R6,GP64,GP64-R2-R6
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips -O2 -verify-machineinstrs | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -O2 -verify-machineinstrs | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MMR3,MM32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips -O2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -O2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MMR6,MM32
 
 
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/addrspacecast.ll b/llvm/test/CodeGen/Mips/llvm-ir/addrspacecast.ll
index bddbdc667bd6e..71a9ca80b95fa 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/addrspacecast.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/addrspacecast.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL
 
 ; Address spaces 1-255 are software defined.
 define ptr @cast(ptr %arg) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/arith-fp.ll b/llvm/test/CodeGen/Mips/llvm-ir/arith-fp.ll
index 33a86bbc5dee6..7b56deae3c090 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/arith-fp.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/arith-fp.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
+; RUN: llc -mtriple=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
 
 define double @add_d(double %a, double %b) {
 ; MIPS32:     add.d   {{.*}}         # <MCInst #{{[0-9]+}} FADD_D32
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/atomicrmx.ll b/llvm/test/CodeGen/Mips/llvm-ir/atomicrmx.ll
index 2e03ee9868a97..3a1a9bee56c25 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/atomicrmx.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/atomicrmx.ll
@@ -1,6 +1,6 @@
-; RUN: llc -asm-show-inst  -march=mipsel -mcpu=mips32r6 < %s | \
+; RUN: llc -asm-show-inst  -mtriple=mipsel -mcpu=mips32r6 < %s | \
 ; RUN:    FileCheck %s -check-prefix=CHK32
-; RUN: llc -asm-show-inst  -march=mips64el -mcpu=mips64r6 < %s | \
+; RUN: llc -asm-show-inst  -mtriple=mips64el -mcpu=mips64r6 < %s | \
 ; RUN:    FileCheck %s -check-prefix=CHK64
 
 @a = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/bitcast.ll b/llvm/test/CodeGen/Mips/llvm-ir/bitcast.ll
index a649234342dd2..a4a2f920a7d7d 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/bitcast.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/bitcast.ll
@@ -1,12 +1,12 @@
-; RUN: llc -march=mips -mcpu=mips32r2                         -asm-show-inst \
+; RUN: llc -mtriple=mips -mcpu=mips32r2                         -asm-show-inst \
 ; RUN: < %s | FileCheck %s --check-prefix=MIPS32R2
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst \
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst \
 ; RUN: < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst \
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst \
 ; RUN: < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst \
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst \
 ; RUN: < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst \
 ; RUN: < %s | FileCheck %s --check-prefix=MMR6
 
 define double @mthc1(i64 %a) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/call.ll b/llvm/test/CodeGen/Mips/llvm-ir/call.ll
index a259bdd04c2e4..a962e1763618a 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/call.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/call.ll
@@ -1,29 +1,29 @@
 ; Test the 'call' instruction and the tailcall variant.
 
-; RUN: llc -march=mips   -mcpu=mips32   -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r2 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r3 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r5 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r6 -relocation-model=pic -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,R6C
-; RUN: llc -march=mips   -mcpu=mips32r6 -relocation-model=pic -mattr=+fp64,+nooddspreg -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,R6C
-; RUN: llc -march=mips64 -mcpu=mips4    -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64   -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r2 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r3 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r5 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r6 -relocation-model=pic  -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,R6C
-; RUN: llc -march=mips   -mcpu=mips32   -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r2 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r3 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r5 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
-; RUN: llc -march=mips   -mcpu=mips32r6 -relocation-model=pic -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=R6C
-; RUN: llc -march=mips   -mcpu=mips32r6 -relocation-model=pic -mattr=+fp64,+nooddspreg -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=R6C
-; RUN: llc -march=mips64 -mcpu=mips4    -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64   -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r2 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r3 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r5 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
-; RUN: llc -march=mips64 -mcpu=mips64r6 -relocation-model=pic  -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32   -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r3 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r5 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -relocation-model=pic -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -relocation-model=pic -mattr=+fp64,+nooddspreg -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,O32,R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips4    -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic  -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefixes=ALL,N64,R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32   -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r3 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r5 -relocation-model=pic  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -relocation-model=pic -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=R6C
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -relocation-model=pic -mattr=+fp64,+nooddspreg -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=O32 -check-prefix=R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips4    -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic   -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=NOT-R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic  -disable-mips-delay-filler  -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=ALL -check-prefix=N64 -check-prefix=R6C
 
 declare void @extern_void_void()
 declare i32 @extern_i32_void()
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/cvt.ll b/llvm/test/CodeGen/Mips/llvm-ir/cvt.ll
index 04368f202fa56..0901c8cbfda7f 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/cvt.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/cvt.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
+; RUN: llc -mtriple=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
 
 ; TODO: Test for cvt_w_d is missing, could not generate instruction
 
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/extractelement.ll b/llvm/test/CodeGen/Mips/llvm-ir/extractelement.ll
index 4f926cbee0b21..290491a13d773 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/extractelement.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/extractelement.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL
 
 ; This test triggered a bug in the vector splitting where the type legalizer
 ; attempted to extract the element with by storing the vector, then reading
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/indirectbr.ll b/llvm/test/CodeGen/Mips/llvm-ir/indirectbr.ll
index aebeac9e5bd21..c5176669fec2e 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/indirectbr.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/indirectbr.ll
@@ -1,16 +1,16 @@
 ; Test all important variants of the unconditional 'br' instruction.
 
-; RUN: llc -march=mips   -mcpu=mips32   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,R6C
-; RUN: llc -march=mips64 -mcpu=mips4    -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,R6
+; RUN: llc -mtriple=mips   -mcpu=mips32   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips4    -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,R6
 
 define i32 @br(ptr %addr) {
 ; ALL-LABEL: br:
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/lh_lhu.ll b/llvm/test/CodeGen/Mips/llvm-ir/lh_lhu.ll
index 4e434617d39ef..f38aa78dd587e 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/lh_lhu.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/lh_lhu.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips -relocation-model=pic | FileCheck %s
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic | FileCheck %s
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | FileCheck %s
 
 @us = global i16 0, align 2
 
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/load-atomic.ll b/llvm/test/CodeGen/Mips/llvm-ir/load-atomic.ll
index f401b0a1d6c9e..16c09003298c2 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/load-atomic.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/load-atomic.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=ALL
-; RUN: llc -march=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefix=ALL
-; RUN: llc -march=mips64 -mcpu=mips64r2 < %s | \
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=ALL
+; RUN: llc -mtriple=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefix=ALL
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 < %s | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,M64
-; RUN: llc -march=mips64 -mcpu=mips64r6 < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 < %s | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,M64
 
 define i8 @load_i8(ptr %ptr) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/mul.ll b/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
index 19955e4f9bbd5..00b91d1413cfe 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/mul.ll
@@ -1,30 +1,30 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,M2,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,32R1-R5,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,32R1-R5,32R2-R5,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,32R1-R5,32R2-R5,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r5 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,32R1-R5,32R2-R5,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,32R6,GP32
-; RUN: llc < %s -march=mips64 -mcpu=mips4 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,M4,GP64-NOT-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,64R1-R5,GP64-NOT-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,64R1-R5,GP64-NOT-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,64R1-R5,GP64-NOT-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,64R1-R5,GP64-NOT-R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,64R6
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=MM32,MM32R3
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic | \
 ; RUN:   FileCheck %s -check-prefixes=MM32,MM32R6
 
 define signext i1 @mul_i1(i1 signext %a, i1 signext %b) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/nan-fp-attr.ll b/llvm/test/CodeGen/Mips/llvm-ir/nan-fp-attr.ll
index 918cda6b38c26..34c7720ecbe03 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/nan-fp-attr.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/nan-fp-attr.ll
@@ -1,20 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s --check-prefix=MIPS32R1
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s --check-prefix=MIPS32R2
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS32R2-ABS2K8
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+abs2008,+fp64 | FileCheck %s --check-prefix=MIPS32R2-ABS2K8
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+fp64 | FileCheck %s --check-prefix=MIPS32R2
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s --check-prefix=MIPS32R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s --check-prefix=MIPS64R1
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s --check-prefix=MIPS64R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS64R2-ABS2K8
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s --check-prefix=MIPS64R6
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS64R6-ABS2K8
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips | FileCheck %s --check-prefix=MM
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips,+abs2008 | FileCheck %s --check-prefix=MM-ABS2K8
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips,+abs2008,+fp64 | FileCheck %s --check-prefix=MM-ABS2K8
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips,+fp64 | FileCheck %s --check-prefix=MM
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s --check-prefix=MMR6
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s --check-prefix=MIPS32R1
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s --check-prefix=MIPS32R2
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS32R2-ABS2K8
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+abs2008,+fp64 | FileCheck %s --check-prefix=MIPS32R2-ABS2K8
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+fp64 | FileCheck %s --check-prefix=MIPS32R2
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s --check-prefix=MIPS32R6
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s --check-prefix=MIPS64R1
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s --check-prefix=MIPS64R2
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS64R2-ABS2K8
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s --check-prefix=MIPS64R6
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -mattr=+abs2008 | FileCheck %s --check-prefix=MIPS64R6-ABS2K8
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips | FileCheck %s --check-prefix=MM
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips,+abs2008 | FileCheck %s --check-prefix=MM-ABS2K8
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips,+abs2008,+fp64 | FileCheck %s --check-prefix=MM-ABS2K8
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips,+fp64 | FileCheck %s --check-prefix=MM
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s --check-prefix=MMR6
 
 ; Test that the instruction selection for the case of `abs.s` and `abs.d`
 ; matches the expected behaviour. In the default case with NaNs and no "abs2008"
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/not.ll b/llvm/test/CodeGen/Mips/llvm-ir/not.ll
index 03ba8e562ef05..58ed0e66d5e21 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/not.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/not.ll
@@ -1,30 +1,30 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s -check-prefixes=ALL,GP32
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP32
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,GP64
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MM,MM32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MM,MM32
 
 define signext i1 @not_i1(i1 signext %a) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/ret.ll b/llvm/test/CodeGen/Mips/llvm-ir/ret.ll
index 7b37784370662..550ee0395ab77 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/ret.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/ret.ll
@@ -7,23 +7,23 @@
 ; affects it and it's undesirable to repeat the non-pointer returns for each
 ; relocation model.
 
-; RUN: llc -march=mips   -mcpu=mips32   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,NO-MTHC1,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
-; RUN: llc -march=mips   -mcpu=mips32r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,R6C
-; RUN: llc -march=mips64 -mcpu=mips4    -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
-; RUN: llc -march=mips64 -mcpu=mips64r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,NO-MTHC1,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,NOT-R6
+; RUN: llc -mtriple=mips   -mcpu=mips32r6 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR32,MTHC1,R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips4    -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64   -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -asm-show-inst < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,NOT-R6
 
 ; FIXME: for the test ret_double_0x0, the delay slot of jr cannot be filled
 ;        as mthc1 has unmodeled side effects. This is an artifact of our backend.
 ;        Force the delay slot filler off to check that the sequence jr $ra; nop is
 ;        turned into jic 0, $ra.
 
-; RUN: llc -march=mips64 -mcpu=mips64r6 -asm-show-inst -disable-mips-delay-filler < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,R6C
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -asm-show-inst -disable-mips-delay-filler < %s | FileCheck %s -check-prefixes=ALL,GPR64,DMTC1,R6C
 
 define void @ret_void() {
 ; ALL-LABEL: ret_void:
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/select-int.ll b/llvm/test/CodeGen/Mips/llvm-ir/select-int.ll
index 8bf83c5c18ce7..12728177380fc 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/select-int.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/select-int.ll
@@ -1,32 +1,32 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,M2,M2-M3
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-32
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-32
-; RUN: llc < %s -march=mips -mcpu=mips32r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-32
-; RUN: llc < %s -march=mips -mcpu=mips32r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-32
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,SEL,SEL-32
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,M3,M2-M3
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-64
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,CMOV,CMOV-64
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,SEL,SEL-64
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips -asm-show-inst | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -asm-show-inst | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MM32R3
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=ALL,MMR6,MM32R6
 
 define signext i1 @tst_select_i1_i1(i1 signext %s,
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/sqrt.ll b/llvm/test/CodeGen/Mips/llvm-ir/sqrt.ll
index 05776886dd947..f4b86257b5b70 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/sqrt.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/sqrt.ll
@@ -1,11 +1,11 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -mattr=+micromips | FileCheck %s
-; RUN: llc < %s -march=mips -mcpu=mips32r2 -mattr=+micromips | FileCheck %s
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s
-; RUN: llc -march=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -mattr=+micromips | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
 
 define float @sqrt_fn(float %value) #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/store-atomic.ll b/llvm/test/CodeGen/Mips/llvm-ir/store-atomic.ll
index 09713805c101c..b9ffbff0c6bc6 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/store-atomic.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/store-atomic.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=ALL
-; RUN: llc -march=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefix=ALL
-; RUN: llc -march=mips64 -mcpu=mips64r2 < %s | \
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=ALL
+; RUN: llc -mtriple=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefix=ALL
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 < %s | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,M64
-; RUN: llc -march=mips64 -mcpu=mips64r6 < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 < %s | \
 ; RUN:    FileCheck %s -check-prefixes=ALL,M64
 
 define void @store_i8(ptr %ptr, i8 signext %v) {
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/sub.ll b/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
index 51dcccefc84d3..b465e24d47a05 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/sub.ll
@@ -1,32 +1,32 @@
-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips2 | FileCheck %s \
 ; RUN:    -check-prefixes=NOT-R2-R6,GP32,NOT-MM,PRE4
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32 | FileCheck %s \
 ; RUN:    -check-prefixes=NOT-R2-R6,GP32,NOT-MM
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP32,NOT-MM
-; RUN: llc < %s -march=mips -mcpu=mips32r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP32,NOT-MM
-; RUN: llc < %s -march=mips -mcpu=mips32r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP32,NOT-MM
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP32,NOT-MM
-; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips -verify-machineinstrs | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -verify-machineinstrs | FileCheck %s \
 ; RUN:    -check-prefixes=MM32,MMR3
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips | FileCheck %s \
 ; RUN:    -check-prefixes=MM32,MMR6
-; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 | FileCheck %s \
 ; RUN:    -check-prefixes=NOT-R2-R6,GP64,NOT-MM,GP64-NOT-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 | FileCheck %s \
 ; RUN:    -check-prefixes=NOT-R2-R6,GP64,NOT-MM,GP64-NOT-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s \
 ; RUN:    -check-prefixes=NOT-R2-R6,GP64,NOT-MM,GP64-NOT-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP64,NOT-MM,GP64-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r3 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP64,NOT-MM,GP64-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r5 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP64,NOT-MM,GP64-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
+; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 | FileCheck %s \
 ; RUN:    -check-prefixes=R2-R6,GP64,NOT-MM,GP64-R2
 
 define signext i1 @sub_i1(i1 signext %a, i1 signext %b) {
diff --git a/llvm/test/CodeGen/Mips/load-store-left-right.ll b/llvm/test/CodeGen/Mips/load-store-left-right.ll
index 3c3110341df26..3925b73527b43 100644
--- a/llvm/test/CodeGen/Mips/load-store-left-right.ll
+++ b/llvm/test/CodeGen/Mips/load-store-left-right.ll
@@ -1,18 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mipsel   -mcpu=mips32              -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EL %s
-; RUN: llc -march=mips     -mcpu=mips32              -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EB %s
-; RUN: llc -march=mipsel   -mcpu=mips32r2            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EL %s
-; RUN: llc -march=mips     -mcpu=mips32r2            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EB %s
-; RUN: llc -march=mipsel   -mcpu=mips32r6            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32R6,MIPS32R6-EL %s
-; RUN: llc -march=mips     -mcpu=mips32r6            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32R6,MIPS32R6-EB %s
-; RUN: llc -march=mips64el -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EL %s
-; RUN: llc -march=mips64   -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EB %s
-; RUN: llc -march=mips64el -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EL %s
-; RUN: llc -march=mips64   -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EB %s
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64R2-EL %s
-; RUN: llc -march=mips64   -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64R2-EB %s
-; RUN: llc -march=mips64el -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64R6 %s
-; RUN: llc -march=mips64   -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64R6 %s
+; RUN: llc -mtriple=mipsel   -mcpu=mips32              -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EL %s
+; RUN: llc -mtriple=mips     -mcpu=mips32              -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EB %s
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r2            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EL %s
+; RUN: llc -mtriple=mips     -mcpu=mips32r2            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32,MIPS32-EB %s
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r6            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32R6,MIPS32R6-EL %s
+; RUN: llc -mtriple=mips     -mcpu=mips32r6            -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS32R6,MIPS32R6-EB %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EL %s
+; RUN: llc -mtriple=mips64   -mcpu=mips4    -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EB %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EL %s
+; RUN: llc -mtriple=mips64   -mcpu=mips64   -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64-EB %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64R2-EL %s
+; RUN: llc -mtriple=mips64   -mcpu=mips64r2 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64,MIPS64R2-EB %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64R6 %s
+; RUN: llc -mtriple=mips64   -mcpu=mips64r6 -target-abi=n64 -relocation-model=pic < %s | FileCheck -check-prefixes=MIPS64R6 %s
 
 %struct.SLL = type { i64 }
 %struct.SI = type { i32 }
diff --git a/llvm/test/CodeGen/Mips/long-call-attr.ll b/llvm/test/CodeGen/Mips/long-call-attr.ll
index beda290a9725b..1c25a36cf1236 100644
--- a/llvm/test/CodeGen/Mips/long-call-attr.ll
+++ b/llvm/test/CodeGen/Mips/long-call-attr.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=O32 %s
-; RUN: llc -march=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=O32 %s
-; RUN: llc -march=mips64 -target-abi n64 --mattr=+long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips64 -target-abi n64 --mattr=+long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=N64 %s
-; RUN: llc -march=mips64 -target-abi n64 --mattr=-long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips64 -target-abi n64 --mattr=-long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=N64 %s
 
 declare void @far() #0
diff --git a/llvm/test/CodeGen/Mips/long-call-mcount.ll b/llvm/test/CodeGen/Mips/long-call-mcount.ll
index 40d3b74ee5797..2e90a47ad0f01 100644
--- a/llvm/test/CodeGen/Mips/long-call-mcount.ll
+++ b/llvm/test/CodeGen/Mips/long-call-mcount.ll
@@ -1,7 +1,7 @@
 ; Check call to mcount in case of long/short call options.
-; RUN: llc -march=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefixes=CHECK,LONG %s
-; RUN: llc -march=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
+; RUN: llc -mtriple=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
 ; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefixes=CHECK,SHORT %s
 
 define void @foo() {
diff --git a/llvm/test/CodeGen/Mips/long-calls.ll b/llvm/test/CodeGen/Mips/long-calls.ll
index 99fce03cd0059..5ee1ee1ed575e 100644
--- a/llvm/test/CodeGen/Mips/long-calls.ll
+++ b/llvm/test/CodeGen/Mips/long-calls.ll
@@ -1,19 +1,19 @@
-; RUN: llc -march=mips -mattr=-long-calls %s -o - \
+; RUN: llc -mtriple=mips -mattr=-long-calls %s -o - \
 ; RUN:   | FileCheck -check-prefix=OFF %s
-; RUN: llc -march=mips -mattr=+long-calls,+noabicalls %s -o - \
+; RUN: llc -mtriple=mips -mattr=+long-calls,+noabicalls %s -o - \
 ; RUN:   | FileCheck -check-prefix=ON32 %s
 
-; RUN: llc -march=mips -mattr=+long-calls,-noabicalls %s -o - \
+; RUN: llc -mtriple=mips -mattr=+long-calls,-noabicalls %s -o - \
 ; RUN:   | FileCheck -check-prefix=OFF %s
 
-; RUN: llc -march=mips64 -target-abi n32 -mattr=-long-calls %s -o - \
+; RUN: llc -mtriple=mips64 -target-abi n32 -mattr=-long-calls %s -o - \
 ; RUN:   | FileCheck -check-prefix=OFF %s
-; RUN: llc -march=mips64 -target-abi n32 -mattr=+long-calls,+noabicalls %s -o - \
+; RUN: llc -mtriple=mips64 -target-abi n32 -mattr=+long-calls,+noabicalls %s -o - \
 ; RUN:   | FileCheck -check-prefix=ON32 %s
 
-; RUN: llc -march=mips64 -target-abi n64 -mattr=-long-calls %s -o - \
+; RUN: llc -mtriple=mips64 -target-abi n64 -mattr=-long-calls %s -o - \
 ; RUN:   | FileCheck -check-prefix=OFF %s
-; RUN: llc -march=mips64 -target-abi n64 -mattr=+long-calls,+noabicalls %s -o - \
+; RUN: llc -mtriple=mips64 -target-abi n64 -mattr=+long-calls,+noabicalls %s -o - \
 ; RUN:   | FileCheck -check-prefix=ON64 %s
 
 declare void @callee()
diff --git a/llvm/test/CodeGen/Mips/longbranch/compact-branches-long-branch.ll b/llvm/test/CodeGen/Mips/longbranch/compact-branches-long-branch.ll
index 709cd477a778e..3f6ac94801eb3 100644
--- a/llvm/test/CodeGen/Mips/longbranch/compact-branches-long-branch.ll
+++ b/llvm/test/CodeGen/Mips/longbranch/compact-branches-long-branch.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -mcpu=mips32r6 -force-mips-long-branch | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -force-mips-long-branch | FileCheck %s
 
 ; Check that when MIPS32R6 with the static relocation model with the usage of
 ; long branches, that there is a nop between any compact branch and the static
diff --git a/llvm/test/CodeGen/Mips/machineverifier.ll b/llvm/test/CodeGen/Mips/machineverifier.ll
index 39d2a7e4a9360..73cb2c62ee055 100644
--- a/llvm/test/CodeGen/Mips/machineverifier.ll
+++ b/llvm/test/CodeGen/Mips/machineverifier.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel -verify-machineinstrs
+; RUN: llc < %s -mtriple=mipsel -verify-machineinstrs
 ; Make sure machine verifier understands the last instruction of a basic block
 ; is not the terminator instruction after delay slot filler pass is run.
 
diff --git a/llvm/test/CodeGen/Mips/madd-msub.ll b/llvm/test/CodeGen/Mips/madd-msub.ll
index 9f7145685ed31..6bb1ea48184f9 100644
--- a/llvm/test/CodeGen/Mips/madd-msub.ll
+++ b/llvm/test/CodeGen/Mips/madd-msub.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mcpu=mips32 < %s | FileCheck %s -check-prefixes=32
-; RUN: llc -march=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=32
-; RUN: llc -march=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefixes=32R6
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=dsp < %s | FileCheck %s -check-prefix=DSP
-; RUN: llc -march=mips -mcpu=mips64   -target-abi n64 < %s | FileCheck %s -check-prefixes=64
-; RUN: llc -march=mips -mcpu=mips64r2 -target-abi n64 < %s | FileCheck %s -check-prefixes=64
-; RUN: llc -march=mips -mcpu=mips64r6 -target-abi n64 < %s | FileCheck %s -check-prefixes=64R6
-; RUN: llc -march=mips -mattr=mips16 < %s | FileCheck %s -check-prefixes=16
+; RUN: llc -mtriple=mips -mcpu=mips32 < %s | FileCheck %s -check-prefixes=32
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefixes=32
+; RUN: llc -mtriple=mips -mcpu=mips32r6 < %s | FileCheck %s -check-prefixes=32R6
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=dsp < %s | FileCheck %s -check-prefix=DSP
+; RUN: llc -mtriple=mips -mcpu=mips64   -target-abi n64 < %s | FileCheck %s -check-prefixes=64
+; RUN: llc -mtriple=mips -mcpu=mips64r2 -target-abi n64 < %s | FileCheck %s -check-prefixes=64
+; RUN: llc -mtriple=mips -mcpu=mips64r6 -target-abi n64 < %s | FileCheck %s -check-prefixes=64R6
+; RUN: llc -mtriple=mips -mattr=mips16 < %s | FileCheck %s -check-prefixes=16
 
 define i64 @madd1(i32 %a, i32 %b, i32 %c) nounwind readnone {
 ; 32-LABEL: madd1:
diff --git a/llvm/test/CodeGen/Mips/mcount.ll b/llvm/test/CodeGen/Mips/mcount.ll
index 8a129536d9769..b45b59eedbc9c 100644
--- a/llvm/test/CodeGen/Mips/mcount.ll
+++ b/llvm/test/CodeGen/Mips/mcount.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 
-; RUN: llc -march=mips -verify-machineinstrs \
+; RUN: llc -mtriple=mips -verify-machineinstrs \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS32
-; RUN: llc -march=mips -verify-machineinstrs -relocation-model=pic \
+; RUN: llc -mtriple=mips -verify-machineinstrs -relocation-model=pic \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-PIC
-; RUN: llc -march=mips64 -verify-machineinstrs \
+; RUN: llc -mtriple=mips64 -verify-machineinstrs \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS64
-; RUN: llc -march=mips64 -verify-machineinstrs -relocation-model=pic \
+; RUN: llc -mtriple=mips64 -verify-machineinstrs -relocation-model=pic \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS64-PIC
-; RUN: llc -march=mips -verify-machineinstrs -mattr=+micromips \
+; RUN: llc -mtriple=mips -verify-machineinstrs -mattr=+micromips \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-MM
-; RUN: llc -march=mips -verify-machineinstrs -relocation-model=pic -mattr=+micromips \
+; RUN: llc -mtriple=mips -verify-machineinstrs -relocation-model=pic -mattr=+micromips \
 ; RUN:   < %s | FileCheck %s -check-prefix=MIPS32-MM-PIC
 
 ; Test that checks ABI for _mcount calls.
diff --git a/llvm/test/CodeGen/Mips/memcpy.ll b/llvm/test/CodeGen/Mips/memcpy.ll
index 554452457166d..009a2f99f5ce3 100644
--- a/llvm/test/CodeGen/Mips/memcpy.ll
+++ b/llvm/test/CodeGen/Mips/memcpy.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s 
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s 
 
 %struct.S1 = type { i32, [41 x i8] }
 
diff --git a/llvm/test/CodeGen/Mips/micromips-addiu.ll b/llvm/test/CodeGen/Mips/micromips-addiu.ll
index 5165994126a2a..4129e6f6e0b48 100644
--- a/llvm/test/CodeGen/Mips/micromips-addiu.ll
+++ b/llvm/test/CodeGen/Mips/micromips-addiu.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 @x = global i32 65504, align 4
diff --git a/llvm/test/CodeGen/Mips/micromips-addu16.ll b/llvm/test/CodeGen/Mips/micromips-addu16.ll
index 04243fc899f86..75a0f2dd4147e 100644
--- a/llvm/test/CodeGen/Mips/micromips-addu16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-addu16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 define i32 @main() {
diff --git a/llvm/test/CodeGen/Mips/micromips-and16.ll b/llvm/test/CodeGen/Mips/micromips-and16.ll
index 66cfb0807f9d9..8a4da68244284 100644
--- a/llvm/test/CodeGen/Mips/micromips-and16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-and16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 define i32 @main() {
diff --git a/llvm/test/CodeGen/Mips/micromips-andi.ll b/llvm/test/CodeGen/Mips/micromips-andi.ll
index c84a99fbef415..8841e402f2b92 100644
--- a/llvm/test/CodeGen/Mips/micromips-andi.ll
+++ b/llvm/test/CodeGen/Mips/micromips-andi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 @x = global i32 65504, align 4
diff --git a/llvm/test/CodeGen/Mips/micromips-attr.ll b/llvm/test/CodeGen/Mips/micromips-attr.ll
index 1ea9b136e1d40..8e70cc616e0b7 100644
--- a/llvm/test/CodeGen/Mips/micromips-attr.ll
+++ b/llvm/test/CodeGen/Mips/micromips-attr.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32 --mattr=-micromips < %s | FileCheck %s 
+; RUN: llc -mtriple=mips -mcpu=mips32 --mattr=-micromips < %s | FileCheck %s 
 
 define void @foo() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/micromips-compact-branches.ll b/llvm/test/CodeGen/Mips/micromips-compact-branches.ll
index 3b05e9e174f35..fb04f8f5ec6b1 100644
--- a/llvm/test/CodeGen/Mips/micromips-compact-branches.ll
+++ b/llvm/test/CodeGen/Mips/micromips-compact-branches.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -march=mipsel -mattr=micromips -filetype=asm -O3 \
+; RUN: llc %s -mtriple=mipsel -mattr=micromips -filetype=asm -O3 \
 ; RUN: -disable-mips-delay-filler -relocation-model=pic -o - | FileCheck %s
 
 define void @main() nounwind uwtable {
diff --git a/llvm/test/CodeGen/Mips/micromips-compact-jump.ll b/llvm/test/CodeGen/Mips/micromips-compact-jump.ll
index 70cff84e967f0..a4c5e5fc492ce 100644
--- a/llvm/test/CodeGen/Mips/micromips-compact-jump.ll
+++ b/llvm/test/CodeGen/Mips/micromips-compact-jump.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -disable-mips-delay-filler -O3 < %s | FileCheck %s
 
 define i32 @foo(i32 signext %a) #0 {
diff --git a/llvm/test/CodeGen/Mips/micromips-gp-rc.ll b/llvm/test/CodeGen/Mips/micromips-gp-rc.ll
index 95f557770a10d..ae1a93e884883 100644
--- a/llvm/test/CodeGen/Mips/micromips-gp-rc.ll
+++ b/llvm/test/CodeGen/Mips/micromips-gp-rc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 @g = external global i32
diff --git a/llvm/test/CodeGen/Mips/micromips-li.ll b/llvm/test/CodeGen/Mips/micromips-li.ll
index bfac23c25ea5b..ebe7e3c90ac36 100644
--- a/llvm/test/CodeGen/Mips/micromips-li.ll
+++ b/llvm/test/CodeGen/Mips/micromips-li.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 @x = external global i32
diff --git a/llvm/test/CodeGen/Mips/micromips-load-effective-address.ll b/llvm/test/CodeGen/Mips/micromips-load-effective-address.ll
index 5215c3a519989..b41a4080e64ec 100644
--- a/llvm/test/CodeGen/Mips/micromips-load-effective-address.ll
+++ b/llvm/test/CodeGen/Mips/micromips-load-effective-address.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -march=mipsel -mattr=micromips -filetype=asm \
+; RUN: llc %s -mtriple=mipsel -mattr=micromips -filetype=asm \
 ; RUN: -relocation-model=pic -O3 -o - | FileCheck %s
 
 define i32 @sum(ptr %x, ptr %y) nounwind uwtable {
diff --git a/llvm/test/CodeGen/Mips/micromips-lwc1-swc1.ll b/llvm/test/CodeGen/Mips/micromips-lwc1-swc1.ll
index 29d46bed4226d..4884d0a22a244 100644
--- a/llvm/test/CodeGen/Mips/micromips-lwc1-swc1.ll
+++ b/llvm/test/CodeGen/Mips/micromips-lwc1-swc1.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips \
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips \
 ; RUN:   -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MM32
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips \
 ; RUN:   -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MM32
 
diff --git a/llvm/test/CodeGen/Mips/micromips-not16.ll b/llvm/test/CodeGen/Mips/micromips-not16.ll
index 2def472fce40a..ef1ff01f9b2c7 100644
--- a/llvm/test/CodeGen/Mips/micromips-not16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-not16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 define i32 @main() {
diff --git a/llvm/test/CodeGen/Mips/micromips-or16.ll b/llvm/test/CodeGen/Mips/micromips-or16.ll
index ae2c53884ef73..d82d3f68fc38e 100644
--- a/llvm/test/CodeGen/Mips/micromips-or16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-or16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
-; RUN: llc -O0 -march=mips -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -O0 -mtriple=mips -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -asm-show-inst < %s | FileCheck %s
 
 ; Branch instruction added to enable FastISel::selectOperator
diff --git a/llvm/test/CodeGen/Mips/micromips-shift.ll b/llvm/test/CodeGen/Mips/micromips-shift.ll
index 11472b8ac99fa..9d2a5ec075ed2 100644
--- a/llvm/test/CodeGen/Mips/micromips-shift.ll
+++ b/llvm/test/CodeGen/Mips/micromips-shift.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mcpu=mips32r6 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r6 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 @a = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir b/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
index 216fec6a7f6ef..cadb440cfdab1 100644
--- a/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
+++ b/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips -mcpu=mips32r2 -mattr=+micromips %s -o - \
+# RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+micromips %s -o - \
 # RUN:     -start-after=block-placement | FileCheck %s
 
 # Test that the micromips jal instruction is correctly handled by the delay slot
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-addiur1sp-addiusp.ll b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-addiur1sp-addiusp.ll
index 8949b19f4ada0..cedd76ebea009 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-addiur1sp-addiusp.ll
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-addiur1sp-addiusp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
 
 define i32 @f1() {
 entry:
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll
index 4cada64548f47..ca4d6856f87f1 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
 
 define void @f1(ptr %p) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwsp-swsp.ll b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwsp-swsp.ll
index 46a963e2373f3..5d594e27c2fcc 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwsp-swsp.ll
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwsp-swsp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -asm-show-inst -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips -asm-show-inst -verify-machineinstrs < %s | FileCheck %s
 
 ; Function Attrs: nounwind
 define i32 @function1(ptr %f) {
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-xor16.ll b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-xor16.ll
index e953df62bf2ad..3120b1a5b6d66 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-xor16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-xor16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s
 
 ; Function Attrs: nounwind readnone
 define i1 @fun(i32 %a, i32 %b) {
diff --git a/llvm/test/CodeGen/Mips/micromips-subu16.ll b/llvm/test/CodeGen/Mips/micromips-subu16.ll
index 874f6dfbcbfa3..972cca99ba1b8 100644
--- a/llvm/test/CodeGen/Mips/micromips-subu16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-subu16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 define i32 @main() {
diff --git a/llvm/test/CodeGen/Mips/micromips-sw-lw-16.ll b/llvm/test/CodeGen/Mips/micromips-sw-lw-16.ll
index c11c817b41651..890e6d002db51 100644
--- a/llvm/test/CodeGen/Mips/micromips-sw-lw-16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-sw-lw-16.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -march=mipsel -mattr=micromips -filetype=asm \
+; RUN: llc %s -mtriple=mipsel -mattr=micromips -filetype=asm \
 ; RUN: -relocation-model=pic -O3 -o - | FileCheck %s
 
 ; Function Attrs: noinline nounwind
diff --git a/llvm/test/CodeGen/Mips/micromips-xor16.ll b/llvm/test/CodeGen/Mips/micromips-xor16.ll
index a09531bc706cb..3bf6b8bfbb4c3 100644
--- a/llvm/test/CodeGen/Mips/micromips-xor16.ll
+++ b/llvm/test/CodeGen/Mips/micromips-xor16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+micromips \
 ; RUN:   -relocation-model=pic -O3 < %s | FileCheck %s
 
 define i32 @main() {
diff --git a/llvm/test/CodeGen/Mips/mips16_32_1.ll b/llvm/test/CodeGen/Mips/mips16_32_1.ll
index bf95c06a49328..0d020221c5efc 100644
--- a/llvm/test/CodeGen/Mips/mips16_32_1.ll
+++ b/llvm/test/CodeGen/Mips/mips16_32_1.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s -mips-mixed-16-32  | FileCheck %s 
-; RUN: llc  -march=mipsel -mcpu=mips32 -relocation-model=pic -O3 < %s -mips-mixed-16-32  | FileCheck %s 
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s -mips-mixed-16-32  | FileCheck %s 
+; RUN: llc  -mtriple=mipsel -mcpu=mips32 -relocation-model=pic -O3 < %s -mips-mixed-16-32  | FileCheck %s 
 
 define void @foo() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips16fpe.ll b/llvm/test/CodeGen/Mips/mips16fpe.ll
index 310213b43b002..7c6fdd532414c 100644
--- a/llvm/test/CodeGen/Mips/mips16fpe.ll
+++ b/llvm/test/CodeGen/Mips/mips16fpe.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic -O3 \
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s -check-prefix=16hf
 
 @x = global float 5.000000e+00, align 4
diff --git a/llvm/test/CodeGen/Mips/mips3-spill-slot.ll b/llvm/test/CodeGen/Mips/mips3-spill-slot.ll
index 4eabbeb867e38..d1cfce09e67ed 100644
--- a/llvm/test/CodeGen/Mips/mips3-spill-slot.ll
+++ b/llvm/test/CodeGen/Mips/mips3-spill-slot.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips3 < %s 2>&1 | FileCheck %s --check-prefix=CHECK
+; RUN: llc -mtriple=mips64 -mcpu=mips3 < %s 2>&1 | FileCheck %s --check-prefix=CHECK
 ; This testcase is from PR35859.
 ; Check that spill slot has the correct size for mips3 and n64 ABI.
 ; Previously, this test case would fail during register
diff --git a/llvm/test/CodeGen/Mips/mips32r6/compatibility.ll b/llvm/test/CodeGen/Mips/mips32r6/compatibility.ll
index 8eac8d4683d15..4e8b588345536 100644
--- a/llvm/test/CodeGen/Mips/mips32r6/compatibility.ll
+++ b/llvm/test/CodeGen/Mips/mips32r6/compatibility.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mipsel -mcpu=mips32r6 < %s | FileCheck %s
-; RUN: not llc -march=mipsel -mcpu=mips32r6 -mattr=+dsp < %s 2>&1 | FileCheck --check-prefix=DSP %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r6 < %s | FileCheck %s
+; RUN: not llc -mtriple=mipsel -mcpu=mips32r6 -mattr=+dsp < %s 2>&1 | FileCheck --check-prefix=DSP %s
 
 ; CHECK: foo:
 ; DSP: MIPS32r6 is not compatible with the DSP ASE
diff --git a/llvm/test/CodeGen/Mips/mips64-f128-call.ll b/llvm/test/CodeGen/Mips/mips64-f128-call.ll
index 879dc902eeb33..58ea4c424fe20 100644
--- a/llvm/test/CodeGen/Mips/mips64-f128-call.ll
+++ b/llvm/test/CodeGen/Mips/mips64-f128-call.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64el -mcpu=mips64r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 < %s | FileCheck %s
 
 @gld0 = external global fp128
 @gld1 = external global fp128
diff --git a/llvm/test/CodeGen/Mips/mips64-libcall.ll b/llvm/test/CodeGen/Mips/mips64-libcall.ll
index 7c0e5b2bbfdce..9de35a893f4c5 100644
--- a/llvm/test/CodeGen/Mips/mips64-libcall.ll
+++ b/llvm/test/CodeGen/Mips/mips64-libcall.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mips64el -mcpu=mips64r2 -O3 < %s |\
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -O3 < %s |\
 ; RUN: FileCheck %s -check-prefix=HARD
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+soft-float < %s |\
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+soft-float < %s |\
 ; RUN: FileCheck %s -check-prefix=SOFT
 
 ; Check that %add is not passed in an integer register.
diff --git a/llvm/test/CodeGen/Mips/mips64-sret.ll b/llvm/test/CodeGen/Mips/mips64-sret.ll
index 9f4bc9ddfcc54..4c05ec097b068 100644
--- a/llvm/test/CodeGen/Mips/mips64-sret.ll
+++ b/llvm/test/CodeGen/Mips/mips64-sret.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 < %s | FileCheck %s
 
 define void @foo(ptr noalias sret(i32) %agg.result) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64directive.ll b/llvm/test/CodeGen/Mips/mips64directive.ll
index 6d6674496f7d0..3d5a32f43982e 100644
--- a/llvm/test/CodeGen/Mips/mips64directive.ll
+++ b/llvm/test/CodeGen/Mips/mips64directive.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
 
 @gl = global i64 1250999896321, align 8
 
diff --git a/llvm/test/CodeGen/Mips/mips64ext.ll b/llvm/test/CodeGen/Mips/mips64ext.ll
index 9c1243b8f18c6..0e863001973cc 100644
--- a/llvm/test/CodeGen/Mips/mips64ext.ll
+++ b/llvm/test/CodeGen/Mips/mips64ext.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
 
 define i64 @zext64_32(i32 %a) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64extins.ll b/llvm/test/CodeGen/Mips/mips64extins.ll
index f160603c2507e..8621938092183 100644
--- a/llvm/test/CodeGen/Mips/mips64extins.ll
+++ b/llvm/test/CodeGen/Mips/mips64extins.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 | FileCheck %s
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 | FileCheck %s
 
 define i64 @dext(i64 %i) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64fpimm0.ll b/llvm/test/CodeGen/Mips/mips64fpimm0.ll
index 0296cb523f98f..a8d7f432eba46 100644
--- a/llvm/test/CodeGen/Mips/mips64fpimm0.ll
+++ b/llvm/test/CodeGen/Mips/mips64fpimm0.ll
@@ -1,5 +1,5 @@
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi=n64 | FileCheck %s
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi=n64 | FileCheck %s
 
 define double @foo1() nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64fpldst.ll b/llvm/test/CodeGen/Mips/mips64fpldst.ll
index 7811ed85f3adb..ad09f78b3184c 100644
--- a/llvm/test/CodeGen/Mips/mips64fpldst.ll
+++ b/llvm/test/CodeGen/Mips/mips64fpldst.ll
@@ -1,7 +1,7 @@
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
 
 @f0 = common global float 0.000000e+00, align 4
 @d0 = common global double 0.000000e+00, align 8
diff --git a/llvm/test/CodeGen/Mips/mips64imm.ll b/llvm/test/CodeGen/Mips/mips64imm.ll
index 993c7c77e35c1..a78e1dce518ae 100644
--- a/llvm/test/CodeGen/Mips/mips64imm.ll
+++ b/llvm/test/CodeGen/Mips/mips64imm.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s
 
 define i32 @foo1() nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64instrs.ll b/llvm/test/CodeGen/Mips/mips64instrs.ll
index e8b630b004cb0..2dc88dfffa255 100644
--- a/llvm/test/CodeGen/Mips/mips64instrs.ll
+++ b/llvm/test/CodeGen/Mips/mips64instrs.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips64el -mcpu=mips4 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,MIPS4,ACCMULDIV %s
-; RUN: llc -march=mips64el -mcpu=mips64 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,ACCMULDIV %s
-; RUN: llc -march=mips64el -mcpu=mips64r2 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,ACCMULDIV %s
-; RUN: llc -march=mips64el -mcpu=mips64r6 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,GPRMULDIV %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,MIPS4,ACCMULDIV %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,ACCMULDIV %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,ACCMULDIV %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,HAS-DCLO,GPRMULDIV %s
 
 @gll0 = common global i64 0, align 8
 @gll1 = common global i64 0, align 8
diff --git a/llvm/test/CodeGen/Mips/mips64intldst.ll b/llvm/test/CodeGen/Mips/mips64intldst.ll
index 89e13247da014..7df92dd8028ef 100644
--- a/llvm/test/CodeGen/Mips/mips64intldst.ll
+++ b/llvm/test/CodeGen/Mips/mips64intldst.ll
@@ -1,7 +1,7 @@
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
-; RUN: llc  < %s -march=mips64el -mcpu=mips4 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
-; RUN: llc  < %s -march=mips64el -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips4 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N64
+; RUN: llc  < %s -mtriple=mips64el -mcpu=mips64 -target-abi n32 -relocation-model=pic | FileCheck %s -check-prefix=CHECK-N32
 
 @c = common global i8 0, align 4
 @s = common global i16 0, align 4
diff --git a/llvm/test/CodeGen/Mips/mips64lea.ll b/llvm/test/CodeGen/Mips/mips64lea.ll
index 69c0796035166..83f4239930533 100644
--- a/llvm/test/CodeGen/Mips/mips64lea.ll
+++ b/llvm/test/CodeGen/Mips/mips64lea.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s
 
 define void @foo3() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64muldiv.ll b/llvm/test/CodeGen/Mips/mips64muldiv.ll
index d1292be504e64..89dd524269491 100644
--- a/llvm/test/CodeGen/Mips/mips64muldiv.ll
+++ b/llvm/test/CodeGen/Mips/mips64muldiv.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefixes=ALL,ACC
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefixes=ALL,ACC
-; RUN: llc -march=mips64el -mcpu=mips64r2 < %s | FileCheck %s -check-prefixes=ALL,ACC
-; RUN: llc -march=mips64el -mcpu=mips64r6 < %s | FileCheck %s -check-prefixes=ALL,GPR
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefixes=ALL,ACC
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefixes=ALL,ACC
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 < %s | FileCheck %s -check-prefixes=ALL,ACC
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 < %s | FileCheck %s -check-prefixes=ALL,GPR
 
 ; FileCheck prefixes:
 ;   ALL - All targets
diff --git a/llvm/test/CodeGen/Mips/mips64r6/compatibility.ll b/llvm/test/CodeGen/Mips/mips64r6/compatibility.ll
index 174f4ce1771ad..6484173e7c339 100644
--- a/llvm/test/CodeGen/Mips/mips64r6/compatibility.ll
+++ b/llvm/test/CodeGen/Mips/mips64r6/compatibility.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mipsel -mcpu=mips64r6 -target-abi n64 < %s | FileCheck %s
-; RUN: not llc -march=mipsel -mcpu=mips64r6 -target-abi n64 -mattr=+dsp < %s 2>&1 | FileCheck --check-prefix=DSP %s
+; RUN: llc -mtriple=mipsel -mcpu=mips64r6 -target-abi n64 < %s | FileCheck %s
+; RUN: not llc -mtriple=mipsel -mcpu=mips64r6 -target-abi n64 -mattr=+dsp < %s 2>&1 | FileCheck --check-prefix=DSP %s
 
 ; CHECK: foo:
 ; DSP: MIPS64r6 is not compatible with the DSP ASE
diff --git a/llvm/test/CodeGen/Mips/mips64shift.ll b/llvm/test/CodeGen/Mips/mips64shift.ll
index ca09d2f5b2904..d5b85ea65ce38 100644
--- a/llvm/test/CodeGen/Mips/mips64shift.ll
+++ b/llvm/test/CodeGen/Mips/mips64shift.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64el -mcpu=mips64r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 < %s | FileCheck %s
 
 define i64 @f0(i64 %a0, i64 %a1) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64signextendsesf.ll b/llvm/test/CodeGen/Mips/mips64signextendsesf.ll
index 831469ab06247..6160cd1dcffed 100644
--- a/llvm/test/CodeGen/Mips/mips64signextendsesf.ll
+++ b/llvm/test/CodeGen/Mips/mips64signextendsesf.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+soft-float -O2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+soft-float -O2 < %s | FileCheck %s
 
 define void @foosf() #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/mips64sinttofpsf.ll b/llvm/test/CodeGen/Mips/mips64sinttofpsf.ll
index 12c353d72c56c..10cc70e34065b 100644
--- a/llvm/test/CodeGen/Mips/mips64sinttofpsf.ll
+++ b/llvm/test/CodeGen/Mips/mips64sinttofpsf.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+soft-float -O0 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+soft-float -O0 < %s | FileCheck %s
 
 
 define double @foo() #0 {
diff --git a/llvm/test/CodeGen/Mips/mipslopat.ll b/llvm/test/CodeGen/Mips/mipslopat.ll
index c6143844a8d4d..109bf5ebb2651 100644
--- a/llvm/test/CodeGen/Mips/mipslopat.ll
+++ b/llvm/test/CodeGen/Mips/mipslopat.ll
@@ -1,5 +1,5 @@
 ; This test does not check the machine code output.   
-; RUN: llc -march=mips < %s 
+; RUN: llc -mtriple=mips < %s 
 
 @stat_vol_ptr_int = internal global ptr null, align 4
 @stat_ptr_vol_int = internal global ptr null, align 4
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
index 6002705965cae..453a0b1b5e27b 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips64 -target-abi n64 -start-before=finalize-isel \
+# RUN: llc -mtriple=mips64 -target-abi n64 -start-before=finalize-isel \
 # RUN:     -stop-after=finalize-isel -relocation-model=pic -mattr=+xgot \
 # RUN:     -o /dev/null %s
 
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
index d9703c74fc5ce..8a20b5354f7c4 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips -start-before=finalize-isel \
+# RUN: llc -mtriple=mips -start-before=finalize-isel \
 # RUN:     -stop-after=finalize-isel -relocation-model=pic \
 # RUN:     -o /dev/null %s
 
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
index 803a2d701fae4..5151cb1c89029 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips64 -target-abi n64 -start-before=finalize-isel \
+# RUN: llc -mtriple=mips64 -target-abi n64 -start-before=finalize-isel \
 # RUN:     -stop-after=finalize-isel -relocation-model=pic \
 # RUN:     -o /dev/null %s
 
diff --git a/llvm/test/CodeGen/Mips/misha.ll b/llvm/test/CodeGen/Mips/misha.ll
index b8e45c633aaa4..40a738fc8a5a3 100644
--- a/llvm/test/CodeGen/Mips/misha.ll
+++ b/llvm/test/CodeGen/Mips/misha.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 define i32 @sumc(ptr nocapture %to, ptr nocapture %from, i32) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/msa/2r.ll b/llvm/test/CodeGen/Mips/msa/2r.ll
index b7ea3fc11c6e3..32a8734a4a170 100644
--- a/llvm/test/CodeGen/Mips/msa/2r.ll
+++ b/llvm/test/CodeGen/Mips/msa/2r.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the 2R instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_nloc_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_nloc_b_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll b/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
index f369a9ebca384..615bbbad66ae0 100644
--- a/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
+++ b/llvm/test/CodeGen/Mips/msa/2r_vector_scalar.ll
@@ -1,13 +1,13 @@
 ; Test the MSA intrinsics that are encoded with the 2R instruction format and
 ; convert scalars to vectors.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
 
 @llvm_mips_fill_b_ARG1 = global i32 23, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf.ll b/llvm/test/CodeGen/Mips/msa/2rf.ll
index 6cdf5b7b8b423..4a272e3fa17d1 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the 2RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_flog2_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_flog2_w_RES  = global <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf_exup.ll b/llvm/test/CodeGen/Mips/msa/2rf_exup.ll
index f8bdf866c5f82..cf2e604c2623f 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf_exup.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf_exup.ll
@@ -1,8 +1,8 @@
 ; Test the MSA floating point conversion intrinsics (e.g. float->double) that
 ; are encoded with the 2RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fexupl_w_ARG1 = global <8 x half> <half 0.000000e+00, half 1.000000e+00, half 2.000000e+00, half 3.000000e+00, half 4.000000e+00, half 5.000000e+00, half 6.000000e+00, half 7.000000e+00>, align 16
 @llvm_mips_fexupl_w_RES  = global <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf_float_int.ll b/llvm/test/CodeGen/Mips/msa/2rf_float_int.ll
index da83b7eb180b7..b8593a7fb9368 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf_float_int.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf_float_int.ll
@@ -1,8 +1,8 @@
 ; Test the MSA integer to floating point conversion intrinsics that are encoded
 ; with the 2RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_ffint_s_w_ARG1 = global <4 x i32> <i32 0, i32 1, i32 2, i32 3>, align 16
 @llvm_mips_ffint_s_w_RES  = global <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf_fq.ll b/llvm/test/CodeGen/Mips/msa/2rf_fq.ll
index 2d773bfda17c8..9a5c9c14b47e8 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf_fq.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf_fq.ll
@@ -1,8 +1,8 @@
 ; Test the MSA fixed-point to floating point conversion intrinsics that are
 ; encoded with the 2RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ffql_w_ARG1 = global <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, align 16
 @llvm_mips_ffql_w_RES  = global <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf_int_float.ll b/llvm/test/CodeGen/Mips/msa/2rf_int_float.ll
index eeac8d4495716..ec700da6f3c06 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf_int_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf_int_float.ll
@@ -2,8 +2,8 @@
 ; 2RF instruction format. This includes conversions but other instructions such
 ; as fclass are also here.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_fclass_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fclass_w_RES  = global <4 x i32> <i32 0, i32 0, i32 0, i32 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/2rf_tq.ll b/llvm/test/CodeGen/Mips/msa/2rf_tq.ll
index 110da06777813..e3fdbecc6b7eb 100644
--- a/llvm/test/CodeGen/Mips/msa/2rf_tq.ll
+++ b/llvm/test/CodeGen/Mips/msa/2rf_tq.ll
@@ -1,8 +1,8 @@
 ; Test the MSA floating-point to fixed-point conversion intrinsics that are
 ; encoded with the 2RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ftq_h_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_ftq_h_ARG2 = global <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-a.ll b/llvm/test/CodeGen/Mips/msa/3r-a.ll
index 31646350b6802..69e862850a473 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-a.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-a.ll
@@ -1,11 +1,11 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'a'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 ; It should fail to compile without fp64.
-; RUN: not llc -march=mips -mattr=+msa < %s 2>&1 | \
+; RUN: not llc -mtriple=mips -mattr=+msa < %s 2>&1 | \
 ; RUN:    FileCheck -check-prefix=FP32ERROR %s
 ; FP32ERROR: LLVM ERROR: MSA requires a 64-bit FPU register file (FR=1 mode).
 
diff --git a/llvm/test/CodeGen/Mips/msa/3r-b.ll b/llvm/test/CodeGen/Mips/msa/3r-b.ll
index f824a6527d72d..f2c5ebb4a580a 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-b.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-b.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'b'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_bclr_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_bclr_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-c.ll b/llvm/test/CodeGen/Mips/msa/3r-c.ll
index 8af06b3f20bd4..38cb386b5a99b 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-c.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-c.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'c'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ceq_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_ceq_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-d.ll b/llvm/test/CodeGen/Mips/msa/3r-d.ll
index b40d2661ee6aa..116e0ada6de7b 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-d.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-d.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'd'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_div_s_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_div_s_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-i.ll b/llvm/test/CodeGen/Mips/msa/3r-i.ll
index c06d79a975bf1..e20064bff84e9 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-i.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-i.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'i'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ilvev_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_ilvev_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-m.ll b/llvm/test/CodeGen/Mips/msa/3r-m.ll
index 855ceb3dd8890..580ae4b41c621 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-m.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-m.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'm'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_max_a_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_max_a_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-p.ll b/llvm/test/CodeGen/Mips/msa/3r-p.ll
index 063da01c29db6..455ebb4a31616 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-p.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-p.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'p'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_pckev_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_pckev_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-s.ll b/llvm/test/CodeGen/Mips/msa/3r-s.ll
index 6c673c543bf2f..6ea52fa856716 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-s.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-s.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 's'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_sld_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_sld_b_ARG2 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r-v.ll b/llvm/test/CodeGen/Mips/msa/3r-v.ll
index 80828a07907cd..c71efdc0bb745 100644
--- a/llvm/test/CodeGen/Mips/msa/3r-v.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r-v.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format.
 ; There are lots of these so this covers those beginning with 'v'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_vshf_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_vshf_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r_4r.ll b/llvm/test/CodeGen/Mips/msa/3r_4r.ll
index abeaee682fb48..d32af452c4fee 100644
--- a/llvm/test/CodeGen/Mips/msa/3r_4r.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r_4r.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3R instruction format and
 ; use the result as a third operand.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_maddv_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_maddv_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r_4r_widen.ll b/llvm/test/CodeGen/Mips/msa/3r_4r_widen.ll
index 4b286a0e93cbd..13546b6bf6fc3 100644
--- a/llvm/test/CodeGen/Mips/msa/3r_4r_widen.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r_4r_widen.ll
@@ -2,8 +2,8 @@
 ; use the result as a third operand and results in wider elements than the
 ; operands had.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_dpadd_s_h_ARG2 = global <16 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23>, align 16
 @llvm_mips_dpadd_s_h_ARG3 = global <16 x i8> <i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3r_splat.ll b/llvm/test/CodeGen/Mips/msa/3r_splat.ll
index e8d9d23fa9b49..c8c7f0585412c 100644
--- a/llvm/test/CodeGen/Mips/msa/3r_splat.ll
+++ b/llvm/test/CodeGen/Mips/msa/3r_splat.ll
@@ -1,9 +1,9 @@
 ; Test the MSA splat intrinsics that are encoded with the 3R instruction
 ; format.
 
-; RUN: llc -march=mips -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefix=MIPS32 %s
-; RUN: llc -march=mipsel -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:     FileCheck -check-prefix=MIPS32 %s
 
 @llvm_mips_splat_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf.ll b/llvm/test/CodeGen/Mips/msa/3rf.ll
index 9bae9ba530453..36114fa9094b5 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the 3RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fadd_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fadd_w_ARG2 = global <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_4rf.ll b/llvm/test/CodeGen/Mips/msa/3rf_4rf.ll
index 6142ada9fef91..94c66715035a3 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_4rf.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_4rf.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3RF instruction format and
 ; use the result as a third operand.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fmadd_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fmadd_w_ARG2 = global <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_4rf_q.ll b/llvm/test/CodeGen/Mips/msa/3rf_4rf_q.ll
index f397644df3919..049042df39c11 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_4rf_q.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_4rf_q.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3RF instruction format and
 ; use the result as a third operand and perform fixed-point operations.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_madd_q_h_ARG1 = global <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, align 16
 @llvm_mips_madd_q_h_ARG2 = global <8 x i16> <i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_exdo.ll b/llvm/test/CodeGen/Mips/msa/3rf_exdo.ll
index 70da349d0f13e..b0ee043b8f371 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_exdo.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_exdo.ll
@@ -1,8 +1,8 @@
 ; Test the MSA floating-point conversion intrinsics that are encoded with the
 ; 3RF instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fexdo_h_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fexdo_h_ARG2 = global <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_float_int.ll b/llvm/test/CodeGen/Mips/msa/3rf_float_int.ll
index 4c1328b49be56..49e31465d8477 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_float_int.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_float_int.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3RF instruction format and
 ; take an integer as an operand.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fexp2_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fexp2_w_ARG2 = global <4 x i32> <i32 4, i32 5, i32 6, i32 7>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_int_float.ll b/llvm/test/CodeGen/Mips/msa/3rf_int_float.ll
index 7e186beb32050..7c1b2ffd100c3 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_int_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_int_float.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the 3RF instruction format and
 ; produce an integer as a result.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_fcaf_w_ARG1 = global <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, align 16
 @llvm_mips_fcaf_w_ARG2 = global <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/3rf_q.ll b/llvm/test/CodeGen/Mips/msa/3rf_q.ll
index 5e3358ccd063a..fcae0713056e3 100644
--- a/llvm/test/CodeGen/Mips/msa/3rf_q.ll
+++ b/llvm/test/CodeGen/Mips/msa/3rf_q.ll
@@ -1,8 +1,8 @@
 ; Test the MSA fixed-point intrinsics that are encoded with the 3RF instruction
 ; format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_mul_q_h_ARG1 = global <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, align 16
 @llvm_mips_mul_q_h_ARG2 = global <8 x i16> <i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/arithmetic.ll b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
index 62fd35a69abaa..a262ce183d74e 100644
--- a/llvm/test/CodeGen/Mips/msa/arithmetic.ll
+++ b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=ALL,MIPS
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=ALL,MIPSEL
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=ALL,MIPS
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=ALL,MIPSEL
 
 define void @add_v16i8(ptr %c, ptr %a, ptr %b) nounwind {
 ; ALL-LABEL: add_v16i8:
diff --git a/llvm/test/CodeGen/Mips/msa/arithmetic_float.ll b/llvm/test/CodeGen/Mips/msa/arithmetic_float.ll
index b2ce43171aeb1..84496e012908a 100644
--- a/llvm/test/CodeGen/Mips/msa/arithmetic_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/arithmetic_float.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 define void @add_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
   ; CHECK: add_v4f32:
diff --git a/llvm/test/CodeGen/Mips/msa/basic_operations.ll b/llvm/test/CodeGen/Mips/msa/basic_operations.ll
index 1f6c430a932e0..820259d7c7bc2 100644
--- a/llvm/test/CodeGen/Mips/msa/basic_operations.ll
+++ b/llvm/test/CodeGen/Mips/msa/basic_operations.ll
@@ -1,20 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic \
+; RUN: llc -mtriple=mips -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic \
 ; RUN:   -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=O32,O32-BE %s
-; RUN: llc -march=mipsel -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r5 -mattr=+msa,+fp64 -relocation-model=pic \
 ; RUN:   -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=O32,O32-LE %s
-; RUN: llc -march=mips64 -mcpu=mips64r5 -target-abi n32 -mattr=+msa,+fp64 \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -target-abi n32 -mattr=+msa,+fp64 \
 ; RUN:   -relocation-model=pic -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=N32,N32-BE %s
-; RUN: llc -march=mips64el -mcpu=mips64r5 -target-abi n32 -mattr=+msa,+fp64 \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r5 -target-abi n32 -mattr=+msa,+fp64 \
 ; RUN:   -relocation-model=pic -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=N32,N32-LE %s
-; RUN: llc -march=mips64 -mcpu=mips64r5 -mattr=+msa,+fp64 -relocation-model=pic \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -mattr=+msa,+fp64 -relocation-model=pic \
 ; RUN:   -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=N64,N64-BE %s
-; RUN: llc -march=mips64el -mcpu=mips64r5 -mattr=+msa,+fp64 -relocation-model=pic \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r5 -mattr=+msa,+fp64 -relocation-model=pic \
 ; RUN:   -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=N64,N64-LE %s
 
diff --git a/llvm/test/CodeGen/Mips/msa/basic_operations_float.ll b/llvm/test/CodeGen/Mips/msa/basic_operations_float.ll
index 83ac551c8dc94..a1b5215d3ea20 100644
--- a/llvm/test/CodeGen/Mips/msa/basic_operations_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/basic_operations_float.ll
@@ -1,14 +1,14 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,O32 %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,O32 %s
-; RUN: llc -march=mips64 -target-abi=n32 -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64 -target-abi=n32 -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64el -target-abi=n32 -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64el -target-abi=n32 -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,N32 %s
-; RUN: llc -march=mips64 -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64 -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,N64 %s
-; RUN: llc -march=mips64el -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64el -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,N64 %s
 
 @v4f32 = global <4 x float> <float 0.0, float 0.0, float 0.0, float 0.0>
diff --git a/llvm/test/CodeGen/Mips/msa/bit.ll b/llvm/test/CodeGen/Mips/msa/bit.ll
index 1b2012cec5f5a..cc200eb915866 100644
--- a/llvm/test/CodeGen/Mips/msa/bit.ll
+++ b/llvm/test/CodeGen/Mips/msa/bit.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the BIT instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_sat_s_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_sat_s_b_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/bitcast.ll b/llvm/test/CodeGen/Mips/msa/bitcast.ll
index 11c5a5fb42c79..12b118b0af53c 100644
--- a/llvm/test/CodeGen/Mips/msa/bitcast.ll
+++ b/llvm/test/CodeGen/Mips/msa/bitcast.ll
@@ -1,7 +1,7 @@
 ; Test the bitcast operation for big-endian and little-endian.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=BIGENDIAN %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=LITENDIAN %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=BIGENDIAN %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=LITENDIAN %s
 
 define void @v16i8_to_v16i8(ptr %src, ptr %dst) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/msa/bitwise.ll b/llvm/test/CodeGen/Mips/msa/bitwise.ll
index c7790b71b9636..def77c8c47266 100644
--- a/llvm/test/CodeGen/Mips/msa/bitwise.ll
+++ b/llvm/test/CodeGen/Mips/msa/bitwise.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPS
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPSEL
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPS
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPSEL
 
 define void @and_v16i8(ptr %c, ptr %a, ptr %b) nounwind {
 ; CHECK-LABEL: and_v16i8:
diff --git a/llvm/test/CodeGen/Mips/msa/bmzi_bmnzi.ll b/llvm/test/CodeGen/Mips/msa/bmzi_bmnzi.ll
index ab2c36eae67da..5bc8b27bf3366 100644
--- a/llvm/test/CodeGen/Mips/msa/bmzi_bmnzi.ll
+++ b/llvm/test/CodeGen/Mips/msa/bmzi_bmnzi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_bmnzi_b_ARG1 = global <16 x i8> <i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>, align 16
 @llvm_mips_bmnzi_b_ARG2 = global <16 x i8> zeroinitializer, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/compare.ll b/llvm/test/CodeGen/Mips/msa/compare.ll
index a3910bde8cd1c..5027c45ecf52e 100644
--- a/llvm/test/CodeGen/Mips/msa/compare.ll
+++ b/llvm/test/CodeGen/Mips/msa/compare.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 define void @ceq_v16i8(ptr %c, ptr %a, ptr %b) nounwind {
   ; CHECK: ceq_v16i8:
diff --git a/llvm/test/CodeGen/Mips/msa/compare_float.ll b/llvm/test/CodeGen/Mips/msa/compare_float.ll
index cd4924eca44cd..396d8c421d123 100644
--- a/llvm/test/CodeGen/Mips/msa/compare_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/compare_float.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 declare <4 x float> @llvm.mips.fmax.w(<4 x float>, <4 x float>) nounwind
 declare <2 x double> @llvm.mips.fmax.d(<2 x double>, <2 x double>) nounwind
diff --git a/llvm/test/CodeGen/Mips/msa/elm_copy.ll b/llvm/test/CodeGen/Mips/msa/elm_copy.ll
index 6e0ee2da0920f..232f09fc3a668 100644
--- a/llvm/test/CodeGen/Mips/msa/elm_copy.ll
+++ b/llvm/test/CodeGen/Mips/msa/elm_copy.ll
@@ -1,13 +1,13 @@
 ; Test the MSA intrinsics that are encoded with the ELM instruction format and
 ; are element extraction operations.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
 
 @llvm_mips_copy_s_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/elm_cxcmsa.ll b/llvm/test/CodeGen/Mips/msa/elm_cxcmsa.ll
index 7d44620e25790..dab5e0c1b93a3 100644
--- a/llvm/test/CodeGen/Mips/msa/elm_cxcmsa.ll
+++ b/llvm/test/CodeGen/Mips/msa/elm_cxcmsa.ll
@@ -1,8 +1,8 @@
 ; Test the MSA ctcmsa and cfcmsa intrinsics (which are encoded with the ELM
 ; instruction format).
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -verify-machineinstrs < %s | FileCheck %s
 
 define i32 @msa_ir_cfcmsa_test() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/msa/elm_insv.ll b/llvm/test/CodeGen/Mips/msa/elm_insv.ll
index 6c00483cf6537..b5b22fd8ee0c9 100644
--- a/llvm/test/CodeGen/Mips/msa/elm_insv.ll
+++ b/llvm/test/CodeGen/Mips/msa/elm_insv.ll
@@ -1,13 +1,13 @@
 ; Test the MSA element insertion intrinsics that are encoded with the ELM
 ; instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
+; RUN: llc -mtriple=mips64el -mcpu=mips64r2 -mattr=+msa,+fp64 -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=MIPS-ANY,MIPS64
 
 @llvm_mips_insert_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/elm_move.ll b/llvm/test/CodeGen/Mips/msa/elm_move.ll
index 4065fc753a55e..9e13d52f28bc2 100644
--- a/llvm/test/CodeGen/Mips/msa/elm_move.ll
+++ b/llvm/test/CodeGen/Mips/msa/elm_move.ll
@@ -1,8 +1,8 @@
 ; Test the MSA move intrinsics (which are encoded with the ELM instruction
 ; format).
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_move_vb_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_move_vb_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/elm_shift_slide.ll b/llvm/test/CodeGen/Mips/msa/elm_shift_slide.ll
index 548cdf394ac85..330da8b04469e 100644
--- a/llvm/test/CodeGen/Mips/msa/elm_shift_slide.ll
+++ b/llvm/test/CodeGen/Mips/msa/elm_shift_slide.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the ELM instruction format and
 ; are either shifts or slides.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_sldi_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_sldi_b_ARG2 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/emergency-spill.mir b/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
index 2089464528661..dcc64f58c5646 100644
--- a/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
+++ b/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
@@ -1,4 +1,4 @@
-# RUN: llc %s -start-after=shrink-wrap -march=mips64 -mcpu=mips64r6 -mattr=+fp64,+msa -o /dev/null
+# RUN: llc %s -start-after=shrink-wrap -mtriple=mips64 -mcpu=mips64r6 -mattr=+fp64,+msa -o /dev/null
 
 # Test that estimated size of the stack leads to the creation of an emergency
 # spill when MSA is in use. Previously, this test case would fail during
diff --git a/llvm/test/CodeGen/Mips/msa/endian.ll b/llvm/test/CodeGen/Mips/msa/endian.ll
index 63aa3f6e18724..e7ed31d145270 100644
--- a/llvm/test/CodeGen/Mips/msa/endian.ll
+++ b/llvm/test/CodeGen/Mips/msa/endian.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=BIGENDIAN %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=LITENDIAN %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=BIGENDIAN %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck -check-prefix=LITENDIAN %s
 
 @v16i8 = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
 @v8i16 = global <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
diff --git a/llvm/test/CodeGen/Mips/msa/fexuprl.ll b/llvm/test/CodeGen/Mips/msa/fexuprl.ll
index abd9d164d7c6b..e25ae39e7e4eb 100644
--- a/llvm/test/CodeGen/Mips/msa/fexuprl.ll
+++ b/llvm/test/CodeGen/Mips/msa/fexuprl.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mips -mcpu=mips32r5 -mattr=+fp64,+msa | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -mattr=+fp64,+msa | FileCheck %s
 
 ; Test that fexup[rl].w don't crash LLVM during type legalization.
 
diff --git a/llvm/test/CodeGen/Mips/msa/frameindex.ll b/llvm/test/CodeGen/Mips/msa/frameindex.ll
index 1ee527bd88a4d..4d7fc78595f57 100644
--- a/llvm/test/CodeGen/Mips/msa/frameindex.ll
+++ b/llvm/test/CodeGen/Mips/msa/frameindex.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
 
 define void @loadstore_v16i8_near() nounwind {
   ; CHECK: loadstore_v16i8_near:
diff --git a/llvm/test/CodeGen/Mips/msa/i10.ll b/llvm/test/CodeGen/Mips/msa/i10.ll
index e130d6df4b90c..a9f95df0d0e83 100644
--- a/llvm/test/CodeGen/Mips/msa/i10.ll
+++ b/llvm/test/CodeGen/Mips/msa/i10.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the I10 instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_bnz_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 
diff --git a/llvm/test/CodeGen/Mips/msa/i5-a.ll b/llvm/test/CodeGen/Mips/msa/i5-a.ll
index 7fd14da4b5f06..b46d0afcb8304 100644
--- a/llvm/test/CodeGen/Mips/msa/i5-a.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5-a.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the I5 instruction format.
 ; There are lots of these so this covers those beginning with 'a'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_addvi_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_addvi_b_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/i5-b.ll b/llvm/test/CodeGen/Mips/msa/i5-b.ll
index 4a36bfe540d14..1aff85d3327bc 100644
--- a/llvm/test/CodeGen/Mips/msa/i5-b.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5-b.ll
@@ -2,8 +2,8 @@
 ; Test the MSA intrinsics that are encoded with the I5 instruction format.
 ; There are lots of these so this covers those beginning with 'b'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_bclri_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_bclri_b_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/i5-c.ll b/llvm/test/CodeGen/Mips/msa/i5-c.ll
index 96f5e6286276a..fb4a21bf4ed3c 100644
--- a/llvm/test/CodeGen/Mips/msa/i5-c.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5-c.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the I5 instruction format.
 ; There are lots of these so this covers those beginning with 'c'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ceqi_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_ceqi_b_RES1 = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/i5-m.ll b/llvm/test/CodeGen/Mips/msa/i5-m.ll
index 74599185963a6..60ed8b7c86165 100644
--- a/llvm/test/CodeGen/Mips/msa/i5-m.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5-m.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the I5 instruction format.
 ; There are lots of these so this covers those beginning with 'm'
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_maxi_s_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_maxi_s_b_RES1 = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/i5-s.ll b/llvm/test/CodeGen/Mips/msa/i5-s.ll
index 0ba71fa9acd3f..754c4c57d7556 100644
--- a/llvm/test/CodeGen/Mips/msa/i5-s.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5-s.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r5 < %s | FileCheck %s
 
 ; Test the MSA intrinsics that are encoded with the I5 instruction format.
 ; There are lots of these so this covers those beginning with 's'
diff --git a/llvm/test/CodeGen/Mips/msa/i5_ld_st.ll b/llvm/test/CodeGen/Mips/msa/i5_ld_st.ll
index e55799cf17667..66cb32eb88246 100644
--- a/llvm/test/CodeGen/Mips/msa/i5_ld_st.ll
+++ b/llvm/test/CodeGen/Mips/msa/i5_ld_st.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the I5 instruction format and
 ; are loads or stores.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_ld_b_ARG = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_ld_b_RES = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/i8.ll b/llvm/test/CodeGen/Mips/msa/i8.ll
index 89f5725c17357..e4c1affc701fb 100644
--- a/llvm/test/CodeGen/Mips/msa/i8.ll
+++ b/llvm/test/CodeGen/Mips/msa/i8.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the I8 instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_andi_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_andi_b_RES  = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/immediates-bad.ll b/llvm/test/CodeGen/Mips/msa/immediates-bad.ll
index bbb76f3b8b6bf..f8bb5b36c44c8 100644
--- a/llvm/test/CodeGen/Mips/msa/immediates-bad.ll
+++ b/llvm/test/CodeGen/Mips/msa/immediates-bad.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s 2> %t1
+; RUN: not --crash llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s 2> %t1
 ; RUN: FileCheck %s < %t1
 
 ; Test that the immediate intrinsics with out of range values trigger an error.
diff --git a/llvm/test/CodeGen/Mips/msa/immediates.ll b/llvm/test/CodeGen/Mips/msa/immediates.ll
index 5808643aded1c..ba1ae74f227b5 100644
--- a/llvm/test/CodeGen/Mips/msa/immediates.ll
+++ b/llvm/test/CodeGen/Mips/msa/immediates.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:      | FileCheck %s -check-prefixes=MSA,MSA32
-; RUN: llc -march=mips64 -mattr=+msa,+fp64 -relocation-model=pic -target-abi n32 < %s \
+; RUN: llc -mtriple=mips64 -mattr=+msa,+fp64 -relocation-model=pic -target-abi n32 < %s \
 ; RUN:      | FileCheck %s -check-prefix=MSA64N32
-; RUN: llc -march=mips64 -mattr=+msa,+fp64 -relocation-model=pic -target-abi n64 < %s \
+; RUN: llc -mtriple=mips64 -mattr=+msa,+fp64 -relocation-model=pic -target-abi n64 < %s \
 ; RUN:      | FileCheck %s -check-prefixes=MSA,MSA64N64
 
 ; Test that the immediate intrinsics don't crash LLVM.
diff --git a/llvm/test/CodeGen/Mips/msa/inline-asm.ll b/llvm/test/CodeGen/Mips/msa/inline-asm.ll
index f84b11e05387e..a41f96047ba7a 100644
--- a/llvm/test/CodeGen/Mips/msa/inline-asm.ll
+++ b/llvm/test/CodeGen/Mips/msa/inline-asm.ll
@@ -1,6 +1,6 @@
 ; A basic inline assembly test
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @v4i32_r  = global <4 x i32> zeroinitializer, align 16
 
diff --git a/llvm/test/CodeGen/Mips/msa/ldr_str.ll b/llvm/test/CodeGen/Mips/msa/ldr_str.ll
index 74f84aa9bbd7a..ff766491834e5 100644
--- a/llvm/test/CodeGen/Mips/msa/ldr_str.ll
+++ b/llvm/test/CodeGen/Mips/msa/ldr_str.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips     -mcpu=mips32r5 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R5-EB
-; RUN: llc -march=mipsel   -mcpu=mips32r5 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R5-EL
-; RUN: llc -march=mips     -mcpu=mips32r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R6-EB
-; RUN: llc -march=mipsel   -mcpu=mips32r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R6-EL
-; RUN: llc -march=mips64   -mcpu=mips64r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS64R6
-; RUN: llc -march=mips64el -mcpu=mips64r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS64R6
+; RUN: llc -mtriple=mips     -mcpu=mips32r5 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R5-EB
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r5 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R5-EL
+; RUN: llc -mtriple=mips     -mcpu=mips32r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R6-EB
+; RUN: llc -mtriple=mipsel   -mcpu=mips32r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS32R6-EL
+; RUN: llc -mtriple=mips64   -mcpu=mips64r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS64R6
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -mattr=+msa,+fp64 -O0 < %s | FileCheck %s --check-prefix=MIPS64R6
 
 ; Test intrinsics for 4-byte and 8-byte MSA load and stores.
 
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s1704963983.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s1704963983.ll
index db2b52028a892..d658dbdce639f 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s1704963983.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s1704963983.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a
 ; "Unexpected illegal type!" assertion.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s1935737938.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s1935737938.ll
index a220b9887f901..77fa9f4715211 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s1935737938.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s1935737938.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a
 ; `Opc && "Cannot copy registers"' assertion.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2090927243-simplified.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2090927243-simplified.ll
index e27ca939d42da..d4b80c63aea85 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2090927243-simplified.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2090927243-simplified.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a "Cannot select ..." error.
 ; This was because undef's are ignored when checking if a vector constant is a
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2501752154-simplified.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2501752154-simplified.ll
index 91f2ca3d8d504..cd20c98e2fe42 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2501752154-simplified.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2501752154-simplified.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a "Cannot select ..." error.
 ; This happened because the legalizer treated undef's in the <4 x float>
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2704903805.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2704903805.ll
index d1d2f0db69dcf..596dc2594c9b1 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s2704903805.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s2704903805.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA after dereferencing a null this pointer.
 ; It should at least successfully build.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3861334421.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3861334421.ll
index c3158801bee57..137e7cf4efa6a 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3861334421.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3861334421.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a
 ; "Don't know how to expand this condition!" unreachable.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3926023935.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3926023935.ll
index ace3f54b33a37..c8123e9a8c8e2 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3926023935.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3926023935.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a
 ; "Type for zero vector elements is not legal" assertion.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3997499501.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3997499501.ll
index d88dfb6bd29ae..cea8ac8f9fffa 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s3997499501.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s3997499501.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed to select instructions for extract_vector_elt for
 ; v4f32 on MSA.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s449609655-simplified.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s449609655-simplified.ll
index de3f5a27eb463..81d6830a19f16 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s449609655-simplified.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s449609655-simplified.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test is based on an llvm-stress generated test case with seed=449609655
 
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s525530439.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s525530439.ll
index 0185c8d3eb336..aad184962c44d 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s525530439.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s525530439.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed for MSA with a
 ; `Num < NumOperands && "Invalid child # of SDNode!"' assertion.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-s997348632.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-s997348632.ll
index ae02778783e36..d3b65b5177e35 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-s997348632.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-s997348632.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed to select instructions for extract_vector_elt for
 ; v2f64 on MSA.
diff --git a/llvm/test/CodeGen/Mips/msa/llvm-stress-sz1-s742806235.ll b/llvm/test/CodeGen/Mips/msa/llvm-stress-sz1-s742806235.ll
index 16900fbf48385..8fc5de240650d 100644
--- a/llvm/test/CodeGen/Mips/msa/llvm-stress-sz1-s742806235.ll
+++ b/llvm/test/CodeGen/Mips/msa/llvm-stress-sz1-s742806235.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s
-; RUN: llc -march=mipsel < %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mips < %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s
+; RUN: llc -mtriple=mipsel < %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s
 
 ; This test originally failed to select code for a truncstore of a
 ; build_vector.
diff --git a/llvm/test/CodeGen/Mips/msa/msa-nooddspreg.ll b/llvm/test/CodeGen/Mips/msa/msa-nooddspreg.ll
index 3071a358d6dc1..8deff611f66b2 100644
--- a/llvm/test/CodeGen/Mips/msa/msa-nooddspreg.ll
+++ b/llvm/test/CodeGen/Mips/msa/msa-nooddspreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32r5 -mattr=+fp64,+msa,+nooddspreg < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32r5 -mattr=+fp64,+msa,+nooddspreg < %s | FileCheck %s
 
 ; Test that the register allocator honours +nooddspreg and does not pick an odd
 ; single precision subregister of an MSA register.
diff --git a/llvm/test/CodeGen/Mips/msa/nori.b.ll b/llvm/test/CodeGen/Mips/msa/nori.b.ll
index ea0d2a958a292..3332dd34d2dec 100644
--- a/llvm/test/CodeGen/Mips/msa/nori.b.ll
+++ b/llvm/test/CodeGen/Mips/msa/nori.b.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips64el -mcpu=mips64r6 -mattr=+msa,+fp64 < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el -mcpu=mips64r6 -mattr=+msa,+fp64 < %s | FileCheck %s
 
 ; Test that simply checks if it will finish when value 255 (-1) appears as
 ; immediate in 'nori.b' instruction.
diff --git a/llvm/test/CodeGen/Mips/msa/remat-ldi.ll b/llvm/test/CodeGen/Mips/msa/remat-ldi.ll
index 64f976c97a358..cd52077173dc3 100644
--- a/llvm/test/CodeGen/Mips/msa/remat-ldi.ll
+++ b/llvm/test/CodeGen/Mips/msa/remat-ldi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O3 -march=mipsel -mcpu=mips32r6 -mattr=+fp64,+msa %s -o - | FileCheck %s
+; RUN: llc -O3 -mtriple=mipsel -mcpu=mips32r6 -mattr=+fp64,+msa %s -o - | FileCheck %s
 
 ; Test that checks if spill for ldi can be avoided and instruction will be
 ; rematerialized.
diff --git a/llvm/test/CodeGen/Mips/msa/shift-dagcombine.ll b/llvm/test/CodeGen/Mips/msa/shift-dagcombine.ll
index 1f8572751c1a4..3f35a5be849ec 100644
--- a/llvm/test/CodeGen/Mips/msa/shift-dagcombine.ll
+++ b/llvm/test/CodeGen/Mips/msa/shift-dagcombine.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 define void @ashr_v4i32(ptr %c) nounwind {
   ; CHECK-LABEL: ashr_v4i32:
diff --git a/llvm/test/CodeGen/Mips/msa/shift_constant_pool.ll b/llvm/test/CodeGen/Mips/msa/shift_constant_pool.ll
index 9312a05f56960..2636f37a06ae6 100644
--- a/llvm/test/CodeGen/Mips/msa/shift_constant_pool.ll
+++ b/llvm/test/CodeGen/Mips/msa/shift_constant_pool.ll
@@ -1,13 +1,13 @@
 ; Test whether the following functions, with vectors featuring negative or values larger than the element
 ; bit size have their results of operations generated correctly when placed into constant pools
 
-; RUN: llc -march=mips64 -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64 -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,MIPS64 %s
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,MIPS32 %s
-; RUN: llc -march=mips64el -mattr=+msa,+fp64 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips64el -mattr=+msa,+fp64 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,MIPS64 %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefixes=ALL,MIPS32 %s
 
 @llvm_mips_bclr_w_test_const_vec_res = global <4 x i32> zeroinitializer, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/shift_no_and.ll b/llvm/test/CodeGen/Mips/msa/shift_no_and.ll
index d975f43e0c06b..58d20458afc74 100644
--- a/llvm/test/CodeGen/Mips/msa/shift_no_and.ll
+++ b/llvm/test/CodeGen/Mips/msa/shift_no_and.ll
@@ -1,7 +1,7 @@
 ; Test the absence of the andi.b / and.v instructions
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_bclr_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 @llvm_mips_bclr_b_ARG2 = global <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/shuffle.ll b/llvm/test/CodeGen/Mips/msa/shuffle.ll
index 61b42f5df441b..e93bb7cdd1128 100644
--- a/llvm/test/CodeGen/Mips/msa/shuffle.ll
+++ b/llvm/test/CodeGen/Mips/msa/shuffle.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 define void @vshf_v16i8_0(ptr %c, ptr %a, ptr %b) nounwind {
   ; CHECK-LABEL: vshf_v16i8_0:
diff --git a/llvm/test/CodeGen/Mips/msa/special.ll b/llvm/test/CodeGen/Mips/msa/special.ll
index 3e392110cccfe..df19d94eeed32 100644
--- a/llvm/test/CodeGen/Mips/msa/special.ll
+++ b/llvm/test/CodeGen/Mips/msa/special.ll
@@ -1,12 +1,12 @@
 ; Test the MSA intrinsics that are encoded with the SPECIAL instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | \
 ; RUN:   FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 < %s | \
 ; RUN:   FileCheck %s --check-prefix=MIPS64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+msa < %s | \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+msa < %s | \
 ; RUN:   FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64r6 -mattr=+msa < %s | \
+; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -mattr=+msa < %s | \
 ; RUN:   FileCheck %s --check-prefix=MIPS64
 
 define i32 @llvm_mips_lsa_test(i32 %a, i32 %b) nounwind {
diff --git a/llvm/test/CodeGen/Mips/msa/spill.ll b/llvm/test/CodeGen/Mips/msa/spill.ll
index 29f02920b10c6..51759f917a32c 100644
--- a/llvm/test/CodeGen/Mips/msa/spill.ll
+++ b/llvm/test/CodeGen/Mips/msa/spill.ll
@@ -1,8 +1,8 @@
 ; Test that the correct instruction is chosen for spill and reload by trying
 ; to have 33 live MSA registers simultaneously
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 define i32 @test_i8(ptr %p0, ptr %q1) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/msa/vec.ll b/llvm/test/CodeGen/Mips/msa/vec.ll
index cc4eba6c95bf1..8f3a822704fea 100644
--- a/llvm/test/CodeGen/Mips/msa/vec.ll
+++ b/llvm/test/CodeGen/Mips/msa/vec.ll
@@ -1,8 +1,8 @@
 ; Test the MSA intrinsics that are encoded with the VEC instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefix=ANYENDIAN %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 -relocation-model=pic < %s \
 ; RUN:   | FileCheck -check-prefix=ANYENDIAN %s
 
 @llvm_mips_and_v_b_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
diff --git a/llvm/test/CodeGen/Mips/msa/vecs10.ll b/llvm/test/CodeGen/Mips/msa/vecs10.ll
index ce61efc33f3a4..12da397a5d018 100644
--- a/llvm/test/CodeGen/Mips/msa/vecs10.ll
+++ b/llvm/test/CodeGen/Mips/msa/vecs10.ll
@@ -1,7 +1,7 @@
 ; Test the MSA intrinsics that are encoded with the VECS10 instruction format.
 
-; RUN: llc -march=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 @llvm_mips_bnz_v_ARG1 = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
 
diff --git a/llvm/test/CodeGen/Mips/mul.ll b/llvm/test/CodeGen/Mips/mul.ll
index 41ea550dbce7a..f5efed6415dc7 100644
--- a/llvm/test/CodeGen/Mips/mul.ll
+++ b/llvm/test/CodeGen/Mips/mul.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 5, align 4
 @jjjj = global i32 -6, align 4
diff --git a/llvm/test/CodeGen/Mips/mulll.ll b/llvm/test/CodeGen/Mips/mulll.ll
index 08813809bd5f0..c65c1bbd980c6 100644
--- a/llvm/test/CodeGen/Mips/mulll.ll
+++ b/llvm/test/CodeGen/Mips/mulll.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i64 5, align 8
 @jjjj = global i64 -6, align 8
diff --git a/llvm/test/CodeGen/Mips/mulull.ll b/llvm/test/CodeGen/Mips/mulull.ll
index 749adfa055729..129054776a03c 100644
--- a/llvm/test/CodeGen/Mips/mulull.ll
+++ b/llvm/test/CodeGen/Mips/mulull.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i64 5, align 8
 @jjjj = global i64 6, align 8
diff --git a/llvm/test/CodeGen/Mips/nacl-reserved-regs.ll b/llvm/test/CodeGen/Mips/nacl-reserved-regs.ll
index b3768b0887477..b03418dff6ccc 100644
--- a/llvm/test/CodeGen/Mips/nacl-reserved-regs.ll
+++ b/llvm/test/CodeGen/Mips/nacl-reserved-regs.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -O3 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -O3 < %s | FileCheck %s
 ; RUN: llc -mtriple=mipsel-none-nacl-gnu -O3 < %s \
 ; RUN:  | FileCheck %s -check-prefix=CHECK-NACL
 
diff --git a/llvm/test/CodeGen/Mips/neg1.ll b/llvm/test/CodeGen/Mips/neg1.ll
index 7f35a9884a10c..00d76be167bce 100644
--- a/llvm/test/CodeGen/Mips/neg1.ll
+++ b/llvm/test/CodeGen/Mips/neg1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10, align 4
 @.str = private unnamed_addr constant [5 x i8] c"%i \0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/nmadd.ll b/llvm/test/CodeGen/Mips/nmadd.ll
index 0fb534fbe3c70..57774b184d8c5 100644
--- a/llvm/test/CodeGen/Mips/nmadd.ll
+++ b/llvm/test/CodeGen/Mips/nmadd.ll
@@ -1,15 +1,15 @@
 ; Check whether nmadd/nmsub instructions are properly generated
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2 -mattr=+fp64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r2 -mattr=micromips -enable-no-nans-fp-math -asm-show-inst | FileCheck %s -check-prefixes=ALL,CHECK-NM,CHECK-MM
-; RUN: llc < %s -march=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
-; RUN: llc < %s -march=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
-; RUN: llc < %s -march=mips64el -mcpu=mips4    -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
-; RUN: llc < %s -march=mipsel   -mcpu=mips32   -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r6 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
-; RUN: llc < %s -march=mips64el -mcpu=mips3    -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM-64
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r6 -mattr=micromips -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
-; RUN: llc < %s -march=mipsel   -mcpu=mips32r3 -mattr=micromips -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2 -mattr=+fp64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r2 -mattr=micromips -enable-no-nans-fp-math -asm-show-inst | FileCheck %s -check-prefixes=ALL,CHECK-NM,CHECK-MM
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64   -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips64r2 -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips4    -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM-64
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32   -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r6 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
+; RUN: llc < %s -mtriple=mips64el -mcpu=mips3    -target-abi=n64 -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM-64
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r6 -mattr=micromips -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NOT-NM
+; RUN: llc < %s -mtriple=mipsel   -mcpu=mips32r3 -mattr=micromips -enable-no-nans-fp-math | FileCheck %s -check-prefixes=ALL,CHECK-NM
 
 define float @add1(float %f, float %g, float %h) local_unnamed_addr #0 {
 entry:
diff --git a/llvm/test/CodeGen/Mips/no-odd-spreg-msa.ll b/llvm/test/CodeGen/Mips/no-odd-spreg-msa.ll
index a71a645a40174..7c9f375cbb9ad 100644
--- a/llvm/test/CodeGen/Mips/no-odd-spreg-msa.ll
+++ b/llvm/test/CodeGen/Mips/no-odd-spreg-msa.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel -mcpu=mips32r5 -mattr=+fp64,+msa,-nooddspreg \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r5 -mattr=+fp64,+msa,-nooddspreg \
 ; RUN:   -verify-machineinstrs -no-integrated-as -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,ODDSPREG
-; RUN: llc -march=mipsel -mcpu=mips32r5 -mattr=+fp64,+msa,+nooddspreg \
+; RUN: llc -mtriple=mipsel -mcpu=mips32r5 -mattr=+fp64,+msa,+nooddspreg \
 ; RUN:   -verify-machineinstrs -no-integrated-as -relocation-model=pic < %s | \
 ; RUN:   FileCheck %s -check-prefixes=ALL,NOODDSPREG
 
diff --git a/llvm/test/CodeGen/Mips/not1.ll b/llvm/test/CodeGen/Mips/not1.ll
index 30b6f45a3b218..7893cb08f719d 100644
--- a/llvm/test/CodeGen/Mips/not1.ll
+++ b/llvm/test/CodeGen/Mips/not1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @x = global i32 65504, align 4
 @y = global i32 60929, align 4
diff --git a/llvm/test/CodeGen/Mips/null.ll b/llvm/test/CodeGen/Mips/null.ll
index 67e346d959f96..9f7c6eace93ab 100644
--- a/llvm/test/CodeGen/Mips/null.ll
+++ b/llvm/test/CodeGen/Mips/null.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 < %s | FileCheck %s -check-prefix=16
 
 
 define i32 @main() nounwind {
diff --git a/llvm/test/CodeGen/Mips/o32_cc.ll b/llvm/test/CodeGen/Mips/o32_cc.ll
index cef1290a75bda..fb6529ebe1472 100644
--- a/llvm/test/CodeGen/Mips/o32_cc.ll
+++ b/llvm/test/CodeGen/Mips/o32_cc.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel < %s | FileCheck -check-prefix=ALL %s
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck -check-prefixes=ALL,NO-MFHC1 %s
-; RUN: llc -march=mipsel -mcpu=mips32r2              < %s | FileCheck -check-prefixes=ALL,HAS-MFHC1 %s
-; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck -check-prefixes=ALL,HAS-MFHC1 %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck -check-prefix=ALL %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck -check-prefixes=ALL,NO-MFHC1 %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2              < %s | FileCheck -check-prefixes=ALL,HAS-MFHC1 %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -mattr=+fp64 < %s | FileCheck -check-prefixes=ALL,HAS-MFHC1 %s
 
 ; $f12, $f14
 ; ALL-LABEL: testlowercall0:
diff --git a/llvm/test/CodeGen/Mips/o32_cc_vararg.ll b/llvm/test/CodeGen/Mips/o32_cc_vararg.ll
index 750457eac5e92..f509f49a341a9 100644
--- a/llvm/test/CodeGen/Mips/o32_cc_vararg.ll
+++ b/llvm/test/CodeGen/Mips/o32_cc_vararg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -pre-RA-sched=source < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -pre-RA-sched=source < %s | FileCheck %s
 
 ; All test functions do the same thing - they return the first variable
 ; argument.
diff --git a/llvm/test/CodeGen/Mips/octeon.ll b/llvm/test/CodeGen/Mips/octeon.ll
index 11e93736f5cf0..09a35e05cb454 100644
--- a/llvm/test/CodeGen/Mips/octeon.ll
+++ b/llvm/test/CodeGen/Mips/octeon.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O1 < %s -march=mips64 -mcpu=octeon | FileCheck %s -check-prefixes=ALL,OCTEON
-; RUN: llc -O1 < %s -march=mips64 -mcpu=mips64 | FileCheck %s -check-prefixes=ALL,MIPS64
-; RUN: llc -O1 < %s -march=mips64 -mcpu=octeon -relocation-model=pic | FileCheck %s -check-prefixes=ALL,OCTEON-PIC
+; RUN: llc -O1 < %s -mtriple=mips64 -mcpu=octeon | FileCheck %s -check-prefixes=ALL,OCTEON
+; RUN: llc -O1 < %s -mtriple=mips64 -mcpu=mips64 | FileCheck %s -check-prefixes=ALL,MIPS64
+; RUN: llc -O1 < %s -mtriple=mips64 -mcpu=octeon -relocation-model=pic | FileCheck %s -check-prefixes=ALL,OCTEON-PIC
 
 define i64 @addi64(i64 %a, i64 %b) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/octeon_popcnt.ll b/llvm/test/CodeGen/Mips/octeon_popcnt.ll
index 13488ed59504c..84fb0bdc5afcd 100644
--- a/llvm/test/CodeGen/Mips/octeon_popcnt.ll
+++ b/llvm/test/CodeGen/Mips/octeon_popcnt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O1 -march=mips64 -mcpu=octeon < %s | FileCheck %s -check-prefix=OCTEON
-; RUN: llc -O1 -march=mips64 -mcpu=mips64 < %s | FileCheck %s -check-prefix=MIPS64
+; RUN: llc -O1 -mtriple=mips64 -mcpu=octeon < %s | FileCheck %s -check-prefix=OCTEON
+; RUN: llc -O1 -mtriple=mips64 -mcpu=mips64 < %s | FileCheck %s -check-prefix=MIPS64
 
 define i8 @cnt8(i8 %x) nounwind readnone {
   %cnt = tail call i8 @llvm.ctpop.i8(i8 %x)
diff --git a/llvm/test/CodeGen/Mips/optimize-fp-math.ll b/llvm/test/CodeGen/Mips/optimize-fp-math.ll
index 7886f29f5cef2..7124013714bb1 100644
--- a/llvm/test/CodeGen/Mips/optimize-fp-math.ll
+++ b/llvm/test/CodeGen/Mips/optimize-fp-math.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
 
 ; 32-LABEL: test_sqrtf_float_:
 ; 32: sqrt.s $f[[R0:[0-9]+]], $f{{[0-9]+}}
diff --git a/llvm/test/CodeGen/Mips/or1.ll b/llvm/test/CodeGen/Mips/or1.ll
index 94e314491a7c8..2657270bee947 100644
--- a/llvm/test/CodeGen/Mips/or1.ll
+++ b/llvm/test/CodeGen/Mips/or1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @x = global i32 65504, align 4
 @y = global i32 60929, align 4
diff --git a/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll b/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
index 730e01d8f114d..63a730c186167 100644
--- a/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
+++ b/llvm/test/CodeGen/Mips/pbqp-reserved-physreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -regalloc=pbqp <%s > %t
+; RUN: llc -mtriple=mips -regalloc=pbqp <%s > %t
 ; ModuleID = 'bugpoint-reduced-simplified.bc'
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/Mips/pr33682.ll b/llvm/test/CodeGen/Mips/pr33682.ll
index 93ef4f900db07..545b25c18fae8 100644
--- a/llvm/test/CodeGen/Mips/pr33682.ll
+++ b/llvm/test/CodeGen/Mips/pr33682.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -mcpu=mips32  < %s | FileCheck %s --check-prefixes=ALL,BE
-; RUN: llc -march=mipsel -mcpu=mips32  < %s | FileCheck %s --check-prefixes=ALL,LE
+; RUN: llc -mtriple=mips -mcpu=mips32  < %s | FileCheck %s --check-prefixes=ALL,BE
+; RUN: llc -mtriple=mipsel -mcpu=mips32  < %s | FileCheck %s --check-prefixes=ALL,LE
 
 ; Verify visitTRUNCATE respects endianness when transforming trunc to insert_vector_elt.
 
diff --git a/llvm/test/CodeGen/Mips/pr33978.ll b/llvm/test/CodeGen/Mips/pr33978.ll
index 921fa543cfda9..136de5edfd337 100644
--- a/llvm/test/CodeGen/Mips/pr33978.ll
+++ b/llvm/test/CodeGen/Mips/pr33978.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32r2 < %s -o /dev/null
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s -o /dev/null
 
 ; Test that SelectionDAG does not crash during DAGCombine when two pointers
 ; to the stack match with differing bases and offsets when expanding memcpy.
diff --git a/llvm/test/CodeGen/Mips/prevent-hoisting.ll b/llvm/test/CodeGen/Mips/prevent-hoisting.ll
index e44b895689b49..05b7e964c9ae0 100644
--- a/llvm/test/CodeGen/Mips/prevent-hoisting.ll
+++ b/llvm/test/CodeGen/Mips/prevent-hoisting.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -O3 -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -O3 -relocation-model=pic < %s | FileCheck %s
 
 
 ; MIPS direct branches implicitly define register $at. This test makes sure that
diff --git a/llvm/test/CodeGen/Mips/private.ll b/llvm/test/CodeGen/Mips/private.ll
index e93c2d7fc5d9e..be936ddde0072 100644
--- a/llvm/test/CodeGen/Mips/private.ll
+++ b/llvm/test/CodeGen/Mips/private.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that the 'private' is used correctly.
 ;
-; RUN: llc -march=mips -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -relocation-model=pic < %s | FileCheck %s
 
 define private void @foo() {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/Mips/ra-allocatable.ll b/llvm/test/CodeGen/Mips/ra-allocatable.ll
index dcbf90393576a..52ec4cf77ce44 100644
--- a/llvm/test/CodeGen/Mips/ra-allocatable.ll
+++ b/llvm/test/CodeGen/Mips/ra-allocatable.ll
@@ -1,4 +1,4 @@
-; RUN: llc  < %s -march=mipsel | FileCheck %s
+; RUN: llc  < %s -mtriple=mipsel | FileCheck %s
 
 @a0 = external global i32
 @b0 = external global ptr
diff --git a/llvm/test/CodeGen/Mips/rem.ll b/llvm/test/CodeGen/Mips/rem.ll
index 525ab633ca019..c7b4da98d63e4 100644
--- a/llvm/test/CodeGen/Mips/rem.ll
+++ b/llvm/test/CodeGen/Mips/rem.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 103, align 4
 @jjjj = global i32 -4, align 4
diff --git a/llvm/test/CodeGen/Mips/remat-immed-load.ll b/llvm/test/CodeGen/Mips/remat-immed-load.ll
index 3d37b43bbc630..ec32acb2b78f7 100644
--- a/llvm/test/CodeGen/Mips/remat-immed-load.ll
+++ b/llvm/test/CodeGen/Mips/remat-immed-load.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips4 -target-abi=n64 < %s | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 -target-abi=n64 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips4 -target-abi=n64 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 -target-abi=n64 < %s | FileCheck %s -check-prefix=64
 
 define void @f0() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/remu.ll b/llvm/test/CodeGen/Mips/remu.ll
index 3d5f5d7629d95..6008d2d06a640 100644
--- a/llvm/test/CodeGen/Mips/remu.ll
+++ b/llvm/test/CodeGen/Mips/remu.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @iiii = global i32 103, align 4
 @jjjj = global i32 4, align 4
diff --git a/llvm/test/CodeGen/Mips/return-vector.ll b/llvm/test/CodeGen/Mips/return-vector.ll
index c59695d18734e..c4fe4467bdcce 100644
--- a/llvm/test/CodeGen/Mips/return-vector.ll
+++ b/llvm/test/CodeGen/Mips/return-vector.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 
 ; Check that function accesses vector return value from stack in cases when
diff --git a/llvm/test/CodeGen/Mips/return_address.ll b/llvm/test/CodeGen/Mips/return_address.ll
index 65b4ff5733f43..0d02c1b3a9202 100644
--- a/llvm/test/CodeGen/Mips/return_address.ll
+++ b/llvm/test/CodeGen/Mips/return_address.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -verify-machineinstrs < %s | FileCheck %s
 
 define ptr @f1() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/Mips/return_address_err.ll b/llvm/test/CodeGen/Mips/return_address_err.ll
index bf2ab3c164886..9a8fe1d0545f5 100644
--- a/llvm/test/CodeGen/Mips/return_address_err.ll
+++ b/llvm/test/CodeGen/Mips/return_address_err.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -march=mips < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=mips < %s 2>&1 | FileCheck %s
 
 declare ptr @llvm.returnaddress(i32) nounwind readnone
 
diff --git a/llvm/test/CodeGen/Mips/rotate.ll b/llvm/test/CodeGen/Mips/rotate.ll
index 77936b7bef9b6..9063cc163891f 100644
--- a/llvm/test/CodeGen/Mips/rotate.ll
+++ b/llvm/test/CodeGen/Mips/rotate.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s
 ; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips32r2 -mattr=+mips16 < %s | FileCheck %s -check-prefix=mips16
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips < %s | FileCheck %s \
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=+micromips < %s | FileCheck %s \
 ; RUN:    -check-prefix=MM32
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips < %s | FileCheck %s \
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips < %s | FileCheck %s \
 ; RUN:    -check-prefix=MM32
 
 ; CHECK:  rotrv $2, $4
diff --git a/llvm/test/CodeGen/Mips/sb1.ll b/llvm/test/CodeGen/Mips/sb1.ll
index 8b79f39d2c70f..daecafc8418e0 100644
--- a/llvm/test/CodeGen/Mips/sb1.ll
+++ b/llvm/test/CodeGen/Mips/sb1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 97, align 4
 @c = common global i8 0, align 1
diff --git a/llvm/test/CodeGen/Mips/selTBteqzCmpi.ll b/llvm/test/CodeGen/Mips/selTBteqzCmpi.ll
index a81393b0b0807..6520bcdc0622a 100644
--- a/llvm/test/CodeGen/Mips/selTBteqzCmpi.ll
+++ b/llvm/test/CodeGen/Mips/selTBteqzCmpi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 1, align 4
 @j = global i32 2, align 4
diff --git a/llvm/test/CodeGen/Mips/selTBtnezCmpi.ll b/llvm/test/CodeGen/Mips/selTBtnezCmpi.ll
index e703e317a0fb8..9c3089d56ae32 100644
--- a/llvm/test/CodeGen/Mips/selTBtnezCmpi.ll
+++ b/llvm/test/CodeGen/Mips/selTBtnezCmpi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 1, align 4
 @j = global i32 2, align 4
diff --git a/llvm/test/CodeGen/Mips/selTBtnezSlti.ll b/llvm/test/CodeGen/Mips/selTBtnezSlti.ll
index 132d5ed770207..1e32d9f2be1f6 100644
--- a/llvm/test/CodeGen/Mips/selTBtnezSlti.ll
+++ b/llvm/test/CodeGen/Mips/selTBtnezSlti.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 1, align 4
 @j = global i32 2, align 4
diff --git a/llvm/test/CodeGen/Mips/selectcc.ll b/llvm/test/CodeGen/Mips/selectcc.ll
index 1bec29e8bcd5f..532fc006ccd45 100644
--- a/llvm/test/CodeGen/Mips/selectcc.ll
+++ b/llvm/test/CodeGen/Mips/selectcc.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mipsel -mcpu=mips32 -relocation-model=pic < %s
-; RUN: llc -march=mipsel -mcpu=mips32 -pre-RA-sched=source -relocation-model=pic < %s | FileCheck %s --check-prefix=SOURCE-SCHED
-; RUN: llc -march=mipsel -mcpu=mips32r2 -relocation-model=pic < %s
-; RUN: llc -march=mipsel -mcpu=mips32r2 -pre-RA-sched=source -relocation-model=pic < %s | FileCheck %s --check-prefix=SOURCE-SCHED
+; RUN: llc -mtriple=mipsel -mcpu=mips32 -relocation-model=pic < %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32 -pre-RA-sched=source -relocation-model=pic < %s | FileCheck %s --check-prefix=SOURCE-SCHED
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -relocation-model=pic < %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32r2 -pre-RA-sched=source -relocation-model=pic < %s | FileCheck %s --check-prefix=SOURCE-SCHED
 
 @gf0 = external global float
 @gf1 = external global float
diff --git a/llvm/test/CodeGen/Mips/selectiondag-optlevel.ll b/llvm/test/CodeGen/Mips/selectiondag-optlevel.ll
index 9632eac84419c..048a362ab1322 100644
--- a/llvm/test/CodeGen/Mips/selectiondag-optlevel.ll
+++ b/llvm/test/CodeGen/Mips/selectiondag-optlevel.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mips -fast-isel=false -O0 < %s 2>&1 | FileCheck %s -check-prefix=O0
-; RUN: llc -march=mips -fast-isel=false -O2 < %s 2>&1 | FileCheck %s -check-prefix=O2
+; RUN: llc -mtriple=mips -fast-isel=false -O0 < %s 2>&1 | FileCheck %s -check-prefix=O0
+; RUN: llc -mtriple=mips -fast-isel=false -O2 < %s 2>&1 | FileCheck %s -check-prefix=O2
 
 ; At -O0, DAGCombine won't try to merge these consecutive loads but it will at
 ; -O2.
diff --git a/llvm/test/CodeGen/Mips/seleq.ll b/llvm/test/CodeGen/Mips/seleq.ll
index ecbeb2b51e3d7..fcf12202b40ca 100644
--- a/llvm/test/CodeGen/Mips/seleq.ll
+++ b/llvm/test/CodeGen/Mips/seleq.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/seleqk.ll b/llvm/test/CodeGen/Mips/seleqk.ll
index 911c6f1996b67..6292494671717 100644
--- a/llvm/test/CodeGen/Mips/seleqk.ll
+++ b/llvm/test/CodeGen/Mips/seleqk.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selgek.ll b/llvm/test/CodeGen/Mips/selgek.ll
index a909bb543538b..9c9e77ae236a7 100644
--- a/llvm/test/CodeGen/Mips/selgek.ll
+++ b/llvm/test/CodeGen/Mips/selgek.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selgt.ll b/llvm/test/CodeGen/Mips/selgt.ll
index 30d7f8ff3a695..a59c89f875173 100644
--- a/llvm/test/CodeGen/Mips/selgt.ll
+++ b/llvm/test/CodeGen/Mips/selgt.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selle.ll b/llvm/test/CodeGen/Mips/selle.ll
index bccc3de56705e..59fe279966183 100644
--- a/llvm/test/CodeGen/Mips/selle.ll
+++ b/llvm/test/CodeGen/Mips/selle.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selltk.ll b/llvm/test/CodeGen/Mips/selltk.ll
index b070c301b0199..2661ea1dead26 100644
--- a/llvm/test/CodeGen/Mips/selltk.ll
+++ b/llvm/test/CodeGen/Mips/selltk.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selne.ll b/llvm/test/CodeGen/Mips/selne.ll
index 6fe9e48279877..0d117c4d76601 100644
--- a/llvm/test/CodeGen/Mips/selne.ll
+++ b/llvm/test/CodeGen/Mips/selne.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selnek.ll b/llvm/test/CodeGen/Mips/selnek.ll
index f38ab246e60f4..ecc68842cc623 100644
--- a/llvm/test/CodeGen/Mips/selnek.ll
+++ b/llvm/test/CodeGen/Mips/selnek.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
+; RUN: llc -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/selpat.ll b/llvm/test/CodeGen/Mips/selpat.ll
index d765acbab33a1..ad263bb41295b 100644
--- a/llvm/test/CodeGen/Mips/selpat.ll
+++ b/llvm/test/CodeGen/Mips/selpat.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @t = global i32 10, align 4
 @f = global i32 199, align 4
diff --git a/llvm/test/CodeGen/Mips/setcc-se.ll b/llvm/test/CodeGen/Mips/setcc-se.ll
index 0e74db3c67866..26fd2845f0817 100644
--- a/llvm/test/CodeGen/Mips/setcc-se.ll
+++ b/llvm/test/CodeGen/Mips/setcc-se.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
 
 @g1 = external global i32
 
diff --git a/llvm/test/CodeGen/Mips/seteq.ll b/llvm/test/CodeGen/Mips/seteq.ll
index 37fd1634f796a..8bbde5de812c8 100644
--- a/llvm/test/CodeGen/Mips/seteq.ll
+++ b/llvm/test/CodeGen/Mips/seteq.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @i = global i32 1, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/seteqz.ll b/llvm/test/CodeGen/Mips/seteqz.ll
index c71ed1d91f72b..006752d6e28df 100644
--- a/llvm/test/CodeGen/Mips/seteqz.ll
+++ b/llvm/test/CodeGen/Mips/seteqz.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @i = global i32 0, align 4
 @j = global i32 99, align 4
diff --git a/llvm/test/CodeGen/Mips/setge.ll b/llvm/test/CodeGen/Mips/setge.ll
index 6fb549e24b3fa..e6021150ab7d4 100644
--- a/llvm/test/CodeGen/Mips/setge.ll
+++ b/llvm/test/CodeGen/Mips/setge.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 -5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setgek.ll b/llvm/test/CodeGen/Mips/setgek.ll
index 661a3f160e325..066b65b8be326 100644
--- a/llvm/test/CodeGen/Mips/setgek.ll
+++ b/llvm/test/CodeGen/Mips/setgek.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @k = global i32 10, align 4
 @r1 = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/setle.ll b/llvm/test/CodeGen/Mips/setle.ll
index 35453b8f5f6af..ecd779cc04b00 100644
--- a/llvm/test/CodeGen/Mips/setle.ll
+++ b/llvm/test/CodeGen/Mips/setle.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 -5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setlt.ll b/llvm/test/CodeGen/Mips/setlt.ll
index fcf9a25be72c0..234d1bc2b3c20 100644
--- a/llvm/test/CodeGen/Mips/setlt.ll
+++ b/llvm/test/CodeGen/Mips/setlt.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 -5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setltk.ll b/llvm/test/CodeGen/Mips/setltk.ll
index 95aa9ea07d245..4be83bb49541e 100644
--- a/llvm/test/CodeGen/Mips/setltk.ll
+++ b/llvm/test/CodeGen/Mips/setltk.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mattr=micromips -mcpu=mips32r6 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mattr=micromips -mcpu=mips32r6 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 -5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setne.ll b/llvm/test/CodeGen/Mips/setne.ll
index d9396b1bf7754..5a37c88ea59b0 100644
--- a/llvm/test/CodeGen/Mips/setne.ll
+++ b/llvm/test/CodeGen/Mips/setne.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @i = global i32 1, align 4
 @j = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setuge.ll b/llvm/test/CodeGen/Mips/setuge.ll
index afdd4ef733758..121daf3272469 100644
--- a/llvm/test/CodeGen/Mips/setuge.ll
+++ b/llvm/test/CodeGen/Mips/setuge.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setugt.ll b/llvm/test/CodeGen/Mips/setugt.ll
index afe4e4b09c781..2a08d80e0f426 100644
--- a/llvm/test/CodeGen/Mips/setugt.ll
+++ b/llvm/test/CodeGen/Mips/setugt.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setule.ll b/llvm/test/CodeGen/Mips/setule.ll
index 024e7701fad4d..63ee5257ec9a9 100644
--- a/llvm/test/CodeGen/Mips/setule.ll
+++ b/llvm/test/CodeGen/Mips/setule.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setult.ll b/llvm/test/CodeGen/Mips/setult.ll
index ef354ec0971e4..0272dd300b153 100644
--- a/llvm/test/CodeGen/Mips/setult.ll
+++ b/llvm/test/CodeGen/Mips/setult.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/setultk.ll b/llvm/test/CodeGen/Mips/setultk.ll
index 9801d388d2232..8f06fb324b5aa 100644
--- a/llvm/test/CodeGen/Mips/setultk.ll
+++ b/llvm/test/CodeGen/Mips/setultk.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=MMR6
 
 @j = global i32 5, align 4
 @k = global i32 10, align 4
diff --git a/llvm/test/CodeGen/Mips/sh1.ll b/llvm/test/CodeGen/Mips/sh1.ll
index 6bfa9f56231f5..d3bdc15eedc3e 100644
--- a/llvm/test/CodeGen/Mips/sh1.ll
+++ b/llvm/test/CodeGen/Mips/sh1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 97, align 4
 @s = common global i16 0, align 2
diff --git a/llvm/test/CodeGen/Mips/shift-parts.ll b/llvm/test/CodeGen/Mips/shift-parts.ll
index 38cbf28108cae..c77ec0249ae10 100644
--- a/llvm/test/CodeGen/Mips/shift-parts.ll
+++ b/llvm/test/CodeGen/Mips/shift-parts.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 define i64 @shl0(i64 %a, i32 %b) nounwind readnone {
 entry:
diff --git a/llvm/test/CodeGen/Mips/sint-fp-store_pattern.ll b/llvm/test/CodeGen/Mips/sint-fp-store_pattern.ll
index 4d15f428e44be..b79d25dfe5ed7 100644
--- a/llvm/test/CodeGen/Mips/sint-fp-store_pattern.ll
+++ b/llvm/test/CodeGen/Mips/sint-fp-store_pattern.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %s | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %s | FileCheck %s -check-prefix=64
 
 @gint_ = external global i32
 @gLL_ = external global i64
diff --git a/llvm/test/CodeGen/Mips/sitofp-selectcc-opt.ll b/llvm/test/CodeGen/Mips/sitofp-selectcc-opt.ll
index 3e2de21b63cae..0f406f0695479 100644
--- a/llvm/test/CodeGen/Mips/sitofp-selectcc-opt.ll
+++ b/llvm/test/CodeGen/Mips/sitofp-selectcc-opt.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 @foo12.d4 = internal unnamed_addr global double 0.000000e+00, align 8
 
diff --git a/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir b/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
index 45807b2e48b0d..b87ee412038ca 100644
--- a/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
+++ b/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips %s -start-after=xray-instrumentation -o - -show-mc-encoding | FileCheck %s
+# RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=+micromips %s -start-after=xray-instrumentation -o - -show-mc-encoding | FileCheck %s
 
 # Test that the 'sll $zero, $zero, 0' is correctly recognized as a real
 # instruction rather than some unimplemented opcode for the purposes of
diff --git a/llvm/test/CodeGen/Mips/sll1.ll b/llvm/test/CodeGen/Mips/sll1.ll
index bf36682100d9f..79b8d4bf70b36 100644
--- a/llvm/test/CodeGen/Mips/sll1.ll
+++ b/llvm/test/CodeGen/Mips/sll1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10, align 4
 @j = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/sll2.ll b/llvm/test/CodeGen/Mips/sll2.ll
index 5de0b561cc989..c3d763f0551fa 100644
--- a/llvm/test/CodeGen/Mips/sll2.ll
+++ b/llvm/test/CodeGen/Mips/sll2.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10, align 4
 @j = global i32 4, align 4
diff --git a/llvm/test/CodeGen/Mips/slt.ll b/llvm/test/CodeGen/Mips/slt.ll
index a17f5ab0e57dd..01dc1299f4f13 100644
--- a/llvm/test/CodeGen/Mips/slt.ll
+++ b/llvm/test/CodeGen/Mips/slt.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=micromips -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mcpu=mips32r3 -mattr=micromips -relocation-model=pic < %s | FileCheck %s
 
 define i32 @slt(i32 signext %a) nounwind readnone {
   %1 = icmp slt i32 %a, 7
diff --git a/llvm/test/CodeGen/Mips/spill-copy-acreg.ll b/llvm/test/CodeGen/Mips/spill-copy-acreg.ll
index ea24bebe3e5f9..71aab3832d0bf 100644
--- a/llvm/test/CodeGen/Mips/spill-copy-acreg.ll
+++ b/llvm/test/CodeGen/Mips/spill-copy-acreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=+dsp < %s
+; RUN: llc -mtriple=mipsel -mattr=+dsp < %s
 
 @g1 = common global i64 0, align 8
 @g2 = common global i64 0, align 8
diff --git a/llvm/test/CodeGen/Mips/sra1.ll b/llvm/test/CodeGen/Mips/sra1.ll
index 08286760d5cda..ad8382f238719 100644
--- a/llvm/test/CodeGen/Mips/sra1.ll
+++ b/llvm/test/CodeGen/Mips/sra1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 -354, align 4
 @.str = private unnamed_addr constant [5 x i8] c"%i \0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/sra2.ll b/llvm/test/CodeGen/Mips/sra2.ll
index 2dd966b0ff843..50b00c935b9bb 100644
--- a/llvm/test/CodeGen/Mips/sra2.ll
+++ b/llvm/test/CodeGen/Mips/sra2.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 -354, align 4
 @j = global i32 3, align 4
diff --git a/llvm/test/CodeGen/Mips/srl1.ll b/llvm/test/CodeGen/Mips/srl1.ll
index 21cdeda4af448..e9fb916758a77 100644
--- a/llvm/test/CodeGen/Mips/srl1.ll
+++ b/llvm/test/CodeGen/Mips/srl1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10654, align 4
 @j = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/srl2.ll b/llvm/test/CodeGen/Mips/srl2.ll
index b433893822dc5..17895ff85d08d 100644
--- a/llvm/test/CodeGen/Mips/srl2.ll
+++ b/llvm/test/CodeGen/Mips/srl2.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10654, align 4
 @j = global i32 0, align 4
diff --git a/llvm/test/CodeGen/Mips/stack-alignment.ll b/llvm/test/CodeGen/Mips/stack-alignment.ll
index 2ae917f50be10..22275a53f38ed 100644
--- a/llvm/test/CodeGen/Mips/stack-alignment.ll
+++ b/llvm/test/CodeGen/Mips/stack-alignment.ll
@@ -1,12 +1,12 @@
 ; RUN: split-file %s %t
 ; RUN: cat %t/main.ll %t/_32.ll > %t/32.ll
-; RUN: llc -march=mipsel < %t/main.ll | FileCheck %s -check-prefix=32
-; RUN: llc -march=mipsel < %t/32.ll | FileCheck %s -check-prefix=A32-32
-; RUN: llc -march=mipsel -mattr=+fp64,+mips32r2 < %t/main.ll | FileCheck %s -check-prefix=32
-; RUN: llc -march=mips64el -mcpu=mips3 < %t/main.ll | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips4 < %t/main.ll | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 < %t/main.ll | FileCheck %s -check-prefix=64
-; RUN: llc -march=mips64el -mcpu=mips64 < %t/32.ll | FileCheck %s -check-prefix=A32-64
+; RUN: llc -mtriple=mipsel < %t/main.ll | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mipsel < %t/32.ll | FileCheck %s -check-prefix=A32-32
+; RUN: llc -mtriple=mipsel -mattr=+fp64,+mips32r2 < %t/main.ll | FileCheck %s -check-prefix=32
+; RUN: llc -mtriple=mips64el -mcpu=mips3 < %t/main.ll | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips4 < %t/main.ll | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %t/main.ll | FileCheck %s -check-prefix=64
+; RUN: llc -mtriple=mips64el -mcpu=mips64 < %t/32.ll | FileCheck %s -check-prefix=A32-64
 
 ;--- main.ll
 ; 32:      addiu  $sp, $sp, -8
diff --git a/llvm/test/CodeGen/Mips/stackcoloring.ll b/llvm/test/CodeGen/Mips/stackcoloring.ll
index aeedbc8abccd0..12ccabd52e476 100644
--- a/llvm/test/CodeGen/Mips/stackcoloring.ll
+++ b/llvm/test/CodeGen/Mips/stackcoloring.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic < %s | FileCheck %s
 
 @g1 = external global ptr
 
diff --git a/llvm/test/CodeGen/Mips/stchar.ll b/llvm/test/CodeGen/Mips/stchar.ll
index 0a2dfbe1183a6..f9bb1b0c2fb11 100644
--- a/llvm/test/CodeGen/Mips/stchar.ll
+++ b/llvm/test/CodeGen/Mips/stchar.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16_h
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16_b
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16_h
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16_b
 
 @.str = private unnamed_addr constant [9 x i8] c"%hd %c \0A\00", align 1
 @sp = common global ptr null, align 4
diff --git a/llvm/test/CodeGen/Mips/stldst.ll b/llvm/test/CodeGen/Mips/stldst.ll
index 0208d62b4f8ab..6f1ed4c78f3ba 100644
--- a/llvm/test/CodeGen/Mips/stldst.ll
+++ b/llvm/test/CodeGen/Mips/stldst.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @kkkk = global i32 67, align 4
 @llll = global i32 33, align 4
diff --git a/llvm/test/CodeGen/Mips/sub1.ll b/llvm/test/CodeGen/Mips/sub1.ll
index 8e30a1732ce4e..8f1e941b99d5b 100644
--- a/llvm/test/CodeGen/Mips/sub1.ll
+++ b/llvm/test/CodeGen/Mips/sub1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10, align 4
 @.str = private unnamed_addr constant [4 x i8] c"%i\0A\00", align 1
diff --git a/llvm/test/CodeGen/Mips/sub2.ll b/llvm/test/CodeGen/Mips/sub2.ll
index 5d6296d279e46..c2a404dbc5c38 100644
--- a/llvm/test/CodeGen/Mips/sub2.ll
+++ b/llvm/test/CodeGen/Mips/sub2.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @i = global i32 10, align 4
 @j = global i32 20, align 4
diff --git a/llvm/test/CodeGen/Mips/swzero.ll b/llvm/test/CodeGen/Mips/swzero.ll
index cc17a1559db96..6059839e2de18 100644
--- a/llvm/test/CodeGen/Mips/swzero.ll
+++ b/llvm/test/CodeGen/Mips/swzero.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
 
 %struct.unaligned = type <{ i32 }>
 
diff --git a/llvm/test/CodeGen/Mips/tailcall/tail-call-arguments-clobber.ll b/llvm/test/CodeGen/Mips/tailcall/tail-call-arguments-clobber.ll
index dda24a2ca163f..c7ad22cf427b5 100644
--- a/llvm/test/CodeGen/Mips/tailcall/tail-call-arguments-clobber.ll
+++ b/llvm/test/CodeGen/Mips/tailcall/tail-call-arguments-clobber.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips -mcpu=mips32 -O0 -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck \
+; RUN: llc -mtriple=mips -mcpu=mips32 -O0 -relocation-model=pic -mips-tail-calls=1 < %s | FileCheck \
 ; RUN:     %s -check-prefix=MIPS32
-; RUN: llc -march=mips64 -mcpu=mips64 -O0 -relocation-model=pic -target-abi n64 \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -O0 -relocation-model=pic -target-abi n64 \
 ; RUN:     -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=MIPS64
-; RUN: llc -march=mips64 -mcpu=mips64 -O0 -relocation-model=pic -target-abi n32 \
+; RUN: llc -mtriple=mips64 -mcpu=mips64 -O0 -relocation-model=pic -target-abi n32 \
 ; RUN:     -mips-tail-calls=1 < %s | FileCheck %s -check-prefix=MIPS64
 
 
diff --git a/llvm/test/CodeGen/Mips/thread-pointer.ll b/llvm/test/CodeGen/Mips/thread-pointer.ll
index b460d6dd0a18b..30d3c9385b980 100644
--- a/llvm/test/CodeGen/Mips/thread-pointer.ll
+++ b/llvm/test/CodeGen/Mips/thread-pointer.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=mips < %s | FileCheck %s
-; RUN: llc -march=mips64 < %s | FileCheck %s
-; RUN: llc -march=mipsel < %s | FileCheck %s
-; RUN: llc -march=mips64el < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips64 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s | FileCheck %s
+; RUN: llc -mtriple=mips64el < %s | FileCheck %s
 
 declare ptr @llvm.thread.pointer() nounwind readnone
 
diff --git a/llvm/test/CodeGen/Mips/tls-alias.ll b/llvm/test/CodeGen/Mips/tls-alias.ll
index 1842830880778..a2ba2250995f8 100644
--- a/llvm/test/CodeGen/Mips/tls-alias.ll
+++ b/llvm/test/CodeGen/Mips/tls-alias.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -disable-mips-delay-filler < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -relocation-model=pic -disable-mips-delay-filler < %s | FileCheck %s
 
 @foo = thread_local global i32 42
 @bar = hidden thread_local alias i32, ptr @foo
diff --git a/llvm/test/CodeGen/Mips/tnaked.ll b/llvm/test/CodeGen/Mips/tnaked.ll
index 803fef965bb17..ac54f2faae3f5 100644
--- a/llvm/test/CodeGen/Mips/tnaked.ll
+++ b/llvm/test/CodeGen/Mips/tnaked.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple=mipsel < %s -verify-machineinstrs | FileCheck %s
 
 
 define void @tnaked() #0 {
diff --git a/llvm/test/CodeGen/Mips/trap.ll b/llvm/test/CodeGen/Mips/trap.ll
index beb4b894632b8..e824d01d5f7c6 100644
--- a/llvm/test/CodeGen/Mips/trap.ll
+++ b/llvm/test/CodeGen/Mips/trap.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mcpu=mips32 < %s | FileCheck %s
 
 declare void @llvm.trap()
 
diff --git a/llvm/test/CodeGen/Mips/uitofp.ll b/llvm/test/CodeGen/Mips/uitofp.ll
index 08cd692deb3ca..1682a627155ac 100644
--- a/llvm/test/CodeGen/Mips/uitofp.ll
+++ b/llvm/test/CodeGen/Mips/uitofp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=mips -mattr=+single-float < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mattr=+single-float < %s | FileCheck %s
 
 define void @f0() nounwind {
 ; CHECK-LABEL: f0:
diff --git a/llvm/test/CodeGen/Mips/ul1.ll b/llvm/test/CodeGen/Mips/ul1.ll
index 5e3fc20dd39f8..f21050aec11f1 100644
--- a/llvm/test/CodeGen/Mips/ul1.ll
+++ b/llvm/test/CodeGen/Mips/ul1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 %struct.ua = type <{ i16, i32 }>
 
 @foo = common global %struct.ua zeroinitializer, align 1
diff --git a/llvm/test/CodeGen/Mips/unaligned-memops.ll b/llvm/test/CodeGen/Mips/unaligned-memops.ll
index 58475a216d6cc..caaa212a40cd5 100644
--- a/llvm/test/CodeGen/Mips/unaligned-memops.ll
+++ b/llvm/test/CodeGen/Mips/unaligned-memops.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -march=mips -mcpu=mips32r2 -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+micromips -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=MICROMIPS
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=MIPS
+; RUN: llc -mtriple=mips -mcpu=mips32r2 -mattr=+micromips -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=MICROMIPS
 
 ; Test that the correct ISA version of the unaligned memory operations is
 ; selected up front.
diff --git a/llvm/test/CodeGen/Mips/unalignedload.ll b/llvm/test/CodeGen/Mips/unalignedload.ll
index da57b92e8f6df..030c640522752 100644
--- a/llvm/test/CodeGen/Mips/unalignedload.ll
+++ b/llvm/test/CodeGen/Mips/unalignedload.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc  < %s -march=mipsel -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EL
-; RUN: llc  < %s -march=mips   -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EB
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EL
-; RUN: llc  < %s -march=mips   -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EB
-; RUN: llc  < %s -march=mipsel -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32R6-EL
-; RUN: llc  < %s -march=mips   -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32R6-EB
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EL
+; RUN: llc  < %s -mtriple=mips   -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EB
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EL
+; RUN: llc  < %s -mtriple=mips   -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32-EB
+; RUN: llc  < %s -mtriple=mipsel -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32R6-EL
+; RUN: llc  < %s -mtriple=mips   -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=MIPS32R6-EB
 
 %struct.S2 = type { %struct.S1, %struct.S1 }
 %struct.S1 = type { i8, i8 }
diff --git a/llvm/test/CodeGen/Mips/vector-load-store.ll b/llvm/test/CodeGen/Mips/vector-load-store.ll
index 142f383b72c7a..daa4022912162 100644
--- a/llvm/test/CodeGen/Mips/vector-load-store.ll
+++ b/llvm/test/CodeGen/Mips/vector-load-store.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mattr=+dsp < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel -mattr=+dsp < %s | FileCheck %s
 
 @g1 = common global <2 x i16> zeroinitializer, align 4
 @g0 = common global <2 x i16> zeroinitializer, align 4
diff --git a/llvm/test/CodeGen/Mips/vector-setcc.ll b/llvm/test/CodeGen/Mips/vector-setcc.ll
index f8e6d5e675202..57787c937a04f 100644
--- a/llvm/test/CodeGen/Mips/vector-setcc.ll
+++ b/llvm/test/CodeGen/Mips/vector-setcc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel < %s
+; RUN: llc -mtriple=mipsel < %s
 
 @a = common global <4 x i32> zeroinitializer, align 16
 @b = common global <4 x i32> zeroinitializer, align 16
diff --git a/llvm/test/CodeGen/Mips/vr4300-mulbranch.ll b/llvm/test/CodeGen/Mips/vr4300-mulbranch.ll
index c3f15fb6afa66..e86097633b7fa 100644
--- a/llvm/test/CodeGen/Mips/vr4300-mulbranch.ll
+++ b/llvm/test/CodeGen/Mips/vr4300-mulbranch.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mfix4300 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mfix4300 -verify-machineinstrs < %s | FileCheck %s
 
 ; Function Attrs: nounwind
 define dso_local void @fun_s(float %a) local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/Mips/vr4300-mulmul.ll b/llvm/test/CodeGen/Mips/vr4300-mulmul.ll
index 33d909c7b4369..823b7d0a2abf3 100644
--- a/llvm/test/CodeGen/Mips/vr4300-mulmul.ll
+++ b/llvm/test/CodeGen/Mips/vr4300-mulmul.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -mfix4300 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=mips -mfix4300 -verify-machineinstrs < %s | FileCheck %s
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
 define dso_local float @fun_s(float %x) local_unnamed_addr !dbg !7  {
diff --git a/llvm/test/CodeGen/Mips/weak.ll b/llvm/test/CodeGen/Mips/weak.ll
index fdc0a1d885413..021cc6b9d7265 100644
--- a/llvm/test/CodeGen/Mips/weak.ll
+++ b/llvm/test/CodeGen/Mips/weak.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips < %s | FileCheck %s
+; RUN: llc -mtriple=mips < %s | FileCheck %s
 
 @t = common global ptr null, align 4
 
diff --git a/llvm/test/CodeGen/Mips/whitespace.ll b/llvm/test/CodeGen/Mips/whitespace.ll
index 6710b73acbf1c..51dd13231fd0a 100644
--- a/llvm/test/CodeGen/Mips/whitespace.ll
+++ b/llvm/test/CodeGen/Mips/whitespace.ll
@@ -1,6 +1,6 @@
 ; Test that the instructions have the correct whitespace.
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck -strict-whitespace %s -check-prefix=16
-; RUN: llc  -march=mips -mcpu=mips32r2 < %s | FileCheck %s -strict-whitespace -check-prefix=32R2
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic < %s | FileCheck -strict-whitespace %s -check-prefix=16
+; RUN: llc  -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -strict-whitespace -check-prefix=32R2
 
 @main.L = internal unnamed_addr constant [5 x ptr] [ptr blockaddress(@main, %L1), ptr blockaddress(@main, %L2), ptr blockaddress(@main, %L3), ptr blockaddress(@main, %L4), ptr null], align 4
 @str = private unnamed_addr constant [2 x i8] c"A\00"
diff --git a/llvm/test/CodeGen/Mips/xor1.ll b/llvm/test/CodeGen/Mips/xor1.ll
index dcdfbf538ffd5..3dca266bf93d2 100644
--- a/llvm/test/CodeGen/Mips/xor1.ll
+++ b/llvm/test/CodeGen/Mips/xor1.ll
@@ -1,4 +1,4 @@
-; RUN: llc  -march=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
+; RUN: llc  -mtriple=mipsel -mattr=mips16 -relocation-model=pic -O3 < %s | FileCheck %s -check-prefix=16
 
 @x = global i32 65504, align 4
 @y = global i32 60929, align 4
diff --git a/llvm/test/CodeGen/Mips/zeroreg.ll b/llvm/test/CodeGen/Mips/zeroreg.ll
index f2abdb61503c5..91705489b7319 100644
--- a/llvm/test/CodeGen/Mips/zeroreg.ll
+++ b/llvm/test/CodeGen/Mips/zeroreg.ll
@@ -1,10 +1,10 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32-CMOV
-; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32-CMOV
-; RUN: llc < %s -march=mipsel -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32R6
-; RUN: llc < %s -march=mipsel -mcpu=mips4    -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
-; RUN: llc < %s -march=mipsel -mcpu=mips64   -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
-; RUN: llc < %s -march=mipsel -mcpu=mips64r2 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
-; RUN: llc < %s -march=mipsel -mcpu=mips64r6 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64R6
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32   -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32-CMOV
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r2 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32-CMOV
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips32r6 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,32R6
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips4    -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips64   -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips64r2 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64-CMOV
+; RUN: llc < %s -mtriple=mipsel -mcpu=mips64r6 -target-abi n64 -relocation-model=pic | FileCheck %s -check-prefixes=ALL,64R6
 
 @g1 = external global i32
 
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 80815b3ca37c0..eee31be80e982 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -204,7 +204,7 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ;
 ; SM80-LABEL: test_faddx2(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -212,22 +212,20 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    add.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_faddx2(
 ; SM80-FTZ:       {
-; SM80-FTZ-NEXT:    .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-NEXT:    .reg .f32 %f<7>;
 ; SM80-FTZ-EMPTY:
@@ -235,16 +233,14 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
 ; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
 ; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs1;
 ; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs3;
 ; SM80-FTZ-NEXT:    add.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs4;
 ; SM80-FTZ-NEXT:    add.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
@@ -311,7 +307,7 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ;
 ; SM80-LABEL: test_fsubx2(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -319,22 +315,20 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    sub.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fsubx2(
 ; SM80-FTZ:       {
-; SM80-FTZ-NEXT:    .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-NEXT:    .reg .f32 %f<7>;
 ; SM80-FTZ-EMPTY:
@@ -342,16 +336,14 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
 ; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
 ; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs1;
 ; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs3;
 ; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs4;
 ; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
@@ -418,7 +410,7 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ;
 ; SM80-LABEL: test_fmulx2(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -426,22 +418,20 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    mul.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    mul.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fmulx2(
 ; SM80-FTZ:       {
-; SM80-FTZ-NEXT:    .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-NEXT:    .reg .f32 %f<7>;
 ; SM80-FTZ-EMPTY:
@@ -449,16 +439,14 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
 ; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
 ; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs1;
 ; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs3;
 ; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs4;
 ; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
@@ -525,7 +513,7 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ;
 ; SM80-LABEL: test_fdiv(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -533,22 +521,20 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    div.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    div.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fdiv(
 ; SM80-FTZ:       {
-; SM80-FTZ-NEXT:    .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-NEXT:    .reg .f32 %f<7>;
 ; SM80-FTZ-EMPTY:
@@ -556,22 +542,20 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
 ; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
 ; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f1, %rs1;
 ; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f2, %rs3;
 ; SM80-FTZ-NEXT:    div.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %f5, %rs4;
 ; SM80-FTZ-NEXT:    div.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_fdiv(
 ; SM90:       {
-; SM90-NEXT:    .reg .b16 %rs<7>;
+; SM90-NEXT:    .reg .b16 %rs<5>;
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-NEXT:    .reg .f32 %f<7>;
 ; SM90-EMPTY:
@@ -579,16 +563,14 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
 ; SM90-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM90-NEXT:    div.rn.f32 %f3, %f2, %f1;
-; SM90-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM90-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM90-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM90-NEXT:    div.rn.f32 %f6, %f5, %f4;
-; SM90-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM90-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 3e54aaf558072..b317697210850 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -10,20 +10,18 @@ declare <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a) #0
 define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
 ; CHECK-LABEL: test_sin(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .f32 %f<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_sin_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; CHECK-NEXT:    sin.approx.f32 %f2, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; CHECK-NEXT:    sin.approx.f32 %f4, %f3;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
@@ -33,20 +31,18 @@ define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
 define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 {
 ; CHECK-LABEL: test_cos(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .f32 %f<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_cos_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; CHECK-NEXT:    cos.approx.f32 %f2, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; CHECK-NEXT:    cos.approx.f32 %f4, %f3;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
index e545d4c117791..7f29676686a26 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
@@ -22,20 +22,18 @@ define <2 x bfloat> @test_ret_const() #0 {
 define <2 x bfloat> @test_fadd_imm_0(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_fadd_imm_0(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<5>;
+; SM80-NEXT:    .reg .b16 %rs<3>;
 ; SM80-NEXT:    .reg .b32 %r<3>;
 ; SM80-NEXT:    .reg .f32 %f<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fadd_imm_0_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
-; SM80-NEXT:    add.rn.f32 %f2, %f1, 0f40000000;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
-; SM80-NEXT:    add.rn.f32 %f4, %f3, 0f3F800000;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
+; SM80-NEXT:    add.rn.f32 %f4, %f3, 0f40000000;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r2;
 ; SM80-NEXT:    ret;
 ;
@@ -84,7 +82,7 @@ define bfloat @test_fadd_imm_1(bfloat %a) #0 {
 define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-LABEL: test_fsubx2(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -92,16 +90,14 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    sub.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
@@ -122,7 +118,7 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-LABEL: test_fmulx2(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b16 %rs<5>;
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-NEXT:    .reg .f32 %f<7>;
 ; SM80-EMPTY:
@@ -130,16 +126,14 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; SM80-NEXT:    mul.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; SM80-NEXT:    mul.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
@@ -160,7 +154,7 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fdiv(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b16 %rs<5>;
 ; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-NEXT:    .reg .f32 %f<7>;
 ; CHECK-EMPTY:
@@ -168,16 +162,14 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; CHECK-NEXT:    cvt.f32.bf16 %f2, %rs3;
 ; CHECK-NEXT:    div.rn.f32 %f3, %f2, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
-; CHECK-NEXT:    cvt.f32.bf16 %f4, %rs1;
-; CHECK-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; CHECK-NEXT:    cvt.f32.bf16 %f4, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f5, %rs4;
 ; CHECK-NEXT:    div.rn.f32 %f6, %f5, %f4;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
-; CHECK-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r3, %f6, %f3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
@@ -418,15 +410,12 @@ define <2 x bfloat> @test_select_cc_bf16_f32(<2 x bfloat> %a, <2 x bfloat> %b,
 define <2 x bfloat> @test_fptrunc_2xfloat(<2 x float> %a) #0 {
 ; CHECK-LABEL: test_fptrunc_2xfloat(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .f32 %f<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.f32 {%f1, %f2}, [test_fptrunc_2xfloat_param_0];
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs1, %f2;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
-; CHECK-NEXT:    mov.b32 %r1, {%rs2, %rs1};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r1, %f2, %f1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %r = fptrunc <2 x float> %a to <2 x bfloat>
@@ -503,20 +492,18 @@ declare <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bf
 define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 {
 ; CHECK-LABEL: test_sqrt(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .f32 %f<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; CHECK-NEXT:    sqrt.rn.f32 %f2, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; CHECK-NEXT:    sqrt.rn.f32 %f4, %f3;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a)
@@ -556,7 +543,7 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 {
 define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-LABEL: test_fabs_add(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<11>;
+; SM80-NEXT:    .reg .b16 %rs<7>;
 ; SM80-NEXT:    .reg .b32 %r<6>;
 ; SM80-NEXT:    .reg .f32 %f<11>;
 ; SM80-EMPTY:
@@ -564,25 +551,21 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_1];
 ; SM80-NEXT:    ld.param.b32 %r2, [test_fabs_add_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    add.rn.f32 %f2, %f1, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; SM80-NEXT:    add.rn.f32 %f4, %f3, %f3;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r3, {%rs4, %rs3};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r3, %f4, %f2;
 ; SM80-NEXT:    abs.bf16x2 %r4, %r3;
-; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; SM80-NEXT:    cvt.f32.bf16 %f5, %rs6;
-; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f6, %rs8;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f6, %rs5;
 ; SM80-NEXT:    add.rn.f32 %f7, %f5, %f6;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs9, %f7;
-; SM80-NEXT:    cvt.f32.bf16 %f8, %rs5;
-; SM80-NEXT:    cvt.f32.bf16 %f9, %rs7;
+; SM80-NEXT:    cvt.f32.bf16 %f8, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f9, %rs6;
 ; SM80-NEXT:    add.rn.f32 %f10, %f8, %f9;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs10, %f10;
-; SM80-NEXT:    mov.b32 %r5, {%rs10, %rs9};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %f10, %f7;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM80-NEXT:    ret;
 ;
@@ -637,20 +620,18 @@ define <2 x bfloat> @test_maxnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_floor(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<5>;
+; SM80-NEXT:    .reg .b16 %rs<3>;
 ; SM80-NEXT:    .reg .b32 %r<3>;
 ; SM80-NEXT:    .reg .f32 %f<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_floor_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    cvt.rmi.f32.f32 %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; SM80-NEXT:    cvt.rmi.f32.f32 %f4, %f3;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r2;
 ; SM80-NEXT:    ret;
 ;
@@ -674,20 +655,18 @@ define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
 define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_ceil(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<5>;
+; SM80-NEXT:    .reg .b16 %rs<3>;
 ; SM80-NEXT:    .reg .b32 %r<3>;
 ; SM80-NEXT:    .reg .f32 %f<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_ceil_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    cvt.rpi.f32.f32 %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; SM80-NEXT:    cvt.rpi.f32.f32 %f4, %f3;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r2;
 ; SM80-NEXT:    ret;
 ;
@@ -711,20 +690,18 @@ define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
 define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_trunc(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<5>;
+; SM80-NEXT:    .reg .b16 %rs<3>;
 ; SM80-NEXT:    .reg .b32 %r<3>;
 ; SM80-NEXT:    .reg .f32 %f<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_trunc_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    cvt.rzi.f32.f32 %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; SM80-NEXT:    cvt.rzi.f32.f32 %f4, %f3;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r2;
 ; SM80-NEXT:    ret;
 ;
@@ -748,20 +725,18 @@ define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
 define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_rint(
 ; SM80:       {
-; SM80-NEXT:    .reg .b16 %rs<5>;
+; SM80-NEXT:    .reg .b16 %rs<3>;
 ; SM80-NEXT:    .reg .b32 %r<3>;
 ; SM80-NEXT:    .reg .f32 %f<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_rint_param_0];
 ; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; SM80-NEXT:    cvt.rni.f32.f32 %f2, %f1;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT:    cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs2;
 ; SM80-NEXT:    cvt.rni.f32.f32 %f4, %f3;
-; SM80-NEXT:    cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT:    mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r2, %f4, %f2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r2;
 ; SM80-NEXT:    ret;
 ;
@@ -786,14 +761,14 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
 ; CHECK-LABEL: test_round(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-NEXT:    .reg .f32 %f<17>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_round_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %f1, %rs1;
 ; CHECK-NEXT:    mov.b32 %r2, %f1;
 ; CHECK-NEXT:    and.b32 %r3, %r2, -2147483648;
 ; CHECK-NEXT:    or.b32 %r4, %r3, 1056964608;
@@ -806,8 +781,7 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
 ; CHECK-NEXT:    cvt.rzi.f32.f32 %f7, %f1;
 ; CHECK-NEXT:    setp.lt.f32 %p2, %f5, 0f3F000000;
 ; CHECK-NEXT:    selp.f32 %f8, %f7, %f6, %p2;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs3, %f8;
-; CHECK-NEXT:    cvt.f32.bf16 %f9, %rs1;
+; CHECK-NEXT:    cvt.f32.bf16 %f9, %rs2;
 ; CHECK-NEXT:    mov.b32 %r5, %f9;
 ; CHECK-NEXT:    and.b32 %r6, %r5, -2147483648;
 ; CHECK-NEXT:    or.b32 %r7, %r6, 1056964608;
@@ -820,8 +794,7 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
 ; CHECK-NEXT:    cvt.rzi.f32.f32 %f15, %f9;
 ; CHECK-NEXT:    setp.lt.f32 %p4, %f13, 0f3F000000;
 ; CHECK-NEXT:    selp.f32 %f16, %f15, %f14, %p4;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f16;
-; CHECK-NEXT:    mov.b32 %r8, {%rs4, %rs3};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r8, %f16, %f8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a)
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm80.ll b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
index 4e30cebfe9025..1106a062f1d56 100644
--- a/llvm/test/CodeGen/NVPTX/convert-sm80.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
@@ -1,41 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s
 ; RUN: %if ptxas-11.0 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | %ptxas-verify -arch=sm_80 %}
 
 
-; CHECK-LABEL: cvt_rn_bf16x2_f32
 define <2 x bfloat> @cvt_rn_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.bf16x2.f32
-  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rn_bf16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_bf16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_bf16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2)
+  ret <2 x bfloat> %val
 }
 
-; CHECK-LABEL: cvt_rn_relu_bf16x2_f32
 define <2 x bfloat> @cvt_rn_relu_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.relu.bf16x2.f32
-%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rn_relu_bf16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_bf16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_relu_bf16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.relu.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2)
+  ret <2 x bfloat> %val
 }
 
-; CHECK-LABEL: cvt_rz_bf16x2_f32
 define <2 x bfloat> @cvt_rz_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.bf16x2.f32
-  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rz_bf16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_bf16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_bf16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2)
+  ret <2 x bfloat> %val
 }
 
-; CHECK-LABEL: cvt_rz_relu_bf16x2_f32
 define <2 x bfloat> @cvt_rz_relu_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.relu.bf16x2.f32
-%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rz_relu_bf16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_relu_bf16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_relu_bf16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.relu.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2)
+  ret <2 x bfloat> %val
 }
 
 declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float, float)
@@ -43,40 +72,68 @@ declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float, float)
 declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float, float)
 declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float, float)
 
-; CHECK-LABEL: cvt_rn_f16x2_f32
 define <2 x half> @cvt_rn_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.f16x2.f32
-  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rn_f16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_f16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_f16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2)
+  ret <2 x half> %val
 }
 
-; CHECK-LABEL: cvt_rn_relu_f16x2_f32
 define <2 x half> @cvt_rn_relu_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.relu.f16x2.f32
-%val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rn_relu_f16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_f16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_relu_f16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.relu.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2)
+  ret <2 x half> %val
 }
 
-; CHECK-LABEL: cvt_rz_f16x2_f32
 define <2 x half> @cvt_rz_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.f16x2.f32
-  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rz_f16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_f16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_f16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2)
+  ret <2 x half> %val
 }
 
-; CHECK-LABEL: cvt_rz_relu_f16x2_f32
 define <2 x half> @cvt_rz_relu_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.relu.f16x2.f32
-%val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rz_relu_f16x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_relu_f16x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_relu_f16x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2)
+  ret <2 x half> %val
 }
 
 declare <2 x half> @llvm.nvvm.ff2f16x2.rn(float, float)
@@ -84,40 +141,64 @@ declare <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float, float)
 declare <2 x half> @llvm.nvvm.ff2f16x2.rz(float, float)
 declare <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float, float)
 
-; CHECK-LABEL: cvt_rn_bf16_f32
 define bfloat @cvt_rn_bf16_f32(float %f1) {
-
-; CHECK: cvt.rn.bf16.f32
-  %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rn_bf16_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_bf16_f32_param_0];
+; CHECK-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    ret;
+  %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1)
+  ret bfloat %val
 }
 
-; CHECK-LABEL: cvt_rn_relu_bf16_f32
 define bfloat @cvt_rn_relu_bf16_f32(float %f1) {
-
-; CHECK: cvt.rn.relu.bf16.f32
-%val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rn_relu_bf16_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_bf16_f32_param_0];
+; CHECK-NEXT:    cvt.rn.relu.bf16.f32 %rs1, %f1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    ret;
+  %val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1)
+  ret bfloat %val
 }
 
-; CHECK-LABEL: cvt_rz_bf16_f32
 define bfloat @cvt_rz_bf16_f32(float %f1) {
-
-; CHECK: cvt.rz.bf16.f32
-  %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rz_bf16_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_bf16_f32_param_0];
+; CHECK-NEXT:    cvt.rz.bf16.f32 %rs1, %f1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    ret;
+  %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1)
+  ret bfloat %val
 }
 
-; CHECK-LABEL: cvt_rz_relu_bf16_f32
 define bfloat @cvt_rz_relu_bf16_f32(float %f1) {
-
-; CHECK: cvt.rz.relu.bf16.f32
-%val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rz_relu_bf16_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_relu_bf16_f32_param_0];
+; CHECK-NEXT:    cvt.rz.relu.bf16.f32 %rs1, %f1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    ret;
+  %val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1)
+  ret bfloat %val
 }
 
 declare bfloat @llvm.nvvm.f2bf16.rn(float)
@@ -125,13 +206,58 @@ declare bfloat @llvm.nvvm.f2bf16.rn.relu(float)
 declare bfloat @llvm.nvvm.f2bf16.rz(float)
 declare bfloat @llvm.nvvm.f2bf16.rz.relu(float)
 
-; CHECK-LABEL: cvt_rna_tf32_f32
 define i32 @cvt_rna_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rna_tf32_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rna_tf32_f32_param_0];
+; CHECK-NEXT:    cvt.rna.tf32.f32 %r1, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1)
+  ret i32 %val
+}
 
-; CHECK: cvt.rna.tf32.f32
-  %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1);
+declare i32 @llvm.nvvm.f2tf32.rna(float)
 
-ret i32 %val
+
+define <2 x bfloat> @fold_ff2bf16x2(float %lo, float %hi) {
+; CHECK-LABEL: fold_ff2bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [fold_ff2bf16x2_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [fold_ff2bf16x2_param_1];
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r1, %f2, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %loh = fptrunc float %lo to bfloat
+  %hih = fptrunc float %hi to bfloat
+  %v0 = insertelement <2 x bfloat> poison, bfloat %loh, i64 0
+  %v1 = insertelement <2 x bfloat> %v0, bfloat %hih, i64 1
+  ret <2 x bfloat> %v1
 }
 
-declare i32 @llvm.nvvm.f2tf32.rna(float)
+define <2 x half> @fold_ff2f16x2(float %lo, float %hi) {
+; CHECK-LABEL: fold_ff2f16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [fold_ff2f16x2_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [fold_ff2f16x2_param_1];
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r1, %f2, %f1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %loh = fptrunc float %lo to half
+  %hih = fptrunc float %hi to half
+  %v0 = insertelement <2 x half> poison, half %loh, i64 0
+  %v1 = insertelement <2 x half> %v0, half %hih, i64 1
+  ret <2 x half> %v1
+}
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
index 8cc4548f6e85e..40771784d1b28 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -959,7 +959,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
 ; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<15>;
+; CHECK-NEXT:    .reg .b16 %rs<13>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
 ; CHECK-NEXT:    .reg .f32 %f<11>;
 ; CHECK-EMPTY:
@@ -971,40 +971,38 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs2;
 ; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-NEXT:    mov.b32 %f1, %r8;
 ; CHECK-NEXT:    add.f32 %f2, %f1, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT:    cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r9, %rs1;
 ; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-NEXT:    mov.b32 %f3, %r10;
 ; CHECK-NEXT:    add.f32 %f4, %f3, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs6, %f4;
 ; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT:    cvt.u32.u16 %r11, %rs7;
 ; CHECK-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-NEXT:    mov.b32 %f5, %r12;
 ; CHECK-NEXT:    cvt.u32.u16 %r13, %rs6;
 ; CHECK-NEXT:    shl.b32 %r14, %r13, 16;
 ; CHECK-NEXT:    mov.b32 %f6, %r14;
 ; CHECK-NEXT:    add.f32 %f7, %f5, %f6;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs8;
 ; CHECK-NEXT:    shl.b32 %r16, %r15, 16;
 ; CHECK-NEXT:    mov.b32 %f8, %r16;
 ; CHECK-NEXT:    cvt.u32.u16 %r17, %rs4;
 ; CHECK-NEXT:    shl.b32 %r18, %r17, 16;
 ; CHECK-NEXT:    mov.b32 %f9, %r18;
 ; CHECK-NEXT:    add.f32 %f10, %f8, %f9;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-NEXT:    mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r19, %f10, %f7;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r19;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<13>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<20>;
 ; CHECK-FTZ-NEXT:    .reg .f32 %f<11>;
 ; CHECK-FTZ-EMPTY:
@@ -1016,34 +1014,32 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f1, %r8;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f2, %f1, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r9, %rs1;
 ; CHECK-FTZ-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f3, %r10;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f4, %f3, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f4;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs7;
 ; CHECK-FTZ-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f5, %r12;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r13, %rs6;
 ; CHECK-FTZ-NEXT:    shl.b32 %r14, %r13, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f6, %r14;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f7, %f5, %f6;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r15, %rs8;
 ; CHECK-FTZ-NEXT:    shl.b32 %r16, %r15, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f8, %r16;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r17, %rs4;
 ; CHECK-FTZ-NEXT:    shl.b32 %r18, %r17, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f9, %r18;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f10, %f8, %f9;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-FTZ-NEXT:    mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r19, %f10, %f7;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r19;
 ; CHECK-FTZ-NEXT:    ret;
 ;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index 16219aa9da095..4d61ce1478953 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -642,7 +642,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
 ; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<11>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-NEXT:    .reg .f32 %f<9>;
 ; CHECK-EMPTY:
@@ -652,12 +652,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-NEXT:    shl.b32 %r6, %r5, 16;
 ; CHECK-NEXT:    mov.b32 %f1, %r6;
 ; CHECK-NEXT:    add.f32 %f2, %f1, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs1;
 ; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-NEXT:    mov.b32 %f3, %r8;
 ; CHECK-NEXT:    add.f32 %f4, %f3, 0f40E00000;
@@ -666,19 +666,17 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-NEXT:    mov.b32 %f5, %r10;
 ; CHECK-NEXT:    add.f32 %f6, %f5, %f3;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs8, %f6;
 ; CHECK-NEXT:    cvt.u32.u16 %r11, %rs4;
 ; CHECK-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-NEXT:    mov.b32 %f7, %r12;
 ; CHECK-NEXT:    add.f32 %f8, %f7, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-NEXT:    mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r13, %f8, %f6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
 ; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<14>;
 ; CHECK-FTZ-NEXT:    .reg .f32 %f<9>;
 ; CHECK-FTZ-EMPTY:
@@ -688,12 +686,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r6, %r5, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f1, %r6;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f2, %f1, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs1;
 ; CHECK-FTZ-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f3, %r8;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f4, %f3, 0f40E00000;
@@ -702,13 +700,11 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f5, %r10;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f6, %f5, %f3;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs8, %f6;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs4;
 ; CHECK-FTZ-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f7, %r12;
 ; CHECK-FTZ-NEXT:    add.ftz.f32 %f8, %f7, %f1;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-FTZ-NEXT:    mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r13, %f8, %f6;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r13;
 ; CHECK-FTZ-NEXT:    ret;
 ;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
index af21bada7783b..043c0fcca6371 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -694,7 +694,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)  {
 ; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<15>;
+; CHECK-NEXT:    .reg .b16 %rs<13>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
 ; CHECK-NEXT:    .reg .f32 %f<11>;
 ; CHECK-EMPTY:
@@ -706,40 +706,38 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs2;
 ; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-NEXT:    mov.b32 %f1, %r8;
 ; CHECK-NEXT:    add.rn.f32 %f2, %f1, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT:    cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r9, %rs1;
 ; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-NEXT:    mov.b32 %f3, %r10;
 ; CHECK-NEXT:    add.rn.f32 %f4, %f3, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs6, %f4;
 ; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT:    cvt.u32.u16 %r11, %rs7;
 ; CHECK-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-NEXT:    mov.b32 %f5, %r12;
 ; CHECK-NEXT:    cvt.u32.u16 %r13, %rs6;
 ; CHECK-NEXT:    shl.b32 %r14, %r13, 16;
 ; CHECK-NEXT:    mov.b32 %f6, %r14;
 ; CHECK-NEXT:    add.rn.f32 %f7, %f5, %f6;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs8;
 ; CHECK-NEXT:    shl.b32 %r16, %r15, 16;
 ; CHECK-NEXT:    mov.b32 %f8, %r16;
 ; CHECK-NEXT:    cvt.u32.u16 %r17, %rs4;
 ; CHECK-NEXT:    shl.b32 %r18, %r17, 16;
 ; CHECK-NEXT:    mov.b32 %f9, %r18;
 ; CHECK-NEXT:    add.rn.f32 %f10, %f8, %f9;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-NEXT:    mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r19, %f10, %f7;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r19;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<13>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<20>;
 ; CHECK-FTZ-NEXT:    .reg .f32 %f<11>;
 ; CHECK-FTZ-EMPTY:
@@ -751,34 +749,32 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f1, %r8;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f2, %f1, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r9, %rs1;
 ; CHECK-FTZ-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f3, %r10;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f4, %f3, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs6, %f4;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs7;
 ; CHECK-FTZ-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f5, %r12;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r13, %rs6;
 ; CHECK-FTZ-NEXT:    shl.b32 %r14, %r13, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f6, %r14;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f7, %f5, %f6;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r15, %rs8;
 ; CHECK-FTZ-NEXT:    shl.b32 %r16, %r15, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f8, %r16;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r17, %rs4;
 ; CHECK-FTZ-NEXT:    shl.b32 %r18, %r17, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f9, %r18;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f10, %f8, %f9;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-FTZ-NEXT:    mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r19, %f10, %f7;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r19;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -1637,7 +1633,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)  {
 ; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<11>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-NEXT:    .reg .f32 %f<9>;
 ; CHECK-EMPTY:
@@ -1647,12 +1643,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-NEXT:    shl.b32 %r6, %r5, 16;
 ; CHECK-NEXT:    mov.b32 %f1, %r6;
 ; CHECK-NEXT:    add.rn.f32 %f2, %f1, 0f40E00000;
 ; CHECK-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs1;
 ; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-NEXT:    mov.b32 %f3, %r8;
 ; CHECK-NEXT:    add.rn.f32 %f4, %f3, 0f40E00000;
@@ -1661,19 +1657,17 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-NEXT:    mov.b32 %f5, %r10;
 ; CHECK-NEXT:    add.rn.f32 %f6, %f5, %f3;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs8, %f6;
 ; CHECK-NEXT:    cvt.u32.u16 %r11, %rs4;
 ; CHECK-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-NEXT:    mov.b32 %f7, %r12;
 ; CHECK-NEXT:    add.rn.f32 %f8, %f7, %f1;
-; CHECK-NEXT:    cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-NEXT:    mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r13, %f8, %f6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
 ; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<14>;
 ; CHECK-FTZ-NEXT:    .reg .f32 %f<9>;
 ; CHECK-FTZ-EMPTY:
@@ -1683,12 +1677,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r6, %r5, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f1, %r6;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f2, %f1, 0f40E00000;
 ; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT:    cvt.u32.u16 %r7, %rs1;
 ; CHECK-FTZ-NEXT:    shl.b32 %r8, %r7, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f3, %r8;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f4, %f3, 0f40E00000;
@@ -1697,13 +1691,11 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    shl.b32 %r10, %r9, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f5, %r10;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f6, %f5, %f3;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs8, %f6;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r11, %rs4;
 ; CHECK-FTZ-NEXT:    shl.b32 %r12, %r11, 16;
 ; CHECK-FTZ-NEXT:    mov.b32 %f7, %r12;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f32 %f8, %f7, %f1;
-; CHECK-FTZ-NEXT:    cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-FTZ-NEXT:    mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r13, %f8, %f6;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r13;
 ; CHECK-FTZ-NEXT:    ret;
 ;
diff --git a/llvm/test/CodeGen/NVPTX/i1-icmp.ll b/llvm/test/CodeGen/NVPTX/i1-icmp.ll
new file mode 100644
index 0000000000000..b189f5ff8bca8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/i1-icmp.ll
@@ -0,0 +1,317 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
+
+target triple = "nvptx-nvidia-cuda"
+
+define i32 @icmp_i1_eq(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_eq(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_eq_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_eq_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    xor.pred %p3, %p1, %p2;
+; CHECK-NEXT:    @%p3 bra $L__BB0_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB0_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp eq i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_ne(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_ne(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<5>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_ne_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_ne_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    xor.pred %p3, %p1, %p2;
+; CHECK-NEXT:    not.pred %p4, %p3;
+; CHECK-NEXT:    @%p4 bra $L__BB1_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB1_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp ne i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_sgt(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_sgt(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_sgt_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_sgt_param_1];
+; CHECK-NEXT:    setp.lt.s32 %p2, %r2, 2;
+; CHECK-NEXT:    or.pred %p3, %p1, %p2;
+; CHECK-NEXT:    @%p3 bra $L__BB2_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB2_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp sgt i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_slt(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_slt(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_slt_param_0];
+; CHECK-NEXT:    setp.lt.s32 %p1, %r1, 2;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_slt_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    or.pred %p3, %p2, %p1;
+; CHECK-NEXT:    @%p3 bra $L__BB3_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB3_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp slt i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_sge(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_sge(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_sge_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_sge_param_1];
+; CHECK-NEXT:    setp.lt.s32 %p2, %r2, 2;
+; CHECK-NEXT:    and.pred %p3, %p1, %p2;
+; CHECK-NEXT:    @%p3 bra $L__BB4_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB4_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp sge i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_sle(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_sle(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_sle_param_0];
+; CHECK-NEXT:    setp.lt.s32 %p1, %r1, 2;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_sle_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    and.pred %p3, %p2, %p1;
+; CHECK-NEXT:    @%p3 bra $L__BB5_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB5_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp sle i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_uge(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_uge(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_uge_param_0];
+; CHECK-NEXT:    setp.lt.s32 %p1, %r1, 2;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_uge_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    and.pred %p3, %p2, %p1;
+; CHECK-NEXT:    @%p3 bra $L__BB6_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB6_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp uge i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_ugt(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_ugt(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_ugt_param_0];
+; CHECK-NEXT:    setp.lt.s32 %p1, %r1, 2;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_ugt_param_1];
+; CHECK-NEXT:    setp.gt.s32 %p2, %r2, 1;
+; CHECK-NEXT:    or.pred %p3, %p2, %p1;
+; CHECK-NEXT:    @%p3 bra $L__BB7_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB7_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp ugt i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_ule(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_ule(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_ule_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_ule_param_1];
+; CHECK-NEXT:    setp.lt.s32 %p2, %r2, 2;
+; CHECK-NEXT:    and.pred %p3, %p1, %p2;
+; CHECK-NEXT:    @%p3 bra $L__BB8_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB8_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp ule i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
+define i32 @icmp_i1_ult(i32 %a, i32 %b) {
+; CHECK-LABEL: icmp_i1_ult(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [icmp_i1_ult_param_0];
+; CHECK-NEXT:    setp.gt.s32 %p1, %r1, 1;
+; CHECK-NEXT:    ld.param.u32 %r2, [icmp_i1_ult_param_1];
+; CHECK-NEXT:    setp.lt.s32 %p2, %r2, 2;
+; CHECK-NEXT:    or.pred %p3, %p1, %p2;
+; CHECK-NEXT:    @%p3 bra $L__BB9_2;
+; CHECK-NEXT:  // %bb.1: // %bb1
+; CHECK-NEXT:    mov.b32 %r4, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+; CHECK-NEXT:  $L__BB9_2: // %bb2
+; CHECK-NEXT:    mov.b32 %r3, 127;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %p1 = icmp sgt i32 %a, 1
+  %p2 = icmp sgt i32 %b, 1
+  %c = icmp ult i1 %p1, %p2
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 127
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/aix-cpu-version-multifunction.ll b/llvm/test/CodeGen/PowerPC/aix-cpu-version-multifunction.ll
new file mode 100644
index 0000000000000..2cdb864f450bd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-cpu-version-multifunction.ll
@@ -0,0 +1,30 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff  < %s | FileCheck %s
+
+; For the .machine directive emitted on AIX, the "target-cpu" attribute that is
+; the newest will be used as the CPU for the module (in this case, PWR10).
+
+; CHECK:      .file "file.c"
+; CHECK-NEXT: .csect ..text..[PR],5
+; CHECK-NEXT: .rename ..text..[PR],""
+; CHECK-NEXT: .machine "PWR10"
+; CHECK-NOT:  .machine "PWR8"
+
+source_filename = "file.c"
+
+define dso_local signext i32 @testFunc1() #0 {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, ptr %retval, align 4
+  ret i32 0
+}
+
+define dso_local signext i32 @testFunc2() #1 {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, ptr %retval, align 4
+  ret i32 0
+}
+
+attributes #0 = { "target-cpu" = "pwr8" }
+attributes #1 = { "target-cpu" = "pwr10" }
+
diff --git a/llvm/test/CodeGen/PowerPC/aix-cpu-version.ll b/llvm/test/CodeGen/PowerPC/aix-cpu-version.ll
new file mode 100644
index 0000000000000..630a09cc58dc6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-cpu-version.ll
@@ -0,0 +1,17 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff  < %s | FileCheck %s
+
+; CHECK:      .file "1.c"
+; CHECK-NEXT: .csect ..text..[PR],5
+; CHECK-NEXT: .rename ..text..[PR],""
+; CHECK-NEXT: .machine "PWR8"
+
+source_filename = "1.c"
+
+define dso_local signext i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, ptr %retval, align 4
+  ret i32 0
+}
+
+attributes #0 = {"target-cpu"="pwr8"}
diff --git a/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll b/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
index 173c58567e40b..0cca7eab91047 100644
--- a/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
@@ -68,8 +68,7 @@ declare extern_weak void @foo_ext_weak(ptr)
 ; CHECKSYM-NEXT:     Value (SymbolTableIndex): 0x0
 ; CHECKSYM-NEXT:     Section: N_DEBUG
 ; CHECKSYM-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; CHECKSYM32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; CHECKSYM64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; CHECKSYM-NEXT:     CPU Version ID: TCPU_COM (0x3)
 ; CHECKSYM-NEXT:     StorageClass: C_FILE (0x67)
 ; CHECKSYM-NEXT:     NumberOfAuxEntries: 2
 ; CHECKSYM:        Symbol {
diff --git a/llvm/test/CodeGen/PowerPC/aix-extern.ll b/llvm/test/CodeGen/PowerPC/aix-extern.ll
index ff2a803608807..71b17ad35c70d 100644
--- a/llvm/test/CodeGen/PowerPC/aix-extern.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-extern.ll
@@ -91,8 +91,7 @@ declare i32 @bar_extern(ptr)
 ; CHECKSYM-NEXT:     Value (SymbolTableIndex): 0x0
 ; CHECKSYM-NEXT:     Section: N_DEBUG
 ; CHECKSYM-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; CHECKSYM32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; CHECKSYM64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; CHECKSYM-NEXT:     CPU Version ID: TCPU_COM (0x3)
 ; CHECKSYM-NEXT:     StorageClass: C_FILE (0x67)
 ; CHECKSYM-NEXT:     NumberOfAuxEntries: 2
 ; CHECKSYM:        Symbol {
diff --git a/llvm/test/CodeGen/PowerPC/aix-filename-c.ll b/llvm/test/CodeGen/PowerPC/aix-filename-c.ll
index c4202a0c58cee..1fec0665c4ca8 100644
--- a/llvm/test/CodeGen/PowerPC/aix-filename-c.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-filename-c.ll
@@ -1,12 +1,57 @@
-; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
-; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ,OBJ32 %s
-; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -filetype=obj -o %t64.o < %s
-; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ,OBJ64 %s
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr9  < %s | FileCheck --check-prefixes=ASM %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr9 -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ32 %s
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr9 -filetype=obj -o %t64.o < %s
+; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ64 %s
 
 source_filename = "1.c"
 
-; OBJ: Name: .file
-; OBJ: Source Language ID: TB_C (0x0)
-; OBJ32: CPU Version ID: TCPU_COM (0x3)
-; OBJ64: CPU Version ID: TCPU_PPC64 (0x2)
-; OBJ: Name: 1.c
+; ASM:      .file   "1.c",,"LLVM{{.*}}"
+; ASM-NEXT: .csect ..text..[PR],5
+; ASM-NEXT: .rename	..text..[PR],""
+; ASM-NEXT: .machine   "PWR9"
+
+; OBJ32:      Symbol {
+; OBJ32-NEXT:   Index: 0
+; OBJ32-NEXT:   Name: .file
+; OBJ32-NEXT:   Value (SymbolTableIndex): 0x0
+; OBJ32-NEXT:   Section: N_DEBUG
+; OBJ32-NEXT:   Source Language ID: TB_C (0x0)
+; OBJ32-NEXT:   CPU Version ID: TCPU_PWR9 (0x1A)
+; OBJ32-NEXT:   StorageClass: C_FILE (0x67)
+; OBJ32-NEXT:   NumberOfAuxEntries: 2
+; OBJ32-NEXT:   File Auxiliary Entry {
+; OBJ32-NEXT:     Index: 1
+; OBJ32-NEXT:     Name: 1.c
+; OBJ32-NEXT:     Type: XFT_FN (0x0)
+; OBJ32-NEXT:   }
+; OBJ32-NEXT:   File Auxiliary Entry {
+; OBJ32-NEXT:     Index: 2
+; OBJ32-NEXT:     Name: LLVM
+; OBJ32-NEXT:     Type: XFT_CV (0x2)
+; OBJ32-NEXT:   }
+; OBJ32-NEXT: }
+
+; OBJ64:      Symbol {
+; OBJ64-NEXT:   Index: 0
+; OBJ64-NEXT:   Name: .file
+; OBJ64-NEXT:   Value (SymbolTableIndex): 0x0
+; OBJ64-NEXT:   Section: N_DEBUG
+; OBJ64-NEXT:   Source Language ID: TB_C (0x0)
+; OBJ64-NEXT:   CPU Version ID: TCPU_PWR9 (0x1A)
+; OBJ64-NEXT:   StorageClass: C_FILE (0x67)
+; OBJ64-NEXT:   NumberOfAuxEntries: 2
+; OBJ64-NEXT:   File Auxiliary Entry {
+; OBJ64-NEXT:     Index: 1
+; OBJ64-NEXT:     Name: 1.c
+; OBJ64-NEXT:     Type: XFT_FN (0x0)
+; OBJ64-NEXT:     Auxiliary Type: AUX_FILE (0xFC)
+; OBJ64-NEXT:   }
+; OBJ64-NEXT:   File Auxiliary Entry {
+; OBJ64-NEXT:     Index: 2
+; OBJ64-NEXT:     Name: LLVM
+; OBJ64-NEXT:     Type: XFT_CV (0x2)
+; OBJ64-NEXT:     Auxiliary Type: AUX_FILE (0xFC)
+; OBJ64-NEXT:   }
+; OBJ64-NEXT: }
diff --git a/llvm/test/CodeGen/PowerPC/aix-filename-cpp.ll b/llvm/test/CodeGen/PowerPC/aix-filename-cpp.ll
index 802281b6c1eaa..873619d20cd2c 100644
--- a/llvm/test/CodeGen/PowerPC/aix-filename-cpp.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-filename-cpp.ll
@@ -1,12 +1,11 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
-; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ,OBJ32 %s
+; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ %s
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -filetype=obj -o %t64.o < %s
-; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ,OBJ64 %s
+; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ %s
 
 source_filename = "1.cpp"
 
 ; OBJ: Name: .file
 ; OBJ: Source Language ID: TB_CPLUSPLUS (0x9)
-; OBJ32: CPU Version ID: TCPU_COM (0x3)
-; OBJ64: CPU Version ID: TCPU_PPC64 (0x2)
+; OBJ: CPU Version ID: TCPU_PWR7 (0x18)
 ; OBJ: Name: 1.cpp
diff --git a/llvm/test/CodeGen/PowerPC/aix-filename-f.ll b/llvm/test/CodeGen/PowerPC/aix-filename-f.ll
index 99036bde702d6..1167148d21a7f 100644
--- a/llvm/test/CodeGen/PowerPC/aix-filename-f.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-filename-f.ll
@@ -1,12 +1,11 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
-; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ,OBJ32 %s
+; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefixes=OBJ %s
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -filetype=obj -o %t64.o < %s
-; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ,OBJ64 %s
+; RUN: llvm-readobj --symbols %t64.o | FileCheck --check-prefixes=OBJ %s
 
 source_filename = "1.f95"
 
 ; OBJ: Name: .file
 ; OBJ: Source Language ID: TB_Fortran (0x1)
-; OBJ32: CPU Version ID: TCPU_COM (0x3)
-; OBJ64: CPU Version ID: TCPU_PPC64 (0x2)
+; OBJ: CPU Version ID: TCPU_PWR7 (0x18)
 ; OBJ: Name: 1.f95
diff --git a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
index 4cca1b4d6f7ba..50221acc2b3ad 100644
--- a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
@@ -17,7 +17,7 @@ entry:
 ; CHECK-NEXT:     Value (SymbolTableIndex): 0x0
 ; CHECK-NEXT:     Section: N_DEBUG
 ; CHECK-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; CHECK-NEXT:     CPU Version ID: TCPU_COM (0x3)
+; CHECK-NEXT:     CPU Version ID: TCPU_PWR7 (0x18)
 ; CHECK-NEXT:     StorageClass: C_FILE (0x67)
 ; CHECK-NEXT:     NumberOfAuxEntries: 2
 ; CHECK-NEXT:     File Auxiliary Entry {
diff --git a/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll b/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll
index 50677f36e3f7a..e2a3febd6c976 100644
--- a/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll
@@ -44,8 +44,7 @@ declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg)
 ; CHECKSYM-NEXT:     Value (SymbolTableIndex): 0x0
 ; CHECKSYM-NEXT:     Section: N_DEBUG
 ; CHECKSYM-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; CHECKSYM32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; CHECKSYM64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; CHECKSYM-NEXT:     CPU Version ID: TCPU_COM (0x3)
 ; CHECKSYM-NEXT:     StorageClass: C_FILE (0x67)
 ; CHECKSYM-NEXT:     NumberOfAuxEntries: 2
 ; CHECKSYM:   }
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-variables.ll b/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-variables.ll
index f9a1a61617761..3c84aa1225d2e 100644
--- a/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-variables.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-variables.ll
@@ -70,7 +70,7 @@
 ; SYMS-NEXT:     Value (SymbolTableIndex): 0x0
 ; SYMS-NEXT:     Section: N_DEBUG
 ; SYMS-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; SYMS-NEXT:     CPU Version ID: TCPU_COM (0x3)
+; SYMS-NEXT:     CPU Version ID: TCPU_PWR7 (0x18)
 ; SYMS-NEXT:     StorageClass: C_FILE (0x67)
 ; SYMS-NEXT:     NumberOfAuxEntries: 2
 ; SYMS:         Symbol {
diff --git a/llvm/test/CodeGen/PowerPC/aix-weak.ll b/llvm/test/CodeGen/PowerPC/aix-weak.ll
index 7bf80ad19e9e0..3b2917ec97284 100644
--- a/llvm/test/CodeGen/PowerPC/aix-weak.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-weak.ll
@@ -104,8 +104,7 @@ entry:
 ; CHECKSYM-NEXT:     Value (SymbolTableIndex): 0x0
 ; CHECKSYM-NEXT:     Section: N_DEBUG
 ; CHECKSYM-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; CHECKSYM32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; CHECKSYM64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; CHECKSYM-NEXT:     CPU Version ID: TCPU_COM (0x3)
 ; CHECKSYM-NEXT:     StorageClass: C_FILE (0x67)
 ; CHECKSYM-NEXT:     NumberOfAuxEntries: 2
 ; CHECKSYM:        Symbol {
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
index de937386b8b7d..10b04b570fa32 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
@@ -45,8 +45,10 @@
 
 ; CHECK-NOT: .toc
 
-; CHECK:  .file
-; CHECK-NEXT:      .csect ..text..[PR],5
+; CHECK:      .file
+; CHECK-NEXT: .csect ..text..[PR],5
+; CHECK-NEXT: .rename ..text..[PR],""
+; CHECK-NEXT: .machine "PWR7"
 
 ; CHECK:      .csect .data[RW],5
 ; CHECK-NEXT: .globl  ivar
@@ -212,8 +214,7 @@
 ; SYMS-NEXT:     Value (SymbolTableIndex): 0x0
 ; SYMS-NEXT:     Section: N_DEBUG
 ; SYMS-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; SYMS32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; SYMS64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; SYMS-NEXT:     CPU Version ID: TCPU_PWR7 (0x18)
 ; SYMS-NEXT:     StorageClass: C_FILE (0x67)
 ; SYMS-NEXT:     NumberOfAuxEntries: 2
 ; SYMS-NEXT:     File Auxiliary Entry {
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
index 5462240846994..edc74e393108f 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
@@ -4,14 +4,14 @@
 ; tests for XCOFF object files.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -xcoff-traceback-table=false \
-; RUN:     -mtriple powerpc-ibm-aix-xcoff  -data-sections=false -ppc-merge-string-pool=false \
+; RUN:     -mtriple powerpc-ibm-aix-xcoff  -data-sections=false \
 ; RUN:     -global-merge-all-const=false < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -xcoff-traceback-table=false \
-; RUN:     -mtriple powerpc64-ibm-aix-xcoff -data-sections=false -ppc-merge-string-pool=false \
+; RUN:     -mtriple powerpc64-ibm-aix-xcoff -data-sections=false \
 ; RUN:     -global-merge-all-const=false < %s | FileCheck %s
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
-; RUN:     -xcoff-traceback-table=false -data-sections=false -ppc-merge-string-pool=false \
+; RUN:     -xcoff-traceback-table=false -data-sections=false \
 ; RUN:     -global-merge-all-const=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
index 950d65b7e2913..77dd56f803117 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
@@ -163,8 +163,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     Value (SymbolTableIndex): 0x0
 ; SYM-NEXT:     Section: N_DEBUG
 ; SYM-NEXT:     Source Language ID: TB_CPLUSPLUS (0x9)
-; SYM32-NEXT:   CPU Version ID: TCPU_COM (0x3)
-; SYM64-NEXT:   CPU Version ID: TCPU_PPC64 (0x2)
+; SYM-NEXT:     CPU Version ID: TCPU_COM (0x3)
 ; SYM-NEXT:     StorageClass: C_FILE (0x67)
 ; SYM-NEXT:     NumberOfAuxEntries: 2
 ; SYM-NEXT:     File Auxiliary Entry {
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
index 0fa47373964a0..9b0a3fe0a716f 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
@@ -1,16 +1,16 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -data-sections=false -ppc-merge-string-pool=false < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -data-sections=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff -data-sections=false -ppc-merge-string-pool=false < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff -data-sections=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -data-sections=false -ppc-merge-string-pool=false \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc-ibm-aix-xcoff -data-sections=false \
 ; RUN:   -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --section-headers --file-header %t.o | \
 ; RUN:   FileCheck --check-prefixes=OBJ,OBJ32 %s
 ; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefixes=SYMS,SYMS32 %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=DIS %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff -data-sections=false -ppc-merge-string-pool=false \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple powerpc64-ibm-aix-xcoff -data-sections=false \
 ; RUN:   -filetype=obj -o %t64.o < %s
 ; RUN: llvm-readobj --section-headers --file-header %t64.o | \
 ; RUN:   FileCheck --check-prefixes=OBJ,OBJ64 %s
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-used-with-stringpool.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-used-with-stringpool.ll
index bbcba59e2e33a..5e4784ac2904c 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-used-with-stringpool.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-used-with-stringpool.ll
@@ -1,4 +1,4 @@
-;; Test that the string pooling pass does not pool globals that are
+;; Test that the global merge pass does not pool globals that are
 ;; in llvm.used or in llvm.compiler.used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple powerpc-ibm-aix-xcoff \
diff --git a/llvm/test/CodeGen/PowerPC/data-align.ll b/llvm/test/CodeGen/PowerPC/data-align.ll
new file mode 100644
index 0000000000000..bfedec139369c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/data-align.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=powerpc-unknown-linux | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux | FileCheck %s
+
+; CHECK:      .set .Li8,
+; CHECK-NEXT:  .size	.Li8, 1
+@i8 = private constant i8 42
+
+; CHECK:      .set .Li16,
+; CHECK-NEXT: .size	.Li16, 2
+@i16 = private constant i16 42
+
+; CHECK:      .set .Li32,
+; CHECK-NEXT: .size	.Li32, 4
+@i32 = private constant i32 42
+
+; CHECK:      .set .Li64,
+; CHECK-NEXT: .size	.Li64, 8
+@i64 = private constant i64 42
+
+; CHECK:        .set .Li128,
+; CHECK-NEXT:	.size	.Li128, 16
+@i128 = private constant i128 42
+
diff --git a/llvm/test/CodeGen/PowerPC/frameindex-negative-offset.ll b/llvm/test/CodeGen/PowerPC/frameindex-negative-offset.ll
new file mode 100644
index 0000000000000..c2208bba31aef
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/frameindex-negative-offset.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu < %s | FileCheck %s
+
+define ptr @test() nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stwu 1, -16(1)
+; CHECK-NEXT:    addi 3, 1, 10
+; CHECK-NEXT:    addi 1, 1, 16
+; CHECK-NEXT:    blr
+  %alloca = alloca i32
+  %gep = getelementptr i8, ptr %alloca, i32 -2
+  ret ptr %gep
+}
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll
index 03a830e087d26..10179bba136f9 100644
--- a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll
@@ -4,7 +4,7 @@
 @id = private unnamed_addr constant [4 x i8] c"@id\00", align 1
 @id2 = private unnamed_addr constant [5 x i8] c"@id2\00", align 1
 
-; Higher-aligned dummy to make sure it is first in the string pool.
+; Higher-aligned dummy to make sure it is first in the global merge pool.
 @dummy = private unnamed_addr constant [1 x i32] [i32 42], align 4
 
 define ptr @test1() personality ptr @__gnu_objc_personality_v0 {
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
index a726e7741cf0f..aa0b441646fd3 100644
--- a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
@@ -258,8 +258,8 @@ entry:
   ret void
 }
 
-; Check the contents of the TLS data and the __ModuleStringPool structure to
-; check that TLS data has been skipped during string pool merging.
+; Check the contents of the TLS data and the _MergedGlobals structure to
+; check that TLS data has been skipped during global merge.
 
 ; CHECK64: 	.csect a[TL],2
 ; CHECK64-NEXT:	.lglobl	a[TL]
diff --git a/llvm/test/CodeGen/PowerPC/pr116071.ll b/llvm/test/CodeGen/PowerPC/pr116071.ll
new file mode 100644
index 0000000000000..29f11fc1d3a63
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr116071.ll
@@ -0,0 +1,26 @@
+; RUN: llc -disable-ppc-vsx-fma-mutation=false -mcpu=pwr10 -verify-machineinstrs \
+; RUN:   -ppc-asm-full-reg-names -mtriple powerpc64-ibm-aix7.2.0.0 < %s | FileCheck %s 
+
+target datalayout = "E-m:a-Fi64-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+
+define void @initial(<2 x double> %0){
+entry:
+  %1 = fmul <2 x double> %0, zeroinitializer
+  br label %for.cond251.preheader.lr.ph
+
+for.cond251.preheader.lr.ph:                      ; preds = %for.cond251.preheader.lr.ph, %entry
+  %2 = phi double [ %3, %for.cond251.preheader.lr.ph ], [ 0.000000e+00, %entry ]
+  %3 = phi double [ %7, %for.cond251.preheader.lr.ph ], [ 0.000000e+00, %entry ]
+  %add737 = fadd double %3, %2
+  %4 = insertelement <2 x double> zeroinitializer, double %add737, i64 0
+  %5 = fmul contract <2 x double> %4, zeroinitializer
+  %6 = fadd contract <2 x double> %1, %5
+  %7 = extractelement <2 x double> %6, i64 0
+  br label %for.cond251.preheader.lr.ph
+}
+
+; CHECK:        xsadddp f4, f3, f4
+; CHECK-NEXT:   xxmrghd vs5, vs4, vs2
+; CHECK-NEXT:   fmr f4, f3
+; CHECK-NEXT:   xvmaddmdp vs5, vs0, vs1
+; CHECK-NEXT:   fmr f3, f5
diff --git a/llvm/test/CodeGen/PowerPC/pr118695.ll b/llvm/test/CodeGen/PowerPC/pr118695.ll
new file mode 100644
index 0000000000000..719a5d38b02c4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr118695.ll
@@ -0,0 +1,17 @@
+; RUN: llc < %s -verify-machineinstrs -mtriple=powerpc-aix- -mcpu=pwr10 | FileCheck %s
+
+; CHECK:      # %bb.0:                                # %bb
+; CHECK-NEXT:   lwz 3, L..C0(2)                         # @dvar
+; CHECK-NEXT:   plxv 0, -152758(3), 0
+; CHECK-NEXT:   stxv 0, 0(3)
+; CHECK-NEXT:   blr
+
+%0 = type <{ double }>
+@dvar = external global [2352637 x %0]
+
+define void @Test() {
+bb:
+	%i9 = load <2 x double>, ptr getelementptr inbounds (i8, ptr @dvar, i64 -152758), align 8
+	store <2 x double> %i9, ptr @dvar, align 8
+	ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/pr38087.ll b/llvm/test/CodeGen/PowerPC/pr38087.ll
index 1216fa9cf8f26..933bf12cddaa6 100644
--- a/llvm/test/CodeGen/PowerPC/pr38087.ll
+++ b/llvm/test/CodeGen/PowerPC/pr38087.ll
@@ -11,9 +11,9 @@ declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) #0
 define void @draw_llvm_vs_variant0(<4 x float> %x) {
 ; CHECK-LABEL: draw_llvm_vs_variant0:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lxsd v3, 0(r3)
-; CHECK-NEXT:    vmrghh v3, v3, v3
+; CHECK-NEXT:    lxsihzx v3, 0, r3
 ; CHECK-NEXT:    vextsh2w v3, v3
+; CHECK-NEXT:    xxmrghw v3, v3, v3
 ; CHECK-NEXT:    xvcvsxwsp vs0, v3
 ; CHECK-NEXT:    xxspltw vs0, vs0, 2
 ; CHECK-NEXT:    xvmaddasp vs0, v2, v2
diff --git a/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll b/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
index fd5f26ba35742..c0f3b60122521 100644
--- a/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
+++ b/llvm/test/CodeGen/PowerPC/saddo-ssubo.ll
@@ -129,12 +129,11 @@ entry:
 define i1 @test_ssubo_i32(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_ssubo_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub 5, 3, 4
-; CHECK-NEXT:    cmpwi 1, 4, 0
-; CHECK-NEXT:    cmpw 5, 3
-; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:    creqv 20, 5, 0
-; CHECK-NEXT:    isel 3, 0, 3, 20
+; CHECK-NEXT:    xor 5, 4, 3
+; CHECK-NEXT:    sub 4, 3, 4
+; CHECK-NEXT:    xor 3, 4, 3
+; CHECK-NEXT:    and 3, 5, 3
+; CHECK-NEXT:    srwi 3, 3, 31
 ; CHECK-NEXT:    blr
 entry:
   %res = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 %b) nounwind
@@ -145,12 +144,11 @@ entry:
 define i1 @test_ssubo_i64(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_ssubo_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sub 5, 3, 4
-; CHECK-NEXT:    cmpdi 1, 4, 0
-; CHECK-NEXT:    cmpd 5, 3
-; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:    creqv 20, 5, 0
-; CHECK-NEXT:    isel 3, 0, 3, 20
+; CHECK-NEXT:    xor 5, 4, 3
+; CHECK-NEXT:    sub 4, 3, 4
+; CHECK-NEXT:    xor 3, 4, 3
+; CHECK-NEXT:    and 3, 5, 3
+; CHECK-NEXT:    rldicl 3, 3, 1, 63
 ; CHECK-NEXT:    blr
 entry:
   %res = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 %a, i64 %b) nounwind
diff --git a/llvm/test/CodeGen/PowerPC/scalar_vector_test_5.ll b/llvm/test/CodeGen/PowerPC/scalar_vector_test_5.ll
index c1059be946a5f..b6799c8a88e0c 100644
--- a/llvm/test/CodeGen/PowerPC/scalar_vector_test_5.ll
+++ b/llvm/test/CodeGen/PowerPC/scalar_vector_test_5.ll
@@ -16,10 +16,8 @@ define i8 @scalar_to_vector_half(ptr nocapture readonly %ad) {
 ;
 ; P9BE-LABEL: scalar_to_vector_half:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    lxsihzx v2, 0, r3
-; P9BE-NEXT:    li r3, 0
-; P9BE-NEXT:    vsplth v2, v2, 3
-; P9BE-NEXT:    vextublx r3, r3, v2
+; P9BE-NEXT:    lhz r3, 0(r3)
+; P9BE-NEXT:    srwi r3, r3, 24
 ; P9BE-NEXT:    blr
 ;
 ; P8LE-LABEL: scalar_to_vector_half:
@@ -30,10 +28,7 @@ define i8 @scalar_to_vector_half(ptr nocapture readonly %ad) {
 ; P8BE-LABEL: scalar_to_vector_half:
 ; P8BE:       # %bb.0: # %entry
 ; P8BE-NEXT:    lhz r3, 0(r3)
-; P8BE-NEXT:    sldi r3, r3, 48
-; P8BE-NEXT:    mtfprd f0, r3
-; P8BE-NEXT:    mffprd r3, f0
-; P8BE-NEXT:    rldicl r3, r3, 8, 56
+; P8BE-NEXT:    srwi r3, r3, 24
 ; P8BE-NEXT:    blr
 entry:
     %0 = load <2 x i8>, ptr %ad, align 1
diff --git a/llvm/test/CodeGen/PowerPC/ssubo-32.ll b/llvm/test/CodeGen/PowerPC/ssubo-32.ll
new file mode 100644
index 0000000000000..488d1e26fa36e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ssubo-32.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-aix | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-aix | FileCheck %s
+
+define i1 @subovfi_i32(i32 noundef %a, i32 noundef %b, ptr %c)  {
+; CHECK-LABEL: subovfi_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xor 6, 4, 3
+; CHECK-NEXT:    sub 4, 3, 4
+; CHECK-NEXT:    xor 3, 4, 3
+; CHECK-NEXT:    stw 4, 0(5)
+; CHECK-NEXT:    and 3, 6, 3
+; CHECK-NEXT:    srwi 3, 3, 31
+; CHECK-NEXT:    blr
+entry:
+  %0 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 %b)
+  %1 = extractvalue { i32, i1 } %0, 1
+  %2 = extractvalue { i32, i1 } %0, 0
+  store i32 %2, ptr %c, align 4
+  ret i1 %1
+}
+
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32)
+
diff --git a/llvm/test/CodeGen/PowerPC/vcmp-setbc-quad.ll b/llvm/test/CodeGen/PowerPC/vcmp-setbc-quad.ll
new file mode 100644
index 0000000000000..4c8d34895f6b2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vcmp-setbc-quad.ll
@@ -0,0 +1,15 @@
+; RUN: llc -mcpu=pwr10 -mtriple=powerpc64-ibm-aix -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -mcpu=pwr10 -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+define range(i64 -2147483648, 2147483648) i64 @cmpgt(<1 x i128> noundef %a, <1 x i128> noundef %b) local_unnamed_addr {
+; CHECK: vcmpgtuq. v2, v3, v2
+; CHECK: setbc r3, 4*cr6+lt
+entry:
+  %0 = tail call i32 @llvm.ppc.altivec.vcmpgtuq.p(i32 2, <1 x i128> %b, <1 x i128> %a)
+  %conv = sext i32 %0 to i64
+  ret i64 %conv
+}
+
+declare i32 @llvm.ppc.altivec.vcmpgtuq.p(i32, <1 x i128>, <1 x i128>)
diff --git a/llvm/test/CodeGen/PowerPC/vcmp-setbc.ll b/llvm/test/CodeGen/PowerPC/vcmp-setbc.ll
new file mode 100644
index 0000000000000..2c9088b61b034
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vcmp-setbc.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mcpu=pwr10 -mtriple=powerpc64-ibm-aix -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -mcpu=pwr10 -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -mcpu=pwr10 -mtriple=powerpc-ibm-aix -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+define signext i32 @cmpgtw(<4 x i32> noundef %a, <4 x i32> noundef %b) local_unnamed_addr {
+; CHECK: vcmpgtsw. v2, v2, v3
+; CHECK: setbc r3, 4*cr6+lt
+entry:
+  %0 = tail call i32 @llvm.ppc.altivec.vcmpgtsw.p(i32 2, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define signext i32 @cmpanynew(<4 x i32> noundef %a, <4 x i32> noundef %b) local_unnamed_addr {
+; CHECK: vcmpequw. v2, v2, v3
+; CHECK: setbcr r3, 4*cr6+lt
+entry:
+  %0 = tail call i32 @llvm.ppc.altivec.vcmpequw.p(i32 3, <4 x i32> %a, <4 x i32> %b)
+  ret i32 %0
+}
+
+define signext i32 @cmpallneh(<8 x i16> noundef %a, <8 x i16> noundef %b) local_unnamed_addr {
+; CHECK: vcmpequh. v2, v2, v3
+; CHECK: setbc r3, 4*cr6+eq
+entry:
+  %0 = tail call i32 @llvm.ppc.altivec.vcmpequh.p(i32 0, <8 x i16> %a, <8 x i16> %b)
+  ret i32 %0
+}
+
+define signext i32 @cmpeqb(<16 x i8> noundef %a, <16 x i8> noundef %b) local_unnamed_addr {
+; CHECK: vcmpequb. v2, v2, v3
+; CHECK: setbcr r3, 4*cr6+eq
+entry:
+  %0 = tail call i32 @llvm.ppc.altivec.vcmpequb.p(i32 1, <16 x i8> %a, <16 x i8> %b)
+  ret i32 %0
+}
+
+declare i32 @llvm.ppc.altivec.vcmpgtsw.p(i32, <4 x i32>, <4 x i32>)
+
+declare i32 @llvm.ppc.altivec.vcmpequw.p(i32, <4 x i32>, <4 x i32>)
+
+declare i32 @llvm.ppc.altivec.vcmpequh.p(i32, <8 x i16>, <8 x i16>)
+
+declare i32 @llvm.ppc.altivec.vcmpequb.p(i32, <16 x i8>, <16 x i8>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
index 828c6053c8ff7..534fec21ce7c4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
@@ -161,8 +161,8 @@ define double @fsgnj_d(double %a, double %b) nounwind {
 ; RV32I-LABEL: fsgnj_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a4, a2, -1
-; RV32I-NEXT:    and a1, a1, a4
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
 ; RV32I-NEXT:    and a2, a3, a2
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -170,10 +170,10 @@ define double @fsgnj_d(double %a, double %b) nounwind {
 ; RV64I-LABEL: fsgnj_d:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a2, -1
-; RV64I-NEXT:    slli a3, a2, 63
-; RV64I-NEXT:    srli a2, a2, 1
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = call double @llvm.copysign.f64(double %a, double %b)
@@ -241,9 +241,9 @@ define double @fsgnjn_d(double %a, double %b) nounwind {
 ; RV32I-LABEL: fsgnjn_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    slli a1, a1, 1
 ; RV32I-NEXT:    xor a3, a3, a2
-; RV32I-NEXT:    addi a4, a2, -1
-; RV32I-NEXT:    and a1, a1, a4
+; RV32I-NEXT:    srli a1, a1, 1
 ; RV32I-NEXT:    and a2, a3, a2
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -251,11 +251,11 @@ define double @fsgnjn_d(double %a, double %b) nounwind {
 ; RV64I-LABEL: fsgnjn_d:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a2, -1
-; RV64I-NEXT:    slli a3, a2, 63
-; RV64I-NEXT:    srli a2, a2, 1
-; RV64I-NEXT:    xor a1, a1, a3
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    xor a1, a1, a2
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = fneg double %b
@@ -281,9 +281,8 @@ define double @fabs_d(double %a, double %b) nounwind {
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __adddf3
 ; RV32I-NEXT:    mv a3, a1
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
 ; RV32I-NEXT:    mv a2, a0
 ; RV32I-NEXT:    call __adddf3
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -296,9 +295,8 @@ define double @fabs_d(double %a, double %b) nounwind {
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __adddf3
 ; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:    slli a0, a0, 1
 ; RV64I-NEXT:    srli a0, a0, 1
-; RV64I-NEXT:    and a0, a1, a0
 ; RV64I-NEXT:    call __adddf3
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-convert.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-convert.ll
index 7133d5c100e75..81d3381449bc8 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-convert.ll
@@ -3,12 +3,34 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -global-isel -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv32 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
 
 define float @fcvt_s_d(double %a) nounwind {
 ; CHECKIFD-LABEL: fcvt_s_d:
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.s.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __truncdfsf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __truncdfsf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptrunc double %a to float
   ret float %1
 }
@@ -18,6 +40,24 @@ define double @fcvt_d_s(float %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.s fa0, fa0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __extendsfdf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __extendsfdf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fpext float %a to double
   ret double %1
 }
@@ -27,6 +67,24 @@ define i32 @fcvt_w_d(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixdfsi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfsi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi double %a to i32
   ret i32 %1
 }
@@ -36,6 +94,24 @@ define i32 @fcvt_wu_d(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunsdfsi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui double %a to i32
   ret i32 %1
 }
@@ -60,6 +136,34 @@ define i32 @fcvt_wu_d_multiple_use(double %x, ptr %y) nounwind {
 ; RV64IFD-NEXT:    li a0, 1
 ; RV64IFD-NEXT:  .LBB4_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_d_multiple_use:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunsdfsi
+; RV32I-NEXT:    bnez a0, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_d_multiple_use:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi
+; RV64I-NEXT:    slli a1, a0, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    bnez a1, .LBB4_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:  .LBB4_2:
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = fptoui double %x to i32
   %b = icmp eq i32 %a, 0
   %c = select i1 %b, i32 1, i32 %a
@@ -71,6 +175,25 @@ define double @fcvt_d_w(i32 %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_w:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_w:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    call __floatsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i32 %a to double
   ret double %1
 }
@@ -81,6 +204,26 @@ define double @fcvt_d_w_load(ptr %p) nounwind {
 ; CHECKIFD-NEXT:    lw a0, 0(a0)
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_w_load:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    call __floatsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_w_load:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    call __floatsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = load i32, ptr %p
   %1 = sitofp i32 %a to double
   ret double %1
@@ -91,6 +234,25 @@ define double @fcvt_d_wu(i32 %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_wu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_wu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    call __floatunsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i32 %a to double
   ret double %1
 }
@@ -107,6 +269,26 @@ define double @fcvt_d_wu_load(ptr %p) nounwind {
 ; RV64IFD-NEXT:    lwu a0, 0(a0)
 ; RV64IFD-NEXT:    fcvt.d.wu fa0, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_wu_load:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    call __floatunsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_wu_load:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    call __floatunsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = load i32, ptr %p
   %1 = uitofp i32 %a to double
   ret double %1
@@ -126,6 +308,24 @@ define i64 @fcvt_l_d(double %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_l_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixdfdi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_l_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfdi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi double %a to i64
   ret i64 %1
 }
@@ -144,6 +344,24 @@ define i64 @fcvt_lu_d(double %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_lu_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunsdfdi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_lu_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfdi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui double %a to i64
   ret i64 %1
 }
@@ -164,6 +382,24 @@ define i64 @fmv_x_d(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    fadd.d fa5, fa0, fa1
 ; RV64IFD-NEXT:    fmv.x.d a0, fa5
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fmv_x_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __adddf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmv_x_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fadd double %a, %b
   %2 = bitcast double %1 to i64
   ret i64 %2
@@ -183,6 +419,24 @@ define double @fcvt_d_l(i64 %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.d.l fa0, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_l:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatdidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_l:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatdidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i64 %a to double
   ret double %1
 }
@@ -201,6 +455,24 @@ define double @fcvt_d_lu(i64 %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.d.lu fa0, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_lu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatundidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_lu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatundidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i64 %a to double
   ret double %1
 }
@@ -225,6 +497,24 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind {
 ; RV64IFD-NEXT:    fmv.d.x fa4, a1
 ; RV64IFD-NEXT:    fadd.d fa0, fa5, fa4
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fmv_d_x:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __adddf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmv_d_x:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = bitcast i64 %a to double
   %2 = bitcast i64 %b to double
   %3 = fadd double %1, %2
@@ -236,6 +526,24 @@ define double @fcvt_d_w_i8(i8 signext %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_w_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_w_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i8 %a to double
   ret double %1
 }
@@ -245,6 +553,24 @@ define double @fcvt_d_wu_i8(i8 zeroext %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_wu_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_wu_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatunsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i8 %a to double
   ret double %1
 }
@@ -254,6 +580,24 @@ define double @fcvt_d_w_i16(i16 signext %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_w_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_w_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i16 %a to double
   ret double %1
 }
@@ -263,6 +607,24 @@ define double @fcvt_d_wu_i16(i16 zeroext %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_wu_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsidf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_wu_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatunsidf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i16 %a to double
   ret double %1
 }
@@ -281,6 +643,43 @@ define signext i32 @fcvt_d_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fcvt.d.w fa5, a0
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_w_demanded_bits:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    addi s1, a0, 1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __floatsidf
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_w_demanded_bits:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    addiw s1, a0, 1
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __floatsidf
+; RV64I-NEXT:    sd a0, 0(s0)
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = sitofp i32 %3 to double
   store double %4, ptr %1, align 8
@@ -301,6 +700,43 @@ define signext i32 @fcvt_d_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fcvt.d.wu fa5, a0
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_d_wu_demanded_bits:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    addi s1, a0, 1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __floatunsidf
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_d_wu_demanded_bits:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    addiw s1, a0, 1
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __floatunsidf
+; RV64I-NEXT:    sd a0, 0(s0)
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = uitofp i32 %3 to double
   store double %4, ptr %1, align 8
@@ -321,6 +757,28 @@ define signext i16 @fcvt_w_s_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    slli a0, a0, 48
 ; RV64IFD-NEXT:    srai a0, a0, 48
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_s_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixdfsi
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_s_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfsi
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi double %a to i16
   ret i16 %1
 }
@@ -329,18 +787,38 @@ define zeroext i16 @fcvt_wu_s_i16(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_wu_s_i16:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    fcvt.wu.d a0, fa0, rtz
-; RV32IFD-NEXT:    lui a1, 16
-; RV32IFD-NEXT:    addi a1, a1, -1
-; RV32IFD-NEXT:    and a0, a0, a1
+; RV32IFD-NEXT:    slli a0, a0, 16
+; RV32IFD-NEXT:    srli a0, a0, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fcvt_wu_s_i16:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.wu.d a0, fa0, rtz
-; RV64IFD-NEXT:    lui a1, 16
-; RV64IFD-NEXT:    addiw a1, a1, -1
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    slli a0, a0, 48
+; RV64IFD-NEXT:    srli a0, a0, 48
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunsdfsi
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui double %a to i16
   ret i16 %1
 }
@@ -359,6 +837,28 @@ define signext i8 @fcvt_w_s_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    slli a0, a0, 56
 ; RV64IFD-NEXT:    srai a0, a0, 56
 ; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_s_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixdfsi
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_s_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfsi
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi double %a to i8
   ret i8 %1
 }
@@ -369,6 +869,26 @@ define zeroext i8 @fcvt_wu_s_i8(double %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    andi a0, a0, 255
 ; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunsdfsi
+; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi
+; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui double %a to i8
   ret i8 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-intrinsics.ll
index 2b67d5c7ac570..8d77d41ab6b45 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-intrinsics.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d \
-; RUN:   -verify-machineinstrs -target-abi=ilp32d \
+; RUN:   -target-abi=ilp32d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+d \
-; RUN:   -verify-machineinstrs -target-abi=lp64d \
+; RUN:   -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel  \
-; RUN:   -verify-machineinstrs | FileCheck -check-prefix=RV32I %s
+; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel  \
-; RUN:   -verify-machineinstrs | FileCheck -check-prefix=RV64I %s
+; RUN:   | FileCheck -check-prefix=RV64I %s
 
 declare double @llvm.sqrt.f64(double)
 
@@ -39,517 +39,1385 @@ define double @sqrt_f64(double %a) nounwind {
   ret double %1
 }
 
-declare double @llvm.fma.f64(double, double, double)
-
-define double @fma_f64(double %a, double %b, double %c) nounwind {
-; CHECKIFD-LABEL: fma_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
-; CHECKIFD-NEXT:    ret
-;
-; RV32I-LABEL: fma_f64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fma
-; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    addi sp, sp, 16
-; RV32I-NEXT:    ret
+define double @powi_f64(double %a, i32 %b) nounwind {
+; RV32IFD-LABEL: powi_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __powidf2
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
 ;
-; RV64I-LABEL: fma_f64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fma
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-  %1 = call double @llvm.fma.f64(double %a, double %b, double %c)
-  ret double %1
-}
-
-declare double @llvm.fmuladd.f64(double, double, double)
-
-define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
-; CHECKIFD-LABEL: fmuladd_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
-; CHECKIFD-NEXT:    ret
+; RV64IFD-LABEL: powi_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    sext.w a0, a0
+; RV64IFD-NEXT:    call __powidf2
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: fmuladd_f64:
+; RV32I-LABEL: powi_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    mv s0, a4
-; RV32I-NEXT:    mv s1, a5
-; RV32I-NEXT:    call __muldf3
-; RV32I-NEXT:    mv a2, s0
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    call __adddf3
+; RV32I-NEXT:    call __powidf2
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fmuladd_f64:
+; RV64I-LABEL: powi_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a2
-; RV64I-NEXT:    call __muldf3
-; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __powidf2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
+  %1 = call double @llvm.powi.f64.i32(double %a, i32 %b)
   ret double %1
 }
 
-declare double @llvm.fabs.f64(double)
+declare double @llvm.sin.f64(double)
 
-define double @fabs_f64(double %a) nounwind {
-; CHECKIFD-LABEL: fabs_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fabs.d fa0, fa0
-; CHECKIFD-NEXT:    ret
-;
-; RV32I-LABEL: fabs_f64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    ret
+define double @sin_f64(double %a) nounwind {
+; RV32IFD-LABEL: sin_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call sin
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
 ;
-; RV64I-LABEL: fabs_f64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a1, -1
-; RV64I-NEXT:    srli a1, a1, 1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    ret
-  %1 = call double @llvm.fabs.f64(double %a)
-  ret double %1
-}
-
-declare double @llvm.minnum.f64(double, double)
-
-define double @minnum_f64(double %a, double %b) nounwind {
-; CHECKIFD-LABEL: minnum_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fmin.d fa0, fa0, fa1
-; CHECKIFD-NEXT:    ret
+; RV64IFD-LABEL: sin_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call sin
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: minnum_f64:
+; RV32I-LABEL: sin_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fmin
+; RV32I-NEXT:    call sin
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: minnum_f64:
+; RV64I-LABEL: sin_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fmin
+; RV64I-NEXT:    call sin
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.minnum.f64(double %a, double %b)
+  %1 = call double @llvm.sin.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.maxnum.f64(double, double)
+declare double @llvm.cos.f64(double)
 
-define double @maxnum_f64(double %a, double %b) nounwind {
-; CHECKIFD-LABEL: maxnum_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fmax.d fa0, fa0, fa1
-; CHECKIFD-NEXT:    ret
+define double @cos_f64(double %a) nounwind {
+; RV32IFD-LABEL: cos_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call cos
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
 ;
-; RV32I-LABEL: maxnum_f64:
+; RV64IFD-LABEL: cos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call cos
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: cos_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fmax
+; RV32I-NEXT:    call cos
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: maxnum_f64:
+; RV64I-LABEL: cos_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fmax
+; RV64I-NEXT:    call cos
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.maxnum.f64(double %a, double %b)
+  %1 = call double @llvm.cos.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.copysign.f64(double, double)
-
-define double @copysign_f64(double %a, double %b) nounwind {
-; CHECKIFD-LABEL: copysign_f64:
-; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fsgnj.d fa0, fa0, fa1
-; CHECKIFD-NEXT:    ret
+; The sin+cos combination results in an FSINCOS SelectionDAG node.
+define double @sincos_f64(double %a) nounwind {
+; RV32IFD-LABEL: sincos_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -32
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    fsd fs0, 16(sp) # 8-byte Folded Spill
+; RV32IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
+; RV32IFD-NEXT:    fmv.d fs0, fa0
+; RV32IFD-NEXT:    call sin
+; RV32IFD-NEXT:    fmv.d fs1, fa0
+; RV32IFD-NEXT:    fmv.d fa0, fs0
+; RV32IFD-NEXT:    call cos
+; RV32IFD-NEXT:    fadd.d fa0, fs1, fa0
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
+; RV32IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    ret
 ;
-; RV32I-LABEL: copysign_f64:
+; RV64IFD-LABEL: sincos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -32
+; RV64IFD-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fsd fs0, 16(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fmv.d fs0, fa0
+; RV64IFD-NEXT:    call sin
+; RV64IFD-NEXT:    fmv.d fs1, fa0
+; RV64IFD-NEXT:    fmv.d fa0, fs0
+; RV64IFD-NEXT:    call cos
+; RV64IFD-NEXT:    fadd.d fa0, fs1, fa0
+; RV64IFD-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 32
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: sincos_f64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a4, a2, -1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    and a2, a3, a2
-; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    mv s1, a1
+; RV32I-NEXT:    call sin
+; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call cos
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    call __adddf3
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: copysign_f64:
+; RV64I-LABEL: sincos_f64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a2, -1
-; RV64I-NEXT:    slli a3, a2, 63
-; RV64I-NEXT:    srli a2, a2, 1
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    and a1, a1, a3
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    call sin
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call cos
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.copysign.f64(double %a, double %b)
-  ret double %1
+  %1 = call double @llvm.sin.f64(double %a)
+  %2 = call double @llvm.cos.f64(double %a)
+  %3 = fadd double %1, %2
+  ret double %3
 }
 
-declare double @llvm.floor.f64(double)
+declare double @llvm.pow.f64(double, double)
 
-define double @floor_f64(double %a) nounwind {
-; RV32IFD-LABEL: floor_f64:
+define double @pow_f64(double %a, double %b) nounwind {
+; RV32IFD-LABEL: pow_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call floor
+; RV32IFD-NEXT:    call pow
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: floor_f64:
+; RV64IFD-LABEL: pow_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call floor
+; RV64IFD-NEXT:    call pow
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: floor_f64:
+; RV32I-LABEL: pow_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call floor
+; RV32I-NEXT:    call pow
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: floor_f64:
+; RV64I-LABEL: pow_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call floor
+; RV64I-NEXT:    call pow
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.floor.f64(double %a)
+  %1 = call double @llvm.pow.f64(double %a, double %b)
   ret double %1
 }
 
-declare double @llvm.ceil.f64(double)
+declare double @llvm.exp.f64(double)
 
-define double @ceil_f64(double %a) nounwind {
-; RV32IFD-LABEL: ceil_f64:
+define double @exp_f64(double %a) nounwind {
+; RV32IFD-LABEL: exp_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call ceil
+; RV32IFD-NEXT:    call exp
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: ceil_f64:
+; RV64IFD-LABEL: exp_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call ceil
+; RV64IFD-NEXT:    call exp
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: ceil_f64:
+; RV32I-LABEL: exp_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call ceil
+; RV32I-NEXT:    call exp
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: ceil_f64:
+; RV64I-LABEL: exp_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call ceil
+; RV64I-NEXT:    call exp
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.ceil.f64(double %a)
+  %1 = call double @llvm.exp.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.trunc.f64(double)
+declare double @llvm.exp2.f64(double)
 
-define double @trunc_f64(double %a) nounwind {
-; RV32IFD-LABEL: trunc_f64:
+define double @exp2_f64(double %a) nounwind {
+; RV32IFD-LABEL: exp2_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call trunc
+; RV32IFD-NEXT:    call exp2
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: trunc_f64:
+; RV64IFD-LABEL: exp2_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call trunc
+; RV64IFD-NEXT:    call exp2
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: trunc_f64:
+; RV32I-LABEL: exp2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call trunc
+; RV32I-NEXT:    call exp2
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: trunc_f64:
+; RV64I-LABEL: exp2_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call trunc
+; RV64I-NEXT:    call exp2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.trunc.f64(double %a)
+  %1 = call double @llvm.exp2.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.rint.f64(double)
-
-define double @rint_f64(double %a) nounwind {
-; RV32IFD-LABEL: rint_f64:
+define double @exp10_f64(double %a) nounwind {
+; RV32IFD-LABEL: exp10_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call rint
+; RV32IFD-NEXT:    call exp10
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: rint_f64:
+; RV64IFD-LABEL: exp10_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call rint
+; RV64IFD-NEXT:    call exp10
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: rint_f64:
+; RV32I-LABEL: exp10_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call rint
+; RV32I-NEXT:    call exp10
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: rint_f64:
+; RV64I-LABEL: exp10_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call rint
+; RV64I-NEXT:    call exp10
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.rint.f64(double %a)
+  %1 = call double @llvm.exp10.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.nearbyint.f64(double)
+declare double @llvm.log.f64(double)
 
-define double @nearbyint_f64(double %a) nounwind {
-; RV32IFD-LABEL: nearbyint_f64:
+define double @log_f64(double %a) nounwind {
+; RV32IFD-LABEL: log_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call nearbyint
+; RV32IFD-NEXT:    call log
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: nearbyint_f64:
+; RV64IFD-LABEL: log_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call nearbyint
+; RV64IFD-NEXT:    call log
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: nearbyint_f64:
+; RV32I-LABEL: log_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call nearbyint
+; RV32I-NEXT:    call log
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: nearbyint_f64:
+; RV64I-LABEL: log_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call nearbyint
+; RV64I-NEXT:    call log
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.nearbyint.f64(double %a)
+  %1 = call double @llvm.log.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.round.f64(double)
+declare double @llvm.log10.f64(double)
 
-define double @round_f64(double %a) nounwind {
-; RV32IFD-LABEL: round_f64:
+define double @log10_f64(double %a) nounwind {
+; RV32IFD-LABEL: log10_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call round
+; RV32IFD-NEXT:    call log10
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: round_f64:
+; RV64IFD-LABEL: log10_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call round
+; RV64IFD-NEXT:    call log10
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: round_f64:
+; RV32I-LABEL: log10_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call round
+; RV32I-NEXT:    call log10
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: round_f64:
+; RV64I-LABEL: log10_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call round
+; RV64I-NEXT:    call log10
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.round.f64(double %a)
+  %1 = call double @llvm.log10.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.roundeven.f64(double)
+declare double @llvm.log2.f64(double)
 
-define double @roundeven_f64(double %a) nounwind {
-; RV32IFD-LABEL: roundeven_f64:
+define double @log2_f64(double %a) nounwind {
+; RV32IFD-LABEL: log2_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call roundeven
+; RV32IFD-NEXT:    call log2
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: roundeven_f64:
+; RV64IFD-LABEL: log2_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call roundeven
+; RV64IFD-NEXT:    call log2
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
-; RV32I-LABEL: roundeven_f64:
+; RV32I-LABEL: log2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call roundeven
+; RV32I-NEXT:    call log2
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: roundeven_f64:
+; RV64I-LABEL: log2_f64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call roundeven
+; RV64I-NEXT:    call log2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call double @llvm.roundeven.f64(double %a)
+  %1 = call double @llvm.log2.f64(double %a)
   ret double %1
 }
 
-declare i1 @llvm.is.fpclass.f64(double, i32)
-define i1 @isnan_d_fpclass(double %x) {
-; CHECKIFD-LABEL: isnan_d_fpclass:
+declare double @llvm.fma.f64(double, double, double)
+
+define double @fma_f64(double %a, double %b, double %c) nounwind {
+; CHECKIFD-LABEL: fma_f64:
 ; CHECKIFD:       # %bb.0:
-; CHECKIFD-NEXT:    fclass.d a0, fa0
-; CHECKIFD-NEXT:    andi a0, a0, 768
-; CHECKIFD-NEXT:    snez a0, a0
+; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
-; RV32I-LABEL: isnan_d_fpclass:
+; RV32I-LABEL: fma_f64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a3, a2, -1
-; RV32I-NEXT:    lui a2, 524032
-; RV32I-NEXT:    and a1, a1, a3
-; RV32I-NEXT:    beq a1, a2, .LBB14_2
-; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    sltu a0, a2, a1
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fma
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB14_2:
-; RV32I-NEXT:    snez a0, a0
+;
+; RV64I-LABEL: fma_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fma
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.fma.f64(double %a, double %b, double %c)
+  ret double %1
+}
+
+declare double @llvm.fmuladd.f64(double, double, double)
+
+define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
+; CHECKIFD-LABEL: fmuladd_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fmuladd_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a4
+; RV32I-NEXT:    mv s1, a5
+; RV32I-NEXT:    call __muldf3
+; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a3, s1
+; RV32I-NEXT:    call __adddf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmuladd_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    call __muldf3
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
+  ret double %1
+}
+
+declare double @llvm.fabs.f64(double)
+
+define double @fabs_f64(double %a) nounwind {
+; CHECKIFD-LABEL: fabs_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fabs.d fa0, fa0
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: fabs_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fabs_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.fabs.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.minnum.f64(double, double)
+
+define double @minnum_f64(double %a, double %b) nounwind {
+; CHECKIFD-LABEL: minnum_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fmin.d fa0, fa0, fa1
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: minnum_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fmin
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: minnum_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmin
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.minnum.f64(double %a, double %b)
+  ret double %1
+}
+
+declare double @llvm.maxnum.f64(double, double)
+
+define double @maxnum_f64(double %a, double %b) nounwind {
+; CHECKIFD-LABEL: maxnum_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fmax.d fa0, fa0, fa1
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: maxnum_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fmax
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: maxnum_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmax
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.maxnum.f64(double %a, double %b)
+  ret double %1
+}
+
+declare double @llvm.copysign.f64(double, double)
+
+define double @copysign_f64(double %a, double %b) nounwind {
+; CHECKIFD-LABEL: copysign_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fsgnj.d fa0, fa0, fa1
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: copysign_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    and a2, a3, a2
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: copysign_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a2, -1
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.copysign.f64(double %a, double %b)
+  ret double %1
+}
+
+declare double @llvm.floor.f64(double)
+
+define double @floor_f64(double %a) nounwind {
+; RV32IFD-LABEL: floor_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call floor
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: floor_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call floor
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: floor_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call floor
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: floor_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call floor
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.floor.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.ceil.f64(double)
+
+define double @ceil_f64(double %a) nounwind {
+; RV32IFD-LABEL: ceil_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call ceil
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: ceil_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call ceil
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: ceil_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call ceil
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ceil_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call ceil
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.ceil.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.trunc.f64(double)
+
+define double @trunc_f64(double %a) nounwind {
+; RV32IFD-LABEL: trunc_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call trunc
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: trunc_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call trunc
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: trunc_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call trunc
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: trunc_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call trunc
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.trunc.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.rint.f64(double)
+
+define double @rint_f64(double %a) nounwind {
+; RV32IFD-LABEL: rint_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call rint
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: rint_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call rint
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: rint_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call rint
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rint_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call rint
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.rint.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.nearbyint.f64(double)
+
+define double @nearbyint_f64(double %a) nounwind {
+; RV32IFD-LABEL: nearbyint_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call nearbyint
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: nearbyint_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call nearbyint
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: nearbyint_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call nearbyint
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: nearbyint_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call nearbyint
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.nearbyint.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.round.f64(double)
+
+define double @round_f64(double %a) nounwind {
+; RV32IFD-LABEL: round_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call round
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: round_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call round
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: round_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call round
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: round_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call round
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.round.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.roundeven.f64(double)
+
+define double @roundeven_f64(double %a) nounwind {
+; RV32IFD-LABEL: roundeven_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call roundeven
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: roundeven_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call roundeven
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: roundeven_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call roundeven
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: roundeven_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call roundeven
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.roundeven.f64(double %a)
+  ret double %1
+}
+
+declare i1 @llvm.is.fpclass.f64(double, i32)
+define i1 @isnan_d_fpclass(double %x) {
+; CHECKIFD-LABEL: isnan_d_fpclass:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fclass.d a0, fa0
+; CHECKIFD-NEXT:    andi a0, a0, 768
+; CHECKIFD-NEXT:    snez a0, a0
+; CHECKIFD-NEXT:    ret
+;
+; RV32I-LABEL: isnan_d_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524032
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    beq a1, a2, .LBB25_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sltu a0, a2, a1
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB25_2:
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnan_d_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 2047
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    slli a1, a1, 52
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    sltu a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f64(double %x, i32 3)  ; nan
+  ret i1 %1
+}
+
+define double @tan_f64(double %a) nounwind {
+; RV32IFD-LABEL: tan_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call tan
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: tan_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call tan
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: tan_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tan
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isnan_d_fpclass:
+; RV64I-LABEL: tan_f64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a1, -1
-; RV64I-NEXT:    li a2, 2047
-; RV64I-NEXT:    srli a1, a1, 1
-; RV64I-NEXT:    slli a2, a2, 52
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    sltu a0, a2, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tan
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f64(double %x, i32 3)  ; nan
-  ret i1 %1
+  %1 = call double @llvm.tan.f64(double %a)
+  ret double %1
+}
+
+define double @ldexp_double(double %x, i32 %y) nounwind {
+; RV32IFD-LABEL: ldexp_double:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call ldexp
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: ldexp_double:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    sext.w a0, a0
+; RV64IFD-NEXT:    call ldexp
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: ldexp_double:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call ldexp
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ldexp_double:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call ldexp
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %z = call double @llvm.ldexp.f64.i32(double %x, i32 %y)
+  ret double %z
+}
+
+define double @asin_f64(double %a) nounwind {
+; RV32IFD-LABEL: asin_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call asin
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: asin_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call asin
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: asin_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asin
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asin
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.asin.f64(double %a)
+  ret double %1
+}
+
+define double @acos_f64(double %a) nounwind {
+; RV32IFD-LABEL: acos_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call acos
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: acos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call acos
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: acos_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acos
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acos
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.acos.f64(double %a)
+  ret double %1
+}
+
+define double @atan_f64(double %a) nounwind {
+; RV32IFD-LABEL: atan_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call atan
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: atan_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call atan
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: atan_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.atan.f64(double %a)
+  ret double %1
+}
+
+define double @atan2_f64(double %a, double %b) nounwind {
+; RV32IFD-LABEL: atan2_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call atan2
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: atan2_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call atan2
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: atan2_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan2_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.atan2.f64(double %a, double %b)
+  ret double %1
+}
+
+define double @sinh_f64(double %a) nounwind {
+; RV32IFD-LABEL: sinh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call sinh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: sinh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call sinh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: sinh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.sinh.f64(double %a)
+  ret double %1
+}
+
+define double @cosh_f64(double %a) nounwind {
+; RV32IFD-LABEL: cosh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call cosh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: cosh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call cosh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: cosh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call cosh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call cosh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.cosh.f64(double %a)
+  ret double %1
+}
+
+define double @tanh_f64(double %a) nounwind {
+; RV32IFD-LABEL: tanh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call tanh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: tanh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call tanh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32I-LABEL: tanh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.tanh.f64(double %a)
+  ret double %1
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-zfa.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-zfa.ll
new file mode 100644
index 0000000000000..385156b3b99d4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-zfa.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=riscv32 -mattr=+zfa,d -global-isel < %s \
+; RUN: | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfa,d -global-isel < %s \
+; RUN: | FileCheck %s
+
+
+define double @fceil(double %a) {
+; CHECK-LABEL: fceil:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rup
+; CHECK-NEXT:    ret
+  %b = call double @llvm.ceil.f64(double %a)
+  ret double %b
+}
+
+define double @ffloor(double %a) {
+; CHECK-LABEL: ffloor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rdn
+; CHECK-NEXT:    ret
+  %b = call double @llvm.floor.f64(double %a)
+  ret double %b
+}
+
+define double @frint(double %a) {
+; CHECK-LABEL: frint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    froundnx.d fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call double @llvm.rint.f64(double %a)
+  ret double %b
+}
+
+define double @fnearbyint(double %a) {
+; CHECK-LABEL: fnearbyint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call double @llvm.nearbyint.f64(double %a)
+  ret double %b
+}
+
+define double @fround(double %a) {
+; CHECK-LABEL: fround:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rmm
+; CHECK-NEXT:    ret
+  %b = call double @llvm.round.f64(double %a)
+  ret double %b
+}
+
+define double @froundeven(double %a) {
+; CHECK-LABEL: froundeven:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rne
+; CHECK-NEXT:    ret
+  %b = call double @llvm.roundeven.f64(double %a)
+  ret double %b
+}
+
+define double @ftrunc(double %a) {
+; CHECK-LABEL: ftrunc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rtz
+; CHECK-NEXT:    ret
+  %b = call double @llvm.trunc.f64(double %a)
+  ret double %b
+}
+
+define double @fmaximum(double %a, double %b) {
+; CHECK-LABEL: fmaximum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmaxm.d fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call double @llvm.maximum.f64(double %a, double %b)
+  ret double %c
+}
+
+define double @fminimum(double %a, double %b) {
+; CHECK-LABEL: fminimum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fminm.d fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call double @llvm.minimum.f64(double %a, double %b)
+  ret double %c
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-arith.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-arith.ll
index f527298916150..3a60856665742 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-arith.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-arith.ll
@@ -161,8 +161,8 @@ define float @fsgnj_s(float %a, float %b) nounwind {
 ; RV32I-LABEL: fsgnj_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a3, a2, -1
-; RV32I-NEXT:    and a0, a0, a3
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
 ; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    ret
@@ -170,8 +170,8 @@ define float @fsgnj_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fsgnj_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 524288
-; RV64I-NEXT:    addiw a3, a2, -1
-; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -238,11 +238,11 @@ define float @fsgnjn_s(float %a, float %b) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    call __addsf3
 ; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    slli s0, s0, 1
 ; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    addi a2, a1, -1
-; RV32I-NEXT:    and a2, s0, a2
+; RV32I-NEXT:    srli s0, s0, 1
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, s0, a0
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -256,11 +256,11 @@ define float @fsgnjn_s(float %a, float %b) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    call __addsf3
 ; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    slli s0, s0, 33
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    addiw a2, a1, -1
-; RV64I-NEXT:    and a2, s0, a2
+; RV64I-NEXT:    srli s0, s0, 33
 ; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, s0, a0
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -287,9 +287,8 @@ define float @fabs_s(float %a, float %b) nounwind {
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __addsf3
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lui a0, 524288
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
 ; RV32I-NEXT:    call __addsf3
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -301,9 +300,8 @@ define float @fabs_s(float %a, float %b) nounwind {
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __addsf3
 ; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lui a0, 524288
-; RV64I-NEXT:    addiw a0, a0, -1
-; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
 ; RV64I-NEXT:    call __addsf3
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-convert.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-convert.ll
index e6df28f5f28d1..51df36f5eee05 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-convert.ll
@@ -3,12 +3,34 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefixes=CHECKIF,RV32IF %s
 ; RUN: llc -mtriple=riscv64 -global-isel -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: llc -mtriple=riscv32 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -global-isel -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
 
 define i32 @fcvt_w_s(float %a) nounwind {
 ; CHECKIF-LABEL: fcvt_w_s:
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixsfsi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi float %a to i32
   ret i32 %1
 }
@@ -18,6 +40,24 @@ define i32 @fcvt_wu_s(float %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunssfsi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui float %a to i32
   ret i32 %1
 }
@@ -44,6 +84,34 @@ define i32 @fcvt_wu_s_multiple_use(float %x, ptr %y) nounwind {
 ; RV64IF-NEXT:    li a0, 1
 ; RV64IF-NEXT:  .LBB2_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s_multiple_use:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunssfsi
+; RV32I-NEXT:    bnez a0, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s_multiple_use:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi
+; RV64I-NEXT:    slli a1, a0, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    bnez a1, .LBB2_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:  .LBB2_2:
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = fptoui float %x to i32
   %b = icmp eq i32 %a, 0
   %c = select i1 %b, i32 1, i32 %a
@@ -63,6 +131,25 @@ define signext i32 @fmv_x_w(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    fmv.x.w a0, fa5
 ; RV64IF-NEXT:    sext.w a0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fmv_x_w:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __addsf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmv_x_w:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
 ; Ensure fmv.x.w is generated even for a soft float calling convention
   %1 = fadd float %a, %b
   %2 = bitcast float %1 to i32
@@ -74,6 +161,25 @@ define float @fcvt_s_w(i32 %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_w:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_w:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i32 %a to float
   ret float %1
 }
@@ -84,6 +190,26 @@ define float @fcvt_s_w_load(ptr %p) nounwind {
 ; CHECKIF-NEXT:    lw a0, 0(a0)
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_w_load:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    call __floatsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_w_load:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = load i32, ptr %p
   %1 = sitofp i32 %a to float
   ret float %1
@@ -94,6 +220,25 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_wu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_wu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i32 %a to float
   ret float %1
 }
@@ -110,6 +255,26 @@ define float @fcvt_s_wu_load(ptr %p) nounwind {
 ; RV64IF-NEXT:    lwu a0, 0(a0)
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_wu_load:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    call __floatunsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_wu_load:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %a = load i32, ptr %p
   %1 = uitofp i32 %a to float
   ret float %1
@@ -122,6 +287,24 @@ define float @fmv_w_x(i32 %a, i32 %b) nounwind {
 ; CHECKIF-NEXT:    fmv.w.x fa4, a1
 ; CHECKIF-NEXT:    fadd.s fa0, fa5, fa4
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fmv_w_x:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __addsf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmv_w_x:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
 ; Ensure fmv.w.x is generated even for a soft float calling convention
   %1 = bitcast i32 %a to float
   %2 = bitcast i32 %b to float
@@ -143,6 +326,24 @@ define i64 @fcvt_l_s(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_l_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixsfdi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_l_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfdi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi float %a to i64
   ret i64 %1
 }
@@ -161,6 +362,24 @@ define i64 @fcvt_lu_s(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_lu_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunssfdi
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_lu_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfdi
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui float %a to i64
   ret i64 %1
 }
@@ -179,6 +398,24 @@ define float @fcvt_s_l(i64 %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.l fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_l:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatdisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_l:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatdisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i64 %a to float
   ret float %1
 }
@@ -197,6 +434,24 @@ define float @fcvt_s_lu(i64 %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.lu fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_lu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatundisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_lu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatundisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i64 %a to float
   ret float %1
 }
@@ -206,6 +461,24 @@ define float @fcvt_s_w_i8(i8 signext %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_w_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_w_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i8 %a to float
   ret float %1
 }
@@ -215,6 +488,24 @@ define float @fcvt_s_wu_i8(i8 zeroext %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_wu_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_wu_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i8 %a to float
   ret float %1
 }
@@ -224,6 +515,24 @@ define float @fcvt_s_w_i16(i16 signext %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_w_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_w_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = sitofp i16 %a to float
   ret float %1
 }
@@ -233,6 +542,24 @@ define float @fcvt_s_wu_i16(i16 zeroext %a) nounwind {
 ; CHECKIF:       # %bb.0:
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_wu_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __floatunsisf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_wu_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = uitofp i16 %a to float
   ret float %1
 }
@@ -252,6 +579,42 @@ define signext i32 @fcvt_s_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_w_demanded_bits:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    addi s1, a0, 1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __floatsisf
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_w_demanded_bits:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    addiw s1, a0, 1
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __floatsisf
+; RV64I-NEXT:    sw a0, 0(s0)
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = sitofp i32 %3 to float
   store float %4, ptr %1, align 4
@@ -273,6 +636,42 @@ define signext i32 @fcvt_s_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fcvt.s.wu fa5, a0
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_s_wu_demanded_bits:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    addi s1, a0, 1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __floatunsisf
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_s_wu_demanded_bits:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    addiw s1, a0, 1
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __floatunsisf
+; RV64I-NEXT:    sw a0, 0(s0)
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = uitofp i32 %3 to float
   store float %4, ptr %1, align 4
@@ -293,6 +692,28 @@ define signext i16 @fcvt_w_s_i16(float %a) nounwind {
 ; RV64IF-NEXT:    slli a0, a0, 48
 ; RV64IF-NEXT:    srai a0, a0, 48
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_s_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixsfsi
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_s_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi float %a to i16
   ret i16 %1
 }
@@ -301,18 +722,38 @@ define zeroext i16 @fcvt_wu_s_i16(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_wu_s_i16:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fcvt.wu.s a0, fa0, rtz
-; RV32IF-NEXT:    lui a1, 16
-; RV32IF-NEXT:    addi a1, a1, -1
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    slli a0, a0, 16
+; RV32IF-NEXT:    srli a0, a0, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcvt_wu_s_i16:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rtz
-; RV64IF-NEXT:    lui a1, 16
-; RV64IF-NEXT:    addiw a1, a1, -1
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    slli a0, a0, 48
+; RV64IF-NEXT:    srli a0, a0, 48
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunssfsi
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui float %a to i16
   ret i16 %1
 }
@@ -331,6 +772,28 @@ define signext i8 @fcvt_w_s_i8(float %a) nounwind {
 ; RV64IF-NEXT:    slli a0, a0, 56
 ; RV64IF-NEXT:    srai a0, a0, 56
 ; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_w_s_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixsfsi
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_w_s_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptosi float %a to i8
   ret i8 %1
 }
@@ -341,6 +804,26 @@ define zeroext i8 @fcvt_wu_s_i8(float %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; CHECKIF-NEXT:    andi a0, a0, 255
 ; CHECKIF-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_wu_s_i8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __fixunssfsi
+; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_wu_s_i8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi
+; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
   %1 = fptoui float %a to i8
   ret i8 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-fclass.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-fclass.ll
new file mode 100644
index 0000000000000..b846302c8a3c0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-fclass.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=riscv32 -mattr=+f -global-isel \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=riscv64 -mattr=+f -global-isel \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK
+
+define i1 @fpclass(ptr %x) {
+; CHECK-LABEL: fpclass:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    flw fa5, 0(a0)
+; CHECK-NEXT:    fclass.s a0, fa5
+; CHECK-NEXT:    andi a0, a0, 927
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %a = load float, ptr %x
+  %cmp = call i1 @llvm.is.fpclass.f32(float %a, i32 639)
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
index 4d2b74ec735a1..35c7fdfb33fe4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
@@ -1,20 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+f \
-; RUN:   -verify-machineinstrs -target-abi=ilp32f \
+; RUN:   -target-abi=ilp32f \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d \
-; RUN:   -verify-machineinstrs -target-abi=ilp32f \
+; RUN:   -target-abi=ilp32f \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+f \
-; RUN:   -verify-machineinstrs -target-abi=lp64f \
+; RUN:   -target-abi=lp64f \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel  -mattr=+d \
-; RUN:   -verify-machineinstrs -target-abi=lp64d \
+; RUN:   -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel \
-; RUN:   -verify-machineinstrs | FileCheck -check-prefix=RV32I %s
+; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel \
-; RUN:    -verify-machineinstrs | FileCheck -check-prefix=RV64I %s
+; RUN:   | FileCheck -check-prefix=RV64I %s
 
 define float @sqrt_f32(float %a) nounwind {
 ; RV32IF-LABEL: sqrt_f32:
@@ -48,897 +48,1708 @@ define float @sqrt_f32(float %a) nounwind {
   ret float %1
 }
 
-define float @fma_f32(float %a, float %b, float %c) nounwind {
-; RV32IF-LABEL: fma_f32:
+define float @powi_f32(float %a, i32 %b) nounwind {
+; RV32IF-LABEL: powi_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __powisf2
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: fma_f32:
+; RV64IF-LABEL: powi_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sext.w a0, a0
+; RV64IF-NEXT:    call __powisf2
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: fma_f32:
+; RV32I-LABEL: powi_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fmaf
+; RV32I-NEXT:    call __powisf2
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fma_f32:
+; RV64I-LABEL: powi_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __powisf2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
+  %1 = call float @llvm.powi.f32.i32(float %a, i32 %b)
   ret float %1
 }
 
-define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
-; RV32IF-LABEL: fmuladd_f32:
+define float @sin_f32(float %a) nounwind {
+; RV32IF-LABEL: sin_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call sinf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: fmuladd_f32:
+; RV64IF-LABEL: sin_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call sinf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: fmuladd_f32:
+; RV32I-LABEL: sin_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    mv s0, a2
-; RV32I-NEXT:    call __mulsf3
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __addsf3
+; RV32I-NEXT:    call sinf
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fmuladd_f32:
+; RV64I-LABEL: sin_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a2
-; RV64I-NEXT:    call __mulsf3
-; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call sinf
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
-  ret float %1
-}
-
-define float @fabs_f32(float %a) nounwind {
-; RV32IF-LABEL: fabs_f32:
-; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fabs.s fa0, fa0
-; RV32IF-NEXT:    ret
-;
-; RV64IF-LABEL: fabs_f32:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fabs.s fa0, fa0
-; RV64IF-NEXT:    ret
-;
-; RV32I-LABEL: fabs_f32:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: fabs_f32:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    ret
-  %1 = call float @llvm.fabs.f32(float %a)
+  %1 = call float @llvm.sin.f32(float %a)
   ret float %1
 }
 
-define float @minnum_f32(float %a, float %b) nounwind {
-; RV32IF-LABEL: minnum_f32:
+define float @cos_f32(float %a) nounwind {
+; RV32IF-LABEL: cos_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmin.s fa0, fa0, fa1
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call cosf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: minnum_f32:
+; RV64IF-LABEL: cos_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmin.s fa0, fa0, fa1
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call cosf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: minnum_f32:
+; RV32I-LABEL: cos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fminf
+; RV32I-NEXT:    call cosf
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: minnum_f32:
+; RV64I-LABEL: cos_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fminf
+; RV64I-NEXT:    call cosf
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.minnum.f32(float %a, float %b)
+  %1 = call float @llvm.cos.f32(float %a)
   ret float %1
 }
 
-define float @maxnum_f32(float %a, float %b) nounwind {
-; RV32IF-LABEL: maxnum_f32:
+; The sin+cos combination results in an FSINCOS SelectionDAG node.
+define float @sincos_f32(float %a) nounwind {
+; RV32IF-LABEL: sincos_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmax.s fa0, fa0, fa1
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    fsw fs0, 8(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    fsw fs1, 4(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    fmv.s fs0, fa0
+; RV32IF-NEXT:    call sinf
+; RV32IF-NEXT:    fmv.s fs1, fa0
+; RV32IF-NEXT:    fmv.s fa0, fs0
+; RV32IF-NEXT:    call cosf
+; RV32IF-NEXT:    fadd.s fa0, fs1, fa0
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    flw fs0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    flw fs1, 4(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: maxnum_f32:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmax.s fa0, fa0, fa1
-; RV64IF-NEXT:    ret
-;
-; RV32I-LABEL: maxnum_f32:
+; RV32I-LABEL: sincos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call fmaxf
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    call sinf
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call cosf
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call __addsf3
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: maxnum_f32:
+; RV64I-LABEL: sincos_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call fmaxf
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    call sinf
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call cosf
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.maxnum.f32(float %a, float %b)
-  ret float %1
+  %1 = call float @llvm.sin.f32(float %a)
+  %2 = call float @llvm.cos.f32(float %a)
+  %3 = fadd float %1, %2
+  ret float %3
 }
 
-define float @copysign_f32(float %a, float %b) nounwind {
-; RV32IF-LABEL: copysign_f32:
+define float @pow_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: pow_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call powf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: copysign_f32:
+; RV64IF-LABEL: pow_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call powf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: copysign_f32:
+; RV32I-LABEL: pow_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 524288
-; RV32I-NEXT:    addi a3, a2, -1
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call powf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: copysign_f32:
+; RV64I-LABEL: pow_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a2, 524288
-; RV64I-NEXT:    addiw a3, a2, -1
-; RV64I-NEXT:    and a0, a0, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call powf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.copysign.f32(float %a, float %b)
+  %1 = call float @llvm.pow.f32(float %a, float %b)
   ret float %1
 }
 
-define float @ceil_f32(float %a) nounwind {
-; RV32IF-LABEL: ceil_f32:
+define float @exp_f32(float %a) nounwind {
+; RV32IF-LABEL: exp_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call ceilf
+; RV32IF-NEXT:    call expf
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: ceil_f32:
+; RV64IF-LABEL: exp_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call ceilf
+; RV64IF-NEXT:    call expf
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: ceil_f32:
+; RV32I-LABEL: exp_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call ceilf
+; RV32I-NEXT:    call expf
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: ceil_f32:
+; RV64I-LABEL: exp_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call ceilf
+; RV64I-NEXT:    call expf
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.ceil.f32(float %a)
+  %1 = call float @llvm.exp.f32(float %a)
   ret float %1
 }
 
-define float @trunc_f32(float %a) nounwind {
-; RV32IF-LABEL: trunc_f32:
+define float @exp2_f32(float %a) nounwind {
+; RV32IF-LABEL: exp2_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call truncf
+; RV32IF-NEXT:    call exp2f
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: trunc_f32:
+; RV64IF-LABEL: exp2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call truncf
+; RV64IF-NEXT:    call exp2f
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: trunc_f32:
+; RV32I-LABEL: exp2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call truncf
+; RV32I-NEXT:    call exp2f
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: trunc_f32:
+; RV64I-LABEL: exp2_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call truncf
+; RV64I-NEXT:    call exp2f
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.trunc.f32(float %a)
+  %1 = call float @llvm.exp2.f32(float %a)
   ret float %1
 }
 
-define float @rint_f32(float %a) nounwind {
-; RV32IF-LABEL: rint_f32:
+define float @exp10_f32(float %a) nounwind {
+; RV32IF-LABEL: exp10_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call rintf
+; RV32IF-NEXT:    call exp10f
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: rint_f32:
+; RV64IF-LABEL: exp10_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call rintf
+; RV64IF-NEXT:    call exp10f
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: rint_f32:
+; RV32I-LABEL: exp10_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call rintf
+; RV32I-NEXT:    call exp10f
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: rint_f32:
+; RV64I-LABEL: exp10_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call rintf
+; RV64I-NEXT:    call exp10f
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.rint.f32(float %a)
+  %1 = call float @llvm.exp10.f32(float %a)
   ret float %1
 }
 
-define float @nearbyint_f32(float %a) nounwind {
-; RV32IF-LABEL: nearbyint_f32:
+define float @log_f32(float %a) nounwind {
+; RV32IF-LABEL: log_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call nearbyintf
+; RV32IF-NEXT:    call logf
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: nearbyint_f32:
+; RV64IF-LABEL: log_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call nearbyintf
+; RV64IF-NEXT:    call logf
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: nearbyint_f32:
+; RV32I-LABEL: log_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call nearbyintf
+; RV32I-NEXT:    call logf
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: nearbyint_f32:
+; RV64I-LABEL: log_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call nearbyintf
+; RV64I-NEXT:    call logf
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.nearbyint.f32(float %a)
+  %1 = call float @llvm.log.f32(float %a)
   ret float %1
 }
 
-define float @round_f32(float %a) nounwind {
-; RV32IF-LABEL: round_f32:
+define float @log10_f32(float %a) nounwind {
+; RV32IF-LABEL: log10_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call roundf
+; RV32IF-NEXT:    call log10f
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: round_f32:
+; RV64IF-LABEL: log10_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call roundf
+; RV64IF-NEXT:    call log10f
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: round_f32:
+; RV32I-LABEL: log10_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call roundf
+; RV32I-NEXT:    call log10f
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: round_f32:
+; RV64I-LABEL: log10_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call roundf
+; RV64I-NEXT:    call log10f
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.round.f32(float %a)
+  %1 = call float @llvm.log10.f32(float %a)
   ret float %1
 }
 
-define float @roundeven_f32(float %a) nounwind {
-; RV32IF-LABEL: roundeven_f32:
+define float @log2_f32(float %a) nounwind {
+; RV32IF-LABEL: log2_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    call roundevenf
+; RV32IF-NEXT:    call log2f
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: roundeven_f32:
+; RV64IF-LABEL: log2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call roundevenf
+; RV64IF-NEXT:    call log2f
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: roundeven_f32:
+; RV32I-LABEL: log2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call roundevenf
+; RV32I-NEXT:    call log2f
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: roundeven_f32:
+; RV64I-LABEL: log2_f32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call roundevenf
+; RV64I-NEXT:    call log2f
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call float @llvm.roundeven.f32(float %a)
+  %1 = call float @llvm.log2.f32(float %a)
   ret float %1
 }
 
-define i1 @fpclass(float %x) {
-; RV32IF-LABEL: fpclass:
+define float @fma_f32(float %a, float %b, float %c) nounwind {
+; RV32IF-LABEL: fma_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 927
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: fpclass:
+; RV64IF-LABEL: fma_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 927
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: fpclass:
+; RV32I-LABEL: fma_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    lui a3, 2048
-; RV32I-NEXT:    lui a4, 1046528
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    addi a3, a3, -1
-; RV32I-NEXT:    and a1, a0, a1
-; RV32I-NEXT:    addi a5, a1, -1
-; RV32I-NEXT:    sltu a3, a5, a3
-; RV32I-NEXT:    lui a5, 520192
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    add a4, a1, a4
-; RV32I-NEXT:    sltu a4, a4, a5
-; RV32I-NEXT:    xor a5, a1, a2
-; RV32I-NEXT:    sltu a2, a2, a1
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    seqz a5, a5
-; RV32I-NEXT:    and a3, a3, a0
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    and a0, a4, a0
-; RV32I-NEXT:    or a1, a1, a3
-; RV32I-NEXT:    or a0, a2, a0
-; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fmaf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fpclass:
+; RV64I-LABEL: fma_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    slli a3, a0, 32
-; RV64I-NEXT:    li a4, 1
-; RV64I-NEXT:    lui a5, 2048
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    seqz a1, a0
-; RV64I-NEXT:    xor a6, a0, a2
-; RV64I-NEXT:    seqz a6, a6
-; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    lui a6, 520192
-; RV64I-NEXT:    srli a3, a3, 32
-; RV64I-NEXT:    xor a3, a3, a0
-; RV64I-NEXT:    sub a4, a0, a4
-; RV64I-NEXT:    sltu a2, a2, a0
-; RV64I-NEXT:    sub a0, a0, a5
-; RV64I-NEXT:    addiw a5, a5, -1
-; RV64I-NEXT:    snez a3, a3
-; RV64I-NEXT:    slli a4, a4, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a4, a4, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    sltu a4, a4, a5
-; RV64I-NEXT:    or a1, a1, a2
-; RV64I-NEXT:    sltu a0, a0, a6
-; RV64I-NEXT:    and a4, a4, a3
-; RV64I-NEXT:    or a1, a1, a4
-; RV64I-NEXT:    and a0, a0, a3
-; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmaf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %cmp = call i1 @llvm.is.fpclass.f32(float %x, i32 639)
-  ret i1 %cmp
+  %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
+  ret float %1
 }
 
-define i1 @isnan_fpclass(float %x) {
-; RV32IF-LABEL: isnan_fpclass:
-; RV32IF:       # %bb.0:
+define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
+; RV32IF-LABEL: fmuladd_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmuladd_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fmuladd_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    call __mulsf3
+; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    call __addsf3
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fmuladd_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    call __mulsf3
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
+  ret float %1
+}
+
+define float @fabs_f32(float %a) nounwind {
+; RV32IF-LABEL: fabs_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fabs.s fa0, fa0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fabs_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fabs.s fa0, fa0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fabs_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fabs_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.fabs.f32(float %a)
+  ret float %1
+}
+
+define float @minnum_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: minnum_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmin.s fa0, fa0, fa1
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: minnum_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmin.s fa0, fa0, fa1
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: minnum_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fminf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: minnum_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fminf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.minnum.f32(float %a, float %b)
+  ret float %1
+}
+
+define float @maxnum_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: maxnum_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmax.s fa0, fa0, fa1
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: maxnum_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmax.s fa0, fa0, fa1
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: maxnum_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call fmaxf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: maxnum_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmaxf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.maxnum.f32(float %a, float %b)
+  ret float %1
+}
+
+define float @copysign_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: copysign_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: copysign_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: copysign_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: copysign_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %1
+}
+
+define float @ceil_f32(float %a) nounwind {
+; RV32IF-LABEL: ceil_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call ceilf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: ceil_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call ceilf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: ceil_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call ceilf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ceil_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call ceilf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.ceil.f32(float %a)
+  ret float %1
+}
+
+define float @trunc_f32(float %a) nounwind {
+; RV32IF-LABEL: trunc_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call truncf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: trunc_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call truncf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: trunc_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call truncf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: trunc_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call truncf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.trunc.f32(float %a)
+  ret float %1
+}
+
+define float @rint_f32(float %a) nounwind {
+; RV32IF-LABEL: rint_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call rintf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: rint_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call rintf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: rint_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call rintf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: rint_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call rintf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.rint.f32(float %a)
+  ret float %1
+}
+
+define float @nearbyint_f32(float %a) nounwind {
+; RV32IF-LABEL: nearbyint_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call nearbyintf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: nearbyint_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call nearbyintf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: nearbyint_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call nearbyintf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: nearbyint_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call nearbyintf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.nearbyint.f32(float %a)
+  ret float %1
+}
+
+define float @round_f32(float %a) nounwind {
+; RV32IF-LABEL: round_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call roundf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: round_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call roundf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: round_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call roundf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: round_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call roundf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.round.f32(float %a)
+  ret float %1
+}
+
+define float @roundeven_f32(float %a) nounwind {
+; RV32IF-LABEL: roundeven_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call roundevenf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: roundeven_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call roundevenf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: roundeven_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call roundevenf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: roundeven_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call roundevenf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.roundeven.f32(float %a)
+  ret float %1
+}
+
+define i1 @fpclass(float %x) {
+; RV32IF-LABEL: fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 927
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 927
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a2, a0, 1
+; RV32I-NEXT:    lui a3, 2048
+; RV32I-NEXT:    lui a4, 1046528
+; RV32I-NEXT:    srli a2, a2, 1
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    addi a5, a2, -1
+; RV32I-NEXT:    sltu a3, a5, a3
+; RV32I-NEXT:    lui a5, 520192
+; RV32I-NEXT:    xor a0, a0, a2
+; RV32I-NEXT:    add a4, a2, a4
+; RV32I-NEXT:    sltu a4, a4, a5
+; RV32I-NEXT:    xor a5, a2, a1
+; RV32I-NEXT:    sltu a1, a1, a2
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    and a3, a3, a0
+; RV32I-NEXT:    or a2, a2, a5
+; RV32I-NEXT:    and a0, a4, a0
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a2, a0, 33
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    li a3, 1
+; RV64I-NEXT:    lui a4, 2048
+; RV64I-NEXT:    srli a2, a2, 33
+; RV64I-NEXT:    seqz a5, a2
+; RV64I-NEXT:    xor a6, a2, a1
+; RV64I-NEXT:    seqz a6, a6
+; RV64I-NEXT:    or a5, a5, a6
+; RV64I-NEXT:    lui a6, 520192
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    xor a0, a0, a2
+; RV64I-NEXT:    sub a3, a2, a3
+; RV64I-NEXT:    sltu a1, a1, a2
+; RV64I-NEXT:    sub a2, a2, a4
+; RV64I-NEXT:    addiw a4, a4, -1
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    slli a3, a3, 32
+; RV64I-NEXT:    slli a2, a2, 32
+; RV64I-NEXT:    srli a3, a3, 32
+; RV64I-NEXT:    srli a2, a2, 32
+; RV64I-NEXT:    sltu a3, a3, a4
+; RV64I-NEXT:    or a1, a5, a1
+; RV64I-NEXT:    sltu a2, a2, a6
+; RV64I-NEXT:    and a3, a3, a0
+; RV64I-NEXT:    or a1, a1, a3
+; RV64I-NEXT:    and a0, a2, a0
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+  %cmp = call i1 @llvm.is.fpclass.f32(float %x, i32 639)
+  ret i1 %cmp
+}
+
+define i1 @isnan_fpclass(float %x) {
+; RV32IF-LABEL: isnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 768
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 768
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    sltu a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    sltu a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 3)  ; nan
+  ret i1 %1
+}
+
+define i1 @isqnan_fpclass(float %x) {
+; RV32IF-LABEL: isqnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 512
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isqnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 512
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isqnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 523264
+; RV32I-NEXT:    sltu a0, a0, a1
+; RV32I-NEXT:    xori a0, a0, 1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isqnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 523264
+; RV64I-NEXT:    sltu a0, a0, a1
+; RV64I-NEXT:    xori a0, a0, 1
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 2)  ; qnan
+  ret i1 %1
+}
+
+define i1 @issnan_fpclass(float %x) {
+; RV32IF-LABEL: issnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 256
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: issnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 256
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: issnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    lui a2, 523264
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    sltu a1, a1, a0
+; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: issnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    lui a2, 523264
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    sltu a1, a1, a0
+; RV64I-NEXT:    sltu a0, a0, a2
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 1)  ; snan
+  ret i1 %1
+}
+
+define i1 @isinf_fpclass(float %x) {
+; RV32IF-LABEL: isinf_fpclass:
+; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 768
+; RV32IF-NEXT:    andi a0, a0, 129
 ; RV32IF-NEXT:    snez a0, a0
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isnan_fpclass:
+; RV64IF-LABEL: isinf_fpclass:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 768
+; RV64IF-NEXT:    andi a0, a0, 129
 ; RV64IF-NEXT:    snez a0, a0
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isnan_fpclass:
+; RV32I-LABEL: isinf_fpclass:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    sltu a0, a2, a0
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isnan_fpclass:
+; RV64I-LABEL: isinf_fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    sltu a0, a2, a0
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 3)  ; nan
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 516)  ; 0x204 = "inf"
   ret i1 %1
 }
 
-define i1 @isqnan_fpclass(float %x) {
-; RV32IF-LABEL: isqnan_fpclass:
+define i1 @isposinf_fpclass(float %x) {
+; RV32IF-LABEL: isposinf_fpclass:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 512
+; RV32IF-NEXT:    andi a0, a0, 128
 ; RV32IF-NEXT:    snez a0, a0
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isqnan_fpclass:
+; RV64IF-LABEL: isposinf_fpclass:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 512
+; RV64IF-NEXT:    andi a0, a0, 128
 ; RV64IF-NEXT:    snez a0, a0
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isqnan_fpclass:
+; RV32I-LABEL: isposinf_fpclass:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 523264
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isposinf_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 512)  ; 0x200 = "+inf"
+  ret i1 %1
+}
+
+define i1 @isneginf_fpclass(float %x) {
+; RV32IF-LABEL: isneginf_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 1
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isneginf_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 1
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isneginf_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1046528
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isneginf_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    li a1, 511
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 23
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 4)  ; "-inf"
+  ret i1 %1
+}
+
+define i1 @isfinite_fpclass(float %x) {
+; RV32IF-LABEL: isfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 126
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 126
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
 ; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isqnan_fpclass:
+; RV64I-LABEL: isfinite_fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    lui a1, 523264
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
 ; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 2)  ; qnan
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 504)  ; 0x1f8 = "finite"
   ret i1 %1
 }
 
-define i1 @issnan_fpclass(float %x) {
-; RV32IF-LABEL: issnan_fpclass:
+define i1 @isposfinite_fpclass(float %x) {
+; RV32IF-LABEL: isposfinite_fpclass:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 256
+; RV32IF-NEXT:    andi a0, a0, 112
 ; RV32IF-NEXT:    snez a0, a0
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: issnan_fpclass:
+; RV64IF-LABEL: isposfinite_fpclass:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 256
+; RV64IF-NEXT:    andi a0, a0, 112
 ; RV64IF-NEXT:    snez a0, a0
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: issnan_fpclass:
+; RV32I-LABEL: isposfinite_fpclass:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 523264
-; RV32I-NEXT:    sltu a2, a2, a0
+; RV32I-NEXT:    lui a1, 522240
 ; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    and a0, a2, a0
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: issnan_fpclass:
+; RV64I-LABEL: isposfinite_fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    lui a1, 523264
-; RV64I-NEXT:    sltu a2, a2, a0
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    and a0, a2, a0
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 1)  ; snan
-  ret i1 %1
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 448)  ; 0x1c0 = "+finite"
+  ret i1 %1
+}
+
+define i1 @isnegfinite_fpclass(float %x) {
+; RV32IF-LABEL: isnegfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 14
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isnegfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 14
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isnegfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a2, a0, 1
+; RV32I-NEXT:    srli a2, a2, 1
+; RV32I-NEXT:    xor a0, a0, a2
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    sltu a1, a2, a1
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnegfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a2, a0, 33
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a2, a2, 33
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    xor a0, a0, a2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    sltu a1, a2, a1
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 56)  ; 0x38 = "-finite"
+  ret i1 %1
+}
+
+define i1 @isnotfinite_fpclass(float %x) {
+; RV32IF-LABEL: isnotfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 897
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: isnotfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 897
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: isnotfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    xor a2, a0, a1
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    sltu a0, a1, a0
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnotfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    xor a2, a0, a1
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    sltu a0, a1, a0
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 519)  ; ox207 = "inf|nan"
+  ret i1 %1
+}
+
+define float @tan_f32(float %a) nounwind {
+; RV32IF-LABEL: tan_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call tanf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: tan_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call tanf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: tan_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tan_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.tan.f32(float %a)
+  ret float %1
+}
+
+define float @ldexp_float(float %x, i32 %y) nounwind {
+; RV32IF-LABEL: ldexp_float:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call ldexpf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: ldexp_float:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sext.w a0, a0
+; RV64IF-NEXT:    call ldexpf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32I-LABEL: ldexp_float:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call ldexpf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ldexp_float:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call ldexpf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %z = call float @llvm.ldexp.f32.i32(float %x, i32 %y)
+  ret float %z
 }
 
-define i1 @isinf_fpclass(float %x) {
-; RV32IF-LABEL: isinf_fpclass:
+define float @asin_f32(float %a) nounwind {
+; RV32IF-LABEL: asin_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 129
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call asinf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isinf_fpclass:
+; RV64IF-LABEL: asin_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 129
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call asinf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isinf_fpclass:
+; RV32I-LABEL: asin_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    xor a0, a0, a2
-; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asinf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isinf_fpclass:
+; RV64I-LABEL: asin_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    xor a0, a0, a2
-; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asinf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 516)  ; 0x204 = "inf"
-  ret i1 %1
+  %1 = call float @llvm.asin.f32(float %a)
+  ret float %1
 }
 
-define i1 @isposinf_fpclass(float %x) {
-; RV32IF-LABEL: isposinf_fpclass:
+define float @acos_f32(float %a) nounwind {
+; RV32IF-LABEL: acos_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 128
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call acosf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isposinf_fpclass:
+; RV64IF-LABEL: acos_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 128
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call acosf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isposinf_fpclass:
+; RV32I-LABEL: acos_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 522240
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acosf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isposinf_fpclass:
+; RV64I-LABEL: acos_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 522240
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acosf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 512)  ; 0x200 = "+inf"
-  ret i1 %1
+  %1 = call float @llvm.acos.f32(float %a)
+  ret float %1
 }
 
-define i1 @isneginf_fpclass(float %x) {
-; RV32IF-LABEL: isneginf_fpclass:
+define float @atan_f32(float %a) nounwind {
+; RV32IF-LABEL: atan_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 1
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call atanf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isneginf_fpclass:
+; RV64IF-LABEL: atan_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 1
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call atanf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isneginf_fpclass:
+; RV32I-LABEL: atan_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 1046528
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atanf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isneginf_fpclass:
+; RV64I-LABEL: atan_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    li a1, 511
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    slli a1, a1, 23
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atanf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 4)  ; "-inf"
-  ret i1 %1
+  %1 = call float @llvm.atan.f32(float %a)
+  ret float %1
 }
 
-define i1 @isfinite_fpclass(float %x) {
-; RV32IF-LABEL: isfinite_fpclass:
+define float @atan2_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: atan2_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 126
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call atan2f
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isfinite_fpclass:
+; RV64IF-LABEL: atan2_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 126
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call atan2f
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isfinite_fpclass:
+; RV32I-LABEL: atan2_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan2f
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isfinite_fpclass:
+; RV64I-LABEL: atan2_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    sltu a0, a0, a2
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan2f
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 504)  ; 0x1f8 = "finite"
-  ret i1 %1
+  %1 = call float @llvm.atan2.f32(float %a, float %b)
+  ret float %1
 }
 
-define i1 @isposfinite_fpclass(float %x) {
-; RV32IF-LABEL: isposfinite_fpclass:
+define float @sinh_f32(float %a) nounwind {
+; RV32IF-LABEL: sinh_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 112
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call sinhf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isposfinite_fpclass:
+; RV64IF-LABEL: sinh_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 112
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call sinhf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isposfinite_fpclass:
+; RV32I-LABEL: sinh_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 522240
-; RV32I-NEXT:    sltu a0, a0, a1
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isposfinite_fpclass:
+; RV64I-LABEL: sinh_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 522240
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    sltu a0, a0, a1
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 448)  ; 0x1c0 = "+finite"
-  ret i1 %1
+  %1 = call float @llvm.sinh.f32(float %a)
+  ret float %1
 }
 
-define i1 @isnegfinite_fpclass(float %x) {
-; RV32IF-LABEL: isnegfinite_fpclass:
+define float @cosh_f32(float %a) nounwind {
+; RV32IF-LABEL: cosh_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 14
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call coshf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isnegfinite_fpclass:
+; RV64IF-LABEL: cosh_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 14
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call coshf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isnegfinite_fpclass:
+; RV32I-LABEL: cosh_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a1, a0, a1
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    sltu a1, a1, a2
-; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call coshf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isnegfinite_fpclass:
+; RV64I-LABEL: cosh_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a1, a0, a1
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    sltu a1, a1, a2
-; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call coshf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 56)  ; 0x38 = "-finite"
-  ret i1 %1
+  %1 = call float @llvm.cosh.f32(float %a)
+  ret float %1
 }
 
-define i1 @isnotfinite_fpclass(float %x) {
-; RV32IF-LABEL: isnotfinite_fpclass:
+define float @tanh_f32(float %a) nounwind {
+; RV32IF-LABEL: tanh_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fclass.s a0, fa0
-; RV32IF-NEXT:    andi a0, a0, 897
-; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call tanhf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
-; RV64IF-LABEL: isnotfinite_fpclass:
+; RV64IF-LABEL: tanh_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fclass.s a0, fa0
-; RV64IF-NEXT:    andi a0, a0, 897
-; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call tanhf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
-; RV32I-LABEL: isnotfinite_fpclass:
+; RV32I-LABEL: tanh_f32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    lui a2, 522240
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    xor a1, a0, a2
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    sltu a0, a2, a0
-; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: isnotfinite_fpclass:
+; RV64I-LABEL: tanh_f32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 524288
-; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    xor a1, a0, a2
-; RV64I-NEXT:    seqz a1, a1
-; RV64I-NEXT:    sltu a0, a2, a0
-; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
-  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 519)  ; ox207 = "inf|nan"
-  ret i1 %1
+  %1 = call float @llvm.tanh.f32(float %a)
+  ret float %1
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-zfa.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-zfa.ll
new file mode 100644
index 0000000000000..b95200bf90904
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-zfa.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=riscv32 -mattr=+zfa -global-isel < %s \
+; RUN: | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfa -global-isel < %s \
+; RUN: | FileCheck %s
+
+
+define float @fceil(float %a) {
+; CHECK-LABEL: fceil:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rup
+; CHECK-NEXT:    ret
+  %b = call float @llvm.ceil.f32(float %a)
+  ret float %b
+}
+
+define float @ffloor(float %a) {
+; CHECK-LABEL: ffloor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rdn
+; CHECK-NEXT:    ret
+  %b = call float @llvm.floor.f32(float %a)
+  ret float %b
+}
+
+define float @frint(float %a) {
+; CHECK-LABEL: frint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    froundnx.s fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.rint.f32(float %a)
+  ret float %b
+}
+
+define float @fnearbyint(float %a) {
+; CHECK-LABEL: fnearbyint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.nearbyint.f32(float %a)
+  ret float %b
+}
+
+define float @fround(float %a) {
+; CHECK-LABEL: fround:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rmm
+; CHECK-NEXT:    ret
+  %b = call float @llvm.round.f32(float %a)
+  ret float %b
+}
+
+define float @froundeven(float %a) {
+; CHECK-LABEL: froundeven:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rne
+; CHECK-NEXT:    ret
+  %b = call float @llvm.roundeven.f32(float %a)
+  ret float %b
+}
+
+define float @ftrunc(float %a) {
+; CHECK-LABEL: ftrunc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rtz
+; CHECK-NEXT:    ret
+  %b = call float @llvm.trunc.f32(float %a)
+  ret float %b
+}
+
+define float @fmaximum(float %a, float %b) {
+; CHECK-LABEL: fmaximum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmaxm.s fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call float @llvm.maximum.f32(float %a, float %b)
+  ret float %c
+}
+
+define float @fminimum(float %a, float %b) {
+; CHECK-LABEL: fminimum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fminm.s fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call float @llvm.minimum.f32(float %a, float %b)
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll b/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll
new file mode 100644
index 0000000000000..2fc25fbb39bb9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll
@@ -0,0 +1,536 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -global-isel < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64I %s
+; RUN: llc -mtriple=riscv64 -global-isel -mattr=+d < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK,RV64D %s
+
+; FIXME: Support RV32.
+
+define fp128 @fadd(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: fadd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+ %a = fadd fp128 %x, %y
+  ret fp128 %a
+}
+
+define fp128 @fsub(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: fsub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __subtf3
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fsub fp128 %x, %y
+  ret fp128 %a
+}
+
+define fp128 @fmul(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: fmul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __multf3
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fmul fp128 %x, %y
+  ret fp128 %a
+}
+
+define fp128 @fdiv(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: fdiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __divtf3
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fdiv fp128 %x, %y
+  ret fp128 %a
+}
+
+define fp128 @frem(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: frem:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call fmodl
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = frem fp128 %x, %y
+  ret fp128 %a
+}
+
+define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind {
+; CHECK-LABEL: fma:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call fmal
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = call fp128 @llvm.fma.f128(fp128 %x, fp128 %y, fp128 %z)
+  ret fp128 %a
+}
+
+define fp128 @fneg(fp128 %x) {
+; CHECK-LABEL: fneg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, -1
+; CHECK-NEXT:    slli a2, a2, 63
+; CHECK-NEXT:    xor a1, a1, a2
+; CHECK-NEXT:    ret
+  %a = fneg fp128 %x
+  ret fp128 %a
+}
+
+define fp128 @fabs(fp128 %x) {
+; CHECK-LABEL: fabs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    ret
+  %a = call fp128 @llvm.fabs.f128(fp128 %x)
+  ret fp128 %a
+}
+
+define fp128 @fcopysign(fp128 %x, fp128 %y) {
+; CHECK-LABEL: fcopysign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, -1
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    slli a2, a2, 63
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    and a2, a3, a2
+; CHECK-NEXT:    or a1, a1, a2
+; CHECK-NEXT:    ret
+  %a = call fp128 @llvm.copysign.f128(fp128 %x, fp128 %y)
+  ret fp128 %a
+}
+
+define i1 @fcmp(fp128 %x, fp128 %y) nounwind {
+; CHECK-LABEL: fcmp:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __eqtf2
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    srli a0, a0, 32
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fcmp oeq fp128 %x, %y
+  ret i1 %a
+}
+
+define fp128 @constant(fp128 %x) nounwind {
+; CHECK-LABEL: constant:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    lui a2, %hi(.LCPI10_0)
+; CHECK-NEXT:    addi a3, a2, %lo(.LCPI10_0)
+; CHECK-NEXT:    ld a2, 0(a3)
+; CHECK-NEXT:    ld a3, 8(a3)
+; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fadd fp128 %x, 0xL00000000000000007FFF000000000000
+  ret fp128 %a
+}
+
+define fp128 @fpext_f32(float %x, float %y) nounwind {
+; RV64I-LABEL: fpext_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call __extendsftf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64D-LABEL: fpext_f32:
+; RV64D:       # %bb.0:
+; RV64D-NEXT:    addi sp, sp, -16
+; RV64D-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fadd.s fa0, fa0, fa1
+; RV64D-NEXT:    call __extendsftf2
+; RV64D-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    addi sp, sp, 16
+; RV64D-NEXT:    ret
+  %a = fadd float %x, %y
+  %b = fpext float %a to fp128
+  ret fp128 %b
+}
+
+define fp128 @fpext_f64(double %x, double %y) nounwind {
+; RV64I-LABEL: fpext_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    call __extenddftf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64D-LABEL: fpext_f64:
+; RV64D:       # %bb.0:
+; RV64D-NEXT:    addi sp, sp, -16
+; RV64D-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fadd.d fa0, fa0, fa1
+; RV64D-NEXT:    call __extenddftf2
+; RV64D-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    addi sp, sp, 16
+; RV64D-NEXT:    ret
+  %a = fadd double %x, %y
+  %b = fpext double %a to fp128
+  ret fp128 %b
+}
+
+define float @fptrunc_f32(fp128 %x, float %y) nounwind {
+; RV64I-LABEL: fptrunc_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    call __trunctfsf2
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64D-LABEL: fptrunc_f32:
+; RV64D:       # %bb.0:
+; RV64D-NEXT:    addi sp, sp, -16
+; RV64D-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fmv.s fs0, fa0
+; RV64D-NEXT:    call __trunctfsf2
+; RV64D-NEXT:    fadd.s fa0, fa0, fs0
+; RV64D-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    addi sp, sp, 16
+; RV64D-NEXT:    ret
+  %a = fptrunc fp128 %x to float
+  %b = fadd float %a, %y
+  ret float %b
+}
+
+define double @fptrunc_f64(fp128 %x, double %y) nounwind {
+; RV64I-LABEL: fptrunc_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    call __trunctfdf2
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __adddf3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64D-LABEL: fptrunc_f64:
+; RV64D:       # %bb.0:
+; RV64D-NEXT:    addi sp, sp, -16
+; RV64D-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    fmv.d fs0, fa0
+; RV64D-NEXT:    call __trunctfdf2
+; RV64D-NEXT:    fadd.d fa0, fa0, fs0
+; RV64D-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    addi sp, sp, 16
+; RV64D-NEXT:    ret
+  %a = fptrunc fp128 %x to double
+  %b = fadd double %a, %y
+  ret double %b
+}
+
+define i8 @fptosi_i8(fp128 %x) nounwind {
+; CHECK-LABEL: fptosi_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixtfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptosi fp128 %x to i8
+  ret i8 %a
+}
+
+define i16 @fptosi_i16(fp128 %x) nounwind {
+; CHECK-LABEL: fptosi_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixtfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptosi fp128 %x to i16
+  ret i16 %a
+}
+
+define i32 @fptosi_i32(fp128 %x) nounwind {
+; CHECK-LABEL: fptosi_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixtfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptosi fp128 %x to i32
+  ret i32 %a
+}
+
+define i64 @fptosi_i64(fp128 %x) nounwind {
+; CHECK-LABEL: fptosi_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixtfdi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptosi fp128 %x to i64
+  ret i64 %a
+}
+
+define i128 @fptosi_i128(fp128 %x) nounwind {
+; CHECK-LABEL: fptosi_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixtfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptosi fp128 %x to i128
+  ret i128 %a
+}
+
+define i8 @fptoui_i8(fp128 %x) nounwind {
+; CHECK-LABEL: fptoui_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunstfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptoui fp128 %x to i8
+  ret i8 %a
+}
+
+define i16 @fptoui_i16(fp128 %x) nounwind {
+; CHECK-LABEL: fptoui_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunstfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptoui fp128 %x to i16
+  ret i16 %a
+}
+
+define i32 @fptoui_i32(fp128 %x) nounwind {
+; CHECK-LABEL: fptoui_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunstfsi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptoui fp128 %x to i32
+  ret i32 %a
+}
+
+define i64 @fptoui_i64(fp128 %x) nounwind {
+; CHECK-LABEL: fptoui_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunstfdi
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptoui fp128 %x to i64
+  ret i64 %a
+}
+
+define i128 @fptoui_i128(fp128 %x) nounwind {
+; CHECK-LABEL: fptoui_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunstfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = fptoui fp128 %x to i128
+  ret i128 %a
+}
+
+define fp128 @sitofp_i8(i8 %x) nounwind {
+; CHECK-LABEL: sitofp_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    slli a0, a0, 56
+; CHECK-NEXT:    srai a0, a0, 56
+; CHECK-NEXT:    call __floatsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = sitofp i8 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @sitofp_i16(i16 %x) nounwind {
+; CHECK-LABEL: sitofp_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    slli a0, a0, 48
+; CHECK-NEXT:    srai a0, a0, 48
+; CHECK-NEXT:    call __floatsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = sitofp i16 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @sitofp_i32(i32 %x) nounwind {
+; CHECK-LABEL: sitofp_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sext.w a0, a0
+; CHECK-NEXT:    call __floatsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = sitofp i32 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @sitofp_i64(i64 %x) nounwind {
+; CHECK-LABEL: sitofp_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __floatditf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = sitofp i64 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @sitofp_i128(i128 %x) nounwind {
+; CHECK-LABEL: sitofp_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __floattitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = sitofp i128 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @uitofp_i8(i8 %x) nounwind {
+; CHECK-LABEL: uitofp_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    andi a0, a0, 255
+; CHECK-NEXT:    call __floatunsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = uitofp i8 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @uitofp_i16(i16 %x) nounwind {
+; CHECK-LABEL: uitofp_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    slli a0, a0, 48
+; CHECK-NEXT:    srli a0, a0, 48
+; CHECK-NEXT:    call __floatunsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = uitofp i16 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @uitofp_i32(i32 %x) nounwind {
+; CHECK-LABEL: uitofp_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sext.w a0, a0
+; CHECK-NEXT:    call __floatunsitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = uitofp i32 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @uitofp_i64(i64 %x) nounwind {
+; CHECK-LABEL: uitofp_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __floatunditf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = uitofp i64 %x to fp128
+  ret fp128 %a
+}
+
+define fp128 @uitofp_i128(i128 %x) nounwind {
+; CHECK-LABEL: uitofp_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __floatuntitf
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %a = uitofp i128 %x to fp128
+  ret fp128 %a
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/half-zfa.ll b/llvm/test/CodeGen/RISCV/GlobalISel/half-zfa.ll
new file mode 100644
index 0000000000000..2ccb3c63e3f21
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/half-zfa.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=riscv32 -mattr=+zfa,+zfh -global-isel < %s \
+; RUN: | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfa,+zfh -global-isel < %s \
+; RUN: | FileCheck %s
+
+
+define half @fceil(half %a) {
+; CHECK-LABEL: fceil:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rup
+; CHECK-NEXT:    ret
+  %b = call half @llvm.ceil.f16(half %a)
+  ret half %b
+}
+
+define half @ffloor(half %a) {
+; CHECK-LABEL: ffloor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rdn
+; CHECK-NEXT:    ret
+  %b = call half @llvm.floor.f16(half %a)
+  ret half %b
+}
+
+define half @frint(half %a) {
+; CHECK-LABEL: frint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    froundnx.h fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call half @llvm.rint.f16(half %a)
+  ret half %b
+}
+
+define half @fnearbyint(half %a) {
+; CHECK-LABEL: fnearbyint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0
+; CHECK-NEXT:    ret
+  %b = call half @llvm.nearbyint.f16(half %a)
+  ret half %b
+}
+
+define half @fround(half %a) {
+; CHECK-LABEL: fround:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rmm
+; CHECK-NEXT:    ret
+  %b = call half @llvm.round.f16(half %a)
+  ret half %b
+}
+
+define half @froundeven(half %a) {
+; CHECK-LABEL: froundeven:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rne
+; CHECK-NEXT:    ret
+  %b = call half @llvm.roundeven.f16(half %a)
+  ret half %b
+}
+
+define half @ftrunc(half %a) {
+; CHECK-LABEL: ftrunc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rtz
+; CHECK-NEXT:    ret
+  %b = call half @llvm.trunc.f16(half %a)
+  ret half %b
+}
+
+define half @fmaximum(half %a, half %b) {
+; CHECK-LABEL: fmaximum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmaxm.h fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call half @llvm.maximum.f16(half %a, half %b)
+  ret half %c
+}
+
+define half @fminimum(half %a, half %b) {
+; CHECK-LABEL: fminimum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fminm.h fa0, fa0, fa1
+; CHECK-NEXT:    ret
+  %c = call half @llvm.minimum.f16(half %a, half %b)
+  ret half %c
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv32.mir
index 39d0ee7c382df..e21be2afebd20 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv32.mir
@@ -102,10 +102,9 @@ body:             |
     ; RV32I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:gpr = ADD [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[LUI:%[0-9]+]]:gpr = LUI 16
-    ; RV32I-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], -1
-    ; RV32I-NEXT: [[AND:%[0-9]+]]:gpr = AND [[ADD]], [[ADDI]]
-    ; RV32I-NEXT: $x10 = COPY [[AND]]
+    ; RV32I-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[ADD]], 16
+    ; RV32I-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[SLLI]], 16
+    ; RV32I-NEXT: $x10 = COPY [[SRLI]]
     ; RV32I-NEXT: PseudoRET implicit $x10
     %0:gprb(s32) = COPY $x10
     %1:gprb(s32) = COPY $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index fb00e4860ddf2..3fcaa81e1a552 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -825,6 +825,299 @@ define i32 @caller_many_scalars() nounwind {
   ret i32 %1
 }
 
+
+; Check that i128 and fp128 are passed indirectly
+
+define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
+  ; RV32I-LABEL: name: callee_large_scalars
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY]](p0) :: (load (s128), align 8)
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
+  ; RV32I-NEXT:   [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128))
+  ; RV32I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s128), [[LOAD1]]
+  ; RV32I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV32I-NEXT:   $x10 = COPY [[ZEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %b_bitcast = bitcast fp128 %b to i128
+  %1 = icmp eq i128 %a, %b_bitcast
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @caller_large_scalars() nounwind {
+  ; ILP32-LABEL: name: caller_large_scalars
+  ; ILP32: bb.1 (%ir-block.0):
+  ; ILP32-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; ILP32-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32-NEXT:   G_STORE [[C]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32-NEXT:   G_STORE [[C1]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; ILP32-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32F-LABEL: name: caller_large_scalars
+  ; ILP32F: bb.1 (%ir-block.0):
+  ; ILP32F-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; ILP32F-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32F-NEXT:   G_STORE [[C]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32F-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32F-NEXT:   G_STORE [[C1]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32F-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32F-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; ILP32F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32F-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32D-LABEL: name: caller_large_scalars
+  ; ILP32D: bb.1 (%ir-block.0):
+  ; ILP32D-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; ILP32D-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32D-NEXT:   G_STORE [[C]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32D-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32D-NEXT:   G_STORE [[C1]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32D-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32D-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; ILP32D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32D-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32D-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_large_scalars(i128 1, fp128 0xL00000000000000007FFF000000000000)
+  ret i32 %1
+}
+
+; Check that arguments larger than 2*xlen are handled correctly when their
+; address is passed on the stack rather than in memory
+
+; Must keep define on a single line due to an update_llc_test_checks.py limitation
+define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i128 %h, i32 %i, fp128 %j) nounwind {
+  ; RV32I-LABEL: name: callee_large_scalars_exhausted_regs
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32I-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32I-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32I-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32I-NEXT:   [[COPY7:%[0-9]+]]:_(p0) = COPY $x17
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY7]](p0) :: (load (s128), align 8)
+  ; RV32I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; RV32I-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s32) from %fixed-stack.1, align 16)
+  ; RV32I-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV32I-NEXT:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (load (p0) from %fixed-stack.0)
+  ; RV32I-NEXT:   [[LOAD3:%[0-9]+]]:_(s128) = G_LOAD [[LOAD2]](p0) :: (load (s128))
+  ; RV32I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s128), [[LOAD3]]
+  ; RV32I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV32I-NEXT:   $x10 = COPY [[ZEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %j_bitcast = bitcast fp128 %j to i128
+  %1 = icmp eq i128 %h, %j_bitcast
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @caller_large_scalars_exhausted_regs() nounwind {
+  ; ILP32-LABEL: name: caller_large_scalars_exhausted_regs
+  ; ILP32: bb.1 (%ir-block.0):
+  ; ILP32-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; ILP32-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; ILP32-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; ILP32-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; ILP32-NEXT:   [[C7:%[0-9]+]]:_(s128) = G_CONSTANT i128 8
+  ; ILP32-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+  ; ILP32-NEXT:   [[C9:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32-NEXT:   G_STORE [[C7]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; ILP32-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; ILP32-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s32)
+  ; ILP32-NEXT:   G_STORE [[C8]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; ILP32-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32-NEXT:   G_STORE [[C9]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s32)
+  ; ILP32-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 4)
+  ; ILP32-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32-NEXT:   $x12 = COPY [[C2]](s32)
+  ; ILP32-NEXT:   $x13 = COPY [[C3]](s32)
+  ; ILP32-NEXT:   $x14 = COPY [[C4]](s32)
+  ; ILP32-NEXT:   $x15 = COPY [[C5]](s32)
+  ; ILP32-NEXT:   $x16 = COPY [[C6]](s32)
+  ; ILP32-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; ILP32-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; ILP32-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32F-LABEL: name: caller_large_scalars_exhausted_regs
+  ; ILP32F: bb.1 (%ir-block.0):
+  ; ILP32F-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32F-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32F-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; ILP32F-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32F-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; ILP32F-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; ILP32F-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; ILP32F-NEXT:   [[C7:%[0-9]+]]:_(s128) = G_CONSTANT i128 8
+  ; ILP32F-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+  ; ILP32F-NEXT:   [[C9:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32F-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32F-NEXT:   G_STORE [[C7]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32F-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; ILP32F-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; ILP32F-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s32)
+  ; ILP32F-NEXT:   G_STORE [[C8]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; ILP32F-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32F-NEXT:   G_STORE [[C9]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32F-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32F-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s32)
+  ; ILP32F-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 4)
+  ; ILP32F-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32F-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32F-NEXT:   $x12 = COPY [[C2]](s32)
+  ; ILP32F-NEXT:   $x13 = COPY [[C3]](s32)
+  ; ILP32F-NEXT:   $x14 = COPY [[C4]](s32)
+  ; ILP32F-NEXT:   $x15 = COPY [[C5]](s32)
+  ; ILP32F-NEXT:   $x16 = COPY [[C6]](s32)
+  ; ILP32F-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; ILP32F-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32F-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; ILP32F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32D-LABEL: name: caller_large_scalars_exhausted_regs
+  ; ILP32D: bb.1 (%ir-block.0):
+  ; ILP32D-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32D-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32D-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; ILP32D-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32D-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; ILP32D-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; ILP32D-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; ILP32D-NEXT:   [[C7:%[0-9]+]]:_(s128) = G_CONSTANT i128 8
+  ; ILP32D-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+  ; ILP32D-NEXT:   [[C9:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; ILP32D-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; ILP32D-NEXT:   G_STORE [[C7]](s128), [[FRAME_INDEX]](p0) :: (store (s128) into %stack.0, align 8)
+  ; ILP32D-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; ILP32D-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; ILP32D-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s32)
+  ; ILP32D-NEXT:   G_STORE [[C8]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; ILP32D-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; ILP32D-NEXT:   G_STORE [[C9]](s128), [[FRAME_INDEX1]](p0) :: (store (s128) into %stack.1)
+  ; ILP32D-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; ILP32D-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s32)
+  ; ILP32D-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 4)
+  ; ILP32D-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32D-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32D-NEXT:   $x12 = COPY [[C2]](s32)
+  ; ILP32D-NEXT:   $x13 = COPY [[C3]](s32)
+  ; ILP32D-NEXT:   $x14 = COPY [[C4]](s32)
+  ; ILP32D-NEXT:   $x15 = COPY [[C5]](s32)
+  ; ILP32D-NEXT:   $x16 = COPY [[C6]](s32)
+  ; ILP32D-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; ILP32D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; ILP32D-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32D-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; ILP32D-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_large_scalars_exhausted_regs(
+      i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i128 8, i32 9,
+      fp128 0xL00000000000000007FFF000000000000)
+  ret i32 %1
+}
+
+; Check passing of coerced integer arrays
+
+define i32 @callee_small_coerced_struct([2 x i32] %a.coerce) nounwind {
+  ; RV32I-LABEL: name: callee_small_coerced_struct
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+  ; RV32I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV32I-NEXT:   $x10 = COPY [[ZEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = extractvalue [2 x i32] %a.coerce, 0
+  %2 = extractvalue [2 x i32] %a.coerce, 1
+  %3 = icmp eq i32 %1, %2
+  %4 = zext i1 %3 to i32
+  ret i32 %4
+}
+
+define i32 @caller_small_coerced_struct() nounwind {
+  ; ILP32-LABEL: name: caller_small_coerced_struct
+  ; ILP32: bb.1 (%ir-block.0):
+  ; ILP32-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32F-LABEL: name: caller_small_coerced_struct
+  ; ILP32F: bb.1 (%ir-block.0):
+  ; ILP32F-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32F-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32F-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32F-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32F-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; ILP32D-LABEL: name: caller_small_coerced_struct
+  ; ILP32D: bb.1 (%ir-block.0):
+  ; ILP32D-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; ILP32D-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; ILP32D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   $x10 = COPY [[C]](s32)
+  ; ILP32D-NEXT:   $x11 = COPY [[C1]](s32)
+  ; ILP32D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; ILP32D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; ILP32D-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; ILP32D-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; ILP32D-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_small_coerced_struct([2 x i32] [i32 1, i32 2])
+  ret i32 %1
+}
+
 ; Check return of 2x xlen scalars
 
 define i64 @callee_small_scalar_ret() nounwind {
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
index 3a0831b3c5214..17c6e55fa8d2c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -83,6 +83,82 @@ define i64 @caller_i128_in_regs() nounwind {
   ret i64 %1
 }
 
+define i64 @callee_i128_fp128_in_regs(i128 %a, fp128 %b) nounwind {
+  ; RV64I-LABEL: name: callee_i128_fp128_in_regs
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11, $x12, $x13
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[COPY1]](s64)
+  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; RV64I-NEXT:   [[COPY3:%[0-9]+]]:_(s64) = COPY $x13
+  ; RV64I-NEXT:   [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY2]](s64), [[COPY3]](s64)
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[MV]](s128), [[MV1]]
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   $x10 = COPY [[ZEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %b_bitcast = bitcast fp128 %b to i128
+  %1 = icmp eq i128 %a, %b_bitcast
+  %2 = zext i1 %1 to i64
+  ret i64 %2
+}
+
+define i64 @caller_i128_fp128_in_regs() nounwind {
+  ; LP64-LABEL: name: caller_i128_fp128_in_regs
+  ; LP64: bb.1 (%ir-block.0):
+  ; LP64-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; LP64-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; LP64-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C]](s128)
+  ; LP64-NEXT:   [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C1]](s128)
+  ; LP64-NEXT:   $x10 = COPY [[UV]](s64)
+  ; LP64-NEXT:   $x11 = COPY [[UV1]](s64)
+  ; LP64-NEXT:   $x12 = COPY [[UV2]](s64)
+  ; LP64-NEXT:   $x13 = COPY [[UV3]](s64)
+  ; LP64-NEXT:   PseudoCALL target-flags(riscv-call) @callee_i128_fp128_in_regs, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10
+  ; LP64-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64F-LABEL: name: caller_i128_fp128_in_regs
+  ; LP64F: bb.1 (%ir-block.0):
+  ; LP64F-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; LP64F-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; LP64F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C]](s128)
+  ; LP64F-NEXT:   [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C1]](s128)
+  ; LP64F-NEXT:   $x10 = COPY [[UV]](s64)
+  ; LP64F-NEXT:   $x11 = COPY [[UV1]](s64)
+  ; LP64F-NEXT:   $x12 = COPY [[UV2]](s64)
+  ; LP64F-NEXT:   $x13 = COPY [[UV3]](s64)
+  ; LP64F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_i128_fp128_in_regs, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10
+  ; LP64F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64F-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64D-LABEL: name: caller_i128_fp128_in_regs
+  ; LP64D: bb.1 (%ir-block.0):
+  ; LP64D-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 1
+  ; LP64D-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; LP64D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C]](s128)
+  ; LP64D-NEXT:   [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C1]](s128)
+  ; LP64D-NEXT:   $x10 = COPY [[UV]](s64)
+  ; LP64D-NEXT:   $x11 = COPY [[UV1]](s64)
+  ; LP64D-NEXT:   $x12 = COPY [[UV2]](s64)
+  ; LP64D-NEXT:   $x13 = COPY [[UV3]](s64)
+  ; LP64D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_i128_fp128_in_regs, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10
+  ; LP64D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64D-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64D-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_i128_fp128_in_regs(i128 1, fp128 0xL00000000000000007FFF000000000000)
+  ret i64 %1
+}
+
 ; Check the correct handling of passing of values that are larger that 2*XLen.
 
 define i32 @caller_i256_indirect_reference_in_stack() {
@@ -480,6 +556,296 @@ define i32 @caller_many_scalars() nounwind {
   ret i32 %1
 }
 
+; Check that i256 is passed indirectly.
+
+define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
+  ; RV64I-LABEL: name: callee_large_scalars
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+  ; RV64I-NEXT:   [[LOAD:%[0-9]+]]:_(s256) = G_LOAD [[COPY]](p0) :: (load (s256), align 16)
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
+  ; RV64I-NEXT:   [[LOAD1:%[0-9]+]]:_(s256) = G_LOAD [[COPY1]](p0) :: (load (s256), align 16)
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s256), [[LOAD1]]
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   $x10 = COPY [[ZEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = icmp eq i256 %a, %b
+  %2 = zext i1 %1 to i64
+  ret i64 %2
+}
+
+define i64 @caller_large_scalars() nounwind {
+  ; LP64-LABEL: name: caller_large_scalars
+  ; LP64: bb.1 (%ir-block.0):
+  ; LP64-NEXT:   [[C:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+  ; LP64-NEXT:   [[C1:%[0-9]+]]:_(s256) = G_CONSTANT i256 2
+  ; LP64-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64-NEXT:   G_STORE [[C]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64-NEXT:   G_STORE [[C1]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; LP64-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; LP64-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64F-LABEL: name: caller_large_scalars
+  ; LP64F: bb.1 (%ir-block.0):
+  ; LP64F-NEXT:   [[C:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+  ; LP64F-NEXT:   [[C1:%[0-9]+]]:_(s256) = G_CONSTANT i256 2
+  ; LP64F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64F-NEXT:   G_STORE [[C]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64F-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64F-NEXT:   G_STORE [[C1]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64F-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; LP64F-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; LP64F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64F-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64D-LABEL: name: caller_large_scalars
+  ; LP64D: bb.1 (%ir-block.0):
+  ; LP64D-NEXT:   [[C:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+  ; LP64D-NEXT:   [[C1:%[0-9]+]]:_(s256) = G_CONSTANT i256 2
+  ; LP64D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64D-NEXT:   G_STORE [[C]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64D-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64D-NEXT:   G_STORE [[C1]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64D-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; LP64D-NEXT:   $x11 = COPY [[FRAME_INDEX1]](p0)
+  ; LP64D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64D-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64D-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_large_scalars(i256 1, i256 2)
+  ret i64 %1
+}
+
+; Check that arguments larger than 2*xlen are handled correctly when their
+; address is passed on the stack rather than in memory
+
+; Must keep define on a single line due to an update_llc_test_checks.py limitation
+define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i256 %h, i64 %i, i256 %j) nounwind {
+  ; RV64I-LABEL: name: callee_large_scalars_exhausted_regs
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; RV64I-NEXT:   [[COPY3:%[0-9]+]]:_(s64) = COPY $x13
+  ; RV64I-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY $x14
+  ; RV64I-NEXT:   [[COPY5:%[0-9]+]]:_(s64) = COPY $x15
+  ; RV64I-NEXT:   [[COPY6:%[0-9]+]]:_(s64) = COPY $x16
+  ; RV64I-NEXT:   [[COPY7:%[0-9]+]]:_(p0) = COPY $x17
+  ; RV64I-NEXT:   [[LOAD:%[0-9]+]]:_(s256) = G_LOAD [[COPY7]](p0) :: (load (s256), align 16)
+  ; RV64I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; RV64I-NEXT:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s64) from %fixed-stack.1, align 16)
+  ; RV64I-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV64I-NEXT:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (load (p0) from %fixed-stack.0)
+  ; RV64I-NEXT:   [[LOAD3:%[0-9]+]]:_(s256) = G_LOAD [[LOAD2]](p0) :: (load (s256), align 16)
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s256), [[LOAD3]]
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   $x10 = COPY [[ZEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = icmp eq i256 %h, %j
+  %2 = zext i1 %1 to i64
+  ret i64 %2
+}
+
+define i64 @caller_large_scalars_exhausted_regs() nounwind {
+  ; LP64-LABEL: name: caller_large_scalars_exhausted_regs
+  ; LP64: bb.1 (%ir-block.0):
+  ; LP64-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+  ; LP64-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; LP64-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+  ; LP64-NEXT:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
+  ; LP64-NEXT:   [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+  ; LP64-NEXT:   [[C7:%[0-9]+]]:_(s256) = G_CONSTANT i256 8
+  ; LP64-NEXT:   [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 9
+  ; LP64-NEXT:   [[C9:%[0-9]+]]:_(s256) = G_CONSTANT i256 10
+  ; LP64-NEXT:   ADJCALLSTACKDOWN 16, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64-NEXT:   G_STORE [[C7]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; LP64-NEXT:   [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; LP64-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s64)
+  ; LP64-NEXT:   G_STORE [[C8]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
+  ; LP64-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64-NEXT:   G_STORE [[C9]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64-NEXT:   [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; LP64-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s64)
+  ; LP64-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 8)
+  ; LP64-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64-NEXT:   $x12 = COPY [[C2]](s64)
+  ; LP64-NEXT:   $x13 = COPY [[C3]](s64)
+  ; LP64-NEXT:   $x14 = COPY [[C4]](s64)
+  ; LP64-NEXT:   $x15 = COPY [[C5]](s64)
+  ; LP64-NEXT:   $x16 = COPY [[C6]](s64)
+  ; LP64-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; LP64-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; LP64-NEXT:   ADJCALLSTACKUP 16, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64-NEXT:   $x10 = COPY [[COPY1]](s64)
+  ; LP64-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64F-LABEL: name: caller_large_scalars_exhausted_regs
+  ; LP64F: bb.1 (%ir-block.0):
+  ; LP64F-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64F-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64F-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+  ; LP64F-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; LP64F-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+  ; LP64F-NEXT:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
+  ; LP64F-NEXT:   [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+  ; LP64F-NEXT:   [[C7:%[0-9]+]]:_(s256) = G_CONSTANT i256 8
+  ; LP64F-NEXT:   [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 9
+  ; LP64F-NEXT:   [[C9:%[0-9]+]]:_(s256) = G_CONSTANT i256 10
+  ; LP64F-NEXT:   ADJCALLSTACKDOWN 16, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64F-NEXT:   G_STORE [[C7]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64F-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; LP64F-NEXT:   [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; LP64F-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s64)
+  ; LP64F-NEXT:   G_STORE [[C8]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
+  ; LP64F-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64F-NEXT:   G_STORE [[C9]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64F-NEXT:   [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; LP64F-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s64)
+  ; LP64F-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 8)
+  ; LP64F-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64F-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64F-NEXT:   $x12 = COPY [[C2]](s64)
+  ; LP64F-NEXT:   $x13 = COPY [[C3]](s64)
+  ; LP64F-NEXT:   $x14 = COPY [[C4]](s64)
+  ; LP64F-NEXT:   $x15 = COPY [[C5]](s64)
+  ; LP64F-NEXT:   $x16 = COPY [[C6]](s64)
+  ; LP64F-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; LP64F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; LP64F-NEXT:   ADJCALLSTACKUP 16, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64F-NEXT:   $x10 = COPY [[COPY1]](s64)
+  ; LP64F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64D-LABEL: name: caller_large_scalars_exhausted_regs
+  ; LP64D: bb.1 (%ir-block.0):
+  ; LP64D-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64D-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64D-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+  ; LP64D-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; LP64D-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+  ; LP64D-NEXT:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
+  ; LP64D-NEXT:   [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+  ; LP64D-NEXT:   [[C7:%[0-9]+]]:_(s256) = G_CONSTANT i256 8
+  ; LP64D-NEXT:   [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 9
+  ; LP64D-NEXT:   [[C9:%[0-9]+]]:_(s256) = G_CONSTANT i256 10
+  ; LP64D-NEXT:   ADJCALLSTACKDOWN 16, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; LP64D-NEXT:   G_STORE [[C7]](s256), [[FRAME_INDEX]](p0) :: (store (s256) into %stack.0, align 16)
+  ; LP64D-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; LP64D-NEXT:   [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; LP64D-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C10]](s64)
+  ; LP64D-NEXT:   G_STORE [[C8]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
+  ; LP64D-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+  ; LP64D-NEXT:   G_STORE [[C9]](s256), [[FRAME_INDEX1]](p0) :: (store (s256) into %stack.1, align 16)
+  ; LP64D-NEXT:   [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; LP64D-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C11]](s64)
+  ; LP64D-NEXT:   G_STORE [[FRAME_INDEX1]](p0), [[PTR_ADD1]](p0) :: (store (p0) into stack + 8)
+  ; LP64D-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64D-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64D-NEXT:   $x12 = COPY [[C2]](s64)
+  ; LP64D-NEXT:   $x13 = COPY [[C3]](s64)
+  ; LP64D-NEXT:   $x14 = COPY [[C4]](s64)
+  ; LP64D-NEXT:   $x15 = COPY [[C5]](s64)
+  ; LP64D-NEXT:   $x16 = COPY [[C6]](s64)
+  ; LP64D-NEXT:   $x17 = COPY [[FRAME_INDEX]](p0)
+  ; LP64D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_scalars_exhausted_regs, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; LP64D-NEXT:   ADJCALLSTACKUP 16, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64D-NEXT:   $x10 = COPY [[COPY1]](s64)
+  ; LP64D-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_large_scalars_exhausted_regs(
+      i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i256 8, i64 9,
+      i256 10)
+  ret i64 %1
+}
+
+; Check passing of coerced integer arrays
+
+define i64 @callee_small_coerced_struct([2 x i64] %a.coerce) nounwind {
+  ; RV64I-LABEL: name: callee_small_coerced_struct
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s64), [[COPY1]]
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   $x10 = COPY [[ZEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = extractvalue [2 x i64] %a.coerce, 0
+  %2 = extractvalue [2 x i64] %a.coerce, 1
+  %3 = icmp eq i64 %1, %2
+  %4 = zext i1 %3 to i64
+  ret i64 %4
+}
+
+define i64 @caller_small_coerced_struct() nounwind {
+  ; LP64-LABEL: name: caller_small_coerced_struct
+  ; LP64: bb.1 (%ir-block.0):
+  ; LP64-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64F-LABEL: name: caller_small_coerced_struct
+  ; LP64F: bb.1 (%ir-block.0):
+  ; LP64F-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64F-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64F-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64F-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64F-NEXT:   PseudoRET implicit $x10
+  ;
+  ; LP64D-LABEL: name: caller_small_coerced_struct
+  ; LP64D: bb.1 (%ir-block.0):
+  ; LP64D-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; LP64D-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; LP64D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   $x10 = COPY [[C]](s64)
+  ; LP64D-NEXT:   $x11 = COPY [[C1]](s64)
+  ; LP64D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_coerced_struct, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+  ; LP64D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64D-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; LP64D-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_small_coerced_struct([2 x i64] [i64 1, i64 2])
+  ret i64 %1
+}
+
 ; Check return of 2x xlen scalars
 
 define i128 @callee_small_scalar_ret() nounwind {
@@ -541,6 +907,51 @@ define i64 @caller_small_scalar_ret() nounwind {
   ret i64 %3
 }
 
+define fp128 @callee_fp128_ret() nounwind {
+  ; RV64I-LABEL: name: callee_fp128_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s128) = G_FCONSTANT fp128 0xL00000000000000007FFF000000000000
+  ; RV64I-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C]](s128)
+  ; RV64I-NEXT:   $x10 = COPY [[UV]](s64)
+  ; RV64I-NEXT:   $x11 = COPY [[UV1]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10, implicit $x11
+  ret fp128 0xL00000000000000007FFF000000000000
+}
+
+define void @caller_fp128_ret() nounwind {
+  ; LP64-LABEL: name: caller_fp128_ret
+  ; LP64: bb.1 (%ir-block.0):
+  ; LP64-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   PseudoCALL target-flags(riscv-call) @callee_fp128_ret, csr_ilp32_lp64, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; LP64-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; LP64-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[COPY1]](s64)
+  ; LP64-NEXT:   PseudoRET
+  ;
+  ; LP64F-LABEL: name: caller_fp128_ret
+  ; LP64F: bb.1 (%ir-block.0):
+  ; LP64F-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   PseudoCALL target-flags(riscv-call) @callee_fp128_ret, csr_ilp32f_lp64f, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; LP64F-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64F-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64F-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; LP64F-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[COPY1]](s64)
+  ; LP64F-NEXT:   PseudoRET
+  ;
+  ; LP64D-LABEL: name: caller_fp128_ret
+  ; LP64D: bb.1 (%ir-block.0):
+  ; LP64D-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   PseudoCALL target-flags(riscv-call) @callee_fp128_ret, csr_ilp32d_lp64d, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; LP64D-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; LP64D-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; LP64D-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; LP64D-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[COPY1]](s64)
+  ; LP64D-NEXT:   PseudoRET
+  %1 = call fp128 @callee_fp128_ret()
+  ret void
+}
+
 ; Check return of 2x xlen structs
 
 %struct.small = type { i64, ptr }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calls.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calls.ll
index b06b539ded19b..34e29718d0e9a 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calls.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calls.ll
@@ -350,31 +350,6 @@ entry:
 %struct.Foo = type { i32, i32, i32, i16, i8 }
 @foo = global %struct.Foo { i32 1, i32 2, i32 3, i16 4, i8 5 }, align 4
 
-declare void @void_byval_args(ptr byval(%struct.Foo) %f)
-
-define void @test_void_byval_args() {
-  ; RV32I-LABEL: name: test_void_byval_args
-  ; RV32I: bb.1.entry:
-  ; RV32I-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @foo
-  ; RV32I-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-  ; RV32I-NEXT:   $x10 = COPY [[GV]](p0)
-  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @void_byval_args, csr_ilp32_lp64, implicit-def $x1, implicit $x10
-  ; RV32I-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-  ; RV32I-NEXT:   PseudoRET
-  ;
-  ; RV64I-LABEL: name: test_void_byval_args
-  ; RV64I: bb.1.entry:
-  ; RV64I-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @foo
-  ; RV64I-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-  ; RV64I-NEXT:   $x10 = COPY [[GV]](p0)
-  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @void_byval_args, csr_ilp32_lp64, implicit-def $x1, implicit $x10
-  ; RV64I-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-  ; RV64I-NEXT:   PseudoRET
-entry:
-  call void @void_byval_args(ptr byval(%struct.Foo) @foo)
-  ret void
-}
-
 declare void @void_sret_args(ptr sret(%struct.Foo) %f)
 
 define void @test_void_sret_args() {
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/knownbits-copy-crash.mir b/llvm/test/CodeGen/RISCV/GlobalISel/knownbits-copy-crash.mir
new file mode 100644
index 0000000000000..99be88b55ea81
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/knownbits-copy-crash.mir
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -mattr=+zbb,+f -run-pass=instruction-select %s -o - | FileCheck %s
+
+---
+name:            foo
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: foo
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY]], %bb.0, %6, %bb.1
+  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = ADDIW [[PHI]], 0
+  ; CHECK-NEXT:   [[FCVT_S_W:%[0-9]+]]:fpr32 = nofpexcept FCVT_S_W [[ADDIW]], 7
+  ; CHECK-NEXT:   [[FCVT_W_S:%[0-9]+]]:gpr = nofpexcept FCVT_W_S [[FCVT_S_W]], 1
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  bb.1:
+    %7:gprb(s64) = G_CONSTANT i64 0
+    %3:gprb(s32) = G_TRUNC %7(s64)
+
+  bb.2:
+    %0:gprb(s32) = G_PHI %3(s32), %bb.1, %2(s32), %bb.2
+    %6:gprb(s64) = G_SEXT %0(s32)
+    %1:fprb(s32) = G_SITOFP %6(s64)
+    %5:gprb(s64) = G_FCVT_W_RV64 %1(s32), 1
+    %2:gprb(s32) = G_TRUNC %5(s64)
+    G_BR %bb.2
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 74749d8f1944b..a27e2b80cd98f 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -73,6 +73,14 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 #
+# DEBUG-NEXT: G_ABDS (opcode 65): 1 type index, 0 imm indices
+# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+#
+# DEBUG-NEXT:G_ABDU (opcode 66): 1 type index, 0 imm indices
+# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+#
 # DEBUG-NEXT: G_IMPLICIT_DEF (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
@@ -356,11 +364,12 @@
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_SCMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_UCMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_SELECT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
@@ -477,32 +486,40 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FPOW (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FPOWI (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FEXP (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FEXP2 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FEXP10 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FLOG (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FLOG2 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FLOG10 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FLDEXP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FFREXP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
@@ -523,12 +540,12 @@
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_SITOFP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_UITOFP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FPTOSI_SAT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -563,6 +580,7 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FMINIMUM (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: .. opcode 212 is aliased to 213
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FMAXIMUM (opcode {{[0-9]+}}): 1 type index
@@ -677,38 +695,47 @@
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FCOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSIN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSINCOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FTAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FACOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FASIN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FATAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FATAN2 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FCOSH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSINH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FTANH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-phi-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-phi-rv64.mir
index 7111f7429e75c..980682616d1d3 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-phi-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-phi-rv64.mir
@@ -139,6 +139,7 @@ body:             |
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
   ; CHECK-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
   ; CHECK-NEXT:   G_BRCOND [[AND]](s64), %bb.2
@@ -147,9 +148,12 @@ body:             |
   ; CHECK-NEXT: bb.1.cond.false:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+  ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cond.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[COPY2]](s64), %bb.1, [[COPY1]](s64), %bb.0
-  ; CHECK-NEXT:   $x10 = COPY [[PHI]](s64)
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[TRUNC1]](s32), %bb.1, [[TRUNC]](s32), %bb.0
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[PHI]](s32)
+  ; CHECK-NEXT:   $x10 = COPY [[ANYEXT]](s64)
   ; CHECK-NEXT:   PseudoRET implicit $x10
   bb.1.entry:
     liveins: $x10, $x11, $x12
@@ -199,6 +203,7 @@ body:             |
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
   ; CHECK-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
   ; CHECK-NEXT:   G_BRCOND [[AND]](s64), %bb.2
@@ -207,9 +212,12 @@ body:             |
   ; CHECK-NEXT: bb.1.cond.false:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+  ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cond.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[COPY2]](s64), %bb.1, [[COPY1]](s64), %bb.0
-  ; CHECK-NEXT:   $x10 = COPY [[PHI]](s64)
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[TRUNC1]](s32), %bb.1, [[TRUNC]](s32), %bb.0
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[PHI]](s32)
+  ; CHECK-NEXT:   $x10 = COPY [[ANYEXT]](s64)
   ; CHECK-NEXT:   PseudoRET implicit $x10
   bb.1.entry:
     liveins: $x10, $x11, $x12
@@ -259,6 +267,7 @@ body:             |
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
   ; CHECK-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
   ; CHECK-NEXT:   G_BRCOND [[AND]](s64), %bb.2
@@ -267,9 +276,12 @@ body:             |
   ; CHECK-NEXT: bb.1.cond.false:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+  ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cond.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[COPY2]](s64), %bb.1, [[COPY1]](s64), %bb.0
-  ; CHECK-NEXT:   $x10 = COPY [[PHI]](s64)
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[TRUNC1]](s32), %bb.1, [[TRUNC]](s32), %bb.0
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[PHI]](s32)
+  ; CHECK-NEXT:   $x10 = COPY [[ANYEXT]](s64)
   ; CHECK-NEXT:   PseudoRET implicit $x10
   bb.1.entry:
     liveins: $x10, $x11, $x12
@@ -318,7 +330,9 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
   ; CHECK-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
   ; CHECK-NEXT:   G_BRCOND [[AND]](s64), %bb.2
@@ -328,8 +342,9 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.cond.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[COPY2]](s64), %bb.1, [[COPY1]](s64), %bb.0
-  ; CHECK-NEXT:   $x10 = COPY [[PHI]](s64)
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[TRUNC1]](s32), %bb.1, [[TRUNC]](s32), %bb.0
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[PHI]](s32)
+  ; CHECK-NEXT:   $x10 = COPY [[ANYEXT]](s64)
   ; CHECK-NEXT:   PseudoRET implicit $x10
   bb.1.entry:
     liveins: $x10, $x11, $x12
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv32.mir
new file mode 100644
index 0000000000000..5c3d7e5975f1f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv32.mir
@@ -0,0 +1,58 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_ucmp_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: test_ucmp_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[C]], [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[C2]], [[SELECT]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_SCMP %0(s32), %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_scmp_i32
+alignment:       4
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: test_scmp_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[C]], [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[C2]], [[SELECT]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_SCMP %0(s32), %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv64.mir
new file mode 100644
index 0000000000000..ccade88ffae7c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-threeway-cmp-rv64.mir
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_ucmp_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: test_ucmp_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C1]](s64)
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[SEXT_INREG]](s64), [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[SEXT_INREG]](s64), [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s64), [[TRUNC2]], [[SELECT]]
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SELECT1]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[SEXT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32) = G_SCMP %0(s32), %1
+    %5:_(s64) = G_SEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_scmp_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: test_scmp_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C1]](s64)
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[SEXT_INREG]](s64), [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[SEXT_INREG]](s64), [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s64), [[TRUNC2]], [[SELECT]]
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SELECT1]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[SEXT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32) = G_SCMP %0(s32), %1
+    %5:_(s64) = G_SEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/phi.ll b/llvm/test/CodeGen/RISCV/GlobalISel/phi.ll
new file mode 100644
index 0000000000000..494d2c94c181d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/phi.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECKI,RV32
+; RUN: llc -mtriple=riscv64 -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECKI,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+f -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECKIF,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+f -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECKIF,RV64
+
+define i1 @phi_i1(i1 %cnd, i1 %a, i1 %b) {
+; CHECK-LABEL: phi_i1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a3, a0
+; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    andi a3, a3, 1
+; CHECK-NEXT:    bnez a3, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB0_2: # %cond.end
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i1 [ %b, %cond.false ], [ %a, %entry ]
+  ret i1 %cond
+}
+
+define i8 @phi_i8(i1 %cnd, i8 %a, i8 %b) {
+; CHECK-LABEL: phi_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a3, a0
+; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    andi a3, a3, 1
+; CHECK-NEXT:    bnez a3, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB1_2: # %cond.end
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i8 [ %b, %cond.false ], [ %a, %entry ]
+  ret i8 %cond
+}
+
+define i16 @phi_i16(i1 %cnd, i16 %a, i16 %b) {
+; CHECK-LABEL: phi_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a3, a0
+; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    andi a3, a3, 1
+; CHECK-NEXT:    bnez a3, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB2_2: # %cond.end
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i16 [ %b, %cond.false ], [ %a, %entry ]
+  ret i16 %cond
+}
+
+define i32 @phi_i32(i1 %cnd, i32 %a, i32 %b) {
+; CHECK-LABEL: phi_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a3, a0
+; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    andi a3, a3, 1
+; CHECK-NEXT:    bnez a3, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB3_2: # %cond.end
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i32 [ %b, %cond.false ], [ %a, %entry ]
+  ret i32 %cond
+}
+
+define i64 @phi_i64(i1 %cnd, i64 %a, i64 %b) {
+; RV32-LABEL: phi_i64:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    mv a5, a0
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    mv a1, a2
+; RV32-NEXT:    andi a5, a5, 1
+; RV32-NEXT:    bnez a5, .LBB4_2
+; RV32-NEXT:  # %bb.1: # %cond.false
+; RV32-NEXT:    mv a0, a3
+; RV32-NEXT:    mv a1, a4
+; RV32-NEXT:  .LBB4_2: # %cond.end
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: phi_i64:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    mv a3, a0
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    andi a3, a3, 1
+; RV64-NEXT:    bnez a3, .LBB4_2
+; RV64-NEXT:  # %bb.1: # %cond.false
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:  .LBB4_2: # %cond.end
+; RV64-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i64 [ %b, %cond.false ], [ %a, %entry ]
+  ret i64 %cond
+}
+
+define ptr @phi_ptr(i1 %cnd, ptr %a, ptr %b) {
+; CHECK-LABEL: phi_ptr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mv a3, a0
+; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    andi a3, a3, 1
+; CHECK-NEXT:    bnez a3, .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB5_2: # %cond.end
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi ptr [ %b, %cond.false ], [ %a, %entry ]
+  ret ptr %cond
+}
+
+define float @phi_float(i1 %cnd, float %a, float %b) {
+; CHECKI-LABEL: phi_float:
+; CHECKI:       # %bb.0: # %entry
+; CHECKI-NEXT:    mv a3, a0
+; CHECKI-NEXT:    mv a0, a1
+; CHECKI-NEXT:    andi a3, a3, 1
+; CHECKI-NEXT:    bnez a3, .LBB6_2
+; CHECKI-NEXT:  # %bb.1: # %cond.false
+; CHECKI-NEXT:    mv a0, a2
+; CHECKI-NEXT:  .LBB6_2: # %cond.end
+; CHECKI-NEXT:    ret
+;
+; CHECKIF-LABEL: phi_float:
+; CHECKIF:       # %bb.0: # %entry
+; CHECKIF-NEXT:    andi a0, a0, 1
+; CHECKIF-NEXT:    bnez a0, .LBB6_2
+; CHECKIF-NEXT:  # %bb.1: # %cond.false
+; CHECKIF-NEXT:    fmv.s fa0, fa1
+; CHECKIF-NEXT:  .LBB6_2: # %cond.end
+; CHECKIF-NEXT:    ret
+entry:
+  br i1 %cnd, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi float [ %b, %cond.false ], [ %a, %entry ]
+  ret float %cond
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/riscv-unsupported.ll b/llvm/test/CodeGen/RISCV/GlobalISel/riscv-unsupported.ll
new file mode 100644
index 0000000000000..073b5186aad71
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/riscv-unsupported.ll
@@ -0,0 +1,16 @@
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -global-isel -global-isel-abort=2 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -global-isel -global-isel-abort=2 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s -check-prefixes=CHECK
+
+; This file checks that we use the fallback path for things that are known to
+; be unsupported on the RISC-V target. It should progressively shrink in size.
+
+%byval.class = type { i32 }
+
+declare void @test_byval_arg(ptr byval(%byval.class) %x)
+
+define void @test_byval_param(ptr %x) {
+; CHECK: remark: {{.*}} unable to translate instruction: call
+; CHECK-LABEL: warning: Instruction selection used fallback path for test_byval_param
+  call void @test_byval_arg(ptr byval(%byval.class) %x)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb-zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb-zbkb.ll
index 68bf9240ccd1d..ababec16f7f8f 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb-zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb-zbkb.ll
@@ -361,9 +361,8 @@ define i8 @srai_i8(i8 %a) nounwind {
 define i16 @srli_i16(i16 %a) nounwind {
 ; RV32I-LABEL: srli_i16:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    srli a0, a0, 6
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
index 7f22127ad3536..338925059862c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
@@ -885,9 +885,8 @@ define i64 @abs_i64(i64 %x) {
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV32I-LABEL: zexth_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: zexth_i32:
@@ -901,9 +900,8 @@ define i32 @zexth_i32(i32 %a) nounwind {
 define i64 @zexth_i64(i64 %a) nounwind {
 ; RV32I-LABEL: zexth_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbkb.ll
index a647eae82dddf..b214cf68ddce8 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbkb.ll
@@ -7,9 +7,8 @@
 define i32 @pack_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: pack_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    or a0, a1, a0
 ; RV32I-NEXT:    ret
@@ -257,9 +256,8 @@ define void @packh_i16_3(i8 zeroext %0, i8 zeroext %1, i8 zeroext %2, ptr %p) {
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV32I-LABEL: zexth_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBKB-LABEL: zexth_i32:
@@ -273,9 +271,8 @@ define i32 @zexth_i32(i32 %a) nounwind {
 define i64 @zexth_i64(i64 %a) nounwind {
 ; RV32I-LABEL: zexth_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ;
@@ -291,9 +288,8 @@ define i64 @zexth_i64(i64 %a) nounwind {
 define i32 @zext_i16_to_i32(i16 %a) nounwind {
 ; RV32I-LABEL: zext_i16_to_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBKB-LABEL: zext_i16_to_i32:
@@ -307,9 +303,8 @@ define i32 @zext_i16_to_i32(i16 %a) nounwind {
 define i64 @zext_i16_to_i64(i16 %a) nounwind {
 ; RV32I-LABEL: zext_i16_to_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zba.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zba.ll
index 9584270d8e66f..993ba19caa6b4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zba.ll
@@ -1520,19 +1520,17 @@ entry:
 define i64 @srli_slli_i16(i64 %1) {
 ; RV64I-LABEL: srli_slli_i16:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
 ; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    slli a0, a0, 4
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBANOZBB-LABEL: srli_slli_i16:
 ; RV64ZBANOZBB:       # %bb.0: # %entry
-; RV64ZBANOZBB-NEXT:    lui a1, 16
-; RV64ZBANOZBB-NEXT:    addiw a1, a1, -1
 ; RV64ZBANOZBB-NEXT:    srli a0, a0, 2
-; RV64ZBANOZBB-NEXT:    and a0, a0, a1
+; RV64ZBANOZBB-NEXT:    slli a0, a0, 48
+; RV64ZBANOZBB-NEXT:    srli a0, a0, 48
 ; RV64ZBANOZBB-NEXT:    slli a0, a0, 4
 ; RV64ZBANOZBB-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
index 5beebe5bbcd97..a29219bfde06b 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb-zbkb.ll
@@ -108,9 +108,7 @@ define signext i32 @rol_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: rol_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    neg a2, a1
-; RV64I-NEXT:    andi a1, a1, 31
 ; RV64I-NEXT:    sllw a1, a0, a1
-; RV64I-NEXT:    andi a2, a2, 31
 ; RV64I-NEXT:    srlw a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
@@ -128,9 +126,7 @@ define void @rol_i32_nosext(i32 signext %a, i32 signext %b, ptr %x) nounwind {
 ; RV64I-LABEL: rol_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    neg a3, a1
-; RV64I-NEXT:    andi a1, a1, 31
 ; RV64I-NEXT:    sllw a1, a0, a1
-; RV64I-NEXT:    andi a3, a3, 31
 ; RV64I-NEXT:    srlw a0, a0, a3
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    sw a0, 0(a2)
@@ -151,9 +147,7 @@ define signext i32 @rol_i32_neg_constant_rhs(i32 signext %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -2
 ; RV64I-NEXT:    neg a2, a0
-; RV64I-NEXT:    andi a0, a0, 31
 ; RV64I-NEXT:    sllw a0, a1, a0
-; RV64I-NEXT:    andi a2, a2, 31
 ; RV64I-NEXT:    srlw a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -192,9 +186,7 @@ define signext i32 @ror_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: ror_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    neg a2, a1
-; RV64I-NEXT:    andi a1, a1, 31
 ; RV64I-NEXT:    srlw a1, a0, a1
-; RV64I-NEXT:    andi a2, a2, 31
 ; RV64I-NEXT:    sllw a0, a0, a2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
@@ -212,9 +204,7 @@ define void @ror_i32_nosext(i32 signext %a, i32 signext %b, ptr %x) nounwind {
 ; RV64I-LABEL: ror_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    neg a3, a1
-; RV64I-NEXT:    andi a1, a1, 31
 ; RV64I-NEXT:    srlw a1, a0, a1
-; RV64I-NEXT:    andi a3, a3, 31
 ; RV64I-NEXT:    sllw a0, a0, a3
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    sw a0, 0(a2)
@@ -235,9 +225,7 @@ define signext i32 @ror_i32_neg_constant_rhs(i32 signext %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -2
 ; RV64I-NEXT:    neg a2, a0
-; RV64I-NEXT:    andi a0, a0, 31
 ; RV64I-NEXT:    srlw a0, a1, a0
-; RV64I-NEXT:    andi a2, a2, 31
 ; RV64I-NEXT:    sllw a1, a1, a2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -451,9 +439,8 @@ define i8 @srai_i8(i8 %a) nounwind {
 define i16 @srli_i16(i16 %a) nounwind {
 ; RV64I-LABEL: srli_i16:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    srli a0, a0, 6
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
index 16c588fa2f2ce..4e12644a22b42 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
@@ -1146,9 +1146,8 @@ define i64 @abs_i64(i64 %x) {
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV64I-LABEL: zexth_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: zexth_i32:
@@ -1162,9 +1161,8 @@ define i32 @zexth_i32(i32 %a) nounwind {
 define i64 @zexth_i64(i64 %a) nounwind {
 ; RV64I-LABEL: zexth_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: zexth_i64:
@@ -1271,3 +1269,69 @@ define i64 @bswap_i64(i64 %a) {
   %1 = call i64 @llvm.bswap.i64(i64 %a)
   ret i64 %1
 }
+
+; This creates a i16->i32 G_ZEXT that we need to be able to select
+define i32 @zext_i16_i32(i1 %z, ptr %x, i32 %y) {
+; RV64I-LABEL: zext_i16_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    bnez a3, .LBB35_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB35_2:
+; RV64I-NEXT:    lh a0, 0(a1)
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: zext_i16_i32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    andi a3, a0, 1
+; RV64ZBB-NEXT:    bnez a3, .LBB35_2
+; RV64ZBB-NEXT:  # %bb.1:
+; RV64ZBB-NEXT:    mv a0, a2
+; RV64ZBB-NEXT:    ret
+; RV64ZBB-NEXT:  .LBB35_2:
+; RV64ZBB-NEXT:    lh a0, 0(a1)
+; RV64ZBB-NEXT:    zext.h a0, a0
+; RV64ZBB-NEXT:    ret
+  %w = load i16, ptr %x
+  %a = freeze i16 %w
+  %b = zext i16 %a to i32
+  %c = select i1 %z, i32 %b, i32 %y
+  ret i32 %c
+}
+
+; This creates a i16->i32 G_SEXT that we need to be able to select
+define i32 @sext_i16_i32(i1 %z, ptr %x, i32 %y) {
+; RV64I-LABEL: sext_i16_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    bnez a3, .LBB36_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB36_2:
+; RV64I-NEXT:    lh a0, 0(a1)
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: sext_i16_i32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    andi a3, a0, 1
+; RV64ZBB-NEXT:    bnez a3, .LBB36_2
+; RV64ZBB-NEXT:  # %bb.1:
+; RV64ZBB-NEXT:    mv a0, a2
+; RV64ZBB-NEXT:    ret
+; RV64ZBB-NEXT:  .LBB36_2:
+; RV64ZBB-NEXT:    lh a0, 0(a1)
+; RV64ZBB-NEXT:    sext.h a0, a0
+; RV64ZBB-NEXT:    ret
+  %w = load i16, ptr %x
+  %a = freeze i16 %w
+  %b = sext i16 %a to i32
+  %c = select i1 %z, i32 %b, i32 %y
+  ret i32 %c
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
index 338476a1bec83..bf430c618afca 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
@@ -8,12 +8,10 @@
 define signext i32 @pack_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: pack_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addi a2, a2, -1
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    slliw a1, a1, 16
 ; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBKB-LABEL: pack_i32:
@@ -334,9 +332,8 @@ define signext i32 @pack_i32_allWUsers(i16 zeroext %0, i16 zeroext %1, i16 zeroe
 ; RV64I-LABEL: pack_i32_allWUsers:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    add a0, a1, a0
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    slli a0, a0, 16
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    sext.w a0, a0
@@ -382,9 +379,8 @@ define i64 @pack_i64_imm() {
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV64I-LABEL: zexth_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBKB-LABEL: zexth_i32:
@@ -398,9 +394,8 @@ define i32 @zexth_i32(i32 %a) nounwind {
 define i64 @zexth_i64(i64 %a) nounwind {
 ; RV64I-LABEL: zexth_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBKB-LABEL: zexth_i64:
@@ -414,9 +409,8 @@ define i64 @zexth_i64(i64 %a) nounwind {
 define i32 @zext_i16_to_i32(i16 %a) nounwind {
 ; RV64I-LABEL: zext_i16_to_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBKB-LABEL: zext_i16_to_i32:
@@ -430,9 +424,8 @@ define i32 @zext_i16_to_i32(i16 %a) nounwind {
 define i64 @zext_i16_to_i64(i16 %a) nounwind {
 ; RV64I-LABEL: zext_i16_to_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBKB-LABEL: zext_i16_to_i64:
@@ -442,3 +435,70 @@ define i64 @zext_i16_to_i64(i16 %a) nounwind {
   %1 = zext i16 %a to i64
   ret i64 %1
 }
+
+; This creates a i16->i32 G_ZEXT that we need to be able to select
+define i32 @zext_i16_i32_2(i1 %z, ptr %x, i32 %y) {
+; RV64I-LABEL: zext_i16_i32_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    bnez a3, .LBB20_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB20_2:
+; RV64I-NEXT:    lh a0, 0(a1)
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: zext_i16_i32_2:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    andi a3, a0, 1
+; RV64ZBKB-NEXT:    bnez a3, .LBB20_2
+; RV64ZBKB-NEXT:  # %bb.1:
+; RV64ZBKB-NEXT:    mv a0, a2
+; RV64ZBKB-NEXT:    ret
+; RV64ZBKB-NEXT:  .LBB20_2:
+; RV64ZBKB-NEXT:    lh a0, 0(a1)
+; RV64ZBKB-NEXT:    zext.h a0, a0
+; RV64ZBKB-NEXT:    ret
+  %w = load i16, ptr %x
+  %a = freeze i16 %w
+  %b = zext i16 %a to i32
+  %c = select i1 %z, i32 %b, i32 %y
+  ret i32 %c
+}
+
+; This creates a i16->i32 G_SEXT that we need to be able to select
+define i32 @sext_i16_i32(i1 %z, ptr %x, i32 %y) {
+; RV64I-LABEL: sext_i16_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    bnez a3, .LBB21_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB21_2:
+; RV64I-NEXT:    lh a0, 0(a1)
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: sext_i16_i32:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    andi a3, a0, 1
+; RV64ZBKB-NEXT:    bnez a3, .LBB21_2
+; RV64ZBKB-NEXT:  # %bb.1:
+; RV64ZBKB-NEXT:    mv a0, a2
+; RV64ZBKB-NEXT:    ret
+; RV64ZBKB-NEXT:  .LBB21_2:
+; RV64ZBKB-NEXT:    lh a0, 0(a1)
+; RV64ZBKB-NEXT:    slli a0, a0, 48
+; RV64ZBKB-NEXT:    srai a0, a0, 48
+; RV64ZBKB-NEXT:    ret
+  %w = load i16, ptr %x
+  %a = freeze i16 %w
+  %b = sext i16 %a to i32
+  %c = select i1 %z, i32 %b, i32 %y
+  ret i32 %c
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/scmp.ll b/llvm/test/CodeGen/RISCV/GlobalISel/scmp.ll
new file mode 100644
index 0000000000000..0f2b6281b6f88
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/scmp.ll
@@ -0,0 +1,291 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -global-isel | FileCheck %s --check-prefix=RV32I
+; RUN: llc < %s -mtriple=riscv64 -global-isel | FileCheck %s --check-prefix=RV64I
+
+define i8 @scmp.8.8(i8 signext %x, i8 signext %y) nounwind {
+; RV32I-LABEL: scmp.8.8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    blt a1, a0, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    blt a2, a1, .LBB0_3
+; RV32I-NEXT:    j .LBB0_4
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bge a2, a1, .LBB0_4
+; RV32I-NEXT:  .LBB0_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB0_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.8.8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt a1, a0, .LBB0_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB0_3
+; RV64I-NEXT:    j .LBB0_4
+; RV64I-NEXT:  .LBB0_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB0_4
+; RV64I-NEXT:  .LBB0_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB0_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.scmp(i8 %x, i8 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.16(i16 signext %x, i16 signext %y) nounwind {
+; RV32I-LABEL: scmp.8.16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    blt a1, a0, .LBB1_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    blt a2, a1, .LBB1_3
+; RV32I-NEXT:    j .LBB1_4
+; RV32I-NEXT:  .LBB1_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bge a2, a1, .LBB1_4
+; RV32I-NEXT:  .LBB1_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB1_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.8.16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt a1, a0, .LBB1_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB1_3
+; RV64I-NEXT:    j .LBB1_4
+; RV64I-NEXT:  .LBB1_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB1_4
+; RV64I-NEXT:  .LBB1_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB1_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.scmp(i16 %x, i16 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: scmp.8.32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    blt a1, a0, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    blt a2, a1, .LBB2_3
+; RV32I-NEXT:    j .LBB2_4
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bge a2, a1, .LBB2_4
+; RV32I-NEXT:  .LBB2_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB2_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.8.32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a2, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    blt a1, a2, .LBB2_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB2_3
+; RV64I-NEXT:    j .LBB2_4
+; RV64I-NEXT:  .LBB2_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB2_4
+; RV64I-NEXT:  .LBB2_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB2_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  ret i8 %1
+}
+
+define i8 @scmp.8.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: scmp.8.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beq a1, a3, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    slt a4, a3, a1
+; RV32I-NEXT:    bnez a4, .LBB3_3
+; RV32I-NEXT:    j .LBB3_4
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    sltu a4, a2, a0
+; RV32I-NEXT:    beqz a4, .LBB3_4
+; RV32I-NEXT:  .LBB3_3:
+; RV32I-NEXT:    li a4, 1
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    beq a1, a3, .LBB3_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    slt a0, a1, a3
+; RV32I-NEXT:    bnez a0, .LBB3_7
+; RV32I-NEXT:    j .LBB3_8
+; RV32I-NEXT:  .LBB3_6:
+; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    beqz a0, .LBB3_8
+; RV32I-NEXT:  .LBB3_7:
+; RV32I-NEXT:    li a4, -1
+; RV32I-NEXT:  .LBB3_8:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.8.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt a1, a0, .LBB3_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB3_3
+; RV64I-NEXT:    j .LBB3_4
+; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB3_4
+; RV64I-NEXT:  .LBB3_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB3_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.scmp(i64 %x, i64 %y)
+  ret i8 %1
+}
+
+define i32 @scmp.32.32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: scmp.32.32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    blt a1, a0, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    blt a2, a1, .LBB4_3
+; RV32I-NEXT:    j .LBB4_4
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bge a2, a1, .LBB4_4
+; RV32I-NEXT:  .LBB4_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB4_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.32.32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a2, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    blt a1, a2, .LBB4_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB4_3
+; RV64I-NEXT:    j .LBB4_4
+; RV64I-NEXT:  .LBB4_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB4_4
+; RV64I-NEXT:  .LBB4_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB4_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.scmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @scmp.32.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: scmp.32.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beq a1, a3, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    slt a4, a3, a1
+; RV32I-NEXT:    bnez a4, .LBB5_3
+; RV32I-NEXT:    j .LBB5_4
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    sltu a4, a2, a0
+; RV32I-NEXT:    beqz a4, .LBB5_4
+; RV32I-NEXT:  .LBB5_3:
+; RV32I-NEXT:    li a4, 1
+; RV32I-NEXT:  .LBB5_4:
+; RV32I-NEXT:    beq a1, a3, .LBB5_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    slt a0, a1, a3
+; RV32I-NEXT:    bnez a0, .LBB5_7
+; RV32I-NEXT:    j .LBB5_8
+; RV32I-NEXT:  .LBB5_6:
+; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    beqz a0, .LBB5_8
+; RV32I-NEXT:  .LBB5_7:
+; RV32I-NEXT:    li a4, -1
+; RV32I-NEXT:  .LBB5_8:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.32.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt a1, a0, .LBB5_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB5_3
+; RV64I-NEXT:    j .LBB5_4
+; RV64I-NEXT:  .LBB5_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB5_4
+; RV64I-NEXT:  .LBB5_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB5_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.scmp(i64 %x, i64 %y)
+  ret i32 %1
+}
+
+define i64 @scmp.64.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: scmp.64.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a4, a0
+; RV32I-NEXT:    beq a1, a3, .LBB6_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    slt a0, a3, a1
+; RV32I-NEXT:    bnez a0, .LBB6_3
+; RV32I-NEXT:    j .LBB6_4
+; RV32I-NEXT:  .LBB6_2:
+; RV32I-NEXT:    sltu a0, a2, a4
+; RV32I-NEXT:    beqz a0, .LBB6_4
+; RV32I-NEXT:  .LBB6_3:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:  .LBB6_4:
+; RV32I-NEXT:    beq a1, a3, .LBB6_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    slt a1, a1, a3
+; RV32I-NEXT:    bnez a1, .LBB6_7
+; RV32I-NEXT:    j .LBB6_8
+; RV32I-NEXT:  .LBB6_6:
+; RV32I-NEXT:    sltu a1, a4, a2
+; RV32I-NEXT:    beqz a1, .LBB6_8
+; RV32I-NEXT:  .LBB6_7:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:    li a1, -1
+; RV32I-NEXT:  .LBB6_8:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: scmp.64.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt a1, a0, .LBB6_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    blt a2, a1, .LBB6_3
+; RV64I-NEXT:    j .LBB6_4
+; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bge a2, a1, .LBB6_4
+; RV64I-NEXT:  .LBB6_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB6_4:
+; RV64I-NEXT:    ret
+  %1 = call i64 @llvm.scmp(i64 %x, i64 %y)
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/ucmp.ll b/llvm/test/CodeGen/RISCV/GlobalISel/ucmp.ll
new file mode 100644
index 0000000000000..e2a95eb974342
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/ucmp.ll
@@ -0,0 +1,370 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -global-isel | FileCheck %s --check-prefix=RV32I
+; RUN: llc < %s -mtriple=riscv64 -global-isel | FileCheck %s --check-prefix=RV64I
+
+define i8 @ucmp.8.8(i8 zeroext %x, i8 zeroext %y) nounwind {
+; RV32I-LABEL: ucmp.8.8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB0_3
+; RV32I-NEXT:    j .LBB0_4
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB0_4
+; RV32I-NEXT:  .LBB0_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB0_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.8.8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB0_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB0_3
+; RV64I-NEXT:    j .LBB0_4
+; RV64I-NEXT:  .LBB0_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB0_4
+; RV64I-NEXT:  .LBB0_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB0_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.ucmp(i8 %x, i8 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.16(i16 zeroext %x, i16 zeroext %y) nounwind {
+; RV32I-LABEL: ucmp.8.16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB1_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB1_3
+; RV32I-NEXT:    j .LBB1_4
+; RV32I-NEXT:  .LBB1_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB1_4
+; RV32I-NEXT:  .LBB1_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB1_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.8.16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB1_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB1_3
+; RV64I-NEXT:    j .LBB1_4
+; RV64I-NEXT:  .LBB1_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB1_4
+; RV64I-NEXT:  .LBB1_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB1_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.ucmp(i16 %x, i16 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: ucmp.8.32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB2_3
+; RV32I-NEXT:    j .LBB2_4
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB2_4
+; RV32I-NEXT:  .LBB2_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB2_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.8.32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a2, a1, 32
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srli a2, a2, 32
+; RV64I-NEXT:    bltu a2, a1, .LBB2_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a1, a2, .LBB2_3
+; RV64I-NEXT:    j .LBB2_4
+; RV64I-NEXT:  .LBB2_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a1, a2, .LBB2_4
+; RV64I-NEXT:  .LBB2_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB2_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  ret i8 %1
+}
+
+define i8 @ucmp.8.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: ucmp.8.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beq a1, a3, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sltu a4, a3, a1
+; RV32I-NEXT:    bnez a4, .LBB3_3
+; RV32I-NEXT:    j .LBB3_4
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    sltu a4, a2, a0
+; RV32I-NEXT:    beqz a4, .LBB3_4
+; RV32I-NEXT:  .LBB3_3:
+; RV32I-NEXT:    li a4, 1
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    beq a1, a3, .LBB3_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    sltu a0, a1, a3
+; RV32I-NEXT:    bnez a0, .LBB3_7
+; RV32I-NEXT:    j .LBB3_8
+; RV32I-NEXT:  .LBB3_6:
+; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    beqz a0, .LBB3_8
+; RV32I-NEXT:  .LBB3_7:
+; RV32I-NEXT:    li a4, -1
+; RV32I-NEXT:  .LBB3_8:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.8.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB3_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB3_3
+; RV64I-NEXT:    j .LBB3_4
+; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB3_4
+; RV64I-NEXT:  .LBB3_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB3_4:
+; RV64I-NEXT:    ret
+  %1 = call i8 @llvm.ucmp(i64 %x, i64 %y)
+  ret i8 %1
+}
+
+define i32 @ucmp.32.32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: ucmp.32.32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB4_3
+; RV32I-NEXT:    j .LBB4_4
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB4_4
+; RV32I-NEXT:  .LBB4_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB4_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.32.32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a2, a1, 32
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srli a2, a2, 32
+; RV64I-NEXT:    bltu a2, a1, .LBB4_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a1, a2, .LBB4_3
+; RV64I-NEXT:    j .LBB4_4
+; RV64I-NEXT:  .LBB4_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a1, a2, .LBB4_4
+; RV64I-NEXT:  .LBB4_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB4_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @ucmp.32.32_sext(i32 signext %x, i32 signext %y) nounwind {
+; RV32I-LABEL: ucmp.32.32_sext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB5_3
+; RV32I-NEXT:    j .LBB5_4
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB5_4
+; RV32I-NEXT:  .LBB5_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB5_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.32.32_sext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a2, a1, 32
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    srli a2, a2, 32
+; RV64I-NEXT:    bltu a2, a1, .LBB5_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a1, a2, .LBB5_3
+; RV64I-NEXT:    j .LBB5_4
+; RV64I-NEXT:  .LBB5_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a1, a2, .LBB5_4
+; RV64I-NEXT:  .LBB5_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB5_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @ucmp.32.32_zext(i32 zeroext %x, i32 zeroext %y) nounwind {
+; RV32I-LABEL: ucmp.32.32_zext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    bltu a1, a0, .LBB6_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    bltu a2, a1, .LBB6_3
+; RV32I-NEXT:    j .LBB6_4
+; RV32I-NEXT:  .LBB6_2:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:    bgeu a2, a1, .LBB6_4
+; RV32I-NEXT:  .LBB6_3:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:  .LBB6_4:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.32.32_zext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB6_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB6_3
+; RV64I-NEXT:    j .LBB6_4
+; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB6_4
+; RV64I-NEXT:  .LBB6_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB6_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
+  ret i32 %1
+}
+
+define i32 @ucmp.32.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: ucmp.32.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beq a1, a3, .LBB7_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sltu a4, a3, a1
+; RV32I-NEXT:    bnez a4, .LBB7_3
+; RV32I-NEXT:    j .LBB7_4
+; RV32I-NEXT:  .LBB7_2:
+; RV32I-NEXT:    sltu a4, a2, a0
+; RV32I-NEXT:    beqz a4, .LBB7_4
+; RV32I-NEXT:  .LBB7_3:
+; RV32I-NEXT:    li a4, 1
+; RV32I-NEXT:  .LBB7_4:
+; RV32I-NEXT:    beq a1, a3, .LBB7_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    sltu a0, a1, a3
+; RV32I-NEXT:    bnez a0, .LBB7_7
+; RV32I-NEXT:    j .LBB7_8
+; RV32I-NEXT:  .LBB7_6:
+; RV32I-NEXT:    sltu a0, a0, a2
+; RV32I-NEXT:    beqz a0, .LBB7_8
+; RV32I-NEXT:  .LBB7_7:
+; RV32I-NEXT:    li a4, -1
+; RV32I-NEXT:  .LBB7_8:
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.32.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB7_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB7_3
+; RV64I-NEXT:    j .LBB7_4
+; RV64I-NEXT:  .LBB7_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB7_4
+; RV64I-NEXT:  .LBB7_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB7_4:
+; RV64I-NEXT:    ret
+  %1 = call i32 @llvm.ucmp(i64 %x, i64 %y)
+  ret i32 %1
+}
+
+define i64 @ucmp.64.64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: ucmp.64.64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a4, a0
+; RV32I-NEXT:    beq a1, a3, .LBB8_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    sltu a0, a3, a1
+; RV32I-NEXT:    bnez a0, .LBB8_3
+; RV32I-NEXT:    j .LBB8_4
+; RV32I-NEXT:  .LBB8_2:
+; RV32I-NEXT:    sltu a0, a2, a4
+; RV32I-NEXT:    beqz a0, .LBB8_4
+; RV32I-NEXT:  .LBB8_3:
+; RV32I-NEXT:    li a0, 1
+; RV32I-NEXT:  .LBB8_4:
+; RV32I-NEXT:    beq a1, a3, .LBB8_6
+; RV32I-NEXT:  # %bb.5:
+; RV32I-NEXT:    sltu a1, a1, a3
+; RV32I-NEXT:    bnez a1, .LBB8_7
+; RV32I-NEXT:    j .LBB8_8
+; RV32I-NEXT:  .LBB8_6:
+; RV32I-NEXT:    sltu a1, a4, a2
+; RV32I-NEXT:    beqz a1, .LBB8_8
+; RV32I-NEXT:  .LBB8_7:
+; RV32I-NEXT:    li a0, -1
+; RV32I-NEXT:    li a1, -1
+; RV32I-NEXT:  .LBB8_8:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ucmp.64.64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu a1, a0, .LBB8_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    li a0, 0
+; RV64I-NEXT:    bltu a2, a1, .LBB8_3
+; RV64I-NEXT:    j .LBB8_4
+; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    bgeu a2, a1, .LBB8_4
+; RV64I-NEXT:  .LBB8_3:
+; RV64I-NEXT:    li a0, -1
+; RV64I-NEXT:  .LBB8_4:
+; RV64I-NEXT:    ret
+  %1 = call i64 @llvm.ucmp(i64 %x, i64 %y)
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll b/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll
new file mode 100644
index 0000000000000..47b6c07cc699e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=riscv64 < %s | FileCheck -check-prefix=RV64 %s
+
+define void @add_sext_shl_moreOneUse_add(ptr %array1, i32 %a, i32 %b) {
+; RV64-LABEL: add_sext_shl_moreOneUse_add:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a3, a1, 5
+; RV64-NEXT:    sext.w a1, a1
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a2, 20(a0)
+; RV64-NEXT:    sw a2, 24(a0)
+; RV64-NEXT:    sw a3, 140(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  store i32 %b, ptr %arrayidx
+  %add3 = add nsw i32 %a, 6
+  %idxprom4 = sext i32 %add3 to i64
+  %arrayidx5 = getelementptr inbounds i32, ptr %array1, i64 %idxprom4
+  store i32 %b, ptr %arrayidx5
+  %add6 = add nsw i32 %a, 35
+  %idxprom7 = sext i32 %add6 to i64
+  %arrayidx8 = getelementptr inbounds i32, ptr %array1, i64 %idxprom7
+  store i32 %add, ptr %arrayidx8
+  ret void
+}
+
+define void @add_sext_shl_moreOneUse_addexceedsign12(ptr %array1, i32 %a, i32 %b) {
+; RV64-LABEL: add_sext_shl_moreOneUse_addexceedsign12:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a3, a1, 2047
+; RV64-NEXT:    lui a4, 2
+; RV64-NEXT:    sext.w a1, a1
+; RV64-NEXT:    addi a3, a3, 1
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a0, a4
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    sw a2, 0(a0)
+; RV64-NEXT:    sw a3, 4(a0)
+; RV64-NEXT:    sw a2, 120(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 2048
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  store i32 %b, ptr %arrayidx
+  %0 = sext i32 %a to i64
+  %1 = getelementptr i32, ptr %array1, i64 %0
+  %arrayidx3 = getelementptr i8, ptr %1, i64 8196
+  store i32 %add, ptr %arrayidx3
+  %arrayidx6 = getelementptr i8, ptr %1, i64 8312
+  store i32 %b, ptr %arrayidx6
+  ret void
+}
+
+define void @add_sext_shl_moreOneUse_sext(ptr %array1, i32 %a, i32 %b) {
+; RV64-LABEL: add_sext_shl_moreOneUse_sext:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    sext.w a1, a1
+; RV64-NEXT:    addi a3, a1, 5
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a2, 20(a0)
+; RV64-NEXT:    sw a2, 24(a0)
+; RV64-NEXT:    sd a3, 140(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  store i32 %b, ptr %arrayidx
+  %add3 = add nsw i32 %a, 6
+  %idxprom4 = sext i32 %add3 to i64
+  %arrayidx5 = getelementptr inbounds i32, ptr %array1, i64 %idxprom4
+  store i32 %b, ptr %arrayidx5
+  %add6 = add nsw i32 %a, 35
+  %idxprom7 = sext i32 %add6 to i64
+  %arrayidx8 = getelementptr inbounds i32, ptr %array1, i64 %idxprom7
+  store i64 %idxprom, ptr %arrayidx8
+  ret void
+}
+
+; test of jumpping, find add's operand has one more use can simplified
+define void @add_sext_shl_moreOneUse_add_inSelect(ptr %array1, i32 signext  %a, i32 %b, i32 signext %x) {
+; RV64-LABEL: add_sext_shl_moreOneUse_add_inSelect:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a4, a1, 5
+; RV64-NEXT:    mv a5, a4
+; RV64-NEXT:    bgtz a3, .LBB3_2
+; RV64-NEXT:  # %bb.1: # %entry
+; RV64-NEXT:    mv a5, a2
+; RV64-NEXT:  .LBB3_2: # %entry
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a5, 20(a0)
+; RV64-NEXT:    sw a5, 24(a0)
+; RV64-NEXT:    sw a4, 140(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %cmp = icmp sgt i32 %x, 0
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  %add.b = select i1 %cmp, i32 %add, i32 %b
+  store i32 %add.b, ptr %arrayidx
+  %add5 = add nsw i32 %a, 6
+  %idxprom6 = sext i32 %add5 to i64
+  %arrayidx7 = getelementptr inbounds i32, ptr %array1, i64 %idxprom6
+  store i32 %add.b, ptr %arrayidx7
+  %add8 = add nsw i32 %a, 35
+  %idxprom9 = sext i32 %add8 to i64
+  %arrayidx10 = getelementptr inbounds i32, ptr %array1, i64 %idxprom9
+  store i32 %add, ptr %arrayidx10
+  ret void
+}
+
+define void @add_sext_shl_moreOneUse_add_inSelect_addexceedsign12(ptr %array1, i32 signext  %a, i32 %b, i32 signext %x) {
+; RV64-LABEL: add_sext_shl_moreOneUse_add_inSelect_addexceedsign12:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a4, a1, 2047
+; RV64-NEXT:    lui a5, 2
+; RV64-NEXT:    slli a6, a1, 2
+; RV64-NEXT:    addi a1, a4, 1
+; RV64-NEXT:    add a0, a0, a6
+; RV64-NEXT:    add a0, a0, a5
+; RV64-NEXT:    mv a4, a1
+; RV64-NEXT:    bgtz a3, .LBB4_2
+; RV64-NEXT:  # %bb.1: # %entry
+; RV64-NEXT:    mv a4, a2
+; RV64-NEXT:  .LBB4_2: # %entry
+; RV64-NEXT:    sw a4, 0(a0)
+; RV64-NEXT:    sw a4, 4(a0)
+; RV64-NEXT:    sw a1, 120(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 2048
+  %cmp = icmp sgt i32 %x, 0
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  %add.b = select i1 %cmp, i32 %add, i32 %b
+  store i32 %add.b, ptr %arrayidx
+  %0 = sext i32 %a to i64
+  %1 = getelementptr i32, ptr %array1, i64 %0
+  %arrayidx7 = getelementptr i8, ptr %1, i64 8196
+  store i32 %add.b, ptr %arrayidx7
+  %arrayidx10 = getelementptr i8, ptr %1, i64 8312
+  store i32 %add, ptr %arrayidx10
+  ret void
+}
+
+define void @add_shl_moreOneUse_inSelect(ptr %array1, i64 %a, i64 %b, i64 %x) {
+; RV64-LABEL: add_shl_moreOneUse_inSelect:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a4, a1, 5
+; RV64-NEXT:    mv a5, a4
+; RV64-NEXT:    bgtz a3, .LBB5_2
+; RV64-NEXT:  # %bb.1: # %entry
+; RV64-NEXT:    mv a5, a2
+; RV64-NEXT:  .LBB5_2: # %entry
+; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sd a5, 40(a0)
+; RV64-NEXT:    sd a5, 48(a0)
+; RV64-NEXT:    sd a4, 280(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i64 %a, 5
+  %cmp = icmp sgt i64 %x, 0
+  %spec.select = select i1 %cmp, i64 %add, i64 %b
+  %0 = getelementptr inbounds i64, ptr %array1, i64 %add
+  store i64 %spec.select, ptr %0
+  %add3 = add nsw i64 %a, 6
+  %arrayidx4 = getelementptr inbounds i64, ptr %array1, i64 %add3
+  store i64 %spec.select, ptr %arrayidx4
+  %add5 = add nsw i64 %a, 35
+  %arrayidx6 = getelementptr inbounds i64, ptr %array1, i64 %add5
+  store i64 %add, ptr %arrayidx6
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/add_shl_constant.ll b/llvm/test/CodeGen/RISCV/add_shl_constant.ll
new file mode 100644
index 0000000000000..71b61868b8c84
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/add_shl_constant.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32  < %s | FileCheck -check-prefix=RV32 %s
+
+define i32 @add_shl_oneUse(i32 %x, i32 %y) nounwind {
+; RV32-LABEL: add_shl_oneUse:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    addi a0, a0, 984
+; RV32-NEXT:    ret
+  %add.0 = add i32 %x, 123
+  %shl = shl i32 %add.0, 3
+  %add.1 = add i32 %shl, %y
+  ret i32 %add.1
+}
+
+define void @add_shl_moreOneUse_inStore(ptr %array1, i32 %a, i32 %b)  {
+; RV32-LABEL: add_shl_moreOneUse_inStore:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi a3, a1, 5
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    sw a2, 20(a0)
+; RV32-NEXT:    sw a2, 24(a0)
+; RV32-NEXT:    sw a3, 140(a0)
+; RV32-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i32 %add
+  store i32 %b, ptr %arrayidx
+  %0 = getelementptr i32, ptr %array1, i32 %a
+  %arrayidx3 = getelementptr i8, ptr %0, i32 24
+  store i32 %b, ptr %arrayidx3
+  %arrayidx5 = getelementptr i8, ptr %0, i32 140
+  store i32 %add, ptr %arrayidx5
+  ret void
+}
+
+define void @add_shl_moreOneUse_inStore_addexceedsign12(ptr %array1, i32 %a, i32 %b)  {
+; RV32-LABEL: add_shl_moreOneUse_inStore_addexceedsign12:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi a3, a1, 2047
+; RV32-NEXT:    lui a4, 2
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    addi a3, a3, 1
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    add a0, a0, a4
+; RV32-NEXT:    sw a2, 0(a0)
+; RV32-NEXT:    sw a3, 4(a0)
+; RV32-NEXT:    sw a2, 120(a0)
+; RV32-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 2048
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i32 %add
+  store i32 %b, ptr %arrayidx
+  %0 = getelementptr i32, ptr %array1, i32 %a
+  %arrayidx2 = getelementptr i8, ptr %0, i32 8196
+  store i32 %add, ptr %arrayidx2
+  %arrayidx4 = getelementptr i8, ptr %0, i32 8312
+  store i32 %b, ptr %arrayidx4
+  ret void
+}
+
+define void @add_shl_moreOneUse_inSelect(ptr %array1, i32 %a, i32 %b, i32 %x) {
+; RV32-LABEL: add_shl_moreOneUse_inSelect:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi a4, a1, 5
+; RV32-NEXT:    mv a5, a4
+; RV32-NEXT:    bgtz a3, .LBB3_2
+; RV32-NEXT:  # %bb.1: # %entry
+; RV32-NEXT:    mv a5, a2
+; RV32-NEXT:  .LBB3_2: # %entry
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    sw a5, 20(a0)
+; RV32-NEXT:    sw a5, 24(a0)
+; RV32-NEXT:    sw a4, 140(a0)
+; RV32-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %cmp = icmp sgt i32 %x, 0
+  %cond = select i1 %cmp, i32 %add, i32 %b
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i32 %add
+  store i32 %cond, ptr %arrayidx
+  %0 = getelementptr i32, ptr %array1, i32 %a
+  %arrayidx2 = getelementptr i32, ptr %0, i32 6
+  store i32 %cond, ptr %arrayidx2
+  %arrayidx4 = getelementptr i32, ptr %0, i32 35
+  store i32 %add, ptr %arrayidx4
+  ret void
+}
+
+define void @add_shl_moreOneUse_inSelect_addexceedsign12(ptr %array1, i32 %a, i32 %b, i32 %x) {
+; RV32-LABEL: add_shl_moreOneUse_inSelect_addexceedsign12:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi a4, a1, 2047
+; RV32-NEXT:    addi a4, a4, 1
+; RV32-NEXT:    mv a5, a4
+; RV32-NEXT:    bgtz a3, .LBB4_2
+; RV32-NEXT:  # %bb.1: # %entry
+; RV32-NEXT:    mv a5, a2
+; RV32-NEXT:  .LBB4_2: # %entry
+; RV32-NEXT:    lui a2, 2
+; RV32-NEXT:    slli a1, a1, 2
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    add a0, a0, a2
+; RV32-NEXT:    sw a5, 0(a0)
+; RV32-NEXT:    sw a5, 4(a0)
+; RV32-NEXT:    sw a4, 120(a0)
+; RV32-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 2048
+  %cmp = icmp sgt i32 %x, 0
+  %spec.select = select i1 %cmp, i32 %add, i32 %b
+  %0 = getelementptr inbounds i32, ptr %array1, i32 %add
+  store i32 %spec.select, ptr %0, align 4
+  %1 = getelementptr i32, ptr %array1, i32 %a
+  %arrayidx4 = getelementptr i8, ptr %1, i32 8196
+  store i32 %spec.select, ptr %arrayidx4
+  %arrayidx6 = getelementptr i8, ptr %1, i32 8312
+  store i32 %add, ptr %arrayidx6
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index e03ab078da627..356dce2979565 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -81,6 +81,9 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+xtheadmempair %s -o - | FileCheck --check-prefix=RV32XTHEADMEMPAIR %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xtheadsync %s -o - | FileCheck --check-prefix=RV32XTHEADSYNC %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xwchc %s -o - | FileCheck --check-prefix=RV32XWCHC %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcia %s -o - | FileCheck --check-prefix=RV32XQCIA %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcicsr %s -o - | FileCheck --check-prefix=RV32XQCICSR %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcisls %s -o - | FileCheck --check-prefix=RV32XQCISLS %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zaamo %s -o - | FileCheck --check-prefix=RV32ZAAMO %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zalrsc %s -o - | FileCheck --check-prefix=RV32ZALRSC %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zca %s -o - | FileCheck --check-prefixes=CHECK,RV32ZCA %s
@@ -385,6 +388,9 @@
 ; RV32XTHEADMEMPAIR: .attribute 5, "rv32i2p1_xtheadmempair1p0"
 ; RV32XTHEADSYNC: .attribute 5, "rv32i2p1_xtheadsync1p0"
 ; RV32XWCHC: .attribute 5, "rv32i2p1_xwchc2p2"
+; RV32XQCIA: .attribute 5, "rv32i2p1_xqcia0p2"
+; RV32XQCICSR: .attribute 5, "rv32i2p1_xqcicsr0p2"
+; RV32XQCISLS: .attribute 5, "rv32i2p1_xqcisls0p2"
 ; RV32ZAAMO: .attribute 5, "rv32i2p1_zaamo1p0"
 ; RV32ZALRSC: .attribute 5, "rv32i2p1_zalrsc1p0"
 ; RV32ZCA: .attribute 5, "rv32i2p1_zca1p0"
diff --git a/llvm/test/CodeGen/RISCV/convert-highly-predictable-select-to-branch.ll b/llvm/test/CodeGen/RISCV/convert-highly-predictable-select-to-branch.ll
index 1abd774d7a487..87c666399e03c 100644
--- a/llvm/test/CodeGen/RISCV/convert-highly-predictable-select-to-branch.ll
+++ b/llvm/test/CodeGen/RISCV/convert-highly-predictable-select-to-branch.ll
@@ -19,7 +19,7 @@ entry:
 }
 
 ; Test has highly predictable select according to profile data,
-; which should be transformed to a branch on cores with enabled FeaturePredictableSelectIsExpensive
+; which should be transformed to a branch on cores with enabled TunePredictableSelectIsExpensive
 define i32 @test2(i32 %a) {
 ; CHEAP-LABEL: test2:
 ; CHEAP:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
index f463bb2009f95..fddb86de58f51 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
@@ -435,6 +435,414 @@ define double @tan_f64(double %a) nounwind strictfp {
   ret double %1
 }
 
+define double @asin_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: asin_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call asin
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: asin_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call asin
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: asin_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call asin
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: asin_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call asin
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: asin_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asin
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asin
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.asin.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+define double @acos_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: acos_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call acos
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: acos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call acos
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: acos_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call acos
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: acos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call acos
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: acos_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acos
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acos
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.acos.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+define double @atan_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: atan_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call atan
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: atan_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call atan
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: atan_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call atan
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: atan_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call atan
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: atan_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.atan.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+declare double @llvm.experimental.constrained.atan2.f64(double, double, metadata, metadata)
+
+define double @atan2_f64(double %a, double %b) nounwind strictfp {
+; RV32IFD-LABEL: atan2_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call atan2
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: atan2_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call atan2
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: atan2_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call atan2
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: atan2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call atan2
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: atan2_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan2_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.atan2.f64(double %a, double %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+define double @sinh_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: sinh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call sinh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: sinh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call sinh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: sinh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call sinh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: sinh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call sinh
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: sinh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.sinh.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+define double @cosh_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: cosh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call cosh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: cosh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call cosh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: cosh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call cosh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: cosh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call cosh
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: cosh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call cosh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call cosh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.cosh.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
+define double @tanh_f64(double %a) nounwind strictfp {
+; RV32IFD-LABEL: tanh_f64:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call tanh
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: tanh_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call tanh
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IZFINXZDINX-LABEL: tanh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call tanh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: tanh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call tanh
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: tanh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.experimental.constrained.tanh.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret double %1
+}
+
 declare double @llvm.experimental.constrained.pow.f64(double, double, metadata, metadata)
 
 define double @pow_f64(double %a, double %b) nounwind strictfp {
@@ -1636,66 +2044,6 @@ define i64 @llround_f64(double %a) nounwind strictfp {
   ret i64 %1
 }
 
-declare double @llvm.experimental.constrained.atan2.f64(double, double, metadata, metadata)
-
-define double @atan2_f64(double %a, double %b) nounwind strictfp {
-; RV32IFD-LABEL: atan2_f64:
-; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    call atan2
-; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 16
-; RV32IFD-NEXT:    ret
-;
-; RV64IFD-LABEL: atan2_f64:
-; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    call atan2
-; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IFD-NEXT:    addi sp, sp, 16
-; RV64IFD-NEXT:    ret
-;
-; RV32IZFINXZDINX-LABEL: atan2_f64:
-; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
-; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    call atan2
-; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
-; RV32IZFINXZDINX-NEXT:    ret
-;
-; RV64IZFINXZDINX-LABEL: atan2_f64:
-; RV64IZFINXZDINX:       # %bb.0:
-; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
-; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINXZDINX-NEXT:    call atan2
-; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
-; RV64IZFINXZDINX-NEXT:    ret
-;
-; RV32I-LABEL: atan2_f64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    call atan2
-; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    addi sp, sp, 16
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: atan2_f64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    call atan2
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-  %1 = call double @llvm.experimental.constrained.atan2.f64(double %a, double %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
-  ret double %1
-}
-
 define double @ldexp_f64(double %x, i32 signext %y) nounwind {
 ; RV32IFD-LABEL: ldexp_f64:
 ; RV32IFD:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
index ebeca7c0c362a..2ae39e244cfeb 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
@@ -453,6 +453,45 @@ define double @exp2_f64(double %a) nounwind {
   ret double %1
 }
 
+define double @exp10_f64(double %a) nounwind {
+; CHECKIFD-LABEL: exp10_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail exp10
+;
+; RV32IZFINXZDINX-LABEL: exp10_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call exp10
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: exp10_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail exp10
+;
+; RV32I-LABEL: exp10_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call exp10
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: exp10_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call exp10
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.exp10.f64(double %a)
+  ret double %1
+}
+
 declare double @llvm.log.f64(double)
 
 define double @log_f64(double %a) nounwind {
@@ -844,16 +883,16 @@ define double @floor_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: floor_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI17_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI18_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB17_2
+; RV64IFD-NEXT:    beqz a0, .LBB18_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rdn
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0, rdn
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB17_2:
+; RV64IFD-NEXT:  .LBB18_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: floor_f64:
@@ -871,12 +910,12 @@ define double @floor_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB17_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB18_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rdn
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rdn
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB17_2:
+; RV64IZFINXZDINX-NEXT:  .LBB18_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: floor_f64:
@@ -909,16 +948,16 @@ define double @ceil_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: ceil_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI18_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI19_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI19_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB18_2
+; RV64IFD-NEXT:    beqz a0, .LBB19_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rup
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0, rup
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB18_2:
+; RV64IFD-NEXT:  .LBB19_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: ceil_f64:
@@ -936,12 +975,12 @@ define double @ceil_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB18_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB19_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rup
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rup
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB18_2:
+; RV64IZFINXZDINX-NEXT:  .LBB19_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: ceil_f64:
@@ -974,16 +1013,16 @@ define double @trunc_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: trunc_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI19_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI20_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB19_2
+; RV64IFD-NEXT:    beqz a0, .LBB20_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0, rtz
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB19_2:
+; RV64IFD-NEXT:  .LBB20_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: trunc_f64:
@@ -1001,12 +1040,12 @@ define double @trunc_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB19_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB20_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rtz
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rtz
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB19_2:
+; RV64IZFINXZDINX-NEXT:  .LBB20_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: trunc_f64:
@@ -1039,16 +1078,16 @@ define double @rint_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: rint_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI20_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI21_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI21_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB20_2
+; RV64IFD-NEXT:    beqz a0, .LBB21_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB20_2:
+; RV64IFD-NEXT:  .LBB21_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: rint_f64:
@@ -1066,12 +1105,12 @@ define double @rint_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB20_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB21_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB20_2:
+; RV64IZFINXZDINX-NEXT:  .LBB21_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: rint_f64:
@@ -1145,16 +1184,16 @@ define double @round_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: round_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI22_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI23_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI23_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB22_2
+; RV64IFD-NEXT:    beqz a0, .LBB23_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0, rmm
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB22_2:
+; RV64IFD-NEXT:  .LBB23_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: round_f64:
@@ -1172,12 +1211,12 @@ define double @round_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB22_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB23_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rmm
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rmm
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB22_2:
+; RV64IZFINXZDINX-NEXT:  .LBB23_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: round_f64:
@@ -1210,16 +1249,16 @@ define double @roundeven_f64(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: roundeven_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI23_0)
-; RV64IFD-NEXT:    fld fa5, %lo(.LCPI23_0)(a0)
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI24_0)
+; RV64IFD-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; RV64IFD-NEXT:    fabs.d fa4, fa0
 ; RV64IFD-NEXT:    flt.d a0, fa4, fa5
-; RV64IFD-NEXT:    beqz a0, .LBB23_2
+; RV64IFD-NEXT:    beqz a0, .LBB24_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rne
 ; RV64IFD-NEXT:    fcvt.d.l fa5, a0, rne
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
-; RV64IFD-NEXT:  .LBB23_2:
+; RV64IFD-NEXT:  .LBB24_2:
 ; RV64IFD-NEXT:    ret
 ;
 ; RV32IZFINXZDINX-LABEL: roundeven_f64:
@@ -1237,12 +1276,12 @@ define double @roundeven_f64(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    slli a1, a1, 52
 ; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
 ; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
-; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB23_2
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB24_2
 ; RV64IZFINXZDINX-NEXT:  # %bb.1:
 ; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rne
 ; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rne
 ; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
-; RV64IZFINXZDINX-NEXT:  .LBB23_2:
+; RV64IZFINXZDINX-NEXT:  .LBB24_2:
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: roundeven_f64:
@@ -1524,11 +1563,11 @@ define i1 @isnan_d_fpclass(double %x) {
 ; RV32I-NEXT:    slli a1, a1, 1
 ; RV32I-NEXT:    srli a1, a1, 1
 ; RV32I-NEXT:    lui a2, 524032
-; RV32I-NEXT:    beq a1, a2, .LBB29_2
+; RV32I-NEXT:    beq a1, a2, .LBB30_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    slt a0, a2, a1
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB29_2:
+; RV32I-NEXT:  .LBB30_2:
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    ret
 ;
@@ -1544,6 +1583,45 @@ define i1 @isnan_d_fpclass(double %x) {
   ret i1 %1
 }
 
+define double @tan_f64(double %a) nounwind {
+; CHECKIFD-LABEL: tan_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail tan
+;
+; RV32IZFINXZDINX-LABEL: tan_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call tan
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: tan_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail tan
+;
+; RV32I-LABEL: tan_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tan
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tan_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tan
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.tan.f64(double %a)
+  ret double %1
+}
+
 declare double @llvm.maximumnum.f64(double, double)
 
 define double @maximumnum_double(double %x, double %y) {
@@ -1769,3 +1847,276 @@ define {double, i32} @frexp_double(double %x) nounwind {
   %a = call {double, i32} @llvm.frexp.f64.i32(double %x)
   ret {double, i32} %a
 }
+
+define double @asin_f64(double %a) nounwind {
+; CHECKIFD-LABEL: asin_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail asin
+;
+; RV32IZFINXZDINX-LABEL: asin_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call asin
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: asin_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail asin
+;
+; RV32I-LABEL: asin_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asin
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asin
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.asin.f64(double %a)
+  ret double %1
+}
+
+define double @acos_f64(double %a) nounwind {
+; CHECKIFD-LABEL: acos_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail acos
+;
+; RV32IZFINXZDINX-LABEL: acos_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call acos
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: acos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail acos
+;
+; RV32I-LABEL: acos_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acos
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acos
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.acos.f64(double %a)
+  ret double %1
+}
+
+define double @atan_f64(double %a) nounwind {
+; CHECKIFD-LABEL: atan_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail atan
+;
+; RV32IZFINXZDINX-LABEL: atan_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call atan
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: atan_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail atan
+;
+; RV32I-LABEL: atan_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.atan.f64(double %a)
+  ret double %1
+}
+
+define double @atan2_f64(double %a, double %b) nounwind {
+; CHECKIFD-LABEL: atan2_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail atan2
+;
+; RV32IZFINXZDINX-LABEL: atan2_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call atan2
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: atan2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail atan2
+;
+; RV32I-LABEL: atan2_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan2_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.atan2.f64(double %a, double %b)
+  ret double %1
+}
+
+define double @sinh_f64(double %a) nounwind {
+; CHECKIFD-LABEL: sinh_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail sinh
+;
+; RV32IZFINXZDINX-LABEL: sinh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call sinh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: sinh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail sinh
+;
+; RV32I-LABEL: sinh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.sinh.f64(double %a)
+  ret double %1
+}
+
+define double @cosh_f64(double %a) nounwind {
+; CHECKIFD-LABEL: cosh_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail cosh
+;
+; RV32IZFINXZDINX-LABEL: cosh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call cosh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: cosh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail cosh
+;
+; RV32I-LABEL: cosh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call cosh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call cosh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.cosh.f64(double %a)
+  ret double %1
+}
+
+define double @tanh_f64(double %a) nounwind {
+; CHECKIFD-LABEL: tanh_f64:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    tail tanh
+;
+; RV32IZFINXZDINX-LABEL: tanh_f64:
+; RV32IZFINXZDINX:       # %bb.0:
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    call tanh
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV32IZFINXZDINX-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: tanh_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail tanh
+;
+; RV32I-LABEL: tanh_f64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanh
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanh
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call double @llvm.tanh.f64(double %a)
+  ret double %1
+}
diff --git a/llvm/test/CodeGen/RISCV/double-zfa.ll b/llvm/test/CodeGen/RISCV/double-zfa.ll
index f8c10e1943947..2f35496b9b32c 100644
--- a/llvm/test/CodeGen/RISCV/double-zfa.ll
+++ b/llvm/test/CodeGen/RISCV/double-zfa.ll
@@ -243,6 +243,17 @@ define double @fround_d_5(double %a) nounwind {
 
 declare double @nearbyint(double) nounwind readnone
 
+define double @fround_d_6(double %a) nounwind {
+; CHECK-LABEL: fround_d_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.d fa0, fa0, rne
+; CHECK-NEXT:    ret
+  %call = tail call double @llvm.roundeven.f64(double %a) nounwind readnone
+  ret double %call
+}
+
+declare double @llvm.roundeven.f64(double) nounwind readnone
+
 
 define double @froundnx_d(double %a) nounwind {
 ; CHECK-LABEL: froundnx_d:
@@ -378,10 +389,10 @@ define double @select_loadfpimm(double %x) nounwind {
 ; RV32IDZFA-NEXT:    fcvt.d.w fa5, zero
 ; RV32IDZFA-NEXT:    fle.d a0, fa5, fa0
 ; RV32IDZFA-NEXT:    fli.d fa0, 0.5
-; RV32IDZFA-NEXT:    bnez a0, .LBB35_2
+; RV32IDZFA-NEXT:    bnez a0, .LBB36_2
 ; RV32IDZFA-NEXT:  # %bb.1:
 ; RV32IDZFA-NEXT:    fneg.d fa0, fa0
-; RV32IDZFA-NEXT:  .LBB35_2: # %entry
+; RV32IDZFA-NEXT:  .LBB36_2: # %entry
 ; RV32IDZFA-NEXT:    ret
 ;
 ; RV64DZFA-LABEL: select_loadfpimm:
@@ -389,10 +400,10 @@ define double @select_loadfpimm(double %x) nounwind {
 ; RV64DZFA-NEXT:    fmv.d.x fa5, zero
 ; RV64DZFA-NEXT:    fle.d a0, fa5, fa0
 ; RV64DZFA-NEXT:    fli.d fa0, 0.5
-; RV64DZFA-NEXT:    bnez a0, .LBB35_2
+; RV64DZFA-NEXT:    bnez a0, .LBB36_2
 ; RV64DZFA-NEXT:  # %bb.1:
 ; RV64DZFA-NEXT:    fneg.d fa0, fa0
-; RV64DZFA-NEXT:  .LBB35_2: # %entry
+; RV64DZFA-NEXT:  .LBB36_2: # %entry
 ; RV64DZFA-NEXT:    ret
 entry:
   %cmp = fcmp ult double %x, 0.000000e+00
diff --git a/llvm/test/CodeGen/RISCV/fixed-csr.ll b/llvm/test/CodeGen/RISCV/fixed-csr.ll
new file mode 100644
index 0000000000000..f39085132e4a2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fixed-csr.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -mattr=+reserve-x24 < %s | FileCheck %s
+
+define noundef signext i32 @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li s8, 321
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+  tail call void @llvm.write_register.i64(metadata !0, i64 321)
+  ret i32 0
+}
+
+declare void @llvm.write_register.i64(metadata, i64)
+
+define noundef signext i32 @bar() nounwind {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd s9, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    li s8, 321
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ld s9, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  tail call void asm sideeffect "", "~{x25}"() #3
+  tail call void @llvm.write_register.i64(metadata !0, i64 321)
+  ret i32 0
+}
+
+!llvm.named.register.x24 = !{!0}
+!0 = !{!"x24"}
+
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
index 4c383be1ac42c..8b883f781c9d9 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
@@ -414,6 +414,180 @@ define float @tan_f32(float %a) nounwind strictfp {
   ret float %1
 }
 
+define float @asin_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: asin_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call asinf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: asin_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call asinf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: asin_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call asinf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: asin_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call asinf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: asin_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asinf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asinf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.asin.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
+define float @acos_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: acos_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call acosf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: acos_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call acosf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: acos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call acosf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: acos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call acosf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: acos_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acosf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acosf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.acos.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
+define float @atan_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: atan_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call atanf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: atan_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call atanf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: atan_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call atanf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: atan_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call atanf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: atan_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atanf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atanf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.atan.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
 declare float @llvm.experimental.constrained.atan2.f32(float, float, metadata, metadata)
 
 define float @atan2_f32(float %a, float %b) nounwind strictfp {
@@ -474,6 +648,180 @@ define float @atan2_f32(float %a, float %b) nounwind strictfp {
   ret float %1
 }
 
+define float @sinh_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: sinh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call sinhf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: sinh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call sinhf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: sinh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call sinhf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sinh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call sinhf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: sinh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.sinh.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
+define float @cosh_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: cosh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call coshf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: cosh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call coshf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: cosh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call coshf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: cosh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call coshf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: cosh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call coshf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call coshf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.cosh.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
+define float @tanh_f32(float %a) nounwind strictfp {
+; RV32IF-LABEL: tanh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call tanhf
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: tanh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call tanhf
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: tanh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call tanhf
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: tanh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call tanhf
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: tanh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.experimental.constrained.tanh.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+  ret float %1
+}
+
 declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
 
 define float @pow_f32(float %a, float %b) nounwind strictfp {
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index d42afd504e5dc..e27b5e27dec1d 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -420,6 +420,44 @@ define float @exp2_f32(float %a) nounwind {
   ret float %1
 }
 
+define float @exp10_f32(float %a) nounwind {
+; RV32IF-LABEL: exp10_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail exp10f
+;
+; RV32IZFINX-LABEL: exp10_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail exp10f
+;
+; RV64IF-LABEL: exp10_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail exp10f
+;
+; RV64IZFINX-LABEL: exp10_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail exp10f
+;
+; RV32I-LABEL: exp10_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call exp10f
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: exp10_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call exp10f
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.exp10.f32(float %a)
+  ret float %1
+}
+
 declare float @llvm.log.f32(float)
 
 define float @log_f32(float %a) nounwind {
@@ -834,12 +872,12 @@ define float @floor_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB17_2
+; RV32IF-NEXT:    beqz a0, .LBB18_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rdn
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0, rdn
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB17_2:
+; RV32IF-NEXT:  .LBB18_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: floor_f32:
@@ -847,12 +885,12 @@ define float @floor_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB17_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB18_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB17_2:
+; RV32IZFINX-NEXT:  .LBB18_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: floor_f32:
@@ -861,12 +899,12 @@ define float @floor_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB17_2
+; RV64IF-NEXT:    beqz a0, .LBB18_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rdn
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0, rdn
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB17_2:
+; RV64IF-NEXT:  .LBB18_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: floor_f32:
@@ -874,12 +912,12 @@ define float @floor_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB17_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB18_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB17_2:
+; RV64IZFINX-NEXT:  .LBB18_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: floor_f32:
@@ -912,12 +950,12 @@ define float @ceil_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB18_2
+; RV32IF-NEXT:    beqz a0, .LBB19_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rup
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0, rup
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB18_2:
+; RV32IF-NEXT:  .LBB19_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: ceil_f32:
@@ -925,12 +963,12 @@ define float @ceil_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB18_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB19_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rup
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rup
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB18_2:
+; RV32IZFINX-NEXT:  .LBB19_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: ceil_f32:
@@ -939,12 +977,12 @@ define float @ceil_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB18_2
+; RV64IF-NEXT:    beqz a0, .LBB19_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rup
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0, rup
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB18_2:
+; RV64IF-NEXT:  .LBB19_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: ceil_f32:
@@ -952,12 +990,12 @@ define float @ceil_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB18_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB19_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rup
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rup
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB18_2:
+; RV64IZFINX-NEXT:  .LBB19_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: ceil_f32:
@@ -990,12 +1028,12 @@ define float @trunc_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB19_2
+; RV32IF-NEXT:    beqz a0, .LBB20_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0, rtz
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB19_2:
+; RV32IF-NEXT:  .LBB20_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: trunc_f32:
@@ -1003,12 +1041,12 @@ define float @trunc_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB19_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB20_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB19_2:
+; RV32IZFINX-NEXT:  .LBB20_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: trunc_f32:
@@ -1017,12 +1055,12 @@ define float @trunc_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB19_2
+; RV64IF-NEXT:    beqz a0, .LBB20_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0, rtz
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB19_2:
+; RV64IF-NEXT:  .LBB20_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: trunc_f32:
@@ -1030,12 +1068,12 @@ define float @trunc_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB19_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB20_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB19_2:
+; RV64IZFINX-NEXT:  .LBB20_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: trunc_f32:
@@ -1068,12 +1106,12 @@ define float @rint_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB20_2
+; RV32IF-NEXT:    beqz a0, .LBB21_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB20_2:
+; RV32IF-NEXT:  .LBB21_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: rint_f32:
@@ -1081,12 +1119,12 @@ define float @rint_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB20_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB21_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB20_2:
+; RV32IZFINX-NEXT:  .LBB21_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: rint_f32:
@@ -1095,12 +1133,12 @@ define float @rint_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB20_2
+; RV64IF-NEXT:    beqz a0, .LBB21_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB20_2:
+; RV64IF-NEXT:  .LBB21_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: rint_f32:
@@ -1108,12 +1146,12 @@ define float @rint_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB20_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB21_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB20_2:
+; RV64IZFINX-NEXT:  .LBB21_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: rint_f32:
@@ -1186,12 +1224,12 @@ define float @round_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB22_2
+; RV32IF-NEXT:    beqz a0, .LBB23_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0, rmm
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB22_2:
+; RV32IF-NEXT:  .LBB23_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: round_f32:
@@ -1199,12 +1237,12 @@ define float @round_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB22_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB23_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB22_2:
+; RV32IZFINX-NEXT:  .LBB23_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: round_f32:
@@ -1213,12 +1251,12 @@ define float @round_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB22_2
+; RV64IF-NEXT:    beqz a0, .LBB23_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0, rmm
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB22_2:
+; RV64IF-NEXT:  .LBB23_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: round_f32:
@@ -1226,12 +1264,12 @@ define float @round_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB22_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB23_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB22_2:
+; RV64IZFINX-NEXT:  .LBB23_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: round_f32:
@@ -1264,12 +1302,12 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fabs.s fa4, fa0
 ; RV32IF-NEXT:    flt.s a0, fa4, fa5
-; RV32IF-NEXT:    beqz a0, .LBB23_2
+; RV32IF-NEXT:    beqz a0, .LBB24_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rne
 ; RV32IF-NEXT:    fcvt.s.w fa5, a0, rne
 ; RV32IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV32IF-NEXT:  .LBB23_2:
+; RV32IF-NEXT:  .LBB24_2:
 ; RV32IF-NEXT:    ret
 ;
 ; RV32IZFINX-LABEL: roundeven_f32:
@@ -1277,12 +1315,12 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV32IZFINX-NEXT:    lui a1, 307200
 ; RV32IZFINX-NEXT:    fabs.s a2, a0
 ; RV32IZFINX-NEXT:    flt.s a1, a2, a1
-; RV32IZFINX-NEXT:    beqz a1, .LBB23_2
+; RV32IZFINX-NEXT:    beqz a1, .LBB24_2
 ; RV32IZFINX-NEXT:  # %bb.1:
 ; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rne
 ; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rne
 ; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV32IZFINX-NEXT:  .LBB23_2:
+; RV32IZFINX-NEXT:  .LBB24_2:
 ; RV32IZFINX-NEXT:    ret
 ;
 ; RV64IF-LABEL: roundeven_f32:
@@ -1291,12 +1329,12 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fabs.s fa4, fa0
 ; RV64IF-NEXT:    flt.s a0, fa4, fa5
-; RV64IF-NEXT:    beqz a0, .LBB23_2
+; RV64IF-NEXT:    beqz a0, .LBB24_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rne
 ; RV64IF-NEXT:    fcvt.s.w fa5, a0, rne
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
-; RV64IF-NEXT:  .LBB23_2:
+; RV64IF-NEXT:  .LBB24_2:
 ; RV64IF-NEXT:    ret
 ;
 ; RV64IZFINX-LABEL: roundeven_f32:
@@ -1304,12 +1342,12 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV64IZFINX-NEXT:    lui a1, 307200
 ; RV64IZFINX-NEXT:    fabs.s a2, a0
 ; RV64IZFINX-NEXT:    flt.s a1, a2, a1
-; RV64IZFINX-NEXT:    beqz a1, .LBB23_2
+; RV64IZFINX-NEXT:    beqz a1, .LBB24_2
 ; RV64IZFINX-NEXT:  # %bb.1:
 ; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rne
 ; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rne
 ; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
-; RV64IZFINX-NEXT:  .LBB23_2:
+; RV64IZFINX-NEXT:  .LBB24_2:
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: roundeven_f32:
@@ -2139,6 +2177,44 @@ define i1 @isnotfinite_fpclass(float %x) {
   ret i1 %1
 }
 
+define float @tan_f32(float %a) nounwind {
+; RV32IF-LABEL: tan_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail tanf
+;
+; RV32IZFINX-LABEL: tan_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail tanf
+;
+; RV64IF-LABEL: tan_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail tanf
+;
+; RV64IZFINX-LABEL: tan_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail tanf
+;
+; RV32I-LABEL: tan_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tan_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.tan.f32(float %a)
+  ret float %1
+}
+
 declare float @llvm.maximumnum.f32(float, float)
 
 define float @maximumnum_float(float %x, float %y) {
@@ -2360,3 +2436,269 @@ define {float, i32} @frexp_float(float %x) nounwind {
   %a = call {float, i32} @llvm.frexp.f32.i32(float %x)
   ret {float, i32} %a
 }
+
+define float @asin_f32(float %a) nounwind {
+; RV32IF-LABEL: asin_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail asinf
+;
+; RV32IZFINX-LABEL: asin_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail asinf
+;
+; RV64IF-LABEL: asin_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail asinf
+;
+; RV64IZFINX-LABEL: asin_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail asinf
+;
+; RV32I-LABEL: asin_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call asinf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call asinf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.asin.f32(float %a)
+  ret float %1
+}
+
+define float @acos_f32(float %a) nounwind {
+; RV32IF-LABEL: acos_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail acosf
+;
+; RV32IZFINX-LABEL: acos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail acosf
+;
+; RV64IF-LABEL: acos_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail acosf
+;
+; RV64IZFINX-LABEL: acos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail acosf
+;
+; RV32I-LABEL: acos_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call acosf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call acosf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.acos.f32(float %a)
+  ret float %1
+}
+
+define float @atan_f32(float %a) nounwind {
+; RV32IF-LABEL: atan_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail atanf
+;
+; RV32IZFINX-LABEL: atan_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail atanf
+;
+; RV64IF-LABEL: atan_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail atanf
+;
+; RV64IZFINX-LABEL: atan_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail atanf
+;
+; RV32I-LABEL: atan_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atanf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atanf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.atan.f32(float %a)
+  ret float %1
+}
+
+define float @atan2_f32(float %a, float %b) nounwind {
+; RV32IF-LABEL: atan2_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail atan2f
+;
+; RV32IZFINX-LABEL: atan2_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail atan2f
+;
+; RV64IF-LABEL: atan2_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail atan2f
+;
+; RV64IZFINX-LABEL: atan2_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail atan2f
+;
+; RV32I-LABEL: atan2_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call atan2f
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan2_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call atan2f
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.atan2.f32(float %a, float %b)
+  ret float %1
+}
+
+define float @sinh_f32(float %a) nounwind {
+; RV32IF-LABEL: sinh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail sinhf
+;
+; RV32IZFINX-LABEL: sinh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail sinhf
+;
+; RV64IF-LABEL: sinh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail sinhf
+;
+; RV64IZFINX-LABEL: sinh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail sinhf
+;
+; RV32I-LABEL: sinh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call sinhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.sinh.f32(float %a)
+  ret float %1
+}
+
+define float @cosh_f32(float %a) nounwind {
+; RV32IF-LABEL: cosh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail coshf
+;
+; RV32IZFINX-LABEL: cosh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail coshf
+;
+; RV64IF-LABEL: cosh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail coshf
+;
+; RV64IZFINX-LABEL: cosh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail coshf
+;
+; RV32I-LABEL: cosh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call coshf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call coshf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.cosh.f32(float %a)
+  ret float %1
+}
+
+define float @tanh_f32(float %a) nounwind {
+; RV32IF-LABEL: tanh_f32:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    tail tanhf
+;
+; RV32IZFINX-LABEL: tanh_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail tanhf
+;
+; RV64IF-LABEL: tanh_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    tail tanhf
+;
+; RV64IZFINX-LABEL: tanh_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    tail tanhf
+;
+; RV32I-LABEL: tanh_f32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call tanhf
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call tanhf
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = call float @llvm.tanh.f32(float %a)
+  ret float %1
+}
diff --git a/llvm/test/CodeGen/RISCV/float-zfa.ll b/llvm/test/CodeGen/RISCV/float-zfa.ll
index 5f0f45eed98ff..aec5ac75a9795 100644
--- a/llvm/test/CodeGen/RISCV/float-zfa.ll
+++ b/llvm/test/CodeGen/RISCV/float-zfa.ll
@@ -198,6 +198,17 @@ define float @fround_s_5(float %a) nounwind {
 
 declare float @nearbyintf(float) nounwind readnone
 
+define float @fround_s_6(float %a) nounwind {
+; CHECK-LABEL: fround_s_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.s fa0, fa0, rne
+; CHECK-NEXT:    ret
+  %call = tail call float @llvm.roundeven.f32(float %a) nounwind readnone
+  ret float %call
+}
+
+declare float @llvm.roundeven.f32(float) nounwind readnone
+
 
 define float @froundnx_s(float %a) nounwind {
 ; CHECK-LABEL: froundnx_s:
@@ -317,10 +328,10 @@ define float @select_loadfpimm(float %x) nounwind {
 ; CHECK-NEXT:    fmv.w.x fa5, zero
 ; CHECK-NEXT:    fle.s a0, fa5, fa0
 ; CHECK-NEXT:    fli.s fa0, 0.5
-; CHECK-NEXT:    bnez a0, .LBB30_2
+; CHECK-NEXT:    bnez a0, .LBB31_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    fneg.s fa0, fa0
-; CHECK-NEXT:  .LBB30_2: # %entry
+; CHECK-NEXT:  .LBB31_2: # %entry
 ; CHECK-NEXT:    ret
 entry:
   %cmp = fcmp ult float %x, 0.000000e+00
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index 581ee5cd2304d..a8e26f7686e50 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -91,3 +91,101 @@ define i32 @test_add_and_fptosi() nounwind {
   %4 = fptosi fp128 %3 to i32
   ret i32 %4
 }
+
+define fp128 @fmaximum(fp128 %x, fp128 %y) {
+; RV32I-LABEL: fmaximum:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    .cfi_def_cfa_offset 64
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a7, 4(a2)
+; RV32I-NEXT:    lw t0, 8(a2)
+; RV32I-NEXT:    lw a2, 12(a2)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 8(sp)
+; RV32I-NEXT:    sw a7, 12(sp)
+; RV32I-NEXT:    sw t0, 16(sp)
+; RV32I-NEXT:    sw a2, 20(sp)
+; RV32I-NEXT:    addi a0, sp, 40
+; RV32I-NEXT:    addi a1, sp, 24
+; RV32I-NEXT:    addi a2, sp, 8
+; RV32I-NEXT:    sw a3, 24(sp)
+; RV32I-NEXT:    sw a4, 28(sp)
+; RV32I-NEXT:    sw a5, 32(sp)
+; RV32I-NEXT:    sw a6, 36(sp)
+; RV32I-NEXT:    call fmaximuml
+; RV32I-NEXT:    lw a0, 40(sp)
+; RV32I-NEXT:    lw a1, 44(sp)
+; RV32I-NEXT:    lw a2, 48(sp)
+; RV32I-NEXT:    lw a3, 52(sp)
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    sw a2, 8(s0)
+; RV32I-NEXT:    sw a3, 12(s0)
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+  %a = call fp128 @llvm.maximum.fp128(fp128 %x, fp128 %y)
+  ret fp128 %a
+}
+
+define fp128 @fminimum(fp128 %x, fp128 %y) {
+; RV32I-LABEL: fminimum:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    .cfi_def_cfa_offset 64
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a7, 4(a2)
+; RV32I-NEXT:    lw t0, 8(a2)
+; RV32I-NEXT:    lw a2, 12(a2)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 8(sp)
+; RV32I-NEXT:    sw a7, 12(sp)
+; RV32I-NEXT:    sw t0, 16(sp)
+; RV32I-NEXT:    sw a2, 20(sp)
+; RV32I-NEXT:    addi a0, sp, 40
+; RV32I-NEXT:    addi a1, sp, 24
+; RV32I-NEXT:    addi a2, sp, 8
+; RV32I-NEXT:    sw a3, 24(sp)
+; RV32I-NEXT:    sw a4, 28(sp)
+; RV32I-NEXT:    sw a5, 32(sp)
+; RV32I-NEXT:    sw a6, 36(sp)
+; RV32I-NEXT:    call fminimuml
+; RV32I-NEXT:    lw a0, 40(sp)
+; RV32I-NEXT:    lw a1, 44(sp)
+; RV32I-NEXT:    lw a2, 48(sp)
+; RV32I-NEXT:    lw a3, 52(sp)
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    sw a2, 8(s0)
+; RV32I-NEXT:    sw a3, 12(s0)
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+  %a = call fp128 @llvm.minimum.fp128(fp128 %x, fp128 %y)
+  ret fp128 %a
+}
diff --git a/llvm/test/CodeGen/RISCV/half-convert-strict.ll b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
index 2f6c30ff210e3..0a04d44893e75 100644
--- a/llvm/test/CodeGen/RISCV/half-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
@@ -2510,3 +2510,473 @@ define signext i32 @fcvt_h_wu_demanded_bits(i32 signext %0, ptr %1) strictfp {
   store half %4, ptr %1, align 2
   ret i32 %3
 }
+
+define half @fcvt_h_q(fp128 %a) nounwind strictfp {
+; RV32IZFH-LABEL: fcvt_h_q:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -32
+; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    lw a1, 0(a0)
+; RV32IZFH-NEXT:    lw a2, 4(a0)
+; RV32IZFH-NEXT:    lw a3, 8(a0)
+; RV32IZFH-NEXT:    lw a4, 12(a0)
+; RV32IZFH-NEXT:    addi a0, sp, 8
+; RV32IZFH-NEXT:    sw a1, 8(sp)
+; RV32IZFH-NEXT:    sw a2, 12(sp)
+; RV32IZFH-NEXT:    sw a3, 16(sp)
+; RV32IZFH-NEXT:    sw a4, 20(sp)
+; RV32IZFH-NEXT:    call __trunctfhf2
+; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 32
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_q:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    call __trunctfhf2
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: fcvt_h_q:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -32
+; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    lw a1, 0(a0)
+; RV32IZHINX-NEXT:    lw a2, 4(a0)
+; RV32IZHINX-NEXT:    lw a3, 8(a0)
+; RV32IZHINX-NEXT:    lw a4, 12(a0)
+; RV32IZHINX-NEXT:    addi a0, sp, 8
+; RV32IZHINX-NEXT:    sw a1, 8(sp)
+; RV32IZHINX-NEXT:    sw a2, 12(sp)
+; RV32IZHINX-NEXT:    sw a3, 16(sp)
+; RV32IZHINX-NEXT:    sw a4, 20(sp)
+; RV32IZHINX-NEXT:    call __trunctfhf2
+; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 32
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: fcvt_h_q:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    call __trunctfhf2
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_q:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -32
+; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    lw a1, 0(a0)
+; RV32IDZFH-NEXT:    lw a2, 4(a0)
+; RV32IDZFH-NEXT:    lw a3, 8(a0)
+; RV32IDZFH-NEXT:    lw a4, 12(a0)
+; RV32IDZFH-NEXT:    addi a0, sp, 8
+; RV32IDZFH-NEXT:    sw a1, 8(sp)
+; RV32IDZFH-NEXT:    sw a2, 12(sp)
+; RV32IDZFH-NEXT:    sw a3, 16(sp)
+; RV32IDZFH-NEXT:    sw a4, 20(sp)
+; RV32IDZFH-NEXT:    call __trunctfhf2
+; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 32
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_q:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    addi sp, sp, -16
+; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IDZFH-NEXT:    call __trunctfhf2
+; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IDZFH-NEXT:    addi sp, sp, 16
+; RV64IDZFH-NEXT:    ret
+;
+; RV32IZDINXZHINX-LABEL: fcvt_h_q:
+; RV32IZDINXZHINX:       # %bb.0:
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
+; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    lw a1, 0(a0)
+; RV32IZDINXZHINX-NEXT:    lw a2, 4(a0)
+; RV32IZDINXZHINX-NEXT:    lw a3, 8(a0)
+; RV32IZDINXZHINX-NEXT:    lw a4, 12(a0)
+; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
+; RV32IZDINXZHINX-NEXT:    sw a1, 8(sp)
+; RV32IZDINXZHINX-NEXT:    sw a2, 12(sp)
+; RV32IZDINXZHINX-NEXT:    sw a3, 16(sp)
+; RV32IZDINXZHINX-NEXT:    sw a4, 20(sp)
+; RV32IZDINXZHINX-NEXT:    call __trunctfhf2
+; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
+; RV32IZDINXZHINX-NEXT:    ret
+;
+; RV64IZDINXZHINX-LABEL: fcvt_h_q:
+; RV64IZDINXZHINX:       # %bb.0:
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
+; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZDINXZHINX-NEXT:    call __trunctfhf2
+; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
+; RV64IZDINXZHINX-NEXT:    ret
+;
+; CHECK32-IZFHMIN-LABEL: fcvt_h_q:
+; CHECK32-IZFHMIN:       # %bb.0:
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZFHMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    lw a1, 0(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a2, 4(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a3, 8(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a4, 12(a0)
+; CHECK32-IZFHMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZFHMIN-NEXT:    sw a1, 8(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a2, 12(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a3, 16(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a4, 20(sp)
+; CHECK32-IZFHMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZFHMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZFHMIN-NEXT:    ret
+;
+; CHECK64-IZFHMIN-LABEL: fcvt_h_q:
+; CHECK64-IZFHMIN:       # %bb.0:
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFHMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZFHMIN-NEXT:    ret
+;
+; CHECK32-IZHINXMIN-LABEL: fcvt_h_q:
+; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    lw a1, 0(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a2, 4(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a3, 8(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a4, 12(a0)
+; CHECK32-IZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINXMIN-NEXT:    sw a1, 8(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a2, 12(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a3, 16(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a4, 20(sp)
+; CHECK32-IZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZHINXMIN-LABEL: fcvt_h_q:
+; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINXMIN-NEXT:    ret
+;
+; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_q:
+; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a1, 0(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 4(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 8(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a4, 12(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a1, 8(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 12(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 16(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a4, 20(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_q:
+; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_q:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -32
+; CHECK32-D-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    lw a1, 0(a0)
+; CHECK32-D-NEXT:    lw a2, 4(a0)
+; CHECK32-D-NEXT:    lw a3, 8(a0)
+; CHECK32-D-NEXT:    lw a4, 12(a0)
+; CHECK32-D-NEXT:    addi a0, sp, 8
+; CHECK32-D-NEXT:    sw a1, 8(sp)
+; CHECK32-D-NEXT:    sw a2, 12(sp)
+; CHECK32-D-NEXT:    sw a3, 16(sp)
+; CHECK32-D-NEXT:    sw a4, 20(sp)
+; CHECK32-D-NEXT:    call __trunctfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 32
+; CHECK32-D-NEXT:    ret
+  %1 = call half @llvm.experimental.constrained.fptrunc.f16.f128(fp128 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret half %1
+}
+
+define fp128 @fcvt_q_h(half %a) nounwind strictfp {
+; RV32IZFH-LABEL: fcvt_q_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -32
+; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    mv s0, a0
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    addi a0, sp, 8
+; RV32IZFH-NEXT:    call __extendsftf2
+; RV32IZFH-NEXT:    lw a0, 8(sp)
+; RV32IZFH-NEXT:    lw a1, 12(sp)
+; RV32IZFH-NEXT:    lw a2, 16(sp)
+; RV32IZFH-NEXT:    lw a3, 20(sp)
+; RV32IZFH-NEXT:    sw a0, 0(s0)
+; RV32IZFH-NEXT:    sw a1, 4(s0)
+; RV32IZFH-NEXT:    sw a2, 8(s0)
+; RV32IZFH-NEXT:    sw a3, 12(s0)
+; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 32
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_q_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call __extendsftf2
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: fcvt_q_h:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -32
+; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    mv s0, a0
+; RV32IZHINX-NEXT:    fcvt.s.h a1, a1
+; RV32IZHINX-NEXT:    addi a0, sp, 8
+; RV32IZHINX-NEXT:    call __extendsftf2
+; RV32IZHINX-NEXT:    lw a0, 8(sp)
+; RV32IZHINX-NEXT:    lw a1, 12(sp)
+; RV32IZHINX-NEXT:    lw a2, 16(sp)
+; RV32IZHINX-NEXT:    lw a3, 20(sp)
+; RV32IZHINX-NEXT:    sw a0, 0(s0)
+; RV32IZHINX-NEXT:    sw a1, 4(s0)
+; RV32IZHINX-NEXT:    sw a2, 8(s0)
+; RV32IZHINX-NEXT:    sw a3, 12(s0)
+; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 32
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: fcvt_q_h:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call __extendsftf2
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_q_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -32
+; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    mv s0, a0
+; RV32IDZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IDZFH-NEXT:    addi a0, sp, 8
+; RV32IDZFH-NEXT:    call __extendsftf2
+; RV32IDZFH-NEXT:    lw a0, 8(sp)
+; RV32IDZFH-NEXT:    lw a1, 12(sp)
+; RV32IDZFH-NEXT:    lw a2, 16(sp)
+; RV32IDZFH-NEXT:    lw a3, 20(sp)
+; RV32IDZFH-NEXT:    sw a0, 0(s0)
+; RV32IDZFH-NEXT:    sw a1, 4(s0)
+; RV32IDZFH-NEXT:    sw a2, 8(s0)
+; RV32IDZFH-NEXT:    sw a3, 12(s0)
+; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 32
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_q_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    addi sp, sp, -16
+; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IDZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IDZFH-NEXT:    call __extendsftf2
+; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IDZFH-NEXT:    addi sp, sp, 16
+; RV64IDZFH-NEXT:    ret
+;
+; RV32IZDINXZHINX-LABEL: fcvt_q_h:
+; RV32IZDINXZHINX:       # %bb.0:
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
+; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    mv s0, a0
+; RV32IZDINXZHINX-NEXT:    fcvt.s.h a1, a1
+; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
+; RV32IZDINXZHINX-NEXT:    call __extendsftf2
+; RV32IZDINXZHINX-NEXT:    lw a0, 8(sp)
+; RV32IZDINXZHINX-NEXT:    lw a1, 12(sp)
+; RV32IZDINXZHINX-NEXT:    lw a2, 16(sp)
+; RV32IZDINXZHINX-NEXT:    lw a3, 20(sp)
+; RV32IZDINXZHINX-NEXT:    sw a0, 0(s0)
+; RV32IZDINXZHINX-NEXT:    sw a1, 4(s0)
+; RV32IZDINXZHINX-NEXT:    sw a2, 8(s0)
+; RV32IZDINXZHINX-NEXT:    sw a3, 12(s0)
+; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
+; RV32IZDINXZHINX-NEXT:    ret
+;
+; RV64IZDINXZHINX-LABEL: fcvt_q_h:
+; RV64IZDINXZHINX:       # %bb.0:
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
+; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZDINXZHINX-NEXT:    call __extendsftf2
+; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
+; RV64IZDINXZHINX-NEXT:    ret
+;
+; CHECK32-IZFHMIN-LABEL: fcvt_q_h:
+; CHECK32-IZFHMIN:       # %bb.0:
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZFHMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    mv s0, a0
+; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; CHECK32-IZFHMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZFHMIN-NEXT:    call __extendsftf2
+; CHECK32-IZFHMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZFHMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZFHMIN-NEXT:    ret
+;
+; CHECK64-IZFHMIN-LABEL: fcvt_q_h:
+; CHECK64-IZFHMIN:       # %bb.0:
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; CHECK64-IZFHMIN-NEXT:    call __extendsftf2
+; CHECK64-IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZFHMIN-NEXT:    ret
+;
+; CHECK32-IZHINXMIN-LABEL: fcvt_q_h:
+; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    mv s0, a0
+; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a1, a1
+; CHECK32-IZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINXMIN-NEXT:    call __extendsftf2
+; CHECK32-IZHINXMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZHINXMIN-LABEL: fcvt_q_h:
+; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECK64-IZHINXMIN-NEXT:    call __extendsftf2
+; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINXMIN-NEXT:    ret
+;
+; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_q_h:
+; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    mv s0, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a1, a1
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZDINXZHINXMIN-NEXT:    call __extendsftf2
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_q_h:
+; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECK64-IZDINXZHINXMIN-NEXT:    call __extendsftf2
+; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_q_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -32
+; CHECK32-D-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    mv s0, a0
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    addi a0, sp, 8
+; CHECK32-D-NEXT:    call __extendsftf2
+; CHECK32-D-NEXT:    lw a0, 8(sp)
+; CHECK32-D-NEXT:    lw a1, 12(sp)
+; CHECK32-D-NEXT:    lw a2, 16(sp)
+; CHECK32-D-NEXT:    lw a3, 20(sp)
+; CHECK32-D-NEXT:    sw a0, 0(s0)
+; CHECK32-D-NEXT:    sw a1, 4(s0)
+; CHECK32-D-NEXT:    sw a2, 8(s0)
+; CHECK32-D-NEXT:    sw a3, 12(s0)
+; CHECK32-D-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 32
+; CHECK32-D-NEXT:    ret
+  %1 = call fp128 @llvm.experimental.constrained.fpext.f128.f16(half %a, metadata !"fpexcept.strict")
+  ret fp128 %1
+}
+declare fp128 @llvm.experimental.constrained.fpext.f128.f16(half, metadata)
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index 01ffcab1a6556..cf57ecd6cd1e4 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -8587,3 +8587,811 @@ start:
   %0 = tail call i32 @llvm.fptosi.sat.i32.f16(half %a)
   ret i32 %0
 }
+
+; Test lib call lowering for fp128->half to make sure it matches the ABI.
+define void @fcvt_h_q(fp128 %x, ptr %y) nounwind {
+; RV32IZFH-LABEL: fcvt_h_q:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -32
+; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    lw a2, 0(a0)
+; RV32IZFH-NEXT:    lw a3, 4(a0)
+; RV32IZFH-NEXT:    lw a4, 8(a0)
+; RV32IZFH-NEXT:    lw a5, 12(a0)
+; RV32IZFH-NEXT:    mv s0, a1
+; RV32IZFH-NEXT:    addi a0, sp, 8
+; RV32IZFH-NEXT:    sw a2, 8(sp)
+; RV32IZFH-NEXT:    sw a3, 12(sp)
+; RV32IZFH-NEXT:    sw a4, 16(sp)
+; RV32IZFH-NEXT:    sw a5, 20(sp)
+; RV32IZFH-NEXT:    call __trunctfhf2
+; RV32IZFH-NEXT:    fsh fa0, 0(s0)
+; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 32
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_q:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    mv s0, a2
+; RV64IZFH-NEXT:    call __trunctfhf2
+; RV64IZFH-NEXT:    fsh fa0, 0(s0)
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_q:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -32
+; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    lw a2, 0(a0)
+; RV32IDZFH-NEXT:    lw a3, 4(a0)
+; RV32IDZFH-NEXT:    lw a4, 8(a0)
+; RV32IDZFH-NEXT:    lw a5, 12(a0)
+; RV32IDZFH-NEXT:    mv s0, a1
+; RV32IDZFH-NEXT:    addi a0, sp, 8
+; RV32IDZFH-NEXT:    sw a2, 8(sp)
+; RV32IDZFH-NEXT:    sw a3, 12(sp)
+; RV32IDZFH-NEXT:    sw a4, 16(sp)
+; RV32IDZFH-NEXT:    sw a5, 20(sp)
+; RV32IDZFH-NEXT:    call __trunctfhf2
+; RV32IDZFH-NEXT:    fsh fa0, 0(s0)
+; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 32
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_q:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    addi sp, sp, -16
+; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IDZFH-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64IDZFH-NEXT:    mv s0, a2
+; RV64IDZFH-NEXT:    call __trunctfhf2
+; RV64IDZFH-NEXT:    fsh fa0, 0(s0)
+; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IDZFH-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IDZFH-NEXT:    addi sp, sp, 16
+; RV64IDZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: fcvt_h_q:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -32
+; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    lw a2, 0(a0)
+; RV32IZHINX-NEXT:    lw a3, 4(a0)
+; RV32IZHINX-NEXT:    lw a4, 8(a0)
+; RV32IZHINX-NEXT:    lw a5, 12(a0)
+; RV32IZHINX-NEXT:    mv s0, a1
+; RV32IZHINX-NEXT:    addi a0, sp, 8
+; RV32IZHINX-NEXT:    sw a2, 8(sp)
+; RV32IZHINX-NEXT:    sw a3, 12(sp)
+; RV32IZHINX-NEXT:    sw a4, 16(sp)
+; RV32IZHINX-NEXT:    sw a5, 20(sp)
+; RV32IZHINX-NEXT:    call __trunctfhf2
+; RV32IZHINX-NEXT:    sh a0, 0(s0)
+; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 32
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: fcvt_h_q:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    mv s0, a2
+; RV64IZHINX-NEXT:    call __trunctfhf2
+; RV64IZHINX-NEXT:    sh a0, 0(s0)
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32IZDINXZHINX-LABEL: fcvt_h_q:
+; RV32IZDINXZHINX:       # %bb.0:
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
+; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    lw a2, 0(a0)
+; RV32IZDINXZHINX-NEXT:    lw a3, 4(a0)
+; RV32IZDINXZHINX-NEXT:    lw a4, 8(a0)
+; RV32IZDINXZHINX-NEXT:    lw a5, 12(a0)
+; RV32IZDINXZHINX-NEXT:    mv s0, a1
+; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
+; RV32IZDINXZHINX-NEXT:    sw a2, 8(sp)
+; RV32IZDINXZHINX-NEXT:    sw a3, 12(sp)
+; RV32IZDINXZHINX-NEXT:    sw a4, 16(sp)
+; RV32IZDINXZHINX-NEXT:    sw a5, 20(sp)
+; RV32IZDINXZHINX-NEXT:    call __trunctfhf2
+; RV32IZDINXZHINX-NEXT:    sh a0, 0(s0)
+; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
+; RV32IZDINXZHINX-NEXT:    ret
+;
+; RV64IZDINXZHINX-LABEL: fcvt_h_q:
+; RV64IZDINXZHINX:       # %bb.0:
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
+; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZDINXZHINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64IZDINXZHINX-NEXT:    mv s0, a2
+; RV64IZDINXZHINX-NEXT:    call __trunctfhf2
+; RV64IZDINXZHINX-NEXT:    sh a0, 0(s0)
+; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZDINXZHINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
+; RV64IZDINXZHINX-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_h_q:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
+; RV32I-NEXT:    lw a4, 8(a0)
+; RV32I-NEXT:    lw a5, 12(a0)
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    addi a0, sp, 8
+; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    sw a4, 16(sp)
+; RV32I-NEXT:    sw a5, 20(sp)
+; RV32I-NEXT:    call __trunctfhf2
+; RV32I-NEXT:    sh a0, 0(s0)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_h_q:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a2
+; RV64I-NEXT:    call __trunctfhf2
+; RV64I-NEXT:    sh a0, 0(s0)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32ID-ILP32-LABEL: fcvt_h_q:
+; RV32ID-ILP32:       # %bb.0:
+; RV32ID-ILP32-NEXT:    addi sp, sp, -32
+; RV32ID-ILP32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    lw a2, 0(a0)
+; RV32ID-ILP32-NEXT:    lw a3, 4(a0)
+; RV32ID-ILP32-NEXT:    lw a4, 8(a0)
+; RV32ID-ILP32-NEXT:    lw a5, 12(a0)
+; RV32ID-ILP32-NEXT:    mv s0, a1
+; RV32ID-ILP32-NEXT:    addi a0, sp, 8
+; RV32ID-ILP32-NEXT:    sw a2, 8(sp)
+; RV32ID-ILP32-NEXT:    sw a3, 12(sp)
+; RV32ID-ILP32-NEXT:    sw a4, 16(sp)
+; RV32ID-ILP32-NEXT:    sw a5, 20(sp)
+; RV32ID-ILP32-NEXT:    call __trunctfhf2
+; RV32ID-ILP32-NEXT:    sh a0, 0(s0)
+; RV32ID-ILP32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    addi sp, sp, 32
+; RV32ID-ILP32-NEXT:    ret
+;
+; RV64ID-LP64-LABEL: fcvt_h_q:
+; RV64ID-LP64:       # %bb.0:
+; RV64ID-LP64-NEXT:    addi sp, sp, -16
+; RV64ID-LP64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-LP64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64ID-LP64-NEXT:    mv s0, a2
+; RV64ID-LP64-NEXT:    call __trunctfhf2
+; RV64ID-LP64-NEXT:    sh a0, 0(s0)
+; RV64ID-LP64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-LP64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64ID-LP64-NEXT:    addi sp, sp, 16
+; RV64ID-LP64-NEXT:    ret
+;
+; RV32ID-LABEL: fcvt_h_q:
+; RV32ID:       # %bb.0:
+; RV32ID-NEXT:    addi sp, sp, -32
+; RV32ID-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    lw a2, 0(a0)
+; RV32ID-NEXT:    lw a3, 4(a0)
+; RV32ID-NEXT:    lw a4, 8(a0)
+; RV32ID-NEXT:    lw a5, 12(a0)
+; RV32ID-NEXT:    mv s0, a1
+; RV32ID-NEXT:    addi a0, sp, 8
+; RV32ID-NEXT:    sw a2, 8(sp)
+; RV32ID-NEXT:    sw a3, 12(sp)
+; RV32ID-NEXT:    sw a4, 16(sp)
+; RV32ID-NEXT:    sw a5, 20(sp)
+; RV32ID-NEXT:    call __trunctfhf2
+; RV32ID-NEXT:    fmv.x.w a0, fa0
+; RV32ID-NEXT:    sh a0, 0(s0)
+; RV32ID-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    addi sp, sp, 32
+; RV32ID-NEXT:    ret
+;
+; RV64ID-LABEL: fcvt_h_q:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    addi sp, sp, -16
+; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64ID-NEXT:    mv s0, a2
+; RV64ID-NEXT:    call __trunctfhf2
+; RV64ID-NEXT:    fmv.x.w a0, fa0
+; RV64ID-NEXT:    sh a0, 0(s0)
+; RV64ID-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64ID-NEXT:    addi sp, sp, 16
+; RV64ID-NEXT:    ret
+;
+; CHECK32-IZFHMIN-LABEL: fcvt_h_q:
+; CHECK32-IZFHMIN:       # %bb.0:
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZFHMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    lw a2, 0(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a3, 4(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a4, 8(a0)
+; CHECK32-IZFHMIN-NEXT:    lw a5, 12(a0)
+; CHECK32-IZFHMIN-NEXT:    mv s0, a1
+; CHECK32-IZFHMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZFHMIN-NEXT:    sw a2, 8(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a3, 12(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a4, 16(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a5, 20(sp)
+; CHECK32-IZFHMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZFHMIN-NEXT:    fsh fa0, 0(s0)
+; CHECK32-IZFHMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZFHMIN-NEXT:    ret
+;
+; CHECK64-IZFHMIN-LABEL: fcvt_h_q:
+; CHECK64-IZFHMIN:       # %bb.0:
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFHMIN-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; CHECK64-IZFHMIN-NEXT:    mv s0, a2
+; CHECK64-IZFHMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZFHMIN-NEXT:    fsh fa0, 0(s0)
+; CHECK64-IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFHMIN-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZFHMIN-NEXT:    ret
+;
+; CHECK32-IZHINXMIN-LABEL: fcvt_h_q:
+; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    lw a2, 0(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a3, 4(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a4, 8(a0)
+; CHECK32-IZHINXMIN-NEXT:    lw a5, 12(a0)
+; CHECK32-IZHINXMIN-NEXT:    mv s0, a1
+; CHECK32-IZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINXMIN-NEXT:    sw a2, 8(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a3, 12(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a4, 16(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a5, 20(sp)
+; CHECK32-IZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZHINXMIN-NEXT:    sh a0, 0(s0)
+; CHECK32-IZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZHINXMIN-LABEL: fcvt_h_q:
+; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINXMIN-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; CHECK64-IZHINXMIN-NEXT:    mv s0, a2
+; CHECK64-IZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZHINXMIN-NEXT:    sh a0, 0(s0)
+; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINXMIN-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINXMIN-NEXT:    ret
+;
+; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_q:
+; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 0(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 4(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a4, 8(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a5, 12(a0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    mv s0, a1
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 8(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 12(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a4, 16(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a5, 20(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK32-IZDINXZHINXMIN-NEXT:    sh a0, 0(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_q:
+; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZDINXZHINXMIN-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; CHECK64-IZDINXZHINXMIN-NEXT:    mv s0, a2
+; CHECK64-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
+; CHECK64-IZDINXZHINXMIN-NEXT:    sh a0, 0(s0)
+; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZDINXZHINXMIN-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+  %a = fptrunc fp128 %x to half
+  store half %a, ptr %y
+  ret void
+}
+
+; Test lib call lowering for half->fp128 to make sure it follows the ABI.
+define fp128 @fcvt_q_h(ptr %x) {
+; RV32IZFH-LABEL: fcvt_q_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -32
+; RV32IZFH-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    .cfi_offset ra, -4
+; RV32IZFH-NEXT:    .cfi_offset s0, -8
+; RV32IZFH-NEXT:    mv s0, a0
+; RV32IZFH-NEXT:    flh fa5, 0(a1)
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa5
+; RV32IZFH-NEXT:    addi a0, sp, 8
+; RV32IZFH-NEXT:    call __extendsftf2
+; RV32IZFH-NEXT:    lw a0, 8(sp)
+; RV32IZFH-NEXT:    lw a1, 12(sp)
+; RV32IZFH-NEXT:    lw a2, 16(sp)
+; RV32IZFH-NEXT:    lw a3, 20(sp)
+; RV32IZFH-NEXT:    sw a0, 0(s0)
+; RV32IZFH-NEXT:    sw a1, 4(s0)
+; RV32IZFH-NEXT:    sw a2, 8(s0)
+; RV32IZFH-NEXT:    sw a3, 12(s0)
+; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    .cfi_restore ra
+; RV32IZFH-NEXT:    .cfi_restore s0
+; RV32IZFH-NEXT:    addi sp, sp, 32
+; RV32IZFH-NEXT:    .cfi_def_cfa_offset 0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_q_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    .cfi_def_cfa_offset 16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    .cfi_offset ra, -8
+; RV64IZFH-NEXT:    flh fa5, 0(a0)
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa5
+; RV64IZFH-NEXT:    call __extendsftf2
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    .cfi_restore ra
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    .cfi_def_cfa_offset 0
+; RV64IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_q_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -32
+; RV32IDZFH-NEXT:    .cfi_def_cfa_offset 32
+; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    .cfi_offset ra, -4
+; RV32IDZFH-NEXT:    .cfi_offset s0, -8
+; RV32IDZFH-NEXT:    mv s0, a0
+; RV32IDZFH-NEXT:    flh fa5, 0(a1)
+; RV32IDZFH-NEXT:    fcvt.s.h fa0, fa5
+; RV32IDZFH-NEXT:    addi a0, sp, 8
+; RV32IDZFH-NEXT:    call __extendsftf2
+; RV32IDZFH-NEXT:    lw a0, 8(sp)
+; RV32IDZFH-NEXT:    lw a1, 12(sp)
+; RV32IDZFH-NEXT:    lw a2, 16(sp)
+; RV32IDZFH-NEXT:    lw a3, 20(sp)
+; RV32IDZFH-NEXT:    sw a0, 0(s0)
+; RV32IDZFH-NEXT:    sw a1, 4(s0)
+; RV32IDZFH-NEXT:    sw a2, 8(s0)
+; RV32IDZFH-NEXT:    sw a3, 12(s0)
+; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    .cfi_restore ra
+; RV32IDZFH-NEXT:    .cfi_restore s0
+; RV32IDZFH-NEXT:    addi sp, sp, 32
+; RV32IDZFH-NEXT:    .cfi_def_cfa_offset 0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_q_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    addi sp, sp, -16
+; RV64IDZFH-NEXT:    .cfi_def_cfa_offset 16
+; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IDZFH-NEXT:    .cfi_offset ra, -8
+; RV64IDZFH-NEXT:    flh fa5, 0(a0)
+; RV64IDZFH-NEXT:    fcvt.s.h fa0, fa5
+; RV64IDZFH-NEXT:    call __extendsftf2
+; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IDZFH-NEXT:    .cfi_restore ra
+; RV64IDZFH-NEXT:    addi sp, sp, 16
+; RV64IDZFH-NEXT:    .cfi_def_cfa_offset 0
+; RV64IDZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: fcvt_q_h:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -32
+; RV32IZHINX-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    .cfi_offset ra, -4
+; RV32IZHINX-NEXT:    .cfi_offset s0, -8
+; RV32IZHINX-NEXT:    mv s0, a0
+; RV32IZHINX-NEXT:    lh a0, 0(a1)
+; RV32IZHINX-NEXT:    fcvt.s.h a1, a0
+; RV32IZHINX-NEXT:    addi a0, sp, 8
+; RV32IZHINX-NEXT:    call __extendsftf2
+; RV32IZHINX-NEXT:    lw a0, 8(sp)
+; RV32IZHINX-NEXT:    lw a1, 12(sp)
+; RV32IZHINX-NEXT:    lw a2, 16(sp)
+; RV32IZHINX-NEXT:    lw a3, 20(sp)
+; RV32IZHINX-NEXT:    sw a0, 0(s0)
+; RV32IZHINX-NEXT:    sw a1, 4(s0)
+; RV32IZHINX-NEXT:    sw a2, 8(s0)
+; RV32IZHINX-NEXT:    sw a3, 12(s0)
+; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    .cfi_restore ra
+; RV32IZHINX-NEXT:    .cfi_restore s0
+; RV32IZHINX-NEXT:    addi sp, sp, 32
+; RV32IZHINX-NEXT:    .cfi_def_cfa_offset 0
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: fcvt_q_h:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    .cfi_def_cfa_offset 16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    .cfi_offset ra, -8
+; RV64IZHINX-NEXT:    lh a0, 0(a0)
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call __extendsftf2
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    .cfi_restore ra
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    .cfi_def_cfa_offset 0
+; RV64IZHINX-NEXT:    ret
+;
+; RV32IZDINXZHINX-LABEL: fcvt_q_h:
+; RV32IZDINXZHINX:       # %bb.0:
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
+; RV32IZDINXZHINX-NEXT:    .cfi_def_cfa_offset 32
+; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IZDINXZHINX-NEXT:    .cfi_offset ra, -4
+; RV32IZDINXZHINX-NEXT:    .cfi_offset s0, -8
+; RV32IZDINXZHINX-NEXT:    mv s0, a0
+; RV32IZDINXZHINX-NEXT:    lh a0, 0(a1)
+; RV32IZDINXZHINX-NEXT:    fcvt.s.h a1, a0
+; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
+; RV32IZDINXZHINX-NEXT:    call __extendsftf2
+; RV32IZDINXZHINX-NEXT:    lw a0, 8(sp)
+; RV32IZDINXZHINX-NEXT:    lw a1, 12(sp)
+; RV32IZDINXZHINX-NEXT:    lw a2, 16(sp)
+; RV32IZDINXZHINX-NEXT:    lw a3, 20(sp)
+; RV32IZDINXZHINX-NEXT:    sw a0, 0(s0)
+; RV32IZDINXZHINX-NEXT:    sw a1, 4(s0)
+; RV32IZDINXZHINX-NEXT:    sw a2, 8(s0)
+; RV32IZDINXZHINX-NEXT:    sw a3, 12(s0)
+; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IZDINXZHINX-NEXT:    .cfi_restore ra
+; RV32IZDINXZHINX-NEXT:    .cfi_restore s0
+; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
+; RV32IZDINXZHINX-NEXT:    .cfi_def_cfa_offset 0
+; RV32IZDINXZHINX-NEXT:    ret
+;
+; RV64IZDINXZHINX-LABEL: fcvt_q_h:
+; RV64IZDINXZHINX:       # %bb.0:
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
+; RV64IZDINXZHINX-NEXT:    .cfi_def_cfa_offset 16
+; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZDINXZHINX-NEXT:    .cfi_offset ra, -8
+; RV64IZDINXZHINX-NEXT:    lh a0, 0(a0)
+; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZDINXZHINX-NEXT:    call __extendsftf2
+; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZDINXZHINX-NEXT:    .cfi_restore ra
+; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
+; RV64IZDINXZHINX-NEXT:    .cfi_def_cfa_offset 0
+; RV64IZDINXZHINX-NEXT:    ret
+;
+; RV32I-LABEL: fcvt_q_h:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    .cfi_def_cfa_offset 32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lh a0, 0(a1)
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    addi a0, sp, 8
+; RV32I-NEXT:    call __extendsftf2
+; RV32I-NEXT:    lw a0, 8(sp)
+; RV32I-NEXT:    lw a1, 12(sp)
+; RV32I-NEXT:    lw a2, 16(sp)
+; RV32I-NEXT:    lw a3, 20(sp)
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    sw a2, 8(s0)
+; RV32I-NEXT:    sw a3, 12(s0)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fcvt_q_h:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    lh a0, 0(a0)
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call __extendsftf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32ID-ILP32-LABEL: fcvt_q_h:
+; RV32ID-ILP32:       # %bb.0:
+; RV32ID-ILP32-NEXT:    addi sp, sp, -32
+; RV32ID-ILP32-NEXT:    .cfi_def_cfa_offset 32
+; RV32ID-ILP32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    .cfi_offset ra, -4
+; RV32ID-ILP32-NEXT:    .cfi_offset s0, -8
+; RV32ID-ILP32-NEXT:    mv s0, a0
+; RV32ID-ILP32-NEXT:    lhu a0, 0(a1)
+; RV32ID-ILP32-NEXT:    call __extendhfsf2
+; RV32ID-ILP32-NEXT:    mv a1, a0
+; RV32ID-ILP32-NEXT:    addi a0, sp, 8
+; RV32ID-ILP32-NEXT:    call __extendsftf2
+; RV32ID-ILP32-NEXT:    lw a0, 8(sp)
+; RV32ID-ILP32-NEXT:    lw a1, 12(sp)
+; RV32ID-ILP32-NEXT:    lw a2, 16(sp)
+; RV32ID-ILP32-NEXT:    lw a3, 20(sp)
+; RV32ID-ILP32-NEXT:    sw a0, 0(s0)
+; RV32ID-ILP32-NEXT:    sw a1, 4(s0)
+; RV32ID-ILP32-NEXT:    sw a2, 8(s0)
+; RV32ID-ILP32-NEXT:    sw a3, 12(s0)
+; RV32ID-ILP32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    .cfi_restore ra
+; RV32ID-ILP32-NEXT:    .cfi_restore s0
+; RV32ID-ILP32-NEXT:    addi sp, sp, 32
+; RV32ID-ILP32-NEXT:    .cfi_def_cfa_offset 0
+; RV32ID-ILP32-NEXT:    ret
+;
+; RV64ID-LP64-LABEL: fcvt_q_h:
+; RV64ID-LP64:       # %bb.0:
+; RV64ID-LP64-NEXT:    addi sp, sp, -16
+; RV64ID-LP64-NEXT:    .cfi_def_cfa_offset 16
+; RV64ID-LP64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-LP64-NEXT:    .cfi_offset ra, -8
+; RV64ID-LP64-NEXT:    lhu a0, 0(a0)
+; RV64ID-LP64-NEXT:    call __extendhfsf2
+; RV64ID-LP64-NEXT:    call __extendsftf2
+; RV64ID-LP64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-LP64-NEXT:    .cfi_restore ra
+; RV64ID-LP64-NEXT:    addi sp, sp, 16
+; RV64ID-LP64-NEXT:    .cfi_def_cfa_offset 0
+; RV64ID-LP64-NEXT:    ret
+;
+; RV32ID-LABEL: fcvt_q_h:
+; RV32ID:       # %bb.0:
+; RV32ID-NEXT:    addi sp, sp, -32
+; RV32ID-NEXT:    .cfi_def_cfa_offset 32
+; RV32ID-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    .cfi_offset ra, -4
+; RV32ID-NEXT:    .cfi_offset s0, -8
+; RV32ID-NEXT:    mv s0, a0
+; RV32ID-NEXT:    lhu a0, 0(a1)
+; RV32ID-NEXT:    fmv.w.x fa0, a0
+; RV32ID-NEXT:    call __extendhfsf2
+; RV32ID-NEXT:    addi a0, sp, 8
+; RV32ID-NEXT:    call __extendsftf2
+; RV32ID-NEXT:    lw a0, 8(sp)
+; RV32ID-NEXT:    lw a1, 12(sp)
+; RV32ID-NEXT:    lw a2, 16(sp)
+; RV32ID-NEXT:    lw a3, 20(sp)
+; RV32ID-NEXT:    sw a0, 0(s0)
+; RV32ID-NEXT:    sw a1, 4(s0)
+; RV32ID-NEXT:    sw a2, 8(s0)
+; RV32ID-NEXT:    sw a3, 12(s0)
+; RV32ID-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    .cfi_restore ra
+; RV32ID-NEXT:    .cfi_restore s0
+; RV32ID-NEXT:    addi sp, sp, 32
+; RV32ID-NEXT:    .cfi_def_cfa_offset 0
+; RV32ID-NEXT:    ret
+;
+; RV64ID-LABEL: fcvt_q_h:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    addi sp, sp, -16
+; RV64ID-NEXT:    .cfi_def_cfa_offset 16
+; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-NEXT:    .cfi_offset ra, -8
+; RV64ID-NEXT:    lhu a0, 0(a0)
+; RV64ID-NEXT:    fmv.w.x fa0, a0
+; RV64ID-NEXT:    call __extendhfsf2
+; RV64ID-NEXT:    call __extendsftf2
+; RV64ID-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-NEXT:    .cfi_restore ra
+; RV64ID-NEXT:    addi sp, sp, 16
+; RV64ID-NEXT:    .cfi_def_cfa_offset 0
+; RV64ID-NEXT:    ret
+;
+; CHECK32-IZFHMIN-LABEL: fcvt_q_h:
+; CHECK32-IZFHMIN:       # %bb.0:
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZFHMIN-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32-IZFHMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZFHMIN-NEXT:    .cfi_offset ra, -4
+; CHECK32-IZFHMIN-NEXT:    .cfi_offset s0, -8
+; CHECK32-IZFHMIN-NEXT:    mv s0, a0
+; CHECK32-IZFHMIN-NEXT:    flh fa5, 0(a1)
+; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa0, fa5
+; CHECK32-IZFHMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZFHMIN-NEXT:    call __extendsftf2
+; CHECK32-IZFHMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZFHMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZFHMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZFHMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZFHMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZFHMIN-NEXT:    .cfi_restore ra
+; CHECK32-IZFHMIN-NEXT:    .cfi_restore s0
+; CHECK32-IZFHMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZFHMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK32-IZFHMIN-NEXT:    ret
+;
+; CHECK64-IZFHMIN-LABEL: fcvt_q_h:
+; CHECK64-IZFHMIN:       # %bb.0:
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZFHMIN-NEXT:    .cfi_def_cfa_offset 16
+; CHECK64-IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFHMIN-NEXT:    .cfi_offset ra, -8
+; CHECK64-IZFHMIN-NEXT:    flh fa5, 0(a0)
+; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa0, fa5
+; CHECK64-IZFHMIN-NEXT:    call __extendsftf2
+; CHECK64-IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFHMIN-NEXT:    .cfi_restore ra
+; CHECK64-IZFHMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZFHMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK64-IZFHMIN-NEXT:    ret
+;
+; CHECK32-IZHINXMIN-LABEL: fcvt_q_h:
+; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINXMIN-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32-IZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZHINXMIN-NEXT:    .cfi_offset ra, -4
+; CHECK32-IZHINXMIN-NEXT:    .cfi_offset s0, -8
+; CHECK32-IZHINXMIN-NEXT:    mv s0, a0
+; CHECK32-IZHINXMIN-NEXT:    lh a0, 0(a1)
+; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a1, a0
+; CHECK32-IZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINXMIN-NEXT:    call __extendsftf2
+; CHECK32-IZHINXMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZHINXMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZHINXMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZHINXMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZHINXMIN-NEXT:    .cfi_restore ra
+; CHECK32-IZHINXMIN-NEXT:    .cfi_restore s0
+; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINXMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK32-IZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZHINXMIN-LABEL: fcvt_q_h:
+; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINXMIN-NEXT:    .cfi_def_cfa_offset 16
+; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINXMIN-NEXT:    .cfi_offset ra, -8
+; CHECK64-IZHINXMIN-NEXT:    lh a0, 0(a0)
+; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECK64-IZHINXMIN-NEXT:    call __extendsftf2
+; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINXMIN-NEXT:    .cfi_restore ra
+; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINXMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK64-IZHINXMIN-NEXT:    ret
+;
+; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_q_h:
+; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_offset ra, -4
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_offset s0, -8
+; CHECK32-IZDINXZHINXMIN-NEXT:    mv s0, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    lh a0, 0(a1)
+; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a1, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
+; CHECK32-IZDINXZHINXMIN-NEXT:    call __extendsftf2
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a0, 8(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a1, 12(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 16(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 20(sp)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a0, 0(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a1, 4(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 8(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 12(s0)
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_restore ra
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_restore s0
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
+; CHECK32-IZDINXZHINXMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK32-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_q_h:
+; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
+; CHECK64-IZDINXZHINXMIN-NEXT:    .cfi_def_cfa_offset 16
+; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZDINXZHINXMIN-NEXT:    .cfi_offset ra, -8
+; CHECK64-IZDINXZHINXMIN-NEXT:    lh a0, 0(a0)
+; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECK64-IZDINXZHINXMIN-NEXT:    call __extendsftf2
+; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZDINXZHINXMIN-NEXT:    .cfi_restore ra
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
+; CHECK64-IZDINXZHINXMIN-NEXT:    .cfi_def_cfa_offset 0
+; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+  %a = load half, ptr %x
+  %b = fpext half %a to fp128
+  ret fp128 %b
+}
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 8f19424742775..7fcad77c7c17b 100644
--- a/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -1213,6 +1213,124 @@ define half @exp2_f16(half %a) nounwind {
   ret half %1
 }
 
+define half @exp10_f16(half %a) nounwind {
+; RV32IZFH-LABEL: exp10_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call exp10f
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: exp10_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call exp10f
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: exp10_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call exp10f
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: exp10_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call exp10f
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: exp10_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call exp10f
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: exp10_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call exp10f
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: exp10_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call exp10f
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: exp10_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call exp10f
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: exp10_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call exp10f
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: exp10_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call exp10f
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.exp10.f16(half %a)
+  ret half %1
+}
+
 declare half @llvm.log.f16(half)
 
 define half @log_f16(half %a) nounwind {
@@ -2121,16 +2239,16 @@ declare half @llvm.floor.f16(half)
 define half @floor_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: floor_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI17_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI17_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB17_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB18_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0, rdn
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0, rdn
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB17_2:
+; CHECKIZFH-NEXT:  .LBB18_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: floor_f16:
@@ -2139,12 +2257,12 @@ define half @floor_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB17_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB18_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0, rdn
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1, rdn
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB17_2:
+; CHECKIZHINX-NEXT:  .LBB18_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: floor_f16:
@@ -2180,12 +2298,12 @@ define half @floor_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB17_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB18_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rdn
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0, rdn
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB17_2:
+; CHECKIZFHMIN-NEXT:  .LBB18_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2195,12 +2313,12 @@ define half @floor_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB17_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB18_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0, rdn
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1, rdn
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB17_2:
+; CHECKIZHINXMIN-NEXT:  .LBB18_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.floor.f16(half %a)
@@ -2212,16 +2330,16 @@ declare half @llvm.ceil.f16(half)
 define half @ceil_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: ceil_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI18_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI19_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI19_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB18_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB19_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0, rup
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0, rup
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB18_2:
+; CHECKIZFH-NEXT:  .LBB19_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: ceil_f16:
@@ -2230,12 +2348,12 @@ define half @ceil_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB18_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB19_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0, rup
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1, rup
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB18_2:
+; CHECKIZHINX-NEXT:  .LBB19_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: ceil_f16:
@@ -2271,12 +2389,12 @@ define half @ceil_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB18_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB19_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rup
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0, rup
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB18_2:
+; CHECKIZFHMIN-NEXT:  .LBB19_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2286,12 +2404,12 @@ define half @ceil_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB18_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB19_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0, rup
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1, rup
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB18_2:
+; CHECKIZHINXMIN-NEXT:  .LBB19_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.ceil.f16(half %a)
@@ -2303,16 +2421,16 @@ declare half @llvm.trunc.f16(half)
 define half @trunc_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: trunc_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI19_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI19_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB19_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB20_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0, rtz
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0, rtz
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB19_2:
+; CHECKIZFH-NEXT:  .LBB20_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: trunc_f16:
@@ -2321,12 +2439,12 @@ define half @trunc_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB19_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB20_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0, rtz
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1, rtz
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB19_2:
+; CHECKIZHINX-NEXT:  .LBB20_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: trunc_f16:
@@ -2362,12 +2480,12 @@ define half @trunc_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB19_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB20_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rtz
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0, rtz
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB19_2:
+; CHECKIZFHMIN-NEXT:  .LBB20_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2377,12 +2495,12 @@ define half @trunc_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB19_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB20_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0, rtz
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1, rtz
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB19_2:
+; CHECKIZHINXMIN-NEXT:  .LBB20_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.trunc.f16(half %a)
@@ -2394,16 +2512,16 @@ declare half @llvm.rint.f16(half)
 define half @rint_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: rint_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI20_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI21_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI21_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB20_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB21_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB20_2:
+; CHECKIZFH-NEXT:  .LBB21_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: rint_f16:
@@ -2412,12 +2530,12 @@ define half @rint_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB20_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB21_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB20_2:
+; CHECKIZHINX-NEXT:  .LBB21_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: rint_f16:
@@ -2453,12 +2571,12 @@ define half @rint_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB20_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB21_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB20_2:
+; CHECKIZFHMIN-NEXT:  .LBB21_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2468,12 +2586,12 @@ define half @rint_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB20_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB21_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB20_2:
+; CHECKIZHINXMIN-NEXT:  .LBB21_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.rint.f16(half %a)
@@ -2605,16 +2723,16 @@ declare half @llvm.round.f16(half)
 define half @round_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: round_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI22_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI23_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI23_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB22_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB23_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0, rmm
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0, rmm
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB22_2:
+; CHECKIZFH-NEXT:  .LBB23_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: round_f16:
@@ -2623,12 +2741,12 @@ define half @round_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB22_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB23_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0, rmm
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1, rmm
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB22_2:
+; CHECKIZHINX-NEXT:  .LBB23_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: round_f16:
@@ -2664,12 +2782,12 @@ define half @round_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB22_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB23_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rmm
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0, rmm
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB22_2:
+; CHECKIZFHMIN-NEXT:  .LBB23_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2679,12 +2797,12 @@ define half @round_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB22_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB23_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0, rmm
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1, rmm
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB22_2:
+; CHECKIZHINXMIN-NEXT:  .LBB23_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.round.f16(half %a)
@@ -2696,16 +2814,16 @@ declare half @llvm.roundeven.f16(half)
 define half @roundeven_f16(half %a) nounwind {
 ; CHECKIZFH-LABEL: roundeven_f16:
 ; CHECKIZFH:       # %bb.0:
-; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI23_0)
-; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI23_0)(a0)
+; CHECKIZFH-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECKIZFH-NEXT:    flh fa5, %lo(.LCPI24_0)(a0)
 ; CHECKIZFH-NEXT:    fabs.h fa4, fa0
 ; CHECKIZFH-NEXT:    flt.h a0, fa4, fa5
-; CHECKIZFH-NEXT:    beqz a0, .LBB23_2
+; CHECKIZFH-NEXT:    beqz a0, .LBB24_2
 ; CHECKIZFH-NEXT:  # %bb.1:
 ; CHECKIZFH-NEXT:    fcvt.w.h a0, fa0, rne
 ; CHECKIZFH-NEXT:    fcvt.h.w fa5, a0, rne
 ; CHECKIZFH-NEXT:    fsgnj.h fa0, fa5, fa0
-; CHECKIZFH-NEXT:  .LBB23_2:
+; CHECKIZFH-NEXT:  .LBB24_2:
 ; CHECKIZFH-NEXT:    ret
 ;
 ; CHECKIZHINX-LABEL: roundeven_f16:
@@ -2714,12 +2832,12 @@ define half @roundeven_f16(half %a) nounwind {
 ; CHECKIZHINX-NEXT:    slli a1, a1, 10
 ; CHECKIZHINX-NEXT:    fabs.h a2, a0
 ; CHECKIZHINX-NEXT:    flt.h a1, a2, a1
-; CHECKIZHINX-NEXT:    beqz a1, .LBB23_2
+; CHECKIZHINX-NEXT:    beqz a1, .LBB24_2
 ; CHECKIZHINX-NEXT:  # %bb.1:
 ; CHECKIZHINX-NEXT:    fcvt.w.h a1, a0, rne
 ; CHECKIZHINX-NEXT:    fcvt.h.w a1, a1, rne
 ; CHECKIZHINX-NEXT:    fsgnj.h a0, a1, a0
-; CHECKIZHINX-NEXT:  .LBB23_2:
+; CHECKIZHINX-NEXT:  .LBB24_2:
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: roundeven_f16:
@@ -2755,12 +2873,12 @@ define half @roundeven_f16(half %a) nounwind {
 ; CHECKIZFHMIN-NEXT:    fmv.w.x fa4, a0
 ; CHECKIZFHMIN-NEXT:    fabs.s fa3, fa5
 ; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
-; CHECKIZFHMIN-NEXT:    beqz a0, .LBB23_2
+; CHECKIZFHMIN-NEXT:    beqz a0, .LBB24_2
 ; CHECKIZFHMIN-NEXT:  # %bb.1:
 ; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rne
 ; CHECKIZFHMIN-NEXT:    fcvt.s.w fa4, a0, rne
 ; CHECKIZFHMIN-NEXT:    fsgnj.s fa5, fa4, fa5
-; CHECKIZFHMIN-NEXT:  .LBB23_2:
+; CHECKIZFHMIN-NEXT:  .LBB24_2:
 ; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
 ; CHECKIZFHMIN-NEXT:    ret
 ;
@@ -2770,12 +2888,12 @@ define half @roundeven_f16(half %a) nounwind {
 ; CHECKIZHINXMIN-NEXT:    lui a1, 307200
 ; CHECKIZHINXMIN-NEXT:    fabs.s a2, a0
 ; CHECKIZHINXMIN-NEXT:    flt.s a1, a2, a1
-; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB23_2
+; CHECKIZHINXMIN-NEXT:    beqz a1, .LBB24_2
 ; CHECKIZHINXMIN-NEXT:  # %bb.1:
 ; CHECKIZHINXMIN-NEXT:    fcvt.w.s a1, a0, rne
 ; CHECKIZHINXMIN-NEXT:    fcvt.s.w a1, a1, rne
 ; CHECKIZHINXMIN-NEXT:    fsgnj.s a0, a1, a0
-; CHECKIZHINXMIN-NEXT:  .LBB23_2:
+; CHECKIZHINXMIN-NEXT:  .LBB24_2:
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.roundeven.f16(half %a)
@@ -3450,3 +3568,863 @@ define {half, i32} @frexp_half(half %x) nounwind {
   %a = call {half, i32} @llvm.frexp.f16.i32(half %x)
   ret {half, i32} %a
 }
+
+define half @asin_f16(half %a) nounwind {
+; RV32IZFH-LABEL: asin_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call asinf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: asin_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call asinf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: asin_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call asinf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: asin_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call asinf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: asin_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call asinf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: asin_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call asinf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: asin_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call asinf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: asin_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call asinf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: asin_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call asinf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: asin_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call asinf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.asin.f16(half %a)
+  ret half %1
+}
+
+define half @acos_f16(half %a) nounwind {
+; RV32IZFH-LABEL: acos_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call acosf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: acos_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call acosf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: acos_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call acosf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: acos_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call acosf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: acos_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call acosf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: acos_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call acosf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: acos_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call acosf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: acos_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call acosf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: acos_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call acosf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: acos_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call acosf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.acos.f16(half %a)
+  ret half %1
+}
+
+define half @atan_f16(half %a) nounwind {
+; RV32IZFH-LABEL: atan_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call atanf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: atan_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call atanf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: atan_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call atanf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: atan_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call atanf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: atan_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call atanf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call atanf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: atan_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call atanf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: atan_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call atanf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: atan_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call atanf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: atan_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call atanf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.atan.f16(half %a)
+  ret half %1
+}
+
+define half @atan2_f16(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: atan2_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    fcvt.s.h fa1, fa1
+; RV32IZFH-NEXT:    call atan2f
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: atan2_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    fcvt.s.h fa1, fa1
+; RV64IZFH-NEXT:    call atan2f
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: atan2_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    fcvt.s.h a1, a1
+; RV32IZHINX-NEXT:    call atan2f
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: atan2_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    fcvt.s.h a1, a1
+; RV64IZHINX-NEXT:    call atan2f
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: atan2_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi s2, a1, -1
+; RV32I-NEXT:    and a0, a0, s2
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    and a0, s0, s2
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    call atan2f
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: atan2_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    and a0, a0, s2
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    and a0, s0, s2
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call atan2f
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: atan2_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa1, fa1
+; RV32IZFHMIN-NEXT:    call atan2f
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: atan2_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa1, fa1
+; RV64IZFHMIN-NEXT:    call atan2f
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: atan2_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a1, a1
+; RV32IZHINXMIN-NEXT:    call atan2f
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: atan2_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a1, a1
+; RV64IZHINXMIN-NEXT:    call atan2f
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.atan2.f16(half %a, half %b)
+  ret half %1
+}
+
+define half @sinh_f16(half %a) nounwind {
+; RV32IZFH-LABEL: sinh_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call sinhf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: sinh_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call sinhf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: sinh_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call sinhf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: sinh_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call sinhf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: sinh_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call sinhf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: sinh_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call sinhf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: sinh_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call sinhf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: sinh_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call sinhf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: sinh_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call sinhf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: sinh_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call sinhf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.sinh.f16(half %a)
+  ret half %1
+}
+
+define half @cosh_f16(half %a) nounwind {
+; RV32IZFH-LABEL: cosh_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call coshf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: cosh_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call coshf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: cosh_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call coshf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: cosh_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call coshf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: cosh_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call coshf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: cosh_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call coshf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: cosh_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call coshf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: cosh_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call coshf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: cosh_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call coshf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: cosh_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call coshf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.cosh.f16(half %a)
+  ret half %1
+}
+
+define half @tanh_f16(half %a) nounwind {
+; RV32IZFH-LABEL: tanh_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call tanhf
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: tanh_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call tanhf
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV32IZHINX-LABEL: tanh_f16:
+; RV32IZHINX:       # %bb.0:
+; RV32IZHINX-NEXT:    addi sp, sp, -16
+; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINX-NEXT:    call tanhf
+; RV32IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINX-NEXT:    addi sp, sp, 16
+; RV32IZHINX-NEXT:    ret
+;
+; RV64IZHINX-LABEL: tanh_f16:
+; RV64IZHINX:       # %bb.0:
+; RV64IZHINX-NEXT:    addi sp, sp, -16
+; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    call tanhf
+; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINX-NEXT:    addi sp, sp, 16
+; RV64IZHINX-NEXT:    ret
+;
+; RV32I-LABEL: tanh_f16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    call __extendhfsf2
+; RV32I-NEXT:    call tanhf
+; RV32I-NEXT:    call __truncsfhf2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: tanh_f16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    call __extendhfsf2
+; RV64I-NEXT:    call tanhf
+; RV64I-NEXT:    call __truncsfhf2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: tanh_f16:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    addi sp, sp, -16
+; RV32IZFHMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFHMIN-NEXT:    call tanhf
+; RV32IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFHMIN-NEXT:    addi sp, sp, 16
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: tanh_f16:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    addi sp, sp, -16
+; RV64IZFHMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFHMIN-NEXT:    call tanhf
+; RV64IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFHMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFHMIN-NEXT:    addi sp, sp, 16
+; RV64IZFHMIN-NEXT:    ret
+;
+; RV32IZHINXMIN-LABEL: tanh_f16:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call tanhf
+; RV32IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: tanh_f16:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call tanhf
+; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+  %1 = call half @llvm.tanh.f16(half %a)
+  ret half %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-zfa.ll b/llvm/test/CodeGen/RISCV/half-zfa.ll
index 54ec90b8eab3e..960c7c4a73e4f 100644
--- a/llvm/test/CodeGen/RISCV/half-zfa.ll
+++ b/llvm/test/CodeGen/RISCV/half-zfa.ll
@@ -140,6 +140,24 @@ define half @fround_h_5(half %a) nounwind {
 
 declare half @llvm.nearbyint.f16(half) nounwind readnone
 
+define half @fround_h_6(half %a) nounwind {
+; CHECK-LABEL: fround_h_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fround.h fa0, fa0, rne
+; CHECK-NEXT:    ret
+;
+; ZFHMIN-LABEL: fround_h_6:
+; ZFHMIN:       # %bb.0:
+; ZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; ZFHMIN-NEXT:    fround.s fa5, fa5, rne
+; ZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; ZFHMIN-NEXT:    ret
+  %call = tail call half @llvm.roundeven.f16(half %a) nounwind readnone
+  ret half %call
+}
+
+declare half @llvm.roundeven.f16(half) nounwind readnone
+
 
 define half @froundnx_h(half %a) nounwind {
 ; CHECK-LABEL: froundnx_h:
@@ -321,10 +339,10 @@ define half @select_loadfpimm(half %x) nounwind {
 ; CHECK-NEXT:    fmv.h.x fa5, zero
 ; CHECK-NEXT:    fle.h a0, fa5, fa0
 ; CHECK-NEXT:    fli.h fa0, 0.5
-; CHECK-NEXT:    bnez a0, .LBB16_2
+; CHECK-NEXT:    bnez a0, .LBB17_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    fneg.h fa0, fa0
-; CHECK-NEXT:  .LBB16_2: # %entry
+; CHECK-NEXT:  .LBB17_2: # %entry
 ; CHECK-NEXT:    ret
 ;
 ; ZFHMIN-LABEL: select_loadfpimm:
@@ -334,8 +352,8 @@ define half @select_loadfpimm(half %x) nounwind {
 ; ZFHMIN-NEXT:    fle.s a0, fa4, fa5
 ; ZFHMIN-NEXT:    xori a0, a0, 1
 ; ZFHMIN-NEXT:    slli a0, a0, 1
-; ZFHMIN-NEXT:    lui a1, %hi(.LCPI16_0)
-; ZFHMIN-NEXT:    addi a1, a1, %lo(.LCPI16_0)
+; ZFHMIN-NEXT:    lui a1, %hi(.LCPI17_0)
+; ZFHMIN-NEXT:    addi a1, a1, %lo(.LCPI17_0)
 ; ZFHMIN-NEXT:    add a0, a1, a0
 ; ZFHMIN-NEXT:    flh fa0, 0(a0)
 ; ZFHMIN-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll b/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll
index c04e4fea7b2c2..64957ed6e4ba6 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll
@@ -45,6 +45,7 @@ define <vscale x 1 x i8> @constraint_vd(<vscale x 1 x i8> %0, <vscale x 1 x i8>
 define <vscale x 1 x i1> @constraint_vm(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1) nounwind {
 ; RV32I-LABEL: constraint_vm:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32I-NEXT:    vmv1r.v v9, v0
 ; RV32I-NEXT:    vmv1r.v v0, v8
 ; RV32I-NEXT:    #APP
@@ -54,6 +55,7 @@ define <vscale x 1 x i1> @constraint_vm(<vscale x 1 x i1> %0, <vscale x 1 x i1>
 ;
 ; RV64I-LABEL: constraint_vm:
 ; RV64I:       # %bb.0:
+; RV64I-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64I-NEXT:    vmv1r.v v9, v0
 ; RV64I-NEXT:    vmv1r.v v0, v8
 ; RV64I-NEXT:    #APP
diff --git a/llvm/test/CodeGen/RISCV/llvm.exp10.ll b/llvm/test/CodeGen/RISCV/llvm.exp10.ll
index 15a123400fd4f..7b199504837e8 100644
--- a/llvm/test/CodeGen/RISCV/llvm.exp10.ll
+++ b/llvm/test/CodeGen/RISCV/llvm.exp10.ll
@@ -6,64 +6,19 @@
 ; RUN:   -verify-machineinstrs -target-abi=lp64d < %s \
 ; RUN:   | FileCheck -check-prefixes=CHECK,RV64IFD %s
 
-declare half @llvm.exp10.f16(half)
 declare <1 x half> @llvm.exp10.v1f16(<1 x half>)
 declare <2 x half> @llvm.exp10.v2f16(<2 x half>)
 declare <3 x half> @llvm.exp10.v3f16(<3 x half>)
 declare <4 x half> @llvm.exp10.v4f16(<4 x half>)
-declare float @llvm.exp10.f32(float)
 declare <1 x float> @llvm.exp10.v1f32(<1 x float>)
 declare <2 x float> @llvm.exp10.v2f32(<2 x float>)
 declare <3 x float> @llvm.exp10.v3f32(<3 x float>)
 declare <4 x float> @llvm.exp10.v4f32(<4 x float>)
-declare double @llvm.exp10.f64(double)
 declare <1 x double> @llvm.exp10.v1f64(<1 x double>)
 declare <2 x double> @llvm.exp10.v2f64(<2 x double>)
 declare <3 x double> @llvm.exp10.v3f64(<3 x double>)
 declare <4 x double> @llvm.exp10.v4f64(<4 x double>)
 
-define half @exp10_f16(half %x) {
-; RV32IFD-LABEL: exp10_f16:
-; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    .cfi_def_cfa_offset 16
-; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    .cfi_offset ra, -4
-; RV32IFD-NEXT:    call __extendhfsf2
-; RV32IFD-NEXT:    call exp10f
-; RV32IFD-NEXT:    call __truncsfhf2
-; RV32IFD-NEXT:    fmv.x.w a0, fa0
-; RV32IFD-NEXT:    lui a1, 1048560
-; RV32IFD-NEXT:    or a0, a0, a1
-; RV32IFD-NEXT:    fmv.w.x fa0, a0
-; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    .cfi_restore ra
-; RV32IFD-NEXT:    addi sp, sp, 16
-; RV32IFD-NEXT:    .cfi_def_cfa_offset 0
-; RV32IFD-NEXT:    ret
-;
-; RV64IFD-LABEL: exp10_f16:
-; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    .cfi_def_cfa_offset 16
-; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFD-NEXT:    .cfi_offset ra, -8
-; RV64IFD-NEXT:    call __extendhfsf2
-; RV64IFD-NEXT:    call exp10f
-; RV64IFD-NEXT:    call __truncsfhf2
-; RV64IFD-NEXT:    fmv.x.w a0, fa0
-; RV64IFD-NEXT:    lui a1, 1048560
-; RV64IFD-NEXT:    or a0, a0, a1
-; RV64IFD-NEXT:    fmv.w.x fa0, a0
-; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IFD-NEXT:    .cfi_restore ra
-; RV64IFD-NEXT:    addi sp, sp, 16
-; RV64IFD-NEXT:    .cfi_def_cfa_offset 0
-; RV64IFD-NEXT:    ret
-  %r = call half @llvm.exp10.f16(half %x)
-  ret half %r
-}
-
 define <1 x half> @exp10_v1f16(<1 x half> %x) {
 ; RV32IFD-LABEL: exp10_v1f16:
 ; RV32IFD:       # %bb.0:
@@ -443,14 +398,6 @@ define <4 x half> @exp10_v4f16(<4 x half> %x) {
   ret <4 x half> %r
 }
 
-define float @exp10_f32(float %x) {
-; CHECK-LABEL: exp10_f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    tail exp10f
-  %r = call float @llvm.exp10.f32(float %x)
-  ret float %r
-}
-
 define <1 x float> @exp10_v1f32(<1 x float> %x) {
 ; RV32IFD-LABEL: exp10_v1f32:
 ; RV32IFD:       # %bb.0:
@@ -733,14 +680,6 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
   ret <4 x float> %r
 }
 
-define double @exp10_f64(double %x) {
-; CHECK-LABEL: exp10_f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    tail exp10
-  %r = call double @llvm.exp10.f64(double %x)
-  ret double %r
-}
-
 ; FIXME: Broken
 ; define <1 x double> @exp10_v1f64(<1 x double> %x) {
 ;   %r = call <1 x double> @llvm.exp10.v1f64(<1 x double> %x)
@@ -994,3 +933,5 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
   %r = call <4 x double> @llvm.exp10.v4f64(<4 x double> %x)
   ret <4 x double> %r
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
index c2882fd46c17d..3f758e25c4217 100644
--- a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
+++ b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
@@ -4,7 +4,14 @@
 ;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion -mcpu=sifive-u74 \
 ;RUN:   -target-abi=lp64f | FileCheck %s --check-prefix=FUSION
 ;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion,+use-postra-scheduler -mcpu=sifive-u74 \
-;RUN:   -target-abi=lp64f | FileCheck %s --check-prefixes=FUSION-POSTRA
+;RUN:   -misched-postra-direction=topdown -target-abi=lp64f \
+;RUN:   | FileCheck %s --check-prefixes=FUSION-POSTRA,FUSION-POSTRA-TOPDOWN
+;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion,+use-postra-scheduler -mcpu=sifive-u74 \
+;RUN:   -misched-postra-direction=bottomup -target-abi=lp64f \
+;RUN:   | FileCheck %s --check-prefixes=FUSION-POSTRA,FUSION-POSTRA-BOTTOMUP
+;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion,+use-postra-scheduler -mcpu=sifive-u74 \
+;RUN:   -misched-postra-direction=bidirectional -target-abi=lp64f \
+;RUN:   | FileCheck %s --check-prefixes=FUSION-POSTRA,FUSION-POSTRA-BIDIRECTIONAL
 
 @.str = private constant [4 x i8] c"%f\0A\00", align 1
 
@@ -23,12 +30,26 @@ define void @foo(i32 signext %0, i32 signext %1) {
 ; FUSION-NEXT:    fcvt.s.w fa0, a1
 ; FUSION-NEXT:    tail bar
 ;
-; FUSION-POSTRA-LABEL: foo:
-; FUSION-POSTRA:       # %bb.0:
-; FUSION-POSTRA-NEXT:    lui a0, %hi(.L.str)
-; FUSION-POSTRA-NEXT:    addi a0, a0, %lo(.L.str)
-; FUSION-POSTRA-NEXT:    fcvt.s.w fa0, a1
-; FUSION-POSTRA-NEXT:    tail bar
+; FUSION-POSTRA-TOPDOWN-LABEL: foo:
+; FUSION-POSTRA-TOPDOWN:       # %bb.0:
+; FUSION-POSTRA-TOPDOWN-NEXT:    lui a0, %hi(.L.str)
+; FUSION-POSTRA-TOPDOWN-NEXT:    addi a0, a0, %lo(.L.str)
+; FUSION-POSTRA-TOPDOWN-NEXT:    fcvt.s.w fa0, a1
+; FUSION-POSTRA-TOPDOWN-NEXT:    tail bar
+;
+; FUSION-POSTRA-BOTTOMUP-LABEL: foo:
+; FUSION-POSTRA-BOTTOMUP:       # %bb.0:
+; FUSION-POSTRA-BOTTOMUP-NEXT:    fcvt.s.w fa0, a1
+; FUSION-POSTRA-BOTTOMUP-NEXT:    lui a0, %hi(.L.str)
+; FUSION-POSTRA-BOTTOMUP-NEXT:    addi a0, a0, %lo(.L.str)
+; FUSION-POSTRA-BOTTOMUP-NEXT:    tail bar
+;
+; FUSION-POSTRA-BIDIRECTIONAL-LABEL: foo:
+; FUSION-POSTRA-BIDIRECTIONAL:       # %bb.0:
+; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    lui a0, %hi(.L.str)
+; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    addi a0, a0, %lo(.L.str)
+; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    fcvt.s.w fa0, a1
+; FUSION-POSTRA-BIDIRECTIONAL-NEXT:    tail bar
   %3 = sitofp i32 %1 to float
   tail call void @bar(ptr @.str, float %3)
   ret void
diff --git a/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll b/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
deleted file mode 100644
index c1e7b682200eb..0000000000000
--- a/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
+++ /dev/null
@@ -1,124 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefix=RV64 %s
-
-define void @test(ptr nocapture noundef writeonly %array1, i32 noundef signext %a, i32 noundef signext %b) {
-; RV64-LABEL: test:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addiw a3, a1, 5
-; RV64-NEXT:    slli a1, a1, 2
-; RV64-NEXT:    slli a4, a3, 2
-; RV64-NEXT:    add a1, a1, a0
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    sw a2, 0(a0)
-; RV64-NEXT:    sw a2, 24(a1)
-; RV64-NEXT:    sw a3, 140(a1)
-; RV64-NEXT:    ret
-entry:
-  %add = add nsw i32 %a, 5
-  %idxprom = sext i32 %add to i64
-  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
-  store i32 %b, ptr %arrayidx, align 4
-  %add3 = add nsw i32 %a, 6
-  %idxprom4 = sext i32 %add3 to i64
-  %arrayidx5 = getelementptr inbounds i32, ptr %array1, i64 %idxprom4
-  store i32 %b, ptr %arrayidx5, align 4
-  %add6 = add nsw i32 %a, 35
-  %idxprom7 = sext i32 %add6 to i64
-  %arrayidx8 = getelementptr inbounds i32, ptr %array1, i64 %idxprom7
-  store i32 %add, ptr %arrayidx8, align 4
-  ret void
-}
-
-; test of jumpping, find add's operand has one more use can simplified
-define void @test1(ptr nocapture noundef %array1, i32 noundef signext %a, i32 noundef signext %b, i32 noundef signext %x) {
-; RV64-LABEL: test1:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addiw a4, a1, 5
-; RV64-NEXT:    slli a5, a4, 2
-; RV64-NEXT:    add a5, a0, a5
-; RV64-NEXT:    mv a6, a4
-; RV64-NEXT:    bgtz a3, .LBB1_2
-; RV64-NEXT:  # %bb.1: # %entry
-; RV64-NEXT:    mv a6, a2
-; RV64-NEXT:  .LBB1_2: # %entry
-; RV64-NEXT:    slli a1, a1, 2
-; RV64-NEXT:    add a0, a1, a0
-; RV64-NEXT:    sw a6, 0(a5)
-; RV64-NEXT:    sw a6, 24(a0)
-; RV64-NEXT:    sw a4, 140(a0)
-; RV64-NEXT:    ret
-entry:
-  %add = add nsw i32 %a, 5
-  %cmp = icmp sgt i32 %x, 0
-  %idxprom = sext i32 %add to i64
-  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
-  %add.b = select i1 %cmp, i32 %add, i32 %b
-  store i32 %add.b, ptr %arrayidx, align 4
-  %add5 = add nsw i32 %a, 6
-  %idxprom6 = sext i32 %add5 to i64
-  %arrayidx7 = getelementptr inbounds i32, ptr %array1, i64 %idxprom6
-  store i32 %add.b, ptr %arrayidx7, align 4
-  %add8 = add nsw i32 %a, 35
-  %idxprom9 = sext i32 %add8 to i64
-  %arrayidx10 = getelementptr inbounds i32, ptr %array1, i64 %idxprom9
-  store i32 %add, ptr %arrayidx10, align 4
-  ret void
-}
-
-define void @test2(ptr nocapture noundef writeonly %array1, i64 noundef %a, i64 noundef %b) local_unnamed_addr #0 {
-; RV64-LABEL: test2:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addi a3, a1, 5
-; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    slli a4, a3, 3
-; RV64-NEXT:    add a1, a1, a0
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    sd a2, 0(a0)
-; RV64-NEXT:    sd a2, 48(a1)
-; RV64-NEXT:    sd a3, 280(a1)
-; RV64-NEXT:    ret
-entry:
-  %add = add nsw i64 %a, 5
-  %arrayidx = getelementptr inbounds i64, ptr %array1, i64 %add
-  store i64 %b, ptr %arrayidx, align 8
-  %add2 = add nsw i64 %a, 6
-  %arrayidx3 = getelementptr inbounds i64, ptr %array1, i64 %add2
-  store i64 %b, ptr %arrayidx3, align 8
-  %add4 = add nsw i64 %a, 35
-  %arrayidx5 = getelementptr inbounds i64, ptr %array1, i64 %add4
-  store i64 %add, ptr %arrayidx5, align 8
-  ret void
-}
-
-define void @test3(ptr nocapture noundef %array1, i64 noundef %a, i64 noundef %b, i64 noundef %x) {
-; RV64-LABEL: test3:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addi a4, a1, 5
-; RV64-NEXT:    mv a5, a4
-; RV64-NEXT:    bgtz a3, .LBB3_2
-; RV64-NEXT:  # %bb.1: # %entry
-; RV64-NEXT:    mv a5, a2
-; RV64-NEXT:  .LBB3_2: # %entry
-; RV64-NEXT:    slli a2, a4, 3
-; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    add a2, a0, a2
-; RV64-NEXT:    add a0, a1, a0
-; RV64-NEXT:    sd a5, 0(a2)
-; RV64-NEXT:    sd a5, 48(a0)
-; RV64-NEXT:    sd a4, 280(a0)
-; RV64-NEXT:    ret
-entry:
-  %add = add nsw i64 %a, 5
-  %cmp = icmp sgt i64 %x, 0
-  %spec.select = select i1 %cmp, i64 %add, i64 %b
-  %0 = getelementptr inbounds i64, ptr %array1, i64 %add
-  store i64 %spec.select, ptr %0, align 8
-  %add3 = add nsw i64 %a, 6
-  %arrayidx4 = getelementptr inbounds i64, ptr %array1, i64 %add3
-  store i64 %spec.select, ptr %arrayidx4, align 8
-  %add5 = add nsw i64 %a, 35
-  %arrayidx6 = getelementptr inbounds i64, ptr %array1, i64 %add5
-  store i64 %add, ptr %arrayidx6, align 8
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rv64-float-convert-strict.ll b/llvm/test/CodeGen/RISCV/rv64-float-convert-strict.ll
index 86d1801928ea1..7065a7120175d 100644
--- a/llvm/test/CodeGen/RISCV/rv64-float-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-float-convert-strict.ll
@@ -7,65 +7,27 @@
 ; RUN:   -disable-strictnode-mutation | FileCheck %s -check-prefixes=CHECK,RV64IFINX
 
 define i128 @fptosi_f32_to_i128(float %a) nounwind strictfp {
-; RV64I-LABEL: fptosi_f32_to_i128:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    call __fixsfti
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-;
-; RV64IF-LABEL: fptosi_f32_to_i128:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call __fixsfti
-; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IF-NEXT:    addi sp, sp, 16
-; RV64IF-NEXT:    ret
-;
-; RV64IFINX-LABEL: fptosi_f32_to_i128:
-; RV64IFINX:       # %bb.0:
-; RV64IFINX-NEXT:    addi sp, sp, -16
-; RV64IFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFINX-NEXT:    call __fixsfti
-; RV64IFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IFINX-NEXT:    addi sp, sp, 16
-; RV64IFINX-NEXT:    ret
+; CHECK-LABEL: fptosi_f32_to_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixsfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %1 = call i128 @llvm.experimental.constrained.fptosi.i128.f32(float %a, metadata !"fpexcept.strict")
   ret i128 %1
 }
 
 define i128 @fptoui_f32_to_i128(float %a) nounwind strictfp {
-; RV64I-LABEL: fptoui_f32_to_i128:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    call __fixunssfti
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-;
-; RV64IF-LABEL: fptoui_f32_to_i128:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call __fixunssfti
-; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IF-NEXT:    addi sp, sp, 16
-; RV64IF-NEXT:    ret
-;
-; RV64IFINX-LABEL: fptoui_f32_to_i128:
-; RV64IFINX:       # %bb.0:
-; RV64IFINX-NEXT:    addi sp, sp, -16
-; RV64IFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IFINX-NEXT:    call __fixunssfti
-; RV64IFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IFINX-NEXT:    addi sp, sp, 16
-; RV64IFINX-NEXT:    ret
+; CHECK-LABEL: fptoui_f32_to_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunssfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %1 = call i128 @llvm.experimental.constrained.fptoui.i128.f32(float %a, metadata !"fpexcept.strict")
   ret i128 %1
 }
@@ -95,3 +57,7 @@ define float @uitofp_i128_to_f32(i128 %a) nounwind strictfp {
   %1 = call float @llvm.experimental.constrained.uitofp.f32.i128(i128 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret float %1
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV64I: {{.*}}
+; RV64IF: {{.*}}
+; RV64IFINX: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
index 0cdd92fbaf916..e387586cd46c2 100644
--- a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
@@ -7,65 +7,27 @@
 ; RUN:   | FileCheck %s -check-prefixes=CHECK,RV64IZFINX
 
 define i128 @fptosi_f32_to_i128(float %a) nounwind {
-; RV64I-LABEL: fptosi_f32_to_i128:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    call __fixsfti
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-;
-; RV64IF-LABEL: fptosi_f32_to_i128:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call __fixsfti
-; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IF-NEXT:    addi sp, sp, 16
-; RV64IF-NEXT:    ret
-;
-; RV64IZFINX-LABEL: fptosi_f32_to_i128:
-; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call __fixsfti
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; CHECK-LABEL: fptosi_f32_to_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixsfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %1 = fptosi float %a to i128
   ret i128 %1
 }
 
 define i128 @fptoui_f32_to_i128(float %a) nounwind {
-; RV64I-LABEL: fptoui_f32_to_i128:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    call __fixunssfti
-; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 16
-; RV64I-NEXT:    ret
-;
-; RV64IF-LABEL: fptoui_f32_to_i128:
-; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IF-NEXT:    call __fixunssfti
-; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IF-NEXT:    addi sp, sp, 16
-; RV64IF-NEXT:    ret
-;
-; RV64IZFINX-LABEL: fptoui_f32_to_i128:
-; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call __fixunssfti
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; CHECK-LABEL: fptoui_f32_to_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call __fixunssfti
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %1 = fptoui float %a to i128
   ret i128 %1
 }
@@ -107,38 +69,38 @@ define i128 @fptosi_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a1, 1044480
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    sext.w a0, s0
-; RV64I-NEXT:    call __fixsfti
 ; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    call __fixsfti
+; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    mv s3, a1
 ; RV64I-NEXT:    li s5, -1
-; RV64I-NEXT:    bgez s1, .LBB4_2
+; RV64I-NEXT:    bgez s2, .LBB4_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    slli s3, s5, 63
 ; RV64I-NEXT:  .LBB4_2:
 ; RV64I-NEXT:    lui a1, 520192
 ; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    blez a0, .LBB4_4
 ; RV64I-NEXT:  # %bb.3:
 ; RV64I-NEXT:    srli s3, s5, 1
 ; RV64I-NEXT:  .LBB4_4:
-; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s1
 ; RV64I-NEXT:    call __unordsf2
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    slti a1, s1, 0
+; RV64I-NEXT:    slti a1, s2, 0
 ; RV64I-NEXT:    sgtz a2, s4
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    addi a3, a1, -1
 ; RV64I-NEXT:    and a1, a0, s3
-; RV64I-NEXT:    and a3, a3, s2
+; RV64I-NEXT:    and a3, a3, s0
 ; RV64I-NEXT:    neg a2, a2
 ; RV64I-NEXT:    or a2, a2, a3
 ; RV64I-NEXT:    and a0, a0, a2
@@ -249,7 +211,7 @@ define i128 @fptoui_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    call __gesf2
 ; RV64I-NEXT:    slti a0, a0, 0
 ; RV64I-NEXT:    addi s2, a0, -1
-; RV64I-NEXT:    sext.w a0, s0
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfti
 ; RV64I-NEXT:    and a0, s2, a0
 ; RV64I-NEXT:    and a1, s2, a1
diff --git a/llvm/test/CodeGen/RISCV/rv64-half-convert-strict.ll b/llvm/test/CodeGen/RISCV/rv64-half-convert-strict.ll
index e9554bce1d501..67aa4ca4c9409 100644
--- a/llvm/test/CodeGen/RISCV/rv64-half-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-half-convert-strict.ll
@@ -15,7 +15,6 @@ define i128 @fptosi_f16_to_i128(half %a) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __extendhfsf2
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    call __fixsfti
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -48,7 +47,6 @@ define i128 @fptoui_f16_to_i128(half %a) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __extendhfsf2
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    call __fixunssfti
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
index a717c6c71f2ec..ea582ac258b71 100644
--- a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
@@ -76,7 +76,6 @@ define i128 @fptosi_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __extendhfsf2
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    call __fixsfti
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -109,7 +108,6 @@ define i128 @fptoui_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __extendhfsf2
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    call __fixunssfti
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -148,13 +146,13 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __extendhfsf2
-; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    lui a1, 1044480
 ; RV64I-NEXT:    call __gesf2
 ; RV64I-NEXT:    mv s0, a0
-; RV64I-NEXT:    sext.w a0, s1
+; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __fixsfti
-; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    mv s3, a1
 ; RV64I-NEXT:    li s5, -1
 ; RV64I-NEXT:    bgez s0, .LBB4_2
@@ -163,15 +161,15 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:  .LBB4_2:
 ; RV64I-NEXT:    lui a1, 520192
 ; RV64I-NEXT:    addiw a1, a1, -1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    blez a0, .LBB4_4
 ; RV64I-NEXT:  # %bb.3:
 ; RV64I-NEXT:    srli s3, s5, 1
 ; RV64I-NEXT:  .LBB4_4:
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __unordsf2
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    sgtz a1, s4
@@ -180,7 +178,7 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    neg a3, a1
 ; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a0, s3
-; RV64I-NEXT:    and a2, a2, s2
+; RV64I-NEXT:    and a2, a2, s1
 ; RV64I-NEXT:    or a2, a3, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
@@ -292,7 +290,7 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    call __gesf2
 ; RV64I-NEXT:    slti a0, a0, 0
 ; RV64I-NEXT:    addi s2, a0, -1
-; RV64I-NEXT:    sext.w a0, s0
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfti
 ; RV64I-NEXT:    and a0, s2, a0
 ; RV64I-NEXT:    and a1, s2, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/abs-vp.ll b/llvm/test/CodeGen/RISCV/rvv/abs-vp.ll
index 163d9145bc362..ee0016ec080e2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/abs-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/abs-vp.ll
@@ -567,6 +567,7 @@ define <vscale x 16 x i64> @vp_abs_nxv16i64(<vscale x 16 x i64> %va, <vscale x 1
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -576,7 +577,6 @@ define <vscale x 16 x i64> @vp_abs_nxv16i64(<vscale x 16 x i64> %va, <vscale x 1
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll b/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
index 66a1178cddb66..10d24927d9b78 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
@@ -3075,6 +3075,7 @@ define <vscale x 64 x i16> @vp_bitreverse_nxv64i16(<vscale x 64 x i16> %va, <vsc
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -3086,7 +3087,6 @@ define <vscale x 64 x i16> @vp_bitreverse_nxv64i16(<vscale x 64 x i16> %va, <vsc
 ; CHECK-NEXT:    lui a2, 3
 ; CHECK-NEXT:    srli a4, a3, 1
 ; CHECK-NEXT:    slli a3, a3, 2
-; CHECK-NEXT:    vsetvli a5, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a4
 ; CHECK-NEXT:    sub a4, a0, a3
 ; CHECK-NEXT:    sltu a5, a0, a4
@@ -3158,11 +3158,11 @@ define <vscale x 64 x i16> @vp_bitreverse_nxv64i16(<vscale x 64 x i16> %va, <vsc
 ;
 ; CHECK-ZVBB-LABEL: vp_bitreverse_nxv64i16:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 1
 ; CHECK-ZVBB-NEXT:    slli a1, a1, 2
-; CHECK-ZVBB-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sub a2, a0, a1
 ; CHECK-ZVBB-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll b/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
index 1c95ec8fafd4f..0dc1d0c32ac44 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
@@ -1584,6 +1584,7 @@ define <vscale x 64 x i16> @vp_bswap_nxv64i16(<vscale x 64 x i16> %va, <vscale x
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -1593,7 +1594,6 @@ define <vscale x 64 x i16> @vp_bswap_nxv64i16(<vscale x 64 x i16> %va, <vscale x
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 1
 ; CHECK-NEXT:    slli a1, a1, 2
-; CHECK-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -1631,11 +1631,11 @@ define <vscale x 64 x i16> @vp_bswap_nxv64i16(<vscale x 64 x i16> %va, <vscale x
 ;
 ; CHECK-ZVKB-LABEL: vp_bswap_nxv64i16:
 ; CHECK-ZVKB:       # %bb.0:
+; CHECK-ZVKB-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-ZVKB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVKB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVKB-NEXT:    srli a2, a1, 1
 ; CHECK-ZVKB-NEXT:    slli a1, a1, 2
-; CHECK-ZVKB-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
 ; CHECK-ZVKB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVKB-NEXT:    sub a2, a0, a1
 ; CHECK-ZVKB-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
index a4e5ab661c528..15f6ca600cb37 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
@@ -336,6 +336,7 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_call_nxv32i32_nxv32i32_i32(<vsca
 ; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    li a3, 2
 ; RV32-NEXT:    vs8r.v v16, (a1)
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv8r.v v8, v0
 ; RV32-NEXT:    vmv8r.v v16, v24
 ; RV32-NEXT:    call ext2
@@ -374,6 +375,7 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_call_nxv32i32_nxv32i32_i32(<vsca
 ; RV64-NEXT:    add a1, a3, a1
 ; RV64-NEXT:    li a3, 2
 ; RV64-NEXT:    vs8r.v v16, (a1)
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv8r.v v8, v0
 ; RV64-NEXT:    vmv8r.v v16, v24
 ; RV64-NEXT:    call ext2
@@ -451,6 +453,7 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_call_nxv32i32_nxv32i32_nxv32i32_
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 128
 ; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv8r.v v16, v0
 ; RV32-NEXT:    call ext3
 ; RV32-NEXT:    addi sp, s0, -144
@@ -523,6 +526,7 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_call_nxv32i32_nxv32i32_nxv32i32_
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 128
 ; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv8r.v v16, v0
 ; RV64-NEXT:    call ext3
 ; RV64-NEXT:    addi sp, s0, -144
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
index 9b27116fef7ca..2e181e0914c88 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
@@ -103,6 +103,7 @@ define target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @caller_tuple_return(
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    call callee_tuple_return
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv2r.v v6, v8
 ; RV32-NEXT:    vmv2r.v v8, v10
 ; RV32-NEXT:    vmv2r.v v10, v6
@@ -119,6 +120,7 @@ define target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @caller_tuple_return(
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
 ; RV64-NEXT:    call callee_tuple_return
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv2r.v v6, v8
 ; RV64-NEXT:    vmv2r.v v8, v10
 ; RV64-NEXT:    vmv2r.v v10, v6
@@ -144,6 +146,7 @@ define void @caller_tuple_argument(target("riscv.vector.tuple", <vscale x 16 x i
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv2r.v v6, v8
 ; RV32-NEXT:    vmv2r.v v8, v10
 ; RV32-NEXT:    vmv2r.v v10, v6
@@ -160,6 +163,7 @@ define void @caller_tuple_argument(target("riscv.vector.tuple", <vscale x 16 x i
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv2r.v v6, v8
 ; RV64-NEXT:    vmv2r.v v8, v10
 ; RV64-NEXT:    vmv2r.v v10, v6
diff --git a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
index 7d0b0118a7272..a81f0ac982f56 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.ceil.nxv4bf16(<vscale x 4 x bfloat>, <vsc
 define <vscale x 4 x bfloat> @vp_ceil_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.ceil.nxv8bf16(<vscale x 8 x bfloat>, <vsc
 define <vscale x 8 x bfloat> @vp_ceil_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.ceil.nxv16bf16(<vscale x 16 x bfloat>, <
 define <vscale x 16 x bfloat> @vp_ceil_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -279,9 +279,9 @@ define <vscale x 32 x bfloat> @vp_ceil_vv_nxv32bf16(<vscale x 32 x bfloat> %va,
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -582,8 +582,8 @@ define <vscale x 4 x half> @vp_ceil_vv_nxv4f16(<vscale x 4 x half> %va, <vscale
 ;
 ; ZVFHMIN-LABEL: vp_ceil_vv_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -649,10 +649,10 @@ declare <vscale x 8 x half> @llvm.vp.ceil.nxv8f16(<vscale x 8 x half>, <vscale x
 define <vscale x 8 x half> @vp_ceil_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_ceil_vv_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -668,8 +668,8 @@ define <vscale x 8 x half> @vp_ceil_vv_nxv8f16(<vscale x 8 x half> %va, <vscale
 ;
 ; ZVFHMIN-LABEL: vp_ceil_vv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -735,10 +735,10 @@ declare <vscale x 16 x half> @llvm.vp.ceil.nxv16f16(<vscale x 16 x half>, <vscal
 define <vscale x 16 x half> @vp_ceil_vv_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_ceil_vv_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -754,8 +754,8 @@ define <vscale x 16 x half> @vp_ceil_vv_nxv16f16(<vscale x 16 x half> %va, <vsca
 ;
 ; ZVFHMIN-LABEL: vp_ceil_vv_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -821,10 +821,10 @@ declare <vscale x 32 x half> @llvm.vp.ceil.nxv32f16(<vscale x 32 x half>, <vscal
 define <vscale x 32 x half> @vp_ceil_vv_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_ceil_vv_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -846,9 +846,9 @@ define <vscale x 32 x half> @vp_ceil_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1068,8 +1068,8 @@ declare <vscale x 4 x float> @llvm.vp.ceil.nxv4f32(<vscale x 4 x float>, <vscale
 define <vscale x 4 x float> @vp_ceil_vv_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1112,8 +1112,8 @@ declare <vscale x 8 x float> @llvm.vp.ceil.nxv8f32(<vscale x 8 x float>, <vscale
 define <vscale x 8 x float> @vp_ceil_vv_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1156,8 +1156,8 @@ declare <vscale x 16 x float> @llvm.vp.ceil.nxv16f32(<vscale x 16 x float>, <vsc
 define <vscale x 16 x float> @vp_ceil_vv_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1242,10 +1242,10 @@ declare <vscale x 2 x double> @llvm.vp.ceil.nxv2f64(<vscale x 2 x double>, <vsca
 define <vscale x 2 x double> @vp_ceil_vv_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1286,10 +1286,10 @@ declare <vscale x 4 x double> @llvm.vp.ceil.nxv4f64(<vscale x 4 x double>, <vsca
 define <vscale x 4 x double> @vp_ceil_vv_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1330,10 +1330,10 @@ declare <vscale x 7 x double> @llvm.vp.ceil.nxv7f64(<vscale x 7 x double>, <vsca
 define <vscale x 7 x double> @vp_ceil_vv_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1374,10 +1374,10 @@ declare <vscale x 8 x double> @llvm.vp.ceil.nxv8f64(<vscale x 8 x double>, <vsca
 define <vscale x 8 x double> @vp_ceil_vv_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_vv_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1425,13 +1425,13 @@ define <vscale x 16 x double> @vp_ceil_vv_nxv16f64(<vscale x 16 x double> %va, <
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/compressstore.ll b/llvm/test/CodeGen/RISCV/rvv/compressstore.ll
index bfb2d0a3accc4..08bf82d4bc796 100644
--- a/llvm/test/CodeGen/RISCV/rvv/compressstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/compressstore.ll
@@ -197,9 +197,9 @@ entry:
 define void @test_compresstore_v256i8(ptr %p, <256 x i1> %mask, <256 x i8> %data) {
 ; RV64-LABEL: test_compresstore_v256i8:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v7, v8
 ; RV64-NEXT:    li a2, 128
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v9, v0, 1
 ; RV64-NEXT:    vmv.x.s a3, v0
 ; RV64-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
@@ -230,9 +230,9 @@ define void @test_compresstore_v256i8(ptr %p, <256 x i1> %mask, <256 x i8> %data
 ; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    sub sp, sp, a2
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vmv8r.v v24, v16
 ; RV32-NEXT:    li a2, 128
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v9, v0, 1
 ; RV32-NEXT:    li a3, 32
 ; RV32-NEXT:    vmv.x.s a4, v0
@@ -652,15 +652,14 @@ define void @test_compresstore_v64i32(ptr %p, <64 x i1> %mask, <64 x i32> %data)
 ; RV64-NEXT:    vse32.v v24, (a0)
 ; RV64-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v0, 4
-; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; RV64-NEXT:    vmv.x.s a2, v0
 ; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vmv.x.s a1, v0
 ; RV64-NEXT:    vcompress.vm v24, v16, v8
-; RV64-NEXT:    vcpop.m a1, v8
-; RV64-NEXT:    cpopw a2, a2
-; RV64-NEXT:    slli a2, a2, 2
-; RV64-NEXT:    add a0, a0, a2
-; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vcpop.m a2, v8
+; RV64-NEXT:    cpopw a1, a1
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV64-NEXT:    vse32.v v24, (a0)
 ; RV64-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll b/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
index ad176df71397e..f6c26bbba89fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
@@ -18,11 +18,11 @@
 define void @constant_folding_crash(ptr %v54, <4 x ptr> %lanes.a, <4 x ptr> %lanes.b, <4 x i1> %sel) {
 ; RV32-LABEL: constant_folding_crash:
 ; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v10, v0
 ; RV32-NEXT:    lw a0, 8(a0)
 ; RV32-NEXT:    andi a0, a0, 1
 ; RV32-NEXT:    seqz a0, a0
-; RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv.v.x v11, a0
 ; RV32-NEXT:    vmsne.vi v0, v11, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
@@ -43,11 +43,11 @@ define void @constant_folding_crash(ptr %v54, <4 x ptr> %lanes.a, <4 x ptr> %lan
 ;
 ; RV64-LABEL: constant_folding_crash:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    ld a0, 8(a0)
 ; RV64-NEXT:    andi a0, a0, 1
 ; RV64-NEXT:    seqz a0, a0
-; RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; RV64-NEXT:    vmv.v.x v13, a0
 ; RV64-NEXT:    vmsne.vi v0, v13, 0
 ; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
index f56a792fdef6a..ce4bc48dff042 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
@@ -1235,13 +1235,13 @@ declare <vscale x 16 x i64> @llvm.vp.ctlz.nxv16i64(<vscale x 16 x i64>, i1 immar
 define <vscale x 16 x i64> @vp_ctlz_nxv16i64(<vscale x 16 x i64> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv16i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    fsrmi a4, 1
 ; CHECK-NEXT:    li a2, 52
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    sub a5, a0, a1
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a5
 ; CHECK-NEXT:    addi a3, a3, -1
@@ -1270,11 +1270,11 @@ define <vscale x 16 x i64> @vp_ctlz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv16i64:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 3
 ; CHECK-ZVBB-NEXT:    sub a3, a0, a1
-; CHECK-ZVBB-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sltu a2, a0, a3
 ; CHECK-ZVBB-NEXT:    addi a2, a2, -1
@@ -2465,12 +2465,12 @@ define <vscale x 8 x i64> @vp_ctlz_zero_undef_nxv8i64_unmasked(<vscale x 8 x i64
 define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64(<vscale x 16 x i64> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv16i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a4, a0, a1
-; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a4
 ; CHECK-NEXT:    addi a2, a2, -1
@@ -2497,11 +2497,11 @@ define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv16i64:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 3
 ; CHECK-ZVBB-NEXT:    sub a3, a0, a1
-; CHECK-ZVBB-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sltu a2, a0, a3
 ; CHECK-ZVBB-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
index 9e75dc9dccffd..52ddd9ab2f832 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
@@ -2022,6 +2022,7 @@ define <vscale x 16 x i64> @vp_ctpop_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
+; RV32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v7, v0
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 24
@@ -2032,7 +2033,6 @@ define <vscale x 16 x i64> @vp_ctpop_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    lui a2, 349525
 ; RV32-NEXT:    srli a3, a1, 3
-; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vx v0, v0, a3
 ; RV32-NEXT:    sub a3, a0, a1
 ; RV32-NEXT:    addi a2, a2, 1365
@@ -2294,11 +2294,11 @@ define <vscale x 16 x i64> @vp_ctpop_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ;
 ; CHECK-ZVBB-LABEL: vp_ctpop_nxv16i64:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 3
 ; CHECK-ZVBB-NEXT:    sub a3, a0, a1
-; CHECK-ZVBB-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sltu a2, a0, a3
 ; CHECK-ZVBB-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
index 9e6295b664417..766717d92a749 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
@@ -2246,6 +2246,7 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
+; RV32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v24, v0
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 5
@@ -2256,7 +2257,6 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    li a2, 1
 ; RV32-NEXT:    srli a3, a1, 3
 ; RV32-NEXT:    sub a4, a0, a1
-; RV32-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vx v0, v0, a3
 ; RV32-NEXT:    sltu a3, a0, a4
 ; RV32-NEXT:    addi a3, a3, -1
@@ -2499,6 +2499,7 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    slli a1, a1, 4
 ; RV64-NEXT:    sub sp, sp, a1
 ; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmv1r.v v24, v0
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    slli a1, a1, 3
@@ -2517,7 +2518,6 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    addiw a4, a4, 819
 ; RV64-NEXT:    addiw a5, a5, -241
 ; RV64-NEXT:    addiw t1, a6, 257
-; RV64-NEXT:    vsetvli a6, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vx v0, v0, a7
 ; RV64-NEXT:    slli a7, a3, 32
 ; RV64-NEXT:    add a7, a3, a7
@@ -2586,11 +2586,11 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ;
 ; CHECK-ZVBB-LABEL: vp_cttz_nxv16i64:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 3
 ; CHECK-ZVBB-NEXT:    sub a3, a0, a1
-; CHECK-ZVBB-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sltu a2, a0, a3
 ; CHECK-ZVBB-NEXT:    addi a2, a2, -1
@@ -4002,6 +4002,7 @@ define <vscale x 16 x i64> @vp_cttz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -4012,7 +4013,6 @@ define <vscale x 16 x i64> @vp_cttz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a4, a0, a1
-; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a4
 ; CHECK-NEXT:    addi a2, a2, -1
@@ -4057,11 +4057,11 @@ define <vscale x 16 x i64> @vp_cttz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ;
 ; CHECK-ZVBB-LABEL: vp_cttz_zero_undef_nxv16i64:
 ; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vmv1r.v v24, v0
 ; CHECK-ZVBB-NEXT:    csrr a1, vlenb
 ; CHECK-ZVBB-NEXT:    srli a2, a1, 3
 ; CHECK-ZVBB-NEXT:    sub a3, a0, a1
-; CHECK-ZVBB-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-ZVBB-NEXT:    sltu a2, a0, a3
 ; CHECK-ZVBB-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/expandload.ll b/llvm/test/CodeGen/RISCV/rvv/expandload.ll
index b32d85bb1943a..a35cf14203f78 100644
--- a/llvm/test/CodeGen/RISCV/rvv/expandload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/expandload.ll
@@ -227,9 +227,9 @@ define <256 x i8> @test_expandload_v256i8(ptr %base, <256 x i1> %mask, <256 x i8
 ; CHECK-RV32-NEXT:    add a2, sp, a2
 ; CHECK-RV32-NEXT:    addi a2, a2, 16
 ; CHECK-RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv1r.v v7, v8
 ; CHECK-RV32-NEXT:    li a2, 128
-; CHECK-RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-RV32-NEXT:    vslidedown.vi v9, v0, 1
 ; CHECK-RV32-NEXT:    li a3, 32
 ; CHECK-RV32-NEXT:    vmv.x.s a4, v0
@@ -338,9 +338,9 @@ define <256 x i8> @test_expandload_v256i8(ptr %base, <256 x i1> %mask, <256 x i8
 ; CHECK-RV64-NEXT:    add a2, sp, a2
 ; CHECK-RV64-NEXT:    addi a2, a2, 16
 ; CHECK-RV64-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv1r.v v7, v8
 ; CHECK-RV64-NEXT:    li a2, 128
-; CHECK-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-RV64-NEXT:    vslidedown.vi v9, v0, 1
 ; CHECK-RV64-NEXT:    vmv.x.s a3, v0
 ; CHECK-RV64-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
@@ -1626,8 +1626,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a1, .LBB61_30
 ; CHECK-RV32-NEXT:  .LBB61_29: # %cond.load109
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 29, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 28
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -1639,8 +1639,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a2, .LBB61_32
 ; CHECK-RV32-NEXT:  # %bb.31: # %cond.load113
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 30, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a2
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 29
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -1786,9 +1786,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a3, .LBB61_66
 ; CHECK-RV32-NEXT:  .LBB61_65: # %cond.load241
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 62
 ; CHECK-RV32-NEXT:    li a4, 61
@@ -1939,9 +1938,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a2, .LBB61_100
 ; CHECK-RV32-NEXT:  .LBB61_99: # %cond.load369
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 94
 ; CHECK-RV32-NEXT:    li a4, 93
@@ -2092,9 +2090,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a3, .LBB61_134
 ; CHECK-RV32-NEXT:  .LBB61_133: # %cond.load497
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 126
 ; CHECK-RV32-NEXT:    li a4, 125
@@ -2245,9 +2242,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a2, .LBB61_168
 ; CHECK-RV32-NEXT:  .LBB61_167: # %cond.load625
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 158
 ; CHECK-RV32-NEXT:    li a4, 157
@@ -2398,9 +2394,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a3, .LBB61_202
 ; CHECK-RV32-NEXT:  .LBB61_201: # %cond.load753
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 190
 ; CHECK-RV32-NEXT:    li a4, 189
@@ -2551,9 +2546,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a2, .LBB61_236
 ; CHECK-RV32-NEXT:  .LBB61_235: # %cond.load881
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 222
 ; CHECK-RV32-NEXT:    li a4, 221
@@ -2704,9 +2698,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    bgez a3, .LBB61_270
 ; CHECK-RV32-NEXT:  .LBB61_269: # %cond.load1009
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 254
 ; CHECK-RV32-NEXT:    li a4, 253
@@ -3907,10 +3900,9 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_2
 ; CHECK-RV32-NEXT:  .LBB61_545: # %cond.load1
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
+; CHECK-RV32-NEXT:    vsetivli zero, 2, e8, m1, tu, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
-; CHECK-RV32-NEXT:    vsetivli zero, 2, e8, m1, tu, ma
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
 ; CHECK-RV32-NEXT:    vmv1r.v v16, v8
@@ -3920,8 +3912,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_3
 ; CHECK-RV32-NEXT:  .LBB61_546: # %cond.load5
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 3, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3932,8 +3924,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_4
 ; CHECK-RV32-NEXT:  .LBB61_547: # %cond.load9
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3944,8 +3936,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_5
 ; CHECK-RV32-NEXT:  .LBB61_548: # %cond.load13
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 5, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 4
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3956,8 +3948,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_6
 ; CHECK-RV32-NEXT:  .LBB61_549: # %cond.load17
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 6, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 5
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3968,8 +3960,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_7
 ; CHECK-RV32-NEXT:  .LBB61_550: # %cond.load21
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 7, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 6
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3980,8 +3972,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_8
 ; CHECK-RV32-NEXT:  .LBB61_551: # %cond.load25
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 7
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -3992,8 +3984,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_9
 ; CHECK-RV32-NEXT:  .LBB61_552: # %cond.load29
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 9, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 8
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4004,8 +3996,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_10
 ; CHECK-RV32-NEXT:  .LBB61_553: # %cond.load33
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 10, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 9
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4016,8 +4008,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_11
 ; CHECK-RV32-NEXT:  .LBB61_554: # %cond.load37
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 11, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 10
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4028,8 +4020,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_12
 ; CHECK-RV32-NEXT:  .LBB61_555: # %cond.load41
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 12, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 11
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4040,8 +4032,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_13
 ; CHECK-RV32-NEXT:  .LBB61_556: # %cond.load45
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 13, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 12
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4052,8 +4044,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_14
 ; CHECK-RV32-NEXT:  .LBB61_557: # %cond.load49
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 14, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 13
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4064,8 +4056,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_15
 ; CHECK-RV32-NEXT:  .LBB61_558: # %cond.load53
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 15, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 14
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4076,8 +4068,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_16
 ; CHECK-RV32-NEXT:  .LBB61_559: # %cond.load57
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 16, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 15
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4088,8 +4080,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_17
 ; CHECK-RV32-NEXT:  .LBB61_560: # %cond.load61
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 17, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 16
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4100,8 +4092,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_18
 ; CHECK-RV32-NEXT:  .LBB61_561: # %cond.load65
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 18, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 17
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4112,8 +4104,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_19
 ; CHECK-RV32-NEXT:  .LBB61_562: # %cond.load69
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 19, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 18
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4124,8 +4116,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_20
 ; CHECK-RV32-NEXT:  .LBB61_563: # %cond.load73
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 20, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 19
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4136,8 +4128,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_21
 ; CHECK-RV32-NEXT:  .LBB61_564: # %cond.load77
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 21, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 20
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4148,8 +4140,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_22
 ; CHECK-RV32-NEXT:  .LBB61_565: # %cond.load81
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 22, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 21
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4160,8 +4152,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_23
 ; CHECK-RV32-NEXT:  .LBB61_566: # %cond.load85
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 23, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 22
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4172,8 +4164,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_24
 ; CHECK-RV32-NEXT:  .LBB61_567: # %cond.load89
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 24, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 23
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4184,8 +4176,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_25
 ; CHECK-RV32-NEXT:  .LBB61_568: # %cond.load93
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 25, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 24
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4196,8 +4188,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_26
 ; CHECK-RV32-NEXT:  .LBB61_569: # %cond.load97
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 26, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 25
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4208,8 +4200,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_27
 ; CHECK-RV32-NEXT:  .LBB61_570: # %cond.load101
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 27, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 26
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4220,8 +4212,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_28
 ; CHECK-RV32-NEXT:  .LBB61_571: # %cond.load105
 ; CHECK-RV32-NEXT:    lbu a1, 0(a0)
-; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vsetivli zero, 28, e8, m1, tu, ma
+; CHECK-RV32-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV32-NEXT:    vslideup.vi v8, v9, 27
 ; CHECK-RV32-NEXT:    addi a0, a0, 1
@@ -4247,9 +4239,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_36
 ; CHECK-RV32-NEXT:  .LBB61_573: # %cond.load125
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 33
 ; CHECK-RV32-NEXT:    li a4, 32
@@ -4263,9 +4254,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_37
 ; CHECK-RV32-NEXT:  .LBB61_574: # %cond.load129
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 34
 ; CHECK-RV32-NEXT:    li a4, 33
@@ -4279,9 +4269,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_38
 ; CHECK-RV32-NEXT:  .LBB61_575: # %cond.load133
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 35
 ; CHECK-RV32-NEXT:    li a4, 34
@@ -4295,9 +4284,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_39
 ; CHECK-RV32-NEXT:  .LBB61_576: # %cond.load137
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 36
 ; CHECK-RV32-NEXT:    li a4, 35
@@ -4311,9 +4299,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_40
 ; CHECK-RV32-NEXT:  .LBB61_577: # %cond.load141
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 37
 ; CHECK-RV32-NEXT:    li a4, 36
@@ -4327,9 +4314,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_41
 ; CHECK-RV32-NEXT:  .LBB61_578: # %cond.load145
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 38
 ; CHECK-RV32-NEXT:    li a4, 37
@@ -4343,9 +4329,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_42
 ; CHECK-RV32-NEXT:  .LBB61_579: # %cond.load149
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 39
 ; CHECK-RV32-NEXT:    li a4, 38
@@ -4359,9 +4344,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_43
 ; CHECK-RV32-NEXT:  .LBB61_580: # %cond.load153
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 40
 ; CHECK-RV32-NEXT:    li a4, 39
@@ -4375,9 +4359,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_44
 ; CHECK-RV32-NEXT:  .LBB61_581: # %cond.load157
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 41
 ; CHECK-RV32-NEXT:    li a4, 40
@@ -4391,9 +4374,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_45
 ; CHECK-RV32-NEXT:  .LBB61_582: # %cond.load161
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 42
 ; CHECK-RV32-NEXT:    li a4, 41
@@ -4407,9 +4389,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_46
 ; CHECK-RV32-NEXT:  .LBB61_583: # %cond.load165
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 43
 ; CHECK-RV32-NEXT:    li a4, 42
@@ -4423,9 +4404,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_47
 ; CHECK-RV32-NEXT:  .LBB61_584: # %cond.load169
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 44
 ; CHECK-RV32-NEXT:    li a4, 43
@@ -4439,9 +4419,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_48
 ; CHECK-RV32-NEXT:  .LBB61_585: # %cond.load173
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 45
 ; CHECK-RV32-NEXT:    li a4, 44
@@ -4455,9 +4434,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_49
 ; CHECK-RV32-NEXT:  .LBB61_586: # %cond.load177
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 46
 ; CHECK-RV32-NEXT:    li a4, 45
@@ -4471,9 +4449,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_50
 ; CHECK-RV32-NEXT:  .LBB61_587: # %cond.load181
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 47
 ; CHECK-RV32-NEXT:    li a4, 46
@@ -4487,9 +4464,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_51
 ; CHECK-RV32-NEXT:  .LBB61_588: # %cond.load185
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 48
 ; CHECK-RV32-NEXT:    li a4, 47
@@ -4503,9 +4479,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_52
 ; CHECK-RV32-NEXT:  .LBB61_589: # %cond.load189
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 49
 ; CHECK-RV32-NEXT:    li a4, 48
@@ -4519,9 +4494,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_53
 ; CHECK-RV32-NEXT:  .LBB61_590: # %cond.load193
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 50
 ; CHECK-RV32-NEXT:    li a4, 49
@@ -4535,9 +4509,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_54
 ; CHECK-RV32-NEXT:  .LBB61_591: # %cond.load197
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 51
 ; CHECK-RV32-NEXT:    li a4, 50
@@ -4551,9 +4524,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_55
 ; CHECK-RV32-NEXT:  .LBB61_592: # %cond.load201
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 52
 ; CHECK-RV32-NEXT:    li a4, 51
@@ -4567,9 +4539,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_56
 ; CHECK-RV32-NEXT:  .LBB61_593: # %cond.load205
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 53
 ; CHECK-RV32-NEXT:    li a4, 52
@@ -4583,9 +4554,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_57
 ; CHECK-RV32-NEXT:  .LBB61_594: # %cond.load209
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 54
 ; CHECK-RV32-NEXT:    li a4, 53
@@ -4599,9 +4569,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_58
 ; CHECK-RV32-NEXT:  .LBB61_595: # %cond.load213
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 55
 ; CHECK-RV32-NEXT:    li a4, 54
@@ -4615,9 +4584,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_59
 ; CHECK-RV32-NEXT:  .LBB61_596: # %cond.load217
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 56
 ; CHECK-RV32-NEXT:    li a4, 55
@@ -4631,9 +4599,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_60
 ; CHECK-RV32-NEXT:  .LBB61_597: # %cond.load221
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 57
 ; CHECK-RV32-NEXT:    li a4, 56
@@ -4647,9 +4614,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_61
 ; CHECK-RV32-NEXT:  .LBB61_598: # %cond.load225
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 58
 ; CHECK-RV32-NEXT:    li a4, 57
@@ -4663,9 +4629,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_62
 ; CHECK-RV32-NEXT:  .LBB61_599: # %cond.load229
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 59
 ; CHECK-RV32-NEXT:    li a4, 58
@@ -4679,9 +4644,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_63
 ; CHECK-RV32-NEXT:  .LBB61_600: # %cond.load233
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 60
 ; CHECK-RV32-NEXT:    li a4, 59
@@ -4695,9 +4659,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_64
 ; CHECK-RV32-NEXT:  .LBB61_601: # %cond.load237
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v9, a3
 ; CHECK-RV32-NEXT:    li a3, 61
 ; CHECK-RV32-NEXT:    li a4, 60
@@ -4727,9 +4690,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_70
 ; CHECK-RV32-NEXT:  .LBB61_603: # %cond.load253
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 65
 ; CHECK-RV32-NEXT:    li a4, 64
@@ -4743,9 +4705,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_71
 ; CHECK-RV32-NEXT:  .LBB61_604: # %cond.load257
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 66
 ; CHECK-RV32-NEXT:    li a4, 65
@@ -4759,9 +4720,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_72
 ; CHECK-RV32-NEXT:  .LBB61_605: # %cond.load261
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 67
 ; CHECK-RV32-NEXT:    li a4, 66
@@ -4775,9 +4735,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_73
 ; CHECK-RV32-NEXT:  .LBB61_606: # %cond.load265
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 68
 ; CHECK-RV32-NEXT:    li a4, 67
@@ -4791,9 +4750,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_74
 ; CHECK-RV32-NEXT:  .LBB61_607: # %cond.load269
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 69
 ; CHECK-RV32-NEXT:    li a4, 68
@@ -4807,9 +4765,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_75
 ; CHECK-RV32-NEXT:  .LBB61_608: # %cond.load273
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 70
 ; CHECK-RV32-NEXT:    li a4, 69
@@ -4823,9 +4780,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_76
 ; CHECK-RV32-NEXT:  .LBB61_609: # %cond.load277
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 71
 ; CHECK-RV32-NEXT:    li a4, 70
@@ -4839,9 +4795,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_77
 ; CHECK-RV32-NEXT:  .LBB61_610: # %cond.load281
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 72
 ; CHECK-RV32-NEXT:    li a4, 71
@@ -4855,9 +4810,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_78
 ; CHECK-RV32-NEXT:  .LBB61_611: # %cond.load285
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 73
 ; CHECK-RV32-NEXT:    li a4, 72
@@ -4871,9 +4825,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_79
 ; CHECK-RV32-NEXT:  .LBB61_612: # %cond.load289
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 74
 ; CHECK-RV32-NEXT:    li a4, 73
@@ -4887,9 +4840,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_80
 ; CHECK-RV32-NEXT:  .LBB61_613: # %cond.load293
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 75
 ; CHECK-RV32-NEXT:    li a4, 74
@@ -4903,9 +4855,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_81
 ; CHECK-RV32-NEXT:  .LBB61_614: # %cond.load297
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 76
 ; CHECK-RV32-NEXT:    li a4, 75
@@ -4919,9 +4870,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_82
 ; CHECK-RV32-NEXT:  .LBB61_615: # %cond.load301
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 77
 ; CHECK-RV32-NEXT:    li a4, 76
@@ -4935,9 +4885,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_83
 ; CHECK-RV32-NEXT:  .LBB61_616: # %cond.load305
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 78
 ; CHECK-RV32-NEXT:    li a4, 77
@@ -4951,9 +4900,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_84
 ; CHECK-RV32-NEXT:  .LBB61_617: # %cond.load309
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 79
 ; CHECK-RV32-NEXT:    li a4, 78
@@ -4967,9 +4915,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_85
 ; CHECK-RV32-NEXT:  .LBB61_618: # %cond.load313
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 80
 ; CHECK-RV32-NEXT:    li a4, 79
@@ -4983,9 +4930,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_86
 ; CHECK-RV32-NEXT:  .LBB61_619: # %cond.load317
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 81
 ; CHECK-RV32-NEXT:    li a4, 80
@@ -4999,9 +4945,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_87
 ; CHECK-RV32-NEXT:  .LBB61_620: # %cond.load321
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 82
 ; CHECK-RV32-NEXT:    li a4, 81
@@ -5015,9 +4960,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_88
 ; CHECK-RV32-NEXT:  .LBB61_621: # %cond.load325
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 83
 ; CHECK-RV32-NEXT:    li a4, 82
@@ -5031,9 +4975,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_89
 ; CHECK-RV32-NEXT:  .LBB61_622: # %cond.load329
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 84
 ; CHECK-RV32-NEXT:    li a4, 83
@@ -5047,9 +4990,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_90
 ; CHECK-RV32-NEXT:  .LBB61_623: # %cond.load333
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 85
 ; CHECK-RV32-NEXT:    li a4, 84
@@ -5063,9 +5005,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_91
 ; CHECK-RV32-NEXT:  .LBB61_624: # %cond.load337
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 86
 ; CHECK-RV32-NEXT:    li a4, 85
@@ -5079,9 +5020,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_92
 ; CHECK-RV32-NEXT:  .LBB61_625: # %cond.load341
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 87
 ; CHECK-RV32-NEXT:    li a4, 86
@@ -5095,9 +5035,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_93
 ; CHECK-RV32-NEXT:  .LBB61_626: # %cond.load345
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 88
 ; CHECK-RV32-NEXT:    li a4, 87
@@ -5111,9 +5050,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_94
 ; CHECK-RV32-NEXT:  .LBB61_627: # %cond.load349
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 89
 ; CHECK-RV32-NEXT:    li a4, 88
@@ -5127,9 +5065,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_95
 ; CHECK-RV32-NEXT:  .LBB61_628: # %cond.load353
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 90
 ; CHECK-RV32-NEXT:    li a4, 89
@@ -5143,9 +5080,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_96
 ; CHECK-RV32-NEXT:  .LBB61_629: # %cond.load357
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 91
 ; CHECK-RV32-NEXT:    li a4, 90
@@ -5159,9 +5095,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_97
 ; CHECK-RV32-NEXT:  .LBB61_630: # %cond.load361
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 92
 ; CHECK-RV32-NEXT:    li a4, 91
@@ -5175,9 +5110,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_98
 ; CHECK-RV32-NEXT:  .LBB61_631: # %cond.load365
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV32-NEXT:    li a2, 93
 ; CHECK-RV32-NEXT:    li a4, 92
@@ -5207,9 +5141,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_104
 ; CHECK-RV32-NEXT:  .LBB61_633: # %cond.load381
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 97
 ; CHECK-RV32-NEXT:    li a4, 96
@@ -5223,9 +5156,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_105
 ; CHECK-RV32-NEXT:  .LBB61_634: # %cond.load385
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 98
 ; CHECK-RV32-NEXT:    li a4, 97
@@ -5239,9 +5171,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_106
 ; CHECK-RV32-NEXT:  .LBB61_635: # %cond.load389
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 99
 ; CHECK-RV32-NEXT:    li a4, 98
@@ -5255,9 +5186,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_107
 ; CHECK-RV32-NEXT:  .LBB61_636: # %cond.load393
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 100
 ; CHECK-RV32-NEXT:    li a4, 99
@@ -5271,9 +5201,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_108
 ; CHECK-RV32-NEXT:  .LBB61_637: # %cond.load397
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 101
 ; CHECK-RV32-NEXT:    li a4, 100
@@ -5287,9 +5216,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_109
 ; CHECK-RV32-NEXT:  .LBB61_638: # %cond.load401
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 102
 ; CHECK-RV32-NEXT:    li a4, 101
@@ -5303,9 +5231,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_110
 ; CHECK-RV32-NEXT:  .LBB61_639: # %cond.load405
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 103
 ; CHECK-RV32-NEXT:    li a4, 102
@@ -5319,9 +5246,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_111
 ; CHECK-RV32-NEXT:  .LBB61_640: # %cond.load409
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 104
 ; CHECK-RV32-NEXT:    li a4, 103
@@ -5335,9 +5261,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_112
 ; CHECK-RV32-NEXT:  .LBB61_641: # %cond.load413
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 105
 ; CHECK-RV32-NEXT:    li a4, 104
@@ -5351,9 +5276,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_113
 ; CHECK-RV32-NEXT:  .LBB61_642: # %cond.load417
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 106
 ; CHECK-RV32-NEXT:    li a4, 105
@@ -5367,9 +5291,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_114
 ; CHECK-RV32-NEXT:  .LBB61_643: # %cond.load421
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 107
 ; CHECK-RV32-NEXT:    li a4, 106
@@ -5383,9 +5306,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_115
 ; CHECK-RV32-NEXT:  .LBB61_644: # %cond.load425
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 108
 ; CHECK-RV32-NEXT:    li a4, 107
@@ -5399,9 +5321,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_116
 ; CHECK-RV32-NEXT:  .LBB61_645: # %cond.load429
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 109
 ; CHECK-RV32-NEXT:    li a4, 108
@@ -5415,9 +5336,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_117
 ; CHECK-RV32-NEXT:  .LBB61_646: # %cond.load433
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 110
 ; CHECK-RV32-NEXT:    li a4, 109
@@ -5431,9 +5351,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_118
 ; CHECK-RV32-NEXT:  .LBB61_647: # %cond.load437
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 111
 ; CHECK-RV32-NEXT:    li a4, 110
@@ -5447,9 +5366,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_119
 ; CHECK-RV32-NEXT:  .LBB61_648: # %cond.load441
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 112
 ; CHECK-RV32-NEXT:    li a4, 111
@@ -5463,9 +5381,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_120
 ; CHECK-RV32-NEXT:  .LBB61_649: # %cond.load445
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 113
 ; CHECK-RV32-NEXT:    li a4, 112
@@ -5479,9 +5396,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_121
 ; CHECK-RV32-NEXT:  .LBB61_650: # %cond.load449
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 114
 ; CHECK-RV32-NEXT:    li a4, 113
@@ -5495,9 +5411,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_122
 ; CHECK-RV32-NEXT:  .LBB61_651: # %cond.load453
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 115
 ; CHECK-RV32-NEXT:    li a4, 114
@@ -5511,9 +5426,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_123
 ; CHECK-RV32-NEXT:  .LBB61_652: # %cond.load457
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 116
 ; CHECK-RV32-NEXT:    li a4, 115
@@ -5527,9 +5441,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_124
 ; CHECK-RV32-NEXT:  .LBB61_653: # %cond.load461
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 117
 ; CHECK-RV32-NEXT:    li a4, 116
@@ -5543,9 +5456,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_125
 ; CHECK-RV32-NEXT:  .LBB61_654: # %cond.load465
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 118
 ; CHECK-RV32-NEXT:    li a4, 117
@@ -5559,9 +5471,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_126
 ; CHECK-RV32-NEXT:  .LBB61_655: # %cond.load469
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 119
 ; CHECK-RV32-NEXT:    li a4, 118
@@ -5575,9 +5486,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_127
 ; CHECK-RV32-NEXT:  .LBB61_656: # %cond.load473
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 120
 ; CHECK-RV32-NEXT:    li a4, 119
@@ -5591,9 +5501,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_128
 ; CHECK-RV32-NEXT:  .LBB61_657: # %cond.load477
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 121
 ; CHECK-RV32-NEXT:    li a4, 120
@@ -5607,9 +5516,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_129
 ; CHECK-RV32-NEXT:  .LBB61_658: # %cond.load481
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 122
 ; CHECK-RV32-NEXT:    li a4, 121
@@ -5623,9 +5531,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_130
 ; CHECK-RV32-NEXT:  .LBB61_659: # %cond.load485
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 123
 ; CHECK-RV32-NEXT:    li a4, 122
@@ -5639,9 +5546,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_131
 ; CHECK-RV32-NEXT:  .LBB61_660: # %cond.load489
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 124
 ; CHECK-RV32-NEXT:    li a4, 123
@@ -5655,9 +5561,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_132
 ; CHECK-RV32-NEXT:  .LBB61_661: # %cond.load493
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v10, a3
 ; CHECK-RV32-NEXT:    li a3, 125
 ; CHECK-RV32-NEXT:    li a4, 124
@@ -5687,9 +5592,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_138
 ; CHECK-RV32-NEXT:  .LBB61_663: # %cond.load509
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 129
 ; CHECK-RV32-NEXT:    li a4, 128
@@ -5703,9 +5607,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_139
 ; CHECK-RV32-NEXT:  .LBB61_664: # %cond.load513
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 130
 ; CHECK-RV32-NEXT:    li a4, 129
@@ -5719,9 +5622,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_140
 ; CHECK-RV32-NEXT:  .LBB61_665: # %cond.load517
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 131
 ; CHECK-RV32-NEXT:    li a4, 130
@@ -5735,9 +5637,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_141
 ; CHECK-RV32-NEXT:  .LBB61_666: # %cond.load521
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 132
 ; CHECK-RV32-NEXT:    li a4, 131
@@ -5751,9 +5652,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_142
 ; CHECK-RV32-NEXT:  .LBB61_667: # %cond.load525
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 133
 ; CHECK-RV32-NEXT:    li a4, 132
@@ -5767,9 +5667,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_143
 ; CHECK-RV32-NEXT:  .LBB61_668: # %cond.load529
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 134
 ; CHECK-RV32-NEXT:    li a4, 133
@@ -5783,9 +5682,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_144
 ; CHECK-RV32-NEXT:  .LBB61_669: # %cond.load533
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 135
 ; CHECK-RV32-NEXT:    li a4, 134
@@ -5799,9 +5697,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_145
 ; CHECK-RV32-NEXT:  .LBB61_670: # %cond.load537
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 136
 ; CHECK-RV32-NEXT:    li a4, 135
@@ -5815,9 +5712,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_146
 ; CHECK-RV32-NEXT:  .LBB61_671: # %cond.load541
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 137
 ; CHECK-RV32-NEXT:    li a4, 136
@@ -5831,9 +5727,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_147
 ; CHECK-RV32-NEXT:  .LBB61_672: # %cond.load545
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 138
 ; CHECK-RV32-NEXT:    li a4, 137
@@ -5847,9 +5742,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_148
 ; CHECK-RV32-NEXT:  .LBB61_673: # %cond.load549
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 139
 ; CHECK-RV32-NEXT:    li a4, 138
@@ -5863,9 +5757,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_149
 ; CHECK-RV32-NEXT:  .LBB61_674: # %cond.load553
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 140
 ; CHECK-RV32-NEXT:    li a4, 139
@@ -5879,9 +5772,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_150
 ; CHECK-RV32-NEXT:  .LBB61_675: # %cond.load557
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 141
 ; CHECK-RV32-NEXT:    li a4, 140
@@ -5895,9 +5787,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_151
 ; CHECK-RV32-NEXT:  .LBB61_676: # %cond.load561
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 142
 ; CHECK-RV32-NEXT:    li a4, 141
@@ -5911,9 +5802,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_152
 ; CHECK-RV32-NEXT:  .LBB61_677: # %cond.load565
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 143
 ; CHECK-RV32-NEXT:    li a4, 142
@@ -5927,9 +5817,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_153
 ; CHECK-RV32-NEXT:  .LBB61_678: # %cond.load569
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 144
 ; CHECK-RV32-NEXT:    li a4, 143
@@ -5943,9 +5832,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_154
 ; CHECK-RV32-NEXT:  .LBB61_679: # %cond.load573
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 145
 ; CHECK-RV32-NEXT:    li a4, 144
@@ -5959,9 +5847,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_155
 ; CHECK-RV32-NEXT:  .LBB61_680: # %cond.load577
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 146
 ; CHECK-RV32-NEXT:    li a4, 145
@@ -5975,9 +5862,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_156
 ; CHECK-RV32-NEXT:  .LBB61_681: # %cond.load581
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 147
 ; CHECK-RV32-NEXT:    li a4, 146
@@ -5991,9 +5877,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_157
 ; CHECK-RV32-NEXT:  .LBB61_682: # %cond.load585
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 148
 ; CHECK-RV32-NEXT:    li a4, 147
@@ -6007,9 +5892,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_158
 ; CHECK-RV32-NEXT:  .LBB61_683: # %cond.load589
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 149
 ; CHECK-RV32-NEXT:    li a4, 148
@@ -6023,9 +5907,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_159
 ; CHECK-RV32-NEXT:  .LBB61_684: # %cond.load593
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 150
 ; CHECK-RV32-NEXT:    li a4, 149
@@ -6039,9 +5922,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_160
 ; CHECK-RV32-NEXT:  .LBB61_685: # %cond.load597
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 151
 ; CHECK-RV32-NEXT:    li a4, 150
@@ -6055,9 +5937,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_161
 ; CHECK-RV32-NEXT:  .LBB61_686: # %cond.load601
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 152
 ; CHECK-RV32-NEXT:    li a4, 151
@@ -6071,9 +5952,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_162
 ; CHECK-RV32-NEXT:  .LBB61_687: # %cond.load605
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 153
 ; CHECK-RV32-NEXT:    li a4, 152
@@ -6087,9 +5967,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_163
 ; CHECK-RV32-NEXT:  .LBB61_688: # %cond.load609
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 154
 ; CHECK-RV32-NEXT:    li a4, 153
@@ -6103,9 +5982,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_164
 ; CHECK-RV32-NEXT:  .LBB61_689: # %cond.load613
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 155
 ; CHECK-RV32-NEXT:    li a4, 154
@@ -6119,9 +5997,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_165
 ; CHECK-RV32-NEXT:  .LBB61_690: # %cond.load617
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 156
 ; CHECK-RV32-NEXT:    li a4, 155
@@ -6135,9 +6012,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_166
 ; CHECK-RV32-NEXT:  .LBB61_691: # %cond.load621
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 157
 ; CHECK-RV32-NEXT:    li a4, 156
@@ -6167,9 +6043,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_172
 ; CHECK-RV32-NEXT:  .LBB61_693: # %cond.load637
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 161
 ; CHECK-RV32-NEXT:    li a4, 160
@@ -6183,9 +6058,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_173
 ; CHECK-RV32-NEXT:  .LBB61_694: # %cond.load641
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 162
 ; CHECK-RV32-NEXT:    li a4, 161
@@ -6199,9 +6073,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_174
 ; CHECK-RV32-NEXT:  .LBB61_695: # %cond.load645
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 163
 ; CHECK-RV32-NEXT:    li a4, 162
@@ -6215,9 +6088,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_175
 ; CHECK-RV32-NEXT:  .LBB61_696: # %cond.load649
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 164
 ; CHECK-RV32-NEXT:    li a4, 163
@@ -6231,9 +6103,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_176
 ; CHECK-RV32-NEXT:  .LBB61_697: # %cond.load653
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 165
 ; CHECK-RV32-NEXT:    li a4, 164
@@ -6247,9 +6118,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_177
 ; CHECK-RV32-NEXT:  .LBB61_698: # %cond.load657
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 166
 ; CHECK-RV32-NEXT:    li a4, 165
@@ -6263,9 +6133,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_178
 ; CHECK-RV32-NEXT:  .LBB61_699: # %cond.load661
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 167
 ; CHECK-RV32-NEXT:    li a4, 166
@@ -6279,9 +6148,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_179
 ; CHECK-RV32-NEXT:  .LBB61_700: # %cond.load665
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 168
 ; CHECK-RV32-NEXT:    li a4, 167
@@ -6295,9 +6163,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_180
 ; CHECK-RV32-NEXT:  .LBB61_701: # %cond.load669
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 169
 ; CHECK-RV32-NEXT:    li a4, 168
@@ -6311,9 +6178,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_181
 ; CHECK-RV32-NEXT:  .LBB61_702: # %cond.load673
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 170
 ; CHECK-RV32-NEXT:    li a4, 169
@@ -6327,9 +6193,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_182
 ; CHECK-RV32-NEXT:  .LBB61_703: # %cond.load677
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 171
 ; CHECK-RV32-NEXT:    li a4, 170
@@ -6343,9 +6208,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_183
 ; CHECK-RV32-NEXT:  .LBB61_704: # %cond.load681
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 172
 ; CHECK-RV32-NEXT:    li a4, 171
@@ -6359,9 +6223,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_184
 ; CHECK-RV32-NEXT:  .LBB61_705: # %cond.load685
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 173
 ; CHECK-RV32-NEXT:    li a4, 172
@@ -6375,9 +6238,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_185
 ; CHECK-RV32-NEXT:  .LBB61_706: # %cond.load689
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 174
 ; CHECK-RV32-NEXT:    li a4, 173
@@ -6391,9 +6253,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_186
 ; CHECK-RV32-NEXT:  .LBB61_707: # %cond.load693
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 175
 ; CHECK-RV32-NEXT:    li a4, 174
@@ -6407,9 +6268,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_187
 ; CHECK-RV32-NEXT:  .LBB61_708: # %cond.load697
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 176
 ; CHECK-RV32-NEXT:    li a4, 175
@@ -6423,9 +6283,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_188
 ; CHECK-RV32-NEXT:  .LBB61_709: # %cond.load701
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 177
 ; CHECK-RV32-NEXT:    li a4, 176
@@ -6439,9 +6298,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_189
 ; CHECK-RV32-NEXT:  .LBB61_710: # %cond.load705
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 178
 ; CHECK-RV32-NEXT:    li a4, 177
@@ -6455,9 +6313,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_190
 ; CHECK-RV32-NEXT:  .LBB61_711: # %cond.load709
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 179
 ; CHECK-RV32-NEXT:    li a4, 178
@@ -6471,9 +6328,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_191
 ; CHECK-RV32-NEXT:  .LBB61_712: # %cond.load713
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 180
 ; CHECK-RV32-NEXT:    li a4, 179
@@ -6487,9 +6343,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_192
 ; CHECK-RV32-NEXT:  .LBB61_713: # %cond.load717
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 181
 ; CHECK-RV32-NEXT:    li a4, 180
@@ -6503,9 +6358,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_193
 ; CHECK-RV32-NEXT:  .LBB61_714: # %cond.load721
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 182
 ; CHECK-RV32-NEXT:    li a4, 181
@@ -6519,9 +6373,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_194
 ; CHECK-RV32-NEXT:  .LBB61_715: # %cond.load725
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 183
 ; CHECK-RV32-NEXT:    li a4, 182
@@ -6535,9 +6388,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_195
 ; CHECK-RV32-NEXT:  .LBB61_716: # %cond.load729
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 184
 ; CHECK-RV32-NEXT:    li a4, 183
@@ -6551,9 +6403,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_196
 ; CHECK-RV32-NEXT:  .LBB61_717: # %cond.load733
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 185
 ; CHECK-RV32-NEXT:    li a4, 184
@@ -6567,9 +6418,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_197
 ; CHECK-RV32-NEXT:  .LBB61_718: # %cond.load737
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 186
 ; CHECK-RV32-NEXT:    li a4, 185
@@ -6583,9 +6433,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_198
 ; CHECK-RV32-NEXT:  .LBB61_719: # %cond.load741
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 187
 ; CHECK-RV32-NEXT:    li a4, 186
@@ -6599,9 +6448,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_199
 ; CHECK-RV32-NEXT:  .LBB61_720: # %cond.load745
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 188
 ; CHECK-RV32-NEXT:    li a4, 187
@@ -6615,9 +6463,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_200
 ; CHECK-RV32-NEXT:  .LBB61_721: # %cond.load749
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 189
 ; CHECK-RV32-NEXT:    li a4, 188
@@ -6647,9 +6494,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_206
 ; CHECK-RV32-NEXT:  .LBB61_723: # %cond.load765
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 193
 ; CHECK-RV32-NEXT:    li a4, 192
@@ -6663,9 +6509,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_207
 ; CHECK-RV32-NEXT:  .LBB61_724: # %cond.load769
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 194
 ; CHECK-RV32-NEXT:    li a4, 193
@@ -6679,9 +6524,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_208
 ; CHECK-RV32-NEXT:  .LBB61_725: # %cond.load773
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 195
 ; CHECK-RV32-NEXT:    li a4, 194
@@ -6695,9 +6539,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_209
 ; CHECK-RV32-NEXT:  .LBB61_726: # %cond.load777
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 196
 ; CHECK-RV32-NEXT:    li a4, 195
@@ -6711,9 +6554,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_210
 ; CHECK-RV32-NEXT:  .LBB61_727: # %cond.load781
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 197
 ; CHECK-RV32-NEXT:    li a4, 196
@@ -6727,9 +6569,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_211
 ; CHECK-RV32-NEXT:  .LBB61_728: # %cond.load785
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 198
 ; CHECK-RV32-NEXT:    li a4, 197
@@ -6743,9 +6584,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_212
 ; CHECK-RV32-NEXT:  .LBB61_729: # %cond.load789
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 199
 ; CHECK-RV32-NEXT:    li a4, 198
@@ -6759,9 +6599,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_213
 ; CHECK-RV32-NEXT:  .LBB61_730: # %cond.load793
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 200
 ; CHECK-RV32-NEXT:    li a4, 199
@@ -6775,9 +6614,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_214
 ; CHECK-RV32-NEXT:  .LBB61_731: # %cond.load797
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 201
 ; CHECK-RV32-NEXT:    li a4, 200
@@ -6791,9 +6629,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_215
 ; CHECK-RV32-NEXT:  .LBB61_732: # %cond.load801
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 202
 ; CHECK-RV32-NEXT:    li a4, 201
@@ -6807,9 +6644,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_216
 ; CHECK-RV32-NEXT:  .LBB61_733: # %cond.load805
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 203
 ; CHECK-RV32-NEXT:    li a4, 202
@@ -6823,9 +6659,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_217
 ; CHECK-RV32-NEXT:  .LBB61_734: # %cond.load809
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 204
 ; CHECK-RV32-NEXT:    li a4, 203
@@ -6839,9 +6674,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_218
 ; CHECK-RV32-NEXT:  .LBB61_735: # %cond.load813
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 205
 ; CHECK-RV32-NEXT:    li a4, 204
@@ -6855,9 +6689,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_219
 ; CHECK-RV32-NEXT:  .LBB61_736: # %cond.load817
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 206
 ; CHECK-RV32-NEXT:    li a4, 205
@@ -6871,9 +6704,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_220
 ; CHECK-RV32-NEXT:  .LBB61_737: # %cond.load821
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 207
 ; CHECK-RV32-NEXT:    li a4, 206
@@ -6887,9 +6719,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_221
 ; CHECK-RV32-NEXT:  .LBB61_738: # %cond.load825
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 208
 ; CHECK-RV32-NEXT:    li a4, 207
@@ -6903,9 +6734,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_222
 ; CHECK-RV32-NEXT:  .LBB61_739: # %cond.load829
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 209
 ; CHECK-RV32-NEXT:    li a4, 208
@@ -6919,9 +6749,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_223
 ; CHECK-RV32-NEXT:  .LBB61_740: # %cond.load833
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 210
 ; CHECK-RV32-NEXT:    li a4, 209
@@ -6935,9 +6764,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_224
 ; CHECK-RV32-NEXT:  .LBB61_741: # %cond.load837
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 211
 ; CHECK-RV32-NEXT:    li a4, 210
@@ -6951,9 +6779,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_225
 ; CHECK-RV32-NEXT:  .LBB61_742: # %cond.load841
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 212
 ; CHECK-RV32-NEXT:    li a4, 211
@@ -6967,9 +6794,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_226
 ; CHECK-RV32-NEXT:  .LBB61_743: # %cond.load845
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 213
 ; CHECK-RV32-NEXT:    li a4, 212
@@ -6983,9 +6809,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_227
 ; CHECK-RV32-NEXT:  .LBB61_744: # %cond.load849
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 214
 ; CHECK-RV32-NEXT:    li a4, 213
@@ -6999,9 +6824,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_228
 ; CHECK-RV32-NEXT:  .LBB61_745: # %cond.load853
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 215
 ; CHECK-RV32-NEXT:    li a4, 214
@@ -7015,9 +6839,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_229
 ; CHECK-RV32-NEXT:  .LBB61_746: # %cond.load857
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 216
 ; CHECK-RV32-NEXT:    li a4, 215
@@ -7031,9 +6854,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_230
 ; CHECK-RV32-NEXT:  .LBB61_747: # %cond.load861
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 217
 ; CHECK-RV32-NEXT:    li a4, 216
@@ -7047,9 +6869,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_231
 ; CHECK-RV32-NEXT:  .LBB61_748: # %cond.load865
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 218
 ; CHECK-RV32-NEXT:    li a4, 217
@@ -7063,9 +6884,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_232
 ; CHECK-RV32-NEXT:  .LBB61_749: # %cond.load869
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 219
 ; CHECK-RV32-NEXT:    li a4, 218
@@ -7079,9 +6899,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_233
 ; CHECK-RV32-NEXT:  .LBB61_750: # %cond.load873
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 220
 ; CHECK-RV32-NEXT:    li a4, 219
@@ -7095,9 +6914,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_234
 ; CHECK-RV32-NEXT:  .LBB61_751: # %cond.load877
 ; CHECK-RV32-NEXT:    lbu a2, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v24, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV32-NEXT:    li a2, 221
 ; CHECK-RV32-NEXT:    li a4, 220
@@ -7127,9 +6945,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_240
 ; CHECK-RV32-NEXT:  .LBB61_753: # %cond.load893
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 225
 ; CHECK-RV32-NEXT:    li a4, 224
@@ -7143,9 +6960,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_241
 ; CHECK-RV32-NEXT:  .LBB61_754: # %cond.load897
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 226
 ; CHECK-RV32-NEXT:    li a4, 225
@@ -7159,9 +6975,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_242
 ; CHECK-RV32-NEXT:  .LBB61_755: # %cond.load901
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 227
 ; CHECK-RV32-NEXT:    li a4, 226
@@ -7175,9 +6990,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_243
 ; CHECK-RV32-NEXT:  .LBB61_756: # %cond.load905
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 228
 ; CHECK-RV32-NEXT:    li a4, 227
@@ -7191,9 +7005,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_244
 ; CHECK-RV32-NEXT:  .LBB61_757: # %cond.load909
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 229
 ; CHECK-RV32-NEXT:    li a4, 228
@@ -7207,9 +7020,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_245
 ; CHECK-RV32-NEXT:  .LBB61_758: # %cond.load913
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 230
 ; CHECK-RV32-NEXT:    li a4, 229
@@ -7223,9 +7035,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_246
 ; CHECK-RV32-NEXT:  .LBB61_759: # %cond.load917
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 231
 ; CHECK-RV32-NEXT:    li a4, 230
@@ -7239,9 +7050,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_247
 ; CHECK-RV32-NEXT:  .LBB61_760: # %cond.load921
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 232
 ; CHECK-RV32-NEXT:    li a4, 231
@@ -7255,9 +7065,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_248
 ; CHECK-RV32-NEXT:  .LBB61_761: # %cond.load925
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 233
 ; CHECK-RV32-NEXT:    li a4, 232
@@ -7271,9 +7080,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_249
 ; CHECK-RV32-NEXT:  .LBB61_762: # %cond.load929
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 234
 ; CHECK-RV32-NEXT:    li a4, 233
@@ -7287,9 +7095,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_250
 ; CHECK-RV32-NEXT:  .LBB61_763: # %cond.load933
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 235
 ; CHECK-RV32-NEXT:    li a4, 234
@@ -7303,9 +7110,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_251
 ; CHECK-RV32-NEXT:  .LBB61_764: # %cond.load937
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 236
 ; CHECK-RV32-NEXT:    li a4, 235
@@ -7319,9 +7125,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_252
 ; CHECK-RV32-NEXT:  .LBB61_765: # %cond.load941
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 237
 ; CHECK-RV32-NEXT:    li a4, 236
@@ -7335,9 +7140,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_253
 ; CHECK-RV32-NEXT:  .LBB61_766: # %cond.load945
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 238
 ; CHECK-RV32-NEXT:    li a4, 237
@@ -7351,9 +7155,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_254
 ; CHECK-RV32-NEXT:  .LBB61_767: # %cond.load949
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 239
 ; CHECK-RV32-NEXT:    li a4, 238
@@ -7367,9 +7170,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_255
 ; CHECK-RV32-NEXT:  .LBB61_768: # %cond.load953
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 240
 ; CHECK-RV32-NEXT:    li a4, 239
@@ -7383,9 +7185,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_256
 ; CHECK-RV32-NEXT:  .LBB61_769: # %cond.load957
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 241
 ; CHECK-RV32-NEXT:    li a4, 240
@@ -7399,9 +7200,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_257
 ; CHECK-RV32-NEXT:  .LBB61_770: # %cond.load961
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 242
 ; CHECK-RV32-NEXT:    li a4, 241
@@ -7415,9 +7215,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_258
 ; CHECK-RV32-NEXT:  .LBB61_771: # %cond.load965
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 243
 ; CHECK-RV32-NEXT:    li a4, 242
@@ -7431,9 +7230,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_259
 ; CHECK-RV32-NEXT:  .LBB61_772: # %cond.load969
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 244
 ; CHECK-RV32-NEXT:    li a4, 243
@@ -7447,9 +7245,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_260
 ; CHECK-RV32-NEXT:  .LBB61_773: # %cond.load973
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 245
 ; CHECK-RV32-NEXT:    li a4, 244
@@ -7463,9 +7260,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_261
 ; CHECK-RV32-NEXT:  .LBB61_774: # %cond.load977
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 246
 ; CHECK-RV32-NEXT:    li a4, 245
@@ -7479,9 +7275,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_262
 ; CHECK-RV32-NEXT:  .LBB61_775: # %cond.load981
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 247
 ; CHECK-RV32-NEXT:    li a4, 246
@@ -7495,9 +7290,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_263
 ; CHECK-RV32-NEXT:  .LBB61_776: # %cond.load985
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 248
 ; CHECK-RV32-NEXT:    li a4, 247
@@ -7511,9 +7305,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_264
 ; CHECK-RV32-NEXT:  .LBB61_777: # %cond.load989
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 249
 ; CHECK-RV32-NEXT:    li a4, 248
@@ -7527,9 +7320,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_265
 ; CHECK-RV32-NEXT:  .LBB61_778: # %cond.load993
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 250
 ; CHECK-RV32-NEXT:    li a4, 249
@@ -7543,9 +7335,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_266
 ; CHECK-RV32-NEXT:  .LBB61_779: # %cond.load997
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 251
 ; CHECK-RV32-NEXT:    li a4, 250
@@ -7559,9 +7350,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_267
 ; CHECK-RV32-NEXT:  .LBB61_780: # %cond.load1001
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 252
 ; CHECK-RV32-NEXT:    li a4, 251
@@ -7575,9 +7365,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV32-NEXT:    j .LBB61_268
 ; CHECK-RV32-NEXT:  .LBB61_781: # %cond.load1005
 ; CHECK-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-RV32-NEXT:    li a4, 512
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv8r.v v16, v8
-; CHECK-RV32-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv.s.x v12, a3
 ; CHECK-RV32-NEXT:    li a3, 253
 ; CHECK-RV32-NEXT:    li a4, 252
@@ -10998,9 +10787,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    bgez a1, .LBB61_63
 ; CHECK-RV64-NEXT:  .LBB61_62: # %cond.load241
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 62
 ; CHECK-RV64-NEXT:    li a3, 61
@@ -11279,9 +11067,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    bgez a2, .LBB61_129
 ; CHECK-RV64-NEXT:  .LBB61_128: # %cond.load497
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 126
 ; CHECK-RV64-NEXT:    li a3, 125
@@ -11560,9 +11347,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    bgez a1, .LBB61_195
 ; CHECK-RV64-NEXT:  .LBB61_194: # %cond.load753
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 190
 ; CHECK-RV64-NEXT:    li a3, 189
@@ -11841,9 +11627,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    bgez a2, .LBB61_261
 ; CHECK-RV64-NEXT:  .LBB61_260: # %cond.load1009
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 254
 ; CHECK-RV64-NEXT:    li a3, 253
@@ -12968,10 +12753,9 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_2
 ; CHECK-RV64-NEXT:  .LBB61_528: # %cond.load1
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
+; CHECK-RV64-NEXT:    vsetivli zero, 2, e8, m1, tu, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
-; CHECK-RV64-NEXT:    vsetivli zero, 2, e8, m1, tu, ma
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
 ; CHECK-RV64-NEXT:    vmv1r.v v16, v8
@@ -12981,8 +12765,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_3
 ; CHECK-RV64-NEXT:  .LBB61_529: # %cond.load5
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 3, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -12993,8 +12777,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_4
 ; CHECK-RV64-NEXT:  .LBB61_530: # %cond.load9
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13005,8 +12789,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_5
 ; CHECK-RV64-NEXT:  .LBB61_531: # %cond.load13
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 5, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 4
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13017,8 +12801,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_6
 ; CHECK-RV64-NEXT:  .LBB61_532: # %cond.load17
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 6, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 5
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13029,8 +12813,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_7
 ; CHECK-RV64-NEXT:  .LBB61_533: # %cond.load21
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 7, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 6
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13041,8 +12825,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_8
 ; CHECK-RV64-NEXT:  .LBB61_534: # %cond.load25
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 7
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13053,8 +12837,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_9
 ; CHECK-RV64-NEXT:  .LBB61_535: # %cond.load29
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 9, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 8
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13065,8 +12849,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_10
 ; CHECK-RV64-NEXT:  .LBB61_536: # %cond.load33
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 10, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 9
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13077,8 +12861,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_11
 ; CHECK-RV64-NEXT:  .LBB61_537: # %cond.load37
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 11, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 10
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13089,8 +12873,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_12
 ; CHECK-RV64-NEXT:  .LBB61_538: # %cond.load41
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 12, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 11
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13101,8 +12885,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_13
 ; CHECK-RV64-NEXT:  .LBB61_539: # %cond.load45
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 13, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 12
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13113,8 +12897,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_14
 ; CHECK-RV64-NEXT:  .LBB61_540: # %cond.load49
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 14, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 13
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13125,8 +12909,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_15
 ; CHECK-RV64-NEXT:  .LBB61_541: # %cond.load53
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 15, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 14
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13137,8 +12921,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_16
 ; CHECK-RV64-NEXT:  .LBB61_542: # %cond.load57
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 16, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 15
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13149,8 +12933,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_17
 ; CHECK-RV64-NEXT:  .LBB61_543: # %cond.load61
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 17, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 16
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13161,8 +12945,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_18
 ; CHECK-RV64-NEXT:  .LBB61_544: # %cond.load65
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 18, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 17
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13173,8 +12957,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_19
 ; CHECK-RV64-NEXT:  .LBB61_545: # %cond.load69
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 19, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 18
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13185,8 +12969,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_20
 ; CHECK-RV64-NEXT:  .LBB61_546: # %cond.load73
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 20, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 19
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13197,8 +12981,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_21
 ; CHECK-RV64-NEXT:  .LBB61_547: # %cond.load77
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 21, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 20
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13209,8 +12993,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_22
 ; CHECK-RV64-NEXT:  .LBB61_548: # %cond.load81
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 22, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 21
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13221,8 +13005,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_23
 ; CHECK-RV64-NEXT:  .LBB61_549: # %cond.load85
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 23, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 22
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13233,8 +13017,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_24
 ; CHECK-RV64-NEXT:  .LBB61_550: # %cond.load89
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 24, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 23
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13245,8 +13029,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_25
 ; CHECK-RV64-NEXT:  .LBB61_551: # %cond.load93
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 25, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 24
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13257,8 +13041,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_26
 ; CHECK-RV64-NEXT:  .LBB61_552: # %cond.load97
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 26, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 25
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13269,8 +13053,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_27
 ; CHECK-RV64-NEXT:  .LBB61_553: # %cond.load101
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 27, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 26
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13281,8 +13065,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_28
 ; CHECK-RV64-NEXT:  .LBB61_554: # %cond.load105
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 28, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 27
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13293,8 +13077,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_29
 ; CHECK-RV64-NEXT:  .LBB61_555: # %cond.load109
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 29, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 28
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13305,8 +13089,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_30
 ; CHECK-RV64-NEXT:  .LBB61_556: # %cond.load113
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 30, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 29
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13317,8 +13101,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_31
 ; CHECK-RV64-NEXT:  .LBB61_557: # %cond.load117
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vsetivli zero, 31, e8, m1, tu, ma
+; CHECK-RV64-NEXT:    vmv8r.v v16, v8
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    vslideup.vi v8, v9, 30
 ; CHECK-RV64-NEXT:    addi a0, a0, 1
@@ -13329,9 +13113,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_32
 ; CHECK-RV64-NEXT:  .LBB61_558: # %cond.load121
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 32
 ; CHECK-RV64-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
@@ -13344,9 +13127,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_33
 ; CHECK-RV64-NEXT:  .LBB61_559: # %cond.load125
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 33
 ; CHECK-RV64-NEXT:    li a3, 32
@@ -13360,9 +13142,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_34
 ; CHECK-RV64-NEXT:  .LBB61_560: # %cond.load129
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 34
 ; CHECK-RV64-NEXT:    li a3, 33
@@ -13376,9 +13157,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_35
 ; CHECK-RV64-NEXT:  .LBB61_561: # %cond.load133
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 35
 ; CHECK-RV64-NEXT:    li a3, 34
@@ -13392,9 +13172,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_36
 ; CHECK-RV64-NEXT:  .LBB61_562: # %cond.load137
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 36
 ; CHECK-RV64-NEXT:    li a3, 35
@@ -13408,9 +13187,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_37
 ; CHECK-RV64-NEXT:  .LBB61_563: # %cond.load141
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 37
 ; CHECK-RV64-NEXT:    li a3, 36
@@ -13424,9 +13202,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_38
 ; CHECK-RV64-NEXT:  .LBB61_564: # %cond.load145
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 38
 ; CHECK-RV64-NEXT:    li a3, 37
@@ -13440,9 +13217,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_39
 ; CHECK-RV64-NEXT:  .LBB61_565: # %cond.load149
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 39
 ; CHECK-RV64-NEXT:    li a3, 38
@@ -13456,9 +13232,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_40
 ; CHECK-RV64-NEXT:  .LBB61_566: # %cond.load153
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 40
 ; CHECK-RV64-NEXT:    li a3, 39
@@ -13472,9 +13247,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_41
 ; CHECK-RV64-NEXT:  .LBB61_567: # %cond.load157
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 41
 ; CHECK-RV64-NEXT:    li a3, 40
@@ -13488,9 +13262,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_42
 ; CHECK-RV64-NEXT:  .LBB61_568: # %cond.load161
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 42
 ; CHECK-RV64-NEXT:    li a3, 41
@@ -13504,9 +13277,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_43
 ; CHECK-RV64-NEXT:  .LBB61_569: # %cond.load165
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 43
 ; CHECK-RV64-NEXT:    li a3, 42
@@ -13520,9 +13292,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_44
 ; CHECK-RV64-NEXT:  .LBB61_570: # %cond.load169
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 44
 ; CHECK-RV64-NEXT:    li a3, 43
@@ -13536,9 +13307,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_45
 ; CHECK-RV64-NEXT:  .LBB61_571: # %cond.load173
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 45
 ; CHECK-RV64-NEXT:    li a3, 44
@@ -13552,9 +13322,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_46
 ; CHECK-RV64-NEXT:  .LBB61_572: # %cond.load177
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 46
 ; CHECK-RV64-NEXT:    li a3, 45
@@ -13568,9 +13337,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_47
 ; CHECK-RV64-NEXT:  .LBB61_573: # %cond.load181
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 47
 ; CHECK-RV64-NEXT:    li a3, 46
@@ -13584,9 +13352,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_48
 ; CHECK-RV64-NEXT:  .LBB61_574: # %cond.load185
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 48
 ; CHECK-RV64-NEXT:    li a3, 47
@@ -13600,9 +13367,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_49
 ; CHECK-RV64-NEXT:  .LBB61_575: # %cond.load189
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 49
 ; CHECK-RV64-NEXT:    li a3, 48
@@ -13616,9 +13382,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_50
 ; CHECK-RV64-NEXT:  .LBB61_576: # %cond.load193
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 50
 ; CHECK-RV64-NEXT:    li a3, 49
@@ -13632,9 +13397,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_51
 ; CHECK-RV64-NEXT:  .LBB61_577: # %cond.load197
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 51
 ; CHECK-RV64-NEXT:    li a3, 50
@@ -13648,9 +13412,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_52
 ; CHECK-RV64-NEXT:  .LBB61_578: # %cond.load201
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 52
 ; CHECK-RV64-NEXT:    li a3, 51
@@ -13664,9 +13427,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_53
 ; CHECK-RV64-NEXT:  .LBB61_579: # %cond.load205
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 53
 ; CHECK-RV64-NEXT:    li a3, 52
@@ -13680,9 +13442,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_54
 ; CHECK-RV64-NEXT:  .LBB61_580: # %cond.load209
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 54
 ; CHECK-RV64-NEXT:    li a3, 53
@@ -13696,9 +13457,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_55
 ; CHECK-RV64-NEXT:  .LBB61_581: # %cond.load213
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 55
 ; CHECK-RV64-NEXT:    li a3, 54
@@ -13712,9 +13472,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_56
 ; CHECK-RV64-NEXT:  .LBB61_582: # %cond.load217
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 56
 ; CHECK-RV64-NEXT:    li a3, 55
@@ -13728,9 +13487,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_57
 ; CHECK-RV64-NEXT:  .LBB61_583: # %cond.load221
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 57
 ; CHECK-RV64-NEXT:    li a3, 56
@@ -13744,9 +13502,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_58
 ; CHECK-RV64-NEXT:  .LBB61_584: # %cond.load225
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 58
 ; CHECK-RV64-NEXT:    li a3, 57
@@ -13760,9 +13517,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_59
 ; CHECK-RV64-NEXT:  .LBB61_585: # %cond.load229
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 59
 ; CHECK-RV64-NEXT:    li a3, 58
@@ -13776,9 +13532,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_60
 ; CHECK-RV64-NEXT:  .LBB61_586: # %cond.load233
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 60
 ; CHECK-RV64-NEXT:    li a3, 59
@@ -13792,9 +13547,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_61
 ; CHECK-RV64-NEXT:  .LBB61_587: # %cond.load237
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v9, a1
 ; CHECK-RV64-NEXT:    li a1, 61
 ; CHECK-RV64-NEXT:    li a3, 60
@@ -13824,9 +13578,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_67
 ; CHECK-RV64-NEXT:  .LBB61_589: # %cond.load253
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 65
 ; CHECK-RV64-NEXT:    li a3, 64
@@ -13840,9 +13593,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_68
 ; CHECK-RV64-NEXT:  .LBB61_590: # %cond.load257
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 66
 ; CHECK-RV64-NEXT:    li a3, 65
@@ -13856,9 +13608,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_69
 ; CHECK-RV64-NEXT:  .LBB61_591: # %cond.load261
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 67
 ; CHECK-RV64-NEXT:    li a3, 66
@@ -13872,9 +13623,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_70
 ; CHECK-RV64-NEXT:  .LBB61_592: # %cond.load265
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 68
 ; CHECK-RV64-NEXT:    li a3, 67
@@ -13888,9 +13638,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_71
 ; CHECK-RV64-NEXT:  .LBB61_593: # %cond.load269
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 69
 ; CHECK-RV64-NEXT:    li a3, 68
@@ -13904,9 +13653,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_72
 ; CHECK-RV64-NEXT:  .LBB61_594: # %cond.load273
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 70
 ; CHECK-RV64-NEXT:    li a3, 69
@@ -13920,9 +13668,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_73
 ; CHECK-RV64-NEXT:  .LBB61_595: # %cond.load277
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 71
 ; CHECK-RV64-NEXT:    li a3, 70
@@ -13936,9 +13683,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_74
 ; CHECK-RV64-NEXT:  .LBB61_596: # %cond.load281
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 72
 ; CHECK-RV64-NEXT:    li a3, 71
@@ -13952,9 +13698,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_75
 ; CHECK-RV64-NEXT:  .LBB61_597: # %cond.load285
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 73
 ; CHECK-RV64-NEXT:    li a3, 72
@@ -13968,9 +13713,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_76
 ; CHECK-RV64-NEXT:  .LBB61_598: # %cond.load289
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 74
 ; CHECK-RV64-NEXT:    li a3, 73
@@ -13984,9 +13728,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_77
 ; CHECK-RV64-NEXT:  .LBB61_599: # %cond.load293
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 75
 ; CHECK-RV64-NEXT:    li a3, 74
@@ -14000,9 +13743,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_78
 ; CHECK-RV64-NEXT:  .LBB61_600: # %cond.load297
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 76
 ; CHECK-RV64-NEXT:    li a3, 75
@@ -14016,9 +13758,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_79
 ; CHECK-RV64-NEXT:  .LBB61_601: # %cond.load301
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 77
 ; CHECK-RV64-NEXT:    li a3, 76
@@ -14032,9 +13773,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_80
 ; CHECK-RV64-NEXT:  .LBB61_602: # %cond.load305
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 78
 ; CHECK-RV64-NEXT:    li a3, 77
@@ -14048,9 +13788,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_81
 ; CHECK-RV64-NEXT:  .LBB61_603: # %cond.load309
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 79
 ; CHECK-RV64-NEXT:    li a3, 78
@@ -14064,9 +13803,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_82
 ; CHECK-RV64-NEXT:  .LBB61_604: # %cond.load313
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 80
 ; CHECK-RV64-NEXT:    li a3, 79
@@ -14080,9 +13818,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_83
 ; CHECK-RV64-NEXT:  .LBB61_605: # %cond.load317
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 81
 ; CHECK-RV64-NEXT:    li a3, 80
@@ -14096,9 +13833,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_84
 ; CHECK-RV64-NEXT:  .LBB61_606: # %cond.load321
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 82
 ; CHECK-RV64-NEXT:    li a3, 81
@@ -14112,9 +13848,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_85
 ; CHECK-RV64-NEXT:  .LBB61_607: # %cond.load325
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 83
 ; CHECK-RV64-NEXT:    li a3, 82
@@ -14128,9 +13863,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_86
 ; CHECK-RV64-NEXT:  .LBB61_608: # %cond.load329
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 84
 ; CHECK-RV64-NEXT:    li a3, 83
@@ -14144,9 +13878,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_87
 ; CHECK-RV64-NEXT:  .LBB61_609: # %cond.load333
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 85
 ; CHECK-RV64-NEXT:    li a3, 84
@@ -14160,9 +13893,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_88
 ; CHECK-RV64-NEXT:  .LBB61_610: # %cond.load337
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 86
 ; CHECK-RV64-NEXT:    li a3, 85
@@ -14176,9 +13908,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_89
 ; CHECK-RV64-NEXT:  .LBB61_611: # %cond.load341
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 87
 ; CHECK-RV64-NEXT:    li a3, 86
@@ -14192,9 +13923,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_90
 ; CHECK-RV64-NEXT:  .LBB61_612: # %cond.load345
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 88
 ; CHECK-RV64-NEXT:    li a3, 87
@@ -14208,9 +13938,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_91
 ; CHECK-RV64-NEXT:  .LBB61_613: # %cond.load349
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 89
 ; CHECK-RV64-NEXT:    li a3, 88
@@ -14224,9 +13953,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_92
 ; CHECK-RV64-NEXT:  .LBB61_614: # %cond.load353
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 90
 ; CHECK-RV64-NEXT:    li a3, 89
@@ -14240,9 +13968,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_93
 ; CHECK-RV64-NEXT:  .LBB61_615: # %cond.load357
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 91
 ; CHECK-RV64-NEXT:    li a3, 90
@@ -14256,9 +13983,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_94
 ; CHECK-RV64-NEXT:  .LBB61_616: # %cond.load361
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 92
 ; CHECK-RV64-NEXT:    li a3, 91
@@ -14272,9 +13998,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_95
 ; CHECK-RV64-NEXT:  .LBB61_617: # %cond.load365
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 93
 ; CHECK-RV64-NEXT:    li a3, 92
@@ -14288,9 +14013,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_96
 ; CHECK-RV64-NEXT:  .LBB61_618: # %cond.load369
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 94
 ; CHECK-RV64-NEXT:    li a3, 93
@@ -14304,9 +14028,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_97
 ; CHECK-RV64-NEXT:  .LBB61_619: # %cond.load373
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 95
 ; CHECK-RV64-NEXT:    li a3, 94
@@ -14320,9 +14043,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_98
 ; CHECK-RV64-NEXT:  .LBB61_620: # %cond.load377
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 96
 ; CHECK-RV64-NEXT:    li a3, 95
@@ -14336,9 +14058,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_99
 ; CHECK-RV64-NEXT:  .LBB61_621: # %cond.load381
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 97
 ; CHECK-RV64-NEXT:    li a3, 96
@@ -14352,9 +14073,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_100
 ; CHECK-RV64-NEXT:  .LBB61_622: # %cond.load385
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 98
 ; CHECK-RV64-NEXT:    li a3, 97
@@ -14368,9 +14088,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_101
 ; CHECK-RV64-NEXT:  .LBB61_623: # %cond.load389
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 99
 ; CHECK-RV64-NEXT:    li a3, 98
@@ -14384,9 +14103,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_102
 ; CHECK-RV64-NEXT:  .LBB61_624: # %cond.load393
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 100
 ; CHECK-RV64-NEXT:    li a3, 99
@@ -14400,9 +14118,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_103
 ; CHECK-RV64-NEXT:  .LBB61_625: # %cond.load397
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 101
 ; CHECK-RV64-NEXT:    li a3, 100
@@ -14416,9 +14133,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_104
 ; CHECK-RV64-NEXT:  .LBB61_626: # %cond.load401
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 102
 ; CHECK-RV64-NEXT:    li a3, 101
@@ -14432,9 +14148,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_105
 ; CHECK-RV64-NEXT:  .LBB61_627: # %cond.load405
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 103
 ; CHECK-RV64-NEXT:    li a3, 102
@@ -14448,9 +14163,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_106
 ; CHECK-RV64-NEXT:  .LBB61_628: # %cond.load409
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 104
 ; CHECK-RV64-NEXT:    li a3, 103
@@ -14464,9 +14178,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_107
 ; CHECK-RV64-NEXT:  .LBB61_629: # %cond.load413
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 105
 ; CHECK-RV64-NEXT:    li a3, 104
@@ -14480,9 +14193,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_108
 ; CHECK-RV64-NEXT:  .LBB61_630: # %cond.load417
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 106
 ; CHECK-RV64-NEXT:    li a3, 105
@@ -14496,9 +14208,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_109
 ; CHECK-RV64-NEXT:  .LBB61_631: # %cond.load421
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 107
 ; CHECK-RV64-NEXT:    li a3, 106
@@ -14512,9 +14223,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_110
 ; CHECK-RV64-NEXT:  .LBB61_632: # %cond.load425
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 108
 ; CHECK-RV64-NEXT:    li a3, 107
@@ -14528,9 +14238,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_111
 ; CHECK-RV64-NEXT:  .LBB61_633: # %cond.load429
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 109
 ; CHECK-RV64-NEXT:    li a3, 108
@@ -14544,9 +14253,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_112
 ; CHECK-RV64-NEXT:  .LBB61_634: # %cond.load433
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 110
 ; CHECK-RV64-NEXT:    li a3, 109
@@ -14560,9 +14268,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_113
 ; CHECK-RV64-NEXT:  .LBB61_635: # %cond.load437
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 111
 ; CHECK-RV64-NEXT:    li a3, 110
@@ -14576,9 +14283,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_114
 ; CHECK-RV64-NEXT:  .LBB61_636: # %cond.load441
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 112
 ; CHECK-RV64-NEXT:    li a3, 111
@@ -14592,9 +14298,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_115
 ; CHECK-RV64-NEXT:  .LBB61_637: # %cond.load445
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 113
 ; CHECK-RV64-NEXT:    li a3, 112
@@ -14608,9 +14313,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_116
 ; CHECK-RV64-NEXT:  .LBB61_638: # %cond.load449
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 114
 ; CHECK-RV64-NEXT:    li a3, 113
@@ -14624,9 +14328,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_117
 ; CHECK-RV64-NEXT:  .LBB61_639: # %cond.load453
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 115
 ; CHECK-RV64-NEXT:    li a3, 114
@@ -14640,9 +14343,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_118
 ; CHECK-RV64-NEXT:  .LBB61_640: # %cond.load457
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 116
 ; CHECK-RV64-NEXT:    li a3, 115
@@ -14656,9 +14358,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_119
 ; CHECK-RV64-NEXT:  .LBB61_641: # %cond.load461
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 117
 ; CHECK-RV64-NEXT:    li a3, 116
@@ -14672,9 +14373,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_120
 ; CHECK-RV64-NEXT:  .LBB61_642: # %cond.load465
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 118
 ; CHECK-RV64-NEXT:    li a3, 117
@@ -14688,9 +14388,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_121
 ; CHECK-RV64-NEXT:  .LBB61_643: # %cond.load469
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 119
 ; CHECK-RV64-NEXT:    li a3, 118
@@ -14704,9 +14403,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_122
 ; CHECK-RV64-NEXT:  .LBB61_644: # %cond.load473
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 120
 ; CHECK-RV64-NEXT:    li a3, 119
@@ -14720,9 +14418,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_123
 ; CHECK-RV64-NEXT:  .LBB61_645: # %cond.load477
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 121
 ; CHECK-RV64-NEXT:    li a3, 120
@@ -14736,9 +14433,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_124
 ; CHECK-RV64-NEXT:  .LBB61_646: # %cond.load481
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 122
 ; CHECK-RV64-NEXT:    li a3, 121
@@ -14752,9 +14448,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_125
 ; CHECK-RV64-NEXT:  .LBB61_647: # %cond.load485
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 123
 ; CHECK-RV64-NEXT:    li a3, 122
@@ -14768,9 +14463,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_126
 ; CHECK-RV64-NEXT:  .LBB61_648: # %cond.load489
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 124
 ; CHECK-RV64-NEXT:    li a3, 123
@@ -14784,9 +14478,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_127
 ; CHECK-RV64-NEXT:  .LBB61_649: # %cond.load493
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v10, a2
 ; CHECK-RV64-NEXT:    li a2, 125
 ; CHECK-RV64-NEXT:    li a3, 124
@@ -14816,9 +14509,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_133
 ; CHECK-RV64-NEXT:  .LBB61_651: # %cond.load509
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 129
 ; CHECK-RV64-NEXT:    li a3, 128
@@ -14832,9 +14524,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_134
 ; CHECK-RV64-NEXT:  .LBB61_652: # %cond.load513
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 130
 ; CHECK-RV64-NEXT:    li a3, 129
@@ -14848,9 +14539,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_135
 ; CHECK-RV64-NEXT:  .LBB61_653: # %cond.load517
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 131
 ; CHECK-RV64-NEXT:    li a3, 130
@@ -14864,9 +14554,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_136
 ; CHECK-RV64-NEXT:  .LBB61_654: # %cond.load521
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 132
 ; CHECK-RV64-NEXT:    li a3, 131
@@ -14880,9 +14569,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_137
 ; CHECK-RV64-NEXT:  .LBB61_655: # %cond.load525
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 133
 ; CHECK-RV64-NEXT:    li a3, 132
@@ -14896,9 +14584,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_138
 ; CHECK-RV64-NEXT:  .LBB61_656: # %cond.load529
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 134
 ; CHECK-RV64-NEXT:    li a3, 133
@@ -14912,9 +14599,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_139
 ; CHECK-RV64-NEXT:  .LBB61_657: # %cond.load533
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 135
 ; CHECK-RV64-NEXT:    li a3, 134
@@ -14928,9 +14614,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_140
 ; CHECK-RV64-NEXT:  .LBB61_658: # %cond.load537
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 136
 ; CHECK-RV64-NEXT:    li a3, 135
@@ -14944,9 +14629,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_141
 ; CHECK-RV64-NEXT:  .LBB61_659: # %cond.load541
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 137
 ; CHECK-RV64-NEXT:    li a3, 136
@@ -14960,9 +14644,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_142
 ; CHECK-RV64-NEXT:  .LBB61_660: # %cond.load545
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 138
 ; CHECK-RV64-NEXT:    li a3, 137
@@ -14976,9 +14659,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_143
 ; CHECK-RV64-NEXT:  .LBB61_661: # %cond.load549
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 139
 ; CHECK-RV64-NEXT:    li a3, 138
@@ -14992,9 +14674,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_144
 ; CHECK-RV64-NEXT:  .LBB61_662: # %cond.load553
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 140
 ; CHECK-RV64-NEXT:    li a3, 139
@@ -15008,9 +14689,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_145
 ; CHECK-RV64-NEXT:  .LBB61_663: # %cond.load557
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 141
 ; CHECK-RV64-NEXT:    li a3, 140
@@ -15024,9 +14704,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_146
 ; CHECK-RV64-NEXT:  .LBB61_664: # %cond.load561
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 142
 ; CHECK-RV64-NEXT:    li a3, 141
@@ -15040,9 +14719,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_147
 ; CHECK-RV64-NEXT:  .LBB61_665: # %cond.load565
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 143
 ; CHECK-RV64-NEXT:    li a3, 142
@@ -15056,9 +14734,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_148
 ; CHECK-RV64-NEXT:  .LBB61_666: # %cond.load569
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 144
 ; CHECK-RV64-NEXT:    li a3, 143
@@ -15072,9 +14749,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_149
 ; CHECK-RV64-NEXT:  .LBB61_667: # %cond.load573
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 145
 ; CHECK-RV64-NEXT:    li a3, 144
@@ -15088,9 +14764,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_150
 ; CHECK-RV64-NEXT:  .LBB61_668: # %cond.load577
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 146
 ; CHECK-RV64-NEXT:    li a3, 145
@@ -15104,9 +14779,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_151
 ; CHECK-RV64-NEXT:  .LBB61_669: # %cond.load581
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 147
 ; CHECK-RV64-NEXT:    li a3, 146
@@ -15120,9 +14794,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_152
 ; CHECK-RV64-NEXT:  .LBB61_670: # %cond.load585
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 148
 ; CHECK-RV64-NEXT:    li a3, 147
@@ -15136,9 +14809,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_153
 ; CHECK-RV64-NEXT:  .LBB61_671: # %cond.load589
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 149
 ; CHECK-RV64-NEXT:    li a3, 148
@@ -15152,9 +14824,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_154
 ; CHECK-RV64-NEXT:  .LBB61_672: # %cond.load593
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 150
 ; CHECK-RV64-NEXT:    li a3, 149
@@ -15168,9 +14839,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_155
 ; CHECK-RV64-NEXT:  .LBB61_673: # %cond.load597
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 151
 ; CHECK-RV64-NEXT:    li a3, 150
@@ -15184,9 +14854,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_156
 ; CHECK-RV64-NEXT:  .LBB61_674: # %cond.load601
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 152
 ; CHECK-RV64-NEXT:    li a3, 151
@@ -15200,9 +14869,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_157
 ; CHECK-RV64-NEXT:  .LBB61_675: # %cond.load605
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 153
 ; CHECK-RV64-NEXT:    li a3, 152
@@ -15216,9 +14884,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_158
 ; CHECK-RV64-NEXT:  .LBB61_676: # %cond.load609
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 154
 ; CHECK-RV64-NEXT:    li a3, 153
@@ -15232,9 +14899,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_159
 ; CHECK-RV64-NEXT:  .LBB61_677: # %cond.load613
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 155
 ; CHECK-RV64-NEXT:    li a3, 154
@@ -15248,9 +14914,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_160
 ; CHECK-RV64-NEXT:  .LBB61_678: # %cond.load617
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 156
 ; CHECK-RV64-NEXT:    li a3, 155
@@ -15264,9 +14929,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_161
 ; CHECK-RV64-NEXT:  .LBB61_679: # %cond.load621
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 157
 ; CHECK-RV64-NEXT:    li a3, 156
@@ -15280,9 +14944,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_162
 ; CHECK-RV64-NEXT:  .LBB61_680: # %cond.load625
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 158
 ; CHECK-RV64-NEXT:    li a3, 157
@@ -15296,9 +14959,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_163
 ; CHECK-RV64-NEXT:  .LBB61_681: # %cond.load629
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 159
 ; CHECK-RV64-NEXT:    li a3, 158
@@ -15312,9 +14974,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_164
 ; CHECK-RV64-NEXT:  .LBB61_682: # %cond.load633
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 160
 ; CHECK-RV64-NEXT:    li a3, 159
@@ -15328,9 +14989,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_165
 ; CHECK-RV64-NEXT:  .LBB61_683: # %cond.load637
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 161
 ; CHECK-RV64-NEXT:    li a3, 160
@@ -15344,9 +15004,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_166
 ; CHECK-RV64-NEXT:  .LBB61_684: # %cond.load641
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 162
 ; CHECK-RV64-NEXT:    li a3, 161
@@ -15360,9 +15019,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_167
 ; CHECK-RV64-NEXT:  .LBB61_685: # %cond.load645
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 163
 ; CHECK-RV64-NEXT:    li a3, 162
@@ -15376,9 +15034,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_168
 ; CHECK-RV64-NEXT:  .LBB61_686: # %cond.load649
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 164
 ; CHECK-RV64-NEXT:    li a3, 163
@@ -15392,9 +15049,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_169
 ; CHECK-RV64-NEXT:  .LBB61_687: # %cond.load653
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 165
 ; CHECK-RV64-NEXT:    li a3, 164
@@ -15408,9 +15064,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_170
 ; CHECK-RV64-NEXT:  .LBB61_688: # %cond.load657
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 166
 ; CHECK-RV64-NEXT:    li a3, 165
@@ -15424,9 +15079,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_171
 ; CHECK-RV64-NEXT:  .LBB61_689: # %cond.load661
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 167
 ; CHECK-RV64-NEXT:    li a3, 166
@@ -15440,9 +15094,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_172
 ; CHECK-RV64-NEXT:  .LBB61_690: # %cond.load665
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 168
 ; CHECK-RV64-NEXT:    li a3, 167
@@ -15456,9 +15109,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_173
 ; CHECK-RV64-NEXT:  .LBB61_691: # %cond.load669
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 169
 ; CHECK-RV64-NEXT:    li a3, 168
@@ -15472,9 +15124,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_174
 ; CHECK-RV64-NEXT:  .LBB61_692: # %cond.load673
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 170
 ; CHECK-RV64-NEXT:    li a3, 169
@@ -15488,9 +15139,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_175
 ; CHECK-RV64-NEXT:  .LBB61_693: # %cond.load677
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 171
 ; CHECK-RV64-NEXT:    li a3, 170
@@ -15504,9 +15154,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_176
 ; CHECK-RV64-NEXT:  .LBB61_694: # %cond.load681
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 172
 ; CHECK-RV64-NEXT:    li a3, 171
@@ -15520,9 +15169,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_177
 ; CHECK-RV64-NEXT:  .LBB61_695: # %cond.load685
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 173
 ; CHECK-RV64-NEXT:    li a3, 172
@@ -15536,9 +15184,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_178
 ; CHECK-RV64-NEXT:  .LBB61_696: # %cond.load689
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 174
 ; CHECK-RV64-NEXT:    li a3, 173
@@ -15552,9 +15199,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_179
 ; CHECK-RV64-NEXT:  .LBB61_697: # %cond.load693
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 175
 ; CHECK-RV64-NEXT:    li a3, 174
@@ -15568,9 +15214,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_180
 ; CHECK-RV64-NEXT:  .LBB61_698: # %cond.load697
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 176
 ; CHECK-RV64-NEXT:    li a3, 175
@@ -15584,9 +15229,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_181
 ; CHECK-RV64-NEXT:  .LBB61_699: # %cond.load701
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 177
 ; CHECK-RV64-NEXT:    li a3, 176
@@ -15600,9 +15244,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_182
 ; CHECK-RV64-NEXT:  .LBB61_700: # %cond.load705
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 178
 ; CHECK-RV64-NEXT:    li a3, 177
@@ -15616,9 +15259,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_183
 ; CHECK-RV64-NEXT:  .LBB61_701: # %cond.load709
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 179
 ; CHECK-RV64-NEXT:    li a3, 178
@@ -15632,9 +15274,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_184
 ; CHECK-RV64-NEXT:  .LBB61_702: # %cond.load713
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 180
 ; CHECK-RV64-NEXT:    li a3, 179
@@ -15648,9 +15289,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_185
 ; CHECK-RV64-NEXT:  .LBB61_703: # %cond.load717
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 181
 ; CHECK-RV64-NEXT:    li a3, 180
@@ -15664,9 +15304,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_186
 ; CHECK-RV64-NEXT:  .LBB61_704: # %cond.load721
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 182
 ; CHECK-RV64-NEXT:    li a3, 181
@@ -15680,9 +15319,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_187
 ; CHECK-RV64-NEXT:  .LBB61_705: # %cond.load725
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 183
 ; CHECK-RV64-NEXT:    li a3, 182
@@ -15696,9 +15334,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_188
 ; CHECK-RV64-NEXT:  .LBB61_706: # %cond.load729
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 184
 ; CHECK-RV64-NEXT:    li a3, 183
@@ -15712,9 +15349,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_189
 ; CHECK-RV64-NEXT:  .LBB61_707: # %cond.load733
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 185
 ; CHECK-RV64-NEXT:    li a3, 184
@@ -15728,9 +15364,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_190
 ; CHECK-RV64-NEXT:  .LBB61_708: # %cond.load737
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 186
 ; CHECK-RV64-NEXT:    li a3, 185
@@ -15744,9 +15379,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_191
 ; CHECK-RV64-NEXT:  .LBB61_709: # %cond.load741
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 187
 ; CHECK-RV64-NEXT:    li a3, 186
@@ -15760,9 +15394,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_192
 ; CHECK-RV64-NEXT:  .LBB61_710: # %cond.load745
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 188
 ; CHECK-RV64-NEXT:    li a3, 187
@@ -15776,9 +15409,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_193
 ; CHECK-RV64-NEXT:  .LBB61_711: # %cond.load749
 ; CHECK-RV64-NEXT:    lbu a1, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a1
 ; CHECK-RV64-NEXT:    li a1, 189
 ; CHECK-RV64-NEXT:    li a3, 188
@@ -15808,9 +15440,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_199
 ; CHECK-RV64-NEXT:  .LBB61_713: # %cond.load765
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 193
 ; CHECK-RV64-NEXT:    li a3, 192
@@ -15824,9 +15455,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_200
 ; CHECK-RV64-NEXT:  .LBB61_714: # %cond.load769
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 194
 ; CHECK-RV64-NEXT:    li a3, 193
@@ -15840,9 +15470,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_201
 ; CHECK-RV64-NEXT:  .LBB61_715: # %cond.load773
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 195
 ; CHECK-RV64-NEXT:    li a3, 194
@@ -15856,9 +15485,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_202
 ; CHECK-RV64-NEXT:  .LBB61_716: # %cond.load777
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 196
 ; CHECK-RV64-NEXT:    li a3, 195
@@ -15872,9 +15500,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_203
 ; CHECK-RV64-NEXT:  .LBB61_717: # %cond.load781
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 197
 ; CHECK-RV64-NEXT:    li a3, 196
@@ -15888,9 +15515,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_204
 ; CHECK-RV64-NEXT:  .LBB61_718: # %cond.load785
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 198
 ; CHECK-RV64-NEXT:    li a3, 197
@@ -15904,9 +15530,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_205
 ; CHECK-RV64-NEXT:  .LBB61_719: # %cond.load789
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 199
 ; CHECK-RV64-NEXT:    li a3, 198
@@ -15920,9 +15545,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_206
 ; CHECK-RV64-NEXT:  .LBB61_720: # %cond.load793
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 200
 ; CHECK-RV64-NEXT:    li a3, 199
@@ -15936,9 +15560,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_207
 ; CHECK-RV64-NEXT:  .LBB61_721: # %cond.load797
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 201
 ; CHECK-RV64-NEXT:    li a3, 200
@@ -15952,9 +15575,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_208
 ; CHECK-RV64-NEXT:  .LBB61_722: # %cond.load801
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 202
 ; CHECK-RV64-NEXT:    li a3, 201
@@ -15968,9 +15590,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_209
 ; CHECK-RV64-NEXT:  .LBB61_723: # %cond.load805
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 203
 ; CHECK-RV64-NEXT:    li a3, 202
@@ -15984,9 +15605,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_210
 ; CHECK-RV64-NEXT:  .LBB61_724: # %cond.load809
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 204
 ; CHECK-RV64-NEXT:    li a3, 203
@@ -16000,9 +15620,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_211
 ; CHECK-RV64-NEXT:  .LBB61_725: # %cond.load813
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 205
 ; CHECK-RV64-NEXT:    li a3, 204
@@ -16016,9 +15635,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_212
 ; CHECK-RV64-NEXT:  .LBB61_726: # %cond.load817
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 206
 ; CHECK-RV64-NEXT:    li a3, 205
@@ -16032,9 +15650,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_213
 ; CHECK-RV64-NEXT:  .LBB61_727: # %cond.load821
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 207
 ; CHECK-RV64-NEXT:    li a3, 206
@@ -16048,9 +15665,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_214
 ; CHECK-RV64-NEXT:  .LBB61_728: # %cond.load825
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 208
 ; CHECK-RV64-NEXT:    li a3, 207
@@ -16064,9 +15680,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_215
 ; CHECK-RV64-NEXT:  .LBB61_729: # %cond.load829
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 209
 ; CHECK-RV64-NEXT:    li a3, 208
@@ -16080,9 +15695,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_216
 ; CHECK-RV64-NEXT:  .LBB61_730: # %cond.load833
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 210
 ; CHECK-RV64-NEXT:    li a3, 209
@@ -16096,9 +15710,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_217
 ; CHECK-RV64-NEXT:  .LBB61_731: # %cond.load837
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 211
 ; CHECK-RV64-NEXT:    li a3, 210
@@ -16112,9 +15725,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_218
 ; CHECK-RV64-NEXT:  .LBB61_732: # %cond.load841
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 212
 ; CHECK-RV64-NEXT:    li a3, 211
@@ -16128,9 +15740,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_219
 ; CHECK-RV64-NEXT:  .LBB61_733: # %cond.load845
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 213
 ; CHECK-RV64-NEXT:    li a3, 212
@@ -16144,9 +15755,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_220
 ; CHECK-RV64-NEXT:  .LBB61_734: # %cond.load849
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 214
 ; CHECK-RV64-NEXT:    li a3, 213
@@ -16160,9 +15770,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_221
 ; CHECK-RV64-NEXT:  .LBB61_735: # %cond.load853
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 215
 ; CHECK-RV64-NEXT:    li a3, 214
@@ -16176,9 +15785,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_222
 ; CHECK-RV64-NEXT:  .LBB61_736: # %cond.load857
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 216
 ; CHECK-RV64-NEXT:    li a3, 215
@@ -16192,9 +15800,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_223
 ; CHECK-RV64-NEXT:  .LBB61_737: # %cond.load861
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 217
 ; CHECK-RV64-NEXT:    li a3, 216
@@ -16208,9 +15815,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_224
 ; CHECK-RV64-NEXT:  .LBB61_738: # %cond.load865
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 218
 ; CHECK-RV64-NEXT:    li a3, 217
@@ -16224,9 +15830,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_225
 ; CHECK-RV64-NEXT:  .LBB61_739: # %cond.load869
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 219
 ; CHECK-RV64-NEXT:    li a3, 218
@@ -16240,9 +15845,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_226
 ; CHECK-RV64-NEXT:  .LBB61_740: # %cond.load873
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 220
 ; CHECK-RV64-NEXT:    li a3, 219
@@ -16256,9 +15860,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_227
 ; CHECK-RV64-NEXT:  .LBB61_741: # %cond.load877
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 221
 ; CHECK-RV64-NEXT:    li a3, 220
@@ -16272,9 +15875,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_228
 ; CHECK-RV64-NEXT:  .LBB61_742: # %cond.load881
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 222
 ; CHECK-RV64-NEXT:    li a3, 221
@@ -16288,9 +15890,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_229
 ; CHECK-RV64-NEXT:  .LBB61_743: # %cond.load885
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 223
 ; CHECK-RV64-NEXT:    li a3, 222
@@ -16304,9 +15905,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_230
 ; CHECK-RV64-NEXT:  .LBB61_744: # %cond.load889
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 224
 ; CHECK-RV64-NEXT:    li a3, 223
@@ -16320,9 +15920,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_231
 ; CHECK-RV64-NEXT:  .LBB61_745: # %cond.load893
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 225
 ; CHECK-RV64-NEXT:    li a3, 224
@@ -16336,9 +15935,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_232
 ; CHECK-RV64-NEXT:  .LBB61_746: # %cond.load897
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 226
 ; CHECK-RV64-NEXT:    li a3, 225
@@ -16352,9 +15950,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_233
 ; CHECK-RV64-NEXT:  .LBB61_747: # %cond.load901
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 227
 ; CHECK-RV64-NEXT:    li a3, 226
@@ -16368,9 +15965,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_234
 ; CHECK-RV64-NEXT:  .LBB61_748: # %cond.load905
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 228
 ; CHECK-RV64-NEXT:    li a3, 227
@@ -16384,9 +15980,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_235
 ; CHECK-RV64-NEXT:  .LBB61_749: # %cond.load909
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 229
 ; CHECK-RV64-NEXT:    li a3, 228
@@ -16400,9 +15995,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_236
 ; CHECK-RV64-NEXT:  .LBB61_750: # %cond.load913
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 230
 ; CHECK-RV64-NEXT:    li a3, 229
@@ -16416,9 +16010,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_237
 ; CHECK-RV64-NEXT:  .LBB61_751: # %cond.load917
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 231
 ; CHECK-RV64-NEXT:    li a3, 230
@@ -16432,9 +16025,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_238
 ; CHECK-RV64-NEXT:  .LBB61_752: # %cond.load921
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 232
 ; CHECK-RV64-NEXT:    li a3, 231
@@ -16448,9 +16040,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_239
 ; CHECK-RV64-NEXT:  .LBB61_753: # %cond.load925
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 233
 ; CHECK-RV64-NEXT:    li a3, 232
@@ -16464,9 +16055,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_240
 ; CHECK-RV64-NEXT:  .LBB61_754: # %cond.load929
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 234
 ; CHECK-RV64-NEXT:    li a3, 233
@@ -16480,9 +16070,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_241
 ; CHECK-RV64-NEXT:  .LBB61_755: # %cond.load933
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 235
 ; CHECK-RV64-NEXT:    li a3, 234
@@ -16496,9 +16085,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_242
 ; CHECK-RV64-NEXT:  .LBB61_756: # %cond.load937
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 236
 ; CHECK-RV64-NEXT:    li a3, 235
@@ -16512,9 +16100,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_243
 ; CHECK-RV64-NEXT:  .LBB61_757: # %cond.load941
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 237
 ; CHECK-RV64-NEXT:    li a3, 236
@@ -16528,9 +16115,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_244
 ; CHECK-RV64-NEXT:  .LBB61_758: # %cond.load945
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 238
 ; CHECK-RV64-NEXT:    li a3, 237
@@ -16544,9 +16130,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_245
 ; CHECK-RV64-NEXT:  .LBB61_759: # %cond.load949
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 239
 ; CHECK-RV64-NEXT:    li a3, 238
@@ -16560,9 +16145,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_246
 ; CHECK-RV64-NEXT:  .LBB61_760: # %cond.load953
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 240
 ; CHECK-RV64-NEXT:    li a3, 239
@@ -16576,9 +16160,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_247
 ; CHECK-RV64-NEXT:  .LBB61_761: # %cond.load957
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 241
 ; CHECK-RV64-NEXT:    li a3, 240
@@ -16592,9 +16175,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_248
 ; CHECK-RV64-NEXT:  .LBB61_762: # %cond.load961
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 242
 ; CHECK-RV64-NEXT:    li a3, 241
@@ -16608,9 +16190,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_249
 ; CHECK-RV64-NEXT:  .LBB61_763: # %cond.load965
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 243
 ; CHECK-RV64-NEXT:    li a3, 242
@@ -16624,9 +16205,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_250
 ; CHECK-RV64-NEXT:  .LBB61_764: # %cond.load969
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 244
 ; CHECK-RV64-NEXT:    li a3, 243
@@ -16640,9 +16220,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_251
 ; CHECK-RV64-NEXT:  .LBB61_765: # %cond.load973
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 245
 ; CHECK-RV64-NEXT:    li a3, 244
@@ -16656,9 +16235,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_252
 ; CHECK-RV64-NEXT:  .LBB61_766: # %cond.load977
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 246
 ; CHECK-RV64-NEXT:    li a3, 245
@@ -16672,9 +16250,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_253
 ; CHECK-RV64-NEXT:  .LBB61_767: # %cond.load981
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 247
 ; CHECK-RV64-NEXT:    li a3, 246
@@ -16688,9 +16265,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_254
 ; CHECK-RV64-NEXT:  .LBB61_768: # %cond.load985
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 248
 ; CHECK-RV64-NEXT:    li a3, 247
@@ -16704,9 +16280,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_255
 ; CHECK-RV64-NEXT:  .LBB61_769: # %cond.load989
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 249
 ; CHECK-RV64-NEXT:    li a3, 248
@@ -16720,9 +16295,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_256
 ; CHECK-RV64-NEXT:  .LBB61_770: # %cond.load993
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 250
 ; CHECK-RV64-NEXT:    li a3, 249
@@ -16736,9 +16310,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_257
 ; CHECK-RV64-NEXT:  .LBB61_771: # %cond.load997
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 251
 ; CHECK-RV64-NEXT:    li a3, 250
@@ -16752,9 +16325,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_258
 ; CHECK-RV64-NEXT:  .LBB61_772: # %cond.load1001
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 252
 ; CHECK-RV64-NEXT:    li a3, 251
@@ -16768,9 +16340,8 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
 ; CHECK-RV64-NEXT:    j .LBB61_259
 ; CHECK-RV64-NEXT:  .LBB61_773: # %cond.load1005
 ; CHECK-RV64-NEXT:    lbu a2, 0(a0)
-; CHECK-RV64-NEXT:    li a3, 512
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv8r.v v16, v8
-; CHECK-RV64-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv.s.x v12, a2
 ; CHECK-RV64-NEXT:    li a2, 253
 ; CHECK-RV64-NEXT:    li a3, 252
diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
index 869478a1efa78..83637e4a71d45 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
@@ -13,6 +13,7 @@ define <vscale x 4 x i32> @extract_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec) {
 define <vscale x 4 x i32> @extract_nxv8i32_nxv4i32_4(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv4i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
   %c = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> %vec, i64 4)
@@ -30,6 +31,7 @@ define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_0(<vscale x 8 x i32> %vec) {
 define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_2(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv2i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, i64 2)
@@ -39,6 +41,7 @@ define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_2(<vscale x 8 x i32> %vec) {
 define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_4(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv2i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, i64 4)
@@ -48,6 +51,7 @@ define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_4(<vscale x 8 x i32> %vec) {
 define <vscale x 2 x i32> @extract_nxv8i32_nxv2i32_6(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv2i32_6:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v11
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, i64 6)
@@ -65,6 +69,7 @@ define <vscale x 8 x i32> @extract_nxv16i32_nxv8i32_0(<vscale x 16 x i32> %vec)
 define <vscale x 8 x i32> @extract_nxv16i32_nxv8i32_8(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv8i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
   %c = call <vscale x 8 x i32> @llvm.vector.extract.nxv8i32.nxv16i32(<vscale x 16 x i32> %vec, i64 8)
@@ -82,6 +87,7 @@ define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_0(<vscale x 16 x i32> %vec)
 define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_4(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv4i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
   %c = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, i64 4)
@@ -91,6 +97,7 @@ define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_4(<vscale x 16 x i32> %vec)
 define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_8(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv4i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
   %c = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, i64 8)
@@ -100,6 +107,7 @@ define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_8(<vscale x 16 x i32> %vec)
 define <vscale x 4 x i32> @extract_nxv16i32_nxv4i32_12(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv4i32_12:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v14
 ; CHECK-NEXT:    ret
   %c = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, i64 12)
@@ -117,6 +125,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_0(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_2(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 2)
@@ -126,6 +135,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_2(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_4(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 4)
@@ -135,6 +145,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_4(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_6(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_6:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v11
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 6)
@@ -144,6 +155,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_6(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_8(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v12
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 8)
@@ -153,6 +165,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_8(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_10(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_10:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v13
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 10)
@@ -162,6 +175,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_10(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_12(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_12:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v14
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 12)
@@ -171,6 +185,7 @@ define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_12(<vscale x 16 x i32> %vec)
 define <vscale x 2 x i32> @extract_nxv16i32_nxv2i32_14(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv2i32_14:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v15
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i32> @llvm.vector.extract.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 14)
@@ -224,6 +239,7 @@ define <vscale x 1 x i32> @extract_nxv16i32_nxv1i32_15(<vscale x 16 x i32> %vec)
 define <vscale x 1 x i32> @extract_nxv16i32_nxv1i32_2(<vscale x 16 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv16i32_nxv1i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv16i32(<vscale x 16 x i32> %vec, i64 2)
@@ -287,6 +303,7 @@ define <vscale x 2 x i8> @extract_nxv32i8_nxv2i8_6(<vscale x 32 x i8> %vec) {
 define <vscale x 2 x i8> @extract_nxv32i8_nxv2i8_8(<vscale x 32 x i8> %vec) {
 ; CHECK-LABEL: extract_nxv32i8_nxv2i8_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x i8> @llvm.vector.extract.nxv2i8.nxv32i8(<vscale x 32 x i8> %vec, i64 8)
@@ -357,6 +374,7 @@ define <vscale x 2 x half> @extract_nxv2f16_nxv16f16_2(<vscale x 16 x half> %vec
 define <vscale x 2 x half> @extract_nxv2f16_nxv16f16_4(<vscale x 16 x half> %vec) {
 ; CHECK-LABEL: extract_nxv2f16_nxv16f16_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x half> @llvm.vector.extract.nxv2f16.nxv16f16(<vscale x 16 x half> %vec, i64 4)
@@ -504,6 +522,7 @@ define <vscale x 2 x bfloat> @extract_nxv2bf16_nxv16bf16_2(<vscale x 16 x bfloat
 define <vscale x 2 x bfloat> @extract_nxv2bf16_nxv16bf16_4(<vscale x 16 x bfloat> %vec) {
 ; CHECK-LABEL: extract_nxv2bf16_nxv16bf16_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %c = call <vscale x 2 x bfloat> @llvm.vector.extract.nxv2bf16.nxv16bf16(<vscale x 16 x bfloat> %vec, i64 4)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-i8-index-cornercase.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-i8-index-cornercase.ll
index ce83e2d8a6220..1752dfd50d0c5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-i8-index-cornercase.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-i8-index-cornercase.ll
@@ -16,10 +16,10 @@ define <512 x i8> @single_source(<512 x i8> %a) {
 ; CHECK-NEXT:    addi s0, sp, 1536
 ; CHECK-NEXT:    .cfi_def_cfa s0, 0
 ; CHECK-NEXT:    andi sp, sp, -512
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    li a0, 512
 ; CHECK-NEXT:    addi a1, sp, 512
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.x.s a2, v16
 ; CHECK-NEXT:    vslidedown.vi v24, v16, 5
 ; CHECK-NEXT:    li a3, 432
@@ -103,11 +103,16 @@ define <512 x i8> @two_source(<512 x i8> %a, <512 x i8> %b) {
 ; CHECK-NEXT:    .cfi_offset s0, -16
 ; CHECK-NEXT:    addi s0, sp, 1536
 ; CHECK-NEXT:    .cfi_def_cfa s0, 0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    andi sp, sp, -512
+; CHECK-NEXT:    addi a0, sp, 1520
+; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a0, 512
 ; CHECK-NEXT:    addi a1, sp, 512
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v0, v24, 5
 ; CHECK-NEXT:    vmv.x.s a2, v24
 ; CHECK-NEXT:    li a3, 432
@@ -132,32 +137,22 @@ define <512 x i8> @two_source(<512 x i8> %a, <512 x i8> %b) {
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m8, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v24, a1
 ; CHECK-NEXT:    li a1, 501
-; CHECK-NEXT:    vmv.s.x v24, a3
+; CHECK-NEXT:    lui a2, %hi(.LCPI2_1)
+; CHECK-NEXT:    addi a2, a2, %lo(.LCPI2_1)
+; CHECK-NEXT:    vsetivli zero, 8, e64, m1, ta, ma
+; CHECK-NEXT:    vle64.v v0, (a2)
 ; CHECK-NEXT:    li a2, 500
+; CHECK-NEXT:    vmv.s.x v24, a3
+; CHECK-NEXT:    lui a3, %hi(.LCPI2_0)
+; CHECK-NEXT:    addi a3, a3, %lo(.LCPI2_0)
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vle8.v v16, (a3)
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v24, a2
-; CHECK-NEXT:    lui a1, 2761
-; CHECK-NEXT:    vsetivli zero, 8, e64, m1, ta, ma
-; CHECK-NEXT:    vmv.v.i v24, 0
-; CHECK-NEXT:    lui a2, 4
-; CHECK-NEXT:    vmv.s.x v25, a2
-; CHECK-NEXT:    lui a2, 1047552
-; CHECK-NEXT:    addi a2, a2, 1
-; CHECK-NEXT:    slli a2, a2, 23
-; CHECK-NEXT:    addi a2, a2, 1
-; CHECK-NEXT:    slli a2, a2, 18
-; CHECK-NEXT:    vslide1down.vx v0, v24, a2
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    slli a1, a1, 25
-; CHECK-NEXT:    addi a1, a1, 501
-; CHECK-NEXT:    slli a1, a1, 13
-; CHECK-NEXT:    addi a1, a1, 512
-; CHECK-NEXT:    vsetivli zero, 7, e64, m1, tu, ma
-; CHECK-NEXT:    vslideup.vi v0, v25, 6
-; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    vmv.v.x v24, a1
+; CHECK-NEXT:    addi a1, sp, 1520
+; CHECK-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
-; CHECK-NEXT:    vrgather.vv v8, v16, v24, v0.t
+; CHECK-NEXT:    vrgather.vv v8, v24, v16, v0.t
 ; CHECK-NEXT:    addi sp, s0, -1536
 ; CHECK-NEXT:    .cfi_def_cfa sp, 1536
 ; CHECK-NEXT:    ld ra, 1528(sp) # 8-byte Folded Reload
@@ -167,7 +162,7 @@ define <512 x i8> @two_source(<512 x i8> %a, <512 x i8> %b) {
 ; CHECK-NEXT:    addi sp, sp, 1536
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
-  %res = shufflevector <512 x i8> %a, <512 x i8> %b, <512 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 548, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 4, i32 574, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 473, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 674, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 500, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 555>
+  %res = shufflevector <512 x i8> %a, <512 x i8> %b, <512 x i32> <i32 0, i32 512, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 512, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 512, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 548, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 4, i32 574, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 512, i32 473, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 674, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 500, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 555>
   ret <512 x i8> %res
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
index 3eb5d36b4896a..5ea4924468595 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
@@ -1659,6 +1659,7 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vmv8r.v v24, v8
 ; RV32-NEXT:    lui a2, 1044480
 ; RV32-NEXT:    lui a3, 61681
@@ -1677,7 +1678,6 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV32-NEXT:    sw a3, 36(sp)
 ; RV32-NEXT:    addi a3, sp, 16
 ; RV32-NEXT:    addi a4, a5, 1365
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    addi a5, a6, -256
 ; RV32-NEXT:    sw a4, 24(sp)
@@ -2055,6 +2055,7 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vmv8r.v v24, v8
 ; RV32-NEXT:    lui a2, 1044480
 ; RV32-NEXT:    lui a3, 61681
@@ -2073,7 +2074,6 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV32-NEXT:    sw a3, 36(sp)
 ; RV32-NEXT:    addi a3, sp, 16
 ; RV32-NEXT:    addi a4, a5, 1365
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    addi a5, a6, -256
 ; RV32-NEXT:    sw a4, 24(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
index ee953a66a004f..60a9948198c8f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
@@ -180,6 +180,7 @@ define fastcc <32 x i32> @ret_v32i32_call_v32i32_v32i32_i32(<32 x i32> %x, <32 x
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset ra, -8
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a1, 2
 ; CHECK-NEXT:    vmv8r.v v8, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv.ll
index 73e148edbe2d6..f42b4a3a26aad 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv.ll
@@ -180,6 +180,7 @@ define <32 x i32> @ret_v32i32_call_v32i32_v32i32_i32(<32 x i32> %x, <32 x i32> %
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset ra, -8
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a1, 2
 ; CHECK-NEXT:    vmv8r.v v8, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
index 511242aa677c2..5fe55583f314c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
@@ -194,8 +194,8 @@ define <8 x half> @vp_ceil_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ;
 ; ZVFHMIN-LABEL: vp_ceil_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -261,10 +261,10 @@ declare <16 x half> @llvm.vp.ceil.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_ceil_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_ceil_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI6_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI6_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -280,8 +280,8 @@ define <16 x half> @vp_ceil_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %e
 ;
 ; ZVFHMIN-LABEL: vp_ceil_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -431,8 +431,8 @@ declare <8 x float> @llvm.vp.ceil.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_ceil_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -475,8 +475,8 @@ declare <16 x float> @llvm.vp.ceil.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_ceil_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -561,10 +561,10 @@ declare <4 x double> @llvm.vp.ceil.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_ceil_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -605,10 +605,10 @@ declare <8 x double> @llvm.vp.ceil.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_ceil_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -649,10 +649,10 @@ declare <15 x double> @llvm.vp.ceil.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_ceil_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -693,10 +693,10 @@ declare <16 x double> @llvm.vp.ceil.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_ceil_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ceil_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -743,6 +743,7 @@ define <32 x double> @vp_ceil_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -750,7 +751,6 @@ define <32 x double> @vp_ceil_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
index 5e73e6df9170c..a5a1061842427 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
@@ -1796,6 +1796,7 @@ define <32 x i64> @vp_ctpop_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv8r.v v24, v16
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    lui a2, 209715
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
index ae5dbfa4bf30b..ede25d2c9bb07 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
@@ -24,19 +24,20 @@ define {<16 x i1>, <16 x i1>} @vector_deinterleave_load_v16i1_v32i1(ptr %p) {
 ; CHECK-NEXT:    vadd.vi v12, v11, -16
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v0, v8, 2
-; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vadd.vi v11, v11, -15
 ; CHECK-NEXT:    vmerge.vim v13, v10, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vim v14, v10, 1, v0
-; CHECK-NEXT:    vnsrl.wi v8, v14, 0
+; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v10, v8, 0
+; CHECK-NEXT:    vnsrl.wi v8, v8, 8
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vrgather.vv v8, v13, v12, v0.t
-; CHECK-NEXT:    vnsrl.wi v12, v14, 8
-; CHECK-NEXT:    vmsne.vi v10, v8, 0
-; CHECK-NEXT:    vrgather.vv v12, v13, v11, v0.t
-; CHECK-NEXT:    vmsne.vi v8, v12, 0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; CHECK-NEXT:    vrgather.vv v10, v13, v12, v0.t
+; CHECK-NEXT:    vrgather.vv v8, v13, v11, v0.t
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    vmsne.vi v8, v8, 0
 ; CHECK-NEXT:    ret
   %vec = load <32 x i1>, ptr %p
   %retval = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
index 02e99ea513e69..49255320c40a6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
@@ -194,8 +194,8 @@ define <8 x half> @vp_floor_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ;
 ; ZVFHMIN-LABEL: vp_floor_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -261,10 +261,10 @@ declare <16 x half> @llvm.vp.floor.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_floor_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_floor_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI6_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI6_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -280,8 +280,8 @@ define <16 x half> @vp_floor_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ;
 ; ZVFHMIN-LABEL: vp_floor_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -431,8 +431,8 @@ declare <8 x float> @llvm.vp.floor.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_floor_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -475,8 +475,8 @@ declare <16 x float> @llvm.vp.floor.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_floor_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -561,10 +561,10 @@ declare <4 x double> @llvm.vp.floor.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_floor_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -605,10 +605,10 @@ declare <8 x double> @llvm.vp.floor.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_floor_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -649,10 +649,10 @@ declare <15 x double> @llvm.vp.floor.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_floor_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -693,10 +693,10 @@ declare <16 x double> @llvm.vp.floor.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_floor_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -743,6 +743,7 @@ define <32 x double> @vp_floor_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -750,7 +751,6 @@ define <32 x double> @vp_floor_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
index 9a3838d57a0b0..11f92555f56cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum-vp.ll
@@ -13,8 +13,8 @@ declare <2 x half> @llvm.vp.maximum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
 define <2 x half> @vfmax_vv_v2f16(<2 x half> %va, <2 x half> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_v2f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -26,8 +26,8 @@ define <2 x half> @vfmax_vv_v2f16(<2 x half> %va, <2 x half> %vb, <2 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -52,24 +52,24 @@ define <2 x half> @vfmax_vv_v2f16_unmasked(<2 x half> %va, <2 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v2f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -83,8 +83,8 @@ declare <4 x half> @llvm.vp.maximum.v4f16(<4 x half>, <4 x half>, <4 x i1>, i32)
 define <4 x half> @vfmax_vv_v4f16(<4 x half> %va, <4 x half> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_v4f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -96,8 +96,8 @@ define <4 x half> @vfmax_vv_v4f16(<4 x half> %va, <4 x half> %vb, <4 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -122,24 +122,24 @@ define <4 x half> @vfmax_vv_v4f16_unmasked(<4 x half> %va, <4 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v4f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -153,8 +153,8 @@ declare <8 x half> @llvm.vp.maximum.v8f16(<8 x half>, <8 x half>, <8 x i1>, i32)
 define <8 x half> @vfmax_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_v8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -166,8 +166,8 @@ define <8 x half> @vfmax_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -194,25 +194,25 @@ define <8 x half> @vfmax_vv_v8f16_unmasked(<8 x half> %va, <8 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v8f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v10, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v10, v0
-; ZVFHMIN-NEXT:    vfmax.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -225,8 +225,8 @@ declare <16 x half> @llvm.vp.maximum.v16f16(<16 x half>, <16 x half>, <16 x i1>,
 define <16 x half> @vfmax_vv_v16f16(<16 x half> %va, <16 x half> %vb, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v13
 ; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -240,8 +240,8 @@ define <16 x half> @vfmax_vv_v16f16(<16 x half> %va, <16 x half> %vb, <16 x i1>
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -268,25 +268,25 @@ define <16 x half> @vfmax_vv_v16f16_unmasked(<16 x half> %va, <16 x half> %vb, i
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v14
+; ZVFH-NEXT:    vfmax.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_v16f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v12, v16, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v16, v12, v0
-; ZVFHMIN-NEXT:    vfmax.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmax.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -299,8 +299,8 @@ declare <2 x float> @llvm.vp.maximum.v2f32(<2 x float>, <2 x float>, <2 x i1>, i
 define <2 x float> @vfmax_vv_v2f32(<2 x float> %va, <2 x float> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -318,11 +318,10 @@ define <2 x float> @vfmax_vv_v2f32_unmasked(<2 x float> %va, <2 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.vp.maximum.v2f32(<2 x float> %va, <2 x float> %vb, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x float> %v
@@ -333,8 +332,8 @@ declare <4 x float> @llvm.vp.maximum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i
 define <4 x float> @vfmax_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -352,11 +351,10 @@ define <4 x float> @vfmax_vv_v4f32_unmasked(<4 x float> %va, <4 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.vp.maximum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x float> %v
@@ -367,8 +365,8 @@ declare <8 x float> @llvm.vp.maximum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i
 define <8 x float> @vfmax_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -388,11 +386,10 @@ define <8 x float> @vfmax_vv_v8f32_unmasked(<8 x float> %va, <8 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.vp.maximum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x float> %v
@@ -403,8 +400,8 @@ declare <16 x float> @llvm.vp.maximum.v16f32(<16 x float>, <16 x float>, <16 x i
 define <16 x float> @vfmax_vv_v16f32(<16 x float> %va, <16 x float> %vb, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -424,11 +421,10 @@ define <16 x float> @vfmax_vv_v16f32_unmasked(<16 x float> %va, <16 x float> %vb
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.vp.maximum.v16f32(<16 x float> %va, <16 x float> %vb, <16 x i1> splat (i1 true), i32 %evl)
   ret <16 x float> %v
@@ -439,8 +435,8 @@ declare <2 x double> @llvm.vp.maximum.v2f64(<2 x double>, <2 x double>, <2 x i1>
 define <2 x double> @vfmax_vv_v2f64(<2 x double> %va, <2 x double> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -458,11 +454,10 @@ define <2 x double> @vfmax_vv_v2f64_unmasked(<2 x double> %va, <2 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.vp.maximum.v2f64(<2 x double> %va, <2 x double> %vb, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x double> %v
@@ -473,8 +468,8 @@ declare <4 x double> @llvm.vp.maximum.v4f64(<4 x double>, <4 x double>, <4 x i1>
 define <4 x double> @vfmax_vv_v4f64(<4 x double> %va, <4 x double> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -494,11 +489,10 @@ define <4 x double> @vfmax_vv_v4f64_unmasked(<4 x double> %va, <4 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.vp.maximum.v4f64(<4 x double> %va, <4 x double> %vb, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x double> %v
@@ -509,8 +503,8 @@ declare <8 x double> @llvm.vp.maximum.v8f64(<8 x double>, <8 x double>, <8 x i1>
 define <8 x double> @vfmax_vv_v8f64(<8 x double> %va, <8 x double> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -530,11 +524,10 @@ define <8 x double> @vfmax_vv_v8f64_unmasked(<8 x double> %va, <8 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.vp.maximum.v8f64(<8 x double> %va, <8 x double> %vb, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x double> %v
@@ -551,8 +544,8 @@ define <16 x double> @vfmax_vv_v16f64(<16 x double> %va, <16 x double> %vb, <16
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -602,6 +595,7 @@ define <32 x double> @vfmax_vv_v32f64(<32 x double> %va, <32 x double> %vb, <32
 ; CHECK-NEXT:    slli a1, a1, 5
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 4
@@ -615,7 +609,6 @@ define <32 x double> @vfmax_vv_v32f64(<32 x double> %va, <32 x double> %vb, <32
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    addi a1, a0, 128
-; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vle64.v v16, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
index 900e02876cbe1..e17ad303eddb8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
@@ -15,25 +15,25 @@ define <2 x half> @vfmax_v2f16_vv(<2 x half> %a, <2 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v2f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -48,25 +48,25 @@ define <4 x half> @vfmax_v4f16_vv(<4 x half> %a, <4 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v4f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -81,25 +81,25 @@ define <8 x half> @vfmax_v8f16_vv(<8 x half> %a, <8 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v8f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v12, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
-; ZVFHMIN-NEXT:    vfmax.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -114,25 +114,25 @@ define <16 x half> @vfmax_v16f16_vv(<16 x half> %a, <16 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v14
+; ZVFH-NEXT:    vfmax.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v16f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v10
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v16, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
-; ZVFHMIN-NEXT:    vfmax.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmax.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -147,11 +147,10 @@ define <2 x float> @vfmax_v2f32_vv(<2 x float> %a, <2 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %v
@@ -164,11 +163,10 @@ define <4 x float> @vfmax_v4f32_vv(<4 x float> %a, <4 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.maximum.v4f32(<4 x float> %a, <4 x float> %b)
   ret <4 x float> %v
@@ -181,11 +179,10 @@ define <8 x float> @vfmax_v8f32_vv(<8 x float> %a, <8 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.maximum.v8f32(<8 x float> %a, <8 x float> %b)
   ret <8 x float> %v
@@ -198,11 +195,10 @@ define <16 x float> @vfmax_v16f32_vv(<16 x float> %a, <16 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %a, <16 x float> %b)
   ret <16 x float> %v
@@ -215,11 +211,10 @@ define <2 x double> @vfmax_v2f64_vv(<2 x double> %a, <2 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.maximum.v2f64(<2 x double> %a, <2 x double> %b)
   ret <2 x double> %v
@@ -232,11 +227,10 @@ define <4 x double> @vfmax_v4f64_vv(<4 x double> %a, <4 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.maximum.v4f64(<4 x double> %a, <4 x double> %b)
   ret <4 x double> %v
@@ -249,11 +243,10 @@ define <8 x double> @vfmax_v8f64_vv(<8 x double> %a, <8 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.maximum.v8f64(<8 x double> %a, <8 x double> %b)
   ret <8 x double> %v
@@ -302,32 +295,30 @@ define <2 x half> @vfmax_v2f16_vv_nnana(<2 x half> %a, <2 x half> %b) {
 ; ZVFH-LABEL: vfmax_v2f16_vv_nnana:
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfadd.vv v10, v8, v8
 ; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFH-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v8
-; ZVFH-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFH-NEXT:    vfmax.vv v8, v11, v8
+; ZVFH-NEXT:    vfadd.vv v8, v8, v8
+; ZVFH-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFH-NEXT:    vmerge.vvm v8, v8, v9, v0
+; ZVFH-NEXT:    vfmax.vv v8, v10, v8
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v2f16_vv_nnana:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v11, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v11, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v9, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -341,13 +332,12 @@ define <2 x half> @vfmax_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
 ; ZVFH-LABEL: vfmax_v2f16_vv_nnanb:
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfadd.vv v10, v9, v9
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v9, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v9
-; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfadd.vv v9, v9, v9
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_v2f16_vv_nnanb:
@@ -360,12 +350,11 @@ define <2 x half> @vfmax_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v10
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v9, v0
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
index 4a7f888fbced4..3fb586b67a21b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum-vp.ll
@@ -13,8 +13,8 @@ declare <2 x half> @llvm.vp.minimum.v2f16(<2 x half>, <2 x half>, <2 x i1>, i32)
 define <2 x half> @vfmin_vv_v2f16(<2 x half> %va, <2 x half> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_v2f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -26,8 +26,8 @@ define <2 x half> @vfmin_vv_v2f16(<2 x half> %va, <2 x half> %vb, <2 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -52,24 +52,24 @@ define <2 x half> @vfmin_vv_v2f16_unmasked(<2 x half> %va, <2 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v2f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -83,8 +83,8 @@ declare <4 x half> @llvm.vp.minimum.v4f16(<4 x half>, <4 x half>, <4 x i1>, i32)
 define <4 x half> @vfmin_vv_v4f16(<4 x half> %va, <4 x half> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_v4f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -96,8 +96,8 @@ define <4 x half> @vfmin_vv_v4f16(<4 x half> %va, <4 x half> %vb, <4 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -122,24 +122,24 @@ define <4 x half> @vfmin_vv_v4f16_unmasked(<4 x half> %va, <4 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v4f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -153,8 +153,8 @@ declare <8 x half> @llvm.vp.minimum.v8f16(<8 x half>, <8 x half>, <8 x i1>, i32)
 define <8 x half> @vfmin_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_v8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -166,8 +166,8 @@ define <8 x half> @vfmin_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -194,25 +194,25 @@ define <8 x half> @vfmin_vv_v8f16_unmasked(<8 x half> %va, <8 x half> %vb, i32 z
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v8f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v10, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v10, v0
-; ZVFHMIN-NEXT:    vfmin.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -225,8 +225,8 @@ declare <16 x half> @llvm.vp.minimum.v16f16(<16 x half>, <16 x half>, <16 x i1>,
 define <16 x half> @vfmin_vv_v16f16(<16 x half> %va, <16 x half> %vb, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v13
 ; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -240,8 +240,8 @@ define <16 x half> @vfmin_vv_v16f16(<16 x half> %va, <16 x half> %vb, <16 x i1>
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -268,25 +268,25 @@ define <16 x half> @vfmin_vv_v16f16_unmasked(<16 x half> %va, <16 x half> %vb, i
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v14
+; ZVFH-NEXT:    vfmin.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_v16f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v12, v16, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v16, v12, v0
-; ZVFHMIN-NEXT:    vfmin.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmin.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -299,8 +299,8 @@ declare <2 x float> @llvm.vp.minimum.v2f32(<2 x float>, <2 x float>, <2 x i1>, i
 define <2 x float> @vfmin_vv_v2f32(<2 x float> %va, <2 x float> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -318,11 +318,10 @@ define <2 x float> @vfmin_vv_v2f32_unmasked(<2 x float> %va, <2 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.vp.minimum.v2f32(<2 x float> %va, <2 x float> %vb, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x float> %v
@@ -333,8 +332,8 @@ declare <4 x float> @llvm.vp.minimum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i
 define <4 x float> @vfmin_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -352,11 +351,10 @@ define <4 x float> @vfmin_vv_v4f32_unmasked(<4 x float> %va, <4 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.vp.minimum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x float> %v
@@ -367,8 +365,8 @@ declare <8 x float> @llvm.vp.minimum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i
 define <8 x float> @vfmin_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -388,11 +386,10 @@ define <8 x float> @vfmin_vv_v8f32_unmasked(<8 x float> %va, <8 x float> %vb, i3
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.vp.minimum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x float> %v
@@ -403,8 +400,8 @@ declare <16 x float> @llvm.vp.minimum.v16f32(<16 x float>, <16 x float>, <16 x i
 define <16 x float> @vfmin_vv_v16f32(<16 x float> %va, <16 x float> %vb, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -424,11 +421,10 @@ define <16 x float> @vfmin_vv_v16f32_unmasked(<16 x float> %va, <16 x float> %vb
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.vp.minimum.v16f32(<16 x float> %va, <16 x float> %vb, <16 x i1> splat (i1 true), i32 %evl)
   ret <16 x float> %v
@@ -439,8 +435,8 @@ declare <2 x double> @llvm.vp.minimum.v2f64(<2 x double>, <2 x double>, <2 x i1>
 define <2 x double> @vfmin_vv_v2f64(<2 x double> %va, <2 x double> %vb, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -458,11 +454,10 @@ define <2 x double> @vfmin_vv_v2f64_unmasked(<2 x double> %va, <2 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.vp.minimum.v2f64(<2 x double> %va, <2 x double> %vb, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x double> %v
@@ -473,8 +468,8 @@ declare <4 x double> @llvm.vp.minimum.v4f64(<4 x double>, <4 x double>, <4 x i1>
 define <4 x double> @vfmin_vv_v4f64(<4 x double> %va, <4 x double> %vb, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -494,11 +489,10 @@ define <4 x double> @vfmin_vv_v4f64_unmasked(<4 x double> %va, <4 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.vp.minimum.v4f64(<4 x double> %va, <4 x double> %vb, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x double> %v
@@ -509,8 +503,8 @@ declare <8 x double> @llvm.vp.minimum.v8f64(<8 x double>, <8 x double>, <8 x i1>
 define <8 x double> @vfmin_vv_v8f64(<8 x double> %va, <8 x double> %vb, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -530,11 +524,10 @@ define <8 x double> @vfmin_vv_v8f64_unmasked(<8 x double> %va, <8 x double> %vb,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.vp.minimum.v8f64(<8 x double> %va, <8 x double> %vb, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x double> %v
@@ -551,8 +544,8 @@ define <16 x double> @vfmin_vv_v16f64(<16 x double> %va, <16 x double> %vb, <16
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -602,6 +595,7 @@ define <32 x double> @vfmin_vv_v32f64(<32 x double> %va, <32 x double> %vb, <32
 ; CHECK-NEXT:    slli a1, a1, 5
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 4
@@ -615,7 +609,6 @@ define <32 x double> @vfmin_vv_v32f64(<32 x double> %va, <32 x double> %vb, <32
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    addi a1, a0, 128
-; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vle64.v v16, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
index db970c89d935c..1362055c4dabf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
@@ -15,25 +15,25 @@ define <2 x half> @vfmin_v2f16_vv(<2 x half> %a, <2 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v2f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -48,25 +48,25 @@ define <4 x half> @vfmin_v4f16_vv(<4 x half> %a, <4 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v4f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -81,25 +81,25 @@ define <8 x half> @vfmin_v8f16_vv(<8 x half> %a, <8 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v8f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v12, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
-; ZVFHMIN-NEXT:    vfmin.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -114,25 +114,25 @@ define <16 x half> @vfmin_v16f16_vv(<16 x half> %a, <16 x half> %b) {
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v14
+; ZVFH-NEXT:    vfmin.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v16f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v10
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v16, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
-; ZVFHMIN-NEXT:    vfmin.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmin.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -147,11 +147,10 @@ define <2 x float> @vfmin_v2f32_vv(<2 x float> %a, <2 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %v
@@ -164,11 +163,10 @@ define <4 x float> @vfmin_v4f32_vv(<4 x float> %a, <4 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
   ret <4 x float> %v
@@ -181,11 +179,10 @@ define <8 x float> @vfmin_v8f32_vv(<8 x float> %a, <8 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.minimum.v8f32(<8 x float> %a, <8 x float> %b)
   ret <8 x float> %v
@@ -198,11 +195,10 @@ define <16 x float> @vfmin_v16f32_vv(<16 x float> %a, <16 x float> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.minimum.v16f32(<16 x float> %a, <16 x float> %b)
   ret <16 x float> %v
@@ -215,11 +211,10 @@ define <2 x double> @vfmin_v2f64_vv(<2 x double> %a, <2 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.minimum.v2f64(<2 x double> %a, <2 x double> %b)
   ret <2 x double> %v
@@ -232,11 +227,10 @@ define <4 x double> @vfmin_v4f64_vv(<4 x double> %a, <4 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.minimum.v4f64(<4 x double> %a, <4 x double> %b)
   ret <4 x double> %v
@@ -249,11 +243,10 @@ define <8 x double> @vfmin_v8f64_vv(<8 x double> %a, <8 x double> %b) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.minimum.v8f64(<8 x double> %a, <8 x double> %b)
   ret <8 x double> %v
@@ -302,32 +295,30 @@ define <2 x half> @vfmin_v2f16_vv_nnana(<2 x half> %a, <2 x half> %b) {
 ; ZVFH-LABEL: vfmin_v2f16_vv_nnana:
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfadd.vv v10, v8, v8
 ; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFH-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v8
-; ZVFH-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFH-NEXT:    vfmin.vv v8, v11, v8
+; ZVFH-NEXT:    vfadd.vv v8, v8, v8
+; ZVFH-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFH-NEXT:    vmerge.vvm v8, v8, v9, v0
+; ZVFH-NEXT:    vfmin.vv v8, v10, v8
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v2f16_vv_nnana:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v11, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v11, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v9, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -341,13 +332,12 @@ define <2 x half> @vfmin_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
 ; ZVFH-LABEL: vfmin_v2f16_vv_nnanb:
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfadd.vv v10, v9, v9
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v9, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v9
-; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfadd.vv v9, v9, v9
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_v2f16_vv_nnanb:
@@ -360,12 +350,11 @@ define <2 x half> @vfmin_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v10
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v9, v0
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index b0f8bc9dcc6bd..e82891f90d85e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -40,16 +40,16 @@ define <4 x float> @hang_when_merging_stores_after_legalization(<8 x float> %x,
 ; CHECK-LABEL: hang_when_merging_stores_after_legalization:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vmv.v.i v12, -14
+; CHECK-NEXT:    vid.v v14
 ; CHECK-NEXT:    li a0, 7
+; CHECK-NEXT:    vmadd.vx v14, a0, v12
+; CHECK-NEXT:    li a0, 129
+; CHECK-NEXT:    vmv.s.x v15, a0
 ; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vmul.vx v14, v12, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v14
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v8, v14, -14
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v10, v8, v0.t
+; CHECK-NEXT:    vcompress.vm v12, v8, v15
+; CHECK-NEXT:    vrgatherei16.vv v12, v10, v14, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v12
 ; CHECK-NEXT:    ret
   %z = shufflevector <8 x float> %x, <8 x float> %y, <4 x i32> <i32 0, i32 7, i32 8, i32 15>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
index e4b8e9debad27..4b09b571b9406 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-interleave.ll
@@ -38,8 +38,8 @@ define <4 x float> @interleave_v2f32(<2 x float> %x, <2 x float> %y) {
 define <4 x double> @interleave_v2f64(<2 x double> %x, <2 x double> %y) {
 ; V128-LABEL: interleave_v2f64:
 ; V128:       # %bb.0:
-; V128-NEXT:    vmv1r.v v12, v9
 ; V128-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; V128-NEXT:    vmv1r.v v12, v9
 ; V128-NEXT:    vid.v v9
 ; V128-NEXT:    vmv.v.i v0, 10
 ; V128-NEXT:    vsrl.vi v14, v9, 1
@@ -242,33 +242,27 @@ define <64 x float> @interleave_v32f32(<32 x float> %x, <32 x float> %y) {
 ; V128-NEXT:    slli a0, a0, 3
 ; V128-NEXT:    sub sp, sp, a0
 ; V128-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; V128-NEXT:    vmv8r.v v24, v16
-; V128-NEXT:    vmv8r.v v16, v8
-; V128-NEXT:    vmv8r.v v8, v24
 ; V128-NEXT:    addi a0, sp, 16
-; V128-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; V128-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; V128-NEXT:    vslidedown.vi v0, v24, 16
-; V128-NEXT:    li a0, -1
-; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v24, v8, v0
-; V128-NEXT:    vwmaccu.vx v24, a0, v0
+; V128-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; V128-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; V128-NEXT:    vslidedown.vi v0, v16, 16
+; V128-NEXT:    vslidedown.vi v24, v16, 16
+; V128-NEXT:    li a0, 32
 ; V128-NEXT:    lui a1, 699051
-; V128-NEXT:    li a2, 32
-; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v8, v0, v16
+; V128-NEXT:    vslidedown.vi v0, v8, 16
+; V128-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; V128-NEXT:    vzext.vf2 v8, v24
 ; V128-NEXT:    addi a1, a1, -1366
+; V128-NEXT:    vzext.vf2 v24, v0
 ; V128-NEXT:    vmv.s.x v0, a1
-; V128-NEXT:    vwmaccu.vx v8, a0, v16
-; V128-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; V128-NEXT:    vmerge.vvm v24, v8, v24, v0
-; V128-NEXT:    addi a1, sp, 16
-; V128-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; V128-NEXT:    vsll.vx v8, v8, a0
+; V128-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; V128-NEXT:    vmerge.vvm v24, v24, v8, v0
+; V128-NEXT:    addi a0, sp, 16
+; V128-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v0, v16, v8
-; V128-NEXT:    vwmaccu.vx v0, a0, v8
+; V128-NEXT:    vwaddu.vv v0, v8, v16
+; V128-NEXT:    li a0, -1
+; V128-NEXT:    vwmaccu.vx v0, a0, v16
 ; V128-NEXT:    vmv8r.v v8, v0
 ; V128-NEXT:    vmv8r.v v16, v24
 ; V128-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
index c803b15913bb3..0db45ae71bc8a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
@@ -138,17 +138,17 @@ define <4 x double> @vrgather_shuffle_xv_v4f64(<4 x double> %x) {
 define <4 x double> @vrgather_shuffle_vx_v4f64(<4 x double> %x) {
 ; CHECK-LABEL: vrgather_shuffle_vx_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    lui a0, %hi(.LCPI9_0)
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI9_0)(a0)
-; CHECK-NEXT:    li a0, 3
-; CHECK-NEXT:    vmul.vx v12, v10, a0
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 9
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vcompress.vm v12, v8, v10
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v0, 3
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; CHECK-NEXT:    vfmv.v.f v10, fa5
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    vmerge.vvm v8, v8, v12, v0
 ; CHECK-NEXT:    ret
   %s = shufflevector <4 x double> %x, <4 x double> <double 2.0, double 2.0, double 2.0, double 2.0>, <4 x i32> <i32 0, i32 3, i32 6, i32 5>
   ret <4 x double> %s
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
index e64c7c87132ee..582706e4dfa18 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
@@ -97,9 +97,9 @@ declare <32 x float> @llvm.vp.fptrunc.v32f64.v32f32(<32 x double>, <32 x i1>, i3
 define <32 x float> @vfptrunc_v32f32_v32f64(<32 x double> %a, <32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vfptrunc_v32f32_v32f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v12, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB7_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fshr-fshl-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fshr-fshl-vp.ll
index a68dc11f3d21e..fc7cd94ca3de8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fshr-fshl-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fshr-fshl-vp.ll
@@ -712,8 +712,8 @@ define <16 x i64> @fshl_v16i64(<16 x i64> %a, <16 x i64> %b, <16 x i64> %c, <16
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    vle64.v v24, (a0)
 ; CHECK-NEXT:    li a0, 63
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
index 1fbc8dfd688c4..62e7e3b109902 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
@@ -133,6 +133,7 @@ define <vscale x 2 x i32> @insert_nxv8i32_v4i32_0(<vscale x 2 x i32> %vec, <4 x
 ;
 ; VLS-LABEL: insert_nxv8i32_v4i32_0:
 ; VLS:       # %bb.0:
+; VLS-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; VLS-NEXT:    vmv1r.v v8, v9
 ; VLS-NEXT:    ret
   %v = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> %vec, <4 x i32> %subvec, i64 0)
@@ -143,6 +144,7 @@ define <vscale x 2 x i32> @insert_nxv8i32_v4i32_0(<vscale x 2 x i32> %vec, <4 x
 define <4 x i32> @insert_v4i32_v4i32_0(<4 x i32> %vec, <4 x i32> %subvec) {
 ; CHECK-LABEL: insert_v4i32_v4i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %v = call <4 x i32> @llvm.vector.insert.v4i32.v4i32(<4 x i32> %vec, <4 x i32> %subvec, i64 0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
index 66af5718fb9dc..da7cdf3ba8ec0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
@@ -51,8 +51,8 @@ define <4 x i32> @interleave_v2i32(<2 x i32> %x, <2 x i32> %y) {
 define <4 x i64> @interleave_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; V128-LABEL: interleave_v2i64:
 ; V128:       # %bb.0:
-; V128-NEXT:    vmv1r.v v12, v9
 ; V128-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; V128-NEXT:    vmv1r.v v12, v9
 ; V128-NEXT:    vid.v v9
 ; V128-NEXT:    vmv.v.i v0, 10
 ; V128-NEXT:    vsrl.vi v14, v9, 1
@@ -186,35 +186,29 @@ define <4 x i32> @interleave_v4i32_offset_2(<4 x i32> %x, <4 x i32> %y) {
 define <4 x i32> @interleave_v4i32_offset_1(<4 x i32> %x, <4 x i32> %y) {
 ; V128-LABEL: interleave_v4i32_offset_1:
 ; V128:       # %bb.0:
-; V128-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; V128-NEXT:    vwaddu.vv v10, v8, v8
-; V128-NEXT:    li a0, -1
 ; V128-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; V128-NEXT:    vid.v v11
-; V128-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; V128-NEXT:    vwmaccu.vx v10, a0, v8
-; V128-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; V128-NEXT:    vsrl.vi v8, v11, 1
+; V128-NEXT:    vid.v v10
 ; V128-NEXT:    vmv.v.i v0, 10
-; V128-NEXT:    vadd.vi v8, v8, 1
-; V128-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; V128-NEXT:    vsrl.vi v10, v10, 1
+; V128-NEXT:    vadd.vi v11, v10, 1
+; V128-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; V128-NEXT:    vzext.vf2 v10, v8
+; V128-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; V128-NEXT:    vrgather.vv v10, v9, v11, v0.t
 ; V128-NEXT:    vmv.v.v v8, v10
 ; V128-NEXT:    ret
 ;
 ; V512-LABEL: interleave_v4i32_offset_1:
 ; V512:       # %bb.0:
-; V512-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; V512-NEXT:    vwaddu.vv v10, v8, v8
-; V512-NEXT:    li a0, -1
 ; V512-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
-; V512-NEXT:    vid.v v11
-; V512-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; V512-NEXT:    vwmaccu.vx v10, a0, v8
-; V512-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
-; V512-NEXT:    vsrl.vi v8, v11, 1
+; V512-NEXT:    vid.v v10
 ; V512-NEXT:    vmv.v.i v0, 10
-; V512-NEXT:    vadd.vi v8, v8, 1
-; V512-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; V512-NEXT:    vsrl.vi v10, v10, 1
+; V512-NEXT:    vadd.vi v11, v10, 1
+; V512-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; V512-NEXT:    vzext.vf2 v10, v8
+; V512-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
+; V512-NEXT:    vrgather.vv v10, v9, v11, v0.t
 ; V512-NEXT:    vmv1r.v v8, v10
 ; V512-NEXT:    ret
   %a = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 1, i32 6>
@@ -411,33 +405,27 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; V128-NEXT:    slli a0, a0, 3
 ; V128-NEXT:    sub sp, sp, a0
 ; V128-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; V128-NEXT:    vmv8r.v v24, v16
-; V128-NEXT:    vmv8r.v v16, v8
-; V128-NEXT:    vmv8r.v v8, v24
 ; V128-NEXT:    addi a0, sp, 16
-; V128-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; V128-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; V128-NEXT:    vslidedown.vi v0, v24, 16
-; V128-NEXT:    li a0, -1
-; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v24, v8, v0
-; V128-NEXT:    vwmaccu.vx v24, a0, v0
+; V128-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; V128-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; V128-NEXT:    vslidedown.vi v0, v16, 16
+; V128-NEXT:    vslidedown.vi v24, v16, 16
+; V128-NEXT:    li a0, 32
 ; V128-NEXT:    lui a1, 699051
-; V128-NEXT:    li a2, 32
-; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v8, v0, v16
+; V128-NEXT:    vslidedown.vi v0, v8, 16
+; V128-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; V128-NEXT:    vzext.vf2 v8, v24
 ; V128-NEXT:    addi a1, a1, -1366
+; V128-NEXT:    vzext.vf2 v24, v0
 ; V128-NEXT:    vmv.s.x v0, a1
-; V128-NEXT:    vwmaccu.vx v8, a0, v16
-; V128-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; V128-NEXT:    vmerge.vvm v24, v8, v24, v0
-; V128-NEXT:    addi a1, sp, 16
-; V128-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; V128-NEXT:    vsll.vx v8, v8, a0
+; V128-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; V128-NEXT:    vmerge.vvm v24, v24, v8, v0
+; V128-NEXT:    addi a0, sp, 16
+; V128-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; V128-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; V128-NEXT:    vwaddu.vv v0, v16, v8
-; V128-NEXT:    vwmaccu.vx v0, a0, v8
+; V128-NEXT:    vwaddu.vv v0, v8, v16
+; V128-NEXT:    li a0, -1
+; V128-NEXT:    vwmaccu.vx v0, a0, v16
 ; V128-NEXT:    vmv8r.v v8, v0
 ; V128-NEXT:    vmv8r.v v16, v24
 ; V128-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
index dbfe7bb51dbff..1c6e1a37fa8af 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
@@ -113,14 +113,12 @@ define <4 x i16> @vrgather_shuffle_xv_v4i16(<4 x i16> %x) {
 define <4 x i16> @vrgather_shuffle_vx_v4i16(<4 x i16> %x) {
 ; CHECK-LABEL: vrgather_shuffle_vx_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v9, 9
 ; CHECK-NEXT:    vmv.v.i v0, 3
-; CHECK-NEXT:    vmul.vx v10, v9, a0
-; CHECK-NEXT:    vmv.v.i v9, 5
-; CHECK-NEXT:    vrgather.vv v9, v8, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
+; CHECK-NEXT:    vmv.v.i v8, 5
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; CHECK-NEXT:    ret
   %s = shufflevector <4 x i16> %x, <4 x i16> <i16 5, i16 5, i16 5, i16 5>, <4 x i32> <i32 0, i32 3, i32 6, i32 5>
   ret <4 x i16> %s
@@ -723,23 +721,12 @@ define <8 x i32> @shuffle_v8i32_2(<8 x i32> %x, <8 x i32> %y) {
 define <8 x i8> @shuffle_v64i8_v8i8(<64 x i8> %wide.vec) {
 ; CHECK-LABEL: shuffle_v64i8_v8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    li a1, 240
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    lui a1, 98561
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vsll.vi v14, v12, 3
-; CHECK-NEXT:    vrgather.vv v12, v8, v14
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    addi a1, a1, -2048
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vrgather.vv v12, v8, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v12
+; CHECK-NEXT:    vnsrl.wi v12, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %s = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56>
   ret <8 x i8> %s
@@ -748,11 +735,10 @@ define <8 x i8> @shuffle_v64i8_v8i8(<64 x i8> %wide.vec) {
 define <8 x i8> @shuffle_compress_singlesrc_e8(<8 x i8> %v) {
 ; CHECK-LABEL: shuffle_compress_singlesrc_e8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI49_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI49_0)
+; CHECK-NEXT:    li a0, 181
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vle8.v v10, (a0)
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.s.x v10, a0
+; CHECK-NEXT:    vcompress.vm v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %out = shufflevector <8 x i8> %v, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>
@@ -762,11 +748,10 @@ define <8 x i8> @shuffle_compress_singlesrc_e8(<8 x i8> %v) {
 define <8 x i16> @shuffle_compress_singlesrc_e16(<8 x i16> %v) {
 ; CHECK-LABEL: shuffle_compress_singlesrc_e16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI50_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI50_0)
+; CHECK-NEXT:    li a0, 181
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vle16.v v10, (a0)
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.s.x v10, a0
+; CHECK-NEXT:    vcompress.vm v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
   %out = shufflevector <8 x i16> %v, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>
@@ -776,11 +761,10 @@ define <8 x i16> @shuffle_compress_singlesrc_e16(<8 x i16> %v) {
 define <8 x i32> @shuffle_compress_singlesrc_e32(<8 x i32> %v) {
 ; CHECK-LABEL: shuffle_compress_singlesrc_e32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI51_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI51_0)
+; CHECK-NEXT:    li a0, 115
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT:    vle16.v v12, (a0)
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
+; CHECK-NEXT:    vmv.s.x v12, a0
+; CHECK-NEXT:    vcompress.vm v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %out = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 undef>
@@ -790,11 +774,10 @@ define <8 x i32> @shuffle_compress_singlesrc_e32(<8 x i32> %v) {
 define <8 x i64> @shuffle_compress_singlesrc_e64(<8 x i64> %v) {
 ; CHECK-LABEL: shuffle_compress_singlesrc_e64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI52_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI52_0)
+; CHECK-NEXT:    li a0, 181
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; CHECK-NEXT:    vle16.v v16, (a0)
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
+; CHECK-NEXT:    vmv.s.x v16, a0
+; CHECK-NEXT:    vcompress.vm v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %out = shufflevector <8 x i64> %v, <8 x i64> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>
@@ -815,31 +798,56 @@ define <8 x i32> @shuffle_compress_singlesrc_gaps_e32(<8 x i32> %v) {
   ret <8 x i32> %out
 }
 
-define <8 x i32> @shuffle_decompress2_singlesrc_e32(<8 x i32> %v) {
-; CHECK-LABEL: shuffle_decompress2_singlesrc_e32:
+define <8 x i32> @shuffle_spread2_singlesrc_e32(<8 x i32> %v) {
+; CHECK-LABEL: shuffle_spread2_singlesrc_e32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vwaddu.vv v10, v8, v8
-; CHECK-NEXT:    li a0, -1
-; CHECK-NEXT:    vwmaccu.vx v10, a0, v8
-; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v10, v8
+; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %out = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3, i32 undef>
   ret <8 x i32> %out
 }
 
-define <8 x i32> @shuffle_decompress3_singlesrc_e32(<8 x i32> %v) {
-; RV32-LABEL: shuffle_decompress3_singlesrc_e32:
+define <8 x i32> @shuffle_spread2_singlesrc_e32_index1(<8 x i32> %v) {
+; CHECK-LABEL: shuffle_spread2_singlesrc_e32_index1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v10, v8
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsll.vx v8, v10, a0
+; CHECK-NEXT:    ret
+  %out = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3>
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @shuffle_spread2_singlesrc_e32_index2(<8 x i32> %v) {
+; CHECK-LABEL: shuffle_spread2_singlesrc_e32_index2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vsrl.vi v10, v10, 1
+; CHECK-NEXT:    vadd.vi v12, v10, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %out = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 undef, i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef>
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @shuffle_spread3_singlesrc_e32(<8 x i32> %v) {
+; RV32-LABEL: shuffle_spread3_singlesrc_e32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a0, %hi(.LCPI55_0)
-; RV32-NEXT:    addi a0, a0, %lo(.LCPI55_0)
+; RV32-NEXT:    lui a0, %hi(.LCPI57_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI57_0)
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vle16.v v12, (a0)
 ; RV32-NEXT:    vrgatherei16.vv v10, v8, v12
 ; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
-; RV64-LABEL: shuffle_decompress3_singlesrc_e32:
+; RV64-LABEL: shuffle_spread3_singlesrc_e32:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a0, 32769
 ; RV64-NEXT:    slli a0, a0, 21
@@ -854,8 +862,8 @@ define <8 x i32> @shuffle_decompress3_singlesrc_e32(<8 x i32> %v) {
 }
 
 ; TODO: This should be a single vslideup.vi
-define <8 x i32> @shuffle_decompress4_singlesrc_e32(<8 x i32> %v) {
-; CHECK-LABEL: shuffle_decompress4_singlesrc_e32:
+define <8 x i32> @shuffle_spread4_singlesrc_e32(<8 x i32> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
@@ -868,25 +876,81 @@ define <8 x i32> @shuffle_decompress4_singlesrc_e32(<8 x i32> %v) {
   ret <8 x i32> %out
 }
 
-; TODO: This should be either a single vslideup.vi or two widening interleaves.
-define <8 x i8> @shuffle_decompress4_singlesrc_e8(<8 x i8> %v) {
-; CHECK-LABEL: shuffle_decompress4_singlesrc_e8:
+define <16 x i8> @shuffle_spread4_singlesrc_e8_idx0(<16 x i8> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e8_idx0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v9, v8
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i8> %v, <16 x i8> poison, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @shuffle_spread4_singlesrc_e8_idx1(<16 x i8> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e8_idx1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v9, v8
+; CHECK-NEXT:    vsll.vi v8, v9, 8
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i8> %v, <16 x i8> poison, <16 x i32> <i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef>
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @shuffle_spread4_singlesrc_e8_idx2(<16 x i8> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e8_idx2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v9, v8
+; CHECK-NEXT:    vsll.vi v8, v9, 16
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i8> %v, <16 x i8> poison, <16 x i32> <i32 undef, i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3, i32 undef>
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @shuffle_spread4_singlesrc_e8_idx3(<16 x i8> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e8_idx3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v9, v8
+; CHECK-NEXT:    vsll.vi v8, v9, 24
+; CHECK-NEXT:    ret
+  %out = shufflevector <16 x i8> %v, <16 x i8> poison, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3>
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @shuffle_spread4_singlesrc_e8_idx4(<16 x i8> %v) {
+; CHECK-LABEL: shuffle_spread4_singlesrc_e8_idx4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vsrl.vi v10, v9, 2
+; CHECK-NEXT:    vsrl.vi v9, v9, 2
+; CHECK-NEXT:    vadd.vi v10, v9, -1
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %out = shufflevector <8 x i8> %v, <8 x i8> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef>
-  ret <8 x i8> %out
+  %out = shufflevector <16 x i8> %v, <16 x i8> poison, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %out
+}
+
+
+define <32 x i8> @shuffle_spread8_singlesrc_e8(<32 x i8> %v) {
+; CHECK-LABEL: shuffle_spread8_singlesrc_e8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf8 v10, v8
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %out = shufflevector <32 x i8> %v, <32 x i8> poison, <32 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <32 x i8> %out
 }
 
 define <8 x i32> @shuffle_decompress_singlesrc_e32(<8 x i32> %v) {
 ; CHECK-LABEL: shuffle_decompress_singlesrc_e32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI58_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI58_0)
+; CHECK-NEXT:    lui a0, %hi(.LCPI65_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI65_0)
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vle16.v v12, (a0)
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
@@ -896,6 +960,22 @@ define <8 x i32> @shuffle_decompress_singlesrc_e32(<8 x i32> %v) {
   ret <8 x i32> %out
 }
 
+; TODO: This should be a single vslideup.vi
+define <8 x i8> @shuffle_decompress_singlesrc_e8(<8 x i8> %v) {
+; CHECK-LABEL: shuffle_decompress_singlesrc_e8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, %hi(.LCPI66_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI66_0)
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vle8.v v10, (a0)
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %out = shufflevector <8 x i8> %v, <8 x i8> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 4>
+  ret <8 x i8> %out
+}
+
+
 define <8 x i32> @shuffle_repeat2_singlesrc_e32(<8 x i32> %v) {
 ; CHECK-LABEL: shuffle_repeat2_singlesrc_e32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
index 30e41f2f526e5..b65352aed2d52 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
@@ -16,58 +16,53 @@ define <4 x i1> @load_large_vector(ptr %p) {
 ; ZVE32X-NEXT:    ld a7, 72(a0)
 ; ZVE32X-NEXT:    ld a0, 80(a0)
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmv.s.x v9, zero
-; ZVE32X-NEXT:    vmv.v.i v10, 0
+; ZVE32X-NEXT:    vmv.s.x v8, zero
+; ZVE32X-NEXT:    vmv.v.i v9, 0
 ; ZVE32X-NEXT:    xor a3, a3, a4
 ; ZVE32X-NEXT:    xor a1, a1, a2
 ; ZVE32X-NEXT:    xor a2, a5, a6
 ; ZVE32X-NEXT:    xor a0, a7, a0
 ; ZVE32X-NEXT:    snez a3, a3
 ; ZVE32X-NEXT:    snez a1, a1
-; ZVE32X-NEXT:    snez a2, a2
-; ZVE32X-NEXT:    snez a0, a0
-; ZVE32X-NEXT:    vmv.s.x v8, a3
+; ZVE32X-NEXT:    vmv.s.x v10, a3
 ; ZVE32X-NEXT:    vmv.s.x v11, a1
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vand.vi v8, v8, 1
-; ZVE32X-NEXT:    vand.vi v11, v11, 1
-; ZVE32X-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32X-NEXT:    vmsne.vi v8, v11, 0
-; ZVE32X-NEXT:    vmerge.vim v11, v9, 1, v0
-; ZVE32X-NEXT:    vmv1r.v v0, v8
+; ZVE32X-NEXT:    vand.vi v10, v10, 1
+; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
+; ZVE32X-NEXT:    vand.vi v10, v11, 1
+; ZVE32X-NEXT:    vmerge.vim v11, v8, 1, v0
+; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
+; ZVE32X-NEXT:    snez a1, a2
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmerge.vim v8, v10, 1, v0
+; ZVE32X-NEXT:    vmerge.vim v10, v9, 1, v0
 ; ZVE32X-NEXT:    vsetivli zero, 2, e8, mf4, tu, ma
-; ZVE32X-NEXT:    vslideup.vi v8, v11, 1
-; ZVE32X-NEXT:    vmv.s.x v11, a2
-; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vand.vi v11, v11, 1
+; ZVE32X-NEXT:    vslideup.vi v10, v11, 1
+; ZVE32X-NEXT:    vmv.s.x v11, a1
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmsne.vi v0, v8, 0
+; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmsne.vi v8, v11, 0
+; ZVE32X-NEXT:    vand.vi v10, v11, 1
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmerge.vim v11, v10, 1, v0
-; ZVE32X-NEXT:    vmv1r.v v0, v8
+; ZVE32X-NEXT:    vmerge.vim v11, v9, 1, v0
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmerge.vim v8, v9, 1, v0
+; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
+; ZVE32X-NEXT:    snez a0, a0
+; ZVE32X-NEXT:    vmerge.vim v10, v8, 1, v0
 ; ZVE32X-NEXT:    vsetivli zero, 3, e8, mf4, tu, ma
-; ZVE32X-NEXT:    vslideup.vi v11, v8, 2
-; ZVE32X-NEXT:    vmv.s.x v8, a0
-; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vand.vi v8, v8, 1
+; ZVE32X-NEXT:    vslideup.vi v11, v10, 2
+; ZVE32X-NEXT:    vmv.s.x v10, a0
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; ZVE32X-NEXT:    vmsne.vi v0, v11, 0
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmsne.vi v8, v8, 0
+; ZVE32X-NEXT:    vand.vi v10, v10, 1
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmerge.vim v10, v10, 1, v0
-; ZVE32X-NEXT:    vmv1r.v v0, v8
+; ZVE32X-NEXT:    vmerge.vim v9, v9, 1, v0
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmerge.vim v8, v9, 1, v0
-; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vslideup.vi v10, v8, 3
 ; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
+; ZVE32X-NEXT:    vmerge.vim v8, v8, 1, v0
+; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVE32X-NEXT:    vslideup.vi v9, v8, 3
+; ZVE32X-NEXT:    vmsne.vi v0, v9, 0
 ; ZVE32X-NEXT:    ret
 ;
 ; ZVE64X-LABEL: load_large_vector:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
index fa1377406d697..8833634be1a0e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
@@ -183,463 +183,406 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 81
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a3, a2, 6
+; RV32-NEXT:    add a2, a3, a2
 ; RV32-NEXT:    sub sp, sp, a2
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd1, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 81 * vlenb
-; RV32-NEXT:    addi a3, a1, 128
-; RV32-NEXT:    addi a4, a1, 256
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc1, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 65 * vlenb
+; RV32-NEXT:    addi a3, a1, 256
+; RV32-NEXT:    addi a4, a1, 128
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    lui a5, 12
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; RV32-NEXT:    vle32.v v16, (a4)
+; RV32-NEXT:    lui a5, 12291
+; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
+; RV32-NEXT:    vle32.v v24, (a1)
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    li a6, 41
+; RV32-NEXT:    mul a1, a1, a6
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    lui a1, %hi(.LCPI8_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_0)
+; RV32-NEXT:    vle16.v v4, (a1)
+; RV32-NEXT:    lui a1, 1
+; RV32-NEXT:    addi a5, a5, 3
+; RV32-NEXT:    vle32.v v8, (a4)
 ; RV32-NEXT:    csrr a4, vlenb
 ; RV32-NEXT:    li a6, 57
 ; RV32-NEXT:    mul a4, a4, a6
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a4, %hi(.LCPI8_0)
-; RV32-NEXT:    addi a4, a4, %lo(.LCPI8_0)
-; RV32-NEXT:    vmv.s.x v1, a5
-; RV32-NEXT:    lui a5, %hi(.LCPI8_1)
-; RV32-NEXT:    addi a5, a5, %lo(.LCPI8_1)
-; RV32-NEXT:    vle16.v v4, (a4)
-; RV32-NEXT:    lui a4, 1
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vslideup.vi v12, v16, 4
-; RV32-NEXT:    csrr a6, vlenb
-; RV32-NEXT:    li a7, 37
-; RV32-NEXT:    mul a6, a6, a7
-; RV32-NEXT:    add a6, sp, a6
-; RV32-NEXT:    addi a6, a6, 16
-; RV32-NEXT:    vs4r.v v12, (a6) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; RV32-NEXT:    vslidedown.vi v16, v16, 16
-; RV32-NEXT:    csrr a6, vlenb
-; RV32-NEXT:    li a7, 45
-; RV32-NEXT:    mul a6, a6, a7
-; RV32-NEXT:    add a6, sp, a6
-; RV32-NEXT:    addi a6, a6, 16
-; RV32-NEXT:    vs8r.v v16, (a6) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v1
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vslideup.vi v12, v16, 10, v0.t
-; RV32-NEXT:    vmv.v.v v28, v12
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vle16.v v24, (a5)
-; RV32-NEXT:    vle32.v v8, (a1)
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a5, a1, 6
-; RV32-NEXT:    add a1, a5, a1
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a1, a1, -64
 ; RV32-NEXT:    vle32.v v16, (a3)
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
-; RV32-NEXT:    mul a1, a1, a3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, a4, -64
+; RV32-NEXT:    vmv.s.x v3, a5
 ; RV32-NEXT:    vmv.s.x v0, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 5
-; RV32-NEXT:    add a1, a3, a1
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vrgatherei16.vv v16, v8, v4
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
+; RV32-NEXT:    li a3, 13
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 12, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v28, v16
+; RV32-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vcompress.vm v8, v24, v3
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 41
+; RV32-NEXT:    li a3, 57
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v28, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v8, v24, v4, v0.t
+; RV32-NEXT:    lui a1, 12
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 49
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vslideup.vi v12, v16, 4
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a4, a3, 4
+; RV32-NEXT:    add a3, a4, a3
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs4r.v v12, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; RV32-NEXT:    vslidedown.vi v24, v16, 16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 57
+; RV32-NEXT:    li a3, 25
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vslideup.vi v12, v8, 2
-; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    vslideup.vi v12, v24, 10, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 45
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a3, a1, 5
+; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vslideup.vi v12, v16, 8, v0.t
+; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetivli zero, 12, e32, m4, tu, ma
+; RV32-NEXT:    vmv.v.v v12, v8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 53
+; RV32-NEXT:    li a3, 21
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_2)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_2)
-; RV32-NEXT:    lui a3, %hi(.LCPI8_3)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_3)
-; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; RV32-NEXT:    vle16.v v12, (a1)
-; RV32-NEXT:    vle16.v v8, (a3)
+; RV32-NEXT:    lui a1, 49164
+; RV32-NEXT:    lui a3, %hi(.LCPI8_1)
+; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_1)
+; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
+; RV32-NEXT:    vle16.v v28, (a3)
+; RV32-NEXT:    addi a1, a1, 12
+; RV32-NEXT:    vmv.s.x v20, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 25
+; RV32-NEXT:    li a3, 41
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_4)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_4)
-; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV32-NEXT:    vle16.v v2, (a1)
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 6
-; RV32-NEXT:    add a1, a3, a1
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v24, v16, v12
+; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vcompress.vm v8, v0, v20
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 5
-; RV32-NEXT:    add a1, a3, a1
+; RV32-NEXT:    li a3, 13
+; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
+; RV32-NEXT:    li a3, 57
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v8, v16, v28, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 25
+; RV32-NEXT:    li a3, 49
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v4, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v24, v8, v4, v0.t
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
+; RV32-NEXT:    vslideup.vi v12, v16, 2
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 53
+; RV32-NEXT:    li a3, 25
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vslideup.vi v12, v24, 8, v0.t
 ; RV32-NEXT:    vsetivli zero, 12, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v8, v24
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 53
-; RV32-NEXT:    mul a1, a1, a3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vmv.v.v v12, v8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 57
+; RV32-NEXT:    li a3, 13
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v8, v24, v2
-; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    lui a1, 196656
+; RV32-NEXT:    lui a3, %hi(.LCPI8_2)
+; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_2)
+; RV32-NEXT:    li a4, 960
+; RV32-NEXT:    lui a5, %hi(.LCPI8_3)
+; RV32-NEXT:    addi a5, a5, %lo(.LCPI8_3)
+; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    vmv.s.x v0, a4
+; RV32-NEXT:    csrr a4, vlenb
+; RV32-NEXT:    add a4, sp, a4
+; RV32-NEXT:    addi a4, a4, 16
+; RV32-NEXT:    vs1r.v v0, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vle16.v v4, (a3)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle16.v v8, (a5)
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a4, a3, 3
+; RV32-NEXT:    add a3, a4, a3
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs2r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vmv.s.x v22, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 45
+; RV32-NEXT:    li a3, 41
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vslideup.vi v8, v24, 6, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 5
-; RV32-NEXT:    add a1, a3, a1
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_5)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_5)
-; RV32-NEXT:    lui a3, %hi(.LCPI8_6)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_6)
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vle16.v v24, (a1)
-; RV32-NEXT:    vle16.v v4, (a3)
-; RV32-NEXT:    li a1, 960
-; RV32-NEXT:    vmv.s.x v28, a1
-; RV32-NEXT:    vrgatherei16.vv v8, v16, v24
-; RV32-NEXT:    vmv1r.v v0, v28
+; RV32-NEXT:    vcompress.vm v8, v24, v22
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
+; RV32-NEXT:    li a3, 57
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vrgatherei16.vv v8, v16, v4, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 25
+; RV32-NEXT:    li a3, 49
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_7)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_7)
-; RV32-NEXT:    lui a3, %hi(.LCPI8_8)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_8)
-; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV32-NEXT:    vle16.v v8, (a1)
-; RV32-NEXT:    lui a1, %hi(.LCPI8_9)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_9)
-; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; RV32-NEXT:    vle16.v v12, (a3)
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 13
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 16
-; RV32-NEXT:    vs4r.v v12, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vle16.v v12, (a1)
+; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 2
+; RV32-NEXT:    slli a3, a1, 3
 ; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 57
-; RV32-NEXT:    mul a1, a1, a3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl2r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v20, v16, v8
-; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    vrgatherei16.vv v12, v0, v16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 45
+; RV32-NEXT:    li a3, 25
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vmv4r.v v24, v8
-; RV32-NEXT:    vslideup.vi v20, v8, 4, v0.t
+; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 21
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a3, a1, 5
+; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vslideup.vi v12, v16, 6, v0.t
+; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
+; RV32-NEXT:    vmv.v.v v12, v8
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 6
+; RV32-NEXT:    slli a3, a1, 3
 ; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    lui a1, 786624
+; RV32-NEXT:    lui a3, %hi(.LCPI8_4)
+; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_4)
+; RV32-NEXT:    lui a4, %hi(.LCPI8_5)
+; RV32-NEXT:    addi a4, a4, %lo(.LCPI8_5)
+; RV32-NEXT:    addi a1, a1, 192
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vle16.v v8, (a3)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle16.v v12, (a4)
+; RV32-NEXT:    vmv.s.x v14, a1
+; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
+; RV32-NEXT:    vcompress.vm v16, v24, v14
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 13
-; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v8, v0, v16
-; RV32-NEXT:    vmv1r.v v0, v28
+; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
+; RV32-NEXT:    li a3, 57
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v16, v24, v8, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 2
-; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v8, v16, v28, v0.t
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 13
+; RV32-NEXT:    li a3, 49
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_10)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_10)
+; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vle16.v v12, (a1)
-; RV32-NEXT:    lui a1, 15
-; RV32-NEXT:    vmv.s.x v3, a1
+; RV32-NEXT:    vrgatherei16.vv v4, v0, v12
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 57
+; RV32-NEXT:    li a3, 25
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vslideup.vi v8, v16, 6
-; RV32-NEXT:    vmv1r.v v0, v3
-; RV32-NEXT:    vrgatherei16.vv v8, v24, v12, v0.t
+; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 57
-; RV32-NEXT:    mul a1, a1, a3
+; RV32-NEXT:    slli a3, a1, 5
+; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs4r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_11)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_11)
-; RV32-NEXT:    lui a3, %hi(.LCPI8_12)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_12)
+; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vslideup.vi v4, v8, 4, v0.t
+; RV32-NEXT:    lui a1, 768
+; RV32-NEXT:    lui a3, %hi(.LCPI8_6)
+; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_6)
+; RV32-NEXT:    li a4, 1008
+; RV32-NEXT:    addi a1, a1, 768
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vle16.v v28, (a1)
-; RV32-NEXT:    vle16.v v4, (a3)
-; RV32-NEXT:    li a1, 1008
-; RV32-NEXT:    vmv.s.x v0, a1
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 2
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vle16.v v8, (a3)
+; RV32-NEXT:    vmv.s.x v1, a4
+; RV32-NEXT:    vmv.s.x v12, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 6
-; RV32-NEXT:    add a1, a3, a1
+; RV32-NEXT:    li a3, 41
+; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v8, v16, v28
+; RV32-NEXT:    vcompress.vm v24, v16, v12
+; RV32-NEXT:    vmv1r.v v0, v1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 73
+; RV32-NEXT:    li a3, 57
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v8, v16, v4, v0.t
+; RV32-NEXT:    vrgatherei16.vv v24, v16, v8, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 2
-; RV32-NEXT:    add a1, a3, a1
+; RV32-NEXT:    li a3, 25
+; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    lui a1, %hi(.LCPI8_13)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_13)
-; RV32-NEXT:    lui a3, %hi(.LCPI8_14)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_14)
-; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    lui a1, %hi(.LCPI8_7)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_7)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
 ; RV32-NEXT:    vle16.v v8, (a1)
-; RV32-NEXT:    lui a1, %hi(.LCPI8_15)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_15)
-; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; RV32-NEXT:    vle16.v v28, (a3)
-; RV32-NEXT:    vle16.v v12, (a1)
+; RV32-NEXT:    lui a1, 15
+; RV32-NEXT:    vmv.s.x v0, a1
 ; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv1r.v v0, v3
+; RV32-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 37
+; RV32-NEXT:    li a3, 49
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vslideup.vi v20, v16, 6
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a3, a1, 5
 ; RV32-NEXT:    add a1, a3, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v20, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v20, v24, v8, v0.t
+; RV32-NEXT:    lui a1, 3073
+; RV32-NEXT:    lui a3, %hi(.LCPI8_8)
+; RV32-NEXT:    addi a3, a3, %lo(.LCPI8_8)
+; RV32-NEXT:    lui a4, %hi(.LCPI8_9)
+; RV32-NEXT:    addi a4, a4, %lo(.LCPI8_9)
+; RV32-NEXT:    addi a1, a1, -1024
+; RV32-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
+; RV32-NEXT:    vle16.v v16, (a3)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle16.v v2, (a4)
+; RV32-NEXT:    vmv.s.x v0, a1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a3, 25
+; RV32-NEXT:    li a3, 41
 ; RV32-NEXT:    mul a1, a1, a3
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v20, v8
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a3, a1, 6
-; RV32-NEXT:    add a1, a3, a1
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v8, v0, v28
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 2
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vcompress.vm v8, v24, v0
+; RV32-NEXT:    vmv1r.v v0, v1
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 73
+; RV32-NEXT:    li a2, 57
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vrgatherei16.vv v8, v24, v16, v0.t
 ; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vl4r.v v4, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vrgatherei16.vv v8, v24, v4, v0.t
+; RV32-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 21
-; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    slli a2, a1, 4
+; RV32-NEXT:    add a1, a2, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 13
-; RV32-NEXT:    mul a1, a1, a2
+; RV32-NEXT:    slli a2, a1, 5
+; RV32-NEXT:    add a1, a2, a1
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
-; RV32-NEXT:    vmv.v.v v28, v0
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, mu
+; RV32-NEXT:    vrgatherei16.vv v12, v24, v2, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 57
-; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl4r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
+; RV32-NEXT:    vmv.v.v v4, v24
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a2, a1, 2
-; RV32-NEXT:    add a1, a2, a1
+; RV32-NEXT:    li a2, 25
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vmv.v.v v24, v0
-; RV32-NEXT:    vmv.v.v v16, v8
+; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vmv.v.v v20, v24
+; RV32-NEXT:    vmv.v.v v12, v8
 ; RV32-NEXT:    addi a1, a0, 320
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v16, (a1)
+; RV32-NEXT:    vse32.v v12, (a1)
 ; RV32-NEXT:    addi a1, a0, 256
-; RV32-NEXT:    vse32.v v24, (a1)
+; RV32-NEXT:    vse32.v v20, (a1)
 ; RV32-NEXT:    addi a1, a0, 192
-; RV32-NEXT:    vse32.v v28, (a1)
+; RV32-NEXT:    vse32.v v4, (a1)
 ; RV32-NEXT:    addi a1, a0, 128
-; RV32-NEXT:    vse32.v v20, (a1)
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a3, a2, 3
+; RV32-NEXT:    add a2, a3, a2
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vse32.v v8, (a1)
 ; RV32-NEXT:    addi a1, a0, 64
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 53
+; RV32-NEXT:    li a3, 13
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vse32.v v8, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 41
+; RV32-NEXT:    li a2, 21
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 81
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a1, a0, 6
+; RV32-NEXT:    add a0, a1, a0
 ; RV32-NEXT:    add sp, sp, a0
 ; RV32-NEXT:    .cfi_def_cfa sp, 16
 ; RV32-NEXT:    addi sp, sp, 16
@@ -651,366 +594,457 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    csrr a2, vlenb
-; RV64-NEXT:    slli a2, a2, 6
+; RV64-NEXT:    li a3, 67
+; RV64-NEXT:    mul a2, a2, a3
 ; RV64-NEXT:    sub sp, sp, a2
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 64 * vlenb
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc3, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 67 * vlenb
 ; RV64-NEXT:    addi a2, a1, 128
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a1)
 ; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    li a4, 48
+; RV64-NEXT:    li a4, 59
 ; RV64-NEXT:    mul a3, a3, a4
 ; RV64-NEXT:    add a3, sp, a3
 ; RV64-NEXT:    addi a3, a3, 16
 ; RV64-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV64-NEXT:    addi a1, a1, 256
-; RV64-NEXT:    vle64.v v16, (a1)
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a3, 20
-; RV64-NEXT:    mul a1, a1, a3
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    li a1, 128
-; RV64-NEXT:    vid.v v10
-; RV64-NEXT:    vmv.s.x v1, a1
-; RV64-NEXT:    li a1, 6
-; RV64-NEXT:    vmul.vx v2, v10, a1
-; RV64-NEXT:    li a1, 56
+; RV64-NEXT:    li a3, 128
+; RV64-NEXT:    vle64.v v24, (a1)
+; RV64-NEXT:    lui a1, 1
+; RV64-NEXT:    vid.v v8
+; RV64-NEXT:    vmv.s.x v0, a3
+; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    li a4, 30
+; RV64-NEXT:    mul a3, a3, a4
+; RV64-NEXT:    add a3, sp, a3
+; RV64-NEXT:    addi a3, a3, 16
+; RV64-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    li a3, 6
+; RV64-NEXT:    vmul.vx v6, v8, a3
+; RV64-NEXT:    li a3, 56
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vrgather.vi v12, v16, 4
+; RV64-NEXT:    vrgather.vi v8, v24, 4
+; RV64-NEXT:    csrr a4, vlenb
+; RV64-NEXT:    li a5, 22
+; RV64-NEXT:    mul a4, a4, a5
+; RV64-NEXT:    add a4, sp, a4
+; RV64-NEXT:    addi a4, a4, 16
+; RV64-NEXT:    vs8r.v v24, (a4) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
-; RV64-NEXT:    vslidedown.vi v16, v16, 8
+; RV64-NEXT:    vslidedown.vi v16, v24, 8
+; RV64-NEXT:    csrr a4, vlenb
+; RV64-NEXT:    li a5, 39
+; RV64-NEXT:    mul a4, a4, a5
+; RV64-NEXT:    add a4, sp, a4
+; RV64-NEXT:    addi a4, a4, 16
+; RV64-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
+; RV64-NEXT:    vrgather.vi v8, v16, 2, v0.t
+; RV64-NEXT:    vmv.v.v v20, v8
+; RV64-NEXT:    vmv.s.x v8, a3
 ; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    li a4, 36
+; RV64-NEXT:    li a4, 55
 ; RV64-NEXT:    mul a3, a3, a4
 ; RV64-NEXT:    add a3, sp, a3
 ; RV64-NEXT:    addi a3, a3, 16
-; RV64-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv1r.v v0, v1
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v12, v16, 2, v0.t
+; RV64-NEXT:    vs1r.v v8, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    addi a3, a1, 65
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vle64.v v16, (a2)
+; RV64-NEXT:    vle64.v v8, (a2)
 ; RV64-NEXT:    csrr a2, vlenb
-; RV64-NEXT:    li a3, 56
+; RV64-NEXT:    li a4, 47
+; RV64-NEXT:    mul a2, a2, a4
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv.s.x v16, a3
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 35
 ; RV64-NEXT:    mul a2, a2, a3
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    addi a2, a2, 16
-; RV64-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv.s.x v7, a1
+; RV64-NEXT:    vs1r.v v16, (a2) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v10, v2, -16
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 48
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vadd.vi v16, v6, -16
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a3, a2, 5
+; RV64-NEXT:    sub a2, a3, a2
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs2r.v v16, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv2r.v v18, v6
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 12
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs2r.v v6, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 59
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v0, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 35
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl1r.v v16, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v24, v16, v2
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v24, v16, v10, v0.t
+; RV64-NEXT:    vcompress.vm v24, v0, v16
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl1r.v v0, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a3, a2, 5
+; RV64-NEXT:    sub a2, a3, a2
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl2r.v v16, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v24, v8, v16, v0.t
 ; RV64-NEXT:    vsetivli zero, 6, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v12, v24
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 20
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.v v20, v24
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 18
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v20, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 22
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v12, v16, 5
-; RV64-NEXT:    vmv1r.v v0, v1
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 36
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgather.vi v12, v16, 3, v0.t
-; RV64-NEXT:    vmv.v.v v20, v12
+; RV64-NEXT:    vrgather.vi v8, v24, 5
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 30
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl1r.v v0, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 39
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgather.vi v8, v24, 3, v0.t
+; RV64-NEXT:    vmv.v.v v20, v8
+; RV64-NEXT:    lui a2, 2
+; RV64-NEXT:    addi a2, a2, 130
+; RV64-NEXT:    vmv.s.x v8, a2
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v16, v2, 1
-; RV64-NEXT:    vadd.vi v18, v2, -15
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 48
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vadd.vi v16, v18, -15
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 59
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v0, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v24, v8, v16
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v24, v8, v18, v0.t
+; RV64-NEXT:    vcompress.vm v24, v0, v8
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl1r.v v0, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 47
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v24, v8, v16, v0.t
 ; RV64-NEXT:    vsetivli zero, 6, e64, m4, tu, ma
 ; RV64-NEXT:    vmv.v.v v20, v24
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 12
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addi a1, a1, 7
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 14
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v20, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    lui a2, 16
+; RV64-NEXT:    addi a2, a2, 7
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.i v9, 6
-; RV64-NEXT:    vmv.v.x v10, a1
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 20
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.i v8, 6
+; RV64-NEXT:    vmv.v.x v9, a2
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 22
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v12, v16, v8
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgatherei16.vv v12, v16, v9
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 44
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vrgatherei16.vv v12, v16, v10
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a3, a2, 5
+; RV64-NEXT:    sub a2, a3, a2
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv4r.v v8, v16
 ; RV64-NEXT:    vrgather.vi v12, v16, 2
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 35
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
 ; RV64-NEXT:    vrgather.vi v12, v16, 3
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 28
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    li a1, 24
-; RV64-NEXT:    vmv.s.x v7, a1
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    lui a2, 4
+; RV64-NEXT:    li a3, 24
+; RV64-NEXT:    addi a2, a2, 260
+; RV64-NEXT:    vmv.s.x v0, a3
+; RV64-NEXT:    addi a3, sp, 16
+; RV64-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv.s.x v24, a2
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 12
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl2r.v v2, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v10, v2, 2
-; RV64-NEXT:    vadd.vi v4, v2, -14
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 48
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vadd.vi v6, v2, -14
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 59
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v24, v16, v10
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v24, v8, v4, v0.t
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 20
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vcompress.vm v8, v16, v24
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 47
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v8, v16, v6, v0.t
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 22
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 30
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl1r.v v1, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vmv1r.v v0, v1
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 36
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 44
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v28, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 39
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl4r.v v28, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
 ; RV64-NEXT:    vrgather.vi v28, v24, 4, v0.t
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 44
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v28, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v28, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    lui a2, 8
+; RV64-NEXT:    addi a2, a2, 520
+; RV64-NEXT:    vmv.s.x v7, a2
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v30, v2, 3
-; RV64-NEXT:    vadd.vi v28, v2, -13
+; RV64-NEXT:    vadd.vi v4, v2, -13
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 59
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v8, v16, v30
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v8, v16, v28, v0.t
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    vcompress.vm v8, v24, v7
+; RV64-NEXT:    addi a2, sp, 16
+; RV64-NEXT:    vl1r.v v0, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v8, v16, v4, v0.t
+; RV64-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v1
-; RV64-NEXT:    vmv4r.v v16, v24
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v4, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 39
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a3, a2, 5
+; RV64-NEXT:    sub a2, a3, a2
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgather.vi v4, v24, 5, v0.t
-; RV64-NEXT:    lui a1, 96
-; RV64-NEXT:    li a2, 192
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vrgather.vi v8, v16, 5, v0.t
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a3, a2, 5
+; RV64-NEXT:    sub a2, a3, a2
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v8, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    lui a2, 96
+; RV64-NEXT:    li a3, 192
+; RV64-NEXT:    vmv.s.x v1, a3
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v9, a1
-; RV64-NEXT:    vmv1r.v v0, v8
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.x v8, a2
+; RV64-NEXT:    vmv1r.v v0, v1
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 35
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl4r.v v12, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v12, v24, v9, v0.t
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    li a1, 28
-; RV64-NEXT:    vmv.s.x v1, a1
+; RV64-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 35
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    li a2, 1040
+; RV64-NEXT:    li a3, 28
+; RV64-NEXT:    vmv.s.x v20, a2
+; RV64-NEXT:    vmv.s.x v0, a3
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 30
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs1r.v v0, (a2) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v10, v2, 4
-; RV64-NEXT:    vadd.vi v12, v2, -12
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 48
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vadd.vi v22, v2, -12
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 59
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v16, v24, v10
-; RV64-NEXT:    vmv1r.v v0, v1
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vrgatherei16.vv v16, v24, v12, v0.t
-; RV64-NEXT:    lui a1, 112
-; RV64-NEXT:    addi a1, a1, 1
+; RV64-NEXT:    vcompress.vm v8, v24, v20
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 47
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v8, v24, v22, v0.t
+; RV64-NEXT:    lui a2, 112
+; RV64-NEXT:    addi a2, a2, 1
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v9, a1
-; RV64-NEXT:    vmv1r.v v0, v8
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 28
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 36
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.x v12, a2
+; RV64-NEXT:    vmv1r.v v0, v1
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl4r.v v4, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v12, v24, v9, v0.t
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 28
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v12, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 44
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v20, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 20
-; RV64-NEXT:    mul a1, a1, a2
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vrgatherei16.vv v4, v16, v12, v0.t
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl4r.v v12, (a2) # Unknown-size Folded Reload
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 22
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v20, v8
+; RV64-NEXT:    vmv.v.v v12, v16
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    li a3, 55
+; RV64-NEXT:    mul a2, a2, a3
+; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    addi a2, a2, 16
+; RV64-NEXT:    vs4r.v v12, (a2) # Unknown-size Folded Spill
+; RV64-NEXT:    addi a1, a1, -2016
+; RV64-NEXT:    vmv.s.x v12, a1
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 44
+; RV64-NEXT:    li a2, 59
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs4r.v v20, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v20, v2, 5
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-NEXT:    vcompress.vm v16, v24, v12
+; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; RV64-NEXT:    vadd.vi v12, v2, -11
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 48
+; RV64-NEXT:    li a2, 30
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-NEXT:    vrgatherei16.vv v8, v24, v20
-; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; RV64-NEXT:    vadd.vi v20, v2, -11
-; RV64-NEXT:    vmv1r.v v0, v1
+; RV64-NEXT:    vl1r.v v0, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 56
+; RV64-NEXT:    li a2, 47
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v8, v24, v20, v0.t
-; RV64-NEXT:    vmv4r.v v12, v4
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vl8r.v v0, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
-; RV64-NEXT:    vmv.v.v v12, v0
+; RV64-NEXT:    vrgatherei16.vv v16, v24, v12, v0.t
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 5
+; RV64-NEXT:    slli a2, a1, 5
+; RV64-NEXT:    sub a1, a2, a1
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v20, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmv.v.v v20, v16
+; RV64-NEXT:    vl4r.v v12, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetivli zero, 5, e64, m4, tu, ma
+; RV64-NEXT:    vmv.v.v v12, v24
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 28
+; RV64-NEXT:    li a2, 35
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vl4r.v v16, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    vmv.v.v v16, v8
-; RV64-NEXT:    addi a1, a0, 320
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vse64.v v16, (a1)
+; RV64-NEXT:    vl4r.v v20, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vmv.v.v v20, v8
+; RV64-NEXT:    vmv4r.v v8, v4
+; RV64-NEXT:    vmv.v.v v8, v16
 ; RV64-NEXT:    addi a1, a0, 256
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vse64.v v20, (a1)
+; RV64-NEXT:    addi a1, a0, 320
+; RV64-NEXT:    vse64.v v8, (a1)
 ; RV64-NEXT:    addi a1, a0, 192
 ; RV64-NEXT:    vse64.v v12, (a1)
 ; RV64-NEXT:    addi a1, a0, 128
 ; RV64-NEXT:    csrr a2, vlenb
-; RV64-NEXT:    li a3, 44
+; RV64-NEXT:    li a3, 55
 ; RV64-NEXT:    mul a2, a2, a3
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    addi a2, a2, 16
@@ -1018,20 +1052,22 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vse64.v v8, (a1)
 ; RV64-NEXT:    addi a1, a0, 64
 ; RV64-NEXT:    csrr a2, vlenb
-; RV64-NEXT:    li a3, 12
+; RV64-NEXT:    li a3, 14
 ; RV64-NEXT:    mul a2, a2, a3
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    addi a2, a2, 16
 ; RV64-NEXT:    vl4r.v v8, (a2) # Unknown-size Folded Reload
 ; RV64-NEXT:    vse64.v v8, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
+; RV64-NEXT:    li a2, 18
+; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl4r.v v8, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vse64.v v8, (a0)
 ; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a0, a0, 6
+; RV64-NEXT:    li a1, 67
+; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    add sp, sp, a0
 ; RV64-NEXT:    .cfi_def_cfa sp, 16
 ; RV64-NEXT:    addi sp, sp, 16
@@ -1211,3 +1247,93 @@ define <4 x i8> @load_factor8_one_active(ptr %ptr) vscale_range(8,1024) {
   %v0 = shufflevector <32 x i8> %interleaved.vec, <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
   ret <4 x i8> %v0
 }
+
+define void @load_factor4_one_active_storeback(ptr %ptr) {
+; CHECK-LABEL: load_factor4_one_active_storeback:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vlse32.v v8, (a0), a1
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    ret
+  %interleaved.vec = load <16 x i32>, ptr %ptr
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  store <4 x i32> %v0, ptr %ptr
+  ret void
+}
+
+; TODO: This should be a strided load
+define void @load_factor4_one_active_storeback_full(ptr %ptr) {
+; CHECK-LABEL: load_factor4_one_active_storeback_full:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v12, v8, 4
+; CHECK-NEXT:    vmv1r.v v13, v8
+; CHECK-NEXT:    vmv1r.v v14, v12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v16, v8, 8
+; CHECK-NEXT:    vmv1r.v v15, v16
+; CHECK-NEXT:    vslidedown.vi v16, v8, 12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsseg4e32.v v13, (a0)
+; CHECK-NEXT:    ret
+  %interleaved.vec = load <16 x i32>, ptr %ptr
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  store <16 x i32> %v0, ptr %ptr
+  ret void
+}
+
+define void @store_factor4_one_active(ptr %ptr, <4 x i32> %v) {
+; CHECK-LABEL: store_factor4_one_active:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsse32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %v0 = shufflevector <4 x i32> %v, <4 x i32> poison, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3,  i32 undef, i32 undef, i32 undef>
+  store <16 x i32> %v0, ptr %ptr
+  ret void
+}
+
+define void @store_factor4_one_active_idx1(ptr %ptr, <4 x i32> %v) {
+; CHECK-LABEL: store_factor4_one_active_idx1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, 4
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsse32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %v0 = shufflevector <4 x i32> %v, <4 x i32> poison, <16 x i32> <i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3,  i32 undef, i32 undef>
+  store <16 x i32> %v0, ptr %ptr
+  ret void
+}
+
+define void @store_factor4_one_active_fullwidth(ptr %ptr, <16 x i32> %v) {
+; CHECK-LABEL: store_factor4_one_active_fullwidth:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
+; CHECK-NEXT:    vsse32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %v0 = shufflevector <16 x i32> %v, <16 x i32> poison, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3,  i32 undef, i32 undef, i32 undef>
+  store <16 x i32> %v0, ptr %ptr
+  ret void
+}
+
+; TODO: This could be a vslidedown followed by a strided store
+define void @store_factor4_one_active_slidedown(ptr %ptr, <4 x i32> %v) {
+; CHECK-LABEL: store_factor4_one_active_slidedown:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v9, v8, 1
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vmv.v.v v11, v10
+; CHECK-NEXT:    vmv.v.v v12, v10
+; CHECK-NEXT:    vsseg4e32.v v9, (a0)
+; CHECK-NEXT:    ret
+  %v0 = shufflevector <4 x i32> %v, <4 x i32> poison, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 4,  i32 undef, i32 undef, i32 undef>
+  store <16 x i32> %v0, ptr %ptr
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 6cc3f7e76797b..141d54cf585f2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -556,11 +556,13 @@ define <4 x i8> @mgather_truemask_v4i8(<4 x ptr> %ptrs, <4 x i8> %passthru) {
 define <4 x i8> @mgather_falsemask_v4i8(<4 x ptr> %ptrs, <4 x i8> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4i8:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4i8:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -733,13 +735,13 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB12_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB12_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB12_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB12_15
 ; RV64ZVE32F-NEXT:  .LBB12_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB12_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB12_16
 ; RV64ZVE32F-NEXT:  .LBB12_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB12_9
@@ -756,14 +758,31 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB12_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB12_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e8, mf2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB12_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB12_16
-; RV64ZVE32F-NEXT:  .LBB12_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB12_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lbu a0, 0(a0)
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB12_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB12_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB12_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
@@ -772,7 +791,7 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB12_6
-; RV64ZVE32F-NEXT:  .LBB12_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB12_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -783,7 +802,7 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB12_7
-; RV64ZVE32F-NEXT:  .LBB12_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB12_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 5, e8, mf2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -793,26 +812,6 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB12_8
 ; RV64ZVE32F-NEXT:    j .LBB12_9
-; RV64ZVE32F-NEXT:  .LBB12_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e8, mf2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB12_11
-; RV64ZVE32F-NEXT:  .LBB12_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lbu a0, 0(a0)
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i8, ptr %base, <8 x i8> %idxs
   %v = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> %ptrs, i32 1, <8 x i1> %m, <8 x i8> %passthru)
   ret <8 x i8> %v
@@ -1253,11 +1252,13 @@ define <4 x i16> @mgather_truemask_v4i16(<4 x ptr> %ptrs, <4 x i16> %passthru) {
 define <4 x i16> @mgather_falsemask_v4i16(<4 x ptr> %ptrs, <4 x i16> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4i16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4i16:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -1435,13 +1436,13 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB23_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB23_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB23_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB23_15
 ; RV64ZVE32F-NEXT:  .LBB23_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB23_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB23_16
 ; RV64ZVE32F-NEXT:  .LBB23_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB23_9
@@ -1460,14 +1461,35 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB23_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB23_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB23_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB23_16
-; RV64ZVE32F-NEXT:  .LBB23_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB23_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB23_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB23_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB23_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -1478,7 +1500,7 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB23_6
-; RV64ZVE32F-NEXT:  .LBB23_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB23_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -1491,7 +1513,7 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB23_7
-; RV64ZVE32F-NEXT:  .LBB23_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB23_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1504,30 +1526,6 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB23_8
 ; RV64ZVE32F-NEXT:    j .LBB23_9
-; RV64ZVE32F-NEXT:  .LBB23_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB23_11
-; RV64ZVE32F-NEXT:  .LBB23_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i8> %idxs
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
   ret <8 x i16> %v
@@ -1587,13 +1585,13 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB24_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB24_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB24_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB24_15
 ; RV64ZVE32F-NEXT:  .LBB24_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB24_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB24_16
 ; RV64ZVE32F-NEXT:  .LBB24_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB24_9
@@ -1612,14 +1610,35 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB24_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB24_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB24_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB24_16
-; RV64ZVE32F-NEXT:  .LBB24_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB24_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB24_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB24_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB24_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -1630,7 +1649,7 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB24_6
-; RV64ZVE32F-NEXT:  .LBB24_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB24_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -1643,7 +1662,7 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB24_7
-; RV64ZVE32F-NEXT:  .LBB24_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB24_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1656,30 +1675,6 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB24_8
 ; RV64ZVE32F-NEXT:    j .LBB24_9
-; RV64ZVE32F-NEXT:  .LBB24_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB24_11
-; RV64ZVE32F-NEXT:  .LBB24_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %eidxs = sext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i16> %eidxs
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
@@ -1740,13 +1735,13 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB25_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB25_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB25_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB25_15
 ; RV64ZVE32F-NEXT:  .LBB25_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB25_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB25_16
 ; RV64ZVE32F-NEXT:  .LBB25_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB25_9
@@ -1766,14 +1761,37 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB25_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB25_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB25_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB25_16
-; RV64ZVE32F-NEXT:  .LBB25_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB25_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB25_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB25_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB25_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1785,7 +1803,7 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB25_6
-; RV64ZVE32F-NEXT:  .LBB25_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB25_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -1799,7 +1817,7 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB25_7
-; RV64ZVE32F-NEXT:  .LBB25_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB25_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -1813,32 +1831,6 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB25_8
 ; RV64ZVE32F-NEXT:    j .LBB25_9
-; RV64ZVE32F-NEXT:  .LBB25_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB25_11
-; RV64ZVE32F-NEXT:  .LBB25_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i16> %eidxs
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
@@ -1896,13 +1888,13 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB26_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB26_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB26_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB26_15
 ; RV64ZVE32F-NEXT:  .LBB26_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB26_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB26_16
 ; RV64ZVE32F-NEXT:  .LBB26_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB26_9
@@ -1920,14 +1912,33 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB26_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB26_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB26_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB26_16
-; RV64ZVE32F-NEXT:  .LBB26_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB26_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB26_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB26_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB26_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -1937,7 +1948,7 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB26_6
-; RV64ZVE32F-NEXT:  .LBB26_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB26_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -1949,7 +1960,7 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB26_7
-; RV64ZVE32F-NEXT:  .LBB26_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB26_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1960,28 +1971,6 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB26_8
 ; RV64ZVE32F-NEXT:    j .LBB26_9
-; RV64ZVE32F-NEXT:  .LBB26_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB26_11
-; RV64ZVE32F-NEXT:  .LBB26_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i16> %idxs
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
   ret <8 x i16> %v
@@ -2311,11 +2300,13 @@ define <4 x i32> @mgather_truemask_v4i32(<4 x ptr> %ptrs, <4 x i32> %passthru) {
 define <4 x i32> @mgather_falsemask_v4i32(<4 x ptr> %ptrs, <4 x i32> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4i32:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4i32:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -2492,13 +2483,13 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB35_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB35_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB35_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB35_15
 ; RV64ZVE32F-NEXT:  .LBB35_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB35_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB35_16
 ; RV64ZVE32F-NEXT:  .LBB35_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB35_9
@@ -2517,14 +2508,35 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB35_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB35_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB35_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB35_16
-; RV64ZVE32F-NEXT:  .LBB35_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB35_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB35_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB35_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB35_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -2535,7 +2547,7 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB35_6
-; RV64ZVE32F-NEXT:  .LBB35_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB35_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -2548,7 +2560,7 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB35_7
-; RV64ZVE32F-NEXT:  .LBB35_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB35_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2561,30 +2573,6 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB35_8
 ; RV64ZVE32F-NEXT:    j .LBB35_9
-; RV64ZVE32F-NEXT:  .LBB35_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB35_11
-; RV64ZVE32F-NEXT:  .LBB35_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i8> %idxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
   ret <8 x i32> %v
@@ -2643,13 +2631,13 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB36_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB36_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB36_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB36_15
 ; RV64ZVE32F-NEXT:  .LBB36_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB36_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB36_16
 ; RV64ZVE32F-NEXT:  .LBB36_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB36_9
@@ -2668,14 +2656,35 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB36_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB36_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB36_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB36_16
-; RV64ZVE32F-NEXT:  .LBB36_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB36_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB36_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB36_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB36_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -2686,7 +2695,7 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB36_6
-; RV64ZVE32F-NEXT:  .LBB36_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB36_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -2699,7 +2708,7 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB36_7
-; RV64ZVE32F-NEXT:  .LBB36_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB36_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2712,30 +2721,6 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB36_8
 ; RV64ZVE32F-NEXT:    j .LBB36_9
-; RV64ZVE32F-NEXT:  .LBB36_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB36_11
-; RV64ZVE32F-NEXT:  .LBB36_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = sext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
@@ -2798,13 +2783,13 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB37_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB37_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB37_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB37_15
 ; RV64ZVE32F-NEXT:  .LBB37_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB37_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB37_16
 ; RV64ZVE32F-NEXT:  .LBB37_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB37_9
@@ -2824,14 +2809,37 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB37_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB37_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB37_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB37_16
-; RV64ZVE32F-NEXT:  .LBB37_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB37_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB37_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB37_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB37_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2843,7 +2851,7 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB37_6
-; RV64ZVE32F-NEXT:  .LBB37_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB37_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -2857,7 +2865,7 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB37_7
-; RV64ZVE32F-NEXT:  .LBB37_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB37_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -2871,32 +2879,6 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB37_8
 ; RV64ZVE32F-NEXT:    j .LBB37_9
-; RV64ZVE32F-NEXT:  .LBB37_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB37_11
-; RV64ZVE32F-NEXT:  .LBB37_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
@@ -2957,13 +2939,13 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB38_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB38_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB38_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB38_15
 ; RV64ZVE32F-NEXT:  .LBB38_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB38_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB38_16
 ; RV64ZVE32F-NEXT:  .LBB38_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB38_9
@@ -2982,14 +2964,35 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB38_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB38_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB38_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB38_16
-; RV64ZVE32F-NEXT:  .LBB38_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB38_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB38_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB38_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB38_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -3000,7 +3003,7 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB38_6
-; RV64ZVE32F-NEXT:  .LBB38_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB38_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -3013,7 +3016,7 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB38_7
-; RV64ZVE32F-NEXT:  .LBB38_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB38_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -3026,30 +3029,6 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB38_8
 ; RV64ZVE32F-NEXT:    j .LBB38_9
-; RV64ZVE32F-NEXT:  .LBB38_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB38_11
-; RV64ZVE32F-NEXT:  .LBB38_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i16> %idxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
   ret <8 x i32> %v
@@ -3109,13 +3088,13 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB39_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB39_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB39_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB39_15
 ; RV64ZVE32F-NEXT:  .LBB39_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB39_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB39_16
 ; RV64ZVE32F-NEXT:  .LBB39_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB39_9
@@ -3134,14 +3113,35 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB39_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB39_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB39_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB39_16
-; RV64ZVE32F-NEXT:  .LBB39_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB39_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB39_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB39_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB39_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -3152,7 +3152,7 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB39_6
-; RV64ZVE32F-NEXT:  .LBB39_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB39_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -3165,7 +3165,7 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB39_7
-; RV64ZVE32F-NEXT:  .LBB39_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB39_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -3178,30 +3178,6 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB39_8
 ; RV64ZVE32F-NEXT:    j .LBB39_9
-; RV64ZVE32F-NEXT:  .LBB39_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB39_11
-; RV64ZVE32F-NEXT:  .LBB39_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = sext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
@@ -3265,13 +3241,13 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a3, a2, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a3, .LBB40_12
+; RV64ZVE32F-NEXT:    bnez a3, .LBB40_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
-; RV64ZVE32F-NEXT:    bnez a3, .LBB40_13
+; RV64ZVE32F-NEXT:    bnez a3, .LBB40_15
 ; RV64ZVE32F-NEXT:  .LBB40_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
-; RV64ZVE32F-NEXT:    bnez a3, .LBB40_14
+; RV64ZVE32F-NEXT:    bnez a3, .LBB40_16
 ; RV64ZVE32F-NEXT:  .LBB40_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB40_9
@@ -3291,14 +3267,37 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a3, a2, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a3, .LBB40_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a3, .LBB40_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV64ZVE32F-NEXT:    and a3, a3, a1
+; RV64ZVE32F-NEXT:    slli a3, a3, 2
+; RV64ZVE32F-NEXT:    add a3, a0, a3
+; RV64ZVE32F-NEXT:    lw a3, 0(a3)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a3
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB40_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
-; RV64ZVE32F-NEXT:    bnez a2, .LBB40_16
-; RV64ZVE32F-NEXT:  .LBB40_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a2, .LBB40_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    and a1, a2, a1
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB40_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB40_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB40_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
 ; RV64ZVE32F-NEXT:    slli a3, a3, 2
@@ -3310,7 +3309,7 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB40_6
-; RV64ZVE32F-NEXT:  .LBB40_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB40_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
@@ -3324,7 +3323,7 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB40_7
-; RV64ZVE32F-NEXT:  .LBB40_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB40_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -3338,32 +3337,6 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    bnez a3, .LBB40_8
 ; RV64ZVE32F-NEXT:    j .LBB40_9
-; RV64ZVE32F-NEXT:  .LBB40_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
-; RV64ZVE32F-NEXT:    and a3, a3, a1
-; RV64ZVE32F-NEXT:    slli a3, a3, 2
-; RV64ZVE32F-NEXT:    add a3, a0, a3
-; RV64ZVE32F-NEXT:    lw a3, 0(a3)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a3
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a2, a2, -128
-; RV64ZVE32F-NEXT:    beqz a2, .LBB40_11
-; RV64ZVE32F-NEXT:  .LBB40_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    and a1, a2, a1
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %eidxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
@@ -3420,13 +3393,13 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB41_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB41_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB41_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB41_15
 ; RV64ZVE32F-NEXT:  .LBB41_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB41_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB41_16
 ; RV64ZVE32F-NEXT:  .LBB41_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB41_9
@@ -3444,14 +3417,33 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v12, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB41_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB41_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lw a2, 0(a2)
+; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB41_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB41_16
-; RV64ZVE32F-NEXT:  .LBB41_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB41_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB41_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB41_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB41_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -3461,7 +3453,7 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v9, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB41_6
-; RV64ZVE32F-NEXT:  .LBB41_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB41_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -3472,7 +3464,7 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB41_7
-; RV64ZVE32F-NEXT:  .LBB41_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB41_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -3483,28 +3475,6 @@ define <8 x i32> @mgather_baseidx_v8i32(ptr %base, <8 x i32> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB41_8
 ; RV64ZVE32F-NEXT:    j .LBB41_9
-; RV64ZVE32F-NEXT:  .LBB41_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB41_11
-; RV64ZVE32F-NEXT:  .LBB41_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %idxs
   %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
   ret <8 x i32> %v
@@ -3822,11 +3792,13 @@ define <4 x i64> @mgather_truemask_v4i64(<4 x ptr> %ptrs, <4 x i64> %passthru) {
 define <4 x i64> @mgather_falsemask_v4i64(<4 x ptr> %ptrs, <4 x i64> %passthru) {
 ; RV32V-LABEL: mgather_falsemask_v4i64:
 ; RV32V:       # %bb.0:
+; RV32V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32V-NEXT:    vmv2r.v v8, v10
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4i64:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv2r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -7113,11 +7085,13 @@ define <4 x bfloat> @mgather_truemask_v4bf16(<4 x ptr> %ptrs, <4 x bfloat> %pass
 define <4 x bfloat> @mgather_falsemask_v4bf16(<4 x ptr> %ptrs, <4 x bfloat> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4bf16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4bf16:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -7295,13 +7269,13 @@ define <8 x bfloat> @mgather_baseidx_v8i8_v8bf16(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB64_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB64_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB64_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB64_15
 ; RV64ZVE32F-NEXT:  .LBB64_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB64_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB64_16
 ; RV64ZVE32F-NEXT:  .LBB64_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB64_9
@@ -7320,14 +7294,35 @@ define <8 x bfloat> @mgather_baseidx_v8i8_v8bf16(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB64_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB64_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB64_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB64_16
-; RV64ZVE32F-NEXT:  .LBB64_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB64_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB64_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB64_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB64_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -7338,7 +7333,7 @@ define <8 x bfloat> @mgather_baseidx_v8i8_v8bf16(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB64_6
-; RV64ZVE32F-NEXT:  .LBB64_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB64_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -7351,7 +7346,7 @@ define <8 x bfloat> @mgather_baseidx_v8i8_v8bf16(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB64_7
-; RV64ZVE32F-NEXT:  .LBB64_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB64_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -7364,30 +7359,6 @@ define <8 x bfloat> @mgather_baseidx_v8i8_v8bf16(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB64_8
 ; RV64ZVE32F-NEXT:    j .LBB64_9
-; RV64ZVE32F-NEXT:  .LBB64_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB64_11
-; RV64ZVE32F-NEXT:  .LBB64_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds bfloat, ptr %base, <8 x i8> %idxs
   %v = call <8 x bfloat> @llvm.masked.gather.v8bf16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x bfloat> %passthru)
   ret <8 x bfloat> %v
@@ -7447,13 +7418,13 @@ define <8 x bfloat> @mgather_baseidx_sext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB65_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB65_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB65_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB65_15
 ; RV64ZVE32F-NEXT:  .LBB65_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB65_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB65_16
 ; RV64ZVE32F-NEXT:  .LBB65_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB65_9
@@ -7472,14 +7443,35 @@ define <8 x bfloat> @mgather_baseidx_sext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB65_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB65_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB65_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB65_16
-; RV64ZVE32F-NEXT:  .LBB65_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB65_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB65_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB65_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB65_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -7490,7 +7482,7 @@ define <8 x bfloat> @mgather_baseidx_sext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB65_6
-; RV64ZVE32F-NEXT:  .LBB65_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB65_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -7503,7 +7495,7 @@ define <8 x bfloat> @mgather_baseidx_sext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB65_7
-; RV64ZVE32F-NEXT:  .LBB65_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB65_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -7516,30 +7508,6 @@ define <8 x bfloat> @mgather_baseidx_sext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB65_8
 ; RV64ZVE32F-NEXT:    j .LBB65_9
-; RV64ZVE32F-NEXT:  .LBB65_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB65_11
-; RV64ZVE32F-NEXT:  .LBB65_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %eidxs = sext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds bfloat, ptr %base, <8 x i16> %eidxs
   %v = call <8 x bfloat> @llvm.masked.gather.v8bf16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x bfloat> %passthru)
@@ -7600,13 +7568,13 @@ define <8 x bfloat> @mgather_baseidx_zext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB66_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB66_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB66_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB66_15
 ; RV64ZVE32F-NEXT:  .LBB66_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB66_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB66_16
 ; RV64ZVE32F-NEXT:  .LBB66_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB66_9
@@ -7626,14 +7594,37 @@ define <8 x bfloat> @mgather_baseidx_zext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB66_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB66_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB66_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB66_16
-; RV64ZVE32F-NEXT:  .LBB66_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB66_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB66_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB66_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB66_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -7645,7 +7636,7 @@ define <8 x bfloat> @mgather_baseidx_zext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB66_6
-; RV64ZVE32F-NEXT:  .LBB66_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB66_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -7658,47 +7649,21 @@ define <8 x bfloat> @mgather_baseidx_zext_v8i8_v8bf16(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    beqz a2, .LBB66_7
-; RV64ZVE32F-NEXT:  .LBB66_14: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV64ZVE32F-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 4
-; RV64ZVE32F-NEXT:    andi a2, a1, 32
-; RV64ZVE32F-NEXT:    bnez a2, .LBB66_8
-; RV64ZVE32F-NEXT:    j .LBB66_9
-; RV64ZVE32F-NEXT:  .LBB66_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB66_11
-; RV64ZVE32F-NEXT:  .LBB66_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
+; RV64ZVE32F-NEXT:    beqz a2, .LBB66_7
+; RV64ZVE32F-NEXT:  .LBB66_16: # %cond.load10
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
+; RV64ZVE32F-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 4
+; RV64ZVE32F-NEXT:    andi a2, a1, 32
+; RV64ZVE32F-NEXT:    bnez a2, .LBB66_8
+; RV64ZVE32F-NEXT:    j .LBB66_9
   %eidxs = zext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds bfloat, ptr %base, <8 x i16> %eidxs
   %v = call <8 x bfloat> @llvm.masked.gather.v8bf16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x bfloat> %passthru)
@@ -7756,13 +7721,13 @@ define <8 x bfloat> @mgather_baseidx_v8bf16(ptr %base, <8 x i16> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB67_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB67_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB67_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB67_15
 ; RV64ZVE32F-NEXT:  .LBB67_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB67_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB67_16
 ; RV64ZVE32F-NEXT:  .LBB67_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB67_9
@@ -7780,14 +7745,33 @@ define <8 x bfloat> @mgather_baseidx_v8bf16(ptr %base, <8 x i16> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB67_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB67_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-NEXT:  .LBB67_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB67_16
-; RV64ZVE32F-NEXT:  .LBB67_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB67_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-NEXT:  .LBB67_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB67_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB67_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -7797,7 +7781,7 @@ define <8 x bfloat> @mgather_baseidx_v8bf16(ptr %base, <8 x i16> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB67_6
-; RV64ZVE32F-NEXT:  .LBB67_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB67_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -7809,7 +7793,7 @@ define <8 x bfloat> @mgather_baseidx_v8bf16(ptr %base, <8 x i16> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB67_7
-; RV64ZVE32F-NEXT:  .LBB67_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB67_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -7820,28 +7804,6 @@ define <8 x bfloat> @mgather_baseidx_v8bf16(ptr %base, <8 x i16> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB67_8
 ; RV64ZVE32F-NEXT:    j .LBB67_9
-; RV64ZVE32F-NEXT:  .LBB67_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB67_11
-; RV64ZVE32F-NEXT:  .LBB67_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds bfloat, ptr %base, <8 x i16> %idxs
   %v = call <8 x bfloat> @llvm.masked.gather.v8bf16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x bfloat> %passthru)
   ret <8 x bfloat> %v
@@ -8135,11 +8097,13 @@ define <4 x half> @mgather_truemask_v4f16(<4 x ptr> %ptrs, <4 x half> %passthru)
 define <4 x half> @mgather_falsemask_v4f16(<4 x ptr> %ptrs, <4 x half> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4f16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4f16:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -8410,13 +8374,13 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_12
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_14
 ; RV64ZVE32F-ZVFH-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_13
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_15
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB74_6: # %else8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_14
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_16
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB74_7: # %else11
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB74_9
@@ -8435,14 +8399,35 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_15
-; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB74_11
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFH-NEXT:  .LBB74_11: # %else17
 ; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    bnez a1, .LBB74_16
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_11: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB74_13
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB74_13: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFH-NEXT:    ret
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_12: # %cond.load4
+; RV64ZVE32F-ZVFH-NEXT:  .LBB74_14: # %cond.load4
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
@@ -8453,7 +8438,7 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB74_6
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_13: # %cond.load7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB74_15: # %cond.load7
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
@@ -8466,7 +8451,7 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB74_7
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_14: # %cond.load10
+; RV64ZVE32F-ZVFH-NEXT:  .LBB74_16: # %cond.load10
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
@@ -8479,30 +8464,6 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB74_8
 ; RV64ZVE32F-ZVFH-NEXT:    j .LBB74_9
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_15: # %cond.load16
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB74_11
-; RV64ZVE32F-ZVFH-NEXT:  .LBB74_16: # %cond.load19
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFH-NEXT:    ret
 ;
 ; RV64ZVE32F-ZVFHMIN-LABEL: mgather_baseidx_v8i8_v8f16:
 ; RV64ZVE32F-ZVFHMIN:       # %bb.0:
@@ -8537,13 +8498,13 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_12
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_14
 ; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_13
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_15
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_6: # %else8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_14
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_16
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_7: # %else11
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB74_9
@@ -8562,14 +8523,35 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_15
-; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB74_11
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_11: # %else17
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a1, .LBB74_16
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_11: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB74_13
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_13: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFHMIN-NEXT:    ret
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_12: # %cond.load4
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_14: # %cond.load4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
@@ -8580,7 +8562,7 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB74_6
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_13: # %cond.load7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_15: # %cond.load7
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
@@ -8593,7 +8575,7 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB74_7
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_14: # %cond.load10
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_16: # %cond.load10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
@@ -8606,30 +8588,6 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB74_8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    j .LBB74_9
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_15: # %cond.load16
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB74_11
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB74_16: # %cond.load19
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFHMIN-NEXT:    ret
   %ptrs = getelementptr inbounds half, ptr %base, <8 x i8> %idxs
   %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
   ret <8 x half> %v
@@ -8689,13 +8647,13 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_12
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_14
 ; RV64ZVE32F-ZVFH-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_13
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_15
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB75_6: # %else8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_14
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_16
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB75_7: # %else11
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB75_9
@@ -8714,14 +8672,35 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_15
-; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB75_11
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFH-NEXT:  .LBB75_11: # %else17
 ; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    bnez a1, .LBB75_16
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_11: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB75_13
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB75_13: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFH-NEXT:    ret
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_12: # %cond.load4
+; RV64ZVE32F-ZVFH-NEXT:  .LBB75_14: # %cond.load4
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
@@ -8732,7 +8711,7 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB75_6
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_13: # %cond.load7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB75_15: # %cond.load7
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
@@ -8745,7 +8724,7 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB75_7
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_14: # %cond.load10
+; RV64ZVE32F-ZVFH-NEXT:  .LBB75_16: # %cond.load10
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
@@ -8758,30 +8737,6 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB75_8
 ; RV64ZVE32F-ZVFH-NEXT:    j .LBB75_9
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_15: # %cond.load16
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB75_11
-; RV64ZVE32F-ZVFH-NEXT:  .LBB75_16: # %cond.load19
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFH-NEXT:    ret
 ;
 ; RV64ZVE32F-ZVFHMIN-LABEL: mgather_baseidx_sext_v8i8_v8f16:
 ; RV64ZVE32F-ZVFHMIN:       # %bb.0:
@@ -8816,13 +8771,13 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_12
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_14
 ; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_13
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_15
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_6: # %else8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_14
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_16
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_7: # %else11
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB75_9
@@ -8841,14 +8796,35 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_15
-; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB75_11
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_11: # %else17
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a1, .LBB75_16
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_11: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB75_13
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_13: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFHMIN-NEXT:    ret
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_12: # %cond.load4
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_14: # %cond.load4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
@@ -8859,7 +8835,7 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB75_6
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_13: # %cond.load7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_15: # %cond.load7
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
@@ -8872,7 +8848,7 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB75_7
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_14: # %cond.load10
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_16: # %cond.load10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
@@ -8885,30 +8861,6 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB75_8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    j .LBB75_9
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_15: # %cond.load16
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB75_11
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB75_16: # %cond.load19
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFHMIN-NEXT:    ret
   %eidxs = sext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds half, ptr %base, <8 x i16> %eidxs
   %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
@@ -8969,13 +8921,13 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_12
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_14
 ; RV64ZVE32F-ZVFH-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_13
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_15
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB76_6: # %else8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_14
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_16
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB76_7: # %else11
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB76_9
@@ -8995,14 +8947,37 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_15
-; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB76_11
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFH-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFH-NEXT:  .LBB76_11: # %else17
 ; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    bnez a1, .LBB76_16
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_11: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB76_13
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
+; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB76_13: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFH-NEXT:    ret
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_12: # %cond.load4
+; RV64ZVE32F-ZVFH-NEXT:  .LBB76_14: # %cond.load4
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
@@ -9014,7 +8989,7 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB76_6
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_13: # %cond.load7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB76_15: # %cond.load7
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
@@ -9028,7 +9003,7 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB76_7
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_14: # %cond.load10
+; RV64ZVE32F-ZVFH-NEXT:  .LBB76_16: # %cond.load10
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a2, 255
@@ -9042,32 +9017,6 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB76_8
 ; RV64ZVE32F-ZVFH-NEXT:    j .LBB76_9
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_15: # %cond.load16
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFH-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB76_11
-; RV64ZVE32F-ZVFH-NEXT:  .LBB76_16: # %cond.load19
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
-; RV64ZVE32F-ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFH-NEXT:    ret
 ;
 ; RV64ZVE32F-ZVFHMIN-LABEL: mgather_baseidx_zext_v8i8_v8f16:
 ; RV64ZVE32F-ZVFHMIN:       # %bb.0:
@@ -9104,13 +9053,13 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_12
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_14
 ; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_13
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_15
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_6: # %else8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_14
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_16
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_7: # %else11
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB76_9
@@ -9130,14 +9079,37 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_15
-; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB76_11
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_11: # %else17
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a1, .LBB76_16
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_11: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB76_13
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_13: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFHMIN-NEXT:    ret
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_12: # %cond.load4
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_14: # %cond.load4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
@@ -9149,7 +9121,7 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB76_6
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_13: # %cond.load7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_15: # %cond.load7
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
@@ -9163,7 +9135,7 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB76_7
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_14: # %cond.load10
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_16: # %cond.load10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a2, 255
@@ -9177,32 +9149,6 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB76_8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    j .LBB76_9
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_15: # %cond.load16
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB76_11
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB76_16: # %cond.load19
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFHMIN-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i16>
   %ptrs = getelementptr inbounds half, ptr %base, <8 x i16> %eidxs
   %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
@@ -9260,13 +9206,13 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_12
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_14
 ; RV64ZVE32F-ZVFH-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_13
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_15
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB77_6: # %else8
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_14
+; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_16
 ; RV64ZVE32F-ZVFH-NEXT:  .LBB77_7: # %else11
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB77_9
@@ -9284,14 +9230,33 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_15
-; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB77_11
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFH-NEXT:  .LBB77_11: # %else17
 ; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    bnez a1, .LBB77_16
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_11: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB77_13
+; RV64ZVE32F-ZVFH-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
+; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB77_13: # %else20
+; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFH-NEXT:    ret
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_12: # %cond.load4
+; RV64ZVE32F-ZVFH-NEXT:  .LBB77_14: # %cond.load4
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
@@ -9301,7 +9266,7 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB77_6
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_13: # %cond.load7
+; RV64ZVE32F-ZVFH-NEXT:  .LBB77_15: # %cond.load7
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
@@ -9313,7 +9278,7 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFH-NEXT:    beqz a2, .LBB77_7
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_14: # %cond.load10
+; RV64ZVE32F-ZVFH-NEXT:  .LBB77_16: # %cond.load10
 ; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
 ; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
@@ -9324,28 +9289,6 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFH-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFH-NEXT:    bnez a2, .LBB77_8
 ; RV64ZVE32F-ZVFH-NEXT:    j .LBB77_9
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_15: # %cond.load16
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFH-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFH-NEXT:    beqz a1, .LBB77_11
-; RV64ZVE32F-ZVFH-NEXT:  .LBB77_16: # %cond.load19
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFH-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFH-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFH-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFH-NEXT:    flh fa5, 0(a0)
-; RV64ZVE32F-ZVFH-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-ZVFH-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFH-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFH-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFH-NEXT:    ret
 ;
 ; RV64ZVE32F-ZVFHMIN-LABEL: mgather_baseidx_v8f16:
 ; RV64ZVE32F-ZVFHMIN:       # %bb.0:
@@ -9379,13 +9322,13 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_12
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_14
 ; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_13
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_15
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_6: # %else8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_14
+; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_16
 ; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_7: # %else11
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB77_9
@@ -9403,14 +9346,33 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v10, 2
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_15
-; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB77_11
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_11: # %else17
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a1, .LBB77_16
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_11: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB77_13
+; RV64ZVE32F-ZVFHMIN-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
+; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
+; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_13: # %else20
+; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-ZVFHMIN-NEXT:    ret
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_12: # %cond.load4
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_14: # %cond.load4
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
@@ -9420,7 +9382,7 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v11, 2
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB77_6
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_13: # %cond.load7
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_15: # %cond.load7
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
@@ -9432,7 +9394,7 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 3
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a2, .LBB77_7
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_14: # %cond.load10
+; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_16: # %cond.load10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 5, e16, m1, tu, ma
 ; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
@@ -9443,28 +9405,6 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-ZVFHMIN-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-ZVFHMIN-NEXT:    bnez a2, .LBB77_8
 ; RV64ZVE32F-ZVFHMIN-NEXT:    j .LBB77_9
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_15: # %cond.load16
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a2, a2, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a2, a0, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v10, 6
-; RV64ZVE32F-ZVFHMIN-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-ZVFHMIN-NEXT:    beqz a1, .LBB77_11
-; RV64ZVE32F-ZVFHMIN-NEXT:  .LBB77_16: # %cond.load19
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-ZVFHMIN-NEXT:    slli a1, a1, 1
-; RV64ZVE32F-ZVFHMIN-NEXT:    add a0, a0, a1
-; RV64ZVE32F-ZVFHMIN-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv.s.x v8, a0
-; RV64ZVE32F-ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64ZVE32F-ZVFHMIN-NEXT:    vslideup.vi v9, v8, 7
-; RV64ZVE32F-ZVFHMIN-NEXT:    vmv1r.v v8, v9
-; RV64ZVE32F-ZVFHMIN-NEXT:    ret
   %ptrs = getelementptr inbounds half, ptr %base, <8 x i16> %idxs
   %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
   ret <8 x half> %v
@@ -9666,11 +9606,13 @@ define <4 x float> @mgather_truemask_v4f32(<4 x ptr> %ptrs, <4 x float> %passthr
 define <4 x float> @mgather_falsemask_v4f32(<4 x ptr> %ptrs, <4 x float> %passthru) {
 ; RV32-LABEL: mgather_falsemask_v4f32:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4f32:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv1r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -9847,13 +9789,13 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB84_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB84_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB84_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB84_15
 ; RV64ZVE32F-NEXT:  .LBB84_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB84_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB84_16
 ; RV64ZVE32F-NEXT:  .LBB84_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB84_9
@@ -9872,14 +9814,35 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB84_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB84_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB84_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB84_16
-; RV64ZVE32F-NEXT:  .LBB84_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB84_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB84_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB84_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB84_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -9890,7 +9853,7 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB84_6
-; RV64ZVE32F-NEXT:  .LBB84_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB84_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -9903,7 +9866,7 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB84_7
-; RV64ZVE32F-NEXT:  .LBB84_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB84_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -9916,30 +9879,6 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB84_8
 ; RV64ZVE32F-NEXT:    j .LBB84_9
-; RV64ZVE32F-NEXT:  .LBB84_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB84_11
-; RV64ZVE32F-NEXT:  .LBB84_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i8> %idxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
   ret <8 x float> %v
@@ -9998,13 +9937,13 @@ define <8 x float> @mgather_baseidx_sext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB85_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB85_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB85_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB85_15
 ; RV64ZVE32F-NEXT:  .LBB85_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB85_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB85_16
 ; RV64ZVE32F-NEXT:  .LBB85_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB85_9
@@ -10023,74 +9962,71 @@ define <8 x float> @mgather_baseidx_sext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB85_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB85_16
-; RV64ZVE32F-NEXT:  .LBB85_11: # %else20
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB85_12: # %cond.load4
+; RV64ZVE32F-NEXT:    beqz a2, .LBB85_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
-; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    beqz a2, .LBB85_6
-; RV64ZVE32F-NEXT:  .LBB85_13: # %cond.load7
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB85_11: # %else17
+; RV64ZVE32F-NEXT:    andi a1, a1, -128
+; RV64ZVE32F-NEXT:    beqz a1, .LBB85_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
-; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    beqz a2, .LBB85_7
-; RV64ZVE32F-NEXT:  .LBB85_14: # %cond.load10
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB85_13: # %else20
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 4
-; RV64ZVE32F-NEXT:    andi a2, a1, 32
-; RV64ZVE32F-NEXT:    bnez a2, .LBB85_8
-; RV64ZVE32F-NEXT:    j .LBB85_9
-; RV64ZVE32F-NEXT:  .LBB85_15: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
+; RV64ZVE32F-NEXT:    ret
+; RV64ZVE32F-NEXT:  .LBB85_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB85_11
-; RV64ZVE32F-NEXT:  .LBB85_16: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 3, e32, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
+; RV64ZVE32F-NEXT:    andi a2, a1, 8
+; RV64ZVE32F-NEXT:    beqz a2, .LBB85_6
+; RV64ZVE32F-NEXT:  .LBB85_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
+; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    andi a2, a1, 16
+; RV64ZVE32F-NEXT:    beqz a2, .LBB85_7
+; RV64ZVE32F-NEXT:  .LBB85_16: # %cond.load10
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 4
+; RV64ZVE32F-NEXT:    andi a2, a1, 32
+; RV64ZVE32F-NEXT:    bnez a2, .LBB85_8
+; RV64ZVE32F-NEXT:    j .LBB85_9
   %eidxs = sext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
@@ -10153,13 +10089,13 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB86_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB86_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB86_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB86_15
 ; RV64ZVE32F-NEXT:  .LBB86_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB86_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB86_16
 ; RV64ZVE32F-NEXT:  .LBB86_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB86_9
@@ -10179,14 +10115,37 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB86_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB86_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    andi a2, a2, 255
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB86_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB86_16
-; RV64ZVE32F-NEXT:  .LBB86_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB86_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    andi a1, a1, 255
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB86_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB86_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB86_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -10198,7 +10157,7 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB86_6
-; RV64ZVE32F-NEXT:  .LBB86_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB86_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -10212,7 +10171,7 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB86_7
-; RV64ZVE32F-NEXT:  .LBB86_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB86_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -10226,32 +10185,6 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB86_8
 ; RV64ZVE32F-NEXT:    j .LBB86_9
-; RV64ZVE32F-NEXT:  .LBB86_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    andi a2, a2, 255
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB86_11
-; RV64ZVE32F-NEXT:  .LBB86_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    andi a1, a1, 255
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = zext <8 x i8> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
@@ -10312,13 +10245,13 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB87_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB87_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB87_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB87_15
 ; RV64ZVE32F-NEXT:  .LBB87_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB87_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB87_16
 ; RV64ZVE32F-NEXT:  .LBB87_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_9
@@ -10337,14 +10270,35 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB87_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB87_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB87_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB87_16
-; RV64ZVE32F-NEXT:  .LBB87_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB87_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB87_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB87_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB87_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -10355,7 +10309,7 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_6
-; RV64ZVE32F-NEXT:  .LBB87_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB87_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -10368,7 +10322,7 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_7
-; RV64ZVE32F-NEXT:  .LBB87_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB87_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -10381,30 +10335,6 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB87_8
 ; RV64ZVE32F-NEXT:    j .LBB87_9
-; RV64ZVE32F-NEXT:  .LBB87_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB87_11
-; RV64ZVE32F-NEXT:  .LBB87_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i16> %idxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
   ret <8 x float> %v
@@ -10464,13 +10394,13 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB88_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB88_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB88_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB88_15
 ; RV64ZVE32F-NEXT:  .LBB88_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB88_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB88_16
 ; RV64ZVE32F-NEXT:  .LBB88_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_9
@@ -10489,14 +10419,35 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB88_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB88_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB88_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB88_16
-; RV64ZVE32F-NEXT:  .LBB88_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB88_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB88_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB88_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB88_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -10507,7 +10458,7 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_6
-; RV64ZVE32F-NEXT:  .LBB88_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB88_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -10520,7 +10471,7 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_7
-; RV64ZVE32F-NEXT:  .LBB88_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB88_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -10533,30 +10484,6 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB88_8
 ; RV64ZVE32F-NEXT:    j .LBB88_9
-; RV64ZVE32F-NEXT:  .LBB88_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB88_11
-; RV64ZVE32F-NEXT:  .LBB88_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = sext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
@@ -10620,13 +10547,13 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a3, .LBB89_12
+; RV64ZVE32F-NEXT:    bnez a3, .LBB89_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
-; RV64ZVE32F-NEXT:    bnez a3, .LBB89_13
+; RV64ZVE32F-NEXT:    bnez a3, .LBB89_15
 ; RV64ZVE32F-NEXT:  .LBB89_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
-; RV64ZVE32F-NEXT:    bnez a3, .LBB89_14
+; RV64ZVE32F-NEXT:    bnez a3, .LBB89_16
 ; RV64ZVE32F-NEXT:  .LBB89_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB89_9
@@ -10646,14 +10573,37 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 2
-; RV64ZVE32F-NEXT:    bnez a3, .LBB89_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a3, .LBB89_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV64ZVE32F-NEXT:    and a3, a3, a1
+; RV64ZVE32F-NEXT:    slli a3, a3, 2
+; RV64ZVE32F-NEXT:    add a3, a0, a3
+; RV64ZVE32F-NEXT:    flw fa5, 0(a3)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB89_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
-; RV64ZVE32F-NEXT:    bnez a2, .LBB89_16
-; RV64ZVE32F-NEXT:  .LBB89_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a2, .LBB89_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    and a1, a2, a1
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB89_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB89_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB89_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
 ; RV64ZVE32F-NEXT:    slli a3, a3, 2
@@ -10665,7 +10615,7 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 2
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB89_6
-; RV64ZVE32F-NEXT:  .LBB89_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB89_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
@@ -10679,7 +10629,7 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB89_7
-; RV64ZVE32F-NEXT:  .LBB89_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB89_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -10693,32 +10643,6 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    bnez a3, .LBB89_8
 ; RV64ZVE32F-NEXT:    j .LBB89_9
-; RV64ZVE32F-NEXT:  .LBB89_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
-; RV64ZVE32F-NEXT:    and a3, a3, a1
-; RV64ZVE32F-NEXT:    slli a3, a3, 2
-; RV64ZVE32F-NEXT:    add a3, a0, a3
-; RV64ZVE32F-NEXT:    flw fa5, 0(a3)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a2, a2, -128
-; RV64ZVE32F-NEXT:    beqz a2, .LBB89_11
-; RV64ZVE32F-NEXT:  .LBB89_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    and a1, a2, a1
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %eidxs = zext <8 x i16> %idxs to <8 x i32>
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %eidxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
@@ -10775,13 +10699,13 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ; RV64ZVE32F-NEXT:    andi a2, a1, 4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB90_12
+; RV64ZVE32F-NEXT:    bnez a2, .LBB90_14
 ; RV64ZVE32F-NEXT:  # %bb.5: # %else5
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
-; RV64ZVE32F-NEXT:    bnez a2, .LBB90_13
+; RV64ZVE32F-NEXT:    bnez a2, .LBB90_15
 ; RV64ZVE32F-NEXT:  .LBB90_6: # %else8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
-; RV64ZVE32F-NEXT:    bnez a2, .LBB90_14
+; RV64ZVE32F-NEXT:    bnez a2, .LBB90_16
 ; RV64ZVE32F-NEXT:  .LBB90_7: # %else11
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB90_9
@@ -10799,14 +10723,33 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v12, 2
-; RV64ZVE32F-NEXT:    bnez a2, .LBB90_15
-; RV64ZVE32F-NEXT:  # %bb.10: # %else17
+; RV64ZVE32F-NEXT:    beqz a2, .LBB90_11
+; RV64ZVE32F-NEXT:  # %bb.10: # %cond.load16
+; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV64ZVE32F-NEXT:    slli a2, a2, 2
+; RV64ZVE32F-NEXT:    add a2, a0, a2
+; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
+; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
+; RV64ZVE32F-NEXT:  .LBB90_11: # %else17
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    bnez a1, .LBB90_16
-; RV64ZVE32F-NEXT:  .LBB90_11: # %else20
+; RV64ZVE32F-NEXT:    beqz a1, .LBB90_13
+; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
+; RV64ZVE32F-NEXT:    slli a1, a1, 2
+; RV64ZVE32F-NEXT:    add a0, a0, a1
+; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
+; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
+; RV64ZVE32F-NEXT:  .LBB90_13: # %else20
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
-; RV64ZVE32F-NEXT:  .LBB90_12: # %cond.load4
+; RV64ZVE32F-NEXT:  .LBB90_14: # %cond.load4
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -10816,7 +10759,7 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v9, 2
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB90_6
-; RV64ZVE32F-NEXT:  .LBB90_13: # %cond.load7
+; RV64ZVE32F-NEXT:  .LBB90_15: # %cond.load7
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -10827,7 +10770,7 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB90_7
-; RV64ZVE32F-NEXT:  .LBB90_14: # %cond.load10
+; RV64ZVE32F-NEXT:  .LBB90_16: # %cond.load10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -10838,28 +10781,6 @@ define <8 x float> @mgather_baseidx_v8f32(ptr %base, <8 x i32> %idxs, <8 x i1> %
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB90_8
 ; RV64ZVE32F-NEXT:    j .LBB90_9
-; RV64ZVE32F-NEXT:  .LBB90_15: # %cond.load16
-; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV64ZVE32F-NEXT:    slli a2, a2, 2
-; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 7, e32, m2, tu, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 6
-; RV64ZVE32F-NEXT:    andi a1, a1, -128
-; RV64ZVE32F-NEXT:    beqz a1, .LBB90_11
-; RV64ZVE32F-NEXT:  .LBB90_16: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV64ZVE32F-NEXT:    slli a1, a1, 2
-; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    flw fa5, 0(a0)
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 7
-; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
-; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %idxs
   %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
   ret <8 x float> %v
@@ -11135,11 +11056,13 @@ define <4 x double> @mgather_truemask_v4f64(<4 x ptr> %ptrs, <4 x double> %passt
 define <4 x double> @mgather_falsemask_v4f64(<4 x ptr> %ptrs, <4 x double> %passthru) {
 ; RV32V-LABEL: mgather_falsemask_v4f64:
 ; RV32V:       # %bb.0:
+; RV32V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32V-NEXT:    vmv2r.v v8, v10
 ; RV32V-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_falsemask_v4f64:
 ; RV64V:       # %bb.0:
+; RV64V-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64V-NEXT:    vmv2r.v v8, v10
 ; RV64V-NEXT:    ret
 ;
@@ -13700,6 +13623,7 @@ define <16 x i8> @mgather_baseidx_v16i8(ptr %base, <16 x i8> %idxs, <16 x i1> %m
 ; RV64ZVE32F-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v8, 15
 ; RV64ZVE32F-NEXT:  .LBB107_24: # %else44
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB107_25: # %cond.load4
@@ -14086,6 +14010,7 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v8, 31
 ; RV64ZVE32F-NEXT:  .LBB108_48: # %else92
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB108_49: # %cond.load4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
index e0cf39c75da24..69903d77084bf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
@@ -318,6 +318,7 @@ define <128 x i16> @masked_load_v128i16(ptr %a, <128 x i1> %mask) {
 define <256 x i8> @masked_load_v256i8(ptr %a, <256 x i1> %mask) {
 ; CHECK-LABEL: masked_load_v256i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v16, v8
 ; CHECK-NEXT:    li a1, 128
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
index 46c2033d28b38..80a9143d1ad8b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
@@ -135,10 +135,10 @@ declare <16 x half> @llvm.vp.nearbyint.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_nearbyint_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v16f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI6_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI6_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -263,8 +263,8 @@ declare <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_nearbyint_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -307,8 +307,8 @@ declare <16 x float> @llvm.vp.nearbyint.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_nearbyint_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -393,10 +393,10 @@ declare <4 x double> @llvm.vp.nearbyint.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_nearbyint_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -437,10 +437,10 @@ declare <8 x double> @llvm.vp.nearbyint.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_nearbyint_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -481,10 +481,10 @@ declare <15 x double> @llvm.vp.nearbyint.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_nearbyint_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -525,10 +525,10 @@ declare <16 x double> @llvm.vp.nearbyint.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_nearbyint_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -569,9 +569,9 @@ declare <32 x double> @llvm.vp.nearbyint.v32f64(<32 x double>, <32 x i1>, i32)
 define <32 x double> @vp_nearbyint_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_v32f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v6, v0
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v7, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-mask-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-mask-vp.ll
index ad358d7320240..276f6b077931b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-mask-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-mask-vp.ll
@@ -23,9 +23,9 @@ declare i1 @llvm.vp.reduce.or.v1i1(i1, <1 x i1>, <1 x i1>, i32)
 define zeroext i1 @vpreduce_or_v1i1(i1 zeroext %s, <1 x i1> %v, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -39,9 +39,9 @@ declare i1 @llvm.vp.reduce.xor.v1i1(i1, <1 x i1>, <1 x i1>, i32)
 define zeroext i1 @vpreduce_xor_v1i1(i1 zeroext %s, <1 x i1> %v, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -71,9 +71,9 @@ declare i1 @llvm.vp.reduce.or.v2i1(i1, <2 x i1>, <2 x i1>, i32)
 define zeroext i1 @vpreduce_or_v2i1(i1 zeroext %s, <2 x i1> %v, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -87,9 +87,9 @@ declare i1 @llvm.vp.reduce.xor.v2i1(i1, <2 x i1>, <2 x i1>, i32)
 define zeroext i1 @vpreduce_xor_v2i1(i1 zeroext %s, <2 x i1> %v, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -119,9 +119,9 @@ declare i1 @llvm.vp.reduce.or.v4i1(i1, <4 x i1>, <4 x i1>, i32)
 define zeroext i1 @vpreduce_or_v4i1(i1 zeroext %s, <4 x i1> %v, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -135,9 +135,9 @@ declare i1 @llvm.vp.reduce.xor.v4i1(i1, <4 x i1>, <4 x i1>, i32)
 define zeroext i1 @vpreduce_xor_v4i1(i1 zeroext %s, <4 x i1> %v, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -167,9 +167,9 @@ declare i1 @llvm.vp.reduce.or.v8i1(i1, <8 x i1>, <8 x i1>, i32)
 define zeroext i1 @vpreduce_or_v8i1(i1 zeroext %s, <8 x i1> %v, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -183,9 +183,9 @@ declare i1 @llvm.vp.reduce.xor.v8i1(i1, <8 x i1>, <8 x i1>, i32)
 define zeroext i1 @vpreduce_xor_v8i1(i1 zeroext %s, <8 x i1> %v, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -231,6 +231,7 @@ declare i1 @llvm.vp.reduce.and.v256i1(i1, <256 x i1>, <256 x i1>, i32)
 define zeroext i1 @vpreduce_and_v256i1(i1 zeroext %s, <256 x i1> %v, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_and_v256i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v9
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    li a3, 128
@@ -265,9 +266,9 @@ declare i1 @llvm.vp.reduce.or.v16i1(i1, <16 x i1>, <16 x i1>, i32)
 define zeroext i1 @vpreduce_or_v16i1(i1 zeroext %s, <16 x i1> %v, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -281,9 +282,9 @@ declare i1 @llvm.vp.reduce.xor.v16i1(i1, <16 x i1>, <16 x i1>, i32)
 define zeroext i1 @vpreduce_xor_v16i1(i1 zeroext %s, <16 x i1> %v, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -297,9 +298,9 @@ declare i1 @llvm.vp.reduce.add.v1i1(i1, <1 x i1>, <1 x i1>, i32)
 define zeroext i1 @vpreduce_add_v1i1(i1 zeroext %s, <1 x i1> %v, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -313,9 +314,9 @@ declare i1 @llvm.vp.reduce.add.v2i1(i1, <2 x i1>, <2 x i1>, i32)
 define zeroext i1 @vpreduce_add_v2i1(i1 zeroext %s, <2 x i1> %v, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -329,9 +330,9 @@ declare i1 @llvm.vp.reduce.add.v4i1(i1, <4 x i1>, <4 x i1>, i32)
 define zeroext i1 @vpreduce_add_v4i1(i1 zeroext %s, <4 x i1> %v, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -345,9 +346,9 @@ declare i1 @llvm.vp.reduce.add.v8i1(i1, <8 x i1>, <8 x i1>, i32)
 define zeroext i1 @vpreduce_add_v8i1(i1 zeroext %s, <8 x i1> %v, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -361,9 +362,9 @@ declare i1 @llvm.vp.reduce.add.v16i1(i1, <16 x i1>, <16 x i1>, i32)
 define zeroext i1 @vpreduce_add_v16i1(i1 zeroext %s, <16 x i1> %v, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -489,9 +490,9 @@ declare i1 @llvm.vp.reduce.smin.v1i1(i1, <1 x i1>, <1 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v1i1(i1 zeroext %s, <1 x i1> %v, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -505,9 +506,9 @@ declare i1 @llvm.vp.reduce.smin.v2i1(i1, <2 x i1>, <2 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v2i1(i1 zeroext %s, <2 x i1> %v, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -521,9 +522,9 @@ declare i1 @llvm.vp.reduce.smin.v4i1(i1, <4 x i1>, <4 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v4i1(i1 zeroext %s, <4 x i1> %v, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -537,9 +538,9 @@ declare i1 @llvm.vp.reduce.smin.v8i1(i1, <8 x i1>, <8 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v8i1(i1 zeroext %s, <8 x i1> %v, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -553,9 +554,9 @@ declare i1 @llvm.vp.reduce.smin.v16i1(i1, <16 x i1>, <16 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v16i1(i1 zeroext %s, <16 x i1> %v, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -569,9 +570,9 @@ declare i1 @llvm.vp.reduce.smin.v32i1(i1, <32 x i1>, <32 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v32i1(i1 zeroext %s, <32 x i1> %v, <32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -585,9 +586,9 @@ declare i1 @llvm.vp.reduce.smin.v64i1(i1, <64 x i1>, <64 x i1>, i32)
 define zeroext i1 @vpreduce_smin_v64i1(i1 zeroext %s, <64 x i1> %v, <64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_v64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -601,9 +602,9 @@ declare i1 @llvm.vp.reduce.umax.v1i1(i1, <1 x i1>, <1 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v1i1(i1 zeroext %s, <1 x i1> %v, <1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -617,9 +618,9 @@ declare i1 @llvm.vp.reduce.umax.v2i1(i1, <2 x i1>, <2 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v2i1(i1 zeroext %s, <2 x i1> %v, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -633,9 +634,9 @@ declare i1 @llvm.vp.reduce.umax.v4i1(i1, <4 x i1>, <4 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v4i1(i1 zeroext %s, <4 x i1> %v, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -649,9 +650,9 @@ declare i1 @llvm.vp.reduce.umax.v8i1(i1, <8 x i1>, <8 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v8i1(i1 zeroext %s, <8 x i1> %v, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -665,9 +666,9 @@ declare i1 @llvm.vp.reduce.umax.v16i1(i1, <16 x i1>, <16 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v16i1(i1 zeroext %s, <16 x i1> %v, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -681,9 +682,9 @@ declare i1 @llvm.vp.reduce.umax.v32i1(i1, <32 x i1>, <32 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v32i1(i1 zeroext %s, <32 x i1> %v, <32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -697,9 +698,9 @@ declare i1 @llvm.vp.reduce.umax.v64i1(i1, <64 x i1>, <64 x i1>, i32)
 define zeroext i1 @vpreduce_umax_v64i1(i1 zeroext %s, <64 x i1> %v, <64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_v64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
index b8617fda3aa7e..266772d36ee9c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-rint-vp.ll
@@ -123,10 +123,10 @@ declare <16 x half> @llvm.vp.rint.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_rint_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v16f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI6_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI6_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -239,8 +239,8 @@ declare <8 x float> @llvm.vp.rint.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_rint_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -279,8 +279,8 @@ declare <16 x float> @llvm.vp.rint.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_rint_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -357,10 +357,10 @@ declare <4 x double> @llvm.vp.rint.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_rint_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -397,10 +397,10 @@ declare <8 x double> @llvm.vp.rint.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_rint_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -437,10 +437,10 @@ declare <15 x double> @llvm.vp.rint.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_rint_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -477,10 +477,10 @@ declare <16 x double> @llvm.vp.rint.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_rint_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -517,9 +517,9 @@ declare <32 x double> @llvm.vp.rint.v32f64(<32 x double>, <32 x i1>, i32)
 define <32 x double> @vp_rint_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_v32f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v6, v0
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v7, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
index 820a05e3d6042..232a8a4827cb1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
@@ -194,8 +194,8 @@ define <8 x half> @vp_round_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ;
 ; ZVFHMIN-LABEL: vp_round_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -261,10 +261,10 @@ declare <16 x half> @llvm.vp.round.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_round_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_round_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI6_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI6_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -280,8 +280,8 @@ define <16 x half> @vp_round_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ;
 ; ZVFHMIN-LABEL: vp_round_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -431,8 +431,8 @@ declare <8 x float> @llvm.vp.round.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_round_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -475,8 +475,8 @@ declare <16 x float> @llvm.vp.round.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_round_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -561,10 +561,10 @@ declare <4 x double> @llvm.vp.round.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_round_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -605,10 +605,10 @@ declare <8 x double> @llvm.vp.round.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_round_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -649,10 +649,10 @@ declare <15 x double> @llvm.vp.round.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_round_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -693,10 +693,10 @@ declare <16 x double> @llvm.vp.round.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_round_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -743,6 +743,7 @@ define <32 x double> @vp_round_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -750,7 +751,6 @@ define <32 x double> @vp_round_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
index 8391c7939180a..7c80c037403c2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
@@ -194,8 +194,8 @@ define <8 x half> @vp_roundeven_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %
 ;
 ; ZVFHMIN-LABEL: vp_roundeven_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -261,10 +261,10 @@ declare <16 x half> @llvm.vp.roundeven.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_roundeven_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundeven_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI6_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI6_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -280,8 +280,8 @@ define <16 x half> @vp_roundeven_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroe
 ;
 ; ZVFHMIN-LABEL: vp_roundeven_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -431,8 +431,8 @@ declare <8 x float> @llvm.vp.roundeven.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_roundeven_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -475,8 +475,8 @@ declare <16 x float> @llvm.vp.roundeven.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_roundeven_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -561,10 +561,10 @@ declare <4 x double> @llvm.vp.roundeven.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_roundeven_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -605,10 +605,10 @@ declare <8 x double> @llvm.vp.roundeven.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_roundeven_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -649,10 +649,10 @@ declare <15 x double> @llvm.vp.roundeven.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_roundeven_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -693,10 +693,10 @@ declare <16 x double> @llvm.vp.roundeven.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_roundeven_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -743,6 +743,7 @@ define <32 x double> @vp_roundeven_v32f64(<32 x double> %va, <32 x i1> %m, i32 z
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -750,7 +751,6 @@ define <32 x double> @vp_roundeven_v32f64(<32 x double> %va, <32 x i1> %m, i32 z
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
index 8c38d24460265..65a4725267cd3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
@@ -194,8 +194,8 @@ define <8 x half> @vp_roundtozero_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext
 ;
 ; ZVFHMIN-LABEL: vp_roundtozero_v8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -261,10 +261,10 @@ declare <16 x half> @llvm.vp.roundtozero.v16f16(<16 x half>, <16 x i1>, i32)
 define <16 x half> @vp_roundtozero_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundtozero_v16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI6_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI6_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI6_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -280,8 +280,8 @@ define <16 x half> @vp_roundtozero_v16f16(<16 x half> %va, <16 x i1> %m, i32 zer
 ;
 ; ZVFHMIN-LABEL: vp_roundtozero_v16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -431,8 +431,8 @@ declare <8 x float> @llvm.vp.roundtozero.v8f32(<8 x float>, <8 x i1>, i32)
 define <8 x float> @vp_roundtozero_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -475,8 +475,8 @@ declare <16 x float> @llvm.vp.roundtozero.v16f32(<16 x float>, <16 x i1>, i32)
 define <16 x float> @vp_roundtozero_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -561,10 +561,10 @@ declare <4 x double> @llvm.vp.roundtozero.v4f64(<4 x double>, <4 x i1>, i32)
 define <4 x double> @vp_roundtozero_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI18_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI18_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI18_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -605,10 +605,10 @@ declare <8 x double> @llvm.vp.roundtozero.v8f64(<8 x double>, <8 x i1>, i32)
 define <8 x double> @vp_roundtozero_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI20_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI20_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -649,10 +649,10 @@ declare <15 x double> @llvm.vp.roundtozero.v15f64(<15 x double>, <15 x i1>, i32)
 define <15 x double> @vp_roundtozero_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v15f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI22_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI22_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI22_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -693,10 +693,10 @@ declare <16 x double> @llvm.vp.roundtozero.v16f64(<16 x double>, <16 x i1>, i32)
 define <16 x double> @vp_roundtozero_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_v16f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI24_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI24_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI24_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -743,6 +743,7 @@ define <32 x double> @vp_roundtozero_v32f64(<32 x double> %va, <32 x i1> %m, i32
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -750,7 +751,6 @@ define <32 x double> @vp_roundtozero_v32f64(<32 x double> %va, <32 x i1> %m, i32
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB26_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
index d52c42891fcc3..69d6ffa9f300c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
@@ -598,6 +598,7 @@ define <256 x i1> @icmp_eq_vv_v256i8(<256 x i8> %va, <256 x i8> %vb, <256 x i1>
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -648,6 +649,7 @@ define <256 x i1> @icmp_eq_vv_v256i8(<256 x i8> %va, <256 x i8> %vb, <256 x i1>
 define <256 x i1> @icmp_eq_vx_v256i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_v256i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
@@ -677,6 +679,7 @@ define <256 x i1> @icmp_eq_vx_v256i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 z
 define <256 x i1> @icmp_eq_vx_swap_v256i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_swap_v256i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
index 8b18be908089f..66f95b7077672 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
@@ -97,33 +97,30 @@ define <4 x i32> @v4i32_v8i32(<8 x i32>) {
 define <4 x i32> @v4i32_v16i32(<16 x i32>) {
 ; RV32-LABEL: v4i32_v16i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
-; RV32-NEXT:    vslidedown.vi v16, v8, 8
-; RV32-NEXT:    vmv4r.v v20, v8
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV32-NEXT:    vmv.v.i v8, 1
-; RV32-NEXT:    vmv2r.v v22, v12
-; RV32-NEXT:    vmv.v.i v10, 6
+; RV32-NEXT:    vmv.v.i v12, 1
+; RV32-NEXT:    vmv.v.i v14, 6
 ; RV32-NEXT:    li a0, 32
 ; RV32-NEXT:    vmv.v.i v0, 10
 ; RV32-NEXT:    vsetivli zero, 2, e16, m1, tu, ma
-; RV32-NEXT:    vslideup.vi v10, v8, 1
+; RV32-NEXT:    vslideup.vi v14, v12, 1
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wx v12, v8, a0
+; RV32-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 8
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV32-NEXT:    vnsrl.wx v8, v20, a0
-; RV32-NEXT:    vrgatherei16.vv v8, v16, v10, v0.t
+; RV32-NEXT:    vrgatherei16.vv v12, v8, v14, v0.t
+; RV32-NEXT:    vmv1r.v v8, v12
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: v4i32_v16i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
-; RV64-NEXT:    vslidedown.vi v16, v8, 8
-; RV64-NEXT:    vmv4r.v v20, v8
 ; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    vmv2r.v v22, v12
-; RV64-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vmv.v.i v0, 10
-; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-NEXT:    vnsrl.wx v8, v20, a0
+; RV64-NEXT:    vnsrl.wx v12, v8, a0
+; RV64-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v8, v8, 8
 ; RV64-NEXT:    li a0, 3
 ; RV64-NEXT:    slli a0, a0, 33
 ; RV64-NEXT:    addi a0, a0, 1
@@ -131,7 +128,8 @@ define <4 x i32> @v4i32_v16i32(<16 x i32>) {
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vmv.v.x v10, a0
 ; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v8, v16, v10, v0.t
+; RV64-NEXT:    vrgatherei16.vv v12, v8, v10, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
 ; RV64-NEXT:    ret
   %2 = shufflevector <16 x i32> %0, <16 x i32> poison, <4 x i32> <i32 1, i32 9, i32 5, i32 14>
   ret <4 x i32> %2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll
index 38026bb591f79..f2353e7d028bd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll
@@ -8,8 +8,8 @@
 define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) {
 ; VLA-LABEL: concat_2xv4i32:
 ; VLA:       # %bb.0:
-; VLA-NEXT:    vmv1r.v v10, v9
 ; VLA-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; VLA-NEXT:    vmv1r.v v10, v9
 ; VLA-NEXT:    vslideup.vi v8, v10, 4
 ; VLA-NEXT:    ret
 ;
@@ -32,9 +32,9 @@ define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x
 ;
 ; VLS-LABEL: concat_4xv2i32:
 ; VLS:       # %bb.0:
+; VLS-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; VLS-NEXT:    vmv1r.v v13, v10
 ; VLS-NEXT:    vmv1r.v v12, v8
-; VLS-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; VLS-NEXT:    vslideup.vi v13, v11, 2
 ; VLS-NEXT:    vslideup.vi v12, v9, 2
 ; VLS-NEXT:    vmv2r.v v8, v12
@@ -62,9 +62,9 @@ define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x
 ;
 ; VLS-LABEL: concat_8xv1i32:
 ; VLS:       # %bb.0:
+; VLS-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; VLS-NEXT:    vmv1r.v v17, v12
 ; VLS-NEXT:    vmv1r.v v16, v8
-; VLS-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; VLS-NEXT:    vslideup.vi v14, v15, 1
 ; VLS-NEXT:    vslideup.vi v17, v13, 1
 ; VLS-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -89,8 +89,8 @@ define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x
 define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) {
 ; VLA-LABEL: concat_2xv8i32:
 ; VLA:       # %bb.0:
-; VLA-NEXT:    vmv2r.v v12, v10
 ; VLA-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; VLA-NEXT:    vmv2r.v v12, v10
 ; VLA-NEXT:    vslideup.vi v8, v12, 8
 ; VLA-NEXT:    ret
 ;
@@ -104,10 +104,10 @@ define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) {
 define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
 ; VLA-LABEL: concat_4xv4i32:
 ; VLA:       # %bb.0:
+; VLA-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; VLA-NEXT:    vmv1r.v v14, v11
 ; VLA-NEXT:    vmv1r.v v12, v10
 ; VLA-NEXT:    vmv1r.v v10, v9
-; VLA-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; VLA-NEXT:    vslideup.vi v12, v14, 4
 ; VLA-NEXT:    vslideup.vi v8, v10, 4
 ; VLA-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
@@ -140,11 +140,11 @@ define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x
 ;
 ; VLS-LABEL: concat_8xv2i32:
 ; VLS:       # %bb.0:
+; VLS-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; VLS-NEXT:    vmv1r.v v19, v14
 ; VLS-NEXT:    vmv1r.v v18, v12
 ; VLS-NEXT:    vmv1r.v v17, v10
 ; VLS-NEXT:    vmv1r.v v16, v8
-; VLS-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; VLS-NEXT:    vslideup.vi v19, v15, 2
 ; VLS-NEXT:    vslideup.vi v18, v13, 2
 ; VLS-NEXT:    vslideup.vi v17, v11, 2
@@ -164,6 +164,7 @@ define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x
 define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) {
 ; VLA-LABEL: concat_2xv16i32:
 ; VLA:       # %bb.0:
+; VLA-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; VLA-NEXT:    vmv4r.v v16, v12
 ; VLA-NEXT:    li a0, 32
 ; VLA-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -180,11 +181,11 @@ define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) {
 define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ; VLA-LABEL: concat_4xv8i32:
 ; VLA:       # %bb.0:
+; VLA-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; VLA-NEXT:    vmv2r.v v20, v14
 ; VLA-NEXT:    vmv2r.v v16, v12
 ; VLA-NEXT:    vmv2r.v v12, v10
 ; VLA-NEXT:    li a0, 32
-; VLA-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; VLA-NEXT:    vslideup.vi v16, v20, 8
 ; VLA-NEXT:    vslideup.vi v8, v12, 8
 ; VLA-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -203,6 +204,7 @@ define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x
 define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h) {
 ; VLA-LABEL: concat_8xv4i32:
 ; VLA:       # %bb.0:
+; VLA-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; VLA-NEXT:    vmv1r.v v18, v15
 ; VLA-NEXT:    vmv1r.v v20, v14
 ; VLA-NEXT:    vmv1r.v v14, v13
@@ -211,7 +213,6 @@ define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x
 ; VLA-NEXT:    vmv1r.v v12, v10
 ; VLA-NEXT:    vmv1r.v v10, v9
 ; VLA-NEXT:    li a0, 32
-; VLA-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; VLA-NEXT:    vslideup.vi v20, v18, 4
 ; VLA-NEXT:    vslideup.vi v16, v14, 4
 ; VLA-NEXT:    vslideup.vi v12, v22, 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
index f04faf5cd2c54..188ef8fe35a4a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+v \
 ; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
-; RUN:   | FileCheck %s --check-prefixes=CHECK,RV32V
+; RUN:   | FileCheck %s --check-prefix=CHECK
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+v \
 ; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
-; RUN:   | FileCheck %s --check-prefixes=CHECK,RV64V
+; RUN:   | FileCheck %s --check-prefix=CHECK
 
 define void @deinterleave3_0_i8(ptr %in, ptr %out) {
 ; CHECK-LABEL: deinterleave3_0_i8:
@@ -12,13 +12,15 @@ define void @deinterleave3_0_i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vmv.v.i v9, -8
+; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    li a0, 3
-; CHECK-NEXT:    vmul.vx v9, v9, a0
+; CHECK-NEXT:    vmadd.vx v10, a0, v9
+; CHECK-NEXT:    li a0, 73
+; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    li a0, 56
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vadd.vi v10, v9, -8
-; CHECK-NEXT:    vrgather.vv v11, v8, v9
+; CHECK-NEXT:    vcompress.vm v11, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
@@ -37,22 +39,20 @@ define void @deinterleave3_8_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 1
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    li a0, 3
-; CHECK-NEXT:    vmadd.vx v10, a0, v9
+; CHECK-NEXT:    li a0, 146
+; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    li a0, 24
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vsrl.vi v10, v8, 8
+; CHECK-NEXT:    vsrl.vi v9, v8, 8
 ; CHECK-NEXT:    vsll.vi v8, v8, 8
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vor.vv v8, v8, v10
+; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
 entry:
@@ -67,22 +67,12 @@ define void @deinterleave4_0_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    li a0, -1
-; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 4
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vwaddu.vv v10, v8, v9
-; CHECK-NEXT:    vwmaccu.vx v10, a0, v9
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vsll.vi v9, v9, 2
-; CHECK-NEXT:    vadd.vi v9, v9, -8
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v8, v9, v0.t
-; CHECK-NEXT:    vse8.v v10, (a1)
+; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
@@ -96,24 +86,16 @@ define void @deinterleave4_8_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 8
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, -9
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    li a0, 5
-; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vmacc.vx v9, a0, v10
-; CHECK-NEXT:    vsll.vi v10, v10, 2
-; CHECK-NEXT:    vadd.vi v10, v10, 1
-; CHECK-NEXT:    vrgather.vv v11, v8, v10
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vv v11, v8, v9, v0.t
-; CHECK-NEXT:    vse8.v v11, (a1)
+; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
-  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 undef, i32 undef, i32 undef, i32 undef>
   store <8 x i8> %shuffle.i5, ptr %out, align 1
   ret void
 }
@@ -124,12 +106,14 @@ define void @deinterleave5_0_i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vmv.v.i v9, -8
+; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    li a0, 5
+; CHECK-NEXT:    vmadd.vx v10, a0, v9
+; CHECK-NEXT:    li a0, 33
 ; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vmul.vx v9, v9, a0
-; CHECK-NEXT:    vadd.vi v10, v9, -8
-; CHECK-NEXT:    vrgather.vv v11, v8, v9
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vcompress.vm v11, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
@@ -148,18 +132,16 @@ define void @deinterleave5_8_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 1
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    li a0, 5
-; CHECK-NEXT:    vmadd.vx v10, a0, v9
+; CHECK-NEXT:    li a0, 66
 ; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vi v9, v8, 3, v0.t
-; CHECK-NEXT:    vse8.v v9, (a1)
+; CHECK-NEXT:    vrgather.vi v10, v8, 3, v0.t
+; CHECK-NEXT:    vse8.v v10, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
@@ -173,12 +155,11 @@ define void @deinterleave6_0_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    li a0, 6
+; CHECK-NEXT:    li a0, 65
 ; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vmul.vx v9, v9, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v9
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
@@ -197,18 +178,16 @@ define void @deinterleave6_8_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 1
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    li a0, 6
-; CHECK-NEXT:    vmadd.vx v10, a0, v9
+; CHECK-NEXT:    li a0, 130
 ; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vi v9, v8, 5, v0.t
-; CHECK-NEXT:    vse8.v v9, (a1)
+; CHECK-NEXT:    vrgather.vi v10, v8, 5, v0.t
+; CHECK-NEXT:    vse8.v v10, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
@@ -222,12 +201,11 @@ define void @deinterleave7_0_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    li a0, 7
+; CHECK-NEXT:    li a0, 129
 ; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vmul.vx v9, v9, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v9
+; CHECK-NEXT:    vmv.s.x v9, a0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vcompress.vm v10, v8, v9
 ; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
@@ -271,10 +249,12 @@ define void @deinterleave8_0_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 8
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -290,12 +270,14 @@ define void @deinterleave8_8_i8(ptr %in, ptr %out) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vmv.v.i v0, -3
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vi v9, v8, 1, v0.t
-; CHECK-NEXT:    vse8.v v9, (a1)
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
@@ -303,6 +285,94 @@ entry:
   store <8 x i8> %shuffle.i5, ptr %out, align 1
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32V: {{.*}}
-; RV64V: {{.*}}
+
+; Exercise the high lmul case
+define void @deinterleave7_0_i64(ptr %in, ptr %out) {
+; CHECK-LABEL: deinterleave7_0_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    li a0, 129
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 4
+; CHECK-NEXT:    vmv.s.x v16, a0
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vcompress.vm v20, v8, v16
+; CHECK-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 8
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
+; CHECK-NEXT:    vrgather.vi v20, v8, 6, v0.t
+; CHECK-NEXT:    vse64.v v20, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i64>, ptr %in
+  %shuffle.i5 = shufflevector <16 x i64> %0, <16 x i64> poison, <8 x i32> <i32 0, i32 7, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  store <8 x i64> %shuffle.i5, ptr %out
+  ret void
+}
+
+; Store back only the active subvector
+define void @deinterleave4_0_i8_subvec(ptr %in, ptr %out) {
+; CHECK-LABEL: deinterleave4_0_i8_subvec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vse8.v v8, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %in, align 1
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  store <4 x i8> %shuffle.i5, ptr %out, align 1
+  ret void
+}
+
+; Store back only the active subvector
+define void @deinterleave7_0_i32_subvec(ptr %in, ptr %out) {
+; CHECK-LABEL: deinterleave7_0_i32_subvec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    li a0, 129
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 4
+; CHECK-NEXT:    vmv.s.x v12, a0
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vcompress.vm v14, v8, v12
+; CHECK-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 8
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; CHECK-NEXT:    vrgather.vi v14, v8, 6, v0.t
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v14, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i32>, ptr %in
+  %shuffle.i5 = shufflevector <16 x i32> %0, <16 x i32> poison, <3 x i32> <i32 0, i32 7, i32 14>
+  store <3 x i32> %shuffle.i5, ptr %out
+  ret void
+}
+
+; Store back only the active subvector
+define void @deinterleave8_0_i8_subvec(ptr %in, ptr %out) {
+; CHECK-LABEL: deinterleave8_0_i8_subvec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslidedown.vi v9, v8, 8
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    vmv.x.s a2, v9
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a2
+; CHECK-NEXT:    vse8.v v8, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %in, align 1
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+  store <2 x i8> %shuffle.i5, ptr %out, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
index d461fa8378cff..cadee8acf27d6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-exact-vlen.ll
@@ -108,6 +108,7 @@ define <4 x i64> @m2_splat_into_identity(<4 x i64> %v1) vscale_range(2,2) {
 define <4 x i64> @m2_broadcast_i128(<4 x i64> %v1) vscale_range(2,2) {
 ; CHECK-LABEL: m2_broadcast_i128:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    ret
   %res = shufflevector <4 x i64> %v1, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -117,6 +118,7 @@ define <4 x i64> @m2_broadcast_i128(<4 x i64> %v1) vscale_range(2,2) {
 define <8 x i64> @m4_broadcast_i128(<8 x i64> %v1) vscale_range(2,2) {
 ; CHECK-LABEL: m4_broadcast_i128:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vmv1r.v v11, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 407535831aeda..f7647ff38c8a0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -966,9 +966,9 @@ define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
 define <32 x i8> @reverse_v32i8_2(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: reverse_v32i8_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    addi a1, a0, -1
 ; CHECK-NEXT:    vrsub.vx v12, v12, a1
@@ -1035,9 +1035,9 @@ define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
 define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: reverse_v16i16_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
 ; CHECK-NEXT:    srli a1, a0, 1
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1060,9 +1060,9 @@ define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
 define <32 x i16> @reverse_v32i16_2(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: reverse_v32i16_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    lui a1, 16
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1116,9 +1116,9 @@ define <4 x i32> @reverse_v4i32_2(<2 x i32> %a, < 2 x i32> %b) {
 define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: reverse_v8i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
 ; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1142,9 +1142,9 @@ define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
 define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: reverse_v16i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1170,9 +1170,9 @@ define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
 define <32 x i32> @reverse_v32i32_2(<16 x i32> %a, <16 x i32> %b) {
 ; CHECK-LABEL: reverse_v32i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v16, v12
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1219,9 +1219,9 @@ define <4 x i64> @reverse_v4i64_2(<2 x i64> %a, < 2 x i64> %b) {
 define <8 x i64> @reverse_v8i64_2(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: reverse_v8i64_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    srli a1, a0, 3
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1289,9 +1289,9 @@ define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
 define <16 x half> @reverse_v16f16_2(<8 x half> %a, <8 x half> %b) {
 ; CHECK-LABEL: reverse_v16f16_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
 ; CHECK-NEXT:    srli a1, a0, 1
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1361,9 +1361,9 @@ define <4 x float> @reverse_v4f32_2(<2 x float> %a, <2 x float> %b) {
 define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: reverse_v8f32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
 ; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1387,9 +1387,9 @@ define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
 define <16 x float> @reverse_v16f32_2(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: reverse_v16f32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    addi a1, a1, -1
@@ -1430,9 +1430,9 @@ define <4 x double> @reverse_v4f64_2(<2 x double> %a, < 2 x double> %b) {
 define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: reverse_v8f64_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    srli a1, a0, 3
 ; CHECK-NEXT:    addi a1, a1, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
index c37c3a9ee0ea0..49f6acf9ba8c9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
@@ -415,8 +415,8 @@ define <4 x i8> @vslide1up_4xi8_neg_incorrect_insert3(<4 x i8> %v, i8 %b) {
 define <2 x i8> @vslide1up_4xi8_neg_length_changing(<4 x i8> %v, i8 %b) {
 ; CHECK-LABEL: vslide1up_4xi8_neg_length_changing:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 8f6240e112cdd..847ef9a7b3601 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -441,28 +441,75 @@ entry:
 }
 
 define void @vnsrl_0_i8_single_src(ptr %in, ptr %out) {
-; CHECK-LABEL: vnsrl_0_i8_single_src:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
-; CHECK-NEXT:    vse8.v v8, (a1)
-; CHECK-NEXT:    ret
+; V-LABEL: vnsrl_0_i8_single_src:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT:    vle8.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vse8.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i8_single_src:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
+; ZVE32F-NEXT:    vse8.v v8, (a1)
+; ZVE32F-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in, align 1
-  %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-  store <8 x i8> %shuffle.i5, ptr %out, align 1
+  %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  store <4 x i8> %shuffle.i5, ptr %out, align 1
   ret void
 }
 
-define void @vnsrl_0_i8_single_src2(ptr %in, ptr %out) {
-; CHECK-LABEL: vnsrl_0_i8_single_src2:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
-; CHECK-NEXT:    vse8.v v8, (a1)
-; CHECK-NEXT:    ret
+define void @vnsrl_8_i8_single_src(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_8_i8_single_src:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT:    vle8.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; V-NEXT:    vnsrl.wi v8, v8, 8
+; V-NEXT:    vse8.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_8_i8_single_src:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 8
+; ZVE32F-NEXT:    vse8.v v8, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <8 x i8>, ptr %in, align 1
+  %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  store <4 x i8> %shuffle.i5, ptr %out, align 1
+  ret void
+}
+
+define void @vnsrl_0_i8_single_wideuse(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_i8_single_wideuse:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT:    vle8.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT:    vse8.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i8_single_wideuse:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
+; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vse8.v v8, (a1)
+; ZVE32F-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in, align 1
   %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -473,17 +520,35 @@ entry:
 ; Can't match the m8 result type as the source would have to be m16 which
 ; isn't a legal type.
 define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
-; CHECK-LABEL: vnsrl_0_i32_single_src_m8:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vid.v v16
-; CHECK-NEXT:    vadd.vv v16, v16, v16
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v24, v8, v16
-; CHECK-NEXT:    vse32.v v24, (a1)
-; CHECK-NEXT:    ret
+; V-LABEL: vnsrl_0_i32_single_src_m8:
+; V:       # %bb.0: # %entry
+; V-NEXT:    li a2, 64
+; V-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; V-NEXT:    vle32.v v8, (a0)
+; V-NEXT:    lui a0, 341
+; V-NEXT:    addiw a0, a0, 1365
+; V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; V-NEXT:    vmv.s.x v16, a0
+; V-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; V-NEXT:    vcompress.vm v24, v8, v16
+; V-NEXT:    vse32.v v24, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i32_single_src_m8:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    li a2, 64
+; ZVE32F-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZVE32F-NEXT:    vle32.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
+; ZVE32F-NEXT:    vmv.v.i v16, 0
+; ZVE32F-NEXT:    lui a0, 341
+; ZVE32F-NEXT:    addi a0, a0, 1365
+; ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
+; ZVE32F-NEXT:    vmv.s.x v16, a0
+; ZVE32F-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZVE32F-NEXT:    vcompress.vm v24, v8, v16
+; ZVE32F-NEXT:    vse32.v v24, (a1)
+; ZVE32F-NEXT:    ret
 entry:
   %0 = load <64 x i32>, ptr %in, align 4
   %shuffle.i5 = shufflevector <64 x i32> %0, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
index 1a08c613ca36a..29d9a8a9b060c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
@@ -62,8 +62,8 @@ define void @gather_masked(ptr noalias nocapture %A, ptr noalias nocapture reado
 ; CHECK-NEXT:    li a4, 5
 ; CHECK-NEXT:  .LBB1_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
+; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vlse8.v v9, (a1), a4, v0.t
 ; CHECK-NEXT:    vle8.v v10, (a0)
 ; CHECK-NEXT:    vadd.vv v9, v10, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll
index 1c2c90478a1f7..4b7f82f94f5e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll
@@ -542,6 +542,7 @@ declare <3 x double> @llvm.experimental.vp.strided.load.v3f64.p0.i32(ptr, i32, <
 define <32 x double> @strided_vpload_v32f64(ptr %ptr, i32 signext %stride, <32 x i1> %m, i32 zeroext %evl) nounwind {
 ; CHECK-LABEL: strided_vpload_v32f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    li a4, 16
 ; CHECK-NEXT:    mv a3, a2
@@ -598,6 +599,7 @@ declare <32 x double> @llvm.experimental.vp.strided.load.v32f64.p0.i32(ptr, i32,
 define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask, i32 zeroext %evl) {
 ; CHECK-RV32-LABEL: strided_load_v33f64:
 ; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv1r.v v8, v0
 ; CHECK-RV32-NEXT:    li a5, 32
 ; CHECK-RV32-NEXT:    mv a3, a4
@@ -648,6 +650,7 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask
 ;
 ; CHECK-RV64-LABEL: strided_load_v33f64:
 ; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv1r.v v8, v0
 ; CHECK-RV64-NEXT:    li a5, 32
 ; CHECK-RV64-NEXT:    mv a4, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
index 12893ec55cda7..a91dee1cb245f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
@@ -53,9 +53,9 @@ declare <128 x i7> @llvm.vp.trunc.v128i7.v128i16(<128 x i16>, <128 x i1>, i32)
 define <128 x i7> @vtrunc_v128i7_v128i16(<128 x i16> %a, <128 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vtrunc_v128i7_v128i16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v12, v0, 8
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:    bltu a0, a1, .LBB4_2
@@ -231,6 +231,7 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze
 ; CHECK-NEXT:    mul a2, a2, a3
 ; CHECK-NEXT:    sub sp, sp, a2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 72 * vlenb
+; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    li a3, 24
@@ -243,7 +244,6 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v6, v0, 8
 ; CHECK-NEXT:    addi a2, a1, 512
 ; CHECK-NEXT:    addi a3, a1, 640
@@ -541,9 +541,9 @@ declare <32 x i32> @llvm.vp.trunc.v32i32.v32i64(<32 x i64>, <32 x i1>, i32)
 define <32 x i32> @vtrunc_v32i32_v32i64(<32 x i64> %a, <32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vtrunc_v32i32_v32i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    li a2, 16
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v12, v0, 2
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    bltu a0, a2, .LBB17_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
index db03dc3d5ab1e..6d9f69f436fc4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
@@ -80,14 +80,8 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV32-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a0, v0
 ; RV32-SLOW-NEXT:    andi a1, a0, 1
-; RV32-SLOW-NEXT:    bnez a1, .LBB4_3
-; RV32-SLOW-NEXT:  # %bb.1: # %else
-; RV32-SLOW-NEXT:    andi a0, a0, 2
-; RV32-SLOW-NEXT:    bnez a0, .LBB4_4
-; RV32-SLOW-NEXT:  .LBB4_2: # %else2
-; RV32-SLOW-NEXT:    vmv1r.v v8, v9
-; RV32-SLOW-NEXT:    ret
-; RV32-SLOW-NEXT:  .LBB4_3: # %cond.load
+; RV32-SLOW-NEXT:    beqz a1, .LBB4_2
+; RV32-SLOW-NEXT:  # %bb.1: # %cond.load
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV32-SLOW-NEXT:    lbu a2, 1(a1)
@@ -96,9 +90,10 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV32-SLOW-NEXT:    or a1, a2, a1
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e16, m2, tu, ma
 ; RV32-SLOW-NEXT:    vmv.s.x v9, a1
+; RV32-SLOW-NEXT:  .LBB4_2: # %else
 ; RV32-SLOW-NEXT:    andi a0, a0, 2
-; RV32-SLOW-NEXT:    beqz a0, .LBB4_2
-; RV32-SLOW-NEXT:  .LBB4_4: # %cond.load1
+; RV32-SLOW-NEXT:    beqz a0, .LBB4_4
+; RV32-SLOW-NEXT:  # %bb.3: # %cond.load1
 ; RV32-SLOW-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; RV32-SLOW-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-SLOW-NEXT:    vmv.x.s a0, v8
@@ -109,6 +104,8 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV32-SLOW-NEXT:    vmv.s.x v8, a0
 ; RV32-SLOW-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; RV32-SLOW-NEXT:    vslideup.vi v9, v8, 1
+; RV32-SLOW-NEXT:  .LBB4_4: # %else2
+; RV32-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-SLOW-NEXT:    vmv1r.v v8, v9
 ; RV32-SLOW-NEXT:    ret
 ;
@@ -117,14 +114,8 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV64-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a0, v0
 ; RV64-SLOW-NEXT:    andi a1, a0, 1
-; RV64-SLOW-NEXT:    bnez a1, .LBB4_3
-; RV64-SLOW-NEXT:  # %bb.1: # %else
-; RV64-SLOW-NEXT:    andi a0, a0, 2
-; RV64-SLOW-NEXT:    bnez a0, .LBB4_4
-; RV64-SLOW-NEXT:  .LBB4_2: # %else2
-; RV64-SLOW-NEXT:    vmv1r.v v8, v9
-; RV64-SLOW-NEXT:    ret
-; RV64-SLOW-NEXT:  .LBB4_3: # %cond.load
+; RV64-SLOW-NEXT:    beqz a1, .LBB4_2
+; RV64-SLOW-NEXT:  # %bb.1: # %cond.load
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV64-SLOW-NEXT:    lbu a2, 1(a1)
@@ -133,9 +124,10 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV64-SLOW-NEXT:    or a1, a2, a1
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e16, m2, tu, ma
 ; RV64-SLOW-NEXT:    vmv.s.x v9, a1
+; RV64-SLOW-NEXT:  .LBB4_2: # %else
 ; RV64-SLOW-NEXT:    andi a0, a0, 2
-; RV64-SLOW-NEXT:    beqz a0, .LBB4_2
-; RV64-SLOW-NEXT:  .LBB4_4: # %cond.load1
+; RV64-SLOW-NEXT:    beqz a0, .LBB4_4
+; RV64-SLOW-NEXT:  # %bb.3: # %cond.load1
 ; RV64-SLOW-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-SLOW-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-SLOW-NEXT:    vmv.x.s a0, v8
@@ -146,6 +138,8 @@ define <2 x i16> @mgather_v2i16_align1(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i16> %
 ; RV64-SLOW-NEXT:    vmv.s.x v8, a0
 ; RV64-SLOW-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; RV64-SLOW-NEXT:    vslideup.vi v9, v8, 1
+; RV64-SLOW-NEXT:  .LBB4_4: # %else2
+; RV64-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-SLOW-NEXT:    vmv1r.v v8, v9
 ; RV64-SLOW-NEXT:    ret
 ;
@@ -174,23 +168,18 @@ define <2 x i64> @mgather_v2i64_align4(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i64> %
 ; RV32-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a0, v0
 ; RV32-SLOW-NEXT:    andi a1, a0, 1
-; RV32-SLOW-NEXT:    bnez a1, .LBB5_3
-; RV32-SLOW-NEXT:  # %bb.1: # %else
-; RV32-SLOW-NEXT:    andi a0, a0, 2
-; RV32-SLOW-NEXT:    bnez a0, .LBB5_4
-; RV32-SLOW-NEXT:  .LBB5_2: # %else2
-; RV32-SLOW-NEXT:    vmv1r.v v8, v9
-; RV32-SLOW-NEXT:    ret
-; RV32-SLOW-NEXT:  .LBB5_3: # %cond.load
+; RV32-SLOW-NEXT:    beqz a1, .LBB5_2
+; RV32-SLOW-NEXT:  # %bb.1: # %cond.load
 ; RV32-SLOW-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV32-SLOW-NEXT:    lw a2, 0(a1)
 ; RV32-SLOW-NEXT:    lw a1, 4(a1)
 ; RV32-SLOW-NEXT:    vslide1down.vx v9, v9, a2
 ; RV32-SLOW-NEXT:    vslide1down.vx v9, v9, a1
+; RV32-SLOW-NEXT:  .LBB5_2: # %else
 ; RV32-SLOW-NEXT:    andi a0, a0, 2
-; RV32-SLOW-NEXT:    beqz a0, .LBB5_2
-; RV32-SLOW-NEXT:  .LBB5_4: # %cond.load1
+; RV32-SLOW-NEXT:    beqz a0, .LBB5_4
+; RV32-SLOW-NEXT:  # %bb.3: # %cond.load1
 ; RV32-SLOW-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; RV32-SLOW-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-SLOW-NEXT:    vmv.x.s a0, v8
@@ -201,6 +190,8 @@ define <2 x i64> @mgather_v2i64_align4(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i64> %
 ; RV32-SLOW-NEXT:    vslide1down.vx v8, v8, a0
 ; RV32-SLOW-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-SLOW-NEXT:    vslideup.vi v9, v8, 1
+; RV32-SLOW-NEXT:  .LBB5_4: # %else2
+; RV32-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-SLOW-NEXT:    vmv1r.v v8, v9
 ; RV32-SLOW-NEXT:    ret
 ;
@@ -209,14 +200,8 @@ define <2 x i64> @mgather_v2i64_align4(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i64> %
 ; RV64-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a0, v0
 ; RV64-SLOW-NEXT:    andi a1, a0, 1
-; RV64-SLOW-NEXT:    bnez a1, .LBB5_3
-; RV64-SLOW-NEXT:  # %bb.1: # %else
-; RV64-SLOW-NEXT:    andi a0, a0, 2
-; RV64-SLOW-NEXT:    bnez a0, .LBB5_4
-; RV64-SLOW-NEXT:  .LBB5_2: # %else2
-; RV64-SLOW-NEXT:    vmv1r.v v8, v9
-; RV64-SLOW-NEXT:    ret
-; RV64-SLOW-NEXT:  .LBB5_3: # %cond.load
+; RV64-SLOW-NEXT:    beqz a1, .LBB5_2
+; RV64-SLOW-NEXT:  # %bb.1: # %cond.load
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e64, m8, tu, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV64-SLOW-NEXT:    lwu a2, 4(a1)
@@ -224,9 +209,10 @@ define <2 x i64> @mgather_v2i64_align4(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i64> %
 ; RV64-SLOW-NEXT:    slli a2, a2, 32
 ; RV64-SLOW-NEXT:    or a1, a2, a1
 ; RV64-SLOW-NEXT:    vmv.s.x v9, a1
+; RV64-SLOW-NEXT:  .LBB5_2: # %else
 ; RV64-SLOW-NEXT:    andi a0, a0, 2
-; RV64-SLOW-NEXT:    beqz a0, .LBB5_2
-; RV64-SLOW-NEXT:  .LBB5_4: # %cond.load1
+; RV64-SLOW-NEXT:    beqz a0, .LBB5_4
+; RV64-SLOW-NEXT:  # %bb.3: # %cond.load1
 ; RV64-SLOW-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-SLOW-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-SLOW-NEXT:    vmv.x.s a0, v8
@@ -236,6 +222,8 @@ define <2 x i64> @mgather_v2i64_align4(<2 x ptr> %ptrs, <2 x i1> %m, <2 x i64> %
 ; RV64-SLOW-NEXT:    or a0, a1, a0
 ; RV64-SLOW-NEXT:    vmv.s.x v8, a0
 ; RV64-SLOW-NEXT:    vslideup.vi v9, v8, 1
+; RV64-SLOW-NEXT:  .LBB5_4: # %else2
+; RV64-SLOW-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-SLOW-NEXT:    vmv1r.v v8, v9
 ; RV64-SLOW-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
index 5be1a771eb279..7ee8179acfdb9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
@@ -363,6 +363,7 @@ declare <256 x i8> @llvm.vp.add.v258i8(<256 x i8>, <256 x i8>, <256 x i1>, i32)
 define <256 x i8> @vadd_vi_v258i8(<256 x i8> %va, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vadd_vi_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
index ac48542ca9ebb..fec54b36042fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
@@ -267,6 +267,7 @@ declare <256 x i8> @llvm.vp.smax.v258i8(<256 x i8>, <256 x i8>, <256 x i1>, i32)
 define <256 x i8> @vmax_vx_v258i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmax_vx_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
index 794eef6ed40b2..7ca0dbd9adffc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
@@ -266,6 +266,7 @@ declare <256 x i8> @llvm.vp.umax.v258i8(<256 x i8>, <256 x i8>, <256 x i1>, i32)
 define <256 x i8> @vmaxu_vx_v258i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmaxu_vx_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
index 34011f6bd8acd..ea75742ca6e43 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
@@ -267,6 +267,7 @@ declare <256 x i8> @llvm.vp.smin.v258i8(<256 x i8>, <256 x i8>, <256 x i1>, i32)
 define <256 x i8> @vmin_vx_v258i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmin_vx_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
index 79e72b7d9cac9..f4f54db64018d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
@@ -266,6 +266,7 @@ declare <256 x i8> @llvm.vp.umin.v258i8(<256 x i8>, <256 x i8>, <256 x i1>, i32)
 define <256 x i8> @vminu_vx_v258i8(<256 x i8> %va, i8 %b, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vminu_vx_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a3, 128
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
index 24e75cde2ce91..df9ff0fc39a7e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpgather.ll
@@ -2617,8 +2617,8 @@ define <32 x double> @vpgather_baseidx_zext_v32i32_v32f64(ptr %base, <32 x i32>
 define <32 x double> @vpgather_baseidx_v32f64(ptr %base, <32 x i64> %idxs, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_v32f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vmv1r.v v7, v0
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vmv1r.v v7, v0
 ; RV32-NEXT:    vnsrl.wi v24, v16, 0
 ; RV32-NEXT:    vnsrl.wi v16, v8, 0
 ; RV32-NEXT:    li a2, 32
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
index 71f497e4c7be4..6c9989775f790 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll
@@ -394,6 +394,7 @@ declare <33 x double> @llvm.vp.load.v33f64.p0(ptr, <33 x i1>, i32)
 define <33 x double> @vpload_v33f64(ptr %ptr, <33 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpload_v33f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    li a4, 32
 ; CHECK-NEXT:    mv a3, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
index a11c2b6bca12e..a53d33e6120d5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
@@ -1181,9 +1181,9 @@ define <32 x double> @vpmerge_vv_v32f64(<32 x double> %va, <32 x double> %vb, <3
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    addi a1, a0, 128
-; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vle64.v v24, (a1)
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    li a1, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
index 44d4a8a1e04cd..0d31ec5f78435 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
@@ -763,3 +763,475 @@ define zeroext i1 @vreduce_add_v64i1(<64 x i1> %v) {
   %red = call i1 @llvm.vector.reduce.add.v64i1(<64 x i1> %v)
   ret i1 %red
 }
+
+declare i1 @llvm.vector.reduce.or.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_or_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_or_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vcpop.m a0, v0
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_xor_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vcpop.m a0, v0
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_and_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_and_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnot.m v8, v0
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_umax_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vcpop.m a0, v0
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_smax_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnot.m v8, v0
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_umin_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnot.m v8, v0
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.v128i1(<128 x i1>)
+
+define zeroext i1 @vreduce_smin_v128i1(<128 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_v128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vcpop.m a0, v0
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.v128i1(<128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_or_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_or_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_xor_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_and_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_and_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnand.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_umax_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_smax_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnand.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_umin_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmnand.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.v256i1(<256 x i1>)
+
+define zeroext i1 @vreduce_smin_v256i1(<256 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.v256i1(<256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_or_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_or_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_xor_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v8, v8, v10
+; CHECK-NEXT:    vmxor.mm v9, v0, v9
+; CHECK-NEXT:    vmxor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_and_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_and_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_umax_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_smax_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_umin_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.v512i1(<512 x i1>)
+
+define zeroext i1 @vreduce_smin_v512i1(<512 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.v512i1(<512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_or_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_or_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_xor_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v10, v10, v14
+; CHECK-NEXT:    vmxor.mm v8, v8, v12
+; CHECK-NEXT:    vmxor.mm v9, v9, v13
+; CHECK-NEXT:    vmxor.mm v11, v0, v11
+; CHECK-NEXT:    vmxor.mm v8, v8, v10
+; CHECK-NEXT:    vmxor.mm v9, v11, v9
+; CHECK-NEXT:    vmxor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_and_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_and_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_umax_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_smax_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_umin_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmnand.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.v1024i1(<1024 x i1>)
+
+define zeroext i1 @vreduce_smin_v1024i1(<1024 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_v1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 128
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.v1024i1(<1024 x i1> %v)
+  ret i1 %red
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
index 888fc79f0122d..7afd31fdd663c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
@@ -372,6 +372,7 @@ declare <256 x i8> @llvm.vp.sadd.sat.v258i8(<256 x i8>, <256 x i8>, <256 x i1>,
 define <256 x i8> @vsadd_vi_v258i8(<256 x i8> %va, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsadd_vi_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
index e1d57ae1e6741..f61b112fd8024 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
@@ -368,6 +368,7 @@ declare <256 x i8> @llvm.vp.uadd.sat.v258i8(<256 x i8>, <256 x i8>, <256 x i1>,
 define <256 x i8> @vsaddu_vi_v258i8(<256 x i8> %va, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsaddu_vi_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
index 1d8af4c46cc07..5d407caf71514 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll
@@ -163,6 +163,7 @@ define <256 x i8> @select_v256i8(<256 x i1> %a, <256 x i8> %b, <256 x i8> %c, i3
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v6, v8
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    li a2, 128
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
index 8fad3db55f9bc..6ddf2e464750e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
@@ -384,6 +384,7 @@ declare <256 x i8> @llvm.vp.ssub.sat.v258i8(<256 x i8>, <256 x i8>, <256 x i1>,
 define <256 x i8> @vssub_vi_v258i8(<256 x i8> %va, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssub_vi_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    addi a3, a1, -128
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
index ca35aa6c4a94c..c403593894794 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
@@ -379,6 +379,7 @@ declare <256 x i8> @llvm.vp.usub.sat.v258i8(<256 x i8>, <256 x i8>, <256 x i1>,
 define <256 x i8> @vssubu_vi_v258i8(<256 x i8> %va, <256 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssubu_vi_v258i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    addi a3, a1, -128
diff --git a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
index e6dfe5e78cdb4..74a00c655d526 100644
--- a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.floor.nxv4bf16(<vscale x 4 x bfloat>, <vs
 define <vscale x 4 x bfloat> @vp_floor_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.floor.nxv8bf16(<vscale x 8 x bfloat>, <vs
 define <vscale x 8 x bfloat> @vp_floor_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.floor.nxv16bf16(<vscale x 16 x bfloat>,
 define <vscale x 16 x bfloat> @vp_floor_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -279,9 +279,9 @@ define <vscale x 32 x bfloat> @vp_floor_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -582,8 +582,8 @@ define <vscale x 4 x half> @vp_floor_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vp_floor_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -649,10 +649,10 @@ declare <vscale x 8 x half> @llvm.vp.floor.nxv8f16(<vscale x 8 x half>, <vscale
 define <vscale x 8 x half> @vp_floor_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_floor_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -668,8 +668,8 @@ define <vscale x 8 x half> @vp_floor_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vp_floor_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -735,10 +735,10 @@ declare <vscale x 16 x half> @llvm.vp.floor.nxv16f16(<vscale x 16 x half>, <vsca
 define <vscale x 16 x half> @vp_floor_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_floor_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -754,8 +754,8 @@ define <vscale x 16 x half> @vp_floor_nxv16f16(<vscale x 16 x half> %va, <vscale
 ;
 ; ZVFHMIN-LABEL: vp_floor_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -821,10 +821,10 @@ declare <vscale x 32 x half> @llvm.vp.floor.nxv32f16(<vscale x 32 x half>, <vsca
 define <vscale x 32 x half> @vp_floor_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_floor_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -846,9 +846,9 @@ define <vscale x 32 x half> @vp_floor_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1068,8 +1068,8 @@ declare <vscale x 4 x float> @llvm.vp.floor.nxv4f32(<vscale x 4 x float>, <vscal
 define <vscale x 4 x float> @vp_floor_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1112,8 +1112,8 @@ declare <vscale x 8 x float> @llvm.vp.floor.nxv8f32(<vscale x 8 x float>, <vscal
 define <vscale x 8 x float> @vp_floor_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1156,8 +1156,8 @@ declare <vscale x 16 x float> @llvm.vp.floor.nxv16f32(<vscale x 16 x float>, <vs
 define <vscale x 16 x float> @vp_floor_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1242,10 +1242,10 @@ declare <vscale x 2 x double> @llvm.vp.floor.nxv2f64(<vscale x 2 x double>, <vsc
 define <vscale x 2 x double> @vp_floor_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1286,10 +1286,10 @@ declare <vscale x 4 x double> @llvm.vp.floor.nxv4f64(<vscale x 4 x double>, <vsc
 define <vscale x 4 x double> @vp_floor_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1330,10 +1330,10 @@ declare <vscale x 7 x double> @llvm.vp.floor.nxv7f64(<vscale x 7 x double>, <vsc
 define <vscale x 7 x double> @vp_floor_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1374,10 +1374,10 @@ declare <vscale x 8 x double> @llvm.vp.floor.nxv8f64(<vscale x 8 x double>, <vsc
 define <vscale x 8 x double> @vp_floor_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_floor_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1425,13 +1425,13 @@ define <vscale x 16 x double> @vp_floor_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
index 734dd5e33c4fc..8b527fb152d68 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
@@ -22,15 +22,16 @@ define <vscale x 1 x bfloat> @vfmax_nxv1bf16_vv(<vscale x 1 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmax_nxv1bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v9, v9
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v11, v9, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
-; CHECK-NEXT:    vfmax.vv v9, v8, v11
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
+; CHECK-NEXT:    vfmax.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -44,15 +45,16 @@ define <vscale x 2 x bfloat> @vfmax_nxv2bf16_vv(<vscale x 2 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmax_nxv2bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v9, v9
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v11, v9, v10, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
-; CHECK-NEXT:    vfmax.vv v9, v8, v11
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
+; CHECK-NEXT:    vfmax.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -66,15 +68,16 @@ define <vscale x 4 x bfloat> @vfmax_nxv4bf16_vv(<vscale x 4 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmax_nxv4bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v12, v12
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v12, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v12, v0
-; CHECK-NEXT:    vfmax.vv v10, v8, v14
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vmerge.vvm v10, v12, v10, v0
+; CHECK-NEXT:    vfmax.vv v10, v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -88,15 +91,16 @@ define <vscale x 8 x bfloat> @vfmax_nxv8bf16_vv(<vscale x 8 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmax_nxv8bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v10
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v16, v16
-; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v16, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v16, v0
-; CHECK-NEXT:    vfmax.vv v12, v8, v20
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmerge.vvm v12, v16, v12, v0
+; CHECK-NEXT:    vfmax.vv v12, v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -153,6 +157,7 @@ define <vscale x 32 x bfloat> @vfmax_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFH-NEXT:    slli a0, a0, 1
 ; ZVFH-NEXT:    add a0, a0, a1
 ; ZVFH-NEXT:    sub sp, sp, a0
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv8r.v v24, v16
 ; ZVFH-NEXT:    csrr a0, vlenb
 ; ZVFH-NEXT:    slli a0, a0, 3
@@ -160,7 +165,6 @@ define <vscale x 32 x bfloat> @vfmax_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFH-NEXT:    addi a0, a0, 16
 ; ZVFH-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFH-NEXT:    vmv8r.v v0, v8
-; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v16, v24
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v8, v0
 ; ZVFH-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -228,6 +232,7 @@ define <vscale x 32 x bfloat> @vfmax_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFHMIN-NEXT:    li a1, 24
 ; ZVFHMIN-NEXT:    mul a0, a0, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
@@ -235,7 +240,6 @@ define <vscale x 32 x bfloat> @vfmax_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v16, v24
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v8, v0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -304,25 +308,25 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vv(<vscale x 1 x half> %a, <vscale x 1
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv1f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -337,25 +341,25 @@ define <vscale x 2 x half> @vfmax_nxv2f16_vv(<vscale x 2 x half> %a, <vscale x 2
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv2f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -370,25 +374,25 @@ define <vscale x 4 x half> @vfmax_nxv4f16_vv(<vscale x 4 x half> %a, <vscale x 4
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv4f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v12, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
-; ZVFHMIN-NEXT:    vfmax.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -403,25 +407,25 @@ define <vscale x 8 x half> @vfmax_nxv8f16_vv(<vscale x 8 x half> %a, <vscale x 8
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v14
+; ZVFH-NEXT:    vfmax.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv8f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v10
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v16, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
-; ZVFHMIN-NEXT:    vfmax.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmax.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -436,11 +440,10 @@ define <vscale x 16 x half> @vfmax_nxv16f16_vv(<vscale x 16 x half> %a, <vscale
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v16, v12, v12
-; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
-; ZVFH-NEXT:    vmv1r.v v0, v16
+; ZVFH-NEXT:    vmerge.vvm v16, v8, v12, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v12, v12
 ; ZVFH-NEXT:    vmerge.vvm v8, v12, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v20
+; ZVFH-NEXT:    vfmax.vv v8, v8, v16
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_nxv16f16_vv:
@@ -498,6 +501,7 @@ define <vscale x 32 x half> @vfmax_nxv32f16_vv(<vscale x 32 x half> %a, <vscale
 ; ZVFHMIN-NEXT:    li a1, 24
 ; ZVFHMIN-NEXT:    mul a0, a0, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
@@ -505,7 +509,6 @@ define <vscale x 32 x half> @vfmax_nxv32f16_vv(<vscale x 32 x half> %a, <vscale
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v24
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -574,11 +577,10 @@ define <vscale x 1 x float> @vfmax_nxv1f32_vv(<vscale x 1 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.maximum.nxv1f32(<vscale x 1 x float> %a, <vscale x 1 x float> %b)
   ret <vscale x 1 x float> %v
@@ -591,11 +593,10 @@ define <vscale x 2 x float> @vfmax_nxv2f32_vv(<vscale x 2 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.maximum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
   ret <vscale x 2 x float> %v
@@ -608,11 +609,10 @@ define <vscale x 4 x float> @vfmax_nxv4f32_vv(<vscale x 4 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.maximum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %v
@@ -625,11 +625,10 @@ define <vscale x 8 x float> @vfmax_nxv8f32_vv(<vscale x 8 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.maximum.nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b)
   ret <vscale x 8 x float> %v
@@ -659,11 +658,10 @@ define <vscale x 1 x double> @vfmax_nxv1f64_vv(<vscale x 1 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.maximum.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b)
   ret <vscale x 1 x double> %v
@@ -676,11 +674,10 @@ define <vscale x 2 x double> @vfmax_nxv2f64_vv(<vscale x 2 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.maximum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %v
@@ -693,11 +690,10 @@ define <vscale x 4 x double> @vfmax_nxv4f64_vv(<vscale x 4 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.maximum.nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b)
   ret <vscale x 4 x double> %v
@@ -755,19 +751,18 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vv_nnana(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v11, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v11, v0
-; ZVFHMIN-NEXT:    vfmax.vv v9, v9, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -796,12 +791,11 @@ define <vscale x 1 x half> @vfmax_nxv1f16_vv_nnanb(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v10
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v9, v0
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
index e90d3e3f3e829..7649d60def111 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
@@ -19,8 +19,8 @@ declare <vscale x 1 x bfloat> @llvm.vp.maximum.nxv1bf16(<vscale x 1 x bfloat>, <
 define <vscale x 1 x bfloat> @vfmax_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -45,13 +45,14 @@ define <vscale x 1 x bfloat> @vfmax_vv_nxv1bf16_unmasked(<vscale x 1 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v11, v11
-; CHECK-NEXT:    vmerge.vvm v9, v10, v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; CHECK-NEXT:    vfmax.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
@@ -65,8 +66,8 @@ declare <vscale x 2 x bfloat> @llvm.vp.maximum.nxv2bf16(<vscale x 2 x bfloat>, <
 define <vscale x 2 x bfloat> @vfmax_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -91,13 +92,14 @@ define <vscale x 2 x bfloat> @vfmax_vv_nxv2bf16_unmasked(<vscale x 2 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v11, v11
-; CHECK-NEXT:    vmerge.vvm v9, v10, v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; CHECK-NEXT:    vfmax.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
@@ -111,8 +113,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.maximum.nxv4bf16(<vscale x 4 x bfloat>, <
 define <vscale x 4 x bfloat> @vfmax_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -139,14 +141,15 @@ define <vscale x 4 x bfloat> @vfmax_vv_nxv4bf16_unmasked(<vscale x 4 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmerge.vvm v14, v10, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v12, v10, v0
-; CHECK-NEXT:    vfmax.vv v10, v8, v14
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vmerge.vvm v10, v12, v10, v0
+; CHECK-NEXT:    vfmax.vv v10, v10, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -159,8 +162,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.maximum.nxv8bf16(<vscale x 8 x bfloat>, <
 define <vscale x 8 x bfloat> @vfmax_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -187,14 +190,15 @@ define <vscale x 8 x bfloat> @vfmax_vv_nxv8bf16_unmasked(<vscale x 8 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v12, v12
-; CHECK-NEXT:    vmfeq.vv v8, v16, v16
-; CHECK-NEXT:    vmerge.vvm v20, v12, v16, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v16, v12, v0
-; CHECK-NEXT:    vfmax.vv v12, v8, v20
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v16, v0
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmerge.vvm v12, v16, v12, v0
+; CHECK-NEXT:    vfmax.vv v12, v12, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -213,8 +217,8 @@ define <vscale x 16 x bfloat> @vfmax_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v24, v24, v0.t
@@ -565,8 +569,8 @@ declare <vscale x 1 x half> @llvm.vp.maximum.nxv1f16(<vscale x 1 x half>, <vscal
 define <vscale x 1 x half> @vfmax_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -578,8 +582,8 @@ define <vscale x 1 x half> @vfmax_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -604,24 +608,24 @@ define <vscale x 1 x half> @vfmax_vv_nxv1f16_unmasked(<vscale x 1 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv1f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -635,8 +639,8 @@ declare <vscale x 2 x half> @llvm.vp.maximum.nxv2f16(<vscale x 2 x half>, <vscal
 define <vscale x 2 x half> @vfmax_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_nxv2f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -648,8 +652,8 @@ define <vscale x 2 x half> @vfmax_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -674,24 +678,24 @@ define <vscale x 2 x half> @vfmax_vv_nxv2f16_unmasked(<vscale x 2 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv2f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -705,8 +709,8 @@ declare <vscale x 4 x half> @llvm.vp.maximum.nxv4f16(<vscale x 4 x half>, <vscal
 define <vscale x 4 x half> @vfmax_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_nxv4f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -718,8 +722,8 @@ define <vscale x 4 x half> @vfmax_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -746,25 +750,25 @@ define <vscale x 4 x half> @vfmax_vv_nxv4f16_unmasked(<vscale x 4 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v11
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv4f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v10, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v10, v0
-; ZVFHMIN-NEXT:    vfmax.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmax.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -777,8 +781,8 @@ declare <vscale x 8 x half> @llvm.vp.maximum.nxv8f16(<vscale x 8 x half>, <vscal
 define <vscale x 8 x half> @vfmax_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v13
 ; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -792,8 +796,8 @@ define <vscale x 8 x half> @vfmax_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -820,25 +824,25 @@ define <vscale x 8 x half> @vfmax_vv_nxv8f16_unmasked(<vscale x 8 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v14
+; ZVFH-NEXT:    vfmax.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv8f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v12, v16, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v16, v12, v0
-; ZVFHMIN-NEXT:    vfmax.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmax.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -851,8 +855,8 @@ declare <vscale x 16 x half> @llvm.vp.maximum.nxv16f16(<vscale x 16 x half>, <vs
 define <vscale x 16 x half> @vfmax_vv_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %vb, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmax_vv_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
 ; ZVFH-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v17
 ; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -872,8 +876,8 @@ define <vscale x 16 x half> @vfmax_vv_nxv16f16(<vscale x 16 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v24, v24, v0.t
@@ -909,11 +913,10 @@ define <vscale x 16 x half> @vfmax_vv_nxv16f16_unmasked(<vscale x 16 x half> %va
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v16, v12, v12
-; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
-; ZVFH-NEXT:    vmv1r.v v0, v16
+; ZVFH-NEXT:    vmerge.vvm v16, v8, v12, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v12, v12
 ; ZVFH-NEXT:    vmerge.vvm v8, v12, v8, v0
-; ZVFH-NEXT:    vfmax.vv v8, v8, v20
+; ZVFH-NEXT:    vfmax.vv v8, v8, v16
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmax_vv_nxv16f16_unmasked:
@@ -961,8 +964,8 @@ define <vscale x 32 x half> @vfmax_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFH-NEXT:    slli a1, a1, 3
 ; ZVFH-NEXT:    sub sp, sp, a1
 ; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; ZVFH-NEXT:    vmv1r.v v7, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v7, v0
 ; ZVFH-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v25
 ; ZVFH-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -1277,8 +1280,8 @@ declare <vscale x 1 x float> @llvm.vp.maximum.nxv1f32(<vscale x 1 x float>, <vsc
 define <vscale x 1 x float> @vfmax_vv_nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x float> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv1f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1296,11 +1299,10 @@ define <vscale x 1 x float> @vfmax_vv_nxv1f32_unmasked(<vscale x 1 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.vp.maximum.nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x float> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x float> %v
@@ -1311,8 +1313,8 @@ declare <vscale x 2 x float> @llvm.vp.maximum.nxv2f32(<vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @vfmax_vv_nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1330,11 +1332,10 @@ define <vscale x 2 x float> @vfmax_vv_nxv2f32_unmasked(<vscale x 2 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.vp.maximum.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x float> %v
@@ -1345,8 +1346,8 @@ declare <vscale x 4 x float> @llvm.vp.maximum.nxv4f32(<vscale x 4 x float>, <vsc
 define <vscale x 4 x float> @vfmax_vv_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x float> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -1366,11 +1367,10 @@ define <vscale x 4 x float> @vfmax_vv_nxv4f32_unmasked(<vscale x 4 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.vp.maximum.nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x float> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x float> %v
@@ -1381,8 +1381,8 @@ declare <vscale x 8 x float> @llvm.vp.maximum.nxv8f32(<vscale x 8 x float>, <vsc
 define <vscale x 8 x float> @vfmax_vv_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x float> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -1402,11 +1402,10 @@ define <vscale x 8 x float> @vfmax_vv_nxv8f32_unmasked(<vscale x 8 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.vp.maximum.nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x float> %vb, <vscale x 8 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 8 x float> %v
@@ -1417,8 +1416,8 @@ declare <vscale x 1 x double> @llvm.vp.maximum.nxv1f64(<vscale x 1 x double>, <v
 define <vscale x 1 x double> @vfmax_vv_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1436,11 +1435,10 @@ define <vscale x 1 x double> @vfmax_vv_nxv1f64_unmasked(<vscale x 1 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v11
+; CHECK-NEXT:    vfmax.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.vp.maximum.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x double> %v
@@ -1451,8 +1449,8 @@ declare <vscale x 2 x double> @llvm.vp.maximum.nxv2f64(<vscale x 2 x double>, <v
 define <vscale x 2 x double> @vfmax_vv_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x double> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -1472,11 +1470,10 @@ define <vscale x 2 x double> @vfmax_vv_nxv2f64_unmasked(<vscale x 2 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v14
+; CHECK-NEXT:    vfmax.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.vp.maximum.nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x double> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x double> %v
@@ -1487,8 +1484,8 @@ declare <vscale x 4 x double> @llvm.vp.maximum.nxv4f64(<vscale x 4 x double>, <v
 define <vscale x 4 x double> @vfmax_vv_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x double> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmax_vv_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -1508,11 +1505,10 @@ define <vscale x 4 x double> @vfmax_vv_nxv4f64_unmasked(<vscale x 4 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmax.vv v8, v8, v20
+; CHECK-NEXT:    vfmax.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.vp.maximum.nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x double> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x double> %v
@@ -1529,8 +1525,8 @@ define <vscale x 8 x double> @vfmax_vv_nxv8f64(<vscale x 8 x double> %va, <vscal
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -1581,6 +1577,7 @@ define <vscale x 16 x double> @vfmax_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -1592,7 +1589,6 @@ define <vscale x 16 x double> @vfmax_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    srli a4, a1, 3
-; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a4
 ; CHECK-NEXT:    sub a4, a2, a1
 ; CHECK-NEXT:    add a3, a0, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
index 21251ee2f3c63..16c119dd87fe0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
@@ -22,15 +22,16 @@ define <vscale x 1 x bfloat> @vfmin_nxv1bf16_vv(<vscale x 1 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmin_nxv1bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v9, v9
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v11, v9, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
-; CHECK-NEXT:    vfmin.vv v9, v8, v11
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
+; CHECK-NEXT:    vfmin.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -44,15 +45,16 @@ define <vscale x 2 x bfloat> @vfmin_nxv2bf16_vv(<vscale x 2 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmin_nxv2bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v9, v9
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v11, v9, v10, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
-; CHECK-NEXT:    vfmin.vv v9, v8, v11
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
+; CHECK-NEXT:    vfmin.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
 ; CHECK-NEXT:    ret
@@ -66,15 +68,16 @@ define <vscale x 4 x bfloat> @vfmin_nxv4bf16_vv(<vscale x 4 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmin_nxv4bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v12, v12
-; CHECK-NEXT:    vmfeq.vv v8, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v12, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v12, v0
-; CHECK-NEXT:    vfmin.vv v10, v8, v14
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vmerge.vvm v10, v12, v10, v0
+; CHECK-NEXT:    vfmin.vv v10, v10, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -88,15 +91,16 @@ define <vscale x 8 x bfloat> @vfmin_nxv8bf16_vv(<vscale x 8 x bfloat> %a, <vscal
 ; CHECK-LABEL: vfmin_nxv8bf16_vv:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v10
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v16, v16
-; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v16, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v16, v0
-; CHECK-NEXT:    vfmin.vv v12, v8, v20
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmerge.vvm v12, v16, v12, v0
+; CHECK-NEXT:    vfmin.vv v12, v12, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -153,6 +157,7 @@ define <vscale x 32 x bfloat> @vfmin_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFH-NEXT:    slli a0, a0, 1
 ; ZVFH-NEXT:    add a0, a0, a1
 ; ZVFH-NEXT:    sub sp, sp, a0
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv8r.v v24, v16
 ; ZVFH-NEXT:    csrr a0, vlenb
 ; ZVFH-NEXT:    slli a0, a0, 3
@@ -160,7 +165,6 @@ define <vscale x 32 x bfloat> @vfmin_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFH-NEXT:    addi a0, a0, 16
 ; ZVFH-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFH-NEXT:    vmv8r.v v0, v8
-; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v16, v24
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v8, v0
 ; ZVFH-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -228,6 +232,7 @@ define <vscale x 32 x bfloat> @vfmin_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFHMIN-NEXT:    li a1, 24
 ; ZVFHMIN-NEXT:    mul a0, a0, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
@@ -235,7 +240,6 @@ define <vscale x 32 x bfloat> @vfmin_nxv32bf16_vv(<vscale x 32 x bfloat> %a, <vs
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v16, v24
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v8, v0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -304,25 +308,25 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vv(<vscale x 1 x half> %a, <vscale x 1
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv1f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -337,25 +341,25 @@ define <vscale x 2 x half> @vfmin_nxv2f16_vv(<vscale x 2 x half> %a, <vscale x 2
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv2f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v11, v9, v10, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v9, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v11
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -370,25 +374,25 @@ define <vscale x 4 x half> @vfmin_nxv4f16_vv(<vscale x 4 x half> %a, <vscale x 4
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv4f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v12, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
-; ZVFHMIN-NEXT:    vfmin.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -403,25 +407,25 @@ define <vscale x 8 x half> @vfmin_nxv8f16_vv(<vscale x 8 x half> %a, <vscale x 8
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v14
+; ZVFH-NEXT:    vfmin.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv8f16_vv:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v10
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v16, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
 ; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
-; ZVFHMIN-NEXT:    vfmin.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmin.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -436,11 +440,10 @@ define <vscale x 16 x half> @vfmin_nxv16f16_vv(<vscale x 16 x half> %a, <vscale
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v16, v12, v12
-; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
-; ZVFH-NEXT:    vmv1r.v v0, v16
+; ZVFH-NEXT:    vmerge.vvm v16, v8, v12, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v12, v12
 ; ZVFH-NEXT:    vmerge.vvm v8, v12, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v20
+; ZVFH-NEXT:    vfmin.vv v8, v8, v16
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_nxv16f16_vv:
@@ -498,6 +501,7 @@ define <vscale x 32 x half> @vfmin_nxv32f16_vv(<vscale x 32 x half> %a, <vscale
 ; ZVFHMIN-NEXT:    li a1, 24
 ; ZVFHMIN-NEXT:    mul a0, a0, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
@@ -505,7 +509,6 @@ define <vscale x 32 x half> @vfmin_nxv32f16_vv(<vscale x 32 x half> %a, <vscale
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v24
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
@@ -574,11 +577,10 @@ define <vscale x 1 x float> @vfmin_nxv1f32_vv(<vscale x 1 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.minimum.nxv1f32(<vscale x 1 x float> %a, <vscale x 1 x float> %b)
   ret <vscale x 1 x float> %v
@@ -591,11 +593,10 @@ define <vscale x 2 x float> @vfmin_nxv2f32_vv(<vscale x 2 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.minimum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
   ret <vscale x 2 x float> %v
@@ -608,11 +609,10 @@ define <vscale x 4 x float> @vfmin_nxv4f32_vv(<vscale x 4 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.minimum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %v
@@ -625,11 +625,10 @@ define <vscale x 8 x float> @vfmin_nxv8f32_vv(<vscale x 8 x float> %a, <vscale x
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.minimum.nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b)
   ret <vscale x 8 x float> %v
@@ -659,11 +658,10 @@ define <vscale x 1 x double> @vfmin_nxv1f64_vv(<vscale x 1 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.minimum.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b)
   ret <vscale x 1 x double> %v
@@ -676,11 +674,10 @@ define <vscale x 2 x double> @vfmin_nxv2f64_vv(<vscale x 2 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.minimum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %v
@@ -693,11 +690,10 @@ define <vscale x 4 x double> @vfmin_nxv4f64_vv(<vscale x 4 x double> %a, <vscale
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.minimum.nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b)
   ret <vscale x 4 x double> %v
@@ -755,19 +751,18 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vv_nnana(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfadd.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v9, v10, v10
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v9
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v11, v10, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v11, v0
-; ZVFHMIN-NEXT:    vfmin.vv v9, v9, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
 ; ZVFHMIN-NEXT:    ret
@@ -796,12 +791,11 @@ define <vscale x 1 x half> @vfmin_nxv1f16_vv_nnanb(<vscale x 1 x half> %a, <vsca
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v10
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v9, v0
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v9, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v9, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
index 818a90607ea07..8e448fcda9c5d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
@@ -19,8 +19,8 @@ declare <vscale x 1 x bfloat> @llvm.vp.minimum.nxv1bf16(<vscale x 1 x bfloat>, <
 define <vscale x 1 x bfloat> @vfmin_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv1bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -45,13 +45,14 @@ define <vscale x 1 x bfloat> @vfmin_vv_nxv1bf16_unmasked(<vscale x 1 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v11, v11
-; CHECK-NEXT:    vmerge.vvm v9, v10, v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; CHECK-NEXT:    vfmin.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
@@ -65,8 +66,8 @@ declare <vscale x 2 x bfloat> @llvm.vp.minimum.nxv2bf16(<vscale x 2 x bfloat>, <
 define <vscale x 2 x bfloat> @vfmin_vv_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -91,13 +92,14 @@ define <vscale x 2 x bfloat> @vfmin_vv_nxv2bf16_unmasked(<vscale x 2 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v11, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v11, v11
-; CHECK-NEXT:    vmerge.vvm v9, v10, v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v10, v8, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v8
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; CHECK-NEXT:    vfmin.vv v9, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
@@ -111,8 +113,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.minimum.nxv4bf16(<vscale x 4 x bfloat>, <
 define <vscale x 4 x bfloat> @vfmin_vv_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x bfloat> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -139,14 +141,15 @@ define <vscale x 4 x bfloat> @vfmin_vv_nxv4bf16_unmasked(<vscale x 4 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v10, v10
-; CHECK-NEXT:    vmfeq.vv v8, v12, v12
-; CHECK-NEXT:    vmerge.vvm v14, v10, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v12, v10, v0
-; CHECK-NEXT:    vfmin.vv v10, v8, v14
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v9
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v10, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
+; CHECK-NEXT:    vmerge.vvm v10, v12, v10, v0
+; CHECK-NEXT:    vfmin.vv v10, v10, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
 ; CHECK-NEXT:    ret
@@ -159,8 +162,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.minimum.nxv8bf16(<vscale x 8 x bfloat>, <
 define <vscale x 8 x bfloat> @vfmin_vv_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -187,14 +190,15 @@ define <vscale x 8 x bfloat> @vfmin_vv_nxv8bf16_unmasked(<vscale x 8 x bfloat> %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v12, v12
-; CHECK-NEXT:    vmfeq.vv v8, v16, v16
-; CHECK-NEXT:    vmerge.vvm v20, v12, v16, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v16, v12, v0
-; CHECK-NEXT:    vfmin.vv v12, v8, v20
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v10
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmerge.vvm v8, v12, v16, v0
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmerge.vvm v12, v16, v12, v0
+; CHECK-NEXT:    vfmin.vv v12, v12, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
 ; CHECK-NEXT:    ret
@@ -213,8 +217,8 @@ define <vscale x 16 x bfloat> @vfmin_vv_nxv16bf16(<vscale x 16 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v24, v24, v0.t
@@ -565,8 +569,8 @@ declare <vscale x 1 x half> @llvm.vp.minimum.nxv1f16(<vscale x 1 x half>, <vscal
 define <vscale x 1 x half> @vfmin_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -578,8 +582,8 @@ define <vscale x 1 x half> @vfmin_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv1f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -604,24 +608,24 @@ define <vscale x 1 x half> @vfmin_vv_nxv1f16_unmasked(<vscale x 1 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv1f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -635,8 +639,8 @@ declare <vscale x 2 x half> @llvm.vp.minimum.nxv2f16(<vscale x 2 x half>, <vscal
 define <vscale x 2 x half> @vfmin_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_nxv2f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -648,8 +652,8 @@ define <vscale x 2 x half> @vfmin_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv2f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v11, v11, v0.t
@@ -674,24 +678,24 @@ define <vscale x 2 x half> @vfmin_vv_nxv2f16_unmasked(<vscale x 2 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv1r.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv2f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v11, v11
-; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v11, v0
-; ZVFHMIN-NEXT:    vmv.v.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v11, v10, v0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v9, v10, v8, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v9
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
@@ -705,8 +709,8 @@ declare <vscale x 4 x half> @llvm.vp.minimum.nxv4f16(<vscale x 4 x half>, <vscal
 define <vscale x 4 x half> @vfmin_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x half> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_nxv4f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; ZVFH-NEXT:    vmv1r.v v0, v10
@@ -718,8 +722,8 @@ define <vscale x 4 x half> @vfmin_vv_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12, v0.t
@@ -746,25 +750,25 @@ define <vscale x 4 x half> @vfmin_vv_nxv4f16_unmasked(<vscale x 4 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v10, v9, v9
-; ZVFH-NEXT:    vmerge.vvm v11, v8, v9, v0
-; ZVFH-NEXT:    vmv.v.v v0, v10
+; ZVFH-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v9, v9
 ; ZVFH-NEXT:    vmerge.vvm v8, v9, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v11
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv4f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v10, v10
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v12, v12
-; ZVFHMIN-NEXT:    vmerge.vvm v14, v10, v12, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v10, v0
-; ZVFHMIN-NEXT:    vfmin.vv v10, v8, v14
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v9
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v10, v12, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v12, v10, v0
+; ZVFHMIN-NEXT:    vfmin.vv v10, v10, v8
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
 ; ZVFHMIN-NEXT:    ret
@@ -777,8 +781,8 @@ declare <vscale x 8 x half> @llvm.vp.minimum.nxv8f16(<vscale x 8 x half>, <vscal
 define <vscale x 8 x half> @vfmin_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
 ; ZVFH-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v13
 ; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -792,8 +796,8 @@ define <vscale x 8 x half> @vfmin_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16, v0.t
@@ -820,25 +824,25 @@ define <vscale x 8 x half> @vfmin_vv_nxv8f16_unmasked(<vscale x 8 x half> %va, <
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v12, v10, v10
-; ZVFH-NEXT:    vmerge.vvm v14, v8, v10, v0
-; ZVFH-NEXT:    vmv1r.v v0, v12
+; ZVFH-NEXT:    vmerge.vvm v12, v8, v10, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
 ; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v14
+; ZVFH-NEXT:    vfmin.vv v8, v8, v12
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv8f16_unmasked:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v0, v12, v12
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v16
-; ZVFHMIN-NEXT:    vmerge.vvm v20, v12, v16, v0
-; ZVFHMIN-NEXT:    vmv1r.v v0, v8
-; ZVFHMIN-NEXT:    vmerge.vvm v8, v16, v12, v0
-; ZVFHMIN-NEXT:    vfmin.vv v12, v8, v20
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v10
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v12, v16, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v16, v16
+; ZVFHMIN-NEXT:    vmerge.vvm v12, v16, v12, v0
+; ZVFHMIN-NEXT:    vfmin.vv v12, v12, v8
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
 ; ZVFHMIN-NEXT:    ret
@@ -851,8 +855,8 @@ declare <vscale x 16 x half> @llvm.vp.minimum.nxv16f16(<vscale x 16 x half>, <vs
 define <vscale x 16 x half> @vfmin_vv_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x half> %vb, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmin_vv_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
 ; ZVFH-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v17
 ; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -872,8 +876,8 @@ define <vscale x 16 x half> @vfmin_vv_nxv16f16(<vscale x 16 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v24, v24, v0.t
@@ -909,11 +913,10 @@ define <vscale x 16 x half> @vfmin_vv_nxv16f16_unmasked(<vscale x 16 x half> %va
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v0, v8, v8
-; ZVFH-NEXT:    vmfeq.vv v16, v12, v12
-; ZVFH-NEXT:    vmerge.vvm v20, v8, v12, v0
-; ZVFH-NEXT:    vmv1r.v v0, v16
+; ZVFH-NEXT:    vmerge.vvm v16, v8, v12, v0
+; ZVFH-NEXT:    vmfeq.vv v0, v12, v12
 ; ZVFH-NEXT:    vmerge.vvm v8, v12, v8, v0
-; ZVFH-NEXT:    vfmin.vv v8, v8, v20
+; ZVFH-NEXT:    vfmin.vv v8, v8, v16
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vfmin_vv_nxv16f16_unmasked:
@@ -961,8 +964,8 @@ define <vscale x 32 x half> @vfmin_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFH-NEXT:    slli a1, a1, 3
 ; ZVFH-NEXT:    sub sp, sp, a1
 ; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; ZVFH-NEXT:    vmv1r.v v7, v0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v7, v0
 ; ZVFH-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; ZVFH-NEXT:    vmv1r.v v0, v25
 ; ZVFH-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -1277,8 +1280,8 @@ declare <vscale x 1 x float> @llvm.vp.minimum.nxv1f32(<vscale x 1 x float>, <vsc
 define <vscale x 1 x float> @vfmin_vv_nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x float> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv1f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1296,11 +1299,10 @@ define <vscale x 1 x float> @vfmin_vv_nxv1f32_unmasked(<vscale x 1 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.vp.minimum.nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x float> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x float> %v
@@ -1311,8 +1313,8 @@ declare <vscale x 2 x float> @llvm.vp.minimum.nxv2f32(<vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @vfmin_vv_nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1330,11 +1332,10 @@ define <vscale x 2 x float> @vfmin_vv_nxv2f32_unmasked(<vscale x 2 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.vp.minimum.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x float> %v
@@ -1345,8 +1346,8 @@ declare <vscale x 4 x float> @llvm.vp.minimum.nxv4f32(<vscale x 4 x float>, <vsc
 define <vscale x 4 x float> @vfmin_vv_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x float> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -1366,11 +1367,10 @@ define <vscale x 4 x float> @vfmin_vv_nxv4f32_unmasked(<vscale x 4 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.vp.minimum.nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x float> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x float> %v
@@ -1381,8 +1381,8 @@ declare <vscale x 8 x float> @llvm.vp.minimum.nxv8f32(<vscale x 8 x float>, <vsc
 define <vscale x 8 x float> @vfmin_vv_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x float> %vb, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -1402,11 +1402,10 @@ define <vscale x 8 x float> @vfmin_vv_nxv8f32_unmasked(<vscale x 8 x float> %va,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.vp.minimum.nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x float> %vb, <vscale x 8 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 8 x float> %v
@@ -1417,8 +1416,8 @@ declare <vscale x 1 x double> @llvm.vp.minimum.nxv1f64(<vscale x 1 x double>, <v
 define <vscale x 1 x double> @vfmin_vv_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8, v0.t
 ; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
@@ -1436,11 +1435,10 @@ define <vscale x 1 x double> @vfmin_vv_nxv1f64_unmasked(<vscale x 1 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v10, v9, v9
-; CHECK-NEXT:    vmerge.vvm v11, v8, v9, v0
-; CHECK-NEXT:    vmv.v.v v0, v10
+; CHECK-NEXT:    vmerge.vvm v10, v8, v9, v0
+; CHECK-NEXT:    vmfeq.vv v0, v9, v9
 ; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v11
+; CHECK-NEXT:    vfmin.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.vp.minimum.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x double> %v
@@ -1451,8 +1449,8 @@ declare <vscale x 2 x double> @llvm.vp.minimum.nxv2f64(<vscale x 2 x double>, <v
 define <vscale x 2 x double> @vfmin_vv_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x double> %vb, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vmfeq.vv v13, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
@@ -1472,11 +1470,10 @@ define <vscale x 2 x double> @vfmin_vv_nxv2f64_unmasked(<vscale x 2 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v12, v10, v10
-; CHECK-NEXT:    vmerge.vvm v14, v8, v10, v0
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmerge.vvm v12, v8, v10, v0
+; CHECK-NEXT:    vmfeq.vv v0, v10, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v14
+; CHECK-NEXT:    vfmin.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.vp.minimum.nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x double> %vb, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x double> %v
@@ -1487,8 +1484,8 @@ declare <vscale x 4 x double> @llvm.vp.minimum.nxv4f64(<vscale x 4 x double>, <v
 define <vscale x 4 x double> @vfmin_vv_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x double> %vb, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfmin_vv_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vmfeq.vv v17, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
@@ -1508,11 +1505,10 @@ define <vscale x 4 x double> @vfmin_vv_nxv4f64_unmasked(<vscale x 4 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v0, v8, v8
-; CHECK-NEXT:    vmfeq.vv v16, v12, v12
-; CHECK-NEXT:    vmerge.vvm v20, v8, v12, v0
-; CHECK-NEXT:    vmv1r.v v0, v16
+; CHECK-NEXT:    vmerge.vvm v16, v8, v12, v0
+; CHECK-NEXT:    vmfeq.vv v0, v12, v12
 ; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
-; CHECK-NEXT:    vfmin.vv v8, v8, v20
+; CHECK-NEXT:    vfmin.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.vp.minimum.nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x double> %vb, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x double> %v
@@ -1529,8 +1525,8 @@ define <vscale x 8 x double> @vfmin_vv_nxv8f64(<vscale x 8 x double> %va, <vscal
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vmerge.vvm v24, v8, v16, v0
@@ -1581,6 +1577,7 @@ define <vscale x 16 x double> @vfmin_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -1592,7 +1589,6 @@ define <vscale x 16 x double> @vfmin_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    srli a4, a1, 3
-; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a4
 ; CHECK-NEXT:    sub a4, a2, a1
 ; CHECK-NEXT:    add a3, a0, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
index 025874a1a74e2..5e657a93ec0d6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
@@ -18,8 +18,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV32-NEXT:  .LBB0_1: # %for.body
 ; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32-NEXT:    th.lrb a0, a1, a0, 0
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, tu, ma
 ; RV32-NEXT:    vmv1r.v v9, v8
-; RV32-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
 ; RV32-NEXT:    vmv.s.x v9, a0
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; RV32-NEXT:    vmseq.vi v9, v9, 0
@@ -45,8 +45,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    th.lrb a0, a1, a0, 0
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, tu, ma
 ; RV64-NEXT:    vmv1r.v v9, v8
-; RV64-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
 ; RV64-NEXT:    vmv.s.x v9, a0
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; RV64-NEXT:    vmseq.vi v9, v9, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
index c7e3c8cb51982..b569efc7447da 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
@@ -703,10 +703,10 @@ define <vscale x 16 x i32> @fshl_v16i32(<vscale x 16 x i32> %a, <vscale x 16 x i
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
 ; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vand.vx v8, v24, a0, v0.t
 ; CHECK-NEXT:    vsll.vv v8, v16, v8, v0.t
 ; CHECK-NEXT:    vnot.v v16, v24, v0.t
@@ -882,10 +882,10 @@ define <vscale x 7 x i64> @fshl_v7i64(<vscale x 7 x i64> %a, <vscale x 7 x i64>
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
 ; CHECK-NEXT:    li a0, 63
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vand.vx v8, v24, a0, v0.t
 ; CHECK-NEXT:    vsll.vv v8, v16, v8, v0.t
 ; CHECK-NEXT:    vnot.v v16, v24, v0.t
@@ -953,10 +953,10 @@ define <vscale x 8 x i64> @fshl_v8i64(<vscale x 8 x i64> %a, <vscale x 8 x i64>
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
 ; CHECK-NEXT:    li a0, 63
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vand.vx v8, v24, a0, v0.t
 ; CHECK-NEXT:    vsll.vv v8, v16, v8, v0.t
 ; CHECK-NEXT:    vnot.v v16, v24, v0.t
@@ -988,6 +988,7 @@ define <vscale x 16 x i64> @fshr_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    li a3, 24
@@ -1022,7 +1023,6 @@ define <vscale x 16 x i64> @fshr_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    add a3, sp, a3
 ; CHECK-NEXT:    addi a3, a3, 16
 ; CHECK-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a6
 ; CHECK-NEXT:    li a3, 63
 ; CHECK-NEXT:    csrr a6, vlenb
@@ -1173,6 +1173,7 @@ define <vscale x 16 x i64> @fshl_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 5
@@ -1189,7 +1190,6 @@ define <vscale x 16 x i64> @fshl_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    slli a5, a3, 3
 ; CHECK-NEXT:    srli a1, a3, 3
 ; CHECK-NEXT:    sub a6, a4, a3
-; CHECK-NEXT:    vsetvli a7, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a1
 ; CHECK-NEXT:    add a1, a2, a5
 ; CHECK-NEXT:    vl8re64.v v8, (a1)
diff --git a/llvm/test/CodeGen/RISCV/rvv/inline-asm.ll b/llvm/test/CodeGen/RISCV/rvv/inline-asm.ll
index 967a58b45a599..daacc853963fc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/inline-asm.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/inline-asm.ll
@@ -365,11 +365,13 @@ entry:
 define <vscale x 4 x i8> @test_specify_reg_mf2(<vscale x 4 x i8> %in, <vscale x 4 x i8> %in2) nounwind {
 ; CHECK-LABEL: test_specify_reg_mf2:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v2, v9
 ; CHECK-NEXT:    vmv1r.v v1, v8
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    vadd.vv v0, v1, v2
 ; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    ret
 entry:
@@ -380,11 +382,13 @@ entry:
 define <vscale x 8 x i8> @test_specify_reg_m1(<vscale x 8 x i8> %in, <vscale x 8 x i8> %in2) nounwind {
 ; CHECK-LABEL: test_specify_reg_m1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v2, v9
 ; CHECK-NEXT:    vmv1r.v v1, v8
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    vadd.vv v0, v1, v2
 ; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    ret
 entry:
@@ -395,11 +399,13 @@ entry:
 define <vscale x 16 x i8> @test_specify_reg_m2(<vscale x 16 x i8> %in, <vscale x 16 x i8> %in2) nounwind {
 ; CHECK-LABEL: test_specify_reg_m2:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v4, v10
 ; CHECK-NEXT:    vmv2r.v v2, v8
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    vadd.vv v0, v2, v4
 ; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v0
 ; CHECK-NEXT:    ret
 entry:
@@ -410,6 +416,7 @@ entry:
 define <vscale x 1 x i1> @test_specify_reg_mask(<vscale x 1 x i1> %in, <vscale x 1 x i1> %in2) nounwind {
 ; CHECK-LABEL: test_specify_reg_mask:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v2, v8
 ; CHECK-NEXT:    vmv1r.v v1, v0
 ; CHECK-NEXT:    #APP
diff --git a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
index 8925a9e0cee32..ca9cec921b3cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
@@ -5,6 +5,7 @@
 define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv4i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv4i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec, i64 0)
@@ -14,6 +15,7 @@ define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec, <vs
 define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_4(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv4i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv4i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec, i64 4)
@@ -23,6 +25,7 @@ define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_4(<vscale x 8 x i32> %vec, <vs
 define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_0(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv2i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec, i64 0)
@@ -32,6 +35,7 @@ define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_0(<vscale x 8 x i32> %vec, <vs
 define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_2(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv2i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec, i64 2)
@@ -41,6 +45,7 @@ define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_2(<vscale x 8 x i32> %vec, <vs
 define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_4(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv2i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec, i64 4)
@@ -50,6 +55,7 @@ define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_4(<vscale x 8 x i32> %vec, <vs
 define <vscale x 8 x i32> @insert_nxv8i32_nxv2i32_6(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv2i32_6:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv2i32.nxv8i32(<vscale x 8 x i32> %vec, <vscale x 2 x i32> %subvec, i64 6)
@@ -86,6 +92,7 @@ define <vscale x 4 x i8> @insert_nxv1i8_nxv4i8_3(<vscale x 4 x i8> %vec, <vscale
 define <vscale x 16 x i32> @insert_nxv16i32_nxv8i32_0(<vscale x 16 x i32> %vec, <vscale x 8 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv8i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv8i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 8 x i32> %subvec, i64 0)
@@ -95,6 +102,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv8i32_0(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv8i32_8(<vscale x 16 x i32> %vec, <vscale x 8 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv8i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v12, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv8i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 8 x i32> %subvec, i64 8)
@@ -104,6 +112,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv8i32_8(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_0(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv4i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec, i64 0)
@@ -113,6 +122,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_0(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_4(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv4i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v10, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec, i64 4)
@@ -122,6 +132,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_4(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_8(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv4i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v12, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec, i64 8)
@@ -131,6 +142,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_8(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_12(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv4i32_12:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v14, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv4i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 4 x i32> %subvec, i64 12)
@@ -140,6 +152,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv4i32_12(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_0(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 0)
@@ -149,6 +162,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_0(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_2(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 2)
@@ -158,6 +172,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_2(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_4(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_4:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 4)
@@ -167,6 +182,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_4(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_6(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_6:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 6)
@@ -176,6 +192,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_6(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_8(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v12, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 8)
@@ -185,6 +202,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_8(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_10(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_10:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v13, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 10)
@@ -194,6 +212,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_10(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_12(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_12:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v14, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 12)
@@ -203,6 +222,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_12(<vscale x 16 x i32> %vec,
 define <vscale x 16 x i32> @insert_nxv16i32_nxv2i32_14(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv16i32_nxv2i32_14:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v15, v16
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv2i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 2 x i32> %subvec, i64 14)
@@ -512,6 +532,7 @@ define <vscale x 2 x i64> @insert_nxv2i64_nxv3i64(<3 x i64> %sv) #0 {
 define <vscale x 8 x i32> @insert_insert_combine(<2 x i32> %subvec) {
 ; CHECK-LABEL: insert_insert_combine:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    ret
   %inner = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v2i32(<vscale x 4 x i32> undef, <2 x i32> %subvec, i64 0)
@@ -524,6 +545,7 @@ define <vscale x 8 x i32> @insert_insert_combine(<2 x i32> %subvec) {
 define <vscale x 8 x i32> @insert_insert_combine2(<vscale x 2 x i32> %subvec) {
 ; CHECK-LABEL: insert_insert_combine2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    ret
   %inner = call <vscale x 4 x i32> @llvm.vector.insert.nxv2i32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 2 x i32> %subvec, i64 0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index e70dcd16d02cd..5d730da09ef83 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -6,8 +6,8 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
 ; CHECK-LABEL: match_nxv16i8_v1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v10, v8, v12
 ; CHECK-NEXT:    vmand.mm v0, v10, v0
 ; CHECK-NEXT:    ret
   %r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <1 x i8> %op2, <vscale x 16 x i1> %mask)
@@ -17,14 +17,12 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
 define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8> %op2, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: match_nxv16i8_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v10, v10, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmor.mm v8, v11, v10
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v10, v8, v12
+; CHECK-NEXT:    vmor.mm v8, v10, v14
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
   %r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <2 x i8> %op2, <vscale x 16 x i1> %mask)
@@ -34,21 +32,17 @@ define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8>
 define <vscale x 16 x i1> @match_nxv16i8_v4i8(<vscale x 16 x i8> %op1, <4 x i8> %op2, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: match_nxv16i8_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v11, v10, 1
-; CHECK-NEXT:    vslidedown.vi v12, v10, 2
-; CHECK-NEXT:    vslidedown.vi v10, v10, 3
-; CHECK-NEXT:    vmv.x.s a1, v11
-; CHECK-NEXT:    vsetvli a2, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmor.mm v11, v11, v12
-; CHECK-NEXT:    vmor.mm v10, v11, v10
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v15, v8, v12
+; CHECK-NEXT:    vmor.mm v12, v15, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 2
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 3
+; CHECK-NEXT:    vmor.mm v10, v12, v13
+; CHECK-NEXT:    vmseq.vv v11, v8, v14
 ; CHECK-NEXT:    vmor.mm v8, v10, v11
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -59,37 +53,29 @@ define <vscale x 16 x i1> @match_nxv16i8_v4i8(<vscale x 16 x i8> %op1, <4 x i8>
 define <vscale x 16 x i1> @match_nxv16i8_v8i8(<vscale x 16 x i8> %op1, <8 x i8> %op2, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: match_nxv16i8_v8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v11, v10, 1
-; CHECK-NEXT:    vslidedown.vi v12, v10, 2
-; CHECK-NEXT:    vmv.x.s a1, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 3
-; CHECK-NEXT:    vmv.x.s a2, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 4
-; CHECK-NEXT:    vmv.x.s a3, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 5
-; CHECK-NEXT:    vmv.x.s a4, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 6
-; CHECK-NEXT:    vslidedown.vi v10, v10, 7
-; CHECK-NEXT:    vmv.x.s a5, v11
-; CHECK-NEXT:    vsetvli a6, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a2
-; CHECK-NEXT:    vmor.mm v11, v11, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a3
-; CHECK-NEXT:    vmor.mm v10, v11, v10
-; CHECK-NEXT:    vmseq.vx v11, v8, a4
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a5
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v15, v8, v12
+; CHECK-NEXT:    vmor.mm v12, v15, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 2
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 3
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 4
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 5
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 6
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 7
+; CHECK-NEXT:    vmor.mm v10, v12, v13
+; CHECK-NEXT:    vmseq.vv v11, v8, v14
 ; CHECK-NEXT:    vmor.mm v8, v10, v11
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -100,69 +86,53 @@ define <vscale x 16 x i1> @match_nxv16i8_v8i8(<vscale x 16 x i8> %op1, <8 x i8>
 define <vscale x 16 x i1> @match_nxv16i8_v16i8(<vscale x 16 x i8> %op1, <16 x i8> %op2, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: match_nxv16i8_v16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v11, v10, 1
-; CHECK-NEXT:    vslidedown.vi v12, v10, 2
-; CHECK-NEXT:    vmv.x.s a1, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 3
-; CHECK-NEXT:    vmv.x.s a2, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 4
-; CHECK-NEXT:    vmv.x.s a3, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 5
-; CHECK-NEXT:    vmv.x.s a4, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 6
-; CHECK-NEXT:    vmv.x.s a5, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 7
-; CHECK-NEXT:    vmv.x.s a6, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 8
-; CHECK-NEXT:    vmv.x.s a7, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 9
-; CHECK-NEXT:    vmv.x.s t0, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 10
-; CHECK-NEXT:    vmv.x.s t1, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 11
-; CHECK-NEXT:    vmv.x.s t2, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 12
-; CHECK-NEXT:    vmv.x.s t3, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 13
-; CHECK-NEXT:    vmv.x.s t4, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 14
-; CHECK-NEXT:    vslidedown.vi v10, v10, 15
-; CHECK-NEXT:    vmv.x.s t5, v11
-; CHECK-NEXT:    vsetvli t6, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a2
-; CHECK-NEXT:    vmor.mm v11, v11, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a3
-; CHECK-NEXT:    vmor.mm v10, v11, v10
-; CHECK-NEXT:    vmseq.vx v11, v8, a4
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a5
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a6
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a7
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, t0
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, t1
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, t2
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, t3
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, t4
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, t5
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v15, v8, v12
+; CHECK-NEXT:    vmor.mm v12, v15, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 2
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 3
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 4
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 5
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 6
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 7
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 8
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 9
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 10
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 11
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 12
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 13
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 14
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 15
+; CHECK-NEXT:    vmor.mm v10, v12, v13
+; CHECK-NEXT:    vmseq.vv v11, v8, v14
 ; CHECK-NEXT:    vmor.mm v8, v10, v11
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -173,9 +143,10 @@ define <vscale x 16 x i1> @match_nxv16i8_v16i8(<vscale x 16 x i8> %op1, <16 x i8
 define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask) {
 ; CHECK-LABEL: match_v16i8_v1i8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vrgather.vi v10, v9, 0
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v9
-; CHECK-NEXT:    vmseq.vx v8, v8, a0
+; CHECK-NEXT:    vmseq.vv v8, v8, v10
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
   %r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
@@ -185,14 +156,12 @@ define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mas
 define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask) {
 ; CHECK-LABEL: match_v16i8_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v9
-; CHECK-NEXT:    vslidedown.vi v9, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v9
-; CHECK-NEXT:    vmseq.vx v8, v8, a0
-; CHECK-NEXT:    vmor.mm v8, v10, v8
+; CHECK-NEXT:    vrgather.vi v10, v9, 1
+; CHECK-NEXT:    vrgather.vi v11, v9, 0
+; CHECK-NEXT:    vmseq.vv v9, v8, v10
+; CHECK-NEXT:    vmseq.vv v8, v8, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v9
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
   %r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask)
@@ -202,21 +171,17 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
 define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask) {
 ; CHECK-LABEL: match_v16i8_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v9
-; CHECK-NEXT:    vslidedown.vi v10, v9, 1
-; CHECK-NEXT:    vslidedown.vi v11, v9, 2
-; CHECK-NEXT:    vslidedown.vi v9, v9, 3
-; CHECK-NEXT:    vmv.x.s a1, v10
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v9
-; CHECK-NEXT:    vmseq.vx v9, v8, a0
-; CHECK-NEXT:    vmor.mm v10, v10, v11
+; CHECK-NEXT:    vrgather.vi v10, v9, 1
+; CHECK-NEXT:    vrgather.vi v11, v9, 0
+; CHECK-NEXT:    vmseq.vv v10, v8, v10
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
+; CHECK-NEXT:    vmor.mm v10, v11, v10
+; CHECK-NEXT:    vrgather.vi v11, v9, 2
+; CHECK-NEXT:    vrgather.vi v12, v9, 3
+; CHECK-NEXT:    vmseq.vv v9, v8, v11
 ; CHECK-NEXT:    vmor.mm v9, v10, v9
-; CHECK-NEXT:    vmseq.vx v8, v8, a1
+; CHECK-NEXT:    vmseq.vv v8, v8, v12
 ; CHECK-NEXT:    vmor.mm v8, v9, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -227,37 +192,29 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
 define <16 x i1> @match_v16i8_v8i8(<16 x i8> %op1, <8 x i8> %op2, <16 x i1> %mask) {
 ; CHECK-LABEL: match_v16i8_v8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v9
-; CHECK-NEXT:    vslidedown.vi v10, v9, 1
-; CHECK-NEXT:    vslidedown.vi v11, v9, 2
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vslidedown.vi v10, v9, 3
-; CHECK-NEXT:    vmv.x.s a2, v11
-; CHECK-NEXT:    vslidedown.vi v11, v9, 4
-; CHECK-NEXT:    vmv.x.s a3, v10
-; CHECK-NEXT:    vslidedown.vi v10, v9, 5
-; CHECK-NEXT:    vmv.x.s a4, v11
-; CHECK-NEXT:    vslidedown.vi v11, v9, 6
-; CHECK-NEXT:    vslidedown.vi v9, v9, 7
-; CHECK-NEXT:    vmv.x.s a5, v10
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v9
-; CHECK-NEXT:    vmseq.vx v9, v8, a2
+; CHECK-NEXT:    vrgather.vi v10, v9, 1
+; CHECK-NEXT:    vrgather.vi v11, v9, 0
+; CHECK-NEXT:    vmseq.vv v10, v8, v10
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
+; CHECK-NEXT:    vmor.mm v10, v11, v10
+; CHECK-NEXT:    vrgather.vi v11, v9, 2
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
 ; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a3
+; CHECK-NEXT:    vrgather.vi v11, v9, 3
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
+; CHECK-NEXT:    vmor.mm v10, v10, v11
+; CHECK-NEXT:    vrgather.vi v11, v9, 4
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
+; CHECK-NEXT:    vmor.mm v10, v10, v11
+; CHECK-NEXT:    vrgather.vi v11, v9, 5
+; CHECK-NEXT:    vmseq.vv v11, v8, v11
+; CHECK-NEXT:    vmor.mm v10, v10, v11
+; CHECK-NEXT:    vrgather.vi v11, v9, 6
+; CHECK-NEXT:    vrgather.vi v12, v9, 7
+; CHECK-NEXT:    vmseq.vv v9, v8, v11
 ; CHECK-NEXT:    vmor.mm v9, v10, v9
-; CHECK-NEXT:    vmseq.vx v10, v8, a4
-; CHECK-NEXT:    vmor.mm v9, v9, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a5
-; CHECK-NEXT:    vmor.mm v9, v9, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmor.mm v9, v9, v11
-; CHECK-NEXT:    vmor.mm v9, v9, v10
-; CHECK-NEXT:    vmseq.vx v8, v8, a1
+; CHECK-NEXT:    vmseq.vv v8, v8, v12
 ; CHECK-NEXT:    vmor.mm v8, v9, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -358,37 +315,29 @@ define <8 x i1> @match_v8i8_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
 define <vscale x 8 x i1> @match_nxv8i16_v8i16(<vscale x 8 x i16> %op1, <8 x i16> %op2, <vscale x 8 x i1> %mask) {
 ; CHECK-LABEL: match_nxv8i16_v8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v11, v10, 1
-; CHECK-NEXT:    vslidedown.vi v12, v10, 2
-; CHECK-NEXT:    vmv.x.s a1, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 3
-; CHECK-NEXT:    vmv.x.s a2, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 4
-; CHECK-NEXT:    vmv.x.s a3, v11
-; CHECK-NEXT:    vslidedown.vi v11, v10, 5
-; CHECK-NEXT:    vmv.x.s a4, v12
-; CHECK-NEXT:    vslidedown.vi v12, v10, 6
-; CHECK-NEXT:    vslidedown.vi v10, v10, 7
-; CHECK-NEXT:    vmv.x.s a5, v11
-; CHECK-NEXT:    vsetvli a6, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a2
-; CHECK-NEXT:    vmor.mm v11, v11, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a3
-; CHECK-NEXT:    vmor.mm v10, v11, v10
-; CHECK-NEXT:    vmseq.vx v11, v8, a4
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a5
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmor.mm v10, v10, v12
-; CHECK-NEXT:    vmor.mm v10, v10, v11
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v15, v8, v12
+; CHECK-NEXT:    vmor.mm v12, v15, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 2
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 3
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 4
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 5
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vmor.mm v12, v12, v13
+; CHECK-NEXT:    vrgather.vi v14, v10, 6
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 7
+; CHECK-NEXT:    vmor.mm v10, v12, v13
+; CHECK-NEXT:    vmseq.vv v11, v8, v14
 ; CHECK-NEXT:    vmor.mm v8, v10, v11
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -1281,21 +1230,17 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
 define <vscale x 4 x i1> @match_nxv4xi32_v4i32(<vscale x 4 x i32> %op1, <4 x i32> %op2, <vscale x 4 x i1> %mask) {
 ; CHECK-LABEL: match_nxv4xi32_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v10
-; CHECK-NEXT:    vslidedown.vi v11, v10, 1
-; CHECK-NEXT:    vslidedown.vi v12, v10, 2
-; CHECK-NEXT:    vslidedown.vi v10, v10, 3
-; CHECK-NEXT:    vmv.x.s a1, v11
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmseq.vx v11, v8, a0
-; CHECK-NEXT:    vmv.x.s a0, v12
-; CHECK-NEXT:    vmseq.vx v12, v8, a1
-; CHECK-NEXT:    vmv.x.s a1, v10
-; CHECK-NEXT:    vmseq.vx v10, v8, a0
-; CHECK-NEXT:    vmor.mm v11, v11, v12
-; CHECK-NEXT:    vmor.mm v10, v11, v10
-; CHECK-NEXT:    vmseq.vx v11, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v15, v8, v12
+; CHECK-NEXT:    vmor.mm v12, v15, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 2
+; CHECK-NEXT:    vmseq.vv v13, v8, v14
+; CHECK-NEXT:    vrgather.vi v14, v10, 3
+; CHECK-NEXT:    vmor.mm v10, v12, v13
+; CHECK-NEXT:    vmseq.vv v11, v8, v14
 ; CHECK-NEXT:    vmor.mm v8, v10, v11
 ; CHECK-NEXT:    vmand.mm v0, v8, v0
 ; CHECK-NEXT:    ret
@@ -1304,48 +1249,16 @@ define <vscale x 4 x i1> @match_nxv4xi32_v4i32(<vscale x 4 x i32> %op1, <4 x i32
 }
 
 define <vscale x 2 x i1> @match_nxv2xi64_v2i64(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask) {
-; RV32-LABEL: match_nxv2xi64_v2i64:
-; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vmv.x.s a0, v10
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vslidedown.vi v11, v10, 1
-; RV32-NEXT:    addi a2, sp, 8
-; RV32-NEXT:    vsrl.vx v10, v10, a1
-; RV32-NEXT:    vmv.x.s a3, v11
-; RV32-NEXT:    vsrl.vx v11, v11, a1
-; RV32-NEXT:    vmv.x.s a1, v10
-; RV32-NEXT:    sw a0, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    vmv.x.s a0, v11
-; RV32-NEXT:    sw a3, 0(sp)
-; RV32-NEXT:    sw a0, 4(sp)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32-NEXT:    vlse64.v v10, (a2), zero
-; RV32-NEXT:    vlse64.v v12, (a0), zero
-; RV32-NEXT:    vmseq.vv v14, v8, v10
-; RV32-NEXT:    vmseq.vv v10, v8, v12
-; RV32-NEXT:    vmor.mm v8, v14, v10
-; RV32-NEXT:    vmand.mm v0, v8, v0
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: match_nxv2xi64_v2i64:
-; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vmv.x.s a0, v10
-; RV64-NEXT:    vslidedown.vi v10, v10, 1
-; RV64-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV64-NEXT:    vmseq.vx v11, v8, a0
-; RV64-NEXT:    vmv.x.s a0, v10
-; RV64-NEXT:    vmseq.vx v10, v8, a0
-; RV64-NEXT:    vmor.mm v8, v11, v10
-; RV64-NEXT:    vmand.mm v0, v8, v0
-; RV64-NEXT:    ret
+; CHECK-LABEL: match_nxv2xi64_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrgather.vi v12, v10, 1
+; CHECK-NEXT:    vmseq.vv v14, v8, v12
+; CHECK-NEXT:    vrgather.vi v12, v10, 0
+; CHECK-NEXT:    vmseq.vv v10, v8, v12
+; CHECK-NEXT:    vmor.mm v8, v10, v14
+; CHECK-NEXT:    vmand.mm v0, v8, v0
+; CHECK-NEXT:    ret
   %r = tail call <vscale x 2 x i1> @llvm.experimental.vector.match(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask)
   ret <vscale x 2 x i1> %r
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/llrint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/llrint-vp.ll
index ffb9bf76fb4fa..166dba6a56524 100644
--- a/llvm/test/CodeGen/RISCV/rvv/llrint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/llrint-vp.ll
@@ -55,11 +55,11 @@ declare <vscale x 8 x i64> @llvm.vp.llrint.nxv8i64.nxv8f32(<vscale x 8 x float>,
 define <vscale x 16 x i64> @llrint_nxv16i64_nxv16f32(<vscale x 16 x float> %x, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: llrint_nxv16i64_nxv16f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/lrint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/lrint-vp.ll
index 9991bbc9725ba..21045b69a8b5d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/lrint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/lrint-vp.ll
@@ -117,11 +117,11 @@ define <vscale x 16 x iXLen> @lrint_nxv16f32(<vscale x 16 x float> %x, <vscale x
 ;
 ; RV64-i64-LABEL: lrint_nxv16f32:
 ; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64-i64-NEXT:    vmv1r.v v24, v0
 ; RV64-i64-NEXT:    csrr a1, vlenb
 ; RV64-i64-NEXT:    srli a2, a1, 3
 ; RV64-i64-NEXT:    sub a3, a0, a1
-; RV64-i64-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vx v0, v0, a2
 ; RV64-i64-NEXT:    sltu a2, a0, a3
 ; RV64-i64-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
index 420597b009f33..cd678c9740721 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
@@ -1288,9 +1288,9 @@ declare <vscale x 1 x i8> @llvm.riscv.viota.mask.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_viota_mask_m_nxv1i8_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv1i8_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    viota.m v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1312,9 +1312,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsbf.mask.nxv1i1(
 define <vscale x 1 x i1> @intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmsbf.m v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
@@ -1443,9 +1443,9 @@ declare <vscale x 64 x i1> @llvm.riscv.vmsbf.mask.nxv64i1(
 define <vscale x 64 x i1> @intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vmsbf.m v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index a3eddbcc2baed..9ee2324f615dd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -221,11 +221,13 @@ define <vscale x 4 x i8> @mgather_truemask_nxv4i8(<vscale x 4 x ptr> %ptrs, <vsc
 define <vscale x 4 x i8> @mgather_falsemask_nxv4i8(<vscale x 4 x ptr> %ptrs, <vscale x 4 x i8> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4i8:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4i8:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 1, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i8> %passthru)
@@ -442,11 +444,13 @@ define <vscale x 4 x i16> @mgather_truemask_nxv4i16(<vscale x 4 x ptr> %ptrs, <v
 define <vscale x 4 x i16> @mgather_falsemask_nxv4i16(<vscale x 4 x ptr> %ptrs, <vscale x 4 x i16> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4i16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4i16:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i16> %passthru)
@@ -686,11 +690,13 @@ define <vscale x 4 x i32> @mgather_truemask_nxv4i32(<vscale x 4 x ptr> %ptrs, <v
 define <vscale x 4 x i32> @mgather_falsemask_nxv4i32(<vscale x 4 x ptr> %ptrs, <vscale x 4 x i32> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4i32:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv2r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4i32:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv2r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %passthru)
@@ -949,6 +955,7 @@ define <vscale x 4 x i64> @mgather_truemask_nxv4i64(<vscale x 4 x ptr> %ptrs, <v
 define <vscale x 4 x i64> @mgather_falsemask_nxv4i64(<vscale x 4 x ptr> %ptrs, <vscale x 4 x i64> %passthru) {
 ; CHECK-LABEL: mgather_falsemask_nxv4i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i64> %passthru)
@@ -1232,12 +1239,12 @@ define void @mgather_nxv16i64(<vscale x 8 x ptr> %ptrs0, <vscale x 8 x ptr> %ptr
 ; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; RV64-NEXT:    addi a3, sp, 16
 ; RV64-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmv8r.v v16, v8
 ; RV64-NEXT:    vl8re64.v v24, (a0)
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    vl8re64.v v8, (a1)
 ; RV64-NEXT:    srli a1, a0, 3
-; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vx v7, v0, a1
 ; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
 ; RV64-NEXT:    vluxei64.v v24, (zero), v16, v0.t
@@ -1348,11 +1355,13 @@ define <vscale x 4 x bfloat> @mgather_truemask_nxv4bf16(<vscale x 4 x ptr> %ptrs
 define <vscale x 4 x bfloat> @mgather_falsemask_nxv4bf16(<vscale x 4 x ptr> %ptrs, <vscale x 4 x bfloat> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4bf16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4bf16:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x bfloat> %passthru)
@@ -1549,11 +1558,13 @@ define <vscale x 4 x half> @mgather_truemask_nxv4f16(<vscale x 4 x ptr> %ptrs, <
 define <vscale x 4 x half> @mgather_falsemask_nxv4f16(<vscale x 4 x ptr> %ptrs, <vscale x 4 x half> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4f16:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4f16:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x half> %passthru)
@@ -1749,11 +1760,13 @@ define <vscale x 4 x float> @mgather_truemask_nxv4f32(<vscale x 4 x ptr> %ptrs,
 define <vscale x 4 x float> @mgather_falsemask_nxv4f32(<vscale x 4 x ptr> %ptrs, <vscale x 4 x float> %passthru) {
 ; RV32-LABEL: mgather_falsemask_nxv4f32:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV32-NEXT:    vmv2r.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mgather_falsemask_nxv4f32:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv2r.v v8, v12
 ; RV64-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> %passthru)
@@ -2012,6 +2025,7 @@ define <vscale x 4 x double> @mgather_truemask_nxv4f64(<vscale x 4 x ptr> %ptrs,
 define <vscale x 4 x double> @mgather_falsemask_nxv4f64(<vscale x 4 x ptr> %ptrs, <vscale x 4 x double> %passthru) {
 ; CHECK-LABEL: mgather_falsemask_nxv4f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0(<vscale x 4 x ptr> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x double> %passthru)
@@ -2317,8 +2331,8 @@ define <vscale x 32 x i8> @mgather_baseidx_nxv32i8(ptr %base, <vscale x 32 x i8>
 ;
 ; RV64-LABEL: mgather_baseidx_nxv32i8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vmv1r.v v16, v0
 ; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; RV64-NEXT:    vmv1r.v v16, v0
 ; RV64-NEXT:    vsext.vf8 v24, v8
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
index 72c251ce985cb..77a1f508d2218 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
@@ -2009,11 +2009,11 @@ define void @mscatter_baseidx_nxv16i16_nxv16f64(<vscale x 8 x double> %val0, <vs
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; RV32-NEXT:    addi a2, sp, 16
 ; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv8r.v v16, v8
 ; RV32-NEXT:    vl4re16.v v8, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    srli a1, a1, 3
-; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vx v7, v0, a1
 ; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; RV32-NEXT:    vsext.vf2 v24, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
index c49a55319a3c4..9298e8b520bd9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
@@ -1169,10 +1169,10 @@ define <vscale x 32 x i8> @reverse_nxv32i8(<vscale x 32 x i8> %a) {
 define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ; RV32-BITS-UNKNOWN-LABEL: reverse_nxv64i8:
 ; RV32-BITS-UNKNOWN:       # %bb.0:
+; RV32-BITS-UNKNOWN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; RV32-BITS-UNKNOWN-NEXT:    vmv8r.v v16, v8
 ; RV32-BITS-UNKNOWN-NEXT:    csrr a0, vlenb
 ; RV32-BITS-UNKNOWN-NEXT:    addi a0, a0, -1
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; RV32-BITS-UNKNOWN-NEXT:    vid.v v8
 ; RV32-BITS-UNKNOWN-NEXT:    vrsub.vx v24, v8, a0
 ; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
@@ -1188,10 +1188,10 @@ define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ;
 ; RV32-BITS-256-LABEL: reverse_nxv64i8:
 ; RV32-BITS-256:       # %bb.0:
+; RV32-BITS-256-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; RV32-BITS-256-NEXT:    vmv8r.v v16, v8
 ; RV32-BITS-256-NEXT:    csrr a0, vlenb
 ; RV32-BITS-256-NEXT:    addi a0, a0, -1
-; RV32-BITS-256-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; RV32-BITS-256-NEXT:    vid.v v8
 ; RV32-BITS-256-NEXT:    vrsub.vx v24, v8, a0
 ; RV32-BITS-256-NEXT:    vrgather.vv v15, v16, v24
@@ -1206,10 +1206,10 @@ define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ;
 ; RV32-BITS-512-LABEL: reverse_nxv64i8:
 ; RV32-BITS-512:       # %bb.0:
+; RV32-BITS-512-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; RV32-BITS-512-NEXT:    vmv8r.v v16, v8
 ; RV32-BITS-512-NEXT:    csrr a0, vlenb
 ; RV32-BITS-512-NEXT:    addi a0, a0, -1
-; RV32-BITS-512-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; RV32-BITS-512-NEXT:    vid.v v8
 ; RV32-BITS-512-NEXT:    vrsub.vx v24, v8, a0
 ; RV32-BITS-512-NEXT:    vrgather.vv v15, v16, v24
@@ -1224,10 +1224,10 @@ define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ;
 ; RV64-BITS-UNKNOWN-LABEL: reverse_nxv64i8:
 ; RV64-BITS-UNKNOWN:       # %bb.0:
+; RV64-BITS-UNKNOWN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; RV64-BITS-UNKNOWN-NEXT:    vmv8r.v v16, v8
 ; RV64-BITS-UNKNOWN-NEXT:    csrr a0, vlenb
 ; RV64-BITS-UNKNOWN-NEXT:    addi a0, a0, -1
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; RV64-BITS-UNKNOWN-NEXT:    vid.v v8
 ; RV64-BITS-UNKNOWN-NEXT:    vrsub.vx v24, v8, a0
 ; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
@@ -1243,10 +1243,10 @@ define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ;
 ; RV64-BITS-256-LABEL: reverse_nxv64i8:
 ; RV64-BITS-256:       # %bb.0:
+; RV64-BITS-256-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; RV64-BITS-256-NEXT:    vmv8r.v v16, v8
 ; RV64-BITS-256-NEXT:    csrr a0, vlenb
 ; RV64-BITS-256-NEXT:    addi a0, a0, -1
-; RV64-BITS-256-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; RV64-BITS-256-NEXT:    vid.v v8
 ; RV64-BITS-256-NEXT:    vrsub.vx v24, v8, a0
 ; RV64-BITS-256-NEXT:    vrgather.vv v15, v16, v24
@@ -1261,10 +1261,10 @@ define <vscale x 64 x i8> @reverse_nxv64i8(<vscale x 64 x i8> %a) {
 ;
 ; RV64-BITS-512-LABEL: reverse_nxv64i8:
 ; RV64-BITS-512:       # %bb.0:
+; RV64-BITS-512-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; RV64-BITS-512-NEXT:    vmv8r.v v16, v8
 ; RV64-BITS-512-NEXT:    csrr a0, vlenb
 ; RV64-BITS-512-NEXT:    addi a0, a0, -1
-; RV64-BITS-512-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; RV64-BITS-512-NEXT:    vid.v v8
 ; RV64-BITS-512-NEXT:    vrsub.vx v24, v8, a0
 ; RV64-BITS-512-NEXT:    vrgather.vv v15, v16, v24
@@ -1367,11 +1367,11 @@ define <vscale x 16 x i16> @reverse_nxv16i16(<vscale x 16 x i16> %a) {
 define <vscale x 32 x i16> @reverse_nxv32i16(<vscale x 32 x i16> %a) {
 ; CHECK-LABEL: reverse_nxv32i16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1458,11 +1458,11 @@ define <vscale x 8 x i32> @reverse_nxv8i32(<vscale x 8 x i32> %a) {
 define <vscale x 16 x i32> @reverse_nxv16i32(<vscale x 16 x i32> %a) {
 ; CHECK-LABEL: reverse_nxv16i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1533,11 +1533,11 @@ define <vscale x 4 x i64> @reverse_nxv4i64(<vscale x 4 x i64> %a) {
 define <vscale x 8 x i64> @reverse_nxv8i64(<vscale x 8 x i64> %a) {
 ; CHECK-LABEL: reverse_nxv8i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1644,11 +1644,11 @@ define <vscale x 16 x bfloat> @reverse_nxv16bf16(<vscale x 16 x bfloat> %a) {
 define <vscale x 32 x bfloat> @reverse_nxv32bf16(<vscale x 32 x bfloat> %a) {
 ; CHECK-LABEL: reverse_nxv32bf16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1751,11 +1751,11 @@ define <vscale x 16 x half> @reverse_nxv16f16(<vscale x 16 x half> %a) {
 define <vscale x 32 x half> @reverse_nxv32f16(<vscale x 32 x half> %a) {
 ; CHECK-LABEL: reverse_nxv32f16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1842,11 +1842,11 @@ define <vscale x 8 x float> @reverse_nxv8f32(<vscale x 8 x float> %a) {
 define <vscale x 16 x float> @reverse_nxv16f32(<vscale x 16 x float> %a) {
 ; CHECK-LABEL: reverse_nxv16f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
@@ -1917,11 +1917,11 @@ define <vscale x 4 x double> @reverse_nxv4f64(<vscale x 4 x double> %a) {
 define <vscale x 8 x double> @reverse_nxv8f64(<vscale x 8 x double> %a) {
 ; CHECK-LABEL: reverse_nxv8f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    vrsub.vx v24, v8, a0
 ; CHECK-NEXT:    vrgather.vv v15, v16, v24
diff --git a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
index 94fce80ad3b8e..9aa26e59c6a03 100644
--- a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.nearbyint.nxv4bf16(<vscale x 4 x bfloat>,
 define <vscale x 4 x bfloat> @vp_nearbyint_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.nearbyint.nxv8bf16(<vscale x 8 x bfloat>,
 define <vscale x 8 x bfloat> @vp_nearbyint_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.nearbyint.nxv16bf16(<vscale x 16 x bfloa
 define <vscale x 16 x bfloat> @vp_nearbyint_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -273,9 +273,9 @@ declare <vscale x 32 x bfloat> @llvm.vp.nearbyint.nxv32bf16(<vscale x 32 x bfloa
 define <vscale x 32 x bfloat> @vp_nearbyint_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv32bf16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -566,8 +566,8 @@ define <vscale x 4 x half> @vp_nearbyint_nxv4f16(<vscale x 4 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -633,10 +633,10 @@ declare <vscale x 8 x half> @llvm.vp.nearbyint.nxv8f16(<vscale x 8 x half>, <vsc
 define <vscale x 8 x half> @vp_nearbyint_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_nearbyint_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -652,8 +652,8 @@ define <vscale x 8 x half> @vp_nearbyint_nxv8f16(<vscale x 8 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -719,10 +719,10 @@ declare <vscale x 16 x half> @llvm.vp.nearbyint.nxv16f16(<vscale x 16 x half>, <
 define <vscale x 16 x half> @vp_nearbyint_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_nearbyint_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -738,8 +738,8 @@ define <vscale x 16 x half> @vp_nearbyint_nxv16f16(<vscale x 16 x half> %va, <vs
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -805,10 +805,10 @@ declare <vscale x 32 x half> @llvm.vp.nearbyint.nxv32f16(<vscale x 32 x half>, <
 define <vscale x 32 x half> @vp_nearbyint_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_nearbyint_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -824,9 +824,9 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16(<vscale x 32 x half> %va, <vs
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv32f16:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1036,8 +1036,8 @@ declare <vscale x 4 x float> @llvm.vp.nearbyint.nxv4f32(<vscale x 4 x float>, <v
 define <vscale x 4 x float> @vp_nearbyint_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1080,8 +1080,8 @@ declare <vscale x 8 x float> @llvm.vp.nearbyint.nxv8f32(<vscale x 8 x float>, <v
 define <vscale x 8 x float> @vp_nearbyint_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1124,8 +1124,8 @@ declare <vscale x 16 x float> @llvm.vp.nearbyint.nxv16f32(<vscale x 16 x float>,
 define <vscale x 16 x float> @vp_nearbyint_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1210,10 +1210,10 @@ declare <vscale x 2 x double> @llvm.vp.nearbyint.nxv2f64(<vscale x 2 x double>,
 define <vscale x 2 x double> @vp_nearbyint_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1254,10 +1254,10 @@ declare <vscale x 4 x double> @llvm.vp.nearbyint.nxv4f64(<vscale x 4 x double>,
 define <vscale x 4 x double> @vp_nearbyint_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1298,10 +1298,10 @@ declare <vscale x 7 x double> @llvm.vp.nearbyint.nxv7f64(<vscale x 7 x double>,
 define <vscale x 7 x double> @vp_nearbyint_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1342,10 +1342,10 @@ declare <vscale x 8 x double> @llvm.vp.nearbyint.nxv8f64(<vscale x 8 x double>,
 define <vscale x 8 x double> @vp_nearbyint_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1387,13 +1387,13 @@ declare <vscale x 16 x double> @llvm.vp.nearbyint.nxv16f64(<vscale x 16 x double
 define <vscale x 16 x double> @vp_nearbyint_nxv16f64(<vscale x 16 x double> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_nearbyint_nxv16f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr88576.ll b/llvm/test/CodeGen/RISCV/rvv/pr88576.ll
index 37c67b9ff2f6a..dd7debd3ab046 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr88576.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr88576.ll
@@ -23,10 +23,10 @@ define i1 @foo(<vscale x 16 x i8> %x, i64 %y) {
 ; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    sub sp, sp, a2
 ; CHECK-NEXT:    andi sp, sp, -64
+; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    addi a2, sp, 64
 ; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    add a0, a2, a0
 ; CHECK-NEXT:    add a1, a2, a1
@@ -53,8 +53,8 @@ define i1 @foo(<vscale x 16 x i8> %x, i64 %y) {
 define i8 @bar(<vscale x 128 x i1> %x, i64 %y) {
 ; CHECK-LABEL: bar:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v1, v8
 ; CHECK-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v1, v8
 ; CHECK-NEXT:    vslidedown.vx v8, v0, a0
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
index 2a69dd31118bd..70ea1bc78d2e5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rint-vp.ll
@@ -109,8 +109,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.rint.nxv4bf16(<vscale x 4 x bfloat>, <vsc
 define <vscale x 4 x bfloat> @vp_rint_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -157,8 +157,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.rint.nxv8bf16(<vscale x 8 x bfloat>, <vsc
 define <vscale x 8 x bfloat> @vp_rint_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -205,8 +205,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.rint.nxv16bf16(<vscale x 16 x bfloat>, <
 define <vscale x 16 x bfloat> @vp_rint_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -259,9 +259,9 @@ define <vscale x 32 x bfloat> @vp_rint_nxv32bf16(<vscale x 32 x bfloat> %va, <vs
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -535,8 +535,8 @@ define <vscale x 4 x half> @vp_rint_nxv4f16(<vscale x 4 x half> %va, <vscale x 4
 ;
 ; ZVFHMIN-LABEL: vp_rint_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -596,10 +596,10 @@ declare <vscale x 8 x half> @llvm.vp.rint.nxv8f16(<vscale x 8 x half>, <vscale x
 define <vscale x 8 x half> @vp_rint_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_rint_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -613,8 +613,8 @@ define <vscale x 8 x half> @vp_rint_nxv8f16(<vscale x 8 x half> %va, <vscale x 8
 ;
 ; ZVFHMIN-LABEL: vp_rint_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -674,10 +674,10 @@ declare <vscale x 16 x half> @llvm.vp.rint.nxv16f16(<vscale x 16 x half>, <vscal
 define <vscale x 16 x half> @vp_rint_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_rint_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -691,8 +691,8 @@ define <vscale x 16 x half> @vp_rint_nxv16f16(<vscale x 16 x half> %va, <vscale
 ;
 ; ZVFHMIN-LABEL: vp_rint_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -752,10 +752,10 @@ declare <vscale x 32 x half> @llvm.vp.rint.nxv32f16(<vscale x 32 x half>, <vscal
 define <vscale x 32 x half> @vp_rint_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_rint_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -775,9 +775,9 @@ define <vscale x 32 x half> @vp_rint_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -978,8 +978,8 @@ declare <vscale x 4 x float> @llvm.vp.rint.nxv4f32(<vscale x 4 x float>, <vscale
 define <vscale x 4 x float> @vp_rint_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1018,8 +1018,8 @@ declare <vscale x 8 x float> @llvm.vp.rint.nxv8f32(<vscale x 8 x float>, <vscale
 define <vscale x 8 x float> @vp_rint_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1058,8 +1058,8 @@ declare <vscale x 16 x float> @llvm.vp.rint.nxv16f32(<vscale x 16 x float>, <vsc
 define <vscale x 16 x float> @vp_rint_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1136,10 +1136,10 @@ declare <vscale x 2 x double> @llvm.vp.rint.nxv2f64(<vscale x 2 x double>, <vsca
 define <vscale x 2 x double> @vp_rint_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1176,10 +1176,10 @@ declare <vscale x 4 x double> @llvm.vp.rint.nxv4f64(<vscale x 4 x double>, <vsca
 define <vscale x 4 x double> @vp_rint_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1216,10 +1216,10 @@ declare <vscale x 7 x double> @llvm.vp.rint.nxv7f64(<vscale x 7 x double>, <vsca
 define <vscale x 7 x double> @vp_rint_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1256,10 +1256,10 @@ declare <vscale x 8 x double> @llvm.vp.rint.nxv8f64(<vscale x 8 x double>, <vsca
 define <vscale x 8 x double> @vp_rint_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_rint_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1303,13 +1303,13 @@ define <vscale x 16 x double> @vp_rint_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
index 8a10e75333ad0..4e15f13570583 100644
--- a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.round.nxv4bf16(<vscale x 4 x bfloat>, <vs
 define <vscale x 4 x bfloat> @vp_round_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.round.nxv8bf16(<vscale x 8 x bfloat>, <vs
 define <vscale x 8 x bfloat> @vp_round_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.round.nxv16bf16(<vscale x 16 x bfloat>,
 define <vscale x 16 x bfloat> @vp_round_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -279,9 +279,9 @@ define <vscale x 32 x bfloat> @vp_round_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -582,8 +582,8 @@ define <vscale x 4 x half> @vp_round_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vp_round_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -649,10 +649,10 @@ declare <vscale x 8 x half> @llvm.vp.round.nxv8f16(<vscale x 8 x half>, <vscale
 define <vscale x 8 x half> @vp_round_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_round_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -668,8 +668,8 @@ define <vscale x 8 x half> @vp_round_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ;
 ; ZVFHMIN-LABEL: vp_round_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -735,10 +735,10 @@ declare <vscale x 16 x half> @llvm.vp.round.nxv16f16(<vscale x 16 x half>, <vsca
 define <vscale x 16 x half> @vp_round_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_round_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -754,8 +754,8 @@ define <vscale x 16 x half> @vp_round_nxv16f16(<vscale x 16 x half> %va, <vscale
 ;
 ; ZVFHMIN-LABEL: vp_round_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -821,10 +821,10 @@ declare <vscale x 32 x half> @llvm.vp.round.nxv32f16(<vscale x 32 x half>, <vsca
 define <vscale x 32 x half> @vp_round_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_round_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -846,9 +846,9 @@ define <vscale x 32 x half> @vp_round_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1068,8 +1068,8 @@ declare <vscale x 4 x float> @llvm.vp.round.nxv4f32(<vscale x 4 x float>, <vscal
 define <vscale x 4 x float> @vp_round_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1112,8 +1112,8 @@ declare <vscale x 8 x float> @llvm.vp.round.nxv8f32(<vscale x 8 x float>, <vscal
 define <vscale x 8 x float> @vp_round_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1156,8 +1156,8 @@ declare <vscale x 16 x float> @llvm.vp.round.nxv16f32(<vscale x 16 x float>, <vs
 define <vscale x 16 x float> @vp_round_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1242,10 +1242,10 @@ declare <vscale x 2 x double> @llvm.vp.round.nxv2f64(<vscale x 2 x double>, <vsc
 define <vscale x 2 x double> @vp_round_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1286,10 +1286,10 @@ declare <vscale x 4 x double> @llvm.vp.round.nxv4f64(<vscale x 4 x double>, <vsc
 define <vscale x 4 x double> @vp_round_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1330,10 +1330,10 @@ declare <vscale x 7 x double> @llvm.vp.round.nxv7f64(<vscale x 7 x double>, <vsc
 define <vscale x 7 x double> @vp_round_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1374,10 +1374,10 @@ declare <vscale x 8 x double> @llvm.vp.round.nxv8f64(<vscale x 8 x double>, <vsc
 define <vscale x 8 x double> @vp_round_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_round_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1425,13 +1425,13 @@ define <vscale x 16 x double> @vp_round_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
index 4cd909e4b0a63..f1510a8c53181 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.roundeven.nxv4bf16(<vscale x 4 x bfloat>,
 define <vscale x 4 x bfloat> @vp_roundeven_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.roundeven.nxv8bf16(<vscale x 8 x bfloat>,
 define <vscale x 8 x bfloat> @vp_roundeven_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.roundeven.nxv16bf16(<vscale x 16 x bfloa
 define <vscale x 16 x bfloat> @vp_roundeven_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -279,9 +279,9 @@ define <vscale x 32 x bfloat> @vp_roundeven_nxv32bf16(<vscale x 32 x bfloat> %va
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -582,8 +582,8 @@ define <vscale x 4 x half> @vp_roundeven_nxv4f16(<vscale x 4 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vp_roundeven_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -649,10 +649,10 @@ declare <vscale x 8 x half> @llvm.vp.roundeven.nxv8f16(<vscale x 8 x half>, <vsc
 define <vscale x 8 x half> @vp_roundeven_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundeven_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -668,8 +668,8 @@ define <vscale x 8 x half> @vp_roundeven_nxv8f16(<vscale x 8 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vp_roundeven_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -735,10 +735,10 @@ declare <vscale x 16 x half> @llvm.vp.roundeven.nxv16f16(<vscale x 16 x half>, <
 define <vscale x 16 x half> @vp_roundeven_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundeven_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -754,8 +754,8 @@ define <vscale x 16 x half> @vp_roundeven_nxv16f16(<vscale x 16 x half> %va, <vs
 ;
 ; ZVFHMIN-LABEL: vp_roundeven_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -821,10 +821,10 @@ declare <vscale x 32 x half> @llvm.vp.roundeven.nxv32f16(<vscale x 32 x half>, <
 define <vscale x 32 x half> @vp_roundeven_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundeven_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -846,9 +846,9 @@ define <vscale x 32 x half> @vp_roundeven_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1068,8 +1068,8 @@ declare <vscale x 4 x float> @llvm.vp.roundeven.nxv4f32(<vscale x 4 x float>, <v
 define <vscale x 4 x float> @vp_roundeven_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1112,8 +1112,8 @@ declare <vscale x 8 x float> @llvm.vp.roundeven.nxv8f32(<vscale x 8 x float>, <v
 define <vscale x 8 x float> @vp_roundeven_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1156,8 +1156,8 @@ declare <vscale x 16 x float> @llvm.vp.roundeven.nxv16f32(<vscale x 16 x float>,
 define <vscale x 16 x float> @vp_roundeven_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1242,10 +1242,10 @@ declare <vscale x 2 x double> @llvm.vp.roundeven.nxv2f64(<vscale x 2 x double>,
 define <vscale x 2 x double> @vp_roundeven_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1286,10 +1286,10 @@ declare <vscale x 4 x double> @llvm.vp.roundeven.nxv4f64(<vscale x 4 x double>,
 define <vscale x 4 x double> @vp_roundeven_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1330,10 +1330,10 @@ declare <vscale x 7 x double> @llvm.vp.roundeven.nxv7f64(<vscale x 7 x double>,
 define <vscale x 7 x double> @vp_roundeven_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1374,10 +1374,10 @@ declare <vscale x 8 x double> @llvm.vp.roundeven.nxv8f64(<vscale x 8 x double>,
 define <vscale x 8 x double> @vp_roundeven_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundeven_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1425,13 +1425,13 @@ define <vscale x 16 x double> @vp_roundeven_nxv16f64(<vscale x 16 x double> %va,
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
index 96c821a76ae84..dd7db58ccdf34 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
@@ -117,8 +117,8 @@ declare <vscale x 4 x bfloat> @llvm.vp.roundtozero.nxv4bf16(<vscale x 4 x bfloat
 define <vscale x 4 x bfloat> @vp_roundtozero_nxv4bf16(<vscale x 4 x bfloat> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv4bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -169,8 +169,8 @@ declare <vscale x 8 x bfloat> @llvm.vp.roundtozero.nxv8bf16(<vscale x 8 x bfloat
 define <vscale x 8 x bfloat> @vp_roundtozero_nxv8bf16(<vscale x 8 x bfloat> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv8bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -221,8 +221,8 @@ declare <vscale x 16 x bfloat> @llvm.vp.roundtozero.nxv16bf16(<vscale x 16 x bfl
 define <vscale x 16 x bfloat> @vp_roundtozero_nxv16bf16(<vscale x 16 x bfloat> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv16bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    lui a1, 307200
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -279,9 +279,9 @@ define <vscale x 32 x bfloat> @vp_roundtozero_nxv32bf16(<vscale x 32 x bfloat> %
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    lui a3, 307200
 ; CHECK-NEXT:    slli a1, a2, 1
@@ -582,8 +582,8 @@ define <vscale x 4 x half> @vp_roundtozero_nxv4f16(<vscale x 4 x half> %va, <vsc
 ;
 ; ZVFHMIN-LABEL: vp_roundtozero_nxv4f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v9, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
@@ -649,10 +649,10 @@ declare <vscale x 8 x half> @llvm.vp.roundtozero.nxv8f16(<vscale x 8 x half>, <v
 define <vscale x 8 x half> @vp_roundtozero_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundtozero_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v10, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI18_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vmv1r.v v10, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI18_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI18_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -668,8 +668,8 @@ define <vscale x 8 x half> @vp_roundtozero_nxv8f16(<vscale x 8 x half> %va, <vsc
 ;
 ; ZVFHMIN-LABEL: vp_roundtozero_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v10, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
@@ -735,10 +735,10 @@ declare <vscale x 16 x half> @llvm.vp.roundtozero.nxv16f16(<vscale x 16 x half>,
 define <vscale x 16 x half> @vp_roundtozero_nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundtozero_nxv16f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v12, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI20_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFH-NEXT:    vmv1r.v v12, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI20_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -754,8 +754,8 @@ define <vscale x 16 x half> @vp_roundtozero_nxv16f16(<vscale x 16 x half> %va, <
 ;
 ; ZVFHMIN-LABEL: vp_roundtozero_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v12, v0
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    lui a1, 307200
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -821,10 +821,10 @@ declare <vscale x 32 x half> @llvm.vp.roundtozero.nxv32f16(<vscale x 32 x half>,
 define <vscale x 32 x half> @vp_roundtozero_nxv32f16(<vscale x 32 x half> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vp_roundtozero_nxv32f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vmv1r.v v16, v0
-; ZVFH-NEXT:    lui a1, %hi(.LCPI22_0)
-; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a1)
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v16, v0
+; ZVFH-NEXT:    lui a0, %hi(.LCPI22_0)
+; ZVFH-NEXT:    flh fa5, %lo(.LCPI22_0)(a0)
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -846,9 +846,9 @@ define <vscale x 32 x half> @vp_roundtozero_nxv32f16(<vscale x 32 x half> %va, <
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    lui a3, 307200
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
@@ -1068,8 +1068,8 @@ declare <vscale x 4 x float> @llvm.vp.roundtozero.nxv4f32(<vscale x 4 x float>,
 define <vscale x 4 x float> @vp_roundtozero_nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1112,8 +1112,8 @@ declare <vscale x 8 x float> @llvm.vp.roundtozero.nxv8f32(<vscale x 8 x float>,
 define <vscale x 8 x float> @vp_roundtozero_nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1156,8 +1156,8 @@ declare <vscale x 16 x float> @llvm.vp.roundtozero.nxv16f32(<vscale x 16 x float
 define <vscale x 16 x float> @vp_roundtozero_nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    lui a0, 307200
 ; CHECK-NEXT:    fmv.w.x fa5, a0
@@ -1242,10 +1242,10 @@ declare <vscale x 2 x double> @llvm.vp.roundtozero.nxv2f64(<vscale x 2 x double>
 define <vscale x 2 x double> @vp_roundtozero_nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI36_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI36_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI36_0)(a0)
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
@@ -1286,10 +1286,10 @@ declare <vscale x 4 x double> @llvm.vp.roundtozero.nxv4f64(<vscale x 4 x double>
 define <vscale x 4 x double> @vp_roundtozero_nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI38_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vmv1r.v v12, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI38_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI38_0)(a0)
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
@@ -1330,10 +1330,10 @@ declare <vscale x 7 x double> @llvm.vp.roundtozero.nxv7f64(<vscale x 7 x double>
 define <vscale x 7 x double> @vp_roundtozero_nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv7f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI40_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI40_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI40_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1374,10 +1374,10 @@ declare <vscale x 8 x double> @llvm.vp.roundtozero.nxv8f64(<vscale x 8 x double>
 define <vscale x 8 x double> @vp_roundtozero_nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_roundtozero_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v16, v0
-; CHECK-NEXT:    lui a1, %hi(.LCPI42_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a1)
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v16, v0
+; CHECK-NEXT:    lui a0, %hi(.LCPI42_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI42_0)(a0)
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
@@ -1425,13 +1425,13 @@ define <vscale x 16 x double> @vp_roundtozero_nxv16f64(<vscale x 16 x double> %v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    lui a2, %hi(.LCPI44_0)
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI44_0)(a2)
 ; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v0, a3
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
index aef160049106b..f636ab9ebd0ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
@@ -17,6 +17,7 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    slli a1, a1, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a1
 ; SPILL-O0-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; SPILL-O0-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; SPILL-O0-NEXT:    vmv1r.v v10, v9
 ; SPILL-O0-NEXT:    vmv1r.v v9, v8
 ; SPILL-O0-NEXT:    csrr a1, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
index b34952b64f09e..adb15f02e33a4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
@@ -15,6 +15,7 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -90,6 +91,7 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -166,6 +168,7 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
@@ -246,6 +249,7 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs4r.v v8, (a0) # Unknown-size Folded Spill
@@ -326,6 +330,7 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
index c7c44fb0e1215..2cd80ef79bd82 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
@@ -20,6 +20,7 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    slli a1, a1, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a1
 ; SPILL-O0-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; SPILL-O0-NEXT:    vmv1r.v v10, v9
 ; SPILL-O0-NEXT:    vmv1r.v v9, v8
 ; SPILL-O0-NEXT:    csrr a1, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
index 361adb55ef12f..ff0f1d7748668 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
@@ -15,6 +15,7 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -90,6 +91,7 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -166,6 +168,7 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
@@ -246,6 +249,7 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs4r.v v8, (a0) # Unknown-size Folded Spill
@@ -326,6 +330,7 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
index b27ba14e85c83..53d1666c30e96 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
@@ -47,6 +47,7 @@ define <vscale x 16 x i32> @foo(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5,
 ; CHECK-NEXT:    vs8r.v v8, (t1)
 ; CHECK-NEXT:    sd t1, 0(sp)
 ; CHECK-NEXT:    sd t0, 8(sp)
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    call bar
 ; CHECK-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index 23ebfade6f6b0..d329979857a6b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -941,8 +941,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vredsum.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @vredsum(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %x, <vscale x 2 x i32> %y, <vscale x 2 x i1> %m, i64 %vl) {
 ; CHECK-LABEL: vredsum:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
+; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vredsum.vs v11, v9, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v11, v0
 ; CHECK-NEXT:    ret
@@ -965,8 +965,8 @@ define <vscale x 2 x float> @vfredusum(<vscale x 2 x float> %passthru, <vscale x
 ; CHECK-LABEL: vfredusum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fsrmi a1, 0
-; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
+; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vfredusum.vs v11, v9, v10
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v11, v0
 ; CHECK-NEXT:    fsrm a1
@@ -1016,8 +1016,8 @@ define <vscale x 2 x float> @vfredusum_allones_mask(<vscale x 2 x float> %passth
 define <vscale x 2 x i32> @unfoldable_vredsum_allones_mask_diff_vl(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %x, <vscale x 2 x i32> %y) {
 ; CHECK-LABEL: unfoldable_vredsum_allones_mask_diff_vl:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, tu, ma
+; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vredsum.vs v11, v9, v10
 ; CHECK-NEXT:    vsetivli zero, 1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.v.v v8, v11
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 6c11e9413525e..70b53841bff4c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -1473,6 +1473,7 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    add a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    mv a3, a1
@@ -1504,7 +1505,6 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    add t0, sp, t0
 ; CHECK-NEXT:    addi t0, t0, 16
 ; CHECK-NEXT:    vs1r.v v24, (t0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v25, v24, a1
 ; CHECK-NEXT:    vsetvli t0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v24, v25, a3
@@ -3720,6 +3720,7 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFH-NEXT:    slli a1, a1, 4
 ; ZVFH-NEXT:    sub sp, sp, a1
 ; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFH-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v24, v0
 ; ZVFH-NEXT:    csrr a1, vlenb
 ; ZVFH-NEXT:    slli a1, a1, 3
@@ -3738,7 +3739,6 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFH-NEXT:    vl8re16.v v0, (a0)
 ; ZVFH-NEXT:    addi a0, sp, 16
 ; ZVFH-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; ZVFH-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; ZVFH-NEXT:    vslidedown.vx v0, v24, a1
 ; ZVFH-NEXT:    and a4, a4, a5
 ; ZVFH-NEXT:    vsetvli zero, a4, e16, m8, ta, ma
@@ -3781,6 +3781,7 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    add a1, a1, a3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v24, v0
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    mv a3, a1
@@ -3812,7 +3813,6 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    add t0, sp, t0
 ; ZVFHMIN-NEXT:    addi t0, t0, 16
 ; ZVFHMIN-NEXT:    vs1r.v v24, (t0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v25, v24, a1
 ; ZVFHMIN-NEXT:    vsetvli t0, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v24, v25, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
index e8099c2d08a5f..2392cb77c753d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
@@ -1092,6 +1092,7 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -1099,7 +1100,6 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vsetvli a4, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vlm.v v0, (a2)
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a2, a0, a1
@@ -1143,8 +1143,8 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 define <vscale x 128 x i1> @icmp_eq_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -1173,8 +1173,8 @@ define <vscale x 128 x i1> @icmp_eq_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b,
 define <vscale x 128 x i1> @icmp_eq_vx_swap_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_swap_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -2244,6 +2244,7 @@ define <vscale x 32 x i1> @icmp_eq_vv_nxv32i32(<vscale x 32 x i32> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -2262,7 +2263,6 @@ define <vscale x 32 x i1> @icmp_eq_vv_nxv32i32(<vscale x 32 x i32> %va, <vscale
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a1
 ; CHECK-NEXT:    and a4, a4, a5
 ; CHECK-NEXT:    vsetvli zero, a4, e32, m8, ta, ma
@@ -2299,11 +2299,11 @@ define <vscale x 32 x i1> @icmp_eq_vv_nxv32i32(<vscale x 32 x i32> %va, <vscale
 define <vscale x 32 x i1> @icmp_eq_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    srli a2, a3, 2
 ; CHECK-NEXT:    slli a3, a3, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a4, a1, a3
 ; CHECK-NEXT:    sltu a5, a1, a4
@@ -2332,11 +2332,11 @@ define <vscale x 32 x i1> @icmp_eq_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b,
 define <vscale x 32 x i1> @icmp_eq_vx_swap_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: icmp_eq_vx_swap_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    srli a2, a3, 2
 ; CHECK-NEXT:    slli a3, a3, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a4, a1, a3
 ; CHECK-NEXT:    sltu a5, a1, a4
diff --git a/llvm/test/CodeGen/RISCV/rvv/shrinkwrap.ll b/llvm/test/CodeGen/RISCV/rvv/shrinkwrap.ll
new file mode 100644
index 0000000000000..d12f2c889650f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/shrinkwrap.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v < %s | FileCheck --check-prefix=RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v < %s | FileCheck --check-prefix=RV64 %s
+
+; FIXME: We can rematerialize "addi s0, a2, 32" (ideally along the edge
+; %do.call -> %exit), and shrink wrap this routine
+define void @vecaddr_straightline(i32 zeroext %a, ptr %p) {
+; RV32-LABEL: vecaddr_straightline:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    .cfi_offset s0, -8
+; RV32-NEXT:    addi s0, a1, 32
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vle32.v v8, (s0)
+; RV32-NEXT:    vadd.vi v8, v8, 1
+; RV32-NEXT:    li a1, 57
+; RV32-NEXT:    vse32.v v8, (s0)
+; RV32-NEXT:    beq a0, a1, .LBB0_2
+; RV32-NEXT:  # %bb.1: # %do_call
+; RV32-NEXT:    call foo
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:  .LBB0_2: # %exit
+; RV32-NEXT:    vle32.v v8, (s0)
+; RV32-NEXT:    vadd.vi v8, v8, 1
+; RV32-NEXT:    vse32.v v8, (s0)
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    .cfi_restore s0
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vecaddr_straightline:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    .cfi_offset s0, -16
+; RV64-NEXT:    addi s0, a1, 32
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vle32.v v8, (s0)
+; RV64-NEXT:    vadd.vi v8, v8, 1
+; RV64-NEXT:    li a1, 57
+; RV64-NEXT:    vse32.v v8, (s0)
+; RV64-NEXT:    beq a0, a1, .LBB0_2
+; RV64-NEXT:  # %bb.1: # %do_call
+; RV64-NEXT:    call foo
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:  .LBB0_2: # %exit
+; RV64-NEXT:    vle32.v v8, (s0)
+; RV64-NEXT:    vadd.vi v8, v8, 1
+; RV64-NEXT:    vse32.v v8, (s0)
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    .cfi_restore s0
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %gep = getelementptr i8, ptr %p, i32 32
+  %v1 = load <4 x i32>, ptr %gep
+  %v2 = add <4 x i32> %v1, splat (i32 1)
+  store <4 x i32> %v2, ptr %gep
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  call i32 @foo()
+  br label %exit
+exit:
+  %v3 = load <4 x i32>, ptr %gep
+  %v4 = add <4 x i32> %v3, splat (i32 1)
+  store <4 x i32> %v4, ptr %gep
+  ret void
+}
+
+; In this case, the second use is in a loop, so using a callee
+; saved register to avoid a remat is the profitable choice.
+; FIXME: We can shrink wrap the frame setup around the loop
+; and avoid it along the %bb.0 -> %exit edge
+define void @vecaddr_loop(i32 zeroext %a, ptr %p) {
+; RV32-LABEL: vecaddr_loop:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    .cfi_offset s0, -8
+; RV32-NEXT:    addi s0, a1, 32
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vle32.v v8, (s0)
+; RV32-NEXT:    vadd.vi v8, v8, 1
+; RV32-NEXT:    li a1, 57
+; RV32-NEXT:    vse32.v v8, (s0)
+; RV32-NEXT:    beq a0, a1, .LBB1_2
+; RV32-NEXT:  .LBB1_1: # %do_call
+; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32-NEXT:    call foo
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vle32.v v8, (s0)
+; RV32-NEXT:    vadd.vi v8, v8, 1
+; RV32-NEXT:    vse32.v v8, (s0)
+; RV32-NEXT:    bnez a0, .LBB1_1
+; RV32-NEXT:  .LBB1_2: # %exit
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    .cfi_restore s0
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vecaddr_loop:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    .cfi_offset s0, -16
+; RV64-NEXT:    addi s0, a1, 32
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vle32.v v8, (s0)
+; RV64-NEXT:    vadd.vi v8, v8, 1
+; RV64-NEXT:    li a1, 57
+; RV64-NEXT:    vse32.v v8, (s0)
+; RV64-NEXT:    beq a0, a1, .LBB1_2
+; RV64-NEXT:  .LBB1_1: # %do_call
+; RV64-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64-NEXT:    call foo
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vle32.v v8, (s0)
+; RV64-NEXT:    vadd.vi v8, v8, 1
+; RV64-NEXT:    vse32.v v8, (s0)
+; RV64-NEXT:    bnez a0, .LBB1_1
+; RV64-NEXT:  .LBB1_2: # %exit
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    .cfi_restore s0
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %gep = getelementptr i8, ptr %p, i32 32
+  %v1 = load <4 x i32>, ptr %gep
+  %v2 = add <4 x i32> %v1, splat (i32 1)
+  store <4 x i32> %v2, ptr %gep
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  %b = call i32 @foo()
+  %v3 = load <4 x i32>, ptr %gep
+  %v4 = add <4 x i32> %v3, splat (i32 1)
+  store <4 x i32> %v4, ptr %gep
+
+  %cmp1 = icmp eq i32 %b, 0
+  br i1 %cmp1, label %exit, label %do_call
+exit:
+  ret void
+}
+
+declare zeroext i32 @foo()
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index c91b02e8f15e4..75a5eea1cb409 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -4865,10 +4865,10 @@ declare <4 x i1> @llvm.vp.icmp.v4i32(<4 x i32>, <4 x i32>, metadata, <4 x i1>, i
 define void @sink_splat_vp_icmp(ptr nocapture %x, i32 signext %y, <4 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: sink_splat_vp_icmp:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    lui a3, 1
 ; CHECK-NEXT:    add a3, a0, a3
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v9, 0
 ; CHECK-NEXT:  .LBB102_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -4906,10 +4906,10 @@ declare <4 x i1> @llvm.vp.fcmp.v4f32(<4 x float>, <4 x float>, metadata, <4 x i1
 define void @sink_splat_vp_fcmp(ptr nocapture %x, float %y, <4 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: sink_splat_vp_fcmp:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    lui a2, 1
 ; CHECK-NEXT:    add a2, a0, a2
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v9, 0
 ; CHECK-NEXT:  .LBB103_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -5498,3 +5498,86 @@ vector.body:                                      ; preds = %vector.body, %entry
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
+
+define void @sink_splat_vp_select_op1(ptr %a, i32 %x, i32 %vl) {
+; CHECK-LABEL: sink_splat_vp_select_op1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a4, 1
+; CHECK-NEXT:    li a3, 42
+; CHECK-NEXT:    slli a5, a2, 32
+; CHECK-NEXT:    add a2, a0, a4
+; CHECK-NEXT:    srli a4, a5, 32
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:  .LBB119_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmseq.vx v0, v8, a3
+; CHECK-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vxm v8, v8, a1, v0
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    bne a0, a2, .LBB119_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, ptr %a, i64 %index
+  %load = load <4 x i32>, ptr %0, align 4
+  %cond = icmp eq <4 x i32> %load, splat (i32 42)
+  %1 = call <4 x i32> @llvm.vp.select(<4 x i1> %cond, <4 x i32> %broadcast.splat, <4 x i32> %load, i32 %vl)
+  store <4 x i32> %1, ptr %0, align 4
+  %index.next = add nuw i64 %index, 4
+  %2 = icmp eq i64 %index.next, 1024
+  br i1 %2, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @sink_splat_vp_select_op2(ptr %a, i32 %x, i32 %vl) {
+; CHECK-LABEL: sink_splat_vp_select_op2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a1
+; CHECK-NEXT:    lui a3, 1
+; CHECK-NEXT:    li a1, 42
+; CHECK-NEXT:    slli a4, a2, 32
+; CHECK-NEXT:    add a2, a0, a3
+; CHECK-NEXT:    srli a3, a4, 32
+; CHECK-NEXT:  .LBB120_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vle32.v v9, (a0)
+; CHECK-NEXT:    vmseq.vx v0, v9, a1
+; CHECK-NEXT:    vsetvli zero, a3, e32, m1, ta, ma
+; CHECK-NEXT:    vmerge.vvm v9, v8, v9, v0
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v9, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    bne a0, a2, .LBB120_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, ptr %a, i64 %index
+  %load = load <4 x i32>, ptr %0, align 4
+  %cond = icmp eq <4 x i32> %load, splat (i32 42)
+  %1 = call <4 x i32> @llvm.vp.select(<4 x i1> %cond, <4 x i32> %load, <4 x i32> %broadcast.splat, i32 %vl)
+  store <4 x i32> %1, ptr %0, align 4
+  %index.next = add nuw i64 %index, 4
+  %2 = icmp eq i64 %index.next, 1024
+  br i1 %2, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
index 371055704c090..03b090def5119 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
@@ -12,15 +12,14 @@ define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, -1
-; CHECK-NEXT:    vsll.vv v10, v8, v9
 ; CHECK-NEXT:    srli a1, a0, 1
-; CHECK-NEXT:    vsra.vv v9, v10, v9
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
-; CHECK-NEXT:    vmv.v.x v9, a1
 ; CHECK-NEXT:    slli a0, a0, 63
-; CHECK-NEXT:    vmerge.vxm v9, v9, a0, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
+; CHECK-NEXT:    vmv.v.x v10, a1
+; CHECK-NEXT:    vmerge.vxm v10, v10, a0, v0
+; CHECK-NEXT:    vsll.vv v11, v8, v9
+; CHECK-NEXT:    vsra.vv v9, v11, v9
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
+; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
 ; CHECK-NEXT:    ret
   %tmp = call <2 x i64> @llvm.sshl.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %tmp
@@ -32,14 +31,13 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 524288
-; CHECK-NEXT:    vsll.vv v10, v8, v9
 ; CHECK-NEXT:    addi a1, a0, -1
-; CHECK-NEXT:    vsra.vv v9, v10, v9
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vmerge.vxm v9, v9, a0, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
+; CHECK-NEXT:    vmv.v.x v10, a1
+; CHECK-NEXT:    vmerge.vxm v10, v10, a0, v0
+; CHECK-NEXT:    vsll.vv v11, v8, v9
+; CHECK-NEXT:    vsra.vv v9, v11, v9
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
+; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.sshl.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %tmp
@@ -51,14 +49,13 @@ define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 8
-; CHECK-NEXT:    vsll.vv v10, v8, v9
 ; CHECK-NEXT:    addi a1, a0, -1
-; CHECK-NEXT:    vsra.vv v9, v10, v9
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
-; CHECK-NEXT:    vmv.v.x v9, a1
-; CHECK-NEXT:    vmerge.vxm v9, v9, a0, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
+; CHECK-NEXT:    vmv.v.x v10, a1
+; CHECK-NEXT:    vmerge.vxm v10, v10, a0, v0
+; CHECK-NEXT:    vsll.vv v11, v8, v9
+; CHECK-NEXT:    vsra.vv v9, v11, v9
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
+; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
 ; CHECK-NEXT:    ret
   %tmp = call <8 x i16> @llvm.sshl.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %tmp
@@ -70,14 +67,13 @@ define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, 127
-; CHECK-NEXT:    vsll.vv v10, v8, v9
-; CHECK-NEXT:    vsra.vv v9, v10, v9
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
-; CHECK-NEXT:    vmv.v.x v9, a0
+; CHECK-NEXT:    vmv.v.x v10, a0
 ; CHECK-NEXT:    li a0, 128
-; CHECK-NEXT:    vmerge.vxm v9, v9, a0, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
-; CHECK-NEXT:    vmerge.vvm v8, v10, v9, v0
+; CHECK-NEXT:    vmerge.vxm v10, v10, a0, v0
+; CHECK-NEXT:    vsll.vv v11, v8, v9
+; CHECK-NEXT:    vsra.vv v9, v11, v9
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
+; CHECK-NEXT:    vmerge.vvm v8, v11, v10, v0
 ; CHECK-NEXT:    ret
   %tmp = call <16 x i8> @llvm.sshl.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
   ret <16 x i8> %tmp
@@ -94,15 +90,14 @@ define <vscale x 2 x i64> @vec_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64>
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, -1
-; CHECK-NEXT:    vsll.vv v12, v8, v10
 ; CHECK-NEXT:    srli a1, a0, 1
-; CHECK-NEXT:    vsra.vv v14, v12, v10
-; CHECK-NEXT:    vmsne.vv v10, v8, v14
-; CHECK-NEXT:    vmv.v.x v8, a1
 ; CHECK-NEXT:    slli a0, a0, 63
-; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    vmv.v.x v12, a1
+; CHECK-NEXT:    vmerge.vxm v12, v12, a0, v0
+; CHECK-NEXT:    vsll.vv v14, v8, v10
+; CHECK-NEXT:    vsra.vv v10, v14, v10
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmerge.vvm v8, v14, v12, v0
 ; CHECK-NEXT:    ret
   %tmp = call <vscale x 2 x i64> @llvm.sshl.sat.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
   ret <vscale x 2 x i64> %tmp
@@ -114,14 +109,13 @@ define <vscale x 4 x i32> @vec_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32>
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 524288
-; CHECK-NEXT:    vsll.vv v12, v8, v10
 ; CHECK-NEXT:    addi a1, a0, -1
-; CHECK-NEXT:    vsra.vv v14, v12, v10
-; CHECK-NEXT:    vmsne.vv v10, v8, v14
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    vmv.v.x v12, a1
+; CHECK-NEXT:    vmerge.vxm v12, v12, a0, v0
+; CHECK-NEXT:    vsll.vv v14, v8, v10
+; CHECK-NEXT:    vsra.vv v10, v14, v10
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmerge.vvm v8, v14, v12, v0
 ; CHECK-NEXT:    ret
   %tmp = call <vscale x 4 x i32> @llvm.sshl.sat.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
   ret <vscale x 4 x i32> %tmp
@@ -133,14 +127,13 @@ define <vscale x 8 x i16> @vec_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16>
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 8
-; CHECK-NEXT:    vsll.vv v12, v8, v10
 ; CHECK-NEXT:    addi a1, a0, -1
-; CHECK-NEXT:    vsra.vv v14, v12, v10
-; CHECK-NEXT:    vmsne.vv v10, v8, v14
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    vmv.v.x v12, a1
+; CHECK-NEXT:    vmerge.vxm v12, v12, a0, v0
+; CHECK-NEXT:    vsll.vv v14, v8, v10
+; CHECK-NEXT:    vsra.vv v10, v14, v10
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmerge.vvm v8, v14, v12, v0
 ; CHECK-NEXT:    ret
   %tmp = call <vscale x 8 x i16> @llvm.sshl.sat.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y)
   ret <vscale x 8 x i16> %tmp
@@ -152,14 +145,13 @@ define <vscale x 16 x i8> @vec_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8>
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, 127
-; CHECK-NEXT:    vsll.vv v12, v8, v10
-; CHECK-NEXT:    vsra.vv v14, v12, v10
-; CHECK-NEXT:    vmsne.vv v10, v8, v14
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.v.x v12, a0
 ; CHECK-NEXT:    li a0, 128
-; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
+; CHECK-NEXT:    vmerge.vxm v12, v12, a0, v0
+; CHECK-NEXT:    vsll.vv v14, v8, v10
+; CHECK-NEXT:    vsra.vv v10, v14, v10
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmerge.vvm v8, v14, v12, v0
 ; CHECK-NEXT:    ret
   %tmp = call <vscale x 16 x i8> @llvm.sshl.sat.nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y)
   ret <vscale x 16 x i8> %tmp
diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
index f8315de324e42..ecd098edb30ae 100644
--- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
@@ -663,6 +663,7 @@ declare <vscale x 3 x double> @llvm.experimental.vp.strided.load.nxv3f64.p0.i32(
 define <vscale x 16 x double> @strided_load_nxv16f64(ptr %ptr, i64 %stride, <vscale x 16 x i1> %mask, i32 zeroext %evl) {
 ; CHECK-RV32-LABEL: strided_load_nxv16f64:
 ; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv1r.v v9, v0
 ; CHECK-RV32-NEXT:    csrr a4, vlenb
 ; CHECK-RV32-NEXT:    sub a2, a3, a4
@@ -688,6 +689,7 @@ define <vscale x 16 x double> @strided_load_nxv16f64(ptr %ptr, i64 %stride, <vsc
 ;
 ; CHECK-RV64-LABEL: strided_load_nxv16f64:
 ; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv1r.v v9, v0
 ; CHECK-RV64-NEXT:    csrr a4, vlenb
 ; CHECK-RV64-NEXT:    sub a3, a2, a4
@@ -765,6 +767,7 @@ declare <vscale x 16 x double> @llvm.experimental.vp.strided.load.nxv16f64.p0.i6
 define <vscale x 16 x double> @strided_load_nxv17f64(ptr %ptr, i64 %stride, <vscale x 17 x i1> %mask, i32 zeroext %evl, ptr %hi_ptr) {
 ; CHECK-RV32-LABEL: strided_load_nxv17f64:
 ; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV32-NEXT:    vmv1r.v v8, v0
 ; CHECK-RV32-NEXT:    csrr a2, vlenb
 ; CHECK-RV32-NEXT:    slli a7, a2, 1
@@ -812,6 +815,7 @@ define <vscale x 16 x double> @strided_load_nxv17f64(ptr %ptr, i64 %stride, <vsc
 ;
 ; CHECK-RV64-LABEL: strided_load_nxv17f64:
 ; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-RV64-NEXT:    vmv1r.v v8, v0
 ; CHECK-RV64-NEXT:    csrr a4, vlenb
 ; CHECK-RV64-NEXT:    slli a7, a4, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpstore.ll
index 98ec99bcfea33..9dcee7e5cb7d1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/strided-vpstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpstore.ll
@@ -615,6 +615,7 @@ define void @strided_store_nxv17f64(<vscale x 17 x double> %v, ptr %ptr, i32 sig
 ; CHECK-NEXT:    slli a4, a4, 3
 ; CHECK-NEXT:    sub sp, sp, a4
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    addi a4, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
index ab13c78da05e8..c9f9a79733003 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
@@ -158,8 +158,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vrgatherei16.vv.nxv8i8.i64(<vscale x 8 x i
 define void @repeat_shuffle(<2 x double> %v, ptr noalias %q) {
 ; CHECK-LABEL: repeat_shuffle:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v10, v8
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmv2r.v v10, v8
 ; CHECK-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-NEXT:    vse64.v v10, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
index ebd550013ec78..fee6799e992f3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
@@ -565,8 +565,8 @@ declare <vscale x 128 x i8> @llvm.vp.add.nxv128i8(<vscale x 128 x i8>, <vscale x
 define <vscale x 128 x i8> @vadd_vi_nxv128i8(<vscale x 128 x i8> %va, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vadd_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -1343,11 +1343,11 @@ declare <vscale x 32 x i32> @llvm.vp.add.nxv32i32(<vscale x 32 x i32>, <vscale x
 define <vscale x 32 x i32> @vadd_vi_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vadd_vi_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -1399,11 +1399,11 @@ declare i32 @llvm.vscale.i32()
 define <vscale x 32 x i32> @vadd_vi_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m) {
 ; CHECK-LABEL: vadd_vi_nxv32i32_evl_nx8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a2, a0, 2
 ; CHECK-NEXT:    slli a1, a0, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vcpop.ll b/llvm/test/CodeGen/RISCV/rvv/vcpop.ll
index e59a9174b03d9..6b35e4767b239 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vcpop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vcpop.ll
@@ -43,9 +43,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv1i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -97,9 +97,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv2i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv2i1(<vscale x 2 x i1> %0, <vscale x 2 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv2i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -137,9 +137,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv4i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv4i1(<vscale x 4 x i1> %0, <vscale x 4 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv4i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -177,9 +177,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv8i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv8i1(<vscale x 8 x i1> %0, <vscale x 8 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv8i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -217,9 +217,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv16i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv16i1(<vscale x 16 x i1> %0, <vscale x 16 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv16i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -257,9 +257,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv32i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv32i1(<vscale x 32 x i1> %0, <vscale x 32 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv32i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -297,9 +297,9 @@ declare iXLen @llvm.riscv.vcpop.mask.iXLen.nxv64i1(
 define iXLen @intrinsic_vcpop_mask_m_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vcpop_mask_m_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 54d2f3f68989b..2fc5b40a89afa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -7,8 +7,8 @@
 define {<16 x i1>, <16 x i1>} @vector_deinterleave_v16i1_v32i1(<32 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_v16i1_v32i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    vslidedown.vi v0, v0, 2
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
@@ -19,18 +19,19 @@ define {<16 x i1>, <16 x i1>} @vector_deinterleave_v16i1_v32i1(<32 x i1> %vec) {
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
-; CHECK-NEXT:    vmerge.vim v14, v10, 1, v0
-; CHECK-NEXT:    vadd.vi v8, v12, -16
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
+; CHECK-NEXT:    vadd.vi v10, v12, -16
 ; CHECK-NEXT:    vadd.vi v12, v12, -15
-; CHECK-NEXT:    vnsrl.wi v10, v14, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v13, v8, 0
+; CHECK-NEXT:    vnsrl.wi v8, v8, 8
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vrgather.vv v10, v11, v8, v0.t
-; CHECK-NEXT:    vnsrl.wi v8, v14, 8
-; CHECK-NEXT:    vmsne.vi v10, v10, 0
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; CHECK-NEXT:    vrgather.vv v13, v11, v10, v0.t
 ; CHECK-NEXT:    vrgather.vv v8, v11, v12, v0.t
+; CHECK-NEXT:    vmsne.vi v0, v13, 0
 ; CHECK-NEXT:    vmsne.vi v8, v8, 0
-; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
 %retval = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec)
 ret {<16 x i1>, <16 x i1>} %retval
@@ -95,26 +96,25 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
 ; CHECK-LABEL: vector_deinterleave_v4i64_v8i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vmv.v.i v14, 5
+; CHECK-NEXT:    vid.v v15
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v16, v8, 4
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vadd.vv v14, v12, v12
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v14
+; CHECK-NEXT:    vmv.v.i v18, 10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vcompress.vm v12, v8, v14
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v10, v14, -4
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v10, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v15, v14, 1
+; CHECK-NEXT:    vadd.vv v14, v15, v15
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v15
+; CHECK-NEXT:    vcompress.vm v10, v8, v18
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v8, v14, -3
+; CHECK-NEXT:    vadd.vi v8, v14, -4
+; CHECK-NEXT:    vadd.vi v9, v14, -3
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v16, v8, v0.t
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vrgatherei16.vv v10, v16, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec)
@@ -124,26 +124,27 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
 define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
 ; CHECK-LABEL: vector_deinterleave_v8i64_v16i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vmv.v.i v0, -16
+; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v24, v8, 8
+; CHECK-NEXT:    vmv.s.x v12, a0
+; CHECK-NEXT:    li a0, 170
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vadd.vv v20, v16, v16
+; CHECK-NEXT:    vmv.s.x v21, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v16, v8, v20
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v12, v20, -8
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v16, v24, v12, v0.t
+; CHECK-NEXT:    vcompress.vm v16, v8, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v21, v20, 1
+; CHECK-NEXT:    vadd.vi v22, v20, -8
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v21
+; CHECK-NEXT:    vcompress.vm v12, v8, v21
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vadd.vi v8, v20, -7
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v16, v24, v22, v0.t
 ; CHECK-NEXT:    vrgatherei16.vv v12, v24, v8, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -241,26 +242,25 @@ define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double
 ; CHECK-LABEL: vector_deinterleave_v4f64_v8f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vmv.v.i v14, 5
+; CHECK-NEXT:    vid.v v15
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v16, v8, 4
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v0, 12
-; CHECK-NEXT:    vadd.vv v14, v12, v12
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v14
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v10, v14, -4
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v10, v0.t
+; CHECK-NEXT:    vmv.v.i v18, 10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vcompress.vm v12, v8, v14
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v15, v14, 1
+; CHECK-NEXT:    vadd.vv v14, v15, v15
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v15
+; CHECK-NEXT:    vcompress.vm v10, v8, v18
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vadd.vi v8, v14, -3
+; CHECK-NEXT:    vadd.vi v8, v14, -4
+; CHECK-NEXT:    vadd.vi v9, v14, -3
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v16, v8, v0.t
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vrgatherei16.vv v10, v16, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 %retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index 39a1bfcda3d83..4338d1f61af72 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -106,95 +106,55 @@ define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_load_nxv8i6
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a2, 40
+; CHECK-NEXT:    li a2, 24
 ; CHECK-NEXT:    mul a1, a1, a2
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    vl8re64.v v16, (a0)
 ; CHECK-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vid.v v8
 ; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    vadd.vv v24, v8, v8
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 4
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vmseq.vi v24, v8, 0
 ; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vadd.vi v8, v24, 1
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vrgather.vv v8, v16, v24
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v24, v16, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmnot.m v6, v24
+; CHECK-NEXT:    vcompress.vm v8, v16, v24
+; CHECK-NEXT:    vmv1r.v v13, v24
+; CHECK-NEXT:    vcompress.vm v24, v16, v6
+; CHECK-NEXT:    vmv1r.v v12, v6
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v24, v16, v0
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vcompress.vm v0, v16, v13
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v16, v24, v0
+; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vcompress.vm v0, v16, v12
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v12, v16
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v28, v16
 ; CHECK-NEXT:    vmv8r.v v16, v24
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 40
+; CHECK-NEXT:    li a1, 24
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    .cfi_def_cfa sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index f20a90a422313..2291475cef014 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -73,12 +73,13 @@ define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vadd.vv v16, v12, v12
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
-; CHECK-NEXT:    vadd.vi v16, v16, 1
-; CHECK-NEXT:    vrgather.vv v20, v8, v16
+; CHECK-NEXT:    vand.vi v12, v12, 1
+; CHECK-NEXT:    vmseq.vi v16, v12, 0
+; CHECK-NEXT:    vcompress.vm v12, v8, v16
+; CHECK-NEXT:    vmnot.m v14, v16
+; CHECK-NEXT:    vcompress.vm v16, v8, v14
 ; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    vmv2r.v v10, v20
+; CHECK-NEXT:    vmv2r.v v10, v16
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %vec)
 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
@@ -89,12 +90,13 @@ define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vid.v v16
-; CHECK-NEXT:    vadd.vv v24, v16, v16
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
-; CHECK-NEXT:    vadd.vi v24, v24, 1
-; CHECK-NEXT:    vrgather.vv v0, v8, v24
+; CHECK-NEXT:    vand.vi v16, v16, 1
+; CHECK-NEXT:    vmseq.vi v24, v16, 0
+; CHECK-NEXT:    vcompress.vm v16, v8, v24
+; CHECK-NEXT:    vmnot.m v20, v24
+; CHECK-NEXT:    vcompress.vm v24, v8, v20
 ; CHECK-NEXT:    vmv4r.v v8, v16
-; CHECK-NEXT:    vmv4r.v v12, v0
+; CHECK-NEXT:    vmv4r.v v12, v24
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)
 ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
@@ -131,8 +133,8 @@ ret {<vscale x 64 x i1>, <vscale x 64 x i1>} %retval
 define {<vscale x 64 x i8>, <vscale x 64 x i8>} @vector_deinterleave_nxv64i8_nxv128i8(<vscale x 128 x i8> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv64i8_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vnsrl.wi v8, v24, 0
 ; CHECK-NEXT:    vnsrl.wi v0, v24, 8
 ; CHECK-NEXT:    vnsrl.wi v12, v16, 0
@@ -146,8 +148,8 @@ ret {<vscale x 64 x i8>, <vscale x 64 x i8>} %retval
 define {<vscale x 32 x i16>, <vscale x 32 x i16>} @vector_deinterleave_nxv32i16_nxv64i16(<vscale x 64 x i16> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv32i16_nxv64i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vnsrl.wi v8, v24, 0
 ; CHECK-NEXT:    vnsrl.wi v0, v24, 16
 ; CHECK-NEXT:    vnsrl.wi v12, v16, 0
@@ -161,9 +163,9 @@ ret {<vscale x 32 x i16>, <vscale x 32 x i16>} %retval
 define {<vscale x 16 x i32>, <vscale x 16 x i32>} @vector_deinterleave_nxv16i32_nxvv32i32(<vscale x 32 x i32> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv16i32_nxvv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v16
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wx v20, v24, a0
 ; CHECK-NEXT:    vnsrl.wx v16, v8, a0
 ; CHECK-NEXT:    vnsrl.wi v0, v8, 0
@@ -180,66 +182,50 @@ define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    sub sp, sp, a0
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv8r.v v24, v8
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vadd.vv v0, v8, v8
-; CHECK-NEXT:    vrgather.vv v8, v24, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 24
 ; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v16, v8, v0
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vadd.vi v8, v0, 1
-; CHECK-NEXT:    vrgather.vv v0, v24, v8
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v16, v24, v8
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vid.v v16
+; CHECK-NEXT:    vand.vi v24, v16, 1
+; CHECK-NEXT:    vmseq.vi v16, v24, 0
+; CHECK-NEXT:    vcompress.vm v24, v8, v16
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmnot.m v17, v16
+; CHECK-NEXT:    vcompress.vm v0, v8, v17
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vcompress.vm v24, v8, v16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vcompress.vm v24, v8, v17
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v20, v8
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmv4r.v v4, v8
+; CHECK-NEXT:    vmv4r.v v4, v24
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    .cfi_def_cfa sp, 16
 ; CHECK-NEXT:    addi sp, sp, 16
@@ -366,12 +352,13 @@ define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vadd.vv v16, v12, v12
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
-; CHECK-NEXT:    vadd.vi v16, v16, 1
-; CHECK-NEXT:    vrgather.vv v20, v8, v16
+; CHECK-NEXT:    vand.vi v12, v12, 1
+; CHECK-NEXT:    vmseq.vi v16, v12, 0
+; CHECK-NEXT:    vcompress.vm v12, v8, v16
+; CHECK-NEXT:    vmnot.m v14, v16
+; CHECK-NEXT:    vcompress.vm v16, v8, v14
 ; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    vmv2r.v v10, v20
+; CHECK-NEXT:    vmv2r.v v10, v16
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %vec)
 ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
@@ -387,8 +374,8 @@ declare {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave
 define {<vscale x 32 x bfloat>, <vscale x 32 x bfloat>} @vector_deinterleave_nxv32bf16_nxv64bf16(<vscale x 64 x bfloat> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv32bf16_nxv64bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vnsrl.wi v8, v24, 0
 ; CHECK-NEXT:    vnsrl.wi v0, v24, 16
 ; CHECK-NEXT:    vnsrl.wi v12, v16, 0
@@ -402,8 +389,8 @@ ret {<vscale x 32 x bfloat>, <vscale x 32 x bfloat>} %retval
 define {<vscale x 32 x half>, <vscale x 32 x half>} @vector_deinterleave_nxv32f16_nxv64f16(<vscale x 64 x half> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv32f16_nxv64f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vnsrl.wi v8, v24, 0
 ; CHECK-NEXT:    vnsrl.wi v0, v24, 16
 ; CHECK-NEXT:    vnsrl.wi v12, v16, 0
@@ -417,9 +404,9 @@ ret {<vscale x 32 x half>, <vscale x 32 x half>} %retval
 define {<vscale x 16 x float>, <vscale x 16 x float>} @vector_deinterleave_nxv16f32_nxv32f32(<vscale x 32 x float> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv16f32_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v16
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wx v20, v24, a0
 ; CHECK-NEXT:    vnsrl.wx v16, v8, a0
 ; CHECK-NEXT:    vnsrl.wi v0, v8, 0
@@ -436,66 +423,50 @@ define {<vscale x 8 x double>, <vscale x 8 x double>} @vector_deinterleave_nxv8f
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    sub sp, sp, a0
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv8r.v v24, v8
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vadd.vv v0, v8, v8
-; CHECK-NEXT:    vrgather.vv v8, v24, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 24
 ; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v16, v8, v0
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vadd.vi v8, v0, 1
-; CHECK-NEXT:    vrgather.vv v0, v24, v8
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vrgather.vv v16, v24, v8
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vid.v v16
+; CHECK-NEXT:    vand.vi v24, v16, 1
+; CHECK-NEXT:    vmseq.vi v16, v24, 0
+; CHECK-NEXT:    vcompress.vm v24, v8, v16
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmnot.m v17, v16
+; CHECK-NEXT:    vcompress.vm v0, v8, v17
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vcompress.vm v24, v8, v16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vcompress.vm v24, v8, v17
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v20, v8
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmv4r.v v4, v8
+; CHECK-NEXT:    vmv4r.v v4, v24
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    .cfi_def_cfa sp, 16
 ; CHECK-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
index 7b0ac01918b9b..08aa02c7e869a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
@@ -91,10 +91,10 @@ define <8 x i32> @vector_interleave_v8i32_v4i32(<4 x i32> %a, <4 x i32> %b) {
 define <4 x i64> @vector_interleave_v4i64_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: vector_interleave_v4i64_v2i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    lui a0, 12304
 ; CHECK-NEXT:    addi a0, a0, 512
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
 ; CHECK-NEXT:    vmv.s.x v10, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
@@ -106,10 +106,10 @@ define <4 x i64> @vector_interleave_v4i64_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; ZVBB-LABEL: vector_interleave_v4i64_v2i64:
 ; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; ZVBB-NEXT:    vmv1r.v v10, v9
 ; ZVBB-NEXT:    lui a0, 12304
 ; ZVBB-NEXT:    addi a0, a0, 512
-; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; ZVBB-NEXT:    vslideup.vi v8, v10, 2
 ; ZVBB-NEXT:    vmv.s.x v10, a0
 ; ZVBB-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
@@ -239,10 +239,10 @@ define <8 x float> @vector_interleave_v8f32_v4f32(<4 x float> %a, <4 x float> %b
 define <4 x double> @vector_interleave_v4f64_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: vector_interleave_v4f64_v2f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    lui a0, 12304
 ; CHECK-NEXT:    addi a0, a0, 512
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
 ; CHECK-NEXT:    vmv.s.x v10, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
@@ -254,10 +254,10 @@ define <4 x double> @vector_interleave_v4f64_v2f64(<2 x double> %a, <2 x double>
 ;
 ; ZVBB-LABEL: vector_interleave_v4f64_v2f64:
 ; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; ZVBB-NEXT:    vmv1r.v v10, v9
 ; ZVBB-NEXT:    lui a0, 12304
 ; ZVBB-NEXT:    addi a0, a0, 512
-; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; ZVBB-NEXT:    vslideup.vi v8, v10, 2
 ; ZVBB-NEXT:    vmv.s.x v10, a0
 ; ZVBB-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index bc203e215d878..9b78f31d399d9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -9,9 +9,9 @@
 define void @vector_interleave_store_nxv32i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, ptr %p) {
 ; CHECK-LABEL: vector_interleave_store_nxv32i1_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    li a1, -1
 ; CHECK-NEXT:    csrr a2, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index 26e9afcb1d109..864acb320d8fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -11,9 +11,9 @@
 define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
 ; CHECK-LABEL: vector_interleave_nxv32i1_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    csrr a1, vlenb
@@ -32,9 +32,9 @@ define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1>
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32i1_nxv16i1:
 ; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, mu
 ; ZVBB-NEXT:    vmv1r.v v9, v0
 ; ZVBB-NEXT:    vmv1r.v v0, v8
-; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, mu
 ; ZVBB-NEXT:    vmv.v.i v10, 0
 ; ZVBB-NEXT:    li a0, 1
 ; ZVBB-NEXT:    csrr a1, vlenb
@@ -160,9 +160,9 @@ declare <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64>,
 define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) {
 ; CHECK-LABEL: vector_interleave_nxv128i1_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v24, 0
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vmerge.vim v16, v24, 1, v0
@@ -203,8 +203,8 @@ define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv64i1(<vscale x 64 x i1
 define <vscale x 128 x i8> @vector_interleave_nxv128i8_nxv64i8(<vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {
 ; CHECK-LABEL: vector_interleave_nxv128i8_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -215,8 +215,8 @@ define <vscale x 128 x i8> @vector_interleave_nxv128i8_nxv64i8(<vscale x 64 x i8
 ;
 ; ZVBB-LABEL: vector_interleave_nxv128i8_nxv64i8:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vwsll.vi v8, v16, 8
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 8
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
@@ -230,8 +230,8 @@ define <vscale x 128 x i8> @vector_interleave_nxv128i8_nxv64i8(<vscale x 64 x i8
 define <vscale x 64 x i16> @vector_interleave_nxv64i16_nxv32i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b) {
 ; CHECK-LABEL: vector_interleave_nxv64i16_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -242,8 +242,8 @@ define <vscale x 64 x i16> @vector_interleave_nxv64i16_nxv32i16(<vscale x 32 x i
 ;
 ; ZVBB-LABEL: vector_interleave_nxv64i16_nxv32i16:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vwsll.vi v8, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
@@ -257,8 +257,8 @@ define <vscale x 64 x i16> @vector_interleave_nxv64i16_nxv32i16(<vscale x 32 x i
 define <vscale x 32 x i32> @vector_interleave_nxv32i32_nxv16i32(<vscale x 16 x i32> %a, <vscale x 16 x i32> %b) {
 ; CHECK-LABEL: vector_interleave_nxv32i32_nxv16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -269,9 +269,9 @@ define <vscale x 32 x i32> @vector_interleave_nxv32i32_nxv16i32(<vscale x 16 x i
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32i32_nxv16i32:
 ; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    li a0, 32
-; ZVBB-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; ZVBB-NEXT:    vwsll.vx v8, v16, a0
 ; ZVBB-NEXT:    vwsll.vx v0, v20, a0
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
@@ -575,8 +575,8 @@ declare <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x dou
 define <vscale x 64 x bfloat> @vector_interleave_nxv64bf16_nxv32bf16(<vscale x 32 x bfloat> %a, <vscale x 32 x bfloat> %b) {
 ; CHECK-LABEL: vector_interleave_nxv64bf16_nxv32bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -587,8 +587,8 @@ define <vscale x 64 x bfloat> @vector_interleave_nxv64bf16_nxv32bf16(<vscale x 3
 ;
 ; ZVBB-LABEL: vector_interleave_nxv64bf16_nxv32bf16:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vwsll.vi v8, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
@@ -602,8 +602,8 @@ define <vscale x 64 x bfloat> @vector_interleave_nxv64bf16_nxv32bf16(<vscale x 3
 define <vscale x 64 x half> @vector_interleave_nxv64f16_nxv32f16(<vscale x 32 x half> %a, <vscale x 32 x half> %b) {
 ; CHECK-LABEL: vector_interleave_nxv64f16_nxv32f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -614,8 +614,8 @@ define <vscale x 64 x half> @vector_interleave_nxv64f16_nxv32f16(<vscale x 32 x
 ;
 ; ZVBB-LABEL: vector_interleave_nxv64f16_nxv32f16:
 ; ZVBB:       # %bb.0:
-; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    vwsll.vi v8, v16, 16
 ; ZVBB-NEXT:    vwsll.vi v0, v20, 16
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
@@ -629,8 +629,8 @@ define <vscale x 64 x half> @vector_interleave_nxv64f16_nxv32f16(<vscale x 32 x
 define <vscale x 32 x float> @vector_interleave_nxv32f32_nxv16f32(<vscale x 16 x float> %a, <vscale x 16 x float> %b) {
 ; CHECK-LABEL: vector_interleave_nxv32f32_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv8r.v v24, v8
 ; CHECK-NEXT:    vwaddu.vv v8, v24, v16
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwaddu.vv v0, v28, v20
@@ -641,9 +641,9 @@ define <vscale x 32 x float> @vector_interleave_nxv32f32_nxv16f32(<vscale x 16 x
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32f32_nxv16f32:
 ; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; ZVBB-NEXT:    vmv8r.v v24, v8
 ; ZVBB-NEXT:    li a0, 32
-; ZVBB-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; ZVBB-NEXT:    vwsll.vx v8, v16, a0
 ; ZVBB-NEXT:    vwsll.vx v0, v20, a0
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v24
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll b/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
index 79bd60d1702f3..6435c1c14e061 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
@@ -120,8 +120,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_passthru(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: vadd_vv_passthru:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
+; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vadd.vv v10, v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v8
@@ -152,8 +152,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_passthru_negative(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: vadd_vv_passthru_negative:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
+; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vadd.vv v10, v8, v9
 ; CHECK-NEXT:    vadd.vv v9, v8, v10
 ; CHECK-NEXT:    vadd.vv v8, v8, v9
@@ -183,8 +183,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_mask(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %m) nounwind {
 ; CHECK-LABEL: vadd_vv_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
+; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vadd.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v8, v0.t
@@ -218,8 +218,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_mask_negative(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %m, <vscale x 1 x i1> %m2) nounwind {
 ; CHECK-LABEL: vadd_vv_mask_negative:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
+; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vadd.vv v11, v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v11, v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll b/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
index 6a72043ca7e8e..90d798b167cfc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
@@ -11,9 +11,9 @@ declare <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1>, <vscale
 define <vscale x 1 x i1> @splice_nxv1i1_offset_negone(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv1i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -33,9 +33,9 @@ define <vscale x 1 x i1> @splice_nxv1i1_offset_negone(<vscale x 1 x i1> %a, <vsc
 define <vscale x 1 x i1> @splice_nxv1i1_offset_max(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv1i1_offset_max:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -59,9 +59,9 @@ declare <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1>, <vscale
 define <vscale x 2 x i1> @splice_nxv2i1_offset_negone(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -81,9 +81,9 @@ define <vscale x 2 x i1> @splice_nxv2i1_offset_negone(<vscale x 2 x i1> %a, <vsc
 define <vscale x 2 x i1> @splice_nxv2i1_offset_max(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i1_offset_max:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -107,9 +107,9 @@ declare <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1>, <vscale
 define <vscale x 4 x i1> @splice_nxv4i1_offset_negone(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv4i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -129,9 +129,9 @@ define <vscale x 4 x i1> @splice_nxv4i1_offset_negone(<vscale x 4 x i1> %a, <vsc
 define <vscale x 4 x i1> @splice_nxv4i1_offset_max(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv4i1_offset_max:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -155,9 +155,9 @@ declare <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1>, <vscale
 define <vscale x 8 x i1> @splice_nxv8i1_offset_negone(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv8i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -176,9 +176,9 @@ define <vscale x 8 x i1> @splice_nxv8i1_offset_negone(<vscale x 8 x i1> %a, <vsc
 define <vscale x 8 x i1> @splice_nxv8i1_offset_max(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv8i1_offset_max:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v10, v8, 1, v0
@@ -201,9 +201,9 @@ declare <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1>, <vsca
 define <vscale x 16 x i1> @splice_nxv16i1_offset_negone(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv16i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
@@ -223,9 +223,9 @@ define <vscale x 16 x i1> @splice_nxv16i1_offset_negone(<vscale x 16 x i1> %a, <
 define <vscale x 16 x i1> @splice_nxv16i1_offset_max(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv16i1_offset_max:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
@@ -249,9 +249,9 @@ declare <vscale x 32 x i1> @llvm.vector.splice.nxv32i1(<vscale x 32 x i1>, <vsca
 define <vscale x 32 x i1> @splice_nxv32i1_offset_negone(<vscale x 32 x i1> %a, <vscale x 32 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv32i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
@@ -296,9 +296,9 @@ declare <vscale x 64 x i1> @llvm.vector.splice.nxv64i1(<vscale x 64 x i1>, <vsca
 define <vscale x 64 x i1> @splice_nxv64i1_offset_negone(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv64i1_offset_negone:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v24, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    vmerge.vim v16, v24, 1, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
index 2c92a5da8eecb..8f9f9c4256c8f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfabs-vp.ll
@@ -462,11 +462,11 @@ declare <vscale x 16 x double> @llvm.vp.fabs.nxv16f64(<vscale x 16 x double>, <v
 define <vscale x 16 x double> @vfabs_vv_nxv16f64(<vscale x 16 x double> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfabs_vv_nxv16f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
index 1953cfd2a0169..87bc9f27d6dc9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd-vp.ll
@@ -411,11 +411,11 @@ define <vscale x 32 x bfloat> @vfadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
@@ -518,12 +518,12 @@ define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bf
 ; CHECK-NEXT:    add a1, a2, a1
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v20
 ; CHECK-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.x v16, a1
@@ -604,10 +604,10 @@ define <vscale x 32 x bfloat> @vfadd_vf_nxv32bf16_unmasked(<vscale x 32 x bfloat
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmset.m v7
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
@@ -1205,11 +1205,11 @@ define <vscale x 32 x half> @vfadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
@@ -1324,12 +1324,12 @@ define <vscale x 32 x half> @vfadd_vf_nxv32f16(<vscale x 32 x half> %va, half %b
 ; ZVFHMIN-NEXT:    add a1, a2, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
 ; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
@@ -1416,10 +1416,10 @@ define <vscale x 32 x half> @vfadd_vf_nxv32f16_unmasked(<vscale x 32 x half> %va
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v7
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
index ccd286b7ee5fd..061af454aa8ba 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv-vp.ll
@@ -373,11 +373,11 @@ define <vscale x 32 x bfloat> @vfdiv_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
@@ -480,12 +480,12 @@ define <vscale x 32 x bfloat> @vfdiv_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bf
 ; CHECK-NEXT:    add a1, a2, a1
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v20
 ; CHECK-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.x v16, a1
@@ -566,10 +566,10 @@ define <vscale x 32 x bfloat> @vfdiv_vf_nxv32bf16_unmasked(<vscale x 32 x bfloat
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmset.m v7
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
@@ -1117,11 +1117,11 @@ define <vscale x 32 x half> @vfdiv_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
@@ -1236,12 +1236,12 @@ define <vscale x 32 x half> @vfdiv_vf_nxv32f16(<vscale x 32 x half> %va, half %b
 ; ZVFHMIN-NEXT:    add a1, a2, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
 ; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
@@ -1328,10 +1328,10 @@ define <vscale x 32 x half> @vfdiv_vf_nxv32f16_unmasked(<vscale x 32 x half> %va
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v7
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfirst.ll b/llvm/test/CodeGen/RISCV/rvv/vfirst.ll
index eafd605c6110e..c510121ee3ebe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfirst.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfirst.ll
@@ -43,9 +43,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv1i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -97,9 +97,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv2i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv2i1(<vscale x 2 x i1> %0, <vscale x 2 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv2i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -137,9 +137,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv4i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv4i1(<vscale x 4 x i1> %0, <vscale x 4 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv4i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -177,9 +177,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv8i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv8i1(<vscale x 8 x i1> %0, <vscale x 8 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv8i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -217,9 +217,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv16i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv16i1(<vscale x 16 x i1> %0, <vscale x 16 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv16i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -257,9 +257,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv32i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv32i1(<vscale x 32 x i1> %0, <vscale x 32 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv32i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -297,9 +297,9 @@ declare iXLen @llvm.riscv.vfirst.mask.iXLen.nxv64i1(
 define iXLen @intrinsic_vfirst_mask_m_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
index fd518d9be786d..7ca1983e8b32c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
@@ -628,6 +628,7 @@ define <vscale x 32 x bfloat> @vfma_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vs
 ; CHECK-NEXT:    add a2, a2, a3
 ; CHECK-NEXT:    sub sp, sp, a2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vl8re16.v v0, (a0)
 ; CHECK-NEXT:    csrr a2, vlenb
@@ -639,7 +640,6 @@ define <vscale x 32 x bfloat> @vfma_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vs
 ; CHECK-NEXT:    add a4, sp, a4
 ; CHECK-NEXT:    addi a4, a4, 16
 ; CHECK-NEXT:    vs1r.v v24, (a4) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v24, v24, a2
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
@@ -2193,6 +2193,7 @@ define <vscale x 32 x half> @vfma_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    add a2, a2, a3
 ; ZVFHMIN-NEXT:    sub sp, sp, a2
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v24, v0
 ; ZVFHMIN-NEXT:    vl8re16.v v0, (a0)
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
@@ -2204,7 +2205,6 @@ define <vscale x 32 x half> @vfma_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    add a4, sp, a4
 ; ZVFHMIN-NEXT:    addi a4, a4, 16
 ; ZVFHMIN-NEXT:    vs1r.v v24, (a4) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v24, v24, a2
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    slli a2, a2, 3
@@ -3662,6 +3662,7 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    add a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -3708,7 +3709,6 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    and a0, a7, a6
 ; CHECK-NEXT:    csrr a2, vlenb
@@ -7796,9 +7796,9 @@ define <vscale x 16 x half> @vfnmadd_vv_nxv16f16(<vscale x 16 x half> %va, <vsca
 ;
 ; ZVFHMIN-LABEL: vfnmadd_vv_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv4r.v v4, v8
 ; ZVFHMIN-NEXT:    lui a1, 8
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v12, v12, a1, v0.t
 ; ZVFHMIN-NEXT:    vxor.vx v16, v16, a1, v0.t
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
@@ -8152,9 +8152,9 @@ define <vscale x 16 x half> @vfnmadd_vf_nxv16f16_neg_splat_commute(<vscale x 16
 ;
 ; ZVFHMIN-LABEL: vfnmadd_vf_nxv16f16_neg_splat_commute:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv4r.v v4, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
 ; ZVFHMIN-NEXT:    lui a1, 8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
@@ -8253,9 +8253,9 @@ define <vscale x 16 x half> @vfnmsub_vv_nxv16f16(<vscale x 16 x half> %va, <vsca
 ;
 ; ZVFHMIN-LABEL: vfnmsub_vv_nxv16f16:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv4r.v v4, v8
 ; ZVFHMIN-NEXT:    lui a1, 8
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v12, v12, a1, v0.t
 ; ZVFHMIN-NEXT:    vxor.vx v16, v16, a1, v0.t
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
@@ -8553,9 +8553,9 @@ define <vscale x 16 x half> @vfnmsub_vf_nxv16f16_neg_splat(<vscale x 16 x half>
 ;
 ; ZVFHMIN-LABEL: vfnmsub_vf_nxv16f16_neg_splat:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv4r.v v4, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
-; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
 ; ZVFHMIN-NEXT:    lui a1, 8
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
@@ -8712,11 +8712,11 @@ define <vscale x 32 x half> @vfmsub_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    add a2, sp, a2
 ; ZVFHMIN-NEXT:    addi a2, a2, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v8
 ; ZVFHMIN-NEXT:    vl8re16.v v8, (a0)
 ; ZVFHMIN-NEXT:    lui a2, 8
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v8, v8, a2, v0.t
 ; ZVFHMIN-NEXT:    slli a2, a0, 1
 ; ZVFHMIN-NEXT:    mv a3, a1
@@ -9277,10 +9277,10 @@ define <vscale x 32 x half> @vfmsub_vf_nxv32f16_unmasked(<vscale x 32 x half> %v
 ; ZVFHMIN-NEXT:    add a1, a1, a2
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
 ; ZVFHMIN-NEXT:    lui a1, 8
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v7
 ; ZVFHMIN-NEXT:    csrr a3, vlenb
 ; ZVFHMIN-NEXT:    csrr a4, vlenb
@@ -10001,11 +10001,11 @@ define <vscale x 32 x half> @vfnmadd_vv_nxv32f16_unmasked_commuted(<vscale x 32
 ; ZVFHMIN-NEXT:    add a2, a2, a3
 ; ZVFHMIN-NEXT:    sub sp, sp, a2
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    vl8re16.v v24, (a0)
 ; ZVFHMIN-NEXT:    lui a2, 8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v8
 ; ZVFHMIN-NEXT:    csrr a3, vlenb
 ; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
@@ -11154,11 +11154,11 @@ define <vscale x 32 x half> @vfnmadd_vf_nxv32f16_neg_splat_unmasked_commute(<vsc
 ; ZVFHMIN-NEXT:    add a1, a1, a2
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    lui a2, 8
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v8
 ; ZVFHMIN-NEXT:    csrr a3, vlenb
 ; ZVFHMIN-NEXT:    vmv.v.x v24, a1
@@ -11766,11 +11766,11 @@ define <vscale x 32 x half> @vfnmsub_vv_nxv32f16_unmasked_commuted(<vscale x 32
 ; ZVFHMIN-NEXT:    add a2, a2, a3
 ; ZVFHMIN-NEXT:    sub sp, sp, a2
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x29, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 41 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    vl8re16.v v24, (a0)
 ; ZVFHMIN-NEXT:    lui a2, 8
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v8
 ; ZVFHMIN-NEXT:    csrr a3, vlenb
 ; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
@@ -11924,11 +11924,11 @@ define <vscale x 32 x half> @vfnmsub_vf_nxv32f16(<vscale x 32 x half> %va, half
 ; ZVFHMIN-NEXT:    add a1, a1, a2
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
 ; ZVFHMIN-NEXT:    lui a3, 8
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
-; ZVFHMIN-NEXT:    vsetvli a4, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a2
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    slli a2, a2, 5
@@ -12075,11 +12075,11 @@ define <vscale x 32 x half> @vfnmsub_vf_nxv32f16_commute(<vscale x 32 x half> %v
 ; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
 ; ZVFHMIN-NEXT:    lui a3, 8
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
-; ZVFHMIN-NEXT:    vsetvli a4, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a2
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    slli a2, a2, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
index 1d471ab2404b1..a4f3a7d3a09a8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd-constrained-sdnode.ll
@@ -227,6 +227,7 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; CHECK-NEXT:    slli a1, a1, 5
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v0, v16
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -237,7 +238,6 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 24
@@ -314,6 +314,7 @@ define <vscale x 32 x bfloat> @vfmadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v24, v16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -321,7 +322,6 @@ define <vscale x 32 x bfloat> @vfmadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    fmv.x.h a0, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -664,6 +664,7 @@ define <vscale x 32 x half> @vfmadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -674,7 +675,6 @@ define <vscale x 32 x half> @vfmadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    li a1, 24
@@ -757,6 +757,7 @@ define <vscale x 32 x half> @vfmadd_vf_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    mul a0, a0, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
@@ -764,7 +765,6 @@ define <vscale x 32 x half> @vfmadd_vf_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
index 88fd81a5a2f7b..109023032e6ab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd-sdnode.ll
@@ -226,6 +226,7 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFH-NEXT:    slli a1, a1, 5
 ; ZVFH-NEXT:    sub sp, sp, a1
 ; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFH-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv8r.v v0, v16
 ; ZVFH-NEXT:    addi a1, sp, 16
 ; ZVFH-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -236,7 +237,6 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFH-NEXT:    add a0, sp, a0
 ; ZVFH-NEXT:    addi a0, a0, 16
 ; ZVFH-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v24, v16
 ; ZVFH-NEXT:    csrr a0, vlenb
 ; ZVFH-NEXT:    slli a0, a0, 3
@@ -316,6 +316,7 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -326,7 +327,6 @@ define <vscale x 32 x bfloat> @vfmadd_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    li a1, 24
@@ -402,12 +402,12 @@ define <vscale x 32 x bfloat> @vfmadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFH-NEXT:    slli a0, a0, 5
 ; ZVFH-NEXT:    sub sp, sp, a0
 ; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv8r.v v0, v16
 ; ZVFH-NEXT:    addi a0, sp, 16
 ; ZVFH-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFH-NEXT:    vmv8r.v v16, v8
 ; ZVFH-NEXT:    fmv.x.h a0, fa0
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vfwcvtbf16.f.f.v v24, v16
 ; ZVFH-NEXT:    csrr a1, vlenb
 ; ZVFH-NEXT:    slli a1, a1, 4
@@ -498,12 +498,12 @@ define <vscale x 32 x bfloat> @vfmadd_vf_nxv32bf16(<vscale x 32 x bfloat> %va, <
 ; ZVFHMIN-NEXT:    slli a0, a0, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    addi a0, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvtbf16.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
@@ -875,6 +875,7 @@ define <vscale x 32 x half> @vfmadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -885,7 +886,6 @@ define <vscale x 32 x half> @vfmadd_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    li a1, 24
@@ -967,12 +967,12 @@ define <vscale x 32 x half> @vfmadd_vf_nxv32f16(<vscale x 32 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    slli a0, a0, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a0
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    addi a0, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
index dafcf8a1410d3..02d6229e99248 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmax-vp.ll
@@ -183,11 +183,11 @@ define <vscale x 32 x bfloat> @vfmax_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
@@ -516,11 +516,11 @@ define <vscale x 32 x half> @vfmax_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
index b3df6572f7936..f7f80299785d4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmin-vp.ll
@@ -183,11 +183,11 @@ define <vscale x 32 x bfloat> @vfmin_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
@@ -516,11 +516,11 @@ define <vscale x 32 x half> @vfmin_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
index f4a236df4c9e4..7e5523044a010 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul-vp.ll
@@ -495,11 +495,11 @@ define <vscale x 32 x half> @vfmul_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
@@ -614,12 +614,12 @@ define <vscale x 32 x half> @vfmul_vf_nxv32f16(<vscale x 32 x half> %va, half %b
 ; ZVFHMIN-NEXT:    add a1, a2, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
 ; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
@@ -706,10 +706,10 @@ define <vscale x 32 x half> @vfmul_vf_nxv32f16_unmasked(<vscale x 32 x half> %va
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v7
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
index d1702268f829f..901f3cd63fa9e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
@@ -1112,6 +1112,7 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    li a3, 24
@@ -1154,7 +1155,6 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    and a0, a7, a6
 ; CHECK-NEXT:    csrr a2, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
index 343098e87649e..bbab056f0ff46 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfneg-vp.ll
@@ -450,11 +450,11 @@ declare <vscale x 16 x double> @llvm.vp.fneg.nxv16f64(<vscale x 16 x double>, <v
 define <vscale x 16 x double> @vfneg_vv_nxv16f64(<vscale x 16 x double> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfneg_vv_nxv16f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
index 3705e73fda492..b8ec285b5c34e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmadd-constrained-sdnode.ll
@@ -329,6 +329,7 @@ define <vscale x 32 x half> @vfnmsub_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    li a2, 24
@@ -338,7 +339,6 @@ define <vscale x 32 x half> @vfnmsub_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vl8re16.v v16, (a0)
 ; ZVFHMIN-NEXT:    lui a0, 8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v8, v0, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
index 80edf0e3a4d81..742083f6f684b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsub-constrained-sdnode.ll
@@ -310,6 +310,7 @@ define <vscale x 32 x half> @vfnmsub_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    mul a1, a1, a2
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
@@ -323,7 +324,6 @@ define <vscale x 32 x half> @vfnmsub_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    vl8re16.v v24, (a0)
 ; ZVFHMIN-NEXT:    lui a0, 8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v0
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vxor.vx v0, v24, a0
@@ -389,6 +389,7 @@ define <vscale x 32 x half> @vfnmsub_vf_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v8
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 4
@@ -396,7 +397,6 @@ define <vscale x 32 x half> @vfnmsub_vf_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v24, a0
 ; ZVFHMIN-NEXT:    lui a0, 8
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
index 341fe678183b6..6debc47460683 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
@@ -96,11 +96,11 @@ declare <vscale x 32 x float> @llvm.vp.fpext.nxv32f32.nxv32f16(<vscale x 32 x ha
 define <vscale x 32 x float> @vfpext_nxv32f16_nxv32f32(<vscale x 32 x half> %a, <vscale x 32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vfpext_nxv32f16_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
index cf195c7c0935e..d990c74c67d5a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptosi-vp.ll
@@ -508,11 +508,11 @@ declare <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f32(<vscale x 32 x flo
 define <vscale x 32 x i16> @vfptosi_nxv32i16_nxv32f32(<vscale x 32 x float> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptosi_nxv32i16_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -538,11 +538,11 @@ declare <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f32(<vscale x 32 x flo
 define <vscale x 32 x i32> @vfptosi_nxv32i32_nxv32f32(<vscale x 32 x float> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptosi_nxv32i32_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
index 952d28604b86c..3b24a648d97f5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoui-vp.ll
@@ -508,11 +508,11 @@ declare <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f32(<vscale x 32 x flo
 define <vscale x 32 x i16> @vfptoui_nxv32i16_nxv32f32(<vscale x 32 x float> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptoui_nxv32i16_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -538,11 +538,11 @@ declare <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f32(<vscale x 32 x flo
 define <vscale x 32 x i32> @vfptoui_nxv32i32_nxv32f32(<vscale x 32 x float> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfptoui_nxv32i32_nxv32f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
index 874813f057595..63156e1399293 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
@@ -102,13 +102,13 @@ define <vscale x 16 x float> @vfptrunc_nxv16f32_nxv16f64(<vscale x 16 x double>
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
@@ -147,6 +147,7 @@ define <vscale x 32 x float> @vfptrunc_nxv32f32_nxv32f64(<vscale x 32 x double>
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -160,7 +161,6 @@ define <vscale x 32 x float> @vfptrunc_nxv32f32_nxv32f64(<vscale x 32 x double>
 ; CHECK-NEXT:    srli a5, a1, 2
 ; CHECK-NEXT:    slli a6, a1, 3
 ; CHECK-NEXT:    slli a4, a1, 1
-; CHECK-NEXT:    vsetvli a7, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v16, v0, a5
 ; CHECK-NEXT:    add a6, a0, a6
 ; CHECK-NEXT:    sub a5, a2, a4
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
index 8edcf23988c7f..8e57be1e0697c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt-vp.ll
@@ -167,12 +167,12 @@ declare <vscale x 32 x bfloat> @llvm.vp.sqrt.nxv32bf16(<vscale x 32 x bfloat>, <
 define <vscale x 32 x bfloat> @vfsqrt_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfsqrt_vv_nxv32bf16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v16, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
@@ -452,12 +452,12 @@ define <vscale x 32 x half> @vfsqrt_vv_nxv32f16(<vscale x 32 x half> %va, <vscal
 ;
 ; ZVFHMIN-LABEL: vfsqrt_vv_nxv32f16:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v16, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
 ; ZVFHMIN-NEXT:    sub a3, a0, a1
-; ZVFHMIN-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v0, v0, a2
 ; ZVFHMIN-NEXT:    sltu a2, a0, a3
 ; ZVFHMIN-NEXT:    addi a2, a2, -1
@@ -749,11 +749,11 @@ declare <vscale x 16 x double> @llvm.vp.sqrt.nxv16f64(<vscale x 16 x double>, <v
 define <vscale x 16 x double> @vfsqrt_vv_nxv16f64(<vscale x 16 x double> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vfsqrt_vv_nxv16f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
index 25a80e66c4a52..d034f65479a15 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsub-vp.ll
@@ -373,11 +373,11 @@ define <vscale x 32 x bfloat> @vfsub_vv_nxv32bf16(<vscale x 32 x bfloat> %va, <v
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
 ; CHECK-NEXT:    slli a1, a2, 1
 ; CHECK-NEXT:    srli a2, a2, 2
@@ -480,12 +480,12 @@ define <vscale x 32 x bfloat> @vfsub_vf_nxv32bf16(<vscale x 32 x bfloat> %va, bf
 ; CHECK-NEXT:    add a1, a2, a1
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v8, v20
 ; CHECK-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.x v16, a1
@@ -566,10 +566,10 @@ define <vscale x 32 x bfloat> @vfsub_vf_nxv32bf16_unmasked(<vscale x 32 x bfloat
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    fmv.x.h a1, fa0
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmset.m v7
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
@@ -1117,11 +1117,11 @@ define <vscale x 32 x half> @vfsub_vv_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    srli a2, a2, 2
@@ -1236,12 +1236,12 @@ define <vscale x 32 x half> @vfsub_vf_nxv32f16(<vscale x 32 x half> %va, half %b
 ; ZVFHMIN-NEXT:    add a1, a2, a1
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
 ; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv.v.x v16, a1
@@ -1328,10 +1328,10 @@ define <vscale x 32 x half> @vfsub_vf_nxv32f16_unmasked(<vscale x 32 x half> %va
 ; ZVFHMIN-NEXT:    slli a1, a1, 4
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v16, v8
 ; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmset.m v7
 ; ZVFHMIN-NEXT:    addi a3, sp, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index 11f603b56b6e5..0215e6a80d09a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -72,17 +72,17 @@ define <vscale x 4 x i32> @vsub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vsub.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vsub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vsub_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vsub.vv v8, v8, v10
-; VLOPT-NEXT:    vsub.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -92,17 +92,17 @@ define <vscale x 4 x i32> @vsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vsub.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vsub.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vsub_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vsub.vx v10, v8, a0
-; VLOPT-NEXT:    vsub.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -132,17 +132,257 @@ define <vscale x 4 x i32> @vrsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vrsub.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vsub.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vrsub_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vrsub.vx v10, v8, a0
-; VLOPT-NEXT:    vsub.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vrsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vand_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vand_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vand.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vand_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vand.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vand.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vand_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vand_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vand.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vand_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vand.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vand.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vand_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vand_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vand.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vand_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vand.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vand.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vor_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vor_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vor.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vor_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vor.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vor.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vor_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vor_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vor.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vor_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vor.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vor.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vor_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vor_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vor.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vor_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vor.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vor.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vxor_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vxor_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vxor.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vxor_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vxor.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vxor.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vxor_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vxor_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vxor.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vxor_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vxor.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vxor.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vxor_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vxor_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vxor.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vxor_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vxor.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vxor.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsll_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsll_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsll.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsll_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsll.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsll_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsll_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsll.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsll_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsll.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsll_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsll_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsll.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsll_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsll.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -167,6 +407,126 @@ define <vscale x 4 x i64> @vwaddu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
   ret <vscale x 4 x i64> %2
 }
 
+define <vscale x 4 x i32> @vsrl_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsrl_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsrl.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsrl_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsrl.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsrl_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsrl_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsrl.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsrl_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsrl.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsrl_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsrl_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsrl.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsrl_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsrl.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsra_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsra_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsra.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsra_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsra.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsra_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsra_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsra.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsra_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsra.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsra_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsra_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsra.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsra_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsra.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
 define <vscale x 4 x i64> @vwaddu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-LABEL: vwaddu_vx:
 ; NOVLOPT:       # %bb.0:
@@ -602,26 +962,6 @@ define <vscale x 4 x i64> @vzext_vf8(<vscale x 4 x i8> %a, <vscale x 4 x i64> %b
   ret <vscale x 4 x i64> %2
 }
 
-define <vscale x 4 x i32> @vsll_vi(<vscale x 4 x i32> %a, iXLen %vl) {
-; NOVLOPT-LABEL: vsll_vi:
-; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vsll.vi v10, v8, 5
-; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vsll.vv v8, v10, v8
-; NOVLOPT-NEXT:    ret
-;
-; VLOPT-LABEL: vsll_vi:
-; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; VLOPT-NEXT:    vsll.vi v10, v8, 5
-; VLOPT-NEXT:    vsll.vv v8, v10, v8
-; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
-  ret <vscale x 4 x i32> %2
-}
-
 define <vscale x 4 x i16> @vnsrl_wi(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, iXLen %vl) {
 ; NOVLOPT-LABEL: vnsrl_wi:
 ; NOVLOPT:       # %bb.0:
@@ -808,17 +1148,17 @@ define <vscale x 4 x i32> @vmul_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmul_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmul.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -828,17 +1168,17 @@ define <vscale x 4 x i32> @vmul_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmul.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmul_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmul.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -848,17 +1188,17 @@ define <vscale x 4 x i32> @vmulh_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulh.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulh_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulh.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulh.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -868,17 +1208,17 @@ define <vscale x 4 x i32> @vmulh_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulh.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulh_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulh.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulh.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -888,17 +1228,17 @@ define <vscale x 4 x i32> @vmulhu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulhu.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulhu_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulhu.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -908,17 +1248,17 @@ define <vscale x 4 x i32> @vmulhu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulhu.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulhu_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulhu.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -928,17 +1268,17 @@ define <vscale x 4 x i32> @vmulhsu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32>
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulhsu.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulhsu_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulhsu.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhsu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -948,17 +1288,17 @@ define <vscale x 4 x i32> @vmulhsu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl)
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vmulhsu.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vmulhsu_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vmulhsu.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhsu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -968,17 +1308,17 @@ define <vscale x 4 x i32> @vdivu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vdivu.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vdivu_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vdivu.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vdivu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -988,17 +1328,17 @@ define <vscale x 4 x i32> @vdivu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vdivu.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vdivu_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vdivu.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vdivu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1008,17 +1348,17 @@ define <vscale x 4 x i32> @vdiv_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vdiv.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vdiv_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vdiv.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vdiv.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1028,17 +1368,17 @@ define <vscale x 4 x i32> @vdiv_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vdiv.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vdiv_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vdiv.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vdiv.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1048,17 +1388,17 @@ define <vscale x 4 x i32> @vremu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vremu.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vremu_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vremu.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vremu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1068,17 +1408,17 @@ define <vscale x 4 x i32> @vremu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vremu.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vremu_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vremu.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vremu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1088,17 +1428,17 @@ define <vscale x 4 x i32> @vrem_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vrem.vv v8, v8, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vrem_vv:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; VLOPT-NEXT:    vrem.vv v8, v8, v10
-; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vrem.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1108,17 +1448,17 @@ define <vscale x 4 x i32> @vrem_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
 ; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; NOVLOPT-NEXT:    vrem.vx v10, v8, a0
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vrem_vx:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; VLOPT-NEXT:    vrem.vx v10, v8, a0
-; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
 ; VLOPT-NEXT:    ret
   %1 = call <vscale x 4 x i32> @llvm.riscv.vrem.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -1248,44 +1588,325 @@ define <vscale x 4 x i64> @vwmulu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
   ret <vscale x 4 x i64> %2
 }
 
-define <vscale x 4 x i32> @vwmacc_vx(<vscale x 4 x i16> %a, i16 %b, iXLen %vl) {
+define <vscale x 4 x i32> @vwmacc_vv(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, <vscale x 4 x i32> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vwmacc_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmacc.vv v8, v10, v11
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmacc_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmacc.vv v8, v10, v11
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.nxv4i16(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %d, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmacc_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmacc_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmacc.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmacc_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vmacc.vv v8, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmacc.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmacc_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmacc_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmv2r.v v10, v8
+; NOVLOPT-NEXT:    vmacc.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmacc_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; VLOPT-NEXT:    vmv2r.v v10, v8
+; VLOPT-NEXT:    vmacc.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmacc.nxv4i32.i32(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i32> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmadd_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmadd_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vmadd.vv v8, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmadd_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmv2r.v v10, v8
+; NOVLOPT-NEXT:    vmadd.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmadd_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; VLOPT-NEXT:    vmv2r.v v10, v8
+; VLOPT-NEXT:    vmadd.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i32> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vnmsac_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnmsac_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vnmsac.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnmsac_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vnmsac.vv v8, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnmsac.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vnmsac_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnmsac_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmv2r.v v10, v8
+; NOVLOPT-NEXT:    vnmsac.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnmsac_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; VLOPT-NEXT:    vmv2r.v v10, v8
+; VLOPT-NEXT:    vnmsac.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnmsac.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i32> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vnmsub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnmsub_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vnmsub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnmsub_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; VLOPT-NEXT:    vnmsub.vv v8, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnmsub.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vnmsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnmsub_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, tu, ma
+; NOVLOPT-NEXT:    vmv2r.v v10, v8
+; NOVLOPT-NEXT:    vnmsub.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnmsub_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; VLOPT-NEXT:    vmv2r.v v10, v8
+; VLOPT-NEXT:    vnmsub.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vnmsub.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, i32 %b, <vscale x 4 x i32> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmacc_vx(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen %vl) {
 ; NOVLOPT-LABEL: vwmacc_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
-; NOVLOPT-NEXT:    vwmacc.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmacc.vx v8, a0, v10
 ; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vwmacc_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
-; VLOPT-NEXT:    vwmacc.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmacc.vx v8, a0, v10
 ; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.i16(<vscale x 4 x i32> poison, i16 %b, <vscale x 4 x i16> %a, iXLen -1, iXLen 0)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.i16(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
   %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
-define <vscale x 4 x i32> @vwmaccu_vx(<vscale x 4 x i16> %a, i16 %b, iXLen %vl) {
+define <vscale x 4 x i32> @vwmaccu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, <vscale x 4 x i32> %d, iXLen %vl) {
+; NOVLOPT-LABEL: vwmaccu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmaccu.vv v8, v10, v11
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmaccu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmaccu.vv v8, v10, v11
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccu.nxv4i32.nxv4i16(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %d, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmaccu_vx(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, <vscale x 4 x i32> %d, i32 %e, iXLen %vl) {
 ; NOVLOPT-LABEL: vwmaccu_vx:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
-; NOVLOPT-NEXT:    vwmaccu.vx v10, a0, v8
-; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmaccu.vx v8, a0, v10
+; NOVLOPT-NEXT:    vsetvli zero, a2, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v12
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vwmaccu_vx:
 ; VLOPT:       # %bb.0:
-; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
-; VLOPT-NEXT:    vwmaccu.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, a2, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmaccu.vx v8, a0, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccu.nxv4i32.i16(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %d, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmaccsu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vwmaccsu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmaccsu.vv v8, v10, v11
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmaccsu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmaccsu.vv v8, v10, v11
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccsu.nxv4i32.nxv4i16(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmaccsu_vx(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vwmaccsu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmaccsu.vx v8, a0, v10
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmaccsu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmaccsu.vx v8, a0, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccsu.nxv4i32.i16(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmaccus_vx(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vwmaccus_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, tu, ma
+; NOVLOPT-NEXT:    vwmaccus.vx v8, a0, v10
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmaccus_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; VLOPT-NEXT:    vwmaccus.vx v8, a0, v10
 ; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccu.nxv4i32.i16(<vscale x 4 x i32> poison, i16 %b, <vscale x 4 x i16> %a, iXLen -1, iXLen 0)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccus.nxv4i32.i16(<vscale x 4 x i32> %a, i16 %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
   %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
@@ -1330,6 +1951,28 @@ define <vscale x 4 x i32> @vmv_v_x(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
   ret <vscale x 4 x i32> %2
 }
 
+; The vmv.v.v is optimized away if we use a vadd as the user.
+define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_v:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vmv.v.v v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_v:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
+; VLOPT-NEXT:    vmv.v.v v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
+; VLOPT-NEXT:    ret
+  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.v.nxv1i8.nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, iXLen -1)
+  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl)
+  ret <vscale x 1 x i8> %3
+}
+
 define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
 ; NOVLOPT-LABEL: vwsll_vi:
 ; NOVLOPT:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir
new file mode 100644
index 0000000000000..1071ee5361085
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir
@@ -0,0 +1,485 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vl-optimizer -verify-machineinstrs | FileCheck %s
+
+---
+name: vop_vv
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vv
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vop_vv_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vv_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+
+...
+---
+name: vop_vv_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vv_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_vv_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_vv_vd_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd_incompatible_eew
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_vv_vd_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vd_incompatible_emul
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e8 */, 0
+...
+---
+name: vwop_vv_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs2
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_vv_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs2_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_vv_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVWADD_VV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_vv_vs1
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs1
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_vv_vs1_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs1_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_VV_M1 $noreg, $noreg, %x, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_VV_M1 $noreg, $noreg, %x, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_vv_vs1_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_vv_vs1_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVWADD_VV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vwop_wv_vd_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd_incompatible_eew
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vd_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vd_incompatible_emul
+    ; CHECK: early-clobber %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVWADD_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 4 /* e8 */, 0
+...
+---
+name: vwop_wv_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs2
+    ; CHECK: %x:vrm2 = PseudoVADD_VV_M2 $noreg, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vrm2 = PseudoVADD_VV_M2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vrm2 = PseudoVWADD_WV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs2_incompatible_eew
+    ; CHECK: %x:vrm2 = PseudoVADD_VV_M2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vrm2 = PseudoVADD_VV_M2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_WV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVWADD_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVWADD_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vs1
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs1
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vs1_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs1_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vwop_wv_vs1_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vwop_wv_vs1_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vrm2 = PseudoVWADD_WV_M1 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vop_vf2_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vd
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF2_M1 $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF2_M1 $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf2_vd_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vd_incompatible_eew
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF2_M1 $noreg, $noreg, -1, 5 /* e32 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF2_M1 $noreg, $noreg, -1, 5 /* e32 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf2_vd_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vd_incompatible_emul
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF2_MF2 $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF2_MF2 $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf2_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vs2
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf2_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vs2_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf2_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf2_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF2_M1 $noreg, %x, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf4_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vd
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF4_M1 $noreg, $noreg, 1, 5 /* e32 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF4_M1 $noreg, $noreg, -1, 5 /* e32 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf4_vd_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vd_incompatible_eew
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF4_M1 $noreg, $noreg, -1, 5 /* e32 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF4_M1 $noreg, $noreg, -1, 5 /* e32 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 4 /* e16 */, 0
+...
+---
+name: vop_vf4_vd_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vd_incompatible_emul
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF4_MF2 $noreg, $noreg, -1, 5 /* e32 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF4_MF2 $noreg, $noreg, -1, 5 /* e32 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf4_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vs2
+    ; CHECK: %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf4_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vs2_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf4_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf4_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF4_M1 $noreg, %x, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf8_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vd
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, 1, 6 /* e64 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 6 /* e64 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, -1, 6 /* e64 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 6 /* e64 */, 0
+...
+---
+name: vop_vf8_vd_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vd_incompatible_eew
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, -1, 6 /* e64 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, -1, 6 /* e64 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 5 /* e32 */, 0
+...
+---
+name: vop_vf8_vd_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vd_incompatible_emul
+    ; CHECK: early-clobber %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, -1, 6 /* e64 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 6 /* e64 */, 0 /* tu, mu */
+    %x:vr = PseudoVZEXT_VF8_M1 $noreg, $noreg, -1, 6 /* e64 */, 0
+    %y:vr = PseudoVADD_VV_MF2 $noreg, %x, $noreg, 1, 6 /* e64 */, 0
+...
+---
+name: vop_vf8_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vs2
+    ; CHECK: %x:vr = PseudoVADD_VV_MF8 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF8 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0
+...
+---
+name: vop_vf8_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vs2_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_MF8 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF8 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0
+...
+---
+name: vop_vf8_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vop_vf8_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: early-clobber %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVZEXT_VF8_M1 $noreg, %x, 1, 6 /* e64 */, 0
+...
+# TODO: VNSRL_WV isn't yet a supported instruction for VL reduction
+---
+name: vnop_wv_vd
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd
+    ; CHECK: early-clobber %x:vr = PseudoVNSRL_WV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVNSRL_WV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vd_unsupported_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd_unsupported_eew
+    ; CHECK: early-clobber %x:vr = PseudoVNSRL_WV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVNSRL_WV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vd_unsupported_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vd_unsupported_emul
+    ; CHECK: %x:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVADD_VV_M1 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs2
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs2_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs2_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs2_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs2_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, %x, $noreg, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs1
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs1
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, 1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs1_incompatible_eew
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs1_incompatible_eew
+    ; CHECK: %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF2 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
+---
+name: vnop_wv_vs1_incompatible_emul
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: vnop_wv_vs1_incompatible_emul
+    ; CHECK: %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0 /* tu, mu */
+    %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0
+    %y:vr = PseudoVNSRL_WV_MF2 $noreg, $noreg, %x, 1, 3 /* e8 */, 0
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
index 1a1472fcfc66f..1efe96c975754 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
@@ -111,8 +111,8 @@ define <vscale x 4 x i32> @different_vl_with_ta(<vscale x 4 x i32> %a, <vscale x
 define <vscale x 4 x i32> @different_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
 ; CHECK-LABEL: different_vl_with_tu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v14, v10
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT:    vmv2r.v v14, v10
 ; CHECK-NEXT:    vadd.vv v14, v10, v12
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    vadd.vv v8, v14, v10
@@ -126,8 +126,8 @@ define <vscale x 4 x i32> @different_vl_with_tu(<vscale x 4 x i32> %passthru, <v
 define <vscale x 4 x i32> @different_imm_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
 ; CHECK-LABEL: different_imm_vl_with_tu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v14, v10
 ; CHECK-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; CHECK-NEXT:    vmv2r.v v14, v10
 ; CHECK-NEXT:    vadd.vv v14, v10, v12
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; CHECK-NEXT:    vadd.vv v8, v14, v10
@@ -136,3 +136,17 @@ define <vscale x 4 x i32> @different_imm_vl_with_tu(<vscale x 4 x i32> %passthru
   %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen 4)
   ret <vscale x 4 x i32> %w
 }
+
+define <vscale x 4 x i32> @dont_optimize_tied_def(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen %vl) {
+; CHECK-LABEL: dont_optimize_tied_def:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vwmacc.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; CHECK-NEXT:    vwmacc.vv v8, v10, v11
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.nxv4i16(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.nxv4i16(<vscale x 4 x i32> %1, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c, iXLen %vl, iXLen 0)
+  ret <vscale x 4 x i32> %2
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
index 1516d656663b6..d8bff08ea5513 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
@@ -51,9 +51,9 @@ entry:
 define <vscale x 16 x i16> @test_vlseg2ff_mask_dead_vl(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: test_vlseg2ff_mask_dead_vl:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
index b89097b8ff974..03aee1881503a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32.ll
@@ -25,9 +25,9 @@ entry:
 define <vscale x 1 x i8> @test_vlseg2ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -64,9 +64,9 @@ entry:
 define <vscale x 2 x i8> @test_vlseg2ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -103,9 +103,9 @@ entry:
 define <vscale x 4 x i8> @test_vlseg2ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -142,9 +142,9 @@ entry:
 define <vscale x 8 x i8> @test_vlseg2ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -181,9 +181,9 @@ entry:
 define <vscale x 16 x i8> @test_vlseg2ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -220,9 +220,9 @@ entry:
 define <vscale x 32 x i8> @test_vlseg2ff_mask_nxv32i8_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 32 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv32i8_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -259,10 +259,10 @@ entry:
 define <vscale x 1 x i8> @test_vlseg3ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_3t(target("riscv.vector.tuple", <vscale x 1 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -299,10 +299,10 @@ entry:
 define <vscale x 2 x i8> @test_vlseg3ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -339,10 +339,10 @@ entry:
 define <vscale x 4 x i8> @test_vlseg3ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -379,10 +379,10 @@ entry:
 define <vscale x 8 x i8> @test_vlseg3ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -419,10 +419,10 @@ entry:
 define <vscale x 16 x i8> @test_vlseg3ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -459,11 +459,11 @@ entry:
 define <vscale x 1 x i8> @test_vlseg4ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_4t(target("riscv.vector.tuple", <vscale x 1 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -500,11 +500,11 @@ entry:
 define <vscale x 2 x i8> @test_vlseg4ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -541,11 +541,11 @@ entry:
 define <vscale x 4 x i8> @test_vlseg4ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -582,11 +582,11 @@ entry:
 define <vscale x 8 x i8> @test_vlseg4ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -623,11 +623,11 @@ entry:
 define <vscale x 16 x i8> @test_vlseg4ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -664,12 +664,12 @@ entry:
 define <vscale x 1 x i8> @test_vlseg5ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_5t(target("riscv.vector.tuple", <vscale x 1 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -706,12 +706,12 @@ entry:
 define <vscale x 2 x i8> @test_vlseg5ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -748,12 +748,12 @@ entry:
 define <vscale x 4 x i8> @test_vlseg5ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -790,12 +790,12 @@ entry:
 define <vscale x 8 x i8> @test_vlseg5ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -832,13 +832,13 @@ entry:
 define <vscale x 1 x i8> @test_vlseg6ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_6t(target("riscv.vector.tuple", <vscale x 1 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -875,13 +875,13 @@ entry:
 define <vscale x 2 x i8> @test_vlseg6ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -918,13 +918,13 @@ entry:
 define <vscale x 4 x i8> @test_vlseg6ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -961,13 +961,13 @@ entry:
 define <vscale x 8 x i8> @test_vlseg6ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1004,6 +1004,7 @@ entry:
 define <vscale x 1 x i8> @test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7t(target("riscv.vector.tuple", <vscale x 1 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1011,7 +1012,6 @@ define <vscale x 1 x i8> @test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1048,6 +1048,7 @@ entry:
 define <vscale x 2 x i8> @test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1055,7 +1056,6 @@ define <vscale x 2 x i8> @test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1092,6 +1092,7 @@ entry:
 define <vscale x 4 x i8> @test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1099,7 +1100,6 @@ define <vscale x 4 x i8> @test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1136,6 +1136,7 @@ entry:
 define <vscale x 8 x i8> @test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1143,7 +1144,6 @@ define <vscale x 8 x i8> @test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1180,6 +1180,7 @@ entry:
 define <vscale x 1 x i8> @test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8t(target("riscv.vector.tuple", <vscale x 1 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1188,7 +1189,6 @@ define <vscale x 1 x i8> @test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1225,6 +1225,7 @@ entry:
 define <vscale x 2 x i8> @test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1233,7 +1234,6 @@ define <vscale x 2 x i8> @test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1270,6 +1270,7 @@ entry:
 define <vscale x 4 x i8> @test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1278,7 +1279,6 @@ define <vscale x 4 x i8> @test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1315,6 +1315,7 @@ entry:
 define <vscale x 8 x i8> @test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1323,7 +1324,6 @@ define <vscale x 8 x i8> @test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1359,9 +1359,9 @@ entry:
 define <vscale x 1 x i16> @test_vlseg2ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1397,9 +1397,9 @@ entry:
 define <vscale x 2 x i16> @test_vlseg2ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1435,9 +1435,9 @@ entry:
 define <vscale x 4 x i16> @test_vlseg2ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1473,9 +1473,9 @@ entry:
 define <vscale x 8 x i16> @test_vlseg2ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1511,9 +1511,9 @@ entry:
 define <vscale x 16 x i16> @test_vlseg2ff_mask_nxv16i16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16i16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1549,10 +1549,10 @@ entry:
 define <vscale x 1 x i16> @test_vlseg3ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1588,10 +1588,10 @@ entry:
 define <vscale x 2 x i16> @test_vlseg3ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1627,10 +1627,10 @@ entry:
 define <vscale x 4 x i16> @test_vlseg3ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1666,10 +1666,10 @@ entry:
 define <vscale x 8 x i16> @test_vlseg3ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1705,11 +1705,11 @@ entry:
 define <vscale x 1 x i16> @test_vlseg4ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1745,11 +1745,11 @@ entry:
 define <vscale x 2 x i16> @test_vlseg4ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1785,11 +1785,11 @@ entry:
 define <vscale x 4 x i16> @test_vlseg4ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1825,11 +1825,11 @@ entry:
 define <vscale x 8 x i16> @test_vlseg4ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1865,12 +1865,12 @@ entry:
 define <vscale x 1 x i16> @test_vlseg5ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1906,12 +1906,12 @@ entry:
 define <vscale x 2 x i16> @test_vlseg5ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1947,12 +1947,12 @@ entry:
 define <vscale x 4 x i16> @test_vlseg5ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -1988,13 +1988,13 @@ entry:
 define <vscale x 1 x i16> @test_vlseg6ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2030,13 +2030,13 @@ entry:
 define <vscale x 2 x i16> @test_vlseg6ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2072,13 +2072,13 @@ entry:
 define <vscale x 4 x i16> @test_vlseg6ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2114,6 +2114,7 @@ entry:
 define <vscale x 1 x i16> @test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2121,7 +2122,6 @@ define <vscale x 1 x i16> @test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2157,6 +2157,7 @@ entry:
 define <vscale x 2 x i16> @test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2164,7 +2165,6 @@ define <vscale x 2 x i16> @test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2200,6 +2200,7 @@ entry:
 define <vscale x 4 x i16> @test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2207,7 +2208,6 @@ define <vscale x 4 x i16> @test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2243,6 +2243,7 @@ entry:
 define <vscale x 1 x i16> @test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2251,7 +2252,6 @@ define <vscale x 1 x i16> @test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2287,6 +2287,7 @@ entry:
 define <vscale x 2 x i16> @test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2295,7 +2296,6 @@ define <vscale x 2 x i16> @test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2331,6 +2331,7 @@ entry:
 define <vscale x 4 x i16> @test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2339,7 +2340,6 @@ define <vscale x 4 x i16> @test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2375,9 +2375,9 @@ entry:
 define <vscale x 1 x i32> @test_vlseg2ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2413,9 +2413,9 @@ entry:
 define <vscale x 2 x i32> @test_vlseg2ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2451,9 +2451,9 @@ entry:
 define <vscale x 4 x i32> @test_vlseg2ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2489,9 +2489,9 @@ entry:
 define <vscale x 8 x i32> @test_vlseg2ff_mask_nxv8i32_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i32_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2527,10 +2527,10 @@ entry:
 define <vscale x 1 x i32> @test_vlseg3ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2566,10 +2566,10 @@ entry:
 define <vscale x 2 x i32> @test_vlseg3ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2605,10 +2605,10 @@ entry:
 define <vscale x 4 x i32> @test_vlseg3ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2644,11 +2644,11 @@ entry:
 define <vscale x 1 x i32> @test_vlseg4ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2684,11 +2684,11 @@ entry:
 define <vscale x 2 x i32> @test_vlseg4ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2724,11 +2724,11 @@ entry:
 define <vscale x 4 x i32> @test_vlseg4ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2764,12 +2764,12 @@ entry:
 define <vscale x 1 x i32> @test_vlseg5ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2805,12 +2805,12 @@ entry:
 define <vscale x 2 x i32> @test_vlseg5ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2846,13 +2846,13 @@ entry:
 define <vscale x 1 x i32> @test_vlseg6ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2888,13 +2888,13 @@ entry:
 define <vscale x 2 x i32> @test_vlseg6ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2930,6 +2930,7 @@ entry:
 define <vscale x 1 x i32> @test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2937,7 +2938,6 @@ define <vscale x 1 x i32> @test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -2973,6 +2973,7 @@ entry:
 define <vscale x 2 x i32> @test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2980,7 +2981,6 @@ define <vscale x 2 x i32> @test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3016,6 +3016,7 @@ entry:
 define <vscale x 1 x i32> @test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3024,7 +3025,6 @@ define <vscale x 1 x i32> @test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3060,6 +3060,7 @@ entry:
 define <vscale x 2 x i32> @test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3068,7 +3069,6 @@ define <vscale x 2 x i32> @test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3104,9 +3104,9 @@ entry:
 define <vscale x 1 x i64> @test_vlseg2ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3142,9 +3142,9 @@ entry:
 define <vscale x 2 x i64> @test_vlseg2ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3180,9 +3180,9 @@ entry:
 define <vscale x 4 x i64> @test_vlseg2ff_mask_nxv4i64_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i64_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3218,10 +3218,10 @@ entry:
 define <vscale x 1 x i64> @test_vlseg3ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3257,10 +3257,10 @@ entry:
 define <vscale x 2 x i64> @test_vlseg3ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3296,11 +3296,11 @@ entry:
 define <vscale x 1 x i64> @test_vlseg4ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3336,11 +3336,11 @@ entry:
 define <vscale x 2 x i64> @test_vlseg4ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3376,12 +3376,12 @@ entry:
 define <vscale x 1 x i64> @test_vlseg5ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3417,13 +3417,13 @@ entry:
 define <vscale x 1 x i64> @test_vlseg6ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3459,6 +3459,7 @@ entry:
 define <vscale x 1 x i64> @test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3466,7 +3467,6 @@ define <vscale x 1 x i64> @test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3502,6 +3502,7 @@ entry:
 define <vscale x 1 x i64> @test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3510,7 +3511,6 @@ define <vscale x 1 x i64> @test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3545,9 +3545,9 @@ entry:
 define <vscale x 1 x half> @test_vlseg2ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3582,9 +3582,9 @@ entry:
 define <vscale x 2 x half> @test_vlseg2ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3619,9 +3619,9 @@ entry:
 define <vscale x 4 x half> @test_vlseg2ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3656,9 +3656,9 @@ entry:
 define <vscale x 8 x half> @test_vlseg2ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3693,9 +3693,9 @@ entry:
 define <vscale x 16 x half> @test_vlseg2ff_mask_nxv16f16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16f16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3730,10 +3730,10 @@ entry:
 define <vscale x 1 x half> @test_vlseg3ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3768,10 +3768,10 @@ entry:
 define <vscale x 2 x half> @test_vlseg3ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3806,10 +3806,10 @@ entry:
 define <vscale x 4 x half> @test_vlseg3ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3844,10 +3844,10 @@ entry:
 define <vscale x 8 x half> @test_vlseg3ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3882,11 +3882,11 @@ entry:
 define <vscale x 1 x half> @test_vlseg4ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3921,11 +3921,11 @@ entry:
 define <vscale x 2 x half> @test_vlseg4ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3960,11 +3960,11 @@ entry:
 define <vscale x 4 x half> @test_vlseg4ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -3999,11 +3999,11 @@ entry:
 define <vscale x 8 x half> @test_vlseg4ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4038,12 +4038,12 @@ entry:
 define <vscale x 1 x half> @test_vlseg5ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4078,12 +4078,12 @@ entry:
 define <vscale x 2 x half> @test_vlseg5ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4118,12 +4118,12 @@ entry:
 define <vscale x 4 x half> @test_vlseg5ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4158,13 +4158,13 @@ entry:
 define <vscale x 1 x half> @test_vlseg6ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4199,13 +4199,13 @@ entry:
 define <vscale x 2 x half> @test_vlseg6ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4240,13 +4240,13 @@ entry:
 define <vscale x 4 x half> @test_vlseg6ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4281,6 +4281,7 @@ entry:
 define <vscale x 1 x half> @test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4288,7 +4289,6 @@ define <vscale x 1 x half> @test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4323,6 +4323,7 @@ entry:
 define <vscale x 2 x half> @test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4330,7 +4331,6 @@ define <vscale x 2 x half> @test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4365,6 +4365,7 @@ entry:
 define <vscale x 4 x half> @test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4372,7 +4373,6 @@ define <vscale x 4 x half> @test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4407,6 +4407,7 @@ entry:
 define <vscale x 1 x half> @test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4415,7 +4416,6 @@ define <vscale x 1 x half> @test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4450,6 +4450,7 @@ entry:
 define <vscale x 2 x half> @test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4458,7 +4459,6 @@ define <vscale x 2 x half> @test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4493,6 +4493,7 @@ entry:
 define <vscale x 4 x half> @test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4501,7 +4502,6 @@ define <vscale x 4 x half> @test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4536,9 +4536,9 @@ entry:
 define <vscale x 1 x float> @test_vlseg2ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4573,9 +4573,9 @@ entry:
 define <vscale x 2 x float> @test_vlseg2ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4610,9 +4610,9 @@ entry:
 define <vscale x 4 x float> @test_vlseg2ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4647,9 +4647,9 @@ entry:
 define <vscale x 8 x float> @test_vlseg2ff_mask_nxv8f32_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8f32_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4684,10 +4684,10 @@ entry:
 define <vscale x 1 x float> @test_vlseg3ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4722,10 +4722,10 @@ entry:
 define <vscale x 2 x float> @test_vlseg3ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4760,10 +4760,10 @@ entry:
 define <vscale x 4 x float> @test_vlseg3ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4798,11 +4798,11 @@ entry:
 define <vscale x 1 x float> @test_vlseg4ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4837,11 +4837,11 @@ entry:
 define <vscale x 2 x float> @test_vlseg4ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4876,11 +4876,11 @@ entry:
 define <vscale x 4 x float> @test_vlseg4ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4915,12 +4915,12 @@ entry:
 define <vscale x 1 x float> @test_vlseg5ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4955,12 +4955,12 @@ entry:
 define <vscale x 2 x float> @test_vlseg5ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -4995,13 +4995,13 @@ entry:
 define <vscale x 1 x float> @test_vlseg6ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5036,13 +5036,13 @@ entry:
 define <vscale x 2 x float> @test_vlseg6ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5077,6 +5077,7 @@ entry:
 define <vscale x 1 x float> @test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5084,7 +5085,6 @@ define <vscale x 1 x float> @test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5119,6 +5119,7 @@ entry:
 define <vscale x 2 x float> @test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5126,7 +5127,6 @@ define <vscale x 2 x float> @test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5161,6 +5161,7 @@ entry:
 define <vscale x 1 x float> @test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5169,7 +5170,6 @@ define <vscale x 1 x float> @test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5204,6 +5204,7 @@ entry:
 define <vscale x 2 x float> @test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5212,7 +5213,6 @@ define <vscale x 2 x float> @test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5247,9 +5247,9 @@ entry:
 define <vscale x 1 x double> @test_vlseg2ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5284,9 +5284,9 @@ entry:
 define <vscale x 2 x double> @test_vlseg2ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5321,9 +5321,9 @@ entry:
 define <vscale x 4 x double> @test_vlseg2ff_mask_nxv4f64_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f64_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5358,10 +5358,10 @@ entry:
 define <vscale x 1 x double> @test_vlseg3ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5396,10 +5396,10 @@ entry:
 define <vscale x 2 x double> @test_vlseg3ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5434,11 +5434,11 @@ entry:
 define <vscale x 1 x double> @test_vlseg4ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5473,11 +5473,11 @@ entry:
 define <vscale x 2 x double> @test_vlseg4ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5512,12 +5512,12 @@ entry:
 define <vscale x 1 x double> @test_vlseg5ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5552,13 +5552,13 @@ entry:
 define <vscale x 1 x double> @test_vlseg6ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5593,6 +5593,7 @@ entry:
 define <vscale x 1 x double> @test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5600,7 +5601,6 @@ define <vscale x 1 x double> @test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5635,6 +5635,7 @@ entry:
 define <vscale x 1 x double> @test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5643,7 +5644,6 @@ define <vscale x 1 x double> @test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5678,9 +5678,9 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg2ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5715,9 +5715,9 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg2ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5752,9 +5752,9 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg2ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5789,9 +5789,9 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg2ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5826,9 +5826,9 @@ entry:
 define <vscale x 16 x bfloat> @test_vlseg2ff_mask_nxv16bf16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16bf16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5863,10 +5863,10 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg3ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5901,10 +5901,10 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg3ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5939,10 +5939,10 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg3ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -5977,10 +5977,10 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg3ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6015,11 +6015,11 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg4ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6054,11 +6054,11 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg4ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6093,11 +6093,11 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg4ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6132,11 +6132,11 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg4ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i32 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6171,12 +6171,12 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg5ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6211,12 +6211,12 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg5ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6251,12 +6251,12 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg5ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6291,13 +6291,13 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg6ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6332,13 +6332,13 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg6ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6373,13 +6373,13 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg6ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6414,6 +6414,7 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6421,7 +6422,6 @@ define <vscale x 1 x bfloat> @test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6456,6 +6456,7 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6463,7 +6464,6 @@ define <vscale x 2 x bfloat> @test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6498,6 +6498,7 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6505,7 +6506,6 @@ define <vscale x 4 x bfloat> @test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6540,6 +6540,7 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6548,7 +6549,6 @@ define <vscale x 1 x bfloat> @test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6583,6 +6583,7 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6591,7 +6592,6 @@ define <vscale x 2 x bfloat> @test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
@@ -6626,6 +6626,7 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i32 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6634,7 +6635,6 @@ define <vscale x 4 x bfloat> @test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sw a0, 0(a2)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
index 3dc0db90b6d85..05a5be295cc71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
@@ -51,9 +51,9 @@ entry:
 define <vscale x 16 x i16> @test_vlseg2ff_mask_dead_vl(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: test_vlseg2ff_mask_dead_vl:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
index 68acb3beb0686..85008ff97143f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64.ll
@@ -25,9 +25,9 @@ entry:
 define <vscale x 1 x i8> @test_vlseg2ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_2t(target("riscv.vector.tuple", <vscale x 1 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -64,9 +64,9 @@ entry:
 define <vscale x 2 x i8> @test_vlseg2ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -103,9 +103,9 @@ entry:
 define <vscale x 4 x i8> @test_vlseg2ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -142,9 +142,9 @@ entry:
 define <vscale x 8 x i8> @test_vlseg2ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -181,9 +181,9 @@ entry:
 define <vscale x 16 x i8> @test_vlseg2ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -220,9 +220,9 @@ entry:
 define <vscale x 32 x i8> @test_vlseg2ff_mask_nxv32i8_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 32 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv32i8_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e8ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -259,10 +259,10 @@ entry:
 define <vscale x 1 x i8> @test_vlseg3ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_3t(target("riscv.vector.tuple", <vscale x 1 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -299,10 +299,10 @@ entry:
 define <vscale x 2 x i8> @test_vlseg3ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -339,10 +339,10 @@ entry:
 define <vscale x 4 x i8> @test_vlseg3ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -379,10 +379,10 @@ entry:
 define <vscale x 8 x i8> @test_vlseg3ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -419,10 +419,10 @@ entry:
 define <vscale x 16 x i8> @test_vlseg3ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -459,11 +459,11 @@ entry:
 define <vscale x 1 x i8> @test_vlseg4ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_4t(target("riscv.vector.tuple", <vscale x 1 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -500,11 +500,11 @@ entry:
 define <vscale x 2 x i8> @test_vlseg4ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -541,11 +541,11 @@ entry:
 define <vscale x 4 x i8> @test_vlseg4ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -582,11 +582,11 @@ entry:
 define <vscale x 8 x i8> @test_vlseg4ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -623,11 +623,11 @@ entry:
 define <vscale x 16 x i8> @test_vlseg4ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv16i8_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e8ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -664,12 +664,12 @@ entry:
 define <vscale x 1 x i8> @test_vlseg5ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_5t(target("riscv.vector.tuple", <vscale x 1 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -706,12 +706,12 @@ entry:
 define <vscale x 2 x i8> @test_vlseg5ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -748,12 +748,12 @@ entry:
 define <vscale x 4 x i8> @test_vlseg5ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -790,12 +790,12 @@ entry:
 define <vscale x 8 x i8> @test_vlseg5ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -832,13 +832,13 @@ entry:
 define <vscale x 1 x i8> @test_vlseg6ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_6t(target("riscv.vector.tuple", <vscale x 1 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -875,13 +875,13 @@ entry:
 define <vscale x 2 x i8> @test_vlseg6ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -918,13 +918,13 @@ entry:
 define <vscale x 4 x i8> @test_vlseg6ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -961,13 +961,13 @@ entry:
 define <vscale x 8 x i8> @test_vlseg6ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1004,6 +1004,7 @@ entry:
 define <vscale x 1 x i8> @test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7t(target("riscv.vector.tuple", <vscale x 1 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1011,7 +1012,6 @@ define <vscale x 1 x i8> @test_vlseg7ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1048,6 +1048,7 @@ entry:
 define <vscale x 2 x i8> @test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1055,7 +1056,6 @@ define <vscale x 2 x i8> @test_vlseg7ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1092,6 +1092,7 @@ entry:
 define <vscale x 4 x i8> @test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1099,7 +1100,6 @@ define <vscale x 4 x i8> @test_vlseg7ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1136,6 +1136,7 @@ entry:
 define <vscale x 8 x i8> @test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1143,7 +1144,6 @@ define <vscale x 8 x i8> @test_vlseg7ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_7
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1180,6 +1180,7 @@ entry:
 define <vscale x 1 x i8> @test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8t(target("riscv.vector.tuple", <vscale x 1 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1188,7 +1189,6 @@ define <vscale x 1 x i8> @test_vlseg8ff_mask_nxv1i8_triscv.vector.tuple_nxv1i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1225,6 +1225,7 @@ entry:
 define <vscale x 2 x i8> @test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1233,7 +1234,6 @@ define <vscale x 2 x i8> @test_vlseg8ff_mask_nxv2i8_triscv.vector.tuple_nxv2i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1270,6 +1270,7 @@ entry:
 define <vscale x 4 x i8> @test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1278,7 +1279,6 @@ define <vscale x 4 x i8> @test_vlseg8ff_mask_nxv4i8_triscv.vector.tuple_nxv4i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1315,6 +1315,7 @@ entry:
 define <vscale x 8 x i8> @test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -1323,7 +1324,6 @@ define <vscale x 8 x i8> @test_vlseg8ff_mask_nxv8i8_triscv.vector.tuple_nxv8i8_8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e8ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1359,9 +1359,9 @@ entry:
 define <vscale x 1 x i16> @test_vlseg2ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1397,9 +1397,9 @@ entry:
 define <vscale x 2 x i16> @test_vlseg2ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1435,9 +1435,9 @@ entry:
 define <vscale x 4 x i16> @test_vlseg2ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1473,9 +1473,9 @@ entry:
 define <vscale x 8 x i16> @test_vlseg2ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1511,9 +1511,9 @@ entry:
 define <vscale x 16 x i16> @test_vlseg2ff_mask_nxv16i16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16i16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1549,10 +1549,10 @@ entry:
 define <vscale x 1 x i16> @test_vlseg3ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1588,10 +1588,10 @@ entry:
 define <vscale x 2 x i16> @test_vlseg3ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1627,10 +1627,10 @@ entry:
 define <vscale x 4 x i16> @test_vlseg3ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1666,10 +1666,10 @@ entry:
 define <vscale x 8 x i16> @test_vlseg3ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1705,11 +1705,11 @@ entry:
 define <vscale x 1 x i16> @test_vlseg4ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1745,11 +1745,11 @@ entry:
 define <vscale x 2 x i16> @test_vlseg4ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1785,11 +1785,11 @@ entry:
 define <vscale x 4 x i16> @test_vlseg4ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1825,11 +1825,11 @@ entry:
 define <vscale x 8 x i16> @test_vlseg4ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8i16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1865,12 +1865,12 @@ entry:
 define <vscale x 1 x i16> @test_vlseg5ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1906,12 +1906,12 @@ entry:
 define <vscale x 2 x i16> @test_vlseg5ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1947,12 +1947,12 @@ entry:
 define <vscale x 4 x i16> @test_vlseg5ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -1988,13 +1988,13 @@ entry:
 define <vscale x 1 x i16> @test_vlseg6ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2030,13 +2030,13 @@ entry:
 define <vscale x 2 x i16> @test_vlseg6ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2072,13 +2072,13 @@ entry:
 define <vscale x 4 x i16> @test_vlseg6ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2114,6 +2114,7 @@ entry:
 define <vscale x 1 x i16> @test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2121,7 +2122,6 @@ define <vscale x 1 x i16> @test_vlseg7ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2157,6 +2157,7 @@ entry:
 define <vscale x 2 x i16> @test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2164,7 +2165,6 @@ define <vscale x 2 x i16> @test_vlseg7ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2200,6 +2200,7 @@ entry:
 define <vscale x 4 x i16> @test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2207,7 +2208,6 @@ define <vscale x 4 x i16> @test_vlseg7ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2243,6 +2243,7 @@ entry:
 define <vscale x 1 x i16> @test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2251,7 +2252,6 @@ define <vscale x 1 x i16> @test_vlseg8ff_mask_nxv1i16_triscv.vector.tuple_nxv2i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2287,6 +2287,7 @@ entry:
 define <vscale x 2 x i16> @test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2295,7 +2296,6 @@ define <vscale x 2 x i16> @test_vlseg8ff_mask_nxv2i16_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2331,6 +2331,7 @@ entry:
 define <vscale x 4 x i16> @test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2339,7 +2340,6 @@ define <vscale x 4 x i16> @test_vlseg8ff_mask_nxv4i16_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2375,9 +2375,9 @@ entry:
 define <vscale x 1 x i32> @test_vlseg2ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2413,9 +2413,9 @@ entry:
 define <vscale x 2 x i32> @test_vlseg2ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2451,9 +2451,9 @@ entry:
 define <vscale x 4 x i32> @test_vlseg2ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2489,9 +2489,9 @@ entry:
 define <vscale x 8 x i32> @test_vlseg2ff_mask_nxv8i32_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8i32_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2527,10 +2527,10 @@ entry:
 define <vscale x 1 x i32> @test_vlseg3ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2566,10 +2566,10 @@ entry:
 define <vscale x 2 x i32> @test_vlseg3ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2605,10 +2605,10 @@ entry:
 define <vscale x 4 x i32> @test_vlseg3ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2644,11 +2644,11 @@ entry:
 define <vscale x 1 x i32> @test_vlseg4ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2684,11 +2684,11 @@ entry:
 define <vscale x 2 x i32> @test_vlseg4ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2724,11 +2724,11 @@ entry:
 define <vscale x 4 x i32> @test_vlseg4ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4i32_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2764,12 +2764,12 @@ entry:
 define <vscale x 1 x i32> @test_vlseg5ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2805,12 +2805,12 @@ entry:
 define <vscale x 2 x i32> @test_vlseg5ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2846,13 +2846,13 @@ entry:
 define <vscale x 1 x i32> @test_vlseg6ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2888,13 +2888,13 @@ entry:
 define <vscale x 2 x i32> @test_vlseg6ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2930,6 +2930,7 @@ entry:
 define <vscale x 1 x i32> @test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2937,7 +2938,6 @@ define <vscale x 1 x i32> @test_vlseg7ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -2973,6 +2973,7 @@ entry:
 define <vscale x 2 x i32> @test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -2980,7 +2981,6 @@ define <vscale x 2 x i32> @test_vlseg7ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3016,6 +3016,7 @@ entry:
 define <vscale x 1 x i32> @test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3024,7 +3025,6 @@ define <vscale x 1 x i32> @test_vlseg8ff_mask_nxv1i32_triscv.vector.tuple_nxv4i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3060,6 +3060,7 @@ entry:
 define <vscale x 2 x i32> @test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3068,7 +3069,6 @@ define <vscale x 2 x i32> @test_vlseg8ff_mask_nxv2i32_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3104,9 +3104,9 @@ entry:
 define <vscale x 1 x i64> @test_vlseg2ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3142,9 +3142,9 @@ entry:
 define <vscale x 2 x i64> @test_vlseg2ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3180,9 +3180,9 @@ entry:
 define <vscale x 4 x i64> @test_vlseg2ff_mask_nxv4i64_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4i64_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3218,10 +3218,10 @@ entry:
 define <vscale x 1 x i64> @test_vlseg3ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3257,10 +3257,10 @@ entry:
 define <vscale x 2 x i64> @test_vlseg3ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3296,11 +3296,11 @@ entry:
 define <vscale x 1 x i64> @test_vlseg4ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3336,11 +3336,11 @@ entry:
 define <vscale x 2 x i64> @test_vlseg4ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2i64_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3376,12 +3376,12 @@ entry:
 define <vscale x 1 x i64> @test_vlseg5ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3417,13 +3417,13 @@ entry:
 define <vscale x 1 x i64> @test_vlseg6ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3459,6 +3459,7 @@ entry:
 define <vscale x 1 x i64> @test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3466,7 +3467,6 @@ define <vscale x 1 x i64> @test_vlseg7ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3502,6 +3502,7 @@ entry:
 define <vscale x 1 x i64> @test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -3510,7 +3511,6 @@ define <vscale x 1 x i64> @test_vlseg8ff_mask_nxv1i64_triscv.vector.tuple_nxv8i8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3545,9 +3545,9 @@ entry:
 define <vscale x 1 x half> @test_vlseg2ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3582,9 +3582,9 @@ entry:
 define <vscale x 2 x half> @test_vlseg2ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3619,9 +3619,9 @@ entry:
 define <vscale x 4 x half> @test_vlseg2ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3656,9 +3656,9 @@ entry:
 define <vscale x 8 x half> @test_vlseg2ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3693,9 +3693,9 @@ entry:
 define <vscale x 16 x half> @test_vlseg2ff_mask_nxv16f16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16f16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3730,10 +3730,10 @@ entry:
 define <vscale x 1 x half> @test_vlseg3ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3768,10 +3768,10 @@ entry:
 define <vscale x 2 x half> @test_vlseg3ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3806,10 +3806,10 @@ entry:
 define <vscale x 4 x half> @test_vlseg3ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3844,10 +3844,10 @@ entry:
 define <vscale x 8 x half> @test_vlseg3ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3882,11 +3882,11 @@ entry:
 define <vscale x 1 x half> @test_vlseg4ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3921,11 +3921,11 @@ entry:
 define <vscale x 2 x half> @test_vlseg4ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3960,11 +3960,11 @@ entry:
 define <vscale x 4 x half> @test_vlseg4ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -3999,11 +3999,11 @@ entry:
 define <vscale x 8 x half> @test_vlseg4ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8f16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4038,12 +4038,12 @@ entry:
 define <vscale x 1 x half> @test_vlseg5ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4078,12 +4078,12 @@ entry:
 define <vscale x 2 x half> @test_vlseg5ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4118,12 +4118,12 @@ entry:
 define <vscale x 4 x half> @test_vlseg5ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4158,13 +4158,13 @@ entry:
 define <vscale x 1 x half> @test_vlseg6ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4199,13 +4199,13 @@ entry:
 define <vscale x 2 x half> @test_vlseg6ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4240,13 +4240,13 @@ entry:
 define <vscale x 4 x half> @test_vlseg6ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4281,6 +4281,7 @@ entry:
 define <vscale x 1 x half> @test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4288,7 +4289,6 @@ define <vscale x 1 x half> @test_vlseg7ff_mask_nxv1f16_triscv.vector.tuple_nxv2i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4323,6 +4323,7 @@ entry:
 define <vscale x 2 x half> @test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4330,7 +4331,6 @@ define <vscale x 2 x half> @test_vlseg7ff_mask_nxv2f16_triscv.vector.tuple_nxv4i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4365,6 +4365,7 @@ entry:
 define <vscale x 4 x half> @test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4372,7 +4373,6 @@ define <vscale x 4 x half> @test_vlseg7ff_mask_nxv4f16_triscv.vector.tuple_nxv8i
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4407,6 +4407,7 @@ entry:
 define <vscale x 1 x half> @test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4415,7 +4416,6 @@ define <vscale x 1 x half> @test_vlseg8ff_mask_nxv1f16_triscv.vector.tuple_nxv2i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4450,6 +4450,7 @@ entry:
 define <vscale x 2 x half> @test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4458,7 +4459,6 @@ define <vscale x 2 x half> @test_vlseg8ff_mask_nxv2f16_triscv.vector.tuple_nxv4i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4493,6 +4493,7 @@ entry:
 define <vscale x 4 x half> @test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -4501,7 +4502,6 @@ define <vscale x 4 x half> @test_vlseg8ff_mask_nxv4f16_triscv.vector.tuple_nxv8i
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4536,9 +4536,9 @@ entry:
 define <vscale x 1 x float> @test_vlseg2ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4573,9 +4573,9 @@ entry:
 define <vscale x 2 x float> @test_vlseg2ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4610,9 +4610,9 @@ entry:
 define <vscale x 4 x float> @test_vlseg2ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4647,9 +4647,9 @@ entry:
 define <vscale x 8 x float> @test_vlseg2ff_mask_nxv8f32_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8f32_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e32ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4684,10 +4684,10 @@ entry:
 define <vscale x 1 x float> @test_vlseg3ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4722,10 +4722,10 @@ entry:
 define <vscale x 2 x float> @test_vlseg3ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4760,10 +4760,10 @@ entry:
 define <vscale x 4 x float> @test_vlseg3ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4798,11 +4798,11 @@ entry:
 define <vscale x 1 x float> @test_vlseg4ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4837,11 +4837,11 @@ entry:
 define <vscale x 2 x float> @test_vlseg4ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4876,11 +4876,11 @@ entry:
 define <vscale x 4 x float> @test_vlseg4ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4f32_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e32ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4915,12 +4915,12 @@ entry:
 define <vscale x 1 x float> @test_vlseg5ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4955,12 +4955,12 @@ entry:
 define <vscale x 2 x float> @test_vlseg5ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -4995,13 +4995,13 @@ entry:
 define <vscale x 1 x float> @test_vlseg6ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5036,13 +5036,13 @@ entry:
 define <vscale x 2 x float> @test_vlseg6ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5077,6 +5077,7 @@ entry:
 define <vscale x 1 x float> @test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5084,7 +5085,6 @@ define <vscale x 1 x float> @test_vlseg7ff_mask_nxv1f32_triscv.vector.tuple_nxv4
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5119,6 +5119,7 @@ entry:
 define <vscale x 2 x float> @test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5126,7 +5127,6 @@ define <vscale x 2 x float> @test_vlseg7ff_mask_nxv2f32_triscv.vector.tuple_nxv8
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5161,6 +5161,7 @@ entry:
 define <vscale x 1 x float> @test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5169,7 +5170,6 @@ define <vscale x 1 x float> @test_vlseg8ff_mask_nxv1f32_triscv.vector.tuple_nxv4
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5204,6 +5204,7 @@ entry:
 define <vscale x 2 x float> @test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5212,7 +5213,6 @@ define <vscale x 2 x float> @test_vlseg8ff_mask_nxv2f32_triscv.vector.tuple_nxv8
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e32ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5247,9 +5247,9 @@ entry:
 define <vscale x 1 x double> @test_vlseg2ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5284,9 +5284,9 @@ entry:
 define <vscale x 2 x double> @test_vlseg2ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5321,9 +5321,9 @@ entry:
 define <vscale x 4 x double> @test_vlseg2ff_mask_nxv4f64_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4f64_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e64ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5358,10 +5358,10 @@ entry:
 define <vscale x 1 x double> @test_vlseg3ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5396,10 +5396,10 @@ entry:
 define <vscale x 2 x double> @test_vlseg3ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5434,11 +5434,11 @@ entry:
 define <vscale x 1 x double> @test_vlseg4ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5473,11 +5473,11 @@ entry:
 define <vscale x 2 x double> @test_vlseg4ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2f64_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e64ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5512,12 +5512,12 @@ entry:
 define <vscale x 1 x double> @test_vlseg5ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5552,13 +5552,13 @@ entry:
 define <vscale x 1 x double> @test_vlseg6ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5593,6 +5593,7 @@ entry:
 define <vscale x 1 x double> @test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5600,7 +5601,6 @@ define <vscale x 1 x double> @test_vlseg7ff_mask_nxv1f64_triscv.vector.tuple_nxv
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5635,6 +5635,7 @@ entry:
 define <vscale x 1 x double> @test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -5643,7 +5644,6 @@ define <vscale x 1 x double> @test_vlseg8ff_mask_nxv1f64_triscv.vector.tuple_nxv
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e64ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5678,9 +5678,9 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg2ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_2t(target("riscv.vector.tuple", <vscale x 2 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5715,9 +5715,9 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg2ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_2t(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5752,9 +5752,9 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg2ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_2t(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5789,9 +5789,9 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg2ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_2t(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5826,9 +5826,9 @@ entry:
 define <vscale x 16 x bfloat> @test_vlseg2ff_mask_nxv16bf16_triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg2ff_mask_nxv16bf16_triscv.vector.tuple_nxv32i8_2t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv4r.v v4, v8
 ; CHECK-NEXT:    vmv4r.v v8, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vlseg2e16ff.v v4, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5863,10 +5863,10 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg3ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_3t(target("riscv.vector.tuple", <vscale x 2 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5901,10 +5901,10 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg3ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_3t(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5939,10 +5939,10 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg3ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_3t(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -5977,10 +5977,10 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg3ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_3t(target("riscv.vector.tuple", <vscale x 16 x i8>, 3) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg3ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_3t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg3e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6015,11 +6015,11 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg4ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_4t(target("riscv.vector.tuple", <vscale x 2 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6054,11 +6054,11 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg4ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_4t(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6093,11 +6093,11 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg4ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6132,11 +6132,11 @@ entry:
 define <vscale x 8 x bfloat> @test_vlseg4ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) %val, ptr %base, i64 %vl, <vscale x 8 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg4ff_mask_nxv8bf16_triscv.vector.tuple_nxv16i8_4t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv2r.v v6, v8
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    vmv2r.v v10, v12
 ; CHECK-NEXT:    vmv2r.v v12, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vlseg4e16ff.v v6, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6171,12 +6171,12 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg5ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_5t(target("riscv.vector.tuple", <vscale x 2 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6211,12 +6211,12 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg5ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_5t(target("riscv.vector.tuple", <vscale x 4 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6251,12 +6251,12 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg5ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_5t(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg5ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_5t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg5e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6291,13 +6291,13 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg6ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_6t(target("riscv.vector.tuple", <vscale x 2 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6332,13 +6332,13 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg6ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_6t(target("riscv.vector.tuple", <vscale x 4 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6373,13 +6373,13 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg6ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_6t(target("riscv.vector.tuple", <vscale x 8 x i8>, 6) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg6ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_6t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vmv1r.v v10, v11
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg6e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6414,6 +6414,7 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_7t(target("riscv.vector.tuple", <vscale x 2 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6421,7 +6422,6 @@ define <vscale x 1 x bfloat> @test_vlseg7ff_mask_nxv1bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6456,6 +6456,7 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_7t(target("riscv.vector.tuple", <vscale x 4 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6463,7 +6464,6 @@ define <vscale x 2 x bfloat> @test_vlseg7ff_mask_nxv2bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6498,6 +6498,7 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_7t(target("riscv.vector.tuple", <vscale x 8 x i8>, 7) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_7t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6505,7 +6506,6 @@ define <vscale x 4 x bfloat> @test_vlseg7ff_mask_nxv4bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v11, v12
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg7e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6540,6 +6540,7 @@ entry:
 define <vscale x 1 x bfloat> @test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_8t(target("riscv.vector.tuple", <vscale x 2 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 1 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nxv2i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6548,7 +6549,6 @@ define <vscale x 1 x bfloat> @test_vlseg8ff_mask_nxv1bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6583,6 +6583,7 @@ entry:
 define <vscale x 2 x bfloat> @test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_8t(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 2 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nxv4i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6591,7 +6592,6 @@ define <vscale x 2 x bfloat> @test_vlseg8ff_mask_nxv2bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
@@ -6626,6 +6626,7 @@ entry:
 define <vscale x 4 x bfloat> @test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %val, ptr %base, i64 %vl, <vscale x 4 x i1> %mask, ptr %outvl) {
 ; CHECK-LABEL: test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nxv8i8_8t:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v7, v8
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -6634,7 +6635,6 @@ define <vscale x 4 x bfloat> @test_vlseg8ff_mask_nxv4bf16_triscv.vector.tuple_nx
 ; CHECK-NEXT:    vmv1r.v v12, v13
 ; CHECK-NEXT:    vmv1r.v v13, v14
 ; CHECK-NEXT:    vmv1r.v v14, v15
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vlseg8e16ff.v v7, (a0), v0.t
 ; CHECK-NEXT:    csrr a0, vl
 ; CHECK-NEXT:    sd a0, 0(a2)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
index 0b553d3cd6fdf..b839cd595f3be 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
@@ -412,8 +412,8 @@ declare <vscale x 128 x i8> @llvm.vp.smax.nxv128i8(<vscale x 128 x i8>, <vscale
 define <vscale x 128 x i8> @vmax_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmax_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -974,11 +974,11 @@ declare <vscale x 32 x i32> @llvm.vp.smax.nxv32i32(<vscale x 32 x i32>, <vscale
 define <vscale x 32 x i32> @vmax_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmax_vx_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    srli a3, a2, 2
 ; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
@@ -1034,11 +1034,11 @@ declare i32 @llvm.vscale.i32()
 define <vscale x 32 x i32> @vmax_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
 ; CHECK-LABEL: vmax_vx_nxv32i32_evl_nx8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a3, a1, 2
 ; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
index f6be882f74206..99e0dfaf90a2d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
@@ -410,8 +410,8 @@ declare <vscale x 128 x i8> @llvm.vp.umax.nxv128i8(<vscale x 128 x i8>, <vscale
 define <vscale x 128 x i8> @vmaxu_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmaxu_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -972,11 +972,11 @@ declare <vscale x 32 x i32> @llvm.vp.umax.nxv32i32(<vscale x 32 x i32>, <vscale
 define <vscale x 32 x i32> @vmaxu_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmaxu_vx_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    srli a3, a2, 2
 ; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
@@ -1032,11 +1032,11 @@ declare i32 @llvm.vscale.i32()
 define <vscale x 32 x i32> @vmaxu_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
 ; CHECK-LABEL: vmaxu_vx_nxv32i32_evl_nx8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a3, a1, 2
 ; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll b/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
index 9ca78c872befd..babf8de57b7ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfeq.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmfeq_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmfeq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v10
+; CHECK-NEXT:    vmfeq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfeq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfeq.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmfeq_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmfeq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v12
+; CHECK-NEXT:    vmfeq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfeq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmfeq.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmfeq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v10
+; CHECK-NEXT:    vmfeq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfeq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmfeq_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmfeq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v12
+; CHECK-NEXT:    vmfeq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfeq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfeq.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmfeq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfeq.vv v0, v8, v9
 ; CHECK-NEXT:    vmfeq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmfeq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v10
+; CHECK-NEXT:    vmfeq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfeq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmfeq.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmfeq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfeq.vv v0, v8, v12
+; CHECK-NEXT:    vmfeq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfeq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfeq.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfeq.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfeq.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfeq.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfeq.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmfeq_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmfeq.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmfeq_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfeq.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfeq.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfeq.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfeq.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmfeq_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfeq.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfeq.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmfeq_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfeq.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmfeq_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmfeq.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfge.ll b/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
index 7cf18a7015812..4a9dd2f7d769d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfge.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmfge_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmfge_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmfge_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmfge_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfge.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmfge_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v12, v8
+; CHECK-NEXT:    vmfle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmfge.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmfge_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmfge_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmfge_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfge.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmfge_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v12, v8
+; CHECK-NEXT:    vmfle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfge.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmfge_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v9, v8
 ; CHECK-NEXT:    vmfle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmfge_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmfge.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmfge_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v12, v8
+; CHECK-NEXT:    vmfle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfge.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfge.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmfge_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfge.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmfge_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfge.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmfge_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfge.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmfge_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmfge.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmfge.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmfge_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmfge.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfge.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmfge_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfge.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmfge_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfge.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmfge_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmfge.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfge.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmfge_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmfge.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfge.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmfge_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmfge.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfge.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmfge_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmfge.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfge.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmfge_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmfge.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll b/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
index b78f2da4ae254..c9c5e84937cec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfgt.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmfgt_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v10, v8
+; CHECK-NEXT:    vmflt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfgt.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmfgt_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v12, v8
+; CHECK-NEXT:    vmflt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmfgt.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v10, v8
+; CHECK-NEXT:    vmflt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmfgt_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v12, v8
+; CHECK-NEXT:    vmflt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfgt.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v9, v8
 ; CHECK-NEXT:    vmflt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v10, v8
+; CHECK-NEXT:    vmflt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmfgt.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v12, v8
+; CHECK-NEXT:    vmflt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfgt.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfgt.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfgt.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfgt.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfgt.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmfgt_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmfgt.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmfgt_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfgt.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfgt.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfgt.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfgt.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmfgt_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfgt.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmfgt_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfgt.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmfgt_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfgt.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmfgt_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmfgt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfle.ll b/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
index 940e4d043f63f..77d8dda258961 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfle.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmfle_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmfle_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmfle_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmfle_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfle.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmfle_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmfle.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmfle_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmfle_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmfle_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmfle_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfle.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmfle_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmfle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfle.vv v0, v8, v9
 ; CHECK-NEXT:    vmfle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmfle_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmfle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmfle.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmfle_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmfle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfle.vv v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfle.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfle.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmfle_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfle.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmfle_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfle.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmfle_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfle.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmfle_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmfle.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmfle.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmfle_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmfle.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfle.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmfle_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfle.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmfle_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfle.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmfle_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmfle.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfle.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmfle_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmfle.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfle.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmfle_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmfle.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfle.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmfle_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmfle.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfle.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmfle_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmfle.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmflt.ll b/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
index 10ddfb8f014ed..0fdae8abe8f6b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmflt.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmflt_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmflt_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmflt_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmflt_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v10
+; CHECK-NEXT:    vmflt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmflt.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmflt_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v12
+; CHECK-NEXT:    vmflt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmflt.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmflt_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmflt_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmflt_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v10
+; CHECK-NEXT:    vmflt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmflt_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v12
+; CHECK-NEXT:    vmflt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmflt.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmflt_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmflt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmflt.vv v0, v8, v9
 ; CHECK-NEXT:    vmflt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmflt_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmflt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v10
+; CHECK-NEXT:    vmflt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmflt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmflt_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmflt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmflt.vv v0, v8, v12
+; CHECK-NEXT:    vmflt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmflt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmflt.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmflt.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmflt_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmflt.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmflt_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmflt.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmflt_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmflt.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmflt_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmflt.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmflt_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmflt.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmflt_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmflt.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmflt_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmflt.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmflt_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmflt.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmflt_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmflt.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmflt_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmflt.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmflt_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmflt.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmflt_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmfne.ll b/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
index 4d8a95de1d3de..1d0227f793728 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmfne.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmfne_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmfne_mask_vv_nxv2f16_nxv2f16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmfne_mask_vv_nxv4f16_nxv4f16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,11 +188,10 @@ define <vscale x 8 x i1> @intrinsic_vmfne_mask_vv_nxv8f16_nxv8f16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmfne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v10
+; CHECK-NEXT:    vmfne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfne.nxv8f16(
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmfne_mask_vv_nxv16f16_nxv16f16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmfne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v12
+; CHECK-NEXT:    vmfne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmfne.nxv16f16(
@@ -295,9 +290,8 @@ define <vscale x 1 x i1> @intrinsic_vmfne_mask_vv_nxv1f32_nxv1f32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -347,9 +341,8 @@ define <vscale x 2 x i1> @intrinsic_vmfne_mask_vv_nxv2f32_nxv2f32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,11 +392,10 @@ define <vscale x 4 x i1> @intrinsic_vmfne_mask_vv_nxv4f32_nxv4f32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmfne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v10
+; CHECK-NEXT:    vmfne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f32(
@@ -451,11 +443,10 @@ define <vscale x 8 x i1> @intrinsic_vmfne_mask_vv_nxv8f32_nxv8f32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmfne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v12
+; CHECK-NEXT:    vmfne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmfne.nxv8f32(
@@ -503,9 +494,8 @@ define <vscale x 1 x i1> @intrinsic_vmfne_mask_vv_nxv1f64_nxv1f64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmfne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmfne.vv v0, v8, v9
 ; CHECK-NEXT:    vmfne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -555,11 +545,10 @@ define <vscale x 2 x i1> @intrinsic_vmfne_mask_vv_nxv2f64_nxv2f64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmfne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v10
+; CHECK-NEXT:    vmfne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmfne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmfne.nxv2f64(
@@ -607,11 +596,10 @@ define <vscale x 4 x i1> @intrinsic_vmfne_mask_vv_nxv4f64_nxv4f64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmfne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmfne.vv v0, v8, v12
+; CHECK-NEXT:    vmfne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmfne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmfne.nxv4f64(
@@ -658,9 +646,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfne.mask.nxv1f16.f16(
 define <vscale x 1 x i1> @intrinsic_vmfne_mask_vf_nxv1f16_f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -705,9 +693,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfne.mask.nxv2f16.f16(
 define <vscale x 2 x i1> @intrinsic_vmfne_mask_vf_nxv2f16_f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -752,9 +740,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfne.mask.nxv4f16.f16(
 define <vscale x 4 x i1> @intrinsic_vmfne_mask_vf_nxv4f16_f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -799,9 +787,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfne.mask.nxv8f16.f16(
 define <vscale x 8 x i1> @intrinsic_vmfne_mask_vf_nxv8f16_f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmfne.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -846,9 +834,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmfne.mask.nxv16f16.f16(
 define <vscale x 16 x i1> @intrinsic_vmfne_mask_vf_nxv16f16_f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmfne.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -893,9 +881,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfne.mask.nxv1f32.f32(
 define <vscale x 1 x i1> @intrinsic_vmfne_mask_vf_nxv1f32_f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -940,9 +928,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfne.mask.nxv2f32.f32(
 define <vscale x 2 x i1> @intrinsic_vmfne_mask_vf_nxv2f32_f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -987,9 +975,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfne.mask.nxv4f32.f32(
 define <vscale x 4 x i1> @intrinsic_vmfne_mask_vf_nxv4f32_f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmfne.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1034,9 +1022,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmfne.mask.nxv8f32.f32(
 define <vscale x 8 x i1> @intrinsic_vmfne_mask_vf_nxv8f32_f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmfne.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1081,9 +1069,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmfne.mask.nxv1f64.f64(
 define <vscale x 1 x i1> @intrinsic_vmfne_mask_vf_nxv1f64_f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmfne.vf v10, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1128,9 +1116,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmfne.mask.nxv2f64.f64(
 define <vscale x 2 x i1> @intrinsic_vmfne_mask_vf_nxv2f64_f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmfne.vf v11, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1175,9 +1163,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmfne.mask.nxv4f64.f64(
 define <vscale x 4 x i1> @intrinsic_vmfne_mask_vf_nxv4f64_f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmfne.vf v13, v8, fa0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
index 8690014cc2c9d..3441934fb1550 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
@@ -412,8 +412,8 @@ declare <vscale x 128 x i8> @llvm.vp.smin.nxv128i8(<vscale x 128 x i8>, <vscale
 define <vscale x 128 x i8> @vmin_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmin_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -974,11 +974,11 @@ declare <vscale x 32 x i32> @llvm.vp.smin.nxv32i32(<vscale x 32 x i32>, <vscale
 define <vscale x 32 x i32> @vmin_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmin_vx_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    srli a3, a2, 2
 ; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
@@ -1034,11 +1034,11 @@ declare i32 @llvm.vscale.i32()
 define <vscale x 32 x i32> @vmin_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
 ; CHECK-LABEL: vmin_vx_nxv32i32_evl_nx8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a3, a1, 2
 ; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
index 414807829d563..bbf4c886bfe9f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
@@ -410,8 +410,8 @@ declare <vscale x 128 x i8> @llvm.vp.umin.nxv128i8(<vscale x 128 x i8>, <vscale
 define <vscale x 128 x i8> @vminu_vx_nxv128i8(<vscale x 128 x i8> %va, i8 %b, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vminu_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -972,11 +972,11 @@ declare <vscale x 32 x i32> @llvm.vp.umin.nxv32i32(<vscale x 32 x i32>, <vscale
 define <vscale x 32 x i32> @vminu_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vminu_vx_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    srli a3, a2, 2
 ; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
@@ -1032,11 +1032,11 @@ declare i32 @llvm.vscale.i32()
 define <vscale x 32 x i32> @vminu_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
 ; CHECK-LABEL: vminu_vx_nxv32i32_evl_nx8:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a3, a1, 2
 ; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a3
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    sltu a4, a1, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
index d1f344d52763d..80e74faa8cd91 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll
@@ -31,9 +31,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsbf.mask.nxv1i1(
 define <vscale x 1 x i1> @intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -73,9 +73,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsbf.mask.nxv2i1(
 define <vscale x 2 x i1> @intrinsic_vmsbf_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1> %0, <vscale x 2 x i1> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv2i1_nxv2i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -115,9 +115,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsbf.mask.nxv4i1(
 define <vscale x 4 x i1> @intrinsic_vmsbf_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1> %0, <vscale x 4 x i1> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv4i1_nxv4i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -157,9 +157,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsbf.mask.nxv8i1(
 define <vscale x 8 x i1> @intrinsic_vmsbf_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1> %0, <vscale x 8 x i1> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv8i1_nxv8i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -199,9 +199,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsbf.mask.nxv16i1(
 define <vscale x 16 x i1> @intrinsic_vmsbf_mask_m_nxv16i1_nxv16i1(<vscale x 16 x i1> %0, <vscale x 16 x i1> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv16i1_nxv16i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -241,9 +241,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsbf.mask.nxv32i1(
 define <vscale x 32 x i1> @intrinsic_vmsbf_mask_m_nxv32i1_nxv32i1(<vscale x 32 x i1> %0, <vscale x 32 x i1> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv32i1_nxv32i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -283,9 +283,9 @@ declare <vscale x 64 x i1> @llvm.riscv.vmsbf.mask.nxv64i1(
 define <vscale x 64 x i1> @intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, <vscale x 64 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsbf.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmseq.ll b/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
index c5769e0d1e519..6407f39a65e8b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmseq_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmseq_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmseq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v10
+; CHECK-NEXT:    vmseq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmseq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmseq.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmseq_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmseq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v12
+; CHECK-NEXT:    vmseq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmseq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmseq.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmseq_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmseq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v10
+; CHECK-NEXT:    vmseq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmseq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmseq.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmseq_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmseq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v12
+; CHECK-NEXT:    vmseq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmseq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmseq.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmseq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v10
+; CHECK-NEXT:    vmseq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmseq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmseq.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmseq_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmseq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v12
+; CHECK-NEXT:    vmseq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmseq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmseq.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmseq.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    vmseq.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmseq.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v10
+; CHECK-NEXT:    vmseq.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmseq.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmseq.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmseq.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmseq.vv v0, v8, v12
+; CHECK-NEXT:    vmseq.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmseq.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmseq.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmseq.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmseq.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmseq.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmseq.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmseq.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmseq_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmseq.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmseq_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmseq.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmseq.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmseq.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmseq.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmseq.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmseq_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmseq.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmseq.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmseq.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmseq.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmseq_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmseq.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmseq_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmseq.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmseq_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmseq.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmseq_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmseq_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmseq_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmseq_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmseq_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmseq_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmseq.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmseq_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmseq.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsge.ll b/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
index e6d775dee5b22..45e3840f7e673 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsge_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsge_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v10, v8
+; CHECK-NEXT:    vmsle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsge.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsge_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v12, v8
+; CHECK-NEXT:    vmsle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsge.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsge_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v10, v8
+; CHECK-NEXT:    vmsle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsge.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsge_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v12, v8
+; CHECK-NEXT:    vmsle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsge.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v10, v8
+; CHECK-NEXT:    vmsle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsge.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsge_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v12, v8
+; CHECK-NEXT:    vmsle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsge.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v9, v8
 ; CHECK-NEXT:    vmsle.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v10, v8
+; CHECK-NEXT:    vmsle.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsge.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsge_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v12, v8
+; CHECK-NEXT:    vmsle.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsge.nxv4i64(
@@ -971,9 +953,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsge.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1019,9 +1001,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsge.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1067,9 +1049,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsge.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1087,11 +1069,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i8_i8_1(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv4i8_i8_1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    li a1, 99
+; CHECK-NEXT:    li a0, 99
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsgt.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1154,9 +1136,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsge.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1202,9 +1184,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsge.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsge_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1250,9 +1232,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsge.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsge_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1298,9 +1280,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsge.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsge.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1394,9 +1376,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsge.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1442,9 +1424,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsge.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1490,9 +1472,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsge.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsge_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1538,9 +1520,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsge.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1586,9 +1568,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsge.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1634,9 +1616,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsge.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1682,9 +1664,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsge.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1757,9 +1739,9 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsge_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v10, v9
 ; RV64-NEXT:    ret
@@ -1832,9 +1814,9 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsge_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v11, v10
 ; RV64-NEXT:    ret
@@ -1907,9 +1889,9 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsge_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v13, v12
 ; RV64-NEXT:    ret
@@ -1942,9 +1924,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1977,9 +1959,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2043,9 +2025,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2078,9 +2060,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2113,9 +2095,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsge_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2148,9 +2130,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsge_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2183,9 +2165,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2218,9 +2200,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2253,9 +2235,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2288,9 +2270,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2323,9 +2305,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsge_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2358,9 +2340,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2393,9 +2375,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2428,9 +2410,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2463,9 +2445,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsge_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2498,9 +2480,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsge_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2533,9 +2515,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsge_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2568,9 +2550,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsge_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsge_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
index 68aa912ae42ea..e42be4faafefc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v10, v8
+; CHECK-NEXT:    vmsleu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgeu.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsgeu_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v12, v8
+; CHECK-NEXT:    vmsleu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsgeu.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v10, v8
+; CHECK-NEXT:    vmsleu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgeu.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vv_nxv16i16_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v12, v8
+; CHECK-NEXT:    vmsleu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgeu.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v10, v8
+; CHECK-NEXT:    vmsleu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgeu.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v12, v8
+; CHECK-NEXT:    vmsleu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgeu.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsleu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v10, v8
+; CHECK-NEXT:    vmsleu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsgeu.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v12, v8
+; CHECK-NEXT:    vmsleu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgeu.nxv4i64(
@@ -971,9 +953,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgeu.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1019,9 +1001,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgeu.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1067,9 +1049,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgeu.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1115,9 +1097,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgeu.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1163,9 +1145,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgeu.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1211,9 +1193,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsgeu.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsgeu_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1259,9 +1241,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgeu.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1307,9 +1289,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgeu.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1355,9 +1337,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgeu.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1403,9 +1385,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgeu.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1451,9 +1433,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgeu.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1499,9 +1481,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgeu.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1547,9 +1529,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgeu.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v10, v9
 ; CHECK-NEXT:    ret
@@ -1595,9 +1577,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgeu.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v11, v10
 ; CHECK-NEXT:    ret
@@ -1643,9 +1625,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgeu.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmxor.mm v0, v13, v12
 ; CHECK-NEXT:    ret
@@ -1718,9 +1700,9 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgeu_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v10, v9
 ; RV64-NEXT:    ret
@@ -1793,9 +1775,9 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgeu_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v11, v10
 ; RV64-NEXT:    ret
@@ -1868,9 +1850,9 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgeu_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmxor.mm v0, v13, v12
 ; RV64-NEXT:    ret
@@ -1903,9 +1885,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1938,9 +1920,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1973,9 +1955,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1993,11 +1975,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vi_nxv4i8_i8_1(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv4i8_i8_1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
-; CHECK-NEXT:    li a1, 99
+; CHECK-NEXT:    li a0, 99
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsgtu.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2029,9 +2011,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, -9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2064,9 +2046,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2099,9 +2081,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsgeu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2134,9 +2116,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2232,9 +2214,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2267,9 +2249,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2302,9 +2284,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgeu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2337,9 +2319,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2372,9 +2354,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2407,9 +2389,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2442,9 +2424,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgeu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2477,9 +2459,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 14, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2512,9 +2494,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, -16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2547,9 +2529,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgeu_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, -14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
index 98a03a2c56280..62ac44bfdf38c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v10, v8
+; CHECK-NEXT:    vmslt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgt.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsgt_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v12, v8
+; CHECK-NEXT:    vmslt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsgt.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v10, v8
+; CHECK-NEXT:    vmslt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgt.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v12, v8
+; CHECK-NEXT:    vmslt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgt.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v10, v8
+; CHECK-NEXT:    vmslt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgt.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v12, v8
+; CHECK-NEXT:    vmslt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgt.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v9, v8
 ; CHECK-NEXT:    vmslt.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v10, v8
+; CHECK-NEXT:    vmslt.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsgt.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v12, v8
+; CHECK-NEXT:    vmslt.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgt.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgt.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgt.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgt.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgt.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgt.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsgt.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsgt_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgt.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgt.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgt.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgt.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgt.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgt.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgt.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgt.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgt.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgt_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsgt.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgt_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsgt.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgt_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsgt.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsgt_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgt_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgt_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsgt.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
index 69b22573c289e..d57b9cd5bae53 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v10, v8
+; CHECK-NEXT:    vmsltu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgtu.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsgtu_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v12, v8
+; CHECK-NEXT:    vmsltu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsgtu.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v10, v8
+; CHECK-NEXT:    vmsltu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgtu.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vv_nxv16i16_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v12, v8
+; CHECK-NEXT:    vmsltu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsgtu.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v10, v8
+; CHECK-NEXT:    vmsltu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgtu.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v12, v8
+; CHECK-NEXT:    vmsltu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsgtu.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v9, v8
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v9, v8
 ; CHECK-NEXT:    vmsltu.vv v11, v10, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v10, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v10, v8
+; CHECK-NEXT:    vmsltu.vv v14, v12, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v12, v10, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsgtu.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v12, v8
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v12, v8
+; CHECK-NEXT:    vmsltu.vv v20, v16, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v16, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsgtu.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgtu.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgtu.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgtu.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgtu.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgtu.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsgtu.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsgtu_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgtu.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgtu.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgtu.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgtu.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsgtu.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsgtu.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgtu.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsgtu.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsgtu.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsgtu.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsgtu.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsgtu.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsgtu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsgtu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsgtu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsgtu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsif.ll b/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
index 1dc52eb55455b..9c70dcab1efde 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsif.ll
@@ -31,9 +31,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsif.mask.nxv1i1(
 define <vscale x 1 x i1> @intrinsic_vmsif_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv1i1_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -73,9 +73,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsif.mask.nxv2i1(
 define <vscale x 2 x i1> @intrinsic_vmsif_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1> %0, <vscale x 2 x i1> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv2i1_nxv2i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -115,9 +115,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsif.mask.nxv4i1(
 define <vscale x 4 x i1> @intrinsic_vmsif_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1> %0, <vscale x 4 x i1> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv4i1_nxv4i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -157,9 +157,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsif.mask.nxv8i1(
 define <vscale x 8 x i1> @intrinsic_vmsif_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1> %0, <vscale x 8 x i1> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv8i1_nxv8i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -199,9 +199,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsif.mask.nxv16i1(
 define <vscale x 16 x i1> @intrinsic_vmsif_mask_m_nxv16i1_nxv16i1(<vscale x 16 x i1> %0, <vscale x 16 x i1> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv16i1_nxv16i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -241,9 +241,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsif.mask.nxv32i1(
 define <vscale x 32 x i1> @intrinsic_vmsif_mask_m_nxv32i1_nxv32i1(<vscale x 32 x i1> %0, <vscale x 32 x i1> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv32i1_nxv32i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -283,9 +283,9 @@ declare <vscale x 64 x i1> @llvm.riscv.vmsif.mask.nxv64i1(
 define <vscale x 64 x i1> @intrinsic_vmsif_mask_m_nxv64i1_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, <vscale x 64 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsif_mask_m_nxv64i1_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsif.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsle.ll b/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
index c8794e1b63900..9653dfd2518d8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsle_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsle_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v10
+; CHECK-NEXT:    vmsle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsle.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsle_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v12
+; CHECK-NEXT:    vmsle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsle.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsle_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v10
+; CHECK-NEXT:    vmsle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsle.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsle_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v12
+; CHECK-NEXT:    vmsle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsle.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v10
+; CHECK-NEXT:    vmsle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsle.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsle_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v12
+; CHECK-NEXT:    vmsle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsle.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsle.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsle.vv v0, v8, v9
 ; CHECK-NEXT:    vmsle.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsle.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v10
+; CHECK-NEXT:    vmsle.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsle.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsle.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsle.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsle.vv v0, v8, v12
+; CHECK-NEXT:    vmsle.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsle.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsle.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsle.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsle.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsle.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsle.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsle.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsle_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsle.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsle_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsle.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsle.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsle.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsle.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsle.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsle_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsle.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsle.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsle.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsle.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsle_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsle.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsle_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsle.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsle_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsle.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsle_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsle_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsle_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsle_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsle_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsle_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsle_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
index 86dc48d51cc2b..25ecfa65c7c48 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v10
+; CHECK-NEXT:    vmsleu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsleu.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsleu_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v12
+; CHECK-NEXT:    vmsleu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsleu.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v10
+; CHECK-NEXT:    vmsleu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsleu.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vv_nxv16i16_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v12
+; CHECK-NEXT:    vmsleu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsleu.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v10
+; CHECK-NEXT:    vmsleu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsleu.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v12
+; CHECK-NEXT:    vmsleu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsleu.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsleu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsleu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsleu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsleu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v10
+; CHECK-NEXT:    vmsleu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsleu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsleu.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsleu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsleu.vv v0, v8, v12
+; CHECK-NEXT:    vmsleu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsleu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsleu.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsleu.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsleu.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsleu.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsleu.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsleu.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsleu.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsleu_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsleu.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsleu.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsleu.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsleu.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsleu.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsleu.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsleu.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsleu.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsleu.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsleu_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsleu.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsleu_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsleu.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsleu_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsleu.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsleu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsleu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsleu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmslt.ll b/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
index 8d57f2adc5386..c17495e3b2119 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmslt_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmslt_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v10
+; CHECK-NEXT:    vmslt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmslt.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmslt_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v12
+; CHECK-NEXT:    vmslt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmslt.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmslt_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v10
+; CHECK-NEXT:    vmslt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmslt.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmslt_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v12
+; CHECK-NEXT:    vmslt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmslt.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v10
+; CHECK-NEXT:    vmslt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmslt.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmslt_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v12
+; CHECK-NEXT:    vmslt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmslt.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmslt.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmslt.vv v0, v8, v9
 ; CHECK-NEXT:    vmslt.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmslt.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v10
+; CHECK-NEXT:    vmslt.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmslt.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmslt.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmslt.vv v0, v8, v12
+; CHECK-NEXT:    vmslt.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmslt.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmslt.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmslt.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmslt_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmslt_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmslt.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmslt_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmslt.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmslt_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmslt.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, -9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmslt_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmslt.vx v10, v8, zero, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsle.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsle.vi v11, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsle.vi v13, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
index 627b594308713..a37a02848365d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v10
+; CHECK-NEXT:    vmsltu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsltu.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsltu_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v12
+; CHECK-NEXT:    vmsltu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsltu.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v10
+; CHECK-NEXT:    vmsltu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsltu.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vv_nxv16i16_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v12
+; CHECK-NEXT:    vmsltu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsltu.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v10
+; CHECK-NEXT:    vmsltu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsltu.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v12
+; CHECK-NEXT:    vmsltu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsltu.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsltu.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsltu.vv v0, v8, v9
 ; CHECK-NEXT:    vmsltu.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsltu.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v10
+; CHECK-NEXT:    vmsltu.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsltu.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsltu.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsltu.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsltu.vv v0, v8, v12
+; CHECK-NEXT:    vmsltu.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsltu.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsltu.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsltu.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsltu_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsltu_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsltu.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsltu_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsltu.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsltu_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsltu.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, -9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsltu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsltu.vx v10, v8, zero, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v10, v8, 14, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v11, v8, -16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsleu.vi v13, v8, -14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsne.ll b/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
index 47d1048f46cab..ed41a18dcc8d3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
@@ -35,9 +35,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vv_nxv1i8_nxv1i8(<vscale x 1 x i1
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -87,9 +86,8 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vv_nxv2i8_nxv2i8(<vscale x 2 x i1
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -139,9 +137,8 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vv_nxv4i8_nxv4i8(<vscale x 4 x i1
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -191,9 +188,8 @@ define <vscale x 8 x i1> @intrinsic_vmsne_mask_vv_nxv8i8_nxv8i8(<vscale x 8 x i1
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -243,11 +239,10 @@ define <vscale x 16 x i1> @intrinsic_vmsne_mask_vv_nxv16i8_nxv16i8(<vscale x 16
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    vmsne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmsne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsne.nxv16i8(
@@ -295,11 +290,10 @@ define <vscale x 32 x i1> @intrinsic_vmsne_mask_vv_nxv32i8_nxv32i8(<vscale x 32
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    vmsne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v12
+; CHECK-NEXT:    vmsne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 32 x i1> @llvm.riscv.vmsne.nxv32i8(
@@ -347,9 +341,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vv_nxv1i16_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -399,9 +392,8 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vv_nxv2i16_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -451,9 +443,8 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vv_nxv4i16_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -503,11 +494,10 @@ define <vscale x 8 x i1> @intrinsic_vmsne_mask_vv_nxv8i16_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    vmsne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmsne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsne.nxv8i16(
@@ -555,11 +545,10 @@ define <vscale x 16 x i1> @intrinsic_vmsne_mask_vv_nxv16i16_nxv16i16(<vscale x 1
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    vmsne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v12
+; CHECK-NEXT:    vmsne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 16 x i1> @llvm.riscv.vmsne.nxv16i16(
@@ -607,9 +596,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vv_nxv1i32_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -659,9 +647,8 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vv_nxv2i32_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -711,11 +698,10 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vv_nxv4i32_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    vmsne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmsne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsne.nxv4i32(
@@ -763,11 +749,10 @@ define <vscale x 8 x i1> @intrinsic_vmsne_mask_vv_nxv8i32_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    vmsne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v12
+; CHECK-NEXT:    vmsne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 8 x i1> @llvm.riscv.vmsne.nxv8i32(
@@ -815,9 +800,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vv_nxv1i64_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    vmsne.vv v8, v8, v9
 ; CHECK-NEXT:    vmv1r.v v11, v0
-; CHECK-NEXT:    vmv.v.v v0, v8
+; CHECK-NEXT:    vmsne.vv v0, v8, v9
 ; CHECK-NEXT:    vmsne.vv v11, v9, v10, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v11
 ; CHECK-NEXT:    ret
@@ -867,11 +851,10 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vv_nxv2i64_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    vmsne.vv v14, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v14, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v10
+; CHECK-NEXT:    vmsne.vv v14, v10, v12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v14
-; CHECK-NEXT:    vmsne.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 2 x i1> @llvm.riscv.vmsne.nxv2i64(
@@ -919,11 +902,10 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vv_nxv4i64_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    vmsne.vv v20, v8, v12
-; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    vmv1r.v v20, v0
+; CHECK-NEXT:    vmsne.vv v0, v8, v12
+; CHECK-NEXT:    vmsne.vv v20, v12, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v20
-; CHECK-NEXT:    vmsne.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
 entry:
   %mask = call <vscale x 4 x i1> @llvm.riscv.vmsne.nxv4i64(
@@ -970,9 +952,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsne.mask.nxv1i8.i8(
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vx_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1017,9 +999,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsne.mask.nxv2i8.i8(
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vx_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1064,9 +1046,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsne.mask.nxv4i8.i8(
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vx_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1111,9 +1093,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsne.mask.nxv8i8.i8(
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vx_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1158,9 +1140,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsne.mask.nxv16i8.i8(
 define <vscale x 16 x i1> @intrinsic_vmsne_mask_vx_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1205,9 +1187,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsne.mask.nxv32i8.i8(
 define <vscale x 32 x i1> @intrinsic_vmsne_mask_vx_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1252,9 +1234,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsne.mask.nxv1i16.i16(
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vx_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1299,9 +1281,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsne.mask.nxv2i16.i16(
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vx_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1346,9 +1328,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsne.mask.nxv4i16.i16(
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vx_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1393,9 +1375,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsne.mask.nxv8i16.i16(
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vx_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1440,9 +1422,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsne.mask.nxv16i16.i16(
 define <vscale x 16 x i1> @intrinsic_vmsne_mask_vx_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1487,9 +1469,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsne.mask.nxv1i32.i32(
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vx_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1534,9 +1516,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsne.mask.nxv2i32.i32(
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vx_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1581,9 +1563,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsne.mask.nxv4i32.i32(
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vx_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vx v11, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -1628,9 +1610,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsne.mask.nxv8i32.i32(
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vx_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vx v13, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -1702,9 +1684,9 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsne_mask_vx_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmv1r.v v10, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vmsne.vx v10, v8, a0, v0.t
 ; RV64-NEXT:    vmv.v.v v0, v10
 ; RV64-NEXT:    ret
@@ -1776,9 +1758,9 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsne_mask_vx_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmv1r.v v11, v0
 ; RV64-NEXT:    vmv1r.v v0, v10
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vmsne.vx v11, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v11
 ; RV64-NEXT:    ret
@@ -1850,9 +1832,9 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ;
 ; RV64-LABEL: intrinsic_vmsne_mask_vx_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmv1r.v v13, v0
 ; RV64-NEXT:    vmv1r.v v0, v12
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vmsne.vx v13, v8, a0, v0.t
 ; RV64-NEXT:    vmv1r.v v0, v13
 ; RV64-NEXT:    ret
@@ -1885,9 +1867,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1920,9 +1902,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1955,9 +1937,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -1990,9 +1972,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2025,9 +2007,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsne_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2060,9 +2042,9 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsne_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2095,9 +2077,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2130,9 +2112,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2165,9 +2147,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2200,9 +2182,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2235,9 +2217,9 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsne_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2270,9 +2252,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2305,9 +2287,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2340,9 +2322,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2375,9 +2357,9 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsne_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
@@ -2410,9 +2392,9 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsne_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vmsne.vi v10, v8, 9, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -2445,9 +2427,9 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsne_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vmsne.vi v11, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    ret
@@ -2480,9 +2462,9 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsne_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v13, v0
 ; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vmsne.vi v13, v8, 9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsof.ll b/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
index b0a28e6e455b0..4b818a2b1e58f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsof.ll
@@ -31,9 +31,9 @@ declare <vscale x 1 x i1> @llvm.riscv.vmsof.mask.nxv1i1(
 define <vscale x 1 x i1> @intrinsic_vmsof_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv1i1_nxv1i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -73,9 +73,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsof.mask.nxv2i1(
 define <vscale x 2 x i1> @intrinsic_vmsof_mask_m_nxv2i1_nxv2i1(<vscale x 2 x i1> %0, <vscale x 2 x i1> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv2i1_nxv2i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -115,9 +115,9 @@ declare <vscale x 4 x i1> @llvm.riscv.vmsof.mask.nxv4i1(
 define <vscale x 4 x i1> @intrinsic_vmsof_mask_m_nxv4i1_nxv4i1(<vscale x 4 x i1> %0, <vscale x 4 x i1> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv4i1_nxv4i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -157,9 +157,9 @@ declare <vscale x 8 x i1> @llvm.riscv.vmsof.mask.nxv8i1(
 define <vscale x 8 x i1> @intrinsic_vmsof_mask_m_nxv8i1_nxv8i1(<vscale x 8 x i1> %0, <vscale x 8 x i1> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv8i1_nxv8i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv.v.v v0, v10
 ; CHECK-NEXT:    ret
@@ -199,9 +199,9 @@ declare <vscale x 16 x i1> @llvm.riscv.vmsof.mask.nxv16i1(
 define <vscale x 16 x i1> @intrinsic_vmsof_mask_m_nxv16i1_nxv16i1(<vscale x 16 x i1> %0, <vscale x 16 x i1> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv16i1_nxv16i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -241,9 +241,9 @@ declare <vscale x 32 x i1> @llvm.riscv.vmsof.mask.nxv32i1(
 define <vscale x 32 x i1> @intrinsic_vmsof_mask_m_nxv32i1_nxv32i1(<vscale x 32 x i1> %0, <vscale x 32 x i1> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv32i1_nxv32i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
@@ -283,9 +283,9 @@ declare <vscale x 64 x i1> @llvm.riscv.vmsof.mask.nxv64i1(
 define <vscale x 64 x i1> @intrinsic_vmsof_mask_m_nxv64i1_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, <vscale x 64 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsof_mask_m_nxv64i1_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, mu
 ; CHECK-NEXT:    vmsof.m v10, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
index 7f248a39b54fa..6345b90db23b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
@@ -49,8 +49,8 @@ define <vscale x 4 x i32> @vadd_same_passthru(<vscale x 4 x i32> %passthru, <vsc
 define <vscale x 4 x i32> @unfoldable_diff_avl_unknown(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
 ; CHECK-LABEL: unfoldable_diff_avl_unknown:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v14, v8
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT:    vmv2r.v v14, v8
 ; CHECK-NEXT:    vadd.vv v14, v10, v12
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    vmv.v.v v8, v14
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-cttz-elts.ll b/llvm/test/CodeGen/RISCV/rvv/vp-cttz-elts.ll
index f7ca65801dc87..b316f5f878816 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-cttz-elts.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-cttz-elts.ll
@@ -5,9 +5,9 @@
 define iXLen @bool_vec(<vscale x 2 x i1> %src, <vscale x 2 x i1> %m, i32 %evl) {
 ; RV32-LABEL: bool_vec:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v9, v0
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; RV32-NEXT:    vfirst.m a1, v9, v0.t
 ; RV32-NEXT:    bltz a1, .LBB0_2
 ; RV32-NEXT:  # %bb.1:
@@ -17,6 +17,7 @@ define iXLen @bool_vec(<vscale x 2 x i1> %src, <vscale x 2 x i1> %m, i32 %evl) {
 ;
 ; RV64-LABEL: bool_vec:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v9, v0
 ; RV64-NEXT:    slli a0, a0, 32
 ; RV64-NEXT:    srli a0, a0, 32
@@ -35,14 +36,15 @@ define iXLen @bool_vec(<vscale x 2 x i1> %src, <vscale x 2 x i1> %m, i32 %evl) {
 define iXLen @bool_vec_zero_poison(<vscale x 2 x i1> %src, <vscale x 2 x i1> %m, i32 %evl) {
 ; RV32-LABEL: bool_vec_zero_poison:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v9, v0
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; RV32-NEXT:    vfirst.m a0, v9, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bool_vec_zero_poison:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64-NEXT:    vmv1r.v v9, v0
 ; RV64-NEXT:    slli a0, a0, 32
 ; RV64-NEXT:    srli a0, a0, 32
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-select.ll b/llvm/test/CodeGen/RISCV/rvv/vp-select.ll
index c8a048971a803..652baf6692341 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-select.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-select.ll
@@ -12,6 +12,7 @@ define <vscale x 1 x i64> @all_ones(<vscale x 1 x i64> %true, <vscale x 1 x i64>
 define <vscale x 1 x i64> @all_zeroes(<vscale x 1 x i64> %true, <vscale x 1 x i64> %false, i32 %evl) {
 ; CHECK-LABEL: all_zeroes:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x i64> @llvm.vp.select.nxv1i64(<vscale x 1 x i1> splat (i1 false), <vscale x 1 x i64> %true, <vscale x 1 x i64> %false, i32 %evl)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
index 2a137099bcb0f..745cec4e7c4f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
@@ -10,9 +10,9 @@ declare <16 x i1> @llvm.experimental.vp.splice.v16i1(<16 x i1>, <16 x i1>, i32,
 define <2 x i1> @test_vp_splice_v2i1(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -34,9 +34,9 @@ define <2 x i1> @test_vp_splice_v2i1(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %ev
 define <2 x i1> @test_vp_splice_v2i1_negative_offset(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -58,9 +58,9 @@ define <2 x i1> @test_vp_splice_v2i1_negative_offset(<2 x i1> %va, <2 x i1> %vb,
 define <2 x i1> @test_vp_splice_v2i1_masked(<2 x i1> %va, <2 x i1> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -83,9 +83,9 @@ define <2 x i1> @test_vp_splice_v2i1_masked(<2 x i1> %va, <2 x i1> %vb, <2 x i1>
 define <4 x i1> @test_vp_splice_v4i1(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -107,9 +107,9 @@ define <4 x i1> @test_vp_splice_v4i1(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %ev
 define <4 x i1> @test_vp_splice_v4i1_negative_offset(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -131,9 +131,9 @@ define <4 x i1> @test_vp_splice_v4i1_negative_offset(<4 x i1> %va, <4 x i1> %vb,
 define <4 x i1> @test_vp_splice_v4i1_masked(<4 x i1> %va, <4 x i1> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -156,9 +156,9 @@ define <4 x i1> @test_vp_splice_v4i1_masked(<4 x i1> %va, <4 x i1> %vb, <4 x i1>
 define <8 x i1> @test_vp_splice_v8i1(<8 x i1> %va, <8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -180,9 +180,9 @@ define <8 x i1> @test_vp_splice_v8i1(<8 x i1> %va, <8 x i1> %vb, i32 zeroext %ev
 define <8 x i1> @test_vp_splice_v8i1_negative_offset(<8 x i1> %va, <8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v8i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -204,9 +204,9 @@ define <8 x i1> @test_vp_splice_v8i1_negative_offset(<8 x i1> %va, <8 x i1> %vb,
 define <8 x i1> @test_vp_splice_v8i1_masked(<8 x i1> %va, <8 x i1> %vb, <8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v8i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -229,9 +229,9 @@ define <8 x i1> @test_vp_splice_v8i1_masked(<8 x i1> %va, <8 x i1> %vb, <8 x i1>
 define <16 x i1> @test_vp_splice_v16i1(<16 x i1> %va, <16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
@@ -253,9 +253,9 @@ define <16 x i1> @test_vp_splice_v16i1(<16 x i1> %va, <16 x i1> %vb, i32 zeroext
 define <16 x i1> @test_vp_splice_v16i1_negative_offset(<16 x i1> %va, <16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v16i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
@@ -277,9 +277,9 @@ define <16 x i1> @test_vp_splice_v16i1_negative_offset(<16 x i1> %va, <16 x i1>
 define <16 x i1> @test_vp_splice_v16i1_masked(<16 x i1> %va, <16 x i1> %vb, <16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v16i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
index fc446d0a3a88a..3b0b183537468 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
@@ -13,9 +13,9 @@ declare <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i
 define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -37,9 +37,9 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x
 define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv1i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -61,9 +61,9 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1
 define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv1i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
@@ -86,9 +86,9 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <v
 define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -110,9 +110,9 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x
 define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv2i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -134,9 +134,9 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1
 define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv2i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
@@ -159,9 +159,9 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <v
 define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -183,9 +183,9 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x
 define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv4i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -207,9 +207,9 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1
 define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv4i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
@@ -232,9 +232,9 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <v
 define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
@@ -256,9 +256,9 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x
 define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv8i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
@@ -280,9 +280,9 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1
 define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv8i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
@@ -305,9 +305,9 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <v
 define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmerge.vim v10, v10, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
@@ -329,9 +329,9 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscal
 define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv16i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmerge.vim v10, v10, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
@@ -353,9 +353,9 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x
 define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, <vscale x 16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv16i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
@@ -379,9 +379,9 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va,
 define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
@@ -403,9 +403,9 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscal
 define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv32i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
@@ -427,9 +427,9 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x
 define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, <vscale x 32 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv32i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
@@ -453,9 +453,9 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va,
 define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
@@ -477,9 +477,9 @@ define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscal
 define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv64i1_negative_offset:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
@@ -501,9 +501,9 @@ define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x
 define <vscale x 64 x i1> @test_vp_splice_nxv64i1_masked(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, <vscale x 64 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv64i1_masked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
index 3e423c8ec9903..ca52ce6e2c4a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpgather-sdnode.ll
@@ -258,12 +258,12 @@ declare <vscale x 32 x i8> @llvm.vp.gather.nxv32i8.nxv32p0(<vscale x 32 x ptr>,
 define <vscale x 32 x i8> @vpgather_baseidx_nxv32i8(ptr %base, <vscale x 32 x i8> %idxs, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv32i8:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a2, a3, 1
 ; RV32-NEXT:    srli a3, a3, 2
 ; RV32-NEXT:    sub a4, a1, a2
-; RV32-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
 ; RV32-NEXT:    vslidedown.vx v0, v0, a3
 ; RV32-NEXT:    sltu a3, a1, a4
 ; RV32-NEXT:    addi a3, a3, -1
@@ -285,12 +285,12 @@ define <vscale x 32 x i8> @vpgather_baseidx_nxv32i8(ptr %base, <vscale x 32 x i8
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv32i8:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    csrr a2, vlenb
 ; RV64-NEXT:    slli a3, a2, 1
 ; RV64-NEXT:    srli a4, a2, 2
 ; RV64-NEXT:    sub a5, a1, a3
-; RV64-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
 ; RV64-NEXT:    vslidedown.vx v13, v0, a4
 ; RV64-NEXT:    sltu a4, a1, a5
 ; RV64-NEXT:    addi a4, a4, -1
@@ -2457,11 +2457,11 @@ declare <vscale x 16 x double> @llvm.vp.gather.nxv16f64.nxv16p0(<vscale x 16 x p
 define <vscale x 16 x double> @vpgather_nxv16f64(<vscale x 16 x ptr> %ptrs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_nxv16f64:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v24, v0
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    sub a2, a0, a1
 ; RV32-NEXT:    srli a3, a1, 3
-; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vx v0, v0, a3
 ; RV32-NEXT:    sltu a3, a0, a2
 ; RV32-NEXT:    addi a3, a3, -1
@@ -2480,11 +2480,11 @@ define <vscale x 16 x double> @vpgather_nxv16f64(<vscale x 16 x ptr> %ptrs, <vsc
 ;
 ; RV64-LABEL: vpgather_nxv16f64:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmv1r.v v24, v0
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    sub a2, a0, a1
 ; RV64-NEXT:    srli a3, a1, 3
-; RV64-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vx v0, v0, a3
 ; RV64-NEXT:    sltu a3, a0, a2
 ; RV64-NEXT:    addi a3, a3, -1
@@ -2506,8 +2506,8 @@ define <vscale x 16 x double> @vpgather_nxv16f64(<vscale x 16 x ptr> %ptrs, <vsc
 define <vscale x 16 x double> @vpgather_baseidx_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vsext.vf2 v16, v8
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    vsll.vi v24, v16, 3
@@ -2531,8 +2531,8 @@ define <vscale x 16 x double> @vpgather_baseidx_nxv16i16_nxv16f64(ptr %base, <vs
 ;
 ; RV64-LABEL: vpgather_baseidx_nxv16i16_nxv16f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vsext.vf4 v16, v10
 ; RV64-NEXT:    csrr a2, vlenb
 ; RV64-NEXT:    vsll.vi v16, v16, 3
@@ -2564,8 +2564,8 @@ define <vscale x 16 x double> @vpgather_baseidx_nxv16i16_nxv16f64(ptr %base, <vs
 define <vscale x 16 x double> @vpgather_baseidx_sext_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_sext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vsext.vf2 v16, v8
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    vsll.vi v24, v16, 3
@@ -2589,8 +2589,8 @@ define <vscale x 16 x double> @vpgather_baseidx_sext_nxv16i16_nxv16f64(ptr %base
 ;
 ; RV64-LABEL: vpgather_baseidx_sext_nxv16i16_nxv16f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vsext.vf4 v16, v10
 ; RV64-NEXT:    csrr a2, vlenb
 ; RV64-NEXT:    vsll.vi v16, v16, 3
@@ -2623,8 +2623,8 @@ define <vscale x 16 x double> @vpgather_baseidx_sext_nxv16i16_nxv16f64(ptr %base
 define <vscale x 16 x double> @vpgather_baseidx_zext_nxv16i16_nxv16f64(ptr %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vpgather_baseidx_zext_nxv16i16_nxv16f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv1r.v v12, v0
 ; RV32-NEXT:    vzext.vf2 v16, v8
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    vsll.vi v24, v16, 3
@@ -2648,8 +2648,8 @@ define <vscale x 16 x double> @vpgather_baseidx_zext_nxv16i16_nxv16f64(ptr %base
 ;
 ; RV64-LABEL: vpgather_baseidx_zext_nxv16i16_nxv16f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV64-NEXT:    vmv1r.v v12, v0
 ; RV64-NEXT:    vzext.vf2 v16, v8
 ; RV64-NEXT:    csrr a2, vlenb
 ; RV64-NEXT:    vsll.vi v24, v16, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload.ll b/llvm/test/CodeGen/RISCV/rvv/vpload.ll
index bd7ea6c19d0b3..0844180e49612 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpload.ll
@@ -522,12 +522,12 @@ declare <vscale x 16 x double> @llvm.vp.load.nxv16f64.p0(ptr, <vscale x 16 x i1>
 define <vscale x 16 x double> @vpload_nxv16f64(ptr %ptr, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpload_nxv16f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    sub a3, a1, a2
 ; CHECK-NEXT:    slli a4, a2, 3
 ; CHECK-NEXT:    srli a5, a2, 3
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a5
 ; CHECK-NEXT:    sltu a5, a1, a3
 ; CHECK-NEXT:    addi a5, a5, -1
@@ -561,6 +561,7 @@ declare <vscale x 16 x double> @llvm.vector.extract.nxv16f64(<vscale x 17 x doub
 define <vscale x 16 x double> @vpload_nxv17f64(ptr %ptr, ptr %out, <vscale x 17 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpload_nxv17f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v8, v0
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    slli a5, a3, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
index f029d0b1b01bc..88a8ebcc90054 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
@@ -361,12 +361,12 @@ define <vscale x 128 x i8> @vpmerge_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    vmv8r.v v24, v16
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vsetvli a4, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vlm.v v0, (a2)
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a2, a0, a1
@@ -401,8 +401,8 @@ define <vscale x 128 x i8> @vpmerge_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 define <vscale x 128 x i8> @vpmerge_vx_nxv128i8(i8 %a, <vscale x 128 x i8> %vb, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpmerge_vx_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a1)
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
@@ -429,8 +429,8 @@ define <vscale x 128 x i8> @vpmerge_vx_nxv128i8(i8 %a, <vscale x 128 x i8> %vb,
 define <vscale x 128 x i8> @vpmerge_vi_nxv128i8(<vscale x 128 x i8> %vb, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpmerge_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
index 8978dc268d4e5..7e7da529bf3d7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll
@@ -468,6 +468,7 @@ define void @vpstore_nxv17f64(<vscale x 17 x double> %val, ptr %ptr, <vscale x 1
 ; CHECK-NEXT:    slli a3, a3, 3
 ; CHECK-NEXT:    sub sp, sp, a3
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-mask-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-mask-vp.ll
index 39666bb6119a0..18d20f66987b2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-mask-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-mask-vp.ll
@@ -23,9 +23,9 @@ declare i1 @llvm.vp.reduce.or.nxv1i1(i1, <vscale x 1 x i1>, <vscale x 1 x i1>, i
 define zeroext i1 @vpreduce_or_nxv1i1(i1 zeroext %s, <vscale x 1 x i1> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -39,9 +39,9 @@ declare i1 @llvm.vp.reduce.xor.nxv1i1(i1, <vscale x 1 x i1>, <vscale x 1 x i1>,
 define zeroext i1 @vpreduce_xor_nxv1i1(i1 zeroext %s, <vscale x 1 x i1> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -71,9 +71,9 @@ declare i1 @llvm.vp.reduce.or.nxv2i1(i1, <vscale x 2 x i1>, <vscale x 2 x i1>, i
 define zeroext i1 @vpreduce_or_nxv2i1(i1 zeroext %s, <vscale x 2 x i1> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -87,9 +87,9 @@ declare i1 @llvm.vp.reduce.xor.nxv2i1(i1, <vscale x 2 x i1>, <vscale x 2 x i1>,
 define zeroext i1 @vpreduce_xor_nxv2i1(i1 zeroext %s, <vscale x 2 x i1> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -119,9 +119,9 @@ declare i1 @llvm.vp.reduce.or.nxv4i1(i1, <vscale x 4 x i1>, <vscale x 4 x i1>, i
 define zeroext i1 @vpreduce_or_nxv4i1(i1 zeroext %s, <vscale x 4 x i1> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -135,9 +135,9 @@ declare i1 @llvm.vp.reduce.xor.nxv4i1(i1, <vscale x 4 x i1>, <vscale x 4 x i1>,
 define zeroext i1 @vpreduce_xor_nxv4i1(i1 zeroext %s, <vscale x 4 x i1> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -167,9 +167,9 @@ declare i1 @llvm.vp.reduce.or.nxv8i1(i1, <vscale x 8 x i1>, <vscale x 8 x i1>, i
 define zeroext i1 @vpreduce_or_nxv8i1(i1 zeroext %s, <vscale x 8 x i1> %v, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -183,9 +183,9 @@ declare i1 @llvm.vp.reduce.xor.nxv8i1(i1, <vscale x 8 x i1>, <vscale x 8 x i1>,
 define zeroext i1 @vpreduce_xor_nxv8i1(i1 zeroext %s, <vscale x 8 x i1> %v, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -215,9 +215,9 @@ declare i1 @llvm.vp.reduce.or.nxv16i1(i1, <vscale x 16 x i1>, <vscale x 16 x i1>
 define zeroext i1 @vpreduce_or_nxv16i1(i1 zeroext %s, <vscale x 16 x i1> %v, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -231,9 +231,9 @@ declare i1 @llvm.vp.reduce.xor.nxv16i1(i1, <vscale x 16 x i1>, <vscale x 16 x i1
 define zeroext i1 @vpreduce_xor_nxv16i1(i1 zeroext %s, <vscale x 16 x i1> %v, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -263,9 +263,9 @@ declare i1 @llvm.vp.reduce.or.nxv32i1(i1, <vscale x 32 x i1>, <vscale x 32 x i1>
 define zeroext i1 @vpreduce_or_nxv32i1(i1 zeroext %s, <vscale x 32 x i1> %v, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -279,9 +279,9 @@ declare i1 @llvm.vp.reduce.xor.nxv32i1(i1, <vscale x 32 x i1>, <vscale x 32 x i1
 define zeroext i1 @vpreduce_xor_nxv32i1(i1 zeroext %s, <vscale x 32 x i1> %v, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -295,9 +295,9 @@ declare i1 @llvm.vp.reduce.or.nxv40i1(i1, <vscale x 40 x i1>, <vscale x 40 x i1>
 define zeroext i1 @vpreduce_or_nxv40i1(i1 zeroext %s, <vscale x 40 x i1> %v, <vscale x 40 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv40i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -327,9 +327,9 @@ declare i1 @llvm.vp.reduce.or.nxv64i1(i1, <vscale x 64 x i1>, <vscale x 64 x i1>
 define zeroext i1 @vpreduce_or_nxv64i1(i1 zeroext %s, <vscale x 64 x i1> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -343,9 +343,9 @@ declare i1 @llvm.vp.reduce.xor.nxv64i1(i1, <vscale x 64 x i1>, <vscale x 64 x i1
 define zeroext i1 @vpreduce_xor_nxv64i1(i1 zeroext %s, <vscale x 64 x i1> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_xor_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -359,6 +359,7 @@ declare i1 @llvm.vp.reduce.or.nxv128i1(i1, <vscale x 128 x i1>, <vscale x 128 x
 define zeroext i1 @vpreduce_or_nxv128i1(i1 zeroext %s, <vscale x 128 x i1> %v, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_or_nxv128i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
@@ -390,9 +391,9 @@ declare i1 @llvm.vp.reduce.add.nxv1i1(i1, <vscale x 1 x i1>, <vscale x 1 x i1>,
 define zeroext i1 @vpreduce_add_nxv1i1(i1 zeroext %s, <vscale x 1 x i1> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -406,9 +407,9 @@ declare i1 @llvm.vp.reduce.add.nxv2i1(i1, <vscale x 2 x i1>, <vscale x 2 x i1>,
 define zeroext i1 @vpreduce_add_nxv2i1(i1 zeroext %s, <vscale x 2 x i1> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -422,9 +423,9 @@ declare i1 @llvm.vp.reduce.add.nxv4i1(i1, <vscale x 4 x i1>, <vscale x 4 x i1>,
 define zeroext i1 @vpreduce_add_nxv4i1(i1 zeroext %s, <vscale x 4 x i1> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -438,9 +439,9 @@ declare i1 @llvm.vp.reduce.add.nxv8i1(i1, <vscale x 8 x i1>, <vscale x 8 x i1>,
 define zeroext i1 @vpreduce_add_nxv8i1(i1 zeroext %s, <vscale x 8 x i1> %v, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -454,9 +455,9 @@ declare i1 @llvm.vp.reduce.add.nxv16i1(i1, <vscale x 16 x i1>, <vscale x 16 x i1
 define zeroext i1 @vpreduce_add_nxv16i1(i1 zeroext %s, <vscale x 16 x i1> %v, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -470,9 +471,9 @@ declare i1 @llvm.vp.reduce.add.nxv32i1(i1, <vscale x 32 x i1>, <vscale x 32 x i1
 define zeroext i1 @vpreduce_add_nxv32i1(i1 zeroext %s, <vscale x 32 x i1> %v, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -486,9 +487,9 @@ declare i1 @llvm.vp.reduce.add.nxv64i1(i1, <vscale x 64 x i1>, <vscale x 64 x i1
 define zeroext i1 @vpreduce_add_nxv64i1(i1 zeroext %s, <vscale x 64 x i1> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_add_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    andi a1, a1, 1
 ; CHECK-NEXT:    xor a0, a1, a0
@@ -615,9 +616,9 @@ declare i1 @llvm.vp.reduce.smin.nxv1i1(i1, <vscale x 1 x i1>, <vscale x 1 x i1>,
 define zeroext i1 @vpreduce_smin_nxv1i1(i1 zeroext %s, <vscale x 1 x i1> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -631,9 +632,9 @@ declare i1 @llvm.vp.reduce.smin.nxv2i1(i1, <vscale x 2 x i1>, <vscale x 2 x i1>,
 define zeroext i1 @vpreduce_smin_nxv2i1(i1 zeroext %s, <vscale x 2 x i1> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -647,9 +648,9 @@ declare i1 @llvm.vp.reduce.smin.nxv4i1(i1, <vscale x 4 x i1>, <vscale x 4 x i1>,
 define zeroext i1 @vpreduce_smin_nxv4i1(i1 zeroext %s, <vscale x 4 x i1> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -663,9 +664,9 @@ declare i1 @llvm.vp.reduce.smin.nxv8i1(i1, <vscale x 8 x i1>, <vscale x 8 x i1>,
 define zeroext i1 @vpreduce_smin_nxv8i1(i1 zeroext %s, <vscale x 8 x i1> %v, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -679,9 +680,9 @@ declare i1 @llvm.vp.reduce.smin.nxv16i1(i1, <vscale x 16 x i1>, <vscale x 16 x i
 define zeroext i1 @vpreduce_smin_nxv16i1(i1 zeroext %s, <vscale x 16 x i1> %v, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -695,9 +696,9 @@ declare i1 @llvm.vp.reduce.smin.nxv32i1(i1, <vscale x 32 x i1>, <vscale x 32 x i
 define zeroext i1 @vpreduce_smin_nxv32i1(i1 zeroext %s, <vscale x 32 x i1> %v, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -711,9 +712,9 @@ declare i1 @llvm.vp.reduce.smin.nxv64i1(i1, <vscale x 64 x i1>, <vscale x 64 x i
 define zeroext i1 @vpreduce_smin_nxv64i1(i1 zeroext %s, <vscale x 64 x i1> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_smin_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -727,9 +728,9 @@ declare i1 @llvm.vp.reduce.umax.nxv1i1(i1, <vscale x 1 x i1>, <vscale x 1 x i1>,
 define zeroext i1 @vpreduce_umax_nxv1i1(i1 zeroext %s, <vscale x 1 x i1> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -743,9 +744,9 @@ declare i1 @llvm.vp.reduce.umax.nxv2i1(i1, <vscale x 2 x i1>, <vscale x 2 x i1>,
 define zeroext i1 @vpreduce_umax_nxv2i1(i1 zeroext %s, <vscale x 2 x i1> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv2i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -759,9 +760,9 @@ declare i1 @llvm.vp.reduce.umax.nxv4i1(i1, <vscale x 4 x i1>, <vscale x 4 x i1>,
 define zeroext i1 @vpreduce_umax_nxv4i1(i1 zeroext %s, <vscale x 4 x i1> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv4i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -775,9 +776,9 @@ declare i1 @llvm.vp.reduce.umax.nxv8i1(i1, <vscale x 8 x i1>, <vscale x 8 x i1>,
 define zeroext i1 @vpreduce_umax_nxv8i1(i1 zeroext %s, <vscale x 8 x i1> %v, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv8i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -791,9 +792,9 @@ declare i1 @llvm.vp.reduce.umax.nxv16i1(i1, <vscale x 16 x i1>, <vscale x 16 x i
 define zeroext i1 @vpreduce_umax_nxv16i1(i1 zeroext %s, <vscale x 16 x i1> %v, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv16i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -807,9 +808,9 @@ declare i1 @llvm.vp.reduce.umax.nxv32i1(i1, <vscale x 32 x i1>, <vscale x 32 x i
 define zeroext i1 @vpreduce_umax_nxv32i1(i1 zeroext %s, <vscale x 32 x i1> %v, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv32i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
@@ -823,9 +824,9 @@ declare i1 @llvm.vp.reduce.umax.nxv64i1(i1, <vscale x 64 x i1>, <vscale x 64 x i
 define zeroext i1 @vpreduce_umax_nxv64i1(i1 zeroext %s, <vscale x 64 x i1> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vpreduce_umax_nxv64i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vcpop.m a1, v9, v0.t
 ; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    or a0, a1, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-mask.ll
index fcd7482dffe2e..d99fd036b4fc9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-mask.ll
@@ -750,3 +750,561 @@ define zeroext i1 @vreduce_add_nxv64i1(<vscale x 64 x i1> %v) {
   %red = call i1 @llvm.vector.reduce.add.nxv64i1(<vscale x 64 x i1> %v)
   ret i1 %red
 }
+
+declare i1 @llvm.vector.reduce.or.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_or_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_or_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_xor_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_and_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_and_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v0, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_umax_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_smax_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v0, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_umin_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v0, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.nxv128i1(<vscale x 128 x i1>)
+
+define zeroext i1 @vreduce_smin_nxv128i1(<vscale x 128 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_nxv128i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v0, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.nxv128i1(<vscale x 128 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_or_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_or_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_xor_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v8, v8, v10
+; CHECK-NEXT:    vmxor.mm v9, v0, v9
+; CHECK-NEXT:    vmxor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_and_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_and_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_umax_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_smax_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_umin_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v0, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.nxv256i1(<vscale x 256 x i1>)
+
+define zeroext i1 @vreduce_smin_nxv256i1(<vscale x 256 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_nxv256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v0, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.nxv256i1(<vscale x 256 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_or_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_or_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_xor_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v10, v10, v14
+; CHECK-NEXT:    vmxor.mm v8, v8, v12
+; CHECK-NEXT:    vmxor.mm v9, v9, v13
+; CHECK-NEXT:    vmxor.mm v11, v0, v11
+; CHECK-NEXT:    vmxor.mm v8, v8, v10
+; CHECK-NEXT:    vmxor.mm v9, v11, v9
+; CHECK-NEXT:    vmxor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_and_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_and_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_umax_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_smax_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_umin_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v0, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.nxv512i1(<vscale x 512 x i1>)
+
+define zeroext i1 @vreduce_smin_nxv512i1(<vscale x 512 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_nxv512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v0, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.nxv512i1(<vscale x 512 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.or.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_or_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_or_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v14, v14, v22
+; CHECK-NEXT:    vmor.mm v10, v10, v18
+; CHECK-NEXT:    vmor.mm v12, v12, v20
+; CHECK-NEXT:    vmor.mm v8, v8, v16
+; CHECK-NEXT:    vmor.mm v13, v13, v21
+; CHECK-NEXT:    vmor.mm v9, v9, v17
+; CHECK-NEXT:    vmor.mm v11, v11, v19
+; CHECK-NEXT:    vmor.mm v15, v0, v15
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v15, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.or.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.xor.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_xor_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_xor_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmxor.mm v14, v14, v22
+; CHECK-NEXT:    vmxor.mm v10, v10, v18
+; CHECK-NEXT:    vmxor.mm v12, v12, v20
+; CHECK-NEXT:    vmxor.mm v8, v8, v16
+; CHECK-NEXT:    vmxor.mm v13, v13, v21
+; CHECK-NEXT:    vmxor.mm v9, v9, v17
+; CHECK-NEXT:    vmxor.mm v11, v11, v19
+; CHECK-NEXT:    vmxor.mm v15, v0, v15
+; CHECK-NEXT:    vmxor.mm v10, v10, v14
+; CHECK-NEXT:    vmxor.mm v8, v8, v12
+; CHECK-NEXT:    vmxor.mm v9, v9, v13
+; CHECK-NEXT:    vmxor.mm v11, v15, v11
+; CHECK-NEXT:    vmxor.mm v8, v8, v10
+; CHECK-NEXT:    vmxor.mm v9, v11, v9
+; CHECK-NEXT:    vmxor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.xor.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.and.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_and_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_and_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v14, v14, v22
+; CHECK-NEXT:    vmand.mm v10, v10, v18
+; CHECK-NEXT:    vmand.mm v12, v12, v20
+; CHECK-NEXT:    vmand.mm v8, v8, v16
+; CHECK-NEXT:    vmand.mm v13, v13, v21
+; CHECK-NEXT:    vmand.mm v9, v9, v17
+; CHECK-NEXT:    vmand.mm v11, v11, v19
+; CHECK-NEXT:    vmand.mm v15, v0, v15
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v15, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.and.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umax.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_umax_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_umax_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v14, v14, v22
+; CHECK-NEXT:    vmor.mm v10, v10, v18
+; CHECK-NEXT:    vmor.mm v12, v12, v20
+; CHECK-NEXT:    vmor.mm v8, v8, v16
+; CHECK-NEXT:    vmor.mm v13, v13, v21
+; CHECK-NEXT:    vmor.mm v9, v9, v17
+; CHECK-NEXT:    vmor.mm v11, v11, v19
+; CHECK-NEXT:    vmor.mm v15, v0, v15
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v15, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umax.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smax.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_smax_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_smax_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v14, v14, v22
+; CHECK-NEXT:    vmand.mm v10, v10, v18
+; CHECK-NEXT:    vmand.mm v12, v12, v20
+; CHECK-NEXT:    vmand.mm v8, v8, v16
+; CHECK-NEXT:    vmand.mm v13, v13, v21
+; CHECK-NEXT:    vmand.mm v9, v9, v17
+; CHECK-NEXT:    vmand.mm v11, v11, v19
+; CHECK-NEXT:    vmand.mm v15, v0, v15
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v15, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smax.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.umin.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_umin_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_umin_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmand.mm v14, v14, v22
+; CHECK-NEXT:    vmand.mm v10, v10, v18
+; CHECK-NEXT:    vmand.mm v12, v12, v20
+; CHECK-NEXT:    vmand.mm v8, v8, v16
+; CHECK-NEXT:    vmand.mm v13, v13, v21
+; CHECK-NEXT:    vmand.mm v9, v9, v17
+; CHECK-NEXT:    vmand.mm v11, v11, v19
+; CHECK-NEXT:    vmand.mm v15, v0, v15
+; CHECK-NEXT:    vmand.mm v10, v10, v14
+; CHECK-NEXT:    vmand.mm v8, v8, v12
+; CHECK-NEXT:    vmand.mm v9, v9, v13
+; CHECK-NEXT:    vmand.mm v11, v15, v11
+; CHECK-NEXT:    vmand.mm v8, v8, v10
+; CHECK-NEXT:    vmand.mm v9, v11, v9
+; CHECK-NEXT:    vmand.mm v8, v9, v8
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.umin.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
+
+declare i1 @llvm.vector.reduce.smin.nxv1024i1(<vscale x 1024 x i1>)
+
+define zeroext i1 @vreduce_smin_nxv1024i1(<vscale x 1024 x i1> %v) {
+; CHECK-LABEL: vreduce_smin_nxv1024i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmor.mm v14, v14, v22
+; CHECK-NEXT:    vmor.mm v10, v10, v18
+; CHECK-NEXT:    vmor.mm v12, v12, v20
+; CHECK-NEXT:    vmor.mm v8, v8, v16
+; CHECK-NEXT:    vmor.mm v13, v13, v21
+; CHECK-NEXT:    vmor.mm v9, v9, v17
+; CHECK-NEXT:    vmor.mm v11, v11, v19
+; CHECK-NEXT:    vmor.mm v15, v0, v15
+; CHECK-NEXT:    vmor.mm v10, v10, v14
+; CHECK-NEXT:    vmor.mm v8, v8, v12
+; CHECK-NEXT:    vmor.mm v9, v9, v13
+; CHECK-NEXT:    vmor.mm v11, v15, v11
+; CHECK-NEXT:    vmor.mm v8, v8, v10
+; CHECK-NEXT:    vmor.mm v9, v11, v9
+; CHECK-NEXT:    vmor.mm v8, v9, v8
+; CHECK-NEXT:    vcpop.m a0, v8
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %red = call i1 @llvm.vector.reduce.smin.nxv1024i1(<vscale x 1024 x i1> %v)
+  ret i1 %red
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
index 1779fc12095e8..7b460f2c058f8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
@@ -22,8 +22,8 @@ define internal void @foo(<vscale x 1 x i16> %v15, <vscale x 1 x i16> %0, <vscal
 ; NOSUBREG-NEXT:  .LBB0_1: # %loopIR3.i.i
 ; NOSUBREG-NEXT:    # =>This Inner Loop Header: Depth=1
 ; NOSUBREG-NEXT:    vl1r.v v9, (zero)
-; NOSUBREG-NEXT:    vmv1r.v v13, v12
 ; NOSUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
+; NOSUBREG-NEXT:    vmv1r.v v13, v12
 ; NOSUBREG-NEXT:    vrgatherei16.vv v13, v9, v10
 ; NOSUBREG-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; NOSUBREG-NEXT:    vand.vv v9, v8, v13
@@ -42,8 +42,8 @@ define internal void @foo(<vscale x 1 x i16> %v15, <vscale x 1 x i16> %0, <vscal
 ; SUBREG-NEXT:  .LBB0_1: # %loopIR3.i.i
 ; SUBREG-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SUBREG-NEXT:    vl1r.v v9, (zero)
-; SUBREG-NEXT:    vmv1r.v v13, v12
 ; SUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
+; SUBREG-NEXT:    vmv1r.v v13, v12
 ; SUBREG-NEXT:    vrgatherei16.vv v13, v9, v10
 ; SUBREG-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; SUBREG-NEXT:    vand.vv v9, v8, v13
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
index 12c439346e356..3421c6af334bc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
@@ -572,8 +572,8 @@ declare <vscale x 128 x i8> @llvm.vp.sadd.sat.nxv128i8(<vscale x 128 x i8>, <vsc
 define <vscale x 128 x i8> @vsadd_vi_nxv128i8(<vscale x 128 x i8> %va, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsadd_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -1350,11 +1350,11 @@ declare <vscale x 32 x i32> @llvm.vp.sadd.sat.nxv32i32(<vscale x 32 x i32>, <vsc
 define <vscale x 32 x i32> @vsadd_vi_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsadd_vi_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
index d962f703abfd2..180e0799044e8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
@@ -571,8 +571,8 @@ declare <vscale x 128 x i8> @llvm.vp.uadd.sat.nxv128i8(<vscale x 128 x i8>, <vsc
 define <vscale x 128 x i8> @vsaddu_vi_nxv128i8(<vscale x 128 x i8> %va, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsaddu_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -1349,11 +1349,11 @@ declare <vscale x 32 x i32> @llvm.vp.uadd.sat.nxv32i32(<vscale x 32 x i32>, <vsc
 define <vscale x 32 x i32> @vsaddu_vi_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsaddu_vi_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-bf16.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-bf16.ll
index a63d14e8b6c04..b793531e34069 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-bf16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-bf16.ll
@@ -126,6 +126,7 @@ define <vscale x 8 x bfloat> @vmerge_truelhs_nxv8bf16_0(<vscale x 8 x bfloat> %v
 define <vscale x 8 x bfloat> @vmerge_falselhs_nxv8bf16_0(<vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb) {
 ; CHECK-LABEL: vmerge_falselhs_nxv8bf16_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
   %vc = select <vscale x 8 x i1> zeroinitializer, <vscale x 8 x bfloat> %va, <vscale x 8 x bfloat> %vb
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
index 1fc33dc73a27d..be2fc6955294d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll
@@ -175,6 +175,7 @@ define <vscale x 8 x half> @vmerge_truelhs_nxv8f16_0(<vscale x 8 x half> %va, <v
 define <vscale x 8 x half> @vmerge_falselhs_nxv8f16_0(<vscale x 8 x half> %va, <vscale x 8 x half> %vb) {
 ; CHECK-LABEL: vmerge_falselhs_nxv8f16_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
   %vc = select <vscale x 8 x i1> zeroinitializer, <vscale x 8 x half> %va, <vscale x 8 x half> %vb
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
index 9cafa28eb429f..4ec9e344e6278 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
@@ -803,6 +803,7 @@ define <vscale x 8 x i64> @vmerge_truelhs_nxv8i64_0(<vscale x 8 x i64> %va, <vsc
 define <vscale x 8 x i64> @vmerge_falselhs_nxv8i64_0(<vscale x 8 x i64> %va, <vscale x 8 x i64> %vb) {
 ; CHECK-LABEL: vmerge_falselhs_nxv8i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
   %vc = select <vscale x 8 x i1> zeroinitializer, <vscale x 8 x i64> %va, <vscale x 8 x i64> %vb
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index bb51f0592dc17..d4ebe27420d7b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -362,6 +362,7 @@ define <vscale x 32 x i32> @select_nxv32i32(<vscale x 32 x i1> %a, <vscale x 32
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    slli a4, a3, 3
@@ -375,7 +376,6 @@ define <vscale x 32 x i32> @select_nxv32i32(<vscale x 32 x i1> %a, <vscale x 32
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a3
 ; CHECK-NEXT:    and a4, a4, a5
 ; CHECK-NEXT:    vsetvli zero, a4, e32, m8, ta, ma
@@ -421,6 +421,7 @@ define <vscale x 32 x i32> @select_evl_nxv32i32(<vscale x 32 x i1> %a, <vscale x
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
@@ -434,7 +435,6 @@ define <vscale x 32 x i32> @select_evl_nxv32i32(<vscale x 32 x i1> %a, <vscale x
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a4
 ; CHECK-NEXT:    and a3, a3, a5
 ; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
@@ -710,6 +710,7 @@ define <vscale x 16 x double> @select_nxv16f64(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
@@ -722,7 +723,6 @@ define <vscale x 16 x double> @select_nxv16f64(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    vl8re64.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a3
 ; CHECK-NEXT:    and a4, a5, a4
 ; CHECK-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
@@ -835,6 +835,7 @@ define <vscale x 2 x i1> @select_cond_x_cond(<vscale x 2 x i1> %x, <vscale x 2 x
 define <vscale x 2 x i1> @select_undef_T_F(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, i32 zeroext %evl) {
 ; CHECK-LABEL: select_undef_T_F:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> poison, <vscale x 2 x i1> %x, <vscale x 2 x i1> %y, i32 %evl)
@@ -852,6 +853,7 @@ define <vscale x 2 x i1> @select_undef_undef_F(<vscale x 2 x i1> %x, i32 zeroext
 define <vscale x 2 x i1> @select_unknown_undef_F(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, i32 zeroext %evl) {
 ; CHECK-LABEL: select_unknown_undef_F:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> %x, <vscale x 2 x i1> undef, <vscale x 2 x i1> %y, i32 %evl)
@@ -861,6 +863,7 @@ define <vscale x 2 x i1> @select_unknown_undef_F(<vscale x 2 x i1> %x, <vscale x
 define <vscale x 2 x i1> @select_unknown_T_undef(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, i32 zeroext %evl) {
 ; CHECK-LABEL: select_unknown_T_undef:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, <vscale x 2 x i1> poison, i32 %evl)
@@ -870,6 +873,7 @@ define <vscale x 2 x i1> @select_unknown_T_undef(<vscale x 2 x i1> %x, <vscale x
 define <vscale x 2 x i1> @select_false_T_F(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, <vscale x 2 x i1> %z, i32 zeroext %evl) {
 ; CHECK-LABEL: select_false_T_F:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> %y, <vscale x 2 x i1> %z, i32 %evl)
@@ -879,6 +883,7 @@ define <vscale x 2 x i1> @select_false_T_F(<vscale x 2 x i1> %x, <vscale x 2 x i
 define <vscale x 2 x i1> @select_unknown_T_T(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, i32 zeroext %evl) {
 ; CHECK-LABEL: select_unknown_T_T:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i1> @llvm.vp.select.nxv2i1(<vscale x 2 x i1> %x, <vscale x 2 x i1> %y, <vscale x 2 x i1> %y, i32 %evl)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-O0.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-O0.ll
index 33acfb7dceb94..d8620d0658261 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-O0.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-O0.ll
@@ -18,11 +18,11 @@ declare <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64(
 define <2 x double> @fixed_length(<2 x double> %a, <2 x double> %b) nounwind {
 ; CHECK-LABEL: fixed_length:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    # kill: def $v11 killed $v10
 ; CHECK-NEXT:    # kill: def $v9 killed $v8
 ; CHECK-NEXT:    # implicit-def: $v9
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vfadd.vv v9, v8, v10
 ; CHECK-NEXT:    # implicit-def: $v8
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10
@@ -36,9 +36,9 @@ entry:
 define <vscale x 1 x double> @scalable(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: scalable:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    # implicit-def: $v9
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfadd.vv v9, v8, v10
 ; CHECK-NEXT:    # implicit-def: $v8
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10
@@ -53,8 +53,8 @@ entry:
 define <vscale x 1 x double> @intrinsic_same_vlmax(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: intrinsic_same_vlmax:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    # implicit-def: $v9
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    vfadd.vv v9, v8, v10
@@ -81,8 +81,8 @@ entry:
 define <vscale x 1 x double> @intrinsic_same_avl_imm(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: intrinsic_same_avl_imm:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    vsetivli a0, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    # implicit-def: $v9
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    vfadd.vv v9, v8, v10
@@ -108,6 +108,7 @@ entry:
 define <vscale x 1 x double> @intrinsic_same_avl_reg(i64 %avl, <vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: intrinsic_same_avl_reg:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    vsetvli a0, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    # implicit-def: $v9
@@ -135,6 +136,7 @@ entry:
 define <vscale x 1 x double> @intrinsic_diff_avl_reg(i64 %avl, i64 %avl2, <vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: intrinsic_diff_avl_reg:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v10, v9
 ; CHECK-NEXT:    vsetvli a0, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    # implicit-def: $v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
index bf93f5cc1f6f2..55cefbbea81b2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
@@ -835,8 +835,8 @@ body:             |
   ; CHECK-NEXT:   PseudoBR %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_1:%[0-9]+]]:gpr = PseudoVSETVLIX0 $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   $v0 = COPY %mask
-  ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
   ; CHECK-NEXT:   PseudoVSOXEI64_V_M1_MF8_MASK [[COPY]], %b, %idxs, $v0, -1, 3 /* e8 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   PseudoRET
   bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index b0cb6bc6125dd..8b48dc43eca29 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -377,8 +377,8 @@ entry:
 define <vscale x 1 x double> @test19(<vscale x 1 x double> %a, double %b) nounwind {
 ; CHECK-LABEL: test19:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 1, e64, m8, tu, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, tu, ma
 ; CHECK-NEXT:    vfmv.s.f v9, fa0
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfadd.vv v8, v9, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll
index d3b905ef897b1..3c91131fe4d12 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll
@@ -151,11 +151,11 @@ declare <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i8(<vscale x 32 x i8>, <
 define <vscale x 32 x i32> @vsext_nxv32i8_nxv32i32(<vscale x 32 x i8> %a, <vscale x 32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vsext_nxv32i8_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
index 581cc666b6cbd..44d3ee96f5e61 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsitofp-vp.ll
@@ -508,11 +508,11 @@ declare <vscale x 32 x half> @llvm.vp.sitofp.nxv32f16.nxv32i32(<vscale x 32 x i3
 define <vscale x 32 x half> @vsitofp_nxv32f16_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vsitofp_nxv32f16_nxv32i32:
 ; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v24, v0
 ; ZVFH-NEXT:    csrr a1, vlenb
 ; ZVFH-NEXT:    srli a2, a1, 2
 ; ZVFH-NEXT:    slli a1, a1, 1
-; ZVFH-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; ZVFH-NEXT:    vslidedown.vx v0, v0, a2
 ; ZVFH-NEXT:    sub a2, a0, a1
 ; ZVFH-NEXT:    sltu a3, a0, a2
@@ -532,11 +532,11 @@ define <vscale x 32 x half> @vsitofp_nxv32f16_nxv32i32(<vscale x 32 x i32> %va,
 ;
 ; ZVFHMIN-LABEL: vsitofp_nxv32f16_nxv32i32:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    srli a2, a1, 2
 ; ZVFHMIN-NEXT:    slli a1, a1, 1
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v0, v0, a2
 ; ZVFHMIN-NEXT:    sub a2, a0, a1
 ; ZVFHMIN-NEXT:    sltu a3, a0, a2
@@ -566,11 +566,11 @@ declare <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i32(<vscale x 32 x i
 define <vscale x 32 x float> @vsitofp_nxv32f32_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vsitofp_nxv32f32_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
index f9c24eeec31c5..7ee6ea9e19df0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
@@ -590,8 +590,8 @@ declare <vscale x 128 x i8> @llvm.vp.ssub.sat.nxv128i8(<vscale x 128 x i8>, <vsc
 define <vscale x 128 x i8> @vssub_vi_nxv128i8(<vscale x 128 x i8> %va, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssub_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -1392,11 +1392,11 @@ declare <vscale x 32 x i32> @llvm.vp.ssub.sat.nxv32i32(<vscale x 32 x i32>, <vsc
 define <vscale x 32 x i32> @vssub_vi_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssub_vi_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
index 04a1b522a8a33..7674a457ca961 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
@@ -588,8 +588,8 @@ declare <vscale x 128 x i8> @llvm.vp.usub.sat.nxv128i8(<vscale x 128 x i8>, <vsc
 define <vscale x 128 x i8> @vssubu_vi_nxv128i8(<vscale x 128 x i8> %va, <vscale x 128 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssubu_vi_nxv128i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -1390,11 +1390,11 @@ declare <vscale x 32 x i32> @llvm.vp.usub.sat.nxv32i32(<vscale x 32 x i32>, <vsc
 define <vscale x 32 x i32> @vssubu_vi_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vssubu_vi_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vtrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vtrunc-vp.ll
index e62b7a0039638..fd5bf4ebcede8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vtrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vtrunc-vp.ll
@@ -157,11 +157,11 @@ declare <vscale x 15 x i16> @llvm.vp.trunc.nxv15i16.nxv15i64(<vscale x 15 x i64>
 define <vscale x 15 x i16> @vtrunc_nxv15i16_nxv15i64(<vscale x 15 x i64> %a, <vscale x 15 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vtrunc_nxv15i16_nxv15i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    sub a3, a0, a1
-; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sltu a2, a0, a3
 ; CHECK-NEXT:    addi a2, a2, -1
@@ -214,11 +214,11 @@ declare <vscale x 32 x i7> @llvm.vp.trunc.nxv32i7.nxv32i32(<vscale x 32 x i32>,
 define <vscale x 32 x i7> @vtrunc_nxv32i7_nxv32i32(<vscale x 32 x i32> %a, <vscale x 32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vtrunc_nxv32i7_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -248,11 +248,11 @@ declare <vscale x 32 x i8> @llvm.vp.trunc.nxv32i8.nxv32i32(<vscale x 32 x i32>,
 define <vscale x 32 x i8> @vtrunc_nxv32i8_nxv32i32(<vscale x 32 x i32> %a, <vscale x 32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vtrunc_nxv32i8_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
@@ -288,6 +288,7 @@ define <vscale x 32 x i32> @vtrunc_nxv32i64_nxv32i32(<vscale x 32 x i64> %a, <vs
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -301,7 +302,6 @@ define <vscale x 32 x i32> @vtrunc_nxv32i64_nxv32i32(<vscale x 32 x i64> %a, <vs
 ; CHECK-NEXT:    srli a5, a1, 2
 ; CHECK-NEXT:    slli a6, a1, 3
 ; CHECK-NEXT:    slli a4, a1, 1
-; CHECK-NEXT:    vsetvli a7, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v16, v0, a5
 ; CHECK-NEXT:    add a6, a0, a6
 ; CHECK-NEXT:    sub a5, a2, a4
diff --git a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
index 2c5a01279d5d3..f17cdd9c72bfc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vuitofp-vp.ll
@@ -500,11 +500,11 @@ declare <vscale x 32 x half> @llvm.vp.uitofp.nxv32f16.nxv32i32(<vscale x 32 x i3
 define <vscale x 32 x half> @vuitofp_nxv32f16_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vuitofp_nxv32f16_nxv32i32:
 ; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v24, v0
 ; ZVFH-NEXT:    csrr a1, vlenb
 ; ZVFH-NEXT:    srli a2, a1, 2
 ; ZVFH-NEXT:    slli a1, a1, 1
-; ZVFH-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; ZVFH-NEXT:    vslidedown.vx v0, v0, a2
 ; ZVFH-NEXT:    sub a2, a0, a1
 ; ZVFH-NEXT:    sltu a3, a0, a2
@@ -524,11 +524,11 @@ define <vscale x 32 x half> @vuitofp_nxv32f16_nxv32i32(<vscale x 32 x i32> %va,
 ;
 ; ZVFHMIN-LABEL: vuitofp_nxv32f16_nxv32i32:
 ; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v7, v0
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
 ; ZVFHMIN-NEXT:    srli a2, a1, 2
 ; ZVFHMIN-NEXT:    slli a1, a1, 1
-; ZVFHMIN-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslidedown.vx v0, v0, a2
 ; ZVFHMIN-NEXT:    sub a2, a0, a1
 ; ZVFHMIN-NEXT:    sltu a3, a0, a2
@@ -558,11 +558,11 @@ declare <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i32(<vscale x 32 x i
 define <vscale x 32 x float> @vuitofp_nxv32f32_nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vuitofp_nxv32f32_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
new file mode 100644
index 0000000000000..c35f05be304cc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
@@ -0,0 +1,666 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+m \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=RV32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+m -mcpu=sifive-p670 \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=RV64P670
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+m -mcpu=spacemit-x60 \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=RV64X60
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+m \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=RV64
+
+
+; test1
+define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_dst_stride, ptr nocapture noundef readonly %src1, i32 noundef signext %i_src1_stride, ptr nocapture noundef readonly %src2, i32 noundef signext %i_src2_stride, i32 noundef signext %i_width, i32 noundef signext %i_height) {
+; RV32-LABEL: test1:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    blez a7, .LBB0_17
+; RV32-NEXT:  # %bb.1: # %for.cond1.preheader.lr.ph
+; RV32-NEXT:    blez a6, .LBB0_17
+; RV32-NEXT:  # %bb.2: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    addi t0, a7, -1
+; RV32-NEXT:    csrr t2, vlenb
+; RV32-NEXT:    mul t3, a1, t0
+; RV32-NEXT:    mul t4, a3, t0
+; RV32-NEXT:    mul t5, a5, t0
+; RV32-NEXT:    slli t1, t2, 1
+; RV32-NEXT:    li t6, 32
+; RV32-NEXT:    mv t0, t1
+; RV32-NEXT:    bnez zero, .LBB0_4
+; RV32-NEXT:  # %bb.3: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    li t0, 32
+; RV32-NEXT:  .LBB0_4: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s2, 4(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset s0, -4
+; RV32-NEXT:    .cfi_offset s1, -8
+; RV32-NEXT:    .cfi_offset s2, -12
+; RV32-NEXT:    add t3, a0, t3
+; RV32-NEXT:    add t4, a2, t4
+; RV32-NEXT:    add s0, a4, t5
+; RV32-NEXT:    bltu t6, t1, .LBB0_6
+; RV32-NEXT:  # %bb.5: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    li t1, 32
+; RV32-NEXT:  .LBB0_6: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    add t3, t3, a6
+; RV32-NEXT:    add t5, t4, a6
+; RV32-NEXT:    add t4, s0, a6
+; RV32-NEXT:    beqz zero, .LBB0_8
+; RV32-NEXT:  # %bb.7: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    mv t1, t0
+; RV32-NEXT:  .LBB0_8: # %for.cond1.preheader.us.preheader
+; RV32-NEXT:    li t0, 0
+; RV32-NEXT:    sltu t5, a0, t5
+; RV32-NEXT:    sltu t6, a2, t3
+; RV32-NEXT:    and t5, t5, t6
+; RV32-NEXT:    sltu t4, a0, t4
+; RV32-NEXT:    sltu t3, a4, t3
+; RV32-NEXT:    and t3, t4, t3
+; RV32-NEXT:    or t4, a1, a3
+; RV32-NEXT:    slti t4, t4, 0
+; RV32-NEXT:    or t4, t5, t4
+; RV32-NEXT:    or t5, a1, a5
+; RV32-NEXT:    sltu t1, a6, t1
+; RV32-NEXT:    slti t5, t5, 0
+; RV32-NEXT:    or t3, t3, t5
+; RV32-NEXT:    or t3, t4, t3
+; RV32-NEXT:    or t1, t1, t3
+; RV32-NEXT:    andi t1, t1, 1
+; RV32-NEXT:    slli t2, t2, 1
+; RV32-NEXT:    j .LBB0_10
+; RV32-NEXT:  .LBB0_9: # %for.cond1.for.cond.cleanup3_crit_edge.us
+; RV32-NEXT:    # in Loop: Header=BB0_10 Depth=1
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    add a2, a2, a3
+; RV32-NEXT:    addi t0, t0, 1
+; RV32-NEXT:    add a4, a4, a5
+; RV32-NEXT:    beq t0, a7, .LBB0_16
+; RV32-NEXT:  .LBB0_10: # %for.cond1.preheader.us
+; RV32-NEXT:    # =>This Loop Header: Depth=1
+; RV32-NEXT:    # Child Loop BB0_13 Depth 2
+; RV32-NEXT:    # Child Loop BB0_15 Depth 2
+; RV32-NEXT:    beqz t1, .LBB0_12
+; RV32-NEXT:  # %bb.11: # in Loop: Header=BB0_10 Depth=1
+; RV32-NEXT:    li t4, 0
+; RV32-NEXT:    li t3, 0
+; RV32-NEXT:    j .LBB0_15
+; RV32-NEXT:  .LBB0_12: # %vector.ph
+; RV32-NEXT:    # in Loop: Header=BB0_10 Depth=1
+; RV32-NEXT:    li t3, 0
+; RV32-NEXT:    neg t4, t2
+; RV32-NEXT:    and t4, t4, a6
+; RV32-NEXT:    csrwi vxrm, 0
+; RV32-NEXT:    li t6, 0
+; RV32-NEXT:    li t5, 0
+; RV32-NEXT:    vsetvli s0, zero, e8, m2, ta, ma
+; RV32-NEXT:  .LBB0_13: # %vector.body
+; RV32-NEXT:    # Parent Loop BB0_10 Depth=1
+; RV32-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32-NEXT:    add s0, a2, t6
+; RV32-NEXT:    add s1, a4, t6
+; RV32-NEXT:    vl2r.v v8, (s0)
+; RV32-NEXT:    add s0, a0, t6
+; RV32-NEXT:    vl2r.v v10, (s1)
+; RV32-NEXT:    add s1, t6, t2
+; RV32-NEXT:    sltu t6, s1, t6
+; RV32-NEXT:    add t5, t5, t6
+; RV32-NEXT:    xor t6, s1, t4
+; RV32-NEXT:    vaaddu.vv v8, v8, v10
+; RV32-NEXT:    or s2, t6, t5
+; RV32-NEXT:    vs2r.v v8, (s0)
+; RV32-NEXT:    mv t6, s1
+; RV32-NEXT:    bnez s2, .LBB0_13
+; RV32-NEXT:  # %bb.14: # %middle.block
+; RV32-NEXT:    # in Loop: Header=BB0_10 Depth=1
+; RV32-NEXT:    beq t4, a6, .LBB0_9
+; RV32-NEXT:  .LBB0_15: # %for.body4.us
+; RV32-NEXT:    # Parent Loop BB0_10 Depth=1
+; RV32-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32-NEXT:    add t5, a2, t4
+; RV32-NEXT:    add t6, a4, t4
+; RV32-NEXT:    add s0, a0, t4
+; RV32-NEXT:    lbu t5, 0(t5)
+; RV32-NEXT:    lbu t6, 0(t6)
+; RV32-NEXT:    addi t4, t4, 1
+; RV32-NEXT:    seqz s1, t4
+; RV32-NEXT:    add t3, t3, s1
+; RV32-NEXT:    add t5, t5, t6
+; RV32-NEXT:    xor t6, t4, a6
+; RV32-NEXT:    addi t5, t5, 1
+; RV32-NEXT:    srli t5, t5, 1
+; RV32-NEXT:    or t6, t6, t3
+; RV32-NEXT:    sb t5, 0(s0)
+; RV32-NEXT:    bnez t6, .LBB0_15
+; RV32-NEXT:    j .LBB0_9
+; RV32-NEXT:  .LBB0_16:
+; RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s2, 4(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore s0
+; RV32-NEXT:    .cfi_restore s1
+; RV32-NEXT:    .cfi_restore s2
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:  .LBB0_17: # %for.cond.cleanup
+; RV32-NEXT:    ret
+;
+; RV64P670-LABEL: test1:
+; RV64P670:       # %bb.0: # %entry
+; RV64P670-NEXT:    csrwi vxrm, 0
+; RV64P670-NEXT:    blez a7, .LBB0_12
+; RV64P670-NEXT:  # %bb.1: # %for.cond1.preheader.lr.ph
+; RV64P670-NEXT:    blez a6, .LBB0_12
+; RV64P670-NEXT:  # %bb.2: # %for.cond1.preheader.us.preheader
+; RV64P670-NEXT:    addi sp, sp, -48
+; RV64P670-NEXT:    .cfi_def_cfa_offset 48
+; RV64P670-NEXT:    sd s0, 40(sp) # 8-byte Folded Spill
+; RV64P670-NEXT:    sd s1, 32(sp) # 8-byte Folded Spill
+; RV64P670-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64P670-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64P670-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64P670-NEXT:    .cfi_offset s0, -8
+; RV64P670-NEXT:    .cfi_offset s1, -16
+; RV64P670-NEXT:    .cfi_offset s2, -24
+; RV64P670-NEXT:    .cfi_offset s3, -32
+; RV64P670-NEXT:    .cfi_offset s4, -40
+; RV64P670-NEXT:    addi s1, a7, -1
+; RV64P670-NEXT:    add s0, a0, a6
+; RV64P670-NEXT:    li t0, 0
+; RV64P670-NEXT:    li t1, 0
+; RV64P670-NEXT:    zext.w s1, s1
+; RV64P670-NEXT:    mul t2, a1, s1
+; RV64P670-NEXT:    add t4, s0, t2
+; RV64P670-NEXT:    mul t2, a3, s1
+; RV64P670-NEXT:    add s0, a2, a6
+; RV64P670-NEXT:    mul s1, a5, s1
+; RV64P670-NEXT:    add t3, s0, t2
+; RV64P670-NEXT:    add s0, a4, a6
+; RV64P670-NEXT:    csrr t2, vlenb
+; RV64P670-NEXT:    add t5, s0, s1
+; RV64P670-NEXT:    sltu s1, a0, t3
+; RV64P670-NEXT:    sltu s0, a2, t4
+; RV64P670-NEXT:    slli t3, t2, 1
+; RV64P670-NEXT:    and s0, s0, s1
+; RV64P670-NEXT:    or s1, a1, a3
+; RV64P670-NEXT:    slti s1, s1, 0
+; RV64P670-NEXT:    or t6, s0, s1
+; RV64P670-NEXT:    sltu s1, a0, t5
+; RV64P670-NEXT:    sltu s0, a4, t4
+; RV64P670-NEXT:    mv t5, a0
+; RV64P670-NEXT:    and s0, s0, s1
+; RV64P670-NEXT:    or s1, a1, a5
+; RV64P670-NEXT:    slti s1, s1, 0
+; RV64P670-NEXT:    or s0, s0, s1
+; RV64P670-NEXT:    li s1, 32
+; RV64P670-NEXT:    maxu s1, t3, s1
+; RV64P670-NEXT:    or s0, t6, s0
+; RV64P670-NEXT:    sltu s1, a6, s1
+; RV64P670-NEXT:    or s0, s0, s1
+; RV64P670-NEXT:    andi t4, s0, 1
+; RV64P670-NEXT:    j .LBB0_4
+; RV64P670-NEXT:  .LBB0_3: # %for.cond1.for.cond.cleanup3_crit_edge.us
+; RV64P670-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64P670-NEXT:    add t5, t5, a1
+; RV64P670-NEXT:    add a2, a2, a3
+; RV64P670-NEXT:    add a4, a4, a5
+; RV64P670-NEXT:    addiw t1, t1, 1
+; RV64P670-NEXT:    addi t0, t0, 1
+; RV64P670-NEXT:    beq t1, a7, .LBB0_11
+; RV64P670-NEXT:  .LBB0_4: # %for.cond1.preheader.us
+; RV64P670-NEXT:    # =>This Loop Header: Depth=1
+; RV64P670-NEXT:    # Child Loop BB0_7 Depth 2
+; RV64P670-NEXT:    # Child Loop BB0_10 Depth 2
+; RV64P670-NEXT:    beqz t4, .LBB0_6
+; RV64P670-NEXT:  # %bb.5: # in Loop: Header=BB0_4 Depth=1
+; RV64P670-NEXT:    li t6, 0
+; RV64P670-NEXT:    j .LBB0_9
+; RV64P670-NEXT:  .LBB0_6: # %vector.ph
+; RV64P670-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64P670-NEXT:    slli s1, t2, 28
+; RV64P670-NEXT:    mv s2, a2
+; RV64P670-NEXT:    mv s3, a4
+; RV64P670-NEXT:    mv s4, t5
+; RV64P670-NEXT:    sub s1, s1, t3
+; RV64P670-NEXT:    vsetvli s0, zero, e8, m2, ta, ma
+; RV64P670-NEXT:    and t6, s1, a6
+; RV64P670-NEXT:    mv s1, t6
+; RV64P670-NEXT:  .LBB0_7: # %vector.body
+; RV64P670-NEXT:    # Parent Loop BB0_4 Depth=1
+; RV64P670-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64P670-NEXT:    vl2r.v v8, (s2)
+; RV64P670-NEXT:    sub s1, s1, t3
+; RV64P670-NEXT:    add s2, s2, t3
+; RV64P670-NEXT:    vl2r.v v10, (s3)
+; RV64P670-NEXT:    add s3, s3, t3
+; RV64P670-NEXT:    vaaddu.vv v8, v8, v10
+; RV64P670-NEXT:    vs2r.v v8, (s4)
+; RV64P670-NEXT:    add s4, s4, t3
+; RV64P670-NEXT:    bnez s1, .LBB0_7
+; RV64P670-NEXT:  # %bb.8: # %middle.block
+; RV64P670-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64P670-NEXT:    beq t6, a6, .LBB0_3
+; RV64P670-NEXT:  .LBB0_9: # %for.body4.us.preheader
+; RV64P670-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64P670-NEXT:    mul s2, a1, t0
+; RV64P670-NEXT:    add s0, a0, a6
+; RV64P670-NEXT:    add s1, t5, t6
+; RV64P670-NEXT:    add s4, a4, t6
+; RV64P670-NEXT:    add t6, t6, a2
+; RV64P670-NEXT:    add s2, s2, s0
+; RV64P670-NEXT:  .LBB0_10: # %for.body4.us
+; RV64P670-NEXT:    # Parent Loop BB0_4 Depth=1
+; RV64P670-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64P670-NEXT:    lbu s3, 0(t6)
+; RV64P670-NEXT:    lbu s0, 0(s4)
+; RV64P670-NEXT:    addi s4, s4, 1
+; RV64P670-NEXT:    addi t6, t6, 1
+; RV64P670-NEXT:    add s0, s0, s3
+; RV64P670-NEXT:    addi s0, s0, 1
+; RV64P670-NEXT:    srli s0, s0, 1
+; RV64P670-NEXT:    sb s0, 0(s1)
+; RV64P670-NEXT:    addi s1, s1, 1
+; RV64P670-NEXT:    bne s1, s2, .LBB0_10
+; RV64P670-NEXT:    j .LBB0_3
+; RV64P670-NEXT:  .LBB0_11:
+; RV64P670-NEXT:    ld s0, 40(sp) # 8-byte Folded Reload
+; RV64P670-NEXT:    ld s1, 32(sp) # 8-byte Folded Reload
+; RV64P670-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64P670-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64P670-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64P670-NEXT:    .cfi_restore s0
+; RV64P670-NEXT:    .cfi_restore s1
+; RV64P670-NEXT:    .cfi_restore s2
+; RV64P670-NEXT:    .cfi_restore s3
+; RV64P670-NEXT:    .cfi_restore s4
+; RV64P670-NEXT:    addi sp, sp, 48
+; RV64P670-NEXT:    .cfi_def_cfa_offset 0
+; RV64P670-NEXT:  .LBB0_12: # %for.cond.cleanup
+; RV64P670-NEXT:    ret
+;
+; RV64X60-LABEL: test1:
+; RV64X60:       # %bb.0: # %entry
+; RV64X60-NEXT:    csrwi vxrm, 0
+; RV64X60-NEXT:    blez a7, .LBB0_12
+; RV64X60-NEXT:  # %bb.1: # %for.cond1.preheader.lr.ph
+; RV64X60-NEXT:    blez a6, .LBB0_12
+; RV64X60-NEXT:  # %bb.2: # %for.cond1.preheader.us.preheader
+; RV64X60-NEXT:    addi sp, sp, -48
+; RV64X60-NEXT:    .cfi_def_cfa_offset 48
+; RV64X60-NEXT:    sd s0, 40(sp) # 8-byte Folded Spill
+; RV64X60-NEXT:    sd s1, 32(sp) # 8-byte Folded Spill
+; RV64X60-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64X60-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64X60-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64X60-NEXT:    .cfi_offset s0, -8
+; RV64X60-NEXT:    .cfi_offset s1, -16
+; RV64X60-NEXT:    .cfi_offset s2, -24
+; RV64X60-NEXT:    .cfi_offset s3, -32
+; RV64X60-NEXT:    .cfi_offset s4, -40
+; RV64X60-NEXT:    li t0, 0
+; RV64X60-NEXT:    li t1, 0
+; RV64X60-NEXT:    addi t2, a7, -1
+; RV64X60-NEXT:    add t4, a0, a6
+; RV64X60-NEXT:    add t5, a2, a6
+; RV64X60-NEXT:    add t3, a4, a6
+; RV64X60-NEXT:    zext.w s0, t2
+; RV64X60-NEXT:    mul s1, a1, s0
+; RV64X60-NEXT:    add t4, t4, s1
+; RV64X60-NEXT:    mul s1, a3, s0
+; RV64X60-NEXT:    add t5, t5, s1
+; RV64X60-NEXT:    csrr t2, vlenb
+; RV64X60-NEXT:    mul s1, a5, s0
+; RV64X60-NEXT:    add t3, t3, s1
+; RV64X60-NEXT:    sltu s1, a0, t5
+; RV64X60-NEXT:    sltu s0, a2, t4
+; RV64X60-NEXT:    and t6, s1, s0
+; RV64X60-NEXT:    li t5, 32
+; RV64X60-NEXT:    sltu s1, a0, t3
+; RV64X60-NEXT:    sltu s0, a4, t4
+; RV64X60-NEXT:    and t3, s1, s0
+; RV64X60-NEXT:    or s1, a1, a3
+; RV64X60-NEXT:    slti s1, s1, 0
+; RV64X60-NEXT:    or t4, t6, s1
+; RV64X60-NEXT:    or s0, a1, a5
+; RV64X60-NEXT:    slti s0, s0, 0
+; RV64X60-NEXT:    or s0, t3, s0
+; RV64X60-NEXT:    slli t3, t2, 1
+; RV64X60-NEXT:    maxu s1, t3, t5
+; RV64X60-NEXT:    or s0, t4, s0
+; RV64X60-NEXT:    sltu s1, a6, s1
+; RV64X60-NEXT:    or s0, s0, s1
+; RV64X60-NEXT:    andi t4, s0, 1
+; RV64X60-NEXT:    mv t5, a0
+; RV64X60-NEXT:    j .LBB0_4
+; RV64X60-NEXT:  .LBB0_3: # %for.cond1.for.cond.cleanup3_crit_edge.us
+; RV64X60-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64X60-NEXT:    add t5, t5, a1
+; RV64X60-NEXT:    add a2, a2, a3
+; RV64X60-NEXT:    add a4, a4, a5
+; RV64X60-NEXT:    addiw t1, t1, 1
+; RV64X60-NEXT:    addi t0, t0, 1
+; RV64X60-NEXT:    beq t1, a7, .LBB0_11
+; RV64X60-NEXT:  .LBB0_4: # %for.cond1.preheader.us
+; RV64X60-NEXT:    # =>This Loop Header: Depth=1
+; RV64X60-NEXT:    # Child Loop BB0_7 Depth 2
+; RV64X60-NEXT:    # Child Loop BB0_10 Depth 2
+; RV64X60-NEXT:    beqz t4, .LBB0_6
+; RV64X60-NEXT:  # %bb.5: # in Loop: Header=BB0_4 Depth=1
+; RV64X60-NEXT:    li t6, 0
+; RV64X60-NEXT:    j .LBB0_9
+; RV64X60-NEXT:  .LBB0_6: # %vector.ph
+; RV64X60-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64X60-NEXT:    slli s1, t2, 28
+; RV64X60-NEXT:    sub s1, s1, t3
+; RV64X60-NEXT:    and t6, s1, a6
+; RV64X60-NEXT:    mv s2, a2
+; RV64X60-NEXT:    mv s3, a4
+; RV64X60-NEXT:    mv s4, t5
+; RV64X60-NEXT:    mv s1, t6
+; RV64X60-NEXT:    vsetvli s0, zero, e8, m2, ta, ma
+; RV64X60-NEXT:  .LBB0_7: # %vector.body
+; RV64X60-NEXT:    # Parent Loop BB0_4 Depth=1
+; RV64X60-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64X60-NEXT:    vl2r.v v8, (s2)
+; RV64X60-NEXT:    vl2r.v v10, (s3)
+; RV64X60-NEXT:    sub s1, s1, t3
+; RV64X60-NEXT:    add s3, s3, t3
+; RV64X60-NEXT:    vaaddu.vv v8, v8, v10
+; RV64X60-NEXT:    vs2r.v v8, (s4)
+; RV64X60-NEXT:    add s4, s4, t3
+; RV64X60-NEXT:    add s2, s2, t3
+; RV64X60-NEXT:    bnez s1, .LBB0_7
+; RV64X60-NEXT:  # %bb.8: # %middle.block
+; RV64X60-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64X60-NEXT:    beq t6, a6, .LBB0_3
+; RV64X60-NEXT:  .LBB0_9: # %for.body4.us.preheader
+; RV64X60-NEXT:    # in Loop: Header=BB0_4 Depth=1
+; RV64X60-NEXT:    mul s2, a1, t0
+; RV64X60-NEXT:    add s1, a0, a6
+; RV64X60-NEXT:    add s0, t5, t6
+; RV64X60-NEXT:    add s2, s2, s1
+; RV64X60-NEXT:    add s4, a4, t6
+; RV64X60-NEXT:    add t6, t6, a2
+; RV64X60-NEXT:  .LBB0_10: # %for.body4.us
+; RV64X60-NEXT:    # Parent Loop BB0_4 Depth=1
+; RV64X60-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64X60-NEXT:    lbu s3, 0(t6)
+; RV64X60-NEXT:    lbu s1, 0(s4)
+; RV64X60-NEXT:    add s1, s1, s3
+; RV64X60-NEXT:    addi s1, s1, 1
+; RV64X60-NEXT:    srli s1, s1, 1
+; RV64X60-NEXT:    sb s1, 0(s0)
+; RV64X60-NEXT:    addi s0, s0, 1
+; RV64X60-NEXT:    addi s4, s4, 1
+; RV64X60-NEXT:    addi t6, t6, 1
+; RV64X60-NEXT:    bne s0, s2, .LBB0_10
+; RV64X60-NEXT:    j .LBB0_3
+; RV64X60-NEXT:  .LBB0_11:
+; RV64X60-NEXT:    ld s0, 40(sp) # 8-byte Folded Reload
+; RV64X60-NEXT:    ld s1, 32(sp) # 8-byte Folded Reload
+; RV64X60-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64X60-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64X60-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64X60-NEXT:    .cfi_restore s0
+; RV64X60-NEXT:    .cfi_restore s1
+; RV64X60-NEXT:    .cfi_restore s2
+; RV64X60-NEXT:    .cfi_restore s3
+; RV64X60-NEXT:    .cfi_restore s4
+; RV64X60-NEXT:    addi sp, sp, 48
+; RV64X60-NEXT:    .cfi_def_cfa_offset 0
+; RV64X60-NEXT:  .LBB0_12: # %for.cond.cleanup
+; RV64X60-NEXT:    ret
+;
+; RV64-LABEL: test1:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    blez a7, .LBB0_14
+; RV64-NEXT:  # %bb.1: # %for.cond1.preheader.lr.ph
+; RV64-NEXT:    blez a6, .LBB0_14
+; RV64-NEXT:  # %bb.2: # %for.cond1.preheader.us.preheader
+; RV64-NEXT:    addi sp, sp, -48
+; RV64-NEXT:    .cfi_def_cfa_offset 48
+; RV64-NEXT:    sd s0, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s1, 32(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset s0, -8
+; RV64-NEXT:    .cfi_offset s1, -16
+; RV64-NEXT:    .cfi_offset s2, -24
+; RV64-NEXT:    .cfi_offset s3, -32
+; RV64-NEXT:    .cfi_offset s4, -40
+; RV64-NEXT:    addi t1, a7, -1
+; RV64-NEXT:    add t5, a0, a6
+; RV64-NEXT:    add s0, a2, a6
+; RV64-NEXT:    add t6, a4, a6
+; RV64-NEXT:    csrr t0, vlenb
+; RV64-NEXT:    li t2, 32
+; RV64-NEXT:    slli t1, t1, 32
+; RV64-NEXT:    srli t3, t1, 32
+; RV64-NEXT:    mul t1, a1, t3
+; RV64-NEXT:    add t5, t5, t1
+; RV64-NEXT:    mul t1, a3, t3
+; RV64-NEXT:    add s0, s0, t1
+; RV64-NEXT:    slli t1, t0, 1
+; RV64-NEXT:    mul t3, a5, t3
+; RV64-NEXT:    add t6, t6, t3
+; RV64-NEXT:    mv t4, t1
+; RV64-NEXT:    bltu t2, t1, .LBB0_4
+; RV64-NEXT:  # %bb.3: # %for.cond1.preheader.us.preheader
+; RV64-NEXT:    li t4, 32
+; RV64-NEXT:  .LBB0_4: # %for.cond1.preheader.us.preheader
+; RV64-NEXT:    li t2, 0
+; RV64-NEXT:    li t3, 0
+; RV64-NEXT:    sltu s0, a0, s0
+; RV64-NEXT:    sltu s1, a2, t5
+; RV64-NEXT:    and s0, s0, s1
+; RV64-NEXT:    sltu t6, a0, t6
+; RV64-NEXT:    sltu t5, a4, t5
+; RV64-NEXT:    and t5, t6, t5
+; RV64-NEXT:    or t6, a1, a3
+; RV64-NEXT:    slti t6, t6, 0
+; RV64-NEXT:    or t6, s0, t6
+; RV64-NEXT:    or s0, a1, a5
+; RV64-NEXT:    slti s0, s0, 0
+; RV64-NEXT:    or t5, t5, s0
+; RV64-NEXT:    or t5, t6, t5
+; RV64-NEXT:    sltu t4, a6, t4
+; RV64-NEXT:    or t4, t4, t5
+; RV64-NEXT:    andi t4, t4, 1
+; RV64-NEXT:    mv t5, a0
+; RV64-NEXT:    j .LBB0_6
+; RV64-NEXT:  .LBB0_5: # %for.cond1.for.cond.cleanup3_crit_edge.us
+; RV64-NEXT:    # in Loop: Header=BB0_6 Depth=1
+; RV64-NEXT:    add t5, t5, a1
+; RV64-NEXT:    add a2, a2, a3
+; RV64-NEXT:    add a4, a4, a5
+; RV64-NEXT:    addiw t3, t3, 1
+; RV64-NEXT:    addi t2, t2, 1
+; RV64-NEXT:    beq t3, a7, .LBB0_13
+; RV64-NEXT:  .LBB0_6: # %for.cond1.preheader.us
+; RV64-NEXT:    # =>This Loop Header: Depth=1
+; RV64-NEXT:    # Child Loop BB0_9 Depth 2
+; RV64-NEXT:    # Child Loop BB0_12 Depth 2
+; RV64-NEXT:    beqz t4, .LBB0_8
+; RV64-NEXT:  # %bb.7: # in Loop: Header=BB0_6 Depth=1
+; RV64-NEXT:    li t6, 0
+; RV64-NEXT:    j .LBB0_11
+; RV64-NEXT:  .LBB0_8: # %vector.ph
+; RV64-NEXT:    # in Loop: Header=BB0_6 Depth=1
+; RV64-NEXT:    slli t6, t0, 28
+; RV64-NEXT:    sub t6, t6, t1
+; RV64-NEXT:    and t6, t6, a6
+; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    mv s0, a2
+; RV64-NEXT:    mv s1, a4
+; RV64-NEXT:    mv s2, t5
+; RV64-NEXT:    mv s3, t6
+; RV64-NEXT:    vsetvli s4, zero, e8, m2, ta, ma
+; RV64-NEXT:  .LBB0_9: # %vector.body
+; RV64-NEXT:    # Parent Loop BB0_6 Depth=1
+; RV64-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64-NEXT:    vl2r.v v8, (s0)
+; RV64-NEXT:    vl2r.v v10, (s1)
+; RV64-NEXT:    sub s3, s3, t1
+; RV64-NEXT:    add s1, s1, t1
+; RV64-NEXT:    vaaddu.vv v8, v8, v10
+; RV64-NEXT:    vs2r.v v8, (s2)
+; RV64-NEXT:    add s2, s2, t1
+; RV64-NEXT:    add s0, s0, t1
+; RV64-NEXT:    bnez s3, .LBB0_9
+; RV64-NEXT:  # %bb.10: # %middle.block
+; RV64-NEXT:    # in Loop: Header=BB0_6 Depth=1
+; RV64-NEXT:    beq t6, a6, .LBB0_5
+; RV64-NEXT:  .LBB0_11: # %for.body4.us.preheader
+; RV64-NEXT:    # in Loop: Header=BB0_6 Depth=1
+; RV64-NEXT:    mul s1, a1, t2
+; RV64-NEXT:    add s2, a0, a6
+; RV64-NEXT:    add s0, t5, t6
+; RV64-NEXT:    add s1, s2, s1
+; RV64-NEXT:    add s2, a4, t6
+; RV64-NEXT:    add t6, a2, t6
+; RV64-NEXT:  .LBB0_12: # %for.body4.us
+; RV64-NEXT:    # Parent Loop BB0_6 Depth=1
+; RV64-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64-NEXT:    lbu s3, 0(t6)
+; RV64-NEXT:    lbu s4, 0(s2)
+; RV64-NEXT:    add s3, s3, s4
+; RV64-NEXT:    addi s3, s3, 1
+; RV64-NEXT:    srli s3, s3, 1
+; RV64-NEXT:    sb s3, 0(s0)
+; RV64-NEXT:    addi s0, s0, 1
+; RV64-NEXT:    addi s2, s2, 1
+; RV64-NEXT:    addi t6, t6, 1
+; RV64-NEXT:    bne s0, s1, .LBB0_12
+; RV64-NEXT:    j .LBB0_5
+; RV64-NEXT:  .LBB0_13:
+; RV64-NEXT:    ld s0, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s1, 32(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore s0
+; RV64-NEXT:    .cfi_restore s1
+; RV64-NEXT:    .cfi_restore s2
+; RV64-NEXT:    .cfi_restore s3
+; RV64-NEXT:    .cfi_restore s4
+; RV64-NEXT:    addi sp, sp, 48
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:  .LBB0_14: # %for.cond.cleanup
+; RV64-NEXT:    ret
+entry:
+  %cmp29 = icmp sgt i32 %i_height, 0
+  br i1 %cmp29, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp227 = icmp sgt i32 %i_width, 0
+  %idx.ext = sext i32 %i_dst_stride to i64
+  %idx.ext12 = sext i32 %i_src1_stride to i64
+  %idx.ext14 = sext i32 %i_src2_stride to i64
+  br i1 %cmp227, label %for.cond1.preheader.us.preheader, label %for.cond.cleanup
+
+for.cond1.preheader.us.preheader:                 ; preds = %for.cond1.preheader.lr.ph
+  %wide.trip.count = zext nneg i32 %i_width to i64
+  %0 = add nsw i32 %i_height, -1
+  %1 = zext i32 %0 to i64
+  %2 = mul nsw i64 %idx.ext, %1
+  %3 = getelementptr i8, ptr %dst, i64 %2
+  %scevgep = getelementptr i8, ptr %3, i64 %wide.trip.count
+  %4 = mul nsw i64 %idx.ext12, %1
+  %5 = getelementptr i8, ptr %src1, i64 %4
+  %scevgep36 = getelementptr i8, ptr %5, i64 %wide.trip.count
+  %6 = mul nsw i64 %idx.ext14, %1
+  %7 = getelementptr i8, ptr %src2, i64 %6
+  %scevgep37 = getelementptr i8, ptr %7, i64 %wide.trip.count
+  %8 = tail call i64 @llvm.vscale.i64()
+  %9 = shl nuw nsw i64 %8, 4
+  %10 = tail call i64 @llvm.umax.i64(i64 %9, i64 32)
+  %min.iters.check = icmp ugt i64 %10, %wide.trip.count
+  %bound0 = icmp ult ptr %dst, %scevgep36
+  %bound1 = icmp ult ptr %src1, %scevgep
+  %found.conflict = and i1 %bound0, %bound1
+  %11 = or i32 %i_dst_stride, %i_src1_stride
+  %12 = icmp slt i32 %11, 0
+  %13 = or i1 %found.conflict, %12
+  %bound039 = icmp ult ptr %dst, %scevgep37
+  %bound140 = icmp ult ptr %src2, %scevgep
+  %found.conflict41 = and i1 %bound039, %bound140
+  %14 = or i32 %i_dst_stride, %i_src2_stride
+  %15 = icmp slt i32 %14, 0
+  %16 = or i1 %found.conflict41, %15
+  %conflict.rdx = or i1 %13, %16
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.preheader.us.preheader, %for.cond1.for.cond.cleanup3_crit_edge.us
+  %y.033.us = phi i32 [ %inc17.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %dst.addr.032.us = phi ptr [ %add.ptr.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %dst, %for.cond1.preheader.us.preheader ]
+  %src1.addr.031.us = phi ptr [ %add.ptr13.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %src1, %for.cond1.preheader.us.preheader ]
+  %src2.addr.030.us = phi ptr [ %add.ptr15.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %src2, %for.cond1.preheader.us.preheader ]
+  %brmerge = select i1 %min.iters.check, i1 true, i1 %conflict.rdx
+  br i1 %brmerge, label %for.body4.us.preheader, label %vector.ph
+
+vector.ph:                                        ; preds = %for.cond1.preheader.us
+  %17 = tail call i64 @llvm.vscale.i64()
+  %.neg = mul nuw nsw i64 %17, 2147483632
+  %n.vec = and i64 %.neg, %wide.trip.count
+  %18 = tail call i64 @llvm.vscale.i64()
+  %19 = shl nuw nsw i64 %18, 4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %20 = getelementptr inbounds i8, ptr %src1.addr.031.us, i64 %index
+  %wide.load = load <vscale x 16 x i8>, ptr %20, align 1
+  %21 = zext <vscale x 16 x i8> %wide.load to <vscale x 16 x i16>
+  %22 = getelementptr inbounds i8, ptr %src2.addr.030.us, i64 %index
+  %wide.load44 = load <vscale x 16 x i8>, ptr %22, align 1
+  %23 = zext <vscale x 16 x i8> %wide.load44 to <vscale x 16 x i16>
+  %24 = add nuw nsw <vscale x 16 x i16> %21, shufflevector (<vscale x 16 x i16> insertelement (<vscale x 16 x i16> poison, i16 1, i64 0), <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer)
+  %25 = add nuw nsw <vscale x 16 x i16> %24, %23
+  %26 = lshr <vscale x 16 x i16> %25, shufflevector (<vscale x 16 x i16> insertelement (<vscale x 16 x i16> poison, i16 1, i64 0), <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer)
+  %27 = trunc <vscale x 16 x i16> %26 to <vscale x 16 x i8>
+  %28 = getelementptr inbounds i8, ptr %dst.addr.032.us, i64 %index
+  store <vscale x 16 x i8> %27, ptr %28, align 1
+  %index.next = add nuw i64 %index, %19
+  %29 = icmp eq i64 %index.next, %n.vec
+  br i1 %29, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
+  br i1 %cmp.n, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.preheader
+
+for.body4.us.preheader:                           ; preds = %for.cond1.preheader.us, %middle.block
+  %indvars.iv.ph = phi i64 [ 0, %for.cond1.preheader.us ], [ %n.vec, %middle.block ]
+  br label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us.preheader, %for.body4.us
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body4.us ], [ %indvars.iv.ph, %for.body4.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i8, ptr %src1.addr.031.us, i64 %indvars.iv
+  %30 = load i8, ptr %arrayidx.us, align 1
+  %conv.us = zext i8 %30 to i16
+  %arrayidx6.us = getelementptr inbounds i8, ptr %src2.addr.030.us, i64 %indvars.iv
+  %31 = load i8, ptr %arrayidx6.us, align 1
+  %conv7.us = zext i8 %31 to i16
+  %add.us = add nuw nsw i16 %conv.us, 1
+  %add8.us = add nuw nsw i16 %add.us, %conv7.us
+  %shr.us = lshr i16 %add8.us, 1
+  %conv9.us = trunc nuw i16 %shr.us to i8
+  %arrayidx11.us = getelementptr inbounds i8, ptr %dst.addr.032.us, i64 %indvars.iv
+  store i8 %conv9.us, ptr %arrayidx11.us, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us, %middle.block
+  %add.ptr.us = getelementptr inbounds i8, ptr %dst.addr.032.us, i64 %idx.ext
+  %add.ptr13.us = getelementptr inbounds i8, ptr %src1.addr.031.us, i64 %idx.ext12
+  %add.ptr15.us = getelementptr inbounds i8, ptr %src2.addr.030.us, i64 %idx.ext14
+  %inc17.us = add nuw nsw i32 %y.033.us, 1
+  %exitcond35.not = icmp eq i32 %inc17.us, %i_height
+  br i1 %exitcond35.not, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll
index 10e655c844540..934d7eb43ac2a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll
@@ -151,11 +151,11 @@ declare <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i8(<vscale x 32 x i8>, <
 define <vscale x 32 x i32> @vzext_nxv32i8_nxv32i32(<vscale x 32 x i8> %a, <vscale x 32 x i1> %m, i32 zeroext %vl) {
 ; CHECK-LABEL: vzext_nxv32i8_nxv32i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v12, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 2
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/shrinkwrap.ll b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
index 40577701e1e24..90f9509c72373 100644
--- a/llvm/test/CodeGen/RISCV/shrinkwrap.ll
+++ b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
@@ -163,3 +163,462 @@ if.then:
 if.end:
   ret void
 }
+
+; FIXME: Rematerialize "li s1, 57" for the second use instead of spilling s1,
+; and then shrink wrap the result avoiding frame setup on the %bb.0 -> %exit
+; edge.
+define void @li_straightline_a(i32 zeroext %a, i32 zeroext %b) {
+; RV32I-SW-NO-LABEL: li_straightline_a:
+; RV32I-SW-NO:       # %bb.0:
+; RV32I-SW-NO-NEXT:    addi sp, sp, -16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NO-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NO-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NO-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-NO-NEXT:    li s1, 57
+; RV32I-SW-NO-NEXT:    beq a0, s1, .LBB2_3
+; RV32I-SW-NO-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NO-NEXT:    mv s0, a1
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:    beq s0, s1, .LBB2_3
+; RV32I-SW-NO-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:  .LBB2_3: # %exit
+; RV32I-SW-NO-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    .cfi_restore ra
+; RV32I-SW-NO-NEXT:    .cfi_restore s0
+; RV32I-SW-NO-NEXT:    .cfi_restore s1
+; RV32I-SW-NO-NEXT:    addi sp, sp, 16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NO-NEXT:    ret
+;
+; RV32I-SW-LABEL: li_straightline_a:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-NEXT:    li s1, 57
+; RV32I-SW-NEXT:    beq a0, s1, .LBB2_3
+; RV32I-SW-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NEXT:    mv s0, a1
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:    beq s0, s1, .LBB2_3
+; RV32I-SW-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:  .LBB2_3: # %exit
+; RV32I-SW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    .cfi_restore ra
+; RV32I-SW-NEXT:    .cfi_restore s0
+; RV32I-SW-NEXT:    .cfi_restore s1
+; RV32I-SW-NEXT:    addi sp, sp, 16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NEXT:    ret
+;
+; RV32I-SW-SR-LABEL: li_straightline_a:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_2
+; RV32I-SW-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-SR-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-SR-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-SR-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-SR-NEXT:    li s1, 57
+; RV32I-SW-SR-NEXT:    beq a0, s1, .LBB2_3
+; RV32I-SW-SR-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-SR-NEXT:    mv s0, a1
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:    beq s0, s1, .LBB2_3
+; RV32I-SW-SR-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:  .LBB2_3: # %exit
+; RV32I-SW-SR-NEXT:    tail __riscv_restore_2
+;
+; RV64I-SW-LABEL: li_straightline_a:
+; RV64I-SW:       # %bb.0:
+; RV64I-SW-NEXT:    addi sp, sp, -32
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 32
+; RV64I-SW-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    .cfi_offset ra, -8
+; RV64I-SW-NEXT:    .cfi_offset s0, -16
+; RV64I-SW-NEXT:    .cfi_offset s1, -24
+; RV64I-SW-NEXT:    li s1, 57
+; RV64I-SW-NEXT:    beq a0, s1, .LBB2_3
+; RV64I-SW-NEXT:  # %bb.1: # %do_call
+; RV64I-SW-NEXT:    mv s0, a1
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:    beq s0, s1, .LBB2_3
+; RV64I-SW-NEXT:  # %bb.2: # %do_call2
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:  .LBB2_3: # %exit
+; RV64I-SW-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    .cfi_restore ra
+; RV64I-SW-NEXT:    .cfi_restore s0
+; RV64I-SW-NEXT:    .cfi_restore s1
+; RV64I-SW-NEXT:    addi sp, sp, 32
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-SW-NEXT:    ret
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  call i32 @foo()
+  %cmp1 = icmp eq i32 %b, 57
+  br i1 %cmp1, label %exit, label %do_call2
+do_call2:
+  call i32 @foo()
+  br label %exit
+exit:
+  ret void
+}
+
+; FIXME: The "mv s0, a1" is only required along the do_call path, and can
+; be sunk there.  We can also shrink wrap to avoid the frame setup along
+; the %bb.0 -> %exit edge.
+define void @li_straightline_b(i32 zeroext %a, i32 zeroext %b) {
+; RV32I-SW-NO-LABEL: li_straightline_b:
+; RV32I-SW-NO:       # %bb.0:
+; RV32I-SW-NO-NEXT:    addi sp, sp, -16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NO-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NO-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NO-NEXT:    mv s0, a1
+; RV32I-SW-NO-NEXT:    li a1, 57
+; RV32I-SW-NO-NEXT:    beq a0, a1, .LBB3_3
+; RV32I-SW-NO-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:    li a0, 57
+; RV32I-SW-NO-NEXT:    beq s0, a0, .LBB3_3
+; RV32I-SW-NO-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:  .LBB3_3: # %exit
+; RV32I-SW-NO-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    .cfi_restore ra
+; RV32I-SW-NO-NEXT:    .cfi_restore s0
+; RV32I-SW-NO-NEXT:    addi sp, sp, 16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NO-NEXT:    ret
+;
+; RV32I-SW-LABEL: li_straightline_b:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NEXT:    mv s0, a1
+; RV32I-SW-NEXT:    li a1, 57
+; RV32I-SW-NEXT:    beq a0, a1, .LBB3_3
+; RV32I-SW-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:    li a0, 57
+; RV32I-SW-NEXT:    beq s0, a0, .LBB3_3
+; RV32I-SW-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:  .LBB3_3: # %exit
+; RV32I-SW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    .cfi_restore ra
+; RV32I-SW-NEXT:    .cfi_restore s0
+; RV32I-SW-NEXT:    addi sp, sp, 16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NEXT:    ret
+;
+; RV32I-SW-SR-LABEL: li_straightline_b:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_1
+; RV32I-SW-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-SR-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-SR-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-SR-NEXT:    mv s0, a1
+; RV32I-SW-SR-NEXT:    li a1, 57
+; RV32I-SW-SR-NEXT:    beq a0, a1, .LBB3_3
+; RV32I-SW-SR-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:    li a0, 57
+; RV32I-SW-SR-NEXT:    beq s0, a0, .LBB3_3
+; RV32I-SW-SR-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:  .LBB3_3: # %exit
+; RV32I-SW-SR-NEXT:    tail __riscv_restore_1
+;
+; RV64I-SW-LABEL: li_straightline_b:
+; RV64I-SW:       # %bb.0:
+; RV64I-SW-NEXT:    addi sp, sp, -16
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV64I-SW-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    .cfi_offset ra, -8
+; RV64I-SW-NEXT:    .cfi_offset s0, -16
+; RV64I-SW-NEXT:    mv s0, a1
+; RV64I-SW-NEXT:    li a1, 57
+; RV64I-SW-NEXT:    beq a0, a1, .LBB3_3
+; RV64I-SW-NEXT:  # %bb.1: # %do_call
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:    li a0, 57
+; RV64I-SW-NEXT:    beq s0, a0, .LBB3_3
+; RV64I-SW-NEXT:  # %bb.2: # %do_call2
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:  .LBB3_3: # %exit
+; RV64I-SW-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    .cfi_restore ra
+; RV64I-SW-NEXT:    .cfi_restore s0
+; RV64I-SW-NEXT:    addi sp, sp, 16
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-SW-NEXT:    ret
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  call i32 @foo()
+  br label %next
+next:
+  %cmp1 = icmp eq i32 %b, 57
+  br i1 %cmp1, label %exit, label %do_call2
+do_call2:
+  call i32 @foo()
+  br label %exit
+exit:
+  ret void
+}
+
+; A further variant of "b" to show that we can shrink wrap this case
+; if the second use of 57 is remat, and we eliminate the second live
+; argument.
+define void @li_straightline_c(i32 zeroext %a) {
+; RV32I-SW-NO-LABEL: li_straightline_c:
+; RV32I-SW-NO:       # %bb.0:
+; RV32I-SW-NO-NEXT:    addi sp, sp, -16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NO-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NO-NEXT:    li a1, 57
+; RV32I-SW-NO-NEXT:    beq a0, a1, .LBB4_3
+; RV32I-SW-NO-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:    li a1, 57
+; RV32I-SW-NO-NEXT:    beq a0, a1, .LBB4_3
+; RV32I-SW-NO-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:  .LBB4_3: # %exit
+; RV32I-SW-NO-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    .cfi_restore ra
+; RV32I-SW-NO-NEXT:    addi sp, sp, 16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NO-NEXT:    ret
+;
+; RV32I-SW-LABEL: li_straightline_c:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    li a1, 57
+; RV32I-SW-NEXT:    beq a0, a1, .LBB4_4
+; RV32I-SW-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:    li a1, 57
+; RV32I-SW-NEXT:    beq a0, a1, .LBB4_3
+; RV32I-SW-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:  .LBB4_3:
+; RV32I-SW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    .cfi_restore ra
+; RV32I-SW-NEXT:    addi sp, sp, 16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NEXT:  .LBB4_4: # %exit
+; RV32I-SW-NEXT:    ret
+;
+; RV32I-SW-SR-LABEL: li_straightline_c:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    li a1, 57
+; RV32I-SW-SR-NEXT:    beq a0, a1, .LBB4_4
+; RV32I-SW-SR-NEXT:  # %bb.1: # %do_call
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_0
+; RV32I-SW-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-SR-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:    li a1, 57
+; RV32I-SW-SR-NEXT:    beq a0, a1, .LBB4_3
+; RV32I-SW-SR-NEXT:  # %bb.2: # %do_call2
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:  .LBB4_3:
+; RV32I-SW-SR-NEXT:    tail __riscv_restore_0
+; RV32I-SW-SR-NEXT:    j .LBB4_4
+; RV32I-SW-SR-NEXT:  .LBB4_4: # %exit
+; RV32I-SW-SR-NEXT:    ret
+;
+; RV64I-SW-LABEL: li_straightline_c:
+; RV64I-SW:       # %bb.0:
+; RV64I-SW-NEXT:    li a1, 57
+; RV64I-SW-NEXT:    beq a0, a1, .LBB4_4
+; RV64I-SW-NEXT:  # %bb.1: # %do_call
+; RV64I-SW-NEXT:    addi sp, sp, -16
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV64I-SW-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    .cfi_offset ra, -8
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:    li a1, 57
+; RV64I-SW-NEXT:    beq a0, a1, .LBB4_3
+; RV64I-SW-NEXT:  # %bb.2: # %do_call2
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:  .LBB4_3:
+; RV64I-SW-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    .cfi_restore ra
+; RV64I-SW-NEXT:    addi sp, sp, 16
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-SW-NEXT:  .LBB4_4: # %exit
+; RV64I-SW-NEXT:    ret
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  %b = call i32 @foo()
+  br label %next
+next:
+  %cmp1 = icmp eq i32 %b, 57
+  br i1 %cmp1, label %exit, label %do_call2
+do_call2:
+  call i32 @foo()
+  br label %exit
+exit:
+  ret void
+}
+
+
+; In this case, the second use is in a loop, so using a callee
+; saved register to avoid a remat is the profitable choice.
+; FIXME: We can shrink wrap the frame setup around the loop
+; and avoid it along the %bb.0 -> %exit edge
+define void @li_loop(i32 zeroext %a, i32 zeroext %b) {
+; RV32I-SW-NO-LABEL: li_loop:
+; RV32I-SW-NO:       # %bb.0:
+; RV32I-SW-NO-NEXT:    addi sp, sp, -16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NO-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-SW-NO-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NO-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NO-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-NO-NEXT:    li s1, 57
+; RV32I-SW-NO-NEXT:    beq a0, s1, .LBB5_3
+; RV32I-SW-NO-NEXT:  # %bb.1: # %do_call.preheader
+; RV32I-SW-NO-NEXT:    mv s0, a1
+; RV32I-SW-NO-NEXT:  .LBB5_2: # %do_call
+; RV32I-SW-NO-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-SW-NO-NEXT:    call foo
+; RV32I-SW-NO-NEXT:    bne s0, s1, .LBB5_2
+; RV32I-SW-NO-NEXT:  .LBB5_3: # %exit
+; RV32I-SW-NO-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-SW-NO-NEXT:    .cfi_restore ra
+; RV32I-SW-NO-NEXT:    .cfi_restore s0
+; RV32I-SW-NO-NEXT:    .cfi_restore s1
+; RV32I-SW-NO-NEXT:    addi sp, sp, 16
+; RV32I-SW-NO-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NO-NEXT:    ret
+;
+; RV32I-SW-LABEL: li_loop:
+; RV32I-SW:       # %bb.0:
+; RV32I-SW-NEXT:    addi sp, sp, -16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-NEXT:    li s1, 57
+; RV32I-SW-NEXT:    beq a0, s1, .LBB5_3
+; RV32I-SW-NEXT:  # %bb.1: # %do_call.preheader
+; RV32I-SW-NEXT:    mv s0, a1
+; RV32I-SW-NEXT:  .LBB5_2: # %do_call
+; RV32I-SW-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-SW-NEXT:    call foo
+; RV32I-SW-NEXT:    bne s0, s1, .LBB5_2
+; RV32I-SW-NEXT:  .LBB5_3: # %exit
+; RV32I-SW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    .cfi_restore ra
+; RV32I-SW-NEXT:    .cfi_restore s0
+; RV32I-SW-NEXT:    .cfi_restore s1
+; RV32I-SW-NEXT:    addi sp, sp, 16
+; RV32I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-SW-NEXT:    ret
+;
+; RV32I-SW-SR-LABEL: li_loop:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_2
+; RV32I-SW-SR-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-SW-SR-NEXT:    .cfi_offset ra, -4
+; RV32I-SW-SR-NEXT:    .cfi_offset s0, -8
+; RV32I-SW-SR-NEXT:    .cfi_offset s1, -12
+; RV32I-SW-SR-NEXT:    li s1, 57
+; RV32I-SW-SR-NEXT:    beq a0, s1, .LBB5_3
+; RV32I-SW-SR-NEXT:  # %bb.1: # %do_call.preheader
+; RV32I-SW-SR-NEXT:    mv s0, a1
+; RV32I-SW-SR-NEXT:  .LBB5_2: # %do_call
+; RV32I-SW-SR-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-SW-SR-NEXT:    call foo
+; RV32I-SW-SR-NEXT:    bne s0, s1, .LBB5_2
+; RV32I-SW-SR-NEXT:  .LBB5_3: # %exit
+; RV32I-SW-SR-NEXT:    tail __riscv_restore_2
+;
+; RV64I-SW-LABEL: li_loop:
+; RV64I-SW:       # %bb.0:
+; RV64I-SW-NEXT:    addi sp, sp, -32
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 32
+; RV64I-SW-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-SW-NEXT:    .cfi_offset ra, -8
+; RV64I-SW-NEXT:    .cfi_offset s0, -16
+; RV64I-SW-NEXT:    .cfi_offset s1, -24
+; RV64I-SW-NEXT:    li s1, 57
+; RV64I-SW-NEXT:    beq a0, s1, .LBB5_3
+; RV64I-SW-NEXT:  # %bb.1: # %do_call.preheader
+; RV64I-SW-NEXT:    mv s0, a1
+; RV64I-SW-NEXT:  .LBB5_2: # %do_call
+; RV64I-SW-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-SW-NEXT:    call foo
+; RV64I-SW-NEXT:    bne s0, s1, .LBB5_2
+; RV64I-SW-NEXT:  .LBB5_3: # %exit
+; RV64I-SW-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-SW-NEXT:    .cfi_restore ra
+; RV64I-SW-NEXT:    .cfi_restore s0
+; RV64I-SW-NEXT:    .cfi_restore s1
+; RV64I-SW-NEXT:    addi sp, sp, 32
+; RV64I-SW-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-SW-NEXT:    ret
+  %cmp0 = icmp eq i32 %a, 57
+  br i1 %cmp0, label %exit, label %do_call
+do_call:
+  call i32 @foo()
+  %cmp1 = icmp eq i32 %b, 57
+  br i1 %cmp1, label %exit, label %do_call
+exit:
+  ret void
+}
+
+declare zeroext i32 @foo()
diff --git a/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll b/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll
new file mode 100644
index 0000000000000..882d0b814063d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll
@@ -0,0 +1,345 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -O2 < %s -verify-machineinstrs \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv32 -mattr=+m -O2 < %s -verify-machineinstrs \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+
+; Tests copied from PowerPC.
+
+; Free probe
+define i8 @f0() #0 nounwind {
+; RV64I-LABEL: f0:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 0(sp)
+; RV64I-NEXT:    lbu a0, 0(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f0:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 0(sp)
+; RV32I-NEXT:    lbu a0, 0(sp)
+; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 64
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f1() #0 nounwind {
+; RV64I-LABEL: f1:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 4096
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f2() #0 nounwind {
+; RV64I-LABEL: f2:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    lui t2, 1
+; RV64I-NEXT:  .LBB2_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB2_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f2:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    lui t2, 1
+; RV32I-NEXT:  .LBB2_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB2_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f3() #0 "stack-probe-size"="32768" nounwind {
+; RV64I-LABEL: f3:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 8
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    lui a0, 8
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f3:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 8
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    lui a0, 8
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+; Same as f2, but without protection.
+define i8 @f4() nounwind {
+; RV64I-LABEL: f4:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f4:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi a0, a0, 16
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f5() #0 "stack-probe-size"="65536" nounwind {
+; RV64I-LABEL: f5:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:  .LBB5_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB5_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 256
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f5:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 256
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:  .LBB5_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB5_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 256
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1048576
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f6() #0 nounwind {
+; RV64I-LABEL: f6:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 262144
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    lui t2, 1
+; RV64I-NEXT:  .LBB6_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB6_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f6:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 262144
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    lui t2, 1
+; RV32I-NEXT:  .LBB6_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB6_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 262144
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1073741824
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f7() #0 "stack-probe-size"="65536" nounwind {
+; RV64I-LABEL: f7:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 244128
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:  .LBB7_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB7_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    lui a0, 13
+; RV64I-NEXT:    addiw a0, a0, -1520
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 9(sp)
+; RV64I-NEXT:    lbu a0, 9(sp)
+; RV64I-NEXT:    lui a1, 244141
+; RV64I-NEXT:    addiw a1, a1, -1520
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f7:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 244128
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:  .LBB7_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB7_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    lui a0, 13
+; RV32I-NEXT:    addi a0, a0, -1520
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 9(sp)
+; RV32I-NEXT:    lbu a0, 9(sp)
+; RV32I-NEXT:    lui a1, 244141
+; RV32I-NEXT:    addi a1, a1, -1520
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1000000007
+  %b = getelementptr inbounds i8, ptr %a, i64 101
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+attributes #0 = { "probe-stack"="inline-asm" }
diff --git a/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll b/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll
new file mode 100644
index 0000000000000..18af080e86747
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll
@@ -0,0 +1,541 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -O2 < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv32 -mattr=+m -O2 < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+
+; Tests copied from PowerPC.
+
+; Free probe
+define i8 @f0() #0 {
+; RV64I-LABEL: f0:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    .cfi_def_cfa_offset 64
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 0(sp)
+; RV64I-NEXT:    lbu a0, 0(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f0:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    .cfi_def_cfa_offset 64
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 0(sp)
+; RV32I-NEXT:    lbu a0, 0(sp)
+; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 64
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f1() #0 {
+; RV64I-LABEL: f1:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    .cfi_def_cfa_offset 4096
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 4112
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    .cfi_def_cfa_offset 4096
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 4112
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 4096
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f2() #0 {
+; RV64I-LABEL: f2:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa t1, 65536
+; RV64I-NEXT:    lui t2, 1
+; RV64I-NEXT:  .LBB2_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB2_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    .cfi_def_cfa_register sp
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 65552
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f2:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa t1, 65536
+; RV32I-NEXT:    lui t2, 1
+; RV32I-NEXT:  .LBB2_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB2_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    .cfi_def_cfa_register sp
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 65552
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f3() #0 "stack-probe-size"="32768" {
+; RV64I-LABEL: f3:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 8
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    .cfi_def_cfa_offset 32768
+; RV64I-NEXT:    lui a0, 8
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    .cfi_def_cfa_offset 65536
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 65552
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f3:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 8
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    .cfi_def_cfa_offset 32768
+; RV32I-NEXT:    lui a0, 8
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    .cfi_def_cfa_offset 65536
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 65552
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+; Same as f2, but without protection.
+define i8 @f4() {
+; RV64I-LABEL: f4:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa_offset 65552
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f4:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi a0, a0, 16
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa_offset 65552
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f5() #0 "stack-probe-size"="65536" {
+; RV64I-LABEL: f5:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa t1, 1048576
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:  .LBB5_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB5_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    .cfi_def_cfa_register sp
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 1048592
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 256
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f5:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 256
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa t1, 1048576
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:  .LBB5_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB5_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    .cfi_def_cfa_register sp
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 1048592
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 256
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1048576
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f6() #0 {
+; RV64I-LABEL: f6:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 262144
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa t1, 1073741824
+; RV64I-NEXT:    lui t2, 1
+; RV64I-NEXT:  .LBB6_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB6_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    .cfi_def_cfa_register sp
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 1073741840
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 16(sp)
+; RV64I-NEXT:    lbu a0, 16(sp)
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f6:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 262144
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa t1, 1073741824
+; RV32I-NEXT:    lui t2, 1
+; RV32I-NEXT:  .LBB6_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB6_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    .cfi_def_cfa_register sp
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 1073741840
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 16(sp)
+; RV32I-NEXT:    lbu a0, 16(sp)
+; RV32I-NEXT:    lui a1, 262144
+; RV32I-NEXT:    addi a1, a1, 16
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1073741824
+  %b = getelementptr inbounds i8, ptr %a, i64 63
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+define i8 @f7() #0 "stack-probe-size"="65536" {
+; RV64I-LABEL: f7:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, 244128
+; RV64I-NEXT:    sub t1, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa t1, 999948288
+; RV64I-NEXT:    lui t2, 16
+; RV64I-NEXT:  .LBB7_1: # %entry
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sub sp, sp, t2
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    bne sp, t1, .LBB7_1
+; RV64I-NEXT:  # %bb.2: # %entry
+; RV64I-NEXT:    .cfi_def_cfa_register sp
+; RV64I-NEXT:    lui a0, 13
+; RV64I-NEXT:    addiw a0, a0, -1520
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    .cfi_def_cfa_offset 1000000016
+; RV64I-NEXT:    li a0, 3
+; RV64I-NEXT:    sb a0, 9(sp)
+; RV64I-NEXT:    lbu a0, 9(sp)
+; RV64I-NEXT:    lui a1, 244141
+; RV64I-NEXT:    addiw a1, a1, -1520
+; RV64I-NEXT:    add sp, sp, a1
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f7:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 244128
+; RV32I-NEXT:    sub t1, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa t1, 999948288
+; RV32I-NEXT:    lui t2, 16
+; RV32I-NEXT:  .LBB7_1: # %entry
+; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT:    sub sp, sp, t2
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    bne sp, t1, .LBB7_1
+; RV32I-NEXT:  # %bb.2: # %entry
+; RV32I-NEXT:    .cfi_def_cfa_register sp
+; RV32I-NEXT:    lui a0, 13
+; RV32I-NEXT:    addi a0, a0, -1520
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    .cfi_def_cfa_offset 1000000016
+; RV32I-NEXT:    li a0, 3
+; RV32I-NEXT:    sb a0, 9(sp)
+; RV32I-NEXT:    lbu a0, 9(sp)
+; RV32I-NEXT:    lui a1, 244141
+; RV32I-NEXT:    addi a1, a1, -1520
+; RV32I-NEXT:    add sp, sp, a1
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+entry:
+  %a = alloca i8, i64 1000000007
+  %b = getelementptr inbounds i8, ptr %a, i64 101
+  store volatile i8 3, ptr %a
+  %c = load volatile i8, ptr %a
+  ret i8 %c
+}
+
+; alloca + align < probe_size
+define i32 @f8(i64 %i) local_unnamed_addr #0 {
+; RV64I-LABEL: f8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -832
+; RV64I-NEXT:    .cfi_def_cfa_offset 832
+; RV64I-NEXT:    sd ra, 824(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 816(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    addi s0, sp, 832
+; RV64I-NEXT:    .cfi_def_cfa s0, 0
+; RV64I-NEXT:    andi sp, sp, -64
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    lw a0, 0(sp)
+; RV64I-NEXT:    addi sp, s0, -832
+; RV64I-NEXT:    .cfi_def_cfa sp, 832
+; RV64I-NEXT:    ld ra, 824(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 816(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    addi sp, sp, 832
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -832
+; RV32I-NEXT:    .cfi_def_cfa_offset 832
+; RV32I-NEXT:    sw ra, 828(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 824(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    addi s0, sp, 832
+; RV32I-NEXT:    .cfi_def_cfa s0, 0
+; RV32I-NEXT:    andi sp, sp, -64
+; RV32I-NEXT:    slli a0, a0, 2
+; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    li a1, 1
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    lw a0, 0(sp)
+; RV32I-NEXT:    addi sp, s0, -832
+; RV32I-NEXT:    .cfi_def_cfa sp, 832
+; RV32I-NEXT:    lw ra, 828(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 824(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    addi sp, sp, 832
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+  %a = alloca i32, i32 200, align 64
+  %b = getelementptr inbounds i32, ptr %a, i64 %i
+  store volatile i32 1, ptr %b
+  %c = load volatile i32, ptr %a
+  ret i32 %c
+}
+
+; alloca > probe_size, align > probe_size
+define i32 @f9(i64 %i) local_unnamed_addr #0 {
+; RV64I-LABEL: f9:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -2032
+; RV64I-NEXT:    .cfi_def_cfa_offset 2032
+; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    addi s0, sp, 2032
+; RV64I-NEXT:    .cfi_def_cfa s0, 0
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    sub sp, sp, a1
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    sub sp, sp, a1
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    andi sp, sp, -2048
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    addi a1, sp, 2047
+; RV64I-NEXT:    addi a1, a1, 1
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    add a0, sp, a0
+; RV64I-NEXT:    lw a0, -2048(a0)
+; RV64I-NEXT:    addi sp, s0, -2032
+; RV64I-NEXT:    .cfi_def_cfa sp, 2032
+; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore ra
+; RV64I-NEXT:    .cfi_restore s0
+; RV64I-NEXT:    addi sp, sp, 2032
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV32I-LABEL: f9:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    .cfi_def_cfa_offset 2032
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    addi s0, sp, 2032
+; RV32I-NEXT:    .cfi_def_cfa s0, 0
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    sub sp, sp, a1
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    sub sp, sp, a1
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    andi sp, sp, -2048
+; RV32I-NEXT:    slli a0, a0, 2
+; RV32I-NEXT:    addi a1, sp, 2047
+; RV32I-NEXT:    addi a1, a1, 1
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    li a1, 1
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    add a0, sp, a0
+; RV32I-NEXT:    lw a0, -2048(a0)
+; RV32I-NEXT:    addi sp, s0, -2032
+; RV32I-NEXT:    .cfi_def_cfa sp, 2032
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore ra
+; RV32I-NEXT:    .cfi_restore s0
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+  %a = alloca i32, i32 2000, align 2048
+  %b = getelementptr inbounds i32, ptr %a, i64 %i
+  store volatile i32 1, ptr %b
+  %c = load volatile i32, ptr %a
+  ret i32 %c
+}
+
+attributes #0 = { "probe-stack"="inline-asm" }
diff --git a/llvm/test/CodeGen/RISCV/vector-tuple-zeroinitializer.ll b/llvm/test/CodeGen/RISCV/vector-tuple-zeroinitializer.ll
new file mode 100644
index 0000000000000..fb1104e0a3b80
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/vector-tuple-zeroinitializer.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+v \
+; RUN:   -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=riscv64 -mattr=+v \
+; RUN:   -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK
+
+define target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @test_tuple_zero_power_of_2() {
+; CHECK-LABEL: test_tuple_zero_power_of_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    ret
+entry:
+  ret target("riscv.vector.tuple", <vscale x 16 x i8>, 2) zeroinitializer
+}
+
+define target("riscv.vector.tuple", <vscale x 16 x i8>, 3) @test_tuple_zero_non_power_of_2() {
+; CHECK-LABEL: test_tuple_zero_non_power_of_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    ret
+entry:
+  ret target("riscv.vector.tuple", <vscale x 16 x i8>, 3) zeroinitializer
+}
+
+define target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @test_tuple_zero_insert1(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: test_tuple_zero_insert1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = call target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv16i8_2t.nxv4i32(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) zeroinitializer, <vscale x 4 x i32> %a, i32 0)
+  ret target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %1
+}
+
+define target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @test_tuple_zero_insert2(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: test_tuple_zero_insert2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v6, 0
+; CHECK-NEXT:    vmv2r.v v10, v8
+; CHECK-NEXT:    vmv2r.v v8, v6
+; CHECK-NEXT:    ret
+entry:
+  %1 = call target("riscv.vector.tuple", <vscale x 16 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv16i8_2t.nxv4i32(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) zeroinitializer, <vscale x 4 x i32> %a, i32 1)
+  ret target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %1
+}
diff --git a/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll
new file mode 100644
index 0000000000000..ec2b986d174d9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mtriple=riscv32-unknown-linux-gnu -mattr=+d,+c < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=riscv64-unknown-linux-gnu -mattr=+d,+c < %s | FileCheck --check-prefix=CHECK --check-prefix=CHECK-RISCV64 %s
+
+define i32 @foo() nounwind "function-instrument"="xray-always" {
+; CHECK:                        .p2align 2
+; CHECK-LABEL:                  .Lxray_sled_0:
+; CHECK-NEXT:                   j .Ltmp0
+; CHECK-COUNT-21:               nop
+; CHECK-RISCV64-COUNT-12:       nop
+; CHECK-LABEL:                  .Ltmp0:
+  ret i32 0
+; CHECK:                        .p2align 2
+; CHECK-LABEL:                  .Lxray_sled_1:
+; CHECK-NEXT:                   j .Ltmp1
+; CHECK-COUNT-21:               nop
+; CHECK-RISCV64-COUNT-12:       nop
+; CHECK-LABEL:                  .Ltmp1:
+; CHECK-NEXT:                   ret
+}
+; CHECK:                        .section xray_instr_map,"ao",@progbits,foo
+; CHECK-LABEL:                  .Lxray_sleds_start0:
+; CHECK:                        .Lxray_sled_0-[[TMP:.Ltmp[0-9]+]]
+; CHECK:                        .Lxray_sled_1-[[TMP:.Ltmp[0-9]+]]
+; CHECK-LABEL:                  .Lxray_sleds_end0:
diff --git a/llvm/test/CodeGen/SPIRV/bitcast.ll b/llvm/test/CodeGen/SPIRV/bitcast.ll
index 242c5a46583c2..d6c985dbadcc4 100644
--- a/llvm/test/CodeGen/SPIRV/bitcast.ll
+++ b/llvm/test/CodeGen/SPIRV/bitcast.ll
@@ -6,7 +6,7 @@
 ; CHECK-SPIRV-DAG: %[[#TyHalf:]] = OpTypeFloat 16
 ; CHECK-SPIRV-DAG: %[[#Arg32:]] = OpFunctionParameter %[[#TyInt32]]
 ; CHECK-SPIRV-DAG: %[[#Arg16:]] = OpUConvert %[[#TyInt16]] %[[#Arg32]]
-; CHECK-SPIRV-DAG: %[[#ValHalf:]] = OpBitcast %[[#TyHalf]] %8
+; CHECK-SPIRV-DAG: %[[#ValHalf:]] = OpBitcast %[[#TyHalf]] %[[#Arg16:]]
 ; CHECK-SPIRV-DAG: %[[#ValHalf2:]] = OpFMul %[[#TyHalf]] %[[#ValHalf]] %[[#ValHalf]]
 ; CHECK-SPIRV-DAG: %[[#Res16:]] = OpBitcast %[[#TyInt16]] %[[#ValHalf2]]
 ; CHECK-SPIRV-DAG: OpReturnValue %[[#Res16]]
diff --git a/llvm/test/CodeGen/SPIRV/branching/OpSwitchBranches.ll b/llvm/test/CodeGen/SPIRV/branching/OpSwitchBranches.ll
index bc61e951fc18f..3665c07cfcf53 100644
--- a/llvm/test/CodeGen/SPIRV/branching/OpSwitchBranches.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/OpSwitchBranches.ll
@@ -34,16 +34,16 @@ end:
   %result = load i32, ptr %alloc
   ret i32 %result
 
-; CHECK-SPIRV:      %[[#DEFAULT]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE3]] = OpLabel
 ; CHECK-SPIRV:      OpBranch %[[#END:]]
 
-; CHECK-SPIRV:      %[[#CASE1]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE2]] = OpLabel
 ; CHECK-SPIRV:      OpBranch %[[#END]]
 
-; CHECK-SPIRV:      %[[#CASE2]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE1]] = OpLabel
 ; CHECK-SPIRV:      OpBranch %[[#END]]
 
-; CHECK-SPIRV:      %[[#CASE3]] = OpLabel
+; CHECK-SPIRV:      %[[#DEFAULT]] = OpLabel
 ; CHECK-SPIRV:      OpBranch %[[#END]]
 
 ; CHECK-SPIRV:      %[[#END]] = OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/branching/OpSwitchUnreachable.ll b/llvm/test/CodeGen/SPIRV/branching/OpSwitchUnreachable.ll
index 6bf58d1c86fd5..7fbd06c67b56e 100644
--- a/llvm/test/CodeGen/SPIRV/branching/OpSwitchUnreachable.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/OpSwitchUnreachable.ll
@@ -12,14 +12,13 @@ define void @test_switch_with_unreachable_block(i1 %a) {
     i32 1, label %reachable
   ]
 
-; CHECK-SPIRV:      %[[#UNREACHABLE]] = OpLabel
-; CHECK-SPIRV-NEXT: OpUnreachable
-
 ; CHECK-SPIRV-NEXT: %[[#REACHABLE]] = OpLabel
 reachable:
 ; CHECK-SPIRV-NEXT: OpReturn
   ret void
 
+; CHECK-SPIRV:      %[[#UNREACHABLE]] = OpLabel
+; CHECK-SPIRV-NEXT: OpUnreachable
 unreachable:
   unreachable
 }
diff --git a/llvm/test/CodeGen/SPIRV/branching/Two_OpSwitch_same_register.ll b/llvm/test/CodeGen/SPIRV/branching/Two_OpSwitch_same_register.ll
index 59a40dce343a3..7a134193de2a6 100644
--- a/llvm/test/CodeGen/SPIRV/branching/Two_OpSwitch_same_register.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/Two_OpSwitch_same_register.ll
@@ -32,19 +32,19 @@ case4:
 default2:
   ret void
 
-; CHECK-SPIRV:      %[[#CASE1]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE2]] = OpLabel
 ; CHECK-SPIRV-NEXT: OpBranch %[[#DEFAULT1]]
 
-; CHECK-SPIRV:      %[[#CASE2]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE1]] = OpLabel
 ; CHECK-SPIRV-NEXT: OpBranch %[[#DEFAULT1]]
 
 ; CHECK-SPIRV:      %[[#DEFAULT1]] = OpLabel
 ; CHECK-SPIRV-NEXT:      OpSwitch %[[#REGISTER]] %[[#DEFAULT2:]] 0 %[[#CASE3:]] 1 %[[#CASE4:]]
 
-; CHECK-SPIRV:      %[[#CASE3]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE4]] = OpLabel
 ; CHECK-SPIRV-NEXT: OpBranch %[[#DEFAULT2]]
 
-; CHECK-SPIRV:      %[[#CASE4:]] = OpLabel
+; CHECK-SPIRV:      %[[#CASE3]] = OpLabel
 ; CHECK-SPIRV-NEXT: OpBranch %[[#DEFAULT2]]
 
 ; CHECK-SPIRV:      %[[#DEFAULT2]] = OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/branching/if-merging.ll b/llvm/test/CodeGen/SPIRV/branching/if-merging.ll
index 52eeb216234e5..18fbe417addf8 100644
--- a/llvm/test/CodeGen/SPIRV/branching/if-merging.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/if-merging.ll
@@ -1,12 +1,12 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 
 ;; NOTE: This does not check for structured control-flow operations.
 
 ; CHECK-DAG: OpName [[FOO:%.+]] "foo"
 ; CHECK-DAG: OpName [[BAR:%.+]] "bar"
 
-; CHECK-DAG: [[I32:%.+]] = OpTypeInt 32
-; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
+; CHECK-DAG: %[[#I32:]] = OpTypeInt 32
+; CHECK-DAG: %[[#BOOL:]] = OpTypeBool
 
 declare i32 @foo()
 declare i32 @bar()
@@ -30,23 +30,24 @@ merge_label:
 }
 
 ; CHECK: OpFunction
-; CHECK: [[A:%.+]] = OpFunctionParameter [[I32]]
-; CHECK: [[B:%.+]] = OpFunctionParameter [[I32]]
+; CHECK: %[[#A:]] = OpFunctionParameter %[[#I32]]
+; CHECK: %[[#B:]] = OpFunctionParameter %[[#I32]]
 
-; CHECK: [[ENTRY:%.+]] = OpLabel
-; CHECK: [[COND:%.+]] = OpIEqual [[BOOL]] [[A]] [[B]]
-; CHECK: OpBranchConditional [[COND]] [[TRUE_LABEL:%.+]] [[FALSE_LABEL:%.+]]
+; CHECK: %[[#ENTRY:]] = OpLabel
+; CHECK: %[[#COND:]] = OpIEqual %[[#BOOL]] %[[#A]] %[[#B]]
+; CHECK: OpBranchConditional %[[#COND]] %[[#TRUE_LABEL:]] %[[#FALSE_LABEL:]]
 
-; CHECK: [[TRUE_LABEL]] = OpLabel
-; CHECK: [[V1:%.+]] = OpFunctionCall [[I32]] [[FOO]]
-; CHECK: OpBranch [[MERGE_LABEL:%.+]]
+; CHECK: %[[#FALSE_LABEL]] = OpLabel
+; CHECK: %[[#V2:]] = OpFunctionCall %[[#I32]] [[BAR]]
+; CHECK: OpBranch %[[#MERGE_LABEL:]]
 
-; CHECK: [[FALSE_LABEL]] = OpLabel
-; CHECK: [[V2:%.+]] = OpFunctionCall [[I32]] [[BAR]]
-; CHECK: OpBranch [[MERGE_LABEL]]
+; CHECK: %[[#TRUE_LABEL]] = OpLabel
+; CHECK: %[[#V1:]] = OpFunctionCall %[[#I32]] [[FOO]]
+; CHECK: OpBranch %[[#MERGE_LABEL]]
 
-; CHECK: [[MERGE_LABEL]] = OpLabel
-; CHECK-NEXT: [[V:%.+]] = OpPhi [[I32]] [[V1]] [[TRUE_LABEL]] [[V2]] [[FALSE_LABEL]]
+
+; CHECK: %[[#MERGE_LABEL]] = OpLabel
+; CHECK-NEXT: [[V:%.+]] = OpPhi %[[#I32]] %[[#V1]] %[[#TRUE_LABEL]] %[[#V2]] %[[#FALSE_LABEL]]
 ; CHECK: OpReturnValue [[V]]
 
 ; CHECK-NEXT: OpFunctionEnd
diff --git a/llvm/test/CodeGen/SPIRV/branching/if-non-merging.ll b/llvm/test/CodeGen/SPIRV/branching/if-non-merging.ll
index 319abda86b046..b664a65a5829c 100644
--- a/llvm/test/CodeGen/SPIRV/branching/if-non-merging.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/if-non-merging.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 
 ; CHECK-DAG: [[I32:%.+]] = OpTypeInt 32
 ; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
@@ -21,7 +21,9 @@ false_label:
 ; CHECK: [[ENTRY:%.+]] = OpLabel
 ; CHECK: [[COND:%.+]] = OpIEqual [[BOOL]] [[A]] [[B]]
 ; CHECK: OpBranchConditional [[COND]] [[TRUE_LABEL:%.+]] [[FALSE_LABEL:%.+]]
-; CHECK: [[TRUE_LABEL]] = OpLabel
-; CHECK: OpReturnValue [[TRUE]]
+
 ; CHECK: [[FALSE_LABEL]] = OpLabel
 ; CHECK: OpReturnValue [[FALSE]]
+
+; CHECK: [[TRUE_LABEL]] = OpLabel
+; CHECK: OpReturnValue [[TRUE]]
diff --git a/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll b/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
index 9981e0d86eaa3..25c2c8627de4e 100644
--- a/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpFunction
diff --git a/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
index d12914d378542..6073b0c2b9276 100644
--- a/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
+++ b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
@@ -7,37 +7,37 @@
 ; CHECK-MIR-DAG: [[type_i32:%[0-9]+\:type]] = OpTypeInt 32, 0
 ; CHECK-MIR-DAG: [[encoding_signedchar:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 5
 ; CHECK-MIR-DAG: [[encoding_float:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 3
-; CHECK-MIR-DAG: [[flag_zero:%[0-9]+\:iid\(s32\)]] = OpConstantNull [[type_i32]]
+; CHECK-MIR-DAG: [[flag_zero:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 0
 ; CHECK-MIR-DAG: [[str_bool:%[0-9]+\:id\(s32\)]] = OpString 1819242338, 0
 ; CHECK-MIR-DAG: [[size_8bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 8
 ; CHECK-MIR-DAG: [[encoding_boolean:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 2
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_bool]], [[size_8bits]], [[encoding_boolean]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_bool]], [[size_8bits]], [[encoding_boolean]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_int:%[0-9]+\:id\(s32\)]] = OpString 7630441
 ; CHECK-MIR-DAG: [[size_32bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 32
 ; CHECK-MIR-DAG: [[encoding_signed:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 4
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_int]], [[size_32bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_int]], [[size_32bits]], [[encoding_signed]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_short:%[0-9]+\:id\(s32\)]] = OpString 1919903859, 116
 ; CHECK-MIR-DAG: [[size_16bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 16
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_short]], [[size_16bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_short]], [[size_16bits]], [[encoding_signed]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_char:%[0-9]+\:id\(s32\)]] = OpString 1918986339, 0
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_char]], [[size_8bits]], [[encoding_signedchar]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_char]], [[size_8bits]], [[encoding_signedchar]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_long:%[0-9]+\:id\(s32\)]] = OpString 1735290732, 0
 ; CHECK-MIR-DAG: [[size_64bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 64
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_long]], [[size_64bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_long]], [[size_64bits]], [[encoding_signed]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_uint:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1953392928, 0
 ; CHECK-MIR-DAG: [[encoding_unsigned:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 6
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_uint]], [[size_32bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_uint]], [[size_32bits]], [[encoding_unsigned]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_ushort:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1869116192, 29810
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_ushort]], [[size_16bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_ushort]], [[size_16bits]], [[encoding_unsigned]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_uchar:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1634231072, 114
 ; CHECK-MIR-DAG: [[encoding_unsignedchar:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 7
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_uchar]], [[size_8bits]], [[encoding_unsignedchar]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_uchar]], [[size_8bits]], [[encoding_unsignedchar]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_ulong:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1852795936, 103
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_ulong]], [[size_64bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_ulong]], [[size_64bits]], [[encoding_unsigned]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_float:%[0-9]+\:id\(s32\)]] = OpString 1634692198, 116
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_float]], [[size_32bits]], [[encoding_float]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_float]], [[size_32bits]], [[encoding_float]], [[flag_zero]]
 ; CHECK-MIR-DAG: [[str_double:%[0-9]+\:id\(s32\)]] = OpString 1651863396, 25964
-; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_double]], [[size_64bits]], [[encoding_float]], [[flag_zero]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 2, [[str_double]], [[size_64bits]], [[encoding_float]], [[flag_zero]]
 
 ; CHECK-SPIRV: [[ext_inst_non_semantic:%[0-9]+]] = OpExtInstImport "NonSemantic.Shader.DebugInfo.100"
 ; CHECK-SPIRV-DAG: [[str_bool:%[0-9]+]] = OpString "bool"
@@ -60,7 +60,7 @@
 ; CHECK-SPIRV-DAG: [[type_int16:%[0-9]+]] = OpTypeInt 16 0
 ; CHECK-SPIRV-DAG: [[type_int32:%[0-9]+]] = OpTypeInt 32 0
 ; CHECK-SPIRV-DAG: [[encoding_signedchar:%[0-9]+]] = OpConstant [[type_int32]] 5
-; CHECK-SPIRV-DAG: [[flag_zero:%[0-9]+]] = OpConstantNull [[type_int32]]
+; CHECK-SPIRV-DAG: [[flag_zero:%[0-9]+]] = OpConstant [[type_int32]] 0
 ; CHECK-SPIRV-DAG: [[encoding_float:%[0-9]+]] = OpConstant [[type_int32]] 3
 ; CHECK-SPIRV-DAG: [[size_8bit:%[0-9]+]] = OpConstant [[type_int32]] 8
 ; CHECK-SPIRV-DAG: [[encoding_boolean:%[0-9]+]] = OpConstant [[type_int32]] 2
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_optnone.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_optnone.ll
new file mode 100644
index 0000000000000..e21d99badea06
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_optnone.ll
@@ -0,0 +1,20 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_optnone %s -o - | FileCheck %s --check-prefixes=CHECK-EXTENSION
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-NO-EXTENSION
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_EXT_optnone %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-EXTENSION: OpCapability OptNoneEXT
+; CHECK-EXTENSION: OpExtension "SPV_EXT_optnone"
+; CHECK-NO-EXTENSION-NOT: OpCapability OptNoneINTEL
+; CHECK-NO-EXTENSION-NOT: OpCapability OptNoneEXT
+; CHECK-NO-EXTENSION-NOT: OpExtension "SPV_INTEL_optnone"
+; CHECK-NO-EXTENSION-NOT: OpExtension "SPV_EXT_optnone"
+
+define spir_func void @foo() #0 {
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] DontInline|OptNoneEXT %[[#]]
+entry:
+  ret void
+}
+
+attributes #0 = { nounwind optnone noinline }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
index 3ebfa1d8c8a9d..6aeb29df9f7bd 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
@@ -1,7 +1,6 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK-DAG: OpCapability Int8
 ; CHECK-DAG: OpCapability FunctionPointersINTEL
 ; CHECK-DAG: OpCapability Int64
 ; CHECK: OpExtension "SPV_INTEL_function_pointers"
@@ -9,19 +8,14 @@
 ; CHECK-DAG: %[[TyVoid:.*]] = OpTypeVoid
 ; CHECK-DAG: %[[TyInt64:.*]] = OpTypeInt 64 0
 ; CHECK-DAG: %[[TyFun:.*]] = OpTypeFunction %[[TyInt64]] %[[TyInt64]]
-; CHECK-DAG: %[[TyInt8:.*]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[TyPtrFunCodeSection:.*]] = OpTypePointer CodeSectionINTEL %[[TyFun]]
 ; CHECK-DAG: %[[ConstFunFp:.*]] = OpConstantFunctionPointerINTEL %[[TyPtrFunCodeSection]] %[[DefFunFp:.*]]
 ; CHECK-DAG: %[[TyPtrFun:.*]] = OpTypePointer Function %[[TyFun]]
 ; CHECK-DAG: %[[TyPtrPtrFun:.*]] = OpTypePointer Function %[[TyPtrFun]]
-; CHECK-DAG: %[[TyPtrInt8:.*]] = OpTypePointer Function %[[TyInt8]]
-; CHECK-DAG: %[[TyPtrPtrInt8:.*]] = OpTypePointer Function %[[TyPtrInt8]]
 ; CHECK: OpFunction
-; CHECK: %[[Var:.*]] = OpVariable %[[TyPtrPtrInt8]] Function
-; CHECK: %[[SAddr:.*]] = OpBitcast %[[TyPtrPtrFun]] %[[Var]]
-; CHECK: OpStore %[[SAddr]] %[[ConstFunFp]]
-; CHECK: %[[LAddr:.*]] = OpBitcast %[[TyPtrPtrFun]] %[[Var]]
-; CHECK: %[[FP:.*]] = OpLoad %[[TyPtrFun]] %[[LAddr]]
+; CHECK: %[[Var:.*]] = OpVariable %[[TyPtrPtrFun]] Function
+; CHECK: OpStore %[[Var]] %[[ConstFunFp]]
+; CHECK: %[[FP:.*]] = OpLoad %[[TyPtrFun]] %[[Var]]
 ; CHECK: OpFunctionPointerCallINTEL %[[TyInt64]] %[[FP]] %[[#]]
 ; CHECK: OpFunctionEnd
  
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
index eb7b1dffaee50..9fa46f50a2e89 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpCapability Int8
@@ -12,13 +12,15 @@
 ; CHECK-DAG: OpName %[[test:.*]] "test"
 ; CHECK-DAG: %[[TyVoid:.*]] = OpTypeVoid
 ; CHECK-DAG: %[[TyFloat32:.*]] = OpTypeFloat 32
-; CHECK-DAG: %[[TyInt8:.*]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[TyInt64:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[TyInt8:.*]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[TyPtrInt8:.*]] = OpTypePointer Function %[[TyInt8]]
-; CHECK-DAG: %[[TyFp:.*]] = OpTypeFunction %[[TyFloat32]] %[[TyPtrInt8]]
-; CHECK-DAG: %[[TyPtrFp:.*]] = OpTypePointer Function %[[TyFp]]
-; CHECK-DAG: %[[TyBar:.*]] = OpTypeFunction %[[TyInt64]] %[[TyPtrFp]] %[[TyPtrInt8]]
+; CHECK-DAG: %[[TyUncompleteFp:.*]] = OpTypeFunction %[[TyFloat32]] %[[TyPtrInt8]]
+; CHECK-DAG: %[[TyPtrUncompleteFp:.*]] = OpTypePointer Function %[[TyUncompleteFp]]
+; CHECK-DAG: %[[TyBar:.*]] = OpTypeFunction %[[TyInt64]] %[[TyPtrUncompleteFp]] %[[TyPtrInt8]]
 ; CHECK-DAG: %[[TyPtrBar:.*]] = OpTypePointer Function %[[TyBar]]
+; CHECK-DAG: %[[TyFp:.*]] = OpTypeFunction %[[TyFloat32]] %[[TyPtrBar]]
+; CHECK-DAG: %[[TyPtrFp:.*]] = OpTypePointer Function %[[TyFp]]
 ; CHECK-DAG: %[[TyTest:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrFp]] %[[TyPtrInt8]] %[[TyPtrBar]]
 ; CHECK: %[[test]] = OpFunction %[[TyVoid]] None %[[TyTest]]
 ; CHECK: %[[fp]] = OpFunctionParameter %[[TyPtrFp]]
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
new file mode 100644
index 0000000000000..c0b23a324992e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
@@ -0,0 +1,32 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: ***BF16ComponentsINTEL type interpretations require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: CooperativeMatrixBFloat16ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+  %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1)
+  %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K)
+  %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64)
+  tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1)
+  ret void
+}
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
+
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
new file mode 100644
index 0000000000000..a4b2c4be5084b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -0,0 +1,46 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const12:]] = OpConstant %[[#Int32Ty]] 12
+; CHECK-DAG: %[[#Const48:]] = OpConstant %[[#Int32Ty]] 48
+; CHECK-DAG: %[[#Const0:]] = OpConstant %[[#Int32Ty]] 0
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32Ty]] 3
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32Ty]] 1
+; CHECK-DAG: %[[#MatTy1:]] = OpTypeCooperativeMatrixKHR %[[#Int32Ty]] %[[#Const3]] %[[#Const12]] %[[#Const12]] %[[#Const2]]
+; CHECK-DAG: %[[#MatTy2:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const3]] %[[#Const12]] %[[#Const48]] %[[#Const0]]
+; CHECK-DAG: %[[#MatTy3:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const2]] %[[#Const48]] %[[#Const12]] %[[#Const1]]
+; CHECK: OpCooperativeMatrixConstructCheckedINTEL %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy3]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue)
+  %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1)
+  %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K)
+  %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+  tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1)
+  ret void
+}
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef)
+
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
new file mode 100644
index 0000000000000..e7ec5e186d16a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
@@ -0,0 +1,24 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixGetElementCoordINTEL requires the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixInvocationInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#MatrixTy:]] = OpTypeCooperativeMatrixKHR
+
+; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]]
+; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]]
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+  %coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx)
+  ret void
+}
+declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
new file mode 100644
index 0000000000000..8b336df1c2c20
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
@@ -0,0 +1,58 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: PackedINTEL layout require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability PackedCooperativeMatrixINTEL
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue)
+  %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1)
+  %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K)
+  %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+  tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1)
+  ret void
+}
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue)
+  %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1)
+  %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K)
+  %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+  tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1)
+  ret void
+}
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef)
+
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
+
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
new file mode 100644
index 0000000000000..8573e09284403
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
@@ -0,0 +1,27 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixPrefetchINTEL instruction requires the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpExtension "SPV_INTEL_joint_matrix"
+
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) {
+entry:
+  tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0)
+  tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K)
+  tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1)
+  ret void
+}
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
new file mode 100644
index 0000000000000..d3ada306d2c2f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
@@ -0,0 +1,32 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: MatrixAAndBTF32ComponentsINTEL type interpretation require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: OpCapability CooperativeMatrixTF32ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL
+
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) {
+entry:
+  %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue)
+  %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1)
+  %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K)
+  %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32)
+  tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1)
+  ret void
+}
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef)
+
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
+
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/builtin-op-wrappers.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/builtin-op-wrappers.ll
new file mode 100644
index 0000000000000..c30370c179516
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/builtin-op-wrappers.ll
@@ -0,0 +1,115 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_media_block_io %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_media_block_io %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: Capability SubgroupImageMediaBlockIOINTEL
+; CHECK: Extension "SPV_INTEL_media_block_io"
+; CHECK-COUNT-14: SubgroupImageMediaBlockReadINTEL
+; CHECK-COUNT-14: SubgroupImageMediaBlockWriteINTEL
+
+define spir_kernel void @intel_media_block_test(<2 x i32> %edgeCoord, ptr addrspace(1) %image_in, ptr addrspace(1) %image_out) !kernel_arg_addr_space !6 !kernel_arg_access_qual !7 !kernel_arg_type !8 !kernel_arg_base_type !8 {
+entry:
+  %call = call spir_func i8 @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_RcharPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call1 = call spir_func <2 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call2 = call spir_func <4 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call3 = call spir_func <8 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call4 = call spir_func <16 x i8> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rchar16PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call5 = call spir_func i16 @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_RshortPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call6 = call spir_func <2 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call7 = call spir_func <4 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call8 = call spir_func <8 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call9 = call spir_func <16 x i16> @_Z49__spirv_SubgroupImageMediaBlockReadINTEL_Rshort16PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call10 = call spir_func i32 @_Z45__spirv_SubgroupImageMediaBlockReadINTEL_RintPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call11 = call spir_func <2 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call12 = call spir_func <4 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  %call13 = call spir_func <8 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1) %image_in, <2 x i32> %edgeCoord, i32 1, i32 16)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiic(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, i8 %call)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv2_c(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <2 x i8> %call1)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_c(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <4 x i8> %call2)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_c(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <8 x i8> %call3)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv16_c(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <16 x i8> %call4)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiis(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, i16 %call5)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv2_s(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <2 x i16> %call6)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_s(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <4 x i16> %call7)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_s(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <8 x i16> %call8)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv16_s(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <16 x i16> %call9)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiii(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, i32 %call10)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiS2_(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <2 x i32> %call11)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_i(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <4 x i32> %call12)
+  call spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_i(ptr addrspace(1) %image_out, <2 x i32> %edgeCoord, i32 1, i32 16, <8 x i32> %call13)
+  ret void
+}
+
+declare spir_func i8 @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_RcharPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <2 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <4 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <8 x i8> @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_Rchar8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <16 x i8> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rchar16PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func i16 @_Z47__spirv_SubgroupImageMediaBlockReadINTEL_RshortPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <2 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <4 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <8 x i16> @_Z48__spirv_SubgroupImageMediaBlockReadINTEL_Rshort8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <16 x i16> @_Z49__spirv_SubgroupImageMediaBlockReadINTEL_Rshort16PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func i32 @_Z45__spirv_SubgroupImageMediaBlockReadINTEL_RintPU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <2 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint2PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <4 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint4PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func <8 x i32> @_Z46__spirv_SubgroupImageMediaBlockReadINTEL_Rint8PU3AS133__spirv_Image__void_1_0_0_0_0_0_0Dv2_iii(ptr addrspace(1), <2 x i32>, i32, i32)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiic(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, i8)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv2_c(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <2 x i8>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_c(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <4 x i8>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_c(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <8 x i8>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv16_c(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <16 x i8>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiis(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, i16)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv2_s(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <2 x i16>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_s(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <4 x i16>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_s(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <8 x i16>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv16_s(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <16 x i16>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiii(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, i32)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiS2_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <2 x i32>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv4_i(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <4 x i32>)
+
+declare spir_func void @_Z41__spirv_SubgroupImageMediaBlockWriteINTELPU3AS133__spirv_Image__void_1_0_0_0_0_0_1Dv2_iiiDv8_i(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, i32, i32, <8 x i32>)
+
+!spirv.MemoryModel = !{!0}
+!opencl.enable.FP_CONTRACT = !{}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!2}
+!opencl.used.extensions = !{!3}
+!opencl.used.optional.core.features = !{!4}
+!spirv.Generator = !{!5}
+
+!0 = !{i32 1, i32 2}
+!1 = !{i32 3, i32 200000}
+!2 = !{i32 2, i32 0}
+!3 = !{}
+!4 = !{!"cl_images"}
+!5 = !{i16 6, i16 14}
+!6 = !{i32 0, i32 1, i32 1}
+!7 = !{!"none", !"read_only", !"write_only"}
+!8 = !{!"int2", !"image2d_t", !"image2d_t"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/cl_intel_media_block_io.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/cl_intel_media_block_io.ll
new file mode 100644
index 0000000000000..735094c7c8862
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_media_block_io/cl_intel_media_block_io.ll
@@ -0,0 +1,115 @@
+; Compiled from https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_media_block_io/SPV_INTEL_media_block_io.cl
+
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; CHECK-ERROR: LLVM ERROR: intel_sub_group_media_block_read_uc: the builtin requires the following SPIR-V extension: SPV_INTEL_media_block_io
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_media_block_io %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_media_block_io %s -o - -filetype=obj | spirv-val %}
+; CHECK: Capability SubgroupImageMediaBlockIOINTEL
+; CHECK: Extension "SPV_INTEL_media_block_io"
+
+; CHECK-COUNT-14: SubgroupImageMediaBlockReadINTEL
+; CHECK-COUNT-14: SubgroupImageMediaBlockWriteINTEL
+
+define spir_kernel void @intel_media_block_test(<2 x i32> noundef %edgeCoord, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !6 {
+entry:
+  %call = tail call spir_func zeroext i8 @_Z35intel_sub_group_media_block_read_ucDv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call1 = tail call spir_func <2 x i8> @_Z36intel_sub_group_media_block_read_uc2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call2 = tail call spir_func <4 x i8> @_Z36intel_sub_group_media_block_read_uc4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call3 = tail call spir_func <8 x i8> @_Z36intel_sub_group_media_block_read_uc8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call4 = tail call spir_func <16 x i8> @_Z37intel_sub_group_media_block_read_uc16Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call5 = tail call spir_func zeroext i16 @_Z35intel_sub_group_media_block_read_usDv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call6 = tail call spir_func <2 x i16> @_Z36intel_sub_group_media_block_read_us2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call7 = tail call spir_func <4 x i16> @_Z36intel_sub_group_media_block_read_us4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call8 = tail call spir_func <8 x i16> @_Z36intel_sub_group_media_block_read_us8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call9 = tail call spir_func <16 x i16> @_Z37intel_sub_group_media_block_read_us16Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call10 = tail call spir_func i32 @_Z35intel_sub_group_media_block_read_uiDv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call11 = tail call spir_func <2 x i32> @_Z36intel_sub_group_media_block_read_ui2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call12 = tail call spir_func <4 x i32> @_Z36intel_sub_group_media_block_read_ui4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  %call13 = tail call spir_func <8 x i32> @_Z36intel_sub_group_media_block_read_ui8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) %src_luma_image)
+  tail call spir_func void @_Z36intel_sub_group_media_block_write_ucDv2_iiih14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, i8 noundef zeroext %call, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_uc2Dv2_iiiDv2_h14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <2 x i8> noundef %call1, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_uc4Dv2_iiiDv4_h14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <4 x i8> noundef %call2, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_uc8Dv2_iiiDv8_h14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <8 x i8> noundef %call3, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z38intel_sub_group_media_block_write_uc16Dv2_iiiDv16_h14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <16 x i8> noundef %call4, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z36intel_sub_group_media_block_write_usDv2_iiit14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, i16 noundef zeroext %call5, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_us2Dv2_iiiDv2_t14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <2 x i16> noundef %call6, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_us4Dv2_iiiDv4_t14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <4 x i16> noundef %call7, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_us8Dv2_iiiDv8_t14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <8 x i16> noundef %call8, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z38intel_sub_group_media_block_write_us16Dv2_iiiDv16_t14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <16 x i16> noundef %call9, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z36intel_sub_group_media_block_write_uiDv2_iiij14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, i32 noundef %call10, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_ui2Dv2_iiiDv2_j14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <2 x i32> noundef %call11, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_ui4Dv2_iiiDv4_j14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <4 x i32> noundef %call12, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  tail call spir_func void @_Z37intel_sub_group_media_block_write_ui8Dv2_iiiDv8_j14ocl_image2d_wo(<2 x i32> noundef %edgeCoord, i32 noundef 1, i32 noundef 16, <8 x i32> noundef %call13, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) %dst_luma_image)
+  ret void
+}
+
+declare spir_func zeroext i8 @_Z35intel_sub_group_media_block_read_ucDv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <2 x i8> @_Z36intel_sub_group_media_block_read_uc2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <4 x i8> @_Z36intel_sub_group_media_block_read_uc4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <8 x i8> @_Z36intel_sub_group_media_block_read_uc8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <16 x i8> @_Z37intel_sub_group_media_block_read_uc16Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func zeroext i16 @_Z35intel_sub_group_media_block_read_usDv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <2 x i16> @_Z36intel_sub_group_media_block_read_us2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <4 x i16> @_Z36intel_sub_group_media_block_read_us4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <8 x i16> @_Z36intel_sub_group_media_block_read_us8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <16 x i16> @_Z37intel_sub_group_media_block_read_us16Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func i32 @_Z35intel_sub_group_media_block_read_uiDv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <2 x i32> @_Z36intel_sub_group_media_block_read_ui2Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <4 x i32> @_Z36intel_sub_group_media_block_read_ui4Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func <8 x i32> @_Z36intel_sub_group_media_block_read_ui8Dv2_iii14ocl_image2d_ro(<2 x i32> noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0))
+
+declare spir_func void @_Z36intel_sub_group_media_block_write_ucDv2_iiih14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, i8 noundef zeroext, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_uc2Dv2_iiiDv2_h14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <2 x i8> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_uc4Dv2_iiiDv4_h14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <4 x i8> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_uc8Dv2_iiiDv8_h14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <8 x i8> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z38intel_sub_group_media_block_write_uc16Dv2_iiiDv16_h14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <16 x i8> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z36intel_sub_group_media_block_write_usDv2_iiit14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, i16 noundef zeroext, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_us2Dv2_iiiDv2_t14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <2 x i16> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_us4Dv2_iiiDv4_t14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <4 x i16> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_us8Dv2_iiiDv8_t14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <8 x i16> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z38intel_sub_group_media_block_write_us16Dv2_iiiDv16_t14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <16 x i16> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z36intel_sub_group_media_block_write_uiDv2_iiij14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, i32 noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_ui2Dv2_iiiDv2_j14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <2 x i32> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_ui4Dv2_iiiDv4_j14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <4 x i32> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+declare spir_func void @_Z37intel_sub_group_media_block_write_ui8Dv2_iiiDv8_j14ocl_image2d_wo(<2 x i32> noundef, i32 noundef, i32 noundef, <8 x i32> noundef, target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1))
+
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{!"clang version 20.0.0git (https://github.com/llvm/llvm-project.git 32da1fd8c7d45d5209c6c781910c51940779ec52)"}
+!3 = !{i32 0, i32 1, i32 1}
+!4 = !{!"none", !"read_only", !"write_only"}
+!5 = !{!"int2", !"image2d_t", !"image2d_t"}
+!6 = !{!"int __attribute__((ext_vector_type(2)))", !"image2d_t", !"image2d_t"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
index 1744ec9680401..9830b8b4cd2d8 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
@@ -1,25 +1,20 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_optnone %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXTENSION
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NO-EXTENSION
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_optnone %s -o - | FileCheck %s --check-prefixes=CHECK-EXTENSION
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-NO-EXTENSION
 
-; CHECK-EXTENSION: OpCapability OptNoneINTEL
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_optnone %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-EXTENSION: OpCapability OptNoneEXT
 ; CHECK-EXTENSION: OpExtension "SPV_INTEL_optnone"
 ; CHECK-NO-EXTENSION-NOT: OpCapability OptNoneINTEL
+; CHECK-NO-EXTENSION-NOT: OpCapability OptNoneEXT
 ; CHECK-NO-EXTENSION-NOT: OpExtension "SPV_INTEL_optnone"
+; CHECK-NO-EXTENSION-NOT: OpExtension "SPV_EXT_optnone"
 
-; Function Attrs: nounwind optnone noinline
 define spir_func void @_Z3foov() #0 {
-; CHECK-LABEL: _Z3foov
-; CHECK:       %4 = OpFunction %2 DontInline %3
-; CHECK-NEXT:    %5 = OpLabel
-; CHECK-NEXT:    OpReturn
-; CHECK-NEXT:    OpFunctionEnd
+; CHECK-EXTENSION: %[[#]] = OpFunction %[[#]] DontInline|OptNoneEXT %[[#]]
 entry:
   ret void
 }
 
 attributes #0 = { nounwind optnone noinline }
-
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-EXTENSION: {{.*}}
-; CHECK-NO-EXTENSION: {{.*}}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
index 9374e154a0239..457b3a14fd8fc 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
@@ -37,7 +37,8 @@
 
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-ERROR: LLVM ERROR: intel_sub_group_shuffle: the builtin requires the following SPIR-V extension: SPV_INTEL_subgroups
 
@@ -76,9 +77,6 @@
 
 ; CHECK-SPIRV-LABEL: Return
 
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
-target triple = "spir64"
-
 %opencl.image2d_ro_t = type opaque
 %opencl.image2d_wo_t = type opaque
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_subgroup_local_block_io.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_subgroup_local_block_io.ll
new file mode 100644
index 0000000000000..8274c917bbe21
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_subgroup_local_block_io.ll
@@ -0,0 +1,156 @@
+; Modified from: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
+
+;Source:
+;void __kernel test(float2 x, uint c,
+;                   read_only image2d_t image_in,
+;                   write_only image2d_t image_out,
+;                   int2 coord,
+;                   __local uint* p,
+;                   __local ushort* sp,
+;                   __local uchar* cp,
+;                   __local ulong* lp) {
+;
+;    uint2 ui2 = intel_sub_group_block_read2(image_in, coord);
+;    intel_sub_group_block_write2(image_out, coord, ui2);
+;    ui2 = intel_sub_group_block_read2(p);
+;    intel_sub_group_block_write2(p, ui2);
+;
+;    ushort2 us2 = intel_sub_group_block_read_us2(image_in, coord);
+;    intel_sub_group_block_write_us2(image_out, coord, us2);
+;    us2 = intel_sub_group_block_read_us2(sp);
+;    intel_sub_group_block_write_us2(sp, us2);
+;
+;    uchar2 uc2 = intel_sub_group_block_read_uc2(image_in, coord);
+;    intel_sub_group_block_write_uc2(image_out, coord, uc2);
+;    uc2 = intel_sub_group_block_read_uc2(cp);
+;    intel_sub_group_block_write_uc2(cp, uc2);
+;
+;    ulong2 ul2 = intel_sub_group_block_read_ul2(image_in, coord);
+;    intel_sub_group_block_write_ul2(image_out, coord, ul2);
+;    ul2 = intel_sub_group_block_read_ul2(lp);
+;    intel_sub_group_block_write_ul2(lp, ul2);
+;}
+
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-ERROR: LLVM ERROR: intel_sub_group_block_read2: the builtin requires the following SPIR-V extension: SPV_INTEL_subgroups
+
+; CHECK-DAG: Capability SubgroupBufferBlockIOINTEL
+; CHECK-DAG: Capability SubgroupImageBlockIOINTEL
+; CHECK: Extension "SPV_INTEL_subgroups"
+
+; CHECK-SPIRV-LABEL: Function
+; CHECK-SPIRV-LABEL: Label
+
+; CHECK: SubgroupImageBlockReadINTEL
+; CHECK: SubgroupImageBlockWriteINTEL
+; CHECK: SubgroupBlockReadINTEL
+; CHECK: SubgroupBlockWriteINTEL
+
+; CHECK: SubgroupImageBlockReadINTEL
+; CHECK: SubgroupImageBlockWriteINTEL
+; CHECK: SubgroupBlockReadINTEL
+; CHECK: SubgroupBlockWriteINTEL
+
+; CHECK: SubgroupImageBlockReadINTEL
+; CHECK: SubgroupImageBlockWriteINTEL
+; CHECK: SubgroupBlockReadINTEL
+; CHECK: SubgroupBlockWriteINTEL
+
+; CHECK: SubgroupImageBlockReadINTEL
+; CHECK: SubgroupImageBlockWriteINTEL
+; CHECK: SubgroupBlockReadINTEL
+; CHECK: SubgroupBlockWriteINTEL
+
+; CHECK-SPIRV-LABEL: Return
+
+%opencl.image2d_ro_t = type opaque
+%opencl.image2d_wo_t = type opaque
+
+; Function Attrs: convergent nounwind
+define spir_kernel void @test(<2 x float> %x, i32 %c, ptr addrspace(3) %image_in, ptr addrspace(3) %image_out, <2 x i32> %coord, ptr addrspace(3) %p, ptr addrspace(3) %sp, ptr addrspace(3) %cp, ptr addrspace(3) %lp) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 !kernel_arg_name !6 {
+entry:
+  %call4 = tail call spir_func <2 x i32> @_Z27intel_sub_group_block_read214ocl_image2d_roDv2_i(ptr addrspace(3) %image_in, <2 x i32> %coord)
+  tail call spir_func void @_Z28intel_sub_group_block_write214ocl_image2d_woDv2_iDv2_j(ptr addrspace(3) %image_out, <2 x i32> %coord, <2 x i32> %call4)
+  %call5 = tail call spir_func <2 x i32> @_Z27intel_sub_group_block_read2PU3AS1Kj(ptr addrspace(3) %p)
+  tail call spir_func void @_Z28intel_sub_group_block_write2PU3AS1jDv2_j(ptr addrspace(3) %p, <2 x i32> %call5)
+
+  %call6 = tail call spir_func <2 x i16> @_Z30intel_sub_group_block_read_us214ocl_image2d_roDv2_i(ptr addrspace(3) %image_in, <2 x i32> %coord)
+  tail call spir_func void @_Z31intel_sub_group_block_write_us214ocl_image2d_woDv2_iDv2_t(ptr addrspace(3) %image_out, <2 x i32> %coord, <2 x i16> %call6)
+  %call7 = tail call spir_func <2 x i16> @_Z30intel_sub_group_block_read_us2PU3AS1Kt(ptr addrspace(3) %sp)
+  tail call spir_func void @_Z31intel_sub_group_block_write_us2PU3AS1tDv2_t(ptr addrspace(3) %sp, <2 x i16> %call7)
+
+  %call8 = tail call spir_func <2 x i8> @_Z30intel_sub_group_block_read_uc214ocl_image2d_roDv2_i(ptr addrspace(3) %image_in, <2 x i32> %coord)
+  tail call spir_func void @_Z31intel_sub_group_block_write_uc214ocl_image2d_woDv2_iDv2_h(ptr addrspace(3) %image_out, <2 x i32> %coord, <2 x i8> %call8)
+  %call9 = tail call spir_func <2 x i8> @_Z30intel_sub_group_block_read_uc2PU3AS1Kh(ptr addrspace(3) %cp)
+  tail call spir_func void @_Z31intel_sub_group_block_write_uc2PU3AS1hDv2_h(ptr addrspace(3) %cp, <2 x i8> %call9)
+
+  %call10 = tail call spir_func <2 x i64> @_Z30intel_sub_group_block_read_ul214ocl_image2d_roDv2_i(ptr addrspace(3) %image_in, <2 x i32> %coord)
+  tail call spir_func void @_Z31intel_sub_group_block_write_ul214ocl_image2d_woDv2_iDv2_m(ptr addrspace(3) %image_out, <2 x i32> %coord, <2 x i64> %call10)
+  %call11 = tail call spir_func <2 x i64> @_Z30intel_sub_group_block_read_ul2PU3AS1Km(ptr addrspace(3) %lp)
+  tail call spir_func void @_Z31intel_sub_group_block_write_ul2PU3AS1mDv2_m(ptr addrspace(3) %lp, <2 x i64> %call11)
+
+  ret void
+}
+
+; Function Attrs: convergent
+declare spir_func <2 x i32> @_Z27intel_sub_group_block_read214ocl_image2d_roDv2_i(ptr addrspace(3), <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func void @_Z28intel_sub_group_block_write214ocl_image2d_woDv2_iDv2_j(ptr addrspace(3), <2 x i32>, <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i32> @_Z27intel_sub_group_block_read2PU3AS1Kj(ptr addrspace(3))
+
+; Function Attrs: convergent
+declare spir_func void @_Z28intel_sub_group_block_write2PU3AS1jDv2_j(ptr addrspace(3), <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i16> @_Z30intel_sub_group_block_read_us214ocl_image2d_roDv2_i(ptr addrspace(3), <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_us214ocl_image2d_woDv2_iDv2_t(ptr addrspace(3), <2 x i32>, <2 x i16>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i16> @_Z30intel_sub_group_block_read_us2PU3AS1Kt(ptr addrspace(3))
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_us2PU3AS1tDv2_t(ptr addrspace(3), <2 x i16>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i8> @_Z30intel_sub_group_block_read_uc214ocl_image2d_roDv2_i(ptr addrspace(3), <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_uc214ocl_image2d_woDv2_iDv2_h(ptr addrspace(3), <2 x i32>, <2 x i8>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i8> @_Z30intel_sub_group_block_read_uc2PU3AS1Kh(ptr addrspace(3))
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_uc2PU3AS1hDv2_h(ptr addrspace(3), <2 x i8>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i64> @_Z30intel_sub_group_block_read_ul214ocl_image2d_roDv2_i(ptr addrspace(3), <2 x i32>)
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_ul214ocl_image2d_woDv2_iDv2_m(ptr addrspace(3), <2 x i32>, <2 x i64>)
+
+; Function Attrs: convergent
+declare spir_func <2 x i64> @_Z30intel_sub_group_block_read_ul2PU3AS1Km(ptr addrspace(3))
+
+; Function Attrs: convergent
+declare spir_func void @_Z31intel_sub_group_block_write_ul2PU3AS1mDv2_m(ptr addrspace(3), <2 x i64>)
+
+!opencl.ocl.version = !{!0}
+!opencl.spir.version = !{!0}
+
+!0 = !{i32 1, i32 2}
+!1 = !{i32 0, i32 0, i32 1, i32 1, i32 0, i32 1, i32 1, i32 1, i32 1}
+!2 = !{!"none", !"none", !"read_only", !"write_only", !"none", !"none", !"none", !"none", !"none"}
+!3 = !{!"float2", !"uint", !"image2d_t", !"image2d_t", !"int2", !"uint*", !"ushort*", !"uchar*", !"ulong*"}
+!4 = !{!"float __attribute__((ext_vector_type(2)))", !"uint", !"image2d_t", !"image2d_t", !"int __attribute__((ext_vector_type(2)))", !"uint*", !"ushort*", !"uchar*", !"ulong*"}
+!5 = !{!"", !"", !"", !"", !"", !"", !"", !"", !""}
+!6 = !{!"x", !"c", !"image_in", !"image_out", !"coord", !"p", !"sp", !"cp", !"lp"}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
index 1c41c7331cda8..e290c1eaeabad 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
@@ -21,7 +21,7 @@
 ; CHECK: %[[#Load1:]] = OpCooperativeMatrixLoadKHR %[[#MatTy2]]
 ; CHECK: OpCooperativeMatrixLengthKHR %[[#Int32Ty]] %[[#MatTy2:]]
 ; CHECK: OpCooperativeMatrixLoadKHR %[[#MatTy3]]
-; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] %[[#]] %[[#]] %[[#]] MatrixCSignedComponentsKHR|MatrixResultSignedComponentsKHR
 ; CHECK: OpCooperativeMatrixStoreKHR
 
 define spir_kernel void @matr_mult(ptr addrspace(1) align 1 %_arg_accA, ptr addrspace(1) align 1 %_arg_accB, ptr addrspace(1) align 4 %_arg_accC, i64 %_arg_N, i64 %_arg_K) {
diff --git a/llvm/test/CodeGen/SPIRV/half_no_extension.ll b/llvm/test/CodeGen/SPIRV/half_no_extension.ll
index a5b0ec9c92d23..63cecbc7399ae 100644
--- a/llvm/test/CodeGen/SPIRV/half_no_extension.ll
+++ b/llvm/test/CodeGen/SPIRV/half_no_extension.ll
@@ -5,10 +5,11 @@
 ;;   vstorea_half4_rtp( data, 0, f );
 ;; }
 
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK-SPIRV:     OpCapability Float16Buffer
-; CHECK-SPIRV-NOT: OpCapability Float16
+; CHECK-DAG: OpCapability Float16Buffer
+; CHECK-DAG: OpCapability Float16
 
 define spir_kernel void @test(<4 x float> addrspace(1)* %p, half addrspace(1)* %f) {
 entry:
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupThreadID.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupThreadID.ll
new file mode 100644
index 0000000000000..a88debf97fa7b
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupThreadID.ll
@@ -0,0 +1,76 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
+
+; This file generated from the following command:
+; clang -cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -finclude-default-header - -o - <<EOF
+; [shader("compute")]
+; [numthreads(1,1,1)]
+; void main(uint3 ID : SV_GroupThreadID) {}
+; EOF
+
+; CHECK-DAG:        %[[#int:]] = OpTypeInt 32 0
+; CHECK-DAG:        %[[#v3int:]] = OpTypeVector %[[#int]] 3
+; CHECK-DAG:        %[[#ptr_Input_v3int:]] = OpTypePointer Input %[[#v3int]]
+; CHECK-DAG:        %[[#tempvar:]] = OpUndef %[[#v3int]]
+; CHECK-DAG:        %[[#LocalInvocationId:]] = OpVariable %[[#ptr_Input_v3int]] Input
+
+; CHECK-DAG:        OpEntryPoint GLCompute {{.*}} %[[#LocalInvocationId]]
+; CHECK-DAG:        OpName %[[#LocalInvocationId]] "__spirv_BuiltInLocalInvocationId"
+; CHECK-DAG:        OpDecorate %[[#LocalInvocationId]] LinkageAttributes "__spirv_BuiltInLocalInvocationId" Import
+; CHECK-DAG:        OpDecorate %[[#LocalInvocationId]] BuiltIn LocalInvocationId
+
+; ModuleID = '-'
+source_filename = "-"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spirv-unknown-vulkan-library"
+
+; Function Attrs: noinline norecurse nounwind optnone
+define internal spir_func void @main(<3 x i32> noundef %ID) #0 {
+entry:
+  %ID.addr = alloca <3 x i32>, align 16
+  store <3 x i32> %ID, ptr %ID.addr, align 16
+  ret void
+}
+
+; Function Attrs: norecurse
+define void @main.1() #1 {
+entry:
+
+; CHECK:        %[[#load:]] = OpLoad %[[#v3int]] %[[#LocalInvocationId]]
+; CHECK:        %[[#load0:]] = OpCompositeExtract %[[#int]] %[[#load]] 0
+  %0 = call i32 @llvm.spv.thread.id.in.group(i32 0)
+
+; CHECK:        %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load0]] %[[#tempvar]] 0
+  %1 = insertelement <3 x i32> poison, i32 %0, i64 0
+
+; CHECK:        %[[#load:]] = OpLoad %[[#v3int]] %[[#LocalInvocationId]]
+; CHECK:        %[[#load1:]] = OpCompositeExtract %[[#int]] %[[#load]] 1
+  %2 = call i32 @llvm.spv.thread.id.in.group(i32 1)
+
+; CHECK:        %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load1]] %[[#tempvar]] 1
+  %3 = insertelement <3 x i32> %1, i32 %2, i64 1
+
+; CHECK:        %[[#load:]] = OpLoad %[[#v3int]] %[[#LocalInvocationId]]
+; CHECK:        %[[#load2:]] = OpCompositeExtract %[[#int]] %[[#load]] 2
+  %4 = call i32 @llvm.spv.thread.id.in.group(i32 2)
+
+; CHECK:        %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load2]] %[[#tempvar]] 2
+  %5 = insertelement <3 x i32> %3, i32 %4, i64 2
+
+  call void @main(<3 x i32> %5)
+  ret void
+}
+
+; Function Attrs: nounwind willreturn memory(none)
+declare i32 @llvm.spv.thread.id.in.group(i32) #2
+
+attributes #0 = { noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{!"clang version 19.0.0git (git@github.com:llvm/llvm-project.git 91600507765679e92434ec7c5edb883bf01f847f)"}
diff --git a/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll b/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
index 354434c37201d..82b115c77987f 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
@@ -2,10 +2,6 @@
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --translator-compatibility-mode %s -o - -filetype=obj | spirv-val %}
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; Modifying the block ordering prevents the pointer types to correctly be deduced. Not sure why, but looks
-; orthogonal to the block sorting.
-; XFAIL: *
-
 ; CHECK-DAG: OpName %[[Test1:.*]] "test1"
 ; CHECK-DAG: OpName %[[Foo:.*]] "foo"
 ; CHECK-DAG: OpName %[[Bar:.*]] "bar"
@@ -17,16 +13,16 @@
 ; CHECK-DAG: %[[Struct2:.*]] = OpTypeStruct %[[Struct1]]
 ; CHECK-DAG: %[[StructPtr:.*]] = OpTypePointer Function %[[Struct2]]
 ; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
-; CHECK-DAG: %[[FooType:.*]] = OpTypeFunction %[[StructPtr:.*]] %[[StructPtr]] %[[StructPtr]] %[[Bool]]
+; CHECK-DAG: %[[FooType:.*]] = OpTypeFunction %[[StructPtr]] %[[StructPtr]] %[[StructPtr]] %[[Bool]]
 ; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[CharPtr:.*]] = OpTypePointer Function %[[Char]]
 
 ; CHECK: %[[Test1]] = OpFunction
-; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Foo]]
-; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Bar]]
+; CHECK: OpFunctionCall %[[StructPtr]] %[[Foo]]
+; CHECK: OpFunctionCall %[[CharPtr]] %[[Bar]]
 ; CHECK: OpFunctionEnd
 
-; CHECK: %[[Foo]] = OpFunction %[[StructPtr:.*]] None %[[FooType]]
+; CHECK: %[[Foo]] = OpFunction %[[StructPtr]] None %[[FooType]]
 ; CHECK: %[[Arg1:.*]] = OpFunctionParameter %[[StructPtr]]
 ; CHECK: %[[Arg2:.*]] = OpFunctionParameter
 ; CHECK: %[[Sw:.*]] = OpFunctionParameter
@@ -34,17 +30,18 @@
 ; CHECK: OpReturnValue %[[Res]]
 ; CHECK: OpReturnValue %[[Arg2]]
 
-; CHECK: %[[Bar]] = OpFunction %[[StructPtr:.*]] None %[[#]]
-; CHECK: %[[BarArg:.*]] = OpFunctionParameter
-; CHECK: %[[BarRes:.*]] = OpInBoundsPtrAccessChain %[[CharPtr]] %[[BarArg]] %[[#]]
-; CHECK: %[[BarResCasted:.*]] = OpBitcast %[[StructPtr]] %[[BarRes]]
+; CHECK: %[[Bar]] = OpFunction %[[CharPtr]] None %[[#]]
+; CHECK: %[[BarArg:.*]] = OpFunctionParameter %[[StructPtr]]
+; CHECK: %[[BarArgCasted:.*]] = OpBitcast %[[CharPtr]] %[[BarArg]]
+; CHECK: %[[BarRes:.*]] = OpInBoundsPtrAccessChain %[[CharPtr]] %[[BarArgCasted]] %[[#]]
 ; CHECK: %[[BarResStruct:.*]] = OpInBoundsPtrAccessChain %[[StructPtr]] %[[#]] %[[#]]
-; CHECK: OpReturnValue %[[BarResStruct]]
-; CHECK: OpReturnValue %[[BarResCasted]]
+; CHECK: %[[BarResStructCasted:.*]] = OpBitcast %[[CharPtr]] %[[BarResStruct]]
+; CHECK: OpReturnValue %[[BarResStructCasted]]
+; CHECK: OpReturnValue %[[BarRes]]
 
 ; CHECK: %[[Test2]] = OpFunction
-; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Foo]]
-; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Bar]]
+; CHECK: OpFunctionCall %[[StructPtr]] %[[Foo]]
+; CHECK: OpFunctionCall %[[CharPtr]] %[[Bar]]
 ; CHECK: OpFunctionEnd
 
 %struct = type { %array }
diff --git a/llvm/test/CodeGen/SPIRV/keep-tracked-const.ll b/llvm/test/CodeGen/SPIRV/keep-tracked-const.ll
index 0c25d8a67aca0..0dc86233f8e16 100644
--- a/llvm/test/CodeGen/SPIRV/keep-tracked-const.ll
+++ b/llvm/test/CodeGen/SPIRV/keep-tracked-const.ll
@@ -1,6 +1,6 @@
 ; This test case ensures that cleaning of temporary constants doesn't purge tracked ones.
 
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV: %[[#Int:]] = OpTypeInt 32 0
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/scmp.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/scmp.ll
new file mode 100644
index 0000000000000..6638204717517
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/scmp.ll
@@ -0,0 +1,148 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[TypeI8:.*]] = OpTypeInt 8 0 
+; CHECK-DAG: %[[TypeI16:.*]] = OpTypeInt 16 0 
+; CHECK-DAG: %[[TypeI32:.*]] = OpTypeInt 32 0 
+; CHECK-DAG: %[[TypeI64:.*]] = OpTypeInt 64 0 
+
+; CHECK-DAG: %[[CmpI64ConstMinusOne:.*]] = OpConstant %[[TypeI64]] 18446744073709551615
+
+; CHECK-DAG: %[[CmpI8ConstOne:.*]] = OpConstant %[[TypeI8]] 1 
+; CHECK-DAG: %[[CmpI8ConstZero:.*]] = OpConstantNull %[[TypeI8]]
+; CHECK-DAG: %[[CmpI8ConstMinusOne:.*]] = OpConstant %[[TypeI8]] 255 
+
+; CHECK-DAG: %[[CmpI16ConstOne:.*]] = OpConstant %[[TypeI16]] 1 
+; CHECK-DAG: %[[CmpI16ConstZero:.*]] = OpConstantNull %[[TypeI16]]
+; CHECK-DAG: %[[CmpI16ConstMinusOne:.*]] = OpConstant %[[TypeI16]] 65535 
+
+; CHECK-DAG: %[[CmpI32ConstOne:.*]] = OpConstant %[[TypeI32]] 1 
+; CHECK-DAG: %[[CmpI32ConstZero:.*]] = OpConstantNull %[[TypeI32]]
+; CHECK-DAG: %[[CmpI32ConstMinusOne:.*]] = OpConstant %[[TypeI32]] 4294967295 
+
+; CHECK-DAG: %[[CmpI64ConstOne:.*]] = OpConstant %[[TypeI64]] 1 
+; CHECK-DAG: %[[CmpI64ConstZero:.*]] = OpConstantNull %[[TypeI64]]
+
+; CHECK-DAG: %[[TypeBool:.*]] = OpTypeBool
+; CHECK-DAG: %[[TypeVBool:.*]] = OpTypeVector %[[TypeBool]] 4
+
+; CHECK-DAG: %[[TypeV4I8:.*]] = OpTypeVector %[[TypeI8]] 4 
+; CHECK-DAG: %[[TypeV4I16:.*]] = OpTypeVector %[[TypeI16]] 4 
+; CHECK-DAG: %[[TypeV4I32:.*]] = OpTypeVector %[[TypeI32]] 4 
+; CHECK-DAG: %[[TypeV4I64:.*]] = OpTypeVector %[[TypeI64]] 4 
+
+; CHECK-DAG: %[[V4I8ConstOne:.*]] = OpConstantComposite %[[TypeV4I8]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]]
+; CHECK-DAG: %[[V4I8ConstZero:.*]] = OpConstantNull %[[TypeV4I8]]
+; CHECK-DAG: %[[V4I8ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I8]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I16ConstOne:.*]] = OpConstantComposite %[[TypeV4I16]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]]
+; CHECK-DAG: %[[V4I16ConstZero:.*]] = OpConstantNull %[[TypeV4I16]]
+; CHECK-DAG: %[[V4I16ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I16]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I32ConstOne:.*]] = OpConstantComposite %[[TypeV4I32]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]]
+; CHECK-DAG: %[[V4I32ConstZero:.*]] = OpConstantNull %[[TypeV4I32]]
+; CHECK-DAG: %[[V4I32ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I32]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I64ConstOne:.*]] = OpConstantComposite %[[TypeV4I64]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]]
+; CHECK-DAG: %[[V4I64ConstZero:.*]] = OpConstantNull %[[TypeV4I64]]
+; CHECK-DAG: %[[V4I64ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I64]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]]
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI8R1:.*]] = OpSLessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI8R2:.*]] = OpSLessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI8R1:.*]] = OpSelect %[[TypeI8]] %[[CmpI8R2]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstZero]]
+; CHECK: %[[SelI8R2:.*]] = OpSelect %[[TypeI8]] %[[CmpI8R1]] %[[SelI8R1]] %[[CmpI8ConstOne]] 
+; CHECK: OpReturnValue %[[SelI8R2]] 
+define range(i8 -1, 2) i8 @test_i8(i8 noundef %0, i8 noundef %1) {
+  %3 = tail call i8 @llvm.scmp.i8.i8(i8 %0, i8 %1)
+  ret i8 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI16R1:.*]] = OpSLessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI16R2:.*]] = OpSLessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI16R1:.*]] = OpSelect %[[TypeI16]] %[[CmpI16R2]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstZero]]
+; CHECK: %[[SelI16R2:.*]] = OpSelect %[[TypeI16]] %[[CmpI16R1]] %[[SelI16R1]] %[[CmpI16ConstOne]] 
+; CHECK: OpReturnValue %[[SelI16R2]] 
+define range(i16 -1, 2) i16 @test_i16(i16 noundef %0, i16 noundef %1) {
+  %3 = tail call i16 @llvm.scmp.i16.i16(i16 %0, i16 %1)
+  ret i16 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI32R1:.*]] = OpSLessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI32R2:.*]] = OpSLessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI32R1:.*]] = OpSelect %[[TypeI32]] %[[CmpI32R2]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstZero]]
+; CHECK: %[[SelI32R2:.*]] = OpSelect %[[TypeI32]] %[[CmpI32R1]] %[[SelI32R1]] %[[CmpI32ConstOne]] 
+; CHECK: OpReturnValue %[[SelI32R2]] 
+define range(i32 -1, 2) i32 @test_i32(i32 noundef %0, i32 noundef %1) {
+  %3 = tail call i32 @llvm.scmp.i32.i32(i32 %0, i32 %1)
+  ret i32 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI64R1:.*]] = OpSLessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI64R2:.*]] = OpSLessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI64R1:.*]] = OpSelect %[[TypeI64]] %[[CmpI64R2]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstZero]]
+; CHECK: %[[SelI64R2:.*]] = OpSelect %[[TypeI64]] %[[CmpI64R1]] %[[SelI64R1]] %[[CmpI64ConstOne]] 
+; CHECK: OpReturnValue %[[SelI64R2]] 
+define range(i64 -1, 2) i64 @test_i64(i64 noundef %0, i64 noundef %1) {
+  %3 = tail call i64 @llvm.scmp.i64.i64(i64 %0, i64 %1)
+  ret i64 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I8R1:.*]] = OpSLessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I8R2:.*]] = OpSLessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I8R1:.*]] = OpSelect %[[TypeV4I8]] %[[V4I8R2]] %[[V4I8ConstMinusOne]] %[[V4I8ConstZero]]
+; CHECK: %[[SelectV4I8R2:.*]] = OpSelect %[[TypeV4I8]] %[[V4I8R1]] %[[SelectV4I8R1]] %[[V4I8ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I8R2]] 
+define range(i8 -1, 2) <4 x i8> @test_v4i8(<4 x i8> noundef %0, <4 x i8> noundef %1) {
+  %3 = tail call <4 x i8> @llvm.scmp.v4i8.v4i8(<4 x i8> %0, <4 x i8> %1)
+  ret <4 x i8> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I16R1:.*]] = OpSLessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I16R2:.*]] = OpSLessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I16R1:.*]] = OpSelect %[[TypeV4I16]] %[[V4I16R2]] %[[V4I16ConstMinusOne]] %[[V4I16ConstZero]]
+; CHECK: %[[SelectV4I16R2:.*]] = OpSelect %[[TypeV4I16]] %[[V4I16R1]] %[[SelectV4I16R1]] %[[V4I16ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I16R2]] 
+define range(i16 -1, 2) <4 x i16> @test_v4i16(<4 x i16> noundef %0, <4 x i16> noundef %1) {
+  %3 = tail call <4 x i16> @llvm.scmp.v4i16.v4i16(<4 x i16> %0, <4 x i16> %1)
+  ret <4 x i16> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I32R1:.*]] = OpSLessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I32R2:.*]] = OpSLessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I32R1:.*]] = OpSelect %[[TypeV4I32]] %[[V4I32R2]] %[[V4I32ConstMinusOne]] %[[V4I32ConstZero]]
+; CHECK: %[[SelectV4I32R2:.*]] = OpSelect %[[TypeV4I32]] %[[V4I32R1]] %[[SelectV4I32R1]] %[[V4I32ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I32R2]] 
+define range(i32 -1, 2) <4 x i32> @test_v4i32(<4 x i32> noundef %0, <4 x i32> noundef %1) {
+  %3 = tail call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %0, <4 x i32> %1)
+  ret <4 x i32> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I64R1:.*]] = OpSLessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I64R2:.*]] = OpSLessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I64R1:.*]] = OpSelect %[[TypeV4I64]] %[[V4I64R2]] %[[V4I64ConstMinusOne]] %[[V4I64ConstZero]]
+; CHECK: %[[SelectV4I64R2:.*]] = OpSelect %[[TypeV4I64]] %[[V4I64R1]] %[[SelectV4I64R1]] %[[V4I64ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I64R2]] 
+define range(i64 -1, 2) <4 x i64> @test_v4i64(<4 x i64> noundef %0, <4 x i64> noundef %1) {
+  %3 = tail call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %0, <4 x i64> %1)
+  ret <4 x i64> %3
+}
+
+declare i8 @llvm.scmp.i8.i8(i8, i8)
+declare i16 @llvm.scmp.i16.i16(i16, i16)
+declare i32 @llvm.scmp.i32.i32(i32, i32)
+declare i64 @llvm.scmp.i64.i64(i64, i64)
+declare <4 x i8> @llvm.scmp.v4i8.v4i8(<4 x i8>, <4 x i8>)
+declare <4 x i16> @llvm.scmp.v4i16.v4i16(<4 x i16>, <4 x i16>)
+declare <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64>, <4 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ucmp.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ucmp.ll
new file mode 100644
index 0000000000000..a5c2899131990
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ucmp.ll
@@ -0,0 +1,148 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[TypeI8:.*]] = OpTypeInt 8 0 
+; CHECK-DAG: %[[TypeI16:.*]] = OpTypeInt 16 0 
+; CHECK-DAG: %[[TypeI32:.*]] = OpTypeInt 32 0 
+; CHECK-DAG: %[[TypeI64:.*]] = OpTypeInt 64 0 
+
+; CHECK-DAG: %[[CmpI64ConstMinusOne:.*]] = OpConstant %[[TypeI64]] 18446744073709551615
+
+; CHECK-DAG: %[[CmpI8ConstOne:.*]] = OpConstant %[[TypeI8]] 1 
+; CHECK-DAG: %[[CmpI8ConstZero:.*]] = OpConstantNull %[[TypeI8]]
+; CHECK-DAG: %[[CmpI8ConstMinusOne:.*]] = OpConstant %[[TypeI8]] 255 
+
+; CHECK-DAG: %[[CmpI16ConstOne:.*]] = OpConstant %[[TypeI16]] 1 
+; CHECK-DAG: %[[CmpI16ConstZero:.*]] = OpConstantNull %[[TypeI16]]
+; CHECK-DAG: %[[CmpI16ConstMinusOne:.*]] = OpConstant %[[TypeI16]] 65535 
+
+; CHECK-DAG: %[[CmpI32ConstOne:.*]] = OpConstant %[[TypeI32]] 1 
+; CHECK-DAG: %[[CmpI32ConstZero:.*]] = OpConstantNull %[[TypeI32]]
+; CHECK-DAG: %[[CmpI32ConstMinusOne:.*]] = OpConstant %[[TypeI32]] 4294967295 
+
+; CHECK-DAG: %[[CmpI64ConstOne:.*]] = OpConstant %[[TypeI64]] 1 
+; CHECK-DAG: %[[CmpI64ConstZero:.*]] = OpConstantNull %[[TypeI64]]
+
+; CHECK-DAG: %[[TypeBool:.*]] = OpTypeBool
+; CHECK-DAG: %[[TypeVBool:.*]] = OpTypeVector %[[TypeBool]] 4
+
+; CHECK-DAG: %[[TypeV4I8:.*]] = OpTypeVector %[[TypeI8]] 4 
+; CHECK-DAG: %[[TypeV4I16:.*]] = OpTypeVector %[[TypeI16]] 4 
+; CHECK-DAG: %[[TypeV4I32:.*]] = OpTypeVector %[[TypeI32]] 4 
+; CHECK-DAG: %[[TypeV4I64:.*]] = OpTypeVector %[[TypeI64]] 4 
+
+; CHECK-DAG: %[[V4I8ConstOne:.*]] = OpConstantComposite %[[TypeV4I8]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]] %[[CmpI8ConstOne]]
+; CHECK-DAG: %[[V4I8ConstZero:.*]] = OpConstantNull %[[TypeV4I8]]
+; CHECK-DAG: %[[V4I8ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I8]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I16ConstOne:.*]] = OpConstantComposite %[[TypeV4I16]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]] %[[CmpI16ConstOne]]
+; CHECK-DAG: %[[V4I16ConstZero:.*]] = OpConstantNull %[[TypeV4I16]]
+; CHECK-DAG: %[[V4I16ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I16]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I32ConstOne:.*]] = OpConstantComposite %[[TypeV4I32]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]] %[[CmpI32ConstOne]]
+; CHECK-DAG: %[[V4I32ConstZero:.*]] = OpConstantNull %[[TypeV4I32]]
+; CHECK-DAG: %[[V4I32ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I32]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstMinusOne]]
+
+; CHECK-DAG: %[[V4I64ConstOne:.*]] = OpConstantComposite %[[TypeV4I64]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]] %[[CmpI64ConstOne]]
+; CHECK-DAG: %[[V4I64ConstZero:.*]] = OpConstantNull %[[TypeV4I64]]
+; CHECK-DAG: %[[V4I64ConstMinusOne:.*]] = OpConstantComposite %[[TypeV4I64]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstMinusOne]]
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI8R1:.*]] = OpULessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI8R2:.*]] = OpULessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI8R1:.*]] = OpSelect %[[TypeI8]] %[[CmpI8R2]] %[[CmpI8ConstMinusOne]] %[[CmpI8ConstZero]]
+; CHECK: %[[SelI8R2:.*]] = OpSelect %[[TypeI8]] %[[CmpI8R1]] %[[SelI8R1]] %[[CmpI8ConstOne]] 
+; CHECK: OpReturnValue %[[SelI8R2]] 
+define range(i8 -1, 2) i8 @test_i8(i8 noundef %0, i8 noundef %1) {
+  %3 = tail call i8 @llvm.ucmp.i8.i8(i8 %0, i8 %1)
+  ret i8 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI16R1:.*]] = OpULessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI16R2:.*]] = OpULessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI16R1:.*]] = OpSelect %[[TypeI16]] %[[CmpI16R2]] %[[CmpI16ConstMinusOne]] %[[CmpI16ConstZero]]
+; CHECK: %[[SelI16R2:.*]] = OpSelect %[[TypeI16]] %[[CmpI16R1]] %[[SelI16R1]] %[[CmpI16ConstOne]] 
+; CHECK: OpReturnValue %[[SelI16R2]] 
+define range(i16 -1, 2) i16 @test_i16(i16 noundef %0, i16 noundef %1) {
+  %3 = tail call i16 @llvm.ucmp.i16.i16(i16 %0, i16 %1)
+  ret i16 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI32R1:.*]] = OpULessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI32R2:.*]] = OpULessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI32R1:.*]] = OpSelect %[[TypeI32]] %[[CmpI32R2]] %[[CmpI32ConstMinusOne]] %[[CmpI32ConstZero]]
+; CHECK: %[[SelI32R2:.*]] = OpSelect %[[TypeI32]] %[[CmpI32R1]] %[[SelI32R1]] %[[CmpI32ConstOne]] 
+; CHECK: OpReturnValue %[[SelI32R2]] 
+define range(i32 -1, 2) i32 @test_i32(i32 noundef %0, i32 noundef %1) {
+  %3 = tail call i32 @llvm.ucmp.i32.i32(i32 %0, i32 %1)
+  ret i32 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[CmpI64R1:.*]] = OpULessThanEqual %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[CmpI64R2:.*]] = OpULessThan %[[TypeBool]] %[[#]] %[[#]]
+; CHECK: %[[SelI64R1:.*]] = OpSelect %[[TypeI64]] %[[CmpI64R2]] %[[CmpI64ConstMinusOne]] %[[CmpI64ConstZero]]
+; CHECK: %[[SelI64R2:.*]] = OpSelect %[[TypeI64]] %[[CmpI64R1]] %[[SelI64R1]] %[[CmpI64ConstOne]] 
+; CHECK: OpReturnValue %[[SelI64R2]] 
+define range(i64 -1, 2) i64 @test_i64(i64 noundef %0, i64 noundef %1) {
+  %3 = tail call i64 @llvm.ucmp.i64.i64(i64 %0, i64 %1)
+  ret i64 %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I8R1:.*]] = OpULessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I8R2:.*]] = OpULessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I8R1:.*]] = OpSelect %[[TypeV4I8]] %[[V4I8R2]] %[[V4I8ConstMinusOne]] %[[V4I8ConstZero]]
+; CHECK: %[[SelectV4I8R2:.*]] = OpSelect %[[TypeV4I8]] %[[V4I8R1]] %[[SelectV4I8R1]] %[[V4I8ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I8R2]] 
+define range(i8 -1, 2) <4 x i8> @test_v4i8(<4 x i8> noundef %0, <4 x i8> noundef %1) {
+  %3 = tail call <4 x i8> @llvm.ucmp.v4i8.v4i8(<4 x i8> %0, <4 x i8> %1)
+  ret <4 x i8> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I16R1:.*]] = OpULessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I16R2:.*]] = OpULessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I16R1:.*]] = OpSelect %[[TypeV4I16]] %[[V4I16R2]] %[[V4I16ConstMinusOne]] %[[V4I16ConstZero]]
+; CHECK: %[[SelectV4I16R2:.*]] = OpSelect %[[TypeV4I16]] %[[V4I16R1]] %[[SelectV4I16R1]] %[[V4I16ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I16R2]] 
+define range(i16 -1, 2) <4 x i16> @test_v4i16(<4 x i16> noundef %0, <4 x i16> noundef %1) {
+  %3 = tail call <4 x i16> @llvm.ucmp.v4i16.v4i16(<4 x i16> %0, <4 x i16> %1)
+  ret <4 x i16> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I32R1:.*]] = OpULessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I32R2:.*]] = OpULessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I32R1:.*]] = OpSelect %[[TypeV4I32]] %[[V4I32R2]] %[[V4I32ConstMinusOne]] %[[V4I32ConstZero]]
+; CHECK: %[[SelectV4I32R2:.*]] = OpSelect %[[TypeV4I32]] %[[V4I32R1]] %[[SelectV4I32R1]] %[[V4I32ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I32R2]] 
+define range(i32 -1, 2) <4 x i32> @test_v4i32(<4 x i32> noundef %0, <4 x i32> noundef %1) {
+  %3 = tail call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %0, <4 x i32> %1)
+  ret <4 x i32> %3
+}
+
+; CHECK: OpFunction
+; CHECK: %[[V4I64R1:.*]] = OpULessThanEqual %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[V4I64R2:.*]] = OpULessThan %[[TypeVBool]] %[[#]] %[[#]]
+; CHECK: %[[SelectV4I64R1:.*]] = OpSelect %[[TypeV4I64]] %[[V4I64R2]] %[[V4I64ConstMinusOne]] %[[V4I64ConstZero]]
+; CHECK: %[[SelectV4I64R2:.*]] = OpSelect %[[TypeV4I64]] %[[V4I64R1]] %[[SelectV4I64R1]] %[[V4I64ConstOne]] 
+; CHECK: OpReturnValue %[[SelectV4I64R2]] 
+define range(i64 -1, 2) <4 x i64> @test_v4i64(<4 x i64> noundef %0, <4 x i64> noundef %1) {
+  %3 = tail call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %0, <4 x i64> %1)
+  ret <4 x i64> %3
+}
+
+declare i8 @llvm.ucmp.i8.i8(i8, i8)
+declare i16 @llvm.ucmp.i16.i16(i16, i16)
+declare i32 @llvm.ucmp.i32.i32(i32, i32)
+declare i64 @llvm.ucmp.i64.i64(i64, i64)
+declare <4 x i8> @llvm.ucmp.v4i8.v4i8(<4 x i8>, <4 x i8>)
+declare <4 x i16> @llvm.ucmp.v4i16.v4i16(<4 x i16>, <4 x i16>)
+declare <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64>, <4 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/opencl/vload2.ll b/llvm/test/CodeGen/SPIRV/opencl/vload2.ll
index 592de33d4d393..1a1b6c484e74f 100644
--- a/llvm/test/CodeGen/SPIRV/opencl/vload2.ll
+++ b/llvm/test/CodeGen/SPIRV/opencl/vload2.ll
@@ -1,9 +1,14 @@
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; This test only intends to check the vloadn builtin name resolution.
-; The calls to the OpenCL builtins are not valid and will not pass SPIR-V validation.
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#IMPORT:]] = OpExtInstImport "OpenCL.std"
 
+; CHECK-DAG: OpName %[[#CALL1:]] "call1"
+; CHECK-DAG: OpName %[[#CALL2:]] "call2"
+; CHECK-DAG: OpName %[[#CALL3:]] "call3"
+; CHECK-DAG: OpName %[[#CALL4:]] "call4"
+; CHECK-DAG: OpName %[[#CALL5:]] "call5"
+
 ; CHECK-DAG: %[[#INT8:]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[#INT16:]] = OpTypeInt 16 0
 ; CHECK-DAG: %[[#INT32:]] = OpTypeInt 32 0
@@ -23,20 +28,20 @@
 ; CHECK: %[[#OFFSET:]] = OpFunctionParameter %[[#INT64]]
 
 define spir_kernel void @test_fn(i64 %offset, ptr addrspace(1) %address) {
-; CHECK: %[[#CASTorPARAMofPTRI8:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT8]]{{.*}}
-; CHECK: %[[#]] = OpExtInst %[[#VINT8]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI8]] 2
+; CHECK-DAG: %[[#CASTorPARAMofPTRI8:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT8]]{{.*}}
+; CHECK-DAG: %[[#CALL1]] = OpExtInst %[[#VINT8]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI8]] 2
   %call1 = call spir_func <2 x i8> @_Z6vload2mPU3AS1Kc(i64 %offset, ptr addrspace(1) %address)
-; CHECK: %[[#CASTorPARAMofPTRI16:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT16]]{{.*}}
-; CHECK: %[[#]] = OpExtInst %[[#VINT16]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI16]] 2
+; CHECK-DAG: %[[#CASTorPARAMofPTRI16:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT16]]{{.*}}
+; CHECK-DAG: %[[#CALL2]] = OpExtInst %[[#VINT16]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI16]] 2
   %call2 = call spir_func <2 x i16> @_Z6vload2mPU3AS1Ks(i64 %offset, ptr addrspace(1) %address)
-; CHECK: %[[#CASTorPARAMofPTRI32:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT32]]{{.*}}
-; CHECK: %[[#]] = OpExtInst %[[#VINT32]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI32]] 2
+; CHECK-DAG: %[[#CASTorPARAMofPTRI32:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT32]]{{.*}}
+; CHECK-DAG: %[[#CALL3]] = OpExtInst %[[#VINT32]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI32]] 2
   %call3 = call spir_func <2 x i32> @_Z6vload2mPU3AS1Ki(i64 %offset, ptr addrspace(1) %address)
-; CHECK: %[[#CASTorPARAMofPTRI64:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT64]]{{.*}}
-; CHECK: %[[#]] = OpExtInst %[[#VINT64]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI64]] 2
+; CHECK-DAG: %[[#CASTorPARAMofPTRI64:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRINT64]]{{.*}}
+; CHECK-DAG: %[[#CALL4]] = OpExtInst %[[#VINT64]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRI64]] 2
   %call4 = call spir_func <2 x i64> @_Z6vload2mPU3AS1Kl(i64 %offset, ptr addrspace(1) %address)
-; CHECK: %[[#CASTorPARAMofPTRFLOAT:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRFLOAT]]{{.*}}
-; CHECK: %[[#]] = OpExtInst %[[#VFLOAT]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRFLOAT]] 2
+; CHECK-DAG: %[[#CASTorPARAMofPTRFLOAT:]] = {{OpBitcast|OpFunctionParameter}}{{.*}}%[[#PTRFLOAT]]{{.*}}
+; CHECK-DAG: %[[#CALL5]] = OpExtInst %[[#VFLOAT]] %[[#IMPORT]] vloadn %[[#OFFSET]] %[[#CASTorPARAMofPTRFLOAT]] 2
   %call5 = call spir_func <2 x float> @_Z6vload2mPU3AS1Kf(i64 %offset, ptr addrspace(1) %address)
   ret void
 }
diff --git a/llvm/test/CodeGen/SPIRV/phi-insert-point.ll b/llvm/test/CodeGen/SPIRV/phi-insert-point.ll
index 41cc514f4dbae..70d121cdf4b3a 100644
--- a/llvm/test/CodeGen/SPIRV/phi-insert-point.ll
+++ b/llvm/test/CodeGen/SPIRV/phi-insert-point.ll
@@ -2,7 +2,7 @@
 ; operand are inserted at the correct positions, and don't break rules of
 ; instruction domination and PHI nodes grouping at top of basic block.
 
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpName %[[#Foo:]] "foo"
diff --git a/llvm/test/CodeGen/SPIRV/phi-ptrcast-dominate.ll b/llvm/test/CodeGen/SPIRV/phi-ptrcast-dominate.ll
index 102eb76356fe3..bc090ce55fbec 100644
--- a/llvm/test/CodeGen/SPIRV/phi-ptrcast-dominate.ll
+++ b/llvm/test/CodeGen/SPIRV/phi-ptrcast-dominate.ll
@@ -3,7 +3,7 @@
 ; positions, and don't break rules of instruction domination and PHI nodes
 ; grouping at top of basic block.
 
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 
diff --git a/llvm/test/CodeGen/SPIRV/phi-spvintrinsic-dominate.ll b/llvm/test/CodeGen/SPIRV/phi-spvintrinsic-dominate.ll
index 471ab03ed89f6..45831ce97263d 100644
--- a/llvm/test/CodeGen/SPIRV/phi-spvintrinsic-dominate.ll
+++ b/llvm/test/CodeGen/SPIRV/phi-spvintrinsic-dominate.ll
@@ -3,7 +3,7 @@
 ; positions, and don't break rules of instruction domination and PHI nodes
 ; grouping at top of basic block.
 
-; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpFunction
diff --git a/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll b/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
index cd1a1b0080c62..55d638f80cc55 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
@@ -15,7 +15,7 @@
 ; CHECK-DAG: %[[GenPtrChar:.*]] = OpTypePointer Generic %[[Char]]
 ; CHECK-DAG: %[[CWPtrChar:.*]] = OpTypePointer CrossWorkgroup %[[Char]]
 ; CHECK-DAG: %[[Arr1:.*]] = OpTypeArray %[[CWPtrChar]] %[[#]]
-; CHECK-DAG: %[[Struct1:.*]] = OpTypeStruct %8
+; CHECK-DAG: %[[Struct1:.*]] = OpTypeStruct %[[Arr1]]
 ; CHECK-DAG: %[[Arr2:.*]] = OpTypeArray %[[GenPtrChar]] %[[#]]
 ; CHECK-DAG: %[[Struct2:.*]] = OpTypeStruct %[[Arr2]]
 ; CHECK-DAG: %[[GenPtr:.*]] = OpTypePointer Generic %[[Int]]
diff --git a/llvm/test/CodeGen/SPIRV/pointers/builtin-ret-reg-type.ll b/llvm/test/CodeGen/SPIRV/pointers/builtin-ret-reg-type.ll
new file mode 100644
index 0000000000000..a846e1936d7ac
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/builtin-ret-reg-type.ll
@@ -0,0 +1,50 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers which were
+; used as return values in call lowering. Pass criterion is that spirv-val considers output valid.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_half = type { half }
+%t_i17 = type { [17 x i32] }
+%t_h17 = type { [17 x %t_half] }
+
+define internal spir_func void @foo(i64 %arrayinit.cur.add_4, half %r1, ptr addrspace(4) noundef align 8 dereferenceable_or_null(72) %this) {
+entry:
+  %r_3 = alloca %t_h17, align 8
+  %p_src = alloca %t_i17, align 4
+  %p_src4 = addrspacecast ptr %p_src to ptr addrspace(4)
+  %call_2 = call spir_func noundef ptr @_Z42__spirv_GenericCastToPtrExplicit_ToPrivatePvi(ptr addrspace(4) noundef %p_src4, i32 noundef 7)
+  br label %l_body
+
+l_body:                                           ; preds = %l_body, %entry
+  %l_done = icmp eq i64 %arrayinit.cur.add_4, 34
+  br i1 %l_done, label %exit, label %l_body
+
+exit:                                             ; preds = %l_body
+  %0 = addrspacecast ptr %call_2 to ptr addrspace(4)
+  %call_6 = call spir_func noundef ptr @_Z42__spirv_GenericCastToPtrExplicit_ToPrivatePvi(ptr addrspace(4) noundef %0, i32 noundef 7)
+  br label %for.cond_3
+
+for.cond_3:                                       ; preds = %for.body_3, %exit
+  %lsr.iv1 = phi ptr [ %scevgep2, %for.body_3 ], [ %call_6, %exit ]
+  %lsr.iv = phi ptr [ %scevgep, %for.body_3 ], [ %r_3, %exit ]
+  %i.0_3 = phi i64 [ 0, %exit ], [ %inc_3, %for.body_3 ]
+  %cmp_3 = icmp ult i64 %i.0_3, 17
+  br i1 %cmp_3, label %for.body_3, label %exit2
+
+for.body_3:                                       ; preds = %for.cond_3
+  %call2_5 = call spir_func noundef half @_Z17__spirv_ocl_frexpDF16_PU3AS0i(half noundef %r1, ptr noundef %lsr.iv1)
+  store half %call2_5, ptr %lsr.iv, align 2
+  %inc_3 = add nuw nsw i64 %i.0_3, 1
+  %scevgep = getelementptr i8, ptr %lsr.iv, i64 2
+  %scevgep2 = getelementptr i8, ptr %lsr.iv1, i64 4
+  br label %for.cond_3
+
+exit2:                                            ; preds = %for.cond_3
+  ret void
+}
+
+declare dso_local spir_func noundef ptr @_Z42__spirv_GenericCastToPtrExplicit_ToPrivatePvi(ptr addrspace(4) noundef, i32 noundef)
+declare dso_local spir_func noundef half @_Z17__spirv_ocl_frexpDF16_PU3AS0i(half noundef, ptr noundef)
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+declare void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
diff --git a/llvm/test/CodeGen/SPIRV/pointers/gep-types-1.ll b/llvm/test/CodeGen/SPIRV/pointers/gep-types-1.ll
new file mode 100644
index 0000000000000..0e2730e18bf38
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/gep-types-1.ll
@@ -0,0 +1,42 @@
+; The goal of the test is to ensure that type inference doesn't break validity of the generated SPIR-V code.
+; The only pass criterion is that spirv-val considers output valid.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Char:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#PtrChar:]] = OpTypePointer Function %[[#Char]]
+; CHECK-DAG: %[[#PtrCharCW:]] = OpTypePointer CrossWorkgroup %[[#Char]]
+; CHECK-DAG: %[[#PtrCharGen:]] = OpTypePointer Generic %[[#Char]]
+; CHECK-DAG: %[[#Struct:]] = OpTypeStruct %[[#]] %[[#]] %[[#]]
+; CHECK-DAG: %[[#PtrInt:]] = OpTypePointer Function %[[#Int]]
+; CHECK-DAG: %[[#PtrPtrCharGen:]] = OpTypePointer Function %[[#PtrCharGen]]
+; CHECK-DAG: %[[#PtrStruct:]] = OpTypePointer Function %[[#Struct]]
+; CHECK: OpFunction
+; CHECK: %[[#Arg1:]] = OpFunctionParameter %[[#Int]]
+; CHECK: %[[#Arg2:]] = OpFunctionParameter %[[#PtrCharCW]]
+; CHECK: %[[#Kernel:]] = OpVariable %[[#PtrStruct]] Function
+; CHECK: %[[#IntKernel:]] = OpBitcast %[[#PtrInt]] %[[#Kernel]]
+; CHECK: OpStore %[[#IntKernel]] %[[#Arg1]]
+; CHECK: %[[#CharKernel:]] = OpBitcast %[[#PtrChar]] %[[#Kernel]]
+; CHECK: %[[#P:]] = OpInBoundsPtrAccessChain %[[#PtrChar]] %[[#CharKernel]] %[[#]]
+; CHECK: %[[#R0:]] = OpPtrCastToGeneric %[[#PtrCharGen]] %[[#Arg2]]
+; CHECK: %[[#P2:]] = OpBitcast %[[#PtrPtrCharGen]] %[[#P]]
+; CHECK: OpStore %[[#P2]] %[[#R0]]
+; CHECK: %[[#P3:]] = OpBitcast %[[#PtrPtrCharGen]] %[[#P]]
+; CHECK: %[[#]] = OpLoad %[[#PtrCharGen]] %[[#P3]]
+
+%"class.std::complex" = type { { double, double } }
+%class.anon = type { i32, ptr addrspace(4), [2 x [2 x %"class.std::complex"]] }
+
+define weak_odr dso_local spir_kernel void @foo(i32 noundef %_arg_N, ptr addrspace(1) noundef align 8 %_arg_p) {
+entry:
+  %Kernel = alloca %class.anon, align 8
+  store i32 %_arg_N, ptr %Kernel, align 8
+  %p = getelementptr inbounds i8, ptr %Kernel, i64 8
+  %r0 = addrspacecast ptr addrspace(1) %_arg_p to ptr addrspace(4)
+  store ptr addrspace(4) %r0, ptr %p, align 8
+  %r3 = load ptr addrspace(4), ptr %p, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/gep-types-2.ll b/llvm/test/CodeGen/SPIRV/pointers/gep-types-2.ll
new file mode 100644
index 0000000000000..d94da31890ab1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/gep-types-2.ll
@@ -0,0 +1,36 @@
+; The goal of the test is to ensure that type inference doesn't break validity of the generated SPIR-V code.
+; The only pass criterion is that spirv-val considers output valid.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpFunction
+
+%class.anon = type { i32, ptr addrspace(4)}
+
+define weak_odr dso_local spir_kernel void @foo(i32 noundef %_arg_N, i1 %fl) {
+entry:
+  %__SYCLKernel = alloca %class.anon, align 8
+  store i32 %_arg_N, ptr %__SYCLKernel, align 8
+  br label %arinit
+
+arinit:
+  %scevgep3 = getelementptr nuw i8, ptr %__SYCLKernel, i64 24
+  br label %for.cond.i
+
+for.cond.i:
+  %lsr.iv4 = phi ptr [ %scevgep5, %for.body.i ], [ %scevgep3, %arinit ]
+  br i1 %fl, label %for.body.i, label %exit
+
+for.body.i:
+  %scevgep6 = getelementptr i8, ptr %lsr.iv4, i64 -8
+  %_M_value.imag.i.i = load double, ptr %lsr.iv4, align 8
+  %scevgep5 = getelementptr i8, ptr %lsr.iv4, i64 32
+  br label %for.cond.i
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/global-addrspacecast.ll b/llvm/test/CodeGen/SPIRV/pointers/global-addrspacecast.ll
new file mode 100644
index 0000000000000..544c657da8488
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/global-addrspacecast.ll
@@ -0,0 +1,17 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+@PrivInternal = internal addrspace(10) global i32 456
+; CHECK-DAG:  %[[#type:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#ptrty:]] = OpTypePointer Private %[[#type]]
+; CHECK-DAG: %[[#value:]] = OpConstant %[[#type]] 456
+; CHECK-DAG:   %[[#var:]] = OpVariable %[[#ptrty]] Private %[[#value]]
+
+define spir_kernel void @Foo() {
+  %p = addrspacecast ptr addrspace(10) @PrivInternal to ptr
+  %v = load i32, ptr %p, align 4
+  ret void
+; CHECK:      OpLabel
+; CHECK-NEXT: OpLoad %[[#type]] %[[#var]] Aligned 4
+; CHECK-Next: OpReturn
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll b/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
index 7982893a0a913..16c20f9067e6e 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
@@ -11,7 +11,7 @@
 ; CHECK-DAG: %[[TyStruct:.*]] = OpTypeStruct %[[TyI64]] %[[TyI64]]
 ; CHECK-DAG: %[[Const128:.*]] = OpConstant %[[TyI64]] 128
 ; CHECK-DAG: %[[GlobalValue]] = OpVariable
-; CHECK-DAG: %[[PtrToInt:.*]] = OpSpecConstantOp %[[TyI64]] 117 %12
+; CHECK-DAG: %[[PtrToInt:.*]] = OpSpecConstantOp %[[TyI64]] 117 %[[GlobalValue]]
 ; TODO: The following bitcast line looks unneeded and we may expect it to be removed in future
 ; CHECK-DAG: %[[UseGlobalValue:.*]] = OpSpecConstantOp %[[TyI64]] 124 %[[PtrToInt]]
 ; CHECK-DAG: %[[ConstComposite:.*]] = OpConstantComposite %[[TyStruct]] %[[Const128]] %[[UseGlobalValue]]
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-chain-types.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-chain-types.ll
new file mode 100644
index 0000000000000..a9e79df259c4f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-chain-types.ll
@@ -0,0 +1,82 @@
+; The goal of the test case is to ensure that correct types are applied to PHI's as arguments of other PHI's.
+; Pass criterion is that spirv-val considers output valid.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+
+; CHECK-DAG: OpName %[[#Foo:]] "foo"
+; CHECK-DAG: OpName %[[#FooVal1:]] "val1"
+; CHECK-DAG: OpName %[[#FooVal2:]] "val2"
+; CHECK-DAG: OpName %[[#FooVal3:]] "val3"
+; CHECK-DAG: OpName %[[#Bar:]] "bar"
+; CHECK-DAG: OpName %[[#BarVal1:]] "val1"
+; CHECK-DAG: OpName %[[#BarVal2:]] "val2"
+; CHECK-DAG: OpName %[[#BarVal3:]] "val3"
+
+; CHECK-DAG: %[[#Short:]] = OpTypeInt 16 0
+; CHECK-DAG: %[[#ShortGenPtr:]] = OpTypePointer Generic %[[#Short]]
+; CHECK-DAG: %[[#ShortWrkPtr:]] = OpTypePointer Workgroup %[[#Short]]
+; CHECK-DAG: %[[#G1:]] = OpVariable %[[#ShortWrkPtr]] Workgroup
+
+; CHECK: %[[#Foo:]] = OpFunction %[[#]] None %[[#]]
+; CHECK: %[[#FooArgP:]] = OpFunctionParameter %[[#ShortGenPtr]]
+; CHECK: OpFunctionParameter
+; CHECK: OpFunctionParameter
+; CHECK: OpFunctionParameter
+; CHECK: %[[#FooG1:]] = OpPtrCastToGeneric %[[#ShortGenPtr]] %[[#G1]]
+; CHECK: %[[#FooVal2]] = OpPhi %[[#ShortGenPtr]] %[[#FooArgP]] %[[#]] %[[#FooVal3]] %[[#]]
+; CHECK: %[[#FooVal1]] = OpPhi %[[#ShortGenPtr]] %[[#FooG1]] %[[#]] %[[#FooVal2]] %[[#]]
+; CHECK: %[[#FooVal3]] = OpLoad %[[#ShortGenPtr]] %[[#]]
+
+; CHECK: %[[#Bar:]] = OpFunction %[[#]] None %[[#]]
+; CHECK: %[[#BarArgP:]] = OpFunctionParameter %[[#ShortGenPtr]]
+; CHECK: OpFunctionParameter
+; CHECK: OpFunctionParameter
+; CHECK: OpFunctionParameter
+; CHECK: %[[#BarVal3]] = OpLoad %[[#ShortGenPtr]] %[[#]]
+; CHECK: %[[#BarG1:]] = OpPtrCastToGeneric %[[#ShortGenPtr]] %[[#G1]]
+; CHECK: %[[#BarVal1]] = OpPhi %[[#ShortGenPtr]] %[[#BarG1]] %[[#]] %[[#BarVal2]] %[[#]]
+; CHECK: %[[#BarVal2]] = OpPhi %[[#ShortGenPtr]] %[[#BarArgP]] %[[#]] %[[#BarVal3]] %[[#]]
+
+@G1 = internal addrspace(3) global i16 undef, align 8
+@G2 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+
+define spir_kernel void @foo(ptr addrspace(4) %p, i1 %f1, i1 %f2, i1 %f3) {
+entry:
+  br label %l1
+
+l1:
+  br i1 %f1, label %l2, label %exit
+
+l2:
+  %val2 = phi ptr addrspace(4) [ %p, %l1 ], [ %val3, %l3 ]
+  %val1 = phi ptr addrspace(4) [ addrspacecast (ptr addrspace(3) @G1 to ptr addrspace(4)), %l1 ], [ %val2, %l3 ]
+  br i1 %f2, label %l3, label %exit
+
+l3:
+  %val3 = load ptr addrspace(4), ptr addrspace(3) @G2, align 8
+  br i1 %f3, label %l2, label %exit
+
+exit:
+  ret void
+}
+
+define spir_kernel void @bar(ptr addrspace(4) %p, i1 %f1, i1 %f2, i1 %f3) {
+entry:
+  %val3 = load ptr addrspace(4), ptr addrspace(3) @G2, align 8
+  br label %l1
+
+l3:
+  br i1 %f3, label %l2, label %exit
+
+l1:
+  br i1 %f1, label %l2, label %exit
+
+l2:
+  %val1 = phi ptr addrspace(4) [ addrspacecast (ptr addrspace(3) @G1 to ptr addrspace(4)), %l1 ], [ %val2, %l3 ]
+  %val2 = phi ptr addrspace(4) [ %p, %l1 ], [ %val3, %l3 ]
+  br i1 %f2, label %l3, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
index 07824d4ed6cd8..f4c8c5a79bcb7 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types.ll
@@ -1,15 +1,14 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK: %[[#Char:]] = OpTypeInt 8 0
-; CHECK: %[[#PtrChar:]] = OpTypePointer Function %[[#Char]]
-; CHECK: %[[#Int:]] = OpTypeInt 32 0
-; CHECK: %[[#PtrInt:]] = OpTypePointer Function %[[#Int]]
+; CHECK-DAG: %[[#Char:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#PtrChar:]] = OpTypePointer Function %[[#Char]]
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#PtrInt:]] = OpTypePointer Function %[[#Int]]
 ; CHECK: %[[#R1:]] = OpFunctionCall %[[#PtrChar]] %[[#]]
 ; CHECK: %[[#R2:]] = OpFunctionCall %[[#PtrInt]] %[[#]]
 ; CHECK: %[[#Casted:]] = OpBitcast %[[#PtrChar]] %[[#R2]]
 ; CHECK: OpPhi %[[#PtrChar]] %[[#R1]] %[[#]] %[[#Casted]] %[[#]]
-; CHECK: OpPhi %[[#PtrChar]] %[[#R1]] %[[#]] %[[#Casted]] %[[#]]
 
 define ptr @foo(i1 %arg) {
 entry:
diff --git a/llvm/test/CodeGen/SPIRV/pointers/tangled-ret.ll b/llvm/test/CodeGen/SPIRV/pointers/tangled-ret.ll
new file mode 100644
index 0000000000000..985893029db89
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/tangled-ret.ll
@@ -0,0 +1,235 @@
+; The only pass criterion is that spirv-val considers output valid.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%subgr = type { i64, i64 }
+%t_range = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_arr2 = type { [4 x i32] }
+
+define internal spir_func noundef i32 @geti32() {
+entry:
+  ret i32 100
+}
+
+define internal spir_func noundef i64 @geti64() {
+entry:
+  ret i64 200
+}
+
+define internal spir_func void @enable_if(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %this, i64 noundef %dim0) {
+entry:
+  %this.addr = alloca ptr addrspace(4), align 8
+  %dim0.addr = alloca i64, align 8
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  store i64 %dim0, ptr %dim0.addr, align 8
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %0 = load i64, ptr %dim0.addr, align 8
+  call spir_func void @enable_if_2(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %this1, i64 noundef %0)
+  ret void
+}
+
+
+define internal spir_func void @test(ptr addrspace(4) noundef align 8 dereferenceable_or_null(16) %this, ptr addrspace(4) noundef align 4 dereferenceable(16) %bits, ptr noundef byval(%t_range) align 8 %pos) {
+entry:
+  %this.addr = alloca ptr addrspace(4), align 8
+  %bits.addr = alloca ptr addrspace(4), align 8
+  %cur_pos = alloca i64, align 8
+  %__range4 = alloca ptr addrspace(4), align 8
+  %__begin0 = alloca ptr addrspace(4), align 8
+  %__end0 = alloca ptr addrspace(4), align 8
+  %cleanup.dest.slot = alloca i32, align 4
+  %elem = alloca ptr addrspace(4), align 8
+  %agg.tmp = alloca %t_range, align 8
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  store ptr addrspace(4) %bits, ptr %bits.addr, align 8
+  %pos.ascast = addrspacecast ptr %pos to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %call = call spir_func noundef i64 @getp(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %pos.ascast, i32 noundef 0)
+  store i64 %call, ptr %cur_pos, align 8
+  %0 = load ptr addrspace(4), ptr %bits.addr, align 8
+  store ptr addrspace(4) %0, ptr %__range4, align 8
+  %1 = load ptr addrspace(4), ptr %__range4, align 8
+  %call2 = call spir_func noundef ptr addrspace(4) @beginp(ptr addrspace(4) noundef align 4 dereferenceable_or_null(16) %1)
+  store ptr addrspace(4) %call2, ptr %__begin0, align 8
+  %2 = load ptr addrspace(4), ptr %__range4, align 8
+  %call3 = call spir_func noundef ptr addrspace(4) @endp(ptr addrspace(4) noundef align 4 dereferenceable_or_null(16) %2)
+  store ptr addrspace(4) %call3, ptr %__end0, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %3 = load ptr addrspace(4), ptr %__begin0, align 8
+  %4 = load ptr addrspace(4), ptr %__end0, align 8
+  %cmp = icmp ne ptr addrspace(4) %3, %4
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %5 = load ptr addrspace(4), ptr %__begin0, align 8
+  store ptr addrspace(4) %5, ptr %elem, align 8
+  %6 = load i64, ptr %cur_pos, align 8
+  %call4 = call spir_func noundef i32 @maskp(ptr addrspace(4) noundef align 8 dereferenceable_or_null(16) %this1)
+  %conv = zext i32 %call4 to i64
+  %cmp5 = icmp ult i64 %6, %conv
+  br i1 %cmp5, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %7 = load ptr addrspace(4), ptr %elem, align 8
+  %8 = load i64, ptr %cur_pos, align 8
+  call spir_func void @enable_if(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %agg.tmp.ascast, i64 noundef %8)
+  call spir_func void @extract_bits(ptr addrspace(4) noundef align 8 dereferenceable_or_null(16) %this1, ptr addrspace(4) noundef align 4 dereferenceable(4) %7, ptr noundef byval(%t_range) align 8 %agg.tmp)
+  %9 = load i64, ptr %cur_pos, align 8
+  %add = add i64 %9, 32
+  store i64 %add, ptr %cur_pos, align 8
+  br label %if.end
+
+if.else:                                          ; preds = %for.body
+  %10 = load ptr addrspace(4), ptr %elem, align 8
+  store i32 0, ptr addrspace(4) %10, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %11 = load ptr addrspace(4), ptr %__begin0, align 8
+  %incdec.ptr = getelementptr inbounds nuw i32, ptr addrspace(4) %11, i32 1
+  store ptr addrspace(4) %incdec.ptr, ptr %__begin0, align 8
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+define internal spir_func noundef i64 @getp(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %this, i32 noundef %dimension) {
+entry:
+  %this.addr.i = alloca ptr addrspace(4), align 8
+  %dimension.addr.i = alloca i32, align 4
+  %retval = alloca i64, align 8
+  %this.addr = alloca ptr addrspace(4), align 8
+  %dimension.addr = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  store i32 %dimension, ptr %dimension.addr, align 4
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %0 = load i32, ptr %dimension.addr, align 4
+  store ptr addrspace(4) %this1, ptr %this.addr.i, align 8
+  store i32 %0, ptr %dimension.addr.i, align 4
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i, align 8
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load i32, ptr %dimension.addr, align 4
+  %idxprom = sext i32 %1 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array1, i64 0, i64 %idxprom
+  %2 = load i64, ptr addrspace(4) %arrayidx, align 8
+  ret i64 %2
+}
+
+define internal spir_func noundef ptr addrspace(4) @beginp(ptr addrspace(4) noundef align 4 dereferenceable_or_null(16) %this) {
+entry:
+  %retval = alloca ptr addrspace(4), align 8
+  %this.addr = alloca ptr addrspace(4), align 8
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %MData1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %arraydecay2 = bitcast ptr addrspace(4) %MData1 to ptr addrspace(4)
+  ret ptr addrspace(4) %arraydecay2
+}
+
+define internal spir_func noundef ptr addrspace(4) @endp(ptr addrspace(4) noundef align 4 dereferenceable_or_null(16) %this) {
+entry:
+  %retval = alloca ptr addrspace(4), align 8
+  %this.addr = alloca ptr addrspace(4), align 8
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %MData1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %arraydecay2 = bitcast ptr addrspace(4) %MData1 to ptr addrspace(4)
+  %add.ptr = getelementptr inbounds nuw i32, ptr addrspace(4) %arraydecay2, i64 4
+  ret ptr addrspace(4) %add.ptr
+}
+
+define internal spir_func noundef i32 @maskp(ptr addrspace(4) noundef align 8 dereferenceable_or_null(16) %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4), align 8
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %bits_num = getelementptr inbounds nuw %subgr, ptr addrspace(4) %this1, i32 0, i32 1
+  %0 = load i64, ptr addrspace(4) %bits_num, align 8
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+define internal spir_func void @enable_if_2(ptr addrspace(4) noundef align 8 dereferenceable_or_null(8) %this, i64 noundef %dim0) {
+entry:
+  %this.addr = alloca ptr addrspace(4), align 8
+  %dim0.addr = alloca i64, align 8
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  store i64 %dim0, ptr %dim0.addr, align 8
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %dim0.addr, align 8
+  store i64 %0, ptr addrspace(4) %common_array1, align 8
+  ret void
+}
+
+define internal spir_func void @extract_bits(ptr addrspace(4) noundef align 8 dereferenceable_or_null(16) %this, ptr addrspace(4) noundef align 4 dereferenceable(4) %bits, ptr noundef byval(%t_range) align 8 %pos) {
+entry:
+  %this.addr = alloca ptr addrspace(4), align 8
+  %bits.addr = alloca ptr addrspace(4), align 8
+  %Res = alloca i64, align 8
+  store ptr addrspace(4) %this, ptr %this.addr, align 8
+  store ptr addrspace(4) %bits, ptr %bits.addr, align 8
+  %pos.ascast = addrspacecast ptr %pos to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr, align 8
+  %Bits1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr addrspace(4) %Bits1, align 8
+  store i64 %0, ptr %Res, align 8
+  %bits_num = getelementptr inbounds nuw %subgr, ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load i64, ptr addrspace(4) %bits_num, align 8
+  %call = call spir_func noundef i64 @geti64()
+  %2 = load i64, ptr %Res, align 8
+  %and = and i64 %2, %call
+  store i64 %and, ptr %Res, align 8
+  %call2 = call spir_func noundef i64 @geti64()
+  %call3 = call spir_func noundef i32 @geti32()
+  %conv = zext i32 %call3 to i64
+  %cmp = icmp ult i64 %call2, %conv
+  br i1 %cmp, label %if.then, label %if.else
+
+if.else:                                          ; preds = %entry
+  %3 = load ptr addrspace(4), ptr %bits.addr, align 8
+  store i32 0, ptr addrspace(4) %3, align 4
+  br label %if.end11
+
+if.then:                                          ; preds = %entry
+  %call4 = call spir_func noundef i64 @geti64()
+  %cmp5 = icmp ugt i64 %call4, 0
+  br i1 %cmp5, label %if.then6, label %if.end
+
+if.then6:                                         ; preds = %if.then
+  %call7 = call spir_func noundef i64 @geti64()
+  %4 = load i64, ptr %Res, align 8
+  %shr = lshr i64 %4, %call7
+  store i64 %shr, ptr %Res, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then6, %if.then
+  %call8 = call spir_func noundef i64 @geti64()
+  %5 = load i64, ptr %Res, align 8
+  %and9 = and i64 %5, %call8
+  store i64 %and9, ptr %Res, align 8
+  %6 = load i64, ptr %Res, align 8
+  %conv10 = trunc i64 %6 to i32
+  %7 = load ptr addrspace(4), ptr %bits.addr, align 8
+  store i32 %conv10, ptr addrspace(4) %7, align 4
+  br label %if.end11
+
+if.end11:                                         ; preds = %if.else, %if.end
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-via-store-load-args-rev.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-via-store-load-args-rev.ll
new file mode 100644
index 0000000000000..b0047ba82c36d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-via-store-load-args-rev.ll
@@ -0,0 +1,64 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - --translator-compatibility-mode | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[#Bar:]] "bar"
+; CHECK-DAG: OpName %[[#Foo:]] "foo"
+; CHECK-DAG: OpName %[[#Test:]] "test"
+; CHECK-DAG: %[[#Void:]] = OpTypeVoid
+; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#LongArr:]] = OpTypeArray %[[#Long]] %[[#]]
+; CHECK-DAG: %[[#StructLongArr:]] = OpTypeStruct %[[#LongArr]]
+; CHECK-DAG: %[[#Struct:]] = OpTypeStruct %[[#StructLongArr]]
+; CHECK-DAG: %[[#StructGenPtr:]] = OpTypePointer Generic %[[#Struct]]
+; CHECK-DAG: %[[#StructFunPtr:]] = OpTypePointer Function %[[#Struct]]
+; CHECK-DAG: %[[#StructGenGenPtr:]] = OpTypePointer Generic %[[#StructGenPtr]]
+; CHECK-DAG: %[[#StructFunGenPtr:]] = OpTypePointer Function %[[#StructGenPtr]]
+
+; CHECK: %[[#Bar]] = OpFunction
+; CHECK: %[[#BarVar:]] = OpVariable %[[#StructFunPtr]] Function
+; CHECK: %[[#BarVarToGen:]] = OpPtrCastToGeneric %[[#StructGenPtr]] %[[#BarVar]]
+; CHECK: %[[#]] = OpFunctionCall %[[#Void]] %[[#Foo]] %[[#BarVarToGen]]
+
+; CHECK: %[[#Foo]] = OpFunction
+; CHECK: %[[#FooArg1:]] = OpFunctionParameter %[[#StructGenPtr]]
+; CHECK: %[[#FooVar:]] = OpVariable %[[#StructFunGenPtr]] Function
+; CHECK: %[[#FooVarToGen:]] = OpPtrCastToGeneric %[[#StructGenGenPtr]] %[[#FooVar]]
+; CHECK: OpStore %[[#FooVarToGen]] %[[#FooArg1]]
+; CHECK: %[[#FooLoad:]] = OpLoad %[[#StructGenPtr]] %[[#FooVarToGen]]
+; CHECK: %[[#]] = OpFunctionCall %[[#Void:]] %[[#Test]] %[[#FooLoad:]]
+
+; CHECK: %[[#Test]] = OpFunction
+; CHECK: %[[#TestArg1:]] = OpFunctionParameter %[[#StructGenPtr]]
+; CHECK: %[[#TestVar:]] = OpVariable %[[#StructFunGenPtr]] Function
+; CHECK: %[[#TestVarToGen:]] = OpPtrCastToGeneric %[[#StructGenGenPtr]] %[[#TestVar]]
+; CHECK: OpStore %[[#TestVarToGen]] %[[#TestArg1]]
+
+%t_range = type { %t_arr }
+%t_arr = type { [1 x i64] }
+
+define internal spir_func void @bar() {
+  %GlobalOffset = alloca %t_range, align 8
+  %GlobalOffset.ascast = addrspacecast ptr %GlobalOffset to ptr addrspace(4)
+  call spir_func void @foo(ptr addrspace(4) noundef align 8 dereferenceable(8) %GlobalOffset.ascast)
+  ret void
+}
+
+define internal spir_func void @foo(ptr addrspace(4) noundef align 8 dereferenceable(8) %Offset) {
+entry:
+  %Offset.addr = alloca ptr addrspace(4), align 8
+  %Offset.addr.ascast = addrspacecast ptr %Offset.addr to ptr addrspace(4)
+  store ptr addrspace(4) %Offset, ptr addrspace(4) %Offset.addr.ascast, align 8
+  %r2 = load ptr addrspace(4), ptr addrspace(4) %Offset.addr.ascast, align 8
+  call spir_func void @test(ptr addrspace(4) noundef align 8 dereferenceable(8) %r2)
+  ret void
+}
+
+define void @test(ptr addrspace(4) noundef align 8 dereferenceable(8) %offset) {
+  %offset.addr = alloca ptr addrspace(4), align 8
+  %offset.addr.ascast = addrspacecast ptr %offset.addr to ptr addrspace(4)
+  store ptr addrspace(4) %offset, ptr addrspace(4) %offset.addr.ascast, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class-vk.ll b/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class-vk.ll
new file mode 100644
index 0000000000000..e8b1dc263f150
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class-vk.ll
@@ -0,0 +1,15 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+
+; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 456
+; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer Private %[[#U32]]
+; CHECK-DAG: %[[#VAR:]] = OpVariable %[[#VTYPE]] Private %[[#VAL]]
+; CHECK-NOT: OpDecorate %[[#VAR]] LinkageAttributes
+@PrivInternal = internal addrspace(10) global i32 456
+
+define void @main() {
+  %l = load i32, ptr addrspace(10) @PrivInternal
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class.ll b/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class.ll
index 2d4c805ac9df1..a1ded0569d67e 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/variables-storage-class.ll
@@ -1,17 +1,29 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
+; CHECK-DAG: %[[#U8:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+
+; CHECK-DAG: %[[#TYPE:]] = OpTypePointer CrossWorkgroup %[[#U8]]
+; CHECK-DAG: %[[#VAL:]] = OpConstantNull %[[#TYPE]]
+; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer CrossWorkgroup %[[#TYPE]]
+; CHECK-DAG: %[[#PTR:]] = OpVariable %[[#VTYPE]] CrossWorkgroup %[[#VAL]]
 @Ptr = addrspace(1) global ptr addrspace(1) null
-@Init = private addrspace(2) constant i32 123
 
-; CHECK-DAG: %[[#PTR:]] = OpVariable %[[#]] UniformConstant %[[#]]
-; CHECK-DAG: %[[#INIT:]] = OpVariable %[[#]] CrossWorkgroup %[[#]]
+; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 123
+; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer UniformConstant %[[#U32]]
+; CHECK-DAG: %[[#INIT:]] = OpVariable %[[#VTYPE]] UniformConstant %[[#VAL]]
+@Init = private addrspace(2) constant i32 123
 
-; CHECK: %[[#]] = OpLoad %[[#]] %[[#INIT]] Aligned 8
-; CHECK: OpCopyMemorySized %[[#]] %[[#PTR]] %[[#]] Aligned 4
+; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 456
+; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer Private %[[#U32]]
+; CHECK-DAG: %[[#]] = OpVariable %[[#VTYPE]] Private %[[#VAL]]
+@PrivInternal = internal addrspace(10) global i32 456
 
 define spir_kernel void @Foo() {
+  ; CHECK: %[[#]] = OpLoad %[[#]] %[[#PTR]] Aligned 8
   %l = load ptr addrspace(1), ptr addrspace(1) @Ptr, align 8
+  ; CHECK: OpCopyMemorySized %[[#]] %[[#INIT]] %[[#]] Aligned 4
   call void @llvm.memcpy.p1.p2.i64(ptr addrspace(1) align 4 %l, ptr addrspace(2) align 1 @Init, i64 4, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
index 5585e4a07590a..75fe865c9be9d 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-if.ll
@@ -14,18 +14,18 @@ entry:
   %1 = alloca i32, align 4
   br i1 true, label %left, label %right
 
-; CHECK: %[[#left]] = OpLabel
-; CHECK:              OpBranch %[[#merge]]
-left:
-  store i32 0, ptr %1
-  br label %end
-
 ; CHECK: %[[#right]] = OpLabel
 ; CHECK:               OpBranch %[[#merge]]
 right:
   store i32 0, ptr %1
   br label %end
 
+; CHECK: %[[#left]] = OpLabel
+; CHECK:              OpBranch %[[#merge]]
+left:
+  store i32 0, ptr %1
+  br label %end
+
 ; CHECK: %[[#merge]] = OpLabel
 ; CHECK:               OpReturnValue %[[#]]
 end:
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
index ded9c335c5a25..29091534bf1d2 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-loop.ll
@@ -21,6 +21,11 @@ header:
   %2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
   br i1 true, label %body, label %merge
 
+; CHECK: %[[#merge]] = OpLabel
+; CHECK:               OpReturnValue %[[#]]
+merge:
+  ret i32 0
+
 ; CHECK: %[[#body]] = OpLabel
 ; CHECK:              OpBranch %[[#continue]]
 body:
@@ -31,11 +36,6 @@ continue:
   br label %header
 ; CHECK: %[[#continue]] = OpLabel
 ; CHECK:                  OpBranch %[[#header]]
-
-; CHECK: %[[#merge]] = OpLabel
-; CHECK:               OpReturnValue %[[#]]
-merge:
-  ret i32 0
 }
 
 ; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
index a43d25e9b06d5..8ad1758a14c84 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll
@@ -18,18 +18,18 @@ entry:
   %0 = call token @llvm.experimental.convergence.entry()
   br i1 true, label %left, label %right
 
-; CHECK:      %[[#left]] = OpLabel
-; CHECK-NEXT:              OpStore %[[#var]] %[[#int_0]]
-; CHECK-NEXT:              OpBranch %[[#merge]]
-left:
-  br label %end
-
 ; CHECK:      %[[#right]] = OpLabel
 ; CHECK-NEXT:               OpStore %[[#var]] %[[#int_1]]
 ; CHECK-NEXT:               OpBranch %[[#merge]]
 right:
   br label %end
 
+; CHECK:      %[[#left]] = OpLabel
+; CHECK-NEXT:              OpStore %[[#var]] %[[#int_0]]
+; CHECK-NEXT:              OpBranch %[[#merge]]
+left:
+  br label %end
+
 ; CHECK: %[[#merge]] = OpLabel
 ; CHECK:  %[[#tmp:]] = OpLoad %[[#]] %[[#var]]
 ; CHECK:               OpReturnValue %[[#tmp]]
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
index 86033608deb6e..e4cb1ddc594b0 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll
@@ -55,18 +55,18 @@ entry:
   %var = alloca i32
   br i1 true, label %a_true, label %a_false
 
-; CHECK: %[[#a_true]] = OpLabel
-; CHECK:                OpStore %[[#r2m_a]] %[[#true]]
-; CHECK:                OpBranch %[[#a_merge]]
-a_true:
-  br label %a_merge
-
 ; CHECK: %[[#a_false]] = OpLabel
 ; CHECK:                 OpStore %[[#r2m_a]] %[[#false]]
 ; CHECK:                 OpBranch %[[#a_merge]]
 a_false:
   br label %a_merge
 
+; CHECK: %[[#a_true]] = OpLabel
+; CHECK:                OpStore %[[#r2m_a]] %[[#true]]
+; CHECK:                OpBranch %[[#a_merge]]
+a_true:
+  br label %a_merge
+
 ; CHECK: %[[#a_merge]] = OpLabel
 ; CHECK:    %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_a]]
 ; CHECK:                 OpSelectionMerge %[[#b_merge:]]
@@ -89,6 +89,14 @@ b_merge:
   %f1 = call spir_func noundef i32 @_Z2fnv() #4 [ "convergencectrl"(token %0) ]
   br i1 true, label %c_true, label %c_false
 
+; CHECK: %[[#c_false]] = OpLabel
+; CHECK:        %[[#]] = OpFunctionCall
+; CHECK:                 OpStore %[[#r2m_b]] %[[#]]
+; CHECK:                 OpBranch %[[#c_merge]]
+c_false:
+  %f3 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ]
+  br label %c_merge
+
 ; CHECK: %[[#c_true]] = OpLabel
 ; CHECK:       %[[#]] = OpFunctionCall
 ; CHECK:                OpStore %[[#r2m_b]] %[[#]]
@@ -97,13 +105,6 @@ c_true:
   %f2 = call spir_func noundef i32 @_Z3fn1v() #4 [ "convergencectrl"(token %0) ]
   br label %c_merge
 
-; CHECK: %[[#c_false]] = OpLabel
-; CHECK:        %[[#]] = OpFunctionCall
-; CHECK:                 OpStore %[[#r2m_b]] %[[#]]
-; CHECK:                 OpBranch %[[#c_merge]]
-c_false:
-  %f3 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ]
-  br label %c_merge
 
 ; CHECK: %[[#c_merge]] = OpLabel
 ; CHECK:    %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_b]]
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
index 12e6473c15c75..2c12e09e223c7 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.continue.ll
@@ -50,43 +50,49 @@
 ; CHECK: %[[#func_12:]] = OpFunction %[[#uint:]] DontInline %[[#]]
 ; CHECK:    %[[#bb44:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb45:]] = OpLabel
+; CHECK:     %[[#bb45]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb46:]] %[[#bb47:]] None
 ; CHECK:                  OpBranchConditional %[[#]] %[[#bb48:]] %[[#bb46:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb49:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb50:]]
-; CHECK:    %[[#bb50:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb49:]]
-; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb47:]]
-; CHECK:    %[[#bb47:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb46:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb51:]]
-; CHECK:    %[[#bb51:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb52:]] %[[#bb53:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb54:]] %[[#bb52:]]
-; CHECK:    %[[#bb54:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb55:]]
-; CHECK:    %[[#bb55:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb56:]] %[[#bb57:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb58:]] %[[#bb56:]]
-; CHECK:    %[[#bb58:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb57:]]
-; CHECK:    %[[#bb57:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb55:]]
-; CHECK:    %[[#bb56:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb53:]]
-; CHECK:    %[[#bb53:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb51:]]
-; CHECK:    %[[#bb52:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
+  ; CHECK:     %[[#bb46]] = OpLabel
+  ; CHECK:                  OpBranch %[[#bb51:]]
+  ; CHECK:     %[[#bb51]] = OpLabel
+  ; CHECK:                  OpLoopMerge %[[#bb52:]] %[[#bb53:]] None
+  ; CHECK:                  OpBranchConditional %[[#]] %[[#bb54:]] %[[#bb52]]
+    ; CHECK:     %[[#bb52]] = OpLabel
+    ; CHECK:                  OpReturnValue %[[#]]
+
+    ; CHECK:     %[[#bb54]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb55:]]
+    ; CHECK:     %[[#bb55]] = OpLabel
+    ; CHECK:                  OpLoopMerge %[[#bb56:]] %[[#bb57:]] None
+    ; CHECK:                  OpBranchConditional %[[#]] %[[#bb58:]] %[[#bb56]]
+      ; CHECK:     %[[#bb56]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb53]]
+      ; CHECK:     %[[#bb53]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb51]]
+
+      ; CHECK:     %[[#bb58]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb57]]
+      ; CHECK:     %[[#bb57]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb55]]
+  ; CHECK:     %[[#bb48]] = OpLabel
+  ; CHECK:                  OpSelectionMerge %[[#bb49:]] None
+  ; CHECK:                  OpBranchConditional %[[#]] %[[#bb49]] %[[#bb50:]]
+    ; CHECK:     %[[#bb50]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb49]]
+
+    ; CHECK:     %[[#bb49]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb47]]
+
+; CHECK:     %[[#bb47]] = OpLabel
+; CHECK:                  OpBranch %[[#bb45]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_40:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb59:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_42:]] = OpFunction %[[#void:]] None %[[#]]
 ; CHECK:    %[[#bb60:]] = OpLabel
 ; CHECK:                  OpReturn
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
index ddd4c5222301a..35a36b468ea52 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.nested.ll
@@ -28,38 +28,43 @@
 ; CHECK: %[[#func_11:]] = OpFunction %[[#uint:]] DontInline %[[#]]
 ; CHECK:    %[[#bb41:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb42:]]
-; CHECK:    %[[#bb42:]] = OpLabel
+; CHECK:     %[[#bb42]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb43:]] %[[#bb44:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb45:]] %[[#bb43:]]
-; CHECK:    %[[#bb45:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb46:]]
-; CHECK:    %[[#bb46:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb47:]] %[[#bb48:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb47:]]
-; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb50:]]
-; CHECK:    %[[#bb50:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb51:]] %[[#bb52:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb53:]] %[[#bb51:]]
-; CHECK:    %[[#bb53:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb52:]]
-; CHECK:    %[[#bb52:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb50:]]
-; CHECK:    %[[#bb51:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb48:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb46:]]
-; CHECK:    %[[#bb47:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb44:]]
-; CHECK:    %[[#bb44:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb42:]]
-; CHECK:    %[[#bb43:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb45:]] %[[#bb43]]
+  ; CHECK:     %[[#bb43]] = OpLabel
+  ; CHECK:                  OpReturnValue %[[#]]
+
+  ; CHECK:     %[[#bb45]] = OpLabel
+  ; CHECK:                  OpBranch %[[#bb46:]]
+  ; CHECK:     %[[#bb46]] = OpLabel
+  ; CHECK:                  OpLoopMerge %[[#bb47:]] %[[#bb48:]] None
+  ; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb47]]
+    ; CHECK:     %[[#bb47]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb44]]
+    ; CHECK:     %[[#bb44]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb42]]
+
+    ; CHECK:     %[[#bb49]] = OpLabel
+    ; CHECK:                  OpBranch %[[#bb50:]]
+    ; CHECK:     %[[#bb50]] = OpLabel
+    ; CHECK:                  OpLoopMerge %[[#bb51:]] %[[#bb52:]] None
+    ; CHECK:                  OpBranchConditional %[[#]] %[[#bb53:]] %[[#bb51]]
+      ; CHECK:     %[[#bb51]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb48]]
+      ; CHECK:     %[[#bb48]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb46]]
+
+      ; CHECK:     %[[#bb53]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb52]]
+      ; CHECK:     %[[#bb52]] = OpLabel
+      ; CHECK:                  OpBranch %[[#bb50]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_37:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb54:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_39:]] = OpFunction %[[#void:]] None %[[#]]
 ; CHECK:    %[[#bb55:]] = OpLabel
 ; CHECK:                  OpReturn
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
index 3e898c4f11b7e..b7167e01e9224 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll
@@ -37,28 +37,13 @@ for.cond:                                         ; preds = %for.inc, %entry
   ; CHECK: OpBranchConditional %[[#]] %[[#for_body:]] %[[#for_end]]
   br i1 %cmp, label %for.body, label %for.end
 
-for.body:                                         ; preds = %for.cond
-  %3 = load i32, ptr %i, align 4
-  store i32 %3, ptr %val, align 4
-  br label %for.inc
-  ; CHECK: %[[#for_body]] = OpLabel
-  ; CHECK:                 OpBranch %[[#for_inc]]
-
-for.inc:                                          ; preds = %for.body
-  %4 = load i32, ptr %i, align 4
-  %inc = add nsw i32 %4, 1
-  store i32 %inc, ptr %i, align 4
-  br label %for.cond
-  ; CHECK: %[[#for_inc]] = OpLabel
-  ; CHECK:                 OpBranch %[[#for_cond]]
-
 for.end:                                          ; preds = %for.cond
   br label %for.cond1
   ; CHECK: %[[#for_end]] = OpLabel
   ; CHECK:                 OpBranch %[[#for_cond1:]]
 
 for.cond1:                                        ; preds = %for.cond1, %for.end
-  %5 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+  %3 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
   store i32 0, ptr %val, align 4
   br label %for.cond1
   ; CHECK: %[[#for_cond1]] = OpLabel
@@ -67,6 +52,23 @@ for.cond1:                                        ; preds = %for.cond1, %for.end
 
   ; CHECK: %[[#unreachable]] = OpLabel
   ; CHECK-NEXT:                OpUnreachable
+
+for.body:                                         ; preds = %for.cond
+  %4 = load i32, ptr %i, align 4
+  store i32 %4, ptr %val, align 4
+  br label %for.inc
+  ; CHECK: %[[#for_body]] = OpLabel
+  ; CHECK:                 OpBranch %[[#for_inc]]
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+  ; CHECK: %[[#for_inc]] = OpLabel
+  ; CHECK:                 OpBranch %[[#for_cond]]
+
+
 }
 
 ; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
index 07c20ebadd159..a5db8f53d2b35 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.short-circuited-cond.ll
@@ -42,115 +42,146 @@
 ;   process();
 ; }
 
-; CHECK: %[[#func_14:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb87:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb88:]]
-; CHECK:    %[[#bb88:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb89:]] %[[#bb90:]] None
-; CHECK:                  OpBranch %[[#bb91:]]
-; CHECK:    %[[#bb91:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb92:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb93:]] %[[#bb92:]]
-; CHECK:    %[[#bb93:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb92:]]
-; CHECK:    %[[#bb92:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb94:]] %[[#bb89:]]
-; CHECK:    %[[#bb94:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb90:]]
-; CHECK:    %[[#bb89:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb95:]]
-; CHECK:    %[[#bb90:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb88:]]
-; CHECK:    %[[#bb95:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb96:]] %[[#bb97:]] None
-; CHECK:                  OpBranch %[[#bb98:]]
-; CHECK:    %[[#bb98:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb99:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb99:]] %[[#bb100:]]
-; CHECK:   %[[#bb100:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb99:]]
-; CHECK:    %[[#bb99:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb101:]] %[[#bb96:]]
-; CHECK:   %[[#bb101:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb97:]]
-; CHECK:    %[[#bb96:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb102:]]
-; CHECK:    %[[#bb97:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb95:]]
-; CHECK:   %[[#bb102:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb103:]] %[[#bb104:]] None
-; CHECK:                  OpBranch %[[#bb105:]]
-; CHECK:   %[[#bb105:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb106:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb107:]] %[[#bb106:]]
-; CHECK:   %[[#bb107:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb108:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb109:]] %[[#bb110:]]
-; CHECK:   %[[#bb109:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb111:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb111:]] %[[#bb112:]]
-; CHECK:   %[[#bb110:]] = OpLabel
-; CHECK:   %[[#bb112:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb111:]] %[[#bb113:]]
-; CHECK:   %[[#bb113:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb111:]]
-; CHECK:   %[[#bb111:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb114:]] %[[#bb108:]]
-; CHECK:   %[[#bb114:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb108:]]
-; CHECK:   %[[#bb108:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb106:]]
-; CHECK:   %[[#bb106:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb115:]] %[[#bb103:]]
-; CHECK:   %[[#bb115:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb104:]]
-; CHECK:   %[[#bb103:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb116:]]
-; CHECK:   %[[#bb104:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb102:]]
-; CHECK:   %[[#bb116:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb117:]] %[[#bb118:]] None
-; CHECK:                  OpBranch %[[#bb119:]]
-; CHECK:   %[[#bb119:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb120:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb121:]] %[[#bb122:]]
-; CHECK:   %[[#bb121:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb120:]]
-; CHECK:   %[[#bb122:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb120:]]
-; CHECK:   %[[#bb120:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb123:]] %[[#bb117:]]
-; CHECK:   %[[#bb123:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb118:]]
-; CHECK:   %[[#bb117:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb124:]]
-; CHECK:   %[[#bb118:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb116:]]
-; CHECK:   %[[#bb124:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb125:]] %[[#bb126:]] None
-; CHECK:                  OpBranch %[[#bb127:]]
-; CHECK:   %[[#bb127:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb128:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb129:]] %[[#bb128:]]
-; CHECK:   %[[#bb129:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb128:]]
-; CHECK:   %[[#bb128:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb130:]] %[[#bb125:]]
-; CHECK:   %[[#bb130:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb126:]]
-; CHECK:   %[[#bb126:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb124:]]
-; CHECK:   %[[#bb125:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
-; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_83:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:   %[[#bb131:]] = OpLabel
-; CHECK:                  OpReturn
-; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_85:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:   %[[#bb132:]] = OpLabel
-; CHECK:                  OpReturn
-; CHECK:                  OpFunctionEnd
+; CHECK:     %[[#process:]] = OpFunction %[[#uint:]] DontInline %[[#]]
+; CHECK:        %[[#bb86:]] = OpLabel
+; CHECK:                      OpBranch %[[#bb87:]]
+  ; CHECK:         %[[#bb87]] = OpLabel
+  ; CHECK:                      OpLoopMerge %[[#bb89:]] %[[#bb90:]] None
+  ; CHECK:                      OpBranch %[[#bb91:]]
+  ; CHECK:         %[[#bb91]] = OpLabel
+  ; CHECK:                      OpSelectionMerge %[[#bb92:]] None
+  ; CHECK:                      OpBranchConditional %[[#]] %[[#bb93:]] %[[#bb94:]]
+    ; CHECK:         %[[#bb94]] = OpLabel
+    ; CHECK:                      OpBranch %[[#bb92]]
+
+    ; CHECK:         %[[#bb93]] = OpLabel
+    ; CHECK:                      OpBranch %[[#bb92]]
+    ; CHECK:         %[[#bb92]] = OpLabel
+    ; CHECK:                      OpBranchConditional %[[#]] %[[#bb96:]] %[[#bb89]]
+      ; CHECK:         %[[#bb89]] = OpLabel
+      ; CHECK:                      OpBranch %[[#bb97:]]
+      ; CHECK:         %[[#bb97]] = OpLabel
+      ; CHECK:                      OpLoopMerge %[[#bb99:]] %[[#bb100:]] None
+      ; CHECK:                      OpBranch %[[#bb101:]]
+      ; CHECK:        %[[#bb101]] = OpLabel
+      ; CHECK:                      OpSelectionMerge %[[#bb102:]] None
+      ; CHECK:                      OpBranchConditional %[[#]] %[[#bb103:]] %[[#bb104:]]
+        ; CHECK:        %[[#bb104]] = OpLabel
+        ; CHECK:                      OpBranch %[[#bb102]]
+
+        ; CHECK:        %[[#bb103]] = OpLabel
+        ; CHECK:                      OpBranch %[[#bb102]]
+        ; CHECK:        %[[#bb102]] = OpLabel
+        ; CHECK:                      OpBranchConditional %[[#]] %[[#bb106:]] %[[#bb99]]
+          ; CHECK:         %[[#bb99]] = OpLabel
+          ; CHECK:                      OpBranch %[[#bb107:]]
+          ; CHECK:        %[[#bb107]] = OpLabel
+          ; CHECK:                      OpLoopMerge %[[#bb109:]] %[[#bb110:]] None
+          ; CHECK:                      OpBranch %[[#bb111:]]
+          ; CHECK:        %[[#bb111]] = OpLabel
+          ; CHECK:                      OpSelectionMerge %[[#bb112:]] None
+          ; CHECK:                      OpBranchConditional %[[#]] %[[#bb113:]] %[[#bb114:]]
+            ; CHECK:        %[[#bb114]] = OpLabel
+            ; CHECK:                      OpBranch %[[#bb112]]
+
+            ; CHECK:        %[[#bb113]] = OpLabel
+            ; CHECK:                      OpSelectionMerge %[[#bb116:]] None
+            ; CHECK:                      OpBranchConditional %[[#]] %[[#bb117:]] %[[#bb116]]
+              ; CHECK:        %[[#bb117]] = OpLabel
+              ; CHECK:                      OpSelectionMerge %[[#bb118:]] None
+              ; CHECK:                      OpBranchConditional %[[#]] %[[#bb119:]] %[[#bb120:]]
+                ; CHECK:        %[[#bb120]] = OpLabel
+                ; CHECK:                      OpSelectionMerge %[[#bb122:]] None
+                ; CHECK:                      OpBranchConditional %[[#]] %[[#bb122]] %[[#bb123:]]
+                  ; CHECK:        %[[#bb123]] = OpLabel
+                  ; CHECK:                      OpBranch %[[#bb122]]
+                  ; CHECK:        %[[#bb122]] = OpLabel
+                  ; CHECK:                      OpBranchConditional %[[#]] %[[#bb126:]] %[[#bb118]]
+                    ; CHECK:        %[[#bb126]] = OpLabel
+                    ; CHECK:                      OpBranch %[[#bb118]]
+
+                ; CHECK:        %[[#bb119]] = OpLabel
+                ; CHECK:                      OpBranch %[[#bb118]]
+                ; CHECK:        %[[#bb118]] = OpLabel
+                ; CHECK:                      OpBranchConditional %[[#]] %[[#bb116]] %[[#bb129:]]
+                  ; CHECK:        %[[#bb129]] = OpLabel
+                  ; CHECK:                      OpBranch %[[#bb116]]
+                  ; CHECK:        %[[#bb116]] = OpLabel
+                  ; CHECK:                      OpBranchConditional %[[#]] %[[#bb112]] %[[#bb133:]]
+                    ; CHECK:        %[[#bb133]] = OpLabel
+                    ; CHECK:                      OpBranch %[[#bb112]]
+                    ; CHECK:        %[[#bb112]] = OpLabel
+                    ; CHECK:                      OpSelectionMerge %[[#bb136:]] None
+                    ; CHECK:                      OpBranchConditional %[[#]] %[[#bb136]] %[[#bb137:]]
+                    ; CHECK:        %[[#bb137]] = OpLabel
+                    ; CHECK-NEXT:                 OpUnreachable
+
+                    ; CHECK:        %[[#bb136]] = OpLabel
+                    ; CHECK:                      OpBranchConditional %[[#]] %[[#bb138:]] %[[#bb109]]
+                      ; CHECK:        %[[#bb109]] = OpLabel
+                      ; CHECK:                      OpBranch %[[#bb139:]]
+                      ; CHECK:        %[[#bb139]] = OpLabel
+                      ; CHECK:                      OpLoopMerge %[[#bb141:]] %[[#bb142:]] None
+                      ; CHECK:                      OpBranch %[[#bb143:]]
+                      ; CHECK:        %[[#bb143]] = OpLabel
+                      ; CHECK:                      OpSelectionMerge %[[#bb144:]] None
+                      ; CHECK:                      OpBranchConditional %[[#]] %[[#bb145:]] %[[#bb146:]]
+                        ; CHECK:        %[[#bb146]] = OpLabel
+                        ; CHECK:                      OpBranch %[[#bb144]]
+
+                        ; CHECK:        %[[#bb145]] = OpLabel
+                        ; CHECK:                      OpBranch %[[#bb144]]
+                      ; CHECK:        %[[#bb144]] = OpLabel
+                      ; CHECK:                      OpBranchConditional %[[#]] %[[#bb149:]] %[[#bb141]]
+                        ; CHECK:        %[[#bb141]] = OpLabel
+                        ; CHECK:                      OpBranch %[[#bb150:]]
+                        ; CHECK:        %[[#bb150]] = OpLabel
+                        ; CHECK:                      OpLoopMerge %[[#bb153:]] %[[#bb154:]] None
+                        ; CHECK:                      OpBranch %[[#bb155:]]
+                        ; CHECK:        %[[#bb155]] = OpLabel
+                        ; CHECK:                      OpSelectionMerge %[[#bb156:]] None
+                        ; CHECK:                      OpBranchConditional %[[#]] %[[#bb157:]] %[[#bb158:]]
+                          ; CHECK:        %[[#bb158]] = OpLabel
+                          ; CHECK:                      OpBranch %[[#bb156]]
+
+                          ; CHECK:        %[[#bb157]] = OpLabel
+                          ; CHECK:                      OpBranch %[[#bb156]]
+                        ; CHECK:        %[[#bb156]] = OpLabel
+                        ; CHECK:                      OpBranchConditional %[[#]] %[[#bb160:]] %[[#bb153]]
+                          ; CHECK:        %[[#bb153]] = OpLabel
+                          ; CHECK:                      OpReturnValue %[[#]]
+
+                          ; CHECK:        %[[#bb160]] = OpLabel
+                          ; CHECK:                      OpBranch %[[#bb154]]
+                          ; CHECK:        %[[#bb154]] = OpLabel
+                          ; CHECK:                      OpBranch %[[#bb150]]
+                        ; CHECK:        %[[#bb149]] = OpLabel
+                        ; CHECK:                      OpBranch %[[#bb142]]
+                        ; CHECK:        %[[#bb142]] = OpLabel
+                        ; CHECK:                      OpBranch %[[#bb139]]
+                      ; CHECK:        %[[#bb138]] = OpLabel
+                      ; CHECK:                      OpBranch %[[#bb110]]
+                      ; CHECK:        %[[#bb110]] = OpLabel
+                      ; CHECK:                      OpBranch %[[#bb107]]
+          ; CHECK:        %[[#bb106]] = OpLabel
+          ; CHECK:                      OpBranch %[[#bb100]]
+          ; CHECK:        %[[#bb100]] = OpLabel
+          ; CHECK:                      OpBranch %[[#bb97]]
+      ; CHECK:         %[[#bb96]] = OpLabel
+      ; CHECK:                      OpBranch %[[#bb90]]
+      ; CHECK:         %[[#bb90]] = OpLabel
+      ; CHECK:                      OpBranch %[[#bb87]]
+; CHECK:                      OpFunctionEnd
+
+; CHECK:   %[[#func_main:]] = OpFunction %[[#void:]] DontInline %[[#]]
+; CHECK:       %[[#bb172:]] = OpLabel
+; CHECK:                      OpReturn
+; CHECK:                      OpFunctionEnd
+
+; CHECK: %[[#func_main_1:]] = OpFunction %[[#void]] None %[[#]]
+; CHECK:       %[[#bb173:]] = OpLabel
+; CHECK:                      OpReturn
+; CHECK:                      OpFunctionEnd
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
index 93effc141fc81..1a241d69154dd 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.for.ll
@@ -49,72 +49,73 @@
 ; CHECK: %[[#func_18:]] = OpFunction %[[#uint:]] DontInline %[[#]]
 ; CHECK:    %[[#bb65:]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb66:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb67:]] %[[#bb66:]]
-; CHECK:    %[[#bb67:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb66:]]
-; CHECK:    %[[#bb66:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb67:]] %[[#bb66]]
+; CHECK:     %[[#bb67]] = OpLabel
+; CHECK:                  OpBranch %[[#bb66]]
+; CHECK:     %[[#bb66]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb68:]]
-; CHECK:    %[[#bb68:]] = OpLabel
+; CHECK:     %[[#bb68]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb69:]] %[[#bb70:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb71:]] %[[#bb69:]]
-; CHECK:    %[[#bb71:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb71:]] %[[#bb69]]
+; CHECK:     %[[#bb69]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb72:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb73:]] %[[#bb72:]]
-; CHECK:    %[[#bb73:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb74:]]
-; CHECK:    %[[#bb74:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb75:]] %[[#bb76:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb77:]] %[[#bb75:]]
-; CHECK:    %[[#bb77:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb73:]] %[[#bb72]]
+; CHECK:     %[[#bb73]] = OpLabel
+; CHECK:                  OpBranch %[[#bb72]]
+; CHECK:     %[[#bb72]] = OpLabel
+; CHECK:                  OpReturnValue %[[#]]
+; CHECK:     %[[#bb71]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb74:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb75:]] %[[#bb74]]
+; CHECK:     %[[#bb75]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb76:]]
-; CHECK:    %[[#bb76:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb74:]]
-; CHECK:    %[[#bb75:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb72:]]
-; CHECK:    %[[#bb72:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb78:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb79:]] %[[#bb80:]]
-; CHECK:    %[[#bb79:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb78:]]
-; CHECK:    %[[#bb80:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb81:]]
-; CHECK:    %[[#bb81:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb82:]] %[[#bb83:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb84:]] %[[#bb82:]]
-; CHECK:    %[[#bb84:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb85:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb86:]] %[[#bb85:]]
-; CHECK:    %[[#bb86:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb85:]]
-; CHECK:    %[[#bb85:]] = OpLabel
+; CHECK:     %[[#bb76]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb77:]] %[[#bb78:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb79:]] %[[#bb77]]
+; CHECK:     %[[#bb77]] = OpLabel
+; CHECK:                  OpBranch %[[#bb74]]
+; CHECK:     %[[#bb74]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb80:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb81:]] %[[#bb82:]]
+; CHECK:     %[[#bb82]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb83:]]
-; CHECK:    %[[#bb83:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb81:]]
-; CHECK:    %[[#bb82:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb78:]]
-; CHECK:    %[[#bb78:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb70:]]
-; CHECK:    %[[#bb70:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb68:]]
-; CHECK:    %[[#bb69:]] = OpLabel
+; CHECK:     %[[#bb83]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb84:]] %[[#bb85:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb86:]] %[[#bb84]]
+; CHECK:     %[[#bb84]] = OpLabel
+; CHECK:                  OpBranch %[[#bb80]]
+; CHECK:     %[[#bb86]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb87:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb88:]] %[[#bb87:]]
-; CHECK:    %[[#bb88:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb87:]]
-; CHECK:    %[[#bb87:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb88:]] %[[#bb87]]
+; CHECK:     %[[#bb88]] = OpLabel
+; CHECK:                  OpBranch %[[#bb87]]
+; CHECK:     %[[#bb87]] = OpLabel
+; CHECK:                  OpBranch %[[#bb85]]
+; CHECK:     %[[#bb85]] = OpLabel
+; CHECK:                  OpBranch %[[#bb83]]
+; CHECK:     %[[#bb81]] = OpLabel
+; CHECK:                  OpBranch %[[#bb80]]
+; CHECK:     %[[#bb80]] = OpLabel
+; CHECK:                  OpBranch %[[#bb70]]
+; CHECK:     %[[#bb70]] = OpLabel
+; CHECK:                  OpBranch %[[#bb68]]
+; CHECK:     %[[#bb79]] = OpLabel
+; CHECK:                  OpBranch %[[#bb78]]
+; CHECK:     %[[#bb78]] = OpLabel
+; CHECK:                  OpBranch %[[#bb76]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_61:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb89:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_63:]] = OpFunction %[[#void:]] None %[[#]]
+
+; CHECK: %[[#func_63:]] = OpFunction %[[#void]] None %[[#]]
 ; CHECK:    %[[#bb90:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
 
 
-
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
index a69475a59db6f..64fc651e94bb5 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.if.nested.ll
@@ -34,35 +34,35 @@
 ; CHECK:    %[[#bb30:]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb31:]] None
 ; CHECK:                  OpBranchConditional %[[#]] %[[#bb32:]] %[[#bb33:]]
-; CHECK:    %[[#bb32:]] = OpLabel
+; CHECK:     %[[#bb33]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb34:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb35:]] %[[#bb34:]]
-; CHECK:    %[[#bb33:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb36:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb37:]] %[[#bb38:]]
-; CHECK:    %[[#bb35:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb34:]]
-; CHECK:    %[[#bb37:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb36:]]
-; CHECK:    %[[#bb38:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb35:]] %[[#bb36:]]
+; CHECK:     %[[#bb36]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb37:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb38:]] %[[#bb37]]
+; CHECK:     %[[#bb38]] = OpLabel
+; CHECK:                  OpBranch %[[#bb37]]
+; CHECK:     %[[#bb37]] = OpLabel
+; CHECK:                  OpBranch %[[#bb34]]
+; CHECK:     %[[#bb35]] = OpLabel
+; CHECK:                  OpBranch %[[#bb34]]
+; CHECK:     %[[#bb34]] = OpLabel
+; CHECK:                  OpBranch %[[#bb31]]
+; CHECK:     %[[#bb32]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb39:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb40:]] %[[#bb39:]]
-; CHECK:    %[[#bb34:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb31:]]
-; CHECK:    %[[#bb40:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb39:]]
-; CHECK:    %[[#bb39:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb36:]]
-; CHECK:    %[[#bb36:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb31:]]
-; CHECK:    %[[#bb31:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb40:]] %[[#bb39]]
+; CHECK:     %[[#bb40]] = OpLabel
+; CHECK:                  OpBranch %[[#bb39]]
+; CHECK:     %[[#bb39]] = OpLabel
+; CHECK:                  OpBranch %[[#bb31]]
+; CHECK:     %[[#bb31]] = OpLabel
 ; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
 ; CHECK: %[[#func_26:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb41:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_28:]] = OpFunction %[[#void:]] None %[[#]]
+; CHECK: %[[#func_28:]] = OpFunction %[[#void]] None %[[#]]
 ; CHECK:    %[[#bb42:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
index dfaca85be2280..1cb2d6d542a5b 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.return.early.ll
@@ -59,51 +59,53 @@
 ; }
 
 ; CHECK: %[[#func_13:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb43:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb44:]]
-; CHECK:    %[[#bb44:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb45:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb46:]] %[[#bb47:]]
 ; CHECK:    %[[#bb46:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb47:]]
+; CHECK:     %[[#bb47]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb48:]] None
-; CHECK:                  OpSwitch %[[#]] %[[#bb48:]] 1 %[[#bb49:]] 2 %[[#bb50:]] 5 %[[#bb51:]]
-; CHECK:    %[[#bb47:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb48:]]
-; CHECK:    %[[#bb50:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb48:]]
-; CHECK:    %[[#bb51:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb48:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb52:]] %[[#bb45:]]
-; CHECK:    %[[#bb52:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb53:]]
-; CHECK:    %[[#bb53:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb54:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb54:]] %[[#bb55:]]
-; CHECK:    %[[#bb55:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb54:]]
-; CHECK:    %[[#bb54:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb56:]] %[[#bb45:]]
-; CHECK:    %[[#bb56:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb50:]]
+; CHECK:     %[[#bb50]] = OpLabel
+; CHECK:                  OpBranch %[[#bb48]]
+; CHECK:     %[[#bb49]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb51:]] None
+; CHECK:                  OpSwitch %[[#]] %[[#bb51]] 1 %[[#bb52:]] 2 %[[#bb53:]] 5 %[[#bb54:]]
+; CHECK:     %[[#bb54]] = OpLabel
+; CHECK:                  OpBranch %[[#bb51]]
+; CHECK:     %[[#bb53]] = OpLabel
+; CHECK:                  OpBranch %[[#bb51]]
+; CHECK:     %[[#bb52]] = OpLabel
+; CHECK:                  OpBranch %[[#bb51]]
+; CHECK:     %[[#bb51]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb55:]] %[[#bb48]]
+; CHECK:     %[[#bb55]] = OpLabel
+; CHECK:                  OpBranch %[[#bb56:]]
+; CHECK:     %[[#bb56]] = OpLabel
 ; CHECK:                  OpSelectionMerge %[[#bb57:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb57:]] %[[#bb58:]]
-; CHECK:    %[[#bb58:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb57:]]
-; CHECK:    %[[#bb57:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb59:]] %[[#bb45:]]
-; CHECK:    %[[#bb59:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb45:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb57]] %[[#bb58:]]
+; CHECK:     %[[#bb58]] = OpLabel
+; CHECK:                  OpBranch %[[#bb57]]
+; CHECK:     %[[#bb57]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb59:]] %[[#bb48]]
+; CHECK:     %[[#bb59]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb60:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb60]] %[[#bb61:]]
+; CHECK:     %[[#bb61]] = OpLabel
+; CHECK:                  OpBranch %[[#bb60]]
+; CHECK:     %[[#bb60]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb62:]] %[[#bb48]]
+; CHECK:     %[[#bb62]] = OpLabel
+; CHECK:                  OpBranch %[[#bb48]]
+; CHECK:     %[[#bb48]] = OpLabel
 ; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_39:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:    %[[#bb60:]] = OpLabel
+
+; CHECK: %[[#func_42:]] = OpFunction %[[#void:]] DontInline %[[#]]
+; CHECK:    %[[#bb63:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_41:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:    %[[#bb61:]] = OpLabel
+
+; CHECK: %[[#func_44:]] = OpFunction %[[#void]] None %[[#]]
+; CHECK:    %[[#bb64:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
index cf50b982b23dc..58f0f1c6bf053 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.switch.ifstmt.simple2.ll
@@ -1,10 +1,9 @@
 ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
+; static int foo() { return 200; }
 ;
-; int foo() { return 200; }
-;
-; int process() {
+; static int process() {
 ;   int a = 0;
 ;   int b = 0;
 ;   int c = 0;
@@ -45,198 +44,167 @@
 ;   process();
 ; }
 
-; CHECK: %[[#func_18:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb52:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
-; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_19:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb53:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb54:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb55:]] %[[#bb56:]]
-; CHECK:    %[[#bb55:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb57:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb58:]] %[[#bb59:]]
-; CHECK:    %[[#bb56:]] = OpLabel
-; CHECK:    %[[#bb58:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb60:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb61:]] %[[#bb62:]]
-; CHECK:    %[[#bb59:]] = OpLabel
-; CHECK:    %[[#bb61:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb63:]] None
-; CHECK:                  OpSwitch %[[#]] %[[#bb64:]] 1 %[[#bb65:]] 2 %[[#bb63:]] 3 %[[#bb66:]] 140 %[[#bb67:]] 4 %[[#bb68:]] 5 %[[#bb69:]] 6 %[[#bb70:]] 7 %[[#bb71:]]
-; CHECK:    %[[#bb62:]] = OpLabel
-; CHECK:    %[[#bb64:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb65:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb66:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb67:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb68:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb69:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb70:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb71:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb63:]]
-; CHECK:    %[[#bb63:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb72:]] None
-; CHECK:                  OpSwitch %[[#]] %[[#bb73:]] 1 %[[#bb72:]] 2 %[[#bb74:]] 3 %[[#bb75:]]
-; CHECK:    %[[#bb73:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb72:]]
-; CHECK:    %[[#bb74:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb72:]]
-; CHECK:    %[[#bb75:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb72:]]
-; CHECK:    %[[#bb72:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb60:]]
-; CHECK:    %[[#bb60:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb76:]] None
-; CHECK:                  OpSwitch %[[#]] %[[#bb77:]] 1 %[[#bb76:]] 2 %[[#bb78:]]
-; CHECK:    %[[#bb77:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb76:]]
-; CHECK:    %[[#bb78:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb76:]]
-; CHECK:    %[[#bb76:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb57:]]
-; CHECK:    %[[#bb57:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb54:]]
-; CHECK:    %[[#bb54:]] = OpLabel
-; CHECK:                  OpReturnValue %[[#]]
-; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_48:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:    %[[#bb79:]] = OpLabel
-; CHECK:                  OpReturn
-; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_50:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:    %[[#bb80:]] = OpLabel
-; CHECK:                  OpReturn
-; CHECK:                  OpFunctionEnd
+; CHECK: %[[#func:]] = OpFunction %[[#]] DontInline %[[#]]
+; CHECK: %[[#bb30:]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb31:]] None
+; CHECK:               OpBranchConditional %[[#]] %[[#bb32:]] %[[#bb33:]]
+
+; CHECK:  %[[#bb33]] = OpLabel
+; CHECK:               OpUnreachable
+
+; CHECK:  %[[#bb32]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb34:]] None
+; CHECK:               OpBranchConditional %[[#]] %[[#bb35:]] %[[#bb36:]]
+
+; CHECK:  %[[#bb36]] = OpLabel
+; CHECK:               OpUnreachable
+
+; CHECK:  %[[#bb35]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb37:]] None
+; CHECK:               OpBranchConditional %[[#]] %[[#bb38:]] %[[#bb39:]]
+
+; CHECK:  %[[#bb39]] = OpLabel
+; CHECK:               OpUnreachable
+
+; CHECK:  %[[#bb38]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb40:]] None
+; CHECK:               OpSwitch %[[#]] %[[#bb41:]] 1 %[[#bb42:]] 2 %[[#bb43:]] 3 %[[#bb44:]] 140 %[[#bb45:]] 4 %[[#bb46:]] 5 %[[#bb47:]] 6 %[[#bb48:]] 7 %[[#bb49:]]
+
+; CHECK:  %[[#bb49]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb48]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb47]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb46]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb45]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb44]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb43]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb42]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+; CHECK:  %[[#bb41]] = OpLabel
+; CHECK:               OpBranch %[[#bb40]]
+
+; CHECK:  %[[#bb40]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb50:]] None
+; CHECK:               OpSwitch %[[#]] %[[#bb50]] 1 %[[#bb51:]] 2 %[[#bb52:]] 3 %[[#bb53:]]
+; CHECK:  %[[#bb53]] = OpLabel
+; CHECK:               OpBranch %[[#bb50]]
+; CHECK:  %[[#bb52]] = OpLabel
+; CHECK:               OpBranch %[[#bb50]]
+; CHECK:  %[[#bb51]] = OpLabel
+; CHECK:               OpBranch %[[#bb50]]
+; CHECK:  %[[#bb50]] = OpLabel
+; CHECK:               OpBranch %[[#bb37]]
+
+; CHECK:  %[[#bb37]] = OpLabel
+; CHECK:               OpSelectionMerge %[[#bb54:]] None
+; CHECK:               OpSwitch %[[#]] %[[#bb54]] 1 %[[#bb55:]] 2 %[[#bb56:]]
+; CHECK:  %[[#bb56]] = OpLabel
+; CHECK:               OpBranch %[[#bb54]]
+; CHECK:  %[[#bb55]] = OpLabel
+; CHECK:               OpBranch %[[#bb54]]
+; CHECK:  %[[#bb54]] = OpLabel
+; CHECK:               OpBranch %[[#bb34]]
+
+; CHECK:  %[[#bb34]] = OpLabel
+; CHECK:               OpBranch %[[#bb31]]
+
+; CHECK:  %[[#bb31]] = OpLabel
+; CHECK:               OpReturn
+; CHECK:               OpFunctionEnd
 
 
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
-; Function Attrs: convergent noinline norecurse nounwind optnone
-define spir_func noundef i32 @_Z3foov() #0 {
-entry:
-  %0 = call token @llvm.experimental.convergence.entry()
-  ret i32 200
-}
-
-; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
-declare token @llvm.experimental.convergence.entry() #1
-
-; Function Attrs: convergent noinline norecurse nounwind optnone
-define spir_func noundef i32 @_Z7processv() #0 {
+; Function Attrs: convergent noinline norecurse
+define void @main() #0 {
 entry:
+  %a.i = alloca i32, align 4
+  %b.i = alloca i32, align 4
+  %c.i = alloca i32, align 4
+  %r.i = alloca i32, align 4
+  %s.i = alloca i32, align 4
+  %t.i = alloca i32, align 4
+  %d.i = alloca i32, align 4
   %0 = call token @llvm.experimental.convergence.entry()
-  %a = alloca i32, align 4
-  %b = alloca i32, align 4
-  %c = alloca i32, align 4
-  %r = alloca i32, align 4
-  %s = alloca i32, align 4
-  %t = alloca i32, align 4
-  %d = alloca i32, align 4
-  store i32 0, ptr %a, align 4
-  store i32 0, ptr %b, align 4
-  store i32 0, ptr %c, align 4
-  store i32 20, ptr %r, align 4
-  store i32 40, ptr %s, align 4
-  store i32 140, ptr %t, align 4
-  store i32 5, ptr %d, align 4
-  %1 = load i32, ptr %d, align 4
-  switch i32 %1, label %sw.default [
-    i32 1, label %sw.bb
-    i32 2, label %sw.bb3
-    i32 3, label %sw.bb5
-    i32 140, label %sw.bb7
-    i32 4, label %sw.bb9
-    i32 5, label %sw.bb9
-    i32 6, label %sw.bb11
-    i32 7, label %sw.bb12
+  store i32 0, ptr %a.i, align 4
+  store i32 0, ptr %b.i, align 4
+  store i32 0, ptr %c.i, align 4
+  store i32 20, ptr %r.i, align 4
+  store i32 40, ptr %s.i, align 4
+  store i32 140, ptr %t.i, align 4
+  store i32 5, ptr %d.i, align 4
+  %1 = load i32, ptr %d.i, align 4
+  switch i32 %1, label %sw.default.i [
+    i32 1, label %sw.bb.i
+    i32 2, label %sw.bb3.i
+    i32 3, label %sw.bb5.i
+    i32 140, label %sw.bb7.i
+    i32 4, label %sw.bb9.i
+    i32 5, label %sw.bb9.i
+    i32 6, label %sw.bb11.i
+    i32 7, label %sw.bb12.i
   ]
 
-sw.bb:                                            ; preds = %entry
-  %2 = load i32, ptr %b, align 4
-  %add = add nsw i32 %2, 1
-  store i32 %add, ptr %b, align 4
-  %call1 = call spir_func noundef i32 @_Z3foov() #3 [ "convergencectrl"(token %0) ]
-  %3 = load i32, ptr %c, align 4
-  %add2 = add nsw i32 %3, %call1
-  store i32 %add2, ptr %c, align 4
-  br label %sw.bb3
-
-sw.bb3:                                           ; preds = %entry, %sw.bb
-  %4 = load i32, ptr %b, align 4
-  %add4 = add nsw i32 %4, 2
-  store i32 %add4, ptr %b, align 4
-  br label %sw.epilog
-
-sw.bb5:                                           ; preds = %entry
-  %5 = load i32, ptr %b, align 4
-  %add6 = add nsw i32 %5, 3
-  store i32 %add6, ptr %b, align 4
-  br label %sw.epilog
-
-sw.bb7:                                           ; preds = %entry
-  %6 = load i32, ptr %b, align 4
-  %add8 = add nsw i32 %6, 140
-  store i32 %add8, ptr %b, align 4
-  br label %sw.bb9
-
-sw.bb9:                                           ; preds = %entry, %entry, %sw.bb7
-  %7 = load i32, ptr %b, align 4
-  %add10 = add nsw i32 %7, 5
-  store i32 %add10, ptr %b, align 4
-  br label %sw.epilog
-
-sw.bb11:                                          ; preds = %entry
-  br label %sw.bb12
-
-sw.bb12:                                          ; preds = %entry, %sw.bb11
-  br label %sw.epilog
-
-sw.default:                                       ; preds = %entry
-  br label %sw.epilog
-
-sw.epilog:                                        ; preds = %sw.default, %sw.bb12, %sw.bb9, %sw.bb5, %sw.bb3
-  %8 = load i32, ptr %a, align 4
-  %9 = load i32, ptr %b, align 4
-  %add13 = add nsw i32 %8, %9
-  %10 = load i32, ptr %c, align 4
-  %add14 = add nsw i32 %add13, %10
-  ret i32 %add14
-}
-
-; Function Attrs: convergent noinline norecurse nounwind optnone
-define internal spir_func void @main() #0 {
-entry:
-  %0 = call token @llvm.experimental.convergence.entry()
-  %call1 = call spir_func noundef i32 @_Z7processv() #3 [ "convergencectrl"(token %0) ]
+sw.bb.i:
+  %2 = load i32, ptr %b.i, align 4
+  %add.i = add nsw i32 %2, 1
+  store i32 %add.i, ptr %b.i, align 4
+  %3 = load i32, ptr %c.i, align 4
+  %add2.i = add nsw i32 %3, 200
+  store i32 %add2.i, ptr %c.i, align 4
+  br label %sw.bb3.i
+
+sw.bb3.i:
+  %4 = load i32, ptr %b.i, align 4
+  %add4.i = add nsw i32 %4, 2
+  store i32 %add4.i, ptr %b.i, align 4
+  br label %_ZL7processv.exit
+
+sw.bb5.i:
+  %5 = load i32, ptr %b.i, align 4
+  %add6.i = add nsw i32 %5, 3
+  store i32 %add6.i, ptr %b.i, align 4
+  br label %_ZL7processv.exit
+
+sw.bb7.i:
+  %6 = load i32, ptr %b.i, align 4
+  %add8.i = add nsw i32 %6, 140
+  store i32 %add8.i, ptr %b.i, align 4
+  br label %sw.bb9.i
+
+sw.bb9.i:
+  %7 = load i32, ptr %b.i, align 4
+  %add10.i = add nsw i32 %7, 5
+  store i32 %add10.i, ptr %b.i, align 4
+  br label %_ZL7processv.exit
+
+sw.bb11.i:
+  br label %sw.bb12.i
+
+sw.bb12.i:
+  br label %_ZL7processv.exit
+
+sw.default.i:
+  br label %_ZL7processv.exit
+
+_ZL7processv.exit:
+  %8 = load i32, ptr %a.i, align 4
+  %9 = load i32, ptr %b.i, align 4
+  %add13.i = add nsw i32 %8, %9
+  %10 = load i32, ptr %c.i, align 4
+  %add14.i = add nsw i32 %add13.i, %10
   ret void
 }
 
-; Function Attrs: convergent norecurse
-define void @main.1() #2 {
-entry:
-  call void @main()
-  ret void
-}
+declare token @llvm.experimental.convergence.entry() #1
 
-attributes #0 = { convergent noinline norecurse nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
-attributes #2 = { convergent norecurse "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #3 = { convergent }
-
-!llvm.module.flags = !{!0, !1, !2}
-
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
-!2 = !{i32 7, !"frame-pointer", i32 2}
-
-
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.ll
index 6c53008386f49..e5545c62db583 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.continue.ll
@@ -45,54 +45,54 @@
 ; }
 
 ; CHECK: %[[#func_15:]] = OpFunction %[[#uint:]] DontInline %[[#]]
-; CHECK:    %[[#bb35:]] = OpLabel
+; CHECK:    %[[#bb36:]] = OpLabel
 ; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_16:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:    %[[#bb36:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb37:]]
 ; CHECK:    %[[#bb37:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb38:]] %[[#bb39:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb40:]] %[[#bb38:]]
-; CHECK:    %[[#bb40:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb41:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb41:]] %[[#bb42:]]
-; CHECK:    %[[#bb42:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb43:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb43:]] %[[#bb44:]]
-; CHECK:    %[[#bb44:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb43:]]
-; CHECK:    %[[#bb43:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb41:]]
-; CHECK:    %[[#bb41:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb39:]]
-; CHECK:    %[[#bb39:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb37:]]
-; CHECK:    %[[#bb38:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb45:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb46:]] %[[#bb47:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb46:]] %[[#bb48:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb49:]]
-; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb50:]] %[[#bb51:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb51:]] %[[#bb50:]]
-; CHECK:    %[[#bb51:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb49:]]
-; CHECK:    %[[#bb50:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb47:]]
-; CHECK:    %[[#bb47:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb45:]]
-; CHECK:    %[[#bb46:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb38:]]
+; CHECK:     %[[#bb38]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb39:]] %[[#bb40:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb41:]] %[[#bb39]]
+; CHECK:     %[[#bb39]] = OpLabel
+; CHECK:                  OpBranch %[[#bb42:]]
+; CHECK:     %[[#bb42]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb43:]] %[[#bb44:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb43]] %[[#bb45:]]
+; CHECK:     %[[#bb45]] = OpLabel
+; CHECK:                  OpBranch %[[#bb46:]]
+; CHECK:     %[[#bb46]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb47:]] %[[#bb48:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb48]] %[[#bb47]]
+; CHECK:     %[[#bb47]] = OpLabel
+; CHECK:                  OpBranch %[[#bb44]]
+; CHECK:     %[[#bb44]] = OpLabel
+; CHECK:                  OpBranch %[[#bb42]]
+; CHECK:     %[[#bb48]] = OpLabel
+; CHECK:                  OpBranch %[[#bb46]]
+; CHECK:     %[[#bb43]] = OpLabel
+; CHECK:     %[[#bb41]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb49:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb49]] %[[#bb50:]]
+; CHECK:     %[[#bb50]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb51:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb51]] %[[#bb52:]]
+; CHECK:     %[[#bb52]] = OpLabel
+; CHECK:                  OpBranch %[[#bb51]]
+; CHECK:     %[[#bb51]] = OpLabel
+; CHECK:                  OpBranch %[[#bb49]]
+; CHECK:     %[[#bb49]] = OpLabel
+; CHECK:                  OpBranch %[[#bb40]]
+; CHECK:     %[[#bb40]] = OpLabel
+; CHECK:                  OpBranch %[[#bb38]]
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_33:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:    %[[#bb52:]] = OpLabel
+
+; CHECK: %[[#func_34:]] = OpFunction %[[#void]] None %[[#]]
+; CHECK:    %[[#bb53:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
 
-
-
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.ll
index 716c48d391fe2..59410ec347f5b 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.nested.ll
@@ -25,29 +25,29 @@
 ; CHECK: %[[#func_12:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb39:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb40:]]
-; CHECK:    %[[#bb40:]] = OpLabel
+; CHECK:     %[[#bb40]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb41:]] %[[#bb42:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb43:]] %[[#bb41:]]
-; CHECK:    %[[#bb43:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb43:]] %[[#bb41]]
+; CHECK:     %[[#bb41]] = OpLabel
+; CHECK:                  OpReturn
+; CHECK:     %[[#bb43]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb44:]]
-; CHECK:    %[[#bb44:]] = OpLabel
+; CHECK:     %[[#bb44]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb45:]] %[[#bb46:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb47:]] %[[#bb45:]]
-; CHECK:    %[[#bb47:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb47:]] %[[#bb45]]
+; CHECK:     %[[#bb45]] = OpLabel
+; CHECK:                  OpBranch %[[#bb42]]
+; CHECK:     %[[#bb42]] = OpLabel
+; CHECK:                  OpBranch %[[#bb40]]
+; CHECK:     %[[#bb47]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb48:]] %[[#bb49:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb49:]] %[[#bb48:]]
-; CHECK:    %[[#bb49:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb47:]]
-; CHECK:    %[[#bb48:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb46:]]
-; CHECK:    %[[#bb46:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb44:]]
-; CHECK:    %[[#bb45:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb42:]]
-; CHECK:    %[[#bb42:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb40:]]
-; CHECK:    %[[#bb41:]] = OpLabel
-; CHECK:                  OpReturn
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb49]] %[[#bb48]]
+; CHECK:     %[[#bb48]] = OpLabel
+; CHECK:                  OpBranch %[[#bb46]]
+; CHECK:     %[[#bb46]] = OpLabel
+; CHECK:                  OpBranch %[[#bb44]]
+; CHECK:     %[[#bb49]] = OpLabel
+; CHECK:                  OpBranch %[[#bb47]]
 ; CHECK:                  OpFunctionEnd
 ; CHECK: %[[#func_37:]] = OpFunction %[[#void:]] None %[[#]]
 ; CHECK:    %[[#bb50:]] = OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.ll
index 2f1d228d8ffdf..e1aa00a3a174b 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.plain.ll
@@ -43,23 +43,26 @@
 ; CHECK:    %[[#bb20:]] = OpLabel
 ; CHECK:                  OpReturnValue %[[#]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_12:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb21:]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb22:]]
-; CHECK:    %[[#bb22:]] = OpLabel
+; CHECK:     %[[#bb22]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb23:]] %[[#bb24:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb24:]] %[[#bb23:]]
-; CHECK:    %[[#bb24:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb22:]]
-; CHECK:    %[[#bb23:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb24]] %[[#bb23]]
+; CHECK:     %[[#bb23]] = OpLabel
 ; CHECK:                  OpBranch %[[#bb25:]]
-; CHECK:    %[[#bb25:]] = OpLabel
+; CHECK:     %[[#bb25]] = OpLabel
 ; CHECK:                  OpLoopMerge %[[#bb26:]] %[[#bb27:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb26:]] %[[#bb27:]]
-; CHECK:    %[[#bb27:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb25:]]
-; CHECK:    %[[#bb26:]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb26]] %[[#bb27]]
+; CHECK:     %[[#bb27]] = OpLabel
+; CHECK:                  OpBranch %[[#bb25]]
+; CHECK:     %[[#bb26]] = OpLabel
+; CHECK-NEXT:             OpUnreachable
+; CHECK:     %[[#bb24]] = OpLabel
+; CHECK:                  OpBranch %[[#bb22]]
 ; CHECK:                  OpFunctionEnd
+
 ; CHECK: %[[#func_18:]] = OpFunction %[[#void:]] None %[[#]]
 ; CHECK:    %[[#bb28:]] = OpLabel
 ; CHECK:                  OpReturn
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.ll
index 79ce0c15e5c43..7f77b6da4ed46 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.while.short-circuited-cond.ll
@@ -21,104 +21,123 @@
 ;   }
 ; }
 
-; CHECK: %[[#func_10:]] = OpFunction %[[#void:]] DontInline %[[#]]
-; CHECK:    %[[#bb54:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb55:]]
-; CHECK:    %[[#bb55:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb56:]] %[[#bb57:]] None
-; CHECK:                  OpBranch %[[#bb58:]]
-; CHECK:    %[[#bb58:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb59:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb60:]] %[[#bb59:]]
-; CHECK:    %[[#bb60:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb59:]]
-; CHECK:    %[[#bb59:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb57:]] %[[#bb56:]]
-; CHECK:    %[[#bb57:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb55:]]
-; CHECK:    %[[#bb56:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb61:]]
-; CHECK:    %[[#bb61:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb62:]] %[[#bb63:]] None
-; CHECK:                  OpBranch %[[#bb64:]]
-; CHECK:    %[[#bb64:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb65:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb65:]] %[[#bb66:]]
-; CHECK:    %[[#bb66:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb65:]]
-; CHECK:    %[[#bb65:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb63:]] %[[#bb62:]]
-; CHECK:    %[[#bb63:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb61:]]
-; CHECK:    %[[#bb62:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb67:]]
-; CHECK:    %[[#bb67:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb68:]] %[[#bb69:]] None
-; CHECK:                  OpBranch %[[#bb70:]]
-; CHECK:    %[[#bb70:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb71:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb72:]] %[[#bb71:]]
-; CHECK:    %[[#bb72:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb73:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb74:]] %[[#bb75:]]
-; CHECK:    %[[#bb74:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb76:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb76:]] %[[#bb77:]]
-; CHECK:    %[[#bb75:]] = OpLabel
-; CHECK:    %[[#bb77:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb76:]] %[[#bb78:]]
-; CHECK:    %[[#bb78:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb76:]]
-; CHECK:    %[[#bb76:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb79:]] %[[#bb73:]]
-; CHECK:    %[[#bb79:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb73:]]
-; CHECK:    %[[#bb73:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb71:]]
-; CHECK:    %[[#bb71:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb69:]] %[[#bb68:]]
-; CHECK:    %[[#bb69:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb67:]]
-; CHECK:    %[[#bb68:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb80:]]
-; CHECK:    %[[#bb80:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb81:]] %[[#bb82:]] None
-; CHECK:                  OpBranch %[[#bb83:]]
-; CHECK:    %[[#bb83:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb84:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb85:]] %[[#bb86:]]
-; CHECK:    %[[#bb85:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb84:]]
-; CHECK:    %[[#bb86:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb84:]]
-; CHECK:    %[[#bb84:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb82:]] %[[#bb81:]]
-; CHECK:    %[[#bb82:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb80:]]
-; CHECK:    %[[#bb81:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb87:]]
+; CHECK: %[[#func_11:]] = OpFunction %[[#void:]] DontInline %[[#]]
 ; CHECK:    %[[#bb87:]] = OpLabel
-; CHECK:                  OpLoopMerge %[[#bb88:]] %[[#bb89:]] None
-; CHECK:                  OpBranch %[[#bb90:]]
-; CHECK:    %[[#bb90:]] = OpLabel
-; CHECK:                  OpSelectionMerge %[[#bb91:]] None
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb92:]] %[[#bb91:]]
-; CHECK:    %[[#bb92:]] = OpLabel
+; CHECK:                  OpBranch %[[#bb88:]]
+; CHECK:     %[[#bb88]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb89:]] %[[#bb90:]] None
 ; CHECK:                  OpBranch %[[#bb91:]]
-; CHECK:    %[[#bb91:]] = OpLabel
-; CHECK:                  OpBranchConditional %[[#]] %[[#bb89:]] %[[#bb88:]]
-; CHECK:    %[[#bb89:]] = OpLabel
-; CHECK:                  OpBranch %[[#bb87:]]
-; CHECK:    %[[#bb88:]] = OpLabel
+; CHECK:     %[[#bb91]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb92:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb93:]] %[[#bb94:]]
+; CHECK:     %[[#bb94]] = OpLabel
+; CHECK:                  OpBranch %[[#bb92]]
+; CHECK:     %[[#bb93]] = OpLabel
+; CHECK:                  OpBranch %[[#bb92]]
+; CHECK:     %[[#bb92]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb90]] %[[#bb89]]
+; CHECK:     %[[#bb89]] = OpLabel
+; CHECK:                  OpBranch %[[#bb95:]]
+; CHECK:     %[[#bb95]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb96:]] %[[#bb97:]] None
+; CHECK:                  OpBranch %[[#bb98:]]
+; CHECK:     %[[#bb98]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb99:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb100:]] %[[#bb101:]]
+; CHECK:    %[[#bb101]] = OpLabel
+; CHECK:                  OpBranch %[[#bb99]]
+; CHECK:    %[[#bb100]] = OpLabel
+; CHECK:                  OpBranch %[[#bb99]]
+; CHECK:     %[[#bb99]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb97]] %[[#bb96]]
+; CHECK:     %[[#bb96]] = OpLabel
+; CHECK:                  OpBranch %[[#bb102:]]
+; CHECK:    %[[#bb102]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb103:]] %[[#bb104:]] None
+; CHECK:                  OpBranch %[[#bb105:]]
+; CHECK:    %[[#bb105]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb106:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb107:]] %[[#bb108:]]
+; CHECK:    %[[#bb108]] = OpLabel
+; CHECK:                  OpBranch %[[#bb106]]
+; CHECK:    %[[#bb107]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb109:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb110:]] %[[#bb109]]
+; CHECK:    %[[#bb110]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb111:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb112:]] %[[#bb113:]]
+; CHECK:    %[[#bb113]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb114:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb114]] %[[#bb115:]]
+; CHECK:    %[[#bb115]] = OpLabel
+; CHECK:                  OpBranch %[[#bb114]]
+; CHECK:    %[[#bb114]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb116:]] %[[#bb111]]
+; CHECK:    %[[#bb116]] = OpLabel
+; CHECK:                  OpBranch %[[#bb111]]
+; CHECK:    %[[#bb112]] = OpLabel
+; CHECK:                  OpBranch %[[#bb111]]
+; CHECK:    %[[#bb111]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb109]] %[[#bb117:]]
+; CHECK:    %[[#bb117]] = OpLabel
+; CHECK:                  OpBranch %[[#bb109]]
+; CHECK:    %[[#bb109]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb106]] %[[#bb118:]]
+; CHECK:    %[[#bb118]] = OpLabel
+; CHECK:                  OpBranch %[[#bb106]]
+; CHECK:    %[[#bb106]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb119:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb119]] %[[#bb120:]]
+; CHECK:    %[[#bb120]] = OpLabel
+; CHECK:    %[[#bb119]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb104]] %[[#bb103]]
+; CHECK:    %[[#bb103]] = OpLabel
+; CHECK:                  OpBranch %[[#bb121:]]
+; CHECK:    %[[#bb121]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb122:]] %[[#bb123:]] None
+; CHECK:                  OpBranch %[[#bb124:]]
+; CHECK:    %[[#bb124]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb125:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb126:]] %[[#bb127:]]
+; CHECK:    %[[#bb127]] = OpLabel
+; CHECK:                  OpBranch %[[#bb125]]
+; CHECK:    %[[#bb126]] = OpLabel
+; CHECK:                  OpBranch %[[#bb125]]
+; CHECK:    %[[#bb125]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb123]] %[[#bb122]]
+; CHECK:    %[[#bb122]] = OpLabel
+; CHECK:                  OpBranch %[[#bb128:]]
+; CHECK:    %[[#bb128]] = OpLabel
+; CHECK:                  OpLoopMerge %[[#bb129:]] %[[#bb130:]] None
+; CHECK:                  OpBranch %[[#bb131:]]
+; CHECK:    %[[#bb131]] = OpLabel
+; CHECK:                  OpSelectionMerge %[[#bb132:]] None
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb133:]] %[[#bb134:]]
+; CHECK:    %[[#bb134]] = OpLabel
+; CHECK:                  OpBranch %[[#bb132]]
+; CHECK:    %[[#bb133]] = OpLabel
+; CHECK:                  OpBranch %[[#bb132]]
+; CHECK:    %[[#bb132]] = OpLabel
+; CHECK:                  OpBranchConditional %[[#]] %[[#bb130]] %[[#bb129]]
+; CHECK:    %[[#bb129]] = OpLabel
 ; CHECK:                  OpReturn
+; CHECK:    %[[#bb130]] = OpLabel
+; CHECK:                  OpBranch %[[#bb128]]
+; CHECK:    %[[#bb123]] = OpLabel
+; CHECK:                  OpBranch %[[#bb121]]
+; CHECK:    %[[#bb104]] = OpLabel
+; CHECK:                  OpBranch %[[#bb102]]
+; CHECK:     %[[#bb97]] = OpLabel
+; CHECK:                  OpBranch %[[#bb95]]
+; CHECK:     %[[#bb90]] = OpLabel
+; CHECK:                  OpBranch %[[#bb88]]
 ; CHECK:                  OpFunctionEnd
-; CHECK: %[[#func_52:]] = OpFunction %[[#void:]] None %[[#]]
-; CHECK:    %[[#bb93:]] = OpLabel
+
+; CHECK: %[[#func_85:]] = OpFunction %[[#void]] None %[[#]]
+; CHECK:   %[[#bb135:]] = OpLabel
 ; CHECK:                  OpReturn
 ; CHECK:                  OpFunctionEnd
 
-
-
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 target triple = "spirv-unknown-vulkan1.3-compute"
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
index 71f3ce9263da5..3b3841142263a 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll
@@ -38,18 +38,18 @@ entry:
   %b = alloca i32, align 4
   br i1 true, label %cond1_true, label %cond1_false
 
-; CHECK:  %[[#cond1_true]] = OpLabel
-; CHECK:                     OpStore %[[#reg_0]] %[[#]]
-; CHECK:                     OpBranch %[[#cond1_merge]]
-cond1_true:
-  %2 = load i32, ptr %a, align 4
-  br label %cond1_merge
-
 ; CHECK:  %[[#cond1_false]] = OpLabel
 ; CHECK:                      OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                      OpBranch %[[#cond1_merge]]
 cond1_false:
-  %3 = load i32, ptr %b, align 4
+  %2 = load i32, ptr %b, align 4
+  br label %cond1_merge
+
+; CHECK:  %[[#cond1_true]] = OpLabel
+; CHECK:                     OpStore %[[#reg_0]] %[[#]]
+; CHECK:                     OpBranch %[[#cond1_merge]]
+cond1_true:
+  %3 = load i32, ptr %a, align 4
   br label %cond1_merge
 
 ; CHECK: %[[#cond1_merge]] = OpLabel
@@ -58,7 +58,7 @@ cond1_false:
 ; CHECK:                     OpSelectionMerge %[[#cond2_merge:]] None
 ; CHECK:                     OpBranchConditional %[[#cond]] %[[#cond2_true:]] %[[#cond2_merge]]
 cond1_merge:
-  %cond = phi i32 [ %2, %cond1_true ], [ %3, %cond1_false ]
+  %cond = phi i32 [ %3, %cond1_true ], [ %2, %cond1_false ]
   %tobool1 = icmp ne i32 %cond, 0
   br i1 %tobool1, label %cond2_true, label %cond2_merge
 
@@ -76,14 +76,6 @@ cond2_merge:
   %call2 = call spir_func noundef i32 @fn() #4 [ "convergencectrl"(token %0) ]
   br i1 true, label %cond3_true, label %cond3_false
 
-; CHECK:  %[[#cond3_true]] = OpLabel
-; CHECK:                     OpFunctionCall
-; CHECK:                     OpStore %[[#reg_1]] %[[#]]
-; CHECK:                     OpBranch %[[#cond3_merge]]
-cond3_true:
-  %call5 = call spir_func noundef i32 @fn1() #4 [ "convergencectrl"(token %0) ]
-  br label %cond3_merge
-
 ; CHECK:  %[[#cond3_false]] = OpLabel
 ; CHECK:                      OpFunctionCall
 ; CHECK:                      OpStore %[[#reg_1]] %[[#]]
@@ -92,6 +84,14 @@ cond3_false:
   %call7 = call spir_func noundef i32 @fn2() #4 [ "convergencectrl"(token %0) ]
   br label %cond3_merge
 
+; CHECK:  %[[#cond3_true]] = OpLabel
+; CHECK:                     OpFunctionCall
+; CHECK:                     OpStore %[[#reg_1]] %[[#]]
+; CHECK:                     OpBranch %[[#cond3_merge]]
+cond3_true:
+  %call5 = call spir_func noundef i32 @fn1() #4 [ "convergencectrl"(token %0) ]
+  br label %cond3_merge
+
 ; CHECK:  %[[#cond3_merge]] = OpLabel
 ; CHECK:         %[[#tmp:]] = OpLoad %[[#]] %[[#reg_1]]
 ; CHECK:       %[[#cond:]] = OpINotEqual %[[#]] %[[#tmp]] %[[#]]
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
index 05071d03164d5..b95642f51c9fe 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-continue.ll
@@ -21,7 +21,7 @@ entry:
 ; Here a the loop header had to be split in two:
 ; - 1 header for the loop
 ; - 1 header for the condition.
-; In SPIR-V, a loop header cannot directly 
+
 ; CHECK: %[[#do_header:]] = OpLabel
 ; CHECK:                    OpLoopMerge %[[#do_merge:]] %[[#do_latch:]] None
 ; CHECK:                    OpBranch %[[#new_header:]]
@@ -34,27 +34,21 @@ do_header:
   store i32 0, ptr %var
   br i1 true, label %if.then, label %if.end
 
-; CHECK: %[[#if_then]] = OpLabel
-; CHECK:                 OpBranch %[[#if_merge]]
-if.then:
-  store i32 0, ptr %var
-  br label %do_latch
-
 ; CHECK: %[[#if_end]] = OpLabel
 ; CHECK:                OpBranch %[[#if_merge]]
 if.end:
   store i32 0, ptr %var
   br label %do_latch
 
+; CHECK: %[[#if_then]] = OpLabel
+; CHECK:                 OpBranch %[[#if_merge]]
+if.then:
+  store i32 0, ptr %var
+  br label %do_latch
+
 ; CHECK: %[[#if_merge]] = OpLabel
 ; CHECK:                  OpBranchConditional %[[#]] %[[#do_latch]] %[[#do_merge]]
 
-; CHECK: %[[#do_latch]] = OpLabel
-; CHECK:                  OpBranch %[[#do_header]]
-do_latch:
-  store i32 0, ptr %var
-  br i1 true, label %do_header, label %do.end
-
 ; CHECK: %[[#do_merge]] = OpLabel
 ; CHECK:                  OpBranch %[[#do2_header:]]
 do.end:
@@ -80,12 +74,6 @@ do3_header:
 ; CHECK: %[[#do3_body]] = OpLabel
 ; CHECK:                  OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
 
-; CHECK: %[[#do3_continue]] = OpLabel
-; CHECK:                      OpBranch %[[#do3_header]]
-do3_continue:
-  store i32 0, ptr %var
-  br i1 true, label %do3_header, label %do3_merge
-
 ; CHECK: %[[#do3_merge]] = OpLabel
 ; CHECK:                   OpBranch %[[#do2_new_latch:]]
 do3_merge:
@@ -95,16 +83,28 @@ do3_merge:
 ; CHECK: %[[#do2_new_latch]] = OpLabel
 ; CHECK:                       OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
 
+; CHECK: %[[#do2_merge]] = OpLabel
+; CHECK:                   OpReturn
+do2_merge:
+  ret void
+
 ; CHECK: %[[#do2_continue]] = OpLabel
 ; CHECK:                      OpBranch %[[#do2_header]]
 do2_continue:
   store i32 0, ptr %var
   br i1 true, label %do2_header, label %do2_merge
 
-; CHECK: %[[#do2_merge]] = OpLabel
-; CHECK:                   OpReturn
-do2_merge:
-  ret void
+; CHECK: %[[#do3_continue]] = OpLabel
+; CHECK:                      OpBranch %[[#do3_header]]
+do3_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do3_header, label %do3_merge
+
+; CHECK: %[[#do_latch]] = OpLabel
+; CHECK:                  OpBranch %[[#do_header]]
+do_latch:
+  store i32 0, ptr %var
+  br i1 true, label %do_header, label %do.end
 }
 
 declare token @llvm.experimental.convergence.entry() #1
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
index bef95f5f63bf7..4ea5fcde348a3 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-nested.ll
@@ -37,11 +37,6 @@ do3_header:
 
 ; CHECK:        %[[#do3_cond]] = OpLabel
 ; CHECK:                         OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
-; CHECK:    %[[#do3_continue]] = OpLabel
-; CHECK:                         OpBranch %[[#do3_header]]
-do3_continue:
-  store i32 0, ptr %var
-  br i1 true, label %do3_header, label %do3_merge
 
 ; CHECK:    %[[#do3_merge]] = OpLabel
 ; CHECK:                      OpBranch %[[#do2_cond:]]
@@ -51,30 +46,38 @@ do3_merge:
 
 ; CHECK:        %[[#do2_cond]] = OpLabel
 ; CHECK:                         OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
-; CHECK:    %[[#do2_continue]] = OpLabel
-; CHECK:                         OpBranch %[[#do2_header]]
-do2_continue:
-  store i32 0, ptr %var
-  br i1 true, label %do2_header, label %do2_merge
 
 ; CHECK:    %[[#do2_merge]] = OpLabel
 ; CHECK:                      OpBranch %[[#do1_cond:]]
+
+; CHECK:        %[[#do1_cond]] = OpLabel
+; CHECK:                         OpBranchConditional %[[#]] %[[#do1_continue]] %[[#do1_merge]]
 do2_merge:
   store i32 0, ptr %var
   br label %do1_continue
 
-; CHECK:        %[[#do1_cond]] = OpLabel
-; CHECK:                         OpBranchConditional %[[#]] %[[#do1_continue]] %[[#do1_merge]]
+; CHECK:    %[[#do1_merge]] = OpLabel
+; CHECK:                      OpReturn
+do1_merge:
+  ret void
+
 ; CHECK:    %[[#do1_continue]] = OpLabel
 ; CHECK:                         OpBranch %[[#do1_header]]
 do1_continue:
   store i32 0, ptr %var
   br i1 true, label %do1_header, label %do1_merge
 
-; CHECK:    %[[#do1_merge]] = OpLabel
-; CHECK:                      OpReturn
-do1_merge:
-  ret void
+; CHECK:    %[[#do2_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do2_header]]
+do2_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do2_header, label %do2_merge
+
+; CHECK:    %[[#do3_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do3_header]]
+do3_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do3_header, label %do3_merge
 }
 
 declare token @llvm.experimental.convergence.entry() #1
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
index 9f84fc317021f..1a419918790bc 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/do-plain.ll
@@ -24,18 +24,13 @@ entry:
 ; CHECK:    %[[#do1_header]] = OpLabel
 ; CHECK:                       OpLoopMerge %[[#do1_merge:]] %[[#do1_continue:]] None
 ; CHECK:                       OpBranch %[[#do1_cond:]]
-do1_header:
-  %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
-  store i32 0, ptr %var
-  br label %do1_continue
 
 ; CHECK:        %[[#do1_cond]] = OpLabel
 ; CHECK:                         OpBranchConditional %[[#]] %[[#do1_continue]] %[[#do1_merge]]
-; CHECK:    %[[#do1_continue]] = OpLabel
-; CHECK:                         OpBranch %[[#do1_header]]
-do1_continue:
+do1_header:
+  %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
   store i32 0, ptr %var
-  br i1 true, label %do1_header, label %do1_merge
+  br label %do1_continue
 
 ; CHECK: %[[#do1_merge]] = OpLabel
 ; CHECK:                   OpBranch %[[#do2_header:]]
@@ -53,11 +48,6 @@ do2_header:
 
 ; CHECK:        %[[#do2_cond]] = OpLabel
 ; CHECK:                         OpBranchConditional %[[#]] %[[#do2_continue]] %[[#do2_merge]]
-; CHECK:    %[[#do2_continue]] = OpLabel
-; CHECK:                         OpBranch %[[#do2_header]]
-do2_continue:
-  store i32 0, ptr %var
-  br i1 true, label %do2_header, label %do2_merge
 
 ; CHECK: %[[#do2_merge]] = OpLabel
 ; CHECK:                   OpBranch %[[#do3_header:]]
@@ -75,16 +65,29 @@ do3_header:
 
 ; CHECK:        %[[#do3_cond]] = OpLabel
 ; CHECK:                         OpBranchConditional %[[#]] %[[#do3_continue]] %[[#do3_merge]]
+
+; CHECK: %[[#do3_merge]] = OpLabel
+; CHECK:                   OpReturn
+do3_merge:
+  ret void
+
 ; CHECK:    %[[#do3_continue]] = OpLabel
 ; CHECK:                         OpBranch %[[#do3_header]]
 do3_continue:
   store i32 0, ptr %var
   br i1 true, label %do3_header, label %do3_merge
 
-; CHECK: %[[#do3_merge]] = OpLabel
-; CHECK:                   OpReturn
-do3_merge:
-  ret void
+; CHECK:    %[[#do2_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do2_header]]
+do2_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do2_header, label %do2_merge
+
+; CHECK:    %[[#do1_continue]] = OpLabel
+; CHECK:                         OpBranch %[[#do1_header]]
+do1_continue:
+  store i32 0, ptr %var
+  br i1 true, label %do1_header, label %do1_merge
 }
 
 
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
index 235f15b064470..732d8161766f2 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll
@@ -17,16 +17,12 @@ define internal spir_func void @main() #3 {
 ; CHECK:                      OpSelectionMerge %[[#merge:]] None
 ; CHECK:                      OpBranchConditional %[[#]] %[[#new_header:]] %[[#unreachable:]]
 
-; CHECK:     %[[#new_header]] = OpLabel
-; CHECK:                        OpSelectionMerge %[[#new_merge:]] None
-; CHECK:                        OpBranchConditional %[[#]] %[[#taint_true_merge:]] %[[#br_false:]]
-
 ; CHECK:       %[[#unreachable]] = OpLabel
 ; CHECK-NEXT:                      OpUnreachable
 
-; CHECK: %[[#taint_true_merge]] = OpLabel
-; CHECK:                          OpStore %[[#switch_0]] %[[#int_1]]
-; CHECK:                          OpBranch %[[#new_merge]]
+; CHECK:     %[[#new_header]] = OpLabel
+; CHECK:                        OpSelectionMerge %[[#new_merge:]] None
+; CHECK:                        OpBranchConditional %[[#]] %[[#taint_true_merge:]] %[[#br_false:]]
 
 ; CHECK:      %[[#br_false]] = OpLabel
 ; CHECK-DAG:                   OpStore %[[#switch_1]] %[[#int_0]]
@@ -47,6 +43,10 @@ define internal spir_func void @main() #3 {
 ; CHECK:                          OpStore %[[#switch_0]] %[[#int_1]]
 ; CHECK:                          OpBranch %[[#new_merge]]
 
+; CHECK: %[[#taint_true_merge]] = OpLabel
+; CHECK:                          OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK:                          OpBranch %[[#new_merge]]
+
 ; CHECK:      %[[#new_merge]] = OpLabel
 ; CHECK:             %[[#tmp:]] = OpLoad %[[#]] %[[#switch_0]]
 ; CHECK:            %[[#cond:]] = OpIEqual %[[#]] %[[#int_0]] %[[#tmp]]
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
index 5a5ea002e0942..4f8babb7f436f 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll
@@ -43,14 +43,6 @@ header:
   %2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
   br i1 true, label %left, label %right
 
-; CHECK:     %[[#left]] = OpLabel
-; CHECK-DAG:              OpStore %[[#switch_0]] %[[#int_0]]
-; CHECK-DAG:              OpStore %[[#variable]] %[[#int_1]]
-; CHECK:                  OpBranchConditional %[[#]] %[[#inner_merge]] %[[#left_next:]]
-left:
-  store i32 1, ptr %var
-  br i1 true, label %merge, label %left_next
-
 ; CHECK:     %[[#right]] = OpLabel
 ; CHECK-DAG:               OpStore %[[#switch_0]] %[[#int_0]]
 ; CHECK-DAG:               OpStore %[[#variable]] %[[#int_2]]
@@ -59,14 +51,6 @@ right:
   store i32 2, ptr %var
   br i1 true, label %merge, label %right_next
 
-; CHECK:     %[[#left_next]] = OpLabel
-; CHECK-DAG:                   OpStore %[[#switch_0]] %[[#int_1]]
-; CHECK-DAG:                   OpStore %[[#variable]] %[[#int_3]]
-; CHECK:                       OpBranch %[[#inner_merge]]
-left_next:
-  store i32 3, ptr %var
-  br label %continue
-
 ; CHECK:     %[[#right_next]] = OpLabel
 ; CHECK-DAG:                    OpStore %[[#switch_0]] %[[#int_1]]
 ; CHECK-DAG:                    OpStore %[[#variable]] %[[#int_4]]
@@ -75,6 +59,22 @@ right_next:
   store i32 4, ptr %var
   br label %continue
 
+; CHECK:     %[[#left]] = OpLabel
+; CHECK-DAG:              OpStore %[[#switch_0]] %[[#int_0]]
+; CHECK-DAG:              OpStore %[[#variable]] %[[#int_1]]
+; CHECK:                  OpBranchConditional %[[#]] %[[#inner_merge]] %[[#left_next:]]
+left:
+  store i32 1, ptr %var
+  br i1 true, label %merge, label %left_next
+
+; CHECK:     %[[#left_next]] = OpLabel
+; CHECK-DAG:                   OpStore %[[#switch_0]] %[[#int_1]]
+; CHECK-DAG:                   OpStore %[[#variable]] %[[#int_3]]
+; CHECK:                       OpBranch %[[#inner_merge]]
+left_next:
+  store i32 3, ptr %var
+  br label %continue
+
 ; CHECK: %[[#inner_merge]] = OpLabel
 ; CHECK:        %[[#tmp:]] = OpLoad %[[#]] %[[#switch_0]]
 ; CHECK:       %[[#cond:]] = OpIEqual %[[#]] %[[#int_0]] %[[#tmp]]
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
index 9b7f1e0f92019..eccd36d02a0da 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-simple-while-identity.ll
@@ -22,6 +22,11 @@ while.cond:
   %cmp = icmp ne i32 %2, 0
   br i1 %cmp, label %while.body, label %while.end
 
+; CHECK:        %[[#while_end]] = OpLabel
+; CHECK-NEXT:                     OpReturn
+while.end:
+  ret void
+
 ; CHECK:   %[[#while_body]] = OpLabel
 ; CHECK:                      OpBranch %[[#while_cond]]
 while.body:
@@ -29,10 +34,6 @@ while.body:
   store i32 %3, ptr %idx, align 4
   br label %while.cond
 
-; CHECK:        %[[#while_end]] = OpLabel
-; CHECK-NEXT:                     OpReturn
-while.end:
-  ret void
 }
 
 declare token @llvm.experimental.convergence.entry() #2
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
index e27aca8784286..5e64c24b78784 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll
@@ -39,11 +39,11 @@ while.body:
     i32 5, label %sw.bb2
   ]
 
-; CHECK: %[[#case_1]] = OpLabel
+; CHECK: %[[#case_5]] = OpLabel
 ; CHECK:                OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                OpBranch %[[#switch_end]]
-sw.bb:
-  store i32 1, ptr %a, align 4
+sw.bb2:
+  store i32 5, ptr %a, align 4
   br label %while.end
 
 ; CHECK: %[[#case_2]] = OpLabel
@@ -53,11 +53,11 @@ sw.bb1:
   store i32 3, ptr %a, align 4
   br label %while.end
 
-; CHECK: %[[#case_5]] = OpLabel
+; CHECK: %[[#case_1]] = OpLabel
 ; CHECK:                OpStore %[[#reg_0]] %[[#]]
 ; CHECK:                OpBranch %[[#switch_end]]
-sw.bb2:
-  store i32 5, ptr %a, align 4
+sw.bb:
+  store i32 1, ptr %a, align 4
   br label %while.end
 
 ; CHECK: %[[#switch_end]] = OpLabel
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpConstantSampler.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpConstantSampler.ll
index 7e0d13490fcf6..0169cc8b8fd81 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/OpConstantSampler.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/OpConstantSampler.ll
@@ -12,8 +12,8 @@
 
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
 
-; CHECK-SPIRV: %[[#SamplerID0:]] = OpConstantSampler %[[#]] Repeat 1 Nearest
-; CHECK-SPIRV: %[[#SamplerID1:]] = OpConstantSampler %[[#]] None 0 Nearest
+; CHECK-SPIRV-DAG: %[[#SamplerID0:]] = OpConstantSampler %[[#]] Repeat 1 Nearest
+; CHECK-SPIRV-DAG: %[[#SamplerID1:]] = OpConstantSampler %[[#]] None 0 Nearest
 ; CHECK-SPIRV: %[[#]] = OpSampledImage %[[#]] %[[#]] %[[#SamplerID0]]
 ; CHECK-SPIRV: %[[#]] = OpSampledImage %[[#]] %[[#]] %[[#SamplerID1]]
 
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll
index 54b2c78674776..2cba0f6ebd74b 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll
@@ -2,9 +2,7 @@
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV-DAG: %[[#Char:]] = OpTypeInt 8 0
-; CHECK-SPIRV-DAG: %[[#GlobalCharPtr:]] = OpTypePointer CrossWorkgroup %[[#Char]]
 ; CHECK-SPIRV-DAG: %[[#LocalCharPtr:]] = OpTypePointer Workgroup %[[#Char]]
-; CHECK-SPIRV-DAG: %[[#PrivateCharPtr:]] = OpTypePointer Function %[[#Char]]
 ; CHECK-SPIRV-DAG: %[[#GenericCharPtr:]] = OpTypePointer Generic %[[#Char]]
 
 ; CHECK-SPIRV-DAG: %[[#Int:]] = OpTypeInt 32 0
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll
index c98fef3631e04..63b517c805494 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll
@@ -12,8 +12,8 @@
 ;;     }
 ;; }
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
-; XFAIL: *
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 %struct.Node = type { %struct.Node.0 addrspace(1)* }
 %struct.Node.0 = type opaque
@@ -25,8 +25,8 @@ entry:
 for.cond:                                         ; preds = %for.inc, %entry
   %pNode.0 = phi %struct.Node addrspace(1)* [ %pNodes, %entry ], [ %1, %for.inc ]
   %j.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-; CHECK-SPIRV:      %[[#]] = OpPhi %[[#]] %[[#]] %[[#]] %[[#BitcastResultId:]] %[[#]]
-; CHECK-SPIRV-NEXT: OpPhi
+; CHECK:      %[[#]] = OpPhi %[[#]] %[[#]] %[[#]] %[[#BitcastResultId:]] %[[#]]
+; CHECK-NEXT: OpPhi
 
   %cmp = icmp slt i32 %j.0, 10
   br i1 %cmp, label %for.body, label %for.end
@@ -36,8 +36,8 @@ for.body:                                         ; preds = %for.cond
 
   %0 = load %struct.Node.0 addrspace(1)*, %struct.Node.0 addrspace(1)* addrspace(1)* %pNext, align 4
   %1 = bitcast %struct.Node.0 addrspace(1)* %0 to %struct.Node addrspace(1)*
-; CHECK-SPIRV: %[[#LoadResultId:]] = OpLoad %[[#]]
-; CHECK-SPIRV: %[[#BitcastResultId]] = OpBitcast %[[#]] %[[#LoadResultId]]
+; CHECK: %[[#LoadResultId:]] = OpLoad %[[#]]
+; CHECK: %[[#BitcastResultId]] = OpBitcast %[[#]] %[[#LoadResultId]]
 
   br label %for.inc
 
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spirv-event-null.ll b/llvm/test/CodeGen/SPIRV/transcoding/spirv-event-null.ll
index fcb61911e0d29..c8691c32710ad 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/spirv-event-null.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/spirv-event-null.ll
@@ -13,13 +13,32 @@
 ; CHECK-DAG: %[[#TyChar:]] = OpTypeInt 8 0
 ; CHECK-DAG: %[[#TyV4:]] = OpTypeVector %[[#TyChar]] 4
 ; CHECK-DAG: %[[#TyStructV4:]] = OpTypeStruct %[[#TyV4]]
-; CHECK-DAG: %[[#TyPtrSV4_W:]] = OpTypePointer Workgroup %[[#TyStructV4]]
 ; CHECK-DAG: %[[#TyPtrSV4_CW:]] = OpTypePointer CrossWorkgroup %[[#TyStructV4]]
 ; CHECK-DAG: %[[#TyPtrV4_W:]] = OpTypePointer Workgroup %[[#TyV4]]
 ; CHECK-DAG: %[[#TyPtrV4_CW:]] = OpTypePointer CrossWorkgroup %[[#TyV4]]
+; CHECK-DAG: %[[#TyHalf:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#TyHalfV2:]] = OpTypeVector %[[#TyHalf]] 2
+; CHECK-DAG: %[[#TyHalfV2_W:]] = OpTypePointer Workgroup %[[#TyHalfV2]]
+; CHECK-DAG: %[[#TyHalfV2_CW:]] = OpTypePointer CrossWorkgroup %[[#TyHalfV2]]
 
 ; Check correct translation of __spirv_GroupAsyncCopy and target("spirv.Event") zeroinitializer
 
+; CHECK: OpFunction
+; CHECK: %[[#HalfA1:]] = OpFunctionParameter %[[#TyHalfV2_W:]]
+; CHECK: %[[#HalfA2:]] = OpFunctionParameter %[[#TyHalfV2_CW:]]
+; CHECK: OpGroupAsyncCopy %[[#TyEvent]] %[[#]] %[[#HalfA1]] %[[#HalfA2]] %[[#]] %[[#]] %[[#ConstEvent]]
+; CHECK: OpFunctionEnd
+
+%StructEvent = type { target("spirv.Event") }
+
+define spir_kernel void @test_half(ptr addrspace(3) %_arg1, ptr addrspace(1) %_arg2) {
+entry:
+  %r = tail call spir_func target("spirv.Event") @_Z22__spirv_GroupAsyncCopyjPU3AS3Dv2_DF16_PU3AS1KS_mm9ocl_event(i32 2, ptr addrspace(3) %_arg1, ptr addrspace(1) %_arg2, i64 16, i64 10, target("spirv.Event") zeroinitializer)
+  ret void
+}
+
+declare dso_local spir_func target("spirv.Event") @_Z22__spirv_GroupAsyncCopyjPU3AS3Dv2_DF16_PU3AS1KS_mm9ocl_event(i32 noundef, ptr addrspace(3) noundef, ptr addrspace(1) noundef, i64 noundef, i64 noundef, target("spirv.Event"))
+
 ; CHECK: OpFunction
 ; CHECK: OpFunctionParameter
 ; CHECK: %[[#Src:]] = OpFunctionParameter
@@ -30,8 +49,6 @@
 ; CHECK: OpStore %[[#EventVar]] %[[#CopyRes]]
 ; CHECK: OpFunctionEnd
 
-%StructEvent = type { target("spirv.Event") }
-
 define spir_kernel void @foo(ptr addrspace(1) %_arg_out_ptr, ptr addrspace(3) %_arg_local_acc) {
 entry:
   %var = alloca %StructEvent
@@ -50,14 +67,11 @@ declare dso_local spir_func target("spirv.Event") @_Z22__spirv_GroupAsyncCopyjPU
 ; and %_arg_Local and %_arg are source/destination arguments in OpGroupAsyncCopy
 
 ; CHECK: OpFunction
-; CHECK: %[[#BarArg1:]] = OpFunctionParameter %[[#TyPtrSV4_W]]
+; CHECK: %[[#BarArg1:]] = OpFunctionParameter %[[#TyPtrV4_W]]
 ; CHECK: %[[#BarArg2:]] = OpFunctionParameter %[[#TyPtrSV4_CW]]
 ; CHECK: %[[#EventVarBar:]] = OpVariable %[[#TyStructPtr]] Function
 ; CHECK: %[[#EventVarBarCasted2:]] = OpBitcast %[[#TyEventPtr]] %[[#EventVarBar]]
-; CHECK: %[[#SrcBar:]] = OpInBoundsPtrAccessChain %[[#TyPtrSV4_CW]] %[[#BarArg2]] %[[#]]
-; CHECK-DAG: %[[#BarArg1Casted:]] = OpBitcast %[[#TyPtrV4_W]] %[[#BarArg1]]
-; CHECK-DAG: %[[#SrcBarCasted:]] = OpBitcast %[[#TyPtrV4_CW]] %[[#SrcBar]]
-; CHECK: %[[#ResBar:]] = OpGroupAsyncCopy %[[#TyEvent]] %[[#]] %[[#BarArg1Casted]] %[[#SrcBarCasted]] %[[#]] %[[#]] %[[#ConstEvent]]
+; CHECK: %[[#ResBar:]] = OpGroupAsyncCopy %[[#TyEvent]] %[[#]] %[[#BarArg1]] %[[#]] %[[#]] %[[#]] %[[#ConstEvent]]
 ; CHECK: %[[#EventVarBarCasted:]] = OpBitcast %[[#TyEventPtr]] %[[#EventVarBar]]
 ; CHECK: OpStore %[[#EventVarBarCasted]] %[[#ResBar]]
 ; CHECK: %[[#EventVarBarGen:]] = OpPtrCastToGeneric %[[#TyEventPtrGen]] %[[#EventVarBarCasted2]]
diff --git a/llvm/test/CodeGen/SPIRV/validate/sycl-hier-par-basic.ll b/llvm/test/CodeGen/SPIRV/validate/sycl-hier-par-basic.ll
new file mode 100644
index 0000000000000..77ed1d6fecf9a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/validate/sycl-hier-par-basic.ll
@@ -0,0 +1,974 @@
+; This is an excerpt from the SYCL end-to-end test suite, cleaned out from unrelevant details,
+; that reproduced multiple cases of the issues when OpPhi's result type mismatches with operand types.
+; The only pass criterion is that spirv-val considers output valid.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%struct.PFWGFunctor = type { i64, i64, i32, i32, %"class.sycl::_V1::accessor" }
+%"class.sycl::_V1::accessor" = type { %"class.sycl::_V1::detail::AccessorImplDevice", %union.anon }
+%"class.sycl::_V1::detail::AccessorImplDevice" = type { %"class.sycl::_V1::range", %"class.sycl::_V1::range", %"class.sycl::_V1::range" }
+%"class.sycl::_V1::range" = type { %"class.sycl::_V1::detail::array" }
+%"class.sycl::_V1::detail::array" = type { [1 x i64] }
+%union.anon = type { ptr addrspace(1) }
+%class.anon.2 = type { %"class.sycl::_V1::accessor" }
+%"class.sycl::_V1::group" = type { %"class.sycl::_V1::range", %"class.sycl::_V1::range", %"class.sycl::_V1::range", %"class.sycl::_V1::range" }
+%"class.sycl::_V1::group.15" = type { %"class.sycl::_V1::range.16", %"class.sycl::_V1::range.16", %"class.sycl::_V1::range.16", %"class.sycl::_V1::range.16" }
+%"class.sycl::_V1::range.16" = type { %"class.sycl::_V1::detail::array.17" }
+%"class.sycl::_V1::detail::array.17" = type { [2 x i64] }
+%"class.sycl::_V1::private_memory" = type { %struct.MyStruct }
+%struct.MyStruct = type { i32, i32 }
+
+@GFunctor = internal addrspace(3) global %struct.PFWGFunctor undef, align 8
+@WI.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WI.1 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WI.2 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WI.3 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WI.4 = internal unnamed_addr addrspace(3) global i32 undef, align 8
+@WI.6 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCnt = internal unnamed_addr addrspace(3) global i32 undef, align 4
+@__spirv_BuiltInNumWorkgroups = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+@GKernel1 = internal addrspace(3) global %class.anon.2 undef, align 8
+@GCnt2 = internal unnamed_addr addrspace(3) global i32 undef, align 4
+@GKernel2 = internal addrspace(3) global %class.anon.2 undef, align 8
+@GCnt3 = internal unnamed_addr addrspace(3) global i32 undef, align 4
+@GKernel3 = internal addrspace(3) global %class.anon.2 undef, align 8
+@GCnt4 = internal unnamed_addr addrspace(3) global i32 undef, align 4
+@GKernel4 = internal addrspace(3) global %class.anon.2 undef, align 8
+@GCnt5 = internal unnamed_addr addrspace(3) global i32 undef, align 4
+@__spirv_BuiltInLocalInvocationIndex = external local_unnamed_addr addrspace(1) constant i64, align 8
+@GThis = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GAsCast = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCmp = internal unnamed_addr addrspace(3) global i1 undef, align 1
+@WGCopy = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@WGCopy.1.0 = internal unnamed_addr addrspace(3) global i64 undef, align 16
+@WGCopy.1.1 = internal unnamed_addr addrspace(3) global i64 undef, align 16
+@WGCopy.1.2 = internal unnamed_addr addrspace(3) global i64 undef, align 16
+@WGCopy.1.3 = internal unnamed_addr addrspace(3) global i64 undef, align 16
+@WGCopy.1.4 = internal unnamed_addr addrspace(3) global i32 undef, align 16
+@WGCopy.1.5 = internal unnamed_addr addrspace(3) global i32 undef, align 16
+@WGCopy.1.6 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 16
+@ArgShadow = internal unnamed_addr addrspace(3) global %"class.sycl::_V1::group" undef, align 16
+@GAsCast2 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCmp2 = internal unnamed_addr addrspace(3) global i1 undef, align 1
+@WGCopy.3.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.4.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.5.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.6.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@ArgShadow.7 = internal unnamed_addr addrspace(3) global %"class.sycl::_V1::group" undef, align 16
+@GAscast3 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCmp3 = internal unnamed_addr addrspace(3) global i1 undef, align 1
+@WGCopy.9.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.10.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@ArgShadow.11 = internal unnamed_addr addrspace(3) global %"class.sycl::_V1::group" undef, align 16
+@GAsCast4 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCmp4 = internal unnamed_addr addrspace(3) global i1 undef, align 1
+@WGCopy.13.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.13.1 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.14.0 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@WGCopy.14.1 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@WGCopy.15.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.15.1 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.16.0 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@WGCopy.16.1 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@ArgShadow.17 = internal unnamed_addr addrspace(3) global %"class.sycl::_V1::group.15" undef, align 16
+@GAsCast5 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@GCmp5 = internal unnamed_addr addrspace(3) global i1 undef, align 1
+@WGCopy.19.0 = internal unnamed_addr addrspace(3) global i64 undef, align 8
+@WGCopy.20.0 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@WGCopy.20.1 = internal unnamed_addr addrspace(3) global ptr addrspace(4) undef, align 8
+@ArgShadow.21 = internal unnamed_addr addrspace(3) global %"class.sycl::_V1::group" undef, align 16
+@__spirv_BuiltInGlobalInvocationId = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInGlobalSize = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInLocalInvocationId = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInWorkgroupId = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInWorkgroupSize = external dso_local local_unnamed_addr addrspace(1) constant <3 x i64>, align 32
+
+; Function Attrs: convergent mustprogress norecurse nounwind
+define weak_odr dso_local spir_kernel void @_ZTS11PFWGFunctor(i64 noundef %_arg_wg_chunk, i64 noundef %_arg_range_length, i32 noundef %_arg_n_iter, i32 noundef %_arg_addend, ptr addrspace(1) noundef align 4 %_arg_dev_ptr, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr1, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr2, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr3) {
+entry:
+  %agg.tmp67 = alloca %"class.sycl::_V1::group", align 8
+  store i64 %_arg_wg_chunk, ptr addrspace(3) @GFunctor, align 8
+  store i64 %_arg_range_length, ptr addrspace(3) undef, align 8
+  store i32 %_arg_n_iter, ptr addrspace(3) undef, align 8
+  store i32 %_arg_addend, ptr addrspace(3) undef, align 4
+  %0 = load i64, ptr %_arg_dev_ptr1, align 8
+  %1 = load i64, ptr %_arg_dev_ptr2, align 8
+  %2 = load i64, ptr %_arg_dev_ptr3, align 8
+  store i64 %2, ptr addrspace(3) undef, align 8
+  store i64 %0, ptr addrspace(3) undef, align 8
+  store i64 %1, ptr addrspace(3) undef, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_dev_ptr, i64 %2
+  store ptr addrspace(1) %add.ptr.i, ptr addrspace(3) undef, align 8
+  %3 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %4 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %5 = load i64, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %6 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %agg.tmp67)
+  store i64 %3, ptr %agg.tmp67, align 1
+  %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  store i64 %4, ptr %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 16
+  store i64 %5, ptr %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  store i64 %6, ptr %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx, align 1
+  %7 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationIndex, align 8
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %cmpz15.i = icmp eq i64 %7, 0
+  br i1 %cmpz15.i, label %leader.i, label %merge.i
+
+leader.i:                                         ; preds = %entry
+  call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow, ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, i64 32, i1 false)
+  br label %merge.i
+
+merge.i:                                          ; preds = %leader.i, %entry
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.memcpy.p0.p3.i64(ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow, i64 32, i1 false)
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader.i, label %wg_cf.i
+
+wg_leader.i:                                      ; preds = %merge.i
+  %g.ascast.i = addrspacecast ptr %agg.tmp67 to ptr addrspace(4)
+  store ptr addrspace(4) %g.ascast.i, ptr addrspace(3) @GAsCast, align 8
+  store ptr addrspace(4) addrspacecast (ptr addrspace(3) @GFunctor to ptr addrspace(4)), ptr addrspace(3) @GThis, align 8
+  %8 = load i32, ptr addrspace(3) undef, align 4
+  %9 = load i64, ptr addrspace(3) @GFunctor, align 8
+  %index.i = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  %10 = load i64, ptr %index.i, align 8
+  %mul.i = mul i64 %9, %10
+  %localRange.i = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  %11 = load i64, ptr %localRange.i, align 8
+  %12 = load i64, ptr addrspace(3) undef, align 8
+  store i64 %9, ptr addrspace(3) @WI.0, align 8
+  store i64 %11, ptr addrspace(3) @WI.1, align 8
+  store i64 %mul.i, ptr addrspace(3) @WI.2, align 8
+  store i64 %12, ptr addrspace(3) @WI.3, align 8
+  store i32 %8, ptr addrspace(3) @WI.4, align 8
+  store ptr addrspace(4) undef, ptr addrspace(3) @WI.6, align 8
+  store i32 0, ptr addrspace(3) @GCnt, align 4
+  br label %wg_cf.i
+
+wg_cf.i:                                          ; preds = %wg_leader.i, %merge.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_this1.i = load ptr addrspace(4), ptr addrspace(3) @GThis, align 8
+  %n_iter.i = getelementptr inbounds i8, ptr addrspace(4) %wg_val_this1.i, i64 16
+  %13 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %wg_cf11.i, %wg_cf.i
+  %agg.tmp.i.sroa.0.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.0.0.copyload13, %wg_cf11.i ]
+  %agg.tmp.i.sroa.6.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.6.0.copyload15, %wg_cf11.i ]
+  %agg.tmp.i.sroa.7.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.7.0.copyload17, %wg_cf11.i ]
+  %agg.tmp.i.sroa.8.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.8.0.copyload19, %wg_cf11.i ]
+  %agg.tmp.i.sroa.9.0 = phi i32 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.9.0.copyload21, %wg_cf11.i ]
+  %agg.tmp.i.sroa.10.0 = phi i32 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.10.0.copyload23, %wg_cf11.i ]
+  %agg.tmp.i.sroa.11.0 = phi ptr addrspace(4) [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.11.0.copyload25, %wg_cf11.i ]
+  %this.addr.0.i = phi ptr addrspace(4) [ addrspacecast (ptr addrspace(3) @GFunctor to ptr addrspace(4)), %wg_cf.i ], [ %mat_ld13.i, %wg_cf11.i ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader4.i, label %wg_cf5.i
+
+wg_leader4.i:                                     ; preds = %for.cond.i
+  %14 = load i32, ptr addrspace(3) @GCnt, align 4
+  %15 = load i32, ptr addrspace(4) %n_iter.i, align 8
+  %cmp.i = icmp slt i32 %14, %15
+  store i1 %cmp.i, ptr addrspace(3) @GCmp, align 1
+  br label %wg_cf5.i
+
+wg_cf5.i:                                         ; preds = %wg_leader4.i, %for.cond.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_cmp.i = load i1, ptr addrspace(3) @GCmp, align 1
+  br i1 %wg_val_cmp.i, label %for.body.i, label %_ZNK11PFWGFunctorclEN4sycl3_V15groupILi1EEE.exit
+
+for.body.i:                                       ; preds = %wg_cf5.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader7.i, label %wg_cf8.i
+
+wg_leader7.i:                                     ; preds = %for.body.i
+  %agg.tmp.i.sroa.0.0.copyload = load i64, ptr addrspace(3) @WI.0, align 8
+  %agg.tmp.i.sroa.6.0.copyload = load i64, ptr addrspace(3) @WI.1, align 8
+  %agg.tmp.i.sroa.7.0.copyload = load i64, ptr addrspace(3) @WI.2, align 8
+  %agg.tmp.i.sroa.8.0.copyload = load i64, ptr addrspace(3) @WI.3, align 8
+  %agg.tmp.i.sroa.9.0.copyload = load i32, ptr addrspace(3) @WI.4, align 8
+  %agg.tmp.i.sroa.11.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WI.6, align 8
+  br label %wg_cf8.i
+
+wg_cf8.i:                                         ; preds = %wg_leader7.i, %for.body.i
+  %agg.tmp.i.sroa.0.1 = phi i64 [ %agg.tmp.i.sroa.0.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.0.0, %for.body.i ]
+  %agg.tmp.i.sroa.6.1 = phi i64 [ %agg.tmp.i.sroa.6.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.6.0, %for.body.i ]
+  %agg.tmp.i.sroa.7.1 = phi i64 [ %agg.tmp.i.sroa.7.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.7.0, %for.body.i ]
+  %agg.tmp.i.sroa.8.1 = phi i64 [ %agg.tmp.i.sroa.8.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.8.0, %for.body.i ]
+  %agg.tmp.i.sroa.9.1 = phi i32 [ %agg.tmp.i.sroa.9.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.9.0, %for.body.i ]
+  %agg.tmp.i.sroa.11.1 = phi ptr addrspace(4) [ %agg.tmp.i.sroa.11.0.copyload, %wg_leader7.i ], [ %agg.tmp.i.sroa.11.0, %for.body.i ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %TestMat.i, label %LeaderMat.i
+
+TestMat.i:                                        ; preds = %wg_cf8.i
+  store i64 %agg.tmp.i.sroa.0.1, ptr addrspace(3) @WGCopy.1.0, align 16
+  store i64 %agg.tmp.i.sroa.6.1, ptr addrspace(3) @WGCopy.1.1, align 16
+  store i64 %agg.tmp.i.sroa.7.1, ptr addrspace(3) @WGCopy.1.2, align 16
+  store i64 %agg.tmp.i.sroa.8.1, ptr addrspace(3) @WGCopy.1.3, align 16
+  store i32 %agg.tmp.i.sroa.9.1, ptr addrspace(3) @WGCopy.1.4, align 16
+  store i32 %agg.tmp.i.sroa.10.0, ptr addrspace(3) @WGCopy.1.5, align 16
+  store ptr addrspace(4) %agg.tmp.i.sroa.11.1, ptr addrspace(3) @WGCopy.1.6, align 16
+  store ptr addrspace(4) %this.addr.0.i, ptr addrspace(3) @WGCopy, align 8
+  br label %LeaderMat.i
+
+LeaderMat.i:                                      ; preds = %TestMat.i, %wg_cf8.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %mat_ld13.i = load ptr addrspace(4), ptr addrspace(3) @WGCopy, align 8
+  %agg.tmp.i.sroa.0.0.copyload13 = load i64, ptr addrspace(3) @WGCopy.1.0, align 16
+  %agg.tmp.i.sroa.6.0.copyload15 = load i64, ptr addrspace(3) @WGCopy.1.1, align 16
+  %agg.tmp.i.sroa.7.0.copyload17 = load i64, ptr addrspace(3) @WGCopy.1.2, align 16
+  %agg.tmp.i.sroa.8.0.copyload19 = load i64, ptr addrspace(3) @WGCopy.1.3, align 16
+  %agg.tmp.i.sroa.9.0.copyload21 = load i32, ptr addrspace(3) @WGCopy.1.4, align 16
+  %agg.tmp.i.sroa.10.0.copyload23 = load i32, ptr addrspace(3) @WGCopy.1.5, align 16
+  %agg.tmp.i.sroa.11.0.copyload25 = load ptr addrspace(4), ptr addrspace(3) @WGCopy.1.6, align 16
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %cmp.not.i.i = icmp ult i64 %13, %agg.tmp.i.sroa.0.0.copyload13
+  br i1 %cmp.not.i.i, label %if.end.i.i, label %lexit1
+
+if.end.i.i:                                       ; preds = %LeaderMat.i
+  %add.i.i = add i64 %agg.tmp.i.sroa.0.0.copyload13, %agg.tmp.i.sroa.6.0.copyload15
+  %sub.i.i = add i64 %add.i.i, -1
+  %div.i.i = udiv i64 %sub.i.i, %agg.tmp.i.sroa.6.0.copyload15
+  %mul.i.i = mul i64 %13, %div.i.i
+  %add4.i.i = add i64 %agg.tmp.i.sroa.7.0.copyload17, %mul.i.i
+  %add6.i.i = add i64 %add4.i.i, %div.i.i
+  %.sroa.speculated.i.i = call i64 @llvm.umin.i64(i64 %agg.tmp.i.sroa.8.0.copyload19, i64 %add6.i.i)
+  %16 = getelementptr inbounds i8, ptr addrspace(4) %agg.tmp.i.sroa.11.0.copyload25, i64 24
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %for.body.i.i, %if.end.i.i
+  %ind.0.i.i = phi i64 [ %add4.i.i, %if.end.i.i ], [ %inc.i.i, %for.body.i.i ]
+  %cmp8.i.i = icmp ult i64 %ind.0.i.i, %.sroa.speculated.i.i
+  br i1 %cmp8.i.i, label %for.body.i.i, label %lexit1
+
+for.body.i.i:                                     ; preds = %for.cond.i.i
+  %17 = load ptr addrspace(1), ptr addrspace(4) %16, align 8
+  %arrayidx.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %17, i64 %ind.0.i.i
+  %18 = load i32, ptr addrspace(1) %arrayidx.i.i.i, align 4
+  %add10.i.i = add nsw i32 %18, %agg.tmp.i.sroa.9.0.copyload21
+  store i32 %add10.i.i, ptr addrspace(1) %arrayidx.i.i.i, align 4
+  %inc.i.i = add nuw i64 %ind.0.i.i, 1
+  br label %for.cond.i.i
+
+lexit1: ; preds = %for.cond.i.i, %LeaderMat.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader10.i, label %wg_cf11.i
+
+wg_leader10.i:                                    ; preds = %lexit1
+  %19 = load i32, ptr addrspace(3) @GCnt, align 4
+  %inc.i = add nsw i32 %19, 1
+  store i32 %inc.i, ptr addrspace(3) @GCnt, align 4
+  br label %wg_cf11.i
+
+wg_cf11.i:                                        ; preds = %wg_leader10.i, %lexit1
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br label %for.cond.i
+
+_ZNK11PFWGFunctorclEN4sycl3_V15groupILi1EEE.exit: ; preds = %wg_cf5.i
+  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %agg.tmp67)
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+
+; Function Attrs: convergent nounwind
+declare dso_local spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef, i32 noundef, i32 noundef)
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memcpy.p0.p3.i64(ptr noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg)
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i64 @llvm.umin.i64(i64, i64)
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+; Function Attrs: convergent mustprogress norecurse nounwind
+define weak_odr dso_local spir_kernel void @bar(ptr addrspace(1) noundef align 4 %_arg_dev_ptr, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr1, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr2, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr3) {
+entry:
+  %agg.tmp67 = alloca %"class.sycl::_V1::group", align 8
+  %0 = load i64, ptr %_arg_dev_ptr1, align 8
+  %1 = load i64, ptr %_arg_dev_ptr2, align 8
+  %2 = load i64, ptr %_arg_dev_ptr3, align 8
+  store i64 %2, ptr addrspace(3) @GKernel1, align 8
+  store i64 %0, ptr addrspace(3) undef, align 8
+  store i64 %1, ptr addrspace(3) undef, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_dev_ptr, i64 %2
+  store ptr addrspace(1) %add.ptr.i, ptr addrspace(3) undef, align 8
+  %3 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %4 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %5 = load i64, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %6 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %agg.tmp67)
+  store i64 %3, ptr %agg.tmp67, align 1
+  %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  store i64 %4, ptr %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 16
+  store i64 %5, ptr %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  store i64 %6, ptr %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx, align 1
+  %7 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationIndex, align 8
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %cmpz27.i = icmp eq i64 %7, 0
+  br i1 %cmpz27.i, label %leader.i, label %merge.i
+
+leader.i:                                         ; preds = %entry
+  call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.7, ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, i64 32, i1 false)
+  br label %merge.i
+
+merge.i:                                          ; preds = %leader.i, %entry
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.memcpy.p0.p3.i64(ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.7, i64 32, i1 false)
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz27.i, label %wg_leader.i, label %wg_cf.i
+
+wg_leader.i:                                      ; preds = %merge.i
+  %g.ascast.i = addrspacecast ptr %agg.tmp67 to ptr addrspace(4)
+  store ptr addrspace(4) %g.ascast.i, ptr addrspace(3) @GAsCast2, align 8
+  store i32 0, ptr addrspace(3) @GCnt2, align 4
+  br label %wg_cf.i
+
+wg_cf.i:                                          ; preds = %wg_leader.i, %merge.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %8 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, align 32
+  %9 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  %cmp.i.i.i.i.i.i = icmp ult i64 %8, 2147483648
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %wg_cf18.i, %wg_cf.i
+  %agg.tmp5.i.sroa.0.0 = phi i64 [ undef, %wg_cf.i ], [ %18, %wg_cf18.i ]
+  %agg.tmp4.i.sroa.0.0 = phi i64 [ undef, %wg_cf.i ], [ %17, %wg_cf18.i ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz27.i, label %wg_leader8.i, label %wg_cf9.i
+
+wg_leader8.i:                                     ; preds = %for.cond.i
+  %10 = load i32, ptr addrspace(3) @GCnt2, align 4
+  %cmp.i = icmp slt i32 %10, 2
+  store i1 %cmp.i, ptr addrspace(3) @GCmp2, align 1
+  br label %wg_cf9.i
+
+wg_cf9.i:                                         ; preds = %wg_leader8.i, %for.cond.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_cmp.i = load i1, ptr addrspace(3) @GCmp2, align 1
+  br i1 %wg_val_cmp.i, label %for.body.i, label %lexit2
+
+for.body.i:                                       ; preds = %wg_cf9.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz27.i, label %TestMat25.i, label %LeaderMat22.i
+
+TestMat25.i:                                      ; preds = %for.body.i
+  store i64 %agg.tmp5.i.sroa.0.0, ptr addrspace(3) @WGCopy.6.0, align 8
+  store i64 ptrtoint (ptr addrspace(4) addrspacecast (ptr addrspace(3) @GKernel1 to ptr addrspace(4)) to i64), ptr addrspace(3) @WGCopy.4.0, align 8
+  store i64 5, ptr addrspace(3) @WGCopy.3.0, align 8
+  store i64 %agg.tmp4.i.sroa.0.0, ptr addrspace(3) @WGCopy.5.0, align 8
+  br label %LeaderMat22.i
+
+LeaderMat22.i:                                    ; preds = %TestMat25.i, %for.body.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %11 = load i64, ptr addrspace(3) @WGCopy.3.0, align 8
+  %12 = load i64, ptr addrspace(3) @WGCopy.4.0, align 8
+  %13 = inttoptr i64 %12 to ptr addrspace(4)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %14 = getelementptr inbounds i8, ptr addrspace(4) %13, i64 24
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %for.body.i.i, %LeaderMat22.i
+  %storemerge.i.i = phi i64 [ %9, %LeaderMat22.i ], [ %add.i.i, %for.body.i.i ]
+  %cmp.i.i = icmp ult i64 %storemerge.i.i, %11
+  br i1 %cmp.i.i, label %for.body.i.i, label %lexit3
+
+for.body.i.i:                                     ; preds = %for.cond.i.i
+  call void @llvm.assume(i1 %cmp.i.i.i.i.i.i)
+  %15 = load ptr addrspace(1), ptr addrspace(4) %14, align 8
+  %arrayidx.i.i.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %15, i64 %8
+  %16 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %inc.i.i.i.i = add nsw i32 %16, 1
+  store i32 %inc.i.i.i.i, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %add.i.i = add i64 %storemerge.i.i, %4
+  br label %for.cond.i.i
+
+lexit3: ; preds = %for.cond.i.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz27.i, label %TestMat.i, label %LeaderMat.i
+
+TestMat.i:                                        ; preds = %lexit3
+  store i64 ptrtoint (ptr addrspace(4) addrspacecast (ptr addrspace(3) @GKernel1 to ptr addrspace(4)) to i64), ptr addrspace(3) @WGCopy.6.0, align 8
+  store i64 %12, ptr addrspace(3) @WGCopy.4.0, align 8
+  store i64 %11, ptr addrspace(3) @WGCopy.3.0, align 8
+  store i64 2, ptr addrspace(3) @WGCopy.5.0, align 8
+  br label %LeaderMat.i
+
+LeaderMat.i:                                      ; preds = %TestMat.i, %lexit3
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %17 = load i64, ptr addrspace(3) @WGCopy.5.0, align 8
+  %18 = load i64, ptr addrspace(3) @WGCopy.6.0, align 8
+  %19 = inttoptr i64 %18 to ptr addrspace(4)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %20 = getelementptr inbounds i8, ptr addrspace(4) %19, i64 24
+  br label %for.cond.i.i19
+
+for.cond.i.i19:                                   ; preds = %for.body.i.i22, %LeaderMat.i
+  %storemerge.i.i20 = phi i64 [ %9, %LeaderMat.i ], [ %add.i.i26, %for.body.i.i22 ]
+  %cmp.i.i21 = icmp ult i64 %storemerge.i.i20, %17
+  br i1 %cmp.i.i21, label %for.body.i.i22, label %lexit4
+
+for.body.i.i22:                                   ; preds = %for.cond.i.i19
+  call void @llvm.assume(i1 %cmp.i.i.i.i.i.i)
+  %21 = load ptr addrspace(1), ptr addrspace(4) %20, align 8
+  %arrayidx.i.i.i.i.i23 = getelementptr inbounds i32, ptr addrspace(1) %21, i64 %8
+  %22 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i23, align 4
+  %inc.i.i.i.i25 = add nsw i32 %22, 1
+  store i32 %inc.i.i.i.i25, ptr addrspace(1) %arrayidx.i.i.i.i.i23, align 4
+  %add.i.i26 = add i64 %storemerge.i.i20, %4
+  br label %for.cond.i.i19
+
+lexit4: ; preds = %for.cond.i.i19
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz27.i, label %wg_leader17.i, label %wg_cf18.i
+
+wg_leader17.i:                                    ; preds = %lexit4
+  %23 = load i32, ptr addrspace(3) @GCnt2, align 4
+  %inc.i = add nsw i32 %23, 1
+  store i32 %inc.i, ptr addrspace(3) @GCnt2, align 4
+  br label %wg_cf18.i
+
+wg_cf18.i:                                        ; preds = %wg_leader17.i, %lexit4
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br label %for.cond.i
+
+lexit2: ; preds = %wg_cf9.i
+  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %agg.tmp67)
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef)
+
+; Function Attrs: convergent mustprogress norecurse nounwind
+define weak_odr dso_local spir_kernel void @test1(ptr addrspace(1) noundef align 4 %_arg_dev_ptr, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr1, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr2, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr3) {
+entry:
+  %agg.tmp67 = alloca %"class.sycl::_V1::group", align 8
+  %0 = load i64, ptr %_arg_dev_ptr1, align 8
+  %1 = load i64, ptr %_arg_dev_ptr2, align 8
+  %2 = load i64, ptr %_arg_dev_ptr3, align 8
+  store i64 %2, ptr addrspace(3) @GKernel2, align 8
+  store i64 %0, ptr addrspace(3) undef, align 8
+  store i64 %1, ptr addrspace(3) undef, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_dev_ptr, i64 %2
+  store ptr addrspace(1) %add.ptr.i, ptr addrspace(3) undef, align 8
+  %3 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %4 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %5 = load i64, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %6 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %agg.tmp67)
+  store i64 %3, ptr %agg.tmp67, align 1
+  %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  store i64 %4, ptr %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 16
+  store i64 %5, ptr %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  store i64 %6, ptr %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx, align 1
+  %7 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationIndex, align 8
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %cmpz15.i = icmp eq i64 %7, 0
+  br i1 %cmpz15.i, label %leader.i, label %merge.i
+
+leader.i:                                         ; preds = %entry
+  call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.11, ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, i64 32, i1 false)
+  br label %merge.i
+
+merge.i:                                          ; preds = %leader.i, %entry
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.memcpy.p0.p3.i64(ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.11, i64 32, i1 false)
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader.i, label %wg_cf.i
+
+wg_leader.i:                                      ; preds = %merge.i
+  %g.ascast.i = addrspacecast ptr %agg.tmp67 to ptr addrspace(4)
+  store ptr addrspace(4) %g.ascast.i, ptr addrspace(3) @GAscast3, align 8
+  store i32 0, ptr addrspace(3) @GCnt3, align 4
+  br label %wg_cf.i
+
+wg_cf.i:                                          ; preds = %wg_leader.i, %merge.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %8 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, align 32
+  %9 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  %cmp.i.i.i.i.i.i = icmp ult i64 %8, 2147483648
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %wg_cf11.i, %wg_cf.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader4.i, label %wg_cf5.i
+
+wg_leader4.i:                                     ; preds = %for.cond.i
+  %10 = load i32, ptr addrspace(3) @GCnt3, align 4
+  %cmp.i = icmp slt i32 %10, 2
+  store i1 %cmp.i, ptr addrspace(3) @GCmp3, align 1
+  br label %wg_cf5.i
+
+wg_cf5.i:                                         ; preds = %wg_leader4.i, %for.cond.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_cmp.i = load i1, ptr addrspace(3) @GCmp3, align 1
+  br i1 %wg_val_cmp.i, label %for.body.i, label %lexit6
+
+for.body.i:                                       ; preds = %wg_cf5.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %TestMat.i, label %LeaderMat.i
+
+TestMat.i:                                        ; preds = %for.body.i
+  store i64 ptrtoint (ptr addrspace(4) addrspacecast (ptr addrspace(3) @GKernel2 to ptr addrspace(4)) to i64), ptr addrspace(3) @WGCopy.10.0, align 8
+  store i64 5, ptr addrspace(3) @WGCopy.9.0, align 8
+  br label %LeaderMat.i
+
+LeaderMat.i:                                      ; preds = %TestMat.i, %for.body.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %11 = load i64, ptr addrspace(3) @WGCopy.9.0, align 8
+  %12 = load i64, ptr addrspace(3) @WGCopy.10.0, align 8
+  %13 = inttoptr i64 %12 to ptr addrspace(4)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %14 = getelementptr inbounds i8, ptr addrspace(4) %13, i64 24
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %for.body.i.i, %LeaderMat.i
+  %storemerge.i.i = phi i64 [ %9, %LeaderMat.i ], [ %add.i.i, %for.body.i.i ]
+  %cmp.i.i = icmp ult i64 %storemerge.i.i, %11
+  br i1 %cmp.i.i, label %for.body.i.i, label %lexit7
+
+for.body.i.i:                                     ; preds = %for.cond.i.i
+  %cmp5.not.i.i.i.i.i.i = icmp ne i64 %storemerge.i.i, %9
+  %cond.i.i.i.i = zext i1 %cmp5.not.i.i.i.i.i.i to i32
+  call void @llvm.assume(i1 %cmp.i.i.i.i.i.i)
+  %15 = load ptr addrspace(1), ptr addrspace(4) %14, align 8
+  %arrayidx.i.i.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %15, i64 %8
+  %16 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %add.i.i.i.i = add nsw i32 %16, %cond.i.i.i.i
+  store i32 %add.i.i.i.i, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %add.i.i = add i64 %storemerge.i.i, %4
+  br label %for.cond.i.i
+
+lexit7: ; preds = %for.cond.i.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz15.i, label %wg_leader10.i, label %wg_cf11.i
+
+wg_leader10.i:                                    ; preds = %lexit7
+  %17 = load i32, ptr addrspace(3) @GCnt3, align 4
+  %inc.i = add nsw i32 %17, 1
+  store i32 %inc.i, ptr addrspace(3) @GCnt3, align 4
+  br label %wg_cf11.i
+
+wg_cf11.i:                                        ; preds = %wg_leader10.i, %lexit7
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br label %for.cond.i
+
+lexit6: ; preds = %wg_cf5.i
+  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %agg.tmp67)
+  ret void
+}
+
+; Function Attrs: convergent mustprogress norecurse nounwind
+define weak_odr dso_local spir_kernel void @test2(ptr addrspace(1) noundef align 4 %_arg_dev_ptr, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr1, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr2, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr3) {
+entry:
+  %priv.i = alloca %"class.sycl::_V1::private_memory", align 4
+  %agg.tmp67 = alloca %"class.sycl::_V1::group.15", align 8
+  %0 = load i64, ptr %_arg_dev_ptr1, align 8
+  %1 = load i64, ptr %_arg_dev_ptr2, align 8
+  %2 = load i64, ptr %_arg_dev_ptr3, align 8
+  store i64 %2, ptr addrspace(3) @GKernel3, align 8
+  store i64 %0, ptr addrspace(3) undef, align 8
+  store i64 %1, ptr addrspace(3) undef, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_dev_ptr, i64 %2
+  store ptr addrspace(1) %add.ptr.i, ptr addrspace(3) undef, align 8
+  %3 = load i64, ptr addrspace(1) undef, align 8
+  %4 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %5 = load i64, ptr addrspace(1) undef, align 8
+  %6 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %7 = load i64, ptr addrspace(1) undef, align 8
+  %8 = load i64, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %9 = load i64, ptr addrspace(1) undef, align 8
+  %10 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  call void @llvm.lifetime.start.p0(i64 64, ptr nonnull %agg.tmp67)
+  store i64 %3, ptr %agg.tmp67, align 1
+  %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  store i64 %4, ptr %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 16
+  store i64 %5, ptr %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  store i64 %6, ptr %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.5.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 32
+  store i64 %7, ptr %agg.tmp6.sroa.5.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.6.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 40
+  store i64 %8, ptr %agg.tmp6.sroa.6.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.7.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 48
+  store i64 %9, ptr %agg.tmp6.sroa.7.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.8.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 56
+  store i64 %10, ptr %agg.tmp6.sroa.8.0.agg.tmp67.sroa_idx, align 1
+  %11 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationIndex, align 8
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %cmpz32.i = icmp eq i64 %11, 0
+  br i1 %cmpz32.i, label %leader.i, label %merge.i
+
+leader.i:                                         ; preds = %entry
+  call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef align 16 dereferenceable(64) @ArgShadow.17, ptr noundef nonnull align 8 dereferenceable(64) %agg.tmp67, i64 64, i1 false)
+  br label %merge.i
+
+merge.i:                                          ; preds = %leader.i, %entry
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.memcpy.p0.p3.i64(ptr noundef nonnull align 8 dereferenceable(64) %agg.tmp67, ptr addrspace(3) noundef align 16 dereferenceable(64) @ArgShadow.17, i64 64, i1 false)
+  %priv.ascast.i = addrspacecast ptr %priv.i to ptr addrspace(4)
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader.i, label %wg_cf.i
+
+wg_leader.i:                                      ; preds = %merge.i
+  %g.ascast.i = addrspacecast ptr %agg.tmp67 to ptr addrspace(4)
+  store ptr addrspace(4) %g.ascast.i, ptr addrspace(3) @GAsCast4, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %priv.i)
+  store i32 0, ptr addrspace(3) @GCnt4, align 4
+  br label %wg_cf.i
+
+wg_cf.i:                                          ; preds = %wg_leader.i, %merge.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %12 = load i64, ptr addrspace(1) undef, align 8
+  %13 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, align 32
+  %14 = load i64, ptr addrspace(1) undef, align 8
+  %15 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  %mul.i.i.i.i.i.i = mul i64 %12, %4
+  %add.i.i.i.i.i.i = add i64 %mul.i.i.i.i.i.i, %13
+  %cmp.i.i.i.i.i.i = icmp ult i64 %add.i.i.i.i.i.i, 2147483648
+  %conv.i.i.i.i.i = trunc i64 %add.i.i.i.i.i.i to i32
+  %y.i.i.i.i.i = getelementptr inbounds i8, ptr %priv.i, i64 4
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %wg_cf20.i, %wg_cf.i
+  %agg.tmp6.i.sroa.9.0 = phi ptr addrspace(4) [ undef, %wg_cf.i ], [ %agg.tmp6.i.sroa.9.0.copyload40, %wg_cf20.i ]
+  %agg.tmp5.i.sroa.0.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp5.i.sroa.0.0.copyload44, %wg_cf20.i ]
+  %agg.tmp5.i.sroa.8.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp5.i.sroa.8.0.copyload48, %wg_cf20.i ]
+  %agg.tmp2.i.sroa.0.0 = phi ptr addrspace(4) [ undef, %wg_cf.i ], [ %agg.tmp2.i.sroa.0.0.copyload52, %wg_cf20.i ]
+  %agg.tmp2.i.sroa.8.0 = phi ptr addrspace(4) [ undef, %wg_cf.i ], [ %agg.tmp2.i.sroa.8.0.copyload56, %wg_cf20.i ]
+  %agg.tmp.i.sroa.0.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.0.0.copyload60, %wg_cf20.i ]
+  %agg.tmp.i.sroa.8.0 = phi i64 [ undef, %wg_cf.i ], [ %agg.tmp.i.sroa.8.0.copyload64, %wg_cf20.i ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader10.i, label %wg_cf11.i
+
+wg_leader10.i:                                    ; preds = %for.cond.i
+  %16 = load i32, ptr addrspace(3) @GCnt4, align 4
+  %cmp.i = icmp slt i32 %16, 2
+  store i1 %cmp.i, ptr addrspace(3) @GCmp4, align 1
+  br label %wg_cf11.i
+
+wg_cf11.i:                                        ; preds = %wg_leader10.i, %for.cond.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_cmp.i = load i1, ptr addrspace(3) @GCmp4, align 1
+  br i1 %wg_val_cmp.i, label %for.body.i, label %for.end.i
+
+for.body.i:                                       ; preds = %wg_cf11.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader13.i, label %wg_cf14.i
+
+wg_leader13.i:                                    ; preds = %for.body.i
+  br label %wg_cf14.i
+
+wg_cf14.i:                                        ; preds = %wg_leader13.i, %for.body.i
+  %agg.tmp2.i.sroa.0.1 = phi ptr addrspace(4) [ addrspacecast (ptr addrspace(3) @GKernel3 to ptr addrspace(4)), %wg_leader13.i ], [ %agg.tmp2.i.sroa.0.0, %for.body.i ]
+  %agg.tmp2.i.sroa.8.1 = phi ptr addrspace(4) [ %priv.ascast.i, %wg_leader13.i ], [ %agg.tmp2.i.sroa.8.0, %for.body.i ]
+  %agg.tmp.i.sroa.0.1 = phi i64 [ 7, %wg_leader13.i ], [ %agg.tmp.i.sroa.0.0, %for.body.i ]
+  %agg.tmp.i.sroa.8.1 = phi i64 [ 3, %wg_leader13.i ], [ %agg.tmp.i.sroa.8.0, %for.body.i ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %TestMat30.i, label %LeaderMat27.i
+
+TestMat30.i:                                      ; preds = %wg_cf14.i
+  store i64 %agg.tmp.i.sroa.0.1, ptr addrspace(3) @WGCopy.13.0, align 8
+  store i64 %agg.tmp.i.sroa.8.1, ptr addrspace(3) @WGCopy.13.1, align 8
+  store ptr addrspace(4) %agg.tmp2.i.sroa.0.1, ptr addrspace(3) @WGCopy.14.0, align 8
+  store ptr addrspace(4) %agg.tmp2.i.sroa.8.1, ptr addrspace(3) @WGCopy.14.1, align 8
+  store i64 %agg.tmp5.i.sroa.0.0, ptr addrspace(3) @WGCopy.15.0, align 8
+  store i64 %agg.tmp5.i.sroa.8.0, ptr addrspace(3) @WGCopy.15.1, align 8
+  store ptr addrspace(4) %priv.ascast.i, ptr addrspace(3) @WGCopy.16.0, align 8
+  store ptr addrspace(4) %agg.tmp6.i.sroa.9.0, ptr addrspace(3) @WGCopy.16.1, align 8
+  br label %LeaderMat27.i
+
+LeaderMat27.i:                                    ; preds = %TestMat30.i, %wg_cf14.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %agg.tmp6.i.sroa.0.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WGCopy.16.0, align 8
+  %agg.tmp6.i.sroa.9.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WGCopy.16.1, align 8
+  %agg.tmp5.i.sroa.0.0.copyload = load i64, ptr addrspace(3) @WGCopy.15.0, align 8
+  %agg.tmp5.i.sroa.8.0.copyload = load i64, ptr addrspace(3) @WGCopy.15.1, align 8
+  %agg.tmp2.i.sroa.0.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WGCopy.14.0, align 8
+  %agg.tmp.i.sroa.0.0.copyload = load i64, ptr addrspace(3) @WGCopy.13.0, align 8
+  %agg.tmp.i.sroa.8.0.copyload = load i64, ptr addrspace(3) @WGCopy.13.1, align 8
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %17 = getelementptr inbounds i8, ptr addrspace(4) %agg.tmp2.i.sroa.0.0.copyload, i64 24
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %lexit10, %LeaderMat27.i
+  %storemerge.i.i = phi i64 [ %14, %LeaderMat27.i ], [ %add.i.i, %lexit10 ]
+  %cmp.i.i = icmp ult i64 %storemerge.i.i, %agg.tmp.i.sroa.0.0.copyload
+  br i1 %cmp.i.i, label %for.cond.i.i.i, label %lexit11
+
+for.cond.i.i.i:                                   ; preds = %for.body.i.i.i, %for.cond.i.i
+  %storemerge.i.i.i = phi i64 [ %add.i.i.i, %for.body.i.i.i ], [ %15, %for.cond.i.i ]
+  %cmp.i.i.i = icmp ult i64 %storemerge.i.i.i, %agg.tmp.i.sroa.8.0.copyload
+  br i1 %cmp.i.i.i, label %for.body.i.i.i, label %lexit10
+
+for.body.i.i.i:                                   ; preds = %for.cond.i.i.i
+  call void @llvm.assume(i1 %cmp.i.i.i.i.i.i)
+  %18 = load ptr addrspace(1), ptr addrspace(4) %17, align 8
+  %arrayidx.i.i.i.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %18, i64 %add.i.i.i.i.i.i
+  %19 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i.i, align 4
+  %inc.i.i.i.i.i = add nsw i32 %19, 1
+  store i32 %inc.i.i.i.i.i, ptr addrspace(1) %arrayidx.i.i.i.i.i.i, align 4
+  store i32 %conv.i.i.i.i.i, ptr %priv.i, align 4
+  store i32 5, ptr %y.i.i.i.i.i, align 4
+  %add.i.i.i = add i64 %storemerge.i.i.i, %6
+  br label %for.cond.i.i.i
+
+lexit10: ; preds = %for.cond.i.i.i
+  %add.i.i = add i64 %storemerge.i.i, %5
+  br label %for.cond.i.i
+
+lexit11: ; preds = %for.cond.i.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader16.i, label %wg_cf17.i
+
+wg_leader16.i:                                    ; preds = %lexit11
+  br label %wg_cf17.i
+
+wg_cf17.i:                                        ; preds = %wg_leader16.i, %lexit11
+  %agg.tmp6.i.sroa.0.1 = phi ptr addrspace(4) [ %priv.ascast.i, %wg_leader16.i ], [ %agg.tmp6.i.sroa.0.0.copyload, %lexit11 ]
+  %agg.tmp6.i.sroa.9.1 = phi ptr addrspace(4) [ addrspacecast (ptr addrspace(3) @GKernel3 to ptr addrspace(4)), %wg_leader16.i ], [ %agg.tmp6.i.sroa.9.0.copyload, %lexit11 ]
+  %agg.tmp5.i.sroa.0.1 = phi i64 [ 7, %wg_leader16.i ], [ %agg.tmp5.i.sroa.0.0.copyload, %lexit11 ]
+  %agg.tmp5.i.sroa.8.1 = phi i64 [ 3, %wg_leader16.i ], [ %agg.tmp5.i.sroa.8.0.copyload, %lexit11 ]
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %TestMat.i, label %LeaderMat.i
+
+TestMat.i:                                        ; preds = %wg_cf17.i
+  store i64 %agg.tmp.i.sroa.0.0.copyload, ptr addrspace(3) @WGCopy.13.0, align 8
+  store i64 %agg.tmp.i.sroa.8.0.copyload, ptr addrspace(3) @WGCopy.13.1, align 8
+  store ptr addrspace(4) %agg.tmp2.i.sroa.0.0.copyload, ptr addrspace(3) @WGCopy.14.0, align 8
+  store ptr addrspace(4) %priv.ascast.i, ptr addrspace(3) @WGCopy.14.1, align 8
+  store i64 %agg.tmp5.i.sroa.0.1, ptr addrspace(3) @WGCopy.15.0, align 8
+  store i64 %agg.tmp5.i.sroa.8.1, ptr addrspace(3) @WGCopy.15.1, align 8
+  store ptr addrspace(4) %agg.tmp6.i.sroa.0.1, ptr addrspace(3) @WGCopy.16.0, align 8
+  store ptr addrspace(4) %agg.tmp6.i.sroa.9.1, ptr addrspace(3) @WGCopy.16.1, align 8
+  br label %LeaderMat.i
+
+LeaderMat.i:                                      ; preds = %TestMat.i, %wg_cf17.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %agg.tmp6.i.sroa.9.0.copyload40 = load ptr addrspace(4), ptr addrspace(3) @WGCopy.16.1, align 8
+  %agg.tmp5.i.sroa.0.0.copyload44 = load i64, ptr addrspace(3) @WGCopy.15.0, align 8
+  %agg.tmp5.i.sroa.8.0.copyload48 = load i64, ptr addrspace(3) @WGCopy.15.1, align 8
+  %agg.tmp2.i.sroa.0.0.copyload52 = load ptr addrspace(4), ptr addrspace(3) @WGCopy.14.0, align 8
+  %agg.tmp2.i.sroa.8.0.copyload56 = load ptr addrspace(4), ptr addrspace(3) @WGCopy.14.1, align 8
+  %agg.tmp.i.sroa.0.0.copyload60 = load i64, ptr addrspace(3) @WGCopy.13.0, align 8
+  %agg.tmp.i.sroa.8.0.copyload64 = load i64, ptr addrspace(3) @WGCopy.13.1, align 8
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %20 = getelementptr inbounds i8, ptr addrspace(4) %agg.tmp6.i.sroa.9.0.copyload40, i64 24
+  br label %for.cond.i.i25
+
+for.cond.i.i25:                                   ; preds = %lexit12, %LeaderMat.i
+  %storemerge.i.i26 = phi i64 [ %14, %LeaderMat.i ], [ %add.i.i31, %lexit12 ]
+  %cmp.i.i27 = icmp ult i64 %storemerge.i.i26, %agg.tmp5.i.sroa.0.0.copyload44
+  br i1 %cmp.i.i27, label %for.cond.i.i.i28, label %lexit13
+
+for.cond.i.i.i28:                                 ; preds = %for.body.i.i.i32, %for.cond.i.i25
+  %storemerge.i.i.i29 = phi i64 [ %add.i.i.i35, %for.body.i.i.i32 ], [ %15, %for.cond.i.i25 ]
+  %cmp.i.i.i30 = icmp ult i64 %storemerge.i.i.i29, %agg.tmp5.i.sroa.8.0.copyload48
+  br i1 %cmp.i.i.i30, label %for.body.i.i.i32, label %lexit12
+
+for.body.i.i.i32:                                 ; preds = %for.cond.i.i.i28
+  %21 = load i32, ptr %priv.i, align 4
+  %22 = load i32, ptr %y.i.i.i.i.i, align 4
+  %add.i.i.i.i.i = add nsw i32 %21, %22
+  call void @llvm.assume(i1 %cmp.i.i.i.i.i.i)
+  %23 = load ptr addrspace(1), ptr addrspace(4) %20, align 8
+  %arrayidx.i.i.i.i.i.i33 = getelementptr inbounds i32, ptr addrspace(1) %23, i64 %add.i.i.i.i.i.i
+  %24 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i.i33, align 4
+  %add4.i.i.i.i.i = add nsw i32 %24, %add.i.i.i.i.i
+  store i32 %add4.i.i.i.i.i, ptr addrspace(1) %arrayidx.i.i.i.i.i.i33, align 4
+  %add.i.i.i35 = add i64 %storemerge.i.i.i29, %6
+  br label %for.cond.i.i.i28
+
+lexit12: ; preds = %for.cond.i.i.i28
+  %add.i.i31 = add i64 %storemerge.i.i26, %5
+  br label %for.cond.i.i25
+
+lexit13: ; preds = %for.cond.i.i25
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader19.i, label %wg_cf20.i
+
+wg_leader19.i:                                    ; preds = %lexit13
+  %25 = load i32, ptr addrspace(3) @GCnt4, align 4
+  %inc.i = add nsw i32 %25, 1
+  store i32 %inc.i, ptr addrspace(3) @GCnt4, align 4
+  br label %wg_cf20.i
+
+wg_cf20.i:                                        ; preds = %wg_leader19.i, %lexit13
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br label %for.cond.i
+
+for.end.i:                                        ; preds = %wg_cf11.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz32.i, label %wg_leader22.i, label %lexit14
+
+wg_leader22.i:                                    ; preds = %for.end.i
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %priv.i)
+  br label %lexit14
+
+lexit14: ; preds = %wg_leader22.i, %for.end.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.lifetime.end.p0(i64 64, ptr nonnull %agg.tmp67)
+  ret void
+}
+
+; Function Attrs: convergent mustprogress norecurse nounwind
+define weak_odr dso_local spir_kernel void @test3(ptr addrspace(1) noundef align 4 %_arg_dev_ptr, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr1, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr2, ptr noundef byval(%"class.sycl::_V1::range") align 8 %_arg_dev_ptr3) {
+entry:
+  %agg.tmp67 = alloca %"class.sycl::_V1::group", align 8
+  %0 = load i64, ptr %_arg_dev_ptr1, align 8
+  %1 = load i64, ptr %_arg_dev_ptr2, align 8
+  %2 = load i64, ptr %_arg_dev_ptr3, align 8
+  store i64 %2, ptr addrspace(3) @GKernel4, align 8
+  store i64 %0, ptr addrspace(3) undef, align 8
+  store i64 %1, ptr addrspace(3) undef, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_dev_ptr, i64 %2
+  store ptr addrspace(1) %add.ptr.i, ptr addrspace(3) undef, align 8
+  %3 = load i64, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %4 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %5 = load i64, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %6 = load i64, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %agg.tmp67)
+  store i64 %3, ptr %agg.tmp67, align 1
+  %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 8
+  store i64 %4, ptr %agg.tmp6.sroa.2.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 16
+  store i64 %5, ptr %agg.tmp6.sroa.3.0.agg.tmp67.sroa_idx, align 1
+  %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx = getelementptr inbounds i8, ptr %agg.tmp67, i64 24
+  store i64 %6, ptr %agg.tmp6.sroa.4.0.agg.tmp67.sroa_idx, align 1
+  %7 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationIndex, align 8
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %cmpz16.i = icmp eq i64 %7, 0
+  br i1 %cmpz16.i, label %leader.i, label %merge.i
+
+leader.i:                                         ; preds = %entry
+  call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.21, ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, i64 32, i1 false)
+  br label %merge.i
+
+merge.i:                                          ; preds = %leader.i, %entry
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call void @llvm.memcpy.p0.p3.i64(ptr noundef nonnull align 8 dereferenceable(32) %agg.tmp67, ptr addrspace(3) noundef align 16 dereferenceable(32) @ArgShadow.21, i64 32, i1 false)
+  tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz16.i, label %wg_leader.i, label %wg_cf.i
+
+wg_leader.i:                                      ; preds = %merge.i
+  %g.ascast.i = addrspacecast ptr %agg.tmp67 to ptr addrspace(4)
+  store ptr addrspace(4) %g.ascast.i, ptr addrspace(3) @GAsCast5, align 8
+  store i32 0, ptr addrspace(3) @GCnt5, align 4
+  br label %wg_cf.i
+
+wg_cf.i:                                          ; preds = %wg_leader.i, %merge.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_g.ascast.i = load ptr addrspace(4), ptr addrspace(3) @GAsCast5, align 8
+  %8 = load i64, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  %9 = trunc i64 %4 to i32
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %wg_cf12.i, %wg_cf.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz16.i, label %wg_leader5.i, label %wg_cf6.i
+
+wg_leader5.i:                                     ; preds = %for.cond.i
+  %10 = load i32, ptr addrspace(3) @GCnt5, align 4
+  %cmp.i = icmp slt i32 %10, 2
+  store i1 %cmp.i, ptr addrspace(3) @GCmp5, align 1
+  br label %wg_cf6.i
+
+wg_cf6.i:                                         ; preds = %wg_leader5.i, %for.cond.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %wg_val_cmp.i = load i1, ptr addrspace(3) @GCmp5, align 1
+  br i1 %wg_val_cmp.i, label %for.body.i, label %lexit20
+
+for.body.i:                                       ; preds = %wg_cf6.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz16.i, label %TestMat.i, label %LeaderMat.i
+
+TestMat.i:                                        ; preds = %for.body.i
+  store ptr addrspace(4) %wg_val_g.ascast.i, ptr addrspace(3) @WGCopy.20.0, align 8
+  store ptr addrspace(4) addrspacecast (ptr addrspace(3) @GKernel4 to ptr addrspace(4)), ptr addrspace(3) @WGCopy.20.1, align 8
+  store i64 5, ptr addrspace(3) @WGCopy.19.0, align 8
+  br label %LeaderMat.i
+
+LeaderMat.i:                                      ; preds = %TestMat.i, %for.body.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  %11 = load i64, ptr addrspace(3) @WGCopy.19.0, align 8
+  %agg.tmp2.i.sroa.0.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WGCopy.20.0, align 8
+  %agg.tmp2.i.sroa.6.0.copyload = load ptr addrspace(4), ptr addrspace(3) @WGCopy.20.1, align 8
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  %index.i.i.i.i.i = getelementptr inbounds i8, ptr addrspace(4) %agg.tmp2.i.sroa.0.0.copyload, i64 24
+  %12 = getelementptr inbounds i8, ptr addrspace(4) %agg.tmp2.i.sroa.6.0.copyload, i64 24
+  %13 = trunc i64 %11 to i32
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %for.body.i.i, %LeaderMat.i
+  %storemerge.i.i = phi i64 [ %8, %LeaderMat.i ], [ %add.i.i, %for.body.i.i ]
+  %cmp.i.i = icmp ult i64 %storemerge.i.i, %11
+  br i1 %cmp.i.i, label %for.body.i.i, label %lexit21
+
+for.body.i.i:                                     ; preds = %for.cond.i.i
+  %14 = load i64, ptr addrspace(4) %index.i.i.i.i.i, align 8
+  %mul.i.i.i.i = mul i64 %14, 10
+  %mul3.i.i.i.i = shl i64 %storemerge.i.i, 1
+  %add.i.i.i.i = add i64 %mul.i.i.i.i, %mul3.i.i.i.i
+  %15 = load ptr addrspace(1), ptr addrspace(4) %12, align 8
+  %arrayidx.i.i.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %15, i64 %add.i.i.i.i
+  %16 = load i32, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %conv9.i.i.i.i = add i32 %16, %13
+  store i32 %conv9.i.i.i.i, ptr addrspace(1) %arrayidx.i.i.i.i.i, align 4
+  %add14.i.i.i.i = or disjoint i64 %add.i.i.i.i, 1
+  %17 = load ptr addrspace(1), ptr addrspace(4) %12, align 8
+  %arrayidx.i25.i.i.i.i = getelementptr inbounds i32, ptr addrspace(1) %17, i64 %add14.i.i.i.i
+  %18 = load i32, ptr addrspace(1) %arrayidx.i25.i.i.i.i, align 4
+  %conv18.i.i.i.i = add i32 %18, %9
+  store i32 %conv18.i.i.i.i, ptr addrspace(1) %arrayidx.i25.i.i.i.i, align 4
+  %add.i.i = add i64 %storemerge.i.i, %4
+  br label %for.cond.i.i
+
+lexit21: ; preds = %for.cond.i.i
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 2, i32 noundef 2, i32 noundef 272)
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br i1 %cmpz16.i, label %wg_leader11.i, label %wg_cf12.i
+
+wg_leader11.i:                                    ; preds = %lexit21
+  %19 = load i32, ptr addrspace(3) @GCnt5, align 4
+  %inc.i = add nsw i32 %19, 1
+  store i32 %inc.i, ptr addrspace(3) @GCnt5, align 4
+  br label %wg_cf12.i
+
+wg_cf12.i:                                        ; preds = %wg_leader11.i, %lexit21
+  call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+  br label %for.cond.i
+
+lexit20: ; preds = %wg_cf6.i
+  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %agg.tmp67)
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/validate/sycl-tangle-group-algorithms.ll b/llvm/test/CodeGen/SPIRV/validate/sycl-tangle-group-algorithms.ll
new file mode 100644
index 0000000000000..b6b919f36d92c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/validate/sycl-tangle-group-algorithms.ll
@@ -0,0 +1,4673 @@
+; This is an excerpt from the SYCL end-to-end test suite, cleaned out from
+; unrelevant details, that reproduced cases of invalid SPIR-V generation due
+; to wrong types, deduced from the input LLVM IR. Namely, this test case covers
+; cases of type mismatch when null pointer constant is used in different
+; contexts and so with different pointee types, and intertwined
+; load/store/function call LLVM IR input with bitcasts inserted between
+; instruction uses.
+
+; The only pass criterion is that spirv-val considers output valid.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.5-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%"nd_item" = type { i8 }
+%struct.AssertHappened = type { i32, [257 x i8], [257 x i8], [129 x i8], i32, i64, i64, i64, i64, i64, i64 }
+%"range" = type { %"detail::array" }
+%"detail::array" = type { [1 x i64] }
+%class.anon = type { %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor" }
+%"accessor" = type { %"detail::AccessorImplDevice", %union.anon }
+%"detail::AccessorImplDevice" = type { %"range", %"range", %"range" }
+%union.anon = type { ptr addrspace(1) }
+%class.anon.6 = type { ptr addrspace(4), ptr addrspace(4), ptr addrspace(4), ptr addrspace(4) }
+%"group" = type { %"range", %"range", %"range", %"range" }
+%"item" = type { %"detail::AccessorImplDevice" }
+%"item.22" = type { %"sd_ItemBase.23" }
+%"sd_ItemBase.23" = type { %"range", %"range" }
+%"tangle_group" = type { %"ss_sub_group_mask" }
+%"ss_sub_group_mask" = type { i64, i64 }
+%class.anon.8 = type { %"accessor", %"accessor", [8 x i8], %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor", %"accessor" }
+%"vec.16" = type { %"struct.std::array.20" }
+%"struct.std::array.20" = type { [4 x i32] }
+%class.anon.15 = type { ptr addrspace(4), ptr addrspace(4), ptr addrspace(4) }
+%class.anon.7 = type { ptr addrspace(4), ptr addrspace(4) }
+
+@.str = private unnamed_addr addrspace(1) constant [21 x i8] c"bits_num <= max_bits\00", align 1
+@.str.1 = private unnamed_addr addrspace(1) constant [17 x i8] c"subgroupmask.hpp\00", align 1
+@__PRETTY_FUNCTION1 = private unnamed_addr addrspace(1) constant [32 x i8] c"subgroup_mask(BitsType, size_t)\00", align 1
+@.str.2 = private unnamed_addr addrspace(1) constant [15 x i8] c"bn <= max_bits\00", align 1
+@__PRETTY_FUNCTION2 = private unnamed_addr addrspace(1) constant [52 x i8] c"BitsType subgroup_mask::valuable_bits(size_t) const\00", align 1
+@__spirv_BuiltInSubgroupMaxSize = external dso_local addrspace(1) constant i32, align 4
+@__spirv_BuiltInSubgroupLocalInvocationId = external dso_local addrspace(1) constant i32, align 4
+@_ZSt6ignore = linkonce_odr dso_local addrspace(1) constant %"nd_item" undef, align 1
+@__spirv_BuiltInNumWorkgroups = external dso_local addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInGlobalOffset = external dso_local addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInGlobalInvocationId = external dso_local addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInGlobalSize = external dso_local addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInLocalInvocationId = external dso_local addrspace(1) constant <3 x i64>, align 32
+@SPIR_AssertHappenedMem = linkonce_odr dso_local addrspace(1) global %struct.AssertHappened zeroinitializer
+@__spirv_BuiltInWorkgroupId = external dso_local addrspace(1) constant <3 x i64>, align 32
+@__spirv_BuiltInWorkgroupSize = external dso_local addrspace(1) constant <3 x i64>, align 32
+
+
+define weak_odr dso_local spir_kernel void @TestKernel(ptr addrspace(1) %_arg_TmpAcc, ptr byval(%"range") %_arg_TmpAcc1, ptr byval(%"range") %_arg_TmpAcc2, ptr byval(%"range") %_arg_TmpAcc3, ptr addrspace(1) align 1 %_arg_BarrierAcc, ptr byval(%"range") %_arg_BarrierAcc4, ptr byval(%"range") %_arg_BarrierAcc5, ptr byval(%"range") %_arg_BarrierAcc6, ptr addrspace(1) align 1 %_arg_BroadcastAcc, ptr byval(%"range") %_arg_BroadcastAcc7, ptr byval(%"range") %_arg_BroadcastAcc8, ptr byval(%"range") %_arg_BroadcastAcc9, ptr addrspace(1) align 1 %_arg_AnyAcc, ptr byval(%"range") %_arg_AnyAcc10, ptr byval(%"range") %_arg_AnyAcc11, ptr byval(%"range") %_arg_AnyAcc12, ptr addrspace(1) align 1 %_arg_AllAcc, ptr byval(%"range") %_arg_AllAcc13, ptr byval(%"range") %_arg_AllAcc14, ptr byval(%"range") %_arg_AllAcc15, ptr addrspace(1) align 1 %_arg_NoneAcc, ptr byval(%"range") %_arg_NoneAcc16, ptr byval(%"range") %_arg_NoneAcc17, ptr byval(%"range") %_arg_NoneAcc18, ptr addrspace(1) align 1 %_arg_ReduceAcc, ptr byval(%"range") %_arg_ReduceAcc19, ptr byval(%"range") %_arg_ReduceAcc20, ptr byval(%"range") %_arg_ReduceAcc21, ptr addrspace(1) align 1 %_arg_ExScanAcc, ptr byval(%"range") %_arg_ExScanAcc22, ptr byval(%"range") %_arg_ExScanAcc23, ptr byval(%"range") %_arg_ExScanAcc24, ptr addrspace(1) align 1 %_arg_IncScanAcc, ptr byval(%"range") %_arg_IncScanAcc25, ptr byval(%"range") %_arg_IncScanAcc26, ptr byval(%"range") %_arg_IncScanAcc27, ptr addrspace(1) align 1 %_arg_ShiftLeftAcc, ptr byval(%"range") %_arg_ShiftLeftAcc28, ptr byval(%"range") %_arg_ShiftLeftAcc29, ptr byval(%"range") %_arg_ShiftLeftAcc30, ptr addrspace(1) align 1 %_arg_ShiftRightAcc, ptr byval(%"range") %_arg_ShiftRightAcc31, ptr byval(%"range") %_arg_ShiftRightAcc32, ptr byval(%"range") %_arg_ShiftRightAcc33, ptr addrspace(1) align 1 %_arg_SelectAcc, ptr byval(%"range") %_arg_SelectAcc34, ptr byval(%"range") %_arg_SelectAcc35, ptr byval(%"range") %_arg_SelectAcc36, ptr addrspace(1) align 1 %_arg_PermuteXorAcc, ptr byval(%"range") %_arg_PermuteXorAcc37, ptr byval(%"range") %_arg_PermuteXorAcc38, ptr byval(%"range") %_arg_PermuteXorAcc39) {
+entry:
+  %_arg_TmpAcc.addr = alloca ptr addrspace(1)
+  %_arg_BarrierAcc.addr = alloca ptr addrspace(1)
+  %_arg_BroadcastAcc.addr = alloca ptr addrspace(1)
+  %_arg_AnyAcc.addr = alloca ptr addrspace(1)
+  %_arg_AllAcc.addr = alloca ptr addrspace(1)
+  %_arg_NoneAcc.addr = alloca ptr addrspace(1)
+  %_arg_ReduceAcc.addr = alloca ptr addrspace(1)
+  %_arg_ExScanAcc.addr = alloca ptr addrspace(1)
+  %_arg_IncScanAcc.addr = alloca ptr addrspace(1)
+  %_arg_ShiftLeftAcc.addr = alloca ptr addrspace(1)
+  %_arg_ShiftRightAcc.addr = alloca ptr addrspace(1)
+  %_arg_SelectAcc.addr = alloca ptr addrspace(1)
+  %_arg_PermuteXorAcc.addr = alloca ptr addrspace(1)
+  %Kernel = alloca %class.anon
+  %agg.tmp = alloca %"range"
+  %agg.tmp41 = alloca %"range"
+  %agg.tmp42 = alloca %"range"
+  %agg.tmp44 = alloca %"range"
+  %agg.tmp45 = alloca %"range"
+  %agg.tmp46 = alloca %"range"
+  %agg.tmp48 = alloca %"range"
+  %agg.tmp49 = alloca %"range"
+  %agg.tmp50 = alloca %"range"
+  %agg.tmp52 = alloca %"range"
+  %agg.tmp53 = alloca %"range"
+  %agg.tmp54 = alloca %"range"
+  %agg.tmp56 = alloca %"range"
+  %agg.tmp57 = alloca %"range"
+  %agg.tmp58 = alloca %"range"
+  %agg.tmp60 = alloca %"range"
+  %agg.tmp61 = alloca %"range"
+  %agg.tmp62 = alloca %"range"
+  %agg.tmp64 = alloca %"range"
+  %agg.tmp65 = alloca %"range"
+  %agg.tmp66 = alloca %"range"
+  %agg.tmp68 = alloca %"range"
+  %agg.tmp69 = alloca %"range"
+  %agg.tmp70 = alloca %"range"
+  %agg.tmp72 = alloca %"range"
+  %agg.tmp73 = alloca %"range"
+  %agg.tmp74 = alloca %"range"
+  %agg.tmp76 = alloca %"range"
+  %agg.tmp77 = alloca %"range"
+  %agg.tmp78 = alloca %"range"
+  %agg.tmp80 = alloca %"range"
+  %agg.tmp81 = alloca %"range"
+  %agg.tmp82 = alloca %"range"
+  %agg.tmp84 = alloca %"range"
+  %agg.tmp85 = alloca %"range"
+  %agg.tmp86 = alloca %"range"
+  %agg.tmp88 = alloca %"range"
+  %agg.tmp89 = alloca %"range"
+  %agg.tmp90 = alloca %"range"
+  %agg.tmp91 = alloca %"nd_item", align 1
+  %Kernel.ascast = addrspacecast ptr %Kernel to ptr addrspace(4)
+  %agg.tmp91.ascast = addrspacecast ptr %agg.tmp91 to ptr addrspace(4)
+  store ptr addrspace(1) %_arg_TmpAcc, ptr %_arg_TmpAcc.addr
+  store ptr addrspace(1) %_arg_BarrierAcc, ptr %_arg_BarrierAcc.addr
+  store ptr addrspace(1) %_arg_BroadcastAcc, ptr %_arg_BroadcastAcc.addr
+  store ptr addrspace(1) %_arg_AnyAcc, ptr %_arg_AnyAcc.addr
+  store ptr addrspace(1) %_arg_AllAcc, ptr %_arg_AllAcc.addr
+  store ptr addrspace(1) %_arg_NoneAcc, ptr %_arg_NoneAcc.addr
+  store ptr addrspace(1) %_arg_ReduceAcc, ptr %_arg_ReduceAcc.addr
+  store ptr addrspace(1) %_arg_ExScanAcc, ptr %_arg_ExScanAcc.addr
+  store ptr addrspace(1) %_arg_IncScanAcc, ptr %_arg_IncScanAcc.addr
+  store ptr addrspace(1) %_arg_ShiftLeftAcc, ptr %_arg_ShiftLeftAcc.addr
+  store ptr addrspace(1) %_arg_ShiftRightAcc, ptr %_arg_ShiftRightAcc.addr
+  store ptr addrspace(1) %_arg_SelectAcc, ptr %_arg_SelectAcc.addr
+  store ptr addrspace(1) %_arg_PermuteXorAcc, ptr %_arg_PermuteXorAcc.addr
+  %TmpAcc1 = bitcast ptr addrspace(4) %Kernel.ascast to ptr addrspace(4)
+  call spir_func void @Foo1(ptr addrspace(4) %TmpAcc1) 
+  %BarrierAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 1
+  call spir_func void @Foo2(ptr addrspace(4) %BarrierAcc) 
+  %BroadcastAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 2
+  call spir_func void @Foo2(ptr addrspace(4) %BroadcastAcc) 
+  %AnyAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 3
+  call spir_func void @Foo2(ptr addrspace(4) %AnyAcc) 
+  %AllAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 4
+  call spir_func void @Foo2(ptr addrspace(4) %AllAcc) 
+  %NoneAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 5
+  call spir_func void @Foo2(ptr addrspace(4) %NoneAcc) 
+  %ReduceAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 6
+  call spir_func void @Foo2(ptr addrspace(4) %ReduceAcc) 
+  %ExScanAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 7
+  call spir_func void @Foo2(ptr addrspace(4) %ExScanAcc) 
+  %IncScanAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 8
+  call spir_func void @Foo2(ptr addrspace(4) %IncScanAcc) 
+  %ShiftLeftAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 9
+  call spir_func void @Foo2(ptr addrspace(4) %ShiftLeftAcc) 
+  %ShiftRightAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 10
+  call spir_func void @Foo2(ptr addrspace(4) %ShiftRightAcc) 
+  %SelectAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 11
+  call spir_func void @Foo2(ptr addrspace(4) %SelectAcc) 
+  %PermuteXorAcc = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 12
+  call spir_func void @Foo2(ptr addrspace(4) %PermuteXorAcc) 
+  %TmpAcc402 = bitcast ptr addrspace(4) %Kernel.ascast to ptr addrspace(4)
+  %0 = load ptr addrspace(1), ptr %_arg_TmpAcc.addr
+  call spir_func void @Foo3(ptr addrspace(4) %TmpAcc402, ptr addrspace(1) %0, ptr byval(%"range") %agg.tmp, ptr byval(%"range") %agg.tmp41, ptr byval(%"range") %agg.tmp42) 
+  %BarrierAcc43 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 1
+  %1 = load ptr addrspace(1), ptr %_arg_BarrierAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %BarrierAcc43, ptr addrspace(1) %1, ptr byval(%"range") %agg.tmp44, ptr byval(%"range") %agg.tmp45, ptr byval(%"range") %agg.tmp46) 
+  %BroadcastAcc47 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 2
+  %2 = load ptr addrspace(1), ptr %_arg_BroadcastAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %BroadcastAcc47, ptr addrspace(1) %2, ptr byval(%"range") %agg.tmp48, ptr byval(%"range") %agg.tmp49, ptr byval(%"range") %agg.tmp50) 
+  %AnyAcc51 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 3
+  %3 = load ptr addrspace(1), ptr %_arg_AnyAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %AnyAcc51, ptr addrspace(1) %3, ptr byval(%"range") %agg.tmp52, ptr byval(%"range") %agg.tmp53, ptr byval(%"range") %agg.tmp54) 
+  %AllAcc55 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 4
+  %4 = load ptr addrspace(1), ptr %_arg_AllAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %AllAcc55, ptr addrspace(1) %4, ptr byval(%"range") %agg.tmp56, ptr byval(%"range") %agg.tmp57, ptr byval(%"range") %agg.tmp58) 
+  %NoneAcc59 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 5
+  %5 = load ptr addrspace(1), ptr %_arg_NoneAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %NoneAcc59, ptr addrspace(1) %5, ptr byval(%"range") %agg.tmp60, ptr byval(%"range") %agg.tmp61, ptr byval(%"range") %agg.tmp62) 
+  %ReduceAcc63 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 6
+  %6 = load ptr addrspace(1), ptr %_arg_ReduceAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %ReduceAcc63, ptr addrspace(1) %6, ptr byval(%"range") %agg.tmp64, ptr byval(%"range") %agg.tmp65, ptr byval(%"range") %agg.tmp66) 
+  %ExScanAcc67 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 7
+  %7 = load ptr addrspace(1), ptr %_arg_ExScanAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %ExScanAcc67, ptr addrspace(1) %7, ptr byval(%"range") %agg.tmp68, ptr byval(%"range") %agg.tmp69, ptr byval(%"range") %agg.tmp70) 
+  %IncScanAcc71 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 8
+  %8 = load ptr addrspace(1), ptr %_arg_IncScanAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %IncScanAcc71, ptr addrspace(1) %8, ptr byval(%"range") %agg.tmp72, ptr byval(%"range") %agg.tmp73, ptr byval(%"range") %agg.tmp74) 
+  %ShiftLeftAcc75 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 9
+  %9 = load ptr addrspace(1), ptr %_arg_ShiftLeftAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %ShiftLeftAcc75, ptr addrspace(1) %9, ptr byval(%"range") %agg.tmp76, ptr byval(%"range") %agg.tmp77, ptr byval(%"range") %agg.tmp78) 
+  %ShiftRightAcc79 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 10
+  %10 = load ptr addrspace(1), ptr %_arg_ShiftRightAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %ShiftRightAcc79, ptr addrspace(1) %10, ptr byval(%"range") %agg.tmp80, ptr byval(%"range") %agg.tmp81, ptr byval(%"range") %agg.tmp82) 
+  %SelectAcc83 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 11
+  %11 = load ptr addrspace(1), ptr %_arg_SelectAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %SelectAcc83, ptr addrspace(1) %11, ptr byval(%"range") %agg.tmp84, ptr byval(%"range") %agg.tmp85, ptr byval(%"range") %agg.tmp86) 
+  %PermuteXorAcc87 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %Kernel.ascast, i32 0, i32 12
+  %12 = load ptr addrspace(1), ptr %_arg_PermuteXorAcc.addr
+  call spir_func void @Foo4(ptr addrspace(4) %PermuteXorAcc87, ptr addrspace(1) %12, ptr byval(%"range") %agg.tmp88, ptr byval(%"range") %agg.tmp89, ptr byval(%"range") %agg.tmp90) 
+  %call = call spir_func ptr addrspace(4) @Foo5() 
+  call spir_func void @Foo6(ptr addrspace(4) dead_on_unwind writable sret(%"nd_item") align 1 %agg.tmp91.ascast, ptr addrspace(4) %call) 
+  call spir_func void @Foo22(ptr addrspace(4) %Kernel.ascast, ptr byval(%"nd_item") align 1 %agg.tmp91) 
+  ret void
+}
+
+define internal spir_func void @Foo1(ptr addrspace(4) %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"range"
+  %agg.tmp2 = alloca %"range"
+  %agg.tmp3 = alloca %"range"
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  %agg.tmp2.ascast = addrspacecast ptr %agg.tmp2 to ptr addrspace(4)
+  %agg.tmp3.ascast = addrspacecast ptr %agg.tmp3 to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  call void @llvm.memset.p0.i64(ptr %agg.tmp, i8 0, i64 8, i1 false)
+  call spir_func void @Foo11(ptr addrspace(4) %agg.tmp.ascast) 
+  call spir_func void @Foo12(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp2.ascast) 
+  call spir_func void @Foo12(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp3.ascast) 
+  call spir_func void @Foo10(ptr addrspace(4) %impl1, ptr byval(%"range") %agg.tmp, ptr byval(%"range") %agg.tmp2, ptr byval(%"range") %agg.tmp3) 
+  ret void
+}
+
+
+define internal spir_func void @Foo2(ptr addrspace(4) %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"range"
+  %agg.tmp2 = alloca %"range"
+  %agg.tmp3 = alloca %"range"
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  %agg.tmp2.ascast = addrspacecast ptr %agg.tmp2 to ptr addrspace(4)
+  %agg.tmp3.ascast = addrspacecast ptr %agg.tmp3 to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  call void @llvm.memset.p0.i64(ptr %agg.tmp, i8 0, i64 8, i1 false)
+  call spir_func void @Foo11(ptr addrspace(4) %agg.tmp.ascast) 
+  call spir_func void @Foo12(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp2.ascast) 
+  call spir_func void @Foo12(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp3.ascast) 
+  call spir_func void @Foo10(ptr addrspace(4) %impl1, ptr byval(%"range") %agg.tmp, ptr byval(%"range") %agg.tmp2, ptr byval(%"range") %agg.tmp3) 
+  ret void
+}
+
+
+
+
+define internal spir_func void @Foo3(ptr addrspace(4) %this, ptr addrspace(1) %Ptr, ptr byval(%"range") %AccessRange, ptr byval(%"range") %MemRange, ptr byval(%"range") %Offset) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %Ptr.addr = alloca ptr addrspace(1)
+  %ref.tmp = alloca %class.anon.6
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(1) %Ptr, ptr %Ptr.addr
+  %AccessRange.ascast = addrspacecast ptr %AccessRange to ptr addrspace(4)
+  %MemRange.ascast = addrspacecast ptr %MemRange to ptr addrspace(4)
+  %Offset.ascast = addrspacecast ptr %Offset to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load ptr addrspace(1), ptr %Ptr.addr
+  %1 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  store ptr addrspace(1) %0, ptr addrspace(4) %1
+  %2 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %2
+  %Offset2 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %Offset.ascast, ptr %Offset2
+  %AccessRange3 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 2
+  store ptr addrspace(4) %AccessRange.ascast, ptr %AccessRange3
+  %MemRange4 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 3
+  store ptr addrspace(4) %MemRange.ascast, ptr %MemRange4
+  call spir_func void @Foo13(ptr addrspace(4) %ref.tmp.ascast) 
+  %call = call spir_func i64 @Foo21(ptr addrspace(4) %this1) 
+  %3 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  %4 = load ptr addrspace(1), ptr addrspace(4) %3
+  %add.ptr = getelementptr inbounds nuw i64, ptr addrspace(1) %4, i64 %call
+  store ptr addrspace(1) %add.ptr, ptr addrspace(4) %3
+  ret void
+}
+
+
+define internal spir_func void @Foo4(ptr addrspace(4) %this, ptr addrspace(1) %Ptr, ptr byval(%"range") %AccessRange, ptr byval(%"range") %MemRange, ptr byval(%"range") %Offset) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %Ptr.addr = alloca ptr addrspace(1)
+  %ref.tmp = alloca %class.anon.6
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(1) %Ptr, ptr %Ptr.addr
+  %AccessRange.ascast = addrspacecast ptr %AccessRange to ptr addrspace(4)
+  %MemRange.ascast = addrspacecast ptr %MemRange to ptr addrspace(4)
+  %Offset.ascast = addrspacecast ptr %Offset to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load ptr addrspace(1), ptr %Ptr.addr
+  %1 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  store ptr addrspace(1) %0, ptr addrspace(4) %1
+  %2 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %2
+  %Offset2 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %Offset.ascast, ptr %Offset2
+  %AccessRange3 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 2
+  store ptr addrspace(4) %AccessRange.ascast, ptr %AccessRange3
+  %MemRange4 = getelementptr inbounds %class.anon.6, ptr %ref.tmp, i32 0, i32 3
+  store ptr addrspace(4) %MemRange.ascast, ptr %MemRange4
+  call spir_func void @Foo30(ptr addrspace(4) %ref.tmp.ascast) 
+  %call = call spir_func i64 @Foo32(ptr addrspace(4) %this1) 
+  %3 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  %4 = load ptr addrspace(1), ptr addrspace(4) %3
+  %add.ptr = getelementptr inbounds nuw i8, ptr addrspace(1) %4, i64 %call
+  store ptr addrspace(1) %add.ptr, ptr addrspace(4) %3
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo5() {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  ret ptr addrspace(4) null
+}
+
+
+define internal spir_func void @Foo6(ptr addrspace(4) dead_on_unwind noalias writable sret(%"nd_item") align 1 %agg.result, ptr addrspace(4) %0) {
+entry:
+  %.addr = alloca ptr addrspace(4)
+  %GlobalSize = alloca %"range"
+  %LocalSize = alloca %"range"
+  %GroupRange = alloca %"range"
+  %GroupId = alloca %"range"
+  %GlobalId = alloca %"range"
+  %LocalId = alloca %"range"
+  %GlobalOffset = alloca %"range"
+  %Group = alloca %"group"
+  %GlobalItem = alloca %"item"
+  %LocalItem = alloca %"item.22"
+  %cleanup.dest.slot = alloca i32, align 4
+  %GlobalSize.ascast = addrspacecast ptr %GlobalSize to ptr addrspace(4)
+  %LocalSize.ascast = addrspacecast ptr %LocalSize to ptr addrspace(4)
+  %GroupRange.ascast = addrspacecast ptr %GroupRange to ptr addrspace(4)
+  %GroupId.ascast = addrspacecast ptr %GroupId to ptr addrspace(4)
+  %GlobalId.ascast = addrspacecast ptr %GlobalId to ptr addrspace(4)
+  %LocalId.ascast = addrspacecast ptr %LocalId to ptr addrspace(4)
+  %GlobalOffset.ascast = addrspacecast ptr %GlobalOffset to ptr addrspace(4)
+  %Group.ascast = addrspacecast ptr %Group to ptr addrspace(4)
+  %GlobalItem.ascast = addrspacecast ptr %GlobalItem to ptr addrspace(4)
+  %LocalItem.ascast = addrspacecast ptr %LocalItem to ptr addrspace(4)
+  store ptr addrspace(4) %0, ptr %.addr
+  call spir_func void @Foo7(ptr addrspace(4) dead_on_unwind writable sret(%"range") %GlobalSize.ascast) 
+  call spir_func void @Init1(ptr addrspace(4) dead_on_unwind writable sret(%"range") %LocalSize.ascast) 
+  call spir_func void @Init2(ptr addrspace(4) dead_on_unwind writable sret(%"range") %GroupRange.ascast) 
+  call spir_func void @Init3(ptr addrspace(4) dead_on_unwind writable sret(%"range") %GroupId.ascast) 
+  call spir_func void @Init6(ptr addrspace(4) dead_on_unwind writable sret(%"range") %GlobalId.ascast) 
+  call spir_func void @Init4(ptr addrspace(4) dead_on_unwind writable sret(%"range") %LocalId.ascast) 
+  call spir_func void @Init5(ptr addrspace(4) dead_on_unwind writable sret(%"range") %GlobalOffset.ascast) 
+  call spir_func void @Foo23(ptr addrspace(4) dead_on_unwind writable sret(%"group") %Group.ascast, ptr addrspace(4) %GlobalSize.ascast, ptr addrspace(4) %LocalSize.ascast, ptr addrspace(4) %GroupRange.ascast, ptr addrspace(4) %GroupId.ascast) 
+  call spir_func void @Foo24(ptr addrspace(4) dead_on_unwind writable sret(%"item") %GlobalItem.ascast, ptr addrspace(4) %GlobalSize.ascast, ptr addrspace(4) %GlobalId.ascast, ptr addrspace(4) %GlobalOffset.ascast) 
+  call spir_func void @Foo25(ptr addrspace(4) dead_on_unwind writable sret(%"item.22") %LocalItem.ascast, ptr addrspace(4) %LocalSize.ascast, ptr addrspace(4) %LocalId.ascast) 
+  call spir_func void @Foo26(ptr addrspace(4) dead_on_unwind writable sret(%"nd_item") align 1 %agg.result, ptr addrspace(4) %GlobalItem.ascast, ptr addrspace(4) %LocalItem.ascast, ptr addrspace(4) %Group.ascast) 
+  ret void
+}
+
+
+define internal spir_func void @Foo22(ptr addrspace(4) %this, ptr byval(%"nd_item") align 1 %item) {
+entry:
+  %this.addr.i76 = alloca ptr addrspace(4)
+  %WI.addr.i = alloca i64
+  %TangleLeader.addr.i = alloca i64
+  %TangleSize.addr.i = alloca i64
+  %agg.tmp.i = alloca %"range"
+  %agg.tmp2.i = alloca %"tangle_group"
+  %Visible.i = alloca i64
+  %Other.i = alloca i64
+  %agg.tmp5.i = alloca %"range"
+  %agg.tmp8.i = alloca %"range"
+  %OriginalLID.i = alloca i32, align 4
+  %LID.i = alloca i32, align 4
+  %BroadcastResult.i = alloca i32, align 4
+  %agg.tmp12.i = alloca %"tangle_group"
+  %agg.tmp15.i = alloca %"range"
+  %AnyResult.i = alloca i8, align 1
+  %agg.tmp18.i = alloca %"tangle_group"
+  %agg.tmp24.i = alloca %"range"
+  %AllResult.i = alloca i8, align 1
+  %agg.tmp27.i = alloca %"tangle_group"
+  %agg.tmp35.i = alloca %"range"
+  %NoneResult.i = alloca i8, align 1
+  %agg.tmp38.i = alloca %"tangle_group"
+  %agg.tmp46.i = alloca %"range"
+  %ReduceResult.i = alloca i32, align 4
+  %agg.tmp49.i = alloca %"tangle_group"
+  %agg.tmp50.i = alloca %"nd_item", align 1
+  %agg.tmp54.i = alloca %"range"
+  %ExScanResult.i = alloca i32, align 4
+  %agg.tmp57.i = alloca %"tangle_group"
+  %agg.tmp58.i = alloca %"nd_item", align 1
+  %agg.tmp61.i = alloca %"range"
+  %IncScanResult.i = alloca i32, align 4
+  %agg.tmp64.i = alloca %"tangle_group"
+  %agg.tmp65.i = alloca %"nd_item", align 1
+  %agg.tmp69.i = alloca %"range"
+  %ShiftLeftResult.i = alloca i32, align 4
+  %agg.tmp72.i = alloca %"tangle_group"
+  %agg.tmp79.i = alloca %"range"
+  %ShiftRightResult.i = alloca i32, align 4
+  %agg.tmp82.i = alloca %"tangle_group"
+  %agg.tmp88.i = alloca %"range"
+  %SelectResult.i = alloca i32, align 4
+  %agg.tmp91.i = alloca %"tangle_group"
+  %agg.tmp92.i = alloca %"range"
+  %ref.tmp.i = alloca %"range"
+  %ref.tmp93.i = alloca %"range"
+  %ref.tmp94.i = alloca i32, align 4
+  %agg.tmp100.i = alloca %"range"
+  %PermuteXorResult.i = alloca i32, align 4
+  %agg.tmp103.i = alloca %"tangle_group"
+  %agg.tmp106.i = alloca %"range"
+  %agg.tmp18.ascast.ascast75 = alloca %"nd_item"
+  %agg.tmp17.ascast.ascast74 = alloca %"tangle_group"
+  %retval.i66 = alloca i64
+  %this.addr.i67 = alloca ptr addrspace(4)
+  %Result.i68 = alloca i64
+  %retval.i58 = alloca i64
+  %this.addr.i59 = alloca ptr addrspace(4)
+  %Result.i60 = alloca i64
+  %retval.i50 = alloca i64
+  %this.addr.i51 = alloca ptr addrspace(4)
+  %Result.i52 = alloca i64
+  %retval.i42 = alloca i64
+  %this.addr.i43 = alloca ptr addrspace(4)
+  %Result.i44 = alloca i64
+  %retval.i = alloca i64
+  %this.addr.i = alloca ptr addrspace(4)
+  %Result.i = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %WI = alloca %"range"
+  %SG = alloca %"nd_item", align 1
+  %BranchBody = alloca %class.anon.8
+  %ref.tmp = alloca %"range"
+  %ref.tmp15 = alloca i32, align 4
+  %Tangle = alloca %"tangle_group"
+  %agg.tmp = alloca %"nd_item", align 1
+  %TangleLeader = alloca i64
+  %TangleSize = alloca i64
+  %IsMember = alloca %"nd_item", align 1
+  %agg.tmp17 = alloca %"tangle_group"
+  %agg.tmp18 = alloca %"nd_item", align 1
+  %ref.tmp19 = alloca %"range"
+  %ref.tmp20 = alloca i32, align 4
+  %Tangle24 = alloca %"tangle_group"
+  %agg.tmp25 = alloca %"nd_item", align 1
+  %TangleLeader26 = alloca i64
+  %TangleSize27 = alloca i64
+  %IsMember28 = alloca %"nd_item", align 1
+  %agg.tmp30 = alloca %"tangle_group"
+  %agg.tmp31 = alloca %"nd_item", align 1
+  %Tangle33 = alloca %"tangle_group"
+  %agg.tmp34 = alloca %"nd_item", align 1
+  %TangleLeader35 = alloca i64
+  %TangleSize36 = alloca i64
+  %IsMember37 = alloca %"nd_item", align 1
+  %agg.tmp39 = alloca %"tangle_group"
+  %agg.tmp40 = alloca %"nd_item", align 1
+  %WI.ascast = addrspacecast ptr %WI to ptr addrspace(4)
+  %SG.ascast = addrspacecast ptr %SG to ptr addrspace(4)
+  %BranchBody.ascast = addrspacecast ptr %BranchBody to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %ref.tmp15.ascast = addrspacecast ptr %ref.tmp15 to ptr addrspace(4)
+  %Tangle.ascast = addrspacecast ptr %Tangle to ptr addrspace(4)
+  %IsMember.ascast = addrspacecast ptr %IsMember to ptr addrspace(4)
+  %ref.tmp19.ascast = addrspacecast ptr %ref.tmp19 to ptr addrspace(4)
+  %ref.tmp20.ascast = addrspacecast ptr %ref.tmp20 to ptr addrspace(4)
+  %Tangle24.ascast = addrspacecast ptr %Tangle24 to ptr addrspace(4)
+  %IsMember28.ascast = addrspacecast ptr %IsMember28 to ptr addrspace(4)
+  %Tangle33.ascast = addrspacecast ptr %Tangle33 to ptr addrspace(4)
+  %IsMember37.ascast = addrspacecast ptr %IsMember37 to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %item.ascast = addrspacecast ptr %item to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Foo40(ptr addrspace(4) dead_on_unwind writable sret(%"range") %WI.ascast, ptr addrspace(4) align 1 %item.ascast) 
+  call spir_func void @Foo41(ptr addrspace(4) dead_on_unwind writable sret(%"nd_item") align 1 %SG.ascast, ptr addrspace(4) align 1 %item.ascast) 
+  %TmpAcc1 = bitcast ptr %BranchBody to ptr
+  %TmpAcc22 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %BarrierAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 1
+  %BarrierAcc3 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 1
+  %0 = getelementptr inbounds i8, ptr addrspace(4) %BranchBody.ascast, i64 64
+  %BroadcastAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 3
+  %BroadcastAcc4 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 2
+  %AnyAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 4
+  %AnyAcc5 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 3
+  %AllAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 5
+  %AllAcc6 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 4
+  %NoneAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 6
+  %NoneAcc7 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 5
+  %ReduceAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 7
+  %ReduceAcc8 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 6
+  %ExScanAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 8
+  %ExScanAcc9 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 7
+  %IncScanAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 9
+  %IncScanAcc10 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 8
+  %ShiftLeftAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 10
+  %ShiftLeftAcc11 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 9
+  %ShiftRightAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 11
+  %ShiftRightAcc12 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 10
+  %SelectAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 12
+  %SelectAcc13 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 11
+  %PermuteXorAcc = getelementptr inbounds %class.anon.8, ptr %BranchBody, i32 0, i32 13
+  %PermuteXorAcc14 = getelementptr inbounds nuw %class.anon, ptr addrspace(4) %this1, i32 0, i32 12
+  store i32 4, ptr %ref.tmp15, align 4
+  call spir_func void @Foo42(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %WI.ascast, ptr addrspace(4) align 4 %ref.tmp15.ascast) 
+  %retval.ascast.i69 = addrspacecast ptr %retval.i66 to ptr addrspace(4)
+  store ptr addrspace(4) %ref.tmp.ascast, ptr %this.addr.i67
+  %this1.i72 = load ptr addrspace(4), ptr %this.addr.i67
+  %1 = load i64, ptr addrspace(4) %this1.i72
+  store i64 %1, ptr %Result.i68
+  %2 = load i64, ptr %Result.i68
+  %tobool = icmp ne i64 %2, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.else:                                          ; preds = %entry
+  store i32 24, ptr %ref.tmp20, align 4
+  call spir_func void @Foo42(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp19.ascast, ptr addrspace(4) %WI.ascast, ptr addrspace(4) align 4 %ref.tmp20.ascast) 
+  %retval.ascast.i53 = addrspacecast ptr %retval.i50 to ptr addrspace(4)
+  store ptr addrspace(4) %ref.tmp19.ascast, ptr %this.addr.i51
+  %this1.i56 = load ptr addrspace(4), ptr %this.addr.i51
+  %3 = load i64, ptr addrspace(4) %this1.i56
+  store i64 %3, ptr %Result.i52
+  %4 = load i64, ptr %Result.i52
+  %tobool22 = icmp ne i64 %4, 0
+  br i1 %tobool22, label %if.then23, label %if.else32
+
+if.else32:                                        ; preds = %if.else
+  call spir_func void @Foo43(ptr addrspace(4) dead_on_unwind writable sret(%"tangle_group") %Tangle33.ascast, ptr byval(%"nd_item") align 1 %agg.tmp34) 
+  store i64 24, ptr %TangleLeader35
+  store i64 8, ptr %TangleSize36
+  %retval.ascast.i = addrspacecast ptr %retval.i to ptr addrspace(4)
+  store ptr addrspace(4) %WI.ascast, ptr %this.addr.i
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i
+  %5 = load i64, ptr addrspace(4) %this1.i
+  store i64 %5, ptr %Result.i
+  %6 = load i64, ptr %Result.i
+  %7 = load i64, ptr %TangleLeader35
+  %8 = load i64, ptr %TangleSize36
+  call spir_func void @Foo69(ptr addrspace(4) %BranchBody.ascast, i64 %6, ptr byval(%"tangle_group") %agg.tmp39, i64 %7, i64 %8, ptr byval(%"nd_item") align 1 %agg.tmp40) 
+  br label %if.end41
+
+if.then23:                                        ; preds = %if.else
+  call spir_func void @Foo43(ptr addrspace(4) dead_on_unwind writable sret(%"tangle_group") %Tangle24.ascast, ptr byval(%"nd_item") align 1 %agg.tmp25) 
+  store i64 4, ptr %TangleLeader26
+  store i64 20, ptr %TangleSize27
+  %retval.ascast.i45 = addrspacecast ptr %retval.i42 to ptr addrspace(4)
+  store ptr addrspace(4) %WI.ascast, ptr %this.addr.i43
+  %this1.i48 = load ptr addrspace(4), ptr %this.addr.i43
+  %9 = load i64, ptr addrspace(4) %this1.i48
+  store i64 %9, ptr %Result.i44
+  %10 = load i64, ptr %Result.i44
+  %11 = load i64, ptr %TangleLeader26
+  %12 = load i64, ptr %TangleSize27
+  call spir_func void @Foo68(ptr addrspace(4) %BranchBody.ascast, i64 %10, ptr byval(%"tangle_group") %agg.tmp30, i64 %11, i64 %12, ptr byval(%"nd_item") align 1 %agg.tmp31) 
+  br label %if.end41
+
+if.then:                                          ; preds = %entry
+  call spir_func void @Foo43(ptr addrspace(4) dead_on_unwind writable sret(%"tangle_group") %Tangle.ascast, ptr byval(%"nd_item") align 1 %agg.tmp) 
+  store i64 0, ptr %TangleLeader
+  store i64 4, ptr %TangleSize
+  %retval.ascast.i61 = addrspacecast ptr %retval.i58 to ptr addrspace(4)
+  store ptr addrspace(4) %WI.ascast, ptr %this.addr.i59
+  %this1.i64 = load ptr addrspace(4), ptr %this.addr.i59
+  %13 = load i64, ptr addrspace(4) %this1.i64
+  store i64 %13, ptr %Result.i60
+  %14 = load i64, ptr %Result.i60
+  %15 = load i64, ptr %TangleLeader
+  %16 = load i64, ptr %TangleSize
+  %TangleSize.addr.ascast.i = addrspacecast ptr %TangleSize.addr.i to ptr addrspace(4)
+  %agg.tmp.ascast.i = addrspacecast ptr %agg.tmp.i to ptr addrspace(4)
+  %agg.tmp5.ascast.i = addrspacecast ptr %agg.tmp5.i to ptr addrspace(4)
+  %agg.tmp8.ascast.i = addrspacecast ptr %agg.tmp8.i to ptr addrspace(4)
+  %agg.tmp15.ascast.i = addrspacecast ptr %agg.tmp15.i to ptr addrspace(4)
+  %agg.tmp24.ascast.i = addrspacecast ptr %agg.tmp24.i to ptr addrspace(4)
+  %agg.tmp35.ascast.i = addrspacecast ptr %agg.tmp35.i to ptr addrspace(4)
+  %agg.tmp46.ascast.i = addrspacecast ptr %agg.tmp46.i to ptr addrspace(4)
+  %agg.tmp50.ascast.i = addrspacecast ptr %agg.tmp50.i to ptr addrspace(4)
+  %agg.tmp54.ascast.i = addrspacecast ptr %agg.tmp54.i to ptr addrspace(4)
+  %agg.tmp58.ascast.i = addrspacecast ptr %agg.tmp58.i to ptr addrspace(4)
+  %agg.tmp61.ascast.i = addrspacecast ptr %agg.tmp61.i to ptr addrspace(4)
+  %agg.tmp65.ascast.i = addrspacecast ptr %agg.tmp65.i to ptr addrspace(4)
+  %agg.tmp69.ascast.i = addrspacecast ptr %agg.tmp69.i to ptr addrspace(4)
+  %agg.tmp79.ascast.i = addrspacecast ptr %agg.tmp79.i to ptr addrspace(4)
+  %agg.tmp88.ascast.i = addrspacecast ptr %agg.tmp88.i to ptr addrspace(4)
+  %agg.tmp92.ascast.i = addrspacecast ptr %agg.tmp92.i to ptr addrspace(4)
+  %ref.tmp.ascast.i = addrspacecast ptr %ref.tmp.i to ptr addrspace(4)
+  %ref.tmp93.ascast.i = addrspacecast ptr %ref.tmp93.i to ptr addrspace(4)
+  %ref.tmp94.ascast.i = addrspacecast ptr %ref.tmp94.i to ptr addrspace(4)
+  %agg.tmp100.ascast.i = addrspacecast ptr %agg.tmp100.i to ptr addrspace(4)
+  %agg.tmp106.ascast.i = addrspacecast ptr %agg.tmp106.i to ptr addrspace(4)
+  store ptr addrspace(4) %BranchBody.ascast, ptr %this.addr.i76
+  store i64 %14, ptr %WI.addr.i
+  %Tangle.ascast.i = addrspacecast ptr %agg.tmp17.ascast.ascast74 to ptr addrspace(4)
+  store i64 %15, ptr %TangleLeader.addr.i
+  store i64 %16, ptr %TangleSize.addr.i
+  %IsMember.ascast.i = addrspacecast ptr %agg.tmp18.ascast.ascast75 to ptr addrspace(4)
+  %this1.i78 = load ptr addrspace(4), ptr %this.addr.i76
+  %17 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp.ascast.i, i64 %17) 
+  %call.i = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %this1.i78, ptr byval(%"range") %agg.tmp.i) 
+  store i64 1, ptr addrspace(4) %call.i
+  call spir_func void @Foo75(ptr byval(%"tangle_group") %agg.tmp2.i, i32 1) 
+  store i64 0, ptr %Visible.i
+  store i64 0, ptr %Other.i
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %if.end.i, %if.then
+  %18 = load i64, ptr %Other.i
+  %cmp.i79 = icmp ult i64 %18, 32
+  br i1 %cmp.i79, label %for.body.i, label %for.cond.cleanup.i
+
+for.cond.cleanup.i:                               ; preds = %for.cond.i
+  %19 = load i64, ptr %Visible.i
+  %20 = load i64, ptr %TangleSize.addr.i
+  %cmp7.i = icmp eq i64 %19, %20
+  %BarrierAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 1
+  %21 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp8.ascast.i, i64 %21) 
+  %call9.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BarrierAcc.i, ptr byval(%"range") %agg.tmp8.i) 
+  %storedv.i = zext i1 %cmp7.i to i8
+  store i8 %storedv.i, ptr addrspace(4) %call9.i, align 1
+  %22 = getelementptr inbounds i8, ptr addrspace(4) %this1.i78, i64 64
+  %call10.i = call spir_func i32 @Foo76(ptr addrspace(4) align 1 %22) 
+  store i32 %call10.i, ptr %OriginalLID.i, align 4
+  %call11.i = call spir_func i32 @Foo90(ptr addrspace(4) %Tangle.ascast.i) 
+  store i32 %call11.i, ptr %LID.i, align 4
+  %23 = load i32, ptr %OriginalLID.i, align 4
+  %call13.i = call spir_func i32 @Foo91(ptr byval(%"tangle_group") %agg.tmp12.i, i32 %23, i32 0) 
+  store i32 %call13.i, ptr %BroadcastResult.i, align 4
+  %24 = load i32, ptr %BroadcastResult.i, align 4
+  %conv.i = zext i32 %24 to i64
+  %25 = load i64, ptr %TangleLeader.addr.i
+  %cmp14.i = icmp eq i64 %conv.i, %25
+  %BroadcastAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 3
+  %26 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp15.ascast.i, i64 %26) 
+  %call16.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BroadcastAcc.i, ptr byval(%"range") %agg.tmp15.i) 
+  %storedv17.i = zext i1 %cmp14.i to i8
+  store i8 %storedv17.i, ptr addrspace(4) %call16.i, align 1
+  %27 = load i32, ptr %LID.i, align 4
+  %cmp19.i = icmp eq i32 %27, 0
+  %call20.i = call spir_func zeroext i1 @Foo92(ptr byval(%"tangle_group") %agg.tmp18.i, i1 zeroext %cmp19.i) 
+  %storedv21.i = zext i1 %call20.i to i8
+  store i8 %storedv21.i, ptr %AnyResult.i, align 1
+  %28 = load i8, ptr %AnyResult.i, align 1  
+  %loadedv.i = trunc i8 %28 to i1
+  %conv22.i = zext i1 %loadedv.i to i32
+  %AnyAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 4
+  %29 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp24.ascast.i, i64 %29) 
+  %call25.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AnyAcc.i, ptr byval(%"range") %agg.tmp24.i) 
+  %storedv26.i = zext i1 %loadedv.i to i8
+  store i8 %storedv26.i, ptr addrspace(4) %call25.i, align 1
+  %30 = load i32, ptr %LID.i, align 4
+  %conv28.i = zext i32 %30 to i64
+  %31 = load i64, ptr %TangleSize.addr.i
+  %cmp29.i = icmp ult i64 %conv28.i, %31
+  %call30.i = call spir_func zeroext i1 @Foo67(ptr byval(%"tangle_group") %agg.tmp27.i, i1 zeroext %cmp29.i) 
+  %storedv31.i = zext i1 %call30.i to i8
+  store i8 %storedv31.i, ptr %AllResult.i, align 1
+  %32 = load i8, ptr %AllResult.i, align 1  
+  %loadedv32.i = trunc i8 %32 to i1
+  %conv33.i = zext i1 %loadedv32.i to i32
+  %AllAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 5
+  %33 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp35.ascast.i, i64 %33) 
+  %call36.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AllAcc.i, ptr byval(%"range") %agg.tmp35.i) 
+  %storedv37.i = zext i1 %loadedv32.i to i8
+  store i8 %storedv37.i, ptr addrspace(4) %call36.i, align 1
+  %34 = load i32, ptr %LID.i, align 4
+  %conv39.i = zext i32 %34 to i64
+  %35 = load i64, ptr %TangleSize.addr.i
+  %cmp40.i = icmp uge i64 %conv39.i, %35
+  %call41.i = call spir_func zeroext i1 @Foo65(ptr byval(%"tangle_group") %agg.tmp38.i, i1 zeroext %cmp40.i) 
+  %storedv42.i = zext i1 %call41.i to i8
+  store i8 %storedv42.i, ptr %NoneResult.i, align 1
+  %36 = load i8, ptr %NoneResult.i, align 1  
+  %loadedv43.i = trunc i8 %36 to i1
+  %conv44.i = zext i1 %loadedv43.i to i32
+  %NoneAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 6
+  %37 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp46.ascast.i, i64 %37) 
+  %call47.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %NoneAcc.i, ptr byval(%"range") %agg.tmp46.i) 
+  %storedv48.i = zext i1 %loadedv43.i to i8
+  store i8 %storedv48.i, ptr addrspace(4) %call47.i, align 1
+  %call51.i = call spir_func i32 @Foo64(ptr byval(%"tangle_group") %agg.tmp49.i, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp50.i) 
+  store i32 %call51.i, ptr %ReduceResult.i, align 4
+  %38 = load i32, ptr %ReduceResult.i, align 4
+  %conv52.i = zext i32 %38 to i64
+  %39 = load i64, ptr %TangleSize.addr.i
+  %cmp53.i = icmp eq i64 %conv52.i, %39
+  %ReduceAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 7
+  %40 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp54.ascast.i, i64 %40) 
+  %call55.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ReduceAcc.i, ptr byval(%"range") %agg.tmp54.i) 
+  %storedv56.i = zext i1 %cmp53.i to i8
+  store i8 %storedv56.i, ptr addrspace(4) %call55.i, align 1
+  %call59.i = call spir_func i32 @Foo63(ptr byval(%"tangle_group") %agg.tmp57.i, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp58.i) 
+  store i32 %call59.i, ptr %ExScanResult.i, align 4
+  %41 = load i32, ptr %ExScanResult.i, align 4
+  %42 = load i32, ptr %LID.i, align 4
+  %cmp60.i = icmp eq i32 %41, %42
+  %ExScanAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 8
+  %43 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp61.ascast.i, i64 %43) 
+  %call62.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ExScanAcc.i, ptr byval(%"range") %agg.tmp61.i) 
+  %storedv63.i = zext i1 %cmp60.i to i8
+  store i8 %storedv63.i, ptr addrspace(4) %call62.i, align 1
+  %call66.i = call spir_func i32 @Foo62(ptr byval(%"tangle_group") %agg.tmp64.i, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp65.i) 
+  store i32 %call66.i, ptr %IncScanResult.i, align 4
+  %44 = load i32, ptr %IncScanResult.i, align 4
+  %45 = load i32, ptr %LID.i, align 4
+  %add67.i = add i32 %45, 1
+  %cmp68.i = icmp eq i32 %44, %add67.i
+  %IncScanAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 9
+  %46 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp69.ascast.i, i64 %46) 
+  %call70.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %IncScanAcc.i, ptr byval(%"range") %agg.tmp69.i) 
+  %storedv71.i = zext i1 %cmp68.i to i8
+  store i8 %storedv71.i, ptr addrspace(4) %call70.i, align 1
+  %47 = load i32, ptr %LID.i, align 4
+  %call73.i = call spir_func i32 @Foo73(ptr byval(%"tangle_group") %agg.tmp72.i, i32 %47, i32 2) 
+  store i32 %call73.i, ptr %ShiftLeftResult.i, align 4
+  %48 = load i32, ptr %LID.i, align 4
+  %add74.i = add i32 %48, 2
+  %conv75.i = zext i32 %add74.i to i64
+  %49 = load i64, ptr %TangleSize.addr.i
+  %cmp76.i = icmp uge i64 %conv75.i, %49
+  br i1 %cmp76.i, label %lor.end.i, label %lor.rhs.i
+
+lor.rhs.i:                                        ; preds = %for.cond.cleanup.i
+  %50 = load i32, ptr %ShiftLeftResult.i, align 4
+  %51 = load i32, ptr %LID.i, align 4
+  %add77.i = add i32 %51, 2
+  %cmp78.i = icmp eq i32 %50, %add77.i
+  br label %lor.end.i
+
+lor.end.i:                                        ; preds = %lor.rhs.i, %for.cond.cleanup.i
+  %52 = phi i1 [ true, %for.cond.cleanup.i ], [ %cmp78.i, %lor.rhs.i ]
+  %ShiftLeftAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 10
+  %53 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp79.ascast.i, i64 %53) 
+  %call80.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftLeftAcc.i, ptr byval(%"range") %agg.tmp79.i) 
+  %storedv81.i = zext i1 %52 to i8
+  store i8 %storedv81.i, ptr addrspace(4) %call80.i, align 1
+  %54 = load i32, ptr %LID.i, align 4
+  %call83.i = call spir_func i32 @Foo53(ptr byval(%"tangle_group") %agg.tmp82.i, i32 %54, i32 2) 
+  store i32 %call83.i, ptr %ShiftRightResult.i, align 4
+  %55 = load i32, ptr %LID.i, align 4
+  %cmp84.i = icmp ult i32 %55, 2
+  br i1 %cmp84.i, label %l1.exit, label %lor.rhs85.i
+
+lor.rhs85.i:                                      ; preds = %lor.end.i
+  %56 = load i32, ptr %ShiftRightResult.i, align 4
+  %57 = load i32, ptr %LID.i, align 4
+  %sub.i = sub i32 %57, 2
+  %cmp86.i = icmp eq i32 %56, %sub.i
+  br label %l1.exit
+
+l1.exit: ; preds = %lor.rhs85.i, %lor.end.i
+  %58 = phi i1 [ true, %lor.end.i ], [ %cmp86.i, %lor.rhs85.i ]
+  %ShiftRightAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 11
+  %59 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp88.ascast.i, i64 %59) 
+  %call89.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftRightAcc.i, ptr byval(%"range") %agg.tmp88.i) 
+  %storedv90.i = zext i1 %58 to i8
+  store i8 %storedv90.i, ptr addrspace(4) %call89.i, align 1
+  %60 = load i32, ptr %LID.i, align 4
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp93.ascast.i, ptr addrspace(4) %Tangle.ascast.i) 
+  store i32 2, ptr %ref.tmp94.i, align 4
+  call spir_func void @Foo55(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast.i, ptr addrspace(4) %ref.tmp93.ascast.i, ptr addrspace(4) align 4 %ref.tmp94.ascast.i) 
+  call spir_func void @Foo56(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp92.ascast.i, ptr addrspace(4) %ref.tmp.ascast.i, ptr addrspace(4) %TangleSize.addr.ascast.i) 
+  %call95.i = call spir_func i32 @Foo57(ptr byval(%"tangle_group") %agg.tmp91.i, i32 %60, ptr byval(%"range") %agg.tmp92.i) 
+  store i32 %call95.i, ptr %SelectResult.i, align 4
+  %61 = load i32, ptr %SelectResult.i, align 4
+  %conv96.i = zext i32 %61 to i64
+  %62 = load i32, ptr %LID.i, align 4
+  %add97.i = add i32 %62, 2
+  %conv98.i = zext i32 %add97.i to i64
+  %63 = load i64, ptr %TangleSize.addr.i
+  %rem.i = urem i64 %conv98.i, %63
+  %cmp99.i = icmp eq i64 %conv96.i, %rem.i
+  %SelectAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 12
+  %64 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp100.ascast.i, i64 %64) 
+  %call101.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %SelectAcc.i, ptr byval(%"range") %agg.tmp100.i) 
+  %storedv102.i = zext i1 %cmp99.i to i8
+  store i8 %storedv102.i, ptr addrspace(4) %call101.i, align 1
+  %65 = load i32, ptr %LID.i, align 4
+  %call104.i = call spir_func i32 @Foo58(ptr byval(%"tangle_group") %agg.tmp103.i, i32 %65, i32 2) 
+  store i32 %call104.i, ptr %PermuteXorResult.i, align 4
+  %66 = load i32, ptr %PermuteXorResult.i, align 4
+  %67 = load i32, ptr %LID.i, align 4
+  %xor.i = xor i32 %67, 2
+  %cmp105.i = icmp eq i32 %66, %xor.i
+  %PermuteXorAcc.i = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1.i78, i32 0, i32 13
+  %68 = load i64, ptr %WI.addr.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp106.ascast.i, i64 %68) 
+  %call107.i = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %PermuteXorAcc.i, ptr byval(%"range") %agg.tmp106.i) 
+  %storedv108.i = zext i1 %cmp105.i to i8
+  store i8 %storedv108.i, ptr addrspace(4) %call107.i, align 1
+  br label %if.end41
+
+if.end41:                                         ; preds = %if.then23, %if.else32, %l1.exit
+  ret void
+
+for.body.i:                                       ; preds = %for.cond.i
+  %69 = load i64, ptr %Other.i
+  %call3.i = call spir_func zeroext i1 @Foo71(ptr addrspace(4) align 1 %IsMember.ascast.i, i64 %69) 
+  br i1 %call3.i, label %if.then.i, label %if.end.i
+
+if.then.i:                                        ; preds = %for.body.i
+  %70 = load i64, ptr %Other.i
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp5.ascast.i, i64 %70) 
+  %call6.i = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %this1.i78, ptr byval(%"range") %agg.tmp5.i) 
+  %71 = load i64, ptr addrspace(4) %call6.i
+  %72 = load i64, ptr %Visible.i
+  %add.i = add i64 %72, %71
+  store i64 %add.i, ptr %Visible.i
+  br label %if.end.i
+
+if.end.i:                                         ; preds = %if.then.i, %for.body.i
+  %73 = load i64, ptr %Other.i
+  %inc.i = add i64 %73, 1
+  store i64 %inc.i, ptr %Other.i
+  br label %for.cond.i
+}
+
+define internal spir_func void @Foo40(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) align 1 %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Init6(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+define internal spir_func void @Foo41(ptr addrspace(4) dead_on_unwind noalias writable sret(%"nd_item") align 1 %agg.result, ptr addrspace(4) align 1 %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  ret void
+}
+
+
+
+
+define internal spir_func void @Foo42(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %lhs, ptr addrspace(4) align 4 %rhs) {
+entry:
+  %lhs.addr = alloca ptr addrspace(4)
+  %rhs.addr = alloca ptr addrspace(4)
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  store ptr addrspace(4) %lhs, ptr %lhs.addr
+  store ptr addrspace(4) %rhs, ptr %rhs.addr
+  call spir_func void @Foo11(ptr addrspace(4) %agg.result) 
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %1 = load ptr addrspace(4), ptr %lhs.addr
+  %common_array1 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %2 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array1, i64 0, i64 %idxprom
+  %3 = load i64, ptr addrspace(4) %arrayidx
+  %4 = load ptr addrspace(4), ptr %rhs.addr
+  %5 = load i32, ptr addrspace(4) %4, align 4
+  %conv = sext i32 %5 to i64
+  %cmp1 = icmp ult i64 %3, %conv
+  %conv2 = zext i1 %cmp1 to i64
+  %common_array32 = bitcast ptr addrspace(4) %agg.result to ptr addrspace(4)
+  %6 = load i32, ptr %i, align 4
+  %idxprom4 = sext i32 %6 to i64
+  %arrayidx5 = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array32, i64 0, i64 %idxprom4
+  store i64 %conv2, ptr addrspace(4) %arrayidx5
+  %7 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+}
+
+declare void @llvm.assume(i1) 
+
+
+define internal spir_func void @Foo43(ptr addrspace(4) dead_on_unwind noalias writable sret(%"tangle_group") %agg.result, ptr byval(%"nd_item") align 1 %group) {
+entry:
+  %mask = alloca %"ss_sub_group_mask"
+  %agg.tmp = alloca %"nd_item", align 1
+  %agg.tmp1 = alloca %"ss_sub_group_mask"
+  %cleanup.dest.slot = alloca i32, align 4
+  %mask.ascast = addrspacecast ptr %mask to ptr addrspace(4)
+  %group.ascast = addrspacecast ptr %group to ptr addrspace(4)
+  call spir_func void @Foo44(ptr addrspace(4) dead_on_unwind writable sret(%"ss_sub_group_mask") %mask.ascast, ptr byval(%"nd_item") align 1 %agg.tmp, i1 zeroext true) 
+  call spir_func void @Foo45(ptr addrspace(4) %agg.result, ptr byval(%"ss_sub_group_mask") %agg.tmp1) 
+  ret void
+}
+
+
+define internal spir_func void @Foo46(ptr addrspace(4) %this, i64 %dim0) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %dim0.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %dim0, ptr %dim0.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %dim0.addr
+  call spir_func void @Foo60(ptr addrspace(4) %this1, i64 %0) 
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %this, ptr byval(%"range") %Index) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %LinearIndex = alloca i64
+  %agg.tmp = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %call = call spir_func i64 @Foo93(ptr addrspace(4) %this1, ptr byval(%"range") %agg.tmp) 
+  store i64 %call, ptr %LinearIndex
+  %call2 = call spir_func ptr addrspace(1) @Foo94(ptr addrspace(4) %this1) 
+  %0 = load i64, ptr %LinearIndex
+  %arrayidx = getelementptr inbounds nuw i64, ptr addrspace(1) %call2, i64 %0
+  %arrayidx.ascast = addrspacecast ptr addrspace(1) %arrayidx to ptr addrspace(4)
+  ret ptr addrspace(4) %arrayidx.ascast
+}
+
+
+define internal spir_func void @Foo75(ptr byval(%"tangle_group") %G, i32 %FenceScope) {
+entry:
+  %FenceScope.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  store i32 %FenceScope, ptr %FenceScope.addr, align 4
+  %0 = load i32, ptr %FenceScope.addr, align 4
+  call spir_func void @Foo95(ptr byval(%"tangle_group") %agg.tmp, i32 %0, i32 5) 
+  ret void
+}
+
+
+define internal spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %this, ptr byval(%"range") %Index) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %LinearIndex = alloca i64
+  %agg.tmp = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %call = call spir_func i64 @Foo77(ptr addrspace(4) %this1, ptr byval(%"range") %agg.tmp) 
+  store i64 %call, ptr %LinearIndex
+  %call2 = call spir_func ptr addrspace(1) @Foo78(ptr addrspace(4) %this1) 
+  %0 = load i64, ptr %LinearIndex
+  %arrayidx = getelementptr inbounds nuw i8, ptr addrspace(1) %call2, i64 %0
+  %arrayidx.ascast = addrspacecast ptr addrspace(1) %arrayidx to ptr addrspace(4)
+  ret ptr addrspace(4) %arrayidx.ascast
+}
+
+
+define internal spir_func i32 @Foo76(ptr addrspace(4) align 1 %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Foo96(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) align 1 %this1) 
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %ref.tmp.ascast, i32 0) 
+  %0 = load i64, ptr addrspace(4) %call
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+
+define internal spir_func i32 @Foo90(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %this1) 
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %ref.tmp.ascast, i32 0) 
+  %0 = load i64, ptr addrspace(4) %call
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+
+define internal spir_func i32 @Foo91(ptr byval(%"tangle_group") %g, i32 %x, i32 %linear_local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %linear_local_id.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %agg.tmp2 = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %agg.tmp1.ascast = addrspacecast ptr %agg.tmp1 to ptr addrspace(4)
+  %agg.tmp2.ascast = addrspacecast ptr %agg.tmp2 to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %linear_local_id, ptr %linear_local_id.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  call spir_func void @Foo97(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp2.ascast, ptr addrspace(4) %g.ascast) 
+  %1 = load i32, ptr %linear_local_id.addr, align 4
+  %conv = zext i32 %1 to i64
+  call spir_func void @Foo98(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp1.ascast, ptr byval(%"range") %agg.tmp2, i64 %conv) 
+  %call = call spir_func i32 @Bar69(ptr byval(%"tangle_group") %agg.tmp, i32 %0, ptr byval(%"range") %agg.tmp1) 
+  ret i32 %call
+}
+
+
+define internal spir_func zeroext i1 @Foo92(ptr byval(%"tangle_group") %g, i1 zeroext %pred) {
+entry:
+  %retval = alloca i1, align 1
+  %pred.addr = alloca i8, align 1
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %storedv = zext i1 %pred to i8
+  store i8 %storedv, ptr %pred.addr, align 1
+  %0 = load i8, ptr %pred.addr, align 1  
+  %loadedv = trunc i8 %0 to i1
+  %call = call spir_func zeroext i1 @Bar10(ptr byval(%"tangle_group") %agg.tmp, i1 zeroext %loadedv) 
+  ret i1 %call
+}
+
+
+define internal spir_func zeroext i1 @Foo67(ptr byval(%"tangle_group") %g, i1 zeroext %pred) {
+entry:
+  %retval = alloca i1, align 1
+  %pred.addr = alloca i8, align 1
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %storedv = zext i1 %pred to i8
+  store i8 %storedv, ptr %pred.addr, align 1
+  %0 = load i8, ptr %pred.addr, align 1  
+  %loadedv = trunc i8 %0 to i1
+  %call = call spir_func zeroext i1 @Foo66(ptr byval(%"tangle_group") %agg.tmp, i1 zeroext %loadedv) 
+  ret i1 %call
+}
+
+
+define internal spir_func zeroext i1 @Foo65(ptr byval(%"tangle_group") %g, i1 zeroext %pred) {
+entry:
+  %retval = alloca i1, align 1
+  %pred.addr = alloca i8, align 1
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %storedv = zext i1 %pred to i8
+  store i8 %storedv, ptr %pred.addr, align 1
+  %0 = load i8, ptr %pred.addr, align 1  
+  %loadedv = trunc i8 %0 to i1
+  %lnot = xor i1 %loadedv, true
+  %call = call spir_func zeroext i1 @Foo66(ptr byval(%"tangle_group") %agg.tmp, i1 zeroext %lnot) 
+  ret i1 %call
+}
+
+
+define internal spir_func i32 @Foo64(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"nd_item") align 1 %binary_op) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %binary_op.ascast = addrspacecast ptr %binary_op to ptr addrspace(4)
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar11(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %0, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Foo63(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"nd_item") align 1 %binary_op) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %res = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %binary_op.ascast = addrspacecast ptr %binary_op to ptr addrspace(4)
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar12(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %0, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  store i32 %call, ptr %res, align 4
+  %1 = load i32, ptr %res, align 4
+  ret i32 %1
+}
+
+
+define internal spir_func i32 @Foo62(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"nd_item") align 1 %binary_op) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %binary_op.ascast = addrspacecast ptr %binary_op to ptr addrspace(4)
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Foo61(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %0, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Foo73(ptr byval(%"tangle_group") %g, i32 %x, i32 %delta) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %delta.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %delta, ptr %delta.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  %1 = load i32, ptr %delta.addr, align 4
+  %call = call spir_func i32 @Foo72(ptr byval(%"tangle_group") %agg.tmp, i32 %0, i32 %1) 
+  ret i32 %call
+}
+
+
+define internal spir_func zeroext i1 @Foo71(ptr addrspace(4) align 1 %this, i64 %Other) {
+entry:
+  %retval = alloca i1, align 1
+  %this.addr = alloca ptr addrspace(4)
+  %Other.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %Other, ptr %Other.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %Other.addr
+  %cmp = icmp ult i64 %0, 4
+  ret i1 %cmp
+}
+
+
+define internal spir_func i32 @Foo53(ptr byval(%"tangle_group") %g, i32 %x, i32 %delta) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %delta.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %delta, ptr %delta.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  %1 = load i32, ptr %delta.addr, align 4
+  %call = call spir_func i32 @Foo52(ptr byval(%"tangle_group") %agg.tmp, i32 %0, i32 %1) 
+  ret i32 %call
+}
+
+
+define internal spir_func void @Foo51(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"ss_sub_group_mask"
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Mask1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %call = call spir_func i32 @Foo47(ptr byval(%"ss_sub_group_mask") %agg.tmp) 
+  %conv = zext i32 %call to i64
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %conv) 
+  ret void
+}
+
+
+define internal spir_func void @Foo55(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %lhs, ptr addrspace(4) align 4 %rhs) {
+entry:
+  %lhs.addr = alloca ptr addrspace(4)
+  %rhs.addr = alloca ptr addrspace(4)
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  store ptr addrspace(4) %lhs, ptr %lhs.addr
+  store ptr addrspace(4) %rhs, ptr %rhs.addr
+  call spir_func void @Foo11(ptr addrspace(4) %agg.result) 
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %1 = load ptr addrspace(4), ptr %lhs.addr
+  %common_array2 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %2 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array2, i64 0, i64 %idxprom
+  %3 = load i64, ptr addrspace(4) %arrayidx
+  %4 = load ptr addrspace(4), ptr %rhs.addr
+  %5 = load i32, ptr addrspace(4) %4, align 4
+  %conv = sext i32 %5 to i64
+  %add = add i64 %3, %conv
+  %common_array13 = bitcast ptr addrspace(4) %agg.result to ptr addrspace(4)
+  %6 = load i32, ptr %i, align 4
+  %idxprom2 = sext i32 %6 to i64
+  %arrayidx3 = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array13, i64 0, i64 %idxprom2
+  store i64 %add, ptr addrspace(4) %arrayidx3
+  %7 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+}
+
+
+define internal spir_func void @Foo56(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %lhs, ptr addrspace(4) %rhs) {
+entry:
+  %lhs.addr = alloca ptr addrspace(4)
+  %rhs.addr = alloca ptr addrspace(4)
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  store ptr addrspace(4) %lhs, ptr %lhs.addr
+  store ptr addrspace(4) %rhs, ptr %rhs.addr
+  call spir_func void @Foo11(ptr addrspace(4) %agg.result) 
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %1 = load ptr addrspace(4), ptr %lhs.addr
+  %common_array2 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %2 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array2, i64 0, i64 %idxprom
+  %3 = load i64, ptr addrspace(4) %arrayidx
+  %4 = load ptr addrspace(4), ptr %rhs.addr
+  %5 = load i64, ptr addrspace(4) %4
+  %rem = urem i64 %3, %5
+  %common_array13 = bitcast ptr addrspace(4) %agg.result to ptr addrspace(4)
+  %6 = load i32, ptr %i, align 4
+  %idxprom2 = sext i32 %6 to i64
+  %arrayidx3 = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array13, i64 0, i64 %idxprom2
+  store i64 %rem, ptr addrspace(4) %arrayidx3
+  %7 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+}
+
+
+define internal spir_func i32 @Foo57(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"range") %local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Foo59(ptr byval(%"tangle_group") %agg.tmp, i32 %0, ptr byval(%"range") %agg.tmp1) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Foo58(ptr byval(%"tangle_group") %g, i32 %x, i32 %mask) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %mask.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %agg.tmp1.ascast = addrspacecast ptr %agg.tmp1 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %mask, ptr %mask.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  %1 = load i32, ptr %mask.addr, align 4
+  %conv = zext i32 %1 to i64
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp1.ascast, i64 %conv) 
+  %call = call spir_func i32 @Bar13(ptr byval(%"tangle_group") %agg.tmp, i32 %0, ptr byval(%"range") %agg.tmp1) 
+  ret i32 %call
+}
+
+
+define internal spir_func void @Foo68(ptr addrspace(4) %this, i64 %WI, ptr byval(%"tangle_group") %Tangle, i64 %TangleLeader, i64 %TangleSize, ptr byval(%"nd_item") align 1 %IsMember) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %WI.addr = alloca i64
+  %TangleLeader.addr = alloca i64
+  %TangleSize.addr = alloca i64
+  %agg.tmp = alloca %"range"
+  %agg.tmp2 = alloca %"tangle_group"
+  %Visible = alloca i64
+  %Other = alloca i64
+  %cleanup.dest.slot = alloca i32, align 4
+  %agg.tmp5 = alloca %"range"
+  %agg.tmp8 = alloca %"range"
+  %OriginalLID = alloca i32, align 4
+  %LID = alloca i32, align 4
+  %BroadcastResult = alloca i32, align 4
+  %agg.tmp12 = alloca %"tangle_group"
+  %agg.tmp15 = alloca %"range"
+  %AnyResult = alloca i8, align 1
+  %agg.tmp18 = alloca %"tangle_group"
+  %agg.tmp24 = alloca %"range"
+  %AllResult = alloca i8, align 1
+  %agg.tmp27 = alloca %"tangle_group"
+  %agg.tmp35 = alloca %"range"
+  %NoneResult = alloca i8, align 1
+  %agg.tmp38 = alloca %"tangle_group"
+  %agg.tmp46 = alloca %"range"
+  %ReduceResult = alloca i32, align 4
+  %agg.tmp49 = alloca %"tangle_group"
+  %agg.tmp50 = alloca %"nd_item", align 1
+  %agg.tmp54 = alloca %"range"
+  %ExScanResult = alloca i32, align 4
+  %agg.tmp57 = alloca %"tangle_group"
+  %agg.tmp58 = alloca %"nd_item", align 1
+  %agg.tmp61 = alloca %"range"
+  %IncScanResult = alloca i32, align 4
+  %agg.tmp64 = alloca %"tangle_group"
+  %agg.tmp65 = alloca %"nd_item", align 1
+  %agg.tmp69 = alloca %"range"
+  %ShiftLeftResult = alloca i32, align 4
+  %agg.tmp72 = alloca %"tangle_group"
+  %agg.tmp79 = alloca %"range"
+  %ShiftRightResult = alloca i32, align 4
+  %agg.tmp82 = alloca %"tangle_group"
+  %agg.tmp88 = alloca %"range"
+  %SelectResult = alloca i32, align 4
+  %agg.tmp91 = alloca %"tangle_group"
+  %agg.tmp92 = alloca %"range"
+  %ref.tmp = alloca %"range"
+  %ref.tmp93 = alloca %"range"
+  %ref.tmp94 = alloca i32, align 4
+  %agg.tmp100 = alloca %"range"
+  %PermuteXorResult = alloca i32, align 4
+  %agg.tmp103 = alloca %"tangle_group"
+  %agg.tmp106 = alloca %"range"
+  %TangleSize.addr.ascast = addrspacecast ptr %TangleSize.addr to ptr addrspace(4)
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  %agg.tmp5.ascast = addrspacecast ptr %agg.tmp5 to ptr addrspace(4)
+  %agg.tmp8.ascast = addrspacecast ptr %agg.tmp8 to ptr addrspace(4)
+  %agg.tmp15.ascast = addrspacecast ptr %agg.tmp15 to ptr addrspace(4)
+  %agg.tmp24.ascast = addrspacecast ptr %agg.tmp24 to ptr addrspace(4)
+  %agg.tmp35.ascast = addrspacecast ptr %agg.tmp35 to ptr addrspace(4)
+  %agg.tmp46.ascast = addrspacecast ptr %agg.tmp46 to ptr addrspace(4)
+  %agg.tmp54.ascast = addrspacecast ptr %agg.tmp54 to ptr addrspace(4)
+  %agg.tmp61.ascast = addrspacecast ptr %agg.tmp61 to ptr addrspace(4)
+  %agg.tmp69.ascast = addrspacecast ptr %agg.tmp69 to ptr addrspace(4)
+  %agg.tmp79.ascast = addrspacecast ptr %agg.tmp79 to ptr addrspace(4)
+  %agg.tmp88.ascast = addrspacecast ptr %agg.tmp88 to ptr addrspace(4)
+  %agg.tmp92.ascast = addrspacecast ptr %agg.tmp92 to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %ref.tmp93.ascast = addrspacecast ptr %ref.tmp93 to ptr addrspace(4)
+  %ref.tmp94.ascast = addrspacecast ptr %ref.tmp94 to ptr addrspace(4)
+  %agg.tmp100.ascast = addrspacecast ptr %agg.tmp100 to ptr addrspace(4)
+  %agg.tmp106.ascast = addrspacecast ptr %agg.tmp106 to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %WI, ptr %WI.addr
+  %Tangle.ascast = addrspacecast ptr %Tangle to ptr addrspace(4)
+  store i64 %TangleLeader, ptr %TangleLeader.addr
+  store i64 %TangleSize, ptr %TangleSize.addr
+  %IsMember.ascast = addrspacecast ptr %IsMember to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %TmpAcc1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp.ascast, i64 %0) 
+  %call = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %TmpAcc1, ptr byval(%"range") %agg.tmp) 
+  store i64 1, ptr addrspace(4) %call
+  call spir_func void @Foo75(ptr byval(%"tangle_group") %agg.tmp2, i32 1) 
+  store i64 0, ptr %Visible
+  store i64 0, ptr %Other
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %1 = load i64, ptr %Other
+  %cmp = icmp ult i64 %1, 32
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %2 = load i64, ptr %Visible
+  %3 = load i64, ptr %TangleSize.addr
+  %cmp7 = icmp eq i64 %2, %3
+  %BarrierAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 1
+  %4 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp8.ascast, i64 %4) 
+  %call9 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BarrierAcc, ptr byval(%"range") %agg.tmp8) 
+  %storedv = zext i1 %cmp7 to i8
+  store i8 %storedv, ptr addrspace(4) %call9, align 1
+  %5 = getelementptr inbounds i8, ptr addrspace(4) %this1, i64 64
+  %call10 = call spir_func i32 @Foo76(ptr addrspace(4) align 1 %5) 
+  store i32 %call10, ptr %OriginalLID, align 4
+  %call11 = call spir_func i32 @Foo90(ptr addrspace(4) %Tangle.ascast) 
+  store i32 %call11, ptr %LID, align 4
+  %6 = load i32, ptr %OriginalLID, align 4
+  %call13 = call spir_func i32 @Foo91(ptr byval(%"tangle_group") %agg.tmp12, i32 %6, i32 0) 
+  store i32 %call13, ptr %BroadcastResult, align 4
+  %7 = load i32, ptr %BroadcastResult, align 4
+  %conv = zext i32 %7 to i64
+  %8 = load i64, ptr %TangleLeader.addr
+  %cmp14 = icmp eq i64 %conv, %8
+  %BroadcastAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 3
+  %9 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp15.ascast, i64 %9) 
+  %call16 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BroadcastAcc, ptr byval(%"range") %agg.tmp15) 
+  %storedv17 = zext i1 %cmp14 to i8
+  store i8 %storedv17, ptr addrspace(4) %call16, align 1
+  %10 = load i32, ptr %LID, align 4
+  %cmp19 = icmp eq i32 %10, 0
+  %call20 = call spir_func zeroext i1 @Foo92(ptr byval(%"tangle_group") %agg.tmp18, i1 zeroext %cmp19) 
+  %storedv21 = zext i1 %call20 to i8
+  store i8 %storedv21, ptr %AnyResult, align 1
+  %11 = load i8, ptr %AnyResult, align 1  
+  %loadedv = trunc i8 %11 to i1
+  %conv22 = zext i1 %loadedv to i32
+  %cmp23 = icmp eq i32 %conv22, 1
+  %AnyAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 4
+  %12 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp24.ascast, i64 %12) 
+  %call25 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AnyAcc, ptr byval(%"range") %agg.tmp24) 
+  %storedv26 = zext i1 %cmp23 to i8
+  store i8 %storedv26, ptr addrspace(4) %call25, align 1
+  %13 = load i32, ptr %LID, align 4
+  %conv28 = zext i32 %13 to i64
+  %14 = load i64, ptr %TangleSize.addr
+  %cmp29 = icmp ult i64 %conv28, %14
+  %call30 = call spir_func zeroext i1 @Foo67(ptr byval(%"tangle_group") %agg.tmp27, i1 zeroext %cmp29) 
+  %storedv31 = zext i1 %call30 to i8
+  store i8 %storedv31, ptr %AllResult, align 1
+  %15 = load i8, ptr %AllResult, align 1  
+  %loadedv32 = trunc i8 %15 to i1
+  %conv33 = zext i1 %loadedv32 to i32
+  %cmp34 = icmp eq i32 %conv33, 1
+  %AllAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 5
+  %16 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp35.ascast, i64 %16) 
+  %call36 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AllAcc, ptr byval(%"range") %agg.tmp35) 
+  %storedv37 = zext i1 %cmp34 to i8
+  store i8 %storedv37, ptr addrspace(4) %call36, align 1
+  %17 = load i32, ptr %LID, align 4
+  %conv39 = zext i32 %17 to i64
+  %18 = load i64, ptr %TangleSize.addr
+  %cmp40 = icmp uge i64 %conv39, %18
+  %call41 = call spir_func zeroext i1 @Foo65(ptr byval(%"tangle_group") %agg.tmp38, i1 zeroext %cmp40) 
+  %storedv42 = zext i1 %call41 to i8
+  store i8 %storedv42, ptr %NoneResult, align 1
+  %19 = load i8, ptr %NoneResult, align 1  
+  %loadedv43 = trunc i8 %19 to i1
+  %conv44 = zext i1 %loadedv43 to i32
+  %cmp45 = icmp eq i32 %conv44, 1
+  %NoneAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 6
+  %20 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp46.ascast, i64 %20) 
+  %call47 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %NoneAcc, ptr byval(%"range") %agg.tmp46) 
+  %storedv48 = zext i1 %cmp45 to i8
+  store i8 %storedv48, ptr addrspace(4) %call47, align 1
+  %call51 = call spir_func i32 @Foo64(ptr byval(%"tangle_group") %agg.tmp49, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp50) 
+  store i32 %call51, ptr %ReduceResult, align 4
+  %21 = load i32, ptr %ReduceResult, align 4
+  %conv52 = zext i32 %21 to i64
+  %22 = load i64, ptr %TangleSize.addr
+  %cmp53 = icmp eq i64 %conv52, %22
+  %ReduceAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 7
+  %23 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp54.ascast, i64 %23) 
+  %call55 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ReduceAcc, ptr byval(%"range") %agg.tmp54) 
+  %storedv56 = zext i1 %cmp53 to i8
+  store i8 %storedv56, ptr addrspace(4) %call55, align 1
+  %call59 = call spir_func i32 @Foo63(ptr byval(%"tangle_group") %agg.tmp57, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp58) 
+  store i32 %call59, ptr %ExScanResult, align 4
+  %24 = load i32, ptr %ExScanResult, align 4
+  %25 = load i32, ptr %LID, align 4
+  %cmp60 = icmp eq i32 %24, %25
+  %ExScanAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 8
+  %26 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp61.ascast, i64 %26) 
+  %call62 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ExScanAcc, ptr byval(%"range") %agg.tmp61) 
+  %storedv63 = zext i1 %cmp60 to i8
+  store i8 %storedv63, ptr addrspace(4) %call62, align 1
+  %call66 = call spir_func i32 @Foo62(ptr byval(%"tangle_group") %agg.tmp64, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp65) 
+  store i32 %call66, ptr %IncScanResult, align 4
+  %27 = load i32, ptr %IncScanResult, align 4
+  %28 = load i32, ptr %LID, align 4
+  %add67 = add i32 %28, 1
+  %cmp68 = icmp eq i32 %27, %add67
+  %IncScanAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 9
+  %29 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp69.ascast, i64 %29) 
+  %call70 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %IncScanAcc, ptr byval(%"range") %agg.tmp69) 
+  %storedv71 = zext i1 %cmp68 to i8
+  store i8 %storedv71, ptr addrspace(4) %call70, align 1
+  %30 = load i32, ptr %LID, align 4
+  %call73 = call spir_func i32 @Foo73(ptr byval(%"tangle_group") %agg.tmp72, i32 %30, i32 2) 
+  store i32 %call73, ptr %ShiftLeftResult, align 4
+  %31 = load i32, ptr %LID, align 4
+  %add74 = add i32 %31, 2
+  %conv75 = zext i32 %add74 to i64
+  %32 = load i64, ptr %TangleSize.addr
+  %cmp76 = icmp uge i64 %conv75, %32
+  br i1 %cmp76, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %for.cond.cleanup
+  %33 = load i32, ptr %ShiftLeftResult, align 4
+  %34 = load i32, ptr %LID, align 4
+  %add77 = add i32 %34, 2
+  %cmp78 = icmp eq i32 %33, %add77
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %for.cond.cleanup
+  %35 = phi i1 [ true, %for.cond.cleanup ], [ %cmp78, %lor.rhs ]
+  %ShiftLeftAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 10
+  %36 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp79.ascast, i64 %36) 
+  %call80 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftLeftAcc, ptr byval(%"range") %agg.tmp79) 
+  %storedv81 = zext i1 %35 to i8
+  store i8 %storedv81, ptr addrspace(4) %call80, align 1
+  %37 = load i32, ptr %LID, align 4
+  %call83 = call spir_func i32 @Foo53(ptr byval(%"tangle_group") %agg.tmp82, i32 %37, i32 2) 
+  store i32 %call83, ptr %ShiftRightResult, align 4
+  %38 = load i32, ptr %LID, align 4
+  %cmp84 = icmp ult i32 %38, 2
+  br i1 %cmp84, label %lor.end87, label %lor.rhs85
+
+lor.rhs85:                                        ; preds = %lor.end
+  %39 = load i32, ptr %ShiftRightResult, align 4
+  %40 = load i32, ptr %LID, align 4
+  %sub = sub i32 %40, 2
+  %cmp86 = icmp eq i32 %39, %sub
+  br label %lor.end87
+
+lor.end87:                                        ; preds = %lor.rhs85, %lor.end
+  %41 = phi i1 [ true, %lor.end ], [ %cmp86, %lor.rhs85 ]
+  %ShiftRightAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 11
+  %42 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp88.ascast, i64 %42) 
+  %call89 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftRightAcc, ptr byval(%"range") %agg.tmp88) 
+  %storedv90 = zext i1 %41 to i8
+  store i8 %storedv90, ptr addrspace(4) %call89, align 1
+  %43 = load i32, ptr %LID, align 4
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp93.ascast, ptr addrspace(4) %Tangle.ascast) 
+  store i32 2, ptr %ref.tmp94, align 4
+  call spir_func void @Foo55(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %ref.tmp93.ascast, ptr addrspace(4) align 4 %ref.tmp94.ascast) 
+  call spir_func void @Foo56(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp92.ascast, ptr addrspace(4) %ref.tmp.ascast, ptr addrspace(4) %TangleSize.addr.ascast) 
+  %call95 = call spir_func i32 @Foo57(ptr byval(%"tangle_group") %agg.tmp91, i32 %43, ptr byval(%"range") %agg.tmp92) 
+  store i32 %call95, ptr %SelectResult, align 4
+  %44 = load i32, ptr %SelectResult, align 4
+  %conv96 = zext i32 %44 to i64
+  %45 = load i32, ptr %LID, align 4
+  %add97 = add i32 %45, 2
+  %conv98 = zext i32 %add97 to i64
+  %46 = load i64, ptr %TangleSize.addr
+  %rem = urem i64 %conv98, %46
+  %cmp99 = icmp eq i64 %conv96, %rem
+  %SelectAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 12
+  %47 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp100.ascast, i64 %47) 
+  %call101 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %SelectAcc, ptr byval(%"range") %agg.tmp100) 
+  %storedv102 = zext i1 %cmp99 to i8
+  store i8 %storedv102, ptr addrspace(4) %call101, align 1
+  %48 = load i32, ptr %LID, align 4
+  %call104 = call spir_func i32 @Foo58(ptr byval(%"tangle_group") %agg.tmp103, i32 %48, i32 2) 
+  store i32 %call104, ptr %PermuteXorResult, align 4
+  %49 = load i32, ptr %PermuteXorResult, align 4
+  %50 = load i32, ptr %LID, align 4
+  %xor = xor i32 %50, 2
+  %cmp105 = icmp eq i32 %49, %xor
+  %PermuteXorAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 13
+  %51 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp106.ascast, i64 %51) 
+  %call107 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %PermuteXorAcc, ptr byval(%"range") %agg.tmp106) 
+  %storedv108 = zext i1 %cmp105 to i8
+  store i8 %storedv108, ptr addrspace(4) %call107, align 1
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %52 = load i64, ptr %Other
+  %call3 = call spir_func zeroext i1 @Foo74(ptr addrspace(4) align 1 %IsMember.ascast, i64 %52) 
+  br i1 %call3, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %TmpAcc42 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %53 = load i64, ptr %Other
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp5.ascast, i64 %53) 
+  %call6 = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %TmpAcc42, ptr byval(%"range") %agg.tmp5) 
+  %54 = load i64, ptr addrspace(4) %call6
+  %55 = load i64, ptr %Visible
+  %add = add i64 %55, %54
+  store i64 %add, ptr %Visible
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %56 = load i64, ptr %Other
+  %inc = add i64 %56, 1
+  store i64 %inc, ptr %Other
+  br label %for.cond
+}
+
+
+define internal spir_func void @Foo69(ptr addrspace(4) %this, i64 %WI, ptr byval(%"tangle_group") %Tangle, i64 %TangleLeader, i64 %TangleSize, ptr byval(%"nd_item") align 1 %IsMember) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %WI.addr = alloca i64
+  %TangleLeader.addr = alloca i64
+  %TangleSize.addr = alloca i64
+  %agg.tmp = alloca %"range"
+  %agg.tmp2 = alloca %"tangle_group"
+  %Visible = alloca i64
+  %Other = alloca i64
+  %cleanup.dest.slot = alloca i32, align 4
+  %agg.tmp5 = alloca %"range"
+  %agg.tmp8 = alloca %"range"
+  %OriginalLID = alloca i32, align 4
+  %LID = alloca i32, align 4
+  %BroadcastResult = alloca i32, align 4
+  %agg.tmp12 = alloca %"tangle_group"
+  %agg.tmp15 = alloca %"range"
+  %AnyResult = alloca i8, align 1
+  %agg.tmp18 = alloca %"tangle_group"
+  %agg.tmp24 = alloca %"range"
+  %AllResult = alloca i8, align 1
+  %agg.tmp27 = alloca %"tangle_group"
+  %agg.tmp35 = alloca %"range"
+  %NoneResult = alloca i8, align 1
+  %agg.tmp38 = alloca %"tangle_group"
+  %agg.tmp46 = alloca %"range"
+  %ReduceResult = alloca i32, align 4
+  %agg.tmp49 = alloca %"tangle_group"
+  %agg.tmp50 = alloca %"nd_item", align 1
+  %agg.tmp54 = alloca %"range"
+  %ExScanResult = alloca i32, align 4
+  %agg.tmp57 = alloca %"tangle_group"
+  %agg.tmp58 = alloca %"nd_item", align 1
+  %agg.tmp61 = alloca %"range"
+  %IncScanResult = alloca i32, align 4
+  %agg.tmp64 = alloca %"tangle_group"
+  %agg.tmp65 = alloca %"nd_item", align 1
+  %agg.tmp69 = alloca %"range"
+  %ShiftLeftResult = alloca i32, align 4
+  %agg.tmp72 = alloca %"tangle_group"
+  %agg.tmp79 = alloca %"range"
+  %ShiftRightResult = alloca i32, align 4
+  %agg.tmp82 = alloca %"tangle_group"
+  %agg.tmp88 = alloca %"range"
+  %SelectResult = alloca i32, align 4
+  %agg.tmp91 = alloca %"tangle_group"
+  %agg.tmp92 = alloca %"range"
+  %ref.tmp = alloca %"range"
+  %ref.tmp93 = alloca %"range"
+  %ref.tmp94 = alloca i32, align 4
+  %agg.tmp100 = alloca %"range"
+  %PermuteXorResult = alloca i32, align 4
+  %agg.tmp103 = alloca %"tangle_group"
+  %agg.tmp106 = alloca %"range"
+  %TangleSize.addr.ascast = addrspacecast ptr %TangleSize.addr to ptr addrspace(4)
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  %agg.tmp5.ascast = addrspacecast ptr %agg.tmp5 to ptr addrspace(4)
+  %agg.tmp8.ascast = addrspacecast ptr %agg.tmp8 to ptr addrspace(4)
+  %agg.tmp15.ascast = addrspacecast ptr %agg.tmp15 to ptr addrspace(4)
+  %agg.tmp24.ascast = addrspacecast ptr %agg.tmp24 to ptr addrspace(4)
+  %agg.tmp35.ascast = addrspacecast ptr %agg.tmp35 to ptr addrspace(4)
+  %agg.tmp46.ascast = addrspacecast ptr %agg.tmp46 to ptr addrspace(4)
+  %agg.tmp54.ascast = addrspacecast ptr %agg.tmp54 to ptr addrspace(4)
+  %agg.tmp61.ascast = addrspacecast ptr %agg.tmp61 to ptr addrspace(4)
+  %agg.tmp69.ascast = addrspacecast ptr %agg.tmp69 to ptr addrspace(4)
+  %agg.tmp79.ascast = addrspacecast ptr %agg.tmp79 to ptr addrspace(4)
+  %agg.tmp88.ascast = addrspacecast ptr %agg.tmp88 to ptr addrspace(4)
+  %agg.tmp92.ascast = addrspacecast ptr %agg.tmp92 to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %ref.tmp93.ascast = addrspacecast ptr %ref.tmp93 to ptr addrspace(4)
+  %ref.tmp94.ascast = addrspacecast ptr %ref.tmp94 to ptr addrspace(4)
+  %agg.tmp100.ascast = addrspacecast ptr %agg.tmp100 to ptr addrspace(4)
+  %agg.tmp106.ascast = addrspacecast ptr %agg.tmp106 to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %WI, ptr %WI.addr
+  %Tangle.ascast = addrspacecast ptr %Tangle to ptr addrspace(4)
+  store i64 %TangleLeader, ptr %TangleLeader.addr
+  store i64 %TangleSize, ptr %TangleSize.addr
+  %IsMember.ascast = addrspacecast ptr %IsMember to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %TmpAcc1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp.ascast, i64 %0) 
+  %call = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %TmpAcc1, ptr byval(%"range") %agg.tmp) 
+  store i64 1, ptr addrspace(4) %call
+  call spir_func void @Foo75(ptr byval(%"tangle_group") %agg.tmp2, i32 1) 
+  store i64 0, ptr %Visible
+  store i64 0, ptr %Other
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %1 = load i64, ptr %Other
+  %cmp = icmp ult i64 %1, 32
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %2 = load i64, ptr %Visible
+  %3 = load i64, ptr %TangleSize.addr
+  %cmp7 = icmp eq i64 %2, %3
+  %BarrierAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 1
+  %4 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp8.ascast, i64 %4) 
+  %call9 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BarrierAcc, ptr byval(%"range") %agg.tmp8) 
+  %storedv = zext i1 %cmp7 to i8
+  store i8 %storedv, ptr addrspace(4) %call9, align 1
+  %5 = getelementptr inbounds i8, ptr addrspace(4) %this1, i64 64
+  %call10 = call spir_func i32 @Foo76(ptr addrspace(4) align 1 %5) 
+  store i32 %call10, ptr %OriginalLID, align 4
+  %call11 = call spir_func i32 @Foo90(ptr addrspace(4) %Tangle.ascast) 
+  store i32 %call11, ptr %LID, align 4
+  %6 = load i32, ptr %OriginalLID, align 4
+  %call13 = call spir_func i32 @Foo91(ptr byval(%"tangle_group") %agg.tmp12, i32 %6, i32 0) 
+  store i32 %call13, ptr %BroadcastResult, align 4
+  %7 = load i32, ptr %BroadcastResult, align 4
+  %conv = zext i32 %7 to i64
+  %8 = load i64, ptr %TangleLeader.addr
+  %cmp14 = icmp eq i64 %conv, %8
+  %BroadcastAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 3
+  %9 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp15.ascast, i64 %9) 
+  %call16 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %BroadcastAcc, ptr byval(%"range") %agg.tmp15) 
+  %storedv17 = zext i1 %cmp14 to i8
+  store i8 %storedv17, ptr addrspace(4) %call16, align 1
+  %10 = load i32, ptr %LID, align 4
+  %cmp19 = icmp eq i32 %10, 0
+  %call20 = call spir_func zeroext i1 @Foo92(ptr byval(%"tangle_group") %agg.tmp18, i1 zeroext %cmp19) 
+  %storedv21 = zext i1 %call20 to i8
+  store i8 %storedv21, ptr %AnyResult, align 1
+  %11 = load i8, ptr %AnyResult, align 1  
+  %loadedv = trunc i8 %11 to i1
+  %conv22 = zext i1 %loadedv to i32
+  %cmp23 = icmp eq i32 %conv22, 1
+  %AnyAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 4
+  %12 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp24.ascast, i64 %12) 
+  %call25 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AnyAcc, ptr byval(%"range") %agg.tmp24) 
+  %storedv26 = zext i1 %cmp23 to i8
+  store i8 %storedv26, ptr addrspace(4) %call25, align 1
+  %13 = load i32, ptr %LID, align 4
+  %conv28 = zext i32 %13 to i64
+  %14 = load i64, ptr %TangleSize.addr
+  %cmp29 = icmp ult i64 %conv28, %14
+  %call30 = call spir_func zeroext i1 @Foo67(ptr byval(%"tangle_group") %agg.tmp27, i1 zeroext %cmp29) 
+  %storedv31 = zext i1 %call30 to i8
+  store i8 %storedv31, ptr %AllResult, align 1
+  %15 = load i8, ptr %AllResult, align 1  
+  %loadedv32 = trunc i8 %15 to i1
+  %conv33 = zext i1 %loadedv32 to i32
+  %cmp34 = icmp eq i32 %conv33, 1
+  %AllAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 5
+  %16 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp35.ascast, i64 %16) 
+  %call36 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %AllAcc, ptr byval(%"range") %agg.tmp35) 
+  %storedv37 = zext i1 %cmp34 to i8
+  store i8 %storedv37, ptr addrspace(4) %call36, align 1
+  %17 = load i32, ptr %LID, align 4
+  %conv39 = zext i32 %17 to i64
+  %18 = load i64, ptr %TangleSize.addr
+  %cmp40 = icmp uge i64 %conv39, %18
+  %call41 = call spir_func zeroext i1 @Foo65(ptr byval(%"tangle_group") %agg.tmp38, i1 zeroext %cmp40) 
+  %storedv42 = zext i1 %call41 to i8
+  store i8 %storedv42, ptr %NoneResult, align 1
+  %19 = load i8, ptr %NoneResult, align 1  
+  %loadedv43 = trunc i8 %19 to i1
+  %conv44 = zext i1 %loadedv43 to i32
+  %cmp45 = icmp eq i32 %conv44, 1
+  %NoneAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 6
+  %20 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp46.ascast, i64 %20) 
+  %call47 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %NoneAcc, ptr byval(%"range") %agg.tmp46) 
+  %storedv48 = zext i1 %cmp45 to i8
+  store i8 %storedv48, ptr addrspace(4) %call47, align 1
+  %call51 = call spir_func i32 @Foo64(ptr byval(%"tangle_group") %agg.tmp49, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp50) 
+  store i32 %call51, ptr %ReduceResult, align 4
+  %21 = load i32, ptr %ReduceResult, align 4
+  %conv52 = zext i32 %21 to i64
+  %22 = load i64, ptr %TangleSize.addr
+  %cmp53 = icmp eq i64 %conv52, %22
+  %ReduceAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 7
+  %23 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp54.ascast, i64 %23) 
+  %call55 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ReduceAcc, ptr byval(%"range") %agg.tmp54) 
+  %storedv56 = zext i1 %cmp53 to i8
+  store i8 %storedv56, ptr addrspace(4) %call55, align 1
+  %call59 = call spir_func i32 @Foo63(ptr byval(%"tangle_group") %agg.tmp57, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp58) 
+  store i32 %call59, ptr %ExScanResult, align 4
+  %24 = load i32, ptr %ExScanResult, align 4
+  %25 = load i32, ptr %LID, align 4
+  %cmp60 = icmp eq i32 %24, %25
+  %ExScanAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 8
+  %26 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp61.ascast, i64 %26) 
+  %call62 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ExScanAcc, ptr byval(%"range") %agg.tmp61) 
+  %storedv63 = zext i1 %cmp60 to i8
+  store i8 %storedv63, ptr addrspace(4) %call62, align 1
+  %call66 = call spir_func i32 @Foo62(ptr byval(%"tangle_group") %agg.tmp64, i32 1, ptr byval(%"nd_item") align 1 %agg.tmp65) 
+  store i32 %call66, ptr %IncScanResult, align 4
+  %27 = load i32, ptr %IncScanResult, align 4
+  %28 = load i32, ptr %LID, align 4
+  %add67 = add i32 %28, 1
+  %cmp68 = icmp eq i32 %27, %add67
+  %IncScanAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 9
+  %29 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp69.ascast, i64 %29) 
+  %call70 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %IncScanAcc, ptr byval(%"range") %agg.tmp69) 
+  %storedv71 = zext i1 %cmp68 to i8
+  store i8 %storedv71, ptr addrspace(4) %call70, align 1
+  %30 = load i32, ptr %LID, align 4
+  %call73 = call spir_func i32 @Foo73(ptr byval(%"tangle_group") %agg.tmp72, i32 %30, i32 2) 
+  store i32 %call73, ptr %ShiftLeftResult, align 4
+  %31 = load i32, ptr %LID, align 4
+  %add74 = add i32 %31, 2
+  %conv75 = zext i32 %add74 to i64
+  %32 = load i64, ptr %TangleSize.addr
+  %cmp76 = icmp uge i64 %conv75, %32
+  br i1 %cmp76, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %for.cond.cleanup
+  %33 = load i32, ptr %ShiftLeftResult, align 4
+  %34 = load i32, ptr %LID, align 4
+  %add77 = add i32 %34, 2
+  %cmp78 = icmp eq i32 %33, %add77
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %for.cond.cleanup
+  %35 = phi i1 [ true, %for.cond.cleanup ], [ %cmp78, %lor.rhs ]
+  %ShiftLeftAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 10
+  %36 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp79.ascast, i64 %36) 
+  %call80 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftLeftAcc, ptr byval(%"range") %agg.tmp79) 
+  %storedv81 = zext i1 %35 to i8
+  store i8 %storedv81, ptr addrspace(4) %call80, align 1
+  %37 = load i32, ptr %LID, align 4
+  %call83 = call spir_func i32 @Foo53(ptr byval(%"tangle_group") %agg.tmp82, i32 %37, i32 2) 
+  store i32 %call83, ptr %ShiftRightResult, align 4
+  %38 = load i32, ptr %LID, align 4
+  %cmp84 = icmp ult i32 %38, 2
+  br i1 %cmp84, label %lor.end87, label %lor.rhs85
+
+lor.rhs85:                                        ; preds = %lor.end
+  %39 = load i32, ptr %ShiftRightResult, align 4
+  %40 = load i32, ptr %LID, align 4
+  %sub = sub i32 %40, 2
+  %cmp86 = icmp eq i32 %39, %sub
+  br label %lor.end87
+
+lor.end87:                                        ; preds = %lor.rhs85, %lor.end
+  %41 = phi i1 [ true, %lor.end ], [ %cmp86, %lor.rhs85 ]
+  %ShiftRightAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 11
+  %42 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp88.ascast, i64 %42) 
+  %call89 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %ShiftRightAcc, ptr byval(%"range") %agg.tmp88) 
+  %storedv90 = zext i1 %41 to i8
+  store i8 %storedv90, ptr addrspace(4) %call89, align 1
+  %43 = load i32, ptr %LID, align 4
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp93.ascast, ptr addrspace(4) %Tangle.ascast) 
+  store i32 2, ptr %ref.tmp94, align 4
+  call spir_func void @Foo55(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %ref.tmp93.ascast, ptr addrspace(4) align 4 %ref.tmp94.ascast) 
+  call spir_func void @Foo56(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.tmp92.ascast, ptr addrspace(4) %ref.tmp.ascast, ptr addrspace(4) %TangleSize.addr.ascast) 
+  %call95 = call spir_func i32 @Foo57(ptr byval(%"tangle_group") %agg.tmp91, i32 %43, ptr byval(%"range") %agg.tmp92) 
+  store i32 %call95, ptr %SelectResult, align 4
+  %44 = load i32, ptr %SelectResult, align 4
+  %conv96 = zext i32 %44 to i64
+  %45 = load i32, ptr %LID, align 4
+  %add97 = add i32 %45, 2
+  %conv98 = zext i32 %add97 to i64
+  %46 = load i64, ptr %TangleSize.addr
+  %rem = urem i64 %conv98, %46
+  %cmp99 = icmp eq i64 %conv96, %rem
+  %SelectAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 12
+  %47 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp100.ascast, i64 %47) 
+  %call101 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %SelectAcc, ptr byval(%"range") %agg.tmp100) 
+  %storedv102 = zext i1 %cmp99 to i8
+  store i8 %storedv102, ptr addrspace(4) %call101, align 1
+  %48 = load i32, ptr %LID, align 4
+  %call104 = call spir_func i32 @Foo58(ptr byval(%"tangle_group") %agg.tmp103, i32 %48, i32 2) 
+  store i32 %call104, ptr %PermuteXorResult, align 4
+  %49 = load i32, ptr %PermuteXorResult, align 4
+  %50 = load i32, ptr %LID, align 4
+  %xor = xor i32 %50, 2
+  %cmp105 = icmp eq i32 %49, %xor
+  %PermuteXorAcc = getelementptr inbounds nuw %class.anon.8, ptr addrspace(4) %this1, i32 0, i32 13
+  %51 = load i64, ptr %WI.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp106.ascast, i64 %51) 
+  %call107 = call spir_func align 1 ptr addrspace(4) @Foo54(ptr addrspace(4) %PermuteXorAcc, ptr byval(%"range") %agg.tmp106) 
+  %storedv108 = zext i1 %cmp105 to i8
+  store i8 %storedv108, ptr addrspace(4) %call107, align 1
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %52 = load i64, ptr %Other
+  %call3 = call spir_func zeroext i1 @Bar14(ptr addrspace(4) align 1 %IsMember.ascast, i64 %52) 
+  br i1 %call3, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %TmpAcc42 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %53 = load i64, ptr %Other
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp5.ascast, i64 %53) 
+  %call6 = call spir_func ptr addrspace(4) @Foo70(ptr addrspace(4) %TmpAcc42, ptr byval(%"range") %agg.tmp5) 
+  %54 = load i64, ptr addrspace(4) %call6
+  %55 = load i64, ptr %Visible
+  %add = add i64 %55, %54
+  store i64 %add, ptr %Visible
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %56 = load i64, ptr %Other
+  %inc = add i64 %56, 1
+  store i64 %inc, ptr %Other
+  br label %for.cond
+}
+
+
+define internal spir_func zeroext i1 @Bar14(ptr addrspace(4) align 1 %this, i64 %Other) {
+entry:
+  %retval = alloca i1, align 1
+  %this.addr = alloca ptr addrspace(4)
+  %Other.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %Other, ptr %Other.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %Other.addr
+  %cmp = icmp uge i64 %0, 24
+  br i1 %cmp, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %entry
+  %1 = load i64, ptr %Other.addr
+  %cmp2 = icmp ult i64 %1, 32
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  %2 = phi i1 [ false, %entry ], [ %cmp2, %land.rhs ]
+  ret i1 %2
+}
+
+
+define internal spir_func zeroext i1 @Foo74(ptr addrspace(4) align 1 %this, i64 %Other) {
+entry:
+  %retval = alloca i1, align 1
+  %this.addr = alloca ptr addrspace(4)
+  %Other.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %Other, ptr %Other.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %Other.addr
+  %cmp = icmp uge i64 %0, 4
+  br i1 %cmp, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %entry
+  %1 = load i64, ptr %Other.addr
+  %cmp2 = icmp ult i64 %1, 24
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  %2 = phi i1 [ false, %entry ], [ %cmp2, %land.rhs ]
+  ret i1 %2
+}
+
+
+define internal spir_func i32 @Bar13(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"range") %mask) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %TargetLocalId = alloca %"range"
+  %ref.tmp = alloca %"range"
+  %TargetId = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %x.addr.ascast = addrspacecast ptr %x.addr to ptr addrspace(4)
+  %TargetLocalId.ascast = addrspacecast ptr %TargetLocalId to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %mask.ascast = addrspacecast ptr %mask to ptr addrspace(4)
+  %0 = addrspacecast ptr addrspace(1) @_ZSt6ignore to ptr addrspace(4)
+  %call = call spir_func align 1 ptr addrspace(4) @Bar15(ptr addrspace(4) align 1 %0, ptr addrspace(4) %g.ascast) 
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %g.ascast) 
+  call spir_func void @Bar16(ptr addrspace(4) dead_on_unwind writable sret(%"range") %TargetLocalId.ascast, ptr addrspace(4) %ref.tmp.ascast, ptr addrspace(4) %mask.ascast) 
+  %call2 = call spir_func i32 @Foo48(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"range") %agg.tmp1) 
+  store i32 %call2, ptr %TargetId, align 4
+  %call3 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %x.addr.ascast) 
+  %1 = load i32, ptr %TargetId, align 4
+  %call4 = call spir_func i32 @Foo50(i32 3, i32 %call3, i32 %1) 
+  ret i32 %call4
+}
+
+
+define internal spir_func align 1 ptr addrspace(4) @Bar15(ptr addrspace(4) align 1 %this, ptr addrspace(4) %0) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %0, ptr %.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  ret ptr addrspace(4) %this1
+}
+
+
+define internal spir_func void @Bar16(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %lhs, ptr addrspace(4) %rhs) {
+entry:
+  %lhs.addr = alloca ptr addrspace(4)
+  %rhs.addr = alloca ptr addrspace(4)
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  store ptr addrspace(4) %lhs, ptr %lhs.addr
+  store ptr addrspace(4) %rhs, ptr %rhs.addr
+  call spir_func void @Foo11(ptr addrspace(4) %agg.result) 
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %1 = load ptr addrspace(4), ptr %lhs.addr
+  %common_array2 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %2 = load i32, ptr %i, align 4
+  %idxprom = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array2, i64 0, i64 %idxprom
+  %3 = load i64, ptr addrspace(4) %arrayidx
+  %4 = load ptr addrspace(4), ptr %rhs.addr
+  %common_array13 = bitcast ptr addrspace(4) %4 to ptr addrspace(4)
+  %5 = load i32, ptr %i, align 4
+  %idxprom2 = sext i32 %5 to i64
+  %arrayidx3 = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array13, i64 0, i64 %idxprom2
+  %6 = load i64, ptr addrspace(4) %arrayidx3
+  %xor = xor i64 %3, %6
+  %common_array44 = bitcast ptr addrspace(4) %agg.result to ptr addrspace(4)
+  %7 = load i32, ptr %i, align 4
+  %idxprom5 = sext i32 %7 to i64
+  %arrayidx6 = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array44, i64 0, i64 %idxprom5
+  store i64 %xor, ptr addrspace(4) %arrayidx6
+  %8 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %8, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+}
+
+
+define internal spir_func i32 @Foo48(ptr byval(%"tangle_group") %g, ptr byval(%"range") %local_id) {
+entry:
+  %retval.i = alloca i64
+  %this.addr.i = alloca ptr addrspace(4)
+  %Result.i = alloca i64
+  %retval = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %local_id.ascast = addrspacecast ptr %local_id to ptr addrspace(4)
+  %retval.ascast.i = addrspacecast ptr %retval.i to ptr addrspace(4)
+  store ptr addrspace(4) %local_id.ascast, ptr %this.addr.i
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i
+  %0 = load i64, ptr addrspace(4) %this1.i
+  store i64 %0, ptr %Result.i
+  %1 = load i64, ptr %Result.i
+  %conv = trunc i64 %1 to i32
+  %call1 = call spir_func i32 @Bar17(ptr byval(%"tangle_group") %agg.tmp, i32 %conv) 
+  ret i32 %call1
+}
+
+
+define internal spir_func i32 @Foo49(ptr addrspace(4) align 4 %x) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %x, ptr %x.addr
+  %0 = load ptr addrspace(4), ptr %x.addr
+  %1 = load i32, ptr addrspace(4) %0, align 4
+  ret i32 %1
+}
+
+declare dso_local spir_func i32 @Foo50(i32, i32, i32) 
+
+
+define internal spir_func i32 @Bar17(ptr byval(%"tangle_group") %Group, i32 %Id) {
+entry:
+  %retval = alloca i32, align 4
+  %Id.addr = alloca i32, align 4
+  %MemberMask = alloca %"vec.16", align 16
+  %agg.tmp = alloca %"ss_sub_group_mask"
+  %agg.tmp1 = alloca %"tangle_group"
+  %Count = alloca i32, align 4
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %b = alloca i32, align 4
+  %MemberMask.ascast = addrspacecast ptr %MemberMask to ptr addrspace(4)
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  store i32 %Id, ptr %Id.addr, align 4
+  call spir_func void @Bar18(ptr addrspace(4) dead_on_unwind writable sret(%"ss_sub_group_mask") %agg.tmp.ascast, ptr byval(%"tangle_group") %agg.tmp1) 
+  call spir_func void @Bar19(ptr addrspace(4) dead_on_unwind writable sret(%"vec.16") align 16 %MemberMask.ascast, ptr byval(%"ss_sub_group_mask") %agg.tmp) 
+  store i32 0, ptr %Count, align 4
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 4
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  store i32 2, ptr %cleanup.dest.slot, align 4
+  br label %cleanup12
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, ptr %b, align 4
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %if.end8, %for.body
+  %1 = load i32, ptr %b, align 4
+  %cmp3 = icmp slt i32 %1, 32
+  br i1 %cmp3, label %for.body5, label %for.cond.cleanup4
+
+for.cond.cleanup4:                                ; preds = %for.cond2
+  store i32 5, ptr %cleanup.dest.slot, align 4
+  br label %cleanup
+
+for.body5:                                        ; preds = %for.cond2
+  %2 = load i32, ptr %i, align 4
+  %call = call spir_func align 4 ptr addrspace(4) @Bar20(ptr addrspace(4) align 16 %MemberMask.ascast, i32 %2) 
+  %3 = load i32, ptr addrspace(4) %call, align 4
+  %4 = load i32, ptr %b, align 4
+  %shl = shl i32 1, %4
+  %and = and i32 %3, %shl
+  %tobool = icmp ne i32 %and, 0
+  br i1 %tobool, label %if.then, label %if.end8
+
+if.then:                                          ; preds = %for.body5
+  %5 = load i32, ptr %Count, align 4
+  %6 = load i32, ptr %Id.addr, align 4
+  %cmp6 = icmp eq i32 %5, %6
+  br i1 %cmp6, label %if.then7, label %if.end
+
+if.end:                                           ; preds = %if.then
+  %7 = load i32, ptr %Count, align 4
+  %inc = add i32 %7, 1
+  store i32 %inc, ptr %Count, align 4
+  br label %if.end8
+
+if.end8:                                          ; preds = %if.end, %for.body5
+  %8 = load i32, ptr %b, align 4
+  %inc9 = add nsw i32 %8, 1
+  store i32 %inc9, ptr %b, align 4
+  br label %for.cond2
+
+if.then7:                                         ; preds = %if.then
+  %9 = load i32, ptr %i, align 4
+  %mul = mul nsw i32 %9, 32
+  %10 = load i32, ptr %b, align 4
+  %add = add nsw i32 %mul, %10
+  store i32 %add, ptr %retval, align 4
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then7, %for.cond.cleanup4
+  %cleanup.dest = load i32, ptr %cleanup.dest.slot, align 4
+  %cond = icmp eq i32 %cleanup.dest, 5
+  br i1 %cond, label %for.end, label %cleanup12
+
+for.end:                                          ; preds = %cleanup
+  %11 = load i32, ptr %i, align 4
+  %inc11 = add nsw i32 %11, 1
+  store i32 %inc11, ptr %i, align 4
+  br label %for.cond
+
+cleanup12:                                        ; preds = %cleanup, %for.cond.cleanup
+  %cleanup.dest13 = load i32, ptr %cleanup.dest.slot, align 4
+  %cond1 = icmp eq i32 %cleanup.dest13, 2
+  br i1 %cond1, label %for.end14, label %cleanup15
+
+for.end14:                                        ; preds = %cleanup12
+  %12 = load i32, ptr %Count, align 4
+  store i32 %12, ptr %retval, align 4
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  br label %cleanup15
+
+cleanup15:                                        ; preds = %cleanup12, %for.end14
+  %13 = load i32, ptr %retval, align 4
+  ret i32 %13
+}
+
+
+define internal spir_func void @Bar18(ptr addrspace(4) dead_on_unwind noalias writable sret(%"ss_sub_group_mask") %agg.result, ptr byval(%"tangle_group") %Group) {
+entry:
+  %Mask1 = bitcast ptr %Group to ptr
+  ret void
+}
+
+
+define internal spir_func void @Bar19(ptr addrspace(4) dead_on_unwind noalias writable sret(%"vec.16") align 16 %agg.result, ptr byval(%"ss_sub_group_mask") %Mask) {
+entry:
+  %TmpMArray = alloca %"struct.std::array.20", align 4
+  %agg.tmp = alloca %"range"
+  %i = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %cleanup.dest.slot2 = alloca i32, align 4
+  %TmpMArray.ascast = addrspacecast ptr %TmpMArray to ptr addrspace(4)
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  %Mask.ascast = addrspacecast ptr %Mask to ptr addrspace(4)
+  call spir_func void @Bar50(ptr addrspace(4) align 4 %TmpMArray.ascast) 
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp.ascast, i64 0) 
+  call spir_func void @Bar51(ptr addrspace(4) %Mask.ascast, ptr addrspace(4) align 4 %TmpMArray.ascast, ptr byval(%"range") %agg.tmp) 
+  store i32 0, ptr %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, ptr %i, align 4
+  %cmp = icmp slt i32 %0, 4
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, ptr %i, align 4
+  %conv = sext i32 %1 to i64
+  %call = call spir_func align 4 ptr addrspace(4) @Bar57(ptr addrspace(4) align 4 %TmpMArray.ascast, i64 %conv) 
+  %2 = load i32, ptr addrspace(4) %call, align 4
+  %3 = load i32, ptr %i, align 4
+  %call1 = call spir_func align 4 ptr addrspace(4) @Bar20(ptr addrspace(4) align 16 %agg.result, i32 %3) 
+  store i32 %2, ptr addrspace(4) %call1, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %4 = load i32, ptr %i, align 4
+  %inc = add nsw i32 %4, 1
+  store i32 %inc, ptr %i, align 4
+  br label %for.cond
+}
+
+
+define internal spir_func align 4 ptr addrspace(4) @Bar20(ptr addrspace(4) align 16 %this, i32 %i) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %i.addr = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i32 %i, ptr %i.addr, align 4
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %m_Data1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i32, ptr %i.addr, align 4
+  %conv = sext i32 %0 to i64
+  %call = call spir_func align 4 ptr addrspace(4) @_ZNSt5arrayIjLm4EEixEm(ptr addrspace(4) align 4 %m_Data1, i64 %conv) 
+  ret ptr addrspace(4) %call
+}
+
+
+define internal spir_func align 4 ptr addrspace(4) @_ZNSt5arrayIjLm4EEixEm(ptr addrspace(4) align 4 %this, i64 %__n) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %__n.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %__n, ptr %__n.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %_M_elems1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %__n.addr
+  %call = call spir_func align 4 ptr addrspace(4) @_ZNSt14__array_traitsIjLm4EE6_S_refERA4_Kjm(ptr addrspace(4) align 4 %_M_elems1, i64 %0) 
+  ret ptr addrspace(4) %call
+}
+
+
+define internal spir_func align 4 ptr addrspace(4) @_ZNSt14__array_traitsIjLm4EE6_S_refERA4_Kjm(ptr addrspace(4) align 4 %__t, i64 %__n) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %__t.addr = alloca ptr addrspace(4)
+  %__n.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %__t, ptr %__t.addr
+  store i64 %__n, ptr %__n.addr
+  %0 = load ptr addrspace(4), ptr %__t.addr
+  %1 = load i64, ptr %__n.addr
+  %arrayidx = getelementptr inbounds nuw [4 x i32], ptr addrspace(4) %0, i64 0, i64 %1
+  ret ptr addrspace(4) %arrayidx
+}
+
+
+define internal spir_func void @Bar50(ptr addrspace(4) align 4 %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = inttoptr i64 16 to ptr addrspace(4)
+  br label %arrayinit.body
+
+arrayinit.body:                                   ; preds = %arrayinit.body, %entry
+  %lsr.iv = phi i64 [ %lsr.iv.next, %arrayinit.body ], [ 0, %entry ]
+  %scevgep = getelementptr i8, ptr addrspace(4) %this1, i64 %lsr.iv
+  store i32 0, ptr addrspace(4) %scevgep, align 4
+  %lsr.iv.next = add nuw nsw i64 %lsr.iv, 4
+  %lsr.iv.next1 = inttoptr i64 %lsr.iv.next to ptr addrspace(4)
+  %arrayinit.done = icmp eq ptr addrspace(4) %lsr.iv.next1, %0
+  br i1 %arrayinit.done, label %arrayinit.end2, label %arrayinit.body
+
+arrayinit.end2:                                   ; preds = %arrayinit.body
+  ret void
+}
+
+
+define internal spir_func void @Bar51(ptr addrspace(4) %this, ptr addrspace(4) align 4 %bits, ptr byval(%"range") %pos) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %bits.addr = alloca ptr addrspace(4)
+  %cur_pos = alloca i64
+  %__range4 = alloca ptr addrspace(4)
+  %__begin0 = alloca ptr addrspace(4)
+  %__end0 = alloca ptr addrspace(4)
+  %cleanup.dest.slot = alloca i32, align 4
+  %elem = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"range"
+  %agg.tmp.ascast = addrspacecast ptr %agg.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %bits, ptr %bits.addr
+  %pos.ascast = addrspacecast ptr %pos to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %call = call spir_func i64 @Bar52(ptr addrspace(4) %pos.ascast, i32 0) 
+  store i64 %call, ptr %cur_pos
+  %0 = load ptr addrspace(4), ptr %bits.addr
+  store ptr addrspace(4) %0, ptr %__range4
+  %1 = load ptr addrspace(4), ptr %__range4
+  %call2 = call spir_func ptr addrspace(4) @Bar53(ptr addrspace(4) align 4 %1) 
+  store ptr addrspace(4) %call2, ptr %__begin0
+  %2 = load ptr addrspace(4), ptr %__range4
+  %call3 = call spir_func ptr addrspace(4) @Bar54(ptr addrspace(4) align 4 %2) 
+  store ptr addrspace(4) %call3, ptr %__end0
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %3 = load ptr addrspace(4), ptr %__begin0
+  %4 = load ptr addrspace(4), ptr %__end0
+  %cmp = icmp ne ptr addrspace(4) %3, %4
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %5 = load ptr addrspace(4), ptr %__begin0
+  store ptr addrspace(4) %5, ptr %elem
+  %6 = load i64, ptr %cur_pos
+  %call4 = call spir_func i32 @Bar55(ptr addrspace(4) %this1) 
+  %conv = zext i32 %call4 to i64
+  %cmp5 = icmp ult i64 %6, %conv
+  br i1 %cmp5, label %if.then, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %7 = load ptr addrspace(4), ptr %elem
+  store i32 0, ptr addrspace(4) %7, align 4
+  br label %if.end
+
+if.then:                                          ; preds = %for.body
+  %8 = load ptr addrspace(4), ptr %elem
+  %9 = load i64, ptr %cur_pos
+  call spir_func void @Foo46(ptr addrspace(4) %agg.tmp.ascast, i64 %9) 
+  call spir_func void @Bar56(ptr addrspace(4) %this1, ptr addrspace(4) align 4 %8, ptr byval(%"range") %agg.tmp) 
+  %10 = load i64, ptr %cur_pos
+  %add = add i64 %10, 32
+  store i64 %add, ptr %cur_pos
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %11 = load ptr addrspace(4), ptr %__begin0
+  %incdec.ptr = getelementptr inbounds nuw i32, ptr addrspace(4) %11, i32 1
+  store ptr addrspace(4) %incdec.ptr, ptr %__begin0
+  br label %for.cond
+}
+
+
+define internal spir_func align 4 ptr addrspace(4) @Bar57(ptr addrspace(4) align 4 %this, i64 %index) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %index.addr = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %index, ptr %index.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %MData1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %index.addr
+  %arrayidx = getelementptr inbounds nuw [4 x i32], ptr addrspace(4) %MData1, i64 0, i64 %0
+  ret ptr addrspace(4) %arrayidx
+}
+
+
+define internal spir_func i64 @Bar52(ptr addrspace(4) %this, i32 %dimension) {
+entry:
+  %this.addr.i = alloca ptr addrspace(4)
+  %dimension.addr.i = alloca i32, align 4
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %dimension.addr = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i32 %dimension, ptr %dimension.addr, align 4
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i32, ptr %dimension.addr, align 4
+  store ptr addrspace(4) %this1, ptr %this.addr.i
+  store i32 %0, ptr %dimension.addr.i, align 4
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load i32, ptr %dimension.addr, align 4
+  %idxprom = sext i32 %1 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array1, i64 0, i64 %idxprom
+  %2 = load i64, ptr addrspace(4) %arrayidx
+  ret i64 %2
+}
+
+
+define internal spir_func ptr addrspace(4) @Bar53(ptr addrspace(4) align 4 %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+
+
+  %this.addr1 = bitcast ptr %this.addr to ptr
+  %this.addr2 = bitcast ptr %this.addr1 to ptr
+  %this1 = load ptr addrspace(4), ptr %this.addr2
+
+
+
+;  %this1 = load ptr addrspace(4), ptr %this.addr
+  %MData1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %arraydecay2 = bitcast ptr addrspace(4) %MData1 to ptr addrspace(4)
+  ret ptr addrspace(4) %arraydecay2
+}
+
+
+define internal spir_func ptr addrspace(4) @Bar54(ptr addrspace(4) align 4 %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+
+
+  %this.addr1 = bitcast ptr %this.addr to ptr
+  %this.addr2 = bitcast ptr %this.addr1 to ptr
+  %this1 = load ptr addrspace(4), ptr %this.addr2
+
+;  %this1 = load ptr addrspace(4), ptr %this.addr
+  %MData1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %arraydecay2 = bitcast ptr addrspace(4) %MData1 to ptr addrspace(4)
+  %add.ptr = getelementptr inbounds nuw i32, ptr addrspace(4) %arraydecay2, i64 4
+  ret ptr addrspace(4) %add.ptr
+}
+
+
+define internal spir_func i32 @Bar55(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %bits_num = getelementptr inbounds nuw %"ss_sub_group_mask", ptr addrspace(4) %this1, i32 0, i32 1
+  %0 = load i64, ptr addrspace(4) %bits_num
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+
+define internal spir_func void @Bar56(ptr addrspace(4) %this, ptr addrspace(4) align 4 %bits, ptr byval(%"range") %pos) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %bits.addr = alloca ptr addrspace(4)
+  %Res = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %bits, ptr %bits.addr
+  %pos.ascast = addrspacecast ptr %pos to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Bits1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr addrspace(4) %Bits1
+  store i64 %0, ptr %Res
+  %bits_num = getelementptr inbounds nuw %"ss_sub_group_mask", ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load i64, ptr addrspace(4) %bits_num
+  %call = call spir_func i64 @Bar58(ptr addrspace(4) %this1, i64 %1) 
+  %2 = load i64, ptr %Res
+  %and = and i64 %2, %call
+  store i64 %and, ptr %Res
+  %call2 = call spir_func i64 @Bar52(ptr addrspace(4) %pos.ascast, i32 0) 
+  %call3 = call spir_func i32 @Bar55(ptr addrspace(4) %this1) 
+  %conv = zext i32 %call3 to i64
+  %cmp = icmp ult i64 %call2, %conv
+  br i1 %cmp, label %if.then, label %if.else
+
+if.else:                                          ; preds = %entry
+  %3 = load ptr addrspace(4), ptr %bits.addr
+  store i32 0, ptr addrspace(4) %3, align 4
+  br label %if.end11
+
+if.then:                                          ; preds = %entry
+  %call4 = call spir_func i64 @Bar52(ptr addrspace(4) %pos.ascast, i32 0) 
+  %cmp5 = icmp ugt i64 %call4, 0
+  br i1 %cmp5, label %if.then6, label %if.end
+
+if.then6:                                         ; preds = %if.then
+  %call7 = call spir_func i64 @Bar52(ptr addrspace(4) %pos.ascast, i32 0) 
+  %4 = load i64, ptr %Res
+  %shr = lshr i64 %4, %call7
+  store i64 %shr, ptr %Res
+  br label %if.end
+
+if.end:                                           ; preds = %if.then6, %if.then
+  %call8 = call spir_func i64 @Bar58(ptr addrspace(4) %this1, i64 32) 
+  %5 = load i64, ptr %Res
+  %and9 = and i64 %5, %call8
+  store i64 %and9, ptr %Res
+  %6 = load i64, ptr %Res
+  %conv10 = trunc i64 %6 to i32
+  %7 = load ptr addrspace(4), ptr %bits.addr
+  store i32 %conv10, ptr addrspace(4) %7, align 4
+  br label %if.end11
+
+if.end11:                                         ; preds = %if.else, %if.end
+  ret void
+}
+
+
+define internal spir_func i64 @Bar58(ptr addrspace(4) %this, i64 %bn) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %bn.addr = alloca i64
+  %one = alloca i64
+  %cleanup.dest.slot = alloca i32, align 4
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %bn, ptr %bn.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %bn.addr
+  %cmp = icmp ule i64 %0, 64
+  %1 = addrspacecast ptr addrspace(1) @.str.2 to ptr addrspace(4)
+  %2 = addrspacecast ptr addrspace(1) @.str.1 to ptr addrspace(4)
+  %3 = addrspacecast ptr addrspace(1) @__PRETTY_FUNCTION2 to ptr addrspace(4)
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  call spir_func void @__assert_fail(ptr addrspace(4) %1, ptr addrspace(4) %2, i32 327, ptr addrspace(4) %3) 
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  store i64 1, ptr %one
+  %4 = load i64, ptr %bn.addr
+  %cmp2 = icmp eq i64 %4, 64
+  br i1 %cmp2, label %if.then, label %if.end
+
+if.end:                                           ; preds = %cond.end
+  %5 = load i64, ptr %one
+  %6 = load i64, ptr %bn.addr
+  %shl = shl i64 %5, %6
+  %7 = load i64, ptr %one
+  %sub3 = sub i64 %shl, %7
+  store i64 %sub3, ptr %retval
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  br label %cleanup
+
+if.then:                                          ; preds = %cond.end
+  %8 = load i64, ptr %one
+  %sub = sub i64 0, %8
+  store i64 %sub, ptr %retval
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end, %if.then
+  %9 = load i64, ptr %retval
+  ret i64 %9
+}
+
+
+
+
+define internal spir_func void @Foo11(ptr addrspace(4) %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Foo60(ptr addrspace(4) %this1, i64 0) 
+  ret void
+}
+
+
+define internal spir_func void @Foo60(ptr addrspace(4) %this, i64 %dim0) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %dim0.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %dim0, ptr %dim0.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %dim0.addr
+  store i64 %0, ptr addrspace(4) %common_array1
+  ret void
+}
+
+
+define internal spir_func i32 @Foo59(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"range") %local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %LocalId = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %x.addr.ascast = addrspacecast ptr %x.addr to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %call = call spir_func i32 @Foo48(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"range") %agg.tmp1) 
+  store i32 %call, ptr %LocalId, align 4
+  %0 = addrspacecast ptr addrspace(1) @_ZSt6ignore to ptr addrspace(4)
+  %call2 = call spir_func align 1 ptr addrspace(4) @Bar15(ptr addrspace(4) align 1 %0, ptr addrspace(4) %g.ascast) 
+  %call3 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %x.addr.ascast) 
+  %1 = load i32, ptr %LocalId, align 4
+  %call4 = call spir_func i32 @Foo50(i32 3, i32 %call3, i32 %1) 
+  ret i32 %call4
+}
+
+
+define internal spir_func i32 @Foo47(ptr byval(%"ss_sub_group_mask") %Mask) {
+entry:
+  %retval = alloca i32, align 4
+  %MemberMask = alloca %"vec.16", align 16
+  %agg.tmp = alloca %"ss_sub_group_mask"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %MemberMask.ascast = addrspacecast ptr %MemberMask to ptr addrspace(4)
+  call spir_func void @Bar19(ptr addrspace(4) dead_on_unwind writable sret(%"vec.16") align 16 %MemberMask.ascast, ptr byval(%"ss_sub_group_mask") %agg.tmp) 
+  %call = call spir_func <4 x i32> @Bar59(ptr addrspace(4) align 16 %MemberMask.ascast) 
+  %call1 = call spir_func i32 @_Z37__spirv_GroupNonUniformBallotBitCountN5__spv5Scope4FlagEiDv4_j(i32 3, i32 2, <4 x i32> %call) 
+  ret i32 %call1
+}
+
+
+define internal spir_func <4 x i32> @Bar59(ptr addrspace(4) align 16 %x) {
+entry:
+  %retval = alloca <4 x i32>, align 16
+  %x.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %x, ptr %x.addr
+  %0 = load ptr addrspace(4), ptr %x.addr
+  %call = call spir_func <4 x i32> @Bar60(ptr addrspace(4) align 16 %0) 
+  ret <4 x i32> %call
+}
+
+declare dso_local spir_func i32 @_Z37__spirv_GroupNonUniformBallotBitCountN5__spv5Scope4FlagEiDv4_j(i32, i32, <4 x i32>) 
+
+
+define internal spir_func <4 x i32> @Bar60(ptr addrspace(4) align 16 %from) {
+entry:
+  %retval = alloca <4 x i32>, align 16
+  %from.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %from, ptr %from.addr
+  %0 = load ptr addrspace(4), ptr %from.addr
+  %1 = load <4 x i32>, ptr addrspace(4) %0, align 16
+  ret <4 x i32> %1
+}
+
+
+define internal spir_func i32 @Foo52(ptr byval(%"tangle_group") %g, i32 %x, i32 %delta) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %delta.addr = alloca i32, align 4
+  %TargetLocalId = alloca %"range"
+  %TargetId = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp3 = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %x.addr.ascast = addrspacecast ptr %x.addr to ptr addrspace(4)
+  %TargetLocalId.ascast = addrspacecast ptr %TargetLocalId to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %delta, ptr %delta.addr, align 4
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %TargetLocalId.ascast, ptr addrspace(4) %g.ascast) 
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %TargetLocalId.ascast, i32 0) 
+  %0 = load i64, ptr addrspace(4) %call
+  %1 = load i32, ptr %delta.addr, align 4
+  %conv = zext i32 %1 to i64
+  %cmp = icmp uge i64 %0, %conv
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, ptr %delta.addr, align 4
+  %conv1 = zext i32 %2 to i64
+  %call2 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %TargetLocalId.ascast, i32 0) 
+  %3 = load i64, ptr addrspace(4) %call2
+  %sub = sub i64 %3, %conv1
+  store i64 %sub, ptr addrspace(4) %call2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %call4 = call spir_func i32 @Foo48(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"range") %agg.tmp3) 
+  store i32 %call4, ptr %TargetId, align 4
+  %call5 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %x.addr.ascast) 
+  %4 = load i32, ptr %TargetId, align 4
+  %call6 = call spir_func i32 @Foo50(i32 3, i32 %call5, i32 %4) 
+  ret i32 %call6
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %this, i32 %dimension) {
+entry:
+  %this.addr.i = alloca ptr addrspace(4)
+  %dimension.addr.i = alloca i32, align 4
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %dimension.addr = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i32 %dimension, ptr %dimension.addr, align 4
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i32, ptr %dimension.addr, align 4
+  store ptr addrspace(4) %this1, ptr %this.addr.i
+  store i32 %0, ptr %dimension.addr.i, align 4
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load i32, ptr %dimension.addr, align 4
+  %idxprom = sext i32 %1 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array1, i64 0, i64 %idxprom
+  ret ptr addrspace(4) %arrayidx
+}
+
+
+define internal spir_func i32 @Foo72(ptr byval(%"tangle_group") %g, i32 %x, i32 %delta) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %delta.addr = alloca i32, align 4
+  %TargetLocalId = alloca %"range"
+  %TargetId = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp6 = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %x.addr.ascast = addrspacecast ptr %x.addr to ptr addrspace(4)
+  %TargetLocalId.ascast = addrspacecast ptr %TargetLocalId to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i32 %delta, ptr %delta.addr, align 4
+  call spir_func void @Foo51(ptr addrspace(4) dead_on_unwind writable sret(%"range") %TargetLocalId.ascast, ptr addrspace(4) %g.ascast) 
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %TargetLocalId.ascast, i32 0) 
+  %0 = load i64, ptr addrspace(4) %call
+  %1 = load i32, ptr %delta.addr, align 4
+  %conv = zext i32 %1 to i64
+  %add = add i64 %0, %conv
+  %call1 = call spir_func i32 @Bar61(ptr addrspace(4) %g.ascast) 
+  %conv2 = zext i32 %call1 to i64
+  %cmp = icmp ult i64 %add, %conv2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, ptr %delta.addr, align 4
+  %conv3 = zext i32 %2 to i64
+  %call4 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %TargetLocalId.ascast, i32 0) 
+  %3 = load i64, ptr addrspace(4) %call4
+  %add5 = add i64 %3, %conv3
+  store i64 %add5, ptr addrspace(4) %call4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %call7 = call spir_func i32 @Foo48(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"range") %agg.tmp6) 
+  store i32 %call7, ptr %TargetId, align 4
+  %call8 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %x.addr.ascast) 
+  %4 = load i32, ptr %TargetId, align 4
+  %call9 = call spir_func i32 @Foo50(i32 3, i32 %call8, i32 %4) 
+  ret i32 %call9
+}
+
+
+define internal spir_func i32 @Bar61(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  call spir_func void @Foo97(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) %this1) 
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %ref.tmp.ascast, i32 0) 
+  %0 = load i64, ptr addrspace(4) %call
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+
+define internal spir_func void @Foo97(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Mask1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %call = call spir_func i32 @Bar62(ptr addrspace(4) %Mask1) 
+  %conv = zext i32 %call to i64
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 %conv) 
+  ret void
+}
+
+
+define internal spir_func i32 @Bar62(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i32, align 4
+  %this.addr = alloca ptr addrspace(4)
+  %count = alloca i32, align 4
+  %word = alloca i64
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  store i32 0, ptr %count, align 4
+  %Bits1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr addrspace(4) %Bits1
+  %bits_num = getelementptr inbounds nuw %"ss_sub_group_mask", ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load i64, ptr addrspace(4) %bits_num
+  %call = call spir_func i64 @Bar58(ptr addrspace(4) %this1, i64 %1) 
+  %and = and i64 %0, %call
+  store i64 %and, ptr %word
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %2 = load i64, ptr %word
+  %tobool = icmp ne i64 %2, 0
+  br i1 %tobool, label %while.body, label %while.end
+
+while.end:                                        ; preds = %while.cond
+  %3 = load i32, ptr %count, align 4
+  ret i32 %3
+
+while.body:                                       ; preds = %while.cond
+  %4 = load i64, ptr %word
+  %sub = sub i64 %4, 1
+  %5 = load i64, ptr %word
+  %and2 = and i64 %5, %sub
+  store i64 %and2, ptr %word
+  %6 = load i32, ptr %count, align 4
+  %inc = add i32 %6, 1
+  store i32 %inc, ptr %count, align 4
+  br label %while.cond
+}
+
+
+define internal spir_func void @Foo9(ptr addrspace(4) %this, i64 %dim0) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %dim0.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %dim0, ptr %dim0.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i64, ptr %dim0.addr
+  call spir_func void @Foo60(ptr addrspace(4) %this1, i64 %0) 
+  ret void
+}
+
+
+define internal spir_func i32 @Foo61(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar63(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %4, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar63(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar64(ptr byval(%"tangle_group") %agg.tmp, i32 %4) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar64(ptr byval(%"tangle_group") %0, i32 %x)   {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %Arg = alloca i32, align 4
+  %Ret = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %2 = load i32, ptr %x.addr, align 4
+  store i32 %2, ptr %Arg, align 4
+  %3 = load i32, ptr %Arg, align 4
+  %call = call spir_func i32 @_Z27__spirv_GroupNonUniformIAddIiET_N5__spv5Scope4FlagEjS0_(i32 3, i32 1, i32 %3) 
+  store i32 %call, ptr %Ret, align 4
+  %4 = load i32, ptr %Ret, align 4
+  ret i32 %4
+}
+
+declare dso_local spir_func i32 @_Z27__spirv_GroupNonUniformIAddIiET_N5__spv5Scope4FlagEjS0_(i32, i32, i32) 
+
+
+define internal spir_func i32 @Bar12(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar65(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %4, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar65(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar66(ptr byval(%"tangle_group") %agg.tmp, i32 %4) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar66(ptr byval(%"tangle_group") %0, i32 %x)   {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %Arg = alloca i32, align 4
+  %Ret = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %2 = load i32, ptr %x.addr, align 4
+  store i32 %2, ptr %Arg, align 4
+  %3 = load i32, ptr %Arg, align 4
+  %call = call spir_func i32 @_Z27__spirv_GroupNonUniformIAddIiET_N5__spv5Scope4FlagEjS0_(i32 3, i32 2, i32 %3) 
+  store i32 %call, ptr %Ret, align 4
+  %4 = load i32, ptr %Ret, align 4
+  ret i32 %4
+}
+
+
+define internal spir_func i32 @Bar11(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"nd_item", align 1
+  %agg.tmp2 = alloca %"nd_item", align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar67(ptr byval(%"tangle_group") %agg.tmp, ptr byval(%"nd_item") align 1 %agg.tmp1, i32 %4, ptr byval(%"nd_item") align 1 %agg.tmp2) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar67(ptr byval(%"tangle_group") %g, ptr byval(%"nd_item") align 1 %0, i32 %x, ptr byval(%"nd_item") align 1 %1){
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %2 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %3 = addrspacecast ptr %1 to ptr addrspace(4)
+  %4 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar68(ptr byval(%"tangle_group") %agg.tmp, i32 %4) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar68(ptr byval(%"tangle_group") %0, i32 %x)   {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %Arg = alloca i32, align 4
+  %Ret = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %2 = load i32, ptr %x.addr, align 4
+  store i32 %2, ptr %Arg, align 4
+  %3 = load i32, ptr %Arg, align 4
+  %call = call spir_func i32 @_Z27__spirv_GroupNonUniformIAddIiET_N5__spv5Scope4FlagEjS0_(i32 3, i32 0, i32 %3) 
+  store i32 %call, ptr %Ret, align 4
+  %4 = load i32, ptr %Ret, align 4
+  ret i32 %4
+}
+
+
+define internal spir_func zeroext i1 @Foo66(ptr byval(%"tangle_group") %0, i1 zeroext %pred) {
+entry:
+  %retval = alloca i1, align 1
+  %pred.addr = alloca i8, align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  %storedv = zext i1 %pred to i8
+  store i8 %storedv, ptr %pred.addr, align 1
+  %2 = load i8, ptr %pred.addr, align 1  
+  %loadedv = trunc i8 %2 to i1
+  %call = call spir_func zeroext i1 @Foo99(i32 3, i1 zeroext %loadedv) 
+  ret i1 %call
+}
+
+declare dso_local spir_func zeroext i1 @Foo99(i32, i1 zeroext) 
+
+
+define internal spir_func zeroext i1 @Bar10(ptr byval(%"tangle_group") %0, i1 zeroext %pred) {
+entry:
+  %retval = alloca i1, align 1
+  %pred.addr = alloca i8, align 1
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  %storedv = zext i1 %pred to i8
+  store i8 %storedv, ptr %pred.addr, align 1
+  %2 = load i8, ptr %pred.addr, align 1  
+  %loadedv = trunc i8 %2 to i1
+  %call = call spir_func zeroext i1 @_Z26__spirv_GroupNonUniformAnyN5__spv5Scope4FlagEb(i32 3, i1 zeroext %loadedv) 
+  ret i1 %call
+}
+
+declare dso_local spir_func zeroext i1 @_Z26__spirv_GroupNonUniformAnyN5__spv5Scope4FlagEb(i32, i1 zeroext) 
+
+
+define internal spir_func void @Foo98(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr byval(%"range") %0, i64 %linear_id)   {
+entry:
+  %linear_id.addr = alloca i64
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store i64 %linear_id, ptr %linear_id.addr
+  %2 = load i64, ptr %linear_id.addr
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %2) 
+  ret void
+}
+
+
+define internal spir_func i32 @Bar69(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"range") %local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %agg.tmp1 = alloca %"range"
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func i32 @Bar70(ptr byval(%"tangle_group") %agg.tmp, i32 %0, ptr byval(%"range") %agg.tmp1) 
+  ret i32 %call
+}
+
+
+define internal spir_func i32 @Bar70(ptr byval(%"tangle_group") %g, i32 %x, ptr byval(%"range") %local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %VecId = alloca %"range"
+  %OCLX = alloca i32, align 4
+  %WideOCLX = alloca i32, align 4
+  %OCLId = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %VecId.ascast = addrspacecast ptr %VecId to ptr addrspace(4)
+  %OCLX.ascast = addrspacecast ptr %OCLX to ptr addrspace(4)
+  %WideOCLX.ascast = addrspacecast ptr %WideOCLX to ptr addrspace(4)
+  %OCLId.ascast = addrspacecast ptr %OCLId to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  %local_id.ascast = addrspacecast ptr %local_id to ptr addrspace(4)
+  %0 = load i32, ptr %x.addr, align 4
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %local_id.ascast, i32 0) 
+  %1 = load i64, ptr addrspace(4) %call
+  %call1 = call spir_func i32 @Bar71(ptr byval(%"tangle_group") %agg.tmp, i32 %0, i64 %1) 
+  ret i32 %call1
+}
+
+
+define internal spir_func i32 @Bar71(ptr byval(%"tangle_group") %g, i32 %x, i64 %local_id) {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %local_id.addr = alloca i64
+  %LocalId = alloca i32, align 4
+  %agg.tmp = alloca %"tangle_group"
+  %GroupLocalId = alloca i32, align 4
+  %OCLX = alloca i32, align 4
+  %WideOCLX = alloca i32, align 4
+  %OCLId = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %x.addr.ascast = addrspacecast ptr %x.addr to ptr addrspace(4)
+  %GroupLocalId.ascast = addrspacecast ptr %GroupLocalId to ptr addrspace(4)
+  store i32 %x, ptr %x.addr, align 4
+  store i64 %local_id, ptr %local_id.addr
+  %0 = load i64, ptr %local_id.addr
+  %conv = trunc i64 %0 to i32
+  %call = call spir_func i32 @Bar17(ptr byval(%"tangle_group") %agg.tmp, i32 %conv) 
+  store i32 %call, ptr %LocalId, align 4
+  %1 = load i32, ptr %LocalId, align 4
+  store i32 %1, ptr %GroupLocalId, align 4
+  %call1 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %x.addr.ascast) 
+  store i32 %call1, ptr %OCLX, align 4
+  %2 = load i32, ptr %OCLX, align 4
+  store i32 %2, ptr %WideOCLX, align 4
+  %call2 = call spir_func i32 @Foo49(ptr addrspace(4) align 4 %GroupLocalId.ascast) 
+  store i32 %call2, ptr %OCLId, align 4
+  %3 = load i32, ptr %WideOCLX, align 4
+  %4 = load i32, ptr %OCLId, align 4
+  %call3 = call spir_func i32 @_Z32__spirv_GroupNonUniformBroadcastIjjET_N5__spv5Scope4FlagES0_T0_(i32 3, i32 %3, i32 %4) 
+  ret i32 %call3
+}
+
+declare dso_local spir_func i32 @_Z32__spirv_GroupNonUniformBroadcastIjjET_N5__spv5Scope4FlagES0_T0_(i32, i32, i32) 
+
+
+define internal spir_func void @Foo96(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) align 1 %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %call = call spir_func i32 @_Z33__spirv_SubgroupLocalInvocationIdv() 
+  %conv = zext i32 %call to i64
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %conv) 
+  ret void
+}
+
+
+define internal spir_func i32 @_Z33__spirv_SubgroupLocalInvocationIdv()   {
+entry:
+  %retval = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4
+  ret i32 %0
+}
+
+
+define internal spir_func i64 @Foo77(ptr addrspace(4) %this, ptr byval(%"range") %Id) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %Result = alloca i64
+  %ref.tmp = alloca %class.anon.15
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %Result.ascast = addrspacecast ptr %Result to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %Id.ascast = addrspacecast ptr %Id to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  store i64 0, ptr %Result
+  %0 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %0
+  %Result2 = getelementptr inbounds %class.anon.15, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %Result.ascast, ptr %Result2
+  %Id3 = getelementptr inbounds %class.anon.15, ptr %ref.tmp, i32 0, i32 2
+  store ptr addrspace(4) %Id.ascast, ptr %Id3
+  call spir_func void @Foo79(ptr addrspace(4) %ref.tmp.ascast) 
+  %1 = load i64, ptr %Result
+  ret i64 %1
+}
+
+
+define internal spir_func ptr addrspace(1) @Foo78(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(1)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load ptr addrspace(1), ptr addrspace(4) %0
+  ret ptr addrspace(1) %1
+}
+
+
+define internal spir_func void @Foo79(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Foo80(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func void @Foo80(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Foo81(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %this) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  ret i64 0
+}
+
+
+define internal spir_func void @Foo81(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %Result = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %Result
+  %3 = load i64, ptr addrspace(4) %2
+  %call = call spir_func ptr addrspace(4) @Bar72(ptr addrspace(4) %1) 
+  %4 = load i64, ptr %I.addr
+  %conv = trunc i64 %4 to i32
+  %call2 = call spir_func i64 @Foo37(ptr addrspace(4) %call, i32 %conv) 
+  %mul = mul i64 %3, %call2
+  %Id = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 2
+  %5 = load ptr addrspace(4), ptr addrspace(4) %Id
+  %6 = load i64, ptr %I.addr
+  %conv3 = trunc i64 %6 to i32
+  %call4 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %5, i32 %conv3) 
+  %7 = load i64, ptr addrspace(4) %call4
+  %add = add i64 %mul, %7
+  %Result5 = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 1
+  %8 = load ptr addrspace(4), ptr addrspace(4) %Result5
+  store i64 %add, ptr addrspace(4) %8
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Bar72(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  ret ptr addrspace(4) %MemRange
+}
+
+
+define internal spir_func i64 @Foo37(ptr addrspace(4) %this, i32 %dimension) {
+entry:
+  %this.addr.i = alloca ptr addrspace(4)
+  %dimension.addr.i = alloca i32, align 4
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %dimension.addr = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i32 %dimension, ptr %dimension.addr, align 4
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = load i32, ptr %dimension.addr, align 4
+  store ptr addrspace(4) %this1, ptr %this.addr.i
+  store i32 %0, ptr %dimension.addr.i, align 4
+  %this1.i = load ptr addrspace(4), ptr %this.addr.i
+  %common_array1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load i32, ptr %dimension.addr, align 4
+  %idxprom = sext i32 %1 to i64
+  %arrayidx = getelementptr inbounds [1 x i64], ptr addrspace(4) %common_array1, i64 0, i64 %idxprom
+  %2 = load i64, ptr addrspace(4) %arrayidx
+  ret i64 %2
+}
+
+
+define internal spir_func void @Foo95(ptr byval(%"tangle_group") %g, i32 %FenceScope, i32 %Order) {
+entry:
+  %FenceScope.addr = alloca i32, align 4
+  %Order.addr = alloca i32, align 4
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  store i32 %FenceScope, ptr %FenceScope.addr, align 4
+  store i32 %Order, ptr %Order.addr, align 4
+  %0 = load i32, ptr %FenceScope.addr, align 4
+  %call = call spir_func i32 @Bar73(i32 %0) 
+  %1 = load i32, ptr %Order.addr, align 4
+  %call1 = call spir_func i32 @Bar74(i32 %1) 
+  %or = or i32 %call1, 128
+  %or2 = or i32 %or, 256
+  %or3 = or i32 %or2, 512
+  call spir_func void @_Z21__spirv_MemoryBarrierjj(i32 %call, i32 %or3) 
+  ret void
+}
+
+
+define internal spir_func i32 @Bar73(i32 %Scope){
+entry:
+  %retval = alloca i32, align 4
+  %Scope.addr = alloca i32, align 4
+  store i32 %Scope, ptr %Scope.addr, align 4
+  %0 = load i32, ptr %Scope.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 4, label %sw.bb4
+  ]
+
+sw.bb4:                                           ; preds = %entry
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+sw.bb3:                                           ; preds = %entry
+  store i32 1, ptr %retval, align 4
+  br label %return
+
+sw.bb2:                                           ; preds = %entry
+  store i32 2, ptr %retval, align 4
+  br label %return
+
+sw.bb1:                                           ; preds = %entry
+  store i32 3, ptr %retval, align 4
+  br label %return
+
+sw.bb:                                            ; preds = %entry
+  store i32 4, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %sw.bb4, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  %1 = load i32, ptr %retval, align 4
+  ret i32 %1
+
+sw.epilog:                                        ; preds = %entry
+  unreachable
+}
+
+
+define internal spir_func i32 @Bar74(i32 %Order){
+entry:
+  %retval = alloca i32, align 4
+  %Order.addr = alloca i32, align 4
+  %SpvOrder = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store i32 %Order, ptr %Order.addr, align 4
+  store i32 0, ptr %SpvOrder, align 4
+  %0 = load i32, ptr %Order.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 2, label %sw.bb1
+    i32 1, label %sw.bb1
+    i32 3, label %sw.bb2
+    i32 4, label %sw.bb3
+    i32 5, label %sw.bb4
+  ]
+
+sw.bb4:                                           ; preds = %entry
+  store i32 16, ptr %SpvOrder, align 4
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %entry
+  store i32 8, ptr %SpvOrder, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  store i32 4, ptr %SpvOrder, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry, %entry
+  store i32 2, ptr %SpvOrder, align 4
+  br label %sw.epilog
+
+sw.bb:                                            ; preds = %entry
+  store i32 0, ptr %SpvOrder, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %sw.bb4, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb, %entry
+  %1 = load i32, ptr %SpvOrder, align 4
+  %or = or i32 %1, 128
+  %or5 = or i32 %or, 256
+  %or6 = or i32 %or5, 512
+  ret i32 %or6
+}
+
+declare dso_local spir_func void @_Z21__spirv_MemoryBarrierjj(i32, i32) 
+
+
+define internal spir_func i64 @Foo93(ptr addrspace(4) %this, ptr byval(%"range") %Id) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %Result = alloca i64
+  %ref.tmp = alloca %class.anon.15
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %Result.ascast = addrspacecast ptr %Result to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %Id.ascast = addrspacecast ptr %Id to ptr addrspace(4)
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  store i64 0, ptr %Result
+  %0 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %0
+  %Result2 = getelementptr inbounds %class.anon.15, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %Result.ascast, ptr %Result2
+  %Id3 = getelementptr inbounds %class.anon.15, ptr %ref.tmp, i32 0, i32 2
+  store ptr addrspace(4) %Id.ascast, ptr %Id3
+  call spir_func void @Bar75(ptr addrspace(4) %ref.tmp.ascast) 
+  %1 = load i64, ptr %Result
+  ret i64 %1
+}
+
+
+define internal spir_func ptr addrspace(1) @Foo94(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(1)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = getelementptr inbounds nuw %"accessor", ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load ptr addrspace(1), ptr addrspace(4) %0
+  ret ptr addrspace(1) %1
+}
+
+
+define internal spir_func void @Bar75(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Bar76(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func void @Bar76(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Bar767(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func void @Bar767(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %Result = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %Result
+  %3 = load i64, ptr addrspace(4) %2
+  %call = call spir_func ptr addrspace(4) @Bar78(ptr addrspace(4) %1) 
+  %4 = load i64, ptr %I.addr
+  %conv = trunc i64 %4 to i32
+  %call2 = call spir_func i64 @Foo37(ptr addrspace(4) %call, i32 %conv) 
+  %mul = mul i64 %3, %call2
+  %Id = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 2
+  %5 = load ptr addrspace(4), ptr addrspace(4) %Id
+  %6 = load i64, ptr %I.addr
+  %conv3 = trunc i64 %6 to i32
+  %call4 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %5, i32 %conv3) 
+  %7 = load i64, ptr addrspace(4) %call4
+  %add = add i64 %mul, %7
+  %Result5 = getelementptr inbounds nuw %class.anon.15, ptr addrspace(4) %this1, i32 0, i32 1
+  %8 = load ptr addrspace(4), ptr addrspace(4) %Result5
+  store i64 %add, ptr addrspace(4) %8
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Bar78(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  ret ptr addrspace(4) %MemRange
+}
+
+
+define internal spir_func void @Foo44(ptr addrspace(4) dead_on_unwind noalias writable sret(%"ss_sub_group_mask") %agg.result, ptr byval(%"nd_item") align 1 %g, i1 zeroext %predicate) {
+entry:
+  %predicate.addr = alloca i8, align 1
+  %res = alloca <4 x i32>, align 16
+  %val = alloca i64
+  %ref.tmp = alloca %"range"
+  %cleanup.dest.slot = alloca i32, align 4
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %g.ascast = addrspacecast ptr %g to ptr addrspace(4)
+  %storedv = zext i1 %predicate to i8
+  store i8 %storedv, ptr %predicate.addr, align 1
+  %0 = load i8, ptr %predicate.addr, align 1  
+  %loadedv = trunc i8 %0 to i1
+  %call = call spir_func <4 x i32> @_Z29__spirv_GroupNonUniformBallotjb(i32 3, i1 zeroext %loadedv) 
+  store <4 x i32> %call, ptr %res, align 16
+  %1 = load <4 x i32>, ptr %res, align 16
+  %vecext = extractelement <4 x i32> %1, i32 0
+  %conv = zext i32 %vecext to i64
+  store i64 %conv, ptr %val
+  %2 = load <4 x i32>, ptr %res, align 16
+  %vecext1 = extractelement <4 x i32> %2, i32 1
+  %conv2 = zext i32 %vecext1 to i64
+  %shl = shl i64 %conv2, 32
+  %3 = load i64, ptr %val
+  %or = or i64 %3, %shl
+  store i64 %or, ptr %val
+  %4 = load i64, ptr %val
+  call spir_func void @Bar79(ptr addrspace(4) dead_on_unwind writable sret(%"range") %ref.tmp.ascast, ptr addrspace(4) align 1 %g.ascast) 
+  %call3 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %ref.tmp.ascast, i32 0) 
+  %5 = load i64, ptr addrspace(4) %call3
+  call spir_func void @Bar80(ptr addrspace(4) dead_on_unwind writable sret(%"ss_sub_group_mask") %agg.result, i64 %4, i64 %5) 
+  ret void
+}
+
+
+define internal spir_func void @Foo45(ptr addrspace(4) %this, ptr byval(%"ss_sub_group_mask") %m) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Mask1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  ret void
+}
+
+declare dso_local spir_func <4 x i32> @_Z29__spirv_GroupNonUniformBallotjb(i32, i1 zeroext) 
+
+
+define internal spir_func void @Bar79(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result, ptr addrspace(4) align 1 %this) {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %call = call spir_func i32 @_Z23__spirv_SubgroupMaxSizev() 
+  %conv = zext i32 %call to i64
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 %conv) 
+  ret void
+}
+
+
+define internal spir_func void @Bar80(ptr addrspace(4) dead_on_unwind noalias writable sret(%"ss_sub_group_mask") %agg.result, i64 %Bits, i64 %BitsNum) {
+entry:
+  %Bits.addr = alloca i64
+  %BitsNum.addr = alloca i64
+  store i64 %Bits, ptr %Bits.addr
+  store i64 %BitsNum, ptr %BitsNum.addr
+  %0 = load i64, ptr %Bits.addr
+  %1 = load i64, ptr %BitsNum.addr
+  call spir_func void @Bar81(ptr addrspace(4) %agg.result, i64 %0, i64 %1) 
+  ret void
+}
+
+
+define internal spir_func void @Bar81(ptr addrspace(4) %this, i64 %rhs, i64 %bn) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %rhs.addr = alloca i64
+  %bn.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %rhs, ptr %rhs.addr
+  store i64 %bn, ptr %bn.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Bits1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load i64, ptr %rhs.addr
+  %1 = load i64, ptr %bn.addr
+  %call = call spir_func i64 @Bar58(ptr addrspace(4) %this1, i64 %1) 
+  %and = and i64 %0, %call
+  store i64 %and, ptr addrspace(4) %Bits1
+  %bits_num = getelementptr inbounds nuw %"ss_sub_group_mask", ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load i64, ptr %bn.addr
+  store i64 %2, ptr addrspace(4) %bits_num
+  %bits_num2 = getelementptr inbounds nuw %"ss_sub_group_mask", ptr addrspace(4) %this1, i32 0, i32 1
+  %3 = load i64, ptr addrspace(4) %bits_num2
+  %cmp = icmp ule i64 %3, 64
+  %4 = addrspacecast ptr addrspace(1) @.str to ptr addrspace(4)
+  %5 = addrspacecast ptr addrspace(1) @.str.1 to ptr addrspace(4)
+  %6 = addrspacecast ptr addrspace(1) @__PRETTY_FUNCTION1 to ptr addrspace(4)
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  call spir_func void @__assert_fail(ptr addrspace(4) %4, ptr addrspace(4) %5, i32 324, ptr addrspace(4) %6) 
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  ret void
+}
+
+
+define internal spir_func i32 @_Z23__spirv_SubgroupMaxSizev()   {
+entry:
+  %retval = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupMaxSize, align 4
+  ret i32 %0
+}
+
+
+define internal spir_func void @Init6(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @Inv1(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Inv1(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @Inv2() 
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @Inv2() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z28__spirv_GlobalInvocationId_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @Foo7(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @Foo8(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Init1(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @Inv3(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Init2(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @InitSize1(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Init3(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @InitSize2(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Init4(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @InitSize3(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Init5(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @InitSize4(ptr addrspace(4) dead_on_unwind writable sret(%"range") %agg.result) 
+  ret void
+}
+
+
+define internal spir_func void @Foo23(ptr addrspace(4) dead_on_unwind noalias writable sret(%"group") %agg.result, ptr addrspace(4) %Global, ptr addrspace(4) %Local, ptr addrspace(4) %Group, ptr addrspace(4) %Index) {
+entry:
+  %Global.addr = alloca ptr addrspace(4)
+  %Local.addr = alloca ptr addrspace(4)
+  %Group.addr = alloca ptr addrspace(4)
+  %Index.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"range"
+  store ptr addrspace(4) %Global, ptr %Global.addr
+  store ptr addrspace(4) %Local, ptr %Local.addr
+  store ptr addrspace(4) %Group, ptr %Group.addr
+  store ptr addrspace(4) %Index, ptr %Index.addr
+  %0 = load ptr addrspace(4), ptr %Global.addr
+  %1 = load ptr addrspace(4), ptr %Local.addr
+  %2 = load ptr addrspace(4), ptr %Group.addr
+  %3 = load ptr addrspace(4), ptr %Index.addr
+  call spir_func void @Bar82(ptr addrspace(4) %agg.result, ptr addrspace(4) %0, ptr addrspace(4) %1, ptr byval(%"range") %agg.tmp, ptr addrspace(4) %3) 
+  ret void
+}
+
+
+define internal spir_func void @Foo24(ptr addrspace(4) dead_on_unwind noalias writable sret(%"item") %agg.result, ptr addrspace(4) %Extent, ptr addrspace(4) %Index, ptr addrspace(4) %Offset) {
+entry:
+  %Extent.addr = alloca ptr addrspace(4)
+  %Index.addr = alloca ptr addrspace(4)
+  %Offset.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %Extent, ptr %Extent.addr
+  store ptr addrspace(4) %Index, ptr %Index.addr
+  store ptr addrspace(4) %Offset, ptr %Offset.addr
+  %0 = load ptr addrspace(4), ptr %Extent.addr
+  %1 = load ptr addrspace(4), ptr %Index.addr
+  %2 = load ptr addrspace(4), ptr %Offset.addr
+  call spir_func void @Foo29(ptr addrspace(4) %agg.result, ptr addrspace(4) %0, ptr addrspace(4) %1, ptr addrspace(4) %2) 
+  ret void
+}
+
+
+define internal spir_func void @Foo25(ptr addrspace(4) dead_on_unwind noalias writable sret(%"item.22") %agg.result, ptr addrspace(4) %Extent, ptr addrspace(4) %Index) {
+entry:
+  %Extent.addr = alloca ptr addrspace(4)
+  %Index.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %Extent, ptr %Extent.addr
+  store ptr addrspace(4) %Index, ptr %Index.addr
+  %0 = load ptr addrspace(4), ptr %Extent.addr
+  %1 = load ptr addrspace(4), ptr %Index.addr
+  call spir_func void @Foo27(ptr addrspace(4) %agg.result, ptr addrspace(4) %0, ptr addrspace(4) %1) 
+  ret void
+}
+
+
+define internal spir_func void @Foo26(ptr addrspace(4) dead_on_unwind noalias writable sret(%"nd_item") align 1 %agg.result, ptr addrspace(4) %Global, ptr addrspace(4) %Local, ptr addrspace(4) %Group) {
+entry:
+  %Global.addr = alloca ptr addrspace(4)
+  %Local.addr = alloca ptr addrspace(4)
+  %Group.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %Global, ptr %Global.addr
+  store ptr addrspace(4) %Local, ptr %Local.addr
+  store ptr addrspace(4) %Group, ptr %Group.addr
+  %0 = load ptr addrspace(4), ptr %Global.addr
+  %1 = load ptr addrspace(4), ptr %Local.addr
+  %2 = load ptr addrspace(4), ptr %Group.addr
+  call spir_func void @Foo28(ptr addrspace(4) align 1 %agg.result, ptr addrspace(4) %0, ptr addrspace(4) %1, ptr addrspace(4) %2) 
+  ret void
+}
+
+
+define internal spir_func void @Foo28(ptr addrspace(4) align 1 %this, ptr addrspace(4) %0, ptr addrspace(4) %1, ptr addrspace(4) %2) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %.addr = alloca ptr addrspace(4)
+  %.addr1 = alloca ptr addrspace(4)
+  %.addr2 = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %0, ptr %.addr
+  store ptr addrspace(4) %1, ptr %.addr1
+  store ptr addrspace(4) %2, ptr %.addr2
+  %this3 = load ptr addrspace(4), ptr %this.addr
+  ret void
+}
+
+
+define internal spir_func void @Foo27(ptr addrspace(4) %this, ptr addrspace(4) %extent, ptr addrspace(4) %index) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %extent.addr = alloca ptr addrspace(4)
+  %index.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %extent, ptr %extent.addr
+  store ptr addrspace(4) %index, ptr %index.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %MImpl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MExtent2 = bitcast ptr addrspace(4) %MImpl1 to ptr addrspace(4)
+  %0 = load ptr addrspace(4), ptr %extent.addr
+  %MIndex = getelementptr inbounds nuw %"sd_ItemBase.23", ptr addrspace(4) %MImpl1, i32 0, i32 1
+  %1 = load ptr addrspace(4), ptr %index.addr
+  ret void
+}
+
+
+
+
+define internal spir_func void @Foo29(ptr addrspace(4) %this, ptr addrspace(4) %extent, ptr addrspace(4) %index, ptr addrspace(4) %offset) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %extent.addr = alloca ptr addrspace(4)
+  %index.addr = alloca ptr addrspace(4)
+  %offset.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %extent, ptr %extent.addr
+  store ptr addrspace(4) %index, ptr %index.addr
+  store ptr addrspace(4) %offset, ptr %offset.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %MImpl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MExtent2 = bitcast ptr addrspace(4) %MImpl1 to ptr addrspace(4)
+  %0 = load ptr addrspace(4), ptr %extent.addr
+  %MIndex = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %MImpl1, i32 0, i32 1
+  %1 = load ptr addrspace(4), ptr %index.addr
+  %MOffset = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %MImpl1, i32 0, i32 2
+  %2 = load ptr addrspace(4), ptr %offset.addr
+  ret void
+}
+
+
+define internal spir_func void @Bar82(ptr addrspace(4) %this, ptr addrspace(4) %G, ptr addrspace(4) %L, ptr byval(%"range") %GroupRange, ptr addrspace(4) %I) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %G.addr = alloca ptr addrspace(4)
+  %L.addr = alloca ptr addrspace(4)
+  %I.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  store ptr addrspace(4) %G, ptr %G.addr
+  store ptr addrspace(4) %L, ptr %L.addr
+  store ptr addrspace(4) %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %globalRange1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %0 = load ptr addrspace(4), ptr %G.addr
+  %localRange = getelementptr inbounds nuw %"group", ptr addrspace(4) %this1, i32 0, i32 1
+  %1 = load ptr addrspace(4), ptr %L.addr
+  %groupRange = getelementptr inbounds nuw %"group", ptr addrspace(4) %this1, i32 0, i32 2
+  %index = getelementptr inbounds nuw %"group", ptr addrspace(4) %this1, i32 0, i32 3
+  %2 = load ptr addrspace(4), ptr %I.addr
+  ret void
+}
+
+
+define internal spir_func void @InitSize4(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv15getGlobalOffsetILi0EEEmv() 
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv15getGlobalOffsetILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z22__spirv_GlobalOffset_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z22__spirv_GlobalOffset_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInGlobalOffset, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @InitSize3(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv20getLocalInvocationIdILi0EEEmv() 
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv20getLocalInvocationIdILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z27__spirv_LocalInvocationId_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z27__spirv_LocalInvocationId_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @InitSize2(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv14getWorkgroupIdILi0EEEmv() 
+  call spir_func void @Foo46(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv14getWorkgroupIdILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z21__spirv_WorkgroupId_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z21__spirv_WorkgroupId_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInWorkgroupId, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @InitSize1(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv16getNumWorkgroupsILi0EEEmv() 
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv16getNumWorkgroupsILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z23__spirv_NumWorkgroups_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z23__spirv_NumWorkgroups_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInNumWorkgroups, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @Inv3(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv16getWorkgroupSizeILi0EEEmv() 
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv16getWorkgroupSizeILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z23__spirv_WorkgroupSize_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z23__spirv_WorkgroupSize_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInWorkgroupSize, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @Foo8(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  %call = call spir_func i64 @_ZN7__spirv13getGlobalSizeILi0EEEmv() 
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func i64 @_ZN7__spirv13getGlobalSizeILi0EEEmv() {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %call = call spir_func i64 @_Z20__spirv_GlobalSize_xv() 
+  ret i64 %call
+}
+
+
+define internal spir_func i64 @_Z20__spirv_GlobalSize_xv()   {
+entry:
+  %retval = alloca i64
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %0 = load <3 x i64>, ptr addrspace(1) @__spirv_BuiltInGlobalSize, align 32
+  %1 = extractelement <3 x i64> %0, i64 0
+  ret i64 %1
+}
+
+
+define internal spir_func void @Foo30(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Foo33(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func i64 @Foo32(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %TotalOffset = alloca i64
+  %ref.tmp = alloca %class.anon.7
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %TotalOffset.ascast = addrspacecast ptr %TotalOffset to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  store i64 0, ptr %TotalOffset
+  %0 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %0
+  %TotalOffset2 = getelementptr inbounds %class.anon.7, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %TotalOffset.ascast, ptr %TotalOffset2
+  call spir_func void @Foo34(ptr addrspace(4) %ref.tmp.ascast) 
+  %1 = load i64, ptr %TotalOffset
+  ret i64 %1
+}
+
+
+define internal spir_func void @Foo34(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Foo35(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func void @Foo35(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Foo36(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func void @Foo36(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %TotalOffset = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset
+  %3 = load i64, ptr addrspace(4) %2
+  %impl1 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  %4 = load i64, ptr %I.addr
+  %conv = trunc i64 %4 to i32
+  %call = call spir_func i64 @Foo37(ptr addrspace(4) %MemRange, i32 %conv) 
+  %mul = mul i64 %3, %call
+  %TotalOffset2 = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %5 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset2
+  store i64 %mul, ptr addrspace(4) %5
+  %impl32 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %Offset3 = bitcast ptr addrspace(4) %impl32 to ptr addrspace(4)
+  %6 = load i64, ptr %I.addr
+  %conv4 = trunc i64 %6 to i32
+  %call5 = call spir_func i64 @Foo37(ptr addrspace(4) %Offset3, i32 %conv4) 
+  %TotalOffset6 = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %7 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset6
+  %8 = load i64, ptr addrspace(4) %7
+  %add = add i64 %8, %call5
+  store i64 %add, ptr addrspace(4) %7
+  ret void
+}
+
+
+define internal spir_func void @Foo33(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Foo38(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func void @Foo38(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %Offset = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %Offset
+  %3 = load i64, ptr %I.addr
+  %conv = trunc i64 %3 to i32
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %2, i32 %conv) 
+  %4 = load i64, ptr addrspace(4) %call
+  %call2 = call spir_func ptr addrspace(4) @Foo39(ptr addrspace(4) %1) 
+  %5 = load i64, ptr %I.addr
+  %conv3 = trunc i64 %5 to i32
+  %call4 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call2, i32 %conv3) 
+  store i64 %4, ptr addrspace(4) %call4
+  %AccessRange = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 2
+  %6 = load ptr addrspace(4), ptr addrspace(4) %AccessRange
+  %7 = load i64, ptr %I.addr
+  %conv5 = trunc i64 %7 to i32
+  %call6 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %6, i32 %conv5) 
+  %8 = load i64, ptr addrspace(4) %call6
+  %call7 = call spir_func ptr addrspace(4) @Foo40A(ptr addrspace(4) %1) 
+  %9 = load i64, ptr %I.addr
+  %conv8 = trunc i64 %9 to i32
+  %call9 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call7, i32 %conv8) 
+  store i64 %8, ptr addrspace(4) %call9
+  %MemRange = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 3
+  %10 = load ptr addrspace(4), ptr addrspace(4) %MemRange
+  %11 = load i64, ptr %I.addr
+  %conv10 = trunc i64 %11 to i32
+  %call11 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %10, i32 %conv10) 
+  %12 = load i64, ptr addrspace(4) %call11
+  %call12 = call spir_func ptr addrspace(4) @Foo41A(ptr addrspace(4) %1) 
+  %13 = load i64, ptr %I.addr
+  %conv13 = trunc i64 %13 to i32
+  %call14 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call12, i32 %conv13) 
+  store i64 %12, ptr addrspace(4) %call14
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo39(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %Offset2 = bitcast ptr addrspace(4) %impl1 to ptr addrspace(4)
+  ret ptr addrspace(4) %Offset2
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo40A(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %AccessRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 1
+  ret ptr addrspace(4) %AccessRange
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo41A(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  ret ptr addrspace(4) %MemRange
+}
+
+
+define internal spir_func void @Foo13(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Foo14(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func i64 @Foo21(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca i64
+  %this.addr = alloca ptr addrspace(4)
+  %TotalOffset = alloca i64
+  %ref.tmp = alloca %class.anon.7
+  %cleanup.dest.slot = alloca i32, align 4
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  %TotalOffset.ascast = addrspacecast ptr %TotalOffset to ptr addrspace(4)
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  store i64 0, ptr %TotalOffset
+  %0 = bitcast ptr %ref.tmp to ptr
+  store ptr addrspace(4) %this1, ptr %0
+  %TotalOffset2 = getelementptr inbounds %class.anon.7, ptr %ref.tmp, i32 0, i32 1
+  store ptr addrspace(4) %TotalOffset.ascast, ptr %TotalOffset2
+  call spir_func void @Bar83(ptr addrspace(4) %ref.tmp.ascast) 
+  %1 = load i64, ptr %TotalOffset
+  ret i64 %1
+}
+
+
+define internal spir_func void @Bar83(ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %agg.tmp = alloca %"nd_item", align 1
+  store ptr addrspace(4) %f, ptr %f.addr
+  %0 = load ptr addrspace(4), ptr %f.addr
+  call spir_func void @Bar84(ptr byval(%"nd_item") align 1 %agg.tmp, ptr addrspace(4) %0) 
+  ret void
+}
+
+
+define internal spir_func void @Bar84(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Bar85(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func void @Bar85(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %TotalOffset = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset
+  %3 = load i64, ptr addrspace(4) %2
+  %impl1 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  %4 = load i64, ptr %I.addr
+  %conv = trunc i64 %4 to i32
+  %call = call spir_func i64 @Foo37(ptr addrspace(4) %MemRange, i32 %conv) 
+  %mul = mul i64 %3, %call
+  %TotalOffset2 = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %5 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset2
+  store i64 %mul, ptr addrspace(4) %5
+  %impl32 = bitcast ptr addrspace(4) %1 to ptr addrspace(4)
+  %Offset3 = bitcast ptr addrspace(4) %impl32 to ptr addrspace(4)
+  %6 = load i64, ptr %I.addr
+  %conv4 = trunc i64 %6 to i32
+  %call5 = call spir_func i64 @Foo37(ptr addrspace(4) %Offset3, i32 %conv4) 
+  %TotalOffset6 = getelementptr inbounds nuw %class.anon.7, ptr addrspace(4) %this1, i32 0, i32 1
+  %7 = load ptr addrspace(4), ptr addrspace(4) %TotalOffset6
+  %8 = load i64, ptr addrspace(4) %7
+  %add = add i64 %8, %call5
+  store i64 %add, ptr addrspace(4) %7
+  ret void
+}
+
+
+define internal spir_func void @Foo14(ptr byval(%"nd_item") align 1 %0, ptr addrspace(4) %f) {
+entry:
+  %f.addr = alloca ptr addrspace(4)
+  %ref.tmp = alloca %"nd_item", align 1
+  %ref.tmp.ascast = addrspacecast ptr %ref.tmp to ptr addrspace(4)
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  store ptr addrspace(4) %f, ptr %f.addr
+  %2 = load ptr addrspace(4), ptr %f.addr
+  %call = call spir_func i64 @_ZNKSt17integral_constantImLm0EEcvmEv(ptr addrspace(4) align 1 %ref.tmp.ascast) 
+  call spir_func void @Foo15(ptr addrspace(4) %2, i64 %call) 
+  ret void
+}
+
+
+define internal spir_func void @Foo15(ptr addrspace(4) %this, i64 %I)  align 2  {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  %I.addr = alloca i64
+  store ptr addrspace(4) %this, ptr %this.addr
+  store i64 %I, ptr %I.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %0 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %1 = load ptr addrspace(4), ptr addrspace(4) %0
+  %Offset = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 1
+  %2 = load ptr addrspace(4), ptr addrspace(4) %Offset
+  %3 = load i64, ptr %I.addr
+  %conv = trunc i64 %3 to i32
+  %call = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %2, i32 %conv) 
+  %4 = load i64, ptr addrspace(4) %call
+  %call2 = call spir_func ptr addrspace(4) @Foo17(ptr addrspace(4) %1) 
+  %5 = load i64, ptr %I.addr
+  %conv3 = trunc i64 %5 to i32
+  %call4 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call2, i32 %conv3) 
+  store i64 %4, ptr addrspace(4) %call4
+  %AccessRange = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 2
+  %6 = load ptr addrspace(4), ptr addrspace(4) %AccessRange
+  %7 = load i64, ptr %I.addr
+  %conv5 = trunc i64 %7 to i32
+  %call6 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %6, i32 %conv5) 
+  %8 = load i64, ptr addrspace(4) %call6
+  %call7 = call spir_func ptr addrspace(4) @Foo18(ptr addrspace(4) %1) 
+  %9 = load i64, ptr %I.addr
+  %conv8 = trunc i64 %9 to i32
+  %call9 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call7, i32 %conv8) 
+  store i64 %8, ptr addrspace(4) %call9
+  %MemRange = getelementptr inbounds nuw %class.anon.6, ptr addrspace(4) %this1, i32 0, i32 3
+  %10 = load ptr addrspace(4), ptr addrspace(4) %MemRange
+  %11 = load i64, ptr %I.addr
+  %conv10 = trunc i64 %11 to i32
+  %call11 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %10, i32 %conv10) 
+  %12 = load i64, ptr addrspace(4) %call11
+  %call12 = call spir_func ptr addrspace(4) @Foo19(ptr addrspace(4) %1) 
+  %13 = load i64, ptr %I.addr
+  %conv13 = trunc i64 %13 to i32
+  %call14 = call spir_func ptr addrspace(4) @Foo16(ptr addrspace(4) %call12, i32 %conv13) 
+  store i64 %12, ptr addrspace(4) %call14
+  ret void
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo17(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %Offset2 = bitcast ptr addrspace(4) %impl1 to ptr addrspace(4)
+  ret ptr addrspace(4) %Offset2
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo18(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %AccessRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 1
+  ret ptr addrspace(4) %AccessRange
+}
+
+
+define internal spir_func ptr addrspace(4) @Foo19(ptr addrspace(4) %this) {
+entry:
+  %retval = alloca ptr addrspace(4)
+  %this.addr = alloca ptr addrspace(4)
+  %retval.ascast = addrspacecast ptr %retval to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %impl1 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %impl1, i32 0, i32 2
+  ret ptr addrspace(4) %MemRange
+}
+
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) 
+
+
+define internal spir_func void @Foo12(ptr addrspace(4) dead_on_unwind noalias writable sret(%"range") %agg.result) {
+entry:
+  call spir_func void @Foo9(ptr addrspace(4) %agg.result, i64 0) 
+  ret void
+}
+
+
+define internal spir_func void @Foo10(ptr addrspace(4) %this, ptr byval(%"range") %Offset, ptr byval(%"range") %AccessRange, ptr byval(%"range") %MemoryRange) unnamed_addr {
+entry:
+  %this.addr = alloca ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr %this.addr
+  %this1 = load ptr addrspace(4), ptr %this.addr
+  %Offset21 = bitcast ptr addrspace(4) %this1 to ptr addrspace(4)
+  %AccessRange3 = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %this1, i32 0, i32 1
+  %MemRange = getelementptr inbounds nuw %"detail::AccessorImplDevice", ptr addrspace(4) %this1, i32 0, i32 2
+  ret void
+}
+
+
+define internal spir_func void @__assert_fail(ptr addrspace(4) %expr, ptr addrspace(4) %file, i32 %line, ptr addrspace(4) %func)   {
+entry:
+  %call = tail call spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() 
+  %call1 = tail call spir_func i64 @_Z28__spirv_GlobalInvocationId_yv() 
+  %call2 = tail call spir_func i64 @_Z28__spirv_GlobalInvocationId_zv() 
+  %call3 = tail call spir_func i64 @_Z27__spirv_LocalInvocationId_xv() 
+  %call4 = tail call spir_func i64 @_Z27__spirv_LocalInvocationId_yv() 
+  %call5 = tail call spir_func i64 @_Z27__spirv_LocalInvocationId_zv() 
+  tail call spir_func void @__devicelib_assert_fail(ptr addrspace(4) %expr, ptr addrspace(4) %file, i32 %line, ptr addrspace(4) %func, i64 %call, i64 %call1, i64 %call2, i64 %call3, i64 %call4, i64 %call5) 
+  ret void
+}
+
+
+define internal spir_func i64 @_Z28__spirv_GlobalInvocationId_yv() local_unnamed_addr   {
+entry:
+  %0 = getelementptr inbounds i8, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, i64 8
+  %1 = load i64, ptr addrspace(1) %0
+  ret i64 %1
+}
+
+
+define internal spir_func i64 @_Z28__spirv_GlobalInvocationId_zv() local_unnamed_addr   {
+entry:
+  %0 = getelementptr inbounds i8, ptr addrspace(1) @__spirv_BuiltInGlobalInvocationId, i64 16
+  %1 = load i64, ptr addrspace(1) %0, align 16
+  ret i64 %1
+}
+
+
+define internal spir_func i64 @_Z27__spirv_LocalInvocationId_yv() local_unnamed_addr   {
+entry:
+  %0 = getelementptr inbounds i8, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, i64 8
+  %1 = load i64, ptr addrspace(1) %0
+  ret i64 %1
+}
+
+
+define internal spir_func i64 @_Z27__spirv_LocalInvocationId_zv() local_unnamed_addr   {
+entry:
+  %0 = getelementptr inbounds i8, ptr addrspace(1) @__spirv_BuiltInLocalInvocationId, i64 16
+  %1 = load i64, ptr addrspace(1) %0, align 16
+  ret i64 %1
+}
+
+
+define internal spir_func void @__devicelib_assert_fail(ptr addrspace(4) %expr, ptr addrspace(4) %file, i32 %line, ptr addrspace(4) %func, i64 %gid0, i64 %gid1, i64 %gid2, i64 %lid0, i64 %lid1, i64 %lid2) local_unnamed_addr   {
+entry:
+  %call.i = tail call spir_func i32 @_Z29__spirv_AtomicCompareExchangePU3AS1iN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_ii(ptr addrspace(1) @SPIR_AssertHappenedMem, i32 1, i32 16, i32 16, i32 1, i32 0) 
+  %cmp = icmp eq i32 %call.i, 0
+  %0 = getelementptr inbounds nuw i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 4
+  %1 = getelementptr inbounds nuw i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 261
+  %2 = getelementptr inbounds nuw i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 518
+  br i1 %cmp, label %if.then, label %if.end82
+
+if.then:                                          ; preds = %entry
+  store i32 %line, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 648)
+  store i64 %gid0, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 656)
+  store i64 %gid1, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 664)
+  store i64 %gid2, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 672)
+  store i64 %lid0, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 680)
+  store i64 %lid1, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 688)
+  store i64 %lid2, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @SPIR_AssertHappenedMem, i64 696)
+  %tobool.not = icmp eq ptr addrspace(4) %expr, null
+  br i1 %tobool.not, label %if.end, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %if.then
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.preheader, %for.inc
+  %ExprLength.0 = phi i32 [ %inc, %for.inc ], [ 0, %for.cond.preheader ]
+  %C.0 = phi ptr addrspace(4) [ %incdec.ptr, %for.inc ], [ %expr, %for.cond.preheader ]
+  %3 = load i8, ptr addrspace(4) %C.0, align 1
+  %cmp2.not = icmp eq i8 %3, 0
+  br i1 %cmp2.not, label %if.end, label %for.inc
+
+for.inc:                                          ; preds = %for.cond
+  %incdec.ptr = getelementptr inbounds nuw i8, ptr addrspace(4) %C.0, i64 1
+  %inc = add nuw nsw i32 %ExprLength.0, 1
+  br label %for.cond
+
+if.end:                                           ; preds = %for.cond, %if.then
+  %ExprLength.1 = phi i32 [ 0, %if.then ], [ %ExprLength.0, %for.cond ]
+  %tobool3.not = icmp eq ptr addrspace(4) %file, null
+  br i1 %tobool3.not, label %if.end16, label %for.cond6.preheader
+
+for.cond6.preheader:                              ; preds = %if.end
+  br label %for.cond6
+
+for.cond6:                                        ; preds = %for.cond6.preheader, %for.inc12
+  %FileLength.0 = phi i32 [ %inc14, %for.inc12 ], [ 0, %for.cond6.preheader ]
+  %C5.0 = phi ptr addrspace(4) [ %incdec.ptr13, %for.inc12 ], [ %file, %for.cond6.preheader ]
+  %4 = load i8, ptr addrspace(4) %C5.0, align 1
+  %cmp8.not = icmp eq i8 %4, 0
+  br i1 %cmp8.not, label %if.end16, label %for.inc12
+
+for.inc12:                                        ; preds = %for.cond6
+  %incdec.ptr13 = getelementptr inbounds nuw i8, ptr addrspace(4) %C5.0, i64 1
+  %inc14 = add nuw nsw i32 %FileLength.0, 1
+  br label %for.cond6
+
+if.end16:                                         ; preds = %for.cond6, %if.end
+  %FileLength.1 = phi i32 [ 0, %if.end ], [ %FileLength.0, %for.cond6 ]
+  %tobool17.not = icmp eq ptr addrspace(4) %func, null
+  br i1 %tobool17.not, label %if.end30.thread, label %for.cond20.preheader
+
+for.cond20.preheader:                             ; preds = %if.end16
+  br label %for.cond20
+
+for.cond20:                                       ; preds = %for.cond20.preheader, %for.inc26
+  %FuncLength.0 = phi i32 [ %inc28, %for.inc26 ], [ 0, %for.cond20.preheader ]
+  %C19.0 = phi ptr addrspace(4) [ %incdec.ptr27, %for.inc26 ], [ %func, %for.cond20.preheader ]
+  %5 = load i8, ptr addrspace(4) %C19.0, align 1
+  %cmp22.not = icmp eq i8 %5, 0
+  br i1 %cmp22.not, label %if.end30, label %for.inc26
+
+for.inc26:                                        ; preds = %for.cond20
+  %incdec.ptr27 = getelementptr inbounds nuw i8, ptr addrspace(4) %C19.0, i64 1
+  %inc28 = add i32 %FuncLength.0, 1
+  br label %for.cond20
+
+if.end30:                                         ; preds = %for.cond20
+  %spec.select = tail call i32 @llvm.umin.i32(i32 %ExprLength.1, i32 256)
+  %MaxFileIdx.0 = tail call i32 @llvm.umin.i32(i32 %FileLength.1, i32 256)
+  %spec.select126 = tail call i32 @llvm.umin.i32(i32 %FuncLength.0, i32 128)
+  br label %6
+
+if.end30.thread:                                  ; preds = %if.end16
+  %spec.select116 = tail call i32 @llvm.umin.i32(i32 %ExprLength.1, i32 256)
+  %MaxFileIdx.0118 = tail call i32 @llvm.umin.i32(i32 %FileLength.1, i32 256)
+  br label %6
+
+6:                                                ; preds = %if.end30, %if.end30.thread
+  %MaxFileIdx.0124 = phi i32 [ %MaxFileIdx.0118, %if.end30.thread ], [ %MaxFileIdx.0, %if.end30 ]
+  %spec.select122 = phi i32 [ %spec.select116, %if.end30.thread ], [ %spec.select, %if.end30 ]
+  %7 = phi i32 [ 0, %if.end30.thread ], [ %spec.select126, %if.end30 ]
+  br label %for.cond40
+
+for.cond40:                                       ; preds = %for.body44, %6
+  %lsr.iv9 = phi ptr addrspace(4) [ %scevgep10, %for.body44 ], [ %expr, %6 ]
+  %lsr.iv7 = phi ptr addrspace(1) [ %scevgep8, %for.body44 ], [ %0, %6 ]
+  %Idx.0 = phi i32 [ 0, %6 ], [ %inc48, %for.body44 ]
+  %cmp41 = icmp ult i32 %Idx.0, %spec.select122
+  br i1 %cmp41, label %for.body44, label %for.cond.cleanup42
+
+for.cond.cleanup42:                               ; preds = %for.cond40
+  %idxprom50 = zext nneg i32 %spec.select122 to i64
+  %arrayidx51 = getelementptr inbounds [257 x i8], ptr addrspace(1) %0, i64 0, i64 %idxprom50
+  store i8 0, ptr addrspace(1) %arrayidx51, align 1
+  br label %for.cond53
+
+for.cond53:                                       ; preds = %for.body57, %for.cond.cleanup42
+  %lsr.iv5 = phi ptr addrspace(4) [ %scevgep6, %for.body57 ], [ %file, %for.cond.cleanup42 ]
+  %lsr.iv3 = phi ptr addrspace(1) [ %scevgep4, %for.body57 ], [ %1, %for.cond.cleanup42 ]
+  %Idx52.0 = phi i32 [ 0, %for.cond.cleanup42 ], [ %inc63, %for.body57 ]
+  %cmp54 = icmp ult i32 %Idx52.0, %MaxFileIdx.0124
+  br i1 %cmp54, label %for.body57, label %for.cond.cleanup55
+
+for.cond.cleanup55:                               ; preds = %for.cond53
+  %idxprom65 = zext nneg i32 %MaxFileIdx.0124 to i64
+  %arrayidx66 = getelementptr inbounds [257 x i8], ptr addrspace(1) %1, i64 0, i64 %idxprom65
+  store i8 0, ptr addrspace(1) %arrayidx66, align 1
+  br label %for.cond68
+
+for.cond68:                                       ; preds = %for.body72, %for.cond.cleanup55
+  %lsr.iv1 = phi ptr addrspace(4) [ %scevgep2, %for.body72 ], [ %func, %for.cond.cleanup55 ]
+  %lsr.iv = phi ptr addrspace(1) [ %scevgep, %for.body72 ], [ %2, %for.cond.cleanup55 ]
+  %Idx67.0 = phi i32 [ 0, %for.cond.cleanup55 ], [ %inc78, %for.body72 ]
+  %cmp69 = icmp ult i32 %Idx67.0, %7
+  br i1 %cmp69, label %for.body72, label %for.cond.cleanup70
+
+for.cond.cleanup70:                               ; preds = %for.cond68
+  %idxprom80 = zext nneg i32 %7 to i64
+  %arrayidx81 = getelementptr inbounds [129 x i8], ptr addrspace(1) %2, i64 0, i64 %idxprom80
+  store i8 0, ptr addrspace(1) %arrayidx81, align 1
+  tail call spir_func void @_Z19__spirv_AtomicStorePU3AS1iN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagEi(ptr addrspace(1) @SPIR_AssertHappenedMem, i32 1, i32 16, i32 2) 
+  br label %if.end82
+
+if.end82:                                         ; preds = %for.cond.cleanup70, %entry
+  ret void
+
+for.body72:                                       ; preds = %for.cond68
+  %8 = load i8, ptr addrspace(4) %lsr.iv1, align 1
+  store i8 %8, ptr addrspace(1) %lsr.iv, align 1
+  %inc78 = add nuw nsw i32 %Idx67.0, 1
+  %scevgep = getelementptr i8, ptr addrspace(1) %lsr.iv, i64 1
+  %scevgep2 = getelementptr i8, ptr addrspace(4) %lsr.iv1, i64 1
+  br label %for.cond68
+
+for.body57:                                       ; preds = %for.cond53
+  %9 = load i8, ptr addrspace(4) %lsr.iv5, align 1
+  store i8 %9, ptr addrspace(1) %lsr.iv3, align 1
+  %inc63 = add nuw nsw i32 %Idx52.0, 1
+  %scevgep4 = getelementptr i8, ptr addrspace(1) %lsr.iv3, i64 1
+  %scevgep6 = getelementptr i8, ptr addrspace(4) %lsr.iv5, i64 1
+  br label %for.cond53
+
+for.body44:                                       ; preds = %for.cond40
+  %10 = load i8, ptr addrspace(4) %lsr.iv9, align 1
+  store i8 %10, ptr addrspace(1) %lsr.iv7, align 1
+  %inc48 = add nuw nsw i32 %Idx.0, 1
+  %scevgep8 = getelementptr i8, ptr addrspace(1) %lsr.iv7, i64 1
+  %scevgep10 = getelementptr i8, ptr addrspace(4) %lsr.iv9, i64 1
+  br label %for.cond40
+}
+
+declare extern_weak dso_local spir_func i32 @_Z29__spirv_AtomicCompareExchangePU3AS1iN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_ii(ptr addrspace(1), i32, i32, i32, i32, i32) local_unnamed_addr 
+declare extern_weak dso_local spir_func void @_Z19__spirv_AtomicStorePU3AS1iN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagEi(ptr addrspace(1), i32, i32, i32) local_unnamed_addr 
+declare i32 @llvm.umin.i32(i32, i32) 
diff --git a/llvm/test/CodeGen/SystemZ/builtin-longjmp.ll b/llvm/test/CodeGen/SystemZ/builtin-longjmp.ll
new file mode 100644
index 0000000000000..c1d3dc54b2fd6
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/builtin-longjmp.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test longjmp load from jmp_buf.
+; Frame pointer from Slot 1.
+; Jump address from Slot 2.
+; Backchain Value from Slot 3.
+; Stack Pointer from Slot 4.
+; Literal Pool Pointer from Slot 5.
+
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu | FileCheck %s
+
+@buf = global [20 x ptr] zeroinitializer, align 8
+
+define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r11, %r15, 88(%r15)
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    lg %r2, 8(%r1)
+; CHECK-NEXT:    lg %r11, 0(%r1)
+; CHECK-NEXT:    lg %r13, 32(%r1)
+; CHECK-NEXT:    lg %r15, 24(%r1)
+; CHECK-NEXT:    br %r2
+entry:
+  tail call void @llvm.eh.sjlj.longjmp(ptr nonnull @buf)
+  unreachable
+}
+
+define void @bar()  "backchain" {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r11, %r15, 88(%r15)
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    lg %r2, 8(%r1)
+; CHECK-NEXT:    lg %r11, 0(%r1)
+; CHECK-NEXT:    lg %r13, 32(%r1)
+; CHECK-NEXT:    lg %r3, 16(%r1)
+; CHECK-NEXT:    lg %r15, 24(%r1)
+; CHECK-NEXT:    stg %r3, 0(%r15)
+; CHECK-NEXT:    br %r2
+entry:
+  tail call void @llvm.eh.sjlj.longjmp(ptr nonnull @buf)
+  unreachable
+}
diff --git a/llvm/test/CodeGen/SystemZ/builtin-setjmp-alloca.ll b/llvm/test/CodeGen/SystemZ/builtin-setjmp-alloca.ll
new file mode 100644
index 0000000000000..f714599f9a8f2
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/builtin-setjmp-alloca.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test setjmp  store to jmp_buf when frame pointer is used and saved
+; because of variable size alloca.
+; Frame Pointer is stored in slot 1.
+; Return address in slot 2.
+; Backchain value is stored in slot 3 for -mbackchain option.
+; Stack Pointer in slot 4.
+; Clobber %r6-%r15, %f8-%f15.
+
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i32 @llvm.eh.sjlj.setjmp(ptr)
+@buf = global [20 x ptr] zeroinitializer, align 8
+
+define signext i32 @foo() "frame-pointer"="all" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r6, %r15, 48(%r15)
+; CHECK-NEXT:    .cfi_offset %r6, -112
+; CHECK-NEXT:    .cfi_offset %r7, -104
+; CHECK-NEXT:    .cfi_offset %r8, -96
+; CHECK-NEXT:    .cfi_offset %r9, -88
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -240
+; CHECK-NEXT:    .cfi_def_cfa_offset 400
+; CHECK-NEXT:    lgr %r11, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r11
+; CHECK-NEXT:    std %f8, 232(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f9, 224(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f10, 216(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f11, 208(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f12, 200(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f13, 192(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f14, 184(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f15, 176(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    .cfi_offset %f11, -192
+; CHECK-NEXT:    .cfi_offset %f12, -200
+; CHECK-NEXT:    .cfi_offset %f13, -208
+; CHECK-NEXT:    .cfi_offset %f14, -216
+; CHECK-NEXT:    .cfi_offset %f15, -224
+; CHECK-NEXT:    la %r0, 160(%r11)
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    stg %r0, 168(%r11)
+; CHECK-NEXT:    mvhi 160(%r11), 10
+; CHECK-NEXT:    larl %r0, .LBB0_1
+; CHECK-NEXT:    stg %r0, 8(%r1)
+; CHECK-NEXT:    stg %r11, 0(%r1)
+; CHECK-NEXT:    stg %r15, 24(%r1)
+; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:    # %entry
+; CHECK-NEXT:  .LBB0_2: # %entry
+; CHECK-NEXT:    lg %r1, 168(%r11)
+; CHECK-NEXT:    lgf %r2, 0(%r1)
+; CHECK-NEXT:    ld %f8, 232(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f9, 224(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f10, 216(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f11, 208(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f12, 200(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f13, 192(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f14, 184(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f15, 176(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    lmg %r6, %r15, 288(%r11)
+; CHECK-NEXT:    br %r14
+entry:
+  %val = alloca ptr, align 8
+  %0 = alloca i8, i64 4, align 8
+  store ptr %0, ptr %val, align 8
+  %1 = load ptr, ptr %val, align 8
+  store volatile i32 10, ptr %1, align 4
+  %2 = call i32 @llvm.eh.sjlj.setjmp(ptr @buf)
+  %3 = load ptr, ptr %val, align 8
+  %4 = load volatile i32, ptr %3, align 4
+  ret i32 %4
+}
+
+define signext i32 @foo1() "backchain" "frame-pointer"="all" {
+; CHECK-LABEL: foo1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r6, %r15, 48(%r15)
+; CHECK-NEXT:    .cfi_offset %r6, -112
+; CHECK-NEXT:    .cfi_offset %r7, -104
+; CHECK-NEXT:    .cfi_offset %r8, -96
+; CHECK-NEXT:    .cfi_offset %r9, -88
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lgr %r1, %r15
+; CHECK-NEXT:    aghi %r15, -240
+; CHECK-NEXT:    .cfi_def_cfa_offset 400
+; CHECK-NEXT:    stg %r1, 0(%r15)
+; CHECK-NEXT:    lgr %r11, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r11
+; CHECK-NEXT:    std %f8, 232(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f9, 224(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f10, 216(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f11, 208(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f12, 200(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f13, 192(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f14, 184(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f15, 176(%r11) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    .cfi_offset %f11, -192
+; CHECK-NEXT:    .cfi_offset %f12, -200
+; CHECK-NEXT:    .cfi_offset %f13, -208
+; CHECK-NEXT:    .cfi_offset %f14, -216
+; CHECK-NEXT:    .cfi_offset %f15, -224
+; CHECK-NEXT:    la %r0, 160(%r11)
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    stg %r0, 168(%r11)
+; CHECK-NEXT:    mvhi 160(%r11), 10
+; CHECK-NEXT:    larl %r0, .LBB1_1
+; CHECK-NEXT:    stg %r0, 8(%r1)
+; CHECK-NEXT:    stg %r11, 0(%r1)
+; CHECK-NEXT:    stg %r15, 24(%r1)
+; CHECK-NEXT:    lg %r0, 0(%r15)
+; CHECK-NEXT:    stg %r0, 16(%r1)
+; CHECK-NEXT:  .LBB1_1: # Block address taken
+; CHECK-NEXT:    # %entry
+; CHECK-NEXT:  .LBB1_2: # %entry
+; CHECK-NEXT:    lg %r1, 168(%r11)
+; CHECK-NEXT:    lgf %r2, 0(%r1)
+; CHECK-NEXT:    ld %f8, 232(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f9, 224(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f10, 216(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f11, 208(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f12, 200(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f13, 192(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f14, 184(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f15, 176(%r11) # 8-byte Folded Reload
+; CHECK-NEXT:    lmg %r6, %r15, 288(%r11)
+; CHECK-NEXT:    br %r14
+entry:
+  %val = alloca ptr, align 8
+  %0 = alloca i8, i64 4, align 8
+  store ptr %0, ptr %val, align 8
+  %1 = load ptr, ptr %val, align 8
+  store volatile i32 10, ptr %1, align 4
+  %2 = call i32 @llvm.eh.sjlj.setjmp(ptr @buf)
+  %3 = load ptr, ptr %val, align 8
+  %4 = load volatile i32, ptr %3, align 4
+  ret i32 %4
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/builtin-setjmp-spills.ll b/llvm/test/CodeGen/SystemZ/builtin-setjmp-spills.ll
new file mode 100644
index 0000000000000..65657ec9f1826
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/builtin-setjmp-spills.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Simulate register pressure around setjmp call and check all virtual registers
+; are saved to stack before setjmp call and restored from stack after the call.
+; And these registers are not live across the setjmp call.
+; setjmp storing to jmp_buf.
+; Return address in slot 2.
+; Stack Pointer in slot 4.
+; Clobber %r6-%r15, %f8-%f15.
+
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i32 @llvm.eh.sjlj.setjmp(ptr)
+
+@t = global i32 0, align 4
+@s = global i32 0, align 4
+@r = global i32 0, align 4
+@q = global i32 0, align 4
+@p = global i32 0, align 4
+@o = global i32 0, align 4
+@n = global i32 0, align 4
+@m = global i32 0, align 4
+@l = global i32 0, align 4
+@k = global i32 0, align 4
+@j = global i32 0, align 4
+@i = global i32 0, align 4
+@h = global i32 0, align 4
+@g = global i32 0, align 4
+@f = global i32 0, align 4
+@e = global i32 0, align 4
+@d = global i32 0, align 4
+@c = global i32 0, align 4
+@b = global i32 0, align 4
+@a = global i32 0, align 4
+@buf = global [10 x ptr] zeroinitializer, align 8
+
+define signext i32 @func() {
+; CHECK-LABEL: func:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r6, %r15, 48(%r15)
+; CHECK-NEXT:    .cfi_offset %r6, -112
+; CHECK-NEXT:    .cfi_offset %r7, -104
+; CHECK-NEXT:    .cfi_offset %r8, -96
+; CHECK-NEXT:    .cfi_offset %r9, -88
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -384
+; CHECK-NEXT:    .cfi_def_cfa_offset 544
+; CHECK-NEXT:    std %f8, 376(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f9, 368(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f10, 360(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f11, 352(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f12, 344(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f13, 336(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f14, 328(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f15, 320(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    .cfi_offset %f11, -192
+; CHECK-NEXT:    .cfi_offset %f12, -200
+; CHECK-NEXT:    .cfi_offset %f13, -208
+; CHECK-NEXT:    .cfi_offset %f14, -216
+; CHECK-NEXT:    .cfi_offset %f15, -224
+; CHECK-NEXT:    lgrl %r1, t@GOT
+; CHECK-NEXT:    lgrl %r2, s@GOT
+; CHECK-NEXT:    stg %r1, 312(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    lgrl %r1, r@GOT
+; CHECK-NEXT:    lgrl %r3, q@GOT
+; CHECK-NEXT:    stg %r2, 304(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    lgrl %r2, p@GOT
+; CHECK-NEXT:    stg %r1, 296(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    stg %r3, 288(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r3), 1
+; CHECK-NEXT:    lgrl %r1, o@GOT
+; CHECK-NEXT:    stg %r2, 280(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    lgrl %r2, n@GOT
+; CHECK-NEXT:    lgrl %r3, m@GOT
+; CHECK-NEXT:    stg %r1, 272(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    lgrl %r1, l@GOT
+; CHECK-NEXT:    stg %r2, 264(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    stg %r3, 256(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r3), 1
+; CHECK-NEXT:    lgrl %r2, k@GOT
+; CHECK-NEXT:    stg %r1, 248(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    lgrl %r1, j@GOT
+; CHECK-NEXT:    lgrl %r3, i@GOT
+; CHECK-NEXT:    stg %r2, 240(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    lgrl %r2, h@GOT
+; CHECK-NEXT:    stg %r1, 232(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    stg %r3, 224(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r3), 1
+; CHECK-NEXT:    lgrl %r1, g@GOT
+; CHECK-NEXT:    stg %r2, 216(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    lgrl %r2, f@GOT
+; CHECK-NEXT:    lgrl %r3, e@GOT
+; CHECK-NEXT:    stg %r1, 208(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    lgrl %r1, d@GOT
+; CHECK-NEXT:    stg %r2, 200(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    stg %r3, 192(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r3), 1
+; CHECK-NEXT:    lgrl %r2, c@GOT
+; CHECK-NEXT:    stg %r1, 184(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r1), 1
+; CHECK-NEXT:    lgrl %r3, b@GOT
+; CHECK-NEXT:    lgrl %r4, a@GOT
+; CHECK-NEXT:    stg %r2, 176(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r2), 1
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    stg %r3, 168(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r3), 1
+; CHECK-NEXT:    stg %r4, 160(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    mvhi 0(%r4), 1
+; CHECK-NEXT:    larl %r0, .LBB0_2
+; CHECK-NEXT:    stg %r0, 8(%r1)
+; CHECK-NEXT:    stg %r15, 24(%r1)
+; CHECK-NEXT:  .LBB0_1: # %entry
+; CHECK-NEXT:    lhi %r0, 0
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:  .LBB0_2: # Block address taken
+; CHECK-NEXT:    # %entry
+; CHECK-NEXT:    lhi %r0, 1
+; CHECK-NEXT:  .LBB0_3: # %entry
+; CHECK-NEXT:    lg %r1, 160(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 168(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 176(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 184(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 192(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 200(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 208(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 216(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 224(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 232(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 240(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 248(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 256(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 264(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 272(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 280(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 288(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 296(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 304(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lg %r1, 312(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    a %r0, 0(%r1)
+; CHECK-NEXT:    lgfr %r2, %r0
+; CHECK-NEXT:    ld %f8, 376(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f9, 368(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f10, 360(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f11, 352(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f12, 344(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f13, 336(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f14, 328(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f15, 320(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    lmg %r6, %r15, 432(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  store i32 1, ptr @t, align 4
+  store i32 1, ptr @s, align 4
+  store i32 1, ptr @r, align 4
+  store i32 1, ptr @q, align 4
+  store i32 1, ptr @p, align 4
+  store i32 1, ptr @o, align 4
+  store i32 1, ptr @n, align 4
+  store i32 1, ptr @m, align 4
+  store i32 1, ptr @l, align 4
+  store i32 1, ptr @k, align 4
+  store i32 1, ptr @j, align 4
+  store i32 1, ptr @i, align 4
+  store i32 1, ptr @h, align 4
+  store i32 1, ptr @g, align 4
+  store i32 1, ptr @f, align 4
+  store i32 1, ptr @e, align 4
+  store i32 1, ptr @d, align 4
+  store i32 1, ptr @c, align 4
+  store i32 1, ptr @b, align 4
+  store i32 1, ptr @a, align 4
+  %0 = tail call i32 @llvm.eh.sjlj.setjmp(ptr nonnull @buf)
+  %1 = load i32, ptr @a, align 4
+  %2 = load i32, ptr @b, align 4
+  %3 = load i32, ptr @c, align 4
+  %4 = load i32, ptr @d, align 4
+  %5 = load i32, ptr @e, align 4
+  %6 = load i32, ptr @f, align 4
+  %7 = load i32, ptr @g, align 4
+  %8 = load i32, ptr @h, align 4
+  %9 = load i32, ptr @i, align 4
+  %10 = load i32, ptr @j, align 4
+  %11 = load i32, ptr @k, align 4
+  %12 = load i32, ptr @l, align 4
+  %13 = load i32, ptr @m, align 4
+  %14 = load i32, ptr @n, align 4
+  %15 = load i32, ptr @o, align 4
+  %16 = load i32, ptr @p, align 4
+  %17 = load i32, ptr @q, align 4
+  %18 = load i32, ptr @r, align 4
+  %19 = load i32, ptr @s, align 4
+  %20 = load i32, ptr @t, align 4
+  %add = add i32 %1, %0
+  %add1 = add i32 %add, %2
+  %add2 = add i32 %add1, %3
+  %add3 = add i32 %add2, %4
+  %add4 = add i32 %add3, %5
+  %add5 = add i32 %add4, %6
+  %add6 = add i32 %add5, %7
+  %add7 = add i32 %add6, %8
+  %add8 = add i32 %add7, %9
+  %add9 = add i32 %add8, %10
+  %add10 = add i32 %add9, %11
+  %add11 = add i32 %add10, %12
+  %add12 = add i32 %add11, %13
+  %add13 = add i32 %add12, %14
+  %add14 = add i32 %add13, %15
+  %add15 = add i32 %add14, %16
+  %add16 = add i32 %add15, %17
+  %add17 = add i32 %add16, %18
+  %add18 = add i32 %add17, %19
+  %add19 = add i32 %add18, %20
+  ret i32 %add19
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/builtin-setjmp.ll b/llvm/test/CodeGen/SystemZ/builtin-setjmp.ll
new file mode 100644
index 0000000000000..cfeba5a051ad1
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/builtin-setjmp.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test setjmp store jmp_buf
+; Return address in slot 2.
+; Backchain value is stored in slot 3 for -mbackchain option.
+; Stack Pointer in slot 4.
+; Clobber %r6-%r15, %f8-%f15.
+
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i32 @llvm.eh.sjlj.setjmp(ptr)
+@buf = global [20 x ptr] zeroinitializer, align 8
+
+define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r6, %r15, 48(%r15)
+; CHECK-NEXT:    .cfi_offset %r6, -112
+; CHECK-NEXT:    .cfi_offset %r7, -104
+; CHECK-NEXT:    .cfi_offset %r8, -96
+; CHECK-NEXT:    .cfi_offset %r9, -88
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -64
+; CHECK-NEXT:    .cfi_def_cfa_offset 224
+; CHECK-NEXT:    std %f8, 56(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f9, 48(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f10, 40(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f11, 32(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f12, 24(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f13, 16(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f14, 8(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f15, 0(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    .cfi_offset %f11, -192
+; CHECK-NEXT:    .cfi_offset %f12, -200
+; CHECK-NEXT:    .cfi_offset %f13, -208
+; CHECK-NEXT:    .cfi_offset %f14, -216
+; CHECK-NEXT:    .cfi_offset %f15, -224
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    larl %r0, .LBB0_1
+; CHECK-NEXT:    stg %r0, 8(%r1)
+; CHECK-NEXT:    stg %r15, 24(%r1)
+; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:    # %entry
+; CHECK-NEXT:  .LBB0_2: # %entry
+; CHECK-NEXT:    ld %f8, 56(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f9, 48(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f10, 40(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f11, 32(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f12, 24(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f13, 16(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f14, 8(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f15, 0(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    lmg %r6, %r15, 112(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i32 @llvm.eh.sjlj.setjmp(ptr nonnull @buf)
+  ret void
+}
+
+define void @foo1() "backchain" {
+; CHECK-LABEL: foo1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r6, %r15, 48(%r15)
+; CHECK-NEXT:    .cfi_offset %r6, -112
+; CHECK-NEXT:    .cfi_offset %r7, -104
+; CHECK-NEXT:    .cfi_offset %r8, -96
+; CHECK-NEXT:    .cfi_offset %r9, -88
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lgr %r1, %r15
+; CHECK-NEXT:    aghi %r15, -64
+; CHECK-NEXT:    .cfi_def_cfa_offset 224
+; CHECK-NEXT:    stg %r1, 0(%r15)
+; CHECK-NEXT:    std %f8, 56(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f9, 48(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f10, 40(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f11, 32(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f12, 24(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f13, 16(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f14, 8(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    std %f15, 0(%r15) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    .cfi_offset %f11, -192
+; CHECK-NEXT:    .cfi_offset %f12, -200
+; CHECK-NEXT:    .cfi_offset %f13, -208
+; CHECK-NEXT:    .cfi_offset %f14, -216
+; CHECK-NEXT:    .cfi_offset %f15, -224
+; CHECK-NEXT:    lgrl %r1, buf@GOT
+; CHECK-NEXT:    larl %r0, .LBB1_1
+; CHECK-NEXT:    stg %r0, 8(%r1)
+; CHECK-NEXT:    stg %r15, 24(%r1)
+; CHECK-NEXT:    lg %r0, 0(%r15)
+; CHECK-NEXT:    stg %r0, 16(%r1)
+; CHECK-NEXT:  .LBB1_1: # Block address taken
+; CHECK-NEXT:    # %entry
+; CHECK-NEXT:  .LBB1_2: # %entry
+; CHECK-NEXT:    ld %f8, 56(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f9, 48(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f10, 40(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f11, 32(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f12, 24(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f13, 16(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f14, 8(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %f15, 0(%r15) # 8-byte Folded Reload
+; CHECK-NEXT:    lmg %r6, %r15, 112(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i32 @llvm.eh.sjlj.setjmp(ptr nonnull @buf)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update-remat.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update-remat.mir
index f7b58f04fd7ba..e3207df799449 100644
--- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update-remat.mir
+++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update-remat.mir
@@ -29,13 +29,12 @@ tracksRegLiveness: true
 body:             |
   bb.0:
 
-    ; CHECK-LABEL
     ; CHECK-LABEL: name: main
     ; CHECK: [[LGHI:%[0-9]+]]:gr64bit = LGHI 25
-    ; CHECK: CHIMux [[LGHI]].subreg_l32, 0, implicit-def $cc
-    ; CHECK: [[LGHI1:%[0-9]+]]:gr64bit = LGHI 25
-    ; CHECK: undef [[LGHI1]].subreg_l32:gr64bit = MSFI [[LGHI1]].subreg_l32, -117440512
-    ; CHECK: Return implicit [[LGHI1]].subreg_l32
+    ; CHECK-NEXT: CHIMux [[LGHI]].subreg_l32, 0, implicit-def $cc
+    ; CHECK-NEXT: [[LGHI1:%[0-9]+]]:gr64bit = LGHI 25
+    ; CHECK-NEXT: undef [[LGHI1:%[0-9]+]].subreg_l32:gr64bit = MSFI [[LGHI1]].subreg_l32, -117440512
+    ; CHECK-NEXT: Return implicit [[LGHI1]].subreg_l32
     %0:gr64bit = LGHI 25
     %1:gr32bit = COPY %0.subreg_l32
     %1:gr32bit = MSFI %1, -117440512
diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir
index bf5855010bf90..c805d31aa11c6 100644
--- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir
+++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir
@@ -25,12 +25,12 @@ body:             |
 
     ; CHECK-LABEL: name: main
     ; CHECK: [[LGHI:%[0-9]+]]:gr64bit = LGHI 43
-    ; CHECK: [[LGHI1:%[0-9]+]]:gr64bit = LGHI 43
-    ; CHECK: [[LGHI1]].subreg_l32:gr64bit = MSR [[LGHI1]].subreg_l32, [[LGHI1]].subreg_l32
-    ; CHECK: [[LGHI1]].subreg_l32:gr64bit = AHIMux [[LGHI1]].subreg_l32, 9, implicit-def dead $cc
-    ; CHECK: undef %3.subreg_l64:gr128bit = LGFI -245143785, implicit [[LGHI1]].subreg_l32
-    ; CHECK: [[DLGR:%[0-9]+]]:gr128bit = DLGR [[DLGR]], [[LGHI]]
-    ; CHECK: Return implicit [[DLGR]]
+    ; CHECK-NEXT: [[LGHI1:%[0-9]+]]:gr64bit = LGHI 43
+    ; CHECK-NEXT: undef [[LGHI1:%[0-9]+]].subreg_l32:gr64bit = MSR [[LGHI1]].subreg_l32, [[LGHI1]].subreg_l32
+    ; CHECK-NEXT: [[LGHI1:%[0-9]+]].subreg_l32:gr64bit = AHIMux [[LGHI1]].subreg_l32, 9, implicit-def dead $cc
+    ; CHECK-NEXT: undef [[LGFI:%[0-9]+]].subreg_l64:gr128bit = LGFI -245143785, implicit [[LGHI1]].subreg_l32
+    ; CHECK-NEXT: [[LGFI:%[0-9]+]]:gr128bit = DLGR [[LGFI]], [[LGHI]]
+    ; CHECK-NEXT: Return implicit [[LGFI]]
     %0:gr64bit = LGHI 43
     %1:gr32bit = COPY %0.subreg_l32
     %1:gr32bit = MSR %1, %1
@@ -55,20 +55,23 @@ liveins:         []
 body:             |
   ; CHECK-LABEL: name: segfault
   ; CHECK: bb.0:
-  ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[LGHI:%[0-9]+]]:addr64bit = LGHI 0
-  ; CHECK: bb.1:
-  ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   ADJCALLSTACKDOWN 0, 0
-  ; CHECK:   [[LGFR:%[0-9]+]]:gr64bit = LGFR [[LGHI]].subreg_l32
-  ; CHECK:   $r2d = LGHI 123
-  ; CHECK:   $r3d = LGHI 0
-  ; CHECK:   $r4d = LGHI 0
-  ; CHECK:   $r5d = COPY [[LGFR]]
-  ; CHECK:   KILL killed $r2d, killed $r3d, killed $r4d, $r5d, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc
-  ; CHECK:   ADJCALLSTACKUP 0, 0
-  ; CHECK:   [[LGHI]]:addr64bit = nuw nsw LA [[LGHI]], 1, $noreg
-  ; CHECK:   J %bb.1
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LGHI:%[0-9]+]]:addr64bit = LGHI 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0
+  ; CHECK-NEXT:   [[LGFR:%[0-9]+]]:gr64bit = LGFR [[LGHI]].subreg_l32
+  ; CHECK-NEXT:   $r2d = LGHI 123
+  ; CHECK-NEXT:   $r3d = LGHI 0
+  ; CHECK-NEXT:   $r4d = LGHI 0
+  ; CHECK-NEXT:   $r5d = COPY [[LGFR]]
+  ; CHECK-NEXT:   KILL killed $r2d, killed $r3d, killed $r4d, $r5d, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0
+  ; CHECK-NEXT:   [[LGHI:%[0-9]+]]:addr64bit = nuw nsw LA [[LGHI]], 1, $noreg
+  ; CHECK-NEXT:   J %bb.1
   bb.0:
     successors: %bb.1(0x80000000)
 
diff --git a/llvm/test/CodeGen/SystemZ/regcoal-undef-lane-4-rm-cp-commuting-def.mir b/llvm/test/CodeGen/SystemZ/regcoal-undef-lane-4-rm-cp-commuting-def.mir
index b1a0a309e1e20..cd8f14b28844b 100644
--- a/llvm/test/CodeGen/SystemZ/regcoal-undef-lane-4-rm-cp-commuting-def.mir
+++ b/llvm/test/CodeGen/SystemZ/regcoal-undef-lane-4-rm-cp-commuting-def.mir
@@ -15,56 +15,70 @@
 # only partial defined before being fully copied.
 #
 # PR40215.
+
+
+# The coalescer should have been able to swap the operands of
+# the OR, hence eliminating the copy of %20 and %18.
+# This is also visible here because the ROSBG operand was LHS
+# before the coalescer, now it is RHS.
 ---
 name:            main
 alignment:       16
 tracksRegLiveness: true
-machineFunctionInfo: {}
 body:             |
   ; CHECK-LABEL: name: main
   ; CHECK: bb.0:
-  ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[LGHI:%[0-9]+]]:addr64bit = LGHI -30
-  ; CHECK:   [[LHIMux:%[0-9]+]]:grx32bit = LHIMux 1
-  ; CHECK:   [[LHIMux1:%[0-9]+]]:grx32bit = LHIMux 0
-  ; CHECK:   undef %20.subreg_l32:gr64bit = IMPLICIT_DEF
-  ; CHECK: bb.1:
-  ; CHECK:   successors: %bb.3(0x00000001), %bb.4(0x7fffffff)
-  ; CHECK:   CHIMux [[LHIMux]], 0, implicit-def $cc
-  ; CHECK:   BRC 14, 6, %bb.3, implicit killed $cc
-  ; CHECK:   J %bb.4
-  ; CHECK: bb.2:
-  ; CHECK:   successors:
-  ; CHECK:   STMux %20.subreg_l32, undef %8:addr64bit, 0, $noreg :: (store (s32) into `ptr undef`)
-  ; CHECK: bb.3:
-  ; CHECK:   successors:
-  ; CHECK: bb.4:
-  ; CHECK:   successors: %bb.5(0x30000000), %bb.6(0x50000000)
-  ; CHECK:   [[LAY:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19838, $noreg
-  ; CHECK:   [[LAY1:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19840, $noreg
-  ; CHECK:   [[LAY2:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19844, $noreg
-  ; CHECK:   [[OGR:%[0-9]+]]:gr64bit = OGR [[OGR]], [[LAY]], implicit-def dead $cc
-  ; CHECK:   undef %19.subreg_l32:gr64bit = AHIMuxK [[LGHI]].subreg_l32, 19843, implicit-def dead $cc
-  ; CHECK:   [[ROSBG:%[0-9]+]]:gr64bit = ROSBG [[ROSBG]], [[OGR]], 32, 63, 0, implicit-def dead $cc
-  ; The coalescer should have been able to swap the operands of
-  ; the OR, hence eliminating the copy of %20 and %18.
-  ; This is also visible here because the ROSBG operand was LHS
-  ; before the coalescer, now it is RHS.
-  ; CHECK:   %20.subreg_l32:gr64bit = OR %20.subreg_l32, [[ROSBG]].subreg_l32, implicit-def dead $cc
-  ; CHECK:   [[ROSBG1:%[0-9]+]]:gr64bit = ROSBG [[ROSBG1]], [[LAY1]], 32, 63, 0, implicit-def dead $cc
-  ; CHECK:   CHIMux [[LHIMux1]], 0, implicit-def $cc
-  ; CHECK:   BRC 14, 6, %bb.6, implicit killed $cc
-  ; CHECK:   J %bb.5
-  ; CHECK: bb.5:
-  ; CHECK:   successors: %bb.6(0x80000000)
-  ; CHECK: bb.6:
-  ; CHECK:   successors: %bb.2(0x00000001), %bb.7(0x7fffffff)
-  ; CHECK:   [[LGHI]]:addr64bit = nuw nsw LA [[LGHI]], 6, $noreg
-  ; CHECK:   CGHI [[LGHI]], 0, implicit-def $cc
-  ; CHECK:   BRC 14, 8, %bb.2, implicit killed $cc
-  ; CHECK: bb.7:
-  ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   J %bb.1
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LGHI:%[0-9]+]]:addr64bit = LGHI -30
+  ; CHECK-NEXT:   [[LHIMux:%[0-9]+]]:grx32bit = LHIMux 1
+  ; CHECK-NEXT:   [[LHIMux1:%[0-9]+]]:grx32bit = LHIMux 0
+  ; CHECK-NEXT:   undef [[DEF:%[0-9]+]].subreg_l32:gr64bit = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x00000001), %bb.4(0x7fffffff)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   CHIMux [[LHIMux]], 0, implicit-def $cc
+  ; CHECK-NEXT:   BRC 14, 6, %bb.3, implicit killed $cc
+  ; CHECK-NEXT:   J %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors:
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STMux [[DEF]].subreg_l32, undef %8:addr64bit, 0, $noreg :: (store (s32) into `ptr undef`)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors:
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.5(0x30000000), %bb.6(0x50000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LAY:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19838, $noreg
+  ; CHECK-NEXT:   [[LAY1:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19840, $noreg
+  ; CHECK-NEXT:   [[LAY2:%[0-9]+]]:gr64bit = LAY [[LGHI]], 19844, $noreg
+  ; CHECK-NEXT:   [[LAY2:%[0-9]+]]:gr64bit = OGR [[LAY2]], [[LAY]], implicit-def dead $cc
+  ; CHECK-NEXT:   undef [[AHIMuxK:%[0-9]+]].subreg_l32:gr64bit = AHIMuxK [[LGHI]].subreg_l32, 19843, implicit-def dead $cc
+  ; CHECK-NEXT:   [[AHIMuxK:%[0-9]+]]:gr64bit = ROSBG [[AHIMuxK]], [[LAY2]], 32, 63, 0, implicit-def dead $cc
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]].subreg_l32:gr64bit = OR [[DEF]].subreg_l32, [[AHIMuxK]].subreg_l32, implicit-def dead $cc
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gr64bit = ROSBG [[DEF]], [[LAY1]], 32, 63, 0, implicit-def dead $cc
+  ; CHECK-NEXT:   CHIMux [[LHIMux1]], 0, implicit-def $cc
+  ; CHECK-NEXT:   BRC 14, 6, %bb.6, implicit killed $cc
+  ; CHECK-NEXT:   J %bb.5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   successors: %bb.6(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6:
+  ; CHECK-NEXT:   successors: %bb.2(0x00000001), %bb.7(0x7fffffff)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LGHI:%[0-9]+]]:addr64bit = nuw nsw LA [[LGHI]], 6, $noreg
+  ; CHECK-NEXT:   CGHI [[LGHI]], 0, implicit-def $cc
+  ; CHECK-NEXT:   BRC 14, 8, %bb.2, implicit killed $cc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.7:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   J %bb.1
   bb.0:
     %6:gr64bit = LGHI -30
     %8:grx32bit = LHIMux 1
diff --git a/llvm/test/CodeGen/SystemZ/regcoal_undefsrc.mir b/llvm/test/CodeGen/SystemZ/regcoal_undefsrc.mir
index eb1fd4e9c1487..3ee857b7b1b18 100644
--- a/llvm/test/CodeGen/SystemZ/regcoal_undefsrc.mir
+++ b/llvm/test/CodeGen/SystemZ/regcoal_undefsrc.mir
@@ -1,55 +1,38 @@
-# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -O3 -start-before=livevars %s -o /dev/null 2>&1
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -start-before=livevars -stop-after=register-coalescer %s -o - | FileCheck %s
 # Test that coalesing of an empty live range (undef) does not cause failure.
 
---- |
-  define dso_local void @fun(ptr %src, ptr %dst) #0 {
-    %1 = sdiv i64 poison, poison
-    %2 = load i32, ptr %src, align 4
-    br i1 poison, label %6, label %3
-  
-  3:                                                ; preds = %0
-    %4 = trunc i64 %1 to i32
-    %5 = udiv i32 %4, %2
-    br label %6
-  
-  6:                                                ; preds = %3, %0
-    %7 = phi i32 [ %5, %3 ], [ 1, %0 ]
-    store i32 %7, ptr %dst, align 4
-    ret void
-  }
-
-...
 ---
 name:            fun
-alignment:       16
 tracksRegLiveness: true
-registers:
-  - { id: 0, class: gr64bit }
-  - { id: 1, class: gr32bit }
-  - { id: 2, class: grx32bit }
-  - { id: 3, class: grx32bit }
-  - { id: 4, class: addr64bit }
-  - { id: 5, class: addr64bit }
-  - { id: 6, class: grx32bit }
-  - { id: 7, class: grx32bit }
-  - { id: 8, class: grx32bit }
-  - { id: 9, class: gr64bit }
-  - { id: 10, class: gr64bit }
-  - { id: 11, class: gr128bit }
-  - { id: 12, class: gr128bit }
-  - { id: 13, class: gr128bit }
-  - { id: 14, class: gr128bit }
-  - { id: 15, class: gr64bit }
-liveins:
-  - { reg: '$r2d', virtual-reg: '%4' }
-  - { reg: '$r3d', virtual-reg: '%5' }
-frameInfo:
-  maxAlignment:    1
-machineFunctionInfo: {}
 body:             |
-  bb.0 (%ir-block.0):
+  ; CHECK-LABEL: name: fun
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $r2d, $r3d
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:addr64bit = COPY $r3d
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:addr64bit = COPY $r2d
+  ; CHECK-NEXT:   [[LHIMux:%[0-9]+]]:grx32bit = LHIMux 1
+  ; CHECK-NEXT:   [[LHIMux1:%[0-9]+]]:grx32bit = LHIMux 0
+  ; CHECK-NEXT:   CHIMux [[LHIMux1]], 0, implicit-def $cc
+  ; CHECK-NEXT:   BRC 14, 6, %bb.2, implicit killed $cc
+  ; CHECK-NEXT:   J %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LMux:%[0-9]+]]:gr32bit = LMux [[COPY1]], 0, $noreg :: (load (s32))
+  ; CHECK-NEXT:   undef [[LLILL:%[0-9]+]].subreg_h64:gr128bit = LLILL 0
+  ; CHECK-NEXT:   [[LLILL:%[0-9]+]]:gr128bit = DLR [[LLILL]], [[LMux]]
+  ; CHECK-NEXT:   [[LHIMux:%[0-9]+]]:grx32bit = COPY [[LLILL]].subreg_ll32
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   STMux [[LHIMux]], [[COPY]], 0, $noreg :: (store (s32))
+  ; CHECK-NEXT:   Return
+  bb.0:
     liveins: $r2d, $r3d
-  
+
     %5:addr64bit = COPY $r3d
     %4:addr64bit = COPY $r2d
     %6:grx32bit = LHIMux 1
@@ -57,18 +40,20 @@ body:             |
     CHIMux killed %7, 0, implicit-def $cc
     BRC 14, 6, %bb.2, implicit $cc
     J %bb.1
-  
-  bb.1 (%ir-block.3):
-    %1:gr32bit = LMux %4, 0, $noreg :: (load (s32) from %ir.src)
+
+  bb.1:
+    %1:gr32bit = LMux %4, 0, $noreg :: (load (s32))
     %15:gr64bit = LLILL 0
     %14:gr128bit = INSERT_SUBREG undef %13:gr128bit, %15, %subreg.subreg_h64
     %11:gr128bit = INSERT_SUBREG %14, undef %9:gr64bit, %subreg.subreg_l64
     %12:gr128bit = DLR %11, %1
     %2:grx32bit = COPY %12.subreg_ll32
-  
-  bb.2 (%ir-block.6):
+
+  bb.2:
     %3:grx32bit = PHI %6, %bb.0, %2, %bb.1
-    STMux %3, %5, 0, $noreg :: (store (s32) into %ir.dst)
+    STMux %3, %5, 0, $noreg :: (store (s32))
     Return
 
 ...
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/Thumb/arm_q15_to_q31.ll b/llvm/test/CodeGen/Thumb/arm_q15_to_q31.ll
index fdfbf3393098e..9d69417e8f6b5 100644
--- a/llvm/test/CodeGen/Thumb/arm_q15_to_q31.ll
+++ b/llvm/test/CodeGen/Thumb/arm_q15_to_q31.ll
@@ -501,9 +501,8 @@ define void @arm_q15_to_q31_altorder(ptr nocapture noundef readonly %pSrc, ptr n
 ; CHECK-NEXT:    lsls r2, r2, #16
 ; CHECK-NEXT:    str r2, [r1, #48]
 ; CHECK-NEXT:    adds r1, #64
-; CHECK-NEXT:    subs r3, r3, #4
 ; CHECK-NEXT:    adds r0, #32
-; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    subs r3, r3, #4
 ; CHECK-NEXT:    bne .LBB1_5
 ; CHECK-NEXT:  .LBB1_6: @ %while.end
 ; CHECK-NEXT:    movs r2, #3
diff --git a/llvm/test/CodeGen/Thumb/frame-chain.ll b/llvm/test/CodeGen/Thumb/frame-chain.ll
index e68fc626be981..134362cfd0cb6 100644
--- a/llvm/test/CodeGen/Thumb/frame-chain.ll
+++ b/llvm/test/CodeGen/Thumb/frame-chain.ll
@@ -23,9 +23,9 @@ define dso_local noundef i32 @leaf(i32 noundef %0) {
 ; LEAF-FP-AAPCS:       @ %bb.0:
 ; LEAF-FP-AAPCS-NEXT:    .save {lr}
 ; LEAF-FP-AAPCS-NEXT:    push {lr}
-; LEAF-FP-AAPCS-NEXT:    mov lr, r11
+; LEAF-FP-AAPCS-NEXT:    mov r3, r11
 ; LEAF-FP-AAPCS-NEXT:    .save {r11}
-; LEAF-FP-AAPCS-NEXT:    push {lr}
+; LEAF-FP-AAPCS-NEXT:    push {r3}
 ; LEAF-FP-AAPCS-NEXT:    .setfp r11, sp
 ; LEAF-FP-AAPCS-NEXT:    mov r11, sp
 ; LEAF-FP-AAPCS-NEXT:    .pad #4
@@ -80,9 +80,9 @@ define dso_local noundef i32 @non_leaf(i32 noundef %0) {
 ; FP-AAPCS:       @ %bb.0:
 ; FP-AAPCS-NEXT:    .save {lr}
 ; FP-AAPCS-NEXT:    push {lr}
-; FP-AAPCS-NEXT:    mov lr, r11
+; FP-AAPCS-NEXT:    mov r3, r11
 ; FP-AAPCS-NEXT:    .save {r11}
-; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    push {r3}
 ; FP-AAPCS-NEXT:    .setfp r11, sp
 ; FP-AAPCS-NEXT:    mov r11, sp
 ; FP-AAPCS-NEXT:    .pad #8
@@ -161,9 +161,9 @@ define dso_local void @required_fp(i32 %0, i32 %1) {
 ; FP-AAPCS:       @ %bb.0:
 ; FP-AAPCS-NEXT:    .save {lr}
 ; FP-AAPCS-NEXT:    push {lr}
-; FP-AAPCS-NEXT:    mov lr, r11
+; FP-AAPCS-NEXT:    mov r3, r11
 ; FP-AAPCS-NEXT:    .save {r11}
-; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    push {r3}
 ; FP-AAPCS-NEXT:    .setfp r11, sp
 ; FP-AAPCS-NEXT:    mov r11, sp
 ; FP-AAPCS-NEXT:    .save {r4, r6}
@@ -227,9 +227,9 @@ define dso_local void @required_fp(i32 %0, i32 %1) {
 ; NOFP-AAPCS:       @ %bb.0:
 ; NOFP-AAPCS-NEXT:    .save {lr}
 ; NOFP-AAPCS-NEXT:    push {lr}
-; NOFP-AAPCS-NEXT:    mov lr, r11
+; NOFP-AAPCS-NEXT:    mov r3, r11
 ; NOFP-AAPCS-NEXT:    .save {r11}
-; NOFP-AAPCS-NEXT:    push {lr}
+; NOFP-AAPCS-NEXT:    push {r3}
 ; NOFP-AAPCS-NEXT:    .setfp r11, sp
 ; NOFP-AAPCS-NEXT:    mov r11, sp
 ; NOFP-AAPCS-NEXT:    .save {r4, r6}
diff --git a/llvm/test/CodeGen/Thumb/returnaddress.ll b/llvm/test/CodeGen/Thumb/returnaddress.ll
new file mode 100644
index 0000000000000..c13daf6b8c91c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb/returnaddress.ll
@@ -0,0 +1,440 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi -frame-pointer=none                          -verify-machineinstrs | FileCheck %s --check-prefix=FP-NONE
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi -frame-pointer=all                           -verify-machineinstrs | FileCheck %s --check-prefix=FP-ALL
+; RUN: llc < %s -mtriple=thumbv6m-none-eabi -frame-pointer=all -mattr=+aapcs-frame-chain -verify-machineinstrs | FileCheck %s --check-prefix=FP-AAPCS
+
+define void @ra_call() {
+; FP-NONE-LABEL: ra_call:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r7, lr}
+; FP-NONE-NEXT:    push {r7, lr}
+; FP-NONE-NEXT:    mov r0, lr
+; FP-NONE-NEXT:    bl sink_ptr
+; FP-NONE-NEXT:    pop {r7, pc}
+;
+; FP-ALL-LABEL: ra_call:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r7, lr}
+; FP-ALL-NEXT:    push {r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp
+; FP-ALL-NEXT:    add r7, sp, #0
+; FP-ALL-NEXT:    mov r0, lr
+; FP-ALL-NEXT:    bl sink_ptr
+; FP-ALL-NEXT:    pop {r7, pc}
+;
+; FP-AAPCS-LABEL: ra_call:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    mov r0, lr
+; FP-AAPCS-NEXT:    bl sink_ptr
+; FP-AAPCS-NEXT:    pop {r0}
+; FP-AAPCS-NEXT:    mov r11, r0
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  tail call void @sink_ptr(ptr %r)
+  ret void
+}
+
+define ptr @ra_return() {
+; FP-NONE-LABEL: ra_return:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    mov r0, lr
+; FP-NONE-NEXT:    bx lr
+;
+; FP-ALL-LABEL: ra_return:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r7, lr}
+; FP-ALL-NEXT:    push {r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp
+; FP-ALL-NEXT:    add r7, sp, #0
+; FP-ALL-NEXT:    mov r0, lr
+; FP-ALL-NEXT:    pop {r7, pc}
+;
+; FP-AAPCS-LABEL: ra_return:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    mov r0, lr
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %r
+}
+
+define ptr @callee_saved_low() {
+; FP-NONE-LABEL: callee_saved_low:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r4, r5, r7, lr}
+; FP-NONE-NEXT:    push {r4, r5, r7, lr}
+; FP-NONE-NEXT:    mov r0, lr
+; FP-NONE-NEXT:    @APP
+; FP-NONE-NEXT:    @NO_APP
+; FP-NONE-NEXT:    pop {r4, r5, r7, pc}
+;
+; FP-ALL-LABEL: callee_saved_low:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r4, r5, r7, lr}
+; FP-ALL-NEXT:    push {r4, r5, r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp, #8
+; FP-ALL-NEXT:    add r7, sp, #8
+; FP-ALL-NEXT:    mov r0, lr
+; FP-ALL-NEXT:    @APP
+; FP-ALL-NEXT:    @NO_APP
+; FP-ALL-NEXT:    pop {r4, r5, r7, pc}
+;
+; FP-AAPCS-LABEL: callee_saved_low:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    .save {r4, r5}
+; FP-AAPCS-NEXT:    push {r4, r5}
+; FP-AAPCS-NEXT:    mov r0, lr
+; FP-AAPCS-NEXT:    @APP
+; FP-AAPCS-NEXT:    @NO_APP
+; FP-AAPCS-NEXT:    pop {r4, r5}
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  call void asm sideeffect "", "~{r4},~{r5}"()
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %r
+}
+
+define ptr @callee_saved_high() {
+; FP-NONE-LABEL: callee_saved_high:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    mov r3, r9
+; FP-NONE-NEXT:    mov r2, r8
+; FP-NONE-NEXT:    .save {r8, r9}
+; FP-NONE-NEXT:    push {r2, r3}
+; FP-NONE-NEXT:    mov r0, lr
+; FP-NONE-NEXT:    @APP
+; FP-NONE-NEXT:    @NO_APP
+; FP-NONE-NEXT:    pop {r1, r2}
+; FP-NONE-NEXT:    mov r8, r1
+; FP-NONE-NEXT:    mov r9, r2
+; FP-NONE-NEXT:    bx lr
+;
+; FP-ALL-LABEL: callee_saved_high:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r7, lr}
+; FP-ALL-NEXT:    push {r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp
+; FP-ALL-NEXT:    add r7, sp, #0
+; FP-ALL-NEXT:    mov r3, r9
+; FP-ALL-NEXT:    mov r2, r8
+; FP-ALL-NEXT:    .save {r8, r9}
+; FP-ALL-NEXT:    push {r2, r3}
+; FP-ALL-NEXT:    mov r0, lr
+; FP-ALL-NEXT:    @APP
+; FP-ALL-NEXT:    @NO_APP
+; FP-ALL-NEXT:    pop {r1, r2}
+; FP-ALL-NEXT:    mov r8, r1
+; FP-ALL-NEXT:    mov r9, r2
+; FP-ALL-NEXT:    pop {r7, pc}
+;
+; FP-AAPCS-LABEL: callee_saved_high:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    mov r3, r9
+; FP-AAPCS-NEXT:    mov r2, r8
+; FP-AAPCS-NEXT:    .save {r8, r9}
+; FP-AAPCS-NEXT:    push {r2, r3}
+; FP-AAPCS-NEXT:    mov r0, lr
+; FP-AAPCS-NEXT:    @APP
+; FP-AAPCS-NEXT:    @NO_APP
+; FP-AAPCS-NEXT:    pop {r1, r2}
+; FP-AAPCS-NEXT:    mov r8, r1
+; FP-AAPCS-NEXT:    mov r9, r2
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  call void asm sideeffect "", "~{r8},~{r9}"()
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %r
+}
+
+define ptr @large_alloca() {
+; FP-NONE-LABEL: large_alloca:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r4, r5, r6, lr}
+; FP-NONE-NEXT:    push {r4, r5, r6, lr}
+; FP-NONE-NEXT:    ldr r6, .LCPI4_0
+; FP-NONE-NEXT:    .pad #2000
+; FP-NONE-NEXT:    add sp, r6
+; FP-NONE-NEXT:    mov r4, lr
+; FP-NONE-NEXT:    mov r0, sp
+; FP-NONE-NEXT:    bl sink_ptr
+; FP-NONE-NEXT:    mov r0, r4
+; FP-NONE-NEXT:    ldr r6, .LCPI4_1
+; FP-NONE-NEXT:    add sp, r6
+; FP-NONE-NEXT:    pop {r4, r5, r6, pc}
+; FP-NONE-NEXT:    .p2align 2
+; FP-NONE-NEXT:  @ %bb.1:
+; FP-NONE-NEXT:  .LCPI4_0:
+; FP-NONE-NEXT:    .long 4294965296 @ 0xfffff830
+; FP-NONE-NEXT:  .LCPI4_1:
+; FP-NONE-NEXT:    .long 2000 @ 0x7d0
+;
+; FP-ALL-LABEL: large_alloca:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r4, r6, r7, lr}
+; FP-ALL-NEXT:    push {r4, r6, r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp, #8
+; FP-ALL-NEXT:    add r7, sp, #8
+; FP-ALL-NEXT:    ldr r6, .LCPI4_0
+; FP-ALL-NEXT:    .pad #2000
+; FP-ALL-NEXT:    add sp, r6
+; FP-ALL-NEXT:    mov r4, lr
+; FP-ALL-NEXT:    mov r0, sp
+; FP-ALL-NEXT:    bl sink_ptr
+; FP-ALL-NEXT:    mov r0, r4
+; FP-ALL-NEXT:    subs r6, r7, #7
+; FP-ALL-NEXT:    subs r6, #1
+; FP-ALL-NEXT:    mov sp, r6
+; FP-ALL-NEXT:    pop {r4, r6, r7, pc}
+; FP-ALL-NEXT:    .p2align 2
+; FP-ALL-NEXT:  @ %bb.1:
+; FP-ALL-NEXT:  .LCPI4_0:
+; FP-ALL-NEXT:    .long 4294965296 @ 0xfffff830
+;
+; FP-AAPCS-LABEL: large_alloca:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    .save {r4, r7}
+; FP-AAPCS-NEXT:    push {r4, r7}
+; FP-AAPCS-NEXT:    ldr r7, .LCPI4_0
+; FP-AAPCS-NEXT:    .pad #2000
+; FP-AAPCS-NEXT:    add sp, r7
+; FP-AAPCS-NEXT:    mov r4, lr
+; FP-AAPCS-NEXT:    mov r0, sp
+; FP-AAPCS-NEXT:    bl sink_ptr
+; FP-AAPCS-NEXT:    mov r0, r4
+; FP-AAPCS-NEXT:    mov r7, r11
+; FP-AAPCS-NEXT:    subs r7, #8
+; FP-AAPCS-NEXT:    mov sp, r7
+; FP-AAPCS-NEXT:    pop {r4, r7}
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+; FP-AAPCS-NEXT:    .p2align 2
+; FP-AAPCS-NEXT:  @ %bb.1:
+; FP-AAPCS-NEXT:  .LCPI4_0:
+; FP-AAPCS-NEXT:    .long 4294965296 @ 0xfffff830
+entry:
+  %big = alloca i8, i32 2000
+  tail call void @sink_ptr(ptr %big)
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %r
+}
+
+define ptr @var_alloca(i32 %size) {
+; FP-NONE-LABEL: var_alloca:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r4, r6, r7, lr}
+; FP-NONE-NEXT:    push {r4, r6, r7, lr}
+; FP-NONE-NEXT:    .setfp r7, sp, #8
+; FP-NONE-NEXT:    add r7, sp, #8
+; FP-NONE-NEXT:    mov r6, sp
+; FP-NONE-NEXT:    mov r4, lr
+; FP-NONE-NEXT:    adds r0, r0, #7
+; FP-NONE-NEXT:    movs r1, #7
+; FP-NONE-NEXT:    bics r0, r1
+; FP-NONE-NEXT:    mov r1, sp
+; FP-NONE-NEXT:    subs r0, r1, r0
+; FP-NONE-NEXT:    mov sp, r0
+; FP-NONE-NEXT:    bl sink_ptr
+; FP-NONE-NEXT:    mov r0, r4
+; FP-NONE-NEXT:    subs r6, r7, #7
+; FP-NONE-NEXT:    subs r6, #1
+; FP-NONE-NEXT:    mov sp, r6
+; FP-NONE-NEXT:    pop {r4, r6, r7, pc}
+;
+; FP-ALL-LABEL: var_alloca:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r4, r6, r7, lr}
+; FP-ALL-NEXT:    push {r4, r6, r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp, #8
+; FP-ALL-NEXT:    add r7, sp, #8
+; FP-ALL-NEXT:    mov r6, sp
+; FP-ALL-NEXT:    mov r4, lr
+; FP-ALL-NEXT:    adds r0, r0, #7
+; FP-ALL-NEXT:    movs r1, #7
+; FP-ALL-NEXT:    bics r0, r1
+; FP-ALL-NEXT:    mov r1, sp
+; FP-ALL-NEXT:    subs r0, r1, r0
+; FP-ALL-NEXT:    mov sp, r0
+; FP-ALL-NEXT:    bl sink_ptr
+; FP-ALL-NEXT:    mov r0, r4
+; FP-ALL-NEXT:    subs r6, r7, #7
+; FP-ALL-NEXT:    subs r6, #1
+; FP-ALL-NEXT:    mov sp, r6
+; FP-ALL-NEXT:    pop {r4, r6, r7, pc}
+;
+; FP-AAPCS-LABEL: var_alloca:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    .save {r4, r6}
+; FP-AAPCS-NEXT:    push {r4, r6}
+; FP-AAPCS-NEXT:    mov r6, sp
+; FP-AAPCS-NEXT:    mov r4, lr
+; FP-AAPCS-NEXT:    adds r0, r0, #7
+; FP-AAPCS-NEXT:    movs r1, #7
+; FP-AAPCS-NEXT:    bics r0, r1
+; FP-AAPCS-NEXT:    mov r1, sp
+; FP-AAPCS-NEXT:    subs r0, r1, r0
+; FP-AAPCS-NEXT:    mov sp, r0
+; FP-AAPCS-NEXT:    bl sink_ptr
+; FP-AAPCS-NEXT:    mov r0, r4
+; FP-AAPCS-NEXT:    mov r6, r11
+; FP-AAPCS-NEXT:    subs r6, #8
+; FP-AAPCS-NEXT:    mov sp, r6
+; FP-AAPCS-NEXT:    pop {r4, r6}
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  %var = alloca i8, i32 %size
+  tail call void @sink_ptr(ptr %var)
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %r
+}
+
+define i32 @all_arg_regs(i32 %a, i32 %b, i32 %c, i32 %d) {
+; FP-NONE-LABEL: all_arg_regs:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r4, lr}
+; FP-NONE-NEXT:    push {r4, lr}
+; FP-NONE-NEXT:    mov r4, lr
+; FP-NONE-NEXT:    adds r0, r0, r1
+; FP-NONE-NEXT:    adds r0, r0, r2
+; FP-NONE-NEXT:    adds r0, r0, r3
+; FP-NONE-NEXT:    adds r0, r0, r4
+; FP-NONE-NEXT:    pop {r4, pc}
+;
+; FP-ALL-LABEL: all_arg_regs:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r4, r6, r7, lr}
+; FP-ALL-NEXT:    push {r4, r6, r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp, #8
+; FP-ALL-NEXT:    add r7, sp, #8
+; FP-ALL-NEXT:    mov r4, lr
+; FP-ALL-NEXT:    adds r0, r0, r1
+; FP-ALL-NEXT:    adds r0, r0, r2
+; FP-ALL-NEXT:    adds r0, r0, r3
+; FP-ALL-NEXT:    adds r0, r0, r4
+; FP-ALL-NEXT:    pop {r4, r6, r7, pc}
+;
+; FP-AAPCS-LABEL: all_arg_regs:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov lr, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    .save {r4, r7}
+; FP-AAPCS-NEXT:    push {r4, r7}
+; FP-AAPCS-NEXT:    ldr r7, [sp, #12]
+; FP-AAPCS-NEXT:    mov lr, r7
+; FP-AAPCS-NEXT:    mov r4, lr
+; FP-AAPCS-NEXT:    adds r0, r0, r1
+; FP-AAPCS-NEXT:    adds r0, r0, r2
+; FP-AAPCS-NEXT:    adds r0, r0, r3
+; FP-AAPCS-NEXT:    adds r0, r0, r4
+; FP-AAPCS-NEXT:    pop {r4, r7}
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  %r = tail call ptr @llvm.returnaddress(i32 0)
+  %ri = ptrtoint ptr %r to i32
+  %t1 = add i32 %a, %b
+  %t2 = add i32 %t1, %c
+  %t3 = add i32 %t2, %d
+  %t4 = add i32 %t3, %ri
+  ret i32 %t4
+}
+
+define ptr @ra_depth_1() {
+; FP-NONE-LABEL: ra_depth_1:
+; FP-NONE:       @ %bb.0: @ %entry
+; FP-NONE-NEXT:    .save {r7, lr}
+; FP-NONE-NEXT:    push {r7, lr}
+; FP-NONE-NEXT:    .setfp r7, sp
+; FP-NONE-NEXT:    add r7, sp, #0
+; FP-NONE-NEXT:    ldr r0, [r7]
+; FP-NONE-NEXT:    ldr r0, [r0, #4]
+; FP-NONE-NEXT:    pop {r7, pc}
+;
+; FP-ALL-LABEL: ra_depth_1:
+; FP-ALL:       @ %bb.0: @ %entry
+; FP-ALL-NEXT:    .save {r7, lr}
+; FP-ALL-NEXT:    push {r7, lr}
+; FP-ALL-NEXT:    .setfp r7, sp
+; FP-ALL-NEXT:    add r7, sp, #0
+; FP-ALL-NEXT:    ldr r0, [r7]
+; FP-ALL-NEXT:    ldr r0, [r0, #4]
+; FP-ALL-NEXT:    pop {r7, pc}
+;
+; FP-AAPCS-LABEL: ra_depth_1:
+; FP-AAPCS:       @ %bb.0: @ %entry
+; FP-AAPCS-NEXT:    .save {lr}
+; FP-AAPCS-NEXT:    push {lr}
+; FP-AAPCS-NEXT:    mov r3, r11
+; FP-AAPCS-NEXT:    .save {r11}
+; FP-AAPCS-NEXT:    push {r3}
+; FP-AAPCS-NEXT:    .setfp r11, sp
+; FP-AAPCS-NEXT:    mov r11, sp
+; FP-AAPCS-NEXT:    mov r0, r11
+; FP-AAPCS-NEXT:    ldr r0, [r0]
+; FP-AAPCS-NEXT:    ldr r0, [r0, #4]
+; FP-AAPCS-NEXT:    pop {r1}
+; FP-AAPCS-NEXT:    mov r11, r1
+; FP-AAPCS-NEXT:    pop {pc}
+entry:
+  %r = tail call ptr @llvm.returnaddress(i32 1)
+  ret ptr %r
+}
+
+declare void @sink_ptr(ptr)
diff --git a/llvm/test/CodeGen/Thumb/select.ll b/llvm/test/CodeGen/Thumb/select.ll
index 0065616c4646e..89cc4c10fe51d 100644
--- a/llvm/test/CodeGen/Thumb/select.ll
+++ b/llvm/test/CodeGen/Thumb/select.ll
@@ -94,8 +94,8 @@ entry:
 ; CHECK-LABEL: f8:
 ; CHECK: cmp r0, r1
 ; CHECK: blt
-; CHECK: movs
-; CHECK: cmp r0, r1
+; CHECK: mov
+; CHECK: mov
 ; CHECK: blt
 ; CHECK: movs
 ; CHECK: movs
diff --git a/llvm/test/CodeGen/Thumb/smul_fix_sat.ll b/llvm/test/CodeGen/Thumb/smul_fix_sat.ll
index f8557419c4199..8bc39ea0370a6 100644
--- a/llvm/test/CodeGen/Thumb/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/Thumb/smul_fix_sat.ll
@@ -49,25 +49,24 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    sub sp, #36
 ; ARM-NEXT:    str r3, [sp, #28] @ 4-byte Spill
 ; ARM-NEXT:    mov r6, r1
-; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    movs r4, #0
-; ARM-NEXT:    str r4, [sp, #32] @ 4-byte Spill
 ; ARM-NEXT:    mov r5, r0
-; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r7, r2
-; ARM-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; ARM-NEXT:    str r2, [sp, #12] @ 4-byte Spill
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; ARM-NEXT:    str r1, [sp, #24] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #32] @ 4-byte Spill
 ; ARM-NEXT:    mov r0, r6
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r7
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r6, r1
-; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:    adds r7, r0, r1
 ; ARM-NEXT:    adcs r6, r4
 ; ARM-NEXT:    mov r0, r5
@@ -77,118 +76,108 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r0, r0, r7
-; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; ARM-NEXT:    adcs r1, r4
 ; ARM-NEXT:    adds r0, r6, r1
-; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    mov r6, r4
 ; ARM-NEXT:    adcs r6, r4
-; ARM-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:    mov r0, r7
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r5
+; ARM-NEXT:    str r4, [sp, #32] @ 4-byte Spill
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    adcs r5, r6
+; ARM-NEXT:    mov r4, r7
 ; ARM-NEXT:    asrs r2, r7, #31
-; ARM-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; ARM-NEXT:    ldr r4, [sp, #28] @ 4-byte Reload
-; ARM-NEXT:    mov r1, r4
+; ARM-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
+; ARM-NEXT:    mov r1, r7
 ; ARM-NEXT:    mov r3, r2
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r6, r0
-; ARM-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; ARM-NEXT:    asrs r0, r4, #31
+; ARM-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    asrs r0, r7, #31
 ; ARM-NEXT:    mov r1, r0
-; ARM-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; ARM-NEXT:    mov r3, r7
+; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r0, r0, r6
-; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; ARM-NEXT:    adcs r1, r2
 ; ARM-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; ARM-NEXT:    adds r0, r2, r0
+; ARM-NEXT:    adcs r1, r2
+; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    adds r3, r2, r0
 ; ARM-NEXT:    adcs r1, r5
-; ARM-NEXT:    rsbs r5, r1, #0
-; ARM-NEXT:    adcs r5, r1
-; ARM-NEXT:    movs r2, #1
-; ARM-NEXT:    str r0, [sp, #28] @ 4-byte Spill
-; ARM-NEXT:    cmp r0, #1
-; ARM-NEXT:    mov r3, r2
+; ARM-NEXT:    rsbs r2, r1, #0
+; ARM-NEXT:    adcs r2, r1
+; ARM-NEXT:    movs r0, #1
+; ARM-NEXT:    cmp r3, #1
+; ARM-NEXT:    mov r5, r0
 ; ARM-NEXT:    bhi .LBB1_2
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
+; ARM-NEXT:    ldr r5, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:  .LBB1_2:
-; ARM-NEXT:    ands r5, r3
+; ARM-NEXT:    ands r2, r5
 ; ARM-NEXT:    cmp r1, #0
-; ARM-NEXT:    mov r3, r2
+; ARM-NEXT:    mov r5, r0
 ; ARM-NEXT:    bgt .LBB1_4
 ; ARM-NEXT:  @ %bb.3:
-; ARM-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
+; ARM-NEXT:    ldr r5, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:  .LBB1_4:
-; ARM-NEXT:    orrs r3, r5
-; ARM-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
-; ARM-NEXT:    mvns r6, r0
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    str r6, [sp, #20] @ 4-byte Spill
-; ARM-NEXT:    bne .LBB1_6
+; ARM-NEXT:    orrs r5, r2
+; ARM-NEXT:    lsls r2, r3, #30
+; ARM-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    lsrs r4, r6, #2
+; ARM-NEXT:    adds r2, r2, r4
+; ARM-NEXT:    lsls r4, r6, #30
+; ARM-NEXT:    ldr r6, [sp, #24] @ 4-byte Reload
+; ARM-NEXT:    lsrs r6, r6, #2
+; ARM-NEXT:    adds r7, r4, r6
+; ARM-NEXT:    ldr r4, [sp, #32] @ 4-byte Reload
+; ARM-NEXT:    mvns r6, r4
+; ARM-NEXT:    cmp r5, #0
+; ARM-NEXT:    beq .LBB1_6
 ; ARM-NEXT:  @ %bb.5:
-; ARM-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; ARM-NEXT:    lsls r0, r0, #30
-; ARM-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
-; ARM-NEXT:    lsrs r4, r4, #2
-; ARM-NEXT:    adds r0, r0, r4
-; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    ldr r2, .LCPI1_0
 ; ARM-NEXT:  .LBB1_6:
-; ARM-NEXT:    adds r0, r1, #1
-; ARM-NEXT:    rsbs r7, r0, #0
-; ARM-NEXT:    adcs r7, r0
-; ARM-NEXT:    mvns r0, r2
-; ARM-NEXT:    ldr r5, [sp, #28] @ 4-byte Reload
-; ARM-NEXT:    cmp r5, r0
-; ARM-NEXT:    mov r0, r2
-; ARM-NEXT:    blo .LBB1_8
+; ARM-NEXT:    mov r5, r6
+; ARM-NEXT:    bne .LBB1_8
 ; ARM-NEXT:  @ %bb.7:
-; ARM-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
+; ARM-NEXT:    mov r5, r7
 ; ARM-NEXT:  .LBB1_8:
-; ARM-NEXT:    ands r7, r0
-; ARM-NEXT:    cmp r1, r6
-; ARM-NEXT:    mov r6, r2
-; ARM-NEXT:    bge .LBB1_12
+; ARM-NEXT:    adds r4, r1, #1
+; ARM-NEXT:    rsbs r7, r4, #0
+; ARM-NEXT:    adcs r7, r4
+; ARM-NEXT:    mvns r4, r0
+; ARM-NEXT:    cmp r3, r4
+; ARM-NEXT:    mov r3, r0
+; ARM-NEXT:    blo .LBB1_10
 ; ARM-NEXT:  @ %bb.9:
-; ARM-NEXT:    orrs r6, r7
-; ARM-NEXT:    beq .LBB1_13
+; ARM-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:  .LBB1_10:
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    bne .LBB1_14
-; ARM-NEXT:  .LBB1_11:
-; ARM-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
-; ARM-NEXT:    lsls r0, r0, #30
-; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; ARM-NEXT:    lsrs r1, r1, #2
-; ARM-NEXT:    adds r1, r0, r1
-; ARM-NEXT:    cmp r6, #0
-; ARM-NEXT:    bne .LBB1_15
-; ARM-NEXT:    b .LBB1_16
+; ARM-NEXT:    ands r7, r3
+; ARM-NEXT:    cmp r1, r6
+; ARM-NEXT:    mov r3, r0
+; ARM-NEXT:    blt .LBB1_12
+; ARM-NEXT:  @ %bb.11:
+; ARM-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:  .LBB1_12:
-; ARM-NEXT:    ldr r6, [sp, #32] @ 4-byte Reload
-; ARM-NEXT:    orrs r6, r7
-; ARM-NEXT:    bne .LBB1_10
-; ARM-NEXT:  .LBB1_13:
-; ARM-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; ARM-NEXT:    str r0, [sp, #32] @ 4-byte Spill
+; ARM-NEXT:    orrs r3, r7
+; ARM-NEXT:    lsls r1, r0, #31
 ; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    beq .LBB1_11
+; ARM-NEXT:    bne .LBB1_14
+; ARM-NEXT:  @ %bb.13:
+; ARM-NEXT:    str r5, [sp, #32] @ 4-byte Spill
 ; ARM-NEXT:  .LBB1_14:
-; ARM-NEXT:    ldr r1, .LCPI1_0
-; ARM-NEXT:    cmp r6, #0
-; ARM-NEXT:    beq .LBB1_16
-; ARM-NEXT:  .LBB1_15:
-; ARM-NEXT:    lsls r1, r2, #31
+; ARM-NEXT:    bne .LBB1_16
+; ARM-NEXT:  @ %bb.15:
+; ARM-NEXT:    mov r1, r2
 ; ARM-NEXT:  .LBB1_16:
 ; ARM-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
 ; ARM-NEXT:    add sp, #36
@@ -282,44 +271,44 @@ define i64 @func5(i64 %x, i64 %y) {
 ; ARM-NEXT:    push {r4, r5, r6, r7, lr}
 ; ARM-NEXT:    .pad #28
 ; ARM-NEXT:    sub sp, #28
-; ARM-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; ARM-NEXT:    mov r5, r2
+; ARM-NEXT:    str r3, [sp, #24] @ 4-byte Spill
+; ARM-NEXT:    mov r4, r2
 ; ARM-NEXT:    str r2, [sp, #16] @ 4-byte Spill
-; ARM-NEXT:    mov r6, r1
+; ARM-NEXT:    mov r5, r1
 ; ARM-NEXT:    movs r7, #0
-; ARM-NEXT:    mov r4, r0
+; ARM-NEXT:    mov r6, r0
 ; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; ARM-NEXT:    mov r1, r7
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; ARM-NEXT:    str r1, [sp, #24] @ 4-byte Spill
-; ARM-NEXT:    mov r0, r6
+; ARM-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    mov r0, r5
 ; ARM-NEXT:    mov r1, r7
-; ARM-NEXT:    mov r2, r5
+; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; ARM-NEXT:    mov r4, r1
+; ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; ARM-NEXT:    adcs r5, r7
-; ARM-NEXT:    mov r0, r4
+; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    adcs r4, r7
+; ARM-NEXT:    mov r0, r6
 ; ARM-NEXT:    mov r1, r7
-; ARM-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; ARM-NEXT:    mov r2, r4
+; ARM-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
+; ARM-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r2
-; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
 ; ARM-NEXT:    adcs r1, r7
-; ARM-NEXT:    adds r0, r5, r1
+; ARM-NEXT:    adds r0, r4, r1
 ; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; ARM-NEXT:    mov r5, r7
-; ARM-NEXT:    adcs r5, r7
-; ARM-NEXT:    mov r0, r6
+; ARM-NEXT:    mov r6, r7
+; ARM-NEXT:    adcs r6, r7
+; ARM-NEXT:    mov r0, r5
 ; ARM-NEXT:    mov r1, r7
+; ARM-NEXT:    ldr r4, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
@@ -327,50 +316,48 @@ define i64 @func5(i64 %x, i64 %y) {
 ; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
 ; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; ARM-NEXT:    adcs r7, r5
-; ARM-NEXT:    asrs r2, r6, #31
+; ARM-NEXT:    adcs r7, r6
+; ARM-NEXT:    asrs r2, r5, #31
 ; ARM-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; ARM-NEXT:    mov r5, r4
+; ARM-NEXT:    mov r6, r4
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r3, r2
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r4, r0
 ; ARM-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; ARM-NEXT:    asrs r0, r5, #31
+; ARM-NEXT:    asrs r0, r6, #31
 ; ARM-NEXT:    mov r1, r0
 ; ARM-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; ARM-NEXT:    mov r3, r6
+; ARM-NEXT:    mov r3, r5
 ; ARM-NEXT:    bl __aeabi_lmul
+; ARM-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r4
-; ARM-NEXT:    ldr r4, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    adcs r1, r2
 ; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; ARM-NEXT:    adds r3, r2, r0
+; ARM-NEXT:    adds r2, r2, r0
 ; ARM-NEXT:    adcs r1, r7
-; ARM-NEXT:    asrs r2, r4, #31
-; ARM-NEXT:    eors r1, r2
-; ARM-NEXT:    eors r3, r2
-; ARM-NEXT:    orrs r3, r1
-; ARM-NEXT:    eors r6, r5
-; ARM-NEXT:    asrs r1, r6, #31
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    bne .LBB4_3
+; ARM-NEXT:    asrs r0, r3, #31
+; ARM-NEXT:    eors r1, r0
+; ARM-NEXT:    eors r2, r0
+; ARM-NEXT:    orrs r2, r1
+; ARM-NEXT:    eors r5, r6
+; ARM-NEXT:    asrs r0, r5, #31
+; ARM-NEXT:    ldr r1, .LCPI4_0
+; ARM-NEXT:    eors r1, r0
+; ARM-NEXT:    mvns r0, r0
+; ARM-NEXT:    cmp r2, #0
+; ARM-NEXT:    beq .LBB4_3
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; ARM-NEXT:    cmp r3, #0
 ; ARM-NEXT:    beq .LBB4_4
 ; ARM-NEXT:  .LBB4_2:
-; ARM-NEXT:    ldr r2, .LCPI4_0
-; ARM-NEXT:    eors r1, r2
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
 ; ARM-NEXT:  .LBB4_3:
-; ARM-NEXT:    mvns r0, r1
-; ARM-NEXT:    cmp r3, #0
+; ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:    bne .LBB4_2
 ; ARM-NEXT:  .LBB4_4:
-; ARM-NEXT:    mov r1, r4
+; ARM-NEXT:    mov r1, r3
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
 ; ARM-NEXT:    .p2align 2
@@ -483,79 +470,67 @@ define i64 @func7(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    mov r2, r1
 ; ARM-NEXT:    adds r0, r0, r4
-; ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; ARM-NEXT:    adcs r2, r1
-; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; ARM-NEXT:    adds r0, r1, r0
-; ARM-NEXT:    adcs r2, r7
-; ARM-NEXT:    rsbs r5, r2, #0
-; ARM-NEXT:    adcs r5, r2
-; ARM-NEXT:    movs r4, #1
+; ARM-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    adcs r1, r2
+; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    adds r0, r2, r0
+; ARM-NEXT:    adcs r1, r7
+; ARM-NEXT:    rsbs r5, r1, #0
+; ARM-NEXT:    adcs r5, r1
+; ARM-NEXT:    movs r2, #1
 ; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
 ; ARM-NEXT:    cmp r0, #0
-; ARM-NEXT:    mov r3, r4
-; ARM-NEXT:    bmi .LBB6_2
+; ARM-NEXT:    mov r3, r2
+; ARM-NEXT:    bge .LBB6_2
 ; ARM-NEXT:  @ %bb.1:
 ; ARM-NEXT:    mov r3, r6
 ; ARM-NEXT:  .LBB6_2:
-; ARM-NEXT:    ands r5, r3
-; ARM-NEXT:    cmp r2, #0
-; ARM-NEXT:    mov r1, r4
-; ARM-NEXT:    mov r3, r4
-; ARM-NEXT:    bgt .LBB6_4
+; ARM-NEXT:    mov r4, r2
+; ARM-NEXT:    bmi .LBB6_4
 ; ARM-NEXT:  @ %bb.3:
-; ARM-NEXT:    mov r3, r6
+; ARM-NEXT:    mov r4, r6
 ; ARM-NEXT:  .LBB6_4:
-; ARM-NEXT:    orrs r3, r5
-; ARM-NEXT:    mvns r4, r6
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    mov r5, r4
-; ARM-NEXT:    bne .LBB6_6
+; ARM-NEXT:    ands r5, r4
+; ARM-NEXT:    cmp r1, #0
+; ARM-NEXT:    mov r7, r2
+; ARM-NEXT:    bgt .LBB6_6
 ; ARM-NEXT:  @ %bb.5:
-; ARM-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    mov r7, r6
 ; ARM-NEXT:  .LBB6_6:
-; ARM-NEXT:    adds r0, r2, #1
-; ARM-NEXT:    rsbs r7, r0, #0
-; ARM-NEXT:    adcs r7, r0
-; ARM-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; ARM-NEXT:    cmp r0, #0
-; ARM-NEXT:    mov r0, r1
-; ARM-NEXT:    bge .LBB6_8
+; ARM-NEXT:    orrs r7, r5
+; ARM-NEXT:    mvns r4, r6
+; ARM-NEXT:    cmp r7, #0
+; ARM-NEXT:    beq .LBB6_8
 ; ARM-NEXT:  @ %bb.7:
-; ARM-NEXT:    mov r0, r6
+; ARM-NEXT:    ldr r0, .LCPI6_0
+; ARM-NEXT:    str r0, [sp, #24] @ 4-byte Spill
 ; ARM-NEXT:  .LBB6_8:
-; ARM-NEXT:    ands r7, r0
-; ARM-NEXT:    cmp r2, r4
-; ARM-NEXT:    mov r0, r1
-; ARM-NEXT:    mov r2, r1
-; ARM-NEXT:    bge .LBB6_12
+; ARM-NEXT:    mov r5, r4
+; ARM-NEXT:    bne .LBB6_10
 ; ARM-NEXT:  @ %bb.9:
-; ARM-NEXT:    orrs r2, r7
-; ARM-NEXT:    beq .LBB6_13
+; ARM-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:  .LBB6_10:
+; ARM-NEXT:    adds r0, r1, #1
+; ARM-NEXT:    rsbs r7, r0, #0
+; ARM-NEXT:    adcs r7, r0
+; ARM-NEXT:    ands r7, r3
+; ARM-NEXT:    cmp r1, r4
+; ARM-NEXT:    mov r3, r2
+; ARM-NEXT:    blt .LBB6_12
+; ARM-NEXT:  @ %bb.11:
+; ARM-NEXT:    mov r3, r6
+; ARM-NEXT:  .LBB6_12:
+; ARM-NEXT:    orrs r3, r7
+; ARM-NEXT:    lsls r1, r2, #31
 ; ARM-NEXT:    cmp r3, #0
 ; ARM-NEXT:    bne .LBB6_14
-; ARM-NEXT:  .LBB6_11:
-; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; ARM-NEXT:    cmp r2, #0
-; ARM-NEXT:    bne .LBB6_15
-; ARM-NEXT:    b .LBB6_16
-; ARM-NEXT:  .LBB6_12:
-; ARM-NEXT:    mov r2, r6
-; ARM-NEXT:    orrs r2, r7
-; ARM-NEXT:    bne .LBB6_10
-; ARM-NEXT:  .LBB6_13:
+; ARM-NEXT:  @ %bb.13:
 ; ARM-NEXT:    mov r6, r5
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    beq .LBB6_11
 ; ARM-NEXT:  .LBB6_14:
-; ARM-NEXT:    ldr r1, .LCPI6_0
-; ARM-NEXT:    cmp r2, #0
-; ARM-NEXT:    beq .LBB6_16
-; ARM-NEXT:  .LBB6_15:
-; ARM-NEXT:    lsls r1, r0, #31
+; ARM-NEXT:    bne .LBB6_16
+; ARM-NEXT:  @ %bb.15:
+; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:  .LBB6_16:
 ; ARM-NEXT:    mov r0, r6
 ; ARM-NEXT:    add sp, #28
@@ -577,23 +552,23 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    sub sp, #28
 ; ARM-NEXT:    str r3, [sp, #24] @ 4-byte Spill
 ; ARM-NEXT:    mov r5, r2
-; ARM-NEXT:    str r2, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r2, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    mov r4, r1
 ; ARM-NEXT:    str r1, [sp, #20] @ 4-byte Spill
 ; ARM-NEXT:    movs r7, #0
 ; ARM-NEXT:    mov r6, r0
-; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
 ; ARM-NEXT:    mov r1, r7
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    mov r0, r4
 ; ARM-NEXT:    mov r1, r7
 ; ARM-NEXT:    mov r2, r5
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    adds r4, r0, r1
 ; ARM-NEXT:    adcs r5, r7
 ; ARM-NEXT:    mov r0, r6
@@ -603,10 +578,10 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r0, r0, r4
-; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    adcs r1, r7
 ; ARM-NEXT:    adds r0, r5, r1
-; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    mov r4, r7
 ; ARM-NEXT:    adcs r4, r7
 ; ARM-NEXT:    ldr r5, [sp, #20] @ 4-byte Reload
@@ -616,75 +591,70 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r3, r7
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r6, r1
-; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    adcs r6, r4
 ; ARM-NEXT:    asrs r2, r5, #31
-; ARM-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; ARM-NEXT:    ldr r4, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r3, r2
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r5, r0
-; ARM-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    asrs r0, r4, #31
 ; ARM-NEXT:    mov r1, r0
-; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r0, r0, r5
-; ARM-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; ARM-NEXT:    adcs r1, r2
 ; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; ARM-NEXT:    adds r0, r2, r0
+; ARM-NEXT:    adcs r1, r2
+; ARM-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    adds r2, r2, r0
 ; ARM-NEXT:    adcs r1, r6
-; ARM-NEXT:    ldr r2, .LCPI7_0
-; ARM-NEXT:    cmp r1, r2
-; ARM-NEXT:    bgt .LBB7_2
+; ARM-NEXT:    lsls r0, r1, #1
+; ARM-NEXT:    lsrs r3, r2, #31
+; ARM-NEXT:    adds r0, r0, r3
+; ARM-NEXT:    lsls r2, r2, #1
+; ARM-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    lsrs r3, r3, #31
+; ARM-NEXT:    adds r2, r2, r3
+; ARM-NEXT:    mvns r3, r7
+; ARM-NEXT:    ldr r4, .LCPI7_1
+; ARM-NEXT:    cmp r1, r4
+; ARM-NEXT:    ble .LBB7_2
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    lsls r3, r0, #1
-; ARM-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
-; ARM-NEXT:    lsrs r4, r4, #31
-; ARM-NEXT:    adds r5, r3, r4
-; ARM-NEXT:    b .LBB7_3
+; ARM-NEXT:    ldr r0, .LCPI7_0
 ; ARM-NEXT:  .LBB7_2:
-; ARM-NEXT:    mvns r5, r7
-; ARM-NEXT:  .LBB7_3:
-; ARM-NEXT:    movs r3, #3
-; ARM-NEXT:    lsls r3, r3, #30
-; ARM-NEXT:    cmp r1, r3
-; ARM-NEXT:    blt .LBB7_5
-; ARM-NEXT:  @ %bb.4:
-; ARM-NEXT:    mov r7, r5
-; ARM-NEXT:  .LBB7_5:
-; ARM-NEXT:    cmp r1, r2
-; ARM-NEXT:    bgt .LBB7_7
-; ARM-NEXT:  @ %bb.6:
-; ARM-NEXT:    lsls r2, r1, #1
-; ARM-NEXT:    lsrs r0, r0, #31
-; ARM-NEXT:    adds r2, r2, r0
-; ARM-NEXT:    cmp r1, r3
+; ARM-NEXT:    bgt .LBB7_4
+; ARM-NEXT:  @ %bb.3:
+; ARM-NEXT:    mov r3, r2
+; ARM-NEXT:  .LBB7_4:
+; ARM-NEXT:    movs r2, #1
+; ARM-NEXT:    lsls r2, r2, #31
+; ARM-NEXT:    movs r4, #3
+; ARM-NEXT:    lsls r4, r4, #30
+; ARM-NEXT:    cmp r1, r4
+; ARM-NEXT:    blt .LBB7_6
+; ARM-NEXT:  @ %bb.5:
+; ARM-NEXT:    mov r7, r3
+; ARM-NEXT:  .LBB7_6:
 ; ARM-NEXT:    blt .LBB7_8
-; ARM-NEXT:    b .LBB7_9
-; ARM-NEXT:  .LBB7_7:
-; ARM-NEXT:    ldr r2, .LCPI7_1
-; ARM-NEXT:    cmp r1, r3
-; ARM-NEXT:    bge .LBB7_9
+; ARM-NEXT:  @ %bb.7:
+; ARM-NEXT:    mov r2, r0
 ; ARM-NEXT:  .LBB7_8:
-; ARM-NEXT:    movs r0, #1
-; ARM-NEXT:    lsls r2, r0, #31
-; ARM-NEXT:  .LBB7_9:
 ; ARM-NEXT:    mov r0, r7
 ; ARM-NEXT:    mov r1, r2
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
 ; ARM-NEXT:    .p2align 2
-; ARM-NEXT:  @ %bb.10:
+; ARM-NEXT:  @ %bb.9:
 ; ARM-NEXT:  .LCPI7_0:
-; ARM-NEXT:    .long 1073741823 @ 0x3fffffff
-; ARM-NEXT:  .LCPI7_1:
 ; ARM-NEXT:    .long 2147483647 @ 0x7fffffff
+; ARM-NEXT:  .LCPI7_1:
+; ARM-NEXT:    .long 1073741823 @ 0x3fffffff
   %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63)
   ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/Thumb/stack-guard-xo.ll b/llvm/test/CodeGen/Thumb/stack-guard-xo.ll
index c344e4f7fc022..421583d07ac7f 100644
--- a/llvm/test/CodeGen/Thumb/stack-guard-xo.ll
+++ b/llvm/test/CodeGen/Thumb/stack-guard-xo.ll
@@ -33,9 +33,9 @@ define dso_local i32 @main() #0 {
 ; V6M-LABEL: main:
 ; V6M:       @ %bb.0: @ %entry
 ; V6M-NEXT:    push {r7, lr}
-; V6M-NEXT:    sub sp, #
+; V6M-NEXT:    sub sp, #24
 ; V6M-NEXT:    movs r0, #0
-; V6M-NEXT:    str r0, [sp, #
+; V6M-NEXT:    str r0, [sp, #4]
 ; V6M-NEXT:    mrs r12, apsr
 ; V6M-NEXT:    movs r0, :upper8_15:__stack_chk_guard
 ; V6M-NEXT:    lsls r0, r0, #8
@@ -46,10 +46,10 @@ define dso_local i32 @main() #0 {
 ; V6M-NEXT:    adds r0, :lower0_7:__stack_chk_guard
 ; V6M-NEXT:    msr apsr, r12
 ; V6M-NEXT:    ldr r0, [r0]
-; V6M-NEXT:    str r0, [sp, #
-; V6M-NEXT:    add r0, sp, #
+; V6M-NEXT:    str r0, [sp, #20]
+; V6M-NEXT:    add r0, sp, #8
 ; V6M-NEXT:    ldrb r0, [r0]
-; V6M-NEXT:    ldr r1, [sp, #
+; V6M-NEXT:    ldr r1, [sp, #20]
 ; V6M-NEXT:    mrs r12, apsr
 ; V6M-NEXT:    movs r2, :upper8_15:__stack_chk_guard
 ; V6M-NEXT:    lsls r2, r2, #8
@@ -63,7 +63,7 @@ define dso_local i32 @main() #0 {
 ; V6M-NEXT:    cmp r2, r1
 ; V6M-NEXT:    bne .LBB0_2
 ; V6M-NEXT:  @ %bb.1: @ %entry
-; V6M-NEXT:    add sp, #
+; V6M-NEXT:    add sp, #24
 ; V6M-NEXT:    pop {r7, pc}
 ; V6M-NEXT:  .LBB0_2: @ %entry
 ; V6M-NEXT:    bl __stack_chk_fail
@@ -105,77 +105,177 @@ entry:
 @bb = hidden local_unnamed_addr global i64 0, align 8
 
 define dso_local i64 @cc() local_unnamed_addr #1 {
+; CHECK-LABEL: cc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    movs r0, #1
+; CHECK-NEXT:    lsls r3, r0, #31
+; CHECK-NEXT:    ldr r0, .LCPI1_0
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    asrs r4, r2, #31
+; CHECK-NEXT:    eors r3, r4
+; CHECK-NEXT:    ldr r0, .LCPI1_1
+; CHECK-NEXT:    ldm r0!, {r1, r5}
+; CHECK-NEXT:    subs r0, r2, r1
+; CHECK-NEXT:    sbcs r3, r5
+; CHECK-NEXT:    subs r0, r2, r1
+; CHECK-NEXT:    ldr r1, .LCPI1_2
+; CHECK-NEXT:    ldr r1, [r1]
+; CHECK-NEXT:    str r1, [sp, #4]
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    sbcs r1, r5
+; CHECK-NEXT:    ands r3, r4
+; CHECK-NEXT:    ands r2, r0
+; CHECK-NEXT:    mov r4, r2
+; CHECK-NEXT:    orrs r4, r3
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    beq .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %entry
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:  .LBB1_2: @ %entry
+; CHECK-NEXT:    beq .LBB1_4
+; CHECK-NEXT:  @ %bb.3: @ %entry
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:  .LBB1_4: @ %entry
+; CHECK-NEXT:    ldr r2, [sp, #4]
+; CHECK-NEXT:    ldr r3, .LCPI1_2
+; CHECK-NEXT:    ldr r3, [r3]
+; CHECK-NEXT:    cmp r3, r2
+; CHECK-NEXT:    bne .LBB1_6
+; CHECK-NEXT:  @ %bb.5: @ %entry
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB1_6: @ %entry
+; CHECK-NEXT:    bl __stack_chk_fail
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.7:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long aa
+; CHECK-NEXT:  .LCPI1_1:
+; CHECK-NEXT:    .long bb
+; CHECK-NEXT:  .LCPI1_2:
+; CHECK-NEXT:    .long __stack_chk_guard
+;
 ; V6M-LABEL: cc:
 ; V6M:       @ %bb.0: @ %entry
-; V6M-NEXT:	push	{r4, r5, r7, lr}
-; V6M-NEXT:	sub	sp, #8
-; V6M-NEXT:	movs	r0, #1
-; V6M-NEXT:	lsls	r3, r0, #31
-; V6M-NEXT:	movs	r0, :upper8_15:aa
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :upper0_7:aa
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :lower8_15:aa
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :lower0_7:aa
-; V6M-NEXT:	ldr	r2, [r0]
-; V6M-NEXT:	asrs	r4, r2, #31
-; V6M-NEXT:	eors	r3, r4
-; V6M-NEXT:	movs	r0, :upper8_15:bb
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :upper0_7:bb
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :lower8_15:bb
-; V6M-NEXT:	lsls	r0, r0, #8
-; V6M-NEXT:	adds	r0, :lower0_7:bb
-; V6M-NEXT:	ldm	r0!, {r1, r5}
-; V6M-NEXT:	subs	r0, r2, r1
-; V6M-NEXT:	sbcs	r3, r5
-; V6M-NEXT:	subs	r0, r2, r1
-; V6M-NEXT:	mrs	r12, apsr
-; V6M-NEXT:	movs	r1, :upper8_15:__stack_chk_guard
-; V6M-NEXT:	lsls	r1, r1, #8
-; V6M-NEXT:	adds	r1, :upper0_7:__stack_chk_guard
-; V6M-NEXT:	lsls	r1, r1, #8
-; V6M-NEXT:	adds	r1, :lower8_15:__stack_chk_guard
-; V6M-NEXT:	lsls	r1, r1, #8
-; V6M-NEXT:	adds	r1, :lower0_7:__stack_chk_guard
-; V6M-NEXT:	msr	apsr, r12
-; V6M-NEXT:	ldr	r1, [r1]
-; V6M-NEXT:	str	r1, [sp, #4]
-; V6M-NEXT:	mov	r1, r4
-; V6M-NEXT:	sbcs	r1, r5
-; V6M-NEXT:	ands	r3, r4
-; V6M-NEXT:	ands	r2, r0
-; V6M-NEXT:	mov	r4, r2
-; V6M-NEXT:	orrs	r4, r3
-; V6M-NEXT:	beq	.LBB1_2
-; V6M-NEXT:   @ %bb.1:                                @ %entry
-; V6M-NEXT:	mov	r1, r3
-; V6M-NEXT:   .LBB1_2:                                @ %entry
-; V6M-NEXT:	cmp	r4, #0
-; V6M-NEXT:	beq	.LBB1_4
-; V6M-NEXT:   @ %bb.3:                                @ %entry
-; V6M-NEXT:	mov	r0, r2
-; V6M-NEXT:   .LBB1_4:                                @ %entry
-; V6M-NEXT:	ldr	r2, [sp, #4]
-; V6M-NEXT:	mrs	r12, apsr
-; V6M-NEXT:	movs	r3, :upper8_15:__stack_chk_guard
-; V6M-NEXT:	lsls	r3, r3, #8
-; V6M-NEXT:	adds	r3, :upper0_7:__stack_chk_guard
-; V6M-NEXT:	lsls	r3, r3, #8
-; V6M-NEXT:	adds	r3, :lower8_15:__stack_chk_guard
-; V6M-NEXT:	lsls	r3, r3, #8
-; V6M-NEXT:	adds	r3, :lower0_7:__stack_chk_guard
-; V6M-NEXT:	msr	apsr, r12
-; V6M-NEXT:	ldr	r3, [r3]
-; V6M-NEXT:	cmp	r3, r2
-; V6M-NEXT:	bne	.LBB1_6
-; V6M-NEXT:   @ %bb.5:                                @ %entry
-; V6M-NEXT:	add	sp, #8
-; V6M-NEXT:	pop	{r4, r5, r7, pc}
-; V6M-NEXT:   .LBB1_6:                                @ %entry
-; V6M-NEXT:	bl	__stack_chk_fail
+; V6M-NEXT:    push {r4, r5, r7, lr}
+; V6M-NEXT:    sub sp, #8
+; V6M-NEXT:    movs r0, #1
+; V6M-NEXT:    lsls r3, r0, #31
+; V6M-NEXT:    movs r0, :upper8_15:aa
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :upper0_7:aa
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :lower8_15:aa
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :lower0_7:aa
+; V6M-NEXT:    ldr r2, [r0]
+; V6M-NEXT:    asrs r4, r2, #31
+; V6M-NEXT:    eors r3, r4
+; V6M-NEXT:    movs r0, :upper8_15:bb
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :upper0_7:bb
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :lower8_15:bb
+; V6M-NEXT:    lsls r0, r0, #8
+; V6M-NEXT:    adds r0, :lower0_7:bb
+; V6M-NEXT:    ldm r0!, {r1, r5}
+; V6M-NEXT:    subs r0, r2, r1
+; V6M-NEXT:    sbcs r3, r5
+; V6M-NEXT:    subs r0, r2, r1
+; V6M-NEXT:    mrs r12, apsr
+; V6M-NEXT:    movs r1, :upper8_15:__stack_chk_guard
+; V6M-NEXT:    lsls r1, r1, #8
+; V6M-NEXT:    adds r1, :upper0_7:__stack_chk_guard
+; V6M-NEXT:    lsls r1, r1, #8
+; V6M-NEXT:    adds r1, :lower8_15:__stack_chk_guard
+; V6M-NEXT:    lsls r1, r1, #8
+; V6M-NEXT:    adds r1, :lower0_7:__stack_chk_guard
+; V6M-NEXT:    msr apsr, r12
+; V6M-NEXT:    ldr r1, [r1]
+; V6M-NEXT:    str r1, [sp, #4]
+; V6M-NEXT:    mov r1, r4
+; V6M-NEXT:    sbcs r1, r5
+; V6M-NEXT:    ands r3, r4
+; V6M-NEXT:    ands r2, r0
+; V6M-NEXT:    mov r4, r2
+; V6M-NEXT:    orrs r4, r3
+; V6M-NEXT:    cmp r4, #0
+; V6M-NEXT:    beq .LBB1_2
+; V6M-NEXT:  @ %bb.1: @ %entry
+; V6M-NEXT:    mov r1, r3
+; V6M-NEXT:  .LBB1_2: @ %entry
+; V6M-NEXT:    beq .LBB1_4
+; V6M-NEXT:  @ %bb.3: @ %entry
+; V6M-NEXT:    mov r0, r2
+; V6M-NEXT:  .LBB1_4: @ %entry
+; V6M-NEXT:    ldr r2, [sp, #4]
+; V6M-NEXT:    mrs r12, apsr
+; V6M-NEXT:    movs r3, :upper8_15:__stack_chk_guard
+; V6M-NEXT:    lsls r3, r3, #8
+; V6M-NEXT:    adds r3, :upper0_7:__stack_chk_guard
+; V6M-NEXT:    lsls r3, r3, #8
+; V6M-NEXT:    adds r3, :lower8_15:__stack_chk_guard
+; V6M-NEXT:    lsls r3, r3, #8
+; V6M-NEXT:    adds r3, :lower0_7:__stack_chk_guard
+; V6M-NEXT:    msr apsr, r12
+; V6M-NEXT:    ldr r3, [r3]
+; V6M-NEXT:    cmp r3, r2
+; V6M-NEXT:    bne .LBB1_6
+; V6M-NEXT:  @ %bb.5: @ %entry
+; V6M-NEXT:    add sp, #8
+; V6M-NEXT:    pop {r4, r5, r7, pc}
+; V6M-NEXT:  .LBB1_6: @ %entry
+; V6M-NEXT:    bl __stack_chk_fail
+;
+; V8MBASE-LABEL: cc:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    push {r4, r5, r7, lr}
+; V8MBASE-NEXT:    sub sp, #8
+; V8MBASE-NEXT:    movs r0, #1
+; V8MBASE-NEXT:    lsls r3, r0, #31
+; V8MBASE-NEXT:    movw r0, :lower16:aa
+; V8MBASE-NEXT:    movt r0, :upper16:aa
+; V8MBASE-NEXT:    ldr r2, [r0]
+; V8MBASE-NEXT:    asrs r4, r2, #31
+; V8MBASE-NEXT:    eors r3, r4
+; V8MBASE-NEXT:    movw r0, :lower16:bb
+; V8MBASE-NEXT:    movt r0, :upper16:bb
+; V8MBASE-NEXT:    ldm r0!, {r1, r5}
+; V8MBASE-NEXT:    subs r0, r2, r1
+; V8MBASE-NEXT:    sbcs r3, r5
+; V8MBASE-NEXT:    subs r0, r2, r1
+; V8MBASE-NEXT:    movw r1, :lower16:__stack_chk_guard
+; V8MBASE-NEXT:    movt r1, :upper16:__stack_chk_guard
+; V8MBASE-NEXT:    ldr r1, [r1]
+; V8MBASE-NEXT:    str r1, [sp, #4]
+; V8MBASE-NEXT:    mov r1, r4
+; V8MBASE-NEXT:    sbcs r1, r5
+; V8MBASE-NEXT:    ands r3, r4
+; V8MBASE-NEXT:    ands r2, r0
+; V8MBASE-NEXT:    mov r4, r2
+; V8MBASE-NEXT:    orrs r4, r3
+; V8MBASE-NEXT:    cmp r4, #0
+; V8MBASE-NEXT:    beq .LBB1_2
+; V8MBASE-NEXT:  @ %bb.1: @ %entry
+; V8MBASE-NEXT:    mov r1, r3
+; V8MBASE-NEXT:  .LBB1_2: @ %entry
+; V8MBASE-NEXT:    beq .LBB1_4
+; V8MBASE-NEXT:  @ %bb.3: @ %entry
+; V8MBASE-NEXT:    mov r0, r2
+; V8MBASE-NEXT:  .LBB1_4: @ %entry
+; V8MBASE-NEXT:    ldr r2, [sp, #4]
+; V8MBASE-NEXT:    movw r3, :lower16:__stack_chk_guard
+; V8MBASE-NEXT:    movt r3, :upper16:__stack_chk_guard
+; V8MBASE-NEXT:    ldr r3, [r3]
+; V8MBASE-NEXT:    cmp r3, r2
+; V8MBASE-NEXT:    bne .LBB1_6
+; V8MBASE-NEXT:  @ %bb.5: @ %entry
+; V8MBASE-NEXT:    add sp, #8
+; V8MBASE-NEXT:    pop {r4, r5, r7, pc}
+; V8MBASE-NEXT:  .LBB1_6: @ %entry
+; V8MBASE-NEXT:    bl __stack_chk_fail
 
 entry:
   %0 = load i32, ptr @aa, align 4
diff --git a/llvm/test/CodeGen/Thumb/umul_fix_sat.ll b/llvm/test/CodeGen/Thumb/umul_fix_sat.ll
index fa88024315211..a43b22102c24b 100644
--- a/llvm/test/CodeGen/Thumb/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/Thumb/umul_fix_sat.ll
@@ -38,26 +38,26 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    sub sp, #28
 ; ARM-NEXT:    str r3, [sp, #24] @ 4-byte Spill
 ; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    movs r4, #0
 ; ARM-NEXT:    mov r6, r0
-; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r7, r2
-; ARM-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; ARM-NEXT:    str r2, [sp, #12] @ 4-byte Spill
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    str r0, [sp] @ 4-byte Spill
-; ARM-NEXT:    str r1, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    mov r0, r5
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r7
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    adcs r5, r4
 ; ARM-NEXT:    mov r0, r6
 ; ARM-NEXT:    mov r1, r4
@@ -65,33 +65,33 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r2, r7
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r2
-; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    adcs r1, r4
 ; ARM-NEXT:    adds r0, r5, r1
-; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp] @ 4-byte Spill
 ; ARM-NEXT:    mov r6, r4
 ; ARM-NEXT:    adcs r6, r4
-; ARM-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:    mov r0, r5
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r7
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r7, r1
-; ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp] @ 4-byte Spill
 ; ARM-NEXT:    adcs r7, r6
-; ARM-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r6, r0
 ; ARM-NEXT:    str r1, [sp, #24] @ 4-byte Spill
-; ARM-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; ARM-NEXT:    mov r1, r5
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r4
@@ -99,34 +99,33 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    adds r0, r0, r6
 ; ARM-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    adcs r1, r2
-; ARM-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; ARM-NEXT:    adds r2, r2, r0
+; ARM-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; ARM-NEXT:    adds r0, r2, r0
 ; ARM-NEXT:    adcs r1, r7
-; ARM-NEXT:    lsrs r3, r2, #2
-; ARM-NEXT:    orrs r3, r1
+; ARM-NEXT:    lsrs r5, r0, #2
+; ARM-NEXT:    orrs r5, r1
+; ARM-NEXT:    lsls r0, r0, #30
+; ARM-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    lsrs r1, r3, #2
+; ARM-NEXT:    adds r2, r0, r1
+; ARM-NEXT:    lsls r0, r3, #30
+; ARM-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    lsrs r1, r1, #2
+; ARM-NEXT:    adds r3, r0, r1
 ; ARM-NEXT:    mvns r1, r4
-; ARM-NEXT:    cmp r3, #0
+; ARM-NEXT:    cmp r5, #0
 ; ARM-NEXT:    mov r0, r1
 ; ARM-NEXT:    beq .LBB1_3
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    cmp r3, #0
 ; ARM-NEXT:    beq .LBB1_4
 ; ARM-NEXT:  .LBB1_2:
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
 ; ARM-NEXT:  .LBB1_3:
-; ARM-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; ARM-NEXT:    lsls r0, r0, #30
-; ARM-NEXT:    ldr r4, [sp] @ 4-byte Reload
-; ARM-NEXT:    lsrs r4, r4, #2
-; ARM-NEXT:    adds r0, r0, r4
-; ARM-NEXT:    cmp r3, #0
+; ARM-NEXT:    mov r0, r3
 ; ARM-NEXT:    bne .LBB1_2
 ; ARM-NEXT:  .LBB1_4:
-; ARM-NEXT:    lsls r1, r2, #30
-; ARM-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; ARM-NEXT:    lsrs r2, r2, #2
-; ARM-NEXT:    adds r1, r1, r2
+; ARM-NEXT:    mov r1, r2
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
   %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 2)
@@ -225,17 +224,16 @@ define i64 @func5(i64 %x, i64 %y) {
 ; ARM-NEXT:    mov r3, r5
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r3, r1, r6
-; ARM-NEXT:    mov r6, r5
-; ARM-NEXT:    adcs r6, r5
-; ARM-NEXT:    orrs r6, r4
+; ARM-NEXT:    mov r2, r5
+; ARM-NEXT:    adcs r2, r5
+; ARM-NEXT:    orrs r2, r4
 ; ARM-NEXT:    mvns r1, r5
-; ARM-NEXT:    cmp r6, #0
+; ARM-NEXT:    cmp r2, #0
 ; ARM-NEXT:    mov r2, r1
 ; ARM-NEXT:    bne .LBB4_2
 ; ARM-NEXT:  @ %bb.1:
 ; ARM-NEXT:    mov r2, r0
 ; ARM-NEXT:  .LBB4_2:
-; ARM-NEXT:    cmp r6, #0
 ; ARM-NEXT:    bne .LBB4_4
 ; ARM-NEXT:  @ %bb.3:
 ; ARM-NEXT:    mov r1, r3
@@ -399,25 +397,27 @@ define i64 @func7(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
+; ARM-NEXT:    mov r2, r1
 ; ARM-NEXT:    adds r0, r0, r5
-; ARM-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; ARM-NEXT:    adcs r1, r2
-; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; ARM-NEXT:    adds r3, r2, r0
-; ARM-NEXT:    adcs r1, r6
-; ARM-NEXT:    mvns r2, r4
-; ARM-NEXT:    cmp r1, #0
-; ARM-NEXT:    mov r0, r2
-; ARM-NEXT:    bne .LBB7_2
+; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; ARM-NEXT:    adcs r2, r1
+; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    adds r3, r1, r0
+; ARM-NEXT:    adcs r2, r6
+; ARM-NEXT:    mvns r1, r4
+; ARM-NEXT:    cmp r2, #0
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    beq .LBB7_3
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    beq .LBB7_4
 ; ARM-NEXT:  .LBB7_2:
-; ARM-NEXT:    cmp r1, #0
-; ARM-NEXT:    bne .LBB7_4
-; ARM-NEXT:  @ %bb.3:
-; ARM-NEXT:    mov r2, r3
+; ARM-NEXT:    add sp, #28
+; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
+; ARM-NEXT:  .LBB7_3:
+; ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    bne .LBB7_2
 ; ARM-NEXT:  .LBB7_4:
-; ARM-NEXT:    mov r1, r2
+; ARM-NEXT:    mov r1, r3
 ; ARM-NEXT:    add sp, #28
 ; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
   %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 32)
@@ -433,23 +433,23 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    sub sp, #28
 ; ARM-NEXT:    str r3, [sp, #24] @ 4-byte Spill
 ; ARM-NEXT:    mov r7, r2
-; ARM-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; ARM-NEXT:    str r2, [sp, #16] @ 4-byte Spill
 ; ARM-NEXT:    mov r5, r1
-; ARM-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
 ; ARM-NEXT:    movs r4, #0
 ; ARM-NEXT:    mov r6, r0
-; ARM-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
-; ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r1, [sp, #20] @ 4-byte Spill
 ; ARM-NEXT:    mov r0, r5
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r7
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r7, r1
-; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; ARM-NEXT:    adds r5, r0, r1
 ; ARM-NEXT:    adcs r7, r4
 ; ARM-NEXT:    mov r0, r6
@@ -459,31 +459,31 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    adds r0, r0, r5
-; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; ARM-NEXT:    adcs r1, r4
 ; ARM-NEXT:    adds r0, r7, r1
-; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    mov r5, r4
 ; ARM-NEXT:    adcs r5, r4
-; ARM-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
+; ARM-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
 ; ARM-NEXT:    mov r0, r7
 ; ARM-NEXT:    mov r1, r4
 ; ARM-NEXT:    mov r2, r6
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r6, r1
-; ARM-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
 ; ARM-NEXT:    adds r0, r0, r1
-; ARM-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; ARM-NEXT:    adcs r6, r5
-; ARM-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
 ; ARM-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r4
 ; ARM-NEXT:    bl __aeabi_lmul
 ; ARM-NEXT:    mov r5, r0
 ; ARM-NEXT:    str r1, [sp, #24] @ 4-byte Spill
-; ARM-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; ARM-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; ARM-NEXT:    mov r1, r7
 ; ARM-NEXT:    mov r2, r4
 ; ARM-NEXT:    mov r3, r4
@@ -491,25 +491,28 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
 ; ARM-NEXT:    adds r0, r0, r5
 ; ARM-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; ARM-NEXT:    adcs r1, r2
-; ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; ARM-NEXT:    adds r5, r2, r0
+; ARM-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; ARM-NEXT:    adds r0, r2, r0
 ; ARM-NEXT:    adcs r1, r6
-; ARM-NEXT:    lsrs r3, r5, #31
-; ARM-NEXT:    mvns r2, r4
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    mov r0, r2
-; ARM-NEXT:    bne .LBB8_2
+; ARM-NEXT:    lsls r1, r1, #1
+; ARM-NEXT:    lsrs r5, r0, #31
+; ARM-NEXT:    adds r2, r1, r5
+; ARM-NEXT:    lsls r0, r0, #1
+; ARM-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; ARM-NEXT:    lsrs r1, r1, #31
+; ARM-NEXT:    adds r3, r0, r1
+; ARM-NEXT:    mvns r1, r4
+; ARM-NEXT:    cmp r5, #0
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    beq .LBB8_3
 ; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:    lsls r0, r5, #1
-; ARM-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
-; ARM-NEXT:    lsrs r4, r4, #31
-; ARM-NEXT:    adds r0, r0, r4
+; ARM-NEXT:    beq .LBB8_4
 ; ARM-NEXT:  .LBB8_2:
-; ARM-NEXT:    cmp r3, #0
-; ARM-NEXT:    bne .LBB8_4
-; ARM-NEXT:  @ %bb.3:
-; ARM-NEXT:    lsls r1, r1, #1
-; ARM-NEXT:    adds r2, r1, r3
+; ARM-NEXT:    add sp, #28
+; ARM-NEXT:    pop {r4, r5, r6, r7, pc}
+; ARM-NEXT:  .LBB8_3:
+; ARM-NEXT:    mov r0, r3
+; ARM-NEXT:    bne .LBB8_2
 ; ARM-NEXT:  .LBB8_4:
 ; ARM-NEXT:    mov r1, r2
 ; ARM-NEXT:    add sp, #28
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/arm_cmplx_dot_prod_f32.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/arm_cmplx_dot_prod_f32.ll
index 13080fcfa1357..a87d363fa61ee 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/arm_cmplx_dot_prod_f32.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/arm_cmplx_dot_prod_f32.ll
@@ -15,13 +15,13 @@ define void @arm_cmplx_dot_prod_f32(ptr %pSrcA, ptr %pSrcB, i32 %numSamples, ptr
 ; CHECK-NEXT:    lsrs r4, r2, #2
 ; CHECK-NEXT:    mov.w lr, #2
 ; CHECK-NEXT:    cmp r4, #2
+; CHECK-NEXT:    vldrw.u32 q2, [r1], #32
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #32
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    lsrlt.w lr, r2, #2
 ; CHECK-NEXT:    rsb r4, lr, r2, lsr #2
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #32
-; CHECK-NEXT:    add.w lr, r4, #1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #32
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, r4, #1
 ; CHECK-NEXT:  .LBB0_2: @ %while.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vcmla.f32 q0, q1, q2, #0
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
index b9a80af649f29..3c1510623e5c4 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
@@ -7,26 +7,26 @@ define void @foo(ptr nocapture readonly %st, ptr %x) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    ldrd r12, r2, [r0]
-; CHECK-NEXT:    ldrd r4, r3, [r0, #8]
-; CHECK-NEXT:    rsb r12, r12, r2, lsl #1
+; CHECK-NEXT:    ldrd r12, r3, [r0]
+; CHECK-NEXT:    ldrd r4, r2, [r0, #8]
+; CHECK-NEXT:    rsb r12, r12, r3, lsl #1
 ; CHECK-NEXT:    dlstp.16 lr, r12
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q0, [r3], #16
+; CHECK-NEXT:    vldrh.u16 q0, [r2], #16
 ; CHECK-NEXT:    vstrh.16 q0, [r4], #16
 ; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %do.end
-; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    ldr r3, [r0]
 ; CHECK-NEXT:    ldr r0, [r0, #8]
 ; CHECK-NEXT:    add.w r0, r0, r12, lsl #1
-; CHECK-NEXT:    mov.w r3, #6144
-; CHECK-NEXT:    dlstp.16 lr, r2
+; CHECK-NEXT:    mov.w r2, #6144
+; CHECK-NEXT:    dlstp.16 lr, r3
 ; CHECK-NEXT:  .LBB0_3: @ %do.body6
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
 ; CHECK-NEXT:    vcvt.f16.s16 q0, q0
-; CHECK-NEXT:    vmul.f16 q0, q0, r3
+; CHECK-NEXT:    vmul.f16 q0, q0, r2
 ; CHECK-NEXT:    vstrh.16 q0, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB0_3
 ; CHECK-NEXT:  @ %bb.4: @ %do.end13
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index 2fdf534d52656..6cb98557c9bc1 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -9,12 +9,12 @@ define arm_aapcs_vfpcc void @fast_float_mul(ptr nocapture %a, ptr nocapture read
 ; CHECK-NEXT:    beq.w .LBB0_11
 ; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
 ; CHECK-NEXT:    add.w r4, r2, r3, lsl #2
-; CHECK-NEXT:    add.w lr, r0, r3, lsl #2
 ; CHECK-NEXT:    cmp r4, r0
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    cmp lr, r2
-; CHECK-NEXT:    csel r12, zr, r4, ls
-; CHECK-NEXT:    cmp lr, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r12, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    csel r12, zr, r12, ls
+; CHECK-NEXT:    cmp r4, r1
 ; CHECK-NEXT:    add.w r4, r1, r3, lsl #2
 ; CHECK-NEXT:    cset lr, hi
 ; CHECK-NEXT:    cmp r4, r0
diff --git a/llvm/test/CodeGen/Thumb2/float-ops.ll b/llvm/test/CodeGen/Thumb2/float-ops.ll
index d2b1dd6f05a3f..191c775be3420 100644
--- a/llvm/test/CodeGen/Thumb2/float-ops.ll
+++ b/llvm/test/CodeGen/Thumb2/float-ops.ll
@@ -289,15 +289,15 @@ define float @select_f(float %a, float %b, i1 %c) {
 define double @select_d(double %a, double %b, i1 %c) {
 ; CHECK-LABEL: select_d:
 ; NOREGS: ldr{{(.w)?}}     [[REG:r[0-9]+]], [sp]
-; NOREGS: ands    [[REG]], [[REG]], #1
-; ONLYREGS: ands    r0, r0, #1
+; NOREGS: lsls.w    [[REG]], [[REG]], #31
+; ONLYREGS: lsls    r0, r0, #31
 ; NOREGS-DAG: moveq   r0, r2
 ; NOREGS-DAG: moveq   r1, r3
-; ONLYREGS-DAG: csel   r0, r0, r2
-; ONLYREGS-DAG: csel   r1, r1, r3
-; SP: ands r0, r0, #1
+; ONLYREGS-DAG: csel   r0, r2, r1
+; ONLYREGS-DAG: csel   r1, r12, r3
 ; SP-DAG: vmov [[ALO:r[0-9]+]], [[AHI:r[0-9]+]], d0
 ; SP-DAG: vmov [[BLO:r[0-9]+]], [[BHI:r[0-9]+]], d1
+; SP: lsls r0, r0, #31
 ; SP: itt ne
 ; SP-DAG: movne [[BLO]], [[ALO]]
 ; SP-DAG: movne [[BHI]], [[AHI]]
diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
index 7087041e8dace..d076cb00ad7e0 100644
--- a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
@@ -489,10 +489,10 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
 ; CHECK-NEXT:    @ Parent Loop BB1_10 Depth=2
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=3
 ; CHECK-NEXT:    vqadd.u32 q2, q5, r1
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r1, #4
 ; CHECK-NEXT:    vcmp.u32 hi, q7, q2
 ; CHECK-NEXT:    vshl.i32 q2, q1, #2
-; CHECK-NEXT:    add.w r1, r1, #4
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vadd.i32 q2, q2, r8
 ; CHECK-NEXT:    vadd.i32 q1, q1, r9
 ; CHECK-NEXT:    vpst
@@ -508,10 +508,10 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
 ; CHECK-NEXT:    @ Parent Loop BB1_10 Depth=2
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=3
 ; CHECK-NEXT:    vqadd.u32 q2, q5, r1
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r1, #4
 ; CHECK-NEXT:    vcmp.u32 hi, q6, q2
 ; CHECK-NEXT:    vshl.i32 q2, q1, #2
-; CHECK-NEXT:    add.w r1, r1, #4
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vadd.i32 q2, q2, r8
 ; CHECK-NEXT:    vadd.i32 q1, q1, r9
 ; CHECK-NEXT:    vpst
diff --git a/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll b/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
index 67723e8aa41ad..9ee6ec345d964 100644
--- a/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
@@ -411,12 +411,12 @@ define i32 @nested_smin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    mvn r3, #-2147483648
 ; CHECK-NEXT:    mvn r2, #-2147483648
-; CHECK-NEXT:    vminv.s32 r3, q0
-; CHECK-NEXT:    vminv.s32 r2, q1
-; CHECK-NEXT:    cmp r3, r0
-; CHECK-NEXT:    csel r0, r3, r0, lt
-; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    csel r1, r2, r1, lt
+; CHECK-NEXT:    vminv.s32 r3, q1
+; CHECK-NEXT:    vminv.s32 r2, q0
+; CHECK-NEXT:    cmp r3, r1
+; CHECK-NEXT:    csel r1, r3, r1, lt
+; CHECK-NEXT:    cmp r2, r0
+; CHECK-NEXT:    csel r0, r2, r0, lt
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    bx lr
@@ -433,12 +433,12 @@ define i32 @nested_smax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    mov.w r3, #-2147483648
 ; CHECK-NEXT:    mov.w r2, #-2147483648
-; CHECK-NEXT:    vmaxv.s32 r3, q0
-; CHECK-NEXT:    vmaxv.s32 r2, q1
-; CHECK-NEXT:    cmp r3, r0
-; CHECK-NEXT:    csel r0, r3, r0, gt
-; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    csel r1, r2, r1, gt
+; CHECK-NEXT:    vmaxv.s32 r3, q1
+; CHECK-NEXT:    vmaxv.s32 r2, q0
+; CHECK-NEXT:    cmp r3, r1
+; CHECK-NEXT:    csel r1, r3, r1, gt
+; CHECK-NEXT:    cmp r2, r0
+; CHECK-NEXT:    csel r0, r2, r0, gt
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    bx lr
@@ -455,12 +455,12 @@ define i32 @nested_umin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    mov.w r3, #-1
 ; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    vminv.u32 r3, q0
-; CHECK-NEXT:    vminv.u32 r2, q1
-; CHECK-NEXT:    cmp r3, r0
-; CHECK-NEXT:    csel r0, r3, r0, lo
-; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    csel r1, r2, r1, lo
+; CHECK-NEXT:    vminv.u32 r3, q1
+; CHECK-NEXT:    vminv.u32 r2, q0
+; CHECK-NEXT:    cmp r3, r1
+; CHECK-NEXT:    csel r1, r3, r1, lo
+; CHECK-NEXT:    cmp r2, r0
+; CHECK-NEXT:    csel r0, r2, r0, lo
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lo
 ; CHECK-NEXT:    bx lr
@@ -477,12 +477,12 @@ define i32 @nested_umax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmaxv.u32 r3, q0
-; CHECK-NEXT:    vmaxv.u32 r2, q1
-; CHECK-NEXT:    cmp r3, r0
-; CHECK-NEXT:    csel r0, r3, r0, hi
-; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    csel r1, r2, r1, hi
+; CHECK-NEXT:    vmaxv.u32 r3, q1
+; CHECK-NEXT:    vmaxv.u32 r2, q0
+; CHECK-NEXT:    cmp r3, r1
+; CHECK-NEXT:    csel r1, r3, r1, hi
+; CHECK-NEXT:    cmp r2, r0
+; CHECK-NEXT:    csel r0, r2, r0, hi
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index 1c95d28b5eed1..c8dd949ca9d88 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -1365,8 +1365,8 @@ define void @arm_biquad_cascade_df2T_f16(ptr nocapture readonly %S, ptr nocaptur
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
 ; CHECK-NEXT:    vstr.16 s5, [r6, #2]
 ; CHECK-NEXT:    add.w r12, r12, #10
+; CHECK-NEXT:    adds r6, #4
 ; CHECK-NEXT:    subs.w r9, r9, #1
-; CHECK-NEXT:    add.w r6, r6, #4
 ; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    beq .LBB17_8
 ; CHECK-NEXT:  .LBB17_3: @ %do.body
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 808626d9a0aeb..495ffe809f70f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1375,8 +1375,8 @@ define arm_aapcs_vfpcc void @arm_biquad_cascade_stereo_df2T_f32(ptr nocapture re
 ; CHECK-NEXT:    le lr, .LBB17_3
 ; CHECK-NEXT:  @ %bb.4: @ %bb75
 ; CHECK-NEXT:    @ in Loop: Header=BB17_2 Depth=1
+; CHECK-NEXT:    adds r3, #20
 ; CHECK-NEXT:    subs.w r12, r12, #1
-; CHECK-NEXT:    add.w r3, r3, #20
 ; CHECK-NEXT:    vstrb.8 q3, [r0], #16
 ; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    bne .LBB17_2
@@ -1514,8 +1514,8 @@ define arm_aapcs_vfpcc void @fms(ptr nocapture readonly %pSrc1, ptr nocapture re
 ; CHECK-NEXT:    le lr, .LBB18_3
 ; CHECK-NEXT:  @ %bb.4: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB18_2 Depth=1
+; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    subs.w r12, r12, #1
-; CHECK-NEXT:    add.w r2, r2, #4
 ; CHECK-NEXT:    bne .LBB18_2
 ; CHECK-NEXT:  .LBB18_5: @ %do.end
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
@@ -1918,8 +1918,8 @@ define void @arm_biquad_cascade_df2T_f32(ptr nocapture readonly %S, ptr nocaptur
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    vstr s6, [r6, #4]
 ; CHECK-NEXT:    add.w r12, r12, #20
+; CHECK-NEXT:    adds r6, #8
 ; CHECK-NEXT:    subs r0, #1
-; CHECK-NEXT:    add.w r6, r6, #8
 ; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    beq .LBB20_8
 ; CHECK-NEXT:  .LBB20_3: @ %do.body
diff --git a/llvm/test/CodeGen/Thumb2/mve-fmas.ll b/llvm/test/CodeGen/Thumb2/mve-fmas.ll
index 377440e1bbc93..94921c78ad912 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fmas.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fmas.ll
@@ -893,19 +893,19 @@ define arm_aapcs_vfpcc <4 x float> @vfma32_v1_pred(<4 x float> %src1, <4 x float
 ;
 ; CHECK-MVE-LABEL: vfma32_v1_pred:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, #0
-; CHECK-MVE-NEXT:    vmov.f32 s14, s2
-; CHECK-MVE-NEXT:    vmov.f32 s12, s3
-; CHECK-MVE-NEXT:    vmla.f32 s14, s6, s10
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vmov.f32 s12, s2
+; CHECK-MVE-NEXT:    vmov.f32 s14, s3
+; CHECK-MVE-NEXT:    vmla.f32 s12, s6, s10
 ; CHECK-MVE-NEXT:    vmov.f32 s10, s1
-; CHECK-MVE-NEXT:    vmla.f32 s12, s7, s11
+; CHECK-MVE-NEXT:    vmla.f32 s14, s7, s11
 ; CHECK-MVE-NEXT:    vmla.f32 s10, s5, s9
 ; CHECK-MVE-NEXT:    vmov.f32 s9, s0
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-MVE-NEXT:    vmla.f32 s9, s4, s8
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -913,13 +913,13 @@ define arm_aapcs_vfpcc <4 x float> @vfma32_v1_pred(<4 x float> %src1, <4 x float
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s12
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s14
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s12
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s10
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s14
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s0, s9
 ; CHECK-MVE-NEXT:    bx lr
 entry:
@@ -946,19 +946,19 @@ define arm_aapcs_vfpcc <4 x float> @vfma32_v2_pred(<4 x float> %src1, <4 x float
 ;
 ; CHECK-MVE-LABEL: vfma32_v2_pred:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, #0
-; CHECK-MVE-NEXT:    vmov.f32 s14, s2
-; CHECK-MVE-NEXT:    vmov.f32 s12, s3
-; CHECK-MVE-NEXT:    vmla.f32 s14, s6, s10
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vmov.f32 s12, s2
+; CHECK-MVE-NEXT:    vmov.f32 s14, s3
+; CHECK-MVE-NEXT:    vmla.f32 s12, s6, s10
 ; CHECK-MVE-NEXT:    vmov.f32 s10, s1
-; CHECK-MVE-NEXT:    vmla.f32 s12, s7, s11
+; CHECK-MVE-NEXT:    vmla.f32 s14, s7, s11
 ; CHECK-MVE-NEXT:    vmla.f32 s10, s5, s9
 ; CHECK-MVE-NEXT:    vmov.f32 s9, s0
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-MVE-NEXT:    vmla.f32 s9, s4, s8
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -966,13 +966,13 @@ define arm_aapcs_vfpcc <4 x float> @vfma32_v2_pred(<4 x float> %src1, <4 x float
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s12
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s14
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s12
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s10
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s14
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s0, s9
 ; CHECK-MVE-NEXT:    bx lr
 entry:
@@ -999,19 +999,19 @@ define arm_aapcs_vfpcc <4 x float> @vfms32_pred(<4 x float> %src1, <4 x float> %
 ;
 ; CHECK-MVE-LABEL: vfms32_pred:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, #0
-; CHECK-MVE-NEXT:    vmov.f32 s14, s2
-; CHECK-MVE-NEXT:    vmov.f32 s12, s3
-; CHECK-MVE-NEXT:    vmls.f32 s14, s6, s10
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vmov.f32 s12, s2
+; CHECK-MVE-NEXT:    vmov.f32 s14, s3
+; CHECK-MVE-NEXT:    vmls.f32 s12, s6, s10
 ; CHECK-MVE-NEXT:    vmov.f32 s10, s1
-; CHECK-MVE-NEXT:    vmls.f32 s12, s7, s11
+; CHECK-MVE-NEXT:    vmls.f32 s14, s7, s11
 ; CHECK-MVE-NEXT:    vmls.f32 s10, s5, s9
 ; CHECK-MVE-NEXT:    vmov.f32 s9, s0
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-MVE-NEXT:    vmls.f32 s9, s4, s8
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1019,13 +1019,13 @@ define arm_aapcs_vfpcc <4 x float> @vfms32_pred(<4 x float> %src1, <4 x float> %
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s12
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s14
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s12
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s10
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s14
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s0, s9
 ; CHECK-MVE-NEXT:    bx lr
 entry:
@@ -1055,33 +1055,33 @@ define arm_aapcs_vfpcc <4 x float> @vfmar32_pred(<4 x float> %src1, <4 x float>
 ;
 ; CHECK-MVE-LABEL: vfmar32_pred:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, #0
-; CHECK-MVE-NEXT:    vmov.f32 s10, s3
-; CHECK-MVE-NEXT:    vmov.f32 s12, s2
-; CHECK-MVE-NEXT:    vmov.f32 s14, s1
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vmov.f32 s10, s2
+; CHECK-MVE-NEXT:    vmov.f32 s12, s1
+; CHECK-MVE-NEXT:    vmov.f32 s14, s3
 ; CHECK-MVE-NEXT:    vmov.f32 s9, s0
-; CHECK-MVE-NEXT:    vmla.f32 s10, s7, s8
-; CHECK-MVE-NEXT:    vmla.f32 s12, s6, s8
-; CHECK-MVE-NEXT:    vmla.f32 s14, s5, s8
+; CHECK-MVE-NEXT:    vmla.f32 s10, s6, s8
+; CHECK-MVE-NEXT:    vmla.f32 s12, s5, s8
+; CHECK-MVE-NEXT:    vmla.f32 s14, s7, s8
 ; CHECK-MVE-NEXT:    vmla.f32 s9, s4, s8
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s6, #0
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s10
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s12
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s14
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s10
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s12
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s14
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s0, s9
 ; CHECK-MVE-NEXT:    bx lr
 entry:
@@ -1112,32 +1112,32 @@ define arm_aapcs_vfpcc <4 x float> @vfmas32_pred(<4 x float> %src1, <4 x float>
 ;
 ; CHECK-MVE-LABEL: vfmas32_pred:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
 ; CHECK-MVE-NEXT:    vmov.f32 s10, s8
 ; CHECK-MVE-NEXT:    vmov.f32 s12, s8
 ; CHECK-MVE-NEXT:    vmov.f32 s14, s8
 ; CHECK-MVE-NEXT:    vmla.f32 s8, s0, s4
-; CHECK-MVE-NEXT:    vmla.f32 s10, s3, s7
-; CHECK-MVE-NEXT:    vmla.f32 s12, s2, s6
-; CHECK-MVE-NEXT:    vmla.f32 s14, s1, s5
+; CHECK-MVE-NEXT:    vmla.f32 s10, s2, s6
+; CHECK-MVE-NEXT:    vmla.f32 s12, s1, s5
+; CHECK-MVE-NEXT:    vmla.f32 s14, s3, s7
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s7, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s6, #0
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s10
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s12
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s14
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s2, s10
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s1, s12
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s3, s14
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s0, s8
 ; CHECK-MVE-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
index f2ac526892180..742f2a75a1aa8 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
@@ -668,62 +668,63 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-LABEL: stest_f64i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov r0, r1, d9
+; CHECK-NEXT:    vmov r0, r1, d8
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    vmov r12, lr, d8
-; CHECK-NEXT:    subs.w r4, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r4, r1, r9
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r3, r3, r4, ne
-; CHECK-NEXT:    csel r2, r2, r4, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
+; CHECK-NEXT:    vmov r12, lr, d9
+; CHECK-NEXT:    subs.w r5, r0, #-1
+; CHECK-NEXT:    mvn r4, #-2147483648
+; CHECK-NEXT:    sbcs.w r5, r1, r4
+; CHECK-NEXT:    sbcs r5, r2, #0
+; CHECK-NEXT:    mov.w r7, #-2147483648
+; CHECK-NEXT:    sbcs r5, r3, #0
 ; CHECK-NEXT:    cset r5, lt
 ; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    csel r3, r3, r5, ne
+; CHECK-NEXT:    csel r2, r2, r5, ne
+; CHECK-NEXT:    mov.w r5, #-1
+; CHECK-NEXT:    csel r1, r1, r4, ne
+; CHECK-NEXT:    csel r0, r0, r5, ne
+; CHECK-NEXT:    rsbs r6, r0, #0
+; CHECK-NEXT:    sbcs.w r6, r7, r1
+; CHECK-NEXT:    sbcs.w r2, r5, r2
+; CHECK-NEXT:    sbcs.w r2, r5, r3
+; CHECK-NEXT:    csel r8, r1, r7, lt
+; CHECK-NEXT:    cset r1, lt
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csel r9, r0, r1, ne
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
+; CHECK-NEXT:    sbcs.w r6, r1, r4
 ; CHECK-NEXT:    sbcs r6, r2, #0
 ; CHECK-NEXT:    sbcs r6, r3, #0
 ; CHECK-NEXT:    cset r6, lt
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
+; CHECK-NEXT:    csel r0, r0, r5, ne
 ; CHECK-NEXT:    csel r3, r3, r6, ne
 ; CHECK-NEXT:    csel r2, r2, r6, ne
+; CHECK-NEXT:    csel r1, r1, r4, ne
 ; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    sbcs.w r6, r7, r1
+; CHECK-NEXT:    sbcs.w r2, r5, r2
+; CHECK-NEXT:    sbcs.w r2, r5, r3
 ; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
+; CHECK-NEXT:    csel r1, r1, r7, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
+; CHECK-NEXT:    vmov q0[2], q0[0], r9, r0
+; CHECK-NEXT:    vmov q0[3], q0[1], r8, r1
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -737,38 +738,33 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @utest_f64i64(<2 x double> %x) {
 ; CHECK-LABEL: utest_f64i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vmov r4, r1, d8
+; CHECK-NEXT:    vmov r12, lr, d8
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    cset r6, lo
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r7, r0, r6, ne
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    cset r2, lo
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r4, r1, r2, ne
+; CHECK-NEXT:    csel r5, r0, r2, ne
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixunsdfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lo
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r4
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -780,10 +776,8 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-LABEL: ustest_f64i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
@@ -793,47 +787,44 @@ define arm_aapcs_vfpcc <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    subs r4, r2, #1
 ; CHECK-NEXT:    sbcs r4, r3, #0
 ; CHECK-NEXT:    mov.w r8, #1
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    movs r7, #0
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
+; CHECK-NEXT:    cset r4, lt
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    csel r0, r0, r4, ne
+; CHECK-NEXT:    csel r3, r3, r4, ne
+; CHECK-NEXT:    csel r1, r1, r4, ne
 ; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r4, r1, r5, ne
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    sbcs.w r1, r7, r4
-; CHECK-NEXT:    sbcs.w r1, r7, r2
-; CHECK-NEXT:    sbcs.w r1, r7, r3
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r9, r0, r6, ne
+; CHECK-NEXT:    rsbs r5, r0, #0
+; CHECK-NEXT:    mov.w r4, #0
+; CHECK-NEXT:    sbcs.w r5, r4, r1
+; CHECK-NEXT:    sbcs.w r2, r4, r2
+; CHECK-NEXT:    sbcs.w r2, r4, r3
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r5, r1, r2, ne
+; CHECK-NEXT:    csel r7, r0, r2, ne
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    subs r5, r2, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
+; CHECK-NEXT:    subs r6, r2, #1
+; CHECK-NEXT:    sbcs r6, r3, #0
+; CHECK-NEXT:    cset r6, lt
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    csel r0, r0, r6, ne
+; CHECK-NEXT:    csel r3, r3, r6, ne
+; CHECK-NEXT:    csel r1, r1, r6, ne
 ; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r1, r1, r5, ne
-; CHECK-NEXT:    rsbs r5, r0, #0
-; CHECK-NEXT:    sbcs.w r5, r7, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    rsbs r6, r0, #0
+; CHECK-NEXT:    sbcs.w r6, r4, r1
+; CHECK-NEXT:    sbcs.w r2, r4, r2
+; CHECK-NEXT:    sbcs.w r2, r4, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r4, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r9
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -847,59 +838,54 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: stest_f32i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    vmov r9, r0, d0
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    vmov r0, r9, d0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r10, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r10
-; CHECK-NEXT:    mov.w r4, #-1
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs.w r7, r1, r5
+; CHECK-NEXT:    mov.w r6, #-1
 ; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    mov.w r11, #-2147483648
 ; CHECK-NEXT:    sbcs r7, r3, #0
 ; CHECK-NEXT:    cset r7, lt
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r5, r0, r4, ne
+; CHECK-NEXT:    csel r0, r0, r6, ne
 ; CHECK-NEXT:    csel r3, r3, r7, ne
 ; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    rsbs r0, r5, #0
-; CHECK-NEXT:    sbcs.w r0, r11, r1
-; CHECK-NEXT:    sbcs.w r0, r4, r2
-; CHECK-NEXT:    sbcs.w r0, r4, r3
-; CHECK-NEXT:    cset r6, lt
+; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    rsbs r4, r0, #0
+; CHECK-NEXT:    mov.w r7, #-2147483648
+; CHECK-NEXT:    sbcs.w r4, r7, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
+; CHECK-NEXT:    csel r8, r1, r7, lt
+; CHECK-NEXT:    cset r1, lt
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csel r10, r0, r1, ne
 ; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r8, r1, r11, ne
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    sbcs.w r7, r1, r10
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r4, ne
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    rsbs r7, r0, #0
-; CHECK-NEXT:    sbcs.w r7, r11, r1
-; CHECK-NEXT:    sbcs.w r2, r4, r2
-; CHECK-NEXT:    sbcs.w r2, r4, r3
+; CHECK-NEXT:    subs.w r4, r0, #-1
+; CHECK-NEXT:    sbcs.w r4, r1, r5
+; CHECK-NEXT:    sbcs r4, r2, #0
+; CHECK-NEXT:    sbcs r4, r3, #0
+; CHECK-NEXT:    cset r4, lt
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    csel r0, r0, r6, ne
+; CHECK-NEXT:    csel r3, r3, r4, ne
+; CHECK-NEXT:    csel r2, r2, r4, ne
+; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    rsbs r5, r0, #0
+; CHECK-NEXT:    sbcs.w r5, r7, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
 ; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r11, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
+; CHECK-NEXT:    csel r1, r1, r7, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:    vmov q0[2], q0[0], r10, r0
+; CHECK-NEXT:    vmov q0[3], q0[1], r8, r1
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -913,33 +899,27 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @utest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: utest_f32i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    vmov r4, r0, d0
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    cset r6, lo
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r7, r0, r6, ne
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    sbcs r2, r3, #0
+; CHECK-NEXT:    cset r2, lo
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r6, r0, r2, ne
 ; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    csel r5, r1, r2, ne
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lo
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -951,54 +931,49 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: ustest_f32i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    vmov r6, r0, d0
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vmov r5, r0, d0
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs r5, r2, #1
+; CHECK-NEXT:    subs r4, r2, #1
 ; CHECK-NEXT:    mov.w r8, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
+; CHECK-NEXT:    sbcs r4, r3, #0
+; CHECK-NEXT:    mov.w r6, #0
 ; CHECK-NEXT:    cset r4, lt
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    csel r0, r0, r4, ne
 ; CHECK-NEXT:    csel r3, r3, r4, ne
-; CHECK-NEXT:    csel r5, r1, r4, ne
+; CHECK-NEXT:    csel r1, r1, r4, ne
 ; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    sbcs.w r1, r4, r5
-; CHECK-NEXT:    sbcs.w r1, r4, r2
-; CHECK-NEXT:    sbcs.w r1, r4, r3
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r9, r0, r7, ne
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    rsbs r4, r0, #0
+; CHECK-NEXT:    sbcs.w r4, r6, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r7, r0, r2, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    csel r4, r1, r2, ne
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs r6, r2, #1
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r6, ne
+; CHECK-NEXT:    subs r5, r2, #1
+; CHECK-NEXT:    sbcs r5, r3, #0
+; CHECK-NEXT:    cset r5, lt
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    csel r0, r0, r5, ne
+; CHECK-NEXT:    csel r3, r3, r5, ne
+; CHECK-NEXT:    csel r1, r1, r5, ne
 ; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r1, r1, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r4, r1
-; CHECK-NEXT:    sbcs.w r2, r4, r2
-; CHECK-NEXT:    sbcs.w r2, r4, r3
+; CHECK-NEXT:    rsbs r5, r0, #0
+; CHECK-NEXT:    sbcs.w r5, r6, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r3, r5, r7, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r9
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -1066,39 +1041,39 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    rsbs r1, r0, #0
+; CHECK-NEXT:    rsbs r4, r0, #0
 ; CHECK-NEXT:    mov.w r5, #0
-; CHECK-NEXT:    sbcs.w r1, r5, r4
-; CHECK-NEXT:    sbcs.w r1, r5, r2
-; CHECK-NEXT:    sbcs.w r1, r5, r3
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r8, r0, r6, ne
+; CHECK-NEXT:    sbcs.w r4, r5, r1
+; CHECK-NEXT:    sbcs.w r2, r5, r2
+; CHECK-NEXT:    sbcs.w r2, r5, r3
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r6, r0, r2, ne
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
+; CHECK-NEXT:    csel r7, r1, r2, ne
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    rsbs r7, r0, #0
-; CHECK-NEXT:    sbcs.w r7, r5, r1
+; CHECK-NEXT:    rsbs r4, r0, #0
+; CHECK-NEXT:    sbcs.w r4, r5, r1
 ; CHECK-NEXT:    sbcs.w r2, r5, r2
 ; CHECK-NEXT:    sbcs.w r2, r5, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r4, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r8
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -1748,62 +1723,63 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-LABEL: stest_f64i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    vmov r12, lr, d8
-; CHECK-NEXT:    subs.w r4, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r4, r1, r9
-; CHECK-NEXT:    sbcs r4, r2, #0
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    sbcs r4, r3, #0
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r3, r3, r4, ne
-; CHECK-NEXT:    csel r2, r2, r4, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
+; CHECK-NEXT:    subs.w r5, r0, #-1
+; CHECK-NEXT:    mvn r4, #-2147483648
+; CHECK-NEXT:    sbcs.w r5, r1, r4
+; CHECK-NEXT:    sbcs r5, r2, #0
+; CHECK-NEXT:    mov.w r6, #-1
+; CHECK-NEXT:    sbcs r5, r3, #0
 ; CHECK-NEXT:    cset r5, lt
 ; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    csel r0, r0, r6, ne
+; CHECK-NEXT:    csel r3, r3, r5, ne
+; CHECK-NEXT:    csel r2, r2, r5, ne
+; CHECK-NEXT:    csel r1, r1, r4, ne
+; CHECK-NEXT:    rsbs r7, r0, #0
+; CHECK-NEXT:    mov.w r5, #-2147483648
+; CHECK-NEXT:    sbcs.w r7, r5, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r9, r0, r2, ne
+; CHECK-NEXT:    csel r8, r1, r5, ne
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
-; CHECK-NEXT:    sbcs r6, r2, #0
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r2, r2, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    subs.w r7, r0, #-1
+; CHECK-NEXT:    sbcs.w r7, r1, r4
+; CHECK-NEXT:    sbcs r7, r2, #0
+; CHECK-NEXT:    sbcs r7, r3, #0
+; CHECK-NEXT:    cset r7, lt
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    csel r0, r0, r6, ne
+; CHECK-NEXT:    csel r3, r3, r7, ne
+; CHECK-NEXT:    csel r2, r2, r7, ne
+; CHECK-NEXT:    csel r1, r1, r4, ne
+; CHECK-NEXT:    rsbs r7, r0, #0
+; CHECK-NEXT:    sbcs.w r7, r5, r1
+; CHECK-NEXT:    sbcs.w r2, r6, r2
+; CHECK-NEXT:    sbcs.w r2, r6, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
+; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r9
 ; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -1815,38 +1791,33 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @utest_f64i64_mm(<2 x double> %x) {
 ; CHECK-LABEL: utest_f64i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vmov r4, r1, d8
+; CHECK-NEXT:    vmov r12, lr, d8
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    cset r6, lo
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r7, r0, r6, ne
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    cset r2, lo
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r4, r1, r2, ne
+; CHECK-NEXT:    csel r5, r0, r2, ne
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixunsdfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lo
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r4
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1857,49 +1828,43 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-LABEL: ustest_f64i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    vmov r4, r1, d8
+; CHECK-NEXT:    vmov r12, lr, d8
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r6, r0, r7, ne
-; CHECK-NEXT:    csel r5, r3, r7, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r6, #0
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r5, r0, r2, ne
+; CHECK-NEXT:    csel r0, r3, r2, ne
+; CHECK-NEXT:    csel r4, r1, r2, ne
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    movmi r4, #0
+; CHECK-NEXT:    movmi r5, #0
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    mov r1, lr
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r3, r3, r2, ne
+; CHECK-NEXT:    csel r1, r1, r2, ne
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
-; CHECK-NEXT:    csel r7, r8, r7, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r7, #0
+; CHECK-NEXT:    csel r2, r3, r2, ne
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    movmi r0, #0
 ; CHECK-NEXT:    movmi r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r4
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1911,59 +1876,54 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: stest_f32i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    vmov r9, r0, d0
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    vmov r8, r0, d0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r10, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r10
-; CHECK-NEXT:    mov.w r4, #-1
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs.w r7, r1, r5
+; CHECK-NEXT:    mov.w r6, #-2147483648
 ; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    mov.w r11, #-2147483648
 ; CHECK-NEXT:    sbcs r7, r3, #0
 ; CHECK-NEXT:    cset r7, lt
 ; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r5, r0, r4, ne
 ; CHECK-NEXT:    csel r3, r3, r7, ne
 ; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    rsbs r0, r5, #0
-; CHECK-NEXT:    sbcs.w r0, r11, r1
-; CHECK-NEXT:    sbcs.w r0, r4, r2
-; CHECK-NEXT:    sbcs.w r0, r4, r3
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r8, r1, r11, ne
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    sbcs.w r7, r1, r10
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r4, ne
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    rsbs r7, r0, #0
-; CHECK-NEXT:    sbcs.w r7, r11, r1
-; CHECK-NEXT:    sbcs.w r2, r4, r2
-; CHECK-NEXT:    sbcs.w r2, r4, r3
+; CHECK-NEXT:    mov.w r7, #-1
+; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    csel r0, r0, r7, ne
+; CHECK-NEXT:    rsbs r4, r0, #0
+; CHECK-NEXT:    sbcs.w r4, r6, r1
+; CHECK-NEXT:    sbcs.w r2, r7, r2
+; CHECK-NEXT:    sbcs.w r2, r7, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r11, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
+; CHECK-NEXT:    csel r10, r0, r2, ne
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    csel r9, r1, r6, ne
+; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    subs.w r4, r0, #-1
+; CHECK-NEXT:    sbcs.w r4, r1, r5
+; CHECK-NEXT:    sbcs r4, r2, #0
+; CHECK-NEXT:    sbcs r4, r3, #0
+; CHECK-NEXT:    cset r4, lt
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    csel r0, r0, r7, ne
+; CHECK-NEXT:    csel r3, r3, r4, ne
+; CHECK-NEXT:    csel r2, r2, r4, ne
+; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    rsbs r5, r0, #0
+; CHECK-NEXT:    sbcs.w r5, r6, r1
+; CHECK-NEXT:    sbcs.w r2, r7, r2
+; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:    csel r1, r1, r6, ne
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r10
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r9
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -1975,33 +1935,27 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @utest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: utest_f32i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    vmov r4, r0, d0
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    cset r6, lo
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r7, r0, r6, ne
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    sbcs r2, r3, #0
+; CHECK-NEXT:    cset r2, lo
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r6, r0, r2, ne
 ; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    csel r5, r1, r2, ne
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lo
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r3, r5, r6, ne
-; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -2012,44 +1966,37 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: ustest_f32i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vmov r5, r0, d0
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vmov r4, r0, d0
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r6, r0, r7, ne
-; CHECK-NEXT:    csel r4, r3, r7, ne
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it mi
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    sbcs r2, r3, #0
+; CHECK-NEXT:    cset r2, lt
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csel r6, r0, r2, ne
+; CHECK-NEXT:    csel r0, r3, r2, ne
+; CHECK-NEXT:    csel r5, r1, r2, ne
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    movmi r5, #0
 ; CHECK-NEXT:    movmi r6, #0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    sbcs r2, r3, #0
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r3, r3, r2, ne
+; CHECK-NEXT:    csel r1, r1, r2, ne
 ; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
-; CHECK-NEXT:    csel r7, r8, r7, ne
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r7, #0
+; CHECK-NEXT:    csel r2, r3, r2, ne
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    movmi r0, #0
 ; CHECK-NEXT:    movmi r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -2112,8 +2059,8 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
@@ -2122,24 +2069,19 @@ define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    mov r6, r3
-; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r4, #0
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r0, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
-; CHECK-NEXT:    it mi
+; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    movmi r5, #0
+; CHECK-NEXT:    movmi r4, #0
+; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it mi
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    movmi r0, #0
 ; CHECK-NEXT:    movmi r1, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
 ; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll b/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
index 75b6cb3e1272b..77548b49d77f2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
@@ -281,49 +281,51 @@ declare <6 x i32> @llvm.fptosi.sat.v6f64.v6i32 (<6 x double>)
 define arm_aapcs_vfpcc <1 x i32> @test_signed_v1f64_v1i32(<1 x double> %f) {
 ; CHECK-LABEL: test_signed_v1f64_v1i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldr d1, .LCPI8_0
 ; CHECK-NEXT:    vmov r5, r4, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI8_1
-; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    bl __aeabi_d2iz
+; CHECK-NEXT:    vldr d0, .LCPI8_1
 ; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_d2iz
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r6, #-2147483648
+; CHECK-NEXT:    moveq.w r7, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    mvnne r7, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI8_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI8_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI8_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <1 x i32> @llvm.fptosi.sat.v1f64.v1i32(<1 x double> %f)
     ret <1 x i32> %x
 }
@@ -337,115 +339,82 @@ define arm_aapcs_vfpcc <2 x i32> @test_signed_v2f64_v2i32(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI9_0
-; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r8, d9
+; CHECK-NEXT:    vmov r7, r3, d0
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI9_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r11, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vldr d0, .LCPI9_1
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-2147483648
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vmov r11, r10, d8
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq.w r4, #-1
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    mvnne r5, #-2147483648
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq.w r7, #-2147483648
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    mvnne r7, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -472,90 +441,53 @@ define arm_aapcs_vfpcc <3 x i32> @test_signed_v3f64_v3i32(<3 x double> %f) {
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    .pad #24
 ; CHECK-NEXT:    sub sp, #24
-; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    vmov.f32 s17, s1
+; CHECK-NEXT:    vmov.f32 s18, s0
+; CHECK-NEXT:    vmov.f32 s19, s1
 ; CHECK-NEXT:    vldr d0, .LCPI10_0
-; CHECK-NEXT:    vmov r4, r6, d1
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    vmov.f32 s18, s4
-; CHECK-NEXT:    vmov.f32 s19, s5
-; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI10_1
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    vmov r2, r8, d0
-; CHECK-NEXT:    str r2, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    str.w r8, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    vmov r5, r7, d9
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r10, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    vmov r10, r7, d1
+; CHECK-NEXT:    vmov r6, r3, d0
+; CHECK-NEXT:    vmov.f32 s16, s4
+; CHECK-NEXT:    vmov.f32 s17, s5
+; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r10
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r10
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vldr d0, .LCPI10_1
 ; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    vmov r1, r0, d9
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r9, r8, d8
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    strd r1, r0, [sp, #12] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r10
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp.w r11, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r6, #-2147483648
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    moveq.w r11, #-2147483648
+; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r10
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    vmov r9, r8, d8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    mvnne r11, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #0
-; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    movne.w r11, #0
+; CHECK-NEXT:    ldr.w r10, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r0, r9
@@ -565,31 +497,67 @@ define arm_aapcs_vfpcc <3 x i32> @test_signed_v3f64_v3i32(<3 x double> %f) {
 ; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r7, #-2147483648
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r7, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov.32 q0[1], r10
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r7, r6
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq.w r6, #-2147483648
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov.32 q0[1], r11
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r7
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI10_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI10_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI10_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <3 x i32> @llvm.fptosi.sat.v3f64.v3i32(<3 x double> %f)
     ret <3 x i32> %x
 }
@@ -603,95 +571,86 @@ define arm_aapcs_vfpcc <4 x i32> @test_signed_v4f64_v4i32(<4 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI11_0
-; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vmov r5, r6, d10
-; CHECK-NEXT:    vmov r9, r3, d0
-; CHECK-NEXT:    str r3, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI11_1
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r9, r3
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    vmov r11, r1, d11
+; CHECK-NEXT:    vldr d0, .LCPI11_1
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r11, r0, d11
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    vmov r7, r10, d8
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    str r2, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r0, #-2147483648
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r0, #-2147483648
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    moveq.w r4, #-2147483648
+; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr.w r8, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    str.w r9, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r4, r9
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    str.w r9, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r10
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r6, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    moveq.w r9, #-2147483648
+; CHECK-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    mvnne r9, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #0
-; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r9, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r11
@@ -701,11 +660,17 @@ define arm_aapcs_vfpcc <4 x i32> @test_signed_v4f64_v4i32(<4 x double> %f) {
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r8, #-2147483648
+; CHECK-NEXT:    ldr.w r10, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r11
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    vmov r7, r4, d9
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r8, #-2147483648
@@ -713,51 +678,50 @@ define arm_aapcs_vfpcc <4 x i32> @test_signed_v4f64_v4i32(<4 x double> %f) {
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    ldr r3, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    moveq.w r6, #-2147483648
+; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r5, #-2147483648
+; CHECK-NEXT:    mvnne r6, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r6, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r5, r8
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r9, r0
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r8
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI11_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI11_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI11_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <4 x i32> @llvm.fptosi.sat.v4f64.v4i32(<4 x double> %f)
     ret <4 x i32> %x
 }
@@ -774,186 +738,186 @@ define arm_aapcs_vfpcc <5 x i32> @test_signed_v5f64_v5i32(<5 x double> %f) {
 ; CHECK-NEXT:    .pad #32
 ; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    vmov.f32 s17, s1
 ; CHECK-NEXT:    vldr d0, .LCPI12_0
-; CHECK-NEXT:    vmov r5, r4, d4
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    vmov r7, r5, d4
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    vmov.f32 s20, s6
-; CHECK-NEXT:    vmov.f32 s18, s4
+; CHECK-NEXT:    vmov.f32 s18, s6
+; CHECK-NEXT:    vmov.f32 s20, s4
 ; CHECK-NEXT:    vmov.f32 s22, s2
-; CHECK-NEXT:    vmov.f32 s21, s7
-; CHECK-NEXT:    vmov.f32 s19, s5
+; CHECK-NEXT:    vmov.f32 s19, s7
+; CHECK-NEXT:    vmov.f32 s21, s5
 ; CHECK-NEXT:    vmov.f32 s23, s3
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    strd r2, r3, [sp, #20] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI12_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    vmov r8, r0, d11
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    vmov r9, r6, d10
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r1, r0, d10
+; CHECK-NEXT:    vldr d0, .LCPI12_1
+; CHECK-NEXT:    vmov r6, r8, d9
+; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    vmov r10, r3, d0
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r0, d11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r11, r3
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r11, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    moveq.w r4, #-2147483648
+; CHECK-NEXT:    str.w r10, [sp] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r11, #-2147483648
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    str.w r11, [r7, #16]
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    ldr.w r10, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    ldr r7, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r4, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    ldr.w r11, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    str r4, [r0, #16]
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r0, #-2147483648
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r10, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r0, #-2147483648
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    str r5, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    mov r6, r11
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-2147483648
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    vmov r11, r4, d9
+; CHECK-NEXT:    moveq.w r8, #-2147483648
+; CHECK-NEXT:    ldr.w r11, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r10, #-2147483648
+; CHECK-NEXT:    mvnne r8, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    ldr.w r9, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r7, #-2147483648
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq.w r4, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r7, #-2147483648
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    vmov r7, r6, d8
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r6, #-2147483648
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r6, r7
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r0
+; CHECK-NEXT:    movne r5, #0
 ; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r8, r0
+; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
@@ -962,11 +926,11 @@ define arm_aapcs_vfpcc <5 x i32> @test_signed_v5f64_v5i32(<5 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI12_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI12_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI12_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <5 x i32> @llvm.fptosi.sat.v5f64.v5i32(<5 x double> %f)
     ret <5 x i32> %x
 }
@@ -983,182 +947,180 @@ define arm_aapcs_vfpcc <6 x i32> @test_signed_v6f64_v6i32(<6 x double> %f) {
 ; CHECK-NEXT:    .pad #40
 ; CHECK-NEXT:    sub sp, #40
 ; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
 ; CHECK-NEXT:    vmov.f32 s17, s1
 ; CHECK-NEXT:    vldr d0, .LCPI13_0
-; CHECK-NEXT:    vmov r9, r4, d5
-; CHECK-NEXT:    vmov r2, r6, d0
-; CHECK-NEXT:    vmov.f32 s22, s8
-; CHECK-NEXT:    vmov.f32 s20, s6
+; CHECK-NEXT:    vmov r6, r4, d5
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov.f32 s20, s8
+; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s18, s4
 ; CHECK-NEXT:    vmov.f32 s24, s2
-; CHECK-NEXT:    vmov.f32 s23, s9
-; CHECK-NEXT:    vmov.f32 s21, s7
+; CHECK-NEXT:    vmov.f32 s21, s9
+; CHECK-NEXT:    vmov.f32 s23, s7
 ; CHECK-NEXT:    vmov.f32 s19, s5
 ; CHECK-NEXT:    vmov.f32 s25, s3
-; CHECK-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    str r6, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI13_1
+; CHECK-NEXT:    str r2, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r2, r3, [sp, #32] @ 8-byte Folded Spill
+; CHECK-NEXT:    str r3, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    vmov r8, r0, d10
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    vmov r7, r5, d11
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    vldr d0, .LCPI13_1
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    vmov r1, r0, d12
-; CHECK-NEXT:    strd r1, r0, [sp, #12] @ 8-byte Folded Spill
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r7, r8, d10
+; CHECK-NEXT:    vmov r11, r10, d11
+; CHECK-NEXT:    str r3, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    mov r9, r2
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r4
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r10, #-2147483648
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr.w r11, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    str.w r10, [r11, #20]
-; CHECK-NEXT:    ldr.w r10, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #32] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r6, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    str r5, [r6, #20]
+; CHECK-NEXT:    ldr r2, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    ldr r3, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r6, #-2147483648
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    vmov r2, r1, d9
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    strd r2, r1, [sp] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r6, #-2147483648
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #0
-; CHECK-NEXT:    str.w r6, [r11, #16]
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr r4, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r7, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    ldr r5, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    str r5, [r6, #16]
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    ldr.w r8, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    ldr.w r9, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-2147483648
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r10, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    moveq.w r0, #-2147483648
+; CHECK-NEXT:    ldr r7, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r4, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r5, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r8, #-2147483648
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    moveq.w r10, #-2147483648
+; CHECK-NEXT:    ldr.w r11, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    vmov r7, r6, d9
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r8, #-2147483648
+; CHECK-NEXT:    mvnne r10, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    ldr.w r11, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    ldr r3, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #32] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r5, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r4, #-2147483648
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    ldr.w r8, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
@@ -1166,38 +1128,39 @@ define arm_aapcs_vfpcc <6 x i32> @test_signed_v6f64_v6i32(<6 x double> %f) {
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r3, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #32] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    ldr r2, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vmov q0[2], q0[0], r5, r4
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[3], q0[1], r8, r10
+; CHECK-NEXT:    vmov q0[3], q0[1], r10, r0
+; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #40
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12}
@@ -1206,11 +1169,11 @@ define arm_aapcs_vfpcc <6 x i32> @test_signed_v6f64_v6i32(<6 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI13_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI13_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI13_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <6 x i32> @llvm.fptosi.sat.v6f64.v6i32(<6 x double> %f)
     ret <6 x i32> %x
 }
@@ -1791,18 +1754,10 @@ define arm_aapcs_vfpcc <4 x i50> @test_signed_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    vmov r0, s19
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    vldr s22, .LCPI28_0
 ; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vldr s20, .LCPI28_1
@@ -1811,48 +1766,60 @@ define arm_aapcs_vfpcc <4 x i50> @test_signed_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movwlt r11, #0
 ; CHECK-NEXT:    movtlt r11, #65534
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    vcmp.f32 s17, s17
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r11, #65535
 ; CHECK-NEXT:    movtgt r11, #1
-; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s22
 ; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs.w r9, #0
 ; CHECK-NEXT:    movvs.w r11, #0
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
-; CHECK-NEXT:    ittt lt
+; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    movtlt r5, #65534
+; CHECK-NEXT:    vcmp.f32 s18, s20
+; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r6, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s18
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r5, #65535
 ; CHECK-NEXT:    movtgt r5, #1
+; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s19, s22
 ; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs r6, #0
 ; CHECK-NEXT:    movvs r5, #0
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vcmp.f32 s19, s22
+; CHECK-NEXT:    mov r7, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt lt
+; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    movtlt r7, #65534
-; CHECK-NEXT:    movlt r4, #0
 ; CHECK-NEXT:    vcmp.f32 s19, s20
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r4, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r7, #65535
 ; CHECK-NEXT:    movtgt r7, #1
+; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r4, #-1
 ; CHECK-NEXT:    vcmp.f32 s19, s19
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1864,9 +1831,9 @@ define arm_aapcs_vfpcc <4 x i50> @test_signed_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    bfc r5, #18, #14
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt lt
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movtlt r1, #65534
-; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1880,11 +1847,10 @@ define arm_aapcs_vfpcc <4 x i50> @test_signed_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    str.w r0, [r8]
-; CHECK-NEXT:    lsrs r0, r7, #10
+; CHECK-NEXT:    lsr.w r0, r7, #10
 ; CHECK-NEXT:    bfc r7, #18, #14
 ; CHECK-NEXT:    bfc r11, #18, #14
 ; CHECK-NEXT:    lsll r4, r7, #22
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    orr.w r3, r5, r7
 ; CHECK-NEXT:    str.w r3, [r8, #20]
 ; CHECK-NEXT:    orr.w r2, r2, r4
@@ -2029,52 +1995,23 @@ define arm_aapcs_vfpcc <4 x i100> @test_signed_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    vmov r6, s17
 ; CHECK-NEXT:    vldr s22, .LCPI30_0
-; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vldr s20, .LCPI30_1
-; CHECK-NEXT:    mov r10, r2
-; CHECK-NEXT:    vcmp.f32 s17, s22
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    mvnlt r3, #7
-; CHECK-NEXT:    movlt.w r10, #0
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt.w r11, #0
-; CHECK-NEXT:    vcmp.f32 s17, s20
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r11, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    movgt.w r10, #-1
-; CHECK-NEXT:    movgt r3, #7
-; CHECK-NEXT:    vcmp.f32 s17, s17
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    itt vs
-; CHECK-NEXT:    movvs.w r10, #0
-; CHECK-NEXT:    movvs r7, #0
-; CHECK-NEXT:    str r7, [sp] @ 4-byte Spill
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs.w r11, #0
-; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    vmov r7, s19
 ; CHECK-NEXT:    vcmp.f32 s18, s22
-; CHECK-NEXT:    mov r5, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    mvnlt r5, #7
+; CHECK-NEXT:    mvnlt r3, #7
 ; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt r5, #7
+; CHECK-NEXT:    movgt r3, #7
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
@@ -2086,40 +2023,67 @@ define arm_aapcs_vfpcc <4 x i100> @test_signed_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    str.w r1, [r4, #29]
-; CHECK-NEXT:    vmov r1, s19
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    str.w r0, [r4, #25]
 ; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r3, #0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    vcmp.f32 s17, s22
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itttt lt
+; CHECK-NEXT:    mvnlt r3, #7
+; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    vcmp.f32 s17, s20
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itttt gt
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
+; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    movgt r3, #7
+; CHECK-NEXT:    vcmp.f32 s17, s17
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r3, #0
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    movvs r6, #0
+; CHECK-NEXT:    movvs.w r10, #0
 ; CHECK-NEXT:    movvs r5, #0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov r0, s16
 ; CHECK-NEXT:    vcmp.f32 s19, s22
-; CHECK-NEXT:    mov r9, r1
-; CHECK-NEXT:    mov r6, r2
-; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    mov r11, r1
+; CHECK-NEXT:    mov r8, r2
+; CHECK-NEXT:    mov r9, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    mvnlt r8, #7
-; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    mvnlt r9, #7
+; CHECK-NEXT:    movlt.w r8, #0
+; CHECK-NEXT:    movlt.w r11, #0
 ; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    vcmp.f32 s19, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    movgt.w r9, #-1
-; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movgt.w r8, #7
+; CHECK-NEXT:    movgt.w r11, #-1
+; CHECK-NEXT:    movgt.w r8, #-1
+; CHECK-NEXT:    movgt.w r9, #7
 ; CHECK-NEXT:    vcmp.f32 s19, s19
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt vs
-; CHECK-NEXT:    movvs.w r8, #0
-; CHECK-NEXT:    movvs r6, #0
 ; CHECK-NEXT:    movvs.w r9, #0
+; CHECK-NEXT:    movvs.w r8, #0
+; CHECK-NEXT:    movvs.w r11, #0
 ; CHECK-NEXT:    movvs r7, #0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s16, s22
@@ -2148,30 +2112,30 @@ define arm_aapcs_vfpcc <4 x i100> @test_signed_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    str r0, [r4]
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r9, #28
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    orr.w r1, r9, r6, lsl #4
-; CHECK-NEXT:    str.w r1, [r4, #45]
-; CHECK-NEXT:    and r1, r8, #15
+; CHECK-NEXT:    lsrl r0, r11, #28
+; CHECK-NEXT:    and r1, r9, #15
 ; CHECK-NEXT:    str.w r0, [r4, #41]
-; CHECK-NEXT:    and r0, r5, #15
-; CHECK-NEXT:    lsrl r6, r1, #28
-; CHECK-NEXT:    strb.w r6, [r4, #49]
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    lsrl r0, r5, #28
+; CHECK-NEXT:    str r0, [r4, #16]
+; CHECK-NEXT:    orr.w r0, r11, r8, lsl #4
+; CHECK-NEXT:    lsrl r8, r1, #28
+; CHECK-NEXT:    str.w r0, [r4, #45]
+; CHECK-NEXT:    strb.w r8, [r4, #49]
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r0, #15
 ; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
 ; CHECK-NEXT:    str.w r0, [r4, #37]
-; CHECK-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    lsrl r0, r11, #28
-; CHECK-NEXT:    orr.w r1, r11, r10, lsl #4
-; CHECK-NEXT:    strd r0, r1, [r4, #16]
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r5, r6, lsl #4
+; CHECK-NEXT:    str r0, [r4, #20]
+; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
-; CHECK-NEXT:    lsrl r10, r1, #28
-; CHECK-NEXT:    strb.w r10, [r4, #24]
+; CHECK-NEXT:    lsrl r6, r1, #28
+; CHECK-NEXT:    strb r6, [r4, #24]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    and r0, r3, #15
-; CHECK-NEXT:    orr.w r0, r0, r2, lsl #4
+; CHECK-NEXT:    orr.w r0, r0, r10, lsl #4
 ; CHECK-NEXT:    str r0, [r4, #12]
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
@@ -2200,61 +2164,58 @@ define arm_aapcs_vfpcc <4 x i128> @test_signed_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov r5, s18
 ; CHECK-NEXT:    vldr s22, .LCPI31_0
-; CHECK-NEXT:    vmov r7, s16
 ; CHECK-NEXT:    vldr s20, .LCPI31_1
-; CHECK-NEXT:    vmov r6, s17
+; CHECK-NEXT:    add.w r12, r4, #48
 ; CHECK-NEXT:    vcmp.f32 s19, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s19, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s19, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s19, s19
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s19, s19
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    movvs r5, #0
-; CHECK-NEXT:    strd r5, r1, [r4, #48]
-; CHECK-NEXT:    strd r2, r3, [r4, #56]
+; CHECK-NEXT:    movvs r0, #0
+; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    vmov r7, s16
+; CHECK-NEXT:    vmov r6, s17
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s18, s22
 ; CHECK-NEXT:    add.w r12, r4, #32
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r6
@@ -2262,52 +2223,48 @@ define arm_aapcs_vfpcc <4 x i128> @test_signed_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-NEXT:    vcmp.f32 s17, s22
 ; CHECK-NEXT:    add.w r12, r4, #16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s17, s17
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s17, s17
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s16, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm r4!, {r0, r1, r2, r3}
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
@@ -2346,70 +2303,72 @@ define arm_aapcs_vfpcc <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI32_0
 ; CHECK-NEXT:    vmov r8, r7, d8
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #12] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI32_1
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r10, r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2iz
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vldr d0, .LCPI32_1
+; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    vmov r6, r5, d9
+; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r11, #-1
+; CHECK-NEXT:    moveq.w r9, #-1
+; CHECK-NEXT:    mov r10, r3
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r8
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    vmov r6, r5, d9
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
+; CHECK-NEXT:    movne.w r9, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    and r0, r11, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #12] @ 8-byte Folded Reload
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    and r0, r9, #1
+; CHECK-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    movs r4, #0
 ; CHECK-NEXT:    bfi r4, r0, #0, #1
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2iz
 ; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    moveq.w r7, #-1
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r7, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
@@ -2419,20 +2378,20 @@ define arm_aapcs_vfpcc <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-NEXT:    and r0, r7, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r4, r0, #1, #1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    strb r4, [r0]
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI32_0:
-; CHECK-NEXT:    .long 0 @ double 0
-; CHECK-NEXT:    .long 0
-; CHECK-NEXT:  .LCPI32_1:
 ; CHECK-NEXT:    .long 0 @ double -1
 ; CHECK-NEXT:    .long 3220176896
+; CHECK-NEXT:  .LCPI32_1:
+; CHECK-NEXT:    .long 0 @ double 0
+; CHECK-NEXT:    .long 0
     %x = call <2 x i1> @llvm.fptosi.sat.v2f64.v2i1(<2 x double> %f)
     ret <2 x i1> %x
 }
@@ -2446,115 +2405,82 @@ define arm_aapcs_vfpcc <2 x i8> @test_signed_v2f64_v2i8(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI33_0
-; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r8, d9
+; CHECK-NEXT:    vmov r7, r3, d0
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI33_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r11, #127
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #127
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vldr d0, .LCPI33_1
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r5, #127
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vmov r11, r10, d8
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq.w r4, #-1
+; CHECK-NEXT:    mvneq r5, #127
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne r5, #127
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    mvneq r7, #127
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movne r7, #127
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2579,118 +2505,84 @@ define arm_aapcs_vfpcc <2 x i13> @test_signed_v2f64_v2i13(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI34_0
-; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r8, d9
+; CHECK-NEXT:    vmov r7, r3, d0
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI34_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vldr d0, .LCPI34_1
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vmov r11, r10, d8
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    movweq r5, #61440
+; CHECK-NEXT:    movteq r5, #65535
+; CHECK-NEXT:    moveq.w r4, #-1
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r5, #4095
 ; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    movweq r7, #61440
+; CHECK-NEXT:    movteq r7, #65535
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r5, #61440
-; CHECK-NEXT:    movtne r5, #65535
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r11, #61440
-; CHECK-NEXT:    movtne r11, #65535
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r11, #4095
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r5, #4095
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movwne r7, #4095
+; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r0
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2715,118 +2607,84 @@ define arm_aapcs_vfpcc <2 x i16> @test_signed_v2f64_v2i16(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI35_0
-; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r8, d9
+; CHECK-NEXT:    vmov r7, r3, d0
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI35_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vldr d0, .LCPI35_1
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vmov r11, r10, d8
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    movweq r5, #32768
+; CHECK-NEXT:    movteq r5, #65535
+; CHECK-NEXT:    moveq.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r5, #32767
 ; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    movweq r7, #32768
+; CHECK-NEXT:    movteq r7, #65535
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r5, #32768
-; CHECK-NEXT:    movtne r5, #65535
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r11, #32768
-; CHECK-NEXT:    movtne r11, #65535
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r11, #32767
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r5, #32767
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movwne r7, #32767
+; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r0
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2851,118 +2709,94 @@ define arm_aapcs_vfpcc <2 x i19> @test_signed_v2f64_v2i19(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI36_0
-; CHECK-NEXT:    vmov r7, r6, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r5, r4, d9
+; CHECK-NEXT:    vmov r7, r6, d0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    strd r5, r4, [sp, #12] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    vldr d0, .LCPI36_1
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r8, r0, d8
-; CHECK-NEXT:    vmov r11, r10, d0
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    clz r0, r4
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    ittt ne
-; CHECK-NEXT:    movwne r9, #0
-; CHECK-NEXT:    movtne r9, #65532
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    vmov r11, r5, d8
+; CHECK-NEXT:    mov r9, r1
+; CHECK-NEXT:    vmov r10, r0, d0
 ; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    movweq r8, #0
+; CHECK-NEXT:    movteq r8, #65532
+; CHECK-NEXT:    moveq.w r9, #-1
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    str r5, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r9, #65535
-; CHECK-NEXT:    movtne r9, #3
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r4, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r7, #0
+; CHECK-NEXT:    movteq r7, #65532
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movwne r7, #65535
+; CHECK-NEXT:    movtne r7, #3
+; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r11, r4
-; CHECK-NEXT:    lsr.w r10, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    movtne r4, #65532
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r4, #65535
-; CHECK-NEXT:    movtne r4, #3
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    movwne r8, #65535
+; CHECK-NEXT:    movtne r8, #3
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r9
-; CHECK-NEXT:    vmov q0[3], q0[1], r7, r0
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r8
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r9
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2987,115 +2821,82 @@ define arm_aapcs_vfpcc <2 x i32> @test_signed_v2f64_v2i32_duplicate(<2 x double>
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI37_0
-; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r9, r8, d9
+; CHECK-NEXT:    vmov r7, r3, d0
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI37_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r11, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vldr d0, .LCPI37_1
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-2147483648
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r5, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vmov r11, r10, d8
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq.w r4, #-1
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq.w r7, #-2147483648
+; CHECK-NEXT:    moveq.w r6, #-1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    mvnne r7, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -3124,114 +2925,84 @@ define arm_aapcs_vfpcc <2 x i50> @test_signed_v2f64_v2i50(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI38_0
-; CHECK-NEXT:    vmov r7, r6, d9
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r5, r4, d9
+; CHECK-NEXT:    vmov r6, r7, d0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    strd r4, r5, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    vmov r5, r10, d8
 ; CHECK-NEXT:    vldr d0, .LCPI38_1
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r8, r0, d8
-; CHECK-NEXT:    vmov r11, r10, d0
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    clz r0, r4
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    movtne r1, #65534
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    csel r11, r0, r11, ne
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r4, #0
+; CHECK-NEXT:    movteq r4, #65534
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    csel r7, r0, r7, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r6, #0
+; CHECK-NEXT:    movteq r6, #65534
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    movwne r6, #65535
+; CHECK-NEXT:    movtne r6, #1
+; CHECK-NEXT:    ldrd r9, r0, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne.w r11, #-1
+; CHECK-NEXT:    movwne r4, #65535
+; CHECK-NEXT:    movtne r4, #1
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r5, #65535
-; CHECK-NEXT:    movtne r5, #1
-; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #0
-; CHECK-NEXT:    ldr r4, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r11, r4
-; CHECK-NEXT:    lsr.w r10, r0, #5
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    movtne r7, #65534
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r7, #65535
-; CHECK-NEXT:    movtne r7, #1
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r11, #0
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r9
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r7, r1
+; CHECK-NEXT:    vmov q0[2], q0[0], r7, r11
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r6, r4
 ; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -3257,115 +3028,80 @@ define arm_aapcs_vfpcc <2 x i64> @test_signed_v2f64_v2i64(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI39_0
 ; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r5, d0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vldr d0, .LCPI39_1
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r6, r5, d0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vldr d0, .LCPI39_1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    csel r9, r0, r9, ne
+; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
+; CHECK-NEXT:    vmov r6, r10, d8
+; CHECK-NEXT:    strd r2, r3, [sp] @ 8-byte Folded Spill
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq.w r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r8
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #-1
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    vmov r10, r7, d8
-; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #1
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    movne.w r9, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    bl __aeabi_d2lz
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq.w r7, #-2147483648
+; CHECK-NEXT:    ldrd r2, r3, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    csel r5, r0, r5, ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    mvnne r7, #-2147483648
 ; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
+; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-2147483648
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r2, #-2147483648
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r2, #0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-2147483648
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r4, #-2147483648
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r4
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -3394,258 +3130,252 @@ define arm_aapcs_vfpcc <2 x i100> @test_signed_v2f64_v2i100(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #48
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI40_0
-; CHECK-NEXT:    vmov r10, r9, d8
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    vmov r7, r3, d0
-; CHECK-NEXT:    str r3, [sp, #40] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    str r7, [sp, #44] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI40_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    vmov r8, r3, d0
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r11, r3
+; CHECK-NEXT:    vmov r7, r6, d8
+; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    vmov r10, r9, d0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __fixdfti
+; CHECK-NEXT:    vldr d0, .LCPI40_1
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    strd r1, r0, [sp, #16] @ 8-byte Folded Spill
 ; CHECK-NEXT:    csel r4, r2, r4, ne
-; CHECK-NEXT:    str r3, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    vmov r5, r11, d0
+; CHECK-NEXT:    str r3, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [r6, #8]
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    ldr r5, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    str.w r8, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    str.w r8, [sp, #44] @ 4-byte Spill
+; CHECK-NEXT:    str.w r4, [r8, #8]
+; CHECK-NEXT:    str.w r9, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    csel r7, r1, r0, ne
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    csel r4, r1, r0, ne
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    str.w r11, [sp, #40] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    str r7, [r6, #4]
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    ldr r2, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    str r6, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    str.w r11, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str.w r4, [r8, #4]
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    csel r7, r1, r0, ne
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    csel r4, r1, r0, ne
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
-; CHECK-NEXT:    str.w r10, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    str.w r9, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r7, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    str r6, [sp, #28] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    vmov r9, r8, d9
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    str r7, [r6]
-; CHECK-NEXT:    ldr r6, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r10, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    str r4, [r0]
+; CHECK-NEXT:    ldr.w r11, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    strd r2, r3, [sp, #4] @ 8-byte Folded Spill
-; CHECK-NEXT:    csel r11, r1, r11, ne
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    strd r2, r3, [sp, #12] @ 8-byte Folded Spill
+; CHECK-NEXT:    csel r7, r1, r4, ne
+; CHECK-NEXT:    mov r4, r5
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    ldr r5, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #-1
+; CHECK-NEXT:    movne.w r7, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r5, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r6, r10
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    csel r10, r4, r0, ne
+; CHECK-NEXT:    csel r6, r6, r0, ne
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #-1
+; CHECK-NEXT:    movne.w r6, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r7, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    ldr r3, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    str.w r10, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    lsrl r10, r11, #28
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    movne r0, #0
+; CHECK-NEXT:    ldr r1, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    lsrl r0, r7, #28
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    str r0, [r1, #16]
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r6, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    ldr.w r11, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    csel r4, r1, r0, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    orr.w r0, r11, r4, lsl #4
+; CHECK-NEXT:    orr.w r0, r7, r4, lsl #4
+; CHECK-NEXT:    ldr r7, [sp, #44] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    strd r10, r0, [r6, #16]
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    ldr.w r11, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r7, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    str r0, [r7, #20]
 ; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    mvneq r0, #7
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #7
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mvneq r6, #7
+; CHECK-NEXT:    mov r10, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #7
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    and r1, r5, #15
-; CHECK-NEXT:    mov r8, r6
+; CHECK-NEXT:    movne r0, #0
+; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r4, r1, #28
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    strb r4, [r6, #24]
-; CHECK-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    strb r4, [r7, #24]
 ; CHECK-NEXT:    ldr r5, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    ldr r6, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    ldr r3, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    ldr r4, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    mvneq r0, #7
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mvneq r4, #7
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #7
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    movne r4, #7
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    and r0, r4, #15
+; CHECK-NEXT:    movne r0, #0
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r0, #15
 ; CHECK-NEXT:    orr.w r0, r0, r1, lsl #4
-; CHECK-NEXT:    str.w r0, [r8, #12]
+; CHECK-NEXT:    str r0, [r7, #12]
 ; CHECK-NEXT:    add sp, #48
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -3653,11 +3383,11 @@ define arm_aapcs_vfpcc <2 x i100> @test_signed_v2f64_v2i100(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI40_0:
-; CHECK-NEXT:    .long 4294967295 @ double 6.3382530011411463E+29
-; CHECK-NEXT:    .long 1176502271
-; CHECK-NEXT:  .LCPI40_1:
 ; CHECK-NEXT:    .long 0 @ double -6.338253001141147E+29
 ; CHECK-NEXT:    .long 3323985920
+; CHECK-NEXT:  .LCPI40_1:
+; CHECK-NEXT:    .long 4294967295 @ double 6.3382530011411463E+29
+; CHECK-NEXT:    .long 1176502271
     %x = call <2 x i100> @llvm.fptosi.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
@@ -3676,247 +3406,237 @@ define arm_aapcs_vfpcc <2 x i128> @test_signed_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI41_0
 ; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r11, r3
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI41_1
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    vmov r4, r3, d0
-; CHECK-NEXT:    str r3, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r4, r2
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r10, r3
-; CHECK-NEXT:    strd r2, r1, [sp] @ 8-byte Folded Spill
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    vldr d0, .LCPI41_1
+; CHECK-NEXT:    mov r5, r3
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r10, r11, d0
+; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-2147483648
+; CHECK-NEXT:    moveq.w r5, #-2147483648
+; CHECK-NEXT:    str.w r11, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r8
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r10, #-2147483648
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    str.w r10, [r6, #28]
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr.w r9, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    mov r5, r11
-; CHECK-NEXT:    str.w r11, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r10, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    str.w r5, [r9, #28]
 ; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r11, r4
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r4, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    csel r4, r1, r0, ne
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    csel r5, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    str.w r10, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    str.w r5, [r9, #24]
+; CHECK-NEXT:    mov r11, r6
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    str r4, [r6, #24]
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r2, r8
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    csel r4, r1, r0, ne
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [r6, #20]
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr.w r10, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r11, r6
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    str.w r4, [r9, #20]
 ; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r5, d8
+; CHECK-NEXT:    mov r10, r9
+; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r9, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr.w r11, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    vmov r6, r5, d8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str.w r4, [r11, #16]
+; CHECK-NEXT:    str.w r4, [r10, #16]
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r9, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
 ; CHECK-NEXT:    ldr.w r8, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r2, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    strd r2, r1, [sp] @ 8-byte Folded Spill
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    mov r9, r3
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r4, #-2147483648
+; CHECK-NEXT:    moveq.w r9, #-2147483648
+; CHECK-NEXT:    ldr.w r10, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    mov r4, r2
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r4, #-2147483648
+; CHECK-NEXT:    mvnne r9, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    str.w r4, [r10, #12]
-; CHECK-NEXT:    ldr.w r11, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    str.w r9, [r7, #12]
+; CHECK-NEXT:    ldr.w r9, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r4, r4, r0, ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    csel r7, r1, r0, ne
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    str.w r7, [r10, #8]
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str r4, [r7, #8]
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    csel r7, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    str.w r7, [r10, #4]
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str r4, [r7, #4]
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    csel r7, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    str.w r7, [r10]
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str r4, [r7]
 ; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -3924,11 +3644,11 @@ define arm_aapcs_vfpcc <2 x i128> @test_signed_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI41_0:
-; CHECK-NEXT:    .long 4294967295 @ double 1.7014118346046921E+38
-; CHECK-NEXT:    .long 1205862399
-; CHECK-NEXT:  .LCPI41_1:
 ; CHECK-NEXT:    .long 0 @ double -1.7014118346046923E+38
 ; CHECK-NEXT:    .long 3353346048
+; CHECK-NEXT:  .LCPI41_1:
+; CHECK-NEXT:    .long 4294967295 @ double 1.7014118346046921E+38
+; CHECK-NEXT:    .long 1205862399
     %x = call <2 x i128> @llvm.fptosi.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
@@ -4559,101 +4279,103 @@ define arm_aapcs_vfpcc <8 x i50> @test_signed_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    .pad #24
 ; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s17
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s16
 ; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcvtb.f32.f16 s28, s17
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    vmov r0, s28
-; CHECK-NEXT:    mov r6, r1
-; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcvtt.f32.f16 s30, s16
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, s30
-; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s17
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    vldr s22, .LCPI48_0
+; CHECK-NEXT:    vldr s20, .LCPI48_1
+; CHECK-NEXT:    vcmp.f32 s24, s22
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itt lt
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movtlt r1, #65534
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    vcmp.f32 s24, s20
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r2, #-1
+; CHECK-NEXT:    vcmp.f32 s24, s24
+; CHECK-NEXT:    itt gt
+; CHECK-NEXT:    movwgt r1, #65535
+; CHECK-NEXT:    movtgt r1, #1
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r2, #0
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r1, #0
+; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcvtb.f32.f16 s26, s18
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s17
 ; CHECK-NEXT:    mov r2, r0
-; CHECK-NEXT:    vmov r0, s26
-; CHECK-NEXT:    vldr s20, .LCPI48_0
-; CHECK-NEXT:    vldr s22, .LCPI48_1
-; CHECK-NEXT:    vcmp.f32 s30, s20
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vcmp.f32 s26, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movtlt r1, #65534
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s30, s22
+; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s30, s30
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r1, #65535
 ; CHECK-NEXT:    movtgt r1, #1
+; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r2, #-1
+; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s20
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r2, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
+; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_f2lz
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s18
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    vcmp.f32 s24, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s22
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    ittt lt
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    movtlt r5, #65534
-; CHECK-NEXT:    movlt r4, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s28
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r5, #65535
-; CHECK-NEXT:    movtgt r5, #1
-; CHECK-NEXT:    movgt.w r4, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itt lt
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movtlt r1, #65534
 ; CHECK-NEXT:    vcmp.f32 s24, s20
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r5, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    str r5, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    ittt lt
-; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movtlt r6, #65534
-; CHECK-NEXT:    movlt.w r8, #0
-; CHECK-NEXT:    vcmp.f32 s24, s22
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r6, #65535
-; CHECK-NEXT:    movtgt r6, #1
-; CHECK-NEXT:    movgt.w r8, #-1
+; CHECK-NEXT:    itt gt
+; CHECK-NEXT:    movwgt r1, #65535
+; CHECK-NEXT:    movtgt r1, #1
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs.w r8, #0
-; CHECK-NEXT:    str.w r8, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    movvs r2, #0
+; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r6, #0
-; CHECK-NEXT:    str r6, [sp] @ 4-byte Spill
+; CHECK-NEXT:    movvs r1, #0
+; CHECK-NEXT:    str r1, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s26, s20
+; CHECK-NEXT:    vcmp.f32 s26, s22
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt lt
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r6, #0
 ; CHECK-NEXT:    movtlt r6, #65534
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    vcmp.f32 s26, s22
+; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    vcvtt.f32.f16 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
@@ -4664,144 +4386,144 @@ define arm_aapcs_vfpcc <8 x i50> @test_signed_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    str.w r0, [r7, #25]
+; CHECK-NEXT:    str.w r0, [r4, #25]
 ; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r6, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s18, s20
+; CHECK-NEXT:    vcmp.f32 s18, s22
 ; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    mov r9, r1
-; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    ittt lt
-; CHECK-NEXT:    movlt.w r8, #0
 ; CHECK-NEXT:    movwlt r9, #0
 ; CHECK-NEXT:    movtlt r9, #65534
+; CHECK-NEXT:    movlt.w r8, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s18, s19
 ; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    movgt.w r8, #-1
 ; CHECK-NEXT:    movwgt r9, #65535
 ; CHECK-NEXT:    movtgt r9, #1
-; CHECK-NEXT:    movgt.w r8, #-1
 ; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs.w r8, #0
 ; CHECK-NEXT:    movvs.w r9, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s18, s20
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    ittt lt
-; CHECK-NEXT:    movlt.w r10, #0
-; CHECK-NEXT:    movwlt r11, #0
-; CHECK-NEXT:    movtlt r11, #65534
+; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movtlt r5, #65534
+; CHECK-NEXT:    movlt.w r11, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vcvtt.f32.f16 s18, s19
 ; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r11, #65535
-; CHECK-NEXT:    movtgt r11, #1
-; CHECK-NEXT:    movgt.w r10, #-1
+; CHECK-NEXT:    movgt.w r11, #-1
+; CHECK-NEXT:    movwgt r5, #65535
+; CHECK-NEXT:    movtgt r5, #1
 ; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt vs
-; CHECK-NEXT:    movvs.w r10, #0
 ; CHECK-NEXT:    movvs.w r11, #0
+; CHECK-NEXT:    movvs r5, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    vcvtb.f32.f16 s16, s16
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r10, r0
 ; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vcmp.f32 s18, s20
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    vcmp.f32 s18, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt lt
-; CHECK-NEXT:    movlt r4, #0
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    movtlt r5, #65534
-; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    movlt r7, #0
+; CHECK-NEXT:    movtlt r7, #65534
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r5, #65535
-; CHECK-NEXT:    movtgt r5, #1
-; CHECK-NEXT:    movgt.w r4, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
+; CHECK-NEXT:    movwgt r7, #65535
+; CHECK-NEXT:    movtgt r7, #1
 ; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt vs
-; CHECK-NEXT:    movvs r4, #0
-; CHECK-NEXT:    movvs r5, #0
+; CHECK-NEXT:    movvs.w r10, #0
+; CHECK-NEXT:    movvs r7, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s16, s20
-; CHECK-NEXT:    bfc r11, #18, #14
+; CHECK-NEXT:    vcmp.f32 s16, s22
+; CHECK-NEXT:    bfc r5, #18, #14
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt lt
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movtlt r1, #65534
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    vcmp.f32 s16, s22
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    vcmp.f32 s16, s20
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movwgt r1, #65535
 ; CHECK-NEXT:    movtgt r1, #1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    vcmp.f32 s16, s16
-; CHECK-NEXT:    lsrl r2, r11, #28
+; CHECK-NEXT:    lsrl r2, r5, #28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    str r0, [r7]
-; CHECK-NEXT:    lsrs r0, r5, #10
-; CHECK-NEXT:    bfc r5, #18, #14
+; CHECK-NEXT:    str r0, [r4]
+; CHECK-NEXT:    lsr.w r0, r7, #10
+; CHECK-NEXT:    bfc r7, #18, #14
 ; CHECK-NEXT:    bfc r9, #18, #14
-; CHECK-NEXT:    lsll r4, r5, #22
+; CHECK-NEXT:    lsll r10, r7, #22
 ; CHECK-NEXT:    bfc r6, #18, #14
-; CHECK-NEXT:    orr.w r3, r11, r5
-; CHECK-NEXT:    str.w r3, [r7, #45]
-; CHECK-NEXT:    orrs r2, r4
-; CHECK-NEXT:    str.w r2, [r7, #41]
-; CHECK-NEXT:    strb.w r0, [r7, #49]
+; CHECK-NEXT:    orr.w r3, r5, r7
+; CHECK-NEXT:    str.w r3, [r4, #45]
+; CHECK-NEXT:    orr.w r2, r2, r10
+; CHECK-NEXT:    str.w r2, [r4, #41]
+; CHECK-NEXT:    strb.w r0, [r4, #49]
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    lsrl r0, r9, #14
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    orr.w r2, r9, r10, lsl #4
-; CHECK-NEXT:    str.w r2, [r7, #37]
-; CHECK-NEXT:    str.w r0, [r7, #33]
+; CHECK-NEXT:    orr.w r2, r9, r11, lsl #4
+; CHECK-NEXT:    str.w r2, [r4, #37]
+; CHECK-NEXT:    str.w r0, [r4, #33]
 ; CHECK-NEXT:    orr.w r0, r6, r8, lsl #18
-; CHECK-NEXT:    str.w r0, [r7, #29]
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    str.w r0, [r4, #29]
 ; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
-; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    bfc r5, #18, #14
-; CHECK-NEXT:    lsr.w r0, r3, #10
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    lsr.w r5, r3, #10
 ; CHECK-NEXT:    bfc r3, #18, #14
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    lsll r6, r3, #22
-; CHECK-NEXT:    lsrl r2, r5, #28
-; CHECK-NEXT:    orr.w r3, r3, r5
-; CHECK-NEXT:    str r3, [r7, #20]
-; CHECK-NEXT:    orr.w r2, r2, r6
-; CHECK-NEXT:    str r2, [r7, #16]
-; CHECK-NEXT:    strb r0, [r7, #24]
+; CHECK-NEXT:    lsll r0, r3, #22
+; CHECK-NEXT:    mov r7, r3
+; CHECK-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    bfc r3, #18, #14
+; CHECK-NEXT:    lsrl r2, r3, #28
+; CHECK-NEXT:    orr.w r3, r3, r7
+; CHECK-NEXT:    str r3, [r4, #20]
+; CHECK-NEXT:    orr.w r2, r2, r0
+; CHECK-NEXT:    str r2, [r4, #16]
+; CHECK-NEXT:    strb r5, [r4, #24]
 ; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    bfc r3, #18, #14
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    lsrl r0, r3, #14
-; CHECK-NEXT:    orr.w r2, r3, r4, lsl #4
-; CHECK-NEXT:    strd r0, r2, [r7, #8]
+; CHECK-NEXT:    orr.w r2, r3, r6, lsl #4
+; CHECK-NEXT:    strd r0, r2, [r4, #8]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    bfc r1, #18, #14
-; CHECK-NEXT:    orr.w r0, r1, r6, lsl #18
-; CHECK-NEXT:    str r0, [r7, #4]
+; CHECK-NEXT:    orr.w r0, r1, r7, lsl #18
+; CHECK-NEXT:    str r0, [r4, #4]
 ; CHECK-NEXT:    add sp, #24
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 2
@@ -4830,37 +4552,37 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    vcvtb.f32.f16 s26, s19
 ; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    vmov r0, s26
-; CHECK-NEXT:    vldr s30, .LCPI49_0
-; CHECK-NEXT:    vldr s28, .LCPI49_1
+; CHECK-NEXT:    vldr s28, .LCPI49_0
+; CHECK-NEXT:    vldr s30, .LCPI49_1
 ; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    vcmp.f32 s24, s30
-; CHECK-NEXT:    vcvtt.f32.f16 s22, s18
+; CHECK-NEXT:    vcmp.f32 s24, s28
+; CHECK-NEXT:    vcvtt.f32.f16 s20, s16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r8, #-2147483648
 ; CHECK-NEXT:    movlt.w r9, #0
-; CHECK-NEXT:    vcmp.f32 s24, s28
-; CHECK-NEXT:    vcvtt.f32.f16 s20, s16
+; CHECK-NEXT:    vcmp.f32 s24, s30
+; CHECK-NEXT:    vcvtt.f32.f16 s22, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    mvngt r8, #-2147483648
 ; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vmov r4, s22
 ; CHECK-NEXT:    vmov r6, s20
+; CHECK-NEXT:    vmov r4, s22
 ; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs.w r8, #0
 ; CHECK-NEXT:    movvs.w r9, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    vcmp.f32 s26, s30
+; CHECK-NEXT:    vcmp.f32 s26, s28
 ; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r10, #0
 ; CHECK-NEXT:    movlt.w r11, #-2147483648
-; CHECK-NEXT:    vcmp.f32 s26, s28
+; CHECK-NEXT:    vcmp.f32 s26, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    itt gt
@@ -4873,13 +4595,13 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    movvs.w r11, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vcmp.f32 s22, s30
+; CHECK-NEXT:    vcmp.f32 s22, s28
 ; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r4, #-2147483648
 ; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    vcmp.f32 s22, s28
+; CHECK-NEXT:    vcmp.f32 s22, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    itt gt
@@ -4895,12 +4617,12 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov r0, s16
 ; CHECK-NEXT:    mov r6, r1
-; CHECK-NEXT:    vcmp.f32 s20, s30
+; CHECK-NEXT:    vcmp.f32 s20, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r6, #-2147483648
 ; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    vcmp.f32 s20, s28
+; CHECK-NEXT:    vcmp.f32 s20, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r7, #-1
@@ -4911,12 +4633,12 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    movvs r6, #0
 ; CHECK-NEXT:    movvs r7, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s16, s30
+; CHECK-NEXT:    vcmp.f32 s16, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt.w r1, #-2147483648
-; CHECK-NEXT:    vcmp.f32 s16, s28
+; CHECK-NEXT:    vcmp.f32 s16, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    itt gt
@@ -4932,11 +4654,11 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    vmov q5[3], q5[1], r1, r6
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s16, s30
+; CHECK-NEXT:    vcmp.f32 s16, s28
 ; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    mov r6, r1
-; CHECK-NEXT:    vcmp.f32 s16, s28
+; CHECK-NEXT:    vcmp.f32 s16, s30
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r6, #-2147483648
 ; CHECK-NEXT:    movlt r7, #0
@@ -4952,12 +4674,12 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    movvs r6, #0
 ; CHECK-NEXT:    movvs r7, #0
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s16, s30
+; CHECK-NEXT:    vcmp.f32 s16, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt.w r1, #-2147483648
-; CHECK-NEXT:    vcmp.f32 s16, s28
+; CHECK-NEXT:    vcmp.f32 s16, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    itt gt
@@ -4973,13 +4695,13 @@ define arm_aapcs_vfpcc <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    vmov q6[3], q6[1], r1, r6
 ; CHECK-NEXT:    bl __aeabi_f2lz
-; CHECK-NEXT:    vcmp.f32 s16, s30
+; CHECK-NEXT:    vcmp.f32 s16, s28
 ; CHECK-NEXT:    vmov q3[2], q3[0], r10, r9
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt.w r1, #-2147483648
-; CHECK-NEXT:    vcmp.f32 s16, s28
+; CHECK-NEXT:    vcmp.f32 s16, s30
 ; CHECK-NEXT:    vmov q3[3], q3[1], r11, r8
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
@@ -5016,77 +4738,109 @@ define arm_aapcs_vfpcc <8 x i100> @test_signed_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    .pad #56
-; CHECK-NEXT:    sub sp, #56
+; CHECK-NEXT:    .pad #48
+; CHECK-NEXT:    sub sp, #48
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vcvtb.f32.f16 s24, s17
 ; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcvtt.f32.f16 s26, s17
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s18
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    vmov r0, s26
 ; CHECK-NEXT:    vldr s22, .LCPI50_0
 ; CHECK-NEXT:    vldr s20, .LCPI50_1
+; CHECK-NEXT:    mov r9, r1
 ; CHECK-NEXT:    vcmp.f32 s24, s22
+; CHECK-NEXT:    mov r10, r2
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    mvnlt r3, #7
-; CHECK-NEXT:    movlt r2, #0
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movlt.w r8, #0
+; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    movlt.w r10, #0
 ; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    movgt.w r2, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
+; CHECK-NEXT:    movgt.w r9, #-1
+; CHECK-NEXT:    movgt.w r8, #-1
 ; CHECK-NEXT:    movgt r3, #7
 ; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    str r3, [sp, #52] @ 4-byte Spill
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str r2, [sp, #48] @ 4-byte Spill
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r7, #0
-; CHECK-NEXT:    str r7, [sp, #40] @ 4-byte Spill
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    movvs.w r8, #0
+; CHECK-NEXT:    movvs.w r9, #0
+; CHECK-NEXT:    movvs.w r10, #0
 ; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    vcvtb.f32.f16 s24, s19
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    vcmp.f32 s26, s22
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s18
+; CHECK-NEXT:    mov r7, r2
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    mvnlt r3, #7
+; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt r7, #0
+; CHECK-NEXT:    vcmp.f32 s26, s20
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itttt gt
+; CHECK-NEXT:    movgt.w r7, #-1
+; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt r3, #7
+; CHECK-NEXT:    vcmp.f32 s26, s26
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r3, #0
+; CHECK-NEXT:    str r3, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    movvs r5, #0
+; CHECK-NEXT:    movvs r6, #0
+; CHECK-NEXT:    movvs r7, #0
+; CHECK-NEXT:    bl __fixsfti
+; CHECK-NEXT:    vcmp.f32 s24, s22
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    vcmp.f32 s26, s20
+; CHECK-NEXT:    mvnlt r3, #7
+; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    itttt gt
+; CHECK-NEXT:    movgt r3, #7
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    movgt r3, #7
-; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str r2, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    str.w r2, [r4, #83]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    str r1, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    str.w r1, [r4, #79]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s16
+; CHECK-NEXT:    str.w r0, [r4, #75]
 ; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    str.w r7, [r4, #58]
+; CHECK-NEXT:    str.w r6, [r4, #54]
+; CHECK-NEXT:    str.w r5, [r4, #50]
+; CHECK-NEXT:    str.w r10, [r4, #33]
+; CHECK-NEXT:    str.w r9, [r4, #29]
+; CHECK-NEXT:    str.w r8, [r4, #25]
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r3, #0
+; CHECK-NEXT:    str r3, [sp, #28] @ 4-byte Spill
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s24, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
@@ -5106,155 +4860,119 @@ define arm_aapcs_vfpcc <8 x i100> @test_signed_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    vcvtb.f32.f16 s24, s17
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s17
+; CHECK-NEXT:    str r0, [sp, #40] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #44] @ 4-byte Spill
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s18
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    mov r8, r1
 ; CHECK-NEXT:    vcmp.f32 s24, s22
-; CHECK-NEXT:    mov r6, r2
+; CHECK-NEXT:    vcvtt.f32.f16 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    mvnlt r3, #7
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    movlt.w r8, #0
-; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movgt.w r8, #-1
-; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    movgt.w r1, #-1
+; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt r3, #7
 ; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    ittt vs
-; CHECK-NEXT:    movvs r5, #0
-; CHECK-NEXT:    movvs.w r8, #0
-; CHECK-NEXT:    movvs r6, #0
+; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r2, #0
+; CHECK-NEXT:    str r2, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r1, #0
+; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    it vs
+; CHECK-NEXT:    movvs r0, #0
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s18, s22
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    mov r11, r1
-; CHECK-NEXT:    mov r4, r2
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    mov r8, r2
 ; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    mvnlt r3, #7
+; CHECK-NEXT:    movlt.w r8, #0
+; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt.w r11, #0
-; CHECK-NEXT:    movlt r4, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s18, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s19
+; CHECK-NEXT:    vcvtt.f32.f16 s18, s19
 ; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r4, #-1
-; CHECK-NEXT:    movgt.w r11, #-1
 ; CHECK-NEXT:    movgt.w r7, #-1
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r8, #-1
 ; CHECK-NEXT:    movgt r3, #7
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    movvs.w r8, #0
+; CHECK-NEXT:    movvs r5, #0
 ; CHECK-NEXT:    movvs r7, #0
-; CHECK-NEXT:    movvs.w r11, #0
-; CHECK-NEXT:    movvs r4, #0
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s18, s22
-; CHECK-NEXT:    mov r9, r3
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
-; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r2, #0
-; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    mvnlt r9, #7
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s18
-; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r9, #7
-; CHECK-NEXT:    movgt.w r0, #-1
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str.w r2, [r10, #83]
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    str.w r1, [r10, #79]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    vcvtt.f32.f16 s18, s19
-; CHECK-NEXT:    str.w r0, [r10, #75]
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    str.w r4, [r10, #58]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    str.w r11, [r10, #54]
-; CHECK-NEXT:    str.w r7, [r10, #50]
-; CHECK-NEXT:    str.w r6, [r10, #33]
-; CHECK-NEXT:    str.w r8, [r10, #29]
-; CHECK-NEXT:    str.w r5, [r10, #25]
-; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs.w r9, #0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcvtb.f32.f16 s16, s16
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    mov r9, r1
 ; CHECK-NEXT:    vcmp.f32 s18, s22
-; CHECK-NEXT:    mov r6, r2
-; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    mov r10, r2
+; CHECK-NEXT:    mov r11, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    mvnlt r4, #7
+; CHECK-NEXT:    mvnlt r11, #7
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt.w r9, #0
 ; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r5, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
 ; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movgt r4, #7
+; CHECK-NEXT:    movgt.w r9, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
+; CHECK-NEXT:    movgt.w r11, #7
 ; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt vs
-; CHECK-NEXT:    movvs r4, #0
+; CHECK-NEXT:    movvs.w r11, #0
+; CHECK-NEXT:    movvs.w r10, #0
+; CHECK-NEXT:    movvs.w r9, #0
 ; CHECK-NEXT:    movvs r6, #0
-; CHECK-NEXT:    movvs r7, #0
-; CHECK-NEXT:    movvs r5, #0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    vcmp.f32 s16, s22
+; CHECK-NEXT:    mov r12, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    mvnlt r3, #7
+; CHECK-NEXT:    mvnlt r12, #7
 ; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt r3, #7
+; CHECK-NEXT:    movgt.w r12, #7
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
@@ -5262,74 +4980,73 @@ define arm_aapcs_vfpcc <8 x i100> @test_signed_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    str.w r2, [r10, #8]
+; CHECK-NEXT:    str r2, [r4, #8]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    str.w r1, [r10, #4]
+; CHECK-NEXT:    str r1, [r4, #4]
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
-; CHECK-NEXT:    str.w r0, [r10]
+; CHECK-NEXT:    str r0, [r4]
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    lsrl r0, r9, #28
+; CHECK-NEXT:    str.w r0, [r4, #91]
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    lsrl r0, r7, #28
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    orr.w r1, r7, r6, lsl #4
-; CHECK-NEXT:    str.w r1, [r10, #95]
-; CHECK-NEXT:    and r1, r4, #15
-; CHECK-NEXT:    str.w r0, [r10, #91]
-; CHECK-NEXT:    and r0, r9, #15
-; CHECK-NEXT:    lsrl r6, r1, #28
-; CHECK-NEXT:    strb.w r6, [r10, #99]
-; CHECK-NEXT:    orr.w r0, r0, r5, lsl #4
-; CHECK-NEXT:    str.w r0, [r10, #87]
-; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    str.w r0, [r4, #66]
+; CHECK-NEXT:    ldr.w lr, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, lr
+; CHECK-NEXT:    lsrl r0, r3, #28
+; CHECK-NEXT:    str.w r0, [r4, #41]
+; CHECK-NEXT:    ldrd r0, r1, [sp, #40] @ 8-byte Folded Reload
 ; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    str.w r1, [r10, #70]
-; CHECK-NEXT:    str.w r0, [r10, #66]
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    and r1, r0, #15
-; CHECK-NEXT:    lsrl r2, r1, #28
-; CHECK-NEXT:    strb.w r2, [r10, #74]
+; CHECK-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; CHECK-NEXT:    and r1, r11, #15
+; CHECK-NEXT:    str r0, [r4, #16]
+; CHECK-NEXT:    orr.w r0, r9, r10, lsl #4
+; CHECK-NEXT:    lsrl r10, r1, #28
+; CHECK-NEXT:    str.w r0, [r4, #95]
+; CHECK-NEXT:    strb.w r10, [r4, #99]
+; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r0, #15
+; CHECK-NEXT:    orr.w r0, r0, r6, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #87]
+; CHECK-NEXT:    orr.w r0, r7, r8, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #70]
 ; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    and r1, r0, #15
+; CHECK-NEXT:    lsrl r8, r1, #28
+; CHECK-NEXT:    strb.w r8, [r4, #74]
+; CHECK-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
 ; CHECK-NEXT:    and r0, r0, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
-; CHECK-NEXT:    str.w r0, [r10, #62]
-; CHECK-NEXT:    ldr r7, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    str.w r1, [r10, #45]
-; CHECK-NEXT:    str.w r0, [r10, #41]
-; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r0, r5, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #62]
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r3, r2, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #45]
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r2, r1, #28
-; CHECK-NEXT:    strb.w r2, [r10, #49]
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    strb.w r2, [r4, #49]
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
 ; CHECK-NEXT:    and r0, r0, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
-; CHECK-NEXT:    str.w r0, [r10, #37]
-; CHECK-NEXT:    ldr r7, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #48] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    strd r0, r1, [r10, #16]
-; CHECK-NEXT:    ldr r0, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r0, lr, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #37]
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #4
+; CHECK-NEXT:    str r0, [r4, #20]
+; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r2, r1, #28
-; CHECK-NEXT:    strb.w r2, [r10, #24]
+; CHECK-NEXT:    strb r2, [r4, #24]
 ; CHECK-NEXT:    it vs
-; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    and r0, r3, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
-; CHECK-NEXT:    str.w r0, [r10, #12]
-; CHECK-NEXT:    add sp, #56
+; CHECK-NEXT:    movvs.w r12, #0
+; CHECK-NEXT:    ldr r1, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r12, #15
+; CHECK-NEXT:    orr.w r0, r0, r1, lsl #4
+; CHECK-NEXT:    str r0, [r4, #12]
+; CHECK-NEXT:    add sp, #48
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -5346,62 +5063,63 @@ define arm_aapcs_vfpcc <8 x i100> @test_signed_v8f16_v8i100(<8 x half> %f) {
 define arm_aapcs_vfpcc <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-LABEL: test_signed_v8f16_v8i128:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vcvtt.f32.f16 s26, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s28, s19
-; CHECK-NEXT:    vmov r0, s26
-; CHECK-NEXT:    vcvtb.f32.f16 s24, s17
-; CHECK-NEXT:    vldr s20, .LCPI51_0
-; CHECK-NEXT:    vmov r5, s28
-; CHECK-NEXT:    vmov r8, s24
-; CHECK-NEXT:    vcvtt.f32.f16 s30, s18
+; CHECK-NEXT:    vcvtt.f32.f16 s30, s19
+; CHECK-NEXT:    vcvtb.f32.f16 s20, s16
+; CHECK-NEXT:    vmov r0, s30
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s19
+; CHECK-NEXT:    vldr s22, .LCPI51_0
+; CHECK-NEXT:    vmov r5, s20
+; CHECK-NEXT:    vmov r7, s26
+; CHECK-NEXT:    vcvtt.f32.f16 s28, s18
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vldr s22, .LCPI51_1
+; CHECK-NEXT:    vldr s24, .LCPI51_1
 ; CHECK-NEXT:    add.w r12, r4, #112
-; CHECK-NEXT:    vmov r6, s30
-; CHECK-NEXT:    vcmp.f32 s26, s22
+; CHECK-NEXT:    vmov r6, s28
+; CHECK-NEXT:    vcvtb.f32.f16 s18, s18
+; CHECK-NEXT:    vcmp.f32 s30, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s30, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s30, s30
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vcvtb.f32.f16 s26, s18
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s28, s22
+; CHECK-NEXT:    vcmp.f32 s26, s24
 ; CHECK-NEXT:    add.w r12, r4, #96
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s26, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s28
+; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
@@ -5410,27 +5128,26 @@ define arm_aapcs_vfpcc <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    vmov r7, s26
-; CHECK-NEXT:    vcvtt.f32.f16 s28, s17
+; CHECK-NEXT:    vmov r7, s18
+; CHECK-NEXT:    vcvtt.f32.f16 s26, s17
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s30, s22
+; CHECK-NEXT:    vcmp.f32 s28, s24
 ; CHECK-NEXT:    add.w r12, r4, #80
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s30, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s28, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s30, s30
+; CHECK-NEXT:    vcmp.f32 s28, s28
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
@@ -5439,155 +5156,145 @@ define arm_aapcs_vfpcc <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    vmov r5, s28
-; CHECK-NEXT:    vcvtt.f32.f16 s18, s16
+; CHECK-NEXT:    vmov r6, s26
+; CHECK-NEXT:    vcvtb.f32.f16 s28, s17
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s26, s22
+; CHECK-NEXT:    vcmp.f32 s18, s24
 ; CHECK-NEXT:    add.w r12, r4, #64
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    vcvtt.f32.f16 s16, s16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    it vs
+; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s16
-; CHECK-NEXT:    vmov r6, s18
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    vmov r7, s28
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s28, s22
+; CHECK-NEXT:    vcmp.f32 s26, s24
 ; CHECK-NEXT:    add.w r12, r4, #48
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s26, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s28
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s26, s26
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r3, #0
-; CHECK-NEXT:    it vs
+; CHECK-NEXT:    ittt vs
 ; CHECK-NEXT:    movvs r2, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt vs
 ; CHECK-NEXT:    movvs r1, #0
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    vmov r7, s16
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    vmov r6, s16
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s24, s22
+; CHECK-NEXT:    vcmp.f32 s28, s24
 ; CHECK-NEXT:    add.w r12, r4, #32
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s28, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s24, s24
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s28, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s18, s22
+; CHECK-NEXT:    vcmp.f32 s16, s24
 ; CHECK-NEXT:    add.w r12, r4, #16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s16, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s18
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    vcmp.f32 s16, s22
+; CHECK-NEXT:    vcmp.f32 s20, s24
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r3, #-2147483648
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    vcmp.f32 s20, s22
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s16
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    mvngt r3, #-2147483648
+; CHECK-NEXT:    vcmp.f32 s20, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt vs
+; CHECK-NEXT:    itttt vs
 ; CHECK-NEXT:    movvs r3, #0
 ; CHECK-NEXT:    movvs r2, #0
 ; CHECK-NEXT:    movvs r1, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it vs
 ; CHECK-NEXT:    movvs r0, #0
 ; CHECK-NEXT:    stm r4!, {r0, r1, r2, r3}
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI51_0:
diff --git a/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll b/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
index 13609bd1903f2..ee040feca4240 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
@@ -268,41 +268,38 @@ declare <6 x i32> @llvm.fptoui.sat.v6f64.v6i32 (<6 x double>)
 define arm_aapcs_vfpcc <1 x i32> @test_unsigned_v1f64_v1i32(<1 x double> %f) {
 ; CHECK-LABEL: test_unsigned_v1f64_v1i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    vldr d1, .LCPI8_0
 ; CHECK-NEXT:    vmov r4, r5, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
 ; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI8_1
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2uiz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
+; CHECK-NEXT:    vldr d0, .LCPI8_1
 ; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI8_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI8_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI8_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <1 x i32> @llvm.fptoui.sat.v1f64.v1i32(<1 x double> %f)
     ret <1 x i32> %x
 }
@@ -316,84 +313,60 @@ define arm_aapcs_vfpcc <2 x i32> @test_unsigned_v2f64_v2i32(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI9_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI9_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -418,99 +391,93 @@ define arm_aapcs_vfpcc <3 x i32> @test_unsigned_v3f64_v3i32(<3 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov.f32 s18, s0
 ; CHECK-NEXT:    vmov.f32 s19, s1
 ; CHECK-NEXT:    vldr d0, .LCPI10_0
-; CHECK-NEXT:    vmov r4, r5, d1
-; CHECK-NEXT:    vmov r9, r7, d0
+; CHECK-NEXT:    vmov r8, r9, d1
+; CHECK-NEXT:    vmov r5, r4, d0
 ; CHECK-NEXT:    vmov.f32 s16, s4
 ; CHECK-NEXT:    vmov.f32 s17, s5
-; CHECK-NEXT:    str.w r9, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI10_1
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    vmov r11, r3, d0
-; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    str r4, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r9
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r10, r8, d8
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r6, ne
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    vldr d0, .LCPI10_1
+; CHECK-NEXT:    vmov r11, r1, d9
+; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    vmov r7, r6, d8
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    csel r10, r0, r10, ne
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r9, r2
+; CHECK-NEXT:    mov r8, r3
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    csel r6, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    csel r5, r0, r4, ne
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    ldrd r2, r3, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    vmov.32 q0[1], r10
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov.32 q0[1], r1
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI10_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI10_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI10_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <3 x i32> @llvm.fptoui.sat.v3f64.v3i32(<3 x double> %f)
     ret <3 x i32> %x
 }
@@ -529,103 +496,103 @@ define arm_aapcs_vfpcc <4 x i32> @test_unsigned_v4f64_v4i32(<4 x double> %f) {
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI11_0
 ; CHECK-NEXT:    vmov q5, q1
-; CHECK-NEXT:    vmov r7, r9, d0
-; CHECK-NEXT:    vmov r4, r5, d10
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    vmov r8, r9, d10
+; CHECK-NEXT:    vmov r2, r11, d0
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI11_1
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    vmov r5, r1, d11
+; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    vmov r6, r7, d8
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r2, r3, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    csel r4, r0, r10, ne
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    strd r5, r1, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r2
+; CHECK-NEXT:    mov r9, r3
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    mov r5, r7
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    mov r10, r11
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r10, r8, d8
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r6, ne
-; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vmov r11, r5, d11
-; CHECK-NEXT:    mov r4, r7
-; CHECK-NEXT:    str r7, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r8, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    csel r8, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r4
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #-1
-; CHECK-NEXT:    ldr.w r10, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldr.w r11, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    ldr.w r9, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r1, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r9
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r4, r5, d9
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r6, r0, r7, ne
-; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    ldr.w r9, [sp] @ 4-byte Reload
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r5, r0, r4, ne
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    vmov r7, r6, d9
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r2, r3, [sp, #16] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r4
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r8, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r6
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldrd r1, r0, [sp, #12] @ 8-byte Folded Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r1, r0
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r5
 ; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    add sp, #4
@@ -633,11 +600,11 @@ define arm_aapcs_vfpcc <4 x i32> @test_unsigned_v4f64_v4i32(<4 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI11_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI11_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI11_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <4 x i32> @llvm.fptoui.sat.v4f64.v4i32(<4 x double> %f)
     ret <4 x i32> %x
 }
@@ -651,162 +618,151 @@ define arm_aapcs_vfpcc <5 x i32> @test_unsigned_v5f64_v5i32(<5 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov.f32 s17, s1
 ; CHECK-NEXT:    vldr d0, .LCPI12_0
-; CHECK-NEXT:    vmov r5, r6, d4
-; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    vmov r6, r11, d4
+; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    vmov.f32 s20, s6
-; CHECK-NEXT:    vmov.f32 s18, s4
+; CHECK-NEXT:    vmov.f32 s18, s6
+; CHECK-NEXT:    vmov.f32 s20, s4
 ; CHECK-NEXT:    vmov.f32 s22, s2
-; CHECK-NEXT:    vmov.f32 s21, s7
-; CHECK-NEXT:    vmov.f32 s19, s5
+; CHECK-NEXT:    vmov.f32 s19, s7
+; CHECK-NEXT:    vmov.f32 s21, s5
 ; CHECK-NEXT:    vmov.f32 s23, s3
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    strd r2, r3, [sp, #32] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI12_1
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    vmov r7, r3, d0
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str r7, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vmov r8, r1, d11
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    vmov r6, r9, d10
-; CHECK-NEXT:    csel r0, r0, r11, ne
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vmov r2, r1, d9
-; CHECK-NEXT:    strd r2, r1, [sp, #16] @ 8-byte Folded Spill
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r4, #16]
+; CHECK-NEXT:    vldr d0, .LCPI12_1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov r10, r9, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    csel r4, r0, r4, ne
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr r5, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    ldr.w r10, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    vmov r5, r1, d10
+; CHECK-NEXT:    strd r5, r1, [sp, #12] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r11, r2
+; CHECK-NEXT:    mov r5, r3
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r7, #16]
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r0, r10
 ; CHECK-NEXT:    mov r1, r9
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    csel r0, r0, r4, ne
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r11, r10
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    mov r10, r11
+; CHECK-NEXT:    mov r11, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r10, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    mov r5, r6
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r9, r7
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    csel r0, r0, r6, ne
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r8, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r9, r0, r4, ne
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    mov r11, r10
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r8, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    ldr.w r11, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    csel r4, r0, r5, ne
-; CHECK-NEXT:    vmov r5, r6, d8
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    ldr r3, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r7, r0, r4, ne
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    vmov r4, r5, d8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r7
+; CHECK-NEXT:    vmov q0[3], q0[1], r9, r0
 ; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI12_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI12_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI12_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <5 x i32> @llvm.fptoui.sat.v5f64.v5i32(<5 x double> %f)
     ret <5 x i32> %x
 }
@@ -823,172 +779,161 @@ define arm_aapcs_vfpcc <6 x i32> @test_unsigned_v6f64_v6i32(<6 x double> %f) {
 ; CHECK-NEXT:    .pad #40
 ; CHECK-NEXT:    sub sp, #40
 ; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    str r0, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    vmov.f32 s17, s1
 ; CHECK-NEXT:    vldr d0, .LCPI13_0
 ; CHECK-NEXT:    vmov r5, r6, d5
-; CHECK-NEXT:    vmov r11, r3, d0
-; CHECK-NEXT:    vmov.f32 s22, s8
-; CHECK-NEXT:    vmov.f32 s20, s6
+; CHECK-NEXT:    vmov r10, r3, d0
+; CHECK-NEXT:    vmov.f32 s20, s8
+; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s18, s4
 ; CHECK-NEXT:    vmov.f32 s24, s2
-; CHECK-NEXT:    vmov.f32 s23, s9
-; CHECK-NEXT:    vmov.f32 s21, s7
+; CHECK-NEXT:    vmov.f32 s21, s9
+; CHECK-NEXT:    vmov.f32 s23, s7
 ; CHECK-NEXT:    vmov.f32 s19, s5
 ; CHECK-NEXT:    vmov.f32 s25, s3
-; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    str.w r11, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI13_1
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    vmov r4, r9, d0
-; CHECK-NEXT:    str r4, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r10, r1, d10
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    vmov r5, r6, d11
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    vmov r2, r1, d12
-; CHECK-NEXT:    strd r2, r1, [sp, #12] @ 8-byte Folded Spill
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r7, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    str r0, [r7, #20]
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    ldr.w r8, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    vmov r9, r1, d11
+; CHECK-NEXT:    vldr d0, .LCPI13_1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov r8, r11, d10
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    csel r4, r0, r4, ne
 ; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    str r1, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    vmov r7, r1, d12
+; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r5, r2
+; CHECK-NEXT:    strd r7, r1, [sp, #24] @ 8-byte Folded Spill
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldr r7, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    str r4, [r7, #20]
+; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vmov r2, r1, d9
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r0, r4, ne
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    strd r2, r1, [sp, #4] @ 8-byte Folded Spill
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r7, #16]
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r5, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    mov r8, r9
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r8, r5
+; CHECK-NEXT:    strd r2, r1, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    ldr.w r11, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r7, #16]
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    ldr r5, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    csel r0, r0, r7, ne
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr r4, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r9, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r4, r0, r4, ne
 ; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    str.w r8, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldr r5, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    ldr r7, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    str r4, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r6, r7
-; CHECK-NEXT:    mov r10, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    csel r9, r0, r7, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    csel r9, r0, r4, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r9, #-1
-; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    ldr r5, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r5, r6, d8
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    ldr r3, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r8, r0, r4, ne
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    vmov r4, r5, d8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #-1
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[3], q0[1], r9, r0
+; CHECK-NEXT:    movne.w r6, #-1
 ; CHECK-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r8
+; CHECK-NEXT:    vmov q0[3], q0[1], r9, r0
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #40
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12}
@@ -997,11 +942,11 @@ define arm_aapcs_vfpcc <6 x i32> @test_unsigned_v6f64_v6i32(<6 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI13_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI13_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI13_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <6 x i32> @llvm.fptoui.sat.v6f64.v6i32(<6 x double> %f)
     ret <6 x i32> %x
 }
@@ -1480,65 +1425,66 @@ define arm_aapcs_vfpcc <4 x i50> @test_unsigned_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10}
 ; CHECK-NEXT:    vpush {d8, d9, d10}
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    bl __aeabi_f2ulz
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    vmov r0, s19
-; CHECK-NEXT:    vldr s20, .LCPI28_0
-; CHECK-NEXT:    vcmp.f32 s18, #0
-; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    vcmp.f32 s16, #0
+; CHECK-NEXT:    mov r9, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vmov r7, s18
+; CHECK-NEXT:    vldr s20, .LCPI28_0
 ; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt.w r8, #0
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    vcmp.f32 s18, s20
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vmov r5, s16
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r7, #65535
-; CHECK-NEXT:    movtgt r7, #3
-; CHECK-NEXT:    movgt.w r8, #-1
+; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vcmp.f32 s19, #0
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r10, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r10, #0
 ; CHECK-NEXT:    movlt r4, #0
-; CHECK-NEXT:    vcmp.f32 s19, s20
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movwgt r10, #65535
-; CHECK-NEXT:    movtgt r10, #3
-; CHECK-NEXT:    movgt.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    vcmp.f32 s16, #0
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vcmp.f32 s18, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    vcmp.f32 s16, s20
-; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f32 s19, s20
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r0, #-1
-; CHECK-NEXT:    str.w r0, [r9]
-; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    movgt.w r7, #-1
+; CHECK-NEXT:    itt gt
+; CHECK-NEXT:    movwgt r1, #65535
+; CHECK-NEXT:    movtgt r1, #3
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r4, #-1
+; CHECK-NEXT:    itt gt
+; CHECK-NEXT:    movwgt r10, #65535
+; CHECK-NEXT:    movtgt r10, #3
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    bfc r1, #18, #14
-; CHECK-NEXT:    bfc r7, #18, #14
-; CHECK-NEXT:    mov r6, r8
-; CHECK-NEXT:    lsll r4, r1, #22
-; CHECK-NEXT:    lsrl r6, r7, #28
+; CHECK-NEXT:    bfc r3, #18, #14
+; CHECK-NEXT:    mov r6, r7
+; CHECK-NEXT:    vcmp.f32 s16, s20
+; CHECK-NEXT:    lsll r4, r3, #22
+; CHECK-NEXT:    lsrl r6, r1, #28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    str.w r5, [r8]
 ; CHECK-NEXT:    itt gt
-; CHECK-NEXT:    movwgt r5, #65535
-; CHECK-NEXT:    movtgt r5, #3
-; CHECK-NEXT:    orrs r1, r7
-; CHECK-NEXT:    str.w r1, [r9, #20]
+; CHECK-NEXT:    movwgt r9, #65535
+; CHECK-NEXT:    movtgt r9, #3
+; CHECK-NEXT:    orrs r1, r3
+; CHECK-NEXT:    str.w r1, [r8, #20]
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s17, #0
 ; CHECK-NEXT:    orr.w r2, r6, r4
@@ -1547,24 +1493,23 @@ define arm_aapcs_vfpcc <4 x i50> @test_unsigned_v4f32_v4i50(<4 x float> %f) {
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    vcmp.f32 s17, s20
-; CHECK-NEXT:    bfc r5, #18, #14
+; CHECK-NEXT:    bfc r9, #18, #14
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r1, #65535
 ; CHECK-NEXT:    movtgt r1, #3
-; CHECK-NEXT:    str.w r2, [r9, #16]
+; CHECK-NEXT:    str.w r2, [r8, #16]
 ; CHECK-NEXT:    lsr.w r2, r10, #10
-; CHECK-NEXT:    strb.w r2, [r9, #24]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    strb.w r2, [r8, #24]
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    bfc r1, #18, #14
-; CHECK-NEXT:    orr.w r0, r5, r0, lsl #18
 ; CHECK-NEXT:    lsrl r2, r1, #14
-; CHECK-NEXT:    orr.w r1, r1, r8, lsl #4
-; CHECK-NEXT:    strd r2, r1, [r9, #8]
-; CHECK-NEXT:    str.w r0, [r9, #4]
+; CHECK-NEXT:    orr.w r0, r9, r0, lsl #18
+; CHECK-NEXT:    orr.w r1, r1, r7, lsl #4
+; CHECK-NEXT:    strd r2, r1, [r8, #8]
+; CHECK-NEXT:    str.w r0, [r8, #4]
 ; CHECK-NEXT:    vpop {d8, d9, d10}
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 ; CHECK-NEXT:    .p2align 2
@@ -1670,36 +1615,14 @@ define arm_aapcs_vfpcc <4 x i100> @test_unsigned_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    vldr s20, .LCPI30_0
-; CHECK-NEXT:    vcmp.f32 s17, #0
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    mov r6, r2
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movlt r3, #0
-; CHECK-NEXT:    vcmp.f32 s17, s20
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #15
-; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    movgt.w r5, #-1
-; CHECK-NEXT:    str r5, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    vmov r6, s17
+; CHECK-NEXT:    vldr s20, .LCPI30_0
 ; CHECK-NEXT:    vcmp.f32 s18, #0
-; CHECK-NEXT:    mov r10, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt r3, #0
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
@@ -1711,16 +1634,37 @@ define arm_aapcs_vfpcc <4 x i100> @test_unsigned_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    str.w r1, [r4, #29]
-; CHECK-NEXT:    vmov r1, s19
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    str.w r0, [r4, #25]
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r10, #15
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    movgt r3, #15
+; CHECK-NEXT:    vmov r7, s19
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    vcmp.f32 s17, #0
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    itttt lt
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s17, s20
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt r3, #15
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov r0, s16
 ; CHECK-NEXT:    vcmp.f32 s19, #0
 ; CHECK-NEXT:    mov r9, r1
@@ -1729,7 +1673,7 @@ define arm_aapcs_vfpcc <4 x i100> @test_unsigned_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt.w r9, #0
-; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    movlt.w r8, #0
 ; CHECK-NEXT:    movlt.w r11, #0
 ; CHECK-NEXT:    vcmp.f32 s19, s20
@@ -1737,7 +1681,7 @@ define arm_aapcs_vfpcc <4 x i100> @test_unsigned_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r11, #15
 ; CHECK-NEXT:    movgt.w r8, #-1
-; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r7, #-1
 ; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s16, #0
@@ -1758,31 +1702,31 @@ define arm_aapcs_vfpcc <4 x i100> @test_unsigned_v4f32_v4i100(<4 x float> %f) {
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    str r0, [r4]
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    lsrl r0, r9, #28
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    orr.w r1, r9, r8, lsl #4
-; CHECK-NEXT:    str.w r1, [r4, #45]
 ; CHECK-NEXT:    and r1, r11, #15
 ; CHECK-NEXT:    str.w r0, [r4, #41]
-; CHECK-NEXT:    and r0, r10, #15
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    lsrl r0, r5, #28
+; CHECK-NEXT:    str r0, [r4, #16]
+; CHECK-NEXT:    orr.w r0, r9, r8, lsl #4
 ; CHECK-NEXT:    lsrl r8, r1, #28
+; CHECK-NEXT:    str.w r0, [r4, #45]
 ; CHECK-NEXT:    strb.w r8, [r4, #49]
-; CHECK-NEXT:    orr.w r0, r0, r5, lsl #4
-; CHECK-NEXT:    str.w r0, [r4, #37]
-; CHECK-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    lsrl r0, r7, #28
-; CHECK-NEXT:    orr.w r1, r7, r6, lsl #4
-; CHECK-NEXT:    strd r0, r1, [r4, #16]
 ; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r0, #15
+; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #37]
+; CHECK-NEXT:    orr.w r0, r5, r6, lsl #4
+; CHECK-NEXT:    str r0, [r4, #20]
+; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r6, r1, #28
 ; CHECK-NEXT:    strb r6, [r4, #24]
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
 ; CHECK-NEXT:    and r0, r3, #15
-; CHECK-NEXT:    orr.w r0, r0, r2, lsl #4
+; CHECK-NEXT:    orr.w r0, r0, r10, lsl #4
 ; CHECK-NEXT:    str r0, [r4, #12]
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9, d10}
@@ -1809,13 +1753,13 @@ define arm_aapcs_vfpcc <4 x i128> @test_unsigned_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov r5, s18
 ; CHECK-NEXT:    vldr s20, .LCPI31_0
 ; CHECK-NEXT:    vcmp.f32 s19, #0
+; CHECK-NEXT:    add.w r12, r4, #48
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
@@ -1823,32 +1767,29 @@ define arm_aapcs_vfpcc <4 x i128> @test_unsigned_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    movgt.w r5, #-1
-; CHECK-NEXT:    strd r5, r1, [r4, #48]
-; CHECK-NEXT:    vmov r6, s17
+; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    vmov r7, s16
-; CHECK-NEXT:    strd r2, r3, [r4, #56]
+; CHECK-NEXT:    vmov r6, s17
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s18, #0
 ; CHECK-NEXT:    add.w r12, r4, #32
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r6
@@ -1856,38 +1797,34 @@ define arm_aapcs_vfpcc <4 x i128> @test_unsigned_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-NEXT:    vcmp.f32 s17, #0
 ; CHECK-NEXT:    add.w r12, r4, #16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s17, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s16, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm r4!, {r0, r1, r2, r3}
 ; CHECK-NEXT:    vpop {d8, d9, d10}
@@ -1928,57 +1865,55 @@ define arm_aapcs_vfpcc <2 x i1> @test_unsigned_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI32_0
-; CHECK-NEXT:    vmov r5, r6, d8
+; CHECK-NEXT:    vmov r4, r8, d8
 ; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    vmov r10, r3, d0
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI32_1
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    vmov r4, r11, d0
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2uiz
+; CHECK-NEXT:    vldr d0, .LCPI32_1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    csel r7, r0, r9, ne
+; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    vmov r6, r5, d9
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    movs r7, #0
-; CHECK-NEXT:    and r0, r0, #1
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    movne r7, #1
+; CHECK-NEXT:    and r0, r7, #1
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    movs r7, #0
 ; CHECK-NEXT:    bfi r7, r0, #0, #1
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_d2uiz
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r0, r4, ne
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    movne r4, #1
+; CHECK-NEXT:    and r0, r4, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r7, r0, #1, #1
 ; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
@@ -1990,11 +1925,11 @@ define arm_aapcs_vfpcc <2 x i1> @test_unsigned_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI32_0:
-; CHECK-NEXT:    .long 0 @ double 1
-; CHECK-NEXT:    .long 1072693248
-; CHECK-NEXT:  .LCPI32_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI32_1:
+; CHECK-NEXT:    .long 0 @ double 1
+; CHECK-NEXT:    .long 1072693248
     %x = call <2 x i1> @llvm.fptoui.sat.v2f64.v2i1(<2 x double> %f)
     ret <2 x i1> %x
 }
@@ -2008,84 +1943,60 @@ define arm_aapcs_vfpcc <2 x i8> @test_unsigned_v2f64_v2i8(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI33_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI33_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #255
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    movne.w r9, #255
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #255
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r6, #255
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2110,84 +2021,60 @@ define arm_aapcs_vfpcc <2 x i13> @test_unsigned_v2f64_v2i13(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI34_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI34_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r5, #8191
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    movwne r9, #8191
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #8191
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r6, #8191
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2212,84 +2099,60 @@ define arm_aapcs_vfpcc <2 x i16> @test_unsigned_v2f64_v2i16(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI35_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI35_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r5, #65535
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    movwne r9, #65535
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r6, #65535
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2318,79 +2181,61 @@ define arm_aapcs_vfpcc <2 x i19> @test_unsigned_v2f64_v2i19(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI36_0
-; CHECK-NEXT:    vmov r11, r10, d8
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI36_1
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov r11, r5, d8
+; CHECK-NEXT:    vmov r6, r7, d0
+; CHECK-NEXT:    str r5, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    vmov r5, r7, d0
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    vmov r8, r6, d9
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    vldr d0, .LCPI36_1
+; CHECK-NEXT:    vmov r5, r8, d9
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov r3, r2, d0
+; CHECK-NEXT:    csel r9, r1, r4, ne
+; CHECK-NEXT:    csel r10, r0, r4, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    csel r6, r0, r4, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    csel r4, r1, r4, ne
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    movwne r6, #65535
+; CHECK-NEXT:    movtne r6, #7
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r5, #65535
-; CHECK-NEXT:    movtne r5, #7
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    movtne r0, #7
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movwne r10, #65535
+; CHECK-NEXT:    movtne r10, #7
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r10, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r9, r4
 ; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -2398,11 +2243,11 @@ define arm_aapcs_vfpcc <2 x i19> @test_unsigned_v2f64_v2i19(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI36_0:
-; CHECK-NEXT:    .long 0 @ double 524287
-; CHECK-NEXT:    .long 1092616188
-; CHECK-NEXT:  .LCPI36_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI36_1:
+; CHECK-NEXT:    .long 0 @ double 524287
+; CHECK-NEXT:    .long 1092616188
     %x = call <2 x i19> @llvm.fptoui.sat.v2f64.v2i19(<2 x double> %f)
     ret <2 x i19> %x
 }
@@ -2416,84 +2261,60 @@ define arm_aapcs_vfpcc <2 x i32> @test_unsigned_v2f64_v2i32_duplicate(<2 x doubl
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI37_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI37_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #0
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2522,79 +2343,61 @@ define arm_aapcs_vfpcc <2 x i50> @test_unsigned_v2f64_v2i50(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI38_0
-; CHECK-NEXT:    vmov r11, r10, d8
-; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI38_1
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov r11, r5, d8
+; CHECK-NEXT:    vmov r6, r7, d0
+; CHECK-NEXT:    str r5, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    vmov r5, r7, d0
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    vmov r8, r6, d9
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    vldr d0, .LCPI38_1
+; CHECK-NEXT:    vmov r5, r8, d9
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vmov r3, r2, d0
+; CHECK-NEXT:    csel r10, r0, r4, ne
+; CHECK-NEXT:    csel r9, r1, r4, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    lsr.w r9, r0, #5
-; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    csel r6, r1, r4, ne
+; CHECK-NEXT:    csel r4, r0, r4, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movwne r6, #65535
+; CHECK-NEXT:    movtne r6, #3
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r10
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r7, #65535
-; CHECK-NEXT:    movtne r7, #3
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r1, #65535
-; CHECK-NEXT:    movtne r1, #3
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movwne r9, #65535
+; CHECK-NEXT:    movtne r9, #3
+; CHECK-NEXT:    movne.w r10, #-1
+; CHECK-NEXT:    vmov q0[2], q0[0], r10, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r9, r6
 ; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -2602,11 +2405,11 @@ define arm_aapcs_vfpcc <2 x i50> @test_unsigned_v2f64_v2i50(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI38_0:
-; CHECK-NEXT:    .long 4294967288 @ double 1125899906842623
-; CHECK-NEXT:    .long 1125122047
-; CHECK-NEXT:  .LCPI38_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI38_1:
+; CHECK-NEXT:    .long 4294967288 @ double 1125899906842623
+; CHECK-NEXT:    .long 1125122047
     %x = call <2 x i50> @llvm.fptoui.sat.v2f64.v2i50(<2 x double> %f)
     ret <2 x i50> %x
 }
@@ -2620,84 +2423,60 @@ define arm_aapcs_vfpcc <2 x i64> @test_unsigned_v2f64_v2i64(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI39_0
 ; CHECK-NEXT:    vmov r6, r7, d9
-; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    vmov r11, r3, d0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    vldr d0, .LCPI39_1
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vmov r9, r8, d0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    vmov r11, r4, d8
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    csel r9, r0, r8, ne
+; CHECK-NEXT:    csel r8, r1, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    vmov r5, r4, d8
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r7, #1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r8, #-1
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    mov r2, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    lsrs r4, r0, #5
-; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #-1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    csel r7, r1, r6, ne
+; CHECK-NEXT:    csel r6, r0, r6, ne
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r8
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2726,192 +2505,186 @@ define arm_aapcs_vfpcc <2 x i100> @test_unsigned_v2f64_v2i100(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #48
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI40_0
-; CHECK-NEXT:    vmov r6, r5, d8
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r2, r7, d0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    mov r9, r2
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI40_1
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r2, [sp, #44] @ 4-byte Spill
-; CHECK-NEXT:    mov r10, r3
+; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r9, r2
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r5, r4
 ; CHECK-NEXT:    bl __fixunsdfti
+; CHECK-NEXT:    vldr d0, .LCPI40_1
 ; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    strd r1, r0, [sp, #20] @ 8-byte Folded Spill
-; CHECK-NEXT:    csel r0, r2, r8, ne
-; CHECK-NEXT:    str r3, [sp, #40] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r11, #0
+; CHECK-NEXT:    str r3, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    csel r4, r2, r8, ne
+; CHECK-NEXT:    vmov r10, r3, d0
+; CHECK-NEXT:    strd r1, r0, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    str r5, [sp, #40] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r4, #8]
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r6, #8]
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    ldr r7, [sp, #36] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    str r7, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r11, r6
-; CHECK-NEXT:    ldr r6, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r4, #4]
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r11
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    strd r4, r11, [sp, #28] @ 8-byte Folded Spill
-; CHECK-NEXT:    str r5, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    str.w r10, [sp, #44] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r6, #4]
+; CHECK-NEXT:    mov r5, r6
+; CHECK-NEXT:    str r6, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r11
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    ldr r6, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    str.w r11, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r4, r1, r0, ne
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r6, r8
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    vmov r8, r11, d9
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r4]
-; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r5, r9
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r5]
+; CHECK-NEXT:    mov r10, r9
+; CHECK-NEXT:    str.w r9, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    mov r5, r7
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    mov r6, r10
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    mov r9, r0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    add.w r12, sp, #16
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    stm.w r12, {r0, r2, r3} @ 12-byte Folded Spill
-; CHECK-NEXT:    csel r9, r1, r10, ne
+; CHECK-NEXT:    ldr r4, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    strd r3, r0, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    csel r7, r1, r9, ne
+; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #-1
-; CHECK-NEXT:    mov r7, r5
-; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r10, r4
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    str r6, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r5, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    str r6, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r7, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    csel r4, r1, r0, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    csel r9, r1, r0, ne
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r9
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    ldr.w r9, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    lsrl r0, r7, #28
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    str.w r0, [r9, #16]
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    lsrl r4, r9, #28
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    ldr r4, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    csel r10, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    csel r6, r1, r0, ne
-; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    ldr r5, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    orr.w r0, r9, r6, lsl #4
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    strd r4, r0, [r5, #16]
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr.w r9, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    movne.w r10, #-1
+; CHECK-NEXT:    orr.w r0, r7, r10, lsl #4
+; CHECK-NEXT:    str.w r0, [r9, #20]
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    ldr.w r11, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r8, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    mov r2, r4
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    csel r7, r1, r0, ne
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r8, r6
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #15
-; CHECK-NEXT:    and r1, r0, #15
-; CHECK-NEXT:    lsrl r6, r1, #28
-; CHECK-NEXT:    strb r6, [r5, #24]
-; CHECK-NEXT:    ldr r6, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    ldr r4, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    movne r7, #15
+; CHECK-NEXT:    and r1, r7, #15
+; CHECK-NEXT:    lsrl r10, r1, #28
+; CHECK-NEXT:    strb.w r10, [r9, #24]
+; CHECK-NEXT:    ldr r6, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r4, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    ldrd r2, r3, [sp, #32] @ 8-byte Folded Reload
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r7, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #15
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    and r0, r0, #15
+; CHECK-NEXT:    movne r7, #15
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r7, #15
 ; CHECK-NEXT:    orr.w r0, r0, r1, lsl #4
-; CHECK-NEXT:    str r0, [r5, #12]
+; CHECK-NEXT:    str.w r0, [r9, #12]
 ; CHECK-NEXT:    add sp, #48
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
@@ -2919,11 +2692,11 @@ define arm_aapcs_vfpcc <2 x i100> @test_unsigned_v2f64_v2i100(<2 x double> %f) {
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI40_0:
-; CHECK-NEXT:    .long 4294967295 @ double 1.2676506002282293E+30
-; CHECK-NEXT:    .long 1177550847
-; CHECK-NEXT:  .LCPI40_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI40_1:
+; CHECK-NEXT:    .long 4294967295 @ double 1.2676506002282293E+30
+; CHECK-NEXT:    .long 1177550847
     %x = call <2 x i100> @llvm.fptoui.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
@@ -2937,196 +2710,185 @@ define arm_aapcs_vfpcc <2 x i128> @test_unsigned_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI41_0
 ; CHECK-NEXT:    vmov r8, r7, d9
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    vmov r6, r4, d0
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r2, r9, d0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r11, r2
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __fixunsdfti
 ; CHECK-NEXT:    vldr d0, .LCPI41_1
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    csel r6, r3, r6, ne
+; CHECK-NEXT:    vmov r10, r5, d0
+; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    vmov r10, r11, d0
+; CHECK-NEXT:    str r5, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    strd r1, r0, [sp, #16] @ 8-byte Folded Spill
-; CHECK-NEXT:    csel r0, r3, r5, ne
-; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r5, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    str r0, [r5, #28]
+; CHECK-NEXT:    movne.w r6, #-1
+; CHECK-NEXT:    str r6, [r4, #28]
+; CHECK-NEXT:    str.w r11, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    csel r5, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    str r6, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    str.w r10, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    str r5, [r4, #24]
+; CHECK-NEXT:    mov r5, r4
+; CHECK-NEXT:    mov r4, r9
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r5, #24]
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    csel r9, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    str r4, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    str.w r9, [r5, #20]
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    vmov r6, r11, d8
+; CHECK-NEXT:    mov r9, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    vmov r6, r5, d8
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr.w r9, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    str.w r0, [r9, #20]
+; CHECK-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    mov r7, r11
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r11, r9
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str.w r0, [r9, #16]
-; CHECK-NEXT:    ldr.w r8, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r5, #16]
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr.w r9, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    ldr r5, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    strd r1, r0, [sp, #16] @ 8-byte Folded Spill
-; CHECK-NEXT:    csel r0, r3, r7, ne
-; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    strd r1, r0, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    csel r4, r3, r8, ne
+; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str.w r0, [r11, #12]
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    mov r7, r11
+; CHECK-NEXT:    str r4, [r7, #12]
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r4, r1, r0, ne
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r10, r4
-; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r7, #8]
 ; CHECK-NEXT:    bl __aeabi_dcmpge
 ; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r7, #8]
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r11, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r7, #4]
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r7, #4]
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    csel r4, r1, r0, ne
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r1, r11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [r7]
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    str r4, [r7]
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI41_0:
-; CHECK-NEXT:    .long 4294967295 @ double 3.4028236692093843E+38
-; CHECK-NEXT:    .long 1206910975
-; CHECK-NEXT:  .LCPI41_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI41_1:
+; CHECK-NEXT:    .long 4294967295 @ double 3.4028236692093843E+38
+; CHECK-NEXT:    .long 1206910975
     %x = call <2 x i128> @llvm.fptoui.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
@@ -3571,81 +3333,86 @@ define arm_aapcs_vfpcc <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    .pad #24
 ; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vcvtb.f32.f16 s22, s17
+; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s18
+; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r0, s22
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    vcvtb.f32.f16 s24, s18
-; CHECK-NEXT:    mov r2, r0
-; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vcvtt.f32.f16 s26, s17
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    vmov r0, s26
 ; CHECK-NEXT:    vcvtt.f32.f16 s20, s18
-; CHECK-NEXT:    vldr s18, .LCPI48_0
+; CHECK-NEXT:    vcvtb.f32.f16 s24, s17
 ; CHECK-NEXT:    vcmp.f32 s22, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcvtt.f32.f16 s26, s17
-; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    vcmp.f32 s22, s18
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt gt
-; CHECK-NEXT:    movwgt r1, #65535
-; CHECK-NEXT:    movtgt r1, #3
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    vmov r5, s26
-; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    vmov r6, s20
+; CHECK-NEXT:    vmov r8, s20
+; CHECK-NEXT:    vldr s18, .LCPI48_0
+; CHECK-NEXT:    vmov r9, s24
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r6, #0
 ; CHECK-NEXT:    bl __aeabi_f2ulz
+; CHECK-NEXT:    vcmp.f32 s26, #0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r0, r9
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s24, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r7, #0
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    vcmp.f32 s24, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    vcmp.f32 s26, s18
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r1, #65535
 ; CHECK-NEXT:    movtgt r1, #3
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vcmp.f32 s22, s18
+; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    vcmp.f32 s26, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    vcmp.f32 s26, s18
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    str r7, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    itt gt
-; CHECK-NEXT:    movwgt r1, #65535
-; CHECK-NEXT:    movtgt r1, #3
-; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str.w r7, [r4, #25]
+; CHECK-NEXT:    movwgt r5, #65535
+; CHECK-NEXT:    movtgt r5, #3
+; CHECK-NEXT:    str r5, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    str.w r6, [r10, #25]
+; CHECK-NEXT:    itt gt
+; CHECK-NEXT:    movwgt r4, #65535
+; CHECK-NEXT:    movtgt r4, #3
+; CHECK-NEXT:    str r4, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s20, #0
 ; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s20, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s20, s19
-; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    vmov r0, s20
 ; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt r7, #0
+; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    movlt r6, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movwgt r7, #65535
-; CHECK-NEXT:    movtgt r7, #3
+; CHECK-NEXT:    movwgt r5, #65535
+; CHECK-NEXT:    movtgt r5, #3
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s20, #0
 ; CHECK-NEXT:    mov r9, r0
@@ -3668,16 +3435,16 @@ define arm_aapcs_vfpcc <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s20, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s20, s16
-; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r7, r1
 ; CHECK-NEXT:    vmov r0, s20
 ; CHECK-NEXT:    itt lt
-; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    movlt.w r10, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r10, #-1
-; CHECK-NEXT:    movwgt r5, #65535
-; CHECK-NEXT:    movtgt r5, #3
+; CHECK-NEXT:    movwgt r7, #65535
+; CHECK-NEXT:    movtgt r7, #3
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s20, #0
 ; CHECK-NEXT:    mov r8, r1
@@ -3690,36 +3457,36 @@ define arm_aapcs_vfpcc <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    ldr r4, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    str r0, [r4]
-; CHECK-NEXT:    lsrs r0, r5, #10
-; CHECK-NEXT:    bfc r5, #18, #14
-; CHECK-NEXT:    lsll r10, r5, #22
 ; CHECK-NEXT:    lsrl r2, r11, #28
-; CHECK-NEXT:    orr.w r1, r11, r5
+; CHECK-NEXT:    bfc r5, #18, #14
+; CHECK-NEXT:    vcvtt.f32.f16 s16, s16
+; CHECK-NEXT:    str r0, [r4]
+; CHECK-NEXT:    lsr.w r0, r7, #10
+; CHECK-NEXT:    bfc r7, #18, #14
+; CHECK-NEXT:    lsll r10, r7, #22
+; CHECK-NEXT:    orr.w r1, r11, r7
 ; CHECK-NEXT:    str.w r1, [r4, #45]
 ; CHECK-NEXT:    orr.w r1, r2, r10
 ; CHECK-NEXT:    str.w r1, [r4, #41]
 ; CHECK-NEXT:    strb.w r0, [r4, #49]
-; CHECK-NEXT:    bfc r7, #18, #14
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    vcvtt.f32.f16 s16, s16
-; CHECK-NEXT:    lsrl r0, r7, #14
-; CHECK-NEXT:    mov r5, r4
-; CHECK-NEXT:    orr.w r1, r7, r9, lsl #4
+; CHECK-NEXT:    lsrl r0, r5, #14
+; CHECK-NEXT:    mov r7, r4
+; CHECK-NEXT:    orr.w r1, r5, r9, lsl #4
 ; CHECK-NEXT:    str.w r1, [r4, #37]
 ; CHECK-NEXT:    str.w r0, [r4, #33]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    bfc r0, #18, #14
 ; CHECK-NEXT:    orr.w r0, r0, r6, lsl #18
 ; CHECK-NEXT:    str.w r0, [r4, #29]
 ; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    ldr.w r9, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    bfc r1, #18, #14
 ; CHECK-NEXT:    bfc r3, #18, #14
 ; CHECK-NEXT:    mov r6, r9
@@ -3729,7 +3496,7 @@ define arm_aapcs_vfpcc <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    movwgt r8, #65535
 ; CHECK-NEXT:    movtgt r8, #3
 ; CHECK-NEXT:    orrs r1, r3
-; CHECK-NEXT:    str r1, [r5, #20]
+; CHECK-NEXT:    str r1, [r7, #20]
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    vcmp.f32 s16, #0
 ; CHECK-NEXT:    orr.w r2, r6, r4
@@ -3743,19 +3510,18 @@ define arm_aapcs_vfpcc <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movwgt r1, #65535
 ; CHECK-NEXT:    movtgt r1, #3
-; CHECK-NEXT:    str r2, [r5, #16]
-; CHECK-NEXT:    lsrs r2, r7, #10
-; CHECK-NEXT:    strb r2, [r5, #24]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    str r2, [r7, #16]
+; CHECK-NEXT:    lsr.w r2, r5, #10
+; CHECK-NEXT:    strb r2, [r7, #24]
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    bfc r1, #18, #14
-; CHECK-NEXT:    orr.w r0, r8, r0, lsl #18
 ; CHECK-NEXT:    lsrl r2, r1, #14
+; CHECK-NEXT:    orr.w r0, r8, r0, lsl #18
 ; CHECK-NEXT:    orr.w r1, r1, r9, lsl #4
-; CHECK-NEXT:    strd r2, r1, [r5, #8]
-; CHECK-NEXT:    str r0, [r5, #4]
+; CHECK-NEXT:    strd r2, r1, [r7, #8]
+; CHECK-NEXT:    str r0, [r7, #4]
 ; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, #4
@@ -3778,38 +3544,38 @@ define arm_aapcs_vfpcc <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vcvtt.f32.f16 s20, s19
-; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vcvtt.f32.f16 s22, s19
+; CHECK-NEXT:    vmov r0, s22
 ; CHECK-NEXT:    bl __aeabi_f2ulz
-; CHECK-NEXT:    vcvtb.f32.f16 s22, s19
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s19
 ; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov r0, s26
 ; CHECK-NEXT:    vldr s28, .LCPI49_0
-; CHECK-NEXT:    vcmp.f32 s20, #0
+; CHECK-NEXT:    vcmp.f32 s22, #0
 ; CHECK-NEXT:    mov r8, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcvtt.f32.f16 s20, s16
 ; CHECK-NEXT:    vcvtt.f32.f16 s24, s18
-; CHECK-NEXT:    vcvtt.f32.f16 s26, s16
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r9, #0
 ; CHECK-NEXT:    movlt.w r8, #0
-; CHECK-NEXT:    vcmp.f32 s20, s28
+; CHECK-NEXT:    vcmp.f32 s22, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vmov r6, s20
 ; CHECK-NEXT:    vmov r4, s24
-; CHECK-NEXT:    vmov r6, s26
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r8, #-1
 ; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    bl __aeabi_f2ulz
 ; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    vcmp.f32 s22, #0
+; CHECK-NEXT:    vcmp.f32 s26, #0
 ; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt.w r11, #0
 ; CHECK-NEXT:    movlt.w r10, #0
-; CHECK-NEXT:    vcmp.f32 s22, s28
+; CHECK-NEXT:    vcmp.f32 s26, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r10, #-1
@@ -3833,12 +3599,12 @@ define arm_aapcs_vfpcc <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
 ; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    vmov r0, s16
 ; CHECK-NEXT:    mov r6, r1
-; CHECK-NEXT:    vcmp.f32 s26, #0
+; CHECK-NEXT:    vcmp.f32 s20, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt lt
 ; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    vcmp.f32 s26, s28
+; CHECK-NEXT:    vcmp.f32 s20, s28
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r6, #-1
@@ -3931,58 +3697,83 @@ define arm_aapcs_vfpcc <8 x i100> @test_unsigned_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    sub sp, #56
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vcvtt.f32.f16 s22, s16
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s17
 ; CHECK-NEXT:    vmov r0, s22
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s17
-; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    vcvtb.f32.f16 s24, s18
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    vldr s20, .LCPI50_0
 ; CHECK-NEXT:    vcmp.f32 s22, #0
+; CHECK-NEXT:    mov r9, r1
+; CHECK-NEXT:    mov r10, r2
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    movlt.w r8, #0
 ; CHECK-NEXT:    movlt r3, #0
 ; CHECK-NEXT:    vcmp.f32 s22, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
 ; CHECK-NEXT:    str r3, [sp, #52] @ 4-byte Spill
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    str r2, [sp, #48] @ 4-byte Spill
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    str r1, [sp, #44] @ 4-byte Spill
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r7, #-1
-; CHECK-NEXT:    str r7, [sp, #40] @ 4-byte Spill
+; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    movgt.w r8, #-1
+; CHECK-NEXT:    movgt.w r9, #-1
+; CHECK-NEXT:    movgt.w r10, #-1
 ; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s19
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    vcmp.f32 s24, #0
-; CHECK-NEXT:    vcvtt.f32.f16 s22, s18
+; CHECK-NEXT:    mov r7, r2
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    movlt r7, #0
+; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt r5, #0
 ; CHECK-NEXT:    movlt r3, #0
 ; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
-; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #48] @ 4-byte Spill
+; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    movgt.w r5, #-1
+; CHECK-NEXT:    movgt.w r6, #-1
+; CHECK-NEXT:    movgt.w r7, #-1
+; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    vcmp.f32 s22, #0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vcmp.f32 s22, s20
+; CHECK-NEXT:    itttt lt
+; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    movlt r1, #0
+; CHECK-NEXT:    movlt r2, #0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    str r2, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    str.w r2, [r4, #83]
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r1, #-1
+; CHECK-NEXT:    str.w r1, [r4, #79]
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    vcvtt.f32.f16 s22, s16
+; CHECK-NEXT:    str.w r0, [r4, #75]
 ; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    str.w r7, [r4, #58]
+; CHECK-NEXT:    str.w r6, [r4, #54]
+; CHECK-NEXT:    str.w r5, [r4, #50]
+; CHECK-NEXT:    str.w r10, [r4, #33]
+; CHECK-NEXT:    str.w r9, [r4, #29]
+; CHECK-NEXT:    str.w r8, [r4, #25]
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    str r1, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    movgt r3, #15
+; CHECK-NEXT:    str r3, [sp, #40] @ 4-byte Spill
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s22, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
@@ -3995,115 +3786,85 @@ define arm_aapcs_vfpcc <8 x i100> @test_unsigned_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
-; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #44] @ 4-byte Spill
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    str r2, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #32] @ 4-byte Spill
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
-; CHECK-NEXT:    vcvtb.f32.f16 s22, s17
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vcvtt.f32.f16 s22, s17
+; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r0, s22
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s18
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    mov r8, r1
 ; CHECK-NEXT:    vcmp.f32 s22, #0
-; CHECK-NEXT:    mov r6, r2
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movlt.w r8, #0
-; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt r0, #0
+; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
 ; CHECK-NEXT:    vcmp.f32 s22, s20
+; CHECK-NEXT:    vcvtt.f32.f16 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    ittt gt
-; CHECK-NEXT:    movgt.w r9, #-1
-; CHECK-NEXT:    movgt.w r8, #-1
-; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s18, #0
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s19
-; CHECK-NEXT:    mov r11, r1
+; CHECK-NEXT:    str r3, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r2, #-1
+; CHECK-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    mov r7, r2
-; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt.w r11, #0
-; CHECK-NEXT:    movlt r5, #0
-; CHECK-NEXT:    movlt r3, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #15
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
-; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r5, #-1
-; CHECK-NEXT:    movgt.w r11, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s18, #0
-; CHECK-NEXT:    mov r10, r3
+; CHECK-NEXT:    mov r9, r1
+; CHECK-NEXT:    mov r8, r2
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt.w r10, #0
 ; CHECK-NEXT:    movlt r0, #0
-; CHECK-NEXT:    movlt r1, #0
-; CHECK-NEXT:    movlt r2, #0
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    str.w r2, [r4, #83]
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    str.w r1, [r4, #79]
+; CHECK-NEXT:    movlt.w r9, #0
+; CHECK-NEXT:    movlt.w r8, #0
+; CHECK-NEXT:    movlt r3, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
+; CHECK-NEXT:    movgt r3, #15
+; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    movgt.w r8, #-1
+; CHECK-NEXT:    movgt.w r9, #-1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    vcvtt.f32.f16 s18, s19
-; CHECK-NEXT:    str.w r0, [r4, #75]
+; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    str.w r7, [r4, #58]
-; CHECK-NEXT:    str.w r11, [r4, #54]
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    str.w r5, [r4, #50]
-; CHECK-NEXT:    str.w r6, [r4, #33]
-; CHECK-NEXT:    str.w r8, [r4, #29]
-; CHECK-NEXT:    str.w r9, [r4, #25]
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r10, #15
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcvtb.f32.f16 s16, s16
-; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    mov r11, r1
 ; CHECK-NEXT:    vcmp.f32 s18, #0
-; CHECK-NEXT:    mov r6, r2
-; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    mov r10, r2
+; CHECK-NEXT:    mov r7, r3
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r7, #0
-; CHECK-NEXT:    movlt r5, #0
+; CHECK-NEXT:    movlt.w r11, #0
 ; CHECK-NEXT:    movlt r6, #0
-; CHECK-NEXT:    movlt.w r8, #0
+; CHECK-NEXT:    movlt.w r10, #0
+; CHECK-NEXT:    movlt r7, #0
 ; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    itttt gt
-; CHECK-NEXT:    movgt.w r8, #15
+; CHECK-NEXT:    movgt r7, #15
+; CHECK-NEXT:    movgt.w r10, #-1
 ; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    movgt.w r5, #-1
-; CHECK-NEXT:    movgt.w r7, #-1
+; CHECK-NEXT:    movgt.w r11, #-1
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s16, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
@@ -4123,65 +3884,65 @@ define arm_aapcs_vfpcc <8 x i100> @test_unsigned_v8f16_v8i100(<8 x half> %f) {
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    str r0, [r4]
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    lsrl r0, r7, #28
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    orr.w r1, r7, r6, lsl #4
-; CHECK-NEXT:    str.w r1, [r4, #95]
-; CHECK-NEXT:    and r1, r8, #15
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    lsrl r0, r11, #28
+; CHECK-NEXT:    and r1, r7, #15
 ; CHECK-NEXT:    str.w r0, [r4, #91]
-; CHECK-NEXT:    and r0, r10, #15
-; CHECK-NEXT:    lsrl r6, r1, #28
-; CHECK-NEXT:    strb.w r6, [r4, #99]
-; CHECK-NEXT:    orr.w r0, r0, r5, lsl #4
-; CHECK-NEXT:    str.w r0, [r4, #87]
-; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    str.w r1, [r4, #70]
+; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    lsrl r0, r9, #28
 ; CHECK-NEXT:    str.w r0, [r4, #66]
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr.w lr, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, lr
+; CHECK-NEXT:    lsrl r0, r5, #28
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    str.w r0, [r4, #41]
+; CHECK-NEXT:    ldr.w r12, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    lsrl r0, r5, #28
+; CHECK-NEXT:    str r0, [r4, #16]
+; CHECK-NEXT:    orr.w r0, r11, r10, lsl #4
+; CHECK-NEXT:    lsrl r10, r1, #28
+; CHECK-NEXT:    str.w r0, [r4, #95]
+; CHECK-NEXT:    strb.w r10, [r4, #99]
+; CHECK-NEXT:    ldr r0, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    and r0, r0, #15
+; CHECK-NEXT:    orr.w r0, r0, r6, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #87]
+; CHECK-NEXT:    orr.w r0, r9, r8, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #70]
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
-; CHECK-NEXT:    lsrl r2, r1, #28
-; CHECK-NEXT:    strb.w r2, [r4, #74]
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    lsrl r8, r1, #28
+; CHECK-NEXT:    strb.w r8, [r4, #74]
+; CHECK-NEXT:    ldr r0, [sp, #48] @ 4-byte Reload
 ; CHECK-NEXT:    and r0, r0, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #4
 ; CHECK-NEXT:    str.w r0, [r4, #62]
-; CHECK-NEXT:    ldr r7, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    str.w r1, [r4, #45]
-; CHECK-NEXT:    str.w r0, [r4, #41]
-; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #4
+; CHECK-NEXT:    str.w r0, [r4, #45]
+; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r2, r1, #28
 ; CHECK-NEXT:    strb.w r2, [r4, #49]
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #52] @ 4-byte Reload
 ; CHECK-NEXT:    and r0, r0, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
+; CHECK-NEXT:    orr.w r0, r0, lr, lsl #4
 ; CHECK-NEXT:    str.w r0, [r4, #37]
-; CHECK-NEXT:    ldr r7, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #48] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    lsrl r0, r1, #28
-; CHECK-NEXT:    orr.w r1, r1, r2, lsl #4
-; CHECK-NEXT:    strd r0, r1, [r4, #16]
-; CHECK-NEXT:    ldr r0, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    orr.w r0, r5, r2, lsl #4
+; CHECK-NEXT:    str r0, [r4, #20]
+; CHECK-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
 ; CHECK-NEXT:    and r1, r0, #15
 ; CHECK-NEXT:    lsrl r2, r1, #28
 ; CHECK-NEXT:    strb r2, [r4, #24]
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt r3, #15
 ; CHECK-NEXT:    and r0, r3, #15
-; CHECK-NEXT:    orr.w r0, r0, r7, lsl #4
+; CHECK-NEXT:    orr.w r0, r0, r12, lsl #4
 ; CHECK-NEXT:    str r0, [r4, #12]
 ; CHECK-NEXT:    add sp, #56
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12}
@@ -4206,64 +3967,61 @@ define arm_aapcs_vfpcc <8 x i128> @test_unsigned_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vcvtt.f32.f16 s22, s19
-; CHECK-NEXT:    vmov r0, s22
-; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s19
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s16
+; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    vcvtb.f32.f16 s28, s19
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    vmov r0, s28
 ; CHECK-NEXT:    vldr s20, .LCPI51_0
-; CHECK-NEXT:    vcmp.f32 s22, #0
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s18
+; CHECK-NEXT:    vmov r5, s22
+; CHECK-NEXT:    vmov r7, s28
+; CHECK-NEXT:    vcvtt.f32.f16 s26, s18
+; CHECK-NEXT:    bl __fixunssfti
+; CHECK-NEXT:    vcmp.f32 s24, #0
+; CHECK-NEXT:    add.w r12, r4, #112
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s22, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s26, s18
 ; CHECK-NEXT:    itttt lt
-; CHECK-NEXT:    movlt r6, #0
+; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s24, s20
+; CHECK-NEXT:    vcvtb.f32.f16 s18, s18
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    movgt.w r6, #-1
-; CHECK-NEXT:    strd r6, r1, [r4, #112]
-; CHECK-NEXT:    vmov r7, s24
-; CHECK-NEXT:    vmov r5, s26
-; CHECK-NEXT:    vcvtt.f32.f16 s18, s17
-; CHECK-NEXT:    strd r2, r3, [r4, #120]
+; CHECK-NEXT:    movgt.w r0, #-1
+; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    vmov r6, s26
 ; CHECK-NEXT:    bl __fixunssfti
 ; CHECK-NEXT:    vcmp.f32 s28, #0
 ; CHECK-NEXT:    add.w r12, r4, #96
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s28, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s28, s20
+; CHECK-NEXT:    vcvtt.f32.f16 s24, s17
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    it gt
+; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    vmov r6, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s22, s17
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    vmov r7, s18
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s24, #0
+; CHECK-NEXT:    vcmp.f32 s26, #0
 ; CHECK-NEXT:    add.w r12, r4, #80
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s24, s20
+; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
@@ -4272,116 +4030,105 @@ define arm_aapcs_vfpcc <8 x i128> @test_unsigned_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vcvtt.f32.f16 s24, s16
-; CHECK-NEXT:    vmov r7, s22
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    vmov r6, s24
+; CHECK-NEXT:    vcvtb.f32.f16 s26, s17
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s26, #0
+; CHECK-NEXT:    vcmp.f32 s18, #0
 ; CHECK-NEXT:    add.w r12, r4, #64
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s18, s20
+; CHECK-NEXT:    vcvtt.f32.f16 s16, s16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    it gt
+; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    vmov r5, s24
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s16
+; CHECK-NEXT:    vmov r7, s26
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s18, #0
+; CHECK-NEXT:    vcmp.f32 s24, #0
 ; CHECK-NEXT:    add.w r12, r4, #48
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s18, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r3, #-1
-; CHECK-NEXT:    it gt
+; CHECK-NEXT:    ittt gt
 ; CHECK-NEXT:    movgt.w r2, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    itt gt
 ; CHECK-NEXT:    movgt.w r1, #-1
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    vmov r6, s16
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s22, #0
+; CHECK-NEXT:    vcmp.f32 s26, #0
 ; CHECK-NEXT:    add.w r12, r4, #32
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s22, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s26, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s24, #0
+; CHECK-NEXT:    vcmp.f32 s16, #0
 ; CHECK-NEXT:    add.w r12, r4, #16
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s24, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2, r3}
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    vcmp.f32 s16, #0
+; CHECK-NEXT:    vcmp.f32 s22, #0
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    vcmp.f32 s16, s20
 ; CHECK-NEXT:    itttt lt
 ; CHECK-NEXT:    movlt r0, #0
 ; CHECK-NEXT:    movlt r1, #0
 ; CHECK-NEXT:    movlt r2, #0
 ; CHECK-NEXT:    movlt r3, #0
+; CHECK-NEXT:    vcmp.f32 s22, s20
 ; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    ittt gt
+; CHECK-NEXT:    itttt gt
 ; CHECK-NEXT:    movgt.w r3, #-1
 ; CHECK-NEXT:    movgt.w r2, #-1
 ; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    movgt.w r0, #-1
 ; CHECK-NEXT:    stm r4!, {r0, r1, r2, r3}
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14}
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll
index 9738f7ade6fe9..96aff0233e4d9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll
@@ -623,9 +623,9 @@ define arm_aapcs_vfpcc void @three_pointer_iv_v4i32(ptr nocapture readonly %x, p
 ; CHECK-NEXT:    vldrw.u32 q3, [r0, q0]
 ; CHECK-NEXT:    vldrw.u32 q4, [r0, q1, uxtw #2]
 ; CHECK-NEXT:    vldrw.u32 q5, [r0, q2]
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r0, #48
 ; CHECK-NEXT:    vmul.i32 q3, q4, q3
-; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmul.i32 q5, q4, q5
 ; CHECK-NEXT:    vmul.i32 q4, q4, r3
 ; CHECK-NEXT:    vstrw.32 q4, [r1, q1, uxtw #2]
@@ -705,9 +705,9 @@ define arm_aapcs_vfpcc void @three_pointer_iv_v4i8(ptr nocapture readonly %x, pt
 ; CHECK-NEXT:    vldrb.u32 q3, [r0, q0]
 ; CHECK-NEXT:    vldrb.u32 q4, [r0, q1]
 ; CHECK-NEXT:    vldrb.u32 q5, [r0, q2]
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r0, #12
 ; CHECK-NEXT:    vmul.i32 q3, q4, q3
-; CHECK-NEXT:    add.w r0, r0, #12
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmul.i32 q5, q4, q5
 ; CHECK-NEXT:    vmul.i32 q4, q4, r3
 ; CHECK-NEXT:    vstrb.32 q4, [r1, q1]
@@ -793,9 +793,9 @@ define arm_aapcs_vfpcc void @three_pointer_iv_v8i16(ptr nocapture readonly %x, p
 ; CHECK-NEXT:    vldrh.u16 q3, [r0, q0]
 ; CHECK-NEXT:    vldrh.u16 q4, [r0, q1, uxtw #1]
 ; CHECK-NEXT:    vldrh.u16 q5, [r0, q2]
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r0, #48
 ; CHECK-NEXT:    vmul.i16 q3, q4, q3
-; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmul.i16 q5, q4, q5
 ; CHECK-NEXT:    vmul.i16 q4, q4, r3
 ; CHECK-NEXT:    vstrh.16 q4, [r1, q1, uxtw #1]
@@ -887,9 +887,9 @@ define arm_aapcs_vfpcc void @three_pointer_iv_v16i8(ptr nocapture readonly %x, p
 ; CHECK-NEXT:    vldrb.u8 q3, [r0, q0]
 ; CHECK-NEXT:    vldrb.u8 q4, [r0, q1]
 ; CHECK-NEXT:    vldrb.u8 q5, [r0, q2]
-; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    adds r0, #48
 ; CHECK-NEXT:    vmul.i8 q3, q4, q3
-; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmul.i8 q5, q4, q5
 ; CHECK-NEXT:    vmul.i8 q4, q4, r3
 ; CHECK-NEXT:    vstrb.8 q4, [r1, q1]
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
index 967e8a94900e1..2b4c478dfe3e6 100644
--- a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc.ll
@@ -108,7 +108,7 @@ define arm_aapcs_vfpcc <4 x i32> @test_vsbciq_s32(<4 x i32> %a, <4 x i32> %b, pt
 ; CHECK-NEXT:    str r1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
-  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0)
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 536870912)
   %1 = extractvalue { <4 x i32>, i32 } %0, 1
   %2 = lshr i32 %1, 29
   %3 = and i32 %2, 1
@@ -125,6 +125,46 @@ define arm_aapcs_vfpcc <4 x i32> @test_vsbciq_u32(<4 x i32> %a, <4 x i32> %b, pt
 ; CHECK-NEXT:    ubfx r1, r1, #29, #1
 ; CHECK-NEXT:    str r1, [r0]
 ; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 536870912)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 1
+  %2 = lshr i32 %1, 29
+  %3 = and i32 %2, 1
+  store i32 %3, ptr %carry_out, align 4
+  %4 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsbcq_s32_carry_in_zero(<4 x i32> %a, <4 x i32> %b, ptr nocapture %carry_out) {
+; CHECK-LABEL: test_vsbcq_s32_carry_in_zero:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r1
+; CHECK-NEXT:    vsbc.i32 q0, q0, q1
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 1
+  %2 = lshr i32 %1, 29
+  %3 = and i32 %2, 1
+  store i32 %3, ptr %carry_out, align 4
+  %4 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsbcq_u32_carry_in_zero(<4 x i32> %a, <4 x i32> %b, ptr nocapture %carry_out) {
+; CHECK-LABEL: test_vsbcq_u32_carry_in_zero:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r1
+; CHECK-NEXT:    vsbc.i32 q0, q0, q1
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0)
   %1 = extractvalue { <4 x i32>, i32 } %0, 1
@@ -196,7 +236,7 @@ define arm_aapcs_vfpcc <4 x i32> @test_vsbciq_m_s32(<4 x i32> %inactive, <4 x i3
 entry:
   %0 = zext i16 %p to i32
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
-  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 536870912, <4 x i1> %1)
   %3 = extractvalue { <4 x i32>, i32 } %2, 1
   %4 = lshr i32 %3, 29
   %5 = and i32 %4, 1
@@ -215,6 +255,54 @@ define arm_aapcs_vfpcc <4 x i32> @test_vsbciq_m_u32(<4 x i32> %inactive, <4 x i3
 ; CHECK-NEXT:    ubfx r1, r1, #29, #1
 ; CHECK-NEXT:    str r1, [r0]
 ; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 536870912, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 1
+  %4 = lshr i32 %3, 29
+  %5 = and i32 %4, 1
+  store i32 %5, ptr %carry_out, align 4
+  %6 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %6
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsbcq_m_s32_carry_in_zero(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, ptr nocapture %carry_out, i16 zeroext %p) {
+; CHECK-LABEL: test_vsbcq_m_s32_carry_in_zero:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsbct.i32 q0, q1, q2
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.predicated.v4i32.v4i1(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 1
+  %4 = lshr i32 %3, 29
+  %5 = and i32 %4, 1
+  store i32 %5, ptr %carry_out, align 4
+  %6 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %6
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vsbcq_m_u32_carry_in_zero(<4 x i32> %inactive, <4 x i32> %a, <4 x i32> %b, ptr nocapture %carry_out, i16 zeroext %p) {
+; CHECK-LABEL: test_vsbcq_m_u32_carry_in_zero:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmsr fpscr_nzcvqc, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsbct.i32 q0, q1, q2
+; CHECK-NEXT:    vmrs r1, fpscr_nzcvqc
+; CHECK-NEXT:    ubfx r1, r1, #29, #1
+; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = zext i16 %p to i32
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
index fe28f785623ed..acbe48f9e5927 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
@@ -307,82 +307,83 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b)
 ; CHECK-NEXT:    vmov.f32 s10, s7
 ; CHECK-NEXT:    vmov r10, s8
 ; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov r7, s2
-; CHECK-NEXT:    vmov.f32 s2, s1
 ; CHECK-NEXT:    vmov.f32 s6, s5
 ; CHECK-NEXT:    vmov r2, s8
 ; CHECK-NEXT:    asr.w r0, r10, #31
-; CHECK-NEXT:    asrs r5, r7, #31
-; CHECK-NEXT:    adds.w r4, r10, r2
-; CHECK-NEXT:    eor.w r6, r10, r2
+; CHECK-NEXT:    adds.w r6, r10, r2
+; CHECK-NEXT:    eor.w r7, r10, r2
 ; CHECK-NEXT:    adc r3, r0, #0
-; CHECK-NEXT:    asrl r4, r3, r2
-; CHECK-NEXT:    subs r0, r4, r2
+; CHECK-NEXT:    asrl r6, r3, r2
+; CHECK-NEXT:    subs r0, r6, r2
+; CHECK-NEXT:    vmov r6, s2
 ; CHECK-NEXT:    sbc lr, r3, #0
 ; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    vmov.f32 s2, s1
 ; CHECK-NEXT:    umull r0, r8, r0, r2
-; CHECK-NEXT:    adds r4, r7, r3
-; CHECK-NEXT:    eor.w r1, r7, r3
+; CHECK-NEXT:    asrs r5, r6, #31
+; CHECK-NEXT:    adds r4, r6, r3
 ; CHECK-NEXT:    adc r5, r5, #0
+; CHECK-NEXT:    eor.w r1, r6, r3
 ; CHECK-NEXT:    asrl r4, r5, r3
 ; CHECK-NEXT:    subs r4, r4, r3
 ; CHECK-NEXT:    sbc r5, r5, #0
-; CHECK-NEXT:    orrs.w r6, r6, r10, asr #31
+; CHECK-NEXT:    orrs.w r7, r7, r10, asr #31
 ; CHECK-NEXT:    umull r4, r12, r4, r3
 ; CHECK-NEXT:    csetm r9, eq
-; CHECK-NEXT:    orrs.w r1, r1, r7, asr #31
-; CHECK-NEXT:    mov.w r6, #0
+; CHECK-NEXT:    orrs.w r1, r1, r6, asr #31
+; CHECK-NEXT:    mov.w r7, #0
 ; CHECK-NEXT:    csetm r1, eq
-; CHECK-NEXT:    bfi r6, r9, #0, #8
+; CHECK-NEXT:    bfi r7, r9, #0, #8
 ; CHECK-NEXT:    mla r5, r5, r3, r12
-; CHECK-NEXT:    bfi r6, r1, #8, #8
-; CHECK-NEXT:    rsbs r1, r7, #0
+; CHECK-NEXT:    bfi r7, r1, #8, #8
+; CHECK-NEXT:    rsbs r1, r6, #0
+; CHECK-NEXT:    vmsr p0, r7
 ; CHECK-NEXT:    mla r7, lr, r2, r8
 ; CHECK-NEXT:    lsll r4, r5, r1
 ; CHECK-NEXT:    rsb.w r1, r10, #0
+; CHECK-NEXT:    lsll r4, r5, r3
 ; CHECK-NEXT:    lsll r0, r7, r1
-; CHECK-NEXT:    vmov lr, s2
+; CHECK-NEXT:    vmov r3, s2
 ; CHECK-NEXT:    vmov r1, s6
 ; CHECK-NEXT:    lsll r0, r7, r2
-; CHECK-NEXT:    lsll r4, r5, r3
-; CHECK-NEXT:    vmsr p0, r6
 ; CHECK-NEXT:    vmov q3[2], q3[0], r0, r4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    vpsel q2, q3, q2
-; CHECK-NEXT:    adds.w r2, lr, r1
-; CHECK-NEXT:    asr.w r0, lr, #31
-; CHECK-NEXT:    adc r3, r0, #0
-; CHECK-NEXT:    asrl r2, r3, r1
+; CHECK-NEXT:    adds r2, r3, r1
+; CHECK-NEXT:    asr.w r0, r3, #31
+; CHECK-NEXT:    adc r5, r0, #0
+; CHECK-NEXT:    asrl r2, r5, r1
 ; CHECK-NEXT:    subs r0, r2, r1
 ; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    sbc r7, r3, #0
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    umull r0, r6, r0, r1
-; CHECK-NEXT:    asrs r5, r2, #31
-; CHECK-NEXT:    adds r4, r2, r3
-; CHECK-NEXT:    adc r5, r5, #0
-; CHECK-NEXT:    asrl r4, r5, r3
-; CHECK-NEXT:    subs r4, r4, r3
 ; CHECK-NEXT:    sbc r8, r5, #0
-; CHECK-NEXT:    mla r5, r7, r1, r6
-; CHECK-NEXT:    eor.w r6, lr, r1
-; CHECK-NEXT:    orrs.w r6, r6, lr, asr #31
-; CHECK-NEXT:    eor.w r7, r2, r3
+; CHECK-NEXT:    umull r4, lr, r0, r1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    asrs r5, r2, #31
+; CHECK-NEXT:    adds r6, r2, r0
+; CHECK-NEXT:    adc r7, r5, #0
+; CHECK-NEXT:    mla r5, r8, r1, lr
+; CHECK-NEXT:    asrl r6, r7, r0
+; CHECK-NEXT:    subs.w r8, r6, r0
+; CHECK-NEXT:    eor.w r6, r2, r0
+; CHECK-NEXT:    sbc lr, r7, #0
+; CHECK-NEXT:    eor.w r7, r3, r1
+; CHECK-NEXT:    orrs.w r6, r6, r2, asr #31
+; CHECK-NEXT:    orr.w r7, r7, r3, asr #31
+; CHECK-NEXT:    csetm r6, eq
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    bfi r12, r6, #0, #8
 ; CHECK-NEXT:    csetm r6, eq
-; CHECK-NEXT:    orrs.w r7, r7, r2, asr #31
-; CHECK-NEXT:    csetm r7, eq
-; CHECK-NEXT:    rsb.w lr, lr, #0
-; CHECK-NEXT:    bfi r12, r7, #0, #8
-; CHECK-NEXT:    lsll r0, r5, lr
 ; CHECK-NEXT:    bfi r12, r6, #8, #8
-; CHECK-NEXT:    umull r4, r6, r4, r3
-; CHECK-NEXT:    lsll r0, r5, r1
-; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    umull r6, r7, r8, r0
+; CHECK-NEXT:    rsb.w r8, r3, #0
+; CHECK-NEXT:    lsll r4, r5, r8
 ; CHECK-NEXT:    vmsr p0, r12
-; CHECK-NEXT:    mla r7, r8, r3, r6
-; CHECK-NEXT:    lsll r4, r7, r1
-; CHECK-NEXT:    lsll r4, r7, r3
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r0
+; CHECK-NEXT:    mla r3, lr, r0, r7
+; CHECK-NEXT:    lsll r4, r5, r1
+; CHECK-NEXT:    rsbs r1, r2, #0
+; CHECK-NEXT:    lsll r6, r3, r1
+; CHECK-NEXT:    lsll r6, r3, r0
+; CHECK-NEXT:    vmov q0[2], q0[0], r6, r4
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    vmov.f32 s1, s2
 ; CHECK-NEXT:    vmov.f32 s2, s8
diff --git a/llvm/test/CodeGen/Thumb2/mve-minmaxi.ll b/llvm/test/CodeGen/Thumb2/mve-minmaxi.ll
index 55a621eaf4c9c..d9ef1030ee922 100644
--- a/llvm/test/CodeGen/Thumb2/mve-minmaxi.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-minmaxi.ll
@@ -6,8 +6,8 @@ declare i8 @llvm.smax.i8(i8 %a, i8 %b) readnone
 define arm_aapcs_vfpcc i8 @smaxi8(i8 %a, i8 %b) {
 ; CHECK-LABEL: smaxi8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sxtb r0, r0
 ; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    sxtb r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    bx lr
@@ -20,8 +20,8 @@ declare i16 @llvm.smax.i16(i16 %a, i16 %b) readnone
 define arm_aapcs_vfpcc i16 @smaxi16(i16 %a, i16 %b) {
 ; CHECK-LABEL: smaxi16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    bx lr
@@ -48,10 +48,8 @@ define arm_aapcs_vfpcc i64 @smaxi64(i64 %a, i64 %b) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    subs.w r12, r2, r0
 ; CHECK-NEXT:    sbcs.w r12, r3, r1
-; CHECK-NEXT:    cset r12, lt
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lt
+; CHECK-NEXT:    csel r1, r1, r3, lt
 ; CHECK-NEXT:    bx lr
   %c = call i64 @llvm.smax.i64(i64 %a, i64 %b)
   ret i64 %c
@@ -205,10 +203,8 @@ define arm_aapcs_vfpcc <1 x i64> @smax1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    subs.w r12, r2, r0
 ; CHECK-NEXT:    sbcs.w r12, r3, r1
-; CHECK-NEXT:    cset r12, lt
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lt
+; CHECK-NEXT:    csel r1, r1, r3, lt
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
   %c = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b)
@@ -290,8 +286,8 @@ declare i8 @llvm.umax.i8(i8 %a, i8 %b) readnone
 define arm_aapcs_vfpcc i8 @umaxi8(i8 %a, i8 %b) {
 ; CHECK-LABEL: umaxi8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    uxtb r0, r0
 ; CHECK-NEXT:    uxtb r1, r1
+; CHECK-NEXT:    uxtb r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    bx lr
@@ -304,8 +300,8 @@ declare i16 @llvm.umax.i16(i16 %a, i16 %b) readnone
 define arm_aapcs_vfpcc i16 @umaxi16(i16 %a, i16 %b) {
 ; CHECK-LABEL: umaxi16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    uxth r1, r1
+; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    bx lr
@@ -332,10 +328,8 @@ define arm_aapcs_vfpcc i64 @umaxi64(i64 %a, i64 %b) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    subs.w r12, r2, r0
 ; CHECK-NEXT:    sbcs.w r12, r3, r1
-; CHECK-NEXT:    cset r12, lo
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lo
+; CHECK-NEXT:    csel r1, r1, r3, lo
 ; CHECK-NEXT:    bx lr
   %c = call i64 @llvm.umax.i64(i64 %a, i64 %b)
   ret i64 %c
@@ -482,10 +476,8 @@ define arm_aapcs_vfpcc <1 x i64> @umax1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    subs.w r12, r2, r0
 ; CHECK-NEXT:    sbcs.w r12, r3, r1
-; CHECK-NEXT:    cset r12, lo
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lo
+; CHECK-NEXT:    csel r1, r1, r3, lo
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
   %c = call <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b)
@@ -567,8 +559,8 @@ declare i8 @llvm.smin.i8(i8 %a, i8 %b) readnone
 define arm_aapcs_vfpcc i8 @smini8(i8 %a, i8 %b) {
 ; CHECK-LABEL: smini8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sxtb r0, r0
 ; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    sxtb r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    bx lr
@@ -581,8 +573,8 @@ declare i16 @llvm.smin.i16(i16 %a, i16 %b) readnone
 define arm_aapcs_vfpcc i16 @smini16(i16 %a, i16 %b) {
 ; CHECK-LABEL: smini16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    bx lr
@@ -609,10 +601,8 @@ define arm_aapcs_vfpcc i64 @smini64(i64 %a, i64 %b) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    subs.w r12, r0, r2
 ; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    cset r12, lt
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lt
+; CHECK-NEXT:    csel r1, r1, r3, lt
 ; CHECK-NEXT:    bx lr
   %c = call i64 @llvm.smin.i64(i64 %a, i64 %b)
   ret i64 %c
@@ -766,10 +756,8 @@ define arm_aapcs_vfpcc <1 x i64> @smin1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    subs.w r12, r0, r2
 ; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    cset r12, lt
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lt
+; CHECK-NEXT:    csel r1, r1, r3, lt
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
   %c = call <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b)
@@ -851,8 +839,8 @@ declare i8 @llvm.umin.i8(i8 %a, i8 %b) readnone
 define arm_aapcs_vfpcc i8 @umini8(i8 %a, i8 %b) {
 ; CHECK-LABEL: umini8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    uxtb r0, r0
 ; CHECK-NEXT:    uxtb r1, r1
+; CHECK-NEXT:    uxtb r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lo
 ; CHECK-NEXT:    bx lr
@@ -865,8 +853,8 @@ declare i16 @llvm.umin.i16(i16 %a, i16 %b) readnone
 define arm_aapcs_vfpcc i16 @umini16(i16 %a, i16 %b) {
 ; CHECK-LABEL: umini16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    uxth r1, r1
+; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lo
 ; CHECK-NEXT:    bx lr
@@ -893,10 +881,8 @@ define arm_aapcs_vfpcc i64 @umini64(i64 %a, i64 %b) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    subs.w r12, r0, r2
 ; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    cset r12, lo
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lo
+; CHECK-NEXT:    csel r1, r1, r3, lo
 ; CHECK-NEXT:    bx lr
   %c = call i64 @llvm.umin.i64(i64 %a, i64 %b)
   ret i64 %c
@@ -1043,10 +1029,8 @@ define arm_aapcs_vfpcc <1 x i64> @umin1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    subs.w r12, r0, r2
 ; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    cset r12, lo
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    csel r1, r1, r3, ne
+; CHECK-NEXT:    csel r0, r0, r2, lo
+; CHECK-NEXT:    csel r1, r1, r3, lo
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
   %c = call <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pipelineloops.ll b/llvm/test/CodeGen/Thumb2/mve-pipelineloops.ll
index 70957ca950d71..43ed5eefbf4c7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pipelineloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pipelineloops.ll
@@ -13,8 +13,8 @@ define void @arm_cmplx_dot_prod_q15(ptr noundef %pSrcA, ptr noundef %pSrcB, i32
 ; CHECK-NEXT:    movs r6, #2
 ; CHECK-NEXT:    lsrs r7, r2, #3
 ; CHECK-NEXT:    rsb r6, r6, r2, lsr #3
-; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    cmp r7, #2
+; CHECK-NEXT:    mov.w r5, #0
 ; CHECK-NEXT:    csel r7, r6, r5, hs
 ; CHECK-NEXT:    add.w lr, r7, #1
 ; CHECK-NEXT:    mov r4, r5
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
index 101b49fea488a..0bec2b100911c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
@@ -28,11 +28,12 @@ define arm_aapcs_vfpcc <4 x i32> @sext_v4i1_v4f32(<4 x float> %src1, <4 x float>
 ; CHECK-MVE-NEXT:    csetm r1, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmov q0[2], q0[0], r1, r0
-; CHECK-MVE-NEXT:    csetm r2, ne
+; CHECK-MVE-NEXT:    vmov q2[2], q2[0], r1, r0
+; CHECK-MVE-NEXT:    csetm r0, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    csetm r3, ne
-; CHECK-MVE-NEXT:    vmov q0[3], q0[1], r3, r2
+; CHECK-MVE-NEXT:    csetm r1, ne
+; CHECK-MVE-NEXT:    vmov q2[3], q2[1], r1, r0
+; CHECK-MVE-NEXT:    vmov q0, q2
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: sext_v4i1_v4f32:
@@ -65,49 +66,49 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @sext_v8i1_v8f32(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: sext_v8i1_v8f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-MVE-NEXT:    push {r4, r5, r7, lr}
+; CHECK-MVE-NEXT:    .save {r4, lr}
+; CHECK-MVE-NEXT:    push {r4, lr}
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s8, s7
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s6
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s10, s8
+; CHECK-MVE-NEXT:    vmovx.f16 s8, s5
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
 ; CHECK-MVE-NEXT:    csetm r12, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s6, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s10, s8
 ; CHECK-MVE-NEXT:    csetm lr, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s6, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s6, s0
 ; CHECK-MVE-NEXT:    csetm r2, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f16 s6, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s3
 ; CHECK-MVE-NEXT:    csetm r3, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f16 s6, s2
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s4
+; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
 ; CHECK-MVE-NEXT:    csetm r0, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s0
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    csetm r1, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f16 s0, s2
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s0
 ; CHECK-MVE-NEXT:    csetm r4, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov.16 q0[0], r4
-; CHECK-MVE-NEXT:    csetm r5, ne
-; CHECK-MVE-NEXT:    vmov.16 q0[1], r5
-; CHECK-MVE-NEXT:    vmov.16 q0[2], r0
-; CHECK-MVE-NEXT:    vmov.16 q0[3], r1
-; CHECK-MVE-NEXT:    vmov.16 q0[4], r2
-; CHECK-MVE-NEXT:    vmov.16 q0[5], r3
+; CHECK-MVE-NEXT:    vmov.16 q0[1], r0
+; CHECK-MVE-NEXT:    vmov.16 q0[2], r1
+; CHECK-MVE-NEXT:    vmov.16 q0[3], r2
+; CHECK-MVE-NEXT:    vmov.16 q0[4], r3
+; CHECK-MVE-NEXT:    vmov.16 q0[5], lr
 ; CHECK-MVE-NEXT:    vmov.16 q0[6], r12
-; CHECK-MVE-NEXT:    vmov.16 q0[7], lr
-; CHECK-MVE-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-MVE-NEXT:    csetm r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q0[7], r0
+; CHECK-MVE-NEXT:    pop {r4, pc}
 ;
 ; CHECK-MVEFP-LABEL: sext_v8i1_v8f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
@@ -198,22 +199,22 @@ define arm_aapcs_vfpcc <2 x i64> @sext_v2i1_v2f64(<2 x double> %src) {
 ; CHECK-MVEFP-NEXT:    vpush {d8, d9}
 ; CHECK-MVEFP-NEXT:    vmov q4, q0
 ; CHECK-MVEFP-NEXT:    vldr d0, .LCPI6_0
-; CHECK-MVEFP-NEXT:    vmov r0, r1, d9
+; CHECK-MVEFP-NEXT:    vmov r0, r1, d8
 ; CHECK-MVEFP-NEXT:    vmov r4, r5, d0
 ; CHECK-MVEFP-NEXT:    mov r2, r4
 ; CHECK-MVEFP-NEXT:    mov r3, r5
 ; CHECK-MVEFP-NEXT:    bl __aeabi_dcmpeq
 ; CHECK-MVEFP-NEXT:    mov r6, r0
-; CHECK-MVEFP-NEXT:    vmov r0, r1, d8
+; CHECK-MVEFP-NEXT:    vmov r0, r1, d9
 ; CHECK-MVEFP-NEXT:    mov r2, r4
 ; CHECK-MVEFP-NEXT:    mov r3, r5
 ; CHECK-MVEFP-NEXT:    bl __aeabi_dcmpeq
-; CHECK-MVEFP-NEXT:    cmp r6, #0
-; CHECK-MVEFP-NEXT:    csetm r1, eq
 ; CHECK-MVEFP-NEXT:    cmp r0, #0
 ; CHECK-MVEFP-NEXT:    csetm r0, eq
-; CHECK-MVEFP-NEXT:    vmov q0[2], q0[0], r0, r1
-; CHECK-MVEFP-NEXT:    vmov q0[3], q0[1], r0, r1
+; CHECK-MVEFP-NEXT:    cmp r6, #0
+; CHECK-MVEFP-NEXT:    csetm r1, eq
+; CHECK-MVEFP-NEXT:    vmov q0[2], q0[0], r1, r0
+; CHECK-MVEFP-NEXT:    vmov q0[3], q0[1], r1, r0
 ; CHECK-MVEFP-NEXT:    vpop {d8, d9}
 ; CHECK-MVEFP-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-MVEFP-NEXT:    .p2align 3
@@ -245,22 +246,22 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @zext_v4i1_v4f32(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-MVE-LABEL: zext_v4i1_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    vmov.i32 q2, #0x1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    csetm r0, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    csetm r1, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    csetm r2, ne
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vmov q3[2], q3[0], r1, r0
+; CHECK-MVE-NEXT:    csetm r0, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    csetm r3, ne
-; CHECK-MVE-NEXT:    vmov q0[2], q0[0], r3, r2
-; CHECK-MVE-NEXT:    vmov q0[3], q0[1], r1, r0
-; CHECK-MVE-NEXT:    vand q0, q0, q2
+; CHECK-MVE-NEXT:    csetm r1, ne
+; CHECK-MVE-NEXT:    vmov q3[3], q3[1], r1, r0
+; CHECK-MVE-NEXT:    vand q0, q3, q2
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: zext_v4i1_v4f32:
@@ -293,51 +294,51 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @zext_v8i1_v8f32(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-MVE-LABEL: zext_v8i1_v8f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-MVE-NEXT:    push {r4, r5, r7, lr}
-; CHECK-MVE-NEXT:    vmovx.f16 s8, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
-; CHECK-MVE-NEXT:    vcmp.f16 s10, s8
+; CHECK-MVE-NEXT:    .save {r4, lr}
+; CHECK-MVE-NEXT:    push {r4, lr}
+; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmovx.f16 s8, s6
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmovx.f16 s10, s2
-; CHECK-MVE-NEXT:    csetm r12, ne
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s10, s8
-; CHECK-MVE-NEXT:    csetm lr, ne
+; CHECK-MVE-NEXT:    csetm r12, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmovx.f16 s2, s5
 ; CHECK-MVE-NEXT:    vmovx.f16 s6, s1
-; CHECK-MVE-NEXT:    csetm r2, ne
+; CHECK-MVE-NEXT:    csetm lr, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s6, s2
 ; CHECK-MVE-NEXT:    vmovx.f16 s2, s4
 ; CHECK-MVE-NEXT:    vmovx.f16 s6, s0
-; CHECK-MVE-NEXT:    csetm r3, ne
+; CHECK-MVE-NEXT:    csetm r2, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    csetm r0, ne
+; CHECK-MVE-NEXT:    csetm r3, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s6, s2
-; CHECK-MVE-NEXT:    csetm r1, ne
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s3
+; CHECK-MVE-NEXT:    csetm r0, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    csetm r1, ne
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f16 s2, s0
 ; CHECK-MVE-NEXT:    vmov.i16 q0, #0x1
 ; CHECK-MVE-NEXT:    csetm r4, ne
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    csetm r5, ne
-; CHECK-MVE-NEXT:    vmov.16 q1[0], r5
-; CHECK-MVE-NEXT:    vmov.16 q1[1], r4
-; CHECK-MVE-NEXT:    vmov.16 q1[2], r1
-; CHECK-MVE-NEXT:    vmov.16 q1[3], r0
-; CHECK-MVE-NEXT:    vmov.16 q1[4], r3
-; CHECK-MVE-NEXT:    vmov.16 q1[5], r2
-; CHECK-MVE-NEXT:    vmov.16 q1[6], lr
-; CHECK-MVE-NEXT:    vmov.16 q1[7], r12
+; CHECK-MVE-NEXT:    vmov.16 q1[0], r4
+; CHECK-MVE-NEXT:    vmov.16 q1[1], r1
+; CHECK-MVE-NEXT:    vmov.16 q1[2], r0
+; CHECK-MVE-NEXT:    vmov.16 q1[3], r3
+; CHECK-MVE-NEXT:    vmov.16 q1[4], r2
+; CHECK-MVE-NEXT:    vmov.16 q1[5], lr
+; CHECK-MVE-NEXT:    vmov.16 q1[6], r12
+; CHECK-MVE-NEXT:    csetm r0, ne
+; CHECK-MVE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-MVE-NEXT:    vand q0, q1, q0
-; CHECK-MVE-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-MVE-NEXT:    pop {r4, pc}
 ;
 ; CHECK-MVEFP-LABEL: zext_v8i1_v8f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
@@ -614,24 +615,24 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @fptoui_v4i1_v4f32(<4 x float> %src) {
 ; CHECK-MVE-LABEL: fptoui_v4i1_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcvt.s32.f32 s6, s3
-; CHECK-MVE-NEXT:    vldr s8, .LCPI20_0
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s2, s2
-; CHECK-MVE-NEXT:    vcvt.s32.f32 s10, s1
-; CHECK-MVE-NEXT:    vmov.f32 s4, #1.000000e+00
+; CHECK-MVE-NEXT:    vldr s10, .LCPI20_0
+; CHECK-MVE-NEXT:    vcvt.s32.f32 s6, s1
+; CHECK-MVE-NEXT:    vcvt.s32.f32 s4, s3
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s0, s0
-; CHECK-MVE-NEXT:    vmov r0, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r0, s2
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s8, s4
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r0, s10
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s8, s4
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vmov.f32 s8, #1.000000e+00
+; CHECK-MVE-NEXT:    vmov r3, s2
+; CHECK-MVE-NEXT:    vmov r2, s6
+; CHECK-MVE-NEXT:    vmov r1, s4
 ; CHECK-MVE-NEXT:    vmov r0, s0
-; CHECK-MVE-NEXT:    vseleq.f32 s1, s8, s4
+; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s8
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s10, s8
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s10, s8
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s10, s8
 ; CHECK-MVE-NEXT:    bx lr
 ; CHECK-MVE-NEXT:    .p2align 2
 ; CHECK-MVE-NEXT:  @ %bb.1:
@@ -654,24 +655,27 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @fptosi_v4i1_v4f32(<4 x float> %src) {
 ; CHECK-MVE-LABEL: fptosi_v4i1_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcvt.s32.f32 s8, s3
-; CHECK-MVE-NEXT:    vldr s10, .LCPI21_0
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s2, s2
-; CHECK-MVE-NEXT:    vcvt.s32.f32 s6, s1
-; CHECK-MVE-NEXT:    vmov.f32 s4, #1.000000e+00
+; CHECK-MVE-NEXT:    vldr s8, .LCPI21_0
+; CHECK-MVE-NEXT:    vcvt.s32.f32 s4, s1
+; CHECK-MVE-NEXT:    vcvt.s32.f32 s10, s3
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s0, s0
-; CHECK-MVE-NEXT:    vmov r0, s8
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vmov r0, s2
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s10, s4
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vmov r0, s6
-; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s4
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov.f32 s6, #1.000000e+00
+; CHECK-MVE-NEXT:    vmov r3, s2
+; CHECK-MVE-NEXT:    vmov r2, s4
+; CHECK-MVE-NEXT:    vmov r1, s10
 ; CHECK-MVE-NEXT:    vmov r0, s0
-; CHECK-MVE-NEXT:    vseleq.f32 s1, s10, s4
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vseleq.f32 s0, s10, s4
+; CHECK-MVE-NEXT:    lsls r3, r3, #31
+; CHECK-MVE-NEXT:    lsl.w r2, r2, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s2, s8, s6
+; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsl.w r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s1, s8, s6
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsl.w r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s8, s6
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s6
 ; CHECK-MVE-NEXT:    bx lr
 ; CHECK-MVE-NEXT:    .p2align 2
 ; CHECK-MVE-NEXT:  @ %bb.1:
@@ -801,45 +805,45 @@ define arm_aapcs_vfpcc <8 x half> @fptoui_v8i1_v8f16(<8 x half> %src) {
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmovx.f16 s0, s0
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov r0, s4
+; CHECK-MVE-NEXT:    vmov r1, s0
 ; CHECK-MVE-NEXT:    vldr.16 s8, .LCPI24_0
-; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.f16 s6, #1.000000e+00
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s0, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vmov r0, s10
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov r1, s10
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s8, s6
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
+; CHECK-MVE-NEXT:    vins.f16 s0, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s1
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s2
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s2
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s2, s2
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r0, s2
 ; CHECK-MVE-NEXT:    vseleq.f16 s1, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s1, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vins.f16 s1, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s2
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s3
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s2, s2
+; CHECK-MVE-NEXT:    vmov r1, s2
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vmov r1, s10
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s2, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s2, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vmov r0, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vins.f16 s2, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s3
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
+; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s3, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s3, s10
+; CHECK-MVE-NEXT:    vins.f16 s3, s4
 ; CHECK-MVE-NEXT:    bx lr
 ; CHECK-MVE-NEXT:    .p2align 1
 ; CHECK-MVE-NEXT:  @ %bb.1:
@@ -865,45 +869,49 @@ define arm_aapcs_vfpcc <8 x half> @fptosi_v8i1_v8f16(<8 x half> %src) {
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s0
 ; CHECK-MVE-NEXT:    vmovx.f16 s0, s0
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
+; CHECK-MVE-NEXT:    vmov r0, s4
+; CHECK-MVE-NEXT:    vmov r1, s0
 ; CHECK-MVE-NEXT:    vldr.16 s8, .LCPI25_0
-; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.f16 s6, #1.000000e+00
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s1
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s0, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vmov r0, s10
 ; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
+; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s8, s6
+; CHECK-MVE-NEXT:    vmov r1, s10
+; CHECK-MVE-NEXT:    vins.f16 s0, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s1
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s2
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s2
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s2, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    lsls r0, r0, #31
-; CHECK-MVE-NEXT:    vmov r0, s2
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s1, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s1, s10
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vins.f16 s1, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s2
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s2
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s3
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s2, s2
+; CHECK-MVE-NEXT:    vmov r1, s2
 ; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s2, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s2, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s10, s3
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
-; CHECK-MVE-NEXT:    vmov r0, s10
-; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmov r1, s10
+; CHECK-MVE-NEXT:    vins.f16 s2, s4
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s3
 ; CHECK-MVE-NEXT:    vmov r0, s4
-; CHECK-MVE-NEXT:    vseleq.f16 s10, s8, s6
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s4, s8, s6
 ; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s3, s8, s6
-; CHECK-MVE-NEXT:    vins.f16 s3, s10
+; CHECK-MVE-NEXT:    vins.f16 s3, s4
 ; CHECK-MVE-NEXT:    bx lr
 ; CHECK-MVE-NEXT:    .p2align 1
 ; CHECK-MVE-NEXT:  @ %bb.1:
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
index 46406aeebfa4e..ff5ee8929aae8 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -378,23 +378,21 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: cmpeqz_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r0, r1, d2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, r2, d2
+; CHECK-NEXT:    vmov r1, r2, d0
+; CHECK-NEXT:    cset r0, eq
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    cset r1, eq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csinc r0, r1, zr, ne
+; CHECK-NEXT:    csinc r0, r0, zr, ne
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #0, #8
-; CHECK-NEXT:    vmov r0, r2, d1
+; CHECK-NEXT:    vmov r0, r2, d3
 ; CHECK-NEXT:    orrs r0, r2
-; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    cset r0, eq
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    cset r2, eq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csinc r0, r2, zr, ne
+; CHECK-NEXT:    csinc r0, r0, zr, ne
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #8, #8
 ; CHECK-NEXT:    vmsr p0, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
index bf6468baac22b..9400f24e7192c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
@@ -4,15 +4,15 @@
 define void @arm_min_helium_f32(ptr %pSrc, i32 %blockSize, ptr nocapture %pResult, ptr nocapture %pIndex) {
 ; CHECK-LABEL: arm_min_helium_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r6, r7, lr}
-; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    movs r4, #0
 ; CHECK-NEXT:    mov.w r12, #4
-; CHECK-NEXT:    vidup.u32 q2, r6, #1
-; CHECK-NEXT:    movw r4, #54437
-; CHECK-NEXT:    movt r4, #21352
+; CHECK-NEXT:    vidup.u32 q2, r4, #1
+; CHECK-NEXT:    movw r5, #54437
+; CHECK-NEXT:    movt r5, #21352
+; CHECK-NEXT:    vdup.32 q1, r5
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    vdup.32 q1, r4
 ; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
@@ -33,7 +33,7 @@ define void @arm_min_helium_f32(ptr %pSrc, i32 %blockSize, ptr nocapture %pResul
 ; CHECK-NEXT:    vminv.u32 r1, q0
 ; CHECK-NEXT:    str r1, [r3]
 ; CHECK-NEXT:    vstr s8, [r2]
-; CHECK-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:  .LCPI0_0:
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vselect.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vselect.ll
index f70af5661f4c9..a990cee1f5fb3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-vselect.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vselect.ll
@@ -70,29 +70,27 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
 ; CHECK-LABEL: cmpeqz_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, r1, d4
+; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, r2, d0
+; CHECK-NEXT:    vmov r1, r2, d2
+; CHECK-NEXT:    cset r0, eq
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r2, r3, d2
+; CHECK-NEXT:    vmov r2, r3, d4
 ; CHECK-NEXT:    cset r1, eq
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    cset r2, eq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r2, eq
+; CHECK-NEXT:    csel r0, r0, r1, eq
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #0, #8
-; CHECK-NEXT:    vmov r0, r2, d5
-; CHECK-NEXT:    orr.w r12, r0, r2
-; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    vmov r0, r2, d1
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    cset r12, eq
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    vmov r3, r0, d3
+; CHECK-NEXT:    vmov r3, r0, d5
 ; CHECK-NEXT:    cset r2, eq
 ; CHECK-NEXT:    orrs r0, r3
-; CHECK-NEXT:    cset r0, eq
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r2, r0, eq
+; CHECK-NEXT:    csel r0, r12, r2, eq
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #8, #8
 ; CHECK-NEXT:    vmsr p0, r1
@@ -176,29 +174,27 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpnez_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
 ; CHECK-LABEL: cmpnez_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, r1, d4
+; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, r2, d0
+; CHECK-NEXT:    vmov r1, r2, d2
+; CHECK-NEXT:    cset r0, ne
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r2, r3, d2
+; CHECK-NEXT:    vmov r2, r3, d4
 ; CHECK-NEXT:    cset r1, ne
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    cset r2, ne
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r2, ne
+; CHECK-NEXT:    csel r0, r0, r1, ne
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #0, #8
-; CHECK-NEXT:    vmov r0, r2, d5
-; CHECK-NEXT:    orr.w r12, r0, r2
-; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    vmov r0, r2, d1
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    cset r12, ne
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    vmov r3, r0, d3
+; CHECK-NEXT:    vmov r3, r0, d5
 ; CHECK-NEXT:    cset r2, ne
 ; CHECK-NEXT:    orrs r0, r3
-; CHECK-NEXT:    cset r0, ne
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r0, r2, r0, ne
+; CHECK-NEXT:    csel r0, r12, r2, ne
 ; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #8, #8
 ; CHECK-NEXT:    vmsr p0, r1
@@ -284,19 +280,19 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpsltz_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
 ; CHECK-LABEL: cmpsltz_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov r2, s9
 ; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s5
-; CHECK-NEXT:    cmp.w r3, r0, lsr #31
-; CHECK-NEXT:    csel r0, r1, r2, ne
-; CHECK-NEXT:    vmov r1, s11
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    cmp.w r3, r2, lsr #31
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    csel r0, r0, r1, ne
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    asr.w r12, r0, #31
-; CHECK-NEXT:    vmov r2, s3
-; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    cmp.w r3, r1, lsr #31
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    cmp.w r3, r0, lsr #31
 ; CHECK-NEXT:    bfi r3, r12, #0, #8
-; CHECK-NEXT:    csel r0, r2, r0, ne
+; CHECK-NEXT:    csel r0, r1, r2, ne
 ; CHECK-NEXT:    asrs r0, r0, #31
 ; CHECK-NEXT:    bfi r3, r0, #8, #8
 ; CHECK-NEXT:    vmsr p0, r3
@@ -381,34 +377,34 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1_i1(<2 x i64> %a, <2 x i64> %b, i64 %c) {
 ; CHECK-LABEL: cmpeqz_v2i1_i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    orr.w r3, r0, r1
-; CHECK-NEXT:    vmov r0, r1, d2
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, r2, d3
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov r2, r3, d2
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    vmov r3, r4, d3
 ; CHECK-NEXT:    csetm r12, eq
-; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r1, r2, d0
-; CHECK-NEXT:    csetm r4, eq
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r1, r2, d1
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    orrs r3, r4
+; CHECK-NEXT:    vmov r4, r3, d0
+; CHECK-NEXT:    csetm r5, eq
+; CHECK-NEXT:    orrs r3, r4
+; CHECK-NEXT:    vmov r3, r4, d1
 ; CHECK-NEXT:    csetm lr, eq
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    csetm r1, eq
-; CHECK-NEXT:    cbz r3, .LBB15_2
+; CHECK-NEXT:    orrs r3, r4
+; CHECK-NEXT:    csetm r4, eq
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    beq .LBB15_2
 ; CHECK-NEXT:  @ %bb.1: @ %select.false
-; CHECK-NEXT:    bfi r0, r12, #0, #8
-; CHECK-NEXT:    bfi r0, r4, #8, #8
+; CHECK-NEXT:    bfi r2, r12, #0, #8
+; CHECK-NEXT:    bfi r2, r5, #8, #8
 ; CHECK-NEXT:    b .LBB15_3
 ; CHECK-NEXT:  .LBB15_2:
-; CHECK-NEXT:    bfi r0, lr, #0, #8
-; CHECK-NEXT:    bfi r0, r1, #8, #8
+; CHECK-NEXT:    bfi r2, lr, #0, #8
+; CHECK-NEXT:    bfi r2, r4, #8, #8
 ; CHECK-NEXT:  .LBB15_3: @ %select.end
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmsr p0, r2
 ; CHECK-NEXT:    vpsel q0, q0, q1
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %c1 = icmp eq <2 x i64> %a, zeroinitializer
   %c2 = icmp eq <2 x i64> %b, zeroinitializer
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
index 0ff262e6b53ab..f4a0d5120305a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
@@ -458,26 +458,24 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: cmpeqz_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r0, r1, d2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, r2, d2
+; CHECK-NEXT:    vmov r1, r2, d0
+; CHECK-NEXT:    cset r0, eq
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    cset r1, eq
-; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    eoreq r1, r1, #1
-; CHECK-NEXT:    rsbs r0, r1, #0
+; CHECK-NEXT:    eoreq r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    bfi r1, r0, #0, #8
-; CHECK-NEXT:    vmov r0, r2, d1
+; CHECK-NEXT:    vmov r0, r2, d3
 ; CHECK-NEXT:    orrs r0, r2
-; CHECK-NEXT:    vmov r2, r3, d3
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    cset r0, eq
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    cset r2, eq
-; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    eoreq r2, r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    eoreq r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    bfi r1, r0, #8, #8
 ; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index 8eb941371f993..29b56639bd769 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -6,102 +6,99 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr no
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #12
-; CHECK-NEXT:    sub sp, #12
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq.w .LBB0_8
 ; CHECK-NEXT:  @ %bb.1: @ %entry
+; CHECK-NEXT:    mov r11, r2
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    bne .LBB0_3
 ; CHECK-NEXT:  @ %bb.2:
-; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    mov r10, r2
+; CHECK-NEXT:    mov r10, r11
 ; CHECK-NEXT:    b .LBB0_6
 ; CHECK-NEXT:  .LBB0_3: @ %vector.ph
-; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    bic r3, r3, #1
-; CHECK-NEXT:    subs r7, r3, #2
-; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    bic r2, r3, #1
 ; CHECK-NEXT:    adr r4, .LCPI0_0
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    subs r7, r2, #2
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    add.w r10, r11, r2, lsl #2
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
-; CHECK-NEXT:    add.w r10, r2, r3, lsl #2
-; CHECK-NEXT:    add.w r8, r1, r3, lsl #2
-; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
+; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
+; CHECK-NEXT:    add.w r8, r1, r2, lsl #2
+; CHECK-NEXT:    add.w r12, r0, r2, lsl #2
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    vmvn.i32 q1, #0x80000000
 ; CHECK-NEXT:  .LBB0_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrd r4, r3, [r0], #8
+; CHECK-NEXT:    ldrd r4, r2, [r0], #8
 ; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    ldrd r7, r6, [r1], #8
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    smull r4, r11, r7, r4
-; CHECK-NEXT:    asrl r4, r11, #31
+; CHECK-NEXT:    smull r4, r7, r7, r4
+; CHECK-NEXT:    asrl r4, r7, #31
 ; CHECK-NEXT:    rsbs.w r9, r4, #-2147483648
 ; CHECK-NEXT:    mov.w r9, #-1
-; CHECK-NEXT:    sbcs.w r3, r9, r11
+; CHECK-NEXT:    sbcs.w r3, r9, r7
 ; CHECK-NEXT:    csetm r3, lt
 ; CHECK-NEXT:    bfi r5, r3, #0, #8
-; CHECK-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    smull r6, r3, r6, r3
-; CHECK-NEXT:    asrl r6, r3, #31
-; CHECK-NEXT:    rsbs.w r7, r6, #-2147483648
-; CHECK-NEXT:    vmov q2[2], q2[0], r4, r6
-; CHECK-NEXT:    sbcs.w r7, r9, r3
-; CHECK-NEXT:    vmov q2[3], q2[1], r11, r3
-; CHECK-NEXT:    csetm r7, lt
-; CHECK-NEXT:    mvn r6, #-2147483648
-; CHECK-NEXT:    bfi r5, r7, #8, #8
+; CHECK-NEXT:    smull r2, r3, r6, r2
+; CHECK-NEXT:    asrl r2, r3, #31
+; CHECK-NEXT:    rsbs.w r6, r2, #-2147483648
+; CHECK-NEXT:    vmov q2[2], q2[0], r4, r2
+; CHECK-NEXT:    sbcs.w r6, r9, r3
+; CHECK-NEXT:    vmov q2[3], q2[1], r7, r3
+; CHECK-NEXT:    csetm r6, lt
+; CHECK-NEXT:    bfi r5, r6, #8, #8
 ; CHECK-NEXT:    vmsr p0, r5
+; CHECK-NEXT:    mvn r5, #-2147483648
 ; CHECK-NEXT:    vpsel q2, q2, q0
-; CHECK-NEXT:    vmov r3, r4, d4
-; CHECK-NEXT:    subs r3, r3, r6
-; CHECK-NEXT:    sbcs r3, r4, #0
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    csetm r3, lt
-; CHECK-NEXT:    bfi r4, r3, #0, #8
-; CHECK-NEXT:    vmov r3, r5, d5
-; CHECK-NEXT:    subs r3, r3, r6
-; CHECK-NEXT:    sbcs r3, r5, #0
-; CHECK-NEXT:    csetm r3, lt
-; CHECK-NEXT:    bfi r4, r3, #8, #8
-; CHECK-NEXT:    vmsr p0, r4
+; CHECK-NEXT:    vmov r2, r3, d4
+; CHECK-NEXT:    subs r2, r2, r5
+; CHECK-NEXT:    sbcs r2, r3, #0
+; CHECK-NEXT:    mov.w r3, #0
+; CHECK-NEXT:    csetm r2, lt
+; CHECK-NEXT:    bfi r3, r2, #0, #8
+; CHECK-NEXT:    vmov r2, r4, d5
+; CHECK-NEXT:    subs r2, r2, r5
+; CHECK-NEXT:    sbcs r2, r4, #0
+; CHECK-NEXT:    csetm r2, lt
+; CHECK-NEXT:    bfi r3, r2, #8, #8
+; CHECK-NEXT:    vmsr p0, r3
 ; CHECK-NEXT:    vpsel q2, q2, q1
-; CHECK-NEXT:    vmov r3, s10
-; CHECK-NEXT:    vmov r4, s8
-; CHECK-NEXT:    strd r4, r3, [r2], #8
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    strd r3, r2, [r11], #8
 ; CHECK-NEXT:    le lr, .LBB0_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
-; CHECK-NEXT:    ldrd r7, r3, [sp] @ 8-byte Folded Reload
-; CHECK-NEXT:    cmp r7, r3
+; CHECK-NEXT:    ldrd r2, r3, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    beq .LBB0_8
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader
-; CHECK-NEXT:    sub.w lr, r3, r7
+; CHECK-NEXT:    sub.w lr, r3, r2
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    mov.w r1, #-2147483648
-; CHECK-NEXT:    mvn r2, #-2147483648
+; CHECK-NEXT:    mvn r3, #-2147483648
 ; CHECK-NEXT:  .LBB0_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r3, [r12], #4
+; CHECK-NEXT:    ldr r2, [r12], #4
 ; CHECK-NEXT:    ldr r4, [r8], #4
-; CHECK-NEXT:    smull r4, r3, r4, r3
-; CHECK-NEXT:    asrl r4, r3, #31
-; CHECK-NEXT:    subs r5, r1, r4
-; CHECK-NEXT:    sbcs.w r5, r0, r3
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r4, r4, r1, ne
-; CHECK-NEXT:    csel r3, r3, r0, ne
-; CHECK-NEXT:    subs r5, r4, r2
-; CHECK-NEXT:    sbcs r3, r3, #0
-; CHECK-NEXT:    csel r3, r4, r2, lt
-; CHECK-NEXT:    str r3, [r10], #4
+; CHECK-NEXT:    smull r2, r5, r4, r2
+; CHECK-NEXT:    asrl r2, r5, #31
+; CHECK-NEXT:    subs r4, r1, r2
+; CHECK-NEXT:    sbcs.w r4, r0, r5
+; CHECK-NEXT:    csel r2, r2, r1, lt
+; CHECK-NEXT:    csel r4, r5, r0, lt
+; CHECK-NEXT:    subs r5, r2, r3
+; CHECK-NEXT:    sbcs r4, r4, #0
+; CHECK-NEXT:    csel r2, r2, r3, lt
+; CHECK-NEXT:    str r2, [r10], #4
 ; CHECK-NEXT:    le lr, .LBB0_7
 ; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #12
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.9:
@@ -322,10 +319,8 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(ptr nocapture readonly %pSrcA, ptr no
 ; CHECK-NEXT:    asrl r4, r1, #31
 ; CHECK-NEXT:    subs r5, r3, r4
 ; CHECK-NEXT:    sbcs.w r5, r0, r1
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r4, r4, r3, ne
-; CHECK-NEXT:    csel r1, r1, r0, ne
+; CHECK-NEXT:    csel r4, r4, r3, lt
+; CHECK-NEXT:    csel r1, r1, r0, lt
 ; CHECK-NEXT:    subs r5, r4, r2
 ; CHECK-NEXT:    sbcs r1, r1, #0
 ; CHECK-NEXT:    csel r1, r4, r2, lt
diff --git a/llvm/test/CodeGen/Thumb2/mve-vadc-vsbc-spill.ll b/llvm/test/CodeGen/Thumb2/mve-vadc-vsbc-spill.ll
index 7fe8e94589a0c..8a0134a4c5ff2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vadc-vsbc-spill.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vadc-vsbc-spill.ll
@@ -59,7 +59,7 @@ define void @sub_256(<4 x i32> %a_low, <4 x i32> %a_high, <4 x i32> %b_low, <4 x
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    b use_int32x4_t
 entry:
-  %adc_low = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a_low, <4 x i32> %b_low, i32 0)
+  %adc_low = tail call { <4 x i32>, i32 } @llvm.arm.mve.vsbc.v4i32(<4 x i32> %a_low, <4 x i32> %b_low, i32 536870912)
   %carry = extractvalue { <4 x i32>, i32 } %adc_low, 1
   %result_low = extractvalue { <4 x i32>, i32 } %adc_low, 0
   tail call void @use_int32x4_t(<4 x i32> %result_low)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
index 6f2539e3cad9a..cc856730d90cf 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -5,25 +5,25 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, eq
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -41,13 +41,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -57,13 +57,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -82,25 +82,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, gt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, gt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -118,25 +118,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, ge
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ge
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -154,25 +154,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, mi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -190,25 +190,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, ls
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ls
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -226,13 +226,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -242,13 +242,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -295,25 +295,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, hi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, hi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -331,25 +331,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, pl
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, pl
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -367,25 +367,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, lt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, lt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -403,25 +403,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, le
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -439,25 +439,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, vc
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -476,25 +476,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    cset r0, vs
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    cset r1, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    cset r2, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vs
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
index d42c393743f4f..586b731c934be 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -5,25 +5,25 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, eq
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -44,13 +44,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -60,13 +60,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -88,25 +88,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, gt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, gt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -127,25 +127,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, ge
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ge
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -166,25 +166,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, mi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -205,25 +205,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, ls
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ls
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -244,13 +244,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -260,13 +260,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -319,25 +319,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, hi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, hi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -358,25 +358,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, pl
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, pl
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -397,25 +397,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, lt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, lt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -436,25 +436,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, le
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -475,25 +475,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, vc
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -515,25 +515,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    cset r0, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    cset r0, vs
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    cset r1, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    cset r2, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vs
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1614,25 +1614,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, eq
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1653,13 +1653,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1669,13 +1669,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, float %sr
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1697,25 +1697,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, gt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, gt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1736,25 +1736,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, ge
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ge
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1775,25 +1775,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, mi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1814,25 +1814,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, ls
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ls
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1853,13 +1853,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
 ; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1869,13 +1869,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, float %sr
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1928,25 +1928,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, hi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, hi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1967,25 +1967,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, pl
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, pl
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -2006,25 +2006,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, lt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, lt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -2045,25 +2045,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, le
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -2084,25 +2084,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, vc
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -2124,25 +2124,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s0
-; CHECK-MVE-NEXT:    cset r0, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s3
+; CHECK-MVE-NEXT:    cset r0, vs
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s4, s1
 ; CHECK-MVE-NEXT:    cset r1, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s4, s2
 ; CHECK-MVE-NEXT:    cset r2, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vs
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
index 718657839d38d..de8b413bf24e5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -5,25 +5,25 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, eq
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -41,13 +41,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -57,13 +57,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -82,25 +82,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, gt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, gt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -118,25 +118,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, ge
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ge
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -154,25 +154,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, mi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -190,25 +190,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, ls
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ls
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -226,13 +226,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -242,13 +242,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -295,25 +295,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, hi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, hi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -331,25 +331,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, pl
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, pl
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -367,25 +367,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, lt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, lt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -403,25 +403,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, le
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -439,25 +439,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
-; CHECK-MVE-NEXT:    cset r0, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    cset r0, vc
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
 ; CHECK-MVE-NEXT:    cset r1, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
 ; CHECK-MVE-NEXT:    cset r2, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -476,25 +476,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
-; CHECK-MVE-NEXT:    cset r0, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    cset r0, vs
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
 ; CHECK-MVE-NEXT:    cset r1, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
 ; CHECK-MVE-NEXT:    cset r2, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vs
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1530,25 +1530,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, eq
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1566,13 +1566,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1582,13 +1582,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, <4 x floa
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1607,25 +1607,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, mi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, mi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, mi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1643,25 +1643,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, ls
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, ls
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ls
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1679,25 +1679,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, gt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, gt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, gt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1715,25 +1715,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, ge
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, ge
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, ge
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1751,13 +1751,13 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    cset r0, eq
 ; CHECK-MVE-NEXT:    csinc r0, r0, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, eq
 ; CHECK-MVE-NEXT:    csinc r1, r1, zr, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1767,13 +1767,13 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, <4 x floa
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, eq
 ; CHECK-MVE-NEXT:    csinc r3, r3, zr, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1820,25 +1820,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, lt
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, lt
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, lt
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1856,25 +1856,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, le
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, le
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, le
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1892,25 +1892,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, hi
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, hi
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, hi
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1928,25 +1928,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    cset r0, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    cset r0, pl
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    cset r1, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    cset r2, pl
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, pl
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -1964,25 +1964,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
-; CHECK-MVE-NEXT:    cset r0, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    cset r0, vc
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
 ; CHECK-MVE-NEXT:    cset r1, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
 ; CHECK-MVE-NEXT:    cset r2, vc
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vc
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -2001,25 +2001,25 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_r_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s0, s0
-; CHECK-MVE-NEXT:    cset r0, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s3, s3
+; CHECK-MVE-NEXT:    cset r0, vs
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s1
 ; CHECK-MVE-NEXT:    cset r1, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vcmp.f32 s2, s2
 ; CHECK-MVE-NEXT:    cset r2, vs
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    cset r3, vs
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
+; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmaxv-vminv-scalar.ll b/llvm/test/CodeGen/Thumb2/mve-vmaxv-vminv-scalar.ll
index 898380760bd4d..35e578e425e74 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmaxv-vminv-scalar.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmaxv-vminv-scalar.ll
@@ -499,16 +499,12 @@ define arm_aapcs_vfpcc i64 @uminv2i64(<2 x i64> %vec, i64 %min) {
 ; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    subs.w r4, r2, r12
 ; CHECK-NEXT:    sbcs.w r4, r3, lr
-; CHECK-NEXT:    cset r4, lo
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r2, r2, r12, ne
-; CHECK-NEXT:    csel r3, r3, lr, ne
+; CHECK-NEXT:    csel r2, r2, r12, lo
+; CHECK-NEXT:    csel r3, r3, lr, lo
 ; CHECK-NEXT:    subs r4, r2, r0
 ; CHECK-NEXT:    sbcs.w r4, r3, r1
-; CHECK-NEXT:    cset r4, lo
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r2, r0, ne
-; CHECK-NEXT:    csel r1, r3, r1, ne
+; CHECK-NEXT:    csel r0, r2, r0, lo
+; CHECK-NEXT:    csel r1, r3, r1, lo
 ; CHECK-NEXT:    pop {r4, pc}
   %x = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> %vec)
   %cmp = icmp ult i64 %x, %min
@@ -525,16 +521,12 @@ define arm_aapcs_vfpcc i64 @sminv2i64(<2 x i64> %vec, i64 %min) {
 ; CHECK-NEXT:    vmov r2, r3, d0
 ; CHECK-NEXT:    subs.w r4, r2, r12
 ; CHECK-NEXT:    sbcs.w r4, r3, lr
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r2, r2, r12, ne
-; CHECK-NEXT:    csel r3, r3, lr, ne
+; CHECK-NEXT:    csel r2, r2, r12, lt
+; CHECK-NEXT:    csel r3, r3, lr, lt
 ; CHECK-NEXT:    subs r4, r2, r0
 ; CHECK-NEXT:    sbcs.w r4, r3, r1
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r2, r0, ne
-; CHECK-NEXT:    csel r1, r3, r1, ne
+; CHECK-NEXT:    csel r0, r2, r0, lt
+; CHECK-NEXT:    csel r1, r3, r1, lt
 ; CHECK-NEXT:    pop {r4, pc}
   %x = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> %vec)
   %cmp = icmp slt i64 %x, %min
@@ -551,16 +543,12 @@ define arm_aapcs_vfpcc i64 @umaxv2i64(<2 x i64> %vec, i64 %max) {
 ; CHECK-NEXT:    vmov r2, r3, d1
 ; CHECK-NEXT:    subs.w r4, r2, r12
 ; CHECK-NEXT:    sbcs.w r4, r3, lr
-; CHECK-NEXT:    cset r4, lo
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r2, r12, r2, ne
-; CHECK-NEXT:    csel r3, lr, r3, ne
+; CHECK-NEXT:    csel r2, r12, r2, lo
+; CHECK-NEXT:    csel r3, lr, r3, lo
 ; CHECK-NEXT:    subs r4, r0, r2
 ; CHECK-NEXT:    sbcs.w r4, r1, r3
-; CHECK-NEXT:    cset r4, lo
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r2, r0, ne
-; CHECK-NEXT:    csel r1, r3, r1, ne
+; CHECK-NEXT:    csel r0, r2, r0, lo
+; CHECK-NEXT:    csel r1, r3, r1, lo
 ; CHECK-NEXT:    pop {r4, pc}
   %x = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> %vec)
   %cmp = icmp ugt i64 %x, %max
@@ -577,16 +565,12 @@ define arm_aapcs_vfpcc i64 @smaxv2i64(<2 x i64> %vec, i64 %max) {
 ; CHECK-NEXT:    vmov r2, r3, d1
 ; CHECK-NEXT:    subs.w r4, r2, r12
 ; CHECK-NEXT:    sbcs.w r4, r3, lr
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r2, r12, r2, ne
-; CHECK-NEXT:    csel r3, lr, r3, ne
+; CHECK-NEXT:    csel r2, r12, r2, lt
+; CHECK-NEXT:    csel r3, lr, r3, lt
 ; CHECK-NEXT:    subs r4, r0, r2
 ; CHECK-NEXT:    sbcs.w r4, r1, r3
-; CHECK-NEXT:    cset r4, lt
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r0, r2, r0, ne
-; CHECK-NEXT:    csel r1, r3, r1, ne
+; CHECK-NEXT:    csel r0, r2, r0, lt
+; CHECK-NEXT:    csel r1, r3, r1, lt
 ; CHECK-NEXT:    pop {r4, pc}
   %x = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> %vec)
   %cmp = icmp sgt i64 %x, %max
diff --git a/llvm/test/CodeGen/Thumb2/mve-vqdmulh.ll b/llvm/test/CodeGen/Thumb2/mve-vqdmulh.ll
index 642ff69ded33f..bf0d92b5e0303 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vqdmulh.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vqdmulh.ll
@@ -502,8 +502,8 @@ define <2 x i64> @large_i128(<2 x double> %x) {
 ; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    csel r0, r0, r7, ne
 ; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r9, ne
 ; CHECK-NEXT:    csel r1, r1, r7, ne
+; CHECK-NEXT:    csel r2, r2, r9, ne
 ; CHECK-NEXT:    rsbs r7, r0, #0
 ; CHECK-NEXT:    sbcs.w r7, r4, r1
 ; CHECK-NEXT:    sbcs.w r2, r4, r2
@@ -521,8 +521,8 @@ define <2 x i64> @large_i128(<2 x double> %x) {
 ; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    csel r0, r0, r5, ne
 ; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r2, r2, r9, ne
 ; CHECK-NEXT:    csel r1, r1, r5, ne
+; CHECK-NEXT:    csel r2, r2, r9, ne
 ; CHECK-NEXT:    rsbs r5, r0, #0
 ; CHECK-NEXT:    sbcs.w r5, r4, r1
 ; CHECK-NEXT:    sbcs.w r2, r4, r2
diff --git a/llvm/test/CodeGen/Thumb2/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/Thumb2/umulo-128-legalisation-lowering.ll
index cff16c300e703..fe1d06cb39e16 100644
--- a/llvm/test/CodeGen/Thumb2/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/Thumb2/umulo-128-legalisation-lowering.ll
@@ -8,121 +8,119 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; THUMBV7-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; THUMBV7-NEXT:    .pad #44
 ; THUMBV7-NEXT:    sub sp, #44
-; THUMBV7-NEXT:    ldr.w lr, [sp, #88]
-; THUMBV7-NEXT:    mov r11, r0
-; THUMBV7-NEXT:    ldr r4, [sp, #96]
-; THUMBV7-NEXT:    ldr.w r12, [sp, #80]
-; THUMBV7-NEXT:    umull r1, r5, r2, lr
-; THUMBV7-NEXT:    umull r7, r6, r3, r4
-; THUMBV7-NEXT:    str r1, [sp, #40] @ 4-byte Spill
-; THUMBV7-NEXT:    ldr r1, [sp, #100]
-; THUMBV7-NEXT:    umull r4, r0, r4, r2
-; THUMBV7-NEXT:    str r7, [sp, #32] @ 4-byte Spill
-; THUMBV7-NEXT:    umull r7, r1, r1, r2
-; THUMBV7-NEXT:    str r4, [sp, #24] @ 4-byte Spill
-; THUMBV7-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; THUMBV7-NEXT:    ldr r0, [sp, #84]
-; THUMBV7-NEXT:    str r7, [sp, #20] @ 4-byte Spill
-; THUMBV7-NEXT:    ldr r7, [sp, #92]
-; THUMBV7-NEXT:    umull r10, r8, r0, lr
-; THUMBV7-NEXT:    umull r4, r9, r7, r12
-; THUMBV7-NEXT:    str r4, [sp, #8] @ 4-byte Spill
-; THUMBV7-NEXT:    umull r4, r0, r12, lr
-; THUMBV7-NEXT:    mov.w r12, #0
-; THUMBV7-NEXT:    umlal r5, r12, r3, lr
-; THUMBV7-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; THUMBV7-NEXT:    ldr.w r8, [sp, #88]
+; THUMBV7-NEXT:    mov r9, r0
+; THUMBV7-NEXT:    ldr r7, [sp, #96]
+; THUMBV7-NEXT:    ldr.w lr, [sp, #100]
+; THUMBV7-NEXT:    umull r0, r5, r2, r8
+; THUMBV7-NEXT:    ldr r4, [sp, #80]
+; THUMBV7-NEXT:    str r0, [sp, #32] @ 4-byte Spill
+; THUMBV7-NEXT:    umull r1, r0, r3, r7
 ; THUMBV7-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; THUMBV7-NEXT:    umull r4, r2, r2, r7
-; THUMBV7-NEXT:    ldr r0, [sp, #40] @ 4-byte Reload
-; THUMBV7-NEXT:    str r4, [sp, #28] @ 4-byte Spill
+; THUMBV7-NEXT:    umull r0, r11, lr, r2
+; THUMBV7-NEXT:    str r1, [sp, #20] @ 4-byte Spill
+; THUMBV7-NEXT:    ldr r1, [sp, #92]
+; THUMBV7-NEXT:    str r0, [sp] @ 4-byte Spill
+; THUMBV7-NEXT:    umull r0, r10, r7, r2
+; THUMBV7-NEXT:    mov r7, r1
+; THUMBV7-NEXT:    umull r6, r12, r1, r4
+; THUMBV7-NEXT:    str r0, [sp, #40] @ 4-byte Spill
+; THUMBV7-NEXT:    ldr r0, [sp, #84]
+; THUMBV7-NEXT:    str r6, [sp, #24] @ 4-byte Spill
+; THUMBV7-NEXT:    umull r6, r1, r0, r8
+; THUMBV7-NEXT:    str r6, [sp, #16] @ 4-byte Spill
+; THUMBV7-NEXT:    umull r6, r2, r2, r7
+; THUMBV7-NEXT:    mov r7, r4
+; THUMBV7-NEXT:    strd r6, r2, [sp, #8] @ 8-byte Folded Spill
+; THUMBV7-NEXT:    umull r2, r6, r4, r8
 ; THUMBV7-NEXT:    str r2, [sp, #36] @ 4-byte Spill
-; THUMBV7-NEXT:    str.w r0, [r11]
-; THUMBV7-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
+; THUMBV7-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
+; THUMBV7-NEXT:    str r6, [sp, #28] @ 4-byte Spill
+; THUMBV7-NEXT:    movs r6, #0
+; THUMBV7-NEXT:    str.w r2, [r9]
+; THUMBV7-NEXT:    umlal r5, r6, r3, r8
 ; THUMBV7-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
-; THUMBV7-NEXT:    add r2, r0
-; THUMBV7-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; THUMBV7-NEXT:    adds.w lr, r0, r2
+; THUMBV7-NEXT:    ldr r4, [sp] @ 4-byte Reload
+; THUMBV7-NEXT:    add r4, r2
+; THUMBV7-NEXT:    adds.w r2, r10, r4
+; THUMBV7-NEXT:    str r2, [sp, #20] @ 4-byte Spill
 ; THUMBV7-NEXT:    mov.w r2, #0
-; THUMBV7-NEXT:    adc r0, r2, #0
-; THUMBV7-NEXT:    str r0, [sp, #32] @ 4-byte Spill
-; THUMBV7-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; THUMBV7-NEXT:    add.w r4, r10, r0
-; THUMBV7-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; THUMBV7-NEXT:    adds r4, r4, r0
-; THUMBV7-NEXT:    adc r0, r2, #0
-; THUMBV7-NEXT:    str r0, [sp, #40] @ 4-byte Spill
-; THUMBV7-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; THUMBV7-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; THUMBV7-NEXT:    adds.w r10, r2, r0
-; THUMBV7-NEXT:    mov r2, r3
-; THUMBV7-NEXT:    adc.w r0, r4, lr
-; THUMBV7-NEXT:    ldr.w lr, [sp, #100]
+; THUMBV7-NEXT:    adc r2, r2, #0
+; THUMBV7-NEXT:    cmp.w r12, #0
+; THUMBV7-NEXT:    str r2, [sp, #32] @ 4-byte Spill
+; THUMBV7-NEXT:    it ne
+; THUMBV7-NEXT:    movne.w r12, #1
 ; THUMBV7-NEXT:    cmp r1, #0
-; THUMBV7-NEXT:    str r0, [sp, #24] @ 4-byte Spill
+; THUMBV7-NEXT:    ldr r2, [sp, #96]
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r1, #1
-; THUMBV7-NEXT:    cmp r3, #0
-; THUMBV7-NEXT:    mov r0, lr
+; THUMBV7-NEXT:    orrs.w r10, r7, r0
 ; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne r2, #1
-; THUMBV7-NEXT:    cmp.w lr, #0
+; THUMBV7-NEXT:    movne.w r10, #1
+; THUMBV7-NEXT:    orrs.w r7, r2, lr
+; THUMBV7-NEXT:    ldr r2, [sp, #92]
+; THUMBV7-NEXT:    it ne
+; THUMBV7-NEXT:    movne r7, #1
+; THUMBV7-NEXT:    cmp r0, #0
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r0, #1
-; THUMBV7-NEXT:    ldr r4, [sp, #28] @ 4-byte Reload
-; THUMBV7-NEXT:    ands r0, r2
-; THUMBV7-NEXT:    orrs r1, r0
-; THUMBV7-NEXT:    adds r5, r5, r4
-; THUMBV7-NEXT:    str.w r5, [r11, #4]
-; THUMBV7-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
-; THUMBV7-NEXT:    mov.w r5, #0
-; THUMBV7-NEXT:    adcs.w r0, r0, r12
-; THUMBV7-NEXT:    adc r2, r5, #0
-; THUMBV7-NEXT:    cmp r6, #0
+; THUMBV7-NEXT:    cmp r2, #0
+; THUMBV7-NEXT:    mov r4, r2
+; THUMBV7-NEXT:    mov r8, r2
 ; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne r6, #1
-; THUMBV7-NEXT:    orrs r1, r6
-; THUMBV7-NEXT:    ldr r6, [sp, #84]
-; THUMBV7-NEXT:    umlal r0, r2, r3, r7
-; THUMBV7-NEXT:    ldr r3, [sp, #32] @ 4-byte Reload
-; THUMBV7-NEXT:    cmp r7, #0
+; THUMBV7-NEXT:    movne r4, #1
+; THUMBV7-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; THUMBV7-NEXT:    ands r0, r4
+; THUMBV7-NEXT:    movs r4, #0
+; THUMBV7-NEXT:    adds r5, r5, r2
+; THUMBV7-NEXT:    str.w r5, [r9, #4]
+; THUMBV7-NEXT:    orr.w r0, r0, r1
+; THUMBV7-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
+; THUMBV7-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; THUMBV7-NEXT:    and.w r5, r10, r7
+; THUMBV7-NEXT:    orr.w r0, r0, r12
+; THUMBV7-NEXT:    mov.w r12, #0
+; THUMBV7-NEXT:    add r1, r2
+; THUMBV7-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; THUMBV7-NEXT:    adcs r2, r6
+; THUMBV7-NEXT:    ldr r6, [sp, #28] @ 4-byte Reload
+; THUMBV7-NEXT:    adc r7, r4, #0
+; THUMBV7-NEXT:    adds r1, r1, r6
+; THUMBV7-NEXT:    umlal r2, r7, r3, r8
+; THUMBV7-NEXT:    adc r4, r4, #0
+; THUMBV7-NEXT:    orrs r0, r4
+; THUMBV7-NEXT:    orrs r0, r5
+; THUMBV7-NEXT:    ldrd r5, r4, [sp, #36] @ 8-byte Folded Reload
+; THUMBV7-NEXT:    adds r5, r5, r4
+; THUMBV7-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
+; THUMBV7-NEXT:    adcs r1, r4
+; THUMBV7-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
+; THUMBV7-NEXT:    cmp r4, #0
 ; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne r7, #1
-; THUMBV7-NEXT:    orrs r1, r3
-; THUMBV7-NEXT:    mov r3, r6
-; THUMBV7-NEXT:    cmp r6, #0
+; THUMBV7-NEXT:    movne r4, #1
+; THUMBV7-NEXT:    cmp r3, #0
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r3, #1
-; THUMBV7-NEXT:    cmp.w r8, #0
-; THUMBV7-NEXT:    and.w r3, r3, r7
-; THUMBV7-NEXT:    ldr r7, [sp, #80]
-; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne.w r8, #1
-; THUMBV7-NEXT:    cmp.w r9, #0
-; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne.w r9, #1
-; THUMBV7-NEXT:    orrs r7, r6
-; THUMBV7-NEXT:    ldr r6, [sp, #96]
+; THUMBV7-NEXT:    cmp.w lr, #0
 ; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne r7, #1
-; THUMBV7-NEXT:    orr.w r3, r3, r8
-; THUMBV7-NEXT:    orrs.w r6, r6, lr
-; THUMBV7-NEXT:    orr.w r3, r3, r9
+; THUMBV7-NEXT:    movne.w lr, #1
+; THUMBV7-NEXT:    cmp.w r11, #0
 ; THUMBV7-NEXT:    it ne
-; THUMBV7-NEXT:    movne r6, #1
-; THUMBV7-NEXT:    adds.w r0, r0, r10
-; THUMBV7-NEXT:    str.w r0, [r11, #8]
-; THUMBV7-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; THUMBV7-NEXT:    adcs r0, r2
-; THUMBV7-NEXT:    str.w r0, [r11, #12]
-; THUMBV7-NEXT:    ldr r0, [sp, #40] @ 4-byte Reload
-; THUMBV7-NEXT:    and.w r2, r7, r6
-; THUMBV7-NEXT:    orr.w r0, r0, r3
-; THUMBV7-NEXT:    orr.w r0, r0, r2
+; THUMBV7-NEXT:    movne.w r11, #1
+; THUMBV7-NEXT:    adds r2, r2, r5
+; THUMBV7-NEXT:    and.w r3, r3, lr
+; THUMBV7-NEXT:    str.w r2, [r9, #8]
+; THUMBV7-NEXT:    adcs r1, r7
+; THUMBV7-NEXT:    str.w r1, [r9, #12]
+; THUMBV7-NEXT:    orr.w r1, r3, r11
+; THUMBV7-NEXT:    ldr r2, [sp, #32] @ 4-byte Reload
+; THUMBV7-NEXT:    orr.w r1, r1, r4
+; THUMBV7-NEXT:    orr.w r1, r1, r2
 ; THUMBV7-NEXT:    orr.w r0, r0, r1
-; THUMBV7-NEXT:    adc r1, r5, #0
+; THUMBV7-NEXT:    adc r1, r12, #0
 ; THUMBV7-NEXT:    orrs r0, r1
 ; THUMBV7-NEXT:    and r0, r0, #1
-; THUMBV7-NEXT:    strb.w r0, [r11, #16]
+; THUMBV7-NEXT:    strb.w r0, [r9, #16]
 ; THUMBV7-NEXT:    add sp, #44
 ; THUMBV7-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 start:
diff --git a/llvm/test/CodeGen/Thumb2/umulo-64-legalisation-lowering.ll b/llvm/test/CodeGen/Thumb2/umulo-64-legalisation-lowering.ll
index 161adf7e7d763..55e917159fce9 100644
--- a/llvm/test/CodeGen/Thumb2/umulo-64-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/Thumb2/umulo-64-legalisation-lowering.ll
@@ -4,33 +4,32 @@
 define { i64, i8 } @mulodi_test(i64 %l, i64 %r) unnamed_addr #0 {
 ; THUMBV7-LABEL: mulodi_test:
 ; THUMBV7:       @ %bb.0: @ %start
-; THUMBV7-NEXT:    .save {r4, r5, r6, lr}
-; THUMBV7-NEXT:    push {r4, r5, r6, lr}
+; THUMBV7-NEXT:    .save {r4, r5, r7, lr}
+; THUMBV7-NEXT:    push {r4, r5, r7, lr}
 ; THUMBV7-NEXT:    umull r12, lr, r3, r0
-; THUMBV7-NEXT:    movs r6, #0
-; THUMBV7-NEXT:    umull r4, r5, r1, r2
-; THUMBV7-NEXT:    umull r0, r2, r0, r2
-; THUMBV7-NEXT:    add r4, r12
-; THUMBV7-NEXT:    adds.w r12, r2, r4
-; THUMBV7-NEXT:    adc r2, r6, #0
 ; THUMBV7-NEXT:    cmp r3, #0
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r3, #1
 ; THUMBV7-NEXT:    cmp r1, #0
+; THUMBV7-NEXT:    umull r0, r4, r0, r2
+; THUMBV7-NEXT:    umull r2, r5, r1, r2
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r1, #1
+; THUMBV7-NEXT:    ands r1, r3
 ; THUMBV7-NEXT:    cmp r5, #0
-; THUMBV7-NEXT:    and.w r1, r1, r3
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne r5, #1
 ; THUMBV7-NEXT:    orrs r1, r5
 ; THUMBV7-NEXT:    cmp.w lr, #0
 ; THUMBV7-NEXT:    it ne
 ; THUMBV7-NEXT:    movne.w lr, #1
-; THUMBV7-NEXT:    orr.w r1, r1, lr
-; THUMBV7-NEXT:    orrs r2, r1
-; THUMBV7-NEXT:    mov r1, r12
-; THUMBV7-NEXT:    pop {r4, r5, r6, pc}
+; THUMBV7-NEXT:    orr.w r3, r1, lr
+; THUMBV7-NEXT:    add.w r1, r2, r12
+; THUMBV7-NEXT:    movs r2, #0
+; THUMBV7-NEXT:    adds r1, r1, r4
+; THUMBV7-NEXT:    adc r2, r2, #0
+; THUMBV7-NEXT:    orrs r2, r3
+; THUMBV7-NEXT:    pop {r4, r5, r7, pc}
 start:
   %0 = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %l, i64 %r) #2
   %1 = extractvalue { i64, i1 } %0, 0
diff --git a/llvm/test/CodeGen/WebAssembly/call-indirect.ll b/llvm/test/CodeGen/WebAssembly/call-indirect.ll
index 55a654f358490..e0a0d14deacba 100644
--- a/llvm/test/CodeGen/WebAssembly/call-indirect.ll
+++ b/llvm/test/CodeGen/WebAssembly/call-indirect.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -mattr=-reference-types -O2 | FileCheck --check-prefixes=CHECK,NOREF %s
-; RUN: llc < %s -asm-verbose=false -mattr=+reference-types -O2 | FileCheck --check-prefixes=CHECK,REF %s
+; RUN: llc < %s -asm-verbose=false -mattr=-reference-types,-call-indirect-overlong -O2 | FileCheck --check-prefixes=CHECK,NOREF %s
+; RUN: llc < %s -asm-verbose=false -mattr=+call-indirect-overlong -O2 | FileCheck --check-prefixes=CHECK,REF %s
 ; RUN: llc < %s -asm-verbose=false -O2 --filetype=obj | obj2yaml | FileCheck --check-prefix=OBJ %s
 
 ; Test that compilation units with call_indirect but without any
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
index 3b312dabcd84d..ab9023cbac604 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.ll
@@ -1,9 +1,9 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,bulk-memory | FileCheck %s
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,bulk-memory
-; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -verify-machineinstrs -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory | FileCheck %s --check-prefix=NOOPT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
+; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -verify-machineinstrs -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt | FileCheck %s --check-prefix=NOOPT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index 6df626df08883..22fda36c25bfd 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -1,9 +1,9 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,bulk-memory | FileCheck %s
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,bulk-memory
-; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -verify-machineinstrs -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory | FileCheck %s --check-prefix=NOOPT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
-; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
+; RUN: llc < %s -O0 -disable-wasm-fallthrough-return-opt -verify-machineinstrs -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt | FileCheck %s --check-prefix=NOOPT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt -wasm-disable-ehpad-sort -stats 2>&1 | FileCheck %s --check-prefix=NOSORT
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -disable-block-placement -verify-machineinstrs -fast-isel=false -machine-sink-split-probability-threshold=0 -cgp-freq-ratio-to-skip-merge=1000 -wasm-enable-eh -wasm-enable-exnref -exception-model=wasm -mattr=+exception-handling,-bulk-memory,-bulk-memory-opt -wasm-disable-ehpad-sort | FileCheck %s --check-prefix=NOSORT-LOCALS
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/CodeGen/WebAssembly/data-align.ll b/llvm/test/CodeGen/WebAssembly/data-align.ll
new file mode 100644
index 0000000000000..ca7f1226d3ec0
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/data-align.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s -march=wasm32 | FileCheck %s
+; RUN: llc < %s -march=wasm64 | FileCheck %s
+
+; CHECK:      .Li8:
+; CHECK-DAG: .size .Li8, 1
+@i8 = private constant i8 42
+
+; CHECK:      .p2align 1
+; CHECK-NEXT: .Li16:
+; CHECK-DAG:  .size .Li16, 2
+@i16 = private constant i16 42
+
+; CHECK:      .p2align 2
+; CHECK-NEXT: .Li32:
+; CHECK-DAG:  .size .Li32, 4
+@i32 = private constant i32 42
+
+; CHECK:      .p2align 3
+; CHECK-NEXT: .Li64:
+; CHECK-DAG:  .size .Li64, 8
+@i64 = private constant i64 42
+
+; CHECK:      .p2align 4
+; CHECK-NEXT: .Li128:
+; CHECK-DAG:  .size .Li128, 16
+@i128 = private constant i128 42
diff --git a/llvm/test/CodeGen/WebAssembly/disable-feature.ll b/llvm/test/CodeGen/WebAssembly/disable-feature.ll
index 0684432a114df..5f7275f3699ed 100644
--- a/llvm/test/CodeGen/WebAssembly/disable-feature.ll
+++ b/llvm/test/CodeGen/WebAssembly/disable-feature.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mattr=-sign-ext,-bulk-memory | FileCheck %s
+; RUN: llc < %s -mattr=-sign-ext,-bulk-memory,-bulk-memory-opt | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
@@ -21,7 +21,7 @@ define i8 @not_use_extend8_s(i8 %v, i8 %x) {
   ret i8 %a
 }
 
-attributes #0 = { "target-features"="+bulk-memory," }
+attributes #0 = { "target-features"="+bulk-memory-opt" }
 
 declare void @llvm.memset.p0.i32(ptr, i8, i32, i1)
 
diff --git a/llvm/test/CodeGen/WebAssembly/fpclamptosat.ll b/llvm/test/CodeGen/WebAssembly/fpclamptosat.ll
index 58e3f0dc2a93c..137994ceac132 100644
--- a/llvm/test/CodeGen/WebAssembly/fpclamptosat.ll
+++ b/llvm/test/CodeGen/WebAssembly/fpclamptosat.ll
@@ -524,9 +524,7 @@ define i64 @utest_f64i64(double %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixunsdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -563,9 +561,7 @@ define i64 @utest_f64i64_cse_combine(double %x) #0 {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixunsdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -602,9 +598,7 @@ define i64 @ustest_f64i64(double %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -661,9 +655,7 @@ define i64 @ustest_f64i64_cse_combine(double %x) #0 {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -727,9 +719,7 @@ define i64 @utest_f32i64(float %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -781,9 +771,7 @@ define i64 @ustest_f32i64(float %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -840,9 +828,7 @@ define i64 @ustest_f32i64_cse_combine(float %x) #0 {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -910,9 +896,7 @@ define i64 @utesth_f16i64(half %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -981,9 +965,7 @@ define i64 @ustest_f16i64(half %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1471,9 +1453,7 @@ define i64 @utest_f64i64_mm(double %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixunsdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1509,9 +1489,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1573,9 +1551,7 @@ define i64 @utest_f32i64_mm(float %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1611,9 +1587,7 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1679,9 +1653,7 @@ define i64 @utesth_f16i64_mm(half %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1719,9 +1691,7 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
diff --git a/llvm/test/CodeGen/WebAssembly/fpclamptosat_vec.ll b/llvm/test/CodeGen/WebAssembly/fpclamptosat_vec.ll
index 8f85575c1cf43..1feb5feb7a9ee 100644
--- a/llvm/test/CodeGen/WebAssembly/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/WebAssembly/fpclamptosat_vec.ll
@@ -685,19 +685,13 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -800,19 +794,13 @@ define <2 x i64> @utest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixunsdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -863,19 +851,13 @@ define <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -964,19 +946,13 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1079,19 +1055,13 @@ define <2 x i64> @utest_f32i64(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1142,19 +1112,13 @@ define <2 x i64> @ustest_f32i64(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -1245,19 +1209,13 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
@@ -1362,19 +1320,13 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
@@ -1427,19 +1379,13 @@ define <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
@@ -2163,19 +2109,13 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2276,19 +2216,13 @@ define <2 x i64> @utest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixunsdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2338,19 +2272,13 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NEXT:    f64x2.extract_lane 0
 ; CHECK-NEXT:    call __fixdfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2421,19 +2349,13 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2534,19 +2456,13 @@ define <2 x i64> @utest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2596,19 +2512,13 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NEXT:    f32x4.extract_lane 0
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    i64.load 0
@@ -2681,19 +2591,13 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
@@ -2796,19 +2700,13 @@ define <2 x i64> @utesth_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixunssfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
@@ -2860,19 +2758,13 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    call __fixsfti
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 24
 ; CHECK-NEXT:    local.set 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 16
 ; CHECK-NEXT:    local.set 4
 ; CHECK-NEXT:    local.get 2
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    i64.load 0
+; CHECK-NEXT:    i64.load 8
 ; CHECK-NEXT:    local.set 5
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i64.load 0
diff --git a/llvm/test/CodeGen/WebAssembly/function-pointer64.ll b/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
index d5d10b00824fe..2accd4151767f 100644
--- a/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
+++ b/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -mattr=-reference-types -O2 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -mattr=-reference-types,-call-indirect-overlong -O2 | FileCheck %s
 ; RUN: llc < %s -asm-verbose=false -mattr=+reference-types -O2 | FileCheck --check-prefix=REF %s
-; RUN: llc < %s -asm-verbose=false -mattr=-reference-types -O2 --filetype=obj | obj2yaml | FileCheck --check-prefix=YAML %s
+; RUN: llc < %s -asm-verbose=false -mattr=-reference-types,-call-indirect-overlong -O2 --filetype=obj | obj2yaml | FileCheck --check-prefix=YAML %s
 
 ; This tests pointer features that may codegen differently in wasm64.
 
diff --git a/llvm/test/CodeGen/WebAssembly/i128.ll b/llvm/test/CodeGen/WebAssembly/i128.ll
index eae7f5f834dc0..d9bec9b8ae887 100644
--- a/llvm/test/CodeGen/WebAssembly/i128.ll
+++ b/llvm/test/CodeGen/WebAssembly/i128.ll
@@ -63,31 +63,29 @@ define i128 @mul128(i128 %x, i128 %y) {
 ; CHECK:         .functype mul128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push4=, __stack_pointer
-; CHECK-NEXT:    i32.const $push5=, 16
-; CHECK-NEXT:    i32.sub $push9=, $pop4, $pop5
-; CHECK-NEXT:    local.tee $push8=, 5, $pop9
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    local.get $push10=, 4
-; CHECK-NEXT:    call __multi3, $pop14, $pop13, $pop12, $pop11, $pop10
+; CHECK-NEXT:    global.get $push2=, __stack_pointer
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    i32.sub $push7=, $pop2, $pop3
+; CHECK-NEXT:    local.tee $push6=, 5, $pop7
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    local.get $push8=, 4
+; CHECK-NEXT:    call __multi3, $pop12, $pop11, $pop10, $pop9, $pop8
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push0=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop0
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop15, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    i64.store 8($pop16), $pop2
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push1=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop1
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push3=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop3
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.add $push7=, $pop19, $pop6
-; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.add $push5=, $pop17, $pop4
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
 ; CHECK-NEXT:    return
   %a = mul i128 %x, %y
   ret i128 %a
@@ -98,31 +96,29 @@ define i128 @sdiv128(i128 %x, i128 %y) {
 ; CHECK:         .functype sdiv128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push4=, __stack_pointer
-; CHECK-NEXT:    i32.const $push5=, 16
-; CHECK-NEXT:    i32.sub $push9=, $pop4, $pop5
-; CHECK-NEXT:    local.tee $push8=, 5, $pop9
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    local.get $push10=, 4
-; CHECK-NEXT:    call __divti3, $pop14, $pop13, $pop12, $pop11, $pop10
+; CHECK-NEXT:    global.get $push2=, __stack_pointer
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    i32.sub  $push7=, $pop2, $pop3
+; CHECK-NEXT:    local.tee $push6=, 5, $pop7
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    local.get $push8=, 4
+; CHECK-NEXT:    call __divti3, $pop12, $pop11, $pop10, $pop9, $pop8
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push0=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop0
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop15, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    i64.store 8($pop16), $pop2
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push1=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop1
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push3=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop3
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.add $push7=, $pop19, $pop6
-; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.add  $push5=, $pop17, $pop4
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
 ; CHECK-NEXT:    return
   %a = sdiv i128 %x, %y
   ret i128 %a
@@ -133,31 +129,29 @@ define i128 @udiv128(i128 %x, i128 %y) {
 ; CHECK:         .functype udiv128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push4=, __stack_pointer
-; CHECK-NEXT:    i32.const $push5=, 16
-; CHECK-NEXT:    i32.sub $push9=, $pop4, $pop5
-; CHECK-NEXT:    local.tee $push8=, 5, $pop9
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    local.get $push10=, 4
-; CHECK-NEXT:    call __udivti3, $pop14, $pop13, $pop12, $pop11, $pop10
+; CHECK-NEXT:    global.get $push2=, __stack_pointer
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    i32.sub  $push7=, $pop2, $pop3
+; CHECK-NEXT:    local.tee $push6=, 5, $pop7
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    local.get $push8=, 4
+; CHECK-NEXT:    call __udivti3, $pop12, $pop11, $pop10, $pop9, $pop8
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push0=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop0
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop15, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    i64.store 8($pop16), $pop2
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push1=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop1
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push3=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop3
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.add $push7=, $pop19, $pop6
-; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.add  $push5=, $pop17, $pop4
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
 ; CHECK-NEXT:    return
   %a = udiv i128 %x, %y
   ret i128 %a
@@ -168,31 +162,29 @@ define i128 @srem128(i128 %x, i128 %y) {
 ; CHECK:         .functype srem128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push4=, __stack_pointer
-; CHECK-NEXT:    i32.const $push5=, 16
-; CHECK-NEXT:    i32.sub $push9=, $pop4, $pop5
-; CHECK-NEXT:    local.tee $push8=, 5, $pop9
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    local.get $push10=, 4
-; CHECK-NEXT:    call __modti3, $pop14, $pop13, $pop12, $pop11, $pop10
+; CHECK-NEXT:    global.get $push2=, __stack_pointer
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    i32.sub  $push7=, $pop2, $pop3
+; CHECK-NEXT:    local.tee $push6=, 5, $pop7
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    local.get $push8=, 4
+; CHECK-NEXT:    call __modti3, $pop12, $pop11, $pop10, $pop9, $pop8
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push0=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop0
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop15, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    i64.store 8($pop16), $pop2
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push1=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop1
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push3=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop3
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.add $push7=, $pop19, $pop6
-; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.add  $push5=, $pop17, $pop4
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
 ; CHECK-NEXT:    return
   %a = srem i128 %x, %y
   ret i128 %a
@@ -203,31 +195,29 @@ define i128 @urem128(i128 %x, i128 %y) {
 ; CHECK:         .functype urem128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push4=, __stack_pointer
-; CHECK-NEXT:    i32.const $push5=, 16
-; CHECK-NEXT:    i32.sub $push9=, $pop4, $pop5
-; CHECK-NEXT:    local.tee $push8=, 5, $pop9
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    local.get $push10=, 4
-; CHECK-NEXT:    call __umodti3, $pop14, $pop13, $pop12, $pop11, $pop10
+; CHECK-NEXT:    global.get $push2=, __stack_pointer
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    i32.sub  $push7=, $pop2, $pop3
+; CHECK-NEXT:    local.tee $push6=, 5, $pop7
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    local.get $push8=, 4
+; CHECK-NEXT:    call __umodti3, $pop12, $pop11, $pop10, $pop9, $pop8
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push0=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop0
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop15, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    i64.store 8($pop16), $pop2
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push1=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop1
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push3=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop3
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.add $push7=, $pop19, $pop6
-; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.add  $push5=, $pop17, $pop4
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
 ; CHECK-NEXT:    return
   %a = urem i128 %x, %y
   ret i128 %a
@@ -295,31 +285,29 @@ define i128 @shl128(i128 %x, i128 %y) {
 ; CHECK:         .functype shl128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push5=, __stack_pointer
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.sub $push10=, $pop5, $pop6
-; CHECK-NEXT:    local.tee $push9=, 5, $pop10
-; CHECK-NEXT:    global.set __stack_pointer, $pop9
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop11
-; CHECK-NEXT:    call __ashlti3, $pop14, $pop13, $pop12, $pop0
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.sub  $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, 5, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop9
+; CHECK-NEXT:    call __ashlti3, $pop12, $pop11, $pop10, $pop0
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push1=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop1
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push1=, 8
-; CHECK-NEXT:    i32.add $push2=, $pop15, $pop1
-; CHECK-NEXT:    i64.load $push3=, 0($pop2)
-; CHECK-NEXT:    i64.store 8($pop16), $pop3
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push2=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop2
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push4=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop4
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push7=, 16
-; CHECK-NEXT:    i32.add $push8=, $pop19, $pop7
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
+; CHECK-NEXT:    i32.const $push5=, 16
+; CHECK-NEXT:    i32.add  $push6=, $pop17, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
 ; CHECK-NEXT:    return
   %a = shl i128 %x, %y
   ret i128 %a
@@ -330,31 +318,29 @@ define i128 @shr128(i128 %x, i128 %y) {
 ; CHECK:         .functype shr128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push5=, __stack_pointer
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.sub $push10=, $pop5, $pop6
-; CHECK-NEXT:    local.tee $push9=, 5, $pop10
-; CHECK-NEXT:    global.set __stack_pointer, $pop9
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop11
-; CHECK-NEXT:    call __lshrti3, $pop14, $pop13, $pop12, $pop0
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.sub  $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, 5, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop9
+; CHECK-NEXT:    call __lshrti3, $pop12, $pop11, $pop10, $pop0
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push1=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop1
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push1=, 8
-; CHECK-NEXT:    i32.add $push2=, $pop15, $pop1
-; CHECK-NEXT:    i64.load $push3=, 0($pop2)
-; CHECK-NEXT:    i64.store 8($pop16), $pop3
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push2=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop2
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push4=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop4
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push7=, 16
-; CHECK-NEXT:    i32.add $push8=, $pop19, $pop7
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
+; CHECK-NEXT:    i32.const $push5=, 16
+; CHECK-NEXT:    i32.add  $push6=, $pop17, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
 ; CHECK-NEXT:    return
   %a = lshr i128 %x, %y
   ret i128 %a
@@ -365,31 +351,29 @@ define i128 @sar128(i128 %x, i128 %y) {
 ; CHECK:         .functype sar128 (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push5=, __stack_pointer
-; CHECK-NEXT:    i32.const $push6=, 16
-; CHECK-NEXT:    i32.sub $push10=, $pop5, $pop6
-; CHECK-NEXT:    local.tee $push9=, 5, $pop10
-; CHECK-NEXT:    global.set __stack_pointer, $pop9
-; CHECK-NEXT:    local.get $push14=, 5
-; CHECK-NEXT:    local.get $push13=, 1
-; CHECK-NEXT:    local.get $push12=, 2
-; CHECK-NEXT:    local.get $push11=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop11
-; CHECK-NEXT:    call __ashrti3, $pop14, $pop13, $pop12, $pop0
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 16
+; CHECK-NEXT:    i32.sub  $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, 5, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    local.get $push12=, 5
+; CHECK-NEXT:    local.get $push11=, 1
+; CHECK-NEXT:    local.get $push10=, 2
+; CHECK-NEXT:    local.get $push9=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop9
+; CHECK-NEXT:    call __ashrti3, $pop12, $pop11, $pop10, $pop0
+; CHECK-NEXT:    local.get $push14=, 0
+; CHECK-NEXT:    local.get $push13=, 5
+; CHECK-NEXT:    i64.load $push1=, 8($pop13)
+; CHECK-NEXT:    i64.store 8($pop14), $pop1
 ; CHECK-NEXT:    local.get $push16=, 0
 ; CHECK-NEXT:    local.get $push15=, 5
-; CHECK-NEXT:    i32.const $push1=, 8
-; CHECK-NEXT:    i32.add $push2=, $pop15, $pop1
-; CHECK-NEXT:    i64.load $push3=, 0($pop2)
-; CHECK-NEXT:    i64.store 8($pop16), $pop3
-; CHECK-NEXT:    local.get $push18=, 0
+; CHECK-NEXT:    i64.load $push2=, 0($pop15)
+; CHECK-NEXT:    i64.store 0($pop16), $pop2
 ; CHECK-NEXT:    local.get $push17=, 5
-; CHECK-NEXT:    i64.load $push4=, 0($pop17)
-; CHECK-NEXT:    i64.store 0($pop18), $pop4
-; CHECK-NEXT:    local.get $push19=, 5
-; CHECK-NEXT:    i32.const $push7=, 16
-; CHECK-NEXT:    i32.add $push8=, $pop19, $pop7
-; CHECK-NEXT:    global.set __stack_pointer, $pop8
+; CHECK-NEXT:    i32.const $push5=, 16
+; CHECK-NEXT:    i32.add  $push6=, $pop17, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
 ; CHECK-NEXT:    return
   %a = ashr i128 %x, %y
   ret i128 %a
@@ -529,51 +513,45 @@ define i128 @rotl(i128 %x, i128 %y) {
 ; CHECK:         .functype rotl (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push11=, __stack_pointer
-; CHECK-NEXT:    i32.const $push12=, 32
-; CHECK-NEXT:    i32.sub $push23=, $pop11, $pop12
-; CHECK-NEXT:    local.tee $push22=, 5, $pop23
-; CHECK-NEXT:    global.set __stack_pointer, $pop22
-; CHECK-NEXT:    local.get $push24=, 5
-; CHECK-NEXT:    i32.const $push17=, 16
-; CHECK-NEXT:    i32.add $push18=, $pop24, $pop17
-; CHECK-NEXT:    local.get $push27=, 1
-; CHECK-NEXT:    local.get $push26=, 2
-; CHECK-NEXT:    local.get $push25=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push21=, $pop25
-; CHECK-NEXT:    local.tee $push20=, 6, $pop21
-; CHECK-NEXT:    call __ashlti3, $pop18, $pop27, $pop26, $pop20
-; CHECK-NEXT:    local.get $push31=, 5
-; CHECK-NEXT:    local.get $push30=, 1
-; CHECK-NEXT:    local.get $push29=, 2
+; CHECK-NEXT:    global.get $push8=, __stack_pointer
+; CHECK-NEXT:    i32.const $push9=, 32
+; CHECK-NEXT:    i32.sub  $push17=, $pop8, $pop9
+; CHECK-NEXT:    local.tee $push16=, 5, $pop17
+; CHECK-NEXT:    global.set __stack_pointer, $pop16
+; CHECK-NEXT:    local.get $push18=, 5
+; CHECK-NEXT:    i32.const $push12=, 16
+; CHECK-NEXT:    i32.add  $push13=, $pop18, $pop12
+; CHECK-NEXT:    local.get $push21=, 1
+; CHECK-NEXT:    local.get $push20=, 2
+; CHECK-NEXT:    local.get $push19=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push15=, $pop19
+; CHECK-NEXT:    local.tee $push14=, 6, $pop15
+; CHECK-NEXT:    call __ashlti3, $pop13, $pop21, $pop20, $pop14
+; CHECK-NEXT:    local.get $push25=, 5
+; CHECK-NEXT:    local.get $push24=, 1
+; CHECK-NEXT:    local.get $push23=, 2
 ; CHECK-NEXT:    i32.const $push0=, 128
-; CHECK-NEXT:    local.get $push28=, 6
-; CHECK-NEXT:    i32.sub $push1=, $pop0, $pop28
-; CHECK-NEXT:    call __lshrti3, $pop31, $pop30, $pop29, $pop1
-; CHECK-NEXT:    local.get $push34=, 0
+; CHECK-NEXT:    local.get $push22=, 6
+; CHECK-NEXT:    i32.sub  $push1=, $pop0, $pop22
+; CHECK-NEXT:    call __lshrti3, $pop25, $pop24, $pop23, $pop1
+; CHECK-NEXT:    local.get $push28=, 0
+; CHECK-NEXT:    local.get $push26=, 5
+; CHECK-NEXT:    i64.load $push2=, 24($pop26)
+; CHECK-NEXT:    local.get $push27=, 5
+; CHECK-NEXT:    i64.load $push3=, 8($pop27)
+; CHECK-NEXT:    i64.or   $push4=, $pop2, $pop3
+; CHECK-NEXT:    i64.store 8($pop28), $pop4
+; CHECK-NEXT:    local.get $push31=, 0
+; CHECK-NEXT:    local.get $push29=, 5
+; CHECK-NEXT:    i64.load $push5=, 16($pop29)
+; CHECK-NEXT:    local.get $push30=, 5
+; CHECK-NEXT:    i64.load $push6=, 0($pop30)
+; CHECK-NEXT:    i64.or   $push7=, $pop5, $pop6
+; CHECK-NEXT:    i64.store 0($pop31), $pop7
 ; CHECK-NEXT:    local.get $push32=, 5
-; CHECK-NEXT:    i32.const $push15=, 16
-; CHECK-NEXT:    i32.add $push16=, $pop32, $pop15
-; CHECK-NEXT:    i32.const $push2=, 8
-; CHECK-NEXT:    i32.add $push3=, $pop16, $pop2
-; CHECK-NEXT:    i64.load $push4=, 0($pop3)
-; CHECK-NEXT:    local.get $push33=, 5
-; CHECK-NEXT:    i32.const $push19=, 8
-; CHECK-NEXT:    i32.add $push5=, $pop33, $pop19
-; CHECK-NEXT:    i64.load $push6=, 0($pop5)
-; CHECK-NEXT:    i64.or $push7=, $pop4, $pop6
-; CHECK-NEXT:    i64.store 8($pop34), $pop7
-; CHECK-NEXT:    local.get $push37=, 0
-; CHECK-NEXT:    local.get $push35=, 5
-; CHECK-NEXT:    i64.load $push8=, 16($pop35)
-; CHECK-NEXT:    local.get $push36=, 5
-; CHECK-NEXT:    i64.load $push9=, 0($pop36)
-; CHECK-NEXT:    i64.or $push10=, $pop8, $pop9
-; CHECK-NEXT:    i64.store 0($pop37), $pop10
-; CHECK-NEXT:    local.get $push38=, 5
-; CHECK-NEXT:    i32.const $push13=, 32
-; CHECK-NEXT:    i32.add $push14=, $pop38, $pop13
-; CHECK-NEXT:    global.set __stack_pointer, $pop14
+; CHECK-NEXT:    i32.const $push10=, 32
+; CHECK-NEXT:    i32.add  $push11=, $pop32, $pop10
+; CHECK-NEXT:    global.set __stack_pointer, $pop11
 ; CHECK-NEXT:    return
   %z = sub i128 128, %y
   %b = shl i128 %x, %y
@@ -587,53 +565,47 @@ define i128 @masked_rotl(i128 %x, i128 %y) {
 ; CHECK:         .functype masked_rotl (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push13=, __stack_pointer
-; CHECK-NEXT:    i32.const $push14=, 32
-; CHECK-NEXT:    i32.sub $push25=, $pop13, $pop14
-; CHECK-NEXT:    local.tee $push24=, 5, $pop25
-; CHECK-NEXT:    global.set __stack_pointer, $pop24
-; CHECK-NEXT:    local.get $push26=, 5
-; CHECK-NEXT:    i32.const $push19=, 16
-; CHECK-NEXT:    i32.add $push20=, $pop26, $pop19
-; CHECK-NEXT:    local.get $push29=, 1
-; CHECK-NEXT:    local.get $push28=, 2
-; CHECK-NEXT:    local.get $push27=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop27
+; CHECK-NEXT:    global.get $push10=, __stack_pointer
+; CHECK-NEXT:    i32.const $push11=, 32
+; CHECK-NEXT:    i32.sub  $push19=, $pop10, $pop11
+; CHECK-NEXT:    local.tee $push18=, 5, $pop19
+; CHECK-NEXT:    global.set __stack_pointer, $pop18
+; CHECK-NEXT:    local.get $push20=, 5
+; CHECK-NEXT:    i32.const $push14=, 16
+; CHECK-NEXT:    i32.add  $push15=, $pop20, $pop14
+; CHECK-NEXT:    local.get $push23=, 1
+; CHECK-NEXT:    local.get $push22=, 2
+; CHECK-NEXT:    local.get $push21=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop21
 ; CHECK-NEXT:    i32.const $push1=, 127
-; CHECK-NEXT:    i32.and $push23=, $pop0, $pop1
-; CHECK-NEXT:    local.tee $push22=, 6, $pop23
-; CHECK-NEXT:    call __ashlti3, $pop20, $pop29, $pop28, $pop22
-; CHECK-NEXT:    local.get $push33=, 5
-; CHECK-NEXT:    local.get $push32=, 1
-; CHECK-NEXT:    local.get $push31=, 2
+; CHECK-NEXT:    i32.and  $push17=, $pop0, $pop1
+; CHECK-NEXT:    local.tee $push16=, 6, $pop17
+; CHECK-NEXT:    call __ashlti3, $pop15, $pop23, $pop22, $pop16
+; CHECK-NEXT:    local.get $push27=, 5
+; CHECK-NEXT:    local.get $push26=, 1
+; CHECK-NEXT:    local.get $push25=, 2
 ; CHECK-NEXT:    i32.const $push2=, 128
-; CHECK-NEXT:    local.get $push30=, 6
-; CHECK-NEXT:    i32.sub $push3=, $pop2, $pop30
-; CHECK-NEXT:    call __lshrti3, $pop33, $pop32, $pop31, $pop3
-; CHECK-NEXT:    local.get $push36=, 0
+; CHECK-NEXT:    local.get $push24=, 6
+; CHECK-NEXT:    i32.sub  $push3=, $pop2, $pop24
+; CHECK-NEXT:    call __lshrti3, $pop27, $pop26, $pop25, $pop3
+; CHECK-NEXT:    local.get $push30=, 0
+; CHECK-NEXT:    local.get $push28=, 5
+; CHECK-NEXT:    i64.load $push4=, 24($pop28)
+; CHECK-NEXT:    local.get $push29=, 5
+; CHECK-NEXT:    i64.load $push5=, 8($pop29)
+; CHECK-NEXT:    i64.or   $push6=, $pop4, $pop5
+; CHECK-NEXT:    i64.store 8($pop30), $pop6
+; CHECK-NEXT:    local.get $push33=, 0
+; CHECK-NEXT:    local.get $push31=, 5
+; CHECK-NEXT:    i64.load $push7=, 16($pop31)
+; CHECK-NEXT:    local.get $push32=, 5
+; CHECK-NEXT:    i64.load $push8=, 0($pop32)
+; CHECK-NEXT:    i64.or   $push9=, $pop7, $pop8
+; CHECK-NEXT:    i64.store 0($pop33), $pop9
 ; CHECK-NEXT:    local.get $push34=, 5
-; CHECK-NEXT:    i32.const $push17=, 16
-; CHECK-NEXT:    i32.add $push18=, $pop34, $pop17
-; CHECK-NEXT:    i32.const $push4=, 8
-; CHECK-NEXT:    i32.add $push5=, $pop18, $pop4
-; CHECK-NEXT:    i64.load $push6=, 0($pop5)
-; CHECK-NEXT:    local.get $push35=, 5
-; CHECK-NEXT:    i32.const $push21=, 8
-; CHECK-NEXT:    i32.add $push7=, $pop35, $pop21
-; CHECK-NEXT:    i64.load $push8=, 0($pop7)
-; CHECK-NEXT:    i64.or $push9=, $pop6, $pop8
-; CHECK-NEXT:    i64.store 8($pop36), $pop9
-; CHECK-NEXT:    local.get $push39=, 0
-; CHECK-NEXT:    local.get $push37=, 5
-; CHECK-NEXT:    i64.load $push10=, 16($pop37)
-; CHECK-NEXT:    local.get $push38=, 5
-; CHECK-NEXT:    i64.load $push11=, 0($pop38)
-; CHECK-NEXT:    i64.or $push12=, $pop10, $pop11
-; CHECK-NEXT:    i64.store 0($pop39), $pop12
-; CHECK-NEXT:    local.get $push40=, 5
-; CHECK-NEXT:    i32.const $push15=, 32
-; CHECK-NEXT:    i32.add $push16=, $pop40, $pop15
-; CHECK-NEXT:    global.set __stack_pointer, $pop16
+; CHECK-NEXT:    i32.const $push12=, 32
+; CHECK-NEXT:    i32.add  $push13=, $pop34, $pop12
+; CHECK-NEXT:    global.set __stack_pointer, $pop13
 ; CHECK-NEXT:    return
   %a = and i128 %y, 127
   %z = sub i128 128, %a
@@ -648,51 +620,45 @@ define i128 @rotr(i128 %x, i128 %y) {
 ; CHECK:         .functype rotr (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push11=, __stack_pointer
-; CHECK-NEXT:    i32.const $push12=, 32
-; CHECK-NEXT:    i32.sub $push23=, $pop11, $pop12
-; CHECK-NEXT:    local.tee $push22=, 5, $pop23
-; CHECK-NEXT:    global.set __stack_pointer, $pop22
-; CHECK-NEXT:    local.get $push24=, 5
-; CHECK-NEXT:    i32.const $push17=, 16
-; CHECK-NEXT:    i32.add $push18=, $pop24, $pop17
-; CHECK-NEXT:    local.get $push27=, 1
-; CHECK-NEXT:    local.get $push26=, 2
-; CHECK-NEXT:    local.get $push25=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push21=, $pop25
-; CHECK-NEXT:    local.tee $push20=, 6, $pop21
-; CHECK-NEXT:    call __lshrti3, $pop18, $pop27, $pop26, $pop20
-; CHECK-NEXT:    local.get $push31=, 5
-; CHECK-NEXT:    local.get $push30=, 1
-; CHECK-NEXT:    local.get $push29=, 2
+; CHECK-NEXT:    global.get $push8=, __stack_pointer
+; CHECK-NEXT:    i32.const $push9=, 32
+; CHECK-NEXT:    i32.sub  $push17=, $pop8, $pop9
+; CHECK-NEXT:    local.tee $push16=, 5, $pop17
+; CHECK-NEXT:    global.set __stack_pointer, $pop16
+; CHECK-NEXT:    local.get $push18=, 5
+; CHECK-NEXT:    i32.const $push12=, 16
+; CHECK-NEXT:    i32.add  $push13=, $pop18, $pop12
+; CHECK-NEXT:    local.get $push21=, 1
+; CHECK-NEXT:    local.get $push20=, 2
+; CHECK-NEXT:    local.get $push19=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push15=, $pop19
+; CHECK-NEXT:    local.tee $push14=, 6, $pop15
+; CHECK-NEXT:    call __lshrti3, $pop13, $pop21, $pop20, $pop14
+; CHECK-NEXT:    local.get $push25=, 5
+; CHECK-NEXT:    local.get $push24=, 1
+; CHECK-NEXT:    local.get $push23=, 2
 ; CHECK-NEXT:    i32.const $push0=, 128
-; CHECK-NEXT:    local.get $push28=, 6
-; CHECK-NEXT:    i32.sub $push1=, $pop0, $pop28
-; CHECK-NEXT:    call __ashlti3, $pop31, $pop30, $pop29, $pop1
-; CHECK-NEXT:    local.get $push34=, 0
+; CHECK-NEXT:    local.get $push22=, 6
+; CHECK-NEXT:    i32.sub  $push1=, $pop0, $pop22
+; CHECK-NEXT:    call __ashlti3, $pop25, $pop24, $pop23, $pop1
+; CHECK-NEXT:    local.get $push28=, 0
+; CHECK-NEXT:    local.get $push26=, 5
+; CHECK-NEXT:    i64.load $push2=, 24($pop26)
+; CHECK-NEXT:    local.get $push27=, 5
+; CHECK-NEXT:    i64.load $push3=, 8($pop27)
+; CHECK-NEXT:    i64.or   $push4=, $pop2, $pop3
+; CHECK-NEXT:    i64.store 8($pop28), $pop4
+; CHECK-NEXT:    local.get $push31=, 0
+; CHECK-NEXT:    local.get $push29=, 5
+; CHECK-NEXT:    i64.load $push5=, 16($pop29)
+; CHECK-NEXT:    local.get $push30=, 5
+; CHECK-NEXT:    i64.load $push6=, 0($pop30)
+; CHECK-NEXT:    i64.or   $push7=, $pop5, $pop6
+; CHECK-NEXT:    i64.store 0($pop31), $pop7
 ; CHECK-NEXT:    local.get $push32=, 5
-; CHECK-NEXT:    i32.const $push15=, 16
-; CHECK-NEXT:    i32.add $push16=, $pop32, $pop15
-; CHECK-NEXT:    i32.const $push2=, 8
-; CHECK-NEXT:    i32.add $push3=, $pop16, $pop2
-; CHECK-NEXT:    i64.load $push4=, 0($pop3)
-; CHECK-NEXT:    local.get $push33=, 5
-; CHECK-NEXT:    i32.const $push19=, 8
-; CHECK-NEXT:    i32.add $push5=, $pop33, $pop19
-; CHECK-NEXT:    i64.load $push6=, 0($pop5)
-; CHECK-NEXT:    i64.or $push7=, $pop4, $pop6
-; CHECK-NEXT:    i64.store 8($pop34), $pop7
-; CHECK-NEXT:    local.get $push37=, 0
-; CHECK-NEXT:    local.get $push35=, 5
-; CHECK-NEXT:    i64.load $push8=, 16($pop35)
-; CHECK-NEXT:    local.get $push36=, 5
-; CHECK-NEXT:    i64.load $push9=, 0($pop36)
-; CHECK-NEXT:    i64.or $push10=, $pop8, $pop9
-; CHECK-NEXT:    i64.store 0($pop37), $pop10
-; CHECK-NEXT:    local.get $push38=, 5
-; CHECK-NEXT:    i32.const $push13=, 32
-; CHECK-NEXT:    i32.add $push14=, $pop38, $pop13
-; CHECK-NEXT:    global.set __stack_pointer, $pop14
+; CHECK-NEXT:    i32.const $push10=, 32
+; CHECK-NEXT:    i32.add  $push11=, $pop32, $pop10
+; CHECK-NEXT:    global.set __stack_pointer, $pop11
 ; CHECK-NEXT:    return
   %z = sub i128 128, %y
   %b = lshr i128 %x, %y
@@ -706,53 +672,47 @@ define i128 @masked_rotr(i128 %x, i128 %y) {
 ; CHECK:         .functype masked_rotr (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push13=, __stack_pointer
-; CHECK-NEXT:    i32.const $push14=, 32
-; CHECK-NEXT:    i32.sub $push25=, $pop13, $pop14
-; CHECK-NEXT:    local.tee $push24=, 5, $pop25
-; CHECK-NEXT:    global.set __stack_pointer, $pop24
-; CHECK-NEXT:    local.get $push26=, 5
-; CHECK-NEXT:    i32.const $push19=, 16
-; CHECK-NEXT:    i32.add $push20=, $pop26, $pop19
-; CHECK-NEXT:    local.get $push29=, 1
-; CHECK-NEXT:    local.get $push28=, 2
-; CHECK-NEXT:    local.get $push27=, 3
-; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop27
+; CHECK-NEXT:    global.get $push10=, __stack_pointer
+; CHECK-NEXT:    i32.const $push11=, 32
+; CHECK-NEXT:    i32.sub  $push19=, $pop10, $pop11
+; CHECK-NEXT:    local.tee $push18=, 5, $pop19
+; CHECK-NEXT:    global.set __stack_pointer, $pop18
+; CHECK-NEXT:    local.get $push20=, 5
+; CHECK-NEXT:    i32.const $push14=, 16
+; CHECK-NEXT:    i32.add  $push15=, $pop20, $pop14
+; CHECK-NEXT:    local.get $push23=, 1
+; CHECK-NEXT:    local.get $push22=, 2
+; CHECK-NEXT:    local.get $push21=, 3
+; CHECK-NEXT:    i32.wrap_i64 $push0=, $pop21
 ; CHECK-NEXT:    i32.const $push1=, 127
-; CHECK-NEXT:    i32.and $push23=, $pop0, $pop1
-; CHECK-NEXT:    local.tee $push22=, 6, $pop23
-; CHECK-NEXT:    call __lshrti3, $pop20, $pop29, $pop28, $pop22
-; CHECK-NEXT:    local.get $push33=, 5
-; CHECK-NEXT:    local.get $push32=, 1
-; CHECK-NEXT:    local.get $push31=, 2
+; CHECK-NEXT:    i32.and  $push17=, $pop0, $pop1
+; CHECK-NEXT:    local.tee $push16=, 6, $pop17
+; CHECK-NEXT:    call __lshrti3, $pop15, $pop23, $pop22, $pop16
+; CHECK-NEXT:    local.get $push27=, 5
+; CHECK-NEXT:    local.get $push26=, 1
+; CHECK-NEXT:    local.get $push25=, 2
 ; CHECK-NEXT:    i32.const $push2=, 128
-; CHECK-NEXT:    local.get $push30=, 6
-; CHECK-NEXT:    i32.sub $push3=, $pop2, $pop30
-; CHECK-NEXT:    call __ashlti3, $pop33, $pop32, $pop31, $pop3
-; CHECK-NEXT:    local.get $push36=, 0
+; CHECK-NEXT:    local.get $push24=, 6
+; CHECK-NEXT:    i32.sub  $push3=, $pop2, $pop24
+; CHECK-NEXT:    call __ashlti3, $pop27, $pop26, $pop25, $pop3
+; CHECK-NEXT:    local.get $push30=, 0
+; CHECK-NEXT:    local.get $push28=, 5
+; CHECK-NEXT:    i64.load $push4=, 24($pop28)
+; CHECK-NEXT:    local.get $push29=, 5
+; CHECK-NEXT:    i64.load $push5=, 8($pop29)
+; CHECK-NEXT:    i64.or   $push6=, $pop4, $pop5
+; CHECK-NEXT:    i64.store 8($pop30), $pop6
+; CHECK-NEXT:    local.get $push33=, 0
+; CHECK-NEXT:    local.get $push31=, 5
+; CHECK-NEXT:    i64.load $push7=, 16($pop31)
+; CHECK-NEXT:    local.get $push32=, 5
+; CHECK-NEXT:    i64.load $push8=, 0($pop32)
+; CHECK-NEXT:    i64.or   $push9=, $pop7, $pop8
+; CHECK-NEXT:    i64.store 0($pop33), $pop9
 ; CHECK-NEXT:    local.get $push34=, 5
-; CHECK-NEXT:    i32.const $push17=, 16
-; CHECK-NEXT:    i32.add $push18=, $pop34, $pop17
-; CHECK-NEXT:    i32.const $push4=, 8
-; CHECK-NEXT:    i32.add $push5=, $pop18, $pop4
-; CHECK-NEXT:    i64.load $push6=, 0($pop5)
-; CHECK-NEXT:    local.get $push35=, 5
-; CHECK-NEXT:    i32.const $push21=, 8
-; CHECK-NEXT:    i32.add $push7=, $pop35, $pop21
-; CHECK-NEXT:    i64.load $push8=, 0($pop7)
-; CHECK-NEXT:    i64.or $push9=, $pop6, $pop8
-; CHECK-NEXT:    i64.store 8($pop36), $pop9
-; CHECK-NEXT:    local.get $push39=, 0
-; CHECK-NEXT:    local.get $push37=, 5
-; CHECK-NEXT:    i64.load $push10=, 16($pop37)
-; CHECK-NEXT:    local.get $push38=, 5
-; CHECK-NEXT:    i64.load $push11=, 0($pop38)
-; CHECK-NEXT:    i64.or $push12=, $pop10, $pop11
-; CHECK-NEXT:    i64.store 0($pop39), $pop12
-; CHECK-NEXT:    local.get $push40=, 5
-; CHECK-NEXT:    i32.const $push15=, 32
-; CHECK-NEXT:    i32.add $push16=, $pop40, $pop15
-; CHECK-NEXT:    global.set __stack_pointer, $pop16
+; CHECK-NEXT:    i32.const $push12=, 32
+; CHECK-NEXT:    i32.add  $push13=, $pop34, $pop12
+; CHECK-NEXT:    global.set __stack_pointer, $pop13
 ; CHECK-NEXT:    return
   %a = and i128 %y, 127
   %z = sub i128 128, %a
diff --git a/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll b/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll
index 7850559b49b7d..042009b9365e7 100644
--- a/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll
+++ b/llvm/test/CodeGen/WebAssembly/libcalls-trig.ll
@@ -45,139 +45,101 @@ define fp128 @fp128libcalls(fp128 %x) {
 ; CHECK:         .functype fp128libcalls (i32, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push31=, __stack_pointer
-; CHECK-NEXT:    i32.const $push32=, 160
-; CHECK-NEXT:    i32.sub $push81=, $pop31, $pop32
-; CHECK-NEXT:    local.tee $push80=, 3, $pop81
-; CHECK-NEXT:    global.set __stack_pointer, $pop80
-; CHECK-NEXT:    local.get $push82=, 3
-; CHECK-NEXT:    i32.const $push69=, 144
-; CHECK-NEXT:    i32.add $push70=, $pop82, $pop69
-; CHECK-NEXT:    local.get $push84=, 1
-; CHECK-NEXT:    local.get $push83=, 2
-; CHECK-NEXT:    call sinl, $pop70, $pop84, $pop83
-; CHECK-NEXT:    local.get $push85=, 3
-; CHECK-NEXT:    i32.const $push65=, 128
-; CHECK-NEXT:    i32.add $push66=, $pop85, $pop65
-; CHECK-NEXT:    local.get $push86=, 3
-; CHECK-NEXT:    i64.load $push3=, 144($pop86)
-; CHECK-NEXT:    local.get $push87=, 3
-; CHECK-NEXT:    i32.const $push67=, 144
-; CHECK-NEXT:    i32.add $push68=, $pop87, $pop67
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop68, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    call cosl, $pop66, $pop3, $pop2
-; CHECK-NEXT:    local.get $push88=, 3
-; CHECK-NEXT:    i32.const $push61=, 112
-; CHECK-NEXT:    i32.add $push62=, $pop88, $pop61
-; CHECK-NEXT:    local.get $push89=, 3
-; CHECK-NEXT:    i64.load $push6=, 128($pop89)
-; CHECK-NEXT:    local.get $push90=, 3
-; CHECK-NEXT:    i32.const $push63=, 128
-; CHECK-NEXT:    i32.add $push64=, $pop90, $pop63
-; CHECK-NEXT:    i32.const $push79=, 8
-; CHECK-NEXT:    i32.add $push4=, $pop64, $pop79
-; CHECK-NEXT:    i64.load $push5=, 0($pop4)
-; CHECK-NEXT:    call tanl, $pop62, $pop6, $pop5
-; CHECK-NEXT:    local.get $push91=, 3
-; CHECK-NEXT:    i32.const $push57=, 96
-; CHECK-NEXT:    i32.add $push58=, $pop91, $pop57
-; CHECK-NEXT:    local.get $push92=, 3
-; CHECK-NEXT:    i64.load $push9=, 112($pop92)
-; CHECK-NEXT:    local.get $push93=, 3
-; CHECK-NEXT:    i32.const $push59=, 112
-; CHECK-NEXT:    i32.add $push60=, $pop93, $pop59
-; CHECK-NEXT:    i32.const $push78=, 8
-; CHECK-NEXT:    i32.add $push7=, $pop60, $pop78
-; CHECK-NEXT:    i64.load $push8=, 0($pop7)
-; CHECK-NEXT:    call asinl, $pop58, $pop9, $pop8
-; CHECK-NEXT:    local.get $push94=, 3
-; CHECK-NEXT:    i32.const $push53=, 80
-; CHECK-NEXT:    i32.add $push54=, $pop94, $pop53
-; CHECK-NEXT:    local.get $push95=, 3
-; CHECK-NEXT:    i64.load $push12=, 96($pop95)
-; CHECK-NEXT:    local.get $push96=, 3
-; CHECK-NEXT:    i32.const $push55=, 96
-; CHECK-NEXT:    i32.add $push56=, $pop96, $pop55
-; CHECK-NEXT:    i32.const $push77=, 8
-; CHECK-NEXT:    i32.add $push10=, $pop56, $pop77
-; CHECK-NEXT:    i64.load $push11=, 0($pop10)
-; CHECK-NEXT:    call acosl, $pop54, $pop12, $pop11
-; CHECK-NEXT:    local.get $push97=, 3
-; CHECK-NEXT:    i32.const $push49=, 64
-; CHECK-NEXT:    i32.add $push50=, $pop97, $pop49
-; CHECK-NEXT:    local.get $push98=, 3
-; CHECK-NEXT:    i64.load $push15=, 80($pop98)
-; CHECK-NEXT:    local.get $push99=, 3
-; CHECK-NEXT:    i32.const $push51=, 80
-; CHECK-NEXT:    i32.add $push52=, $pop99, $pop51
-; CHECK-NEXT:    i32.const $push76=, 8
-; CHECK-NEXT:    i32.add $push13=, $pop52, $pop76
-; CHECK-NEXT:    i64.load $push14=, 0($pop13)
-; CHECK-NEXT:    call atanl, $pop50, $pop15, $pop14
-; CHECK-NEXT:    local.get $push100=, 3
-; CHECK-NEXT:    i32.const $push45=, 48
-; CHECK-NEXT:    i32.add $push46=, $pop100, $pop45
-; CHECK-NEXT:    local.get $push101=, 3
-; CHECK-NEXT:    i64.load $push18=, 64($pop101)
-; CHECK-NEXT:    local.get $push102=, 3
-; CHECK-NEXT:    i32.const $push47=, 64
-; CHECK-NEXT:    i32.add $push48=, $pop102, $pop47
-; CHECK-NEXT:    i32.const $push75=, 8
-; CHECK-NEXT:    i32.add $push16=, $pop48, $pop75
-; CHECK-NEXT:    i64.load $push17=, 0($pop16)
-; CHECK-NEXT:    call sinhl, $pop46, $pop18, $pop17
-; CHECK-NEXT:    local.get $push103=, 3
-; CHECK-NEXT:    i32.const $push41=, 32
-; CHECK-NEXT:    i32.add $push42=, $pop103, $pop41
-; CHECK-NEXT:    local.get $push104=, 3
-; CHECK-NEXT:    i64.load $push21=, 48($pop104)
-; CHECK-NEXT:    local.get $push105=, 3
-; CHECK-NEXT:    i32.const $push43=, 48
-; CHECK-NEXT:    i32.add $push44=, $pop105, $pop43
-; CHECK-NEXT:    i32.const $push74=, 8
-; CHECK-NEXT:    i32.add $push19=, $pop44, $pop74
-; CHECK-NEXT:    i64.load $push20=, 0($pop19)
-; CHECK-NEXT:    call coshl, $pop42, $pop21, $pop20
-; CHECK-NEXT:    local.get $push106=, 3
-; CHECK-NEXT:    i32.const $push37=, 16
-; CHECK-NEXT:    i32.add $push38=, $pop106, $pop37
-; CHECK-NEXT:    local.get $push107=, 3
-; CHECK-NEXT:    i64.load $push24=, 32($pop107)
-; CHECK-NEXT:    local.get $push108=, 3
-; CHECK-NEXT:    i32.const $push39=, 32
-; CHECK-NEXT:    i32.add $push40=, $pop108, $pop39
-; CHECK-NEXT:    i32.const $push73=, 8
-; CHECK-NEXT:    i32.add $push22=, $pop40, $pop73
-; CHECK-NEXT:    i64.load $push23=, 0($pop22)
-; CHECK-NEXT:    call tanhl, $pop38, $pop24, $pop23
-; CHECK-NEXT:    local.get $push113=, 3
-; CHECK-NEXT:    local.get $push112=, 1
-; CHECK-NEXT:    local.get $push111=, 2
-; CHECK-NEXT:    local.get $push109=, 3
-; CHECK-NEXT:    i64.load $push27=, 16($pop109)
-; CHECK-NEXT:    local.get $push110=, 3
-; CHECK-NEXT:    i32.const $push35=, 16
-; CHECK-NEXT:    i32.add $push36=, $pop110, $pop35
-; CHECK-NEXT:    i32.const $push72=, 8
-; CHECK-NEXT:    i32.add $push25=, $pop36, $pop72
-; CHECK-NEXT:    i64.load $push26=, 0($pop25)
-; CHECK-NEXT:    call atan2l, $pop113, $pop112, $pop111, $pop27, $pop26
-; CHECK-NEXT:    local.get $push115=, 0
-; CHECK-NEXT:    local.get $push114=, 3
-; CHECK-NEXT:    i32.const $push71=, 8
-; CHECK-NEXT:    i32.add $push28=, $pop114, $pop71
-; CHECK-NEXT:    i64.load $push29=, 0($pop28)
-; CHECK-NEXT:    i64.store 8($pop115), $pop29
-; CHECK-NEXT:    local.get $push117=, 0
-; CHECK-NEXT:    local.get $push116=, 3
-; CHECK-NEXT:    i64.load $push30=, 0($pop116)
-; CHECK-NEXT:    i64.store 0($pop117), $pop30
-; CHECK-NEXT:    local.get $push118=, 3
-; CHECK-NEXT:    i32.const $push33=, 160
-; CHECK-NEXT:    i32.add $push34=, $pop118, $pop33
-; CHECK-NEXT:    global.set __stack_pointer, $pop34
+; CHECK-NEXT:    global.get $push20=, __stack_pointer
+; CHECK-NEXT:    i32.const $push21=, 160
+; CHECK-NEXT:    i32.sub  $push43=, $pop20, $pop21
+; CHECK-NEXT:    local.tee $push42=, 3, $pop43
+; CHECK-NEXT:    global.set __stack_pointer, $pop42
+; CHECK-NEXT:    local.get $push44=, 3
+; CHECK-NEXT:    i32.const $push40=, 144
+; CHECK-NEXT:    i32.add  $push41=, $pop44, $pop40
+; CHECK-NEXT:    local.get $push46=, 1
+; CHECK-NEXT:    local.get $push45=, 2
+; CHECK-NEXT:    call sinl, $pop41, $pop46, $pop45
+; CHECK-NEXT:    local.get $push47=, 3
+; CHECK-NEXT:    i32.const $push38=, 128
+; CHECK-NEXT:    i32.add  $push39=, $pop47, $pop38
+; CHECK-NEXT:    local.get $push48=, 3
+; CHECK-NEXT:    i64.load $push1=, 144($pop48)
+; CHECK-NEXT:    local.get $push49=, 3
+; CHECK-NEXT:    i64.load $push0=, 152($pop49)
+; CHECK-NEXT:    call cosl, $pop39, $pop1, $pop0
+; CHECK-NEXT:    local.get $push50=, 3
+; CHECK-NEXT:    i32.const $push36=, 112
+; CHECK-NEXT:    i32.add  $push37=, $pop50, $pop36
+; CHECK-NEXT:    local.get $push51=, 3
+; CHECK-NEXT:    i64.load $push3=, 128($pop51)
+; CHECK-NEXT:    local.get $push52=, 3
+; CHECK-NEXT:    i64.load $push2=, 136($pop52)
+; CHECK-NEXT:    call tanl, $pop37, $pop3, $pop2
+; CHECK-NEXT:    local.get $push53=, 3
+; CHECK-NEXT:    i32.const $push34=, 96
+; CHECK-NEXT:    i32.add  $push35=, $pop53, $pop34
+; CHECK-NEXT:    local.get $push54=, 3
+; CHECK-NEXT:    i64.load $push5=, 112($pop54)
+; CHECK-NEXT:    local.get $push55=, 3
+; CHECK-NEXT:    i64.load $push4=, 120($pop55)
+; CHECK-NEXT:    call asinl, $pop35, $pop5, $pop4
+; CHECK-NEXT:    local.get $push56=, 3
+; CHECK-NEXT:    i32.const $push32=, 80
+; CHECK-NEXT:    i32.add  $push33=, $pop56, $pop32
+; CHECK-NEXT:    local.get $push57=, 3
+; CHECK-NEXT:    i64.load $push7=, 96($pop57)
+; CHECK-NEXT:    local.get $push58=, 3
+; CHECK-NEXT:    i64.load $push6=, 104($pop58)
+; CHECK-NEXT:    call acosl, $pop33, $pop7, $pop6
+; CHECK-NEXT:    local.get $push59=, 3
+; CHECK-NEXT:    i32.const $push30=, 64
+; CHECK-NEXT:    i32.add  $push31=, $pop59, $pop30
+; CHECK-NEXT:    local.get $push60=, 3
+; CHECK-NEXT:    i64.load $push9=, 80($pop60)
+; CHECK-NEXT:    local.get $push61=, 3
+; CHECK-NEXT:    i64.load $push8=, 88($pop61)
+; CHECK-NEXT:    call atanl, $pop31, $pop9, $pop8
+; CHECK-NEXT:    local.get $push62=, 3
+; CHECK-NEXT:    i32.const $push28=, 48
+; CHECK-NEXT:    i32.add  $push29=, $pop62, $pop28
+; CHECK-NEXT:    local.get $push63=, 3
+; CHECK-NEXT:    i64.load $push11=, 64($pop63)
+; CHECK-NEXT:    local.get $push64=, 3
+; CHECK-NEXT:    i64.load $push10=, 72($pop64)
+; CHECK-NEXT:    call sinhl, $pop29, $pop11, $pop10
+; CHECK-NEXT:    local.get $push65=, 3
+; CHECK-NEXT:    i32.const $push26=, 32
+; CHECK-NEXT:    i32.add  $push27=, $pop65, $pop26
+; CHECK-NEXT:    local.get $push66=, 3
+; CHECK-NEXT:    i64.load $push13=, 48($pop66)
+; CHECK-NEXT:    local.get $push67=, 3
+; CHECK-NEXT:    i64.load $push12=, 56($pop67)
+; CHECK-NEXT:    call coshl, $pop27, $pop13, $pop12
+; CHECK-NEXT:    local.get $push68=, 3
+; CHECK-NEXT:    i32.const $push24=, 16
+; CHECK-NEXT:    i32.add  $push25=, $pop68, $pop24
+; CHECK-NEXT:    local.get $push69=, 3
+; CHECK-NEXT:    i64.load $push15=, 32($pop69)
+; CHECK-NEXT:    local.get $push70=, 3
+; CHECK-NEXT:    i64.load $push14=, 40($pop70)
+; CHECK-NEXT:    call tanhl, $pop25, $pop15, $pop14
+; CHECK-NEXT:    local.get $push75=, 3
+; CHECK-NEXT:    local.get $push74=, 1
+; CHECK-NEXT:    local.get $push73=, 2
+; CHECK-NEXT:    local.get $push71=, 3
+; CHECK-NEXT:    i64.load $push17=, 16($pop71)
+; CHECK-NEXT:    local.get $push72=, 3
+; CHECK-NEXT:    i64.load $push16=, 24($pop72)
+; CHECK-NEXT:    call atan2l, $pop75, $pop74, $pop73, $pop17, $pop16
+; CHECK-NEXT:    local.get $push77=, 0
+; CHECK-NEXT:    local.get $push76=, 3
+; CHECK-NEXT:    i64.load $push18=, 8($pop76)
+; CHECK-NEXT:    i64.store 8($pop77), $pop18
+; CHECK-NEXT:    local.get $push79=, 0
+; CHECK-NEXT:    local.get $push78=, 3
+; CHECK-NEXT:    i64.load $push19=, 0($pop78)
+; CHECK-NEXT:    i64.store 0($pop79), $pop19
+; CHECK-NEXT:    local.get $push80=, 3
+; CHECK-NEXT:    i32.const $push22=, 160
+; CHECK-NEXT:    i32.add  $push23=, $pop80, $pop22
+; CHECK-NEXT:    global.set __stack_pointer, $pop23
 ; CHECK-NEXT:    return
   ; libm calls
   %d = call fp128 @llvm.sin.f128(fp128 %x)
diff --git a/llvm/test/CodeGen/WebAssembly/libcalls.ll b/llvm/test/CodeGen/WebAssembly/libcalls.ll
index 70f000664d388..39657fe2c5870 100644
--- a/llvm/test/CodeGen/WebAssembly/libcalls.ll
+++ b/llvm/test/CodeGen/WebAssembly/libcalls.ll
@@ -32,134 +32,100 @@ define fp128 @fp128libcalls(fp128 %x, fp128 %y, i32 %z) {
 ; CHECK:         .functype fp128libcalls (i32, i64, i64, i64, i64, i32) -> ()
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push28=, __stack_pointer
-; CHECK-NEXT:    i32.const $push29=, 144
-; CHECK-NEXT:    i32.sub $push73=, $pop28, $pop29
-; CHECK-NEXT:    local.tee $push72=, 6, $pop73
-; CHECK-NEXT:    global.set __stack_pointer, $pop72
+; CHECK-NEXT:    global.get $push18=, __stack_pointer
+; CHECK-NEXT:    i32.const $push19=, 144
+; CHECK-NEXT:    i32.sub $push39=, $pop18, $pop19
+; CHECK-NEXT:    local.tee $push38=, 6, $pop39
+; CHECK-NEXT:    global.set __stack_pointer, $pop38
+; CHECK-NEXT:    local.get $push40=, 6
+; CHECK-NEXT:    i32.const $push36=, 128
+; CHECK-NEXT:    i32.add $push37=, $pop40, $pop36
+; CHECK-NEXT:    local.get $push44=, 1
+; CHECK-NEXT:    local.get $push43=, 2
+; CHECK-NEXT:    local.get $push42=, 3
+; CHECK-NEXT:    local.get $push41=, 4
+; CHECK-NEXT:    call __addtf3, $pop37, $pop44, $pop43, $pop42, $pop41
+; CHECK-NEXT:    local.get $push45=, 6
+; CHECK-NEXT:    i32.const $push34=, 112
+; CHECK-NEXT:    i32.add $push35=, $pop45, $pop34
+; CHECK-NEXT:    local.get $push46=, 6
+; CHECK-NEXT:    i64.load $push1=, 128($pop46)
+; CHECK-NEXT:    local.get $push47=, 6
+; CHECK-NEXT:    i64.load $push0=, 136($pop47)
+; CHECK-NEXT:    local.get $push49=, 3
+; CHECK-NEXT:    local.get $push48=, 4
+; CHECK-NEXT:    call __multf3, $pop35, $pop1, $pop0, $pop49, $pop48
+; CHECK-NEXT:    local.get $push50=, 6
+; CHECK-NEXT:    i32.const $push32=, 96
+; CHECK-NEXT:    i32.add $push33=, $pop50, $pop32
+; CHECK-NEXT:    local.get $push51=, 6
+; CHECK-NEXT:    i64.load $push3=, 112($pop51)
+; CHECK-NEXT:    local.get $push52=, 6
+; CHECK-NEXT:    i64.load $push2=, 120($pop52)
+; CHECK-NEXT:    local.get $push54=, 3
+; CHECK-NEXT:    local.get $push53=, 4
+; CHECK-NEXT:    call __divtf3, $pop33, $pop3, $pop2, $pop54, $pop53
+; CHECK-NEXT:    local.get $push55=, 6
+; CHECK-NEXT:    i32.const $push30=, 80
+; CHECK-NEXT:    i32.add $push31=, $pop55, $pop30
+; CHECK-NEXT:    local.get $push56=, 6
+; CHECK-NEXT:    i64.load $push5=, 96($pop56)
+; CHECK-NEXT:    local.get $push57=, 6
+; CHECK-NEXT:    i64.load $push4=, 104($pop57)
+; CHECK-NEXT:    call sqrtl, $pop31, $pop5, $pop4
+; CHECK-NEXT:    local.get $push58=, 6
+; CHECK-NEXT:    i32.const $push28=, 64
+; CHECK-NEXT:    i32.add $push29=, $pop58, $pop28
+; CHECK-NEXT:    local.get $push59=, 6
+; CHECK-NEXT:    i64.load $push7=, 80($pop59)
+; CHECK-NEXT:    local.get $push60=, 6
+; CHECK-NEXT:    i64.load $push6=, 88($pop60)
+; CHECK-NEXT:    call floorl, $pop29, $pop7, $pop6
+; CHECK-NEXT:    local.get $push61=, 6
+; CHECK-NEXT:    i32.const $push26=, 48
+; CHECK-NEXT:    i32.add $push27=, $pop61, $pop26
+; CHECK-NEXT:    local.get $push62=, 6
+; CHECK-NEXT:    i64.load $push9=, 64($pop62)
+; CHECK-NEXT:    local.get $push63=, 6
+; CHECK-NEXT:    i64.load $push8=, 72($pop63)
+; CHECK-NEXT:    local.get $push65=, 3
+; CHECK-NEXT:    local.get $push64=, 4
+; CHECK-NEXT:    call powl, $pop27, $pop9, $pop8, $pop65, $pop64
+; CHECK-NEXT:    local.get $push66=, 6
+; CHECK-NEXT:    i32.const $push24=, 32
+; CHECK-NEXT:    i32.add $push25=, $pop66, $pop24
+; CHECK-NEXT:    local.get $push67=, 6
+; CHECK-NEXT:    i64.load $push11=, 48($pop67)
+; CHECK-NEXT:    local.get $push68=, 6
+; CHECK-NEXT:    i64.load $push10=, 56($pop68)
+; CHECK-NEXT:    local.get $push69=, 5
+; CHECK-NEXT:    call __powitf2, $pop25, $pop11, $pop10, $pop69
+; CHECK-NEXT:    local.get $push70=, 6
+; CHECK-NEXT:    i32.const $push22=, 16
+; CHECK-NEXT:    i32.add $push23=, $pop70, $pop22
+; CHECK-NEXT:    local.get $push71=, 6
+; CHECK-NEXT:    i64.load $push13=, 32($pop71)
+; CHECK-NEXT:    local.get $push72=, 6
+; CHECK-NEXT:    i64.load $push12=, 40($pop72)
+; CHECK-NEXT:    call truncl, $pop23, $pop13, $pop12
+; CHECK-NEXT:    local.get $push75=, 6
+; CHECK-NEXT:    local.get $push73=, 6
+; CHECK-NEXT:    i64.load $push15=, 16($pop73)
 ; CHECK-NEXT:    local.get $push74=, 6
-; CHECK-NEXT:    i32.const $push62=, 128
-; CHECK-NEXT:    i32.add $push63=, $pop74, $pop62
-; CHECK-NEXT:    local.get $push78=, 1
-; CHECK-NEXT:    local.get $push77=, 2
-; CHECK-NEXT:    local.get $push76=, 3
-; CHECK-NEXT:    local.get $push75=, 4
-; CHECK-NEXT:    call __addtf3, $pop63, $pop78, $pop77, $pop76, $pop75
-; CHECK-NEXT:    local.get $push79=, 6
-; CHECK-NEXT:    i32.const $push58=, 112
-; CHECK-NEXT:    i32.add $push59=, $pop79, $pop58
+; CHECK-NEXT:    i64.load $push14=, 24($pop74)
+; CHECK-NEXT:    call nearbyintl, $pop75, $pop15, $pop14
+; CHECK-NEXT:    local.get $push77=, 0
+; CHECK-NEXT:    local.get $push76=, 6
+; CHECK-NEXT:    i64.load $push16=, 8($pop76)
+; CHECK-NEXT:    i64.store 8($pop77), $pop16
+; CHECK-NEXT:    local.get $push79=, 0
+; CHECK-NEXT:    local.get $push78=, 6
+; CHECK-NEXT:    i64.load $push17=, 0($pop78)
+; CHECK-NEXT:    i64.store 0($pop79), $pop17
 ; CHECK-NEXT:    local.get $push80=, 6
-; CHECK-NEXT:    i64.load $push3=, 128($pop80)
-; CHECK-NEXT:    local.get $push81=, 6
-; CHECK-NEXT:    i32.const $push60=, 128
-; CHECK-NEXT:    i32.add $push61=, $pop81, $pop60
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push1=, $pop61, $pop0
-; CHECK-NEXT:    i64.load $push2=, 0($pop1)
-; CHECK-NEXT:    local.get $push83=, 3
-; CHECK-NEXT:    local.get $push82=, 4
-; CHECK-NEXT:    call __multf3, $pop59, $pop3, $pop2, $pop83, $pop82
-; CHECK-NEXT:    local.get $push84=, 6
-; CHECK-NEXT:    i32.const $push54=, 96
-; CHECK-NEXT:    i32.add $push55=, $pop84, $pop54
-; CHECK-NEXT:    local.get $push85=, 6
-; CHECK-NEXT:    i64.load $push6=, 112($pop85)
-; CHECK-NEXT:    local.get $push86=, 6
-; CHECK-NEXT:    i32.const $push56=, 112
-; CHECK-NEXT:    i32.add $push57=, $pop86, $pop56
-; CHECK-NEXT:    i32.const $push71=, 8
-; CHECK-NEXT:    i32.add $push4=, $pop57, $pop71
-; CHECK-NEXT:    i64.load $push5=, 0($pop4)
-; CHECK-NEXT:    local.get $push88=, 3
-; CHECK-NEXT:    local.get $push87=, 4
-; CHECK-NEXT:    call __divtf3, $pop55, $pop6, $pop5, $pop88, $pop87
-; CHECK-NEXT:    local.get $push89=, 6
-; CHECK-NEXT:    i32.const $push50=, 80
-; CHECK-NEXT:    i32.add $push51=, $pop89, $pop50
-; CHECK-NEXT:    local.get $push90=, 6
-; CHECK-NEXT:    i64.load $push9=, 96($pop90)
-; CHECK-NEXT:    local.get $push91=, 6
-; CHECK-NEXT:    i32.const $push52=, 96
-; CHECK-NEXT:    i32.add $push53=, $pop91, $pop52
-; CHECK-NEXT:    i32.const $push70=, 8
-; CHECK-NEXT:    i32.add $push7=, $pop53, $pop70
-; CHECK-NEXT:    i64.load $push8=, 0($pop7)
-; CHECK-NEXT:    call sqrtl, $pop51, $pop9, $pop8
-; CHECK-NEXT:    local.get $push92=, 6
-; CHECK-NEXT:    i32.const $push46=, 64
-; CHECK-NEXT:    i32.add $push47=, $pop92, $pop46
-; CHECK-NEXT:    local.get $push93=, 6
-; CHECK-NEXT:    i64.load $push12=, 80($pop93)
-; CHECK-NEXT:    local.get $push94=, 6
-; CHECK-NEXT:    i32.const $push48=, 80
-; CHECK-NEXT:    i32.add $push49=, $pop94, $pop48
-; CHECK-NEXT:    i32.const $push69=, 8
-; CHECK-NEXT:    i32.add $push10=, $pop49, $pop69
-; CHECK-NEXT:    i64.load $push11=, 0($pop10)
-; CHECK-NEXT:    call floorl, $pop47, $pop12, $pop11
-; CHECK-NEXT:    local.get $push95=, 6
-; CHECK-NEXT:    i32.const $push42=, 48
-; CHECK-NEXT:    i32.add $push43=, $pop95, $pop42
-; CHECK-NEXT:    local.get $push96=, 6
-; CHECK-NEXT:    i64.load $push15=, 64($pop96)
-; CHECK-NEXT:    local.get $push97=, 6
-; CHECK-NEXT:    i32.const $push44=, 64
-; CHECK-NEXT:    i32.add $push45=, $pop97, $pop44
-; CHECK-NEXT:    i32.const $push68=, 8
-; CHECK-NEXT:    i32.add $push13=, $pop45, $pop68
-; CHECK-NEXT:    i64.load $push14=, 0($pop13)
-; CHECK-NEXT:    local.get $push99=, 3
-; CHECK-NEXT:    local.get $push98=, 4
-; CHECK-NEXT:    call powl, $pop43, $pop15, $pop14, $pop99, $pop98
-; CHECK-NEXT:    local.get $push100=, 6
-; CHECK-NEXT:    i32.const $push38=, 32
-; CHECK-NEXT:    i32.add $push39=, $pop100, $pop38
-; CHECK-NEXT:    local.get $push101=, 6
-; CHECK-NEXT:    i64.load $push18=, 48($pop101)
-; CHECK-NEXT:    local.get $push102=, 6
-; CHECK-NEXT:    i32.const $push40=, 48
-; CHECK-NEXT:    i32.add $push41=, $pop102, $pop40
-; CHECK-NEXT:    i32.const $push67=, 8
-; CHECK-NEXT:    i32.add $push16=, $pop41, $pop67
-; CHECK-NEXT:    i64.load $push17=, 0($pop16)
-; CHECK-NEXT:    local.get $push103=, 5
-; CHECK-NEXT:    call __powitf2, $pop39, $pop18, $pop17, $pop103
-; CHECK-NEXT:    local.get $push104=, 6
-; CHECK-NEXT:    i32.const $push34=, 16
-; CHECK-NEXT:    i32.add $push35=, $pop104, $pop34
-; CHECK-NEXT:    local.get $push105=, 6
-; CHECK-NEXT:    i64.load $push21=, 32($pop105)
-; CHECK-NEXT:    local.get $push106=, 6
-; CHECK-NEXT:    i32.const $push36=, 32
-; CHECK-NEXT:    i32.add $push37=, $pop106, $pop36
-; CHECK-NEXT:    i32.const $push66=, 8
-; CHECK-NEXT:    i32.add $push19=, $pop37, $pop66
-; CHECK-NEXT:    i64.load $push20=, 0($pop19)
-; CHECK-NEXT:    call truncl, $pop35, $pop21, $pop20
-; CHECK-NEXT:    local.get $push109=, 6
-; CHECK-NEXT:    local.get $push107=, 6
-; CHECK-NEXT:    i64.load $push24=, 16($pop107)
-; CHECK-NEXT:    local.get $push108=, 6
-; CHECK-NEXT:    i32.const $push32=, 16
-; CHECK-NEXT:    i32.add $push33=, $pop108, $pop32
-; CHECK-NEXT:    i32.const $push65=, 8
-; CHECK-NEXT:    i32.add $push22=, $pop33, $pop65
-; CHECK-NEXT:    i64.load $push23=, 0($pop22)
-; CHECK-NEXT:    call nearbyintl, $pop109, $pop24, $pop23
-; CHECK-NEXT:    local.get $push111=, 0
-; CHECK-NEXT:    local.get $push110=, 6
-; CHECK-NEXT:    i32.const $push64=, 8
-; CHECK-NEXT:    i32.add $push25=, $pop110, $pop64
-; CHECK-NEXT:    i64.load $push26=, 0($pop25)
-; CHECK-NEXT:    i64.store 8($pop111), $pop26
-; CHECK-NEXT:    local.get $push113=, 0
-; CHECK-NEXT:    local.get $push112=, 6
-; CHECK-NEXT:    i64.load $push27=, 0($pop112)
-; CHECK-NEXT:    i64.store 0($pop113), $pop27
-; CHECK-NEXT:    local.get $push114=, 6
-; CHECK-NEXT:    i32.const $push30=, 144
-; CHECK-NEXT:    i32.add $push31=, $pop114, $pop30
-; CHECK-NEXT:    global.set __stack_pointer, $pop31
+; CHECK-NEXT:    i32.const $push20=, 144
+; CHECK-NEXT:    i32.add $push21=, $pop80, $pop20
+; CHECK-NEXT:    global.set __stack_pointer, $pop21
 ; CHECK-NEXT:    return
   %a = fadd fp128 %x, %y
   %b = fmul fp128 %a, %y
@@ -180,55 +146,49 @@ define i128 @i128libcalls(i128 %x, i128 %y) {
 ; CHECK:         .functype i128libcalls (i32, i64, i64, i64, i64) -> ()
 ; CHECK-NEXT:    .local i32, i64
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push11=, __stack_pointer
-; CHECK-NEXT:    i32.const $push12=, 32
-; CHECK-NEXT:    i32.sub $push23=, $pop11, $pop12
-; CHECK-NEXT:    local.tee $push22=, 5, $pop23
-; CHECK-NEXT:    global.set __stack_pointer, $pop22
-; CHECK-NEXT:    local.get $push24=, 5
-; CHECK-NEXT:    i32.const $push17=, 16
-; CHECK-NEXT:    i32.add $push18=, $pop24, $pop17
-; CHECK-NEXT:    local.get $push26=, 1
-; CHECK-NEXT:    local.get $push25=, 3
-; CHECK-NEXT:    i64.add $push21=, $pop26, $pop25
-; CHECK-NEXT:    local.tee $push20=, 6, $pop21
-; CHECK-NEXT:    local.get $push28=, 2
-; CHECK-NEXT:    local.get $push27=, 4
-; CHECK-NEXT:    i64.add $push0=, $pop28, $pop27
-; CHECK-NEXT:    local.get $push30=, 6
-; CHECK-NEXT:    local.get $push29=, 1
-; CHECK-NEXT:    i64.lt_u $push1=, $pop30, $pop29
+; CHECK-NEXT:    global.get $push8=, __stack_pointer
+; CHECK-NEXT:    i32.const $push9=, 32
+; CHECK-NEXT:    i32.sub $push17=, $pop8, $pop9
+; CHECK-NEXT:    local.tee $push16=, 5, $pop17
+; CHECK-NEXT:    global.set __stack_pointer, $pop16
+; CHECK-NEXT:    local.get $push18=, 5
+; CHECK-NEXT:    i32.const $push12=, 16
+; CHECK-NEXT:    i32.add $push13=, $pop18, $pop12
+; CHECK-NEXT:    local.get $push20=, 1
+; CHECK-NEXT:    local.get $push19=, 3
+; CHECK-NEXT:    i64.add $push15=, $pop20, $pop19
+; CHECK-NEXT:    local.tee $push14=, 6, $pop15
+; CHECK-NEXT:    local.get $push22=, 2
+; CHECK-NEXT:    local.get $push21=, 4
+; CHECK-NEXT:    i64.add $push0=, $pop22, $pop21
+; CHECK-NEXT:    local.get $push24=, 6
+; CHECK-NEXT:    local.get $push23=, 1
+; CHECK-NEXT:    i64.lt_u $push1=, $pop24, $pop23
 ; CHECK-NEXT:    i64.extend_i32_u $push2=, $pop1
 ; CHECK-NEXT:    i64.add $push3=, $pop0, $pop2
-; CHECK-NEXT:    local.get $push32=, 3
-; CHECK-NEXT:    local.get $push31=, 4
-; CHECK-NEXT:    call __multi3, $pop18, $pop20, $pop3, $pop32, $pop31
-; CHECK-NEXT:    local.get $push37=, 5
-; CHECK-NEXT:    local.get $push33=, 5
-; CHECK-NEXT:    i64.load $push7=, 16($pop33)
+; CHECK-NEXT:    local.get $push26=, 3
+; CHECK-NEXT:    local.get $push25=, 4
+; CHECK-NEXT:    call __multi3, $pop13, $pop14, $pop3, $pop26, $pop25
+; CHECK-NEXT:    local.get $push31=, 5
+; CHECK-NEXT:    local.get $push27=, 5
+; CHECK-NEXT:    i64.load $push5=, 16($pop27)
+; CHECK-NEXT:    local.get $push28=, 5
+; CHECK-NEXT:    i64.load $push4=, 24($pop28)
+; CHECK-NEXT:    local.get $push30=, 3
+; CHECK-NEXT:    local.get $push29=, 4
+; CHECK-NEXT:    call __umodti3, $pop31, $pop5, $pop4, $pop30, $pop29
+; CHECK-NEXT:    local.get $push33=, 0
+; CHECK-NEXT:    local.get $push32=, 5
+; CHECK-NEXT:    i64.load $push6=, 8($pop32)
+; CHECK-NEXT:    i64.store 8($pop33), $pop6
+; CHECK-NEXT:    local.get $push35=, 0
 ; CHECK-NEXT:    local.get $push34=, 5
-; CHECK-NEXT:    i32.const $push15=, 16
-; CHECK-NEXT:    i32.add $push16=, $pop34, $pop15
-; CHECK-NEXT:    i32.const $push4=, 8
-; CHECK-NEXT:    i32.add $push5=, $pop16, $pop4
-; CHECK-NEXT:    i64.load $push6=, 0($pop5)
-; CHECK-NEXT:    local.get $push36=, 3
-; CHECK-NEXT:    local.get $push35=, 4
-; CHECK-NEXT:    call __umodti3, $pop37, $pop7, $pop6, $pop36, $pop35
-; CHECK-NEXT:    local.get $push39=, 0
-; CHECK-NEXT:    local.get $push38=, 5
-; CHECK-NEXT:    i32.const $push19=, 8
-; CHECK-NEXT:    i32.add $push8=, $pop38, $pop19
-; CHECK-NEXT:    i64.load $push9=, 0($pop8)
-; CHECK-NEXT:    i64.store 8($pop39), $pop9
-; CHECK-NEXT:    local.get $push41=, 0
-; CHECK-NEXT:    local.get $push40=, 5
-; CHECK-NEXT:    i64.load $push10=, 0($pop40)
-; CHECK-NEXT:    i64.store 0($pop41), $pop10
-; CHECK-NEXT:    local.get $push42=, 5
-; CHECK-NEXT:    i32.const $push13=, 32
-; CHECK-NEXT:    i32.add $push14=, $pop42, $pop13
-; CHECK-NEXT:    global.set __stack_pointer, $pop14
+; CHECK-NEXT:    i64.load $push7=, 0($pop34)
+; CHECK-NEXT:    i64.store 0($pop35), $pop7
+; CHECK-NEXT:    local.get $push36=, 5
+; CHECK-NEXT:    i32.const $push10=, 32
+; CHECK-NEXT:    i32.add $push11=, $pop36, $pop10
+; CHECK-NEXT:    global.set __stack_pointer, $pop11
 ; CHECK-NEXT:    return
   %a = add i128 %x, %y
   %b = mul i128 %a, %y
@@ -243,7 +203,7 @@ define double @f64libcalls(double %x, double %y, i32 %z) {
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    global.get $push12=, __stack_pointer
 ; CHECK-NEXT:    i32.const $push13=, 16
-; CHECK-NEXT:    i32.sub  $push19=, $pop12, $pop13
+; CHECK-NEXT:    i32.sub $push19=, $pop12, $pop13
 ; CHECK-NEXT:    local.tee $push18=, 3, $pop19
 ; CHECK-NEXT:    global.set __stack_pointer, $pop18
 ; CHECK-NEXT:    local.get $push23=, 0
@@ -263,7 +223,7 @@ define double @f64libcalls(double %x, double %y, i32 %z) {
 ; CHECK-NEXT:    call $push10=, ldexp, $pop23, $pop9
 ; CHECK-NEXT:    local.get $push24=, 3
 ; CHECK-NEXT:    i32.const $push16=, 12
-; CHECK-NEXT:    i32.add  $push17=, $pop24, $pop16
+; CHECK-NEXT:    i32.add $push17=, $pop24, $pop16
 ; CHECK-NEXT:    call $push25=, frexp, $pop10, $pop17
 ; CHECK-NEXT:    local.set 0, $pop25
 ; CHECK-NEXT:    local.get $push26=, 3
@@ -271,7 +231,7 @@ define double @f64libcalls(double %x, double %y, i32 %z) {
 ; CHECK-NEXT:    call escape_value, $pop11
 ; CHECK-NEXT:    local.get $push27=, 3
 ; CHECK-NEXT:    i32.const $push14=, 16
-; CHECK-NEXT:    i32.add  $push15=, $pop27, $pop14
+; CHECK-NEXT:    i32.add $push15=, $pop27, $pop14
 ; CHECK-NEXT:    global.set __stack_pointer, $pop15
 ; CHECK-NEXT:    local.get $push28=, 0
 ; CHECK-NEXT:    return $pop28
diff --git a/llvm/test/CodeGen/WebAssembly/multi-return.ll b/llvm/test/CodeGen/WebAssembly/multi-return.ll
index 293a1b35c39c6..2582a4e1be48c 100644
--- a/llvm/test/CodeGen/WebAssembly/multi-return.ll
+++ b/llvm/test/CodeGen/WebAssembly/multi-return.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
@@ -9,9 +10,20 @@ declare { i64, i128, i192, i128, i64 } @return_multi_multi()
 
 define i64 @test0() {
 ; CHECK-LABEL: test0:
-; CHECK: call    	return_multi_multi
-; CHECK: i64.load	$[[RV:[0-9]+]]=, 8(${{[0-9]+}})
-; CHECK: local.copy	$push8=, $[[RV]]
+; CHECK:         .functype test0 () -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push0=, __stack_pointer
+; CHECK-NEXT:    i32.const $push1=, 96
+; CHECK-NEXT:    i32.sub $push5=, $pop0, $pop1
+; CHECK-NEXT:    local.tee $push4=, $1=, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    call return_multi_multi, $1
+; CHECK-NEXT:    i64.load $0=, 0($1)
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.add $push3=, $1, $pop2
+; CHECK-NEXT:    global.set __stack_pointer, $pop3
+; CHECK-NEXT:    local.copy $push6=, $0
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %t1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 0
   ret i64 %t1
@@ -19,13 +31,22 @@ define i64 @test0() {
 
 define i128 @test1() {
 ; CHECK-LABEL: test1:
-; CHECK: call    	return_multi_multi
-; CHECK: i64.load	$[[RV:[0-9]+]]=, 16($[[SP:[0-9]+]])
-; CHECK: i32.const	$push0=, 24
-; CHECK: i32.add 	$push1=, $[[SP]], $pop0
-; CHECK: i64.load	$push2=, 0($pop1)
-; CHECK: i64.store	8($0), $pop2
-; CHECK: i64.store	0($0), $[[RV]]
+; CHECK:         .functype test1 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push1=, __stack_pointer
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.sub $push6=, $pop1, $pop2
+; CHECK-NEXT:    local.tee $push5=, $2=, $pop6
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
+; CHECK-NEXT:    call return_multi_multi, $2
+; CHECK-NEXT:    i64.load $1=, 24($2)
+; CHECK-NEXT:    i64.load $push0=, 16($2)
+; CHECK-NEXT:    i64.store 0($0), $pop0
+; CHECK-NEXT:    i64.store 8($0), $1
+; CHECK-NEXT:    i32.const $push3=, 96
+; CHECK-NEXT:    i32.add $push4=, $2, $pop3
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %t1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 1
   ret i128 %t1
@@ -33,17 +54,26 @@ define i128 @test1() {
 
 define i192 @test2() {
 ; CHECK-LABEL: test2:
-; CHECK: call    	return_multi_multi
-; CHECK: i32.const	$push0=, 40
-; CHECK: i32.add 	$push1=, $[[SP:[0-9]+]], $pop0
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 0($pop1)
-; CHECK: i64.load	$[[L2:[0-9]+]]=, 32($[[SP]])
-; CHECK: i32.const	$push2=, 48
-; CHECK: i32.add 	$push3=, $[[SP]], $pop2
-; CHECK: i64.load	$push4=, 0($pop3)
-; CHECK: i64.store	16($0), $pop4
-; CHECK: i64.store	0($0), $[[L2]]
-; CHECK: i64.store	8($0), $[[L1]]
+; CHECK:         .functype test2 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 96
+; CHECK-NEXT:    i32.sub $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, $3=, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    call return_multi_multi, $3
+; CHECK-NEXT:    i64.load $1=, 40($3)
+; CHECK-NEXT:    i64.load $2=, 32($3)
+; CHECK-NEXT:    i32.const $push0=, 48
+; CHECK-NEXT:    i32.add $push1=, $3, $pop0
+; CHECK-NEXT:    i64.load $push2=, 0($pop1)
+; CHECK-NEXT:    i64.store 16($0), $pop2
+; CHECK-NEXT:    i64.store 0($0), $2
+; CHECK-NEXT:    i64.store 8($0), $1
+; CHECK-NEXT:    i32.const $push5=, 96
+; CHECK-NEXT:    i32.add $push6=, $3, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %t1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 2
   ret i192 %t1
@@ -51,13 +81,22 @@ define i192 @test2() {
 
 define i128 @test3() {
 ; CHECK-LABEL: test3:
-; CHECK: call    	return_multi_multi
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 56($[[SP:[0-9]+]])
-; CHECK: i32.const	$push0=, 64
-; CHECK: i32.add 	$push1=, $[[SP]], $pop0
-; CHECK: i64.load	$push2=, 0($pop1)
-; CHECK: i64.store	8($0), $pop2
-; CHECK: i64.store	0($0), $[[L1]]
+; CHECK:         .functype test3 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push1=, __stack_pointer
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.sub $push6=, $pop1, $pop2
+; CHECK-NEXT:    local.tee $push5=, $2=, $pop6
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
+; CHECK-NEXT:    call return_multi_multi, $2
+; CHECK-NEXT:    i64.load $1=, 72($2)
+; CHECK-NEXT:    i64.load $push0=, 64($2)
+; CHECK-NEXT:    i64.store 0($0), $pop0
+; CHECK-NEXT:    i64.store 8($0), $1
+; CHECK-NEXT:    i32.const $push3=, 96
+; CHECK-NEXT:    i32.add $push4=, $2, $pop3
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %t1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 3
   ret i128 %t1
@@ -65,9 +104,20 @@ define i128 @test3() {
 
 define i64 @test4() {
 ; CHECK-LABEL: test4:
-; CHECK: call    	return_multi_multi
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 72($[[SP:[0-9]+]])
-; CHECK: local.copy	$push8=, $[[L1]]
+; CHECK:         .functype test4 () -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push0=, __stack_pointer
+; CHECK-NEXT:    i32.const $push1=, 96
+; CHECK-NEXT:    i32.sub $push5=, $pop0, $pop1
+; CHECK-NEXT:    local.tee $push4=, $1=, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    call return_multi_multi, $1
+; CHECK-NEXT:    i64.load $0=, 80($1)
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.add $push3=, $1, $pop2
+; CHECK-NEXT:    global.set __stack_pointer, $pop3
+; CHECK-NEXT:    local.copy $push6=, $0
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %t1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 4
   ret i64 %t1
@@ -77,17 +127,24 @@ define i64 @test4() {
 
 define { i64, i128 } @test5() {
 ; CHECK-LABEL: test5:
-; CHECK: call    	return_multi_multi
-; CHECK: i32.const	$push0=, 24
-; CHECK: i32.add 	$push1=, $[[SP:[0-9]+]], $pop0
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 0($pop1)
-; CHECK: i64.load	$[[L2:[0-9]+]]=, 8($[[SP]])
-; CHECK: i64.load	$push2=, 16($[[SP]])
-; CHECK: i64.store	8($0), $pop2
-; CHECK: i64.store	16($0), $[[L1]]
-; CHECK: i64.store	0($0), $[[L2]]
-; CHECK: i32.const	$push5=, 80
-; CHECK: i32.add 	$push6=, $3, $pop5
+; CHECK:         .functype test5 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push1=, __stack_pointer
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.sub $push6=, $pop1, $pop2
+; CHECK-NEXT:    local.tee $push5=, $3=, $pop6
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
+; CHECK-NEXT:    call return_multi_multi, $3
+; CHECK-NEXT:    i64.load $1=, 24($3)
+; CHECK-NEXT:    i64.load $2=, 16($3)
+; CHECK-NEXT:    i64.load $push0=, 0($3)
+; CHECK-NEXT:    i64.store 0($0), $pop0
+; CHECK-NEXT:    i64.store 16($0), $2
+; CHECK-NEXT:    i64.store 24($0), $1
+; CHECK-NEXT:    i32.const $push3=, 96
+; CHECK-NEXT:    i32.add $push4=, $3, $pop3
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %r0 = extractvalue { i64, i128, i192, i128, i64 } %t0, 0
   %r1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 1
@@ -98,21 +155,26 @@ define { i64, i128 } @test5() {
 
 define { i128, i128 } @test6() {
 ; CHECK-LABEL: test6:
-; CHECK: call    	return_multi_multi
-; CHECK: i32.const	$push0=, 24
-; CHECK: i32.add 	$push1=, $[[SP:[0-9]+]], $pop0
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 0($pop1)
-; CHECK: i32.const	$push2=, 64
-; CHECK: i32.add 	$push3=, $[[SP]], $pop2
-; CHECK: i64.load	$[[L2:[0-9]+]]=, 0($pop3)
-; CHECK: i64.load	$[[L3:[0-9]+]]=, 16($[[SP]])
-; CHECK: i64.load	$push4=, 56($[[SP]])
-; CHECK: i64.store	16($0), $pop4
-; CHECK: i64.store	24($0), $[[L2]]
-; CHECK: i64.store	0($0), $[[L3]]
-; CHECK: i64.store	8($0), $[[L1]]
-; CHECK: i32.const	$push7=, 80
-; CHECK: i32.add	$push8=, $4, $pop7
+; CHECK:         .functype test6 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push1=, __stack_pointer
+; CHECK-NEXT:    i32.const $push2=, 96
+; CHECK-NEXT:    i32.sub $push6=, $pop1, $pop2
+; CHECK-NEXT:    local.tee $push5=, $4=, $pop6
+; CHECK-NEXT:    global.set __stack_pointer, $pop5
+; CHECK-NEXT:    call return_multi_multi, $4
+; CHECK-NEXT:    i64.load $1=, 24($4)
+; CHECK-NEXT:    i64.load $2=, 16($4)
+; CHECK-NEXT:    i64.load $3=, 72($4)
+; CHECK-NEXT:    i64.load $push0=, 64($4)
+; CHECK-NEXT:    i64.store 16($0), $pop0
+; CHECK-NEXT:    i64.store 24($0), $3
+; CHECK-NEXT:    i64.store 0($0), $2
+; CHECK-NEXT:    i64.store 8($0), $1
+; CHECK-NEXT:    i32.const $push3=, 96
+; CHECK-NEXT:    i32.add $push4=, $4, $pop3
+; CHECK-NEXT:    global.set __stack_pointer, $pop4
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %r1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 1
   %r3 = extractvalue { i64, i128, i192, i128, i64 } %t0, 3
@@ -123,21 +185,28 @@ define { i128, i128 } @test6() {
 
 define { i64, i192 } @test7() {
 ; CHECK-LABEL: test7:
-; CHECK: call    	return_multi_multi
-; CHECK: i32.const	$push0=, 40
-; CHECK: i32.add 	$push1=, $[[SP:[0-9]+]], $pop0
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 0($pop1)
-; CHECK: i64.load	$[[L2:[0-9]+]]=, 8($[[SP]])
-; CHECK: i64.load	$[[L3:[0-9]+]]=, 32($[[SP]])
-; CHECK: i32.const	$push2=, 48
-; CHECK: i32.add 	$push3=, $[[SP]], $pop2
-; CHECK: i64.load	$push4=, 0($pop3)
-; CHECK: i64.store	24($0), $pop4
-; CHECK: i64.store	8($0), $[[L3]]
-; CHECK: i64.store	16($0), $[[L1]]
-; CHECK: i64.store	0($0), $[[L2]]
-; CHECK: i32.const	$push7=, 80
-; CHECK: i32.add 	$push8=, $4, $pop7
+; CHECK:         .functype test7 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 96
+; CHECK-NEXT:    i32.sub $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, $4=, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    call return_multi_multi, $4
+; CHECK-NEXT:    i32.const $push0=, 48
+; CHECK-NEXT:    i32.add $push1=, $4, $pop0
+; CHECK-NEXT:    i64.load $1=, 0($pop1)
+; CHECK-NEXT:    i64.load $2=, 40($4)
+; CHECK-NEXT:    i64.load $3=, 32($4)
+; CHECK-NEXT:    i64.load $push2=, 0($4)
+; CHECK-NEXT:    i64.store 0($0), $pop2
+; CHECK-NEXT:    i64.store 32($0), $1
+; CHECK-NEXT:    i64.store 16($0), $3
+; CHECK-NEXT:    i64.store 24($0), $2
+; CHECK-NEXT:    i32.const $push5=, 96
+; CHECK-NEXT:    i32.add $push6=, $4, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %r0 = extractvalue { i64, i128, i192, i128, i64 } %t0, 0
   %r2 = extractvalue { i64, i128, i192, i128, i64 } %t0, 2
@@ -148,33 +217,36 @@ define { i64, i192 } @test7() {
 
 define { i128, i192, i128, i64 } @test8() {
 ; CHECK-LABEL: test8:
-; CHECK: call    	return_multi_multi
-; CHECK: i32.const	$push0=, 64
-; CHECK: i32.add 	$push1=, $[[SP:[0-9]+]], $pop0
-; CHECK: i64.load	$[[L1:[0-9]+]]=, 0($pop1)
-; CHECK: i32.const	$push2=, 40
-; CHECK: i32.add 	$push3=, $[[SP]], $pop2
-; CHECK: i64.load	$[[L2:[0-9]+]]=, 0($pop3)
-; CHECK: i32.const	$push4=, 48
-; CHECK: i32.add 	$push5=, $[[SP]], $pop4
-; CHECK: i64.load	$[[L3:[0-9]+]]=, 0($pop5)
-; CHECK: i32.const	$push6=, 24
-; CHECK: i32.add 	$push7=, $[[SP]], $pop6
-; CHECK: i64.load	$[[L4:[0-9]+]]=, 0($pop7)
-; CHECK: i64.load	$[[L5:[0-9]+]]=, 8($[[SP]])
-; CHECK: i64.load	$[[L6:[0-9]+]]=, 56($[[SP]])
-; CHECK: i64.load	$[[L7:[0-9]+]]=, 32($[[SP]])
-; CHECK: i64.load	$push8=, 16($[[SP]])
-; CHECK: i64.store	40($0), $pop8
-; CHECK: i64.store	48($0), $[[L4]]
-; CHECK: i64.store	32($0), $[[L3]]
-; CHECK: i64.store	16($0), $[[L7]]
-; CHECK: i64.store	24($0), $[[L2]]
-; CHECK: i64.store	0($0), $[[L6]]
-; CHECK: i64.store	8($0), $[[L1]]
-; CHECK: i64.store	56($0), $[[L5]]
-; CHECK: i32.const	$push11=, 80
-; CHECK: i32.add 	$push12=, $8, $pop11
+; CHECK:         .functype test8 (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push3=, __stack_pointer
+; CHECK-NEXT:    i32.const $push4=, 96
+; CHECK-NEXT:    i32.sub $push8=, $pop3, $pop4
+; CHECK-NEXT:    local.tee $push7=, $8=, $pop8
+; CHECK-NEXT:    global.set __stack_pointer, $pop7
+; CHECK-NEXT:    call return_multi_multi, $8
+; CHECK-NEXT:    i32.const $push0=, 48
+; CHECK-NEXT:    i32.add $push1=, $8, $pop0
+; CHECK-NEXT:    i64.load $1=, 0($pop1)
+; CHECK-NEXT:    i64.load $2=, 72($8)
+; CHECK-NEXT:    i64.load $3=, 64($8)
+; CHECK-NEXT:    i64.load $4=, 40($8)
+; CHECK-NEXT:    i64.load $5=, 32($8)
+; CHECK-NEXT:    i64.load $6=, 24($8)
+; CHECK-NEXT:    i64.load $7=, 16($8)
+; CHECK-NEXT:    i64.load $push2=, 0($8)
+; CHECK-NEXT:    i64.store 64($0), $pop2
+; CHECK-NEXT:    i64.store 48($0), $7
+; CHECK-NEXT:    i64.store 56($0), $6
+; CHECK-NEXT:    i64.store 32($0), $1
+; CHECK-NEXT:    i64.store 16($0), $5
+; CHECK-NEXT:    i64.store 24($0), $4
+; CHECK-NEXT:    i64.store 0($0), $3
+; CHECK-NEXT:    i64.store 8($0), $2
+; CHECK-NEXT:    i32.const $push5=, 96
+; CHECK-NEXT:    i32.add $push6=, $8, $pop5
+; CHECK-NEXT:    global.set __stack_pointer, $pop6
+; CHECK-NEXT:    # fallthrough-return
   %t0 = call { i64, i128, i192, i128, i64 } @return_multi_multi()
   %r0 = extractvalue { i64, i128, i192, i128, i64 } %t0, 0
   %r1 = extractvalue { i64, i128, i192, i128, i64 } %t0, 1
diff --git a/llvm/test/CodeGen/WebAssembly/multivalue_libcall.ll b/llvm/test/CodeGen/WebAssembly/multivalue_libcall.ll
index 2958b115df9d3..c1343d32f80e5 100644
--- a/llvm/test/CodeGen/WebAssembly/multivalue_libcall.ll
+++ b/llvm/test/CodeGen/WebAssembly/multivalue_libcall.ll
@@ -34,9 +34,7 @@ define i128 @multivalue_sdiv(i128 %a, i128 %b) {
 ; NO_MULTIVALUE-NEXT:    call __divti3
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
-; NO_MULTIVALUE-NEXT:    i32.const 8
-; NO_MULTIVALUE-NEXT:    i32.add
-; NO_MULTIVALUE-NEXT:    i64.load 0
+; NO_MULTIVALUE-NEXT:    i64.load 8
 ; NO_MULTIVALUE-NEXT:    i64.store 8
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
@@ -80,9 +78,7 @@ define fp128 @multivalue_fsub(fp128 %a, fp128 %b) {
 ; NO_MULTIVALUE-NEXT:    call __subtf3
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
-; NO_MULTIVALUE-NEXT:    i32.const 8
-; NO_MULTIVALUE-NEXT:    i32.add
-; NO_MULTIVALUE-NEXT:    i64.load 0
+; NO_MULTIVALUE-NEXT:    i64.load 8
 ; NO_MULTIVALUE-NEXT:    i64.store 8
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
@@ -125,9 +121,7 @@ define i128 @multivalue_lshr(i128 %a, i128 %b) {
 ; NO_MULTIVALUE-NEXT:    call __ashlti3
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
-; NO_MULTIVALUE-NEXT:    i32.const 8
-; NO_MULTIVALUE-NEXT:    i32.add
-; NO_MULTIVALUE-NEXT:    i64.load 0
+; NO_MULTIVALUE-NEXT:    i64.load 8
 ; NO_MULTIVALUE-NEXT:    i64.store 8
 ; NO_MULTIVALUE-NEXT:    local.get 0
 ; NO_MULTIVALUE-NEXT:    local.get 5
diff --git a/llvm/test/CodeGen/WebAssembly/reference-types.ll b/llvm/test/CodeGen/WebAssembly/reference-types.ll
index 168aaec8f0943..3df383b023726 100644
--- a/llvm/test/CodeGen/WebAssembly/reference-types.ll
+++ b/llvm/test/CodeGen/WebAssembly/reference-types.ll
@@ -7,7 +7,11 @@ define void @reference-types() {
   ret void
 }
 
-; CHECK:      .int8 1
+; CHECK: .section .custom_section.target_features,"",@
+; CHECK-NEXT: .int8 2
+; CHECK-NEXT: .int8 43
+; CHECK-NEXT: .int8 22
+; CHECK-NEXT: .ascii "call-indirect-overlong"
 ; CHECK-NEXT: .int8 43
 ; CHECK-NEXT: .int8 15
 ; CHECK-NEXT: .ascii "reference-types"
diff --git a/llvm/test/CodeGen/WebAssembly/target-features-attrs.ll b/llvm/test/CodeGen/WebAssembly/target-features-attrs.ll
index 25dee51ac8c38..0e46b96591816 100644
--- a/llvm/test/CodeGen/WebAssembly/target-features-attrs.ll
+++ b/llvm/test/CodeGen/WebAssembly/target-features-attrs.ll
@@ -55,11 +55,14 @@ attributes #2 = { "target-features"="+reference-types" }
 ; Features in function attributes:
 ; +atomics, +nontrapping-fptoint, +reference-types
 ; CHECK-LABEL: .custom_section.target_features,"",@
-; CHECK-NEXT: .int8  3
+; CHECK-NEXT: .int8  4
 ; CHECK-NEXT: .int8  43
 ; CHECK-NEXT: .int8  7
 ; CHECK-NEXT: .ascii  "atomics"
 ; CHECK-NEXT: .int8  43
+; CHECK-NEXT: .int8  22
+; CHECK-NEXT: .ascii  "call-indirect-overlong"
+; CHECK-NEXT: .int8  43
 ; CHECK-NEXT: .int8  19
 ; CHECK-NEXT: .ascii  "nontrapping-fptoint"
 ; CHECK-NEXT: .int8  43
@@ -69,11 +72,14 @@ attributes #2 = { "target-features"="+reference-types" }
 ; Features in function attributes + features specified by -mattr= option:
 ; +atomics, +nontrapping-fptoint, +reference-types, +simd128
 ; SIMD128-LABEL: .custom_section.target_features,"",@
-; SIMD128-NEXT: .int8  4
+; SIMD128-NEXT: .int8  5
 ; SIMD128-NEXT: .int8  43
 ; SIMD128-NEXT: .int8  7
 ; SIMD128-NEXT: .ascii  "atomics"
 ; SIMD128-NEXT: .int8  43
+; SIMD128-NEXT: .int8  22
+; SIMD128-NEXT: .ascii  "call-indirect-overlong"
+; SIMD128-NEXT: .int8  43
 ; SIMD128-NEXT: .int8  19
 ; SIMD128-NEXT: .ascii  "nontrapping-fptoint"
 ; SIMD128-NEXT: .int8  43
diff --git a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
index ba10dd94a9838..1c77ad5c049a5 100644
--- a/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
+++ b/llvm/test/CodeGen/WebAssembly/target-features-cpus.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s -mcpu=mvp | FileCheck %s --check-prefixes MVP
 ; RUN: llc < %s -mcpu=generic | FileCheck %s --check-prefixes GENERIC
+; RUN: llc < %s -mcpu=lime1 | FileCheck %s --check-prefixes LIME1
 ; RUN: llc < %s | FileCheck %s --check-prefixes GENERIC
 ; RUN: llc < %s -mcpu=bleeding-edge | FileCheck %s --check-prefixes BLEEDING-EDGE
 
@@ -11,13 +12,19 @@ target triple = "wasm32-unknown-unknown"
 ; mvp: should not contain the target features section
 ; MVP-NOT: .custom_section.target_features,"",@
 
-; generic: +multivalue, +mutable-globals, +reference-types, +sign-ext
+; generic: +call-indirect-overlong, +multivalue, +mutable-globals, +reference-types, +sign-ext
 ; GENERIC-LABEL: .custom_section.target_features,"",@
-; GENERIC-NEXT: .int8  6
+; GENERIC-NEXT: .int8  8
 ; GENERIC-NEXT: .int8  43
 ; GENERIC-NEXT: .int8  11
 ; GENERIC-NEXT: .ascii  "bulk-memory"
 ; GENERIC-NEXT: .int8  43
+; GENERIC-NEXT: .int8  15
+; GENERIC-NEXT: .ascii  "bulk-memory-opt"
+; GENERIC-NEXT: .int8  43
+; GENERIC-NEXT: .int8  22
+; GENERIC-NEXT: .ascii  "call-indirect-overlong"
+; GENERIC-NEXT: .int8  43
 ; GENERIC-NEXT: .int8  10
 ; GENERIC-NEXT: .ascii  "multivalue"
 ; GENERIC-NEXT: .int8  43
@@ -33,12 +40,39 @@ target triple = "wasm32-unknown-unknown"
 ; GENERIC-NEXT: .int8  8
 ; GENERIC-NEXT: .ascii  "sign-ext"
 
-; bleeding-edge: +atomics, +bulk-memory, +exception-handling, +extended-const,
-;                +fp16, +multimemory, +multivalue, +mutable-globals,
-;                +nontrapping-fptoint, +relaxed-simd, +reference-types,
-;                +simd128, +sign-ext, +tail-call
+; lime1: +bulk-memory-opt, +call-indirect-overlong, +extended-const, +multivalue,
+;        +mutable-globals, +nontrapping-fptoint, +sign-ext
+; LIME1-LABEL: .custom_section.target_features,"",@
+; LIME1-NEXT: .int8  7
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  15
+; LIME1-NEXT: .ascii  "bulk-memory-opt"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  22
+; LIME1-NEXT: .ascii  "call-indirect-overlong"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  14
+; LIME1-NEXT: .ascii  "extended-const"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  10
+; LIME1-NEXT: .ascii  "multivalue"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  15
+; LIME1-NEXT: .ascii  "mutable-globals"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  19
+; LIME1-NEXT: .ascii  "nontrapping-fptoint"
+; LIME1-NEXT: .int8  43
+; LIME1-NEXT: .int8  8
+; LIME1-NEXT: .ascii  "sign-ext"
+
+; bleeding-edge: +atomics, +bulk-memory, +bulk-memory-opt,
+;                +call-indirect-overlong, +exception-handling,
+;                +extended-const, +fp16, +multimemory, +multivalue,
+;                +mutable-globals, +nontrapping-fptoint, +relaxed-simd,
+;                +reference-types, +simd128, +sign-ext, +tail-call
 ; BLEEDING-EDGE-LABEL: .section  .custom_section.target_features,"",@
-; BLEEDING-EDGE-NEXT: .int8  14
+; BLEEDING-EDGE-NEXT: .int8  16
 ; BLEEDING-EDGE-NEXT: .int8  43
 ; BLEEDING-EDGE-NEXT: .int8  7
 ; BLEEDING-EDGE-NEXT: .ascii  "atomics"
@@ -46,6 +80,12 @@ target triple = "wasm32-unknown-unknown"
 ; BLEEDING-EDGE-NEXT: .int8  11
 ; BLEEDING-EDGE-NEXT: .ascii  "bulk-memory"
 ; BLEEDING-EDGE-NEXT: .int8  43
+; BLEEDING-EDGE-NEXT: .int8  15
+; BLEEDING-EDGE-NEXT: .ascii  "bulk-memory-opt"
+; BLEEDING-EDGE-NEXT: .int8  43
+; BLEEDING-EDGE-NEXT: .int8  22
+; BLEEDING-EDGE-NEXT: .ascii  "call-indirect-overlong"
+; BLEEDING-EDGE-NEXT: .int8  43
 ; BLEEDING-EDGE-NEXT: .int8  18
 ; BLEEDING-EDGE-NEXT: .ascii  "exception-handling"
 ; BLEEDING-EDGE-NEXT: .int8  43
diff --git a/llvm/test/CodeGen/WebAssembly/target-features-tls.ll b/llvm/test/CodeGen/WebAssembly/target-features-tls.ll
index 45bc06b5d5c96..4abe01a73aeee 100644
--- a/llvm/test/CodeGen/WebAssembly/target-features-tls.ll
+++ b/llvm/test/CodeGen/WebAssembly/target-features-tls.ll
@@ -21,11 +21,14 @@ target triple = "wasm32-unknown-unknown"
 
 ; +bulk-memory
 ; BULK-MEM-LABEL: .custom_section.target_features,"",@
-; BULK-MEM-NEXT: .int8 2
+; BULK-MEM-NEXT: .int8 3
 ; BULK-MEM-NEXT: .int8 43
 ; BULK-MEM-NEXT: .int8 7
 ; BULK-MEM-NEXT: .ascii "atomics"
 ; BULK-MEM-NEXT: .int8 43
 ; BULK-MEM-NEXT: .int8 11
 ; BULK-MEM-NEXT: .ascii "bulk-memory"
+; BULK-MEM-NEXT: .int8 43
+; BULK-MEM-NEXT: .int8 15
+; BULK-MEM-NEXT: .ascii "bulk-memory-opt"
 ; BULK-MEM-NEXT: .tbss.foo,"T",@
diff --git a/llvm/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
index 90e1a176e38f9..110fb2d43580a 100644
--- a/llvm/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
@@ -3,78 +3,76 @@
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; WASM32-LABEL: muloti_test
-; WASM32: global.get      $push18=, __stack_pointer
-; WASM32: i32.const       $push19=, 48
-; WASM32: i32.sub         $push40=, $pop18, $pop19
-; WASM32: local.tee       $push39=, 5, $pop40
-; WASM32: global.set      __stack_pointer, $pop39
-; WASM32: local.get       $push41=, 5
-; WASM32: i32.const       $push24=, 32
-; WASM32: i32.add         $push25=, $pop41, $pop24
-; WASM32: local.get       $push43=, 1
+; WASM32: global.get      $push16=, __stack_pointer
+; WASM32: i32.const       $push17=, 48
+; WASM32: i32.sub         $push38=, $pop16, $pop17
+; WASM32: local.tee       $push37=, 5, $pop38
+; WASM32: global.set      __stack_pointer, $pop37
+; WASM32: local.get       $push39=, 5
+; WASM32: i32.const       $push22=, 32
+; WASM32: i32.add         $push23=, $pop39, $pop22
+; WASM32: local.get       $push41=, 1
 ; WASM32: i64.const       $push0=, 0
-; WASM32: local.get       $push42=, 3
-; WASM32: i64.const       $push38=, 0
-; WASM32: call            __multi3, $pop25, $pop43, $pop0, $pop42, $pop38
-; WASM32: local.get       $push44=, 5
-; WASM32: i32.const       $push22=, 16
-; WASM32: i32.add         $push23=, $pop44, $pop22
-; WASM32: local.get       $push46=, 4
-; WASM32: i64.const       $push37=, 0
-; WASM32: local.get       $push45=, 1
+; WASM32: local.get       $push40=, 3
 ; WASM32: i64.const       $push36=, 0
-; WASM32: call            __multi3, $pop23, $pop46, $pop37, $pop45, $pop36
-; WASM32: local.get       $push49=, 5
-; WASM32: local.get       $push48=, 2
+; WASM32: call __multi3,  $pop23, $pop41, $pop0, $pop40, $pop36
+; WASM32: local.get       $push42=, 5
+; WASM32: i32.const       $push20=, 16
+; WASM32: i32.add         $push21=, $pop42, $pop20
+; WASM32: local.get       $push44=, 4
 ; WASM32: i64.const       $push35=, 0
-; WASM32: local.get       $push47=, 3
+; WASM32: local.get       $push43=, 1
 ; WASM32: i64.const       $push34=, 0
-; WASM32: call            __multi3, $pop49, $pop48, $pop35, $pop47, $pop34
-; WASM32: local.get       $push51=, 0
+; WASM32: call __multi3,  $pop21, $pop44, $pop35, $pop43, $pop34
+; WASM32: local.get       $push47=, 5
+; WASM32: local.get       $push46=, 2
+; WASM32: i64.const       $push33=, 0
+; WASM32: local.get       $push45=, 3
+; WASM32: i64.const       $push32=, 0
+; WASM32: call __multi3,  $pop47, $pop46, $pop33, $pop45, $pop32
+; WASM32: local.get       $push49=, 0
+; WASM32: local.get       $push48=, 5
+; WASM32: i64.load        $push1=, 32($pop48)
+; WASM32: i64.store       0($pop49), $pop1
+; WASM32: local.get       $push53=, 0
 ; WASM32: local.get       $push50=, 5
-; WASM32: i64.load        $push1=, 32($pop50)
-; WASM32: i64.store       0($pop51), $pop1
-; WASM32: local.get       $push55=, 0
+; WASM32: i64.load        $push31=, 40($pop50)
+; WASM32: local.tee       $push30=, 3, $pop31
+; WASM32: local.get       $push51=, 5
+; WASM32: i64.load        $push3=, 0($pop51)
 ; WASM32: local.get       $push52=, 5
-; WASM32: i32.const       $push5=, 40
-; WASM32: i32.add         $push6=, $pop52, $pop5
-; WASM32: i64.load        $push33=, 0($pop6)
-; WASM32: local.tee       $push32=, 3, $pop33
-; WASM32: local.get       $push53=, 5
-; WASM32: i64.load        $push3=, 0($pop53)
-; WASM32: local.get       $push54=, 5
-; WASM32: i64.load        $push2=, 16($pop54)
+; WASM32: i64.load        $push2=, 16($pop52)
 ; WASM32: i64.add         $push4=, $pop3, $pop2
-; WASM32: i64.add         $push31=, $pop32, $pop4
-; WASM32: local.tee       $push30=, 1, $pop31
-; WASM32: i64.store       8($pop55), $pop30
-; WASM32: local.get       $push62=, 0
-; WASM32: local.get       $push56=, 2
-; WASM32: i64.const       $push29=, 0
-; WASM32: i64.ne          $push8=, $pop56, $pop29
-; WASM32: local.get       $push57=, 4
-; WASM32: i64.const       $push28=, 0
-; WASM32: i64.ne          $push7=, $pop57, $pop28
-; WASM32: i32.and         $push9=, $pop8, $pop7
-; WASM32: local.get       $push58=, 5
-; WASM32: i64.load        $push10=, 8($pop58)
+; WASM32: i64.add         $push29=, $pop30, $pop4
+; WASM32: local.tee       $push28=, 1, $pop29
+; WASM32: i64.store       8($pop53), $pop28
+; WASM32: local.get       $push60=, 0
+; WASM32: local.get       $push54=, 2
 ; WASM32: i64.const       $push27=, 0
-; WASM32: i64.ne          $push11=, $pop10, $pop27
-; WASM32: i32.or          $push12=, $pop9, $pop11
-; WASM32: local.get       $push59=, 5
-; WASM32: i64.load        $push13=, 24($pop59)
+; WASM32: i64.ne          $push6=, $pop54, $pop27
+; WASM32: local.get       $push55=, 4
 ; WASM32: i64.const       $push26=, 0
-; WASM32: i64.ne          $push14=, $pop13, $pop26
-; WASM32: i32.or          $push15=, $pop12, $pop14
-; WASM32: local.get       $push61=, 1
-; WASM32: local.get       $push60=, 3
-; WASM32: i64.lt_u        $push16=, $pop61, $pop60
-; WASM32: i32.or          $push17=, $pop15, $pop16
-; WASM32: i32.store8      16($pop62), $pop17
-; WASM32: local.get       $push63=, 5
-; WASM32: i32.const       $push20=, 48
-; WASM32: i32.add         $push21=, $pop63, $pop20
-; WASM32: global.set      __stack_pointer, $pop21
+; WASM32: i64.ne          $push5=, $pop55, $pop26
+; WASM32: i32.and         $push7=, $pop6, $pop5
+; WASM32: local.get       $push56=, 5
+; WASM32: i64.load        $push8=, 8($pop56)
+; WASM32: i64.const       $push25=, 0
+; WASM32: i64.ne          $push9=, $pop8, $pop25
+; WASM32: i32.or          $push10=, $pop7, $pop9
+; WASM32: local.get       $push57=, 5
+; WASM32: i64.load        $push11=, 24($pop57)
+; WASM32: i64.const       $push24=, 0
+; WASM32: i64.ne          $push12=, $pop11, $pop24
+; WASM32: i32.or          $push13=, $pop10, $pop12
+; WASM32: local.get       $push59=, 1
+; WASM32: local.get       $push58=, 3
+; WASM32: i64.lt_u        $push14=, $pop59, $pop58
+; WASM32: i32.or          $push15=, $pop13, $pop14
+; WASM32: i32.store8      16($pop60), $pop15
+; WASM32: local.get       $push61=, 5
+; WASM32: i32.const       $push18=, 48
+; WASM32: i32.add         $push19=, $pop61, $pop18
+; WASM32: global.set      __stack_pointer, $pop19
 
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/WinEH/wineh-scope-statenumbering.ll b/llvm/test/CodeGen/WinEH/wineh-scope-statenumbering.ll
new file mode 100644
index 0000000000000..9cebb9a8330ab
--- /dev/null
+++ b/llvm/test/CodeGen/WinEH/wineh-scope-statenumbering.ll
@@ -0,0 +1,45 @@
+; RUN: opt -mtriple=i386-pc-windows-msvc -S -x86-winehstate < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc19.42.34433"
+
+%struct.Destructor = type { ptr }
+
+define dso_local void @"?HandleDestructorCallWithException@@YAXPA_N@Z"(ptr noundef %destructorCalled) personality ptr @__CxxFrameHandler3 {
+entry:
+  %destructorCalled.addr = alloca ptr, align 4
+  %x = alloca %struct.Destructor, align 4
+  store ptr %destructorCalled, ptr %destructorCalled.addr, align 4
+  %0 = load ptr, ptr %destructorCalled.addr, align 4
+  %call = call x86_thiscallcc noundef ptr @"??0Destructor@@QAE@PA_N@Z"(ptr noundef nonnull align 4 dereferenceable(4) %x, ptr noundef %0)
+  ; CHECK:  store i32 0, ptr %9, align 4
+  ; CHECK-NEXT:  invoke void @llvm.seh.scope.begin()
+  invoke void @llvm.seh.scope.begin()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:
+  store i32 1, ptr inttoptr (i32 1 to ptr), align 4
+  ; CHECK:  store i32 -1, ptr %10, align 4
+  ; CHECK-NEXT:  invoke void @llvm.seh.scope.end()
+  invoke void @llvm.seh.scope.end()
+          to label %invoke.cont1 unwind label %ehcleanup
+
+invoke.cont1:
+  call x86_thiscallcc void @"??1Destructor@@QAE@XZ"(ptr noundef nonnull align 4 dereferenceable(4) %x) #1
+  ret void
+
+ehcleanup:
+  %1 = cleanuppad within none []
+  call x86_thiscallcc void @"??1Destructor@@QAE@XZ"(ptr noundef nonnull align 4 dereferenceable(4) %x) #1 [ "funclet"(token %1) ]
+  cleanupret from %1 unwind to caller
+}
+
+declare dso_local i32 @__CxxFrameHandler3(...)
+declare dso_local void @llvm.seh.scope.begin() #0
+declare dso_local void @llvm.seh.scope.end() #0
+
+declare dso_local x86_thiscallcc noundef ptr @"??0Destructor@@QAE@PA_N@Z"(ptr noundef nonnull returned align 4 dereferenceable(4) %this, ptr noundef %destructorCalled)
+declare dso_local x86_thiscallcc void @"??1Destructor@@QAE@XZ"(ptr noundef nonnull align 4 dereferenceable(4) %this) #1
+
+attributes #0 = { nounwind memory(none) }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/apx/cfcmov.ll b/llvm/test/CodeGen/X86/apx/cfcmov.ll
index f643120c9b50f..37ba3d451c2b1 100644
--- a/llvm/test/CodeGen/X86/apx/cfcmov.ll
+++ b/llvm/test/CodeGen/X86/apx/cfcmov.ll
@@ -93,3 +93,91 @@ define i64 @cfcmov64rr_inv(i64 %0) {
   %3 = select i1 %2, i64 0, i64 %0
   ret i64 %3
 }
+
+define void @cfcmov16mr(ptr %p, i16 %0) {
+; CHECK-LABEL: cfcmov16mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    cmpw %ax, %si
+; CHECK-NEXT:    cfcmovlew %si, (%rdi)
+; CHECK-NEXT:    retq
+  %2 = load i16, ptr %p, align 2
+  %3 = icmp sgt i16 %0, %2
+  %4 = select i1 %3, i16 %2, i16 %0
+  store i16 %4, ptr %p, align 2
+  ret void
+}
+
+define void @cfcmov32mr(ptr %p, i32 %0) {
+; CHECK-LABEL: cfcmov32mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl (%rdi), %esi
+; CHECK-NEXT:    cfcmovgl %esi, (%rdi)
+; CHECK-NEXT:    retq
+  %2 = load i32, ptr %p, align 4
+  %3 = call i32 @llvm.smax.i32(i32 %0, i32 %2)
+  store i32 %3, ptr %p, align 4
+  ret void
+}
+
+define void @cfcmov64mr(ptr %p, i64 %0) {
+; CHECK-LABEL: cfcmov64mr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpq (%rdi), %rsi
+; CHECK-NEXT:    cfcmovgq %rsi, (%rdi)
+; CHECK-NEXT:    retq
+  %2 = load i64, ptr %p, align 8
+  %3 = icmp sgt i64 %0, %2
+  %4 = select i1 %3, i64 %0, i64 %2
+  store i64 %4, ptr %p, align 8
+  ret void
+}
+
+define void @volatileload(ptr %p, i32 %0) {
+; CHECK-LABEL: volatileload:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    cmpl %eax, %esi
+; CHECK-NEXT:    cmovbl %esi, %eax
+; CHECK-NEXT:    movl %eax, (%rdi)
+; CHECK-NEXT:    retq
+  %2 = load volatile i32, ptr %p, align 4
+  %3 = call i32 @llvm.umin.i32(i32 %0, i32 %2)
+  store i32 %3, ptr %p, align 4
+  ret void
+}
+
+define void @atomicstore(ptr %p, i64 %0) {
+; CHECK-LABEL: atomicstore:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    cmpq %rax, %rsi
+; CHECK-NEXT:    cmovaq %rsi, %rax
+; CHECK-NEXT:    movq %rax, (%rdi)
+; CHECK-NEXT:    retq
+  %2 = load i64, ptr %p, align 8
+  %3 = icmp ugt i64 %0, %2
+  %4 = select i1 %3, i64 %0, i64 %2
+  store atomic i64 %4, ptr %p unordered, align 8
+  ret void
+}
+
+define void @loadstorediffptr(ptr %p, i32 %0) {
+; CHECK-LABEL: loadstorediffptr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    cmpl %eax, %esi
+; CHECK-NEXT:    cmovbel %eax, %esi
+; CHECK-NEXT:    movl %esi, 4(%rdi)
+; CHECK-NEXT:    retq
+  %2 = getelementptr [2 x i32], ptr %p, i32 0, i32 0
+  %3 = load i32, ptr %2, align 4
+  %4 = icmp ule i32 %0, %3
+  %5 = select i1 %4, i32 %3, i32 %0
+  %6 = getelementptr [2 x i32], ptr %p, i32 0, i32 1
+  store i32 %5, ptr %6, align 4
+  ret void
+}
+
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
diff --git a/llvm/test/CodeGen/X86/avx10_2-cmp.ll b/llvm/test/CodeGen/X86/avx10_2-cmp.ll
index de0bec7ea2695..140a20c17ea6d 100644
--- a/llvm/test/CodeGen/X86/avx10_2-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx10_2-cmp.ll
@@ -235,3 +235,44 @@ define i1 @dune_mem(ptr %xp, ptr %yp) {
     %1 = fcmp une double %x, %y
     ret i1 %1
 }
+
+define i32 @PR118606(x86_fp80 %val1) #0 {
+; X64-LABEL: PR118606:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldz
+; X64-NEXT:    fucomi %st(1), %st
+; X64-NEXT:    fstp %st(1)
+; X64-NEXT:    fld1
+; X64-NEXT:    fcmovne %st(1), %st
+; X64-NEXT:    fcmovu %st(1), %st
+; X64-NEXT:    fucompi %st(1), %st
+; X64-NEXT:    fstp %st(0)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: PR118606:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldz
+; X86-NEXT:    fucomi %st(1), %st
+; X86-NEXT:    fstp %st(1)
+; X86-NEXT:    fld1
+; X86-NEXT:    fcmovne %st(1), %st
+; X86-NEXT:    fcmovu %st(1), %st
+; X86-NEXT:    fucompi %st(1), %st
+; X86-NEXT:    fstp %st(0)
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+entry:
+  %cmp8 = fcmp oeq x86_fp80 %val1, 0xK00000000000000000000
+  %0 = select i1 %cmp8, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000
+  %cmp64 = fcmp ogt x86_fp80 %0, 0xK00000000000000000000
+  br i1 %cmp64, label %if.then66, label %if.end70
+
+if.then66:                                        ; preds = %entry
+  ret i32 0
+
+if.end70:                                         ; preds = %entry
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/X86/avx10_2fptosi_satcvtds.ll b/llvm/test/CodeGen/X86/avx10_2fptosi_satcvtds.ll
index 4a6556bdc4a91..494e4bc8e068e 100644
--- a/llvm/test/CodeGen/X86/avx10_2fptosi_satcvtds.ll
+++ b/llvm/test/CodeGen/X86/avx10_2fptosi_satcvtds.ll
@@ -85,8 +85,7 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    vcvttpd2qq %xmm1, %xmm1
+; X86-NEXT:    vcvttpd2qq %xmm0, %xmm1
 ; X86-NEXT:    vmovd %xmm1, %esi
 ; X86-NEXT:    xorl %ecx, %ecx
 ; X86-NEXT:    vucomisd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index 1c4bfa8422d81..7ce37c637a79c 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -672,7 +672,7 @@ define <32 x i16> @insert_v32i16(<32 x i16> %x, i16 %y, ptr %ptr) nounwind {
 ; KNL-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm1
 ; KNL-NEXT:    vmovd %edi, %xmm0
 ; KNL-NEXT:    vpbroadcastw %xmm0, %ymm0
-; KNL-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: insert_v32i16:
@@ -1466,7 +1466,7 @@ define zeroext i8 @test_extractelement_varible_v2i1(<2 x i64> %a, <2 x i64> %b,
 ; KNL-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $1, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,8), %eax
@@ -1497,7 +1497,7 @@ define zeroext i8 @test_extractelement_varible_v4i1(<4 x i32> %a, <4 x i32> %b,
 ; KNL-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vpcmpnleud %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $3, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,4), %eax
@@ -1528,7 +1528,7 @@ define zeroext i8 @test_extractelement_varible_v8i1(<8 x i32> %a, <8 x i32> %b,
 ; KNL-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
 ; KNL-NEXT:    vpcmpnleud %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdw %zmm0, %ymm0
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $7, %edi
@@ -1559,7 +1559,7 @@ define zeroext i8 @test_extractelement_varible_v16i1(<16 x i32> %a, <16 x i32> %
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    ## kill: def $edi killed $edi def $rdi
 ; KNL-NEXT:    vpcmpnleud %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdb %zmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $15, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi), %eax
@@ -1641,7 +1641,7 @@ define i32 @test_insertelement_variable_v32i1(<32 x i8> %a, i8 %b, i32 %index) n
 ; KNL-NEXT:    ## kill: def $esi killed $esi def $rsi
 ; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 = ~zmm0
 ; KNL-NEXT:    andl $31, %esi
 ; KNL-NEXT:    testb %dil, %dil
 ; KNL-NEXT:    vmovdqa %ymm0, (%rsp)
@@ -1695,7 +1695,7 @@ define i64 @test_insertelement_variable_v64i1(<64 x i8> %a, i8 %b, i32 %index) n
 ; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
 ; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
 ; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 = ~zmm0
 ; KNL-NEXT:    andl $63, %esi
 ; KNL-NEXT:    testb %dil, %dil
 ; KNL-NEXT:    vmovdqa64 %zmm0, (%rsp)
@@ -1827,7 +1827,7 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
 ; KNL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm2
 ; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; KNL-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm1 = ~zmm1
 ; KNL-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; KNL-NEXT:    vpinsrb $1, 488(%rbp), %xmm2, %xmm2
 ; KNL-NEXT:    vpinsrb $2, 496(%rbp), %xmm2, %xmm2
@@ -1862,7 +1862,7 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
 ; KNL-NEXT:    vpinsrb $15, 728(%rbp), %xmm3, %xmm3
 ; KNL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm2
-; KNL-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm2 = ~zmm2
 ; KNL-NEXT:    cmpb $0, 736(%rbp)
 ; KNL-NEXT:    vmovdqa %ymm0, {{[0-9]+}}(%rsp)
 ; KNL-NEXT:    vmovdqa %ymm2, {{[0-9]+}}(%rsp)
@@ -2063,12 +2063,12 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index
 ; KNL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
 ; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 = ~zmm0
 ; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
 ; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; KNL-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm1 = ~zmm1
 ; KNL-NEXT:    andl $127, %esi
 ; KNL-NEXT:    testb %dil, %dil
 ; KNL-NEXT:    vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
@@ -2159,14 +2159,33 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index
 define void @test_concat_v2i1(ptr %arg, ptr %arg1, ptr %arg2) nounwind {
 ; KNL-LABEL: test_concat_v2i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; KNL-NEXT:    vcvtph2ps %xmm0, %ymm0
-; KNL-NEXT:    vcmpltps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %k0
-; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vcmpltps %zmm0, %zmm1, %k1
+; KNL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; KNL-NEXT:    vcvtph2ps %xmm0, %xmm1
+; KNL-NEXT:    vmovss {{.*#+}} xmm2 = [6.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; KNL-NEXT:    vucomiss %xmm2, %xmm1
+; KNL-NEXT:    setb %al
+; KNL-NEXT:    andl $1, %eax
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
+; KNL-NEXT:    vcvtph2ps %xmm0, %xmm0
+; KNL-NEXT:    vucomiss %xmm2, %xmm0
+; KNL-NEXT:    setb %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vucomiss %xmm2, %xmm1
+; KNL-NEXT:    seta %al
+; KNL-NEXT:    andl $1, %eax
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vucomiss %xmm2, %xmm0
+; KNL-NEXT:    seta %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $1, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k1
 ; KNL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdw %zmm1, %ymm1
 ; KNL-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, (%rdx)
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 24eb9b3715ed6..5ce2b56cbd43a 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1441,30 +1441,56 @@ define <4 x i32> @zext_bool_logic(<4 x i64> %cond1, <4 x i64> %cond2, <4 x i32>
 define void @half_vec_compare(ptr %x, ptr %y) {
 ; KNL-LABEL: half_vec_compare:
 ; KNL:       ## %bb.0: ## %entry
-; KNL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x07]
-; KNL-NEXT:    vcvtph2ps %xmm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x13,0xc0]
-; KNL-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; KNL-NEXT:    vcmpneqps %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfc,0xc2,0xc1,0x04]
-; KNL-NEXT:    vpmovdb %zmm0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0]
+; KNL-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; KNL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07]
+; KNL-NEXT:    vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55]
+; KNL-NEXT:    ## xmm1 = xmm0[1,1,1,1,4,5,6,7]
+; KNL-NEXT:    vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9]
+; KNL-NEXT:    xorl %eax, %eax ## encoding: [0x31,0xc0]
+; KNL-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
+; KNL-NEXT:    vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca]
+; KNL-NEXT:    movl $65535, %ecx ## encoding: [0xb9,0xff,0xff,0x00,0x00]
+; KNL-NEXT:    ## imm = 0xFFFF
+; KNL-NEXT:    movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00]
+; KNL-NEXT:    cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1]
+; KNL-NEXT:    cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1]
+; KNL-NEXT:    vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0]
+; KNL-NEXT:    vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2]
+; KNL-NEXT:    cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1]
+; KNL-NEXT:    cmovpl %ecx, %eax ## encoding: [0x0f,0x4a,0xc1]
+; KNL-NEXT:    vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0]
+; KNL-NEXT:    vpinsrw $1, %edx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc2,0x01]
+; KNL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
 ; KNL-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
 ; KNL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; KNL-NEXT:    vpextrw $0, %xmm0, (%rsi) ## encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
-; KNL-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; KNL-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512BW-LABEL: half_vec_compare:
 ; AVX512BW:       ## %bb.0: ## %entry
-; AVX512BW-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512BW-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x07]
-; AVX512BW-NEXT:    vcvtph2ps %xmm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x13,0xc0]
-; AVX512BW-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; AVX512BW-NEXT:    vcmpneqps %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfc,0xc2,0xc1,0x04]
-; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0]
+; AVX512BW-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512BW-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07]
+; AVX512BW-NEXT:    vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55]
+; AVX512BW-NEXT:    ## xmm1 = xmm0[1,1,1,1,4,5,6,7]
+; AVX512BW-NEXT:    vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9]
+; AVX512BW-NEXT:    xorl %eax, %eax ## encoding: [0x31,0xc0]
+; AVX512BW-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
+; AVX512BW-NEXT:    vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca]
+; AVX512BW-NEXT:    movl $65535, %ecx ## encoding: [0xb9,0xff,0xff,0x00,0x00]
+; AVX512BW-NEXT:    ## imm = 0xFFFF
+; AVX512BW-NEXT:    movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00]
+; AVX512BW-NEXT:    cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1]
+; AVX512BW-NEXT:    cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1]
+; AVX512BW-NEXT:    vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0]
+; AVX512BW-NEXT:    vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2]
+; AVX512BW-NEXT:    cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1]
+; AVX512BW-NEXT:    cmovpl %ecx, %eax ## encoding: [0x0f,0x4a,0xc1]
+; AVX512BW-NEXT:    vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0]
+; AVX512BW-NEXT:    vpinsrw $1, %edx, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc2,0x01]
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
 ; AVX512BW-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
-; AVX512BW-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; AVX512BW-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: half_vec_compare:
diff --git a/llvm/test/CodeGen/X86/avx512cfmulsh-instrinsics.ll b/llvm/test/CodeGen/X86/avx512cfmulsh-instrinsics.ll
index 43e085f37ff67..e449c7192e4bf 100644
--- a/llvm/test/CodeGen/X86/avx512cfmulsh-instrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512cfmulsh-instrinsics.ll
@@ -277,3 +277,15 @@ define <4 x float> @test_int_x86_avx512fp16_maskz_cfcmadd_sh(<4 x float> %x0, <4
   %res = call <4 x float> @llvm.x86.avx512fp16.maskz.vfcmadd.csh(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 9)
   ret <4 x float> %res
 }
+
+define <4 x float> @PR98306() {
+; CHECK-LABEL: PR98306:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kxorw %k0, %k0, %k1
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm1 = [7.8125E-3,1.050912E+6,4.203776E+6,1.6815616E+7]
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [3.2E+1,4.03288064E+8,8.0658432E+8,1.61318502E+9]
+; CHECK-NEXT:    vfmaddcsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512fp16.maskz.vfmadd.csh(<4 x float> <float 7.812500e-03, float 0x4130092000000000, float 0x4150094000000000, float 0x4170096000000000>, <4 x float> <float 2.000000e+00, float 0x4188098000000000, float 0x4198099000000000, float 0x41A809A000000000>, <4 x float> <float 3.200000e+01, float 0x41B809B000000000, float 0x41C809C000000000, float 0x41D809D000000000>, i8 0, i32 4)
+  ret <4 x float> %res
+}
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 27348dd31958f..a6b3e3fd1fd16 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -1983,8 +1983,6 @@ define bfloat @PR115710(fp128 %0) nounwind {
 ; X86-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
 ; X86-NEXT:    vmovups %xmm0, (%esp)
 ; X86-NEXT:    calll __trunctfbf2
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    vmovw %eax, %xmm0
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/combine-fabs.ll b/llvm/test/CodeGen/X86/combine-fabs.ll
index a862ea16a748f..7aa6628cb7f39 100644
--- a/llvm/test/CodeGen/X86/combine-fabs.ll
+++ b/llvm/test/CodeGen/X86/combine-fabs.ll
@@ -135,6 +135,100 @@ define <4 x float> @combine_vec_fabs_fcopysign(<4 x float> %a, <4 x float> %b) {
   ret <4 x float> %2
 }
 
+; store(fabs(load())) - convert scalar to integer
+define void @combine_fabs_int_rmw_f64(ptr %ptr) {
+; SSE-LABEL: combine_fabs_int_rmw_f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    andb $127, 7(%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_fabs_int_rmw_f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    andb $127, 7(%rdi)
+; AVX-NEXT:    retq
+  %1 = load double, ptr %ptr
+  %2 = call double @llvm.fabs.f64(double %1)
+  store double %2, ptr %ptr
+  ret void
+}
+
+define void @combine_fabs_int_f32(ptr %src, ptr %dst) {
+; SSE-LABEL: combine_fabs_int_f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
+; SSE-NEXT:    andl (%rdi), %eax
+; SSE-NEXT:    movl %eax, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_fabs_int_f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
+; AVX-NEXT:    andl (%rdi), %eax
+; AVX-NEXT:    movl %eax, (%rsi)
+; AVX-NEXT:    retq
+  %1 = load float, ptr %src
+  %2 = call float @llvm.fabs.f32(float %1)
+  store float %2, ptr %dst
+  ret void
+}
+
+define void @combine_fabs_int_rmw_bfloat(ptr %ptr) nounwind {
+; SSE-LABEL: combine_fabs_int_rmw_bfloat:
+; SSE:       # %bb.0:
+; SSE-NEXT:    andb $127, 1(%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_fabs_int_rmw_bfloat:
+; AVX:       # %bb.0:
+; AVX-NEXT:    andb $127, 1(%rdi)
+; AVX-NEXT:    retq
+  %1 = load bfloat, ptr %ptr
+  %2 = call bfloat @llvm.fabs.bf16(bfloat %1)
+  store bfloat %2, ptr %ptr
+  ret void
+}
+
+define void @combine_fabs_int_half(ptr %src, ptr %dst) nounwind {
+; SSE-LABEL: combine_fabs_int_half:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movzwl (%rdi), %eax
+; SSE-NEXT:    andl $32767, %eax # imm = 0x7FFF
+; SSE-NEXT:    movw %ax, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_fabs_int_half:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl (%rdi), %eax
+; AVX-NEXT:    andl $32767, %eax # imm = 0x7FFF
+; AVX-NEXT:    movw %ax, (%rsi)
+; AVX-NEXT:    retq
+  %1 = load half, ptr %src
+  %2 = call half @llvm.fabs.f16(half %1)
+  store half %2, ptr %dst
+  ret void
+}
+
+; don't convert vector to scalar
+define void @combine_fabs_vec_int_v4f32(ptr %src, ptr %dst) {
+; SSE-LABEL: combine_fabs_vec_int_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm0
+; SSE-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_fabs_vec_int_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; AVX-NEXT:    vandps (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vmovaps %xmm0, (%rsi)
+; AVX-NEXT:    retq
+  %1 = load <4 x float>, ptr %src
+  %2 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %1)
+  store <4 x float> %2, ptr %dst
+  ret void
+}
+
 declare float @llvm.fabs.f32(float %p)
 declare float @llvm.copysign.f32(float %Mag, float %Sgn)
 
diff --git a/llvm/test/CodeGen/X86/combine-fneg.ll b/llvm/test/CodeGen/X86/combine-fneg.ll
index e4a07348dc96c..8ca7fb81563fa 100644
--- a/llvm/test/CodeGen/X86/combine-fneg.ll
+++ b/llvm/test/CodeGen/X86/combine-fneg.ll
@@ -205,4 +205,113 @@ define <4 x float> @fneg(<4 x float> %Q) nounwind {
   ret <4 x float> %tmp
 }
 
+; store(fneg(load())) - convert scalar to integer
+define void @fneg_int_rmw_half(ptr %ptr) nounwind {
+; X86-SSE-LABEL: fneg_int_rmw_half:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    xorb $-128, 1(%eax)
+; X86-SSE-NEXT:    retl
+;
+; X64-SSE-LABEL: fneg_int_rmw_half:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    xorb $-128, 1(%rdi)
+; X64-SSE-NEXT:    retq
+  %1 = load half, ptr %ptr
+  %2 = fneg half %1
+  store half %2, ptr %ptr
+  ret void
+}
+
+define void @fneg_int_bfloat(ptr %src, ptr %dst) nounwind {
+; X86-SSE-LABEL: fneg_int_bfloat:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    movzwl (%ecx), %ecx
+; X86-SSE-NEXT:    xorl $32768, %ecx # imm = 0x8000
+; X86-SSE-NEXT:    movw %cx, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X64-SSE-LABEL: fneg_int_bfloat:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movzwl (%rdi), %eax
+; X64-SSE-NEXT:    xorl $32768, %eax # imm = 0x8000
+; X64-SSE-NEXT:    movw %ax, (%rsi)
+; X64-SSE-NEXT:    retq
+  %1 = load bfloat, ptr %src
+  %2 = fneg bfloat %1
+  store bfloat %2, ptr %dst
+  ret void
+}
+
+define void @fneg_int_rmw_f32(ptr %ptr) {
+; X86-SSE-LABEL: fneg_int_rmw_f32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    xorb $-128, 3(%eax)
+; X86-SSE-NEXT:    retl
+;
+; X64-SSE-LABEL: fneg_int_rmw_f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    xorb $-128, 3(%rdi)
+; X64-SSE-NEXT:    retq
+  %1 = load float, ptr %ptr
+  %2 = fneg float %1
+  store float %2, ptr %ptr
+  ret void
+}
 
+define void @fneg_int_f64(ptr %src, ptr %dst) {
+; X86-SSE1-LABEL: fneg_int_f64:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    fldl (%ecx)
+; X86-SSE1-NEXT:    fchs
+; X86-SSE1-NEXT:    fstpl (%eax)
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: fneg_int_f64:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: fneg_int_f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; X64-SSE-NEXT:    xorq (%rdi), %rax
+; X64-SSE-NEXT:    movq %rax, (%rsi)
+; X64-SSE-NEXT:    retq
+  %1 = load double, ptr %src
+  %2 = fneg double %1
+  store double %2, ptr %dst
+  ret void
+}
+
+; don't convert vector to scalar
+define void @fneg_int_v4f32(ptr %src, ptr %dst) {
+; X86-SSE-LABEL: fneg_int_v4f32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    movaps (%ecx), %xmm0
+; X86-SSE-NEXT:    xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    movaps %xmm0, (%eax)
+; X86-SSE-NEXT:    retl
+;
+; X64-SSE-LABEL: fneg_int_v4f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movaps (%rdi), %xmm0
+; X64-SSE-NEXT:    xorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE-NEXT:    movaps %xmm0, (%rsi)
+; X64-SSE-NEXT:    retq
+  %1 = load <4 x float>, ptr %src
+  %2 = fneg <4 x float> %1
+  store <4 x float> %2, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/extract-vselect-setcc.ll b/llvm/test/CodeGen/X86/extract-vselect-setcc.ll
new file mode 100644
index 0000000000000..81ab104cab283
--- /dev/null
+++ b/llvm/test/CodeGen/X86/extract-vselect-setcc.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64 | FileCheck %s
+
+define void @PR117684(i1 %cond, <8 x float> %vec, ptr %ptr1, ptr %ptr2) #0 {
+; CHECK-LABEL: PR117684:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vcmpnltps %ymm1, %ymm0, %k1
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; CHECK-NEXT:    vinsertf32x4 $0, %xmm0, %ymm0, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; CHECK-NEXT:    vbroadcastss %xmm2, %ymm2
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    cmoveq %rdx, %rsi
+; CHECK-NEXT:    vmovups %ymm2, (%rsi)
+; CHECK-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vbroadcastss %xmm0, %ymm0
+; CHECK-NEXT:    vmovups %ymm0, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %cmp = fcmp olt <8 x float> %vec, zeroinitializer
+  %sel1 = select <8 x i1> %cmp, <8 x float> zeroinitializer, <8 x float>
+  <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000,
+  float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>
+  %fmul1 = fmul <8 x float> zeroinitializer, %sel1
+  %shuffle = shufflevector <8 x float> %fmul1, <8 x float> zeroinitializer, <8 x i32> zeroinitializer
+  %fmul2 = fmul <8 x float> %shuffle,
+  <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000,
+  float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>
+  %sel2 = select i1 %cond, ptr %ptr1, ptr %ptr2
+  store <8 x float> %fmul2, ptr %sel2, align 4
+  %fmul3 = fmul <8 x float> %shuffle, zeroinitializer
+  store <8 x float> %fmul3, ptr %ptr2, align 4
+  ret void
+}
+
+attributes #0 = { "target-cpu"="skylake-avx512" }
diff --git a/llvm/test/CodeGen/X86/fast-isel-expect.ll b/llvm/test/CodeGen/X86/fast-isel-expect.ll
index 9c2d6488db3c9..e804076304e30 100644
--- a/llvm/test/CodeGen/X86/fast-isel-expect.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-expect.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -O0 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -6,9 +7,17 @@ target triple = "i686-unknown-linux-gnu"
 @glbl = extern_weak constant i8
 
 declare i64 @llvm.expect.i64(i64, i64)
+declare i64 @llvm.expect.with.probability.i64(i64, i64, double)
 
-define void @test() {
-; CHECK: movl $glbl
+define void @test_expect() {
+; CHECK-LABEL: test_expect:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $glbl, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:  .LBB0_1: # %bb1
+; CHECK-NEXT:  .LBB0_2: # %bb2
   %cmp2 = icmp eq ptr @glbl, null
   %ext = zext i1 %cmp2 to i64
   %tmp = call i64 @llvm.expect.i64(i64 %ext, i64 0)
@@ -21,3 +30,25 @@ bb1:
 bb2:
   unreachable
 }
+
+define void @test_expect_with_probability() {
+; CHECK-LABEL: test_expect_with_probability:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $glbl, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jne .LBB1_2
+; CHECK-NEXT:    jmp .LBB1_1
+; CHECK-NEXT:  .LBB1_1: # %bb1
+; CHECK-NEXT:  .LBB1_2: # %bb2
+  %cmp2 = icmp eq ptr @glbl, null
+  %ext = zext i1 %cmp2 to i64
+  %tmp = call i64 @llvm.expect.with.probability.i64(i64 %ext, i64 0, double 0.5)
+  %tmp2 = icmp ne i64 %tmp, 0
+  br i1 %tmp2, label %bb1, label %bb2
+
+bb1:
+  unreachable
+
+bb2:
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index 07701f082b0e2..c6da0c5ca4792 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -207,7 +207,7 @@ define float @test_fmaximum_nnan(float %x, float %y) nounwind {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vaddss %xmm1, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vsubss %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0
+; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isPositiveZero(xmm0)
 ; AVX512DQ-NEXT:    kmovw %k0, %k1
 ; AVX512DQ-NEXT:    vmovaps %xmm2, %xmm1
 ; AVX512DQ-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
@@ -466,7 +466,7 @@ define float @test_fmaximum_combine_cmps(float %x, float %y) nounwind {
 ; AVX512DQ-LABEL: test_fmaximum_combine_cmps:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vdivss %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0
+; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isPositiveZero(xmm0)
 ; AVX512DQ-NEXT:    kmovw %k0, %k1
 ; AVX512DQ-NEXT:    vmovaps %xmm1, %xmm2
 ; AVX512DQ-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
@@ -687,7 +687,7 @@ define double @test_fminimum_nnan(double %x, double %y) "no-nans-fp-math"="true"
 ;
 ; AVX512DQ-LABEL: test_fminimum_nnan:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vfpclasssd $5, %xmm1, %k0
+; AVX512DQ-NEXT:    vfpclasssd $5, %xmm1, %k0 # k0 = isQuietNaN(xmm1) | isNegativeZero(xmm1)
 ; AVX512DQ-NEXT:    kmovw %k0, %k1
 ; AVX512DQ-NEXT:    vmovapd %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
@@ -940,7 +940,7 @@ define float @test_fminimum_combine_cmps(float %x, float %y) nounwind {
 ; AVX512DQ-LABEL: test_fminimum_combine_cmps:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vdivss %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vfpclassss $5, %xmm0, %k0
+; AVX512DQ-NEXT:    vfpclassss $5, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isNegativeZero(xmm0)
 ; AVX512DQ-NEXT:    kmovw %k0, %k1
 ; AVX512DQ-NEXT:    vmovaps %xmm1, %xmm2
 ; AVX512DQ-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
@@ -1384,11 +1384,10 @@ define <4 x float> @test_fmaximum_v4f32_splat(<4 x float> %x, float %y) {
   ret <4 x float> %r
 }
 
-define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
+define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) nounwind {
 ; SSE2-LABEL: test_fmaximum_v4f16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    subq $104, %rsp
-; SSE2-NEXT:    .cfi_def_cfa_offset 112
 ; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1524,13 +1523,11 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
 ; SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    addq $104, %rsp
-; SSE2-NEXT:    .cfi_def_cfa_offset 8
 ; SSE2-NEXT:    retq
 ;
 ; AVX1-LABEL: test_fmaximum_v4f16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    subq $120, %rsp
-; AVX1-NEXT:    .cfi_def_cfa_offset 128
 ; AVX1-NEXT:    vmovaps %xmm0, %xmm2
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1636,37 +1633,179 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
 ; AVX1-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX1-NEXT:    addq $120, %rsp
-; AVX1-NEXT:    .cfi_def_cfa_offset 8
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: test_fmaximum_v4f16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm2
-; AVX512-NEXT:    vcvtph2ps %xmm1, %ymm3
-; AVX512-NEXT:    vcmpltps %ymm2, %ymm3, %ymm4
-; AVX512-NEXT:    vpmovdw %zmm4, %ymm4
-; AVX512-NEXT:    vpblendvb %xmm4, %xmm0, %xmm1, %xmm4
-; AVX512-NEXT:    vcmpunordps %ymm3, %ymm2, %ymm2
-; AVX512-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; AVX512-NEXT:    vpblendvb %xmm2, %xmm3, %xmm4, %xmm2
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm4
-; AVX512-NEXT:    vpblendvb %xmm4, %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm3
-; AVX512-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vcvtph2ps %xmm2, %ymm1
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm1
-; AVX512-NEXT:    vpmovdw %zmm1, %ymm1
-; AVX512-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    pushq %rbp
+; AVX512-NEXT:    pushq %r15
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    pushq %r13
+; AVX512-NEXT:    pushq %r12
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmoval %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %esi
+; AVX512-NEXT:    cmovpl %ecx, %esi
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r8d
+; AVX512-NEXT:    cmovpl %ecx, %r8d
+; AVX512-NEXT:    movl $0, %r11d
+; AVX512-NEXT:    cmoval %ecx, %r11d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r10d
+; AVX512-NEXT:    cmovpl %ecx, %r10d
+; AVX512-NEXT:    movl $0, %ebp
+; AVX512-NEXT:    cmoval %ecx, %ebp
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %ebx
+; AVX512-NEXT:    cmovpl %ecx, %ebx
+; AVX512-NEXT:    movl $0, %r14d
+; AVX512-NEXT:    cmoval %ecx, %r14d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r15d
+; AVX512-NEXT:    cmovpl %ecx, %r15d
+; AVX512-NEXT:    movl $0, %r12d
+; AVX512-NEXT:    cmoval %ecx, %r12d
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r13d
+; AVX512-NEXT:    cmoval %ecx, %r13d
+; AVX512-NEXT:    vmovd %r13d, %xmm2
+; AVX512-NEXT:    vpinsrw $1, %r12d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $2, %r14d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $3, %ebp, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $4, %r11d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $5, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $6, %edi, %xmm2, %xmm2
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmovpl %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm3, %xmm4
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vpinsrw $7, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm2
+; AVX512-NEXT:    vmovd %edi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %r15d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $2, %ebx, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $3, %r10d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $4, %r8d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $5, %esi, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    vpinsrw $7, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm3
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %esi # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %esi
+; AVX512-NEXT:    cmovpl %eax, %esi
+; AVX512-NEXT:    vmovd %esi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $2, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $3, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $4, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    cmovnel %eax, %ecx
+; AVX512-NEXT:    cmovpl %eax, %ecx
+; AVX512-NEXT:    vpinsrw $7, %ecx, %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm5
+; AVX512-NEXT:    vpblendvb %xmm5, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm1, %xmm4
+; AVX512-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    popq %r12
+; AVX512-NEXT:    popq %r13
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    popq %r15
+; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    retq
 ;
 ; X86-LABEL: test_fmaximum_v4f16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    subl $164, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 168
 ; X86-NEXT:    vmovdqa %xmm0, %xmm2
 ; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-NEXT:    vpsrlq $48, %xmm0, %xmm0
@@ -1806,7 +1945,6 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
 ; X86-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
 ; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; X86-NEXT:    addl $164, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
   %r = call <4 x half> @llvm.maximum.v4f16(<4 x half> %x, <4 x half> %y)
   ret <4 x half> %r
diff --git a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
index 332fbf7188af8..2163121410553 100644
--- a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -141,56 +141,61 @@ declare <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half>, <8 x i16>)
 define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) {
 ; CHECK-SSE-LABEL: fmul_pow2_8xhalf:
 ; CHECK-SSE:       # %bb.0:
-; CHECK-SSE-NEXT:    subq $88, %rsp
-; CHECK-SSE-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-SSE-NEXT:    subq $104, %rsp
+; CHECK-SSE-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; CHECK-SSE-NEXT:    pslld $23, %xmm1
 ; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216]
 ; CHECK-SSE-NEXT:    paddd %xmm2, %xmm1
 ; CHECK-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; CHECK-SSE-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
+; CHECK-SSE-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE-NEXT:    pslld $16, %xmm1
+; CHECK-SSE-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
 ; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; CHECK-SSE-NEXT:    pslld $23, %xmm0
 ; CHECK-SSE-NEXT:    paddd %xmm2, %xmm0
 ; CHECK-SSE-NEXT:    cvttps2dq %xmm0, %xmm0
+; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-SSE-NEXT:    pslld $16, %xmm0
-; CHECK-SSE-NEXT:    psrld $16, %xmm0
 ; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-SSE-NEXT:    psrld $16, %xmm0
 ; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    cvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-SSE-NEXT:    psrlq $48, %xmm0
+; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[2,3,2,3]
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[3,3,3,3]
-; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-SSE-NEXT:    cvtdq2ps %xmm1, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
-; CHECK-SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
-; CHECK-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-SSE-NEXT:    psrld $16, %xmm0
 ; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    cvtdq2ps (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-SSE-NEXT:    psrlq $48, %xmm0
+; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    pshufd $238, (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[2,3,2,3]
+; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    pshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[3,3,3,3]
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
 ; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
@@ -202,39 +207,39 @@ define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) {
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    punpcklwd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-SSE-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
 ; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
-; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-SSE-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-SSE-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-SSE-NEXT:    punpckldq (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-SSE-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
 ; CHECK-SSE-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
@@ -246,14 +251,13 @@ define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) {
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-SSE-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; CHECK-SSE-NEXT:    punpcklqdq (%rsp), %xmm1 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm1 = xmm1[0],mem[0]
-; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm0
-; CHECK-SSE-NEXT:    addq $88, %rsp
+; CHECK-SSE-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-SSE-NEXT:    punpckldq (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; CHECK-SSE-NEXT:    addq $104, %rsp
 ; CHECK-SSE-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-SSE-NEXT:    retq
 ;
@@ -1028,17 +1032,17 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
 ; CHECK-SSE-NEXT:    pslld $23, %xmm0
 ; CHECK-SSE-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    cvttps2dq %xmm0, %xmm0
-; CHECK-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [2,2,u,u,u,u,u,u]
-; CHECK-SSE-NEXT:    pxor %xmm0, %xmm0
-; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-SSE-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-SSE-NEXT:    cvtdq2ps %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; CHECK-SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [2,2,u,u,u,u,u,u]
+; CHECK-SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE-NEXT:    psrld $16, %xmm0
+; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = mem[1,1,1,1]
-; CHECK-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    xorps %xmm0, %xmm0
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-SSE-NEXT:    cvtdq2ps %xmm1, %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
@@ -1049,8 +1053,9 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
 ; CHECK-SSE-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-SSE-NEXT:    mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE-NEXT:    callq __truncsfhf2@PLT
-; CHECK-SSE-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-SSE-NEXT:    addq $40, %rsp
 ; CHECK-SSE-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/fp16-libcalls.ll b/llvm/test/CodeGen/X86/fp16-libcalls.ll
index 5c6c3653a27f9..1515cd1366bc6 100644
--- a/llvm/test/CodeGen/X86/fp16-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp16-libcalls.ll
@@ -73,7 +73,7 @@ define void @test_half_copysign(half %a0, half %a1, ptr %p0) nounwind {
 ; FP16-LABEL: test_half_copysign:
 ; FP16:       # %bb.0:
 ; FP16-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; FP16-NEXT:    vpternlogd $202, %xmm1, %xmm0, %xmm2
+; FP16-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & (xmm0 ^ xmm1))
 ; FP16-NEXT:    vmovsh %xmm2, (%rdi)
 ; FP16-NEXT:    retq
 ;
@@ -379,22 +379,10 @@ define void @test_half_fabs(half %a0, ptr %p0) nounwind {
 ;
 ; X86-LABEL: test_half_fabs:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    pinsrw $0, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    pextrw $0, %xmm0, %eax
-; X86-NEXT:    movw %ax, (%esp)
-; X86-NEXT:    calll __extendhfsf2
-; X86-NEXT:    fstps {{[0-9]+}}(%esp)
-; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-NEXT:    movd %xmm0, (%esp)
-; X86-NEXT:    calll __truncsfhf2
-; X86-NEXT:    pextrw $0, %xmm0, %eax
-; X86-NEXT:    movw %ax, (%esi)
-; X86-NEXT:    addl $8, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT:    movw %cx, (%eax)
 ; X86-NEXT:    retl
   %res = call half @llvm.fabs.half(half %a0)
   store half %res, ptr %p0, align 2
@@ -584,22 +572,10 @@ define void @test_half_fneg(half %a0, ptr %p0) nounwind {
 ;
 ; X86-LABEL: test_half_fneg:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    pinsrw $0, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    pextrw $0, %xmm0, %eax
-; X86-NEXT:    movw %ax, (%esp)
-; X86-NEXT:    calll __extendhfsf2
-; X86-NEXT:    fstps {{[0-9]+}}(%esp)
-; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-NEXT:    movd %xmm0, (%esp)
-; X86-NEXT:    calll __truncsfhf2
-; X86-NEXT:    pextrw $0, %xmm0, %eax
-; X86-NEXT:    movw %ax, (%esi)
-; X86-NEXT:    addl $8, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $32768, %ecx # imm = 0x8000
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movw %cx, (%eax)
 ; X86-NEXT:    retl
   %res = fneg half %a0
   store half %res, ptr %p0, align 2
diff --git a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
index 6aad4c2ebba1d..2dedb10d42fb4 100644
--- a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
@@ -731,7 +731,7 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: stest_f16i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2qq %ymm0, %zmm0
 ; AVX512-NEXT:    vpmovsqd %ymm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -894,7 +894,7 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: utesth_f16i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2uqq %ymm0, %zmm0
 ; AVX512-NEXT:    vpmovusqd %ymm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -1031,7 +1031,7 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: ustest_f16i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2qq %ymm0, %zmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
@@ -3343,7 +3343,7 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: stest_f16i32_mm:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2qq %ymm0, %zmm0
 ; AVX512-NEXT:    vpmovsqd %ymm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -3504,7 +3504,7 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: utesth_f16i32_mm:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2uqq %ymm0, %zmm0
 ; AVX512-NEXT:    vpmovusqd %ymm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -3640,7 +3640,7 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) nounwind {
 ;
 ; AVX512-LABEL: ustest_f16i32_mm:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2qq %ymm0, %zmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index fe240286462e9..362b3b945f962 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -630,13 +630,8 @@ define void @pr59677(i32 %x, ptr %out) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    vmovd %eax, %xmm0
-; X86-NEXT:    orl $1, %eax
-; X86-NEXT:    vmovd %eax, %xmm1
-; X86-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
@@ -651,10 +646,6 @@ define void @pr59677(i32 %x, ptr %out) nounwind {
 ; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    movq %rsi, %rbx
 ; X64-NEXT:    vmovd %edi, %xmm0
-; X64-NEXT:    orl $1, %edi
-; X64-NEXT:    vmovd %edi, %xmm1
-; X64-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; X64-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X64-NEXT:    callq sinf@PLT
diff --git a/llvm/test/CodeGen/X86/fsxor-alignment.ll b/llvm/test/CodeGen/X86/fsxor-alignment.ll
index 34624cc4cf8fb..6fa4a310956dc 100644
--- a/llvm/test/CodeGen/X86/fsxor-alignment.ll
+++ b/llvm/test/CodeGen/X86/fsxor-alignment.ll
@@ -5,18 +5,19 @@
 ; to do floating-point negations, because the arguments aren't vectors
 ; and aren't vector-aligned.
 
-define void @foo(ptr %p, ptr %q, float %s, float %y) {
+define void @foo(ptr %p, ptr %q, float %s, float %y) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    xorps %xmm1, %xmm0
-; CHECK-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT:    xorps %xmm1, %xmm2
-; CHECK-NEXT:    movss %xmm0, (%ecx)
-; CHECK-NEXT:    movss %xmm2, (%eax)
+; CHECK-NEXT:    movl $-2147483648, %edx # imm = 0x80000000
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    xorl %edx, %esi
+; CHECK-NEXT:    movl %esi, (%ecx)
+; CHECK-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl %edx, (%eax)
+; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    retl
   %ss = fsub float -0.0, %s
   %yy = fsub float -0.0, %y
diff --git a/llvm/test/CodeGen/X86/gfni-lzcnt.ll b/llvm/test/CodeGen/X86/gfni-lzcnt.ll
new file mode 100644
index 0000000000000..e84af84b36aa9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/gfni-lzcnt.ll
@@ -0,0 +1,510 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2,+gfni | FileCheck %s --check-prefixes=GFNISSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
+
+define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv16i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    pxor %xmm3, %xmm3
+; GFNISSE-NEXT:    pcmpeqb %xmm0, %xmm3
+; GFNISSE-NEXT:    pand %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
+; GFNISSE-NEXT:    paddb %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1OR2-LABEL: testv16i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
+; GFNIAVX1OR2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv16i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
+; GFNIAVX512-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv16i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    pxor %xmm3, %xmm3
+; GFNISSE-NEXT:    pcmpeqb %xmm0, %xmm3
+; GFNISSE-NEXT:    pand %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
+; GFNISSE-NEXT:    paddb %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1OR2-LABEL: testv16i8u:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
+; GFNIAVX1OR2-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv16i8u:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
+; GFNIAVX512-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv32i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    pxor %xmm5, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm6
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
+; GFNISSE-NEXT:    pand %xmm3, %xmm0
+; GFNISSE-NEXT:    paddb %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    pcmpeqb %xmm1, %xmm5
+; GFNISSE-NEXT:    pand %xmm3, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
+; GFNISSE-NEXT:    paddb %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv32i8:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm6
+; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm4
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv32i8:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX2-NEXT:    # ymm1 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv32i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512-NEXT:    # ymm1 = mem[0,1,0,1]
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
+; GFNIAVX512-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; GFNIAVX512-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv32i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    pxor %xmm5, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm6
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
+; GFNISSE-NEXT:    pand %xmm3, %xmm0
+; GFNISSE-NEXT:    paddb %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    pcmpeqb %xmm1, %xmm5
+; GFNISSE-NEXT:    pand %xmm3, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
+; GFNISSE-NEXT:    paddb %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv32i8u:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm6
+; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm4
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv32i8u:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX2-NEXT:    # ymm1 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv32i8u:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512-NEXT:    # ymm1 = mem[0,1,0,1]
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
+; GFNIAVX512-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; GFNIAVX512-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv64i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
+; GFNISSE-NEXT:    movq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm7
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    pxor %xmm5, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
+; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    paddb %xmm8, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; GFNISSE-NEXT:    pand %xmm7, %xmm1
+; GFNISSE-NEXT:    paddb %xmm8, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm2
+; GFNISSE-NEXT:    pand %xmm7, %xmm2
+; GFNISSE-NEXT:    paddb %xmm8, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm4, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
+; GFNISSE-NEXT:    pcmpeqb %xmm4, %xmm5
+; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm4, %xmm3
+; GFNISSE-NEXT:    paddb %xmm5, %xmm3
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv64i8:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv64i8:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
+; GFNIAVX2-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
+; GFNIAVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512VL-LABEL: testv64i8:
+; GFNIAVX512VL:       # %bb.0:
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
+; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    retq
+;
+; GFNIAVX512BW-LABEL: testv64i8:
+; GFNIAVX512BW:       # %bb.0:
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm3
+; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k0
+; GFNIAVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; GFNIAVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpaddb %zmm3, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    retq
+  %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0)
+  ret <64 x i8> %out
+}
+
+define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv64i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
+; GFNISSE-NEXT:    movq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm7
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    pxor %xmm5, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
+; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    paddb %xmm8, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; GFNISSE-NEXT:    pand %xmm7, %xmm1
+; GFNISSE-NEXT:    paddb %xmm8, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm8
+; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm2
+; GFNISSE-NEXT:    pand %xmm7, %xmm2
+; GFNISSE-NEXT:    paddb %xmm8, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
+; GFNISSE-NEXT:    pshufb %xmm4, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
+; GFNISSE-NEXT:    pcmpeqb %xmm4, %xmm5
+; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm4, %xmm3
+; GFNISSE-NEXT:    paddb %xmm5, %xmm3
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv64i8u:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv64i8u:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
+; GFNIAVX2-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
+; GFNIAVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512VL-LABEL: testv64i8u:
+; GFNIAVX512VL:       # %bb.0:
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
+; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    retq
+;
+; GFNIAVX512BW-LABEL: testv64i8u:
+; GFNIAVX512BW:       # %bb.0:
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm3
+; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k0
+; GFNIAVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; GFNIAVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpaddb %zmm3, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    retq
+  %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1)
+  ret <64 x i8> %out
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFNIAVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/gfni-tzcnt.ll b/llvm/test/CodeGen/X86/gfni-tzcnt.ll
new file mode 100644
index 0000000000000..f424483c53e2c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/gfni-tzcnt.ll
@@ -0,0 +1,590 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2,+gfni | FileCheck %s --check-prefixes=GFNISSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
+
+define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv16i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; GFNISSE-NEXT:    paddb %xmm0, %xmm1
+; GFNISSE-NEXT:    pandn %xmm1, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    pand %xmm0, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
+; GFNISSE-NEXT:    paddb %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1OR2-LABEL: testv16i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX1OR2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; GFNIAVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv16i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; GFNIAVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv16i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; GFNISSE-NEXT:    paddb %xmm0, %xmm1
+; GFNISSE-NEXT:    pandn %xmm1, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    pand %xmm0, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
+; GFNISSE-NEXT:    paddb %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1OR2-LABEL: testv16i8u:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX1OR2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; GFNIAVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv16i8u:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1
+; GFNIAVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv32i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    pcmpeqd %xmm4, %xmm4
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
+; GFNISSE-NEXT:    paddb %xmm4, %xmm2
+; GFNISSE-NEXT:    pandn %xmm2, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm3
+; GFNISSE-NEXT:    pand %xmm5, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
+; GFNISSE-NEXT:    pshufb %xmm3, %xmm6
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
+; GFNISSE-NEXT:    paddb %xmm6, %xmm3
+; GFNISSE-NEXT:    paddb %xmm1, %xmm4
+; GFNISSE-NEXT:    pandn %xmm4, %xmm1
+; GFNISSE-NEXT:    pand %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm1
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
+; GFNISSE-NEXT:    paddb %xmm0, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv32i8:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; GFNIAVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
+; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv32i8:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX2-NEXT:    vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv32i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512-NEXT:    vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv32i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    pcmpeqd %xmm4, %xmm4
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
+; GFNISSE-NEXT:    paddb %xmm4, %xmm2
+; GFNISSE-NEXT:    pandn %xmm2, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm3
+; GFNISSE-NEXT:    pand %xmm5, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
+; GFNISSE-NEXT:    pshufb %xmm3, %xmm6
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
+; GFNISSE-NEXT:    paddb %xmm6, %xmm3
+; GFNISSE-NEXT:    paddb %xmm1, %xmm4
+; GFNISSE-NEXT:    pandn %xmm4, %xmm1
+; GFNISSE-NEXT:    pand %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm1
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
+; GFNISSE-NEXT:    paddb %xmm0, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv32i8u:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; GFNIAVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1-NEXT:    vpshufb %xmm4, %xmm5, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm2
+; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv32i8u:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX2-NEXT:    vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512-LABEL: testv32i8u:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512-NEXT:    vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512-NEXT:    # ymm2 = mem[0,1,0,1]
+; GFNIAVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv64i8:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
+; GFNISSE-NEXT:    pcmpeqd %xmm6, %xmm6
+; GFNISSE-NEXT:    paddb %xmm6, %xmm0
+; GFNISSE-NEXT:    pandn %xmm0, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm9
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm0
+; GFNISSE-NEXT:    paddb %xmm9, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm1
+; GFNISSE-NEXT:    paddb %xmm6, %xmm1
+; GFNISSE-NEXT:    pandn %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm1
+; GFNISSE-NEXT:    pand %xmm7, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm1
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm1
+; GFNISSE-NEXT:    paddb %xmm9, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    paddb %xmm6, %xmm5
+; GFNISSE-NEXT:    pandn %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm5
+; GFNISSE-NEXT:    paddb %xmm9, %xmm5
+; GFNISSE-NEXT:    paddb %xmm3, %xmm6
+; GFNISSE-NEXT:    pandn %xmm6, %xmm3
+; GFNISSE-NEXT:    pand %xmm3, %xmm7
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm2
+; GFNISSE-NEXT:    pshufb %xmm7, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm3, %xmm4
+; GFNISSE-NEXT:    paddb %xmm2, %xmm4
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm3
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv64i8:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; GFNIAVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm4
+; GFNIAVX1-NEXT:    vpandn %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm5
+; GFNIAVX1-NEXT:    vpandn %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm0, %xmm5
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vpandn %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv64i8:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX2-NEXT:    # ymm5 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512VL-LABEL: testv64i8:
+; GFNIAVX512VL:       # %bb.0:
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; GFNIAVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; GFNIAVX512VL-NEXT:    vpandn %ymm3, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512VL-NEXT:    # ymm5 = mem[0,1,0,1]
+; GFNIAVX512VL-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    retq
+;
+; GFNIAVX512BW-LABEL: testv64i8:
+; GFNIAVX512BW:       # %bb.0:
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm1
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    retq
+  %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 0)
+  ret <64 x i8> %out
+}
+
+define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
+; GFNISSE-LABEL: testv64i8u:
+; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
+; GFNISSE-NEXT:    pcmpeqd %xmm6, %xmm6
+; GFNISSE-NEXT:    paddb %xmm6, %xmm0
+; GFNISSE-NEXT:    pandn %xmm0, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm0, %xmm9
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm0
+; GFNISSE-NEXT:    paddb %xmm9, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm1
+; GFNISSE-NEXT:    paddb %xmm6, %xmm1
+; GFNISSE-NEXT:    pandn %xmm1, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm1
+; GFNISSE-NEXT:    pand %xmm7, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm1, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm1
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm1
+; GFNISSE-NEXT:    paddb %xmm9, %xmm1
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    paddb %xmm6, %xmm5
+; GFNISSE-NEXT:    pandn %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm9
+; GFNISSE-NEXT:    pshufb %xmm5, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm5
+; GFNISSE-NEXT:    pshufb %xmm2, %xmm5
+; GFNISSE-NEXT:    paddb %xmm9, %xmm5
+; GFNISSE-NEXT:    paddb %xmm3, %xmm6
+; GFNISSE-NEXT:    pandn %xmm6, %xmm3
+; GFNISSE-NEXT:    pand %xmm3, %xmm7
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm2
+; GFNISSE-NEXT:    pshufb %xmm7, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm3
+; GFNISSE-NEXT:    pshufb %xmm3, %xmm4
+; GFNISSE-NEXT:    paddb %xmm2, %xmm4
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm3
+; GFNISSE-NEXT:    retq
+;
+; GFNIAVX1-LABEL: testv64i8u:
+; GFNIAVX1:       # %bb.0:
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; GFNIAVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm4
+; GFNIAVX1-NEXT:    vpandn %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm5
+; GFNIAVX1-NEXT:    vpandn %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm0, %xmm5
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vpandn %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; GFNIAVX1-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpand %xmm4, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
+; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    retq
+;
+; GFNIAVX2-LABEL: testv64i8u:
+; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm0, %ymm4
+; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX2-NEXT:    # ymm5 = mem[0,1,0,1]
+; GFNIAVX2-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpand %ymm3, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    retq
+;
+; GFNIAVX512VL-LABEL: testv64i8u:
+; GFNIAVX512VL:       # %bb.0:
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; GFNIAVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm1, %ymm3
+; GFNIAVX512VL-NEXT:    vpandn %ymm3, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512VL-NEXT:    # ymm5 = mem[0,1,0,1]
+; GFNIAVX512VL-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    retq
+;
+; GFNIAVX512BW-LABEL: testv64i8u:
+; GFNIAVX512BW:       # %bb.0:
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
+; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm1
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0
+; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    retq
+  %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 -1)
+  ret <64 x i8> %out
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFNIAVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/load-partial.ll b/llvm/test/CodeGen/X86/load-partial.ll
index bec6cce889142..dba63582ff08b 100644
--- a/llvm/test/CodeGen/X86/load-partial.ll
+++ b/llvm/test/CodeGen/X86/load-partial.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX
 
 ;
 ; Partial Vector Loads - PR16739
@@ -382,38 +382,24 @@ define dso_local i32 @load_partial_illegal_type()  {
 define dso_local void @PR43227(ptr %explicit_0, ptr %explicit_1) {
 ; SSE-LABEL: PR43227:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE-NEXT:    psrlq $32, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    movdqa %xmm1, 672(%rsi)
-; SSE-NEXT:    movdqa %xmm0, 688(%rsi)
+; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm0, 672(%rsi)
+; SSE-NEXT:    movaps %xmm1, 688(%rsi)
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: PR43227:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovaps %ymm0, 672(%rsi)
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: PR43227:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    vmovdqa %ymm0, 672(%rsi)
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
+; AVX-LABEL: PR43227:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps %ymm0, 672(%rsi)
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %1 = getelementptr i32, ptr %explicit_0, i64 63
   %2 = load <3 x i32>, ptr %1, align 1
   %3 = shufflevector <3 x i32> %2, <3 x i32> undef, <2 x i32> <i32 1, i32 2>
diff --git a/llvm/test/CodeGen/X86/mmx-cvt.ll b/llvm/test/CodeGen/X86/mmx-cvt.ll
index 51a71dab37f6d..07e56a9148cf0 100644
--- a/llvm/test/CodeGen/X86/mmx-cvt.ll
+++ b/llvm/test/CodeGen/X86/mmx-cvt.ll
@@ -279,6 +279,21 @@ define <4 x float> @cvt_v2i32_v2f32(ptr) nounwind {
   ret <4 x float> %8
 }
 
+define noundef <2 x i64> @cvt_f64_v2i64(double %a0) {
+; X86-LABEL: cvt_f64_v2i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    retl
+;
+; X64-LABEL: cvt_f64_v2i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
+; X64-NEXT:    retq
+  %bc = bitcast double %a0 to <1 x i64>
+  %r = shufflevector <1 x i64> %bc, <1 x i64> zeroinitializer, <2 x i32> <i32 0, i32 1>
+  ret <2 x i64> %r
+}
+
 declare <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64>, <1 x i64>)
 declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>)
 declare <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double>)
diff --git a/llvm/test/CodeGen/X86/no-trap-after-noreturn-fastisel.ll b/llvm/test/CodeGen/X86/no-trap-after-noreturn-fastisel.ll
new file mode 100644
index 0000000000000..5149209f79d15
--- /dev/null
+++ b/llvm/test/CodeGen/X86/no-trap-after-noreturn-fastisel.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 -trap-unreachable -no-trap-after-noreturn -fast-isel-abort=3 < %s | FileCheck %s
+
+declare void @foo()
+
+define void @noreturn_unreachable() nounwind {
+; CHECK-LABEL: noreturn_unreachable:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq foo@PLT
+  call void @foo() noreturn
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/pr114360.ll b/llvm/test/CodeGen/X86/pr114360.ll
new file mode 100644
index 0000000000000..cf510854cce66
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr114360.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; REQUIRES: asserts
+; RUN: llc < %s -mtriple=x86_64-- -debug-counter=dagcombine=0 | FileCheck %s
+
+; BUG: shrinkAndImmediate folds away the AND after the ZEXT has already been folded away to SUBREG_TO_REG losing implicit zext.
+define i64 @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $-4294967295, %rax # imm = 0xFFFFFFFF00000001
+; CHECK-NEXT:    movzwl %ax, %eax
+; CHECK-NEXT:    retq
+  %x = bitcast i64 u0xffffffff00000001 to i64
+  %t = trunc i64 %x to i32
+  %a = and i32 %t, 1
+  %e = zext i32 %a to i64
+  ret i64 %e
+}
diff --git a/llvm/test/CodeGen/X86/pr116153.ll b/llvm/test/CodeGen/X86/pr116153.ll
new file mode 100644
index 0000000000000..5c9c2c76131d5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr116153.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define void @_test_func(<16 x half> %0) #0 {
+; CHECK-LABEL: _test_func:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7]
+; CHECK-NEXT:    vcvtph2ps %xmm1, %xmm1
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vucomiss %xmm1, %xmm1
+; CHECK-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; CHECK-NEXT:    movl $0, %edx
+; CHECK-NEXT:    cmovnpl %ecx, %edx
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-NEXT:    vcvtph2ps %xmm1, %xmm1
+; CHECK-NEXT:    vucomiss %xmm1, %xmm1
+; CHECK-NEXT:    movl $0, %esi
+; CHECK-NEXT:    cmovnpl %ecx, %esi
+; CHECK-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[1,1,1,1,4,5,6,7]
+; CHECK-NEXT:    vcvtph2ps %xmm1, %xmm1
+; CHECK-NEXT:    vucomiss %xmm1, %xmm1
+; CHECK-NEXT:    movl $0, %edi
+; CHECK-NEXT:    cmovnpl %ecx, %edi
+; CHECK-NEXT:    vcvtph2ps %xmm0, %xmm0
+; CHECK-NEXT:    vucomiss %xmm0, %xmm0
+; CHECK-NEXT:    cmovnpl %ecx, %eax
+; CHECK-NEXT:    vmovd %eax, %xmm0
+; CHECK-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $2, %esi, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $3, %edx, %xmm0, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vmovdqu %xmm1, 16
+; CHECK-NEXT:    vmovdqu %xmm0, 0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %2 = fcmp ord <16 x half> %0, zeroinitializer
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  %4 = shufflevector <16 x i32> %3, <16 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = tail call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %4, <4 x i32> zeroinitializer)
+  %6 = shufflevector <8 x i16> %5, <8 x i16> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %7 = bitcast <16 x i16> %6 to <32 x i8>
+  store <32 x i8> %7, ptr null, align 1
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>)
+
+attributes #0 = { "target-features"="+aes,+avx,+avx2,+avx512f,+avx512vnni,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fxsr,+mmx,+pclmul,+popcnt,+prfchw,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" }
diff --git a/llvm/test/CodeGen/X86/pr118934.ll b/llvm/test/CodeGen/X86/pr118934.ll
new file mode 100644
index 0000000000000..b68ed2643d8df
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr118934.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+
+define void @PR118934(i1 %b, ptr %f, ptr %k) {
+; X86-LABEL: PR118934:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    andb $1, %dl
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    addb $-2, %dl
+; X86-NEXT:    movsbl %dl, %edx
+; X86-NEXT:    addl $-6, %edx
+; X86-NEXT:    addl $6, %edx
+; X86-NEXT:    movl %edx, (%ecx)
+; X86-NEXT:    addl %edx, %edx
+; X86-NEXT:    movl %edx, (%eax)
+;
+; X64-LABEL: PR118934:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    addb %dil, %dil
+; X64-NEXT:    addb $-2, %dil
+; X64-NEXT:    movsbl %dil, %eax
+; X64-NEXT:    addl $-6, %eax
+; X64-NEXT:    addl $6, %eax
+; X64-NEXT:    movl %eax, (%rsi)
+; X64-NEXT:    addl %eax, %eax
+; X64-NEXT:    movl %eax, (%rdx)
+entry:
+  %0 = select i1 %b, i8 0, i8 -2
+  br label %for.end
+
+for.end:
+  %n.i.i = select i1 poison, i32 6, i32 7
+  %narrow = add nsw i8 %0, -6
+  %add2.i = sext i8 %narrow to i32
+  %conv5.i = add nsw i32 %n.i.i, %add2.i
+  store i32 %conv5.i, ptr %f, align 4
+  %add = shl nsw i32 %conv5.i, 1
+  store i32 %add, ptr %k, align 4
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/pr119158.ll b/llvm/test/CodeGen/X86/pr119158.ll
new file mode 100644
index 0000000000000..ca31df802c913
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr119158.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+define dso_local void @foo() #1 {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %newFuncRoot
+; CHECK-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64]
+; CHECK-NEXT:    vpdpwssd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; CHECK-NEXT:    vpsrld $7, %ymm1, %ymm0
+; CHECK-NEXT:    vpackusdw %ymm0, %ymm0, %ymm0
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; CHECK-NEXT:    vmovdqu %ymm0, (%rax)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+newFuncRoot:
+  br label %loop
+
+loop:                                              ; preds = %newFuncRoot, %loop
+  %0 = load <16 x i8>, ptr poison, align 1
+  %1 = zext <16 x i8> %0 to <16 x i32>
+  %2 = mul nuw nsw <16 x i32> %1, splat (i32 18)
+  %3 = add nuw nsw <16 x i32> zeroinitializer, splat (i32 64)
+  %4 = add nuw nsw <16 x i32> %3, zeroinitializer
+  %5 = add nuw nsw <16 x i32> %4, %2
+  %6 = sub nsw <16 x i32> %5, zeroinitializer
+  %7 = ashr <16 x i32> %6, splat (i32 7)
+  %8 = tail call <16 x i32> @llvm.smin.v16i32(<16 x i32> %7, <16 x i32> splat (i32 255))
+  %9 = tail call <16 x i32> @llvm.smax.v16i32(<16 x i32> %8, <16 x i32> zeroinitializer)
+  %10 = trunc <16 x i32> %9 to <16 x i8>
+  %11 = shufflevector <16 x i8> %10, <16 x i8> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  store <32 x i8> %11, ptr poison, align 1
+  br i1 poison, label %.exitStub, label %loop
+
+.exitStub:                                        ; preds = %loop
+  ret void
+}
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="tigerlake" "target-features"="+adx,+aes,+avx,+avx2,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vp2intersect,+avx512vpopcntdq,+bmi,+bmi2,+clflushopt,+clwb,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fsgsbase,+fxsr,+gfni,+invpcid,+kl,+lzcnt,+mmx,+movbe,+movdir64b,+movdiri,+pclmul,+pku,+popcnt,+prfchw,+rdpid,+rdrnd,+rdseed,+sahf,+sgx,+sha,+shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+widekl,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" }
diff --git a/llvm/test/CodeGen/X86/pr50609.ll b/llvm/test/CodeGen/X86/pr50609.ll
index 3e39c567d79e6..e9c0f10194425 100644
--- a/llvm/test/CodeGen/X86/pr50609.ll
+++ b/llvm/test/CodeGen/X86/pr50609.ll
@@ -11,9 +11,9 @@ define void @PR50609(ptr noalias nocapture %RET, ptr noalias %aFOO, <16 x i32> %
 ; CHECK-NEXT:    vpsrad $31, %xmm2, %xmm3
 ; CHECK-NEXT:    vpsrld $30, %xmm3, %xmm3
 ; CHECK-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; CHECK-NEXT:    vpsrad $2, %xmm2, %xmm2
 ; CHECK-NEXT:    vcvtdq2ps %xmm2, %xmm2
-; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
 ; CHECK-NEXT:    vmaskmovps %ymm2, %ymm0, (%rdi)
 ; CHECK-NEXT:    vmaskmovps %ymm2, %ymm1, 32(%rdi)
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index ef0f3f3e816df..62ee0b298ba91 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -4966,22 +4966,14 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptosi_2f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vpsrld $16, %xmm0, %xmm1
-; F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
-; F16C-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
-; F16C-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; F16C-NEXT:    vcvttps2dq %xmm0, %xmm0
 ; F16C-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: fptosi_2f16_to_4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512-NEXT:    vcvttps2dq %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512-NEXT:    retq
@@ -5084,11 +5076,9 @@ define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptoui_2f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vpsrld $16, %xmm0, %xmm1
-; F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
-; F16C-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
-; F16C-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; F16C-NEXT:    vcvttps2dq %xmm0, %xmm1
 ; F16C-NEXT:    vpsrad $31, %xmm1, %xmm2
 ; F16C-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
@@ -5100,11 +5090,9 @@ define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind {
 ;
 ; AVX512F-LABEL: fptoui_2f16_to_4i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512F-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
 ; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512F-NEXT:    vzeroupper
@@ -5112,11 +5100,7 @@ define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind {
 ;
 ; AVX512-FASTLANE-LABEL: fptoui_2f16_to_4i32:
 ; AVX512-FASTLANE:       # %bb.0:
-; AVX512-FASTLANE-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512-FASTLANE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-FASTLANE-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512-FASTLANE-NEXT:    vcvttps2udq %xmm0, %xmm0
 ; AVX512-FASTLANE-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512-FASTLANE-NEXT:    retq
@@ -5222,7 +5206,7 @@ define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; AVX512F-LABEL: fptoui_4f16_to_4i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
index 8dded96acfc12..bb609b9938a82 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
@@ -123,12 +123,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -140,12 +140,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -157,12 +157,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512DQ-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512DQ-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512DQ-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -174,12 +174,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
@@ -191,12 +191,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512BW-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -208,12 +208,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512BW-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512BW-FCP-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -225,12 +225,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512DQ-BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512DQ-BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512DQ-BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -242,12 +242,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,9]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [3,9]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5]
 ; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm1, 64(%r9)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm3, 64(%r9)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <2 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
index e70975addc676..c2f1723d8031e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
@@ -138,13 +138,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -157,13 +157,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -176,13 +176,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512DQ-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512DQ-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512DQ-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512DQ-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -195,13 +195,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
@@ -214,13 +214,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512BW-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512BW-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -233,13 +233,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512BW-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512BW-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512BW-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512BW-FCP-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -252,13 +252,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512DQ-BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512DQ-BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512DQ-BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512DQ-BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -271,13 +271,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm3, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <2 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index b87abded10819..941b18db0931a 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -5585,12 +5585,12 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-LABEL: store_i8_stride5_vf64:
 ; AVX512BW-FCP:       # %bb.0:
 ; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdx), %ymm21
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm8
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm21, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
+; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %ymm21
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm8, %ymm2
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm21, %ymm2
 ; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm2
@@ -5598,77 +5598,76 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm14, %xmm2, %xmm3
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm12
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm4
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm5
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm5
 ; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1]
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm13
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm3, %xmm9
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm16
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm9
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm5
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm5, %xmm10
 ; AVX512BW-FCP-NEXT:    vpor %xmm9, %xmm10, %xmm9
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm18 = ymm9[0,0,1,1]
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm19
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm9[0,0,1,1]
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm16
 ; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [3,3,3,0,4,4,4,4]
-; AVX512BW-FCP-NEXT:    vpermd %ymm19, %ymm9, %ymm22
+; AVX512BW-FCP-NEXT:    vpermd %ymm16, %ymm9, %ymm22
 ; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %ymm23
 ; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14]
 ; AVX512BW-FCP-NEXT:    movl $138547332, %eax # imm = 0x8421084
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512BW-FCP-NEXT:    vpshufb %ymm10, %ymm23, %ymm22 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm18, %zmm18
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm15, %zmm15
 ; AVX512BW-FCP-NEXT:    movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm18, %zmm1 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %ymm22
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [1,1,2,2,2,2,2,2,27,27,27,27,0,28,28,28]
-; AVX512BW-FCP-NEXT:    vpermi2d %zmm0, %zmm22, %zmm18
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm1 {%k2}
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [9,9,10,10,10,10,10,10,11,11,11,11,0,12,12,12]
+; AVX512BW-FCP-NEXT:    vpermd %zmm0, %zmm15, %zmm15
 ; AVX512BW-FCP-NEXT:    movabsq $4760450083537948804, %rax # imm = 0x4210842108421084
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm18, %zmm1 {%k3}
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm18, %zmm23, %zmm23
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm19[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm19, %zmm24, %zmm24
-; AVX512BW-FCP-NEXT:    vporq %zmm23, %zmm24, %zmm23
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm1 {%k3}
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30]
+; AVX512BW-FCP-NEXT:    vpshufb %zmm15, %zmm22, %zmm22
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512BW-FCP-NEXT:    vpshufb %zmm16, %zmm23, %zmm23
+; AVX512BW-FCP-NEXT:    vporq %zmm22, %zmm23, %zmm22
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128]
+; AVX512BW-FCP-NEXT:    vpshufb %zmm23, %zmm8, %zmm8
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm24, %zmm21, %zmm21
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm25, %zmm8, %zmm8
-; AVX512BW-FCP-NEXT:    vporq %zmm21, %zmm8, %zmm8
+; AVX512BW-FCP-NEXT:    vporq %zmm8, %zmm21, %zmm8
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7]
 ; AVX512BW-FCP-NEXT:    movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm23, %zmm8 {%k3}
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
-; AVX512BW-FCP-NEXT:    vpermi2d %zmm0, %zmm22, %zmm21
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm22, %zmm8 {%k3}
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [12,14,13,13,13,13,12,14,14,14,14,14,15,15,15,15]
+; AVX512BW-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm21
 ; AVX512BW-FCP-NEXT:    movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k4
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm21, %zmm8 {%k4}
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-FCP-NEXT:    vpor %xmm14, %xmm15, %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm12, %xmm17
+; AVX512BW-FCP-NEXT:    vporq %xmm14, %xmm17, %xmm14
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm11
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm11, %zmm11
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm14 = zmm11[0,0,1,1,4,4,5,5]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm13, %xmm11
-; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm16, %xmm15
-; AVX512BW-FCP-NEXT:    vpor %xmm11, %xmm15, %xmm11
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm16[0],xmm13[1],xmm16[1],xmm13[2],xmm16[2],xmm13[3],xmm16[3],xmm13[4],xmm16[4],xmm13[5],xmm16[5],xmm13[6],xmm16[6],xmm13[7],xmm16[7]
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm13, %xmm11
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm18, %xmm17
+; AVX512BW-FCP-NEXT:    vporq %xmm11, %xmm17, %xmm11
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm18[0],xmm13[1],xmm18[1],xmm13[2],xmm18[2],xmm13[3],xmm18[3],xmm13[4],xmm18[4],xmm13[5],xmm18[5],xmm13[6],xmm18[6],xmm13[7],xmm18[7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm13, %xmm13
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm13, %zmm11
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[0,0,1,1,4,4,5,5]
 ; AVX512BW-FCP-NEXT:    movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C
@@ -5682,23 +5681,23 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm11 {%k4}
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
 ; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm14, %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm16, %ymm7
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %ymm18
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm18, %ymm7
 ; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm7, %ymm6
-; AVX512BW-FCP-NEXT:    vpshufb %ymm24, %ymm14, %ymm7
-; AVX512BW-FCP-NEXT:    vpshufb %ymm25, %ymm16, %ymm17
-; AVX512BW-FCP-NEXT:    vporq %ymm7, %ymm17, %ymm7
+; AVX512BW-FCP-NEXT:    vpshufb %ymm23, %ymm14, %ymm7
+; AVX512BW-FCP-NEXT:    vpshufb %ymm24, %ymm18, %ymm19
+; AVX512BW-FCP-NEXT:    vporq %ymm7, %ymm19, %ymm7
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3]
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm7
-; AVX512BW-FCP-NEXT:    vpshufb %ymm18, %ymm7, %ymm17
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm18
-; AVX512BW-FCP-NEXT:    vpshufb %ymm19, %ymm18, %ymm19
-; AVX512BW-FCP-NEXT:    vporq %ymm17, %ymm19, %ymm17
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm17 = ymm17[2,2,3,3]
-; AVX512BW-FCP-NEXT:    vpermd %ymm18, %ymm9, %ymm9
+; AVX512BW-FCP-NEXT:    vpshufb %ymm15, %ymm7, %ymm15
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm19
+; AVX512BW-FCP-NEXT:    vpshufb %ymm16, %ymm19, %ymm16
+; AVX512BW-FCP-NEXT:    vporq %ymm15, %ymm16, %ymm15
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3]
+; AVX512BW-FCP-NEXT:    vpermd %ymm19, %ymm9, %ymm9
 ; AVX512BW-FCP-NEXT:    vpshufb %ymm10, %ymm7, %ymm9 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm9, %zmm9
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm9 {%k2}
 ; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [3,3,3,3,0,4,4,4,12,14,13,13,13,13,12,14]
 ; AVX512BW-FCP-NEXT:    vpermd %zmm13, %zmm6, %zmm6
@@ -5707,15 +5706,15 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm9 {%k1}
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm16[27],zero,zero,ymm16[26],zero,ymm16[28],zero,ymm16[30],zero,zero,ymm16[29],zero,ymm16[31],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm18[27],zero,zero,ymm18[26],zero,ymm18[28],zero,ymm18[30],zero,zero,ymm18[29],zero,ymm18[31],zero
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9]
 ; AVX512BW-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm4
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm2, %xmm2
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm7[26],zero,ymm7[28],zero,zero,zero,zero,ymm7[29],zero,ymm7[31],zero,zero,ymm7[30]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,ymm18[27],zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm19[26],zero,ymm19[28],zero,zero,ymm19[27],zero,ymm19[29],zero,ymm19[31],zero,zero,ymm19[30],zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; AVX512BW-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm3
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k3}
@@ -5909,12 +5908,12 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride5_vf64:
 ; AVX512DQ-BW-FCP:       # %bb.0:
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdx), %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm8
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm21, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm8, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %ymm21
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm8, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm21, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm2
@@ -5922,77 +5921,76 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm14, %xmm2, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm12
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm5
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1]
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm13
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm3, %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm5
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm5, %xmm10
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm9, %xmm10, %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm18 = ymm9[0,0,1,1]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm9[0,0,1,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm16
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [3,3,3,0,4,4,4,4]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm19, %ymm9, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm16, %ymm9, %ymm22
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %ymm23
 ; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14]
 ; AVX512DQ-BW-FCP-NEXT:    movl $138547332, %eax # imm = 0x8421084
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm10, %ymm23, %ymm22 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm18, %zmm18
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm15, %zmm15
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm18, %zmm1 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %ymm22
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [1,1,2,2,2,2,2,2,27,27,27,27,0,28,28,28]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2d %zmm0, %zmm22, %zmm18
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm1 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [9,9,10,10,10,10,10,10,11,11,11,11,0,12,12,12]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %zmm0, %zmm15, %zmm15
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $4760450083537948804, %rax # imm = 0x4210842108421084
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm18, %zmm1 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm18, %zmm23, %zmm23
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm19[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm19, %zmm24, %zmm24
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm23, %zmm24, %zmm23
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm1 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm15, %zmm22, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm16, %zmm23, %zmm23
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm22, %zmm23, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm23, %zmm8, %zmm8
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm24, %zmm21, %zmm21
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm25, %zmm8, %zmm8
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm21, %zmm8, %zmm8
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm8, %zmm21, %zmm8
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm23, %zmm8 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2d %zmm0, %zmm22, %zmm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm22, %zmm8 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [12,14,13,13,13,13,12,14,14,14,14,14,15,15,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm21
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k4
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm21, %zmm8 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm14, %xmm15, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm12, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm14, %xmm17, %xmm14
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm11
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm11, %zmm11
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm14 = zmm11[0,0,1,1,4,4,5,5]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm13, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm16, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm11, %xmm15, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm16[0],xmm13[1],xmm16[1],xmm13[2],xmm16[2],xmm13[3],xmm16[3],xmm13[4],xmm16[4],xmm13[5],xmm16[5],xmm13[6],xmm16[6],xmm13[7],xmm16[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm13, %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm18, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm11, %xmm17, %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm18[0],xmm13[1],xmm18[1],xmm13[2],xmm18[2],xmm13[3],xmm18[3],xmm13[4],xmm18[4],xmm13[5],xmm18[5],xmm13[6],xmm18[6],xmm13[7],xmm18[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm13, %xmm13
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm13, %zmm11
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[0,0,1,1,4,4,5,5]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C
@@ -6006,23 +6004,23 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm11 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm14, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm16, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %ymm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm18, %ymm7
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm7, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm24, %ymm14, %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm25, %ymm16, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm7, %ymm17, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm23, %ymm14, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm24, %ymm18, %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm7, %ymm19, %ymm7
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3]
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm18, %ymm7, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm19, %ymm18, %ymm19
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm17, %ymm19, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm17 = ymm17[2,2,3,3]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm18, %ymm9, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm15, %ymm7, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm16, %ymm19, %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm15, %ymm16, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm19, %ymm9, %ymm9
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm10, %ymm7, %ymm9 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm9, %zmm9
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm9 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [3,3,3,3,0,4,4,4,12,14,13,13,13,13,12,14]
 ; AVX512DQ-BW-FCP-NEXT:    vpermd %zmm13, %zmm6, %zmm6
@@ -6031,15 +6029,15 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm9 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm16[27],zero,zero,ymm16[26],zero,ymm16[28],zero,ymm16[30],zero,zero,ymm16[29],zero,ymm16[31],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm18[27],zero,zero,ymm18[26],zero,ymm18[28],zero,ymm18[30],zero,zero,ymm18[29],zero,ymm18[31],zero
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9]
 ; AVX512DQ-BW-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm2, %xmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm7[26],zero,ymm7[28],zero,zero,zero,zero,ymm7[29],zero,ymm7[31],zero,zero,ymm7[30]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,ymm18[27],zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm19[26],zero,ymm19[28],zero,zero,ymm19[27],zero,ymm19[29],zero,ymm19[31],zero,zero,ymm19[30],zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; AVX512DQ-BW-FCP-NEXT:    vpermt2q %zmm2, %zmm6, %zmm3
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k3}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index 4d9cc3c8a7dcb..37919128ba9f7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -1599,15 +1599,15 @@ define void @store_i8_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
-; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,8,10,9,11]
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-FCP-NEXT:    vpermt2q %zmm3, %zmm4, %zmm5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u]
-; AVX512BW-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [4,6,4,6,0,2,1,3]
+; AVX512BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm4
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u]
+; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,2,0,2,4,6,5,7]
+; AVX512BW-FCP-NEXT:    vpermq %zmm3, %zmm5, %zmm3
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58]
 ; AVX512BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm3 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm4, %zmm3 {%k1}
 ; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,0,2,1,3]
 ; AVX512BW-FCP-NEXT:    vpermq %zmm2, %zmm4, %zmm4
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u]
@@ -1685,15 +1685,15 @@ define void @store_i8_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, (%r9), %zmm2, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,8,10,9,11]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vpermt2q %zmm3, %zmm4, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [4,6,4,6,0,2,1,3]
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,2,0,2,4,6,5,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm3, %zmm5, %zmm3
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58]
 ; AVX512DQ-BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm3 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm4, %zmm3 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,0,2,1,3]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm2, %zmm4, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
index b214bf082f235..eafee9e65345f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
@@ -413,9 +413,12 @@ define half @test_v2f16(<2 x half> %a0) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512F-NEXT:    vcvtph2ps %xmm0, %ymm2
-; AVX512F-NEXT:    vcvtph2ps %xmm1, %ymm3
-; AVX512F-NEXT:    vcmpltps %zmm2, %zmm3, %k1
+; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm2
+; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm3
+; AVX512F-NEXT:    vucomiss %xmm3, %xmm2
+; AVX512F-NEXT:    seta %al
+; AVX512F-NEXT:    negb %al
+; AVX512F-NEXT:    kmovd %eax, %k1
 ; AVX512F-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512F-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
index 9f37df716b6cd..e8f9c7f7b524d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
@@ -412,9 +412,12 @@ define half @test_v2f16(<2 x half> %a0) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512F-NEXT:    vcvtph2ps %xmm0, %ymm2
-; AVX512F-NEXT:    vcvtph2ps %xmm1, %ymm3
-; AVX512F-NEXT:    vcmpltps %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm2
+; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm3
+; AVX512F-NEXT:    xorl %eax, %eax
+; AVX512F-NEXT:    vucomiss %xmm3, %xmm2
+; AVX512F-NEXT:    sbbl %eax, %eax
+; AVX512F-NEXT:    kmovd %eax, %k1
 ; AVX512F-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512F-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 4d4739232e7a6..b5571edfa522c 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -689,7 +689,7 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE-NEXT:    psubq %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = ashr <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -738,7 +738,7 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    psrad %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -787,7 +787,7 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -924,7 +924,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
@@ -1008,7 +1008,7 @@ define <2 x i64> @splatvar_modulo_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwi
 ; X86-SSE-NEXT:    psubq %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <2 x i64> %b, <i64 63, i64 63>
-  %splat = shufflevector <2 x i64> %mod, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %mod, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = ashr <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -1050,7 +1050,7 @@ define <4 x i32> @splatvar_modulo_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwi
 ; X86-SSE-NEXT:    psrad %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <4 x i32> %mod, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %mod, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -1092,7 +1092,7 @@ define <8 x i16> @splatvar_modulo_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwi
 ; X86-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <8 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <8 x i16> %mod, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %mod, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -1234,7 +1234,7 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi
 ; X86-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <16 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <16 x i8> %mod, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %mod, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 1ebefe6d0d0d3..cc64b4375e45b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -732,7 +732,7 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X86-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -794,7 +794,7 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; X86-AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -856,7 +856,7 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; X86-AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -998,7 +998,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; X86-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = ashr <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
@@ -1096,7 +1096,7 @@ define <4 x i64> @splatvar_modulo_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwi
 ; X86-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <4 x i64> %b, <i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <4 x i64> %mod, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %mod, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -1159,7 +1159,7 @@ define <8 x i32> @splatvar_modulo_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwi
 ; X86-AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <8 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <8 x i32> %mod, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %mod, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -1222,7 +1222,7 @@ define <16 x i16> @splatvar_modulo_shift_v16i16(<16 x i16> %a, <16 x i16> %b) no
 ; X86-AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <16 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <16 x i16> %mod, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %mod, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -1369,7 +1369,7 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi
 ; X86-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <32 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <32 x i8> %mod, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %mod, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = ashr <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index b70407c0b96a4..738092a7825e0 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -150,7 +150,7 @@ define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpsraq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %b, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -161,7 +161,7 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 ; ALL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; ALL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %b, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -181,7 +181,7 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsraw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %b, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = ashr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -221,7 +221,7 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512BW-NEXT:    vpternlogq $108, %zmm0, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %b, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = ashr <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
@@ -237,7 +237,7 @@ define <8 x i64> @splatvar_modulo_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwi
 ; ALL-NEXT:    vpsraq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <8 x i64> %b, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <8 x i64> %mod, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %mod, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -249,7 +249,7 @@ define <16 x i32> @splatvar_modulo_shift_v16i32(<16 x i32> %a, <16 x i32> %b) no
 ; ALL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <16 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <16 x i32> %mod, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %mod, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = ashr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -270,7 +270,7 @@ define <32 x i16> @splatvar_modulo_shift_v32i16(<32 x i16> %a, <32 x i16> %b) no
 ; AVX512BW-NEXT:    vpsraw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <32 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <32 x i16> %mod, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %mod, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = ashr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -311,7 +311,7 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi
 ; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <64 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <64 x i8> %mod, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %mod, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = ashr <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 2ec9de0cb447f..95d8c604aace1 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -1197,7 +1197,7 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    psrad %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
   %shift = ashr <2 x i32> %a, %splat
   ret <2 x i32> %shift
 }
@@ -1246,7 +1246,7 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i16> %a, %splat
   ret <4 x i16> %shift
 }
@@ -1295,7 +1295,7 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
   %shift = ashr <2 x i16> %a, %splat
   ret <2 x i16> %shift
 }
@@ -1433,7 +1433,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
   %shift = ashr <8 x i8> %a, %splat
   ret <8 x i8> %shift
 }
@@ -1571,7 +1571,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
   %shift = ashr <4 x i8> %a, %splat
   ret <4 x i8> %shift
 }
@@ -1700,7 +1700,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
   %shift = ashr <2 x i8> %a, %splat
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index 177d8f78b5a8d..b453f925b94e8 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -450,22 +450,22 @@ for.body.preheader40:
 
 vector.ph:
   %n.vec = and i64 %wide.trip.count, 4294967264
-  %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %amt0, i32 0
-  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert22 = insertelement <8 x i32> undef, i32 %amt1, i32 0
-  %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert24 = insertelement <8 x i32> undef, i32 %amt0, i32 0
-  %broadcast.splat25 = shufflevector <8 x i32> %broadcast.splatinsert24, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert26 = insertelement <8 x i32> undef, i32 %amt1, i32 0
-  %broadcast.splat27 = shufflevector <8 x i32> %broadcast.splatinsert26, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert28 = insertelement <8 x i32> undef, i32 %amt0, i32 0
-  %broadcast.splat29 = shufflevector <8 x i32> %broadcast.splatinsert28, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert30 = insertelement <8 x i32> undef, i32 %amt1, i32 0
-  %broadcast.splat31 = shufflevector <8 x i32> %broadcast.splatinsert30, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert32 = insertelement <8 x i32> undef, i32 %amt0, i32 0
-  %broadcast.splat33 = shufflevector <8 x i32> %broadcast.splatinsert32, <8 x i32> undef, <8 x i32> zeroinitializer
-  %broadcast.splatinsert34 = insertelement <8 x i32> undef, i32 %amt1, i32 0
-  %broadcast.splat35 = shufflevector <8 x i32> %broadcast.splatinsert34, <8 x i32> undef, <8 x i32> zeroinitializer
+  %broadcast.splatinsert20 = insertelement <8 x i32> poison, i32 %amt0, i32 0
+  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert22 = insertelement <8 x i32> poison, i32 %amt1, i32 0
+  %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert24 = insertelement <8 x i32> poison, i32 %amt0, i32 0
+  %broadcast.splat25 = shufflevector <8 x i32> %broadcast.splatinsert24, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert26 = insertelement <8 x i32> poison, i32 %amt1, i32 0
+  %broadcast.splat27 = shufflevector <8 x i32> %broadcast.splatinsert26, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert28 = insertelement <8 x i32> poison, i32 %amt0, i32 0
+  %broadcast.splat29 = shufflevector <8 x i32> %broadcast.splatinsert28, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert30 = insertelement <8 x i32> poison, i32 %amt1, i32 0
+  %broadcast.splat31 = shufflevector <8 x i32> %broadcast.splatinsert30, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert32 = insertelement <8 x i32> poison, i32 %amt0, i32 0
+  %broadcast.splat33 = shufflevector <8 x i32> %broadcast.splatinsert32, <8 x i32> poison, <8 x i32> zeroinitializer
+  %broadcast.splatinsert34 = insertelement <8 x i32> poison, i32 %amt1, i32 0
+  %broadcast.splat35 = shufflevector <8 x i32> %broadcast.splatinsert34, <8 x i32> poison, <8 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:
@@ -655,12 +655,12 @@ entry:
 
 vector.ph:
   %n.vec = and i64 %wide.trip.count, 4294967292
-  %splatinsert18 = insertelement <4 x i32> undef, i32 %amt0, i32 0
-  %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer
-  %splatinsert20 = insertelement <4 x i32> undef, i32 %amt1, i32 0
-  %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
-  %splatinsert22 = insertelement <4 x i32> undef, i32 %x, i32 0
-  %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splatinsert18 = insertelement <4 x i32> poison, i32 %amt0, i32 0
+  %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> poison, <4 x i32> zeroinitializer
+  %splatinsert20 = insertelement <4 x i32> poison, i32 %amt1, i32 0
+  %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> poison, <4 x i32> zeroinitializer
+  %splatinsert22 = insertelement <4 x i32> poison, i32 %x, i32 0
+  %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 4caa7da4ce136..467c1574180da 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -544,7 +544,7 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlq %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = lshr <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -593,7 +593,7 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    psrld %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -642,7 +642,7 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -759,7 +759,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
@@ -805,7 +805,7 @@ define <2 x i64> @splatvar_modulo_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwi
 ; X86-SSE-NEXT:    psrlq %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <2 x i64> %b, <i64 63, i64 63>
-  %splat = shufflevector <2 x i64> %mod, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %mod, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = lshr <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -847,7 +847,7 @@ define <4 x i32> @splatvar_modulo_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwi
 ; X86-SSE-NEXT:    psrld %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <4 x i32> %mod, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %mod, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -889,7 +889,7 @@ define <8 x i16> @splatvar_modulo_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwi
 ; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <8 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <8 x i16> %mod, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %mod, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -1011,7 +1011,7 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <16 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <16 x i8> %mod, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %mod, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
@@ -1760,8 +1760,8 @@ define <4 x i32> @vector_variable_shift_right(<4 x i1> %cond, <4 x i32> %x, <4 x
 ; X86-SSE-NEXT:    movl %ebp, %esp
 ; X86-SSE-NEXT:    popl %ebp
 ; X86-SSE-NEXT:    retl
-  %splat1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
-  %splat2 = shufflevector <4 x i32> %y, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat1 = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> zeroinitializer
+  %splat2 = shufflevector <4 x i32> %y, <4 x i32> poison, <4 x i32> zeroinitializer
   %sel = select <4 x i1> %cond, <4 x i32> %splat1, <4 x i32> %splat2
   %sh = lshr <4 x i32> %z, %sel
   ret <4 x i32> %sh
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index cc3b1a72e5b53..ca303b4c7ebf6 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -572,7 +572,7 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -634,7 +634,7 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; X86-AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -696,7 +696,7 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -811,7 +811,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; X86-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = lshr <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
@@ -878,7 +878,7 @@ define <4 x i64> @splatvar_modulo_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwi
 ; X86-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <4 x i64> %b, <i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <4 x i64> %mod, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %mod, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -941,7 +941,7 @@ define <8 x i32> @splatvar_modulo_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwi
 ; X86-AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <8 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <8 x i32> %mod, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %mod, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -1004,7 +1004,7 @@ define <16 x i16> @splatvar_modulo_shift_v16i16(<16 x i16> %a, <16 x i16> %b) no
 ; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <16 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <16 x i16> %mod, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %mod, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -1124,7 +1124,7 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi
 ; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <32 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <32 x i8> %mod, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %mod, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = lshr <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
index 81dc63ba33cb0..103d5702fb93a 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
@@ -114,7 +114,7 @@ define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %b, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -125,7 +125,7 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 ; ALL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; ALL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %b, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -145,7 +145,7 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %b, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = lshr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -176,7 +176,7 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %b, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = lshr <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
@@ -192,7 +192,7 @@ define <8 x i64> @splatvar_modulo_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwi
 ; ALL-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <8 x i64> %b, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <8 x i64> %mod, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %mod, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -204,7 +204,7 @@ define <16 x i32> @splatvar_modulo_shift_v16i32(<16 x i32> %a, <16 x i32> %b) no
 ; ALL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <16 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <16 x i32> %mod, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %mod, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = lshr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -225,7 +225,7 @@ define <32 x i16> @splatvar_modulo_shift_v32i16(<32 x i16> %a, <32 x i16> %b) no
 ; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <32 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <32 x i16> %mod, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %mod, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = lshr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -257,7 +257,7 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi
 ; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <64 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <64 x i8> %mod, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %mod, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = lshr <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index fe349e9ff995d..e0d1c256ebecf 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -963,7 +963,7 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    psrld %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
   %shift = lshr <2 x i32> %a, %splat
   ret <2 x i32> %shift
 }
@@ -1012,7 +1012,7 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i16> %a, %splat
   ret <4 x i16> %shift
 }
@@ -1061,7 +1061,7 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
   %shift = lshr <2 x i16> %a, %splat
   ret <2 x i16> %shift
 }
@@ -1179,7 +1179,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
   %shift = lshr <8 x i8> %a, %splat
   ret <8 x i8> %shift
 }
@@ -1297,7 +1297,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
   %shift = lshr <4 x i8> %a, %splat
   ret <4 x i8> %shift
 }
@@ -1406,7 +1406,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
   %shift = lshr <2 x i8> %a, %splat
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index 902bf8a0e55ce..4dda9ff09cc62 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -455,7 +455,7 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllq %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = shl <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -504,7 +504,7 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    pslld %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -553,7 +553,7 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -666,7 +666,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
@@ -712,7 +712,7 @@ define <2 x i64> @splatvar_modulo_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwi
 ; X86-SSE-NEXT:    psllq %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <2 x i64> %b, <i64 63, i64 63>
-  %splat = shufflevector <2 x i64> %mod, <2 x i64> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i64> %mod, <2 x i64> poison, <2 x i32> zeroinitializer
   %shift = shl <2 x i64> %a, %splat
   ret <2 x i64> %shift
 }
@@ -754,7 +754,7 @@ define <4 x i32> @splatvar_modulo_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwi
 ; X86-SSE-NEXT:    pslld %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <4 x i32> %mod, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i32> %mod, <4 x i32> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
@@ -796,7 +796,7 @@ define <8 x i16> @splatvar_modulo_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwi
 ; X86-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <8 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <8 x i16> %mod, <8 x i16> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i16> %mod, <8 x i16> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
@@ -914,7 +914,7 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %mod = and <16 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <16 x i8> %mod, <16 x i8> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i8> %mod, <16 x i8> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i8> %a, %splat
   ret <16 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index b4880a6cad70e..3c6aa642c778c 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -502,7 +502,7 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -564,7 +564,7 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; X86-AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -626,7 +626,7 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; X86-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -736,7 +736,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; X86-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
-  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = shl <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
@@ -803,7 +803,7 @@ define <4 x i64> @splatvar_modulo_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwi
 ; X86-AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <4 x i64> %b, <i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <4 x i64> %mod, <4 x i64> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i64> %mod, <4 x i64> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i64> %a, %splat
   ret <4 x i64> %shift
 }
@@ -866,7 +866,7 @@ define <8 x i32> @splatvar_modulo_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwi
 ; X86-AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <8 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <8 x i32> %mod, <8 x i32> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i32> %mod, <8 x i32> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
@@ -929,7 +929,7 @@ define <16 x i16> @splatvar_modulo_shift_v16i16(<16 x i16> %a, <16 x i16> %b) no
 ; X86-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <16 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <16 x i16> %mod, <16 x i16> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i16> %mod, <16 x i16> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
@@ -1044,7 +1044,7 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi
 ; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %mod = and <32 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <32 x i8> %mod, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i8> %mod, <32 x i8> poison, <32 x i32> zeroinitializer
   %shift = shl <32 x i8> %a, %splat
   ret <32 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
index cd729ebbc4330..a7fb05fbc5ba9 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
@@ -109,7 +109,7 @@ define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpsllq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %b, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -120,7 +120,7 @@ define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind
 ; ALL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; ALL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %b, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -140,7 +140,7 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %b, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = shl <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -169,7 +169,7 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
-  %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %b, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = shl <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
@@ -185,7 +185,7 @@ define <8 x i64> @splatvar_modulo_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwi
 ; ALL-NEXT:    vpsllq %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <8 x i64> %b, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
-  %splat = shufflevector <8 x i64> %mod, <8 x i64> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i64> %mod, <8 x i64> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i64> %a, %splat
   ret <8 x i64> %shift
 }
@@ -197,7 +197,7 @@ define <16 x i32> @splatvar_modulo_shift_v16i32(<16 x i32> %a, <16 x i32> %b) no
 ; ALL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %mod = and <16 x i32> %b, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
-  %splat = shufflevector <16 x i32> %mod, <16 x i32> undef, <16 x i32> zeroinitializer
+  %splat = shufflevector <16 x i32> %mod, <16 x i32> poison, <16 x i32> zeroinitializer
   %shift = shl <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
@@ -218,7 +218,7 @@ define <32 x i16> @splatvar_modulo_shift_v32i16(<32 x i16> %a, <32 x i16> %b) no
 ; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <32 x i16> %b, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
-  %splat = shufflevector <32 x i16> %mod, <32 x i16> undef, <32 x i32> zeroinitializer
+  %splat = shufflevector <32 x i16> %mod, <32 x i16> poison, <32 x i32> zeroinitializer
   %shift = shl <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
@@ -248,7 +248,7 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi
 ; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %mod = and <64 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
-  %splat = shufflevector <64 x i8> %mod, <64 x i8> undef, <64 x i32> zeroinitializer
+  %splat = shufflevector <64 x i8> %mod, <64 x i8> poison, <64 x i32> zeroinitializer
   %shift = shl <64 x i8> %a, %splat
   ret <64 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index a44120b6d038c..f213db0349c31 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -816,7 +816,7 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X86-SSE-NEXT:    pslld %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
   %shift = shl <2 x i32> %a, %splat
   ret <2 x i32> %shift
 }
@@ -865,7 +865,7 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i16> %a, %splat
   ret <4 x i16> %shift
 }
@@ -914,7 +914,7 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
   %shift = shl <2 x i16> %a, %splat
   ret <2 x i16> %shift
 }
@@ -1027,7 +1027,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
+  %splat = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
   %shift = shl <8 x i8> %a, %splat
   ret <8 x i8> %shift
 }
@@ -1140,7 +1140,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
+  %splat = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
   %shift = shl <4 x i8> %a, %splat
   ret <4 x i8> %shift
 }
@@ -1246,7 +1246,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
-  %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
+  %splat = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
   %shift = shl <2 x i8> %a, %splat
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index 24c05ba9c52f6..9fd8c11ba6c4d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -875,7 +875,7 @@ define <16 x i8> @shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2],xmm1[4],xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 2, i32 20, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 2, i32 20, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1228,7 +1228,7 @@ define <16 x i8> @trunc_v4i32_shuffle(<16 x i8> %a) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1246,33 +1246,33 @@ entry:
   %s.5.4 = shufflevector <16 x i8> %s.1.5, <16 x i8> %s.1.8, <16 x i32> <i32 3, i32 13, i32 19, i32 7, i32 23, i32 11, i32 1, i32 9, i32 16, i32 25, i32 2, i32 7, i32 0, i32 21, i32 23, i32 17>
   %s.6.1 = shufflevector <16 x i8> %s.3.2, <16 x i8> %s.3.2, <16 x i32> <i32 11, i32 2, i32 28, i32 31, i32 27, i32 3, i32 9, i32 27, i32 25, i32 25, i32 14, i32 7, i32 12, i32 28, i32 12, i32 23>
   %s.7.1 = shufflevector <16 x i8> %s.6.1, <16 x i8> %s.3.2, <16 x i32> <i32 15, i32 29, i32 14, i32 0, i32 29, i32 15, i32 26, i32 30, i32 6, i32 7, i32 2, i32 8, i32 12, i32 10, i32 29, i32 17>
-  %s.7.2 = shufflevector <16 x i8> %s.3.2, <16 x i8> %s.5.4, <16 x i32> <i32 3, i32 29, i32 3, i32 19, i32 undef, i32 20, i32 undef, i32 3, i32 27, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
-  %s.16.0 = shufflevector <16 x i8> %s.7.1, <16 x i8> %s.7.2, <16 x i32> <i32 13, i32 1, i32 16, i32 16, i32 6, i32 7, i32 29, i32 18, i32 19, i32 28, i32 undef, i32 undef, i32 31, i32 1, i32 undef, i32 10>
+  %s.7.2 = shufflevector <16 x i8> %s.3.2, <16 x i8> %s.5.4, <16 x i32> <i32 3, i32 29, i32 3, i32 19, i32 poison, i32 20, i32 poison, i32 3, i32 27, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s.16.0 = shufflevector <16 x i8> %s.7.1, <16 x i8> %s.7.2, <16 x i32> <i32 13, i32 1, i32 16, i32 16, i32 6, i32 7, i32 29, i32 18, i32 19, i32 28, i32 poison, i32 poison, i32 31, i32 1, i32 poison, i32 10>
   ret <16 x i8> %s.16.0
 }
 
-define <16 x i8> @undef_test1(<16 x i8> %s.0.5, <16 x i8> %s.0.8, <16 x i8> %s.0.9) noinline nounwind {
+define <16 x i8> @poison_test1(<16 x i8> %s.0.5, <16 x i8> %s.0.8, <16 x i8> %s.0.9) noinline nounwind {
 ; There is nothing interesting to check about these instructions other than
 ; that they survive codegen. However, we actually do better and delete all of
-; them because the result is 'undef'.
+; them because the result is 'poison'.
 ;
-; ALL-LABEL: undef_test1:
+; ALL-LABEL: poison_test1:
 ; ALL:       # %bb.0: # %entry
 ; ALL-NEXT:    retq
 entry:
-  %s.1.8 = shufflevector <16 x i8> %s.0.8, <16 x i8> undef, <16 x i32> <i32 9, i32 9, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 6, i32 undef, i32 6, i32 undef, i32 14, i32 14, i32 undef, i32 undef, i32 0>
-  %s.2.4 = shufflevector <16 x i8> undef, <16 x i8> %s.0.5, <16 x i32> <i32 21, i32 undef, i32 undef, i32 19, i32 undef, i32 undef, i32 29, i32 24, i32 21, i32 23, i32 21, i32 17, i32 19, i32 undef, i32 20, i32 22>
-  %s.2.5 = shufflevector <16 x i8> %s.0.5, <16 x i8> undef, <16 x i32> <i32 3, i32 8, i32 undef, i32 7, i32 undef, i32 10, i32 8, i32 0, i32 15, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 9>
-  %s.2.9 = shufflevector <16 x i8> %s.0.9, <16 x i8> undef, <16 x i32> <i32 7, i32 undef, i32 14, i32 7, i32 8, i32 undef, i32 7, i32 8, i32 5, i32 15, i32 undef, i32 1, i32 11, i32 undef, i32 undef, i32 11>
+  %s.1.8 = shufflevector <16 x i8> %s.0.8, <16 x i8> poison, <16 x i32> <i32 9, i32 9, i32 poison, i32 poison, i32 poison, i32 2, i32 poison, i32 6, i32 poison, i32 6, i32 poison, i32 14, i32 14, i32 poison, i32 poison, i32 0>
+  %s.2.4 = shufflevector <16 x i8> poison, <16 x i8> %s.0.5, <16 x i32> <i32 21, i32 poison, i32 poison, i32 19, i32 poison, i32 poison, i32 29, i32 24, i32 21, i32 23, i32 21, i32 17, i32 19, i32 poison, i32 20, i32 22>
+  %s.2.5 = shufflevector <16 x i8> %s.0.5, <16 x i8> poison, <16 x i32> <i32 3, i32 8, i32 poison, i32 7, i32 poison, i32 10, i32 8, i32 0, i32 15, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 9>
+  %s.2.9 = shufflevector <16 x i8> %s.0.9, <16 x i8> poison, <16 x i32> <i32 7, i32 poison, i32 14, i32 7, i32 8, i32 poison, i32 7, i32 8, i32 5, i32 15, i32 poison, i32 1, i32 11, i32 poison, i32 poison, i32 11>
   %s.3.4 = shufflevector <16 x i8> %s.2.4, <16 x i8> %s.0.5, <16 x i32> <i32 5, i32 0, i32 21, i32 6, i32 15, i32 27, i32 22, i32 21, i32 4, i32 22, i32 19, i32 26, i32 9, i32 26, i32 8, i32 29>
-  %s.3.9 = shufflevector <16 x i8> %s.2.9, <16 x i8> undef, <16 x i32> <i32 8, i32 6, i32 8, i32 1, i32 undef, i32 4, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 6, i32 undef>
+  %s.3.9 = shufflevector <16 x i8> %s.2.9, <16 x i8> poison, <16 x i32> <i32 8, i32 6, i32 8, i32 1, i32 poison, i32 4, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 6, i32 poison>
   %s.4.7 = shufflevector <16 x i8> %s.1.8, <16 x i8> %s.2.9, <16 x i32> <i32 9, i32 0, i32 22, i32 20, i32 24, i32 7, i32 21, i32 17, i32 20, i32 12, i32 19, i32 23, i32 2, i32 9, i32 17, i32 10>
-  %s.4.8 = shufflevector <16 x i8> %s.2.9, <16 x i8> %s.3.9, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 6, i32 10, i32 undef, i32 0, i32 5, i32 undef, i32 9, i32 undef>
-  %s.5.7 = shufflevector <16 x i8> %s.4.7, <16 x i8> %s.4.8, <16 x i32> <i32 16, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %s.8.4 = shufflevector <16 x i8> %s.3.4, <16 x i8> %s.5.7, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 28, i32 undef, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %s.9.4 = shufflevector <16 x i8> %s.8.4, <16 x i8> undef, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 10, i32 5>
-  %s.10.4 = shufflevector <16 x i8> %s.9.4, <16 x i8> undef, <16 x i32> <i32 undef, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %s.12.4 = shufflevector <16 x i8> %s.10.4, <16 x i8> undef, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 13, i32 undef, i32 undef, i32 undef>
+  %s.4.8 = shufflevector <16 x i8> %s.2.9, <16 x i8> %s.3.9, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 6, i32 10, i32 poison, i32 0, i32 5, i32 poison, i32 9, i32 poison>
+  %s.5.7 = shufflevector <16 x i8> %s.4.7, <16 x i8> %s.4.8, <16 x i32> <i32 16, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s.8.4 = shufflevector <16 x i8> %s.3.4, <16 x i8> %s.5.7, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 28, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s.9.4 = shufflevector <16 x i8> %s.8.4, <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 5>
+  %s.10.4 = shufflevector <16 x i8> %s.9.4, <16 x i8> poison, <16 x i32> <i32 poison, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %s.12.4 = shufflevector <16 x i8> %s.10.4, <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 13, i32 poison, i32 poison, i32 poison>
 
   ret <16 x i8> %s.12.4
 }
@@ -1303,7 +1303,7 @@ define <16 x i8> @shuffle_v16i8_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; AVX-NEXT:    movzbl %dil, %eax
 ; AVX-NEXT:    vmovd %eax, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <16 x i8> undef, i8 %i, i32 0
+  %a = insertelement <16 x i8> poison, i8 %i, i32 0
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %shuffle
 }
@@ -1334,7 +1334,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <16 x i8> undef, i8 %i, i32 0
+  %a = insertelement <16 x i8> poison, i8 %i, i32 0
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i8> %shuffle
 }
@@ -1365,8 +1365,8 @@ define <16 x i8> @shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16(
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrb $15, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <16 x i8> undef, i8 %i, i32 0
-  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 16>
+  %a = insertelement <16 x i8> poison, i8 %i, i32 0
+  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -1396,7 +1396,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrb $2, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <16 x i8> undef, i8 %i, i32 3
+  %a = insertelement <16 x i8> poison, i8 %i, i32 3
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 1, i32 19, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %shuffle
 }
@@ -1411,7 +1411,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_16_uu_18_uu(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 undef, i32 18, i32 undef>
+  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 poison, i32 18, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1425,7 +1425,7 @@ define <16 x i8> @shuffle_v16i8_28_uu_30_31_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 28, i32 undef, i32 30, i32 31, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 09, i32 0, i32 0, i32 0, i32 0, i32 0>
+  %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 28, i32 poison, i32 30, i32 31, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 09, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i8> %shuffle
 }
 
@@ -1630,7 +1630,7 @@ define <16 x i8> @shuffle_v16i8_00_uu_uu_uu_uu_uu_uu_uu_01_uu_uu_uu_uu_uu_uu_uu(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1683,7 +1683,7 @@ define <16 x i8> @shuffle_v16i8_00_uu_uu_uu_01_uu_uu_uu_02_uu_uu_uu_03_uu_uu_uu(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1735,7 +1735,7 @@ define <16 x i8> @shuffle_v16i8_00_uu_01_uu_02_uu_03_uu_04_uu_05_uu_06_uu_07_uu(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3, i32 undef, i32 4, i32 undef, i32 5, i32 undef, i32 6, i32 undef, i32 7, i32 undef>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7, i32 poison>
   ret <16 x i8> %shuffle
 }
 
@@ -1843,7 +1843,7 @@ define <16 x i8> @shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00(
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[u,10,2,7],xmm1[6],xmm0[14,7,2],xmm1[2],xmm0[3,1,14],xmm1[2],xmm0[9,11,0]
 ; XOP-NEXT:    retq
 entry:
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 undef, i32 10, i32 2, i32 7, i32 22, i32 14, i32 7, i32 2, i32 18, i32 3, i32 1, i32 14, i32 18, i32 9, i32 11, i32 0>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 poison, i32 10, i32 2, i32 7, i32 22, i32 14, i32 7, i32 2, i32 18, i32 3, i32 1, i32 14, i32 18, i32 9, i32 11, i32 0>
 
   ret <16 x i8> %shuffle
 }
@@ -1889,7 +1889,7 @@ define <16 x i8> @stress_test2(<16 x i8> %s.0.0, <16 x i8> %s.0.1, <16 x i8> %s.
 entry:
   %s.1.0 = shufflevector <16 x i8> %s.0.0, <16 x i8> %s.0.1, <16 x i32> <i32 29, i32 30, i32 2, i32 16, i32 26, i32 21, i32 11, i32 26, i32 26, i32 3, i32 4, i32 5, i32 30, i32 28, i32 15, i32 5>
   %s.1.1 = shufflevector <16 x i8> %s.0.1, <16 x i8> %s.0.2, <16 x i32> <i32 31, i32 1, i32 24, i32 12, i32 28, i32 5, i32 2, i32 9, i32 29, i32 1, i32 31, i32 5, i32 6, i32 17, i32 15, i32 22>
-  %s.2.0 = shufflevector <16 x i8> %s.1.0, <16 x i8> %s.1.1, <16 x i32> <i32 22, i32 1, i32 12, i32 3, i32 30, i32 4, i32 30, i32 undef, i32 1, i32 10, i32 14, i32 18, i32 27, i32 13, i32 16, i32 19>
+  %s.2.0 = shufflevector <16 x i8> %s.1.0, <16 x i8> %s.1.1, <16 x i32> <i32 22, i32 1, i32 12, i32 3, i32 30, i32 4, i32 30, i32 poison, i32 1, i32 10, i32 14, i32 18, i32 27, i32 13, i32 16, i32 19>
 
   ret <16 x i8> %s.2.0
 }
@@ -1910,7 +1910,7 @@ define void @constant_gets_selected(ptr %ptr1, ptr %ptr2) {
 ; AVX-NEXT:    retq
 entry:
   %weird_zero = bitcast <4 x i32> zeroinitializer to <16 x i8>
-  %shuffle.i = shufflevector <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0>, <16 x i8> %weird_zero, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
+  %shuffle.i = shufflevector <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0>, <16 x i8> %weird_zero, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27>
   %weirder_zero = bitcast <16 x i8> %shuffle.i to <4 x i32>
   store <4 x i32> %weirder_zero, ptr %ptr1, align 16
   store <4 x i32> zeroinitializer, ptr %ptr2, align 16
@@ -1973,7 +1973,7 @@ define <16 x i8> @shuffle_v16i8_zz_00_uu_02_03_uu_05_06_zz_08_09_uu_11_12_13_14(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $8, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 16, i32 0, i32 undef, i32 2, i32 3, i32 undef, i32 5, i32 6, i32 16, i32 8, i32 9, i32 undef, i32 11, i32 12, i32 13, i32 14>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 16, i32 0, i32 poison, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 16, i32 8, i32 9, i32 poison, i32 11, i32 12, i32 13, i32 14>
   ret <16 x i8> %shuffle
 }
 
@@ -1987,7 +1987,7 @@ define <16 x i8> @shuffle_v16i8_01_uu_uu_uu_uu_zz_uu_zz_uu_zz_11_zz_13_zz_15_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 undef, i32 16, i32 undef, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 poison, i32 16, i32 poison, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2001,7 +2001,7 @@ define <16 x i8> @shuffle_v16i8_02_03_zz_zz_06_07_uu_uu_uu_uu_uu_uu_14_15_zz_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 2, i32 3, i32 16, i32 16, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 14, i32 15, i32 16, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 2, i32 3, i32 16, i32 16, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 14, i32 15, i32 16, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2015,7 +2015,7 @@ define <16 x i8> @shuffle_v16i8_07_zz_zz_zz_zz_zz_uu_uu_15_uu_uu_uu_uu_uu_zz_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlq $56, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32><i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 poison, i32 poison, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2041,7 +2041,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2079,7 +2079,7 @@ define <16 x i8> @shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; XOP-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
 ; XOP-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2117,7 +2117,7 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06(
 ; XOP-NEXT:    vpsrlq $8, %xmm0, %xmm0
 ; XOP-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
   ret <16 x i8> %shuffle
 }
 
@@ -2159,7 +2159,7 @@ define <16 x i8> @shuffle_v16i8_03_00_01_02_07_04_05_06_11_08_09_10_15_12_13_14(
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $8, %xmm0, %xmm0
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14>
   ret <16 x i8> %shuffle
 }
 
@@ -2197,7 +2197,7 @@ define <16 x i8> @shuffle_v16i8_02_03_04_05_06_07_00_01_10_11_12_13_14_15_08_09(
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq $48, %xmm0, %xmm0
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9>
   ret <16 x i8> %shuffle
 }
 
@@ -2269,7 +2269,7 @@ define <16 x i8> @shuffle_v16i8_uu_02_03_zz_uu_06_07_zz_uu_10_11_zz_uu_14_15_zz(
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrld $8, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 undef, i32 2, i32 3, i32 16, i32 undef, i32 6, i32 7, i32 16, i32 undef, i32 10, i32 11, i32 16, i32 undef, i32 14, i32 15, i32 16>
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 poison, i32 2, i32 3, i32 16, i32 poison, i32 6, i32 7, i32 16, i32 poison, i32 10, i32 11, i32 16, i32 poison, i32 14, i32 15, i32 16>
   ret <16 x i8> %shuffle
 }
 
@@ -2285,9 +2285,9 @@ define <16 x i8> @shuffle_v16i8_bitcast_unpack(<16 x i8> %a, <16 x i8> %b) {
 ; AVX-NEXT:    retq
   %shuffle8  = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 7, i32 23, i32 6, i32 22, i32 5, i32 21, i32 4, i32 20, i32 3, i32 19, i32 2, i32 18, i32 1, i32 17, i32 0, i32 16>
   %bitcast32 = bitcast <16 x i8> %shuffle8 to <4 x float>
-  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %bitcast16 = bitcast <4 x float> %shuffle32 to <8 x i16>
-  %shuffle16 = shufflevector <8 x i16> %bitcast16, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+  %shuffle16 = shufflevector <8 x i16> %bitcast16, <8 x i16> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %bitcast8  = bitcast <8 x i16> %shuffle16 to <16 x i8>
   ret <16 x i8> %bitcast8
 }
@@ -2341,7 +2341,7 @@ define <16 x i8> @insert_dup_mem_v16i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <16 x i32> zeroinitializer
   ret <16 x i8> %tmp3
 }
 
@@ -2400,7 +2400,7 @@ define <16 x i8> @insert_dup_mem_v16i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> zeroinitializer
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <16 x i32> zeroinitializer
   ret <16 x i8> %tmp4
 }
 
@@ -2449,7 +2449,7 @@ define <16 x i8> @insert_dup_elt1_mem_v16i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <16 x i8> %tmp3
 }
 
@@ -2498,7 +2498,7 @@ define <16 x i8> @insert_dup_elt2_mem_v16i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   ret <16 x i8> %tmp3
 }
 
@@ -2566,7 +2566,7 @@ define <16 x i8> @insert_dup_elt1_mem_v16i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <16 x i8> %tmp4
 }
 
@@ -2634,7 +2634,7 @@ define <16 x i8> @insert_dup_elt2_mem_v16i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   ret <16 x i8> %tmp4
 }
 
@@ -2682,10 +2682,10 @@ define <16 x i8> @PR31364(ptr nocapture readonly %a, ptr nocapture readonly %b)
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1],zero,xmm0[1,1,1,1,1,0,0,0]
 ; AVX-NEXT:    retq
   %v0 = load i8, ptr %a, align 1
-  %vecins = insertelement <16 x i8> <i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, i8 %v0, i32 0
+  %vecins = insertelement <16 x i8> <i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, i8 %v0, i32 0
   %v1 = load i8, ptr %b, align 1
   %vecins2 = insertelement <16 x i8> %vecins, i8 %v1, i32 1
-  %result = shufflevector <16 x i8> %vecins2, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 3, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0>
+  %result = shufflevector <16 x i8> %vecins2, <16 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 3, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0>
   ret <16 x i8> %result
 }
 
@@ -2763,11 +2763,11 @@ define <16 x i8> @PR31301(ptr nocapture readonly %x, ptr nocapture readonly %y)
 ; XOPAVX2-NEXT:    retq
 entry:
   %0 = load i8, ptr %x, align 1
-  %1 = insertelement <16 x i8> undef, i8 %0, i32 0
-  %lane = shufflevector <16 x i8> %1, <16 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %1 = insertelement <16 x i8> poison, i8 %0, i32 0
+  %lane = shufflevector <16 x i8> %1, <16 x i8> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %2 = load i8, ptr %y, align 1
-  %3 = insertelement <16 x i8> undef, i8 %2, i32 0
-  %lane3 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %3 = insertelement <16 x i8> poison, i8 %2, i32 0
+  %lane3 = shufflevector <16 x i8> %3, <16 x i8> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %vzip.i = shufflevector <16 x i8> %lane, <16 x i8> %lane3, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %vzip.i
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
index 55f7dec28bfcc..1d389f9817229 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
@@ -300,7 +300,7 @@ define <2 x double> @shuffle_v2f64_u2(<2 x double> %a, <2 x double> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm1[0,0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 undef, i32 2>
+  %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 poison, i32 2>
   ret <2 x double> %shuffle
 }
 define <2 x double> @shuffle_v2f64_3u(<2 x double> %a, <2 x double> %b) {
@@ -314,7 +314,7 @@ define <2 x double> @shuffle_v2f64_3u(<2 x double> %a, <2 x double> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufpd {{.*#+}} xmm0 = xmm1[1,0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 3, i32 undef>
+  %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 3, i32 poison>
   ret <2 x double> %shuffle
 }
 
@@ -847,7 +847,7 @@ define <2 x double> @shuffle_v2f64_bitcast_1z(<2 x double> %a) {
 ; AVX-NEXT:    retq
   %shuffle64 = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> <i32 2, i32 1>
   %bitcast32 = bitcast <2 x double> %shuffle64 to <4 x float>
-  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
   %bitcast64 = bitcast <4 x float> %shuffle32 to <2 x double>
   ret <2 x double> %bitcast64
 }
@@ -880,7 +880,7 @@ define <2 x i64> @shuffle_v2i64_bitcast_z123(<2 x i64> %x) {
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %bitcast32 = bitcast <2 x i64> %x to <4 x float>
-  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> <float 1.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
   %bitcast64 = bitcast <4 x float> %shuffle32 to <2 x i64>
   %and = and <2 x i64> %bitcast64, <i64 -4294967296, i64 -1>
   ret <2 x i64> %and
@@ -896,7 +896,7 @@ define <2 x i64> @insert_reg_and_zero_v2i64(i64 %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovq %rdi, %xmm0
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %shuffle
 }
@@ -912,7 +912,7 @@ define <2 x i64> @insert_mem_and_zero_v2i64(ptr %ptr) {
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
   %a = load i64, ptr %ptr
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %shuffle
 }
@@ -927,7 +927,7 @@ define <2 x double> @insert_reg_and_zero_v2f64(double %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> zeroinitializer, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %shuffle
 }
@@ -943,7 +943,7 @@ define <2 x double> @insert_mem_and_zero_v2f64(ptr %ptr) {
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> zeroinitializer, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %shuffle
 }
@@ -976,7 +976,7 @@ define <2 x i64> @insert_reg_lo_v2i64(i64 %a, <2 x i64> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpinsrq $0, %rdi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %shuffle
 }
@@ -1007,7 +1007,7 @@ define <2 x i64> @insert_mem_lo_v2i64(ptr %ptr, <2 x i64> %b) {
 ; AVX-NEXT:    vpinsrq $0, (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = load i64, ptr %ptr
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %shuffle
 }
@@ -1040,7 +1040,7 @@ define <2 x i64> @insert_reg_hi_v2i64(i64 %a, <2 x i64> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 2, i32 0>
   ret <2 x i64> %shuffle
 }
@@ -1074,7 +1074,7 @@ define <2 x i64> @insert_mem_hi_v2i64(ptr %ptr, <2 x i64> %b) {
 ; AVX-NEXT:    vpinsrq $1, (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = load i64, ptr %ptr
-  %v = insertelement <2 x i64> undef, i64 %a, i32 0
+  %v = insertelement <2 x i64> poison, i64 %a, i32 0
   %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> <i32 2, i32 0>
   ret <2 x i64> %shuffle
 }
@@ -1104,7 +1104,7 @@ define <2 x double> @insert_reg_lo_v2f64(double %a, <2 x double> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %shuffle
 }
@@ -1120,7 +1120,7 @@ define <2 x double> @insert_mem_lo_v2f64(ptr %ptr, <2 x double> %b) {
 ; AVX-NEXT:    vmovlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; AVX-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %shuffle
 }
@@ -1136,7 +1136,7 @@ define <2 x double> @insert_reg_hi_v2f64(double %a, <2 x double> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 2, i32 0>
   ret <2 x double> %shuffle
 }
@@ -1152,7 +1152,7 @@ define <2 x double> @insert_mem_hi_v2f64(ptr %ptr, <2 x double> %b) {
 ; AVX-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; AVX-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <2 x double> undef, double %a, i32 0
+  %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 2, i32 0>
   ret <2 x double> %shuffle
 }
@@ -1182,8 +1182,8 @@ define <2 x double> @insert_dup_reg_v2f64(double %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX-NEXT:    retq
-  %v = insertelement <2 x double> undef, double %a, i32 0
-  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %v = insertelement <2 x double> poison, double %a, i32 0
+  %shuffle = shufflevector <2 x double> %v, <2 x double> poison, <2 x i32> <i32 0, i32 0>
   ret <2 x double> %shuffle
 }
 
@@ -1214,8 +1214,8 @@ define <2 x double> @insert_dup_mem_v2f64(ptr %ptr) {
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <2 x double> undef, double %a, i32 0
-  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %v = insertelement <2 x double> poison, double %a, i32 0
+  %shuffle = shufflevector <2 x double> %v, <2 x double> poison, <2 x i32> <i32 0, i32 0>
   ret <2 x double> %shuffle
 }
 
@@ -1246,7 +1246,7 @@ define <2 x double> @insert_dup_mem128_v2f64(ptr %ptr) nounwind {
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX-NEXT:    retq
   %v = load  <2 x double>,  ptr %ptr
-  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %shuffle = shufflevector <2 x double> %v, <2 x double> poison, <2 x i32> <i32 0, i32 0>
   ret <2 x double> %shuffle
 }
 
@@ -1263,8 +1263,8 @@ define <2 x i64> @insert_dup_mem_v2i64(ptr %ptr) {
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX-NEXT:    retq
   %tmp = load i64, ptr %ptr, align 1
-  %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
-  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %tmp1 = insertelement <2 x i64> poison, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <2 x i32> zeroinitializer
   ret <2 x i64> %tmp2
 }
 
@@ -1281,7 +1281,7 @@ define <2 x double> @shuffle_mem_v2f64_10(ptr %ptr) {
 ; AVX-NEXT:    retq
 
   %a = load <2 x double>, ptr %ptr
-  %shuffle = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+  %shuffle = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x double> %shuffle
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
index 6fe16f85ec6be..025518d1a101e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -725,7 +725,7 @@ define <4 x float> @shuffle_v4f32_zuu4(<4 x float> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,zero,xmm0[0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <4 x i32> <i32 0, i32 undef, i32 undef, i32 4>
+  %shuffle = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <4 x i32> <i32 0, i32 poison, i32 poison, i32 4>
   ret <4 x float> %shuffle
 }
 
@@ -988,7 +988,7 @@ define <4 x float> @shuffle_v4f32_0z2z(<4 x float> %v) {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> %v, <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 0, i32 4, i32 2, i32 4>
+  %shuffle = shufflevector <4 x float> %v, <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 0, i32 4, i32 2, i32 4>
   ret <4 x float> %shuffle
 }
 
@@ -1003,7 +1003,7 @@ define <4 x float> @shuffle_v4f32_u051(<4 x float> %a, <4 x float> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 undef, i32 0, i32 5, i32 1>
+  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 poison, i32 0, i32 5, i32 1>
   ret <4 x float> %shuffle
 }
 
@@ -1044,7 +1044,7 @@ define <4 x float> @shuffle_v4f32_0zz4(<4 x float> %a, <4 x float> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,zero,xmm1[0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> %b, <4 x float> zeroinitializer, <4 x i32> <i32 undef, i32 5, i32 6, i32 0>
+  %shuffle = shufflevector <4 x float> %b, <4 x float> zeroinitializer, <4 x i32> <i32 poison, i32 5, i32 6, i32 0>
   %shuffle1 = shufflevector <4 x float> %a, <4 x float> %shuffle, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x float> %shuffle1
 }
@@ -1086,7 +1086,7 @@ define <4 x float> @shuffle_v4f32_0zz6(<4 x float> %a, <4 x float> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,zero,xmm1[2]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 0, i32 undef, i32 undef, i32 6>
+  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 0, i32 poison, i32 poison, i32 6>
   %shuffle1 = shufflevector <4 x float> zeroinitializer, <4 x float> %shuffle, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
   ret <4 x float> %shuffle1
 }
@@ -1128,7 +1128,7 @@ define <4 x float> @shuffle_v4f32_0z24(<4 x float> %a, <4 x float> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,xmm0[2],xmm1[0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 0, i32 undef, i32 2, i32 4>
+  %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 0, i32 poison, i32 2, i32 4>
   %shuffle1 = shufflevector <4 x float> zeroinitializer, <4 x float> %shuffle, <4 x i32> <i32 4, i32 1, i32 6, i32 7>
   ret <4 x float> %shuffle1
 }
@@ -1292,7 +1292,7 @@ define <4 x i32> @shuffle_v4i32_zuu4(<4 x i32> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x i32> zeroinitializer, <4 x i32> %a, <4 x i32> <i32 0, i32 undef, i32 undef, i32 4>
+  %shuffle = shufflevector <4 x i32> zeroinitializer, <4 x i32> %a, <4 x i32> <i32 0, i32 poison, i32 poison, i32 4>
   ret <4 x i32> %shuffle
 }
 
@@ -1553,7 +1553,7 @@ define <4 x i32> @shuffle_v4i32_2456(<4 x i32> %a, <4 x i32> %b) {
 ; AVX512VL-NEXT:    vpermi2d %xmm0, %xmm1, %xmm2
 ; AVX512VL-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
-  %s1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %s1 = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %s2 = shufflevector <4 x i32> %s1, <4 x i32> %b, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x i32> %s2
 }
@@ -1569,7 +1569,7 @@ define <4 x i32> @shuffle_v4i32_40u1(<4 x i32> %a, <4 x i32> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 4, i32 0, i32 undef, i32 1>
+  %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 4, i32 0, i32 poison, i32 1>
   ret <4 x i32> %shuffle
 }
 
@@ -1631,7 +1631,7 @@ define <4 x i32> @shuffle_v4i32_0u1u(<4 x i32> %a, <4 x i32> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+  %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
   ret <4 x i32> %shuffle
 }
 
@@ -1677,7 +1677,7 @@ define <4 x i32> @shuffle_v4i32_01zu(<4 x i32> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 7, i32 undef>
+  %shuffle = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 7, i32 poison>
   ret <4 x i32> %shuffle
 }
 
@@ -1817,7 +1817,7 @@ define <4 x i32> @shuffle_v4i32_bitcast_0415(<4 x i32> %a, <4 x i32> %b) {
 ; AVX-NEXT:    retq
   %shuffle32 = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 5, i32 0, i32 4>
   %bitcast64 = bitcast <4 x i32> %shuffle32 to <2 x double>
-  %shuffle64 = shufflevector <2 x double> %bitcast64, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+  %shuffle64 = shufflevector <2 x double> %bitcast64, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   %bitcast32 = bitcast <2 x double> %shuffle64 to <4 x i32>
   ret <4 x i32> %bitcast32
 }
@@ -1833,7 +1833,7 @@ define <4 x float> @shuffle_v4f32_bitcast_4401(<4 x float> %a, <4 x i32> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,0],xmm0[0,1]
 ; AVX-NEXT:    retq
-  %1 = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  %1 = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
   %2 = bitcast <4 x i32> %1 to <2 x double>
   %3 = bitcast <4 x float> %a to <2 x double>
   %4 = shufflevector <2 x double> %2, <2 x double> %3, <2 x i32> <i32 0, i32 2>
@@ -1851,7 +1851,7 @@ define <4 x float> @shuffle_v4f32_bitcast_0045(<4 x float> %a, <4 x i32> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,1]
 ; AVX-NEXT:    retq
-  %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  %1 = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
   %2 = bitcast <4 x i32> %b to <4 x float>
   %3 = shufflevector <4 x float> %1, <4 x float> %2, <4 x i32> <i32 1, i32 0, i32 4, i32 5>
   ret <4 x float> %3
@@ -2004,7 +2004,7 @@ define <4 x float> @broadcast_v4f32_0101_from_v2f32(ptr %x) {
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX-NEXT:    retq
   %1 = load <2 x float>, ptr %x, align 1
-  %2 = shufflevector <2 x float> %1, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = shufflevector <2 x float> %1, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   ret <4 x float> %2
 }
 
@@ -2086,7 +2086,7 @@ define <4 x i32> @insert_reg_and_zero_v4i32(i32 %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovd %edi, %xmm0
 ; AVX-NEXT:    retq
-  %v = insertelement <4 x i32> undef, i32 %a, i32 0
+  %v = insertelement <4 x i32> poison, i32 %a, i32 0
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i32> %shuffle
 }
@@ -2102,7 +2102,7 @@ define <4 x i32> @insert_mem_and_zero_v4i32(ptr %ptr) {
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
   %a = load i32, ptr %ptr
-  %v = insertelement <4 x i32> undef, i32 %a, i32 0
+  %v = insertelement <4 x i32> poison, i32 %a, i32 0
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i32> %shuffle
 }
@@ -2140,7 +2140,7 @@ define <4 x float> @insert_reg_and_zero_v4f32(float %a) {
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
-  %v = insertelement <4 x float> undef, float %a, i32 0
+  %v = insertelement <4 x float> poison, float %a, i32 0
   %shuffle = shufflevector <4 x float> %v, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x float> %shuffle
 }
@@ -2156,7 +2156,7 @@ define <4 x float> @insert_mem_and_zero_v4f32(ptr %ptr) {
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
   %a = load float, ptr %ptr
-  %v = insertelement <4 x float> undef, float %a, i32 0
+  %v = insertelement <4 x float> poison, float %a, i32 0
   %shuffle = shufflevector <4 x float> %v, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x float> %shuffle
 }
@@ -2198,7 +2198,7 @@ define <4 x i32> @insert_reg_lo_v4i32(i64 %a, <4 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX2OR512VL-NEXT:    retq
   %a.cast = bitcast i64 %a to <2 x i32>
-  %v = shufflevector <2 x i32> %a.cast, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x i32> %a.cast, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %shuffle
 }
@@ -2231,7 +2231,7 @@ define <4 x i32> @insert_mem_lo_v4i32(ptr %ptr, <4 x i32> %b) {
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX-NEXT:    retq
   %a = load <2 x i32>, ptr %ptr
-  %v = shufflevector <2 x i32> %a, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x i32> %a, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %shuffle
 }
@@ -2249,7 +2249,7 @@ define <4 x i32> @insert_reg_hi_v4i32(i64 %a, <4 x i32> %b) {
 ; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
   %a.cast = bitcast i64 %a to <2 x i32>
-  %v = shufflevector <2 x i32> %a.cast, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x i32> %a.cast, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x i32> %shuffle
 }
@@ -2267,7 +2267,7 @@ define <4 x i32> @insert_mem_hi_v4i32(ptr %ptr, <4 x i32> %b) {
 ; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
   %a = load <2 x i32>, ptr %ptr
-  %v = shufflevector <2 x i32> %a, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x i32> %a, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x i32> %shuffle
 }
@@ -2298,7 +2298,7 @@ define <4 x float> @insert_reg_lo_v4f32(double %a, <4 x float> %b) {
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; AVX-NEXT:    retq
   %a.cast = bitcast double %a to <2 x float>
-  %v = shufflevector <2 x float> %a.cast, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x float> %a.cast, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x float> %v, <4 x float> %b, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x float> %shuffle
 }
@@ -2314,7 +2314,7 @@ define <4 x float> @insert_mem_lo_v4f32(ptr %ptr, <4 x float> %b) {
 ; AVX-NEXT:    vmovlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; AVX-NEXT:    retq
   %a = load <2 x float>, ptr %ptr
-  %v = shufflevector <2 x float> %a, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x float> %a, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x float> %v, <4 x float> %b, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x float> %shuffle
 }
@@ -2331,7 +2331,7 @@ define <4 x float> @insert_reg_hi_v4f32(double %a, <4 x float> %b) {
 ; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX-NEXT:    retq
   %a.cast = bitcast double %a to <2 x float>
-  %v = shufflevector <2 x float> %a.cast, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x float> %a.cast, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x float> %v, <4 x float> %b, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x float> %shuffle
 }
@@ -2347,7 +2347,7 @@ define <4 x float> @insert_mem_hi_v4f32(ptr %ptr, <4 x float> %b) {
 ; AVX-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; AVX-NEXT:    retq
   %a = load <2 x float>, ptr %ptr
-  %v = shufflevector <2 x float> %a, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %v = shufflevector <2 x float> %a, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %shuffle = shufflevector <4 x float> %v, <4 x float> %b, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x float> %shuffle
 }
@@ -2365,7 +2365,7 @@ define <4 x float> @shuffle_mem_v4f32_3210(ptr %ptr) {
 ; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = mem[3,2,1,0]
 ; AVX-NEXT:    retq
   %a = load <4 x float>, ptr %ptr
-  %shuffle = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %shuffle = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %shuffle
 }
 
@@ -2382,7 +2382,7 @@ define <4 x i32> @insert_dup_mem_v4i32(ptr %ptr) {
 ; AVX-NEXT:    retq
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
-  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> poison, <4 x i32> zeroinitializer
   ret <4 x i32> %tmp2
 }
 
@@ -2416,9 +2416,9 @@ define <4 x float> @shuffle_mem_pmovzx_v4f32(ptr %p0, ptr %p1) {
 ; AVX2OR512VL-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX2OR512VL-NEXT:    retq
   %1 = load <2 x float>, ptr %p0
-  %2 = shufflevector <2 x float> %1, <2 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
-  %3 = shufflevector <4 x float> %2, <4 x float> <float undef, float undef, float 0.000000e+00, float 0.000000e+00>, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %4 = shufflevector <2 x float> %1, <2 x float> undef, <4 x i32> zeroinitializer
+  %2 = shufflevector <2 x float> %1, <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
+  %3 = shufflevector <4 x float> %2, <4 x float> <float poison, float poison, float 0.000000e+00, float 0.000000e+00>, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  %4 = shufflevector <2 x float> %1, <2 x float> poison, <4 x i32> zeroinitializer
   store <4 x float> %3, ptr %p1
   ret <4 x float> %4
 }
@@ -2437,7 +2437,7 @@ define <4 x i32> @shuffle_v4i32_z0zX(<4 x i32> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 0, i32 4, i32 undef>
+  %shuffle = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 0, i32 4, i32 poison>
   ret <4 x i32> %shuffle
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index d95bf2a48c3cd..d73cfb379333b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -148,7 +148,7 @@ define <8 x i16> @shuffle_v8i16_u0u1u2u3(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3>
   ret <8 x i16> %shuffle
 }
 define <8 x i16> @shuffle_v8i16_u4u5u6u7(<8 x i16> %a, <8 x i16> %b) {
@@ -161,7 +161,7 @@ define <8 x i16> @shuffle_v8i16_u4u5u6u7(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 4, i32 undef, i32 5, i32 undef, i32 6, i32 undef, i32 7>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
   ret <8 x i16> %shuffle
 }
 define <8 x i16> @shuffle_v8i16_31206745(<8 x i16> %a, <8 x i16> %b) {
@@ -615,7 +615,7 @@ define <8 x i16> @shuffle_v8i16_60514754(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,0,1,10,11,2,3,8,9,14,15,10,11,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 6, i32 0, i32 5, i32 1, i32 4, i32 7, i32 5, i32 4>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 6, i32 0, i32 5, i32 1, i32 4, i32 7, i32 5, i32 4>
   ret <8 x i16> %shuffle
 }
 
@@ -785,7 +785,7 @@ define <8 x i16> @shuffle_v8i16_0X444444(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,8,9,8,9,8,9,8,9,8,9,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 undef, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 poison, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
   ret <8 x i16> %shuffle
 }
 define <8 x i16> @shuffle_v8i16_44X04444(<8 x i16> %a, <8 x i16> %b) {
@@ -810,7 +810,7 @@ define <8 x i16> @shuffle_v8i16_44X04444(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,8,9,8,9,0,1,8,9,8,9,8,9,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 4, i32 undef, i32 0, i32 4, i32 4, i32 4, i32 4>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 4, i32 poison, i32 0, i32 4, i32 4, i32 4, i32 4>
   ret <8 x i16> %shuffle
 }
 define <8 x i16> @shuffle_v8i16_X4404444(<8 x i16> %a, <8 x i16> %b) {
@@ -835,7 +835,7 @@ define <8 x i16> @shuffle_v8i16_X4404444(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,0,1,8,9,8,9,8,9,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 4, i32 4, i32 0, i32 4, i32 4, i32 4, i32 4>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 4, i32 4, i32 0, i32 4, i32 4, i32 4, i32 4>
   ret <8 x i16> %shuffle
 }
 
@@ -861,7 +861,7 @@ define <8 x i16> @shuffle_v8i16_0127XXXX(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,14,15,4,5,14,15,12,13,14,15]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -887,7 +887,7 @@ define <8 x i16> @shuffle_v8i16_XXXX4563(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,6,7,4,5,6,7,8,9,10,11,12,13,6,7]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 4, i32 5, i32 6, i32 3>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 6, i32 3>
   ret <8 x i16> %shuffle
 }
 
@@ -913,7 +913,7 @@ define <8 x i16> @shuffle_v8i16_4563XXXX(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,6,7,8,9,10,11,0,1,2,3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1322,7 +1322,7 @@ define <8 x i16> @shuffle_v8i16_443aXXXX(<8 x i16> %a, <8 x i16> %b) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[8,9,8,9,6,7],xmm1[4,5],xmm0[8,9,10,11,12,13,14,15]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 4, i32 3, i32 10, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 4, i32 4, i32 3, i32 10, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1371,7 +1371,7 @@ define <8 x i16> @shuffle_v8i16_032dXXXX(<8 x i16> %a, <8 x i16> %b) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,6,7,4,5],xmm1[10,11],xmm0[0,1],xmm1[10,11],xmm0[0,1,2,3]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 3, i32 2, i32 13, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 3, i32 2, i32 13, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 define <8 x i16> @shuffle_v8i16_XXXdXXXX(<8 x i16> %a, <8 x i16> %b) {
@@ -1384,7 +1384,7 @@ define <8 x i16> @shuffle_v8i16_XXXdXXXX(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,2,3,3]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 13, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 13, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1416,7 +1416,7 @@ define <8 x i16> @shuffle_v8i16_012dXXXX(<8 x i16> %a, <8 x i16> %b) {
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 13, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 13, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1467,7 +1467,7 @@ define <8 x i16> @shuffle_v8i16_XXXXcde3(<8 x i16> %a, <8 x i16> %b) {
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,6],xmm0[7]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 3>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 3>
   ret <8 x i16> %shuffle
 }
 
@@ -1500,7 +1500,7 @@ define <8 x i16> @shuffle_v8i16_cde3XXXX(<8 x i16> %a, <8 x i16> %b) {
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 12, i32 13, i32 14, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 12, i32 13, i32 14, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1655,7 +1655,7 @@ define <8 x i16> @shuffle_v8i16_XXX1X579(<8 x i16> %a, <8 x i16> %b) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,8,9,10,11,14,15],xmm1[2,3]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 5, i32 7, i32 9>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 5, i32 7, i32 9>
   ret <8 x i16> %shuffle
 }
 
@@ -1706,7 +1706,7 @@ define <8 x i16> @shuffle_v8i16_XX4X8acX(<8 x i16> %a, <8 x i16> %b) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[8,9,10,11,8,9,10,11],xmm1[0,1,4,5,8,9,4,5]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 4, i32 undef, i32 8, i32 10, i32 12, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 4, i32 poison, i32 8, i32 10, i32 12, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1722,7 +1722,7 @@ define <8 x i16> @shuffle_v8i16_8zzzzzzz(i16 %i) {
 ; AVX-NEXT:    movzwl %di, %eax
 ; AVX-NEXT:    vmovd %eax, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <8 x i16> undef, i16 %i, i32 0
+  %a = insertelement <8 x i16> poison, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %shuffle
 }
@@ -1739,7 +1739,7 @@ define <8 x i16> @shuffle_v8i16_z8zzzzzz(i16 %i) {
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <8 x i16> undef, i16 %i, i32 0
+  %a = insertelement <8 x i16> poison, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 2, i32 8, i32 3, i32 7, i32 6, i32 5, i32 4, i32 3>
   ret <8 x i16> %shuffle
 }
@@ -1756,7 +1756,7 @@ define <8 x i16> @shuffle_v8i16_zzzzz8zz(i16 %i) {
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrw $5, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <8 x i16> undef, i16 %i, i32 0
+  %a = insertelement <8 x i16> poison, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 0, i32 0>
   ret <8 x i16> %shuffle
 }
@@ -1773,8 +1773,8 @@ define <8 x i16> @shuffle_v8i16_zuuzuuz8(i16 %i) {
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <8 x i16> undef, i16 %i, i32 0
-  %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 8>
+  %a = insertelement <8 x i16> poison, i16 %i, i32 0
+  %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 8>
   ret <8 x i16> %shuffle
 }
 
@@ -1790,7 +1790,7 @@ define <8 x i16> @shuffle_v8i16_zzBzzzzz(i16 %i) {
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrw $2, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %a = insertelement <8 x i16> undef, i16 %i, i32 3
+  %a = insertelement <8 x i16> poison, i16 %i, i32 3
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 1, i32 11, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %shuffle
 }
@@ -1843,7 +1843,7 @@ define <8 x i16> @shuffle_v8i16_ueuu123u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 14, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 14, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1897,7 +1897,7 @@ define <8 x i16> @shuffle_v8i16_u6uu123u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1911,7 +1911,7 @@ define <8 x i16> @shuffle_v8i16_uuuu123u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -1963,7 +1963,7 @@ define <8 x i16> @shuffle_v8i16_ucdeuu1u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 12, i32 13, i32 14, i32 undef, i32 undef, i32 1, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 12, i32 13, i32 14, i32 poison, i32 poison, i32 1, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2017,7 +2017,7 @@ define <8 x i16> @shuffle_v8i16_u456uu1u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 1, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 1, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2031,7 +2031,7 @@ define <8 x i16> @shuffle_v8i16_u456uuuu(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2087,7 +2087,7 @@ define <8 x i16> @shuffle_v8i16_u456uu9u(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 9, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 9, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2143,7 +2143,7 @@ define <8 x i16> @shuffle_v8i16_u6uu9abu(<8 x i16> %a, <8 x i16> %b) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 9, i32 10, i32 11, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 9, i32 10, i32 11, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2169,7 +2169,7 @@ define <8 x i16> @shuffle_v8i16_0uuu1uuu(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2221,7 +2221,7 @@ define <8 x i16> @shuffle_v8i16_0u1u2u3u(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2347,7 +2347,7 @@ define <8 x i16> @shuffle_v8i16_01u0u110(<8 x i16> %a) {
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,0,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 undef, i32 0, i32 undef, i32 1, i32 1, i32 0>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 poison, i32 0, i32 poison, i32 1, i32 1, i32 0>
   ret <8 x i16> %shuffle
 }
 
@@ -2391,7 +2391,7 @@ define <8 x i16> @shuffle_v8i16_467uu675(<8 x i16> %a) {
 ; XOP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,7,5]
 ; XOP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 4, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7, i32 5>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 4, i32 6, i32 7, i32 poison, i32 poison, i32 6, i32 7, i32 5>
   ret <8 x i16> %shuffle
 }
 
@@ -2493,7 +2493,7 @@ define <8 x i16> @shuffle_v8i16_02460246(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,0,1,4,5,8,9,12,13]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
   ret <8 x i16> %shuffle
 }
 
@@ -2519,7 +2519,7 @@ define <8 x i16> @shuffle_v8i16_04040404(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,0,1,8,9,0,1,8,9,0,1,8,9]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 4, i32 0, i32 4, i32 0, i32 4, i32 0, i32 4>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 0, i32 4, i32 0, i32 4, i32 0, i32 4, i32 0, i32 4>
   ret <8 x i16> %shuffle
 }
 
@@ -2682,7 +2682,7 @@ define <8 x i16> @shuffle_v8i16_zz01zX4X(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 8, i32 8, i32 0, i32 1, i32 8, i32 undef, i32 4, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 8, i32 8, i32 0, i32 1, i32 8, i32 poison, i32 4, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2696,7 +2696,7 @@ define <8 x i16> @shuffle_v8i16_z0X2z456(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $16, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 8, i32 0, i32 undef, i32 2, i32 8, i32 4, i32 5, i32 6>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 8, i32 0, i32 poison, i32 2, i32 8, i32 4, i32 5, i32 6>
   ret <8 x i16> %shuffle
 }
 
@@ -2710,7 +2710,7 @@ define <8 x i16> @shuffle_v8i16_1z3zXz7z(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 1, i32 8, i32 3, i32 8, i32 undef, i32 8, i32 7, i32 8>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 1, i32 8, i32 3, i32 8, i32 poison, i32 8, i32 7, i32 8>
   ret <8 x i16> %shuffle
 }
 
@@ -2724,7 +2724,7 @@ define <8 x i16> @shuffle_v8i16_1X3z567z(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlq $16, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 1, i32 undef, i32 3, i32 8, i32 5, i32 6, i32 7, i32 8>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 1, i32 poison, i32 3, i32 8, i32 5, i32 6, i32 7, i32 8>
   ret <8 x i16> %shuffle
 }
 
@@ -2752,7 +2752,7 @@ define <8 x i16> @shuffle_v8i16_3zXXXzzz(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm0
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 3, i32 8, i32 undef, i32 undef, i32 undef, i32 8, i32 8, i32 8>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32><i32 3, i32 8, i32 poison, i32 poison, i32 poison, i32 8, i32 8, i32 8>
   ret <8 x i16> %shuffle
 }
 
@@ -2766,7 +2766,7 @@ define <8 x i16> @shuffle_v8i16_01u3zzuz(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 undef, i32 3, i32 8, i32 8, i32 undef, i32 8>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 8, i32 8, i32 poison, i32 8>
   ret <8 x i16> %shuffle
 }
 
@@ -2875,7 +2875,7 @@ define <8 x i16> @shuffle_v8i16_fu3ucc5u(<8 x i16> %a, <8 x i16> %b) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm1[14,15,10,11],xmm0[6,7,8,9],xmm1[8,9,8,9],xmm0[10,11,12,13]
 ; XOP-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 15, i32 undef, i32 3, i32 undef, i32 12, i32 12, i32 5, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 15, i32 poison, i32 3, i32 poison, i32 12, i32 12, i32 5, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -2889,7 +2889,7 @@ define <8 x i16> @shuffle_v8i16_8012345u(<8 x i16> %a) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 undef>
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison>
   ret <8 x i16> %shuffle
 }
 
@@ -3035,7 +3035,7 @@ define <8 x i16> @shuffle_v8i16_9zzzuuuu(<8 x i16> %x) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    extrq {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    retq
-  %r = shufflevector <8 x i16> zeroinitializer, <8 x i16> %x, <8 x i32> <i32 9, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x i16> zeroinitializer, <8 x i16> %x, <8 x i32> <i32 9, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %r
 }
 
@@ -3079,7 +3079,7 @@ define <8 x i16> @shuffle_v8i16_2zzzuuuu(<8 x i16> %x) {
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    extrq {{.*#+}} xmm0 = xmm0[4,5],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    retq
-  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 2, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 2, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i16> %r
 }
 
@@ -3123,7 +3123,7 @@ define <8 x i16> @shuffle_v8i16_3uu6zzzz(<8 x i16> %x) {
 ; XOP-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
 ; XOP-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
 ; XOP-NEXT:    retq
-  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 3, i32 undef, i32 undef, i32 6, i32 8, i32 8, i32 8, i32 8>
+  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 3, i32 poison, i32 poison, i32 6, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i16> %r
 }
 
@@ -3194,7 +3194,7 @@ define <8 x i16> @insert_dup_mem_v8i16_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> zeroinitializer
   ret <8 x i16> %tmp3
 }
 
@@ -3236,7 +3236,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <8 x i32> zeroinitializer
   ret <8 x i16> %tmp4
 }
 
@@ -3274,7 +3274,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i16> %tmp3
 }
 
@@ -3324,7 +3324,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 1
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <8 x i16> %tmp3
 }
 
@@ -3379,7 +3379,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_sext_i16(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i16> %tmp4
 }
 
@@ -3446,7 +3446,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_sext_i16(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 1
   %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <8 x i16> %tmp4
 }
 
@@ -3484,7 +3484,7 @@ define <8 x i16> @insert_dup_mem_v8i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> zeroinitializer
   ret <8 x i16> %tmp3
 }
 
@@ -3522,7 +3522,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i16> %tmp3
 }
 
@@ -3560,7 +3560,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <8 x i16> %tmp3
 }
 
@@ -3611,7 +3611,7 @@ define <8 x i16> @insert_dup_elt7_mem_v8i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 1
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   ret <8 x i16> %tmp3
 }
 
@@ -3653,6 +3653,6 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i64
   %tmp2 = insertelement <2 x i64> zeroinitializer, i64 %tmp1, i32 0
   %tmp3 = bitcast <2 x i64> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <8 x i32> zeroinitializer
   ret <8 x i16> %tmp4
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index cf0e40ce52138..2bb570521a304 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -2461,7 +2461,7 @@ define <16 x i16> @shuffle_v16i16_00_uu_uu_00_00_00_00_00_08_08_uu_uu_08_08_14_0
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,u,u,u,u,0,1,0,1,0,1,0,1,0,1,16,17,16,17,u,u,u,u,16,17,16,17,28,29,16,17]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 undef, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 undef, i32 undef, i32 8, i32 8, i32 14, i32 8>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 poison, i32 poison, i32 8, i32 8, i32 14, i32 8>
   ret <16 x i16> %shuffle
 }
 
@@ -2491,7 +2491,7 @@ define <16 x i16> @shuffle_v16i16_07_uu_00_00_00_00_00_00_08_08_uu_uu_08_08_08_1
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15,u,u,0,1,0,1,0,1,0,1,0,1,0,1,16,17,16,17,u,u,u,u,16,17,16,17,16,17,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 7, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 undef, i32 undef, i32 8, i32 8, i32 8, i32 15>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 7, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 poison, i32 poison, i32 8, i32 8, i32 8, i32 15>
   ret <16 x i16> %shuffle
 }
 
@@ -2523,7 +2523,7 @@ define <16 x i16> @shuffle_v16i16_00_uu_uu_02_04_04_uu_06_14_14_uu_12_10_10_08_0
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,u,u,u,u,4,5,8,9,8,9,u,u,12,13,28,29,28,29,u,u,24,25,20,21,20,21,16,17,16,17]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 undef, i32 undef, i32 2, i32 4, i32 4, i32 undef, i32 6, i32 14, i32 14, i32 undef, i32 12, i32 10, i32 10, i32 8, i32 8>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 poison, i32 poison, i32 2, i32 4, i32 4, i32 poison, i32 6, i32 14, i32 14, i32 poison, i32 12, i32 10, i32 10, i32 8, i32 8>
   ret <16 x i16> %shuffle
 }
 
@@ -2557,7 +2557,7 @@ define <16 x i16> @shuffle_v16i16_04_04_04_04_uu_uu_uu_uu_08_08_08_uu_uu_12_12_1
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,8,9,8,9,8,9,u,u,u,u,u,u,u,u,16,17,16,17,16,17,u,u,u,u,24,25,24,25,24,25]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 8, i32 8, i32 undef, i32 undef, i32 12, i32 12, i32 12>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 8, i32 8, i32 poison, i32 poison, i32 12, i32 12, i32 12>
   ret <16 x i16> %shuffle
 }
 
@@ -2887,7 +2887,7 @@ define <16 x i16> @shuffle_v16i16_06_07_01_02_07_00_04_05_14_15_09_10_15_08_12_1
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[12,13,14,15,2,3,4,5,14,15,0,1,8,9,10,11,28,29,30,31,18,19,20,21,30,31,16,17,24,25,26,27]
 ; XOPAVX2-NEXT:    retq
-  %1 = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 6, i32 7, i32 1, i32 2, i32 7, i32 0, i32 4, i32 5, i32 14, i32 15, i32 9, i32 10, i32 15, i32 8, i32 12, i32 13>
+  %1 = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 6, i32 7, i32 1, i32 2, i32 7, i32 0, i32 4, i32 5, i32 14, i32 15, i32 9, i32 10, i32 15, i32 8, i32 12, i32 13>
   ret <16 x i16> %1
 }
 
@@ -3586,7 +3586,7 @@ define <16 x i16> @shuffle_v16i16_uu_00_uu_01_uu_02_uu_11_uu_08_uu_09_uu_10_uu_1
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,0,1,u,u,2,3,u,u,4,5,u,u,14,15,u,u,16,17,u,u,18,19,u,u,20,21,u,u,22,23]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 11, i32 undef, i32 8, i32 undef, i32 9, i32 undef, i32 10, i32 undef, i32 11>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 11, i32 poison, i32 8, i32 poison, i32 9, i32 poison, i32 10, i32 poison, i32 11>
   ret <16 x i16> %shuffle
 }
 
@@ -3626,7 +3626,7 @@ define <16 x i16> @shuffle_v16i16_uu_04_uu_05_uu_06_uu_15_uu_12_uu_13_uu_14_uu_1
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,3,3]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,0,1,u,u,2,3,u,u,4,5,u,u,14,15,u,u,16,17,u,u,18,19,u,u,20,21,u,u,22,23]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 4, i32 undef, i32 5, i32 undef, i32 6, i32 undef, i32 15, i32 undef, i32 12, i32 undef, i32 13, i32 undef, i32 14, i32 undef, i32 15>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 15, i32 poison, i32 12, i32 poison, i32 13, i32 poison, i32 14, i32 poison, i32 15>
   ret <16 x i16> %shuffle
 }
 
@@ -4747,7 +4747,7 @@ define <16 x i16> @shuffle_v16i16_00_uu_04_04_04_04_04_12_08_uu_12_12_12_12_12_1
 ; XOPAVX2-NEXT:    vpsllq $48, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7],ymm1[8,9,10,11,12,13,14],ymm0[15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 undef, i32 4, i32 4, i32 4, i32 4, i32 4, i32 12, i32 8, i32 undef, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 poison, i32 4, i32 4, i32 4, i32 4, i32 4, i32 12, i32 8, i32 poison, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
   ret <16 x i16> %shuffle
 }
 
@@ -4806,7 +4806,7 @@ define <16 x i16> @shuffle_v16i16_04_04_uu_00_04_04_04_12_12_12_uu_08_12_12_12_1
 ; XOPAVX2-NEXT:    vpsllq $48, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7],ymm1[8,9,10,11,12,13,14],ymm0[15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 undef, i32 0, i32 4, i32 4, i32 4, i32 12, i32 12, i32 12, i32 undef, i32 8, i32 12, i32 12, i32 12, i32 12>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 poison, i32 0, i32 4, i32 4, i32 4, i32 12, i32 12, i32 12, i32 poison, i32 8, i32 12, i32 12, i32 12, i32 12>
   ret <16 x i16> %shuffle
 }
 
@@ -4865,7 +4865,7 @@ define <16 x i16> @shuffle_v16i16_uu_04_04_00_04_04_04_12_uu_12_12_08_12_12_12_1
 ; XOPAVX2-NEXT:    vpsllq $48, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7],ymm1[8,9,10,11,12,13,14],ymm0[15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 4, i32 4, i32 0, i32 4, i32 4, i32 4, i32 12, i32 undef, i32 12, i32 12, i32 8, i32 12, i32 12, i32 12, i32 12>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 4, i32 4, i32 0, i32 4, i32 4, i32 4, i32 12, i32 poison, i32 12, i32 12, i32 8, i32 12, i32 12, i32 12, i32 12>
   ret <16 x i16> %shuffle
 }
 
@@ -4897,7 +4897,7 @@ define <16 x i16> @shuffle_v16i16_00_01_02_07_uu_uu_uu_uu_08_09_10_15_uu_uu_uu_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,14,15,4,5,14,15,12,13,14,15,16,17,18,19,20,21,30,31,20,21,30,31,28,29,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 9, i32 10, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 15, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -4937,7 +4937,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_uu_04_05_06_11_uu_uu_uu_uu_12_13_14_1
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,2,3,2]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,8,9,10,11,0,1,2,3,4,5,14,15,16,17,18,19,24,25,26,27,16,17,18,19,20,21,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 4, i32 5, i32 6, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 11>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 6, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 11>
   ret <16 x i16> %shuffle
 }
 
@@ -4969,7 +4969,7 @@ define <16 x i16> @shuffle_v16i16_04_05_06_03_uu_uu_uu_uu_12_13_14_11_uu_uu_uu_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,6,7,8,9,10,11,0,1,2,3,24,25,26,27,28,29,22,23,24,25,26,27,16,17,18,19]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 5, i32 6, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 5, i32 6, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -4999,7 +4999,7 @@ define <16 x i16> @shuffle_v16i16_01_zz_02_zz_04_uu_06_07_08_09_10_11_12_13_14_1
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3],zero,zero,ymm0[4,5],zero,zero,ymm0[8,9,u,u,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 2, i32 16, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 2, i32 16, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %shuffle
 }
 
@@ -5742,7 +5742,7 @@ define <16 x i16> @shuffle_v16i16_04_04_03_18_uu_uu_uu_uu_12_12_11_26_uu_uu_uu_u
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,1,2,3,6,5,6,7]
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,3,2,4,5,6,7,8,8,11,10,12,13,14,15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 3, i32 18, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 12, i32 11, i32 26, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 4, i32 3, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 12, i32 11, i32 26, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -5786,7 +5786,7 @@ define <16 x i16> @shuffle_v16i16_00_03_02_21_uu_uu_uu_uu_08_11_10_29_uu_uu_uu_u
 ; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,6,7,4,5,10,11,0,1,10,11,0,1,2,3,16,17,22,23,20,21,26,27,16,17,26,27,16,17,18,19]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 3, i32 2, i32 21, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 11, i32 10, i32 29, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 3, i32 2, i32 21, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 11, i32 10, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -5795,7 +5795,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_21_uu_uu_uu_uu_uu_uu_uu_29_uu_uu_uu_u
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[2,2,2,2,6,6,6,6]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 21, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 29, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 21, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -5839,7 +5839,7 @@ define <16 x i16> @shuffle_v16i16_00_01_02_21_uu_uu_uu_uu_08_09_10_29_uu_uu_uu_u
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7,8,9,10],ymm1[11],ymm0[12,13,14,15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 21, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 9, i32 10, i32 29, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 21, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -5882,7 +5882,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_uu_20_21_22_11_uu_uu_uu_uu_28_29_30_1
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7],ymm1[8,9,10,11,12,13,14],ymm0[15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 11>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 20, i32 21, i32 22, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 28, i32 29, i32 30, i32 11>
   ret <16 x i16> %shuffle
 }
 
@@ -5927,7 +5927,7 @@ define <16 x i16> @shuffle_v16i16_20_21_22_03_uu_uu_uu_uu_28_29_30_11_uu_uu_uu_u
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,3,6,7,6,7]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6,7,8,9,10],ymm0[11],ymm1[12,13,14,15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 20, i32 21, i32 22, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 20, i32 21, i32 22, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 28, i32 29, i32 30, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6087,7 +6087,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_2
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,7,7,8,9,10,11,12,13,15,15]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 5, i32 7, i32 25, i32 undef, i32 undef, i32 undef, i32 9, i32 undef, i32 13, i32 15, i32 25>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 5, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 9, i32 poison, i32 13, i32 15, i32 25>
   ret <16 x i16> %shuffle
 }
 
@@ -6137,7 +6137,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_04_uu_16_18_20_uu_uu_uu_12_uu_24_26_28_u
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 4, i32 undef, i32 16, i32 18, i32 20, i32 undef, i32 undef, i32 undef, i32 12, i32 undef, i32 24, i32 26, i32 28, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 4, i32 poison, i32 16, i32 18, i32 20, i32 poison, i32 poison, i32 poison, i32 12, i32 poison, i32 24, i32 26, i32 28, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6215,7 +6215,7 @@ define <16 x i16> @shuffle_v16i16_uu_22_uu_uu_01_02_03_uu_uu_30_uu_uu_09_10_11_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9],ymm1[26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 22, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 30, i32 undef, i32 undef, i32 9, i32 10, i32 11, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 22, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 30, i32 poison, i32 poison, i32 9, i32 10, i32 11, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6287,7 +6287,7 @@ define <16 x i16> @shuffle_v16i16_uu_06_uu_uu_01_02_03_uu_uu_14_uu_uu_09_10_11_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 14, i32 undef, i32 undef, i32 9, i32 10, i32 11, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 14, i32 poison, i32 poison, i32 9, i32 10, i32 11, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6317,7 +6317,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_uu_01_02_03_uu_uu_uu_uu_uu_09_10_11_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9],zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 9, i32 10, i32 11, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 9, i32 10, i32 11, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6398,7 +6398,7 @@ define <16 x i16> @shuffle_v16i16_uu_20_21_22_uu_uu_01_uu_uu_28_29_30_uu_uu_09_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5],ymm1[22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 20, i32 21, i32 22, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 undef, i32 undef, i32 9, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 20, i32 21, i32 22, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 28, i32 29, i32 30, i32 poison, i32 poison, i32 9, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6470,7 +6470,7 @@ define <16 x i16> @shuffle_v16i16_uu_04_05_06_uu_uu_01_uu_uu_12_13_14_uu_uu_09_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 1, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 undef, i32 undef, i32 9, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 poison, i32 poison, i32 9, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6500,7 +6500,7 @@ define <16 x i16> @shuffle_v16i16_uu_04_05_06_uu_uu_uu_uu_uu_12_13_14_uu_uu_uu_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6580,7 +6580,7 @@ define <16 x i16> @shuffle_v16i16_uu_04_05_06_uu_uu_17_uu_uu_12_13_14_uu_uu_25_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5],ymm0[22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 17, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 undef, i32 undef, i32 25, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 17, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 poison, i32 poison, i32 25, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6697,7 +6697,7 @@ define <16 x i16> @shuffle_v16i16_uu_06_uu_uu_17_18_19_uu_uu_14_uu_uu_25_26_27_u
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9],ymm0[26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 14, i32 undef, i32 undef, i32 25, i32 26, i32 27, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 14, i32 poison, i32 poison, i32 25, i32 26, i32 27, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6743,7 +6743,7 @@ define <16 x i16> @shuffle_v16i16_23_uu_03_uu_20_20_05_uu_31_uu_11_uu_28_28_13_u
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8,9,10,11],ymm1[12],ymm0[13,14],ymm1[15]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[14,15,14,15,6,7,6,7,8,9,8,9,10,11,14,15,30,31,30,31,22,23,22,23,24,25,24,25,26,27,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 23, i32 undef, i32 3, i32 undef, i32 20, i32 20, i32 5, i32 undef, i32 31, i32 undef, i32 11, i32 undef, i32 28, i32 28, i32 13, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 23, i32 poison, i32 3, i32 poison, i32 20, i32 20, i32 5, i32 poison, i32 31, i32 poison, i32 11, i32 poison, i32 28, i32 28, i32 13, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6771,7 +6771,7 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_0_16_1_17_2_18_3_19(<16 x i16>
 ; XOPAVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19>
   ret <16 x i16> %shuffle
 }
 
@@ -6819,7 +6819,7 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a,
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <16 x i16> %shuffle
 }
 
@@ -6878,7 +6878,7 @@ define <16 x i16> @shuffle_v16i16_4_20_5_21_6_22_7_23_u_u_u_u_u_u_u_u(<16 x i16>
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6922,7 +6922,7 @@ define <16 x i16> @shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -6973,7 +6973,7 @@ define <16 x i16> @shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %shuffle
 }
 
@@ -7069,7 +7069,7 @@ define <16 x i16> @shuffle_v16i16_02_18_03_19_10_26_11_27_00_16_01_17_08_24_09_2
 ; XOPAVX2-NEXT:    retq
   %1 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 0, i32 16, i32 1, i32 17, i32 10, i32 26, i32 11, i32 27, i32 8, i32 24, i32 9, i32 25>
   %2 = bitcast <16 x i16> %1 to <4 x i64>
-  %3 = shufflevector <4 x i64> %2, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  %3 = shufflevector <4 x i64> %2, <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   %4 = bitcast <4 x i64> %3 to <16 x i16>
   ret <16 x i16> %4
 }
@@ -7216,7 +7216,7 @@ define <16 x i16> @shuffle_v16i16_04_06_07_uu_uu_06_07_05_12_14_15_uu_uu_14_15_1
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,7,5,8,9,10,11,12,14,15,13]
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 4, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7, i32 5, i32 12, i32 14, i32 15, i32 undef, i32 undef, i32 14, i32 15, i32 13>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 4, i32 6, i32 7, i32 poison, i32 poison, i32 6, i32 7, i32 5, i32 12, i32 14, i32 15, i32 poison, i32 poison, i32 14, i32 15, i32 13>
   ret <16 x i16> %shuffle
 }
 
@@ -7248,8 +7248,8 @@ define <16 x i16> @shuffle_v16i16_03_02_01_00_04_05_06_07_11_10_09_08_12_13_14_1
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
 ; XOPAVX2-NEXT:    retq
-  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %2 = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
+  %1 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
+  %2 = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %3
 }
@@ -7282,8 +7282,8 @@ define <16 x i16> @shuffle_v16i16_00_01_02_04_07_06_05_04_08_09_10_11_15_14_13_1
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
 ; XOPAVX2-NEXT:    retq
-  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
-  %2 = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
+  %1 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
+  %2 = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %3
 }
@@ -7328,7 +7328,7 @@ define <16 x i16> @shuffle_v16i16_01_00_03_02_05_04_07_06_09_08_11_10_13_12_15_1
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   ret <16 x i16> %shuffle
 }
 
@@ -7372,7 +7372,7 @@ define <16 x i16> @shuffle_v16i16_03_00_01_02_07_04_05_06_11_08_09_10_15_12_13_1
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,0,1,2,4,5,6,7,11,8,9,10,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14>
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14>
   ret <16 x i16> %shuffle
 }
 
@@ -7392,8 +7392,8 @@ define <16 x i16> @concat_v16i16_0_1_2_3_4_5_6_7_24_25_26_27_28_29_30_31(<16 x i
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; ALL-NEXT:    retq
-  %alo = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %bhi = shufflevector <16 x i16> %b, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %alo = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %bhi = shufflevector <16 x i16> %b, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %shuf = shufflevector <8 x i16> %alo, <8 x i16> %bhi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %shuf
 }
@@ -7413,8 +7413,8 @@ define <16 x i16> @concat_v16i16_8_9_10_11_12_13_14_15_24_25_26_27_28_29_30_31_b
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
 ; XOP-NEXT:    retq
-  %ahi = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %bhi = shufflevector <16 x i16> %b, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %ahi = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bhi = shufflevector <16 x i16> %b, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %bc0hi = bitcast <8 x i16> %ahi to <16 x i8>
   %bc1hi = bitcast <8 x i16> %bhi to <16 x i8>
   %shuffle8 = shufflevector <16 x i8> %bc0hi, <16 x i8> %bc1hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -7596,7 +7596,7 @@ define <16 x i16> @PR34369(<16 x i16> %vec, <16 x i16> %mask) {
 ; XOPAVX2-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
 ; XOPAVX2-NEXT:    retq
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 0, i32 0, i32 13, i32 5, i32 2, i32 2, i32 10, i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>
+  %shuf = shufflevector <16 x i16> %vec, <16 x i16> poison, <16 x i32> <i32 3, i32 0, i32 0, i32 13, i32 5, i32 2, i32 2, i32 10, i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>
   %cmp = icmp eq <16 x i16> %mask, zeroinitializer
   %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
   ret <16 x i16> %res
@@ -7631,7 +7631,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> zeroinitializer
   ret <16 x i16> %tmp3
 }
 
@@ -7667,7 +7667,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <16 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <16 x i32> zeroinitializer
   ret <16 x i16> %tmp4
 }
 
@@ -7700,7 +7700,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i32(ptr %ptr) #0 {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <16 x i16> %tmp3
 }
 
@@ -7733,7 +7733,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i32(ptr %ptr) #0 {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 1
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <16 x i16> %tmp3
 }
 
@@ -7766,7 +7766,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> zeroinitializer
   ret <16 x i16> %tmp3
 }
 
@@ -7799,7 +7799,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <16 x i16> %tmp3
 }
 
@@ -7832,7 +7832,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <16 x i16> %tmp3
 }
 
@@ -7863,7 +7863,7 @@ define <16 x i16> @insert_dup_elt7_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 1
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   ret <16 x i16> %tmp3
 }
 
@@ -7899,7 +7899,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16_i64(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i64
   %tmp2 = insertelement <2 x i64> zeroinitializer, i64 %tmp1, i32 0
   %tmp3 = bitcast <2 x i64> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <16 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <16 x i32> zeroinitializer
   ret <16 x i16> %tmp4
 }
 
@@ -7927,7 +7927,7 @@ define <16 x i16> @unpckh_v16i16(<16 x i16> %x, <16 x i16> %y) {
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; XOPAVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; XOPAVX2-NEXT:    retq
-  %unpckh = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 4, i32 28, i32 5, i32 29, i32 6, i32 30, i32 7, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %unpckh = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 4, i32 28, i32 5, i32 29, i32 6, i32 30, i32 7, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i16> %unpckh
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index e29848295eaab..4f42d5c655280 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -2407,7 +2407,7 @@ define <32 x i8> @shuffle_v32i8_01_zz_02_zz_04_uu_06_07_08_09_10_11_12_13_14_15_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1],zero,ymm0[2],zero,ymm0[4,u,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 2, i32 32, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 2, i32 32, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <32 x i8> %shuffle
 }
 
@@ -3290,7 +3290,7 @@ define <32 x i8> @shuffle_v32i8_00_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,16,16,16,u,u,u,u,u,16,16,16,16,16,30,16]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 16, i32 16, i32 16, i32 16, i32 30, i32 16>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 16, i32 16, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 16, i32 16, i32 16, i32 16, i32 30, i32 16>
   ret <32 x i8> %shuffle
 }
 
@@ -3320,7 +3320,7 @@ define <32 x i8> @shuffle_v32i8_uu_14_uu_uu_00_00_00_00_00_00_00_00_00_00_00_00_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,14,u,u,0,0,0,0,0,0,0,0,0,0,0,0,16,16,u,16,u,u,u,u,16,16,16,16,16,16,30,16]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 undef, i32 14, i32 undef, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 undef, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 30, i32 16>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 poison, i32 14, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 poison, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 30, i32 16>
   ret <32 x i8> %shuffle
 }
 
@@ -3350,7 +3350,7 @@ define <32 x i8> @shuffle_v32i8_00_00_00_uu_uu_uu_04_uu_08_08_08_08_uu_uu_12_uu_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,0,0,u,u,u,4,u,8,8,8,8,u,u,12,u,28,28,28,28,u,u,u,24,20,20,20,20,16,16,16,16]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 4, i32 undef, i32 8, i32 8, i32 8, i32 8, i32 undef, i32 undef, i32 12, i32 undef, i32 28, i32 28, i32 28, i32 28, i32 undef, i32 undef, i32 undef, i32 24, i32 20, i32 20, i32 20, i32 20, i32 16, i32 16, i32 16, i32 16>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 4, i32 poison, i32 8, i32 8, i32 8, i32 8, i32 poison, i32 poison, i32 12, i32 poison, i32 28, i32 28, i32 28, i32 28, i32 poison, i32 poison, i32 poison, i32 24, i32 20, i32 20, i32 20, i32 20, i32 16, i32 16, i32 16, i32 16>
   ret <32 x i8> %shuffle
 }
 
@@ -3382,7 +3382,7 @@ define <32 x i8> @shuffle_v32i8_08_08_08_08_08_08_08_08_uu_uu_uu_uu_uu_uu_uu_uu_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,8,8,8,8,8,8,8,u,u,u,u,u,u,u,u,16,16,16,u,u,u,u,u,u,u,24,24,24,24,24,24]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 16, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
   ret <32 x i8> %shuffle
 }
 
@@ -4189,7 +4189,7 @@ define <32 x i8> @shuffle_v32i8_uu_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 undef, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   ret <32 x i8> %shuffle
 }
 
@@ -4219,7 +4219,7 @@ define <32 x i8> @shuffle_v32i8_47_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 47, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 undef, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 47, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   ret <32 x i8> %shuffle
 }
 
@@ -4249,7 +4249,7 @@ define <32 x i8> @shuffle_v32i8_uu_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 undef, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i8> %shuffle
 }
 
@@ -4279,7 +4279,7 @@ define <32 x i8> @shuffle_v32i8_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[15],ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],ymm1[31],ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   ret <32 x i8> %shuffle
 }
 
@@ -4502,7 +4502,7 @@ define <32 x i8> @shuffle_v32i8_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
   ret <32 x i8> %shuffle
 }
 
@@ -4599,7 +4599,7 @@ define <32 x i8> @shuffle_v32i8_15_15_15_15_15_15_15_15_32_32_32_32_32_32_32_32_
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[15,15,15,15,15,15,15,15],xmm1[0,0,0,0,0,0,0,0]
 ; AVX-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i8> %shuffle
 }
 
@@ -4608,7 +4608,7 @@ define <32 x i8> @shuffle_v32i8_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i8> %shuffle
 }
 
@@ -4636,7 +4636,7 @@ define <32 x i8> @shuffle_v32i8_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i8> %shuffle
 }
 
@@ -4720,7 +4720,7 @@ define <32 x i8> @shuffle_v32i8_03_00_01_02_07_04_05_06_11_08_09_10_15_12_13_14_
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,0,1,2,7,4,5,6,11,8,9,10,15,12,13,14,19,16,17,18,23,20,21,22,27,24,25,26,31,28,29,30]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14, i32 19, i32 16, i32 17, i32 18, i32 23, i32 20, i32 21, i32 22, i32 27, i32 24, i32 25, i32 26, i32 31, i32 28, i32 29, i32 30>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14, i32 19, i32 16, i32 17, i32 18, i32 23, i32 20, i32 21, i32 22, i32 27, i32 24, i32 25, i32 26, i32 31, i32 28, i32 29, i32 30>
   ret <32 x i8> %shuffle
 }
 
@@ -4765,7 +4765,7 @@ define <32 x i8> @shuffle_v32i8_02_03_04_05_06_07_00_01_10_11_12_13_14_15_08_09_
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,2,3,0,4,5,6,7,9,10,11,8,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,4,8,9,10,11,13,14,15,12]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25>
   ret <32 x i8> %shuffle
 }
 
@@ -4819,7 +4819,7 @@ define <32 x i8> @shuffle_v32i8_30_31_30_31_30_31_30_31_30_31_30_31_30_31_30_31_
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; XOPAVX2-NEXT:    retq
-  %shuffle = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31>
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31, i32 30, i32 31>
   ret <32 x i8> %shuffle
 }
 
@@ -5029,7 +5029,7 @@ define <4 x i64> @PR28136(<32 x i8> %a0, <32 x i8> %a1) {
 ; XOPAVX2-NEXT:    retq
   %1 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50,i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   %2 = bitcast <32 x i8> %1 to <4 x i64>
-  %3 = shufflevector <4 x i64> %2, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  %3 = shufflevector <4 x i64> %2, <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x i64> %3
 }
 
@@ -5076,9 +5076,9 @@ define <32 x i8> @PR47262(<4 x i64> %a0) {
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,4,8,12,1,5,9,13,2,6,10,14,3,7,11,15,16,20,24,28,17,21,25,29,18,22,26,30,19,23,27,31]
 ; XOPAVX2-NEXT:    retq
-  %t1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  %t1 = shufflevector <4 x i64> %a0, <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   %t2 = bitcast <4 x i64> %t1 to <32 x i8>
-  %t3 = shufflevector <32 x i8> %t2, <32 x i8> undef, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15, i32 16, i32 20, i32 24, i32 28, i32 17, i32 21, i32 25, i32 29, i32 18, i32 22, i32 26, i32 30, i32 19, i32 23, i32 27, i32 31>
+  %t3 = shufflevector <32 x i8> %t2, <32 x i8> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15, i32 16, i32 20, i32 24, i32 28, i32 17, i32 21, i32 25, i32 29, i32 18, i32 22, i32 26, i32 30, i32 19, i32 23, i32 27, i32 31>
     ret <32 x i8> %t3
 }
 
@@ -5163,7 +5163,7 @@ define <4 x i64> @PR66150(ptr %b) {
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19]
 ; XOPAVX2-NEXT:    retq
   %tmp1 = load i32, ptr %b, align 4
-  %tmp2 = insertelement <8 x i32> undef, i32 %tmp1, i64 0
+  %tmp2 = insertelement <8 x i32> poison, i32 %tmp1, i64 0
   %tmp3 = shufflevector <8 x i32> %tmp2, <8 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison>
   %tmp4 = bitcast <8 x i32> %tmp3 to <32 x i8>
   %tmp5 = shufflevector <32 x i8> %tmp4, <32 x i8> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 18, i32 18, i32 18, i32 18, i32 18, i32 18, i32 18, i32 18, i32 19, i32 19, i32 19, i32 19, i32 19, i32 19, i32 19, i32 19>
@@ -5267,7 +5267,7 @@ define <32 x i8> @insert_dup_mem_v32i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> zeroinitializer
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <32 x i32> zeroinitializer
   ret <32 x i8> %tmp3
 }
 
@@ -5303,7 +5303,7 @@ define <32 x i8> @insert_dup_mem_v32i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <32 x i32> zeroinitializer
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <32 x i32> zeroinitializer
   ret <32 x i8> %tmp4
 }
 
@@ -5334,7 +5334,7 @@ define <32 x i8> @insert_dup_elt1_mem_v32i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <32 x i8> %tmp3
 }
 
@@ -5365,7 +5365,7 @@ define <32 x i8> @insert_dup_elt3_mem_v32i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <32 x i8> %tmp3
 }
 
@@ -5412,7 +5412,7 @@ define <32 x i8> @insert_dup_elt1_mem_v32i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <32 x i8> %tmp4
 }
 
@@ -5442,8 +5442,8 @@ define <32 x i8> @zeroable_src_to_zext(<32 x i8> %a0) {
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; XOPAVX2-NEXT:    retq
-  %1 = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %2 = shufflevector <32 x i8> %1, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 8, i32 9, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 20, i32 21, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+  %1 = shufflevector <32 x i8> %a0, <32 x i8> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = shufflevector <32 x i8> %1, <32 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 8, i32 9, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 20, i32 21, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
   ret <32 x i8> %2
 }
 
@@ -5471,7 +5471,7 @@ define <32 x i8> @unpckh_v32i8(<32 x i8> %x, <32 x i8> %y) {
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; XOPAVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
 ; XOPAVX2-NEXT:    retq
-  %unpckh = shufflevector <32 x i8> %x, <32 x i8> %y, <32 x i32> <i32 8, i32 56, i32 9, i32 57, i32 10, i32 58, i32 11, i32 59, i32 12, i32 60, i32 13, i32 61, i32 14, i32 62, i32 15, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %unpckh = shufflevector <32 x i8> %x, <32 x i8> %y, <32 x i32> <i32 8, i32 56, i32 9, i32 57, i32 10, i32 58, i32 11, i32 59, i32 12, i32 60, i32 13, i32 61, i32 14, i32 62, i32 15, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i8> %unpckh
 }
 
@@ -5501,6 +5501,6 @@ define <32 x i8> @shuffle_v16i16_zextinreg_to_v8i32(<32 x i8> %a)  {
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; XOPAVX2-NEXT:    retq
-  %b = shufflevector <32 x i8> %a, <32 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 42, i32 42, i32 2, i32 3, i32 42, i32 42, i32 4, i32 5, i32 42, i32 42, i32 6, i32 7, i32 42, i32 42, i32 8, i32 9, i32 42, i32 42, i32 10, i32 11, i32 42, i32 42, i32 12, i32 13, i32 42, i32 42, i32 14, i32 15, i32 42, i32 42>
+  %b = shufflevector <32 x i8> %a, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 0, i32 1, i32 42, i32 42, i32 2, i32 3, i32 42, i32 42, i32 4, i32 5, i32 42, i32 42, i32 6, i32 7, i32 42, i32 42, i32 8, i32 9, i32 42, i32 42, i32 10, i32 11, i32 42, i32 42, i32 12, i32 13, i32 42, i32 42, i32 14, i32 15, i32 42, i32 42>
   ret <32 x i8> %b
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 779ae33a27715..919869df6c262 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -598,7 +598,7 @@ define <4 x double> @shuffle_v4f64_u062(<4 x double> %a, <4 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 undef, i32 0, i32 6, i32 2>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 poison, i32 0, i32 6, i32 2>
   ret <4 x double> %shuffle
 }
 
@@ -607,7 +607,7 @@ define <4 x double> @shuffle_v4f64_15uu(<4 x double> %a, <4 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
   ret <4 x double> %shuffle
 }
 
@@ -616,7 +616,7 @@ define <4 x double> @shuffle_v4f64_11uu(<4 x double> %a, <4 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1,1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
   ret <4 x double> %shuffle
 }
 
@@ -636,7 +636,7 @@ define <4 x double> @shuffle_v4f64_22uu(<4 x double> %a, <4 x double> %b) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 2, i32 2, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 2, i32 2, i32 poison, i32 poison>
   ret <4 x double> %shuffle
 }
 
@@ -727,7 +727,7 @@ define <4 x double> @shuffle_v4f64_0z3z(<4 x double> %a, <4 x double> %b) {
 ; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 0, i32 4, i32 3, i32 4>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double poison, double poison, double poison>, <4 x i32> <i32 0, i32 4, i32 3, i32 4>
   ret <4 x double> %shuffle
 }
 
@@ -737,7 +737,7 @@ define <4 x double> @shuffle_v4f64_1z2z(<4 x double> %a, <4 x double> %b) {
 ; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[2],ymm1[3]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
+  %1 = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double poison, double poison, double poison>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
   ret <4 x double> %1
 }
 
@@ -782,8 +782,8 @@ define <4 x double> @shuffle_v4f64_0044_v2f64(<2 x double> %a, <2 x double> %b)
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> <i32 0, i32 0>
-  %2 = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %1 = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 0, i32 0>
+  %2 = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> <i32 0, i32 0>
   %3 = shufflevector <2 x double> %1, <2 x double> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x double> %3
 }
@@ -795,8 +795,8 @@ define <4 x double> @shuffle_v4f64_1032_v2f64(<2 x double> %a, <2 x double> %b)
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-  %2 = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+  %1 = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 1, i32 0>
+  %2 = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   %3 = shufflevector <2 x double> %1, <2 x double> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x double> %3
 }
@@ -1228,7 +1228,7 @@ define <4 x i64> @shuffle_v4i64_2u35(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512VL-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512VL-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,1]
 ; AVX512VL-FAST-PERLANE-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 undef, i32 3, i32 5>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 poison, i32 3, i32 5>
   ret <4 x i64> %shuffle
 }
 
@@ -1504,7 +1504,7 @@ define <4 x i64> @shuffle_v4i64_5zuz(<4 x i64> %a) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> zeroinitializer, <4 x i64> %a, <4 x i32> <i32 5, i32 0, i32 undef, i32 0>
+  %shuffle = shufflevector <4 x i64> zeroinitializer, <4 x i64> %a, <4 x i32> <i32 5, i32 0, i32 poison, i32 0>
   ret <4 x i64> %shuffle
 }
 
@@ -1513,7 +1513,7 @@ define <4 x i64> @shuffle_v4i64_40u2(<4 x i64> %a, <4 x i64> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 4, i32 0, i32 undef, i32 2>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 4, i32 0, i32 poison, i32 2>
   ret <4 x i64> %shuffle
 }
 
@@ -1522,7 +1522,7 @@ define <4 x i64> @shuffle_v4i64_15uu(<4 x i64> %a, <4 x i64> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
   ret <4 x i64> %shuffle
 }
 
@@ -1531,7 +1531,7 @@ define <4 x i64> @shuffle_v4i64_11uu(<4 x i64> %a, <4 x i64> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
   ret <4 x i64> %shuffle
 }
 
@@ -1551,7 +1551,7 @@ define <4 x i64> @shuffle_v4i64_22uu(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 2, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 2, i32 poison, i32 poison>
   ret <4 x i64> %shuffle
 }
 
@@ -1591,7 +1591,7 @@ define <4 x i64> @shuffle_v4i64_1z3z(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i32> <i32 1, i32 4, i32 3, i32 4>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 1, i32 4, i32 3, i32 4>
   ret <4 x i64> %shuffle
 }
 
@@ -1616,8 +1616,8 @@ define <4 x i64> @shuffle_v4i64_0044_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,1,4,5,4,5]
 ; AVX512VL-NEXT:    retq
-  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
-  %2 = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %1 = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 0, i32 0>
+  %2 = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> <i32 0, i32 0>
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x i64> %3
 }
@@ -1629,8 +1629,8 @@ define <4 x i64> @shuffle_v4i64_1032_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-  %2 = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+  %1 = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+  %2 = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x i64> %3
 }
@@ -1638,9 +1638,9 @@ define <4 x i64> @shuffle_v4i64_1032_v2i64(<2 x i64> %a, <2 x i64> %b) {
 define <4 x i64> @stress_test1(<4 x i64> %a, <4 x i64> %b) {
 ; ALL-LABEL: stress_test1:
 ; ALL:         retq
-  %c = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> <i32 3, i32 1, i32 1, i32 0>
-  %d = shufflevector <4 x i64> %c, <4 x i64> undef, <4 x i32> <i32 3, i32 undef, i32 2, i32 undef>
-  %e = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> <i32 3, i32 3, i32 1, i32 undef>
+  %c = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> <i32 3, i32 1, i32 1, i32 0>
+  %d = shufflevector <4 x i64> %c, <4 x i64> poison, <4 x i32> <i32 3, i32 poison, i32 2, i32 poison>
+  %e = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> <i32 3, i32 3, i32 1, i32 poison>
   %f = shufflevector <4 x i64> %d, <4 x i64> %e, <4 x i32> <i32 5, i32 1, i32 1, i32 0>
 
   ret <4 x i64> %f
@@ -1651,7 +1651,7 @@ define <4 x i64> @insert_reg_and_zero_v4i64(i64 %a) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovq %rdi, %xmm0
 ; ALL-NEXT:    retq
-  %v = insertelement <4 x i64> undef, i64 %a, i64 0
+  %v = insertelement <4 x i64> poison, i64 %a, i64 0
   %shuffle = shufflevector <4 x i64> %v, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i64> %shuffle
 }
@@ -1662,7 +1662,7 @@ define <4 x i64> @insert_mem_and_zero_v4i64(ptr %ptr) {
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
   %a = load i64, ptr %ptr
-  %v = insertelement <4 x i64> undef, i64 %a, i64 0
+  %v = insertelement <4 x i64> poison, i64 %a, i64 0
   %shuffle = shufflevector <4 x i64> %v, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i64> %shuffle
 }
@@ -1672,7 +1672,7 @@ define <4 x double> @insert_reg_and_zero_v4f64(double %a) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; ALL-NEXT:    retq
-  %v = insertelement <4 x double> undef, double %a, i32 0
+  %v = insertelement <4 x double> poison, double %a, i32 0
   %shuffle = shufflevector <4 x double> %v, <4 x double> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x double> %shuffle
 }
@@ -1683,7 +1683,7 @@ define <4 x double> @insert_mem_and_zero_v4f64(ptr %ptr) {
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <4 x double> undef, double %a, i32 0
+  %v = insertelement <4 x double> poison, double %a, i32 0
   %shuffle = shufflevector <4 x double> %v, <4 x double> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x double> %shuffle
 }
@@ -1694,8 +1694,8 @@ define <4 x double> @splat_mem_v4f64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %a = load double, ptr %ptr
-  %v = insertelement <4 x double> undef, double %a, i32 0
-  %shuffle = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %v = insertelement <4 x double> poison, double %a, i32 0
+  %shuffle = shufflevector <4 x double> %v, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
   ret <4 x double> %shuffle
 }
 
@@ -1705,8 +1705,8 @@ define <4 x i64> @splat_mem_v4i64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %a = load i64, ptr %ptr
-  %v = insertelement <4 x i64> undef, i64 %a, i64 0
-  %shuffle = shufflevector <4 x i64> %v, <4 x i64> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %v = insertelement <4 x i64> poison, i64 %a, i64 0
+  %shuffle = shufflevector <4 x i64> %v, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
   ret <4 x i64> %shuffle
 }
 
@@ -1716,8 +1716,8 @@ define <4 x double> @splat_mem_v4f64_2(ptr %p) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %1 = load double, ptr %p
-  %2 = insertelement <2 x double> undef, double %1, i32 0
-  %3 = shufflevector <2 x double> %2, <2 x double> undef, <4 x i32> zeroinitializer
+  %2 = insertelement <2 x double> poison, double %1, i32 0
+  %3 = shufflevector <2 x double> %2, <2 x double> poison, <4 x i32> zeroinitializer
   ret <4 x double> %3
 }
 
@@ -1737,7 +1737,7 @@ define <4 x double> @splat_v4f64(<2 x double> %r) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; AVX512VL-NEXT:    retq
-  %1 = shufflevector <2 x double> %r, <2 x double> undef, <4 x i32> zeroinitializer
+  %1 = shufflevector <2 x double> %r, <2 x double> poison, <4 x i32> zeroinitializer
   ret <4 x double> %1
 }
 
@@ -1747,7 +1747,7 @@ define <4 x i64> @splat_mem_v4i64_from_v2i64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %v = load <2 x i64>, ptr %ptr
-  %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %shuffle = shufflevector <2 x i64> %v, <2 x i64> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
   ret <4 x i64> %shuffle
 }
 
@@ -1757,7 +1757,7 @@ define <4 x double> @splat_mem_v4f64_from_v2f64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %v = load <2 x double>, ptr %ptr
-  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %shuffle = shufflevector <2 x double> %v, <2 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
   ret <4 x double> %shuffle
 }
 
@@ -1772,7 +1772,7 @@ define <4 x i64> @splat128_mem_v4i64_from_v2i64(ptr %ptr) {
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
 ; AVX512VL-NEXT:    retq
   %v = load <2 x i64>, ptr %ptr
-  %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuffle = shufflevector <2 x i64> %v, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   ret <4 x i64> %shuffle
 }
 
@@ -1782,7 +1782,7 @@ define <4 x double> @splat128_mem_v4f64_from_v2f64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
 ; ALL-NEXT:    retq
   %v = load <2 x double>, ptr %ptr
-  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuffle = shufflevector <2 x double> %v, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   ret <4 x double> %shuffle
 }
 
@@ -1802,9 +1802,9 @@ define <4 x double> @broadcast_v4f64_0000_from_v2i64(<2 x i64> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; AVX512VL-NEXT:    retq
-  %1 = shufflevector <2 x i64> %a0, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %1 = shufflevector <2 x i64> %a0, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = bitcast <4 x i64> %1 to <4 x double>
-  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> zeroinitializer
+  %3 = shufflevector <4 x double> %2, <4 x double> poison, <4 x i32> zeroinitializer
   ret <4 x double> %3
 }
 
@@ -1830,9 +1830,9 @@ define <4 x double> @bitcast_v4f64_0426(<4 x double> %a, <4 x double> %b) {
 ; ALL-NEXT:    retq
   %shuffle64 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
   %bitcast32 = bitcast <4 x double> %shuffle64 to <8 x float>
-  %shuffle32 = shufflevector <8 x float> %bitcast32, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+  %shuffle32 = shufflevector <8 x float> %bitcast32, <8 x float> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   %bitcast16 = bitcast <8 x float> %shuffle32 to <16 x i16>
-  %shuffle16 = shufflevector <16 x i16> %bitcast16, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
+  %shuffle16 = shufflevector <16 x i16> %bitcast16, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
   %bitcast64 = bitcast <16 x i16> %shuffle16 to <4 x double>
   ret <4 x double> %bitcast64
 }
@@ -1868,8 +1868,8 @@ define <4 x i64> @insert_dup_mem_v4i64(ptr %ptr) {
 ; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %tmp = load i64, ptr %ptr, align 1
-  %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
-  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <4 x i32> zeroinitializer
+  %tmp1 = insertelement <2 x i64> poison, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <4 x i32> zeroinitializer
   ret <4 x i64> %tmp2
 }
 
@@ -1910,7 +1910,7 @@ define <4 x i64> @shuffle_v4i64_1230(<4 x i64> %a) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,2,3,0]
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
   ret <4 x i64> %shuffle
 }
 
@@ -1944,7 +1944,7 @@ define <4 x i64> @shuffle_v4i64_z0z3(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512VL-FAST:       # %bb.0:
 ; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31]
 ; AVX512VL-FAST-NEXT:    retq
-  %1 = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i32> <i32 4, i32 0, i32 4, i32 3>
+  %1 = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 0, i32 4, i32 3>
   ret <4 x i64> %1
 }
 
@@ -1978,7 +1978,7 @@ define <4 x i64> @shuffle_v4i64_1z2z(<4 x i64> %a, <4 x i64> %b) {
 ; AVX512VL-FAST:       # %bb.0:
 ; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-FAST-NEXT:    retq
-  %1 = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
+  %1 = shufflevector <4 x i64> %a, <4 x i64> <i64 0, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
   ret <4 x i64> %1
 }
 
@@ -2052,10 +2052,10 @@ define <4 x double> @add_v4f64_024u_135u_reverse(<4 x double> %a, <4 x double> %
 ; AVX512VL-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,3,1]
 ; AVX512VL-NEXT:    retq
-  %shuffle0 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 undef>
-  %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 undef>
+  %shuffle0 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 poison>
+  %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 poison>
   %add = fadd <4 x double> %shuffle0, %shuffle1
-  %shuffle = shufflevector <4 x double> %add, <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+  %shuffle = shufflevector <4 x double> %add, <4 x double> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
   ret <4 x double> %shuffle
 }
 
@@ -2265,7 +2265,7 @@ define <4 x i64> @unpckh_v4i64(<4 x i64> %x, <4 x i64> %y) {
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; ALL-NEXT:    retq
-  %unpckh = shufflevector <4 x i64> %x, <4 x i64> %y, <4 x i32> <i32 1, i32 7, i32 undef, i32 undef>
+  %unpckh = shufflevector <4 x i64> %x, <4 x i64> %y, <4 x i32> <i32 1, i32 7, i32 poison, i32 poison>
   ret <4 x i64> %unpckh
 }
 
@@ -2275,7 +2275,7 @@ define <4 x double> @unpckh_v4f64(<4 x double> %x, <4 x double> %y) {
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; ALL-NEXT:    retq
-  %unpckh = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 1, i32 7, i32 undef, i32 undef>
+  %unpckh = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 1, i32 7, i32 poison, i32 poison>
   ret <4 x double> %unpckh
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index fde3e0aecf5aa..b815cef63bfd4 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -831,7 +831,7 @@ define <8 x float> @shuffle_v8f32_002u6u44(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,2,u,6,u,4,4]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 poison, i32 6, i32 poison, i32 4, i32 4>
   ret <8 x float> %shuffle
 }
 
@@ -840,7 +840,7 @@ define <8 x float> @shuffle_v8f32_00uu66uu(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,u,u,6,6,u,u]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 6, i32 6, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -849,7 +849,7 @@ define <8 x float> @shuffle_v8f32_103245uu(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,4,5,u,u]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -858,7 +858,7 @@ define <8 x float> @shuffle_v8f32_1133uu67(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,1,3,3,u,u,6,7]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 poison, i32 poison, i32 6, i32 7>
   ret <8 x float> %shuffle
 }
 
@@ -867,7 +867,7 @@ define <8 x float> @shuffle_v8f32_0uu354uu(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,u,u,3,5,4,u,u]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 5, i32 4, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -876,7 +876,7 @@ define <8 x float> @shuffle_v8f32_uuu3uu66(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[u,u,u,3,u,u,6,6]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 6>
   ret <8 x float> %shuffle
 }
 
@@ -1320,7 +1320,7 @@ define <8 x float> @shuffle_v8f32_80u1c4u5(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 8, i32 0, i32 undef, i32 1, i32 12, i32 4, i32 undef, i32 5>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 8, i32 0, i32 poison, i32 1, i32 12, i32 4, i32 poison, i32 5>
   ret <8 x float> %shuffle
 }
 
@@ -1329,7 +1329,7 @@ define <8 x float> @shuffle_v8f32_a2u3e6f7(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 10, i32 2, i32 undef, i32 3, i32 14, i32 6, i32 15, i32 7>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 10, i32 2, i32 poison, i32 3, i32 14, i32 6, i32 15, i32 7>
   ret <8 x float> %shuffle
 }
 
@@ -1381,7 +1381,7 @@ define <8 x float> @shuffle_v8f32_01452367(<8 x float> %a) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
   ret <8 x float> %shuffle
 }
 
@@ -1473,7 +1473,7 @@ define <8 x float> @shuffle_v8f32_uuuu1111(<8 x float> %a, <8 x float> %b) {
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 1, i32 1, i32 1>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 1>
   ret <8 x float> %shuffle
 }
 
@@ -1528,7 +1528,7 @@ define <8 x float> @shuffle_v8f32_1188uuuu(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,0]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 8, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 8, i32 8, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -1538,7 +1538,7 @@ define <8 x float> @shuffle_v8f32_uuuu3210(<8 x float> %a, <8 x float> %b) {
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,2,1,0]
 ; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 2, i32 1, i32 0>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 3, i32 2, i32 1, i32 0>
   ret <8 x float> %shuffle
 }
 
@@ -1548,7 +1548,7 @@ define <8 x float> @shuffle_v8f32_uuuu1188(<8 x float> %a, <8 x float> %b) {
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,0]
 ; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 1, i32 8, i32 8>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 1, i32 8, i32 8>
   ret <8 x float> %shuffle
 }
 
@@ -1557,7 +1557,7 @@ define <8 x float> @shuffle_v8f32_1111uuuu(<8 x float> %a, <8 x float> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -1567,7 +1567,7 @@ define <8 x float> @shuffle_v8f32_5555uuuu(<8 x float> %a, <8 x float> %b) {
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %shuffle
 }
 
@@ -1578,8 +1578,8 @@ define <8 x float> @shuffle_v8f32_32107654_v4f32(<4 x float> %a, <4 x float> %b)
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %2 = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %1 = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %2 = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = shufflevector <4 x float> %1, <4 x float> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x float> %3
 }
@@ -2529,7 +2529,7 @@ define <8 x i32> @shuffle_v8i32_002u6u44(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [0,0,2,u,6,u,4,4]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 poison, i32 6, i32 poison, i32 4, i32 4>
   ret <8 x i32> %shuffle
 }
 
@@ -2544,7 +2544,7 @@ define <8 x i32> @shuffle_v8i32_00uu66uu(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [0,0,u,u,6,6,u,u]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 6, i32 6, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -2559,7 +2559,7 @@ define <8 x i32> @shuffle_v8i32_103245uu(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [1,0,3,2,4,5,u,u]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -2574,7 +2574,7 @@ define <8 x i32> @shuffle_v8i32_1133uu67(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [1,1,3,3,u,u,6,7]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 poison, i32 poison, i32 6, i32 7>
   ret <8 x i32> %shuffle
 }
 
@@ -2589,7 +2589,7 @@ define <8 x i32> @shuffle_v8i32_0uu354uu(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [0,u,u,3,5,4,u,u]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 5, i32 4, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -2604,7 +2604,7 @@ define <8 x i32> @shuffle_v8i32_uuu3uu66(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2OR512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [u,u,u,3,u,u,6,6]
 ; AVX2OR512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 6>
   ret <8 x i32> %shuffle
 }
 
@@ -3069,7 +3069,7 @@ define <8 x i32> @shuffle_v8i32_zuu8zuuc(<8 x i32> %a) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19]
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> zeroinitializer, <8 x i32> %a, <8 x i32> <i32 0, i32 undef, i32 undef, i32 8, i32 0, i32 undef, i32 undef, i32 12>
+  %shuffle = shufflevector <8 x i32> zeroinitializer, <8 x i32> %a, <8 x i32> <i32 0, i32 poison, i32 poison, i32 8, i32 0, i32 poison, i32 poison, i32 12>
   ret <8 x i32> %shuffle
 }
 
@@ -3085,7 +3085,7 @@ define <8 x i32> @shuffle_v8i32_9ubzdefz(<8 x i32> %a) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,ymm0[20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> zeroinitializer, <8 x i32> %a, <8 x i32> <i32 9, i32 undef, i32 11, i32 0, i32 13, i32 14, i32 15, i32 0>
+  %shuffle = shufflevector <8 x i32> zeroinitializer, <8 x i32> %a, <8 x i32> <i32 9, i32 poison, i32 11, i32 0, i32 13, i32 14, i32 15, i32 0>
   ret <8 x i32> %shuffle
 }
 
@@ -3094,7 +3094,7 @@ define <8 x i32> @shuffle_v8i32_80u1b4uu(<8 x i32> %a, <8 x i32> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 8, i32 0, i32 undef, i32 1, i32 12, i32 4, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 8, i32 0, i32 poison, i32 1, i32 12, i32 4, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3104,7 +3104,7 @@ define <8 x i32> @shuffle_v8i32_uuuu1111(<8 x i32> %a, <8 x i32> %b) {
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 1, i32 1, i32 1>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i32> %shuffle
 }
 
@@ -3113,7 +3113,7 @@ define <8 x i32> @shuffle_v8i32_2222uuuu(<8 x i32> %a, <8 x i32> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3122,7 +3122,7 @@ define <8 x i32> @shuffle_v8i32_2A3Buuuu(<8 x i32> %a, <8 x i32> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3226,7 +3226,7 @@ define <8 x i32> @shuffle_v8i32_5555uuuu(<8 x i32> %a, <8 x i32> %b) {
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3271,7 +3271,7 @@ define <8 x i32> @shuffle_v8i32_uuuuuu7u(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3282,8 +3282,8 @@ define <8 x i32> @shuffle_v8i32_32107654_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
 ; ALL-NEXT:    retq
-  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %2 = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %1 = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %2 = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i32> %3
 }
@@ -3305,8 +3305,8 @@ define <8 x float> @splat_mem_v8f32_2(ptr %p) {
 ; ALL-NEXT:    vbroadcastss (%rdi), %ymm0
 ; ALL-NEXT:    retq
   %1 = load float, ptr %p
-  %2 = insertelement <4 x float> undef, float %1, i32 0
-  %3 = shufflevector <4 x float> %2, <4 x float> undef, <8 x i32> zeroinitializer
+  %2 = insertelement <4 x float> poison, float %1, i32 0
+  %3 = shufflevector <4 x float> %2, <4 x float> poison, <8 x i32> zeroinitializer
   ret <8 x float> %3
 }
 
@@ -3321,7 +3321,7 @@ define <8 x float> @splat_v8f32(<4 x float> %r) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vbroadcastss %xmm0, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %1 = shufflevector <4 x float> %r, <4 x float> undef, <8 x i32> zeroinitializer
+  %1 = shufflevector <4 x float> %r, <4 x float> poison, <8 x i32> zeroinitializer
   ret <8 x float> %1
 }
 
@@ -3341,7 +3341,7 @@ define <8 x i32> @shuffle_v8i32_z0U2zUz6(<8 x i32> %a) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpsllq $32, %ymm0, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 undef, i32 2, i32 8, i32 undef, i32 8, i32 6>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 poison, i32 2, i32 8, i32 poison, i32 8, i32 6>
   ret <8 x i32> %shuffle
 }
 
@@ -3357,7 +3357,7 @@ define <8 x i32> @shuffle_v8i32_1U3z5zUU(<8 x i32> %a) {
 ; AVX2OR512VL:       # %bb.0:
 ; AVX2OR512VL-NEXT:    vpsrlq $32, %ymm0, %ymm0
 ; AVX2OR512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 undef, i32 3, i32 8, i32 5, i32 8, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 poison, i32 3, i32 8, i32 5, i32 8, i32 poison, i32 poison>
   ret <8 x i32> %shuffle
 }
 
@@ -3448,9 +3448,9 @@ define <8x float> @concat_v2f32_1(ptr %tmp64, ptr %tmp65) {
 entry:
   %tmp74 = load <2 x float>, ptr %tmp65, align 8
   %tmp72 = load <2 x float>, ptr %tmp64, align 8
-  %tmp73 = shufflevector <2 x float> %tmp72, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %tmp75 = shufflevector <2 x float> %tmp74, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %tmp76 = shufflevector <8 x float> %tmp73, <8 x float> %tmp75, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp73 = shufflevector <2 x float> %tmp72, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %tmp75 = shufflevector <2 x float> %tmp74, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %tmp76 = shufflevector <8 x float> %tmp73, <8 x float> %tmp75, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %tmp76
 }
 
@@ -3463,7 +3463,7 @@ define <8x float> @concat_v2f32_2(ptr %tmp64, ptr %tmp65) {
 entry:
   %tmp74 = load <2 x float>, ptr %tmp65, align 8
   %tmp72 = load <2 x float>, ptr %tmp64, align 8
-  %tmp76 = shufflevector <2 x float> %tmp72, <2 x float> %tmp74, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp76 = shufflevector <2 x float> %tmp72, <2 x float> %tmp74, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %tmp76
 }
 
@@ -3477,7 +3477,7 @@ entry:
   %tmp74 = load <2 x float>, ptr %tmp65, align 8
   %tmp72 = load <2 x float>, ptr %tmp64, align 8
   %tmp76 = shufflevector <2 x float> %tmp72, <2 x float> %tmp74, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %res = shufflevector <4 x float> %tmp76, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %res = shufflevector <4 x float> %tmp76, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %res
 }
 
@@ -3487,7 +3487,7 @@ define <8 x i32> @insert_mem_and_zero_v8i32(ptr %ptr) {
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ALL-NEXT:    retq
   %a = load i32, ptr %ptr
-  %v = insertelement <8 x i32> undef, i32 %a, i32 0
+  %v = insertelement <8 x i32> poison, i32 %a, i32 0
   %shuffle = shufflevector <8 x i32> %v, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i32> %shuffle
 }
@@ -3497,8 +3497,8 @@ define <8 x i32> @concat_v8i32_0123CDEF(<8 x i32> %a, <8 x i32> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; ALL-NEXT:    retq
-  %alo = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %bhi = shufflevector <8 x i32> %b, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %alo = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bhi = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuf = shufflevector <4 x i32> %alo, <4 x i32> %bhi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i32> %shuf
 }
@@ -3529,8 +3529,8 @@ define <8 x float> @concat_v8f32_4567CDEF_bc(<8 x float> %f0, <8 x float> %f1) {
 ; ALL-NEXT:    retq
   %a0 = bitcast <8 x float> %f0 to <4 x i64>
   %a1 = bitcast <8 x float> %f1 to <8 x i32>
-  %a0hi = shufflevector <4 x i64> %a0, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
-  %a1hi = shufflevector <8 x i32> %a1, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %a0hi = shufflevector <4 x i64> %a0, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+  %a1hi = shufflevector <8 x i32> %a1, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %bc0hi = bitcast <2 x i64> %a0hi to <2 x i64>
   %bc1hi = bitcast <4 x i32> %a1hi to <2 x i64>
   %shuffle64 = shufflevector <2 x i64> %bc0hi, <2 x i64> %bc1hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3545,7 +3545,7 @@ define <8 x i32> @insert_dup_mem_v8i32(ptr %ptr) {
 ; ALL-NEXT:    retq
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
-  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <8 x i32> zeroinitializer
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> poison, <8 x i32> zeroinitializer
   ret <8 x i32> %tmp2
 }
 
@@ -3589,7 +3589,7 @@ define <8 x i32> @shuffle_v8i32_12345670(<8 x i32> %a) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    valignd {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,0]
 ; AVX512VL-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   ret <8 x i32> %shuffle
 }
 
@@ -3723,7 +3723,7 @@ entry:
   %tmp1 = extractelement <4 x i64> %bc, i32 3
   %tmp2 = bitcast i64 %tmp1 to <2 x float>
   %tmp4 = extractelement <2 x float> %tmp2, i32 1
-  %tmp5 = insertelement <8 x float> undef, float %tmp4, i32 4
+  %tmp5 = insertelement <8 x float> poison, float %tmp4, i32 4
   %tmp6 = insertelement <8 x float> %tmp5, float %z, i32 5
   ret <8 x float> %tmp6
 }
@@ -3736,7 +3736,7 @@ define <8 x i32> @unpckh_v8i32(<8 x i32> %x, <8 x i32> %y) {
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
-  %unpckh = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  %unpckh = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %unpckh
 }
 
@@ -3748,7 +3748,7 @@ define <8 x float> @unpckh_v8f32(<8 x float> %x, <8 x float> %y) {
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
-  %unpckh = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  %unpckh = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %unpckh
 }
 
@@ -3761,7 +3761,7 @@ define <8 x i32> @blend_perm_v8i32(<8 x i32> %x, <8 x i32> %y) {
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
   %unpckh = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-  %r = shufflevector <8 x i32> %unpckh, <8 x i32> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x i32> %unpckh, <8 x i32> poison, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %r
 }
 
@@ -3774,7 +3774,7 @@ define <8 x float> @blend_perm_v8f32(<8 x float> %x, <8 x float> %y) {
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
   %unpckh = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-  %r = shufflevector <8 x float> %unpckh, <8 x float> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x float> %unpckh, <8 x float> poison, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %r
 }
 
@@ -3786,7 +3786,7 @@ define <8 x i32> @unpckh_v8i32_unary(<8 x i32> %x) {
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
-  %r = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x i32> %x, <8 x i32> poison, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %r
 }
 
@@ -3798,7 +3798,7 @@ define <8 x float> @unpckh_v8f32_unary(<8 x float> %x) {
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; ALL-NEXT:    retq
-  %r = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x float> %x, <8 x float> poison, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %r
 }
 
@@ -3824,7 +3824,7 @@ define <8 x i32> @lowhalf_v8i32(<8 x i32> %x, <8 x i32> %y) {
 ; AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm2 = [2,14,3,14]
 ; AVX512VL-NEXT:    vpermt2d %ymm1, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
-  %r = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x i32> %r
 }
 
@@ -3850,7 +3850,7 @@ define <8 x float> @lowhalf_v8f32(<8 x float> %x, <8 x float> %y) {
 ; AVX512VL-NEXT:    vmovaps {{.*#+}} xmm2 = [2,14,3,14]
 ; AVX512VL-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
-  %r = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x float> %r
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index 1b80fcdedb43f..d3b04878dc06d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -83,7 +83,7 @@ define <16 x float> @shuffle_v16f32_01_01_03_00_06_04_05_07_08_08_09_09_15_14_14
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,1,3,0,6,4,5,7,8,8,9,9,15,14,14,12]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a0, <16 x float> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 0, i32 6, i32 4, i32 5, i32 7, i32 8, i32 8, i32 9, i32 9, i32 15, i32 14, i32 14, i32 12>
+  %shuffle = shufflevector <16 x float> %a0, <16 x float> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 0, i32 6, i32 4, i32 5, i32 7, i32 8, i32 8, i32 9, i32 9, i32 15, i32 14, i32 14, i32 12>
   ret <16 x float> %shuffle
 }
 
@@ -166,7 +166,7 @@ define <16 x float> @shuffle_v16f32_03_uu_uu_uu_uu_04_uu_uu_uu_uu_11_uu_uu_uu_uu
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} zmm0 = zmm0[3,0,3,0,7,4,7,4,11,8,11,8,15,12,15,12]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 12>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 12>
   ret <16 x float> %shuffle
 }
 
@@ -176,8 +176,8 @@ define <16 x float> @shuffle_v16f32_00_17_02_19_04_21_06_23_08_25_10_27_12_29_14
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %tmp1 = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> <i32 undef, i32 17, i32 undef, i32 19, i32 undef, i32 5, i32 undef, i32 7, i32 undef, i32 9, i32 undef, i32 11, i32 undef, i32 13, i32 undef, i32 15>
-  %tmp2 = shufflevector <16 x float> %tmp1, <16 x float> <float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef, float 0.000000e+00, float undef>, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
+  %tmp1 = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 poison, i32 17, i32 poison, i32 19, i32 poison, i32 5, i32 poison, i32 7, i32 poison, i32 9, i32 poison, i32 11, i32 poison, i32 13, i32 poison, i32 15>
+  %tmp2 = shufflevector <16 x float> %tmp1, <16 x float> <float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float poison>, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x float> %tmp2
 }
 
@@ -295,7 +295,7 @@ define <16 x float> @shuffle_v16f32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01(<
 ; ALL-NEXT:    vmovaps {{.*#+}} zmm1 = [2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,1]
 ; ALL-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
-  %c = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> <i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 1>
+  %c = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 2, i32 5, i32 poison, i32 poison, i32 7, i32 poison, i32 10, i32 1,  i32 0, i32 5, i32 poison, i32 4, i32 7, i32 poison, i32 10, i32 1>
   ret <16 x float> %c
 }
 
@@ -305,7 +305,7 @@ define <16 x i32> @shuffle_v16i32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01(<16
 ; ALL-NEXT:    vmovaps {{.*#+}} zmm1 = [2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,1]
 ; ALL-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
-  %c = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 1>
+  %c = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 2, i32 5, i32 poison, i32 poison, i32 7, i32 poison, i32 10, i32 1,  i32 0, i32 5, i32 poison, i32 4, i32 7, i32 poison, i32 10, i32 1>
   ret <16 x i32> %c
 }
 
@@ -342,7 +342,7 @@ define <16 x i32> @shuffle_v16i32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05_0
 ; FAST-NEXT:    vmovaps {{.*#+}} zmm1 = [11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4]
 ; FAST-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; FAST-NEXT:    retq
-  %1 = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+  %1 = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <16 x i32> %1
 }
 
@@ -358,7 +358,7 @@ define <16 x float> @shuffle_v16f32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05
 ; FAST-NEXT:    vmovaps {{.*#+}} zmm1 = [11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4]
 ; FAST-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; FAST-NEXT:    retq
-  %1 = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+  %1 = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <16 x float> %1
 }
 
@@ -406,7 +406,7 @@ define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; ALL-NEXT:    retq
-  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <16 x i32> %c
 }
 
@@ -425,7 +425,7 @@ define <8 x i32> @test_v16i32_1_3_5_7_9_11_13_15(<16 x i32> %v) {
 ; FAST-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; FAST-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; FAST-NEXT:    retq
-  %res = shufflevector <16 x i32> %v, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %res = shufflevector <16 x i32> %v, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   ret <8 x i32> %res
 }
 
@@ -446,7 +446,7 @@ define <4 x i32> @test_v16i32_0_1_2_12 (<16 x i32> %v) {
 ; FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; FAST-NEXT:    vzeroupper
 ; FAST-NEXT:    retq
-  %res = shufflevector <16 x i32> %v, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 12>
+  %res = shufflevector <16 x i32> %v, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 12>
   ret <4 x i32> %res
 }
 
@@ -460,7 +460,7 @@ define <4 x i32> @test_v16i32_0_4_8_12(<16 x i32> %v) {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
-  %res = shufflevector <16 x i32> %v, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %res = shufflevector <16 x i32> %v, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
   ret <4 x i32> %res
 }
 
@@ -470,7 +470,7 @@ define <8 x float> @shuffle_v16f32_extract_256(ptr %RET, ptr %a) {
 ; ALL-NEXT:    vmovups 32(%rsi), %ymm0
 ; ALL-NEXT:    retq
   %v_a = load <16 x float>, ptr %a, align 4
-  %v2 = shufflevector <16 x float> %v_a, <16 x float> undef, <8 x i32>  <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %v2 = shufflevector <16 x float> %v_a, <16 x float> poison, <8 x i32>  <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x float> %v2
 }
 
@@ -482,7 +482,7 @@ define <8 x float> @test_v16f32_0_1_2_3_4_6_7_10 (<16 x float> %v) {
 ; ALL-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; ALL-NEXT:    retq
-  %res = shufflevector <16 x float> %v, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 10>
+  %res = shufflevector <16 x float> %v, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 10>
   ret <8 x float> %res
 }
 
@@ -495,7 +495,7 @@ define <4 x float> @test_v16f32_0_1_3_6 (<16 x float> %v) {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
-  %res = shufflevector <16 x float> %v, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 6>
+  %res = shufflevector <16 x float> %v, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 3, i32 6>
   ret <4 x float> %res
 }
 
@@ -532,7 +532,7 @@ define <16 x i32> @insert_mem_and_zero_v16i32(ptr %ptr) {
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ALL-NEXT:    retq
   %a = load i32, ptr %ptr
-  %v = insertelement <16 x i32> undef, i32 %a, i32 0
+  %v = insertelement <16 x i32> poison, i32 %a, i32 0
   %shuffle = shufflevector <16 x i32> %v, <16 x i32> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i32> %shuffle
 }
@@ -581,7 +581,7 @@ define <16 x i32> @shuffle_v16i32_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15_0
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    valignd {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
   ret <16 x i32> %shuffle
 }
 
@@ -599,7 +599,7 @@ define <16 x i32> @shuffle_v16i32_16_16_02_03_20_20_06_07_24_24_10_11_28_28_uu_u
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufps {{.*#+}} zmm0 = zmm1[0,0],zmm0[2,3],zmm1[4,4],zmm0[6,7],zmm1[8,8],zmm0[10,11],zmm1[12,12],zmm0[14,15]
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 16, i32 16, i32 02, i32 03, i32 20, i32 20, i32 06, i32 07, i32 24, i32 24, i32 10, i32 11, i32 28, i32 28, i32 undef, i32 undef>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 16, i32 16, i32 02, i32 03, i32 20, i32 20, i32 06, i32 07, i32 24, i32 24, i32 10, i32 11, i32 28, i32 28, i32 poison, i32 poison>
   ret <16 x i32> %shuffle
 }
 
@@ -641,7 +641,7 @@ define <16 x float> @shuffle_v8f32_v16f32_04_04_04_04_04_04_04_04_04_04_04_04_04
 ; FAST-NEXT:    vbroadcastss {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
 ; FAST-NEXT:    vpermps %zmm0, %zmm1, %zmm0
 ; FAST-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
   ret <16 x float> %shuffle
 }
 
@@ -723,7 +723,7 @@ define <16 x i32> @mask_shuffle_v16i32_02_03_04_05_06_07_08_09_10_11_12_13_14_15
 ; AVX512BW-NEXT:    valignd {{.*#+}} zmm1 {%k1} = zmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32><i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32><i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x i32> %shuffle, <16 x i32> %passthru
   ret <16 x i32> %res
@@ -761,7 +761,7 @@ define <16 x i32> @maskz_shuffle_v16i32_02_03_04_05_06_07_08_09_10_11_12_13_14_1
 ; AVX512BW-NEXT:    kmovd %edi, %k1
 ; AVX512BW-NEXT:    valignd {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1]
 ; AVX512BW-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32><i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32><i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x i32> %shuffle, <16 x i32> zeroinitializer
   ret <16 x i32> %res
@@ -933,7 +933,7 @@ define <16 x i32> @mask_shuffle_v4i32_v16i32_00_01_02_03_00_01_02_03_00_01_02_03
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; ALL-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
 ; ALL-NEXT:    retq
-  %res = shufflevector <4 x i32> %a, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %res = shufflevector <4 x i32> %a, <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   ret <16 x i32> %res
 }
 
@@ -943,7 +943,7 @@ define <16 x float> @mask_shuffle_v4f32_v16f32_00_01_02_03_00_01_02_03_00_01_02_
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
 ; ALL-NEXT:    retq
-  %res = shufflevector <4 x float> %a, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %res = shufflevector <4 x float> %a, <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   ret <16 x float> %res
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
index ca81317eb61a4..2391d508435f2 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
@@ -15,7 +15,7 @@ define <32 x i16> @shuffle_v32i16(<32 x i16> %a)  {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpbroadcastw %xmm0, %zmm0
 ; SKX-NEXT:    retq
-  %c = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> zeroinitializer
+  %c = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> zeroinitializer
   ret <32 x i16> %c
 }
 
@@ -32,7 +32,7 @@ define <32 x i16> @shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_0
 ; SKX-NEXT:    vpbroadcastw {{.*#+}} zmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; SKX-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; SKX-NEXT:    retq
-  %c = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %c = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <32 x i16> %c
 }
 
@@ -58,7 +58,7 @@ define <32 x i16> @shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_
 ; SKX-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [2,5,0,0,7,0,10,1,0,5,0,4,7,0,10,1,2,5,0,0,7,0,10,1,0,5,0,4,7,0,10,31]
 ; SKX-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; SKX-NEXT:    retq
-  %c = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> <i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 1, i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 31>
+  %c = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 2, i32 5, i32 poison, i32 poison, i32 7, i32 poison, i32 10, i32 1,  i32 0, i32 5, i32 poison, i32 4, i32 7, i32 poison, i32 10, i32 1, i32 2, i32 5, i32 poison, i32 poison, i32 7, i32 poison, i32 10, i32 1,  i32 0, i32 5, i32 poison, i32 4, i32 7, i32 poison, i32 10, i32 31>
   ret <32 x i16> %c
 }
 
@@ -94,7 +94,7 @@ define <32 x i16> @shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u(<32 x i1
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
 ; ALL-NEXT:    retq
-  %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i16> %c
 }
 
@@ -103,7 +103,7 @@ define <32 x i16> @shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u(<32 x
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
 ; ALL-NEXT:    retq
-  %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <32 x i16> %c
 }
 
@@ -183,7 +183,7 @@ define <32 x i16> @shuffle_v32i16_01_00_03_02_05_04_07_06_09_08_11_10_13_12_15_1
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vprold $16, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+  %shuffle = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
   ret <32 x i16> %shuffle
 }
 
@@ -192,7 +192,7 @@ define <32 x i16> @shuffle_v32i16_03_00_01_02_07_04_05_06_11_08_09_10_15_12_13_1
 ; ALL:       ## %bb.0:
 ; ALL-NEXT:    vprolq $16, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14, i32 19, i32 16, i32 17, i32 18, i32 23, i32 20, i32 21, i32 22, i32 27, i32 24, i32 25, i32 26, i32 31, i32 28, i32 29, i32 30>
+  %shuffle = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 3, i32 0, i32 1, i32 2, i32 7, i32 4, i32 5, i32 6, i32 11, i32 8, i32 9, i32 10, i32 15, i32 12, i32 13, i32 14, i32 19, i32 16, i32 17, i32 18, i32 23, i32 20, i32 21, i32 22, i32 27, i32 24, i32 25, i32 26, i32 31, i32 28, i32 29, i32 30>
   ret <32 x i16> %shuffle
 }
 
@@ -278,7 +278,7 @@ define <32 x i16> @insert_dup_mem_v32i16_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> zeroinitializer
   ret <32 x i16> %tmp3
 }
 
@@ -297,7 +297,7 @@ define <32 x i16> @insert_dup_mem_v32i16_sext_i16(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <32 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <32 x i32> zeroinitializer
   ret <32 x i16> %tmp4
 }
 
@@ -315,7 +315,7 @@ define <32 x i16> @insert_dup_elt1_mem_v32i16_i32(ptr %ptr) #0 {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <32 x i16> %tmp3
 }
 
@@ -333,7 +333,7 @@ define <32 x i16> @insert_dup_elt3_mem_v32i16_i32(ptr %ptr) #0 {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 1
   %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <32 x i16> %tmp3
 }
 
@@ -351,7 +351,7 @@ define <32 x i16> @insert_dup_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> zeroinitializer
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> zeroinitializer
   ret <32 x i16> %tmp3
 }
 
@@ -369,7 +369,7 @@ define <32 x i16> @insert_dup_elt1_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <32 x i16> %tmp3
 }
 
@@ -387,7 +387,7 @@ define <32 x i16> @insert_dup_elt3_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <32 x i16> %tmp3
 }
 
@@ -405,7 +405,7 @@ define <32 x i16> @insert_dup_elt7_mem_v16i16_i64(ptr %ptr) {
   %tmp = load i64, ptr %ptr, align 4
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 1
   %tmp2 = bitcast <2 x i64> %tmp1 to <8 x i16>
-  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <32 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+  %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> poison, <32 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   ret <32 x i16> %tmp3
 }
 
@@ -424,7 +424,7 @@ define <32 x i16> @insert_dup_mem_v16i16_sext_i16_i64(ptr %ptr) {
   %tmp1 = sext i16 %tmp to i64
   %tmp2 = insertelement <2 x i64> zeroinitializer, i64 %tmp1, i32 0
   %tmp3 = bitcast <2 x i64> %tmp2 to <8 x i16>
-  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <32 x i32> zeroinitializer
+  %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> poison, <32 x i32> zeroinitializer
   ret <32 x i16> %tmp4
 }
 
@@ -459,7 +459,7 @@ define <8 x i16> @pr32967(<32 x i16> %v) {
 ; SKX-NEXT:    vpmovqw %zmm0, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
- %shuffle = shufflevector <32 x i16> %v, <32 x i16> undef, <8 x i32> <i32 1,i32 5,i32 9,i32 13,i32 17,i32 21,i32 25,i32 29>
+ %shuffle = shufflevector <32 x i16> %v, <32 x i16> poison, <8 x i32> <i32 1,i32 5,i32 9,i32 13,i32 17,i32 21,i32 25,i32 29>
  ret <8 x i16> %shuffle
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 469c087ec9c08..62327fb2618ce 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -9,7 +9,7 @@ define <64 x i8> @shuffle_v64i8_02_03_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_u_
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; ALL-NEXT:    retq
-  %b = shufflevector <64 x i8> %a, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %b = shufflevector <64 x i8> %a, <64 x i8> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <64 x i8> %b
 }
 
@@ -194,7 +194,7 @@ define <64 x i8> @shuffle_v64i8_63_62_61_60_59_58_57_56_55_54_53_52_51_50_49_48_
 ; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [63,62,61,60,59,58,57,56,55,54,53,52,51,50,49,48,47,46,45,44,43,42,41,40,39,38,37,36,35,34,33,32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; AVX512VBMI-NEXT:    vpermb %zmm0, %zmm1, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %shuffle = shufflevector <64 x i8> %a, <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuffle = shufflevector <64 x i8> %a, <64 x i8> poison, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <64 x i8> %shuffle
 }
 
@@ -204,7 +204,7 @@ define <64 x i8> @shuffle_v64i8_02_03_04_05_06_07_00_01_10_11_12_13_14_15_08_09_
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vprolq $48, %zmm0, %zmm0
 ; ALL-NEXT:    retq
-  %shuffle = shufflevector <64 x i8> %a, <64 x i8> undef, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 32, i32 33, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 40, i32 41, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 48, i32 49, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 56, i32 57>
+  %shuffle = shufflevector <64 x i8> %a, <64 x i8> poison, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 32, i32 33, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 40, i32 41, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 48, i32 49, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 56, i32 57>
   ret <64 x i8> %shuffle
 }
 
@@ -281,7 +281,7 @@ define <64 x i8> @insert_dup_mem_v64i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <64 x i32> zeroinitializer
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <64 x i32> zeroinitializer
   ret <64 x i8> %tmp3
 }
 
@@ -311,7 +311,7 @@ define <64 x i8> @insert_dup_mem_v64i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <64 x i32> zeroinitializer
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <64 x i32> zeroinitializer
   ret <64 x i8> %tmp4
 }
 
@@ -340,7 +340,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <64 x i8> %tmp3
 }
 
@@ -369,7 +369,7 @@ define <64 x i8> @insert_dup_elt3_mem_v64i8_i32(ptr %ptr) {
   %tmp = load i32, ptr %ptr, align 4
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <64 x i8> %tmp3
 }
 
@@ -409,7 +409,7 @@ define <64 x i8> @insert_dup_elt1_mem_v64i8_sext_i8(ptr %ptr) {
   %tmp1 = sext i8 %tmp to i32
   %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0
   %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8>
-  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <64 x i8> %tmp4
 }
 
@@ -639,7 +639,7 @@ define <64 x i8> @shuffle_v64i8_01_03_07_09_13_15_19_21_25_27_31_33_37_39_43_45_
 ; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,3,7,9,13,15,19,21,25,27,31,33,37,39,43,45,49,51,55,57,61,63,67,69,73,75,79,81,85,87,91,93,97,99,103,105,109,111,115,117,121,123,127,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512VBMI-NEXT:    vpermt2b %zmm1, %zmm2, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 1, i32 3, i32 7, i32 9, i32 13, i32 15, i32 19, i32 21, i32 25, i32 27, i32 31, i32 33, i32 37, i32 39, i32 43, i32 45, i32 49, i32 51, i32 55, i32 57, i32 61, i32 63, i32 67, i32 69, i32 73, i32 75, i32 79, i32 81, i32 85, i32 87, i32 91, i32 93, i32 97, i32 99, i32 103, i32 105, i32 109, i32 111, i32 115, i32 117, i32 121, i32 123, i32 127, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 1, i32 3, i32 7, i32 9, i32 13, i32 15, i32 19, i32 21, i32 25, i32 27, i32 31, i32 33, i32 37, i32 39, i32 43, i32 45, i32 49, i32 51, i32 55, i32 57, i32 61, i32 63, i32 67, i32 69, i32 73, i32 75, i32 79, i32 81, i32 85, i32 87, i32 91, i32 93, i32 97, i32 99, i32 103, i32 105, i32 109, i32 111, i32 115, i32 117, i32 121, i32 123, i32 127, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <64 x i8> %r
 }
 
@@ -731,7 +731,7 @@ define <64 x i8> @shuffle_v64i8_01_05_07_11_13_17_19_23_25_29_31_35_37_41_43_47_
 ; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm1, %zmm2
 ; AVX512VBMI-NEXT:    vmovdqa64 %zmm2, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 1, i32 5, i32 7, i32 11, i32 13, i32 17, i32 19, i32 23, i32 25, i32 29, i32 31, i32 35, i32 37, i32 41, i32 43, i32 47, i32 49, i32 53, i32 55, i32 59, i32 61, i32 65, i32 67, i32 71, i32 73, i32 77, i32 79, i32 83, i32 85, i32 89, i32 91, i32 95, i32 97, i32 101, i32 103, i32 107, i32 109, i32 113, i32 115, i32 119, i32 121, i32 125, i32 127, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 1, i32 5, i32 7, i32 11, i32 13, i32 17, i32 19, i32 23, i32 25, i32 29, i32 31, i32 35, i32 37, i32 41, i32 43, i32 47, i32 49, i32 53, i32 55, i32 59, i32 61, i32 65, i32 67, i32 71, i32 73, i32 77, i32 79, i32 83, i32 85, i32 89, i32 91, i32 95, i32 97, i32 101, i32 103, i32 107, i32 109, i32 113, i32 115, i32 119, i32 121, i32 125, i32 127, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <64 x i8> %r
 }
 
@@ -822,7 +822,7 @@ define <64 x i8> @shuffle_v64i8_02_04_08_10_14_16_20_22_26_28_32_34_38_40_44_46_
 ; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [2,4,8,10,14,16,20,22,26,28,32,34,38,40,44,46,50,52,56,58,62,64,68,70,74,76,80,82,86,88,92,94,98,100,104,106,110,112,116,118,122,124,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512VBMI-NEXT:    vpermt2b %zmm1, %zmm2, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 2, i32 4, i32 8, i32 10, i32 14, i32 16, i32 20, i32 22, i32 26, i32 28, i32 32, i32 34, i32 38, i32 40, i32 44, i32 46, i32 50, i32 52, i32 56, i32 58, i32 62, i32 64, i32 68, i32 70, i32 74, i32 76, i32 80, i32 82, i32 86, i32 88, i32 92, i32 94, i32 98, i32 100, i32 104, i32 106, i32 110, i32 112, i32 116, i32 118, i32 122, i32 124, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 2, i32 4, i32 8, i32 10, i32 14, i32 16, i32 20, i32 22, i32 26, i32 28, i32 32, i32 34, i32 38, i32 40, i32 44, i32 46, i32 50, i32 52, i32 56, i32 58, i32 62, i32 64, i32 68, i32 70, i32 74, i32 76, i32 80, i32 82, i32 86, i32 88, i32 92, i32 94, i32 98, i32 100, i32 104, i32 106, i32 110, i32 112, i32 116, i32 118, i32 122, i32 124, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <64 x i8> %r
 }
 
@@ -914,7 +914,7 @@ define <64 x i8> @shuffle_v64i8_00_04_06_10_12_16_18_22_24_28_30_34_36_40_42_46_
 ; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm1, %zmm2
 ; AVX512VBMI-NEXT:    vmovdqa64 %zmm2, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 0, i32 4, i32 6, i32 10, i32 12, i32 16, i32 18, i32 22, i32 24, i32 28, i32 30, i32 34, i32 36, i32 40, i32 42, i32 46, i32 48, i32 52, i32 54, i32 58, i32 60, i32 64, i32 66, i32 70, i32 72, i32 76, i32 78, i32 82, i32 84, i32 88, i32 90, i32 94, i32 96, i32 100, i32 102, i32 106, i32 108, i32 112, i32 114, i32 118, i32 120, i32 124, i32 126, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %r = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 0, i32 4, i32 6, i32 10, i32 12, i32 16, i32 18, i32 22, i32 24, i32 28, i32 30, i32 34, i32 36, i32 40, i32 42, i32 46, i32 48, i32 52, i32 54, i32 58, i32 60, i32 64, i32 66, i32 70, i32 72, i32 76, i32 78, i32 82, i32 84, i32 88, i32 90, i32 94, i32 96, i32 100, i32 102, i32 106, i32 108, i32 112, i32 114, i32 118, i32 120, i32 124, i32 126, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <64 x i8> %r
 }
 
@@ -1154,8 +1154,8 @@ define <64 x i8> @shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [63,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
-; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm2, %zmm0
+; AVX512BW-NEXT:    valignq {{.*#+}} zmm1 = zmm1[6,7],zmm0[0,1,2,3,4,5]
+; AVX512BW-NEXT:    vpalignr {{.*#+}} zmm0 = zmm1[14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13],zmm1[30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29],zmm1[46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42,43,44,45],zmm1[62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58,59,60,61]
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125:
@@ -1171,8 +1171,8 @@ define <64 x i8> @shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125:
 ; AVX512VBMI:       # %bb.0:
-; AVX512VBMI-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [63,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
-; AVX512VBMI-NEXT:    vpermt2w %zmm1, %zmm2, %zmm0
+; AVX512VBMI-NEXT:    valignq {{.*#+}} zmm1 = zmm1[6,7],zmm0[0,1,2,3,4,5]
+; AVX512VBMI-NEXT:    vpalignr {{.*#+}} zmm0 = zmm1[14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13],zmm1[30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29],zmm1[46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42,43,44,45],zmm1[62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58,59,60,61]
 ; AVX512VBMI-NEXT:    retq
   %r = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125>
   ret <64 x i8> %r
@@ -1670,7 +1670,7 @@ define <64 x i8> @PR54562_ref(<64 x i8> %a0) {
 ; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [1,0,2,1,4,3,5,4,7,6,8,7,10,9,11,10,13,12,14,13,16,15,17,16,19,18,20,19,22,21,23,22,25,24,26,25,28,27,29,28,31,30,32,31,34,33,35,34,37,36,38,37,40,39,41,40,43,42,44,43,46,45,47,46]
 ; AVX512VBMI-NEXT:    vpermb %zmm0, %zmm1, %zmm0
 ; AVX512VBMI-NEXT:    retq
-  %shuffle1 = shufflevector <64 x i8> %a0, <64 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 undef, i32 undef, i32 undef, i32 undef, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle1 = shufflevector <64 x i8> %a0, <64 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 poison, i32 poison, i32 poison, i32 poison>
   %shuffle2 = shufflevector <64 x i8> %shuffle1, <64 x i8> poison, <64 x i32> <i32 1,  i32 0,  i32 2,  i32 1,  i32 4,  i32 3,  i32 5,  i32 4, i32 7,  i32 6,  i32 8,  i32 7,  i32 10, i32 9,  i32 11, i32 10, i32 17, i32 16, i32 18, i32 17, i32 20, i32 19, i32 21, i32 20, i32 23, i32 22, i32 24, i32 23, i32 26, i32 25, i32 27, i32 26, i32 33, i32 32, i32 34, i32 33, i32 36, i32 35, i32 37, i32 36, i32 39, i32 38, i32 40, i32 39, i32 42, i32 41, i32 43, i32 42, i32 49, i32 48, i32 50, i32 49, i32 52, i32 51, i32 53, i32 52, i32 55, i32 54, i32 56, i32 55, i32 58, i32 57, i32 59, i32 58>
   ret <64 x i8> %shuffle2
 }
@@ -1721,7 +1721,7 @@ define void @PR54562_mem(ptr %src, ptr %dst) {
 ; AVX512VBMI-NEXT:    vzeroupper
 ; AVX512VBMI-NEXT:    retq
   %load = load <64 x i8>, ptr %src, align 512
-  %shuffle1 = shufflevector <64 x i8> %load, <64 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 undef, i32 undef, i32 undef, i32 undef, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef>
+  %shuffle1 = shufflevector <64 x i8> %load, <64 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 poison, i32 poison, i32 poison, i32 poison>
   %shuffle2 = shufflevector <64 x i8> %shuffle1, <64 x i8> poison, <64 x i32> <i32 1,  i32 0,  i32 2,  i32 1,  i32 4,  i32 3,  i32 5,  i32 4, i32 7,  i32 6,  i32 8,  i32 7,  i32 10, i32 9,  i32 11, i32 10, i32 17, i32 16, i32 18, i32 17, i32 20, i32 19, i32 21, i32 20, i32 23, i32 22, i32 24, i32 23, i32 26, i32 25, i32 27, i32 26, i32 33, i32 32, i32 34, i32 33, i32 36, i32 35, i32 37, i32 36, i32 39, i32 38, i32 40, i32 39, i32 42, i32 41, i32 43, i32 42, i32 49, i32 48, i32 50, i32 49, i32 52, i32 51, i32 53, i32 52, i32 55, i32 54, i32 56, i32 55, i32 58, i32 57, i32 59, i32 58>
   store <64 x i8> %shuffle2, ptr %dst, align 512
   ret void
@@ -1732,7 +1732,7 @@ define <64 x i8> @shuffle_v32i16_zextinreg_to_v16i32(<64 x i8> %a)  {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; ALL-NEXT:    retq
-  %b = shufflevector <64 x i8> %a, <64 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef
+  %b = shufflevector <64 x i8> %a, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison
 >, <64 x i32> <i32 0, i32 1, i32 84, i32 84, i32 2, i32 3, i32 84, i32 84, i32 4, i32 5, i32 84, i32 84, i32 6, i32 7, i32 84, i32 84, i32 8, i32 9, i32 84, i32 84, i32 10, i32 11, i32 84, i32 84, i32 12, i32 13, i32 84, i32 84, i32 14, i32 15, i32 84, i32 84, i32 16, i32 17, i32 84, i32 84, i32 18, i32 19, i32 84, i32 84, i32 20, i32 21, i32 84, i32 84, i32 22, i32 23, i32 84, i32 84, i32 24, i32 25, i32 84, i32 84, i32 26, i32 27, i32 84, i32 84, i32 28, i32 29, i32 84, i32 84, i32 30, i32 31, i32 84, i32 84>
   ret <64 x i8> %b
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
index a706e4ce00ed3..172ba83d30415 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -696,7 +696,7 @@ define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,2,0,u,u,6,0,u,u,4,0,4,0]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 poison, i32 6, i32 poison, i32 4, i32 4>
   ret <8 x double> %shuffle
 }
 
@@ -712,7 +712,7 @@ define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,u,u,u,u,6,0,6,0,u,u,u,u]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 6, i32 6, i32 poison, i32 poison>
   ret <8 x double> %shuffle
 }
 
@@ -721,7 +721,7 @@ define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,6]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 poison, i32 poison>
   ret <8 x double> %shuffle
 }
 
@@ -730,7 +730,7 @@ define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1,1,3,3,4,4,6,7]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 poison, i32 poison, i32 6, i32 7>
   ret <8 x double> %shuffle
 }
 
@@ -739,7 +739,7 @@ define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0,0,2,3,5,4,6,6]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 5, i32 4, i32 poison, i32 poison>
   ret <8 x double> %shuffle
 }
 
@@ -748,7 +748,7 @@ define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0,0,2,3,4,4,6,6]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 6>
   ret <8 x double> %shuffle
 }
 
@@ -792,7 +792,7 @@ define <8 x double> @shuffle_v8f64_1z2z5z6z(<8 x double> %a, <8 x double> %b) {
 ; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[2],zmm1[3],zmm0[5],zmm1[5],zmm0[6],zmm1[7]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x double> %a, <8 x double> <double 0.000000e+00, double undef, double undef, double undef, double undef, double undef, double undef, double undef>, <8 x i32> <i32 1, i32 8, i32 2, i32 8, i32 5, i32 8, i32 6, i32 8>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> <double 0.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison, double poison>, <8 x i32> <i32 1, i32 8, i32 2, i32 8, i32 5, i32 8, i32 6, i32 8>
   ret <8 x double> %shuffle
 }
 
@@ -801,7 +801,7 @@ define <8 x double> @shuffle_v8f64_23uuuuuu(<8 x double> %a0, <8 x double> %a1)
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %1 = shufflevector <8 x double> %a0, <8 x double> poison, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x double> %1
 }
 
@@ -819,7 +819,7 @@ define <8 x double> @shuffle_v8f64_4567uuuu(<8 x double> %a0, <8 x double> %a1)
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  %1 = shufflevector <8 x double> %a0, <8 x double> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
   ret <8 x double> %1
 }
 
@@ -1497,7 +1497,7 @@ define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,2,0,u,u,6,0,u,u,4,0,4,0]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 poison, i32 6, i32 poison, i32 4, i32 4>
   ret <8 x i64> %shuffle
 }
 
@@ -1513,7 +1513,7 @@ define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,u,u,u,u,6,0,6,0,u,u,u,u]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 6, i32 6, i32 poison, i32 poison>
   ret <8 x i64> %shuffle
 }
 
@@ -1529,7 +1529,7 @@ define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [1,0,0,0,3,0,2,0,4,0,5,0,u,u,u,u]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 poison, i32 poison>
   ret <8 x i64> %shuffle
 }
 
@@ -1545,7 +1545,7 @@ define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [1,0,1,0,3,0,3,0,u,u,u,u,6,0,7,0]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 poison, i32 poison, i32 6, i32 7>
   ret <8 x i64> %shuffle
 }
 
@@ -1561,7 +1561,7 @@ define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,u,u,u,u,3,0,5,0,4,0,u,u,u,u]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 5, i32 4, i32 poison, i32 poison>
   ret <8 x i64> %shuffle
 }
 
@@ -1577,7 +1577,7 @@ define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) {
 ; AVX512F-32-NEXT:    vmovaps {{.*#+}} zmm1 = [u,u,u,u,u,u,3,0,u,u,u,u,6,0,6,0]
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 6, i32 6>
   ret <8 x i64> %shuffle
 }
 
@@ -1790,7 +1790,7 @@ define <8 x double> @shuffle_v8f64_23014567(<8 x double> %a0, <8 x double> %a1)
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[2,3,0,1,4,5,6,7]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a1, <8 x double> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
+  %1 = shufflevector <8 x double> %a1, <8 x double> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
   ret <8 x double> %1
 }
 
@@ -1799,7 +1799,7 @@ define <8 x double> @shuffle_v8f64_2301uu67(<8 x double> %a0, <8 x double> %a1)
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[2,3,0,1,4,5,6,7]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a1, <8 x double> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef, i32 6, i32 7>
+  %1 = shufflevector <8 x double> %a1, <8 x double> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 poison, i32 poison, i32 6, i32 7>
   ret <8 x double> %1
 }
 
@@ -1808,7 +1808,7 @@ define <8 x double> @shuffle_v8f64_2301uuuu(<8 x double> %a0, <8 x double> %a1)
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm1[2,3,0,1]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a1, <8 x double> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+  %1 = shufflevector <8 x double> %a1, <8 x double> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
   ret <8 x double> %1
 }
 
@@ -1817,7 +1817,7 @@ define <8 x double> @shuffle_v8f64_uuu2301(<8 x double> %a0, <8 x double> %a1) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3,2,3,0,1]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x double> %a1, <8 x double> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 3, i32 0, i32 1>
+  %1 = shufflevector <8 x double> %a1, <8 x double> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 3, i32 0, i32 1>
   ret <8 x double> %1
 }
 
@@ -1853,7 +1853,7 @@ define <8 x i64> @shuffle_v8i64_12345670(<8 x i64> %a) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   ret <8 x i64> %shuffle
 }
 
@@ -1893,7 +1893,7 @@ define <8 x i64> @mask_shuffle_v8i64_12345670(<8 x i64> %a, <8 x i64> %passthru,
 ; AVX512F-32-NEXT:    valignq {{.*#+}} zmm1 {%k1} = zmm0[1,2,3,4,5,6,7,0]
 ; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i64> %shuffle, <8 x i64> %passthru
   ret <8 x i64> %res
@@ -1931,7 +1931,7 @@ define <8 x i64> @maskz_shuffle_v8i64_12345670(<8 x i64> %a, i8 %mask) {
 ; AVX512F-32-NEXT:    kmovw %eax, %k1
 ; AVX512F-32-NEXT:    valignq {{.*#+}} zmm0 {%k1} {z} = zmm0[1,2,3,4,5,6,7,0]
 ; AVX512F-32-NEXT:    retl
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i64> %shuffle, <8 x i64> zeroinitializer
   ret <8 x i64> %res
@@ -2068,7 +2068,7 @@ define <8 x i64> @shuffle_v8i64_23uuuuuu(<8 x i64> %a0, <8 x i64> %a1) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> poison, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i64> %1
 }
 
@@ -2086,7 +2086,7 @@ define <8 x i64> @shuffle_v8i64_4567uuuu(<8 x i64> %a0, <8 x i64> %a1) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i64> %1
 }
 
@@ -2095,7 +2095,7 @@ define <8 x i64> @shuffle_v8i64_uu67zzzz(<8 x i64> %a0, <8 x i64> %a1) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %1 = shufflevector <8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i32> <i32 poison, i32 poison, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i64> %1
 }
 
@@ -2105,7 +2105,7 @@ define <8 x double> @shuffle_v4f64_v8f64_22222222(<4 x double> %a) {
 ; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    vbroadcastsd %xmm0, %zmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   ret <8 x double> %shuffle
 }
 
@@ -2115,7 +2115,7 @@ define <8 x i64> @shuffle_v2i64_v8i64_01010101(<2 x i64> %a) {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; ALL-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <2 x i64> %a, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   ret <8 x i64> %shuffle
 }
 
@@ -2125,7 +2125,7 @@ define <8 x double> @shuffle_v2f64_v8f64_01010101(<2 x double> %a) {
 ; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1]
 ; ALL-NEXT:    ret{{[l|q]}}
-  %shuffle = shufflevector <2 x double> %a, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuffle = shufflevector <2 x double> %a, <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   ret <8 x double> %shuffle
 }
 
@@ -2144,7 +2144,7 @@ define <4 x double> @test_v8f64_2346 (<8 x double> %v) {
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-32-NEXT:    retl
-  %res = shufflevector <8 x double> %v, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 6>
+  %res = shufflevector <8 x double> %v, <8 x double> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 6>
   ret <4 x double> %res
 }
 
@@ -2165,7 +2165,7 @@ define <2 x double> @test_v8f64_34 (<8 x double> %v) {
 ; AVX512F-32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-32-NEXT:    vzeroupper
 ; AVX512F-32-NEXT:    retl
-  %res = shufflevector <8 x double> %v, <8 x double> undef, <2 x i32> <i32 3, i32 4>
+  %res = shufflevector <8 x double> %v, <8 x double> poison, <2 x i32> <i32 3, i32 4>
   ret <2 x double> %res
 }
 
@@ -2184,7 +2184,7 @@ define <4 x i64> @test_v8i64_1257 (<8 x i64> %v) {
 ; AVX512F-32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-32-NEXT:    retl
-  %res = shufflevector <8 x i64> %v, <8 x i64> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 7>
+  %res = shufflevector <8 x i64> %v, <8 x i64> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 7>
   ret <4 x i64> %res
 }
 
@@ -2204,7 +2204,7 @@ define <2 x i64> @test_v8i64_2_5 (<8 x i64> %v) {
 ; AVX512F-32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-32-NEXT:    vzeroupper
 ; AVX512F-32-NEXT:    retl
-  %res = shufflevector <8 x i64> %v, <8 x i64> undef, <2 x i32> <i32 2, i32 5>
+  %res = shufflevector <8 x i64> %v, <8 x i64> poison, <2 x i32> <i32 2, i32 5>
   ret <2 x i64> %res
 }
 
@@ -2213,7 +2213,7 @@ define <8 x i64> @test_v8i64_insert_zero_128(<8 x i64> %a) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovaps %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = shufflevector <8 x i64> %a, <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 8, i32 9, i32 8, i32 9>
+  %res = shufflevector <8 x i64> %a, <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 poison, i64 poison, i64 poison, i64 poison>, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 8, i32 9, i32 8, i32 9>
   ret <8 x i64> %res
 }
 
@@ -2222,6 +2222,6 @@ define <8 x i64> @test_v8i64_insert_zero_256(<8 x i64> %a) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovaps %ymm0, %ymm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = shufflevector <8 x i64> %a, <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
+  %res = shufflevector <8 x i64> %a, <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 poison, i64 poison, i64 poison, i64 poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   ret <8 x i64> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
index b4375cfb343b7..376fe37ef1fa8 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
@@ -24,8 +24,7 @@ define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x
 ; X86-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32]
-; X86-NEXT:    vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpermw %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_32i16_identity_mask:
@@ -33,8 +32,7 @@ define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x
 ; X64-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-NEXT:    kmovd %edi, %k1
 ; X64-NEXT:    vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32]
-; X64-NEXT:    vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vpermw %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
   %res0 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <32 x i16> %x0, <32 x i16> %x1, i32 %m)
   %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 63, i16 30, i16 61, i16 28, i16 59, i16 26, i16 57, i16 24, i16 55, i16 22, i16 53, i16 20, i16 51, i16 18, i16 49, i16 16, i16 47, i16 46, i16 13, i16 44, i16 11, i16 42, i16 9, i16 40, i16 7, i16 38, i16 5, i16 36, i16 3, i16 34, i16 1, i16 32>, <32 x i16> %res0, <32 x i16> %res0, i32 %m)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
index 388511ce0741f..f0b70ae26b1f0 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
@@ -19,8 +19,7 @@ define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x
 ; X86-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z}
-; X86-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X86-NEXT:    vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z}
+; X86-NEXT:    vpermw %ymm0, %ymm1, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16i16_identity_mask:
@@ -28,8 +27,7 @@ define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x
 ; X64-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-NEXT:    kmovd %edi, %k1
 ; X64-NEXT:    vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z}
-; X64-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-NEXT:    vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z}
+; X64-NEXT:    vpermw %ymm0, %ymm1, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
   %res0 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <16 x i16> %x0, <16 x i16> %x1, i16 %m)
   %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 30, i16 13, i16 28, i16 11, i16 26, i16 9, i16 24, i16 7, i16 22, i16 5, i16 20, i16 3, i16 18, i16 1, i16 16>, <16 x i16> %res0, <16 x i16> %res0, i16 %m)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
index e87e810971e11..29806cd25fe3f 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
@@ -155,8 +155,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8
 ; X86-AVX512F-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512F-NEXT:    kmovw %eax, %k1
 ; X86-AVX512F-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-AVX512F-NEXT:    vmovapd {{.*#+}} zmm1 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X86-AVX512F-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-AVX512F-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X86-AVX512BW-LABEL: combine_vpermt2var_8f64_identity_mask:
@@ -165,8 +164,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8
 ; X86-AVX512BW-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512BW-NEXT:    kmovd %eax, %k1
 ; X86-AVX512BW-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-AVX512BW-NEXT:    vmovapd {{.*#+}} zmm1 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X86-AVX512BW-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-AVX512BW-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-AVX512BW-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: combine_vpermt2var_8f64_identity_mask:
@@ -174,8 +172,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8
 ; X64-AVX512F-NEXT:    vmovapd {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
 ; X64-AVX512F-NEXT:    kmovw %edi, %k1
 ; X64-AVX512F-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512F-NEXT:    vmovapd {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X64-AVX512F-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: combine_vpermt2var_8f64_identity_mask:
@@ -183,8 +180,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8
 ; X64-AVX512BW-NEXT:    vmovapd {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
 ; X64-AVX512BW-NEXT:    kmovd %edi, %k1
 ; X64-AVX512BW-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512BW-NEXT:    vmovapd {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X64-AVX512BW-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512BW-NEXT:    retq
   %res0 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x double> %x0, <8 x double> %x1, i8 %m)
   %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x double> %res0, <8 x double> %res0, i8 %m)
@@ -259,8 +255,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64>
 ; X86-AVX512F-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512F-NEXT:    kmovw %eax, %k1
 ; X86-AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-AVX512F-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X86-AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-AVX512F-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X86-AVX512BW-LABEL: combine_vpermt2var_8i64_identity_mask:
@@ -269,8 +264,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64>
 ; X86-AVX512BW-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512BW-NEXT:    kmovd %eax, %k1
 ; X86-AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X86-AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-AVX512BW-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-AVX512BW-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: combine_vpermt2var_8i64_identity_mask:
@@ -278,8 +272,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64>
 ; X64-AVX512F-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
 ; X64-AVX512F-NEXT:    kmovw %edi, %k1
 ; X64-AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512F-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X64-AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: combine_vpermt2var_8i64_identity_mask:
@@ -287,8 +280,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64>
 ; X64-AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
 ; X64-AVX512BW-NEXT:    kmovd %edi, %k1
 ; X64-AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512BW-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8]
-; X64-AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512BW-NEXT:    retq
   %res0 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x i64> %x0, <8 x i64> %x1, i8 %m)
   %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x i64> %res0, <8 x i64> %res0, i8 %m)
@@ -309,8 +301,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1
 ; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X86-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: combine_vpermt2var_16f32_identity_mask:
@@ -318,8 +309,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1
 ; X64-AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-AVX512F-NEXT:    kmovw %edi, %k1
 ; X64-AVX512F-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-AVX512F-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: combine_vpermt2var_16f32_identity_mask:
@@ -327,8 +317,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1
 ; X64-AVX512BW-NEXT:    vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-AVX512BW-NEXT:    kmovd %edi, %k1
 ; X64-AVX512BW-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512BW-NEXT:    vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-AVX512BW-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512BW-NEXT:    retq
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x float> %x0, <16 x float> %x1, i16 %m)
   %res1 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x float> %res0, <16 x float> %res0, i16 %m)
@@ -598,8 +587,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x
 ; X86-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X86-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X86-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: combine_vpermt2var_16i32_identity_mask:
@@ -607,8 +595,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x
 ; X64-AVX512F-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-AVX512F-NEXT:    kmovw %edi, %k1
 ; X64-AVX512F-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512F-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-AVX512F-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: combine_vpermt2var_16i32_identity_mask:
@@ -616,8 +603,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x
 ; X64-AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-AVX512BW-NEXT:    kmovd %edi, %k1
 ; X64-AVX512BW-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X64-AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-AVX512BW-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
 ; X64-AVX512BW-NEXT:    retq
   %res0 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x i32> %x0, <16 x i32> %x1, i16 %m)
   %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x i32> %res0, <16 x i32> %res0, i16 %m)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
index 04cfa3cedd58f..ccc78823623df 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
@@ -32,8 +32,7 @@ define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8>
 ; X86-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z}
-; X86-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X86-NEXT:    vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    vpermb %xmm0, %xmm1, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16i8_identity_mask:
@@ -41,8 +40,7 @@ define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8>
 ; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
 ; X64-NEXT:    kmovd %edi, %k1
 ; X64-NEXT:    vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z}
-; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X64-NEXT:    vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z}
+; X64-NEXT:    vpermb %xmm0, %xmm1, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
   %res0 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8> %x0, <16 x i8> %x1, i16 %m)
   %res1 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 30, i8 13, i8 28, i8 11, i8 26, i8 9, i8 24, i8 7, i8 22, i8 5, i8 20, i8 3, i8 18, i8 1, i8 16>, <16 x i8> %res0, <16 x i8> %res0, i16 %m)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-masked.ll b/llvm/test/CodeGen/X86/vector-shuffle-masked.ll
index 1cb914f5b7840..545ffcc9fab86 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-masked.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-masked.ll
@@ -10,7 +10,7 @@ define <4 x i32> @mask_shuffle_v4i32_1234(<4 x i32> %a, <4 x i32> %b, <4 x i32>
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -23,7 +23,7 @@ define <4 x i32> @maskz_shuffle_v4i32_1234(<4 x i32> %a, <4 x i32> %b, i8 %mask)
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -37,7 +37,7 @@ define <4 x i32> @mask_shuffle_v4i32_2345(<4 x i32> %a, <4 x i32> %b, <4 x i32>
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -50,7 +50,7 @@ define <4 x i32> @maskz_shuffle_v4i32_2345(<4 x i32> %a, <4 x i32> %b, i8 %mask)
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -64,7 +64,7 @@ define <2 x i64> @mask_shuffle_v2i64_12(<2 x i64> %a, <2 x i64> %b, <2 x i64> %p
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 1, i32 2>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -77,7 +77,7 @@ define <2 x i64> @maskz_shuffle_v2i64_12(<2 x i64> %a, <2 x i64> %b, i8 %mask) {
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 1, i32 2>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -91,7 +91,7 @@ define <4 x i64> @mask_shuffle_v4i64_1234(<4 x i64> %a, <4 x i64> %b, <4 x i64>
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -104,7 +104,7 @@ define <4 x i64> @maskz_shuffle_v4i64_1234(<4 x i64> %a, <4 x i64> %b, i8 %mask)
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -116,9 +116,9 @@ define <4 x i64> @mask_shuffle_v4i64_1230(<4 x i64> %a, <4 x i64> %passthru, i8
 ; CHECK-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[1,2,3,0]
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -129,9 +129,9 @@ define <4 x i64> @maskz_shuffle_v4i64_1230(<4 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3,0]
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -193,7 +193,7 @@ define <8 x i32> @mask_shuffle_v8i32_12345670(<8 x i32> %a, <8 x i32> %passthru,
 ; CHECK-NEXT:    valignd {{.*#+}} ymm1 {%k1} = ymm0[1,2,3,4,5,6,7,0]
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
   ret <8 x i32> %res
@@ -205,7 +205,7 @@ define <8 x i32> @maskz_shuffle_v8i32_12345670(<8 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    valignd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3,4,5,6,7,0]
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
   ret <8 x i32> %res
@@ -218,7 +218,7 @@ define <8 x i32> @mask_shuffle_v8i32_23456701(<8 x i32> %a, <8 x i32> %passthru,
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
   ret <8 x i32> %res
@@ -231,7 +231,7 @@ define <8 x i32> @maskz_shuffle_v8i32_23456701(<8 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
   ret <8 x i32> %res
@@ -244,9 +244,9 @@ define <4 x i32> @mask_extract_v8i32_v4i32_0(<8 x i32> %a, <4 x i32> %passthru,
 ; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -258,9 +258,9 @@ define <4 x i32> @mask_extract_v8i32_v4i32_0_z(<8 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -273,9 +273,9 @@ define <4 x i32> @mask_extract_v8i32_v4i32_1(<8 x i32> %a, <4 x i32> %passthru,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -287,9 +287,9 @@ define <4 x i32> @mask_extract_v8i32_v4i32_1_z(<8 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti32x4 $1, %ymm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -301,9 +301,9 @@ define <4 x float> @mask_extract_v8f32_v4f32_0(<8 x float> %a, <4 x float> %pass
 ; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -315,9 +315,9 @@ define <4 x float> @mask_extract_v8f32_v4f32_0_z(<8 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -330,9 +330,9 @@ define <4 x float> @mask_extract_v8f32_v4f32_1(<8 x float> %a, <4 x float> %pass
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -344,9 +344,9 @@ define <4 x float> @mask_extract_v8f32_v4f32_1_z(<8 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    vextractf32x4 $1, %ymm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -358,9 +358,9 @@ define <2 x i64> @mask_extract_v4i64_v2i64_0(<4 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -372,9 +372,9 @@ define <2 x i64> @mask_extract_v4i64_v2i64_0_z(<4 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -387,9 +387,9 @@ define <2 x i64> @mask_extract_v4i64_v2i64_1(<4 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -401,9 +401,9 @@ define <2 x i64> @mask_extract_v4i64_v2i64_1_z(<4 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti64x2 $1, %ymm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -415,9 +415,9 @@ define <2 x double> @mask_extract_v4f64_v2f64_0(<4 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -429,9 +429,9 @@ define <2 x double> @mask_extract_v4f64_v2f64_0_z(<4 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -444,9 +444,9 @@ define <2 x double> @mask_extract_v4f64_v2f64_1(<4 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -458,9 +458,9 @@ define <2 x double> @mask_extract_v4f64_v2f64_1_z(<4 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    vextractf64x2 $1, %ymm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -472,9 +472,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_0(<16 x i32> %a, <4 x i32> %passthru
 ; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -486,9 +486,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_0_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -501,9 +501,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_1(<16 x i32> %a, <4 x i32> %passthru
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -515,9 +515,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_1_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti32x4 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -530,9 +530,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_2(<16 x i32> %a, <4 x i32> %passthru
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -545,9 +545,9 @@ define <4 x i32> @mask_extract_v16i32_v4i32_3(<16 x i32> %a, <4 x i32> %passthru
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -559,9 +559,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_0(<16 x float> %a, <4 x float> %pa
 ; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -573,9 +573,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_0_z(<16 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -588,9 +588,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_1(<16 x float> %a, <4 x float> %pa
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -602,9 +602,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_1_z(<16 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    vextractf32x4 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -617,9 +617,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_2(<16 x float> %a, <4 x float> %pa
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -632,9 +632,9 @@ define <4 x float> @mask_extract_v16f32_v4f32_3(<16 x float> %a, <4 x float> %pa
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -645,7 +645,7 @@ define <8 x i32> @mask_extract_v16i32_v8i32_0(<16 x i32> %a, <8 x i32> %passthru
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
   ret <8 x i32> %res
@@ -657,7 +657,7 @@ define <8 x i32> @mask_extract_v16i32_v8i32_0_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
   ret <8 x i32> %res
@@ -670,7 +670,7 @@ define <8 x i32> @mask_extract_v16i32_v8i32_1(<16 x i32> %a, <8 x i32> %passthru
 ; CHECK-NEXT:    vextracti32x8 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
   ret <8 x i32> %res
@@ -682,7 +682,7 @@ define <8 x i32> @mask_extract_v16i32_v8i32_1_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextracti32x8 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
   ret <8 x i32> %res
@@ -694,7 +694,7 @@ define <8 x float> @mask_extract_v16f32_v8f32_0(<16 x float> %a, <8 x float> %pa
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> %passthru
   ret <8 x float> %res
@@ -706,7 +706,7 @@ define <8 x float> @mask_extract_v16f32_v8f32_0_z(<16 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> zeroinitializer
   ret <8 x float> %res
@@ -719,7 +719,7 @@ define <8 x float> @mask_extract_v16f32_v8f32_1(<16 x float> %a, <8 x float> %pa
 ; CHECK-NEXT:    vextractf32x8 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovaps %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> %passthru
   ret <8 x float> %res
@@ -731,7 +731,7 @@ define <8 x float> @mask_extract_v16f32_v8f32_1_z(<16 x float> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextractf32x8 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> zeroinitializer
   ret <8 x float> %res
@@ -744,9 +744,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_0(<8 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -758,9 +758,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_0_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -773,9 +773,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_1(<8 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -787,9 +787,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_1_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti64x2 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -802,9 +802,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_2(<8 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 4, i32 5>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -817,9 +817,9 @@ define <2 x i64> @mask_extract_v8i64_v2i64_3(<8 x i64> %a, <2 x i64> %passthru,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -831,9 +831,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_0(<8 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -845,9 +845,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_0_z(<8 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -860,9 +860,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_1(<8 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -874,9 +874,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_1_z(<8 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    vextractf64x2 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -889,9 +889,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_2(<8 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 4, i32 5>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 4, i32 5>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -904,9 +904,9 @@ define <2 x double> @mask_extract_v8f64_v2f64_3(<8 x double> %a, <2 x double> %p
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -917,9 +917,9 @@ define <4 x i64> @mask_extract_v8i64_v4i64_0(<8 x i64> %a, <4 x i64> %passthru,
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -930,9 +930,9 @@ define <4 x i64> @mask_extract_v8i64_v4i64_0_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -944,9 +944,9 @@ define <4 x i64> @mask_extract_v8i64_v4i64_1(<8 x i64> %a, <4 x i64> %passthru,
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -957,9 +957,9 @@ define <4 x i64> @mask_extract_v8i64_v4i64_1_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -970,9 +970,9 @@ define <4 x double> @mask_extract_v8f64_v4f64_0(<8 x double> %a, <4 x double> %p
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> %passthru
   ret <4 x double> %res
 }
@@ -983,9 +983,9 @@ define <4 x double> @mask_extract_v8f64_v4f64_0_z(<8 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovapd %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
@@ -997,9 +997,9 @@ define <4 x double> @mask_extract_v8f64_v4f64_1(<8 x double> %a, <4 x double> %p
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> %passthru
   ret <4 x double> %res
 }
@@ -1010,9 +1010,9 @@ define <4 x double> @mask_extract_v8f64_v4f64_1_z(<8 x double> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
@@ -1023,7 +1023,7 @@ define <8 x i32> @mask_cast_extract_v8i64_v8i32_0(<8 x i64> %a, <8 x i32> %passt
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> %passthru
@@ -1036,7 +1036,7 @@ define <8 x i32> @mask_cast_extract_v8i64_v8i32_0_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> zeroinitializer
@@ -1050,7 +1050,7 @@ define <8 x i32> @mask_cast_extract_v8i64_v8i32_1(<8 x i64> %a, <8 x i32> %passt
 ; CHECK-NEXT:    vextracti32x8 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> %passthru
@@ -1063,7 +1063,7 @@ define <8 x i32> @mask_cast_extract_v8i64_v8i32_1_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextracti32x8 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> zeroinitializer
@@ -1076,7 +1076,7 @@ define <8 x float> @mask_cast_extract_v8f64_v8f32_0(<8 x double> %a, <8 x float>
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> %passthru
@@ -1089,7 +1089,7 @@ define <8 x float> @mask_cast_extract_v8f64_v8f32_0_z(<8 x double> %a, i8 %mask)
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> zeroinitializer
@@ -1103,7 +1103,7 @@ define <8 x float> @mask_cast_extract_v8f64_v8f32_1(<8 x double> %a, <8 x float>
 ; CHECK-NEXT:    vextractf32x8 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovaps %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> %passthru
@@ -1116,7 +1116,7 @@ define <8 x float> @mask_cast_extract_v8f64_v8f32_1_z(<8 x double> %a, i8 %mask)
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextractf32x8 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> zeroinitializer
@@ -1130,10 +1130,10 @@ define <4 x i32> @mask_cast_extract_v8i64_v4i32_0(<8 x i64> %a, <4 x i32> %passt
 ; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
   %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -1145,10 +1145,10 @@ define <4 x i32> @mask_cast_extract_v8i64_v4i32_0_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
   %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -1161,10 +1161,10 @@ define <4 x i32> @mask_cast_extract_v8i64_v4i32_1(<8 x i64> %a, <4 x i32> %passt
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
   %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -1176,10 +1176,10 @@ define <4 x i32> @mask_cast_extract_v8i64_v4i32_1_z(<8 x i64> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti32x4 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
   %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -1191,10 +1191,10 @@ define <4 x float> @mask_cast_extract_v8f64_v4f32_0(<8 x double> %a, <4 x float>
 ; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 0, i32 1>
   %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -1206,10 +1206,10 @@ define <4 x float> @mask_cast_extract_v8f64_v4f32_0_z(<8 x double> %a, i8 %mask)
 ; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 0, i32 1>
   %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -1222,10 +1222,10 @@ define <4 x float> @mask_cast_extract_v8f64_v4f32_1(<8 x double> %a, <4 x float>
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 2, i32 3>
   %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -1237,10 +1237,10 @@ define <4 x float> @mask_cast_extract_v8f64_v4f32_1_z(<8 x double> %a, i8 %mask)
 ; CHECK-NEXT:    vextractf32x4 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 2, i32 3>
   %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -1251,10 +1251,10 @@ define <4 x i64> @mask_cast_extract_v16i32_v4i64_0(<16 x i32> %a, <4 x i64> %pas
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -1265,10 +1265,10 @@ define <4 x i64> @mask_cast_extract_v16i32_v4i64_0_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -1280,10 +1280,10 @@ define <4 x i64> @mask_cast_extract_v16i32_v4i64_1(<16 x i32> %a, <4 x i64> %pas
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> %passthru
   ret <4 x i64> %res
 }
@@ -1294,10 +1294,10 @@ define <4 x i64> @mask_cast_extract_v16i32_v4i64_1_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> zeroinitializer
   ret <4 x i64> %res
 }
@@ -1308,10 +1308,10 @@ define <4 x double> @mask_cast_extract_v16f32_v4f64_0(<16 x float> %a, <4 x doub
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> %passthru
   ret <4 x double> %res
 }
@@ -1322,10 +1322,10 @@ define <4 x double> @mask_cast_extract_v16f32_v4f64_0_z(<16 x float> %a, i8 %mas
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovapd %ymm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
@@ -1337,10 +1337,10 @@ define <4 x double> @mask_cast_extract_v16f32_v4f64_1(<16 x float> %a, <4 x doub
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> %passthru
   ret <4 x double> %res
 }
@@ -1351,10 +1351,10 @@ define <4 x double> @mask_cast_extract_v16f32_v4f64_1_z(<16 x float> %a, i8 %mas
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
@@ -1366,10 +1366,10 @@ define <2 x i64> @mask_cast_extract_v16i32_v2i64_0(<16 x i32> %a, <2 x i64> %pas
 ; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -1381,10 +1381,10 @@ define <2 x i64> @mask_cast_extract_v16i32_v2i64_0_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -1397,10 +1397,10 @@ define <2 x i64> @mask_cast_extract_v16i32_v2i64_1(<16 x i32> %a, <2 x i64> %pas
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> %passthru
   ret <2 x i64> %res
 }
@@ -1412,10 +1412,10 @@ define <2 x i64> @mask_cast_extract_v16i32_v2i64_1_z(<16 x i32> %a, i8 %mask) {
 ; CHECK-NEXT:    vextracti64x2 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> zeroinitializer
   ret <2 x i64> %res
 }
@@ -1427,10 +1427,10 @@ define <2 x double> @mask_cast_extract_v16f32_v2f64_0(<16 x float> %a, <2 x doub
 ; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -1442,10 +1442,10 @@ define <2 x double> @mask_cast_extract_v16f32_v2f64_0_z(<16 x float> %a, i8 %mas
 ; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -1458,10 +1458,10 @@ define <2 x double> @mask_cast_extract_v16f32_v2f64_1(<16 x float> %a, <2 x doub
 ; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -1473,10 +1473,10 @@ define <2 x double> @mask_cast_extract_v16f32_v2f64_1_z(<16 x float> %a, i8 %mas
 ; CHECK-NEXT:    vextractf64x2 $1, %zmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle = shufflevector <16 x float> %a, <16 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -1488,10 +1488,10 @@ define <2 x double> @broadcast_v4f32_0101_from_v2f32_mask(ptr %x, <2 x double> %
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 {%k1} = mem[0,0]
 ; CHECK-NEXT:    retq
   %q = load double, ptr %x, align 1
-  %vecinit.i = insertelement <2 x double> undef, double %q, i32 0
+  %vecinit.i = insertelement <2 x double> poison, double %q, i32 0
   %vecinit2.i = insertelement <2 x double> %vecinit.i, double %q, i32 1
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %vecinit2.i, <2 x double> %passthru
   ret <2 x double> %res
 }
@@ -1503,10 +1503,10 @@ define <2 x double> @broadcast_v4f32_0101_from_v2f32_maskz(ptr %x, i8 %mask) {
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 {%k1} {z} = mem[0,0]
 ; CHECK-NEXT:    retq
   %q = load double, ptr %x, align 1
-  %vecinit.i = insertelement <2 x double> undef, double %q, i32 0
+  %vecinit.i = insertelement <2 x double> poison, double %q, i32 0
   %vecinit2.i = insertelement <2 x double> %vecinit.i, double %q, i32 1
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res = select <2 x i1> %mask.extract, <2 x double> %vecinit2.i, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
@@ -1518,7 +1518,7 @@ define <8 x float> @test_broadcast_2f64_8f32_mask(ptr%p, i8 %mask, <8 x float> %
 ; CHECK-NEXT:    vbroadcastf32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x double>, ptr%p
- %2 = shufflevector <2 x double> %1, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x double> %1, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <4 x double> %2 to <8 x float>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x float> %3, <8 x float> %passthru
@@ -1532,7 +1532,7 @@ define <8 x float> @test_broadcast_2f64_8f32_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x double>, ptr%p
- %2 = shufflevector <2 x double> %1, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x double> %1, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <4 x double> %2 to <8 x float>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x float> %3, <8 x float> zeroinitializer
@@ -1546,7 +1546,7 @@ define <8 x i32> @test_broadcast_2i64_8i32_mask(ptr%p, i8 %mask, <8 x i32> %pass
 ; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x i64>, ptr%p
- %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x i64> %1, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <4 x i64> %2 to <8 x i32>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i32> %3, <8 x i32> %passthru
@@ -1560,7 +1560,7 @@ define <8 x i32> @test_broadcast_2i64_8i32_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x i64>, ptr%p
- %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x i64> %1, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <4 x i64> %2 to <8 x i32>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i32> %3, <8 x i32> zeroinitializer
@@ -1574,7 +1574,7 @@ define <16 x float> @test_broadcast_2f64_16f32_mask(ptr%p, i16 %mask, <16 x floa
 ; CHECK-NEXT:    vbroadcastf32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x double>, ptr%p
- %2 = shufflevector <2 x double> %1, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x double> %1, <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <8 x double> %2 to <16 x float>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> %passthru
@@ -1588,7 +1588,7 @@ define <16 x float> @test_broadcast_2f64_16f32_maskz(ptr%p, i16 %mask) nounwind
 ; CHECK-NEXT:    vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x double>, ptr%p
- %2 = shufflevector <2 x double> %1, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x double> %1, <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <8 x double> %2 to <16 x float>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> zeroinitializer
@@ -1602,7 +1602,7 @@ define <16 x i32> @test_broadcast_2i64_16i32_mask(ptr%p, i16 %mask, <16 x i32> %
 ; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x i64>, ptr%p
- %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x i64> %1, <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <8 x i64> %2 to <16 x i32>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> %passthru
@@ -1616,7 +1616,7 @@ define <16 x i32> @test_broadcast_2i64_16i32_maskz(ptr%p, i16 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <2 x i64>, ptr%p
- %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+ %2 = shufflevector <2 x i64> %1, <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
  %3 = bitcast <8 x i64> %2 to <16 x i32>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> zeroinitializer
@@ -1630,7 +1630,7 @@ define <16 x float> @test_broadcast_4f64_16f32_mask(ptr%p, i16 %mask, <16 x floa
 ; CHECK-NEXT:    vbroadcastf32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
 ; CHECK-NEXT:    retq
  %1 = load <4 x double>, ptr%p
- %2 = shufflevector <4 x double> %1, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x double> %1, <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x double> %2 to <16 x float>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> %passthru
@@ -1644,7 +1644,7 @@ define <16 x float> @test_broadcast_4f64_16f32_maskz(ptr%p, i16 %mask) nounwind
 ; CHECK-NEXT:    vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
 ; CHECK-NEXT:    retq
  %1 = load <4 x double>, ptr%p
- %2 = shufflevector <4 x double> %1, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x double> %1, <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x double> %2 to <16 x float>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> zeroinitializer
@@ -1658,7 +1658,7 @@ define <16 x i32> @test_broadcast_4i64_16i32_mask(ptr%p, i16 %mask, <16 x i32> %
 ; CHECK-NEXT:    vbroadcasti32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i64>, ptr%p
- %2 = shufflevector <4 x i64> %1, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i64> %1, <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x i64> %2 to <16 x i32>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> %passthru
@@ -1672,7 +1672,7 @@ define <16 x i32> @test_broadcast_4i64_16i32_maskz(ptr%p, i16 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i64>, ptr%p
- %2 = shufflevector <4 x i64> %1, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i64> %1, <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x i64> %2 to <16 x i32>
  %mask.cast = bitcast i16 %mask to <16 x i1>
  %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> zeroinitializer
@@ -1686,10 +1686,10 @@ define <4 x double> @test_broadcast_4f32_4f64_mask(ptr%p, i8 %mask, <4 x double>
 ; CHECK-NEXT:    vbroadcastf64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x float>, ptr%p
- %2 = shufflevector <4 x float> %1, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x float> %1, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x float> %2 to <4 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
- %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %res = select <4 x i1> %mask.extract, <4 x double> %3, <4 x double> %passthru
  ret <4 x double> %res
 }
@@ -1701,10 +1701,10 @@ define <4 x double> @test_broadcast_4f32_4f64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x float>, ptr%p
- %2 = shufflevector <4 x float> %1, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x float> %1, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x float> %2 to <4 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
- %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %res = select <4 x i1> %mask.extract, <4 x double> %3, <4 x double> zeroinitializer
  ret <4 x double> %res
 }
@@ -1716,10 +1716,10 @@ define <4 x i64> @test_broadcast_4i32_4i64_mask(ptr%p, i8 %mask, <4 x i64> %pass
 ; CHECK-NEXT:    vbroadcasti64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i32>, ptr%p
- %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i32> %1, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x i32> %2 to <4 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
- %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %res = select <4 x i1> %mask.extract, <4 x i64> %3, <4 x i64> %passthru
  ret <4 x i64> %res
 }
@@ -1731,10 +1731,10 @@ define <4 x i64> @test_broadcast_4i32_4i64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i32>, ptr%p
- %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i32> %1, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <8 x i32> %2 to <4 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
- %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %res = select <4 x i1> %mask.extract, <4 x i64> %3, <4 x i64> zeroinitializer
  ret <4 x i64> %res
 }
@@ -1746,7 +1746,7 @@ define <8 x double> @test_broadcast_4f32_8f64_mask(ptr%p, i8 %mask, <8 x double>
 ; CHECK-NEXT:    vbroadcastf64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x float>, ptr%p
- %2 = shufflevector <4 x float> %1, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x float> %1, <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <16 x float> %2 to <8 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> %passthru
@@ -1760,7 +1760,7 @@ define <8 x double> @test_broadcast_4f32_8f64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x float>, ptr%p
- %2 = shufflevector <4 x float> %1, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x float> %1, <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <16 x float> %2 to <8 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> zeroinitializer
@@ -1774,7 +1774,7 @@ define <8 x i64> @test_broadcast_4i32_8i64_mask(ptr%p, i8 %mask, <8 x i64> %pass
 ; CHECK-NEXT:    vbroadcasti64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i32>, ptr%p
- %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i32> %1, <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <16 x i32> %2 to <8 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> %passthru
@@ -1788,7 +1788,7 @@ define <8 x i64> @test_broadcast_4i32_8i64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
  %1 = load <4 x i32>, ptr%p
- %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <4 x i32> %1, <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
  %3 = bitcast <16 x i32> %2 to <8 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> zeroinitializer
@@ -1802,7 +1802,7 @@ define <8 x double> @test_broadcast_8f32_8f64_mask(ptr%p, i8 %mask, <8 x double>
 ; CHECK-NEXT:    vbroadcastf64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <8 x float>, ptr%p
- %2 = shufflevector <8 x float> %1, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x float> %1, <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %3 = bitcast <16 x float> %2 to <8 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> %passthru
@@ -1816,7 +1816,7 @@ define <8 x double> @test_broadcast_8f32_8f64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <8 x float>, ptr%p
- %2 = shufflevector <8 x float> %1, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x float> %1, <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %3 = bitcast <16 x float> %2 to <8 x double>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> zeroinitializer
@@ -1830,7 +1830,7 @@ define <8 x i64> @test_broadcast_8i32_8i64_mask(ptr%p, i8 %mask, <8 x i64> %pass
 ; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <8 x i32>, ptr%p
- %2 = shufflevector <8 x i32> %1, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x i32> %1, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %3 = bitcast <16 x i32> %2 to <8 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> %passthru
@@ -1844,7 +1844,7 @@ define <8 x i64> @test_broadcast_8i32_8i64_maskz(ptr%p, i8 %mask) nounwind {
 ; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
 ; CHECK-NEXT:    retq
  %1 = load <8 x i32>, ptr%p
- %2 = shufflevector <8 x i32> %1, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x i32> %1, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %3 = bitcast <16 x i32> %2 to <8 x i64>
  %mask.cast = bitcast i8 %mask to <8 x i1>
  %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> zeroinitializer
@@ -1858,9 +1858,9 @@ define <4 x float> @test_broadcastf32x2_v4f32(<4 x float> %vec, <4 x float> %pas
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <4 x float> %vec, <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuf, <4 x float> %passthru
   ret <4 x float> %res
 }
@@ -1872,9 +1872,9 @@ define <4 x float> @test_broadcastf32x2_v4f32_z(<4 x float> %vec, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <4 x float> %vec, <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x float> %shuf, <4 x float> zeroinitializer
   ret <4 x float> %res
 }
@@ -1886,9 +1886,9 @@ define <4 x i32> @test_broadcasti32x2_v4i32(<4 x i32> %vec, <4 x i32> %passthru,
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} xmm1 {%k1} = xmm0[0,1,0,1]
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuf, <4 x i32> %passthru
   ret <4 x i32> %res
 }
@@ -1899,9 +1899,9 @@ define <4 x i32> @test_broadcasti32x2_v4i32_z(<4 x i32> %vec, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,0,1]
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = select <4 x i1> %mask.extract, <4 x i32> %shuf, <4 x i32> zeroinitializer
   ret <4 x i32> %res
 }
@@ -1913,7 +1913,7 @@ define <8 x float> @test_broadcastf32x2_v8f32(<8 x float> %vec, <8 x float> %pas
 ; CHECK-NEXT:    vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <8 x float> %vec, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuf, <8 x float> %passthru
   ret <8 x float> %res
@@ -1925,7 +1925,7 @@ define <8 x float> @test_broadcastf32x2_v8f32_z(<8 x float> %vec, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <8 x float> %vec, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x float> %shuf, <8 x float> zeroinitializer
   ret <8 x float> %res
@@ -1938,7 +1938,7 @@ define <8 x i32> @test_broadcasti32x2_v8i32(<8 x i32> %vec, <8 x i32> %passthru,
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <8 x i32> %vec, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuf, <8 x i32> %passthru
   ret <8 x i32> %res
@@ -1950,7 +1950,7 @@ define <8 x i32> @test_broadcasti32x2_v8i32_z(<8 x i32> %vec, i8 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <8 x i32> %vec, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuf, <8 x i32> zeroinitializer
   ret <8 x i32> %res
@@ -1962,7 +1962,7 @@ define <16 x float> @test_broadcastf32x2_v16f32_z(<16 x float> %vec, i16 %mask)
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <16 x float> %vec, <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x float> %shuf, <16 x float> zeroinitializer
   ret <16 x float> %res
@@ -1975,7 +1975,7 @@ define <16 x i32> @test_broadcasti32x2_v16i32(<16 x i32> %vec, <16 x i32> %passt
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <16 x i32> %vec, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x i32> %shuf, <16 x i32> %passthru
   ret <16 x i32> %res
@@ -1988,7 +1988,7 @@ define <16 x float> @test_broadcastf32x2_v16f32(<16 x float> %vec, <16 x float>
 ; CHECK-NEXT:    vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    vmovapd %zmm1, %zmm0
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <16 x float> %vec, <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x float> %shuf, <16 x float> %passthru
   ret <16 x float> %res
@@ -2000,7 +2000,7 @@ define <16 x i32> @test_broadcasti32x2_v16i32_z(<16 x i32> %vec, i16 %mask) {
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  %shuf = shufflevector <16 x i32> %vec, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %mask.cast, <16 x i32> %shuf, <16 x i32> zeroinitializer
   ret <16 x i32> %res
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index 3c1eb92e9e3c3..2ac2b48af4ce7 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
 
 define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
 ; CHECK-LABEL: vpdpwssd_test:
@@ -11,3 +12,75 @@ define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
   %4 = tail call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)
   ret <16 x i32> %4
 }
+
+define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: vpdpwssd_v16i32_accumulate:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %zmm1, %zmm0, %zmm2
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %x0 = sext <32 x i16> %a0 to <32 x i32>
+  %x1 = sext <32 x i16> %a1 to <32 x i32>
+  %m = mul nsw <32 x i32> %x0, %x1
+  %lo = shufflevector <32 x i32> %m, <32 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %hi = shufflevector <32 x i32> %m, <32 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  %r0 = add <16 x i32> %lo, %a2
+  %r1 = add <16 x i32> %r0, %hi
+  ret <16 x i32> %r1
+}
+
+define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) {
+; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI:       # %bb.0:
+; AVX512VL-VNNI-NEXT:    vpdpwssd %ymm1, %ymm0, %ymm2
+; AVX512VL-VNNI-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512VL-VNNI-NEXT:    retq
+;
+; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX-VNNI:       # %bb.0:
+; AVX-VNNI-NEXT:    {vex} vpdpwssd %ymm1, %ymm0, %ymm2
+; AVX-VNNI-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX-VNNI-NEXT:    retq
+;
+; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512-VNNI:       # %bb.0:
+; AVX512-VNNI-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX512-VNNI-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-VNNI-NEXT:    retq
+  %x0 = sext <16 x i16> %a0 to <16 x i32>
+  %x1 = sext <16 x i16> %a1 to <16 x i32>
+  %m = mul nsw <16 x i32> %x0, %x1
+  %lo = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %hi = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %r0 = add <8 x i32> %hi, %a2
+  %r1 = add <8 x i32> %lo, %r0
+  ret <8 x i32> %r1
+}
+
+define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
+; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
+; AVX512VL-VNNI:       # %bb.0:
+; AVX512VL-VNNI-NEXT:    vpdpwssd %xmm1, %xmm0, %xmm2
+; AVX512VL-VNNI-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX512VL-VNNI-NEXT:    retq
+;
+; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
+; AVX-VNNI:       # %bb.0:
+; AVX-VNNI-NEXT:    {vex} vpdpwssd %xmm1, %xmm0, %xmm2
+; AVX-VNNI-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX-VNNI-NEXT:    retq
+;
+; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
+; AVX512-VNNI:       # %bb.0:
+; AVX512-VNNI-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX512-VNNI-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-VNNI-NEXT:    retq
+  %x0 = sext <8 x i16> %a0 to <8 x i32>
+  %x1 = sext <8 x i16> %a1 to <8 x i32>
+  %m = mul nsw <8 x i32> %x0, %x1
+  %lo = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %hi = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %r0 = add <4 x i32> %lo, %a2
+  %r1 = add <4 x i32> %hi, %r0
+  ret <4 x i32> %r1
+}
diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll
index 9acd995d612c3..be6ee8f689958 100644
--- a/llvm/test/CodeGen/X86/vselect.ll
+++ b/llvm/test/CodeGen/X86/vselect.ll
@@ -796,3 +796,29 @@ define i64 @vselect_any_extend_vector_inreg_crash(ptr %x) {
   ret i64 %4
 }
 
+; Tests the scalarizeBinOp code in DAGCombiner
+define void @scalarize_binop(<1 x i1> %a) {
+; SSE-LABEL: scalarize_binop:
+; SSE:       # %bb.0: # %bb0
+; SSE-NEXT:    .p2align 4
+; SSE-NEXT:  .LBB35_1: # %bb1
+; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
+; SSE-NEXT:    jmp .LBB35_1
+;
+; AVX-LABEL: scalarize_binop:
+; AVX:       # %bb.0: # %bb0
+; AVX-NEXT:    .p2align 4
+; AVX-NEXT:  .LBB35_1: # %bb1
+; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX-NEXT:    jmp .LBB35_1
+bb0:
+  br label %bb1
+
+bb1:
+  %b = select <1 x i1> %a, <1 x i1> zeroinitializer, <1 x i1> splat (i1 true)
+  br label %bb2
+
+bb2:
+  %c = extractelement <1 x i1> %b, i32 0
+  br label %bb1
+}
diff --git a/llvm/test/CodeGen/X86/widen_conv-3.ll b/llvm/test/CodeGen/X86/widen_conv-3.ll
index 5887834f9af2f..f9b588b8b8915 100644
--- a/llvm/test/CodeGen/X86/widen_conv-3.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-3.ll
@@ -10,7 +10,7 @@ define void @convert_v2i16_to_v2f32(ptr %dst.addr, <2 x i16> %src) nounwind {
 ; X86-SSE2-LABEL: convert_v2i16_to_v2f32:
 ; X86-SSE2:       # %bb.0: # %entry
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X86-SSE2-NEXT:    psrad $16, %xmm0
 ; X86-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
@@ -26,7 +26,7 @@ define void @convert_v2i16_to_v2f32(ptr %dst.addr, <2 x i16> %src) nounwind {
 ;
 ; X64-SSE2-LABEL: convert_v2i16_to_v2f32:
 ; X64-SSE2:       # %bb.0: # %entry
-; X64-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X64-SSE2-NEXT:    psrad $16, %xmm0
 ; X64-SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X64-SSE2-NEXT:    movlps %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/widen_shuffle-1.ll b/llvm/test/CodeGen/X86/widen_shuffle-1.ll
index 925b96f5e346c..3257936f62e3b 100644
--- a/llvm/test/CodeGen/X86/widen_shuffle-1.ll
+++ b/llvm/test/CodeGen/X86/widen_shuffle-1.ll
@@ -71,16 +71,16 @@ define void @shuf3(<4 x float> %tmp10, <4 x float> %vecinit15, ptr %dst) nounwin
 ; X64-NEXT:    retq
 entry:
   %shuffle.i.i.i12 = shufflevector <4 x float> %tmp10, <4 x float> %vecinit15, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-  %tmp25.i.i = shufflevector <4 x float> %shuffle.i.i.i12, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %tmp25.i.i = shufflevector <4 x float> %shuffle.i.i.i12, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %tmp1.i.i = shufflevector <3 x float> %tmp25.i.i, <3 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %tmp3.i13 = shufflevector <4 x float> %tmp1.i.i, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2> ; <<3 x float>>
+  %tmp3.i13 = shufflevector <4 x float> %tmp1.i.i, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2> ; <<3 x float>>
   %tmp6.i14 = shufflevector <3 x float> %tmp3.i13, <3 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %tmp97.i = shufflevector <4 x float> %tmp6.i14, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
-  %tmp2.i18 = shufflevector <3 x float> %tmp97.i, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %tmp97.i = shufflevector <4 x float> %tmp6.i14, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  %tmp2.i18 = shufflevector <3 x float> %tmp97.i, <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %t5 = bitcast <4 x float> %tmp2.i18 to <4 x i32>
   %shr.i.i19 = lshr <4 x i32> %t5, <i32 19, i32 19, i32 19, i32 19>
   %and.i.i20 = and <4 x i32> %shr.i.i19, <i32 4080, i32 4080, i32 4080, i32 4080>
-  %shuffle.i.i.i21 = shufflevector <4 x float> %tmp2.i18, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+  %shuffle.i.i.i21 = shufflevector <4 x float> %tmp2.i18, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
   store <4 x float> %shuffle.i.i.i21, ptr %dst
   ret void
 }
@@ -114,7 +114,7 @@ define void @shuf5(ptr %p) nounwind {
 ; X64-NEXT:    movabsq $2387225703656530209, %rax # imm = 0x2121212121212121
 ; X64-NEXT:    movq %rax, (%rdi)
 ; X64-NEXT:    retq
-  %v = shufflevector <2 x i8> <i8 4, i8 33>, <2 x i8> undef, <8 x i32> <i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %v = shufflevector <2 x i8> <i8 4, i8 33>, <2 x i8> poison, <8 x i32> <i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   store <8 x i8> %v, ptr %p, align 8
   ret void
 }
diff --git a/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir b/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
index 97ff7c4589b44..b8628ac6ae87d 100644
--- a/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
+++ b/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
@@ -1,4 +1,4 @@
-# RUN: llc -run-pass=livedebugvalues -march=mips -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -mtriple=mips -o - %s | FileCheck %s
 #
 #"last-instr-bundled.c"
 #extern void set_cond(int, int*);
diff --git a/llvm/test/DebugInfo/Mips/tls.ll b/llvm/test/DebugInfo/Mips/tls.ll
index baf346fe035ba..927966d514078 100644
--- a/llvm/test/DebugInfo/Mips/tls.ll
+++ b/llvm/test/DebugInfo/Mips/tls.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -march=mips -mcpu=mips32r2 -filetype=asm < %s | FileCheck %s -check-prefix=CHECK-WORD
-; RUN: llc -O0 -march=mips64 -mcpu=mips64r2 -filetype=asm < %s | FileCheck %s -check-prefix=CHECK-DWORD
+; RUN: llc -O0 -mtriple=mips -mcpu=mips32r2 < %s | FileCheck %s -check-prefix=CHECK-WORD
+; RUN: llc -O0 -mtriple=mips64 -mcpu=mips64r2 < %s | FileCheck %s -check-prefix=CHECK-DWORD
 
 @x = thread_local global i32 5, align 4, !dbg !0
 
diff --git a/llvm/test/DebugInfo/XCOFF/empty.ll b/llvm/test/DebugInfo/XCOFF/empty.ll
index 660a1fac6af89..af2f74fb82b8f 100644
--- a/llvm/test/DebugInfo/XCOFF/empty.ll
+++ b/llvm/test/DebugInfo/XCOFF/empty.ll
@@ -37,6 +37,7 @@ entry:
 ; ASM32:               .file   "1.c"
 ; ASM32-NEXT:          .csect ..text..[PR],5
 ; ASM32-NEXT:          .rename ..text..[PR],""
+; ASM32-NEXT:          .machine "COM"
 ; ASM32-NEXT:          .globl  main[DS]                        # -- Begin function main
 ; ASM32-NEXT:          .globl  .main
 ; ASM32-NEXT:          .align  2
@@ -239,6 +240,7 @@ entry:
 ; ASM64:               .file   "1.c"
 ; ASM64-NEXT:          .csect ..text..[PR],5
 ; ASM64-NEXT:          .rename ..text..[PR],""
+; ASM64-NEXT:          .machine "COM"
 ; ASM64-NEXT:          .globl  main[DS]                        # -- Begin function main
 ; ASM64-NEXT:          .globl  .main
 ; ASM64-NEXT:          .align  2
diff --git a/llvm/test/DebugInfo/XCOFF/explicit-section.ll b/llvm/test/DebugInfo/XCOFF/explicit-section.ll
index 345f6f117321b..0ae9289c08df3 100644
--- a/llvm/test/DebugInfo/XCOFF/explicit-section.ll
+++ b/llvm/test/DebugInfo/XCOFF/explicit-section.ll
@@ -45,6 +45,7 @@ entry:
 ; CHECK:               .file   "2.c"
 ; CHECK-NEXT:          .csect ..text..[PR],5
 ; CHECK-NEXT:          .rename ..text..[PR],""
+; CHECK-NEXT:          .machine "COM"
 ; CHECK-NEXT:          .globl  bar[DS]                         # -- Begin function bar
 ; CHECK-NEXT:          .globl  .bar
 ; CHECK-NEXT:          .align  2
diff --git a/llvm/test/DebugInfo/XCOFF/function-sections.ll b/llvm/test/DebugInfo/XCOFF/function-sections.ll
index 6b0bf6be7a252..6a86ae6796fbc 100644
--- a/llvm/test/DebugInfo/XCOFF/function-sections.ll
+++ b/llvm/test/DebugInfo/XCOFF/function-sections.ll
@@ -40,6 +40,7 @@ entry:
 ; CHECK:               .file   "1.c"
 ; CHECK-NEXT:          .csect ..text..[PR],5
 ; CHECK-NEXT:          .rename ..text..[PR],""
+; CHECK-NEXT:          .machine "COM"
 ; CHECK-NEXT:          .csect .foo[PR],5
 ; CHECK-NEXT:          .globl  foo[DS]                         # -- Begin function foo
 ; CHECK-NEXT:          .globl  .foo[PR]
diff --git a/llvm/test/Instrumentation/HeapProfiler/memprof-options.ll b/llvm/test/Instrumentation/HeapProfiler/memprof-options.ll
new file mode 100644
index 0000000000000..8e82d524952dd
--- /dev/null
+++ b/llvm/test/Instrumentation/HeapProfiler/memprof-options.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes='function(memprof),memprof-module' -S | FileCheck %s --check-prefixes=CHECK,EMPTY
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes='function(memprof),memprof-module' -S -memprof-runtime-default-options="verbose=1" | FileCheck %s --check-prefixes=CHECK,VERBOSE
+
+define i32 @main() {
+entry:
+  ret i32 0
+}
+
+; CHECK: $__memprof_default_options_str = comdat any
+; EMPTY: @__memprof_default_options_str = constant [1 x i8] zeroinitializer, comdat
+; VERBOSE: @__memprof_default_options_str = constant [10 x i8] c"verbose=1\00", comdat
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vmul.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vmul.ll
new file mode 100644
index 0000000000000..38d6669671509
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/neon_vmul.ll
@@ -0,0 +1,6173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=msan -S | FileCheck %s
+;
+; Test memory sanitizer instrumentation for Arm vector multiplication
+; instructions.
+;
+; Forked from llvm/test/CodeGen/AArch64/arm64-vmul.ll
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android9001"
+
+define <8 x i16> @smull8h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @smull8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <8 x i8> [[_MSPROP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %temp1 = load <8 x i8>, ptr %A
+  %temp2 = load <8 x i8>, ptr %B
+  %temp3 = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %temp1, <8 x i8> %temp2)
+  ret <8 x i16> %temp3
+}
+
+define <4 x i32> @smull4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smull4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define <2 x i64> @smull2d(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smull2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i64> %temp3
+}
+
+declare <8 x i16>  @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
+
+define <8 x i16> @umull8h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @umull8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <8 x i8> [[_MSPROP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %temp1 = load <8 x i8>, ptr %A
+  %temp2 = load <8 x i8>, ptr %B
+  %temp3 = call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %temp1, <8 x i8> %temp2)
+  ret <8 x i16> %temp3
+}
+
+define <4 x i32> @umull4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umull4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define <2 x i64> @umull2d(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umull2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i64> %temp3
+}
+
+declare <8 x i16>  @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
+
+define <4 x i32> @sqdmull4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmull4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB15:.*]], label %[[BB16:.*]], !prof [[PROF1]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define <2 x i64> @sqdmull2d(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmull2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB15:.*]], label %[[BB16:.*]], !prof [[PROF1]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i64> %temp3
+}
+
+define <4 x i32> @sqdmull2_4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmull2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[LOAD1]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i16> [[_MSLD1]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[LOAD2]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[_MSPROP2]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB15:.*]], label %[[BB16:.*]], !prof [[PROF1]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %load1 = load <8 x i16>, ptr %A
+  %load2 = load <8 x i16>, ptr %B
+  %temp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define <2 x i64> @sqdmull2_2d(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmull2_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[LOAD1]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <4 x i32> [[_MSLD1]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[LOAD2]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[_MSPROP2]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB15:.*]], label %[[BB16:.*]], !prof [[PROF1]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %load1 = load <4 x i32>, ptr %A
+  %load2 = load <4 x i32>, ptr %B
+  %temp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp3 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i64> %temp3
+}
+
+
+declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
+
+define <8 x i16> @pmull8h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @pmull8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i8>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <8 x i8> [[_MSPROP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %temp1 = load <8 x i8>, ptr %A
+  %temp2 = load <8 x i8>, ptr %B
+  %temp3 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %temp1, <8 x i8> %temp2)
+  ret <8 x i16> %temp3
+}
+
+declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
+
+define <4 x i16> @sqdmulh_4h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i16> @sqdmulh_4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i16> %temp3
+}
+
+define <8 x i16> @sqdmulh_8h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @sqdmulh_8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %temp1 = load <8 x i16>, ptr %A
+  %temp2 = load <8 x i16>, ptr %B
+  %temp3 = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %temp1, <8 x i16> %temp2)
+  ret <8 x i16> %temp3
+}
+
+define <2 x i32> @sqdmulh_2s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i32> @sqdmulh_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i32> %temp3
+}
+
+define <4 x i32> @sqdmulh_4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmulh_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %temp1 = load <4 x i32>, ptr %A
+  %temp2 = load <4 x i32>, ptr %B
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %temp1, <4 x i32> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define i32 @sqdmulh_1s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmulh_1s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 [[TMP1]], i32 [[TMP2]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %temp1 = load i32, ptr %A
+  %temp2 = load i32, ptr %B
+  %temp3 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %temp1, i32 %temp2)
+  ret i32 %temp3
+}
+
+declare <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare i32 @llvm.aarch64.neon.sqdmulh.i32(i32, i32) nounwind readnone
+
+define <4 x i16> @sqrdmulh_4h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i16> @sqrdmulh_4h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP3]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i16> %temp3
+}
+
+define <8 x i16> @sqrdmulh_8h(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @sqrdmulh_8h(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %temp1 = load <8 x i16>, ptr %A
+  %temp2 = load <8 x i16>, ptr %B
+  %temp3 = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %temp1, <8 x i16> %temp2)
+  ret <8 x i16> %temp3
+}
+
+define <2 x i32> @sqrdmulh_2s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i32> @sqrdmulh_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i32> %temp3
+}
+
+define <4 x i32> @sqrdmulh_4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqrdmulh_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %temp1 = load <4 x i32>, ptr %A
+  %temp2 = load <4 x i32>, ptr %B
+  %temp3 = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %temp1, <4 x i32> %temp2)
+  ret <4 x i32> %temp3
+}
+
+define i32 @sqrdmulh_1s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqrdmulh_1s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 [[TMP1]], i32 [[TMP2]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %temp1 = load i32, ptr %A
+  %temp2 = load i32, ptr %B
+  %temp3 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %temp1, i32 %temp2)
+  ret i32 %temp3
+}
+
+declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
+declare i32 @llvm.aarch64.neon.sqrdmulh.i32(i32, i32) nounwind readnone
+
+define <2 x float> @fmulx_2s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmulx_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> [[TMP1]], <2 x float> [[TMP2]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP3]]
+;
+  %temp1 = load <2 x float>, ptr %A
+  %temp2 = load <2 x float>, ptr %B
+  %temp3 = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %temp1, <2 x float> %temp2)
+  ret <2 x float> %temp3
+}
+
+define <4 x float> @fmulx_4s(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmulx_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %temp1 = load <4 x float>, ptr %A
+  %temp2 = load <4 x float>, ptr %B
+  %temp3 = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %temp1, <4 x float> %temp2)
+  ret <4 x float> %temp3
+}
+
+define <2 x double> @fmulx_2d(ptr %A, ptr %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmulx_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 193514046488576
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %temp1 = load <2 x double>, ptr %A
+  %temp2 = load <2 x double>, ptr %B
+  %temp3 = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %temp1, <2 x double> %temp2)
+  ret <2 x double> %temp3
+}
+
+declare <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float>, <2 x float>) nounwind readnone
+declare <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float>, <4 x float>) nounwind readnone
+declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>) nounwind readnone
+
+define <4 x i32> @smlal4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smlal4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = add <4 x i32> %temp3, %temp4
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @smlal2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smlal2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = add <2 x i64> %temp3, %temp4
+  ret <2 x i64> %temp5
+}
+
+define void @smlal8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {
+; CHECK-LABEL: define void @smlal8h_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <8 x i8> [[V1:%.*]], <8 x i8> [[V2:%.*]], <8 x i8> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i8> [[V3]], splat (i8 -1)
+; CHECK-NEXT:    [[SMULL_1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[V1]], <8 x i8> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <8 x i16> [[SMULL_1]], splat (i16 257)
+; CHECK-NEXT:    [[SMULL_2:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[V2]], <8 x i8> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <8 x i16> [[ADD_1]], [[SMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <8 x i16> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %smull.1 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v1, <8 x i8> %v3)
+  %add.1 = add <8 x i16> %smull.1, <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>
+  %smull.2 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v2, <8 x i8> %xor)
+  %add.2 = add <8 x i16> %add.1, %smull.2
+  store <8 x i16> %add.2, ptr %dst
+  ret void
+}
+
+define void @smlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {
+; CHECK-LABEL: define void @smlal2d_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <2 x i32> [[V1:%.*]], <2 x i32> [[V2:%.*]], <2 x i32> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[V3]], splat (i32 -1)
+; CHECK-NEXT:    [[SMULL_1:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[V1]], <2 x i32> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <2 x i64> [[SMULL_1]], splat (i64 257)
+; CHECK-NEXT:    [[SMULL_2:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[V2]], <2 x i32> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <2 x i64> [[ADD_1]], [[SMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <2 x i64> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>
+  %smull.1 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v1, <2 x i32> %v3)
+  %add.1 = add <2 x i64> %smull.1, <i64 257, i64 257>
+  %smull.2 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v2, <2 x i32> %xor)
+  %add.2 = add <2 x i64> %add.1, %smull.2
+  store <2 x i64> %add.2, ptr %dst
+  ret void
+}
+
+define <4 x i32> @smlsl4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smlsl4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = sub <4 x i32> %temp3, %temp4
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @smlsl2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smlsl2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = sub <2 x i64> %temp3, %temp4
+  ret <2 x i64> %temp5
+}
+
+define void @smlsl8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {
+; CHECK-LABEL: define void @smlsl8h_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <8 x i8> [[V1:%.*]], <8 x i8> [[V2:%.*]], <8 x i8> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i8> [[V3]], splat (i8 -1)
+; CHECK-NEXT:    [[SMULL_1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[V1]], <8 x i8> [[V3]])
+; CHECK-NEXT:    [[SUB_1:%.*]] = sub <8 x i16> splat (i16 257), [[SMULL_1]]
+; CHECK-NEXT:    [[SMULL_2:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[V2]], <8 x i8> [[XOR]])
+; CHECK-NEXT:    [[SUB_2:%.*]] = sub <8 x i16> [[SUB_1]], [[SMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <8 x i16> [[SUB_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %smull.1 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v1, <8 x i8> %v3)
+  %sub.1 = sub <8 x i16> <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>, %smull.1
+  %smull.2 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v2, <8 x i8> %xor)
+  %sub.2 = sub <8 x i16> %sub.1, %smull.2
+  store <8 x i16> %sub.2, ptr %dst
+  ret void
+}
+
+define void @smlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {
+; CHECK-LABEL: define void @smlsl2d_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <2 x i32> [[V1:%.*]], <2 x i32> [[V2:%.*]], <2 x i32> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[V3]], splat (i32 -1)
+; CHECK-NEXT:    [[SMULL_1:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[V1]], <2 x i32> [[V3]])
+; CHECK-NEXT:    [[SUB_1:%.*]] = sub <2 x i64> splat (i64 257), [[SMULL_1]]
+; CHECK-NEXT:    [[SMULL_2:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[V2]], <2 x i32> [[XOR]])
+; CHECK-NEXT:    [[SUB_2:%.*]] = sub <2 x i64> [[SUB_1]], [[SMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <2 x i64> [[SUB_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>
+  %smull.1 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v1, <2 x i32> %v3)
+  %sub.1 = sub <2 x i64> <i64 257, i64 257>, %smull.1
+  %smull.2 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v2, <2 x i32> %xor)
+  %sub.2 = sub <2 x i64> %sub.1, %smull.2
+  store <2 x i64> %sub.2, ptr %dst
+  ret void
+}
+
+declare <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)
+
+define <4 x i32> @sqdmlal4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlal4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i16> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP4]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %temp3, <4 x i32> %temp4)
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @sqdmlal2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlal2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i32> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i32> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %temp3, <2 x i64> %temp4)
+  ret <2 x i64> %temp5
+}
+
+define <4 x i32> @sqdmlal2_4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlal2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[LOAD1]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i16> [[_MSLD1]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[LOAD2]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i16> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP4]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %load1 = load <8 x i16>, ptr %A
+  %load2 = load <8 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %temp3, <4 x i32> %temp4)
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @sqdmlal2_2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlal2_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[LOAD1]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <4 x i32> [[_MSLD1]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[LOAD2]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i32> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %load1 = load <4 x i32>, ptr %A
+  %load2 = load <4 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %temp3, <2 x i64> %temp4)
+  ret <2 x i64> %temp5
+}
+
+define <4 x i32> @sqdmlsl4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlsl4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i16> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i16> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP4]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %temp3, <4 x i32> %temp4)
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @sqdmlsl2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlsl2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i32> [[_MSLD]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i32> [[_MSLD1]] to i64
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %temp3, <2 x i64> %temp4)
+  ret <2 x i64> %temp5
+}
+
+define <4 x i32> @sqdmlsl2_4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlsl2_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <8 x i16>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <8 x i16>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[LOAD1]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i16> [[_MSLD1]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[LOAD2]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x i16> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP4]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %load1 = load <8 x i16>, ptr %A
+  %load2 = load <8 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %temp3, <4 x i32> %temp4)
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @sqdmlsl2_2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlsl2_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <4 x i32>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[LOAD1]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <4 x i32> [[_MSLD1]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[LOAD2]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <2 x i32> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %load1 = load <4 x i32>, ptr %A
+  %load2 = load <4 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %temp3, <2 x i64> %temp4)
+  ret <2 x i64> %temp5
+}
+
+define <4 x i32> @umlal4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umlal4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = add <4 x i32> %temp3, %temp4
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @umlal2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umlal2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = add <2 x i64> %temp3, %temp4
+  ret <2 x i64> %temp5
+}
+
+define void @umlal8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {
+; CHECK-LABEL: define void @umlal8h_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <8 x i8> [[V1:%.*]], <8 x i8> [[V2:%.*]], <8 x i8> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i8> [[V3]], splat (i8 -1)
+; CHECK-NEXT:    [[UMULL_1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[V1]], <8 x i8> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <8 x i16> [[UMULL_1]], splat (i16 257)
+; CHECK-NEXT:    [[UMULL_2:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[V2]], <8 x i8> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <8 x i16> [[ADD_1]], [[UMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <8 x i16> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %umull.1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v1, <8 x i8> %v3)
+  %add.1 = add <8 x i16> %umull.1, <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>
+  %umull.2 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v2, <8 x i8> %xor)
+  %add.2 = add <8 x i16> %add.1, %umull.2
+  store <8 x i16> %add.2, ptr %dst
+  ret void
+}
+
+define void @umlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {
+; CHECK-LABEL: define void @umlal2d_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <2 x i32> [[V1:%.*]], <2 x i32> [[V2:%.*]], <2 x i32> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[V3]], splat (i32 -1)
+; CHECK-NEXT:    [[UMULL_1:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[V1]], <2 x i32> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = add <2 x i64> [[UMULL_1]], splat (i64 257)
+; CHECK-NEXT:    [[UMULL_2:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[V2]], <2 x i32> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = add <2 x i64> [[ADD_1]], [[UMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <2 x i64> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>
+  %umull.1 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v1, <2 x i32> %v3)
+  %add.1 = add <2 x i64> %umull.1, <i64 257, i64 257>
+  %umull.2 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v2, <2 x i32> %xor)
+  %add.2 = add <2 x i64> %add.1, %umull.2
+  store <2 x i64> %add.2, ptr %dst
+  ret void
+}
+
+define <4 x i32> @umlsl4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umlsl4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+;
+  %temp1 = load <4 x i16>, ptr %A
+  %temp2 = load <4 x i16>, ptr %B
+  %temp3 = load <4 x i32>, ptr %C
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp5 = sub <4 x i32> %temp3, %temp4
+  ret <4 x i32> %temp5
+}
+
+define <2 x i64> @umlsl2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umlsl2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSLD2]], [[TMP19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %temp1 = load <2 x i32>, ptr %A
+  %temp2 = load <2 x i32>, ptr %B
+  %temp3 = load <2 x i64>, ptr %C
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp5 = sub <2 x i64> %temp3, %temp4
+  ret <2 x i64> %temp5
+}
+
+define void @umlsl8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {
+; CHECK-LABEL: define void @umlsl8h_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <8 x i8> [[V1:%.*]], <8 x i8> [[V2:%.*]], <8 x i8> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i8> [[V3]], splat (i8 -1)
+; CHECK-NEXT:    [[UMULL_1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[V1]], <8 x i8> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = sub <8 x i16> splat (i16 257), [[UMULL_1]]
+; CHECK-NEXT:    [[UMULL_2:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[V2]], <8 x i8> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = sub <8 x i16> [[ADD_1]], [[UMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <8 x i16> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %umull.1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v1, <8 x i8> %v3)
+  %add.1 = sub <8 x i16> <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>, %umull.1
+  %umull.2 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v2, <8 x i8> %xor)
+  %add.2 = sub <8 x i16> %add.1, %umull.2
+  store <8 x i16> %add.2, ptr %dst
+  ret void
+}
+
+define void @umlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {
+; CHECK-LABEL: define void @umlsl2d_chain_with_constant(
+; CHECK-SAME: ptr [[DST:%.*]], <2 x i32> [[V1:%.*]], <2 x i32> [[V2:%.*]], <2 x i32> [[V3:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[V3]], splat (i32 -1)
+; CHECK-NEXT:    [[UMULL_1:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[V1]], <2 x i32> [[V3]])
+; CHECK-NEXT:    [[ADD_1:%.*]] = sub <2 x i64> splat (i64 257), [[UMULL_1]]
+; CHECK-NEXT:    [[UMULL_2:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[V2]], <2 x i32> [[XOR]])
+; CHECK-NEXT:    [[ADD_2:%.*]] = sub <2 x i64> [[ADD_1]], [[UMULL_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 193514046488576
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP3]], align 16
+; CHECK-NEXT:    store <2 x i64> [[ADD_2]], ptr [[DST]], align 16
+; CHECK-NEXT:    ret void
+;
+  %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>
+  %umull.1 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v1, <2 x i32> %v3)
+  %add.1 = sub <2 x i64> <i64 257, i64 257>, %umull.1
+  %umull.2 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v2, <2 x i32> %xor)
+  %add.2 = sub <2 x i64> %add.1, %umull.2
+  store <2 x i64> %add.2, ptr %dst
+  ret void
+}
+
+define <2 x float> @fmla_2s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmla_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i32>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[TMP1]], <2 x float> [[TMP2]], <2 x float> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP4]]
+;
+  %temp1 = load <2 x float>, ptr %A
+  %temp2 = load <2 x float>, ptr %B
+  %temp3 = load <2 x float>, ptr %C
+  %temp4 = call <2 x float> @llvm.fma.v2f32(<2 x float> %temp1, <2 x float> %temp2, <2 x float> %temp3)
+  ret <2 x float> %temp4
+}
+
+define <4 x float> @fmla_4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmla_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %temp1 = load <4 x float>, ptr %A
+  %temp2 = load <4 x float>, ptr %B
+  %temp3 = load <4 x float>, ptr %C
+  %temp4 = call <4 x float> @llvm.fma.v4f32(<4 x float> %temp1, <4 x float> %temp2, <4 x float> %temp3)
+  ret <4 x float> %temp4
+}
+
+define <2 x double> @fmla_2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmla_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP2]], <2 x double> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %temp1 = load <2 x double>, ptr %A
+  %temp2 = load <2 x double>, ptr %B
+  %temp3 = load <2 x double>, ptr %C
+  %temp4 = call <2 x double> @llvm.fma.v2f64(<2 x double> %temp1, <2 x double> %temp2, <2 x double> %temp3)
+  ret <2 x double> %temp4
+}
+
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) nounwind readnone
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
+
+define <2 x float> @fmls_2s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmls_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i32>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x float> splat (float -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i32> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[TMP1]], <2 x float> [[TMP4]], <2 x float> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP5]]
+;
+  %temp1 = load <2 x float>, ptr %A
+  %temp2 = load <2 x float>, ptr %B
+  %temp3 = load <2 x float>, ptr %C
+  %temp4 = fsub <2 x float> <float -0.0, float -0.0>, %temp2
+  %temp5 = call <2 x float> @llvm.fma.v2f32(<2 x float> %temp1, <2 x float> %temp4, <2 x float> %temp3)
+  ret <2 x float> %temp5
+}
+
+define <4 x float> @fmls_4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmls_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP4]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP5]]
+;
+  %temp1 = load <4 x float>, ptr %A
+  %temp2 = load <4 x float>, ptr %B
+  %temp3 = load <4 x float>, ptr %C
+  %temp4 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %temp2
+  %temp5 = call <4 x float> @llvm.fma.v4f32(<4 x float> %temp1, <4 x float> %temp4, <4 x float> %temp3)
+  ret <4 x float> %temp5
+}
+
+define <2 x double> @fmls_2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmls_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSLD]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP4]], <2 x double> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP5]]
+;
+  %temp1 = load <2 x double>, ptr %A
+  %temp2 = load <2 x double>, ptr %B
+  %temp3 = load <2 x double>, ptr %C
+  %temp4 = fsub <2 x double> <double -0.0, double -0.0>, %temp2
+  %temp5 = call <2 x double> @llvm.fma.v2f64(<2 x double> %temp1, <2 x double> %temp4, <2 x double> %temp3)
+  ret <2 x double> %temp5
+}
+
+define <2 x float> @fmls_commuted_neg_2s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmls_commuted_neg_2s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i32>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x float> splat (float -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i32> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP1]], <2 x float> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP5]]
+;
+  %temp1 = load <2 x float>, ptr %A
+  %temp2 = load <2 x float>, ptr %B
+  %temp3 = load <2 x float>, ptr %C
+  %temp4 = fsub <2 x float> <float -0.0, float -0.0>, %temp2
+  %temp5 = call <2 x float> @llvm.fma.v2f32(<2 x float> %temp4, <2 x float> %temp1, <2 x float> %temp3)
+  ret <2 x float> %temp5
+}
+
+define <4 x float> @fmls_commuted_neg_4s(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmls_commuted_neg_4s(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <4 x i32>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP4]], <4 x float> [[TMP1]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP5]]
+;
+  %temp1 = load <4 x float>, ptr %A
+  %temp2 = load <4 x float>, ptr %B
+  %temp3 = load <4 x float>, ptr %C
+  %temp4 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %temp2
+  %temp5 = call <4 x float> @llvm.fma.v4f32(<4 x float> %temp4, <4 x float> %temp1, <4 x float> %temp3)
+  ret <4 x float> %temp5
+}
+
+define <2 x double> @fmls_commuted_neg_2d(ptr %A, ptr %B, ptr %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmls_commuted_neg_2d(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 193514046488576
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[B]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], 193514046488576
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, ptr [[C]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP16]], 193514046488576
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    [[_MSLD2:%.*]] = load <2 x i64>, ptr [[TMP18]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[_MSLD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSLD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP4]], <2 x double> [[TMP1]], <2 x double> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP5]]
+;
+  %temp1 = load <2 x double>, ptr %A
+  %temp2 = load <2 x double>, ptr %B
+  %temp3 = load <2 x double>, ptr %C
+  %temp4 = fsub <2 x double> <double -0.0, double -0.0>, %temp2
+  %temp5 = call <2 x double> @llvm.fma.v2f64(<2 x double> %temp4, <2 x double> %temp1, <2 x double> %temp3)
+  ret <2 x double> %temp5
+}
+
+define <2 x float> @fmls_indexed_2s(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x float> @fmls_indexed_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <2 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x float> [[B]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <2 x float> @llvm.fma.v2f32(<2 x float> [[TMP0]], <2 x float> [[LANE]], <2 x float> [[A]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[FMLS1]]
+;
+entry:
+  %0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %c
+  %lane = shufflevector <2 x float> %b, <2 x float> undef, <2 x i32> zeroinitializer
+  %fmls1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %0, <2 x float> %lane, <2 x float> %a)
+  ret <2 x float> %fmls1
+}
+
+define <4 x float> @fmls_indexed_4s(<4 x float> %a, <4 x float> %b, <4 x float> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x float> @fmls_indexed_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP0]], <4 x float> [[LANE]], <4 x float> [[A]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[FMLS1]]
+;
+entry:
+  %0 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %lane = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
+  %fmls1 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %0, <4 x float> %lane, <4 x float> %a)
+  ret <4 x float> %fmls1
+}
+
+define <2 x double> @fmls_indexed_2d(<2 x double> %a, <2 x double> %b, <2 x double> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x double> @fmls_indexed_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x double> [[B]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP0]], <2 x double> [[LANE]], <2 x double> [[A]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[FMLS1]]
+;
+entry:
+  %0 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
+  %lane = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> zeroinitializer
+  %fmls1 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %0, <2 x double> %lane, <2 x double> %a)
+  ret <2 x double> %fmls1
+}
+
+define <2 x float> @fmla_indexed_scalar_2s(<2 x float> %a, <2 x float> %b, float %c) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x float> @fmla_indexed_scalar_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], float [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <2 x float> undef, float [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <2 x float> [[V1]], float [[C]], i32 1
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <2 x float> @llvm.fma.v2f32(<2 x float> [[V1]], <2 x float> [[B]], <2 x float> [[A]]) #[[ATTR7:[0-9]+]]
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <2 x float> undef, float %c, i32 0
+  %v2 = insertelement <2 x float> %v1, float %c, i32 1
+  %fmla1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %v1, <2 x float> %b, <2 x float> %a) nounwind
+  ret <2 x float> %fmla1
+}
+
+define <4 x float> @fmla_indexed_scalar_4s(<4 x float> %a, <4 x float> %b, float %c) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x float> @fmla_indexed_scalar_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], float [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[C]], i32 1
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[C]], i32 2
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <4 x float> [[V3]], float [[C]], i32 3
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <4 x float> @llvm.fma.v4f32(<4 x float> [[V4]], <4 x float> [[B]], <4 x float> [[A]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <4 x float> undef, float %c, i32 0
+  %v2 = insertelement <4 x float> %v1, float %c, i32 1
+  %v3 = insertelement <4 x float> %v2, float %c, i32 2
+  %v4 = insertelement <4 x float> %v3, float %c, i32 3
+  %fmla1 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %v4, <4 x float> %b, <4 x float> %a) nounwind
+  ret <4 x float> %fmla1
+}
+
+define <2 x double> @fmla_indexed_scalar_2d(<2 x double> %a, <2 x double> %b, double %c) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x double> @fmla_indexed_scalar_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], double [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <2 x double> undef, double [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <2 x double> [[V1]], double [[C]], i32 1
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <2 x double> @llvm.fma.v2f64(<2 x double> [[V2]], <2 x double> [[B]], <2 x double> [[A]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <2 x double> undef, double %c, i32 0
+  %v2 = insertelement <2 x double> %v1, double %c, i32 1
+  %fmla1 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %v2, <2 x double> %b, <2 x double> %a) nounwind
+  ret <2 x double> %fmla1
+}
+
+define <2 x float> @fmls_indexed_2s_strict(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <2 x float> @fmls_indexed_2s_strict(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fneg <2 x float> [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x float> [[B]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> [[TMP0]], <2 x float> [[LANE]], <2 x float> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9:[0-9]+]]
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[FMLS1]]
+;
+entry:
+  %0 = fneg <2 x float> %c
+  %lane = shufflevector <2 x float> %b, <2 x float> undef, <2 x i32> zeroinitializer
+  %fmls1 = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> %0, <2 x float> %lane, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <2 x float> %fmls1
+}
+
+define <4 x float> @fmls_indexed_4s_strict(<4 x float> %a, <4 x float> %b, <4 x float> %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <4 x float> @fmls_indexed_4s_strict(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fneg <4 x float> [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> [[TMP0]], <4 x float> [[LANE]], <4 x float> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[FMLS1]]
+;
+entry:
+  %0 = fneg <4 x float> %c
+  %lane = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
+  %fmls1 = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %0, <4 x float> %lane, <4 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <4 x float> %fmls1
+}
+
+define <2 x double> @fmls_indexed_2d_strict(<2 x double> %a, <2 x double> %b, <2 x double> %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <2 x double> @fmls_indexed_2d_strict(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = fneg <2 x double> [[C]]
+; CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x double> [[B]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[FMLS1:%.*]] = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> [[TMP0]], <2 x double> [[LANE]], <2 x double> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[FMLS1]]
+;
+entry:
+  %0 = fneg <2 x double> %c
+  %lane = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> zeroinitializer
+  %fmls1 = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %0, <2 x double> %lane, <2 x double> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <2 x double> %fmls1
+}
+
+define <2 x float> @fmla_indexed_scalar_2s_strict(<2 x float> %a, <2 x float> %b, float %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <2 x float> @fmla_indexed_scalar_2s_strict(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], float [[C:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <2 x float> undef, float [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <2 x float> [[V1]], float [[C]], i32 1
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> [[V2]], <2 x float> [[B]], <2 x float> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9]]
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <2 x float> undef, float %c, i32 0
+  %v2 = insertelement <2 x float> %v1, float %c, i32 1
+  %fmla1 = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> %v2, <2 x float> %b, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <2 x float> %fmla1
+}
+
+define <4 x float> @fmla_indexed_scalar_4s_strict(<4 x float> %a, <4 x float> %b, float %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <4 x float> @fmla_indexed_scalar_4s_strict(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], float [[C:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[C]], i32 1
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[C]], i32 2
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <4 x float> [[V3]], float [[C]], i32 3
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> [[V4]], <4 x float> [[B]], <4 x float> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <4 x float> undef, float %c, i32 0
+  %v2 = insertelement <4 x float> %v1, float %c, i32 1
+  %v3 = insertelement <4 x float> %v2, float %c, i32 2
+  %v4 = insertelement <4 x float> %v3, float %c, i32 3
+  %fmla1 = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %v4, <4 x float> %b, <4 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <4 x float> %fmla1
+}
+
+define <2 x double> @fmla_indexed_scalar_2d_strict(<2 x double> %a, <2 x double> %b, double %c) nounwind readnone ssp strictfp {
+; CHECK-LABEL: define <2 x double> @fmla_indexed_scalar_2d_strict(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], double [[C:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <2 x double> undef, double [[C]], i32 0
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <2 x double> [[V1]], double [[C]], i32 1
+; CHECK-NEXT:    [[FMLA1:%.*]] = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> [[V2]], <2 x double> [[B]], <2 x double> [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR9]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[FMLA1]]
+;
+entry:
+  %v1 = insertelement <2 x double> undef, double %c, i32 0
+  %v2 = insertelement <2 x double> %v1, double %c, i32 1
+  %fmla1 = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %v2, <2 x double> %b, <2 x double> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  ret <2 x double> %fmla1
+}
+
+attributes #0 = { strictfp }
+
+declare <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float>, <2 x float>, <2 x float>, metadata, metadata)
+declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, metadata, metadata)
+declare <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double>, <2 x double>, <2 x double>, metadata, metadata)
+
+define <4 x i16> @mul_4h(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i16> @mul_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i16> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = mul <4 x i16> %A, %temp3
+  ret <4 x i16> %temp4
+}
+
+define <8 x i16> @mul_8h(<8 x i16> %A, <8 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @mul_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i16> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %temp4 = mul <8 x i16> %A, %temp3
+  ret <8 x i16> %temp4
+}
+
+define <2 x i32> @mul_2s(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i32> @mul_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i32> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = mul <2 x i32> %A, %temp3
+  ret <2 x i32> %temp4
+}
+
+define <4 x i32> @mul_4s(<4 x i32> %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @mul_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = mul <4 x i32> %A, %temp3
+  ret <4 x i32> %temp4
+}
+
+define <2 x i64> @mul_2d(<2 x i64> %A, <2 x i64> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @mul_2d(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i64> [[A]], [[B]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %temp1 = mul <2 x i64> %A, %B
+  ret <2 x i64> %temp1
+}
+
+define <2 x float> @fmul_lane_2s(<2 x float> %A, <2 x float> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmul_lane_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[B]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x float> %B, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = fmul <2 x float> %A, %temp3
+  ret <2 x float> %temp4
+}
+
+define <4 x float> @fmul_lane_4s(<4 x float> %A, <4 x float> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmul_lane_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x float> %B, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = fmul <4 x float> %A, %temp3
+  ret <4 x float> %temp4
+}
+
+define <2 x double> @fmul_lane_2d(<2 x double> %A, <2 x double> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmul_lane_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[A]], [[TMP3]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x double> %B, <2 x double> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = fmul <2 x double> %A, %temp3
+  ret <2 x double> %temp4
+}
+
+define float @fmul_lane_s(float %A, <4 x float> %vec) nounwind sanitize_memory {
+; CHECK-LABEL: define float @fmul_lane_s(
+; CHECK-SAME: float [[A:%.*]], <4 x float> [[VEC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x float> [[VEC]], i32 3
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT:    store i32 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %B = extractelement <4 x float> %vec, i32 3
+  %res = fmul float %A, %B
+  ret float %res
+}
+
+define double @fmul_lane_d(double %A, <2 x double> %vec) nounwind sanitize_memory {
+; CHECK-LABEL: define double @fmul_lane_d(
+; CHECK-SAME: double [[A:%.*]], <2 x double> [[VEC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[B:%.*]] = extractelement <2 x double> [[VEC]], i32 1
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul double [[A]], [[B]]
+; CHECK-NEXT:    store i64 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %B = extractelement <2 x double> %vec, i32 1
+  %res = fmul double %A, %B
+  ret double %res
+}
+
+
+
+define <2 x float> @fmulx_lane_2s(<2 x float> %A, <2 x float> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x float> @fmulx_lane_2s(
+; CHECK-SAME: <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[B]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> [[A]], <2 x float> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x float> %B, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %A, <2 x float> %temp3)
+  ret <2 x float> %temp4
+}
+
+define <4 x float> @fmulx_lane_4s(<4 x float> %A, <4 x float> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x float> @fmulx_lane_4s(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> [[A]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x float> %B, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %A, <4 x float> %temp3)
+  ret <4 x float> %temp4
+}
+
+define <2 x double> @fmulx_lane_2d(<2 x double> %A, <2 x double> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x double> @fmulx_lane_2d(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> splat (i64 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> [[A]], <2 x double> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x double> %B, <2 x double> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %A, <2 x double> %temp3)
+  ret <2 x double> %temp4
+}
+
+define <4 x i16> @sqdmulh_lane_4h(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i16> @sqdmulh_lane_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> [[A]], <4 x i16> [[TMP3]])
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %A, <4 x i16> %temp3)
+  ret <4 x i16> %temp4
+}
+
+define <8 x i16> @sqdmulh_lane_8h(<8 x i16> %A, <8 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @sqdmulh_lane_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> [[A]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %A, <8 x i16> %temp3)
+  ret <8 x i16> %temp4
+}
+
+define <2 x i32> @sqdmulh_lane_2s(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i32> @sqdmulh_lane_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> [[A]], <2 x i32> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %A, <2 x i32> %temp3)
+  ret <2 x i32> %temp4
+}
+
+define <4 x i32> @sqdmulh_lane_4s(<4 x i32> %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmulh_lane_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> [[A]], <4 x i32> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %A, <4 x i32> %temp3)
+  ret <4 x i32> %temp4
+}
+
+define i32 @sqdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmulh_lane_1s(
+; CHECK-SAME: i32 [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[B]], i32 1
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 [[A]], i32 [[TMP1]])
+; CHECK-NEXT:    store i32 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %temp1 = extractelement <4 x i32> %B, i32 1
+  %temp2 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %A, i32 %temp1)
+  ret i32 %temp2
+}
+
+define <4 x i16> @sqrdmulh_lane_4h(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i16> @sqrdmulh_lane_4h(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> [[A]], <4 x i16> [[TMP3]])
+; CHECK-NEXT:    store <4 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %A, <4 x i16> %temp3)
+  ret <4 x i16> %temp4
+}
+
+define <8 x i16> @sqrdmulh_lane_8h(<8 x i16> %A, <8 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @sqrdmulh_lane_8h(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> splat (i16 -1), <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> [[A]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP4]]
+;
+  %temp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %A, <8 x i16> %temp3)
+  ret <8 x i16> %temp4
+}
+
+define <2 x i32> @sqrdmulh_lane_2s(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i32> @sqrdmulh_lane_2s(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> [[A]], <2 x i32> [[TMP3]])
+; CHECK-NEXT:    store <2 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %A, <2 x i32> %temp3)
+  ret <2 x i32> %temp4
+}
+
+define <4 x i32> @sqrdmulh_lane_4s(<4 x i32> %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqrdmulh_lane_4s(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> [[A]], <4 x i32> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %A, <4 x i32> %temp3)
+  ret <4 x i32> %temp4
+}
+
+define i32 @sqrdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqrdmulh_lane_1s(
+; CHECK-SAME: i32 [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[B]], i32 1
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 [[A]], i32 [[TMP1]])
+; CHECK-NEXT:    store i32 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %temp1 = extractelement <4 x i32> %B, i32 1
+  %temp2 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %A, i32 %temp1)
+  ret i32 %temp2
+}
+
+define <4 x i32> @sqdmull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmull_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %temp3)
+  ret <4 x i32> %temp4
+}
+
+define <2 x i64> @sqdmull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmull_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %temp3)
+  ret <2 x i64> %temp4
+}
+
+define <4 x i32> @sqdmull2_lane_4s(<8 x i16> %A, <8 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmull2_lane_4s(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i16> [[TMP6]], <8 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  ret <4 x i32> %temp4
+}
+
+define <2 x i64> @sqdmull2_lane_2d(<4 x i32> %A, <4 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmull2_lane_2d(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %temp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  ret <2 x i64> %temp4
+}
+
+define <4 x i32> @umull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umull_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %temp3)
+  ret <4 x i32> %temp4
+}
+
+define <2 x i64> @umull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umull_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %temp3)
+  ret <2 x i64> %temp4
+}
+
+define <4 x i32> @smull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smull_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP3]])
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %temp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %temp3)
+  ret <4 x i32> %temp4
+}
+
+define <2 x i64> @smull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smull_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP3]])
+; CHECK-NEXT:    store <2 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %temp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %temp3)
+  ret <2 x i64> %temp4
+}
+
+define <4 x i32> @smlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smlal_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = add <4 x i32> %C, %temp5
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @smlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smlal_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = add <2 x i64> %C, %temp5
+  ret <2 x i64> %temp6
+}
+
+define <4 x i32> @sqdmlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlal_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[C]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %C, <4 x i32> %temp5)
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @sqdmlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlal_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[C]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %C, <2 x i64> %temp5)
+  ret <2 x i64> %temp6
+}
+
+define <4 x i32> @sqdmlal2_lane_4s(<8 x i16> %A, <8 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlal2_lane_4s(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i16> [[TMP8]], <8 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[C]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp6 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %C, <4 x i32> %temp5)
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @sqdmlal2_lane_2d(<4 x i32> %A, <4 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlal2_lane_2d(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i32> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[C]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp6 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %C, <2 x i64> %temp5)
+  ret <2 x i64> %temp6
+}
+
+define i32 @sqdmlal_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmlal_lane_1s(
+; CHECK-SAME: i32 [[A:%.*]], i16 [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHS:%.*]] = insertelement <4 x i16> undef, i16 [[B]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[RHS:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[PROD_VEC:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[LHS]], <4 x i16> [[RHS]])
+; CHECK-NEXT:    [[PROD:%.*]] = extractelement <4 x i32> [[PROD_VEC]], i32 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.aarch64.neon.sqadd.i32(i32 [[A]], i32 [[PROD]])
+; CHECK-NEXT:    store i32 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %lhs = insertelement <4 x i16> undef, i16 %B, i32 0
+  %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)
+  %prod = extractelement <4 x i32> %prod.vec, i32 0
+  %res = call i32 @llvm.aarch64.neon.sqadd.i32(i32 %A, i32 %prod)
+  ret i32 %res
+}
+declare i32 @llvm.aarch64.neon.sqadd.i32(i32, i32)
+
+define i32 @sqdmlsl_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmlsl_lane_1s(
+; CHECK-SAME: i32 [[A:%.*]], i16 [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP1]], i32 0
+; CHECK-NEXT:    [[LHS:%.*]] = insertelement <4 x i16> undef, i16 [[B]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[RHS:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[PROD_VEC:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[LHS]], <4 x i16> [[RHS]])
+; CHECK-NEXT:    [[PROD:%.*]] = extractelement <4 x i32> [[PROD_VEC]], i32 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.aarch64.neon.sqsub.i32(i32 [[A]], i32 [[PROD]])
+; CHECK-NEXT:    store i32 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %lhs = insertelement <4 x i16> undef, i16 %B, i32 0
+  %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)
+  %prod = extractelement <4 x i32> %prod.vec, i32 0
+  %res = call i32 @llvm.aarch64.neon.sqsub.i32(i32 %A, i32 %prod)
+  ret i32 %res
+}
+declare i32 @llvm.aarch64.neon.sqsub.i32(i32, i32)
+
+define i32 @sqadd_lane1_sqdmull4s(i32 %A, <4 x i16> %B, <4 x i16> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqadd_lane1_sqdmull4s(
+; CHECK-SAME: i32 [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[PROD_VEC:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[B]], <4 x i16> [[C]])
+; CHECK-NEXT:    [[PROD:%.*]] = extractelement <4 x i32> [[PROD_VEC]], i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.aarch64.neon.sqadd.i32(i32 [[A]], i32 [[PROD]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %B, <4 x i16> %C)
+  %prod = extractelement <4 x i32> %prod.vec, i32 1
+  %res = call i32 @llvm.aarch64.neon.sqadd.i32(i32 %A, i32 %prod)
+  ret i32 %res
+}
+
+define i32 @sqsub_lane1_sqdmull4s(i32 %A, <4 x i16> %B, <4 x i16> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqsub_lane1_sqdmull4s(
+; CHECK-SAME: i32 [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[PROD_VEC:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[B]], <4 x i16> [[C]])
+; CHECK-NEXT:    [[PROD:%.*]] = extractelement <4 x i32> [[PROD_VEC]], i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.aarch64.neon.sqsub.i32(i32 [[A]], i32 [[PROD]])
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %B, <4 x i16> %C)
+  %prod = extractelement <4 x i32> %prod.vec, i32 1
+  %res = call i32 @llvm.aarch64.neon.sqsub.i32(i32 %A, i32 %prod)
+  ret i32 %res
+}
+
+define i64 @sqdmlal_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i64 @sqdmlal_lane_1d(
+; CHECK-SAME: i64 [[A:%.*]], i32 [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHS:%.*]] = extractelement <2 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[_MSPROP]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[PROD:%.*]] = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 [[B]], i32 [[RHS]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.aarch64.neon.sqadd.i64(i64 [[A]], i64 [[PROD]])
+; CHECK-NEXT:    store i64 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %rhs = extractelement <2 x i32> %C, i32 1
+  %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)
+  %res = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %A, i64 %prod)
+  ret i64 %res
+}
+declare i64 @llvm.aarch64.neon.sqdmulls.scalar(i32, i32)
+declare i64 @llvm.aarch64.neon.sqadd.i64(i64, i64)
+
+define i64 @sqdmlsl_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i64 @sqdmlsl_lane_1d(
+; CHECK-SAME: i64 [[A:%.*]], i32 [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[RHS:%.*]] = extractelement <2 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[_MSPROP]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[PROD:%.*]] = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 [[B]], i32 [[RHS]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[TMP3]], 0
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.aarch64.neon.sqsub.i64(i64 [[A]], i64 [[PROD]])
+; CHECK-NEXT:    store i64 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %rhs = extractelement <2 x i32> %C, i32 1
+  %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)
+  %res = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %A, i64 %prod)
+  ret i64 %res
+}
+declare i64 @llvm.aarch64.neon.sqsub.i64(i64, i64)
+
+
+define <4 x i32> @umlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umlal_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = add <4 x i32> %C, %temp5
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @umlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umlal_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = add <2 x i64> %C, %temp5
+  ret <2 x i64> %temp6
+}
+
+
+define <4 x i32> @smlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @smlsl_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = sub <4 x i32> %C, %temp5
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @smlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @smlsl_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = sub <2 x i64> %C, %temp5
+  ret <2 x i64> %temp6
+}
+
+define <4 x i32> @sqdmlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlsl_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> [[C]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %C, <4 x i32> %temp5)
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @sqdmlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlsl_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> [[C]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %C, <2 x i64> %temp5)
+  ret <2 x i64> %temp6
+}
+
+define <4 x i32> @sqdmlsl2_lane_4s(<8 x i16> %A, <8 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @sqdmlsl2_lane_4s(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> splat (i16 -1), <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i16> [[TMP8]], <8 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> [[C]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %temp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp6 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %C, <4 x i32> %temp5)
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @sqdmlsl2_lane_2d(<4 x i32> %A, <4 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @sqdmlsl2_lane_2d(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> splat (i32 -1), <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i32> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> [[C]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %temp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %temp1, <2 x i32> %temp2)
+  %temp6 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %C, <2 x i64> %temp5)
+  ret <2 x i64> %temp6
+}
+
+define <4 x i32> @umlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @umlsl_lane_4s(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> splat (i16 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[B]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i16> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[A]], <4 x i16> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+;
+  %temp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp5 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %temp4)
+  %temp6 = sub <4 x i32> %C, %temp5
+  ret <4 x i32> %temp6
+}
+
+define <2 x i64> @umlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @umlsl_lane_2d(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <2 x i64> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> splat (i32 -1), <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i32> [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[_MSPROP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[A]], <2 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[C]], [[TMP5]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP6]]
+;
+  %temp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  %temp5 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %temp4)
+  %temp6 = sub <2 x i64> %C, %temp5
+  ret <2 x i64> %temp6
+}
+
+; Scalar FMULX
+define float @fmulxs(float %a, float %b) nounwind sanitize_memory {
+; CHECK-LABEL: define float @fmulxs(
+; CHECK-SAME: float [[A:%.*]], float [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], 0
+; CHECK-NEXT:    [[FMULX_I:%.*]] = tail call float @llvm.aarch64.neon.fmulx.f32(float [[A]], float [[B]]) #[[ATTR7]]
+; CHECK-NEXT:    store i32 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[FMULX_I]]
+;
+  %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind
+  ret float %fmulx.i
+}
+
+define double @fmulxd(double %a, double %b) nounwind sanitize_memory {
+; CHECK-LABEL: define double @fmulxd(
+; CHECK-SAME: double [[A:%.*]], double [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], 0
+; CHECK-NEXT:    [[FMULX_I:%.*]] = tail call double @llvm.aarch64.neon.fmulx.f64(double [[A]], double [[B]]) #[[ATTR7]]
+; CHECK-NEXT:    store i64 [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[FMULX_I]]
+;
+  %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind
+  ret double %fmulx.i
+}
+
+define float @fmulxs_lane(float %a, <4 x float> %vec) nounwind sanitize_memory {
+; CHECK-LABEL: define float @fmulxs_lane(
+; CHECK-SAME: float [[A:%.*]], <4 x float> [[VEC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x float> [[VEC]], i32 3
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[FMULX_I:%.*]] = tail call float @llvm.aarch64.neon.fmulx.f32(float [[A]], float [[B]]) #[[ATTR7]]
+; CHECK-NEXT:    store i32 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[FMULX_I]]
+;
+  %b = extractelement <4 x float> %vec, i32 3
+  %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind
+  ret float %fmulx.i
+}
+
+define double @fmulxd_lane(double %a, <2 x double> %vec) nounwind sanitize_memory {
+; CHECK-LABEL: define double @fmulxd_lane(
+; CHECK-SAME: double [[A:%.*]], <2 x double> [[VEC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[B:%.*]] = extractelement <2 x double> [[VEC]], i32 1
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[TMP2]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[FMULX_I:%.*]] = tail call double @llvm.aarch64.neon.fmulx.f64(double [[A]], double [[B]]) #[[ATTR7]]
+; CHECK-NEXT:    store i64 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[FMULX_I]]
+;
+  %b = extractelement <2 x double> %vec, i32 1
+  %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind
+  ret double %fmulx.i
+}
+
+declare double @llvm.aarch64.neon.fmulx.f64(double, double) nounwind readnone
+declare float @llvm.aarch64.neon.fmulx.f32(float, float) nounwind readnone
+
+
+define <8 x i16> @smull2_8h_simple(<16 x i8> %a, <16 x i8> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @smull2_8h_simple(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> splat (i8 -1), <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> splat (i8 -1), <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i8> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[_MSPROP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+; CHECK-NEXT:    store <8 x i16> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %3 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %1, <8 x i8> %2) #2
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @foo0(<16 x i8> %a, <16 x i8> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @foo0(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i8> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i8> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <8 x i8> [[_MSPROP3]] to <8 x i16>
+; CHECK-NEXT:    [[VMULL_I_I:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VMULL_I_I]]
+;
+  %temp = bitcast <16 x i8> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>
+  %temp2 = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <8 x i8>
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %temp1, <8 x i8> %temp3) nounwind
+  ret <8 x i16> %vmull.i.i
+}
+
+define <4 x i32> @foo1(<8 x i16> %a, <8 x i16> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @foo1(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i16> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I_I]]
+;
+  %temp = bitcast <8 x i16> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
+define <2 x i64> @foo2(<4 x i32> %a, <4 x i32> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @foo2(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I_I]]
+;
+  %temp = bitcast <4 x i32> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  ret <2 x i64> %vmull2.i.i
+}
+
+define <8 x i16> @foo3(<16 x i8> %a, <16 x i8> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @foo3(
+; CHECK-SAME: <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i8> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i8> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <8 x i8> [[_MSPROP3]] to <8 x i16>
+; CHECK-NEXT:    [[VMULL_I_I:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[VMULL_I_I]]
+;
+  %temp = bitcast <16 x i8> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>
+  %temp2 = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <8 x i8>
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %temp1, <8 x i8> %temp3) nounwind
+  ret <8 x i16> %vmull.i.i
+}
+
+define <4 x i32> @foo4(<8 x i16> %a, <8 x i16> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @foo4(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i16> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I_I]]
+;
+  %temp = bitcast <8 x i16> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
+define <2 x i64> @foo5(<4 x i32> %a, <4 x i32> %b) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @foo5(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I_I]]
+;
+  %temp = bitcast <4 x i32> %a to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  ret <2 x i64> %vmull2.i.i
+}
+
+define <4 x i32> @foo6(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @foo6(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <4 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
+define <4 x i32> @foo6a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @foo6a(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <4 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
+define <2 x i64> @foo7(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @foo7(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <2 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
+define <2 x i64> @foo7a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @foo7a(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <2 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
+
+define <4 x i32> @foo8(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @foo8(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <4 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
+define <4 x i32> @foo8a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @foo8a(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <4 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
+define <2 x i64> @foo9(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @foo9(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <2 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
+define <2 x i64> @foo9a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @foo9a(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I]] to <2 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[SHUFFLE]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
+define <8 x i16> @bar0(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @bar0(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i8> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[_MSPROP3]] to <8 x i16>
+; CHECK-NEXT:    [[VMULL_I_I_I:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i16> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <8 x i16> [[VMULL_I_I_I]], [[A]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[ADD_I]]
+;
+  %temp = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
+  %temp2 = bitcast <16 x i8> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <8 x i8>
+  %vmull.i.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %temp1, <8 x i8> %temp3) nounwind
+  %add.i = add <8 x i16> %vmull.i.i.i, %a
+  ret <8 x i16> %add.i
+}
+
+define <4 x i32> @bar1(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @bar1(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <4 x i32> [[VMULL2_I_I_I]], [[A]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD_I]]
+;
+  %temp = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <4 x i16>
+  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  %add.i = add <4 x i32> %vmull2.i.i.i, %a
+  ret <4 x i32> %add.i
+}
+
+define <2 x i64> @bar2(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @bar2(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <2 x i64> [[VMULL2_I_I_I]], [[A]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD_I]]
+;
+  %temp = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <2 x i32>
+  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  %add.i = add <2 x i64> %vmull2.i.i.i, %a
+  ret <2 x i64> %add.i
+}
+
+define <8 x i16> @bar3(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <8 x i16> @bar3(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <16 x i8> [[B:%.*]], <16 x i8> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <16 x i8> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <8 x i8>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i8> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i8> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[_MSPROP3]] to <8 x i16>
+; CHECK-NEXT:    [[VMULL_I_I_I:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i16> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <8 x i16> [[VMULL_I_I_I]], [[A]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[ADD_I]]
+;
+  %temp = bitcast <16 x i8> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>
+  %temp2 = bitcast <16 x i8> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <8 x i8>
+  %vmull.i.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %temp1, <8 x i8> %temp3) nounwind
+  %add.i = add <8 x i16> %vmull.i.i.i, %a
+  ret <8 x i16> %add.i
+}
+
+define <4 x i32> @bar4(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @bar4(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i16> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i16> [[_MSPROP3]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <4 x i32> [[VMULL2_I_I_I]], [[A]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD_I]]
+;
+  %temp = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <4 x i16>
+  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  %add.i = add <4 x i32> %vmull2.i.i.i, %a
+  ret <4 x i32> %add.i
+}
+
+define <2 x i64> @bar5(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @bar5(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[C]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[_MSPROP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i32> [[_MSPROP3]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add <2 x i64> [[VMULL2_I_I_I]], [[A]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD_I]]
+;
+  %temp = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %c to <2 x i64>
+  %shuffle.i3.i.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i.i to <2 x i32>
+  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  %add.i = add <2 x i64> %vmull2.i.i.i, %a
+  ret <2 x i64> %add.i
+}
+
+define <4 x i32> @mlal2_1(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @mlal2_1(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> splat (i16 -1), <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[SHUFFLE]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i16> [[_MSPROP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i16> [[_MSPROP4]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[VMULL2_I_I]], [[A]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %temp = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %shuffle to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  %add = add <4 x i32> %vmull2.i.i, %a
+  ret <4 x i32> %add
+}
+
+define <2 x i64> @mlal2_2(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @mlal2_2(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> splat (i32 -1), <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i32> [[_MSPROP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i32> [[_MSPROP4]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i64> [[VMULL2_I_I]], [[A]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %temp = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %shuffle to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  %add = add <2 x i64> %vmull2.i.i, %a
+  ret <2 x i64> %add
+}
+
+define <4 x i32> @mlal2_4(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @mlal2_4(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> splat (i16 -1), <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[C]], <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[SHUFFLE]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <4 x i16>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i16> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i16> [[_MSPROP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i16> [[_MSPROP4]] to <4 x i32>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[VMULL2_I_I]], [[A]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  %temp = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %temp2 = bitcast <8 x i16> %shuffle to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %temp1, <4 x i16> %temp3) nounwind
+  %add = add <4 x i32> %vmull2.i.i, %a
+  ret <4 x i32> %add
+}
+
+define <2 x i64> @mlal2_5(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @mlal2_5(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> splat (i32 -1), <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[C]], <2 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <2 x i64>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> splat (i64 -1), <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SHUFFLE_I3_I:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[_MSPROP2]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[SHUFFLE_I3_I]] to <2 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i32> [[_MSPROP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i32> [[_MSPROP4]] to <2 x i64>
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[TMP3]]) #[[ATTR7]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <2 x i64> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i64> [[VMULL2_I_I]], [[A]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> zeroinitializer
+  %temp = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %temp, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %temp2 = bitcast <4 x i32> %shuffle to <2 x i64>
+  %shuffle.i3.i = shufflevector <2 x i64> %temp2, <2 x i64> undef, <1 x i32> <i32 1>
+  %temp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %temp1, <2 x i32> %temp3) nounwind
+  %add = add <2 x i64> %vmull2.i.i, %a
+  ret <2 x i64> %add
+}
+
+; rdar://12328502
+define <2 x double> @vmulq_n_f64(<2 x double> %x, double %y) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x double> @vmulq_n_f64(
+; CHECK-SAME: <2 x double> [[X:%.*]], double [[Y:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x double> undef, double [[Y]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[Y]], i32 1
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul <2 x double> [[VECINIT1_I]], [[X]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[MUL_I]]
+;
+entry:
+  %vecinit.i = insertelement <2 x double> undef, double %y, i32 0
+  %vecinit1.i = insertelement <2 x double> %vecinit.i, double %y, i32 1
+  %mul.i = fmul <2 x double> %vecinit1.i, %x
+  ret <2 x double> %mul.i
+}
+
+define <4 x float> @vmulq_n_f32(<4 x float> %x, float %y) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x float> @vmulq_n_f32(
+; CHECK-SAME: <4 x float> [[X:%.*]], float [[Y:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> undef, float [[Y]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[Y]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT1_I]], float [[Y]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[Y]], i32 3
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul <4 x float> [[VECINIT3_I]], [[X]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[MUL_I]]
+;
+entry:
+  %vecinit.i = insertelement <4 x float> undef, float %y, i32 0
+  %vecinit1.i = insertelement <4 x float> %vecinit.i, float %y, i32 1
+  %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %y, i32 2
+  %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %y, i32 3
+  %mul.i = fmul <4 x float> %vecinit3.i, %x
+  ret <4 x float> %mul.i
+}
+
+define <2 x float> @vmul_n_f32(<2 x float> %x, float %y) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x float> @vmul_n_f32(
+; CHECK-SAME: <2 x float> [[X:%.*]], float [[Y:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x float> undef, float [[Y]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <2 x float> [[VECINIT_I]], float [[Y]], i32 1
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul <2 x float> [[VECINIT1_I]], [[X]]
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[MUL_I]]
+;
+entry:
+  %vecinit.i = insertelement <2 x float> undef, float %y, i32 0
+  %vecinit1.i = insertelement <2 x float> %vecinit.i, float %y, i32 1
+  %mul.i = fmul <2 x float> %vecinit1.i, %x
+  ret <2 x float> %mul.i
+}
+
+define <4 x i16> @vmla_laneq_s16_test(<4 x i16> %a, <4 x i16> %b, <8 x i16> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x i16> @vmla_laneq_s16_test(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]], <8 x i16> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[C]], <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i16> [[SHUFFLE]], [[B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i16> [[MUL]], [[A]]
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[ADD]]
+;
+entry:
+  %shuffle = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %mul = mul <4 x i16> %shuffle, %b
+  %add = add <4 x i16> %mul, %a
+  ret <4 x i16> %add
+}
+
+define <2 x i32> @vmla_laneq_s32_test(<2 x i32> %a, <2 x i32> %b, <4 x i32> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x i32> @vmla_laneq_s32_test(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[C]], <4 x i32> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[SHUFFLE]], [[B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i32> [[MUL]], [[A]]
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i32> [[ADD]]
+;
+entry:
+  %shuffle = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
+  %mul = mul <2 x i32> %shuffle, %b
+  %add = add <2 x i32> %mul, %a
+  ret <2 x i32> %add
+}
+
+define <8 x i16> @not_really_vmlaq_laneq_s16_test(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <8 x i16> @not_really_vmlaq_laneq_s16_test(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i16> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[C]], <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i16> [[SHUFFLE1]], <4 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i16> [[SHUFFLE2]], [[B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <8 x i16> [[MUL]], [[A]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[ADD]]
+;
+entry:
+  %shuffle1 = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle2 = shufflevector <4 x i16> %shuffle1, <4 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %mul = mul <8 x i16> %shuffle2, %b
+  %add = add <8 x i16> %mul, %a
+  ret <8 x i16> %add
+}
+
+define <4 x i32> @not_really_vmlaq_laneq_s32_test(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x i32> @not_really_vmlaq_laneq_s32_test(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[C]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <2 x i32> [[SHUFFLE1]], <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i32> [[SHUFFLE2]], [[B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[MUL]], [[A]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+entry:
+  %shuffle1 = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %shuffle2 = shufflevector <2 x i32> %shuffle1, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %mul = mul <4 x i32> %shuffle2, %b
+  %add = add <4 x i32> %mul, %a
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @vmull_laneq_s16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x i32> @vmull_laneq_s16_test(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[A]], <4 x i16> [[SHUFFLE]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #2
+  ret <4 x i32> %vmull2.i
+}
+
+define <2 x i64> @vmull_laneq_s32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x i64> @vmull_laneq_s32_test(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[A]], <2 x i32> [[SHUFFLE]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #2
+  ret <2 x i64> %vmull2.i
+}
+define <4 x i32> @vmull_laneq_u16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {
+; CHECK-LABEL: define <4 x i32> @vmull_laneq_u16_test(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <8 x i16> [[B:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[A]], <4 x i16> [[SHUFFLE]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I]]
+;
+entry:
+  %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #2
+  ret <4 x i32> %vmull2.i
+}
+
+define <2 x i64> @vmull_laneq_u32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {
+; CHECK-LABEL: define <2 x i64> @vmull_laneq_u32_test(
+; CHECK-SAME: <2 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    [[VMULL2_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[A]], <2 x i32> [[SHUFFLE]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I]]
+;
+entry:
+  %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #2
+  ret <2 x i64> %vmull2.i
+}
+
+define <4 x i32> @vmull_low_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @vmull_low_n_s16_test(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], i32 [[D:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[D]] to i16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x i16> undef, i16 [[CONV]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x i16> [[VECINIT_I]], i16 [[CONV]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x i16> [[VECINIT1_I]], i16 [[CONV]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x i16> [[VECINIT2_I]], i16 [[CONV]], i32 3
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[VECINIT3_I]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I_I]]
+;
+entry:
+  %conv = trunc i32 %d to i16
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 0
+  %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 1
+  %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 2
+  %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 3
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
+define <4 x i32> @vmull_high_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @vmull_high_n_s16_test(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], i32 [[D:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[D]] to i16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x i16> undef, i16 [[CONV]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x i16> [[VECINIT_I]], i16 [[CONV]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x i16> [[VECINIT1_I]], i16 [[CONV]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x i16> [[VECINIT2_I]], i16 [[CONV]], i32 3
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[VECINIT3_I]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I_I]]
+;
+entry:
+  %conv = trunc i32 %d to i16
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 0
+  %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 1
+  %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 2
+  %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 3
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
+define <2 x i64> @vmull_high_n_s32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @vmull_high_n_s32_test(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], i32 [[D:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x i32> undef, i32 [[D]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <2 x i32> [[VECINIT_I]], i32 [[D]], i32 1
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[VECINIT1_I]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %vecinit.i = insertelement <2 x i32> undef, i32 %d, i32 0
+  %vecinit1.i = insertelement <2 x i32> %vecinit.i, i32 %d, i32 1
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %vecinit1.i) nounwind
+  ret <2 x i64> %vmull2.i.i
+}
+
+define <4 x i32> @vmull_high_n_u16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <4 x i32> @vmull_high_n_u16_test(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <8 x i16> [[B:%.*]], <4 x i16> [[C:%.*]], i32 [[D:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[D]] to i16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <4 x i16>
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x i16> undef, i16 [[CONV]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x i16> [[VECINIT_I]], i16 [[CONV]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x i16> [[VECINIT1_I]], i16 [[CONV]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x i16> [[VECINIT2_I]], i16 [[CONV]], i32 3
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[VECINIT3_I]]) #[[ATTR7]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[VMULL2_I_I]]
+;
+entry:
+  %conv = trunc i32 %d to i16
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 0
+  %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 1
+  %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 2
+  %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 3
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
+define <2 x i64> @vmull_high_n_u32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {
+; CHECK-LABEL: define <2 x i64> @vmull_high_n_u32_test(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <4 x i32> [[B:%.*]], <2 x i32> [[C:%.*]], i32 [[D:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[SHUFFLE_I_I:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[SHUFFLE_I_I]] to <2 x i32>
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <2 x i32> undef, i32 [[D]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <2 x i32> [[VECINIT_I]], i32 [[D]], i32 1
+; CHECK-NEXT:    [[VMULL2_I_I:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[TMP1]], <2 x i32> [[VECINIT1_I]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[VMULL2_I_I]]
+;
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>
+  %vecinit.i = insertelement <2 x i32> undef, i32 %d, i32 0
+  %vecinit1.i = insertelement <2 x i32> %vecinit.i, i32 %d, i32 1
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %vecinit1.i) nounwind
+  ret <2 x i64> %vmull2.i.i
+}
+
+define <4 x i32> @vmul_built_dup_test(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: define <4 x i32> @vmul_built_dup_test(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <4 x i32> [[B]], i32 1
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[VGET_LANE]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x i32> [[VECINIT_I]], i32 [[VGET_LANE]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x i32> [[VECINIT1_I]], i32 [[VGET_LANE]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x i32> [[VECINIT2_I]], i32 [[VGET_LANE]], i32 3
+; CHECK-NEXT:    [[PROD:%.*]] = mul <4 x i32> [[A]], [[VECINIT3_I]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[PROD]]
+;
+  %vget_lane = extractelement <4 x i32> %b, i32 1
+  %vecinit.i = insertelement <4 x i32> undef, i32 %vget_lane, i32 0
+  %vecinit1.i = insertelement <4 x i32> %vecinit.i, i32 %vget_lane, i32 1
+  %vecinit2.i = insertelement <4 x i32> %vecinit1.i, i32 %vget_lane, i32 2
+  %vecinit3.i = insertelement <4 x i32> %vecinit2.i, i32 %vget_lane, i32 3
+  %prod = mul <4 x i32> %a, %vecinit3.i
+  ret <4 x i32> %prod
+}
+
+define <4 x i16> @vmul_built_dup_fromsmall_test(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: define <4 x i16> @vmul_built_dup_fromsmall_test(
+; CHECK-SAME: <4 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <4 x i16> [[B]], i32 3
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x i16> undef, i16 [[VGET_LANE]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <4 x i16> [[VECINIT_I]], i16 [[VGET_LANE]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x i16> [[VECINIT1_I]], i16 [[VGET_LANE]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <4 x i16> [[VECINIT2_I]], i16 [[VGET_LANE]], i32 3
+; CHECK-NEXT:    [[PROD:%.*]] = mul <4 x i16> [[A]], [[VECINIT3_I]]
+; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i16> [[PROD]]
+;
+  %vget_lane = extractelement <4 x i16> %b, i32 3
+  %vecinit.i = insertelement <4 x i16> undef, i16 %vget_lane, i32 0
+  %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %vget_lane, i32 1
+  %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %vget_lane, i32 2
+  %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %vget_lane, i32 3
+  %prod = mul <4 x i16> %a, %vecinit3.i
+  ret <4 x i16> %prod
+}
+
+define <8 x i16> @vmulq_built_dup_fromsmall_test(<8 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: define <8 x i16> @vmulq_built_dup_fromsmall_test(
+; CHECK-SAME: <8 x i16> [[A:%.*]], <4 x i16> [[B:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <4 x i16> [[B]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[VGET_LANE]], i32 0
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x i16> [[VECINIT_I]], i16 [[VGET_LANE]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x i16> [[VECINIT1_I]], i16 [[VGET_LANE]], i32 2
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x i16> [[VECINIT2_I]], i16 [[VGET_LANE]], i32 3
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x i16> [[VECINIT3_I]], i16 [[VGET_LANE]], i32 4
+; CHECK-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x i16> [[VECINIT4_I]], i16 [[VGET_LANE]], i32 5
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x i16> [[VECINIT5_I]], i16 [[VGET_LANE]], i32 6
+; CHECK-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x i16> [[VECINIT6_I]], i16 [[VGET_LANE]], i32 7
+; CHECK-NEXT:    [[PROD:%.*]] = mul <8 x i16> [[A]], [[VECINIT7_I]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[PROD]]
+;
+  %vget_lane = extractelement <4 x i16> %b, i32 0
+  %vecinit.i = insertelement <8 x i16> undef, i16 %vget_lane, i32 0
+  %vecinit1.i = insertelement <8 x i16> %vecinit.i, i16 %vget_lane, i32 1
+  %vecinit2.i = insertelement <8 x i16> %vecinit1.i, i16 %vget_lane, i32 2
+  %vecinit3.i = insertelement <8 x i16> %vecinit2.i, i16 %vget_lane, i32 3
+  %vecinit4.i = insertelement <8 x i16> %vecinit3.i, i16 %vget_lane, i32 4
+  %vecinit5.i = insertelement <8 x i16> %vecinit4.i, i16 %vget_lane, i32 5
+  %vecinit6.i = insertelement <8 x i16> %vecinit5.i, i16 %vget_lane, i32 6
+  %vecinit7.i = insertelement <8 x i16> %vecinit6.i, i16 %vget_lane, i32 7
+  %prod = mul <8 x i16> %a, %vecinit7.i
+  ret <8 x i16> %prod
+}
+
+define <2 x i64> @mull_from_two_extracts(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @mull_from_two_extracts(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @mlal_from_two_extracts(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @mlal_from_two_extracts(
+; CHECK-SAME: <2 x i64> [[ACCUM:%.*]], <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    [[SUM:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[ACCUM]], <2 x i64> [[RES]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[SUM]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @mull_from_extract_dup_low(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: define <2 x i64> @mull_from_extract_dup_low(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> undef, i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHSVEC]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @mull_from_extract_dup_high(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: define <2 x i64> @mull_from_extract_dup_high(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], i32 [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHSVEC_TMP:%.*]] = insertelement <2 x i32> undef, i32 [[RHS]], i32 0
+; CHECK-NEXT:    [[RHSVEC:%.*]] = insertelement <2 x i32> [[RHSVEC_TMP]], i32 [[RHS]], i32 1
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHSVEC]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @pmull_from_extract_dup_low(<16 x i8> %lhs, i8 %rhs) {
+; CHECK-LABEL: define <8 x i16> @pmull_from_extract_dup_low(
+; CHECK-SAME: <16 x i8> [[LHS:%.*]], i8 [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHSVEC_0:%.*]] = insertelement <8 x i8> undef, i8 [[RHS]], i32 0
+; CHECK-NEXT:    [[RHSVEC:%.*]] = shufflevector <8 x i8> [[RHSVEC_0]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <16 x i8> [[LHS]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[LHS_HIGH]], <8 x i8> [[RHSVEC]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
+  %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhsvec) nounwind
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @pmull_from_extract_dup_high(<16 x i8> %lhs, i8 %rhs) {
+; CHECK-LABEL: define <8 x i16> @pmull_from_extract_dup_high(
+; CHECK-SAME: <16 x i8> [[LHS:%.*]], i8 [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHSVEC_0:%.*]] = insertelement <8 x i8> undef, i8 [[RHS]], i32 0
+; CHECK-NEXT:    [[RHSVEC:%.*]] = shufflevector <8 x i8> [[RHSVEC_0]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <16 x i8> [[LHS]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RES:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[LHS_HIGH]], <8 x i8> [[RHSVEC]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
+  %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhsvec) nounwind
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @pmull_from_extract_duplane_low(<16 x i8> %lhs, <8 x i8> %rhs) {
+; CHECK-LABEL: define <8 x i16> @pmull_from_extract_duplane_low(
+; CHECK-SAME: <16 x i8> [[LHS:%.*]], <8 x i8> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <16 x i8> [[LHS]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <8 x i8> [[RHS]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[LHS_HIGH]], <8 x i8> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhs.high) nounwind
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @pmull_from_extract_duplane_high(<16 x i8> %lhs, <8 x i8> %rhs) {
+; CHECK-LABEL: define <8 x i16> @pmull_from_extract_duplane_high(
+; CHECK-SAME: <16 x i8> [[LHS:%.*]], <8 x i8> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <16 x i8> [[LHS]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <8 x i8> [[RHS]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[LHS_HIGH]], <8 x i8> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhs.high) nounwind
+  ret <8 x i16> %res
+}
+
+define <2 x i64> @sqdmull_from_extract_duplane_low(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @sqdmull_from_extract_duplane_low(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @sqdmull_from_extract_duplane_high(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @sqdmull_from_extract_duplane_high(
+; CHECK-SAME: <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @sqdmlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @sqdmlal_from_extract_duplane_low(
+; CHECK-SAME: <2 x i64> [[ACCUM:%.*]], <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    [[SUM:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[ACCUM]], <2 x i64> [[RES]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[SUM]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @sqdmlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @sqdmlal_from_extract_duplane_high(
+; CHECK-SAME: <2 x i64> [[ACCUM:%.*]], <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    [[SUM:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[ACCUM]], <2 x i64> [[RES]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[SUM]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @umlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @umlal_from_extract_duplane_low(
+; CHECK-SAME: <2 x i64> [[ACCUM:%.*]], <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <2 x i64> [[ACCUM]], [[RES]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[SUM]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = add <2 x i64> %accum, %res
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @umlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: define <2 x i64> @umlal_from_extract_duplane_high(
+; CHECK-SAME: <2 x i64> [[ACCUM:%.*]], <4 x i32> [[LHS:%.*]], <4 x i32> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[LHS_HIGH:%.*]] = shufflevector <4 x i32> [[LHS]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[RHS_HIGH:%.*]] = shufflevector <4 x i32> [[RHS]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> [[LHS_HIGH]], <2 x i32> [[RHS_HIGH]]) #[[ATTR7]]
+; CHECK-NEXT:    [[SUM:%.*]] = add <2 x i64> [[ACCUM]], [[RES]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[SUM]]
+;
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = add <2 x i64> %accum, %res
+  ret <2 x i64> %sum
+}
+
+define float @scalar_fmla_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {
+; CHECK-LABEL: define float @scalar_fmla_from_extract_v4f32(
+; CHECK-SAME: float [[ACCUM:%.*]], float [[LHS:%.*]], <4 x float> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS:%.*]] = extractelement <4 x float> [[RVEC]], i32 3
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.fma.f32(float [[LHS]], float [[RHS]], float [[ACCUM]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %rhs = extractelement <4 x float> %rvec, i32 3
+  %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
+  ret float %res
+}
+
+define float @scalar_fmla_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {
+; CHECK-LABEL: define float @scalar_fmla_from_extract_v2f32(
+; CHECK-SAME: float [[ACCUM:%.*]], float [[LHS:%.*]], <2 x float> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS:%.*]] = extractelement <2 x float> [[RVEC]], i32 1
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.fma.f32(float [[LHS]], float [[RHS]], float [[ACCUM]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %rhs = extractelement <2 x float> %rvec, i32 1
+  %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
+  ret float %res
+}
+
+define float @scalar_fmls_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {
+; CHECK-LABEL: define float @scalar_fmls_from_extract_v4f32(
+; CHECK-SAME: float [[ACCUM:%.*]], float [[LHS:%.*]], <4 x float> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_SCAL:%.*]] = extractelement <4 x float> [[RVEC]], i32 3
+; CHECK-NEXT:    [[RHS:%.*]] = fsub float -0.000000e+00, [[RHS_SCAL]]
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.fma.f32(float [[LHS]], float [[RHS]], float [[ACCUM]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %rhs.scal = extractelement <4 x float> %rvec, i32 3
+  %rhs = fsub float -0.0, %rhs.scal
+  %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
+  ret float %res
+}
+
+define float @scalar_fmls_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {
+; CHECK-LABEL: define float @scalar_fmls_from_extract_v2f32(
+; CHECK-SAME: float [[ACCUM:%.*]], float [[LHS:%.*]], <2 x float> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_SCAL:%.*]] = extractelement <2 x float> [[RVEC]], i32 1
+; CHECK-NEXT:    [[RHS:%.*]] = fsub float -0.000000e+00, [[RHS_SCAL]]
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.fma.f32(float [[LHS]], float [[RHS]], float [[ACCUM]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %rhs.scal = extractelement <2 x float> %rvec, i32 1
+  %rhs = fsub float -0.0, %rhs.scal
+  %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)
+  ret float %res
+}
+
+declare float @llvm.fma.f32(float, float, float)
+
+define double @scalar_fmla_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {
+; CHECK-LABEL: define double @scalar_fmla_from_extract_v2f64(
+; CHECK-SAME: double [[ACCUM:%.*]], double [[LHS:%.*]], <2 x double> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS:%.*]] = extractelement <2 x double> [[RVEC]], i32 1
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.fma.f64(double [[LHS]], double [[RHS]], double [[ACCUM]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %rhs = extractelement <2 x double> %rvec, i32 1
+  %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)
+  ret double %res
+}
+
+define double @scalar_fmls_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {
+; CHECK-LABEL: define double @scalar_fmls_from_extract_v2f64(
+; CHECK-SAME: double [[ACCUM:%.*]], double [[LHS:%.*]], <2 x double> [[RVEC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_SCAL:%.*]] = extractelement <2 x double> [[RVEC]], i32 1
+; CHECK-NEXT:    [[RHS:%.*]] = fsub double -0.000000e+00, [[RHS_SCAL]]
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.fma.f64(double [[LHS]], double [[RHS]], double [[ACCUM]])
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %rhs.scal = extractelement <2 x double> %rvec, i32 1
+  %rhs = fsub double -0.0, %rhs.scal
+  %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)
+  ret double %res
+}
+
+declare double @llvm.fma.f64(double, double, double)
+
+define <2 x float> @fmls_with_fneg_before_extract_v2f32(<2 x float> %accum, <2 x float> %lhs, <4 x float> %rhs) {
+; CHECK-LABEL: define <2 x float> @fmls_with_fneg_before_extract_v2f32(
+; CHECK-SAME: <2 x float> [[ACCUM:%.*]], <2 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_NEG:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[RHS]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[RHS_NEG]], <4 x float> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[LHS]], <2 x float> [[SPLAT]], <2 x float> [[ACCUM]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[RES]]
+;
+  %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
+  %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>
+  %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)
+  ret <2 x float> %res
+}
+
+define <2 x float> @fmls_with_fneg_before_extract_v2f32_1(<2 x float> %accum, <2 x float> %lhs, <2 x float> %rhs) {
+; CHECK-LABEL: define <2 x float> @fmls_with_fneg_before_extract_v2f32_1(
+; CHECK-SAME: <2 x float> [[ACCUM:%.*]], <2 x float> [[LHS:%.*]], <2 x float> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_NEG:%.*]] = fsub <2 x float> splat (float -0.000000e+00), [[RHS]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x float> [[RHS_NEG]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[LHS]], <2 x float> [[SPLAT]], <2 x float> [[ACCUM]])
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x float> [[RES]]
+;
+  %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs
+  %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+  %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)
+  ret <2 x float> %res
+}
+
+define <4 x float> @fmls_with_fneg_before_extract_v4f32(<4 x float> %accum, <4 x float> %lhs, <4 x float> %rhs) {
+; CHECK-LABEL: define <4 x float> @fmls_with_fneg_before_extract_v4f32(
+; CHECK-SAME: <4 x float> [[ACCUM:%.*]], <4 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_NEG:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[RHS]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[RHS_NEG]], <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[LHS]], <4 x float> [[SPLAT]], <4 x float> [[ACCUM]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
+  %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)
+  ret <4 x float> %res
+}
+
+define <4 x float> @fmls_with_fneg_before_extract_v4f32_1(<4 x float> %accum, <4 x float> %lhs, <2 x float> %rhs) {
+; CHECK-LABEL: define <4 x float> @fmls_with_fneg_before_extract_v4f32_1(
+; CHECK-SAME: <4 x float> [[ACCUM:%.*]], <4 x float> [[LHS:%.*]], <2 x float> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_NEG:%.*]] = fsub <2 x float> splat (float -0.000000e+00), [[RHS]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x float> [[RHS_NEG]], <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[LHS]], <4 x float> [[SPLAT]], <4 x float> [[ACCUM]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs
+  %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)
+  ret <4 x float> %res
+}
+
+define <2 x double> @fmls_with_fneg_before_extract_v2f64(<2 x double> %accum, <2 x double> %lhs, <2 x double> %rhs) {
+; CHECK-LABEL: define <2 x double> @fmls_with_fneg_before_extract_v2f64(
+; CHECK-SAME: <2 x double> [[ACCUM:%.*]], <2 x double> [[LHS:%.*]], <2 x double> [[RHS:%.*]]) {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RHS_NEG:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[RHS]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x double> [[RHS_NEG]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[LHS]], <2 x double> [[SPLAT]], <2 x double> [[ACCUM]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %rhs_neg = fsub <2 x double> <double -0.0, double -0.0>, %rhs
+  %splat = shufflevector <2 x double> %rhs_neg, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %lhs, <2 x double> %splat, <2 x double> %accum)
+  ret <2 x double> %res
+}
+
+define <1 x double> @test_fmul_v1f64(<1 x double> %L, <1 x double> %R) nounwind sanitize_memory {
+; CHECK-LABEL: define <1 x double> @test_fmul_v1f64(
+; CHECK-SAME: <1 x double> [[L:%.*]], <1 x double> [[R:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[PROD:%.*]] = fmul <1 x double> [[L]], [[R]]
+; CHECK-NEXT:    store <1 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x double> [[PROD]]
+;
+  %prod = fmul <1 x double> %L, %R
+  ret <1 x double> %prod
+}
+
+define <1 x double> @test_fdiv_v1f64(<1 x double> %L, <1 x double> %R) nounwind sanitize_memory {
+; CHECK-LABEL: define <1 x double> @test_fdiv_v1f64(
+; CHECK-SAME: <1 x double> [[L:%.*]], <1 x double> [[R:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[PROD:%.*]] = fdiv <1 x double> [[L]], [[R]]
+; CHECK-NEXT:    store <1 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x double> [[PROD]]
+;
+  %prod = fdiv <1 x double> %L, %R
+  ret <1 x double> %prod
+}
+
+define i32 @sqdmlal_s(i16 %A, i16 %B, i32 %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmlal_s(
+; CHECK-SAME: i16 [[A:%.*]], i16 [[B:%.*]], i32 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> undef, i16 [[A]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i16> undef, i16 [[B]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sqadd.i32(i32 [[C]], i32 [[TMP4]])
+; CHECK-NEXT:    store i32 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %temp1 = insertelement <4 x i16> undef, i16 %A, i64 0
+  %temp2 = insertelement <4 x i16> undef, i16 %B, i64 0
+  %temp3 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp4 = extractelement <4 x i32> %temp3, i64 0
+  %temp5 = tail call i32 @llvm.aarch64.neon.sqadd.i32(i32 %C, i32 %temp4)
+  ret i32 %temp5
+}
+
+define i64 @sqdmlal_d(i32 %A, i32 %B, i64 %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i64 @sqdmlal_d(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.aarch64.neon.sqadd.i64(i64 [[C]], i64 [[TMP4]])
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %temp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)
+  %temp5 = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %C, i64 %temp4)
+  ret i64 %temp5
+}
+
+define i32 @sqdmlsl_s(i16 %A, i16 %B, i32 %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i32 @sqdmlsl_s(
+; CHECK-SAME: i16 [[A:%.*]], i16 [[B:%.*]], i32 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> undef, i16 [[A]], i64 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i16> splat (i16 -1), i16 [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i16> undef, i16 [[B]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 [[C]], i32 [[TMP4]])
+; CHECK-NEXT:    store i32 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %temp1 = insertelement <4 x i16> undef, i16 %A, i64 0
+  %temp2 = insertelement <4 x i16> undef, i16 %B, i64 0
+  %temp3 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %temp1, <4 x i16> %temp2)
+  %temp4 = extractelement <4 x i32> %temp3, i64 0
+  %temp5 = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 %C, i32 %temp4)
+  ret i32 %temp5
+}
+
+define i64 @sqdmlsl_d(i32 %A, i32 %B, i64 %C) nounwind sanitize_memory {
+; CHECK-LABEL: define i64 @sqdmlsl_d(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.aarch64.neon.sqsub.i64(i64 [[C]], i64 [[TMP4]])
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %temp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)
+  %temp5 = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %C, i64 %temp4)
+  ret i64 %temp5
+}
+
+define <16 x i8> @test_pmull_64(i64 %l, i64 %r) nounwind sanitize_memory {
+; CHECK-LABEL: define <16 x i8> @test_pmull_64(
+; CHECK-SAME: i64 [[L:%.*]], i64 [[R:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i64 [[_MSPROP1]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i128 [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[VAL:%.*]] = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 [[L]], i64 [[R]])
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[VAL]]
+;
+  %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l, i64 %r)
+  ret <16 x i8> %val
+}
+
+define <16 x i8> @test_pmull_high_64(<2 x i64> %l, <2 x i64> %r) nounwind sanitize_memory {
+; CHECK-LABEL: define <16 x i8> @test_pmull_high_64(
+; CHECK-SAME: <2 x i64> [[L:%.*]], <2 x i64> [[R:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[L_HI:%.*]] = extractelement <2 x i64> [[L]], i32 1
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[R_HI:%.*]] = extractelement <2 x i64> [[R]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP]], [[_MSPROP1]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or i64 [[_MSPROP2]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i64 [[_MSPROP3]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i128 [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[VAL:%.*]] = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 [[L_HI]], i64 [[R_HI]])
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[VAL]]
+;
+  %l_hi = extractelement <2 x i64> %l, i32 1
+  %r_hi = extractelement <2 x i64> %r, i32 1
+  %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l_hi, i64 %r_hi)
+  ret <16 x i8> %val
+}
+
+declare <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
+
+define <1 x i64> @test_mul_v1i64(<1 x i64> %lhs, <1 x i64> %rhs) nounwind sanitize_memory {
+; CHECK-LABEL: define <1 x i64> @test_mul_v1i64(
+; CHECK-SAME: <1 x i64> [[LHS:%.*]], <1 x i64> [[RHS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[PROD:%.*]] = mul <1 x i64> [[LHS]], [[RHS]]
+; CHECK-NEXT:    store <1 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[PROD]]
+;
+  %prod = mul <1 x i64> %lhs, %rhs
+  ret <1 x i64> %prod
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
index 5bb2c3f0e233b..48ecd53b40c72 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
@@ -903,15 +903,8 @@ define <4 x double> @test_x86_avx_round_pd_256(<4 x double> %a0) #0 {
 ; CHECK-LABEL: @test_x86_avx_round_pd_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
   %res = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7) ; <<4 x double>> [#uses=1]
@@ -924,15 +917,8 @@ define <8 x float> @test_x86_avx_round_ps_256(<8 x float> %a0) #0 {
 ; CHECK-LABEL: @test_x86_avx_round_ps_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7) ; <<8 x float>> [#uses=1]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/sse41-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/sse41-intrinsics-x86.ll
index f5e8452fd0d58..1fcab72d571ea 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/sse41-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/sse41-intrinsics-x86.ll
@@ -320,15 +320,8 @@ define <2 x double> @test_x86_sse41_round_pd(<2 x double> %a0) #0 {
 ; CHECK-LABEL: @test_x86_sse41_round_pd(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES]]
 ;
   %res = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7) ; <<2 x double>> [#uses=1]
@@ -341,15 +334,8 @@ define <4 x float> @test_x86_sse41_round_ps(<4 x float> %a0) #0 {
 ; CHECK-LABEL: @test_x86_sse41_round_ps(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES]]
 ;
   %res = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7) ; <<4 x float>> [#uses=1]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
index bd3b9388eadff..3ac0fa2415a0f 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
@@ -940,15 +940,8 @@ define <4 x double> @test_x86_avx_round_pd_256(<4 x double> %a0) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
   %res = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7) ; <<4 x double>> [#uses=1]
@@ -962,15 +955,8 @@ define <8 x float> @test_x86_avx_round_ps_256(<8 x float> %a0) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7) ; <<8 x float>> [#uses=1]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-i386.ll
index a3bda12e395e6..e51c53375d2b5 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-i386.ll
@@ -335,15 +335,8 @@ define <2 x double> @test_x86_sse41_round_pd(<2 x double> %a0) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES]]
 ;
   %res = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7) ; <<2 x double>> [#uses=1]
@@ -357,15 +350,8 @@ define <4 x float> @test_x86_sse41_round_ps(<4 x float> %a0) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> [[A0:%.*]], i32 7)
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES]]
 ;
   %res = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7) ; <<4 x float>> [#uses=1]
diff --git a/llvm/test/Instrumentation/RealtimeSanitizer/rtsan.ll b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan.ll
index a0bc4aef2cc31..c245a85a9a485 100644
--- a/llvm/test/Instrumentation/RealtimeSanitizer/rtsan.ll
+++ b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=rtsan -S | FileCheck %s
+; RUN: opt < %s -passes='function(rtsan),module(rtsan-module)' -S | FileCheck %s
 
 define void @violation() #0 {
   %1 = alloca ptr, align 8
@@ -18,6 +18,10 @@ define noundef i32 @main() #2 {
 
 attributes #0 = { mustprogress noinline sanitize_realtime optnone ssp uwtable(sync) }
 
+; RealtimeSanitizer pass should insert call to initialize the runtime
+; CHECK: @llvm.used = appending global [1 x ptr] [ptr @rtsan.module_ctor]
+; CHECK: @llvm.global_ctors = {{.*}}@rtsan.module_ctor
+
 ; RealtimeSanitizer pass should insert __rtsan_realtime_enter right after function definition
 ; CHECK-LABEL: @violation()
 ; CHECK-NEXT: call{{.*}}@__rtsan_realtime_enter
@@ -25,3 +29,6 @@ attributes #0 = { mustprogress noinline sanitize_realtime optnone ssp uwtable(sy
 ; RealtimeSanitizer pass should insert __rtsan_realtime_exit right before function return
 ; CHECK: call{{.*}}@__rtsan_realtime_exit
 ; CHECK-NEXT: ret{{.*}}void
+
+; CHECK-LABEL: define internal void @rtsan.module_ctor()
+; CHECK-NEXT: call void @__rtsan_ensure_initialized()
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
index 6ade3e32bd279..e57ff078a5274 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
@@ -7,13 +7,14 @@ target triple = "i386-unknown-linux-gnu"
 define i32 @foo() #0 {
 ; CHECK-LABEL: define i32 @foo() comdat {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr inttoptr (i32 ptrtoint (ptr @__sancov_gen_ to i32) to ptr)) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr @__sancov_gen_) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   ret i32 0
 }
 
+; UTC_ARGS: --disable
 ; CHECK-DAG: declare void @__sanitizer_cov_trace_pc_indir(i32)
 ; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp1(i8 zeroext, i8 zeroext)
 ; CHECK-DAG: declare void @__sanitizer_cov_trace_cmp2(i16 zeroext, i16 zeroext)
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/missing_dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/missing_dbg.ll
index 21c6fcdb3a84b..35684346c4d5a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/missing_dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/missing_dbg.ll
@@ -1,38 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=2 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define i32 @with_dbg(ptr %a, ptr %b) !dbg !3 {
+; CHECK-LABEL: define i32 @with_dbg(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) comdat !dbg [[DBG3:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr @__sancov_gen_) #[[ATTR1:[0-9]+]], !dbg [[DBG6:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr getelementptr inbounds ([2 x i32], ptr @__sancov_gen_, i64 0, i64 1)) #[[ATTR1]], !dbg [[DBG7:![0-9]+]]
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 entry:
-  %tmp1 = load i32, ptr %a, align 4
-  %cmp = icmp eq i32 %tmp1, 42
+  %t = load i32, ptr %a, align 4
+  %cmp = icmp eq i32 %t, 42
   br i1 %cmp, label %0, label %1
 0:
-  store i32 %tmp1, ptr %b
+  store i32 %t, ptr %b
   br label %1
 1:
-  ret i32 %tmp1
+  ret i32 %t
 }
-; CHECK-LABEL: @with_dbg
-; CHECK-NEXT:  entry:
-; CHECK:       call void @__sanitizer_cov_trace_pc_guard(ptr @__sancov_gen_) #1, !dbg [[DBG1:![0-9]+]]
-; CHECK:       call void @__sanitizer_cov_trace_pc_guard(ptr inttoptr (i64 add (i64 ptrtoint (ptr @__sancov_gen_ to i64), i64 4) to ptr)) #1, !dbg [[DBG2:![0-9]+]]
 
 define i32 @without_dbg(ptr %a, ptr %b) {
+; CHECK-LABEL: define i32 @without_dbg(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) comdat {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr @__sancov_gen_.1) #[[ATTR1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    call void @__sanitizer_cov_trace_pc_guard(ptr getelementptr inbounds ([2 x i32], ptr @__sancov_gen_.1, i64 0, i64 1)) #[[ATTR1]]
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 entry:
-  %tmp1 = load i32, ptr %a, align 4
-  %cmp = icmp eq i32 %tmp1, 42
+  %t = load i32, ptr %a, align 4
+  %cmp = icmp eq i32 %t, 42
   br i1 %cmp, label %0, label %1
 0:
-  store i32 %tmp1, ptr %b
+  store i32 %t, ptr %b
   br label %1
 1:
-  ret i32 %tmp1
+  ret i32 %t
 }
-; CHECK-LABEL: @without_dbg
-; CHECK-NEXT:  entry:
-; CHECK:       call void @__sanitizer_cov_trace_pc_guard(ptr @__sancov_gen_.1) #1
-; CHECK:       call void @__sanitizer_cov_trace_pc_guard(ptr inttoptr (i64 add (i64 ptrtoint (ptr @__sancov_gen_.1 to i64), i64 4) to ptr)) #1
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2}
@@ -46,5 +67,12 @@ entry:
 !6 = !DILocation(line: 192, scope: !3)
 !7 = !DILocation(line: 0, scope: !3)
 
-; CHECK:       [[DBG1]] = !DILocation(line: 192, scope: !3)
-; CHECK:       [[DBG2]] = !DILocation(line: 0, scope: !3)
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C89, file: [[META1:![0-9]+]], isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false, nameTableKind: None)
+; CHECK: [[META1]] = !DIFile(filename: "foo.c", directory: "")
+; CHECK: [[DBG3]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 190, type: [[META4:![0-9]+]], scopeLine: 192, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]])
+; CHECK: [[META4]] = !DISubroutineType(types: [[META5:![0-9]+]])
+; CHECK: [[META5]] = !{}
+; CHECK: [[DBG6]] = !DILocation(line: 192, scope: [[DBG3]])
+; CHECK: [[DBG7]] = !DILocation(line: 0, scope: [[DBG3]])
+;.
diff --git a/llvm/test/LTO/AMDGPU/closed-world-assumption.ll b/llvm/test/LTO/AMDGPU/closed-world-assumption.ll
new file mode 100644
index 0000000000000..dd084e7f3d9ed
--- /dev/null
+++ b/llvm/test/LTO/AMDGPU/closed-world-assumption.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O3 -debug-only=amdgpu-attributor -o - %s 2>&1 | FileCheck %s --check-prefix=NO-CW
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="lto<O3>" -debug-only=amdgpu-attributor -o - %s 2>&1 | FileCheck %s --check-prefix=NO-CW
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="lto<O3>" -debug-only=amdgpu-attributor -amdgpu-link-time-closed-world=1 -o - %s 2>&1 | FileCheck %s --check-prefix=CW
+
+; REQUIRES: amdgpu-registered-target
+; REQUIRES: asserts
+
+; NO-CW: Module {{.*}} is not assumed to be a closed world.
+; CW: Module {{.*}} is assumed to be a closed world.
+define hidden noundef i32 @_Z3foov() {
+  ret i32 1
+}
diff --git a/llvm/test/MC/AArch64/arm64-elf-relocs.s b/llvm/test/MC/AArch64/arm64-elf-relocs.s
index f679bb4c82827..5c04e899a7b22 100644
--- a/llvm/test/MC/AArch64/arm64-elf-relocs.s
+++ b/llvm/test/MC/AArch64/arm64-elf-relocs.s
@@ -331,6 +331,19 @@ trickQuestion:
 // CHECK-OBJ-LP64: R_AARCH64_GOT_LD_PREL19 sym
 // CHECK-OBJ-LP64: R_AARCH64_GOT_LD_PREL19 sym
 
+   adr x24, #:got_auth:sym
+   adr x24, :got_auth:sym
+   ldr x24, #:got_auth:sym
+   ldr x24, :got_auth:sym
+// CHECK: adr x24, :got_auth:sym
+// CHECK: adr x24, :got_auth:sym
+// CHECK: ldr x24, :got_auth:sym
+// CHECK: ldr x24, :got_auth:sym
+// CHECK-OBJ-LP64: R_AARCH64_AUTH_GOT_ADR_PREL_LO21 sym
+// CHECK-OBJ-LP64: R_AARCH64_AUTH_GOT_ADR_PREL_LO21 sym
+// CHECK-OBJ-LP64: R_AARCH64_AUTH_GOT_LD_PREL19 sym
+// CHECK-OBJ-LP64: R_AARCH64_AUTH_GOT_LD_PREL19 sym
+
 // GOT relocations referencing local symbols are not converted to reference
 // STT_SECTION symbols. https://github.com/llvm/llvm-project/issues/63418
   ldr x0, [x0, :got_lo12:local0]
diff --git a/llvm/test/MC/AArch64/ilp32-diagnostics.s b/llvm/test/MC/AArch64/ilp32-diagnostics.s
index 5d9c6e5626b2b..3c22ff84b0d7e 100644
--- a/llvm/test/MC/AArch64/ilp32-diagnostics.s
+++ b/llvm/test/MC/AArch64/ilp32-diagnostics.s
@@ -87,3 +87,9 @@ ldr x24, [x23, #:got_lo12:sym]
 
 ldr x24, [x23, :gottprel_lo12:sym]
 // ERROR: [[#@LINE-1]]:1: error: ILP32 64-bit load/store relocation not supported (LP64 eqv: TLSIE_LD64_GOTTPREL_LO12_NC)
+
+ldr x24, :got_auth:sym
+// ERROR: [[#@LINE-1]]:1: error: ILP32 LDR AUTH relocation not supported (LP64 eqv: AUTH_GOT_LD_PREL19)
+
+adr x24, :got_auth:sym
+// ERROR: [[#@LINE-1]]:1: error: ILP32 ADR AUTH relocation not supported (LP64 eqv: AUTH_GOT_ADR_PREL_LO21)
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_err.s
index 68442b01bf7d9..1b1d1c04359bd 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_err.s
@@ -75,20 +75,6 @@ v_cvt_f16_u16_e64_dpp v5, s1 row_shl:1 row_mask:0xf bank_mask:0xf
 v_dual_mul_f32 v0, v0, v2 : : v_dual_mul_f32 v1, v1, v3
 // GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression
 
-// On GFX11, v_dot8_i32_i4 is a valid SP3 alias for v_dot8_i32_iu4.
-// However, we intentionally leave it unimplemented because on other
-// processors v_dot8_i32_i4 denotes an instruction of a different
-// behaviour, which is considered potentially dangerous.
-v_dot8_i32_i4 v0, v1, v2, v3
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
-
-// On GFX11, v_dot4_i32_i8 is a valid SP3 alias for v_dot4_i32_iu8.
-// However, we intentionally leave it unimplemented because on other
-// processors v_dot4_i32_i8 denotes an instruction of a different
-// behaviour, which is considered potentially dangerous.
-v_dot4_i32_i8 v0, v1, v2, v3
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
-
 v_dot4c_i32_i8 v0, v1, v2
 // GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s
new file mode 100644
index 0000000000000..bb44217a836d3
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s
@@ -0,0 +1,74 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck --check-prefix=GFX11 %s
+
+v_interp_p10_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x00,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p10_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p10_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p10_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84]
+
+v_interp_p2_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x01,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p2_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p2_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p2_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84]
+
+v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x02,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p10_f16_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p10_f16_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p10_f16_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84]
+
+v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x03,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p2_f16_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p2_f16_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p2_f16_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84]
+
+v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x04,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84]
+
+v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5
+// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x05,0xcd,0x01,0x05,0x0e,0x04]
+
+v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3
+// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24]
+
+v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3
+// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44]
+
+v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3
+// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_alias.s
new file mode 100644
index 0000000000000..32b5eb14a5092
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_alias.s
@@ -0,0 +1,5 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck --check-prefix=GFX11 %s
+
+v_interp_p2_new_f32 v0, v1, v2, v3
+// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
index 7d850ec92aadb..cb7b71935e22a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2.s
@@ -124,50 +124,65 @@ v_add_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
 // W64: v_add_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2
-// GFX11: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+v_add_f16 v5.l, v1.l, v2.l
+// GFX11: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
 
-v_add_f16 v5, v127, v2
-// GFX11: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
+v_add_f16 v5.l, v127.l, v2.l
+// GFX11: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
 
-v_add_f16 v5, s1, v2
-// GFX11: v_add_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x64]
+v_add_f16 v5.l, s1, v2.l
+// GFX11: v_add_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x64]
 
-v_add_f16 v5, s105, v2
-// GFX11: v_add_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x64]
+v_add_f16 v5.l, s105, v2.l
+// GFX11: v_add_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x64]
 
-v_add_f16 v5, vcc_lo, v2
-// GFX11: v_add_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x64]
+v_add_f16 v5.l, vcc_lo, v2.l
+// GFX11: v_add_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x64]
 
-v_add_f16 v5, vcc_hi, v2
-// GFX11: v_add_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x64]
+v_add_f16 v5.l, vcc_hi, v2.l
+// GFX11: v_add_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x64]
 
-v_add_f16 v5, ttmp15, v2
-// GFX11: v_add_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x64]
+v_add_f16 v5.l, ttmp15, v2.l
+// GFX11: v_add_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x64]
 
-v_add_f16 v5, m0, v2
-// GFX11: v_add_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x64]
+v_add_f16 v5.l, m0, v2.l
+// GFX11: v_add_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x64]
 
-v_add_f16 v5, exec_lo, v2
-// GFX11: v_add_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x64]
+v_add_f16 v5.l, exec_lo, v2.l
+// GFX11: v_add_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x64]
 
-v_add_f16 v5, exec_hi, v2
-// GFX11: v_add_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x64]
+v_add_f16 v5.l, exec_hi, v2.l
+// GFX11: v_add_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x64]
 
-v_add_f16 v5, null, v2
-// GFX11: v_add_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x64]
+v_add_f16 v5.l, null, v2.l
+// GFX11: v_add_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x64]
 
-v_add_f16 v5, -1, v2
-// GFX11: v_add_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x64]
+v_add_f16 v5.l, -1, v2.l
+// GFX11: v_add_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x64]
 
-v_add_f16 v5, 0.5, v2
-// GFX11: v_add_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x64]
+v_add_f16 v5.l, 0.5, v2.l
+// GFX11: v_add_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x64]
 
-v_add_f16 v5, src_scc, v2
-// GFX11: v_add_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x64]
+v_add_f16 v5.l, src_scc, v2.l
+// GFX11: v_add_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x64]
 
-v_add_f16 v127, 0xfe0b, v127
-// GFX11: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+v_add_f16 v127.l, 0xfe0b, v127.l
+// GFX11: v_add_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+
+v_add_f16 v5.l, v1.h, v2.l
+// GFX11: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
+
+v_add_f16 v5.l, v127.h, v2.l
+// GFX11: v_add_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x64]
+
+v_add_f16 v127.l, 0.5, v127.l
+// GFX11: v_add_f16_e32 v127.l, 0.5, v127.l       ; encoding: [0xf0,0xfe,0xfe,0x64]
+
+v_add_f16 v5.h, src_scc, v2.h
+// GFX11: v_add_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x65]
+
+v_add_f16 v127.h, 0xfe0b, v127.h
+// GFX11: v_add_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00]
 
 v_add_f32 v5, v1, v2
 // GFX11: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
@@ -649,50 +664,56 @@ v_dot2c_f32_f16 v5, src_scc, v2
 v_dot2c_f32_f16 v255, 0xfe0b, v255
 // GFX11: v_dot2acc_f32_f16 v255, 0xfe0b, v255    ; encoding: [0xff,0xfe,0xff,0x05,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, v1, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, v1.l, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, v1.l, v2.l, 0xfe0b    ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5.l, v127.l, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, v127.l, v2.l, 0xfe0b  ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, v127, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, v1.h, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, v1.h, v2.l, 0xfe0b    ; encoding: [0x81,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, s1, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, v127.h, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, v127.h, v2.l, 0xfe0b  ; encoding: [0xff,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, s105, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, s1, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, s1, v2.l, 0xfe0b      ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, s105, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, s105, v2.l, 0xfe0b    ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, vcc_lo, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, vcc_lo, v2.l, 0xfe0b  ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, ttmp15, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, vcc_hi, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, vcc_hi, v2.l, 0xfe0b  ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, m0, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, ttmp15, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, ttmp15, v2.l, 0xfe0b  ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, exec_lo, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, m0, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, m0, v2.l, 0xfe0b      ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, exec_hi, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, exec_lo, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, exec_lo, v2.l, 0xfe0b ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, null, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, exec_hi, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, exec_hi, v2.l, 0xfe0b ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, -1, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, null, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, null, v2.l, 0xfe0b    ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, 0.5, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, -1, v2.l, 0xfe0b
+// GFX11: v_fmaak_f16 v5.l, -1, v2.l, 0xfe0b      ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, src_scc, v2, 0xfe0b
-// GFX11: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v127.l, 0.5, v127.l, 0xfe0b
+// GFX11: v_fmaak_f16 v127.l, 0.5, v127.l, 0xfe0b ; encoding: [0xf0,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b
-// GFX11: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.h, src_scc, v2.h, 0xfe0b
+// GFX11: v_fmaak_f16 v5.h, src_scc, v2.h, 0xfe0b ; encoding: [0xfd,0x04,0x0b,0x71,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v127.h, 0xfe0b, v127.h, 0xfe0b
+// GFX11: v_fmaak_f16 v127.h, 0xfe0b, v127.h, 0xfe0b ; encoding: [0xff,0xfe,0xff,0x71,0x0b,0xfe,0x00,0x00]
 
 v_fmaak_f32 v5, v1, v2, 0xaf123456
 // GFX11: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
@@ -784,50 +805,56 @@ v_fmac_dx9_zero_f32 v5, src_scc, v2
 v_fmac_dx9_zero_f32 v255, 0xaf123456, v255
 // GFX11: v_fmac_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
 
-v_fmac_f16 v5, v1, v2
-// GFX11: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
+v_fmac_f16 v5.l, v1.l, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, v1.l, v2.l         ; encoding: [0x01,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5.l, v127.l, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, v127.l, v2.l       ; encoding: [0x7f,0x05,0x0a,0x6c]
 
-v_fmac_f16 v5, v127, v2
-// GFX11: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
+v_fmac_f16 v5.l, v1.h, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, v1.h, v2.l         ; encoding: [0x81,0x05,0x0a,0x6c]
 
-v_fmac_f16 v5, s1, v2
-// GFX11: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, v127.h, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, v127.h, v2.l       ; encoding: [0xff,0x05,0x0a,0x6c]
 
-v_fmac_f16 v5, s105, v2
-// GFX11: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, s1, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, s1, v2.l           ; encoding: [0x01,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, vcc_lo, v2
-// GFX11: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, s105, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, s105, v2.l         ; encoding: [0x69,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, vcc_hi, v2
-// GFX11: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, vcc_lo, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, vcc_lo, v2.l       ; encoding: [0x6a,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, ttmp15, v2
-// GFX11: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, vcc_hi, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, vcc_hi, v2.l       ; encoding: [0x6b,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, m0, v2
-// GFX11: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, ttmp15, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, ttmp15, v2.l       ; encoding: [0x7b,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, exec_lo, v2
-// GFX11: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, m0, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, m0, v2.l           ; encoding: [0x7d,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, exec_hi, v2
-// GFX11: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, exec_lo, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, exec_lo, v2.l      ; encoding: [0x7e,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, null, v2
-// GFX11: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, exec_hi, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, exec_hi, v2.l      ; encoding: [0x7f,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, -1, v2
-// GFX11: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, null, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, null, v2.l         ; encoding: [0x7c,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, 0.5, v2
-// GFX11: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, -1, v2.l
+// GFX11: v_fmac_f16_e32 v5.l, -1, v2.l           ; encoding: [0xc1,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, src_scc, v2
-// GFX11: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
+v_fmac_f16 v127.l, 0.5, v127.l
+// GFX11: v_fmac_f16_e32 v127.l, 0.5, v127.l      ; encoding: [0xf0,0xfe,0xfe,0x6c]
 
-v_fmac_f16 v127, 0xfe0b, v127
-// GFX11: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
+v_fmac_f16 v5.h, src_scc, v2.h
+// GFX11: v_fmac_f16_e32 v5.h, src_scc, v2.h      ; encoding: [0xfd,0x04,0x0b,0x6d]
+
+v_fmac_f16 v127.h, 0xfe0b, v127.h
+// GFX11: v_fmac_f16_e32 v127.h, 0xfe0b, v127.h   ; encoding: [0xff,0xfe,0xff,0x6d,0x0b,0xfe,0x00,0x00]
 
 v_fmac_f32 v5, v1, v2
 // GFX11: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
@@ -919,50 +946,56 @@ v_fmac_legacy_f32 v5, src_scc, v2
 v_fmac_legacy_f32 v255, 0xaf123456, v255
 // GFX11: v_fmac_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0d,0x56,0x34,0x12,0xaf]
 
-v_fmamk_f16 v5, v1, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v1.l, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, v1.l, 0xfe0b, v3.l    ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5.l, v127.l, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, v127.l, 0xfe0b, v3.l  ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, v127, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v1.h, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, v1.h, 0xfe0b, v3.l    ; encoding: [0x81,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, s1, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v127.h, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, v127.h, 0xfe0b, v3.l  ; encoding: [0xff,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, s105, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, s1, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, s1, 0xfe0b, v3.l      ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, s105, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, s105, 0xfe0b, v3.l    ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, vcc_lo, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, vcc_lo, 0xfe0b, v3.l  ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, ttmp15, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, vcc_hi, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, vcc_hi, 0xfe0b, v3.l  ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, m0, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, ttmp15, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, ttmp15, 0xfe0b, v3.l  ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, exec_lo, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, m0, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, m0, 0xfe0b, v3.l      ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, exec_hi, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, exec_lo, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, exec_lo, 0xfe0b, v3.l ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, null, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, exec_hi, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, exec_hi, 0xfe0b, v3.l ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, -1, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, null, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, null, 0xfe0b, v3.l    ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, 0.5, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, -1, 0xfe0b, v3.l
+// GFX11: v_fmamk_f16 v5.l, -1, 0xfe0b, v3.l      ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, src_scc, 0xfe0b, v3
-// GFX11: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v127.l, 0.5, 0xfe0b, v127.l
+// GFX11: v_fmamk_f16 v127.l, 0.5, 0xfe0b, v127.l ; encoding: [0xf0,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127
-// GFX11: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.h, src_scc, 0xfe0b, v3.h
+// GFX11: v_fmamk_f16 v5.h, src_scc, 0xfe0b, v3.h ; encoding: [0xfd,0x06,0x0b,0x6f,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v127.h, 0xfe0b, 0xfe0b, v127.h
+// GFX11: v_fmamk_f16 v127.h, 0xfe0b, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x6f,0x0b,0xfe,0x00,0x00]
 
 v_fmamk_f32 v5, v1, 0xaf123456, v3
 // GFX11: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
@@ -2074,50 +2107,65 @@ v_sub_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
 // W64: v_sub_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2
-// GFX11: v_sub_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x66]
+v_sub_f16 v5.l, v1.l, v2.l
+// GFX11: v_sub_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x66]
+
+v_sub_f16 v5.l, v127.l, v2.l
+// GFX11: v_sub_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x66]
+
+v_sub_f16 v5.l, s1, v2.l
+// GFX11: v_sub_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x66]
+
+v_sub_f16 v5.l, s105, v2.l
+// GFX11: v_sub_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x66]
+
+v_sub_f16 v5.l, vcc_lo, v2.l
+// GFX11: v_sub_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x66]
+
+v_sub_f16 v5.l, vcc_hi, v2.l
+// GFX11: v_sub_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x66]
 
-v_sub_f16 v5, v127, v2
-// GFX11: v_sub_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x66]
+v_sub_f16 v5.l, ttmp15, v2.l
+// GFX11: v_sub_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x66]
 
-v_sub_f16 v5, s1, v2
-// GFX11: v_sub_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x66]
+v_sub_f16 v5.l, m0, v2.l
+// GFX11: v_sub_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x66]
 
-v_sub_f16 v5, s105, v2
-// GFX11: v_sub_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x66]
+v_sub_f16 v5.l, exec_lo, v2.l
+// GFX11: v_sub_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x66]
 
-v_sub_f16 v5, vcc_lo, v2
-// GFX11: v_sub_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x66]
+v_sub_f16 v5.l, exec_hi, v2.l
+// GFX11: v_sub_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x66]
 
-v_sub_f16 v5, vcc_hi, v2
-// GFX11: v_sub_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x66]
+v_sub_f16 v5.l, null, v2.l
+// GFX11: v_sub_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x66]
 
-v_sub_f16 v5, ttmp15, v2
-// GFX11: v_sub_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x66]
+v_sub_f16 v5.l, -1, v2.l
+// GFX11: v_sub_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x66]
 
-v_sub_f16 v5, m0, v2
-// GFX11: v_sub_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x66]
+v_sub_f16 v5.l, 0.5, v2.l
+// GFX11: v_sub_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x66]
 
-v_sub_f16 v5, exec_lo, v2
-// GFX11: v_sub_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x66]
+v_sub_f16 v5.l, src_scc, v2.l
+// GFX11: v_sub_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x66]
 
-v_sub_f16 v5, exec_hi, v2
-// GFX11: v_sub_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x66]
+v_sub_f16 v127.l, 0xfe0b, v127.l
+// GFX11: v_sub_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16 v5, null, v2
-// GFX11: v_sub_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x66]
+v_sub_f16 v5.l, v1.h, v2.l
+// GFX11: v_sub_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x66]
 
-v_sub_f16 v5, -1, v2
-// GFX11: v_sub_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x66]
+v_sub_f16 v5.l, v127.h, v2.l
+// GFX11: v_sub_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x66]
 
-v_sub_f16 v5, 0.5, v2
-// GFX11: v_sub_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x66]
+v_sub_f16 v127.l, 0.5, v127.l
+// GFX11: v_sub_f16_e32 v127.l, 0.5, v127.l       ; encoding: [0xf0,0xfe,0xfe,0x66]
 
-v_sub_f16 v5, src_scc, v2
-// GFX11: v_sub_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x66]
+v_sub_f16 v5.h, src_scc, v2.h
+// GFX11: v_sub_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x67]
 
-v_sub_f16 v127, 0xfe0b, v127
-// GFX11: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
+v_sub_f16 v127.h, 0xfe0b, v127.h
+// GFX11: v_sub_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00]
 
 v_sub_f32 v5, v1, v2
 // GFX11: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
index 9d0a407a4cd5e..00353c4cdcb49 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp16.s
@@ -116,47 +116,56 @@ v_add_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0
 // W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+v_add_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+v_add_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 
-v_add_f16 v5, v1, v2 row_mirror
-// GFX11: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_mirror
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_half_mirror
-// GFX11: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shl:1
-// GFX11: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shl:15
-// GFX11: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shr:1
-// GFX11: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shr:15
-// GFX11: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_ror:1
-// GFX11: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_ror:15
-// GFX11: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+v_add_f16 v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 
-v_add_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
+v_add_f16 v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
 
-v_add_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+v_add_f16 v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+
+v_add_f16 v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f16_dpp v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x5f,0x01,0x01]
+
+v_add_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x65,0x81,0x60,0x09,0x13]
+
+v_add_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x65,0xff,0x6f,0xf5,0x30]
 
 v_add_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
@@ -630,47 +639,47 @@ v_dot2c_f32_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 f
 v_dot2c_f32_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_dot2acc_f32_f16_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x05,0xff,0x6f,0xf5,0x30]
 
-v_fmac_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
 
-v_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 
-v_fmac_f16 v5, v1, v2 row_mirror
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_mirror
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_half_mirror
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shl:1
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shl:15
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shr:1
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shr:15
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_ror:1
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_ror:15
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
+v_fmac_f16 v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_fmac_f16_dpp v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x5f,0x01,0x01]
 
-v_fmac_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x09,0x13]
+v_fmac_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_fmac_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x6d,0x81,0x60,0x09,0x13]
 
-v_fmac_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xf5,0x30]
+v_fmac_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_fmac_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x6d,0xff,0x6f,0xf5,0x30]
 
 v_fmac_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
@@ -1666,47 +1675,56 @@ v_sub_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0
 // W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+
+v_sub_f16 v5.l, v1.l, v2.l row_mirror
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+
+v_sub_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_mirror
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_half_mirror
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shl:1
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shl:15
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shr:1
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shr:15
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_ror:1
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 
-v_sub_f16 v5, v1, v2 row_ror:15
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
 
-v_sub_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+v_sub_f16 v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1
+// GFX11: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
 
-v_sub_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+v_sub_f16 v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f16_dpp v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x5f,0x01,0x01]
 
-v_sub_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
+v_sub_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x67,0x81,0x60,0x09,0x13]
 
-v_sub_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
+v_sub_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x67,0xff,0x6f,0xf5,0x30]
 
 v_sub_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
index 6c82b7ea6b15d..489e6d8c9d63a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_dpp8.s
@@ -28,14 +28,23 @@ v_add_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-v_add_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+v_add_f16 v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+
+v_add_f16 v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x77,0x39,0x05]
+
+v_add_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x65,0x81,0x77,0x39,0x05]
+
+v_add_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x65,0xff,0x00,0x00,0x00]
 
 v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
@@ -133,14 +142,17 @@ v_dot2c_f32_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_dot2c_f32_f16 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_dot2acc_f32_f16_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x05,0xff,0x00,0x00,0x00]
 
-v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+v_fmac_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f16_dpp v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x77,0x39,0x05]
 
-v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+v_fmac_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_fmac_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x6d,0x81,0x77,0x39,0x05]
 
-v_fmac_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
+v_fmac_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_fmac_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x6d,0xff,0x00,0x00,0x00]
 
 v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
@@ -355,14 +367,23 @@ v_sub_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+
+v_sub_f16 v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x77,0x39,0x05]
 
-v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x67,0x81,0x77,0x39,0x05]
 
-v_sub_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+v_sub_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x67,0xff,0x00,0x00,0x00]
 
 v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
index 2027a436fa72f..e5504074079e5 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_err.s
@@ -2,32 +2,59 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
-v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_add_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_add_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_add_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_e32 v255.h, v1.h, v2.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_add_f16_e32 v255.l, v1.l, v2.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_add_f16_e32 v5.h, v1.h, v255.h
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_e32 v5.h, v255.h, v2.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_e32 v5.l, v1.l, v255.l
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_e32 v5.l, v255.l, v2.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 v_fmaak_f16_e32 v255.h, v1.h, v2.h, 0xfe0b
 // GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
@@ -227,32 +254,59 @@ v_mul_f16_e32 v5, v1, v255
 v_mul_f16_e32 v5, v255, v2
 // GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_e32 v255, v1, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sub_f16_e32 v5, v1, v255
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sub_f16_e32 v5, v255, v2
-// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_e32 v255.h, v1.h, v2.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_sub_f16_e32 v255.l, v1.l, v2.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.h, v1.h, v255.h
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.h, v255.h, v2.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.l, v1.l, v255.l
+// GFX11: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.l, v255.l, v2.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
index a6dcce40fd0e0..63394502ec14e 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop2_t16_promote.s
@@ -1,33 +1,60 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
 
-v_add_f16 v255, v1, v2
-// GFX11: v_add_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+v_add_f16 v255.h, v1.h, v2.h
+// GFX11: v_add_f16_e64 v255.h, v1.h, v2.h op_sel:[1,1,1] ; encoding: [0xff,0x58,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_add_f16 v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16 v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v255
-// GFX11: v_add_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
+v_add_f16 v255.l, v1.l, v2.l
+// GFX11: v_add_f16_e64 v255.l, v1.l, v2.l        ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_add_f16 v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16 v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v255, v2
-// GFX11: v_add_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
+v_add_f16 v5.h, v1.h, v255.h
+// GFX11: v_add_f16_e64 v5.h, v1.h, v255.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x32,0xd5,0x01,0xff,0x03,0x00]
 
-v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_add_f16 v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_add_f16 v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5.h, v255.h, v2.h
+// GFX11: v_add_f16_e64 v5.h, v255.h, v2.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x32,0xd5,0xff,0x05,0x02,0x00]
+
+v_add_f16 v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_add_f16 v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_add_f16 v5.l, v1.l, v255.l
+// GFX11: v_add_f16_e64 v5.l, v1.l, v255.l        ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
+
+v_add_f16 v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16 v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5.l, v255.l, v2.l
+// GFX11: v_add_f16_e64 v5.l, v255.l, v2.l        ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
+
+v_add_f16 v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_add_f16 v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_fmac_f16 v255, v1, v2
 // GFX11: v_fmac_f16_e64 v255, v1, v2             ; encoding: [0xff,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
@@ -146,32 +173,59 @@ v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
 v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
 // GFX11: v_mul_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_sub_f16 v255, v1, v2
-// GFX11: v_sub_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+v_sub_f16 v255.h, v1.h, v2.h
+// GFX11: v_sub_f16_e64 v255.h, v1.h, v2.h op_sel:[1,1,1] ; encoding: [0xff,0x58,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16 v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v255.l, v1.l, v2.l
+// GFX11: v_sub_f16_e64 v255.l, v1.l, v2.l        ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16 v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.h, v1.h, v255.h
+// GFX11: v_sub_f16_e64 v5.h, v1.h, v255.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x33,0xd5,0x01,0xff,0x03,0x00]
+
+v_sub_f16 v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.h, v255.h, v2.h
+// GFX11: v_sub_f16_e64 v5.h, v255.h, v2.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v1, v255
-// GFX11: v_sub_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
+v_sub_f16 v5.l, v1.l, v255.l
+// GFX11: v_sub_f16_e64 v5.l, v1.l, v255.l        ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
 
-v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v255, v2
-// GFX11: v_sub_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
+v_sub_f16 v5.l, v255.l, v2.l
+// GFX11: v_sub_f16_e64 v5.l, v255.l, v2.l        ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_sub_f16 v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_subrev_f16 v255, v1, v2
 // GFX11: v_subrev_f16_e64 v255, v1, v2           ; encoding: [0xff,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
index 376e6bf968cbc..b649bab532f26 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
@@ -2018,104 +2018,116 @@ v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2
 // W64: v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
 
-v_dot2_bf16_bf16 v5, v1, v2, s3
-// GFX11: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+v_dot2_bf16_bf16 v5.l, v1, v2, s3
+// GFX11: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 
-v_dot2_bf16_bf16 v5, v255, v255, s105
-// GFX11: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+v_dot2_bf16_bf16 v5.l, v255, v255, s105
+// GFX11: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 
-v_dot2_bf16_bf16 v5, s1, s2, v3
-// GFX11: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+v_dot2_bf16_bf16 v5.l, s1, s2, v3.l
+// GFX11: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 
-v_dot2_bf16_bf16 v5, s105, s105, m0
-// GFX11: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+v_dot2_bf16_bf16 v5.l, s105, s105, m0
+// GFX11: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 
-v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255
-// GFX11: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l
+// GFX11: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi
-// GFX11: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi
+// GFX11: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15
-// GFX11: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15
+// GFX11: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 
-v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo
-// GFX11: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo
+// GFX11: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 
-v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b|
-// GFX11: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b|
+// GFX11: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 
-v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo|
-// GFX11: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo|
+// GFX11: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 
-v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc|
-// GFX11: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc|
+// GFX11: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 
-v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| op_sel:[0,0,0,0]
-// GFX11: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi|
+// GFX11: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 
-v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
-// GFX11: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
+// GFX11: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1]
-// GFX11: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null
+// GFX11: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-v_dot2_bf16_bf16 v2, v0, 0x20004000, v2
-// GFX11: v_dot2_bf16_bf16 v2, v0, 0x20004000, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l
+// GFX11: v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 
-v_dot2_bf16_bf16 v2, 0x20004000, v0, v2
-// GFX11: v_dot2_bf16_bf16 v2, 0x20004000, v0, v2 ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l
+// GFX11: v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
 
-v_dot2_f16_f16 v5, v1, v2, s3
-// GFX11: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h
+// GFX11: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-v_dot2_f16_f16 v5, v255, s2, s105
-// GFX11: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX11: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_dot2_f16_f16 v5, s1, v255, exec_hi
-// GFX11: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+v_dot2_f16_f16 v5.l, v1, v2, s3
+// GFX11: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 
-v_dot2_f16_f16 v5, s105, s105, exec_lo
-// GFX11: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+v_dot2_f16_f16 v5.l, v255, s2, s105
+// GFX11: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 
-v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3
-// GFX11: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+v_dot2_f16_f16 v5.l, s1, v255, exec_hi
+// GFX11: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 
-v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255
-// GFX11: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, s105, s105, exec_lo
+// GFX11: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 
-v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX11: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX11: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_dot2_f16_f16 v5, m0, 0.5, m0
-// GFX11: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l
+// GFX11: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX11: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX11: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 
-v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX11: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_dot2_f16_f16 v5.l, m0, 0.5, m0
+// GFX11: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 
-v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b|
-// GFX11: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX11: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 
-v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX11: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX11: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_dot2_f16_f16 v5, 0.5, -m0, 0.5 op_sel:[0,0,0,0]
-// GFX11: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b|
+// GFX11: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
-// GFX11: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX11: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1]
-// GFX11: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5
+// GFX11: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
 
-v_dot2_f16_f16 v2, v0, 0x20004000, v2
-// GFX11: v_dot2_f16_f16 v2, v0, 0x20004000, v2   ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
+// GFX11: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_dot2_f16_f16 v2, 0x20004000, v0, v2
-// GFX11: v_dot2_f16_f16 v2, 0x20004000, v0, v2   ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null
+// GFX11: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+
+v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l
+// GFX11: v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+
+v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l
+// GFX11: v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+
+v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h
+// GFX11: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+
+v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX11: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
 
 v_fma_dx9_zero_f32 v5, v1, v2, s3
 // GFX11: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
index ad1d652a2ac1d..9803b1b050d8a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
@@ -6,3 +6,9 @@ v_cvt_pknorm_i16_f16 v5, v1, v2
 
 v_cvt_pknorm_u16_f16 v5, v1, v2
 // GFX11: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
+
+v_add3_nc_u32 v5, v1, v2, s3
+// GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
+
+v_xor_add_u32 v5, v1, v2, s3
+// GFX11: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
index 3f812625c9773..e6f868d2b40e7 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
@@ -4617,44 +4617,128 @@ v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 ban
 v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
 
-v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h quad_perm:[0,1,2,3]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 row_mirror
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 row_half_mirror
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:1
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi row_shl:15
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo row_shr:1
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| row_shr:15
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:1
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_ror:15
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
 
-v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff]
+
+v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+
+v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x09,0x13]
+
+v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
 // GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX11-ERR: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX11: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
+v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
-// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[0,1,2,3]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
-// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h row_mirror
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 row_half_mirror
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 row_shl:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:15
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi row_shr:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo row_shr:15
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| row_ror:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:15
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01]
 
-v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x09,0x13]
 
-v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX11: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
+v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2-fake16.s
new file mode 100644
index 0000000000000..cf3bc7aba5727
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2-fake16.s
@@ -0,0 +1,1987 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x20,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x20,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x20,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x20,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x20,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x20,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x20,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x20,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x03,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x03,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x03,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_add_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x25,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_and_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_and_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_and_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_and_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1b,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_ashrrev_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_ashrrev_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1a,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_mirror
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_half_mirror
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_ror:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s105 row_ror:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x01,0xd5,0xfa,0x04,0xaa,0x41,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x01,0xd5,0xfa,0x04,0xee,0x21,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_mirror
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_half_mirror
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x01,0xd5,0xfa,0x04,0xaa,0x41,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x01,0xd5,0xfa,0x04,0xea,0x21,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x01,0xd5,0xfa,0xfe,0xf3,0x61,0xff,0x6f,0x05,0x30]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x2f,0xd5,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x2f,0xd5,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x2f,0xd5,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x2f,0xd5,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_fmac_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_fmac_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_fmac_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x36,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_fmac_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_fmac_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x36,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_fmac_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_fmac_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x36,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_fmac_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_fmac_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_fmac_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x2b,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_fmac_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_fmac_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x2b,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_fmac_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_fmac_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x2b,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x10,0x01,0x60,0x09,0x13]
+
+v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0x3b,0xd5,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x05,0x30]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_lshlrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_lshlrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x18,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_lshrrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_lshrrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x19,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_max_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_max_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x39,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_max_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_max_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x39,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_max_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_max_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x39,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_max_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_max_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x10,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_max_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_max_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x10,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_max_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_max_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x10,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_max_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_max_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_max_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_max_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x12,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_max_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_max_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_max_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_max_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x14,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_min_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_min_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x3a,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_min_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_min_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x3a,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_min_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_min_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x3a,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_min_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_min_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x0f,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_min_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_min_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x0f,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_min_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_min_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x0f,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_min_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_min_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_min_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_min_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x11,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_min_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_min_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_min_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_min_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x13,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x07,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x07,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x35,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x35,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x35,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x08,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x08,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x08,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_hi_i32_i24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x0a,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_hi_u32_u24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x0c,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_i32_i24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_i32_i24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x09,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_legacy_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x07,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_legacy_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x07,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_u32_u24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_mul_u32_u24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x0b,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_or_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_or_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_or_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_or_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1c,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x21,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x21,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x21,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x21,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x21,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x21,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x21,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x21,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x04,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x04,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x04,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_sub_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x26,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x22,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x22,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x22,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x22,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x22,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x22,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x22,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x22,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x34,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x34,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x34,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x05,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x05,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x05,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x27,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_xnor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_xnor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1e,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_xor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_xor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1d,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2.s
index fccdf3cf579c2..a63637dc22e3a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vop2.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
@@ -111,47 +111,59 @@ v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mas
 v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x20,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
 
-v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_mirror
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_mirror
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_half_mirror
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shl:1
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shl:15
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shr:1
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shr:15
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_ror:1
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_ror:15
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_f16_e64_dpp v5.h, v1.h, v2.h row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_add_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x09,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x12,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc3,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
 v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1543,47 +1555,59 @@ v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mas
 v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x21,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
 
-v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_mirror
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_mirror
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shl:1
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shl:15
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shr:1
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shr:15
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_ror:1
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_ror:15
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_sub_f16_e64_dpp v5.h, v1.h, v2.h row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_sub_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX11: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x09,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX11: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x12,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX11: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc3,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
 v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX11: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc-fake16.s
new file mode 100644
index 0000000000000..dc16b6ae6eaa2
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc-fake16.s
@@ -0,0 +1,6853 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_class_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_class_f16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_class_f16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_class_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_class_f16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_class_f16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp null, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_class_f16_e64_dpp null, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x01,0x7d,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_class_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_class_f32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_class_f32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_class_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_class_f32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_class_f32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp null, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_class_f32_e64_dpp null, -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x01,0x7e,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x02,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x02,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x02,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x02,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x02,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x02,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x12,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x12,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x12,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x12,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x12,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x12,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x32,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x32,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x42,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x42,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x3a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x3a,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_eq_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_eq_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_eq_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_eq_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_eq_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_eq_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_eq_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_eq_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4a,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_f_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_f_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_f_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x00,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_f_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x00,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_f_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x00,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_f_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x00,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_f_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x00,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_f_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x00,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_f_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_f_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_f_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x10,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_f_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x10,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_f_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x10,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_f_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x10,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_f_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x10,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_f_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x10,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_f_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_f_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_f_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_f_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_f_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_f_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_f_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x40,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_f_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x40,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_f_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_f_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_f_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_f_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_f_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_f_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_f_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x48,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_f_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x48,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x06,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x06,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x06,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x06,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x06,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x06,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x16,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x16,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x16,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x16,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x16,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x16,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x36,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x36,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x46,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x46,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x3e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x3e,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ge_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ge_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ge_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ge_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ge_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ge_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ge_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ge_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4e,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x04,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x04,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x04,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x04,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x04,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x04,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x14,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x14,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x14,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x14,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x14,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x14,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x34,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x34,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x44,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x44,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x3c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x3c,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_gt_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_gt_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_gt_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_gt_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_gt_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_gt_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_gt_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_gt_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4c,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x03,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x03,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x03,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x03,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x03,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x03,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x13,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x13,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x13,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x13,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x13,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x13,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x33,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x33,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x43,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x43,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x3b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x3b,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_le_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_le_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_le_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_le_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_le_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_le_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_le_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_le_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4b,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lg_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lg_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x05,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lg_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x05,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lg_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x05,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lg_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x05,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lg_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x05,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lg_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x05,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lg_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lg_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lg_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x15,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lg_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x15,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lg_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x15,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lg_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x15,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lg_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x15,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lg_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x15,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x01,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x01,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x01,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x01,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x01,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x01,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x11,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x11,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x11,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x11,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x11,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x11,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x31,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x31,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x41,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x41,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x39,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x39,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_lt_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_lt_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_lt_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_lt_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_lt_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_lt_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_lt_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x49,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_lt_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x49,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ne_i16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ne_i16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ne_i16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ne_i16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ne_i16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x35,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ne_i16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x35,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ne_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ne_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ne_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ne_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ne_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ne_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ne_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x45,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ne_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x45,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ne_u16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ne_u16_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ne_u16_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ne_u16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ne_u16_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x3d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ne_u16_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x3d,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ne_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ne_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ne_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ne_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ne_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ne_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ne_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ne_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4d,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_neq_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_neq_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_neq_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_neq_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_neq_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_neq_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_neq_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_neq_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_neq_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_neq_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_neq_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_neq_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_neq_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_neq_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_neq_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nge_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nge_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x09,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nge_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x09,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nge_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x09,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nge_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x09,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nge_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x09,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nge_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x09,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nge_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nge_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nge_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x19,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nge_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x19,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nge_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x19,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nge_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x19,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nge_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x19,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nge_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x19,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ngt_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ngt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ngt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ngt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ngt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ngt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ngt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_ngt_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_ngt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_ngt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_ngt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_ngt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_ngt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_ngt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_ngt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nle_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nle_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nle_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nle_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nle_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nle_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nle_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nle_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nle_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nle_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nle_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nle_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nle_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nle_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nle_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nlg_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nlg_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nlg_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nlg_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nlg_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nlg_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nlg_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nlg_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nlg_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nlg_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nlg_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nlg_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nlg_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nlg_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nlg_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nlt_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nlt_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nlt_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nlt_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nlt_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nlt_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nlt_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_nlt_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_nlt_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_nlt_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_nlt_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_nlt_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_nlt_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_nlt_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_nlt_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_o_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_o_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x07,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_o_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x07,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_o_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x07,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_o_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x07,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_o_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x07,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_o_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x07,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_o_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_o_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_o_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x17,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_o_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x17,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_o_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x17,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_o_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x17,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_o_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x17,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_o_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x17,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_i32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_i32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_i32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_i32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_i32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_i32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_i32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x47,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_i32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x47,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_u32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_u32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_u32_e64_dpp vcc_hi, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_u32_e64_dpp ttmp15, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_u32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_u32_e64_dpp vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_u32_e64_dpp ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x00,0x4f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_u32_e64_dpp null, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x00,0x4f,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x0f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x0f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x0f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x0f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x0f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x0f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_t_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x1f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x1f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_t_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x1f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x1f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x1f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x1f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_u_f16_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_u_f16_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x08,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_u_f16_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x08,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_u_f16_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x08,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_u_f16_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x08,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_u_f16_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x08,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_u_f16_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x08,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3]
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_mirror
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_half_mirror
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_shl:1
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_shl:15
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_shr:1
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_shr:15
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 row_ror:1
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s105, v1, v2 row_ror:15
+// W32: v_cmp_u_f32_e64_dpp s105, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x69,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cmp_u_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cmp_u_f32_e64_dpp vcc_hi, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6b,0x01,0x18,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cmp_u_f32_e64_dpp ttmp15, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7b,0x02,0x18,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0]
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3]
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_mirror
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_half_mirror
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shl:1
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shl:15
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shr:1
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shr:15
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_ror:1
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_ror:15
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cmp_u_f32_e64_dpp s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x68,0x00,0x18,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cmp_u_f32_e64_dpp vcc, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x6a,0x01,0x18,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cmp_u_f32_e64_dpp ttmp[14:15], -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7a,0x02,0x18,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmp_u_f32_e64_dpp null, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7c,0x83,0x18,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc.s
index 996df6995913d..1b9092d30b1b7 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopc.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
 // W32: v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx-fake16.s
new file mode 100644
index 0000000000000..7ed43e8689f69
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx-fake16.s
@@ -0,0 +1,2692 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+
+v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f16_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_class_f16_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x01,0xfd,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
+
+v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_class_f32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x01,0xfe,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x82,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x92,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x92,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x92,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x92,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x80,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x80,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x90,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x90,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x90,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x90,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc0,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_f_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_f_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_f_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_f_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x86,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x96,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x96,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x96,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x96,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x84,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x94,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x94,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x94,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x94,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x83,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x93,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x93,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x93,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x93,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_le_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_le_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_le_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x85,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x95,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lg_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lg_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x95,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x95,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_lg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x95,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x81,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x81,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x81,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x91,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x91,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x91,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x91,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_lt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_lt_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ne_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ne_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ne_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_neq_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_neq_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9d,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_neq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_neq_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9d,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x89,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x99,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nge_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nge_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x99,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nge_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x99,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x99,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9b,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_ngt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_ngt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9b,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_ngt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_ngt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9b,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9c,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nle_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nle_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9c,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nle_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nle_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9c,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9a,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlg_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlg_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9a,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlg_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9a,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_nlt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_nlt_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9e,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_nlt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_nlt_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9e,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x87,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x97,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_o_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_o_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x97,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_o_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_o_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x97,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x97,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_t_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_i32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xc7,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_t_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_cmpx_t_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_cmpx_t_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_u32_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_tru_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x8f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_tru_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_tru_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x8f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x9f,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_tru_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_t_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x9f,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_tru_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_t_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x9f,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_tru_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x9f,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x88,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3]
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_mirror
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_half_mirror
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_shl:1
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_shl:15
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_shr:1
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_shr:15
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_ror:1
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_ror:15
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x98,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmpx_u_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX11: v_cmpx_u_f32_e64_dpp |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x7e,0x01,0x98,0xd4,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cmpx_u_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX11: v_cmpx_u_f32_e64_dpp -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x7e,0x02,0x98,0xd4,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cmpx_u_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX11: v_cmpx_u_f32_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0x7e,0x83,0x98,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
index d97b7eb6bd155..462ad7ba6516d 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16_from_vopcx.s
@@ -1,6 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 
 v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
index ff7b114b128cf..160863b19012d 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
@@ -2974,44 +2974,125 @@ v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0]
 v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
 
-v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX11-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
 
-v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
+v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x03,0x66,0xd6,0xea,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+
+v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x66,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
 // GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX11: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x03,0x67,0xd6,0xea,0x04,0x06,0x63,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
+v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2-fake16.s
new file mode 100644
index 0000000000000..27e83473b9437
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2-fake16.s
@@ -0,0 +1,551 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x20,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x20,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x20,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x20,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x20,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x20,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x20,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x20,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x20,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x03,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x03,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x03,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x25,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x25,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1b,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_and_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1b,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1a,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_ashrrev_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1a,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x01,0xd5,0xe9,0x04,0xaa,0x41,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x01,0xd5,0xea,0x04,0xee,0x21,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x01,0xd5,0xe9,0x04,0xaa,0x41,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x01,0xd5,0xea,0x04,0xea,0x21,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x01,0xd5,0xe9,0xfe,0xf3,0x61,0xff,0x00,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd5,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x2f,0xd5,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd5,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x2f,0xd5,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_fmac_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x36,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_fmac_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x36,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_fmac_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_fmac_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x36,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_fmac_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_fmac_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x36,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_fmac_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_fmac_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fmac_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2b,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_fmac_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_fmac_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x2b,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_fmac_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_fmac_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x2b,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x3b,0xd5,0xea,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x3b,0xd5,0xe9,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x18,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_lshlrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x18,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x19,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_lshrrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x19,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_max_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x39,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_max_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_max_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x39,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_max_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_max_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x39,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_max_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x10,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_max_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_max_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x10,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_max_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_max_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x10,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x12,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_max_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x12,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x14,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x14,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_max_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x14,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_min_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x3a,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_min_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_min_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x3a,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_min_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_min_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x3a,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_min_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0f,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_min_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_min_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x0f,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_min_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_min_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x0f,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x11,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_min_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x11,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x13,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_min_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x13,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x07,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x07,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x35,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x35,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x35,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x08,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x08,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x08,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x08,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0a,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_hi_i32_i24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x0a,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0c,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_hi_u32_u24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x0c,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x09,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_i32_i24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x09,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x07,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x07,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0b,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_mul_u32_u24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x0b,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1c,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_or_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1c,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x21,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x21,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x21,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x21,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x21,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x21,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x21,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x21,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x21,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x21,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x04,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x04,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x04,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x26,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x26,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x22,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x22,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x22,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x22,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x22,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x22,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x22,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x22,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x22,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x22,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_subrev_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x34,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x34,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x34,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_subrev_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x05,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x05,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x05,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x27,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x27,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x27,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1e,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_xnor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1e,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1d,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_xor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1d,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2.s
index 29d65d816ae58..49ee9a7b02bed 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vop2.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
@@ -43,17 +43,29 @@ v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1
 v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x20,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_f16_e64_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x09,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x12,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc3,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
 
 v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -427,17 +439,29 @@ v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1
 v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x21,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_sub_f16_e64_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x09,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX11: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x12,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX11: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc3,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
 
 v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc-fake16.s
new file mode 100644
index 0000000000000..28012b16a2b01
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc-fake16.s
@@ -0,0 +1,2501 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_cmp_class_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_class_f16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x7d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_class_f16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x7d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f16_e64_dpp null, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_class_f16_e64_dpp null, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x01,0x7d,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_class_f32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_class_f32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x7e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_class_f32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_class_f32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x7e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_cmp_class_f32_e64_dpp null, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_class_f32_e64_dpp null, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x01,0x7e,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x02,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x02,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x02,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x02,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x02,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x02,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x12,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x12,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x12,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x12,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x12,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x12,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x12,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x12,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x12,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x32,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x32,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x32,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x32,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x42,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x42,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x42,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x42,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x3a,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x3a,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x3a,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_eq_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_eq_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_eq_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4a,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_eq_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_eq_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4a,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_eq_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_eq_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4a,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_f_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x00,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_f_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x00,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x00,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x00,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_f_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x00,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_f_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x00,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_f_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x10,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x10,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x10,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_f_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x10,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x10,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x10,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x10,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_f_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x10,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_f_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x10,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_f_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_f_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x40,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x40,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_f_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x40,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_f_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x40,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_f_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_f_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_f_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x48,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_f_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x48,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_f_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x48,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_f_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_f_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x48,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x06,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x06,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x06,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x06,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x06,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x06,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x16,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x16,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x16,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x16,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x16,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x16,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x16,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x16,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x16,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x36,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x36,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x36,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x36,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x46,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x46,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x46,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x46,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x3e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x3e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x3e,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ge_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ge_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ge_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ge_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ge_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4e,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ge_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ge_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4e,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x04,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x04,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x04,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x04,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x04,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x04,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x14,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x14,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x14,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x14,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x14,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x14,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x14,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x14,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x14,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x14,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x34,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x34,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x34,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x34,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x44,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x44,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x44,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x44,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x3c,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x3c,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x3c,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_gt_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_gt_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_gt_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4c,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_gt_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_gt_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4c,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_gt_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_gt_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4c,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_le_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x03,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x03,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x03,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x03,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x03,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x03,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_le_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x13,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x13,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x13,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x13,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x13,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x13,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x13,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x13,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x13,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_le_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x33,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x33,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x33,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x33,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_le_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x43,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x43,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x43,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x43,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_le_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x3b,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x3b,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x3b,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_le_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_le_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_le_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4b,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_le_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_le_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4b,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_le_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_le_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4b,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_lg_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x05,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lg_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x05,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x05,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x05,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lg_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x05,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lg_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x05,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_lg_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x15,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x15,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lg_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x15,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lg_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x15,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x15,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x15,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lg_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x15,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lg_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x15,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lg_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lg_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x15,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x01,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x01,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x01,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x01,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x01,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x01,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x11,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x11,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x11,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x11,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x11,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x11,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x11,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x11,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x11,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x31,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x31,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x31,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x31,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x41,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x41,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x41,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x41,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x39,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x39,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x39,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x39,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_lt_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_lt_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_lt_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x49,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_lt_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x49,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_lt_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x49,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_lt_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_lt_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x49,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ne_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ne_i16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x35,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x35,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ne_i16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x35,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ne_i16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x35,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ne_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ne_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x45,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ne_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x45,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ne_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x45,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ne_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u16_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ne_u16_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x3d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u16_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x3d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ne_u16_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x3d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ne_u16_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x3d,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_ne_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ne_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ne_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ne_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ne_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4d,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cmp_ne_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ne_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4d,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_neq_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_neq_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_neq_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_neq_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_neq_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_neq_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_neq_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_neq_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_neq_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_neq_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_neq_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nge_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x09,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nge_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x09,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x09,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x09,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nge_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x09,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nge_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x09,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nge_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x19,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x19,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nge_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x19,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nge_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x19,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x19,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x19,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nge_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x19,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nge_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x19,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nge_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nge_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x19,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_ngt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ngt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ngt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ngt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_ngt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_ngt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_ngt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_ngt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_ngt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_ngt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_ngt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nle_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nle_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nle_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nle_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nle_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nle_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nle_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nle_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nle_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nle_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nle_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nlg_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nlg_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nlg_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nlg_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nlg_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlg_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nlg_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlg_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nlg_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlg_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nlg_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nlt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nlt_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nlt_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nlt_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_nlt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_nlt_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_nlt_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_nlt_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_nlt_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_nlt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_nlt_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_o_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x07,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_o_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x07,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x07,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x07,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_o_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x07,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_o_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x07,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_o_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x17,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x17,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_o_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x17,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_o_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x17,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x17,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x17,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_o_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x17,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_o_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x17,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_o_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_o_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x17,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_t_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_t_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_t_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_i32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_i32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_i32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_i32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_i32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x47,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_i32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_i32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_i32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x47,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_i32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x47,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_i32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x47,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_t_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_u32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_u32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_u32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_u32_e64_dpp vcc_hi, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_u32_e64_dpp ttmp15, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x00,0x4f,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_u32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_u32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_u32_e64_dpp vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x4f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_u32_e64_dpp ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x00,0x4f,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_t_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_u32_e64_dpp null, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x00,0x4f,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmp_tru_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x0f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x0f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x0f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x0f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x0f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x0f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_tru_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_t_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x1f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_t_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x1f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x1f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_t_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x1f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_t_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x1f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:23: error: invalid operand for instruction
+
+v_cmp_tru_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_t_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x1f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_u_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f16_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x08,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_u_f16_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x08,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x08,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f16_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x08,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_u_f16_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x08,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_u_f16_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x08,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmp_u_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f32_e64_dpp s105, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x69,0x00,0x18,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x18,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cmp_u_f32_e64_dpp vcc_hi, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6b,0x01,0x18,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cmp_u_f32_e64_dpp ttmp15, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7b,0x02,0x18,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f32_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x18,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f32_e64_dpp s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x68,0x00,0x18,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cmp_u_f32_e64_dpp vcc, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x01,0x18,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cmp_u_f32_e64_dpp ttmp[14:15], -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7a,0x02,0x18,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_u_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmp_u_f32_e64_dpp null, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7c,0x83,0x18,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc.s
index 4d73ca321465c..1efbd671cfaee 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopc.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_cmp_class_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // W32: v_cmp_class_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx-fake16.s
new file mode 100644
index 0000000000000..e3093d324a722
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx-fake16.s
@@ -0,0 +1,681 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xfd,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_class_f16_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x01,0xfd,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
+
+v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xfe,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_class_f32_e64_dpp -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x01,0xfe,0xd4,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x82,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x82,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x82,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x82,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x92,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x92,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x92,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x92,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc2,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc2,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc2,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xba,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xca,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_eq_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xca,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_eq_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xca,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x80,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x80,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x80,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x80,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x90,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x90,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x90,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x90,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc0,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc0,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc0,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_f_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc8,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_f_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc8,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_f_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_f_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc8,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x86,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x86,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x86,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x86,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x96,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x96,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x96,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x96,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc6,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc6,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc6,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xce,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ge_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xce,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ge_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xce,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x84,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x84,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x84,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x84,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x94,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x94,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x94,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x94,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc4,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc4,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc4,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xcc,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_gt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xcc,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_gt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcc,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x83,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x83,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x83,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x83,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x93,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x93,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x93,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x93,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc3,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc3,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc3,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_le_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xcb,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_le_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xcb,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_le_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcb,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x85,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x85,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x85,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x85,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x95,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lg_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x95,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lg_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x95,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x95,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x81,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x81,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x81,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x81,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x91,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x91,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x91,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x91,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc1,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc1,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc1,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc9,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_lt_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc9,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_lt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_lt_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc9,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ne_i16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ne_i16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc5,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ne_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc5,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ne_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc5,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ne_u16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ne_u16_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xcd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ne_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xcd,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ne_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcd,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_neq_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9d,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_neq_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9d,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_neq_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9d,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x89,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x89,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x89,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x89,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x99,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nge_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x99,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nge_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x99,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nge_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x99,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9b,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_ngt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9b,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_ngt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9b,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_ngt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9b,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9c,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nle_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9c,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nle_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9c,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nle_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9c,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9a,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlg_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9a,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlg_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9a,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlg_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9a,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_nlt_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9e,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_nlt_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9e,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_nlt_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9e,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x87,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x87,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x87,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x87,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x97,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_o_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x97,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_o_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x97,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_o_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x97,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xc7,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_i32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xc7,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_i32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xc7,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_t_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xcf,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_u32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x00,0xcf,0xd4,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_t_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_u32_e64_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x00,0xcf,0xd4,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cmpx_tru_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x8f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x8f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x8f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_tru_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x9f,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_t_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x9f,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_t_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x9f,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_tru_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_t_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x9f,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x88,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f16_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x88,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f16_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x88,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x88,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x98,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_u_f32_e64_dpp |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x01,0x98,0xd4,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX11: v_cmpx_u_f32_e64_dpp -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x7e,0x02,0x98,0xd4,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_u_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX11: v_cmpx_u_f32_e64_dpp -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x7e,0x83,0x98,0xd4,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
index a950c75dbf85d..46f1db837b0dd 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8_from_vopcx.s
@@ -1,6 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2-fake16.s
new file mode 100644
index 0000000000000..83f7c0b922ec0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2-fake16.s
@@ -0,0 +1,2188 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x20,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_add_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x20,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_add_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x20,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_add_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x20,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_add_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x20,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_add_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x20,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_add_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x20,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x20,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX11: v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x20,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_add_f16_e64 v5, v1, v2
+// GFX11: v_add_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f16_e64 v5, v255, v255
+// GFX11: v_add_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f16_e64 v5, s1, s2
+// GFX11: v_add_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_f16_e64 v5, s105, s105
+// GFX11: v_add_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_add_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_add_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_add_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_f16_e64 v5, m0, 0.5
+// GFX11: v_add_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_f16_e64 v5, exec_lo, -1
+// GFX11: v_add_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_f16_e64 v5, |exec_hi|, null
+// GFX11: v_add_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_f16_e64 v5, null, exec_lo
+// GFX11: v_add_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_f16_e64 v5, -1, exec_hi
+// GFX11: v_add_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_add_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_add_f32_e64 v5, v1, v2
+// GFX11: v_add_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f32_e64 v5, v255, v255
+// GFX11: v_add_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x03,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f32_e64 v5, s1, s2
+// GFX11: v_add_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_f32_e64 v5, s105, s105
+// GFX11: v_add_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x03,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_add_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x03,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_add_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x03,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_add_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_add_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x03,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_f32_e64 v5, m0, 0.5
+// GFX11: v_add_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x03,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_f32_e64 v5, exec_lo, -1
+// GFX11: v_add_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x03,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_f32_e64 v5, |exec_hi|, null
+// GFX11: v_add_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x03,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_f32_e64 v5, null, exec_lo
+// GFX11: v_add_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x03,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_f32_e64 v5, -1, exec_hi
+// GFX11: v_add_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x03,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_add_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x03,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_add_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_add_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x03,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_add_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_add_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x03,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32_e64 v5, v1, v2
+// GFX11: v_add_nc_u32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x25,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_nc_u32_e64 v5, v255, v255
+// GFX11: v_add_nc_u32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x25,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_nc_u32_e64 v5, s1, s2
+// GFX11: v_add_nc_u32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x25,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_nc_u32_e64 v5, s105, s105
+// GFX11: v_add_nc_u32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x25,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_add_nc_u32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x25,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_add_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32_e64 v5, ttmp15, src_scc
+// GFX11: v_add_nc_u32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x25,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_nc_u32_e64 v5, m0, 0.5
+// GFX11: v_add_nc_u32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x25,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_nc_u32_e64 v5, exec_lo, -1
+// GFX11: v_add_nc_u32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x25,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_nc_u32_e64 v5, exec_hi, null
+// GFX11: v_add_nc_u32_e64 v5, exec_hi, null      ; encoding: [0x05,0x00,0x25,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_nc_u32_e64 v5, null, exec_lo
+// GFX11: v_add_nc_u32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x25,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_nc_u32_e64 v5, -1, exec_hi
+// GFX11: v_add_nc_u32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x25,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_nc_u32_e64 v5, 0.5, m0
+// GFX11: v_add_nc_u32_e64 v5, 0.5, m0            ; encoding: [0x05,0x00,0x25,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_add_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX11: v_add_nc_u32_e64 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x25,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_add_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX11: v_add_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_and_b32_e64 v5, v1, v2
+// GFX11: v_and_b32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x1b,0xd5,0x01,0x05,0x02,0x00]
+
+v_and_b32_e64 v5, v255, v255
+// GFX11: v_and_b32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x1b,0xd5,0xff,0xff,0x03,0x00]
+
+v_and_b32_e64 v5, s1, s2
+// GFX11: v_and_b32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x1b,0xd5,0x01,0x04,0x00,0x00]
+
+v_and_b32_e64 v5, s105, s105
+// GFX11: v_and_b32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x1b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_and_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_and_b32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x1b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_and_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_and_b32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x1b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_and_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_and_b32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x1b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_and_b32_e64 v5, m0, 0.5
+// GFX11: v_and_b32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x1b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_and_b32_e64 v5, exec_lo, -1
+// GFX11: v_and_b32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x1b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_and_b32_e64 v5, exec_hi, null
+// GFX11: v_and_b32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x1b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_and_b32_e64 v5, null, exec_lo
+// GFX11: v_and_b32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x1b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_and_b32_e64 v5, -1, exec_hi
+// GFX11: v_and_b32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x1b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_and_b32_e64 v5, 0.5, m0
+// GFX11: v_and_b32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x1b,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_and_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_and_b32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x1b,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_and_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_and_b32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x1b,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32_e64 v5, v1, v2
+// GFX11: v_ashrrev_i32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x1a,0xd5,0x01,0x05,0x02,0x00]
+
+v_ashrrev_i32_e64 v5, v255, v255
+// GFX11: v_ashrrev_i32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x1a,0xd5,0xff,0xff,0x03,0x00]
+
+v_ashrrev_i32_e64 v5, s1, s2
+// GFX11: v_ashrrev_i32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x1a,0xd5,0x01,0x04,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, s105, s105
+// GFX11: v_ashrrev_i32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x1a,0xd5,0x69,0xd2,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_ashrrev_i32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x1a,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_ashrrev_i32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32_e64 v5, ttmp15, src_scc
+// GFX11: v_ashrrev_i32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x1a,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, m0, 0.5
+// GFX11: v_ashrrev_i32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x1a,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, exec_lo, -1
+// GFX11: v_ashrrev_i32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x1a,0xd5,0x7e,0x82,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, exec_hi, null
+// GFX11: v_ashrrev_i32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x1a,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, null, exec_lo
+// GFX11: v_ashrrev_i32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x1a,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, -1, exec_hi
+// GFX11: v_ashrrev_i32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x1a,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, 0.5, m0
+// GFX11: v_ashrrev_i32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x1a,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, src_scc, vcc_lo
+// GFX11: v_ashrrev_i32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x1a,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_ashrrev_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_ashrrev_i32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cndmask_b32_e64 v5, v1, 0xaf123456, s3
+// W32: v_cndmask_b32_e64 v5, v1, 0xaf123456, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v255, src_scc, s3
+// W32: v_cndmask_b32_e64 v5, v255, src_scc, s3 ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, s105, s105, s3
+// W32: v_cndmask_b32_e64 v5, s105, s105, s3    ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_lo, v2, s3
+// W32: v_cndmask_b32_e64 v5, vcc_lo, v2, s3    ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_hi, v255, s3
+// W32: v_cndmask_b32_e64 v5, vcc_hi, v255, s3  ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:37: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, ttmp15, ttmp15, s3
+// W32: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, m0, 0.5, s3
+// W32: v_cndmask_b32_e64 v5, m0, 0.5, s3       ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_lo, exec_lo, s3
+// W32: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_hi, -1, s3
+// W32: v_cndmask_b32_e64 v5, exec_hi, -1, s3   ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:36: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, null, exec_hi, s105
+// W32: v_cndmask_b32_e64 v5, null, exec_hi, s105 ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -1, m0, vcc_lo
+// W32: v_cndmask_b32_e64 v5, -1, m0, vcc_lo    ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:31: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc_hi
+// W32: v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc_hi ; encoding: [0x05,0x02,0x01,0xd5,0xf0,0xd4,0xac,0x41]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp15
+// W32: v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp15 ; encoding: [0x05,0x01,0x01,0xd5,0xfd,0xf8,0xec,0x21]
+// W64-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v1, 0xaf123456, s[6:7]
+// W64: v_cndmask_b32_e64 v5, v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v255, src_scc, s[6:7]
+// W64: v_cndmask_b32_e64 v5, v255, src_scc, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, s105, s105, s[6:7]
+// W64: v_cndmask_b32_e64 v5, s105, s105, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7]
+// W64: v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_hi, v255, s[6:7]
+// W64: v_cndmask_b32_e64 v5, vcc_hi, v255, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:37: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, ttmp15, ttmp15, s[6:7]
+// W64: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, m0, 0.5, s[6:7]
+// W64: v_cndmask_b32_e64 v5, m0, 0.5, s[6:7]   ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_lo, exec_lo, s[6:7]
+// W64: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_hi, -1, s[6:7]
+// W64: v_cndmask_b32_e64 v5, exec_hi, -1, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:36: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, null, exec_hi, s[6:7]
+// W64: v_cndmask_b32_e64 v5, null, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -1, m0, s[104:105]
+// W64: v_cndmask_b32_e64 v5, -1, m0, s[104:105] ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:31: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc
+// W64: v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc ; encoding: [0x05,0x02,0x01,0xd5,0xf0,0xd4,0xa8,0x41]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp[14:15]
+// W64: v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp[14:15] ; encoding: [0x05,0x01,0x01,0xd5,0xfd,0xf8,0xe8,0x21]
+// W32-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v255, -|0xaf123456|, -|vcc_hi|, null
+// GFX11: v_cndmask_b32_e64 v255, -|0xaf123456|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x01,0xd5,0xff,0xd6,0xf0,0x61,0x56,0x34,0x12,0xaf]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x05,0x02,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255 ; encoding: [0x05,0x00,0x2f,0xd5,0xff,0xff,0x03,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x04,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105 ; encoding: [0x05,0x00,0x2f,0xd5,0x69,0xd2,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x2f,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x2f,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x2f,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5    ; encoding: [0x05,0x00,0x2f,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x2f,0xd5,0x7e,0x82,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x2f,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x2f,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x2f,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0   ; encoding: [0x05,0x00,0x2f,0xd5,0xf0,0xfa,0x00,0x40]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo|
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x2f,0xd5,0xfd,0xd4,0x00,0x20]
+
+v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0xff,0x83,0x2f,0xd5,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32_e64 v5, v1, v2
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x05,0x02,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, v255, v255
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255 ; encoding: [0x05,0x00,0x2f,0xd5,0xff,0xff,0x03,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, s1, s2
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x04,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, s105, s105
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105 ; encoding: [0x05,0x00,0x2f,0xd5,0x69,0xd2,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x2f,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x2f,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x2f,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, m0, 0.5
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5    ; encoding: [0x05,0x00,0x2f,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, exec_lo, -1
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x2f,0xd5,0x7e,0x82,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, |exec_hi|, null
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x2f,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, null, exec_lo
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x2f,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, -1, exec_hi
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x2f,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, 0.5, -m0
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0   ; encoding: [0x05,0x00,0x2f,0xd5,0xf0,0xfa,0x00,0x40]
+
+v_cvt_pkrtz_f16_f32_e64 v5, -src_scc, |vcc_lo|
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x2f,0xd5,0xfd,0xd4,0x00,0x20]
+
+v_cvt_pkrtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0xff,0x83,0x2f,0xd5,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_fmac_dx9_zero_f32_e64 v5, v1, v2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, v1, v2      ; encoding: [0x05,0x00,0x06,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, v255, v255
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, v255, v255  ; encoding: [0x05,0x00,0x06,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, s1, s2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, s1, s2      ; encoding: [0x05,0x00,0x06,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, s105, s105
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, s105, s105  ; encoding: [0x05,0x00,0x06,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x06,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_fmac_dx9_zero_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x06,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, m0, 0.5
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, m0, 0.5     ; encoding: [0x05,0x00,0x06,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, exec_lo, -1
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x06,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, |exec_hi|, null
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x06,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, null, exec_lo
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x06,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, -1, exec_hi
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x06,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x06,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x06,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_fmac_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x06,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_fmac_f16_e64 v5, v1, v2
+// GFX11: v_fmac_f16_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_f16_e64 v5, v255, v255
+// GFX11: v_fmac_f16_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x36,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_f16_e64 v5, s1, s2
+// GFX11: v_fmac_f16_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x36,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_f16_e64 v5, s105, s105
+// GFX11: v_fmac_f16_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x36,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_fmac_f16_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x36,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_fmac_f16_e64 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x36,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_fmac_f16_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x36,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_f16_e64 v5, m0, 0.5
+// GFX11: v_fmac_f16_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x36,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_f16_e64 v5, exec_lo, -1
+// GFX11: v_fmac_f16_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x36,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_f16_e64 v5, |exec_hi|, null
+// GFX11: v_fmac_f16_e64 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x36,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_f16_e64 v5, null, exec_lo
+// GFX11: v_fmac_f16_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x36,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_f16_e64 v5, -1, exec_hi
+// GFX11: v_fmac_f16_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x36,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_fmac_f16_e64 v5, 0.5, -m0 mul:2       ; encoding: [0x05,0x00,0x36,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_fmac_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x36,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_fmac_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x36,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f32_e64 v5, v1, v2
+// GFX11: v_fmac_f32_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x2b,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_f32_e64 v5, v255, v255
+// GFX11: v_fmac_f32_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x2b,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_f32_e64 v5, s1, s2
+// GFX11: v_fmac_f32_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x2b,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_f32_e64 v5, s105, s105
+// GFX11: v_fmac_f32_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x2b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_fmac_f32_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x2b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_fmac_f32_e64 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x2b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_fmac_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_fmac_f32_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x2b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_f32_e64 v5, m0, 0.5
+// GFX11: v_fmac_f32_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x2b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_f32_e64 v5, exec_lo, -1
+// GFX11: v_fmac_f32_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x2b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_f32_e64 v5, |exec_hi|, null
+// GFX11: v_fmac_f32_e64 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x2b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_f32_e64 v5, null, exec_lo
+// GFX11: v_fmac_f32_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x2b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_f32_e64 v5, -1, exec_hi
+// GFX11: v_fmac_f32_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x2b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_fmac_f32_e64 v5, 0.5, -m0 mul:2       ; encoding: [0x05,0x00,0x2b,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_fmac_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x2b,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_fmac_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x2b,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_fmac_legacy_f32_e64 v5, v1, v2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, v1, v2      ; encoding: [0x05,0x00,0x06,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_legacy_f32_e64 v5, v255, v255
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, v255, v255  ; encoding: [0x05,0x00,0x06,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_legacy_f32_e64 v5, s1, s2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, s1, s2      ; encoding: [0x05,0x00,0x06,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, s105, s105
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, s105, s105  ; encoding: [0x05,0x00,0x06,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x06,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_fmac_legacy_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x06,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_legacy_f32_e64 v5, m0, 0.5
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, m0, 0.5     ; encoding: [0x05,0x00,0x06,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_legacy_f32_e64 v5, exec_lo, -1
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x06,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_legacy_f32_e64 v5, |exec_hi|, null
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x06,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, null, exec_lo
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x06,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, -1, exec_hi
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x06,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_legacy_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x06,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_legacy_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_fmac_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x06,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_legacy_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_fmac_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x06,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_ldexp_f16_e64 v5, v1, v2
+// GFX11: v_ldexp_f16_e64 v5, v1, v2              ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x05,0x02,0x00]
+
+v_ldexp_f16_e64 v5, v255, v255
+// GFX11: v_ldexp_f16_e64 v5, v255, v255          ; encoding: [0x05,0x00,0x3b,0xd5,0xff,0xff,0x03,0x00]
+
+v_ldexp_f16_e64 v5, s1, s2
+// GFX11: v_ldexp_f16_e64 v5, s1, s2              ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x04,0x00,0x00]
+
+v_ldexp_f16_e64 v5, s105, s105
+// GFX11: v_ldexp_f16_e64 v5, s105, s105          ; encoding: [0x05,0x00,0x3b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_ldexp_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_ldexp_f16_e64 v5, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_ldexp_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_ldexp_f16_e64 v5, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3b,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_ldexp_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_ldexp_f16_e64 v5, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_ldexp_f16_e64 v5, m0, 0.5
+// GFX11: v_ldexp_f16_e64 v5, m0, 0.5             ; encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_ldexp_f16_e64 v5, exec_lo, -1
+// GFX11: v_ldexp_f16_e64 v5, exec_lo, -1         ; encoding: [0x05,0x00,0x3b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_ldexp_f16_e64 v5, exec_hi, null
+// GFX11: v_ldexp_f16_e64 v5, exec_hi, null       ; encoding: [0x05,0x00,0x3b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_ldexp_f16_e64 v5, null, exec_lo
+// GFX11: v_ldexp_f16_e64 v5, null, exec_lo       ; encoding: [0x05,0x00,0x3b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_ldexp_f16_e64 v5, -1, exec_hi
+// GFX11: v_ldexp_f16_e64 v5, -1, exec_hi         ; encoding: [0x05,0x00,0x3b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_ldexp_f16_e64 v5, 0.5, m0 mul:2
+// GFX11: v_ldexp_f16_e64 v5, 0.5, m0 mul:2       ; encoding: [0x05,0x00,0x3b,0xd5,0xf0,0xfa,0x00,0x08]
+
+v_ldexp_f16_e64 v5, src_scc, vcc_lo mul:4
+// GFX11: v_ldexp_f16_e64 v5, src_scc, vcc_lo mul:4 ; encoding: [0x05,0x00,0x3b,0xd5,0xfd,0xd4,0x00,0x10]
+
+v_ldexp_f16_e64 v255, -|0xfe0b|, vcc_hi clamp div:2
+// GFX11: v_ldexp_f16_e64 v255, -|0xfe0b|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x3b,0xd5,0xff,0xd6,0x00,0x38,0x0b,0xfe,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, v1, v2
+// GFX11: v_lshlrev_b32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x18,0xd5,0x01,0x05,0x02,0x00]
+
+v_lshlrev_b32_e64 v5, v255, v255
+// GFX11: v_lshlrev_b32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x18,0xd5,0xff,0xff,0x03,0x00]
+
+v_lshlrev_b32_e64 v5, s1, s2
+// GFX11: v_lshlrev_b32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x18,0xd5,0x01,0x04,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, s105, s105
+// GFX11: v_lshlrev_b32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x18,0xd5,0x69,0xd2,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_lshlrev_b32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x18,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_lshlrev_b32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x18,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshlrev_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_lshlrev_b32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x18,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, m0, 0.5
+// GFX11: v_lshlrev_b32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x18,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, exec_lo, -1
+// GFX11: v_lshlrev_b32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x18,0xd5,0x7e,0x82,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, exec_hi, null
+// GFX11: v_lshlrev_b32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x18,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, null, exec_lo
+// GFX11: v_lshlrev_b32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x18,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, -1, exec_hi
+// GFX11: v_lshlrev_b32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x18,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, 0.5, m0
+// GFX11: v_lshlrev_b32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x18,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_lshlrev_b32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x18,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_lshlrev_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_lshlrev_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x18,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32_e64 v5, v1, v2
+// GFX11: v_lshrrev_b32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x19,0xd5,0x01,0x05,0x02,0x00]
+
+v_lshrrev_b32_e64 v5, v255, v255
+// GFX11: v_lshrrev_b32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x19,0xd5,0xff,0xff,0x03,0x00]
+
+v_lshrrev_b32_e64 v5, s1, s2
+// GFX11: v_lshrrev_b32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x19,0xd5,0x01,0x04,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, s105, s105
+// GFX11: v_lshrrev_b32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x19,0xd5,0x69,0xd2,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_lshrrev_b32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x19,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_lshrrev_b32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x19,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_lshrrev_b32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x19,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, m0, 0.5
+// GFX11: v_lshrrev_b32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x19,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, exec_lo, -1
+// GFX11: v_lshrrev_b32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x19,0xd5,0x7e,0x82,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, exec_hi, null
+// GFX11: v_lshrrev_b32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x19,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, null, exec_lo
+// GFX11: v_lshrrev_b32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x19,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, -1, exec_hi
+// GFX11: v_lshrrev_b32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x19,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, 0.5, m0
+// GFX11: v_lshrrev_b32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x19,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_lshrrev_b32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x19,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_lshrrev_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_lshrrev_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x19,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_f16_e64 v5, v1, v2
+// GFX11: v_max_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_f16_e64 v5, v255, v255
+// GFX11: v_max_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_f16_e64 v5, s1, s2
+// GFX11: v_max_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x39,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_f16_e64 v5, s105, s105
+// GFX11: v_max_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x39,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_max_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x39,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_max_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x39,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_max_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_max_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x39,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_f16_e64 v5, m0, 0.5
+// GFX11: v_max_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x39,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_f16_e64 v5, exec_lo, -1
+// GFX11: v_max_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x39,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_f16_e64 v5, |exec_hi|, null
+// GFX11: v_max_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x39,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_f16_e64 v5, null, exec_lo
+// GFX11: v_max_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x39,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_f16_e64 v5, -1, exec_hi
+// GFX11: v_max_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x39,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_max_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x39,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_max_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_max_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x39,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_max_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_max_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x39,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_max_f32_e64 v5, v1, v2
+// GFX11: v_max_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x10,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_f32_e64 v5, v255, v255
+// GFX11: v_max_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x10,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_f32_e64 v5, s1, s2
+// GFX11: v_max_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x10,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_f32_e64 v5, s105, s105
+// GFX11: v_max_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x10,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_max_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x10,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_max_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x10,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_max_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x10,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_f32_e64 v5, m0, 0.5
+// GFX11: v_max_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x10,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_f32_e64 v5, exec_lo, -1
+// GFX11: v_max_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x10,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_f32_e64 v5, |exec_hi|, null
+// GFX11: v_max_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x10,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_f32_e64 v5, null, exec_lo
+// GFX11: v_max_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x10,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_f32_e64 v5, -1, exec_hi
+// GFX11: v_max_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x10,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_max_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x10,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_max_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_max_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x10,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_max_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_max_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x10,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_max_i32_e64 v5, v1, v2
+// GFX11: v_max_i32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x12,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_i32_e64 v5, v255, v255
+// GFX11: v_max_i32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x12,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_i32_e64 v5, s1, s2
+// GFX11: v_max_i32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x12,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_i32_e64 v5, s105, s105
+// GFX11: v_max_i32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x12,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_i32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_max_i32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x12,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_max_i32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x12,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_i32_e64 v5, ttmp15, src_scc
+// GFX11: v_max_i32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x12,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_i32_e64 v5, m0, 0.5
+// GFX11: v_max_i32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x12,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_i32_e64 v5, exec_lo, -1
+// GFX11: v_max_i32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x12,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_i32_e64 v5, exec_hi, null
+// GFX11: v_max_i32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x12,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_i32_e64 v5, null, exec_lo
+// GFX11: v_max_i32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x12,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_i32_e64 v5, -1, exec_hi
+// GFX11: v_max_i32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x12,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_i32_e64 v5, 0.5, m0
+// GFX11: v_max_i32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_max_i32_e64 v5, src_scc, vcc_lo
+// GFX11: v_max_i32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x12,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_max_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_max_i32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x12,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_u32_e64 v5, v1, v2
+// GFX11: v_max_u32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x14,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_u32_e64 v5, v255, v255
+// GFX11: v_max_u32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x14,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_u32_e64 v5, s1, s2
+// GFX11: v_max_u32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x14,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_u32_e64 v5, s105, s105
+// GFX11: v_max_u32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x14,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_u32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_max_u32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x14,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_max_u32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_u32_e64 v5, ttmp15, src_scc
+// GFX11: v_max_u32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x14,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_u32_e64 v5, m0, 0.5
+// GFX11: v_max_u32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x14,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_u32_e64 v5, exec_lo, -1
+// GFX11: v_max_u32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x14,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_u32_e64 v5, exec_hi, null
+// GFX11: v_max_u32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x14,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_u32_e64 v5, null, exec_lo
+// GFX11: v_max_u32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x14,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_u32_e64 v5, -1, exec_hi
+// GFX11: v_max_u32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x14,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_u32_e64 v5, 0.5, m0
+// GFX11: v_max_u32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x14,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_max_u32_e64 v5, src_scc, vcc_lo
+// GFX11: v_max_u32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x14,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_max_u32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_max_u32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x14,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_f16_e64 v5, v1, v2
+// GFX11: v_min_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_f16_e64 v5, v255, v255
+// GFX11: v_min_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_f16_e64 v5, s1, s2
+// GFX11: v_min_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x3a,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_f16_e64 v5, s105, s105
+// GFX11: v_min_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x3a,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_min_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x3a,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_min_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x3a,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_min_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_min_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x3a,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_f16_e64 v5, m0, 0.5
+// GFX11: v_min_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x3a,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_f16_e64 v5, exec_lo, -1
+// GFX11: v_min_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x3a,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_f16_e64 v5, |exec_hi|, null
+// GFX11: v_min_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x3a,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_f16_e64 v5, null, exec_lo
+// GFX11: v_min_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x3a,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_f16_e64 v5, -1, exec_hi
+// GFX11: v_min_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x3a,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_min_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x3a,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_min_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_min_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x3a,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_min_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_min_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x3a,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_min_f32_e64 v5, v1, v2
+// GFX11: v_min_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x0f,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_f32_e64 v5, v255, v255
+// GFX11: v_min_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x0f,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_f32_e64 v5, s1, s2
+// GFX11: v_min_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x0f,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_f32_e64 v5, s105, s105
+// GFX11: v_min_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x0f,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_min_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x0f,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_min_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x0f,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_min_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x0f,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_f32_e64 v5, m0, 0.5
+// GFX11: v_min_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x0f,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_f32_e64 v5, exec_lo, -1
+// GFX11: v_min_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x0f,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_f32_e64 v5, |exec_hi|, null
+// GFX11: v_min_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x0f,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_f32_e64 v5, null, exec_lo
+// GFX11: v_min_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x0f,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_f32_e64 v5, -1, exec_hi
+// GFX11: v_min_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x0f,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_min_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x0f,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_min_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_min_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x0f,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_min_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_min_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x0f,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_min_i32_e64 v5, v1, v2
+// GFX11: v_min_i32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x11,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_i32_e64 v5, v255, v255
+// GFX11: v_min_i32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x11,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_i32_e64 v5, s1, s2
+// GFX11: v_min_i32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x11,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_i32_e64 v5, s105, s105
+// GFX11: v_min_i32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x11,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_i32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_min_i32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x11,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_min_i32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x11,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_i32_e64 v5, ttmp15, src_scc
+// GFX11: v_min_i32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x11,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_i32_e64 v5, m0, 0.5
+// GFX11: v_min_i32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x11,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_i32_e64 v5, exec_lo, -1
+// GFX11: v_min_i32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x11,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_i32_e64 v5, exec_hi, null
+// GFX11: v_min_i32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x11,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_i32_e64 v5, null, exec_lo
+// GFX11: v_min_i32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x11,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_i32_e64 v5, -1, exec_hi
+// GFX11: v_min_i32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x11,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_i32_e64 v5, 0.5, m0
+// GFX11: v_min_i32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_min_i32_e64 v5, src_scc, vcc_lo
+// GFX11: v_min_i32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x11,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_min_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_min_i32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x11,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_u32_e64 v5, v1, v2
+// GFX11: v_min_u32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x13,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_u32_e64 v5, v255, v255
+// GFX11: v_min_u32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x13,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_u32_e64 v5, s1, s2
+// GFX11: v_min_u32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x13,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_u32_e64 v5, s105, s105
+// GFX11: v_min_u32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x13,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_u32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_min_u32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x13,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_min_u32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x13,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_u32_e64 v5, ttmp15, src_scc
+// GFX11: v_min_u32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x13,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_u32_e64 v5, m0, 0.5
+// GFX11: v_min_u32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x13,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_u32_e64 v5, exec_lo, -1
+// GFX11: v_min_u32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x13,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_u32_e64 v5, exec_hi, null
+// GFX11: v_min_u32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x13,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_u32_e64 v5, null, exec_lo
+// GFX11: v_min_u32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x13,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_u32_e64 v5, -1, exec_hi
+// GFX11: v_min_u32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x13,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_u32_e64 v5, 0.5, m0
+// GFX11: v_min_u32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_min_u32_e64 v5, src_scc, vcc_lo
+// GFX11: v_min_u32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x13,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_min_u32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_min_u32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x13,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32_e64 v5, v1, v2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, v1, v2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, v255, v255
+// GFX11: v_mul_dx9_zero_f32_e64 v5, v255, v255   ; encoding: [0x05,0x00,0x07,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, s1, s2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, s1, s2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, s105, s105
+// GFX11: v_mul_dx9_zero_f32_e64 v5, s105, s105   ; encoding: [0x05,0x00,0x07,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x07,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x07,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, m0, 0.5
+// GFX11: v_mul_dx9_zero_f32_e64 v5, m0, 0.5      ; encoding: [0x05,0x00,0x07,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, exec_lo, -1
+// GFX11: v_mul_dx9_zero_f32_e64 v5, exec_lo, -1  ; encoding: [0x05,0x00,0x07,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null
+// GFX11: v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x07,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, null, exec_lo
+// GFX11: v_mul_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x07,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, -1, exec_hi
+// GFX11: v_mul_dx9_zero_f32_e64 v5, -1, exec_hi  ; encoding: [0x05,0x00,0x07,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x07,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x07,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x07,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_f16_e64 v5, v1, v2
+// GFX11: v_mul_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x35,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_f16_e64 v5, v255, v255
+// GFX11: v_mul_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x35,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_f16_e64 v5, s1, s2
+// GFX11: v_mul_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x35,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_f16_e64 v5, s105, s105
+// GFX11: v_mul_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x35,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x35,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_mul_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x35,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_mul_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x35,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_f16_e64 v5, m0, 0.5
+// GFX11: v_mul_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x35,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_f16_e64 v5, exec_lo, -1
+// GFX11: v_mul_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x35,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_f16_e64 v5, |exec_hi|, null
+// GFX11: v_mul_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x35,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_f16_e64 v5, null, exec_lo
+// GFX11: v_mul_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x35,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_f16_e64 v5, -1, exec_hi
+// GFX11: v_mul_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x35,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_mul_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x35,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_mul_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x35,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_mul_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x35,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_mul_f32_e64 v5, v1, v2
+// GFX11: v_mul_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x08,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_f32_e64 v5, v255, v255
+// GFX11: v_mul_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x08,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_f32_e64 v5, s1, s2
+// GFX11: v_mul_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x08,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_f32_e64 v5, s105, s105
+// GFX11: v_mul_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x08,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x08,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x08,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x08,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_f32_e64 v5, m0, 0.5
+// GFX11: v_mul_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x08,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_f32_e64 v5, exec_lo, -1
+// GFX11: v_mul_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x08,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_f32_e64 v5, |exec_hi|, null
+// GFX11: v_mul_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x08,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_f32_e64 v5, null, exec_lo
+// GFX11: v_mul_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x08,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_f32_e64 v5, -1, exec_hi
+// GFX11: v_mul_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x08,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_mul_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x08,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_mul_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x08,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_mul_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x08,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24_e64 v5, v1, v2
+// GFX11: v_mul_hi_i32_i24_e64 v5, v1, v2         ; encoding: [0x05,0x00,0x0a,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_hi_i32_i24_e64 v5, v255, v255
+// GFX11: v_mul_hi_i32_i24_e64 v5, v255, v255     ; encoding: [0x05,0x00,0x0a,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_hi_i32_i24_e64 v5, s1, s2
+// GFX11: v_mul_hi_i32_i24_e64 v5, s1, s2         ; encoding: [0x05,0x00,0x0a,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, s105, s105
+// GFX11: v_mul_hi_i32_i24_e64 v5, s105, s105     ; encoding: [0x05,0x00,0x0a,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_hi_i32_i24_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_hi_i32_i24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_hi_i32_i24_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x0a,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, m0, 0.5
+// GFX11: v_mul_hi_i32_i24_e64 v5, m0, 0.5        ; encoding: [0x05,0x00,0x0a,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, exec_lo, -1
+// GFX11: v_mul_hi_i32_i24_e64 v5, exec_lo, -1    ; encoding: [0x05,0x00,0x0a,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, exec_hi, null
+// GFX11: v_mul_hi_i32_i24_e64 v5, exec_hi, null  ; encoding: [0x05,0x00,0x0a,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, null, exec_lo
+// GFX11: v_mul_hi_i32_i24_e64 v5, null, exec_lo  ; encoding: [0x05,0x00,0x0a,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, -1, exec_hi
+// GFX11: v_mul_hi_i32_i24_e64 v5, -1, exec_hi    ; encoding: [0x05,0x00,0x0a,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, 0.5, m0
+// GFX11: v_mul_hi_i32_i24_e64 v5, 0.5, m0        ; encoding: [0x05,0x00,0x0a,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, src_scc, vcc_lo
+// GFX11: v_mul_hi_i32_i24_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_mul_hi_i32_i24_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x0a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24_e64 v5, v1, v2
+// GFX11: v_mul_hi_u32_u24_e64 v5, v1, v2         ; encoding: [0x05,0x00,0x0c,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_hi_u32_u24_e64 v5, v255, v255
+// GFX11: v_mul_hi_u32_u24_e64 v5, v255, v255     ; encoding: [0x05,0x00,0x0c,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_hi_u32_u24_e64 v5, s1, s2
+// GFX11: v_mul_hi_u32_u24_e64 v5, s1, s2         ; encoding: [0x05,0x00,0x0c,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, s105, s105
+// GFX11: v_mul_hi_u32_u24_e64 v5, s105, s105     ; encoding: [0x05,0x00,0x0c,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_hi_u32_u24_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x0c,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_hi_u32_u24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0c,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_hi_u32_u24_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x0c,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, m0, 0.5
+// GFX11: v_mul_hi_u32_u24_e64 v5, m0, 0.5        ; encoding: [0x05,0x00,0x0c,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, exec_lo, -1
+// GFX11: v_mul_hi_u32_u24_e64 v5, exec_lo, -1    ; encoding: [0x05,0x00,0x0c,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, exec_hi, null
+// GFX11: v_mul_hi_u32_u24_e64 v5, exec_hi, null  ; encoding: [0x05,0x00,0x0c,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, null, exec_lo
+// GFX11: v_mul_hi_u32_u24_e64 v5, null, exec_lo  ; encoding: [0x05,0x00,0x0c,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, -1, exec_hi
+// GFX11: v_mul_hi_u32_u24_e64 v5, -1, exec_hi    ; encoding: [0x05,0x00,0x0c,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, 0.5, m0
+// GFX11: v_mul_hi_u32_u24_e64 v5, 0.5, m0        ; encoding: [0x05,0x00,0x0c,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, src_scc, vcc_lo
+// GFX11: v_mul_hi_u32_u24_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x0c,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_mul_hi_u32_u24_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x0c,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24_e64 v5, v1, v2
+// GFX11: v_mul_i32_i24_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x09,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_i32_i24_e64 v5, v255, v255
+// GFX11: v_mul_i32_i24_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x09,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_i32_i24_e64 v5, s1, s2
+// GFX11: v_mul_i32_i24_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x09,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, s105, s105
+// GFX11: v_mul_i32_i24_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x09,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_i32_i24_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x09,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_i32_i24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x09,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_i32_i24_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x09,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, m0, 0.5
+// GFX11: v_mul_i32_i24_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x09,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, exec_lo, -1
+// GFX11: v_mul_i32_i24_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x09,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, exec_hi, null
+// GFX11: v_mul_i32_i24_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x09,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, null, exec_lo
+// GFX11: v_mul_i32_i24_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x09,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, -1, exec_hi
+// GFX11: v_mul_i32_i24_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x09,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, 0.5, m0
+// GFX11: v_mul_i32_i24_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x09,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, src_scc, vcc_lo
+// GFX11: v_mul_i32_i24_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x09,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_i32_i24_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX11: v_mul_i32_i24_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x09,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32_e64 v5, v1, v2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, v1, v2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_legacy_f32_e64 v5, v255, v255
+// GFX11: v_mul_dx9_zero_f32_e64 v5, v255, v255   ; encoding: [0x05,0x00,0x07,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_legacy_f32_e64 v5, s1, s2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, s1, s2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, s105, s105
+// GFX11: v_mul_dx9_zero_f32_e64 v5, s105, s105   ; encoding: [0x05,0x00,0x07,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x07,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x07,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, m0, 0.5
+// GFX11: v_mul_dx9_zero_f32_e64 v5, m0, 0.5      ; encoding: [0x05,0x00,0x07,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, exec_lo, -1
+// GFX11: v_mul_dx9_zero_f32_e64 v5, exec_lo, -1  ; encoding: [0x05,0x00,0x07,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, |exec_hi|, null
+// GFX11: v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x07,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, null, exec_lo
+// GFX11: v_mul_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x07,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, -1, exec_hi
+// GFX11: v_mul_dx9_zero_f32_e64 v5, -1, exec_hi  ; encoding: [0x05,0x00,0x07,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x07,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_legacy_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x07,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_legacy_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x07,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24_e64 v5, v1, v2
+// GFX11: v_mul_u32_u24_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x0b,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_u32_u24_e64 v5, v255, v255
+// GFX11: v_mul_u32_u24_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x0b,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_u32_u24_e64 v5, s1, s2
+// GFX11: v_mul_u32_u24_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x0b,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, s105, s105
+// GFX11: v_mul_u32_u24_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x0b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, vcc_lo, ttmp15
+// GFX11: v_mul_u32_u24_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x0b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_mul_u32_u24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24_e64 v5, ttmp15, src_scc
+// GFX11: v_mul_u32_u24_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x0b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, m0, 0.5
+// GFX11: v_mul_u32_u24_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x0b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, exec_lo, -1
+// GFX11: v_mul_u32_u24_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x0b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, exec_hi, null
+// GFX11: v_mul_u32_u24_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x0b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, null, exec_lo
+// GFX11: v_mul_u32_u24_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x0b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, -1, exec_hi
+// GFX11: v_mul_u32_u24_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x0b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, 0.5, m0
+// GFX11: v_mul_u32_u24_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, src_scc, vcc_lo
+// GFX11: v_mul_u32_u24_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x0b,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_u32_u24_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX11: v_mul_u32_u24_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x0b,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_or_b32_e64 v5, v1, v2
+// GFX11: v_or_b32_e64 v5, v1, v2                 ; encoding: [0x05,0x00,0x1c,0xd5,0x01,0x05,0x02,0x00]
+
+v_or_b32_e64 v5, v255, v255
+// GFX11: v_or_b32_e64 v5, v255, v255             ; encoding: [0x05,0x00,0x1c,0xd5,0xff,0xff,0x03,0x00]
+
+v_or_b32_e64 v5, s1, s2
+// GFX11: v_or_b32_e64 v5, s1, s2                 ; encoding: [0x05,0x00,0x1c,0xd5,0x01,0x04,0x00,0x00]
+
+v_or_b32_e64 v5, s105, s105
+// GFX11: v_or_b32_e64 v5, s105, s105             ; encoding: [0x05,0x00,0x1c,0xd5,0x69,0xd2,0x00,0x00]
+
+v_or_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_or_b32_e64 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x1c,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_or_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_or_b32_e64 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x1c,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_or_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_or_b32_e64 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x1c,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_or_b32_e64 v5, m0, 0.5
+// GFX11: v_or_b32_e64 v5, m0, 0.5                ; encoding: [0x05,0x00,0x1c,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_or_b32_e64 v5, exec_lo, -1
+// GFX11: v_or_b32_e64 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x1c,0xd5,0x7e,0x82,0x01,0x00]
+
+v_or_b32_e64 v5, exec_hi, null
+// GFX11: v_or_b32_e64 v5, exec_hi, null          ; encoding: [0x05,0x00,0x1c,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_or_b32_e64 v5, null, exec_lo
+// GFX11: v_or_b32_e64 v5, null, exec_lo          ; encoding: [0x05,0x00,0x1c,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_or_b32_e64 v5, -1, exec_hi
+// GFX11: v_or_b32_e64 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x1c,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_or_b32_e64 v5, 0.5, m0
+// GFX11: v_or_b32_e64 v5, 0.5, m0                ; encoding: [0x05,0x00,0x1c,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_or_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_or_b32_e64 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x1c,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_or_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_or_b32_e64 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x1c,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x21,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_sub_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x21,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_sub_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x21,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_sub_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x21,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_sub_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x21,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_sub_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x21,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_sub_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x21,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x21,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX11: v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x21,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_sub_f16_e64 v5, v1, v2
+// GFX11: v_sub_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16_e64 v5, v255, v255
+// GFX11: v_sub_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f16_e64 v5, s1, s2
+// GFX11: v_sub_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f16_e64 v5, s105, s105
+// GFX11: v_sub_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_sub_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_sub_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_sub_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_sub_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_f16_e64 v5, m0, 0.5
+// GFX11: v_sub_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_f16_e64 v5, exec_lo, -1
+// GFX11: v_sub_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_f16_e64 v5, |exec_hi|, null
+// GFX11: v_sub_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_f16_e64 v5, null, exec_lo
+// GFX11: v_sub_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_f16_e64 v5, -1, exec_hi
+// GFX11: v_sub_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_sub_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_sub_f32_e64 v5, v1, v2
+// GFX11: v_sub_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f32_e64 v5, v255, v255
+// GFX11: v_sub_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x04,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f32_e64 v5, s1, s2
+// GFX11: v_sub_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f32_e64 v5, s105, s105
+// GFX11: v_sub_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x04,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_sub_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x04,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_sub_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x04,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_sub_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x04,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_f32_e64 v5, m0, 0.5
+// GFX11: v_sub_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x04,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_f32_e64 v5, exec_lo, -1
+// GFX11: v_sub_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x04,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_f32_e64 v5, |exec_hi|, null
+// GFX11: v_sub_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x04,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_f32_e64 v5, null, exec_lo
+// GFX11: v_sub_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x04,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_f32_e64 v5, -1, exec_hi
+// GFX11: v_sub_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x04,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_sub_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x04,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_sub_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_sub_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x04,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_sub_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_sub_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x04,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32_e64 v5, v1, v2
+// GFX11: v_sub_nc_u32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x26,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_nc_u32_e64 v5, v255, v255
+// GFX11: v_sub_nc_u32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x26,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_nc_u32_e64 v5, s1, s2
+// GFX11: v_sub_nc_u32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x26,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, s105, s105
+// GFX11: v_sub_nc_u32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x26,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_sub_nc_u32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x26,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_sub_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32_e64 v5, ttmp15, src_scc
+// GFX11: v_sub_nc_u32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x26,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, m0, 0.5
+// GFX11: v_sub_nc_u32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x26,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, exec_lo, -1
+// GFX11: v_sub_nc_u32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x26,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, exec_hi, null
+// GFX11: v_sub_nc_u32_e64 v5, exec_hi, null      ; encoding: [0x05,0x00,0x26,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, null, exec_lo
+// GFX11: v_sub_nc_u32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x26,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, -1, exec_hi
+// GFX11: v_sub_nc_u32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x26,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, 0.5, m0
+// GFX11: v_sub_nc_u32_e64 v5, 0.5, m0            ; encoding: [0x05,0x00,0x26,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX11: v_sub_nc_u32_e64 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x26,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_sub_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX11: v_sub_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x22,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_subrev_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x22,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_subrev_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x22,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_subrev_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x22,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_subrev_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x22,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_subrev_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x22,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_subrev_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x22,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_subrev_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x22,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX11: v_subrev_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x22,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_subrev_f16_e64 v5, v1, v2
+// GFX11: v_subrev_f16_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_f16_e64 v5, v255, v255
+// GFX11: v_subrev_f16_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x34,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_f16_e64 v5, s1, s2
+// GFX11: v_subrev_f16_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x34,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_f16_e64 v5, s105, s105
+// GFX11: v_subrev_f16_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x34,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_f16_e64 v5, vcc_lo, ttmp15
+// GFX11: v_subrev_f16_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x34,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX11: v_subrev_f16_e64 v5, vcc_hi, 0xfe0b     ; encoding: [0x05,0x00,0x34,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f16_e64 v5, ttmp15, src_scc
+// GFX11: v_subrev_f16_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x34,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_f16_e64 v5, m0, 0.5
+// GFX11: v_subrev_f16_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x34,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_f16_e64 v5, exec_lo, -1
+// GFX11: v_subrev_f16_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x34,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_f16_e64 v5, |exec_hi|, null
+// GFX11: v_subrev_f16_e64 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x34,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_f16_e64 v5, null, exec_lo
+// GFX11: v_subrev_f16_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x34,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_f16_e64 v5, -1, exec_hi
+// GFX11: v_subrev_f16_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x34,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_f16_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_subrev_f16_e64 v5, 0.5, -m0 mul:2     ; encoding: [0x05,0x00,0x34,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_subrev_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_subrev_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x34,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_subrev_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_subrev_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x34,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f32_e64 v5, v1, v2
+// GFX11: v_subrev_f32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x05,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_f32_e64 v5, v255, v255
+// GFX11: v_subrev_f32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x05,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_f32_e64 v5, s1, s2
+// GFX11: v_subrev_f32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x05,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_f32_e64 v5, s105, s105
+// GFX11: v_subrev_f32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x05,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_f32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_subrev_f32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x05,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_subrev_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x05,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_f32_e64 v5, ttmp15, src_scc
+// GFX11: v_subrev_f32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x05,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_f32_e64 v5, m0, 0.5
+// GFX11: v_subrev_f32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x05,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_f32_e64 v5, exec_lo, -1
+// GFX11: v_subrev_f32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x05,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_f32_e64 v5, |exec_hi|, null
+// GFX11: v_subrev_f32_e64 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x05,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_f32_e64 v5, null, exec_lo
+// GFX11: v_subrev_f32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x05,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_f32_e64 v5, -1, exec_hi
+// GFX11: v_subrev_f32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x05,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_f32_e64 v5, 0.5, -m0 mul:2
+// GFX11: v_subrev_f32_e64 v5, 0.5, -m0 mul:2     ; encoding: [0x05,0x00,0x05,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_subrev_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX11: v_subrev_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x05,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_subrev_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX11: v_subrev_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x05,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32_e64 v5, v1, v2
+// GFX11: v_subrev_nc_u32_e64 v5, v1, v2          ; encoding: [0x05,0x00,0x27,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_nc_u32_e64 v5, v255, v255
+// GFX11: v_subrev_nc_u32_e64 v5, v255, v255      ; encoding: [0x05,0x00,0x27,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_nc_u32_e64 v5, s1, s2
+// GFX11: v_subrev_nc_u32_e64 v5, s1, s2          ; encoding: [0x05,0x00,0x27,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, s105, s105
+// GFX11: v_subrev_nc_u32_e64 v5, s105, s105      ; encoding: [0x05,0x00,0x27,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_subrev_nc_u32_e64 v5, vcc_lo, ttmp15  ; encoding: [0x05,0x00,0x27,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_subrev_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x27,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32_e64 v5, ttmp15, src_scc
+// GFX11: v_subrev_nc_u32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x27,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, m0, 0.5
+// GFX11: v_subrev_nc_u32_e64 v5, m0, 0.5         ; encoding: [0x05,0x00,0x27,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, exec_lo, -1
+// GFX11: v_subrev_nc_u32_e64 v5, exec_lo, -1     ; encoding: [0x05,0x00,0x27,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, exec_hi, null
+// GFX11: v_subrev_nc_u32_e64 v5, exec_hi, null   ; encoding: [0x05,0x00,0x27,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, null, exec_lo
+// GFX11: v_subrev_nc_u32_e64 v5, null, exec_lo   ; encoding: [0x05,0x00,0x27,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, -1, exec_hi
+// GFX11: v_subrev_nc_u32_e64 v5, -1, exec_hi     ; encoding: [0x05,0x00,0x27,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, 0.5, m0
+// GFX11: v_subrev_nc_u32_e64 v5, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX11: v_subrev_nc_u32_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x27,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_subrev_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX11: v_subrev_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x27,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32_e64 v5, v1, v2
+// GFX11: v_xnor_b32_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd5,0x01,0x05,0x02,0x00]
+
+v_xnor_b32_e64 v5, v255, v255
+// GFX11: v_xnor_b32_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd5,0xff,0xff,0x03,0x00]
+
+v_xnor_b32_e64 v5, s1, s2
+// GFX11: v_xnor_b32_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd5,0x01,0x04,0x00,0x00]
+
+v_xnor_b32_e64 v5, s105, s105
+// GFX11: v_xnor_b32_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd5,0x69,0xd2,0x00,0x00]
+
+v_xnor_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_xnor_b32_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_xnor_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_xnor_b32_e64 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_xnor_b32_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_xnor_b32_e64 v5, m0, 0.5
+// GFX11: v_xnor_b32_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_xnor_b32_e64 v5, exec_lo, -1
+// GFX11: v_xnor_b32_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd5,0x7e,0x82,0x01,0x00]
+
+v_xnor_b32_e64 v5, exec_hi, null
+// GFX11: v_xnor_b32_e64 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_xnor_b32_e64 v5, null, exec_lo
+// GFX11: v_xnor_b32_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_xnor_b32_e64 v5, -1, exec_hi
+// GFX11: v_xnor_b32_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_xnor_b32_e64 v5, 0.5, m0
+// GFX11: v_xnor_b32_e64 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_xnor_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_xnor_b32_e64 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_xnor_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_xnor_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_xor_b32_e64 v5, v1, v2
+// GFX11: v_xor_b32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x1d,0xd5,0x01,0x05,0x02,0x00]
+
+v_xor_b32_e64 v5, v255, v255
+// GFX11: v_xor_b32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x1d,0xd5,0xff,0xff,0x03,0x00]
+
+v_xor_b32_e64 v5, s1, s2
+// GFX11: v_xor_b32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x1d,0xd5,0x01,0x04,0x00,0x00]
+
+v_xor_b32_e64 v5, s105, s105
+// GFX11: v_xor_b32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x1d,0xd5,0x69,0xd2,0x00,0x00]
+
+v_xor_b32_e64 v5, vcc_lo, ttmp15
+// GFX11: v_xor_b32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x1d,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_xor_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX11: v_xor_b32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x1d,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_xor_b32_e64 v5, ttmp15, src_scc
+// GFX11: v_xor_b32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x1d,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_xor_b32_e64 v5, m0, 0.5
+// GFX11: v_xor_b32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x1d,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_xor_b32_e64 v5, exec_lo, -1
+// GFX11: v_xor_b32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x1d,0xd5,0x7e,0x82,0x01,0x00]
+
+v_xor_b32_e64 v5, exec_hi, null
+// GFX11: v_xor_b32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x1d,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_xor_b32_e64 v5, null, exec_lo
+// GFX11: v_xor_b32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x1d,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_xor_b32_e64 v5, -1, exec_hi
+// GFX11: v_xor_b32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x1d,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_xor_b32_e64 v5, 0.5, m0
+// GFX11: v_xor_b32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x1d,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_xor_b32_e64 v5, src_scc, vcc_lo
+// GFX11: v_xor_b32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x1d,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_xor_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX11: v_xor_b32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x1d,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
index c56a3514a833a..ee6aa11252609 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
 // W32: v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
@@ -111,50 +111,59 @@ v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
 v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
 // GFX11: v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x20,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-v_add_f16_e64 v5, v1, v2
-// GFX11: v_add_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+v_add_f16_e64 v5.l, v1.l, v2.l
+// GFX11: v_add_f16_e64 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16_e64 v5, v255, v255
-// GFX11: v_add_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
+v_add_f16_e64 v5.l, v255.l, v255.l
+// GFX11: v_add_f16_e64 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
 
-v_add_f16_e64 v5, s1, s2
-// GFX11: v_add_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
+v_add_f16_e64 v5.l, s1, s2
+// GFX11: v_add_f16_e64 v5.l, s1, s2              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
 
-v_add_f16_e64 v5, s105, s105
-// GFX11: v_add_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
+v_add_f16_e64 v5.l, s105, s105
+// GFX11: v_add_f16_e64 v5.l, s105, s105          ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
 
-v_add_f16_e64 v5, vcc_lo, ttmp15
-// GFX11: v_add_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
+v_add_f16_e64 v5.l, vcc_lo, ttmp15
+// GFX11: v_add_f16_e64 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
 
-v_add_f16_e64 v5, vcc_hi, 0xfe0b
-// GFX11: v_add_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+v_add_f16_e64 v5.l, vcc_hi, 0xfe0b
+// GFX11: v_add_f16_e64 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-v_add_f16_e64 v5, ttmp15, src_scc
-// GFX11: v_add_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
+v_add_f16_e64 v5.l, ttmp15, src_scc
+// GFX11: v_add_f16_e64 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
 
-v_add_f16_e64 v5, m0, 0.5
-// GFX11: v_add_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
+v_add_f16_e64 v5.l, m0, 0.5
+// GFX11: v_add_f16_e64 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
 
-v_add_f16_e64 v5, exec_lo, -1
-// GFX11: v_add_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
+v_add_f16_e64 v5.l, exec_lo, -1
+// GFX11: v_add_f16_e64 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
 
-v_add_f16_e64 v5, |exec_hi|, null
-// GFX11: v_add_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
+v_add_f16_e64 v5.l, |exec_hi|, null
+// GFX11: v_add_f16_e64 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
 
-v_add_f16_e64 v5, null, exec_lo
-// GFX11: v_add_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
+v_add_f16_e64 v5.l, null, exec_lo
+// GFX11: v_add_f16_e64 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
 
-v_add_f16_e64 v5, -1, exec_hi
-// GFX11: v_add_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
+v_add_f16_e64 v5.l, -1, exec_hi
+// GFX11: v_add_f16_e64 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
 
-v_add_f16_e64 v5, 0.5, -m0 mul:2
-// GFX11: v_add_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
+v_add_f16_e64 v5.l, 0.5, -m0 mul:2
+// GFX11: v_add_f16_e64 v5.l, 0.5, -m0 mul:2      ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
 
-v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4
-// GFX11: v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
+v_add_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4
+// GFX11: v_add_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
 
-v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
-// GFX11: v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+v_add_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_add_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5.l, v1.h, v2.l
+// GFX11: v_add_f16_e64 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x32,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f16_e64 v5.l, v255.l, v255.h
+// GFX11: v_add_f16_e64 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x32,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_add_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| op_sel:[0,0,1] clamp div:2 ; encoding: [0xff,0xc3,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
 v_add_f32_e64 v5, v1, v2
 // GFX11: v_add_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x00]
@@ -1720,50 +1729,59 @@ v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
 v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
 // GFX11: v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x21,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-v_sub_f16_e64 v5, v1, v2
-// GFX11: v_sub_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+v_sub_f16_e64 v5.l, v1.l, v2.l
+// GFX11: v_sub_f16_e64 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16_e64 v5.l, v255.l, v255.l
+// GFX11: v_sub_f16_e64 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f16_e64 v5.l, s1, s2
+// GFX11: v_sub_f16_e64 v5.l, s1, s2              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f16_e64 v5.l, s105, s105
+// GFX11: v_sub_f16_e64 v5.l, s105, s105          ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
 
-v_sub_f16_e64 v5, v255, v255
-// GFX11: v_sub_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+v_sub_f16_e64 v5.l, vcc_lo, ttmp15
+// GFX11: v_sub_f16_e64 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
 
-v_sub_f16_e64 v5, s1, s2
-// GFX11: v_sub_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+v_sub_f16_e64 v5.l, vcc_hi, 0xfe0b
+// GFX11: v_sub_f16_e64 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, s105, s105
-// GFX11: v_sub_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
+v_sub_f16_e64 v5.l, ttmp15, src_scc
+// GFX11: v_sub_f16_e64 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
 
-v_sub_f16_e64 v5, vcc_lo, ttmp15
-// GFX11: v_sub_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
+v_sub_f16_e64 v5.l, m0, 0.5
+// GFX11: v_sub_f16_e64 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
 
-v_sub_f16_e64 v5, vcc_hi, 0xfe0b
-// GFX11: v_sub_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+v_sub_f16_e64 v5.l, exec_lo, -1
+// GFX11: v_sub_f16_e64 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
 
-v_sub_f16_e64 v5, ttmp15, src_scc
-// GFX11: v_sub_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
+v_sub_f16_e64 v5.l, |exec_hi|, null
+// GFX11: v_sub_f16_e64 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
 
-v_sub_f16_e64 v5, m0, 0.5
-// GFX11: v_sub_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
+v_sub_f16_e64 v5.l, null, exec_lo
+// GFX11: v_sub_f16_e64 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
 
-v_sub_f16_e64 v5, exec_lo, -1
-// GFX11: v_sub_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
+v_sub_f16_e64 v5.l, -1, exec_hi
+// GFX11: v_sub_f16_e64 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, |exec_hi|, null
-// GFX11: v_sub_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
+v_sub_f16_e64 v5.l, 0.5, -m0 mul:2
+// GFX11: v_sub_f16_e64 v5.l, 0.5, -m0 mul:2      ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
 
-v_sub_f16_e64 v5, null, exec_lo
-// GFX11: v_sub_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
+v_sub_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4
+// GFX11: v_sub_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
 
-v_sub_f16_e64 v5, -1, exec_hi
-// GFX11: v_sub_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
+v_sub_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_sub_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, 0.5, -m0 mul:2
-// GFX11: v_sub_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
+v_sub_f16_e64 v5.l, v1.h, v2.l
+// GFX11: v_sub_f16_e64 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x33,0xd5,0x01,0x05,0x02,0x00]
 
-v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4
-// GFX11: v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
+v_sub_f16_e64 v5.l, v255.l, v255.h
+// GFX11: v_sub_f16_e64 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x33,0xd5,0xff,0xff,0x03,0x00]
 
-v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
-// GFX11: v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+v_sub_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX11: v_sub_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| op_sel:[0,0,1] clamp div:2 ; encoding: [0xff,0xc3,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
 v_sub_f32_e64 v5, v1, v2
 // GFX11: v_sub_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx-fake16.s
new file mode 100644
index 0000000000000..26a9bd3ebb1e1
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx-fake16.s
@@ -0,0 +1,4119 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+
+v_cmpx_class_f16_e64 v1, v2
+// GFX11: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_class_f16_e64 v255, v2
+// GFX11: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_class_f16_e64 s1, v2
+// GFX11: v_cmpx_class_f16_e64 s1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00]
+
+v_cmpx_class_f16_e64 s105, v255
+// GFX11: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+
+v_cmpx_class_f16_e64 vcc_lo, s2
+// GFX11: v_cmpx_class_f16_e64 vcc_lo, s2         ; encoding: [0x7e,0x00,0xfd,0xd4,0x6a,0x04,0x00,0x00]
+
+v_cmpx_class_f16_e64 vcc_hi, s105
+// GFX11: v_cmpx_class_f16_e64 vcc_hi, s105       ; encoding: [0x7e,0x00,0xfd,0xd4,0x6b,0xd2,0x00,0x00]
+
+v_cmpx_class_f16_e64 ttmp15, ttmp15
+// GFX11: v_cmpx_class_f16_e64 ttmp15, ttmp15     ; encoding: [0x7e,0x00,0xfd,0xd4,0x7b,0xf6,0x00,0x00]
+
+v_cmpx_class_f16_e64 m0, src_scc
+// GFX11: v_cmpx_class_f16_e64 m0, src_scc        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7d,0xfa,0x01,0x00]
+
+v_cmpx_class_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_class_f16_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_class_f16_e64 exec_hi, null
+// GFX11: v_cmpx_class_f16_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_class_f16_e64 null, exec_lo
+// GFX11: v_cmpx_class_f16_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_class_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_class_f16_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_class_f16_e64 0.5, m0
+// GFX11: v_cmpx_class_f16_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_class_f16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_class_f16_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi
+// GFX11: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_class_f16_e64 v1, 0.5
+// GFX11: v_cmpx_class_f16_e64 v1, 0.5            ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xe1,0x01,0x00]
+
+v_cmpx_class_f32_e64 v1, v2
+// GFX11: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_class_f32_e64 v255, v255
+// GFX11: v_cmpx_class_f32_e64 v255, v255         ; encoding: [0x7e,0x00,0xfe,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_class_f32_e64 s1, s2
+// GFX11: v_cmpx_class_f32_e64 s1, s2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_class_f32_e64 s105, s105
+// GFX11: v_cmpx_class_f32_e64 s105, s105         ; encoding: [0x7e,0x00,0xfe,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_class_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_class_f32_e64 vcc_lo, ttmp15     ; encoding: [0x7e,0x00,0xfe,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_class_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_class_f32_e64 vcc_hi, 0xaf123456 ; encoding: [0x7e,0x00,0xfe,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_class_f32_e64 ttmp15, src_scc    ; encoding: [0x7e,0x00,0xfe,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_class_f32_e64 m0, 0.5
+// GFX11: v_cmpx_class_f32_e64 m0, 0.5            ; encoding: [0x7e,0x00,0xfe,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_class_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_class_f32_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfe,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_class_f32_e64 exec_hi, null
+// GFX11: v_cmpx_class_f32_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_class_f32_e64 null, exec_lo
+// GFX11: v_cmpx_class_f32_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_class_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_class_f32_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfe,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_class_f32_e64 0.5, m0
+// GFX11: v_cmpx_class_f32_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfe,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_class_f32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_class_f32_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfe,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_class_f32_e64 -|0xaf123456|, vcc_hi
+// GFX11: v_cmpx_class_f32_e64 -|0xaf123456|, vcc_hi ; encoding: [0x7e,0x01,0xfe,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f64_e64 v[1:2], v2
+// GFX11: v_cmpx_class_f64_e64 v[1:2], v2         ; encoding: [0x7e,0x00,0xff,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_class_f64_e64 v[1:2], v255
+// GFX11: v_cmpx_class_f64_e64 v[1:2], v255       ; encoding: [0x7e,0x00,0xff,0xd4,0x01,0xff,0x03,0x00]
+
+v_cmpx_class_f64_e64 v[1:2], s2
+// GFX11: v_cmpx_class_f64_e64 v[1:2], s2         ; encoding: [0x7e,0x00,0xff,0xd4,0x01,0x05,0x00,0x00]
+
+v_cmpx_class_f64_e64 v[1:2], s105
+// GFX11: v_cmpx_class_f64_e64 v[1:2], s105       ; encoding: [0x7e,0x00,0xff,0xd4,0x01,0xd3,0x00,0x00]
+
+v_cmpx_class_f64_e64 v[254:255], ttmp15
+// GFX11: v_cmpx_class_f64_e64 v[254:255], ttmp15 ; encoding: [0x7e,0x00,0xff,0xd4,0xfe,0xf7,0x00,0x00]
+
+v_cmpx_class_f64_e64 s[2:3], vcc_hi
+// GFX11: v_cmpx_class_f64_e64 s[2:3], vcc_hi     ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xd6,0x00,0x00]
+
+v_cmpx_class_f64_e64 s[104:105], vcc_lo
+// GFX11: v_cmpx_class_f64_e64 s[104:105], vcc_lo ; encoding: [0x7e,0x00,0xff,0xd4,0x68,0xd4,0x00,0x00]
+
+v_cmpx_class_f64_e64 vcc, m0
+// GFX11: v_cmpx_class_f64_e64 vcc, m0            ; encoding: [0x7e,0x00,0xff,0xd4,0x6a,0xfa,0x00,0x00]
+
+v_cmpx_class_f64_e64 ttmp[14:15], exec_hi
+// GFX11: v_cmpx_class_f64_e64 ttmp[14:15], exec_hi ; encoding: [0x7e,0x00,0xff,0xd4,0x7a,0xfe,0x00,0x00]
+
+v_cmpx_class_f64_e64 exec, exec_lo
+// GFX11: v_cmpx_class_f64_e64 exec, exec_lo      ; encoding: [0x7e,0x00,0xff,0xd4,0x7e,0xfc,0x00,0x00]
+
+v_cmpx_class_f64_e64 null, null
+// GFX11: v_cmpx_class_f64_e64 null, null         ; encoding: [0x7e,0x00,0xff,0xd4,0x7c,0xf8,0x00,0x00]
+
+v_cmpx_class_f64_e64 -1, -1
+// GFX11: v_cmpx_class_f64_e64 -1, -1             ; encoding: [0x7e,0x00,0xff,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_class_f64_e64 0.5, 0.5
+// GFX11: v_cmpx_class_f64_e64 0.5, 0.5           ; encoding: [0x7e,0x00,0xff,0xd4,0xf0,0xe0,0x01,0x00]
+
+v_cmpx_class_f64_e64 -|src_scc|, src_scc
+// GFX11: v_cmpx_class_f64_e64 -|src_scc|, src_scc ; encoding: [0x7e,0x01,0xff,0xd4,0xfd,0xfa,0x01,0x20]
+
+v_cmpx_class_f64_e64 0xaf123456, 0xaf123456
+// GFX11: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f16_e64 v1, v2
+// GFX11: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f16_e64 v255, v255
+// GFX11: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_f16_e64 s1, s2
+// GFX11: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_f16_e64 s105, s105
+// GFX11: v_cmpx_eq_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_f16_e64 m0, 0.5
+// GFX11: v_cmpx_eq_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_eq_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_f16_e64 null, exec_lo
+// GFX11: v_cmpx_eq_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16_e64 0.5, -m0
+// GFX11: v_cmpx_eq_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32_e64 v1, v2
+// GFX11: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f32_e64 v255, v255
+// GFX11: v_cmpx_eq_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x92,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_f32_e64 s1, s2
+// GFX11: v_cmpx_eq_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_f32_e64 s105, s105
+// GFX11: v_cmpx_eq_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x92,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x92,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_eq_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x92,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x92,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_f32_e64 m0, 0.5
+// GFX11: v_cmpx_eq_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x92,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x92,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_eq_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x92,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_f32_e64 null, exec_lo
+// GFX11: v_cmpx_eq_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x92,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x92,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32_e64 0.5, -m0
+// GFX11: v_cmpx_eq_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x92,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_eq_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_eq_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x92,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_eq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_eq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x92,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_eq_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_eq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa2,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_eq_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_eq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa2,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_eq_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa2,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_eq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_eq_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa2,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_eq_f64_e64 null, 0.5
+// GFX11: v_cmpx_eq_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa2,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_f64_e64 -1, -1
+// GFX11: v_cmpx_eq_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa2,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_f64_e64 0.5, null
+// GFX11: v_cmpx_eq_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa2,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_eq_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa2,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_eq_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_eq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa2,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i16_e64 v1, v2
+// GFX11: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_i16_e64 v255, v255
+// GFX11: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_i16_e64 s1, s2
+// GFX11: v_cmpx_eq_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_i16_e64 s105, s105
+// GFX11: v_cmpx_eq_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb2,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_eq_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_i16_e64 m0, 0.5
+// GFX11: v_cmpx_eq_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_i16_e64 exec_hi, null
+// GFX11: v_cmpx_eq_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_i16_e64 null, exec_lo
+// GFX11: v_cmpx_eq_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_i16_e64 0.5, m0
+// GFX11: v_cmpx_eq_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_eq_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32_e64 v1, v2
+// GFX11: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_i32_e64 v255, v255
+// GFX11: v_cmpx_eq_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_i32_e64 s1, s2
+// GFX11: v_cmpx_eq_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_i32_e64 s105, s105
+// GFX11: v_cmpx_eq_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc2,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc2,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_eq_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc2,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc2,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_i32_e64 m0, 0.5
+// GFX11: v_cmpx_eq_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc2,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc2,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_i32_e64 exec_hi, null
+// GFX11: v_cmpx_eq_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_i32_e64 null, exec_lo
+// GFX11: v_cmpx_eq_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc2,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32_e64 0.5, m0
+// GFX11: v_cmpx_eq_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc2,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_eq_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc2,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_eq_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_eq_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_eq_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd2,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_eq_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_eq_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd2,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_eq_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd2,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_eq_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i64_e64 exec, src_scc
+// GFX11: v_cmpx_eq_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd2,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_eq_i64_e64 null, 0.5
+// GFX11: v_cmpx_eq_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd2,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_i64_e64 -1, -1
+// GFX11: v_cmpx_eq_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd2,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_i64_e64 0.5, null
+// GFX11: v_cmpx_eq_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd2,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_i64_e64 src_scc, exec
+// GFX11: v_cmpx_eq_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd2,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_eq_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_eq_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd2,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u16_e64 v1, v2
+// GFX11: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_u16_e64 v255, v255
+// GFX11: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_u16_e64 s1, s2
+// GFX11: v_cmpx_eq_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_u16_e64 s105, s105
+// GFX11: v_cmpx_eq_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xba,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xba,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_eq_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xba,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_u16_e64 m0, 0.5
+// GFX11: v_cmpx_eq_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_u16_e64 exec_hi, null
+// GFX11: v_cmpx_eq_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xba,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_u16_e64 null, exec_lo
+// GFX11: v_cmpx_eq_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xba,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_u16_e64 0.5, m0
+// GFX11: v_cmpx_eq_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_eq_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32_e64 v1, v2
+// GFX11: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_u32_e64 v255, v255
+// GFX11: v_cmpx_eq_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_u32_e64 s1, s2
+// GFX11: v_cmpx_eq_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_u32_e64 s105, s105
+// GFX11: v_cmpx_eq_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xca,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_eq_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xca,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_eq_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xca,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_eq_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xca,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_u32_e64 m0, 0.5
+// GFX11: v_cmpx_eq_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xca,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_eq_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xca,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_u32_e64 exec_hi, null
+// GFX11: v_cmpx_eq_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xca,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_u32_e64 null, exec_lo
+// GFX11: v_cmpx_eq_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xca,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_eq_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xca,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32_e64 0.5, m0
+// GFX11: v_cmpx_eq_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xca,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_eq_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xca,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_eq_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_eq_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xda,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_eq_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xda,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_eq_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xda,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_eq_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xda,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_eq_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xda,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_eq_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xda,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u64_e64 exec, src_scc
+// GFX11: v_cmpx_eq_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xda,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_eq_u64_e64 null, 0.5
+// GFX11: v_cmpx_eq_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xda,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_u64_e64 -1, -1
+// GFX11: v_cmpx_eq_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xda,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_u64_e64 0.5, null
+// GFX11: v_cmpx_eq_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xda,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_u64_e64 src_scc, exec
+// GFX11: v_cmpx_eq_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xda,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_eq_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_eq_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f16_e64 v1, v2
+// GFX11: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_f16_e64 v255, v255
+// GFX11: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_f_f16_e64 s1, s2
+// GFX11: v_cmpx_f_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_f_f16_e64 s105, s105
+// GFX11: v_cmpx_f_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x80,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_f_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_f_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x80,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_f_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_f_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x80,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_f_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_f_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x80,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_f_f16_e64 m0, 0.5
+// GFX11: v_cmpx_f_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x80,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_f_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_f_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x80,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_f_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_f_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x80,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_f_f16_e64 null, exec_lo
+// GFX11: v_cmpx_f_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x80,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_f_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_f_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x80,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_f_f16_e64 0.5, -m0
+// GFX11: v_cmpx_f_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x80,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_f_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_f_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x80,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_f_f32_e64 v1, v2
+// GFX11: v_cmpx_f_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_f32_e64 v255, v255
+// GFX11: v_cmpx_f_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x90,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_f_f32_e64 s1, s2
+// GFX11: v_cmpx_f_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_f_f32_e64 s105, s105
+// GFX11: v_cmpx_f_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x90,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_f_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_f_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x90,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_f_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_f_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x90,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_f_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x90,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_f_f32_e64 m0, 0.5
+// GFX11: v_cmpx_f_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x90,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_f_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_f_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x90,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_f_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_f_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x90,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_f_f32_e64 null, exec_lo
+// GFX11: v_cmpx_f_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x90,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_f_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_f_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x90,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_f_f32_e64 0.5, -m0
+// GFX11: v_cmpx_f_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x90,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_f_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_f_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x90,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_f_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_f_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x90,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_f_f64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xa0,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_f_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa0,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_f_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_f_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa0,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_f_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_f_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa0,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_f_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_f_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa0,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_f_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_f_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa0,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_f_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa0,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_f_f64_e64 null, 0.5
+// GFX11: v_cmpx_f_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa0,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_f_f64_e64 -1, -1
+// GFX11: v_cmpx_f_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa0,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_f_f64_e64 0.5, null
+// GFX11: v_cmpx_f_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa0,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_f_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_f_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa0,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_f_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_f_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa0,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i32_e64 v1, v2
+// GFX11: v_cmpx_f_i32_e64 v1, v2                 ; encoding: [0x7e,0x00,0xc0,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_i32_e64 v255, v255
+// GFX11: v_cmpx_f_i32_e64 v255, v255             ; encoding: [0x7e,0x00,0xc0,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_f_i32_e64 s1, s2
+// GFX11: v_cmpx_f_i32_e64 s1, s2                 ; encoding: [0x7e,0x00,0xc0,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_f_i32_e64 s105, s105
+// GFX11: v_cmpx_f_i32_e64 s105, s105             ; encoding: [0x7e,0x00,0xc0,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_f_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_f_i32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0xc0,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_f_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_f_i32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0xc0,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_f_i32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0xc0,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_f_i32_e64 m0, 0.5
+// GFX11: v_cmpx_f_i32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0xc0,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_f_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_f_i32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0xc0,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_f_i32_e64 exec_hi, null
+// GFX11: v_cmpx_f_i32_e64 exec_hi, null          ; encoding: [0x7e,0x00,0xc0,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_f_i32_e64 null, exec_lo
+// GFX11: v_cmpx_f_i32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0xc0,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_f_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_f_i32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0xc0,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_f_i32_e64 0.5, m0
+// GFX11: v_cmpx_f_i32_e64 0.5, m0                ; encoding: [0x7e,0x00,0xc0,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_f_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_f_i32_e64 src_scc, vcc_lo        ; encoding: [0x7e,0x00,0xc0,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_f_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_f_i32_e64 0xaf123456, vcc_hi     ; encoding: [0x7e,0x00,0xc0,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_f_i64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xd0,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_f_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd0,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_f_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_f_i64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xd0,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_f_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_f_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd0,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_f_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_f_i64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xd0,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_f_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_f_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd0,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_i64_e64 exec, src_scc
+// GFX11: v_cmpx_f_i64_e64 exec, src_scc          ; encoding: [0x7e,0x00,0xd0,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_f_i64_e64 null, 0.5
+// GFX11: v_cmpx_f_i64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xd0,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_f_i64_e64 -1, -1
+// GFX11: v_cmpx_f_i64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xd0,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_f_i64_e64 0.5, null
+// GFX11: v_cmpx_f_i64_e64 0.5, null              ; encoding: [0x7e,0x00,0xd0,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_f_i64_e64 src_scc, exec
+// GFX11: v_cmpx_f_i64_e64 src_scc, exec          ; encoding: [0x7e,0x00,0xd0,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_f_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_f_i64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xd0,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u32_e64 v1, v2
+// GFX11: v_cmpx_f_u32_e64 v1, v2                 ; encoding: [0x7e,0x00,0xc8,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_u32_e64 v255, v255
+// GFX11: v_cmpx_f_u32_e64 v255, v255             ; encoding: [0x7e,0x00,0xc8,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_f_u32_e64 s1, s2
+// GFX11: v_cmpx_f_u32_e64 s1, s2                 ; encoding: [0x7e,0x00,0xc8,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_f_u32_e64 s105, s105
+// GFX11: v_cmpx_f_u32_e64 s105, s105             ; encoding: [0x7e,0x00,0xc8,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_f_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_f_u32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0xc8,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_f_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_f_u32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0xc8,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_f_u32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0xc8,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_f_u32_e64 m0, 0.5
+// GFX11: v_cmpx_f_u32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0xc8,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_f_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_f_u32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0xc8,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_f_u32_e64 exec_hi, null
+// GFX11: v_cmpx_f_u32_e64 exec_hi, null          ; encoding: [0x7e,0x00,0xc8,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_f_u32_e64 null, exec_lo
+// GFX11: v_cmpx_f_u32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0xc8,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_f_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_f_u32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0xc8,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_f_u32_e64 0.5, m0
+// GFX11: v_cmpx_f_u32_e64 0.5, m0                ; encoding: [0x7e,0x00,0xc8,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_f_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_f_u32_e64 src_scc, vcc_lo        ; encoding: [0x7e,0x00,0xc8,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_f_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_f_u32_e64 0xaf123456, vcc_hi     ; encoding: [0x7e,0x00,0xc8,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_f_u64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xd8,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_f_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd8,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_f_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_f_u64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xd8,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_f_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_f_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd8,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_f_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_f_u64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xd8,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_f_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_f_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd8,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_f_u64_e64 exec, src_scc
+// GFX11: v_cmpx_f_u64_e64 exec, src_scc          ; encoding: [0x7e,0x00,0xd8,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_f_u64_e64 null, 0.5
+// GFX11: v_cmpx_f_u64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xd8,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_f_u64_e64 -1, -1
+// GFX11: v_cmpx_f_u64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xd8,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_f_u64_e64 0.5, null
+// GFX11: v_cmpx_f_u64_e64 0.5, null              ; encoding: [0x7e,0x00,0xd8,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_f_u64_e64 src_scc, exec
+// GFX11: v_cmpx_f_u64_e64 src_scc, exec          ; encoding: [0x7e,0x00,0xd8,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_f_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_f_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xd8,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f16_e64 v1, v2
+// GFX11: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f16_e64 v255, v255
+// GFX11: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_f16_e64 s1, s2
+// GFX11: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_f16_e64 s105, s105
+// GFX11: v_cmpx_ge_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_f16_e64 m0, 0.5
+// GFX11: v_cmpx_ge_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_ge_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_f16_e64 null, exec_lo
+// GFX11: v_cmpx_ge_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_f16_e64 0.5, -m0
+// GFX11: v_cmpx_ge_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32_e64 v1, v2
+// GFX11: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f32_e64 v255, v255
+// GFX11: v_cmpx_ge_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x96,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_f32_e64 s1, s2
+// GFX11: v_cmpx_ge_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_f32_e64 s105, s105
+// GFX11: v_cmpx_ge_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x96,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x96,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ge_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x96,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x96,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_f32_e64 m0, 0.5
+// GFX11: v_cmpx_ge_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x96,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x96,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_ge_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x96,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_f32_e64 null, exec_lo
+// GFX11: v_cmpx_ge_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x96,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x96,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32_e64 0.5, -m0
+// GFX11: v_cmpx_ge_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x96,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ge_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_ge_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x96,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_ge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x96,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ge_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa6,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ge_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa6,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ge_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa6,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_ge_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa6,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_ge_f64_e64 null, 0.5
+// GFX11: v_cmpx_ge_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa6,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_f64_e64 -1, -1
+// GFX11: v_cmpx_ge_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa6,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_f64_e64 0.5, null
+// GFX11: v_cmpx_ge_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa6,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_ge_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa6,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_ge_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_ge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa6,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i16_e64 v1, v2
+// GFX11: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_i16_e64 v255, v255
+// GFX11: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_i16_e64 s1, s2
+// GFX11: v_cmpx_ge_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_i16_e64 s105, s105
+// GFX11: v_cmpx_ge_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb6,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ge_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_i16_e64 m0, 0.5
+// GFX11: v_cmpx_ge_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_i16_e64 exec_hi, null
+// GFX11: v_cmpx_ge_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_i16_e64 null, exec_lo
+// GFX11: v_cmpx_ge_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_i16_e64 0.5, m0
+// GFX11: v_cmpx_ge_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ge_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32_e64 v1, v2
+// GFX11: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_i32_e64 v255, v255
+// GFX11: v_cmpx_ge_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_i32_e64 s1, s2
+// GFX11: v_cmpx_ge_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_i32_e64 s105, s105
+// GFX11: v_cmpx_ge_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc6,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc6,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ge_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc6,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc6,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_i32_e64 m0, 0.5
+// GFX11: v_cmpx_ge_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc6,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc6,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_i32_e64 exec_hi, null
+// GFX11: v_cmpx_ge_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_i32_e64 null, exec_lo
+// GFX11: v_cmpx_ge_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc6,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32_e64 0.5, m0
+// GFX11: v_cmpx_ge_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc6,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ge_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc6,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_ge_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ge_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ge_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd6,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ge_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ge_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd6,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ge_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd6,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ge_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i64_e64 exec, src_scc
+// GFX11: v_cmpx_ge_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd6,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ge_i64_e64 null, 0.5
+// GFX11: v_cmpx_ge_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd6,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_i64_e64 -1, -1
+// GFX11: v_cmpx_ge_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd6,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_i64_e64 0.5, null
+// GFX11: v_cmpx_ge_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd6,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_i64_e64 src_scc, exec
+// GFX11: v_cmpx_ge_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd6,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ge_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_ge_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd6,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u16_e64 v1, v2
+// GFX11: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_u16_e64 v255, v255
+// GFX11: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_u16_e64 s1, s2
+// GFX11: v_cmpx_ge_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_u16_e64 s105, s105
+// GFX11: v_cmpx_ge_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbe,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ge_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_u16_e64 m0, 0.5
+// GFX11: v_cmpx_ge_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_u16_e64 exec_hi, null
+// GFX11: v_cmpx_ge_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_u16_e64 null, exec_lo
+// GFX11: v_cmpx_ge_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_u16_e64 0.5, m0
+// GFX11: v_cmpx_ge_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ge_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32_e64 v1, v2
+// GFX11: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_u32_e64 v255, v255
+// GFX11: v_cmpx_ge_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_u32_e64 s1, s2
+// GFX11: v_cmpx_ge_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_u32_e64 s105, s105
+// GFX11: v_cmpx_ge_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xce,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ge_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xce,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ge_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xce,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ge_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xce,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_u32_e64 m0, 0.5
+// GFX11: v_cmpx_ge_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xce,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_ge_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xce,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_u32_e64 exec_hi, null
+// GFX11: v_cmpx_ge_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xce,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_u32_e64 null, exec_lo
+// GFX11: v_cmpx_ge_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xce,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_ge_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xce,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32_e64 0.5, m0
+// GFX11: v_cmpx_ge_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xce,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ge_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xce,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_ge_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ge_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xde,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ge_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xde,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ge_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xde,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ge_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xde,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ge_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xde,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ge_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xde,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u64_e64 exec, src_scc
+// GFX11: v_cmpx_ge_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xde,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ge_u64_e64 null, 0.5
+// GFX11: v_cmpx_ge_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xde,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_u64_e64 -1, -1
+// GFX11: v_cmpx_ge_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xde,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_u64_e64 0.5, null
+// GFX11: v_cmpx_ge_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xde,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_u64_e64 src_scc, exec
+// GFX11: v_cmpx_ge_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xde,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ge_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_ge_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f16_e64 v1, v2
+// GFX11: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f16_e64 v255, v255
+// GFX11: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_f16_e64 s1, s2
+// GFX11: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_f16_e64 s105, s105
+// GFX11: v_cmpx_gt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_f16_e64 m0, 0.5
+// GFX11: v_cmpx_gt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_gt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_f16_e64 null, exec_lo
+// GFX11: v_cmpx_gt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_f16_e64 0.5, -m0
+// GFX11: v_cmpx_gt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32_e64 v1, v2
+// GFX11: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f32_e64 v255, v255
+// GFX11: v_cmpx_gt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x94,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_f32_e64 s1, s2
+// GFX11: v_cmpx_gt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_f32_e64 s105, s105
+// GFX11: v_cmpx_gt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x94,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x94,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_gt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x94,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x94,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_f32_e64 m0, 0.5
+// GFX11: v_cmpx_gt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x94,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x94,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_gt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x94,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_f32_e64 null, exec_lo
+// GFX11: v_cmpx_gt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x94,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x94,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32_e64 0.5, -m0
+// GFX11: v_cmpx_gt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x94,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_gt_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_gt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x94,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_gt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_gt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x94,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_gt_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_gt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa4,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_gt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_gt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa4,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_gt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa4,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_gt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_gt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa4,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_gt_f64_e64 null, 0.5
+// GFX11: v_cmpx_gt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa4,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_f64_e64 -1, -1
+// GFX11: v_cmpx_gt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa4,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_f64_e64 0.5, null
+// GFX11: v_cmpx_gt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa4,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_gt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa4,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_gt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_gt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa4,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i16_e64 v1, v2
+// GFX11: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_i16_e64 v255, v255
+// GFX11: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_i16_e64 s1, s2
+// GFX11: v_cmpx_gt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_i16_e64 s105, s105
+// GFX11: v_cmpx_gt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb4,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_gt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_i16_e64 m0, 0.5
+// GFX11: v_cmpx_gt_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_i16_e64 exec_hi, null
+// GFX11: v_cmpx_gt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_i16_e64 null, exec_lo
+// GFX11: v_cmpx_gt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_i16_e64 0.5, m0
+// GFX11: v_cmpx_gt_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_gt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32_e64 v1, v2
+// GFX11: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_i32_e64 v255, v255
+// GFX11: v_cmpx_gt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_i32_e64 s1, s2
+// GFX11: v_cmpx_gt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_i32_e64 s105, s105
+// GFX11: v_cmpx_gt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc4,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc4,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_gt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc4,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc4,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_i32_e64 m0, 0.5
+// GFX11: v_cmpx_gt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc4,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc4,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_i32_e64 exec_hi, null
+// GFX11: v_cmpx_gt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_i32_e64 null, exec_lo
+// GFX11: v_cmpx_gt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc4,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32_e64 0.5, m0
+// GFX11: v_cmpx_gt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc4,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_gt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc4,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_gt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_gt_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_gt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd4,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_gt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_gt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd4,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_gt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd4,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_gt_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i64_e64 exec, src_scc
+// GFX11: v_cmpx_gt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd4,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_gt_i64_e64 null, 0.5
+// GFX11: v_cmpx_gt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd4,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_i64_e64 -1, -1
+// GFX11: v_cmpx_gt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd4,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_i64_e64 0.5, null
+// GFX11: v_cmpx_gt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd4,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_i64_e64 src_scc, exec
+// GFX11: v_cmpx_gt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd4,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_gt_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_gt_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd4,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u16_e64 v1, v2
+// GFX11: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_u16_e64 v255, v255
+// GFX11: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_u16_e64 s1, s2
+// GFX11: v_cmpx_gt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_u16_e64 s105, s105
+// GFX11: v_cmpx_gt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbc,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_gt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_u16_e64 m0, 0.5
+// GFX11: v_cmpx_gt_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_u16_e64 exec_hi, null
+// GFX11: v_cmpx_gt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_u16_e64 null, exec_lo
+// GFX11: v_cmpx_gt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_u16_e64 0.5, m0
+// GFX11: v_cmpx_gt_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_gt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32_e64 v1, v2
+// GFX11: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_u32_e64 v255, v255
+// GFX11: v_cmpx_gt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_u32_e64 s1, s2
+// GFX11: v_cmpx_gt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_u32_e64 s105, s105
+// GFX11: v_cmpx_gt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcc,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_gt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcc,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_gt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcc,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_gt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcc,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_u32_e64 m0, 0.5
+// GFX11: v_cmpx_gt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcc,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_gt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcc,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_u32_e64 exec_hi, null
+// GFX11: v_cmpx_gt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_u32_e64 null, exec_lo
+// GFX11: v_cmpx_gt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_gt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcc,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32_e64 0.5, m0
+// GFX11: v_cmpx_gt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcc,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_gt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcc,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_gt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_gt_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_gt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdc,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_gt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_gt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdc,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_gt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdc,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_gt_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xdc,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u64_e64 exec, src_scc
+// GFX11: v_cmpx_gt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdc,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_gt_u64_e64 null, 0.5
+// GFX11: v_cmpx_gt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdc,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_u64_e64 -1, -1
+// GFX11: v_cmpx_gt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdc,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_u64_e64 0.5, null
+// GFX11: v_cmpx_gt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdc,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_u64_e64 src_scc, exec
+// GFX11: v_cmpx_gt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdc,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_gt_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_gt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f16_e64 v1, v2
+// GFX11: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f16_e64 v255, v255
+// GFX11: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_f16_e64 s1, s2
+// GFX11: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_f16_e64 s105, s105
+// GFX11: v_cmpx_le_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_le_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_f16_e64 m0, 0.5
+// GFX11: v_cmpx_le_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_le_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_le_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_f16_e64 null, exec_lo
+// GFX11: v_cmpx_le_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_le_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_f16_e64 0.5, -m0
+// GFX11: v_cmpx_le_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_le_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_le_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f32_e64 v1, v2
+// GFX11: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f32_e64 v255, v255
+// GFX11: v_cmpx_le_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x93,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_f32_e64 s1, s2
+// GFX11: v_cmpx_le_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_f32_e64 s105, s105
+// GFX11: v_cmpx_le_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x93,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x93,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_le_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x93,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x93,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_f32_e64 m0, 0.5
+// GFX11: v_cmpx_le_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x93,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_le_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x93,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_le_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x93,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_f32_e64 null, exec_lo
+// GFX11: v_cmpx_le_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x93,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_le_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x93,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_f32_e64 0.5, -m0
+// GFX11: v_cmpx_le_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x93,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_le_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_le_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x93,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_le_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_le_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x93,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_le_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_le_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa3,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_le_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_le_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa3,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_le_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa3,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_le_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_le_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa3,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_le_f64_e64 null, 0.5
+// GFX11: v_cmpx_le_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa3,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_f64_e64 -1, -1
+// GFX11: v_cmpx_le_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa3,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_f64_e64 0.5, null
+// GFX11: v_cmpx_le_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa3,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_le_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa3,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_le_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_le_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa3,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i16_e64 v1, v2
+// GFX11: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_i16_e64 v255, v255
+// GFX11: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_i16_e64 s1, s2
+// GFX11: v_cmpx_le_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_i16_e64 s105, s105
+// GFX11: v_cmpx_le_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb3,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_le_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_i16_e64 m0, 0.5
+// GFX11: v_cmpx_le_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_le_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_i16_e64 exec_hi, null
+// GFX11: v_cmpx_le_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_i16_e64 null, exec_lo
+// GFX11: v_cmpx_le_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_le_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_i16_e64 0.5, m0
+// GFX11: v_cmpx_le_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_le_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i32_e64 v1, v2
+// GFX11: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_i32_e64 v255, v255
+// GFX11: v_cmpx_le_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_i32_e64 s1, s2
+// GFX11: v_cmpx_le_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_i32_e64 s105, s105
+// GFX11: v_cmpx_le_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc3,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc3,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_le_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc3,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc3,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_i32_e64 m0, 0.5
+// GFX11: v_cmpx_le_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc3,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_le_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc3,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_i32_e64 exec_hi, null
+// GFX11: v_cmpx_le_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_i32_e64 null, exec_lo
+// GFX11: v_cmpx_le_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_le_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc3,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_i32_e64 0.5, m0
+// GFX11: v_cmpx_le_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc3,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_le_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc3,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_le_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_le_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_le_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd3,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_le_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_le_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd3,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_le_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd3,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_le_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i64_e64 exec, src_scc
+// GFX11: v_cmpx_le_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd3,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_le_i64_e64 null, 0.5
+// GFX11: v_cmpx_le_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd3,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_i64_e64 -1, -1
+// GFX11: v_cmpx_le_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd3,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_i64_e64 0.5, null
+// GFX11: v_cmpx_le_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd3,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_i64_e64 src_scc, exec
+// GFX11: v_cmpx_le_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd3,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_le_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_le_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd3,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u16_e64 v1, v2
+// GFX11: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_u16_e64 v255, v255
+// GFX11: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_u16_e64 s1, s2
+// GFX11: v_cmpx_le_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_u16_e64 s105, s105
+// GFX11: v_cmpx_le_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbb,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_le_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_u16_e64 m0, 0.5
+// GFX11: v_cmpx_le_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_le_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_u16_e64 exec_hi, null
+// GFX11: v_cmpx_le_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_u16_e64 null, exec_lo
+// GFX11: v_cmpx_le_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_le_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_u16_e64 0.5, m0
+// GFX11: v_cmpx_le_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_le_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u32_e64 v1, v2
+// GFX11: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_u32_e64 v255, v255
+// GFX11: v_cmpx_le_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_u32_e64 s1, s2
+// GFX11: v_cmpx_le_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_u32_e64 s105, s105
+// GFX11: v_cmpx_le_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcb,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_le_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcb,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_le_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcb,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_le_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcb,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_u32_e64 m0, 0.5
+// GFX11: v_cmpx_le_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcb,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_le_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcb,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_u32_e64 exec_hi, null
+// GFX11: v_cmpx_le_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_u32_e64 null, exec_lo
+// GFX11: v_cmpx_le_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_le_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcb,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_u32_e64 0.5, m0
+// GFX11: v_cmpx_le_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcb,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_le_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcb,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_le_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_le_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_le_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdb,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_le_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_le_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdb,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_le_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdb,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_le_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xdb,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u64_e64 exec, src_scc
+// GFX11: v_cmpx_le_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdb,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_le_u64_e64 null, 0.5
+// GFX11: v_cmpx_le_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdb,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_u64_e64 -1, -1
+// GFX11: v_cmpx_le_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdb,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_u64_e64 0.5, null
+// GFX11: v_cmpx_le_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdb,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_u64_e64 src_scc, exec
+// GFX11: v_cmpx_le_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdb,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_le_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_le_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f16_e64 v1, v2
+// GFX11: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f16_e64 v255, v255
+// GFX11: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lg_f16_e64 s1, s2
+// GFX11: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lg_f16_e64 s105, s105
+// GFX11: v_cmpx_lg_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lg_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lg_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lg_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lg_f16_e64 m0, 0.5
+// GFX11: v_cmpx_lg_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lg_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_lg_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lg_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_lg_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lg_f16_e64 null, exec_lo
+// GFX11: v_cmpx_lg_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lg_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_lg_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lg_f16_e64 0.5, -m0
+// GFX11: v_cmpx_lg_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32_e64 v1, v2
+// GFX11: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f32_e64 v255, v255
+// GFX11: v_cmpx_lg_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x95,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lg_f32_e64 s1, s2
+// GFX11: v_cmpx_lg_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lg_f32_e64 s105, s105
+// GFX11: v_cmpx_lg_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x95,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lg_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lg_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x95,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lg_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_lg_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x95,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lg_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x95,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lg_f32_e64 m0, 0.5
+// GFX11: v_cmpx_lg_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x95,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lg_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_lg_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x95,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lg_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_lg_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x95,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lg_f32_e64 null, exec_lo
+// GFX11: v_cmpx_lg_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x95,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lg_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_lg_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x95,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32_e64 0.5, -m0
+// GFX11: v_cmpx_lg_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x95,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lg_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_lg_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x95,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_lg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x95,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_lg_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_lg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa5,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lg_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_lg_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lg_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_lg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa5,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lg_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_lg_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa5,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lg_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_lg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_lg_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa5,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_lg_f64_e64 null, 0.5
+// GFX11: v_cmpx_lg_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa5,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lg_f64_e64 -1, -1
+// GFX11: v_cmpx_lg_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa5,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lg_f64_e64 0.5, null
+// GFX11: v_cmpx_lg_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa5,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lg_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_lg_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa5,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_lg_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_lg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa5,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f16_e64 v1, v2
+// GFX11: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_f16_e64 v255, v255
+// GFX11: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_f16_e64 s1, s2
+// GFX11: v_cmpx_lt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_f16_e64 s105, s105
+// GFX11: v_cmpx_lt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x81,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x81,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_lt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x81,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x81,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_f16_e64 m0, 0.5
+// GFX11: v_cmpx_lt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x81,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x81,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_lt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x81,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_f16_e64 null, exec_lo
+// GFX11: v_cmpx_lt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x81,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x81,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_f16_e64 0.5, -m0
+// GFX11: v_cmpx_lt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x81,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lt_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_lt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x81,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32_e64 v1, v2
+// GFX11: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_f32_e64 v255, v255
+// GFX11: v_cmpx_lt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x91,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_f32_e64 s1, s2
+// GFX11: v_cmpx_lt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_f32_e64 s105, s105
+// GFX11: v_cmpx_lt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x91,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x91,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_lt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x91,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x91,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_f32_e64 m0, 0.5
+// GFX11: v_cmpx_lt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x91,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x91,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_lt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x91,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_f32_e64 null, exec_lo
+// GFX11: v_cmpx_lt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x91,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x91,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32_e64 0.5, -m0
+// GFX11: v_cmpx_lt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x91,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lt_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_lt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x91,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_lt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x91,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_lt_f64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_lt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa1,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_lt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_lt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa1,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_lt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa1,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_lt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_lt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa1,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_lt_f64_e64 null, 0.5
+// GFX11: v_cmpx_lt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa1,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_f64_e64 -1, -1
+// GFX11: v_cmpx_lt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa1,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_f64_e64 0.5, null
+// GFX11: v_cmpx_lt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa1,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_lt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa1,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_lt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_lt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa1,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i16_e64 v1, v2
+// GFX11: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_i16_e64 v255, v255
+// GFX11: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_i16_e64 s1, s2
+// GFX11: v_cmpx_lt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_i16_e64 s105, s105
+// GFX11: v_cmpx_lt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb1,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_lt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_i16_e64 m0, 0.5
+// GFX11: v_cmpx_lt_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_i16_e64 exec_hi, null
+// GFX11: v_cmpx_lt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_i16_e64 null, exec_lo
+// GFX11: v_cmpx_lt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_i16_e64 0.5, m0
+// GFX11: v_cmpx_lt_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_lt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32_e64 v1, v2
+// GFX11: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_i32_e64 v255, v255
+// GFX11: v_cmpx_lt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_i32_e64 s1, s2
+// GFX11: v_cmpx_lt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_i32_e64 s105, s105
+// GFX11: v_cmpx_lt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc1,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc1,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_lt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc1,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc1,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_i32_e64 m0, 0.5
+// GFX11: v_cmpx_lt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc1,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc1,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_i32_e64 exec_hi, null
+// GFX11: v_cmpx_lt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_i32_e64 null, exec_lo
+// GFX11: v_cmpx_lt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc1,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32_e64 0.5, m0
+// GFX11: v_cmpx_lt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc1,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_lt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc1,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_lt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_lt_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_lt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd1,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_lt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_lt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd1,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_lt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd1,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_lt_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i64_e64 exec, src_scc
+// GFX11: v_cmpx_lt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd1,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_lt_i64_e64 null, 0.5
+// GFX11: v_cmpx_lt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd1,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_i64_e64 -1, -1
+// GFX11: v_cmpx_lt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd1,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_i64_e64 0.5, null
+// GFX11: v_cmpx_lt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd1,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_i64_e64 src_scc, exec
+// GFX11: v_cmpx_lt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd1,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_lt_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_lt_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd1,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u16_e64 v1, v2
+// GFX11: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_u16_e64 v255, v255
+// GFX11: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_u16_e64 s1, s2
+// GFX11: v_cmpx_lt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_u16_e64 s105, s105
+// GFX11: v_cmpx_lt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb9,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_lt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_u16_e64 m0, 0.5
+// GFX11: v_cmpx_lt_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_u16_e64 exec_hi, null
+// GFX11: v_cmpx_lt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_u16_e64 null, exec_lo
+// GFX11: v_cmpx_lt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_u16_e64 0.5, m0
+// GFX11: v_cmpx_lt_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_lt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32_e64 v1, v2
+// GFX11: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_u32_e64 v255, v255
+// GFX11: v_cmpx_lt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_u32_e64 s1, s2
+// GFX11: v_cmpx_lt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_u32_e64 s105, s105
+// GFX11: v_cmpx_lt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc9,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_lt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc9,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_lt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc9,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_lt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc9,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_u32_e64 m0, 0.5
+// GFX11: v_cmpx_lt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc9,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_lt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc9,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_u32_e64 exec_hi, null
+// GFX11: v_cmpx_lt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_u32_e64 null, exec_lo
+// GFX11: v_cmpx_lt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_lt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc9,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32_e64 0.5, m0
+// GFX11: v_cmpx_lt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc9,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_lt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc9,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_lt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_lt_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_lt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd9,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_lt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_lt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd9,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_lt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd9,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_lt_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u64_e64 exec, src_scc
+// GFX11: v_cmpx_lt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd9,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_lt_u64_e64 null, 0.5
+// GFX11: v_cmpx_lt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd9,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_u64_e64 -1, -1
+// GFX11: v_cmpx_lt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd9,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_u64_e64 0.5, null
+// GFX11: v_cmpx_lt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd9,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_u64_e64 src_scc, exec
+// GFX11: v_cmpx_lt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd9,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_lt_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_lt_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd9,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i16_e64 v1, v2
+// GFX11: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_i16_e64 v255, v255
+// GFX11: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_i16_e64 s1, s2
+// GFX11: v_cmpx_ne_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_i16_e64 s105, s105
+// GFX11: v_cmpx_ne_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb5,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_i16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ne_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_i16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ne_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ne_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_i16_e64 m0, 0.5
+// GFX11: v_cmpx_ne_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_i16_e64 exec_lo, -1
+// GFX11: v_cmpx_ne_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_i16_e64 exec_hi, null
+// GFX11: v_cmpx_ne_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_i16_e64 null, exec_lo
+// GFX11: v_cmpx_ne_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_i16_e64 -1, exec_hi
+// GFX11: v_cmpx_ne_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_i16_e64 0.5, m0
+// GFX11: v_cmpx_ne_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_i16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ne_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32_e64 v1, v2
+// GFX11: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_i32_e64 v255, v255
+// GFX11: v_cmpx_ne_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_i32_e64 s1, s2
+// GFX11: v_cmpx_ne_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_i32_e64 s105, s105
+// GFX11: v_cmpx_ne_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc5,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ne_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc5,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ne_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc5,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ne_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc5,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_i32_e64 m0, 0.5
+// GFX11: v_cmpx_ne_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc5,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_ne_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc5,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_i32_e64 exec_hi, null
+// GFX11: v_cmpx_ne_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_i32_e64 null, exec_lo
+// GFX11: v_cmpx_ne_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_ne_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc5,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32_e64 0.5, m0
+// GFX11: v_cmpx_ne_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc5,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ne_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc5,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_ne_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ne_i64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ne_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd5,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ne_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ne_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ne_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ne_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd5,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ne_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ne_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd5,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ne_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ne_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i64_e64 exec, src_scc
+// GFX11: v_cmpx_ne_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd5,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ne_i64_e64 null, 0.5
+// GFX11: v_cmpx_ne_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd5,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ne_i64_e64 -1, -1
+// GFX11: v_cmpx_ne_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd5,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ne_i64_e64 0.5, null
+// GFX11: v_cmpx_ne_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd5,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ne_i64_e64 src_scc, exec
+// GFX11: v_cmpx_ne_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd5,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ne_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_ne_i64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xd5,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u16_e64 v1, v2
+// GFX11: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_u16_e64 v255, v255
+// GFX11: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_u16_e64 s1, s2
+// GFX11: v_cmpx_ne_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_u16_e64 s105, s105
+// GFX11: v_cmpx_ne_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbd,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_u16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ne_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_u16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ne_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ne_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_u16_e64 m0, 0.5
+// GFX11: v_cmpx_ne_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_u16_e64 exec_lo, -1
+// GFX11: v_cmpx_ne_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_u16_e64 exec_hi, null
+// GFX11: v_cmpx_ne_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_u16_e64 null, exec_lo
+// GFX11: v_cmpx_ne_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_u16_e64 -1, exec_hi
+// GFX11: v_cmpx_ne_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_u16_e64 0.5, m0
+// GFX11: v_cmpx_ne_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_u16_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ne_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
+// GFX11: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32_e64 v1, v2
+// GFX11: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_u32_e64 v255, v255
+// GFX11: v_cmpx_ne_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_u32_e64 s1, s2
+// GFX11: v_cmpx_ne_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_u32_e64 s105, s105
+// GFX11: v_cmpx_ne_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcd,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ne_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcd,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ne_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcd,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ne_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcd,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_u32_e64 m0, 0.5
+// GFX11: v_cmpx_ne_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcd,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_ne_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_u32_e64 exec_hi, null
+// GFX11: v_cmpx_ne_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_u32_e64 null, exec_lo
+// GFX11: v_cmpx_ne_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_ne_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32_e64 0.5, m0
+// GFX11: v_cmpx_ne_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_ne_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_ne_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ne_u64_e64 v[1:2], v[2:3]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ne_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdd,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ne_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ne_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ne_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ne_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdd,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ne_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ne_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdd,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ne_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ne_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xdd,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u64_e64 exec, src_scc
+// GFX11: v_cmpx_ne_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdd,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ne_u64_e64 null, 0.5
+// GFX11: v_cmpx_ne_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdd,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ne_u64_e64 -1, -1
+// GFX11: v_cmpx_ne_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdd,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ne_u64_e64 0.5, null
+// GFX11: v_cmpx_ne_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdd,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ne_u64_e64 src_scc, exec
+// GFX11: v_cmpx_ne_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdd,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ne_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_ne_u64_e64 0xaf123456, vcc       ; encoding: [0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f16_e64 v1, v2
+// GFX11: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f16_e64 v255, v255
+// GFX11: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_neq_f16_e64 s1, s2
+// GFX11: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_neq_f16_e64 s105, s105
+// GFX11: v_cmpx_neq_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_neq_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_neq_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_neq_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_neq_f16_e64 m0, 0.5
+// GFX11: v_cmpx_neq_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_neq_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_neq_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_neq_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_neq_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_neq_f16_e64 null, exec_lo
+// GFX11: v_cmpx_neq_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_neq_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_neq_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_neq_f16_e64 0.5, -m0
+// GFX11: v_cmpx_neq_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32_e64 v1, v2
+// GFX11: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f32_e64 v255, v255
+// GFX11: v_cmpx_neq_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9d,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_neq_f32_e64 s1, s2
+// GFX11: v_cmpx_neq_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_neq_f32_e64 s105, s105
+// GFX11: v_cmpx_neq_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9d,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_neq_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_neq_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9d,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_neq_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_neq_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_neq_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9d,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_neq_f32_e64 m0, 0.5
+// GFX11: v_cmpx_neq_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9d,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_neq_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_neq_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9d,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_neq_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_neq_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9d,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_neq_f32_e64 null, exec_lo
+// GFX11: v_cmpx_neq_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9d,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_neq_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_neq_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9d,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32_e64 0.5, -m0
+// GFX11: v_cmpx_neq_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9d,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_neq_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_neq_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9d,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_neq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_neq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_neq_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xad,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_neq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xad,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_neq_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_neq_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xad,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_neq_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_neq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xad,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_neq_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_neq_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xad,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_neq_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_neq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xad,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_neq_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xad,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_neq_f64_e64 null, 0.5
+// GFX11: v_cmpx_neq_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xad,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_neq_f64_e64 -1, -1
+// GFX11: v_cmpx_neq_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xad,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_neq_f64_e64 0.5, null
+// GFX11: v_cmpx_neq_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xad,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_neq_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_neq_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xad,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f16_e64 v1, v2
+// GFX11: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f16_e64 v255, v255
+// GFX11: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nge_f16_e64 s1, s2
+// GFX11: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nge_f16_e64 s105, s105
+// GFX11: v_cmpx_nge_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nge_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nge_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nge_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nge_f16_e64 m0, 0.5
+// GFX11: v_cmpx_nge_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nge_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_nge_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nge_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_nge_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nge_f16_e64 null, exec_lo
+// GFX11: v_cmpx_nge_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nge_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_nge_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nge_f16_e64 0.5, -m0
+// GFX11: v_cmpx_nge_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32_e64 v1, v2
+// GFX11: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f32_e64 v255, v255
+// GFX11: v_cmpx_nge_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x99,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nge_f32_e64 s1, s2
+// GFX11: v_cmpx_nge_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nge_f32_e64 s105, s105
+// GFX11: v_cmpx_nge_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x99,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nge_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nge_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x99,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nge_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_nge_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x99,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nge_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x99,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nge_f32_e64 m0, 0.5
+// GFX11: v_cmpx_nge_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x99,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nge_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_nge_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x99,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nge_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_nge_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x99,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nge_f32_e64 null, exec_lo
+// GFX11: v_cmpx_nge_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x99,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nge_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_nge_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x99,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32_e64 0.5, -m0
+// GFX11: v_cmpx_nge_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x99,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nge_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nge_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x99,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x99,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_nge_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_nge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa9,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nge_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_nge_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nge_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_nge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa9,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nge_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_nge_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xa9,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nge_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_nge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_nge_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xa9,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nge_f64_e64 null, 0.5
+// GFX11: v_cmpx_nge_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xa9,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nge_f64_e64 -1, -1
+// GFX11: v_cmpx_nge_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xa9,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nge_f64_e64 0.5, null
+// GFX11: v_cmpx_nge_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xa9,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nge_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_nge_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xa9,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f16_e64 v1, v2
+// GFX11: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16_e64 v255, v255
+// GFX11: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16_e64 s1, s2
+// GFX11: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 s105, s105
+// GFX11: v_cmpx_ngt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ngt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ngt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 m0, 0.5
+// GFX11: v_cmpx_ngt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_ngt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_ngt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 null, exec_lo
+// GFX11: v_cmpx_ngt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_ngt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 0.5, -m0
+// GFX11: v_cmpx_ngt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 v1, v2
+// GFX11: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f32_e64 v255, v255
+// GFX11: v_cmpx_ngt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9b,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ngt_f32_e64 s1, s2
+// GFX11: v_cmpx_ngt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 s105, s105
+// GFX11: v_cmpx_ngt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9b,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_ngt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9b,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_ngt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_ngt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9b,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 m0, 0.5
+// GFX11: v_cmpx_ngt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9b,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_ngt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9b,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_ngt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9b,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 null, exec_lo
+// GFX11: v_cmpx_ngt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9b,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_ngt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9b,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 0.5, -m0
+// GFX11: v_cmpx_ngt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9b,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ngt_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_ngt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9b,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ngt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_ngt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_ngt_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xab,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_ngt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xab,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ngt_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_ngt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xab,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_ngt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xab,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_ngt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xab,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_ngt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xab,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_ngt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xab,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_ngt_f64_e64 null, 0.5
+// GFX11: v_cmpx_ngt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xab,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f64_e64 -1, -1
+// GFX11: v_cmpx_ngt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xab,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ngt_f64_e64 0.5, null
+// GFX11: v_cmpx_ngt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xab,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xab,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f16_e64 v1, v2
+// GFX11: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f16_e64 v255, v255
+// GFX11: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nle_f16_e64 s1, s2
+// GFX11: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nle_f16_e64 s105, s105
+// GFX11: v_cmpx_nle_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nle_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nle_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nle_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nle_f16_e64 m0, 0.5
+// GFX11: v_cmpx_nle_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nle_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_nle_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nle_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_nle_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nle_f16_e64 null, exec_lo
+// GFX11: v_cmpx_nle_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nle_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_nle_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nle_f16_e64 0.5, -m0
+// GFX11: v_cmpx_nle_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32_e64 v1, v2
+// GFX11: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f32_e64 v255, v255
+// GFX11: v_cmpx_nle_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9c,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nle_f32_e64 s1, s2
+// GFX11: v_cmpx_nle_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nle_f32_e64 s105, s105
+// GFX11: v_cmpx_nle_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9c,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nle_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nle_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9c,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nle_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_nle_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nle_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9c,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nle_f32_e64 m0, 0.5
+// GFX11: v_cmpx_nle_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9c,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nle_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_nle_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9c,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nle_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_nle_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9c,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nle_f32_e64 null, exec_lo
+// GFX11: v_cmpx_nle_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9c,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nle_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_nle_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9c,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32_e64 0.5, -m0
+// GFX11: v_cmpx_nle_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9c,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nle_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nle_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9c,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nle_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nle_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_nle_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xac,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_nle_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xac,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nle_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_nle_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xac,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nle_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_nle_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xac,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nle_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_nle_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xac,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nle_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_nle_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xac,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_nle_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xac,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nle_f64_e64 null, 0.5
+// GFX11: v_cmpx_nle_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xac,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nle_f64_e64 -1, -1
+// GFX11: v_cmpx_nle_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xac,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nle_f64_e64 0.5, null
+// GFX11: v_cmpx_nle_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xac,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nle_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_nle_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xac,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f16_e64 v1, v2
+// GFX11: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16_e64 v255, v255
+// GFX11: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlg_f16_e64 s1, s2
+// GFX11: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 s105, s105
+// GFX11: v_cmpx_nlg_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nlg_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nlg_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 m0, 0.5
+// GFX11: v_cmpx_nlg_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_nlg_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_nlg_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 null, exec_lo
+// GFX11: v_cmpx_nlg_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_nlg_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 0.5, -m0
+// GFX11: v_cmpx_nlg_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 v1, v2
+// GFX11: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f32_e64 v255, v255
+// GFX11: v_cmpx_nlg_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9a,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlg_f32_e64 s1, s2
+// GFX11: v_cmpx_nlg_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 s105, s105
+// GFX11: v_cmpx_nlg_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9a,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nlg_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9a,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_nlg_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nlg_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9a,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 m0, 0.5
+// GFX11: v_cmpx_nlg_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9a,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_nlg_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9a,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_nlg_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9a,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 null, exec_lo
+// GFX11: v_cmpx_nlg_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9a,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_nlg_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9a,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 0.5, -m0
+// GFX11: v_cmpx_nlg_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9a,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlg_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nlg_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9a,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nlg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_nlg_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_nlg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xaa,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nlg_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_nlg_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_nlg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xaa,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_nlg_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xaa,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_nlg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xaa,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_nlg_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xaa,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nlg_f64_e64 null, 0.5
+// GFX11: v_cmpx_nlg_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xaa,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f64_e64 -1, -1
+// GFX11: v_cmpx_nlg_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xaa,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nlg_f64_e64 0.5, null
+// GFX11: v_cmpx_nlg_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xaa,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xaa,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f16_e64 v1, v2
+// GFX11: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f16_e64 v255, v255
+// GFX11: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlt_f16_e64 s1, s2
+// GFX11: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 s105, s105
+// GFX11: v_cmpx_nlt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nlt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nlt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 m0, 0.5
+// GFX11: v_cmpx_nlt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_nlt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_nlt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 null, exec_lo
+// GFX11: v_cmpx_nlt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_nlt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 0.5, -m0
+// GFX11: v_cmpx_nlt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 v1, v2
+// GFX11: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f32_e64 v255, v255
+// GFX11: v_cmpx_nlt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9e,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlt_f32_e64 s1, s2
+// GFX11: v_cmpx_nlt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 s105, s105
+// GFX11: v_cmpx_nlt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9e,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_nlt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9e,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_nlt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_nlt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9e,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 m0, 0.5
+// GFX11: v_cmpx_nlt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9e,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_nlt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9e,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_nlt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9e,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 null, exec_lo
+// GFX11: v_cmpx_nlt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9e,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_nlt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9e,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 0.5, -m0
+// GFX11: v_cmpx_nlt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9e,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlt_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_nlt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9e,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_nlt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_nlt_f64_e64 v[1:2], v[2:3]       ; encoding: [0x7e,0x00,0xae,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_nlt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xae,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nlt_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_nlt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xae,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_nlt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xae,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_nlt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xae,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_nlt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xae,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_nlt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xae,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nlt_f64_e64 null, 0.5
+// GFX11: v_cmpx_nlt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xae,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f64_e64 -1, -1
+// GFX11: v_cmpx_nlt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xae,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nlt_f64_e64 0.5, null
+// GFX11: v_cmpx_nlt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xae,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xae,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f16_e64 v1, v2
+// GFX11: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f16_e64 v255, v255
+// GFX11: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_o_f16_e64 s1, s2
+// GFX11: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_o_f16_e64 s105, s105
+// GFX11: v_cmpx_o_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_o_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_o_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_o_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_o_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_o_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_o_f16_e64 m0, 0.5
+// GFX11: v_cmpx_o_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_o_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_o_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_o_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_o_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_o_f16_e64 null, exec_lo
+// GFX11: v_cmpx_o_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_o_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_o_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_o_f16_e64 0.5, -m0
+// GFX11: v_cmpx_o_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_o_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_o_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f32_e64 v1, v2
+// GFX11: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f32_e64 v255, v255
+// GFX11: v_cmpx_o_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x97,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_o_f32_e64 s1, s2
+// GFX11: v_cmpx_o_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_o_f32_e64 s105, s105
+// GFX11: v_cmpx_o_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x97,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_o_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_o_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x97,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_o_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_o_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x97,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_o_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x97,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_o_f32_e64 m0, 0.5
+// GFX11: v_cmpx_o_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x97,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_o_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_o_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x97,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_o_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_o_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x97,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_o_f32_e64 null, exec_lo
+// GFX11: v_cmpx_o_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x97,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_o_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_o_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x97,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_o_f32_e64 0.5, -m0
+// GFX11: v_cmpx_o_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x97,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_o_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_o_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x97,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_o_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_o_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x97,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_o_f64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_o_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa7,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_o_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_o_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_o_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_o_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa7,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_o_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_o_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa7,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_o_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_o_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa7,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_o_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa7,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_o_f64_e64 null, 0.5
+// GFX11: v_cmpx_o_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa7,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_o_f64_e64 -1, -1
+// GFX11: v_cmpx_o_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa7,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_o_f64_e64 0.5, null
+// GFX11: v_cmpx_o_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa7,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_o_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_o_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa7,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f16_e64 v1, v2
+// GFX11: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_f16_e64 v255, v255
+// GFX11: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_t_f16_e64 s1, s2
+// GFX11: v_cmpx_t_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_t_f16_e64 s105, s105
+// GFX11: v_cmpx_t_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x8f,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_t_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x8f,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_t_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_t_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x8f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_t_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x8f,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_t_f16_e64 m0, 0.5
+// GFX11: v_cmpx_t_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x8f,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_t_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_t_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x8f,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_t_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_t_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x8f,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_t_f16_e64 null, exec_lo
+// GFX11: v_cmpx_t_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x8f,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_t_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_t_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x8f,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_t_f16_e64 0.5, -m0
+// GFX11: v_cmpx_t_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x8f,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_t_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_t_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x8f,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_t_f32_e64 v1, v2
+// GFX11: v_cmpx_t_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_f32_e64 v255, v255
+// GFX11: v_cmpx_t_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x9f,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_t_f32_e64 s1, s2
+// GFX11: v_cmpx_t_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_t_f32_e64 s105, s105
+// GFX11: v_cmpx_t_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x9f,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_t_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x9f,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_t_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_t_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x9f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x9f,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_t_f32_e64 m0, 0.5
+// GFX11: v_cmpx_t_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x9f,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_t_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_t_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x9f,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_t_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_t_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x9f,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_t_f32_e64 null, exec_lo
+// GFX11: v_cmpx_t_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x9f,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_t_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_t_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x9f,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_t_f32_e64 0.5, -m0
+// GFX11: v_cmpx_t_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x9f,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_t_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_t_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x9f,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_t_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_t_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9f,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_f64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xaf,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_t_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xaf,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_t_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_t_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xaf,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_t_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_t_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xaf,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_t_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_t_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xaf,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_t_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_t_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xaf,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_t_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xaf,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_t_f64_e64 null, 0.5
+// GFX11: v_cmpx_t_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xaf,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_t_f64_e64 -1, -1
+// GFX11: v_cmpx_t_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xaf,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_t_f64_e64 0.5, null
+// GFX11: v_cmpx_t_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xaf,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_t_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_t_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xaf,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_t_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_t_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaf,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i32_e64 v1, v2
+// GFX11: v_cmpx_t_i32_e64 v1, v2                 ; encoding: [0x7e,0x00,0xc7,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_i32_e64 v255, v255
+// GFX11: v_cmpx_t_i32_e64 v255, v255             ; encoding: [0x7e,0x00,0xc7,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_t_i32_e64 s1, s2
+// GFX11: v_cmpx_t_i32_e64 s1, s2                 ; encoding: [0x7e,0x00,0xc7,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_t_i32_e64 s105, s105
+// GFX11: v_cmpx_t_i32_e64 s105, s105             ; encoding: [0x7e,0x00,0xc7,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_t_i32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_i32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0xc7,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_t_i32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_t_i32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0xc7,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_i32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0xc7,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_t_i32_e64 m0, 0.5
+// GFX11: v_cmpx_t_i32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0xc7,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_t_i32_e64 exec_lo, -1
+// GFX11: v_cmpx_t_i32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0xc7,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_t_i32_e64 exec_hi, null
+// GFX11: v_cmpx_t_i32_e64 exec_hi, null          ; encoding: [0x7e,0x00,0xc7,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_t_i32_e64 null, exec_lo
+// GFX11: v_cmpx_t_i32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0xc7,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_t_i32_e64 -1, exec_hi
+// GFX11: v_cmpx_t_i32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0xc7,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_t_i32_e64 0.5, m0
+// GFX11: v_cmpx_t_i32_e64 0.5, m0                ; encoding: [0x7e,0x00,0xc7,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_t_i32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_t_i32_e64 src_scc, vcc_lo        ; encoding: [0x7e,0x00,0xc7,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_t_i32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_t_i32_e64 0xaf123456, vcc_hi     ; encoding: [0x7e,0x00,0xc7,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_i64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xd7,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_i64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_t_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd7,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_t_i64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_t_i64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xd7,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_t_i64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_t_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd7,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_t_i64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_t_i64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xd7,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_t_i64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_t_i64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xd7,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_i64_e64 exec, src_scc
+// GFX11: v_cmpx_t_i64_e64 exec, src_scc          ; encoding: [0x7e,0x00,0xd7,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_t_i64_e64 null, 0.5
+// GFX11: v_cmpx_t_i64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xd7,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_t_i64_e64 -1, -1
+// GFX11: v_cmpx_t_i64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xd7,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_t_i64_e64 0.5, null
+// GFX11: v_cmpx_t_i64_e64 0.5, null              ; encoding: [0x7e,0x00,0xd7,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_t_i64_e64 src_scc, exec
+// GFX11: v_cmpx_t_i64_e64 src_scc, exec          ; encoding: [0x7e,0x00,0xd7,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_t_i64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_t_i64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xd7,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u32_e64 v1, v2
+// GFX11: v_cmpx_t_u32_e64 v1, v2                 ; encoding: [0x7e,0x00,0xcf,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_u32_e64 v255, v255
+// GFX11: v_cmpx_t_u32_e64 v255, v255             ; encoding: [0x7e,0x00,0xcf,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_t_u32_e64 s1, s2
+// GFX11: v_cmpx_t_u32_e64 s1, s2                 ; encoding: [0x7e,0x00,0xcf,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_t_u32_e64 s105, s105
+// GFX11: v_cmpx_t_u32_e64 s105, s105             ; encoding: [0x7e,0x00,0xcf,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_t_u32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_u32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0xcf,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_t_u32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_t_u32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0xcf,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_u32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0xcf,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_t_u32_e64 m0, 0.5
+// GFX11: v_cmpx_t_u32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0xcf,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_t_u32_e64 exec_lo, -1
+// GFX11: v_cmpx_t_u32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0xcf,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_t_u32_e64 exec_hi, null
+// GFX11: v_cmpx_t_u32_e64 exec_hi, null          ; encoding: [0x7e,0x00,0xcf,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_t_u32_e64 null, exec_lo
+// GFX11: v_cmpx_t_u32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0xcf,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_t_u32_e64 -1, exec_hi
+// GFX11: v_cmpx_t_u32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0xcf,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_t_u32_e64 0.5, m0
+// GFX11: v_cmpx_t_u32_e64 0.5, m0                ; encoding: [0x7e,0x00,0xcf,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_t_u32_e64 src_scc, vcc_lo
+// GFX11: v_cmpx_t_u32_e64 src_scc, vcc_lo        ; encoding: [0x7e,0x00,0xcf,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_t_u32_e64 0xaf123456, vcc_hi
+// GFX11: v_cmpx_t_u32_e64 0xaf123456, vcc_hi     ; encoding: [0x7e,0x00,0xcf,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_u64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xdf,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_t_u64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_t_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdf,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_t_u64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_t_u64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xdf,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_t_u64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_t_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdf,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_t_u64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_t_u64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xdf,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_t_u64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_t_u64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xdf,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_t_u64_e64 exec, src_scc
+// GFX11: v_cmpx_t_u64_e64 exec, src_scc          ; encoding: [0x7e,0x00,0xdf,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_t_u64_e64 null, 0.5
+// GFX11: v_cmpx_t_u64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xdf,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_t_u64_e64 -1, -1
+// GFX11: v_cmpx_t_u64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xdf,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_t_u64_e64 0.5, null
+// GFX11: v_cmpx_t_u64_e64 0.5, null              ; encoding: [0x7e,0x00,0xdf,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_t_u64_e64 src_scc, exec
+// GFX11: v_cmpx_t_u64_e64 src_scc, exec          ; encoding: [0x7e,0x00,0xdf,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_t_u64_e64 0xaf123456, vcc
+// GFX11: v_cmpx_t_u64_e64 0xaf123456, vcc        ; encoding: [0x7e,0x00,0xdf,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f16_e64 v1, v2
+// GFX11: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_tru_f16_e64 v255, v255
+// GFX11: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_tru_f16_e64 s1, s2
+// GFX11: v_cmpx_t_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_tru_f16_e64 s105, s105
+// GFX11: v_cmpx_t_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x8f,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_tru_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x8f,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_tru_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_t_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x8f,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_tru_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x8f,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_tru_f16_e64 m0, 0.5
+// GFX11: v_cmpx_t_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x8f,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_tru_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_t_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x8f,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_tru_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_t_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x8f,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_tru_f16_e64 null, exec_lo
+// GFX11: v_cmpx_t_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x8f,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_tru_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_t_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x8f,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_tru_f16_e64 0.5, -m0
+// GFX11: v_cmpx_t_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x8f,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_tru_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_t_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x8f,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_tru_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_tru_f32_e64 v1, v2
+// GFX11: v_cmpx_t_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_tru_f32_e64 v255, v255
+// GFX11: v_cmpx_t_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x9f,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_tru_f32_e64 s1, s2
+// GFX11: v_cmpx_t_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_tru_f32_e64 s105, s105
+// GFX11: v_cmpx_t_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x9f,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_tru_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_t_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x9f,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_tru_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_t_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x9f,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_t_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x9f,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_tru_f32_e64 m0, 0.5
+// GFX11: v_cmpx_t_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x9f,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_tru_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_t_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x9f,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_tru_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_t_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x9f,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_tru_f32_e64 null, exec_lo
+// GFX11: v_cmpx_t_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x9f,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_tru_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_t_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x9f,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_tru_f32_e64 0.5, -m0
+// GFX11: v_cmpx_t_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x9f,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_tru_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_t_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x9f,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_tru_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_t_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9f,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_f64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xaf,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_tru_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_t_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xaf,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_tru_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_t_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xaf,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_tru_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_t_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xaf,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_tru_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_t_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xaf,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_tru_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_t_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xaf,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_tru_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_t_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xaf,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_tru_f64_e64 null, 0.5
+// GFX11: v_cmpx_t_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xaf,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_tru_f64_e64 -1, -1
+// GFX11: v_cmpx_t_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xaf,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_tru_f64_e64 0.5, null
+// GFX11: v_cmpx_t_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xaf,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_tru_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_t_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xaf,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_tru_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_t_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaf,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f16_e64 v1, v2
+// GFX11: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f16_e64 v255, v255
+// GFX11: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_u_f16_e64 s1, s2
+// GFX11: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_u_f16_e64 s105, s105
+// GFX11: v_cmpx_u_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_u_f16_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_u_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_u_f16_e64 vcc_hi, 0xfe0b
+// GFX11: v_cmpx_u_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f16_e64 ttmp15, src_scc
+// GFX11: v_cmpx_u_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_u_f16_e64 m0, 0.5
+// GFX11: v_cmpx_u_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_u_f16_e64 exec_lo, -1
+// GFX11: v_cmpx_u_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_u_f16_e64 |exec_hi|, null
+// GFX11: v_cmpx_u_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_u_f16_e64 null, exec_lo
+// GFX11: v_cmpx_u_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_u_f16_e64 -1, exec_hi
+// GFX11: v_cmpx_u_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_u_f16_e64 0.5, -m0
+// GFX11: v_cmpx_u_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_u_f16_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_u_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX11: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f32_e64 v1, v2
+// GFX11: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f32_e64 v255, v255
+// GFX11: v_cmpx_u_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x98,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_u_f32_e64 s1, s2
+// GFX11: v_cmpx_u_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_u_f32_e64 s105, s105
+// GFX11: v_cmpx_u_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x98,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_u_f32_e64 vcc_lo, ttmp15
+// GFX11: v_cmpx_u_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x98,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_u_f32_e64 vcc_hi, 0xaf123456
+// GFX11: v_cmpx_u_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x98,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f32_e64 ttmp15, src_scc
+// GFX11: v_cmpx_u_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x98,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_u_f32_e64 m0, 0.5
+// GFX11: v_cmpx_u_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x98,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_u_f32_e64 exec_lo, -1
+// GFX11: v_cmpx_u_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x98,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_u_f32_e64 |exec_hi|, null
+// GFX11: v_cmpx_u_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x98,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_u_f32_e64 null, exec_lo
+// GFX11: v_cmpx_u_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x98,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_u_f32_e64 -1, exec_hi
+// GFX11: v_cmpx_u_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x98,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_u_f32_e64 0.5, -m0
+// GFX11: v_cmpx_u_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x98,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_u_f32_e64 -src_scc, |vcc_lo|
+// GFX11: v_cmpx_u_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x98,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_u_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX11: v_cmpx_u_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x98,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64_e64 v[1:2], v[2:3]
+// GFX11: v_cmpx_u_f64_e64 v[1:2], v[2:3]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f64_e64 v[254:255], v[254:255]
+// GFX11: v_cmpx_u_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa8,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_u_f64_e64 s[2:3], s[4:5]
+// GFX11: v_cmpx_u_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_u_f64_e64 s[104:105], s[104:105]
+// GFX11: v_cmpx_u_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa8,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_u_f64_e64 vcc, ttmp[14:15]
+// GFX11: v_cmpx_u_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa8,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_u_f64_e64 ttmp[14:15], 0xaf123456
+// GFX11: v_cmpx_u_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa8,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64_e64 -|exec|, src_scc
+// GFX11: v_cmpx_u_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa8,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_u_f64_e64 null, 0.5
+// GFX11: v_cmpx_u_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa8,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_u_f64_e64 -1, -1
+// GFX11: v_cmpx_u_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa8,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_u_f64_e64 0.5, null
+// GFX11: v_cmpx_u_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa8,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_u_f64_e64 -|src_scc|, -|exec|
+// GFX11: v_cmpx_u_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa8,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_u_f64_e64 0xaf123456, -|vcc| clamp
+// GFX11: v_cmpx_u_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa8,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
index 55a4fabb4e236..371d29f2a2cb6 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
@@ -1,6 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11 %s
 
 v_cmpx_class_f16_e64 v1, v2
 // GFX11: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3p_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3p_alias.s
new file mode 100644
index 0000000000000..d2fdf013939fb
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3p_alias.s
@@ -0,0 +1,7 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+
+v_dot4_i32_i8 v5, v1, v2, s3
+// GFX11: v_dot4_i32_iu8 v5, v1, v2, s3           ; encoding: [0x05,0x40,0x16,0xcc,0x01,0x05,0x0e,0x18]
+
+v_dot8_i32_i4 v5, v1, v2, s3
+// GFX11: v_dot8_i32_iu4 v5, v1, v2, s3           ; encoding: [0x05,0x40,0x18,0xcc,0x01,0x05,0x0e,0x18]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_alias.s
new file mode 100644
index 0000000000000..2b95e6a948746
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc_alias.s
@@ -0,0 +1,14 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+
+v_cmp_tru_i32 vcc, v1, v2
+// GFX11: v_cmp_t_i32_e32 vcc, v1, v2             ; encoding: [0x01,0x05,0x8e,0x7c]
+
+v_cmp_tru_u32 vcc, v1, v2
+// GFX11: v_cmp_t_u32_e32 vcc, v1, v2             ; encoding: [0x01,0x05,0x9e,0x7c]
+
+v_cmp_tru_i64 vcc, v[1:2], v[2:3]
+// GFX11: v_cmp_t_i64_e32 vcc, v[1:2], v[2:3]     ; encoding: [0x01,0x05,0xae,0x7c]
+
+v_cmp_tru_u64 vcc, v[1:2], v[2:3]
+// GFX11: v_cmp_t_u64_e32 vcc, v[1:2], v[2:3]     ; encoding: [0x01,0x05,0xbe,0x7c]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_alias.s
new file mode 100644
index 0000000000000..636ec3c19d70f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_alias.s
@@ -0,0 +1,14 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+
+v_cmpx_tru_i32 v1, v2
+// GFX11: v_cmpx_t_i32_e32 v1, v2                 ; encoding: [0x01,0x05,0x8e,0x7d]
+
+v_cmpx_tru_u32 v1, v2
+// GFX11: v_cmpx_t_u32_e32 v1, v2                 ; encoding: [0x01,0x05,0x9e,0x7d]
+
+v_cmpx_tru_i64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_i64_e32 v[1:2], v[2:3]         ; encoding: [0x01,0x05,0xae,0x7d]
+
+v_cmpx_tru_u64 v[1:2], v[2:3]
+// GFX11: v_cmpx_t_u64_e32 v[1:2], v[2:3]         ; encoding: [0x01,0x05,0xbe,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopd.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopd.s
index 4b8ed488793db..db05a9f6f1150 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopd.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopd.s
@@ -14817,3 +14817,219 @@ v_dual_subrev_f32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5
 v_dual_subrev_f32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4
 // GFX11: encoding: [0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_bf16 v6, v1, v3
+// GFX11: encoding: [0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_bf16 v6, v255, v3
+// GFX11: encoding: [0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_bf16 v6, v2, v3
+// GFX11: encoding: [0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_bf16 v6, v3, v3
+// GFX11: encoding: [0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_bf16 v6, v4, v3
+// GFX11: encoding: [0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_bf16 v6, s1, v3
+// GFX11: encoding: [0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_bf16 v6, s105, v3
+// GFX11: encoding: [0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_lo, v3
+// GFX11: encoding: [0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_hi, v3
+// GFX11: encoding: [0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, ttmp15, v3
+// GFX11: encoding: [0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_bf16 v6, m0, v3
+// GFX11: encoding: [0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_lo, v3
+// GFX11: encoding: [0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_hi, v3
+// GFX11: encoding: [0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_bf16 v6, null, v3
+// GFX11: encoding: [0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_bf16 v6, -1, v3
+// GFX11: encoding: [0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_bf16 v6, 0.5, v2
+// GFX11: encoding: [0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_bf16 v6, src_scc, v5
+// GFX11: encoding: [0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v4
+// GFX11: encoding: [0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3
+// GFX11: encoding: [0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3
+// GFX11: encoding: [0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3
+// GFX11: encoding: [0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3
+// GFX11: encoding: [0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3
+// GFX11: encoding: [0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3
+// GFX11: encoding: [0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3
+// GFX11: encoding: [0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3
+// GFX11: encoding: [0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3
+// GFX11: encoding: [0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3
+// GFX11: encoding: [0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3
+// GFX11: encoding: [0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3
+// GFX11: encoding: [0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3
+// GFX11: encoding: [0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3
+// GFX11: encoding: [0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3
+// GFX11: encoding: [0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2
+// GFX11: encoding: [0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5
+// GFX11: encoding: [0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4
+// GFX11: encoding: [0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_nc_u32 v6, v1, v3
+// GFX11: encoding: [0x04,0x05,0x60,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_nc_u32 v6, v255, v3
+// GFX11: encoding: [0x01,0x05,0x60,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_nc_u32 v6, v2, v3
+// GFX11: encoding: [0xff,0x05,0x60,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_nc_u32 v6, v3, v3
+// GFX11: encoding: [0x02,0x05,0x60,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_nc_u32 v6, v4, v3
+// GFX11: encoding: [0x03,0x05,0x60,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_nc_u32 v6, s1, v3
+// GFX11: encoding: [0x69,0x04,0x60,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_nc_u32 v6, s105, v3
+// GFX11: encoding: [0x01,0x04,0x60,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_nc_u32 v6, vcc_lo, v3
+// GFX11: encoding: [0x7b,0x04,0x60,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_nc_u32 v6, vcc_hi, v3
+// GFX11: encoding: [0x7f,0x04,0x60,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_nc_u32 v6, ttmp15, v3
+// GFX11: encoding: [0x7e,0x04,0x60,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_nc_u32 v6, m0, v3
+// GFX11: encoding: [0x7d,0x04,0x60,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_nc_u32 v6, exec_lo, v3
+// GFX11: encoding: [0x6b,0x04,0x60,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_nc_u32 v6, exec_hi, v3
+// GFX11: encoding: [0x6a,0x04,0x60,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_nc_u32 v6, null, v3
+// GFX11: encoding: [0xff,0x04,0x60,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_nc_u32 v6, -1, v3
+// GFX11: encoding: [0xfd,0x04,0x60,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_nc_u32 v6, 0.5, v2
+// GFX11: encoding: [0xf0,0x06,0x60,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_nc_u32 v6, src_scc, v5
+// GFX11: encoding: [0xc1,0x08,0x60,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_nc_u32 v255, 0xaf123456, v4
+// GFX11: encoding: [0x7c,0x0a,0x60,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
diff --git a/llvm/test/MC/AMDGPU/gfx11_unsupported.s b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
index c565801d275bb..640ffb65a76bf 100644
--- a/llvm/test/MC/AMDGPU/gfx11_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
@@ -2064,3 +2064,6 @@ ds_subrev_u64 v1, v[2:3]
 
 ds_subrev_rtn_u64 v[5:6], v1, v[2:3]
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2c_f32_bf16 v5, v1, v2
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir_alias.s
new file mode 100644
index 0000000000000..0d9993afa43e7
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir_alias.s
@@ -0,0 +1,8 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s
+
+lds_direct_load v0
+// GFX12: ds_direct_load v0 wait_va_vdst:0 wait_vm_vsrc:0 ; encoding: [0x00,0x00,0x10,0xce]
+
+lds_param_load v0, attr0.x
+// GFX12: ds_param_load v0, attr0.x wait_va_vdst:0 wait_vm_vsrc:0 ; encoding: [0x00,0x00,0x00,0xce]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s
index fceab4b7830f9..b9999b671f7e7 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s
@@ -35,3 +35,9 @@ image_atomic_fmin v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
 
 image_atomic_fmax v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
 // GFX12: image_atomic_max_flt v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x40,0x61,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+bvh_intersect_ray v[4:7], [v9, v10, v[11:13], v[14:16], v[17:19]], s[4:7]
+// GFX12: image_bvh_intersect_ray v[4:7], [v9, v10, v[11:13], v[14:16], v[17:19]], s[4:7] ; encoding: [0x10,0x40,0xc6,0xd3,0x04,0x08,0x00,0x11,0x09,0x0a,0x0b,0x0e]
+
+bvh64_intersect_ray v[4:7], [v[9:10], v11, v[12:14], v[15:17], v[18:20]], s[4:7]
+// GFX12: image_bvh64_intersect_ray v[4:7], [v[9:10], v11, v[12:14], v[15:17], v[18:20]], s[4:7] ; encoding: [0x10,0x80,0xc6,0xd3,0x04,0x08,0x00,0x12,0x09,0x0b,0x0c,0x0f]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
index 0becad77b64dd..6d07c41de6418 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s
@@ -124,50 +124,62 @@ v_add_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
 // W64: v_add_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2
-// GFX12: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+v_add_f16 v5.l, v1.l, v2.l
+// GFX12: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
 
-v_add_f16 v5, v127, v2
-// GFX12: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
+v_add_f16 v5.l, v127.l, v2.l
+// GFX12: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
 
-v_add_f16 v5, s1, v2
-// GFX12: v_add_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x64]
+v_add_f16 v5.l, s1, v2.l
+// GFX12: v_add_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x64]
 
-v_add_f16 v5, s105, v2
-// GFX12: v_add_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x64]
+v_add_f16 v5.l, s105, v2.l
+// GFX12: v_add_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x64]
 
-v_add_f16 v5, vcc_lo, v2
-// GFX12: v_add_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x64]
+v_add_f16 v5.l, vcc_lo, v2.l
+// GFX12: v_add_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x64]
 
-v_add_f16 v5, vcc_hi, v2
-// GFX12: v_add_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x64]
+v_add_f16 v5.l, vcc_hi, v2.l
+// GFX12: v_add_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x64]
 
-v_add_f16 v5, ttmp15, v2
-// GFX12: v_add_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x64]
+v_add_f16 v5.l, ttmp15, v2.l
+// GFX12: v_add_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x64]
 
-v_add_f16 v5, m0, v2
-// GFX12: v_add_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x64]
+v_add_f16 v5.l, m0, v2.l
+// GFX12: v_add_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x64]
 
-v_add_f16 v5, exec_lo, v2
-// GFX12: v_add_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x64]
+v_add_f16 v5.l, exec_lo, v2.l
+// GFX12: v_add_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x64]
 
-v_add_f16 v5, exec_hi, v2
-// GFX12: v_add_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x64]
+v_add_f16 v5.l, exec_hi, v2.l
+// GFX12: v_add_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x64]
 
-v_add_f16 v5, null, v2
-// GFX12: v_add_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x64]
+v_add_f16 v5.l, null, v2.l
+// GFX12: v_add_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x64]
 
-v_add_f16 v5, -1, v2
-// GFX12: v_add_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x64]
+v_add_f16 v5.l, -1, v2.l
+// GFX12: v_add_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x64]
 
-v_add_f16 v5, 0.5, v2
-// GFX12: v_add_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x64]
+v_add_f16 v5.l, 0.5, v2.l
+// GFX12: v_add_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x64]
 
-v_add_f16 v5, src_scc, v2
-// GFX12: v_add_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x64]
+v_add_f16 v5.l, src_scc, v2.l
+// GFX12: v_add_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x64]
 
-v_add_f16 v127, 0xfe0b, v127
-// GFX12: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+v_add_f16 v127.l, 0xfe0b, v127.l
+// GFX12: v_add_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
+
+v_add_f16 v5.l, v1.h, v2.l
+// GFX12: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
+
+v_add_f16 v5.l, v127.h, v2.l
+// GFX12: v_add_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x64]
+
+v_add_f16 v5.h, src_scc, v2.h
+// GFX12: v_add_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x65]
+
+v_add_f16 v127.h, 0xfe0b, v127.h
+// GFX12: v_add_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00]
 
 v_add_f32 v5, v1, v2
 // GFX12: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
@@ -595,50 +607,62 @@ v_cvt_pkrtz_f16_f32 v5, src_scc, v2
 v_cvt_pkrtz_f16_f32 v255, 0xaf123456, v255
 // GFX12: v_cvt_pk_rtz_f16_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x5f,0x56,0x34,0x12,0xaf]
 
-v_fmaak_f16 v5, v1, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, v1, v2, 0xfe0b          ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, v1.l, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, v1.l, v2.l, 0xfe0b    ; encoding: [0x01,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5.l, v127.l, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, v127.l, v2.l, 0xfe0b  ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, v127, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, v127, v2, 0xfe0b        ; encoding: [0x7f,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, s1, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, s1, v2.l, 0xfe0b      ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, s1, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, s1, v2, 0xfe0b          ; encoding: [0x01,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, s105, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, s105, v2.l, 0xfe0b    ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, s105, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, s105, v2, 0xfe0b        ; encoding: [0x69,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, vcc_lo, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, vcc_lo, v2.l, 0xfe0b  ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, vcc_lo, v2, 0xfe0b      ; encoding: [0x6a,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, vcc_hi, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, vcc_hi, v2.l, 0xfe0b  ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, vcc_hi, v2, 0xfe0b      ; encoding: [0x6b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, ttmp15, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, ttmp15, v2.l, 0xfe0b  ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, ttmp15, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, ttmp15, v2, 0xfe0b      ; encoding: [0x7b,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, m0, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, m0, v2.l, 0xfe0b      ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, m0, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, m0, v2, 0xfe0b          ; encoding: [0x7d,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, exec_lo, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, exec_lo, v2.l, 0xfe0b ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, exec_lo, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, exec_lo, v2, 0xfe0b     ; encoding: [0x7e,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, exec_hi, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, exec_hi, v2.l, 0xfe0b ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, exec_hi, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, exec_hi, v2, 0xfe0b     ; encoding: [0x7f,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, null, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, null, v2.l, 0xfe0b    ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, null, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, null, v2, 0xfe0b        ; encoding: [0x7c,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, -1, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, -1, v2.l, 0xfe0b      ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, -1, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, -1, v2, 0xfe0b          ; encoding: [0xc1,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, 0.5, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, 0.5, v2.l, 0xfe0b     ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, 0.5, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, 0.5, v2, 0xfe0b         ; encoding: [0xf0,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, src_scc, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, src_scc, v2.l, 0xfe0b ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v5, src_scc, v2, 0xfe0b
-// GFX12: v_fmaak_f16 v5, src_scc, v2, 0xfe0b     ; encoding: [0xfd,0x04,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v127.l, 0xfe0b, v127.l, 0xfe0b
+// GFX12: v_fmaak_f16 v127.l, 0xfe0b, v127.l, 0xfe0b ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
 
-v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b
-// GFX12: v_fmaak_f16 v127, 0xfe0b, v127, 0xfe0b  ; encoding: [0xff,0xfe,0xfe,0x70,0x0b,0xfe,0x00,0x00]
+v_fmaak_f16 v5.l, v1.h, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, v1.h, v2.l, 0xfe0b    ; encoding: [0x81,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5.l, v127.h, v2.l, 0xfe0b
+// GFX12: v_fmaak_f16 v5.l, v127.h, v2.l, 0xfe0b  ; encoding: [0xff,0x05,0x0a,0x70,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v5.h, src_scc, v2.h, 0xfe0b
+// GFX12: v_fmaak_f16 v5.h, src_scc, v2.h, 0xfe0b ; encoding: [0xfd,0x04,0x0b,0x71,0x0b,0xfe,0x00,0x00]
+
+v_fmaak_f16 v127.h, 0xfe0b, v127.h, 0xfe0b
+// GFX12: v_fmaak_f16 v127.h, 0xfe0b, v127.h, 0xfe0b ; encoding: [0xff,0xfe,0xff,0x71,0x0b,0xfe,0x00,0x00]
 
 v_fmaak_f32 v5, v1, v2, 0xaf123456
 // GFX12: v_fmaak_f32 v5, v1, v2, 0xaf123456      ; encoding: [0x01,0x05,0x0a,0x5a,0x56,0x34,0x12,0xaf]
@@ -685,50 +709,62 @@ v_fmaak_f32 v5, src_scc, v2, 0xaf123456
 v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456
 // GFX12: v_fmaak_f32 v255, 0xaf123456, v255, 0xaf123456 ; encoding: [0xff,0xfe,0xff,0x5b,0x56,0x34,0x12,0xaf]
 
-v_fmac_f16 v5, v1, v2
-// GFX12: v_fmac_f16_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x6c]
+v_fmac_f16 v5.l, v1.l, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, v1.l, v2.l         ; encoding: [0x01,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5.l, v127.l, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, v127.l, v2.l       ; encoding: [0x7f,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5.l, s1, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, s1, v2.l           ; encoding: [0x01,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, v127, v2
-// GFX12: v_fmac_f16_e32 v5, v127, v2             ; encoding: [0x7f,0x05,0x0a,0x6c]
+v_fmac_f16 v5.l, s105, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, s105, v2.l         ; encoding: [0x69,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, s1, v2
-// GFX12: v_fmac_f16_e32 v5, s1, v2               ; encoding: [0x01,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, vcc_lo, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, vcc_lo, v2.l       ; encoding: [0x6a,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, s105, v2
-// GFX12: v_fmac_f16_e32 v5, s105, v2             ; encoding: [0x69,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, vcc_hi, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, vcc_hi, v2.l       ; encoding: [0x6b,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, vcc_lo, v2
-// GFX12: v_fmac_f16_e32 v5, vcc_lo, v2           ; encoding: [0x6a,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, ttmp15, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, ttmp15, v2.l       ; encoding: [0x7b,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, vcc_hi, v2
-// GFX12: v_fmac_f16_e32 v5, vcc_hi, v2           ; encoding: [0x6b,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, m0, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, m0, v2.l           ; encoding: [0x7d,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, ttmp15, v2
-// GFX12: v_fmac_f16_e32 v5, ttmp15, v2           ; encoding: [0x7b,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, exec_lo, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, exec_lo, v2.l      ; encoding: [0x7e,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, m0, v2
-// GFX12: v_fmac_f16_e32 v5, m0, v2               ; encoding: [0x7d,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, exec_hi, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, exec_hi, v2.l      ; encoding: [0x7f,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, exec_lo, v2
-// GFX12: v_fmac_f16_e32 v5, exec_lo, v2          ; encoding: [0x7e,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, null, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, null, v2.l         ; encoding: [0x7c,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, exec_hi, v2
-// GFX12: v_fmac_f16_e32 v5, exec_hi, v2          ; encoding: [0x7f,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, -1, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, -1, v2.l           ; encoding: [0xc1,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, null, v2
-// GFX12: v_fmac_f16_e32 v5, null, v2             ; encoding: [0x7c,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, 0.5, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, 0.5, v2.l          ; encoding: [0xf0,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, -1, v2
-// GFX12: v_fmac_f16_e32 v5, -1, v2               ; encoding: [0xc1,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, src_scc, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, src_scc, v2.l      ; encoding: [0xfd,0x04,0x0a,0x6c]
 
-v_fmac_f16 v5, 0.5, v2
-// GFX12: v_fmac_f16_e32 v5, 0.5, v2              ; encoding: [0xf0,0x04,0x0a,0x6c]
+v_fmac_f16 v127.l, 0xfe0b, v127.l
+// GFX12: v_fmac_f16_e32 v127.l, 0xfe0b, v127.l   ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
 
-v_fmac_f16 v5, src_scc, v2
-// GFX12: v_fmac_f16_e32 v5, src_scc, v2          ; encoding: [0xfd,0x04,0x0a,0x6c]
+v_fmac_f16 v5.l, v1.h, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, v1.h, v2.l         ; encoding: [0x81,0x05,0x0a,0x6c]
 
-v_fmac_f16 v127, 0xfe0b, v127
-// GFX12: v_fmac_f16_e32 v127, 0xfe0b, v127       ; encoding: [0xff,0xfe,0xfe,0x6c,0x0b,0xfe,0x00,0x00]
+v_fmac_f16 v5.l, v127.h, v2.l
+// GFX12: v_fmac_f16_e32 v5.l, v127.h, v2.l       ; encoding: [0xff,0x05,0x0a,0x6c]
+
+v_fmac_f16 v5.h, src_scc, v2.h
+// GFX12: v_fmac_f16_e32 v5.h, src_scc, v2.h      ; encoding: [0xfd,0x04,0x0b,0x6d]
+
+v_fmac_f16 v127.h, 0xfe0b, v127.h
+// GFX12: v_fmac_f16_e32 v127.h, 0xfe0b, v127.h   ; encoding: [0xff,0xfe,0xff,0x6d,0x0b,0xfe,0x00,0x00]
 
 v_fmac_f32 v5, v1, v2
 // GFX12: v_fmac_f32_e32 v5, v1, v2               ; encoding: [0x01,0x05,0x0a,0x56]
@@ -775,50 +811,62 @@ v_fmac_f32 v5, src_scc, v2
 v_fmac_f32 v255, 0xaf123456, v255
 // GFX12: v_fmac_f32_e32 v255, 0xaf123456, v255   ; encoding: [0xff,0xfe,0xff,0x57,0x56,0x34,0x12,0xaf]
 
-v_fmamk_f16 v5, v1, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, v1, 0xfe0b, v3          ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v1.l, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, v1.l, 0xfe0b, v3.l    ; encoding: [0x01,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5.l, v127.l, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, v127.l, 0xfe0b, v3.l  ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5.l, s1, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, s1, 0xfe0b, v3.l      ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v5.l, s105, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, s105, 0xfe0b, v3.l    ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, v127, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, v127, 0xfe0b, v3        ; encoding: [0x7f,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, vcc_lo, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, vcc_lo, 0xfe0b, v3.l  ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, s1, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, s1, 0xfe0b, v3          ; encoding: [0x01,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, vcc_hi, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, vcc_hi, 0xfe0b, v3.l  ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, s105, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, s105, 0xfe0b, v3        ; encoding: [0x69,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, ttmp15, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, ttmp15, 0xfe0b, v3.l  ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, vcc_lo, 0xfe0b, v3      ; encoding: [0x6a,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, m0, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, m0, 0xfe0b, v3.l      ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, vcc_hi, 0xfe0b, v3      ; encoding: [0x6b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, exec_lo, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, exec_lo, 0xfe0b, v3.l ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, ttmp15, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, ttmp15, 0xfe0b, v3      ; encoding: [0x7b,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, exec_hi, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, exec_hi, 0xfe0b, v3.l ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, m0, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, m0, 0xfe0b, v3          ; encoding: [0x7d,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, null, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, null, 0xfe0b, v3.l    ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, exec_lo, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, exec_lo, 0xfe0b, v3     ; encoding: [0x7e,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, -1, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, -1, 0xfe0b, v3.l      ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, exec_hi, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, exec_hi, 0xfe0b, v3     ; encoding: [0x7f,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, 0.5, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, 0.5, 0xfe0b, v3.l     ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, null, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, null, 0xfe0b, v3        ; encoding: [0x7c,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, src_scc, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, src_scc, 0xfe0b, v3.l ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, -1, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, -1, 0xfe0b, v3          ; encoding: [0xc1,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v127.l, 0xfe0b, 0xfe0b, v127.l
+// GFX12: v_fmamk_f16 v127.l, 0xfe0b, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, 0.5, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, 0.5, 0xfe0b, v3         ; encoding: [0xf0,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v1.h, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, v1.h, 0xfe0b, v3.l    ; encoding: [0x81,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v5, src_scc, 0xfe0b, v3
-// GFX12: v_fmamk_f16 v5, src_scc, 0xfe0b, v3     ; encoding: [0xfd,0x06,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.l, v127.h, 0xfe0b, v3.l
+// GFX12: v_fmamk_f16 v5.l, v127.h, 0xfe0b, v3.l  ; encoding: [0xff,0x07,0x0a,0x6e,0x0b,0xfe,0x00,0x00]
 
-v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127
-// GFX12: v_fmamk_f16 v127, 0xfe0b, 0xfe0b, v127  ; encoding: [0xff,0xfe,0xfe,0x6e,0x0b,0xfe,0x00,0x00]
+v_fmamk_f16 v5.h, src_scc, 0xfe0b, v3.h
+// GFX12: v_fmamk_f16 v5.h, src_scc, 0xfe0b, v3.h ; encoding: [0xfd,0x06,0x0b,0x6f,0x0b,0xfe,0x00,0x00]
+
+v_fmamk_f16 v127.h, 0xfe0b, 0xfe0b, v127.h
+// GFX12: v_fmamk_f16 v127.h, 0xfe0b, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x6f,0x0b,0xfe,0x00,0x00]
 
 v_fmamk_f32 v5, v1, 0xaf123456, v3
 // GFX12: v_fmamk_f32 v5, v1, 0xaf123456, v3      ; encoding: [0x01,0x07,0x0a,0x58,0x56,0x34,0x12,0xaf]
@@ -2080,50 +2128,62 @@ v_sub_co_ci_u32 v255, vcc, 0xaf123456, v255, vcc
 // W64: v_sub_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2
-// GFX12: v_sub_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x66]
+v_sub_f16 v5.l, v1.l, v2.l
+// GFX12: v_sub_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x66]
+
+v_sub_f16 v5.l, v127.l, v2.l
+// GFX12: v_sub_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x66]
+
+v_sub_f16 v5.l, s1, v2.l
+// GFX12: v_sub_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x66]
+
+v_sub_f16 v5.l, s105, v2.l
+// GFX12: v_sub_f16_e32 v5.l, s105, v2.l          ; encoding: [0x69,0x04,0x0a,0x66]
+
+v_sub_f16 v5.l, vcc_lo, v2.l
+// GFX12: v_sub_f16_e32 v5.l, vcc_lo, v2.l        ; encoding: [0x6a,0x04,0x0a,0x66]
 
-v_sub_f16 v5, v127, v2
-// GFX12: v_sub_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x66]
+v_sub_f16 v5.l, vcc_hi, v2.l
+// GFX12: v_sub_f16_e32 v5.l, vcc_hi, v2.l        ; encoding: [0x6b,0x04,0x0a,0x66]
 
-v_sub_f16 v5, s1, v2
-// GFX12: v_sub_f16_e32 v5, s1, v2                ; encoding: [0x01,0x04,0x0a,0x66]
+v_sub_f16 v5.l, ttmp15, v2.l
+// GFX12: v_sub_f16_e32 v5.l, ttmp15, v2.l        ; encoding: [0x7b,0x04,0x0a,0x66]
 
-v_sub_f16 v5, s105, v2
-// GFX12: v_sub_f16_e32 v5, s105, v2              ; encoding: [0x69,0x04,0x0a,0x66]
+v_sub_f16 v5.l, m0, v2.l
+// GFX12: v_sub_f16_e32 v5.l, m0, v2.l            ; encoding: [0x7d,0x04,0x0a,0x66]
 
-v_sub_f16 v5, vcc_lo, v2
-// GFX12: v_sub_f16_e32 v5, vcc_lo, v2            ; encoding: [0x6a,0x04,0x0a,0x66]
+v_sub_f16 v5.l, exec_lo, v2.l
+// GFX12: v_sub_f16_e32 v5.l, exec_lo, v2.l       ; encoding: [0x7e,0x04,0x0a,0x66]
 
-v_sub_f16 v5, vcc_hi, v2
-// GFX12: v_sub_f16_e32 v5, vcc_hi, v2            ; encoding: [0x6b,0x04,0x0a,0x66]
+v_sub_f16 v5.l, exec_hi, v2.l
+// GFX12: v_sub_f16_e32 v5.l, exec_hi, v2.l       ; encoding: [0x7f,0x04,0x0a,0x66]
 
-v_sub_f16 v5, ttmp15, v2
-// GFX12: v_sub_f16_e32 v5, ttmp15, v2            ; encoding: [0x7b,0x04,0x0a,0x66]
+v_sub_f16 v5.l, null, v2.l
+// GFX12: v_sub_f16_e32 v5.l, null, v2.l          ; encoding: [0x7c,0x04,0x0a,0x66]
 
-v_sub_f16 v5, m0, v2
-// GFX12: v_sub_f16_e32 v5, m0, v2                ; encoding: [0x7d,0x04,0x0a,0x66]
+v_sub_f16 v5.l, -1, v2.l
+// GFX12: v_sub_f16_e32 v5.l, -1, v2.l            ; encoding: [0xc1,0x04,0x0a,0x66]
 
-v_sub_f16 v5, exec_lo, v2
-// GFX12: v_sub_f16_e32 v5, exec_lo, v2           ; encoding: [0x7e,0x04,0x0a,0x66]
+v_sub_f16 v5.l, 0.5, v2.l
+// GFX12: v_sub_f16_e32 v5.l, 0.5, v2.l           ; encoding: [0xf0,0x04,0x0a,0x66]
 
-v_sub_f16 v5, exec_hi, v2
-// GFX12: v_sub_f16_e32 v5, exec_hi, v2           ; encoding: [0x7f,0x04,0x0a,0x66]
+v_sub_f16 v5.l, src_scc, v2.l
+// GFX12: v_sub_f16_e32 v5.l, src_scc, v2.l       ; encoding: [0xfd,0x04,0x0a,0x66]
 
-v_sub_f16 v5, null, v2
-// GFX12: v_sub_f16_e32 v5, null, v2              ; encoding: [0x7c,0x04,0x0a,0x66]
+v_sub_f16 v127.l, 0xfe0b, v127.l
+// GFX12: v_sub_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16 v5, -1, v2
-// GFX12: v_sub_f16_e32 v5, -1, v2                ; encoding: [0xc1,0x04,0x0a,0x66]
+v_sub_f16 v5.l, v1.h, v2.l
+// GFX12: v_sub_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x66]
 
-v_sub_f16 v5, 0.5, v2
-// GFX12: v_sub_f16_e32 v5, 0.5, v2               ; encoding: [0xf0,0x04,0x0a,0x66]
+v_sub_f16 v5.l, v127.h, v2.l
+// GFX12: v_sub_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x66]
 
-v_sub_f16 v5, src_scc, v2
-// GFX12: v_sub_f16_e32 v5, src_scc, v2           ; encoding: [0xfd,0x04,0x0a,0x66]
+v_sub_f16 v5.h, src_scc, v2.h
+// GFX12: v_sub_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x67]
 
-v_sub_f16 v127, 0xfe0b, v127
-// GFX12: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
+v_sub_f16 v127.h, 0xfe0b, v127.h
+// GFX12: v_sub_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00]
 
 v_sub_f32 v5, v1, v2
 // GFX12: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
index b1d97fbf599fc..2e709832deecc 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
@@ -116,47 +116,53 @@ v_add_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0
 // W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x41,0xff,0x6f,0x05,0x30]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+v_add_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_add_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
+v_add_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0xe4,0x00,0xff]
 
-v_add_f16 v5, v1, v2 row_mirror
-// GFX12: v_add_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_mirror
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x40,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_half_mirror
-// GFX12: v_add_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x41,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shl:1
-// GFX12: v_add_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x01,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shl:15
-// GFX12: v_add_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x0f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shr:1
-// GFX12: v_add_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x11,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_shr:15
-// GFX12: v_add_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_ror:1
-// GFX12: v_add_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x21,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_ror:15
-// GFX12: v_add_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x2f,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
+v_add_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 
-v_add_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+v_add_f16 v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
 
-v_add_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
+v_add_f16 v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x09,0x13]
 
-v_add_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+v_add_f16 v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xf5,0x30]
+
+v_add_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x65,0x81,0x60,0x09,0x13]
+
+v_add_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x65,0xff,0x6f,0xf5,0x30]
 
 v_add_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
@@ -522,47 +528,53 @@ v_cvt_pkrtz_f16_f32 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl
 v_cvt_pkrtz_f16_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x5f,0xff,0x6f,0xf5,0x30]
 
-v_fmac_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1b,0x00,0xff]
+
+v_fmac_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
 
-v_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0xe4,0x00,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_mirror
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_mirror
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x40,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_half_mirror
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x41,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shl:1
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x01,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shl:15
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x0f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shr:1
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x11,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_shr:15
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x1f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_ror:1
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x21,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_ror:15
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x2f,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
 
-v_fmac_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x50,0x01,0xff]
+v_fmac_f16 v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
 
-v_fmac_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x5f,0x01,0x01]
+v_fmac_f16 v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x09,0x13]
 
-v_fmac_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_fmac_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x6c,0x01,0x60,0x09,0x13]
+v_fmac_f16 v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_fmac_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xf5,0x30]
 
-v_fmac_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_fmac_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x6c,0x7f,0x6f,0xf5,0x30]
+v_fmac_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_fmac_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x6d,0x81,0x60,0x09,0x13]
+
+v_fmac_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_fmac_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x6d,0xff,0x6f,0xf5,0x30]
 
 v_fmac_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_fmac_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x56,0x01,0x1b,0x00,0xff]
@@ -1516,47 +1528,53 @@ v_sub_co_ci_u32 v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0
 // W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x43,0xff,0x6f,0x05,0x30]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+
+v_sub_f16 v5.l, v1.l, v2.l row_mirror
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_sub_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0xe4,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_mirror
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x40,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_half_mirror
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x41,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shl:1
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x01,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shl:15
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x0f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shr:1
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x11,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_shr:15
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x1f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_ror:1
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x21,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 
-v_sub_f16 v5, v1, v2 row_ror:15
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x2f,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
 
-v_sub_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
+v_sub_f16 v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
 
-v_sub_f16 v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+v_sub_f16 v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
 
-v_sub_f16 v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x09,0x13]
+v_sub_f16 v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x04,0x0b,0x67,0x81,0x60,0x09,0x13]
 
-v_sub_f16 v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xf5,0x30]
+v_sub_f16 v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xfe,0xff,0x67,0xff,0x6f,0xf5,0x30]
 
 v_sub_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
index bea2eb334dab1..2bf74d8d74291 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
@@ -28,14 +28,20 @@ v_add_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // W64: v_add_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x41,0xff,0x00,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-v_add_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+v_add_f16 v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+
+v_add_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x65,0x81,0x77,0x39,0x05]
+
+v_add_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x65,0xff,0x00,0x00,0x00]
 
 v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
@@ -115,14 +121,20 @@ v_cvt_pkrtz_f16_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cvt_pkrtz_f16_f32 v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cvt_pk_rtz_f16_f32_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x5f,0xff,0x00,0x00,0x00]
 
-v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+v_fmac_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+
+v_fmac_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_fmac_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
 
-v_fmac_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x6c,0x01,0x77,0x39,0x05]
+v_fmac_f16 v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_fmac_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
 
-v_fmac_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_fmac_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x6c,0x7f,0x00,0x00,0x00]
+v_fmac_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_fmac_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x6d,0x81,0x77,0x39,0x05]
+
+v_fmac_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_fmac_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x6d,0xff,0x00,0x00,0x00]
 
 v_fmac_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_fmac_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x56,0x01,0x77,0x39,0x05]
@@ -328,14 +340,20 @@ v_sub_co_ci_u32 v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] fi:0
 // W64: v_sub_co_ci_u32_dpp v255, vcc, v255, v255, vcc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x43,0xff,0x00,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 
-v_sub_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x04,0x0b,0x67,0x81,0x77,0x39,0x05]
 
-v_sub_f16 v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+v_sub_f16 v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xfe,0xff,0x67,0xff,0x00,0x00,0x00]
 
 v_sub_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
index 92729d4bca3ce..389179b56de31 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s
@@ -1,32 +1,59 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
 
-v_add_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_add_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_add_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_add_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_add_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_add_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_e32 v255.h, v1.h, v2.h
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_add_f16_e32 v255.l, v1.l, v2.l
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_add_f16_e32 v5.h, v1.h, v255.h
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_e32 v5.h, v255.h, v2.h
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_add_f16_e32 v5.l, v1.l, v255.l
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_add_f16_e32 v5.l, v255.l, v2.l
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 v_fmaak_f16_e32 v255.h, v1.h, v2.h, 0xfe0b
 // GFX12: :[[@LINE-1]]:17: error: invalid operand for instruction
@@ -217,32 +244,59 @@ v_mul_f16_e32 v5, v1, v255
 v_mul_f16_e32 v5, v255, v2
 // GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sub_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_dpp v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
 
-v_sub_f16_e32 v255, v1, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sub_f16_e32 v5, v1, v255
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sub_f16_e32 v5, v255, v2
-// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+v_sub_f16_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_e32 v255.h, v1.h, v2.h
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_sub_f16_e32 v255.l, v1.l, v2.l
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.h, v1.h, v255.h
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.h, v255.h, v2.h
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.l, v1.l, v255.l
+// GFX12: :[[@LINE-1]]:27: error: invalid operand for instruction
+
+v_sub_f16_e32 v5.l, v255.l, v2.l
+// GFX12: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 v_subrev_f16_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
index e9e91fa70773d..3c281c4855985 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s
@@ -1,32 +1,59 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s
 
-v_add_f16 v255, v1, v2
-// GFX12: v_add_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+v_add_f16 v255.h, v1.h, v2.h
+// GFX12: v_add_f16_e64 v255.h, v1.h, v2.h op_sel:[1,1,1] ; encoding: [0xff,0x58,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_add_f16 v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16 v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v1, v255
-// GFX12: v_add_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
+v_add_f16 v255.l, v1.l, v2.l
+// GFX12: v_add_f16_e64 v255.l, v1.l, v2.l        ; encoding: [0xff,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_add_f16 v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16 v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16 v5, v255, v2
-// GFX12: v_add_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
+v_add_f16 v5.h, v1.h, v255.h
+// GFX12: v_add_f16_e64 v5.h, v1.h, v255.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x32,0xd5,0x01,0xff,0x03,0x00]
 
-v_add_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_add_f16 v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_add_f16 v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5.h, v255.h, v2.h
+// GFX12: v_add_f16_e64 v5.h, v255.h, v2.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x32,0xd5,0xff,0x05,0x02,0x00]
+
+v_add_f16 v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_add_f16 v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+
+v_add_f16 v5.l, v1.l, v255.l
+// GFX12: v_add_f16_e64 v5.l, v1.l, v255.l        ; encoding: [0x05,0x00,0x32,0xd5,0x01,0xff,0x03,0x00]
+
+v_add_f16 v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16 v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16 v5.l, v255.l, v2.l
+// GFX12: v_add_f16_e64 v5.l, v255.l, v2.l        ; encoding: [0x05,0x00,0x32,0xd5,0xff,0x05,0x02,0x00]
+
+v_add_f16 v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+
+v_add_f16 v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_fmac_f16 v255, v1, v2
 // GFX12: v_fmac_f16_e64 v255, v1, v2             ; encoding: [0xff,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
@@ -136,32 +163,59 @@ v_mul_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
 v_mul_f16 v5, v255, v2 quad_perm:[3,2,1,0]
 // GFX12: v_mul_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_sub_f16 v255, v1, v2
-// GFX12: v_sub_f16_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+v_sub_f16 v255.h, v1.h, v2.h
+// GFX12: v_sub_f16_e64 v255.h, v1.h, v2.h op_sel:[1,1,1] ; encoding: [0xff,0x58,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16 v255.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v255.h, v1.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255.h, v1.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v255.l, v1.l, v2.l
+// GFX12: v_sub_f16_e64 v255.l, v1.l, v2.l        ; encoding: [0xff,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16 v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v255.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v255.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.h, v1.h, v255.h
+// GFX12: v_sub_f16_e64 v5.h, v1.h, v255.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x33,0xd5,0x01,0xff,0x03,0x00]
+
+v_sub_f16 v5.h, v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16 v5.h, v1.h, v255.h quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.h, v1.h, v255.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16 v5.h, v255.h, v2.h
+// GFX12: v_sub_f16_e64 v5.h, v255.h, v2.h op_sel:[1,1,1] ; encoding: [0x05,0x58,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.h, v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_sub_f16 v255, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v255, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.h, v255.h, v2.h quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.h, v255.h, v2.h op_sel:[1,1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v1, v255
-// GFX12: v_sub_f16_e64 v5, v1, v255              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
+v_sub_f16 v5.l, v1.l, v255.l
+// GFX12: v_sub_f16_e64 v5.l, v1.l, v255.l        ; encoding: [0x05,0x00,0x33,0xd5,0x01,0xff,0x03,0x00]
 
-v_sub_f16 v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16 v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16 v5, v1, v255 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v1.l, v255.l quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
-v_sub_f16 v5, v255, v2
-// GFX12: v_sub_f16_e64 v5, v255, v2              ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
+v_sub_f16 v5.l, v255.l, v2.l
+// GFX12: v_sub_f16_e64 v5.l, v255.l, v2.l        ; encoding: [0x05,0x00,0x33,0xd5,0xff,0x05,0x02,0x00]
 
-v_sub_f16 v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v255, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
+v_sub_f16 v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v255.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
 
-v_sub_f16 v5, v255, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v255, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
+v_sub_f16 v5.l, v255.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_subrev_f16 v255, v1, v2
 // GFX12: v_subrev_f16_e64 v255, v1, v2           ; encoding: [0xff,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index 69ed1af22b459..c7cd88e81583f 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -2033,92 +2033,116 @@ v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2
 // W64: v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 // W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
 
-v_dot2_bf16_bf16 v5, v1, v2, s3
-// GFX12: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+v_dot2_bf16_bf16 v5.l, v1, v2, s3
+// GFX12: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 
-v_dot2_bf16_bf16 v5, v255, v255, s105
-// GFX12: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+v_dot2_bf16_bf16 v5.l, v255, v255, s105
+// GFX12: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 
-v_dot2_bf16_bf16 v5, s1, s2, v3
-// GFX12: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+v_dot2_bf16_bf16 v5.l, s1, s2, v3.l
+// GFX12: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 
-v_dot2_bf16_bf16 v5, s105, s105, m0
-// GFX12: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+v_dot2_bf16_bf16 v5.l, s105, s105, m0
+// GFX12: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 
-v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255
-// GFX12: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l
+// GFX12: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi
-// GFX12: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi
+// GFX12: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15
-// GFX12: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15
+// GFX12: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 
-v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo
-// GFX12: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo
+// GFX12: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 
-v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b|
-// GFX12: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b|
+// GFX12: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 
-v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo|
-// GFX12: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo|
+// GFX12: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 
-v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc|
-// GFX12: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc|
+// GFX12: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 
-v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| op_sel:[0,0,0,0]
-// GFX12: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi|
+// GFX12: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 
-v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
-// GFX12: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
+// GFX12: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1]
-// GFX12: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null
+// GFX12: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-v_dot2_f16_f16 v5, v1, v2, s3
-// GFX12: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h
+// GFX12: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-v_dot2_f16_f16 v5, v255, s2, s105
-// GFX12: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 
-v_dot2_f16_f16 v5, s1, v255, exec_hi
-// GFX12: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l
+// GFX12: v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 
-v_dot2_f16_f16 v5, s105, s105, exec_lo
-// GFX12: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l
+// GFX12: v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
 
-v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3
-// GFX12: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+v_dot2_f16_f16 v5.l, v1, v2, s3
+// GFX12: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 
-v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255
-// GFX12: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, v255, s2, s105
+// GFX12: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 
-v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
-// GFX12: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+v_dot2_f16_f16 v5.l, s1, v255, exec_hi
+// GFX12: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 
-v_dot2_f16_f16 v5, m0, 0.5, m0
-// GFX12: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+v_dot2_f16_f16 v5.l, s105, s105, exec_lo
+// GFX12: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 
-v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi
-// GFX12: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l
+// GFX12: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 
-v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo|
-// GFX12: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l
+// GFX12: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b|
-// GFX12: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 
-v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc|
-// GFX12: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+v_dot2_f16_f16 v5.l, m0, 0.5, m0
+// GFX12: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 
-v_dot2_f16_f16 v5, 0.5, -m0, 0.5 op_sel:[0,0,0,0]
-// GFX12: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi
+// GFX12: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 
-v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
-// GFX12: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo|
+// GFX12: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1]
-// GFX12: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b|
+// GFX12: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+
+v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc|
+// GFX12: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5
+// GFX12: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0]
+// GFX12: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null
+// GFX12: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+
+v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h
+// GFX12: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+
+v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1
+// GFX12: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l
+// GFX12: v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+
+v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l
+// GFX12: v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
 
 v_fma_dx9_zero_f32 v5, v1, v2, s3
 // GFX12: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s
index 7a1ebbd9e19a8..f5d5bd580cef8 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s
@@ -47,3 +47,9 @@ v_cvt_pknorm_i16_f16 v5, v1, v2
 
 v_cvt_pknorm_u16_f16 v5, v1, v2
 // GFX12: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
+
+v_add3_nc_u32 v5, v1, v2, s3
+// GFX12: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
+
+v_xor_add_u32 v5, v1, v2, s3
+// GFX12: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index 6b7b2ac4d3cc5..894acc5e94e1d 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -5239,47 +5239,59 @@ v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 ban
 v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1
 // GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
 
-v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
-v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX12: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h quad_perm:[0,1,2,3]
+// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
-v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
-// GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX12-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x00,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x00,0x00]
+v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h row_mirror
+// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
+v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 v_minimum_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index 4ea57c4b74ae9..a5bfec80d8039 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -3508,47 +3508,59 @@ v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0]
 v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1
 // GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
 
-v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x66,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
-v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
+v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 v_minimum_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2-fake16.s
new file mode 100644
index 0000000000000..1e78e3866cfb4
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2-fake16.s
@@ -0,0 +1,2269 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x20,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_add_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_add_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x20,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_add_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x20,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_add_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x20,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_add_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x20,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_add_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_add_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x20,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_add_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x20,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x20,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX12: v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x20,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_add_f16_e64 v5, v1, v2
+// GFX12: v_add_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f16_e64 v5, v255, v255
+// GFX12: v_add_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f16_e64 v5, s1, s2
+// GFX12: v_add_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_f16_e64 v5, s105, s105
+// GFX12: v_add_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_add_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_add_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_add_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_f16_e64 v5, m0, 0.5
+// GFX12: v_add_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_f16_e64 v5, exec_lo, -1
+// GFX12: v_add_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_f16_e64 v5, |exec_hi|, null
+// GFX12: v_add_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_f16_e64 v5, null, exec_lo
+// GFX12: v_add_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_f16_e64 v5, -1, exec_hi
+// GFX12: v_add_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_add_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_add_f32_e64 v5, v1, v2
+// GFX12: v_add_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f32_e64 v5, v255, v255
+// GFX12: v_add_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x03,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f32_e64 v5, s1, s2
+// GFX12: v_add_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_f32_e64 v5, s105, s105
+// GFX12: v_add_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x03,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_add_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x03,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_add_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x03,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_add_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_add_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x03,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_f32_e64 v5, m0, 0.5
+// GFX12: v_add_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x03,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_f32_e64 v5, exec_lo, -1
+// GFX12: v_add_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x03,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_f32_e64 v5, |exec_hi|, null
+// GFX12: v_add_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x03,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_f32_e64 v5, null, exec_lo
+// GFX12: v_add_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x03,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_f32_e64 v5, -1, exec_hi
+// GFX12: v_add_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x03,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_add_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x03,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_add_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_add_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x03,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_add_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_add_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x03,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_add_f64_e64 v[5:6], v[2:3], v[4:5]
+// GFX12: v_add_f64_e64 v[5:6], v[2:3], v[4:5]    ; encoding: [0x05,0x00,0x02,0xd5,0x02,0x09,0x02,0x00]
+
+v_add_f64_e64 v[5:6], v[104:105], v[104:105]
+// GFX12: v_add_f64_e64 v[5:6], v[104:105], v[104:105] ; encoding: [0x05,0x00,0x02,0xd5,0x68,0xd1,0x02,0x00]
+
+v_add_f64_e64 v[5:6], s[2:3], s[4:5]
+// GFX12: v_add_f64_e64 v[5:6], s[2:3], s[4:5]    ; encoding: [0x05,0x00,0x02,0xd5,0x02,0x08,0x00,0x00]
+
+v_add_f64_e64 v[5:6], s[104:105], s[104:105]
+// GFX12: v_add_f64_e64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x02,0xd5,0x68,0xd0,0x00,0x00]
+
+v_add_f64_e64 v[5:6], vcc, ttmp[14:15]
+// GFX12: v_add_f64_e64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x02,0xd5,0x6a,0xf4,0x00,0x00]
+
+v_add_f64_e64 v[5:6], ttmp[14:15], 0xaf123456
+// GFX12: v_add_f64_e64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x02,0xd5,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_add_f64_e64 v[5:6], -|exec|, src_scc
+// GFX12: v_add_f64_e64 v[5:6], -|exec|, src_scc  ; encoding: [0x05,0x01,0x02,0xd5,0x7e,0xfa,0x01,0x20]
+
+v_add_f64_e64 v[5:6], null, 0.5
+// GFX12: v_add_f64_e64 v[5:6], null, 0.5         ; encoding: [0x05,0x00,0x02,0xd5,0x7c,0xe0,0x01,0x00]
+
+v_add_f64_e64 v[5:6], -1, -1
+// GFX12: v_add_f64_e64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x02,0xd5,0xc1,0x82,0x01,0x00]
+
+v_add_f64_e64 v[5:6], 0.5, null mul:2
+// GFX12: v_add_f64_e64 v[5:6], 0.5, null mul:2   ; encoding: [0x05,0x00,0x02,0xd5,0xf0,0xf8,0x00,0x08]
+
+v_add_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4
+// GFX12: v_add_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x02,0xd5,0xfd,0xfc,0x00,0x70]
+
+v_add_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2
+// GFX12: v_add_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x02,0xd5,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32_e64 v5, v1, v2
+// GFX12: v_add_nc_u32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x25,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_nc_u32_e64 v5, v255, v255
+// GFX12: v_add_nc_u32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x25,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_nc_u32_e64 v5, s1, s2
+// GFX12: v_add_nc_u32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x25,0xd5,0x01,0x04,0x00,0x00]
+
+v_add_nc_u32_e64 v5, s105, s105
+// GFX12: v_add_nc_u32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x25,0xd5,0x69,0xd2,0x00,0x00]
+
+v_add_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_add_nc_u32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x25,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_add_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_add_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_add_nc_u32_e64 v5, ttmp15, src_scc
+// GFX12: v_add_nc_u32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x25,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_add_nc_u32_e64 v5, m0, 0.5
+// GFX12: v_add_nc_u32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x25,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_add_nc_u32_e64 v5, exec_lo, -1
+// GFX12: v_add_nc_u32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x25,0xd5,0x7e,0x82,0x01,0x00]
+
+v_add_nc_u32_e64 v5, exec_hi, null
+// GFX12: v_add_nc_u32_e64 v5, exec_hi, null      ; encoding: [0x05,0x00,0x25,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_add_nc_u32_e64 v5, null, exec_lo
+// GFX12: v_add_nc_u32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x25,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_add_nc_u32_e64 v5, -1, exec_hi
+// GFX12: v_add_nc_u32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x25,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_add_nc_u32_e64 v5, 0.5, m0
+// GFX12: v_add_nc_u32_e64 v5, 0.5, m0            ; encoding: [0x05,0x00,0x25,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_add_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX12: v_add_nc_u32_e64 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x25,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_add_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX12: v_add_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_and_b32_e64 v5, v1, v2
+// GFX12: v_and_b32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x1b,0xd5,0x01,0x05,0x02,0x00]
+
+v_and_b32_e64 v5, v255, v255
+// GFX12: v_and_b32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x1b,0xd5,0xff,0xff,0x03,0x00]
+
+v_and_b32_e64 v5, s1, s2
+// GFX12: v_and_b32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x1b,0xd5,0x01,0x04,0x00,0x00]
+
+v_and_b32_e64 v5, s105, s105
+// GFX12: v_and_b32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x1b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_and_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_and_b32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x1b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_and_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_and_b32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x1b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_and_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_and_b32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x1b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_and_b32_e64 v5, m0, 0.5
+// GFX12: v_and_b32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x1b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_and_b32_e64 v5, exec_lo, -1
+// GFX12: v_and_b32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x1b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_and_b32_e64 v5, exec_hi, null
+// GFX12: v_and_b32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x1b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_and_b32_e64 v5, null, exec_lo
+// GFX12: v_and_b32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x1b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_and_b32_e64 v5, -1, exec_hi
+// GFX12: v_and_b32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x1b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_and_b32_e64 v5, 0.5, m0
+// GFX12: v_and_b32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x1b,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_and_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_and_b32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x1b,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_and_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_and_b32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x1b,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32_e64 v5, v1, v2
+// GFX12: v_ashrrev_i32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x1a,0xd5,0x01,0x05,0x02,0x00]
+
+v_ashrrev_i32_e64 v5, v255, v255
+// GFX12: v_ashrrev_i32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x1a,0xd5,0xff,0xff,0x03,0x00]
+
+v_ashrrev_i32_e64 v5, s1, s2
+// GFX12: v_ashrrev_i32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x1a,0xd5,0x01,0x04,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, s105, s105
+// GFX12: v_ashrrev_i32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x1a,0xd5,0x69,0xd2,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_ashrrev_i32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x1a,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_ashrrev_i32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_ashrrev_i32_e64 v5, ttmp15, src_scc
+// GFX12: v_ashrrev_i32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x1a,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, m0, 0.5
+// GFX12: v_ashrrev_i32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x1a,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, exec_lo, -1
+// GFX12: v_ashrrev_i32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x1a,0xd5,0x7e,0x82,0x01,0x00]
+
+v_ashrrev_i32_e64 v5, exec_hi, null
+// GFX12: v_ashrrev_i32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x1a,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, null, exec_lo
+// GFX12: v_ashrrev_i32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x1a,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, -1, exec_hi
+// GFX12: v_ashrrev_i32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x1a,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, 0.5, m0
+// GFX12: v_ashrrev_i32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x1a,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_ashrrev_i32_e64 v5, src_scc, vcc_lo
+// GFX12: v_ashrrev_i32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x1a,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_ashrrev_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_ashrrev_i32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cndmask_b32_e64 v5, v1, 0xaf123456, s3
+// W32: v_cndmask_b32_e64 v5, v1, 0xaf123456, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v255, src_scc, s3
+// W32: v_cndmask_b32_e64 v5, v255, src_scc, s3 ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, s105, s105, s3
+// W32: v_cndmask_b32_e64 v5, s105, s105, s3    ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_lo, v2, s3
+// W32: v_cndmask_b32_e64 v5, vcc_lo, v2, s3    ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_hi, v255, s3
+// W32: v_cndmask_b32_e64 v5, vcc_hi, v255, s3  ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:37: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, ttmp15, ttmp15, s3
+// W32: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, m0, 0.5, s3
+// W32: v_cndmask_b32_e64 v5, m0, 0.5, s3       ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_lo, exec_lo, s3
+// W32: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s3 ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_hi, -1, s3
+// W32: v_cndmask_b32_e64 v5, exec_hi, -1, s3   ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:36: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, null, exec_hi, s105
+// W32: v_cndmask_b32_e64 v5, null, exec_hi, s105 ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -1, m0, vcc_lo
+// W32: v_cndmask_b32_e64 v5, -1, m0, vcc_lo    ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:31: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc_hi
+// W32: v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc_hi ; encoding: [0x05,0x02,0x01,0xd5,0xf0,0xd4,0xac,0x41]
+// W64-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp15
+// W32: v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp15 ; encoding: [0x05,0x01,0x01,0xd5,0xfd,0xf8,0xec,0x21]
+// W64-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v1, 0xaf123456, s[6:7]
+// W64: v_cndmask_b32_e64 v5, v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, v255, src_scc, s[6:7]
+// W64: v_cndmask_b32_e64 v5, v255, src_scc, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, s105, s105, s[6:7]
+// W64: v_cndmask_b32_e64 v5, s105, s105, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7]
+// W64: v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, vcc_hi, v255, s[6:7]
+// W64: v_cndmask_b32_e64 v5, vcc_hi, v255, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:37: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, ttmp15, ttmp15, s[6:7]
+// W64: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, m0, 0.5, s[6:7]
+// W64: v_cndmask_b32_e64 v5, m0, 0.5, s[6:7]   ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_lo, exec_lo, s[6:7]
+// W64: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, exec_hi, -1, s[6:7]
+// W64: v_cndmask_b32_e64 v5, exec_hi, -1, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:36: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, null, exec_hi, s[6:7]
+// W64: v_cndmask_b32_e64 v5, null, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -1, m0, s[104:105]
+// W64: v_cndmask_b32_e64 v5, -1, m0, s[104:105] ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:31: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc
+// W64: v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc ; encoding: [0x05,0x02,0x01,0xd5,0xf0,0xd4,0xa8,0x41]
+// W32-ERR: :[[@LINE-2]]:39: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp[14:15]
+// W64: v_cndmask_b32_e64 v5, -|src_scc|, null, ttmp[14:15] ; encoding: [0x05,0x01,0x01,0xd5,0xfd,0xf8,0xe8,0x21]
+// W32-ERR: :[[@LINE-2]]:41: error: invalid operand for instruction
+
+v_cndmask_b32_e64 v255, -|0xaf123456|, -|vcc_hi|, null
+// GFX12: v_cndmask_b32_e64 v255, -|0xaf123456|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x01,0xd5,0xff,0xd6,0xf0,0x61,0x56,0x34,0x12,0xaf]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x05,0x02,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255 ; encoding: [0x05,0x00,0x2f,0xd5,0xff,0xff,0x03,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x04,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105 ; encoding: [0x05,0x00,0x2f,0xd5,0x69,0xd2,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x2f,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x2f,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x2f,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5    ; encoding: [0x05,0x00,0x2f,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x2f,0xd5,0x7e,0x82,0x01,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x2f,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x2f,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x2f,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0   ; encoding: [0x05,0x00,0x2f,0xd5,0xf0,0xfa,0x00,0x40]
+
+v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo|
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x2f,0xd5,0xfd,0xd4,0x00,0x20]
+
+v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0xff,0x83,0x2f,0xd5,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32_e64 v5, v1, v2
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, v1, v2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x05,0x02,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, v255, v255
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, v255, v255 ; encoding: [0x05,0x00,0x2f,0xd5,0xff,0xff,0x03,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, s1, s2
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, s1, s2     ; encoding: [0x05,0x00,0x2f,0xd5,0x01,0x04,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, s105, s105
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, s105, s105 ; encoding: [0x05,0x00,0x2f,0xd5,0x69,0xd2,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x2f,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x2f,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cvt_pkrtz_f16_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x2f,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, m0, 0.5
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, m0, 0.5    ; encoding: [0x05,0x00,0x2f,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, exec_lo, -1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, exec_lo, -1 ; encoding: [0x05,0x00,0x2f,0xd5,0x7e,0x82,0x01,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, |exec_hi|, null
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x2f,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, null, exec_lo
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x2f,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, -1, exec_hi
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, -1, exec_hi ; encoding: [0x05,0x00,0x2f,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64 v5, 0.5, -m0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, 0.5, -m0   ; encoding: [0x05,0x00,0x2f,0xd5,0xf0,0xfa,0x00,0x40]
+
+v_cvt_pkrtz_f16_f32_e64 v5, -src_scc, |vcc_lo|
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x2f,0xd5,0xfd,0xd4,0x00,0x20]
+
+v_cvt_pkrtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp
+// GFX12: v_cvt_pk_rtz_f16_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0xff,0x83,0x2f,0xd5,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_fmac_f16_e64 v5, v1, v2
+// GFX12: v_fmac_f16_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x36,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_f16_e64 v5, v255, v255
+// GFX12: v_fmac_f16_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x36,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_f16_e64 v5, s1, s2
+// GFX12: v_fmac_f16_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x36,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_f16_e64 v5, s105, s105
+// GFX12: v_fmac_f16_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x36,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_fmac_f16_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x36,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_fmac_f16_e64 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x36,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_fmac_f16_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x36,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_f16_e64 v5, m0, 0.5
+// GFX12: v_fmac_f16_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x36,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_f16_e64 v5, exec_lo, -1
+// GFX12: v_fmac_f16_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x36,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_f16_e64 v5, |exec_hi|, null
+// GFX12: v_fmac_f16_e64 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x36,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_f16_e64 v5, null, exec_lo
+// GFX12: v_fmac_f16_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x36,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_f16_e64 v5, -1, exec_hi
+// GFX12: v_fmac_f16_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x36,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_fmac_f16_e64 v5, 0.5, -m0 mul:2       ; encoding: [0x05,0x00,0x36,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_fmac_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x36,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_fmac_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x36,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_fmac_f32_e64 v5, v1, v2
+// GFX12: v_fmac_f32_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x2b,0xd5,0x01,0x05,0x02,0x00]
+
+v_fmac_f32_e64 v5, v255, v255
+// GFX12: v_fmac_f32_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x2b,0xd5,0xff,0xff,0x03,0x00]
+
+v_fmac_f32_e64 v5, s1, s2
+// GFX12: v_fmac_f32_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x2b,0xd5,0x01,0x04,0x00,0x00]
+
+v_fmac_f32_e64 v5, s105, s105
+// GFX12: v_fmac_f32_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x2b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_fmac_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_fmac_f32_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x2b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_fmac_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_fmac_f32_e64 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x2b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_fmac_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_fmac_f32_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x2b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_fmac_f32_e64 v5, m0, 0.5
+// GFX12: v_fmac_f32_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x2b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_fmac_f32_e64 v5, exec_lo, -1
+// GFX12: v_fmac_f32_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x2b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_fmac_f32_e64 v5, |exec_hi|, null
+// GFX12: v_fmac_f32_e64 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x2b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_fmac_f32_e64 v5, null, exec_lo
+// GFX12: v_fmac_f32_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x2b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_fmac_f32_e64 v5, -1, exec_hi
+// GFX12: v_fmac_f32_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x2b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_fmac_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_fmac_f32_e64 v5, 0.5, -m0 mul:2       ; encoding: [0x05,0x00,0x2b,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_fmac_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_fmac_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x2b,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_fmac_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_fmac_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x2b,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_ldexp_f16_e64 v5, v1, v2
+// GFX12: v_ldexp_f16_e64 v5, v1, v2              ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x05,0x02,0x00]
+
+v_ldexp_f16_e64 v5, v255, v255
+// GFX12: v_ldexp_f16_e64 v5, v255, v255          ; encoding: [0x05,0x00,0x3b,0xd5,0xff,0xff,0x03,0x00]
+
+v_ldexp_f16_e64 v5, s1, s2
+// GFX12: v_ldexp_f16_e64 v5, s1, s2              ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x04,0x00,0x00]
+
+v_ldexp_f16_e64 v5, s105, s105
+// GFX12: v_ldexp_f16_e64 v5, s105, s105          ; encoding: [0x05,0x00,0x3b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_ldexp_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_ldexp_f16_e64 v5, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_ldexp_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_ldexp_f16_e64 v5, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3b,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_ldexp_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_ldexp_f16_e64 v5, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_ldexp_f16_e64 v5, m0, 0.5
+// GFX12: v_ldexp_f16_e64 v5, m0, 0.5             ; encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_ldexp_f16_e64 v5, exec_lo, -1
+// GFX12: v_ldexp_f16_e64 v5, exec_lo, -1         ; encoding: [0x05,0x00,0x3b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_ldexp_f16_e64 v5, exec_hi, null
+// GFX12: v_ldexp_f16_e64 v5, exec_hi, null       ; encoding: [0x05,0x00,0x3b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_ldexp_f16_e64 v5, null, exec_lo
+// GFX12: v_ldexp_f16_e64 v5, null, exec_lo       ; encoding: [0x05,0x00,0x3b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_ldexp_f16_e64 v5, -1, exec_hi
+// GFX12: v_ldexp_f16_e64 v5, -1, exec_hi         ; encoding: [0x05,0x00,0x3b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_ldexp_f16_e64 v5, 0.5, m0 mul:2
+// GFX12: v_ldexp_f16_e64 v5, 0.5, m0 mul:2       ; encoding: [0x05,0x00,0x3b,0xd5,0xf0,0xfa,0x00,0x08]
+
+v_ldexp_f16_e64 v5, src_scc, vcc_lo mul:4
+// GFX12: v_ldexp_f16_e64 v5, src_scc, vcc_lo mul:4 ; encoding: [0x05,0x00,0x3b,0xd5,0xfd,0xd4,0x00,0x10]
+
+v_ldexp_f16_e64 v255, -|0xfe0b|, vcc_hi clamp div:2
+// GFX12: v_ldexp_f16_e64 v255, -|0xfe0b|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x3b,0xd5,0xff,0xd6,0x00,0x38,0x0b,0xfe,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, v1, v2
+// GFX12: v_lshlrev_b32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x18,0xd5,0x01,0x05,0x02,0x00]
+
+v_lshlrev_b32_e64 v5, v255, v255
+// GFX12: v_lshlrev_b32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x18,0xd5,0xff,0xff,0x03,0x00]
+
+v_lshlrev_b32_e64 v5, s1, s2
+// GFX12: v_lshlrev_b32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x18,0xd5,0x01,0x04,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, s105, s105
+// GFX12: v_lshlrev_b32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x18,0xd5,0x69,0xd2,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_lshlrev_b32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x18,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_lshlrev_b32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x18,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshlrev_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_lshlrev_b32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x18,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, m0, 0.5
+// GFX12: v_lshlrev_b32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x18,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, exec_lo, -1
+// GFX12: v_lshlrev_b32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x18,0xd5,0x7e,0x82,0x01,0x00]
+
+v_lshlrev_b32_e64 v5, exec_hi, null
+// GFX12: v_lshlrev_b32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x18,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, null, exec_lo
+// GFX12: v_lshlrev_b32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x18,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, -1, exec_hi
+// GFX12: v_lshlrev_b32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x18,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, 0.5, m0
+// GFX12: v_lshlrev_b32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x18,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_lshlrev_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_lshlrev_b32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x18,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_lshlrev_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_lshlrev_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x18,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshlrev_b64_e64 v[5:6], v1, v[2:3]
+// GFX12: v_lshlrev_b64_e64 v[5:6], v1, v[2:3]    ; encoding: [0x05,0x00,0x1f,0xd5,0x01,0x05,0x02,0x00]
+
+v_lshlrev_b64_e64 v[5:6], v255, v[254:255]
+// GFX12: v_lshlrev_b64_e64 v[5:6], v255, v[254:255] ; encoding: [0x05,0x00,0x1f,0xd5,0xff,0xfd,0x03,0x00]
+
+v_lshlrev_b64_e64 v[5:6], v1, vcc
+// GFX12: v_lshlrev_b64_e64 v[5:6], v1, vcc       ; encoding: [0x05,0x00,0x1f,0xd5,0x01,0xd5,0x00,0x00]
+
+v_lshlrev_b64_e64 v[5:6], v255, exec
+// GFX12: v_lshlrev_b64_e64 v[5:6], v255, exec    ; encoding: [0x05,0x00,0x1f,0xd5,0xff,0xfd,0x00,0x00]
+
+v_lshlrev_b64_e64 v[5:6], null, null
+// GFX12: v_lshlrev_b64_e64 v[5:6], null, null    ; encoding: [0x05,0x00,0x1f,0xd5,0x7c,0xf8,0x00,0x00]
+
+v_lshlrev_b64_e64 v[5:6], -1, -1
+// GFX12: v_lshlrev_b64_e64 v[5:6], -1, -1        ; encoding: [0x05,0x00,0x1f,0xd5,0xc1,0x82,0x01,0x00]
+
+v_lshlrev_b64_e64 v[5:6], 0.5, 0xaf123456
+// GFX12: v_lshlrev_b64_e64 v[5:6], 0.5, 0xaf123456 ; encoding: [0x05,0x00,0x1f,0xd5,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshlrev_b64_e64 v[5:6], src_scc, src_scc
+// GFX12: v_lshlrev_b64_e64 v[5:6], src_scc, src_scc ; encoding: [0x05,0x00,0x1f,0xd5,0xfd,0xfa,0x01,0x00]
+
+v_lshlrev_b64_e64 v[254:255], 0xaf123456, 0.5
+// GFX12: v_lshlrev_b64_e64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x1f,0xd5,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32_e64 v5, v1, v2
+// GFX12: v_lshrrev_b32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x19,0xd5,0x01,0x05,0x02,0x00]
+
+v_lshrrev_b32_e64 v5, v255, v255
+// GFX12: v_lshrrev_b32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x19,0xd5,0xff,0xff,0x03,0x00]
+
+v_lshrrev_b32_e64 v5, s1, s2
+// GFX12: v_lshrrev_b32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x19,0xd5,0x01,0x04,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, s105, s105
+// GFX12: v_lshrrev_b32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x19,0xd5,0x69,0xd2,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_lshrrev_b32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x19,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_lshrrev_b32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x19,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_lshrrev_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_lshrrev_b32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x19,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, m0, 0.5
+// GFX12: v_lshrrev_b32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x19,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, exec_lo, -1
+// GFX12: v_lshrrev_b32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x19,0xd5,0x7e,0x82,0x01,0x00]
+
+v_lshrrev_b32_e64 v5, exec_hi, null
+// GFX12: v_lshrrev_b32_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x19,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, null, exec_lo
+// GFX12: v_lshrrev_b32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x19,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, -1, exec_hi
+// GFX12: v_lshrrev_b32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x19,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, 0.5, m0
+// GFX12: v_lshrrev_b32_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x19,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_lshrrev_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_lshrrev_b32_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x19,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_lshrrev_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_lshrrev_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x19,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_num_f16_e64 v5, v1, v2
+// GFX12: v_max_num_f16_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x31,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_num_f16_e64 v5, v255, v255
+// GFX12: v_max_num_f16_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x31,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_num_f16_e64 v5, s1, s2
+// GFX12: v_max_num_f16_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x31,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_num_f16_e64 v5, s105, s105
+// GFX12: v_max_num_f16_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x31,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_num_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_max_num_f16_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x31,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_num_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_max_num_f16_e64 v5, vcc_hi, 0xfe0b    ; encoding: [0x05,0x00,0x31,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_max_num_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_max_num_f16_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x31,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_num_f16_e64 v5, m0, 0.5
+// GFX12: v_max_num_f16_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x31,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_num_f16_e64 v5, exec_lo, -1
+// GFX12: v_max_num_f16_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x31,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_num_f16_e64 v5, |exec_hi|, null
+// GFX12: v_max_num_f16_e64 v5, |exec_hi|, null   ; encoding: [0x05,0x01,0x31,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_num_f16_e64 v5, null, exec_lo
+// GFX12: v_max_num_f16_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x31,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_num_f16_e64 v5, -1, exec_hi
+// GFX12: v_max_num_f16_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x31,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_num_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_max_num_f16_e64 v5, 0.5, -m0 mul:2    ; encoding: [0x05,0x00,0x31,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_max_num_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_max_num_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x31,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_max_num_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_max_num_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x31,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_max_num_f32_e64 v5, v1, v2
+// GFX12: v_max_num_f32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x16,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_num_f32_e64 v5, v255, v255
+// GFX12: v_max_num_f32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x16,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_num_f32_e64 v5, s1, s2
+// GFX12: v_max_num_f32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x16,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_num_f32_e64 v5, s105, s105
+// GFX12: v_max_num_f32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x16,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_num_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_max_num_f32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x16,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_num_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_max_num_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x16,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_num_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_max_num_f32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x16,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_num_f32_e64 v5, m0, 0.5
+// GFX12: v_max_num_f32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x16,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_num_f32_e64 v5, exec_lo, -1
+// GFX12: v_max_num_f32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x16,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_num_f32_e64 v5, |exec_hi|, null
+// GFX12: v_max_num_f32_e64 v5, |exec_hi|, null   ; encoding: [0x05,0x01,0x16,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_num_f32_e64 v5, null, exec_lo
+// GFX12: v_max_num_f32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x16,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_num_f32_e64 v5, -1, exec_hi
+// GFX12: v_max_num_f32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x16,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_num_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_max_num_f32_e64 v5, 0.5, -m0 mul:2    ; encoding: [0x05,0x00,0x16,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_max_num_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_max_num_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x16,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_max_num_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_max_num_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x16,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_max_num_f64_e64 v[5:6], v[2:3], v[4:5]
+// GFX12: v_max_num_f64_e64 v[5:6], v[2:3], v[4:5] ; encoding: [0x05,0x00,0x0e,0xd5,0x02,0x09,0x02,0x00]
+
+v_max_num_f64_e64 v[5:6], v[104:105], v[104:105]
+// GFX12: v_max_num_f64_e64 v[5:6], v[104:105], v[104:105] ; encoding: [0x05,0x00,0x0e,0xd5,0x68,0xd1,0x02,0x00]
+
+v_max_num_f64_e64 v[5:6], s[2:3], s[4:5]
+// GFX12: v_max_num_f64_e64 v[5:6], s[2:3], s[4:5] ; encoding: [0x05,0x00,0x0e,0xd5,0x02,0x08,0x00,0x00]
+
+v_max_num_f64_e64 v[5:6], s[104:105], s[104:105]
+// GFX12: v_max_num_f64_e64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x0e,0xd5,0x68,0xd0,0x00,0x00]
+
+v_max_num_f64_e64 v[5:6], vcc, ttmp[14:15]
+// GFX12: v_max_num_f64_e64 v[5:6], vcc, ttmp[14:15] ; encoding: [0x05,0x00,0x0e,0xd5,0x6a,0xf4,0x00,0x00]
+
+v_max_num_f64_e64 v[5:6], ttmp[14:15], 0xaf123456
+// GFX12: v_max_num_f64_e64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x0e,0xd5,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_num_f64_e64 v[5:6], -|exec|, src_scc
+// GFX12: v_max_num_f64_e64 v[5:6], -|exec|, src_scc ; encoding: [0x05,0x01,0x0e,0xd5,0x7e,0xfa,0x01,0x20]
+
+v_max_num_f64_e64 v[5:6], null, 0.5
+// GFX12: v_max_num_f64_e64 v[5:6], null, 0.5     ; encoding: [0x05,0x00,0x0e,0xd5,0x7c,0xe0,0x01,0x00]
+
+v_max_num_f64_e64 v[5:6], -1, -1
+// GFX12: v_max_num_f64_e64 v[5:6], -1, -1        ; encoding: [0x05,0x00,0x0e,0xd5,0xc1,0x82,0x01,0x00]
+
+v_max_num_f64_e64 v[5:6], 0.5, null mul:2
+// GFX12: v_max_num_f64_e64 v[5:6], 0.5, null mul:2 ; encoding: [0x05,0x00,0x0e,0xd5,0xf0,0xf8,0x00,0x08]
+
+v_max_num_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4
+// GFX12: v_max_num_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x0e,0xd5,0xfd,0xfc,0x00,0x70]
+
+v_max_num_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2
+// GFX12: v_max_num_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x0e,0xd5,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
+
+v_max_i32_e64 v5, v1, v2
+// GFX12: v_max_i32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x12,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_i32_e64 v5, v255, v255
+// GFX12: v_max_i32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x12,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_i32_e64 v5, s1, s2
+// GFX12: v_max_i32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x12,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_i32_e64 v5, s105, s105
+// GFX12: v_max_i32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x12,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_i32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_max_i32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x12,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_max_i32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x12,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_i32_e64 v5, ttmp15, src_scc
+// GFX12: v_max_i32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x12,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_i32_e64 v5, m0, 0.5
+// GFX12: v_max_i32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x12,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_i32_e64 v5, exec_lo, -1
+// GFX12: v_max_i32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x12,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_i32_e64 v5, exec_hi, null
+// GFX12: v_max_i32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x12,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_i32_e64 v5, null, exec_lo
+// GFX12: v_max_i32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x12,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_i32_e64 v5, -1, exec_hi
+// GFX12: v_max_i32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x12,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_i32_e64 v5, 0.5, m0
+// GFX12: v_max_i32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_max_i32_e64 v5, src_scc, vcc_lo
+// GFX12: v_max_i32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x12,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_max_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_max_i32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x12,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_u32_e64 v5, v1, v2
+// GFX12: v_max_u32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x14,0xd5,0x01,0x05,0x02,0x00]
+
+v_max_u32_e64 v5, v255, v255
+// GFX12: v_max_u32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x14,0xd5,0xff,0xff,0x03,0x00]
+
+v_max_u32_e64 v5, s1, s2
+// GFX12: v_max_u32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x14,0xd5,0x01,0x04,0x00,0x00]
+
+v_max_u32_e64 v5, s105, s105
+// GFX12: v_max_u32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x14,0xd5,0x69,0xd2,0x00,0x00]
+
+v_max_u32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_max_u32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x14,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_max_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_max_u32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_max_u32_e64 v5, ttmp15, src_scc
+// GFX12: v_max_u32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x14,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_max_u32_e64 v5, m0, 0.5
+// GFX12: v_max_u32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x14,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_max_u32_e64 v5, exec_lo, -1
+// GFX12: v_max_u32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x14,0xd5,0x7e,0x82,0x01,0x00]
+
+v_max_u32_e64 v5, exec_hi, null
+// GFX12: v_max_u32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x14,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_max_u32_e64 v5, null, exec_lo
+// GFX12: v_max_u32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x14,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_max_u32_e64 v5, -1, exec_hi
+// GFX12: v_max_u32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x14,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_max_u32_e64 v5, 0.5, m0
+// GFX12: v_max_u32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x14,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_max_u32_e64 v5, src_scc, vcc_lo
+// GFX12: v_max_u32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x14,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_max_u32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_max_u32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x14,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_num_f16_e64 v5, v1, v2
+// GFX12: v_min_num_f16_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x30,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_num_f16_e64 v5, v255, v255
+// GFX12: v_min_num_f16_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x30,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_num_f16_e64 v5, s1, s2
+// GFX12: v_min_num_f16_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x30,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_num_f16_e64 v5, s105, s105
+// GFX12: v_min_num_f16_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x30,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_num_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_min_num_f16_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x30,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_num_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_min_num_f16_e64 v5, vcc_hi, 0xfe0b    ; encoding: [0x05,0x00,0x30,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_min_num_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_min_num_f16_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x30,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_num_f16_e64 v5, m0, 0.5
+// GFX12: v_min_num_f16_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x30,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_num_f16_e64 v5, exec_lo, -1
+// GFX12: v_min_num_f16_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x30,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_num_f16_e64 v5, |exec_hi|, null
+// GFX12: v_min_num_f16_e64 v5, |exec_hi|, null   ; encoding: [0x05,0x01,0x30,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_num_f16_e64 v5, null, exec_lo
+// GFX12: v_min_num_f16_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x30,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_num_f16_e64 v5, -1, exec_hi
+// GFX12: v_min_num_f16_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x30,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_num_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_min_num_f16_e64 v5, 0.5, -m0 mul:2    ; encoding: [0x05,0x00,0x30,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_min_num_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_min_num_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x30,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_min_num_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_min_num_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x30,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_min_num_f32_e64 v5, v1, v2
+// GFX12: v_min_num_f32_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x15,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_num_f32_e64 v5, v255, v255
+// GFX12: v_min_num_f32_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x15,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_num_f32_e64 v5, s1, s2
+// GFX12: v_min_num_f32_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x15,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_num_f32_e64 v5, s105, s105
+// GFX12: v_min_num_f32_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x15,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_num_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_min_num_f32_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x15,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_num_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_min_num_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x15,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_num_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_min_num_f32_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x15,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_num_f32_e64 v5, m0, 0.5
+// GFX12: v_min_num_f32_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x15,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_num_f32_e64 v5, exec_lo, -1
+// GFX12: v_min_num_f32_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x15,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_num_f32_e64 v5, |exec_hi|, null
+// GFX12: v_min_num_f32_e64 v5, |exec_hi|, null   ; encoding: [0x05,0x01,0x15,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_num_f32_e64 v5, null, exec_lo
+// GFX12: v_min_num_f32_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x15,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_num_f32_e64 v5, -1, exec_hi
+// GFX12: v_min_num_f32_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x15,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_num_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_min_num_f32_e64 v5, 0.5, -m0 mul:2    ; encoding: [0x05,0x00,0x15,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_min_num_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_min_num_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x15,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_min_num_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_min_num_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x15,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_min_num_f64_e64 v[5:6], v[2:3], v[4:5]
+// GFX12: v_min_num_f64_e64 v[5:6], v[2:3], v[4:5] ; encoding: [0x05,0x00,0x0d,0xd5,0x02,0x09,0x02,0x00]
+
+v_min_num_f64_e64 v[5:6], v[104:105], v[104:105]
+// GFX12: v_min_num_f64_e64 v[5:6], v[104:105], v[104:105] ; encoding: [0x05,0x00,0x0d,0xd5,0x68,0xd1,0x02,0x00]
+
+v_min_num_f64_e64 v[5:6], s[2:3], s[4:5]
+// GFX12: v_min_num_f64_e64 v[5:6], s[2:3], s[4:5] ; encoding: [0x05,0x00,0x0d,0xd5,0x02,0x08,0x00,0x00]
+
+v_min_num_f64_e64 v[5:6], s[104:105], s[104:105]
+// GFX12: v_min_num_f64_e64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x0d,0xd5,0x68,0xd0,0x00,0x00]
+
+v_min_num_f64_e64 v[5:6], vcc, ttmp[14:15]
+// GFX12: v_min_num_f64_e64 v[5:6], vcc, ttmp[14:15] ; encoding: [0x05,0x00,0x0d,0xd5,0x6a,0xf4,0x00,0x00]
+
+v_min_num_f64_e64 v[5:6], ttmp[14:15], 0xaf123456
+// GFX12: v_min_num_f64_e64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x0d,0xd5,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_num_f64_e64 v[5:6], -|exec|, src_scc
+// GFX12: v_min_num_f64_e64 v[5:6], -|exec|, src_scc ; encoding: [0x05,0x01,0x0d,0xd5,0x7e,0xfa,0x01,0x20]
+
+v_min_num_f64_e64 v[5:6], null, 0.5
+// GFX12: v_min_num_f64_e64 v[5:6], null, 0.5     ; encoding: [0x05,0x00,0x0d,0xd5,0x7c,0xe0,0x01,0x00]
+
+v_min_num_f64_e64 v[5:6], -1, -1
+// GFX12: v_min_num_f64_e64 v[5:6], -1, -1        ; encoding: [0x05,0x00,0x0d,0xd5,0xc1,0x82,0x01,0x00]
+
+v_min_num_f64_e64 v[5:6], 0.5, null mul:2
+// GFX12: v_min_num_f64_e64 v[5:6], 0.5, null mul:2 ; encoding: [0x05,0x00,0x0d,0xd5,0xf0,0xf8,0x00,0x08]
+
+v_min_num_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4
+// GFX12: v_min_num_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x0d,0xd5,0xfd,0xfc,0x00,0x70]
+
+v_min_num_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2
+// GFX12: v_min_num_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x0d,0xd5,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
+
+v_min_i32_e64 v5, v1, v2
+// GFX12: v_min_i32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x11,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_i32_e64 v5, v255, v255
+// GFX12: v_min_i32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x11,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_i32_e64 v5, s1, s2
+// GFX12: v_min_i32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x11,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_i32_e64 v5, s105, s105
+// GFX12: v_min_i32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x11,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_i32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_min_i32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x11,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_i32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_min_i32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x11,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_i32_e64 v5, ttmp15, src_scc
+// GFX12: v_min_i32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x11,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_i32_e64 v5, m0, 0.5
+// GFX12: v_min_i32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x11,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_i32_e64 v5, exec_lo, -1
+// GFX12: v_min_i32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x11,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_i32_e64 v5, exec_hi, null
+// GFX12: v_min_i32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x11,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_i32_e64 v5, null, exec_lo
+// GFX12: v_min_i32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x11,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_i32_e64 v5, -1, exec_hi
+// GFX12: v_min_i32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x11,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_i32_e64 v5, 0.5, m0
+// GFX12: v_min_i32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_min_i32_e64 v5, src_scc, vcc_lo
+// GFX12: v_min_i32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x11,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_min_i32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_min_i32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x11,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_u32_e64 v5, v1, v2
+// GFX12: v_min_u32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x13,0xd5,0x01,0x05,0x02,0x00]
+
+v_min_u32_e64 v5, v255, v255
+// GFX12: v_min_u32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x13,0xd5,0xff,0xff,0x03,0x00]
+
+v_min_u32_e64 v5, s1, s2
+// GFX12: v_min_u32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x13,0xd5,0x01,0x04,0x00,0x00]
+
+v_min_u32_e64 v5, s105, s105
+// GFX12: v_min_u32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x13,0xd5,0x69,0xd2,0x00,0x00]
+
+v_min_u32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_min_u32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x13,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_min_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_min_u32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x13,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_min_u32_e64 v5, ttmp15, src_scc
+// GFX12: v_min_u32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x13,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_min_u32_e64 v5, m0, 0.5
+// GFX12: v_min_u32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x13,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_min_u32_e64 v5, exec_lo, -1
+// GFX12: v_min_u32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x13,0xd5,0x7e,0x82,0x01,0x00]
+
+v_min_u32_e64 v5, exec_hi, null
+// GFX12: v_min_u32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x13,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_min_u32_e64 v5, null, exec_lo
+// GFX12: v_min_u32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x13,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_min_u32_e64 v5, -1, exec_hi
+// GFX12: v_min_u32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x13,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_min_u32_e64 v5, 0.5, m0
+// GFX12: v_min_u32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_min_u32_e64 v5, src_scc, vcc_lo
+// GFX12: v_min_u32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x13,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_min_u32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_min_u32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x13,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32_e64 v5, v1, v2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, v1, v2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, v255, v255
+// GFX12: v_mul_dx9_zero_f32_e64 v5, v255, v255   ; encoding: [0x05,0x00,0x07,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, s1, s2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, s1, s2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, s105, s105
+// GFX12: v_mul_dx9_zero_f32_e64 v5, s105, s105   ; encoding: [0x05,0x00,0x07,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x07,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x07,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, m0, 0.5
+// GFX12: v_mul_dx9_zero_f32_e64 v5, m0, 0.5      ; encoding: [0x05,0x00,0x07,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, exec_lo, -1
+// GFX12: v_mul_dx9_zero_f32_e64 v5, exec_lo, -1  ; encoding: [0x05,0x00,0x07,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null
+// GFX12: v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x07,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, null, exec_lo
+// GFX12: v_mul_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x07,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, -1, exec_hi
+// GFX12: v_mul_dx9_zero_f32_e64 v5, -1, exec_hi  ; encoding: [0x05,0x00,0x07,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x07,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x07,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x07,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_f16_e64 v5, v1, v2
+// GFX12: v_mul_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x35,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_f16_e64 v5, v255, v255
+// GFX12: v_mul_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x35,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_f16_e64 v5, s1, s2
+// GFX12: v_mul_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x35,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_f16_e64 v5, s105, s105
+// GFX12: v_mul_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x35,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x35,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_mul_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x35,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_mul_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x35,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_f16_e64 v5, m0, 0.5
+// GFX12: v_mul_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x35,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_f16_e64 v5, exec_lo, -1
+// GFX12: v_mul_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x35,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_f16_e64 v5, |exec_hi|, null
+// GFX12: v_mul_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x35,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_f16_e64 v5, null, exec_lo
+// GFX12: v_mul_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x35,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_f16_e64 v5, -1, exec_hi
+// GFX12: v_mul_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x35,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_mul_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x35,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_mul_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x35,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_mul_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x35,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_mul_f32_e64 v5, v1, v2
+// GFX12: v_mul_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x08,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_f32_e64 v5, v255, v255
+// GFX12: v_mul_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x08,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_f32_e64 v5, s1, s2
+// GFX12: v_mul_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x08,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_f32_e64 v5, s105, s105
+// GFX12: v_mul_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x08,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x08,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x08,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x08,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_f32_e64 v5, m0, 0.5
+// GFX12: v_mul_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x08,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_f32_e64 v5, exec_lo, -1
+// GFX12: v_mul_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x08,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_f32_e64 v5, |exec_hi|, null
+// GFX12: v_mul_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x08,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_f32_e64 v5, null, exec_lo
+// GFX12: v_mul_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x08,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_f32_e64 v5, -1, exec_hi
+// GFX12: v_mul_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x08,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_mul_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x08,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_mul_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x08,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_mul_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x08,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_f64_e64 v[5:6], v[2:3], v[4:5]
+// GFX12: v_mul_f64_e64 v[5:6], v[2:3], v[4:5]    ; encoding: [0x05,0x00,0x06,0xd5,0x02,0x09,0x02,0x00]
+
+v_mul_f64_e64 v[5:6], v[104:105], v[104:105]
+// GFX12: v_mul_f64_e64 v[5:6], v[104:105], v[104:105] ; encoding: [0x05,0x00,0x06,0xd5,0x68,0xd1,0x02,0x00]
+
+v_mul_f64_e64 v[5:6], s[2:3], s[4:5]
+// GFX12: v_mul_f64_e64 v[5:6], s[2:3], s[4:5]    ; encoding: [0x05,0x00,0x06,0xd5,0x02,0x08,0x00,0x00]
+
+v_mul_f64_e64 v[5:6], s[104:105], s[104:105]
+// GFX12: v_mul_f64_e64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x06,0xd5,0x68,0xd0,0x00,0x00]
+
+v_mul_f64_e64 v[5:6], vcc, ttmp[14:15]
+// GFX12: v_mul_f64_e64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x06,0xd5,0x6a,0xf4,0x00,0x00]
+
+v_mul_f64_e64 v[5:6], ttmp[14:15], 0xaf123456
+// GFX12: v_mul_f64_e64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd5,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_f64_e64 v[5:6], -|exec|, src_scc
+// GFX12: v_mul_f64_e64 v[5:6], -|exec|, src_scc  ; encoding: [0x05,0x01,0x06,0xd5,0x7e,0xfa,0x01,0x20]
+
+v_mul_f64_e64 v[5:6], null, 0.5
+// GFX12: v_mul_f64_e64 v[5:6], null, 0.5         ; encoding: [0x05,0x00,0x06,0xd5,0x7c,0xe0,0x01,0x00]
+
+v_mul_f64_e64 v[5:6], -1, -1
+// GFX12: v_mul_f64_e64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x06,0xd5,0xc1,0x82,0x01,0x00]
+
+v_mul_f64_e64 v[5:6], 0.5, null mul:2
+// GFX12: v_mul_f64_e64 v[5:6], 0.5, null mul:2   ; encoding: [0x05,0x00,0x06,0xd5,0xf0,0xf8,0x00,0x08]
+
+v_mul_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4
+// GFX12: v_mul_f64_e64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x06,0xd5,0xfd,0xfc,0x00,0x70]
+
+v_mul_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2
+// GFX12: v_mul_f64_e64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x06,0xd5,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24_e64 v5, v1, v2
+// GFX12: v_mul_hi_i32_i24_e64 v5, v1, v2         ; encoding: [0x05,0x00,0x0a,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_hi_i32_i24_e64 v5, v255, v255
+// GFX12: v_mul_hi_i32_i24_e64 v5, v255, v255     ; encoding: [0x05,0x00,0x0a,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_hi_i32_i24_e64 v5, s1, s2
+// GFX12: v_mul_hi_i32_i24_e64 v5, s1, s2         ; encoding: [0x05,0x00,0x0a,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, s105, s105
+// GFX12: v_mul_hi_i32_i24_e64 v5, s105, s105     ; encoding: [0x05,0x00,0x0a,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_hi_i32_i24_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_hi_i32_i24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_i32_i24_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_hi_i32_i24_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x0a,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, m0, 0.5
+// GFX12: v_mul_hi_i32_i24_e64 v5, m0, 0.5        ; encoding: [0x05,0x00,0x0a,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, exec_lo, -1
+// GFX12: v_mul_hi_i32_i24_e64 v5, exec_lo, -1    ; encoding: [0x05,0x00,0x0a,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_hi_i32_i24_e64 v5, exec_hi, null
+// GFX12: v_mul_hi_i32_i24_e64 v5, exec_hi, null  ; encoding: [0x05,0x00,0x0a,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, null, exec_lo
+// GFX12: v_mul_hi_i32_i24_e64 v5, null, exec_lo  ; encoding: [0x05,0x00,0x0a,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, -1, exec_hi
+// GFX12: v_mul_hi_i32_i24_e64 v5, -1, exec_hi    ; encoding: [0x05,0x00,0x0a,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, 0.5, m0
+// GFX12: v_mul_hi_i32_i24_e64 v5, 0.5, m0        ; encoding: [0x05,0x00,0x0a,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v5, src_scc, vcc_lo
+// GFX12: v_mul_hi_i32_i24_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_hi_i32_i24_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_mul_hi_i32_i24_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x0a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24_e64 v5, v1, v2
+// GFX12: v_mul_hi_u32_u24_e64 v5, v1, v2         ; encoding: [0x05,0x00,0x0c,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_hi_u32_u24_e64 v5, v255, v255
+// GFX12: v_mul_hi_u32_u24_e64 v5, v255, v255     ; encoding: [0x05,0x00,0x0c,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_hi_u32_u24_e64 v5, s1, s2
+// GFX12: v_mul_hi_u32_u24_e64 v5, s1, s2         ; encoding: [0x05,0x00,0x0c,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, s105, s105
+// GFX12: v_mul_hi_u32_u24_e64 v5, s105, s105     ; encoding: [0x05,0x00,0x0c,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_hi_u32_u24_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x0c,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_hi_u32_u24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0c,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_hi_u32_u24_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_hi_u32_u24_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x0c,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, m0, 0.5
+// GFX12: v_mul_hi_u32_u24_e64 v5, m0, 0.5        ; encoding: [0x05,0x00,0x0c,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, exec_lo, -1
+// GFX12: v_mul_hi_u32_u24_e64 v5, exec_lo, -1    ; encoding: [0x05,0x00,0x0c,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_hi_u32_u24_e64 v5, exec_hi, null
+// GFX12: v_mul_hi_u32_u24_e64 v5, exec_hi, null  ; encoding: [0x05,0x00,0x0c,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, null, exec_lo
+// GFX12: v_mul_hi_u32_u24_e64 v5, null, exec_lo  ; encoding: [0x05,0x00,0x0c,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, -1, exec_hi
+// GFX12: v_mul_hi_u32_u24_e64 v5, -1, exec_hi    ; encoding: [0x05,0x00,0x0c,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, 0.5, m0
+// GFX12: v_mul_hi_u32_u24_e64 v5, 0.5, m0        ; encoding: [0x05,0x00,0x0c,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v5, src_scc, vcc_lo
+// GFX12: v_mul_hi_u32_u24_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x0c,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_hi_u32_u24_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_mul_hi_u32_u24_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x0c,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24_e64 v5, v1, v2
+// GFX12: v_mul_i32_i24_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x09,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_i32_i24_e64 v5, v255, v255
+// GFX12: v_mul_i32_i24_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x09,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_i32_i24_e64 v5, s1, s2
+// GFX12: v_mul_i32_i24_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x09,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, s105, s105
+// GFX12: v_mul_i32_i24_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x09,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_i32_i24_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x09,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_i32_i24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x09,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_i32_i24_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_i32_i24_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x09,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, m0, 0.5
+// GFX12: v_mul_i32_i24_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x09,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, exec_lo, -1
+// GFX12: v_mul_i32_i24_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x09,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_i32_i24_e64 v5, exec_hi, null
+// GFX12: v_mul_i32_i24_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x09,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, null, exec_lo
+// GFX12: v_mul_i32_i24_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x09,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, -1, exec_hi
+// GFX12: v_mul_i32_i24_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x09,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, 0.5, m0
+// GFX12: v_mul_i32_i24_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x09,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_i32_i24_e64 v5, src_scc, vcc_lo
+// GFX12: v_mul_i32_i24_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x09,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_i32_i24_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX12: v_mul_i32_i24_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x09,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32_e64 v5, v1, v2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, v1, v2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_legacy_f32_e64 v5, v255, v255
+// GFX12: v_mul_dx9_zero_f32_e64 v5, v255, v255   ; encoding: [0x05,0x00,0x07,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_legacy_f32_e64 v5, s1, s2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, s1, s2       ; encoding: [0x05,0x00,0x07,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, s105, s105
+// GFX12: v_mul_dx9_zero_f32_e64 v5, s105, s105   ; encoding: [0x05,0x00,0x07,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_dx9_zero_f32_e64 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x07,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_dx9_zero_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_legacy_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_dx9_zero_f32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x07,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, m0, 0.5
+// GFX12: v_mul_dx9_zero_f32_e64 v5, m0, 0.5      ; encoding: [0x05,0x00,0x07,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, exec_lo, -1
+// GFX12: v_mul_dx9_zero_f32_e64 v5, exec_lo, -1  ; encoding: [0x05,0x00,0x07,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_legacy_f32_e64 v5, |exec_hi|, null
+// GFX12: v_mul_dx9_zero_f32_e64 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x07,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, null, exec_lo
+// GFX12: v_mul_dx9_zero_f32_e64 v5, null, exec_lo ; encoding: [0x05,0x00,0x07,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, -1, exec_hi
+// GFX12: v_mul_dx9_zero_f32_e64 v5, -1, exec_hi  ; encoding: [0x05,0x00,0x07,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_legacy_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_mul_dx9_zero_f32_e64 v5, 0.5, -m0 mul:2 ; encoding: [0x05,0x00,0x07,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_mul_legacy_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_mul_dx9_zero_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x07,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_mul_legacy_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_mul_dx9_zero_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x07,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24_e64 v5, v1, v2
+// GFX12: v_mul_u32_u24_e64 v5, v1, v2            ; encoding: [0x05,0x00,0x0b,0xd5,0x01,0x05,0x02,0x00]
+
+v_mul_u32_u24_e64 v5, v255, v255
+// GFX12: v_mul_u32_u24_e64 v5, v255, v255        ; encoding: [0x05,0x00,0x0b,0xd5,0xff,0xff,0x03,0x00]
+
+v_mul_u32_u24_e64 v5, s1, s2
+// GFX12: v_mul_u32_u24_e64 v5, s1, s2            ; encoding: [0x05,0x00,0x0b,0xd5,0x01,0x04,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, s105, s105
+// GFX12: v_mul_u32_u24_e64 v5, s105, s105        ; encoding: [0x05,0x00,0x0b,0xd5,0x69,0xd2,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, vcc_lo, ttmp15
+// GFX12: v_mul_u32_u24_e64 v5, vcc_lo, ttmp15    ; encoding: [0x05,0x00,0x0b,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_mul_u32_u24_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_mul_u32_u24_e64 v5, ttmp15, src_scc
+// GFX12: v_mul_u32_u24_e64 v5, ttmp15, src_scc   ; encoding: [0x05,0x00,0x0b,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, m0, 0.5
+// GFX12: v_mul_u32_u24_e64 v5, m0, 0.5           ; encoding: [0x05,0x00,0x0b,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, exec_lo, -1
+// GFX12: v_mul_u32_u24_e64 v5, exec_lo, -1       ; encoding: [0x05,0x00,0x0b,0xd5,0x7e,0x82,0x01,0x00]
+
+v_mul_u32_u24_e64 v5, exec_hi, null
+// GFX12: v_mul_u32_u24_e64 v5, exec_hi, null     ; encoding: [0x05,0x00,0x0b,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, null, exec_lo
+// GFX12: v_mul_u32_u24_e64 v5, null, exec_lo     ; encoding: [0x05,0x00,0x0b,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, -1, exec_hi
+// GFX12: v_mul_u32_u24_e64 v5, -1, exec_hi       ; encoding: [0x05,0x00,0x0b,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, 0.5, m0
+// GFX12: v_mul_u32_u24_e64 v5, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_mul_u32_u24_e64 v5, src_scc, vcc_lo
+// GFX12: v_mul_u32_u24_e64 v5, src_scc, vcc_lo   ; encoding: [0x05,0x00,0x0b,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_mul_u32_u24_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX12: v_mul_u32_u24_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x0b,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_or_b32_e64 v5, v1, v2
+// GFX12: v_or_b32_e64 v5, v1, v2                 ; encoding: [0x05,0x00,0x1c,0xd5,0x01,0x05,0x02,0x00]
+
+v_or_b32_e64 v5, v255, v255
+// GFX12: v_or_b32_e64 v5, v255, v255             ; encoding: [0x05,0x00,0x1c,0xd5,0xff,0xff,0x03,0x00]
+
+v_or_b32_e64 v5, s1, s2
+// GFX12: v_or_b32_e64 v5, s1, s2                 ; encoding: [0x05,0x00,0x1c,0xd5,0x01,0x04,0x00,0x00]
+
+v_or_b32_e64 v5, s105, s105
+// GFX12: v_or_b32_e64 v5, s105, s105             ; encoding: [0x05,0x00,0x1c,0xd5,0x69,0xd2,0x00,0x00]
+
+v_or_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_or_b32_e64 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x1c,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_or_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_or_b32_e64 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x1c,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_or_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_or_b32_e64 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x1c,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_or_b32_e64 v5, m0, 0.5
+// GFX12: v_or_b32_e64 v5, m0, 0.5                ; encoding: [0x05,0x00,0x1c,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_or_b32_e64 v5, exec_lo, -1
+// GFX12: v_or_b32_e64 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x1c,0xd5,0x7e,0x82,0x01,0x00]
+
+v_or_b32_e64 v5, exec_hi, null
+// GFX12: v_or_b32_e64 v5, exec_hi, null          ; encoding: [0x05,0x00,0x1c,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_or_b32_e64 v5, null, exec_lo
+// GFX12: v_or_b32_e64 v5, null, exec_lo          ; encoding: [0x05,0x00,0x1c,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_or_b32_e64 v5, -1, exec_hi
+// GFX12: v_or_b32_e64 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x1c,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_or_b32_e64 v5, 0.5, m0
+// GFX12: v_or_b32_e64 v5, 0.5, m0                ; encoding: [0x05,0x00,0x1c,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_or_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_or_b32_e64 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x1c,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_or_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_or_b32_e64 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x1c,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x21,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_sub_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x21,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_sub_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x21,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_sub_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x21,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_sub_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x21,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_sub_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x21,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_sub_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_sub_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x21,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_sub_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x21,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x21,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:25: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX12: v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x21,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_sub_f16_e64 v5, v1, v2
+// GFX12: v_sub_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16_e64 v5, v255, v255
+// GFX12: v_sub_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f16_e64 v5, s1, s2
+// GFX12: v_sub_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f16_e64 v5, s105, s105
+// GFX12: v_sub_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_sub_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_sub_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_sub_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_sub_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_f16_e64 v5, m0, 0.5
+// GFX12: v_sub_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_f16_e64 v5, exec_lo, -1
+// GFX12: v_sub_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_f16_e64 v5, |exec_hi|, null
+// GFX12: v_sub_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_f16_e64 v5, null, exec_lo
+// GFX12: v_sub_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_f16_e64 v5, -1, exec_hi
+// GFX12: v_sub_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_sub_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_sub_f32_e64 v5, v1, v2
+// GFX12: v_sub_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f32_e64 v5, v255, v255
+// GFX12: v_sub_f32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x04,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f32_e64 v5, s1, s2
+// GFX12: v_sub_f32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f32_e64 v5, s105, s105
+// GFX12: v_sub_f32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x04,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_sub_f32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x04,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_sub_f32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x04,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_sub_f32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x04,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_f32_e64 v5, m0, 0.5
+// GFX12: v_sub_f32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x04,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_f32_e64 v5, exec_lo, -1
+// GFX12: v_sub_f32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x04,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_f32_e64 v5, |exec_hi|, null
+// GFX12: v_sub_f32_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x04,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_f32_e64 v5, null, exec_lo
+// GFX12: v_sub_f32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x04,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_f32_e64 v5, -1, exec_hi
+// GFX12: v_sub_f32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x04,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_sub_f32_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x04,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_sub_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_sub_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x04,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_sub_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_sub_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x04,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32_e64 v5, v1, v2
+// GFX12: v_sub_nc_u32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x26,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_nc_u32_e64 v5, v255, v255
+// GFX12: v_sub_nc_u32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x26,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_nc_u32_e64 v5, s1, s2
+// GFX12: v_sub_nc_u32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x26,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, s105, s105
+// GFX12: v_sub_nc_u32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x26,0xd5,0x69,0xd2,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_sub_nc_u32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x26,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_sub_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_sub_nc_u32_e64 v5, ttmp15, src_scc
+// GFX12: v_sub_nc_u32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x26,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, m0, 0.5
+// GFX12: v_sub_nc_u32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x26,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, exec_lo, -1
+// GFX12: v_sub_nc_u32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x26,0xd5,0x7e,0x82,0x01,0x00]
+
+v_sub_nc_u32_e64 v5, exec_hi, null
+// GFX12: v_sub_nc_u32_e64 v5, exec_hi, null      ; encoding: [0x05,0x00,0x26,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, null, exec_lo
+// GFX12: v_sub_nc_u32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x26,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, -1, exec_hi
+// GFX12: v_sub_nc_u32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x26,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, 0.5, m0
+// GFX12: v_sub_nc_u32_e64 v5, 0.5, m0            ; encoding: [0x05,0x00,0x26,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_sub_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX12: v_sub_nc_u32_e64 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x26,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_sub_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX12: v_sub_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, v255, src_scc, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, v255, src_scc, s3 ; encoding: [0x05,0x06,0x22,0xd5,0xff,0xfb,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, s105, s105, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, s105, s105, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x69,0xd2,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, vcc_lo, v2, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x6a,0x04,0x0e,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, vcc_hi, v255, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x6b,0xfe,0x0f,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, ttmp15, ttmp15, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7b,0xf6,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, m0, 0.5, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, m0, 0.5, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7d,0xe0,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, exec_lo, exec_lo, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7e,0xfc,0x0c,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s6, exec_hi, -1, s3
+// W32: v_subrev_co_ci_u32_e64 v5, s6, exec_hi, -1, s3 ; encoding: [0x05,0x06,0x22,0xd5,0x7f,0x82,0x0d,0x00]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s105, null, exec_hi, s105
+// W32: v_subrev_co_ci_u32_e64 v5, s105, null, exec_hi, s105 ; encoding: [0x05,0x69,0x22,0xd5,0x7c,0xfe,0xa4,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo
+// W32: v_subrev_co_ci_u32_e64 v5, vcc_lo, -1, m0, vcc_lo ; encoding: [0x05,0x6a,0x22,0xd5,0xc1,0xfa,0xa8,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi
+// W32: v_subrev_co_ci_u32_e64 v5, vcc_hi, 0.5, vcc_lo, vcc_hi ; encoding: [0x05,0x6b,0x22,0xd5,0xf0,0xd4,0xac,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15
+// W32: v_subrev_co_ci_u32_e64 v5, ttmp15, src_scc, null, ttmp15 ; encoding: [0x05,0x7b,0x22,0xd5,0xfd,0xf8,0xec,0x01]
+// W64-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], v255, src_scc, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0xff,0xfb,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x69,0xd2,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_lo, v2, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x6a,0x04,0x1a,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], vcc_hi, v255, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x6b,0xfe,0x1b,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7b,0xf6,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7d,0xe0,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7e,0xfc,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], exec_hi, -1, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7f,0x82,0x19,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7]
+// W64: v_subrev_co_ci_u32_e64 v5, s[12:13], null, exec_hi, s[6:7] ; encoding: [0x05,0x0c,0x22,0xd5,0x7c,0xfe,0x18,0x00]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105]
+// W64: v_subrev_co_ci_u32_e64 v5, s[104:105], -1, m0, s[104:105] ; encoding: [0x05,0x68,0x22,0xd5,0xc1,0xfa,0xa0,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc
+// W64: v_subrev_co_ci_u32_e64 v5, vcc, 0.5, vcc_lo, vcc ; encoding: [0x05,0x6a,0x22,0xd5,0xf0,0xd4,0xa8,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
+// W64: v_subrev_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15] ; encoding: [0x05,0x7a,0x22,0xd5,0xfd,0xf8,0xe8,0x01]
+// W32-ERR: :[[@LINE-2]]:28: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
+// GFX12: v_subrev_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x22,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+
+v_subrev_f16_e64 v5, v1, v2
+// GFX12: v_subrev_f16_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x34,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_f16_e64 v5, v255, v255
+// GFX12: v_subrev_f16_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x34,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_f16_e64 v5, s1, s2
+// GFX12: v_subrev_f16_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x34,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_f16_e64 v5, s105, s105
+// GFX12: v_subrev_f16_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x34,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_f16_e64 v5, vcc_lo, ttmp15
+// GFX12: v_subrev_f16_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x34,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_f16_e64 v5, vcc_hi, 0xfe0b
+// GFX12: v_subrev_f16_e64 v5, vcc_hi, 0xfe0b     ; encoding: [0x05,0x00,0x34,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f16_e64 v5, ttmp15, src_scc
+// GFX12: v_subrev_f16_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x34,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_f16_e64 v5, m0, 0.5
+// GFX12: v_subrev_f16_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x34,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_f16_e64 v5, exec_lo, -1
+// GFX12: v_subrev_f16_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x34,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_f16_e64 v5, |exec_hi|, null
+// GFX12: v_subrev_f16_e64 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x34,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_f16_e64 v5, null, exec_lo
+// GFX12: v_subrev_f16_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x34,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_f16_e64 v5, -1, exec_hi
+// GFX12: v_subrev_f16_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x34,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_f16_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_subrev_f16_e64 v5, 0.5, -m0 mul:2     ; encoding: [0x05,0x00,0x34,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_subrev_f16_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_subrev_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x34,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_subrev_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_subrev_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x34,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_subrev_f32_e64 v5, v1, v2
+// GFX12: v_subrev_f32_e64 v5, v1, v2             ; encoding: [0x05,0x00,0x05,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_f32_e64 v5, v255, v255
+// GFX12: v_subrev_f32_e64 v5, v255, v255         ; encoding: [0x05,0x00,0x05,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_f32_e64 v5, s1, s2
+// GFX12: v_subrev_f32_e64 v5, s1, s2             ; encoding: [0x05,0x00,0x05,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_f32_e64 v5, s105, s105
+// GFX12: v_subrev_f32_e64 v5, s105, s105         ; encoding: [0x05,0x00,0x05,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_f32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_subrev_f32_e64 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x05,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_f32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_subrev_f32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x05,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_f32_e64 v5, ttmp15, src_scc
+// GFX12: v_subrev_f32_e64 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x05,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_f32_e64 v5, m0, 0.5
+// GFX12: v_subrev_f32_e64 v5, m0, 0.5            ; encoding: [0x05,0x00,0x05,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_f32_e64 v5, exec_lo, -1
+// GFX12: v_subrev_f32_e64 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x05,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_f32_e64 v5, |exec_hi|, null
+// GFX12: v_subrev_f32_e64 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x05,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_f32_e64 v5, null, exec_lo
+// GFX12: v_subrev_f32_e64 v5, null, exec_lo      ; encoding: [0x05,0x00,0x05,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_f32_e64 v5, -1, exec_hi
+// GFX12: v_subrev_f32_e64 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x05,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_f32_e64 v5, 0.5, -m0 mul:2
+// GFX12: v_subrev_f32_e64 v5, 0.5, -m0 mul:2     ; encoding: [0x05,0x00,0x05,0xd5,0xf0,0xfa,0x00,0x48]
+
+v_subrev_f32_e64 v5, -src_scc, |vcc_lo| mul:4
+// GFX12: v_subrev_f32_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x05,0xd5,0xfd,0xd4,0x00,0x30]
+
+v_subrev_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2
+// GFX12: v_subrev_f32_e64 v255, -|0xaf123456|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x05,0xd5,0xff,0xd6,0x00,0x78,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32_e64 v5, v1, v2
+// GFX12: v_subrev_nc_u32_e64 v5, v1, v2          ; encoding: [0x05,0x00,0x27,0xd5,0x01,0x05,0x02,0x00]
+
+v_subrev_nc_u32_e64 v5, v255, v255
+// GFX12: v_subrev_nc_u32_e64 v5, v255, v255      ; encoding: [0x05,0x00,0x27,0xd5,0xff,0xff,0x03,0x00]
+
+v_subrev_nc_u32_e64 v5, s1, s2
+// GFX12: v_subrev_nc_u32_e64 v5, s1, s2          ; encoding: [0x05,0x00,0x27,0xd5,0x01,0x04,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, s105, s105
+// GFX12: v_subrev_nc_u32_e64 v5, s105, s105      ; encoding: [0x05,0x00,0x27,0xd5,0x69,0xd2,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_subrev_nc_u32_e64 v5, vcc_lo, ttmp15  ; encoding: [0x05,0x00,0x27,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_subrev_nc_u32_e64 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x27,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_subrev_nc_u32_e64 v5, ttmp15, src_scc
+// GFX12: v_subrev_nc_u32_e64 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x27,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, m0, 0.5
+// GFX12: v_subrev_nc_u32_e64 v5, m0, 0.5         ; encoding: [0x05,0x00,0x27,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, exec_lo, -1
+// GFX12: v_subrev_nc_u32_e64 v5, exec_lo, -1     ; encoding: [0x05,0x00,0x27,0xd5,0x7e,0x82,0x01,0x00]
+
+v_subrev_nc_u32_e64 v5, exec_hi, null
+// GFX12: v_subrev_nc_u32_e64 v5, exec_hi, null   ; encoding: [0x05,0x00,0x27,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, null, exec_lo
+// GFX12: v_subrev_nc_u32_e64 v5, null, exec_lo   ; encoding: [0x05,0x00,0x27,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, -1, exec_hi
+// GFX12: v_subrev_nc_u32_e64 v5, -1, exec_hi     ; encoding: [0x05,0x00,0x27,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, 0.5, m0
+// GFX12: v_subrev_nc_u32_e64 v5, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_subrev_nc_u32_e64 v5, src_scc, vcc_lo
+// GFX12: v_subrev_nc_u32_e64 v5, src_scc, vcc_lo ; encoding: [0x05,0x00,0x27,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_subrev_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp
+// GFX12: v_subrev_nc_u32_e64 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x27,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32_e64 v5, v1, v2
+// GFX12: v_xnor_b32_e64 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd5,0x01,0x05,0x02,0x00]
+
+v_xnor_b32_e64 v5, v255, v255
+// GFX12: v_xnor_b32_e64 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd5,0xff,0xff,0x03,0x00]
+
+v_xnor_b32_e64 v5, s1, s2
+// GFX12: v_xnor_b32_e64 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd5,0x01,0x04,0x00,0x00]
+
+v_xnor_b32_e64 v5, s105, s105
+// GFX12: v_xnor_b32_e64 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd5,0x69,0xd2,0x00,0x00]
+
+v_xnor_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_xnor_b32_e64 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_xnor_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_xnor_b32_e64 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_xnor_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_xnor_b32_e64 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_xnor_b32_e64 v5, m0, 0.5
+// GFX12: v_xnor_b32_e64 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_xnor_b32_e64 v5, exec_lo, -1
+// GFX12: v_xnor_b32_e64 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd5,0x7e,0x82,0x01,0x00]
+
+v_xnor_b32_e64 v5, exec_hi, null
+// GFX12: v_xnor_b32_e64 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_xnor_b32_e64 v5, null, exec_lo
+// GFX12: v_xnor_b32_e64 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_xnor_b32_e64 v5, -1, exec_hi
+// GFX12: v_xnor_b32_e64 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_xnor_b32_e64 v5, 0.5, m0
+// GFX12: v_xnor_b32_e64 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_xnor_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_xnor_b32_e64 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_xnor_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_xnor_b32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_xor_b32_e64 v5, v1, v2
+// GFX12: v_xor_b32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x1d,0xd5,0x01,0x05,0x02,0x00]
+
+v_xor_b32_e64 v5, v255, v255
+// GFX12: v_xor_b32_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x1d,0xd5,0xff,0xff,0x03,0x00]
+
+v_xor_b32_e64 v5, s1, s2
+// GFX12: v_xor_b32_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x1d,0xd5,0x01,0x04,0x00,0x00]
+
+v_xor_b32_e64 v5, s105, s105
+// GFX12: v_xor_b32_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x1d,0xd5,0x69,0xd2,0x00,0x00]
+
+v_xor_b32_e64 v5, vcc_lo, ttmp15
+// GFX12: v_xor_b32_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x1d,0xd5,0x6a,0xf6,0x00,0x00]
+
+v_xor_b32_e64 v5, vcc_hi, 0xaf123456
+// GFX12: v_xor_b32_e64 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x1d,0xd5,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_xor_b32_e64 v5, ttmp15, src_scc
+// GFX12: v_xor_b32_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x1d,0xd5,0x7b,0xfa,0x01,0x00]
+
+v_xor_b32_e64 v5, m0, 0.5
+// GFX12: v_xor_b32_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x1d,0xd5,0x7d,0xe0,0x01,0x00]
+
+v_xor_b32_e64 v5, exec_lo, -1
+// GFX12: v_xor_b32_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x1d,0xd5,0x7e,0x82,0x01,0x00]
+
+v_xor_b32_e64 v5, exec_hi, null
+// GFX12: v_xor_b32_e64 v5, exec_hi, null         ; encoding: [0x05,0x00,0x1d,0xd5,0x7f,0xf8,0x00,0x00]
+
+v_xor_b32_e64 v5, null, exec_lo
+// GFX12: v_xor_b32_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x1d,0xd5,0x7c,0xfc,0x00,0x00]
+
+v_xor_b32_e64 v5, -1, exec_hi
+// GFX12: v_xor_b32_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x1d,0xd5,0xc1,0xfe,0x00,0x00]
+
+v_xor_b32_e64 v5, 0.5, m0
+// GFX12: v_xor_b32_e64 v5, 0.5, m0               ; encoding: [0x05,0x00,0x1d,0xd5,0xf0,0xfa,0x00,0x00]
+
+v_xor_b32_e64 v5, src_scc, vcc_lo
+// GFX12: v_xor_b32_e64 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x1d,0xd5,0xfd,0xd4,0x00,0x00]
+
+v_xor_b32_e64 v255, 0xaf123456, vcc_hi
+// GFX12: v_xor_b32_e64 v255, 0xaf123456, vcc_hi  ; encoding: [0xff,0x00,0x1d,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s
index 9675add7fb12d..04caa4a8b5177 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3
 // W32: v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 ; encoding: [0x05,0x06,0x20,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf]
@@ -111,50 +111,59 @@ v_add_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
 v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
 // GFX12: v_add_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x20,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-v_add_f16_e64 v5, v1, v2
-// GFX12: v_add_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
+v_add_f16_e64 v5.l, v1.l, v2.l
+// GFX12: v_add_f16_e64 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x05,0x02,0x00]
 
-v_add_f16_e64 v5, v255, v255
-// GFX12: v_add_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
+v_add_f16_e64 v5.l, v255.l, v255.l
+// GFX12: v_add_f16_e64 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x32,0xd5,0xff,0xff,0x03,0x00]
 
-v_add_f16_e64 v5, s1, s2
-// GFX12: v_add_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
+v_add_f16_e64 v5.l, s1, s2
+// GFX12: v_add_f16_e64 v5.l, s1, s2              ; encoding: [0x05,0x00,0x32,0xd5,0x01,0x04,0x00,0x00]
 
-v_add_f16_e64 v5, s105, s105
-// GFX12: v_add_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
+v_add_f16_e64 v5.l, s105, s105
+// GFX12: v_add_f16_e64 v5.l, s105, s105          ; encoding: [0x05,0x00,0x32,0xd5,0x69,0xd2,0x00,0x00]
 
-v_add_f16_e64 v5, vcc_lo, ttmp15
-// GFX12: v_add_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
+v_add_f16_e64 v5.l, vcc_lo, ttmp15
+// GFX12: v_add_f16_e64 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x32,0xd5,0x6a,0xf6,0x00,0x00]
 
-v_add_f16_e64 v5, vcc_hi, 0xfe0b
-// GFX12: v_add_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+v_add_f16_e64 v5.l, vcc_hi, 0xfe0b
+// GFX12: v_add_f16_e64 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x32,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-v_add_f16_e64 v5, ttmp15, src_scc
-// GFX12: v_add_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
+v_add_f16_e64 v5.l, ttmp15, src_scc
+// GFX12: v_add_f16_e64 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x32,0xd5,0x7b,0xfa,0x01,0x00]
 
-v_add_f16_e64 v5, m0, 0.5
-// GFX12: v_add_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
+v_add_f16_e64 v5.l, m0, 0.5
+// GFX12: v_add_f16_e64 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x32,0xd5,0x7d,0xe0,0x01,0x00]
 
-v_add_f16_e64 v5, exec_lo, -1
-// GFX12: v_add_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
+v_add_f16_e64 v5.l, exec_lo, -1
+// GFX12: v_add_f16_e64 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x32,0xd5,0x7e,0x82,0x01,0x00]
 
-v_add_f16_e64 v5, |exec_hi|, null
-// GFX12: v_add_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
+v_add_f16_e64 v5.l, |exec_hi|, null
+// GFX12: v_add_f16_e64 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x32,0xd5,0x7f,0xf8,0x00,0x00]
 
-v_add_f16_e64 v5, null, exec_lo
-// GFX12: v_add_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
+v_add_f16_e64 v5.l, null, exec_lo
+// GFX12: v_add_f16_e64 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x32,0xd5,0x7c,0xfc,0x00,0x00]
 
-v_add_f16_e64 v5, -1, exec_hi
-// GFX12: v_add_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
+v_add_f16_e64 v5.l, -1, exec_hi
+// GFX12: v_add_f16_e64 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x32,0xd5,0xc1,0xfe,0x00,0x00]
 
-v_add_f16_e64 v5, 0.5, -m0 mul:2
-// GFX12: v_add_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
+v_add_f16_e64 v5.l, 0.5, -m0 mul:2
+// GFX12: v_add_f16_e64 v5.l, 0.5, -m0 mul:2      ; encoding: [0x05,0x00,0x32,0xd5,0xf0,0xfa,0x00,0x48]
 
-v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4
-// GFX12: v_add_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
+v_add_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4
+// GFX12: v_add_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x32,0xd5,0xfd,0xd4,0x00,0x30]
 
-v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
-// GFX12: v_add_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+v_add_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_add_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+
+v_add_f16_e64 v5.l, v1.h, v2.l
+// GFX12: v_add_f16_e64 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x32,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f16_e64 v5.l, v255.l, v255.h
+// GFX12: v_add_f16_e64 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x32,0xd5,0xff,0xff,0x03,0x00]
+
+v_add_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_add_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| op_sel:[0,0,1] clamp div:2 ; encoding: [0xff,0xc3,0x32,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
 v_add_f32_e64 v5, v1, v2
 // GFX12: v_add_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x00]
@@ -1801,50 +1810,59 @@ v_sub_co_ci_u32_e64 v5, ttmp[14:15], src_scc, null, ttmp[14:15]
 v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp
 // GFX12: v_sub_co_ci_u32_e64 v255, null, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0xfc,0x21,0xd5,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-v_sub_f16_e64 v5, v1, v2
-// GFX12: v_sub_f16_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+v_sub_f16_e64 v5.l, v1.l, v2.l
+// GFX12: v_sub_f16_e64 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x05,0x02,0x00]
+
+v_sub_f16_e64 v5.l, v255.l, v255.l
+// GFX12: v_sub_f16_e64 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+
+v_sub_f16_e64 v5.l, s1, s2
+// GFX12: v_sub_f16_e64 v5.l, s1, s2              ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+
+v_sub_f16_e64 v5.l, s105, s105
+// GFX12: v_sub_f16_e64 v5.l, s105, s105          ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
 
-v_sub_f16_e64 v5, v255, v255
-// GFX12: v_sub_f16_e64 v5, v255, v255            ; encoding: [0x05,0x00,0x33,0xd5,0xff,0xff,0x03,0x00]
+v_sub_f16_e64 v5.l, vcc_lo, ttmp15
+// GFX12: v_sub_f16_e64 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
 
-v_sub_f16_e64 v5, s1, s2
-// GFX12: v_sub_f16_e64 v5, s1, s2                ; encoding: [0x05,0x00,0x33,0xd5,0x01,0x04,0x00,0x00]
+v_sub_f16_e64 v5.l, vcc_hi, 0xfe0b
+// GFX12: v_sub_f16_e64 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, s105, s105
-// GFX12: v_sub_f16_e64 v5, s105, s105            ; encoding: [0x05,0x00,0x33,0xd5,0x69,0xd2,0x00,0x00]
+v_sub_f16_e64 v5.l, ttmp15, src_scc
+// GFX12: v_sub_f16_e64 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
 
-v_sub_f16_e64 v5, vcc_lo, ttmp15
-// GFX12: v_sub_f16_e64 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x33,0xd5,0x6a,0xf6,0x00,0x00]
+v_sub_f16_e64 v5.l, m0, 0.5
+// GFX12: v_sub_f16_e64 v5.l, m0, 0.5             ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
 
-v_sub_f16_e64 v5, vcc_hi, 0xfe0b
-// GFX12: v_sub_f16_e64 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x33,0xd5,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+v_sub_f16_e64 v5.l, exec_lo, -1
+// GFX12: v_sub_f16_e64 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
 
-v_sub_f16_e64 v5, ttmp15, src_scc
-// GFX12: v_sub_f16_e64 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x33,0xd5,0x7b,0xfa,0x01,0x00]
+v_sub_f16_e64 v5.l, |exec_hi|, null
+// GFX12: v_sub_f16_e64 v5.l, |exec_hi|, null     ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
 
-v_sub_f16_e64 v5, m0, 0.5
-// GFX12: v_sub_f16_e64 v5, m0, 0.5               ; encoding: [0x05,0x00,0x33,0xd5,0x7d,0xe0,0x01,0x00]
+v_sub_f16_e64 v5.l, null, exec_lo
+// GFX12: v_sub_f16_e64 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
 
-v_sub_f16_e64 v5, exec_lo, -1
-// GFX12: v_sub_f16_e64 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x33,0xd5,0x7e,0x82,0x01,0x00]
+v_sub_f16_e64 v5.l, -1, exec_hi
+// GFX12: v_sub_f16_e64 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, |exec_hi|, null
-// GFX12: v_sub_f16_e64 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x33,0xd5,0x7f,0xf8,0x00,0x00]
+v_sub_f16_e64 v5.l, 0.5, -m0 mul:2
+// GFX12: v_sub_f16_e64 v5.l, 0.5, -m0 mul:2      ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
 
-v_sub_f16_e64 v5, null, exec_lo
-// GFX12: v_sub_f16_e64 v5, null, exec_lo         ; encoding: [0x05,0x00,0x33,0xd5,0x7c,0xfc,0x00,0x00]
+v_sub_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4
+// GFX12: v_sub_f16_e64 v5.l, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
 
-v_sub_f16_e64 v5, -1, exec_hi
-// GFX12: v_sub_f16_e64 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x33,0xd5,0xc1,0xfe,0x00,0x00]
+v_sub_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_sub_f16_e64 v255.l, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
-v_sub_f16_e64 v5, 0.5, -m0 mul:2
-// GFX12: v_sub_f16_e64 v5, 0.5, -m0 mul:2        ; encoding: [0x05,0x00,0x33,0xd5,0xf0,0xfa,0x00,0x48]
+v_sub_f16_e64 v5.l, v1.h, v2.l
+// GFX12: v_sub_f16_e64 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x33,0xd5,0x01,0x05,0x02,0x00]
 
-v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4
-// GFX12: v_sub_f16_e64 v5, -src_scc, |vcc_lo| mul:4 ; encoding: [0x05,0x02,0x33,0xd5,0xfd,0xd4,0x00,0x30]
+v_sub_f16_e64 v5.l, v255.l, v255.h
+// GFX12: v_sub_f16_e64 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x33,0xd5,0xff,0xff,0x03,0x00]
 
-v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2
-// GFX12: v_sub_f16_e64 v255, -|0xfe0b|, -|vcc_hi| clamp div:2 ; encoding: [0xff,0x83,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
+v_sub_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| clamp div:2
+// GFX12: v_sub_f16_e64 v255.h, -|0xfe0b|, -|vcc_hi| op_sel:[0,0,1] clamp div:2 ; encoding: [0xff,0xc3,0x33,0xd5,0xff,0xd6,0x00,0x78,0x0b,0xfe,0x00,0x00]
 
 v_sub_f32_e64 v5, v1, v2
 // GFX12: v_sub_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x04,0xd5,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16-fake16.s
new file mode 100644
index 0000000000000..81c11916f1243
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16-fake16.s
@@ -0,0 +1,2113 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x20,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x20,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x20,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x20,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x20,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x20,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x20,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x20,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_add_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_add_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_add_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_add_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x03,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x03,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x03,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x25,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_add_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x25,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_and_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_and_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_and_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_and_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_and_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_and_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_and_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1b,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_ashrrev_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_ashrrev_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1a,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_mirror
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_half_mirror
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_ror:1
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s105 row_ror:15
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x01,0xd5,0xfa,0x04,0xaa,0x41,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x01,0xd5,0xfa,0x04,0xee,0x21,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_mirror
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_half_mirror
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:1
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:15
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x01,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x01,0xd5,0xfa,0x04,0xaa,0x41,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x01,0xd5,0xfa,0x04,0xea,0x21,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x03,0x01,0xd5,0xfa,0xfe,0xf3,0x61,0xff,0x6f,0x05,0x30]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x2f,0xd5,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x2f,0xd5,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x2f,0xd5,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x2f,0xd5,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x3b,0xd5,0xfa,0x04,0x02,0x10,0x01,0x60,0x09,0x13]
+
+v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0x3b,0xd5,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x05,0x30]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x18,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_lshlrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_lshlrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x18,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x19,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_lshrrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_lshrrev_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x19,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_max_num_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_max_num_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_max_num_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_num_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_max_num_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x31,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_max_num_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_max_num_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x31,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_max_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_max_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x31,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_max_num_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_max_num_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_max_num_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_num_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_max_num_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x16,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_max_num_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_max_num_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x16,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_max_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_max_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x16,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_max_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_max_i32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_max_i32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_max_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x12,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_max_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_max_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x12,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_max_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_max_u32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_max_u32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_max_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_max_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x14,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_max_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_max_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x14,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_min_num_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_min_num_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_min_num_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_num_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_min_num_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x30,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_min_num_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_min_num_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x30,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_min_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_min_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x30,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_min_num_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_min_num_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_min_num_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_num_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_min_num_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x15,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_min_num_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_min_num_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x15,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_min_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_min_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x15,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_min_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_min_i32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_min_i32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_min_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x11,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_min_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_min_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x11,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_min_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_min_u32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_min_u32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_min_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_min_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x13,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_min_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_min_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x13,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x07,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x07,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_mul_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x35,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x35,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x35,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x35,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_mul_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x08,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x08,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x08,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x08,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0a,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_hi_i32_i24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x0a,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_hi_u32_u24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x0c,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x09,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_i32_i24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_i32_i24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x09,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_legacy_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_mul_legacy_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x07,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_mul_legacy_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x07,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x0b,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_mul_u32_u24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_mul_u32_u24_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x0b,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_or_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_or_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_or_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_or_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_or_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1c,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_or_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_or_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1c,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x21,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x21,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x21,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x21,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x21,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x21,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x21,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x21,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x21,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x21,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_sub_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_sub_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x04,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x04,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x04,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x26,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_sub_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x26,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0xe4,0x00,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x01,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x0f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x11,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x22,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x21,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x69,0x22,0xd5,0xfa,0x04,0xa6,0x01,0x01,0x2f,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x6a,0x22,0xd5,0xfa,0x04,0xae,0x01,0x01,0x50,0x01,0xff]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6b,0x22,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W32: v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7b,0x22,0xd5,0xfa,0x04,0xee,0x01,0x01,0x60,0x09,0x13]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x22,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x22,0xd5,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x22,0xd5,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// W64: v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x7a,0x22,0xd5,0xfa,0x04,0xea,0x01,0x01,0x60,0x09,0x13]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x22,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
+
+v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_subrev_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_subrev_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x34,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x34,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x34,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_subrev_f32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, 2.0 row_shl:15
+// GFX12: v_subrev_f32_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x05,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x05,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x05,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x27,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x80,0x27,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_xnor_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_xnor_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_xnor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1e,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_xnor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_xnor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1e,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
+
+v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_mirror
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_half_mirror
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shl:1
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shl:15
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, s2 row_shl:15
+// GFX12: v_xor_b32_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, 10 row_shl:15
+// GFX12: v_xor_b32_e64_dpp v5, v1, 10 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x14,0x01,0x00,0x01,0x0f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shr:1
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_shr:15
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_ror:1
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_ror:15
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
+
+v_xor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x00,0x1d,0xd5,0xfa,0x04,0x02,0x00,0x01,0x60,0x09,0x13]
+
+v_xor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_xor_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x1d,0xd5,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
index 919b1cfa16932..7991b87583aac 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
@@ -111,53 +111,65 @@ v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mas
 v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x20,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
 
-v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_mirror
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_mirror
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_half_mirror
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shl:1
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shl:15
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, s2 row_shl:15
-// GFX12: v_add_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, s2 row_shl:15
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, 2.0 row_shl:15
-// GFX12: v_add_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, 2.0 row_shl:15
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shr:1
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_shr:15
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_ror:1
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_ror:15
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+
+v_add_f16_e64_dpp v5.h, v1.h, v2.h row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_add_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x32,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x09,0x32,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x12,0x32,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+
+v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc3,0x32,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
 v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
@@ -1621,53 +1633,65 @@ v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] row_xmask:0 row_mas
 v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xfc,0x21,0xd5,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x05,0x30]
 
-v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_mirror
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3]
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_mirror
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, s2 row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, 2.0 row_shl:15
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shl:1
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shl:15
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, s2 row_shl:15
-// GFX12: v_sub_f16_e64_dpp v5, v1, s2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x00,0x00,0x01,0x0f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, 2.0 row_shl:15
-// GFX12: v_sub_f16_e64_dpp v5, v1, 2.0 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0xe8,0x01,0x00,0x01,0x0f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shr:1
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_shr:15
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_ror:1
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_ror:15
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
-v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+v_sub_f16_e64_dpp v5.h, v1.h, v2.h row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: v_sub_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x33,0xd5,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
-// GFX12: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
+v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x09,0x33,0xd5,0xfa,0x04,0x02,0x48,0x01,0x5f,0x01,0x01]
 
-v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
-// GFX12: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x12,0x33,0xd5,0xfa,0x04,0x02,0x30,0x01,0x60,0x09,0x13]
 
-v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x83,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
+v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc3,0x33,0xd5,0xfa,0xfe,0x03,0x78,0xff,0x6f,0x05,0x30]
 
 v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_sub_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8-fake16.s
new file mode 100644
index 0000000000000..916f7a740dc38
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8-fake16.s
@@ -0,0 +1,768 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+
+v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x20,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x20,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_add_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x20,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_add_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x20,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x20,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x20,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_add_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x20,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x20,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x20,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x03,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x03,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x03,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x25,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x25,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_and_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_and_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_and_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1b,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_and_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_and_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1b,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_ashrrev_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1a,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ashrrev_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_ashrrev_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1a,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, 10, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, 10, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x14,0x0d,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, s2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, s2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa4,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, s2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, v1, s2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xac,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x01,0xd5,0xe9,0x04,0xaa,0x41,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x01,0xd5,0xea,0x04,0xee,0x21,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, s2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, s2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0x18,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, s2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, s2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x04,0xa0,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, v1, 10, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, v1, 10, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x01,0xd5,0xe9,0x14,0xa1,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:35: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_cndmask_b32_e64_dpp v5, |v1|, -v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x01,0xd5,0xe9,0x04,0xaa,0x41,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_cndmask_b32_e64_dpp v5, -v1, |v2|, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x01,0xd5,0xea,0x04,0xea,0x21,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:38: error: invalid operand for instruction
+
+v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cndmask_b32_e64_dpp v255, -|v255|, -|v255|, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x01,0xd5,0xe9,0xfe,0xf3,0x61,0xff,0x00,0x00,0x00]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd5,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x2f,0xd5,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd5,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x2f,0xd5,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cvt_pkrtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_cvt_pk_rtz_f16_f32_e64_dpp v255, -|v255|, -|v255| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x2f,0xd5,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, s2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, s2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0x04,0x00,0x08,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, 2.0 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, 2.0 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3b,0xd5,0xe9,0xe8,0x01,0x08,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_ldexp_f16_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x3b,0xd5,0xea,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
+
+v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_ldexp_f16_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x3b,0xd5,0xe9,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_lshlrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x18,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshlrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_lshlrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x18,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_lshrrev_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x19,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_lshrrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_lshrrev_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x19,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_max_num_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x31,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_max_num_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_max_num_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x31,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_max_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_max_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x31,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_max_num_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_max_num_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_num_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x16,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_max_num_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_max_num_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x16,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_max_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_max_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x16,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_max_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x12,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_max_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x12,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x14,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x14,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_max_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x14,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_max_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x14,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_max_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_max_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x14,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_min_num_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_min_num_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_min_num_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x30,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_min_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_min_num_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x30,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_min_num_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_min_num_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_num_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x15,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_min_num_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_min_num_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x15,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_min_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_min_num_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x15,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_i32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_i32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_min_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x11,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_min_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x11,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_min_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_min_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x13,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_min_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_min_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x13,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x07,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x07,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x35,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x35,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x35,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x35,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x08,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x08,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x08,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x08,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x08,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x08,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_hi_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0a,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_i32_i24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_hi_i32_i24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x0a,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_hi_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0c,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_hi_u32_u24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_hi_u32_u24_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x0c,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_i32_i24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x09,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_i32_i24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_i32_i24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x09,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_mul_legacy_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x07,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_mul_legacy_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_dx9_zero_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x07,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_mul_u32_u24_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x0b,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_mul_u32_u24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_mul_u32_u24_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x0b,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_or_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_or_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_or_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1c,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_or_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_or_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1c,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x21,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x21,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x21,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_sub_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x21,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_sub_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x21,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x21,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x21,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_sub_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x21,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x21,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:29: error: invalid operand for instruction
+
+v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x21,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x04,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x04,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x04,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x26,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x26,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x22,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, s105, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x69,0x22,0xd5,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_lo, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x22,0xd5,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// W32: v_subrev_co_ci_u32_e64_dpp v5, vcc_hi, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6b,0x22,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W32: v_subrev_co_ci_u32_e64_dpp v5, ttmp15, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7b,0x22,0xd5,0xea,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+// W64-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x22,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, s[104:105], v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x22,0xd5,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0]
+// W64: v_subrev_co_ci_u32_e64_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x22,0xd5,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1
+// W64: v_subrev_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x7a,0x22,0xd5,0xea,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
+// W32-ERR: :[[@LINE-2]]:32: error: invalid operand for instruction
+
+v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_subrev_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x22,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
+
+v_subrev_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x34,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_subrev_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x34,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_subrev_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x34,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_subrev_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f32_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_f32_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x05,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_subrev_f32_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x05,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_subrev_f32_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x05,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x27,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x27,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x27,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_subrev_nc_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x27,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_subrev_nc_u32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x80,0x27,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xnor_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xnor_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_xnor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1e,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xnor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_xnor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1e,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
+
+v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xor_b32_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_xor_b32_e64_dpp v5, v1, 10 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd5,0xe9,0x14,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_xor_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x1d,0xd5,0xea,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_xor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_xor_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x1d,0xd5,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
index 1e589729dc64e..05d3ee1fa853a 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
@@ -43,23 +43,35 @@ v_add_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1
 v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_add_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x20,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, v1.l, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_add_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_add_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
 
-v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_add_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+
+v_add_f16_e64_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x32,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_add_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x09,0x32,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_add_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x12,0x32,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+
+v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_add_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc3,0x32,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
 
 v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -596,23 +608,35 @@ v_sub_co_ci_u32_e64_dpp v5, ttmp[14:15], v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1
 v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_sub_co_ci_u32_e64_dpp v255, null, v255, v255, null clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xfc,0x21,0xd5,0xe9,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, v1.l, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, |v1.l|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.l| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f16_e64_dpp v255.l, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
 
-v_sub_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd5,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x33,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sub_f16_e64_dpp v5, |v1|, -v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sub_f16_e64_dpp v5.l, |v1.h|, -v2.l op_sel:[1,0,0] mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x09,0x33,0xd5,0xe9,0x04,0x02,0x48,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_sub_f16_e64_dpp v5, -v1, |v2| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
+v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_sub_f16_e64_dpp v5.l, -v1.l, |v2.h| op_sel:[0,1,0] mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x12,0x33,0xd5,0xea,0x04,0x02,0x30,0x01,0x77,0x39,0x05]
 
-v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_sub_f16_e64_dpp v255, -|v255|, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x83,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
+v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: v_sub_f16_e64_dpp v255.h, -|v255.l|, -|v255.l| op_sel:[0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc3,0x33,0xd5,0xe9,0xfe,0x03,0x78,0xff,0x00,0x00,0x00]
 
 v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_sub_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s
index 5915cbc011863..fadd24283d211 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s
@@ -5,3 +5,9 @@ v_pk_min_f16 v0, v1, v2
 
 v_pk_max_f16 v0, v1, v2
 // GFX12: v_pk_max_num_f16 v0, v1, v2             ; encoding: [0x00,0x40,0x1c,0xcc,0x01,0x05,0x02,0x18]
+
+v_dot4_i32_i8 v5, v1, v2, s3
+// GFX12: v_dot4_i32_iu8 v5, v1, v2, s3           ; encoding: [0x05,0x40,0x16,0xcc,0x01,0x05,0x0e,0x18]
+
+v_dot8_i32_i4 v5, v1, v2, s3
+// GFX12: v_dot8_i32_iu4 v5, v1, v2, s3           ; encoding: [0x05,0x40,0x18,0xcc,0x01,0x05,0x0e,0x18]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s
index 80b107982ae38..e345f24f9a057 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s
@@ -290,6 +290,78 @@ v_dual_add_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x12,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x18,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x18,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x18,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x18,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x18,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x18,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x18,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x18,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x18,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x18,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x18,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x18,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x18,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x18,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x18,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x18,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x18,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x18,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_add_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x02,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -1370,6 +1442,78 @@ v_dual_cndmask_b32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x52,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_cndmask_b32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x58,0xca,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x58,0xca,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x58,0xca,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x58,0xca,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x58,0xca,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x69,0x04,0x58,0xca,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x01,0x04,0x58,0xca,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7b,0x04,0x58,0xca,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x58,0xca,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x7e,0x04,0x58,0xca,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x58,0xca,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x6b,0x04,0x58,0xca,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x6a,0x04,0x58,0xca,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x58,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x58,0xca,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x58,0xca,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x58,0xca,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x58,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_cndmask_b32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x42,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -2010,84 +2154,1228 @@ v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_sub_f32 v6, 0.5, v2
 // GFX12: encoding: [0xf0,0x06,0x4a,0xca,0xf0,0x04,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5
-// GFX12: encoding: [0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff]
+v_dual_cndmask_b32 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, s105, v2 :: v_dual_subrev_f32 v6, s105, v3
+// GFX12: encoding: [0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, s1, v2 :: v_dual_subrev_f32 v6, s1, v3
+// GFX12: encoding: [0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_subrev_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_cndmask_b32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x08,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x08,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x08,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x08,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x08,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x08,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x08,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x08,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x08,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x08,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x08,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x08,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x08,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x08,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x08,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x08,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x08,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x08,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_add_nc_u32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x20,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_add_nc_u32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x20,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_add_nc_u32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x20,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_add_nc_u32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x20,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_add_nc_u32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x20,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_add_nc_u32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x20,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_add_nc_u32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x20,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_add_nc_u32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x20,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_add_nc_u32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x20,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_add_nc_u32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x20,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_add_nc_u32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x20,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_add_nc_u32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x20,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_add_nc_u32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x20,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_add_nc_u32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x20,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_add_nc_u32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x20,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_add_nc_u32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x20,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_add_nc_u32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x20,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_add_nc_u32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x20,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_and_b32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x24,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_and_b32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x24,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_and_b32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x24,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_and_b32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x24,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_and_b32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x24,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_and_b32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x24,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_and_b32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x24,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_and_b32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x24,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_and_b32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x24,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_and_b32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x24,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_and_b32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x24,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_and_b32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x24,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_and_b32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x24,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_and_b32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x24,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_and_b32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x24,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_and_b32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x24,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_and_b32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x24,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_and_b32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x24,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_cndmask_b32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x12,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_cndmask_b32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x12,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_cndmask_b32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x12,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_cndmask_b32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x12,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_cndmask_b32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x12,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_cndmask_b32 v6, s105, v3
+// GFX12: encoding: [0x69,0x04,0x12,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_cndmask_b32 v6, s1, v3
+// GFX12: encoding: [0x01,0x04,0x12,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_cndmask_b32 v6, ttmp15, v3
+// GFX12: encoding: [0x7b,0x04,0x12,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_cndmask_b32 v6, exec_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x12,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_cndmask_b32 v6, exec_lo, v3
+// GFX12: encoding: [0x7e,0x04,0x12,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_cndmask_b32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x12,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_cndmask_b32 v6, vcc_hi, v3
+// GFX12: encoding: [0x6b,0x04,0x12,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_cndmask_b32 v6, vcc_lo, v3
+// GFX12: encoding: [0x6a,0x04,0x12,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_cndmask_b32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x12,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_cndmask_b32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x12,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_cndmask_b32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x12,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_cndmask_b32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x12,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x12,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x18,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x18,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x18,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x18,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x18,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x18,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x18,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x18,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x18,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x18,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x18,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x18,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x18,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x18,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x18,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x18,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x18,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x18,0xcb,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_fmaak_f32 v6, v255, v3, 0xaf123456
+// GFX12: encoding: [0x04,0x05,0x02,0xcb,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_fmaak_f32 v6, v3, v3, 0xaf123456
+// GFX12: encoding: [0x01,0x05,0x02,0xcb,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_fmaak_f32 v6, v2, v3, 0xaf123456
+// GFX12: encoding: [0xff,0x05,0x02,0xcb,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_fmaak_f32 v6, v4, v3, 0xaf123456
+// GFX12: encoding: [0x02,0x05,0x02,0xcb,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_fmaak_f32 v6, vcc_lo, v3, 0xaf123456
+// GFX12: encoding: [0x03,0x05,0x02,0xcb,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_fmaak_f32 v6, s105, v3, 0xaf123456
+// GFX12: encoding: [0x69,0x04,0x02,0xcb,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_fmaak_f32 v6, s1, v3, 0xaf123456
+// GFX12: encoding: [0x01,0x04,0x02,0xcb,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_fmaak_f32 v6, ttmp15, v3, 0xaf123456
+// GFX12: encoding: [0x7b,0x04,0x02,0xcb,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_fmaak_f32 v6, exec_hi, v3, 0xaf123456
+// GFX12: encoding: [0x7f,0x04,0x02,0xcb,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_fmaak_f32 v6, exec_lo, v3, 0xaf123456
+// GFX12: encoding: [0x7e,0x04,0x02,0xcb,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_fmaak_f32 v6, m0, v3, 0xaf123456
+// GFX12: encoding: [0x7d,0x04,0x02,0xcb,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_fmaak_f32 v6, vcc_hi, v3, 0xaf123456
+// GFX12: encoding: [0x6b,0x04,0x02,0xcb,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_fmaak_f32 v6, null, v3, 0xaf123456
+// GFX12: encoding: [0x6a,0x04,0x02,0xcb,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_fmaak_f32 v6, -1, v3, 0xaf123456
+// GFX12: encoding: [0xfd,0x04,0x02,0xcb,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmaak_f32 v6, 0.5, v2, 0xaf123456
+// GFX12: encoding: [0xf0,0x06,0x02,0xcb,0xf0,0x04,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmaak_f32 v6, src_scc, v5, 0xaf123456
+// GFX12: encoding: [0xc1,0x08,0x02,0xcb,0xfd,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmaak_f32 v255, 0xaf123456, v4, 0xaf123456
+// GFX12: encoding: [0x7c,0x0a,0x02,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_fmac_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x00,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_fmac_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x00,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_fmac_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x00,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_fmac_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x00,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_fmac_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x00,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_fmac_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x00,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_fmac_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x00,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_fmac_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x00,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_fmac_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x00,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_fmac_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x00,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_fmac_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x00,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_fmac_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x00,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_fmac_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x00,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_fmac_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x00,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_fmac_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x00,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmac_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x00,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmac_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x00,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmac_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x00,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v255 :: v_dual_fmamk_f32 v6, v255, 0xaf123456, v4
+// GFX12: encoding: [0x04,0xff,0x05,0xcb,0xff,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v255 :: v_dual_fmamk_f32 v6, v3, 0xaf123456, v4
+// GFX12: encoding: [0x01,0xff,0x05,0xcb,0x03,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v255 :: v_dual_fmamk_f32 v6, v2, 0xaf123456, v4
+// GFX12: encoding: [0xff,0xff,0x05,0xcb,0x02,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v255 :: v_dual_fmamk_f32 v6, v4, 0xaf123456, v4
+// GFX12: encoding: [0x02,0xff,0x05,0xcb,0x04,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v255 :: v_dual_fmamk_f32 v6, vcc_lo, 0xaf123456, v4
+// GFX12: encoding: [0x03,0xff,0x05,0xcb,0x6a,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v255 :: v_dual_fmamk_f32 v6, s105, 0xaf123456, v4
+// GFX12: encoding: [0x69,0xfe,0x05,0xcb,0x69,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v255 :: v_dual_fmamk_f32 v6, s1, 0xaf123456, v4
+// GFX12: encoding: [0x01,0xfe,0x05,0xcb,0x01,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v255 :: v_dual_fmamk_f32 v6, ttmp15, 0xaf123456, v4
+// GFX12: encoding: [0x7b,0xfe,0x05,0xcb,0x7b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v255 :: v_dual_fmamk_f32 v6, exec_hi, 0xaf123456, v4
+// GFX12: encoding: [0x7f,0xfe,0x05,0xcb,0x7f,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v255 :: v_dual_fmamk_f32 v6, exec_lo, 0xaf123456, v4
+// GFX12: encoding: [0x7e,0xfe,0x05,0xcb,0x7e,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v255 :: v_dual_fmamk_f32 v6, m0, 0xaf123456, v4
+// GFX12: encoding: [0x7d,0xfe,0x05,0xcb,0x7d,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v255 :: v_dual_fmamk_f32 v6, vcc_hi, 0xaf123456, v4
+// GFX12: encoding: [0x6b,0xfe,0x05,0xcb,0x6b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v255 :: v_dual_fmamk_f32 v6, null, 0xaf123456, v4
+// GFX12: encoding: [0x6a,0xfe,0x05,0xcb,0x7c,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v255 :: v_dual_fmamk_f32 v6, -1, 0xaf123456, v4
+// GFX12: encoding: [0xfd,0xfe,0x05,0xcb,0xc1,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmamk_f32 v6, 0.5, 0xaf123456, v4
+// GFX12: encoding: [0xf0,0x06,0x04,0xcb,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmamk_f32 v6, src_scc, 0xaf123456, v4
+// GFX12: encoding: [0xc1,0x08,0x04,0xcb,0xfd,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255
+// GFX12: encoding: [0x7c,0x0a,0x04,0xcb,0xff,0xfe,0xff,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_lshlrev_b32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x22,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_lshlrev_b32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x22,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_lshlrev_b32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x22,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_lshlrev_b32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x22,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_lshlrev_b32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x22,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_lshlrev_b32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x22,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_lshlrev_b32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x22,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_lshlrev_b32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x22,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_lshlrev_b32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x22,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_lshlrev_b32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x22,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_lshlrev_b32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x22,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_lshlrev_b32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x22,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_lshlrev_b32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x22,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_lshlrev_b32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x22,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_lshlrev_b32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x22,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_lshlrev_b32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x22,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_lshlrev_b32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x22,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_lshlrev_b32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x22,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_max_num_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x14,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_max_num_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x14,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_max_num_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x14,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_max_num_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x14,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_max_num_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x14,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_max_num_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x14,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_max_num_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x14,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_max_num_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x14,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_max_num_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x14,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_max_num_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x14,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_max_num_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x14,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_max_num_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x14,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_max_num_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x14,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_max_num_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x14,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_max_num_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x14,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_max_num_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x14,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_max_num_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x14,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_max_num_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x14,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_min_num_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x16,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_min_num_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x16,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_min_num_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x16,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_min_num_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x16,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_min_num_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x16,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_min_num_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x16,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_min_num_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x16,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_min_num_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x16,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_min_num_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x16,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_min_num_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x16,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_min_num_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x16,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_min_num_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x16,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_min_num_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x16,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_min_num_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x16,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_min_num_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x16,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_min_num_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x16,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_min_num_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x16,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_min_num_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x16,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v255 :: v_dual_mov_b32 v6, v1
+// GFX12: encoding: [0x04,0xff,0x11,0xcb,0x01,0x01,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v255 :: v_dual_mov_b32 v6, v255
+// GFX12: encoding: [0x01,0xff,0x11,0xcb,0xff,0x01,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v255 :: v_dual_mov_b32 v6, v2
+// GFX12: encoding: [0xff,0xff,0x11,0xcb,0x02,0x01,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v255 :: v_dual_mov_b32 v6, v3
+// GFX12: encoding: [0x02,0xff,0x11,0xcb,0x03,0x01,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v255 :: v_dual_mov_b32 v6, v4
+// GFX12: encoding: [0x03,0xff,0x11,0xcb,0x04,0x01,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v255 :: v_dual_mov_b32 v6, s1
+// GFX12: encoding: [0x69,0xfe,0x11,0xcb,0x01,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v255 :: v_dual_mov_b32 v6, s105
+// GFX12: encoding: [0x01,0xfe,0x11,0xcb,0x69,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v255 :: v_dual_mov_b32 v6, vcc_lo
+// GFX12: encoding: [0x7b,0xfe,0x11,0xcb,0x6a,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v255 :: v_dual_mov_b32 v6, vcc_hi
+// GFX12: encoding: [0x7f,0xfe,0x11,0xcb,0x6b,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v255 :: v_dual_mov_b32 v6, ttmp15
+// GFX12: encoding: [0x7e,0xfe,0x11,0xcb,0x7b,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v255 :: v_dual_mov_b32 v6, m0
+// GFX12: encoding: [0x7d,0xfe,0x11,0xcb,0x7d,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v255 :: v_dual_mov_b32 v6, exec_lo
+// GFX12: encoding: [0x6b,0xfe,0x11,0xcb,0x7e,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v255 :: v_dual_mov_b32 v6, exec_hi
+// GFX12: encoding: [0x6a,0xfe,0x11,0xcb,0x7f,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v255 :: v_dual_mov_b32 v6, null
+// GFX12: encoding: [0xff,0xfe,0x11,0xcb,0x7c,0x00,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v255 :: v_dual_mov_b32 v6, -1
+// GFX12: encoding: [0xfd,0xfe,0x11,0xcb,0xc1,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mov_b32 v6, 0.5
+// GFX12: encoding: [0xf0,0x06,0x10,0xcb,0xf0,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mov_b32 v6, src_scc
+// GFX12: encoding: [0xc1,0x08,0x10,0xcb,0xfd,0x00,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mov_b32 v255, 0xaf123456
+// GFX12: encoding: [0x7c,0x0a,0x10,0xcb,0xff,0x00,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_mul_dx9_zero_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x0e,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_mul_dx9_zero_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x0e,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_mul_dx9_zero_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x0e,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_mul_dx9_zero_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x0e,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_mul_dx9_zero_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x0e,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_mul_dx9_zero_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x0e,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_mul_dx9_zero_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x0e,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_mul_dx9_zero_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x0e,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_mul_dx9_zero_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x0e,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_mul_dx9_zero_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x0e,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_mul_dx9_zero_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x0e,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_mul_dx9_zero_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x0e,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_mul_dx9_zero_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x0e,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_mul_dx9_zero_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x0e,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_mul_dx9_zero_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x0e,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mul_dx9_zero_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x0e,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mul_dx9_zero_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x0e,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mul_dx9_zero_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x0e,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_mul_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x06,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_mul_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x06,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_mul_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x06,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_mul_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x06,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_mul_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x06,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_mul_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x06,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_mul_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x06,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_mul_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x06,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_mul_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x06,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_mul_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x06,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_mul_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x06,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_mul_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x06,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_mul_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x06,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_mul_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x06,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_mul_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x06,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mul_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x06,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mul_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x06,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mul_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x06,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_sub_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x0a,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_sub_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x0a,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_sub_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x0a,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_sub_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x0a,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_sub_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x0a,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_sub_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x0a,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_sub_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x0a,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_sub_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x0a,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_sub_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x0a,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_sub_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x0a,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_sub_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x0a,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_sub_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x0a,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_sub_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x0a,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_sub_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x0a,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_sub_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x0a,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_sub_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x0a,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x0a,0xcb,0xfd,0x0a,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4
-// GFX12: encoding: [0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x0a,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3
-// GFX12: encoding: [0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x0c,0xcb,0x01,0x07,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3
-// GFX12: encoding: [0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x0c,0xcb,0xff,0x07,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3
-// GFX12: encoding: [0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x0c,0xcb,0x02,0x07,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3
-// GFX12: encoding: [0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x0c,0xcb,0x03,0x07,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3
-// GFX12: encoding: [0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x0c,0xcb,0x04,0x07,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, s105, v2 :: v_dual_subrev_f32 v6, s105, v3
-// GFX12: encoding: [0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_subrev_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x0c,0xcb,0x01,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, s1, v2 :: v_dual_subrev_f32 v6, s1, v3
-// GFX12: encoding: [0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_subrev_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x0c,0xcb,0x69,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, ttmp15, v3
-// GFX12: encoding: [0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x0c,0xcb,0x6a,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, exec_hi, v3
-// GFX12: encoding: [0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x0c,0xcb,0x6b,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, exec_lo, v3
-// GFX12: encoding: [0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x0c,0xcb,0x7b,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3
-// GFX12: encoding: [0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x0c,0xcb,0x7d,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3
-// GFX12: encoding: [0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x0c,0xcb,0x7e,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3
-// GFX12: encoding: [0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x0c,0xcb,0x7f,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_subrev_f32 v6, null, v3
-// GFX12: encoding: [0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_subrev_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x0c,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3
-// GFX12: encoding: [0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x0c,0xcb,0xc1,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2
-// GFX12: encoding: [0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x0c,0xcb,0xf0,0x04,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5
-// GFX12: encoding: [0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff]
+v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x0c,0xcb,0xfd,0x0a,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_cndmask_b32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4
-// GFX12: encoding: [0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x0c,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
 v_dual_fmaak_f32 v255, v4, v2, 0xaf123456 :: v_dual_add_f32 v6, v1, v3
@@ -2342,6 +3630,74 @@ v_dual_fmaak_f32 v6, null, v5, 0xaf123456 :: v_dual_cndmask_b32 v255, 0xaf123456
 // GFX12: encoding: [0x7c,0x0a,0x52,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_fmaak_f32 v255, v1, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x58,0xc8,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, v255, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0xff,0x05,0x58,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, v2, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x58,0xc8,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, v3, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0x03,0x05,0x58,0xc8,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, s105, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x69,0x04,0x58,0xc8,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, s1, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x01,0x04,0x58,0xc8,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, ttmp15, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7b,0x04,0x58,0xc8,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, exec_hi, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x7f,0x04,0x58,0xc8,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, exec_lo, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x7e,0x04,0x58,0xc8,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, m0, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x58,0xc8,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, vcc_hi, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x6b,0x04,0x58,0xc8,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, vcc_lo, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x6a,0x04,0x58,0xc8,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, 0xaf123456, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0xff,0x04,0x58,0xc8,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, src_scc, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xfd,0x04,0x58,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, 0.5, v3, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, -1, v2
+// GFX12: encoding: [0xf0,0x06,0x58,0xc8,0xc1,0x04,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v255, -1, v4, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, 0.5, v5
+// GFX12: encoding: [0xc1,0x08,0x58,0xc8,0xf0,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmaak_f32 v6, null, v5, 0xaf123456 :: v_dual_dot2acc_f32_f16 v255, src_scc, v4
+// GFX12: encoding: [0x7c,0x0a,0x58,0xc8,0xfd,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_fmaak_f32 v255, v4, v2, 0xaf123456 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x42,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -3422,6 +4778,78 @@ v_dual_fmac_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x12,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_fmac_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x18,0xc8,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x18,0xc8,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x18,0xc8,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x18,0xc8,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x18,0xc8,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x18,0xc8,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x18,0xc8,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x18,0xc8,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x18,0xc8,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x18,0xc8,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x18,0xc8,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x18,0xc8,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x18,0xc8,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x18,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x18,0xc8,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x18,0xc8,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x18,0xc8,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmac_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x18,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_fmac_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x02,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -4466,6 +5894,74 @@ v_dual_fmamk_f32 v6, null, 0xaf123456, v4 :: v_dual_cndmask_b32 v255, 0xaf123456
 // GFX12: encoding: [0x7c,0x08,0x92,0xc8,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_fmamk_f32 v255, v1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v255, v255
+// GFX12: encoding: [0x01,0xff,0x99,0xc8,0xff,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, v255, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v1, v255
+// GFX12: encoding: [0xff,0xff,0x99,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, v2, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v3, v255
+// GFX12: encoding: [0x02,0xff,0x99,0xc8,0x03,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, v3, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v2, v255
+// GFX12: encoding: [0x03,0xff,0x99,0xc8,0x02,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, s105, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, s105, v255
+// GFX12: encoding: [0x69,0xfe,0x99,0xc8,0x69,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, s1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, s1, v255
+// GFX12: encoding: [0x01,0xfe,0x99,0xc8,0x01,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, ttmp15, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v255
+// GFX12: encoding: [0x7b,0xfe,0x99,0xc8,0x7b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, exec_hi, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v4, v255
+// GFX12: encoding: [0x7f,0xfe,0x99,0xc8,0x04,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, exec_lo, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v255
+// GFX12: encoding: [0x7e,0xfe,0x99,0xc8,0x7e,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, m0, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, m0, v255
+// GFX12: encoding: [0x7d,0xfe,0x99,0xc8,0x7d,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, vcc_hi, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v255
+// GFX12: encoding: [0x6b,0xfe,0x99,0xc8,0x6b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, vcc_lo, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v255
+// GFX12: encoding: [0x6a,0xfe,0x99,0xc8,0x6a,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v255
+// GFX12: encoding: [0xff,0xfe,0x99,0xc8,0x7f,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, src_scc, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, null, v255
+// GFX12: encoding: [0xfd,0xfe,0x99,0xc8,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, 0.5, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xf0,0xfe,0x99,0xc8,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v255, -1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, 0.5, v4
+// GFX12: encoding: [0xc1,0xfe,0x99,0xc8,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_fmamk_f32 v6, null, 0xaf123456, v4 :: v_dual_dot2acc_f32_f16 v255, src_scc, v5
+// GFX12: encoding: [0x7c,0x08,0x98,0xc8,0xfd,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_fmamk_f32 v255, v4, 0xaf123456, v255 :: v_dual_fmaak_f32 v6, v1, v255, 0xaf123456
 // GFX12: encoding: [0x04,0xff,0x83,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -5546,6 +7042,78 @@ v_dual_max_num_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x92,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_max_num_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x98,0xca,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x98,0xca,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x98,0xca,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x98,0xca,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x98,0xca,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x98,0xca,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x98,0xca,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x98,0xca,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x98,0xca,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x98,0xca,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x98,0xca,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x98,0xca,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x98,0xca,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x98,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x98,0xca,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x98,0xca,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x98,0xca,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_max_num_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x98,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_max_num_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x82,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -6626,6 +8194,78 @@ v_dual_min_num_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0xd2,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_min_num_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0xd8,0xca,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0xd8,0xca,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0xd8,0xca,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0xd8,0xca,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0xd8,0xca,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0xd8,0xca,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0xd8,0xca,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0xd8,0xca,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0xd8,0xca,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0xd8,0xca,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0xd8,0xca,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0xd8,0xca,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0xd8,0xca,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0xd8,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0xd8,0xca,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0xd8,0xca,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0xd8,0xca,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_min_num_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0xd8,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_min_num_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0xc2,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -7698,12 +9338,84 @@ v_dual_mov_b32 v255, 0.5 :: v_dual_cndmask_b32 v6, 0.5, v3
 // GFX12: encoding: [0xf0,0x00,0x12,0xca,0xf0,0x06,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_mov_b32 v255, -1 :: v_dual_cndmask_b32 v6, src_scc, v4
-// GFX12: encoding: [0xc1,0x00,0x12,0xca,0xfd,0x08,0x06,0xff]
+v_dual_mov_b32 v255, -1 :: v_dual_cndmask_b32 v6, src_scc, v4
+// GFX12: encoding: [0xc1,0x00,0x12,0xca,0xfd,0x08,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v6, null :: v_dual_cndmask_b32 v255, 0xaf123456, v5
+// GFX12: encoding: [0x7c,0x00,0x12,0xca,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, v4 :: v_dual_dot2acc_f32_f16 v6, v1, v255
+// GFX12: encoding: [0x04,0x01,0x18,0xca,0x01,0xff,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, v1 :: v_dual_dot2acc_f32_f16 v6, v255, v255
+// GFX12: encoding: [0x01,0x01,0x18,0xca,0xff,0xff,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, v255 :: v_dual_dot2acc_f32_f16 v6, v2, v255
+// GFX12: encoding: [0xff,0x01,0x18,0xca,0x02,0xff,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v255
+// GFX12: encoding: [0x02,0x01,0x18,0xca,0x03,0xff,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, v3 :: v_dual_dot2acc_f32_f16 v6, v4, v255
+// GFX12: encoding: [0x03,0x01,0x18,0xca,0x04,0xff,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, s105 :: v_dual_dot2acc_f32_f16 v6, s1, v255
+// GFX12: encoding: [0x69,0x00,0x18,0xca,0x01,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, s1 :: v_dual_dot2acc_f32_f16 v6, s105, v255
+// GFX12: encoding: [0x01,0x00,0x18,0xca,0x69,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, ttmp15 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v255
+// GFX12: encoding: [0x7b,0x00,0x18,0xca,0x6a,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, exec_hi :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v255
+// GFX12: encoding: [0x7f,0x00,0x18,0xca,0x6b,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, exec_lo :: v_dual_dot2acc_f32_f16 v6, ttmp15, v255
+// GFX12: encoding: [0x7e,0x00,0x18,0xca,0x7b,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, m0 :: v_dual_dot2acc_f32_f16 v6, m0, v255
+// GFX12: encoding: [0x7d,0x00,0x18,0xca,0x7d,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, vcc_hi :: v_dual_dot2acc_f32_f16 v6, exec_lo, v255
+// GFX12: encoding: [0x6b,0x00,0x18,0xca,0x7e,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, vcc_lo :: v_dual_dot2acc_f32_f16 v6, exec_hi, v255
+// GFX12: encoding: [0x6a,0x00,0x18,0xca,0x7f,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, null, v255
+// GFX12: encoding: [0xff,0x00,0x18,0xca,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, src_scc :: v_dual_dot2acc_f32_f16 v6, -1, v255
+// GFX12: encoding: [0xfd,0x00,0x18,0xca,0xc1,0xfe,0x07,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, 0.5 :: v_dual_dot2acc_f32_f16 v6, 0.5, v3
+// GFX12: encoding: [0xf0,0x00,0x18,0xca,0xf0,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mov_b32 v255, -1 :: v_dual_dot2acc_f32_f16 v6, src_scc, v4
+// GFX12: encoding: [0xc1,0x00,0x18,0xca,0xfd,0x08,0x06,0xff]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
-v_dual_mov_b32 v6, null :: v_dual_cndmask_b32 v255, 0xaf123456, v5
-// GFX12: encoding: [0x7c,0x00,0x12,0xca,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
+v_dual_mov_b32 v6, null :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v5
+// GFX12: encoding: [0x7c,0x00,0x18,0xca,0xff,0x0a,0xfe,0x06,0x0b,0xfe,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
 v_dual_mov_b32 v255, v4 :: v_dual_fmaak_f32 v6, v1, v255, 0xaf123456
@@ -8786,6 +10498,78 @@ v_dual_mul_dx9_zero_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0xd2,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_mul_dx9_zero_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0xd8,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0xd8,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0xd8,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0xd8,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0xd8,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0xd8,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0xd8,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0xd8,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0xd8,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0xd8,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0xd8,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0xd8,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0xd8,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0xd8,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0xd8,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0xd8,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0xd8,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_dx9_zero_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0xd8,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_mul_dx9_zero_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0xc2,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -9866,6 +11650,78 @@ v_dual_mul_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0xd2,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_mul_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0xd8,0xc8,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0xd8,0xc8,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0xd8,0xc8,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0xd8,0xc8,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0xd8,0xc8,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0xd8,0xc8,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0xd8,0xc8,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0xd8,0xc8,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0xd8,0xc8,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0xd8,0xc8,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0xd8,0xc8,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0xd8,0xc8,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0xd8,0xc8,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0xd8,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0xd8,0xc8,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0xd8,0xc8,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0xd8,0xc8,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_mul_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0xd8,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_mul_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0xc2,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -10946,6 +12802,78 @@ v_dual_sub_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x52,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_sub_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x58,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x58,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x58,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x58,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x58,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x58,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x58,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x58,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x58,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x58,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x58,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x58,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x58,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x58,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x58,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x58,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x58,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_sub_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x58,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_sub_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x42,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -12026,6 +13954,78 @@ v_dual_subrev_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x92,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
 
+v_dual_subrev_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x98,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x98,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x98,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x98,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x98,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x98,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x98,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x98,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x98,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x98,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x98,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x98,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x98,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x98,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x98,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x98,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x98,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_subrev_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x98,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
 v_dual_subrev_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456
 // GFX12: encoding: [0x04,0x05,0x82,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
@@ -12817,3 +14817,219 @@ v_dual_subrev_f32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5
 v_dual_subrev_f32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4
 // GFX12: encoding: [0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_bf16 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_bf16 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_bf16 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_bf16 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_bf16 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_bf16 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_bf16 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_bf16 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_bf16 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_bf16 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_bf16 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_bf16 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v4
+// GFX12: encoding: [0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_nc_u32 v6, v1, v3
+// GFX12: encoding: [0x04,0x05,0x60,0xcb,0x01,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_nc_u32 v6, v255, v3
+// GFX12: encoding: [0x01,0x05,0x60,0xcb,0xff,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_nc_u32 v6, v2, v3
+// GFX12: encoding: [0xff,0x05,0x60,0xcb,0x02,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_nc_u32 v6, v3, v3
+// GFX12: encoding: [0x02,0x05,0x60,0xcb,0x03,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_nc_u32 v6, v4, v3
+// GFX12: encoding: [0x03,0x05,0x60,0xcb,0x04,0x07,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_nc_u32 v6, s1, v3
+// GFX12: encoding: [0x69,0x04,0x60,0xcb,0x01,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_nc_u32 v6, s105, v3
+// GFX12: encoding: [0x01,0x04,0x60,0xcb,0x69,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_nc_u32 v6, vcc_lo, v3
+// GFX12: encoding: [0x7b,0x04,0x60,0xcb,0x6a,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_nc_u32 v6, vcc_hi, v3
+// GFX12: encoding: [0x7f,0x04,0x60,0xcb,0x6b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_nc_u32 v6, ttmp15, v3
+// GFX12: encoding: [0x7e,0x04,0x60,0xcb,0x7b,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_nc_u32 v6, m0, v3
+// GFX12: encoding: [0x7d,0x04,0x60,0xcb,0x7d,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_nc_u32 v6, exec_lo, v3
+// GFX12: encoding: [0x6b,0x04,0x60,0xcb,0x7e,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_nc_u32 v6, exec_hi, v3
+// GFX12: encoding: [0x6a,0x04,0x60,0xcb,0x7f,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_nc_u32 v6, null, v3
+// GFX12: encoding: [0xff,0x04,0x60,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_nc_u32 v6, -1, v3
+// GFX12: encoding: [0xfd,0x04,0x60,0xcb,0xc1,0x06,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_nc_u32 v6, 0.5, v2
+// GFX12: encoding: [0xf0,0x06,0x60,0xcb,0xf0,0x04,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_nc_u32 v6, src_scc, v5
+// GFX12: encoding: [0xc1,0x08,0x60,0xcb,0xfd,0x0a,0x06,0xff]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
+
+v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_nc_u32 v255, 0xaf123456, v4
+// GFX12: encoding: [0x7c,0x0a,0x60,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+// W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error
diff --git a/llvm/test/MC/AMDGPU/gfx12_err.s b/llvm/test/MC/AMDGPU/gfx12_err.s
index d8578d87279d1..d55b86b54ec7d 100644
--- a/llvm/test/MC/AMDGPU/gfx12_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_err.s
@@ -22,20 +22,6 @@ v_cvt_f16_u16_e64_dpp v5, s1 row_shl:1 row_mask:0xf bank_mask:0xf
 v_dual_mul_f32 v0, v0, v2 : : v_dual_mul_f32 v1, v1, v3
 // GFX12-ERR: [[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression
 
-// On GFX12, v_dot8_i32_i4 is a valid SP3 alias for v_dot8_i32_iu4.
-// However, we intentionally leave it unimplemented because on other
-// processors v_dot8_i32_i4 denotes an instruction of a different
-// behaviour, which is considered potentially dangerous.
-v_dot8_i32_i4 v0, v1, v2, v3
-// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
-
-// On GFX12, v_dot4_i32_i8 is a valid SP3 alias for v_dot4_i32_iu8.
-// However, we intentionally leave it unimplemented because on other
-// processors v_dot4_i32_i8 denotes an instruction of a different
-// behaviour, which is considered potentially dangerous.
-v_dot4_i32_i8 v0, v1, v2, v3
-// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
-
 v_dot4c_i32_i8 v0, v1, v2
 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
diff --git a/llvm/test/MC/AMDGPU/gfx950_asm_features.s b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
index 490dbb4cd97fe..389b17296c045 100644
--- a/llvm/test/MC/AMDGPU/gfx950_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
@@ -1551,3 +1551,27 @@ v_cvt_scalef32_sr_pk32_bf6_f32 v[0:5], v[6:37], v38, v39
 // NOT-GFX950: error: instruction not supported on this GPU
 // GFX950: v_cvt_scalef32_sr_pk32_fp6_f32 v[0:5], v[6:37], v38, v39 ; encoding: [0x00,0x00,0x54,0xd2,0x06,0x4d,0x9e,0x04]
 v_cvt_scalef32_sr_pk32_fp6_f32 v[0:5], v[6:37], v38, v39
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x67,0xd2,0x01,0x05,0x02,0x00]
+v_cvt_pk_f16_f32 v5, v1, v2
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x67,0xd2,0xff,0xff,0x03,0x00]
+v_cvt_pk_f16_f32 v5, v255, v255
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x67,0xd2,0x7c,0xe0,0x01,0x00]
+v_cvt_pk_f16_f32 v5, m0, 0.5
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x67,0xd2,0x7e,0x82,0x01,0x00]
+v_cvt_pk_f16_f32 v5, exec_lo, -1
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x67,0xd2,0xc1,0xfe,0x00,0x00]
+v_cvt_pk_f16_f32 v5, -1, exec_hi
+
+// NOT-GFX950: error: instruction not supported on this GPU
+// GFX950: v_cvt_pk_f16_f32 v5, 0.5, m0 mul:2      ; encoding: [0x05,0x00,0x67,0xd2,0xf0,0xf8,0x00,0x08]
+v_cvt_pk_f16_f32 v5, 0.5, m0 mul:2
diff --git a/llvm/test/MC/ARM/invalid-barrier.s b/llvm/test/MC/ARM/invalid-barrier.s
index 29fcd8e74623a..9c2a857becfe9 100644
--- a/llvm/test/MC/ARM/invalid-barrier.s
+++ b/llvm/test/MC/ARM/invalid-barrier.s
@@ -5,24 +5,46 @@
 @ DMB
 @------------------------------------------------------------------------------
         dmb #0x10
+@ CHECK: [[@LINE-1]]:{{.*}}: error: immediate value out of range
         dmb imaginary_scope
-
-@ CHECK: error: immediate value out of range
-@ CHECK: error: invalid operand for instruction
+@ CHECK: [[@LINE-1]]:{{.*}}: error: invalid operand for instruction
+        dmb [r0]
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dmb [], @, -=_+
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dmb ,,,,,
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dmb 3.141
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
 
 @------------------------------------------------------------------------------
 @ DSB
 @------------------------------------------------------------------------------
         dsb #0x10
+@ CHECK: [[@LINE-1]]:{{.*}}: error: immediate value out of range
         dsb imaginary_scope
-@ CHECK: error: immediate value out of range
-@ CHECK: error: invalid operand for instruction
+@ CHECK: [[@LINE-1]]:{{.*}}: error: invalid operand for instruction
+        dsb [r0]
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dsb [], @, -=_+
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dsb ,,,,,
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        dsb 3.141
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
 
 @------------------------------------------------------------------------------
 @ ISB
 @------------------------------------------------------------------------------
         isb #0x1f
+@ CHECK: [[@LINE-1]]:{{.*}}: error: immediate value out of range
         isb imaginary_domain
-
-@ CHECK: error: immediate value out of range
-@ CHECK: error: invalid operand for instruction
+@ CHECK: [[@LINE-1]]:{{.*}}: error: invalid operand for instruction
+        isb [r0]
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        isb [], @, -=_+
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        isb ,,,,,
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
+        isb 3.141
+@ CHECK: [[@LINE-1]]:{{.*}}: error: expected an immediate or barrier type
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt
new file mode 100644
index 0000000000000..afdee5b2eeb06
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX11 %s
+
+0x00,0x05,0x00,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x00,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84]
+
+0x00,0x05,0x01,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x01,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84]
+
+0x00,0x05,0x02,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x02,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84]
+
+0x00,0x05,0x03,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x03,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84]
+
+0x00,0x05,0x04,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x04,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84]
+
+0x00,0x05,0x05,0xcd,0x01,0x05,0x0e,0x04
+# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:5 ; encoding: [0x00,0x05,0x05,0xcd,0x01,0x05,0x0e,0x04]
+
+0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24
+# GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24]
+
+0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44
+# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44]
+
+0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84
+# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
index 372721a17d4d7..e74c3d8032f56 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64,GFX11-REAL16 %s
@@ -76,9 +76,13 @@
 # GFX11-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
 # GFX11-FAKE16: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
 
-0xff,0x05,0x0a,0x64
-# GFX11-REAL16: v_add_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x64]
-# GFX11-FAKE16: v_add_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x64]
+0x81,0x05,0x0a,0x64
+# GFX11-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
+# GFX11-FAKE16: v_add_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x64]
+
+0x7f,0x05,0x0a,0x64
+# GFX11-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l        ; encoding: [0x7f,0x05,0x0a,0x64]
+# GFX11-FAKE16: v_add_f16_e32 v5, v127, v2              ; encoding: [0x7f,0x05,0x0a,0x64]
 
 0x01,0x04,0x0a,0x64
 # GFX11-REAL16: v_add_f16_e32 v5.l, s1, v2.l            ; encoding: [0x01,0x04,0x0a,0x64]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
index 84465624da6ba..4e35ae9d61eb4 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp16.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
@@ -104,17 +104,17 @@
 # GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 # GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x50,0x01,0xff]
 
-0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01
-# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
-# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x5f,0x01,0x01]
+0xfa,0xfe,0xfe,0x64,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_add_f16_dpp v127, v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x5f,0x01,0x01]
 
-0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13
-# GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
-# GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x60,0x01,0x13]
+0xfa,0x04,0x0b,0x65,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_add_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x65,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_mul_i32_i24_e32 v128, 1, v176         ; encoding: [0x81,0x60,0x01,0x13]
 
-0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30
-# GFX11-REAL16: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
-# GFX11-FAKE16: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
+0xfa,0xfe,0xff,0x65,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_add_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x65,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_lshlrev_b32_e32 v126, v255, v183      ; encoding: [0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff
 # GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
@@ -564,7 +564,7 @@
 
 0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff
 # GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
-# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff
+# GFX11-FAKE16: v_ldexp_f16_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x1f,0x01,0xff]
 
 0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff
 # GFX11-REAL16: v_ldexp_f16_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x76,0x01,0x21,0x01,0xff]
@@ -1094,7 +1094,7 @@
 
 0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff
 # GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
-# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff
+# GFX11-FAKE16: v_mul_f16_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x41,0x01,0xff]
 
 0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff
 # GFX11-REAL16: v_mul_f16_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x6a,0x01,0x01,0x01,0xff]
@@ -1488,17 +1488,17 @@
 # GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 # GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x50,0x01,0xff]
 
-0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01
-# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
-# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x5f,0x01,0x01]
+0xfa,0xfe,0xfe,0x66,0x7f,0x5f,0x01,0x01
+# GFX11-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x5f,0x01,0x01]
+# GFX11-FAKE16: v_sub_f16_dpp v127, v127, v127 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x5f,0x01,0x01]
 
-0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13
-# GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
-# GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0a,0x66,0x01,0x60,0x01,0x13]
+0xfa,0x04,0x0b,0x67,0x81,0x60,0x01,0x13
+# GFX11-REAL16: v_sub_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x67,0x81,0x60,0x01,0x13]
+# GFX11-FAKE16: v_mul_i32_i24_e32 v128, 1, v176         ; encoding: [0x81,0x60,0x01,0x13]
 
-0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30
-# GFX11-REAL16: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
-# GFX11-FAKE16: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
+0xfa,0xfe,0xff,0x67,0xff,0x6f,0xfd,0x30
+# GFX11-REAL16: v_sub_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x67,0xff,0x6f,0xfd,0x30]
+# GFX11-FAKE16: v_lshlrev_b32_e32 v126, v255, v183      ; encoding: [0xff,0x6f,0xfd,0x30]
 
 0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff
 # GFX11: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
index 3d4a16d41880a..5fd1145196890 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2_dpp8.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,GFX11-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-FAKE16 %s
@@ -16,9 +16,16 @@
 # GFX11-REAL16: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 # GFX11-FAKE16: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00
-# GFX11-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
-# GFX11-FAKE16: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
+0xe9,0xfe,0xfe,0x64,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_add_f16_dpp v127, v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x64,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x0b,0x65,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_add_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x65,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_dot2acc_f32_f16 v156, v129, v187      ; encoding: [0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0xff,0x65,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_add_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x65,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05
 # GFX11: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
@@ -219,9 +226,16 @@
 # GFX11-REAL16: v_sub_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 # GFX11-FAKE16: v_sub_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x66,0x01,0x77,0x39,0x05]
 
-0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00
-# GFX11-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
-# GFX11-FAKE16: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
+0xe9,0xfe,0xfe,0x66,0x7f,0x77,0x39,0x05
+# GFX11-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x77,0x39,0x05]
+# GFX11-FAKE16: v_sub_f16_dpp v127, v127, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xfe,0xfe,0x66,0x7f,0x77,0x39,0x05]
+
+0xe9,0x04,0x0b,0x67,0x81,0x77,0x39,0x05
+# GFX11-REAL16: v_sub_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x67,0x81,0x77,0x39,0x05]
+# GFX11-FAKE16: v_dot2acc_f32_f16 v156, v129, v187      ; encoding: [0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0xff,0x67,0xff,0x00,0x00,0x00
+# GFX11-REAL16: v_sub_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x67,0xff,0x00,0x00,0x00]
 
 0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05
 # GFX11: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
index 365caa5f9b6d0..60d213f1ff937 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
@@ -1,6582 +1,6710 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-FAKE16 %s
 
-# GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_add3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_add3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_add3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_add3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_add3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_add3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_add3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_add3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_add3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_add3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_add3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_add3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_add3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_add3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_add3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_add3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_add3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_add3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_add3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_add3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_add3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_add3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_add3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_add3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_add3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_add3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# W32: v_add_co_u32 v5, s12, v1, v2              ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_add_co_u32 v5, s[12:13], v1, v2         ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00
+# W32: v_add_co_u32 v5, s12, v1, v2            ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_add_co_u32 v5, s[12:13], v1, v2       ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_add_co_u32 v5, s12, v255, v255          ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_add_co_u32 v5, s[12:13], v255, v255     ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00
+# W32: v_add_co_u32 v5, s12, v255, v255        ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_add_co_u32 v5, s[12:13], v255, v255   ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_add_co_u32 v5, s12, s1, s2              ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], s1, s2         ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00
+# W32: v_add_co_u32 v5, s12, s1, s2            ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], s1, s2       ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, s105, s105          ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], s105, s105     ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_add_co_u32 v5, s12, s105, s105        ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], s105, s105   ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, vcc_lo, ttmp15      ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_add_co_u32 v5, s12, vcc_lo, ttmp15    ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, vcc_hi, 0xaf123456  ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-# W64: v_add_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# W32: v_add_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+# W64: v_add_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_add_co_u32 v5, s12, ttmp15, src_scc     ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_add_co_u32 v5, s12, ttmp15, src_scc   ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, m0, 0.5             ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], m0, 0.5        ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_add_co_u32 v5, s12, m0, 0.5           ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], m0, 0.5      ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, exec_lo, -1         ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], exec_lo, -1    ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_add_co_u32 v5, s12, exec_lo, -1       ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], exec_lo, -1  ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, exec_hi, null       ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], exec_hi, null  ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_add_co_u32 v5, s12, exec_hi, null     ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, null, exec_lo       ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], null, exec_lo  ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_add_co_u32 v5, s12, null, exec_lo     ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s104, -1, exec_hi        ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_add_co_u32 v5, s[104:105], -1, exec_hi  ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_add_co_u32 v5, s104, -1, exec_hi      ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_add_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_add_co_u32 v5, vcc_lo, 0.5, m0          ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_add_co_u32 v5, vcc, 0.5, m0             ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_add_co_u32 v5, vcc_lo, 0.5, m0        ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_add_co_u32 v5, vcc, 0.5, m0           ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
 
-# W32: v_add_co_u32 v5, ttmp14, src_scc, vcc_lo  ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
-# W64: v_add_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00
+# W32: v_add_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
+# W64: v_add_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_add_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_add_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x27,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x27,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_add_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x27,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_add_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x27,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x27,0xd7,0xfe,0xfd,0x03,0x00
+# GFX11: v_add_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x27,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX11: v_add_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x27,0xd7,0x02,0x08,0x00,0x00]
 0x05,0x00,0x27,0xd7,0x02,0x08,0x00,0x00
+# GFX11: v_add_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x27,0xd7,0x02,0x08,0x00,0x00]
 
-# GFX11: v_add_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x27,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x27,0xd7,0x68,0xd0,0x00,0x00
+# GFX11: v_add_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x27,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX11: v_add_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x27,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x27,0xd7,0x6a,0xf4,0x00,0x00
+# GFX11: v_add_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x27,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX11: v_add_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x27,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x27,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_add_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x27,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x27,0xd7,0x7e,0xfa,0x01,0x20]
 0x05,0x01,0x27,0xd7,0x7e,0xfa,0x01,0x20
+# GFX11: v_add_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x27,0xd7,0x7e,0xfa,0x01,0x20]
 
-# GFX11: v_add_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x27,0xd7,0x7c,0xe0,0x01,0x00]
 0x05,0x00,0x27,0xd7,0x7c,0xe0,0x01,0x00
+# GFX11: v_add_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x27,0xd7,0x7c,0xe0,0x01,0x00]
 
-# GFX11: v_add_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x27,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x27,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_add_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x27,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_add_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x27,0xd7,0xf0,0xf8,0x00,0x08]
 0x05,0x00,0x27,0xd7,0xf0,0xf8,0x00,0x08
+# GFX11: v_add_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x27,0xd7,0xf0,0xf8,0x00,0x08]
 
-# GFX11: v_add_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x27,0xd7,0xfd,0xfc,0x00,0x70]
 0x05,0x03,0x27,0xd7,0xfd,0xfc,0x00,0x70
+# GFX11: v_add_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x27,0xd7,0xfd,0xfc,0x00,0x70]
 
-# GFX11: v_add_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x27,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x27,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_add_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x27,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_lshl_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_add_lshl_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_add_lshl_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_add_lshl_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_add_lshl_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_add_lshl_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_add_lshl_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_add_lshl_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_add_lshl_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_add_lshl_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_add_lshl_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_add_lshl_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_lshl_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_add_lshl_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_add_lshl_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_add_lshl_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_add_lshl_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_add_lshl_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_add_lshl_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_add_lshl_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_add_lshl_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_add_lshl_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_lshl_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_add_lshl_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_add_lshl_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_add_lshl_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_add_lshl_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_add_lshl_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_add_lshl_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_add_lshl_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_add_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_add_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_add_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_add_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_add_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_add_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_add_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_add_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_add_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_add_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_add_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_add_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_add_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_add_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_add_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_add_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_add_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_add_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_add_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_add_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_add_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_add_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_add_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x03,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x03,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_alignbit_b32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_alignbit_b32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_alignbit_b32 v5, v255, s2, s3         ; encoding: [0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00]
 0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00
+# GFX11: v_alignbit_b32 v5, v255, s2, s3         ; encoding: [0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00]
 
-# GFX11: v_alignbit_b32 v5, s1, v255, s3         ; encoding: [0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00]
 0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00
+# GFX11: v_alignbit_b32 v5, s1, v255, s3         ; encoding: [0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00]
 
-# GFX11: v_alignbit_b32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01
+# GFX11: v_alignbit_b32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX11: v_alignbit_b32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_alignbit_b32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_alignbit_b32 v5, m0, 0.5, exec_lo     ; encoding: [0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01]
 0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01
+# GFX11: v_alignbit_b32 v5, m0, 0.5, exec_lo     ; encoding: [0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01]
 
-# GFX11: v_alignbit_b32 v5, exec_lo, -1, m0      ; encoding: [0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01]
 0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01
+# GFX11: v_alignbit_b32 v5, exec_lo, -1, m0      ; encoding: [0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01]
 
-# GFX11: v_alignbit_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01]
 0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01
+# GFX11: v_alignbit_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01]
 
-# GFX11: v_alignbit_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01]
 0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01
+# GFX11: v_alignbit_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01]
 
-# GFX11: v_alignbit_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_alignbit_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_alignbit_b32 v5, 0.5, m0, exec_hi     ; encoding: [0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01]
 0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01
+# GFX11: v_alignbit_b32 v5, 0.5, m0, exec_hi     ; encoding: [0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01]
 
-# GFX11: v_alignbit_b32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_alignbit_b32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_alignbyte_b32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_alignbyte_b32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_alignbyte_b32 v5, v255, s2, s3        ; encoding: [0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00]
 0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00
+# GFX11: v_alignbyte_b32 v5, v255, s2, s3        ; encoding: [0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00]
 
-# GFX11: v_alignbyte_b32 v5, s1, v255, s3        ; encoding: [0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00]
 0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00
+# GFX11: v_alignbyte_b32 v5, s1, v255, s3        ; encoding: [0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00]
 
-# GFX11: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01
+# GFX11: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX11: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_alignbyte_b32 v5, m0, 0.5, exec_lo    ; encoding: [0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01]
 0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01
+# GFX11: v_alignbyte_b32 v5, m0, 0.5, exec_lo    ; encoding: [0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01]
 
-# GFX11: v_alignbyte_b32 v5, exec_lo, -1, m0     ; encoding: [0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01]
 0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01
+# GFX11: v_alignbyte_b32 v5, exec_lo, -1, m0     ; encoding: [0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01]
 
-# GFX11: v_alignbyte_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01]
 0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01
+# GFX11: v_alignbyte_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01]
 
-# GFX11: v_alignbyte_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01]
 0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01
+# GFX11: v_alignbyte_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01]
 
-# GFX11: v_alignbyte_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_alignbyte_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_alignbyte_b32 v5, 0.5, m0, exec_hi    ; encoding: [0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01]
 0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01
+# GFX11: v_alignbyte_b32 v5, 0.5, m0, exec_hi    ; encoding: [0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01]
 
-# GFX11: v_alignbyte_b32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_alignbyte_b32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_and_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_and_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_and_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_and_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_and_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_and_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_and_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_and_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_and_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_and_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_and_or_b32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_and_or_b32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_and_or_b32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_and_or_b32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_and_or_b32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_and_or_b32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_and_or_b32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_and_or_b32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_and_or_b32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_and_or_b32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_and_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_and_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_and_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_and_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_and_or_b32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_and_or_b32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_and_or_b32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_and_or_b32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_and_or_b32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_and_or_b32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_and_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_and_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_and_or_b32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_and_or_b32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_and_or_b32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_and_or_b32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_and_or_b32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_and_or_b32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_and_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_and_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, s1, s2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, s1, s2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, s105, s105            ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, s105, s105            ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_ashrrev_i64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00
+# GFX11: v_ashrrev_i64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00
+# GFX11: v_ashrrev_i64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00]
 0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00
+# GFX11: v_ashrrev_i64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00]
 0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00
+# GFX11: v_ashrrev_i64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00
+# GFX11: v_ashrrev_i64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_ashrrev_i64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_ashrrev_i64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_ashrrev_i64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_ashrrev_i64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00]
 0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00
+# GFX11: v_ashrrev_i64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00]
 
-# GFX11: v_ashrrev_i64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_ashrrev_i64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bcnt_u32_b32 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_bcnt_u32_b32 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_bcnt_u32_b32 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_bcnt_u32_b32 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bcnt_u32_b32 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_bcnt_u32_b32 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_bcnt_u32_b32 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_bcnt_u32_b32 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_bcnt_u32_b32 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_bcnt_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_bcnt_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_i32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_bfe_i32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_bfe_i32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_bfe_i32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_bfe_i32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_bfe_i32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_bfe_i32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_bfe_i32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_bfe_i32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_bfe_i32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_bfe_i32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_i32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_i32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_bfe_i32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_bfe_i32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_bfe_i32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_bfe_i32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_bfe_i32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_bfe_i32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_bfe_i32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_bfe_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_i32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_bfe_i32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_bfe_i32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_bfe_i32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_bfe_i32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_bfe_i32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_bfe_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_bfe_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_bfe_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_bfe_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_bfe_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_bfe_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_bfe_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_bfe_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_bfe_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_bfe_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_bfe_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_bfe_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_bfe_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_bfe_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_bfe_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_bfe_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_bfe_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_bfe_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_bfe_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfe_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_bfe_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_bfe_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_bfe_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_bfe_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_bfe_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_bfe_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_bfe_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfi_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_bfi_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_bfi_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_bfi_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_bfi_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_bfi_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_bfi_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_bfi_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_bfi_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_bfi_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_bfi_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_bfi_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfi_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_bfi_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_bfi_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_bfi_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_bfi_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_bfi_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_bfi_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_bfi_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_bfi_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_bfi_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfi_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_bfi_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_bfi_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_bfi_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_bfi_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_bfi_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_bfi_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_bfi_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfm_b32 v5, v1, v2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_bfm_b32 v5, v1, v2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_bfm_b32 v5, v255, v255                ; encoding: [0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_bfm_b32 v5, v255, v255                ; encoding: [0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_bfm_b32 v5, s1, s2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_bfm_b32 v5, s1, s2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, s105, s105                ; encoding: [0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_bfm_b32 v5, s105, s105                ; encoding: [0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_bfm_b32 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, vcc_hi, 0xaf123456        ; encoding: [0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_bfm_b32 v5, vcc_hi, 0xaf123456        ; encoding: [0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_bfm_b32 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_bfm_b32 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_bfm_b32 v5, m0, 0.5                   ; encoding: [0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_bfm_b32 v5, m0, 0.5                   ; encoding: [0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_bfm_b32 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_bfm_b32 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_bfm_b32 v5, exec_hi, null             ; encoding: [0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_bfm_b32 v5, exec_hi, null             ; encoding: [0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, null, exec_lo             ; encoding: [0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_bfm_b32 v5, null, exec_lo             ; encoding: [0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_bfm_b32 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, 0.5, m0                   ; encoding: [0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_bfm_b32 v5, 0.5, m0                   ; encoding: [0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_bfm_b32 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_bfm_b32 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_bfm_b32 v255, 0xaf123456, vcc_hi      ; encoding: [0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_bfm_b32 v255, 0xaf123456, vcc_hi      ; encoding: [0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_cndmask_b16 v5, v1, src_scc, s6         ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
-# W64: v_cndmask_b16 v5, v1, src_scc, s[6:7]     ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
 0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00
+# W32: v_cndmask_b16 v5, v1, src_scc, s6       ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
+# W64: v_cndmask_b16 v5, v1, src_scc, s[6:7]   ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
 
-# W32: v_cndmask_b16 v5, v255, 0x3800, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
-# W64: v_cndmask_b16 v5, v255, 0x3800, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
 0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00
+# W32: v_cndmask_b16 v5, v255, 0x3800, s6      ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+# W64: v_cndmask_b16 v5, v255, 0x3800, s[6:7]  ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
 
-# W32: v_cndmask_b16 v5, s105, s105, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
-# W64: v_cndmask_b16 v5, s105, s105, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00
+# W32: v_cndmask_b16 v5, s105, s105, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
+# W64: v_cndmask_b16 v5, s105, s105, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, vcc_hi, v2, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
-# W64: v_cndmask_b16 v5, vcc_hi, v2, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
 0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00
+# W32: v_cndmask_b16 v5, vcc_hi, v2, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
+# W64: v_cndmask_b16 v5, vcc_hi, v2, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
 
-# W32: v_cndmask_b16 v5, ttmp15, ttmp15, s6      ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
-# W64: v_cndmask_b16 v5, ttmp15, ttmp15, s[6:7]  ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00
+# W32: v_cndmask_b16 v5, ttmp15, ttmp15, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
+# W64: v_cndmask_b16 v5, ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, m0, v255, s6            ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
-# W64: v_cndmask_b16 v5, m0, v255, s[6:7]        ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
 0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00
+# W32: v_cndmask_b16 v5, m0, v255, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
+# W64: v_cndmask_b16 v5, m0, v255, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
 
-# W32: v_cndmask_b16 v5, exec_lo, exec_lo, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
-# W64: v_cndmask_b16 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00
+# W32: v_cndmask_b16 v5, exec_lo, exec_lo, s6  ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
+# W64: v_cndmask_b16 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, exec_hi, exec_hi, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
-# W64: v_cndmask_b16 v5, exec_hi, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00
+# W32: v_cndmask_b16 v5, exec_hi, exec_hi, s6  ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
+# W64: v_cndmask_b16 v5, exec_hi, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, null, m0, s6            ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
-# W64: v_cndmask_b16 v5, null, m0, s[6:7]        ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00
+# W32: v_cndmask_b16 v5, null, m0, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
+# W64: v_cndmask_b16 v5, null, m0, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, -1, -|vcc_lo|, s104     ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
-# W64: v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105] ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
 0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41
+# W32: v_cndmask_b16 v5, -1, -|vcc_lo|, s104   ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
+# W64: v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105] ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
 
-# W32: v_cndmask_b16 v5, 0x3800, -1, vcc_lo      ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
-# W64: v_cndmask_b16 v5, 0x3800, -1, vcc         ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
 0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01
+# W32: v_cndmask_b16 v5, 0x3800, -1, vcc_lo    ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+# W64: v_cndmask_b16 v5, 0x3800, -1, vcc       ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
 
+0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21
 # W32: v_cndmask_b16 v5, -|src_scc|, null, ttmp14 ; encoding: [0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21]
 # W64: v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15] ; encoding: [0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21]
-0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21
 
-# GFX11: v_cndmask_b16 v255, -|0xfe0b|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_cndmask_b16 v255, -|0xfe0b|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cubeid_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_cubeid_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_cubeid_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_cubeid_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_cubeid_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_cubeid_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_cubeid_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_cubeid_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_cubeid_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_cubeid_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_cubeid_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_cubeid_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubeid_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_cubeid_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_cubeid_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_cubeid_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_cubeid_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_cubeid_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_cubeid_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_cubeid_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_cubeid_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_cubeid_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubeid_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_cubeid_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_cubeid_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_cubeid_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_cubeid_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_cubeid_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_cubeid_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_cubeid_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubema_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_cubema_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_cubema_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_cubema_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_cubema_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_cubema_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_cubema_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_cubema_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_cubema_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_cubema_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_cubema_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_cubema_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubema_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_cubema_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_cubema_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_cubema_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_cubema_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_cubema_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_cubema_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_cubema_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_cubema_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_cubema_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubema_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_cubema_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_cubema_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_cubema_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_cubema_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_cubema_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_cubema_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_cubema_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubesc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_cubesc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_cubesc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_cubesc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_cubesc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_cubesc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_cubesc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_cubesc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_cubesc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_cubesc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_cubesc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_cubesc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubesc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_cubesc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_cubesc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_cubesc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_cubesc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_cubesc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_cubesc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_cubesc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_cubesc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_cubesc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubesc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_cubesc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_cubesc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_cubesc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_cubesc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_cubesc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_cubesc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_cubesc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubetc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_cubetc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_cubetc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_cubetc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_cubetc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_cubetc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_cubetc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_cubetc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_cubetc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_cubetc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_cubetc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_cubetc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubetc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_cubetc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_cubetc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_cubetc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_cubetc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_cubetc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_cubetc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_cubetc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_cubetc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_cubetc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cubetc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_cubetc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_cubetc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_cubetc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_cubetc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_cubetc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_i16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_i16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_i16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_i16_i32 v5, v1, v2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, v1, v2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, v255, v255         ; encoding: [0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, v255, v255         ; encoding: [0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, s1, s2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, s1, s2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, s105, s105         ; encoding: [0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, s105, s105         ; encoding: [0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_i16_i32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_i16_i32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_i16_i32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_norm_i16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_norm_i16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_norm_i16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_norm_u16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_norm_u16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_norm_u16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_u16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u16_u32 v5, v1, v2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, v1, v2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, v255, v255         ; encoding: [0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, v255, v255         ; encoding: [0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, s1, s2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, s1, s2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, s105, s105         ; encoding: [0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, s105, s105         ; encoding: [0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u16_u32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u16_u32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u16_u32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u8_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_cvt_pk_u8_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_cvt_pk_u8_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_cvt_pk_u8_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_cvt_pk_u8_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u8_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u8_f32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, exec_lo, -1, vcc_hi ; encoding: [0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, exec_lo, -1, vcc_hi ; encoding: [0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_cvt_pk_u8_f32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_cvt_pk_u8_f32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u8_f32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_u8_f32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_cvt_pk_u8_f32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_cvt_pk_u8_f32 v5, 0.5, m0, 0.5        ; encoding: [0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_cvt_pk_u8_f32 v5, 0.5, m0, 0.5        ; encoding: [0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_cvt_pk_u8_f32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_cvt_pk_u8_f32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_cvt_pk_u8_f32 v255, -|0xaf123456|, vcc_hi, null ; encoding: [0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf]
 0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_u8_f32 v255, -|0xaf123456|, vcc_hi, null ; encoding: [0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_norm_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_norm_i16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_norm_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_norm_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_norm_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_norm_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_cvt_pk_norm_u16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_cvt_pk_norm_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_cvt_pk_norm_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX11: v_cvt_pk_norm_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f16 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_div_fixup_f16 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_div_fixup_f16 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_div_fixup_f16 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_div_fixup_f16 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_div_fixup_f16 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_div_fixup_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_div_fixup_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_div_fixup_f16 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_div_fixup_f16 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_div_fixup_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_div_fixup_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_div_fixup_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_div_fixup_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_div_fixup_f16 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_div_fixup_f16 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_div_fixup_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_div_fixup_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_div_fixup_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_div_fixup_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_div_fixup_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_div_fixup_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_div_fixup_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_div_fixup_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX11: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_div_fixup_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23
+# GFX11: v_div_fixup_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_div_fixup_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_div_fixup_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
 # CHECK: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] mul:2   ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] mul:2 ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_div_fixup_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_div_fixup_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_div_fixup_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_div_fixup_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_div_fixup_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_div_fixup_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_div_fixup_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_div_fixup_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_div_fixup_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_div_fixup_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_div_fixup_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_div_fixup_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_div_fixup_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_div_fixup_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_div_fixup_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_div_fixup_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_div_fixup_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_div_fixup_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_div_fixup_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_div_fixup_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_div_fixup_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_div_fixup_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_div_fixup_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_div_fixup_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_div_fixup_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04
+# GFX11: v_div_fixup_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX11: v_div_fixup_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00]
 0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00
+# GFX11: v_div_fixup_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00]
 
-# GFX11: v_div_fixup_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07]
 0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07
+# GFX11: v_div_fixup_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07]
 
-# GFX11: v_div_fixup_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1]
 0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1
+# GFX11: v_div_fixup_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1]
 
-# GFX11: v_div_fixup_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1]
 0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1
+# GFX11: v_div_fixup_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1]
 
-# GFX11: v_div_fixup_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1]
 0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1
+# GFX11: v_div_fixup_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1]
 
-# GFX11: v_div_fixup_f64 v[5:6], null, 0.5, vcc  ; encoding: [0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01]
 0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01
+# GFX11: v_div_fixup_f64 v[5:6], null, 0.5, vcc  ; encoding: [0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01]
 
-# GFX11: v_div_fixup_f64 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f64 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fixup_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b]
 0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b
+# GFX11: v_div_fixup_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b]
 
-# GFX11: v_div_fixup_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73]
 0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73
+# GFX11: v_div_fixup_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73]
 
-# GFX11: v_div_fixup_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fixup_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fmas_f32 v5, v1, 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fmas_f32 v5, v1, 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fmas_f32 v5, v255, src_scc, src_scc ; encoding: [0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03]
 0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03
+# GFX11: v_div_fmas_f32 v5, v255, src_scc, src_scc ; encoding: [0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03]
 
-# GFX11: v_div_fmas_f32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01
+# GFX11: v_div_fmas_f32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX11: v_div_fmas_f32 v5, vcc_lo, v2, v3       ; encoding: [0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04]
 0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04
+# GFX11: v_div_fmas_f32 v5, vcc_lo, v2, v3       ; encoding: [0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04]
 
-# GFX11: v_div_fmas_f32 v5, vcc_hi, v255, vcc_hi ; encoding: [0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01]
 0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01
+# GFX11: v_div_fmas_f32 v5, vcc_hi, v255, vcc_hi ; encoding: [0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01]
 
-# GFX11: v_div_fmas_f32 v5, -|ttmp15|, -|ttmp15|, ttmp15 ; encoding: [0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61]
 0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61
+# GFX11: v_div_fmas_f32 v5, -|ttmp15|, -|ttmp15|, ttmp15 ; encoding: [0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61]
 
-# GFX11: v_div_fmas_f32 v5, m0, 0.5, v255        ; encoding: [0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07]
 0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07
+# GFX11: v_div_fmas_f32 v5, m0, 0.5, v255        ; encoding: [0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07]
 
-# GFX11: v_div_fmas_f32 v5, -|exec_lo|, exec_lo, -|exec_lo| ; encoding: [0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1]
 0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1
+# GFX11: v_div_fmas_f32 v5, -|exec_lo|, exec_lo, -|exec_lo| ; encoding: [0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1]
 
-# GFX11: v_div_fmas_f32 v5, -|exec_hi|, -|exec_hi|, -|exec_hi| ; encoding: [0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1]
 0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1
+# GFX11: v_div_fmas_f32 v5, -|exec_hi|, -|exec_hi|, -|exec_hi| ; encoding: [0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1]
 
-# GFX11: v_div_fmas_f32 v5, null, m0, -|m0|      ; encoding: [0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81]
 0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81
+# GFX11: v_div_fmas_f32 v5, null, m0, -|m0|      ; encoding: [0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81]
 
-# GFX11: v_div_fmas_f32 v5, -1, -|vcc_lo|, -|vcc_lo| ; encoding: [0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1]
 0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1
+# GFX11: v_div_fmas_f32 v5, -1, -|vcc_lo|, -|vcc_lo| ; encoding: [0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1]
 
-# GFX11: v_div_fmas_f32 v5, 0.5, -|vcc_hi|, 0.5 mul:2 ; encoding: [0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b]
 0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b
+# GFX11: v_div_fmas_f32 v5, 0.5, -|vcc_hi|, 0.5 mul:2 ; encoding: [0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b]
 
-# GFX11: v_div_fmas_f32 v5, src_scc, -1, -1 mul:4 ; encoding: [0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13]
 0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13
+# GFX11: v_div_fmas_f32 v5, src_scc, -1, -1 mul:4 ; encoding: [0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13]
 
-# GFX11: v_div_fmas_f32 v255, -|0xaf123456|, null, null clamp div:2 ; encoding: [0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf]
 0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fmas_f32 v255, -|0xaf123456|, null, null clamp div:2 ; encoding: [0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fmas_f64 v[5:6], v[1:2], 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fmas_f64 v[5:6], v[1:2], 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_div_fmas_f64 v[5:6], v[254:255], src_scc, v[3:4] ; encoding: [0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04]
 0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04
+# GFX11: v_div_fmas_f64 v[5:6], v[254:255], src_scc, v[3:4] ; encoding: [0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04]
 
-# GFX11: v_div_fmas_f64 v[5:6], s[104:105], |s[104:105]|, s[104:105] ; encoding: [0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01]
 0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01
+# GFX11: v_div_fmas_f64 v[5:6], s[104:105], |s[104:105]|, s[104:105] ; encoding: [0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01]
 
-# GFX11: v_div_fmas_f64 v[5:6], -|vcc|, v[2:3], -|v[254:255]| ; encoding: [0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7]
 0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7
+# GFX11: v_div_fmas_f64 v[5:6], -|vcc|, v[2:3], -|v[254:255]| ; encoding: [0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7]
 
-# GFX11: v_div_fmas_f64 v[5:6], -|ttmp[14:15]|, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1]
 0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1
+# GFX11: v_div_fmas_f64 v[5:6], -|ttmp[14:15]|, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1]
 
-# GFX11: v_div_fmas_f64 v[5:6], -|exec|, -|v[254:255]|, null ; encoding: [0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61]
 0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61
+# GFX11: v_div_fmas_f64 v[5:6], -|exec|, -|v[254:255]|, null ; encoding: [0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61]
 
-# GFX11: v_div_fmas_f64 v[5:6], null, 0.5, -src_scc ; encoding: [0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83]
 0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83
+# GFX11: v_div_fmas_f64 v[5:6], null, 0.5, -src_scc ; encoding: [0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83]
 
-# GFX11: v_div_fmas_f64 v[5:6], -1, -exec, |exec| ; encoding: [0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41]
 0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41
+# GFX11: v_div_fmas_f64 v[5:6], -1, -exec, |exec| ; encoding: [0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41]
 
-# GFX11: v_div_fmas_f64 v[5:6], 0.5, -|vcc|, -|vcc| mul:2 ; encoding: [0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9]
 0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9
+# GFX11: v_div_fmas_f64 v[5:6], 0.5, -|vcc|, -|vcc| mul:2 ; encoding: [0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9]
 
-# GFX11: v_div_fmas_f64 v[5:6], -|src_scc|, -1, 0.5 mul:4 ; encoding: [0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33]
 0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33
+# GFX11: v_div_fmas_f64 v[5:6], -|src_scc|, -1, 0.5 mul:4 ; encoding: [0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33]
 
-# GFX11: v_div_fmas_f64 v[254:255], 0xaf123456, null, -1 clamp div:2 ; encoding: [0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf
+# GFX11: v_div_fmas_f64 v[254:255], 0xaf123456, null, -1 clamp div:2 ; encoding: [0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf]
 
-# W32: v_div_scale_f32 v5, vcc_lo, v1, v2, s3    ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
-# W64: v_div_scale_f32 v5, vcc, v1, v2, s3       ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00
+# W32: v_div_scale_f32 v5, vcc_lo, v1, v2, s3  ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
+# W64: v_div_scale_f32 v5, vcc, v1, v2, s3     ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
 
-# W32: v_div_scale_f32 v5, vcc_lo, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
-# W64: v_div_scale_f32 v5, vcc, v255, s2, s105   ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01
+# W32: v_div_scale_f32 v5, vcc_lo, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
+# W64: v_div_scale_f32 v5, vcc, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
 
+0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, s1, v255, exec_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01]
 # W64: v_div_scale_f32 v5, vcc, s1, v255, exec_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01]
-0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01
 
+0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, s105, s105, exec_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01]
 # W64: v_div_scale_f32 v5, vcc, s105, s105, exec_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01]
-0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01
 
+0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04
 # W32: v_div_scale_f32 v5, vcc_lo, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04]
 # W64: v_div_scale_f32 v5, vcc, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04]
-0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04
 
+0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v5, vcc_lo, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v5, vcc, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1
 # W32: v_div_scale_f32 v5, vcc_lo, -ttmp15, -src_scc, -ttmp15 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1]
 # W64: v_div_scale_f32 v5, vcc, -ttmp15, -src_scc, -ttmp15 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1]
-0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1
 
-# W32: v_div_scale_f32 v5, vcc_lo, m0, 0.5, m0   ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
-# W64: v_div_scale_f32 v5, vcc, m0, 0.5, m0      ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01
+# W32: v_div_scale_f32 v5, vcc_lo, m0, 0.5, m0 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64: v_div_scale_f32 v5, vcc, m0, 0.5, m0    ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
 
+0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, exec_lo, -1, vcc_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01]
 # W64: v_div_scale_f32 v5, vcc, exec_lo, -1, vcc_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01]
-0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01
 
+0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1
 # W32: v_div_scale_f32 v5, vcc_lo, -exec_hi, null, -vcc_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1]
 # W64: v_div_scale_f32 v5, vcc, -exec_hi, null, -vcc_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1]
-0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1
 
+0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v5, vcc_lo, null, exec_lo, neg(0xaf123456) ; encoding: [0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v5, vcc, null, exec_lo, neg(0xaf123456) ; encoding: [0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3
 # W32: v_div_scale_f32 v5, vcc_lo, -1, -exec_hi, -src_scc ; encoding: [0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3]
 # W64: v_div_scale_f32 v5, vcc, -1, -exec_hi, -src_scc ; encoding: [0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3]
-0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3
 
+0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b
 # W32: v_div_scale_f32 v5, vcc_lo, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b]
 # W64: v_div_scale_f32 v5, vcc, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b]
-0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b
 
+0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33
 # W32: v_div_scale_f32 v5, vcc_lo, -src_scc, vcc_lo, -1 mul:4 ; encoding: [0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33]
 # W64: v_div_scale_f32 v5, vcc, -src_scc, vcc_lo, -1 mul:4 ; encoding: [0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33]
-0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33
 
+0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v255, vcc_lo, neg(0xaf123456), -vcc_hi, null clamp div:2 ; encoding: [0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v255, vcc, neg(0xaf123456), -vcc_hi, null clamp div:2 ; encoding: [0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
-0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04
 # W32: v_div_scale_f64 v[5:6], vcc_lo, v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04]
 # W64: v_div_scale_f64 v[5:6], vcc, v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04]
-0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04
 
+0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00
 # W32: v_div_scale_f64 v[5:6], vcc_lo, v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00]
 # W64: v_div_scale_f64 v[5:6], vcc, v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00]
-0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00
 
+0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07
 # W32: v_div_scale_f64 v[5:6], vcc_lo, s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07]
 # W64: v_div_scale_f64 v[5:6], vcc, s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07]
-0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07
 
+0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -s[104:105], s[104:105], -s[104:105] ; encoding: [0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1]
 # W64: v_div_scale_f64 v[5:6], vcc, -s[104:105], s[104:105], -s[104:105] ; encoding: [0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1]
-0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1
 
+0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, vcc, -ttmp[14:15], -ttmp[14:15] ; encoding: [0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1]
 # W64: v_div_scale_f64 v[5:6], vcc, vcc, -ttmp[14:15], -ttmp[14:15] ; encoding: [0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1]
-0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1
 
+0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -ttmp[14:15], 0xaf123456, null ; encoding: [0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[5:6], vcc, -ttmp[14:15], 0xaf123456, null ; encoding: [0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -exec, -src_scc, -exec ; encoding: [0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1]
 # W64: v_div_scale_f64 v[5:6], vcc, -exec, -src_scc, -exec ; encoding: [0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1]
-0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1
 
+0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01
 # W32: v_div_scale_f64 v[5:6], vcc_lo, null, 0.5, vcc ; encoding: [0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01]
 # W64: v_div_scale_f64 v[5:6], vcc, null, 0.5, vcc ; encoding: [0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01]
-0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01
 
+0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -1, -1, 0xaf123456 ; encoding: [0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[5:6], vcc, -1, -1, 0xaf123456 ; encoding: [0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b
 # W32: v_div_scale_f64 v[5:6], vcc_lo, 0.5, null, -src_scc mul:2 ; encoding: [0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b]
 # W64: v_div_scale_f64 v[5:6], vcc, 0.5, null, -src_scc mul:2 ; encoding: [0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b]
-0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b
 
+0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -src_scc, -exec, 0.5 mul:4 ; encoding: [0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73]
 # W64: v_div_scale_f64 v[5:6], vcc, -src_scc, -exec, 0.5 mul:4 ; encoding: [0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73]
-0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73
 
+0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[254:255], vcc_lo, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
-0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
 
-# GFX11: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 
-# GFX11: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 
-# GFX11: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 
-# GFX11: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
-0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-# GFX11: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 
-# GFX11: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 
-# GFX11: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 
-# GFX11: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 
-# GFX11: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
-0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23
+0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_bf16_bf16 v2, v0, 0x20004000, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20
+# W32-REAL16: v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W32-FAKE16: v_dot2_bf16_bf16 v2, v0, 0x20004000, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W64-REAL16: v_dot2_bf16_bf16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W64-FAKE16: v_dot2_bf16_bf16 v2, v0, 0x20004000, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 
-# GFX11: v_dot2_bf16_bf16 v2, 0x20004000, v0, v2 ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
 0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20
+# W32-REAL16: v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W32-FAKE16: v_dot2_bf16_bf16 v2, 0x20004000, v0, v2 ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W64-REAL16: v_dot2_bf16_bf16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W64-FAKE16: v_dot2_bf16_bf16 v2, 0x20004000, v0, v2 ; encoding: [0x02,0x00,0x67,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+
+0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+
+0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1
+# W32-REAL16: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1
+# W32-REAL16: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3
+# W32-REAL16: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43
+# W32-REAL16: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
-0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23
+0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23
+# W32-REAL16: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x66,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_dot2_f16_f16 v2, v0, 0x20004000, v2 ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20
+# W32-REAL16: v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W32-FAKE16: v_dot2_f16_f16 v2, v0, 0x20004000, v2   ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W64-REAL16: v_dot2_f16_f16 v2.l, v0, 0x20004000, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
+# W64-FAKE16: v_dot2_f16_f16 v2, v0, 0x20004000, v2   ; encoding: [0x02,0x00,0x66,0xd6,0x00,0xff,0x09,0x04,0x00,0x40,0x00,0x20]
 
-# GFX11: v_dot2_f16_f16 v2, 0x20004000, v0, v2 ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
 0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20
+# W32-REAL16: v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W32-FAKE16: v_dot2_f16_f16 v2, 0x20004000, v0, v2   ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W64-REAL16: v_dot2_f16_f16 v2.l, 0x20004000, v0, v2.l ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+# W64-FAKE16: v_dot2_f16_f16 v2, 0x20004000, v0, v2   ; encoding: [0x02,0x00,0x66,0xd6,0xff,0x00,0x0a,0x04,0x00,0x40,0x00,0x20]
+
+0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+
+0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_fma_dx9_zero_f32 v5, v255, s2, s105   ; encoding: [0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_fma_dx9_zero_f32 v5, v255, s2, s105   ; encoding: [0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_fma_dx9_zero_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_fma_dx9_zero_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_fma_dx9_zero_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_fma_dx9_zero_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_fma_dx9_zero_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_fma_dx9_zero_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_fma_dx9_zero_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_dx9_zero_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_dx9_zero_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_fma_dx9_zero_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_fma_dx9_zero_f32 v5, m0, 0.5, m0      ; encoding: [0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_fma_dx9_zero_f32 v5, m0, 0.5, m0      ; encoding: [0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_fma_dx9_zero_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_fma_dx9_zero_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_fma_dx9_zero_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_fma_dx9_zero_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_fma_dx9_zero_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_dx9_zero_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_dx9_zero_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_fma_dx9_zero_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_fma_dx9_zero_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_fma_dx9_zero_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_fma_dx9_zero_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_fma_dx9_zero_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_fma_dx9_zero_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_dx9_zero_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_fma_f16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_fma_f16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_fma_f16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_fma_f16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_fma_f16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_fma_f16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_fma_f16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_fma_f16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_fma_f16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_fma_f16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_fma_f16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fma_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_fma_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_fma_f16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_fma_f16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_fma_f16 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_fma_f16 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_fma_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_fma_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_fma_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_fma_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fma_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_fma_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_fma_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX11: v_fma_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_fma_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23
+# GFX11: v_fma_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
 # CHECK: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp div:2    ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00
+# GFX11: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp div:2 ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_fma_f32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_fma_f32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_fma_f32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_fma_f32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_fma_f32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_fma_f32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_fma_f32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_fma_f32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_fma_f32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_fma_f32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_fma_f32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_fma_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_fma_f32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_fma_f32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_fma_f32 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_fma_f32 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_fma_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_fma_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_fma_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_fma_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_fma_f32 v5, 0.5, -m0, 0.5 mul:2       ; encoding: [0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_fma_f32 v5, 0.5, -m0, 0.5 mul:2       ; encoding: [0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_fma_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_fma_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_fma_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04
+# GFX11: v_fma_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX11: v_fma_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00]
 0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00
+# GFX11: v_fma_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00]
 
-# GFX11: v_fma_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07]
 0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07
+# GFX11: v_fma_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07]
 
-# GFX11: v_fma_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1]
 0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1
+# GFX11: v_fma_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1]
 
-# GFX11: v_fma_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1]
 0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1
+# GFX11: v_fma_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1]
 
-# GFX11: v_fma_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1]
 0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1
+# GFX11: v_fma_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1]
 
-# GFX11: v_fma_f64 v[5:6], null, 0.5, vcc        ; encoding: [0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01]
 0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01
+# GFX11: v_fma_f64 v[5:6], null, 0.5, vcc        ; encoding: [0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01]
 
-# GFX11: v_fma_f64 v[5:6], -1, -1, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f64 v[5:6], -1, -1, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_fma_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b]
 0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b
+# GFX11: v_fma_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b]
 
-# GFX11: v_fma_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73]
 0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73
+# GFX11: v_fma_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73]
 
-# GFX11: v_fma_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
+# GFX11: v_fma_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_ldexp_f32 v5, v1, v2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_ldexp_f32 v5, v1, v2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_ldexp_f32 v5, v255, v255              ; encoding: [0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_ldexp_f32 v5, v255, v255              ; encoding: [0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_ldexp_f32 v5, s1, s2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_ldexp_f32 v5, s1, s2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, s105, s105              ; encoding: [0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_ldexp_f32 v5, s105, s105              ; encoding: [0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_ldexp_f32 v5, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, vcc_hi, 0xaf123456      ; encoding: [0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_ldexp_f32 v5, vcc_hi, 0xaf123456      ; encoding: [0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_ldexp_f32 v5, ttmp15, src_scc         ; encoding: [0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_ldexp_f32 v5, ttmp15, src_scc         ; encoding: [0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_ldexp_f32 v5, m0, 0.5                 ; encoding: [0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_ldexp_f32 v5, m0, 0.5                 ; encoding: [0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_ldexp_f32 v5, exec_lo, -1             ; encoding: [0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_ldexp_f32 v5, exec_lo, -1             ; encoding: [0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_ldexp_f32 v5, exec_hi, null           ; encoding: [0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_ldexp_f32 v5, exec_hi, null           ; encoding: [0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, null, exec_lo           ; encoding: [0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_ldexp_f32 v5, null, exec_lo           ; encoding: [0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, -1, exec_hi             ; encoding: [0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_ldexp_f32 v5, -1, exec_hi             ; encoding: [0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_ldexp_f32 v5, 0.5, m0 mul:2           ; encoding: [0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08]
 0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08
+# GFX11: v_ldexp_f32 v5, 0.5, m0 mul:2           ; encoding: [0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08]
 
-# GFX11: v_ldexp_f32 v5, src_scc, vcc_lo mul:4   ; encoding: [0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10]
 0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10
+# GFX11: v_ldexp_f32 v5, src_scc, vcc_lo mul:4   ; encoding: [0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10]
 
-# GFX11: v_ldexp_f32 v255, -|0xaf123456|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf]
 0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf
+# GFX11: v_ldexp_f32 v255, -|0xaf123456|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_ldexp_f64 v[5:6], v[1:2], v2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_ldexp_f64 v[5:6], v[1:2], v2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], v[1:2], v255        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00
+# GFX11: v_ldexp_f64 v[5:6], v[1:2], v255        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], v[1:2], s2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], v[1:2], s2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], v[1:2], s105        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], v[1:2], s105        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], v[254:255], ttmp15  ; encoding: [0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], v[254:255], ttmp15  ; encoding: [0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], s[2:3], vcc_hi      ; encoding: [0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], s[2:3], vcc_hi      ; encoding: [0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], s[104:105], vcc_lo  ; encoding: [0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], s[104:105], vcc_lo  ; encoding: [0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], vcc, m0             ; encoding: [0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], vcc, m0             ; encoding: [0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], exec, exec_lo       ; encoding: [0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], exec, exec_lo       ; encoding: [0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], null, null          ; encoding: [0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00
+# GFX11: v_ldexp_f64 v[5:6], null, null          ; encoding: [0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], -1, -1              ; encoding: [0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_ldexp_f64 v[5:6], -1, -1              ; encoding: [0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_ldexp_f64 v[5:6], 0.5, 0.5 mul:2      ; encoding: [0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08]
 0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08
+# GFX11: v_ldexp_f64 v[5:6], 0.5, 0.5 mul:2      ; encoding: [0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08]
 
-# GFX11: v_ldexp_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30]
 0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30
+# GFX11: v_ldexp_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30]
 
-# GFX11: v_ldexp_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf
+# GFX11: v_ldexp_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lerp_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_lerp_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_lerp_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_lerp_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_lerp_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_lerp_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_lerp_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_lerp_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_lerp_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_lerp_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_lerp_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_lerp_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lerp_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_lerp_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_lerp_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_lerp_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_lerp_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_lerp_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_lerp_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_lerp_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_lerp_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_lerp_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lerp_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_lerp_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_lerp_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_lerp_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_lerp_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_lerp_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_lerp_u8 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_lerp_u8 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_add_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_lshl_add_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_lshl_add_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_lshl_add_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_lshl_add_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_lshl_add_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_lshl_add_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_lshl_add_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_lshl_add_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_lshl_add_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_lshl_add_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_add_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_add_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_lshl_add_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_lshl_add_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_lshl_add_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_lshl_add_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_lshl_add_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_lshl_add_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_lshl_add_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_lshl_add_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_add_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_add_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_lshl_add_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_lshl_add_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_lshl_add_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_lshl_add_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_lshl_add_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_lshl_add_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_add_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_or_b32 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_lshl_or_b32 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_lshl_or_b32 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_lshl_or_b32 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_lshl_or_b32 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_lshl_or_b32 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_lshl_or_b32 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_lshl_or_b32 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_lshl_or_b32 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_lshl_or_b32 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_lshl_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_lshl_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_lshl_or_b32 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_lshl_or_b32 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_lshl_or_b32 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_lshl_or_b32 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_lshl_or_b32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_lshl_or_b32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_lshl_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshl_or_b32 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_lshl_or_b32 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_lshl_or_b32 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_lshl_or_b32 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_lshl_or_b32 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_lshl_or_b32 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_lshl_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_lshl_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_lshlrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3c,0xd7,0x01,0xd5,0x00,0x00]
 0x05,0x00,0x3c,0xd7,0x01,0xd5,0x00,0x00
+# GFX11: v_lshlrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3c,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3c,0xd7,0xff,0xfd,0x00,0x00]
 0x05,0x00,0x3c,0xd7,0xff,0xfd,0x00,0x00
+# GFX11: v_lshlrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3c,0xd7,0xff,0xfd,0x00,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3c,0xd7,0x7e,0x04,0x02,0x00]
 0x05,0x00,0x3c,0xd7,0x7e,0x04,0x02,0x00
+# GFX11: v_lshlrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3c,0xd7,0x7e,0x04,0x02,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3c,0xd7,0x7f,0xfc,0x03,0x00]
 0x05,0x00,0x3c,0xd7,0x7f,0xfc,0x03,0x00
+# GFX11: v_lshlrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3c,0xd7,0x7f,0xfc,0x03,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3c,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x3c,0xd7,0x7c,0xf8,0x00,0x00
+# GFX11: v_lshlrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3c,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3c,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x3c,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_lshlrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3c,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_lshlrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3c,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3c,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_lshlrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3c,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshlrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3c,0xd7,0xfd,0xfa,0x01,0x00]
 0x05,0x00,0x3c,0xd7,0xfd,0xfa,0x01,0x00
+# GFX11: v_lshlrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3c,0xd7,0xfd,0xfa,0x01,0x00]
 
-# GFX11: v_lshlrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3c,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 0xfe,0x00,0x3c,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_lshlrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3c,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_lshrrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00
+# GFX11: v_lshrrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00
+# GFX11: v_lshrrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00]
 0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00
+# GFX11: v_lshrrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00]
 0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00
+# GFX11: v_lshrrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00
+# GFX11: v_lshrrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_lshrrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_lshrrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_lshrrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_lshrrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00]
 0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00
+# GFX11: v_lshrrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00]
 
-# GFX11: v_lshrrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_lshrrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_i16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_mad_i16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_mad_i16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_mad_i16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_mad_i16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_mad_i16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_mad_i16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_mad_i16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_mad_i16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_mad_i16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_mad_i16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_i16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_i16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_i16 v5, m0, 0x3800, m0            ; encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_i16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_mad_i16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_mad_i16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_mad_i16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_mad_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_mad_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_mad_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04
+# GFX11: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX11: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00
+# GFX11: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
-# GFX11: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
 0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07
+# GFX11: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
 
-# GFX11: v_mad_i32_i16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01
+# GFX11: v_mad_i32_i16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX11: v_mad_i32_i16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01]
 0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01
+# GFX11: v_mad_i32_i16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01]
 
-# GFX11: v_mad_i32_i16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i32_i16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_i32_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_i32_i16 v5, m0, 0x3800, m0        ; encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_i32_i16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
 0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01
+# GFX11: v_mad_i32_i16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
 
-# GFX11: v_mad_i32_i16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01]
 0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01
+# GFX11: v_mad_i32_i16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01]
 
-# GFX11: v_mad_i32_i16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01]
 0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01
+# GFX11: v_mad_i32_i16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01]
 
-# GFX11: v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_i32_i16 v5, 0x3800, m0, -1
 0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03
+# GFX11: v_mad_i32_i16 v5, 0x3800, m0, -1        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
 0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX11: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
 
-# GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_mad_i32_i24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_mad_i32_i24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_mad_i32_i24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_mad_i32_i24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_mad_i32_i24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_mad_i32_i24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_mad_i32_i24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_mad_i32_i24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_mad_i32_i24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_i32_i24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_i32_i24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_i32_i24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_i32_i24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_i32_i24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_mad_i32_i24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_mad_i32_i24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_mad_i32_i24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_mad_i32_i24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_mad_i32_i24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_i32_i24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_i32_i24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_mad_i32_i24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_mad_i32_i24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_mad_i32_i24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_mad_i32_i24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_mad_i32_i24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_mad_i32_i24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_i32_i24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00
 # W32: v_mad_i64_i32 v[5:6], s12, v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00
 # W32: v_mad_i64_i32 v[5:6], s12, v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00
 # W32: v_mad_i64_i32 v[5:6], s12, s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00]
-0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00
 
+0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00
 # W32: v_mad_i64_i32 v[5:6], s12, s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01
 # W32: v_mad_i64_i32 v[5:6], s12, vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01]
-0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01
 
+0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 # W32: v_mad_i64_i32 v[5:6], s12, vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 
+0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07
 # W32: v_mad_i64_i32 v[5:6], s12, ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07]
-0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07
 
+0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01
 # W32: v_mad_i64_i32 v[5:6], s12, m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01]
-0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01
 
+0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01
 # W32: v_mad_i64_i32 v[5:6], s12, exec_lo, -1, exec ; encoding: [0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], exec_lo, -1, exec ; encoding: [0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01]
-0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01
 
+0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01
 # W32: v_mad_i64_i32 v[5:6], s12, exec_hi, null, vcc ; encoding: [0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], exec_hi, null, vcc ; encoding: [0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01]
-0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01
 
+0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01
 # W32: v_mad_i64_i32 v[5:6], s12, null, exec_lo, null ; encoding: [0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01]
 # W64: v_mad_i64_i32 v[5:6], s[12:13], null, exec_lo, null ; encoding: [0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01]
-0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01
 
+0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03
 # W32: v_mad_i64_i32 v[5:6], s104, -1, exec_hi, -1 ; encoding: [0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03]
 # W64: v_mad_i64_i32 v[5:6], s[104:105], -1, exec_hi, -1 ; encoding: [0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03]
-0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03
 
+0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 # W32: v_mad_i64_i32 v[5:6], vcc_lo, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_mad_i64_i32 v[5:6], vcc, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03
 # W32: v_mad_i64_i32 v[5:6], ttmp14, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03]
 # W64: v_mad_i64_i32 v[5:6], ttmp[14:15], src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03]
-0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03
 
-# GFX11: v_mad_i64_i32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_i64_i32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_mad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_mad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_mad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_mad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_mad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_mad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_mad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_mad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_mad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_mad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_u16 v5, m0, 0x3800, m0            ; encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_mad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_mad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_mad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_mad_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_mad_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_mad_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04
+# GFX11: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX11: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00
+# GFX11: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
-# GFX11: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
 0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07
+# GFX11: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
 
-# GFX11: v_mad_u32_u16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01
+# GFX11: v_mad_u32_u16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX11: v_mad_u32_u16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01]
 0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01
+# GFX11: v_mad_u32_u16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01]
 
-# GFX11: v_mad_u32_u16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u32_u16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_u32_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_u32_u16 v5, m0, 0x3800, m0        ; encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_u32_u16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
 0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01
+# GFX11: v_mad_u32_u16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
 
-# GFX11: v_mad_u32_u16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01]
 0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01
+# GFX11: v_mad_u32_u16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01]
 
-# GFX11: v_mad_u32_u16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01]
 0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01
+# GFX11: v_mad_u32_u16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01]
 
-# GFX11: v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_u32_u16 v5, 0x3800, m0, -1
 0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03
+# GFX11: v_mad_u32_u16 v5, 0x3800, m0, -1        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
 0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX11: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
 
-# GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_mad_u32_u24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_mad_u32_u24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_mad_u32_u24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_mad_u32_u24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_mad_u32_u24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_mad_u32_u24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_mad_u32_u24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_mad_u32_u24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_mad_u32_u24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_u32_u24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_u32_u24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_mad_u32_u24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_mad_u32_u24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mad_u32_u24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_mad_u32_u24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_mad_u32_u24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_mad_u32_u24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_mad_u32_u24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_mad_u32_u24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_u32_u24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mad_u32_u24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_mad_u32_u24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_mad_u32_u24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_mad_u32_u24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_mad_u32_u24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_mad_u32_u24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_mad_u32_u24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_u32_u24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00
 # W32: v_mad_u64_u32 v[5:6], s12, v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00
 # W32: v_mad_u64_u32 v[5:6], s12, v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00
 # W32: v_mad_u64_u32 v[5:6], s12, s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00]
-0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00
 
+0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00
 # W32: v_mad_u64_u32 v[5:6], s12, s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01
 # W32: v_mad_u64_u32 v[5:6], s12, vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01]
-0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01
 
+0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 # W32: v_mad_u64_u32 v[5:6], s12, vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 
+0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07
 # W32: v_mad_u64_u32 v[5:6], s12, ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07]
-0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07
 
+0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01
 # W32: v_mad_u64_u32 v[5:6], s12, m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01]
-0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01
 # W32: v_mad_u64_u32 v[5:6], s12, exec_lo, -1, exec ; encoding: [0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], exec_lo, -1, exec ; encoding: [0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01]
-0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01
 # W32: v_mad_u64_u32 v[5:6], s12, exec_hi, null, vcc ; encoding: [0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], exec_hi, null, vcc ; encoding: [0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01]
-0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01
 # W32: v_mad_u64_u32 v[5:6], s12, null, exec_lo, null ; encoding: [0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01]
 # W64: v_mad_u64_u32 v[5:6], s[12:13], null, exec_lo, null ; encoding: [0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01]
-0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01
 
+0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03
 # W32: v_mad_u64_u32 v[5:6], s104, -1, exec_hi, -1 ; encoding: [0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03]
 # W64: v_mad_u64_u32 v[5:6], s[104:105], -1, exec_hi, -1 ; encoding: [0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03]
-0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03
 
+0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 # W32: v_mad_u64_u32 v[5:6], vcc_lo, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_mad_u64_u32 v[5:6], vcc, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03
 # W32: v_mad_u64_u32 v[5:6], ttmp14, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03]
 # W64: v_mad_u64_u32 v[5:6], ttmp[14:15], src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03]
-0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03
 
-# GFX11: v_mad_u64_u32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mad_u64_u32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4c,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4c,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x4c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x4c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_max3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x4c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_max3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x4c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x4c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x4c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_max3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x4c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x4c,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x4c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x4c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x4c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_max3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x4c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_max3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x4c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x4c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x4c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x4c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x4c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_max3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x4c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_max3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4c,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x4c,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX11: v_max3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4c,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_max3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x4c,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x4c,0xd6,0xfd,0xd4,0x04,0x23
+# GFX11: v_max3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x4c,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_max3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x4c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x4c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x4c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
 # CHECK: v_max3_f16 v5, v255, s2, s105 mul:2     ; encoding: [0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x09]
 0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x09
+# GFX11: v_max3_f16 v5, v255, s2, s105 mul:2     ; encoding: [0x05,0x00,0x4c,0xd6,0xff,0x05,0xa4,0x09]
 
-# GFX11: v_max3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1c,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1c,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1c,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x1c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x1c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_max3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x1c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_max3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_max3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x1c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x1c,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x1c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x1c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x1c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_max3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x1c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_max3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x1c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x1c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x1c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x1c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x1c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_max3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x1c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_max3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x1c,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x1c,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_max3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x1c,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_max3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x1c,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x1c,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_max3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x1c,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_max3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x1c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x1c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x1c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_max3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_max3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_max3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_max3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_max3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_max3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_max3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_max3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_max3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_max3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_max3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_max3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_max3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_max3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_max3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_max3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_max3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_max3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_max3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_max3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_max3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_max3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_max3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_max3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_max3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_max3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_max3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_max3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_max3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_max3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_max3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_max3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_max3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_max3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_max3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_max3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_max3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_max3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_max3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_max3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_max3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_max3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_max3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_max3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_max3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_max3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_max3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_max3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_max3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_max3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_max3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_max3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_max3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_max3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x2a,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2a,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_max_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x2a,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_max_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x2a,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x2a,0xd7,0xfe,0xfd,0x03,0x00
+# GFX11: v_max_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x2a,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX11: v_max_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x2a,0xd7,0x02,0x08,0x00,0x00]
 0x05,0x00,0x2a,0xd7,0x02,0x08,0x00,0x00
+# GFX11: v_max_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x2a,0xd7,0x02,0x08,0x00,0x00]
 
-# GFX11: v_max_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x2a,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x2a,0xd7,0x68,0xd0,0x00,0x00
+# GFX11: v_max_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x2a,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX11: v_max_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x2a,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x2a,0xd7,0x6a,0xf4,0x00,0x00
+# GFX11: v_max_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x2a,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX11: v_max_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x2a,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2a,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_max_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x2a,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_max_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x2a,0xd7,0x7e,0xfa,0x01,0x20]
 0x05,0x01,0x2a,0xd7,0x7e,0xfa,0x01,0x20
+# GFX11: v_max_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x2a,0xd7,0x7e,0xfa,0x01,0x20]
 
-# GFX11: v_max_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x2a,0xd7,0x7c,0xe0,0x01,0x00]
 0x05,0x00,0x2a,0xd7,0x7c,0xe0,0x01,0x00
+# GFX11: v_max_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x2a,0xd7,0x7c,0xe0,0x01,0x00]
 
-# GFX11: v_max_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x2a,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x2a,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_max_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x2a,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_max_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x2a,0xd7,0xf0,0xf8,0x00,0x08]
 0x05,0x00,0x2a,0xd7,0xf0,0xf8,0x00,0x08
+# GFX11: v_max_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x2a,0xd7,0xf0,0xf8,0x00,0x08]
 
-# GFX11: v_max_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x2a,0xd7,0xfd,0xfc,0x00,0x70]
 0x05,0x03,0x2a,0xd7,0xfd,0xfc,0x00,0x70
+# GFX11: v_max_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x2a,0xd7,0xfd,0xfc,0x00,0x70]
 
-# GFX11: v_max_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x2a,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x2a,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_max_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x2a,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_max_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_max_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_max_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_max_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_max_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_max_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_max_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_max_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_max_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_max_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_max_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_max_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_maxmin_f16 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x60,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x60,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_maxmin_f16 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x60,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_maxmin_f16 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x60,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x60,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_maxmin_f16 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x60,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_maxmin_f16 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x60,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x60,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_maxmin_f16 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x60,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_maxmin_f16 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x60,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x60,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_maxmin_f16 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x60,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_maxmin_f16 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x60,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x60,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_maxmin_f16 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x60,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_maxmin_f16 v5, vcc_hi, 0xfe0b, v255   ; encoding: [0x05,0x00,0x60,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x60,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_maxmin_f16 v5, vcc_hi, 0xfe0b, v255   ; encoding: [0x05,0x00,0x60,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_maxmin_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x60,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x60,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_maxmin_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x60,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_maxmin_f16 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x60,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x60,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_maxmin_f16 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x60,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_maxmin_f16 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x60,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x60,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_maxmin_f16 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x60,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_maxmin_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x60,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x60,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_maxmin_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x60,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_maxmin_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x60,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x60,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_maxmin_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x60,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_maxmin_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x60,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x60,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_maxmin_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x60,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_maxmin_f16 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x60,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x60,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_maxmin_f16 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x60,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_maxmin_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x60,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x60,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_maxmin_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x60,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_maxmin_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x60,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0x83,0x60,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00
+# GFX11: v_maxmin_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x60,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_maxmin_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x5e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x5e,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_maxmin_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x5e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_maxmin_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x5e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x5e,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_maxmin_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x5e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_maxmin_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x5e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x5e,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_maxmin_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x5e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_maxmin_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x5e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x5e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_maxmin_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x5e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_maxmin_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x5e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x5e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_maxmin_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x5e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_maxmin_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x5e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x5e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x5e,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x5e,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_maxmin_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x5e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_maxmin_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x5e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x5e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_maxmin_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x5e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_maxmin_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x5e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x5e,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_maxmin_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x5e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_maxmin_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x5e,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x5e,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_maxmin_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x5e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_maxmin_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x5e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x5e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x5e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x5e,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x5e,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_maxmin_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x5e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_maxmin_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x5e,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x5e,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_maxmin_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x5e,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_maxmin_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x5e,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x5e,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_maxmin_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x5e,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_maxmin_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x5e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x5e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x5e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_maxmin_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_maxmin_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_maxmin_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_maxmin_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_maxmin_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_maxmin_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_maxmin_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_maxmin_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_maxmin_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_maxmin_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_maxmin_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_maxmin_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_maxmin_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_maxmin_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_maxmin_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_maxmin_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_maxmin_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_maxmin_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_maxmin_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_maxmin_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_maxmin_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_maxmin_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_maxmin_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_maxmin_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_maxmin_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_maxmin_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_maxmin_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_maxmin_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_maxmin_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_maxmin_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_maxmin_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_maxmin_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_maxmin_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_maxmin_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_maxmin_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_maxmin_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_maxmin_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_maxmin_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_maxmin_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_maxmin_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_maxmin_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_maxmin_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_maxmin_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_maxmin_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_maxmin_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_maxmin_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_maxmin_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_maxmin_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_maxmin_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_maxmin_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mbcnt_hi_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mbcnt_hi_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mbcnt_lo_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mbcnt_lo_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4f,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4f,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4f,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4f,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4f,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4f,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x4f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x4f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_med3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x4f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_med3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x4f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x4f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x4f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_med3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x4f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x4f,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x4f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x4f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x4f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_med3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x4f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_med3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x4f,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x4f,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x4f,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x4f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x4f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_med3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x4f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_med3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX11: v_med3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_med3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x4f,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x4f,0xd6,0xfd,0xd4,0x04,0x23
+# GFX11: v_med3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x4f,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_med3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x4f,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x4f,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x4f,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
 # CHECK: v_med3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] div:2    ; encoding: [0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x5b]
 0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x5b
+# GFX11: v_med3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] div:2 ; encoding: [0x05,0x10,0x4f,0xd6,0xf0,0xfa,0xc0,0x5b]
 
-# GFX11: v_med3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1f,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1f,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1f,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x1f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x1f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_med3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x1f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_med3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_med3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x1f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x1f,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x1f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x1f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x1f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_med3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x1f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_med3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x1f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x1f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x1f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x1f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x1f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_med3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x1f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_med3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x1f,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x1f,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_med3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x1f,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_med3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x1f,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x1f,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_med3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x1f,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_med3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x1f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x1f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x1f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_med3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_med3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_med3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_med3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_med3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_med3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_med3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_med3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_med3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_med3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_med3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_med3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_med3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_med3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_med3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_med3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_med3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_med3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_med3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_med3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_med3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_med3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_med3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_med3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_med3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_med3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_med3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_med3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_med3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_med3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_med3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_med3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_med3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_med3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_med3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_med3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_med3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_med3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_med3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_med3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_med3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_med3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_med3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_med3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_med3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_med3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_med3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_med3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_med3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_med3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_med3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_med3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_med3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_med3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_med3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x49,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x49,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_f16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x49,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x49,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x49,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_f16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x49,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x49,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x49,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_f16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x49,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x49,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x49,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_f16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x49,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x49,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x49,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_f16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x49,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x49,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x49,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_f16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x49,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x49,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x49,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_min3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x49,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_min3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x49,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x49,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_f16 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x49,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_min3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x49,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x49,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_f16 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x49,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x49,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x49,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_min3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x49,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_min3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x49,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x49,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x49,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x49,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x49,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_min3_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x49,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_min3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x49,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x49,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX11: v_min3_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x49,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX11: v_min3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x49,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x49,0xd6,0xfd,0xd4,0x04,0x23
+# GFX11: v_min3_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x49,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX11: v_min3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x49,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x49,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x49,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
 # CHECK: v_min3_f16 v5, m0, 0.5, m0 clamp mul:4  ; encoding: [0x05,0x80,0x49,0xd6,0x7d,0xe0,0xf5,0x11]
 0x05,0x80,0x49,0xd6,0x7d,0xe0,0xf5,0x11
+# GFX11: v_min3_f16 v5, m0, 0.5, m0 clamp mul:4  ; encoding: [0x05,0x80,0x49,0xd6,0x7d,0xe0,0xf5,0x11]
 
-# GFX11: v_min3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x19,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x19,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_f32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x19,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x19,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x19,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_f32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x19,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x19,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x19,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_f32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x19,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x19,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x19,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_f32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x19,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x19,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x19,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_f32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x19,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x19,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x19,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x19,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x19,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x19,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_min3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x19,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_min3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x19,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x19,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_f32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x19,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_min3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x19,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x19,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_f32 v5, |exec_lo|, -1, vcc_hi    ; encoding: [0x05,0x01,0x19,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x19,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x19,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_min3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x19,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_min3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x19,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x19,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x19,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x19,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x19,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_min3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x19,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_min3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x19,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x19,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_min3_f32 v5, 0.5, -m0, 0.5 mul:2      ; encoding: [0x05,0x00,0x19,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_min3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x19,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x19,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_min3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x19,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_min3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x19,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x19,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x19,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_min3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_min3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_min3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_min3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_min3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_min3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_min3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_min3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_min3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_min3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_min3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_min3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_min3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_min3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_min3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_min3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_min3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_min3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_min3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_min3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_min3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX11: v_min3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_min3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_min3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_min3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_min3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX11: v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_min3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_min3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_min3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_min3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_min3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_min3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_min3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_min3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_min3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_min3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_min3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_min3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_min3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_min3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_min3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_min3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_min3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_min3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_min3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_min3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_min3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_min3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_min3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_min3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_min3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_min3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_min3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_min3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x29,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x29,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_min_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x29,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_min_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x29,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x29,0xd7,0xfe,0xfd,0x03,0x00
+# GFX11: v_min_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x29,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX11: v_min_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x29,0xd7,0x02,0x08,0x00,0x00]
 0x05,0x00,0x29,0xd7,0x02,0x08,0x00,0x00
+# GFX11: v_min_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x29,0xd7,0x02,0x08,0x00,0x00]
 
-# GFX11: v_min_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x29,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x29,0xd7,0x68,0xd0,0x00,0x00
+# GFX11: v_min_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x29,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX11: v_min_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x29,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x29,0xd7,0x6a,0xf4,0x00,0x00
+# GFX11: v_min_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x29,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX11: v_min_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x29,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x29,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_min_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x29,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_min_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x29,0xd7,0x7e,0xfa,0x01,0x20]
 0x05,0x01,0x29,0xd7,0x7e,0xfa,0x01,0x20
+# GFX11: v_min_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x29,0xd7,0x7e,0xfa,0x01,0x20]
 
-# GFX11: v_min_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x29,0xd7,0x7c,0xe0,0x01,0x00]
 0x05,0x00,0x29,0xd7,0x7c,0xe0,0x01,0x00
+# GFX11: v_min_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x29,0xd7,0x7c,0xe0,0x01,0x00]
 
-# GFX11: v_min_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x29,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x29,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_min_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x29,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_min_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x29,0xd7,0xf0,0xf8,0x00,0x08]
 0x05,0x00,0x29,0xd7,0xf0,0xf8,0x00,0x08
+# GFX11: v_min_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x29,0xd7,0xf0,0xf8,0x00,0x08]
 
-# GFX11: v_min_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x29,0xd7,0xfd,0xfc,0x00,0x70]
 0x05,0x03,0x29,0xd7,0xfd,0xfc,0x00,0x70
+# GFX11: v_min_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x29,0xd7,0xfd,0xfc,0x00,0x70]
 
-# GFX11: v_min_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x29,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x29,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_min_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x29,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_min_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_min_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_min_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_min_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_min_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_min_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_min_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_min_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_min_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_min_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_min_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_min_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_minmax_f16 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x61,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x61,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_minmax_f16 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x61,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_minmax_f16 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x61,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x61,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_minmax_f16 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x61,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_minmax_f16 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x61,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x61,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_minmax_f16 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x61,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_minmax_f16 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x61,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x61,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_minmax_f16 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x61,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_minmax_f16 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x61,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x61,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_minmax_f16 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x61,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_minmax_f16 v5, vcc_hi, 0xfe0b, v255   ; encoding: [0x05,0x00,0x61,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x61,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_minmax_f16 v5, vcc_hi, 0xfe0b, v255   ; encoding: [0x05,0x00,0x61,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_minmax_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x61,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x61,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_minmax_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x61,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_minmax_f16 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x61,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x61,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_minmax_f16 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x61,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_minmax_f16 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x61,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x61,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_minmax_f16 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x61,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_minmax_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x61,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x61,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_minmax_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x61,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_minmax_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x61,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x61,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX11: v_minmax_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x61,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_minmax_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x61,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x61,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_minmax_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x61,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_minmax_f16 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x61,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x61,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_minmax_f16 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x61,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_minmax_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x61,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x61,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_minmax_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x61,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_minmax_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x61,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0x83,0x61,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00
+# GFX11: v_minmax_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x61,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_minmax_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x5f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x5f,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_minmax_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x5f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_minmax_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x5f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x5f,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_minmax_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x5f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_minmax_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x5f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x5f,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_minmax_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x5f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_minmax_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x5f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x5f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_minmax_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x5f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_minmax_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x5f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x5f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_minmax_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x5f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_minmax_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x5f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x5f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x5f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x5f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_minmax_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x5f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_minmax_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x5f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x5f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_minmax_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x5f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_minmax_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x5f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x5f,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_minmax_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x5f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_minmax_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x5f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x5f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_minmax_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x5f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_minmax_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x5f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x5f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x5f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x5f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x5f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_minmax_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x5f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_minmax_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x5f,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x5f,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_minmax_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x5f,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_minmax_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x5f,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x5f,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_minmax_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x5f,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_minmax_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x5f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x5f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x5f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_minmax_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_minmax_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_minmax_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_minmax_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_minmax_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_minmax_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_minmax_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_minmax_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_minmax_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_minmax_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_minmax_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_minmax_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_minmax_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_minmax_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_minmax_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_minmax_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_minmax_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_minmax_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_minmax_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_minmax_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_minmax_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_minmax_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_minmax_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_minmax_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_minmax_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_minmax_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_minmax_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_minmax_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_minmax_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_minmax_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_minmax_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_minmax_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_minmax_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_minmax_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_minmax_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_minmax_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_minmax_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_minmax_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_minmax_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_minmax_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_minmax_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_minmax_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_minmax_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_minmax_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_minmax_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_minmax_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_minmax_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_minmax_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_minmax_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_minmax_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00]
 0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04]
 0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01]
 0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07]
 0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01]
 0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01]
 0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01]
 0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03]
 0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03]
 
-# GFX11: v_mqsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03]
 0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03
+# GFX11: v_mqsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03]
 
-# GFX11: v_mqsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_mqsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], v2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], v2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], v255, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], v255, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], s2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], s2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], s105, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], v[1:2], s105, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], v[254:255], ttmp15, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], v[254:255], ttmp15, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], s[2:3], vcc_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], s[2:3], vcc_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], s[104:105], vcc_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], s[104:105], vcc_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], vcc, m0, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], vcc, m0, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], ttmp[14:15], exec_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], ttmp[14:15], exec_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], exec, exec_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], exec, exec_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], null, null, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], null, null, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], -1, -1, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], -1, -1, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], 0.5, 0.5, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], 0.5, 0.5, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[5:8], src_scc, src_scc, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07
+# GFX11: v_mqsad_u32_u8 v[5:8], src_scc, src_scc, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07]
 
-# GFX11: v_mqsad_u32_u8 v[252:255], 0xaf123456, 0xaf123456, v[3:6] clamp ; encoding: [0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
+# GFX11: v_mqsad_u32_u8 v[252:255], 0xaf123456, 0xaf123456, v[3:6] clamp ; encoding: [0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_msad_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_msad_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_msad_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_msad_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_msad_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_msad_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_msad_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_msad_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_msad_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_msad_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_msad_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_msad_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_msad_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_msad_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_msad_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_msad_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_msad_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_msad_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_msad_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_msad_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_msad_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_msad_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_msad_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_msad_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_msad_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_msad_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_msad_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_msad_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_msad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_msad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x28,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x28,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mul_f64 v[5:6], v[1:2], v[2:3]        ; encoding: [0x05,0x00,0x28,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mul_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x28,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x28,0xd7,0xfe,0xfd,0x03,0x00
+# GFX11: v_mul_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x28,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX11: v_mul_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x28,0xd7,0x02,0x08,0x00,0x00]
 0x05,0x00,0x28,0xd7,0x02,0x08,0x00,0x00
+# GFX11: v_mul_f64 v[5:6], s[2:3], s[4:5]        ; encoding: [0x05,0x00,0x28,0xd7,0x02,0x08,0x00,0x00]
 
-# GFX11: v_mul_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x28,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x28,0xd7,0x68,0xd0,0x00,0x00
+# GFX11: v_mul_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x28,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX11: v_mul_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x28,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x28,0xd7,0x6a,0xf4,0x00,0x00
+# GFX11: v_mul_f64 v[5:6], vcc, ttmp[14:15]      ; encoding: [0x05,0x00,0x28,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX11: v_mul_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x28,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_f64 v[5:6], ttmp[14:15], 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd7,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x28,0xd7,0x7e,0xfa,0x01,0x20]
 0x05,0x01,0x28,0xd7,0x7e,0xfa,0x01,0x20
+# GFX11: v_mul_f64 v[5:6], -|exec|, src_scc      ; encoding: [0x05,0x01,0x28,0xd7,0x7e,0xfa,0x01,0x20]
 
-# GFX11: v_mul_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x28,0xd7,0x7c,0xe0,0x01,0x00]
 0x05,0x00,0x28,0xd7,0x7c,0xe0,0x01,0x00
+# GFX11: v_mul_f64 v[5:6], null, 0.5             ; encoding: [0x05,0x00,0x28,0xd7,0x7c,0xe0,0x01,0x00]
 
-# GFX11: v_mul_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x28,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x28,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_mul_f64 v[5:6], -1, -1                ; encoding: [0x05,0x00,0x28,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_mul_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x28,0xd7,0xf0,0xf8,0x00,0x08]
 0x05,0x00,0x28,0xd7,0xf0,0xf8,0x00,0x08
+# GFX11: v_mul_f64 v[5:6], 0.5, null mul:2       ; encoding: [0x05,0x00,0x28,0xd7,0xf0,0xf8,0x00,0x08]
 
-# GFX11: v_mul_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x28,0xd7,0xfd,0xfc,0x00,0x70]
 0x05,0x03,0x28,0xd7,0xfd,0xfc,0x00,0x70
+# GFX11: v_mul_f64 v[5:6], -|src_scc|, -|exec| mul:4 ; encoding: [0x05,0x03,0x28,0xd7,0xfd,0xfc,0x00,0x70]
 
-# GFX11: v_mul_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x28,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_f64 v[254:255], 0xaf123456, -|vcc| clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd7,0xff,0xd4,0x00,0x58,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mul_hi_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mul_hi_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_mul_hi_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_mul_hi_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_mul_hi_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_mul_hi_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_mul_hi_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_mul_hi_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_mul_hi_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_mul_hi_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_mul_hi_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_mul_hi_i32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_i32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mul_hi_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mul_hi_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_mul_hi_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_mul_hi_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_hi_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_mul_hi_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_mul_hi_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_mul_hi_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_mul_hi_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_mul_hi_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_mul_hi_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_mul_hi_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_mul_hi_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_hi_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v255.l, 0xfe0b, vcc_hi     ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v255.l, 0xfe0b, vcc_hi     ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_mul_lo_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_mul_lo_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_mul_lo_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_mul_lo_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_mul_lo_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_lo_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mul_lo_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_mul_lo_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_mul_lo_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_mul_lo_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_mul_lo_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_mul_lo_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_mul_lo_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_mul_lo_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_mul_lo_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_mul_lo_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mullit_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_mullit_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_mullit_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_mullit_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_mullit_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_mullit_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_mullit_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_mullit_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_mullit_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_mullit_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_mullit_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_mullit_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mullit_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX11: v_mullit_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX11: v_mullit_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_mullit_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_mullit_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_mullit_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_mullit_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX11: v_mullit_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX11: v_mullit_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX11: v_mullit_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_mullit_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX11: v_mullit_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX11: v_mullit_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX11: v_mullit_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX11: v_mullit_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33
+# GFX11: v_mullit_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX11: v_mullit_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX11: v_mullit_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_or3_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_or3_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_or3_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_or3_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_or3_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_or3_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_or3_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_or3_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_or3_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_or3_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_or3_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_or3_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_or3_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_or3_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_or3_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_or3_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_or3_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_or3_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_or3_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_or3_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_or3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_or3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_or3_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_or3_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_or3_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_or3_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_or3_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_or3_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_or3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_or3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_or_b16 v5.l, v1.l, v2.l               ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_or_b16 v5, v1, v2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_or_b16 v5.l, v1.l, v2.l               ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_or_b16 v5, v1, v2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_or_b16 v5.l, v255.l, v255.l           ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_or_b16 v5, v255, v255                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_or_b16 v5.l, v255.l, v255.l           ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_or_b16 v5, v255, v255                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, s1, s2                   ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, s1, s2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, s1, s2                   ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, s1, s2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, s105, s105               ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, s105, s105                 ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, s105, s105               ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, s105, s105                 ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, vcc_lo, ttmp15           ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, vcc_lo, ttmp15             ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, vcc_lo, ttmp15           ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, vcc_lo, ttmp15             ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, vcc_hi, 0xfe0b           ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, vcc_hi, 0xfe0b             ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, vcc_hi, 0xfe0b           ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, vcc_hi, 0xfe0b             ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, ttmp15, src_scc          ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_or_b16 v5, ttmp15, src_scc            ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_or_b16 v5.l, ttmp15, src_scc          ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_or_b16 v5, ttmp15, src_scc            ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, m0, 0x3800               ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, m0, 0x3800                 ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, m0, 0x3800               ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, m0, 0x3800                 ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, exec_lo, -1              ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_or_b16 v5, exec_lo, -1                ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_or_b16 v5.l, exec_lo, -1              ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_or_b16 v5, exec_lo, -1                ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, exec_hi, null            ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, exec_hi, null              ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, exec_hi, null            ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, exec_hi, null              ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, null, exec_lo            ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, null, exec_lo              ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, null, exec_lo            ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, null, exec_lo              ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, -1, exec_hi              ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, -1, exec_hi                ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, -1, exec_hi              ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, -1, exec_hi                ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, 0x3800, m0               ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, 0x3800, m0                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, 0x3800, m0               ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, 0x3800, m0                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, src_scc, vcc_lo          ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, src_scc, vcc_lo            ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, src_scc, vcc_lo          ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, src_scc, vcc_lo            ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v255.l, 0xfe0b, vcc_hi         ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v255, 0xfe0b, vcc_hi           ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v255.l, 0xfe0b, vcc_hi         ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v255, 0xfe0b, vcc_hi           ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_pack_b32_f16 v5, v1, v2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_pack_b32_f16 v5, v1, v2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_pack_b32_f16 v5, v255, v255           ; encoding: [0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_pack_b32_f16 v5, v255, v255           ; encoding: [0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_pack_b32_f16 v5, s1, s2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, s1, s2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, s105, s105           ; encoding: [0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, s105, s105           ; encoding: [0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_pack_b32_f16 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_pack_b32_f16 v5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_pack_b32_f16 v5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_pack_b32_f16 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_pack_b32_f16 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_pack_b32_f16 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, null, exec_lo        ; encoding: [0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, null, exec_lo        ; encoding: [0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_pack_b32_f16 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_pack_b32_f16 v5, 0.5, -m0             ; encoding: [0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40
+# GFX11: v_pack_b32_f16 v5, 0.5, -m0             ; encoding: [0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX11: v_pack_b32_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20
+# GFX11: v_pack_b32_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX11: v_pack_b32_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX11: v_pack_b32_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_perm_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_perm_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_perm_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_perm_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_perm_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_perm_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_perm_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_perm_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_perm_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_perm_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_perm_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_perm_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_perm_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_perm_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_perm_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_perm_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_perm_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_perm_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_perm_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_perm_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_perm_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_perm_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_perm_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_perm_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_perm_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_perm_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_perm_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_perm_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_perm_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_perm_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_permlane16_b32 v5, v1, s2, s3         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlane16_b32 v5, v1, s2, s3         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlane16_b32 v5, v1, s105, s105     ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01
+# GFX11: v_permlane16_b32 v5, v1, s105, s105     ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01
+# GFX11: v_permlane16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01
+# GFX11: v_permlane16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01
+# GFX11: v_permlane16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, m0, m0         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01
+# GFX11: v_permlane16_b32 v5, v1, m0, m0         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01
+# GFX11: v_permlane16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01
+# GFX11: v_permlane16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01]
 
-# GFX11: v_permlane16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_permlane16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_permlane16_b32 v5, v1, -1, src_scc    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03
+# GFX11: v_permlane16_b32 v5, v1, -1, src_scc    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03]
 
-# GFX11: v_permlane16_b32 v5, v1, 0.5, 0.5       ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03
+# GFX11: v_permlane16_b32 v5, v1, 0.5, 0.5       ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03]
 
-# GFX11: v_permlane16_b32 v5, v1, src_scc, -1    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03
+# GFX11: v_permlane16_b32 v5, v1, src_scc, -1    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03]
 
-# GFX11: v_permlane16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_permlane16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_permlanex16_b32 v5, v1, s2, s3        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlanex16_b32 v5, v1, s2, s3        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlane16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlanex16_b32 v5, v1, s105, s105    ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01
+# GFX11: v_permlanex16_b32 v5, v1, s105, s105    ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01
+# GFX11: v_permlanex16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01
+# GFX11: v_permlanex16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01
+# GFX11: v_permlanex16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, m0, m0        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01
+# GFX11: v_permlanex16_b32 v5, v1, m0, m0        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01
+# GFX11: v_permlanex16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01
+# GFX11: v_permlanex16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01]
 
-# GFX11: v_permlanex16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_permlanex16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_permlanex16_b32 v5, v1, -1, src_scc   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03
+# GFX11: v_permlanex16_b32 v5, v1, -1, src_scc   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03]
 
-# GFX11: v_permlanex16_b32 v5, v1, 0.5, 0.5      ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03
+# GFX11: v_permlanex16_b32 v5, v1, 0.5, 0.5      ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03]
 
-# GFX11: v_permlanex16_b32 v5, v1, src_scc, -1   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03
+# GFX11: v_permlanex16_b32 v5, v1, src_scc, -1   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03]
 
-# GFX11: v_permlanex16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_permlanex16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX11: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00]
 0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00
+# GFX11: v_qsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04]
 0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04
+# GFX11: v_qsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01]
 0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07]
 0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07
+# GFX11: v_qsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01]
 0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01]
 0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01]
 0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01
+# GFX11: v_qsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_qsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03]
 0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03
+# GFX11: v_qsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03]
 
-# GFX11: v_qsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03]
 0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03
+# GFX11: v_qsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03]
 
-# GFX11: v_qsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_qsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_readlane_b32 s5, v1, s2               ; encoding: [0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00
+# GFX11: v_readlane_b32 s5, v1, s2               ; encoding: [0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX11: v_readlane_b32 s5, v1, s105             ; encoding: [0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00
+# GFX11: v_readlane_b32 s5, v1, s105             ; encoding: [0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX11: v_readlane_b32 s105, v1, ttmp15         ; encoding: [0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00]
 0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00
+# GFX11: v_readlane_b32 s105, v1, ttmp15         ; encoding: [0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00]
 
-# GFX11: v_readlane_b32 vcc_lo, v1, vcc_hi       ; encoding: [0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00]
 0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00
+# GFX11: v_readlane_b32 vcc_lo, v1, vcc_hi       ; encoding: [0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00]
 
-# GFX11: v_readlane_b32 vcc_hi, v1, vcc_lo       ; encoding: [0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00]
 0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00
+# GFX11: v_readlane_b32 vcc_hi, v1, vcc_lo       ; encoding: [0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX11: v_readlane_b32 ttmp15, v1, m0           ; encoding: [0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00]
 0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00
+# GFX11: v_readlane_b32 ttmp15, v1, m0           ; encoding: [0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00]
 
-# GFX11: v_readlane_b32 null, v255, null         ; encoding: [0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00]
 0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00
+# GFX11: v_readlane_b32 null, v255, null         ; encoding: [0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00]
 
-# GFX11: v_sad_hi_u8 v5, v1, v2, s3              ; encoding: [0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_sad_hi_u8 v5, v1, v2, s3              ; encoding: [0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_sad_hi_u8 v5, v255, s2, s105          ; encoding: [0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_sad_hi_u8 v5, v255, s2, s105          ; encoding: [0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_sad_hi_u8 v5, s1, v255, exec_hi       ; encoding: [0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_sad_hi_u8 v5, s1, v255, exec_hi       ; encoding: [0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_sad_hi_u8 v5, s105, s105, exec_lo     ; encoding: [0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_sad_hi_u8 v5, s105, s105, exec_lo     ; encoding: [0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_sad_hi_u8 v5, vcc_lo, ttmp15, v3      ; encoding: [0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_sad_hi_u8 v5, vcc_lo, ttmp15, v3      ; encoding: [0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_sad_hi_u8 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_hi_u8 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_hi_u8 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_sad_hi_u8 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_sad_hi_u8 v5, m0, 0.5, m0             ; encoding: [0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_sad_hi_u8 v5, m0, 0.5, m0             ; encoding: [0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_sad_hi_u8 v5, exec_lo, -1, vcc_hi     ; encoding: [0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_sad_hi_u8 v5, exec_lo, -1, vcc_hi     ; encoding: [0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_sad_hi_u8 v5, exec_hi, null, vcc_lo   ; encoding: [0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_sad_hi_u8 v5, exec_hi, null, vcc_lo   ; encoding: [0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_sad_hi_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_hi_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_hi_u8 v5, -1, exec_hi, src_scc    ; encoding: [0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_sad_hi_u8 v5, -1, exec_hi, src_scc    ; encoding: [0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_sad_hi_u8 v5, 0.5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_sad_hi_u8 v5, 0.5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_sad_hi_u8 v5, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_sad_hi_u8 v5, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_sad_hi_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_hi_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_sad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_sad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_sad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_sad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_sad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_sad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_sad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_sad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_sad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_sad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX11: v_sad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_sad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_sad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_sad_u16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_sad_u16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_sad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_sad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_sad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_sad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_sad_u16 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u16 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u16 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_sad_u16 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_sad_u16 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_sad_u16 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_sad_u16 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_sad_u16 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_sad_u16 v255, 0xfe0b, vcc_hi, null clamp ; encoding: [0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX11: v_sad_u16 v255, 0xfe0b, vcc_hi, null clamp ; encoding: [0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX11: v_sad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_sad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_sad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_sad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_sad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_sad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_sad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_sad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_sad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_sad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_sad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_sad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_sad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_sad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_sad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_sad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_sad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_sad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_sad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_sad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_sad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_sad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_sad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_sad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_sad_u32 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u32 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u8 v5, v1, v2, s3                 ; encoding: [0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_sad_u8 v5, v1, v2, s3                 ; encoding: [0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_sad_u8 v5, v255, s2, s105             ; encoding: [0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_sad_u8 v5, v255, s2, s105             ; encoding: [0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_sad_u8 v5, s1, v255, exec_hi          ; encoding: [0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_sad_u8 v5, s1, v255, exec_hi          ; encoding: [0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_sad_u8 v5, s105, s105, exec_lo        ; encoding: [0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_sad_u8 v5, s105, s105, exec_lo        ; encoding: [0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_sad_u8 v5, vcc_lo, ttmp15, v3         ; encoding: [0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_sad_u8 v5, vcc_lo, ttmp15, v3         ; encoding: [0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_sad_u8 v5, vcc_hi, 0xaf123456, v255   ; encoding: [0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u8 v5, vcc_hi, 0xaf123456, v255   ; encoding: [0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u8 v5, ttmp15, src_scc, ttmp15    ; encoding: [0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_sad_u8 v5, ttmp15, src_scc, ttmp15    ; encoding: [0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_sad_u8 v5, m0, 0.5, m0                ; encoding: [0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_sad_u8 v5, m0, 0.5, m0                ; encoding: [0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_sad_u8 v5, exec_lo, -1, vcc_hi        ; encoding: [0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_sad_u8 v5, exec_lo, -1, vcc_hi        ; encoding: [0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_sad_u8 v5, exec_hi, null, vcc_lo      ; encoding: [0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_sad_u8 v5, exec_hi, null, vcc_lo      ; encoding: [0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_sad_u8 v5, null, exec_lo, 0xaf123456  ; encoding: [0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u8 v5, null, exec_lo, 0xaf123456  ; encoding: [0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sad_u8 v5, -1, exec_hi, src_scc       ; encoding: [0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_sad_u8 v5, -1, exec_hi, src_scc       ; encoding: [0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_sad_u8 v5, 0.5, m0, 0.5               ; encoding: [0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_sad_u8 v5, 0.5, m0, 0.5               ; encoding: [0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_sad_u8 v5, src_scc, vcc_lo, -1        ; encoding: [0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_sad_u8 v5, src_scc, vcc_lo, -1        ; encoding: [0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_sad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_sad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# W32: v_sub_co_u32 v5, s12, v1, v2              ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], v1, v2         ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00
+# W32: v_sub_co_u32 v5, s12, v1, v2            ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], v1, v2       ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_sub_co_u32 v5, s12, v255, v255          ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], v255, v255     ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00
+# W32: v_sub_co_u32 v5, s12, v255, v255        ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], v255, v255   ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_sub_co_u32 v5, s12, s1, s2              ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], s1, s2         ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, s1, s2            ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], s1, s2       ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, s105, s105          ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], s105, s105     ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, s105, s105        ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], s105, s105   ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, vcc_lo, ttmp15      ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, vcc_lo, ttmp15    ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, vcc_hi, 0xaf123456  ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-# W64: v_sub_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# W32: v_sub_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+# W64: v_sub_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_sub_co_u32 v5, s12, ttmp15, src_scc     ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, ttmp15, src_scc   ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, m0, 0.5             ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], m0, 0.5        ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, m0, 0.5           ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], m0, 0.5      ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, exec_lo, -1         ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], exec_lo, -1    ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, exec_lo, -1       ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], exec_lo, -1  ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, exec_hi, null       ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], exec_hi, null  ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, exec_hi, null     ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, null, exec_lo       ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], null, exec_lo  ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, null, exec_lo     ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s104, -1, exec_hi        ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[104:105], -1, exec_hi  ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_sub_co_u32 v5, s104, -1, exec_hi      ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, vcc_lo, 0.5, m0          ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_sub_co_u32 v5, vcc, 0.5, m0             ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_sub_co_u32 v5, vcc_lo, 0.5, m0        ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_sub_co_u32 v5, vcc, 0.5, m0           ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, ttmp14, src_scc, vcc_lo  ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
-# W64: v_sub_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00
+# W32: v_sub_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
+# W64: v_sub_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_sub_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_sub_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX11: v_sub_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_sub_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_sub_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00
+# GFX11: v_sub_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX11: v_sub_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_sub_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_sub_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00
+# GFX11: v_sub_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX11: v_sub_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00
+# GFX11: v_sub_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX11: v_sub_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00
+# GFX11: v_sub_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX11: v_sub_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_sub_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_sub_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_sub_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x04,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x04,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# W32: v_subrev_co_u32 v5, s12, v1, v2           ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], v1, v2      ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00
+# W32: v_subrev_co_u32 v5, s12, v1, v2         ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], v1, v2    ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, v255, v255       ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], v255, v255  ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00
+# W32: v_subrev_co_u32 v5, s12, v255, v255     ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], v255, v255 ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, s1, s2           ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], s1, s2      ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, s1, s2         ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], s1, s2    ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, s105, s105       ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], s105, s105  ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, s105, s105     ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], s105, s105 ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, vcc_lo, ttmp15   ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
 
+0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
 # W32: v_subrev_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 # W64: v_subrev_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
 
-# W32: v_subrev_co_u32 v5, s12, ttmp15, src_scc  ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, m0, 0.5          ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], m0, 0.5     ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, m0, 0.5        ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], m0, 0.5   ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, exec_lo, -1      ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], exec_lo, -1 ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, exec_lo, -1    ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], exec_lo, -1 ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, exec_hi, null    ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, exec_hi, null  ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, null, exec_lo    ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, null, exec_lo  ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s104, -1, exec_hi     ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_subrev_co_u32 v5, s104, -1, exec_hi   ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, vcc_lo, 0.5, m0       ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_subrev_co_u32 v5, vcc, 0.5, m0          ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_subrev_co_u32 v5, vcc_lo, 0.5, m0     ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_subrev_co_u32 v5, vcc, 0.5, m0        ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
 
+0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00
 # W32: v_subrev_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00]
 # W64: v_subrev_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00
 
-# GFX11: v_subrev_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_subrev_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_trig_preop_f64 v[5:6], v[1:2], v2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00
+# GFX11: v_trig_preop_f64 v[5:6], v[1:2], v2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], v[1:2], v255   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00
+# GFX11: v_trig_preop_f64 v[5:6], v[1:2], v255   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], v[1:2], s2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], v[1:2], s2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], v[1:2], s105   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], v[1:2], s105   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], v[254:255], ttmp15 ; encoding: [0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], v[254:255], ttmp15 ; encoding: [0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], s[2:3], vcc_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], s[2:3], vcc_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], s[104:105], vcc_lo ; encoding: [0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], s[104:105], vcc_lo ; encoding: [0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], vcc, m0        ; encoding: [0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], vcc, m0        ; encoding: [0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], exec, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], exec, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], null, null     ; encoding: [0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00
+# GFX11: v_trig_preop_f64 v[5:6], null, null     ; encoding: [0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], -1, -1         ; encoding: [0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00
+# GFX11: v_trig_preop_f64 v[5:6], -1, -1         ; encoding: [0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX11: v_trig_preop_f64 v[5:6], 0.5, 0.5 mul:2 ; encoding: [0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08]
 0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08
+# GFX11: v_trig_preop_f64 v[5:6], 0.5, 0.5 mul:2 ; encoding: [0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08]
 
-# GFX11: v_trig_preop_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30]
 0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30
+# GFX11: v_trig_preop_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30]
 
-# GFX11: v_trig_preop_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf
+# GFX11: v_trig_preop_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_writelane_b32 v5, s1, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, s1, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, s105, s2            ; encoding: [0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, s105, s2            ; encoding: [0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, vcc_lo, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, vcc_lo, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, vcc_hi, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, vcc_hi, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, ttmp15, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, ttmp15, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, m0, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, m0, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, exec_lo, s2         ; encoding: [0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00
+# GFX11: v_writelane_b32 v5, exec_lo, s2         ; encoding: [0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, exec_hi, s105       ; encoding: [0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00
+# GFX11: v_writelane_b32 v5, exec_hi, s105       ; encoding: [0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, null, ttmp15        ; encoding: [0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00
+# GFX11: v_writelane_b32 v5, null, ttmp15        ; encoding: [0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, -1, null            ; encoding: [0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00
+# GFX11: v_writelane_b32 v5, -1, null            ; encoding: [0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, 0.5, m0             ; encoding: [0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00
+# GFX11: v_writelane_b32 v5, 0.5, m0             ; encoding: [0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX11: v_writelane_b32 v5, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00
+# GFX11: v_writelane_b32 v5, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX11: v_writelane_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX11: v_writelane_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_xad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_xad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_xad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_xad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_xad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_xad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_xad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_xad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_xad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_xad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_xad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_xad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_xad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_xad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_xad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_xad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_xad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_xad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_xad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_xad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_xad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_xad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_xad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_xad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_xad_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_xad_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xor3_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00
+# GFX11: v_xor3_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX11: v_xor3_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01
+# GFX11: v_xor3_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX11: v_xor3_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01
+# GFX11: v_xor3_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX11: v_xor3_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01
+# GFX11: v_xor3_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX11: v_xor3_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX11: v_xor3_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX11: v_xor3_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX11: v_xor3_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xor3_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01
+# GFX11: v_xor3_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX11: v_xor3_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX11: v_xor3_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX11: v_xor3_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01
+# GFX11: v_xor3_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX11: v_xor3_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX11: v_xor3_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX11: v_xor3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX11: v_xor3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX11: v_xor3_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX11: v_xor3_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX11: v_xor3_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX11: v_xor3_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX11: v_xor3_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03
+# GFX11: v_xor3_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX11: v_xor3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX11: v_xor3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_xor_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_xor_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_xor_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_xor_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_xor_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_xor_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_xor_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_xor_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_xor_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_xor_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_xor_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_xor_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_xor_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_xor_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
index d0bd6398ad10a..35ad673de75db 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
@@ -1,5039 +1,5206 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-FAKE16 %s
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_add3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_add_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_add_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_add_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_add_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX11: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_and_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_and_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_and_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_and_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_bcnt_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_bfe_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_bfe_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_bfe_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_bfe_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_bfi_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_bfi_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_bfm_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_bfm_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s104 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp14 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13
 
-# GFX11: v_cndmask_b16_e64_dpp v5, -v1, |v2|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 ; encoding: [0x05,0x02,0x5d,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x6f,0x09,0x30]
 0x05,0x02,0x5d,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x6f,0x09,0x30
+# GFX11: v_cndmask_b16_e64_dpp v5, -v1, |v2|, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 ; encoding: [0x05,0x02,0x5d,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x6f,0x09,0x30]
 
-# GFX11: v_cndmask_b16_e64_dpp v5, |v1|, -v2, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 ; encoding: [0x05,0x01,0x5d,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x6f,0x09,0x30]
 0x05,0x01,0x5d,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x6f,0x09,0x30
+# GFX11: v_cndmask_b16_e64_dpp v5, |v1|, -v2, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 ; encoding: [0x05,0x01,0x5d,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x6f,0x09,0x30]
 
-# GFX11: v_cndmask_b16_e64_dpp v255, v255, v255, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30
+# GFX11: v_cndmask_b16_e64_dpp v255, v255, v255, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30]
 0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13]
 
-# GFX11: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30]
 0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30
+# GFX11: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_lerp_u8_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_lerp_u8_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x1c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x1c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x1c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x1c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x1c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x1c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x1c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x1c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x1c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x1c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x1c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_max3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x1c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x1c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x1c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x1c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x1c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x1c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x1c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x1c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x1c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x1c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x60,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x60,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x60,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x60,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x60,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x60,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x60,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x60,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x60,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x60,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x60,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x60,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x60,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x60,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x60,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x60,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x60,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x60,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x60,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x60,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_maxmin_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x60,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x60,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_maxmin_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x60,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x5e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x5e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x5e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x5e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x5e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x5e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x5e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x5e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x5e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x5e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x5e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x5e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x5e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x5e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x5e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x5e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x5e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x5e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x5e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_maxmin_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x5e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x5e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_maxmin_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x5e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x1f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x1f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x1f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x1f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x1f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x1f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x1f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x1f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x1f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x1f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x1f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_med3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x1f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x1f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x1f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x1f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x1f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x1f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x1f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x1f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x1f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x1f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x19,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x19,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x19,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x19,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x19,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x19,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x19,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x19,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x19,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x19,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x19,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x19,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x19,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_min3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x19,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x19,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x19,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x19,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x19,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x19,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x19,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x19,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x19,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x19,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x61,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x61,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x61,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x61,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x61,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x61,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x61,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x61,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x61,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x61,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x61,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x61,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x61,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x61,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x61,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x61,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x61,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x61,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x61,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x61,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_minmax_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x61,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x61,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_minmax_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x61,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x5f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x5f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x5f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x5f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x5f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x5f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x5f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x5f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x5f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x5f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x5f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x5f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x5f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x5f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x5f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x5f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x5f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x5f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x5f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_minmax_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x5f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x5f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_minmax_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x5f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_minmax_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_minmax_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_minmax_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_minmax_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX11: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_or3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_or3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_or_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_or_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_perm_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_perm_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_sub_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_sub_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_sub_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_sub_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX11: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX11: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_subrev_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_subrev_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_subrev_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_subrev_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX11: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX11: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_xad_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_xad_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_xor3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_xor3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX11: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX11: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX11: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x4c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x4c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x4c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x4c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x4c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x4c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x4c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x4c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x4c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x4c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x4c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_max3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x4c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x4c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x4c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x4c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x4c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x4c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x4c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x4c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x4c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x4c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x4f,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x4f,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x4f,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x4f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x4f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x4f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x4f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x4f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x4f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x4f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x4f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_med3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x4f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x4f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x4f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x4f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x4f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x4f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x4f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x4f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x4f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x4f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x49,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x49,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x49,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x49,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x49,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x49,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x49,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x49,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x49,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x49,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x49,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x49,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x49,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX11: v_min3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x49,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x49,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x49,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x49,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x49,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x49,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x49,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x49,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x49,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x49,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX11: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX11: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX11: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX11: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-
-# op_sel[1:0] are ignored
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x78,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
-0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
-
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-
-# op_sel[1:0] are ignored
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x78,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
 
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, -|m0| row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, -|m0| row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x66,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, v2, -|exec_hi| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, v2, -|exec_hi| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x66,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+
+0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, -|v2|, -|exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, -|v2|, -|exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x66,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
+
+0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, |v1|, -v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x66,0xd6,0xfa,0x04,0xf2,0x41,0x01,0x50,0x01,0xff]
+
+0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -v1, |v2|, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x66,0xd6,0xfa,0x04,0x06,0x23,0x01,0x5f,0x01,0x01]
+
+0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x66,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
+
+0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+
+0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
+
+0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, |v1|, v2, -vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, |v1|, v2, -vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x67,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -v2, |m0| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -v2, |m0| row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x67,0xd6,0xfa,0x04,0xf6,0x41,0x01,0x21,0x01,0xff]
+
+0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, v2, -|exec_hi| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, v2, -|exec_hi| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x05,0x67,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x2f,0x01,0xff]
+
+0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -|v2|, -|exec_lo| row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x67,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x50,0x01,0xff]
+
+0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -v1, |v2|, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x02,0x67,0xd6,0xfa,0x04,0xf2,0x21,0x01,0x5f,0x01,0x01]
+
+0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x01,0x13
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x01,0x13]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x01,0x13]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x01,0x13]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x03,0x67,0xd6,0xfa,0x04,0x06,0x63,0x01,0x60,0x01,0x13]
+
+0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+
+0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
-0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
index cbf5a3d11e50b..3a37a19d6d3af 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
@@ -1,3053 +1,3214 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64,W64-FAKE16 %s
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_add3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_add3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX11: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_add_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_and_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_and_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_and_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_and_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_bcnt_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bcnt_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_bcnt_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_bfe_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfe_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_bfe_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfi_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_bfi_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_bfm_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_bfm_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_bfm_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s104 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp14 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05
 
-# GFX11: v_cndmask_b16_e64_dpp v5, -v1, |v2|, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x02,0x5d,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x00,0x00,0x00]
 0x05,0x02,0x5d,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x00,0x00,0x00
+# GFX11: v_cndmask_b16_e64_dpp v5, -v1, |v2|, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x02,0x5d,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x00,0x00,0x00]
 
-# GFX11: v_cndmask_b16_e64_dpp v5, |v1|, -v2, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x01,0x5d,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x00,0x00,0x00]
 0x05,0x01,0x5d,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x00,0x00,0x00
+# GFX11: v_cndmask_b16_e64_dpp v5, |v1|, -v2, null dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x01,0x5d,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x00,0x00,0x00]
 
-# GFX11: v_cndmask_b16_e64_dpp v255, v255, v255, null dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00
+# GFX11: v_cndmask_b16_e64_dpp v255, v255, v255, null dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00]
 0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
 
-# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05
+# GFX11: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
 
-# GFX11: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
 0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00
+# GFX11: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lerp_u8_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_lerp_u8_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x1c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x1c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x1c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x1c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x1c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x1c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x1c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x1c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x1c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x1c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x1c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x1c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x1c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x1c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x1c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x1c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x1c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x1c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x1c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x1c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_max3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x1c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_max3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_max3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x60,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x60,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x60,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x60,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x60,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x60,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x60,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x60,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x60,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x60,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x60,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x60,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x60,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x60,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x60,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x60,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x60,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x60,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x60,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x60,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x60,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x60,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x60,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x60,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x60,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_maxmin_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x60,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x5e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x5e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x5e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x5e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x5e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x5e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x5e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x5e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x5e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x5e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x5e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x5e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x5e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x5e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x5e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x5e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x5e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x5e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x5e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x5e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_maxmin_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x5e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x1f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x1f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x1f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x1f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x1f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x1f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x1f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x1f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x1f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x1f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x1f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x1f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x1f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x1f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x1f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x1f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x1f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x1f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x1f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x1f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_med3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x1f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_med3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_med3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x19,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x19,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x19,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x19,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x19,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x19,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x19,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x19,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x19,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x19,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x19,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x19,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x19,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x19,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x19,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x19,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x19,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x19,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x19,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x19,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x19,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x19,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x19,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x19,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x19,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x19,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_min3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x19,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_min3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_min3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x61,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x61,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x61,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x61,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x61,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x61,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x61,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x61,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x61,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x61,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x61,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x61,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x61,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x61,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x61,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x61,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x61,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x61,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x61,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x61,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x61,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x61,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x61,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x61,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x61,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_minmax_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x61,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x5f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x5f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x5f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x5f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x5f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x5f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x5f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x5f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x5f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x5f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x5f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x5f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x5f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x5f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x5f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x5f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x5f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x5f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x5f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x5f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_minmax_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x5f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_minmax_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_minmax_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_minmax_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_msad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX11: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX11: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_or3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_or3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_or3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_or_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_or_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_perm_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_perm_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_perm_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_sad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX11: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_sub_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX11: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX11: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_xad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xad_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_xad_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_xor3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_xor3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX11: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX11: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX11: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX11: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x4c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x4c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x4c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x4c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x4c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x4c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x4c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x4c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x4c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x4c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x4c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x4c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x4c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x4c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x4c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x4c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x4c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX11: v_max3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x4c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x4c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x4c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX11: v_max3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x4c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x4f,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x4f,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x4f,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x4f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x4f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x4f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x4f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x4f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x4f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x4f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x4f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x4f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x4f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x4f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x4f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x4f,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x4f,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX11: v_med3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x4f,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x4f,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x4f,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX11: v_med3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x4f,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x49,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x49,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x49,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x49,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x49,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x49,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x49,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x49,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x49,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x49,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x49,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x49,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x49,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x49,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x49,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x49,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x49,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x49,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x49,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX11: v_min3_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x49,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x49,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x49,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX11: v_min3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x49,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX11: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX11: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX11: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX11: v_pack_b32_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX11: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX11: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX11: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX11: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-
-# op_sel[1:0] are ignored
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x78,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-
-# GFX11: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
-0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
-
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-
-# op_sel[1:0] are ignored
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x78,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
 
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, -|m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x66,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
+
+0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x66,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+
+0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x66,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+
+0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, |v1|, -v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd6,0xe9,0x04,0xf2,0x41,0x01,0x77,0x39,0x05]
+
+0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -v1, |v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x66,0xd6,0xe9,0x04,0x06,0x23,0x01,0x77,0x39,0x05]
+
+0x05,0x03,0x66,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x66,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x66,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x66,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x66,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, |v1|, v2, -vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd6,0xe9,0x04,0xaa,0x81,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -v2, |m0| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x67,0xd6,0xe9,0x04,0xf6,0x41,0x01,0x77,0x39,0x05]
+
+0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, v2, -|exec_hi| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x67,0xd6,0xe9,0x04,0xfe,0xa1,0x01,0x77,0x39,0x05]
+
+0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, -|v2|, -|exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x67,0xd6,0xe9,0x04,0xfa,0xc1,0x01,0x77,0x39,0x05]
+
+0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -v1, |v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x67,0xd6,0xe9,0x04,0xf2,0x21,0x01,0x77,0x39,0x05]
+
+0x05,0x03,0x67,0xd6,0xe9,0x04,0x06,0x63,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x67,0xd6,0xe9,0x04,0x06,0x63,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x67,0xd6,0xe9,0x04,0x06,0x63,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x67,0xd6,0xe9,0x04,0x06,0x63,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x67,0xd6,0xe9,0x04,0x06,0x63,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX11: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
-0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopd.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopd.txt
index 222718b70f0d7..1bdb7136de95f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopd.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vopd.txt
@@ -11111,3 +11111,111 @@
 
 # GFX11: v_dual_subrev_f32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX11: v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_bf16 v6, v1, v3 ; encoding: [0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_bf16 v6, v255, v3 ; encoding: [0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_bf16 v6, v2, v3 ; encoding: [0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_bf16 v6, v3, v3 ; encoding: [0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_bf16 v6, v4, v3 ; encoding: [0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_bf16 v6, s1, v3 ; encoding: [0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_bf16 v6, s105, v3 ; encoding: [0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_bf16 v6, m0, v3 ; encoding: [0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_bf16 v6, null, v3 ; encoding: [0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX11: v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_bf16 v6, -1, v3 ; encoding: [0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_bf16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX11: v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_bf16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX11: v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3 ; encoding: [0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX11: v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
index b120d6a030cc9..23a7cb1dd7b88 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
@@ -124,6 +124,24 @@
 # GFX12-REAL16: v_add_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
 # GFX12-FAKE16: v_add_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00]
 
+0x81,0x05,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x64]
+
+0x81,0x05,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x64]
+
+0xff,0x05,0x0a,0x64
+# GFX12-REAL16: v_add_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x64]
+# GFX12-FAKE16: v_add_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x64]
+
+0xfd,0x04,0x0b,0x65
+# GFX12-REAL16: v_add_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x65]
+
+0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_add_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00]
+
 0x01,0x05,0x0a,0x06
 # GFX12: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
 
@@ -2032,6 +2050,20 @@
 # GFX12-REAL16: v_sub_f16_e32 v127.l, 0xfe0b, v127.l    ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 # GFX12-FAKE16: v_sub_f16_e32 v127, 0xfe0b, v127        ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00]
 
+0x81,0x05,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, v1.h, v2.l          ; encoding: [0x81,0x05,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x66]
+
+0xff,0x05,0x0a,0x66
+# GFX12-REAL16: v_sub_f16_e32 v5.l, v127.h, v2.l        ; encoding: [0xff,0x05,0x0a,0x66]
+# GFX12-FAKE16: v_sub_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x66]
+
+0xfd,0x04,0x0b,0x67
+# GFX12-REAL16: v_sub_f16_e32 v5.h, src_scc, v2.h       ; encoding: [0xfd,0x04,0x0b,0x67]
+
+0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00
+# GFX12-REAL16: v_sub_f16_e32 v127.h, 0xfe0b, v127.h    ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00]
+
 0x01,0x05,0x0a,0x08
 # GFX12: v_sub_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x08]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
index 9a9a1eb2cf959..c0eacd7c593fa 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
@@ -116,6 +116,14 @@
 # GFX12-REAL16: v_add_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 # GFX12-FAKE16: v_add_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x64,0x7f,0x6f,0xfd,0x30]
 
+0xfa,0x04,0x0b,0x65,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_add_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x65,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_mul_i32_i24_e32 v128, 1, v176         ; encoding: [0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0xff,0x65,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_add_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x65,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_lshlrev_b32_e32 v126, v255, v183      ; encoding: [0xff,0x6f,0xfd,0x30]
+
 0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff
 # GFX12: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
 
@@ -1454,6 +1462,14 @@
 # GFX12-REAL16: v_sub_f16_dpp v127.l, -|v127.l|, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 # GFX12-FAKE16: v_sub_f16_dpp v127, -|v127|, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xfe,0x66,0x7f,0x6f,0xfd,0x30]
 
+0xfa,0x04,0x0b,0x67,0x81,0x60,0x01,0x13
+# GFX12-REAL16: v_sub_f16_dpp v5.h, v1.h, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0xfa,0x04,0x0b,0x67,0x81,0x60,0x01,0x13]
+# GFX12-FAKE16: v_mul_i32_i24_e32 v128, 1, v176         ; encoding: [0x81,0x60,0x01,0x13]
+
+0xfa,0xfe,0xff,0x67,0xff,0x6f,0xfd,0x30
+# GFX12-REAL16: v_sub_f16_dpp v127.h, -|v127.h|, -|v127.h| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xfa,0xfe,0xff,0x67,0xff,0x6f,0xfd,0x30]
+# GFX12-FAKE16: v_lshlrev_b32_e32 v126, v255, v183      ; encoding: [0xff,0x6f,0xfd,0x30]
+
 0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff
 # GFX12: v_sub_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x08,0x01,0x1b,0x00,0xff]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
index 0ef2c082f7dda..983b55db71052 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,GFX12-FAKE16 %s
@@ -20,6 +20,13 @@
 # GFX12-REAL16: v_add_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 # GFX12-FAKE16: v_add_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x64,0x7f,0x00,0x00,0x00]
 
+0xe9,0x04,0x0b,0x65,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_add_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x65,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0xff,0x65,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_add_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x65,0xff,0x00,0x00,0x00]
+
 0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05
 # GFX12: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
 
@@ -217,6 +224,13 @@
 # GFX12-REAL16: v_sub_f16_dpp v127.l, v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 # GFX12-FAKE16: v_sub_f16_dpp v127, v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xfe,0x66,0x7f,0x00,0x00,0x00]
 
+0xe9,0x04,0x0b,0x67,0x81,0x77,0x39,0x05
+# GFX12-REAL16: v_sub_f16_dpp v5.h, v1.h, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0b,0x67,0x81,0x77,0x39,0x05]
+# GFX12-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
+
+0xea,0xfe,0xff,0x67,0xff,0x00,0x00,0x00
+# GFX12-REAL16: v_sub_f16_dpp v127.h, v127.h, v127.h dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xea,0xfe,0xff,0x67,0xff,0x00,0x00,0x00]
+
 0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05
 # GFX12: v_sub_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x08,0x01,0x77,0x39,0x05]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
index c87c8855f5cdc..c3bab320b0ba2 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
@@ -1,7721 +1,7844 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16 %s
 
-# GFX12: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_add3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_add3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x55,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_add3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_add3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x55,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_add3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_add3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x55,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_add3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_add3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x55,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_add3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_add3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_add3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x55,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_add3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_add3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x55,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_add3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_add3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x55,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_add3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_add3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x55,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_add3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_add3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_add3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x55,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_add3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_add3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x55,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_add3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_add3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x55,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_add3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_add3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x55,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# W32: v_add_co_u32 v5, s12, v1, v2              ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_add_co_u32 v5, s[12:13], v1, v2         ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00
+# W32: v_add_co_u32 v5, s12, v1, v2            ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_add_co_u32 v5, s[12:13], v1, v2       ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_add_co_u32 v5, s12, v255, v255          ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_add_co_u32 v5, s[12:13], v255, v255     ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00
+# W32: v_add_co_u32 v5, s12, v255, v255        ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_add_co_u32 v5, s[12:13], v255, v255   ; encoding: [0x05,0x0c,0x00,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_add_co_u32 v5, s12, s1, s2              ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], s1, s2         ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00
+# W32: v_add_co_u32 v5, s12, s1, s2            ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], s1, s2       ; encoding: [0x05,0x0c,0x00,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, s105, s105          ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], s105, s105     ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_add_co_u32 v5, s12, s105, s105        ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], s105, s105   ; encoding: [0x05,0x0c,0x00,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, vcc_lo, ttmp15      ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_add_co_u32 v5, s12, vcc_lo, ttmp15    ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x00,0xd7,0x6a,0xf6,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, vcc_hi, 0xaf123456  ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-# W64: v_add_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# W32: v_add_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+# W64: v_add_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x00,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_add_co_u32 v5, s12, ttmp15, src_scc     ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_add_co_u32 v5, s12, ttmp15, src_scc   ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x00,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, m0, 0.5             ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], m0, 0.5        ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_add_co_u32 v5, s12, m0, 0.5           ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], m0, 0.5      ; encoding: [0x05,0x0c,0x00,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, exec_lo, -1         ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_add_co_u32 v5, s[12:13], exec_lo, -1    ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_add_co_u32 v5, s12, exec_lo, -1       ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_add_co_u32 v5, s[12:13], exec_lo, -1  ; encoding: [0x05,0x0c,0x00,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_add_co_u32 v5, s12, exec_hi, null       ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], exec_hi, null  ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_add_co_u32 v5, s12, exec_hi, null     ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x00,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s12, null, exec_lo       ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_add_co_u32 v5, s[12:13], null, exec_lo  ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_add_co_u32 v5, s12, null, exec_lo     ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_add_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x00,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_add_co_u32 v5, s104, -1, exec_hi        ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_add_co_u32 v5, s[104:105], -1, exec_hi  ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_add_co_u32 v5, s104, -1, exec_hi      ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_add_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x00,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_add_co_u32 v5, vcc_lo, 0.5, m0          ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_add_co_u32 v5, vcc, 0.5, m0             ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_add_co_u32 v5, vcc_lo, 0.5, m0        ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_add_co_u32 v5, vcc, 0.5, m0           ; encoding: [0x05,0x6a,0x00,0xd7,0xf0,0xfa,0x00,0x00]
 
-# W32: v_add_co_u32 v5, ttmp14, src_scc, vcc_lo  ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
-# W64: v_add_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00
+# W32: v_add_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
+# W64: v_add_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x00,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_add_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_add_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x00,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_lshl_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_add_lshl_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x47,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_add_lshl_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_add_lshl_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x47,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_add_lshl_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_add_lshl_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x47,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_add_lshl_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_add_lshl_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x47,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_add_lshl_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_add_lshl_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x47,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_add_lshl_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_add_lshl_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x47,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_lshl_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_add_lshl_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x47,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_add_lshl_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_add_lshl_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x47,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_add_lshl_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_add_lshl_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x47,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_add_lshl_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_add_lshl_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x47,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_add_lshl_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_add_lshl_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x47,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_lshl_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_add_lshl_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x47,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_add_lshl_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_add_lshl_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x47,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_add_lshl_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_add_lshl_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x47,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_add_lshl_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_add_lshl_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x47,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v1, v2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v1, v2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v255, v255             ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v255, v255             ; encoding: [0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0d,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0d,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x58,0x0d,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0d,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0d,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_add_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_add_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_add_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_add_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x26,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_add_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_add_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x26,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_add_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x26,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_add_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x26,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_add_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x26,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_add_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_add_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x26,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_add_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_add_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x26,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_add_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_add_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x26,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_add_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_add_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x26,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_add_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x26,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_add_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x26,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_add_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x26,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_add_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x26,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_add_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_add_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x26,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x03,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x03,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x03,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x58,0x03,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x03,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_add_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_add_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x03,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_add_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_alignbit_b32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_alignbit_b32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x16,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_alignbit_b32 v5, v255, s2, s3         ; encoding: [0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00]
 0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00
+# GFX12: v_alignbit_b32 v5, v255, s2, s3         ; encoding: [0x05,0x00,0x16,0xd6,0xff,0x05,0x0c,0x00]
 
-# GFX12: v_alignbit_b32 v5, s1, v255, s3         ; encoding: [0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00]
 0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00
+# GFX12: v_alignbit_b32 v5, s1, v255, s3         ; encoding: [0x05,0x00,0x16,0xd6,0x01,0xfe,0x0f,0x00]
 
-# GFX12: v_alignbit_b32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01
+# GFX12: v_alignbit_b32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX12: v_alignbit_b32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_alignbit_b32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_alignbit_b32 v5, m0, 0.5, exec_lo     ; encoding: [0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01]
 0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01
+# GFX12: v_alignbit_b32 v5, m0, 0.5, exec_lo     ; encoding: [0x05,0x00,0x16,0xd6,0x7d,0xe0,0xf9,0x01]
 
-# GFX12: v_alignbit_b32 v5, exec_lo, -1, m0      ; encoding: [0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01]
 0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01
+# GFX12: v_alignbit_b32 v5, exec_lo, -1, m0      ; encoding: [0x05,0x00,0x16,0xd6,0x7e,0x82,0xf5,0x01]
 
-# GFX12: v_alignbit_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01]
 0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01
+# GFX12: v_alignbit_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x16,0xd6,0x7f,0xf8,0xac,0x01]
 
-# GFX12: v_alignbit_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01]
 0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01
+# GFX12: v_alignbit_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x16,0xd6,0x7c,0xfc,0xa8,0x01]
 
-# GFX12: v_alignbit_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_alignbit_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x16,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_alignbit_b32 v5, 0.5, m0, exec_hi     ; encoding: [0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01]
 0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01
+# GFX12: v_alignbit_b32 v5, 0.5, m0, exec_hi     ; encoding: [0x05,0x00,0x16,0xd6,0xf0,0xfa,0xfc,0x01]
 
-# GFX12: v_alignbit_b32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_alignbit_b32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x16,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_alignbyte_b32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_alignbyte_b32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_alignbyte_b32 v5, v255, s2, s3        ; encoding: [0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00]
 0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00
+# GFX12: v_alignbyte_b32 v5, v255, s2, s3        ; encoding: [0x05,0x00,0x17,0xd6,0xff,0x05,0x0c,0x00]
 
-# GFX12: v_alignbyte_b32 v5, s1, v255, s3        ; encoding: [0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00]
 0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00
+# GFX12: v_alignbyte_b32 v5, s1, v255, s3        ; encoding: [0x05,0x00,0x17,0xd6,0x01,0xfe,0x0f,0x00]
 
-# GFX12: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01
+# GFX12: v_alignbyte_b32 v5, s105, s105, s105    ; encoding: [0x05,0x00,0x17,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX12: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_alignbyte_b32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x17,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_alignbyte_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x17,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_alignbyte_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x17,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_alignbyte_b32 v5, m0, 0.5, exec_lo    ; encoding: [0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01]
 0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01
+# GFX12: v_alignbyte_b32 v5, m0, 0.5, exec_lo    ; encoding: [0x05,0x00,0x17,0xd6,0x7d,0xe0,0xf9,0x01]
 
-# GFX12: v_alignbyte_b32 v5, exec_lo, -1, m0     ; encoding: [0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01]
 0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01
+# GFX12: v_alignbyte_b32 v5, exec_lo, -1, m0     ; encoding: [0x05,0x00,0x17,0xd6,0x7e,0x82,0xf5,0x01]
 
-# GFX12: v_alignbyte_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01]
 0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01
+# GFX12: v_alignbyte_b32 v5, exec_hi, null, vcc_hi ; encoding: [0x05,0x00,0x17,0xd6,0x7f,0xf8,0xac,0x01]
 
-# GFX12: v_alignbyte_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01]
 0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01
+# GFX12: v_alignbyte_b32 v5, null, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x17,0xd6,0x7c,0xfc,0xa8,0x01]
 
-# GFX12: v_alignbyte_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_alignbyte_b32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x17,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_alignbyte_b32 v5, 0.5, m0, exec_hi    ; encoding: [0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01]
 0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01
+# GFX12: v_alignbyte_b32 v5, 0.5, m0, exec_hi    ; encoding: [0x05,0x00,0x17,0xd6,0xf0,0xfa,0xfc,0x01]
 
-# GFX12: v_alignbyte_b32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_alignbyte_b32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x17,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_alignbyte_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x17,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_and_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_and_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x62,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_and_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_and_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_and_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_and_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x62,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_and_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_and_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_and_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_and_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_and_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_and_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_and_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x62,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_and_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_and_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_and_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_and_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_and_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_and_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_and_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_and_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x62,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_and_or_b32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_and_or_b32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x57,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_and_or_b32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_and_or_b32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x57,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_and_or_b32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_and_or_b32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x57,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_and_or_b32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_and_or_b32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x57,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_and_or_b32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_and_or_b32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x57,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_and_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_and_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x57,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_and_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_and_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x57,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_and_or_b32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_and_or_b32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x57,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_and_or_b32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_and_or_b32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x57,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_and_or_b32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_and_or_b32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x57,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_and_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_and_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x57,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_and_or_b32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_and_or_b32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x57,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_and_or_b32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_and_or_b32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x57,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_and_or_b32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_and_or_b32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x57,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_and_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_and_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x57,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, s1, s2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, s1, s2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, s105, s105            ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, s105, s105            ; encoding: [0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v1, v2                ; encoding: [0x05,0x00,0x3a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x3a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_ashrrev_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_ashrrev_i16 v5, v255, v255            ; encoding: [0x05,0x00,0x3a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x3a,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_ashrrev_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x3a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_ashrrev_i64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00
+# GFX12: v_ashrrev_i64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3e,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00
+# GFX12: v_ashrrev_i64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3e,0xd7,0xff,0xfd,0x00,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00]
 0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00
+# GFX12: v_ashrrev_i64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3e,0xd7,0x7e,0x04,0x02,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00]
 0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00
+# GFX12: v_ashrrev_i64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3e,0xd7,0x7f,0xfc,0x03,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00
+# GFX12: v_ashrrev_i64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3e,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00
+# GFX12: v_ashrrev_i64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3e,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX12: v_ashrrev_i64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_ashrrev_i64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3e,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ashrrev_i64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00]
 0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00
+# GFX12: v_ashrrev_i64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3e,0xd7,0xfd,0xfa,0x01,0x00]
 
-# GFX12: v_ashrrev_i64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_ashrrev_i64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3e,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bcnt_u32_b32 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_bcnt_u32_b32 v5, v1, v2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_bcnt_u32_b32 v5, v255, v255           ; encoding: [0x05,0x00,0x1e,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, s1, s2               ; encoding: [0x05,0x00,0x1e,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, s105, s105           ; encoding: [0x05,0x00,0x1e,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x1e,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_bcnt_u32_b32 v5, vcc_hi, 0xaf123456   ; encoding: [0x05,0x00,0x1e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bcnt_u32_b32 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_bcnt_u32_b32 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x1e,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_bcnt_u32_b32 v5, m0, 0.5              ; encoding: [0x05,0x00,0x1e,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_bcnt_u32_b32 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x1e,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, exec_hi, null        ; encoding: [0x05,0x00,0x1e,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, null, exec_lo        ; encoding: [0x05,0x00,0x1e,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x1e,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_bcnt_u32_b32 v5, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x1e,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_bcnt_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_bcnt_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_i32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_bfe_i32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x11,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_bfe_i32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_bfe_i32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x11,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_bfe_i32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_bfe_i32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x11,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_bfe_i32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_bfe_i32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x11,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_bfe_i32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_bfe_i32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x11,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_bfe_i32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_i32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x11,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_i32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_bfe_i32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x11,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_bfe_i32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_bfe_i32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x11,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_bfe_i32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_bfe_i32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x11,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_bfe_i32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_bfe_i32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x11,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_bfe_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x11,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_i32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_bfe_i32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x11,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_bfe_i32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_bfe_i32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_bfe_i32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_bfe_i32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x11,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_bfe_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x11,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_bfe_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x10,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_bfe_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_bfe_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x10,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_bfe_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_bfe_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x10,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_bfe_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_bfe_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x10,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_bfe_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_bfe_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x10,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_bfe_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x10,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_bfe_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x10,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_bfe_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_bfe_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x10,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_bfe_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_bfe_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x10,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_bfe_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_bfe_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x10,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_bfe_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x10,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfe_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_bfe_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x10,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_bfe_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_bfe_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x10,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_bfe_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_bfe_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x10,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_bfe_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_bfe_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x10,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfi_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_bfi_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x12,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_bfi_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_bfi_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x12,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_bfi_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_bfi_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x12,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_bfi_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_bfi_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x12,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_bfi_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_bfi_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x12,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_bfi_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_bfi_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x12,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfi_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_bfi_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x12,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_bfi_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_bfi_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x12,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_bfi_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_bfi_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x12,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_bfi_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_bfi_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x12,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_bfi_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_bfi_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x12,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfi_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_bfi_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x12,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_bfi_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_bfi_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x12,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_bfi_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_bfi_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x12,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_bfi_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_bfi_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x12,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfm_b32 v5, v1, v2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_bfm_b32 v5, v1, v2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_bfm_b32 v5, v255, v255                ; encoding: [0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_bfm_b32 v5, v255, v255                ; encoding: [0x05,0x00,0x1d,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_bfm_b32 v5, s1, s2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_bfm_b32 v5, s1, s2                    ; encoding: [0x05,0x00,0x1d,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, s105, s105                ; encoding: [0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_bfm_b32 v5, s105, s105                ; encoding: [0x05,0x00,0x1d,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_bfm_b32 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x1d,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, vcc_hi, 0xaf123456        ; encoding: [0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_bfm_b32 v5, vcc_hi, 0xaf123456        ; encoding: [0x05,0x00,0x1d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_bfm_b32 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_bfm_b32 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x1d,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_bfm_b32 v5, m0, 0.5                   ; encoding: [0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_bfm_b32 v5, m0, 0.5                   ; encoding: [0x05,0x00,0x1d,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_bfm_b32 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_bfm_b32 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x1d,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_bfm_b32 v5, exec_hi, null             ; encoding: [0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_bfm_b32 v5, exec_hi, null             ; encoding: [0x05,0x00,0x1d,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, null, exec_lo             ; encoding: [0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_bfm_b32 v5, null, exec_lo             ; encoding: [0x05,0x00,0x1d,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_bfm_b32 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x1d,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, 0.5, m0                   ; encoding: [0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_bfm_b32 v5, 0.5, m0                   ; encoding: [0x05,0x00,0x1d,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_bfm_b32 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_bfm_b32 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x1d,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_bfm_b32 v255, 0xaf123456, vcc_hi      ; encoding: [0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_bfm_b32 v255, 0xaf123456, vcc_hi      ; encoding: [0xff,0x00,0x1d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_cndmask_b16 v5, v1, src_scc, s6         ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
-# W64: v_cndmask_b16 v5, v1, src_scc, s[6:7]     ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
 0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00
+# W32: v_cndmask_b16 v5, v1, src_scc, s6       ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
+# W64: v_cndmask_b16 v5, v1, src_scc, s[6:7]   ; encoding: [0x05,0x00,0x5d,0xd6,0x01,0xfb,0x19,0x00]
 
-# W32: v_cndmask_b16 v5, v255, 0x3800, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
-# W64: v_cndmask_b16 v5, v255, 0x3800, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
 0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00
+# W32: v_cndmask_b16 v5, v255, 0x3800, s6      ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+# W64: v_cndmask_b16 v5, v255, 0x3800, s[6:7]  ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
 
-# W32: v_cndmask_b16 v5, s105, s105, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
-# W64: v_cndmask_b16 v5, s105, s105, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00
+# W32: v_cndmask_b16 v5, s105, s105, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
+# W64: v_cndmask_b16 v5, s105, s105, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0x69,0xd2,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, vcc_hi, v2, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
-# W64: v_cndmask_b16 v5, vcc_hi, v2, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
 0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00
+# W32: v_cndmask_b16 v5, vcc_hi, v2, s6        ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
+# W64: v_cndmask_b16 v5, vcc_hi, v2, s[6:7]    ; encoding: [0x05,0x00,0x5d,0xd6,0x6b,0x04,0x1a,0x00]
 
-# W32: v_cndmask_b16 v5, ttmp15, ttmp15, s6      ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
-# W64: v_cndmask_b16 v5, ttmp15, ttmp15, s[6:7]  ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00
+# W32: v_cndmask_b16 v5, ttmp15, ttmp15, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
+# W64: v_cndmask_b16 v5, ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7b,0xf6,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, m0, v255, s6            ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
-# W64: v_cndmask_b16 v5, m0, v255, s[6:7]        ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
 0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00
+# W32: v_cndmask_b16 v5, m0, v255, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
+# W64: v_cndmask_b16 v5, m0, v255, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x7d,0xfe,0x1b,0x00]
 
-# W32: v_cndmask_b16 v5, exec_lo, exec_lo, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
-# W64: v_cndmask_b16 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00
+# W32: v_cndmask_b16 v5, exec_lo, exec_lo, s6  ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
+# W64: v_cndmask_b16 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7e,0xfc,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, exec_hi, exec_hi, s6    ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
-# W64: v_cndmask_b16 v5, exec_hi, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00
+# W32: v_cndmask_b16 v5, exec_hi, exec_hi, s6  ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
+# W64: v_cndmask_b16 v5, exec_hi, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x5d,0xd6,0x7f,0xfe,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, null, m0, s6            ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
-# W64: v_cndmask_b16 v5, null, m0, s[6:7]        ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
 0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00
+# W32: v_cndmask_b16 v5, null, m0, s6          ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
+# W64: v_cndmask_b16 v5, null, m0, s[6:7]      ; encoding: [0x05,0x00,0x5d,0xd6,0x7c,0xfa,0x18,0x00]
 
-# W32: v_cndmask_b16 v5, -1, -|vcc_lo|, s104     ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
-# W64: v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105] ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
 0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41
+# W32: v_cndmask_b16 v5, -1, -|vcc_lo|, s104   ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
+# W64: v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105] ; encoding: [0x05,0x02,0x5d,0xd6,0xc1,0xd4,0xa0,0x41]
 
-# W32: v_cndmask_b16 v5, 0x3800, -1, vcc_lo      ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
-# W64: v_cndmask_b16 v5, 0x3800, -1, vcc         ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
 0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01
+# W32: v_cndmask_b16 v5, 0x3800, -1, vcc_lo    ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+# W64: v_cndmask_b16 v5, 0x3800, -1, vcc       ; encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
 
+0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21
 # W32: v_cndmask_b16 v5, -|src_scc|, null, ttmp14 ; encoding: [0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21]
 # W64: v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15] ; encoding: [0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21]
-0x05,0x01,0x5d,0xd6,0xfd,0xf8,0xe8,0x21
 
-# GFX12: v_cndmask_b16 v255, -|0xfe0b|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_cndmask_b16 v255, -|0xfe0b|, -|vcc_hi|, null ; encoding: [0xff,0x03,0x5d,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_cubeid_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_cubeid_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_cubeid_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_cubeid_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_cubeid_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_cubeid_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_cubeid_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_cubeid_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_cubeid_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_cubeid_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_cubeid_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_cubeid_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubeid_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_cubeid_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_cubeid_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_cubeid_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_cubeid_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_cubeid_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_cubeid_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_cubeid_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_cubeid_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_cubeid_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubeid_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_cubeid_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_cubeid_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_cubeid_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0c,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_cubeid_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_cubeid_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0c,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_cubeid_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_cubeid_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubema_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_cubema_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_cubema_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_cubema_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_cubema_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_cubema_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_cubema_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_cubema_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_cubema_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_cubema_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_cubema_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_cubema_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0f,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubema_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_cubema_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_cubema_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_cubema_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_cubema_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_cubema_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_cubema_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_cubema_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_cubema_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_cubema_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0f,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubema_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_cubema_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_cubema_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_cubema_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0f,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_cubema_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_cubema_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0f,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_cubema_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_cubema_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0f,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubesc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_cubesc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_cubesc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_cubesc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_cubesc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_cubesc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_cubesc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_cubesc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_cubesc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_cubesc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_cubesc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_cubesc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubesc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_cubesc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0d,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_cubesc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_cubesc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_cubesc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_cubesc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_cubesc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_cubesc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0d,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_cubesc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_cubesc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubesc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_cubesc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0d,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_cubesc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_cubesc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0d,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_cubesc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_cubesc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0d,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_cubesc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_cubesc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubetc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_cubetc_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_cubetc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_cubetc_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x0e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_cubetc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_cubetc_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x0e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_cubetc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_cubetc_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x0e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_cubetc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_cubetc_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x0e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_cubetc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_cubetc_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubetc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_cubetc_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x0e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_cubetc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_cubetc_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x0e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_cubetc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_cubetc_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x0e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_cubetc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_cubetc_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x0e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_cubetc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_cubetc_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x0e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cubetc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_cubetc_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x0e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_cubetc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_cubetc_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x0e,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_cubetc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_cubetc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x0e,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_fp8_f32 v1, v2, v3 ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_pk_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3| ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20
+# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
-# GFX12: v_cvt_pk_fp8_f32 v1, s2, 3 ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
 0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00
+# GFX12: v_cvt_pk_fp8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
 
-# GFX12: v_cvt_pk_bf8_f32 v1, v2, v3 ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_pk_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3| ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20
+# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 
-# GFX12: v_cvt_pk_bf8_f32 v1, s2, 3 ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
 0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00
+# GFX12: v_cvt_pk_bf8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 ; encoding: [0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32 v10, s2, v5 ; encoding: [0x0a,0x00,0x6b,0xd7,0x02,0x0a,0x02,0x00]
 0x0a,0x00,0x6b,0xd7,0x02,0x0a,0x02,0x00
+# GFX12: v_cvt_sr_fp8_f32 v10, s2, v5            ; encoding: [0x0a,0x00,0x6b,0xd7,0x02,0x0a,0x02,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32 v5, -|v255|, v4 ; encoding: [0x05,0x01,0x6b,0xd7,0xff,0x09,0x02,0x20]
 0x05,0x01,0x6b,0xd7,0xff,0x09,0x02,0x20
+# GFX12: v_cvt_sr_fp8_f32 v5, -|v255|, v4        ; encoding: [0x05,0x01,0x6b,0xd7,0xff,0x09,0x02,0x20]
 
-# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:1  ; encoding: [0x01,0x20,0x6b,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x20,0x6b,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:1  ; encoding: [0x01,0x20,0x6b,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:2  ; encoding: [0x01,0x40,0x6b,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x40,0x6b,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:2  ; encoding: [0x01,0x40,0x6b,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:3  ; encoding: [0x01,0x60,0x6b,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x60,0x6b,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_fp8_f32 v1, v2, v3 byte_sel:3  ; encoding: [0x01,0x60,0x6b,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 ; encoding: [0x01,0x00,0x6c,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x00,0x6c,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6c,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32 v10, s2, v5 ; encoding: [0x0a,0x00,0x6c,0xd7,0x02,0x0a,0x02,0x00]
 0x0a,0x00,0x6c,0xd7,0x02,0x0a,0x02,0x00
+# GFX12: v_cvt_sr_bf8_f32 v10, s2, v5            ; encoding: [0x0a,0x00,0x6c,0xd7,0x02,0x0a,0x02,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32 v5, -|v255|, v4 ; encoding: [0x05,0x01,0x6c,0xd7,0xff,0x09,0x02,0x20]
 0x05,0x01,0x6c,0xd7,0xff,0x09,0x02,0x20
+# GFX12: v_cvt_sr_bf8_f32 v5, -|v255|, v4        ; encoding: [0x05,0x01,0x6c,0xd7,0xff,0x09,0x02,0x20]
 
-# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:1  ; encoding: [0x01,0x20,0x6c,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x20,0x6c,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:1  ; encoding: [0x01,0x20,0x6c,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:2  ; encoding: [0x01,0x40,0x6c,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x40,0x6c,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:2  ; encoding: [0x01,0x40,0x6c,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:3  ; encoding: [0x01,0x60,0x6c,0xd7,0x02,0x07,0x02,0x00]
 0x01,0x60,0x6c,0xd7,0x02,0x07,0x02,0x00
+# GFX12: v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:3  ; encoding: [0x01,0x60,0x6c,0xd7,0x02,0x07,0x02,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x06,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x06,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x06,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x06,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x06,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_i16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x06,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x06,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x06,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x06,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x06,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x06,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_i16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x06,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x06,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x06,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_i16_i32 v5, v1, v2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, v1, v2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, v255, v255         ; encoding: [0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, v255, v255         ; encoding: [0x05,0x00,0x24,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, s1, s2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, s1, s2             ; encoding: [0x05,0x00,0x24,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, s105, s105         ; encoding: [0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, s105, s105         ; encoding: [0x05,0x00,0x24,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x24,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_i16_i32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_i16_i32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x24,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x24,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x24,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x24,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x24,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x24,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x24,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x24,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_i16_i32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x24,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x12,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x12,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x12,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x12,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x12,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x12,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x12,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x12,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x12,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x12,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_norm_i16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x12,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_norm_i16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_norm_i16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x12,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, v255, v255    ; encoding: [0x05,0x00,0x13,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, s1, s2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, s105, s105    ; encoding: [0x05,0x00,0x13,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x13,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, vcc_hi, 0xfe0b ; encoding: [0x05,0x00,0x13,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x13,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, m0, 0.5       ; encoding: [0x05,0x00,0x13,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x13,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x13,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, null, exec_lo ; encoding: [0x05,0x00,0x13,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x13,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_norm_u16_f16 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x13,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_norm_u16_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_norm_u16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x13,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x07,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, s1, s2             ; encoding: [0x05,0x00,0x07,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, s105, s105         ; encoding: [0x05,0x00,0x07,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x07,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x07,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x07,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x07,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x07,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, |exec_hi|, null    ; encoding: [0x05,0x01,0x07,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x07,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x07,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_u16_f32 v5, 0.5, -m0           ; encoding: [0x05,0x00,0x07,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x07,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x07,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u16_u32 v5, v1, v2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, v1, v2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, v255, v255         ; encoding: [0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, v255, v255         ; encoding: [0x05,0x00,0x23,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, s1, s2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, s1, s2             ; encoding: [0x05,0x00,0x23,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, s105, s105         ; encoding: [0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, s105, s105         ; encoding: [0x05,0x00,0x23,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x00,0x23,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u16_u32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u16_u32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, ttmp15, src_scc    ; encoding: [0x05,0x00,0x23,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x23,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, exec_hi, null      ; encoding: [0x05,0x00,0x23,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, null, exec_lo      ; encoding: [0x05,0x00,0x23,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x23,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, 0.5, m0            ; encoding: [0x05,0x00,0x23,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32 v5, src_scc, vcc_lo    ; encoding: [0x05,0x00,0x23,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u16_u32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x23,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u8_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_cvt_pk_u8_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x26,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_cvt_pk_u8_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x26,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x26,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x26,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_cvt_pk_u8_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x26,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_cvt_pk_u8_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u8_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x26,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u8_f32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x26,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x26,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, exec_lo, -1, vcc_hi ; encoding: [0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, exec_lo, -1, vcc_hi ; encoding: [0x05,0x00,0x26,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_cvt_pk_u8_f32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x26,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_cvt_pk_u8_f32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u8_f32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x26,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_u8_f32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_cvt_pk_u8_f32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x26,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_cvt_pk_u8_f32 v5, 0.5, m0, 0.5        ; encoding: [0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_cvt_pk_u8_f32 v5, 0.5, m0, 0.5        ; encoding: [0x05,0x00,0x26,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_cvt_pk_u8_f32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_cvt_pk_u8_f32 v5, src_scc, vcc_lo, -1 ; encoding: [0x05,0x00,0x26,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_cvt_pk_u8_f32 v255, -|0xaf123456|, vcc_hi, null ; encoding: [0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf]
 0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_u8_f32 v255, -|0xaf123456|, vcc_hi, null ; encoding: [0xff,0x01,0x26,0xd6,0xff,0xd6,0xf0,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x21,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x21,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x21,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x21,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_norm_i16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x21,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x21,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x21,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x21,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x21,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x21,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_norm_i16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x21,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_norm_i16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x21,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_norm_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_norm_i16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x21,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, v1, v2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, v255, v255    ; encoding: [0x05,0x00,0x22,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, s1, s2        ; encoding: [0x05,0x00,0x22,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, s105, s105    ; encoding: [0x05,0x00,0x22,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, vcc_lo, ttmp15 ; encoding: [0x05,0x00,0x22,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_norm_u16_f32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x22,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, ttmp15, src_scc ; encoding: [0x05,0x00,0x22,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, m0, 0.5       ; encoding: [0x05,0x00,0x22,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, exec_lo, -1   ; encoding: [0x05,0x00,0x22,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, |exec_hi|, null ; encoding: [0x05,0x01,0x22,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, null, exec_lo ; encoding: [0x05,0x00,0x22,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32 v5, -1, exec_hi   ; encoding: [0x05,0x00,0x22,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_cvt_pk_norm_u16_f32 v5, 0.5, -m0      ; encoding: [0x05,0x00,0x22,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_cvt_pk_norm_u16_f32 v5, -src_scc, |vcc_lo| ; encoding: [0x05,0x02,0x22,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_cvt_pk_norm_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_cvt_pk_norm_u16_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x22,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f16 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_div_fixup_f16 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x54,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_div_fixup_f16 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_div_fixup_f16 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x54,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_div_fixup_f16 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_div_fixup_f16 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x54,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_div_fixup_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_div_fixup_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x54,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_div_fixup_f16 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_div_fixup_f16 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x54,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_div_fixup_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_div_fixup_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x54,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_div_fixup_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_div_fixup_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x54,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_div_fixup_f16 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_div_fixup_f16 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x54,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_div_fixup_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_div_fixup_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x54,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_div_fixup_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_div_fixup_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x54,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_div_fixup_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_div_fixup_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x54,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_div_fixup_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_div_fixup_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x54,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_div_fixup_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x54,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_div_fixup_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_div_fixup_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x54,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_div_fixup_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_div_fixup_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x54,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_div_fixup_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_div_fixup_f32 v5, v1, v2, s3          ; encoding: [0x05,0x00,0x27,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_div_fixup_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_div_fixup_f32 v5, v255, s2, s105      ; encoding: [0x05,0x00,0x27,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_div_fixup_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_div_fixup_f32 v5, s1, v255, exec_hi   ; encoding: [0x05,0x00,0x27,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_div_fixup_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_div_fixup_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x27,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_div_fixup_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_div_fixup_f32 v5, vcc_lo, ttmp15, v3  ; encoding: [0x05,0x00,0x27,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_div_fixup_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x27,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_div_fixup_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x27,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_div_fixup_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_div_fixup_f32 v5, m0, 0.5, m0         ; encoding: [0x05,0x00,0x27,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_div_fixup_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_div_fixup_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x27,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_div_fixup_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_div_fixup_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x27,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_div_fixup_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x27,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_div_fixup_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x27,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_div_fixup_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_div_fixup_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x27,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_div_fixup_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_div_fixup_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x27,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_div_fixup_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x27,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04
+# GFX12: v_div_fixup_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x28,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX12: v_div_fixup_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00]
 0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00
+# GFX12: v_div_fixup_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x28,0xd6,0xfe,0xfd,0x1b,0x00]
 
-# GFX12: v_div_fixup_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07]
 0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07
+# GFX12: v_div_fixup_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x28,0xd6,0x02,0x08,0xf8,0x07]
 
-# GFX12: v_div_fixup_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1]
 0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1
+# GFX12: v_div_fixup_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x28,0xd6,0x68,0xd0,0xa0,0xa1]
 
-# GFX12: v_div_fixup_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1]
 0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1
+# GFX12: v_div_fixup_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x28,0xd6,0x6a,0xf4,0xe8,0xc1]
 
-# GFX12: v_div_fixup_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x28,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1]
 0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1
+# GFX12: v_div_fixup_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x28,0xd6,0x7e,0xfa,0xf9,0xe1]
 
-# GFX12: v_div_fixup_f64 v[5:6], null, 0.5, vcc  ; encoding: [0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01]
 0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01
+# GFX12: v_div_fixup_f64 v[5:6], null, 0.5, vcc  ; encoding: [0x05,0x00,0x28,0xd6,0x7c,0xe0,0xa9,0x01]
 
-# GFX12: v_div_fixup_f64 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f64 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x28,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fixup_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b]
 0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b
+# GFX12: v_div_fixup_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x28,0xd6,0xf0,0xf8,0xf4,0x8b]
 
-# GFX12: v_div_fixup_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73]
 0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73
+# GFX12: v_div_fixup_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x28,0xd6,0xfd,0xfc,0xc0,0x73]
 
-# GFX12: v_div_fixup_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fixup_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x28,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fmas_f32 v5, v1, 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fmas_f32 v5, v1, 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x37,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fmas_f32 v5, v255, src_scc, src_scc ; encoding: [0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03]
 0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03
+# GFX12: v_div_fmas_f32 v5, v255, src_scc, src_scc ; encoding: [0x05,0x00,0x37,0xd6,0xff,0xfb,0xf5,0x03]
 
-# GFX12: v_div_fmas_f32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01
+# GFX12: v_div_fmas_f32 v5, s105, s105, s105     ; encoding: [0x05,0x00,0x37,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX12: v_div_fmas_f32 v5, vcc_lo, v2, v3       ; encoding: [0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04]
 0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04
+# GFX12: v_div_fmas_f32 v5, vcc_lo, v2, v3       ; encoding: [0x05,0x00,0x37,0xd6,0x6a,0x04,0x0e,0x04]
 
-# GFX12: v_div_fmas_f32 v5, vcc_hi, v255, vcc_hi ; encoding: [0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01]
 0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01
+# GFX12: v_div_fmas_f32 v5, vcc_hi, v255, vcc_hi ; encoding: [0x05,0x00,0x37,0xd6,0x6b,0xfe,0xaf,0x01]
 
-# GFX12: v_div_fmas_f32 v5, -|ttmp15|, -|ttmp15|, ttmp15 ; encoding: [0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61]
 0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61
+# GFX12: v_div_fmas_f32 v5, -|ttmp15|, -|ttmp15|, ttmp15 ; encoding: [0x05,0x03,0x37,0xd6,0x7b,0xf6,0xec,0x61]
 
-# GFX12: v_div_fmas_f32 v5, m0, 0.5, v255        ; encoding: [0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07]
 0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07
+# GFX12: v_div_fmas_f32 v5, m0, 0.5, v255        ; encoding: [0x05,0x00,0x37,0xd6,0x7d,0xe0,0xfd,0x07]
 
-# GFX12: v_div_fmas_f32 v5, -|exec_lo|, exec_lo, -|exec_lo| ; encoding: [0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1]
 0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1
+# GFX12: v_div_fmas_f32 v5, -|exec_lo|, exec_lo, -|exec_lo| ; encoding: [0x05,0x05,0x37,0xd6,0x7e,0xfc,0xf8,0xa1]
 
-# GFX12: v_div_fmas_f32 v5, -|exec_hi|, -|exec_hi|, -|exec_hi| ; encoding: [0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1]
 0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1
+# GFX12: v_div_fmas_f32 v5, -|exec_hi|, -|exec_hi|, -|exec_hi| ; encoding: [0x05,0x07,0x37,0xd6,0x7f,0xfe,0xfc,0xe1]
 
-# GFX12: v_div_fmas_f32 v5, null, m0, -|m0|      ; encoding: [0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81]
 0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81
+# GFX12: v_div_fmas_f32 v5, null, m0, -|m0|      ; encoding: [0x05,0x04,0x37,0xd6,0x7c,0xfa,0xf4,0x81]
 
-# GFX12: v_div_fmas_f32 v5, -1, -|vcc_lo|, -|vcc_lo| ; encoding: [0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1]
 0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1
+# GFX12: v_div_fmas_f32 v5, -1, -|vcc_lo|, -|vcc_lo| ; encoding: [0x05,0x06,0x37,0xd6,0xc1,0xd4,0xa8,0xc1]
 
-# GFX12: v_div_fmas_f32 v5, 0.5, -|vcc_hi|, 0.5 mul:2 ; encoding: [0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b]
 0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b
+# GFX12: v_div_fmas_f32 v5, 0.5, -|vcc_hi|, 0.5 mul:2 ; encoding: [0x05,0x02,0x37,0xd6,0xf0,0xd6,0xc0,0x4b]
 
-# GFX12: v_div_fmas_f32 v5, src_scc, -1, -1 mul:4 ; encoding: [0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13]
 0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13
+# GFX12: v_div_fmas_f32 v5, src_scc, -1, -1 mul:4 ; encoding: [0x05,0x00,0x37,0xd6,0xfd,0x82,0x05,0x13]
 
-# GFX12: v_div_fmas_f32 v255, -|0xaf123456|, null, null clamp div:2 ; encoding: [0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf]
 0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fmas_f32 v255, -|0xaf123456|, null, null clamp div:2 ; encoding: [0xff,0x81,0x37,0xd6,0xff,0xf8,0xf0,0x39,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fmas_f64 v[5:6], v[1:2], 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fmas_f64 v[5:6], v[1:2], 0xaf123456, 0xaf123456 ; encoding: [0x05,0x00,0x38,0xd6,0x01,0xff,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_div_fmas_f64 v[5:6], v[254:255], src_scc, v[3:4] ; encoding: [0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04]
 0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04
+# GFX12: v_div_fmas_f64 v[5:6], v[254:255], src_scc, v[3:4] ; encoding: [0x05,0x00,0x38,0xd6,0xfe,0xfb,0x0d,0x04]
 
-# GFX12: v_div_fmas_f64 v[5:6], s[104:105], |s[104:105]|, s[104:105] ; encoding: [0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01]
 0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01
+# GFX12: v_div_fmas_f64 v[5:6], s[104:105], |s[104:105]|, s[104:105] ; encoding: [0x05,0x02,0x38,0xd6,0x68,0xd0,0xa0,0x01]
 
-# GFX12: v_div_fmas_f64 v[5:6], -|vcc|, v[2:3], -|v[254:255]| ; encoding: [0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7]
 0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7
+# GFX12: v_div_fmas_f64 v[5:6], -|vcc|, v[2:3], -|v[254:255]| ; encoding: [0x05,0x05,0x38,0xd6,0x6a,0x04,0xfa,0xa7]
 
-# GFX12: v_div_fmas_f64 v[5:6], -|ttmp[14:15]|, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1]
 0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1
+# GFX12: v_div_fmas_f64 v[5:6], -|ttmp[14:15]|, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x07,0x38,0xd6,0x7a,0xf4,0xe8,0xe1]
 
-# GFX12: v_div_fmas_f64 v[5:6], -|exec|, -|v[254:255]|, null ; encoding: [0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61]
 0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61
+# GFX12: v_div_fmas_f64 v[5:6], -|exec|, -|v[254:255]|, null ; encoding: [0x05,0x03,0x38,0xd6,0x7e,0xfc,0xf3,0x61]
 
-# GFX12: v_div_fmas_f64 v[5:6], null, 0.5, -src_scc ; encoding: [0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83]
 0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83
+# GFX12: v_div_fmas_f64 v[5:6], null, 0.5, -src_scc ; encoding: [0x05,0x00,0x38,0xd6,0x7c,0xe0,0xf5,0x83]
 
-# GFX12: v_div_fmas_f64 v[5:6], -1, -exec, |exec| ; encoding: [0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41]
 0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41
+# GFX12: v_div_fmas_f64 v[5:6], -1, -exec, |exec| ; encoding: [0x05,0x04,0x38,0xd6,0xc1,0xfc,0xf8,0x41]
 
-# GFX12: v_div_fmas_f64 v[5:6], 0.5, -|vcc|, -|vcc| mul:2 ; encoding: [0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9]
 0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9
+# GFX12: v_div_fmas_f64 v[5:6], 0.5, -|vcc|, -|vcc| mul:2 ; encoding: [0x05,0x06,0x38,0xd6,0xf0,0xd4,0xa8,0xc9]
 
-# GFX12: v_div_fmas_f64 v[5:6], -|src_scc|, -1, 0.5 mul:4 ; encoding: [0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33]
 0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33
+# GFX12: v_div_fmas_f64 v[5:6], -|src_scc|, -1, 0.5 mul:4 ; encoding: [0x05,0x01,0x38,0xd6,0xfd,0x82,0xc1,0x33]
 
-# GFX12: v_div_fmas_f64 v[254:255], 0xaf123456, null, -1 clamp div:2 ; encoding: [0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf
+# GFX12: v_div_fmas_f64 v[254:255], 0xaf123456, null, -1 clamp div:2 ; encoding: [0xfe,0x80,0x38,0xd6,0xff,0xf8,0x04,0x1b,0x56,0x34,0x12,0xaf]
 
-# W32: v_div_scale_f32 v5, vcc_lo, v1, v2, s3    ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
-# W64: v_div_scale_f32 v5, vcc, v1, v2, s3       ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00
+# W32: v_div_scale_f32 v5, vcc_lo, v1, v2, s3  ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
+# W64: v_div_scale_f32 v5, vcc, v1, v2, s3     ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0x05,0x0e,0x00]
 
-# W32: v_div_scale_f32 v5, vcc_lo, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
-# W64: v_div_scale_f32 v5, vcc, v255, s2, s105   ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01
+# W32: v_div_scale_f32 v5, vcc_lo, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
+# W64: v_div_scale_f32 v5, vcc, v255, s2, s105 ; encoding: [0x05,0x6a,0xfc,0xd6,0xff,0x05,0xa4,0x01]
 
+0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, s1, v255, exec_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01]
 # W64: v_div_scale_f32 v5, vcc, s1, v255, exec_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01]
-0x05,0x6a,0xfc,0xd6,0x01,0xfe,0xff,0x01
 
+0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, s105, s105, exec_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01]
 # W64: v_div_scale_f32 v5, vcc, s105, s105, exec_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01]
-0x05,0x6a,0xfc,0xd6,0x69,0xd2,0xf8,0x01
 
+0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04
 # W32: v_div_scale_f32 v5, vcc_lo, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04]
 # W64: v_div_scale_f32 v5, vcc, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04]
-0x05,0x6a,0xfc,0xd6,0x6a,0xf6,0x0c,0x04
 
+0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v5, vcc_lo, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v5, vcc, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfc,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1
 # W32: v_div_scale_f32 v5, vcc_lo, -ttmp15, -src_scc, -ttmp15 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1]
 # W64: v_div_scale_f32 v5, vcc, -ttmp15, -src_scc, -ttmp15 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1]
-0x05,0x6a,0xfc,0xd6,0x7b,0xfa,0xed,0xe1
 
-# W32: v_div_scale_f32 v5, vcc_lo, m0, 0.5, m0   ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
-# W64: v_div_scale_f32 v5, vcc, m0, 0.5, m0      ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01
+# W32: v_div_scale_f32 v5, vcc_lo, m0, 0.5, m0 ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64: v_div_scale_f32 v5, vcc, m0, 0.5, m0    ; encoding: [0x05,0x6a,0xfc,0xd6,0x7d,0xe0,0xf5,0x01]
 
+0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01
 # W32: v_div_scale_f32 v5, vcc_lo, exec_lo, -1, vcc_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01]
 # W64: v_div_scale_f32 v5, vcc, exec_lo, -1, vcc_hi ; encoding: [0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01]
-0x05,0x6a,0xfc,0xd6,0x7e,0x82,0xad,0x01
 
+0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1
 # W32: v_div_scale_f32 v5, vcc_lo, -exec_hi, null, -vcc_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1]
 # W64: v_div_scale_f32 v5, vcc, -exec_hi, null, -vcc_lo ; encoding: [0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1]
-0x05,0x6a,0xfc,0xd6,0x7f,0xf8,0xa8,0xa1
 
+0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v5, vcc_lo, null, exec_lo, neg(0xaf123456) ; encoding: [0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v5, vcc, null, exec_lo, neg(0xaf123456) ; encoding: [0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfc,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3
 # W32: v_div_scale_f32 v5, vcc_lo, -1, -exec_hi, -src_scc ; encoding: [0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3]
 # W64: v_div_scale_f32 v5, vcc, -1, -exec_hi, -src_scc ; encoding: [0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3]
-0x05,0x6a,0xfc,0xd6,0xc1,0xfe,0xf4,0xc3
 
+0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b
 # W32: v_div_scale_f32 v5, vcc_lo, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b]
 # W64: v_div_scale_f32 v5, vcc, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b]
-0x05,0x6a,0xfc,0xd6,0xf0,0xfa,0xc0,0x4b
 
+0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33
 # W32: v_div_scale_f32 v5, vcc_lo, -src_scc, vcc_lo, -1 mul:4 ; encoding: [0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33]
 # W64: v_div_scale_f32 v5, vcc, -src_scc, vcc_lo, -1 mul:4 ; encoding: [0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33]
-0x05,0x6a,0xfc,0xd6,0xfd,0xd4,0x04,0x33
 
+0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f32 v255, vcc_lo, neg(0xaf123456), -vcc_hi, null clamp div:2 ; encoding: [0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f32 v255, vcc, neg(0xaf123456), -vcc_hi, null clamp div:2 ; encoding: [0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
-0xff,0xea,0xfc,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04
 # W32: v_div_scale_f64 v[5:6], vcc_lo, v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04]
 # W64: v_div_scale_f64 v[5:6], vcc, v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04]
-0x05,0x6a,0xfd,0xd6,0x01,0x05,0x0e,0x04
 
+0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00
 # W32: v_div_scale_f64 v[5:6], vcc_lo, v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00]
 # W64: v_div_scale_f64 v[5:6], vcc, v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00]
-0x05,0x6a,0xfd,0xd6,0xfe,0xfd,0x1b,0x00
 
+0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07
 # W32: v_div_scale_f64 v[5:6], vcc_lo, s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07]
 # W64: v_div_scale_f64 v[5:6], vcc, s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07]
-0x05,0x6a,0xfd,0xd6,0x02,0x08,0xf8,0x07
 
+0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -s[104:105], s[104:105], -s[104:105] ; encoding: [0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1]
 # W64: v_div_scale_f64 v[5:6], vcc, -s[104:105], s[104:105], -s[104:105] ; encoding: [0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1]
-0x05,0x6a,0xfd,0xd6,0x68,0xd0,0xa0,0xa1
 
+0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, vcc, -ttmp[14:15], -ttmp[14:15] ; encoding: [0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1]
 # W64: v_div_scale_f64 v[5:6], vcc, vcc, -ttmp[14:15], -ttmp[14:15] ; encoding: [0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1]
-0x05,0x6a,0xfd,0xd6,0x6a,0xf4,0xe8,0xc1
 
+0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -ttmp[14:15], 0xaf123456, null ; encoding: [0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[5:6], vcc, -ttmp[14:15], 0xaf123456, null ; encoding: [0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfd,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -exec, -src_scc, -exec ; encoding: [0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1]
 # W64: v_div_scale_f64 v[5:6], vcc, -exec, -src_scc, -exec ; encoding: [0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1]
-0x05,0x6a,0xfd,0xd6,0x7e,0xfa,0xf9,0xe1
 
+0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01
 # W32: v_div_scale_f64 v[5:6], vcc_lo, null, 0.5, vcc ; encoding: [0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01]
 # W64: v_div_scale_f64 v[5:6], vcc, null, 0.5, vcc ; encoding: [0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01]
-0x05,0x6a,0xfd,0xd6,0x7c,0xe0,0xa9,0x01
 
+0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -1, -1, 0xaf123456 ; encoding: [0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[5:6], vcc, -1, -1, 0xaf123456 ; encoding: [0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfd,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b
 # W32: v_div_scale_f64 v[5:6], vcc_lo, 0.5, null, -src_scc mul:2 ; encoding: [0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b]
 # W64: v_div_scale_f64 v[5:6], vcc, 0.5, null, -src_scc mul:2 ; encoding: [0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b]
-0x05,0x6a,0xfd,0xd6,0xf0,0xf8,0xf4,0x8b
 
+0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73
 # W32: v_div_scale_f64 v[5:6], vcc_lo, -src_scc, -exec, 0.5 mul:4 ; encoding: [0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73]
 # W64: v_div_scale_f64 v[5:6], vcc, -src_scc, -exec, 0.5 mul:4 ; encoding: [0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73]
-0x05,0x6a,0xfd,0xd6,0xfd,0xfc,0xc0,0x73
 
+0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
 # W32: v_div_scale_f64 v[254:255], vcc_lo, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 # W64: v_div_scale_f64 v[254:255], vcc, 0xaf123456, -vcc, -1 clamp div:2 ; encoding: [0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
-0xfe,0xea,0xfd,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
 
-# GFX12: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, v1, v2, s3       ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, v255, v255, s105 ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, v255, v255, s105   ; encoding: [0x05,0x00,0x67,0xd6,0xff,0xff,0xa7,0x01]
 
-# GFX12: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, s1, s2, v3.l     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, s1, s2, v3         ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x04,0x0c,0x04]
 
-# GFX12: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, s105, s105, m0   ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, s105, s105, m0     ; encoding: [0x05,0x00,0x67,0xd6,0x69,0xd2,0xf4,0x01]
 
-# GFX12: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.l ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 ; encoding: [0x05,0x00,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-# GFX12: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x67,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x67,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, |m0|, -1, -vcc_lo ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, |m0|, -1, -vcc_lo  ; encoding: [0x05,0x01,0x67,0xd6,0x7d,0x82,0xa9,0x81]
 
-# GFX12: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -|exec_lo|, null, -|0xfe0b| ; encoding: [0x05,0x05,0x67,0xd6,0x7e,0xf8,0xfc,0xa3,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -|exec_hi|, -|exec_lo|, -|exec_lo| ; encoding: [0x05,0x07,0x67,0xd6,0x7f,0xfc,0xf8,0xe1]
 
-# GFX12: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, null, -exec_hi, |src_scc| ; encoding: [0x05,0x04,0x67,0xd6,0x7c,0xfe,0xf4,0x43]
 
-# GFX12: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -1, -|m0|, -|exec_hi| ; encoding: [0x05,0x06,0x67,0xd6,0xc1,0xfa,0xfc,0xc1]
 
-# GFX12: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x67,0xd6,0xfd,0xd4,0x04,0x23]
+
+0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+
+0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+
+0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07
+# W32-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W32-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-REAL16: v_dot2_bf16_bf16 v5.l, vcc_lo, ttmp15, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
+# W64-FAKE16: v_dot2_bf16_bf16 v5, vcc_lo, ttmp15, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0x6a,0xf6,0xfc,0x07]
 
-# GFX12: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x67,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, v1, v2, s3         ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x66,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, v255, s2, s105     ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x66,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x66,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, s105, s105, exec_lo ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x66,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x66,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.l ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1
+# W32-REAL16: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W32-FAKE16: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
+# W64-FAKE16: v_dot2_f16_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x66,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, m0, 0.5, m0        ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x66,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01
+# W32-REAL16: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W32-FAKE16: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W64-REAL16: v_dot2_f16_f16 v5.l, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
+# W64-FAKE16: v_dot2_f16_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x66,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1
+# W32-REAL16: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W32-FAKE16: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
+# W64-FAKE16: v_dot2_f16_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x66,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x66,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3
+# W32-REAL16: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W32-FAKE16: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
+# W64-FAKE16: v_dot2_f16_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x66,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43
+# W32-REAL16: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W32-FAKE16: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-REAL16: v_dot2_f16_f16 v5.l, 0.5, -m0, 0.5      ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
+# W64-FAKE16: v_dot2_f16_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x66,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23
+# W32-REAL16: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W32-FAKE16: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-REAL16: v_dot2_f16_f16 v5.l, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
+# W64-FAKE16: v_dot2_f16_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x66,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+
+0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+
+0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v5.l, vcc_hi, 0xfe0b, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v5, vcc_hi, 0xfe0b, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
+
+0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16 v255.h, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] ; encoding: [0xff,0x43,0x66,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_fma_dx9_zero_f32 v5, v1, v2, s3       ; encoding: [0x05,0x00,0x09,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_fma_dx9_zero_f32 v5, v255, s2, s105   ; encoding: [0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_fma_dx9_zero_f32 v5, v255, s2, s105   ; encoding: [0x05,0x00,0x09,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_fma_dx9_zero_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_fma_dx9_zero_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x09,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_fma_dx9_zero_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_fma_dx9_zero_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x09,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_fma_dx9_zero_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_fma_dx9_zero_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x09,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_fma_dx9_zero_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_dx9_zero_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x09,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_dx9_zero_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_fma_dx9_zero_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x09,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_fma_dx9_zero_f32 v5, m0, 0.5, m0      ; encoding: [0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_fma_dx9_zero_f32 v5, m0, 0.5, m0      ; encoding: [0x05,0x00,0x09,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_fma_dx9_zero_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_fma_dx9_zero_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x09,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_fma_dx9_zero_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_fma_dx9_zero_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x09,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_fma_dx9_zero_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_dx9_zero_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x09,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_dx9_zero_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_fma_dx9_zero_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x09,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_fma_dx9_zero_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_fma_dx9_zero_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x09,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_fma_dx9_zero_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_fma_dx9_zero_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x09,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_fma_dx9_zero_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_dx9_zero_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x09,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_fma_f16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x48,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_fma_f16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_fma_f16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x48,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_fma_f16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_fma_f16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x48,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_fma_f16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_fma_f16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x48,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_fma_f16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_fma_f16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x48,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_fma_f16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_fma_f16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x48,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fma_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_fma_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x48,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_fma_f16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_fma_f16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x48,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_fma_f16 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_fma_f16 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x48,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_fma_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_fma_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x48,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_fma_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_fma_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x48,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fma_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_fma_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x48,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_fma_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_fma_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x48,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_fma_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_fma_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x48,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_fma_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x48,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_fma_f32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_fma_f32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x13,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_fma_f32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_fma_f32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x13,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_fma_f32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_fma_f32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x13,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_fma_f32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_fma_f32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x13,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_fma_f32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_fma_f32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x13,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_fma_f32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x13,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_fma_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x13,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_fma_f32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_fma_f32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x13,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_fma_f32 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_fma_f32 v5, |exec_lo|, -1, vcc_hi     ; encoding: [0x05,0x01,0x13,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_fma_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_fma_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x13,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_fma_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x13,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_fma_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x13,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_fma_f32 v5, 0.5, -m0, 0.5 mul:2       ; encoding: [0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_fma_f32 v5, 0.5, -m0, 0.5 mul:2       ; encoding: [0x05,0x00,0x13,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_fma_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_fma_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x13,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_fma_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x13,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04
+# GFX12: v_fma_f64 v[5:6], v[1:2], v[2:3], v[3:4] ; encoding: [0x05,0x00,0x14,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX12: v_fma_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00]
 0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00
+# GFX12: v_fma_f64 v[5:6], v[254:255], v[254:255], s[6:7] ; encoding: [0x05,0x00,0x14,0xd6,0xfe,0xfd,0x1b,0x00]
 
-# GFX12: v_fma_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07]
 0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07
+# GFX12: v_fma_f64 v[5:6], s[2:3], s[4:5], v[254:255] ; encoding: [0x05,0x00,0x14,0xd6,0x02,0x08,0xf8,0x07]
 
-# GFX12: v_fma_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1]
 0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1
+# GFX12: v_fma_f64 v[5:6], -|s[104:105]|, s[104:105], -|s[104:105]| ; encoding: [0x05,0x05,0x14,0xd6,0x68,0xd0,0xa0,0xa1]
 
-# GFX12: v_fma_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1]
 0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1
+# GFX12: v_fma_f64 v[5:6], vcc, -|ttmp[14:15]|, -|ttmp[14:15]| ; encoding: [0x05,0x06,0x14,0xd6,0x6a,0xf4,0xe8,0xc1]
 
-# GFX12: v_fma_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f64 v[5:6], -|ttmp[14:15]|, 0xaf123456, null ; encoding: [0x05,0x01,0x14,0xd6,0x7a,0xfe,0xf1,0x21,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1]
 0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1
+# GFX12: v_fma_f64 v[5:6], -|exec|, -|src_scc|, -|exec| ; encoding: [0x05,0x07,0x14,0xd6,0x7e,0xfa,0xf9,0xe1]
 
-# GFX12: v_fma_f64 v[5:6], null, 0.5, vcc        ; encoding: [0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01]
 0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01
+# GFX12: v_fma_f64 v[5:6], null, 0.5, vcc        ; encoding: [0x05,0x00,0x14,0xd6,0x7c,0xe0,0xa9,0x01]
 
-# GFX12: v_fma_f64 v[5:6], -1, -1, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f64 v[5:6], -1, -1, 0xaf123456    ; encoding: [0x05,0x00,0x14,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_fma_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b]
 0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b
+# GFX12: v_fma_f64 v[5:6], 0.5, null, -|src_scc| mul:2 ; encoding: [0x05,0x04,0x14,0xd6,0xf0,0xf8,0xf4,0x8b]
 
-# GFX12: v_fma_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73]
 0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73
+# GFX12: v_fma_f64 v[5:6], -|src_scc|, -|exec|, 0.5 mul:4 ; encoding: [0x05,0x03,0x14,0xd6,0xfd,0xfc,0xc0,0x73]
 
-# GFX12: v_fma_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf
+# GFX12: v_fma_f64 v[254:255], 0xaf123456, -|vcc|, -1 clamp div:2 ; encoding: [0xfe,0x82,0x14,0xd6,0xff,0xd4,0x04,0x5b,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ldexp_f32 v5, v1, v2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_ldexp_f32 v5, v1, v2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_ldexp_f32 v5, v255, v255              ; encoding: [0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_ldexp_f32 v5, v255, v255              ; encoding: [0x05,0x00,0x1c,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_ldexp_f32 v5, s1, s2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_ldexp_f32 v5, s1, s2                  ; encoding: [0x05,0x00,0x1c,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, s105, s105              ; encoding: [0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_ldexp_f32 v5, s105, s105              ; encoding: [0x05,0x00,0x1c,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_ldexp_f32 v5, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x1c,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, vcc_hi, 0xaf123456      ; encoding: [0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_ldexp_f32 v5, vcc_hi, 0xaf123456      ; encoding: [0x05,0x00,0x1c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ldexp_f32 v5, ttmp15, src_scc         ; encoding: [0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_ldexp_f32 v5, ttmp15, src_scc         ; encoding: [0x05,0x00,0x1c,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_ldexp_f32 v5, m0, 0.5                 ; encoding: [0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_ldexp_f32 v5, m0, 0.5                 ; encoding: [0x05,0x00,0x1c,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_ldexp_f32 v5, exec_lo, -1             ; encoding: [0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_ldexp_f32 v5, exec_lo, -1             ; encoding: [0x05,0x00,0x1c,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_ldexp_f32 v5, exec_hi, null           ; encoding: [0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_ldexp_f32 v5, exec_hi, null           ; encoding: [0x05,0x00,0x1c,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, null, exec_lo           ; encoding: [0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_ldexp_f32 v5, null, exec_lo           ; encoding: [0x05,0x00,0x1c,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, -1, exec_hi             ; encoding: [0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_ldexp_f32 v5, -1, exec_hi             ; encoding: [0x05,0x00,0x1c,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_ldexp_f32 v5, 0.5, m0 mul:2           ; encoding: [0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08]
 0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08
+# GFX12: v_ldexp_f32 v5, 0.5, m0 mul:2           ; encoding: [0x05,0x00,0x1c,0xd7,0xf0,0xfa,0x00,0x08]
 
-# GFX12: v_ldexp_f32 v5, src_scc, vcc_lo mul:4   ; encoding: [0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10]
 0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10
+# GFX12: v_ldexp_f32 v5, src_scc, vcc_lo mul:4   ; encoding: [0x05,0x00,0x1c,0xd7,0xfd,0xd4,0x00,0x10]
 
-# GFX12: v_ldexp_f32 v255, -|0xaf123456|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf]
 0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf
+# GFX12: v_ldexp_f32 v255, -|0xaf123456|, vcc_hi clamp div:2 ; encoding: [0xff,0x81,0x1c,0xd7,0xff,0xd6,0x00,0x38,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_ldexp_f64 v[5:6], v[1:2], v2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_ldexp_f64 v[5:6], v[1:2], v2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], v[1:2], v255        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00
+# GFX12: v_ldexp_f64 v[5:6], v[1:2], v255        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], v[1:2], s2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], v[1:2], s2          ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], v[1:2], s105        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], v[1:2], s105        ; encoding: [0x05,0x00,0x2b,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], v[254:255], ttmp15  ; encoding: [0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], v[254:255], ttmp15  ; encoding: [0x05,0x00,0x2b,0xd7,0xfe,0xf7,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], s[2:3], vcc_hi      ; encoding: [0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], s[2:3], vcc_hi      ; encoding: [0x05,0x00,0x2b,0xd7,0x02,0xd6,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], s[104:105], vcc_lo  ; encoding: [0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], s[104:105], vcc_lo  ; encoding: [0x05,0x00,0x2b,0xd7,0x68,0xd4,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], vcc, m0             ; encoding: [0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], vcc, m0             ; encoding: [0x05,0x00,0x2b,0xd7,0x6a,0xfa,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2b,0xd7,0x7a,0xfe,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], exec, exec_lo       ; encoding: [0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], exec, exec_lo       ; encoding: [0x05,0x00,0x2b,0xd7,0x7e,0xfc,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], null, null          ; encoding: [0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00
+# GFX12: v_ldexp_f64 v[5:6], null, null          ; encoding: [0x05,0x00,0x2b,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], -1, -1              ; encoding: [0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00
+# GFX12: v_ldexp_f64 v[5:6], -1, -1              ; encoding: [0x05,0x00,0x2b,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX12: v_ldexp_f64 v[5:6], 0.5, 0.5 mul:2      ; encoding: [0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08]
 0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08
+# GFX12: v_ldexp_f64 v[5:6], 0.5, 0.5 mul:2      ; encoding: [0x05,0x00,0x2b,0xd7,0xf0,0xe0,0x01,0x08]
 
-# GFX12: v_ldexp_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30]
 0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30
+# GFX12: v_ldexp_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2b,0xd7,0xfd,0xfa,0x01,0x30]
 
-# GFX12: v_ldexp_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf
+# GFX12: v_ldexp_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2b,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lerp_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_lerp_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x15,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_lerp_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_lerp_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x15,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_lerp_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_lerp_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x15,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_lerp_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_lerp_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x15,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_lerp_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_lerp_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x15,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_lerp_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_lerp_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x15,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lerp_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_lerp_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x15,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_lerp_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_lerp_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x15,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_lerp_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_lerp_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x15,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_lerp_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_lerp_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x15,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_lerp_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_lerp_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x15,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lerp_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_lerp_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x15,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_lerp_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_lerp_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x15,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_lerp_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_lerp_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x15,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_lerp_u8 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_lerp_u8 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x15,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_add_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_lshl_add_u32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x46,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_lshl_add_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_lshl_add_u32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x46,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_lshl_add_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_lshl_add_u32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x46,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_lshl_add_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_lshl_add_u32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x46,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_lshl_add_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_lshl_add_u32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x46,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_lshl_add_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_add_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x46,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_add_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_lshl_add_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x46,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_lshl_add_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_lshl_add_u32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x46,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_lshl_add_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_lshl_add_u32 v5, exec_lo, -1, vcc_hi  ; encoding: [0x05,0x00,0x46,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_lshl_add_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_lshl_add_u32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x46,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_lshl_add_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_add_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x46,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_add_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_lshl_add_u32 v5, -1, exec_hi, src_scc ; encoding: [0x05,0x00,0x46,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_lshl_add_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_lshl_add_u32 v5, 0.5, m0, 0.5         ; encoding: [0x05,0x00,0x46,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_lshl_add_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_lshl_add_u32 v5, src_scc, vcc_lo, -1  ; encoding: [0x05,0x00,0x46,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_lshl_add_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_add_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x46,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_or_b32 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_lshl_or_b32 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x56,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_lshl_or_b32 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_lshl_or_b32 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x56,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_lshl_or_b32 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_lshl_or_b32 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x56,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_lshl_or_b32 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_lshl_or_b32 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x56,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_lshl_or_b32 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_lshl_or_b32 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x56,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_lshl_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_or_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x56,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_lshl_or_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x56,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_lshl_or_b32 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_lshl_or_b32 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x56,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_lshl_or_b32 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_lshl_or_b32 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x56,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_lshl_or_b32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_lshl_or_b32 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x56,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_lshl_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_or_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x56,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshl_or_b32 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_lshl_or_b32 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x56,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_lshl_or_b32 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_lshl_or_b32 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x56,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_lshl_or_b32 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_lshl_or_b32 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x56,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_lshl_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_lshl_or_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x56,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x38,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x38,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x38,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshlrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshlrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x38,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x38,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshlrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x38,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v1.l, v2.l          ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.l      ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, s1, s2              ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, s1, s2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, s105, s105          ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, s105, s105            ; encoding: [0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, vcc_lo, ttmp15      ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, vcc_hi, 0xfe0b      ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, vcc_hi, 0xfe0b        ; encoding: [0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, ttmp15, src_scc     ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, m0, 0x3800          ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, m0, 0x3800            ; encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, exec_lo, -1         ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, exec_hi, null       ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, exec_hi, null         ; encoding: [0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, null, exec_lo       ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x39,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, -1, exec_hi         ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, 0x3800, m0          ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, 0x3800, m0            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, src_scc, vcc_lo       ; encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v255.l, 0xfe0b, vcc_hi    ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v1, v2                ; encoding: [0x05,0x00,0x39,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x39,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_lshrrev_b16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_lshrrev_b16 v5, v255, v255            ; encoding: [0x05,0x00,0x39,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x39,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_lshrrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16 v255, 0xfe0b, vcc_hi      ; encoding: [0xff,0x00,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x39,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_lshrrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00
+# GFX12: v_lshrrev_b64 v[5:6], v1, vcc           ; encoding: [0x05,0x00,0x3d,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00
+# GFX12: v_lshrrev_b64 v[5:6], v255, exec        ; encoding: [0x05,0x00,0x3d,0xd7,0xff,0xfd,0x00,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00]
 0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00
+# GFX12: v_lshrrev_b64 v[5:6], exec_lo, v[2:3]   ; encoding: [0x05,0x00,0x3d,0xd7,0x7e,0x04,0x02,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00]
 0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00
+# GFX12: v_lshrrev_b64 v[5:6], exec_hi, v[254:255] ; encoding: [0x05,0x00,0x3d,0xd7,0x7f,0xfc,0x03,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00
+# GFX12: v_lshrrev_b64 v[5:6], null, null        ; encoding: [0x05,0x00,0x3d,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00
+# GFX12: v_lshrrev_b64 v[5:6], -1, -1            ; encoding: [0x05,0x00,0x3d,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX12: v_lshrrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_lshrrev_b64 v[5:6], 0.5, 0xaf123456   ; encoding: [0x05,0x00,0x3d,0xd7,0xf0,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_lshrrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00]
 0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00
+# GFX12: v_lshrrev_b64 v[5:6], src_scc, src_scc  ; encoding: [0x05,0x00,0x3d,0xd7,0xfd,0xfa,0x01,0x00]
 
-# GFX12: v_lshrrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_lshrrev_b64 v[254:255], 0xaf123456, 0.5 ; encoding: [0xfe,0x00,0x3d,0xd7,0xff,0xe0,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_i16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_mad_i16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x53,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_mad_i16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_mad_i16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x53,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_mad_i16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_mad_i16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x53,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_mad_i16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_mad_i16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x53,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_mad_i16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_mad_i16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x53,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_mad_i16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_i16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x53,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_i16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_i16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_i16 v5, m0, 0x3800, m0            ; encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_i16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_mad_i16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_mad_i16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_mad_i16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x53,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_mad_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x53,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_mad_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_mad_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x53,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04
+# GFX12: v_mad_i32_i16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX12: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00
+# GFX12: v_mad_i32_i16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xff,0x0f,0x00]
 
-# GFX12: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
 0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07
+# GFX12: v_mad_i32_i16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x5a,0xd6,0x01,0x04,0xfc,0x07]
 
-# GFX12: v_mad_i32_i16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01
+# GFX12: v_mad_i32_i16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x5a,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX12: v_mad_i32_i16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01]
 0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01
+# GFX12: v_mad_i32_i16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x6a,0xf6,0xa8,0x01]
 
-# GFX12: v_mad_i32_i16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_i32_i16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x5a,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_i32_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_i32_i16 v5, m0, 0x3800, m0        ; encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_i32_i16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
 0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01
+# GFX12: v_mad_i32_i16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
 
-# GFX12: v_mad_i32_i16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01]
 0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01
+# GFX12: v_mad_i32_i16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x5a,0xd6,0x7f,0xf8,0xf8,0x01]
 
-# GFX12: v_mad_i32_i16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01]
 0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01
+# GFX12: v_mad_i32_i16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x5a,0xd6,0x7c,0xfc,0xf0,0x01]
 
-# GFX12: v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_i32_i16 v5, 0x3800, m0, -1
 0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03
+# GFX12: v_mad_i32_i16 v5, 0x3800, m0, -1        ; encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
 0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX12: v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
 
-# GFX12: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_i32_i16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x5a,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_mad_i32_i24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_mad_i32_i24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_mad_i32_i24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_mad_i32_i24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_mad_i32_i24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_mad_i32_i24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_mad_i32_i24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_mad_i32_i24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_mad_i32_i24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_mad_i32_i24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_i32_i24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_i32_i24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_i32_i24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_i32_i24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_i32_i24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_mad_i32_i24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_mad_i32_i24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_mad_i32_i24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_mad_i32_i24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_mad_i32_i24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_i32_i24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_i32_i24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_mad_i32_i24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_mad_i32_i24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_mad_i32_i24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0a,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_mad_i32_i24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_mad_i32_i24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_mad_i32_i24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_i32_i24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00
 # W32: v_mad_co_i64_i32 v[5:6], s12, v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0x01,0x05,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00
 # W32: v_mad_co_i64_i32 v[5:6], s12, v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0xff,0xf7,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00
 # W32: v_mad_co_i64_i32 v[5:6], s12, s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00]
-0x05,0x0c,0xff,0xd6,0x01,0x04,0x1a,0x00
 
+0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00
 # W32: v_mad_co_i64_i32 v[5:6], s12, s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00]
-0x05,0x0c,0xff,0xd6,0x69,0xd2,0x18,0x00
 
+0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01
 # W32: v_mad_co_i64_i32 v[5:6], s12, vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01]
-0x05,0x0c,0xff,0xd6,0x6a,0xfe,0xa3,0x01
 
+0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 # W32: v_mad_co_i64_i32 v[5:6], s12, vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0xff,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 
+0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07
 # W32: v_mad_co_i64_i32 v[5:6], s12, ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07]
-0x05,0x0c,0xff,0xd6,0x7b,0xfa,0xf9,0x07
 
+0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01
 # W32: v_mad_co_i64_i32 v[5:6], s12, m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01]
-0x05,0x0c,0xff,0xd6,0x7d,0xe0,0xe9,0x01
 
+0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01
 # W32: v_mad_co_i64_i32 v[5:6], s12, exec_lo, -1, exec ; encoding: [0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], exec_lo, -1, exec ; encoding: [0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01]
-0x05,0x0c,0xff,0xd6,0x7e,0x82,0xf9,0x01
 
+0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01
 # W32: v_mad_co_i64_i32 v[5:6], s12, exec_hi, null, vcc ; encoding: [0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], exec_hi, null, vcc ; encoding: [0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01]
-0x05,0x0c,0xff,0xd6,0x7f,0xf8,0xa8,0x01
 
+0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01
 # W32: v_mad_co_i64_i32 v[5:6], s12, null, exec_lo, null ; encoding: [0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01]
 # W64: v_mad_co_i64_i32 v[5:6], s[12:13], null, exec_lo, null ; encoding: [0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01]
-0x05,0x0c,0xff,0xd6,0x7c,0xfc,0xf0,0x01
 
+0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03
 # W32: v_mad_co_i64_i32 v[5:6], s104, -1, exec_hi, -1 ; encoding: [0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03]
 # W64: v_mad_co_i64_i32 v[5:6], s[104:105], -1, exec_hi, -1 ; encoding: [0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03]
-0x05,0x68,0xff,0xd6,0xc1,0xfe,0x04,0x03
 
+0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 # W32: v_mad_co_i64_i32 v[5:6], vcc_lo, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_mad_co_i64_i32 v[5:6], vcc, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xff,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03
 # W32: v_mad_co_i64_i32 v[5:6], ttmp14, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03]
 # W64: v_mad_co_i64_i32 v[5:6], ttmp[14:15], src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03]
-0x05,0x7a,0xff,0xd6,0xfd,0xd4,0xf4,0x03
 
-# GFX12: v_mad_co_i64_i32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_co_i64_i32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xff,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_mad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x41,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_mad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_mad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x41,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_mad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_mad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x41,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_mad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_mad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x41,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_mad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_mad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x41,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_mad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x41,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_u16 v5, m0, 0x3800, m0            ; encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_mad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_mad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_mad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x41,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_mad_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x41,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_mad_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_mad_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc0,0x41,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04
+# GFX12: v_mad_u32_u16 v5, v1, v2, v3            ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x05,0x0e,0x04]
 
-# GFX12: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00
+# GFX12: v_mad_u32_u16 v5, v255, v255, s3        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xff,0x0f,0x00]
 
-# GFX12: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
 0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07
+# GFX12: v_mad_u32_u16 v5, s1, s2, v255          ; encoding: [0x05,0x00,0x59,0xd6,0x01,0x04,0xfc,0x07]
 
-# GFX12: v_mad_u32_u16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01]
 0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01
+# GFX12: v_mad_u32_u16 v5, s105, s105, s105      ; encoding: [0x05,0x00,0x59,0xd6,0x69,0xd2,0xa4,0x01]
 
-# GFX12: v_mad_u32_u16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01]
 0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01
+# GFX12: v_mad_u32_u16 v5, vcc_lo, ttmp15, vcc_lo ; encoding: [0x05,0x00,0x59,0xd6,0x6a,0xf6,0xa8,0x01]
 
-# GFX12: v_mad_u32_u16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_u32_u16 v5, vcc_hi, 0xfe0b, vcc_hi ; encoding: [0x05,0x00,0x59,0xd6,0x6b,0xfe,0xad,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_u32_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_u32_u16 v5, m0, 0x3800, m0        ; encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_u32_u16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
 0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01
+# GFX12: v_mad_u32_u16 v5, exec_lo, -1, exec_hi  ; encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
 
-# GFX12: v_mad_u32_u16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01]
 0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01
+# GFX12: v_mad_u32_u16 v5, exec_hi, null, exec_lo ; encoding: [0x05,0x00,0x59,0xd6,0x7f,0xf8,0xf8,0x01]
 
-# GFX12: v_mad_u32_u16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01]
 0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01
+# GFX12: v_mad_u32_u16 v5, null, exec_lo, null   ; encoding: [0x05,0x00,0x59,0xd6,0x7c,0xfc,0xf0,0x01]
 
-# GFX12: v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456 ; encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_u32_u16 v5, 0x3800, m0, -1
 0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03
+# GFX12: v_mad_u32_u16 v5, 0x3800, m0, -1        ; encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
 0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03
+# GFX12: v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
 
-# GFX12: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_mad_u32_u16 v255, 0xfe0b, vcc_hi, 0.5 op_sel:[0,1,0,0] clamp ; encoding: [0xff,0x90,0x59,0xd6,0xff,0xd6,0xc0,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_mad_u32_u24 v5, v1, v2, s3            ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_mad_u32_u24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_mad_u32_u24 v5, v255, s2, s105        ; encoding: [0x05,0x00,0x0b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_mad_u32_u24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_mad_u32_u24 v5, s1, v255, exec_hi     ; encoding: [0x05,0x00,0x0b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_mad_u32_u24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_mad_u32_u24 v5, s105, s105, exec_lo   ; encoding: [0x05,0x00,0x0b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_mad_u32_u24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_mad_u32_u24 v5, vcc_lo, ttmp15, v3    ; encoding: [0x05,0x00,0x0b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_mad_u32_u24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_u32_u24 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x0b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_u32_u24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_mad_u32_u24 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x0b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_mad_u32_u24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mad_u32_u24 v5, m0, 0.5, m0           ; encoding: [0x05,0x00,0x0b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_mad_u32_u24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_mad_u32_u24 v5, exec_lo, -1, vcc_hi   ; encoding: [0x05,0x00,0x0b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_mad_u32_u24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_mad_u32_u24 v5, exec_hi, null, vcc_lo ; encoding: [0x05,0x00,0x0b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_mad_u32_u24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_u32_u24 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x0b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mad_u32_u24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_mad_u32_u24 v5, -1, exec_hi, src_scc  ; encoding: [0x05,0x00,0x0b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_mad_u32_u24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_mad_u32_u24 v5, 0.5, m0, 0.5          ; encoding: [0x05,0x00,0x0b,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_mad_u32_u24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_mad_u32_u24 v5, src_scc, vcc_lo, -1   ; encoding: [0x05,0x00,0x0b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_mad_u32_u24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_u32_u24 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x0b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00
 # W32: v_mad_co_u64_u32 v[5:6], s12, v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], v1, s2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0x01,0x05,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00
 # W32: v_mad_co_u64_u32 v[5:6], s12, v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], v255, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0xff,0xf7,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00
 # W32: v_mad_co_u64_u32 v[5:6], s12, s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], s1, v2, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00]
-0x05,0x0c,0xfe,0xd6,0x01,0x04,0x1a,0x00
 
+0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00
 # W32: v_mad_co_u64_u32 v[5:6], s12, s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], s105, s105, s[6:7] ; encoding: [0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00]
-0x05,0x0c,0xfe,0xd6,0x69,0xd2,0x18,0x00
 
+0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01
 # W32: v_mad_co_u64_u32 v[5:6], s12, vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], vcc_lo, v255, s[104:105] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01]
-0x05,0x0c,0xfe,0xd6,0x6a,0xfe,0xa3,0x01
 
+0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 # W32: v_mad_co_u64_u32 v[5:6], s12, vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], vcc_hi, 0xaf123456, v[3:4] ; encoding: [0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0xfe,0xd6,0x6b,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
 
+0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07
 # W32: v_mad_co_u64_u32 v[5:6], s12, ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], ttmp15, src_scc, v[254:255] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07]
-0x05,0x0c,0xfe,0xd6,0x7b,0xfa,0xf9,0x07
 
+0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01
 # W32: v_mad_co_u64_u32 v[5:6], s12, m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], m0, 0.5, ttmp[14:15] ; encoding: [0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01]
-0x05,0x0c,0xfe,0xd6,0x7d,0xe0,0xe9,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01
 # W32: v_mad_co_u64_u32 v[5:6], s12, exec_lo, -1, exec ; encoding: [0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], exec_lo, -1, exec ; encoding: [0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01]
-0x05,0x0c,0xfe,0xd6,0x7e,0x82,0xf9,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01
 # W32: v_mad_co_u64_u32 v[5:6], s12, exec_hi, null, vcc ; encoding: [0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], exec_hi, null, vcc ; encoding: [0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01]
-0x05,0x0c,0xfe,0xd6,0x7f,0xf8,0xa8,0x01
 
+0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01
 # W32: v_mad_co_u64_u32 v[5:6], s12, null, exec_lo, null ; encoding: [0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01]
 # W64: v_mad_co_u64_u32 v[5:6], s[12:13], null, exec_lo, null ; encoding: [0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01]
-0x05,0x0c,0xfe,0xd6,0x7c,0xfc,0xf0,0x01
 
+0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03
 # W32: v_mad_co_u64_u32 v[5:6], s104, -1, exec_hi, -1 ; encoding: [0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03]
 # W64: v_mad_co_u64_u32 v[5:6], s[104:105], -1, exec_hi, -1 ; encoding: [0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03]
-0x05,0x68,0xfe,0xd6,0xc1,0xfe,0x04,0x03
 
+0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 # W32: v_mad_co_u64_u32 v[5:6], vcc_lo, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
 # W64: v_mad_co_u64_u32 v[5:6], vcc, 0.5, m0, 0xaf123456 ; encoding: [0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf]
-0x05,0x6a,0xfe,0xd6,0xf0,0xfa,0xfc,0x03,0x56,0x34,0x12,0xaf
 
+0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03
 # W32: v_mad_co_u64_u32 v[5:6], ttmp14, src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03]
 # W64: v_mad_co_u64_u32 v[5:6], ttmp[14:15], src_scc, vcc_lo, src_scc ; encoding: [0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03]
-0x05,0x7a,0xfe,0xd6,0xfd,0xd4,0xf4,0x03
 
-# GFX12: v_mad_co_u64_u32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mad_co_u64_u32 v[254:255], null, 0xaf123456, vcc_hi, 0.5 clamp ; encoding: [0xfe,0xfc,0xfe,0xd6,0xff,0xd6,0xc0,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2c,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2c,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2c,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x2c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x2c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x2c,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_max3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_max3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_max3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2c,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_max3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_max3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x2c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x2c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x2c,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x2c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x2c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_max3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x2c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_max3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x2c,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x2c,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_max3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x2c,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_max3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x2c,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x2c,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_max3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x2c,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_max3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x2c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x2c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x2c,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2a,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2a,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2a,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2a,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2a,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_max3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2a,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_max3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_max3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2a,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2a,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2a,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_max3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2a,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_max3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2a,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x2a,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2a,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2a,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x2a,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_max3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2a,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_max3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2a,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x2a,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_max3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2a,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_max3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2a,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x2a,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_max3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2a,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_max3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2a,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x2a,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2a,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4d,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_max3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_max3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_max3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_max3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4d,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_max3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4d,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_max3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_max3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_max3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4d,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_max3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1d,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_max3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_max3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_max3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1d,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_max3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1d,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_max3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1d,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_max3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_max3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1d,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_max3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_max3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1d,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_max3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1d,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4e,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_max3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_max3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_max3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_max3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4e,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_max3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4e,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_max3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_max3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_max3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_max3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4e,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_max3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_max3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_max3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_max3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_max3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_max3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_max3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_max3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_max3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_max3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_max3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_max3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1e,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_max3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_max3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_max3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_max3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_max3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_max3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1e,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_max3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1e,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_max3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_max3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1e,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_max3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_max3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1e,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_max3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_max3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1e,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_max3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_max3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1e,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_max_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_max_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_max_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_max_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_max_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_max_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_max_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_max_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_max_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_max_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_max_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0a,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0a,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0a,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0a,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_max_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_max_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_max_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_max_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_max_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_max_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_max_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x09,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_max_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_max_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_max_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_max_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_max_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_max_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_max_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x09,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_max_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_max_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_max_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x09,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x09,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_max_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_max_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_max_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x09,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_maxmin_num_f16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x6b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6b,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maxmin_num_f16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x6b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maxmin_num_f16 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x6b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6b,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maxmin_num_f16 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x6b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maxmin_num_f16 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x6b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6b,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maxmin_num_f16 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x6b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maxmin_num_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maxmin_num_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maxmin_num_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maxmin_num_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maxmin_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x6b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x6b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_maxmin_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x6b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_maxmin_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6b,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6b,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maxmin_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6b,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maxmin_num_f16 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x6b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maxmin_num_f16 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x6b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maxmin_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6b,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maxmin_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maxmin_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6b,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6b,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maxmin_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6b,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maxmin_num_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x6b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x6b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_maxmin_num_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x6b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_maxmin_num_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6b,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6b,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maxmin_num_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6b,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maxmin_num_f16 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6b,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x6b,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_maxmin_num_f16 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6b,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_maxmin_num_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6b,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x6b,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_maxmin_num_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6b,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_maxmin_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6b,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0x83,0x6b,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00
+# GFX12: v_maxmin_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6b,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_maxmin_num_f32 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x69,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x69,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maxmin_num_f32 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x69,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maxmin_num_f32 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x69,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x69,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maxmin_num_f32 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x69,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maxmin_num_f32 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x69,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x69,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maxmin_num_f32 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x69,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maxmin_num_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x69,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x69,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maxmin_num_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x69,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maxmin_num_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x69,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x69,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maxmin_num_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x69,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maxmin_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x69,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x69,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x69,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x69,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x69,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maxmin_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x69,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maxmin_num_f32 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x69,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x69,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maxmin_num_f32 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x69,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maxmin_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x69,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x69,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maxmin_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x69,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maxmin_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x69,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x69,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maxmin_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x69,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maxmin_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x69,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x69,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x69,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x69,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x69,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maxmin_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x69,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maxmin_num_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x69,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x69,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_maxmin_num_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x69,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_maxmin_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x69,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x69,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_maxmin_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x69,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_maxmin_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x69,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x69,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x69,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maxmin_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x64,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maxmin_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maxmin_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x64,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maxmin_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maxmin_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x64,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maxmin_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maxmin_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x64,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maxmin_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maxmin_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x64,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maxmin_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x64,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_maxmin_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x64,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_maxmin_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maxmin_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x64,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maxmin_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maxmin_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x64,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maxmin_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_maxmin_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x64,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_maxmin_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x64,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_maxmin_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x64,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_maxmin_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_maxmin_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x64,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_maxmin_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_maxmin_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x64,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_maxmin_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x64,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maxmin_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x62,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maxmin_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maxmin_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x62,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maxmin_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maxmin_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x62,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maxmin_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maxmin_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x62,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maxmin_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maxmin_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x62,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maxmin_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x62,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_maxmin_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x62,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_maxmin_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maxmin_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x62,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maxmin_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maxmin_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x62,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maxmin_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_maxmin_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x62,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_maxmin_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x62,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maxmin_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_maxmin_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x62,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_maxmin_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_maxmin_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x62,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_maxmin_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_maxmin_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x62,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_maxmin_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_maxmin_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x62,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x20,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x20,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x20,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x20,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mbcnt_hi_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x20,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x20,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x20,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x20,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x20,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x20,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x20,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x20,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mbcnt_hi_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x20,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, v1, v2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, v255, v255       ; encoding: [0x05,0x00,0x1f,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, s1, s2           ; encoding: [0x05,0x00,0x1f,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, s105, s105       ; encoding: [0x05,0x00,0x1f,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, vcc_lo, ttmp15   ; encoding: [0x05,0x00,0x1f,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mbcnt_lo_u32_b32 v5, vcc_hi, 0xaf123456 ; encoding: [0x05,0x00,0x1f,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, ttmp15, src_scc  ; encoding: [0x05,0x00,0x1f,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, m0, 0.5          ; encoding: [0x05,0x00,0x1f,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, exec_lo, -1      ; encoding: [0x05,0x00,0x1f,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, exec_hi, null    ; encoding: [0x05,0x00,0x1f,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, null, exec_lo    ; encoding: [0x05,0x00,0x1f,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, -1, exec_hi      ; encoding: [0x05,0x00,0x1f,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, 0.5, m0          ; encoding: [0x05,0x00,0x1f,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32 v5, src_scc, vcc_lo  ; encoding: [0x05,0x00,0x1f,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mbcnt_lo_u32_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1f,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x32,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x32,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x32,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x32,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x32,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x32,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x32,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x32,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x32,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x32,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x32,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x32,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x32,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x32,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x32,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x32,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x32,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x32,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x32,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x32,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_med3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x32,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_med3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x32,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x32,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x32,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_med3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x32,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x32,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x32,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x32,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x32,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_med3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x32,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_med3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x32,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x32,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x32,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x32,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x32,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_med3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x32,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_med3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x32,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x32,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_med3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x32,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_med3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x32,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x32,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_med3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x32,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_med3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x32,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x32,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x32,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x31,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x31,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x31,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x31,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x31,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x31,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x31,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x31,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x31,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x31,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x31,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x31,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x31,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x31,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x31,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x31,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x31,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x31,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x31,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x31,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_med3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x31,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_med3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x31,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x31,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x31,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_med3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x31,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x31,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x31,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x31,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x31,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_med3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x31,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_med3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x31,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x31,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x31,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x31,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x31,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_med3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x31,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_med3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x31,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x31,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_med3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x31,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_med3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x31,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x31,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_med3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x31,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_med3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x31,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x31,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x31,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x50,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x50,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x50,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x50,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x50,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x50,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_med3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_med3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_med3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_med3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x50,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_med3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x50,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_med3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_med3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_med3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x50,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x20,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x20,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x20,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x20,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x20,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x20,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_med3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x20,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_med3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x20,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_med3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x20,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_med3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x20,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_med3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x20,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_med3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x20,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_med3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_med3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x20,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_med3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_med3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x20,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_med3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x20,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x51,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x51,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x51,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x51,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x51,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x51,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_med3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_med3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_med3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_med3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x51,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_med3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x51,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_med3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_med3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_med3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_med3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x51,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_med3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_med3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x21,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_med3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_med3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x21,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_med3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_med3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x21,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_med3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_med3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x21,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_med3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_med3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x21,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_med3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x21,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_med3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x21,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_med3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_med3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x21,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_med3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_med3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x21,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_med3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_med3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x21,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_med3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x21,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_med3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_med3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x21,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_med3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_med3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x21,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_med3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_med3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x21,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_med3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_med3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x21,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2b,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_num_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2b,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_num_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2b,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_num_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_num_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_num_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x2b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x2b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x2b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2b,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2b,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_min3_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2b,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_min3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_num_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_min3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2b,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2b,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2b,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_min3_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2b,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_min3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x2b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x7c,0x2b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_num_f16 v5, null, exec_lo, -|0xfe0b| op_sel:[1,1,1,1] ; encoding: [0x05,0x7c,0x2b,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x2b,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x0e,0x2b,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_min3_num_f16 v5, -1, -|exec_hi|, -|src_scc| op_sel:[1,0,0,0] ; encoding: [0x05,0x0e,0x2b,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_min3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x2b,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x10,0x2b,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_min3_num_f16 v5, 0.5, -m0, 0.5 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x2b,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_min3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x2b,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x22,0x2b,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_min3_num_f16 v5, -src_scc, |vcc_lo|, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x22,0x2b,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_min3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x2b,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 0xff,0xc3,0x2b,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null op_sel:[0,0,0,1] clamp ; encoding: [0xff,0xc3,0x2b,0xd6,0xff,0xd6,0xf0,0x61,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x29,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x29,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_num_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x29,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x29,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x29,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_num_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x29,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x29,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x29,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_num_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x29,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x29,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x29,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_num_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x29,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x29,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x29,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_num_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x29,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x29,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x29,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x29,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x29,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x29,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_min3_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x29,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_min3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x29,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x29,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_num_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x29,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_min3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x29,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x29,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x29,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x29,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x29,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_min3_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x29,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_min3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x29,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x29,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x29,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x29,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x29,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_min3_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x29,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_min3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x29,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x29,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_min3_num_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x29,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_min3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x29,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x29,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_min3_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x29,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_min3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x29,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x29,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x29,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_i16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_i16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_i16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_i16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_i16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_i16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_min3_i16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_min3_i16 v5, m0, 0x3800, m0
 0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_i16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_min3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_i16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_min3_i16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_min3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_i16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4a,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_min3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_min3_i16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_min3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_i16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4a,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_i32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_i32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_i32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_i32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_i32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1a,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_min3_i32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1a,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_min3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_i32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_min3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_i32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_min3_i32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1a,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_min3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1a,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_min3_i32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1a,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_min3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_min3_i32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1a,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_min3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_min3_i32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1a,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_min3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1a,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_u16 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_u16 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x4b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_u16 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x4b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_u16 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x4b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_u16 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x4b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_u16 v5, vcc_hi, 0xfe0b, v255     ; encoding: [0x05,0x00,0x4b,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_min3_u16 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_min3_u16 v5, m0, 0x3800, m0
 0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_u16 v5, m0, 0x3800, m0           ; encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
 
-# GFX12: v_min3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_u16 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_min3_u16 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x4b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_min3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_u16 v5, null, exec_lo, 0xfe0b op_sel:[1,1,1,1] ; encoding: [0x05,0x78,0x4b,0xd6,0x7c,0xfc,0xfc,0x03,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_min3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0]
 0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_min3_u16 v5, 0x3800, m0, 0x3800 op_sel:[0,1,0,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
 
-# GFX12: v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_min3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_min3_u16 v255, 0xfe0b, vcc_hi, null op_sel:[0,0,0,1] ; encoding: [0xff,0x40,0x4b,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_min3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_min3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_min3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_min3_u32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x1b,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_min3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_min3_u32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x1b,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_min3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_min3_u32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x1b,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_min3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_min3_u32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x1b,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_min3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x1b,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_min3_u32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x1b,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_min3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_min3_u32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x1b,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_min3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_min3_u32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x1b,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_min3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_min3_u32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x1b,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_min3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x1b,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_min3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_min3_u32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x1b,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_min3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_min3_u32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x1b,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_min3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_min3_u32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x1b,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_min3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_min3_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x1b,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_i16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_i16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, s105, s105                ; encoding: [0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_min_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_min_i16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_min_i16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_min_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_min_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_min_i16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_min_i16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_min_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_min_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_min_i16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_min_i16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_i16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0c,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_i16 v5, v255, v255                ; encoding: [0x05,0x00,0x0c,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0c,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_i16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0c,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_u16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_u16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, s1, s2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, s105, s105                ; encoding: [0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_min_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_min_u16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_min_u16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_min_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_min_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_min_u16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_min_u16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_min_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_min_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_min_u16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_min_u16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_min_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_min_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_min_u16 v5, v1, v2                    ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0b,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_min_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_min_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_min_u16 v5, v255, v255                ; encoding: [0x05,0x00,0x0b,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0b,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_min_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_min_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_min_u16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x0b,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_minmax_num_f16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x6a,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6a,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minmax_num_f16 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x6a,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minmax_num_f16 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x6a,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6a,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minmax_num_f16 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x6a,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minmax_num_f16 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x6a,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6a,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minmax_num_f16 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x6a,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minmax_num_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6a,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6a,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minmax_num_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6a,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minmax_num_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6a,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6a,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minmax_num_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6a,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minmax_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x6a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x6a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_minmax_num_f16 v5, vcc_hi, 0xfe0b, v255 ; encoding: [0x05,0x00,0x6a,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_minmax_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6a,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6a,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minmax_num_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6a,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minmax_num_f16 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x6a,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6a,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minmax_num_f16 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x6a,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minmax_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6a,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6a,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minmax_num_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6a,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minmax_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6a,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6a,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minmax_num_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6a,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minmax_num_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x6a,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 0x05,0x04,0x6a,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00
+# GFX12: v_minmax_num_f16 v5, null, exec_lo, -|0xfe0b| ; encoding: [0x05,0x04,0x6a,0xd6,0x7c,0xfc,0xfc,0x83,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_minmax_num_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6a,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6a,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minmax_num_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6a,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minmax_num_f16 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6a,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x6a,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_minmax_num_f16 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6a,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_minmax_num_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6a,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x6a,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_minmax_num_f16 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6a,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_minmax_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6a,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 0xff,0x83,0x6a,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00
+# GFX12: v_minmax_num_f16 v255, -|0xfe0b|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6a,0xd6,0xff,0xd6,0xf0,0x79,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_minmax_num_f32 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x68,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x68,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minmax_num_f32 v5, v1, v2, s3         ; encoding: [0x05,0x00,0x68,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minmax_num_f32 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x68,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x68,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minmax_num_f32 v5, v255, s2, s105     ; encoding: [0x05,0x00,0x68,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minmax_num_f32 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x68,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x68,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minmax_num_f32 v5, s1, v255, exec_hi  ; encoding: [0x05,0x00,0x68,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minmax_num_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x68,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x68,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minmax_num_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x68,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minmax_num_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x68,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x68,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minmax_num_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x68,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minmax_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x68,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x68,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_num_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x68,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x68,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x68,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minmax_num_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x68,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minmax_num_f32 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x68,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x68,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minmax_num_f32 v5, m0, 0.5, m0        ; encoding: [0x05,0x00,0x68,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minmax_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x68,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x68,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minmax_num_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x68,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minmax_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x68,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x68,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minmax_num_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x68,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minmax_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x68,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x68,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_num_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x68,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x68,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x68,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minmax_num_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x68,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minmax_num_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x68,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x68,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_minmax_num_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x68,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_minmax_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x68,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x68,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_minmax_num_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x68,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_minmax_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x68,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x68,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_num_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x68,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minmax_i32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x65,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minmax_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minmax_i32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x65,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minmax_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minmax_i32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x65,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minmax_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minmax_i32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x65,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minmax_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minmax_i32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x65,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minmax_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_i32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x65,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_minmax_i32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x65,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_minmax_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minmax_i32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x65,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minmax_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minmax_i32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x65,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minmax_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_minmax_i32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x65,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_minmax_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_i32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x65,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_minmax_i32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x65,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_minmax_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_minmax_i32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x65,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_minmax_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_minmax_i32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x65,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_minmax_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_i32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x65,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minmax_u32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x63,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minmax_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minmax_u32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x63,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minmax_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minmax_u32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x63,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minmax_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minmax_u32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x63,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minmax_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minmax_u32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x63,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minmax_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_u32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x63,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_minmax_u32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x63,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_minmax_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minmax_u32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x63,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minmax_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minmax_u32 v5, exec_lo, -1, vcc_hi    ; encoding: [0x05,0x00,0x63,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minmax_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_minmax_u32 v5, exec_hi, null, vcc_lo  ; encoding: [0x05,0x00,0x63,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_minmax_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x63,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minmax_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_minmax_u32 v5, -1, exec_hi, src_scc   ; encoding: [0x05,0x00,0x63,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_minmax_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_minmax_u32 v5, 0.5, m0, 0.5           ; encoding: [0x05,0x00,0x63,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_minmax_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_minmax_u32 v5, src_scc, vcc_lo, -1    ; encoding: [0x05,0x00,0x63,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_minmax_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_minmax_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x63,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xea,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xff,0xeb,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0x05,0xe8,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01]
 0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3b,0xd6,0x01,0xd3,0xe8,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00]
 0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3b,0xd6,0xfe,0xf7,0x18,0x00]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04]
 0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3b,0xd6,0x02,0xd6,0x0c,0x04]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01]
 0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3b,0xd6,0x68,0xd4,0xa0,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07]
 0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3b,0xd6,0x6a,0xfa,0xf8,0x07]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01]
 0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3b,0xd6,0x7a,0xfe,0xf0,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01]
 0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3b,0xd6,0x7e,0xfc,0xf8,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01]
 0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3b,0xd6,0x7c,0xf8,0xa8,0x01]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3b,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03]
 0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3b,0xd6,0xf0,0xe0,0xf5,0x03]
 
-# GFX12: v_mqsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03]
 0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03
+# GFX12: v_mqsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3b,0xd6,0xfd,0xfa,0xc1,0x03]
 
-# GFX12: v_mqsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_mqsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3b,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], v2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], v2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf2,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], v255, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], v255, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xff,0xf3,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], s2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], s2, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0x05,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], s105, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], v[1:2], s105, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x01,0xd3,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], v[254:255], ttmp15, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], v[254:255], ttmp15, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfe,0xf7,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], s[2:3], vcc_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], s[2:3], vcc_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x02,0xd6,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], s[104:105], vcc_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], s[104:105], vcc_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x68,0xd4,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], vcc, m0, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], vcc, m0, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x6a,0xfa,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], ttmp[14:15], exec_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], ttmp[14:15], exec_hi, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7a,0xfe,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], exec, exec_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], exec, exec_lo, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7e,0xfc,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], null, null, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07]
 0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], null, null, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0x7c,0xf8,0xf0,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], -1, -1, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], -1, -1, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xc1,0x82,0xf1,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], 0.5, 0.5, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], 0.5, 0.5, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xf0,0xe0,0xf1,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[5:8], src_scc, src_scc, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07]
 0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07
+# GFX12: v_mqsad_u32_u8 v[5:8], src_scc, src_scc, v[252:255] ; encoding: [0x05,0x00,0x3d,0xd6,0xfd,0xfa,0xf1,0x07]
 
-# GFX12: v_mqsad_u32_u8 v[252:255], 0xaf123456, 0xaf123456, v[3:6] clamp ; encoding: [0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf
+# GFX12: v_mqsad_u32_u8 v[252:255], 0xaf123456, 0xaf123456, v[3:6] clamp ; encoding: [0xfc,0x80,0x3d,0xd6,0xff,0xfe,0x0d,0x04,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_msad_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_msad_u8 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x39,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_msad_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_msad_u8 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x39,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_msad_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_msad_u8 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x39,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_msad_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_msad_u8 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x39,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_msad_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_msad_u8 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x39,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_msad_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_msad_u8 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x39,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_msad_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_msad_u8 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x39,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_msad_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_msad_u8 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x39,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_msad_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_msad_u8 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x39,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_msad_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_msad_u8 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x39,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_msad_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_msad_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x39,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_msad_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_msad_u8 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x39,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_msad_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_msad_u8 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x39,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_msad_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_msad_u8 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x39,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_msad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_msad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x39,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_mul_hi_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_mul_hi_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_mul_hi_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x2e,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_mul_hi_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2e,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x2e,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2e,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2e,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_mul_hi_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2e,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_mul_hi_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_mul_hi_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2e,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_mul_hi_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_mul_hi_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2e,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_mul_hi_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2e,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2e,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2e,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2e,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_mul_hi_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2e,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_mul_hi_i32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_i32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2e,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_mul_hi_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_mul_hi_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_mul_hi_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2d,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_mul_hi_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2d,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2d,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2d,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2d,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_hi_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_mul_hi_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2d,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_mul_hi_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_mul_hi_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2d,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_mul_hi_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_mul_hi_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2d,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_mul_hi_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2d,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2d,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2d,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2d,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_mul_hi_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2d,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_mul_hi_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_hi_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2d,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x05,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, -1, exec_hi          ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, 0x3800, m0           ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, 0x3800, m0             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, src_scc, vcc_lo      ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v255.l, 0xfe0b, vcc_hi     ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v255.l, 0xfe0b, vcc_hi     ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x05,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x05,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_mul_lo_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_mul_lo_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x05,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x05,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_mul_lo_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] ; encoding: [0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16 v255, 0xfe0b, vcc_hi       ; encoding: [0xff,0x00,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x40,0x05,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_mul_lo_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_mul_lo_u32 v5, v1, v2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_mul_lo_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_mul_lo_u32 v5, v255, v255             ; encoding: [0x05,0x00,0x2c,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_mul_lo_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, s1, s2                 ; encoding: [0x05,0x00,0x2c,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, s105, s105             ; encoding: [0x05,0x00,0x2c,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x2c,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_lo_u32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x2c,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mul_lo_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_mul_lo_u32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x2c,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_mul_lo_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_mul_lo_u32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x2c,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_mul_lo_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_mul_lo_u32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x2c,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_mul_lo_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x2c,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x2c,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x2c,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x2c,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_mul_lo_u32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x2c,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_mul_lo_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_mul_lo_u32 v255, 0xaf123456, vcc_hi   ; encoding: [0xff,0x00,0x2c,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mullit_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_mullit_f32 v5, v1, v2, s3             ; encoding: [0x05,0x00,0x18,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_mullit_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_mullit_f32 v5, v255, s2, s105         ; encoding: [0x05,0x00,0x18,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_mullit_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_mullit_f32 v5, s1, v255, exec_hi      ; encoding: [0x05,0x00,0x18,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_mullit_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_mullit_f32 v5, s105, s105, exec_lo    ; encoding: [0x05,0x00,0x18,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_mullit_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_mullit_f32 v5, vcc_lo, ttmp15, v3     ; encoding: [0x05,0x00,0x18,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_mullit_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_mullit_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x18,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mullit_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_mullit_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x18,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_mullit_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_mullit_f32 v5, m0, 0.5, m0            ; encoding: [0x05,0x00,0x18,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_mullit_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_mullit_f32 v5, |exec_lo|, -1, vcc_hi  ; encoding: [0x05,0x01,0x18,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_mullit_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_mullit_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x18,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_mullit_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_mullit_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x18,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_mullit_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_mullit_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x18,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_mullit_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_mullit_f32 v5, 0.5, -m0, 0.5 mul:2    ; encoding: [0x05,0x00,0x18,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_mullit_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_mullit_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x18,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_mullit_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_mullit_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x18,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_nop                                   ; encoding: [0x00,0x00,0x80,0xd5,0x00,0x00,0x00,0x00]
 0x00,0x00,0x80,0xd5,0x00,0x00,0x00,0x00
+# GFX12: v_nop                                   ; encoding: [0x00,0x00,0x80,0xd5,0x00,0x00,0x00,0x00]
 
-# GFX12: v_or3_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_or3_b32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x58,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_or3_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_or3_b32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x58,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_or3_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_or3_b32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x58,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_or3_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_or3_b32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x58,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_or3_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_or3_b32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x58,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_or3_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_or3_b32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x58,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_or3_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_or3_b32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x58,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_or3_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_or3_b32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x58,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_or3_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_or3_b32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x58,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_or3_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_or3_b32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x58,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_or3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_or3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x58,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_or3_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_or3_b32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x58,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_or3_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_or3_b32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x58,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_or3_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_or3_b32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x58,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_or3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_or3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x58,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
 
+0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_or_b16 v5.l, v1.l, v2.l               ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_or_b16 v5, v1, v2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_or_b16 v5.l, v1.l, v2.l               ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_or_b16 v5, v1, v2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x63,0xd7,0x01,0x05,0x02,0x00
 
 
+0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_or_b16 v5.l, v255.l, v255.l           ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_or_b16 v5, v255, v255                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_or_b16 v5.l, v255.l, v255.l           ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_or_b16 v5, v255, v255                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x63,0xd7,0xff,0xff,0x03,0x00
 
 
+0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, s1, s2                   ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, s1, s2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, s1, s2                   ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, s1, s2                     ; encoding: [0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x01,0x04,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, s105, s105               ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, s105, s105                 ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, s105, s105               ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, s105, s105                 ; encoding: [0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x69,0xd2,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, vcc_lo, ttmp15           ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, vcc_lo, ttmp15             ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, vcc_lo, ttmp15           ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, vcc_lo, ttmp15             ; encoding: [0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x6a,0xf6,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, vcc_hi, 0xfe0b           ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, vcc_hi, 0xfe0b             ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, vcc_hi, 0xfe0b           ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, vcc_hi, 0xfe0b             ; encoding: [0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, ttmp15, src_scc          ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_or_b16 v5, ttmp15, src_scc            ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_or_b16 v5.l, ttmp15, src_scc          ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_or_b16 v5, ttmp15, src_scc            ; encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, m0, 0x3800               ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, m0, 0x3800                 ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, m0, 0x3800               ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, m0, 0x3800                 ; encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00
 
 
+0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_or_b16 v5.l, exec_lo, -1              ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_or_b16 v5, exec_lo, -1                ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_or_b16 v5.l, exec_lo, -1              ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_or_b16 v5, exec_lo, -1                ; encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00
 
 
+0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, exec_hi, null            ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, exec_hi, null              ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, exec_hi, null            ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, exec_hi, null              ; encoding: [0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7f,0xf8,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, null, exec_lo            ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, null, exec_lo              ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, null, exec_lo            ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, null, exec_lo              ; encoding: [0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x63,0xd7,0x7c,0xfc,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, -1, exec_hi              ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, -1, exec_hi                ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, -1, exec_hi              ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, -1, exec_hi                ; encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, 0x3800, m0               ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, 0x3800, m0                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, 0x3800, m0               ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, 0x3800, m0                 ; encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00
 
 
+0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_or_b16 v5.l, src_scc, vcc_lo          ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_or_b16 v5, src_scc, vcc_lo            ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_or_b16 v5.l, src_scc, vcc_lo          ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_or_b16 v5, src_scc, vcc_lo            ; encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00
 
 
+0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_or_b16 v255.l, 0xfe0b, vcc_hi         ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_or_b16 v255, 0xfe0b, vcc_hi           ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_or_b16 v255.l, 0xfe0b, vcc_hi         ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_or_b16 v255, 0xfe0b, vcc_hi           ; encoding: [0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x63,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_pack_b32_f16 v5, v1, v2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_pack_b32_f16 v5, v1, v2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_pack_b32_f16 v5, v255, v255           ; encoding: [0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_pack_b32_f16 v5, v255, v255           ; encoding: [0x05,0x00,0x11,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_pack_b32_f16 v5, s1, s2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, s1, s2               ; encoding: [0x05,0x00,0x11,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, s105, s105           ; encoding: [0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, s105, s105           ; encoding: [0x05,0x00,0x11,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x11,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x11,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_pack_b32_f16 v5, ttmp15, src_scc      ; encoding: [0x05,0x00,0x11,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_pack_b32_f16 v5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_pack_b32_f16 v5, m0, 0.5              ; encoding: [0x05,0x00,0x11,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_pack_b32_f16 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_pack_b32_f16 v5, exec_lo, -1          ; encoding: [0x05,0x00,0x11,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_pack_b32_f16 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, |exec_hi|, null      ; encoding: [0x05,0x01,0x11,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, null, exec_lo        ; encoding: [0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, null, exec_lo        ; encoding: [0x05,0x00,0x11,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_pack_b32_f16 v5, -1, exec_hi          ; encoding: [0x05,0x00,0x11,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_pack_b32_f16 v5, 0.5, -m0             ; encoding: [0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_pack_b32_f16 v5, 0.5, -m0             ; encoding: [0x05,0x00,0x11,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_pack_b32_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_pack_b32_f16 v5, -src_scc, |vcc_lo| op_sel:[1,0,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_pack_b32_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX12: v_pack_b32_f16 v255, -|0xfe0b|, -|vcc_hi| op_sel:[0,1,0] ; encoding: [0xff,0x13,0x11,0xd7,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_perm_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_perm_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x44,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_perm_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_perm_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x44,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_perm_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_perm_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x44,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_perm_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_perm_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x44,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_perm_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_perm_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x44,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_perm_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_perm_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x44,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_perm_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_perm_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x44,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_perm_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_perm_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x44,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_perm_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_perm_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x44,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_perm_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_perm_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x44,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_perm_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_perm_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x44,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_perm_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_perm_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x44,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_perm_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_perm_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x44,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_perm_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_perm_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x44,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_perm_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_perm_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x44,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_permlane16_b32 v5, v1, s2, s3         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlane16_b32 v5, v1, s2, s3         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlane16_b32 v5, v1, s105, s105     ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01
+# GFX12: v_permlane16_b32 v5, v1, s105, s105     ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd3,0xa4,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01
+# GFX12: v_permlane16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf7,0xec,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01
+# GFX12: v_permlane16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd7,0xfc,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01
+# GFX12: v_permlane16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xd5,0xf8,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, m0, m0         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01
+# GFX12: v_permlane16_b32 v5, v1, m0, m0         ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0xf4,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01
+# GFX12: v_permlane16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xff,0xac,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01]
 0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01
+# GFX12: v_permlane16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfd,0xa8,0x01]
 
-# GFX12: v_permlane16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_permlane16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_permlane16_b32 v5, v1, -1, src_scc    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03
+# GFX12: v_permlane16_b32 v5, v1, -1, src_scc    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0x83,0xf5,0x03]
 
-# GFX12: v_permlane16_b32 v5, v1, 0.5, 0.5       ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03
+# GFX12: v_permlane16_b32 v5, v1, 0.5, 0.5       ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xe1,0xc1,0x03]
 
-# GFX12: v_permlane16_b32 v5, v1, src_scc, -1    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03]
 0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03
+# GFX12: v_permlane16_b32 v5, v1, src_scc, -1    ; encoding: [0x05,0x00,0x5b,0xd6,0x01,0xfb,0x05,0x03]
 
-# GFX12: v_permlane16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_permlane16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5b,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_permlanex16_b32 v5, v1, s2, s3        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlanex16_b32 v5, v1, s2, s3        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlane16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5b,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlanex16_b32 v5, v1, s105, s105    ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01
+# GFX12: v_permlanex16_b32 v5, v1, s105, s105    ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd3,0xa4,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01
+# GFX12: v_permlanex16_b32 v5, v1, ttmp15, ttmp15 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf7,0xec,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01
+# GFX12: v_permlanex16_b32 v5, v1, vcc_hi, exec_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd7,0xfc,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01
+# GFX12: v_permlanex16_b32 v5, v1, vcc_lo, exec_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xd5,0xf8,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, m0, m0        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01
+# GFX12: v_permlanex16_b32 v5, v1, m0, m0        ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0xf4,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01
+# GFX12: v_permlanex16_b32 v5, v1, exec_hi, vcc_hi ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xff,0xac,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01]
 0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01
+# GFX12: v_permlanex16_b32 v5, v1, exec_lo, vcc_lo ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfd,0xa8,0x01]
 
-# GFX12: v_permlanex16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_permlanex16_b32 v5, v1, null, 0xaf123456 ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xf9,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_permlanex16_b32 v5, v1, -1, src_scc   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03
+# GFX12: v_permlanex16_b32 v5, v1, -1, src_scc   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0x83,0xf5,0x03]
 
-# GFX12: v_permlanex16_b32 v5, v1, 0.5, 0.5      ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03
+# GFX12: v_permlanex16_b32 v5, v1, 0.5, 0.5      ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xe1,0xc1,0x03]
 
-# GFX12: v_permlanex16_b32 v5, v1, src_scc, -1   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03]
 0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03
+# GFX12: v_permlanex16_b32 v5, v1, src_scc, -1   ; encoding: [0x05,0x00,0x5c,0xd6,0x01,0xfb,0x05,0x03]
 
-# GFX12: v_permlanex16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_permlanex16_b32 v255, v255, 0xaf123456, null ; encoding: [0xff,0x00,0x5c,0xd6,0xff,0xff,0xf1,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,1] ; encoding: [0x05,0x18,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[1,0] ; encoding: [0x05,0x08,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00
+# GFX12: v_permlanex16_b32 v5, v1, s2, s3 op_sel:[0,1] ; encoding: [0x05,0x10,0x5c,0xd6,0x01,0x05,0x0c,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v1, v2         ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_permlane16_var_b32 v5, v1, v2         ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v1, v255       ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x0f,0xd7,0x01,0xff,0x03,0x00
+# GFX12: v_permlane16_var_b32 v5, v1, v255       ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v255, v0       ; encoding: [0x05,0x00,0x0f,0xd7,0xff,0x01,0x02,0x00]
 0x05,0x00,0x0f,0xd7,0xff,0x01,0x02,0x00
+# GFX12: v_permlane16_var_b32 v5, v255, v0       ; encoding: [0x05,0x00,0x0f,0xd7,0xff,0x01,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v255, v1, v2       ; encoding: [0xff,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00]
 0xff,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_permlane16_var_b32 v255, v1, v2       ; encoding: [0xff,0x00,0x0f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v1, v50 op_sel:[1,1] ; encoding: [0x05,0x18,0x0f,0xd7,0x01,0x65,0x02,0x00]
 0x05,0x18,0x0f,0xd7,0x01,0x65,0x02,0x00
+# GFX12: v_permlane16_var_b32 v5, v1, v50 op_sel:[1,1] ; encoding: [0x05,0x18,0x0f,0xd7,0x01,0x65,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v1, v50        ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0x65,0x02,0x00]
 0x05,0x00,0x0f,0xd7,0x01,0x65,0x02,0x00
+# GFX12: v_permlane16_var_b32 v5, v1, v50        ; encoding: [0x05,0x00,0x0f,0xd7,0x01,0x65,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v5, v1, v50 op_sel:[1,0] ; encoding: [0x05,0x08,0x0f,0xd7,0x01,0x65,0x02,0x00]
 0x05,0x08,0x0f,0xd7,0x01,0x65,0x02,0x00
+# GFX12: v_permlane16_var_b32 v5, v1, v50 op_sel:[1,0] ; encoding: [0x05,0x08,0x0f,0xd7,0x01,0x65,0x02,0x00]
 
-# GFX12: v_permlane16_var_b32 v255, v255, v0 op_sel:[0,1] ; encoding: [0xff,0x10,0x0f,0xd7,0xff,0x01,0x02,0x00]
 0xff,0x10,0x0f,0xd7,0xff,0x01,0x02,0x00
+# GFX12: v_permlane16_var_b32 v255, v255, v0 op_sel:[0,1] ; encoding: [0xff,0x10,0x0f,0xd7,0xff,0x01,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v2        ; encoding: [0x05,0x00,0x10,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x10,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v2        ; encoding: [0x05,0x00,0x10,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v105      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xd3,0x02,0x00]
 0x05,0x00,0x10,0xd7,0x01,0xd3,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v105      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xd3,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v255      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x10,0xd7,0x01,0xff,0x03,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v255      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX12: v_permlanex16_var_b32 v255, v1, v2      ; encoding: [0xff,0x00,0x10,0xd7,0x01,0x05,0x02,0x00]
 0xff,0x00,0x10,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v255, v1, v2      ; encoding: [0xff,0x00,0x10,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v1, v255, v2      ; encoding: [0x01,0x00,0x10,0xd7,0xff,0x05,0x02,0x00]
 0x01,0x00,0x10,0xd7,0xff,0x05,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v1, v255, v2      ; encoding: [0x01,0x00,0x10,0xd7,0xff,0x05,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v100 op_sel:[1,1] ; encoding: [0x05,0x18,0x10,0xd7,0x01,0xc9,0x02,0x00]
 0x05,0x18,0x10,0xd7,0x01,0xc9,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v100 op_sel:[1,1] ; encoding: [0x05,0x18,0x10,0xd7,0x01,0xc9,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v100      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xc9,0x02,0x00]
 0x05,0x00,0x10,0xd7,0x01,0xc9,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v100      ; encoding: [0x05,0x00,0x10,0xd7,0x01,0xc9,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v5, v1, v100 op_sel:[1,0] ; encoding: [0x05,0x08,0x10,0xd7,0x01,0xc9,0x02,0x00]
 0x05,0x08,0x10,0xd7,0x01,0xc9,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v5, v1, v100 op_sel:[1,0] ; encoding: [0x05,0x08,0x10,0xd7,0x01,0xc9,0x02,0x00]
 
-# GFX12: v_permlanex16_var_b32 v255, v255, v100 op_sel:[0,1] ; encoding: [0xff,0x10,0x10,0xd7,0xff,0xc9,0x02,0x00]
 0xff,0x10,0x10,0xd7,0xff,0xc9,0x02,0x00
+# GFX12: v_permlanex16_var_b32 v255, v255, v100 op_sel:[0,1] ; encoding: [0xff,0x10,0x10,0xd7,0xff,0xc9,0x02,0x00]
 
-# GFX12: v_pipeflush                             ; encoding: [0x00,0x00,0x9b,0xd5,0x00,0x00,0x00,0x00]
 0x00,0x00,0x9b,0xd5,0x00,0x00,0x00,0x00
+# GFX12: v_pipeflush                             ; encoding: [0x00,0x00,0x9b,0xd5,0x00,0x00,0x00,0x00]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], v2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xea,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], v255, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xff,0xeb,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], s2, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0x05,0xe8,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01]
 0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], v[1:2], s105, ttmp[14:15] ; encoding: [0x05,0x00,0x3a,0xd6,0x01,0xd3,0xe8,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00]
 0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00
+# GFX12: v_qsad_pk_u16_u8 v[5:6], v[254:255], ttmp15, s[6:7] ; encoding: [0x05,0x00,0x3a,0xd6,0xfe,0xf7,0x18,0x00]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04]
 0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04
+# GFX12: v_qsad_pk_u16_u8 v[5:6], s[2:3], vcc_hi, v[3:4] ; encoding: [0x05,0x00,0x3a,0xd6,0x02,0xd6,0x0c,0x04]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01]
 0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], s[104:105], vcc_lo, s[104:105] ; encoding: [0x05,0x00,0x3a,0xd6,0x68,0xd4,0xa0,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07]
 0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07
+# GFX12: v_qsad_pk_u16_u8 v[5:6], vcc, m0, v[254:255] ; encoding: [0x05,0x00,0x3a,0xd6,0x6a,0xfa,0xf8,0x07]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01]
 0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], ttmp[14:15], exec_hi, null ; encoding: [0x05,0x00,0x3a,0xd6,0x7a,0xfe,0xf0,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01]
 0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], exec, exec_lo, exec ; encoding: [0x05,0x00,0x3a,0xd6,0x7e,0xfc,0xf8,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01]
 0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01
+# GFX12: v_qsad_pk_u16_u8 v[5:6], null, null, vcc ; encoding: [0x05,0x00,0x3a,0xd6,0x7c,0xf8,0xa8,0x01]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_qsad_pk_u16_u8 v[5:6], -1, -1, 0xaf123456 ; encoding: [0x05,0x00,0x3a,0xd6,0xc1,0x82,0xfd,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03]
 0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03
+# GFX12: v_qsad_pk_u16_u8 v[5:6], 0.5, 0.5, src_scc ; encoding: [0x05,0x00,0x3a,0xd6,0xf0,0xe0,0xf5,0x03]
 
-# GFX12: v_qsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03]
 0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03
+# GFX12: v_qsad_pk_u16_u8 v[5:6], src_scc, src_scc, 0.5 ; encoding: [0x05,0x00,0x3a,0xd6,0xfd,0xfa,0xc1,0x03]
 
-# GFX12: v_qsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_qsad_pk_u16_u8 v[254:255], 0xaf123456, 0xaf123456, -1 clamp ; encoding: [0xfe,0x80,0x3a,0xd6,0xff,0xfe,0x05,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_readlane_b32 s5, v1, s2               ; encoding: [0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00
+# GFX12: v_readlane_b32 s5, v1, s2               ; encoding: [0x05,0x00,0x60,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX12: v_readlane_b32 s5, v1, s105             ; encoding: [0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00
+# GFX12: v_readlane_b32 s5, v1, s105             ; encoding: [0x05,0x00,0x60,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX12: v_readlane_b32 s105, v1, ttmp15         ; encoding: [0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00]
 0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00
+# GFX12: v_readlane_b32 s105, v1, ttmp15         ; encoding: [0x69,0x00,0x60,0xd7,0x01,0xf7,0x00,0x00]
 
-# GFX12: v_readlane_b32 vcc_lo, v1, vcc_hi       ; encoding: [0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00]
 0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00
+# GFX12: v_readlane_b32 vcc_lo, v1, vcc_hi       ; encoding: [0x6a,0x00,0x60,0xd7,0x01,0xd7,0x00,0x00]
 
-# GFX12: v_readlane_b32 vcc_hi, v1, vcc_lo       ; encoding: [0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00]
 0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00
+# GFX12: v_readlane_b32 vcc_hi, v1, vcc_lo       ; encoding: [0x6b,0x00,0x60,0xd7,0x01,0xd5,0x00,0x00]
 
-# GFX12: v_readlane_b32 ttmp15, v1, m0           ; encoding: [0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00]
 0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00
+# GFX12: v_readlane_b32 ttmp15, v1, m0           ; encoding: [0x7b,0x00,0x60,0xd7,0x01,0xfb,0x00,0x00]
 
-# GFX12: v_readlane_b32 null, v255, null         ; encoding: [0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00]
 0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00
+# GFX12: v_readlane_b32 null, v255, null         ; encoding: [0x7c,0x00,0x60,0xd7,0xff,0xf9,0x00,0x00]
 
-# GFX12: v_sad_hi_u8 v5, v1, v2, s3              ; encoding: [0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_sad_hi_u8 v5, v1, v2, s3              ; encoding: [0x05,0x00,0x23,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_sad_hi_u8 v5, v255, s2, s105          ; encoding: [0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_sad_hi_u8 v5, v255, s2, s105          ; encoding: [0x05,0x00,0x23,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_sad_hi_u8 v5, s1, v255, exec_hi       ; encoding: [0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_sad_hi_u8 v5, s1, v255, exec_hi       ; encoding: [0x05,0x00,0x23,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_sad_hi_u8 v5, s105, s105, exec_lo     ; encoding: [0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_sad_hi_u8 v5, s105, s105, exec_lo     ; encoding: [0x05,0x00,0x23,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_sad_hi_u8 v5, vcc_lo, ttmp15, v3      ; encoding: [0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_sad_hi_u8 v5, vcc_lo, ttmp15, v3      ; encoding: [0x05,0x00,0x23,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_sad_hi_u8 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_hi_u8 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x23,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_hi_u8 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_sad_hi_u8 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x23,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_sad_hi_u8 v5, m0, 0.5, m0             ; encoding: [0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_sad_hi_u8 v5, m0, 0.5, m0             ; encoding: [0x05,0x00,0x23,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_sad_hi_u8 v5, exec_lo, -1, vcc_hi     ; encoding: [0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_sad_hi_u8 v5, exec_lo, -1, vcc_hi     ; encoding: [0x05,0x00,0x23,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_sad_hi_u8 v5, exec_hi, null, vcc_lo   ; encoding: [0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_sad_hi_u8 v5, exec_hi, null, vcc_lo   ; encoding: [0x05,0x00,0x23,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_sad_hi_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_hi_u8 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x23,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_hi_u8 v5, -1, exec_hi, src_scc    ; encoding: [0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_sad_hi_u8 v5, -1, exec_hi, src_scc    ; encoding: [0x05,0x00,0x23,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_sad_hi_u8 v5, 0.5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_sad_hi_u8 v5, 0.5, m0, 0.5            ; encoding: [0x05,0x00,0x23,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_sad_hi_u8 v5, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_sad_hi_u8 v5, src_scc, vcc_lo, -1     ; encoding: [0x05,0x00,0x23,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_sad_hi_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_hi_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x23,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_sad_u16 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x24,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_sad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_sad_u16 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x24,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_sad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_sad_u16 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x24,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_sad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_sad_u16 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x24,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_sad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_sad_u16 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x24,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_sad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00
+# GFX12: v_sad_u16 v5, vcc_hi, 0xfe0b, v255      ; encoding: [0x05,0x00,0x24,0xd6,0x6b,0xfe,0xfd,0x07,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_sad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_sad_u16 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x24,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_sad_u16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_sad_u16 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x24,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_sad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_sad_u16 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x24,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_sad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_sad_u16 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x24,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_sad_u16 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u16 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x24,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u16 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_sad_u16 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x24,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_sad_u16 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_sad_u16 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x24,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_sad_u16 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_sad_u16 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x24,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_sad_u16 v255, 0xfe0b, vcc_hi, null clamp ; encoding: [0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00
+# GFX12: v_sad_u16 v255, 0xfe0b, vcc_hi, null clamp ; encoding: [0xff,0x80,0x24,0xd6,0xff,0xd6,0xf0,0x01,0x0b,0xfe,0x00,0x00]
 
-# GFX12: v_sad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_sad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x25,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_sad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_sad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x25,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_sad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_sad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x25,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_sad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_sad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x25,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_sad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_sad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x25,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_sad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x25,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_sad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x25,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_sad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_sad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x25,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_sad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_sad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x25,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_sad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_sad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x25,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_sad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x25,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_sad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x25,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_sad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_sad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x25,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_sad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_sad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x25,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_sad_u32 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u32 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x25,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u8 v5, v1, v2, s3                 ; encoding: [0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_sad_u8 v5, v1, v2, s3                 ; encoding: [0x05,0x00,0x22,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_sad_u8 v5, v255, s2, s105             ; encoding: [0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_sad_u8 v5, v255, s2, s105             ; encoding: [0x05,0x00,0x22,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_sad_u8 v5, s1, v255, exec_hi          ; encoding: [0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_sad_u8 v5, s1, v255, exec_hi          ; encoding: [0x05,0x00,0x22,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_sad_u8 v5, s105, s105, exec_lo        ; encoding: [0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_sad_u8 v5, s105, s105, exec_lo        ; encoding: [0x05,0x00,0x22,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_sad_u8 v5, vcc_lo, ttmp15, v3         ; encoding: [0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_sad_u8 v5, vcc_lo, ttmp15, v3         ; encoding: [0x05,0x00,0x22,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_sad_u8 v5, vcc_hi, 0xaf123456, v255   ; encoding: [0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u8 v5, vcc_hi, 0xaf123456, v255   ; encoding: [0x05,0x00,0x22,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u8 v5, ttmp15, src_scc, ttmp15    ; encoding: [0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_sad_u8 v5, ttmp15, src_scc, ttmp15    ; encoding: [0x05,0x00,0x22,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_sad_u8 v5, m0, 0.5, m0                ; encoding: [0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_sad_u8 v5, m0, 0.5, m0                ; encoding: [0x05,0x00,0x22,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_sad_u8 v5, exec_lo, -1, vcc_hi        ; encoding: [0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_sad_u8 v5, exec_lo, -1, vcc_hi        ; encoding: [0x05,0x00,0x22,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_sad_u8 v5, exec_hi, null, vcc_lo      ; encoding: [0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_sad_u8 v5, exec_hi, null, vcc_lo      ; encoding: [0x05,0x00,0x22,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_sad_u8 v5, null, exec_lo, 0xaf123456  ; encoding: [0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u8 v5, null, exec_lo, 0xaf123456  ; encoding: [0x05,0x00,0x22,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sad_u8 v5, -1, exec_hi, src_scc       ; encoding: [0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_sad_u8 v5, -1, exec_hi, src_scc       ; encoding: [0x05,0x00,0x22,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_sad_u8 v5, 0.5, m0, 0.5               ; encoding: [0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_sad_u8 v5, 0.5, m0, 0.5               ; encoding: [0x05,0x00,0x22,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_sad_u8 v5, src_scc, vcc_lo, -1        ; encoding: [0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_sad_u8 v5, src_scc, vcc_lo, -1        ; encoding: [0x05,0x00,0x22,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_sad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_sad_u8 v255, 0xaf123456, vcc_hi, null clamp ; encoding: [0xff,0x80,0x22,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# W32: v_sub_co_u32 v5, s12, v1, v2              ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], v1, v2         ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00
+# W32: v_sub_co_u32 v5, s12, v1, v2            ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], v1, v2       ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_sub_co_u32 v5, s12, v255, v255          ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], v255, v255     ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00
+# W32: v_sub_co_u32 v5, s12, v255, v255        ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], v255, v255   ; encoding: [0x05,0x0c,0x01,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_sub_co_u32 v5, s12, s1, s2              ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], s1, s2         ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, s1, s2            ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], s1, s2       ; encoding: [0x05,0x0c,0x01,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, s105, s105          ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], s105, s105     ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, s105, s105        ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], s105, s105   ; encoding: [0x05,0x0c,0x01,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, vcc_lo, ttmp15      ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, vcc_lo, ttmp15    ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x01,0xd7,0x6a,0xf6,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, vcc_hi, 0xaf123456  ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-# W64: v_sub_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# W32: v_sub_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+# W64: v_sub_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x01,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# W32: v_sub_co_u32 v5, s12, ttmp15, src_scc     ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, ttmp15, src_scc   ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x01,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, m0, 0.5             ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], m0, 0.5        ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, m0, 0.5           ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], m0, 0.5      ; encoding: [0x05,0x0c,0x01,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, exec_lo, -1         ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], exec_lo, -1    ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_sub_co_u32 v5, s12, exec_lo, -1       ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], exec_lo, -1  ; encoding: [0x05,0x0c,0x01,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_sub_co_u32 v5, s12, exec_hi, null       ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], exec_hi, null  ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, exec_hi, null     ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x01,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s12, null, exec_lo       ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[12:13], null, exec_lo  ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_sub_co_u32 v5, s12, null, exec_lo     ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x01,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, s104, -1, exec_hi        ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_sub_co_u32 v5, s[104:105], -1, exec_hi  ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_sub_co_u32 v5, s104, -1, exec_hi      ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_sub_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x01,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, vcc_lo, 0.5, m0          ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_sub_co_u32 v5, vcc, 0.5, m0             ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_sub_co_u32 v5, vcc_lo, 0.5, m0        ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_sub_co_u32 v5, vcc, 0.5, m0           ; encoding: [0x05,0x6a,0x01,0xd7,0xf0,0xfa,0x00,0x00]
 
-# W32: v_sub_co_u32 v5, ttmp14, src_scc, vcc_lo  ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
-# W64: v_sub_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00
+# W32: v_sub_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
+# W64: v_sub_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x01,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_sub_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_sub_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x01,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v1, v2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v1, v2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v255, v255             ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v255, v255             ; encoding: [0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x0e,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, s1, s2                 ; encoding: [0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, s105, s105             ; encoding: [0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x0e,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x58,0x0e,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x0e,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_i16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_i16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x0e,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_sub_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_sub_nc_i32 v5, v1, v2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_sub_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_sub_nc_i32 v5, v255, v255             ; encoding: [0x05,0x00,0x25,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_sub_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, s1, s2                 ; encoding: [0x05,0x00,0x25,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, s105, s105             ; encoding: [0x05,0x00,0x25,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x25,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_sub_nc_i32 v5, vcc_hi, 0xaf123456     ; encoding: [0x05,0x00,0x25,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_sub_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_sub_nc_i32 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x25,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_sub_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_sub_nc_i32 v5, m0, 0.5                ; encoding: [0x05,0x00,0x25,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_sub_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_sub_nc_i32 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x25,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_sub_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, exec_hi, null          ; encoding: [0x05,0x00,0x25,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, null, exec_lo          ; encoding: [0x05,0x00,0x25,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, -1, exec_hi            ; encoding: [0x05,0x00,0x25,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, 0.5, m0                ; encoding: [0x05,0x00,0x25,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_sub_nc_i32 v5, src_scc, vcc_lo        ; encoding: [0x05,0x00,0x25,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_sub_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_sub_nc_i32 v255, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0x80,0x25,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
+0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v1.l, v2.l           ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v1, v2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x04,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.l       ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v255, v255             ; encoding: [0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x04,0xd7,0xff,0xff,0x03,0x00
 
+0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, s1, s2               ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, s1, s2                 ; encoding: [0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x01,0x04,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, s105, s105           ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, s105, s105             ; encoding: [0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x69,0xd2,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, vcc_lo, ttmp15       ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, vcc_lo, ttmp15         ; encoding: [0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x6a,0xf6,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, vcc_hi, 0xfe0b       ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, vcc_hi, 0xfe0b         ; encoding: [0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, ttmp15, src_scc      ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, ttmp15, src_scc        ; encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, m0, 0x3800           ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, m0, 0x3800             ; encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, exec_lo, -1          ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, exec_lo, -1            ; encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00
 
+0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, exec_hi, null        ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, exec_hi, null          ; encoding: [0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7f,0xf8,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, null, exec_lo        ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, null, exec_lo          ; encoding: [0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x04,0xd7,0x7c,0xfc,0x00,0x00
 
+0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.h, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, -1, exec_hi op_sel:[1,1,1] ; encoding: [0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x58,0x04,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, 0x3800, m0 op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00
 
+0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00
 
+0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v1, v2 op_sel:[1,0,0]  ; encoding: [0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x08,0x04,0xd7,0x01,0x05,0x02,0x00
 
+0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_sub_nc_u16 v5.l, v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_sub_nc_u16 v5, v255, v255 op_sel:[0,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x10,0x04,0xd7,0xff,0xff,0x03,0x00
 
+0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16 v255.h, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16 v255, 0xfe0b, vcc_hi op_sel:[0,0,1] clamp ; encoding: [0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# W32: v_subrev_co_u32 v5, s12, v1, v2           ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], v1, v2      ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00
+# W32: v_subrev_co_u32 v5, s12, v1, v2         ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], v1, v2    ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x05,0x02,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, v255, v255       ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], v255, v255  ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00
+# W32: v_subrev_co_u32 v5, s12, v255, v255     ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], v255, v255 ; encoding: [0x05,0x0c,0x02,0xd7,0xff,0xff,0x03,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, s1, s2           ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], s1, s2      ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, s1, s2         ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], s1, s2    ; encoding: [0x05,0x0c,0x02,0xd7,0x01,0x04,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, s105, s105       ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], s105, s105  ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, s105, s105     ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], s105, s105 ; encoding: [0x05,0x0c,0x02,0xd7,0x69,0xd2,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, vcc_lo, ttmp15   ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], vcc_lo, ttmp15 ; encoding: [0x05,0x0c,0x02,0xd7,0x6a,0xf6,0x00,0x00]
 
+0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
 # W32: v_subrev_co_u32 v5, s12, vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 # W64: v_subrev_co_u32 v5, s[12:13], vcc_hi, 0xaf123456 ; encoding: [0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
-0x05,0x0c,0x02,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
 
-# W32: v_subrev_co_u32 v5, s12, ttmp15, src_scc  ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], ttmp15, src_scc ; encoding: [0x05,0x0c,0x02,0xd7,0x7b,0xfa,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, m0, 0.5          ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], m0, 0.5     ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, m0, 0.5        ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], m0, 0.5   ; encoding: [0x05,0x0c,0x02,0xd7,0x7d,0xe0,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, exec_lo, -1      ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], exec_lo, -1 ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00
+# W32: v_subrev_co_u32 v5, s12, exec_lo, -1    ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], exec_lo, -1 ; encoding: [0x05,0x0c,0x02,0xd7,0x7e,0x82,0x01,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, exec_hi, null    ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, exec_hi, null  ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], exec_hi, null ; encoding: [0x05,0x0c,0x02,0xd7,0x7f,0xf8,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s12, null, exec_lo    ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00
+# W32: v_subrev_co_u32 v5, s12, null, exec_lo  ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[12:13], null, exec_lo ; encoding: [0x05,0x0c,0x02,0xd7,0x7c,0xfc,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, s104, -1, exec_hi     ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
-# W64: v_subrev_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00
+# W32: v_subrev_co_u32 v5, s104, -1, exec_hi   ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
+# W64: v_subrev_co_u32 v5, s[104:105], -1, exec_hi ; encoding: [0x05,0x68,0x02,0xd7,0xc1,0xfe,0x00,0x00]
 
-# W32: v_subrev_co_u32 v5, vcc_lo, 0.5, m0       ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
-# W64: v_subrev_co_u32 v5, vcc, 0.5, m0          ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00
+# W32: v_subrev_co_u32 v5, vcc_lo, 0.5, m0     ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
+# W64: v_subrev_co_u32 v5, vcc, 0.5, m0        ; encoding: [0x05,0x6a,0x02,0xd7,0xf0,0xfa,0x00,0x00]
 
+0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00
 # W32: v_subrev_co_u32 v5, ttmp14, src_scc, vcc_lo ; encoding: [0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00]
 # W64: v_subrev_co_u32 v5, ttmp[14:15], src_scc, vcc_lo ; encoding: [0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x7a,0x02,0xd7,0xfd,0xd4,0x00,0x00
 
-# GFX12: v_subrev_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_subrev_co_u32 v255, null, 0xaf123456, vcc_hi clamp ; encoding: [0xff,0xfc,0x02,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_trig_preop_f64 v[5:6], v[1:2], v2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_trig_preop_f64 v[5:6], v[1:2], v2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], v[1:2], v255   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00
+# GFX12: v_trig_preop_f64 v[5:6], v[1:2], v255   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xff,0x03,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], v[1:2], s2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], v[1:2], s2     ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0x05,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], v[1:2], s105   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], v[1:2], s105   ; encoding: [0x05,0x00,0x2f,0xd7,0x01,0xd3,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], v[254:255], ttmp15 ; encoding: [0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], v[254:255], ttmp15 ; encoding: [0x05,0x00,0x2f,0xd7,0xfe,0xf7,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], s[2:3], vcc_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], s[2:3], vcc_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x02,0xd6,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], s[104:105], vcc_lo ; encoding: [0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], s[104:105], vcc_lo ; encoding: [0x05,0x00,0x2f,0xd7,0x68,0xd4,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], vcc, m0        ; encoding: [0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], vcc, m0        ; encoding: [0x05,0x00,0x2f,0xd7,0x6a,0xfa,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], ttmp[14:15], exec_hi ; encoding: [0x05,0x00,0x2f,0xd7,0x7a,0xfe,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], exec, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], exec, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd7,0x7e,0xfc,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], null, null     ; encoding: [0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00]
 0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00
+# GFX12: v_trig_preop_f64 v[5:6], null, null     ; encoding: [0x05,0x00,0x2f,0xd7,0x7c,0xf8,0x00,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], -1, -1         ; encoding: [0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00]
 0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00
+# GFX12: v_trig_preop_f64 v[5:6], -1, -1         ; encoding: [0x05,0x00,0x2f,0xd7,0xc1,0x82,0x01,0x00]
 
-# GFX12: v_trig_preop_f64 v[5:6], 0.5, 0.5 mul:2 ; encoding: [0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08]
 0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08
+# GFX12: v_trig_preop_f64 v[5:6], 0.5, 0.5 mul:2 ; encoding: [0x05,0x00,0x2f,0xd7,0xf0,0xe0,0x01,0x08]
 
-# GFX12: v_trig_preop_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30]
 0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30
+# GFX12: v_trig_preop_f64 v[5:6], -|src_scc|, src_scc mul:4 ; encoding: [0x05,0x01,0x2f,0xd7,0xfd,0xfa,0x01,0x30]
 
-# GFX12: v_trig_preop_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf
+# GFX12: v_trig_preop_f64 v[254:255], 0xaf123456, 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x2f,0xd7,0xff,0xfe,0x01,0x18,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_writelane_b32 v5, s1, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, s1, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, s105, s2            ; encoding: [0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, s105, s2            ; encoding: [0x05,0x00,0x61,0xd7,0x69,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, vcc_lo, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, vcc_lo, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6a,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, vcc_hi, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, vcc_hi, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x6b,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, ttmp15, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, ttmp15, s2          ; encoding: [0x05,0x00,0x61,0xd7,0x7b,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, m0, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, m0, s2              ; encoding: [0x05,0x00,0x61,0xd7,0x7d,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, exec_lo, s2         ; encoding: [0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00
+# GFX12: v_writelane_b32 v5, exec_lo, s2         ; encoding: [0x05,0x00,0x61,0xd7,0x7e,0x04,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, exec_hi, s105       ; encoding: [0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00
+# GFX12: v_writelane_b32 v5, exec_hi, s105       ; encoding: [0x05,0x00,0x61,0xd7,0x7f,0xd2,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, null, ttmp15        ; encoding: [0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00]
 0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00
+# GFX12: v_writelane_b32 v5, null, ttmp15        ; encoding: [0x05,0x00,0x61,0xd7,0x7c,0xf6,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, -1, null            ; encoding: [0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00
+# GFX12: v_writelane_b32 v5, -1, null            ; encoding: [0x05,0x00,0x61,0xd7,0xc1,0xf8,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, 0.5, m0             ; encoding: [0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00
+# GFX12: v_writelane_b32 v5, 0.5, m0             ; encoding: [0x05,0x00,0x61,0xd7,0xf0,0xfa,0x00,0x00]
 
-# GFX12: v_writelane_b32 v5, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00]
 0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00
+# GFX12: v_writelane_b32 v5, src_scc, vcc_lo     ; encoding: [0x05,0x00,0x61,0xd7,0xfd,0xd4,0x00,0x00]
 
-# GFX12: v_writelane_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_writelane_b32 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x61,0xd7,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_xad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_xad_u32 v5, v255, s2, s105            ; encoding: [0x05,0x00,0x45,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_xad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_xad_u32 v5, s1, v255, exec_hi         ; encoding: [0x05,0x00,0x45,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_xad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_xad_u32 v5, s105, s105, exec_lo       ; encoding: [0x05,0x00,0x45,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_xad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_xad_u32 v5, vcc_lo, ttmp15, v3        ; encoding: [0x05,0x00,0x45,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_xad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_xad_u32 v5, vcc_hi, 0xaf123456, v255  ; encoding: [0x05,0x00,0x45,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_xad_u32 v5, ttmp15, src_scc, ttmp15   ; encoding: [0x05,0x00,0x45,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_xad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_xad_u32 v5, m0, 0.5, m0               ; encoding: [0x05,0x00,0x45,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_xad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_xad_u32 v5, exec_lo, -1, vcc_hi       ; encoding: [0x05,0x00,0x45,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_xad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_xad_u32 v5, exec_hi, null, vcc_lo     ; encoding: [0x05,0x00,0x45,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_xad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_xad_u32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x45,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_xad_u32 v5, -1, exec_hi, src_scc      ; encoding: [0x05,0x00,0x45,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_xad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_xad_u32 v5, 0.5, m0, 0.5              ; encoding: [0x05,0x00,0x45,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_xad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_xad_u32 v5, src_scc, vcc_lo, -1       ; encoding: [0x05,0x00,0x45,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_xad_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_xad_u32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x45,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xor3_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_xor3_b32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x40,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_xor3_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_xor3_b32 v5, v255, s2, s105           ; encoding: [0x05,0x00,0x40,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_xor3_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_xor3_b32 v5, s1, v255, exec_hi        ; encoding: [0x05,0x00,0x40,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_xor3_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_xor3_b32 v5, s105, s105, exec_lo      ; encoding: [0x05,0x00,0x40,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_xor3_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_xor3_b32 v5, vcc_lo, ttmp15, v3       ; encoding: [0x05,0x00,0x40,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_xor3_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_xor3_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x40,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xor3_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01]
 0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01
+# GFX12: v_xor3_b32 v5, ttmp15, src_scc, ttmp15  ; encoding: [0x05,0x00,0x40,0xd6,0x7b,0xfa,0xed,0x01]
 
-# GFX12: v_xor3_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_xor3_b32 v5, m0, 0.5, m0              ; encoding: [0x05,0x00,0x40,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_xor3_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_xor3_b32 v5, exec_lo, -1, vcc_hi      ; encoding: [0x05,0x00,0x40,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_xor3_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01]
 0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01
+# GFX12: v_xor3_b32 v5, exec_hi, null, vcc_lo    ; encoding: [0x05,0x00,0x40,0xd6,0x7f,0xf8,0xa8,0x01]
 
-# GFX12: v_xor3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf
+# GFX12: v_xor3_b32 v5, null, exec_lo, 0xaf123456 ; encoding: [0x05,0x00,0x40,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_xor3_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03]
 0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03
+# GFX12: v_xor3_b32 v5, -1, exec_hi, src_scc     ; encoding: [0x05,0x00,0x40,0xd6,0xc1,0xfe,0xf4,0x03]
 
-# GFX12: v_xor3_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03]
 0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03
+# GFX12: v_xor3_b32 v5, 0.5, m0, 0.5             ; encoding: [0x05,0x00,0x40,0xd6,0xf0,0xfa,0xc0,0x03]
 
-# GFX12: v_xor3_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03]
 0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03
+# GFX12: v_xor3_b32 v5, src_scc, vcc_lo, -1      ; encoding: [0x05,0x00,0x40,0xd6,0xfd,0xd4,0x04,0x03]
 
-# GFX12: v_xor3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
+# GFX12: v_xor3_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x40,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
 
 
+0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00
 # W32-REAL16: v_xor_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W32-FAKE16: v_xor_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W64-REAL16: v_xor_b16 v5.l, v1.l, v2.l              ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
 # W64-FAKE16: v_xor_b16 v5, v1, v2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00]
-0x05,0x00,0x64,0xd7,0x01,0x05,0x02,0x00
 
 
+0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00
 # W32-REAL16: v_xor_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W32-FAKE16: v_xor_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W64-REAL16: v_xor_b16 v5.l, v255.l, v255.l          ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
 # W64-FAKE16: v_xor_b16 v5, v255, v255                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00]
-0x05,0x00,0x64,0xd7,0xff,0xff,0x03,0x00
 
 
+0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, s1, s2                  ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, s1, s2                    ; encoding: [0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x01,0x04,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, s105, s105              ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, s105, s105                ; encoding: [0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x69,0xd2,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, vcc_lo, ttmp15          ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, vcc_lo, ttmp15            ; encoding: [0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x6a,0xf6,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, vcc_hi, 0xfe0b          ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, vcc_hi, 0xfe0b            ; encoding: [0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W32-FAKE16: v_xor_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-REAL16: v_xor_b16 v5.l, ttmp15, src_scc         ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 # W64-FAKE16: v_xor_b16 v5, ttmp15, src_scc           ; encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
-0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00
 
+0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, m0, 0x3800              ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, m0, 0x3800                ; encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00
 
 
+0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00
 # W32-REAL16: v_xor_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W32-FAKE16: v_xor_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W64-REAL16: v_xor_b16 v5.l, exec_lo, -1             ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
 # W64-FAKE16: v_xor_b16 v5, exec_lo, -1               ; encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
-0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00
 
 
+0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, exec_hi, null           ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, exec_hi, null             ; encoding: [0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7f,0xf8,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, null, exec_lo           ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, null, exec_lo             ; encoding: [0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00]
-0x05,0x00,0x64,0xd7,0x7c,0xfc,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, -1, exec_hi             ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, -1, exec_hi               ; encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00
 
+0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, 0x3800, m0              ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, 0x3800, m0                ; encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00
 
 
+0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00
 # W32-REAL16: v_xor_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-REAL16: v_xor_b16 v5.l, src_scc, vcc_lo         ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v5, src_scc, vcc_lo           ; encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
-0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00
 
 
+0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 # W32-REAL16: v_xor_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W32-FAKE16: v_xor_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-REAL16: v_xor_b16 v255.l, 0xfe0b, vcc_hi        ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 # W64-FAKE16: v_xor_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
-0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_minimum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_minimum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_minimum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_minimum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_minimum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_minimum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_minimum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_minimum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_minimum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_minimum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_minimum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_minimum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_minimum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_minimum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_minimum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_minimum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_minimum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_minimum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_minimum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_minimum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_minimum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_minimum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_minimum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_minimum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_minimum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_minimum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_minimum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_minimum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX12: v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_maximum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_maximum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_maximum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_maximum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_maximum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_maximum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_maximum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_maximum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_maximum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_maximum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_maximum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_maximum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_maximum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_maximum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_maximum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_maximum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_maximum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_maximum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_maximum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_maximum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_maximum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_maximum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_maximum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_maximum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_maximum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_maximum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
 0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00
+# GFX12: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
 
-# GFX12: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
+# GFX12: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
 0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00
+# GFX12: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
 
-# GFX12: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
+# GFX12: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
 0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00
+# GFX12: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
 0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40
+# GFX12: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
 
-# GFX12: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00
+# GFX12: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
 
-# GFX12: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
 0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00
+# GFX12: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
 0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00
+# GFX12: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00
+# GFX12: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
 
-# GFX12: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
 0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00
+# GFX12: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
 
-# GFX12: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
 0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00
+# GFX12: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
 
-# GFX12: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
 0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00
+# GFX12: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
 
-# GFX12: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
 0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00
+# GFX12: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60]
 0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60
+# GFX12: v_minimum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60]
 
-# GFX12: v_minimum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00]
 0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_minimum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_minimum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40]
 0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40
+# GFX12: v_minimum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40]
 
-# GFX12: v_minimum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_minimum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00]
 0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00]
 0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00
+# GFX12: v_minimum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00]
 0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00
+# GFX12: v_minimum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00
+# GFX12: v_minimum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00]
 0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00
+# GFX12: v_minimum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
 0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf
+# GFX12: v_minimum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
 
-# GFX12: v_minimum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX12: v_minimum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00]
 0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00
+# GFX12: v_minimum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60]
 0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60
+# GFX12: v_maximum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60]
 
-# GFX12: v_maximum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00]
 0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20]
 0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20
+# GFX12: v_maximum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20]
 
-# GFX12: v_maximum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40]
 0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40
+# GFX12: v_maximum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40]
 
-# GFX12: v_maximum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00]
 0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00
+# GFX12: v_maximum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00]
 0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00]
 0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00]
 0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00]
 0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00
+# GFX12: v_maximum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00]
 0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00
+# GFX12: v_maximum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00]
 0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00
+# GFX12: v_maximum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00]
 0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00
+# GFX12: v_maximum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
 0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf
+# GFX12: v_maximum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
 
-# GFX12: v_maximum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00]
 0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00]
 
-# GFX12: v_maximum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00]
 0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00
+# GFX12: v_maximum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00]
 
-# GFX12: v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maximum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maximum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_maximum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_maximum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maximum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maximum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maximum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maximum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_maximum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_maximum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maximum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maximum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_maximum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maximum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maximum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maximum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maximum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maximum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maximum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maximum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maximum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_maximum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maximum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maximum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maximum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minimum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_minimum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minimum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minimum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_minimum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_minimum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minimum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minimum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minimum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minimum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_minimum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_minimum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minimum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minimum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_minimum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minimum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minimum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minimum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minimum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minimum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minimum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minimum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minimum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_minimum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minimum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minimum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minimum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+# GFX12: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# GFX12: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+# GFX12: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+# GFX12: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# GFX12: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maximumminimum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_maximumminimum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximumminimum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maximumminimum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maximumminimum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_maximumminimum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_maximumminimum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maximumminimum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maximumminimum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maximumminimum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maximumminimum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_maximumminimum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_maximumminimum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maximumminimum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maximumminimum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_maximumminimum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximumminimum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maximumminimum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maximumminimum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maximumminimum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maximumminimum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maximumminimum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maximumminimum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maximumminimum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maximumminimum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_maximumminimum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_maximumminimum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maximumminimum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maximumminimum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maximumminimum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+# GFX12: v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimummaximum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minimummaximum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minimummaximum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33]
 0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33
+# GFX12: v_minimummaximum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33]
 
-# GFX12: v_minimummaximum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minimummaximum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minimummaximum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minimummaximum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minimummaximum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b]
 0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b
+# GFX12: v_minimummaximum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b]
 
-# GFX12: v_minimummaximum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minimummaximum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minimummaximum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+# GFX12: v_minimummaximum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimummaximum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minimummaximum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minimummaximum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minimummaximum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minimummaximum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minimummaximum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minimummaximum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minimummaximum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minimummaximum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# GFX12: v_minimummaximum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_minimummaximum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minimummaximum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minimummaximum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minimummaximum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+# GFX12: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+# GFX12: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# GFX12: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
 0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3
+# GFX12: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
 
-# GFX12: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
 0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23
+# GFX12: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
 
-# GFX12: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
 0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1
+# GFX12: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
 
-# GFX12: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
 0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1
+# GFX12: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
 
-# GFX12: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
 0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43
+# GFX12: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
 
-# GFX12: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
 0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01
+# GFX12: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
 
-# GFX12: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+# GFX12: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
 0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01
+# GFX12: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
 
-# GFX12: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
 0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01
+# GFX12: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
 
-# GFX12: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
 0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00
+# GFX12: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
 
-# GFX12: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
 0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01
+# GFX12: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
 
-# GFX12: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+# GFX12: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
 0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04
+# GFX12: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
 
-# GFX12: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
 0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01
+# GFX12: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
 
-# GFX12: v_s_exp_f32 s5, s1                      ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, s1                      ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, s105                    ; encoding: [0x05,0x00,0x80,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, s105                    ; encoding: [0x05,0x00,0x80,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x80,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x80,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x80,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x80,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x80,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x80,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, m0                      ; encoding: [0x05,0x00,0x80,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, m0                      ; encoding: [0x05,0x00,0x80,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x80,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x80,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x80,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x80,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, null                    ; encoding: [0x05,0x00,0x80,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, null                    ; encoding: [0x05,0x00,0x80,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, -1                      ; encoding: [0x05,0x00,0x80,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, -1                      ; encoding: [0x05,0x00,0x80,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x80,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x80,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x80,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x80,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x80,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x80,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 0x69,0x00,0x80,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_s_exp_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x80,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_s_exp_f32 s5, -s1                     ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_exp_f32 s5, -s1                     ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_exp_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x80,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x80,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x80,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x80,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x80,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x80,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_exp_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_exp_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_exp_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_exp_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_exp_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x80,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_exp_f16 s5, s1                      ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, s1                      ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, s105                    ; encoding: [0x05,0x00,0x81,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, s105                    ; encoding: [0x05,0x00,0x81,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x81,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x81,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x81,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x81,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x81,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x81,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, m0                      ; encoding: [0x05,0x00,0x81,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, m0                      ; encoding: [0x05,0x00,0x81,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x81,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x81,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x81,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x81,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, null                    ; encoding: [0x05,0x00,0x81,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, null                    ; encoding: [0x05,0x00,0x81,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, -1                      ; encoding: [0x05,0x00,0x81,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, -1                      ; encoding: [0x05,0x00,0x81,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x81,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x81,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x81,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x81,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x81,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x81,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 0x69,0x00,0x81,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_s_exp_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x81,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, -s1                     ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_exp_f16 s5, -s1                     ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_exp_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x81,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x81,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x81,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x81,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x81,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_exp_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x81,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_exp_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_exp_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_exp_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_exp_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_exp_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_exp_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x81,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_log_f32 s5, s1                      ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, s1                      ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, s105                    ; encoding: [0x05,0x00,0x82,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, s105                    ; encoding: [0x05,0x00,0x82,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x82,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x82,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x82,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x82,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x82,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x82,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, m0                      ; encoding: [0x05,0x00,0x82,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, m0                      ; encoding: [0x05,0x00,0x82,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x82,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x82,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x82,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x82,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, null                    ; encoding: [0x05,0x00,0x82,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, null                    ; encoding: [0x05,0x00,0x82,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, -1                      ; encoding: [0x05,0x00,0x82,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, -1                      ; encoding: [0x05,0x00,0x82,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x82,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x82,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x82,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x82,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x82,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x82,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 0x69,0x00,0x82,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_s_log_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x82,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_s_log_f32 s5, -s1                     ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_log_f32 s5, -s1                     ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_log_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x82,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x82,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x82,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x82,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x82,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x82,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_log_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_log_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_log_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_log_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_log_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x82,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_log_f16 s5, s1                      ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, s1                      ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, s105                    ; encoding: [0x05,0x00,0x83,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, s105                    ; encoding: [0x05,0x00,0x83,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x83,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x83,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x83,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x83,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x83,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x83,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, m0                      ; encoding: [0x05,0x00,0x83,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, m0                      ; encoding: [0x05,0x00,0x83,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x83,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x83,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x83,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x83,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, null                    ; encoding: [0x05,0x00,0x83,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, null                    ; encoding: [0x05,0x00,0x83,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, -1                      ; encoding: [0x05,0x00,0x83,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, -1                      ; encoding: [0x05,0x00,0x83,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x83,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x83,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x83,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x83,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x83,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x83,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 0x69,0x00,0x83,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_s_log_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x83,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, -s1                     ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_log_f16 s5, -s1                     ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_log_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x83,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x83,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x83,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x83,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x83,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_log_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x83,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_log_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_log_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_log_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_log_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_log_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_log_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x83,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_rcp_f32 s5, s1                      ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, s1                      ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, s105                    ; encoding: [0x05,0x00,0x84,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, s105                    ; encoding: [0x05,0x00,0x84,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x84,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x84,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x84,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x84,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x84,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x84,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, m0                      ; encoding: [0x05,0x00,0x84,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, m0                      ; encoding: [0x05,0x00,0x84,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x84,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x84,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x84,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x84,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, null                    ; encoding: [0x05,0x00,0x84,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, null                    ; encoding: [0x05,0x00,0x84,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, -1                      ; encoding: [0x05,0x00,0x84,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, -1                      ; encoding: [0x05,0x00,0x84,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x84,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x84,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x84,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x84,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x84,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x84,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 0x69,0x00,0x84,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_s_rcp_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x84,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_s_rcp_f32 s5, -s1                     ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_rcp_f32 s5, -s1                     ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_rcp_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x84,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x84,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x84,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x84,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x84,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x84,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_rcp_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_rcp_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_rcp_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_rcp_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_rcp_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x84,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_rcp_f16 s5, s1                      ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, s1                      ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, s105                    ; encoding: [0x05,0x00,0x85,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, s105                    ; encoding: [0x05,0x00,0x85,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x85,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x85,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x85,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x85,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x85,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x85,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, m0                      ; encoding: [0x05,0x00,0x85,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, m0                      ; encoding: [0x05,0x00,0x85,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x85,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x85,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x85,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x85,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, null                    ; encoding: [0x05,0x00,0x85,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, null                    ; encoding: [0x05,0x00,0x85,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, -1                      ; encoding: [0x05,0x00,0x85,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, -1                      ; encoding: [0x05,0x00,0x85,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x85,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x85,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x85,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x85,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x85,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x85,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 0x69,0x00,0x85,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_s_rcp_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x85,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, -s1                     ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_rcp_f16 s5, -s1                     ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_rcp_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x85,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x85,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x85,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x85,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x85,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rcp_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x85,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rcp_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_rcp_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_rcp_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_rcp_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_rcp_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_rcp_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x85,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_rsq_f32 s5, s1                      ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, s1                      ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, s105                    ; encoding: [0x05,0x00,0x86,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, s105                    ; encoding: [0x05,0x00,0x86,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x86,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, vcc_lo                  ; encoding: [0x05,0x00,0x86,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x86,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, vcc_hi                  ; encoding: [0x05,0x00,0x86,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x86,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, ttmp15                  ; encoding: [0x05,0x00,0x86,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, m0                      ; encoding: [0x05,0x00,0x86,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, m0                      ; encoding: [0x05,0x00,0x86,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x86,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, exec_lo                 ; encoding: [0x05,0x00,0x86,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x86,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, exec_hi                 ; encoding: [0x05,0x00,0x86,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, null                    ; encoding: [0x05,0x00,0x86,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, null                    ; encoding: [0x05,0x00,0x86,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, -1                      ; encoding: [0x05,0x00,0x86,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, -1                      ; encoding: [0x05,0x00,0x86,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x86,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, 0.5                     ; encoding: [0x05,0x00,0x86,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x86,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x86,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, src_scc                 ; encoding: [0x05,0x00,0x86,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x86,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 0x69,0x00,0x86,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_s_rsq_f32 s105, 0xaf123456            ; encoding: [0x69,0x00,0x86,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_s_rsq_f32 s5, -s1                     ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_rsq_f32 s5, -s1                     ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_rsq_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x86,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x86,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, |s1|                    ; encoding: [0x05,0x01,0x86,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x86,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x86,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f32 s5, s1 clamp                ; encoding: [0x05,0x80,0x86,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_rsq_f32 s5, s1 mul:2                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_rsq_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_rsq_f32 s5, s1 mul:4                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_rsq_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_rsq_f32 s5, s1 div:2                ; encoding: [0x05,0x00,0x86,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_rsq_f16 s5, s1                      ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, s1                      ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, s105                    ; encoding: [0x05,0x00,0x87,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, s105                    ; encoding: [0x05,0x00,0x87,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x87,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, vcc_lo                  ; encoding: [0x05,0x00,0x87,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x87,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, vcc_hi                  ; encoding: [0x05,0x00,0x87,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x87,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, ttmp15                  ; encoding: [0x05,0x00,0x87,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, m0                      ; encoding: [0x05,0x00,0x87,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, m0                      ; encoding: [0x05,0x00,0x87,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x87,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, exec_lo                 ; encoding: [0x05,0x00,0x87,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x87,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, exec_hi                 ; encoding: [0x05,0x00,0x87,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, null                    ; encoding: [0x05,0x00,0x87,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, null                    ; encoding: [0x05,0x00,0x87,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, -1                      ; encoding: [0x05,0x00,0x87,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, -1                      ; encoding: [0x05,0x00,0x87,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x87,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, 0.5                     ; encoding: [0x05,0x00,0x87,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x87,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x87,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, src_scc                 ; encoding: [0x05,0x00,0x87,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x87,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 0x69,0x00,0x87,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_s_rsq_f16 s105, 0xaf12                ; encoding: [0x69,0x00,0x87,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, -s1                     ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_rsq_f16 s5, -s1                     ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_rsq_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x87,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x87,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, |s1|                    ; encoding: [0x05,0x01,0x87,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x87,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x87,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_rsq_f16 s5, s1 clamp                ; encoding: [0x05,0x80,0x87,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_rsq_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_rsq_f16 s5, s1 mul:2                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_rsq_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_rsq_f16 s5, s1 mul:4                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_rsq_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_rsq_f16 s5, s1 div:2                ; encoding: [0x05,0x00,0x87,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_sqrt_f32 s5, s1                     ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, s1                     ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, s105                   ; encoding: [0x05,0x00,0x88,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, s105                   ; encoding: [0x05,0x00,0x88,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, vcc_lo                 ; encoding: [0x05,0x00,0x88,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, vcc_lo                 ; encoding: [0x05,0x00,0x88,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, vcc_hi                 ; encoding: [0x05,0x00,0x88,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, vcc_hi                 ; encoding: [0x05,0x00,0x88,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, ttmp15                 ; encoding: [0x05,0x00,0x88,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, ttmp15                 ; encoding: [0x05,0x00,0x88,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, m0                     ; encoding: [0x05,0x00,0x88,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, m0                     ; encoding: [0x05,0x00,0x88,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, exec_lo                ; encoding: [0x05,0x00,0x88,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, exec_lo                ; encoding: [0x05,0x00,0x88,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, exec_hi                ; encoding: [0x05,0x00,0x88,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, exec_hi                ; encoding: [0x05,0x00,0x88,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, null                   ; encoding: [0x05,0x00,0x88,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, null                   ; encoding: [0x05,0x00,0x88,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, -1                     ; encoding: [0x05,0x00,0x88,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, -1                     ; encoding: [0x05,0x00,0x88,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, 0.5                    ; encoding: [0x05,0x00,0x88,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, 0.5                    ; encoding: [0x05,0x00,0x88,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, src_scc                ; encoding: [0x05,0x00,0x88,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x88,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, src_scc                ; encoding: [0x05,0x00,0x88,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s105, 0xaf123456           ; encoding: [0x69,0x00,0x88,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 0x69,0x00,0x88,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX12: v_s_sqrt_f32 s105, 0xaf123456           ; encoding: [0x69,0x00,0x88,0xd6,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-# GFX12: v_s_sqrt_f32 s5, -s1                    ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_sqrt_f32 s5, -s1                    ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_sqrt_f32 s5, |s1|                   ; encoding: [0x05,0x01,0x88,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x88,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, |s1|                   ; encoding: [0x05,0x01,0x88,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, s1 clamp               ; encoding: [0x05,0x80,0x88,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x88,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f32 s5, s1 clamp               ; encoding: [0x05,0x80,0x88,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f32 s5, s1 mul:2               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_sqrt_f32 s5, s1 mul:2               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_sqrt_f32 s5, s1 mul:4               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_sqrt_f32 s5, s1 mul:4               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_sqrt_f32 s5, s1 div:2               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_sqrt_f32 s5, s1 div:2               ; encoding: [0x05,0x00,0x88,0xd6,0x01,0x00,0x00,0x18]
 
-# GFX12: v_s_sqrt_f16 s5, s1                     ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, s1                     ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, s105                   ; encoding: [0x05,0x00,0x89,0xd6,0x69,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x69,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, s105                   ; encoding: [0x05,0x00,0x89,0xd6,0x69,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, vcc_lo                 ; encoding: [0x05,0x00,0x89,0xd6,0x6a,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x6a,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, vcc_lo                 ; encoding: [0x05,0x00,0x89,0xd6,0x6a,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, vcc_hi                 ; encoding: [0x05,0x00,0x89,0xd6,0x6b,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x6b,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, vcc_hi                 ; encoding: [0x05,0x00,0x89,0xd6,0x6b,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, ttmp15                 ; encoding: [0x05,0x00,0x89,0xd6,0x7b,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x7b,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, ttmp15                 ; encoding: [0x05,0x00,0x89,0xd6,0x7b,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, m0                     ; encoding: [0x05,0x00,0x89,0xd6,0x7d,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x7d,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, m0                     ; encoding: [0x05,0x00,0x89,0xd6,0x7d,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, exec_lo                ; encoding: [0x05,0x00,0x89,0xd6,0x7e,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x7e,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, exec_lo                ; encoding: [0x05,0x00,0x89,0xd6,0x7e,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, exec_hi                ; encoding: [0x05,0x00,0x89,0xd6,0x7f,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x7f,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, exec_hi                ; encoding: [0x05,0x00,0x89,0xd6,0x7f,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, null                   ; encoding: [0x05,0x00,0x89,0xd6,0x7c,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0x7c,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, null                   ; encoding: [0x05,0x00,0x89,0xd6,0x7c,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, -1                     ; encoding: [0x05,0x00,0x89,0xd6,0xc1,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0xc1,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, -1                     ; encoding: [0x05,0x00,0x89,0xd6,0xc1,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, 0.5                    ; encoding: [0x05,0x00,0x89,0xd6,0xf0,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0xf0,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, 0.5                    ; encoding: [0x05,0x00,0x89,0xd6,0xf0,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, src_scc                ; encoding: [0x05,0x00,0x89,0xd6,0xfd,0x00,0x00,0x00]
 0x05,0x00,0x89,0xd6,0xfd,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, src_scc                ; encoding: [0x05,0x00,0x89,0xd6,0xfd,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s105, 0xaf12               ; encoding: [0x69,0x00,0x89,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 0x69,0x00,0x89,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00
+# GFX12: v_s_sqrt_f16 s105, 0xaf12               ; encoding: [0x69,0x00,0x89,0xd6,0xff,0x00,0x00,0x00,0x12,0xaf,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, -s1                    ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x20]
 0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x20
+# GFX12: v_s_sqrt_f16 s5, -s1                    ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x20]
 
-# GFX12: v_s_sqrt_f16 s5, |s1|                   ; encoding: [0x05,0x01,0x89,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x01,0x89,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, |s1|                   ; encoding: [0x05,0x01,0x89,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, s1 clamp               ; encoding: [0x05,0x80,0x89,0xd6,0x01,0x00,0x00,0x00]
 0x05,0x80,0x89,0xd6,0x01,0x00,0x00,0x00
+# GFX12: v_s_sqrt_f16 s5, s1 clamp               ; encoding: [0x05,0x80,0x89,0xd6,0x01,0x00,0x00,0x00]
 
-# GFX12: v_s_sqrt_f16 s5, s1 mul:2               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x08]
 0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x08
+# GFX12: v_s_sqrt_f16 s5, s1 mul:2               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x08]
 
-# GFX12: v_s_sqrt_f16 s5, s1 mul:4               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x10]
 0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x10
+# GFX12: v_s_sqrt_f16 s5, s1 mul:4               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x10]
 
-# GFX12: v_s_sqrt_f16 s5, s1 div:2               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x18]
 0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x18
+# GFX12: v_s_sqrt_f16 s5, s1 div:2               ; encoding: [0x05,0x00,0x89,0xd6,0x01,0x00,0x00,0x18]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index 5081b9811e43e..232ed8d23c9c6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -1,5999 +1,6071 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16 %s
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, 15, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x1e,0x0d,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x1e,0x0d,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, 15, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x1e,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_add3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_add_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_add_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_add_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_add_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_add_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_add_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x00,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX12: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x47,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, null row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x62,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_and_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_and_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x62,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x57,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_and_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_and_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x3a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x3a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_bcnt_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x11,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_bfe_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_bfe_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x10,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_bfe_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_bfe_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x12,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_bfi_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_bfi_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_bfm_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_bfm_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0x1a,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s104 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[104:105] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xa2,0x01,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, vcc_lo row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, vcc row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp14 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp[14:15] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13]
-0x05,0x00,0x5d,0xd6,0xfa,0x04,0xea,0x01,0x01,0x60,0x01,0x13
 
-# GFX12: v_cndmask_b16_e64_dpp v255, v255, v255, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30
+# GFX12: v_cndmask_b16_e64_dpp v255, v255, v255, null row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xfa,0xfe,0xf3,0x01,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0f,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0f,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x0e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x0e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x0e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x0e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x0e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x0e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20
+# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20
+# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v6, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x06,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v6, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v6, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v6, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v255 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x00,0x6c,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v255 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x20,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x20,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x20,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x40,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x40,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x40,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x60,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x60,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x60,0x6c,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v6, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 0x06,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v6, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v6, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v6, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v255 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 0x01,0x00,0x6b,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v255 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x00,0x6b,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x20,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x20,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x20,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x40,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x40,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x40,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x60,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 0x01,0x60,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x60,0x6b,0xd7,0xfa,0x06,0x02,0x00,0x02,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x06,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x06,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x06,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x07,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x07,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x07,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x26,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30]
 0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xfa,0xfe,0xf7,0x23,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x21,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x21,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x22,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x02,0x22,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x13,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x13,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x13,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x13,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x13,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x13,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x08,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1c,0xd7,0xfa,0x04,0x02,0x10,0x01,0x60,0x01,0x13]
 
-# GFX12: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30]
 0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30
+# GFX12: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xfa,0xfe,0x03,0x38,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x15,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_lerp_u8_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_lerp_u8_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x46,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x56,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x38,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x38,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x39,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x39,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x2a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x2a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x2a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x2a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x2a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x2a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x2a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, 15, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x1e,0xa9,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x1e,0xa9,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, 15, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x1e,0xa9,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1d,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1e,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0a,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0a,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x09,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x09,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0xea,0x0d,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0xea,0x0d,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -2.0, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0xea,0x0d,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6b,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6b,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6b,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6b,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6b,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6b,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x6b,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x6b,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x6b,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6b,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6b,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_maxmin_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6b,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x69,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x69,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x69,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x69,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x69,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x69,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x69,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x69,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x69,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x69,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x69,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x69,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x69,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x69,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x69,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x69,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x69,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x69,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x69,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x69,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x69,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x69,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_maxmin_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x69,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x62,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1f,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x31,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x31,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x31,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x31,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x31,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x31,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x31,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x31,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x31,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x31,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x31,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x31,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x31,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x31,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x31,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x31,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x31,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x31,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x31,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x31,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x31,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x31,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x31,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x20,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x21,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x29,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x29,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x29,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x29,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x29,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x29,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x29,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x29,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x29,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x29,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x29,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x29,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x29,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x29,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x29,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x29,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x29,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x29,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x29,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x29,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x29,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x29,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x29,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x1b,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0c,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0c,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0b,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x0b,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6a,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6a,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6a,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6a,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6a,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x6a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x6a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x6a,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_minmax_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_minmax_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6a,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x68,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x68,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x68,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x68,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x68,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x68,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x68,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x68,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x68,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x68,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x68,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x68,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x68,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x68,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x68,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x68,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x68,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x68,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x68,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_minmax_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x68,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x68,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_minmax_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x68,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x65,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_minmax_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_minmax_i32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_minmax_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_minmax_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x39,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x05,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x40,0x05,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x18,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x18,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x18,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x18,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x18,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x18,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13
+# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x06,0x18,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x58,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_or3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_or3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x63,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_or_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_or_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x63,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x44,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_perm_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_perm_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x23,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x24,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x22,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_sub_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_sub_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_sub_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_sub_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_sub_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_sub_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x01,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX12: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x25,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 
-# GFX12: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x0c,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32: v_subrev_co_u32_e64_dpp v5, s104, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64: v_subrev_co_u32_e64_dpp v5, s[104:105], v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x68,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32: v_subrev_co_u32_e64_dpp v5, vcc_lo, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64: v_subrev_co_u32_e64_dpp v5, vcc, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x6a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32: v_subrev_co_u32_e64_dpp v5, ttmp14, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64: v_subrev_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x7a,0x02,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
-# GFX12: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
+# GFX12: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x45,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_xad_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_xad_u32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xee,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x40,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_xor3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_xor3_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x00,0x64,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0x00,0x64,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0d,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0d,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x03,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x12,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x12,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x12,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x13,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x13,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x13,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x54,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x54,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x54,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x54,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x54,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x54,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x54,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x48,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x48,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x48,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x48,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x48,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x48,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x48,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x53,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x53,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x53,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x53,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x53,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x5a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x5a,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x41,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x41,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x41,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x41,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x41,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x00,0x59,0xd6,0xfa,0x04,0x06,0x03,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x08,0x59,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2c,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x2c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x2c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x2c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x2c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x2c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x2c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x2c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x2c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x2c,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x2c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x2c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x2c,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x2c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x2c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x2c,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4d,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4d,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4d,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4d,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4e,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4e,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4e,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4e,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4e,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x32,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x32,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x32,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x32,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x32,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x32,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x32,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x32,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x32,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x32,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x32,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x32,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x32,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x32,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x32,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x32,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x32,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x32,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x32,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x32,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x32,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x32,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x32,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x50,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x50,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x50,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x50,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x50,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x51,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x51,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x51,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x51,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x51,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v255 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s105 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, ttmp15 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2b,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2b,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, |v1|, v2, -m0 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2b,0xd6,0xfa,0x04,0xf6,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2b,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x2b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x7c,0x2b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x2b,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x2b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x0b,0x2b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0b,0x2b,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x2b,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x15,0x2b,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x2b,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x2b,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 0x05,0x26,0x2b,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x26,0x2b,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x2b,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0xc7,0x2b,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc7,0x2b,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4a,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4a,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4a,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4a,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v255 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xae,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4b,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x4b,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x4b,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x10,0x4b,0xd6,0xfa,0x04,0xf2,0x01,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x20,0x4b,0xd6,0xfa,0x04,0x06,0x03,0x01,0x60,0x01,0x13]
 
-# GFX12: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
+# GFX12: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x11,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x11,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13
+# GFX12: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x0a,0x11,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x01,0x13]
 
-# GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x0e,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x0e,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
 
+0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
-0x05,0x00,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
 
+0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
+0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
-0x05,0x58,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
 
+0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01]
-0x05,0x08,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x5f,0x01,0x01
 
+0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13]
-0x05,0x10,0x04,0xd7,0xfa,0x04,0x02,0x00,0x01,0x60,0x01,0x13
 
+0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
-0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 # op_sel[1:0] are ignored
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x78,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+
+0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+
+0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+
+0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 # op_sel[1:0] are ignored
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x78,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+
+0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+
+0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
 0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index 77f05027d1cfd..469b199053d47 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -1,3782 +1,3854 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16 %s
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, 10, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x14,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_add3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_add3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x55,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_add_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_add_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x00,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX12: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_add_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x00,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_add_lshl_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x47,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_add_lshl_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x47,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_add_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_alignbyte_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x17,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_and_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_and_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x62,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_and_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_and_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_and_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x62,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_and_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x57,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_and_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_and_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x57,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x3a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_ashrrev_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_ashrrev_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x3a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_bcnt_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bcnt_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_bcnt_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_bfe_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x11,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfe_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x10,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfe_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_bfe_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x10,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_bfi_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfi_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_bfi_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x12,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_bfm_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_bfm_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_bfm_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s6 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, s104 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, s[104:105] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xa2,0x01,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
 
+0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05
 # W32: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp14 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
 # W64: v_cndmask_b16_e64_dpp v5, v1, v2, ttmp[14:15] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05]
-0x05,0x00,0x5d,0xd6,0xe9,0x04,0xea,0x01,0x01,0x77,0x39,0x05
 
-# GFX12: v_cndmask_b16_e64_dpp v255, v255, v255, null dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00
+# GFX12: v_cndmask_b16_e64_dpp v255, v255, v255, null dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x5d,0xd6,0xea,0xfe,0xf3,0x01,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_cubeid_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0c,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_cubeid_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0c,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0f,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_cubema_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0f,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_cubema_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0f,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_cubesc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0d,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_cubesc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0d,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x0e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x0e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x0e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x0e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x0e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x0e,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, |v1|, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, |v1|, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v255, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x6b,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 0xff,0x01,0x6b,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v255, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x6b,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x20,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x20,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x20,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x40,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x40,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x40,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x60,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x60,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_fp8_f32_e64_dpp v1, v2, v3 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x60,0x6b,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, |v1|, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, |v1|, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v5, -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v255, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x6c,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 0xff,0x01,0x6c,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v255, -|v255|, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x6c,0xd7,0xe9,0xfe,0x03,0x20,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x20,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x20,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x20,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x40,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x40,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x40,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x60,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 0x01,0x60,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05
+# GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, v2, v3 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x60,0x6c,0xd7,0xe9,0x06,0x02,0x00,0x02,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x06,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x06,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x06,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x06,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_i16_i32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x24,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x07,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x07,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x07,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x07,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_u16_u32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x23,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x26,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00]
 0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_u8_f32_e64_dpp v255, -|v255|, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x01,0x26,0xd6,0xea,0xfe,0xf7,0x23,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x21,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x21,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x21,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x22,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x22,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x03,0x22,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x13,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x13,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x13,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x13,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x13,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_fma_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x13,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x08,0x01,0x77,0x39,0x05]
 
-# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05
+# GFX12: v_ldexp_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1c,0xd7,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
 
-# GFX12: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
 0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00
+# GFX12: v_ldexp_f32_e64_dpp v255, -|v255|, v255 clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x81,0x1c,0xd7,0xea,0xfe,0x03,0x38,0xff,0x00,0x00,0x00]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lerp_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x15,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lerp_u8_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_lerp_u8_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x15,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_add_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x46,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_lshl_add_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x46,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_lshl_or_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x56,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_lshl_or_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x56,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x38,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshlrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshlrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x38,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x39,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_lshrrev_b16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_lshrrev_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x39,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_i32_i24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u24_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_u32_u24_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x0b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x2a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x2a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x2a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x2a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x2a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x2a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x2a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x2a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_max3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x2a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1d,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_max3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1e,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_max3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0a,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_max_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x09,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_max_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_max_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x09,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6b,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6b,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6b,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6b,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6b,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6b,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x6b,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6b,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x6b,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x6b,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6b,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_maxmin_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x6b,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x69,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x69,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x69,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x69,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x69,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x69,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x69,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x69,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x69,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x69,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x69,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x69,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x69,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x69,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x69,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x69,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x69,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x69,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x69,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x69,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_maxmin_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x69,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_maxmin_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_maxmin_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x62,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_maxmin_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x62,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_mbcnt_hi_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1f,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_mbcnt_lo_u32_b32_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1f,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x31,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x31,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x31,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x31,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x31,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x31,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x31,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x31,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x31,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x31,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x31,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x31,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x31,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x31,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x31,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x31,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x31,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x31,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x31,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x31,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x31,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x31,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_med3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x31,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x20,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_med3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x20,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x21,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_med3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x21,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x29,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x29,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x29,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x29,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x29,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x29,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x29,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x29,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x29,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x29,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x29,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x29,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x29,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x29,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x29,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x29,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x29,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x29,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x29,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x29,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x29,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x29,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_min3_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x29,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_min3_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x1b,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_min3_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x1b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_i16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0c,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_i16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0c,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_min_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_min_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_min_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x0b,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6a,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6a,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6a,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6a,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x6a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x6a,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x6a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_minmax_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x6a,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x68,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x68,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x68,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x68,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x68,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x68,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x68,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x68,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x68,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x68,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x68,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x68,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x68,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x68,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x68,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_num_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x68,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x68,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x68,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_minmax_num_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x68,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_i32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_minmax_i32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x65,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_minmax_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minmax_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_minmax_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_msad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x39,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_msad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x39,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x05,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_mul_lo_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_mul_lo_u16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x40,0x05,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x18,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x18,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x18,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x18,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x18,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x18,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_mullit_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x18,0xd6,0xe9,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_mullit_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x18,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_or3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x58,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_or3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_or3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x58,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_or_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_or_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_or_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x63,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_or_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_or_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_or_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x63,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_perm_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x44,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_perm_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_perm_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x44,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_hi_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x23,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_sad_hi_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x23,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x24,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_sad_u16_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x24,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_sad_u32_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_sad_u8_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x22,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_sad_u8_e64_dpp v255, v255, v255, src_scc clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x22,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_sub_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_sub_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x01,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX12: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_sub_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x01,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_sub_nc_i32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x25,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_sub_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x25,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
+0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, s12, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, s[12:13], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x0c,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, s104, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, s[104:105], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x68,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x6a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32: v_subrev_co_u32_e64_dpp v5, ttmp14, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64: v_subrev_co_u32_e64_dpp v5, ttmp[14:15], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x7a,0x02,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
-# GFX12: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
+# GFX12: v_subrev_co_u32_e64_dpp v255, null, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xfc,0x02,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_xad_u32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x45,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xad_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_xad_u32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x45,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_xor3_b32_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x40,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_xor3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_xor3_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x40,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_xor_b16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_xor_b16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x64,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_xor_b16_e64_dpp v255.l, v255.l, v255.l dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_xor_b16_e64_dpp v255, v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0x00,0x64,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0d,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0d,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x03,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_add_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x03,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x12,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x12,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x12,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_norm_i16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x12,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x13,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x13,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x13,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_cvt_pk_norm_u16_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x13,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x54,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x54,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x54,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x54,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x54,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x54,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_div_fixup_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x54,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_div_fixup_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x54,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, 4.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0xec,0x0d,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0xec,0x0d,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, 4.0, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0xec,0x0d,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x48,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x48,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x48,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x48,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x48,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x48,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_fma_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x48,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_fma_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x48,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x53,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x53,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x53,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x53,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x53,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x53,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_i32_i16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x5a,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_i32_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x5a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x41,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x41,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x41,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x41,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x41,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x41,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, exec_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x59,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_mad_u32_u16_e64_dpp v5, v1, v2, 0.5 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x59,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_mad_u32_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,1,0,0] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x90,0x59,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2c,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x2c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x2c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x2c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x2c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x2c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x2c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x2c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x2c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x2c,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x2c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x2c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_max3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x2c,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x2c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x2c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_max3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x2c,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4d,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4d,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4d,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4d,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_max3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4d,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4e,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4e,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4e,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4e,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_max3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4e,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_max3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4e,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x32,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x32,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x32,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x32,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x32,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x32,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x32,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x32,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x32,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x32,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x32,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x32,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x32,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x32,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x32,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x32,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x32,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x32,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_med3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x32,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x32,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x32,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_med3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x32,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x50,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x50,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x50,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x50,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x50,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_med3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x50,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x51,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x51,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x51,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x51,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_med3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x51,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_med3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x51,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2b,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2b,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, |v1|, v2, -m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2b,0xd6,0xe9,0x04,0xf6,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2b,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x2b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x7c,0x2b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, -v1, v2, |exec_lo| op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7c,0x2b,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x2b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x0b,0x2b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, null op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0b,0x2b,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x2b,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x15,0x2b,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, -|v1|, v2, -|-1| op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x15,0x2b,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x2b,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x26,0x2b,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_min3_num_f16_e64_dpp v5, v1, -|v2|, -|0.5| op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x26,0x2b,0xd6,0xe9,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x2b,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0xc7,0x2b,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_min3_num_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc7,0x2b,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4a,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4a,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4a,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_i16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4a,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_min3_i16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4a,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xee,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4b,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x4b,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, exec_lo op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x4b,0xd6,0xe9,0x04,0xfa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x4b,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05
+# GFX12: v_min3_u16_e64_dpp v5, v1, v2, -1 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x4b,0xd6,0xe9,0x04,0x06,0x03,0x01,0x77,0x39,0x05]
 
-# GFX12: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
+# GFX12: v_min3_u16_e64_dpp v255, v255, v255, src_scc op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x40,0x4b,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_pack_b32_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x11,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_pack_b32_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x11,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_pack_b32_f16_e64_dpp v5, -v1, |v2| op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0a,0x11,0xd7,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_pack_b32_f16_e64_dpp v255, -|v255|, -|v255| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
+0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x0e,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_i16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_i16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x0e,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x00,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
+0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.h, v1.h, v2.h op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x58,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.h, v2.l op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x08,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # W32-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v5.l, v1.l, v2.h op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v5, v1, v2 op_sel:[0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
-0x05,0x10,0x04,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 
+0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 # W32-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_sub_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 # op_sel[1:0] are ignored
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x78,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-# GFX12: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+
+0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 # op_sel[1:0] are ignored
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x78,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-# GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+
+0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
 
-# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
 0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt
index 51c2906608c83..86764399c91d2 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt
@@ -216,6 +216,60 @@
 # GFX12: v_dual_add_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x12,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x12,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x18,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0x18,0xc9,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x18,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0x18,0xc9,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x18,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0x18,0xc9,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x18,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0x18,0xc9,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x18,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0x18,0xc9,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x18,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0x18,0xc9,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x18,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0x18,0xc9,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x18,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x18,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x18,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x18,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x18,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x18,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x18,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x18,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x18,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x18,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x18,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x18,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x18,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x18,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x18,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x18,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x18,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x18,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x18,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x18,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x18,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x18,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_add_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x02,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x02,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -1026,6 +1080,60 @@
 # GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x52,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x52,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_cndmask_b32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x58,0xca,0x01,0x07,0x06,0xff]
+0x04,0x05,0x58,0xca,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x58,0xca,0xff,0x07,0x06,0xff]
+0x01,0x05,0x58,0xca,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x58,0xca,0x02,0x07,0x06,0xff]
+0xff,0x05,0x58,0xca,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x58,0xca,0x03,0x07,0x06,0xff]
+0x02,0x05,0x58,0xca,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x58,0xca,0x04,0x07,0x06,0xff]
+0x03,0x05,0x58,0xca,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x69,0x04,0x58,0xca,0x69,0x06,0x06,0xff]
+0x69,0x04,0x58,0xca,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x01,0x04,0x58,0xca,0x01,0x06,0x06,0xff]
+0x01,0x04,0x58,0xca,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7b,0x04,0x58,0xca,0x7b,0x06,0x06,0xff]
+0x7b,0x04,0x58,0xca,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x7f,0x04,0x58,0xca,0x7f,0x06,0x06,0xff]
+0x7f,0x04,0x58,0xca,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x7e,0x04,0x58,0xca,0x7e,0x06,0x06,0xff]
+0x7e,0x04,0x58,0xca,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x58,0xca,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x58,0xca,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x6b,0x04,0x58,0xca,0x6b,0x06,0x06,0xff]
+0x6b,0x04,0x58,0xca,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x6a,0x04,0x58,0xca,0x6a,0x06,0x06,0xff]
+0x6a,0x04,0x58,0xca,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x58,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x58,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x58,0xca,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x58,0xca,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x58,0xca,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x58,0xca,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x58,0xca,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x58,0xca,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x58,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x58,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_cndmask_b32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x42,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x42,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -1503,68 +1611,926 @@
 # GFX12: v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_sub_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x4a,0xca,0xc1,0x06,0x06,0xff]
 0xfd,0x04,0x4a,0xca,0xc1,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_sub_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x4a,0xca,0xf0,0x04,0x06,0xff]
-0xf0,0x06,0x4a,0xca,0xf0,0x04,0x06,0xff
+# GFX12: v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_sub_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x4a,0xca,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x4a,0xca,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_cndmask_b32 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff]
+0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff]
+0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff]
+0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff]
+0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff]
+0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, s105, v2 :: v_dual_subrev_f32 v6, s105, v3 ; encoding: [0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff]
+0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, s1, v2 :: v_dual_subrev_f32 v6, s1, v3 ; encoding: [0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff]
+0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, ttmp15, v3 ; encoding: [0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff]
+0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, exec_hi, v3 ; encoding: [0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff]
+0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, exec_lo, v3 ; encoding: [0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff]
+0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3 ; encoding: [0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff]
+0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3 ; encoding: [0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff]
+0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_subrev_f32 v6, null, v3 ; encoding: [0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x08,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x08,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x08,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x08,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x08,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x08,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x08,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x08,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x08,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x08,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x08,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x08,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x08,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x08,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x08,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x08,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x08,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x08,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x08,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x08,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x08,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x08,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x08,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x08,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x08,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x08,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3 ; encoding: [0xff,0x04,0x08,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x08,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x08,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x08,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x08,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x08,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x08,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x08,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x08,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x08,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_add_nc_u32 v6, v1, v3 ; encoding: [0x04,0x05,0x20,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x20,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_add_nc_u32 v6, v255, v3 ; encoding: [0x01,0x05,0x20,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x20,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_add_nc_u32 v6, v2, v3 ; encoding: [0xff,0x05,0x20,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x20,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_add_nc_u32 v6, v3, v3 ; encoding: [0x02,0x05,0x20,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x20,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_add_nc_u32 v6, v4, v3 ; encoding: [0x03,0x05,0x20,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x20,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_add_nc_u32 v6, s1, v3 ; encoding: [0x69,0x04,0x20,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x20,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_add_nc_u32 v6, s105, v3 ; encoding: [0x01,0x04,0x20,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x20,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_add_nc_u32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x20,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x20,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_add_nc_u32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x20,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x20,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_add_nc_u32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x20,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x20,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_add_nc_u32 v6, m0, v3 ; encoding: [0x7d,0x04,0x20,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x20,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_add_nc_u32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x20,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x20,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_add_nc_u32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x20,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x20,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_add_nc_u32 v6, null, v3 ; encoding: [0xff,0x04,0x20,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x20,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_add_nc_u32 v6, -1, v3 ; encoding: [0xfd,0x04,0x20,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x20,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_add_nc_u32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x20,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x20,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_add_nc_u32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x20,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x20,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_add_nc_u32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x20,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x20,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_and_b32 v6, v1, v3 ; encoding: [0x04,0x05,0x24,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x24,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_and_b32 v6, v255, v3 ; encoding: [0x01,0x05,0x24,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x24,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_and_b32 v6, v2, v3 ; encoding: [0xff,0x05,0x24,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x24,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_and_b32 v6, v3, v3 ; encoding: [0x02,0x05,0x24,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x24,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_and_b32 v6, v4, v3 ; encoding: [0x03,0x05,0x24,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x24,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_and_b32 v6, s1, v3 ; encoding: [0x69,0x04,0x24,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x24,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_and_b32 v6, s105, v3 ; encoding: [0x01,0x04,0x24,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x24,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_and_b32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x24,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x24,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_and_b32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x24,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x24,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_and_b32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x24,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x24,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_and_b32 v6, m0, v3 ; encoding: [0x7d,0x04,0x24,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x24,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_and_b32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x24,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x24,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_and_b32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x24,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x24,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_and_b32 v6, null, v3 ; encoding: [0xff,0x04,0x24,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x24,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_and_b32 v6, -1, v3 ; encoding: [0xfd,0x04,0x24,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x24,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_and_b32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x24,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x24,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_and_b32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x24,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x24,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_and_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x24,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x24,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_cndmask_b32 v6, v1, v3 ; encoding: [0x04,0x05,0x12,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x12,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_cndmask_b32 v6, v255, v3 ; encoding: [0x01,0x05,0x12,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x12,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_cndmask_b32 v6, v2, v3 ; encoding: [0xff,0x05,0x12,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x12,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_cndmask_b32 v6, v3, v3 ; encoding: [0x02,0x05,0x12,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x12,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_cndmask_b32 v6, v4, v3 ; encoding: [0x03,0x05,0x12,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x12,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_cndmask_b32 v6, s105, v3 ; encoding: [0x69,0x04,0x12,0xcb,0x69,0x06,0x06,0xff]
+0x69,0x04,0x12,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_cndmask_b32 v6, s1, v3 ; encoding: [0x01,0x04,0x12,0xcb,0x01,0x06,0x06,0xff]
+0x01,0x04,0x12,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_cndmask_b32 v6, ttmp15, v3 ; encoding: [0x7b,0x04,0x12,0xcb,0x7b,0x06,0x06,0xff]
+0x7b,0x04,0x12,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_cndmask_b32 v6, exec_hi, v3 ; encoding: [0x7f,0x04,0x12,0xcb,0x7f,0x06,0x06,0xff]
+0x7f,0x04,0x12,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_cndmask_b32 v6, exec_lo, v3 ; encoding: [0x7e,0x04,0x12,0xcb,0x7e,0x06,0x06,0xff]
+0x7e,0x04,0x12,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_cndmask_b32 v6, m0, v3 ; encoding: [0x7d,0x04,0x12,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x12,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_cndmask_b32 v6, vcc_hi, v3 ; encoding: [0x6b,0x04,0x12,0xcb,0x6b,0x06,0x06,0xff]
+0x6b,0x04,0x12,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_cndmask_b32 v6, vcc_lo, v3 ; encoding: [0x6a,0x04,0x12,0xcb,0x6a,0x06,0x06,0xff]
+0x6a,0x04,0x12,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_cndmask_b32 v6, null, v3 ; encoding: [0xff,0x04,0x12,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x12,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_cndmask_b32 v6, -1, v3 ; encoding: [0xfd,0x04,0x12,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x12,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_cndmask_b32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x12,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x12,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_cndmask_b32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x12,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x12,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x12,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x12,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x18,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x18,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x18,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x18,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x18,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x18,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x18,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x18,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x18,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x18,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x18,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x18,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x18,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x18,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x18,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x18,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x18,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x18,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x18,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x18,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x18,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x18,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x18,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x18,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x18,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x18,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x18,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x18,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x18,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x18,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x18,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x18,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x18,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x18,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x18,0xcb,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x18,0xcb,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_fmaak_f32 v6, v255, v3, 0xaf123456 ; encoding: [0x04,0x05,0x02,0xcb,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x04,0x05,0x02,0xcb,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_fmaak_f32 v6, v3, v3, 0xaf123456 ; encoding: [0x01,0x05,0x02,0xcb,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0x05,0x02,0xcb,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_fmaak_f32 v6, v2, v3, 0xaf123456 ; encoding: [0xff,0x05,0x02,0xcb,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x05,0x02,0xcb,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_fmaak_f32 v6, v4, v3, 0xaf123456 ; encoding: [0x02,0x05,0x02,0xcb,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x02,0x05,0x02,0xcb,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_fmaak_f32 v6, vcc_lo, v3, 0xaf123456 ; encoding: [0x03,0x05,0x02,0xcb,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x03,0x05,0x02,0xcb,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_fmaak_f32 v6, s105, v3, 0xaf123456 ; encoding: [0x69,0x04,0x02,0xcb,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x69,0x04,0x02,0xcb,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_fmaak_f32 v6, s1, v3, 0xaf123456 ; encoding: [0x01,0x04,0x02,0xcb,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0x04,0x02,0xcb,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_fmaak_f32 v6, ttmp15, v3, 0xaf123456 ; encoding: [0x7b,0x04,0x02,0xcb,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7b,0x04,0x02,0xcb,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_fmaak_f32 v6, exec_hi, v3, 0xaf123456 ; encoding: [0x7f,0x04,0x02,0xcb,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7f,0x04,0x02,0xcb,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_fmaak_f32 v6, exec_lo, v3, 0xaf123456 ; encoding: [0x7e,0x04,0x02,0xcb,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7e,0x04,0x02,0xcb,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_fmaak_f32 v6, m0, v3, 0xaf123456 ; encoding: [0x7d,0x04,0x02,0xcb,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7d,0x04,0x02,0xcb,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_fmaak_f32 v6, vcc_hi, v3, 0xaf123456 ; encoding: [0x6b,0x04,0x02,0xcb,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6b,0x04,0x02,0xcb,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_fmaak_f32 v6, null, v3, 0xaf123456 ; encoding: [0x6a,0x04,0x02,0xcb,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6a,0x04,0x02,0xcb,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_fmaak_f32 v6, -1, v3, 0xaf123456 ; encoding: [0xfd,0x04,0x02,0xcb,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xfd,0x04,0x02,0xcb,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmaak_f32 v6, 0.5, v2, 0xaf123456 ; encoding: [0xf0,0x06,0x02,0xcb,0xf0,0x04,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xf0,0x06,0x02,0xcb,0xf0,0x04,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmaak_f32 v6, src_scc, v5, 0xaf123456 ; encoding: [0xc1,0x08,0x02,0xcb,0xfd,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xc1,0x08,0x02,0xcb,0xfd,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmaak_f32 v255, 0xaf123456, v4, 0xaf123456 ; encoding: [0x7c,0x0a,0x02,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x02,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_fmac_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x00,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x00,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_fmac_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x00,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x00,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_fmac_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x00,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x00,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_fmac_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x00,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x00,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_fmac_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x00,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x00,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_fmac_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x00,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x00,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_fmac_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x00,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x00,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_fmac_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x00,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x00,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_fmac_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x00,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x00,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_fmac_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x00,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x00,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_fmac_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x00,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x00,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_fmac_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x00,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x00,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_fmac_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x00,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x00,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_fmac_f32 v6, null, v3 ; encoding: [0xff,0x04,0x00,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x00,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_fmac_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x00,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x00,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmac_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x00,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x00,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmac_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x00,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x00,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmac_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x00,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x00,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v255 :: v_dual_fmamk_f32 v6, v255, 0xaf123456, v4 ; encoding: [0x04,0xff,0x05,0xcb,0xff,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x04,0xff,0x05,0xcb,0xff,0x09,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v255 :: v_dual_fmamk_f32 v6, v3, 0xaf123456, v4 ; encoding: [0x01,0xff,0x05,0xcb,0x03,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0xff,0x05,0xcb,0x03,0x09,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v255 :: v_dual_fmamk_f32 v6, v2, 0xaf123456, v4 ; encoding: [0xff,0xff,0x05,0xcb,0x02,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0xff,0x05,0xcb,0x02,0x09,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v255 :: v_dual_fmamk_f32 v6, v4, 0xaf123456, v4 ; encoding: [0x02,0xff,0x05,0xcb,0x04,0x09,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x02,0xff,0x05,0xcb,0x04,0x09,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v255 :: v_dual_fmamk_f32 v6, vcc_lo, 0xaf123456, v4 ; encoding: [0x03,0xff,0x05,0xcb,0x6a,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x03,0xff,0x05,0xcb,0x6a,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v255 :: v_dual_fmamk_f32 v6, s105, 0xaf123456, v4 ; encoding: [0x69,0xfe,0x05,0xcb,0x69,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x69,0xfe,0x05,0xcb,0x69,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v255 :: v_dual_fmamk_f32 v6, s1, 0xaf123456, v4 ; encoding: [0x01,0xfe,0x05,0xcb,0x01,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0xfe,0x05,0xcb,0x01,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v255 :: v_dual_fmamk_f32 v6, ttmp15, 0xaf123456, v4 ; encoding: [0x7b,0xfe,0x05,0xcb,0x7b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7b,0xfe,0x05,0xcb,0x7b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v255 :: v_dual_fmamk_f32 v6, exec_hi, 0xaf123456, v4 ; encoding: [0x7f,0xfe,0x05,0xcb,0x7f,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7f,0xfe,0x05,0xcb,0x7f,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v255 :: v_dual_fmamk_f32 v6, exec_lo, 0xaf123456, v4 ; encoding: [0x7e,0xfe,0x05,0xcb,0x7e,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7e,0xfe,0x05,0xcb,0x7e,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v255 :: v_dual_fmamk_f32 v6, m0, 0xaf123456, v4 ; encoding: [0x7d,0xfe,0x05,0xcb,0x7d,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7d,0xfe,0x05,0xcb,0x7d,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v255 :: v_dual_fmamk_f32 v6, vcc_hi, 0xaf123456, v4 ; encoding: [0x6b,0xfe,0x05,0xcb,0x6b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6b,0xfe,0x05,0xcb,0x6b,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v255 :: v_dual_fmamk_f32 v6, null, 0xaf123456, v4 ; encoding: [0x6a,0xfe,0x05,0xcb,0x7c,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6a,0xfe,0x05,0xcb,0x7c,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v255 :: v_dual_fmamk_f32 v6, -1, 0xaf123456, v4 ; encoding: [0xfd,0xfe,0x05,0xcb,0xc1,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xfd,0xfe,0x05,0xcb,0xc1,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_fmamk_f32 v6, 0.5, 0xaf123456, v4 ; encoding: [0xf0,0x06,0x04,0xcb,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xf0,0x06,0x04,0xcb,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_fmamk_f32 v6, src_scc, 0xaf123456, v4 ; encoding: [0xc1,0x08,0x04,0xcb,0xfd,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xc1,0x08,0x04,0xcb,0xfd,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 ; encoding: [0x7c,0x0a,0x04,0xcb,0xff,0xfe,0xff,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x04,0xcb,0xff,0xfe,0xff,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_lshlrev_b32 v6, v1, v3 ; encoding: [0x04,0x05,0x22,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x22,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_lshlrev_b32 v6, v255, v3 ; encoding: [0x01,0x05,0x22,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x22,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_lshlrev_b32 v6, v2, v3 ; encoding: [0xff,0x05,0x22,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x22,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_lshlrev_b32 v6, v3, v3 ; encoding: [0x02,0x05,0x22,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x22,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_lshlrev_b32 v6, v4, v3 ; encoding: [0x03,0x05,0x22,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x22,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_lshlrev_b32 v6, s1, v3 ; encoding: [0x69,0x04,0x22,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x22,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_lshlrev_b32 v6, s105, v3 ; encoding: [0x01,0x04,0x22,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x22,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_lshlrev_b32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x22,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x22,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_lshlrev_b32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x22,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x22,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_lshlrev_b32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x22,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x22,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_lshlrev_b32 v6, m0, v3 ; encoding: [0x7d,0x04,0x22,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x22,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_lshlrev_b32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x22,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x22,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_lshlrev_b32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x22,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x22,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_lshlrev_b32 v6, null, v3 ; encoding: [0xff,0x04,0x22,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x22,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_lshlrev_b32 v6, -1, v3 ; encoding: [0xfd,0x04,0x22,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x22,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_lshlrev_b32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x22,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x22,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_lshlrev_b32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x22,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x22,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_lshlrev_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x22,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x22,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_max_num_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x14,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x14,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_max_num_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x14,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x14,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_max_num_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x14,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x14,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_max_num_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x14,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x14,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_max_num_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x14,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x14,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_max_num_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x14,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x14,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_max_num_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x14,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x14,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_max_num_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x14,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x14,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_max_num_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x14,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x14,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_max_num_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x14,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x14,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_max_num_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x14,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x14,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_max_num_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x14,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x14,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_max_num_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x14,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x14,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_max_num_f32 v6, null, v3 ; encoding: [0xff,0x04,0x14,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x14,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_max_num_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x14,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x14,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_max_num_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x14,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x14,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_max_num_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x14,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x14,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_max_num_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x14,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x14,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_min_num_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x16,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x16,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_min_num_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x16,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x16,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_min_num_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x16,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x16,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_min_num_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x16,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x16,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_min_num_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x16,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x16,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_min_num_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x16,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x16,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_min_num_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x16,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x16,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_min_num_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x16,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x16,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_min_num_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x16,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x16,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_min_num_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x16,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x16,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_min_num_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x16,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x16,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_min_num_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x16,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x16,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_min_num_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x16,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x16,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_min_num_f32 v6, null, v3 ; encoding: [0xff,0x04,0x16,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x16,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_min_num_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x16,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x16,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_min_num_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x16,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x16,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_min_num_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x16,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x16,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_min_num_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x16,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x16,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v255 :: v_dual_mov_b32 v6, v1 ; encoding: [0x04,0xff,0x11,0xcb,0x01,0x01,0x06,0xff]
+0x04,0xff,0x11,0xcb,0x01,0x01,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v255 :: v_dual_mov_b32 v6, v255 ; encoding: [0x01,0xff,0x11,0xcb,0xff,0x01,0x06,0xff]
+0x01,0xff,0x11,0xcb,0xff,0x01,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v255 :: v_dual_mov_b32 v6, v2 ; encoding: [0xff,0xff,0x11,0xcb,0x02,0x01,0x06,0xff]
+0xff,0xff,0x11,0xcb,0x02,0x01,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v255 :: v_dual_mov_b32 v6, v3 ; encoding: [0x02,0xff,0x11,0xcb,0x03,0x01,0x06,0xff]
+0x02,0xff,0x11,0xcb,0x03,0x01,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v255 :: v_dual_mov_b32 v6, v4 ; encoding: [0x03,0xff,0x11,0xcb,0x04,0x01,0x06,0xff]
+0x03,0xff,0x11,0xcb,0x04,0x01,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v255 :: v_dual_mov_b32 v6, s1 ; encoding: [0x69,0xfe,0x11,0xcb,0x01,0x00,0x06,0xff]
+0x69,0xfe,0x11,0xcb,0x01,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v255 :: v_dual_mov_b32 v6, s105 ; encoding: [0x01,0xfe,0x11,0xcb,0x69,0x00,0x06,0xff]
+0x01,0xfe,0x11,0xcb,0x69,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v255 :: v_dual_mov_b32 v6, vcc_lo ; encoding: [0x7b,0xfe,0x11,0xcb,0x6a,0x00,0x06,0xff]
+0x7b,0xfe,0x11,0xcb,0x6a,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v255 :: v_dual_mov_b32 v6, vcc_hi ; encoding: [0x7f,0xfe,0x11,0xcb,0x6b,0x00,0x06,0xff]
+0x7f,0xfe,0x11,0xcb,0x6b,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v255 :: v_dual_mov_b32 v6, ttmp15 ; encoding: [0x7e,0xfe,0x11,0xcb,0x7b,0x00,0x06,0xff]
+0x7e,0xfe,0x11,0xcb,0x7b,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v255 :: v_dual_mov_b32 v6, m0 ; encoding: [0x7d,0xfe,0x11,0xcb,0x7d,0x00,0x06,0xff]
+0x7d,0xfe,0x11,0xcb,0x7d,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v255 :: v_dual_mov_b32 v6, exec_lo ; encoding: [0x6b,0xfe,0x11,0xcb,0x7e,0x00,0x06,0xff]
+0x6b,0xfe,0x11,0xcb,0x7e,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v255 :: v_dual_mov_b32 v6, exec_hi ; encoding: [0x6a,0xfe,0x11,0xcb,0x7f,0x00,0x06,0xff]
+0x6a,0xfe,0x11,0xcb,0x7f,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v255 :: v_dual_mov_b32 v6, null ; encoding: [0xff,0xfe,0x11,0xcb,0x7c,0x00,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0xfe,0x11,0xcb,0x7c,0x00,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v255 :: v_dual_mov_b32 v6, -1 ; encoding: [0xfd,0xfe,0x11,0xcb,0xc1,0x00,0x06,0xff]
+0xfd,0xfe,0x11,0xcb,0xc1,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mov_b32 v6, 0.5 ; encoding: [0xf0,0x06,0x10,0xcb,0xf0,0x00,0x06,0xff]
+0xf0,0x06,0x10,0xcb,0xf0,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mov_b32 v6, src_scc ; encoding: [0xc1,0x08,0x10,0xcb,0xfd,0x00,0x06,0xff]
+0xc1,0x08,0x10,0xcb,0xfd,0x00,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mov_b32 v255, 0xaf123456 ; encoding: [0x7c,0x0a,0x10,0xcb,0xff,0x00,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x10,0xcb,0xff,0x00,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_mul_dx9_zero_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x0e,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x0e,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_mul_dx9_zero_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x0e,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x0e,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_mul_dx9_zero_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x0e,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x0e,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_mul_dx9_zero_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x0e,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x0e,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_mul_dx9_zero_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x0e,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x0e,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_mul_dx9_zero_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x0e,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x0e,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_mul_dx9_zero_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x0e,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x0e,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_mul_dx9_zero_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x0e,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x0e,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_mul_dx9_zero_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x0e,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x0e,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_mul_dx9_zero_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x0e,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x0e,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_mul_dx9_zero_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x0e,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x0e,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_mul_dx9_zero_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x0e,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x0e,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_mul_dx9_zero_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x0e,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x0e,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_mul_dx9_zero_f32 v6, null, v3 ; encoding: [0xff,0x04,0x0e,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x0e,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_mul_dx9_zero_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x0e,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x0e,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mul_dx9_zero_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x0e,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x0e,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mul_dx9_zero_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x0e,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x0e,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mul_dx9_zero_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x0e,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x0e,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_mul_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x06,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x06,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_mul_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x06,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x06,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_mul_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x06,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x06,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_mul_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x06,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x06,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_mul_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x06,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x06,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_mul_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x06,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x06,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_mul_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x06,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x06,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_mul_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x06,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x06,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_mul_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x06,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x06,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_mul_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x06,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x06,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_mul_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x06,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x06,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_mul_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x06,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x06,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_mul_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x06,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x06,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_mul_f32 v6, null, v3 ; encoding: [0xff,0x04,0x06,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x06,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_mul_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x06,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x06,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_mul_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x06,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x06,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_mul_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x06,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x06,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_mul_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x06,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x06,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_sub_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x0a,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x0a,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_sub_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x0a,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x0a,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_sub_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x0a,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x0a,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_sub_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x0a,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x0a,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_sub_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x0a,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x0a,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_sub_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x0a,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x0a,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_sub_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x0a,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x0a,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_sub_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x0a,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x0a,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_sub_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x0a,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x0a,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_sub_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x0a,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x0a,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_sub_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x0a,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x0a,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_sub_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x0a,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x0a,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_sub_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x0a,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x0a,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_sub_f32 v6, null, v3 ; encoding: [0xff,0x04,0x0a,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x0a,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_sub_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x0a,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x0a,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_sub_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x0a,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x0a,0xcb,0xf0,0x04,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff]
-0xc1,0x08,0x4a,0xca,0xfd,0x0a,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_sub_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x0a,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x0a,0xcb,0xfd,0x0a,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
-0x7c,0x0a,0x4a,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_sub_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x0a,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x0a,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
-# GFX12: v_dual_cndmask_b32 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff]
-0x04,0x05,0x4c,0xca,0x01,0x07,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, v4, v2 :: v_dual_subrev_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x0c,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x0c,0xcb,0x01,0x07,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff]
-0x01,0x05,0x4c,0xca,0xff,0x07,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, v1, v2 :: v_dual_subrev_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x0c,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x0c,0xcb,0xff,0x07,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff]
-0xff,0x05,0x4c,0xca,0x02,0x07,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, v255, v2 :: v_dual_subrev_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x0c,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x0c,0xcb,0x02,0x07,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff]
-0x02,0x05,0x4c,0xca,0x03,0x07,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, v2, v2 :: v_dual_subrev_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x0c,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x0c,0xcb,0x03,0x07,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff]
-0x03,0x05,0x4c,0xca,0x04,0x07,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, v3, v2 :: v_dual_subrev_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x0c,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x0c,0xcb,0x04,0x07,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, s105, v2 :: v_dual_subrev_f32 v6, s105, v3 ; encoding: [0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff]
-0x69,0x04,0x4c,0xca,0x69,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, s105, v2 :: v_dual_subrev_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x0c,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x0c,0xcb,0x01,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, s1, v2 :: v_dual_subrev_f32 v6, s1, v3 ; encoding: [0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff]
-0x01,0x04,0x4c,0xca,0x01,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, s1, v2 :: v_dual_subrev_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x0c,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x0c,0xcb,0x69,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, ttmp15, v3 ; encoding: [0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff]
-0x7b,0x04,0x4c,0xca,0x7b,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, ttmp15, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x0c,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x0c,0xcb,0x6a,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, exec_hi, v3 ; encoding: [0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff]
-0x7f,0x04,0x4c,0xca,0x7f,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x0c,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x0c,0xcb,0x6b,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, exec_lo, v3 ; encoding: [0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff]
-0x7e,0x04,0x4c,0xca,0x7e,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, exec_lo, v2 :: v_dual_subrev_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x0c,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x0c,0xcb,0x7b,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff]
-0x7d,0x04,0x4c,0xca,0x7d,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, m0, v2 :: v_dual_subrev_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x0c,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x0c,0xcb,0x7d,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, vcc_hi, v3 ; encoding: [0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff]
-0x6b,0x04,0x4c,0xca,0x6b,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_hi, v2 :: v_dual_subrev_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x0c,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x0c,0xcb,0x7e,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, vcc_lo, v3 ; encoding: [0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff]
-0x6a,0x04,0x4c,0xca,0x6a,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, vcc_lo, v2 :: v_dual_subrev_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x0c,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x0c,0xcb,0x7f,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, 0xaf123456, v2 :: v_dual_subrev_f32 v6, null, v3 ; encoding: [0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
-0xff,0x04,0x4c,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v2 :: v_dual_subrev_f32 v6, null, v3 ; encoding: [0xff,0x04,0x0c,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x0c,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
 
-# GFX12: v_dual_cndmask_b32 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff]
-0xfd,0x04,0x4c,0xca,0xc1,0x06,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, src_scc, v2 :: v_dual_subrev_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x0c,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x0c,0xcb,0xc1,0x06,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff]
-0xf0,0x06,0x4c,0xca,0xf0,0x04,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, 0.5, v3 :: v_dual_subrev_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x0c,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x0c,0xcb,0xf0,0x04,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff]
-0xc1,0x08,0x4c,0xca,0xfd,0x0a,0x06,0xff
+# GFX12: v_dual_dot2acc_f32_f16 v255, -1, v4 :: v_dual_subrev_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x0c,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x0c,0xcb,0xfd,0x0a,0x06,0xff
 
-# GFX12: v_dual_cndmask_b32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
-0x7c,0x0a,0x4c,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+# GFX12: v_dual_dot2acc_f32_f16 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x0c,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x0c,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
 # GFX12: v_dual_fmaak_f32 v255, v4, v2, 0xaf123456 :: v_dual_add_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x48,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x48,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
@@ -1755,6 +2721,57 @@
 # GFX12: v_dual_fmaak_f32 v6, null, v5, 0xaf123456 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x52,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x52,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_fmaak_f32 v255, v1, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x58,0xc8,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0x05,0x58,0xc8,0xff,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, v255, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0xff,0x05,0x58,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x05,0x58,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, v2, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x58,0xc8,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x02,0x05,0x58,0xc8,0x03,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, v3, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0x03,0x05,0x58,0xc8,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x03,0x05,0x58,0xc8,0x02,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, s105, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x69,0x04,0x58,0xc8,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x69,0x04,0x58,0xc8,0x69,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, s1, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x01,0x04,0x58,0xc8,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x01,0x04,0x58,0xc8,0x01,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, ttmp15, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7b,0x04,0x58,0xc8,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7b,0x04,0x58,0xc8,0x7b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, exec_hi, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x7f,0x04,0x58,0xc8,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7f,0x04,0x58,0xc8,0x04,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, exec_lo, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x7e,0x04,0x58,0xc8,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7e,0x04,0x58,0xc8,0x7e,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, m0, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x58,0xc8,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x7d,0x04,0x58,0xc8,0x7d,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, vcc_hi, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x6b,0x04,0x58,0xc8,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6b,0x04,0x58,0xc8,0x6b,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, vcc_lo, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x6a,0x04,0x58,0xc8,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0x6a,0x04,0x58,0xc8,0x6a,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, 0xaf123456, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0xff,0x04,0x58,0xc8,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x58,0xc8,0x7f,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, src_scc, v2, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xfd,0x04,0x58,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xfd,0x04,0x58,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, 0.5, v3, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, -1, v2 ; encoding: [0xf0,0x06,0x58,0xc8,0xc1,0x04,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xf0,0x06,0x58,0xc8,0xc1,0x04,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v255, -1, v4, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, 0.5, v5 ; encoding: [0xc1,0x08,0x58,0xc8,0xf0,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xc1,0x08,0x58,0xc8,0xf0,0x0a,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmaak_f32 v6, null, v5, 0xaf123456 :: v_dual_dot2acc_f32_f16 v255, src_scc, v4 ; encoding: [0x7c,0x0a,0x58,0xc8,0xfd,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x58,0xc8,0xfd,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
 # GFX12: v_dual_fmaak_f32 v255, v4, v2, 0xaf123456 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x42,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x42,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -2565,6 +3582,60 @@
 # GFX12: v_dual_fmac_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x12,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x12,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_fmac_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x18,0xc8,0x01,0x07,0x06,0xff]
+0x04,0x05,0x18,0xc8,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x18,0xc8,0xff,0x07,0x06,0xff]
+0x01,0x05,0x18,0xc8,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x18,0xc8,0x02,0x07,0x06,0xff]
+0xff,0x05,0x18,0xc8,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x18,0xc8,0x03,0x07,0x06,0xff]
+0x02,0x05,0x18,0xc8,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x18,0xc8,0x04,0x07,0x06,0xff]
+0x03,0x05,0x18,0xc8,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x18,0xc8,0x01,0x06,0x06,0xff]
+0x69,0x04,0x18,0xc8,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x18,0xc8,0x69,0x06,0x06,0xff]
+0x01,0x04,0x18,0xc8,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x18,0xc8,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x18,0xc8,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x18,0xc8,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x18,0xc8,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x18,0xc8,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x18,0xc8,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x18,0xc8,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x18,0xc8,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x18,0xc8,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x18,0xc8,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x18,0xc8,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x18,0xc8,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x18,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x18,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmac_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x18,0xc8,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x18,0xc8,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x18,0xc8,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x18,0xc8,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x18,0xc8,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x18,0xc8,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_fmac_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x18,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x18,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_fmac_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x02,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x02,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -3348,6 +4419,57 @@
 # GFX12: v_dual_fmamk_f32 v6, null, 0xaf123456, v4 :: v_dual_cndmask_b32 v255, 0xaf123456, v5 ; encoding: [0x7c,0x08,0x92,0xc8,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x08,0x92,0xc8,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_fmamk_f32 v255, v1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v255, v255 ; encoding: [0x01,0xff,0x99,0xc8,0xff,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x01,0xff,0x99,0xc8,0xff,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, v255, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v1, v255 ; encoding: [0xff,0xff,0x99,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+0xff,0xff,0x99,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, v2, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v3, v255 ; encoding: [0x02,0xff,0x99,0xc8,0x03,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x02,0xff,0x99,0xc8,0x03,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, v3, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v2, v255 ; encoding: [0x03,0xff,0x99,0xc8,0x02,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x03,0xff,0x99,0xc8,0x02,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, s105, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, s105, v255 ; encoding: [0x69,0xfe,0x99,0xc8,0x69,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x69,0xfe,0x99,0xc8,0x69,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, s1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, s1, v255 ; encoding: [0x01,0xfe,0x99,0xc8,0x01,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x01,0xfe,0x99,0xc8,0x01,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, ttmp15, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v255 ; encoding: [0x7b,0xfe,0x99,0xc8,0x7b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x7b,0xfe,0x99,0xc8,0x7b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, exec_hi, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, v4, v255 ; encoding: [0x7f,0xfe,0x99,0xc8,0x04,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x7f,0xfe,0x99,0xc8,0x04,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, exec_lo, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v255 ; encoding: [0x7e,0xfe,0x99,0xc8,0x7e,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x7e,0xfe,0x99,0xc8,0x7e,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, m0, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, m0, v255 ; encoding: [0x7d,0xfe,0x99,0xc8,0x7d,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x7d,0xfe,0x99,0xc8,0x7d,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, vcc_hi, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v255 ; encoding: [0x6b,0xfe,0x99,0xc8,0x6b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x6b,0xfe,0x99,0xc8,0x6b,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, vcc_lo, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v255 ; encoding: [0x6a,0xfe,0x99,0xc8,0x6a,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0x6a,0xfe,0x99,0xc8,0x6a,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, 0xaf123456, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v255 ; encoding: [0xff,0xfe,0x99,0xc8,0x7f,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0xff,0xfe,0x99,0xc8,0x7f,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, src_scc, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, null, v255 ; encoding: [0xfd,0xfe,0x99,0xc8,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0xfd,0xfe,0x99,0xc8,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, 0.5, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xf0,0xfe,0x99,0xc8,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xf0,0xfe,0x99,0xc8,0xc1,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v255, -1, 0xaf123456, v255 :: v_dual_dot2acc_f32_f16 v6, 0.5, v4 ; encoding: [0xc1,0xfe,0x99,0xc8,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xc1,0xfe,0x99,0xc8,0xf0,0x08,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_fmamk_f32 v6, null, 0xaf123456, v4 :: v_dual_dot2acc_f32_f16 v255, src_scc, v5 ; encoding: [0x7c,0x08,0x98,0xc8,0xfd,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x08,0x98,0xc8,0xfd,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf
+
 # GFX12: v_dual_fmamk_f32 v255, v4, 0xaf123456, v255 :: v_dual_fmaak_f32 v6, v1, v255, 0xaf123456 ; encoding: [0x04,0xff,0x83,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
 0x04,0xff,0x83,0xc8,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
 
@@ -4158,6 +5280,60 @@
 # GFX12: v_dual_max_num_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x92,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x92,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_max_num_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x98,0xca,0x01,0x07,0x06,0xff]
+0x04,0x05,0x98,0xca,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x98,0xca,0xff,0x07,0x06,0xff]
+0x01,0x05,0x98,0xca,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x98,0xca,0x02,0x07,0x06,0xff]
+0xff,0x05,0x98,0xca,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x98,0xca,0x03,0x07,0x06,0xff]
+0x02,0x05,0x98,0xca,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x98,0xca,0x04,0x07,0x06,0xff]
+0x03,0x05,0x98,0xca,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x98,0xca,0x01,0x06,0x06,0xff]
+0x69,0x04,0x98,0xca,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x98,0xca,0x69,0x06,0x06,0xff]
+0x01,0x04,0x98,0xca,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x98,0xca,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x98,0xca,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x98,0xca,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x98,0xca,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x98,0xca,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x98,0xca,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x98,0xca,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x98,0xca,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x98,0xca,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x98,0xca,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x98,0xca,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x98,0xca,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x98,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x98,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_max_num_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x98,0xca,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x98,0xca,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x98,0xca,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x98,0xca,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x98,0xca,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x98,0xca,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_max_num_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x98,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x98,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_max_num_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x82,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x82,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -4968,6 +6144,60 @@
 # GFX12: v_dual_min_num_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0xd2,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0xd2,0xca,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_min_num_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0xd8,0xca,0x01,0x07,0x06,0xff]
+0x04,0x05,0xd8,0xca,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0xd8,0xca,0xff,0x07,0x06,0xff]
+0x01,0x05,0xd8,0xca,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0xd8,0xca,0x02,0x07,0x06,0xff]
+0xff,0x05,0xd8,0xca,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0xd8,0xca,0x03,0x07,0x06,0xff]
+0x02,0x05,0xd8,0xca,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0xd8,0xca,0x04,0x07,0x06,0xff]
+0x03,0x05,0xd8,0xca,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0xd8,0xca,0x01,0x06,0x06,0xff]
+0x69,0x04,0xd8,0xca,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0xd8,0xca,0x69,0x06,0x06,0xff]
+0x01,0x04,0xd8,0xca,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0xd8,0xca,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0xd8,0xca,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0xd8,0xca,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0xd8,0xca,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0xd8,0xca,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0xd8,0xca,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0xd8,0xca,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0xd8,0xca,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0xd8,0xca,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0xd8,0xca,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0xd8,0xca,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0xd8,0xca,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0xd8,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0xd8,0xca,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_min_num_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0xd8,0xca,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0xd8,0xca,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0xd8,0xca,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0xd8,0xca,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0xd8,0xca,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0xd8,0xca,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_min_num_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0xd8,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0xd8,0xca,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_min_num_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0xc2,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0xc2,0xca,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -5778,6 +7008,60 @@
 # GFX12: v_dual_mov_b32 v6, null :: v_dual_cndmask_b32 v255, 0xaf123456, v5 ; encoding: [0x7c,0x00,0x12,0xca,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x00,0x12,0xca,0xff,0x0a,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_mov_b32 v255, v4 :: v_dual_dot2acc_f32_f16 v6, v1, v255 ; encoding: [0x04,0x01,0x18,0xca,0x01,0xff,0x07,0xff]
+0x04,0x01,0x18,0xca,0x01,0xff,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, v1 :: v_dual_dot2acc_f32_f16 v6, v255, v255 ; encoding: [0x01,0x01,0x18,0xca,0xff,0xff,0x07,0xff]
+0x01,0x01,0x18,0xca,0xff,0xff,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, v255 :: v_dual_dot2acc_f32_f16 v6, v2, v255 ; encoding: [0xff,0x01,0x18,0xca,0x02,0xff,0x07,0xff]
+0xff,0x01,0x18,0xca,0x02,0xff,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v255 ; encoding: [0x02,0x01,0x18,0xca,0x03,0xff,0x07,0xff]
+0x02,0x01,0x18,0xca,0x03,0xff,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, v3 :: v_dual_dot2acc_f32_f16 v6, v4, v255 ; encoding: [0x03,0x01,0x18,0xca,0x04,0xff,0x07,0xff]
+0x03,0x01,0x18,0xca,0x04,0xff,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, s105 :: v_dual_dot2acc_f32_f16 v6, s1, v255 ; encoding: [0x69,0x00,0x18,0xca,0x01,0xfe,0x07,0xff]
+0x69,0x00,0x18,0xca,0x01,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, s1 :: v_dual_dot2acc_f32_f16 v6, s105, v255 ; encoding: [0x01,0x00,0x18,0xca,0x69,0xfe,0x07,0xff]
+0x01,0x00,0x18,0xca,0x69,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, ttmp15 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v255 ; encoding: [0x7b,0x00,0x18,0xca,0x6a,0xfe,0x07,0xff]
+0x7b,0x00,0x18,0xca,0x6a,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, exec_hi :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v255 ; encoding: [0x7f,0x00,0x18,0xca,0x6b,0xfe,0x07,0xff]
+0x7f,0x00,0x18,0xca,0x6b,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, exec_lo :: v_dual_dot2acc_f32_f16 v6, ttmp15, v255 ; encoding: [0x7e,0x00,0x18,0xca,0x7b,0xfe,0x07,0xff]
+0x7e,0x00,0x18,0xca,0x7b,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, m0 :: v_dual_dot2acc_f32_f16 v6, m0, v255 ; encoding: [0x7d,0x00,0x18,0xca,0x7d,0xfe,0x07,0xff]
+0x7d,0x00,0x18,0xca,0x7d,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, vcc_hi :: v_dual_dot2acc_f32_f16 v6, exec_lo, v255 ; encoding: [0x6b,0x00,0x18,0xca,0x7e,0xfe,0x07,0xff]
+0x6b,0x00,0x18,0xca,0x7e,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, vcc_lo :: v_dual_dot2acc_f32_f16 v6, exec_hi, v255 ; encoding: [0x6a,0x00,0x18,0xca,0x7f,0xfe,0x07,0xff]
+0x6a,0x00,0x18,0xca,0x7f,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, 0xaf123456 :: v_dual_dot2acc_f32_f16 v6, null, v255 ; encoding: [0xff,0x00,0x18,0xca,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x00,0x18,0xca,0x7c,0xfe,0x07,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_mov_b32 v255, src_scc :: v_dual_dot2acc_f32_f16 v6, -1, v255 ; encoding: [0xfd,0x00,0x18,0xca,0xc1,0xfe,0x07,0xff]
+0xfd,0x00,0x18,0xca,0xc1,0xfe,0x07,0xff
+
+# GFX12: v_dual_mov_b32 v255, 0.5 :: v_dual_dot2acc_f32_f16 v6, 0.5, v3 ; encoding: [0xf0,0x00,0x18,0xca,0xf0,0x06,0x06,0xff]
+0xf0,0x00,0x18,0xca,0xf0,0x06,0x06,0xff
+
+# GFX12: v_dual_mov_b32 v255, -1 :: v_dual_dot2acc_f32_f16 v6, src_scc, v4 ; encoding: [0xc1,0x00,0x18,0xca,0xfd,0x08,0x06,0xff]
+0xc1,0x00,0x18,0xca,0xfd,0x08,0x06,0xff
+
+# GFX12: v_dual_mov_b32 v6, null :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v5 ; encoding: [0x7c,0x00,0x18,0xca,0xff,0x0a,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x00,0x18,0xca,0xff,0x0a,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_mov_b32 v255, v4 :: v_dual_fmaak_f32 v6, v1, v255, 0xaf123456 ; encoding: [0x04,0x01,0x02,0xca,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x01,0x02,0xca,0x01,0xff,0x07,0xff,0x56,0x34,0x12,0xaf
 
@@ -6588,6 +7872,60 @@
 # GFX12: v_dual_mul_dx9_zero_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0xd2,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0xd2,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_mul_dx9_zero_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0xd8,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0xd8,0xc9,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0xd8,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0xd8,0xc9,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0xd8,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0xd8,0xc9,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0xd8,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0xd8,0xc9,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0xd8,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0xd8,0xc9,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0xd8,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0xd8,0xc9,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0xd8,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0xd8,0xc9,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0xd8,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0xd8,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0xd8,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0xd8,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0xd8,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0xd8,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0xd8,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0xd8,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0xd8,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0xd8,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0xd8,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0xd8,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0xd8,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0xd8,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0xd8,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0xd8,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0xd8,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0xd8,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0xd8,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0xd8,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_mul_dx9_zero_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0xd8,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0xd8,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_mul_dx9_zero_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0xc2,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0xc2,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -7398,6 +8736,60 @@
 # GFX12: v_dual_mul_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0xd2,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0xd2,0xc8,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_mul_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0xd8,0xc8,0x01,0x07,0x06,0xff]
+0x04,0x05,0xd8,0xc8,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0xd8,0xc8,0xff,0x07,0x06,0xff]
+0x01,0x05,0xd8,0xc8,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0xd8,0xc8,0x02,0x07,0x06,0xff]
+0xff,0x05,0xd8,0xc8,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0xd8,0xc8,0x03,0x07,0x06,0xff]
+0x02,0x05,0xd8,0xc8,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0xd8,0xc8,0x04,0x07,0x06,0xff]
+0x03,0x05,0xd8,0xc8,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0xd8,0xc8,0x01,0x06,0x06,0xff]
+0x69,0x04,0xd8,0xc8,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0xd8,0xc8,0x69,0x06,0x06,0xff]
+0x01,0x04,0xd8,0xc8,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0xd8,0xc8,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0xd8,0xc8,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0xd8,0xc8,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0xd8,0xc8,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0xd8,0xc8,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0xd8,0xc8,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0xd8,0xc8,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0xd8,0xc8,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0xd8,0xc8,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0xd8,0xc8,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0xd8,0xc8,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0xd8,0xc8,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0xd8,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0xd8,0xc8,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_mul_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0xd8,0xc8,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0xd8,0xc8,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0xd8,0xc8,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0xd8,0xc8,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0xd8,0xc8,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0xd8,0xc8,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_mul_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0xd8,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0xd8,0xc8,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_mul_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0xc2,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0xc2,0xc8,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -8208,6 +9600,60 @@
 # GFX12: v_dual_sub_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x52,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x52,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_sub_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x58,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0x58,0xc9,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x58,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0x58,0xc9,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x58,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0x58,0xc9,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x58,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0x58,0xc9,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x58,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0x58,0xc9,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x58,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0x58,0xc9,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x58,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0x58,0xc9,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x58,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x58,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x58,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x58,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x58,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x58,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x58,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x58,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x58,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x58,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x58,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x58,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x58,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x58,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_sub_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x58,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x58,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x58,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x58,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x58,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x58,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_sub_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x58,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x58,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_sub_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x42,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x42,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -9018,6 +10464,60 @@
 # GFX12: v_dual_subrev_f32 v6, null, v5 :: v_dual_cndmask_b32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x92,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x92,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
 
+# GFX12: v_dual_subrev_f32 v255, v4, v2 :: v_dual_dot2acc_f32_f16 v6, v1, v3 ; encoding: [0x04,0x05,0x98,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0x98,0xc9,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, v1, v2 :: v_dual_dot2acc_f32_f16 v6, v255, v3 ; encoding: [0x01,0x05,0x98,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0x98,0xc9,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, v255, v2 :: v_dual_dot2acc_f32_f16 v6, v2, v3 ; encoding: [0xff,0x05,0x98,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0x98,0xc9,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, v2, v2 :: v_dual_dot2acc_f32_f16 v6, v3, v3 ; encoding: [0x02,0x05,0x98,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0x98,0xc9,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, v3, v2 :: v_dual_dot2acc_f32_f16 v6, v4, v3 ; encoding: [0x03,0x05,0x98,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0x98,0xc9,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, s105, v2 :: v_dual_dot2acc_f32_f16 v6, s1, v3 ; encoding: [0x69,0x04,0x98,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0x98,0xc9,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, s1, v2 :: v_dual_dot2acc_f32_f16 v6, s105, v3 ; encoding: [0x01,0x04,0x98,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0x98,0xc9,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x98,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x98,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_f16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x98,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x98,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_f16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x98,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x98,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, m0, v2 :: v_dual_dot2acc_f32_f16 v6, m0, v3 ; encoding: [0x7d,0x04,0x98,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x98,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_f16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x98,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x98,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_f16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x98,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x98,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_f16 v6, null, v3 ; encoding: [0xff,0x04,0x98,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x98,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_subrev_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_f16 v6, -1, v3 ; encoding: [0xfd,0x04,0x98,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x98,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_f16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x98,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x98,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v255, -1, v4 :: v_dual_dot2acc_f32_f16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x98,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x98,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_subrev_f32 v6, null, v5 :: v_dual_dot2acc_f32_f16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x98,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x98,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
 # GFX12: v_dual_subrev_f32 v255, v4, v2 :: v_dual_fmaak_f32 v6, v1, v3, 0xaf123456 ; encoding: [0x04,0x05,0x82,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf]
 0x04,0x05,0x82,0xc9,0x01,0x07,0x06,0xff,0x56,0x34,0x12,0xaf
 
@@ -9611,3 +11111,111 @@
 
 # GFX12: v_dual_subrev_f32 v6, null, v5 :: v_dual_subrev_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
 0x7c,0x0a,0x8c,0xc9,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_add_f32 v255, v4, v2 :: v_dual_dot2acc_f32_bf16 v6, v1, v3 ; encoding: [0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff]
+0x04,0x05,0x1a,0xc9,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v1, v2 :: v_dual_dot2acc_f32_bf16 v6, v255, v3 ; encoding: [0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff]
+0x01,0x05,0x1a,0xc9,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v255, v2 :: v_dual_dot2acc_f32_bf16 v6, v2, v3 ; encoding: [0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff]
+0xff,0x05,0x1a,0xc9,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v2, v2 :: v_dual_dot2acc_f32_bf16 v6, v3, v3 ; encoding: [0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff]
+0x02,0x05,0x1a,0xc9,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, v3, v2 :: v_dual_dot2acc_f32_bf16 v6, v4, v3 ; encoding: [0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff]
+0x03,0x05,0x1a,0xc9,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, s105, v2 :: v_dual_dot2acc_f32_bf16 v6, s1, v3 ; encoding: [0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff]
+0x69,0x04,0x1a,0xc9,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, s1, v2 :: v_dual_dot2acc_f32_bf16 v6, s105, v3 ; encoding: [0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff]
+0x01,0x04,0x1a,0xc9,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, ttmp15, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x1a,0xc9,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, exec_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x1a,0xc9,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, exec_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x1a,0xc9,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, m0, v2 :: v_dual_dot2acc_f32_bf16 v6, m0, v3 ; encoding: [0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x1a,0xc9,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, vcc_hi, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x1a,0xc9,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, vcc_lo, v2 :: v_dual_dot2acc_f32_bf16 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x1a,0xc9,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, 0xaf123456, v2 :: v_dual_dot2acc_f32_bf16 v6, null, v3 ; encoding: [0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf]
+0xff,0x04,0x1a,0xc9,0x7c,0x06,0x06,0xff,0x56,0x34,0x12,0xaf
+
+# GFX12: v_dual_add_f32 v255, src_scc, v2 :: v_dual_dot2acc_f32_bf16 v6, -1, v3 ; encoding: [0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x1a,0xc9,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, 0.5, v3 :: v_dual_dot2acc_f32_bf16 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x1a,0xc9,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_add_f32 v255, -1, v4 :: v_dual_dot2acc_f32_bf16 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x1a,0xc9,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_add_f32 v6, null, v5 :: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v4 ; encoding: [0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00]
+0x7c,0x0a,0x1a,0xc9,0xff,0x08,0xfe,0x06,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff]
+0x04,0x05,0x48,0xcb,0x01,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, v1, v2 :: v_dual_add_f32 v6, v255, v3 ; encoding: [0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff]
+0x01,0x05,0x48,0xcb,0xff,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, v255, v2 :: v_dual_add_f32 v6, v2, v3 ; encoding: [0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff]
+0xff,0x05,0x48,0xcb,0x02,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, v2, v2 :: v_dual_add_f32 v6, v3, v3 ; encoding: [0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff]
+0x02,0x05,0x48,0xcb,0x03,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, v3, v2 :: v_dual_add_f32 v6, v4, v3 ; encoding: [0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff]
+0x03,0x05,0x48,0xcb,0x04,0x07,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, s105, v2 :: v_dual_add_f32 v6, s1, v3 ; encoding: [0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff]
+0x69,0x04,0x48,0xcb,0x01,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, s1, v2 :: v_dual_add_f32 v6, s105, v3 ; encoding: [0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff]
+0x01,0x04,0x48,0xcb,0x69,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, ttmp15, v2 :: v_dual_add_f32 v6, vcc_lo, v3 ; encoding: [0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff]
+0x7b,0x04,0x48,0xcb,0x6a,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, exec_hi, v2 :: v_dual_add_f32 v6, vcc_hi, v3 ; encoding: [0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff]
+0x7f,0x04,0x48,0xcb,0x6b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, exec_lo, v2 :: v_dual_add_f32 v6, ttmp15, v3 ; encoding: [0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff]
+0x7e,0x04,0x48,0xcb,0x7b,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, m0, v2 :: v_dual_add_f32 v6, m0, v3 ; encoding: [0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff]
+0x7d,0x04,0x48,0xcb,0x7d,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, vcc_hi, v2 :: v_dual_add_f32 v6, exec_lo, v3 ; encoding: [0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff]
+0x6b,0x04,0x48,0xcb,0x7e,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, vcc_lo, v2 :: v_dual_add_f32 v6, exec_hi, v3 ; encoding: [0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff]
+0x6a,0x04,0x48,0xcb,0x7f,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, 0xfe0b, v2 :: v_dual_add_f32 v6, null, v3 ; encoding: [0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00]
+0xff,0x04,0x48,0xcb,0x7c,0x06,0x06,0xff,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, src_scc, v2 :: v_dual_add_f32 v6, -1, v3 ; encoding: [0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff]
+0xfd,0x04,0x48,0xcb,0xc1,0x06,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, 0.5, v3 :: v_dual_add_f32 v6, 0.5, v2 ; encoding: [0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff]
+0xf0,0x06,0x48,0xcb,0xf0,0x04,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v255, -1, v4 :: v_dual_add_f32 v6, src_scc, v5 ; encoding: [0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff]
+0xc1,0x08,0x48,0xcb,0xfd,0x0a,0x06,0xff
+
+# GFX12: v_dual_dot2acc_f32_bf16 v6, null, v5 :: v_dual_add_f32 v255, 0xaf123456, v4 ; encoding: [0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf]
+0x7c,0x0a,0x48,0xcb,0xff,0x08,0xfe,0x06,0x56,0x34,0x12,0xaf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt
index f86d4325d63e5..7cd97ac87057e 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt
@@ -1146,3 +1146,21 @@
 
 # GFX950: v_cvt_scalef32_sr_pk32_fp6_f32 v[0:5], v[6:37], v38, v39 ; encoding: [0x00,0x00,0x54,0xd2,0x06,0x4d,0x9e,0x04]
 0x00,0x00,0x54,0xd2,0x06,0x4d,0x9e,0x04
+
+# GFX950: v_cvt_pk_f16_f32 v5, v1, v2             ; encoding: [0x05,0x00,0x67,0xd2,0x01,0x05,0x02,0x00]
+0x05,0x00,0x67,0xd2,0x01,0x05,0x02,0x00
+
+# GFX950: v_cvt_pk_f16_f32 v5, v255, v255         ; encoding: [0x05,0x00,0x67,0xd2,0xff,0xff,0x03,0x00]
+0x05,0x00,0x67,0xd2,0xff,0xff,0x03,0x00
+
+# GFX950: v_cvt_pk_f16_f32 v5, m0, 0.5            ; encoding: [0x05,0x00,0x67,0xd2,0x7c,0xe0,0x01,0x00]
+0x05,0x00,0x67,0xd2,0x7c,0xe0,0x01,0x00
+
+# GFX950: v_cvt_pk_f16_f32 v5, exec_lo, -1        ; encoding: [0x05,0x00,0x67,0xd2,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x67,0xd2,0x7e,0x82,0x01,0x00
+
+# GFX950: v_cvt_pk_f16_f32 v5, -1, exec_hi        ; encoding: [0x05,0x00,0x67,0xd2,0xc1,0xfe,0x00,0x00]
+0x05,0x00,0x67,0xd2,0xc1,0xfe,0x00,0x00
+
+# GFX950: v_cvt_pk_f16_f32 v5, 0.5, m0 mul:2      ; encoding: [0x05,0x00,0x67,0xd2,0xf0,0xf8,0x00,0x08]
+0x05,0x00,0x67,0xd2,0xf0,0xf8,0x00,0x08
diff --git a/llvm/test/MC/Disassembler/SystemZ/insns-pcrel.txt b/llvm/test/MC/Disassembler/SystemZ/insns-pcrel.txt
index ef1d0f1970d16..f12441c9c6cba 100644
--- a/llvm/test/MC/Disassembler/SystemZ/insns-pcrel.txt
+++ b/llvm/test/MC/Disassembler/SystemZ/insns-pcrel.txt
@@ -52,7 +52,7 @@
 0xa7 0xf5 0x7f 0xff
 
 # 0x0000003c:
-# CHECK: brcl 0, 0x3c
+# CHECK: jgnop 0x3c
 0xc0 0x04 0x00 0x00 0x00 0x00
 
 # 0x00000042:
@@ -116,15 +116,15 @@
 0xc0 0xf4 0x00 0x00 0x00 0x00
 
 # 0x0000009c:
-# CHECK: brcl 0, 0x9a
+# CHECK: jgnop 0x9a
 0xc0 0x04 0xff 0xff 0xff 0xff
 
 # 0x000000a2:
-# CHECK: brcl 0, 0xffffffff000000a2
+# CHECK: jgnop 0xffffffff000000a2
 0xc0 0x04 0x80 0x00 0x00 0x00
 
 # 0x000000a8:
-# CHECK: brcl 0, 0x1000000a6
+# CHECK: jgnop 0x1000000a6
 0xc0 0x04 0x7f 0xff 0xff 0xff
 
 # 0x000000ae:
@@ -140,7 +140,7 @@
 0xc0 0xf4 0x7f 0xff 0xff 0xff
 
 # 0x000000c0:
-# CHECK: brc 0, 0xc0
+# CHECK: jnop 0xc0
 0xa7 0x04 0x00 0x00
 
 # 0x000000c4:
@@ -204,15 +204,15 @@
 0xa7 0xf4 0x00 0x00
 
 # 0x00000100:
-# CHECK: brc 0, 0xfe
+# CHECK: jnop 0xfe
 0xa7 0x04 0xff 0xff
 
 # 0x00000104:
-# CHECK: brc 0, 0xffffffffffff0104
+# CHECK: jnop 0xffffffffffff0104
 0xa7 0x04 0x80 0x00
 
 # 0x00000108:
-# CHECK: brc 0, 0x10106
+# CHECK: jnop 0x10106
 0xa7 0x04 0x7f 0xff
 
 # 0x0000010c:
diff --git a/llvm/test/MC/Disassembler/SystemZ/insns.txt b/llvm/test/MC/Disassembler/SystemZ/insns.txt
index 07a1ff6d18388..a4e4a2203a467 100644
--- a/llvm/test/MC/Disassembler/SystemZ/insns.txt
+++ b/llvm/test/MC/Disassembler/SystemZ/insns.txt
@@ -1315,7 +1315,7 @@
 # CHECK: bassm %r15, %r1
 0x0c 0xf1
 
-# CHECK: nop 0
+# CHECK: nop
 0x47 0x00 0x00 0x00
 
 # CHECK: nop 4095
diff --git a/llvm/test/MC/RISCV/custom_reloc.s b/llvm/test/MC/RISCV/custom_reloc.s
new file mode 100644
index 0000000000000..d6fea8fac3671
--- /dev/null
+++ b/llvm/test/MC/RISCV/custom_reloc.s
@@ -0,0 +1,35 @@
+# RUN: llvm-mc -triple riscv32 %s \
+# RUN:   | FileCheck -check-prefix=CHECK-ASM %s
+# RUN: llvm-mc -triple riscv64 %s \
+# RUN:   | FileCheck -check-prefix=CHECK-ASM %s
+
+# RUN: llvm-mc -filetype=obj -triple riscv32 %s \
+# RUN:   | llvm-objdump -dr -M no-aliases - \
+# RUN:   | FileCheck -check-prefix=CHECK-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 %s \
+# RUN:   | llvm-objdump -dr -M no-aliases  - \
+# RUN:   | FileCheck -check-prefix=CHECK-OBJ %s
+
+  # CHECK-ASM: .text
+  # CHECK-OBJ: <.text>:
+
+  nop
+  # CHECK-ASM: nop
+  # CHECK-OBJ: addi zero, zero, 0x0
+
+  .reloc ., R_RISCV_VENDOR,    VENDOR_NAME
+  .reloc ., R_RISCV_CUSTOM192, my_foo + 1
+  addi a0, a0, 0
+  # CHECK-ASM: [[L1:.L[^:]+]]:
+  # CHECK-ASM-NEXT: .reloc [[L1]], R_RISCV_VENDOR, VENDOR_NAME
+  # CHECK-ASM-NEXT: [[L2:.L[^:]+]]:
+  # CHECK-ASM-NEXT: .reloc [[L2]], R_RISCV_CUSTOM192, my_foo+1
+  # CHECK-ASM-NEXT: mv a0, a0
+
+  # CHECK-OBJ: addi a0, a0, 0
+  # CHECK-OBJ-NEXT: R_RISCV_VENDOR    VENDOR_NAME
+  # CHECK-OBJ-NEXT: R_RISCV_CUSTOM192 my_foo+0x1
+
+  nop
+  # CHECK-ASM: nop
+  # CHECK-OBJ: addi zero, zero, 0x0
diff --git a/llvm/test/MC/RISCV/xqcia-invalid.s b/llvm/test/MC/RISCV/xqcia-invalid.s
new file mode 100644
index 0000000000000..a410fb63fad9c
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcia-invalid.s
@@ -0,0 +1,201 @@
+# Xqcia - Qualcomm uC Arithmetic Extension
+# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-xqcia < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-PLUS %s
+# RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcia < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
+
+# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+qc.slasat x10, x3, 17
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.slasat x10, x3
+
+# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+qc.slasat x0, x3, x17
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.slasat x10, x0, x17
+
+# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+qc.slasat x10, x3, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.slasat x10, x3, x17
+
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.sllsat x23, x25, 27
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.sllsat x23, x25
+
+# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+qc.sllsat x0, x25, x27
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.sllsat x23, x0, x27
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.sllsat x23, x25, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.sllsat x23, x25, x27
+
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.addsat x17, x14, 7
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.addsat x17, x14
+
+# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+qc.addsat x0, x14, x7
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.addsat x17, x0, x7
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.addsat x17, x14, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.addsat x17, x14, x7
+
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.addusat x8, x18, 28
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.addusat x8, x18
+
+# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.addusat x0, x18, x28
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.addusat x8, x0, x28
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.addusat x8, x18, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.addusat x8, x18, x28
+
+
+# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+qc.subsat x22, x2, 12
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.subsat x22, x2
+
+# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+qc.subsat x0, x2, x12
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.subsat x22, x0, x12
+
+# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+qc.subsat x22, x2, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.subsat x22, x2, x12
+
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.subusat x9, x14, 17
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.subusat x9, x14
+
+# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.subusat x0, x14, x17
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.subusat x9, x0, x17
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.subusat x9, x14, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.subusat x9, x14, x17
+
+
+# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+qc.wrap x3, x30, 23
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.wrap x3, x30
+
+# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+qc.wrap x0, x30, x23
+
+# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+qc.wrap x3, x30, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.wrap x3, x30, x23
+
+
+# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+qc.wrapi x0, 12, 2047
+
+# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+qc.wrapi x0, x12, 2047
+
+# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+qc.wrapi x6, x0, 2047
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.wrapi x6, x12
+
+# CHECK-PLUS: :[[@LINE+1]]:19: error: immediate must be an integer in the range [0, 2047]
+qc.wrapi x6, x12, 2048
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.wrapi x6, x12, 2047
+
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.norm x3, 7
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.norm x3
+
+# CHECK: :[[@LINE+1]]:9: error: invalid operand for instruction
+qc.norm x0, x7
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.norm x3, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.norm x3, x7
+
+
+# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+qc.normu x11, 17
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.normu x11
+
+# CHECK: :[[@LINE+1]]:10: error: invalid operand for instruction
+qc.normu x0, x17
+
+# CHECK: :[[@LINE+1]]:15: error: invalid operand for instruction
+qc.normu x11, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.normu x11, x17
+
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.normeu x26, 31
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.normeu x26
+
+# CHECK: :[[@LINE+1]]:11: error: invalid operand for instruction
+qc.normeu x0, x31
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.normeu x26, x0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcia' (Qualcomm uC Arithmetic Extension)
+qc.normeu x26, x31
diff --git a/llvm/test/MC/RISCV/xqcia-valid.s b/llvm/test/MC/RISCV/xqcia-valid.s
new file mode 100644
index 0000000000000..6bd10492d4d6a
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcia-valid.s
@@ -0,0 +1,55 @@
+# Xqcia - Qualcomm uC Arithmetic Extesnsion
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcia -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcia < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcia -M no-aliases --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcia -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcia < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcia --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+
+# CHECK-INST: qc.slasat    a0, gp, a7
+# CHECK-ENC: encoding: [0x0b,0xb5,0x11,0x15]
+qc.slasat x10, x3, x17
+
+# CHECK-INST: qc.sllsat    s7, s9, s11
+# CHECK-ENC: encoding: [0x8b,0xbb,0xbc,0x19]
+qc.sllsat x23, x25, x27
+
+# CHECK-INST: qc.addsat    a7, a4, t2
+# CHECK-ENC: encoding: [0x8b,0x38,0x77,0x1c]
+qc.addsat x17, x14, x7
+
+# CHECK-INST: qc.addusat   s0, s2, t3
+# CHECK-ENC: encoding: [0x0b,0x34,0xc9,0x1f]
+qc.addusat x8, x18, x28
+
+# CHECK-INST: qc.subsat    s6, sp, a2
+# CHECK-ENC: encoding: [0x0b,0x3b,0xc1,0x20]
+qc.subsat x22, x2, x12
+
+# CHECK-INST: qc.subusat   s1, a4, a7
+# CHECK-ENC: encoding: [0x8b,0x34,0x17,0x23]
+qc.subusat x9, x14, x17
+
+# CHECK-INST: qc.wrap  gp, t5, s7
+# CHECK-ENC: encoding: [0x8b,0x31,0x7f,0x25]
+qc.wrap x3, x30, x23
+
+# CHECK-INST: qc.wrapi   t1, a2, 2047
+# CHECK-ENC: encoding: [0x0b,0x03,0xf6,0x7f]
+qc.wrapi x6, x12, 2047
+
+# CHECK-INST: qc.norm    gp, t2
+# CHECK-ENC: encoding: [0x8b,0xb1,0x03,0x0e]
+qc.norm x3, x7
+
+# CHECK-INST: qc.normu   a1, a7
+# CHECK-ENC: encoding: [0x8b,0xb5,0x08,0x10]
+qc.normu x11, x17
+
+# CHECK-INST: qc.normeu  s10, t6
+# CHECK-ENC: encoding: [0x0b,0xbd,0x0f,0x12]
+qc.normeu x26, x31
diff --git a/llvm/test/MC/RISCV/xqcicsr-invalid.s b/llvm/test/MC/RISCV/xqcicsr-invalid.s
new file mode 100644
index 0000000000000..6d3d62d4b4b41
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcicsr-invalid.s
@@ -0,0 +1,27 @@
+# Xqcicsr - Qualcomm uC CSR Extension
+# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-xqcicsr < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-PLUS %s
+# RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcicsr < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
+
+# CHECK: :[[@LINE+1]]:20: error: invalid operand for instruction
+qc.csrrwr x10, x5, x0
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.csrrwr x10, x5
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcicsr' (Qualcomm uC CSR Extension)
+qc.csrrwr x10, x5, x20
+
+
+# CHECK: :[[@LINE+1]]:21: error: invalid operand for instruction
+qc.csrrwri x20, 31, x0
+
+# CHECK-PLUS: :[[@LINE+1]]:17: error: immediate must be an integer in the range [0, 31]
+qc.csrrwri x20, 45, x12
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.csrrwri x20, 23
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcicsr' (Qualcomm uC CSR Extension)
+qc.csrrwri x30, 31, x12
diff --git a/llvm/test/MC/RISCV/xqcicsr-valid.s b/llvm/test/MC/RISCV/xqcicsr-valid.s
new file mode 100644
index 0000000000000..1236dd622703d
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcicsr-valid.s
@@ -0,0 +1,19 @@
+# Xqcicsr - Qualcomm uC CSR Extension
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcicsr -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcicsr < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcicsr -M no-aliases --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcicsr -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcicsr < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcicsr --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+
+# CHECK-INST: qc.csrrwr  a0, t0, s4
+# CHECK-ENC: encoding: [0x73,0x85,0x42,0x8d]
+qc.csrrwr  x10, x5, x20
+
+# CHECK-INST: qc.csrrwri s4, 31, a2
+# CHECK-ENC: encoding: [0x73,0x8a,0xcf,0x8e]
+qc.csrrwri x20, 31, x12
diff --git a/llvm/test/MC/RISCV/xqcisls-invalid.s b/llvm/test/MC/RISCV/xqcisls-invalid.s
new file mode 100644
index 0000000000000..5e9e54812fee8
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcisls-invalid.s
@@ -0,0 +1,132 @@
+# Xqcisls - Qualcomm uC Scaled Load Store Extension
+# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-xqcisls < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-PLUS %s
+# RUN: not llvm-mc -triple riscv32 -mattr=-experimental-xqcisls < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-MINUS %s
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.lrb x5, x2, x0, 4
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.lrb x5, x2, x4
+
+# CHECK-PLUS: :[[@LINE+1]]:20: error: immediate must be an integer in the range [0, 7]
+qc.lrb x5, x2, x4, 12
+
+# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.lrb x5, 2, x4, 4
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.lrb x5, x2, x4, 4
+
+
+# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+qc.lrh x1, x12, x0, 2
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.lrh x1, x12, x6
+
+# CHECK-PLUS: :[[@LINE+1]]:21: error: immediate must be an integer in the range [0, 7]
+qc.lrh x1, x12, x6, 22
+
+# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.lrh x1, 12, x6, 2
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.lrh x1, x12, x6, 2
+
+
+# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+qc.lrw x15, x7, x0, 1
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.lrw x15, x7, x14
+
+# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+qc.lrw x15, x7, x14, 11
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.lrw x15, 7, x14, 1
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.lrw x15, x7, x14, 1
+
+
+# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+qc.lrbu x9, x11, x0, 7
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.lrbu x9, x11, x4
+
+# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+qc.lrbu x9, x11, x4, 37
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.lrbu x9, 11, x4, 7
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.lrbu x9, x11, x4, 7
+
+
+# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+qc.lrhu x16, x6, x0, 4
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.lrhu x16, x6, x10
+
+# CHECK-PLUS: :[[@LINE+1]]:23: error: immediate must be an integer in the range [0, 7]
+qc.lrhu x16, x6, x10, 44
+
+# CHECK: :[[@LINE+1]]:14: error: invalid operand for instruction
+qc.lrhu x16, 6, x10, 4
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.lrhu x16, x6, x10, 4
+
+
+# CHECK: :[[@LINE+1]]:16: error: invalid operand for instruction
+qc.srb x0, x2, x0, 3
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.srb x0, x2, x8
+
+# CHECK-PLUS: :[[@LINE+1]]:20: error: immediate must be an integer in the range [0, 7]
+qc.srb x0, x2, x8, 93
+
+# CHECK: :[[@LINE+1]]:12: error: invalid operand for instruction
+qc.srb x0, 2, x8, 3
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.srb x0, x2, x8, 3
+
+
+# CHECK: :[[@LINE+1]]:17: error: invalid operand for instruction
+qc.srh x13, x0, x0, 6
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.srh x13, x0, x20
+
+# CHECK-PLUS: :[[@LINE+1]]:22: error: immediate must be an integer in the range [0, 7]
+qc.srh x13, x0, x20, 76
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.srh x13, 0, x20, 6
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.srh x13, x0, x20, 6
+
+
+# CHECK: :[[@LINE+1]]:18: error: invalid operand for instruction
+qc.srw x17, x18, x0, 0
+
+# CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
+qc.srw x17, x18, x19
+
+# CHECK-PLUS: :[[@LINE+1]]:23: error: immediate must be an integer in the range [0, 7]
+qc.srw x17, x18, x19, 10
+
+# CHECK: :[[@LINE+1]]:13: error: invalid operand for instruction
+qc.srw x17, 18, x19, 0
+
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcisls' (Qualcomm uC Scaled Load Store Extension)
+qc.srw x17, x18, x19, 0
diff --git a/llvm/test/MC/RISCV/xqcisls-valid.s b/llvm/test/MC/RISCV/xqcisls-valid.s
new file mode 100644
index 0000000000000..32f64a82985ce
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcisls-valid.s
@@ -0,0 +1,43 @@
+# Xqcisls - Qualcomm uC Scaled Load Store Extension
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcisls -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcisls < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcisls -M no-aliases --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcisls -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcisls < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-xqcisls --no-print-imm-hex -d - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+
+# CHECK-INST: qc.lrb    t0, sp, tp, 4
+# CHECK-ENC: encoding: [0x8b,0x72,0x41,0x88]
+qc.lrb x5, x2, x4, 4
+
+# CHECK-INST: qc.lrh    ra, a2, t1, 2
+# CHECK-ENC: encoding: [0x8b,0x70,0x66,0x94]
+qc.lrh x1, x12, x6, 2
+
+# CHECK-INST: qc.lrw    a5, t2, a4, 1
+# CHECK-ENC: encoding: [0x8b,0xf7,0xe3,0xa2]
+qc.lrw x15, x7, x14, 1
+
+# CHECK-INST: qc.lrbu    s1, a1, tp, 7
+# CHECK-ENC: encoding: [0x8b,0xf4,0x45,0xbe]
+qc.lrbu x9, x11, x4, 7
+
+# CHECK-INST: qc.lrhu    a6, t1, a0, 4
+# CHECK-ENC: encoding: [0x0b,0x78,0xa3,0xc8]
+qc.lrhu x16, x6, x10, 4
+
+# CHECK-INST: qc.srb    zero, sp, s0, 3
+# CHECK-ENC: encoding: [0x2b,0x60,0x81,0xd6]
+qc.srb x0, x2, x8, 3
+
+# CHECK-INST: qc.srh    a3, zero, s4, 6
+# CHECK-ENC: encoding: [0xab,0x66,0x40,0xed]
+qc.srh x13, x0, x20, 6
+
+# CHECK-INST: qc.srw    a7, s2, s3, 0
+# CHECK-ENC: encoding: [0xab,0x68,0x39,0xf1]
+qc.srw x17, x18, x19, 0
diff --git a/llvm/test/MC/SystemZ/insn-good-zos-pcrel.s b/llvm/test/MC/SystemZ/insn-good-zos-pcrel.s
index 734520798baa6..0acbe26d75b15 100644
--- a/llvm/test/MC/SystemZ/insn-good-zos-pcrel.s
+++ b/llvm/test/MC/SystemZ/insn-good-zos-pcrel.s
@@ -3,7 +3,7 @@
 
 *CHECK: brcl	0, FOO                  * encoding: [0xc0,0x04,A,A,A,A]
 *CHECK:  fixup A - offset: 2, value: FOO+2, kind: FK_390_PC32DBL
-*CHECK: brcl	0, FOO                  * encoding: [0xc0,0x04,A,A,A,A]
+*CHECK: jgnop	FOO                     * encoding: [0xc0,0x04,A,A,A,A]
 *CHECK:  fixup A - offset: 2, value: FOO+2, kind: FK_390_PC32DBL
 	brcl	0,FOO
 	jlnop	FOO
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index 09f55049546c2..93f5ff27780ab 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -1398,35 +1398,35 @@
 
 #CHECK: brc	0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	.[[LAB:L.*]]-65536	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brc	0, -0x10000
 	jnop	-0x10000
 
 #CHECK: brc	0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	.[[LAB:L.*]]-2	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brc	0, -2
 	jnop	-2
 
 #CHECK: brc	0, .[[LAB:L.*]]		# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, .[[LAB:L.*]]		# encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	.[[LAB:L.*]]		# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brc	0, 0
 	jnop	0
 
 #CHECK: brc	0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	.[[LAB:L.*]]+65534	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brc	0, 0xfffe
 	jnop	0xfffe
 
 #CHECK: brc	0, foo                  # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, foo                  # encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	foo                     # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	0, foo
 	jnop	foo
@@ -1623,7 +1623,7 @@
 
 #CHECK: brc	0, bar+100              # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, bar+100              # encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	bar+100                 # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	brc	0, bar+100
 	jnop	bar+100
@@ -1735,7 +1735,7 @@
 
 #CHECK: brc	0, bar@PLT              # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
-#CHECK: brc	0, bar@PLT              # encoding: [0xa7,0x04,A,A]
+#CHECK: jnop	bar@PLT                 # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	brc	0, bar@PLT
 	jnop	bar@PLT
@@ -1847,32 +1847,32 @@
 
 #CHECK: brcl	0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	.[[LAB:L.*]]-4294967296    # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
 	brcl	0, -0x100000000
 	jgnop	-0x100000000
 #CHECK: brcl	0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	.[[LAB:L.*]]-2		# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
 	brcl	0, -2
 	jgnop	-2
 #CHECK: brcl	0, .[[LAB:L.*]]		# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, .[[LAB:L.*]]		# encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	.[[LAB:L.*]]		# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
 	brcl	0, 0
 	jgnop	0
 #CHECK: brcl	0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	.[[LAB:L.*]]+4294967294    # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
 	brcl	0, 0xfffffffe
 	jgnop	0xfffffffe
 
 #CHECK: brcl	0, foo                  # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, foo                  # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	foo                     # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	0, foo
 	jgnop	foo
@@ -2065,7 +2065,7 @@
 
 #CHECK: brcl	0, bar+100              # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, bar+100              # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	bar+100                 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	brcl	0, bar+100
 	jgnop	bar+100
@@ -2177,7 +2177,7 @@
 
 #CHECK: brcl	0, bar@PLT              # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
-#CHECK: brcl	0, bar@PLT              # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK: jgnop	bar@PLT                 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	brcl	0, bar@PLT
 	jgnop	bar@PLT
@@ -13142,7 +13142,7 @@
 #CHECK: nop	0                       # encoding: [0x47,0x00,0x00,0x00]
 #CHECK: nop                             # encoding: [0x47,0x00,0x00,0x00]
 #CHECK: nopr	%r7                     # encoding: [0x07,0x07]
-#CHECK: nopr	%r0                 # encoding: [0x07,0x00]
+#CHECK: nopr	                        # encoding: [0x07,0x00]
 
 	nop	0
 	nop
diff --git a/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll b/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
index b321c0c82ad4d..6e9c64604e99d 100644
--- a/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
+++ b/llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll
@@ -1,5 +1,5 @@
-; RUN: llc %s -mattr=-bulk-memory -o - | FileCheck %s
-; RUN: llc %s -mattr=-bulk-memory -o - | llvm-mc -triple=wasm32-unknown-unknown | FileCheck %s
+; RUN: llc %s -mattr=-bulk-memory,-bulk-memory-opt -o - | FileCheck %s
+; RUN: llc %s -mattr=-bulk-memory,-bulk-memory-opt -o - | llvm-mc -triple=wasm32-unknown-unknown | FileCheck %s
 
 ; ModuleID = 'test.c'
 source_filename = "test.c"
diff --git a/llvm/test/MC/WebAssembly/function-alias.ll b/llvm/test/MC/WebAssembly/function-alias.ll
index 036cd7d06e063..3f76516a9bcc8 100644
--- a/llvm/test/MC/WebAssembly/function-alias.ll
+++ b/llvm/test/MC/WebAssembly/function-alias.ll
@@ -1,5 +1,5 @@
-; RUN: llc -filetype=obj %s -mattr=-reference-types -o - | llvm-readobj --symbols - | FileCheck %s
-; RUN: llc -filetype=obj %s -mattr=+reference-types -o - | llvm-readobj --symbols - | FileCheck --check-prefix=REF %s
+; RUN: llc -filetype=obj %s -mattr=-reference-types,-call-indirect-overlong -o - | llvm-readobj --symbols - | FileCheck %s
+; RUN: llc -filetype=obj %s -mattr=+reference-types,-call-indirect-overlong -o - | llvm-readobj --symbols - | FileCheck --check-prefix=REF %s
 
 target triple = "wasm32-unknown-unknown-wasm"
 
diff --git a/llvm/test/MC/WebAssembly/init-array-label.s b/llvm/test/MC/WebAssembly/init-array-label.s
new file mode 100644
index 0000000000000..0b4a5ea2da0b5
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/init-array-label.s
@@ -0,0 +1,91 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj < %s | obj2yaml | FileCheck %s
+
+init1:
+	.functype	init1 () -> ()
+	end_function
+
+init2:
+	.functype	init2 () -> ()
+	end_function
+
+	.section	.init_array.42,"",@
+	.p2align	2, 0x0
+	.int32	init1
+
+	.section	.init_array,"",@
+	.globl	p_init1
+	.p2align	2, 0x0
+p_init1:
+	.int32	init1
+	.size	p_init1, 4
+
+	.section	.init_array,"",@
+	.globl	p_init2
+	.p2align	2, 0x0
+p_init2:
+	.int32	init1
+	.int32	init2
+	.size	p_init2, 8
+
+# CHECK:        - Type:            FUNCTION
+# CHECK-NEXT:     FunctionTypes:   [ 0, 0 ]
+# CHECK-NEXT:   - Type:            DATACOUNT
+# CHECK-NEXT:     Count:           1
+# CHECK-NEXT:   - Type:            CODE
+# CHECK-NEXT:     Functions:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Locals:          []
+# CHECK-NEXT:         Body:            0B
+# CHECK-NEXT:       - Index:           1
+# CHECK-NEXT:         Locals:          []
+# CHECK-NEXT:         Body:            0B
+# CHECK-NEXT:   - Type:            DATA
+# CHECK-NEXT:     Segments:
+# CHECK-NEXT:       - SectionOffset:   6
+# CHECK-NEXT:         InitFlags:       0
+# CHECK-NEXT:         Offset:
+# CHECK-NEXT:           Opcode:          I32_CONST
+# CHECK-NEXT:           Value:           0
+# CHECK-NEXT:         Content:         '000000000000000000000000'
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            linking
+# CHECK-NEXT:     Version:         2
+# CHECK-NEXT:     SymbolTable:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Kind:            FUNCTION
+# CHECK-NEXT:         Name:            init1
+# CHECK-NEXT:         Flags:           [ BINDING_LOCAL ]
+# CHECK-NEXT:         Function:        0
+# CHECK-NEXT:       - Index:           1
+# CHECK-NEXT:         Kind:            FUNCTION
+# CHECK-NEXT:         Name:            init2
+# CHECK-NEXT:         Flags:           [ BINDING_LOCAL ]
+# CHECK-NEXT:         Function:        1
+# CHECK-NEXT:       - Index:           2
+# CHECK-NEXT:         Kind:            DATA
+# CHECK-NEXT:         Name:            p_init1
+# CHECK-NEXT:         Flags:           [  ]
+# CHECK-NEXT:         Segment:         0
+# CHECK-NEXT:         Size:            4
+# CHECK-NEXT:       - Index:           3
+# CHECK-NEXT:         Kind:            DATA
+# CHECK-NEXT:         Name:            p_init2
+# CHECK-NEXT:         Flags:           [  ]
+# CHECK-NEXT:         Segment:         0
+# CHECK-NEXT:         Offset:          4
+# CHECK-NEXT:         Size:            8
+# CHECK-NEXT:     SegmentInfo:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Name:            .init_array
+# CHECK-NEXT:         Alignment:       2
+# CHECK-NEXT:         Flags:           [  ]
+# CHECK-NEXT:     InitFunctions:
+# CHECK-NEXT:       - Priority:        42
+# CHECK-NEXT:         Symbol:          0
+# CHECK-NEXT:       - Priority:        65535
+# CHECK-NEXT:         Symbol:          0
+# CHECK-NEXT:       - Priority:        65535
+# CHECK-NEXT:         Symbol:          0
+# CHECK-NEXT:       - Priority:        65535
+# CHECK-NEXT:         Symbol:          1
+# CHECK-NEXT: ...
diff --git a/llvm/test/MC/WebAssembly/init-array.s b/llvm/test/MC/WebAssembly/init-array.s
new file mode 100644
index 0000000000000..e79fb453ec12a
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/init-array.s
@@ -0,0 +1,49 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj < %s | obj2yaml | FileCheck %s
+
+init1:
+	.functype	init1 () -> ()
+	end_function
+
+init2:
+	.functype	init2 () -> ()
+	end_function
+
+	.section	.init_array,"",@
+	.p2align	2, 0
+	.int32	init1
+
+	.section	.init_array,"",@
+	.p2align	2
+	.int32	init2
+
+# CHECK:        - Type:            FUNCTION
+# CHECK-NEXT:     FunctionTypes:   [ 0, 0 ]
+# CHECK-NEXT:   - Type:            CODE
+# CHECK-NEXT:     Functions:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Locals:          []
+# CHECK-NEXT:         Body:            0B
+# CHECK-NEXT:       - Index:           1
+# CHECK-NEXT:         Locals:          []
+# CHECK-NEXT:         Body:            0B
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            linking
+# CHECK-NEXT:     Version:         2
+# CHECK-NEXT:     SymbolTable:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Kind:            FUNCTION
+# CHECK-NEXT:         Name:            init1
+# CHECK-NEXT:         Flags:           [ BINDING_LOCAL ]
+# CHECK-NEXT:         Function:        0
+# CHECK-NEXT:       - Index:           1
+# CHECK-NEXT:         Kind:            FUNCTION
+# CHECK-NEXT:         Name:            init2
+# CHECK-NEXT:         Flags:           [ BINDING_LOCAL ]
+# CHECK-NEXT:         Function:        1
+# CHECK-NEXT:     InitFunctions:
+# CHECK-NEXT:       - Priority:        65535
+# CHECK-NEXT:         Symbol:          0
+# CHECK-NEXT:       - Priority:        65535
+# CHECK-NEXT:         Symbol:          1
+# CHECK-NEXT: ...
+#
diff --git a/llvm/test/MC/WebAssembly/libcall.ll b/llvm/test/MC/WebAssembly/libcall.ll
index ffd32abe2345b..6f36ab7ad317f 100644
--- a/llvm/test/MC/WebAssembly/libcall.ll
+++ b/llvm/test/MC/WebAssembly/libcall.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -mattr=-bulk-memory %s -o - | obj2yaml | FileCheck %s
+; RUN: llc -filetype=obj -mattr=-bulk-memory,-bulk-memory-opt %s -o - | obj2yaml | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MachineVerifier/RISCV/lit.local.cfg b/llvm/test/MachineVerifier/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000..17351748513d9
--- /dev/null
+++ b/llvm/test/MachineVerifier/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "RISCV" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/MachineVerifier/RISCV/subreg-liveness.mir b/llvm/test/MachineVerifier/RISCV/subreg-liveness.mir
index c69bc1b5eca64..3c7b70efe7199 100644
--- a/llvm/test/MachineVerifier/RISCV/subreg-liveness.mir
+++ b/llvm/test/MachineVerifier/RISCV/subreg-liveness.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=none %s -o - | FileCheck %s
-# REQUIRES: riscv64-registered-target
 
 # During the MachineVerifier, it assumes that used registers have been defined
 # In this test case, while $v12_v13_v14_v15_v16 covers $v14_v15,
@@ -20,8 +19,8 @@ body:             |
     ; CHECK: liveins: $v0, $v8, $v9, $v10, $v11
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: renamable $v16m2 = PseudoVMV_V_I_M2 undef renamable $v16m2, 0, -1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: $v20m2 = VMV2R_V $v14m2, implicit $v12_v13_v14_v15_v16
+    ; CHECK-NEXT: $v20m2 = VMV2R_V $v14m2, implicit $v12_v13_v14_v15_v16, implicit $vtype
     renamable $v16m2 = PseudoVMV_V_I_M2 undef renamable $v16m2, 0, -1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    $v20m2 = VMV2R_V $v14m2, implicit $v12_v13_v14_v15_v16
+    $v20m2 = VMV2R_V $v14m2, implicit $v12_v13_v14_v15_v16, implicit $vtype
 
 ...
diff --git a/llvm/test/MachineVerifier/test_abd_su.mir b/llvm/test/MachineVerifier/test_abd_su.mir
new file mode 100644
index 0000000000000..4f9272f238a7d
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_abd_su.mir
@@ -0,0 +1,33 @@
+# RUN: not --crash llc -verify-machineinstrs -mtriple=arm64 -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+---
+name:            g_abd_su
+body: |
+  bb.0:
+
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(p0) = G_IMPLICIT_DEF
+    %4:_(s1) = G_ABDS %2, %3
+
+    %12:_(s64) = G_IMPLICIT_DEF
+    %13:_(s64) = G_IMPLICIT_DEF
+    %14:_(p0) = G_ABDS %12, %13
+
+    %23:_(<2 x s32>) = G_IMPLICIT_DEF
+    %24:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: Generic vector abds/abdu must preserve number of lanes
+    %5:_(s1) = G_ABDU  %23, %24
+
+    %15:_(s32) = G_CONSTANT i32 0
+    %16:_(s64) = G_CONSTANT i64 2
+    ; CHECK: Generic abds/abdu must have same input types
+    %17:_(s1) = G_ABDU %15, %16
+
+    %18:_(s64) = G_CONSTANT i64 0
+    %19:_(s64) = G_CONSTANT i64 2
+    ; CHECK: Generic abds/abdu must have same input and output types
+    %20:_(s1) = G_ABDU %18, %19
+
+...
+
diff --git a/llvm/test/Other/constant-fold-gep.ll b/llvm/test/Other/constant-fold-gep.ll
index e0d535e695273..a4c55c62840a6 100644
--- a/llvm/test/Other/constant-fold-gep.ll
+++ b/llvm/test/Other/constant-fold-gep.ll
@@ -106,10 +106,10 @@
 
 ; PLAIN: @Y = global ptr getelementptr inbounds ([3 x { i32, i32 }], ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 1), i64 1)
 ; PLAIN: @Z = global ptr getelementptr inbounds (i32, ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 0, i64 1, i32 0), i64 1)
-; OPT: @Y = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 48)
-; OPT: @Z = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 12)
-; TO: @Y = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 48)
-; TO: @Z = local_unnamed_addr global ptr getelementptr inbounds (i8, ptr @ext, i64 12)
+; OPT: @Y = local_unnamed_addr global ptr getelementptr inbounds nuw (i8, ptr @ext, i64 48)
+; OPT: @Z = local_unnamed_addr global ptr getelementptr inbounds nuw (i8, ptr @ext, i64 12)
+; TO: @Y = local_unnamed_addr global ptr getelementptr inbounds nuw (i8, ptr @ext, i64 48)
+; TO: @Z = local_unnamed_addr global ptr getelementptr inbounds nuw (i8, ptr @ext, i64 12)
 
 @ext = external global [3 x { i32, i32 }]
 @Y = global ptr getelementptr inbounds ([3 x { i32, i32 }], ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 1), i64 1)
@@ -433,10 +433,10 @@ define ptr @fO() nounwind {
 ; PLAIN:   ret ptr %t
 ; PLAIN: }
 ; OPT: define ptr @fZ() local_unnamed_addr #0 {
-; OPT:   ret ptr getelementptr inbounds (i8, ptr @ext, i64 12)
+; OPT:   ret ptr getelementptr inbounds nuw (i8, ptr @ext, i64 12)
 ; OPT: }
 ; TO: define ptr @fZ() local_unnamed_addr #0 {
-; TO:   ret ptr getelementptr inbounds (i8, ptr @ext, i64 12)
+; TO:   ret ptr getelementptr inbounds nuw (i8, ptr @ext, i64 12)
 ; TO: }
 ; SCEV: Classifying expressions for: @fZ
 ; SCEV:   %t = bitcast ptr getelementptr inbounds (i32, ptr getelementptr inbounds ([3 x { i32, i32 }], ptr @ext, i64 0, i64 1, i32 0), i64 1) to ptr
@@ -457,14 +457,14 @@ define ptr @different_addrspace() nounwind noinline {
   %p = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(12) @p12 to ptr),
                                   i32 2
   ret ptr %p
-; OPT: ret ptr getelementptr inbounds (i8, ptr addrspacecast (ptr addrspace(12) @p12 to ptr), i64 2)
+; OPT: ret ptr getelementptr inbounds nuw (i8, ptr addrspacecast (ptr addrspace(12) @p12 to ptr), i64 2)
 }
 
 define ptr @same_addrspace() nounwind noinline {
 ; OPT: same_addrspace
   %p = getelementptr inbounds i8, ptr @p0, i32 2
   ret ptr %p
-; OPT: ret ptr getelementptr inbounds (i8, ptr @p0, i64 2)
+; OPT: ret ptr getelementptr inbounds nuw (i8, ptr @p0, i64 2)
 }
 
 @gv1 = internal global i32 1
diff --git a/llvm/test/Other/optimize-inrange-gep.ll b/llvm/test/Other/optimize-inrange-gep.ll
index 66cf7f2c17f98..6dea44005353c 100644
--- a/llvm/test/Other/optimize-inrange-gep.ll
+++ b/llvm/test/Other/optimize-inrange-gep.ll
@@ -20,7 +20,7 @@ define void @foo(ptr %p) {
 ;
 ; CHECK-LABEL: define void @foo(
 ; CHECK-SAME: ptr nocapture writeonly initializes((0, 8)) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    store ptr getelementptr inbounds inrange(-24, 0) (i8, ptr @vtable, i64 24), ptr [[P]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw inrange(-24, 0) (i8, ptr @vtable, i64 24), ptr [[P]], align 8
 ; CHECK-NEXT:    ret void
 ;
   store ptr getelementptr inrange(-24, 0) ({ [3 x ptr], [3 x ptr] }, ptr @vtable, i32 0, i32 0, i32 3), ptr %p
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 7dbaf4390c0f7..ffefaba284299 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 1832 bytes
+// R00O-NEXT:  }; // Size: 1840 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,
diff --git a/llvm/test/TableGen/dag-isel-instrument.td b/llvm/test/TableGen/dag-isel-instrument.td
new file mode 100644
index 0000000000000..2862af03cf16d
--- /dev/null
+++ b/llvm/test/TableGen/dag-isel-instrument.td
@@ -0,0 +1,32 @@
+// RUN: llvm-tblgen -gen-dag-isel -instrument-coverage -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def TestTargetInstrInfo : InstrInfo;
+
+def TestTarget : Target {
+  let InstructionSet = TestTargetInstrInfo;
+}
+
+def REG : Register<"REG">;
+def GPR : RegisterClass<"TestTarget", [i32], 32, (add REG)>;
+
+// CHECK-LABEL: OPC_CheckOpcode, TARGET_VAL(ISD::UDIVREM)
+// CHECK: OPC_EmitNode2None, TARGET_VAL(::INSTR)
+// CHECK-NEXT: Results = #2 #3
+// CHECK-NEXT: OPC_Coverage, COVERAGE_IDX_VAL(0),
+// CHECK-NEXT: OPC_CompleteMatch, 2, 3, 2
+def INSTR : Instruction {
+  let OutOperandList = (outs GPR:$r1, GPR:$r0);
+  let InOperandList = (ins GPR:$t0, GPR:$t1);
+  let Pattern = [(set i32:$r0, i32:$r1, (udivrem i32:$t0, i32:$t1))];
+}
+
+
+// CHECK: getPatternForIndex(unsigned Index)
+// CHECK: static const char *PATTERN_MATCH_TABLE[]
+// CHECK: return StringRef(PATTERN_MATCH_TABLE[Index]);
+
+// CHECK: getIncludePathForIndex(unsigned Index)
+// CHECK: static const char *INCLUDE_PATH_TABLE[]
+// CHECK: return StringRef(INCLUDE_PATH_TABLE[Index]);
diff --git a/llvm/test/TableGen/intrinsic-overload-conflict.td b/llvm/test/TableGen/intrinsic-overload-conflict.td
index 84333119d41f5..13431c3bc49e0 100644
--- a/llvm/test/TableGen/intrinsic-overload-conflict.td
+++ b/llvm/test/TableGen/intrinsic-overload-conflict.td
@@ -6,13 +6,17 @@ include "llvm/IR/Intrinsics.td"
 // CHECK: foo = 1,
 def int_foo : Intrinsic<[llvm_any_ty]>;
 
-// No conflicts, since .bar is not a vaid mangled type.
+// No conflicts, since .bar is not a valid mangled type.
 // CHECK: foo_bar,
 def int_foo_bar : Intrinsic<[llvm_i32_ty]>;
 
 // CHECK: foo_bar_f32,
 def int_foo_bar_f32 : Intrinsic<[llvm_i32_ty]>;
 
+// No conflicts, since i is not a valid mangled type without a bitwidth.
+// CHECK: foo_i
+def int_foo_i : Intrinsic<[llvm_i32_ty]>;
+
 #ifdef CONFLICT
 // CHECK-CONFLICT: error: intrinsic `llvm.foo.a3` cannot share prefix `llvm.foo.a3` with another overloaded intrinsic `llvm.foo`
 // CHECK-CONFLICT: error: intrinsic `llvm.foo.bf16` cannot share prefix `llvm.foo.bf16` with another overloaded intrinsic `llvm.foo`
diff --git a/llvm/test/TableGen/letUnknownValue.td b/llvm/test/TableGen/letUnknownValue.td
new file mode 100644
index 0000000000000..e33dca417f1d1
--- /dev/null
+++ b/llvm/test/TableGen/letUnknownValue.td
@@ -0,0 +1,9 @@
+// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s --strict-whitespace
+
+def {
+  /// Let can only override something that already exists.
+  let abc = [];
+// CHECK: error: Value 'abc' unknown!
+// CHECK-NEXT:{{^}}  let abc = [];
+// CHECK-NEXT:{{^}}      ^
+}
\ No newline at end of file
diff --git a/llvm/test/ThinLTO/X86/distributed_indexes.ll b/llvm/test/ThinLTO/X86/distributed_indexes.ll
index 4f2662b1b34e1..824c582c2025e 100644
--- a/llvm/test/ThinLTO/X86/distributed_indexes.ll
+++ b/llvm/test/ThinLTO/X86/distributed_indexes.ll
@@ -48,6 +48,10 @@
 ; RUN: llvm-dis %t1.bc.thinlto.bc -o - | FileCheck %s --check-prefix=DIS
 ; DIS: aliasee: null
 
+; RUN: opt -passes=function-import -import-all-index -summary-file=%t1.bc.thinlto.bc %t1.bc -S -o - 2>&1 | FileCheck %s --check-prefix=IR
+; Tests that analias definition is imported.
+; IR: define available_externally void @analias
+
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 declare void @g(...)
diff --git a/llvm/test/ThinLTO/X86/import_callee_declaration.ll b/llvm/test/ThinLTO/X86/import_callee_declaration.ll
index 246920e5db0dc..72550fa4d6f0b 100644
--- a/llvm/test/ThinLTO/X86/import_callee_declaration.ll
+++ b/llvm/test/ThinLTO/X86/import_callee_declaration.ll
@@ -34,11 +34,14 @@
 ; RUN:   -r=main.bc,main,px \
 ; RUN:   -r=main.bc,small_func, \
 ; RUN:   -r=main.bc,large_func, \
+; RUN:   -r=main.bc,read_write_global_vars, \
+; RUN:   -r=main.bc,external_func, \
 ; RUN:   -r=lib.bc,callee,pl \
 ; RUN:   -r=lib.bc,large_indirect_callee,px \
 ; RUN:   -r=lib.bc,large_indirect_bar,px \
 ; RUN:   -r=lib.bc,small_func,px \
 ; RUN:   -r=lib.bc,large_func,px \
+; RUN:   -r=lib.bc,read_write_global_vars,px \
 ; RUN:   -r=lib.bc,large_indirect_callee_alias,px \
 ; RUN:   -r=lib.bc,large_indirect_bar_alias,px \
 ; RUN:   -r=lib.bc,calleeAddrs,px -r=lib.bc,calleeAddrs2,px -o summary main.bc lib.bc 2>&1 | FileCheck %s --check-prefix=DUMP
@@ -71,13 +74,22 @@
 ; MAIN-DIS: [[LARGEINDIRECT:\^[0-9]+]] = gv: (guid: 14343440786664691134, summaries: (function: (module: [[LIBMOD]], flags: ({{.*}} importType: declaration), insts: 8, {{.*}})))
 ; MAIN-DIS: gv: (guid: 16730173943625350469, summaries: (alias: (module: [[LIBMOD]], flags: ({{.*}} importType: declaration), aliasee: [[LARGEINDIRECT]])))
 
+; RUN: opt -passes=function-import -import-all-index -summary-file=main.bc.thinlto.bc main.bc -o main-after-import.bc
+; RUN: llvm-dis -o - main-after-import.bc | FileCheck %s --check-prefix=MAIN-IMPORT
+
+; Tests that dso_local attribute is applied on a global var from its summary.
+MAIN-IMPORT: @read_write_global_vars = external dso_local global [1 x ptr]
+
 ; Run in-process ThinLTO and tests that
 ; 1. `callee` remains internalized even if the symbols of its callers
 ; (large_func, large_indirect_callee, large_indirect_bar) are exported as
 ; declarations and visible to main module.
 ; 2. the debugging logs from `function-import` pass are expected.
+; Set relocation model to static so the dso_local attribute from a summary is
+; applied on the global variable declaration.
 
 ; RUN: llvm-lto2 run \
+; RUN:   -relocation-model=static \
 ; RUN:   -debug-only=function-import \
 ; RUN:   -save-temps \
 ; RUN:   -thinlto-threads=1 \
@@ -87,11 +99,14 @@
 ; RUN:   -r=main.bc,main,px \
 ; RUN:   -r=main.bc,small_func, \
 ; RUN:   -r=main.bc,large_func, \
+; RUN:   -r=main.bc,read_write_global_vars, \
+; RUN:   -r=main.bc,external_func, \
 ; RUN:   -r=lib.bc,callee,pl \
 ; RUN:   -r=lib.bc,large_indirect_callee,px \
 ; RUN:   -r=lib.bc,large_indirect_bar,px \
 ; RUN:   -r=lib.bc,small_func,px \
 ; RUN:   -r=lib.bc,large_func,px \
+; RUN:   -r=lib.bc,read_write_global_vars,px \
 ; RUN:   -r=lib.bc,large_indirect_callee_alias,px \
 ; RUN:   -r=lib.bc,large_indirect_bar_alias,px \
 ; RUN:   -r=lib.bc,calleeAddrs,px -r=lib.bc,calleeAddrs2,px -o in-process main.bc lib.bc 2>&1 | FileCheck %s --check-prefix=IMPORTDUMP
@@ -103,7 +118,7 @@
 ; IMPORTDUMP-DAG: Is importing function definition 13568239288960714650 small_indirect_callee from lib.cc
 ; IMPORTDUMP-DAG: Is importing function definition 6976996067367342685 small_func from lib.cc
 ; IMPORTDUMP-DAG: Is importing function declaration 2418497564662708935 large_func from lib.cc
-; IMPORTDUMP-DAG: Not importing global 7680325410415171624 calleeAddrs from lib.cc
+; IMPORTDUMP-DAG: Is importing global declaration 7680325410415171624 calleeAddrs from lib.cc
 ; IMPORTDUMP-DAG: Is importing alias declaration 16730173943625350469 large_indirect_callee_alias from lib.cc
 ; IMPORTDUMP-DAG: Is importing alias declaration 13590951773474913315 large_indirect_bar_alias from lib.cc
 ; IMPORTDUMP-DAG: Not importing function 13770917885399536773 large_indirect_bar
@@ -115,6 +130,8 @@
 ; IMPORT-DAG: define available_externally void @small_func
 ; IMPORT-DAG: define available_externally hidden void @small_indirect_callee
 ; IMPORT-DAG: declare void @large_func
+; Tests that dso_local attribute is applied on a global var from its summary.
+; IMPORT-DAG: @read_write_global_vars = external dso_local global [1 x ptr]
 ; IMPORT-NOT: large_indirect_callee
 ; IMPORT-NOT: large_indirect_callee_alias
 ; IMPORT-NOT: large_indirect_bar
@@ -126,9 +143,14 @@
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
+@read_write_global_vars = external global [1 x ptr]
+
 define i32 @main() {
   call void @small_func()
   call void @large_func()
+  %num = call ptr @external_func(ptr @read_write_global_vars)
+  store ptr %num, ptr getelementptr inbounds ([1 x ptr], ptr @read_write_global_vars, i64 0, i64 0)
+  %res1 = call i32 @external_func(ptr @read_write_global_vars)
   ret i32 0
 }
 
@@ -137,6 +159,8 @@ declare void @small_func()
 ; large_func without attributes
 declare void @large_func()
 
+declare ptr @external_func(ptr)
+
 ;--- lib.ll
 source_filename = "lib.cc"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
@@ -149,6 +173,10 @@ target triple = "x86_64-unknown-linux-gnu"
 ; large_indirect_bar_alias is visible to main.ll but its aliasee isn't.
 @calleeAddrs2 = global [1 x ptr] [ptr @large_indirect_bar_alias]
 
+; @read_write_global_vars is not read-only nor write-only (in main.ll). It's not
+; a constant global var and has references, so it's not importable as a definition.
+@read_write_global_vars = dso_local global [1 x ptr] [ptr @large_indirect_callee]
+
 define void @callee() #1 {
   ret void
 }
diff --git a/llvm/test/Transforms/AggressiveInstCombine/AArch64/or-load.ll b/llvm/test/Transforms/AggressiveInstCombine/AArch64/or-load.ll
index 1400ee7f703ca..10c4c9b0ca4c9 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/AArch64/or-load.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/AArch64/or-load.ll
@@ -1121,19 +1121,19 @@ entry:
 
 define i32 @loadCombine_4consecutive_metadata(ptr %p, ptr %pstr) {
 ; LE-LABEL: @loadCombine_4consecutive_metadata(
-; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[P:%.*]], align 1, !alias.scope !0
-; LE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias !0
+; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[P:%.*]], align 1, !alias.scope [[META0:![0-9]+]]
+; LE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias [[META0]]
 ; LE-NEXT:    ret i32 [[L1]]
 ;
 ; BE-LABEL: @loadCombine_4consecutive_metadata(
 ; BE-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
 ; BE-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i32 2
 ; BE-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[P]], i32 3
-; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1, !alias.scope !0
-; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1, !alias.scope !0
-; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1, !alias.scope !0
-; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1, !alias.scope !0
-; BE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias !0
+; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1, !alias.scope [[META0:![0-9]+]]
+; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias [[META0]]
 ; BE-NEXT:    [[E1:%.*]] = zext i8 [[L1]] to i32
 ; BE-NEXT:    [[E2:%.*]] = zext i8 [[L2]] to i32
 ; BE-NEXT:    [[E3:%.*]] = zext i8 [[L3]] to i32
@@ -1869,7 +1869,7 @@ define i32 @loadCombine_4consecutive_badinsert2(ptr %p) {
 
 define i32 @loadCombine_4consecutive_badinsert3(ptr %p) {
 ; LE-LABEL: @loadCombine_4consecutive_badinsert3(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1
 ; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[TMP1]], align 1
 ; LE-NEXT:    ret i32 [[L1]]
 ;
@@ -2088,7 +2088,7 @@ define i32 @loadCombine_4consecutive_badinsert6(ptr %p) {
 
 define void @nested_gep(ptr %p, ptr %dest) {
 ; LE-LABEL: @nested_gep(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 68
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 68
 ; LE-NEXT:    [[LD2:%.*]] = load i64, ptr [[TMP1]], align 4
 ; LE-NEXT:    [[TRUNC:%.*]] = trunc i64 [[LD2]] to i32
 ; LE-NEXT:    store i32 [[TRUNC]], ptr [[DEST:%.*]], align 4
@@ -2128,7 +2128,7 @@ define void @nested_gep(ptr %p, ptr %dest) {
 
 define void @bitcast_gep(ptr %p, ptr %dest) {
 ; LE-LABEL: @bitcast_gep(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 68
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 68
 ; LE-NEXT:    [[LD2:%.*]] = load i64, ptr [[TMP1]], align 4
 ; LE-NEXT:    [[TRUNC:%.*]] = trunc i64 [[LD2]] to i32
 ; LE-NEXT:    store i32 [[TRUNC]], ptr [[DEST:%.*]], align 4
diff --git a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
index 0aa6f9ecdf884..1b53c8f71222b 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
@@ -1205,19 +1205,19 @@ entry:
 
 define i32 @loadCombine_4consecutive_metadata(ptr %p, ptr %pstr) {
 ; LE-LABEL: @loadCombine_4consecutive_metadata(
-; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[P:%.*]], align 1, !alias.scope !0
-; LE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias !0
+; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[P:%.*]], align 1, !alias.scope [[META0:![0-9]+]]
+; LE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias [[META0]]
 ; LE-NEXT:    ret i32 [[L1]]
 ;
 ; BE-LABEL: @loadCombine_4consecutive_metadata(
 ; BE-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
 ; BE-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i32 2
 ; BE-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[P]], i32 3
-; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1, !alias.scope !0
-; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1, !alias.scope !0
-; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1, !alias.scope !0
-; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1, !alias.scope !0
-; BE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias !0
+; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1, !alias.scope [[META0:![0-9]+]]
+; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1, !alias.scope [[META0]]
+; BE-NEXT:    store i32 25, ptr [[PSTR:%.*]], align 4, !noalias [[META0]]
 ; BE-NEXT:    [[E1:%.*]] = zext i8 [[L1]] to i32
 ; BE-NEXT:    [[E2:%.*]] = zext i8 [[L2]] to i32
 ; BE-NEXT:    [[E3:%.*]] = zext i8 [[L3]] to i32
@@ -2005,7 +2005,7 @@ define i32 @loadCombine_4consecutive_badinsert2(ptr %p) {
 
 define i32 @loadCombine_4consecutive_badinsert3(ptr %p) {
 ; LE-LABEL: @loadCombine_4consecutive_badinsert3(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1
 ; LE-NEXT:    [[L1:%.*]] = load i32, ptr [[TMP1]], align 1
 ; LE-NEXT:    ret i32 [[L1]]
 ;
@@ -2306,7 +2306,7 @@ define i64 @loadCombine_nonConstShift2(ptr %arg, i8 %b) {
 
 define void @nested_gep(ptr %p, ptr %dest) {
 ; LE-LABEL: @nested_gep(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 68
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 68
 ; LE-NEXT:    [[LD2:%.*]] = load i64, ptr [[TMP1]], align 4
 ; LE-NEXT:    [[TRUNC:%.*]] = trunc i64 [[LD2]] to i32
 ; LE-NEXT:    store i32 [[TRUNC]], ptr [[DEST:%.*]], align 4
@@ -2346,7 +2346,7 @@ define void @nested_gep(ptr %p, ptr %dest) {
 
 define void @bitcast_gep(ptr %p, ptr %dest) {
 ; LE-LABEL: @bitcast_gep(
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 68
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 68
 ; LE-NEXT:    [[LD2:%.*]] = load i64, ptr [[TMP1]], align 4
 ; LE-NEXT:    [[TRUNC:%.*]] = trunc i64 [[LD2]] to i32
 ; LE-NEXT:    store i32 [[TRUNC]], ptr [[DEST:%.*]], align 4
@@ -2382,3 +2382,35 @@ define void @bitcast_gep(ptr %p, ptr %dest) {
   store i32 %trunc, ptr %dest, align 4
   ret void
 }
+
+define i32 @loadcombine_consecutive_idx_64(ptr %data) {
+; LE-LABEL: @loadcombine_consecutive_idx_64(
+; LE-NEXT:  entry:
+; LE-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[DATA:%.*]], i64 2149675576
+; LE-NEXT:    [[VAL_2:%.*]] = load i16, ptr [[TMP0]], align 1
+; LE-NEXT:    [[TMP1:%.*]] = zext i16 [[VAL_2]] to i32
+; LE-NEXT:    ret i32 [[TMP1]]
+;
+; BE-LABEL: @loadcombine_consecutive_idx_64(
+; BE-NEXT:  entry:
+; BE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA:%.*]], i64 2149675577
+; BE-NEXT:    [[VAL:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; BE-NEXT:    [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; BE-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 2149675576
+; BE-NEXT:    [[VAL_2:%.*]] = load i8, ptr [[ARRAYIDX_2]], align 1
+; BE-NEXT:    [[CONV_2:%.*]] = zext i8 [[VAL_2]] to i32
+; BE-NEXT:    [[SHL:%.*]] = shl nuw nsw i32 [[CONV]], 8
+; BE-NEXT:    [[OR:%.*]] = or disjoint i32 [[SHL]], [[CONV_2]]
+; BE-NEXT:    ret i32 [[OR]]
+;
+entry:
+  %arrayidx = getelementptr inbounds nuw i8, ptr %data, i64 2149675577
+  %val = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %val to i32
+  %arrayidx.2 = getelementptr inbounds nuw i8, ptr %data, i64 2149675576
+  %val.2 = load i8, ptr %arrayidx.2, align 1
+  %conv.2 = zext i8 %val.2 to i32
+  %shl = shl nuw nsw i32 %conv, 8
+  %or = or disjoint i32 %shl, %conv.2
+  ret i32 %or
+}
diff --git a/llvm/test/Transforms/AtomicExpand/LoongArch/atomicrmw-expand.ll b/llvm/test/Transforms/AtomicExpand/LoongArch/atomicrmw-expand.ll
new file mode 100644
index 0000000000000..11859b27c1026
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/LoongArch/atomicrmw-expand.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --mtriple=loongarch64 --mattr=+d,+lamcas %s | FileCheck %s --check-prefix=NO-EXPAND
+; RUN: opt -S --mtriple=loongarch64 --passes=atomic-expand --mattr=+d,+lamcas %s | FileCheck %s --check-prefix=EXPAND
+
+; When -mlamcas is enabled, all atomicrmw and/or/xor ptr %a,a i8/i16 %b ordering 
+; will be expanded to am{and/or/xo}[_db].w by LoongArchTargetLowering::emitExpandAtomicRMW
+
+define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
+; NO-EXPAND-LABEL: define i8 @atomicrmw_and_i8_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw and ptr [[A]], i8 [[B]] acquire, align 1
+; NO-EXPAND-NEXT:    ret i8 [[TMP1]]
+;
+; EXPAND-LABEL: define i8 @atomicrmw_and_i8_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i8 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr [[ALIGNEDADDR]], i32 [[ANDOPERAND]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; EXPAND-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %1 = atomicrmw and ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind {
+; NO-EXPAND-LABEL: define i16 @atomicrmw_and_i16_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw and ptr [[A]], i16 [[B]] acquire, align 2
+; NO-EXPAND-NEXT:    ret i16 [[TMP1]]
+;
+; EXPAND-LABEL: define i16 @atomicrmw_and_i16_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i16 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr [[ALIGNEDADDR]], i32 [[ANDOPERAND]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; EXPAND-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %1 = atomicrmw and ptr %a, i16 %b acquire
+  ret i16 %1
+
+}
+
+define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind {
+; NO-EXPAND-LABEL: define i8 @atomicrmw_or_i8_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw or ptr [[A]], i8 [[B]] acquire, align 1
+; NO-EXPAND-NEXT:    ret i8 [[TMP1]]
+;
+; EXPAND-LABEL: define i8 @atomicrmw_or_i8_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i8 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; EXPAND-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %1 = atomicrmw or ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind {
+; NO-EXPAND-LABEL: define i16 @atomicrmw_or_i16_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw or ptr [[A]], i16 [[B]] acquire, align 2
+; NO-EXPAND-NEXT:    ret i16 [[TMP1]]
+;
+; EXPAND-LABEL: define i16 @atomicrmw_or_i16_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i16 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; EXPAND-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %1 = atomicrmw or ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind {
+; NO-EXPAND-LABEL: define i8 @atomicrmw_xor_i8_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw xor ptr [[A]], i8 [[B]] acquire, align 1
+; NO-EXPAND-NEXT:    ret i8 [[TMP1]]
+;
+; EXPAND-LABEL: define i8 @atomicrmw_xor_i8_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i8 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; EXPAND-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %1 = atomicrmw xor ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind {
+; NO-EXPAND-LABEL: define i16 @atomicrmw_xor_i16_acquire(
+; NO-EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; NO-EXPAND-NEXT:    [[TMP1:%.*]] = atomicrmw xor ptr [[A]], i16 [[B]] acquire, align 2
+; NO-EXPAND-NEXT:    ret i16 [[TMP1]]
+;
+; EXPAND-LABEL: define i16 @atomicrmw_xor_i16_acquire(
+; EXPAND-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; EXPAND-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[A]], i64 -4)
+; EXPAND-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[A]] to i64
+; EXPAND-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; EXPAND-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; EXPAND-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; EXPAND-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; EXPAND-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; EXPAND-NEXT:    [[TMP3:%.*]] = zext i16 [[B]] to i32
+; EXPAND-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] acquire, align 4
+; EXPAND-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; EXPAND-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; EXPAND-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %1 = atomicrmw xor ptr %a, i16 %b acquire
+  ret i16 %1
+}
diff --git a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
index b94023b97a295..cc51a00db415c 100644
--- a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
+++ b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
@@ -30,11 +30,11 @@ define i1 @test_cmpxchg_seq_cst(ptr %addr, i128 %desire, i128 %new) {
 ;
 ; PWR7-LABEL: @test_cmpxchg_seq_cst(
 ; PWR7-NEXT:  entry:
-; PWR7-NEXT:    [[TMP0:%.*]] = alloca i128, align 8
+; PWR7-NEXT:    [[TMP0:%.*]] = alloca i128, align 16
 ; PWR7-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP0]])
-; PWR7-NEXT:    store i128 [[DESIRE:%.*]], ptr [[TMP0]], align 8
+; PWR7-NEXT:    store i128 [[DESIRE:%.*]], ptr [[TMP0]], align 16
 ; PWR7-NEXT:    [[TMP1:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[ADDR:%.*]], ptr [[TMP0]], i128 [[NEW:%.*]], i32 5, i32 5)
-; PWR7-NEXT:    [[TMP2:%.*]] = load i128, ptr [[TMP0]], align 8
+; PWR7-NEXT:    [[TMP2:%.*]] = load i128, ptr [[TMP0]], align 16
 ; PWR7-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP0]])
 ; PWR7-NEXT:    [[TMP3:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP2]], 0
 ; PWR7-NEXT:    [[TMP4:%.*]] = insertvalue { i128, i1 } [[TMP3]], i1 [[TMP1]], 1
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr118172.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr118172.ll
new file mode 100644
index 0000000000000..a599129466b35
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr118172.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-unknown-unknown < %s | FileCheck %s
+
+; Make sure the nsw flag is dropped when the load ext is combined.
+define i32 @simplify_load_ext_drop_trunc_nsw(ptr %p) {
+; CHECK-LABEL: define i32 @simplify_load_ext_drop_trunc_nsw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[TMP0]] to i8
+; CHECK-NEXT:    [[EXT1:%.*]] = sext i8 [[TRUNC]] to i16
+; CHECK-NEXT:    call void @use(i32 [[TMP0]])
+; CHECK-NEXT:    [[EXT2:%.*]] = zext i16 [[EXT1]] to i32
+; CHECK-NEXT:    ret i32 [[EXT2]]
+;
+entry:
+  %x = load i32, ptr %p, align 4
+  %trunc = trunc nsw i32 %x to i8
+  %ext1 = sext i8 %trunc to i16
+  %conv2 = and i32 %x, 255
+  call void @use(i32 %conv2)
+  %ext2 = zext i16 %ext1 to i32
+  ret i32 %ext2
+}
+
+; Make sure the nsw flag is dropped when the load ext is combined.
+define i32 @simplify_load_ext_drop_shl_nsw(ptr %p) {
+; CHECK-LABEL: define i32 @simplify_load_ext_drop_shl_nsw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP0]], 24
+; CHECK-NEXT:    call void @use(i32 [[TMP0]])
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+entry:
+  %x = load i32, ptr %p, align 4
+  %shl = shl nsw i32 %x, 24
+  %conv2 = and i32 %x, 255
+  call void @use(i32 %conv2)
+  ret i32 %shl
+}
+
+define i32 @simplify_load_ext_keep_trunc_nuw(ptr %p) {
+; CHECK-LABEL: define i32 @simplify_load_ext_keep_trunc_nuw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i32 [[TMP0]] to i8
+; CHECK-NEXT:    [[EXT1:%.*]] = sext i8 [[TRUNC]] to i16
+; CHECK-NEXT:    call void @use(i32 [[TMP0]])
+; CHECK-NEXT:    [[EXT2:%.*]] = zext i16 [[EXT1]] to i32
+; CHECK-NEXT:    ret i32 [[EXT2]]
+;
+entry:
+  %x = load i32, ptr %p, align 4
+  %trunc = trunc nuw i32 %x to i8
+  %ext1 = sext i8 %trunc to i16
+  %conv2 = and i32 %x, 255
+  call void @use(i32 %conv2)
+  %ext2 = zext i16 %ext1 to i32
+  ret i32 %ext2
+}
+
+define i32 @simplify_load_ext_drop_shl_nuw(ptr %p) {
+; CHECK-LABEL: define i32 @simplify_load_ext_drop_shl_nuw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[TMP0]], 24
+; CHECK-NEXT:    call void @use(i32 [[TMP0]])
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+entry:
+  %x = load i32, ptr %p, align 4
+  %shl = shl nuw i32 %x, 24
+  %conv2 = and i32 %x, 255
+  call void @use(i32 %conv2)
+  ret i32 %shl
+}
+
+declare void @use(i32)
diff --git a/llvm/test/Transforms/ConstraintElimination/constraint-overflow.ll b/llvm/test/Transforms/ConstraintElimination/constraint-overflow.ll
index 88f87f4afab28..57b7b11be0cf1 100644
--- a/llvm/test/Transforms/ConstraintElimination/constraint-overflow.ll
+++ b/llvm/test/Transforms/ConstraintElimination/constraint-overflow.ll
@@ -38,3 +38,17 @@ exit:
 }
 
 declare void @llvm.assume(i1)
+
+define i1 @negate_overflow_add_1(i64 %x) {
+; CHECK-LABEL: define i1 @negate_overflow_add_1(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i64 [[X]], -9223372036854775807
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 0, [[SUB]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+entry:
+  %sub = add nsw i64 %x, -9223372036854775807
+  %c = icmp slt i64 0, %sub
+  ret i1 %c
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/gep-arithmetic.ll b/llvm/test/Transforms/ConstraintElimination/gep-arithmetic.ll
index a4d825b327969..23e1698136f5e 100644
--- a/llvm/test/Transforms/ConstraintElimination/gep-arithmetic.ll
+++ b/llvm/test/Transforms/ConstraintElimination/gep-arithmetic.ll
@@ -630,3 +630,138 @@ ptr.check:
 exit:
   ret i4 3
 }
+
+define i1 @test_nusw(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: @test_nusw(
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X_EXT]], [[Y_EXT]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 [[X_EXT]]
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[Y_EXT]]
+; CHECK-NEXT:    ret i1 true
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %cmp1 = icmp ugt i64 %x.ext, %y.ext
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nusw i8, ptr %p, i64 %x.ext
+  %gep.y = getelementptr nusw i8, ptr %p, i64 %y.ext
+  %cmp2 = icmp ugt ptr %gep.x, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_nusw_nested(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: @test_nusw_nested(
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X_EXT]], [[Y_EXT]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 [[X_EXT]]
+; CHECK-NEXT:    [[GEP_X1:%.*]] = getelementptr nusw i8, ptr [[GEP_X]], i64 1
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[Y_EXT]]
+; CHECK-NEXT:    ret i1 true
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %cmp1 = icmp ugt i64 %x.ext, %y.ext
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nusw i8, ptr %p, i64 %x.ext
+  %gep.x1 = getelementptr nusw i8, ptr %gep.x, i64 1
+  %gep.y = getelementptr nusw i8, ptr %p, i64 %y.ext
+  %cmp2 = icmp ugt ptr %gep.x1, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_missing_nusw(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: @test_missing_nusw(
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X_EXT]], [[Y_EXT]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 [[X_EXT]]
+; CHECK-NEXT:    [[GEP_X1:%.*]] = getelementptr i8, ptr [[GEP_X]], i64 1
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[Y_EXT]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt ptr [[GEP_X1]], [[GEP_Y]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %cmp1 = icmp ugt i64 %x.ext, %y.ext
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nusw i8, ptr %p, i64 %x.ext
+  %gep.x1 = getelementptr i8, ptr %gep.x, i64 1
+  %gep.y = getelementptr nusw i8, ptr %p, i64 %y.ext
+  %cmp2 = icmp ugt ptr %gep.x1, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_nuw(ptr %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test_nuw(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[Y]]
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ugt i64 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nuw i8, ptr %p, i64 %x
+  %gep.y = getelementptr nuw i8, ptr %p, i64 %y
+  %cmp2 = icmp ugt ptr %gep.x, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_nuw_nested(ptr %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test_nuw_nested(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[GEP_X1:%.*]] = getelementptr nuw i8, ptr [[GEP_X]], i64 1
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[Y]]
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ugt i64 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nuw i8, ptr %p, i64 %x
+  %gep.x1 = getelementptr nuw i8, ptr %gep.x, i64 1
+  %gep.y = getelementptr nuw i8, ptr %p, i64 %y
+  %cmp2 = icmp ugt ptr %gep.x1, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_nuw_nested_missing_nuw(ptr %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test_nuw_nested_missing_nuw(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[GEP_X1:%.*]] = getelementptr nuw i8, ptr [[GEP_X]], i64 1
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[Y]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt ptr [[GEP_X1]], [[GEP_Y]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i64 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr i8, ptr %p, i64 %x
+  %gep.x1 = getelementptr nuw i8, ptr %gep.x, i64 1
+  %gep.y = getelementptr nuw i8, ptr %p, i64 %y
+  %cmp2 = icmp ugt ptr %gep.x1, %gep.y
+  ret i1 %cmp2
+}
+
+define i1 @test_nuw_incorrect_precondition(ptr %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test_nuw_incorrect_precondition(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[Y]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt ptr [[GEP_X]], [[GEP_Y]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp uge i64 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+  %gep.x = getelementptr nuw i8, ptr %p, i64 %x
+  %gep.y = getelementptr nuw i8, ptr %p, i64 %y
+  %cmp2 = icmp ugt ptr %gep.x, %gep.y
+  ret i1 %cmp2
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/trunc.ll b/llvm/test/Transforms/ConstraintElimination/trunc.ll
new file mode 100644
index 0000000000000..620d978149874
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/trunc.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=constraint-elimination -S %s | FileCheck %s
+
+define i1 @test_icmp_ult_zext_icmp_trunc_nuw(i16 %x, i32 %y) {
+; CHECK-LABEL: define i1 @test_icmp_ult_zext_icmp_trunc_nuw(
+; CHECK-SAME: i16 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[X]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[Y]], [[EXT]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc nuw i32 [[Y]] to i16
+; CHECK-NEXT:    ret i1 false
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = zext i16 %x to i32
+  %cond = icmp ult i32 %y, %ext
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %conv = trunc nuw i32 %y to i16
+  %cmp = icmp eq i16 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i1 @test_icmp_slt_sext_icmp_trunc_nsw(i16 %x, i32 %y) {
+; CHECK-LABEL: define i1 @test_icmp_slt_sext_icmp_trunc_nsw(
+; CHECK-SAME: i16 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[X]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[Y]], [[EXT]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc nsw i32 [[Y]] to i16
+; CHECK-NEXT:    ret i1 false
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = sext i16 %x to i32
+  %cond = icmp slt i32 %y, %ext
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %conv = trunc nsw i32 %y to i16
+  %cmp = icmp slt i16 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i1 @test_icmp_ult_trunc_nsw_nneg_icmp_trunc_nuw(i64 %x, i32 %y) {
+; CHECK-LABEL: define i1 @test_icmp_ult_trunc_nsw_nneg_icmp_trunc_nuw(
+; CHECK-SAME: i64 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nsw i64 [[X]] to i32
+; CHECK-NEXT:    [[NNEG:%.*]] = icmp sgt i64 [[X]], -1
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[Y]], [[EXT]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[NNEG]], [[COND]]
+; CHECK-NEXT:    br i1 [[AND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i32 [[Y]] to i64
+; CHECK-NEXT:    ret i1 false
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = trunc nsw i64 %x to i32
+  %nneg = icmp sgt i64 %x, -1
+  %cond = icmp ult i32 %y, %ext
+  %and = and i1 %nneg, %cond
+  br i1 %and, label %if.then, label %if.else
+
+if.then:
+  %conv = zext i32 %y to i64
+  %cmp = icmp eq i64 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i1 @test2(i32 %n) {
+; CHECK-LABEL: define i1 @test2(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT:    [[END:%.*]] = add nsw i64 [[EXT]], -1
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, %[[IF_THEN]] ], [ [[INDVAR_NEXT:%.*]], %[[FOR_NEXT:.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[INDVAR]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_ELSE]], label %[[FOR_NEXT]]
+; CHECK:       [[FOR_NEXT]]:
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add nuw nsw i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[COND2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nsw i64 [[INDVAR_NEXT]] to i32
+; CHECK-NEXT:    ret i1 true
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %cond = icmp sgt i32 %n, 0
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %ext = zext nneg i32 %n to i64
+  %end = add nsw i64 %ext, -1
+  br label %for.body
+
+for.body:
+  %indvar = phi i64 [ 0, %if.then ], [ %indvar.next, %for.next ]
+  %cmp = icmp eq i64 %indvar, %end
+  br i1 %cmp, label %if.else, label %for.next
+
+for.next:
+  %indvar.next = add nuw nsw i64 %indvar, 1
+  %cond2 = call i1 @cond()
+  br i1 %cond2, label %for.body, label %for.end
+
+for.end:
+  %trunc = trunc nsw i64 %indvar.next to i32
+  %res = icmp sgt i32 %n, %trunc
+  ret i1 %res
+
+if.else:
+  ret i1 false
+}
+
+define i1 @test_icmp_ult_zext_icmp_trunc(i16 %x, i32 %y) {
+; CHECK-LABEL: define i1 @test_icmp_ult_zext_icmp_trunc(
+; CHECK-SAME: i16 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[X]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[Y]], [[EXT]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[Y]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[X]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = zext i16 %x to i32
+  %cond = icmp ult i32 %y, %ext
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %conv = trunc i32 %y to i16
+  %cmp = icmp eq i16 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i1 @test_icmp_ult_zext_icmp_trunc_nuw_i128(i16 %x, i128 %y) {
+; CHECK-LABEL: define i1 @test_icmp_ult_zext_icmp_trunc_nuw_i128(
+; CHECK-SAME: i16 [[X:%.*]], i128 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[X]] to i128
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i128 [[Y]], [[EXT]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc nuw i128 [[Y]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[X]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = zext i16 %x to i128
+  %cond = icmp ult i128 %y, %ext
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %conv = trunc nuw i128 %y to i16
+  %cmp = icmp eq i16 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+; We do not know the sign of %x, so we cannot infer nuw for %ext.
+define i1 @test_icmp_ult_trunc_nsw_icmp_trunc_nuw(i64 %x, i32 %y) {
+; CHECK-LABEL: define i1 @test_icmp_ult_trunc_nsw_icmp_trunc_nuw(
+; CHECK-SAME: i64 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nsw i64 [[X]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[Y]], [[EXT]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i32 [[Y]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[X]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    ret i1 false
+;
+  %ext = trunc nsw i64 %x to i32
+  %cond = icmp ult i32 %y, %ext
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %conv = zext i32 %y to i64
+  %cmp = icmp eq i64 %x, %conv
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+declare void @cond()
diff --git a/llvm/test/Transforms/Coroutines/coro-async.ll b/llvm/test/Transforms/Coroutines/coro-async.ll
index f02d0a242dc99..acc448559c7fa 100644
--- a/llvm/test/Transforms/Coroutines/coro-async.ll
+++ b/llvm/test/Transforms/Coroutines/coro-async.ll
@@ -120,19 +120,19 @@ define void @my_async_function_pa(ptr %ctxt, ptr %task, ptr %actor) {
 ; CHECK-O0-LABEL: define swiftcc void @my_async_function(ptr swiftasync %async.ctxt, ptr %task, ptr %actor)
 ; CHECK-SAME: !dbg ![[SP1:[0-9]+]] {
 ; CHECK: coro.return:
-; CHECK:   [[FRAMEPTR:%.*]] = getelementptr inbounds i8, ptr %async.ctxt, i64 128
-; CHECK:   [[ACTOR_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr %async.ctxt, i64 152
+; CHECK:   [[FRAMEPTR:%.*]] = getelementptr inbounds nuw i8, ptr %async.ctxt, i64 128
+; CHECK:   [[ACTOR_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr %async.ctxt, i64 152
 ; CHECK:   store ptr %actor, ptr [[ACTOR_SPILL_ADDR]]
-; CHECK:   [[ADDR1:%.*]]  = getelementptr inbounds i8, ptr %async.ctxt, i64 144
+; CHECK:   [[ADDR1:%.*]]  = getelementptr inbounds nuw i8, ptr %async.ctxt, i64 144
 ; CHECK:   store ptr %async.ctxt, ptr [[ADDR1]]
-; CHECK:   [[ALLOCA_PRJ2:%.*]] = getelementptr inbounds i8, ptr %async.ctxt, i64 136
+; CHECK:   [[ALLOCA_PRJ2:%.*]] = getelementptr inbounds nuw i8, ptr %async.ctxt, i64 136
 ; CHECK:   store i64 0, ptr [[FRAMEPTR]]
 ; CHECK:   store i64 1, ptr [[ALLOCA_PRJ2]]
 ; CHECK:   tail call void @some_may_write(ptr nonnull [[FRAMEPTR]])
 ; CHECK:   [[CALLEE_CTXT:%.*]] = tail call ptr @llvm.coro.async.context.alloc(ptr %task, ptr nonnull @my_other_async_function_fp)
-; CHECK:   [[CALLEE_CTXT_SPILL:%.*]] = getelementptr inbounds i8, ptr %async.ctxt, i64 160
+; CHECK:   [[CALLEE_CTXT_SPILL:%.*]] = getelementptr inbounds nuw i8, ptr %async.ctxt, i64 160
 ; CHECK:   store ptr [[CALLEE_CTXT]], ptr [[CALLEE_CTXT_SPILL]]
-; CHECK:   [[TYPED_RETURN_TO_CALLER_ADDR:%.*]] = getelementptr inbounds i8, ptr [[CALLEE_CTXT]], i64 8
+; CHECK:   [[TYPED_RETURN_TO_CALLER_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLEE_CTXT]], i64 8
 ; CHECK:   store ptr @my_async_functionTQ0_, ptr [[TYPED_RETURN_TO_CALLER_ADDR]]
 ; CHECK:   store ptr %async.ctxt, ptr [[CALLEE_CTXT]]
 ; Make sure the spill is underaligned to the max context alignment (16).
@@ -148,16 +148,16 @@ define void @my_async_function_pa(ptr %ctxt, ptr %task, ptr %actor) {
 ; CHECK-SAME: !dbg ![[SP2:[0-9]+]] {
 ; CHECK: entryresume.0:
 ; CHECK:   [[CALLER_CONTEXT:%.*]] = load ptr, ptr %0
-; CHECK:   [[FRAME_PTR:%.*]] = getelementptr inbounds i8, ptr [[CALLER_CONTEXT]], i64 128
+; CHECK:   [[FRAME_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 128
 ; CHECK-O0:   [[VECTOR_SPILL_ADDR:%.*]] = getelementptr inbounds %my_async_function.Frame, ptr {{.*}}, i32 0, i32 1
 ; CHECK-O0:   load <4 x double>, ptr [[VECTOR_SPILL_ADDR]], align 16
-; CHECK:   [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[CALLER_CONTEXT]], i64 160
+; CHECK:   [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 160
 ; CHECK:   [[CALLEE_CTXT_RELOAD:%.*]] = load ptr, ptr [[CALLEE_CTXT_SPILL_ADDR]]
-; CHECK:   [[ACTOR_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[CALLER_CONTEXT]], i64 152
+; CHECK:   [[ACTOR_RELOAD_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 152
 ; CHECK:   [[ACTOR_RELOAD:%.*]] = load ptr, ptr [[ACTOR_RELOAD_ADDR]]
-; CHECK:   [[ADDR1:%.*]] = getelementptr inbounds i8, ptr [[CALLER_CONTEXT]], i64 144
+; CHECK:   [[ADDR1:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 144
 ; CHECK:   [[ASYNC_CTXT_RELOAD:%.*]] = load ptr, ptr [[ADDR1]]
-; CHECK:   [[ALLOCA_PRJ2:%.*]] = getelementptr inbounds i8, ptr [[CALLER_CONTEXT]], i64 136
+; CHECK:   [[ALLOCA_PRJ2:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLER_CONTEXT]], i64 136
 ; CHECK:   tail call void @llvm.coro.async.context.dealloc(ptr nonnull [[CALLEE_CTXT_RELOAD]])
 ; CHECK:   [[VAL1:%.*]] = load i64, ptr [[FRAME_PTR]]
 ; CHECK:   tail call void @some_user(i64 [[VAL1]])
@@ -232,7 +232,7 @@ entry:
 ; CHECK-SAME: #[[FRAMEPOINTER]]
 ; CHECK-SAME: !dbg ![[SP4:[0-9]+]]
 ; CHECK: [[CALLEE_CTXT:%.*]] = load ptr, ptr %2
-; CHECK: [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[CALLEE_CTXT]], i64 152
+; CHECK: [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLEE_CTXT]], i64 152
 ; CHECK: store ptr @my_async_function2.resume.1,
 ; CHECK: [[CALLLE_CTXT_RELOAD:%.*]] = load ptr, ptr [[CALLEE_CTXT_SPILL_ADDR]]
 ; CHECK: tail call swiftcc void @asyncSuspend(ptr [[CALLEE_CTXT_RELOAD]]
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
index 2dabc72a51014..b23c5222a3deb 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca-opaque-ptr.ll
@@ -8,7 +8,7 @@ declare {ptr, ptr, i32} @prototype_f(ptr, i1)
 define {ptr, ptr, i32} @f(ptr %buffer, i32 %n, { i32 } %dummy) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  coro.return:
-; CHECK-NEXT:    [[N_VAL_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[BUFFER:%.*]], i64 8
+; CHECK-NEXT:    [[N_VAL_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[BUFFER:%.*]], i64 8
 ; CHECK-NEXT:    store i32 [[N:%.*]], ptr [[N_VAL_SPILL_ADDR]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call ptr @allocate(i32 [[N]])
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[BUFFER]], align 8
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
index 9b00d7929684f..aeb959e5ce711 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
@@ -7,7 +7,7 @@ declare {ptr, ptr, i32} @prototype_f(ptr, i1)
 define {ptr, ptr, i32} @f(ptr %buffer, i32 %n) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  coro.return:
-; CHECK-NEXT:    [[N_VAL_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[BUFFER:%.*]], i64 8
+; CHECK-NEXT:    [[N_VAL_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[BUFFER:%.*]], i64 8
 ; CHECK-NEXT:    store i32 [[N:%.*]], ptr [[N_VAL_SPILL_ADDR]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call ptr @allocate(i32 [[N]])
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[BUFFER]], align 8
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
index b2da9affcd500..fa10ddefee00e 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
@@ -159,7 +159,7 @@ declare void @print(i32)
 ; CHECK-NEXT:  PostSpill:
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call ptr @allocate(i32 16)
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[BUFFER:%.*]], align 8
-; CHECK-NEXT:    [[VAL_SPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[VAL_SPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; CHECK-NEXT:    store i32 [[VAL:%.*]], ptr [[VAL_SPILL_ADDR]], align 4
 ; CHECK-NEXT:    store ptr [[ARRAY:%.*]], ptr [[TMP0]], align 8
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[ARRAY]], align 4
@@ -180,7 +180,7 @@ declare void @print(i32)
 ; CHECK-NEXT:    store i32 0, ptr [[ARRAY_RELOAD]], align 4
 ; CHECK-NEXT:    br label [[COROEND]]
 ; CHECK:       CoroEnd:
-; CHECK-NEXT:    [[VAL_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+; CHECK-NEXT:    [[VAL_RELOAD_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 ; CHECK-NEXT:    [[VAL_RELOAD:%.*]] = load i32, ptr [[VAL_RELOAD_ADDR]], align 4
 ; CHECK-NEXT:    [[NEW_VAL:%.*]] = add i32 [[VAL_RELOAD]], 123
 ; CHECK-NEXT:    tail call void @deallocate(ptr [[TMP2]])
@@ -198,7 +198,7 @@ declare void @print(i32)
 ; CHECK-NEXT:    store i32 10, ptr [[ARRAY_RELOAD]], align 4
 ; CHECK-NEXT:    br label [[COROEND]]
 ; CHECK:       CoroEnd:
-; CHECK-NEXT:    [[VAL_RELOAD_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+; CHECK-NEXT:    [[VAL_RELOAD_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 ; CHECK-NEXT:    [[VAL_RELOAD:%.*]] = load i32, ptr [[VAL_RELOAD_ADDR]], align 4
 ; CHECK-NEXT:    [[NEW_VAL:%.*]] = add i32 [[VAL_RELOAD]], 123
 ; CHECK-NEXT:    tail call void @deallocate(ptr [[TMP2]])
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
index 4c10cb3805c7c..907d7e588ffe0 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
@@ -38,16 +38,16 @@ define i32 @main() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call ptr @allocate(i32 12)
 ; CHECK-NEXT:    store i32 1, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[N_VAL3_SPILL_ADDR_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT:    store i32 1, ptr [[N_VAL3_SPILL_ADDR_I]], align 4, !noalias !0
-; CHECK-NEXT:    [[INPUT_SPILL_ADDR_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
-; CHECK-NEXT:    store i32 2, ptr [[INPUT_SPILL_ADDR_I]], align 4, !noalias !0
-; CHECK-NEXT:    [[INPUT_RELOAD_ADDR13_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
-; CHECK-NEXT:    [[N_VAL3_RELOAD_ADDR11_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT:    store i32 3, ptr [[N_VAL3_RELOAD_ADDR11_I]], align 4, !noalias !3
-; CHECK-NEXT:    store i32 4, ptr [[INPUT_RELOAD_ADDR13_I]], align 4, !noalias !3
-; CHECK-NEXT:    tail call void @print(i32 7), !noalias !6
-; CHECK-NEXT:    tail call void @deallocate(ptr nonnull [[TMP0]]), !noalias !6
+; CHECK-NEXT:    [[N_VAL3_SPILL_ADDR_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    store i32 1, ptr [[N_VAL3_SPILL_ADDR_I]], align 4, !noalias [[META0:![0-9]+]]
+; CHECK-NEXT:    [[INPUT_SPILL_ADDR_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    store i32 2, ptr [[INPUT_SPILL_ADDR_I]], align 4, !noalias [[META0]]
+; CHECK-NEXT:    [[INPUT_RELOAD_ADDR13_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[N_VAL3_RELOAD_ADDR11_I:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    store i32 3, ptr [[N_VAL3_RELOAD_ADDR11_I]], align 4, !noalias [[META3:![0-9]+]]
+; CHECK-NEXT:    store i32 4, ptr [[INPUT_RELOAD_ADDR13_I]], align 4, !noalias [[META3]]
+; CHECK-NEXT:    tail call void @print(i32 7), !noalias [[META6:![0-9]+]]
+; CHECK-NEXT:    tail call void @deallocate(ptr nonnull [[TMP0]]), !noalias [[META6]]
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
diff --git a/llvm/test/Transforms/Coroutines/coro-swifterror.ll b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
index de28ed5608168..899be4a010326 100644
--- a/llvm/test/Transforms/Coroutines/coro-swifterror.ll
+++ b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
@@ -50,7 +50,7 @@ define ptr @g(ptr %buffer, i32 %n) {
 ; CHECK-NEXT:    store ptr null, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    call void @maybeThrow(ptr nonnull swifterror [[TMP0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds i8, ptr [[BUFFER]], i32 4
+; CHECK-NEXT:    [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[BUFFER]], i32 4
 ; CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTSPILL_ADDR]], align 4
 ; CHECK-NEXT:    call void @logError(ptr [[TMP1]])
 ; CHECK-NEXT:    ret ptr @g.resume.0
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
index 3af4c70a5621c..cca648bb6ea23 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
@@ -630,3 +630,95 @@ define i64 @test_shl_nsw_at_use(i64 noundef %x) {
   %res = select i1 %cmp, i64 %shr, i64 0
   ret i64 %res
 }
+
+define i1 @test_icmp_mod(i64 noundef %x) {
+; CHECK-LABEL: @test_icmp_mod(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[X:%.*]], 86400
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[REM]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i64 86400, i64 0
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[COND]], [[REM]]
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %rem = srem i64 %x, 86400
+  %cmp = icmp slt i64 %rem, 0
+  %cond = select i1 %cmp, i64 86400, i64 0
+  %add = add nsw i64 %cond, %rem
+  %cmp1 = icmp ugt i64 %add, 86399
+  ret i1 %cmp1
+}
+
+define i1 @test_icmp_mod_commuted1(i64 noundef %x) {
+; CHECK-LABEL: @test_icmp_mod_commuted1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[X:%.*]], 86400
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[REM]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i64 86400, i64 0
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[REM]], [[COND]]
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %rem = srem i64 %x, 86400
+  %cmp = icmp slt i64 %rem, 0
+  %cond = select i1 %cmp, i64 86400, i64 0
+  %add = add nsw i64 %rem, %cond
+  %cmp1 = icmp ugt i64 %add, 86399
+  ret i1 %cmp1
+}
+
+define i1 @test_icmp_mod_commuted2(i64 noundef %x) {
+; CHECK-LABEL: @test_icmp_mod_commuted2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[X:%.*]], 86400
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[REM]], -1
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i64 0, i64 86400
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[COND]], [[REM]]
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %rem = srem i64 %x, 86400
+  %cmp = icmp sgt i64 %rem, -1
+  %cond = select i1 %cmp, i64 0, i64 86400
+  %add = add nsw i64 %cond, %rem
+  %cmp1 = icmp ugt i64 %add, 86399
+  ret i1 %cmp1
+}
+
+define i1 @test_icmp_mod_undef(i64 %x) {
+; CHECK-LABEL: @test_icmp_mod_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[X:%.*]], 86400
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[REM]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i64 86400, i64 0
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[COND]], [[REM]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i64 [[ADD]], 86399
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+entry:
+  %rem = srem i64 %x, 86400
+  %cmp = icmp slt i64 %rem, 0
+  %cond = select i1 %cmp, i64 86400, i64 0
+  %add = add nsw i64 %cond, %rem
+  %cmp1 = icmp ugt i64 %add, 86399
+  ret i1 %cmp1
+}
+
+define i1 @test_icmp_mod_wrong_range(i64 noundef %x) {
+; CHECK-LABEL: @test_icmp_mod_wrong_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[X:%.*]], 86400
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[REM]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i64 86401, i64 0
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[COND]], [[REM]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i64 [[ADD]], 86399
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+entry:
+  %rem = srem i64 %x, 86400
+  %cmp = icmp slt i64 %rem, 0
+  %cond = select i1 %cmp, i64 86401, i64 0
+  %add = add nsw i64 %cond, %rem
+  %cmp1 = icmp ugt i64 %add, 86399
+  ret i1 %cmp1
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
index 5c6dead841b5b..baf8f386a8cce 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/vectors.ll
@@ -352,3 +352,14 @@ define <2 x i1> @insertelement_fold2() {
   %icmp1 = icmp slt <2 x i32> %ie1, <i32 1024, i32 1024>
   ret <2 x i1> %icmp1
 }
+
+@g = external global i32
+
+define <2 x i16> @insertelement_constexpr() {
+; CHECK-LABEL: define <2 x i16> @insertelement_constexpr() {
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i16> poison, i16 ptrtoint (ptr @g to i16), i32 0
+; CHECK-NEXT:    ret <2 x i16> [[INS]]
+;
+  %ins = insertelement <2 x i16> poison, i16 ptrtoint (ptr @g to i16), i32 0
+  ret <2 x i16> %ins
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/offsetted-overlapping-stores.ll b/llvm/test/Transforms/DeadStoreElimination/offsetted-overlapping-stores.ll
index bdc023128c852..48f0c58ceb053 100644
--- a/llvm/test/Transforms/DeadStoreElimination/offsetted-overlapping-stores.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/offsetted-overlapping-stores.ll
@@ -47,6 +47,90 @@ bb:
   ret void
 }
 
+define void @ScalableVectorTestFullyOverlapping(ptr %arg, i32 %i) vscale_range(1, 2) {
+; CHECK-LABEL: @ScalableVectorTestFullyOverlapping(
+; CHECK-NEXT:    [[I_1:%.*]] = add nuw nsw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[EXT_I_1:%.*]] = zext i32 [[I_1]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I_1:%.*]] = getelementptr inbounds float, ptr [[ARG:%.*]], i64 [[EXT_I_1]]
+; CHECK-NEXT:    store float 0.000000e+00, ptr [[GEP_ARG_I_1]], align 4
+; CHECK-NEXT:    [[EXT_I:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 [[EXT_I]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i.1 = add nuw nsw i32 %i, 1
+  %ext.i.1 = zext i32 %i.1 to i64
+  %gep.arg.i.1 = getelementptr inbounds float, ptr %arg, i64 %ext.i.1
+  store float 0.0, ptr %gep.arg.i.1
+  %ext.i = zext i32 %i to i64
+  %gep.arg.i = getelementptr inbounds float, ptr %arg, i64 %ext.i
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i
+  ret void
+}
+
+define void @ScalableVectorTestFullyOverlapping2(ptr %arg, i32 %i) {
+; CHECK-LABEL: @ScalableVectorTestFullyOverlapping2(
+; CHECK-NEXT:    [[I_1:%.*]] = add nuw nsw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[EXT_I_1:%.*]] = zext i32 [[I_1]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I_1:%.*]] = getelementptr inbounds float, ptr [[ARG:%.*]], i64 [[EXT_I_1]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I_1]], align 8
+; CHECK-NEXT:    [[EXT_I:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 [[EXT_I]]
+; CHECK-NEXT:    store <vscale x 4 x float> zeroinitializer, ptr [[GEP_ARG_I]], align 16
+; CHECK-NEXT:    ret void
+;
+  %i.1 = add nuw nsw i32 %i, 1
+  %ext.i.1 = zext i32 %i.1 to i64
+  %gep.arg.i.1 = getelementptr inbounds float, ptr %arg, i64 %ext.i.1
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i.1
+  %ext.i = zext i32 %i to i64
+  %gep.arg.i = getelementptr inbounds float, ptr %arg, i64 %ext.i
+  store <vscale x 4 x float> zeroinitializer, ptr %gep.arg.i
+  ret void
+}
+
+define void @ScalableVectorTestNonOverlapping(ptr %arg, i32 %i) vscale_range(1, 2) {
+; CHECK-LABEL: @ScalableVectorTestNonOverlapping(
+; CHECK-NEXT:    [[I_10:%.*]] = add nuw nsw i32 [[I:%.*]], 10
+; CHECK-NEXT:    [[EXT_I_10:%.*]] = zext i32 [[I_10]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I_10:%.*]] = getelementptr inbounds float, ptr [[ARG:%.*]], i64 [[EXT_I_10]]
+; CHECK-NEXT:    store float 0.000000e+00, ptr [[GEP_ARG_I_10]], align 4
+; CHECK-NEXT:    [[EXT_I:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 [[EXT_I]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i.10 = add nuw nsw i32 %i, 10
+  %ext.i.10 = zext i32 %i.10 to i64
+  %gep.arg.i.10 = getelementptr inbounds float, ptr %arg, i64 %ext.i.10
+  store float 0.0, ptr %gep.arg.i.10
+  %ext.i = zext i32 %i to i64
+  %gep.arg.i = getelementptr inbounds float, ptr %arg, i64 %ext.i
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i
+  ret void
+}
+
+define void @ScalableVectorTestNonOverlapping2(ptr %arg, i32 %i) vscale_range(1, 2) {
+; CHECK-LABEL: @ScalableVectorTestNonOverlapping2(
+; CHECK-NEXT:    [[I_10:%.*]] = add nuw nsw i32 [[I:%.*]], 10
+; CHECK-NEXT:    [[EXT_I_10:%.*]] = zext i32 [[I_10]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I_10:%.*]] = getelementptr inbounds float, ptr [[ARG:%.*]], i64 [[EXT_I_10]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I_10]], align 8
+; CHECK-NEXT:    [[EXT_I:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 [[EXT_I]]
+; CHECK-NEXT:    store <vscale x 4 x float> zeroinitializer, ptr [[GEP_ARG_I]], align 16
+; CHECK-NEXT:    ret void
+;
+  %i.10 = add nuw nsw i32 %i, 10
+  %ext.i.10 = zext i32 %i.10 to i64
+  %gep.arg.i.10 = getelementptr inbounds float, ptr %arg, i64 %ext.i.10
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i.10
+  %ext.i = zext i32 %i to i64
+  %gep.arg.i = getelementptr inbounds float, ptr %arg, i64 %ext.i
+  store <vscale x 4 x float> zeroinitializer, ptr %gep.arg.i
+  ret void
+}
+
 define void @ArrayTestPartiallyOverlapping(i64 %0) {
 ;
 ; The DSE pass will not kill the store because the overlap is partial
@@ -55,9 +139,9 @@ define void @ArrayTestPartiallyOverlapping(i64 %0) {
 ; CHECK-LABEL: @ArrayTestPartiallyOverlapping(
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP0:%.*]], 10
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [0 x i8], ptr @BUFFER, i64 0, i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP0]], 15
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [0 x i8], ptr @BUFFER, i64 0, i64 [[TMP5]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 15
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [0 x i8], ptr @BUFFER, i64 0, i64 [[TMP4]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    store i64 0, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -97,3 +181,28 @@ bb:
   ret void
 }
 
+define void @ScalableVectorTestPartiallyOverlapping(ptr %arg, i32 %i) {
+;
+; The DSE pass will not kill the store because the overlap is partial
+; and won't fully clobber the original store.
+;
+; CHECK-LABEL: @ScalableVectorTestPartiallyOverlapping(
+; CHECK-NEXT:    [[EXT_I:%.*]] = zext i32 [[I:%.*]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I:%.*]] = getelementptr inbounds float, ptr [[ARG:%.*]], i64 [[EXT_I]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I]], align 8
+; CHECK-NEXT:    [[I_1:%.*]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[EXT_I_1:%.*]] = zext i32 [[I_1]] to i64
+; CHECK-NEXT:    [[GEP_ARG_I_1:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 [[EXT_I_1]]
+; CHECK-NEXT:    store <vscale x 2 x float> zeroinitializer, ptr [[GEP_ARG_I_1]], align 8
+; CHECK-NEXT:    ret void
+;
+  %ext.i = zext i32 %i to i64
+  %gep.arg.i = getelementptr inbounds float, ptr %arg, i64 %ext.i
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i
+  %i.1 = add nuw nsw i32 %i, 1
+  %ext.i.1 = zext i32 %i.1 to i64
+  %gep.arg.i.1 = getelementptr inbounds float, ptr %arg, i64 %ext.i.1
+  store <vscale x 2 x float> zeroinitializer, ptr %gep.arg.i.1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll b/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
index c9a0943de8cd9..7193bb501c89d 100644
--- a/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/stores-of-existing-values.ll
@@ -641,7 +641,7 @@ define void @later_non_removable_store(i1 %c, ptr %p) {
 ; CHECK:       if:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    call void @use(ptr [[P]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    call void @use(ptr [[P]]) #[[ATTR7:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   store i8 1, ptr %p
@@ -655,3 +655,94 @@ exit:
   call void @use(ptr %p) argmemonly
   ret void
 }
+
+define void @scalable_scalable_redundant_store(ptr %ptr) {
+; CHECK-LABEL: @scalable_scalable_redundant_store(
+; CHECK-NEXT:    [[GEP_PTR_2:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 2
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[GEP_PTR_2]], align 16
+; CHECK-NEXT:    store <vscale x 4 x i64> zeroinitializer, ptr [[PTR]], align 32
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.2 = getelementptr i64, ptr %ptr, i64 2
+  store <vscale x 2 x i64> zeroinitializer, ptr %gep.ptr.2
+  store <vscale x 4 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @scalable_scalable_neg(ptr %ptr) {
+; CHECK-LABEL: @scalable_scalable_neg(
+; CHECK-NEXT:    [[GEP_PTR_8:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 8
+; CHECK-NEXT:    store <vscale x 4 x i64> zeroinitializer, ptr [[GEP_PTR_8]], align 32
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[PTR]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.8 = getelementptr i64, ptr %ptr, i64 8
+  store <vscale x 4 x i64> zeroinitializer, ptr %gep.ptr.8
+  store <vscale x 2 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @scalable_scalable_nonconst_offset_neg(ptr %ptr, i64 %i) {
+; CHECK-LABEL: @scalable_scalable_nonconst_offset_neg(
+; CHECK-NEXT:    [[GEP_PTR_I:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[GEP_PTR_I]], align 16
+; CHECK-NEXT:    store <vscale x 4 x i64> zeroinitializer, ptr [[PTR]], align 32
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.i = getelementptr i64, ptr %ptr, i64 %i
+  store <vscale x 2 x i64> zeroinitializer, ptr %gep.ptr.i
+  store <vscale x 4 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @scalable_fixed_redundant_store(ptr %ptr) vscale_range(1, 2) {
+; CHECK-LABEL: @scalable_fixed_redundant_store(
+; CHECK-NEXT:    [[GEP_PTR_2:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 2
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[GEP_PTR_2]], align 16
+; CHECK-NEXT:    store <vscale x 4 x i64> zeroinitializer, ptr [[PTR]], align 32
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.2 = getelementptr i64, ptr %ptr, i64 2
+  store <2 x i64> zeroinitializer, ptr %gep.ptr.2
+  store <vscale x 4 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @scalable_fixed_neg(ptr %ptr) vscale_range(1, 2) {
+; CHECK-LABEL: @scalable_fixed_neg(
+; CHECK-NEXT:    [[GEP_PTR_16:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 16
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr [[GEP_PTR_16]], align 16
+; CHECK-NEXT:    store <vscale x 4 x i64> zeroinitializer, ptr [[PTR]], align 32
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.16 = getelementptr i64, ptr %ptr, i64 16
+  store <2 x i64> zeroinitializer, ptr %gep.ptr.16
+  store <vscale x 4 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @fixed_scalable_redundant_store(ptr %ptr) vscale_range(1, 2) {
+; CHECK-LABEL: @fixed_scalable_redundant_store(
+; CHECK-NEXT:    [[GEP_PTR_2:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 2
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[GEP_PTR_2]], align 16
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr [[PTR]], align 64
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.2 = getelementptr i64, ptr %ptr, i64 2
+  store <vscale x 2 x i64> zeroinitializer, ptr %gep.ptr.2
+  store <8 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
+
+define void @fixed_scalable_neg(ptr %ptr) vscale_range(1, 2) {
+; CHECK-LABEL: @fixed_scalable_neg(
+; CHECK-NEXT:    [[GEP_PTR_2:%.*]] = getelementptr i64, ptr [[PTR:%.*]], i64 2
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[GEP_PTR_2]], align 16
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr [[PTR]], align 32
+; CHECK-NEXT:    ret void
+;
+  %gep.ptr.2 = getelementptr i64, ptr %ptr, i64 2
+  store <vscale x 2 x i64> zeroinitializer, ptr %gep.ptr.2
+  store <4 x i64> zeroinitializer, ptr %ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
index 0c24169d02c2c..0e0854c8807c2 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization-constant-expression.ll
@@ -30,13 +30,13 @@ define internal i64 @zoo(i1 %flag) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[FLAG:%.*]], label [[PLUS:%.*]], label [[MINUS:%.*]]
 ; CHECK:       plus:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @func2.specialized.2(ptr getelementptr inbounds (i8, ptr @Global, i64 8))
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @func2.specialized.2(ptr getelementptr inbounds nuw (i8, ptr @Global, i64 8))
 ; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       minus:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @func2.specialized.1(ptr getelementptr inbounds (i8, ptr @Global, i64 16))
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @func2.specialized.1(ptr getelementptr inbounds nuw (i8, ptr @Global, i64 16))
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ ptrtoint (ptr getelementptr inbounds (i8, ptr @Global, i64 8) to i64), [[PLUS]] ], [ ptrtoint (ptr getelementptr inbounds (i8, ptr @Global, i64 16) to i64), [[MINUS]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @Global, i64 8) to i64), [[PLUS]] ], [ ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @Global, i64 16) to i64), [[MINUS]] ]
 ; CHECK-NEXT:    ret i64 [[TMP2]]
 ;
 entry:
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
index 6a05d5b17dde8..7028edb4732bd 100644
--- a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
@@ -8,7 +8,7 @@ target triple = "i386-apple-darwin11.0.0"
 define void @Bubble() nounwind noinline {
 ; CHECK-LABEL: @Bubble(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP7_PRE:%.*]] = load i32, ptr getelementptr inbounds (i8, ptr @sortlist, i32 4), align 4
+; CHECK-NEXT:    [[TMP7_PRE:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @sortlist, i32 4), align 4
 ; CHECK-NEXT:    br label [[WHILE_BODY5:%.*]]
 ; CHECK:       while.body5:
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP7_PRE]], [[ENTRY:%.*]] ], [ [[TMP71:%.*]], [[IF_END:%.*]] ]
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
index 39790f4cddbb7..a38d3e50a6121 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
@@ -63,8 +63,8 @@ define void @test2(i64 %i) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i64 (...) @goo()
 ; CHECK-NEXT:    store i64 [[CALL]], ptr @g2, align 8
-; CHECK-NEXT:    [[T2_PRE:%.*]] = load i64, ptr getelementptr inbounds (i8, ptr @a, i64 24), align 8
-; CHECK-NEXT:    [[T3_PRE:%.*]] = load i64, ptr getelementptr inbounds (i8, ptr @b, i64 24), align 8
+; CHECK-NEXT:    [[T2_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
+; CHECK-NEXT:    [[T3_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @b, i64 24), align 8
 ; CHECK-NEXT:    [[DOTPRE:%.*]] = mul nsw i64 [[T3_PRE]], [[T2_PRE]]
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
@@ -272,7 +272,7 @@ define void @test6(ptr %ptr, i1 %arg) {
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[TMP0]], [[IF_THEN]] ], [ [[TMP1]], [[WHILE]] ]
-; CHECK-NEXT:    br i1 %arg, label [[WHILE_END:%.*]], label [[WHILE]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[WHILE_END:%.*]], label [[WHILE]]
 ; CHECK:       while.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/GVN/opaque-ptr.ll b/llvm/test/Transforms/GVN/opaque-ptr.ll
index 4a0f9d319501c..8a7e420584859 100644
--- a/llvm/test/Transforms/GVN/opaque-ptr.ll
+++ b/llvm/test/Transforms/GVN/opaque-ptr.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=gvn < %s | FileCheck %s
+; RUN: opt -S -passes=gvn -use-constant-int-for-fixed-length-splat < %s | FileCheck %s
 
 declare void @use(ptr)
 declare void @use.i32(i32)
@@ -58,6 +59,10 @@ define void @gep_cse_offset_canonicalization(ptr %p, i64 %idx, i64 %idx2) {
 ; CHECK-NEXT:    call void @use(ptr [[GEP6]])
 ; CHECK-NEXT:    call void @use(ptr [[GEP6_SAME]])
 ; CHECK-NEXT:    call void @use(ptr [[GEP6_DIFFERENT]])
+; CHECK-NEXT:    %gep7 = getelementptr <16 x i32>, ptr %p, i64 1
+; CHECK-NEXT:    %gep7.different = getelementptr <16 x i32>, ptr %p, <16 x i64> splat (i64 1)
+; CHECK-NEXT:    call void @use(ptr %gep7)
+; CHECK-NEXT:    call void @use(<16 x ptr> %gep7.different)
 ; CHECK-NEXT:    ret void
 ;
   %gep1 = getelementptr i64, ptr %p, i64 1
@@ -101,6 +106,10 @@ define void @gep_cse_offset_canonicalization(ptr %p, i64 %idx, i64 %idx2) {
   call void @use(ptr %gep6)
   call void @use(ptr %gep6.same)
   call void @use(ptr %gep6.different)
+  %gep7 = getelementptr <16 x i32>, ptr %p, i64 1
+  %gep7.different = getelementptr <16 x i32>, ptr %p, <16 x i64> splat (i64 1)
+  call void @use(ptr %gep7)
+  call void @use(<16 x ptr> %gep7.different)
   ret void
 }
 
diff --git a/llvm/test/Transforms/GVN/setjmp.ll b/llvm/test/Transforms/GVN/setjmp.ll
new file mode 100644
index 0000000000000..07b7028346760
--- /dev/null
+++ b/llvm/test/Transforms/GVN/setjmp.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=gvn < %s | FileCheck %s
+
+declare i32 @setjmp() returns_twice
+declare void @longjmp()
+declare ptr @malloc(i64)
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:    [[MALLOC:%.*]] = call noalias ptr @malloc(i64 4)
+; CHECK-NEXT:    store i32 10, ptr [[MALLOC]], align 4
+; CHECK-NEXT:    [[SJ:%.*]] = call i32 @setjmp()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SJ]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    store i32 20, ptr [[MALLOC]], align 4
+; CHECK-NEXT:    call void @longjmp()
+; CHECK-NEXT:    unreachable
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[MALLOC]], align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %malloc = call noalias ptr @malloc(i64 4)
+  store i32 10, ptr %malloc, align 4
+  %sj = call i32 @setjmp()
+  %cmp = icmp eq i32 %sj, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 20, ptr %malloc
+  call void @longjmp()
+  unreachable
+
+if.end:
+  %res = load i32, ptr %malloc
+  ret i32 %res
+}
+
+; We are still allowed to optimize non-volatile accesses to allocas.
+define i32 @test_alloca() {
+; CHECK-LABEL: define i32 @test_alloca() {
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i43, align 8
+; CHECK-NEXT:    store i32 10, ptr [[ALLOC]], align 4
+; CHECK-NEXT:    [[SJ:%.*]] = call i32 @setjmp()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SJ]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    store i32 20, ptr [[ALLOC]], align 4
+; CHECK-NEXT:    call void @longjmp()
+; CHECK-NEXT:    unreachable
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    ret i32 10
+;
+  %alloc = alloca i43
+  store i32 10, ptr %alloc, align 4
+  %sj = call i32 @setjmp()
+  %cmp = icmp eq i32 %sj, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 20, ptr %alloc
+  call void @longjmp()
+  unreachable
+
+if.end:
+  %res = load i32, ptr %alloc
+  ret i32 %res
+}
+
+define i32 @test_alloca_volatile() {
+; CHECK-LABEL: define i32 @test_alloca_volatile() {
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i43, align 8
+; CHECK-NEXT:    store volatile i32 10, ptr [[ALLOC]], align 4
+; CHECK-NEXT:    [[SJ:%.*]] = call i32 @setjmp()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SJ]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    store volatile i32 20, ptr [[ALLOC]], align 4
+; CHECK-NEXT:    call void @longjmp()
+; CHECK-NEXT:    unreachable
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[RES:%.*]] = load volatile i32, ptr [[ALLOC]], align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %alloc = alloca i43
+  store volatile i32 10, ptr %alloc, align 4
+  %sj = call i32 @setjmp()
+  %cmp = icmp eq i32 %sj, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store volatile i32 20, ptr %alloc
+  call void @longjmp()
+  unreachable
+
+if.end:
+  %res = load volatile i32, ptr %alloc
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-addrspace.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-addrspace.ll
new file mode 100644
index 0000000000000..bd0957150748f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-addrspace.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
+; RUN: opt -S -passes=globalopt -o - < %s | FileCheck %s
+
+@g = internal addrspace(200) global ptr null, align 8
+
+;.
+; CHECK: @g.init = internal unnamed_addr addrspace(200) global i1 false
+;.
+define internal i32 @f1() {
+; CHECK-LABEL: define {{[^@]+}}@f1() unnamed_addr {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[G_INIT_VAL:%.*]] = load i1, ptr addrspace(200) @g.init, align 1
+; CHECK-NEXT:    call fastcc void @f2()
+; CHECK-NEXT:    [[NOTINIT:%.*]] = xor i1 [[G_INIT_VAL]], true
+; CHECK-NEXT:    br i1 [[NOTINIT]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[I5:%.*]] = phi i32 [ -1, [[BB2]] ], [ 1, [[BB3]] ]
+; CHECK-NEXT:    ret i32 [[I5]]
+;
+bb:
+  %i = load ptr addrspace(200), ptr addrspace(200) @g, align 8
+  call void @f2()
+  %i1 = icmp eq ptr addrspace(200) %i, null
+  br i1 %i1, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb
+  br label %bb4
+
+bb3:                                              ; preds = %bb
+  br label %bb4
+
+bb4:                                              ; preds = %bb3, %bb2
+  %i5 = phi i32 [ -1, %bb2 ], [ 1, %bb3 ]
+  ret i32 %i5
+}
+
+define internal void @f2() {
+; CHECK-LABEL: define {{[^@]+}}@f2() unnamed_addr {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store i1 true, ptr addrspace(200) @g.init, align 1
+; CHECK-NEXT:    ret void
+;
+bb:
+  %i = call noalias ptr @malloc(i64 4)
+  store ptr %i, ptr addrspace(200) @g, align 8
+  ret void
+}
+
+define dso_local i32 @main() {
+; CHECK-LABEL: define {{[^@]+}}@main() local_unnamed_addr {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store i1 false, ptr addrspace(200) @g.init, align 1
+; CHECK-NEXT:    [[I:%.*]] = call fastcc i32 @f1()
+; CHECK-NEXT:    ret i32 [[I]]
+;
+bb:
+  store ptr null, ptr addrspace(200) @g, align 8
+  %i = call i32 @f1()
+  ret i32 %i
+}
+
+; Function Attrs: allockind("alloc,uninitialized") allocsize(0)
+declare dso_local noalias ptr @malloc(i64) #0
+
+attributes #0 = { allockind("alloc,uninitialized") allocsize(0) }
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
index a3c4002626a70..3f75d0c9880f8 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-exit-no-dl.ll
@@ -14,7 +14,7 @@ define void @foo() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @global, i64 1), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @global, i64 1), align 1
 ; CHECK-NEXT:    br i1 false, label [[BB7:%.*]], label [[BB11:%.*]]
 ; CHECK:       bb7:
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP6]] to i64
diff --git a/llvm/test/Transforms/IndVarSimplify/floating-point-small-iv.ll b/llvm/test/Transforms/IndVarSimplify/floating-point-small-iv.ll
index bebd314f7375a..90d84eb5a1add 100644
--- a/llvm/test/Transforms/IndVarSimplify/floating-point-small-iv.ll
+++ b/llvm/test/Transforms/IndVarSimplify/floating-point-small-iv.ll
@@ -357,7 +357,7 @@ define void @uitofp_fptoui_range_with_negative() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    store i32 100, ptr getelementptr inbounds (i8, ptr @array, i64 400), align 4
+; CHECK-NEXT:    store i32 100, ptr getelementptr inbounds nuw (i8, ptr @array, i64 400), align 4
 ; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[CLEANUP:%.*]]
 ; CHECK:       cleanup:
 ; CHECK-NEXT:    ret void
@@ -418,7 +418,7 @@ define void @uitofp_fptosi_range_with_negative () {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    store i32 100, ptr getelementptr inbounds (i8, ptr @array, i64 400), align 4
+; CHECK-NEXT:    store i32 100, ptr getelementptr inbounds nuw (i8, ptr @array, i64 400), align 4
 ; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[CLEANUP:%.*]]
 ; CHECK:       cleanup:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IndVarSimplify/pr117133.ll b/llvm/test/Transforms/IndVarSimplify/pr117133.ll
new file mode 100644
index 0000000000000..f62dc3f04a096
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr117133.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=indvars < %s | FileCheck %s
+
+define i32 @widget() {
+; CHECK-LABEL: define i32 @widget() {
+; CHECK-NEXT:  [[B:.*:]]
+; CHECK-NEXT:    br label %[[B1:.*]]
+; CHECK:       [[B1]]:
+; CHECK-NEXT:    br i1 true, label %[[B3:.*]], label %[[B8:.*]]
+; CHECK:       [[B3]]:
+; CHECK-NEXT:    br i1 true, label %[[B7:.*]], label %[[B5:.*]]
+; CHECK:       [[B5]]:
+; CHECK-NEXT:    br label %[[B1]]
+; CHECK:       [[B7]]:
+; CHECK-NEXT:    ret i32 5
+; CHECK:       [[B8]]:
+; CHECK-NEXT:    ret i32 7
+;
+b:
+  br label %b1
+
+b1:
+  %phi = phi i32 [ 0, %b ], [ %udiv6, %b5 ]
+  %phi2 = phi i32 [ 1, %b ], [ %add, %b5 ]
+  %icmp = icmp eq i32 %phi, 0
+  br i1 %icmp, label %b3, label %b8
+
+b3:
+  %udiv = udiv i32 10, %phi2
+  %urem = urem i32 %udiv, 10
+  %icmp4 = icmp eq i32 %urem, 0
+  br i1 %icmp4, label %b7, label %b5
+
+b5:
+  %udiv6 = udiv i32 %phi2, 0
+  %add = add i32 %phi2, 1
+  br label %b1
+
+b7:
+  ret i32 5
+
+b8:
+  ret i32 7
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/memcpy-constant-size.ll b/llvm/test/Transforms/Inline/AArch64/memcpy-constant-size.ll
new file mode 100644
index 0000000000000..17f7024ff8905
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/memcpy-constant-size.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt %s -mtriple=arm64-apple-macosx -passes=inline -inline-threshold=2 -inline-call-penalty=5 -S | FileCheck %s
+
+declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
+declare ptr @__memcpy_chk(ptr, ptr, i64, i64)
+declare ptr @__memmove_chk(ptr, ptr, i64, i64)
+declare ptr @__mempcpy_chk(ptr, ptr, i64, i64)
+declare ptr @__memset_chk(ptr, i32, i64, i64)
+
+define void @callee(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @callee
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) {
+; CHECK-NEXT:    [[OBJSIZE:%.*]] = call i64 @llvm.objectsize.i64.p0(ptr [[DST]], i1 false, i1 true, i1 false)
+; CHECK-NEXT:    [[CALL_MEMCPY:%.*]] = call ptr @__memcpy_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 [[OBJSIZE]])
+; CHECK-NEXT:    [[CALL_MEMMOVE:%.*]] = call ptr @__memmove_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 [[OBJSIZE]])
+; CHECK-NEXT:    [[CALL_MEMPCPY:%.*]] = call ptr @__mempcpy_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 [[OBJSIZE]])
+; CHECK-NEXT:    [[CALL_MEMSET:%.*]] = call ptr @__memset_chk(ptr [[DST]], i32 0, i64 [[SIZE]], i64 [[OBJSIZE]])
+; CHECK-NEXT:    ret void
+;
+  %objsize = call i64 @llvm.objectsize.i64.p0(ptr %dst, i1 false, i1 true, i1 false)
+  %call.memcpy = call ptr @__memcpy_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  %call.memmove = call ptr @__memmove_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  %call.mempcpy = call ptr @__mempcpy_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  %call.memset = call ptr @__memset_chk(ptr %dst, i32 0, i64 %size, i64 %objsize)
+  ret void
+}
+
+define void @caller(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @caller
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[OBJSIZE_I:%.*]] = call i64 @llvm.objectsize.i64.p0(ptr [[DST]], i1 false, i1 true, i1 false)
+; CHECK-NEXT:    [[CALL_MEMCPY_I:%.*]] = call ptr @__memcpy_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMMOVE_I:%.*]] = call ptr @__memmove_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMPCPY_I:%.*]] = call ptr @__mempcpy_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMSET_I:%.*]] = call ptr @__memset_chk(ptr [[DST]], i32 0, i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    ret void
+;
+  call void @callee(ptr %dst, ptr %src, i64 4)
+  ret void
+}
+
+define void @objsize_toosmall_callee(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @objsize_toosmall_callee
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) {
+; CHECK-NEXT:    [[CALL_MEMCPY:%.*]] = call ptr @__memcpy_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 1)
+; CHECK-NEXT:    [[CALL_MEMMOVE:%.*]] = call ptr @__memmove_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 1)
+; CHECK-NEXT:    [[CALL_MEMPCPY:%.*]] = call ptr @__mempcpy_chk(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i64 1)
+; CHECK-NEXT:    [[CALL_MEMSET:%.*]] = call ptr @__memset_chk(ptr [[DST]], i32 0, i64 [[SIZE]], i64 1)
+; CHECK-NEXT:    ret void
+;
+  %call.memcpy = call ptr @__memcpy_chk(ptr %dst, ptr %src, i64 %size, i64 1)
+  %call.memmove = call ptr @__memmove_chk(ptr %dst, ptr %src, i64 %size, i64 1)
+  %call.mempcpy = call ptr @__mempcpy_chk(ptr %dst, ptr %src, i64 %size, i64 1)
+  %call.memset = call ptr @__memset_chk(ptr %dst, i32 0, i64 %size, i64 1)
+  ret void
+}
+
+define void @objsize_toosmall_caller(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @objsize_toosmall_caller
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    call void @objsize_toosmall_callee(ptr [[DST]], ptr [[SRC]], i64 4)
+; CHECK-NEXT:    ret void
+;
+  call void @objsize_toosmall_callee(ptr %dst, ptr %src, i64 4)
+  ret void
+}
+
+define void @intrinsics_callee(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @intrinsics_callee
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[DST]], i8 0, i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+  call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+  call void @llvm.memset.p0.i64(ptr %dst, i8 0, i64 %size, i1 false)
+  ret void
+}
+
+define void @intrinsics_caller(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @intrinsics_caller
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[DST]], i8 0, i64 4, i1 false)
+; CHECK-NEXT:    ret void
+;
+  call void @intrinsics_callee(ptr %dst, ptr %src, i64 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/promote-call-bitwidth.ll b/llvm/test/Transforms/Inline/promote-call-bitwidth.ll
new file mode 100644
index 0000000000000..6a0ddb5601253
--- /dev/null
+++ b/llvm/test/Transforms/Inline/promote-call-bitwidth.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=inline < %s | FileCheck %s
+
+;; Check that we correctly use the index size when accumulating offsets during CallPromotion
+
+target datalayout = "p200:128:128:128:64-A200-P200-G200"
+
+define void @test(ptr addrspace(200) %arg1, ptr addrspace(200) %arg2) local_unnamed_addr addrspace(200) {
+; CHECK-LABEL: define {{[^@]+}}@test
+; CHECK-SAME: (ptr addrspace(200) [[ARG1:%.*]], ptr addrspace(200) [[ARG2:%.*]]) local_unnamed_addr addrspace(200) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr addrspace(200), ptr addrspace(200) [[ARG2]], align 16
+; CHECK-NEXT:    call addrspace(200) void [[TMP0]](ptr addrspace(200) [[ARG1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @call_fnptr(ptr addrspace(200) %arg1, ptr addrspace(200) %arg2)
+  ret void
+}
+
+define internal void @call_fnptr(ptr addrspace(200) %this, ptr addrspace(200) %arg) unnamed_addr addrspace(200) align 2 {
+entry:
+  %0 = load ptr addrspace(200), ptr addrspace(200) %arg, align 16
+  call void %0(ptr addrspace(200) %this)
+  ret void
+}
+
+define void @test2(ptr addrspace(200) %this) local_unnamed_addr addrspace(200) {
+; CHECK-LABEL: define {{[^@]+}}@test2
+; CHECK-SAME: (ptr addrspace(200) [[THIS:%.*]]) local_unnamed_addr addrspace(200) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VTABLE_I:%.*]] = load ptr addrspace(200), ptr addrspace(200) [[THIS]], align 16
+; CHECK-NEXT:    [[FN_I:%.*]] = load ptr addrspace(200), ptr addrspace(200) [[VTABLE_I]], align 16
+; CHECK-NEXT:    call addrspace(200) void [[FN_I]](ptr addrspace(200) [[THIS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @call_via_vtable(ptr addrspace(200) %this)
+  ret void
+}
+
+define internal void @call_via_vtable(ptr addrspace(200) %this) unnamed_addr addrspace(200) {
+entry:
+  %vtable = load ptr addrspace(200), ptr addrspace(200) %this, align 16
+  %fn = load ptr addrspace(200), ptr addrspace(200) %vtable, align 16
+  call void %fn(ptr addrspace(200) %this)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll b/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
index 27005f38f29f1..b8d570637963d 100644
--- a/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
+++ b/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
@@ -10,7 +10,7 @@ define i32 @main() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[U:%.*]] = alloca [[STRUCT__1ANON:%.*]], align 8
 ; CHECK-NEXT:    store double 0x7FF0000000000000, ptr [[U]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[TMP6]], 2146435072
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP0]], 2146435072
diff --git a/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll b/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
index c31f6d4b4a1fe..870b71e812826 100644
--- a/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
+++ b/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
@@ -13,7 +13,7 @@ define i32 @bar(i64 %key_token2) nounwind {
 ; CHECK-NEXT:    [[IOSPEC:%.*]] = alloca [[STRUCT_KEY:%.*]], align 8
 ; CHECK-NEXT:    [[RET:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 0, ptr [[IOSPEC]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[IOSPEC]], i32 4
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[IOSPEC]], i32 4
 ; CHECK-NEXT:    store i32 0, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    store i64 [[KEY_TOKEN2:%.*]], ptr [[IOSPEC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (...) @foo(ptr nonnull byval([[STRUCT_KEY]]) align 4 [[IOSPEC]], ptr nonnull [[RET]]) #[[ATTR0:[0-9]+]]
diff --git a/llvm/test/Transforms/InstCombine/AArch64/dmb-intrinsics.ll b/llvm/test/Transforms/InstCombine/AArch64/dmb-intrinsics.ll
index dacdd41301365..a93091a73a065 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/dmb-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/dmb-intrinsics.ll
@@ -35,7 +35,7 @@ define void @simple_nonmatching() #0 {
 define ptr @simple_safe_instruction(ptr %p) #0 {
 ; CHECK-LABEL: define ptr @simple_safe_instruction(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[RES:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; CHECK-NEXT:    [[RES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 ; CHECK-NEXT:    call void @llvm.aarch64.dmb(i32 10)
 ; CHECK-NEXT:    ret ptr [[RES]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/lane-index-simplify-demanded-bits.ll b/llvm/test/Transforms/InstCombine/AMDGPU/lane-index-simplify-demanded-bits.ll
new file mode 100644
index 0000000000000..327d68bdf550e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/lane-index-simplify-demanded-bits.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=instcombine < %s | FileCheck -check-prefixes=CHECK,WAVE64 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mattr=+wavefrontsize32 -passes=instcombine < %s | FileCheck -check-prefixes=CHECK,WAVE32 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64 -passes=instcombine < %s | FileCheck -check-prefixes=CHECK,WAVE64 %s
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.readlane
+; --------------------------------------------------------------------
+
+define i32 @readlane_31(i32 %arg) #0 {
+; CHECK-LABEL: define i32 @readlane_31(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 31)
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 31)
+  ret i32 %res
+}
+
+define i32 @readlane_32(i32 %arg) #0 {
+; WAVE64-LABEL: define i32 @readlane_32(
+; WAVE64-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 32)
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @readlane_32(
+; WAVE32-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 0)
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 32)
+  ret i32 %res
+}
+
+define i32 @readlane_33(i32 %arg) #0 {
+; WAVE64-LABEL: define i32 @readlane_33(
+; WAVE64-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 33)
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @readlane_33(
+; WAVE32-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 1)
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 33)
+  ret i32 %res
+}
+
+define i32 @readlane_63(i32 %arg) #0 {
+; WAVE64-LABEL: define i32 @readlane_63(
+; WAVE64-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 63)
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @readlane_63(
+; WAVE32-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 31)
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 63)
+  ret i32 %res
+}
+
+define i32 @readlane_64(i32 %arg) #0 {
+; CHECK-LABEL: define i32 @readlane_64(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 0)
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 64)
+  ret i32 %res
+}
+
+define i32 @readlane_and_31(i32 %arg, i32 %idx) #0 {
+; WAVE64-LABEL: define i32 @readlane_and_31(
+; WAVE64-SAME: i32 [[ARG:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[IDX_CLAMP:%.*]] = and i32 [[IDX]], 31
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[IDX_CLAMP]])
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @readlane_and_31(
+; WAVE32-SAME: i32 [[ARG:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[IDX]])
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %idx.clamp = and i32 %idx, 31
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 %idx.clamp)
+  ret i32 %res
+}
+
+define i32 @readlane_and_63(i32 %arg, i32 %idx) #0 {
+; CHECK-LABEL: define i32 @readlane_and_63(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[IDX]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %idx.clamp = and i32 %idx, 63
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 %idx.clamp)
+  ret i32 %res
+}
+
+define i32 @readlane_poison(i32 %arg) #0 {
+; CHECK-LABEL: define i32 @readlane_poison(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 poison)
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.readlane.i32(i32 %arg, i32 poison)
+  ret i32 %res
+}
+
+define float @readlane_f32_63(float %arg) #0 {
+; WAVE64-LABEL: define float @readlane_f32_63(
+; WAVE64-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[ARG]], i32 63)
+; WAVE64-NEXT:    ret float [[RES]]
+;
+; WAVE32-LABEL: define float @readlane_f32_63(
+; WAVE32-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[ARG]], i32 31)
+; WAVE32-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.amdgcn.readlane.f32(float %arg, i32 63)
+  ret float %res
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.writelane
+; --------------------------------------------------------------------
+
+define i32 @writelane_31(i32 %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define i32 @writelane_31(
+; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 31, i32 [[ARG1]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 31, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_32(i32 %arg0, i32 %arg1) #0 {
+; WAVE64-LABEL: define i32 @writelane_32(
+; WAVE64-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 32, i32 [[ARG1]])
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @writelane_32(
+; WAVE32-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 0, i32 [[ARG1]])
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 32, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_33(i32 %arg0, i32 %arg1) #0 {
+; WAVE64-LABEL: define i32 @writelane_33(
+; WAVE64-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 33, i32 [[ARG1]])
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @writelane_33(
+; WAVE32-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 1, i32 [[ARG1]])
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 33, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_63(i32 %arg0, i32 %arg1) #0 {
+; WAVE64-LABEL: define i32 @writelane_63(
+; WAVE64-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 63, i32 [[ARG1]])
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @writelane_63(
+; WAVE32-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 31, i32 [[ARG1]])
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 63, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_64(i32 %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define i32 @writelane_64(
+; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 0, i32 [[ARG1]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 64, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_and_31(i32 %arg0, i32 %arg1, i32 %idx) #0 {
+; WAVE64-LABEL: define i32 @writelane_and_31(
+; WAVE64-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[IDX_CLAMP:%.*]] = and i32 [[IDX]], 31
+; WAVE64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 [[IDX_CLAMP]], i32 [[ARG1]])
+; WAVE64-NEXT:    ret i32 [[RES]]
+;
+; WAVE32-LABEL: define i32 @writelane_and_31(
+; WAVE32-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 [[IDX]], i32 [[ARG1]])
+; WAVE32-NEXT:    ret i32 [[RES]]
+;
+  %idx.clamp = and i32 %idx, 31
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 %idx.clamp, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_and_63(i32 %arg0, i32 %arg1, i32 %idx) #0 {
+; CHECK-LABEL: define i32 @writelane_and_63(
+; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 [[IDX]], i32 [[ARG1]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %idx.clamp = and i32 %idx, 63
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 %idx.clamp, i32 %arg1)
+  ret i32 %res
+}
+
+define i32 @writelane_poison(i32 %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define i32 @writelane_poison(
+; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[ARG0]], i32 poison, i32 [[ARG1]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.amdgcn.writelane.i32(i32 %arg0, i32 poison, i32 %arg1)
+  ret i32 %res
+}
+
+define float @writelane_f32_63(float %arg0, float %arg1) #0 {
+; WAVE64-LABEL: define float @writelane_f32_63(
+; WAVE64-SAME: float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE64-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.writelane.f32(float [[ARG0]], i32 63, float [[ARG1]])
+; WAVE64-NEXT:    ret float [[RES]]
+;
+; WAVE32-LABEL: define float @writelane_f32_63(
+; WAVE32-SAME: float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; WAVE32-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.writelane.f32(float [[ARG0]], i32 31, float [[ARG1]])
+; WAVE32-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.amdgcn.writelane.f32(float %arg0, i32 63, float %arg1)
+  ret float %res
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
index c14d61b51ad77..0e5b4dedd4a0d 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
@@ -53,7 +53,7 @@ define i64 @memcpy_constant_arg_ptr_to_alloca_load_atomic(ptr addrspace(4) noali
 ; CHECK-LABEL: @memcpy_constant_arg_ptr_to_alloca_load_atomic(
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i64], align 8, addrspace(5)
 ; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 8 dereferenceable(256) [[ALLOCA]], ptr addrspace(4) noundef align 8 dereferenceable(256) [[ARG:%.*]], i64 256, i1 false)
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i64], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [32 x i64], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load atomic i64, ptr addrspace(5) [[GEP]] syncscope("somescope") acquire, align 8
 ; CHECK-NEXT:    ret i64 [[LOAD]]
 ;
@@ -101,7 +101,7 @@ define amdgpu_kernel void @memcpy_constant_byref_arg_ptr_to_alloca_too_many_byte
 ; CHECK-LABEL: @memcpy_constant_byref_arg_ptr_to_alloca_too_many_bytes(
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i8], align 4, addrspace(5)
 ; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 4 dereferenceable(31) [[ALLOCA]], ptr addrspace(4) noundef align 4 dereferenceable(31) [[ARG:%.*]], i64 31, i1 false)
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [32 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(5) [[GEP]], align 1
 ; CHECK-NEXT:    store i8 [[LOAD]], ptr addrspace(1) [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -120,7 +120,7 @@ define amdgpu_kernel void @memcpy_constant_intrinsic_ptr_to_alloca(ptr addrspace
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i8], align 4, addrspace(5)
 ; CHECK-NEXT:    [[KERNARG_SEGMENT_PTR:%.*]] = call align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
 ; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 4 dereferenceable(32) [[ALLOCA]], ptr addrspace(4) noundef align 16 dereferenceable(32) [[KERNARG_SEGMENT_PTR]], i64 32, i1 false)
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [32 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[IDX:%.*]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(5) [[GEP]], align 1
 ; CHECK-NEXT:    store i8 [[LOAD]], ptr addrspace(1) [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
index f03b633c581d6..9c2476578974a 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
@@ -146,7 +146,7 @@ define void @PR46277(float %0, float %1, float %2, float %3, <4 x float> %4, ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP8]], <4 x float> [[TMP4:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i64 0
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP5:%.*]], i64 4
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5:%.*]], i64 4
 ; CHECK-NEXT:    store float [[TMP10]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i64 1
 ; CHECK-NEXT:    store float [[TMP12]], ptr [[TMP11]], align 4
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
index 9b0387e51eead..8bc9bb2dadb62 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
@@ -146,7 +146,7 @@ define void @PR46277(float %0, float %1, float %2, float %3, <4 x float> %4, ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP8]], <4 x float> [[TMP4:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i64 0
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP5:%.*]], i64 4
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5:%.*]], i64 4
 ; CHECK-NEXT:    store float [[TMP10]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i64 1
 ; CHECK-NEXT:    store float [[TMP12]], ptr [[TMP11]], align 4
diff --git a/llvm/test/Transforms/InstCombine/addrspacecast.ll b/llvm/test/Transforms/InstCombine/addrspacecast.ll
index 35a1066a6b31c..00df545064069 100644
--- a/llvm/test/Transforms/InstCombine/addrspacecast.ll
+++ b/llvm/test/Transforms/InstCombine/addrspacecast.ll
@@ -141,7 +141,7 @@ define i32 @memcpy_addrspacecast() nounwind {
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[LOOP_BODY]] ]
 ; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_INC:%.*]], [[LOOP_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[I]] to i16
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr i8, ptr addrspace(2) getelementptr inbounds (i8, ptr addrspace(2) @const_array, i16 4), i16 [[TMP0]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr i8, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @const_array, i16 4), i16 [[TMP0]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(2) [[PTR]], align 1
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[LOAD]] to i32
 ; CHECK-NEXT:    [[SUM_INC]] = add i32 [[SUM]], [[EXT]]
diff --git a/llvm/test/Transforms/InstCombine/align-addr.ll b/llvm/test/Transforms/InstCombine/align-addr.ll
index 58647dc9595d8..6ef4d85fe4e41 100644
--- a/llvm/test/Transforms/InstCombine/align-addr.ll
+++ b/llvm/test/Transforms/InstCombine/align-addr.ll
@@ -81,7 +81,7 @@ define <16 x i8> @test1_as1(<2 x i64> %x) {
 
 define <16 x i8> @test1_as1_gep(<2 x i64> %x) {
 ; CHECK-LABEL: @test1_as1_gep(
-; CHECK-NEXT:    [[TMP:%.*]] = load <16 x i8>, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @GLOBAL_as1_gep, i32 16), align 1
+; CHECK-NEXT:    [[TMP:%.*]] = load <16 x i8>, ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @GLOBAL_as1_gep, i32 16), align 1
 ; CHECK-NEXT:    ret <16 x i8> [[TMP]]
 ;
   %tmp = load <16 x i8>, ptr addrspace(1) getelementptr ([8 x i32], ptr addrspace(1) @GLOBAL_as1_gep, i16 0, i16 4), align 1
diff --git a/llvm/test/Transforms/InstCombine/and-sub-combine.ll b/llvm/test/Transforms/InstCombine/and-sub-combine.ll
new file mode 100644
index 0000000000000..8b2f91b50beef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-sub-combine.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i8 @and_sub(i8 %a) {
+; CHECK-LABEL: @and_sub(
+; CHECK-NEXT:    [[RET:%.*]] = and i8 [[A:%.*]], 12
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %and1 = and i8 %a, 15
+  %and2 = and i8 %a, 3
+
+  %ret = sub i8 %and1, %and2
+  ret i8 %ret
+}
+
+declare void @use(i8)
+define i8 @and_sub_multi_use(i8 %a) {
+; CHECK-LABEL: @and_sub_multi_use(
+; CHECK-NEXT:    [[AND1:%.*]] = and i8 [[A:%.*]], 15
+; CHECK-NEXT:    call void @use(i8 [[AND1]])
+; CHECK-NEXT:    [[AND2:%.*]] = and i8 [[A]], 3
+; CHECK-NEXT:    call void @use(i8 [[AND2]])
+; CHECK-NEXT:    [[RET:%.*]] = and i8 [[A]], 12
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %and1 = and i8 %a, 15
+  call void @use(i8 %and1)
+  %and2 = and i8 %a, 3
+  call void @use(i8 %and2)
+  %ret = sub i8 %and1, %and2
+  ret i8 %ret
+}
+
+define <2 x i32> @and_sub_vec(<2 x i32> %a) {
+; CHECK-LABEL: @and_sub_vec(
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[A:%.*]], <i32 3, i32 8>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and1 = and <2 x i32> %a, <i32 11, i32 10>
+  %and2 = and <2 x i32> %a, <i32 8, i32 2>
+
+  %ret = sub <2 x i32> %and1, %and2
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @and_sub_vec_posion(<2 x i32> %a) {
+; CHECK-LABEL: @and_sub_vec_posion(
+; CHECK-NEXT:    ret <2 x i32> poison
+;
+  %and1 = and <2 x i32> %a, <i32 11, i32 poison>
+  %and2 = and <2 x i32> %a, <i32 poison, i32 2>
+
+  %ret = sub <2 x i32> %and1, %and2
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @and_sub_vec_undef(<2 x i32> %a) {
+; CHECK-LABEL: @and_sub_vec_undef(
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[A:%.*]], <i32 11, i32 undef>
+; CHECK-NEXT:    [[AND2:%.*]] = and <2 x i32> [[A]], <i32 undef, i32 2>
+; CHECK-NEXT:    [[RET:%.*]] = sub <2 x i32> [[AND1]], [[AND2]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and1 = and <2 x i32> %a, <i32 11, i32 undef>
+  %and2 = and <2 x i32> %a, <i32 undef, i32 2>
+
+  %ret = sub <2 x i32> %and1, %and2
+  ret <2 x i32> %ret
+}
+
+define i8 @and_sub_negtive(i8 %a) {
+; CHECK-LABEL: @and_sub_negtive(
+; CHECK-NEXT:    [[AND1:%.*]] = and i8 [[A:%.*]], 10
+; CHECK-NEXT:    [[AND2:%.*]] = and i8 [[A]], 9
+; CHECK-NEXT:    [[RET:%.*]] = sub nsw i8 [[AND1]], [[AND2]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %and1 = and i8 %a, 10
+  %and2 = and i8 %a, 9
+
+  %ret = sub i8 %and1, %and2
+  ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/array.ll b/llvm/test/Transforms/InstCombine/array.ll
index 4f4ae17bebc50..3edb47dda62cc 100644
--- a/llvm/test/Transforms/InstCombine/array.ll
+++ b/llvm/test/Transforms/InstCombine/array.ll
@@ -97,7 +97,7 @@ define void @test_zext_missing_nneg(ptr %ptr, i32 %a, i32 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[A]], 10
 ; CHECK-NEXT:    [[IDX:%.*]] = zext i32 [[ADD]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IDX]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[PTR]], i64 [[IDX]]
 ; CHECK-NEXT:    store i32 [[B]], ptr [[GEP]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -116,8 +116,8 @@ define ptr @gep_inbounds_add_nsw_nonneg(ptr %ptr, i64 %a, i64 %b) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[A_NNEG]])
 ; CHECK-NEXT:    [[B_NNEG:%.*]] = icmp sgt i64 [[B]], -1
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[B_NNEG]])
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[A]]
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[PTR]], i64 [[A]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 [[B]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
   %a.nneg = icmp sgt i64 %a, -1
@@ -207,8 +207,8 @@ define ptr @gep_inbounds_sext_add_nonneg(ptr %ptr, i32 %a) {
 ; CHECK-NEXT:    [[A_NNEG:%.*]] = icmp sgt i32 [[A]], -1
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[A_NNEG]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[A]] to i64
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[TMP1]]
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 40
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PTR]], i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 40
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
   %a.nneg = icmp sgt i32 %a, -1
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index 2b8ca5d25fd1a..47659ff8c8490 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -6,7 +6,7 @@ declare void @llvm.assume(i1 noundef)
 define void @f1(ptr %a) {
 ; CHECK-LABEL: @f1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 4
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[PTR]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
@@ -50,7 +50,7 @@ define void @f2(ptr %a) {
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A:%.*]], i64 32, i32 24) ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 0
diff --git a/llvm/test/Transforms/InstCombine/assume-loop-align.ll b/llvm/test/Transforms/InstCombine/assume-loop-align.ll
index e7eb18c61b6bb..24fd343d1448e 100644
--- a/llvm/test/Transforms/InstCombine/assume-loop-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-loop-align.ll
@@ -21,10 +21,10 @@ define void @foo(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], 1
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
diff --git a/llvm/test/Transforms/InstCombine/assume-redundant.ll b/llvm/test/Transforms/InstCombine/assume-redundant.ll
index 63f8859371f3a..32d803e35a1c3 100644
--- a/llvm/test/Transforms/InstCombine/assume-redundant.ll
+++ b/llvm/test/Transforms/InstCombine/assume-redundant.ll
@@ -1,7 +1,11 @@
-; RUN: opt -passes='require<domtree>,instcombine,require<domtree>' -S < %s | FileCheck %s
+; RUN: opt -passes='require<domtree>,instcombine<no-verify-fixpoint>,require<domtree>' -S < %s | FileCheck %s
 ; Note: The -loops above can be anything that requires the domtree, and is
 ; necessary to work around a pass-manager bug.
 
+; In _Z3fooR1s we can only infer nuw on the gep after the IV has been
+; simplified, which we can't do in one iteration. (Note that nowadays
+; LoopUnroll would pre-simplify the IV and avoid the issue.)
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll
index a728c294628ca..52f0adf02a396 100644
--- a/llvm/test/Transforms/InstCombine/assume.ll
+++ b/llvm/test/Transforms/InstCombine/assume.ll
@@ -774,7 +774,7 @@ exit:
 
 define void @canonicalize_assume(ptr %0) {
 ; DEFAULT-LABEL: @canonicalize_assume(
-; DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 8
+; DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 8
 ; DEFAULT-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[TMP2]], i64 16) ]
 ; DEFAULT-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll b/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll
index a3cd0328e9b59..41367d55d8894 100644
--- a/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll
+++ b/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll
@@ -231,7 +231,7 @@ define <2 x i8> @vectorized_add(<2 x i1> %c, <2 x i8> %arg) {
 define i64 @pr64669(i64 %a) {
 ; CHECK-LABEL: define i64 @pr64669
 ; CHECK-SAME: (i64 [[A:%.*]]) {
-; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @b, i64 100), @c
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @b, i64 100), @c
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[A]], 1
 ; CHECK-NEXT:    [[ADD:%.*]] = select i1 [[CMP_NOT]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    ret i64 [[ADD]]
diff --git a/llvm/test/Transforms/InstCombine/call-cast-target.ll b/llvm/test/Transforms/InstCombine/call-cast-target.ll
index e5bb929ce92f0..2cedc6c81d735 100644
--- a/llvm/test/Transforms/InstCombine/call-cast-target.ll
+++ b/llvm/test/Transforms/InstCombine/call-cast-target.ll
@@ -114,7 +114,7 @@ define i1 @test5() {
 ; CHECK-LABEL: define i1 @test5() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = alloca { i32, i32 }, align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @fn5(i32 [[TMP2]], i32 [[TMP4]])
 ; CHECK-NEXT:    ret i1 [[TMP5]]
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll b/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
new file mode 100644
index 0000000000000..68049ca230191
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
@@ -0,0 +1,401 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i8 @add_and_sgt(i8 %x) {
+; CHECK-LABEL: define i8 @add_and_sgt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 8)
+; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 16
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %x, 8
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+define i8 @add_sgt_nuw(i8 %x) {
+; CHECK-LABEL: define i8 @add_sgt_nuw(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 8)
+; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 16
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nuw i8 %x, 16
+  %cmp = icmp sgt i8 %x, 8
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+define i8 @sub_and_ugt(i8 %x) {
+; CHECK-LABEL: define i8 @sub_and_ugt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = add nsw i8 [[TMP1]], -50
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %sub = sub nsw i8 %x, 50
+  %cmp = icmp ugt i8 %x, 100
+  %s = select i1 %cmp, i8 50, i8 %sub
+  ret i8 %s
+}
+
+define i8 @sub_ugt_nuw_nsw(i8 %x) {
+; CHECK-LABEL: define i8 @sub_ugt_nuw_nsw(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = add nsw i8 [[TMP1]], -50
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %sub = sub nuw nsw i8 %x, 50
+  %cmp = icmp ugt i8 %x, 100
+  %s = select i1 %cmp, i8 50, i8 %sub
+  ret i8 %s
+}
+
+define i8 @mul_and_ult(i8 %x) {
+; CHECK-LABEL: define i8 @mul_and_ult(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 10)
+; CHECK-NEXT:    [[S:%.*]] = mul nuw i8 [[TMP1]], 10
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = mul nsw i8 %x, 10
+  %cmp = icmp ult i8 10, %x
+  %s = select i1 %cmp, i8 100, i8 %add
+  ret i8 %s
+}
+
+define i8 @mul_and_non_strict_predicate(i8 %x) {
+; CHECK-LABEL: define i8 @mul_and_non_strict_predicate(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 10)
+; CHECK-NEXT:    [[S:%.*]] = mul i8 [[TMP1]], 10
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = mul nsw i8 %x, 10
+  %cmp = icmp sle i8 10, %x
+  %s = select i1 %cmp, i8 100, i8 %add
+  ret i8 %s
+}
+
+define i8 @mul_ult_noflags(i8 %x) {
+; CHECK-LABEL: define i8 @mul_ult_noflags(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 10)
+; CHECK-NEXT:    [[S:%.*]] = mul nuw i8 [[TMP1]], 10
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = mul i8 %x, 10
+  %cmp = icmp ult i8 10, %x
+  %s = select i1 %cmp, i8 100, i8 %add
+  ret i8 %s
+}
+
+define i8 @udiv_and_slt(i8 %x) {
+; CHECK-LABEL: define i8 @udiv_and_slt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = udiv i8 [[X]], 10
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X]], 100
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 10, i8 [[SUB]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %sub = udiv i8 %x, 10
+  %cmp = icmp slt i8 %x, 100
+  %s = select i1 %cmp, i8 10, i8 %sub
+  ret i8 %s
+}
+
+define i8 @udiv_slt_exact(i8 %x) {
+; CHECK-LABEL: define i8 @udiv_slt_exact(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = udiv exact i8 [[X]], 10
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X]], 100
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 10, i8 [[SUB]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %sub = udiv exact i8 %x, 10
+  %cmp = icmp slt i8 %x, 100
+  %s = select i1 %cmp, i8 10, i8 %sub
+  ret i8 %s
+}
+
+define i8 @canonicalize_icmp_operands(i8 %x) {
+; CHECK-LABEL: define i8 @canonicalize_icmp_operands(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 119)
+; CHECK-NEXT:    [[S:%.*]] = add nsw i8 [[TMP1]], 8
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 8
+  %cmp = icmp sle i8 120, %x
+  %s = select i1 %cmp, i8 127, i8 %add
+  ret i8 %s
+}
+
+define i8 @flipped_strictness_predicate_and_constant(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @flipped_strictness_predicate_and_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 1)
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[TMP1]], -2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = add i8 %x, -2
+  %c = icmp ult i8 %x, 2
+  %r = select i1 %c, i8 -1, i8 %a
+  ret i8 %r
+}
+
+declare void @use(i1)
+declare void @use_byte(i8)
+
+define i8 @multi_use_cond_and_sel(i8 %x) {
+; CHECK-LABEL: define i8 @multi_use_cond_and_sel(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 8
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 8)
+; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 16
+; CHECK-NEXT:    call void @use_byte(i8 [[S]])
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %x, 8
+  call void @use(i1 %cmp)
+  %s = select i1 %cmp, i8 %add, i8 24
+  call void @use_byte(i8 %s)
+  ret i8 %s
+}
+
+define void @rust_noop_loop() {
+; CHECK-LABEL: define void @rust_noop_loop() {
+; CHECK-NEXT:  [[START:.*]]:
+; CHECK-NEXT:    br label %[[BB2_I:.*]]
+; CHECK:       [[BB2_I]]:
+; CHECK-NEXT:    [[ITER_SROA_0_07:%.*]] = phi i32 [ 0, %[[START]] ], [ [[SPEC_SELECT5:%.*]], %[[BB2_I]] ]
+; CHECK-NEXT:    [[_0_I3_I:%.*]] = icmp sgt i32 [[ITER_SROA_0_07]], 99
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.smin.i32(i32 [[ITER_SROA_0_07]], i32 99)
+; CHECK-NEXT:    [[SPEC_SELECT5]] = add nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[_0_I3_I]], label %[[BASICBLOCK4:.*]], label %[[BB2_I]]
+; CHECK:       [[BASICBLOCK4]]:
+; CHECK-NEXT:    ret void
+;
+start:
+  br label %bb2.i
+
+bb2.i:
+  %iter.sroa.0.07 = phi i32 [ 0, %start ], [ %spec.select5, %bb2.i ]
+  %_0.i3.i = icmp sgt i32 %iter.sroa.0.07, 99
+  %0 = add nsw i32 %iter.sroa.0.07, 1
+  %spec.select5 = select i1 %_0.i3.i, i32 100, i32 %0
+  %_0.i.not.i = icmp sgt i32 %spec.select5, 100
+  %or.cond = select i1 %_0.i3.i, i1 true, i1 %_0.i.not.i
+  br i1 %or.cond, label %basicblock4, label %bb2.i
+
+basicblock4:
+  ret void
+}
+
+define <2 x i8> @add_non_splat_vector(<2 x i8> %x) {
+; CHECK-LABEL: define <2 x i8> @add_non_splat_vector(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> <i8 0, i8 1>)
+; CHECK-NEXT:    [[S:%.*]] = add nuw <2 x i8> [[TMP1]], <i8 1, i8 0>
+; CHECK-NEXT:    ret <2 x i8> [[S]]
+;
+  %add = add <2 x i8> %x, <i8 1, i8 0>
+  %cmp = icmp sgt <2 x i8> %x, <i8 0, i8 1>
+  %s = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> <i8 1, i8 1>
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @or_splat_vector(<2 x i8> %x) {
+; CHECK-LABEL: define <2 x i8> @or_splat_vector(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> splat (i8 1))
+; CHECK-NEXT:    [[S:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
+; CHECK-NEXT:    ret <2 x i8> [[S]]
+;
+  %add = or <2 x i8> %x, <i8 1, i8 1>
+  %cmp = icmp sgt <2 x i8> %x, <i8 0, i8 0>
+  %s = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> <i8 1, i8 1>
+  ret <2 x i8> %s
+}
+
+define i8 @const_operands_dont_fold_negative(i8 %x) {
+; CHECK-LABEL: define i8 @const_operands_dont_fold_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 18
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 25
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %x, 18
+  %s = select i1 %cmp, i8 %add, i8 25
+  ret i8 %s
+}
+
+define i8 @add_with_poison_negative(i8 %x) {
+; CHECK-LABEL: define i8 @add_with_poison_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    ret i8 25
+;
+  %add = add nsw i8 %x, poison
+  %cmp = icmp sgt i8 %x, 8
+  %s = select i1 %cmp, i8 %add, i8 25
+  ret i8 %s
+}
+
+define i8 @add_with_overflow_negative(i8 %x) {
+; CHECK-LABEL: define i8 @add_with_overflow_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 100
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 119
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 -127, i8 [[ADD]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add i8 %x, 100
+  %cmp = icmp sle i8 120, %x
+  %s = select i1 %cmp, i8 -127, i8 %add
+  ret i8 %s
+}
+
+define <2 x i8> @vector_with_poison_negative(<2 x i8> %x) {
+; CHECK-LABEL: define <2 x i8> @vector_with_poison_negative(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = xor <2 x i8> [[X]], <i8 1, i8 poison>
+; CHECK-NEXT:    [[CMP_INV:%.*]] = icmp slt <2 x i8> [[X]], splat (i8 1)
+; CHECK-NEXT:    [[S:%.*]] = select <2 x i1> [[CMP_INV]], <2 x i8> splat (i8 1), <2 x i8> [[ADD]]
+; CHECK-NEXT:    ret <2 x i8> [[S]]
+;
+  %add = xor <2 x i8> %x, <i8 1, i8 poison>
+  %cmp = icmp sgt <2 x i8> %x, <i8 0, i8 0>
+  %s = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> <i8 1, i8 1>
+  ret <2 x i8> %s
+}
+
+define i8 @multi_use_bop_negative(i8 %x) {
+; CHECK-LABEL: define i8 @multi_use_bop_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], 16
+; CHECK-NEXT:    call void @use_byte(i8 [[ADD]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 7
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 24, i8 [[ADD]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  call void @use_byte(i8 %add)
+  %cmp = icmp sle i8 8, %x
+  %s = select i1 %cmp, i8 24, i8 %add
+  ret i8 %s
+}
+
+define half @float_negative(half %x) {
+; CHECK-LABEL: define half @float_negative(
+; CHECK-SAME: half [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = fmul fast half [[X]], 0xH2E66
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt half [[X]], 0xH5640
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], half 0xH4900, half [[ADD]]
+; CHECK-NEXT:    ret half [[S]]
+;
+  %add = fdiv fast half %x, 10.0
+  %cmp = fcmp ult half 100.0, %x
+  %s = select i1 %cmp, half 10.0, half %add
+  ret half %s
+}
+
+define i8 @poison_false_val_negative(i8 %x) {
+; CHECK-LABEL: define i8 @poison_false_val_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], 16
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %x, 8
+  %s = select i1 %cmp, i8 %add, i8 poison
+  ret i8 %s
+}
+
+define i8 @eq_negative(i8 %x) {
+; CHECK-LABEL: define i8 @eq_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    ret i8 24
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp eq i8 %x, 8
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+define i8 @different_operands_negative(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @different_operands_negative(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[Y]], 8
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 24
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %y, 8
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+define i8 @non_const_bop_operand_negative(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @non_const_bop_operand_negative(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[Y]], 8
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 24
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, %y
+  %cmp = icmp sgt i8 %y, 8
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+define i8 @non_const_cmp_operand_negative(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @non_const_cmp_operand_negative(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 24
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nsw i8 %x, 16
+  %cmp = icmp sgt i8 %x, %y
+  %s = select i1 %cmp, i8 %add, i8 24
+  ret i8 %s
+}
+
+declare i8 @result()
+
+define i8 @non_binop_negative(i8 %x) {
+; CHECK-LABEL: define i8 @non_binop_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[RESULT:%.*]] = call i8 @result()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 16
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[RESULT]], i8 24
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %result = call i8 @result()
+  %cmp = icmp sgt i8 %x, 16
+  %s = select i1 %cmp, i8 %result, i8 24
+  ret i8 %s
+}
+
+define i8 @sub_const_on_lhs_negative(i8 %x) {
+; CHECK-LABEL: define i8 @sub_const_on_lhs_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 50, [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X]], 100
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i8 [[SUB]], i8 50
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %sub = sub nsw i8 50, %x
+  %cmp = icmp ugt i8 %x, 100
+  %s = select i1 %cmp, i8 %sub, i8 50
+  ret i8 %s
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-gep-constglob.ll b/llvm/test/Transforms/InstCombine/canonicalize-gep-constglob.ll
index 53585dfde48b2..1520d6ce59548 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-gep-constglob.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-gep-constglob.ll
@@ -8,7 +8,7 @@ define ptr @x12(i64 %x) {
 ; CHECK-SAME: i64 [[X:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GEP_IDX:%.*]] = mul nsw i64 [[X]], 400
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr getelementptr inbounds (i8, ptr @glob, i64 84), i64 [[GEP_IDX]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr getelementptr inbounds nuw (i8, ptr @glob, i64 84), i64 [[GEP_IDX]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
 entry:
@@ -22,7 +22,7 @@ define ptr @x1y(i64 %x, i64 %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GEP_IDX:%.*]] = mul nsw i64 [[X]], 400
 ; CHECK-NEXT:    [[GEP_IDX1:%.*]] = shl nsw i64 [[Y]], 2
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr getelementptr inbounds (i8, ptr @glob, i64 116), i64 [[GEP_IDX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr getelementptr inbounds nuw (i8, ptr @glob, i64 116), i64 [[GEP_IDX]]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[GEP_IDX1]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
@@ -35,7 +35,7 @@ define ptr @xzy(i64 %x, i64 %y, i64 %z) {
 ; CHECK-LABEL: define ptr @xzy(
 ; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [10 x [10 x [10 x i32]]], ptr getelementptr inbounds (i8, ptr @glob, i64 40), i64 0, i64 [[X]], i64 [[Z]], i64 [[Y]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [10 x [10 x [10 x i32]]], ptr getelementptr inbounds nuw (i8, ptr @glob, i64 40), i64 0, i64 [[X]], i64 [[Z]], i64 [[Y]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
 entry:
@@ -47,7 +47,7 @@ define ptr @zerox(i64 %x) {
 ; CHECK-LABEL: define ptr @zerox(
 ; CHECK-SAME: i64 [[X:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [10 x i32], ptr getelementptr inbounds (i8, ptr @glob, i64 32), i64 0, i64 [[X]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [10 x i32], ptr getelementptr inbounds nuw (i8, ptr @glob, i64 32), i64 0, i64 [[X]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
 entry:
@@ -60,9 +60,9 @@ define i32 @twoloads(i64 %x) {
 ; CHECK-SAME: i64 [[X:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GEP1_IDX:%.*]] = mul nsw i64 [[X]], 400
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr getelementptr inbounds (i8, ptr @glob, i64 134), i64 [[GEP1_IDX]]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr getelementptr inbounds nuw (i8, ptr @glob, i64 134), i64 [[GEP1_IDX]]
 ; CHECK-NEXT:    [[GEP2_IDX:%.*]] = mul nsw i64 [[X]], 400
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr getelementptr inbounds (i8, ptr @glob, i64 132), i64 [[GEP2_IDX]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr getelementptr inbounds nuw (i8, ptr @glob, i64 132), i64 [[GEP2_IDX]]
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[GEP2]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = add i32 [[A]], [[B]]
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
index ca748a9483e9b..0f957e22ad17b 100644
--- a/llvm/test/Transforms/InstCombine/cast.ll
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -2,6 +2,8 @@
 ; Tests to make sure elimination of casts is working correctly
 ; RUN: opt < %s -passes=instcombine -S -data-layout="E-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" | FileCheck %s --check-prefixes=ALL,BE
 ; RUN: opt < %s -passes=instcombine -S -data-layout="e-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" | FileCheck %s --check-prefixes=ALL,LE
+; RUN: opt < %s -passes=instcombine -S -data-layout="E-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" -use-constant-fp-for-fixed-length-splat -use-constant-int-for-fixed-length-splat | FileCheck %s --check-prefixes=ALL,BE
+; RUN: opt < %s -passes=instcombine -S -data-layout="e-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64" -use-constant-fp-for-fixed-length-splat -use-constant-int-for-fixed-length-splat | FileCheck %s --check-prefixes=ALL,LE
 
 declare void @use_i32(i32)
 declare void @use_v2i32(<2 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/cast_phi.ll b/llvm/test/Transforms/InstCombine/cast_phi.ll
index 80b07139faf4b..f289e1459000a 100644
--- a/llvm/test/Transforms/InstCombine/cast_phi.ll
+++ b/llvm/test/Transforms/InstCombine/cast_phi.ll
@@ -9,16 +9,16 @@ define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
 ; CHECK-NEXT:    [[CALLB:%.*]] = alloca [258 x float], align 4
 ; CHECK-NEXT:    [[CONV_I:%.*]] = uitofp i32 [[INUMSTEPS:%.*]] to float
 ; CHECK-NEXT:    [[CONV_I12:%.*]] = zext i32 [[TID:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLA]], i64 0, i64 [[CONV_I12]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [258 x float], ptr [[CALLA]], i64 0, i64 [[CONV_I12]]
 ; CHECK-NEXT:    store float [[CONV_I]], ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLB]], i64 0, i64 [[CONV_I12]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw [258 x float], ptr [[CALLB]], i64 0, i64 [[CONV_I12]]
 ; CHECK-NEXT:    store float [[CONV_I]], ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp eq i32 [[TID]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label [[DOTBB1:%.*]], label [[DOTBB2:%.*]]
 ; CHECK:       .bb1:
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i8, ptr [[CALLA]], i64 1024
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLA]], i64 1024
 ; CHECK-NEXT:    store float [[CONV_I]], ptr [[ARRAYIDX10]], align 4
-; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, ptr [[CALLB]], i64 1024
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds nuw i8, ptr [[CALLB]], i64 1024
 ; CHECK-NEXT:    store float 0.000000e+00, ptr [[ARRAYIDX11]], align 4
 ; CHECK-NEXT:    br label [[DOTBB2]]
 ; CHECK:       .bb2:
@@ -31,8 +31,8 @@ define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[I12_06]], [[BASE:%.*]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[I12_06]], 1
 ; CHECK-NEXT:    [[CONV_I9:%.*]] = sext i32 [[ADD]] to i64
-; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLA]], i64 0, i64 [[CONV_I9]]
-; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLB]], i64 0, i64 [[CONV_I9]]
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds nuw [258 x float], ptr [[CALLA]], i64 0, i64 [[CONV_I9]]
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds nuw [258 x float], ptr [[CALLB]], i64 0, i64 [[CONV_I9]]
 ; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[I12_06]], [[BASE]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[DOTBB4:%.*]], label [[DOTBB5:%.*]]
 ; CHECK:       .bb4:
diff --git a/llvm/test/Transforms/InstCombine/cast_ptr.ll b/llvm/test/Transforms/InstCombine/cast_ptr.ll
index db576b9679b14..40ba21b24435b 100644
--- a/llvm/test/Transforms/InstCombine/cast_ptr.ll
+++ b/llvm/test/Transforms/InstCombine/cast_ptr.ll
@@ -407,7 +407,7 @@ define i32 @ptr_add_in_int_extra_use1(i32 %x) {
 ; CHECK-LABEL: @ptr_add_in_int_extra_use1(
 ; CHECK-NEXT:    [[PTR:%.*]] = inttoptr i32 [[X:%.*]] to ptr
 ; CHECK-NEXT:    call void @use_ptr(ptr [[PTR]])
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 4096
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i32 4096
 ; CHECK-NEXT:    [[R:%.*]] = ptrtoint ptr [[P2]] to i32
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -421,7 +421,7 @@ define i32 @ptr_add_in_int_extra_use1(i32 %x) {
 define i32 @ptr_add_in_int_extra_use2(i32 %x) {
 ; CHECK-LABEL: @ptr_add_in_int_extra_use2(
 ; CHECK-NEXT:    [[PTR:%.*]] = inttoptr i32 [[X:%.*]] to ptr
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 4096
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i32 4096
 ; CHECK-NEXT:    call void @use_ptr(ptr nonnull [[P2]])
 ; CHECK-NEXT:    [[R:%.*]] = ptrtoint ptr [[P2]] to i32
 ; CHECK-NEXT:    ret i32 [[R]]
diff --git a/llvm/test/Transforms/InstCombine/catchswitch-phi.ll b/llvm/test/Transforms/InstCombine/catchswitch-phi.ll
index cb87ee67a4518..720f5258a6346 100644
--- a/llvm/test/Transforms/InstCombine/catchswitch-phi.ll
+++ b/llvm/test/Transforms/InstCombine/catchswitch-phi.ll
@@ -22,11 +22,11 @@ define void @test0(i1 %c1) personality ptr @__gxx_wasm_personality_v0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca [[STRUCT_BLAM:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i32 4
 ; CHECK-NEXT:    invoke void @foo()
 ; CHECK-NEXT:            to label [[BB3:%.*]] unwind label [[BB4:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i32 4
 ; CHECK-NEXT:    invoke void @foo()
 ; CHECK-NEXT:            to label [[BB3]] unwind label [[BB4]]
 ; CHECK:       bb3:
@@ -120,10 +120,10 @@ define void @test1() personality ptr @__gxx_wasm_personality_v0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = catchswitch within none [label %catch.start1] unwind to caller
 ; CHECK:       catch.start1:
 ; CHECK-NEXT:    [[TMP3:%.*]] = catchpad within [[TMP2]] [ptr null]
-; CHECK-NEXT:    [[TOBOOL1_NOT:%.*]] = trunc i8 [[AP_2]] to i1
-; CHECK-NEXT:    br i1 [[TOBOOL1_NOT]], label [[IF_END1:%.*]], label [[IF_THEN1:%.*]]
+; CHECK-NEXT:    [[TOBOOL1:%.*]] = trunc i8 [[AP_2]] to i1
+; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[IF_THEN1:%.*]], label [[IF_END1:%.*]]
 ; CHECK:       if.then1:
-; CHECK-NEXT:    br label [[IF_THEN1]]
+; CHECK-NEXT:    br label [[IF_END1]]
 ; CHECK:       if.end1:
 ; CHECK-NEXT:    catchret from [[TMP3]] to label [[TRY_CONT]]
 ; CHECK:       try.cont:
diff --git a/llvm/test/Transforms/InstCombine/compare-alloca.ll b/llvm/test/Transforms/InstCombine/compare-alloca.ll
index c2639d39bc656..e6342520c6fa9 100644
--- a/llvm/test/Transforms/InstCombine/compare-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/compare-alloca.ll
@@ -75,7 +75,7 @@ define i1 @alloca_argument_compare_escaped_through_store(ptr %arg, ptr %ptr) {
 ; CHECK-LABEL: @alloca_argument_compare_escaped_through_store(
 ; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[ALLOC]], [[ARG:%.*]]
-; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds i8, ptr [[ALLOC]], i32 8
+; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds nuw i8, ptr [[ALLOC]], i32 8
 ; CHECK-NEXT:    store ptr [[P]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/compare-unescaped.ll b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
index 02eb464c814c8..c48c754e3941b 100644
--- a/llvm/test/Transforms/InstCombine/compare-unescaped.ll
+++ b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
@@ -79,7 +79,7 @@ declare void @escape(ptr)
 define i1 @compare_and_call_after() {
 ; CHECK-LABEL: @compare_and_call_after(
 ; CHECK-NEXT:    [[M:%.*]] = call dereferenceable_or_null(24) ptr @malloc(i64 24)
-; CHECK-NEXT:    [[LGP:%.*]] = load ptr, ptr @gp, align 8, !nonnull !0
+; CHECK-NEXT:    [[LGP:%.*]] = load ptr, ptr @gp, align 8, !nonnull [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[M]], [[LGP]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[ESCAPE_CALL:%.*]], label [[JUST_RETURN:%.*]]
 ; CHECK:       escape_call:
@@ -151,7 +151,7 @@ define ptr @compare_ret_escape(ptr %c) {
 ; CHECK:       retst:
 ; CHECK-NEXT:    ret ptr [[M]]
 ; CHECK:       chk:
-; CHECK-NEXT:    [[LGP:%.*]] = load ptr, ptr @gp, align 8, !nonnull !0
+; CHECK-NEXT:    [[LGP:%.*]] = load ptr, ptr @gp, align 8, !nonnull [[META0]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq ptr [[M]], [[LGP]]
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[RETST]], label [[CHK2:%.*]]
 ; CHECK:       chk2:
@@ -382,8 +382,8 @@ define i1 @two_nonnull_mallocs_hidden() {
 ; CHECK-LABEL: @two_nonnull_mallocs_hidden(
 ; CHECK-NEXT:    [[M:%.*]] = call nonnull dereferenceable(4) ptr @malloc(i64 4)
 ; CHECK-NEXT:    [[N:%.*]] = call nonnull dereferenceable(4) ptr @malloc(i64 4)
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[M]], i64 1
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[N]], i64 2
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[M]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds nuw i8, ptr [[N]], i64 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[GEP1]], [[GEP2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll b/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
index dafe3900fef3b..81052194b6b95 100644
--- a/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
+++ b/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
@@ -225,7 +225,7 @@ define i32 @test_cast_gep_large_indices_as() {
 
 define i32 @test_constant_cast_gep_struct_indices_as() {
 ; CHECK-LABEL: @test_constant_cast_gep_struct_indices_as(
-; CHECK-NEXT:    [[Y:%.*]] = load i32, ptr addrspace(3) getelementptr inbounds (i8, ptr addrspace(3) @constant_fold_global_ptr, i16 16), align 4
+; CHECK-NEXT:    [[Y:%.*]] = load i32, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @constant_fold_global_ptr, i16 16), align 4
 ; CHECK-NEXT:    ret i32 [[Y]]
 ;
   %x = getelementptr %struct.foo, ptr addrspace(3) @constant_fold_global_ptr, i18 0, i32 2, i12 2
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-gep.ll b/llvm/test/Transforms/InstCombine/constant-fold-gep.ll
index 54b7a6f66ecd2..321b97f557735 100644
--- a/llvm/test/Transforms/InstCombine/constant-fold-gep.ll
+++ b/llvm/test/Transforms/InstCombine/constant-fold-gep.ll
@@ -12,24 +12,24 @@ target datalayout = "E-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-
 define void @frob() {
 ; CHECK-LABEL: @frob(
 ; CHECK-NEXT:    store i32 1, ptr @Y, align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 4), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 8), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 12), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 16), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 20), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 24), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 28), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 32), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 36), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 40), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 44), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 48), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 52), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 56), align 8
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 60), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 64), align 8
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 68), align 4
-; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds (i8, ptr @Y, i64 72), align 8
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 4), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 8), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 12), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 16), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 20), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 24), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 28), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 32), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 36), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 40), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 44), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 48), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 52), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 56), align 8
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 60), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 64), align 8
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 68), align 4
+; CHECK-NEXT:    store i32 1, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 72), align 8
 ; CHECK-NEXT:    store i32 1, ptr getelementptr (i8, ptr @Y, i64 144), align 8
 ; CHECK-NEXT:    store i32 1, ptr getelementptr (i8, ptr @Y, i64 76), align 8
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/dependent-ivs.ll b/llvm/test/Transforms/InstCombine/dependent-ivs.ll
index 218e619643dfb..bfd01c6b68941 100644
--- a/llvm/test/Transforms/InstCombine/dependent-ivs.ll
+++ b/llvm/test/Transforms/InstCombine/dependent-ivs.ll
@@ -513,7 +513,7 @@ define void @ptr_iv_inbounds(ptr %base, i64 %end) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[IV_PTR:%.*]] = getelementptr inbounds i8, ptr [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 [[IV]]
 ; CHECK-NEXT:    call void @use.p0(ptr [[IV_PTR]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[END]]
diff --git a/llvm/test/Transforms/InstCombine/extractvalue.ll b/llvm/test/Transforms/InstCombine/extractvalue.ll
index 0b7c649839063..ba46dd33a59b0 100644
--- a/llvm/test/Transforms/InstCombine/extractvalue.ll
+++ b/llvm/test/Transforms/InstCombine/extractvalue.ll
@@ -51,7 +51,7 @@ define i32 @foo(i32 %a, i32 %b) {
 ; The new load should be in the same spot as the old load.
 define i32 @extract2gep(ptr %pair, ptr %P) {
 ; CHECK-LABEL: @extract2gep(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[PAIR:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[PAIR:%.*]], i64 4
 ; CHECK-NEXT:    [[E:%.*]] = load i32, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    store i32 0, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
@@ -82,7 +82,7 @@ end:
 ; to a 3-index inbounds gep + smaller load.
 define i16 @doubleextract2gep(ptr %arg) {
 ; CHECK-LABEL: @doubleextract2gep(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG:%.*]], i64 8
 ; CHECK-NEXT:    [[E2:%.*]] = load i16, ptr [[TMP1]], align 2
 ; CHECK-NEXT:    ret i16 [[E2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/fcmp-fadd-select.ll b/llvm/test/Transforms/InstCombine/fcmp-fadd-select.ll
index c49aea3e82b56..0d0af91608e7a 100644
--- a/llvm/test/Transforms/InstCombine/fcmp-fadd-select.ll
+++ b/llvm/test/Transforms/InstCombine/fcmp-fadd-select.ll
@@ -6,7 +6,7 @@
 define float @test_fcmp_ogt_fadd_select_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -19,7 +19,7 @@ define float @test_fcmp_ogt_fadd_select_constant(float %in) {
 define float @test_fcmp_ogt_fadd_select_constant_swapped(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_constant_swapped(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -32,7 +32,7 @@ define float @test_fcmp_ogt_fadd_select_constant_swapped(float %in) {
 define float @test_fcmp_ogt_fadd_select_neg_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_neg_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -45,7 +45,7 @@ define float @test_fcmp_ogt_fadd_select_neg_constant(float %in) {
 define float @test_fcmp_ogt_fadd_select_fastmath_preserve(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_fastmath_preserve(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -58,7 +58,7 @@ define float @test_fcmp_ogt_fadd_select_fastmath_preserve(float %in) {
 define <2 x float> @test_fcmp_ogt_fadd_select_constant_vectors(<2 x float> %in) {
 ; CHECK-LABEL: define <2 x float> @test_fcmp_ogt_fadd_select_constant_vectors(
 ; CHECK-SAME: <2 x float> [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz <2 x float> [[SEL_NEW]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    ret <2 x float> [[ADD_NEW]]
 ;
@@ -74,7 +74,7 @@ define <2 x float> @test_fcmp_ogt_fadd_select_constant_vectors(<2 x float> %in)
 define float @test_fcmp_olt_fadd_select_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_olt_fadd_select_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -87,7 +87,7 @@ define float @test_fcmp_olt_fadd_select_constant(float %in) {
 define float @test_fcmp_olt_fadd_select_constant_swapped(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_olt_fadd_select_constant_swapped(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -100,7 +100,7 @@ define float @test_fcmp_olt_fadd_select_constant_swapped(float %in) {
 define float @test_fcmp_olt_fadd_select_neg_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_olt_fadd_select_neg_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -113,7 +113,7 @@ define float @test_fcmp_olt_fadd_select_neg_constant(float %in) {
 define float @test_fcmp_olt_fadd_select_fastmath_preserve(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_olt_fadd_select_fastmath_preserve(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -126,7 +126,7 @@ define float @test_fcmp_olt_fadd_select_fastmath_preserve(float %in) {
 define <2 x float> @test_fcmp_olt_fadd_select_constant_vectors(<2 x float> %in) {
 ; CHECK-LABEL: define <2 x float> @test_fcmp_olt_fadd_select_constant_vectors(
 ; CHECK-SAME: <2 x float> [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz <2 x float> [[SEL_NEW]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    ret <2 x float> [[ADD_NEW]]
 ;
@@ -142,7 +142,7 @@ define <2 x float> @test_fcmp_olt_fadd_select_constant_vectors(<2 x float> %in)
 define float @test_fcmp_oge_fadd_select_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_oge_fadd_select_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -155,7 +155,7 @@ define float @test_fcmp_oge_fadd_select_constant(float %in) {
 define float @test_fcmp_oge_fadd_select_constant_swapped(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_oge_fadd_select_constant_swapped(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -168,7 +168,7 @@ define float @test_fcmp_oge_fadd_select_constant_swapped(float %in) {
 define float @test_fcmp_oge_fadd_select_neg_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_oge_fadd_select_neg_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -181,7 +181,7 @@ define float @test_fcmp_oge_fadd_select_neg_constant(float %in) {
 define float @test_fcmp_oge_fadd_select_fastmath_preserve(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_oge_fadd_select_fastmath_preserve(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -194,7 +194,7 @@ define float @test_fcmp_oge_fadd_select_fastmath_preserve(float %in) {
 define <2 x float> @test_fcmp_oge_fadd_select_constant_vectors(<2 x float> %in) {
 ; CHECK-LABEL: define <2 x float> @test_fcmp_oge_fadd_select_constant_vectors(
 ; CHECK-SAME: <2 x float> [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz <2 x float> [[SEL_NEW]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    ret <2 x float> [[ADD_NEW]]
 ;
@@ -210,7 +210,7 @@ define <2 x float> @test_fcmp_oge_fadd_select_constant_vectors(<2 x float> %in)
 define float @test_fcmp_ole_fadd_select_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ole_fadd_select_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -223,7 +223,7 @@ define float @test_fcmp_ole_fadd_select_constant(float %in) {
 define float @test_fcmp_ole_fadd_select_constant_swapped(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ole_fadd_select_constant_swapped(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -236,7 +236,7 @@ define float @test_fcmp_ole_fadd_select_constant_swapped(float %in) {
 define float @test_fcmp_ole_fadd_select_neg_constant(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ole_fadd_select_neg_constant(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -249,7 +249,7 @@ define float @test_fcmp_ole_fadd_select_neg_constant(float %in) {
 define float @test_fcmp_ole_fadd_select_fastmath_preserve(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ole_fadd_select_fastmath_preserve(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.minnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -262,7 +262,7 @@ define float @test_fcmp_ole_fadd_select_fastmath_preserve(float %in) {
 define <2 x float> @test_fcmp_ole_fadd_select_constant_vectors(<2 x float> %in) {
 ; CHECK-LABEL: define <2 x float> @test_fcmp_ole_fadd_select_constant_vectors(
 ; CHECK-SAME: <2 x float> [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[IN]], <2 x float> zeroinitializer)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz <2 x float> [[SEL_NEW]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    ret <2 x float> [[ADD_NEW]]
 ;
@@ -637,7 +637,7 @@ define float @test_fcmp_multiple_uses(float %in) {
 define float @test_fcmp_ogt_fadd_select_rewrite_flags1(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_rewrite_flags1(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call reassoc nnan nsz arcp contract afn float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call reassoc nsz arcp contract afn float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd reassoc nnan nsz arcp contract afn float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -650,7 +650,7 @@ define float @test_fcmp_ogt_fadd_select_rewrite_flags1(float %in) {
 define float @test_fcmp_ogt_fadd_select_rewrite_flags2(float %in) {
 ; CHECK-LABEL: define float @test_fcmp_ogt_fadd_select_rewrite_flags2(
 ; CHECK-SAME: float [[IN:%.*]]) {
-; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
+; CHECK-NEXT:    [[SEL_NEW:%.*]] = call nsz float @llvm.maxnum.f32(float [[IN]], float 0.000000e+00)
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd nnan nsz float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
@@ -667,7 +667,7 @@ define float @test_fcmp_ogt_fadd_select_rewrite_and_fastmath(float %in) {
 ; CHECK-NEXT:    [[ADD_NEW:%.*]] = fadd fast float [[SEL_NEW]], 1.000000e+00
 ; CHECK-NEXT:    ret float [[ADD_NEW]]
 ;
-  %cmp1 = fcmp ogt float %in, 0.000000e+00
+  %cmp1 = fcmp nnan ogt float %in, 0.000000e+00
   %add = fadd fast reassoc float %in, 1.000000e+00
   %sel = select fast i1 %cmp1, float %add, float 1.000000e+00
   ret float %sel
diff --git a/llvm/test/Transforms/InstCombine/fcmp-select.ll b/llvm/test/Transforms/InstCombine/fcmp-select.ll
index 028de1ff8a99f..408bc1cdc268f 100644
--- a/llvm/test/Transforms/InstCombine/fcmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/fcmp-select.ll
@@ -219,8 +219,8 @@ define double @test_fcmp_select_clamp(double %x) {
 
 define double @test_fcmp_select_maxnum(double %x) {
 ; CHECK-LABEL: @test_fcmp_select_maxnum(
-; CHECK-NEXT:    [[SEL1:%.*]] = call nnan nsz double @llvm.maxnum.f64(double [[X:%.*]], double 1.000000e+00)
-; CHECK-NEXT:    [[SEL2:%.*]] = call nnan nsz double @llvm.minnum.f64(double [[SEL1]], double 2.550000e+02)
+; CHECK-NEXT:    [[SEL1:%.*]] = call nsz double @llvm.maxnum.f64(double [[X:%.*]], double 1.000000e+00)
+; CHECK-NEXT:    [[SEL2:%.*]] = call nsz double @llvm.minnum.f64(double [[SEL1]], double 2.550000e+02)
 ; CHECK-NEXT:    ret double [[SEL2]]
 ;
   %cmp1 = fcmp ogt double %x, 1.0
diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll
index 70cbb4306ec67..fde0b8ab461a9 100644
--- a/llvm/test/Transforms/InstCombine/fmul.ll
+++ b/llvm/test/Transforms/InstCombine/fmul.ll
@@ -1098,7 +1098,7 @@ define void @fmul_loop_invariant_fdiv(ptr %a, float %x) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext nneg i32 [[I_08]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[F:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[M:%.*]] = fdiv fast float [[F]], [[X:%.*]]
 ; CHECK-NEXT:    store float [[M]], ptr [[ARRAYIDX]], align 4
@@ -1131,7 +1131,7 @@ for.body:
 
 define double @fmul_negated_constant_expression(double %x) {
 ; CHECK-LABEL: @fmul_negated_constant_expression(
-; CHECK-NEXT:    [[FSUB:%.*]] = fneg double bitcast (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @g, i64 16) to i64) to double)
+; CHECK-NEXT:    [[FSUB:%.*]] = fneg double bitcast (i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @g, i64 16) to i64) to double)
 ; CHECK-NEXT:    [[R:%.*]] = fmul double [[X:%.*]], [[FSUB]]
 ; CHECK-NEXT:    ret double [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/fneg.ll b/llvm/test/Transforms/InstCombine/fneg.ll
index 9692005edf2b6..549291f2c4f0d 100644
--- a/llvm/test/Transforms/InstCombine/fneg.ll
+++ b/llvm/test/Transforms/InstCombine/fneg.ll
@@ -1099,7 +1099,7 @@ define float @test_fneg_select_constant_var_multiuse(i1 %cond, float %x) {
 
 define float @test_fneg_select_maxnum(float %x) {
 ; CHECK-LABEL: @test_fneg_select_maxnum(
-; CHECK-NEXT:    [[SEL1:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[X:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    [[SEL1:%.*]] = call nsz float @llvm.maxnum.f32(float [[X:%.*]], float 1.000000e+00)
 ; CHECK-NEXT:    [[NEG:%.*]] = fneg float [[SEL1]]
 ; CHECK-NEXT:    ret float [[NEG]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/force-opaque-ptr.ll b/llvm/test/Transforms/InstCombine/force-opaque-ptr.ll
index 3b799e2fb2d0a..6181aafa4f422 100644
--- a/llvm/test/Transforms/InstCombine/force-opaque-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/force-opaque-ptr.ll
@@ -5,14 +5,14 @@
 
 define ptr @gep_constexpr_gv_1() {
 ; CHECK-LABEL: @gep_constexpr_gv_1(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g, i64 20)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g, i64 20)
 ;
   ret ptr getelementptr([16 x i16], ptr @g, i64 0, i64 10)
 }
 
 define ptr @gep_constexpr_gv_2() {
 ; CHECK-LABEL: @gep_constexpr_gv_2(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g, i64 24)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g, i64 24)
 ;
   ret ptr getelementptr(i32, ptr getelementptr([16 x i16], ptr @g, i64 0, i64 10), i64 1)
 }
diff --git a/llvm/test/Transforms/InstCombine/fortify-folding.ll b/llvm/test/Transforms/InstCombine/fortify-folding.ll
index 988726c99edb7..175975e16e17c 100644
--- a/llvm/test/Transforms/InstCombine/fortify-folding.ll
+++ b/llvm/test/Transforms/InstCombine/fortify-folding.ll
@@ -39,7 +39,7 @@ define ptr @test_memccpy_tail() {
 define ptr @test_mempcpy() {
 ; CHECK-LABEL: @test_mempcpy(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(15) @a, ptr noundef nonnull align 1 dereferenceable(15) @b, i64 15, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i64 15)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i64 15)
 ;
   %ret = call ptr @__mempcpy_chk(ptr @a, ptr @b, i64 15, i64 -1)
   ret ptr %ret
@@ -57,7 +57,7 @@ define ptr @test_not_mempcpy() {
 define ptr @test_mempcpy_tail() {
 ; CHECK-LABEL: @test_mempcpy_tail(
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(15) @a, ptr noundef nonnull align 1 dereferenceable(15) @b, i64 15, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i64 15)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i64 15)
 ;
   %ret = tail call ptr @__mempcpy_chk(ptr @a, ptr @b, i64 15, i64 -1)
   ret ptr %ret
diff --git a/llvm/test/Transforms/InstCombine/fpcast.ll b/llvm/test/Transforms/InstCombine/fpcast.ll
index 69daac773a645..d5290b572aefd 100644
--- a/llvm/test/Transforms/InstCombine/fpcast.ll
+++ b/llvm/test/Transforms/InstCombine/fpcast.ll
@@ -29,6 +29,17 @@ define half @test3(float %a) {
   ret half %c
 }
 
+define half @test3_fast(float %a) {
+; CHECK-LABEL: @test3_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = call float @llvm.fabs.f32(float %a)
+  %c = fptrunc fast float %b to half
+  ret half %c
+}
+
 define half @fneg_fptrunc(float %a) {
 ; CHECK-LABEL: @fneg_fptrunc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
@@ -75,9 +86,31 @@ define <2 x half> @unary_fneg_fptrunc_vec(<2 x float> %a) {
 
 define half @test4-fast(float %a) {
 ; CHECK-LABEL: @test4-fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc fast float [[A:%.*]] to half
 ; CHECK-NEXT:    [[C:%.*]] = fneg fast half [[TMP1]]
 ; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fsub fast float -0.0, %a
+  %c = fptrunc fast float %b to half
+  ret half %c
+}
+
+define half @test4-mixed-fast-1(float %a) {
+; CHECK-LABEL: @test4-mixed-fast-1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fneg half [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fsub float -0.0, %a
+  %c = fptrunc fast float %b to half
+  ret half %c
+}
+
+define half @test4-mixed-fast-2(float %a) {
+; CHECK-LABEL: @test4-mixed-fast-2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fneg half [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
 ;
   %b = fsub fast float -0.0, %a
   %c = fptrunc float %b to half
@@ -86,15 +119,70 @@ define half @test4-fast(float %a) {
 
 define half @test4_unary_fneg-fast(float %a) {
 ; CHECK-LABEL: @test4_unary_fneg-fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc fast float [[A:%.*]] to half
 ; CHECK-NEXT:    [[C:%.*]] = fneg fast half [[TMP1]]
 ; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fneg fast float %a
+  %c = fptrunc fast float %b to half
+  ret half %c
+}
+
+define half @test4_unary_fneg-mixed-fast-1(float %a) {
+; CHECK-LABEL: @test4_unary_fneg-mixed-fast-1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fneg half [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fneg float %a
+  %c = fptrunc fast float %b to half
+  ret half %c
+}
+
+define half @test4_unary_fneg-mixed-fast-2(float %a) {
+; CHECK-LABEL: @test4_unary_fneg-mixed-fast-2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fneg half [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
 ;
   %b = fneg fast float %a
   %c = fptrunc float %b to half
   ret half %c
 }
 
+define <2 x half> @test4_unary_fneg-vec-fast(<2 x float> %a) {
+; CHECK-LABEL: @test4_unary_fneg-vec-fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc fast <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[C:%.*]] = fneg fast <2 x half> [[TMP1]]
+; CHECK-NEXT:    ret <2 x half> [[C]]
+;
+  %b = fneg fast <2 x float> %a
+  %c = fptrunc fast <2 x float> %b to <2 x half>
+  ret <2 x half> %c
+}
+
+define <2 x half> @test4_unary_fneg-vec-mixed-fast-1(<2 x float> %a) {
+; CHECK-LABEL: @test4_unary_fneg-vec-mixed-fast-1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[C:%.*]] = fneg <2 x half> [[TMP1]]
+; CHECK-NEXT:    ret <2 x half> [[C]]
+;
+  %b = fneg <2 x float> %a
+  %c = fptrunc fast <2 x float> %b to <2 x half>
+  ret <2 x half> %c
+}
+
+define <2 x half> @test4_unary_fneg-vec-mixed-fast-2(<2 x float> %a) {
+; CHECK-LABEL: @test4_unary_fneg-vec-mixed-fast-2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[C:%.*]] = fneg <2 x half> [[TMP1]]
+; CHECK-NEXT:    ret <2 x half> [[C]]
+;
+  %b = fneg fast <2 x float> %a
+  %c = fptrunc <2 x float> %b to <2 x half>
+  ret <2 x half> %c
+}
+
 define half @test5(float %a, float %b, float %c) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[D:%.*]] = fcmp ogt float [[A:%.*]], [[B:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
index d6706d76056ee..141b44cbbb7a1 100644
--- a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
+++ b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll
@@ -46,6 +46,31 @@ if.end:
   ret i1 %cmp.i
 }
 
+define i1 @test2_or(double %x, i1 %cond) {
+; CHECK-LABEL: define i1 @test2_or(
+; CHECK-SAME: double [[X:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt double [[X]], 0x3EB0C6F7A0000000
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP]], [[COND]]
+; CHECK-NEXT:    br i1 [[OR]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = fcmp olt double %x, 0x3EB0C6F7A0000000
+  %or = or i1 %cmp, %cond
+  br i1 %or, label %if.then, label %if.end
+
+if.then:
+  ret i1 false
+
+if.end:
+  %cmp.i = fcmp oeq double %x, 0.000000e+00
+  ret i1 %cmp.i
+}
+
 define i1 @test3(float %x) {
 ; CHECK-LABEL: define i1 @test3(
 ; CHECK-SAME: float [[X:%.*]]) {
@@ -240,7 +265,6 @@ if.else:
   ret i1 false
 }
 
-; TODO: handle and/or conditions
 define i1 @test11_and(float %x, i1 %cond2) {
 ; CHECK-LABEL: define i1 @test11_and(
 ; CHECK-SAME: float [[X:%.*]], i1 [[COND2:%.*]]) {
@@ -248,8 +272,7 @@ define i1 @test11_and(float %x, i1 %cond2) {
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND]], [[COND2]]
 ; CHECK-NEXT:    br i1 [[AND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[RET1:%.*]] = fcmp oeq float [[X]], 0x7FF0000000000000
-; CHECK-NEXT:    ret i1 [[RET1]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -264,7 +287,6 @@ if.else:
   ret i1 false
 }
 
-; TODO: handle and/or conditions
 define i1 @test12_or(float %x, i1 %cond2) {
 ; CHECK-LABEL: define i1 @test12_or(
 ; CHECK-SAME: float [[X:%.*]], i1 [[COND2:%.*]]) {
@@ -275,7 +297,7 @@ define i1 @test12_or(float %x, i1 %cond2) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    ret i1 false
 ; CHECK:       if.else:
-; CHECK-NEXT:    [[RET:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 783)
+; CHECK-NEXT:    [[RET:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 780)
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/fptrunc.ll b/llvm/test/Transforms/InstCombine/fptrunc.ll
index a4296a326c4bc..0b5d8b3cd06e0 100644
--- a/llvm/test/Transforms/InstCombine/fptrunc.ll
+++ b/llvm/test/Transforms/InstCombine/fptrunc.ll
@@ -61,6 +61,18 @@ define float @fptrunc_select_true_val(float %x, double %y, i1 %cond) {
   ret float %r
 }
 
+define float @fptrunc_fast_select_true_val(float %x, double %y, i1 %cond) {
+; CHECK-LABEL: @fptrunc_fast_select_true_val(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc fast double [[Y:%.*]] to float
+; CHECK-NEXT:    [[NARROW_SEL:%.*]] = select i1 [[COND:%.*]], float [[TMP1]], float [[X:%.*]]
+; CHECK-NEXT:    ret float [[NARROW_SEL]]
+;
+  %e = fpext float %x to double
+  %sel = select fast i1 %cond, double %y, double %e
+  %r = fptrunc fast double %sel to float
+  ret float %r
+}
+
 define <2 x float> @fptrunc_select_false_val(<2 x float> %x, <2 x double> %y, <2 x i1> %cond) {
 ; CHECK-LABEL: @fptrunc_select_false_val(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x double> [[Y:%.*]] to <2 x float>
@@ -73,6 +85,18 @@ define <2 x float> @fptrunc_select_false_val(<2 x float> %x, <2 x double> %y, <2
   ret <2 x float> %r
 }
 
+define <2 x float> @fptrunc_nnan_select_false_val(<2 x float> %x, <2 x double> %y, <2 x i1> %cond) {
+; CHECK-LABEL: @fptrunc_nnan_select_false_val(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc nnan <2 x double> [[Y:%.*]] to <2 x float>
+; CHECK-NEXT:    [[NARROW_SEL:%.*]] = select <2 x i1> [[COND:%.*]], <2 x float> [[X:%.*]], <2 x float> [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[NARROW_SEL]]
+;
+  %e = fpext <2 x float> %x to <2 x double>
+  %sel = select nnan <2 x i1> %cond, <2 x double> %e, <2 x double> %y
+  %r = fptrunc nnan <2 x double> %sel to <2 x float>
+  ret <2 x float> %r
+}
+
 declare void @use(float)
 
 define half @fptrunc_select_true_val_extra_use(half %x, float %y, i1 %cond) {
diff --git a/llvm/test/Transforms/InstCombine/freeze-phi.ll b/llvm/test/Transforms/InstCombine/freeze-phi.ll
index 8bdfc90f20a4e..e94af34bf1cd6 100644
--- a/llvm/test/Transforms/InstCombine/freeze-phi.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-phi.ll
@@ -116,8 +116,8 @@ C:
 define i32 @two_undef(i8 %cond, i32 %x) {
 ; CHECK-LABEL: @two_undef(
 ; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
-; CHECK-NEXT:    i8 0, label [[B:%.*]]
-; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:      i8 0, label [[B:%.*]]
+; CHECK-NEXT:      i8 1, label [[C:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       A:
 ; CHECK-NEXT:    br label [[D:%.*]]
@@ -149,8 +149,8 @@ D:
 define i32 @one_undef(i8 %cond) {
 ; CHECK-LABEL: @one_undef(
 ; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
-; CHECK-NEXT:    i8 0, label [[B:%.*]]
-; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:      i8 0, label [[B:%.*]]
+; CHECK-NEXT:      i8 1, label [[C:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       A:
 ; CHECK-NEXT:    br label [[D:%.*]]
@@ -183,11 +183,11 @@ D:
 define i32 @one_constexpr(i8 %cond, i32 %x) {
 ; CHECK-LABEL: @one_constexpr(
 ; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
-; CHECK-NEXT:    i8 0, label [[B:%.*]]
-; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:      i8 0, label [[B:%.*]]
+; CHECK-NEXT:      i8 1, label [[C:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       A:
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 ptrtoint (ptr getelementptr inbounds (i8, ptr @glb, i64 2) to i32)
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @glb, i64 2) to i32)
 ; CHECK-NEXT:    br label [[D:%.*]]
 ; CHECK:       B:
 ; CHECK-NEXT:    br label [[D]]
diff --git a/llvm/test/Transforms/InstCombine/fsh.ll b/llvm/test/Transforms/InstCombine/fsh.ll
index 79d3360377e50..434cd810296d8 100644
--- a/llvm/test/Transforms/InstCombine/fsh.ll
+++ b/llvm/test/Transforms/InstCombine/fsh.ll
@@ -760,17 +760,17 @@ define i32 @fsh_load_rotate_12(ptr %data) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[DATA:%.*]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[CONV]], 24
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[SHL3:%.*]] = shl nuw nsw i32 [[CONV2]], 16
 ; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[SHL3]], [[SHL]]
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
 ; CHECK-NEXT:    [[CONV5:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[SHL6:%.*]] = shl nuw nsw i32 [[CONV5]], 8
 ; CHECK-NEXT:    [[OR7:%.*]] = or disjoint i32 [[OR]], [[SHL6]]
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
 ; CHECK-NEXT:    [[CONV9:%.*]] = zext i8 [[TMP3]] to i32
 ; CHECK-NEXT:    [[OR10:%.*]] = or disjoint i32 [[OR7]], [[CONV9]]
@@ -808,17 +808,17 @@ define i32 @fsh_load_rotate_25(ptr %data) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[DATA:%.*]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[CONV]], 24
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[SHL3:%.*]] = shl nuw nsw i32 [[CONV2]], 16
 ; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[SHL3]], [[SHL]]
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
 ; CHECK-NEXT:    [[CONV5:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[SHL6:%.*]] = shl nuw nsw i32 [[CONV5]], 8
 ; CHECK-NEXT:    [[OR7:%.*]] = or disjoint i32 [[OR]], [[SHL6]]
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
 ; CHECK-NEXT:    [[CONV9:%.*]] = zext i8 [[TMP3]] to i32
 ; CHECK-NEXT:    [[OR10:%.*]] = or disjoint i32 [[OR7]], [[CONV9]]
diff --git a/llvm/test/Transforms/InstCombine/gep-addrspace.ll b/llvm/test/Transforms/InstCombine/gep-addrspace.ll
index 789a31e593946..0d1850b8fa020 100644
--- a/llvm/test/Transforms/InstCombine/gep-addrspace.ll
+++ b/llvm/test/Transforms/InstCombine/gep-addrspace.ll
@@ -45,7 +45,7 @@ define { i8, i8 } @inbounds_after_addrspacecast() {
 ; CHECK-NEXT:    [[T0:%.*]] = alloca i16, align 2
 ; CHECK-NEXT:    call void @escape_alloca(ptr nonnull [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = addrspacecast ptr [[T0]] to ptr addrspace(11)
-; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds i8, ptr addrspace(11) [[T1]], i64 1
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(11) [[T1]], i64 1
 ; CHECK-NEXT:    [[T3:%.*]] = load i8, ptr addrspace(11) [[T2]], align 1
 ; CHECK-NEXT:    [[INSERT:%.*]] = insertvalue { i8, i8 } zeroinitializer, i8 [[T3]], 1
 ; CHECK-NEXT:    ret { i8, i8 } [[INSERT]]
diff --git a/llvm/test/Transforms/InstCombine/gep-canonicalize-constant-indices.ll b/llvm/test/Transforms/InstCombine/gep-canonicalize-constant-indices.ll
index 849793c7a85af..fa006ff992ba3 100644
--- a/llvm/test/Transforms/InstCombine/gep-canonicalize-constant-indices.ll
+++ b/llvm/test/Transforms/InstCombine/gep-canonicalize-constant-indices.ll
@@ -13,7 +13,7 @@ declare void @use(i1)
 ; result = (((ptr) p + a) + b) + 1
 define ptr @basic(ptr %p, i64 %a, i64 %b) {
 ; CHECK-LABEL: @basic(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[A:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 [[B:%.*]]
 ; CHECK-NEXT:    ret ptr [[TMP3]]
@@ -50,9 +50,9 @@ define ptr @partialConstant2(ptr %p, i64 %a, i64 %b) {
 ; result = ((ptr) p + a) + 3
 define ptr @merge(ptr %p, i64 %a) {
 ; CHECK-LABEL: @merge(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[A:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 ; CHECK-NEXT:    ret ptr [[TMP3]]
 ;
   %1 = getelementptr inbounds i32, ptr %p, i64 1
@@ -67,12 +67,12 @@ define ptr @merge(ptr %p, i64 %a) {
 ; result = (ptr) ((ptr) ((ptr) ptr + a) + (a * b)) + 9
 define ptr @nested(ptr %p, i64 %a, i64 %b) {
 ; CHECK-LABEL: @nested(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[A:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[A]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 128
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 128
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[TMP4]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 16
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
 ; CHECK-NEXT:    ret ptr [[TMP6]]
 ;
   %1 = getelementptr inbounds <3 x i32>, ptr %p, i64 1
@@ -87,7 +87,7 @@ define ptr @nested(ptr %p, i64 %a, i64 %b) {
 ; It is valid to swap if the source operand of the first GEP has multiple uses.
 define ptr @multipleUses1(ptr %p) {
 ; CHECK-LABEL: @multipleUses1(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[P]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[TMP2]]
 ; CHECK-NEXT:    ret ptr [[TMP3]]
@@ -101,7 +101,7 @@ define ptr @multipleUses1(ptr %p) {
 ; It is valid to swap if the second GEP has multiple uses.
 define ptr @multipleUses2(ptr %p, i64 %a) {
 ; CHECK-LABEL: @multipleUses2(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[A:%.*]]
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    ret ptr [[TMP2]]
@@ -115,7 +115,7 @@ define ptr @multipleUses2(ptr %p, i64 %a) {
 ; Negative test. It is not valid to swap if the first GEP has multiple uses.
 define ptr @multipleUses3(ptr %p) {
 ; CHECK-LABEL: @multipleUses3(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[TMP2]]
 ; CHECK-NEXT:    ret ptr [[TMP3]]
diff --git a/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll b/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
index 5f22a35457015..9b266b097057c 100644
--- a/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
+++ b/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
@@ -8,9 +8,9 @@ define i32 @foo(ptr nocapture readnone %match, i32 %cur_match, i32 %best_len, i3
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT2:%.*]] = zext i32 [[CUR_MATCH:%.*]] to i64
-; CHECK-NEXT:    [[ADD_PTR4:%.*]] = getelementptr inbounds i8, ptr [[WIN:%.*]], i64 [[IDX_EXT2]]
+; CHECK-NEXT:    [[ADD_PTR4:%.*]] = getelementptr inbounds nuw i8, ptr [[WIN:%.*]], i64 [[IDX_EXT2]]
 ; CHECK-NEXT:    [[IDX_EXT1:%.*]] = zext i32 [[BEST_LEN:%.*]] to i64
-; CHECK-NEXT:    [[ADD_PTR25:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR4]], i64 [[IDX_EXT1]]
+; CHECK-NEXT:    [[ADD_PTR25:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR4]], i64 [[IDX_EXT1]]
 ; CHECK-NEXT:    [[ADD_PTR36:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR25]], i64 -1
 ; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[ADD_PTR36]], align 4
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp eq i32 [[I1]], [[SCAN_END:%.*]]
@@ -19,8 +19,8 @@ define i32 @foo(ptr nocapture readnone %match, i32 %cur_match, i32 %best_len, i3
 ; CHECK-NEXT:    br label [[IF_THEN:%.*]]
 ; CHECK:       do.body:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[I4:%.*]] to i64
-; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[WIN]], i64 [[IDX_EXT]]
-; CHECK-NEXT:    [[ADD_PTR2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT1]]
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[WIN]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 [[IDX_EXT1]]
 ; CHECK-NEXT:    [[ADD_PTR3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR2]], i64 -1
 ; CHECK-NEXT:    [[I3:%.*]] = load i32, ptr [[ADD_PTR3]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I3]], [[SCAN_END]]
@@ -30,7 +30,7 @@ define i32 @foo(ptr nocapture readnone %match, i32 %cur_match, i32 %best_len, i3
 ; CHECK-NEXT:    [[CHAIN_LENGTH_ADDR_08:%.*]] = phi i32 [ [[CHAIN_LENGTH:%.*]], [[IF_THEN_LR_PH]] ], [ [[DEC:%.*]], [[DO_BODY]] ]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CUR_MATCH_ADDR_09]], [[WMASK:%.*]]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[AND]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PREV:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[PREV:%.*]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[I4]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp ugt i32 [[I4]], [[LIMIT:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP4]], label [[LAND_LHS_TRUE:%.*]], label [[DO_END]]
@@ -94,7 +94,7 @@ define void @PR37005(ptr %base, ptr %in) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[E1:%.*]] = getelementptr inbounds ptr, ptr [[IN:%.*]], i64 undef
-; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8, ptr [[E1]], i64 48
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds nuw i8, ptr [[E1]], i64 48
 ; CHECK-NEXT:    [[E4:%.*]] = getelementptr inbounds ptr, ptr [[E2]], <2 x i64> <i64 0, i64 1>
 ; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint <2 x ptr> [[E4]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i64> [[PI1]], splat (i64 14)
@@ -126,11 +126,11 @@ define void @PR37005_2(ptr %base, ptr %in) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[E1:%.*]] = getelementptr inbounds ptr, ptr [[IN:%.*]], i64 undef
-; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8, ptr [[E1]], i64 48
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds nuw i8, ptr [[E1]], i64 48
 ; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint ptr [[E2]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 [[PI1]], 14
 ; CHECK-NEXT:    [[SL1:%.*]] = and i64 [[TMP0]], 1125899906842496
-; CHECK-NEXT:    [[E5:%.*]] = getelementptr inbounds i8, ptr [[BASE:%.*]], i64 [[SL1]]
+; CHECK-NEXT:    [[E5:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE:%.*]], i64 [[SL1]]
 ; CHECK-NEXT:    [[E6:%.*]] = getelementptr inbounds i8, ptr [[E5]], <2 x i64> <i64 80, i64 60>
 ; CHECK-NEXT:    call void @blackhole(<2 x ptr> [[E6]])
 ; CHECK-NEXT:    br label [[LOOP]]
@@ -156,7 +156,7 @@ define void @PR37005_3(<2 x ptr> %base, ptr %in) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[E1:%.*]] = getelementptr inbounds ptr, ptr [[IN:%.*]], i64 undef
-; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8, ptr [[E1]], i64 48
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds nuw i8, ptr [[E1]], i64 48
 ; CHECK-NEXT:    [[E4:%.*]] = getelementptr inbounds ptr, ptr [[E2]], <2 x i64> <i64 0, i64 1>
 ; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint <2 x ptr> [[E4]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i64> [[PI1]], splat (i64 14)
@@ -222,7 +222,7 @@ define float @gep_cross_loop(ptr %_arg_, ptr %_arg_3, float %_arg_8) {
 ; CHECK:       for.cond.i.i.i.preheader:
 ; CHECK-NEXT:    ret float [[SUM]]
 ; CHECK:       for.body.i:
-; CHECK-NEXT:    [[ARRAYIDX_I84_I:%.*]] = getelementptr inbounds float, ptr [[ADD_PTR]], i64 [[IDX]]
+; CHECK-NEXT:    [[ARRAYIDX_I84_I:%.*]] = getelementptr inbounds nuw float, ptr [[ADD_PTR]], i64 [[IDX]]
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[ARRAYIDX_I84_I]], align 4
 ; CHECK-NEXT:    [[ADD_I]] = fadd fast float [[SUM]], [[I1]]
 ; CHECK-NEXT:    [[ADD11_I]] = add nuw nsw i64 [[IDX]], 1
@@ -259,7 +259,7 @@ define void @only_one_inbounds(ptr %ptr, i1 %c, i32 noundef %arg1, i32 noundef %
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[ARG1_EXT:%.*]] = zext i32 [[ARG1:%.*]] to i64
-; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[ARG1_EXT]]
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[ARG1_EXT]]
 ; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr i8, ptr [[PTR2]], i64 [[ARG2_EXT]]
 ; CHECK-NEXT:    call void @use(ptr [[PTR3]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
@@ -287,7 +287,7 @@ define void @both_inbounds_one_neg(ptr %ptr, i1 %c, i32 noundef %arg) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[ARG_EXT:%.*]] = zext i32 [[ARG:%.*]] to i64
-; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[ARG_EXT]]
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[ARG_EXT]]
 ; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds i8, ptr [[PTR2]], i64 -1
 ; CHECK-NEXT:    call void @use(ptr nonnull [[PTR3]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
@@ -314,8 +314,8 @@ define void @both_inbounds_pos(ptr %ptr, i1 %c, i32 noundef %arg) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[ARG_EXT:%.*]] = zext i32 [[ARG:%.*]] to i64
-; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[ARG_EXT]]
-; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds i8, ptr [[PTR2]], i64 1
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[ARG_EXT]]
+; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 1
 ; CHECK-NEXT:    call void @use(ptr nonnull [[PTR3]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/InstCombine/gep-custom-dl.ll b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
index ed9a5e6497638..4cd711a2582ae 100644
--- a/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
+++ b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
@@ -34,7 +34,7 @@ define ptr @test2(ptr %I) {
 define void @test3(i8 %B) {
 ; This should be turned into a constexpr instead of being an instruction
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds (i8, ptr @Global, i32 4), align 1
+; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds nuw (i8, ptr @Global, i32 4), align 1
 ; CHECK-NEXT:    ret void
 ;
   %A = getelementptr [10 x i8], ptr @Global, i32 0, i32 4
@@ -62,7 +62,7 @@ define void @test_evaluate_gep_nested_as_ptrs(ptr addrspace(2) %B) {
 
 define void @test_evaluate_gep_as_ptrs_array(ptr addrspace(2) %B) {
 ; CHECK-LABEL: @test_evaluate_gep_as_ptrs_array(
-; CHECK-NEXT:    store ptr addrspace(2) [[B:%.*]], ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @arst, i32 16), align 8
+; CHECK-NEXT:    store ptr addrspace(2) [[B:%.*]], ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @arst, i32 16), align 8
 ; CHECK-NEXT:    ret void
 ;
 
@@ -168,7 +168,7 @@ define i32 @test10() {
 define i16 @constant_fold_custom_dl() {
 ; CHECK-LABEL: @constant_fold_custom_dl(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i16 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @X_as1, i32 1000), i16 sub (i16 0, i16 ptrtoint (ptr addrspace(1) @X_as1 to i16))) to i16)
+; CHECK-NEXT:    ret i16 ptrtoint (ptr addrspace(1) getelementptr (i8, ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @X_as1, i32 1000), i16 sub (i16 0, i16 ptrtoint (ptr addrspace(1) @X_as1 to i16))) to i16)
 ;
 
 entry:
diff --git a/llvm/test/Transforms/InstCombine/gep-merge-constant-indices.ll b/llvm/test/Transforms/InstCombine/gep-merge-constant-indices.ll
index edec54495c1ae..46f42e16c83ce 100644
--- a/llvm/test/Transforms/InstCombine/gep-merge-constant-indices.ll
+++ b/llvm/test/Transforms/InstCombine/gep-merge-constant-indices.ll
@@ -12,7 +12,7 @@ target datalayout = "i24:8:8"
 ; result = (ptr) p + 3
 define ptr @mergeBasic(ptr %p) {
 ; CHECK-LABEL: @mergeBasic(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 12
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 12
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i32, ptr %p, i64 1
@@ -24,7 +24,7 @@ define ptr @mergeBasic(ptr %p) {
 ; result = (ptr) p + 10
 define ptr @mergeDifferentTypes(ptr %p) {
 ; CHECK-LABEL: @mergeDifferentTypes(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 10
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 10
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i8, ptr %p, i64 2
@@ -36,7 +36,7 @@ define ptr @mergeDifferentTypes(ptr %p) {
 ; result = (ptr) p + 10
 define ptr @mergeReverse(ptr %p) {
 ; CHECK-LABEL: @mergeReverse(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 10
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 10
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i64, ptr %p, i64 1
@@ -58,7 +58,7 @@ define ptr @zeroSum(ptr %p) {
 ; result = (ptr) ((ptr) p + 1) + 17
 define ptr @array1(ptr %p) {
 ; CHECK-LABEL: @array1(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 37
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 37
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds [20 x i8], ptr %p, i64 1, i64 1
@@ -70,7 +70,7 @@ define ptr @array1(ptr %p) {
 ; result = (ptr) p + 20
 define ptr @array2(ptr %p) {
 ; CHECK-LABEL: @array2(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 20
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 20
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i64, ptr %p, i64 2
@@ -82,7 +82,7 @@ define ptr @array2(ptr %p) {
 ; result = (ptr) p + 36
 define ptr @struct1(ptr %p) {
 ; CHECK-LABEL: @struct1(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 36
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 36
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i64, ptr %p, i64 3
@@ -104,7 +104,7 @@ define ptr @struct2(ptr %p) {
 ; result = (ptr) &((struct.B) p)[0].member2.member0 + 7
 define ptr @structStruct(ptr %p) {
 ; CHECK-LABEL: @structStruct(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 15
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 15
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds %struct.B, ptr %p, i64 0, i32 2, i32 0, i64 3
@@ -132,7 +132,7 @@ define ptr @appendIndex(ptr %p, i64 %i) {
 ; result = (ptr) p + 7
 define ptr @notDivisible(ptr %p) {
 ; CHECK-LABEL: @notDivisible(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 7
 ; CHECK-NEXT:    ret ptr [[TMP1]]
 ;
   %1 = getelementptr inbounds i24, ptr %p, i64 1
@@ -154,7 +154,7 @@ define ptr @partialConstant2(ptr %p, i64 %a) {
 
 define ptr @partialConstant3(ptr %p) {
 ; CHECK-LABEL: @partialConstant3(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = shl nsw i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[DOTOFFS:%.*]] = or disjoint i64 [[DOTIDX]], 16
@@ -186,7 +186,7 @@ define ptr @partialConstantMemberAliasing1(ptr %p, i64 %a) {
 define ptr @partialConstantMemberAliasing2(ptr %p, i64 %a) {
 ; CHECK-LABEL: @partialConstantMemberAliasing2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_C:%.*]], ptr [[P:%.*]], i64 [[A:%.*]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 1
 ; CHECK-NEXT:    ret ptr [[TMP2]]
 ;
   %1 = getelementptr inbounds %struct.C, ptr %p, i64 %a, i32 1
@@ -199,7 +199,7 @@ define ptr @partialConstantMemberAliasing2(ptr %p, i64 %a) {
 define ptr @partialConstantMemberAliasing3(ptr %p, i64 %a) {
 ; CHECK-LABEL: @partialConstantMemberAliasing3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_C:%.*]], ptr [[P:%.*]], i64 [[A:%.*]], i32 2
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
 ; CHECK-NEXT:    ret ptr [[TMP2]]
 ;
   %1 = getelementptr inbounds %struct.C, ptr %p, i64 %a, i32 2
diff --git a/llvm/test/Transforms/InstCombine/gep-vector-indices.ll b/llvm/test/Transforms/InstCombine/gep-vector-indices.ll
index e9534e45ec141..660ce317a0f64 100644
--- a/llvm/test/Transforms/InstCombine/gep-vector-indices.ll
+++ b/llvm/test/Transforms/InstCombine/gep-vector-indices.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=instcombine %s -S | FileCheck %s
+; RUN: opt -passes=instcombine -use-constant-int-for-fixed-length-splat %s -S | FileCheck %s
 
 define ptr @vector_splat_indices_v2i64_ext0(ptr %a) {
 ; CHECK-LABEL: @vector_splat_indices_v2i64_ext0(
@@ -13,7 +14,7 @@ define ptr @vector_splat_indices_v2i64_ext0(ptr %a) {
 
 define ptr @vector_splat_indices_nxv2i64_ext0(ptr %a) {
 ; CHECK-LABEL: @vector_splat_indices_nxv2i64_ext0(
-; CHECK-NEXT:    [[RES:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; CHECK-NEXT:    [[RES:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; CHECK-NEXT:    ret ptr [[RES]]
 ;
   %tmp = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
diff --git a/llvm/test/Transforms/InstCombine/gepphigep.ll b/llvm/test/Transforms/InstCombine/gepphigep.ll
index 93266c6754396..6b5dae93c1e04 100644
--- a/llvm/test/Transforms/InstCombine/gepphigep.ll
+++ b/llvm/test/Transforms/InstCombine/gepphigep.ll
@@ -54,7 +54,7 @@ define i32 @test2(ptr %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19) {
 ; CHECK-NEXT:    store i32 0, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT2]], ptr [[TMP1]], i64 [[TMP19:%.*]]
 ; CHECK-NEXT:    store i32 0, ptr [[TMP20]], align 4
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP10]], i64 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    ret i32 [[TMP25]]
 ;
@@ -86,12 +86,12 @@ define i32 @test3(ptr %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19, i64 %tmp20, i64 %tmp
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ [[TMP19]], [[BB1]] ], [ [[TMP20]], [[BB2]] ]
 ; CHECK-NEXT:    [[TMP22:%.*]] = invoke i32 @foo1(i32 11)
-; CHECK-NEXT:    to label [[BB4:%.*]] unwind label [[BB5:%.*]]
+; CHECK-NEXT:            to label [[BB4:%.*]] unwind label [[BB5:%.*]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       bb5:
 ; CHECK-NEXT:    [[TMP27:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    catch ptr @_ZTIi
+; CHECK-NEXT:            catch ptr @_ZTIi
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT3]], ptr [[DM]], i64 [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds [[STRUCT4:%.*]], ptr [[TMP1]], i64 [[TMP21:%.*]], i32 1, i32 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP35]], align 4
@@ -144,7 +144,7 @@ define ptr @test4(i32 %value, ptr %buffer) {
 ; CHECK:       loop.body:
 ; CHECK-NEXT:    [[BUFFER_PN:%.*]] = phi ptr [ [[BUFFER:%.*]], [[LOOP_HEADER]] ], [ [[LOOPPTR:%.*]], [[LOOP_BODY]] ]
 ; CHECK-NEXT:    [[NEWVAL:%.*]] = phi i32 [ [[VALUE]], [[LOOP_HEADER]] ], [ [[SHR:%.*]], [[LOOP_BODY]] ]
-; CHECK-NEXT:    [[LOOPPTR]] = getelementptr inbounds i8, ptr [[BUFFER_PN]], i64 1
+; CHECK-NEXT:    [[LOOPPTR]] = getelementptr inbounds nuw i8, ptr [[BUFFER_PN]], i64 1
 ; CHECK-NEXT:    [[SHR]] = lshr i32 [[NEWVAL]], 7
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[NEWVAL]], 16383
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP_BODY]], label [[LOOP_EXIT:%.*]]
@@ -152,7 +152,7 @@ define ptr @test4(i32 %value, ptr %buffer) {
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi ptr [ [[LOOPPTR]], [[LOOP_EXIT]] ], [ [[BUFFER]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[INCPTR3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 2
+; CHECK-NEXT:    [[INCPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 2
 ; CHECK-NEXT:    ret ptr [[INCPTR3]]
 ;
 entry:
@@ -189,7 +189,7 @@ define void @test5(ptr %idx, ptr %in) #0 {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[IN:%.*]], align 8
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
 ; CHECK-NEXT:    [[CMP23:%.*]] = icmp eq i8 [[TMP1]], 54
 ; CHECK-NEXT:    br i1 [[CMP23]], label [[WHILE_COND:%.*]], label [[IF_THEN_25:%.*]]
@@ -203,15 +203,15 @@ define void @test5(ptr %idx, ptr %in) #0 {
 ; CHECK-NEXT:    [[LNOT:%.*]] = icmp eq i8 [[AND]], 0
 ; CHECK-NEXT:    br i1 [[LNOT]], label [[WHILE_BODY]], label [[WHILE_COND_33:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[INCDEC_PTR32]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR32]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    br label [[WHILE_COND]]
 ; CHECK:       while.cond.33:
-; CHECK-NEXT:    [[INCDEC_PTR34:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR34:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    br label [[WHILE_COND_57:%.*]]
 ; CHECK:       while.cond.57:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[INCDEC_PTR34]], align 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i16, ptr [[IDX:%.*]], i64 [[TMP4]]
+; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds nuw i16, ptr [[IDX:%.*]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX61]], align 2
 ; CHECK-NEXT:    [[AND63:%.*]] = and i16 [[TMP5]], 2048
 ; CHECK-NEXT:    [[TOBOOL64:%.*]] = icmp eq i16 [[AND63]], 0
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index 2ea65eb443cdf..ec03d9a2dae2b 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -64,7 +64,7 @@ define ptr @test4(ptr %I) {
 define void @test5(i8 %B) {
         ; This should be turned into a constexpr instead of being an instruction
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds (i8, ptr @Global, i64 4), align 1
+; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds nuw (i8, ptr @Global, i64 4), align 1
 ; CHECK-NEXT:    ret void
 ;
   %A = getelementptr [10 x i8], ptr @Global, i64 0, i64 4
@@ -75,7 +75,7 @@ define void @test5(i8 %B) {
 define void @test5_as1(i8 %B) {
         ; This should be turned into a constexpr instead of being an instruction
 ; CHECK-LABEL: @test5_as1(
-; CHECK-NEXT:    store i8 [[B:%.*]], ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @Global_as1, i16 4), align 1
+; CHECK-NEXT:    store i8 [[B:%.*]], ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @Global_as1, i16 4), align 1
 ; CHECK-NEXT:    ret void
 ;
   %A = getelementptr [10 x i8], ptr addrspace(1) @Global_as1, i16 0, i16 4
@@ -103,7 +103,7 @@ define void @test_evaluate_gep_nested_as_ptrs(ptr addrspace(2) %B) {
 
 define void @test_evaluate_gep_as_ptrs_array(ptr addrspace(2) %B) {
 ; CHECK-LABEL: @test_evaluate_gep_as_ptrs_array(
-; CHECK-NEXT:    store ptr addrspace(2) [[B:%.*]], ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @arst, i16 8), align 4
+; CHECK-NEXT:    store ptr addrspace(2) [[B:%.*]], ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @arst, i16 8), align 4
 ; CHECK-NEXT:    ret void
 ;
 
@@ -115,7 +115,7 @@ define void @test_evaluate_gep_as_ptrs_array(ptr addrspace(2) %B) {
 ; This should be turned into a constexpr instead of being an instruction
 define void @test_overaligned_vec(i8 %B) {
 ; CHECK-LABEL: @test_overaligned_vec(
-; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds (i8, ptr @Global, i64 2), align 1
+; CHECK-NEXT:    store i8 [[B:%.*]], ptr getelementptr inbounds nuw (i8, ptr @Global, i64 2), align 1
 ; CHECK-NEXT:    ret void
 ;
   %A = getelementptr <2 x half>, ptr @Global, i64 0, i64 1
@@ -422,7 +422,7 @@ define ptr @test_index_canon_nusw_nuw(ptr %X, i32 %Idx) {
 
 define ptr @test_index_canon_const_expr_inbounds() {
 ; CHECK-LABEL: @test_index_canon_const_expr_inbounds(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @Global, i64 123)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @Global, i64 123)
 ;
   ret ptr getelementptr inbounds (i8, ptr @Global, i32 123)
 }
@@ -443,7 +443,7 @@ define ptr @test_const_gep_gep_nuw() {
 
 define ptr @test_const_gep_gep_nusw_no_overflow() {
 ; CHECK-LABEL: @test_const_gep_gep_nusw_no_overflow(
-; CHECK-NEXT:    ret ptr getelementptr nusw (i8, ptr @Global, i64 246)
+; CHECK-NEXT:    ret ptr getelementptr nusw nuw (i8, ptr @Global, i64 246)
 ;
   ret ptr getelementptr nusw (i8, ptr getelementptr nusw (i8, ptr @Global, i64 123), i64 123)
 }
@@ -598,7 +598,7 @@ define i32 @test21() {
 
 define i1 @test22() {
 ; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult ptr getelementptr inbounds (i8, ptr @A, i64 4), getelementptr (i8, ptr @B, i64 8)
+; CHECK-NEXT:    [[C:%.*]] = icmp ult ptr getelementptr inbounds nuw (i8, ptr @A, i64 4), getelementptr (i8, ptr @B, i64 8)
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %C = icmp ult ptr getelementptr (i32, ptr @A, i64 1),
@@ -686,7 +686,7 @@ define i32 @test28() nounwind  {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ORIENTATIONS:%.*]] = alloca [1 x [1 x %struct.x]], align 8
 ; CHECK-NEXT:    [[T3:%.*]] = call i32 @puts(ptr noundef nonnull dereferenceable(1) @.str) #[[ATTR0]]
-; CHECK-NEXT:    [[T45:%.*]] = getelementptr inbounds i8, ptr [[ORIENTATIONS]], i64 1
+; CHECK-NEXT:    [[T45:%.*]] = getelementptr inbounds nuw i8, ptr [[ORIENTATIONS]], i64 1
 ; CHECK-NEXT:    br label [[BB10:%.*]]
 ; CHECK:       bb10:
 ; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
@@ -796,9 +796,9 @@ define ptr @test32(ptr %v) {
 ; CHECK-LABEL: @test32(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [4 x ptr], align 16
 ; CHECK-NEXT:    store ptr null, ptr [[A]], align 8
-; CHECK-NEXT:    [[D:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; CHECK-NEXT:    [[D:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    store ptr [[V:%.*]], ptr [[D]], align 8
-; CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 16
+; CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
 ; CHECK-NEXT:    [[G:%.*]] = load ptr, ptr [[F]], align 8
 ; CHECK-NEXT:    ret ptr [[G]]
 ;
@@ -890,7 +890,7 @@ entry:
 
 define i32 @test35() nounwind {
 ; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @"\01LC8", ptr nonnull getelementptr inbounds (i8, ptr @s, i64 8)) #[[ATTR0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @"\01LC8", ptr nonnull getelementptr inbounds nuw (i8, ptr @s, i64 8)) #[[ATTR0]]
 ; CHECK-NEXT:    ret i32 0
 ;
   call i32 (ptr, ...) @printf(ptr @"\01LC8",
@@ -935,7 +935,7 @@ declare void @pr10322_f3(ptr)
 define void @pr10322_f1(ptr %foo) {
 ; CHECK-LABEL: @pr10322_f1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, ptr [[FOO:%.*]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO:%.*]], i64 16
 ; CHECK-NEXT:    call void @pr10322_f2(ptr nonnull [[ARRAYIDX8]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @pr10322_f3(ptr nonnull [[ARRAYIDX8]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret void
@@ -975,7 +975,7 @@ declare void @three_gep_h(ptr)
 
 define void @test39(ptr %arg, i8 %arg1) nounwind {
 ; CHECK-LABEL: @test39(
-; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 16
+; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG:%.*]], i64 16
 ; CHECK-NEXT:    [[T2:%.*]] = load ptr, ptr [[T]], align 8
 ; CHECK-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[T2]], i64 -8
 ; CHECK-NEXT:    store i8 [[ARG1:%.*]], ptr [[T4]], align 8
@@ -1439,14 +1439,14 @@ define ptr @gep_of_gep_multiuse_var_and_var(ptr %p, i64 %idx, i64 %idx2) {
 
 define ptr @const_gep_global_di_i8_smaller() {
 ; CHECK-LABEL: @const_gep_global_di_i8_smaller(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g_i32_di, i64 3)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g_i32_di, i64 3)
 ;
   ret ptr getelementptr (i8, ptr @g_i32_di, i64 3)
 }
 
 define ptr @const_gep_global_di_i8_exact() {
 ; CHECK-LABEL: @const_gep_global_di_i8_exact(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g_i32_di, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g_i32_di, i64 4)
 ;
   ret ptr getelementptr (i8, ptr @g_i32_di, i64 4)
 }
@@ -1467,14 +1467,14 @@ define ptr @const_gep_global_di_i64_larger() {
 
 define ptr @const_gep_global_e_smaller() {
 ; CHECK-LABEL: @const_gep_global_e_smaller(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g_i32_e, i64 3)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g_i32_e, i64 3)
 ;
   ret ptr getelementptr (i8, ptr @g_i32_e, i64 3)
 }
 
 define ptr @const_gep_global_e_exact() {
 ; CHECK-LABEL: @const_gep_global_e_exact(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @g_i32_e, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @g_i32_e, i64 4)
 ;
   ret ptr getelementptr (i8, ptr @g_i32_e, i64 4)
 }
@@ -1517,7 +1517,7 @@ define ptr @const_gep_0xi8_global() {
 define ptr @const_gep_chain(ptr %p, i64 %a) {
 ; CHECK-LABEL: @const_gep_chain(
 ; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 [[A:%.*]]
-; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 6
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds nuw i8, ptr [[P1]], i64 6
 ; CHECK-NEXT:    ret ptr [[P4]]
 ;
   %p1 = getelementptr inbounds i8, ptr %p, i64 %a
diff --git a/llvm/test/Transforms/InstCombine/hoist-xor-by-constant-from-xor-by-value.ll b/llvm/test/Transforms/InstCombine/hoist-xor-by-constant-from-xor-by-value.ll
index 39a76ae11ceef..804843779b25e 100644
--- a/llvm/test/Transforms/InstCombine/hoist-xor-by-constant-from-xor-by-value.ll
+++ b/llvm/test/Transforms/InstCombine/hoist-xor-by-constant-from-xor-by-value.ll
@@ -94,7 +94,7 @@ entry:
 
 define i16 @constantexpr2() {
 ; CHECK-LABEL: @constantexpr2(
-; CHECK-NEXT:    [[I0:%.*]] = icmp ne ptr getelementptr inbounds (i8, ptr @global_constant3, i64 40), @global_constant4
+; CHECK-NEXT:    [[I0:%.*]] = icmp ne ptr getelementptr inbounds nuw (i8, ptr @global_constant3, i64 40), @global_constant4
 ; CHECK-NEXT:    [[I1:%.*]] = zext i1 [[I0]] to i16
 ; CHECK-NEXT:    [[I2:%.*]] = load ptr, ptr @global_constant5, align 1
 ; CHECK-NEXT:    [[I3:%.*]] = load i16, ptr [[I2]], align 1
diff --git a/llvm/test/Transforms/InstCombine/icmp-binop.ll b/llvm/test/Transforms/InstCombine/icmp-binop.ll
index 878f39bb7c9a5..356489716fff9 100644
--- a/llvm/test/Transforms/InstCombine/icmp-binop.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-binop.ll
@@ -252,3 +252,110 @@ false:
   call void @use64(i64 %v)
   ret i1 false
 }
+
+define i1 @test_icmp_sgt_and_negpow2_zero(i32 %add) {
+; CHECK-LABEL: @test_icmp_sgt_and_negpow2_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD:%.*]], 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -8
+  %cmp = icmp sgt i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_and_negpow2_one(i32 %add) {
+; CHECK-LABEL: @test_icmp_slt_and_negpow2_one(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD:%.*]], 8
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -8
+  %cmp = icmp slt i32 %and, 1
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_sgt_and_negpow2_nonzero(i32 %add) {
+; CHECK-LABEL: @test_icmp_sgt_and_negpow2_nonzero(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[AND]], -2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -8
+  %cmp = icmp sgt i32 %and, -2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_sgt_and_nonnegpow2_zero(i32 %add) {
+; CHECK-LABEL: @test_icmp_sgt_and_nonnegpow2_zero(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, 8
+  %cmp = icmp sgt i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_ult_and_negpow2_one(i32 %add) {
+; CHECK-LABEL: @test_icmp_ult_and_negpow2_one(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD:%.*]], 8
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -8
+  %cmp = icmp ult i32 %and, 1
+  ret i1 %cmp
+}
+
+define i1 @test_imply_dom_condition(i32 %add) {
+; CHECK-LABEL: @test_imply_dom_condition(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD:%.*]], 7
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 false
+;
+  %and = and i32 %add, -8
+  %cmp = icmp sgt i32 %and, 0
+  tail call void @llvm.assume(i1 %cmp)
+  %min.iters.check = icmp ult i32 %and, 8
+  ret i1 %min.iters.check
+}
+
+define i1 @test_icmp_slt_and_negpow2_c(i32 %add) {
+; CHECK-LABEL: @test_icmp_slt_and_negpow2_c(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD:%.*]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -32
+  %cmp = icmp slt i32 %and, 16
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_and_negpow2_invalid_c(i32 %add) {
+; CHECK-LABEL: @test_icmp_slt_and_negpow2_invalid_c(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD:%.*]], -32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[AND]], 48
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -32
+  %cmp = icmp slt i32 %and, 48
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_sgt_and_negpow2_c(i32 %add) {
+; CHECK-LABEL: @test_icmp_sgt_and_negpow2_c(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD:%.*]], 31
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -32
+  %cmp = icmp sgt i32 %and, 16
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_sgt_and_negpow2_invalid_c(i32 %add) {
+; CHECK-LABEL: @test_icmp_sgt_and_negpow2_invalid_c(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD:%.*]], -32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[AND]], 48
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %add, -32
+  %cmp = icmp sgt i32 %and, 48
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
index 76f8c926e9bec..1296dc6a33b5d 100644
--- a/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
@@ -7,7 +7,7 @@ declare i32 @test58_d(i64 )
 
 define i1 @test59(ptr %foo) {
 ; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[FOO:%.*]], i32 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO:%.*]], i32 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[GEP1]] to i40
 ; CHECK-NEXT:    [[USE:%.*]] = zext i40 [[TMP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
@@ -23,7 +23,7 @@ define i1 @test59(ptr %foo) {
 
 define i1 @test59_as1(ptr addrspace(1) %foo) {
 ; CHECK-LABEL: @test59_as1(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[FOO:%.*]], i16 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[FOO:%.*]], i16 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[GEP1]] to i16
 ; CHECK-NEXT:    [[USE:%.*]] = zext i16 [[TMP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
@@ -146,8 +146,8 @@ define i1 @test61_as1(ptr addrspace(1) %foo, i16 %i, i16 %j) {
 ; Negative test: GEP inbounds may cross sign boundary.
 define i1 @test62(ptr %a) {
 ; CHECK-LABEL: @test62(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 10
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -159,8 +159,8 @@ define i1 @test62(ptr %a) {
 
 define i1 @test62_as1(ptr addrspace(1) %a) {
 ; CHECK-LABEL: @test62_as1(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A:%.*]], i16 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i16 10
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A:%.*]], i16 1
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i16 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr addrspace(1) [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll
index 776716fe90873..f9b90c224d832 100644
--- a/llvm/test/Transforms/InstCombine/icmp-gep.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll
@@ -133,6 +133,54 @@ define i1 @ult_base_inbounds(ptr %x, i64 %y) {
   ret i1 %r
 }
 
+define i1 @ult_base_nusw(ptr %x, i64 %y) {
+; CHECK-LABEL: @ult_base_nusw(
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i64 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr nusw i8, ptr %x, i64 %y
+  %r = icmp ult ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @ugt_base_nuw(ptr %x, i64 %y) {
+; CHECK-LABEL: @ugt_base_nuw(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr nuw i8, ptr %x, i64 %y
+  %r = icmp ugt ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @ugt_base_nusw_nuw(ptr %x, i64 %y) {
+; CHECK-LABEL: @ugt_base_nusw_nuw(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr nusw nuw i8, ptr %x, i64 %y
+  %r = icmp ugt ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @uge_base_nuw(ptr %x, i64 %y) {
+; CHECK-LABEL: @uge_base_nuw(
+; CHECK-NEXT:    ret i1 true
+;
+  %g = getelementptr nuw i8, ptr %x, i64 %y
+  %r = icmp uge ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @uge_base_nusw_nuw(ptr %x, i64 %y) {
+; CHECK-LABEL: @uge_base_nusw_nuw(
+; CHECK-NEXT:    ret i1 true
+;
+  %g = getelementptr nusw nuw i8, ptr %x, i64 %y
+  %r = icmp uge ptr %g, %x
+  ret i1 %r
+}
+
 define i1 @ugt_base_inbounds_commute(i64 %y) {
 ; CHECK-LABEL: @ugt_base_inbounds_commute(
 ; CHECK-NEXT:    [[X:%.*]] = call ptr @getptr()
@@ -244,7 +292,7 @@ declare i32 @test58_d(i64)
 
 define i1 @test59(ptr %foo) {
 ; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[FOO:%.*]], i64 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO:%.*]], i64 8
 ; CHECK-NEXT:    [[USE:%.*]] = ptrtoint ptr [[GEP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
 ; CHECK-NEXT:    ret i1 true
@@ -259,7 +307,7 @@ define i1 @test59(ptr %foo) {
 
 define i1 @test59_as1(ptr addrspace(1) %foo) {
 ; CHECK-LABEL: @test59_as1(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[FOO:%.*]], i16 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[FOO:%.*]], i16 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[GEP1]] to i16
 ; CHECK-NEXT:    [[USE:%.*]] = zext i16 [[TMP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
@@ -285,6 +333,67 @@ define i1 @test60(ptr %foo, i64 %i, i64 %j) {
   ret i1 %cmp
 }
 
+define i1 @test60_nusw(ptr %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_nusw(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nusw i32, ptr %foo, i64 %i
+  %gep2 = getelementptr nusw i8, ptr %foo, i64 %j
+  %cmp = icmp ult ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_nusw_inbounds(ptr %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_nusw_inbounds(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nsw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nusw i32, ptr %foo, i64 %i
+  %gep2 = getelementptr inbounds i8, ptr %foo, i64 %j
+  %cmp = icmp ult ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_nuw(ptr %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_nuw(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %foo, i64 %i
+  %gep2 = getelementptr nuw i8, ptr %foo, i64 %j
+  %cmp = icmp ult ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_nusw_nuw(ptr %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_nusw_nuw(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw nsw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ult i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nusw nuw i32, ptr %foo, i64 %i
+  %gep2 = getelementptr nusw nuw i8, ptr %foo, i64 %j
+  %cmp = icmp ult ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_nusw_nuw_mix(ptr %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_nusw_nuw_mix(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr nuw i32, ptr [[FOO:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr nusw i8, ptr [[FOO]], i64 [[J:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[GEP1]], [[GEP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %foo, i64 %i
+  %gep2 = getelementptr nusw i8, ptr %foo, i64 %j
+  %cmp = icmp ult ptr %gep1, %gep2
+  ret i1 %cmp
+}
+
 define i1 @test_gep_ult_no_inbounds(ptr %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test_gep_ult_no_inbounds(
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, ptr [[FOO:%.*]], i64 [[I:%.*]]
@@ -419,7 +528,7 @@ define i1 @test60_extra_use(ptr %foo, i64 %i, i64 %j) {
 
 define i1 @test60_extra_use_const_operands_inbounds(ptr %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_extra_use_const_operands_inbounds(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[FOO:%.*]], i64 4
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO:%.*]], i64 4
 ; CHECK-NEXT:    call void @use(ptr nonnull [[GEP1]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[J:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
diff --git a/llvm/test/Transforms/InstCombine/icmp-mul.ll b/llvm/test/Transforms/InstCombine/icmp-mul.ll
index c9f9b6d809e8a..49e1e11fe6c36 100644
--- a/llvm/test/Transforms/InstCombine/icmp-mul.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-mul.ll
@@ -1464,3 +1464,107 @@ entry:
   %cmp = icmp slt i8 %mul1, %mul2
   ret i1 %cmp
 }
+
+define i1 @test_icmp_slt_mul_known_sgt(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_slt_mul_known_sgt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[Z:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul nsw i64 %x, %z
+  %mul2 = mul nsw i64 %y, %z
+  %cmp = icmp slt i64 %mul1, %mul2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_sle_mul_known_sgt(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_sle_mul_known_sgt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[Z:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul nsw i64 %x, %z
+  %mul2 = mul nsw i64 %y, %z
+  %cmp = icmp sle i64 %mul1, %mul2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_mul_known_slt(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_mul_known_slt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[Z:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul nsw i64 %x, %z
+  %mul2 = mul nsw i64 %y, %z
+  %cmp = icmp slt i64 %mul2, %mul1
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_mul_known_sgt_commuted1(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_slt_mul_known_sgt_commuted1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[Z:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul nsw i64 %z, %x
+  %mul2 = mul nsw i64 %y, %z
+  %cmp = icmp slt i64 %mul1, %mul2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_mul_known_sgt_commuted2(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_slt_mul_known_sgt_commuted2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[Z:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul nsw i64 %x, %z
+  %mul2 = mul nsw i64 %z, %y
+  %cmp = icmp slt i64 %mul1, %mul2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_mul_unknown(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_slt_mul_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Y:%.*]] = add i64 [[X:%.*]], 1
+; CHECK-NEXT:    [[MUL1:%.*]] = mul nsw i64 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul nsw i64 [[Z]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[MUL1]], [[MUL2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add i64 %x, 1
+  %mul1 = mul nsw i64 %x, %z
+  %mul2 = mul nsw i64 %z, %y
+  %cmp = icmp slt i64 %mul1, %mul2
+  ret i1 %cmp
+}
+
+define i1 @test_icmp_slt_mul_no_nsw(i64 %x, i64 %z) {
+; CHECK-LABEL: @test_icmp_slt_mul_no_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Y:%.*]] = add nsw i64 [[X:%.*]], 1
+; CHECK-NEXT:    [[MUL1:%.*]] = mul i64 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul nsw i64 [[Z]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[MUL1]], [[MUL2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %y = add nsw i64 %x, 1
+  %mul1 = mul i64 %x, %z
+  %mul2 = mul nsw i64 %z, %y
+  %cmp = icmp slt i64 %mul1, %mul2
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-or-of-select-with-zero.ll b/llvm/test/Transforms/InstCombine/icmp-or-of-select-with-zero.ll
index 75301ce5d72a7..43afd3c69f7c3 100644
--- a/llvm/test/Transforms/InstCombine/icmp-or-of-select-with-zero.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-or-of-select-with-zero.ll
@@ -293,3 +293,20 @@ define i1 @src_tv_ne_invert(i1 %c1, i8 %a, i8 %b, i8 %x, i8 %yy) {
   call void @use.i8(i8 %sel_other)
   ret i1 %r
 }
+
+; Make sure we don't crash on this case.
+
+define <4 x i1> @pr119063(<4 x i32> %x, i1 %cond) {
+; CHECK-LABEL: @pr119063(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[SEL]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <4 x i32> [[OR]], zeroinitializer
+; CHECK-NEXT:    ret <4 x i1> [[CMP]]
+;
+entry:
+  %sel = select i1 %cond, <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
+  %or = or <4 x i32> %sel, %x
+  %cmp = icmp ne <4 x i32> %or, zeroinitializer
+  ret <4 x i1> %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index c1b9752607c3d..6e1486660b24d 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -1238,8 +1238,8 @@ declare i32 @test58_d(i64)
 ; Negative test: GEP inbounds may cross sign boundary.
 define i1 @test62(ptr %a) {
 ; CHECK-LABEL: @test62(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1251,8 +1251,8 @@ define i1 @test62(ptr %a) {
 
 define i1 @test62_as1(ptr addrspace(1) %a) {
 ; CHECK-LABEL: @test62_as1(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A:%.*]], i16 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i16 10
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A:%.*]], i16 1
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i16 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr addrspace(1) [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2197,8 +2197,7 @@ define i1 @icmp_ashr_and_overshift(i8 %X) {
 
 define i1 @icmp_and_ashr_neg_and_legal(i8 %x) {
 ; CHECK-LABEL: @icmp_and_ashr_neg_and_legal(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 32
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %ashr = ashr i8 %x, 4
diff --git a/llvm/test/Transforms/InstCombine/inbounds-gep.ll b/llvm/test/Transforms/InstCombine/inbounds-gep.ll
index f2a647c135103..467e5a82f89cc 100644
--- a/llvm/test/Transforms/InstCombine/inbounds-gep.ll
+++ b/llvm/test/Transforms/InstCombine/inbounds-gep.ll
@@ -19,7 +19,7 @@ define void @call1() {
 define void @call2() {
 ; CHECK-LABEL: define void @call2() {
 ; CHECK-NEXT:    [[A:%.*]] = call dereferenceable(8) ptr @g()
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
 ; CHECK-NEXT:    call void @use(ptr nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ;
@@ -45,7 +45,7 @@ define void @call3() {
 define void @alloca() {
 ; CHECK-LABEL: define void @alloca() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
 ; CHECK-NEXT:    call void @use(ptr nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ;
@@ -70,7 +70,7 @@ define void @arg1(ptr %a) {
 define void @arg2(ptr dereferenceable(8) %a) {
 ; CHECK-LABEL: define void @arg2
 ; CHECK-SAME: (ptr dereferenceable(8) [[A:%.*]]) {
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
 ; CHECK-NEXT:    call void @use(ptr nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
index 1110bbc5403e4..53c9736da59c4 100644
--- a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
+++ b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
@@ -1,5 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=instcombine -S  < %s | FileCheck %s
+; RUN: opt -passes='instcombine<no-verify-fixpoint>' -S  < %s | FileCheck %s
+
+; In many of these tests nuw can be inferred on the sunk GEP in the exit
+; block. However, the condition is only converted into the necessary form
+; after it has been sunk and revisited already.
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
 
@@ -131,8 +135,8 @@ define ptr @test3_no_inbounds1(ptr %A, i32 %Offset) {
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[RHS:%.*]] = phi ptr [ [[RHS_NEXT:%.*]], [[BB]] ], [ [[TMP]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[LHS:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 400
-; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr inbounds i8, ptr [[RHS]], i32 4
+; CHECK-NEXT:    [[LHS:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 400
+; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr inbounds nuw i8, ptr [[RHS]], i32 4
 ; CHECK-NEXT:    [[COND:%.*]] = icmp ult ptr [[LHS]], [[RHS]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
 ; CHECK:       bb2:
@@ -160,7 +164,7 @@ define ptr @test3_no_inbounds2(ptr %A, i32 %Offset) {
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[RHS:%.*]] = phi ptr [ [[RHS_NEXT:%.*]], [[BB]] ], [ [[TMP]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[LHS:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 400
+; CHECK-NEXT:    [[LHS:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 400
 ; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr i8, ptr [[RHS]], i32 4
 ; CHECK-NEXT:    [[COND:%.*]] = icmp ult ptr [[LHS]], [[RHS]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
@@ -190,7 +194,7 @@ define ptr @test3_no_inbounds3(ptr %A, i32 %Offset) {
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[RHS:%.*]] = phi ptr [ [[RHS_NEXT:%.*]], [[BB]] ], [ [[TMP]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[LHS:%.*]] = getelementptr i8, ptr [[A]], i32 400
-; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr inbounds i8, ptr [[RHS]], i32 4
+; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr inbounds nuw i8, ptr [[RHS]], i32 4
 ; CHECK-NEXT:    [[COND:%.*]] = icmp ult ptr [[LHS]], [[RHS]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
 ; CHECK:       bb2:
@@ -376,7 +380,7 @@ define i1 @test8(ptr %in, i64 %offset) {
 ; CHECK-NEXT:    [[GEPI8:%.*]] = getelementptr inbounds i8, ptr [[CASTI8]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[LD]] to i32
 ; CHECK-NEXT:    [[PTRCAST:%.*]] = inttoptr i32 [[TMP2]] to ptr
-; CHECK-NEXT:    [[GEPI32:%.*]] = getelementptr inbounds i8, ptr [[PTRCAST]], i32 4
+; CHECK-NEXT:    [[GEPI32:%.*]] = getelementptr inbounds nuw i8, ptr [[PTRCAST]], i32 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[GEPI32]], [[GEPI8]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -393,7 +397,7 @@ entry:
 define void @test_zero_offset_cycle(ptr %arg) {
 ; CHECK-LABEL: @test_zero_offset_cycle(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i32 8
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG:%.*]], i32 8
 ; CHECK-NEXT:    [[GEP_INT:%.*]] = ptrtoint ptr [[GEP]] to i32
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
diff --git a/llvm/test/Transforms/InstCombine/intptr1.ll b/llvm/test/Transforms/InstCombine/intptr1.ll
index 2bf255341f4bb..2f57dc619dbb1 100644
--- a/llvm/test/Transforms/InstCombine/intptr1.ll
+++ b/llvm/test/Transforms/InstCombine/intptr1.ll
@@ -16,8 +16,8 @@ define void @test1(ptr %a, ptr readnone %a_end, ptr %b.i64) {
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[B_ADDR_02_PTR]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[B_ADDR_02_PTR]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_02_PTR]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
@@ -68,8 +68,8 @@ define void @test1_neg(ptr %a, ptr readnone %a_end, ptr %b.i64) {
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[A]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[A]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
@@ -122,8 +122,8 @@ define void @test2(ptr %a, ptr readnone %a_end, ptr %b.float) {
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[B_ADDR_02_PTR]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[B_ADDR_02_PTR]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_02_PTR]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
@@ -172,8 +172,8 @@ define void @test3(ptr %a, ptr readnone %a_end, ptr %b.i8p) {
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[B_ADDR_02_PTR]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[B_ADDR_02_PTR]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_02_PTR]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
@@ -221,8 +221,8 @@ define void @test4(ptr %a, ptr readnone %a_end, ptr %b.float) {
 ; CHECK-NEXT:    [[I1:%.*]] = load float, ptr [[B_ADDR_02_IN]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[B_ADDR_02_IN]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_02_IN]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/InstCombine/intptr2.ll b/llvm/test/Transforms/InstCombine/intptr2.ll
index c256171142b91..f7bdc0895b6bd 100644
--- a/llvm/test/Transforms/InstCombine/intptr2.ll
+++ b/llvm/test/Transforms/InstCombine/intptr2.ll
@@ -15,8 +15,8 @@ define void @test1(ptr %a, ptr readnone %a_end, ptr %b.i) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[B_ADDR_02_IN]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[ADD]] = getelementptr inbounds i8, ptr [[B_ADDR_02_IN]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[ADD]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_02_IN]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/InstCombine/intptr3.ll b/llvm/test/Transforms/InstCombine/intptr3.ll
index 3bb3c7dcb4e1a..03c89368cd0a4 100644
--- a/llvm/test/Transforms/InstCombine/intptr3.ll
+++ b/llvm/test/Transforms/InstCombine/intptr3.ll
@@ -15,8 +15,8 @@ define  void @test(ptr %a, ptr readnone %a_end, i64 %b) unnamed_addr  {
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[B_ADDR_FLOAT]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[L]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds i8, ptr [[B_ADDR_FLOAT]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_FLOAT]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/InstCombine/intptr4.ll b/llvm/test/Transforms/InstCombine/intptr4.ll
index 5998501d82f4f..671ed1ccc3dfb 100644
--- a/llvm/test/Transforms/InstCombine/intptr4.ll
+++ b/llvm/test/Transforms/InstCombine/intptr4.ll
@@ -22,8 +22,8 @@ define  void @test(ptr %a, ptr readnone %a_end, i64 %b, ptr %bf) unnamed_addr  {
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[B_ADDR_FLOAT]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[L]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds i8, ptr [[B_ADDR_I64_PTR]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_I64_PTR]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/InstCombine/intptr5.ll b/llvm/test/Transforms/InstCombine/intptr5.ll
index f992886208f67..fa7a9a785231b 100644
--- a/llvm/test/Transforms/InstCombine/intptr5.ll
+++ b/llvm/test/Transforms/InstCombine/intptr5.ll
@@ -15,7 +15,7 @@ define  void @test(ptr %a, ptr readnone %a_end, i64 %b, ptr %bf) unnamed_addr  {
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[B_PHI:%.*]] = phi ptr [ [[TMP0]], [[BB1]] ], [ [[BF:%.*]], [[BB2]] ]
 ; CHECK-NEXT:    switch i64 [[B]], label [[FOR_BODY:%.*]] [
-; CHECK-NEXT:    i64 1, label [[FOR_BODY]]
+; CHECK-NEXT:      i64 1, label [[FOR_BODY]]
 ; CHECK-NEXT:    ]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
@@ -24,8 +24,8 @@ define  void @test(ptr %a, ptr readnone %a_end, i64 %b, ptr %bf) unnamed_addr  {
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[B_ADDR_FLOAT]], align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[L]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], ptr [[A_ADDR_03]], align 4
-; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds i8, ptr [[B_ADDR_I64_PTR]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_03]], i64 4
+; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_I64_PTR]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_03]], i64 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult ptr [[INCDEC_PTR]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/InstCombine/intptr7.ll b/llvm/test/Transforms/InstCombine/intptr7.ll
index 18fdd208e99a0..3120f028bd59b 100644
--- a/llvm/test/Transforms/InstCombine/intptr7.ll
+++ b/llvm/test/Transforms/InstCombine/intptr7.ll
@@ -6,7 +6,7 @@ define void @matching_phi(i64 %a, ptr %b, i1 %cond) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB2:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 8
+; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 8
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[ADD_INT:%.*]] = add i64 [[A:%.*]], 1
@@ -48,7 +48,7 @@ define void @no_matching_phi(i64 %a, ptr %b, i1 %cond) {
 ; CHECK-LABEL: @no_matching_phi(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_INT:%.*]] = add i64 [[A:%.*]], 1
-; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 8
+; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 8
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[B:%.*]], label [[A:%.*]]
 ; CHECK:       A:
 ; CHECK-NEXT:    [[TMP0:%.*]] = inttoptr i64 [[ADD_INT]] to ptr
diff --git a/llvm/test/Transforms/InstCombine/known-phi-recurse.ll b/llvm/test/Transforms/InstCombine/known-phi-recurse.ll
index fd3728324b8ea..9008ee9ca8e06 100644
--- a/llvm/test/Transforms/InstCombine/known-phi-recurse.ll
+++ b/llvm/test/Transforms/InstCombine/known-phi-recurse.ll
@@ -175,7 +175,7 @@ define i32 @knownbits_phi_select_test1(ptr %p1, ptr %p2, i8 %x) {
 ; CHECK-NEXT:    [[CMP6:%.*]] = icmp eq i8 [[INDVAR1]], 34
 ; CHECK-NEXT:    [[EXT3:%.*]] = zext i1 [[CMP6]] to i32
 ; CHECK-NEXT:    [[INDVAR5_NEXT]] = xor i32 [[INDVAR5]], [[EXT3]]
-; CHECK-NEXT:    [[INDVAR3_NEXT]] = getelementptr inbounds i8, ptr [[INDVAR3]], i64 1
+; CHECK-NEXT:    [[INDVAR3_NEXT]] = getelementptr inbounds nuw i8, ptr [[INDVAR3]], i64 1
 ; CHECK-NEXT:    [[LOAD2]] = load i8, ptr [[INDVAR3_NEXT]], align 1
 ; CHECK-NEXT:    br label [[LOOP]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast-select.ll b/llvm/test/Transforms/InstCombine/load-bitcast-select.ll
index 83a34de768f2e..61afa3ba9d717 100644
--- a/llvm/test/Transforms/InstCombine/load-bitcast-select.ll
+++ b/llvm/test/Transforms/InstCombine/load-bitcast-select.ll
@@ -16,8 +16,8 @@ define void @_Z3foov() {
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[I_0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x float], ptr @a, i64 0, i64 [[TMP0]]
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [1000 x float], ptr @b, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x float], ptr @a, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x float], ptr @b, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp fast olt float [[TMP1]], [[TMP2]]
diff --git a/llvm/test/Transforms/InstCombine/load-cmp.ll b/llvm/test/Transforms/InstCombine/load-cmp.ll
index 12be81b8f815d..531258935bb82 100644
--- a/llvm/test/Transforms/InstCombine/load-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/load-cmp.ll
@@ -339,7 +339,7 @@ define i1 @test10_struct_arr_noinbounds_i64(i64 %x) {
 define i1 @pr93017(i64 %idx) {
 ; CHECK-LABEL: @pr93017(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[IDX:%.*]] to i32
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [2 x ptr], ptr @table, i32 0, i32 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [2 x ptr], ptr @table, i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[V:%.*]] = load ptr, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[V]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
diff --git a/llvm/test/Transforms/InstCombine/lshr.ll b/llvm/test/Transforms/InstCombine/lshr.ll
index 00626015d2ed7..ff358c6bc772a 100644
--- a/llvm/test/Transforms/InstCombine/lshr.ll
+++ b/llvm/test/Transforms/InstCombine/lshr.ll
@@ -1523,3 +1523,65 @@ define <2 x i8> @bool_add_lshr_vec_wrong_shift_amt(<2 x i1> %a, <2 x i1> %b) {
   %lshr = lshr <2 x i8> %add, <i8 1, i8 2>
   ret <2 x i8> %lshr
 }
+
+define i32 @lowbits_of_lshr_mul(i64 %x) {
+; CHECK-LABEL: @lowbits_of_lshr_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = mul i32 [[TMP0]], 15
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %mul = mul i64 %x, 64424509440
+  %shift = lshr i64 %mul, 32
+  %conv = trunc i64 %shift to i32
+  ret i32 %conv
+}
+
+define i32 @lowbits_of_lshr_mul_mask(i32 %x) {
+; CHECK-LABEL: @lowbits_of_lshr_mul_mask(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[X:%.*]], 1600
+; CHECK-NEXT:    [[CONV:%.*]] = and i32 [[TMP0]], 32704
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %mul = mul i32 %x, 104857600
+  %shift = lshr i32 %mul, 16
+  %conv = and i32 %shift, 32767
+  ret i32 %conv
+}
+
+; Negative tests
+
+define i32 @lowbits_of_lshr_mul_mask_multiuse(i32 %x) {
+; CHECK-LABEL: @lowbits_of_lshr_mul_mask_multiuse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 104857600
+; CHECK-NEXT:    call void @use(i32 [[MUL]])
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr exact i32 [[MUL]], 16
+; CHECK-NEXT:    [[CONV:%.*]] = and i32 [[SHIFT]], 32704
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %mul = mul i32 %x, 104857600
+  call void @use(i32 %mul)
+  %shift = lshr i32 %mul, 16
+  %conv = and i32 %shift, 32767
+  ret i32 %conv
+}
+
+define i32 @lowbits_of_lshr_mul_mask_indivisible(i32 %x) {
+; CHECK-LABEL: @lowbits_of_lshr_mul_mask_indivisible(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 25600
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr i32 [[MUL]], 16
+; CHECK-NEXT:    [[CONV:%.*]] = and i32 [[SHIFT]], 32767
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %mul = mul i32 %x, 25600
+  %shift = lshr i32 %mul, 16
+  %conv = and i32 %shift, 32767
+  ret i32 %conv
+}
diff --git a/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
index 7826611ecc165..644795f08ac22 100644
--- a/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
+++ b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
@@ -22,7 +22,7 @@ define void @_Z4testPcl(ptr %out, i64 %size) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_0]], [[SIZE:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[OUT:%.*]], i64 [[I_0]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[OUT:%.*]], i64 [[I_0]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[OUT]], i64 [[I_0]]
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[SIZE]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
diff --git a/llvm/test/Transforms/InstCombine/memccpy.ll b/llvm/test/Transforms/InstCombine/memccpy.ll
index 91a1e5cbc1fe5..f9cc3eaac54e2 100644
--- a/llvm/test/Transforms/InstCombine/memccpy.ll
+++ b/llvm/test/Transforms/InstCombine/memccpy.ll
@@ -11,7 +11,7 @@ declare ptr @memccpy(ptr, ptr, i32, i64)
 define ptr @memccpy_to_memcpy(ptr %dst) {
 ; CHECK-LABEL: @memccpy_to_memcpy(
 ; CHECK-NEXT:    store i64 8245940763182785896, ptr [[DST:%.*]], align 1
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 8
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @hello, i32 114, i64 12) ; 114 is 'r'
@@ -21,7 +21,7 @@ define ptr @memccpy_to_memcpy(ptr %dst) {
 define ptr @memccpy_to_memcpy2(ptr %dst) {
 ; CHECK-LABEL: @memccpy_to_memcpy2(
 ; CHECK-NEXT:    store i64 8245940763182785896, ptr [[DST:%.*]], align 1
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 8
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @hello, i32 114, i64 8); ; 114 is 'r'
@@ -122,7 +122,7 @@ define ptr @memccpy_to_memcpy8(ptr %dst) {
 define ptr @memccpy_to_memcpy9(ptr %dst, i64 %n) {
 ; CHECK-LABEL: @memccpy_to_memcpy9(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(12) [[DST:%.*]], ptr noundef nonnull align 1 dereferenceable(12) @StopCharAfterNulTerminator, i64 12, i1 false)
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 12
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 12
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @StopCharAfterNulTerminator, i32 120, i64 15) ; 120 is 'x'
@@ -132,7 +132,7 @@ define ptr @memccpy_to_memcpy9(ptr %dst, i64 %n) {
 define ptr @memccpy_to_memcpy10(ptr %dst, i64 %n) {
 ; CHECK-LABEL: @memccpy_to_memcpy10(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(11) [[DST:%.*]], ptr noundef nonnull align 1 dereferenceable(11) @StringWithEOF, i64 11, i1 false)
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 11
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 11
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @StringWithEOF, i32 255, i64 15)
@@ -142,7 +142,7 @@ define ptr @memccpy_to_memcpy10(ptr %dst, i64 %n) {
 define ptr @memccpy_to_memcpy11(ptr %dst, i64 %n) {
 ; CHECK-LABEL: @memccpy_to_memcpy11(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(11) [[DST:%.*]], ptr noundef nonnull align 1 dereferenceable(11) @StringWithEOF, i64 11, i1 false)
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 11
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 11
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @StringWithEOF, i32 -1, i64 15)
@@ -152,7 +152,7 @@ define ptr @memccpy_to_memcpy11(ptr %dst, i64 %n) {
 define ptr @memccpy_to_memcpy12(ptr %dst, i64 %n) {
 ; CHECK-LABEL: @memccpy_to_memcpy12(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(11) [[DST:%.*]], ptr noundef nonnull align 1 dereferenceable(11) @StringWithEOF, i64 11, i1 false)
-; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 11
+; CHECK-NEXT:    [[CALL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 11
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
   %call = call ptr @memccpy(ptr %dst, ptr @StringWithEOF, i32 1023, i64 15)
diff --git a/llvm/test/Transforms/InstCombine/memchr-2.ll b/llvm/test/Transforms/InstCombine/memchr-2.ll
index 2e85fe4ad1de0..fc6d674387065 100644
--- a/llvm/test/Transforms/InstCombine/memchr-2.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-2.ll
@@ -51,7 +51,7 @@ define ptr @fold_memchr_a12345_4_3() {
 
 define ptr @fold_memchr_a12345_3_3() {
 ; CHECK-LABEL: @fold_memchr_a12345_3_3(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ;
 
   %res = call ptr @memchr(ptr @a12345, i32 3, i64 3)
@@ -63,7 +63,7 @@ define ptr @fold_memchr_a12345_3_3() {
 
 define ptr @fold_memchr_a12345_3_9() {
 ; CHECK-LABEL: @fold_memchr_a12345_3_9(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ;
 
   %res = call ptr @memchr(ptr @a12345, i32 3, i64 9)
@@ -76,7 +76,7 @@ define ptr @fold_memchr_a12345_3_9() {
 
 define ptr @fold_memchr_a123f45_500_9() {
 ; CHECK-LABEL: @fold_memchr_a123f45_500_9(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a123f45, i64 3)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a123f45, i64 3)
 ;
 
   %res = call ptr @memchr(ptr @a123f45, i32 500, i64 9)
@@ -89,7 +89,7 @@ define ptr @fold_memchr_a123f45_500_9() {
 define ptr @fold_a12345_3_n(i64 %n) {
 ; CHECK-LABEL: @fold_a12345_3_n(
 ; CHECK-NEXT:    [[MEMCHR_CMP:%.*]] = icmp ult i64 [[N:%.*]], 3
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[MEMCHR_CMP]], ptr null, ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[MEMCHR_CMP]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ; CHECK-NEXT:    ret ptr [[RES]]
 ;
 
@@ -104,7 +104,7 @@ define ptr @fold_a12345_3_n(i64 %n) {
 define ptr @fold_a12345_259_n(i64 %n) {
 ; CHECK-LABEL: @fold_a12345_259_n(
 ; CHECK-NEXT:    [[MEMCHR_CMP:%.*]] = icmp ult i64 [[N:%.*]], 3
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[MEMCHR_CMP]], ptr null, ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[MEMCHR_CMP]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ; CHECK-NEXT:    ret ptr [[RES]]
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/memchr-4.ll b/llvm/test/Transforms/InstCombine/memchr-4.ll
index 9aec0f1dfe57c..d7f2f4a17f80f 100644
--- a/llvm/test/Transforms/InstCombine/memchr-4.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-4.ll
@@ -44,7 +44,7 @@ define ptr @call_memchr_ax_2_uimax_p2() {
 
 define ptr @fold_memchr_a12345_3_uimax_p2() {
 ; CHECK-LABEL: @fold_memchr_a12345_3_uimax_p2(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ;
 
   %res = call ptr @memchr(ptr @a12345, i32 3, i64 4294967297)
diff --git a/llvm/test/Transforms/InstCombine/memchr-6.ll b/llvm/test/Transforms/InstCombine/memchr-6.ll
index 28364a92f54df..3c55f418cd8e0 100644
--- a/llvm/test/Transforms/InstCombine/memchr-6.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-6.ll
@@ -69,7 +69,7 @@ define ptr @fold_memchr_a111122_c_n(i32 %C, i64 %N) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[N:%.*]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP4]], ptr getelementptr inbounds (i8, ptr @a111122, i64 4), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP4]], ptr getelementptr inbounds nuw (i8, ptr @a111122, i64 4), ptr null
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[N]], 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP6]], [[TMP5]]
@@ -103,7 +103,7 @@ define ptr @call_memchr_a1110111_c_4(i32 %C) {
 ; CHECK-LABEL: @call_memchr_a1110111_c_4(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a1110111, i64 3), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a1110111, i64 3), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @a1110111, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
diff --git a/llvm/test/Transforms/InstCombine/memchr-7.ll b/llvm/test/Transforms/InstCombine/memchr-7.ll
index 61f1093279f83..e5b0c1df4985d 100644
--- a/llvm/test/Transforms/InstCombine/memchr-7.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-7.ll
@@ -79,7 +79,7 @@ define ptr @memchr_no_zero_cmp2(i32 %c) {
 ; CHECK-LABEL: @memchr_no_zero_cmp2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 10
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @.str.1, i64 1), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @.str.1, i64 1), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 13
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @.str.1, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
diff --git a/llvm/test/Transforms/InstCombine/memchr-8.ll b/llvm/test/Transforms/InstCombine/memchr-8.ll
index b2ac2e6eda9a7..3ad47b09ad3a7 100644
--- a/llvm/test/Transforms/InstCombine/memchr-8.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-8.ll
@@ -15,7 +15,7 @@ declare ptr @memrchr(ptr, i32, i64)
 
 define ptr @call_a_pi32max_p1() {
 ; CHECK-LABEL: @call_a_pi32max_p1(
-; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(2147483647) getelementptr inbounds (i8, ptr @a, i64 2147483648), i32 0, i64 2147483647)
+; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(2147483647) getelementptr inbounds nuw (i8, ptr @a, i64 2147483648), i32 0, i64 2147483647)
 ; CHECK-NEXT:    ret ptr [[CHR]]
 ;
   %ptr = getelementptr <{ i8, [4294967295 x i8] }>, ptr @a, i32 0, i32 1, i32 2147483647
@@ -28,7 +28,7 @@ define ptr @call_a_pi32max_p1() {
 
 define ptr @call_a_pi32max() {
 ; CHECK-LABEL: @call_a_pi32max(
-; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(2147483647) getelementptr inbounds (i8, ptr @a, i64 2147483649), i32 0, i64 2147483647)
+; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(2147483647) getelementptr inbounds nuw (i8, ptr @a, i64 2147483649), i32 0, i64 2147483647)
 ; CHECK-NEXT:    ret ptr [[CHR]]
 ;
   %ptr = getelementptr <{ i8, [4294967295 x i8] }>, ptr @a, i32 0, i32 1, i64 2147483648
@@ -42,7 +42,7 @@ define ptr @call_a_pi32max() {
 
 define ptr @call_a_pui32max() {
 ; CHECK-LABEL: @call_a_pui32max(
-; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(4294967295) getelementptr inbounds (i8, ptr @a, i64 1), i32 0, i64 4294967295)
+; CHECK-NEXT:    [[CHR:%.*]] = tail call ptr @memrchr(ptr noundef nonnull dereferenceable(4294967295) getelementptr inbounds nuw (i8, ptr @a, i64 1), i32 0, i64 4294967295)
 ; CHECK-NEXT:    ret ptr [[CHR]]
 ;
   %ptr = getelementptr <{ i8, [4294967295 x i8] }>, ptr @a, i32 0, i32 1, i32 0
diff --git a/llvm/test/Transforms/InstCombine/memchr-9.ll b/llvm/test/Transforms/InstCombine/memchr-9.ll
index 7a5e6c3f863cf..0cb129749ea6b 100644
--- a/llvm/test/Transforms/InstCombine/memchr-9.ll
+++ b/llvm/test/Transforms/InstCombine/memchr-9.ll
@@ -24,19 +24,19 @@ define void @fold_memchr_A_pIb_cst_cst(ptr %pchr) {
 ; CHECK-NEXT:    [[PST_0_4_4:%.*]] = getelementptr i8, ptr [[PCHR]], i64 16
 ; CHECK-NEXT:    store ptr null, ptr [[PST_0_4_4]], align 8
 ; CHECK-NEXT:    [[PST_1_0_1:%.*]] = getelementptr i8, ptr [[PCHR]], i64 24
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @a, i64 1), ptr [[PST_1_0_1]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @a, i64 1), ptr [[PST_1_0_1]], align 8
 ; CHECK-NEXT:    [[PST_1_0_3:%.*]] = getelementptr i8, ptr [[PCHR]], i64 32
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @a, i64 1), ptr [[PST_1_0_3]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @a, i64 1), ptr [[PST_1_0_3]], align 8
 ; CHECK-NEXT:    [[PST_1_1_1:%.*]] = getelementptr i8, ptr [[PCHR]], i64 40
 ; CHECK-NEXT:    store ptr null, ptr [[PST_1_1_1]], align 8
 ; CHECK-NEXT:    [[PST_1_1_2:%.*]] = getelementptr i8, ptr [[PCHR]], i64 48
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @a, i64 2), ptr [[PST_1_1_2]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @a, i64 2), ptr [[PST_1_1_2]], align 8
 ; CHECK-NEXT:    [[PST_1_3_3:%.*]] = getelementptr i8, ptr [[PCHR]], i64 56
 ; CHECK-NEXT:    store ptr null, ptr [[PST_1_3_3]], align 8
 ; CHECK-NEXT:    [[PST_1_3_4:%.*]] = getelementptr i8, ptr [[PCHR]], i64 64
 ; CHECK-NEXT:    store ptr null, ptr [[PST_1_3_4]], align 8
 ; CHECK-NEXT:    [[PST_1_3_6:%.*]] = getelementptr i8, ptr [[PCHR]], i64 80
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @a, i64 6), ptr [[PST_1_3_6]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @a, i64 6), ptr [[PST_1_3_6]], align 8
 ; CHECK-NEXT:    ret void
 ;
 
@@ -110,25 +110,25 @@ define void @fold_memchr_A_pIb_cst_N(i64 %N, ptr %pchr) {
 ; CHECK-NEXT:    store ptr [[CHR_0_0_N]], ptr [[PCHR:%.*]], align 8
 ; CHECK-NEXT:    [[PST_0_1_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 8
 ; CHECK-NEXT:    [[MEMCHR_CMP1:%.*]] = icmp ult i64 [[N]], 3
-; CHECK-NEXT:    [[CHR_0_1_N:%.*]] = select i1 [[MEMCHR_CMP1]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 2)
+; CHECK-NEXT:    [[CHR_0_1_N:%.*]] = select i1 [[MEMCHR_CMP1]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_0_1_N]], ptr [[PST_0_1_N]], align 8
 ; CHECK-NEXT:    [[PST_0_4_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 16
 ; CHECK-NEXT:    store ptr null, ptr [[PST_0_4_N]], align 8
 ; CHECK-NEXT:    [[PST_1_0_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 24
 ; CHECK-NEXT:    [[MEMCHR_CMP2:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT:    [[CHR_1_0_N:%.*]] = select i1 [[MEMCHR_CMP2]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 1)
+; CHECK-NEXT:    [[CHR_1_0_N:%.*]] = select i1 [[MEMCHR_CMP2]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 1)
 ; CHECK-NEXT:    store ptr [[CHR_1_0_N]], ptr [[PST_1_0_N]], align 8
 ; CHECK-NEXT:    [[PST_1_1_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 32
 ; CHECK-NEXT:    [[MEMCHR_CMP3:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    [[CHR_1_1_N:%.*]] = select i1 [[MEMCHR_CMP3]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 2)
+; CHECK-NEXT:    [[CHR_1_1_N:%.*]] = select i1 [[MEMCHR_CMP3]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_1_1_N]], ptr [[PST_1_1_N]], align 8
 ; CHECK-NEXT:    [[PST_1_2_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 40
 ; CHECK-NEXT:    [[MEMCHR_CMP4:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT:    [[CHR_1_2_N:%.*]] = select i1 [[MEMCHR_CMP4]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 4)
+; CHECK-NEXT:    [[CHR_1_2_N:%.*]] = select i1 [[MEMCHR_CMP4]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 4)
 ; CHECK-NEXT:    store ptr [[CHR_1_2_N]], ptr [[PST_1_2_N]], align 8
 ; CHECK-NEXT:    [[PST_1_3_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 48
 ; CHECK-NEXT:    [[MEMCHR_CMP5:%.*]] = icmp ult i64 [[N]], 6
-; CHECK-NEXT:    [[CHR_1_3_N:%.*]] = select i1 [[MEMCHR_CMP5]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 6)
+; CHECK-NEXT:    [[CHR_1_3_N:%.*]] = select i1 [[MEMCHR_CMP5]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 6)
 ; CHECK-NEXT:    store ptr [[CHR_1_3_N]], ptr [[PST_1_3_N]], align 8
 ; CHECK-NEXT:    [[PST_1_4_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 56
 ; CHECK-NEXT:    store ptr null, ptr [[PST_1_4_N]], align 8
@@ -136,15 +136,15 @@ define void @fold_memchr_A_pIb_cst_N(i64 %N, ptr %pchr) {
 ; CHECK-NEXT:    store ptr null, ptr [[PST_2_0_N]], align 8
 ; CHECK-NEXT:    [[PST_2_1_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 72
 ; CHECK-NEXT:    [[MEMCHR_CMP6:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT:    [[CHR_2_1_N:%.*]] = select i1 [[MEMCHR_CMP6]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 2)
+; CHECK-NEXT:    [[CHR_2_1_N:%.*]] = select i1 [[MEMCHR_CMP6]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_2_1_N]], ptr [[PST_2_1_N]], align 8
 ; CHECK-NEXT:    [[PST_2_2_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 80
 ; CHECK-NEXT:    [[MEMCHR_CMP7:%.*]] = icmp ult i64 [[N]], 3
-; CHECK-NEXT:    [[CHR_2_2_N:%.*]] = select i1 [[MEMCHR_CMP7]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 4)
+; CHECK-NEXT:    [[CHR_2_2_N:%.*]] = select i1 [[MEMCHR_CMP7]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 4)
 ; CHECK-NEXT:    store ptr [[CHR_2_2_N]], ptr [[PST_2_2_N]], align 8
 ; CHECK-NEXT:    [[PST_2_3_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 88
 ; CHECK-NEXT:    [[MEMCHR_CMP8:%.*]] = icmp ult i64 [[N]], 5
-; CHECK-NEXT:    [[CHR_2_3_N:%.*]] = select i1 [[MEMCHR_CMP8]], ptr null, ptr getelementptr inbounds (i8, ptr @a, i64 6)
+; CHECK-NEXT:    [[CHR_2_3_N:%.*]] = select i1 [[MEMCHR_CMP8]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a, i64 6)
 ; CHECK-NEXT:    store ptr [[CHR_2_3_N]], ptr [[PST_2_3_N]], align 8
 ; CHECK-NEXT:    [[PST_2_4_N:%.*]] = getelementptr i8, ptr [[PCHR]], i64 96
 ; CHECK-NEXT:    store ptr null, ptr [[PST_2_4_N]], align 8
@@ -230,13 +230,13 @@ define void @fold_memchr_A_pIb_cst_N(i64 %N, ptr %pchr) {
 
 define void @call_memchr_A_pIb_xs_cst(ptr %pchr) {
 ; CHECK-LABEL: @call_memchr_A_pIb_xs_cst(
-; CHECK-NEXT:    [[CHR_1_0_0_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds (i8, ptr @a, i64 8), i32 0, i64 2)
+; CHECK-NEXT:    [[CHR_1_0_0_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @a, i64 8), i32 0, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_1_0_0_2]], ptr [[PCHR:%.*]], align 8
 ; CHECK-NEXT:    [[PST_1_0_1_2:%.*]] = getelementptr i8, ptr [[PCHR]], i64 8
-; CHECK-NEXT:    [[CHR_1_0_1_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds (i8, ptr @a, i64 8), i32 0, i64 2)
+; CHECK-NEXT:    [[CHR_1_0_1_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @a, i64 8), i32 0, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_1_0_1_2]], ptr [[PST_1_0_1_2]], align 8
 ; CHECK-NEXT:    [[PST_0_0_8_2:%.*]] = getelementptr i8, ptr [[PCHR]], i64 16
-; CHECK-NEXT:    [[CHR_0_0_8_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds (i8, ptr @a, i64 8), i32 0, i64 2)
+; CHECK-NEXT:    [[CHR_0_0_8_2:%.*]] = call ptr @memchr(ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @a, i64 8), i32 0, i64 2)
 ; CHECK-NEXT:    store ptr [[CHR_0_0_8_2]], ptr [[PST_0_0_8_2]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -276,7 +276,7 @@ define void @call_memchr_A_pIb_xs_cst(ptr %pchr) {
 
 define ptr @fold_memchr_gep_gep_gep() {
 ; CHECK-LABEL: @fold_memchr_gep_gep_gep(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @ai64, i64 14)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @ai64, i64 14)
 ;
 
   %p8_1 = getelementptr [2 x i64], ptr @ai64, i64 0, i64 1
@@ -297,10 +297,10 @@ define ptr @fold_memchr_gep_gep_gep() {
 
 define ptr @fold_memchr_union_member() {
 ; BE-CHECK-LABEL: @fold_memchr_union_member(
-; BE-CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @u, i64 5)
+; BE-CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @u, i64 5)
 ;
 ; LE-CHECK-LABEL: @fold_memchr_union_member(
-; LE-CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @u, i64 4)
+; LE-CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @u, i64 4)
 ;
   %pi8u_p1 = getelementptr i8, ptr @u, i64 1
   %pc = call ptr @memchr(ptr %pi8u_p1, i32 34, i64 8)
diff --git a/llvm/test/Transforms/InstCombine/memchr.ll b/llvm/test/Transforms/InstCombine/memchr.ll
index 0cf85a2a1b551..89509b079d849 100644
--- a/llvm/test/Transforms/InstCombine/memchr.ll
+++ b/llvm/test/Transforms/InstCombine/memchr.ll
@@ -17,7 +17,7 @@ declare ptr @memchr(ptr, i32, i32)
 
 define void @test1() {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
   %dst = call ptr @memchr(ptr @hello, i32 119, i32 14)
@@ -37,7 +37,7 @@ define void @test2() {
 
 define void @test3() {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
   %dst = call ptr @memchr(ptr @hello, i32 0, i32 14)
@@ -58,7 +58,7 @@ define void @test4(i32 %chr) {
 
 define void @test5() {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
   %dst = call ptr @memchr(ptr @hello, i32 65280, i32 14)
@@ -68,7 +68,7 @@ define void @test5() {
 
 define void @test6() {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 ; Overflow, but we still find the right thing.
@@ -90,7 +90,7 @@ define void @test7() {
 
 define void @test8() {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hellonull, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hellonull, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
   %dst = call ptr @memchr(ptr @hellonull, i32 119, i32 14)
@@ -100,7 +100,7 @@ define void @test8() {
 
 define void @test9() {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hellonull, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hellonull, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
   %str = getelementptr [14 x i8], ptr @hellonull, i32 0, i32 2
diff --git a/llvm/test/Transforms/InstCombine/memcmp-8.ll b/llvm/test/Transforms/InstCombine/memcmp-8.ll
index 2bc1efad5c77a..031d2c4532d1d 100644
--- a/llvm/test/Transforms/InstCombine/memcmp-8.ll
+++ b/llvm/test/Transforms/InstCombine/memcmp-8.ll
@@ -42,7 +42,7 @@ define i32 @fold_memcmp_a5pi_a5p5_n(i32 %i, i64 %n) {
 ; CHECK-LABEL: @fold_memcmp_a5pi_a5p5_n(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I:%.*]] to i64
 ; CHECK-NEXT:    [[PA5_PI:%.*]] = getelementptr [5 x i8], ptr @a5, i64 0, i64 [[TMP1]]
-; CHECK-NEXT:    [[CMP:%.*]] = call i32 @memcmp(ptr [[PA5_PI]], ptr nonnull getelementptr inbounds (i8, ptr @a5, i64 5), i64 [[N:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = call i32 @memcmp(ptr [[PA5_PI]], ptr nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i32 [[CMP]]
 ;
   %pa5_pi = getelementptr [5 x i8], ptr @a5, i32 0, i32 %i
diff --git a/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
index d6624010acb21..f931b41eb0f71 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
@@ -6,7 +6,7 @@
 define void @test_load(ptr addrspace(1) %out, i64 %x) {
 ; CHECK-LABEL: @test_load(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr addrspace(2) @test.data, i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i32], ptr addrspace(2) @test.data, i64 0, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
@@ -45,7 +45,7 @@ entry:
 define void @test_load_bitcast_chain(ptr addrspace(1) %out, i64 %x) {
 ; CHECK-LABEL: @test_load_bitcast_chain(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr addrspace(2) @test.data, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(2) @test.data, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
@@ -66,7 +66,7 @@ define void @test_call(ptr addrspace(1) %out, i64 %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DATA:%.*]] = alloca [8 x i32], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p2.i64(ptr noundef nonnull align 4 dereferenceable(32) [[DATA]], ptr addrspace(2) noundef align 4 dereferenceable(32) @test.data, i64 32, i1 false)
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @foo(ptr nonnull [[ARRAYIDX]])
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
@@ -87,8 +87,8 @@ define void @test_call_no_null_opt(ptr addrspace(1) %out, i64 %x) #0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DATA:%.*]] = alloca [8 x i32], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p2.i64(ptr noundef nonnull align 4 dereferenceable(32) [[DATA]], ptr addrspace(2) noundef align 4 dereferenceable(32) @test.data, i64 32, i1 false)
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @foo(ptr [[ARRAYIDX]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @foo(ptr nonnull [[ARRAYIDX]])
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret void
@@ -108,7 +108,7 @@ define void @test_load_and_call(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DATA:%.*]] = alloca [8 x i32], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p2.i64(ptr noundef nonnull align 4 dereferenceable(32) [[DATA]], ptr addrspace(2) noundef align 4 dereferenceable(32) @test.data, i64 32, i1 false)
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
@@ -135,11 +135,11 @@ define void @test_load_and_call_no_null_opt(ptr addrspace(1) %out, i64 %x, i64 %
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DATA:%.*]] = alloca [8 x i32], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p2.i64(ptr noundef nonnull align 4 dereferenceable(32) [[DATA]], ptr addrspace(2) noundef align 4 dereferenceable(32) @test.data, i64 32, i1 false)
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i32], ptr [[DATA]], i64 0, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @foo(ptr [[ARRAYIDX]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @foo(ptr nonnull [[ARRAYIDX]])
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[Y:%.*]]
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(1) [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
index 34e6c601f494a..d894266ccaab5 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -220,7 +220,7 @@ define void @test7() {
 define void @test8() {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:    [[AL:%.*]] = alloca [[U:%.*]], align 16
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(20) [[AL]], ptr noundef nonnull align 4 dereferenceable(20) getelementptr inbounds (i8, ptr @H, i64 20), i64 20, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(20) [[AL]], ptr noundef nonnull align 4 dereferenceable(20) getelementptr inbounds nuw (i8, ptr @H, i64 20), i64 20, i1 false)
 ; CHECK-NEXT:    call void @bar(ptr nonnull [[AL]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
@@ -234,7 +234,7 @@ define void @test8() {
 define void @test8_addrspacecast() {
 ; CHECK-LABEL: @test8_addrspacecast(
 ; CHECK-NEXT:    [[AL:%.*]] = alloca [[U:%.*]], align 16
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr noundef nonnull align 16 dereferenceable(20) [[AL]], ptr addrspace(1) noundef align 4 dereferenceable(20) addrspacecast (ptr getelementptr inbounds (i8, ptr @H, i64 20) to ptr addrspace(1)), i64 20, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr noundef nonnull align 16 dereferenceable(20) [[AL]], ptr addrspace(1) noundef align 4 dereferenceable(20) addrspacecast (ptr getelementptr inbounds nuw (i8, ptr @H, i64 20) to ptr addrspace(1)), i64 20, i1 false)
 ; CHECK-NEXT:    call void @bar(ptr nonnull [[AL]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
@@ -246,7 +246,7 @@ define void @test8_addrspacecast() {
 
 define void @test9() {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    call void @bar(ptr nonnull getelementptr inbounds (i8, ptr @H, i64 20)) #[[ATTR3]]
+; CHECK-NEXT:    call void @bar(ptr nonnull getelementptr inbounds nuw (i8, ptr @H, i64 20)) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca %U, align 4
@@ -257,7 +257,7 @@ define void @test9() {
 
 define void @test9_addrspacecast() {
 ; CHECK-LABEL: @test9_addrspacecast(
-; CHECK-NEXT:    call void @bar(ptr nonnull getelementptr inbounds (i8, ptr @H, i64 20)) #[[ATTR3]]
+; CHECK-NEXT:    call void @bar(ptr nonnull getelementptr inbounds nuw (i8, ptr @H, i64 20)) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca %U, align 4
@@ -322,7 +322,7 @@ define float @test11_volatile(i64 %i) {
 ; CHECK-NEXT:    [[A:%.*]] = alloca [4 x float], align 4
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[A]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr align 4 [[A]], ptr addrspace(1) align 4 @I, i64 16, i1 true)
-; CHECK-NEXT:    [[G:%.*]] = getelementptr inbounds [4 x float], ptr [[A]], i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[G:%.*]] = getelementptr inbounds nuw [4 x float], ptr [[A]], i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[G]], align 4
 ; CHECK-NEXT:    ret float [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/memcpy_alloca.ll b/llvm/test/Transforms/InstCombine/memcpy_alloca.ll
index e58e357684e18..b86066c2776e8 100644
--- a/llvm/test/Transforms/InstCombine/memcpy_alloca.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy_alloca.ll
@@ -59,7 +59,7 @@ define void @test5(ptr %dest) {
 define void @test6(ptr %dest) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [7 x i8], align 1
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 2
 ; CHECK-NEXT:    store i16 42, ptr [[P2]], align 2
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(7) [[DEST:%.*]], ptr noundef nonnull align 1 dereferenceable(7) [[P2]], i64 7, i1 false)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/mempcpy.ll b/llvm/test/Transforms/InstCombine/mempcpy.ll
index 82c34f8a864ce..e277c1e14e24b 100644
--- a/llvm/test/Transforms/InstCombine/mempcpy.ll
+++ b/llvm/test/Transforms/InstCombine/mempcpy.ll
@@ -34,7 +34,7 @@ define ptr @memcpy_small_const_n(ptr %d, ptr nocapture readonly %s) {
 ; CHECK-LABEL: @memcpy_small_const_n(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[S:%.*]], align 1
 ; CHECK-NEXT:    store i64 [[TMP1]], ptr [[D:%.*]], align 1
-; CHECK-NEXT:    [[R:%.*]] = getelementptr inbounds i8, ptr [[D]], i64 8
+; CHECK-NEXT:    [[R:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 8
 ; CHECK-NEXT:    ret ptr [[R]]
 ;
   %r = tail call ptr @mempcpy(ptr %d, ptr %s, i64 8)
@@ -44,7 +44,7 @@ define ptr @memcpy_small_const_n(ptr %d, ptr nocapture readonly %s) {
 define ptr @memcpy_big_const_n(ptr %d, ptr nocapture readonly %s) {
 ; CHECK-LABEL: @memcpy_big_const_n(
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(1024) [[D:%.*]], ptr noundef nonnull align 1 dereferenceable(1024) [[S:%.*]], i64 1024, i1 false)
-; CHECK-NEXT:    [[R:%.*]] = getelementptr inbounds i8, ptr [[D]], i64 1024
+; CHECK-NEXT:    [[R:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 1024
 ; CHECK-NEXT:    ret ptr [[R]]
 ;
   %r = tail call ptr @mempcpy(ptr %d, ptr %s, i64 1024)
diff --git a/llvm/test/Transforms/InstCombine/memrchr-3.ll b/llvm/test/Transforms/InstCombine/memrchr-3.ll
index d3619432c0d83..cf3a1f3da2f66 100644
--- a/llvm/test/Transforms/InstCombine/memrchr-3.ll
+++ b/llvm/test/Transforms/InstCombine/memrchr-3.ll
@@ -98,7 +98,7 @@ define ptr @fold_memrchr_ax_c_1(i32 %C) {
 
 define ptr @fold_memrchr_a12345_5_5() {
 ; CHECK-LABEL: @fold_memrchr_a12345_5_5(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 4)
 ;
 
   %ret = call ptr @memrchr(ptr @a12345, i32 5, i64 5)
@@ -122,7 +122,7 @@ define ptr @fold_memrchr_a12345_5_4() {
 
 define ptr @fold_memrchr_a12345_4_5() {
 ; CHECK-LABEL: @fold_memrchr_a12345_4_5(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 3)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 3)
 ;
 
   %ret = call ptr @memrchr(ptr @a12345, i32 4, i64 5)
@@ -147,7 +147,7 @@ define ptr @fold_memrchr_a12345p1_1_4() {
 
 define ptr @fold_memrchr_a12345p1_2_4() {
 ; CHECK-LABEL: @fold_memrchr_a12345p1_2_4(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 1)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 1)
 ;
 
   %ptr = getelementptr [5 x i8], ptr @a12345, i32 0, i32 1
@@ -160,7 +160,7 @@ define ptr @fold_memrchr_a12345p1_2_4() {
 
 define ptr @fold_memrchr_a12345_2_5() {
 ; CHECK-LABEL: @fold_memrchr_a12345_2_5(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a12345, i64 1)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 1)
 ;
 
   %ret = call ptr @memrchr(ptr @a12345, i32 2, i64 5)
@@ -185,7 +185,7 @@ define ptr @fold_memrchr_a12345_0_n(i64 %N) {
 define ptr @fold_memrchr_a12345_3_n(i64 %n) {
 ; CHECK-LABEL: @fold_memrchr_a12345_3_n(
 ; CHECK-NEXT:    [[MEMRCHR_CMP:%.*]] = icmp ult i64 [[N:%.*]], 3
-; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CMP]], ptr null, ptr getelementptr inbounds (i8, ptr @a12345, i64 2)
+; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CMP]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 2)
 ; CHECK-NEXT:    ret ptr [[MEMRCHR_SEL]]
 ;
 
@@ -199,7 +199,7 @@ define ptr @fold_memrchr_a12345_3_n(i64 %n) {
 define ptr @fold_memrchr_a12345_5_n(i64 %n) {
 ; CHECK-LABEL: @fold_memrchr_a12345_5_n(
 ; CHECK-NEXT:    [[MEMRCHR_CMP:%.*]] = icmp ult i64 [[N:%.*]], 5
-; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CMP]], ptr null, ptr getelementptr inbounds (i8, ptr @a12345, i64 4)
+; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CMP]], ptr null, ptr getelementptr inbounds nuw (i8, ptr @a12345, i64 4)
 ; CHECK-NEXT:    ret ptr [[MEMRCHR_SEL]]
 ;
 
@@ -212,7 +212,7 @@ define ptr @fold_memrchr_a12345_5_n(i64 %n) {
 
 define ptr @fold_memrchr_a123123_3_5() {
 ; CHECK-LABEL: @fold_memrchr_a123123_3_5(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a123123, i64 2)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a123123, i64 2)
 ;
 
   %ret = call ptr @memrchr(ptr @a123123, i32 3, i64 5)
@@ -224,7 +224,7 @@ define ptr @fold_memrchr_a123123_3_5() {
 
 define ptr @fold_memrchr_a123123_3_6() {
 ; CHECK-LABEL: @fold_memrchr_a123123_3_6(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a123123, i64 5)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a123123, i64 5)
 ;
 
   %ret = call ptr @memrchr(ptr @a123123, i32 3, i64 6)
@@ -235,7 +235,7 @@ define ptr @fold_memrchr_a123123_3_6() {
 
 define ptr @fold_memrchr_a123123_2_6() {
 ; CHECK-LABEL: @fold_memrchr_a123123_2_6(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a123123, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a123123, i64 4)
 ;
 
   %ret = call ptr @memrchr(ptr @a123123, i32 2, i64 6)
@@ -246,7 +246,7 @@ define ptr @fold_memrchr_a123123_2_6() {
 
 define ptr @fold_memrchr_a123123_1_6() {
 ; CHECK-LABEL: @fold_memrchr_a123123_1_6(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a123123, i64 3)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a123123, i64 3)
 ;
 
   %ret = call ptr @memrchr(ptr @a123123, i32 1, i64 6)
diff --git a/llvm/test/Transforms/InstCombine/memrchr-4.ll b/llvm/test/Transforms/InstCombine/memrchr-4.ll
index 708b4417a7df9..7362916e28fb9 100644
--- a/llvm/test/Transforms/InstCombine/memrchr-4.ll
+++ b/llvm/test/Transforms/InstCombine/memrchr-4.ll
@@ -16,7 +16,7 @@ define ptr @fold_memrchr_a11111_c_5(i32 %C) {
 ; CHECK-LABEL: @fold_memrchr_a11111_c_5(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 1
-; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a11111, i64 4), ptr null
+; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a11111, i64 4), ptr null
 ; CHECK-NEXT:    ret ptr [[MEMRCHR_SEL]]
 ;
 
@@ -51,7 +51,7 @@ define ptr @fold_memrchr_a1110111_c_3(i32 %C) {
 ; CHECK-LABEL: @fold_memrchr_a1110111_c_3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 1
-; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a1110111, i64 2), ptr null
+; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a1110111, i64 2), ptr null
 ; CHECK-NEXT:    ret ptr [[MEMRCHR_SEL]]
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/memset2.ll b/llvm/test/Transforms/InstCombine/memset2.ll
index be5faccca3c43..04a19d6c2f3d1 100644
--- a/llvm/test/Transforms/InstCombine/memset2.ll
+++ b/llvm/test/Transforms/InstCombine/memset2.ll
@@ -9,7 +9,7 @@ define i32 @test(ptr addrspace(1) nocapture %moves) {
 ; CHECK-LABEL: define i32 @test
 ; CHECK-SAME: (ptr addrspace(1) nocapture [[MOVES:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MOVES]], i64 26
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MOVES]], i64 26
 ; CHECK-NEXT:    store i64 0, ptr addrspace(1) [[GEP]], align 1
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll b/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll
index 41e3197e5a2f0..292295a5853cf 100644
--- a/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll
+++ b/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll
@@ -34,9 +34,9 @@ define void @_Z4testv() {
 ; CHECK-NEXT:    store i16 [[I4]], ptr @arr_4, align 2
 ; CHECK-NEXT:    [[I8:%.*]] = sext i16 [[I4]] to i32
 ; CHECK-NEXT:    store i32 [[I8]], ptr @arr_3, align 4
-; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr getelementptr inbounds (i8, ptr @arr_2, i64 4), align 4
-; CHECK-NEXT:    store i16 [[I4]], ptr getelementptr inbounds (i8, ptr @arr_4, i64 2), align 2
-; CHECK-NEXT:    store i32 [[I8]], ptr getelementptr inbounds (i8, ptr @arr_3, i64 4), align 4
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr getelementptr inbounds nuw (i8, ptr @arr_2, i64 4), align 4
+; CHECK-NEXT:    store i16 [[I4]], ptr getelementptr inbounds nuw (i8, ptr @arr_4, i64 2), align 2
+; CHECK-NEXT:    store i32 [[I8]], ptr getelementptr inbounds nuw (i8, ptr @arr_3, i64 4), align 4
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll
index ccdf4400b16b5..2e267958d0476 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fold.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -1358,11 +1358,10 @@ define i8 @PR14613_smax(i8 %x) {
 
 define i8 @PR46271(<2 x i8> %x) {
 ; CHECK-LABEL: @PR46271(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], <i8 poison, i8 -1>
-; CHECK-NEXT:    [[A_INV:%.*]] = icmp slt <2 x i8> [[X]], zeroinitializer
-; CHECK-NEXT:    [[NOT:%.*]] = select <2 x i1> [[A_INV]], <2 x i8> <i8 poison, i8 0>, <2 x i8> [[TMP1]]
+; CHECK-NEXT:    [[NOT:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> splat (i8 -1))
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x i8> [[NOT]], i64 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    [[R1:%.*]] = xor i8 [[R]], -1
+; CHECK-NEXT:    ret i8 [[R1]]
 ;
   %a = icmp sgt <2 x i8> %x, <i8 -1, i8 -1>
   %b = select <2 x i1> %a, <2 x i8> %x, <2 x i8> <i8 poison, i8 -1>
diff --git a/llvm/test/Transforms/InstCombine/minmax-fp.ll b/llvm/test/Transforms/InstCombine/minmax-fp.ll
index 1276b7b3e3867..4fe8cf374344e 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fp.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fp.ll
@@ -321,7 +321,7 @@ define double @fneg_fmin(double %x, double %y) {
 
 define float @maxnum_ogt_fmf_on_select(float %a, float %b) {
 ; CHECK-LABEL: @maxnum_ogt_fmf_on_select(
-; CHECK-NEXT:    [[F:%.*]] = call nnan nsz float @llvm.maxnum.f32(float [[A:%.*]], float [[B:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = call nsz float @llvm.maxnum.f32(float [[A:%.*]], float [[B:%.*]])
 ; CHECK-NEXT:    ret float [[F]]
 ;
   %cond = fcmp ogt float %a, %b
@@ -331,7 +331,7 @@ define float @maxnum_ogt_fmf_on_select(float %a, float %b) {
 
 define <2 x float> @maxnum_oge_fmf_on_select(<2 x float> %a, <2 x float> %b) {
 ; CHECK-LABEL: @maxnum_oge_fmf_on_select(
-; CHECK-NEXT:    [[F:%.*]] = call nnan ninf nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = call ninf nsz <2 x float> @llvm.maxnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %cond = fcmp oge <2 x float> %a, %b
@@ -385,7 +385,7 @@ define float @maxnum_no_nnan(float %a, float %b) {
 
 define float @minnum_olt_fmf_on_select(float %a, float %b) {
 ; CHECK-LABEL: @minnum_olt_fmf_on_select(
-; CHECK-NEXT:    [[F:%.*]] = call nnan nsz float @llvm.minnum.f32(float [[A:%.*]], float [[B:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = call nsz float @llvm.minnum.f32(float [[A:%.*]], float [[B:%.*]])
 ; CHECK-NEXT:    ret float [[F]]
 ;
   %cond = fcmp olt float %a, %b
@@ -395,7 +395,7 @@ define float @minnum_olt_fmf_on_select(float %a, float %b) {
 
 define <2 x float> @minnum_ole_fmf_on_select(<2 x float> %a, <2 x float> %b) {
 ; CHECK-LABEL: @minnum_ole_fmf_on_select(
-; CHECK-NEXT:    [[F:%.*]] = call nnan ninf nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = call ninf nsz <2 x float> @llvm.minnum.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %cond = fcmp ole <2 x float> %a, %b
diff --git a/llvm/test/Transforms/InstCombine/objsize.ll b/llvm/test/Transforms/InstCombine/objsize.ll
index 9a3391d91bab7..1c33412303c19 100644
--- a/llvm/test/Transforms/InstCombine/objsize.ll
+++ b/llvm/test/Transforms/InstCombine/objsize.ll
@@ -64,7 +64,7 @@ define i1 @baz() nounwind {
 define void @test1(ptr %q, i32 %x) nounwind noinline {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.objectsize.i32.p0(ptr getelementptr inbounds (i8, ptr @window, i32 10), i1 false, i1 false, i1 false)
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.objectsize.i32.p0(ptr getelementptr inbounds nuw (i8, ptr @window, i32 10), i1 false, i1 false, i1 false)
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], -1
 ; CHECK-NEXT:    br i1 [[TMP1]], label %"47", label %"46"
 ; CHECK:       "46":
@@ -112,7 +112,7 @@ define void @test3(i1 %c1, ptr %ptr1, ptr %ptr2, ptr %ptr3) nounwind {
 ; CHECK:       bb11:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       bb12:
-; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @__inline_memcpy_chk(ptr nonnull getelementptr inbounds (i8, ptr @array, i32 4), ptr [[PTR3:%.*]], i32 512) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @__inline_memcpy_chk(ptr nonnull getelementptr inbounds nuw (i8, ptr @array, i32 4), ptr [[PTR3:%.*]], i32 512) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    unreachable
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/opaque-ptr.ll b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
index 4e0355e61739a..bac51c82f36dd 100644
--- a/llvm/test/Transforms/InstCombine/opaque-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
@@ -1,5 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+; RUN: opt -S -passes='instcombine<no-verify-fixpoint>' < %s | FileCheck %s
+
+; In the indexed_compare test the comparison is only converted int a form
+; from which we can infer non-negativity after the GEP has already been sunk
+; and revisited.
 
 define ptr @bitcast_opaque_to_opaque(ptr %a) {
 ; CHECK-LABEL: @bitcast_opaque_to_opaque(
@@ -241,7 +245,7 @@ define ptr @geps_combinable_different_elem_type7(ptr %a, i64 %idx) {
 define ptr @geps_combinable_different_elem_type8(ptr %a, i64 %idx) {
 ; CHECK-LABEL: @geps_combinable_different_elem_type8(
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds { { i32, i32 } }, ptr [[A:%.*]], i64 [[IDX:%.*]], i32 0, i32 1
-; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds i8, ptr [[A2]], i64 4
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw i8, ptr [[A2]], i64 4
 ; CHECK-NEXT:    ret ptr [[A3]]
 ;
   %a2 = getelementptr inbounds { { i32, i32 } }, ptr %a, i64 %idx, i32 0, i32 1
@@ -292,7 +296,7 @@ define ptr @geps_combinable_scalable(ptr %a, i64 %idx) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP2]]
-; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds i8, ptr [[A2]], i64 4
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw i8, ptr [[A2]], i64 4
 ; CHECK-NEXT:    ret ptr [[A3]]
 ;
   %a2 = getelementptr inbounds <vscale x 2 x i32>, ptr %a, i64 1
@@ -305,7 +309,7 @@ define ptr @geps_combinable_scalable_vector_array(ptr %a, i64 %idx) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 5
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP2]]
-; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds i8, ptr [[A2]], i64 4
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw i8, ptr [[A2]], i64 4
 ; CHECK-NEXT:    ret ptr [[A3]]
 ;
   %a2 = getelementptr inbounds [4 x <vscale x 2 x i32>], ptr %a, i64 1
@@ -444,7 +448,7 @@ bb2:
 define ptr addrspace(1) @gep_of_addrspace_cast(ptr %ptr) {
 ; CHECK-LABEL: @gep_of_addrspace_cast(
 ; CHECK-NEXT:    [[CAST1:%.*]] = addrspacecast ptr [[PTR:%.*]] to ptr addrspace(1)
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[CAST1]], i64 4
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[CAST1]], i64 4
 ; CHECK-NEXT:    ret ptr addrspace(1) [[GEP]]
 ;
   %cast1 = addrspacecast ptr %ptr to ptr addrspace(1)
@@ -549,7 +553,7 @@ define ptr @phi_of_gep_flags_1(i1 %c, ptr %p) {
 ; CHECK:       else:
 ; CHECK-NEXT:    br label [[JOIN]]
 ; CHECK:       join:
-; CHECK-NEXT:    [[PHI:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[PHI:%.*]] = getelementptr nusw nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    ret ptr [[PHI]]
 ;
   br i1 %c, label %if, label %else
@@ -694,7 +698,7 @@ define ptr @select_of_gep(i1 %c, ptr %p) {
 define ptr @select_of_gep_flags_1(i1 %c, ptr %p) {
 ; CHECK-LABEL: @select_of_gep_flags_1(
 ; CHECK-NEXT:    [[S_V:%.*]] = select i1 [[C:%.*]], i64 4, i64 8
-; CHECK-NEXT:    [[S:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 [[S_V]]
+; CHECK-NEXT:    [[S:%.*]] = getelementptr nusw nuw i8, ptr [[P:%.*]], i64 [[S_V]]
 ; CHECK-NEXT:    ret ptr [[S]]
 ;
   %gep1 = getelementptr inbounds i32, ptr %p, i64 1
diff --git a/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll b/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
index 998169ec2fe26..5d162e5dd770b 100644
--- a/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
+++ b/llvm/test/Transforms/InstCombine/phi-equal-incoming-pointers.ll
@@ -19,7 +19,7 @@ define i32 @test_gep_and_bitcast(i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
@@ -57,7 +57,7 @@ define i32 @test_gep_and_bitcast_arg(ptr %obj, i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ:%.*]], i64 16
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ:%.*]], i64 16
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
@@ -105,7 +105,7 @@ define i32 @test_gep_and_bitcast_phi(i1 %cond, i1 %cond2, i1 %cond3) {
 ; ALL:       bb4:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND3:%.*]], i32 [[RES_PHI]], i32 1
@@ -157,7 +157,7 @@ define i32 @test_gep_i32ptr(i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 64
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 64
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
@@ -196,7 +196,7 @@ define i32 @test_gep_and_bitcast_gep_base_ptr(i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ0]], i64 32
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ0]], i64 32
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
@@ -234,7 +234,7 @@ define i32 @test_gep_and_bitcast_same_bb(i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i32, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    store i32 1, ptr [[PTR_TYPED]], align 4
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i32 [[RES_PHI]], i32 1
@@ -264,11 +264,11 @@ define i32 @test_gep_and_bitcast_same_bb_and_extra_use(i1 %cond, i1 %cond2) {
 ; INSTCOMBINE-LABEL: @test_gep_and_bitcast_same_bb_and_extra_use(
 ; INSTCOMBINE-NEXT:  entry:
 ; INSTCOMBINE-NEXT:    [[OBJ:%.*]] = call ptr @get_ptr.i8()
-; INSTCOMBINE-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; INSTCOMBINE-NEXT:    call void @foo.i32(ptr nonnull [[PTR1]])
 ; INSTCOMBINE-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
 ; INSTCOMBINE:       bb2:
-; INSTCOMBINE-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; INSTCOMBINE-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; INSTCOMBINE-NEXT:    br label [[EXIT]]
 ; INSTCOMBINE:       exit:
 ; INSTCOMBINE-NEXT:    [[PTR_TYPED:%.*]] = phi ptr [ [[PTR1]], [[ENTRY:%.*]] ], [ [[PTR2]], [[BB2]] ]
@@ -280,7 +280,7 @@ define i32 @test_gep_and_bitcast_same_bb_and_extra_use(i1 %cond, i1 %cond2) {
 ; INSTCOMBINEGVN-LABEL: @test_gep_and_bitcast_same_bb_and_extra_use(
 ; INSTCOMBINEGVN-NEXT:  entry:
 ; INSTCOMBINEGVN-NEXT:    [[OBJ:%.*]] = call ptr @get_ptr.i8()
-; INSTCOMBINEGVN-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; INSTCOMBINEGVN-NEXT:    [[PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; INSTCOMBINEGVN-NEXT:    call void @foo.i32(ptr nonnull [[PTR1]])
 ; INSTCOMBINEGVN-NEXT:    br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB2:%.*]]
 ; INSTCOMBINEGVN:       bb2:
@@ -322,7 +322,7 @@ define i8 @test_gep(i1 %cond, i1 %cond2) {
 ; ALL:       bb2:
 ; ALL-NEXT:    br label [[EXIT]]
 ; ALL:       exit:
-; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR_TYPED:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES_PHI:%.*]] = load i8, ptr [[PTR_TYPED]], align 1
 ; ALL-NEXT:    store i8 1, ptr [[PTR_TYPED]], align 1
 ; ALL-NEXT:    [[RES:%.*]] = select i1 [[COND2:%.*]], i8 [[RES_PHI]], i8 1
@@ -357,12 +357,12 @@ define i32 @test_extra_uses(i1 %cond, i1 %cond2) {
 ; ALL-NEXT:    [[OBJ:%.*]] = call ptr @get_ptr.i8()
 ; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; ALL:       bb1:
-; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES1:%.*]] = load i32, ptr [[PTR1]], align 4
 ; ALL-NEXT:    call void @foo.i32(ptr nonnull [[PTR1]])
 ; ALL-NEXT:    br label [[EXIT:%.*]]
 ; ALL:       bb2:
-; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES2:%.*]] = load i32, ptr [[PTR2]], align 4
 ; ALL-NEXT:    call void @foo.i32(ptr nonnull [[PTR2]])
 ; ALL-NEXT:    br label [[EXIT]]
@@ -451,7 +451,7 @@ define i32 @test_extra_uses_multiple_geps(i1 %cond, i1 %cond2) {
 ; ALL-NEXT:    [[OBJ:%.*]] = call ptr @get_ptr.i8()
 ; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; ALL:       bb1:
-; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES1:%.*]] = load i32, ptr [[PTR1]], align 4
 ; ALL-NEXT:    call void @foo.i32(ptr nonnull [[PTR1]])
 ; ALL-NEXT:    br label [[EXIT:%.*]]
@@ -499,12 +499,12 @@ define i8 @test_gep_extra_uses(i1 %cond, i1 %cond2) {
 ; ALL-NEXT:    [[OBJ:%.*]] = call ptr @get_ptr.i8()
 ; ALL-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; ALL:       bb1:
-; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES1:%.*]] = load i8, ptr [[PTR1]], align 1
 ; ALL-NEXT:    call void @foo.i8(ptr nonnull [[PTR1]])
 ; ALL-NEXT:    br label [[EXIT:%.*]]
 ; ALL:       bb2:
-; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i8, ptr [[OBJ]], i64 16
+; ALL-NEXT:    [[PTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[OBJ]], i64 16
 ; ALL-NEXT:    [[RES2:%.*]] = load i8, ptr [[PTR2]], align 1
 ; ALL-NEXT:    call void @foo.i8(ptr nonnull [[PTR2]])
 ; ALL-NEXT:    br label [[EXIT]]
diff --git a/llvm/test/Transforms/InstCombine/phi-timeout.ll b/llvm/test/Transforms/InstCombine/phi-timeout.ll
index f30750ef0455c..bb3717de0ca27 100644
--- a/llvm/test/Transforms/InstCombine/phi-timeout.ll
+++ b/llvm/test/Transforms/InstCombine/phi-timeout.ll
@@ -10,7 +10,7 @@ define void @timeout(ptr nocapture readonly %cinfo, ptr %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i8, ptr [[CINFO:%.*]], i32 4
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw i8, ptr [[CINFO:%.*]], i32 4
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[ARRAYIDX15]], align 2
 ; CHECK-NEXT:    [[CMP17:%.*]] = icmp eq i16 [[L]], 0
 ; CHECK-NEXT:    [[EXTRACT_T1:%.*]] = trunc i16 [[L]] to i8
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
index c3548590d7f57..e3bf5ef673819 100644
--- a/llvm/test/Transforms/InstCombine/phi.ll
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -1756,7 +1756,7 @@ define i1 @pr57488_icmp_of_phi(ptr %ptr.base, i64 %len) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[ACCUM:%.*]] = phi i1 [ [[AND:%.*]], [[LOOP]] ], [ true, [[START:%.*]] ]
 ; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP]] ], [ [[PTR_BASE]], [[START]] ]
-; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
+; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 8
 ; CHECK-NEXT:    [[VAL:%.*]] = load i64, ptr [[PTR]], align 8
 ; CHECK-NEXT:    [[VAL_ZERO:%.*]] = icmp eq i64 [[VAL]], 0
 ; CHECK-NEXT:    [[AND]] = and i1 [[ACCUM]], [[VAL_ZERO]]
@@ -1895,7 +1895,7 @@ define void @simplify_context_instr(ptr %ptr.base, i64 %n) {
 ; CHECK-NEXT:    call void @use(i32 [[SEL]])
 ; CHECK-NEXT:    br label [[LATCH]]
 ; CHECK:       latch:
-; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
+; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    [[CMP_I_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]
 ; CHECK-NEXT:    br i1 [[CMP_I_NOT]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
@@ -2097,7 +2097,7 @@ define i1 @cmp_eq_phi_node_can_fold_1(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2129,12 +2129,12 @@ define i1 @cmp_eq_phi_node_can_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[SUB_IS_ZERO1:%.*]], label [[JOIN]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i8 [[TMP7]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2174,7 +2174,7 @@ define i1 @cmp_eq_phi_node_can_fold_3(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2207,12 +2207,12 @@ define i1 @cmp_eq_phi_node_can_fold_4(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[JOIN]], label [[SUB_IS_ZERO1:%.*]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i8 [[TMP7]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2252,7 +2252,7 @@ define i1 @cmp_ne_phi_node_can_fold_1(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2284,12 +2284,12 @@ define i1 @cmp_ne_phi_node_can_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[SUB_IS_ZERO1:%.*]], label [[JOIN]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i8 [[TMP7]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2329,7 +2329,7 @@ define i1 @cmp_ne_phi_node_can_fold_3(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2361,12 +2361,12 @@ define i1 @cmp_ne_phi_node_can_fold_4(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[JOIN]], label [[SUB_IS_ZERO1:%.*]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i8 [[TMP7]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2406,7 +2406,7 @@ define i1 @cmp_sgt_phi_node_can_fold_1(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2438,12 +2438,12 @@ define i1 @cmp_sgt_phi_node_can_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[JOIN]], label [[SUB_IS_ZERO1:%.*]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i8 [[TMP7]], 0
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2485,7 +2485,7 @@ define i1 @cmp_sgt_phi_node_cant_fold_1(ptr %C) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2520,14 +2520,14 @@ define i1 @cmp_sgt_phi_node_cant_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw i32 [[TMP7]], -49
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[SUB_IS_ZERO1:%.*]], label [[JOIN]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = zext i8 [[TMP11]] to i32
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2596,7 +2596,7 @@ define i1 @cmp_slt_phi_node_can_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 48
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[JOIN:%.*]], label [[SUB_IS_ZERO:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 49
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[JOIN]], label [[SUB_IS_ZERO1:%.*]]
@@ -2639,7 +2639,7 @@ define i1 @cmp_slt_phi_node_cant_fold_1(ptr %C) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
 ; CHECK-NEXT:    br label [[JOIN]]
@@ -2674,14 +2674,14 @@ define i1 @cmp_slt_phi_node_cant_fold_2(ptr %C) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[SUB_IS_ZERO:%.*]], label [[JOIN:%.*]]
 ; CHECK:       sub_is_zero:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw i32 [[TMP7]], -49
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[SUB_IS_ZERO1:%.*]], label [[JOIN]]
 ; CHECK:       sub_is_zero1:
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = zext i8 [[TMP11]] to i32
 ; CHECK-NEXT:    br label [[JOIN]]
diff --git a/llvm/test/Transforms/InstCombine/pr17827.ll b/llvm/test/Transforms/InstCombine/pr17827.ll
index 2f10bb5c7f25f..58b77ec60620e 100644
--- a/llvm/test/Transforms/InstCombine/pr17827.ll
+++ b/llvm/test/Transforms/InstCombine/pr17827.ll
@@ -5,8 +5,7 @@
 define i1 @test_shift_and_cmp_not_changed1(i8 %p) {
 ; CHECK-LABEL: @test_shift_and_cmp_not_changed1(
 ; CHECK-NEXT:    [[SHLP:%.*]] = shl i8 [[P:%.*]], 5
-; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[SHLP]], 64
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shlp = shl i8 %p, 5
@@ -18,10 +17,7 @@ define i1 @test_shift_and_cmp_not_changed1(i8 %p) {
 ; With arithmetic right shift, the comparison should not be modified.
 define i1 @test_shift_and_cmp_not_changed2(i8 %p) {
 ; CHECK-LABEL: @test_shift_and_cmp_not_changed2(
-; CHECK-NEXT:    [[SHLP:%.*]] = ashr i8 [[P:%.*]], 5
-; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %shlp = ashr i8 %p, 5
   %andp = and i8 %shlp, -64
@@ -34,8 +30,7 @@ define i1 @test_shift_and_cmp_not_changed2(i8 %p) {
 define i1 @test_shift_and_cmp_changed1(i8 %p, i8 %q) {
 ; CHECK-LABEL: @test_shift_and_cmp_changed1(
 ; CHECK-NEXT:    [[ANDP:%.*]] = shl i8 [[P:%.*]], 5
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[ANDP]], -64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 33
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %andp = and i8 %p, 6
@@ -50,8 +45,7 @@ define i1 @test_shift_and_cmp_changed1(i8 %p, i8 %q) {
 define <2 x i1> @test_shift_and_cmp_changed1_vec(<2 x i8> %p, <2 x i8> %q) {
 ; CHECK-LABEL: @test_shift_and_cmp_changed1_vec(
 ; CHECK-NEXT:    [[ANDP:%.*]] = shl <2 x i8> [[P:%.*]], splat (i8 5)
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[ANDP]], splat (i8 -64)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], splat (i8 32)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[ANDP]], splat (i8 33)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %andp = and <2 x i8> %p, <i8 6, i8 6>
@@ -91,9 +85,7 @@ define <2 x i1> @test_shift_and_cmp_changed2_vec(<2 x i8> %p) {
 ; nsw on the shift should not affect the comparison.
 define i1 @test_shift_and_cmp_changed3(i8 %p) {
 ; CHECK-LABEL: @test_shift_and_cmp_changed3(
-; CHECK-NEXT:    [[SHLP:%.*]] = shl nsw i8 [[P:%.*]], 5
-; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[P:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shlp = shl nsw i8 %p, 5
diff --git a/llvm/test/Transforms/InstCombine/pr25342.ll b/llvm/test/Transforms/InstCombine/pr25342.ll
index 271d69b141ddd..826070bc4e76b 100644
--- a/llvm/test/Transforms/InstCombine/pr25342.ll
+++ b/llvm/test/Transforms/InstCombine/pr25342.ll
@@ -17,9 +17,9 @@ define void @_Z3fooi(i32 signext %n) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr @dd, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr getelementptr inbounds (i8, ptr @dd, i64 4), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr getelementptr inbounds nuw (i8, ptr @dd, i64 4), align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr @dd2, align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr getelementptr inbounds (i8, ptr @dd2, i64 4), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr getelementptr inbounds nuw (i8, ptr @dd2, i64 4), align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[MUL4_I:%.*]] = fmul float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[SUB_I:%.*]] = fsub float [[MUL_I]], [[MUL4_I]]
@@ -32,7 +32,7 @@ define void @_Z3fooi(i32 signext %n) {
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    store float [[TMP0]], ptr @dd, align 4
-; CHECK-NEXT:    store float [[TMP1]], ptr getelementptr inbounds (i8, ptr @dd, i64 4), align 4
+; CHECK-NEXT:    store float [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @dd, i64 4), align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -84,9 +84,9 @@ define void @multi_phi(i32 signext %n) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr @dd, align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr getelementptr inbounds (i8, ptr @dd, i64 4), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr getelementptr inbounds nuw (i8, ptr @dd, i64 4), align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr @dd2, align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr getelementptr inbounds (i8, ptr @dd2, i64 4), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr getelementptr inbounds nuw (i8, ptr @dd2, i64 4), align 4
 ; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[MUL4_I:%.*]] = fmul float [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[SUB_I:%.*]] = fsub float [[MUL_I]], [[MUL4_I]]
diff --git a/llvm/test/Transforms/InstCombine/pr33453.ll b/llvm/test/Transforms/InstCombine/pr33453.ll
index 4edebdfc91dc4..c8688dae0089d 100644
--- a/llvm/test/Transforms/InstCombine/pr33453.ll
+++ b/llvm/test/Transforms/InstCombine/pr33453.ll
@@ -6,7 +6,7 @@
 
 define float @patatino() {
 ; CHECK-LABEL: @patatino(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @g2, i64 2), @g1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @g2, i64 2), @g1
 ; CHECK-NEXT:    [[FMUL:%.*]] = uitofp i1 [[CMP]] to float
 ; CHECK-NEXT:    ret float [[FMUL]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll b/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
index 4cafc4b1d475c..a9f310b17e6b6 100644
--- a/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
@@ -26,7 +26,7 @@ define <4 x i1> @PR38984_2() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @offsets, align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds (i8, ptr @a, i16 42), <4 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds nuw (i8, ptr @a, i16 42), <4 x i16> [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr null, <4 x i16> [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x ptr> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret <4 x i1> [[TMP4]]
diff --git a/llvm/test/Transforms/InstCombine/pr38984.ll b/llvm/test/Transforms/InstCombine/pr38984.ll
index 9dfc2257dfe83..19fa3346797fb 100644
--- a/llvm/test/Transforms/InstCombine/pr38984.ll
+++ b/llvm/test/Transforms/InstCombine/pr38984.ll
@@ -26,7 +26,7 @@ define <4 x i1> @PR38984_2() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @offsets, align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> <i16 undef, i16 undef, i16 undef, i16 poison>, i16 [[TMP0]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds (i8, ptr @a, i16 42), <4 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds nuw (i8, ptr @a, i16 42), <4 x i16> [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr null, <4 x i16> [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x ptr> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret <4 x i1> [[TMP4]]
diff --git a/llvm/test/Transforms/InstCombine/printf-addrspace.ll b/llvm/test/Transforms/InstCombine/printf-addrspace.ll
new file mode 100644
index 0000000000000..3cb28ce3460bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/printf-addrspace.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the printf library call simplifier works correctly.
+;
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-G200"
+
+@hello_world = addrspace(200) constant [13 x i8] c"hello world\0A\00"
+@percent_s = addrspace(200) constant [4 x i8] c"%s\0A\00"
+
+declare i32 @printf(ptr addrspace(200) , ...)
+declare i32 @puts(ptr addrspace(200))
+
+; Check printf("foo\n") -> puts("foo").
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(ptr addrspace(200) @str)
+; CHECK-NEXT:    ret void
+;
+  call i32 (ptr addrspace(200) , ...) @printf(ptr addrspace(200) @hello_world)
+  ret void
+}
+
+
+; Check printf("%s\n", str) -> puts(str).
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(ptr addrspace(200) @hello_world)
+; CHECK-NEXT:    ret void
+;
+  call i32 (ptr addrspace(200) , ...) @printf(ptr addrspace(200) @percent_s, ptr addrspace(200) @hello_world)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
index a1d10c274c465..f084fe38bb226 100644
--- a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
@@ -15,7 +15,7 @@ define i8 @remove_alloca_use_arg(i1 %cond) {
 ; CHECK:       else:
 ; CHECK-NEXT:    br label [[SINK]]
 ; CHECK:       sink:
-; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds (i8, ptr @g1, i64 2), [[IF]] ], [ getelementptr inbounds (i8, ptr @g1, i64 1), [[ELSE]] ]
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds nuw (i8, ptr @g1, i64 2), [[IF]] ], [ getelementptr inbounds nuw (i8, ptr @g1, i64 1), [[ELSE]] ]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr [[PTR1]], align 1
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
@@ -45,10 +45,10 @@ define i8 @volatile_load_keep_alloca(i1 %cond) {
 ; CHECK-NEXT:    call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef align 4 dereferenceable(32) [[ALLOCA]], ptr noundef nonnull align 16 dereferenceable(32) @g1, i64 32, i1 false)
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[VAL_IF:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ALLOCA]], i64 1
+; CHECK-NEXT:    [[VAL_IF:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ALLOCA]], i64 1
 ; CHECK-NEXT:    br label [[SINK:%.*]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[VAL_ELSE:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ALLOCA]], i64 2
+; CHECK-NEXT:    [[VAL_ELSE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ALLOCA]], i64 2
 ; CHECK-NEXT:    br label [[SINK]]
 ; CHECK:       sink:
 ; CHECK-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[VAL_IF]], [[IF]] ], [ [[VAL_ELSE]], [[ELSE]] ]
@@ -81,10 +81,10 @@ define i8 @no_memcpy_keep_alloca(i1 %cond) {
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i8], align 4, addrspace(1)
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[VAL_IF:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ALLOCA]], i64 1
+; CHECK-NEXT:    [[VAL_IF:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ALLOCA]], i64 1
 ; CHECK-NEXT:    br label [[SINK:%.*]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[VAL_ELSE:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ALLOCA]], i64 2
+; CHECK-NEXT:    [[VAL_ELSE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ALLOCA]], i64 2
 ; CHECK-NEXT:    br label [[SINK]]
 ; CHECK:       sink:
 ; CHECK-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[VAL_IF]], [[IF]] ], [ [[VAL_ELSE]], [[ELSE]] ]
@@ -114,7 +114,7 @@ define i8 @loop_phi_remove_alloca(i1 %cond) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB_0:%.*]]
 ; CHECK:       bb.0:
-; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds (i8, ptr @g1, i64 1), [[ENTRY:%.*]] ], [ getelementptr inbounds (i8, ptr @g1, i64 2), [[BB_1:%.*]] ]
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds nuw (i8, ptr @g1, i64 1), [[ENTRY:%.*]] ], [ getelementptr inbounds nuw (i8, ptr @g1, i64 2), [[BB_1:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB_1]], label [[EXIT:%.*]]
 ; CHECK:       bb.1:
 ; CHECK-NEXT:    br label [[BB_0]]
@@ -171,7 +171,7 @@ define i8 @loop_phi_late_memtransfer_remove_alloca(i1 %cond) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB_0:%.*]]
 ; CHECK:       bb.0:
-; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds (i8, ptr @g1, i64 1), [[ENTRY:%.*]] ], [ getelementptr inbounds (i8, ptr @g1, i64 2), [[BB_1:%.*]] ]
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ getelementptr inbounds nuw (i8, ptr @g1, i64 1), [[ENTRY:%.*]] ], [ getelementptr inbounds nuw (i8, ptr @g1, i64 2), [[BB_1:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB_1]], label [[EXIT:%.*]]
 ; CHECK:       bb.1:
 ; CHECK-NEXT:    br label [[BB_0]]
@@ -259,7 +259,7 @@ define i32 @addrspace_diff_keep_alloca_extra_gep(i1 %cond, ptr %x) {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr noundef nonnull align 1 dereferenceable(32) [[A]], ptr addrspace(1) noundef align 16 dereferenceable(32) @g2, i64 32, i1 false)
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 4
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
 ; CHECK-NEXT:    br label [[JOIN]]
 ; CHECK:       join:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[GEP]], [[IF]] ], [ [[X:%.*]], [[ENTRY:%.*]] ]
@@ -288,7 +288,7 @@ define i32 @addrspace_diff_remove_alloca(i1 %cond) {
 ; CHECK:       if:
 ; CHECK-NEXT:    br label [[JOIN]]
 ; CHECK:       join:
-; CHECK-NEXT:    [[PHI1:%.*]] = phi ptr addrspace(1) [ @g2, [[IF]] ], [ getelementptr inbounds (i8, ptr addrspace(1) @g2, i64 2), [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi ptr addrspace(1) [ @g2, [[IF]] ], [ getelementptr inbounds nuw (i8, ptr addrspace(1) @g2, i64 2), [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[PHI1]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -430,7 +430,7 @@ entry:
 define i8 @select_diff_addrspace_remove_alloca_asan(i1 %cond, ptr %p) sanitize_address {
 ; CHECK-LABEL: @select_diff_addrspace_remove_alloca_asan(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP2:%.*]] = select i1 [[COND:%.*]], ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @g2, i64 4), ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @g2, i64 6)
+; CHECK-NEXT:    [[GEP2:%.*]] = select i1 [[COND:%.*]], ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @g2, i64 4), ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @g2, i64 6)
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(1) [[GEP2]], align 1
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
@@ -466,8 +466,8 @@ define i8 @call_readonly_keep_alloca2() {
 ; CHECK-LABEL: @call_readonly_keep_alloca2(
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i8], align 1, addrspace(1)
 ; CHECK-NEXT:    call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef align 1 dereferenceable(16) [[ALLOCA]], ptr noundef nonnull align 16 dereferenceable(16) @g1, i64 16, i1 false)
-; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ALLOCA]], i64 16
-; CHECK-NEXT:    call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 1 dereferenceable(16) [[A1]], ptr addrspace(1) noundef align 16 dereferenceable(16) @g2, i64 16, i1 false)
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ALLOCA]], i64 16
+; CHECK-NEXT:    call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 dereferenceable(16) [[A1]], ptr addrspace(1) noundef align 16 dereferenceable(16) @g2, i64 16, i1 false)
 ; CHECK-NEXT:    [[P:%.*]] = addrspacecast ptr addrspace(1) [[ALLOCA]] to ptr
 ; CHECK-NEXT:    [[V:%.*]] = call i8 @readonly_callee(ptr [[P]])
 ; CHECK-NEXT:    ret i8 [[V]]
diff --git a/llvm/test/Transforms/InstCombine/ptrmask.ll b/llvm/test/Transforms/InstCombine/ptrmask.ll
index 54d1417ff9fd9..736fe66f0419f 100644
--- a/llvm/test/Transforms/InstCombine/ptrmask.ll
+++ b/llvm/test/Transforms/InstCombine/ptrmask.ll
@@ -411,7 +411,7 @@ define ptr @ptrmask_to_modified_gep_zero_argument() {
 define ptr @ptrmask_to_preserves_inbounds(ptr align 16 %p) {
 ; CHECK-LABEL: define ptr @ptrmask_to_preserves_inbounds
 ; CHECK-SAME: (ptr align 16 [[P:%.*]]) {
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 28
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 28
 ; CHECK-NEXT:    ret ptr [[GEP1]]
 ;
   %gep = getelementptr inbounds i8, ptr %p, i32 31
diff --git a/llvm/test/Transforms/InstCombine/range-check.ll b/llvm/test/Transforms/InstCombine/range-check.ll
index ebb310fb7c1f8..002f0224af6b0 100644
--- a/llvm/test/Transforms/InstCombine/range-check.ll
+++ b/llvm/test/Transforms/InstCombine/range-check.ll
@@ -32,6 +32,23 @@ define i1 @test_and1_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_and1_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_and1_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sge i32 %x, 0
+  %b = icmp slt i64 %x_sext, %n
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_and2(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_and2(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -60,6 +77,23 @@ define i1 @test_and2_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_and2_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_and2_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sgt i32 %x, -1
+  %b = icmp sle i64 %x_sext, %n
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_and3(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_and3(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -86,6 +120,23 @@ define i1 @test_and3_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_and3_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_and3_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sgt i64 %n, %x_sext
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_and4(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_and4(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -112,6 +163,23 @@ define i1 @test_and4_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_and4_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_and4_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sge i64 %n, %x_sext
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_or1(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_or1(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -140,6 +208,23 @@ define i1 @test_or1_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_or1_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_or1_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp slt i32 %x, 0
+  %b = icmp sge i64 %x_sext, %n
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_or2(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_or2(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -168,6 +253,23 @@ define i1 @test_or2_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_or2_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_or2_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sle i32 %x, -1
+  %b = icmp sgt i64 %x_sext, %n
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_or3(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_or3(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -194,6 +296,23 @@ define i1 @test_or3_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_or3_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_or3_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp sle i64 %n, %x_sext
+  %b = icmp slt i32 %x, 0
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
 define i1 @test_or4(i32 %x, i32 %n) {
 ; CHECK-LABEL: @test_or4(
 ; CHECK-NEXT:    [[NN:%.*]] = and i32 [[N:%.*]], 2147483647
@@ -220,6 +339,23 @@ define i1 @test_or4_logical(i32 %x, i32 %n) {
   ret i1 %c
 }
 
+define i1 @test_or4_sext(i32 %x, i64 %n) {
+; CHECK-LABEL: @test_or4_sext(
+; CHECK-NEXT:    [[N_NOT_NEGATIVE:%.*]] = icmp sgt i64 [[NN:%.*]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[N_NOT_NEGATIVE]])
+; CHECK-NEXT:    [[X_SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[NN]], [[X_SEXT]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %n_not_negative = icmp sge i64 %n, 0
+  call void @llvm.assume(i1 %n_not_negative)
+  %x_sext = sext i32 %x to i64
+  %a = icmp slt i64 %n, %x_sext
+  %b = icmp slt i32 %x, 0
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
 ; Negative tests
 
 define i1 @negative1(i32 %x, i32 %n) {
diff --git a/llvm/test/Transforms/InstCombine/rem.ll b/llvm/test/Transforms/InstCombine/rem.ll
index c1d8bb2b02b80..198a7062dca06 100644
--- a/llvm/test/Transforms/InstCombine/rem.ll
+++ b/llvm/test/Transforms/InstCombine/rem.ll
@@ -522,7 +522,7 @@ define i32 @pr27968_0(i1 %c0, ptr %p) {
 ; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @a, i64 8), @b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @a, i64 8), @b
 ; CHECK-NEXT:    br i1 [[CMP]], label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
 ; CHECK:       rem.is.safe:
 ; CHECK-NEXT:    ret i32 0
@@ -593,7 +593,7 @@ define i32 @pr27968_2(i1 %c0, ptr %p) {
 ; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @a, i64 8), @b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @a, i64 8), @b
 ; CHECK-NEXT:    br i1 [[CMP]], label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
 ; CHECK:       rem.is.safe:
 ; CHECK-NEXT:    ret i32 0
diff --git a/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll b/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
index 56580a8b68c68..3c10378d17220 100644
--- a/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
+++ b/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
@@ -67,7 +67,7 @@ define double @test_nnan_flag_enabled(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul nnan double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -101,7 +101,7 @@ define double @test_ninf_flag_enabled(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul ninf double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -135,7 +135,7 @@ define double @test_nsz_flag_enabled(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul nsz double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -169,7 +169,7 @@ define double @test_phi_initalise_to_non_zero(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 1.000000e+00, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul fast double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -205,7 +205,7 @@ define double @test_multiple_phi_operands(ptr %arr_d, i1 %entry_cond) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[ENTRY_2]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ 0.000000e+00, [[ENTRY_2]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul fast double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -244,7 +244,7 @@ define double @test_multiple_phi_operands_with_non_zero(ptr %arr_d, i1 %entry_co
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[ENTRY_2]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi double [ 1.000000e+00, [[ENTRY]] ], [ 0.000000e+00, [[ENTRY_2]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul fast double [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -281,7 +281,7 @@ define i32 @test_int_phi_operands(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -315,7 +315,7 @@ define i32 @test_int_phi_operands_initalise_to_non_zero(ptr %arr_d) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul i32 [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -351,7 +351,7 @@ define i32 @test_multiple_int_phi_operands(ptr %arr_d, i1 %entry_cond) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[ENTRY_2]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ 0, [[ENTRY_2]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul i32 [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
@@ -390,7 +390,7 @@ define i32 @test_multiple_int_phi_operands_initalise_to_non_zero(ptr %arr_d, i1
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[ENTRY_2]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_PROD_01:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ 1, [[ENTRY_2]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul i32 [[F_PROD_01]], [[TMP0]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
index 9236d96f59a55..cfd679c0cc592 100644
--- a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -1793,10 +1793,9 @@ define i32 @not_uadd_sat(i32 %x, i32 %y) {
 
 define i32 @not_uadd_sat2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @not_uadd_sat2(
-; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], -2
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], 1
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 -1
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.umax.i32(i32 [[X1:%.*]], i32 1)
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X]], -2
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %a = add i32 %x, -2
   %c = icmp ugt i32 %x, 1
@@ -2316,3 +2315,39 @@ define i32 @uadd_sat_via_add_swapped_cmp_select_nonstrict(i32 %x, i32 %y) {
   %r = select i1 %c, i32 %a, i32 -1
   ret i32 %r
 }
+
+define i8 @fold_add_umax_to_usub(i8 %a) {
+; CHECK-LABEL: @fold_add_umax_to_usub(
+; CHECK-NEXT:    [[SEL:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %umax = call i8 @llvm.umax.i8(i8 %a, i8 10)
+  %sel = add i8 %umax, -10
+  ret i8 %sel
+}
+
+define i8 @fold_add_umax_to_usub_incorrect_rhs(i8 %a) {
+; CHECK-LABEL: @fold_add_umax_to_usub_incorrect_rhs(
+; CHECK-NEXT:    [[UMAX:%.*]] = call i8 @llvm.umax.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[SEL:%.*]] = add i8 [[UMAX]], -11
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %umax = call i8 @llvm.umax.i8(i8 %a, i8 10)
+  %sel = add i8 %umax, -11
+  ret i8 %sel
+}
+
+define i8 @fold_add_umax_to_usub_multiuse(i8 %a) {
+; CHECK-LABEL: @fold_add_umax_to_usub_multiuse(
+; CHECK-NEXT:    [[UMAX:%.*]] = call i8 @llvm.umax.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    call void @usei8(i8 [[UMAX]])
+; CHECK-NEXT:    [[SEL:%.*]] = add i8 [[UMAX]], -10
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %umax = call i8 @llvm.umax.i8(i8 %a, i8 10)
+  call void @usei8(i8 %umax)
+  %sel = add i8 %umax, -10
+  ret i8 %sel
+}
+
+declare void @usei8(i8)
diff --git a/llvm/test/Transforms/InstSimplify/select-and-cmp.ll b/llvm/test/Transforms/InstCombine/select-and-cmp.ll
similarity index 82%
rename from llvm/test/Transforms/InstSimplify/select-and-cmp.ll
rename to llvm/test/Transforms/InstCombine/select-and-cmp.ll
index 41a4ab96bd62c..50e1493cad5c7 100644
--- a/llvm/test/Transforms/InstSimplify/select-and-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/select-and-cmp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i32 @select_and_icmp(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp(
@@ -100,7 +100,6 @@ define <2 x i8> @select_and_icmp_alt_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z)
   ret <2 x i8> %D
 }
 
-
 define i32 @select_and_icmp_inv(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_inv(
 ; CHECK-NEXT:    ret i32 [[X:%.*]]
@@ -112,14 +111,12 @@ define i32 @select_and_icmp_inv(i32 %x, i32 %y, i32 %z) {
   ret i32 %D
 }
 
-; Negative tests
+
+; Below used to be negative tests in InstSimplify, but are no more negative cases here
+
 define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_1(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
-; CHECK-NEXT:    ret i32 [[D]]
+; CHECK-NEXT:    ret i32 [[X]]
 ;
   %A = icmp eq i32 %x, %z
   %B = icmp ne i32 %y, %z
@@ -130,10 +127,8 @@ define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_2(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[Z]], i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp ne i32 %x, %z
@@ -145,10 +140,8 @@ define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_3(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[X]], i32 [[Z]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp ne i32 %x, %z
@@ -160,10 +153,8 @@ define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_4(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[Z]], i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp eq i32 %x, %z
@@ -173,6 +164,60 @@ define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
   ret i32 %D
 }
 
+define i32 @select_and_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_1(
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_2(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+
+define i32 @select_and_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_3(
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_4(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+
+; Negative tests
+
 define i32 @select_and_icmp_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
 ; CHECK-LABEL: @select_and_icmp_bad_true_val(
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
@@ -233,66 +278,6 @@ define i32 @select_and_icmp_bad_op_2(i32 %x, i32 %y, i32 %z, i32 %k) {
   ret i32 %D
 }
 
-define i32 @select_and_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_and_icmp_alt_bad_1(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp eq i32 %x, %z
-  %B = icmp ne i32 %y, %z
-  %C = and i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_and_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_and_icmp_alt_bad_2(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp ne i32 %x, %z
-  %B = icmp eq i32 %y, %z
-  %C = and i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_and_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_and_icmp_alt_bad_3(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp ne i32 %x, %z
-  %B = icmp ne i32 %y, %z
-  %C = and i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_and_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_and_icmp_alt_bad_4(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp eq i32 %x, %z
-  %B = icmp eq i32 %y, %z
-  %C = or i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
 define i32 @select_and_icmp_alt_bad_5(i32 %x, i32 %y, i32 %z, i32 %k) {
 ; CHECK-LABEL: @select_and_icmp_alt_bad_5(
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[K:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/select-and-or.ll b/llvm/test/Transforms/InstCombine/select-and-or.ll
index c3fbb3fe15e19..b14186d737e98 100644
--- a/llvm/test/Transforms/InstCombine/select-and-or.ll
+++ b/llvm/test/Transforms/InstCombine/select-and-or.ll
@@ -445,7 +445,7 @@ define i1 @demorgan_select_infloop1(i1 %L) {
 define i1 @demorgan_select_infloop2(i1 %L) {
 ; CHECK-LABEL: @demorgan_select_infloop2(
 ; CHECK-NEXT:    [[NOT_L:%.*]] = xor i1 [[L:%.*]], true
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne ptr getelementptr inbounds (i8, ptr @g2, i64 2), @g1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne ptr getelementptr inbounds nuw (i8, ptr @g2, i64 2), @g1
 ; CHECK-NEXT:    [[C15:%.*]] = select i1 [[NOT_L]], i1 [[CMP2]], i1 false
 ; CHECK-NEXT:    ret i1 [[C15]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/select-cmp-br.ll b/llvm/test/Transforms/InstCombine/select-cmp-br.ll
index f1c62a5593f0f..2a8c19ece4b70 100644
--- a/llvm/test/Transforms/InstCombine/select-cmp-br.ll
+++ b/llvm/test/Transforms/InstCombine/select-cmp-br.ll
@@ -12,7 +12,7 @@ define void @test1(ptr %arg) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[M:%.*]] = load ptr, ptr [[ARG:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 16
 ; CHECK-NEXT:    [[N:%.*]] = load ptr, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP5_NOT:%.*]] = icmp eq ptr [[M]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP5_NOT]], label [[BB8:%.*]], label [[BB10:%.*]]
@@ -22,7 +22,7 @@ define void @test1(ptr %arg) {
 ; CHECK-NEXT:    tail call void @bar(ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb10:
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[M]], i64 72
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[M]], i64 72
 ; CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB]]
@@ -54,7 +54,7 @@ define void @test2(ptr %arg) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[M:%.*]] = load ptr, ptr [[ARG:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 16
 ; CHECK-NEXT:    [[N:%.*]] = load ptr, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq ptr [[M]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BB10:%.*]], label [[BB8:%.*]]
@@ -64,7 +64,7 @@ define void @test2(ptr %arg) {
 ; CHECK-NEXT:    tail call void @bar(ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb10:
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[M]], i64 72
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[M]], i64 72
 ; CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB]]
@@ -96,7 +96,7 @@ define void @test3(ptr %arg) {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[M:%.*]] = load ptr, ptr [[ARG:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 16
 ; CHECK-NEXT:    [[N:%.*]] = load ptr, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq ptr [[M]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BB8:%.*]], label [[BB10:%.*]]
@@ -106,7 +106,7 @@ define void @test3(ptr %arg) {
 ; CHECK-NEXT:    tail call void @bar(ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb10:
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[M]], i64 72
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[M]], i64 72
 ; CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB]]
@@ -138,7 +138,7 @@ define void @test4(ptr %arg) {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[M:%.*]] = load ptr, ptr [[ARG:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 16
 ; CHECK-NEXT:    [[N:%.*]] = load ptr, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq ptr [[M]], [[N]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BB10:%.*]], label [[BB8:%.*]]
@@ -148,7 +148,7 @@ define void @test4(ptr %arg) {
 ; CHECK-NEXT:    tail call void @bar(ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb10:
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[M]], i64 72
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[M]], i64 72
 ; CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](ptr nonnull [[ARG]])
 ; CHECK-NEXT:    br label [[BB]]
diff --git a/llvm/test/Transforms/InstCombine/select-gep.ll b/llvm/test/Transforms/InstCombine/select-gep.ll
index 17304801b1b34..d3c357ea9751b 100644
--- a/llvm/test/Transforms/InstCombine/select-gep.ll
+++ b/llvm/test/Transforms/InstCombine/select-gep.ll
@@ -47,7 +47,7 @@ define ptr @test1d(ptr %p, ptr %q) {
 ; CHECK-LABEL: @test1d(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt ptr [[P:%.*]], [[Q:%.*]]
 ; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], ptr [[P]], ptr [[Q]]
-; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr inbounds i8, ptr [[SELECT_V]], i64 16
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr inbounds nuw i8, ptr [[SELECT_V]], i64 16
 ; CHECK-NEXT:    ret ptr [[SELECT]]
 ;
   %gep1 = getelementptr inbounds i32, ptr %p, i64 4
@@ -130,7 +130,7 @@ define ptr @test2c(ptr %p, i64 %x, i64 %y) {
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[P:%.*]], i64 [[X:%.*]]
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp ugt i64 [[X]], [[Y:%.*]]
 ; CHECK-NEXT:    [[SEL_IDX:%.*]] = select i1 [[ICMP]], i64 0, i64 24
-; CHECK-NEXT:    [[SEL:%.*]] = getelementptr inbounds i8, ptr [[GEP1]], i64 [[SEL_IDX]]
+; CHECK-NEXT:    [[SEL:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP1]], i64 [[SEL_IDX]]
 ; CHECK-NEXT:    ret ptr [[SEL]]
 ;
   %gep1 = getelementptr inbounds i32, ptr %p, i64 %x
@@ -146,7 +146,7 @@ define ptr @test2d(ptr %p, i64 %x, i64 %y) {
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[P:%.*]], i64 [[X:%.*]]
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp ugt i64 [[X]], [[Y:%.*]]
 ; CHECK-NEXT:    [[SEL_IDX:%.*]] = select i1 [[ICMP]], i64 24, i64 0
-; CHECK-NEXT:    [[SEL:%.*]] = getelementptr inbounds i8, ptr [[GEP1]], i64 [[SEL_IDX]]
+; CHECK-NEXT:    [[SEL:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP1]], i64 [[SEL_IDX]]
 ; CHECK-NEXT:    ret ptr [[SEL]]
 ;
   %gep1 = getelementptr inbounds i32, ptr %p, i64 %x
diff --git a/llvm/test/Transforms/InstSimplify/select-or-cmp.ll b/llvm/test/Transforms/InstCombine/select-or-cmp.ll
similarity index 81%
rename from llvm/test/Transforms/InstSimplify/select-or-cmp.ll
rename to llvm/test/Transforms/InstCombine/select-or-cmp.ll
index 0e410a9645f0d..72a3747c5e405 100644
--- a/llvm/test/Transforms/InstSimplify/select-or-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/select-or-cmp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i32 @select_or_icmp(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_or_icmp(
@@ -111,13 +111,13 @@ define i32 @select_or_icmp_inv(i32 %x, i32 %y, i32 %z) {
   ret i32 %D
 }
 
-; Negative tests
+
+; Below used to be negative tests in InstSimplify, but are no more negative cases here
+
 define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_1(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[X]], i32 [[Z]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp eq i32 %x, %z
@@ -129,11 +129,7 @@ define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_2(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
-; CHECK-NEXT:    ret i32 [[D]]
+; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %A = icmp ne i32 %x, %z
   %B = icmp eq i32 %y, %z
@@ -144,10 +140,8 @@ define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_3(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[Z]], i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp eq i32 %x, %z
@@ -159,10 +153,8 @@ define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
 
 define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_and_icmp_pred_bad_4(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[X]], i32 [[Z]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %A = icmp ne i32 %x, %z
@@ -172,6 +164,59 @@ define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
   ret i32 %D
 }
 
+define i32 @select_or_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_1(
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_2(
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_3(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_4(
+; CHECK-NEXT:    [[B_NOT:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[B_NOT]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+
+; Negative tests
+
 define i32 @select_or_icmp_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
 ; CHECK-LABEL: @select_or_icmp_bad_true_val(
 ; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
@@ -217,7 +262,6 @@ define i32 @select_or_icmp_bad_op(i32 %x, i32 %y, i32 %z, i32 %k) {
   ret i32 %D
 }
 
-
 define i32 @select_or_icmp_bad_op_2(i32 %x, i32 %y, i32 %z, i32 %k) {
 ; CHECK-LABEL: @select_or_icmp_bad_op_2(
 ; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[K:%.*]]
@@ -233,66 +277,6 @@ define i32 @select_or_icmp_bad_op_2(i32 %x, i32 %y, i32 %z, i32 %k) {
   ret i32 %D
 }
 
-define i32 @select_or_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_or_icmp_alt_bad_1(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp eq i32 %x, %z
-  %B = icmp ne i32 %y, %z
-  %C = or i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_or_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_or_icmp_alt_bad_2(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp ne i32 %x, %z
-  %B = icmp eq i32 %y, %z
-  %C = or i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_or_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_or_icmp_alt_bad_3(
-; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp eq i32 %x, %z
-  %B = icmp eq i32 %y, %z
-  %C = or i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
-define i32 @select_or_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_or_icmp_alt_bad_4(
-; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
-; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[D]]
-;
-  %A = icmp ne i32 %x, %z
-  %B = icmp ne i32 %y, %z
-  %C = and i1 %A, %B
-  %D = select i1 %C, i32 %x, i32 %z
-  ret i32 %D
-}
-
 define i32 @select_or_icmp_alt_bad_5(i32 %x, i32 %y, i32 %z, i32 %k) {
 ; CHECK-LABEL: @select_or_icmp_alt_bad_5(
 ; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[K:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/select-value-equivalence.ll b/llvm/test/Transforms/InstCombine/select-value-equivalence.ll
index d55766ddf4040..812b4d8f6be79 100644
--- a/llvm/test/Transforms/InstCombine/select-value-equivalence.ll
+++ b/llvm/test/Transforms/InstCombine/select-value-equivalence.ll
@@ -322,12 +322,11 @@ define <2 x i8> @select_vec_op_const_no_undef(<2 x i8> %x) {
   ret <2 x i8> %xr
 }
 
-; FIXME: This is a miscompile.
 define <2 x i8> @select_vec_op_const_undef(<2 x i8> %x) {
 ; CHECK-LABEL: define <2 x i8> @select_vec_op_const_undef(
 ; CHECK-SAME: <2 x i8> [[X:%.*]]) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq <2 x i8> [[X]], <i8 1, i8 undef>
-; CHECK-NEXT:    [[XR:%.*]] = select <2 x i1> [[XZ]], <2 x i8> <i8 1, i8 undef>, <2 x i8> <i8 4, i8 3>
+; CHECK-NEXT:    [[XR:%.*]] = select <2 x i1> [[XZ]], <2 x i8> [[X]], <2 x i8> <i8 4, i8 3>
 ; CHECK-NEXT:    ret <2 x i8> [[XR]]
 ;
   %xz = icmp eq <2 x i8> %x, <i8 1, i8 undef>
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index a3221d7388b8f..82c079d681284 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -2989,10 +2989,9 @@ define i8 @select_replacement_loop3(i32 noundef %x) {
 
 define i16 @select_replacement_loop4(i16 noundef %p_12) {
 ; CHECK-LABEL: @select_replacement_loop4(
-; CHECK-NEXT:    [[AND1:%.*]] = and i16 [[P_12:%.*]], 1
-; CHECK-NEXT:    [[CMP21:%.*]] = icmp ult i16 [[P_12]], 2
-; CHECK-NEXT:    [[AND3:%.*]] = select i1 [[CMP21]], i16 [[AND1]], i16 0
-; CHECK-NEXT:    ret i16 [[AND3]]
+; CHECK-NEXT:    [[P_12:%.*]] = call i16 @llvm.umin.i16(i16 [[P_13:%.*]], i16 2)
+; CHECK-NEXT:    [[AND1:%.*]] = and i16 [[P_12]], 1
+; CHECK-NEXT:    ret i16 [[AND1]]
 ;
   %cmp1 = icmp ult i16 %p_12, 2
   %and1 = and i16 %p_12, 1
diff --git a/llvm/test/Transforms/InstCombine/shl-twice-constant.ll b/llvm/test/Transforms/InstCombine/shl-twice-constant.ll
new file mode 100644
index 0000000000000..bbdd7fa3d1c40
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shl-twice-constant.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+@c = external constant i8
+@c2 = external constant i8
+
+define i64 @testfunc() {
+; CHECK-LABEL: @testfunc(
+; CHECK-NEXT:    [[SHL1:%.*]] = shl nuw i64 1, ptrtoint (ptr @c2 to i64)
+; CHECK-NEXT:    [[SHL2:%.*]] = shl i64 [[SHL1]], ptrtoint (ptr @c to i64)
+; CHECK-NEXT:    ret i64 [[SHL2]]
+;
+  %shl1 = shl i64 1, ptrtoint (ptr @c2 to i64)
+  %shl2 = shl i64 %shl1, ptrtoint (ptr @c to i64)
+  ret i64 %shl2
+}
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-i16.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-i16.ll
index 9a08b6b5cf9f7..8bbd61ac6bc09 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-i16.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-i16.ll
@@ -29,11 +29,11 @@ define void @foo(ptr %P, ptr %X) {
 
 define ptr @test1() {
 ; CHECK32-LABEL: @test1(
-; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str, i32 2), i16 103)
+; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str, i32 2), i16 103)
 ; CHECK32-NEXT:    ret ptr [[TMP3]]
 ;
 ; CHECK16-LABEL: @test1(
-; CHECK16-NEXT:    ret ptr getelementptr inbounds (i8, ptr @str, i32 3)
+; CHECK16-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @str, i32 3)
 ;
   %tmp3 = tail call ptr @strchr( ptr getelementptr ([5 x i8], ptr @str, i32 0, i16 2), i16 103 )              ; <ptr> [#uses=1]
   ret ptr %tmp3
@@ -45,11 +45,11 @@ declare ptr @strchr(ptr, i16)
 
 define ptr @test2() {
 ; CHECK32-LABEL: @test2(
-; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str1, i32 2), i16 0)
+; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str1, i32 2), i16 0)
 ; CHECK32-NEXT:    ret ptr [[TMP3]]
 ;
 ; CHECK16-LABEL: @test2(
-; CHECK16-NEXT:    ret ptr getelementptr inbounds (i8, ptr @str1, i32 7)
+; CHECK16-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @str1, i32 7)
 ;
   %tmp3 = tail call ptr @strchr( ptr getelementptr ([8 x i8], ptr @str1, i32 0, i32 2), i16 0 )               ; <ptr> [#uses=1]
   ret ptr %tmp3
@@ -58,7 +58,7 @@ define ptr @test2() {
 define ptr @test3() {
 ; CHECK32-LABEL: @test3(
 ; CHECK32-NEXT:  entry:
-; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str2, i32 1), i16 80)
+; CHECK32-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str2, i32 1), i16 80)
 ; CHECK32-NEXT:    ret ptr [[TMP3]]
 ;
 ; CHECK16-LABEL: @test3(
@@ -143,18 +143,31 @@ declare ptr @fopen(ptr, ptr)
 declare void @exit(i16)
 
 define i32 @PR4645(i1 %c1) {
-; CHECK-LABEL: @PR4645(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[IF_THEN:%.*]]
-; CHECK:       lor.lhs.false:
-; CHECK-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    call void @exit(i16 1)
-; CHECK-NEXT:    br label [[FOR_COND]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    unreachable
-; CHECK:       for.end:
-; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK32-LABEL: @PR4645(
+; CHECK32-NEXT:  entry:
+; CHECK32-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK32:       lor.lhs.false:
+; CHECK32-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
+; CHECK32:       if.then:
+; CHECK32-NEXT:    call void @exit(i16 1)
+; CHECK32-NEXT:    br label [[FOR_COND]]
+; CHECK32:       for.cond:
+; CHECK32-NEXT:    unreachable
+; CHECK32:       for.end:
+; CHECK32-NEXT:    br label [[FOR_COND]]
+;
+; CHECK16-LABEL: @PR4645(
+; CHECK16-NEXT:  entry:
+; CHECK16-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK16:       lor.lhs.false:
+; CHECK16-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
+; CHECK16:       if.then:
+; CHECK16-NEXT:    call void @exit(i16 1) #[[ATTR6:[0-9]+]]
+; CHECK16-NEXT:    br label [[FOR_COND]]
+; CHECK16:       for.cond:
+; CHECK16-NEXT:    unreachable
+; CHECK16:       for.end:
+; CHECK16-NEXT:    br label [[FOR_COND]]
 ;
 entry:
   br label %if.then
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
index c4ae7e7e16bcf..6ce765d99931e 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -29,10 +29,10 @@ define void @foo(ptr %P, ptr %X) {
 
 define ptr @test1() {
 ; CHECK32-LABEL: @test1(
-; CHECK32-NEXT:    ret ptr getelementptr inbounds (i8, ptr @str, i32 3)
+; CHECK32-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @str, i32 3)
 ;
 ; CHECK16-LABEL: @test1(
-; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str, i32 2), i32 103)
+; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str, i32 2), i32 103)
 ; CHECK16-NEXT:    ret ptr [[TMP3]]
 ;
   %tmp3 = tail call ptr @strchr( ptr getelementptr ([5 x i8], ptr @str, i32 0, i32 2), i32 103 )              ; <ptr> [#uses=1]
@@ -45,10 +45,10 @@ declare ptr @strchr(ptr, i32)
 
 define ptr @test2() {
 ; CHECK32-LABEL: @test2(
-; CHECK32-NEXT:    ret ptr getelementptr inbounds (i8, ptr @str1, i32 7)
+; CHECK32-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @str1, i32 7)
 ;
 ; CHECK16-LABEL: @test2(
-; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str1, i32 2), i32 0)
+; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str1, i32 2), i32 0)
 ; CHECK16-NEXT:    ret ptr [[TMP3]]
 ;
   %tmp3 = tail call ptr @strchr( ptr getelementptr ([8 x i8], ptr @str1, i32 0, i32 2), i32 0 )               ; <ptr> [#uses=1]
@@ -62,7 +62,7 @@ define ptr @test3() {
 ;
 ; CHECK16-LABEL: @test3(
 ; CHECK16-NEXT:  entry:
-; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds (i8, ptr @str2, i32 1), i32 80)
+; CHECK16-NEXT:    [[TMP3:%.*]] = tail call ptr @strchr(ptr nonnull getelementptr inbounds nuw (i8, ptr @str2, i32 1), i32 80)
 ; CHECK16-NEXT:    ret ptr [[TMP3]]
 ;
 entry:
@@ -154,18 +154,31 @@ declare ptr @fopen(ptr, ptr)
 declare void @exit(i32)
 
 define i32 @PR4645(i1 %c1) {
-; CHECK-LABEL: @PR4645(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[IF_THEN:%.*]]
-; CHECK:       lor.lhs.false:
-; CHECK-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    call void @exit(i32 1)
-; CHECK-NEXT:    br label [[FOR_COND]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    unreachable
-; CHECK:       for.end:
-; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK32-LABEL: @PR4645(
+; CHECK32-NEXT:  entry:
+; CHECK32-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK32:       lor.lhs.false:
+; CHECK32-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
+; CHECK32:       if.then:
+; CHECK32-NEXT:    call void @exit(i32 1) #[[ATTR6:[0-9]+]]
+; CHECK32-NEXT:    br label [[FOR_COND]]
+; CHECK32:       for.cond:
+; CHECK32-NEXT:    unreachable
+; CHECK32:       for.end:
+; CHECK32-NEXT:    br label [[FOR_COND]]
+;
+; CHECK16-LABEL: @PR4645(
+; CHECK16-NEXT:  entry:
+; CHECK16-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK16:       lor.lhs.false:
+; CHECK16-NEXT:    br i1 [[C1:%.*]], label [[IF_THEN]], label [[FOR_COND:%.*]]
+; CHECK16:       if.then:
+; CHECK16-NEXT:    call void @exit(i32 1)
+; CHECK16-NEXT:    br label [[FOR_COND]]
+; CHECK16:       for.cond:
+; CHECK16-NEXT:    unreachable
+; CHECK16:       for.end:
+; CHECK16-NEXT:    br label [[FOR_COND]]
 ;
 entry:
   br label %if.then
diff --git a/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll b/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
index 1da6903ae4666..ccb96012ae3cc 100644
--- a/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
+++ b/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
@@ -187,7 +187,7 @@ define i32 @sink_gep1(i1 %c) {
 ; CHECK:       early_return:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       use_block:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[VAR]], i64 4
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 4
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @unknown(ptr nonnull [[GEP]]) #[[ATTR1]]
 ; CHECK-NEXT:    ret i32 [[VAR3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/snprintf-2.ll b/llvm/test/Transforms/InstCombine/snprintf-2.ll
index 0465457aacec7..5195b4271ab8c 100644
--- a/llvm/test/Transforms/InstCombine/snprintf-2.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf-2.ll
@@ -32,12 +32,12 @@ define void @fold_snprintf_fmt() {
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 16), align 4
 ; BE-NEXT:    [[PD3:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 24), align 8
 ; BE-NEXT:    store i16 12594, ptr [[PD3]], align 1
-; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[PD3]], i64 2
+; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PD3]], i64 2
 ; BE-NEXT:    store i8 0, ptr [[ENDPTR]], align 1
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 12), align 4
 ; BE-NEXT:    [[PD2:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 16), align 8
 ; BE-NEXT:    store i8 49, ptr [[PD2]], align 1
-; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[PD2]], i64 1
+; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2]], i64 1
 ; BE-NEXT:    store i8 0, ptr [[ENDPTR1]], align 1
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 8), align 4
 ; BE-NEXT:    [[PD1:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 8), align 8
@@ -58,12 +58,12 @@ define void @fold_snprintf_fmt() {
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 16), align 4
 ; LE-NEXT:    [[PD3:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 24), align 8
 ; LE-NEXT:    store i16 12849, ptr [[PD3]], align 1
-; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[PD3]], i64 2
+; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PD3]], i64 2
 ; LE-NEXT:    store i8 0, ptr [[ENDPTR]], align 1
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 12), align 4
 ; LE-NEXT:    [[PD2:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 16), align 8
 ; LE-NEXT:    store i8 49, ptr [[PD2]], align 1
-; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[PD2]], i64 1
+; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2]], i64 1
 ; LE-NEXT:    store i8 0, ptr [[ENDPTR1]], align 1
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 8), align 4
 ; LE-NEXT:    [[PD1:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 8), align 8
diff --git a/llvm/test/Transforms/InstCombine/snprintf-3.ll b/llvm/test/Transforms/InstCombine/snprintf-3.ll
index 7c93580b4ea54..d1375954a15ad 100644
--- a/llvm/test/Transforms/InstCombine/snprintf-3.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf-3.ll
@@ -33,12 +33,12 @@ define void @fold_snprintf_pcnt_s() {
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 16), align 4
 ; BE-NEXT:    [[PD3:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 24), align 8
 ; BE-NEXT:    store i16 12594, ptr [[PD3]], align 1
-; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[PD3]], i64 2
+; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PD3]], i64 2
 ; BE-NEXT:    store i8 0, ptr [[ENDPTR]], align 1
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 12), align 4
 ; BE-NEXT:    [[PD2:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 16), align 8
 ; BE-NEXT:    store i8 49, ptr [[PD2]], align 1
-; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[PD2]], i64 1
+; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2]], i64 1
 ; BE-NEXT:    store i8 0, ptr [[ENDPTR1]], align 1
 ; BE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 8), align 4
 ; BE-NEXT:    [[PD1:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 8), align 8
@@ -59,12 +59,12 @@ define void @fold_snprintf_pcnt_s() {
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 16), align 4
 ; LE-NEXT:    [[PD3:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 24), align 8
 ; LE-NEXT:    store i16 12849, ptr [[PD3]], align 1
-; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[PD3]], i64 2
+; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PD3]], i64 2
 ; LE-NEXT:    store i8 0, ptr [[ENDPTR]], align 1
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 12), align 4
 ; LE-NEXT:    [[PD2:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 16), align 8
 ; LE-NEXT:    store i8 49, ptr [[PD2]], align 1
-; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[PD2]], i64 1
+; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2]], i64 1
 ; LE-NEXT:    store i8 0, ptr [[ENDPTR1]], align 1
 ; LE-NEXT:    store i32 3, ptr getelementptr (i8, ptr @asiz, i64 8), align 4
 ; LE-NEXT:    [[PD1:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 8), align 8
diff --git a/llvm/test/Transforms/InstCombine/snprintf-4.ll b/llvm/test/Transforms/InstCombine/snprintf-4.ll
index 7006838ae9b58..88d9ea06765d0 100644
--- a/llvm/test/Transforms/InstCombine/snprintf-4.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf-4.ll
@@ -21,17 +21,17 @@ define void @fold_snprintf_pcnt_c(i32 %c) {
 ; CHECK-LABEL: @fold_snprintf_pcnt_c(
 ; CHECK-NEXT:    [[PDIMAX:%.*]] = load ptr, ptr @adst, align 8
 ; CHECK-NEXT:    store i8 1, ptr [[PDIMAX]], align 1
-; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds i8, ptr [[PDIMAX]], i64 1
+; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds nuw i8, ptr [[PDIMAX]], i64 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL]], align 1
 ; CHECK-NEXT:    store i32 1, ptr @asiz, align 4
 ; CHECK-NEXT:    [[PD2:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 8), align 8
 ; CHECK-NEXT:    store i8 2, ptr [[PD2]], align 1
-; CHECK-NEXT:    [[NUL1:%.*]] = getelementptr inbounds i8, ptr [[PD2]], i64 1
+; CHECK-NEXT:    [[NUL1:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2]], i64 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL1]], align 1
 ; CHECK-NEXT:    store i32 1, ptr getelementptr (i8, ptr @asiz, i64 4), align 4
 ; CHECK-NEXT:    [[PD2_0:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 16), align 8
 ; CHECK-NEXT:    store i8 0, ptr [[PD2_0]], align 1
-; CHECK-NEXT:    [[NUL2:%.*]] = getelementptr inbounds i8, ptr [[PD2_0]], i64 1
+; CHECK-NEXT:    [[NUL2:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2_0]], i64 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL2]], align 1
 ; CHECK-NEXT:    store i32 1, ptr getelementptr (i8, ptr @asiz, i64 8), align 4
 ; CHECK-NEXT:    [[PD1:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 24), align 8
@@ -41,7 +41,7 @@ define void @fold_snprintf_pcnt_c(i32 %c) {
 ; CHECK-NEXT:    [[PD2_C:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 32), align 8
 ; CHECK-NEXT:    [[CHAR:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    store i8 [[CHAR]], ptr [[PD2_C]], align 1
-; CHECK-NEXT:    [[NUL3:%.*]] = getelementptr inbounds i8, ptr [[PD2_C]], i64 1
+; CHECK-NEXT:    [[NUL3:%.*]] = getelementptr inbounds nuw i8, ptr [[PD2_C]], i64 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL3]], align 1
 ; CHECK-NEXT:    store i32 1, ptr getelementptr (i8, ptr @asiz, i64 16), align 4
 ; CHECK-NEXT:    [[PD1_C:%.*]] = load ptr, ptr getelementptr (i8, ptr @adst, i64 40), align 8
diff --git a/llvm/test/Transforms/InstCombine/snprintf.ll b/llvm/test/Transforms/InstCombine/snprintf.ll
index 4e51040b39807..33285a97e0f33 100644
--- a/llvm/test/Transforms/InstCombine/snprintf.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf.ll
@@ -103,7 +103,7 @@ define i32 @test_char_small_size(ptr %buf) #0 {
 define i32 @test_char_ok_size(ptr %buf) #0 {
 ; CHECK-LABEL: @test_char_ok_size(
 ; CHECK-NEXT:    store i8 65, ptr [[BUF:%.*]], align 1
-; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds i8, ptr [[BUF]], i64 1
+; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL]], align 1
 ; CHECK-NEXT:    ret i32 1
 ;
diff --git a/llvm/test/Transforms/InstCombine/sprintf-1.ll b/llvm/test/Transforms/InstCombine/sprintf-1.ll
index b9354bb4b2990..0749015059415 100644
--- a/llvm/test/Transforms/InstCombine/sprintf-1.ll
+++ b/llvm/test/Transforms/InstCombine/sprintf-1.ll
@@ -56,7 +56,7 @@ define void @test_simplify3(ptr %dst) {
 define void @test_simplify4(ptr %dst) {
 ; CHECK-LABEL: @test_simplify4(
 ; CHECK-NEXT:    store i8 104, ptr [[DST:%.*]], align 1
-; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 1
+; CHECK-NEXT:    [[NUL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i32 1
 ; CHECK-NEXT:    store i8 0, ptr [[NUL]], align 1
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/stpcpy-1.ll b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
index 2ddacb2097442..843fd8c5ed95c 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
@@ -16,7 +16,7 @@ declare ptr @stpcpy(ptr, ptr)
 define ptr @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(6) @a, ptr noundef nonnull align 1 dereferenceable(6) @hello, i32 6, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 5)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 5)
 ;
   %ret = call ptr @stpcpy(ptr @a, ptr @hello)
   ret ptr %ret
@@ -25,7 +25,7 @@ define ptr @test_simplify1() {
 define ptr @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
 ; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) @a)
-; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i8, ptr @a, i32 [[STRLEN]]
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds nuw i8, ptr @a, i32 [[STRLEN]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %ret = call ptr @stpcpy(ptr @a, ptr @a)
@@ -62,7 +62,7 @@ define ptr @test_no_simplify2(ptr %dst, ptr %src) {
 define ptr @test_no_incompatible_attr() {
 ; CHECK-LABEL: @test_no_incompatible_attr(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(6) @a, ptr noundef nonnull align 1 dereferenceable(6) @hello, i32 6, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 5)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 5)
 ;
   %ret = call dereferenceable(1) ptr @stpcpy(ptr @a, ptr @hello)
   ret ptr %ret
diff --git a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
index 2d775f35c8bda..cd3f8f8919720 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
@@ -15,7 +15,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 define ptr @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(12) @a, ptr noundef nonnull align 1 dereferenceable(12) @.str, i32 12, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 11)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 11)
 ;
 
   %ret = call ptr @__stpcpy_chk(ptr @a, ptr @.str, i32 60)
@@ -25,7 +25,7 @@ define ptr @test_simplify1() {
 define ptr @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(12) @a, ptr noundef nonnull align 1 dereferenceable(12) @.str, i32 12, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 11)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 11)
 ;
 
   %ret = call ptr @__stpcpy_chk(ptr @a, ptr @.str, i32 12)
@@ -35,7 +35,7 @@ define ptr @test_simplify2() {
 define ptr @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(12) @a, ptr noundef nonnull align 1 dereferenceable(12) @.str, i32 12, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 11)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 11)
 ;
 
   %ret = call ptr @__stpcpy_chk(ptr @a, ptr @.str, i32 -1)
@@ -45,7 +45,7 @@ define ptr @test_simplify3() {
 define ptr @test_simplify1_tail() {
 ; CHECK-LABEL: @test_simplify1_tail(
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(12) @a, ptr noundef nonnull align 1 dereferenceable(12) @.str, i32 12, i1 false)
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 11)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 11)
 ;
 
   %ret = tail call ptr @__stpcpy_chk(ptr @a, ptr @.str, i32 60)
@@ -80,7 +80,7 @@ define ptr @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
 ; CHECK-NEXT:    [[LEN:%.*]] = call i32 @llvm.objectsize.i32.p0(ptr @a, i1 false, i1 false, i1 false)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @__memcpy_chk(ptr nonnull @a, ptr nonnull @.str, i32 12, i32 [[LEN]])
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a, i32 11)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a, i32 11)
 ;
 
   %len = call i32 @llvm.objectsize.i32.p0(ptr @a, i1 false, i1 false, i1 false)
@@ -93,7 +93,7 @@ define ptr @test_simplify5() {
 define ptr @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
 ; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) @a)
-; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i8, ptr @a, i32 [[STRLEN]]
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds nuw i8, ptr @a, i32 [[STRLEN]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/stpncpy-1.ll b/llvm/test/Transforms/InstCombine/stpncpy-1.ll
index ddfc1c086347f..87f54918b7d25 100644
--- a/llvm/test/Transforms/InstCombine/stpncpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpncpy-1.ll
@@ -47,7 +47,7 @@ define void @fold_stpncpy_overlap(ptr %dst, i64 %n) {
 ; ANY-NEXT:    [[STXNCPY_CHAR0:%.*]] = load i8, ptr [[DST]], align 1
 ; ANY-NEXT:    [[STPNCPY_CHAR0CMP:%.*]] = icmp ne i8 [[STXNCPY_CHAR0]], 0
 ; ANY-NEXT:    [[STPNCPY_SEL_IDX:%.*]] = zext i1 [[STPNCPY_CHAR0CMP]] to i64
-; ANY-NEXT:    [[STPNCPY_SEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[STPNCPY_SEL_IDX]]
+; ANY-NEXT:    [[STPNCPY_SEL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 [[STPNCPY_SEL_IDX]]
 ; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_SEL]])
 ; ANY-NEXT:    ret void
 ;
@@ -141,32 +141,32 @@ define void @fold_stpncpy_s1(ptr %dst) {
 ; BE-LABEL: @fold_stpncpy_s1(
 ; BE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; BE-NEXT:    store i8 52, ptr [[DST]], align 1
-; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; BE-NEXT:    store i16 13312, ptr [[DST]], align 1
-; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(3) @str.6, i64 3, i1 false)
-; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.7, i64 9, i1 false)
-; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; BE-NEXT:    ret void
 ;
 ; LE-LABEL: @fold_stpncpy_s1(
 ; LE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; LE-NEXT:    store i8 52, ptr [[DST]], align 1
-; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; LE-NEXT:    store i16 52, ptr [[DST]], align 1
-; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(3) @str.6, i64 3, i1 false)
-; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.7, i64 9, i1 false)
-; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; LE-NEXT:    ret void
 ;
@@ -204,38 +204,38 @@ define void @fold_stpncpy_s4(ptr %dst, i64 %n) {
 ; BE-LABEL: @fold_stpncpy_s4(
 ; BE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; BE-NEXT:    store i8 49, ptr [[DST]], align 1
-; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; BE-NEXT:    store i16 12594, ptr [[DST]], align 1
-; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @s4, i64 3, i1 false)
-; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; BE-NEXT:    store i32 825373492, ptr [[DST]], align 1
-; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.8, i64 9, i1 false)
-; BE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR3]])
 ; BE-NEXT:    ret void
 ;
 ; LE-LABEL: @fold_stpncpy_s4(
 ; LE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; LE-NEXT:    store i8 49, ptr [[DST]], align 1
-; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; LE-NEXT:    store i16 12849, ptr [[DST]], align 1
-; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @s4, i64 3, i1 false)
-; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; LE-NEXT:    store i32 875770417, ptr [[DST]], align 1
-; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.8, i64 9, i1 false)
-; LE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR3]])
 ; LE-NEXT:    ret void
 ;
@@ -273,11 +273,11 @@ define void @fold_stpncpy_s4(ptr %dst, i64 %n) {
 
 define void @call_stpncpy_xx_n(ptr %dst, i64 %n) {
 ; ANY-LABEL: @call_stpncpy_xx_n(
-; ANY-NEXT:    [[EA1_N:%.*]] = call ptr @stpncpy(ptr [[DST:%.*]], ptr nonnull dereferenceable(2) getelementptr inbounds (i8, ptr @a4, i64 3), i64 [[N:%.*]])
+; ANY-NEXT:    [[EA1_N:%.*]] = call ptr @stpncpy(ptr [[DST:%.*]], ptr nonnull dereferenceable(2) getelementptr inbounds nuw (i8, ptr @a4, i64 3), i64 [[N:%.*]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[EA1_N]])
 ; ANY-NEXT:    [[EA4_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr nonnull dereferenceable(5) @a4, i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[EA4_N]])
-; ANY-NEXT:    [[ES1_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr nonnull dereferenceable(2) getelementptr inbounds (i8, ptr @s4, i64 3), i64 [[N]])
+; ANY-NEXT:    [[ES1_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr nonnull dereferenceable(2) getelementptr inbounds nuw (i8, ptr @s4, i64 3), i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES1_N]])
 ; ANY-NEXT:    [[ES4_N:%.*]] = call ptr @stpncpy(ptr [[DST]], ptr nonnull dereferenceable(5) @s4, i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], ptr [[ES4_N]])
@@ -312,44 +312,44 @@ define void @fold_stpncpy_a4(ptr %dst, i64 %n) {
 ; BE-LABEL: @fold_stpncpy_a4(
 ; BE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; BE-NEXT:    store i8 49, ptr [[DST]], align 1
-; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; BE-NEXT:    store i16 12594, ptr [[DST]], align 1
-; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; BE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a4, i64 3, i1 false)
-; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; BE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; BE-NEXT:    store i32 825373492, ptr [[DST]], align 1
-; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a4, i64 5, i1 false)
-; BE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR3]])
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.9, i64 9, i1 false)
-; BE-NEXT:    [[ENDPTR4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[ENDPTR4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR4]])
 ; BE-NEXT:    ret void
 ;
 ; LE-LABEL: @fold_stpncpy_a4(
 ; LE-NEXT:    call void @sink(ptr [[DST:%.*]], ptr [[DST]])
 ; LE-NEXT:    store i8 49, ptr [[DST]], align 1
-; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[STPNCPY_END:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_END]])
 ; LE-NEXT:    store i16 12849, ptr [[DST]], align 1
-; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; LE-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a4, i64 3, i1 false)
-; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; LE-NEXT:    [[ENDPTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR1]])
 ; LE-NEXT:    store i32 875770417, ptr [[DST]], align 1
-; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[ENDPTR2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR2]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a4, i64 5, i1 false)
-; LE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[ENDPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR3]])
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(9) [[DST]], ptr noundef nonnull align 1 dereferenceable(9) @str.9, i64 9, i1 false)
-; LE-NEXT:    [[ENDPTR4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[ENDPTR4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[ENDPTR4]])
 ; LE-NEXT:    ret void
 ;
@@ -398,7 +398,7 @@ define void @fold_stpncpy_s(ptr %dst, ptr %src) {
 ; ANY-NEXT:    store i8 [[STXNCPY_CHAR0]], ptr [[DST]], align 1
 ; ANY-NEXT:    [[STPNCPY_CHAR0CMP:%.*]] = icmp ne i8 [[STXNCPY_CHAR0]], 0
 ; ANY-NEXT:    [[STPNCPY_SEL_IDX:%.*]] = zext i1 [[STPNCPY_CHAR0CMP]] to i64
-; ANY-NEXT:    [[STPNCPY_SEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[STPNCPY_SEL_IDX]]
+; ANY-NEXT:    [[STPNCPY_SEL:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 [[STPNCPY_SEL_IDX]]
 ; ANY-NEXT:    call void @sink(ptr nonnull [[DST]], ptr nonnull [[STPNCPY_SEL]])
 ; ANY-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/str-int-2.ll b/llvm/test/Transforms/InstCombine/str-int-2.ll
index ae67422d12078..9ff8e059b676f 100644
--- a/llvm/test/Transforms/InstCombine/str-int-2.ll
+++ b/llvm/test/Transforms/InstCombine/str-int-2.ll
@@ -44,7 +44,7 @@ define i64 @strtol_hex() #0 {
 
 define i64 @strtol_endptr_not_null(ptr nonnull %pend) {
 ; CHECK-LABEL: @strtol_endptr_not_null(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @.str, i64 2), ptr [[PEND:%.*]], align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @.str, i64 2), ptr [[PEND:%.*]], align 8
 ; CHECK-NEXT:    ret i64 12
 ;
   %call = call i64 @strtol(ptr @.str, ptr %pend, i32 10)
diff --git a/llvm/test/Transforms/InstCombine/str-int-3.ll b/llvm/test/Transforms/InstCombine/str-int-3.ll
index 100f1a95b1359..3c3a43122abca 100644
--- a/llvm/test/Transforms/InstCombine/str-int-3.ll
+++ b/llvm/test/Transforms/InstCombine/str-int-3.ll
@@ -66,7 +66,7 @@ define void @fold_atoi_member(ptr %pi) {
 
 define void @fold_atoi_offset_out_of_bounds(ptr %pi) {
 ; CHECK-LABEL: @fold_atoi_offset_out_of_bounds(
-; CHECK-NEXT:    [[IA_0_0_32:%.*]] = call i32 @atoi(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a, i64 32))
+; CHECK-NEXT:    [[IA_0_0_32:%.*]] = call i32 @atoi(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a, i64 32))
 ; CHECK-NEXT:    store i32 [[IA_0_0_32]], ptr [[PI:%.*]], align 4
 ; CHECK-NEXT:    [[IA_0_0_33:%.*]] = call i32 @atoi(ptr nocapture getelementptr (i8, ptr @a, i64 33))
 ; CHECK-NEXT:    store i32 [[IA_0_0_33]], ptr [[PI]], align 4
diff --git a/llvm/test/Transforms/InstCombine/str-int-4.ll b/llvm/test/Transforms/InstCombine/str-int-4.ll
index 9173e122f8dd8..7cabb63462c32 100644
--- a/llvm/test/Transforms/InstCombine/str-int-4.ll
+++ b/llvm/test/Transforms/InstCombine/str-int-4.ll
@@ -42,39 +42,39 @@ declare i64 @strtoll(ptr, ptr, i32)
 
 define void @fold_strtol(ptr %ps) {
 ; CHECK-LABEL: @fold_strtol(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    store i32 -123, ptr [[PS:%.*]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS1:%.*]] = getelementptr i8, ptr [[PS]], i64 4
 ; CHECK-NEXT:    store i32 234, ptr [[PS1]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i0, i64 2), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i0, i64 2), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i32 0, ptr [[PS2]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i9, i64 2), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i9, i64 2), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 12
 ; CHECK-NEXT:    store i32 9, ptr [[PS3]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ia, i64 2), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ia, i64 2), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS4:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i32 10, ptr [[PS4]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i19azAZ, i64 6), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i19azAZ, i64 6), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS5:%.*]] = getelementptr i8, ptr [[PS]], i64 20
 ; CHECK-NEXT:    store i32 76095035, ptr [[PS5]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS6:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS6]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mo32min, i64 14), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mo32min, i64 14), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS7:%.*]] = getelementptr i8, ptr [[PS]], i64 28
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS7]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS8:%.*]] = getelementptr i8, ptr [[PS]], i64 32
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS8]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS9:%.*]] = getelementptr i8, ptr [[PS]], i64 36
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS9]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32max, i64 11), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32max, i64 11), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS10:%.*]] = getelementptr i8, ptr [[PS]], i64 40
 ; CHECK-NEXT:    store i32 2147483647, ptr [[PS10]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @x32max, i64 11), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @x32max, i64 11), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS11:%.*]] = getelementptr i8, ptr [[PS]], i64 44
 ; CHECK-NEXT:    store i32 2147483647, ptr [[PS11]], align 4
 ; CHECK-NEXT:    ret void
@@ -181,7 +181,7 @@ define void @call_strtol(ptr %ps) {
 ; CHECK-NEXT:    [[NWS:%.*]] = call i32 @strtol(ptr nonnull @ws, ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS11:%.*]] = getelementptr i8, ptr [[PS]], i64 44
 ; CHECK-NEXT:    store i32 [[NWS]], ptr [[PS11]], align 4
-; CHECK-NEXT:    [[NWSP6:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[NWSP6:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds nuw (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS12:%.*]] = getelementptr i8, ptr [[PS]], i64 48
 ; CHECK-NEXT:    store i32 [[NWSP6]], ptr [[PS12]], align 4
 ; CHECK-NEXT:    [[I0B1:%.*]] = call i32 @strtol(ptr nonnull @i0, ptr nonnull @endptr, i32 1)
@@ -287,15 +287,15 @@ define void @call_strtol(ptr %ps) {
 
 define void @fold_strtoll(ptr %ps) {
 ; CHECK-LABEL: @fold_strtoll(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    store i64 -123, ptr [[PS:%.*]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS1:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i64 234, ptr [[PS1]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64min, i64 21), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64min, i64 21), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 -9223372036854775808, ptr [[PS2]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64max, i64 20), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64max, i64 20), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 9223372036854775807, ptr [[PS3]], align 4
 ; CHECK-NEXT:    ret void
@@ -335,7 +335,7 @@ define void @call_strtoll(ptr %ps) {
 ; CHECK-NEXT:    [[NWS:%.*]] = call i64 @strtoll(ptr nonnull @ws, ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 [[NWS]], ptr [[PS2]], align 4
-; CHECK-NEXT:    [[NWSP6:%.*]] = call i64 @strtoll(ptr nonnull getelementptr inbounds (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[NWSP6:%.*]] = call i64 @strtoll(ptr nonnull getelementptr inbounds nuw (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 [[NWSP6]], ptr [[PS3]], align 4
 ; CHECK-NEXT:    ret void
@@ -375,10 +375,10 @@ define void @call_strtol_trailing_space(ptr %ps) {
 ; CHECK-NEXT:    [[N1:%.*]] = call i32 @strtol(ptr nonnull @i_1_2_3_, ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS1:%.*]] = getelementptr i8, ptr [[PS:%.*]], i64 4
 ; CHECK-NEXT:    store i32 [[N1]], ptr [[PS1]], align 4
-; CHECK-NEXT:    [[N2:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds (i8, ptr @i_1_2_3_, i64 2), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[N2:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds nuw (i8, ptr @i_1_2_3_, i64 2), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i32 [[N2]], ptr [[PS2]], align 4
-; CHECK-NEXT:    [[N3:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds (i8, ptr @i_1_2_3_, i64 4), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[N3:%.*]] = call i32 @strtol(ptr nonnull getelementptr inbounds nuw (i8, ptr @i_1_2_3_, i64 4), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 12
 ; CHECK-NEXT:    store i32 [[N3]], ptr [[PS3]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/str-int-5.ll b/llvm/test/Transforms/InstCombine/str-int-5.ll
index 4ccf7ea6407c2..ea56f3c4982a1 100644
--- a/llvm/test/Transforms/InstCombine/str-int-5.ll
+++ b/llvm/test/Transforms/InstCombine/str-int-5.ll
@@ -46,39 +46,39 @@ declare i64 @strtoull(ptr, ptr, i32)
 
 define void @fold_strtoul(ptr %ps) {
 ; CHECK-LABEL: @fold_strtoul(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    store i32 -123, ptr [[PS:%.*]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS1:%.*]] = getelementptr i8, ptr [[PS]], i64 4
 ; CHECK-NEXT:    store i32 234, ptr [[PS1]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32min_m1, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32min_m1, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i32 2147483647, ptr [[PS2]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 12
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS3]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @o32min, i64 14), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @o32min, i64 14), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS4:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS4]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mo32min, i64 14), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mo32min, i64 14), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS5:%.*]] = getelementptr i8, ptr [[PS]], i64 20
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS5]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @x32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @x32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS6:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS6]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mx32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS7:%.*]] = getelementptr i8, ptr [[PS]], i64 28
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS7]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32max, i64 11), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32max, i64 11), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS8:%.*]] = getelementptr i8, ptr [[PS]], i64 32
 ; CHECK-NEXT:    store i32 2147483647, ptr [[PS8]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @mX01, i64 5), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @mX01, i64 5), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS9:%.*]] = getelementptr i8, ptr [[PS]], i64 36
 ; CHECK-NEXT:    store i32 -1, ptr [[PS9]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32max_p1, i64 11), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32max_p1, i64 11), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS10:%.*]] = getelementptr i8, ptr [[PS]], i64 40
 ; CHECK-NEXT:    store i32 -2147483648, ptr [[PS10]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ui32max, i64 11), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ui32max, i64 11), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS11:%.*]] = getelementptr i8, ptr [[PS]], i64 44
 ; CHECK-NEXT:    store i32 -1, ptr [[PS11]], align 4
 ; CHECK-NEXT:    ret void
@@ -159,7 +159,7 @@ define void @call_strtoul(ptr %ps) {
 ; CHECK-NEXT:    [[NWS:%.*]] = call i32 @strtoul(ptr nonnull @ws, ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i32 [[NWS]], ptr [[PS2]], align 4
-; CHECK-NEXT:    [[NWSP6:%.*]] = call i32 @strtoul(ptr nonnull getelementptr inbounds (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[NWSP6:%.*]] = call i32 @strtoul(ptr nonnull getelementptr inbounds nuw (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 12
 ; CHECK-NEXT:    store i32 [[NWSP6]], ptr [[PS3]], align 4
 ; CHECK-NEXT:    ret void
@@ -195,36 +195,36 @@ define void @call_strtoul(ptr %ps) {
 
 define void @fold_strtoull(ptr %ps) {
 ; CHECK-LABEL: @fold_strtoull(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_im123, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    store i64 -123, ptr [[PS:%.*]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ws_ip234, i64 10), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS1:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i64 234, ptr [[PS1]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64min_m1, i64 21), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64min_m1, i64 21), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 9223372036854775807, ptr [[PS2]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 -2147483648, ptr [[PS3]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @o32min, i64 14), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @o32min, i64 14), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS4:%.*]] = getelementptr i8, ptr [[PS]], i64 32
 ; CHECK-NEXT:    store i64 2147483648, ptr [[PS4]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @x32min, i64 12), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @x32min, i64 12), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS5:%.*]] = getelementptr i8, ptr [[PS]], i64 40
 ; CHECK-NEXT:    store i64 2147483648, ptr [[PS5]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64min, i64 21), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64min, i64 21), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS6:%.*]] = getelementptr i8, ptr [[PS]], i64 48
 ; CHECK-NEXT:    store i64 -9223372036854775808, ptr [[PS6]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64max, i64 20), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64max, i64 20), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS7:%.*]] = getelementptr i8, ptr [[PS]], i64 56
 ; CHECK-NEXT:    store i64 9223372036854775807, ptr [[PS7]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @i64max_p1, i64 20), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @i64max_p1, i64 20), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS8:%.*]] = getelementptr i8, ptr [[PS]], i64 64
 ; CHECK-NEXT:    store i64 -9223372036854775808, ptr [[PS8]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @ui64max, i64 21), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @ui64max, i64 21), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS9:%.*]] = getelementptr i8, ptr [[PS]], i64 72
 ; CHECK-NEXT:    store i64 -1, ptr [[PS9]], align 4
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @x64max, i64 19), ptr @endptr, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @x64max, i64 19), ptr @endptr, align 8
 ; CHECK-NEXT:    [[PS10:%.*]] = getelementptr i8, ptr [[PS]], i64 80
 ; CHECK-NEXT:    store i64 -1, ptr [[PS10]], align 4
 ; CHECK-NEXT:    ret void
@@ -298,7 +298,7 @@ define void @call_strtoull(ptr %ps) {
 ; CHECK-NEXT:    [[NWS:%.*]] = call i64 @strtoull(ptr nonnull @ws, ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 [[NWS]], ptr [[PS2]], align 4
-; CHECK-NEXT:    [[NWSP6:%.*]] = call i64 @strtoull(ptr nonnull getelementptr inbounds (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
+; CHECK-NEXT:    [[NWSP6:%.*]] = call i64 @strtoull(ptr nonnull getelementptr inbounds nuw (i8, ptr @ws, i64 6), ptr nonnull @endptr, i32 10)
 ; CHECK-NEXT:    [[PS3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 [[NWSP6]], ptr [[PS3]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/str-int.ll b/llvm/test/Transforms/InstCombine/str-int.ll
index ee8d04d2f0e2a..33314a18e2563 100644
--- a/llvm/test/Transforms/InstCombine/str-int.ll
+++ b/llvm/test/Transforms/InstCombine/str-int.ll
@@ -45,8 +45,8 @@ define i32 @strtol_hex() #0 {
 
 define i32 @strtol_endptr_not_null(ptr %pend) {
 ; CHECK-LABEL: @strtol_endptr_not_null(
-; CHECK-NEXT:    [[ENDP1:%.*]] = getelementptr inbounds i8, ptr [[PEND:%.*]], i64 8
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @.str, i64 2), ptr [[ENDP1]], align 8
+; CHECK-NEXT:    [[ENDP1:%.*]] = getelementptr inbounds nuw i8, ptr [[PEND:%.*]], i64 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @.str, i64 2), ptr [[ENDP1]], align 8
 ; CHECK-NEXT:    ret i32 12
 ;
   %endp1 = getelementptr inbounds ptr, ptr %pend, i32 1
diff --git a/llvm/test/Transforms/InstCombine/strcall-bad-sig.ll b/llvm/test/Transforms/InstCombine/strcall-bad-sig.ll
index 7d3633a5d2271..cf013dbe6a141 100644
--- a/llvm/test/Transforms/InstCombine/strcall-bad-sig.ll
+++ b/llvm/test/Transforms/InstCombine/strcall-bad-sig.ll
@@ -42,7 +42,7 @@ declare ptr @strncasecmp(ptr, ptr)
 
 define ptr @call_bad_strncasecmp() {
 ; CHECK-LABEL: @call_bad_strncasecmp(
-; CHECK-NEXT:    [[CMP:%.*]] = call ptr @strncasecmp(ptr nonnull @a, ptr nonnull getelementptr inbounds (i8, ptr @a, i64 1))
+; CHECK-NEXT:    [[CMP:%.*]] = call ptr @strncasecmp(ptr nonnull @a, ptr nonnull getelementptr inbounds nuw (i8, ptr @a, i64 1))
 ; CHECK-NEXT:    ret ptr [[CMP]]
 ;
   %p1 = getelementptr [2 x i8], ptr @a, i32 0, i32 1
@@ -55,7 +55,7 @@ declare i1 @strcoll(ptr, ptr, ptr)
 
 define i1 @call_bad_strcoll() {
 ; CHECK-LABEL: @call_bad_strcoll(
-; CHECK-NEXT:    [[I:%.*]] = call i1 @strcoll(ptr nonnull @a, ptr nonnull getelementptr inbounds (i8, ptr @a, i64 1), ptr nonnull @a)
+; CHECK-NEXT:    [[I:%.*]] = call i1 @strcoll(ptr nonnull @a, ptr nonnull getelementptr inbounds nuw (i8, ptr @a, i64 1), ptr nonnull @a)
 ; CHECK-NEXT:    ret i1 [[I]]
 ;
   %p1 = getelementptr [2 x i8], ptr @a, i32 0, i32 1
@@ -80,7 +80,7 @@ declare i1 @strtok(ptr, ptr, i1)
 
 define i1 @call_bad_strtok() {
 ; CHECK-LABEL: @call_bad_strtok(
-; CHECK-NEXT:    [[RET:%.*]] = call i1 @strtok(ptr nonnull @a, ptr nonnull getelementptr inbounds (i8, ptr @a, i64 1), i1 false)
+; CHECK-NEXT:    [[RET:%.*]] = call i1 @strtok(ptr nonnull @a, ptr nonnull getelementptr inbounds nuw (i8, ptr @a, i64 1), i1 false)
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %p1 = getelementptr [2 x i8], ptr @a, i32 0, i32 1
@@ -94,7 +94,7 @@ declare i1 @strtok_r(ptr, ptr)
 
 define i1 @call_bad_strtok_r() {
 ; CHECK-LABEL: @call_bad_strtok_r(
-; CHECK-NEXT:    [[RET:%.*]] = call i1 @strtok_r(ptr nonnull @a, ptr nonnull getelementptr inbounds (i8, ptr @a, i64 1))
+; CHECK-NEXT:    [[RET:%.*]] = call i1 @strtok_r(ptr nonnull @a, ptr nonnull getelementptr inbounds nuw (i8, ptr @a, i64 1))
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %p1 = getelementptr [2 x i8], ptr @a, i32 0, i32 1
@@ -146,7 +146,7 @@ declare ptr @strxfrm(ptr, ptr)
 
 define ptr @call_bad_strxfrm() {
 ; CHECK-LABEL: @call_bad_strxfrm(
-; CHECK-NEXT:    [[RET:%.*]] = call ptr @strxfrm(ptr nonnull @a, ptr nonnull getelementptr inbounds (i8, ptr @a, i64 1))
+; CHECK-NEXT:    [[RET:%.*]] = call ptr @strxfrm(ptr nonnull @a, ptr nonnull getelementptr inbounds nuw (i8, ptr @a, i64 1))
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %p1 = getelementptr [2 x i8], ptr @a, i32 0, i32 1
diff --git a/llvm/test/Transforms/InstCombine/strcall-no-nul.ll b/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
index 96905a273319b..41ee7de5cb40d 100644
--- a/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
+++ b/llvm/test/Transforms/InstCombine/strcall-no-nul.ll
@@ -50,7 +50,7 @@ declare i32 @snprintf(ptr, i64, ptr, ...)
 
 define ptr @fold_strchr_past_end() {
 ; CHECK-LABEL: @fold_strchr_past_end(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @a5, i64 5)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @a5, i64 5)
 ;
   %p = getelementptr [5 x i8], ptr @a5, i32 0, i32 5
   %q = call ptr @strchr(ptr %p, i32 0)
@@ -268,7 +268,7 @@ define void @fold_strcspn_past_end(ptr %poff) {
 
 define i32 @fold_atoi_past_end() {
 ; CHECK-LABEL: @fold_atoi_past_end(
-; CHECK-NEXT:    [[I:%.*]] = call i32 @atoi(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5))
+; CHECK-NEXT:    [[I:%.*]] = call i32 @atoi(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    ret i32 [[I]]
 ;
   %p5 = getelementptr [5 x i8], ptr @a5, i32 0, i32 5
@@ -282,21 +282,21 @@ define i32 @fold_atoi_past_end() {
 
 define void @fold_atol_strtol_past_end(ptr %ps) {
 ; CHECK-LABEL: @fold_atol_strtol_past_end(
-; CHECK-NEXT:    [[I0:%.*]] = call i64 @atol(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5))
+; CHECK-NEXT:    [[I0:%.*]] = call i64 @atol(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    store i64 [[I0]], ptr [[PS:%.*]], align 4
-; CHECK-NEXT:    [[I1:%.*]] = call i64 @atoll(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5))
+; CHECK-NEXT:    [[I1:%.*]] = call i64 @atoll(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[PS]], i64 8
 ; CHECK-NEXT:    store i64 [[I1]], ptr [[P1]], align 4
-; CHECK-NEXT:    [[I2:%.*]] = call i64 @strtol(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5), ptr null, i32 0)
+; CHECK-NEXT:    [[I2:%.*]] = call i64 @strtol(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 0)
 ; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[PS]], i64 16
 ; CHECK-NEXT:    store i64 [[I2]], ptr [[P2]], align 4
-; CHECK-NEXT:    [[I3:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5), ptr null, i32 8)
+; CHECK-NEXT:    [[I3:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 8)
 ; CHECK-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[PS]], i64 24
 ; CHECK-NEXT:    store i64 [[I3]], ptr [[P3]], align 4
-; CHECK-NEXT:    [[I4:%.*]] = call i64 @strtoll(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5), ptr null, i32 10)
+; CHECK-NEXT:    [[I4:%.*]] = call i64 @strtoll(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 10)
 ; CHECK-NEXT:    [[P4:%.*]] = getelementptr i8, ptr [[PS]], i64 32
 ; CHECK-NEXT:    store i64 [[I4]], ptr [[P4]], align 4
-; CHECK-NEXT:    [[I5:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds (i8, ptr @a5, i64 5), ptr null, i32 16)
+; CHECK-NEXT:    [[I5:%.*]] = call i64 @strtoul(ptr nocapture nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5), ptr null, i32 16)
 ; CHECK-NEXT:    [[P5:%.*]] = getelementptr i8, ptr [[PS]], i64 40
 ; CHECK-NEXT:    store i64 [[I5]], ptr [[P5]], align 4
 ; CHECK-NEXT:    ret void
@@ -358,9 +358,9 @@ define void @fold_sprintf_past_end(ptr %pcnt, ptr %dst) {
 
 define void @fold_snprintf_past_end(ptr %pcnt, ptr %dst, i64 %n) {
 ; CHECK-LABEL: @fold_snprintf_past_end(
-; CHECK-NEXT:    [[N5_:%.*]] = call i32 (ptr, i64, ptr, ...) @snprintf(ptr [[DST:%.*]], i64 [[N:%.*]], ptr nonnull getelementptr inbounds (i8, ptr @a5, i64 5))
+; CHECK-NEXT:    [[N5_:%.*]] = call i32 (ptr, i64, ptr, ...) @snprintf(ptr [[DST:%.*]], i64 [[N:%.*]], ptr nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    store i32 [[N5_]], ptr [[PCNT:%.*]], align 4
-; CHECK-NEXT:    [[N05:%.*]] = call i32 (ptr, i64, ptr, ...) @snprintf(ptr [[DST]], i64 [[N]], ptr nonnull @a5, ptr nonnull getelementptr inbounds (i8, ptr @a5, i64 5))
+; CHECK-NEXT:    [[N05:%.*]] = call i32 (ptr, i64, ptr, ...) @snprintf(ptr [[DST]], i64 [[N]], ptr nonnull @a5, ptr nonnull getelementptr inbounds nuw (i8, ptr @a5, i64 5))
 ; CHECK-NEXT:    [[PN05:%.*]] = getelementptr i8, ptr [[PCNT]], i64 4
 ; CHECK-NEXT:    store i32 [[N05]], ptr [[PN05]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/strchr-1.ll b/llvm/test/Transforms/InstCombine/strchr-1.ll
index 50ef6abb2107e..9fe8e75b7d403 100644
--- a/llvm/test/Transforms/InstCombine/strchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strchr-1.ll
@@ -13,7 +13,7 @@ declare ptr @strchr(ptr, i32)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
@@ -35,7 +35,7 @@ define void @test_simplify2() {
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
@@ -58,7 +58,7 @@ define void @test_simplify4(i32 %chr) {
 
 define void @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/strchr-3.ll b/llvm/test/Transforms/InstCombine/strchr-3.ll
index 7cbbdf8c69bc4..36518a0bf496b 100644
--- a/llvm/test/Transforms/InstCombine/strchr-3.ll
+++ b/llvm/test/Transforms/InstCombine/strchr-3.ll
@@ -20,7 +20,7 @@ define ptr @fold_strchr_s1_C(i32 %C) {
 ; CHECK-LABEL: @fold_strchr_s1_C(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @s1, i64 1), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @s1, i64 1), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @s1, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
@@ -36,7 +36,7 @@ define ptr @fold_strchr_s11_C(i32 %C) {
 ; CHECK-LABEL: @fold_strchr_s11_C(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @s11, i64 2), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @s11, i64 2), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @s11, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
@@ -52,7 +52,7 @@ define ptr @fold_strchr_s111_C(i32 %C) {
 ; CHECK-LABEL: @fold_strchr_s111_C(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @s111, i64 3), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @s111, i64 3), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @s111, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
@@ -96,9 +96,9 @@ define ptr @fold_strchr_s21111p1_C(i32 %C) {
 ; CHECK-LABEL: @fold_strchr_s21111p1_C(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @s21111, i64 5), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @s21111, i64 5), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
-; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr getelementptr inbounds (i8, ptr @s21111, i64 1), ptr [[MEMCHR_SEL1]]
+; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr getelementptr inbounds nuw (i8, ptr @s21111, i64 1), ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
 ;
   %ptr = getelementptr inbounds [6 x i8], ptr @s21111, i64 0, i64 1
@@ -113,7 +113,7 @@ define ptr @fold_strchr_s11102_C(i32 %C) {
 ; CHECK-LABEL: @fold_strchr_s11102_C(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds (i8, ptr @s11102, i64 3), ptr null
+; CHECK-NEXT:    [[MEMCHR_SEL1:%.*]] = select i1 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @s11102, i64 3), ptr null
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[TMP1]], 1
 ; CHECK-NEXT:    [[MEMCHR_SEL2:%.*]] = select i1 [[TMP3]], ptr @s11102, ptr [[MEMCHR_SEL1]]
 ; CHECK-NEXT:    ret ptr [[MEMCHR_SEL2]]
diff --git a/llvm/test/Transforms/InstCombine/strcmp-4.ll b/llvm/test/Transforms/InstCombine/strcmp-4.ll
index e96c28b780b28..e16e8b9d6c839 100644
--- a/llvm/test/Transforms/InstCombine/strcmp-4.ll
+++ b/llvm/test/Transforms/InstCombine/strcmp-4.ll
@@ -11,8 +11,8 @@ declare i32 @strcmp(ptr, ptr)
 
 define i32 @fold_strcmp_s3_x_s4_s3(i1 %C) {
 ; CHECK-LABEL: @fold_strcmp_s3_x_s4_s3(
-; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[C:%.*]], ptr getelementptr inbounds (i8, ptr @s9, i64 6), ptr getelementptr inbounds (i8, ptr @s9, i64 5)
-; CHECK-NEXT:    [[CMP:%.*]] = call i32 @strcmp(ptr noundef nonnull dereferenceable(1) [[PTR]], ptr noundef nonnull dereferenceable(4) getelementptr inbounds (i8, ptr @s9, i64 6))
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[C:%.*]], ptr getelementptr inbounds nuw (i8, ptr @s9, i64 6), ptr getelementptr inbounds nuw (i8, ptr @s9, i64 5)
+; CHECK-NEXT:    [[CMP:%.*]] = call i32 @strcmp(ptr noundef nonnull dereferenceable(1) [[PTR]], ptr noundef nonnull dereferenceable(4) getelementptr inbounds nuw (i8, ptr @s9, i64 6))
 ; CHECK-NEXT:    ret i32 [[CMP]]
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/strlcpy-1.ll b/llvm/test/Transforms/InstCombine/strlcpy-1.ll
index 7ca6c1599f191..fd9d0580426f0 100644
--- a/llvm/test/Transforms/InstCombine/strlcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlcpy-1.ll
@@ -109,15 +109,15 @@ define void @fold_strlcpy_s5(ptr %dst) {
 ; BE-NEXT:    store i8 0, ptr [[DST]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; BE-NEXT:    store i8 49, ptr [[DST]], align 1
-; BE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    store i8 0, ptr [[TMP1]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; BE-NEXT:    store i16 12594, ptr [[DST]], align 1
-; BE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; BE-NEXT:    store i8 0, ptr [[TMP2]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(3) @s4, i64 3, i1 false)
-; BE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; BE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; BE-NEXT:    store i8 0, ptr [[TMP3]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @s4, i64 5, i1 false)
@@ -133,15 +133,15 @@ define void @fold_strlcpy_s5(ptr %dst) {
 ; LE-NEXT:    store i8 0, ptr [[DST]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; LE-NEXT:    store i8 49, ptr [[DST]], align 1
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    store i8 0, ptr [[TMP1]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; LE-NEXT:    store i16 12849, ptr [[DST]], align 1
-; LE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; LE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 2
 ; LE-NEXT:    store i8 0, ptr [[TMP2]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(3) [[DST]], ptr noundef nonnull align 1 dereferenceable(3) @s4, i64 3, i1 false)
-; LE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 3
 ; LE-NEXT:    store i8 0, ptr [[TMP3]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 4)
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @s4, i64 5, i1 false)
@@ -235,9 +235,9 @@ define void @call_strlcpy_s0_n(ptr %dst, ptr %s, i64 %n) {
 ; ANY-NEXT:    [[NZ:%.*]] = or i64 [[N]], 1
 ; ANY-NEXT:    [[NS_NZ:%.*]] = call i64 @strlcpy(ptr noundef nonnull dereferenceable(1) [[DST]], ptr noundef nonnull dereferenceable(1) [[S]], i64 [[NZ]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS_NZ]])
-; ANY-NEXT:    [[NS0_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds (i8, ptr @s4, i64 4), i64 [[N]])
+; ANY-NEXT:    [[NS0_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 4), i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS0_N]])
-; ANY-NEXT:    [[NS1_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds (i8, ptr @s4, i64 3), i64 [[N]])
+; ANY-NEXT:    [[NS1_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) getelementptr inbounds nuw (i8, ptr @s4, i64 3), i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS1_N]])
 ; ANY-NEXT:    [[NS4_N:%.*]] = call i64 @strlcpy(ptr [[DST]], ptr noundef nonnull dereferenceable(1) @s4, i64 [[N]])
 ; ANY-NEXT:    call void @sink(ptr [[DST]], i64 [[NS4_N]])
@@ -283,19 +283,19 @@ define void @fold_strlcpy_a5(ptr %dst, i64 %n) {
 ; BE-NEXT:    store i8 0, ptr [[DST]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; BE-NEXT:    store i8 49, ptr [[DST]], align 1
-; BE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; BE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; BE-NEXT:    store i8 0, ptr [[TMP1]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; BE-NEXT:    store i32 825373492, ptr [[DST]], align 1
-; BE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; BE-NEXT:    store i8 0, ptr [[TMP2]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a5, i64 5, i1 false)
-; BE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 5
+; BE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 5
 ; BE-NEXT:    store i8 0, ptr [[TMP3]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; BE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a5, i64 5, i1 false)
-; BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 5
+; BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 5
 ; BE-NEXT:    store i8 0, ptr [[TMP4]], align 1
 ; BE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; BE-NEXT:    ret void
@@ -305,19 +305,19 @@ define void @fold_strlcpy_a5(ptr %dst, i64 %n) {
 ; LE-NEXT:    store i8 0, ptr [[DST]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; LE-NEXT:    store i8 49, ptr [[DST]], align 1
-; LE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 1
+; LE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 1
 ; LE-NEXT:    store i8 0, ptr [[TMP1]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; LE-NEXT:    store i32 875770417, ptr [[DST]], align 1
-; LE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; LE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 4
 ; LE-NEXT:    store i8 0, ptr [[TMP2]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a5, i64 5, i1 false)
-; LE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 5
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 5
 ; LE-NEXT:    store i8 0, ptr [[TMP3]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; LE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(5) [[DST]], ptr noundef nonnull align 1 dereferenceable(5) @a5, i64 5, i1 false)
-; LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 5
+; LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 5
 ; LE-NEXT:    store i8 0, ptr [[TMP4]], align 1
 ; LE-NEXT:    call void @sink(ptr nonnull [[DST]], i64 5)
 ; LE-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/strlen-1.ll b/llvm/test/Transforms/InstCombine/strlen-1.ll
index 8def4dd9747f9..b2755f9f4119f 100644
--- a/llvm/test/Transforms/InstCombine/strlen-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-1.ll
@@ -155,7 +155,7 @@ define i32 @test_no_simplify1() {
 
 define i32 @test_no_simplify2(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify2(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], ptr @null_hello, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [7 x i8], ptr @null_hello, i32 0, i32 [[X:%.*]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -166,8 +166,8 @@ define i32 @test_no_simplify2(i32 %x) {
 
 define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify2_no_null_opt(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], ptr @null_hello, i32 0, i32 [[X:%.*]]
-; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef [[HELLO_P]])
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [7 x i8], ptr @null_hello, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
   %hello_p = getelementptr inbounds [7 x i8], ptr @null_hello, i32 0, i32 %x
@@ -180,7 +180,7 @@ define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 define i32 @test_no_simplify3(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify3(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], ptr @null_hello_mid, i32 0, i32 [[AND]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [13 x i8], ptr @null_hello_mid, i32 0, i32 [[AND]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -193,8 +193,8 @@ define i32 @test_no_simplify3(i32 %x) {
 define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify3_on_null_opt(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], ptr @null_hello_mid, i32 0, i32 [[AND]]
-; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef [[HELLO_P]])
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [13 x i8], ptr @null_hello_mid, i32 0, i32 [[AND]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
   %and = and i32 %x, 15
@@ -235,7 +235,7 @@ define i1 @strlen0_after_write_to_first_byte_global() {
 
 define i1 @strlen0_after_write_to_second_byte_global() {
 ; CHECK-LABEL: @strlen0_after_write_to_second_byte_global(
-; CHECK-NEXT:    store i8 49, ptr getelementptr inbounds (i8, ptr @a, i32 1), align 16
+; CHECK-NEXT:    store i8 49, ptr getelementptr inbounds nuw (i8, ptr @a, i32 1), align 16
 ; CHECK-NEXT:    [[CHAR0:%.*]] = load i8, ptr @a, align 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[CHAR0]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
diff --git a/llvm/test/Transforms/InstCombine/strlen-4.ll b/llvm/test/Transforms/InstCombine/strlen-4.ll
index 58d04e8a0d4be..ca01ce93be883 100644
--- a/llvm/test/Transforms/InstCombine/strlen-4.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-4.ll
@@ -18,7 +18,7 @@ declare i64 @strlen(ptr)
 
 define i64 @fold_strlen_s3_pi_s5(i1 %X, i64 %I) {
 ; CHECK-LABEL: @fold_strlen_s3_pi_s5(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr [[PS3_PI]], ptr @s5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -40,7 +40,7 @@ define i64 @fold_strlen_s3_pi_p1_s5(i1 %0, i64 %1) {
 ; XFAIL-CHECK-NEXT:    [[SEL:%.*]] = select i1 %0, i64 [[DIF_I]], i64 5
 ; XFAIL-CHECK-NEXT:    ret i64 [[SEL]]
 ; CHECK-LABEL: @fold_strlen_s3_pi_p1_s5(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[PS3_PI_P1:%.*]] = getelementptr i8, ptr [[PS3_PI]], i64 1
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS3_PI_P1]], ptr @s5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
@@ -61,7 +61,7 @@ define i64 @fold_strlen_s3_pi_p1_s5(i1 %0, i64 %1) {
 
 define i64 @call_strlen_s5_3_pi_s5(i1 %0, i64 %1) {
 ; CHECK-LABEL: @call_strlen_s5_3_pi_s5(
-; CHECK-NEXT:    [[PS5_3_PI:%.*]] = getelementptr inbounds [10 x i8], ptr @s5_3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_3_PI:%.*]] = getelementptr inbounds nuw [10 x i8], ptr @s5_3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS5_3_PI]], ptr @s5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -78,7 +78,7 @@ define i64 @call_strlen_s5_3_pi_s5(i1 %0, i64 %1) {
 
 define i64 @call_strlen_s5_3_s5_pj(i1 %X, i64 %J) {
 ; CHECK-LABEL: @call_strlen_s5_3_s5_pj(
-; CHECK-NEXT:    [[PS5:%.*]] = getelementptr inbounds [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
+; CHECK-NEXT:    [[PS5:%.*]] = getelementptr inbounds nuw [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr @s5_3, ptr [[PS5]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -95,7 +95,7 @@ define i64 @call_strlen_s5_3_s5_pj(i1 %X, i64 %J) {
 
 define i64 @fold_strlen_s3_s5_pj(i1 %X, i64 %J) {
 ; CHECK-LABEL: @fold_strlen_s3_s5_pj(
-; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
+; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds nuw [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr @s3, ptr [[PS5_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -114,7 +114,7 @@ define i64 @fold_strlen_s3_s5_pj(i1 %X, i64 %J) {
 
 define i64 @call_strlen_s3_s5_3_pj(i1 %0, i64 %1) {
 ; CHECK-LABEL: @call_strlen_s3_s5_3_pj(
-; CHECK-NEXT:    [[PS5_3_PJ:%.*]] = getelementptr inbounds [10 x i8], ptr @s5_3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_3_PJ:%.*]] = getelementptr inbounds nuw [10 x i8], ptr @s5_3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr @s3, ptr [[PS5_3_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -131,8 +131,8 @@ define i64 @call_strlen_s3_s5_3_pj(i1 %0, i64 %1) {
 
 define i64 @fold_strlen_s3_pi_s5_pj(i1 %X, i64 %I, i64 %J) {
 ; CHECK-LABEL: @fold_strlen_s3_pi_s5_pj(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
-; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds nuw [6 x i8], ptr @s5, i64 0, i64 [[J:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], ptr [[PS3_PI]], ptr [[PS5_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @strlen(ptr noundef nonnull dereferenceable(1) [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
diff --git a/llvm/test/Transforms/InstCombine/strlen-6.ll b/llvm/test/Transforms/InstCombine/strlen-6.ll
index 25e653362db81..bae061ac70fb0 100644
--- a/llvm/test/Transforms/InstCombine/strlen-6.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-6.ll
@@ -103,7 +103,7 @@ define i64 @fold_strlen_a_S3_p2_s4_to_1() {
 define void @fold_strlen_a_s3_S4_to_4() {
 ; CHECK-LABEL: @fold_strlen_a_s3_S4_to_4(
 ; CHECK-NEXT:    store i64 4, ptr @ax, align 4
-; CHECK-NEXT:    store i64 4, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 4, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_a5, ptr @a_s3_s4, i32 0, i32 0, i32 4
@@ -125,7 +125,7 @@ define void @fold_strlen_a_s3_S4_to_4() {
 define void @fold_strlen_a_s3_S4_p1_to_3() {
 ; CHECK-LABEL: @fold_strlen_a_s3_S4_p1_to_3(
 ; CHECK-NEXT:    store i64 3, ptr @ax, align 4
-; CHECK-NEXT:    store i64 3, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 3, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_a5, ptr @a_s3_s4, i32 0, i32 0, i32 5
@@ -147,7 +147,7 @@ define void @fold_strlen_a_s3_S4_p1_to_3() {
 define void @fold_strlen_a_s3_i32_S4_to_4() {
 ; CHECK-LABEL: @fold_strlen_a_s3_i32_S4_to_4(
 ; CHECK-NEXT:    store i64 4, ptr @ax, align 4
-; CHECK-NEXT:    store i64 4, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 4, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_i32_a5, ptr @a_s3_i32_s4, i32 0, i32 0, i32 8
@@ -169,7 +169,7 @@ define void @fold_strlen_a_s3_i32_S4_to_4() {
 define void @fold_strlen_a_s3_i32_S4_p1_to_3() {
 ; CHECK-LABEL: @fold_strlen_a_s3_i32_S4_p1_to_3(
 ; CHECK-NEXT:    store i64 3, ptr @ax, align 4
-; CHECK-NEXT:    store i64 3, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 3, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_i32_a5, ptr @a_s3_i32_s4, i32 0, i32 0, i32 9
@@ -191,7 +191,7 @@ define void @fold_strlen_a_s3_i32_S4_p1_to_3() {
 define void @fold_strlen_a_s3_i32_S4_p2_to_2() {
 ; CHECK-LABEL: @fold_strlen_a_s3_i32_S4_p2_to_2(
 ; CHECK-NEXT:    store i64 2, ptr @ax, align 4
-; CHECK-NEXT:    store i64 2, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 2, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_i32_a5, ptr @a_s3_i32_s4, i32 0, i32 0, i32 10
@@ -213,7 +213,7 @@ define void @fold_strlen_a_s3_i32_S4_p2_to_2() {
 define void @fold_strlen_a_s3_i32_S4_p3_to_1() {
 ; CHECK-LABEL: @fold_strlen_a_s3_i32_S4_p3_to_1(
 ; CHECK-NEXT:    store i64 1, ptr @ax, align 4
-; CHECK-NEXT:    store i64 1, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 1, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_i32_a5, ptr @a_s3_i32_s4, i32 0, i32 0, i32 11
@@ -235,7 +235,7 @@ define void @fold_strlen_a_s3_i32_S4_p3_to_1() {
 define void @fold_strlen_a_s3_i32_S4_p4_to_0() {
 ; CHECK-LABEL: @fold_strlen_a_s3_i32_S4_p4_to_0(
 ; CHECK-NEXT:    store i64 0, ptr @ax, align 4
-; CHECK-NEXT:    store i64 0, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 0, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr %struct.A_a4_i32_a5, ptr @a_s3_i32_s4, i32 0, i32 0, i32 12
@@ -257,8 +257,8 @@ define void @fold_strlen_a_s3_i32_S4_p4_to_0() {
 define void @fold_strlen_ax_s() {
 ; CHECK-LABEL: @fold_strlen_ax_s(
 ; CHECK-NEXT:    store i64 3, ptr @ax, align 4
-; CHECK-NEXT:    store i64 5, ptr getelementptr inbounds (i8, ptr @ax, i64 8), align 4
-; CHECK-NEXT:    store i64 7, ptr getelementptr inbounds (i8, ptr @ax, i64 16), align 4
+; CHECK-NEXT:    store i64 5, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 8), align 4
+; CHECK-NEXT:    store i64 7, ptr getelementptr inbounds nuw (i8, ptr @ax, i64 16), align 4
 ; CHECK-NEXT:    ret void
 ;
   %pax_s3 = getelementptr { i8, [4 x i8] }, ptr @ax_s3, i64 0, i32 1, i64 0
diff --git a/llvm/test/Transforms/InstCombine/strncat-2.ll b/llvm/test/Transforms/InstCombine/strncat-2.ll
index 7f1199d908d39..bc857c5b31d87 100644
--- a/llvm/test/Transforms/InstCombine/strncat-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncat-2.ll
@@ -13,7 +13,7 @@ declare ptr @strncat(ptr, ptr, i32)
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
 ; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(ptr noundef nonnull dereferenceable(1) @a)
-; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds i8, ptr @a, i32 [[STRLEN]]
+; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr inbounds nuw i8, ptr @a, i32 [[STRLEN]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(6) [[ENDPTR]], ptr noundef nonnull align 1 dereferenceable(6) @hello, i32 6, i1 false)
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/strnlen-3.ll b/llvm/test/Transforms/InstCombine/strnlen-3.ll
index f988f86fc15b3..3b7e44f8df889 100644
--- a/llvm/test/Transforms/InstCombine/strnlen-3.ll
+++ b/llvm/test/Transforms/InstCombine/strnlen-3.ll
@@ -31,7 +31,7 @@ define i64 @fold_strnlen_sx_pi_0(i64 %i) {
 
 define i64 @call_strnlen_sx_pi_n(i64 %i, i64 %n) {
 ; CHECK-LABEL: @call_strnlen_sx_pi_n(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [0 x i8], ptr @sx, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [0 x i8], ptr @sx, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[PTR]], i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -46,7 +46,7 @@ define i64 @call_strnlen_sx_pi_n(i64 %i, i64 %n) {
 
 define i64 @call_strnlen_a3_pi_2(i64 %i) {
 ; CHECK-LABEL: @call_strnlen_a3_pi_2(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 2)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -61,7 +61,7 @@ define i64 @call_strnlen_a3_pi_2(i64 %i) {
 
 define i64 @call_strnlen_a3_pi_3(i64 %i) {
 ; CHECK-LABEL: @call_strnlen_a3_pi_3(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 3)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -111,7 +111,7 @@ define i64 @fold_strnlen_s5_3_pi_0(i64 zeroext %i) {
 
 define i64 @call_strnlen_s5_3_pi_n(i64 zeroext %i, i64 %n) {
 ; CHECK-LABEL: @call_strnlen_s5_3_pi_n(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [10 x i8], ptr @s5_3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [10 x i8], ptr @s5_3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[PTR]], i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -151,7 +151,7 @@ define i64 @fold_strnlen_s3_n(i64 %n) {
 
 define i64 @fold_strnlen_a3_pi_2(i64 %i) {
 ; CHECK-LABEL: @fold_strnlen_a3_pi_2(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [3 x i8], ptr @a3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 2)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -166,7 +166,7 @@ define i64 @fold_strnlen_a3_pi_2(i64 %i) {
 
 define i64 @fold_strnlen_s3_pi_2(i64 %i) {
 ; CHECK-LABEL: @fold_strnlen_s3_pi_2(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 2)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -181,7 +181,7 @@ define i64 @fold_strnlen_s3_pi_2(i64 %i) {
 
 define i64 @fold_strnlen_s3_pi_3(i64 %i) {
 ; CHECK-LABEL: @fold_strnlen_s3_pi_3(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 3)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -196,7 +196,7 @@ define i64 @fold_strnlen_s3_pi_3(i64 %i) {
 
 define i64 @fold_strnlen_s3_pi_n(i64 %i, i64 %n) {
 ; CHECK-LABEL: @fold_strnlen_s3_pi_n(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[PTR]], i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
@@ -212,7 +212,7 @@ define i64 @fold_strnlen_s3_pi_n(i64 %i, i64 %n) {
 
 define i64 @call_strnlen_s5_3_pi_2(i64 %i) {
 ; CHECK-LABEL: @call_strnlen_s5_3_pi_2(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [10 x i8], ptr @s5_3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [10 x i8], ptr @s5_3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr noundef nonnull dereferenceable(1) [[PTR]], i64 2)
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/strnlen-4.ll b/llvm/test/Transforms/InstCombine/strnlen-4.ll
index 7a222f0993be4..6f73ee47d7b50 100644
--- a/llvm/test/Transforms/InstCombine/strnlen-4.ll
+++ b/llvm/test/Transforms/InstCombine/strnlen-4.ll
@@ -17,7 +17,7 @@ declare i64 @strnlen(ptr, i64)
 
 define i64 @fold_strnlen_s3_pi_s5_n(i1 %C, i64 %i, i64 %n) {
 ; CHECK-LABEL: @fold_strnlen_s3_pi_s5_n(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], ptr [[PTR]], ptr @s5
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[SEL]], i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -57,7 +57,7 @@ define i64 @call_strnlen_s3_pi_xbounds_s5_n(i1 %C, i64 %i, i64 %n) {
 
 define i64 @call_strnlen_s3_pi_sx_n(i1 %C, i64 %i, i64 %n) {
 ; CHECK-LABEL: @call_strnlen_s3_pi_sx_n(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @s3, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], ptr [[PTR]], ptr @sx
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[SEL]], i64 [[N:%.*]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
diff --git a/llvm/test/Transforms/InstCombine/strnlen-5.ll b/llvm/test/Transforms/InstCombine/strnlen-5.ll
index 3866b92ab8b4a..3eb3764955fa5 100644
--- a/llvm/test/Transforms/InstCombine/strnlen-5.ll
+++ b/llvm/test/Transforms/InstCombine/strnlen-5.ll
@@ -164,7 +164,7 @@ define i1 @fold_strnlen_ax_nz_gtz(i64 %n) {
 
 define i1 @fold_strnlen_a5_pi_nz_eqz(i64 %i, i64 %n) {
 ; CHECK-LABEL: @fold_strnlen_a5_pi_nz_eqz(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [5 x i8], ptr @a5, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [5 x i8], ptr @a5, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[CHAR0:%.*]] = load i8, ptr [[PTR]], align 1
 ; CHECK-NEXT:    [[EQZ:%.*]] = icmp eq i8 [[CHAR0]], 0
 ; CHECK-NEXT:    ret i1 [[EQZ]]
@@ -200,7 +200,7 @@ define i1 @fold_strnlen_s5_pi_nz_eqz(i64 %i, i64 %n) {
 
 define i1 @call_strnlen_s5_pi_n_eqz(i64 %i, i64 %n) {
 ; CHECK-LABEL: @call_strnlen_s5_pi_n_eqz(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [6 x i8], ptr @s5, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds nuw [6 x i8], ptr @s5, i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[LEN:%.*]] = call i64 @strnlen(ptr nonnull [[PTR]], i64 [[N:%.*]])
 ; CHECK-NEXT:    [[EQZ:%.*]] = icmp eq i64 [[LEN]], 0
 ; CHECK-NEXT:    ret i1 [[EQZ]]
diff --git a/llvm/test/Transforms/InstCombine/strpbrk-1.ll b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
index b51071df25d20..01dabf8cd1ec5 100644
--- a/llvm/test/Transforms/InstCombine/strpbrk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
@@ -37,7 +37,7 @@ define ptr @test_simplify2(ptr %pat) {
 
 define ptr @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @hello, i32 6)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @hello, i32 6)
 ;
 
   %ret = call ptr @strpbrk(ptr @hello, ptr @w)
diff --git a/llvm/test/Transforms/InstCombine/strrchr-1.ll b/llvm/test/Transforms/InstCombine/strrchr-1.ll
index 0c876b9d2a982..524d9e078cbe0 100644
--- a/llvm/test/Transforms/InstCombine/strrchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strrchr-1.ll
@@ -12,7 +12,7 @@ declare ptr @strrchr(ptr, i32)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 6), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 6), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
@@ -34,7 +34,7 @@ define void @test_simplify2() {
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
@@ -45,7 +45,7 @@ define void @test_simplify3() {
 
 define void @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @hello, i32 13), ptr @chp, align 4
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @hello, i32 13), ptr @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/strrchr-3.ll b/llvm/test/Transforms/InstCombine/strrchr-3.ll
index f25504a8db2b5..fda02114bc94e 100644
--- a/llvm/test/Transforms/InstCombine/strrchr-3.ll
+++ b/llvm/test/Transforms/InstCombine/strrchr-3.ll
@@ -13,7 +13,7 @@ define ptr @fold_strrchr_sp10_x(i32 %c) {
 ; CHECK-LABEL: @fold_strrchr_sp10_x(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i8
 ; CHECK-NEXT:    [[MEMRCHR_CHAR0CMP:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CHAR0CMP]], ptr getelementptr inbounds (i8, ptr @s10, i64 10), ptr null
+; CHECK-NEXT:    [[MEMRCHR_SEL:%.*]] = select i1 [[MEMRCHR_CHAR0CMP]], ptr getelementptr inbounds nuw (i8, ptr @s10, i64 10), ptr null
 ; CHECK-NEXT:    ret ptr [[MEMRCHR_SEL]]
 ;
   %psp10 = getelementptr [11 x i8], ptr @s10, i32 0, i32 10
@@ -26,7 +26,7 @@ define ptr @fold_strrchr_sp10_x(i32 %c) {
 
 define ptr @call_strrchr_sp9_x(i32 %c) {
 ; CHECK-LABEL: @call_strrchr_sp9_x(
-; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(2) getelementptr inbounds (i8, ptr @s10, i64 9), i32 [[C:%.*]], i64 2)
+; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(2) getelementptr inbounds nuw (i8, ptr @s10, i64 9), i32 [[C:%.*]], i64 2)
 ; CHECK-NEXT:    ret ptr [[MEMRCHR]]
 ;
   %psp9 = getelementptr [11 x i8], ptr @s10, i32 0, i32 9
@@ -40,7 +40,7 @@ define ptr @call_strrchr_sp9_x(i32 %c) {
 
 define ptr @call_strrchr_sp2_x(i32 %c) {
 ; CHECK-LABEL: @call_strrchr_sp2_x(
-; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(9) getelementptr inbounds (i8, ptr @s10, i64 2), i32 [[C:%.*]], i64 9)
+; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(9) getelementptr inbounds nuw (i8, ptr @s10, i64 2), i32 [[C:%.*]], i64 9)
 ; CHECK-NEXT:    ret ptr [[MEMRCHR]]
 ;
   %psp2 = getelementptr [11 x i8], ptr @s10, i32 0, i32 2
@@ -53,7 +53,7 @@ define ptr @call_strrchr_sp2_x(i32 %c) {
 
 define ptr @call_strrchr_sp1_x(i32 %c) {
 ; CHECK-LABEL: @call_strrchr_sp1_x(
-; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(10) getelementptr inbounds (i8, ptr @s10, i64 1), i32 [[C:%.*]], i64 10)
+; CHECK-NEXT:    [[MEMRCHR:%.*]] = call ptr @memrchr(ptr noundef nonnull dereferenceable(10) getelementptr inbounds nuw (i8, ptr @s10, i64 1), i32 [[C:%.*]], i64 10)
 ; CHECK-NEXT:    ret ptr [[MEMRCHR]]
 ;
   %psp1 = getelementptr [11 x i8], ptr @s10, i32 0, i32 1
diff --git a/llvm/test/Transforms/InstCombine/strstr-1.ll b/llvm/test/Transforms/InstCombine/strstr-1.ll
index b5f4a2ce288d5..68de7614aad2b 100644
--- a/llvm/test/Transforms/InstCombine/strstr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strstr-1.ll
@@ -37,7 +37,7 @@ define ptr @test_simplify2(ptr %str) {
 
 define ptr @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @.str2, i64 1)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @.str2, i64 1)
 ;
   %ret = call ptr @strstr(ptr @.str2, ptr @.str3)
   ret ptr %ret
diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa-2.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa-2.ll
index b52a062fc6404..5c547ab7def6d 100644
--- a/llvm/test/Transforms/InstCombine/struct-assign-tbaa-2.ll
+++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa-2.ll
@@ -11,10 +11,10 @@ declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounw
 define void @test1(ptr %a1, ptr %a2) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A2:%.*]], i64 2
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[A2:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A2]], align 2, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    store i16 [[TMP0]], ptr [[A1:%.*]], align 2, !tbaa [[TBAA0]]
-; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds i8, ptr [[A1]], i64 2
+; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw i8, ptr [[A1]], i64 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[B]], align 2, !tbaa [[TBAA6:![0-9]+]]
 ; CHECK-NEXT:    store i16 [[TMP1]], ptr [[B2]], align 2, !tbaa [[TBAA6]]
 ; CHECK-NEXT:    ret void
@@ -37,6 +37,8 @@ entry:
 !7 = !{!5, !5, i64 0, i64 2}
 !8 = !{!1, !4, i64 2, i64 2}
 
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
 ;.
 ; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META4:![0-9]+]], i64 0, i64 2}
 ; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"_ZTS1T", [[META4]], i64 0, i64 2, [[META4]], i64 2, i64 2}
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
index f7a54ab2141bd..39d502b23ee31 100644
--- a/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -156,6 +156,62 @@ define i64 @test_inbounds_nuw_two_gep(ptr %base, i64 %idx, i64 %idx2) {
   ret i64 %d
 }
 
+define i64 @test_nusw_two_gep(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_nusw_two_gep(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[IDX2:%.*]], [[IDX:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr nusw [0 x i32], ptr %base, i64 0, i64 %idx
+  %p2 = getelementptr nusw [0 x i32], ptr %base, i64 0, i64 %idx2
+  %i1 = ptrtoint ptr %p1 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_nuw_two_gep(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_nuw_two_gep(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i64 [[IDX2:%.*]], [[IDX:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr nuw [0 x i32], ptr %base, i64 0, i64 %idx
+  %p2 = getelementptr nuw [0 x i32], ptr %base, i64 0, i64 %idx2
+  %i1 = ptrtoint ptr %p1 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_nuw_two_gep_missing_nuw_on_sub(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_nuw_two_gep_missing_nuw_on_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[IDX2:%.*]], [[IDX:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr nuw [0 x i32], ptr %base, i64 0, i64 %idx
+  %p2 = getelementptr nuw [0 x i32], ptr %base, i64 0, i64 %idx2
+  %i1 = ptrtoint ptr %p1 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @test_nuw_two_gep_missing_nuw_on_one_gep(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @test_nuw_two_gep_missing_nuw_on_one_gep(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[IDX2:%.*]], [[IDX:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr nuw [0 x i32], ptr %base, i64 0, i64 %idx
+  %p2 = getelementptr [0 x i32], ptr %base, i64 0, i64 %idx2
+  %i1 = ptrtoint ptr %p1 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub nuw i64 %i2, %i1
+  ret i64 %d
+}
+
 define i64 @test_inbounds_nuw_multi_index(ptr %base, i64 %idx, i64 %idx2) {
 ; CHECK-LABEL: @test_inbounds_nuw_multi_index(
 ; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX:%.*]], 3
@@ -249,7 +305,7 @@ define i16 @test24a_as1(ptr addrspace(1) %P, i16 %A) {
 
 define i64 @test24b(ptr %P, i64 %A){
 ; CHECK-LABEL: @test24b(
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw nsw i64 [[A:%.*]], 1
 ; CHECK-NEXT:    ret i64 [[B_IDX]]
 ;
   %B = getelementptr inbounds [42 x i16], ptr @Arr, i64 0, i64 %A
@@ -260,7 +316,7 @@ define i64 @test24b(ptr %P, i64 %A){
 
 define i64 @test25(ptr %P, i64 %A){
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw nsw i64 [[A:%.*]], 1
 ; CHECK-NEXT:    [[GEPDIFF:%.*]] = add nsw i64 [[B_IDX]], -84
 ; CHECK-NEXT:    ret i64 [[GEPDIFF]]
 ;
@@ -339,7 +395,7 @@ define i64 @negative_ptrtoint_sub_zext_ptrtoint(ptr %p, i32 %offset) {
 define i16 @test25_as1(ptr addrspace(1) %P, i64 %A) {
 ; CHECK-LABEL: @test25_as1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i16
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw i16 [[TMP1]], 1
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw nsw i16 [[TMP1]], 1
 ; CHECK-NEXT:    [[GEPDIFF:%.*]] = add nsw i16 [[B_IDX]], -84
 ; CHECK-NEXT:    ret i16 [[GEPDIFF]]
 ;
@@ -353,7 +409,7 @@ define i16 @test25_as1(ptr addrspace(1) %P, i64 %A) {
 
 define i64 @ptrtoint_sub_zext_ptrtoint_as2_inbounds(i32 %offset) {
 ; CHECK-LABEL: @ptrtoint_sub_zext_ptrtoint_as2_inbounds(
-; CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds bfloat, ptr addrspace(2) @Arr_as2, i32 [[OFFSET:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds nuw bfloat, ptr addrspace(2) @Arr_as2, i32 [[OFFSET:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = ptrtoint ptr addrspace(2) [[A]] to i32
 ; CHECK-NEXT:    [[C:%.*]] = zext i32 [[B]] to i64
 ; CHECK-NEXT:    [[D:%.*]] = sub nsw i64 ptrtoint (ptr addrspace(2) @Arr_as2 to i64), [[C]]
@@ -715,7 +771,7 @@ define i1 @_gep_phi1(ptr %str1) {
 ; CHECK-NEXT:    br i1 [[CMP1_I]], label [[_Z3FOOPKC_EXIT]], label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[STR1]], [[LOR_LHS_FALSE_I]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[TEST_0_I]], align 1
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
@@ -764,7 +820,7 @@ define i1 @_gep_phi2(ptr %str1, i64 %val2) {
 ; CHECK-NEXT:    br i1 [[CMP1_I]], label [[_Z3FOOPKC_EXIT]], label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[STR1]], [[LOR_LHS_FALSE_I]] ]
-; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 1
+; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds nuw i8, ptr [[A_PN_I]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[TEST_0_I]], align 1
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[CMP3_NOT_I]], label [[WHILE_END_I:%.*]], label [[WHILE_COND_I]]
diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll
index 4fc225454d311..e89419d1f3838 100644
--- a/llvm/test/Transforms/InstCombine/sub.ll
+++ b/llvm/test/Transforms/InstCombine/sub.ll
@@ -1125,7 +1125,7 @@ define i64 @test59(ptr %foo, i64 %i) {
 ; CHECK-LABEL: @test59(
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[FOO:%.*]], i64 [[I:%.*]]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 4200
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4200
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 4200
 ; CHECK-NEXT:    store ptr [[GEP1]], ptr @dummy_global1, align 8
 ; CHECK-NEXT:    store ptr [[GEP2]], ptr @dummy_global2, align 8
 ; CHECK-NEXT:    ret i64 [[I]]
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index 15533346b32ec..a85ce716fbdfa 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -1124,3 +1124,84 @@ loop:
 exit:
   ret void
 }
+
+define i1 @trunc_nuw_i1_non_zero(i8 %1) {
+; CHECK-LABEL: @trunc_nuw_i1_non_zero(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP0:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    ret i1 true
+;
+  %3 = icmp ne i8 %1, 0
+  tail call void @llvm.assume(i1 %3)
+  %ret = trunc nuw i8 %1 to i1
+  ret i1 %ret
+}
+
+define i1 @neg_trunc_nuw_i1_maybe_zero(i8 %1) {
+; CHECK-LABEL: @neg_trunc_nuw_i1_maybe_zero(
+; CHECK-NEXT:    [[RET:%.*]] = trunc nuw i8 [[TMP0:%.*]] to i1
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %ret = trunc nuw i8 %1 to i1
+  ret i1 %ret
+}
+
+define i2 @neg_trunc_nuw_i2_non_zero(i8 %1) {
+; CHECK-LABEL: @neg_trunc_nuw_i2_non_zero(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP0:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    [[RET:%.*]] = trunc nuw i8 [[TMP0]] to i2
+; CHECK-NEXT:    ret i2 [[RET]]
+;
+  %3 = icmp ne i8 %1, 0
+  tail call void @llvm.assume(i1 %3)
+  %ret = trunc nuw i8 %1 to i2
+  ret i2 %ret
+}
+
+define i1 @neg_trunc_i1_non_zero(i8 %1) {
+; CHECK-LABEL: @neg_trunc_i1_non_zero(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP0:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    [[RET:%.*]] = trunc i8 [[TMP0]] to i1
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %3 = icmp ne i8 %1, 0
+  tail call void @llvm.assume(i1 %3)
+  %ret = trunc i8 %1 to i1
+  ret i1 %ret
+}
+
+define i1 @trunc_nsw_i1_non_zero(i8 %1) {
+; CHECK-LABEL: @trunc_nsw_i1_non_zero(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP0:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    ret i1 true
+;
+  %3 = icmp ne i8 %1, 0
+  tail call void @llvm.assume(i1 %3)
+  %ret = trunc nsw i8 %1 to i1
+  ret i1 %ret
+}
+
+define i1 @neg_trunc_nsw_i1_maybe_zero(i8 %1) {
+; CHECK-LABEL: @neg_trunc_nsw_i1_maybe_zero(
+; CHECK-NEXT:    [[RET:%.*]] = trunc nsw i8 [[TMP0:%.*]] to i1
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %ret = trunc nsw i8 %1 to i1
+  ret i1 %ret
+}
+
+define i2 @neg_trunc_nsw_i2_non_zero(i8 %1) {
+; CHECK-LABEL: @neg_trunc_nsw_i2_non_zero(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP0:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    [[RET:%.*]] = trunc nsw i8 [[TMP0]] to i2
+; CHECK-NEXT:    ret i2 [[RET]]
+;
+  %3 = icmp ne i8 %1, 0
+  tail call void @llvm.assume(i1 %3)
+  %ret = trunc nsw i8 %1 to i2
+  ret i2 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
index b164dd983a892..178795f9f9a83 100644
--- a/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
@@ -115,7 +115,7 @@ define float @select_max_ugt_2_use_cmp(float %a, float %b) {
 ; CHECK-LABEL: @select_max_ugt_2_use_cmp(
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc ugt float [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    call void @foo(i1 [[CMP]])
-; CHECK-NEXT:    [[SEL:%.*]] = call fast float @llvm.maxnum.f32(float [[A]], float [[B]])
+; CHECK-NEXT:    [[SEL:%.*]] = call reassoc ninf nsz arcp contract afn float @llvm.maxnum.f32(float [[A]], float [[B]])
 ; CHECK-NEXT:    ret float [[SEL]]
 ;
   %cmp = fcmp reassoc ugt float %a, %b
diff --git a/llvm/test/Transforms/InstCombine/unpack-fca.ll b/llvm/test/Transforms/InstCombine/unpack-fca.ll
index b0cd3823dcca9..bd96611a7a651 100644
--- a/llvm/test/Transforms/InstCombine/unpack-fca.ll
+++ b/llvm/test/Transforms/InstCombine/unpack-fca.ll
@@ -24,7 +24,7 @@ define void @storeA(ptr %a.ptr) {
 define void @storeB(ptr %b.ptr) {
 ; CHECK-LABEL: @storeB(
 ; CHECK-NEXT:    store ptr null, ptr [[B_PTR:%.*]], align 8
-; CHECK-NEXT:    [[B_PTR_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[B_PTR]], i64 8
+; CHECK-NEXT:    [[B_PTR_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 8
 ; CHECK-NEXT:    store i64 42, ptr [[B_PTR_REPACK1]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -76,14 +76,14 @@ define void @storeArrayOfB(ptr %ab.ptr, [2 x %B] %ab) {
 ; CHECK-NEXT:    [[AB_ELT:%.*]] = extractvalue [2 x %B] [[AB:%.*]], 0
 ; CHECK-NEXT:    [[AB_ELT_ELT:%.*]] = extractvalue [[B:%.*]] [[AB_ELT]], 0
 ; CHECK-NEXT:    store ptr [[AB_ELT_ELT]], ptr [[AB_PTR:%.*]], align 8
-; CHECK-NEXT:    [[AB_PTR_REPACK3:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 8
+; CHECK-NEXT:    [[AB_PTR_REPACK3:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 8
 ; CHECK-NEXT:    [[AB_ELT_ELT4:%.*]] = extractvalue [[B]] [[AB_ELT]], 1
 ; CHECK-NEXT:    store i64 [[AB_ELT_ELT4]], ptr [[AB_PTR_REPACK3]], align 8
-; CHECK-NEXT:    [[AB_PTR_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 16
+; CHECK-NEXT:    [[AB_PTR_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 16
 ; CHECK-NEXT:    [[AB_ELT2:%.*]] = extractvalue [2 x %B] [[AB]], 1
 ; CHECK-NEXT:    [[AB_ELT2_ELT:%.*]] = extractvalue [[B]] [[AB_ELT2]], 0
 ; CHECK-NEXT:    store ptr [[AB_ELT2_ELT]], ptr [[AB_PTR_REPACK1]], align 8
-; CHECK-NEXT:    [[AB_PTR_REPACK1_REPACK5:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 24
+; CHECK-NEXT:    [[AB_PTR_REPACK1_REPACK5:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 24
 ; CHECK-NEXT:    [[AB_ELT2_ELT6:%.*]] = extractvalue [[B]] [[AB_ELT2]], 1
 ; CHECK-NEXT:    store i64 [[AB_ELT2_ELT6]], ptr [[AB_PTR_REPACK1_REPACK5]], align 8
 ; CHECK-NEXT:    ret void
@@ -106,7 +106,7 @@ define %B @loadB(ptr %b.ptr) {
 ; CHECK-LABEL: @loadB(
 ; CHECK-NEXT:    [[DOTUNPACK:%.*]] = load ptr, ptr [[B_PTR:%.*]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[B:%.*]] poison, ptr [[DOTUNPACK]], 0
-; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds i8, ptr [[B_PTR]], i64 8
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 8
 ; CHECK-NEXT:    [[DOTUNPACK2:%.*]] = load i64, ptr [[DOTELT1]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue [[B]] [[TMP1]], i64 [[DOTUNPACK2]], 1
 ; CHECK-NEXT:    ret [[B]] [[TMP2]]
@@ -162,7 +162,7 @@ define { %A } @structOfA(ptr %sa.ptr) {
 define %B @structB(ptr %b.ptr) {
 ; CHECK-LABEL: @structB(
 ; CHECK-NEXT:    store ptr null, ptr [[B_PTR:%.*]], align 8
-; CHECK-NEXT:    [[B_PTR_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[B_PTR]], i64 8
+; CHECK-NEXT:    [[B_PTR_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[B_PTR]], i64 8
 ; CHECK-NEXT:    store i64 42, ptr [[B_PTR_REPACK1]], align 8
 ; CHECK-NEXT:    ret [[B:%.*]] { ptr null, i64 42 }
 ;
@@ -175,14 +175,14 @@ define [2 x %B] @loadArrayOfB(ptr %ab.ptr) {
 ; CHECK-LABEL: @loadArrayOfB(
 ; CHECK-NEXT:    [[DOTUNPACK_UNPACK:%.*]] = load ptr, ptr [[AB_PTR:%.*]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[B:%.*]] poison, ptr [[DOTUNPACK_UNPACK]], 0
-; CHECK-NEXT:    [[DOTUNPACK_ELT3:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 8
+; CHECK-NEXT:    [[DOTUNPACK_ELT3:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 8
 ; CHECK-NEXT:    [[DOTUNPACK_UNPACK4:%.*]] = load i64, ptr [[DOTUNPACK_ELT3]], align 8
 ; CHECK-NEXT:    [[DOTUNPACK5:%.*]] = insertvalue [[B]] [[TMP1]], i64 [[DOTUNPACK_UNPACK4]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue [2 x %B] poison, [[B]] [[DOTUNPACK5]], 0
-; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 16
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 16
 ; CHECK-NEXT:    [[DOTUNPACK2_UNPACK:%.*]] = load ptr, ptr [[DOTELT1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertvalue [[B]] poison, ptr [[DOTUNPACK2_UNPACK]], 0
-; CHECK-NEXT:    [[DOTUNPACK2_ELT6:%.*]] = getelementptr inbounds i8, ptr [[AB_PTR]], i64 24
+; CHECK-NEXT:    [[DOTUNPACK2_ELT6:%.*]] = getelementptr inbounds nuw i8, ptr [[AB_PTR]], i64 24
 ; CHECK-NEXT:    [[DOTUNPACK2_UNPACK7:%.*]] = load i64, ptr [[DOTUNPACK2_ELT6]], align 8
 ; CHECK-NEXT:    [[DOTUNPACK28:%.*]] = insertvalue [[B]] [[TMP3]], i64 [[DOTUNPACK2_UNPACK7]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertvalue [2 x %B] [[TMP2]], [[B]] [[DOTUNPACK28]], 1
@@ -207,7 +207,7 @@ define [2000 x %B] @loadLargeArrayOfB(ptr %ab.ptr) {
 ; Make sure that we do not increase alignment of packed struct element
 define i32 @packed_alignment(ptr dereferenceable(9) %s) {
 ; CHECK-LABEL: @packed_alignment(
-; CHECK-NEXT:    [[TV_ELT1:%.*]] = getelementptr inbounds i8, ptr [[S:%.*]], i64 5
+; CHECK-NEXT:    [[TV_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[S:%.*]], i64 5
 ; CHECK-NEXT:    [[TV_UNPACK2:%.*]] = load i32, ptr [[TV_ELT1]], align 1
 ; CHECK-NEXT:    ret i32 [[TV_UNPACK2]]
 ;
@@ -222,38 +222,38 @@ define i32 @packed_alignment(ptr dereferenceable(9) %s) {
 define void @check_alignment(ptr %u, ptr %v) {
 ; CHECK-LABEL: @check_alignment(
 ; CHECK-NEXT:    [[DOTUNPACK:%.*]] = load i8, ptr [[U:%.*]], align 8
-; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 1
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 1
 ; CHECK-NEXT:    [[DOTUNPACK2:%.*]] = load i8, ptr [[DOTELT1]], align 1
-; CHECK-NEXT:    [[DOTELT3:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 2
+; CHECK-NEXT:    [[DOTELT3:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 2
 ; CHECK-NEXT:    [[DOTUNPACK4:%.*]] = load i8, ptr [[DOTELT3]], align 2
-; CHECK-NEXT:    [[DOTELT5:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 3
+; CHECK-NEXT:    [[DOTELT5:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 3
 ; CHECK-NEXT:    [[DOTUNPACK6:%.*]] = load i8, ptr [[DOTELT5]], align 1
-; CHECK-NEXT:    [[DOTELT7:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 4
+; CHECK-NEXT:    [[DOTELT7:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 4
 ; CHECK-NEXT:    [[DOTUNPACK8:%.*]] = load i8, ptr [[DOTELT7]], align 4
-; CHECK-NEXT:    [[DOTELT9:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 5
+; CHECK-NEXT:    [[DOTELT9:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 5
 ; CHECK-NEXT:    [[DOTUNPACK10:%.*]] = load i8, ptr [[DOTELT9]], align 1
-; CHECK-NEXT:    [[DOTELT11:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 6
+; CHECK-NEXT:    [[DOTELT11:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 6
 ; CHECK-NEXT:    [[DOTUNPACK12:%.*]] = load i8, ptr [[DOTELT11]], align 2
-; CHECK-NEXT:    [[DOTELT13:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 7
+; CHECK-NEXT:    [[DOTELT13:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 7
 ; CHECK-NEXT:    [[DOTUNPACK14:%.*]] = load i8, ptr [[DOTELT13]], align 1
-; CHECK-NEXT:    [[DOTELT15:%.*]] = getelementptr inbounds i8, ptr [[U]], i64 8
+; CHECK-NEXT:    [[DOTELT15:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 8
 ; CHECK-NEXT:    [[DOTUNPACK16:%.*]] = load i64, ptr [[DOTELT15]], align 8
 ; CHECK-NEXT:    store i8 [[DOTUNPACK]], ptr [[V:%.*]], align 8
-; CHECK-NEXT:    [[V_REPACK17:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 1
+; CHECK-NEXT:    [[V_REPACK17:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 1
 ; CHECK-NEXT:    store i8 [[DOTUNPACK2]], ptr [[V_REPACK17]], align 1
-; CHECK-NEXT:    [[V_REPACK19:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 2
+; CHECK-NEXT:    [[V_REPACK19:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 2
 ; CHECK-NEXT:    store i8 [[DOTUNPACK4]], ptr [[V_REPACK19]], align 2
-; CHECK-NEXT:    [[V_REPACK21:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 3
+; CHECK-NEXT:    [[V_REPACK21:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 3
 ; CHECK-NEXT:    store i8 [[DOTUNPACK6]], ptr [[V_REPACK21]], align 1
-; CHECK-NEXT:    [[V_REPACK23:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 4
+; CHECK-NEXT:    [[V_REPACK23:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 4
 ; CHECK-NEXT:    store i8 [[DOTUNPACK8]], ptr [[V_REPACK23]], align 4
-; CHECK-NEXT:    [[V_REPACK25:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 5
+; CHECK-NEXT:    [[V_REPACK25:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 5
 ; CHECK-NEXT:    store i8 [[DOTUNPACK10]], ptr [[V_REPACK25]], align 1
-; CHECK-NEXT:    [[V_REPACK27:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 6
+; CHECK-NEXT:    [[V_REPACK27:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 6
 ; CHECK-NEXT:    store i8 [[DOTUNPACK12]], ptr [[V_REPACK27]], align 2
-; CHECK-NEXT:    [[V_REPACK29:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 7
+; CHECK-NEXT:    [[V_REPACK29:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 7
 ; CHECK-NEXT:    store i8 [[DOTUNPACK14]], ptr [[V_REPACK29]], align 1
-; CHECK-NEXT:    [[V_REPACK31:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 8
+; CHECK-NEXT:    [[V_REPACK31:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
 ; CHECK-NEXT:    store i64 [[DOTUNPACK16]], ptr [[V_REPACK31]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
index 1509a42872922..3bcb6498095c1 100644
--- a/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
+++ b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
@@ -376,10 +376,8 @@ define i32 @max_sub_ugt_c10(i32 %a) {
 
 define i32 @max_sub_ugt_c910(i32 %a) {
 ; CHECK-LABEL: @max_sub_ugt_c910(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 9
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -10
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A:%.*]], i32 10)
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp ugt i32 %a, 9
   %sub = add i32 %a, -10
@@ -412,10 +410,8 @@ define i32 @max_sub_ugt_c0(i32 %a) {
 
 define i32 @max_sub_ugt_cmiss(i32 %a) {
 ; CHECK-LABEL: @max_sub_ugt_cmiss(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 1
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    [[SUB:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[A:%.*]], i32 2)
+; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp ugt i32 %a, 1
   %sub = add i32 %a, -2
@@ -479,9 +475,8 @@ define i32 @max_sub_ult_c2_oneusesub(i32 %a) {
 
 define i32 @max_sub_ult_c32(i32 %a) {
 ; CHECK-LABEL: @max_sub_ult_c32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = add nsw i32 [[TMP1]], -2
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %cmp = icmp ult i32 %a, 3
@@ -492,9 +487,8 @@ define i32 @max_sub_ult_c32(i32 %a) {
 
 define i32 @max_sub_ugt_c32(i32 %a) {
 ; CHECK-LABEL: @max_sub_ugt_c32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = add nsw i32 [[TMP1]], -2
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %cmp = icmp ugt i32 3, %a
@@ -505,9 +499,8 @@ define i32 @max_sub_ugt_c32(i32 %a) {
 
 define i32 @max_sub_uge_c32(i32 %a) {
 ; CHECK-LABEL: @max_sub_uge_c32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 3
-; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A]], -2
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = add nsw i32 [[TMP1]], -2
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %cmp = icmp uge i32 2, %a
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
index 644f13f8eee63..a240dfe7d271a 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
@@ -566,7 +566,7 @@ define ptr @gep_cvbase_w_s_idx(<2 x ptr> %base, i64 %raw_addr) {
 
 define ptr @gep_cvbase_w_cv_idx(<2 x ptr> %base, i64 %raw_addr) {
 ; CHECK-LABEL: @gep_cvbase_w_cv_idx(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @GLOBAL, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @GLOBAL, i64 4)
 ;
   %gep = getelementptr i32, <2 x ptr> <ptr @GLOBAL, ptr @GLOBAL>, <2 x i64> <i64 0, i64 1>
   %ee = extractelement <2 x ptr> %gep, i32 1
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
index a4b55e2b6cfec..ee7ef9955e643 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -569,7 +569,7 @@ define ptr @gep_cvbase_w_s_idx(<2 x ptr> %base, i64 %raw_addr) {
 
 define ptr @gep_cvbase_w_cv_idx(<2 x ptr> %base, i64 %raw_addr) {
 ; CHECK-LABEL: @gep_cvbase_w_cv_idx(
-; CHECK-NEXT:    ret ptr getelementptr inbounds (i8, ptr @GLOBAL, i64 4)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw (i8, ptr @GLOBAL, i64 4)
 ;
   %gep = getelementptr i32, <2 x ptr> <ptr @GLOBAL, ptr @GLOBAL>, <2 x i64> <i64 0, i64 1>
   %ee = extractelement <2 x ptr> %gep, i32 1
diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
index 4ac1d084cda9a..c8e5c239af528 100644
--- a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
@@ -3,7 +3,7 @@
 
 define <4 x ptr> @PR41270(ptr %x) {
 ; CHECK-LABEL: @PR41270(
-; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 6
+; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds nuw i8, ptr [[X:%.*]], i64 6
 ; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x ptr> poison, ptr [[T3]], i64 0
 ; CHECK-NEXT:    ret <4 x ptr> [[INS2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
index 8b44213333542..3732af6f86141 100644
--- a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
+++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
@@ -3,7 +3,7 @@
 
 define <4 x ptr> @PR41270(ptr %x) {
 ; CHECK-LABEL: @PR41270(
-; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 6
+; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds nuw i8, ptr [[X:%.*]], i64 6
 ; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x ptr> <ptr poison, ptr undef, ptr undef, ptr undef>, ptr [[T3]], i64 0
 ; CHECK-NEXT:    ret <4 x ptr> [[INS2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vscale_gep.ll b/llvm/test/Transforms/InstCombine/vscale_gep.ll
index f424650d21e84..f85b8f9c5c546 100644
--- a/llvm/test/Transforms/InstCombine/vscale_gep.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_gep.ll
@@ -44,7 +44,7 @@ define void @gep_bitcast(ptr %p) {
 define i32 @gep_alloca_inbounds_vscale_zero() {
 ; CHECK-LABEL: @gep_alloca_inbounds_vscale_zero(
 ; CHECK-NEXT:    [[A:%.*]] = alloca <vscale x 4 x i32>, align 16
-; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[TMP]], align 4
 ; CHECK-NEXT:    ret i32 [[LOAD]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/wcslen-1.ll b/llvm/test/Transforms/InstCombine/wcslen-1.ll
index 8833754a53678..497e53bb28b0e 100644
--- a/llvm/test/Transforms/InstCombine/wcslen-1.ll
+++ b/llvm/test/Transforms/InstCombine/wcslen-1.ll
@@ -149,7 +149,7 @@ define i64 @test_no_simplify1() {
 define i64 @test_no_simplify2(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i32], ptr @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [7 x i32], ptr @null_hello, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
@@ -161,8 +161,8 @@ define i64 @test_no_simplify2(i32 %x) {
 define i64 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify2_no_null_opt(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i32], ptr @null_hello, i64 0, i64 [[TMP1]]
-; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr [[HELLO_P]])
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [7 x i32], ptr @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
   %hello_p = getelementptr inbounds [7 x i32], ptr @null_hello, i32 0, i32 %x
@@ -176,7 +176,7 @@ define i64 @test_no_simplify3(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify3(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[AND]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i32], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [13 x i32], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
@@ -190,8 +190,8 @@ define i64 @test_no_simplify3_no_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify3_no_null_opt(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[AND]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i32], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
-; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr [[HELLO_P]])
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [13 x i32], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
   %and = and i32 %x, 15
@@ -231,7 +231,7 @@ define i64 @fold_wcslen_1() {
 ; with an offset that isn't a multiple of the element size).
 define i64 @no_fold_wcslen_1() {
 ; CHECK-LABEL: @no_fold_wcslen_1(
-; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull getelementptr inbounds (i8, ptr @ws, i64 3))
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull getelementptr inbounds nuw (i8, ptr @ws, i64 3))
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
   %p = getelementptr [15 x i8], ptr @ws, i64 0, i64 3
@@ -246,7 +246,7 @@ define i64 @no_fold_wcslen_1() {
 ; with an offset that isn't a multiple of the element size).
 define i64 @no_fold_wcslen_2() {
 ; CHECK-LABEL: @no_fold_wcslen_2(
-; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull getelementptr inbounds (i8, ptr @s8, i64 3))
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull getelementptr inbounds nuw (i8, ptr @s8, i64 3))
 ; CHECK-NEXT:    ret i64 [[LEN]]
 ;
   %p = getelementptr [10 x i8], ptr @s8, i64 0, i64 3
diff --git a/llvm/test/Transforms/InstCombine/wcslen-3.ll b/llvm/test/Transforms/InstCombine/wcslen-3.ll
index 6cabe0570cfe4..65b5adadb805d 100644
--- a/llvm/test/Transforms/InstCombine/wcslen-3.ll
+++ b/llvm/test/Transforms/InstCombine/wcslen-3.ll
@@ -150,7 +150,7 @@ define i64 @test_no_simplify1() {
 define i64 @test_no_simplify2(i16 %x) {
 ; CHECK-LABEL: @test_no_simplify2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[X:%.*]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i16], ptr @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [7 x i16], ptr @null_hello, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
@@ -165,7 +165,7 @@ define i64 @test_no_simplify3(i16 %x) {
 ; CHECK-LABEL: @test_no_simplify3(
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[X:%.*]], 15
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i16 [[AND]] to i64
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i16], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw [13 x i16], ptr @null_hello_mid, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i64 [[HELLO_L]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/wcslen-5.ll b/llvm/test/Transforms/InstCombine/wcslen-5.ll
index 3e0cdf34a6422..8cb37be89b4fd 100644
--- a/llvm/test/Transforms/InstCombine/wcslen-5.ll
+++ b/llvm/test/Transforms/InstCombine/wcslen-5.ll
@@ -19,7 +19,7 @@ declare i64 @wcslen(ptr)
 
 define dso_local i64 @fold_wcslen_s3_pi_s5(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: @fold_wcslen_s3_pi_s5(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS3_PI]], ptr @ws5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -41,8 +41,8 @@ define dso_local i64 @fold_wcslen_s3_pi_p1_s5(i1 zeroext %0, i64 %1) {
 ; XFAIL-CHECK-NEXT:    [[SEL:%.*]] = select i1 %0, i64 [[DIF_I]], i64 5
 ; XFAIL-CHECK-NEXT:    ret i64 [[SEL]]
 ; CHECK-LABEL: @fold_wcslen_s3_pi_p1_s5(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
-; CHECK-NEXT:    [[PS3_PI_P1:%.*]] = getelementptr inbounds i8, ptr [[PS3_PI]], i64 4
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS3_PI_P1:%.*]] = getelementptr inbounds nuw i8, ptr [[PS3_PI]], i64 4
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS3_PI_P1]], ptr @ws5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -62,7 +62,7 @@ define dso_local i64 @fold_wcslen_s3_pi_p1_s5(i1 zeroext %0, i64 %1) {
 
 define dso_local i64 @call_wcslen_s5_3_pi_s5(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: @call_wcslen_s5_3_pi_s5(
-; CHECK-NEXT:    [[PS5_3_PI:%.*]] = getelementptr inbounds [10 x i32], ptr @ws5_3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_3_PI:%.*]] = getelementptr inbounds nuw [10 x i32], ptr @ws5_3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS5_3_PI]], ptr @ws5
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -79,7 +79,7 @@ define dso_local i64 @call_wcslen_s5_3_pi_s5(i1 zeroext %0, i64 %1) {
 
 define dso_local i64 @call_wcslen_s5_3_s5_pj(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: @call_wcslen_s5_3_s5_pj(
-; CHECK-NEXT:    [[PS5:%.*]] = getelementptr inbounds [6 x i32], ptr @ws5, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5:%.*]] = getelementptr inbounds nuw [6 x i32], ptr @ws5, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr @ws5_3, ptr [[PS5]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -96,7 +96,7 @@ define dso_local i64 @call_wcslen_s5_3_s5_pj(i1 zeroext %0, i64 %1) {
 
 define dso_local i64 @fold_wcslen_s3_s5_pj(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: @fold_wcslen_s3_s5_pj(
-; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds [6 x i32], ptr @ws5, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds nuw [6 x i32], ptr @ws5, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr @ws3, ptr [[PS5_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -115,7 +115,7 @@ define dso_local i64 @fold_wcslen_s3_s5_pj(i1 zeroext %0, i64 %1) {
 
 define dso_local i64 @call_wcslen_s3_s5_3_pj(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: @call_wcslen_s3_s5_3_pj(
-; CHECK-NEXT:    [[PS5_3_PJ:%.*]] = getelementptr inbounds [10 x i32], ptr @ws5_3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_3_PJ:%.*]] = getelementptr inbounds nuw [10 x i32], ptr @ws5_3, i64 0, i64 [[TMP1:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr @ws3, ptr [[PS5_3_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
@@ -132,8 +132,8 @@ define dso_local i64 @call_wcslen_s3_s5_3_pj(i1 zeroext %0, i64 %1) {
 
 define dso_local i64 @fold_wcslen_s3_pi_s5_pj(i1 zeroext %0, i64 %1, i64 %2) {
 ; CHECK-LABEL: @fold_wcslen_s3_pi_s5_pj(
-; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
-; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds [6 x i32], ptr @ws5, i64 0, i64 [[TMP2:%.*]]
+; CHECK-NEXT:    [[PS3_PI:%.*]] = getelementptr inbounds nuw [4 x i32], ptr @ws3, i64 0, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[PS5_PJ:%.*]] = getelementptr inbounds nuw [6 x i32], ptr @ws5, i64 0, i64 [[TMP2:%.*]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP0:%.*]], ptr [[PS3_PI]], ptr [[PS5_PJ]]
 ; CHECK-NEXT:    [[LEN:%.*]] = tail call i64 @wcslen(ptr nonnull [[SEL]])
 ; CHECK-NEXT:    ret i64 [[LEN]]
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/abs.ll b/llvm/test/Transforms/InstSimplify/ConstProp/abs.ll
index e313bd453a152..37233b0f29342 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/abs.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/abs.ll
@@ -6,7 +6,7 @@ declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
 
 define i8 @undef_val_min_poison() {
 ; CHECK-LABEL: @undef_val_min_poison(
-; CHECK-NEXT:    ret i8 undef
+; CHECK-NEXT:    ret i8 poison
 ;
   %r = call i8 @llvm.abs.i8(i8 undef, i1 true)
   ret i8 %r
@@ -22,7 +22,7 @@ define i8 @undef_val_min_not_poison() {
 
 define i8 @min_val_min_poison() {
 ; CHECK-LABEL: @min_val_min_poison(
-; CHECK-NEXT:    ret i8 undef
+; CHECK-NEXT:    ret i8 poison
 ;
   %r = call i8 @llvm.abs.i8(i8 -128, i1 true)
   ret i8 %r
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/gep-constanfolding-error.ll b/llvm/test/Transforms/InstSimplify/ConstProp/gep-constanfolding-error.ll
index e5287a45da4b6..a9b473f050d85 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/gep-constanfolding-error.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/gep-constanfolding-error.ll
@@ -44,7 +44,7 @@ entry:
   %9 = add i32 %f.promoted, %smax
   %10 = add i32 %9, 2
   call void @llvm.memset.p0.i32(ptr %scevgep, i8 %conv6, i32 %10, i1 false)
-; CHECK: call void @llvm.memset.p0.i32(ptr getelementptr inbounds (i8, ptr @j, i32 239), i8 %conv6, i32 1, i1 false)
+; CHECK: call void @llvm.memset.p0.i32(ptr getelementptr inbounds nuw (i8, ptr @j, i32 239), i8 %conv6, i32 1, i1 false)
   ret i32 0
 }
 ; Function Attrs: argmemonly nounwind
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/gep.ll b/llvm/test/Transforms/InstSimplify/ConstProp/gep.ll
index b3fe7f36ff97b..763257986e1f7 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/gep.ll
@@ -11,14 +11,14 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define ptr @f0() {
 ; CHECK-LABEL: @f0(
-; CHECK-NEXT:    ret ptr getelementptr inbounds inrange(-16, 8) (i8, ptr @vt, i64 16)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw inrange(-16, 8) (i8, ptr @vt, i64 16)
 ;
   ret ptr getelementptr (ptr, ptr getelementptr inbounds inrange(-8, 16) ([3 x ptr], ptr @vt, i64 0, i64 1), i64 1)
 }
 
 define ptr @f1() {
 ; CHECK-LABEL: @f1(
-; CHECK-NEXT:    ret ptr getelementptr inbounds inrange(-8, 0) (i8, ptr @vt, i64 16)
+; CHECK-NEXT:    ret ptr getelementptr inbounds nuw inrange(-8, 0) (i8, ptr @vt, i64 16)
 ;
   ret ptr getelementptr (ptr, ptr getelementptr inbounds inrange(0, 8) ([3 x ptr], ptr @vt, i64 0, i64 1), i64 1)
 }
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/icmp-global.ll b/llvm/test/Transforms/InstSimplify/ConstProp/icmp-global.ll
index 3ef9552eb79d6..69219562858d0 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/icmp-global.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/icmp-global.ll
@@ -125,7 +125,7 @@ define i1 @global_gep_ugt_null() {
 
 define i1 @global_gep_sgt_null() {
 ; CHECK-LABEL: @global_gep_sgt_null(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds (i8, ptr @g, i64 8), null
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds nuw (i8, ptr @g, i64 8), null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %gep = getelementptr inbounds [2 x i32], ptr @g, i64 1
@@ -233,7 +233,7 @@ define i1 @global_gep_ugt_global() {
 
 define i1 @global_gep_sgt_global() {
 ; CHECK-LABEL: @global_gep_sgt_global(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds (i8, ptr @g, i64 8), @g
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds nuw (i8, ptr @g, i64 8), @g
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %gep = getelementptr inbounds [2 x i32], ptr @g, i64 1
@@ -274,7 +274,7 @@ define i1 @global_gep_ugt_global_gep() {
 ; Should not fold due to signed comparison.
 define i1 @global_gep_sgt_global_gep() {
 ; CHECK-LABEL: @global_gep_sgt_global_gep(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds (i8, ptr @g, i64 4), @g
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt ptr getelementptr inbounds nuw (i8, ptr @g, i64 4), @g
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %gep2 = getelementptr inbounds [2 x i32], ptr @g, i64 0, i64 1
diff --git a/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll b/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
index a3ad5f0139142..d2656e291547c 100644
--- a/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
+++ b/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
@@ -284,3 +284,11 @@ define <1 x i32> @bitcast_constexpr_scalar_fp_to_vector_int() {
   %res = bitcast float 1.0 to <1 x i32>
   ret <1 x i32> %res
 }
+
+define <2 x i64> @bitcast_constexpr_4f32_2i64_1111() {
+; CHECK-LABEL: @bitcast_constexpr_4f32_2i64_1111(
+; CHECK-NEXT:    ret <2 x i64> splat (i64 4575657222473777152)
+;
+  %res = bitcast <4 x float> splat (float 1.0) to <2 x i64>
+  ret <2 x i64> %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/compare.ll b/llvm/test/Transforms/InstSimplify/compare.ll
index 21653d800dce2..222b25836e053 100644
--- a/llvm/test/Transforms/InstSimplify/compare.ll
+++ b/llvm/test/Transforms/InstSimplify/compare.ll
@@ -780,6 +780,32 @@ define i1 @lshr_nonzero_ult(i32 %x) {
   ret i1 %cmp
 }
 
+define i1 @lshr_nonzero_ugt(i32 %x) {
+; CHECK-LABEL: @lshr_nonzero_ugt(
+; CHECK-NEXT:    [[X_NE_0:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NE_0]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_ne_0 = icmp ne i32 %x, 0
+  call void @llvm.assume(i1 %x_ne_0)
+  %lhs = lshr i32 %x, 1
+  %cmp = icmp ugt i32 %lhs, %x
+  ret i1 %cmp
+}
+
+define i1 @lshr_nonzero_ule(i32 %x) {
+; CHECK-LABEL: @lshr_nonzero_ule(
+; CHECK-NEXT:    [[X_NE_0:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NE_0]])
+; CHECK-NEXT:    ret i1 true
+;
+  %x_ne_0 = icmp ne i32 %x, 0
+  call void @llvm.assume(i1 %x_ne_0)
+  %lhs = lshr i32 %x, 1
+  %cmp = icmp ule i32 %lhs, %x
+  ret i1 %cmp
+}
+
 ; Negative test - unknown shift amount
 define i1 @lshr_nonzero_neg_unknown(i32 %x, i32 %c) {
 ; CHECK-LABEL: @lshr_nonzero_neg_unknown(
@@ -3241,7 +3267,7 @@ define i1 @globals_inequal() {
 ; TODO: Never equal
 define i1 @globals_offset_inequal() {
 ; CHECK-LABEL: @globals_offset_inequal(
-; CHECK-NEXT:    [[RES:%.*]] = icmp ne ptr getelementptr inbounds (i8, ptr @A, i32 1), getelementptr inbounds (i8, ptr @B, i32 1)
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne ptr getelementptr inbounds nuw (i8, ptr @A, i32 1), getelementptr inbounds nuw (i8, ptr @B, i32 1)
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %a.off = getelementptr i8, ptr @A, i32 1
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
index b23494fc56aa4..a330f5cbc9268 100644
--- a/llvm/test/Transforms/InstSimplify/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
+; RUN: opt -S -passes=instsimplify -use-constant-int-for-fixed-length-splat < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/InstSimplify/icmp-monotonic.ll b/llvm/test/Transforms/InstSimplify/icmp-monotonic.ll
new file mode 100644
index 0000000000000..e1a4ee91bd15c
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp-monotonic.ll
@@ -0,0 +1,218 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
+
+define i1 @lshr_or_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_uge_swapped(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_uge_swapped(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp uge i32 %op2, %op1
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_ugt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_ugt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 false
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp ugt i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_ult_wrong_pred(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_ult_wrong_pred(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp ult i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_sle_wrong_pred(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_sle_wrong_pred(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp sle i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_swapped_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_swapped_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %z, %x
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_ule_invalid_swapped(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_or_ule_invalid_swapped(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %y, %x
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @and_uadd_sat_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @and_uadd_sat_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = and i32 %x, %y
+  %op2 = call i32 @llvm.uadd.sat(i32 %x, i32 %z)
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @urem_or_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @urem_or_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = urem i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @urem_or_ule_invalid_swapped(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @urem_or_ule_invalid_swapped(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = urem i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = urem i32 %y, %x
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @udiv_or_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @udiv_or_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = udiv i32 %x, %y
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @udiv_or_ule_invalid_swapped(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @udiv_or_ule_invalid_swapped(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = udiv i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = udiv i32 %y, %x
+  %op2 = or i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @usub_sat_uadd_sat_ule(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @usub_sat_uadd_sat_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %op1 = call i32 @llvm.usub.sat(i32 %x, i32 %y)
+  %op2 = call i32 @llvm.uadd.sat(i32 %x, i32 %z)
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @usub_sat_uadd_sat_ule_invalid_swapped(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @usub_sat_uadd_sat_ule_invalid_swapped(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[Y]], i32 [[X]])
+; CHECK-NEXT:    [[OP2:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X]], i32 [[Z]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = call i32 @llvm.usub.sat(i32 %y, i32 %x)
+  %op2 = call i32 @llvm.uadd.sat(i32 %x, i32 %z)
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_ule_no_common_op(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: define i1 @lshr_or_ule_no_common_op(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], i32 [[W:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[W]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %w, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
+
+define i1 @lshr_or_ule_nested(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: define i1 @lshr_or_ule_nested(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], i32 [[W:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[OP2:%.*]] = or i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[OP3:%.*]] = or i32 [[OP2]], [[W]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP3]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = or i32 %x, %z
+  %op3 = or i32 %op2, %w
+  %cmp = icmp ule i32 %op1, %op3
+  ret i1 %cmp
+}
+
+define i1 @lshr_add_ule_non_monotonic(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: define i1 @lshr_add_ule_non_monotonic(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[OP1:%.*]] = lshr i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[OP2:%.*]] = add i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OP1]], [[OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %op1 = lshr i32 %x, %y
+  %op2 = add i32 %x, %z
+  %cmp = icmp ule i32 %op1, %op2
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstSimplify/past-the-end.ll b/llvm/test/Transforms/InstSimplify/past-the-end.ll
index cc2fe6f36fcd8..2517c79fe0cc0 100644
--- a/llvm/test/Transforms/InstSimplify/past-the-end.ll
+++ b/llvm/test/Transforms/InstSimplify/past-the-end.ll
@@ -21,7 +21,7 @@ define zeroext i1 @no_offsets() {
 
 define zeroext i1 @both_past_the_end() {
 ; CHECK-LABEL: @both_past_the_end(
-; CHECK-NEXT:    [[T:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @opte_a, i32 4), getelementptr inbounds (i8, ptr @opte_b, i32 4)
+; CHECK-NEXT:    [[T:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @opte_a, i32 4), getelementptr inbounds nuw (i8, ptr @opte_b, i32 4)
 ; CHECK-NEXT:    ret i1 [[T]]
 ;
   %x = getelementptr i32, ptr @opte_a, i32 1
@@ -36,7 +36,7 @@ define zeroext i1 @both_past_the_end() {
 
 define zeroext i1 @just_one_past_the_end() {
 ; CHECK-LABEL: @just_one_past_the_end(
-; CHECK-NEXT:    [[T:%.*]] = icmp eq ptr getelementptr inbounds (i8, ptr @opte_a, i32 4), @opte_b
+; CHECK-NEXT:    [[T:%.*]] = icmp eq ptr getelementptr inbounds nuw (i8, ptr @opte_a, i32 4), @opte_b
 ; CHECK-NEXT:    ret i1 [[T]]
 ;
   %x = getelementptr i32, ptr @opte_a, i32 1
diff --git a/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll b/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
index cda3f3809606c..fcbca72512a32 100644
--- a/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
+++ b/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
@@ -25,7 +25,7 @@ entry:
 }
 
 ; CHECK-LABEL: define void @f1
-; CHECK: %1 = load ptr, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) @__aaa_struct_ptr to ptr addrspace(4)), i64 16), align 8
+; CHECK: %1 = load ptr, ptr addrspace(4) getelementptr inbounds nuw (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) @__aaa_struct_ptr to ptr addrspace(4)), i64 16), align 8
 
 ; Function Attrs: alwaysinline nounwind
 define void @f1(ptr addrspace(1)) #1 {
diff --git a/llvm/test/Transforms/InstSimplify/sincos.ll b/llvm/test/Transforms/InstSimplify/sincos.ll
new file mode 100644
index 0000000000000..e0f81ee45af05
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/sincos.ll
@@ -0,0 +1,148 @@
+; RUN: opt -S -passes=instsimplify %s | FileCheck %s
+
+define { float, float } @sincos_zero() {
+; CHECK-LABEL: define { float, float } @sincos_zero() {
+; CHECK-NEXT:    ret { float, float } { float 0.000000e+00, float 1.000000e+00 }
+;
+  %ret = call { float, float } @llvm.sincos.f32(float zeroinitializer)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_neg_zero() {
+; CHECK-LABEL: define { float, float } @sincos_neg_zero() {
+; CHECK-NEXT:    ret { float, float } { float -0.000000e+00, float 1.000000e+00 }
+;
+  %ret = call { float, float } @llvm.sincos.f32(float -0.0)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_one() {
+; CHECK-LABEL: define { float, float } @sincos_one() {
+; CHECK-NEXT:    ret { float, float } { float [[$SIN_ONE:.+]], float [[$COS_ONE:.+]] }
+;
+  %ret = call { float, float } @llvm.sincos.f32(float 1.0)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_two() {
+; CHECK-LABEL: define { float, float } @sincos_two() {
+; CHECK-NEXT:    ret { float, float } { float [[$SIN_TWO:.+]], float [[$COS_TWO:.+]] }
+;
+  %ret = call { float, float } @llvm.sincos.f32(float 2.0)
+  ret { float, float } %ret
+}
+
+define { <2 x float>, <2 x float> } @sincos_vector() {
+; CHECK-LABEL: define { <2 x float>, <2 x float> } @sincos_vector() {
+; CHECK-NEXT:    ret { <2 x float>, <2 x float> } { <2 x float> <float [[$SIN_ONE]], float [[$SIN_TWO]]>, <2 x float> <float [[$COS_ONE]], float [[$COS_TWO]]> }
+;
+  %ret = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> <float 1.0, float 2.0>)
+  ret { <2 x float>, <2 x float> } %ret
+}
+
+define { <2 x float>, <2 x float> } @sincos_zero_vector() {
+; CHECK-LABEL: define { <2 x float>, <2 x float> } @sincos_zero_vector() {
+; CHECK-NEXT:    ret { <2 x float>, <2 x float> } { <2 x float> zeroinitializer, <2 x float> splat (float 1.000000e+00) }
+;
+  %ret = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> zeroinitializer)
+  ret { <2 x float>, <2 x float> } %ret
+}
+
+define { float, float } @sincos_poison() {
+; CHECK-LABEL: define { float, float } @sincos_poison() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float poison)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float poison)
+  ret { float, float } %ret
+}
+
+define { <2 x float>, <2 x float> } @sincos_poison_vector() {
+; CHECK-LABEL: define { <2 x float>, <2 x float> } @sincos_poison_vector() {
+; CHECK-NEXT:    [[RET:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> poison)
+; CHECK-NEXT:    ret { <2 x float>, <2 x float> } [[RET]]
+;
+  %ret = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> poison)
+  ret { <2 x float>, <2 x float> } %ret
+}
+
+define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_poison_scalable_vector() {
+; CHECK-LABEL: define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_poison_scalable_vector() {
+; CHECK-NEXT:    [[RET:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:    ret { <vscale x 2 x float>, <vscale x 2 x float> } [[RET]]
+;
+  %ret = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> poison)
+  ret { <vscale x 2 x float>, <vscale x 2 x float> } %ret
+}
+
+define { float, float } @sincos_undef() {
+; CHECK-LABEL: define { float, float } @sincos_undef() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float undef)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float undef)
+  ret { float, float } %ret
+}
+
+define { <2 x float>, <2 x float> } @sincos_undef_vector() {
+; CHECK-LABEL: define { <2 x float>, <2 x float> } @sincos_undef_vector() {
+; CHECK-NEXT:    [[RET:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> undef)
+; CHECK-NEXT:    ret { <2 x float>, <2 x float> } [[RET]]
+;
+  %ret = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> undef)
+  ret { <2 x float>, <2 x float> } %ret
+}
+
+define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_undef_scalable_vector() {
+; CHECK-LABEL: define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_undef_scalable_vector() {
+; CHECK-NEXT:    [[RET:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:    ret { <vscale x 2 x float>, <vscale x 2 x float> } [[RET]]
+;
+  %ret = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> undef)
+  ret { <vscale x 2 x float>, <vscale x 2 x float> } %ret
+}
+
+define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_zero_scalable_vector() {
+; CHECK-LABEL: define { <vscale x 2 x float>, <vscale x 2 x float> } @sincos_zero_scalable_vector() {
+; CHECK-NEXT:    [[RET:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> zeroinitializer)
+; CHECK-NEXT:    ret { <vscale x 2 x float>, <vscale x 2 x float> } [[RET]]
+;
+  %ret = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.sincos.nxv2f32(<vscale x 2 x float> zeroinitializer)
+  ret { <vscale x 2 x float>, <vscale x 2 x float> } %ret
+}
+
+define { float, float } @sincos_inf() {
+; CHECK-LABEL: define { float, float } @sincos_inf() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float 0x7FF0000000000000)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float 0x7FF0000000000000)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_neginf() {
+; CHECK-LABEL: define { float, float } @sincos_neginf() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float 0xFFF0000000000000)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float 0xFFF0000000000000)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_qnan() {
+; CHECK-LABEL: define { float, float } @sincos_qnan() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float 0x7FF8000000000000)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float 0x7FF8000000000000)
+  ret { float, float } %ret
+}
+
+define { float, float } @sincos_snan() {
+; CHECK-LABEL: define { float, float } @sincos_snan() {
+; CHECK-NEXT:    [[RET:%.*]] = call { float, float } @llvm.sincos.f32(float 0x7FF0000020000000)
+; CHECK-NEXT:    ret { float, float } [[RET]]
+;
+  %ret = call { float, float } @llvm.sincos.f32(float bitcast (i32 2139095041 to float))
+  ret { float, float } %ret
+}
diff --git a/llvm/test/Transforms/LoopRotate/dbgvalue.ll b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
index b58d821fbca5d..026306a4637f0 100644
--- a/llvm/test/Transforms/LoopRotate/dbgvalue.ll
+++ b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
@@ -113,7 +113,7 @@ return:                                           ; preds = %if.end
 
 ; Check that the dbg.values move up to being immediately below the PHIs,
 ; using their Values. However once we exit the loop, the x and y values
-; become irrelevant and undef, only z gets an LCSSA PHI to preserve it.
+; become irrelevant and poison, only z gets an LCSSA PHI to preserve it.
 ;
 ; Note that while the icmp is initially undominated by any dbg.value and thus
 ; shouldn't get a variable location, the first iteration is peeled off into the
@@ -138,8 +138,8 @@ define i32 @tak2(i32 %x, i32 %y, i32 %z) nounwind ssp !dbg !21 {
 ; CHECK:      tail call i32 @tak(i32
 ; CHECK: if.end:
 ; CHECK-NEXT: z.tr.lcssa = phi i32
-; CHECK-NEXT: #dbg_value(i32 undef
-; CHECK-NEXT: #dbg_value(i32 undef
+; CHECK-NEXT: #dbg_value(i32 poison
+; CHECK-NEXT: #dbg_value(i32 poison
 ; CHECK-NEXT: #dbg_value(i32 %z.tr.lcssa
 entry:
   br label %tailrecurse
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll
new file mode 100644
index 0000000000000..d27d5e74e28f2
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll
@@ -0,0 +1,357 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-unroll -mcpu=apple-m1 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m2 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m3 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=apple-m4 -S %s | FileCheck --check-prefix=APPLE %s
+; RUN: opt -p loop-unroll -mcpu=cortex-a57 -S %s | FileCheck --check-prefix=OTHER %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define void @small_load_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale) {
+; APPLE-LABEL: define void @small_load_store_loop(
+; APPLE-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]]) #[[ATTR0:[0-9]+]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; APPLE-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 7
+; APPLE-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
+; APPLE-NEXT:    br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]]
+; APPLE:       [[ENTRY_NEW]]:
+; APPLE-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; APPLE-NEXT:    br label %[[LOOP:.*]]
+; APPLE:       [[LOOP]]:
+; APPLE-NEXT:    [[IV_EPIL:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_7:%.*]], %[[LOOP]] ]
+; APPLE-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[LOOP]] ]
+; APPLE-NEXT:    [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_EPIL]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_EPIL:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL]]
+; APPLE-NEXT:    [[L_EPIL:%.*]] = load float, ptr [[GEP_SRC_EPIL]], align 4
+; APPLE-NEXT:    [[GEP_DST_EPIL:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_EPIL]]
+; APPLE-NEXT:    store float [[L_EPIL]], ptr [[GEP_DST_EPIL]], align 4
+; APPLE-NEXT:    [[IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[IV_EPIL]], 1
+; APPLE-NEXT:    [[SCALED_IV_1:%.*]] = mul nuw nsw i64 [[IV_NEXT_EPIL]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_1]]
+; APPLE-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
+; APPLE-NEXT:    [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_EPIL]]
+; APPLE-NEXT:    store float [[L_1]], ptr [[GEP_DST_1]], align 4
+; APPLE-NEXT:    [[IV_NEXT_1:%.*]] = add nuw nsw i64 [[IV_EPIL]], 2
+; APPLE-NEXT:    [[SCALED_IV_2:%.*]] = mul nuw nsw i64 [[IV_NEXT_1]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_2]]
+; APPLE-NEXT:    [[L_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
+; APPLE-NEXT:    [[GEP_DST_2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_1]]
+; APPLE-NEXT:    store float [[L_2]], ptr [[GEP_DST_2]], align 4
+; APPLE-NEXT:    [[IV_NEXT_2:%.*]] = add nuw nsw i64 [[IV_EPIL]], 3
+; APPLE-NEXT:    [[SCALED_IV_3:%.*]] = mul nuw nsw i64 [[IV_NEXT_2]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_3]]
+; APPLE-NEXT:    [[L_3:%.*]] = load float, ptr [[GEP_SRC_3]], align 4
+; APPLE-NEXT:    [[GEP_DST_3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_2]]
+; APPLE-NEXT:    store float [[L_3]], ptr [[GEP_DST_3]], align 4
+; APPLE-NEXT:    [[IV_NEXT_3:%.*]] = add nuw nsw i64 [[IV_EPIL]], 4
+; APPLE-NEXT:    [[SCALED_IV_4:%.*]] = mul nuw nsw i64 [[IV_NEXT_3]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_4]]
+; APPLE-NEXT:    [[L_4:%.*]] = load float, ptr [[GEP_SRC_4]], align 4
+; APPLE-NEXT:    [[GEP_DST_4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_3]]
+; APPLE-NEXT:    store float [[L_4]], ptr [[GEP_DST_4]], align 4
+; APPLE-NEXT:    [[IV_NEXT_4:%.*]] = add nuw nsw i64 [[IV_EPIL]], 5
+; APPLE-NEXT:    [[SCALED_IV_5:%.*]] = mul nuw nsw i64 [[IV_NEXT_4]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_5]]
+; APPLE-NEXT:    [[L_5:%.*]] = load float, ptr [[GEP_SRC_5]], align 4
+; APPLE-NEXT:    [[GEP_DST_5:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_4]]
+; APPLE-NEXT:    store float [[L_5]], ptr [[GEP_DST_5]], align 4
+; APPLE-NEXT:    [[IV_NEXT_5:%.*]] = add nuw nsw i64 [[IV_EPIL]], 6
+; APPLE-NEXT:    [[SCALED_IV_6:%.*]] = mul nuw nsw i64 [[IV_NEXT_5]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_6:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_6]]
+; APPLE-NEXT:    [[L_6:%.*]] = load float, ptr [[GEP_SRC_6]], align 4
+; APPLE-NEXT:    [[GEP_DST_6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_5]]
+; APPLE-NEXT:    store float [[L_6]], ptr [[GEP_DST_6]], align 4
+; APPLE-NEXT:    [[IV_NEXT_6:%.*]] = add nuw nsw i64 [[IV_EPIL]], 7
+; APPLE-NEXT:    [[SCALED_IV_7:%.*]] = mul nuw nsw i64 [[IV_NEXT_6]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_7:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_7]]
+; APPLE-NEXT:    [[L_7:%.*]] = load float, ptr [[GEP_SRC_7]], align 4
+; APPLE-NEXT:    [[GEP_DST_7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT_6]]
+; APPLE-NEXT:    store float [[L_7]], ptr [[GEP_DST_7]], align 4
+; APPLE-NEXT:    [[IV_NEXT_7]] = add nuw nsw i64 [[IV_EPIL]], 8
+; APPLE-NEXT:    [[NITER_NEXT_7]] = add i64 [[NITER]], 8
+; APPLE-NEXT:    [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]]
+; APPLE-NEXT:    br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]]
+; APPLE:       [[EXIT_UNR_LCSSA_LOOPEXIT]]:
+; APPLE-NEXT:    [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ]
+; APPLE-NEXT:    br label %[[EXIT_UNR_LCSSA]]
+; APPLE:       [[EXIT_UNR_LCSSA]]:
+; APPLE-NEXT:    [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ]
+; APPLE-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; APPLE-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]]
+; APPLE:       [[LOOP_EPIL_PREHEADER]]:
+; APPLE-NEXT:    br label %[[LOOP_EPIL:.*]]
+; APPLE:       [[LOOP_EPIL]]:
+; APPLE-NEXT:    [[IV_EPIL1:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL1:%.*]], %[[LOOP_EPIL]] ]
+; APPLE-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ]
+; APPLE-NEXT:    [[SCALED_IV_EPIL1:%.*]] = mul nuw nsw i64 [[IV_EPIL1]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_EPIL1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL1]]
+; APPLE-NEXT:    [[L_EPIL1:%.*]] = load float, ptr [[GEP_SRC_EPIL1]], align 4
+; APPLE-NEXT:    [[GEP_DST_EPIL1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_EPIL1]]
+; APPLE-NEXT:    store float [[L_EPIL1]], ptr [[GEP_DST_EPIL1]], align 4
+; APPLE-NEXT:    [[IV_NEXT_EPIL1]] = add nuw nsw i64 [[IV_EPIL1]], 1
+; APPLE-NEXT:    [[EC_EPIL:%.*]] = icmp eq i64 [[IV_NEXT_EPIL1]], [[N]]
+; APPLE-NEXT:    [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1
+; APPLE-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]]
+; APPLE-NEXT:    br i1 [[EPIL_ITER_CMP]], label %[[LOOP_EPIL]], label %[[EXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; APPLE:       [[EXIT_EPILOG_LCSSA]]:
+; APPLE-NEXT:    br label %[[EXIT]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    ret void
+;
+; OTHER-LABEL: define void @small_load_store_loop(
+; OTHER-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]]) #[[ATTR0:[0-9]+]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; OTHER-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; OTHER-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; OTHER-NEXT:    br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]]
+; OTHER:       [[ENTRY_NEW]]:
+; OTHER-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; OTHER-NEXT:    br label %[[LOOP:.*]]
+; OTHER:       [[LOOP]]:
+; OTHER-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; OTHER-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; OTHER-NEXT:    [[SCALED_IV:%.*]] = mul nuw nsw i64 [[IV]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV]]
+; OTHER-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; OTHER-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; OTHER-NEXT:    store float [[L]], ptr [[GEP_DST]], align 4
+; OTHER-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; OTHER-NEXT:    [[SCALED_IV_1:%.*]] = mul nuw nsw i64 [[IV_NEXT]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_1]]
+; OTHER-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
+; OTHER-NEXT:    [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT]]
+; OTHER-NEXT:    store float [[L_1]], ptr [[GEP_DST_1]], align 4
+; OTHER-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; OTHER-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; OTHER-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; OTHER-NEXT:    br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]]
+; OTHER:       [[EXIT_UNR_LCSSA_LOOPEXIT]]:
+; OTHER-NEXT:    [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; OTHER-NEXT:    br label %[[EXIT_UNR_LCSSA]]
+; OTHER:       [[EXIT_UNR_LCSSA]]:
+; OTHER-NEXT:    [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ]
+; OTHER-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; OTHER-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]]
+; OTHER:       [[LOOP_EPIL_PREHEADER]]:
+; OTHER-NEXT:    br label %[[LOOP_EPIL:.*]]
+; OTHER:       [[LOOP_EPIL]]:
+; OTHER-NEXT:    [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC_EPIL:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL]]
+; OTHER-NEXT:    [[L_EPIL:%.*]] = load float, ptr [[GEP_SRC_EPIL]], align 4
+; OTHER-NEXT:    [[GEP_DST_EPIL:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_UNR]]
+; OTHER-NEXT:    store float [[L_EPIL]], ptr [[GEP_DST_EPIL]], align 4
+; OTHER-NEXT:    br label %[[EXIT]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %scaled.iv = mul nuw nsw i64 %iv, %scale
+  %gep.src = getelementptr inbounds float, ptr %src, i64 %scaled.iv
+  %l = load float, ptr %gep.src, align 4
+  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+  store float %l, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+@A = external constant [9 x i8], align 1
+@B = external constant [8 x i32], align 4
+@C = external constant [8 x i32], align 4
+
+define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x, i32 %width, i32 %t.1, i32 %t.2) {
+; APPLE-LABEL: define void @early_continue_dep_on_load_large(
+; APPLE-SAME: ptr [[P_1:%.*]], ptr [[P_2:%.*]], i64 [[N:%.*]], i32 [[X:%.*]], i32 [[WIDTH:%.*]], i32 [[T_1:%.*]], i32 [[T_2:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    br label %[[LOOP_HEADER:.*]]
+; APPLE:       [[LOOP_HEADER]]:
+; APPLE-NEXT:    [[IV_EPIL:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_LATCH:.*]] ]
+; APPLE-NEXT:    [[GEP_EPIL:%.*]] = getelementptr { i32, i8, i8, [2 x i8] }, ptr [[P_1]], i64 [[IV_EPIL]]
+; APPLE-NEXT:    [[L_1_EPIL:%.*]] = load i32, ptr [[GEP_EPIL]], align 4
+; APPLE-NEXT:    [[CMP6_NOT_EPIL:%.*]] = icmp sgt i32 [[L_1_EPIL]], [[T_1]]
+; APPLE-NEXT:    br i1 [[CMP6_NOT_EPIL]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[THEN]]:
+; APPLE-NEXT:    [[GEP_4_EPIL:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_EPIL]], i64 4
+; APPLE-NEXT:    [[L_2_EPIL:%.*]] = load i8, ptr [[GEP_4_EPIL]], align 4
+; APPLE-NEXT:    [[OR_COND_EPIL:%.*]] = icmp ugt i8 [[L_2_EPIL]], 7
+; APPLE-NEXT:    br i1 [[OR_COND_EPIL]], label %[[MERGE:.*]], label %[[ELSE:.*]]
+; APPLE:       [[ELSE]]:
+; APPLE-NEXT:    [[CONV_I_EPIL:%.*]] = zext nneg i8 [[L_2_EPIL]] to i64
+; APPLE-NEXT:    [[ARRAYIDX_I_EPIL:%.*]] = getelementptr inbounds [9 x i8], ptr @A, i64 0, i64 [[CONV_I_EPIL]]
+; APPLE-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX_I_EPIL]], align 1
+; APPLE-NEXT:    [[IDXPROM_I_EPIL:%.*]] = sext i8 [[TMP27]] to i64
+; APPLE-NEXT:    [[ARRAYIDX_I37_EPIL:%.*]] = getelementptr inbounds [8 x i32], ptr @B, i64 0, i64 [[IDXPROM_I_EPIL]]
+; APPLE-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX_I37_EPIL]], align 4
+; APPLE-NEXT:    [[ARRAYIDX_I42_EPIL:%.*]] = getelementptr inbounds [8 x i32], ptr @C, i64 0, i64 [[IDXPROM_I_EPIL]]
+; APPLE-NEXT:    [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX_I42_EPIL]], align 4
+; APPLE-NEXT:    br label %[[MERGE]]
+; APPLE:       [[MERGE]]:
+; APPLE-NEXT:    [[RETVAL_0_I3851_EPIL:%.*]] = phi i32 [ 0, %[[THEN]] ], [ [[TMP28]], %[[ELSE]] ]
+; APPLE-NEXT:    [[RETVAL_0_I43_EPIL:%.*]] = phi i32 [ 0, %[[THEN]] ], [ [[TMP29]], %[[ELSE]] ]
+; APPLE-NEXT:    [[ADD14_EPIL:%.*]] = add nsw i32 [[RETVAL_0_I43_EPIL]], [[X]]
+; APPLE-NEXT:    [[MUL15_EPIL:%.*]] = mul nsw i32 [[ADD14_EPIL]], [[WIDTH]]
+; APPLE-NEXT:    [[TMP30:%.*]] = trunc nuw nsw i64 [[IV_EPIL]] to i32
+; APPLE-NEXT:    [[ADD16_EPIL:%.*]] = add nsw i32 [[RETVAL_0_I3851_EPIL]], [[TMP30]]
+; APPLE-NEXT:    [[ADD17_EPIL:%.*]] = add nsw i32 [[ADD16_EPIL]], [[MUL15_EPIL]]
+; APPLE-NEXT:    [[IDXPROM18_EPIL:%.*]] = sext i32 [[ADD17_EPIL]] to i64
+; APPLE-NEXT:    [[ARRAYIDX19_EPIL:%.*]] = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr [[P_2]], i64 [[IDXPROM18_EPIL]]
+; APPLE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX19_EPIL]], align 4
+; APPLE-NEXT:    [[SUB_EPIL:%.*]] = sub nsw i32 [[X]], [[RETVAL_0_I43_EPIL]]
+; APPLE-NEXT:    [[MUL21_EPIL:%.*]] = mul nsw i32 [[SUB_EPIL]], [[WIDTH]]
+; APPLE-NEXT:    [[SUB22_EPIL:%.*]] = sub i32 [[TMP30]], [[RETVAL_0_I3851_EPIL]]
+; APPLE-NEXT:    [[ADD23_EPIL:%.*]] = add nsw i32 [[SUB22_EPIL]], [[MUL21_EPIL]]
+; APPLE-NEXT:    [[IDXPROM24_EPIL:%.*]] = sext i32 [[ADD23_EPIL]] to i64
+; APPLE-NEXT:    [[ARRAYIDX25_EPIL:%.*]] = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr [[P_2]], i64 [[IDXPROM24_EPIL]]
+; APPLE-NEXT:    [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX25_EPIL]], align 4
+; APPLE-NEXT:    [[CMP27_EPIL:%.*]] = icmp sgt i32 [[L_1_EPIL]], [[TMP31]]
+; APPLE-NEXT:    [[CMP28_EPIL:%.*]] = icmp sgt i32 [[L_1_EPIL]], [[TMP32]]
+; APPLE-NEXT:    [[AND34_EPIL:%.*]] = and i1 [[CMP27_EPIL]], [[CMP28_EPIL]]
+; APPLE-NEXT:    br i1 [[AND34_EPIL]], label %[[STORE_RES:.*]], label %[[LOOP_LATCH]]
+; APPLE:       [[STORE_RES]]:
+; APPLE-NEXT:    [[CMP32_EPIL:%.*]] = icmp sgt i32 [[L_1_EPIL]], [[T_2]]
+; APPLE-NEXT:    [[GEP_5_EPIL:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_EPIL]], i64 5
+; APPLE-NEXT:    [[RES_EPIL:%.*]] = select i1 [[CMP32_EPIL]], i8 1, i8 2
+; APPLE-NEXT:    store i8 [[RES_EPIL]], ptr [[GEP_5_EPIL]], align 1
+; APPLE-NEXT:    br label %[[LOOP_LATCH]]
+; APPLE:       [[LOOP_LATCH]]:
+; APPLE-NEXT:    [[IV_NEXT_EPIL]] = add nuw nsw i64 [[IV_EPIL]], 1
+; APPLE-NEXT:    [[EC_EPIL:%.*]] = icmp eq i64 [[IV_NEXT_EPIL]], [[N]]
+; APPLE-NEXT:    br i1 [[EC_EPIL]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    ret void
+;
+; OTHER-LABEL: define void @early_continue_dep_on_load_large(
+; OTHER-SAME: ptr [[P_1:%.*]], ptr [[P_2:%.*]], i64 [[N:%.*]], i32 [[X:%.*]], i32 [[WIDTH:%.*]], i32 [[T_1:%.*]], i32 [[T_2:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    br label %[[LOOP_HEADER:.*]]
+; OTHER:       [[LOOP_HEADER]]:
+; OTHER-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; OTHER-NEXT:    [[GEP:%.*]] = getelementptr { i32, i8, i8, [2 x i8] }, ptr [[P_1]], i64 [[IV]]
+; OTHER-NEXT:    [[L_1:%.*]] = load i32, ptr [[GEP]], align 4
+; OTHER-NEXT:    [[CMP6_NOT:%.*]] = icmp sgt i32 [[L_1]], [[T_1]]
+; OTHER-NEXT:    br i1 [[CMP6_NOT]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[THEN]]:
+; OTHER-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP]], i64 4
+; OTHER-NEXT:    [[L_2:%.*]] = load i8, ptr [[GEP_4]], align 4
+; OTHER-NEXT:    [[OR_COND:%.*]] = icmp ugt i8 [[L_2]], 7
+; OTHER-NEXT:    br i1 [[OR_COND]], label %[[MERGE:.*]], label %[[ELSE:.*]]
+; OTHER:       [[ELSE]]:
+; OTHER-NEXT:    [[CONV_I:%.*]] = zext nneg i8 [[L_2]] to i64
+; OTHER-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [9 x i8], ptr @A, i64 0, i64 [[CONV_I]]
+; OTHER-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX_I]], align 1
+; OTHER-NEXT:    [[IDXPROM_I:%.*]] = sext i8 [[TMP0]] to i64
+; OTHER-NEXT:    [[ARRAYIDX_I37:%.*]] = getelementptr inbounds [8 x i32], ptr @B, i64 0, i64 [[IDXPROM_I]]
+; OTHER-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX_I37]], align 4
+; OTHER-NEXT:    [[ARRAYIDX_I42:%.*]] = getelementptr inbounds [8 x i32], ptr @C, i64 0, i64 [[IDXPROM_I]]
+; OTHER-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX_I42]], align 4
+; OTHER-NEXT:    br label %[[MERGE]]
+; OTHER:       [[MERGE]]:
+; OTHER-NEXT:    [[RETVAL_0_I3851:%.*]] = phi i32 [ 0, %[[THEN]] ], [ [[TMP1]], %[[ELSE]] ]
+; OTHER-NEXT:    [[RETVAL_0_I43:%.*]] = phi i32 [ 0, %[[THEN]] ], [ [[TMP2]], %[[ELSE]] ]
+; OTHER-NEXT:    [[ADD14:%.*]] = add nsw i32 [[RETVAL_0_I43]], [[X]]
+; OTHER-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[ADD14]], [[WIDTH]]
+; OTHER-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; OTHER-NEXT:    [[ADD16:%.*]] = add nsw i32 [[RETVAL_0_I3851]], [[TMP3]]
+; OTHER-NEXT:    [[ADD17:%.*]] = add nsw i32 [[ADD16]], [[MUL15]]
+; OTHER-NEXT:    [[IDXPROM18:%.*]] = sext i32 [[ADD17]] to i64
+; OTHER-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr [[P_2]], i64 [[IDXPROM18]]
+; OTHER-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX19]], align 4
+; OTHER-NEXT:    [[SUB:%.*]] = sub nsw i32 [[X]], [[RETVAL_0_I43]]
+; OTHER-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[SUB]], [[WIDTH]]
+; OTHER-NEXT:    [[SUB22:%.*]] = sub i32 [[TMP3]], [[RETVAL_0_I3851]]
+; OTHER-NEXT:    [[ADD23:%.*]] = add nsw i32 [[SUB22]], [[MUL21]]
+; OTHER-NEXT:    [[IDXPROM24:%.*]] = sext i32 [[ADD23]] to i64
+; OTHER-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr [[P_2]], i64 [[IDXPROM24]]
+; OTHER-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX25]], align 4
+; OTHER-NEXT:    [[CMP27:%.*]] = icmp sgt i32 [[L_1]], [[TMP4]]
+; OTHER-NEXT:    [[CMP28:%.*]] = icmp sgt i32 [[L_1]], [[TMP5]]
+; OTHER-NEXT:    [[AND34:%.*]] = and i1 [[CMP27]], [[CMP28]]
+; OTHER-NEXT:    br i1 [[AND34]], label %[[STORE_RES:.*]], label %[[LOOP_LATCH]]
+; OTHER:       [[STORE_RES]]:
+; OTHER-NEXT:    [[CMP32:%.*]] = icmp sgt i32 [[L_1]], [[T_2]]
+; OTHER-NEXT:    [[GEP_5:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP]], i64 5
+; OTHER-NEXT:    [[RES:%.*]] = select i1 [[CMP32]], i8 1, i8 2
+; OTHER-NEXT:    store i8 [[RES]], ptr [[GEP_5]], align 1
+; OTHER-NEXT:    br label %[[LOOP_LATCH]]
+; OTHER:       [[LOOP_LATCH]]:
+; OTHER-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; OTHER-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; OTHER-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr { i32, i8, i8, [2 x i8] }, ptr %p.1, i64 %iv
+  %l.1 = load i32, ptr %gep, align 4
+  %c.1 = icmp sgt i32 %l.1, %t.1
+  br i1 %c.1, label %then, label %loop.latch
+
+then:
+  %gep.4 = getelementptr inbounds nuw i8, ptr %gep, i64 4
+  %l.2 = load i8, ptr %gep.4, align 4
+  %c.2 = icmp ugt i8 %l.2, 7
+  br i1 %c.2, label %merge, label %else
+
+else:
+  %conv.i = zext nneg i8 %l.2 to i64
+  %gep.A = getelementptr inbounds [9 x i8], ptr @A, i64 0, i64 %conv.i
+  %l.3 = load i8, ptr %gep.A, align 1
+  %idxprom.i = sext i8 %l.3 to i64
+  %gep.B = getelementptr inbounds [8 x i32], ptr @B, i64 0, i64 %idxprom.i
+  %l.4 = load i32, ptr %gep.B, align 4
+  %gep.C = getelementptr inbounds [8 x i32], ptr @C, i64 0, i64 %idxprom.i
+  %l.5 = load i32, ptr %gep.C, align 4
+  br label %merge
+
+merge:
+  %merge.1 = phi i32 [ 0, %then ], [ %l.4, %else ]
+  %merge.2 = phi i32 [ 0, %then ], [ %l.5, %else ]
+  %add14 = add nsw i32 %merge.2, %x
+  %mul15 = mul nsw i32 %add14, %width
+  %12 = trunc nuw nsw i64 %iv to i32
+  %add16 = add nsw i32 %merge.1, %12
+  %add17 = add nsw i32 %add16, %mul15
+  %idxprom18 = sext i32 %add17 to i64
+  %gep.p.2 = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr %p.2, i64 %idxprom18
+  %l.6 = load i32, ptr %gep.p.2, align 4
+  %sub = sub nsw i32 %x, %merge.2
+  %mul21 = mul nsw i32 %sub, %width
+  %sub22 = sub i32 %12, %merge.1
+  %add23 = add nsw i32 %sub22, %mul21
+  %idxprom24 = sext i32 %add23 to i64
+  %gep.p2.1 = getelementptr inbounds { i32, i8, i8, [2 x i8] }, ptr %p.2, i64 %idxprom24
+  %l.7 = load i32, ptr %gep.p2.1, align 4
+  %c.3 = icmp sgt i32 %l.1, %l.6
+  %c.4 = icmp sgt i32 %l.1, %l.7
+  %and34 = and i1 %c.3, %c.4
+  br i1 %and34, label %store.res, label %loop.latch
+
+store.res:
+  %c.5 = icmp sgt i32 %l.1, %t.2
+  %gep.5 = getelementptr inbounds nuw i8, ptr %gep, i64 5
+  %res = select i1 %c.5, i8 1, i8 2
+  store i8 %res, ptr %gep.5, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
index cba80a3a2bc5f..b8215d9b6f33c 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
@@ -19,50 +19,50 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
 ; CHECK-A55:       for.body6:
 ; CHECK-A55-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY6_PREHEADER_NEW]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_BODY6]] ]
 ; CHECK-A55-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[FOR_BODY6_PREHEADER_NEW]] ], [ [[NITER_NEXT_3:%.*]], [[FOR_BODY6]] ]
-; CHECK-A55-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[ARG_2:%.*]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-A55-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; CHECK-A55-NEXT:    [[CONV:%.*]] = sext i16 [[TMP1]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i16, ptr [[ARG_3:%.*]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-A55-NEXT:    [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX14]], align 2
 ; CHECK-A55-NEXT:    [[CONV15:%.*]] = sext i16 [[TMP2]] to i32
 ; CHECK-A55-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[CONV15]], [[CONV]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, ptr [[ARG_1:%.*]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-A55-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX20]], align 4
 ; CHECK-A55-NEXT:    [[ADD21:%.*]] = add nsw i32 [[MUL16]], [[TMP3]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21]], ptr [[ARRAYIDX20]], align 4
 ; CHECK-A55-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-A55-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-A55-NEXT:    [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX10_1]], align 2
 ; CHECK-A55-NEXT:    [[CONV_1:%.*]] = sext i16 [[TMP4]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_1:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-A55-NEXT:    [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX14_1]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_1:%.*]] = sext i16 [[TMP5]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_1:%.*]] = mul nsw i32 [[CONV15_1]], [[CONV_1]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_1:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_1:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-A55-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX20_1]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_1:%.*]] = add nsw i32 [[MUL16_1]], [[TMP6]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_1]], ptr [[ARRAYIDX20_1]], align 4
 ; CHECK-A55-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
-; CHECK-A55-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-A55-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX10_2]], align 2
 ; CHECK-A55-NEXT:    [[CONV_2:%.*]] = sext i16 [[TMP7]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_2:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_2:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-A55-NEXT:    [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX14_2]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_2:%.*]] = sext i16 [[TMP8]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_2:%.*]] = mul nsw i32 [[CONV15_2]], [[CONV_2]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-A55-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX20_2]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_2:%.*]] = add nsw i32 [[MUL16_2]], [[TMP9]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_2]], ptr [[ARRAYIDX20_2]], align 4
 ; CHECK-A55-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
-; CHECK-A55-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-A55-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10_3]], align 2
 ; CHECK-A55-NEXT:    [[CONV_3:%.*]] = sext i16 [[TMP10]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_3:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_3:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-A55-NEXT:    [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX14_3]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_3:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_3:%.*]] = mul nsw i32 [[CONV15_3]], [[CONV_3]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-A55-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX20_3]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_3:%.*]] = add nsw i32 [[MUL16_3]], [[TMP12]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_3]], ptr [[ARRAYIDX20_3]], align 4
@@ -75,14 +75,14 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
 ; CHECK-A55-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0
 ; CHECK-A55-NEXT:    br i1 [[LCMP_MOD_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL:%.*]]
 ; CHECK-A55:       for.body6.epil:
-; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_UNR]]
 ; CHECK-A55-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL]], align 2
 ; CHECK-A55-NEXT:    [[CONV_EPIL:%.*]] = sext i16 [[TMP13]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_UNR]]
 ; CHECK-A55-NEXT:    [[TMP14:%.*]] = load i16, ptr [[ARRAYIDX14_EPIL]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_EPIL:%.*]] = sext i16 [[TMP14]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_EPIL:%.*]] = mul nsw i32 [[CONV15_EPIL]], [[CONV_EPIL]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_UNR]]
 ; CHECK-A55-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX20_EPIL]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_EPIL:%.*]] = add nsw i32 [[MUL16_EPIL]], [[TMP15]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_EPIL]], ptr [[ARRAYIDX20_EPIL]], align 4
@@ -90,14 +90,14 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
 ; CHECK-A55-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL_1:%.*]]
 ; CHECK-A55:       for.body6.epil.1:
 ; CHECK-A55-NEXT:    [[INDVARS_IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 1
-; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL_1:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL_1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL]]
 ; CHECK-A55-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL_1]], align 2
 ; CHECK-A55-NEXT:    [[CONV_EPIL_1:%.*]] = sext i16 [[TMP16]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL_1:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL_1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL]]
 ; CHECK-A55-NEXT:    [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX14_EPIL_1]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_EPIL_1:%.*]] = sext i16 [[TMP17]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_EPIL_1:%.*]] = mul nsw i32 [[CONV15_EPIL_1]], [[CONV_EPIL_1]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL_1:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL_1:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL]]
 ; CHECK-A55-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX20_EPIL_1]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_EPIL_1:%.*]] = add nsw i32 [[MUL16_EPIL_1]], [[TMP18]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_EPIL_1]], ptr [[ARRAYIDX20_EPIL_1]], align 4
@@ -105,14 +105,14 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
 ; CHECK-A55-NEXT:    br i1 [[EPIL_ITER_CMP_1_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL_2:%.*]]
 ; CHECK-A55:       for.body6.epil.2:
 ; CHECK-A55-NEXT:    [[INDVARS_IV_NEXT_EPIL_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 2
-; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL_2:%.*]] = getelementptr inbounds i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX10_EPIL_2:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
 ; CHECK-A55-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL_2]], align 2
 ; CHECK-A55-NEXT:    [[CONV_EPIL_2:%.*]] = sext i16 [[TMP19]] to i32
-; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL_2:%.*]] = getelementptr inbounds i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX14_EPIL_2:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
 ; CHECK-A55-NEXT:    [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX14_EPIL_2]], align 2
 ; CHECK-A55-NEXT:    [[CONV15_EPIL_2:%.*]] = sext i16 [[TMP20]] to i32
 ; CHECK-A55-NEXT:    [[MUL16_EPIL_2:%.*]] = mul nsw i32 [[CONV15_EPIL_2]], [[CONV_EPIL_2]]
-; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL_2:%.*]] = getelementptr inbounds i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT:    [[ARRAYIDX20_EPIL_2:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
 ; CHECK-A55-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX20_EPIL_2]], align 4
 ; CHECK-A55-NEXT:    [[ADD21_EPIL_2:%.*]] = add nsw i32 [[MUL16_EPIL_2]], [[TMP21]]
 ; CHECK-A55-NEXT:    store i32 [[ADD21_EPIL_2]], ptr [[ARRAYIDX20_EPIL_2]], align 4
@@ -129,14 +129,14 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
 ; CHECK-GENERIC-NEXT:    br label [[FOR_BODY6:%.*]]
 ; CHECK-GENERIC:       for.body6:
 ; CHECK-GENERIC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY6_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY6]] ]
-; CHECK-GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[ARG_2:%.*]], i64 [[INDVARS_IV]]
+; CHECK-GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-GENERIC-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; CHECK-GENERIC-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-; CHECK-GENERIC-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i16, ptr [[ARG_3:%.*]], i64 [[INDVARS_IV]]
+; CHECK-GENERIC-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_3:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-GENERIC-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX14]], align 2
 ; CHECK-GENERIC-NEXT:    [[CONV15:%.*]] = sext i16 [[TMP1]] to i32
 ; CHECK-GENERIC-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[CONV15]], [[CONV]]
-; CHECK-GENERIC-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, ptr [[ARG_1:%.*]], i64 [[INDVARS_IV]]
+; CHECK-GENERIC-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG_1:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-GENERIC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX20]], align 4
 ; CHECK-GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[MUL16]], [[TMP2]]
 ; CHECK-GENERIC-NEXT:    store i32 [[ADD21]], ptr [[ARRAYIDX20]], align 4
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/lob-unroll.ll b/llvm/test/Transforms/LoopUnroll/ARM/lob-unroll.ll
new file mode 100644
index 0000000000000..b155f5d31045f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/lob-unroll.ll
@@ -0,0 +1,63 @@
+; RUN: opt -mcpu=cortex-m55 -mtriple=thumbv8.1m.main -passes=loop-unroll -S  %s -o - | FileCheck %s --check-prefix=LOB
+
+; This test checks behaviour of loop unrolling on processors with low overhead branching available 
+
+; LOB-CHECK-LABEL: for.body{{.*}}.prol
+; LOB-COUNT-1:     fmul fast float 
+; LOB-CHECK-LABEL: for.body{{.*}}.prol.1
+; LOB-COUNT-1:     fmul fast float 
+; LOB-CHECK-LABEL: for.body{{.*}}.prol.2
+; LOB-COUNT-1:     fmul fast float 
+; LOB-CHECK-LABEL: for.body{{.*}}
+; LOB-COUNT-4:     fmul fast float 
+; LOB-NOT:     fmul fast float 
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite)
+define dso_local void @test(i32 noundef %n, ptr nocapture noundef %pA) local_unnamed_addr #0 {
+entry:
+  %cmp46 = icmp sgt i32 %n, 0
+  br i1 %cmp46, label %for.body, label %for.cond.cleanup
+
+for.cond.loopexit:                                ; preds = %for.cond6.for.cond.cleanup8_crit_edge.us, %for.body
+  %exitcond49.not = icmp eq i32 %add, %n
+  br i1 %exitcond49.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.cond.loopexit
+  %k.047 = phi i32 [ %add, %for.cond.loopexit ], [ 0, %entry ]
+  %add = add nuw nsw i32 %k.047, 1
+  %cmp244 = icmp slt i32 %add, %n
+  br i1 %cmp244, label %for.cond6.preheader.lr.ph, label %for.cond.loopexit
+
+for.cond6.preheader.lr.ph:                        ; preds = %for.body
+  %invariant.gep = getelementptr float, ptr %pA, i32 %k.047
+  br label %for.cond6.preheader.us
+
+for.cond6.preheader.us:                           ; preds = %for.cond6.for.cond.cleanup8_crit_edge.us, %for.cond6.preheader.lr.ph
+  %w.045.us = phi i32 [ %add, %for.cond6.preheader.lr.ph ], [ %inc19.us, %for.cond6.for.cond.cleanup8_crit_edge.us ]
+  %mul.us = mul nuw nsw i32 %w.045.us, %n
+  %0 = getelementptr float, ptr %pA, i32 %mul.us
+  %arrayidx.us = getelementptr float, ptr %0, i32 %k.047
+  br label %for.body9.us
+
+for.body9.us:                                     ; preds = %for.cond6.preheader.us, %for.body9.us
+  %x.043.us = phi i32 [ %add, %for.cond6.preheader.us ], [ %inc.us, %for.body9.us ]
+  %1 = load float, ptr %arrayidx.us, align 4
+  %mul11.us = mul nuw nsw i32 %x.043.us, %n
+  %gep.us = getelementptr float, ptr %invariant.gep, i32 %mul11.us
+  %2 = load float, ptr %gep.us, align 4
+  %mul14.us = fmul fast float %2, %1
+  %arrayidx17.us = getelementptr float, ptr %0, i32 %x.043.us
+  store float %mul14.us, ptr %arrayidx17.us, align 4
+  %inc.us = add nuw nsw i32 %x.043.us, 1
+  %exitcond.not = icmp eq i32 %inc.us, %n
+  br i1 %exitcond.not, label %for.cond6.for.cond.cleanup8_crit_edge.us, label %for.body9.us
+
+for.cond6.for.cond.cleanup8_crit_edge.us:         ; preds = %for.body9.us
+  %inc19.us = add nuw nsw i32 %w.045.us, 1
+  %exitcond48.not = icmp eq i32 %inc19.us, %n
+  br i1 %exitcond48.not, label %for.cond.loopexit, label %for.cond6.preheader.us
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
index 141af3e66949a..302cc96c9961e 100644
--- a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
+++ b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
@@ -17,7 +17,7 @@ define void @test(ptr %x, i32 %n) {
 ; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 4
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[REM]], 1
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY_1:%.*]]
 ; CHECK:       while.body.1:
@@ -28,7 +28,7 @@ define void @test(ptr %x, i32 %n) {
 ; CHECK-NEXT:    store i32 0, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    br label [[IF_END_1]]
 ; CHECK:       if.end.1:
-; CHECK-NEXT:    [[INCDEC_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 8
+; CHECK-NEXT:    [[INCDEC_PTR_1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 8
 ; CHECK-NEXT:    [[CMP_1:%.*]] = icmp sgt i32 [[REM]], 2
 ; CHECK-NEXT:    br i1 [[CMP_1]], label [[WHILE_BODY_2:%.*]], label [[WHILE_END]]
 ; CHECK:       while.body.2:
diff --git a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
index b59f05bd11f0d..ea499e548934f 100644
--- a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
+++ b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
@@ -8,41 +8,41 @@ define hidden void @compile_time_full(ptr nocapture %a, ptr nocapture readonly %
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I:%.*]] = load i8, ptr [[B:%.*]], align 1
 ; CHECK-NEXT:    store i8 [[I]], ptr [[A:%.*]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 1
 ; CHECK-NEXT:    [[I_1:%.*]] = load i8, ptr [[ARRAYIDX_1]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX1_1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 1
 ; CHECK-NEXT:    store i8 [[I_1]], ptr [[ARRAYIDX1_1]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 2
 ; CHECK-NEXT:    [[I_2:%.*]] = load i8, ptr [[ARRAYIDX_2]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX1_2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 2
 ; CHECK-NEXT:    store i8 [[I_2]], ptr [[ARRAYIDX1_2]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 3
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 3
 ; CHECK-NEXT:    [[I_3:%.*]] = load i8, ptr [[ARRAYIDX_3]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 3
+; CHECK-NEXT:    [[ARRAYIDX1_3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 3
 ; CHECK-NEXT:    store i8 [[I_3]], ptr [[ARRAYIDX1_3]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 4
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 4
 ; CHECK-NEXT:    [[I_4:%.*]] = load i8, ptr [[ARRAYIDX_4]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_4:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 4
+; CHECK-NEXT:    [[ARRAYIDX1_4:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 4
 ; CHECK-NEXT:    store i8 [[I_4]], ptr [[ARRAYIDX1_4]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 5
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 5
 ; CHECK-NEXT:    [[I_5:%.*]] = load i8, ptr [[ARRAYIDX_5]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_5:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 5
+; CHECK-NEXT:    [[ARRAYIDX1_5:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 5
 ; CHECK-NEXT:    store i8 [[I_5]], ptr [[ARRAYIDX1_5]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 6
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 6
 ; CHECK-NEXT:    [[I_6:%.*]] = load i8, ptr [[ARRAYIDX_6]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_6:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 6
+; CHECK-NEXT:    [[ARRAYIDX1_6:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 6
 ; CHECK-NEXT:    store i8 [[I_6]], ptr [[ARRAYIDX1_6]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 7
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 7
 ; CHECK-NEXT:    [[I_7:%.*]] = load i8, ptr [[ARRAYIDX_7]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_7:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 7
+; CHECK-NEXT:    [[ARRAYIDX1_7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 7
 ; CHECK-NEXT:    store i8 [[I_7]], ptr [[ARRAYIDX1_7]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 8
+; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 8
 ; CHECK-NEXT:    [[I_8:%.*]] = load i8, ptr [[ARRAYIDX_8]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_8:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 8
+; CHECK-NEXT:    [[ARRAYIDX1_8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 8
 ; CHECK-NEXT:    store i8 [[I_8]], ptr [[ARRAYIDX1_8]], align 1
-; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 9
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i32 9
 ; CHECK-NEXT:    [[I_9:%.*]] = load i8, ptr [[ARRAYIDX_9]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1_9:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 9
+; CHECK-NEXT:    [[ARRAYIDX1_9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 9
 ; CHECK-NEXT:    store i8 [[I_9]], ptr [[ARRAYIDX1_9]], align 1
 ; CHECK-NEXT:    ret void
 ;
@@ -71,28 +71,28 @@ define hidden void @compile_time_partial(ptr nocapture %a, ptr nocapture readonl
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_3:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i32 [[I_07]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[B:%.*]], i32 [[I_07]]
 ; CHECK-NEXT:    [[I:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[I]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i32 [[I_07]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i16, ptr [[A:%.*]], i32 [[I_07]]
 ; CHECK-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    [[INC:%.*]] = or disjoint i32 [[I_07]], 1
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[INC]]
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i16, ptr [[B]], i32 [[INC]]
 ; CHECK-NEXT:    [[I_1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
 ; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[I_1]], 1
-; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INC]]
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw i16, ptr [[A]], i32 [[INC]]
 ; CHECK-NEXT:    store i16 [[ADD_1]], ptr [[ARRAYIDX2_1]], align 2
 ; CHECK-NEXT:    [[INC_1:%.*]] = or disjoint i32 [[I_07]], 2
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[INC_1]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i16, ptr [[B]], i32 [[INC_1]]
 ; CHECK-NEXT:    [[I_2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
 ; CHECK-NEXT:    [[ADD_2:%.*]] = add i16 [[I_2]], 1
-; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INC_1]]
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw i16, ptr [[A]], i32 [[INC_1]]
 ; CHECK-NEXT:    store i16 [[ADD_2]], ptr [[ARRAYIDX2_2]], align 2
 ; CHECK-NEXT:    [[INC_2:%.*]] = or disjoint i32 [[I_07]], 3
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[INC_2]]
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i16, ptr [[B]], i32 [[INC_2]]
 ; CHECK-NEXT:    [[I_3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
 ; CHECK-NEXT:    [[ADD_3:%.*]] = add i16 [[I_3]], 1
-; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INC_2]]
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw i16, ptr [[A]], i32 [[INC_2]]
 ; CHECK-NEXT:    store i16 [[ADD_3]], ptr [[ARRAYIDX2_3]], align 2
 ; CHECK-NEXT:    [[INC_3]] = add nuw nsw i32 [[I_07]], 4
 ; CHECK-NEXT:    [[EXITCOND_NOT_3:%.*]] = icmp eq i32 [[INC_3]], 1000
@@ -203,7 +203,7 @@ define hidden void @dont_unroll_call(ptr nocapture %a, ptr nocapture readonly %b
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I1]], [[I]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[I_013]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str, i32 [[I_013]], i32 [[MUL]])
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str, i32 [[I_013]], i32 [[MUL]])
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_013]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
@@ -238,9 +238,9 @@ define hidden void @dont_unroll_optsize(ptr nocapture %a, ptr nocapture readonly
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i32 [[I_06]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i32 [[I_06]]
 ; CHECK-NEXT:    [[I:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[I_06]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i32 [[I_06]]
 ; CHECK-NEXT:    store i8 [[I]], ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 10
@@ -271,9 +271,9 @@ define hidden void @dont_unroll_minsize(ptr nocapture %a, ptr nocapture readonly
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i32 [[I_06]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i32 [[I_06]]
 ; CHECK-NEXT:    [[I:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[I_06]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i32 [[I_06]]
 ; CHECK-NEXT:    store i8 [[I]], ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 10
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop.ll b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
index a76facae3cb9d..0f58e705b448a 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
@@ -16,20 +16,20 @@ define void @basic(ptr %p, i32 %k) #0 {
 ; CHECK-NEXT:    [[CMP_PEEL_NOT:%.*]] = icmp eq i32 [[K]], 1
 ; CHECK-NEXT:    br i1 [[CMP_PEEL_NOT]], label [[FOR_END]], label [[FOR_BODY_PEEL2:%.*]]
 ; CHECK:       for.body.peel2:
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    store i32 1, ptr [[INCDEC_PTR_PEEL]], align 4
 ; CHECK-NEXT:    [[CMP_PEEL5:%.*]] = icmp sgt i32 [[K]], 2
 ; CHECK-NEXT:    br i1 [[CMP_PEEL5]], label [[FOR_BODY_PEEL7:%.*]], label [[FOR_END]]
 ; CHECK:       for.body.peel7:
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL8:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL8:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 ; CHECK-NEXT:    store i32 2, ptr [[INCDEC_PTR_PEEL3]], align 4
 ; CHECK-NEXT:    [[CMP_PEEL10_NOT:%.*]] = icmp eq i32 [[K]], 3
 ; CHECK-NEXT:    br i1 [[CMP_PEEL10_NOT]], label [[FOR_END]], label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 3, [[FOR_BODY_PEEL7]] ]
 ; CHECK-NEXT:    [[P_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[INCDEC_PTR_PEEL8]], [[FOR_BODY_PEEL7]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[P_ADDR_04]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[P_ADDR_04]], i64 4
 ; CHECK-NEXT:    store i32 [[I_05]], ptr [[P_ADDR_04]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
@@ -76,20 +76,20 @@ define i32 @output(ptr %p, i32 %k) #0 {
 ; CHECK-NEXT:    [[CMP_PEEL_NOT:%.*]] = icmp eq i32 [[K]], 1
 ; CHECK-NEXT:    br i1 [[CMP_PEEL_NOT]], label [[FOR_END]], label [[FOR_BODY_PEEL2:%.*]]
 ; CHECK:       for.body.peel2:
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    store i32 1, ptr [[INCDEC_PTR_PEEL]], align 4
 ; CHECK-NEXT:    [[CMP_PEEL5:%.*]] = icmp sgt i32 [[K]], 2
 ; CHECK-NEXT:    br i1 [[CMP_PEEL5]], label [[FOR_BODY_PEEL7:%.*]], label [[FOR_END]]
 ; CHECK:       for.body.peel7:
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
-; CHECK-NEXT:    [[INCDEC_PTR_PEEL8:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+; CHECK-NEXT:    [[INCDEC_PTR_PEEL8:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
 ; CHECK-NEXT:    store i32 2, ptr [[INCDEC_PTR_PEEL3]], align 4
 ; CHECK-NEXT:    [[CMP_PEEL10_NOT:%.*]] = icmp eq i32 [[K]], 3
 ; CHECK-NEXT:    br i1 [[CMP_PEEL10_NOT]], label [[FOR_END]], label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 3, [[FOR_BODY_PEEL7]] ]
 ; CHECK-NEXT:    [[P_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[INCDEC_PTR_PEEL8]], [[FOR_BODY_PEEL7]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[P_ADDR_04]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[P_ADDR_04]], i64 4
 ; CHECK-NEXT:    store i32 [[I_05]], ptr [[P_ADDR_04]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
index 4307be7a8b5a7..a3cfeac1f86be 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
@@ -66,30 +66,30 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N)
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH_NEW]] ], [ [[INDVARS_IV_NEXT_3]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[C_010:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH_NEW]] ], [ [[ADD_3]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH_NEW]] ], [ [[NITER_NEXT_3:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[MUL]], [[C_010]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX2_1]], align 4
 ; CHECK-NEXT:    [[MUL_1:%.*]] = mul nsw i32 [[TMP10]], [[TMP9]]
 ; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[MUL_1]], [[ADD]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX2_2]], align 4
 ; CHECK-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[TMP12]], [[TMP11]]
 ; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[MUL_2]], [[ADD_1]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX2_3]], align 4
 ; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i32 [[TMP14]], [[TMP13]]
 ; CHECK-NEXT:    [[ADD_3]] = add nsw i32 [[MUL_3]], [[ADD_2]]
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
index da1808fc278c0..40d6ddc27370e 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
@@ -57,15 +57,15 @@ define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK-NEXT:    [[CMP_PROL:%.*]] = icmp eq i8 [[TMP3_PROL]], 0
 ; CHECK-NEXT:    br i1 [[CMP_PROL]], label %[[IF_THEN_PROL:.*]], label %[[FOR_INC_PROL:.*]]
 ; CHECK:       [[IF_THEN_PROL]]:
-; CHECK-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds i8, ptr [[R_022]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds nuw i8, ptr [[R_022]], i64 2
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX_PROL]], align 2
 ; CHECK-NEXT:    store i16 0, ptr [[R_022]], align 2
-; CHECK-NEXT:    [[ARRAYIDX5_PROL:%.*]] = getelementptr inbounds i8, ptr [[R_022]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_PROL:%.*]] = getelementptr inbounds nuw i8, ptr [[R_022]], i64 4
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX5_PROL]], align 2
 ; CHECK-NEXT:    br label %[[FOR_INC_PROL]]
 ; CHECK:       [[FOR_INC_PROL]]:
-; CHECK-NEXT:    [[INCDEC_PTR_PROL:%.*]] = getelementptr inbounds i8, ptr [[A_021]], i64 1
-; CHECK-NEXT:    [[ADD_PTR_PROL:%.*]] = getelementptr inbounds i8, ptr [[R_022]], i64 6
+; CHECK-NEXT:    [[INCDEC_PTR_PROL:%.*]] = getelementptr inbounds nuw i8, ptr [[A_021]], i64 1
+; CHECK-NEXT:    [[ADD_PTR_PROL:%.*]] = getelementptr inbounds nuw i8, ptr [[R_022]], i64 6
 ; CHECK-NEXT:    br label %[[FOR_BODY3_PROL_LOOPEXIT]]
 ; CHECK:       [[FOR_BODY3_PROL_LOOPEXIT]]:
 ; CHECK-NEXT:    [[ADD_PTR_LCSSA_UNR]] = phi ptr [ poison, %[[FOR_BODY3_PREHEADER]] ], [ [[ADD_PTR_PROL]], %[[FOR_INC_PROL]] ]
@@ -82,29 +82,29 @@ define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[FOR_INC:.*]]
 ; CHECK:       [[IF_THEN]]:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[R_117]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 2
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    store i16 0, ptr [[R_117]], align 2
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[R_117]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 4
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX5]], align 2
 ; CHECK-NEXT:    br label %[[FOR_INC]]
 ; CHECK:       [[FOR_INC]]:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[A_116]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[A_116]], i64 1
 ; CHECK-NEXT:    [[DEC18_1]] = add nsw i32 [[DEC18_IN]], -2
 ; CHECK-NEXT:    [[TMP3_1:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
 ; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i8 [[TMP3_1]], 0
 ; CHECK-NEXT:    br i1 [[CMP_1]], label %[[IF_THEN_1:.*]], label %[[FOR_INC_1]]
 ; CHECK:       [[IF_THEN_1]]:
-; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[R_117]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[R_117]], i64 8
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 8
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX_1]], align 2
 ; CHECK-NEXT:    store i16 0, ptr [[ADD_PTR]], align 2
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[R_117]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 10
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX5_1]], align 2
 ; CHECK-NEXT:    br label %[[FOR_INC_1]]
 ; CHECK:       [[FOR_INC_1]]:
-; CHECK-NEXT:    [[INCDEC_PTR_1]] = getelementptr inbounds i8, ptr [[A_116]], i64 2
-; CHECK-NEXT:    [[ADD_PTR_1]] = getelementptr inbounds i8, ptr [[R_117]], i64 12
+; CHECK-NEXT:    [[INCDEC_PTR_1]] = getelementptr inbounds nuw i8, ptr [[A_116]], i64 2
+; CHECK-NEXT:    [[ADD_PTR_1]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 12
 ; CHECK-NEXT:    [[TOBOOL2_1:%.*]] = icmp eq i32 [[DEC18_1]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL2_1]], label %[[FOR_COND_LOOPEXIT_LOOPEXIT]], label %[[FOR_BODY3]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[FOR_END6]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
index c4559dc070e45..1948211858d44 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
@@ -8,42 +8,40 @@ define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,1
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 8, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[DST]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 8)
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = add <vscale x 4 x i64> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 1)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[VAL]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT:    [[TMP15:%.*]] = lshr <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = trunc <vscale x 4 x i64> [[TMP15]] to <vscale x 4 x i8>
+; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP16]], ptr [[TMP17]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr [[TMP17]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 8)
-; CHECK-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 8)
+; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -97,42 +95,40 @@ define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[WIDE_TRIP_COUNT]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[DST]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = add <vscale x 4 x i64> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 1)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 8 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[VAL]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT:    [[TMP15:%.*]] = lshr <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = trunc <vscale x 4 x i64> [[TMP15]] to <vscale x 4 x i8>
+; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw <vscale x 8 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP16]], ptr [[TMP17]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr [[TMP17]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
-; CHECK-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 1acc400a2c9ff..6956030570636 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -455,7 +455,7 @@ define void @latch_branch_cost(ptr %dst) {
 ; PRED-NEXT:    store i8 0, ptr [[TMP24]], align 1
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; PRED:       pred.store.continue14:
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
 ; PRED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 104
 ; PRED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -494,10 +494,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
 ; DEFAULT-NEXT:  entry:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 64, i64 [[TMP2]])
-; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 60
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; DEFAULT:       vector.memcheck:
 ; DEFAULT-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[E]], i64 4
@@ -536,34 +533,55 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; DEFAULT-NEXT:    [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX23]], [[FOUND_CONFLICT26]]
 ; DEFAULT-NEXT:    br i1 [[CONFLICT_RDX27]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; DEFAULT:       vector.ph:
-; DEFAULT-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP7]]
+; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT32:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[E]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT33:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT32]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; DEFAULT:       vector.body:
-; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE37:%.*]] ]
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
-; DEFAULT-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META7:![0-9]+]]
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP12:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META10:![0-9]+]]
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP13:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT29]]
-; DEFAULT-NEXT:    [[TMP14:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META12:![0-9]+]]
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT31:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT30]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP15:%.*]] = icmp ugt <vscale x 4 x i32> [[BROADCAST_SPLAT31]], [[TMP13]]
+; DEFAULT-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META7:![0-9]+]]
+; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <4 x i32> poison, i32 [[TMP9]], i64 0
+; DEFAULT-NEXT:    [[BROADCAST_SPLAT29:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT28]], <4 x i32> poison, <4 x i32> zeroinitializer
+; DEFAULT-NEXT:    [[TMP19:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META10:![0-9]+]]
+; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i64 0
+; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; DEFAULT-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT29]]
+; DEFAULT-NEXT:    [[TMP7:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META12:![0-9]+]]
+; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
+; DEFAULT-NEXT:    [[BROADCAST_SPLAT31:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT30]], <4 x i32> poison, <4 x i32> zeroinitializer
+; DEFAULT-NEXT:    [[TMP8:%.*]] = icmp ugt <4 x i32> [[BROADCAST_SPLAT31]], [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[D]], i64 [[TMP10]]
-; DEFAULT-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> [[BROADCAST_SPLAT33]], i32 4, <vscale x 4 x i1> [[TMP15]]), !alias.scope [[META14:![0-9]+]], !noalias [[META16:![0-9]+]]
+; DEFAULT-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP8]], i32 0
+; DEFAULT-NEXT:    br i1 [[TMP20]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; DEFAULT:       pred.store.if:
+; DEFAULT-NEXT:    [[TMP11:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
+; DEFAULT-NEXT:    store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META14:![0-9]+]], !noalias [[META16:![0-9]+]]
+; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; DEFAULT:       pred.store.continue:
+; DEFAULT-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP8]], i32 1
+; DEFAULT-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]]
+; DEFAULT:       pred.store.if32:
+; DEFAULT-NEXT:    [[TMP13:%.*]] = extractelement <4 x i32> [[TMP6]], i32 1
+; DEFAULT-NEXT:    store i32 [[TMP13]], ptr [[E]], align 4, !alias.scope [[META14]], !noalias [[META16]]
+; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE33]]
+; DEFAULT:       pred.store.continue33:
+; DEFAULT-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP8]], i32 2
+; DEFAULT-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35:%.*]]
+; DEFAULT:       pred.store.if34:
+; DEFAULT-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP6]], i32 2
+; DEFAULT-NEXT:    store i32 [[TMP15]], ptr [[E]], align 4, !alias.scope [[META14]], !noalias [[META16]]
+; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE35]]
+; DEFAULT:       pred.store.continue35:
+; DEFAULT-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP8]], i32 3
+; DEFAULT-NEXT:    br i1 [[TMP21]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37]]
+; DEFAULT:       pred.store.if36:
+; DEFAULT-NEXT:    [[TMP22:%.*]] = extractelement <4 x i32> [[TMP6]], i32 3
+; DEFAULT-NEXT:    store i32 [[TMP22]], ptr [[E]], align 4, !alias.scope [[META14]], !noalias [[META16]]
+; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE37]]
+; DEFAULT:       pred.store.continue37:
 ; DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 0
-; DEFAULT-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> zeroinitializer, ptr [[TMP17]], i32 4, <vscale x 4 x i1> [[TMP15]]), !alias.scope [[META18:![0-9]+]], !noalias [[META19:![0-9]+]]
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; DEFAULT-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr [[TMP17]], i32 4, <4 x i1> [[TMP8]]), !alias.scope [[META18:![0-9]+]], !noalias [[META19:![0-9]+]]
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; DEFAULT-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; DEFAULT:       middle.block:
@@ -634,48 +652,66 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; PRED-NEXT:    [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX23]], [[FOUND_CONFLICT26]]
 ; PRED-NEXT:    br i1 [[CONFLICT_RDX27]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; PRED:       vector.ph:
-; PRED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; PRED-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP4]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]]
+; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
-; PRED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
-; PRED-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP0]], [[TMP11]]
-; PRED-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[TMP0]], [[TMP11]]
+; PRED-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP0]], 4
+; PRED-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[TMP0]], 4
 ; PRED-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT32:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[E]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT33:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT32]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
 ; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE37:%.*]] ]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE37]] ]
 ; PRED-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 0
-; PRED-NEXT:    [[TMP16:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META4:![0-9]+]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP16]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP17:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META7:![0-9]+]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP17]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP18:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT29]]
-; PRED-NEXT:    [[TMP19:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META9:![0-9]+]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP19]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT31:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT30]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP20:%.*]] = icmp ugt <vscale x 4 x i32> [[BROADCAST_SPLAT31]], [[TMP18]]
-; PRED-NEXT:    [[TMP21:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP20]], <vscale x 4 x i1> zeroinitializer
+; PRED-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META4:![0-9]+]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT29:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT28]], <4 x i32> poison, <4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP8:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META7:![0-9]+]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT29]]
+; PRED-NEXT:    [[TMP10:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META9:![0-9]+]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT31:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT30]], <4 x i32> poison, <4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP11:%.*]] = icmp ugt <4 x i32> [[BROADCAST_SPLAT31]], [[TMP9]]
+; PRED-NEXT:    [[TMP25:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
 ; PRED-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[D]], i64 [[TMP15]]
-; PRED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> [[BROADCAST_SPLAT33]], i32 4, <vscale x 4 x i1> [[TMP21]]), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
+; PRED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[TMP25]], i32 0
+; PRED-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; PRED:       pred.store.if:
+; PRED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[TMP9]], i32 0
+; PRED-NEXT:    store i32 [[TMP27]], ptr [[E]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; PRED:       pred.store.continue:
+; PRED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP25]], i32 1
+; PRED-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]]
+; PRED:       pred.store.if32:
+; PRED-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[TMP9]], i32 1
+; PRED-NEXT:    store i32 [[TMP17]], ptr [[E]], align 4, !alias.scope [[META11]], !noalias [[META13]]
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE33]]
+; PRED:       pred.store.continue33:
+; PRED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP25]], i32 2
+; PRED-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35:%.*]]
+; PRED:       pred.store.if34:
+; PRED-NEXT:    [[TMP19:%.*]] = extractelement <4 x i32> [[TMP9]], i32 2
+; PRED-NEXT:    store i32 [[TMP19]], ptr [[E]], align 4, !alias.scope [[META11]], !noalias [[META13]]
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE35]]
+; PRED:       pred.store.continue35:
+; PRED-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP25]], i32 3
+; PRED-NEXT:    br i1 [[TMP20]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37]]
+; PRED:       pred.store.if36:
+; PRED-NEXT:    [[TMP21:%.*]] = extractelement <4 x i32> [[TMP9]], i32 3
+; PRED-NEXT:    store i32 [[TMP21]], ptr [[E]], align 4, !alias.scope [[META11]], !noalias [[META13]]
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE37]]
+; PRED:       pred.store.continue37:
 ; PRED-NEXT:    [[TMP23:%.*]] = getelementptr i32, ptr [[TMP22]], i32 0
-; PRED-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> zeroinitializer, ptr [[TMP23]], i32 4, <vscale x 4 x i1> [[TMP21]]), !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]]
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP14]])
-; PRED-NEXT:    [[TMP24:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 4 x i1> [[TMP24]], i32 0
-; PRED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; PRED-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr [[TMP23]], i32 4, <4 x i1> [[TMP25]]), !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[TMP14]])
+; PRED-NEXT:    [[TMP28:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i1> [[TMP28]], i32 0
+; PRED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
@@ -943,7 +979,7 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
 ; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE14]]
 ; DEFAULT:       pred.store.continue14:
 ; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; DEFAULT-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -1045,7 +1081,7 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE14]]
 ; PRED:       pred.store.continue14:
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; PRED-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -1593,7 +1629,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) optsize {
 ; DEFAULT-NEXT:    store i32 [[TMP10]], ptr [[DST]], align 4
 ; DEFAULT-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; DEFAULT:       pred.store.continue6:
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
 ; DEFAULT-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
@@ -1656,7 +1692,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) optsize {
 ; PRED-NEXT:    store i32 [[TMP10]], ptr [[DST]], align 4
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; PRED:       pred.store.continue6:
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; PRED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
 ; PRED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
index 8cda055a53ad8..6d43c1e1c7d48 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
@@ -91,17 +91,17 @@ define void @test_pr25490(i32 %n, ptr noalias nocapture %a, ptr noalias nocaptur
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP27]] to i32
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
 ; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[TMP28]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV3]], [[CONV]]
 ; CHECK-NEXT:    [[SHR_26:%.*]] = lshr i32 [[MUL]], 8
 ; CHECK-NEXT:    [[CONV4:%.*]] = trunc nuw i32 [[SHR_26]] to i8
 ; CHECK-NEXT:    store i8 [[CONV4]], ptr [[ARRAYIDX2]], align 1
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX8]], align 1
 ; CHECK-NEXT:    [[CONV9:%.*]] = zext i8 [[TMP29]] to i32
 ; CHECK-NEXT:    [[MUL10:%.*]] = mul nuw nsw i32 [[CONV9]], [[CONV]]
@@ -172,7 +172,7 @@ define void @test_shrink_zext_in_preheader(ptr noalias %src, ptr noalias %dst, i
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
 ; CHECK-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1
 ; CHECK-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP9]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
@@ -250,7 +250,7 @@ define void @test_shrink_select(ptr noalias %src, ptr noalias %dst, i32 %A, i1 %
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
 ; CHECK-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1
 ; CHECK-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP9]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
@@ -324,7 +324,7 @@ define void @trunc_invariant_sdiv_result(i32 %a, i32 %b, ptr noalias %src, ptr %
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i16>
@@ -332,7 +332,7 @@ define void @trunc_invariant_sdiv_result(i32 %a, i32 %b, ptr noalias %src, ptr %
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <16 x i16> [[TMP2]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <16 x i16> [[TMP2]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 32
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 32
 ; CHECK-NEXT:    store <16 x i16> [[TMP7]], ptr [[TMP9]], align 2
 ; CHECK-NEXT:    store <16 x i16> [[TMP8]], ptr [[TMP10]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
@@ -462,7 +462,7 @@ define void @old_and_new_size_equalko(ptr noalias %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; CHECK-NEXT:    store <4 x i32> splat (i32 1), ptr [[TMP1]], align 4
 ; CHECK-NEXT:    store <4 x i32> splat (i32 1), ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
index 7c6a881ead108..7645e1094e9ff 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
@@ -143,11 +143,10 @@ define void @test_widen_induction(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -205,10 +204,10 @@ define void @test_widen_induction_variable_start(ptr %A, i64 %N, i64 %start) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[START]], [[N_VEC]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[START]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[START]], [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -322,11 +321,10 @@ define void @test_widen_induction_step_2(ptr %A, i64 %N, i32 %step) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -433,7 +431,7 @@ define void @test_widen_extended_induction(ptr %dst) {
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i8 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i8 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -491,9 +489,8 @@ define void @test_widen_truncated_induction(ptr %A) {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 true, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i8
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i8> poison, i8 [[TMP5]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i8> [[DOTSPLATINSERT]], <2 x i8> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i8> [[DOTSPLAT]], <i8 0, i8 1>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
index aa78113ebaa48..f969be80c3c00 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
@@ -63,11 +63,11 @@ exit:
 ; Similar to the test case above, but checks getVectorCallCost as well.
 declare float @powf(float, float) readnone nounwind
 
-; Ensure the extractvalue + add instructions are hoisted out
+; Ensure the extractvalue instructions are hoisted out
+; CM-LABEL: Checking a loop in 'test_getVectorCallCost'
 ; CM: vector.ph:
 ; CM:  CLONE ir<%a> = extractvalue ir<%sv>
 ; CM:  CLONE ir<%b> = extractvalue ir<%sv>
-; CM:  WIDEN ir<%add> = add ir<%a>, ir<%b>
 ; CM:  Successor(s): vector loop
 
 ; CM: LV: Scalar loop costs: 14.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
new file mode 100644
index 0000000000000..f5ffc731eac84
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
@@ -0,0 +1,145 @@
+; REQUIRES: asserts
+; RUN: opt < %s -mcpu=neoverse-v2 -passes=loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
+
+target triple="aarch64--linux-gnu"
+
+; This test shows that comparison and next iteration IV have zero cost if the
+; vector loop gets executed exactly once with the given VF.
+define i64 @test(ptr %a, ptr %b) #0 {
+; CHECK-LABEL: LV: Checking a loop in 'test'
+; CHECK: Cost of 1 for VF 8: induction instruction   %i.iv.next = add nuw nsw i64 %i.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 1 for VF 8: exit condition instruction   %exitcond.not = icmp eq i64 %i.iv.next, 16
+; CHECK-NEXT: Cost of 0 for VF 8: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 8: 26
+; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 16: 48
+; CHECK: LV: Selecting VF: 16
+entry:
+  br label %for.body
+
+exit:                                 ; preds = %for.body
+  ret i64 %add
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+  %sum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i8, ptr %a, i64 %i.iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %0 to i64
+  %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %i.iv
+  %1 = load i8, ptr %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i64
+  %mul = mul nuw nsw i64 %conv3, %conv
+  %add = add i64 %mul, %sum
+  %i.iv.next = add nuw nsw i64 %i.iv, 1
+  %exitcond.not = icmp eq i64 %i.iv.next, 16
+  br i1 %exitcond.not, label %exit, label %for.body
+}
+
+; Same as above, but in the next iteration IV has extra users, and thus, the cost is not zero.
+define i64 @test_external_iv_user(ptr %a, ptr %b) #0 {
+; CHECK-LABEL: LV: Checking a loop in 'test_external_iv_user'
+; CHECK: Cost of 1 for VF 8: induction instruction   %i.iv.next = add nuw nsw i64 %i.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 1 for VF 8: exit condition instruction   %exitcond.not = icmp eq i64 %i.iv.next, 16
+; CHECK-NEXT: Cost of 0 for VF 8: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 8: 26
+; CHECK-NEXT: Cost of 1 for VF 16: induction instruction   %i.iv.next = add nuw nsw i64 %i.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 16: 49
+; CHECK: LV: Selecting VF: vscale x 2
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+  %sum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %0 to i64
+  %i.iv.next = add nuw nsw i64 %i.iv, 1
+  %arrayidx2 = getelementptr inbounds nuw i8, ptr %b, i64 %i.iv.next
+  %1 = load i8, ptr %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i64
+  %mul = mul nuw nsw i64 %conv3, %conv
+  %add = add i64 %sum, %mul
+  %exitcond.not = icmp eq i64 %i.iv.next, 16
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:                                 ; preds = %for.body
+  ret i64 %add
+}
+
+; Same as above but with two IVs without extra users. They all have zero cost when VF equals the number of iterations.
+define i64 @test_two_ivs(ptr %a, ptr %b, i64 %start) #0 {
+; CHECK-LABEL: LV: Checking a loop in 'test_two_ivs'
+; CHECK: Cost of 1 for VF 8: induction instruction   %i.iv.next = add nuw nsw i64 %i.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 1 for VF 8: induction instruction   %j.iv.next = add nuw nsw i64 %j.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 1 for VF 8: exit condition instruction   %exitcond.not = icmp eq i64 %i.iv.next, 16
+; CHECK-NEXT: Cost of 0 for VF 8: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 8: 27
+; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 16: 48
+; CHECK: LV: Selecting VF: 16
+entry:
+  br label %for.body
+
+exit:                                 ; preds = %for.body
+  ret i64 %add
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
+  %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
+  %sum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i8, ptr %a, i64 %i.iv
+  %0 = load i8, ptr %arrayidx, align 1
+  %conv = zext i8 %0 to i64
+  %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %j.iv
+  %1 = load i8, ptr %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i64
+  %mul = mul nuw nsw i64 %conv3, %conv
+  %add = add i64 %mul, %sum
+  %i.iv.next = add nuw nsw i64 %i.iv, 1
+  %j.iv.next = add nuw nsw i64 %j.iv, 1
+  %exitcond.not = icmp eq i64 %i.iv.next, 16
+  br i1 %exitcond.not, label %exit, label %for.body
+}
+
+define i1 @test_extra_cmp_user(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src) {
+; CHECK-LABEL: LV: Checking a loop in 'test_extra_cmp_user'
+; CHECK: Cost of 4 for VF 8: induction instruction   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 4 for VF 8: exit condition instruction   %exitcond.not = icmp eq i64 %indvars.iv.next, 16
+; CHECK-NEXT: Cost of 0 for VF 8: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 8: 12
+; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK: Cost for VF 16: 4
+; CHECK: LV: Selecting VF: 16
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %src, i64 %indvars.iv
+  %0 = load i8, ptr %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds nuw i8, ptr %dst, i64 %indvars.iv
+  %1 = load i8, ptr %arrayidx2, align 4
+  %add = add nsw i8 %1, %0
+  store i8 %add, ptr %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ret i1 %exitcond.not
+}
+
+attributes #0 = { vscale_range(1, 16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 4faed427b4e17..4bb67c890f3cf 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -8,12 +8,12 @@ target triple = "arm64-apple-macosx14.0.0"
 define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-LABEL: define void @iv_casts(
 ; DEFAULT-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; DEFAULT-NEXT:  iter.check:
+; DEFAULT-NEXT:  entry:
 ; DEFAULT-NEXT:    [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64
 ; DEFAULT-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 16
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; DEFAULT:       vector.memcheck:
@@ -22,12 +22,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
 ; DEFAULT-NEXT:    [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; DEFAULT-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; DEFAULT-NEXT:    br i1 [[DIFF_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
-; DEFAULT:       vector.main.loop.iter.check:
-; DEFAULT-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
-; DEFAULT-NEXT:    [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], [[TMP8]]
-; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK3]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; DEFAULT-NEXT:    br i1 [[DIFF_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; DEFAULT:       vector.ph:
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 16
@@ -73,48 +68,9 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
-; DEFAULT:       vec.epilog.iter.check:
-; DEFAULT-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP46:%.*]] = mul i64 [[TMP45]], 4
-; DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP46]]
-; DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
-; DEFAULT:       vec.epilog.ph:
-; DEFAULT-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT:    [[TMP47:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP48:%.*]] = mul i64 [[TMP47]], 4
-; DEFAULT-NEXT:    [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP48]]
-; DEFAULT-NEXT:    [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
-; DEFAULT-NEXT:    [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP49]], 4
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP51:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT9]] to <vscale x 4 x i16>
-; DEFAULT-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; DEFAULT:       vec.epilog.vector.body:
-; DEFAULT-NEXT:    [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[TMP52:%.*]] = add i64 [[INDEX10]], 0
-; DEFAULT-NEXT:    [[TMP53:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP52]]
-; DEFAULT-NEXT:    [[TMP54:%.*]] = getelementptr i8, ptr [[TMP53]], i32 0
-; DEFAULT-NEXT:    [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP54]], align 1
-; DEFAULT-NEXT:    [[TMP55:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD11]] to <vscale x 4 x i16>
-; DEFAULT-NEXT:    [[TMP56:%.*]] = mul <vscale x 4 x i16> [[TMP55]], [[TMP51]]
-; DEFAULT-NEXT:    [[TMP57:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD11]] to <vscale x 4 x i16>
-; DEFAULT-NEXT:    [[TMP58:%.*]] = or <vscale x 4 x i16> [[TMP56]], [[TMP57]]
-; DEFAULT-NEXT:    [[TMP59:%.*]] = lshr <vscale x 4 x i16> [[TMP58]], trunc (<vscale x 4 x i32> splat (i32 1) to <vscale x 4 x i16>)
-; DEFAULT-NEXT:    [[TMP60:%.*]] = trunc <vscale x 4 x i16> [[TMP59]] to <vscale x 4 x i8>
-; DEFAULT-NEXT:    [[TMP61:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]
-; DEFAULT-NEXT:    [[TMP62:%.*]] = getelementptr i8, ptr [[TMP61]], i32 0
-; DEFAULT-NEXT:    store <vscale x 4 x i8> [[TMP60]], ptr [[TMP62]], align 1
-; DEFAULT-NEXT:    [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP50]]
-; DEFAULT-NEXT:    [[TMP63:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
-; DEFAULT-NEXT:    br i1 [[TMP63]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; DEFAULT:       vec.epilog.middle.block:
-; DEFAULT-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC6]]
-; DEFAULT-NEXT:    br i1 [[CMP_N7]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
-; DEFAULT:       vec.epilog.scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; DEFAULT:       scalar.ph:
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -130,7 +86,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
 ; DEFAULT-NEXT:    store i8 [[CONV36_US]], ptr [[GEP_DST]], align 1
 ; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    ret void
 ;
@@ -277,7 +233,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 1, ptr [[TMP21]], align 4
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; DEFAULT-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -293,7 +249,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 1, ptr [[GEP]], align 4
 ; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    ret void
 ;
@@ -397,6 +353,7 @@ exit:
 }
 
 define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
+;
 ; DEFAULT-LABEL: define void @trunc_ivs_and_store(
 ; DEFAULT-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -445,7 +402,7 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 [[TMP15]], ptr [[TMP24]], align 4
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; DEFAULT-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -464,7 +421,7 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 [[IV_2]], ptr [[GEP]], align 4
 ; DEFAULT-NEXT:    [[IV_1_NEXT]] = add i64 [[IV_1]], 1
 ; DEFAULT-NEXT:    [[EXITCOND_3_NOT:%.*]] = icmp eq i64 [[IV_1]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EXITCOND_3_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EXITCOND_3_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    ret void
 ;
@@ -505,9 +462,9 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ]
-; PRED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ]
+; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE7:%.*]] ]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE7]] ]
+; PRED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE7]] ]
 ; PRED-NEXT:    [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
 ; PRED-NEXT:    [[TMP17:%.*]] = mul <4 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]]
 ; PRED-NEXT:    [[TMP18:%.*]] = zext <4 x i32> [[TMP17]] to <4 x i64>
@@ -521,32 +478,32 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; PRED:       pred.store.continue:
 ; PRED-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 1
-; PRED-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; PRED:       pred.store.if3:
+; PRED-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3:%.*]]
+; PRED:       pred.store.if2:
 ; PRED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP18]], i32 1
 ; PRED-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP24]]
 ; PRED-NEXT:    [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 1
 ; PRED-NEXT:    store i32 [[TMP26]], ptr [[TMP25]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; PRED:       pred.store.continue4:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE3]]
+; PRED:       pred.store.continue3:
 ; PRED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 2
-; PRED-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; PRED:       pred.store.if5:
+; PRED-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
+; PRED:       pred.store.if4:
 ; PRED-NEXT:    [[TMP28:%.*]] = extractelement <4 x i64> [[TMP18]], i32 2
 ; PRED-NEXT:    [[TMP29:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP28]]
 ; PRED-NEXT:    [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 2
 ; PRED-NEXT:    store i32 [[TMP30]], ptr [[TMP29]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
-; PRED:       pred.store.continue6:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE5]]
+; PRED:       pred.store.continue5:
 ; PRED-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 3
-; PRED-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
-; PRED:       pred.store.if7:
+; PRED-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
+; PRED:       pred.store.if6:
 ; PRED-NEXT:    [[TMP32:%.*]] = extractelement <4 x i64> [[TMP18]], i32 3
 ; PRED-NEXT:    [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP32]]
 ; PRED-NEXT:    [[TMP34:%.*]] = add i32 [[OFFSET_IDX]], 3
 ; PRED-NEXT:    store i32 [[TMP34]], ptr [[TMP33]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE8]]
-; PRED:       pred.store.continue8:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE7]]
+; PRED:       pred.store.continue7:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[TMP16]])
 ; PRED-NEXT:    [[TMP35:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
@@ -557,11 +514,11 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
 ; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
-; PRED-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL8]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
 ; PRED-NEXT:    [[IV_1_TRUNC:%.*]] = trunc i64 [[IV_1]] to i32
 ; PRED-NEXT:    [[IV_1_MUL:%.*]] = mul i32 [[MUL]], [[IV_1_TRUNC]]
 ; PRED-NEXT:    [[IV_2_NEXT]] = add i32 [[IV_2]], 1
@@ -643,7 +600,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 [[TMP14]], ptr [[TMP23]], align 4
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; DEFAULT-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -662,7 +619,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; DEFAULT-NEXT:    store i32 [[IV_2]], ptr [[GEP]], align 4
 ; DEFAULT-NEXT:    [[IV_1_NEXT]] = add i64 [[IV_1]], 1
 ; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_1]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    ret void
 ;
@@ -702,9 +659,9 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE7:%.*]] ]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE7]] ]
-; PRED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE7]] ]
+; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
+; PRED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
 ; PRED-NEXT:    [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
 ; PRED-NEXT:    [[TMP16:%.*]] = mul <4 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]]
 ; PRED-NEXT:    [[TMP17:%.*]] = zext <4 x i32> [[TMP16]] to <4 x i64>
@@ -718,32 +675,32 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; PRED:       pred.store.continue:
 ; PRED-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 1
-; PRED-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3:%.*]]
-; PRED:       pred.store.if2:
+; PRED-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; PRED:       pred.store.if1:
 ; PRED-NEXT:    [[TMP23:%.*]] = extractelement <4 x i64> [[TMP17]], i32 1
 ; PRED-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP23]]
 ; PRED-NEXT:    [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], 1
 ; PRED-NEXT:    store i32 [[TMP25]], ptr [[TMP24]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; PRED:       pred.store.continue3:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; PRED:       pred.store.continue2:
 ; PRED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 2
-; PRED-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
-; PRED:       pred.store.if4:
+; PRED-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; PRED:       pred.store.if3:
 ; PRED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i64> [[TMP17]], i32 2
 ; PRED-NEXT:    [[TMP28:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]
 ; PRED-NEXT:    [[TMP29:%.*]] = add i32 [[OFFSET_IDX]], 2
 ; PRED-NEXT:    store i32 [[TMP29]], ptr [[TMP28]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE5]]
-; PRED:       pred.store.continue5:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; PRED:       pred.store.continue4:
 ; PRED-NEXT:    [[TMP30:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 3
-; PRED-NEXT:    br i1 [[TMP30]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
-; PRED:       pred.store.if6:
+; PRED-NEXT:    br i1 [[TMP30]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; PRED:       pred.store.if5:
 ; PRED-NEXT:    [[TMP31:%.*]] = extractelement <4 x i64> [[TMP17]], i32 3
 ; PRED-NEXT:    [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP31]]
 ; PRED-NEXT:    [[TMP33:%.*]] = add i32 [[OFFSET_IDX]], 3
 ; PRED-NEXT:    store i32 [[TMP33]], ptr [[TMP32]], align 4
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE7]]
-; PRED:       pred.store.continue7:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; PRED:       pred.store.continue6:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[TMP15]])
 ; PRED-NEXT:    [[TMP34:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
@@ -754,11 +711,11 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
 ; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
 ; PRED-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
-; PRED-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL7]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
 ; PRED-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV_1]] to i32
 ; PRED-NEXT:    [[IV_MUL:%.*]] = mul i32 [[ADD]], [[IV_TRUNC]]
 ; PRED-NEXT:    [[IV_2_NEXT]] = add i32 [[IV_2]], 1
@@ -771,6 +728,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       exit:
 ; PRED-NEXT:    ret void
 ;
+
 entry:
   %add = add i32 %x, 1
   br label %loop
@@ -824,7 +782,7 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
 ; DEFAULT-NEXT:    store i32 0, ptr [[TMP10]], align 8
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -840,7 +798,7 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
 ; DEFAULT-NEXT:    [[IV_1_NEXT]] = add i32 [[IV_1]], 1
 ; DEFAULT-NEXT:    [[IV_EXT]] = zext i32 [[IV_1_NEXT]] to i64
 ; DEFAULT-NEXT:    [[C:%.*]] = icmp ult i64 [[IV_EXT]], [[N]]
-; DEFAULT-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    ret void
 ;
@@ -865,49 +823,49 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
 ; PRED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX1]], 1
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT3]], <2 x i64> poison, <2 x i32> zeroinitializer
-; PRED-NEXT:    br label [[LOOP:%.*]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT2]], <2 x i64> poison, <2 x i32> zeroinitializer
+; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE5:%.*]] ]
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[INDEX]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; PRED-NEXT:    [[VEC_IV:%.*]] = add <2 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1>
-; PRED-NEXT:    [[TMP7:%.*]] = icmp ule <2 x i64> [[VEC_IV]], [[BROADCAST_SPLAT4]]
+; PRED-NEXT:    [[TMP7:%.*]] = icmp ule <2 x i64> [[VEC_IV]], [[BROADCAST_SPLAT3]]
 ; PRED-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
 ; PRED-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; PRED:       pred.store.if:
-; PRED-NEXT:    [[IV_CONV:%.*]] = add i64 [[INDEX]], 0
-; PRED-NEXT:    [[GEP:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV]], i32 2
-; PRED-NEXT:    store i32 0, ptr [[GEP]], align 8
+; PRED-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; PRED-NEXT:    [[TMP10:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP9]], i32 2
+; PRED-NEXT:    store i32 0, ptr [[TMP10]], align 8
 ; PRED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; PRED:       pred.store.continue:
 ; PRED-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
-; PRED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
-; PRED:       pred.store.if5:
+; PRED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5]]
+; PRED:       pred.store.if4:
 ; PRED-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 1
 ; PRED-NEXT:    [[TMP13:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP12]], i32 2
 ; PRED-NEXT:    store i32 0, ptr [[TMP13]], align 8
-; PRED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
-; PRED:       pred.store.continue6:
+; PRED-NEXT:    br label [[PRED_STORE_CONTINUE5]]
+; PRED:       pred.store.continue5:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; PRED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; PRED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; PRED:       middle.block:
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
 ; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; PRED-NEXT:    br label [[LOOP1:%.*]]
+; PRED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
-; PRED-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP1]] ]
-; PRED-NEXT:    [[IV_CONV1:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[IV_EXT:%.*]], [[LOOP1]] ]
-; PRED-NEXT:    [[GEP1:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV1]], i32 2
-; PRED-NEXT:    store i32 0, ptr [[GEP1]], align 8
+; PRED-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ], [ [[IV_EXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[GEP:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV]], i32 2
+; PRED-NEXT:    store i32 0, ptr [[GEP]], align 8
 ; PRED-NEXT:    [[IV_1_NEXT]] = add i32 [[IV_1]], 1
 ; PRED-NEXT:    [[IV_EXT]] = zext i32 [[IV_1_NEXT]] to i64
 ; PRED-NEXT:    [[C:%.*]] = icmp ult i64 [[IV_EXT]], [[N]]
-; PRED-NEXT:    br i1 [[C]], label [[LOOP1]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; PRED-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
 ; PRED:       exit:
 ; PRED-NEXT:    ret void
 ;
@@ -934,16 +892,15 @@ attributes #0 = { "target-features"="+sve" }
 ; DEFAULT: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; DEFAULT: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; DEFAULT: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
-; DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
-; DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]]}
-; DEFAULT: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}
-; DEFAULT: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+; DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; DEFAULT: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+; DEFAULT: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; DEFAULT: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; DEFAULT: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; DEFAULT: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
 ;.
 ; PRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; PRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index f7f36d011535f..4d070a4d02c90 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -243,9 +243,8 @@ define void @wide_truncated_iv(ptr %dst) {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 192, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 192, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i8
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i8> [[DOTSPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i8> [[DOTSPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index 8320608d67588..3a20e4a0bf8c8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -103,7 +103,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
 ; INTERLEAVE-4-NEXT:    [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC10]]
 ; INTERLEAVE-4-NEXT:    br i1 [[CMP_N11]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-4:       vec.epilog.scalar.ph:
-; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC10]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; INTERLEAVE-4-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-4:       loop:
 ; INTERLEAVE-4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -198,7 +198,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
 ; INTERLEAVE-2-NEXT:    [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC8]]
 ; INTERLEAVE-2-NEXT:    br i1 [[CMP_N9]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-2:       vec.epilog.scalar.ph:
-; INTERLEAVE-2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; INTERLEAVE-2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; INTERLEAVE-2-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-2:       loop:
 ; INTERLEAVE-2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
index fc2f8a0dcabf5..07aa57d329c7d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
@@ -57,8 +57,8 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; INTERLEAVE-4-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; INTERLEAVE-4:       vec.epilog.ph:
-; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; INTERLEAVE-4-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; INTERLEAVE-4-NEXT:    [[N_MOD_VF10:%.*]] = urem i64 [[N]], 4
 ; INTERLEAVE-4-NEXT:    [[N_VEC11:%.*]] = sub i64 [[N]], [[N_MOD_VF10]]
 ; INTERLEAVE-4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
@@ -80,7 +80,7 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-NEXT:    br i1 [[CMP_N16]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; INTERLEAVE-4:       vec.epilog.scalar.ph:
 ; INTERLEAVE-4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ]
+; INTERLEAVE-4-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; INTERLEAVE-4-NEXT:    br label [[LOOP:%.*]]
 ; INTERLEAVE-4:       loop:
 ; INTERLEAVE-4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
index 89d5ba6e879df..6d37dd9a4c209 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
@@ -95,10 +95,10 @@ define void @saddsat(ptr nocapture readonly %pSrc, i16 signext %offset, ptr noca
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PSRC_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL11]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR3:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL14]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_08]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_08]], i64 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[PSRC_ADDR_08]], align 2
 ; CHECK-NEXT:    [[TMP13:%.*]] = tail call i16 @llvm.sadd.sat.i16(i16 [[TMP12]], i16 [[OFFSET]])
-; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds i8, ptr [[PDST_ADDR_07]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_07]], i64 2
 ; CHECK-NEXT:    store i16 [[TMP13]], ptr [[PDST_ADDR_07]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_09]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
@@ -134,6 +134,7 @@ while.end:                                        ; preds = %while.body, %entry
 ; CHECK-COST: Cost of 1 for VF 8: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
 ; CHECK-COST: Cost of 1 for VF 16: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
 
+
 define void @umin(ptr nocapture readonly %pSrc, i8 signext %offset, ptr nocapture noalias %pDst, i32 %blockSize) #0 {
 ; CHECK-LABEL: @umin(
 ; CHECK-NEXT:  entry:
@@ -209,10 +210,10 @@ define void @umin(ptr nocapture readonly %pSrc, i8 signext %offset, ptr nocaptur
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PSRC_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL10]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR3:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL13]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_08]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_08]], i64 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[PSRC_ADDR_08]], align 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = tail call i8 @llvm.umin.i8(i8 [[TMP8]], i8 [[OFFSET]])
-; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds i8, ptr [[PDST_ADDR_07]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_07]], i64 1
 ; CHECK-NEXT:    store i8 [[TMP9]], ptr [[PDST_ADDR_07]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_09]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index 59b879e0867dc..a02323ab2108f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -1,7 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -mattr=+sve 2>%t | FileCheck %s
-; RUN: cat %t | FileCheck %s --check-prefix=DEBUG
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -mattr=+sve 2>%t | FileCheck %s --check-prefixes=CHECK,CHECK-VS1
+; RUN: cat %t | FileCheck %s --check-prefixes=DEBUG,DEBUG-VS1
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -mcpu=neoverse-v1 -sve-tail-folding=disabled 2>%t | FileCheck %s --check-prefixes=CHECK,CHECK-VS2
+; RUN: cat %t | FileCheck %s --check-prefixes=DEBUG,DEBUG-VS2
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -9,8 +11,10 @@ target triple = "aarch64-unknown-linux-gnu"
 ; DEBUG: LV: Found trip count: 0
 ; DEBUG: LV: Found maximum trip count: 19
 ; DEBUG: LV: IC is 1
-; DEBUG: LV: VF is vscale x 8
-; DEBUG: Main Loop VF:vscale x 8, Main Loop UF:1, Epilogue Loop VF:vscale x 4, Epilogue Loop UF:1
+; DEBUG-VS1: LV: VF is vscale x 16
+; DEBUG-VS1: Main Loop VF:vscale x 16, Main Loop UF:1, Epilogue Loop VF:vscale x 8, Epilogue Loop UF:1
+; DEBUG-VS2: LV: VF is vscale x 8
+; DEBUG-VS2: Main Loop VF:vscale x 8, Main Loop UF:1, Epilogue Loop VF:vscale x 4, Epilogue Loop UF:1
 
 ; DEBUG-LABEL: LV: Checking a loop in 'trip_count_too_small'
 ; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar iteration overheads are incurred.
@@ -32,113 +36,221 @@ target triple = "aarch64-unknown-linux-gnu"
 ; DEBUG: Executing best plan with VF=vscale x 16, UF=1
 
 define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef %val) {
-; CHECK-LABEL: define void @low_vf_ic_is_better(
-; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
-; CHECK-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
-; CHECK:       [[ITER_CHECK]]:
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
-; CHECK-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
-; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TC]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TC]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 19, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP6]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP8]], 4294967295
-; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX]]
-; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP21]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP22]], align 1
-; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 8 x i8> [[TMP23]], ptr [[TMP22]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT:    [[IND_END5:%.*]] = add i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 4
-; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP33]]
-; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
-; CHECK:       [[VEC_EPILOG_PH]]:
-; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP35:%.*]] = mul i64 [[TMP34]], 4
-; CHECK-NEXT:    [[N_MOD_VF3:%.*]] = urem i64 [[TMP3]], [[TMP35]]
-; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF3]]
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC4]]
-; CHECK-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 4
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[CONV]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX7:%.*]] = add i64 [[TMP0]], [[INDEX6]]
-; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[OFFSET_IDX7]], 0
-; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[TMP39]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i8>, ptr [[TMP40]], align 1
-; CHECK-NEXT:    [[TMP41:%.*]] = add <vscale x 4 x i8> [[WIDE_LOAD7]], [[BROADCAST_SPLAT9]]
-; CHECK-NEXT:    store <vscale x 4 x i8> [[TMP41]], ptr [[TMP40]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i64 [[INDEX6]], [[TMP37]]
-; CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC4]]
-; CHECK-NEXT:    br i1 [[TMP42]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC4]]
-; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ]
-; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
-; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[TMP43]], [[CONV]]
-; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[TMP44:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP44]], 19
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:       [[WHILE_END_LOOPEXIT]]:
-; CHECK-NEXT:    br label %[[WHILE_END]]
-; CHECK:       [[WHILE_END]]:
-; CHECK-NEXT:    ret void
+; CHECK-VS1-LABEL: define void @low_vf_ic_is_better(
+; CHECK-VS1-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS1-NEXT:  [[ENTRY:.*:]]
+; CHECK-VS1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
+; CHECK-VS1-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
+; CHECK-VS1:       [[ITER_CHECK]]:
+; CHECK-VS1-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-VS1-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-VS1-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-VS1-NEXT:    [[TMP1:%.*]] = add i32 [[TC]], 1
+; CHECK-VS1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-VS1-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
+; CHECK-VS1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
+; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK-VS1:       [[VECTOR_SCEVCHECK]]:
+; CHECK-VS1-NEXT:    [[TMP6:%.*]] = add i32 [[TC]], 1
+; CHECK-VS1-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-VS1-NEXT:    [[TMP8:%.*]] = sub i64 19, [[TMP7]]
+; CHECK-VS1-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-VS1-NEXT:    [[TMP10:%.*]] = add i32 [[TMP6]], [[TMP9]]
+; CHECK-VS1-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP6]]
+; CHECK-VS1-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP8]], 4294967295
+; CHECK-VS1-NEXT:    [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-VS1-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 16
+; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
+; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VS1:       [[VECTOR_PH]]:
+; CHECK-VS1-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 16
+; CHECK-VS1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
+; CHECK-VS1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-VS1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 16
+; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VS1:       [[VECTOR_BODY]]:
+; CHECK-VS1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS1-NEXT:    [[TMP20:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-VS1-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], 0
+; CHECK-VS1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP21]]
+; CHECK-VS1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP22]], i32 0
+; CHECK-VS1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP23]], align 1
+; CHECK-VS1-NEXT:    [[TMP24:%.*]] = add <vscale x 16 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-VS1-NEXT:    store <vscale x 16 x i8> [[TMP24]], ptr [[TMP23]], align 1
+; CHECK-VS1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-VS1-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS1-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VS1:       [[MIDDLE_BLOCK]]:
+; CHECK-VS1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-VS1-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS1:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS1-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS1-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
+; CHECK-VS1-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 8
+; CHECK-VS1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
+; CHECK-VS1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK-VS1:       [[VEC_EPILOG_PH]]:
+; CHECK-VS1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS1-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 8
+; CHECK-VS1-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
+; CHECK-VS1-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
+; CHECK-VS1-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
+; CHECK-VS1-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS1-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
+; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS1-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS1:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS1-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS1-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX5]]
+; CHECK-VS1-NEXT:    [[TMP32:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-VS1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP32]]
+; CHECK-VS1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[TMP33]], i32 0
+; CHECK-VS1-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x i8>, ptr [[TMP34]], align 1
+; CHECK-VS1-NEXT:    [[TMP35:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD6]], [[BROADCAST_SPLAT8]]
+; CHECK-VS1-NEXT:    store <vscale x 8 x i8> [[TMP35]], ptr [[TMP34]], align 1
+; CHECK-VS1-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP31]]
+; CHECK-VS1-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]]
+; CHECK-VS1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VS1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS1-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC3]]
+; CHECK-VS1-NEXT:    br i1 [[CMP_N10]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-VS1:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ]
+; CHECK-VS1-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK-VS1:       [[WHILE_BODY]]:
+; CHECK-VS1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-VS1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[IV]]
+; CHECK-VS1-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-VS1-NEXT:    [[ADD:%.*]] = add i8 [[TMP37]], [[CONV]]
+; CHECK-VS1-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
+; CHECK-VS1-NEXT:    [[TMP38:%.*]] = and i64 [[IV_NEXT]], 4294967295
+; CHECK-VS1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP38]], 19
+; CHECK-VS1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VS1:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-VS1-NEXT:    br label %[[WHILE_END]]
+; CHECK-VS1:       [[WHILE_END]]:
+; CHECK-VS1-NEXT:    ret void
+;
+; CHECK-VS2-LABEL: define void @low_vf_ic_is_better(
+; CHECK-VS2-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS2-NEXT:  [[ENTRY:.*:]]
+; CHECK-VS2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
+; CHECK-VS2-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
+; CHECK-VS2:       [[ITER_CHECK]]:
+; CHECK-VS2-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-VS2-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-VS2-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-VS2-NEXT:    [[TMP1:%.*]] = add i32 [[TC]], 1
+; CHECK-VS2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-VS2-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
+; CHECK-VS2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
+; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK-VS2:       [[VECTOR_SCEVCHECK]]:
+; CHECK-VS2-NEXT:    [[TMP6:%.*]] = add i32 [[TC]], 1
+; CHECK-VS2-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-VS2-NEXT:    [[TMP8:%.*]] = sub i64 19, [[TMP7]]
+; CHECK-VS2-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-VS2-NEXT:    [[TMP10:%.*]] = add i32 [[TMP6]], [[TMP9]]
+; CHECK-VS2-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP6]]
+; CHECK-VS2-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP8]], 4294967295
+; CHECK-VS2-NEXT:    [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-VS2-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS2:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS2-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
+; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VS2:       [[VECTOR_PH]]:
+; CHECK-VS2-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 8
+; CHECK-VS2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
+; CHECK-VS2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-VS2-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VS2:       [[VECTOR_BODY]]:
+; CHECK-VS2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS2-NEXT:    [[TMP20:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-VS2-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], 0
+; CHECK-VS2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP21]]
+; CHECK-VS2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP22]], i32 0
+; CHECK-VS2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP23]], align 1
+; CHECK-VS2-NEXT:    [[TMP24:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-VS2-NEXT:    store <vscale x 8 x i8> [[TMP24]], ptr [[TMP23]], align 1
+; CHECK-VS2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-VS2-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS2-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VS2:       [[MIDDLE_BLOCK]]:
+; CHECK-VS2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-VS2-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS2:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS2-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS2-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
+; CHECK-VS2-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 4
+; CHECK-VS2-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
+; CHECK-VS2-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK-VS2:       [[VEC_EPILOG_PH]]:
+; CHECK-VS2-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS2-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-VS2-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
+; CHECK-VS2-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
+; CHECK-VS2-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
+; CHECK-VS2-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS2-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 4
+; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VS2-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS2:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS2-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX5]]
+; CHECK-VS2-NEXT:    [[TMP32:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-VS2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP32]]
+; CHECK-VS2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[TMP33]], i32 0
+; CHECK-VS2-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i8>, ptr [[TMP34]], align 1
+; CHECK-VS2-NEXT:    [[TMP35:%.*]] = add <vscale x 4 x i8> [[WIDE_LOAD6]], [[BROADCAST_SPLAT8]]
+; CHECK-VS2-NEXT:    store <vscale x 4 x i8> [[TMP35]], ptr [[TMP34]], align 1
+; CHECK-VS2-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP31]]
+; CHECK-VS2-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]]
+; CHECK-VS2-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VS2:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS2-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC3]]
+; CHECK-VS2-NEXT:    br i1 [[CMP_N10]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-VS2:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ]
+; CHECK-VS2-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK-VS2:       [[WHILE_BODY]]:
+; CHECK-VS2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-VS2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[IV]]
+; CHECK-VS2-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-VS2-NEXT:    [[ADD:%.*]] = add i8 [[TMP37]], [[CONV]]
+; CHECK-VS2-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
+; CHECK-VS2-NEXT:    [[TMP38:%.*]] = and i64 [[IV_NEXT]], 4294967295
+; CHECK-VS2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP38]], 19
+; CHECK-VS2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VS2:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-VS2-NEXT:    br label %[[WHILE_END]]
+; CHECK-VS2:       [[WHILE_END]]:
+; CHECK-VS2-NEXT:    ret void
 ;
 entry:
   %cmp7 = icmp ult i32 %tc, 19
@@ -387,11 +499,19 @@ while.end:
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
+; CHECK-VS1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VS1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VS1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VS1: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-VS1: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK-VS1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-VS1: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
+;.
+; CHECK-VS2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VS2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VS2: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VS2: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-VS2: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK-VS2: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-VS2: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 307b47c42a423..0cea16d103678 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt -p loop-vectorize -S %s | FileCheck --check-prefixes=DEFAULT %s
-; RUN: opt -p loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s | FileCheck --check-prefixes=PRED %s
+; RUN: opt -p loop-vectorize -mcpu=neoverse-v1 -S %s | FileCheck --check-prefixes=VSCALEFORTUNING2 %s
+; RUN: opt -p loop-vectorize -mcpu=neoverse-v1 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s | FileCheck --check-prefixes=PRED %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-macosx14.0.0"
@@ -9,104 +10,12 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; DEFAULT-LABEL: define i32 @chained_recurrences(
 ; DEFAULT-SAME: i32 [[X:%.*]], i64 [[Y:%.*]], ptr [[SRC_1:%.*]], i32 [[Z:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
 ; DEFAULT-NEXT:  entry:
-; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
-; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
-; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; DEFAULT:       vector.ph:
-; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
-; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
-; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; DEFAULT-NEXT:    [[TMP13:%.*]] = add i64 [[Y]], 1
-; DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP13]]
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP25:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT7]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP31:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT7]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP33:%.*]] = or <vscale x 4 x i32> [[TMP25]], [[TMP31]]
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP39:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT9]], [[BROADCAST_SPLAT7]]
-; DEFAULT-NEXT:    [[TMP14:%.*]] = and <vscale x 4 x i32> [[TMP39]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP43:%.*]] = xor <vscale x 4 x i32> [[TMP14]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP45:%.*]] = zext <vscale x 4 x i32> [[TMP43]] to <vscale x 4 x i64>
-; DEFAULT-NEXT:    [[TMP61:%.*]] = extractelement <vscale x 4 x i64> [[TMP45]], i32 0
-; DEFAULT-NEXT:    [[TMP62:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP61]]
-; DEFAULT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP62]], i64 0
-; DEFAULT-NEXT:    [[TMP47:%.*]] = shufflevector <vscale x 4 x ptr> [[DOTSPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; DEFAULT-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
-; DEFAULT-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
-; DEFAULT-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP9]]
-; DEFAULT-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; DEFAULT-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], 4
-; DEFAULT-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP11]], 1
-; DEFAULT-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP12]]
-; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
-; DEFAULT:       vector.body:
-; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT5:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT1]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP57:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP58:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP15]], align 4
-; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP18]], i64 0
-; DEFAULT-NEXT:    [[BROADCAST_SPLAT5]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[BROADCAST_SPLAT5]], i32 -1)
-; DEFAULT-NEXT:    [[TMP20]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT5]], <vscale x 4 x i32> [[BROADCAST_SPLAT5]], i32 -1)
-; DEFAULT-NEXT:    [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR2]], <vscale x 4 x i32> [[TMP19]], i32 -1)
-; DEFAULT-NEXT:    [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[TMP19]], <vscale x 4 x i32> [[TMP20]], i32 -1)
-; DEFAULT-NEXT:    [[TMP23:%.*]] = or <vscale x 4 x i32> [[TMP21]], [[BROADCAST_SPLAT7]]
-; DEFAULT-NEXT:    [[TMP24:%.*]] = or <vscale x 4 x i32> [[TMP22]], [[BROADCAST_SPLAT7]]
-; DEFAULT-NEXT:    [[TMP34:%.*]] = shl <vscale x 4 x i32> [[TMP23]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP32:%.*]] = shl <vscale x 4 x i32> [[TMP24]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP29:%.*]] = or <vscale x 4 x i32> [[TMP34]], splat (i32 2)
-; DEFAULT-NEXT:    [[TMP30:%.*]] = or <vscale x 4 x i32> [[TMP32]], splat (i32 2)
-; DEFAULT-NEXT:    [[TMP35:%.*]] = or <vscale x 4 x i32> [[TMP33]], [[TMP29]]
-; DEFAULT-NEXT:    [[TMP36:%.*]] = or <vscale x 4 x i32> [[TMP33]], [[TMP30]]
-; DEFAULT-NEXT:    [[TMP37:%.*]] = or <vscale x 4 x i32> [[TMP35]], [[BROADCAST_SPLAT7]]
-; DEFAULT-NEXT:    [[TMP38:%.*]] = or <vscale x 4 x i32> [[TMP36]], [[BROADCAST_SPLAT7]]
-; DEFAULT-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP47]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; DEFAULT-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP47]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; DEFAULT-NEXT:    [[TMP49:%.*]] = lshr <vscale x 4 x i32> [[TMP37]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP50:%.*]] = lshr <vscale x 4 x i32> [[TMP38]], splat (i32 1)
-; DEFAULT-NEXT:    [[TMP51:%.*]] = zext <vscale x 4 x i32> [[TMP49]] to <vscale x 4 x i64>
-; DEFAULT-NEXT:    [[TMP52:%.*]] = zext <vscale x 4 x i32> [[TMP50]] to <vscale x 4 x i64>
-; DEFAULT-NEXT:    [[TMP53:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP51]]
-; DEFAULT-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP52]]
-; DEFAULT-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP53]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; DEFAULT-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP54]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; DEFAULT-NEXT:    [[TMP55:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
-; DEFAULT-NEXT:    [[TMP56:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[VEC_PHI3]]
-; DEFAULT-NEXT:    [[TMP57]] = or <vscale x 4 x i32> [[TMP55]], [[WIDE_MASKED_GATHER11]]
-; DEFAULT-NEXT:    [[TMP58]] = or <vscale x 4 x i32> [[TMP56]], [[WIDE_MASKED_GATHER12]]
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; DEFAULT-NEXT:    [[TMP59:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP59]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; DEFAULT:       middle.block:
-; DEFAULT-NEXT:    [[BIN_RDX:%.*]] = or <vscale x 4 x i32> [[TMP58]], [[TMP57]]
-; DEFAULT-NEXT:    [[TMP60:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; DEFAULT-NEXT:    [[TMP64:%.*]] = call i32 @llvm.vscale.i32()
-; DEFAULT-NEXT:    [[TMP65:%.*]] = mul i32 [[TMP64]], 4
-; DEFAULT-NEXT:    [[TMP66:%.*]] = sub i32 [[TMP65]], 1
-; DEFAULT-NEXT:    [[VECTOR_RECUR_EXTRACT13:%.*]] = extractelement <vscale x 4 x i32> [[TMP20]], i32 [[TMP66]]
-; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; DEFAULT-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; DEFAULT-NEXT:    [[SCALAR_RECUR_INIT14:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT13]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; DEFAULT-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP60]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
-; DEFAULT-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP68:%.*]], [[LOOP]] ]
-; DEFAULT-NEXT:    [[SCALAR_RECUR15:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT14]], [[SCALAR_PH]] ], [ [[SCALAR_RECUR]], [[LOOP]] ]
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; DEFAULT-NEXT:    [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_2:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP68:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT:    [[SCALAR_RECUR15:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP0]], [[LOOP]] ]
+; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT:    [[SUM_RED:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[RED_2:%.*]], [[LOOP]] ]
 ; DEFAULT-NEXT:    [[TMP67:%.*]] = add i64 [[Y]], 1
 ; DEFAULT-NEXT:    [[GEP_1:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP67]]
 ; DEFAULT-NEXT:    [[TMP68]] = load i32, ptr [[GEP_1]], align 4
@@ -132,109 +41,240 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; DEFAULT-NEXT:    [[RED_1:%.*]] = or i32 [[TMP74]], [[SUM_RED]]
 ; DEFAULT-NEXT:    [[RED_2]] = or i32 [[RED_1]], [[TMP75]]
 ; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[Y]]
-; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
 ; DEFAULT:       exit:
-; DEFAULT-NEXT:    [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], [[LOOP]] ], [ [[TMP60]], [[MIDDLE_BLOCK]] ]
+; DEFAULT-NEXT:    [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], [[LOOP]] ]
 ; DEFAULT-NEXT:    ret i32 [[RED_2_LCSSA]]
 ;
+; VSCALEFORTUNING2-LABEL: define i32 @chained_recurrences(
+; VSCALEFORTUNING2-SAME: i32 [[X:%.*]], i64 [[Y:%.*]], ptr [[SRC_1:%.*]], i32 [[Z:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
+; VSCALEFORTUNING2-NEXT:  entry:
+; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VSCALEFORTUNING2:       vector.ph:
+; VSCALEFORTUNING2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP7:%.*]] = add i64 [[Y]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP7]]
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    [[TMP9:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP10:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = or <vscale x 4 x i32> [[TMP9]], [[TMP10]]
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    [[TMP12:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP13:%.*]] = and <vscale x 4 x i32> [[TMP12]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP14:%.*]] = xor <vscale x 4 x i32> [[TMP13]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP15:%.*]] = zext <vscale x 4 x i32> [[TMP14]] to <vscale x 4 x i64>
+; VSCALEFORTUNING2-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i64> [[TMP15]], i32 0
+; VSCALEFORTUNING2-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP16]]
+; VSCALEFORTUNING2-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP17]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[DOTSPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; VSCALEFORTUNING2-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP20:%.*]] = sub i32 [[TMP19]], 1
+; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP20]]
+; VSCALEFORTUNING2-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
+; VSCALEFORTUNING2-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP22]], 1
+; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP23]]
+; VSCALEFORTUNING2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VSCALEFORTUNING2:       vector.body:
+; VSCALEFORTUNING2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT7:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR4:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT3]], [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP48:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP8]], align 4
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP24]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT7]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    [[TMP25:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[BROADCAST_SPLAT7]], i32 -1)
+; VSCALEFORTUNING2-NEXT:    [[TMP26]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[BROADCAST_SPLAT7]], <vscale x 4 x i32> [[BROADCAST_SPLAT7]], i32 -1)
+; VSCALEFORTUNING2-NEXT:    [[TMP27:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR4]], <vscale x 4 x i32> [[TMP25]], i32 -1)
+; VSCALEFORTUNING2-NEXT:    [[TMP28:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[TMP25]], <vscale x 4 x i32> [[TMP26]], i32 -1)
+; VSCALEFORTUNING2-NEXT:    [[TMP29:%.*]] = or <vscale x 4 x i32> [[TMP27]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP30:%.*]] = or <vscale x 4 x i32> [[TMP28]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP31:%.*]] = shl <vscale x 4 x i32> [[TMP29]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP32:%.*]] = shl <vscale x 4 x i32> [[TMP30]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP33:%.*]] = or <vscale x 4 x i32> [[TMP31]], splat (i32 2)
+; VSCALEFORTUNING2-NEXT:    [[TMP34:%.*]] = or <vscale x 4 x i32> [[TMP32]], splat (i32 2)
+; VSCALEFORTUNING2-NEXT:    [[TMP35:%.*]] = or <vscale x 4 x i32> [[TMP11]], [[TMP33]]
+; VSCALEFORTUNING2-NEXT:    [[TMP36:%.*]] = or <vscale x 4 x i32> [[TMP11]], [[TMP34]]
+; VSCALEFORTUNING2-NEXT:    [[TMP37:%.*]] = or <vscale x 4 x i32> [[TMP35]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP38:%.*]] = or <vscale x 4 x i32> [[TMP36]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[DOTSPLAT]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT:    [[WIDE_MASKED_GATHER8:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[DOTSPLAT]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT:    [[TMP39:%.*]] = lshr <vscale x 4 x i32> [[TMP37]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP40:%.*]] = lshr <vscale x 4 x i32> [[TMP38]], splat (i32 1)
+; VSCALEFORTUNING2-NEXT:    [[TMP41:%.*]] = zext <vscale x 4 x i32> [[TMP39]] to <vscale x 4 x i64>
+; VSCALEFORTUNING2-NEXT:    [[TMP42:%.*]] = zext <vscale x 4 x i32> [[TMP40]] to <vscale x 4 x i64>
+; VSCALEFORTUNING2-NEXT:    [[TMP43:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP41]]
+; VSCALEFORTUNING2-NEXT:    [[TMP44:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP42]]
+; VSCALEFORTUNING2-NEXT:    [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP43]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP44]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; VSCALEFORTUNING2-NEXT:    [[TMP45:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
+; VSCALEFORTUNING2-NEXT:    [[TMP46:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER8]], [[VEC_PHI5]]
+; VSCALEFORTUNING2-NEXT:    [[TMP47]] = or <vscale x 4 x i32> [[TMP45]], [[WIDE_MASKED_GATHER9]]
+; VSCALEFORTUNING2-NEXT:    [[TMP48]] = or <vscale x 4 x i32> [[TMP46]], [[WIDE_MASKED_GATHER10]]
+; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; VSCALEFORTUNING2-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VSCALEFORTUNING2:       middle.block:
+; VSCALEFORTUNING2-NEXT:    [[BIN_RDX:%.*]] = or <vscale x 4 x i32> [[TMP48]], [[TMP47]]
+; VSCALEFORTUNING2-NEXT:    [[TMP50:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; VSCALEFORTUNING2-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vscale.i32()
+; VSCALEFORTUNING2-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP53:%.*]] = sub i32 [[TMP52]], 1
+; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP26]], i32 [[TMP53]]
+; VSCALEFORTUNING2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; VSCALEFORTUNING2:       scalar.ph:
+; VSCALEFORTUNING2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; VSCALEFORTUNING2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP24]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT:    [[SCALAR_RECUR_INIT11:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP50]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT:    br label [[LOOP:%.*]]
+; VSCALEFORTUNING2:       loop:
+; VSCALEFORTUNING2-NEXT:    [[TMP54:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP57:%.*]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[TMP55:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT11]], [[SCALAR_PH]] ], [ [[TMP54]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_2:%.*]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[TMP56:%.*]] = add i64 [[Y]], 1
+; VSCALEFORTUNING2-NEXT:    [[GEP_1:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP56]]
+; VSCALEFORTUNING2-NEXT:    [[TMP57]] = load i32, ptr [[GEP_1]], align 4
+; VSCALEFORTUNING2-NEXT:    [[OR3:%.*]] = or i32 [[TMP55]], [[X]]
+; VSCALEFORTUNING2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; VSCALEFORTUNING2-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP58:%.*]] = shl i32 [[OR3]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP59:%.*]] = or i32 [[TMP58]], 2
+; VSCALEFORTUNING2-NEXT:    [[SHL19:%.*]] = shl i32 [[X]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP60:%.*]] = or i32 [[SHR]], [[SHL19]]
+; VSCALEFORTUNING2-NEXT:    [[TMP61:%.*]] = or i32 [[TMP60]], [[TMP59]]
+; VSCALEFORTUNING2-NEXT:    [[TMP62:%.*]] = or i32 [[TMP61]], [[X]]
+; VSCALEFORTUNING2-NEXT:    [[OR20:%.*]] = or i32 [[Z]], [[X]]
+; VSCALEFORTUNING2-NEXT:    [[NOT:%.*]] = and i32 [[OR20]], 1
+; VSCALEFORTUNING2-NEXT:    [[AND:%.*]] = xor i32 [[NOT]], 1
+; VSCALEFORTUNING2-NEXT:    [[IDX_EXT_1:%.*]] = zext i32 [[AND]] to i64
+; VSCALEFORTUNING2-NEXT:    [[GEP_2:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[IDX_EXT_1]]
+; VSCALEFORTUNING2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[GEP_2]], align 4
+; VSCALEFORTUNING2-NEXT:    [[SHR24:%.*]] = lshr i32 [[TMP62]], 1
+; VSCALEFORTUNING2-NEXT:    [[IDX_EXT_2:%.*]] = zext i32 [[SHR24]] to i64
+; VSCALEFORTUNING2-NEXT:    [[GEP_3:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[IDX_EXT_2]]
+; VSCALEFORTUNING2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[GEP_3]], align 4
+; VSCALEFORTUNING2-NEXT:    [[RED_1:%.*]] = or i32 [[TMP63]], [[SUM_RED]]
+; VSCALEFORTUNING2-NEXT:    [[RED_2]] = or i32 [[RED_1]], [[TMP64]]
+; VSCALEFORTUNING2-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[Y]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; VSCALEFORTUNING2:       exit:
+; VSCALEFORTUNING2-NEXT:    [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], [[LOOP]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT:    ret i32 [[RED_2_LCSSA]]
+;
 ; PRED-LABEL: define i32 @chained_recurrences(
 ; PRED-SAME: i32 [[X:%.*]], i64 [[Y:%.*]], ptr [[SRC_1:%.*]], i32 [[Z:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
 ; PRED-NEXT:  entry:
 ; PRED-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
-; PRED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PRED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; PRED:       vector.ph:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; PRED-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP2]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP5]]
+; PRED-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 1
+; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP3]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; PRED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
-; PRED-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP0]], [[TMP9]]
-; PRED-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], [[TMP9]]
-; PRED-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
+; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP0]], [[TMP7]]
+; PRED-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], [[TMP7]]
+; PRED-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT:    [[TMP19:%.*]] = add i64 [[Y]], 1
-; PRED-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP19]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP25:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT4]], splat (i32 1)
-; PRED-NEXT:    [[TMP28:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT4]], splat (i32 1)
-; PRED-NEXT:    [[TMP29:%.*]] = or <vscale x 4 x i32> [[TMP25]], [[TMP28]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP32:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT6]], [[BROADCAST_SPLAT4]]
-; PRED-NEXT:    [[TMP26:%.*]] = and <vscale x 4 x i32> [[TMP32]], splat (i32 1)
-; PRED-NEXT:    [[TMP34:%.*]] = xor <vscale x 4 x i32> [[TMP26]], splat (i32 1)
-; PRED-NEXT:    [[TMP35:%.*]] = zext <vscale x 4 x i32> [[TMP34]] to <vscale x 4 x i64>
-; PRED-NEXT:    [[TMP46:%.*]] = extractelement <vscale x 4 x i64> [[TMP35]], i32 0
-; PRED-NEXT:    [[TMP47:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP46]]
-; PRED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP47]], i64 0
-; PRED-NEXT:    [[TMP36:%.*]] = shufflevector <vscale x 4 x ptr> [[DOTSPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], 4
-; PRED-NEXT:    [[TMP15:%.*]] = sub i32 [[TMP14]], 1
-; PRED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP15]]
-; PRED-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 4
-; PRED-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP17]], 1
-; PRED-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP18]]
-; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PRED-NEXT:    [[TMP11:%.*]] = add i64 [[Y]], 1
+; PRED-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP11]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP13:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; PRED-NEXT:    [[TMP14:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; PRED-NEXT:    [[TMP15:%.*]] = or <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP16:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT]]
+; PRED-NEXT:    [[TMP17:%.*]] = and <vscale x 4 x i32> [[TMP16]], splat (i32 1)
+; PRED-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i32> [[TMP17]], splat (i32 1)
+; PRED-NEXT:    [[TMP19:%.*]] = zext <vscale x 4 x i32> [[TMP18]] to <vscale x 4 x i64>
+; PRED-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i64> [[TMP19]], i32 0
+; PRED-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP20]]
+; PRED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP21]], i64 0
+; PRED-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[DOTSPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; PRED-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 4
+; PRED-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; PRED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP24]]
+; PRED-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; PRED-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 4
+; PRED-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP26]], 1
+; PRED-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP27]]
+; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       vector.body:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP21]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT:    [[TMP22]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[BROADCAST_SPLAT]], i32 -1)
-; PRED-NEXT:    [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR2]], <vscale x 4 x i32> [[TMP22]], i32 -1)
-; PRED-NEXT:    [[TMP24:%.*]] = or <vscale x 4 x i32> [[TMP23]], [[BROADCAST_SPLAT4]]
-; PRED-NEXT:    [[TMP33:%.*]] = shl <vscale x 4 x i32> [[TMP24]], splat (i32 1)
-; PRED-NEXT:    [[TMP27:%.*]] = or <vscale x 4 x i32> [[TMP33]], splat (i32 2)
-; PRED-NEXT:    [[TMP30:%.*]] = or <vscale x 4 x i32> [[TMP29]], [[TMP27]]
-; PRED-NEXT:    [[TMP31:%.*]] = or <vscale x 4 x i32> [[TMP30]], [[BROADCAST_SPLAT4]]
-; PRED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP36]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; PRED-NEXT:    [[TMP37:%.*]] = lshr <vscale x 4 x i32> [[TMP31]], splat (i32 1)
-; PRED-NEXT:    [[TMP38:%.*]] = zext <vscale x 4 x i32> [[TMP37]] to <vscale x 4 x i64>
-; PRED-NEXT:    [[TMP39:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP38]]
-; PRED-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP39]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; PRED-NEXT:    [[TMP40:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
-; PRED-NEXT:    [[TMP41:%.*]] = or <vscale x 4 x i32> [[TMP40]], [[WIDE_MASKED_GATHER7]]
-; PRED-NEXT:    [[TMP42]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP41]], <vscale x 4 x i32> [[VEC_PHI]]
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
-; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]])
-; PRED-NEXT:    [[TMP43:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP44:%.*]] = extractelement <vscale x 4 x i1> [[TMP43]], i32 0
-; PRED-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], [[ENTRY]] ], [ [[BROADCAST_SPLAT6:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[VECTOR_RECUR4:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT3]], [[ENTRY]] ], [ [[TMP29:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP41:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP12]], align 4
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP28]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT6]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP29]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[BROADCAST_SPLAT6]], i32 -1)
+; PRED-NEXT:    [[TMP30:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR4]], <vscale x 4 x i32> [[TMP29]], i32 -1)
+; PRED-NEXT:    [[TMP31:%.*]] = or <vscale x 4 x i32> [[TMP30]], [[BROADCAST_SPLAT]]
+; PRED-NEXT:    [[TMP32:%.*]] = shl <vscale x 4 x i32> [[TMP31]], splat (i32 1)
+; PRED-NEXT:    [[TMP33:%.*]] = or <vscale x 4 x i32> [[TMP32]], splat (i32 2)
+; PRED-NEXT:    [[TMP34:%.*]] = or <vscale x 4 x i32> [[TMP15]], [[TMP33]]
+; PRED-NEXT:    [[TMP35:%.*]] = or <vscale x 4 x i32> [[TMP34]], [[BROADCAST_SPLAT]]
+; PRED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[DOTSPLAT]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; PRED-NEXT:    [[TMP36:%.*]] = lshr <vscale x 4 x i32> [[TMP35]], splat (i32 1)
+; PRED-NEXT:    [[TMP37:%.*]] = zext <vscale x 4 x i32> [[TMP36]] to <vscale x 4 x i64>
+; PRED-NEXT:    [[TMP38:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP37]]
+; PRED-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP38]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; PRED-NEXT:    [[TMP39:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
+; PRED-NEXT:    [[TMP40:%.*]] = or <vscale x 4 x i32> [[TMP39]], [[WIDE_MASKED_GATHER7]]
+; PRED-NEXT:    [[TMP41]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP40]], <vscale x 4 x i32> [[VEC_PHI]]
+; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[TMP5]]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[IV]], i64 [[TMP10]])
+; PRED-NEXT:    [[TMP42:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP43:%.*]] = extractelement <vscale x 4 x i1> [[TMP42]], i32 0
+; PRED-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PRED:       middle.block:
-; PRED-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP42]])
-; PRED-NEXT:    [[TMP49:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT:    [[TMP50:%.*]] = mul i32 [[TMP49]], 4
-; PRED-NEXT:    [[TMP51:%.*]] = sub i32 [[TMP50]], 1
-; PRED-NEXT:    [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <vscale x 4 x i32> [[TMP22]], i32 [[TMP51]]
+; PRED-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP41]])
+; PRED-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vscale.i32()
+; PRED-NEXT:    [[TMP46:%.*]] = mul i32 [[TMP45]], 4
+; PRED-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP46]], 1
+; PRED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP29]], i32 [[TMP47]]
 ; PRED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; PRED-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; PRED-NEXT:    [[SCALAR_RECUR_INIT9:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; PRED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; PRED-NEXT:    br label [[LOOP:%.*]]
+; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; PRED-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP28]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1]] ]
+; PRED-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1]] ]
+; PRED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP44]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1]] ]
+; PRED-NEXT:    br label [[LOOP1:%.*]]
 ; PRED:       loop:
-; PRED-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP53:%.*]], [[LOOP]] ]
-; PRED-NEXT:    [[SCALAR_RECUR10:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT9]], [[SCALAR_PH]] ], [ [[SCALAR_RECUR]], [[LOOP]] ]
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; PRED-NEXT:    [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_2:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[TMP48:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP53:%.*]], [[LOOP1]] ]
+; PRED-NEXT:    [[SCALAR_RECUR10:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT8]], [[SCALAR_PH]] ], [ [[TMP48]], [[LOOP1]] ]
+; PRED-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[LOOP1]] ]
+; PRED-NEXT:    [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_2:%.*]], [[LOOP1]] ]
 ; PRED-NEXT:    [[TMP52:%.*]] = add i64 [[Y]], 1
 ; PRED-NEXT:    [[GEP_1:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP52]]
 ; PRED-NEXT:    [[TMP53]] = load i32, ptr [[GEP_1]], align 4
 ; PRED-NEXT:    [[OR3:%.*]] = or i32 [[SCALAR_RECUR10]], [[X]]
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; PRED-NEXT:    [[IV_NEXT1]] = add i64 [[IV1]], 1
 ; PRED-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], 1
 ; PRED-NEXT:    [[TMP54:%.*]] = shl i32 [[OR3]], 1
 ; PRED-NEXT:    [[TMP55:%.*]] = or i32 [[TMP54]], 2
@@ -254,10 +294,10 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[TMP60:%.*]] = load i32, ptr [[GEP_3]], align 4
 ; PRED-NEXT:    [[RED_1:%.*]] = or i32 [[TMP59]], [[SUM_RED]]
 ; PRED-NEXT:    [[RED_2]] = or i32 [[RED_1]], [[TMP60]]
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[Y]]
-; PRED-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV1]], [[Y]]
+; PRED-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
 ; PRED:       exit:
-; PRED-NEXT:    [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], [[LOOP]] ], [ [[TMP45]], [[MIDDLE_BLOCK]] ]
+; PRED-NEXT:    [[RED_2_LCSSA:%.*]] = phi i32 [ [[RED_2]], [[LOOP1]] ], [ [[TMP44]], [[MIDDLE_BLOCK]] ]
 ; PRED-NEXT:    ret i32 [[RED_2_LCSSA]]
 ;
 entry:
@@ -336,7 +376,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP22]] = or <vscale x 4 x i16> [[TMP20]], [[VEC_PHI1]]
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       middle.block:
 ; DEFAULT-NEXT:    [[BIN_RDX:%.*]] = or <vscale x 4 x i16> [[TMP22]], [[TMP21]]
 ; DEFAULT-NEXT:    [[TMP24:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX]])
@@ -355,11 +395,71 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[RED_NEXT]] = or i16 [[DIV]], [[RED]]
 ; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; DEFAULT:       exit:
 ; DEFAULT-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], [[LOOP]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ]
 ; DEFAULT-NEXT:    ret i16 [[RED_NEXT_LCSSA]]
 ;
+; VSCALEFORTUNING2-LABEL: define i16 @reduce_udiv(
+; VSCALEFORTUNING2-SAME: ptr [[SRC:%.*]], i16 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; VSCALEFORTUNING2-NEXT:  entry:
+; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VSCALEFORTUNING2:       vector.ph:
+; VSCALEFORTUNING2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VSCALEFORTUNING2:       vector.body:
+; VSCALEFORTUNING2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 0
+; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP7]]
+; VSCALEFORTUNING2-NEXT:    [[TMP9:%.*]] = getelementptr i16, ptr [[TMP8]], i32 0
+; VSCALEFORTUNING2-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP12:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP11]]
+; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP9]], align 2
+; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
+; VSCALEFORTUNING2-NEXT:    [[TMP13:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP14:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT:    [[TMP15]] = or <vscale x 4 x i16> [[TMP13]], [[VEC_PHI]]
+; VSCALEFORTUNING2-NEXT:    [[TMP16]] = or <vscale x 4 x i16> [[TMP14]], [[VEC_PHI1]]
+; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; VSCALEFORTUNING2-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VSCALEFORTUNING2:       middle.block:
+; VSCALEFORTUNING2-NEXT:    [[BIN_RDX:%.*]] = or <vscale x 4 x i16> [[TMP16]], [[TMP15]]
+; VSCALEFORTUNING2-NEXT:    [[TMP18:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[BIN_RDX]])
+; VSCALEFORTUNING2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; VSCALEFORTUNING2:       scalar.ph:
+; VSCALEFORTUNING2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; VSCALEFORTUNING2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT:    br label [[LOOP:%.*]]
+; VSCALEFORTUNING2:       loop:
+; VSCALEFORTUNING2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
+; VSCALEFORTUNING2-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; VSCALEFORTUNING2-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; VSCALEFORTUNING2-NEXT:    [[DIV:%.*]] = udiv i16 [[L]], [[X]]
+; VSCALEFORTUNING2-NEXT:    [[RED_NEXT]] = or i16 [[DIV]], [[RED]]
+; VSCALEFORTUNING2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; VSCALEFORTUNING2-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; VSCALEFORTUNING2-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; VSCALEFORTUNING2:       exit:
+; VSCALEFORTUNING2-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], [[LOOP]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT:    ret i16 [[RED_NEXT_LCSSA]]
+;
 ; PRED-LABEL: define i16 @reduce_udiv(
 ; PRED-SAME: ptr [[SRC:%.*]], i16 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; PRED-NEXT:  entry:
@@ -444,8 +544,13 @@ attributes #0 = { "target-features"="+sve" }
 ; DEFAULT: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; DEFAULT: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
+; VSCALEFORTUNING2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; VSCALEFORTUNING2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; VSCALEFORTUNING2: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VSCALEFORTUNING2: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; VSCALEFORTUNING2: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; VSCALEFORTUNING2: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
 ;.
 ; PRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; PRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
index 225108fe89de0..1565d1ce982e3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
@@ -1,42 +1,39 @@
 ; REQUIRES: asserts
 ; RUN: opt -mtriple=aarch64 -mattr=+sve \
 ; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=GENERIC,VF-VSCALE16
+; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mattr=+sve -mcpu=generic \
 ; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=GENERIC,VF-VSCALE16
+; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v1 \
 ; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=NEOVERSE-V1,VF-VSCALE16
+; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING2
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-n2 \
 ; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=NEOVERSE-N2,VF-VSCALE16
+; RUN:     | FileCheck %s --check-prefixes=VSCALEFORTUNING1
 
 ; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v2 \
 ; RUN:     -force-target-instruction-cost=1 -passes=loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=NEOVERSE-V2,VF-16
+; RUN:     | FileCheck %s --check-prefixes=NEOVERSEV2
 
-; GENERIC: Cost for VF vscale x 2: 11 (Estimated cost per lane: 2.
-; GENERIC: Cost for VF vscale x 4: 11 (Estimated cost per lane: 1.
-; GENERIC: LV: Selecting VF: vscale x 16
+; VSCALEFORTUNING1: Cost for VF vscale x 2: 11 (Estimated cost per lane: 5.
+; VSCALEFORTUNING1: Cost for VF vscale x 4: 11 (Estimated cost per lane: 2.
+; VSCALEFORTUNING1: LV: Selecting VF: vscale x 16
 
-; NEOVERSE-V1: Cost for VF vscale x 2: 11 (Estimated cost per lane: 2.
-; NEOVERSE-V1: Cost for VF vscale x 4: 11 (Estimated cost per lane: 1.
-; NEOVERSE-V1: LV: Selecting VF: vscale x 16
+; VSCALEFORTUNING2: Cost for VF vscale x 2: 11 (Estimated cost per lane: 2.
+; VSCALEFORTUNING2: Cost for VF vscale x 4: 11 (Estimated cost per lane: 1.
+; VSCALEFORTUNING2: LV: Selecting VF: vscale x 16
 
-; NEOVERSE-N2: Cost for VF vscale x 2: 11 (Estimated cost per lane: 5.
-; NEOVERSE-N2: Cost for VF vscale x 4: 11 (Estimated cost per lane: 2.
-; NEOVERSE-N2: LV: Selecting VF: vscale x 16
+; NEOVERSEV2: Cost for VF vscale x 2: 11 (Estimated cost per lane: 5.
+; NEOVERSEV2: Cost for VF vscale x 4: 11 (Estimated cost per lane: 2.
+; NEOVERSEV2: LV: Selecting VF: 16
 
-; NEOVERSE-V2: Cost for VF vscale x 2: 11 (Estimated cost per lane: 5.
-; NEOVERSE-V2: Cost for VF vscale x 4: 11 (Estimated cost per lane: 2.
-; NEOVERSE-V2: LV: Selecting VF: 16
-
-; VF-16: <16 x i8>
-; VF-VSCALE16: <vscale x 16 x i8>
+; VSCALEFORTUNING1: <vscale x 16 x i8>
+; VSCALEFORTUNING2: <vscale x 16 x i8>
+; NEOVERSEV2: <16 x i8>
 define void @test0(ptr %a, ptr %b, ptr %c) #0 {
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-hint.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-hint.ll
index 47b159b1553b8..7cb56bb10809a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-hint.ll
@@ -187,9 +187,9 @@ exit:
 ; CHECK-DBG: LV: User VF=vscale x 4 is unsafe. Ignoring scalable UserVF.
 ; CHECK-DBG: remark: <unknown>:0:0: User-specified vectorization factor vscale x 4 is unsafe. Ignoring the hint to let the compiler pick a more suitable value.
 ; CHECK-DBG: Found feasible scalable VF = vscale x 2
-; CHECK-DBG: LV: Selecting VF: vscale x 2.
+; CHECK-DBG: LV: Selecting VF: 4.
 ; CHECK-LABEL: @test4
-; CHECK: <vscale x 2 x i32>
+; CHECK-NOT: <vscale x 4 x i32>
 define void @test4(ptr %a, ptr %b) #0 {
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scatter-cost.ll
deleted file mode 100644
index ec56b4e7cd1be..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scatter-cost.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: opt -mtriple aarch64 -mcpu=neoverse-v2 -passes="print<cost-model>" -disable-output < %s 2>&1 | FileCheck %s -check-prefix=CHECK-V2
-; RUN: opt -mtriple aarch64 -mattr=+sve2  -passes="print<cost-model>" -disable-output < %s 2>&1 | FileCheck %s -check-prefix=CHECK-GENERIC
-; CHECK-V2: Cost Model: Found an estimated cost of 52 for instruction: call void @llvm.masked.scatter.nxv4f32
-; CHECK-GENERIC: Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4f32
-
-define void @masked_scatter_nxv8f32_i64(<vscale x 4 x float> %data, <vscale x 4 x ptr> %b, <vscale x 4 x i64> %V) #0 {
-  call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> %data, <vscale x 4 x ptr> %b, i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-  ret void
-}
-
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
index 6165a73e77f23..403fc9f316d35 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
@@ -50,10 +50,6 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup.lo
 
 define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
 ; CHECK: LV: Checking a loop in 'multi_user_cmp'
-; CHECK: Cost of 1 for VF 16:
-; CHECK:  any-of reduction   %all.off = select i1 %cmp1, i1 %all.off.next, i1 false
-; CHECK: Cost of 1 for VF 16:
-; CHECK:  any-of reduction   %.any.0.off0 = select i1 %cmp1, i1 true, i1 %any.0.off09
 ; CHECK: Cost of 4 for VF 16: WIDEN ir<%cmp1> = fcmp olt ir<%load1>, ir<0.000000e+00>
 ; CHECK: LV: Selecting VF: 16.
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 6f5b6f48e1fc6..0bcdbed607f0d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -15,23 +15,27 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; DEFAULT:       vector.main.loop.iter.check:
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; DEFAULT-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP3]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DEFAULT:       vector.ph:
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 101, [[TMP5]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
 ; DEFAULT-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 16
+; DEFAULT-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 32
 ; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; DEFAULT:       vector.body:
 ; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
+; DEFAULT-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP23]]
 ; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP10]], align 1
+; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP24]], align 1
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -215,7 +219,7 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 0, [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; DEFAULT:       vec.epilog.scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; DEFAULT-NEXT:    br label [[LOOP:%.*]]
 ; DEFAULT:       loop:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
index a5c38177c2dc3..ab5600a2dc3a6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
@@ -170,7 +170,7 @@ define void @invariant_load_cond(ptr noalias nocapture %a, ptr nocapture readonl
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 168
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 168
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP5]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -201,7 +201,7 @@ define void @invariant_load_cond(ptr noalias nocapture %a, ptr nocapture readonl
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP12]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 168
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 168
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
index 22f3cdb90a59f..754f97c21608a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
@@ -50,8 +50,8 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[N]], 2
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -72,7 +72,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1, [[ITER_CHECK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
index b1534911246bb..9d06363846254 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
@@ -49,8 +49,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[N]], 2
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
@@ -72,7 +72,7 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 5, [[ITER_CHECK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
index 6d29476bbe6dc..af678d7a20f42 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
@@ -46,8 +46,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF3:%.*]] = urem i64 [[N]], 2
 ; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 [[N]], [[N_MOD_VF3]]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -67,7 +67,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
index e437da29b9403..751c121df0526 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
@@ -315,7 +315,7 @@ define void @gather_nxv4i32_ind64_stride2(ptr noalias nocapture %a, ptr noalias
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = shl nuw nsw i64 [[TMP13]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[DOTIDX]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 [[DOTIDX]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[WIDE_MASKED_GATHER]], ptr [[TMP12]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[WIDE_MASKED_GATHER2]], ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 632687bf5a413..28c9c5398a7a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -36,14 +36,14 @@ define void @test_array_load2_store2(i32 %C, i32 %D) #1 {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw <vscale x 4 x i32> [[TMP4]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])
 ; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
@@ -127,7 +127,7 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 {
 ; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> [[TMP8]], i32 2, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> poison)
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[TMP9]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER1]] to <vscale x 4 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul nsw <vscale x 4 x i32> [[BROADCAST_SPLAT3]], [[TMP11]]
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP12]])
@@ -209,7 +209,7 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
@@ -480,7 +480,7 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw <vscale x 4 x i32> [[TMP5]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[INDEX]], 9223372036854775804
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -498,7 +498,7 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[LOAD]], 1
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[LSHR]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[LSHR]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV]], 1022
@@ -566,7 +566,7 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = shl nsw <vscale x 4 x i32> [[TMP13]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[INDEX]], 9223372036854775804
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP15]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -584,7 +584,7 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[LOAD]], 1
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[LSHR]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[LSHR]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index f56fae37b9ef0..1a281fe7c6f7f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -79,19 +79,19 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; SCALAR_TAIL_FOLDING:       if.then:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = zext nneg i32 [[MUL]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP18]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP18]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = zext nneg i32 [[ADD]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP20]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP20]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[SPEC_SELECT_I:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP19]], i8 [[TMP21]])
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = zext nneg i32 [[MUL]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP22]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP22]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SPEC_SELECT_I]], ptr [[ARRAYIDX6]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = zext nneg i32 [[ADD]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP23]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP23]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 [[SUB]], ptr [[ARRAYIDX11]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[FOR_INC]]
 ; SCALAR_TAIL_FOLDING:       for.inc:
@@ -253,14 +253,14 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    [[IX_012:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_012]], 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = zext nneg i32 [[MUL]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP15]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP15]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 1, ptr [[ARRAYIDX]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_012]], [[CONV]]
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; SCALAR_TAIL_FOLDING:       if.then:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = zext nneg i32 [[ADD]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP16]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP16]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 2, ptr [[ARRAYIDX3]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[FOR_INC]]
 ; SCALAR_TAIL_FOLDING:       for.inc:
@@ -417,7 +417,7 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; SCALAR_TAIL_FOLDING:       if.then:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = zext nneg i32 [[MUL]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP16]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP16]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 1, ptr [[ARRAYIDX]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[IF_END]]
 ; SCALAR_TAIL_FOLDING:       if.end:
@@ -426,7 +426,7 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING:       if.then6:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = zext nneg i32 [[ADD]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[TMP17]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP17]]
 ; SCALAR_TAIL_FOLDING-NEXT:    store i8 2, ptr [[ARRAYIDX7]], align 1
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[FOR_INC]]
 ; SCALAR_TAIL_FOLDING:       for.inc:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll
index b84e8de678140..0efdf077dca66 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll
@@ -1,21 +1,25 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -passes=loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
-; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 -S 2>&1 | FileCheck %s
+; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 -S 2>&1 | FileCheck %s --check-prefix=CHECK-VSCALE1
 ; RUN: opt < %s -passes=loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 \
-; RUN:   -mcpu=neoverse-n2 -S 2>&1 | FileCheck %s --check-prefix=CHECK-CPU-NEOVERSE-N2
+; RUN:   -mcpu=neoverse-v1 -S 2>&1 | FileCheck %s --check-prefix=CHECK-VSCALE2
+; RUN: opt < %s -passes=loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
+; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 \
+; RUN:   -mcpu=neoverse-n2 -S 2>&1 | FileCheck %s --check-prefix=CHECK-VSCALE1
 
 target triple="aarch64-unknown-linux-gnu"
 
-; CHECK-LABEL: LV: Checking a loop in 'fadd_strict32'
-; CHECK: Cost of 4 for VF vscale x 2:
-; CHECK:  in-loop reduction   %add = fadd float %0, %sum.07
-; CHECK: Cost of 8 for VF vscale x 4:
-; CHECK:  in-loop reduction   %add = fadd float %0, %sum.07
-; CHECK-CPU-NEOVERSE-N2: Cost of 2 for VF vscale x 2:
-; CHECK-CPU-NEOVERSE-N2:  in-loop reduction   %add = fadd float %0, %sum.07
-; CHECK-CPU-NEOVERSE-N2: Cost of 4 for VF vscale x 4:
-; CHECK-CPU-NEOVERSE-N2:  in-loop reduction   %add = fadd float %0, %sum.07
+; CHECK-VSCALE2-LABEL: LV: Checking a loop in 'fadd_strict32'
+; CHECK-VSCALE2: Cost of 4 for VF vscale x 2:
+; CHECK-VSCALE2:  in-loop reduction   %add = fadd float %0, %sum.07
+; CHECK-VSCALE2: Cost of 8 for VF vscale x 4:
+; CHECK-VSCALE2:  in-loop reduction   %add = fadd float %0, %sum.07
+; CHECK-VSCALE1-LABEL: LV: Checking a loop in 'fadd_strict32'
+; CHECK-VSCALE1: Cost of 2 for VF vscale x 2:
+; CHECK-VSCALE1:  in-loop reduction   %add = fadd float %0, %sum.07
+; CHECK-VSCALE1: Cost of 4 for VF vscale x 4:
+; CHECK-VSCALE1:  in-loop reduction   %add = fadd float %0, %sum.07
 
 define float @fadd_strict32(ptr noalias nocapture readonly %a, i64 %n) #0 {
 entry:
@@ -36,11 +40,12 @@ for.end:
 }
 
 
-; CHECK-LABEL: LV: Checking a loop in 'fadd_strict64'
-; CHECK: Cost of 4 for VF vscale x 2:
-; CHECK:  in-loop reduction   %add = fadd double %0, %sum.07
-; CHECK-CPU-NEOVERSE-N2: Cost of 2 for VF vscale x 2:
-; CHECK-CPU-NEOVERSE-N2:  in-loop reduction   %add = fadd double %0, %sum.07
+; CHECK-VSCALE2-LABEL: LV: Checking a loop in 'fadd_strict64'
+; CHECK-VSCALE2: Cost of 4 for VF vscale x 2:
+; CHECK-VSCALE2:  in-loop reduction   %add = fadd double %0, %sum.07
+; CHECK-VSCALE1-LABEL: LV: Checking a loop in 'fadd_strict64'
+; CHECK-VSCALE1: Cost of 2 for VF vscale x 2:
+; CHECK-VSCALE1:  in-loop reduction   %add = fadd double %0, %sum.07
 
 define double @fadd_strict64(ptr noalias nocapture readonly %a, i64 %n) #0 {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
index ade24b85f5af9..4b096e17a4fa0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
@@ -51,7 +51,7 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = shl nuw nsw i64 [[TMP16]], 4
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 [[DOTIDX]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 [[DOTIDX]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP15]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = add nsw <vscale x 4 x i32> [[TMP10]], splat (i32 1)
@@ -59,7 +59,7 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[DOTIDX5:%.*]] = shl nuw nsw i64 [[TMP21]], 4
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[DOTIDX5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 [[DOTIDX5]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP19]], ptr [[TMP22]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
@@ -75,9 +75,9 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[PTR_014:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I_013:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[PTR_014]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_014]], i64 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[PTR_014]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[PTR_014]], i64 8
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[PTR_014]], i64 8
 ; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP24]], 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_013]]
@@ -181,8 +181,8 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[S_010]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP14]], 1
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[D_09]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[D_09]], i64 4
-; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[S_010]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[D_09]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[S_010]], i64 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_011]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -273,8 +273,8 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
 ; CHECK-NEXT:    [[VAR1:%.*]] = load i32, ptr [[P]], align 8
 ; CHECK-NEXT:    [[VAR2]] = add i32 [[VAR1]], [[VAR0]]
 ; CHECK-NEXT:    store ptr [[P]], ptr [[Q]], align 8
-; CHECK-NEXT:    [[VAR3]] = getelementptr inbounds i8, ptr [[P]], i64 4
-; CHECK-NEXT:    [[VAR4]] = getelementptr inbounds i8, ptr [[Q]], i64 8
+; CHECK-NEXT:    [[VAR3]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[VAR4]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index b8f94b938fa80..84fc963833cf2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -73,7 +73,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
index 716e3c4fbd7a2..013c218da2e16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
@@ -14,10 +14,10 @@ define void @simple_histogram_forced_scalar_interleave(ptr noalias %buckets, ptr
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
index 64746fa131d70..c430e72cea703 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -20,17 +20,17 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; CHECK-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
-; CHECK-NEXT:    [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[IDX_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[ARRAY]], i64 [[IV]]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; CHECK-NEXT:    store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
-; CHECK-NEXT:    [[GEP_OTHER:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_OTHER:%.*]] = getelementptr inbounds nuw i32, ptr [[OTHER]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_OTHER:%.*]] = load i32, ptr [[GEP_OTHER]], align 4
 ; CHECK-NEXT:    [[ADD_OTHER:%.*]] = add i32 [[L_OTHER]], [[IV_TRUNC]]
 ; CHECK-NEXT:    store i32 [[ADD_OTHER]], ptr [[GEP_OTHER]], align 4
@@ -106,7 +106,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    [[GEP_INDICES1:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES1]], align 4
 ; NORMAL_DEP_LIMIT-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; NORMAL_DEP_LIMIT-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
 ; NORMAL_DEP_LIMIT-NEXT:    [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
 ; NORMAL_DEP_LIMIT-NEXT:    store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
index 6257d3325f979..9739611a8b6e4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -41,9 +41,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond [[TC_CHECK]]
 ; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -51,6 +48,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %iv.next, %N
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ;; Check that the vectorized plan contains a histogram recipe instead.
@@ -83,9 +83,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond [[TC_CHECK]]
 ; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -93,6 +90,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %iv.next, %N
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 511d4dacc340c..728d7d1995e08 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -63,7 +63,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -129,7 +129,7 @@ define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indi
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], [[INCVAL]]
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -269,7 +269,7 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX3]], align 4
@@ -346,7 +346,7 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
 ; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i8 [[TMP1]], 1
 ; CHECK-NEXT:    store i8 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -384,10 +384,10 @@ define void @histogram_float(ptr noalias %buckets, ptr readonly %indices, i64 %N
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
 ; CHECK-NEXT:    store float [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -425,12 +425,12 @@ define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %ind
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[INCIDX:%.*]] = getelementptr inbounds i32, ptr [[INCVALS]], i64 [[IV]]
+; CHECK-NEXT:    [[INCIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INCVALS]], i64 [[IV]]
 ; CHECK-NEXT:    [[INCVAL:%.*]] = load i32, ptr [[INCIDX]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], [[INCVAL]]
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -483,7 +483,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = shl nuw nsw i64 [[TMP15]], 4
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[DOTIDX]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 [[DOTIDX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
@@ -506,7 +506,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -557,7 +557,7 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [1048576 x i32], ptr @idx_array, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, <vscale x 4 x i64> [[TMP14]]
@@ -573,10 +573,10 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1048576 x i32], ptr @idx_array, i64 0, i64 [[IV]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP9]] to i64
-; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 [[IDXPROM5]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw [1048576 x i32], ptr @data_array, i64 0, i64 [[IDXPROM5]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX6]], align 4
@@ -786,7 +786,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP17]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP18]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
@@ -831,10 +831,10 @@ define void @simple_histogram_unsafe_alias(ptr %buckets, ptr %indices, i64 %N) #
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index af1c63bc405a3..050bc7c346bf2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -39,9 +39,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -49,6 +46,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -81,9 +81,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -91,6 +88,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ;; If we have a masked variant at one VF and an unmasked variant at a different
@@ -128,9 +128,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -138,6 +135,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -170,9 +170,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -180,6 +177,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ;; If we have two variants at different VFs, neither of which are masked, we
@@ -216,9 +216,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -226,6 +223,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -258,9 +258,6 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -268,6 +265,9 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
 ; CHECK:        IR   %exitcond = icmp eq i64 %indvars.iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 define void @test_v4_v4m(ptr noalias %a, ptr readonly %b) #3 {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
index 68c8e4dd93fd7..bd2e5dcb3dba4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
@@ -23,29 +23,23 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 16
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 1
-; CHECK-NEXT:    [[TMP10:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i16>
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 8 x i16> [[TMP10]], trunc (<vscale x 8 x i32> splat (i32 2) to <vscale x 8 x i16>)
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP4]], splat (i16 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]
+; CHECK-NEXT:    store <16 x i16> [[TMP5]], ptr [[TMP6]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -55,8 +49,8 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP8]] to i32
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2
 ; CHECK-NEXT:    [[CONV1:%.*]] = trunc i32 [[ADD]] to i16
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]]
@@ -105,29 +99,23 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LEN]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 16
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 1
-; CHECK-NEXT:    [[TMP10:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i16>
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 8 x i16> [[TMP10]], trunc (<vscale x 8 x i32> splat (i32 2) to <vscale x 8 x i16>)
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP4]], splat (i16 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]
+; CHECK-NEXT:    store <16 x i16> [[TMP5]], ptr [[TMP6]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -137,8 +125,8 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP8]] to i32
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2
 ; CHECK-NEXT:    [[CONV1:%.*]] = trunc i32 [[ADD]] to i16
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index d0decbff1a462..ce8492cd77362 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -189,7 +189,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
 ; CHECK-NEXT:    [[GEPSRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[DATA:%.*]] = load double, ptr [[GEPSRC]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call double @foo(double [[DATA]], i64 [[INDVARS_IV]]) #[[ATTR5:[0-9]+]]
-; CHECK-NEXT:    [[GEPDST:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[GEPDST:%.*]] = getelementptr inbounds nuw double, ptr [[DST]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store double [[CALL]], ptr [[GEPDST]], align 8
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
index b2695cbf6e968..4ad9987b09069 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --filter "call.*(cos|sin|tan|cbrt|erf|exp[^e]|gamma|log|sqrt|copysign|dim|min|mod|hypot|nextafter|pow|fma)" --version 2
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefix=SLEEF-NEON
-; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s -check-prefix=SLEEF-SVE
-; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefixes=SLEEF-SVE-NOPRED
+; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -mcpu=neoverse-v1 -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s -check-prefix=SLEEF-SVE
+; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -mcpu=neoverse-v1 -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefixes=SLEEF-SVE-NOPRED
 ; RUN: opt -mattr=+neon -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefix=ARMPL-NEON
-; RUN: opt -mattr=+sve -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s -check-prefix=ARMPL-SVE
-; RUN: opt -mattr=+sve -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefixes=ARMPL-SVE-NOPRED
+; RUN: opt -mattr=+sve -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -mcpu=neoverse-v1 -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s -check-prefix=ARMPL-SVE
+; RUN: opt -mattr=+sve -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -mcpu=neoverse-v1 -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefixes=ARMPL-SVE-NOPRED
 
 
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index afc2fd5a049ad..7a67f37c72662 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -37,9 +37,6 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -47,6 +44,9 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT:   IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %cmp = icmp ne i64 %iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -79,9 +79,6 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -89,6 +86,9 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT:   IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %cmp = icmp ne i64 %iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll b/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
index 17da8f314b0e7..ab7bb667f3f36 100644
--- a/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
@@ -14,7 +14,7 @@ define half @vectorize_v2f16_loop(ptr addrspace(1) noalias %s) {
 ; GFX9-NEXT:    [[VEC_PHI:%.*]] = phi <2 x half> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; GFX9-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x half> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; GFX9-NEXT:    [[TMP0:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[S:%.*]], i64 [[INDEX]]
-; GFX9-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP0]], i64 4
+; GFX9-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP0]], i64 4
 ; GFX9-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x half>, ptr addrspace(1) [[TMP0]], align 2
 ; GFX9-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x half>, ptr addrspace(1) [[TMP1]], align 2
 ; GFX9-NEXT:    [[TMP2]] = fadd fast <2 x half> [[VEC_PHI]], [[WIDE_LOAD]]
@@ -44,7 +44,7 @@ define half @vectorize_v2f16_loop(ptr addrspace(1) noalias %s) {
 ; VI-NEXT:    [[VEC_PHI:%.*]] = phi <2 x half> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; VI-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x half> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; VI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[S:%.*]], i64 [[INDEX]]
-; VI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP0]], i64 4
+; VI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP0]], i64 4
 ; VI-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x half>, ptr addrspace(1) [[TMP0]], align 2
 ; VI-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x half>, ptr addrspace(1) [[TMP1]], align 2
 ; VI-NEXT:    [[TMP2]] = fadd fast <2 x half> [[VEC_PHI]], [[WIDE_LOAD]]
@@ -70,7 +70,7 @@ define half @vectorize_v2f16_loop(ptr addrspace(1) noalias %s) {
 ; CI:       for.body:
 ; CI-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CI-NEXT:    [[Q_04:%.*]] = phi half [ 0xH0000, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CI-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[S:%.*]], i64 [[INDVARS_IV]]
+; CI-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw half, ptr addrspace(1) [[S:%.*]], i64 [[INDVARS_IV]]
 ; CI-NEXT:    [[TMP0:%.*]] = load half, ptr addrspace(1) [[ARRAYIDX]], align 2
 ; CI-NEXT:    [[ADD]] = fadd fast half [[Q_04]], [[TMP0]]
 ; CI-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
index fde594d048361..0595bb31a58d8 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll
@@ -49,14 +49,14 @@ define void @arm_abs_q7(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %
 ; CHECK-NEXT:    [[PSRC_ADDR_022:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[BLKCNT_021:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_020:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_022]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_022]], i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[PSRC_ADDR_022]], align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[TMP7]], 0
 ; CHECK-NEXT:    [[CMP5:%.*]] = icmp eq i8 [[TMP7]], -128
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i8 0, [[TMP7]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP5]], i8 127, i8 [[SUB]]
 ; CHECK-NEXT:    [[COND11:%.*]] = select i1 [[CMP1]], i8 [[TMP7]], i8 [[COND]]
-; CHECK-NEXT:    [[INCDEC_PTR13]] = getelementptr inbounds i8, ptr [[PDST_ADDR_020]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR13]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_020]], i32 1
 ; CHECK-NEXT:    store i8 [[COND11]], ptr [[PDST_ADDR_020]], align 1
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_021]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
@@ -144,14 +144,14 @@ define void @arm_abs_q15(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32
 ; CHECK-NEXT:    [[PSRC_ADDR_023:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[BLKCNT_022:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_021:%.*]] = phi ptr [ [[INCDEC_PTR13:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_023]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_023]], i32 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[PSRC_ADDR_023]], align 2
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i16 [[TMP9]], 0
 ; CHECK-NEXT:    [[CMP5:%.*]] = icmp eq i16 [[TMP9]], -32768
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i16 0, [[TMP9]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP5]], i16 32767, i16 [[SUB]]
 ; CHECK-NEXT:    [[COND11:%.*]] = select i1 [[CMP1]], i16 [[TMP9]], i16 [[COND]]
-; CHECK-NEXT:    [[INCDEC_PTR13]] = getelementptr inbounds i8, ptr [[PDST_ADDR_021]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR13]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_021]], i32 2
 ; CHECK-NEXT:    store i16 [[COND11]], ptr [[PDST_ADDR_021]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_022]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
@@ -239,14 +239,14 @@ define void @arm_abs_q31(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32
 ; CHECK-NEXT:    [[PSRC_ADDR_017:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[BLKCNT_016:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_015:%.*]] = phi ptr [ [[INCDEC_PTR7:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_017]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_017]], i32 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[PSRC_ADDR_017]], align 4
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP9]], 0
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[TMP9]], -2147483648
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[TMP9]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP2]], i32 2147483647, i32 [[SUB]]
 ; CHECK-NEXT:    [[COND6:%.*]] = select i1 [[CMP1]], i32 [[TMP9]], i32 [[COND]]
-; CHECK-NEXT:    [[INCDEC_PTR7]] = getelementptr inbounds i8, ptr [[PDST_ADDR_015]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR7]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_015]], i32 4
 ; CHECK-NEXT:    store i32 [[COND6]], ptr [[PDST_ADDR_015]], align 4
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_016]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
index 6011ccfd15614..af43b747c0043 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
@@ -18,7 +18,7 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP0]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -71,7 +71,7 @@ define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD1]])
 ; CHECK-NEXT:    [[TMP8]] = add i32 [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -124,7 +124,7 @@ define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
@@ -176,7 +176,7 @@ define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> splat (i32 -1)
 ; CHECK-NEXT:    [[TMP4]] = and <4 x i32> [[VEC_PHI]], [[TMP3]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
@@ -228,7 +228,7 @@ define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4]] = or <4 x i32> [[VEC_PHI]], [[TMP3]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
@@ -280,7 +280,7 @@ define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4]] = xor <4 x i32> [[VEC_PHI]], [[TMP3]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
@@ -332,7 +332,7 @@ define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
@@ -384,7 +384,7 @@ define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
index 832d4db53036f..6f78982d7ab02 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
@@ -33,7 +33,7 @@ define i32 @mla_i32(ptr noalias nocapture readonly %A, ptr noalias nocapture rea
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP8]])
 ; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -116,7 +116,7 @@ define i32 @mla_i8(ptr noalias nocapture readonly %A, ptr noalias nocapture read
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP8]])
 ; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
@@ -193,7 +193,7 @@ define i32 @add_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
@@ -259,7 +259,7 @@ define i32 @mul_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
@@ -326,7 +326,7 @@ define i32 @and_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
@@ -393,7 +393,7 @@ define i32 @or_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
@@ -460,7 +460,7 @@ define i32 @xor_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
@@ -527,7 +527,7 @@ define float @fadd_f32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
@@ -594,7 +594,7 @@ define float @fmul_f32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP2]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index c115c91cff896..c459dd28fdaee 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -13,7 +13,7 @@ define i64 @add_i64_i64(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[X:%.*]], i32 [[I_08]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[X:%.*]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD]] = add nsw i64 [[TMP0]], [[R_07]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
@@ -76,7 +76,7 @@ define i64 @add_i32_i64(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP5]] to i64
 ; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
@@ -141,7 +141,7 @@ define i64 @add_i16_i64(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_08]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP5]] to i64
 ; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
@@ -206,7 +206,7 @@ define i64 @add_i8_i64(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[I_08]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP5]] to i64
 ; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[R_07]], [[CONV]]
@@ -255,7 +255,7 @@ define i32 @add_i32_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP0]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -301,7 +301,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP1]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4]] = add i32 [[TMP3]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -348,7 +348,7 @@ define i32 @add_i8_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP1]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4]] = add i32 [[TMP3]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -393,7 +393,7 @@ define signext i16 @add_i16_i16(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr [[TMP0]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> zeroinitializer)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[TMP2]] = add i16 [[TMP1]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -439,7 +439,7 @@ define signext i16 @add_i8_i16(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP1]], <16 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4]] = add i16 [[TMP3]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -484,7 +484,7 @@ define zeroext i8 @add_i8_i8(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[TMP0]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> zeroinitializer)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[TMP2]] = add i8 [[TMP1]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -519,9 +519,9 @@ define i64 @mla_i64_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[X:%.*]], i32 [[I_010]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[X:%.*]], i32 [[I_010]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[Y:%.*]], i32 [[I_010]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i64, ptr [[Y:%.*]], i32 [[I_010]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[ADD]] = add nsw i64 [[MUL]], [[R_09]]
@@ -591,9 +591,9 @@ define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_010]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i32 [[I_010]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i32 [[I_010]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[Y]], i32 [[I_010]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[MUL]] to i64
@@ -667,10 +667,10 @@ define i64 @mla_i16_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_012]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_012]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP9]] to i32
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[Y]], i32 [[I_012]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i16, ptr [[Y]], i32 [[I_012]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP10]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
@@ -747,10 +747,10 @@ define i64 @mla_i8_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[X]], i32 [[I_012]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_012]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP9]] to i32
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[Y]], i32 [[I_012]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i32 [[I_012]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP10]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]
@@ -810,7 +810,7 @@ define i32 @mla_i32_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -863,7 +863,7 @@ define i32 @mla_i16_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i3
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP4]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[TMP7]] = add i32 [[TMP6]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -918,7 +918,7 @@ define i32 @mla_i8_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[TMP7]] = add i32 [[TMP6]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -971,7 +971,7 @@ define signext i16 @mla_i16_i16(ptr nocapture readonly %x, ptr nocapture readonl
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> [[TMP2]], <8 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP3]])
 ; CHECK-NEXT:    [[TMP5]] = add i16 [[TMP4]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -1024,7 +1024,7 @@ define signext i16 @mla_i8_i16(ptr nocapture readonly %x, ptr nocapture readonly
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP4]], <16 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP5]])
 ; CHECK-NEXT:    [[TMP7]] = add i16 [[TMP6]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -1077,7 +1077,7 @@ define zeroext i8 @mla_i8_i8(ptr nocapture readonly %x, ptr nocapture readonly %
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[TMP2]], <16 x i8> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[TMP3]])
 ; CHECK-NEXT:    [[TMP5]] = add i8 [[TMP4]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -1417,7 +1417,7 @@ define i32 @mla_i8_i32_multiuse(ptr nocapture readonly %x, ptr nocapture readonl
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP2]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
@@ -1482,7 +1482,7 @@ define i64 @mla_xx_sext_zext(ptr nocapture noundef readonly %x, i32 %n) #0 {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[S_010:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_011]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_011]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP7]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV]], [[CONV]]
@@ -1558,7 +1558,7 @@ define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 no
 ; CHECK-NEXT:    [[I_019:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[T_018:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[S_017:%.*]] = phi i32 [ [[ADD6]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_019]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_019]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP9]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV]], [[CONV]]
@@ -1606,20 +1606,20 @@ define i64 @interleave_doublereduct_i16_i64(ptr %x, ptr %y, i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[T_024:%.*]] = phi i64 [ [[ADD12]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[I_025]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X:%.*]], i32 [[I_025]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i32 [[I_025]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i16, ptr [[Y:%.*]], i32 [[I_025]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
 ; CHECK-NEXT:    [[CONV3:%.*]] = sext i32 [[MUL]] to i64
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[T_024]], [[CONV3]]
 ; CHECK-NEXT:    [[ADD4:%.*]] = or disjoint i32 [[I_025]], 1
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[ADD4]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[ADD4]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX5]], align 2
 ; CHECK-NEXT:    [[CONV6:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i16, ptr [[Y]], i32 [[ADD4]]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i16, ptr [[Y]], i32 [[ADD4]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX8]], align 2
 ; CHECK-NEXT:    [[CONV9:%.*]] = sext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[CONV9]], [[CONV6]]
@@ -1677,7 +1677,7 @@ define i64 @test_std_q31(ptr %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[S_014:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY1]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[I_013:%.*]] = phi i32 [ [[ADD4:%.*]], [[FOR_BODY1]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[T_012:%.*]] = phi i64 [ [[ADD3]], [[FOR_BODY1]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[I_013]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X:%.*]], i32 [[I_013]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[SHR]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
index 025b8738f7829..eaec02038fde1 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
@@ -65,9 +65,9 @@ define float @test(ptr nocapture readonly %pA, ptr nocapture readonly %pB, i32 %
 ; CHECK-NEXT:    [[PB_ADDR_019:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[BLOCKSIZE_ADDR_018:%.*]] = phi i32 [ [[DEC:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ACCUM_017:%.*]] = phi float [ [[ACCUM_1:%.*]], [[IF_END]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PA_ADDR_020]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PA_ADDR_020]], i32 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[PA_ADDR_020]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[PB_ADDR_019]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[PB_ADDR_019]], i32 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[PB_ADDR_019]], align 4
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast une float [[TMP13]], 0.000000e+00
 ; CHECK-NEXT:    [[CMP3:%.*]] = fcmp fast une float [[TMP14]], 0.000000e+00
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
index d49639db3bbfc..07ba7745b34a2 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
@@ -69,10 +69,10 @@ define hidden void @pointer_phi_v4i32_add2(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 8
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[Y]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -123,10 +123,10 @@ define hidden void @pointer_phi_v4i32_add3(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 12
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 12
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[Y]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -219,10 +219,10 @@ define hidden void @pointer_phi_v8i16_add2(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_09:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[L1:%.*]] = load i16, ptr [[A_ADDR_011]], align 2
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_011]], i32 4
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_011]], i32 4
 ; CHECK-NEXT:    [[CONV1:%.*]] = add i16 [[L1]], [[TMP0]]
 ; CHECK-NEXT:    store i16 [[CONV1]], ptr [[B_ADDR_09]], align 2
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_09]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_09]], i32 2
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -258,10 +258,10 @@ define hidden void @pointer_phi_v8i16_add3(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_09:%.*]] = phi ptr [ [[B:%.*]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[L1:%.*]] = load i16, ptr [[A_ADDR_011]], align 2
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_011]], i32 6
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_011]], i32 6
 ; CHECK-NEXT:    [[CONV1:%.*]] = add i16 [[L1]], [[TMP0]]
 ; CHECK-NEXT:    store i16 [[CONV1]], ptr [[B_ADDR_09]], align 2
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_09]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_09]], i32 2
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]]
@@ -311,10 +311,10 @@ define hidden void @pointer_phi_v16i8_add1(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_010]], i32 1
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 1
 ; CHECK-NEXT:    [[CONV1:%.*]] = add i8 [[TMP3]], [[TMP0]]
 ; CHECK-NEXT:    store i8 [[CONV1]], ptr [[B_ADDR_08]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_08]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_09]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -367,10 +367,10 @@ define hidden void @pointer_phi_v16i8_add2(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_09:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_010]], i32 2
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 2
 ; CHECK-NEXT:    [[CONV1:%.*]] = add i8 [[TMP3]], [[TMP0]]
 ; CHECK-NEXT:    store i8 [[CONV1]], ptr [[B_ADDR_08]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_08]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_09]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -407,10 +407,10 @@ define hidden void @pointer_phi_v16i8_add3(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_09:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_08:%.*]] = phi ptr [ [[B:%.*]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[A_ADDR_010]], align 1
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_010]], i32 3
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_010]], i32 3
 ; CHECK-NEXT:    [[CONV1:%.*]] = add i8 [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    store i8 [[CONV1]], ptr [[B_ADDR_08]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_08]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_08]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_09]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]]
@@ -502,10 +502,10 @@ define hidden void @pointer_phi_v4f32_add2(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 8
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 8
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP2]], [[Y]]
 ; CHECK-NEXT:    store float [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -556,10 +556,10 @@ define hidden void @pointer_phi_v4f32_add3(ptr noalias nocapture readonly %A, pt
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 996, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 12
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 12
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP3]], [[Y]]
 ; CHECK-NEXT:    store float [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -649,10 +649,10 @@ define hidden void @pointer_phi_v4half_add2(ptr noalias nocapture readonly %A, p
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load half, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 4
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 4
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP2]], [[Y]]
 ; CHECK-NEXT:    store half [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 2
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
@@ -703,10 +703,10 @@ define hidden void @pointer_phi_v4half_add3(ptr noalias nocapture readonly %A, p
 ; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load half, ptr [[A_ADDR_09]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_09]], i32 6
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_09]], i32 6
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP2]], [[Y]]
 ; CHECK-NEXT:    store half [[ADD]], ptr [[B_ADDR_07]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_07]], i32 2
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
@@ -767,10 +767,10 @@ define hidden void @pointer_phi_v4i32_uf2(ptr noalias nocapture readonly %A, ptr
 ; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 9992, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_08]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_08]], i32 24
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_08]], i32 24
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP6]], [[Y]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[B_ADDR_06]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_06]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_06]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
@@ -842,10 +842,10 @@ define hidden void @pointer_phi_v4i32_uf4(ptr noalias nocapture readonly %A, ptr
 ; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 9984, [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[B_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END2]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR_08]], align 4
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_08]], i32 24
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_08]], i32 24
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP12]], [[Y]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[B_ADDR_06]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[B_ADDR_06]], i32 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[B_ADDR_06]], i32 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
@@ -907,20 +907,20 @@ define hidden void @mult_ptr_iv(ptr noalias nocapture readonly %x, ptr noalias n
 ; CHECK-NEXT:    [[X_ADDR_050:%.*]] = phi ptr [ [[INCDEC_PTR2:%.*]], [[FOR_BODY]] ], [ [[X]], [[ENTRY]] ]
 ; CHECK-NEXT:    [[Z_ADDR_049:%.*]] = phi ptr [ [[INCDEC_PTR34:%.*]], [[FOR_BODY]] ], [ [[Z]], [[ENTRY]] ]
 ; CHECK-NEXT:    [[I_048:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[X_ADDR_050]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[X_ADDR_050]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i8, ptr [[X_ADDR_050]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR2]] = getelementptr inbounds i8, ptr [[X_ADDR_050]], i32 3
+; CHECK-NEXT:    [[INCDEC_PTR2]] = getelementptr inbounds nuw i8, ptr [[X_ADDR_050]], i32 3
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[INCDEC_PTR1]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[TMP10]], 10
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul i8 [[TMP10]], [[TMP11]]
 ; CHECK-NEXT:    [[MUL2:%.*]] = mul i8 [[TMP10]], [[TMP12]]
-; CHECK-NEXT:    [[INCDEC_PTR32:%.*]] = getelementptr inbounds i8, ptr [[Z_ADDR_049]], i32 1
+; CHECK-NEXT:    [[INCDEC_PTR32:%.*]] = getelementptr inbounds nuw i8, ptr [[Z_ADDR_049]], i32 1
 ; CHECK-NEXT:    store i8 [[MUL]], ptr [[Z_ADDR_049]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR33:%.*]] = getelementptr inbounds i8, ptr [[Z_ADDR_049]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR33:%.*]] = getelementptr inbounds nuw i8, ptr [[Z_ADDR_049]], i32 2
 ; CHECK-NEXT:    store i8 [[MUL1]], ptr [[INCDEC_PTR32]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR34]] = getelementptr inbounds i8, ptr [[Z_ADDR_049]], i32 3
+; CHECK-NEXT:    [[INCDEC_PTR34]] = getelementptr inbounds nuw i8, ptr [[Z_ADDR_049]], i32 3
 ; CHECK-NEXT:    store i8 [[MUL2]], ptr [[INCDEC_PTR33]], align 1
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_048]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
index ed6254074c8b5..e9642a55bc341 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
@@ -345,7 +345,7 @@ define void @float(ptr noalias nocapture %A, ptr noalias nocapture readonly %B,
 ; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0({{.*}}%active.lane.mask
 ; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0({{.*}}%active.lane.mask
 ; PREFER-FOLDING: call void @llvm.masked.store.v4f32.p0({{.*}}%active.lane.mask
-; PREFER-FOLDING: %index.next = add i32 %index, 4
+; PREFER-FOLDING: %index.next = add nuw i32 %index, 4
 ; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-loop-hint.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-loop-hint.ll
index 77cc3b70edb93..573a9f75f3054 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-loop-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-loop-hint.ll
@@ -50,7 +50,7 @@ define dso_local void @predicate_loop_hint(ptr noalias nocapture %A, ptr noalias
 ; CHECK:         %[[WML2:.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}<4 x i1> %active.lane.mask
 ; CHECK:         %[[ADD:.*]] = add nsw <4 x i32> %[[WML2]], %[[WML1]]
 ; CHECK:         call void @llvm.masked.store.v4i32.p0(<4 x i32> %[[ADD]], {{.*}}<4 x i1> %active.lane.mask
-; CHECK:         %index.next = add i64 %index, 4
+; CHECK:         %index.next = add nuw i64 %index, 4
 ; CHECK:         br i1 %{{.*}}, label %{{.*}}, label %vector.body, !llvm.loop [[VEC_LOOP2:![0-9]+]]
 ;
 ; CHECK:         br i1 %{{.*}}, label %{{.*}}, label %for.body, !llvm.loop [[SCALAR_LOOP2:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
index 8f3cfc5b20ee9..f0464a085d390 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
@@ -27,7 +27,7 @@ define dso_local void @flag_overrules_hint(ptr noalias nocapture %A, ptr noalias
 ; PREDFLAG:  %wide.masked.load1 = call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %active.lane.mask
 ; PREDFLAG:  %{{.*}} = add nsw <4 x i32> %wide.masked.load1, %wide.masked.load
 ; PREDFLAG:  call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %active.lane.mask
-; PREDFLAG:  %index.next = add i64 %index, 4
+; PREDFLAG:  %index.next = add nuw i64 %index, 4
 ; PREDFLAG:  %[[CMP:.*]] = icmp eq i64 %index.next, 432
 ; PREDFLAG:  br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
index 0add22fb588b8..5f151c260c8fc 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
@@ -113,13 +113,12 @@ define i1 @select_exit_cond(ptr %start, ptr %end, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF24:%.*]] = urem i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_VEC25:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF24]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC25]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>
 ; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
@@ -147,11 +146,11 @@ define i1 @select_exit_cond(ptr %start, ptr %end, i64 %N) {
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i64 [ [[N_VEC25]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL28:%.*]] = phi ptr [ [[IND_END]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX37:%.*]] = phi i64 [ [[TMP54]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[ITER_CHECK]] ], [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX29:%.*]] = phi i64 [ [[TMP54]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP52]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL26]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX37]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX29]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL28]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[TMP53:%.*]] = load i8, ptr [[PTR_IV]], align 1
 ; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[TMP53]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
index ba9d49fc682c4..95a77fb6be354 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
@@ -438,8 +438,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; VF-TWO-CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC17]]
 ; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-TWO-CHECK:       vec.epilog.scalar.ph:
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
-; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-TWO-CHECK:       for.body:
 ; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -597,8 +597,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; VF-FOUR-CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC17]]
 ; VF-FOUR-CHECK-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; VF-FOUR-CHECK:       vec.epilog.scalar.ph:
-; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
-; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL19:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END18]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; VF-FOUR-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF-FOUR-CHECK:       for.body:
 ; VF-FOUR-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 8368022bceed9..14818199072c2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -143,11 +143,11 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
-; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i16> [[VP_OP_LOAD]] to <vscale x 4 x i32>
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vp.sext.nxv4i32.nxv4i16(<vscale x 4 x i16> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP:%.*]] = call <vscale x 4 x i32> @llvm.vp.add.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[VP_OP]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP5]])
-; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i32 [[TMP5]], [[EVL_BASED_IV]]
-; IF-EVL-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL-OUTLOOP:       middle.block:
@@ -194,17 +194,17 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = sub i32 [[N]], [[EVL_BASED_IV]]
-; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = add i32 [[EVL_BASED_IV]], 0
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]]
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0
-; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP8]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
-; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = sext <vscale x 8 x i16> [[VP_OP_LOAD]] to <vscale x 8 x i32>
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, <vscale x 8 x i32> [[TMP9]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[TMP5]], i32 8, i1 true)
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = add i32 [[EVL_BASED_IV]], 0
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP7]]
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0
+; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = call <vscale x 8 x i32> @llvm.vp.sext.nxv8i32.nxv8i16(<vscale x 8 x i16> [[VP_OP_LOAD]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, <vscale x 8 x i32> [[TMP14]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]]
-; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i32 [[TMP5]], [[EVL_BASED_IV]]
-; IF-EVL-INLOOP-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP6]], [[EVL_BASED_IV]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL-INLOOP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
index 44c5348296dea..5c5600b9cfdf8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
@@ -69,7 +69,7 @@ define void @trip3_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[TMP12]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i8> poison)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i8> [[TMP10]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i8.p0(<vscale x 2 x i8> [[TMP13]], ptr [[TMP12]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -138,7 +138,7 @@ define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i8> [[TMP10]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP13]], ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -207,7 +207,7 @@ define void @trip8_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i8> [[TMP10]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP13]], ptr [[TMP12]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 02b7ce3795ad5..305d26d7f3bc1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -55,7 +55,7 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK:       pred.store.continue8:
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index d68556fca4774..fcc01e0ba2e2d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -91,9 +91,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -102,6 +99,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
 ; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
@@ -142,11 +142,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Interleaving is not beneficial.
 ; CHECK-NEXT:  LV: Found a vectorizable loop (vscale x 4) in <stdin>
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
-; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
+; CHECK:       Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  Live-in ir<[[VF:%.+]]> = VF
+; CHECK-NEXT:  Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF
+; CHECK-NEXT:  Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
@@ -159,40 +159,42 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
 ; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%13>, ir<1>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
+; CHECK-NEXT:      WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT:      WIDEN ir<%add9> = add ir<[[L]]>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, ir<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
-; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Successor(s): middle.block
+; CHECK-NEXT:  Successor(s): ir-bb<middle.block>
 ; CHECK-EMPTY:
-; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:  ir-bb<middle.block>:
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VEC_TC]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
-; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT:  No successors
+; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
-; CHECK-NEXT:  scalar.ph:
+; CHECK-NEXT:  ir-bb<scalar.ph>:
+; CHECK-NEXT:    EMIT vp<[[RESUME1:%.+]]> = resume-phi ir<%ind.end>, ir<%0>
+; CHECK-NEXT:    EMIT vp<[[RESUME2:%.+]]>.1 = resume-phi ir<%ind.end3>, ir<%n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
+; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME1]]> from ir-bb<scalar.ph>
+; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME2]]>.1 from ir-bb<scalar.ph>
 ; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK:  LV: Loop does not require scalar epilogue
 ;
@@ -305,9 +307,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -316,6 +315,9 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
 ; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
@@ -356,11 +358,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Interleaving is not beneficial.
 ; CHECK-NEXT:  LV: Found a vectorizable loop (vscale x 4) in <stdin>
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
-; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
+; CHECK:       Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  Live-in ir<[[VF:%.+]]> = VF
+; CHECK-NEXT:  Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF
+; CHECK-NEXT:  Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
@@ -373,40 +375,42 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
 ; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
+; CHECK-NEXT:      WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<[[L]]>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, ir<[[VF]]>
 ; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR]]>, ir<%conv1>
-; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Successor(s): middle.block
+; CHECK-NEXT:  Successor(s): ir-bb<middle.block>
 ; CHECK-EMPTY:
-; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:  ir-bb<middle.block>:
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VEC_TC]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
-; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT:  No successors
+; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
-; CHECK-NEXT:  scalar.ph:
+; CHECK-NEXT:  ir-bb<scalar.ph>:
+; CHECK-NEXT:    EMIT vp<[[RESUME1:%.+]]> = resume-phi ir<%ind.end>, ir<%0>
+; CHECK-NEXT:    EMIT vp<[[RESUME2:%.+]]>.1 = resume-phi ir<%ind.end3>, ir<%n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
+; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME1]]> from ir-bb<scalar.ph>
+; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME2]]>.1 from ir-bb<scalar.ph>
 ; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK:  LV: Loop does not require scalar epilogue
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
index aa7b7a5676eef..3c35c272d3389 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
@@ -31,7 +31,7 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -95,7 +95,7 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x ptr> [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
@@ -158,7 +158,7 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; CHECK-NEXT:    [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
@@ -225,7 +225,7 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
@@ -282,7 +282,7 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
@@ -375,7 +375,7 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
index 1583e071fe7bc..375278eea38f9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
@@ -22,7 +22,7 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr [[TMP5]], i32 4, <vscale x 1 x i1> [[ACTIVE_LANE_MASK]], <vscale x 1 x i32> poison)
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <vscale x 1 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv1i32.p0(<vscale x 1 x i32> [[TMP6]], ptr [[TMP5]], i32 4, <vscale x 1 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -81,7 +81,7 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP7]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr [[TMP7]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index d0e14e5ba44c7..bfdcfbf5139b3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -175,7 +175,7 @@ define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[TMP1]], i32 2)
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[ACTIVE_LANE_MASK]], <2 x i1> [[TMP2]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v2i8.v2p0(<2 x i8> zeroinitializer, <2 x ptr> [[BROADCAST_SPLAT4]], i32 1, <2 x i1> [[TMP3]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index 30f1f0ca75e1d..9bba6e4376612 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -111,7 +111,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ; TF-SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]]
 ; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TF-SCALABLE:       middle.block:
@@ -145,7 +145,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ; TF-FIXEDLEN-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP3]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TF-FIXEDLEN:       middle.block:
@@ -452,7 +452,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; TF-SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP9]]
 ; TF-SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; TF-SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -497,7 +497,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[PREDPHI]], ptr [[TMP4]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; TF-FIXEDLEN-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -648,7 +648,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ; TF-SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]]
 ; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TF-SCALABLE:       middle.block:
@@ -682,7 +682,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ; TF-FIXEDLEN-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP3]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TF-FIXEDLEN:       middle.block:
@@ -819,7 +819,7 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; TF-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]]
 ; TF-SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; TF-SCALABLE:       middle.block:
@@ -853,7 +853,7 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; TF-FIXEDLEN-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP2]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; TF-FIXEDLEN:       middle.block:
@@ -1015,7 +1015,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-SCALABLE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP9]]
 ; TF-SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; TF-SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1076,7 +1076,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-FIXEDLEN-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; TF-FIXEDLEN:       middle.block:
@@ -1255,7 +1255,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; TF-SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP9]]
 ; TF-SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; TF-SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1300,7 +1300,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP4]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; TF-FIXEDLEN-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1449,7 +1449,7 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]]
 ; TF-SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0
 ; TF-SCALABLE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; TF-SCALABLE:       middle.block:
@@ -1483,7 +1483,7 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-FIXEDLEN-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]]
 ; TF-FIXEDLEN-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP2]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
-; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; TF-FIXEDLEN-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1028
 ; TF-FIXEDLEN-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; TF-FIXEDLEN:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
index f18a098e162a8..40ae96f378b6c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
@@ -47,8 +47,8 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -141,8 +141,8 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -235,8 +235,8 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -329,8 +329,8 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -423,8 +423,8 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -517,8 +517,8 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -611,8 +611,8 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -705,8 +705,8 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -799,8 +799,8 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -893,8 +893,8 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -987,8 +987,8 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1081,8 +1081,8 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1175,8 +1175,8 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP17]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1272,8 +1272,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1367,8 +1367,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1462,8 +1462,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1557,8 +1557,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -1705,8 +1705,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
new file mode 100644
index 0000000000000..66a5e086a0117
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
@@ -0,0 +1,1000 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-tail-folding-style=data-with-evl \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=IF-EVL
+
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-tail-folding-style=none \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP
+
+define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL-LABEL: define void @vp_smax(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = sub i64 [[A1]], [[C3]]
+; IF-EVL-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP26]], [[TMP25]]
+; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP29:%.*]] = call <vscale x 4 x i32> @llvm.vp.smax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VP_OP_LOAD5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP29]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP20]], i32 [[TMP21]])
+; IF-EVL-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_smax(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP0]], i32 [[TMP1]])
+; NO-VP-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.smax.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL-LABEL: define void @vp_smin(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = sub i64 [[A1]], [[C3]]
+; IF-EVL-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP26]], [[TMP25]]
+; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP29:%.*]] = call <vscale x 4 x i32> @llvm.vp.smin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VP_OP_LOAD5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP29]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP20]], i32 [[TMP21]])
+; IF-EVL-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_smin(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.smin.i32(i32 [[TMP0]], i32 [[TMP1]])
+; NO-VP-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.smin.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL-LABEL: define void @vp_umax(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = sub i64 [[A1]], [[C3]]
+; IF-EVL-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP26]], [[TMP25]]
+; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP29:%.*]] = call <vscale x 4 x i32> @llvm.vp.umax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VP_OP_LOAD5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP29]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.umax.i32(i32 [[TMP20]], i32 [[TMP21]])
+; IF-EVL-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_umax(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.umax.i32(i32 [[TMP0]], i32 [[TMP1]])
+; NO-VP-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.umax.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL-LABEL: define void @vp_umin(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = sub i64 [[A1]], [[C3]]
+; IF-EVL-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP26]], [[TMP25]]
+; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP29:%.*]] = call <vscale x 4 x i32> @llvm.vp.umin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VP_OP_LOAD5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP29]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.umin.i32(i32 [[TMP20]], i32 [[TMP21]])
+; IF-EVL-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_umin(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[DOT:%.*]] = tail call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP1]])
+; NO-VP-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[DOT]], ptr [[GEP11]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.umin.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
+define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_ctlz(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vp.ctlz.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], i1 true, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP24]], ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[TMP19:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[TMP18]], i1 true)
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[TMP19]], ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_ctlz(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 true)
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[TMP1]], ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %1 = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %0, i1 true)
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %1, ptr %gep3, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_cttz(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP9]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP14:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP14]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x i32> @llvm.vp.cttz.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], i1 true, <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP17]], ptr align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; IF-EVL-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP13]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP20]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = tail call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP22]], i1 true)
+; IF-EVL-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[TMP23]], ptr [[GEP3]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_cttz(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = tail call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP0]], i1 true)
+; NO-VP-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[TMP1]], ptr [[GEP3]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %1 = tail call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %0, i1 true)
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %1, ptr %gep3, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_lrint(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.vp.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.vp.trunc.nxv4i32.nxv4i64(<vscale x 4 x i64> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP15]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load float, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV2:%.*]] = fpext float [[TMP20]] to double
+; IF-EVL-NEXT:    [[TMP21:%.*]] = tail call i64 @llvm.lrint.i64.f64(double [[CONV2]])
+; IF-EVL-NEXT:    [[CONV3:%.*]] = trunc i64 [[TMP21]] to i32
+; IF-EVL-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[CONV3]], ptr [[GEP5]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_lrint(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load float, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV2:%.*]] = fpext float [[TMP0]] to double
+; NO-VP-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.lrint.i64.f64(double [[CONV2]])
+; NO-VP-NEXT:    [[CONV3:%.*]] = trunc i64 [[TMP1]] to i32
+; NO-VP-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[CONV3]], ptr [[GEP5]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fpext float %0 to double
+  %1 = tail call i64 @llvm.lrint.i64.f64(double %conv2)
+  %conv3 = trunc i64 %1 to i32
+  %gep5 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv3, ptr %gep5, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_llrint(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.vp.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.vp.trunc.nxv4i32.nxv4i64(<vscale x 4 x i64> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP15]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load float, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV2:%.*]] = fpext float [[TMP20]] to double
+; IF-EVL-NEXT:    [[TMP21:%.*]] = tail call i64 @llvm.llrint.i64.f64(double [[CONV2]])
+; IF-EVL-NEXT:    [[CONV3:%.*]] = trunc i64 [[TMP21]] to i32
+; IF-EVL-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[CONV3]], ptr [[GEP5]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_llrint(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load float, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV2:%.*]] = fpext float [[TMP0]] to double
+; NO-VP-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.llrint.i64.f64(double [[CONV2]])
+; NO-VP-NEXT:    [[CONV3:%.*]] = trunc i64 [[TMP1]] to i32
+; NO-VP-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[CONV3]], ptr [[GEP5]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fpext float %0 to double
+  %1 = tail call i64 @llvm.llrint.i64.f64(double %conv2)
+  %conv3 = trunc i64 %1 to i32
+  %gep5 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv3, ptr %gep5, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_abs(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_abs(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vp.abs.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], i1 true, <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP24]], ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[COND:%.*]] = tail call i32 @llvm.abs.i32(i32 [[TMP18]], i1 true)
+; IF-EVL-NEXT:    [[GEP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[COND]], ptr [[GEP9]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_abs(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[COND:%.*]] = tail call i32 @llvm.abs.i32(i32 [[TMP0]], i1 true)
+; NO-VP-NEXT:    [[GEP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[COND]], ptr [[GEP9]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %cond = tail call i32 @llvm.abs.i32(i32 %0, i1 true)
+  %gep9 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %cond, ptr %gep9, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.ctlz.i32(i32, i1 immarg)
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+declare i64 @llvm.lrint.i64.f64(double)
+declare i64 @llvm.llrint.i64.f64(double)
+declare i32 @llvm.abs.i32(i32, i1 immarg)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
new file mode 100644
index 0000000000000..4557e95f1e1b6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -0,0 +1,1109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-tail-folding-style=data-with-evl \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=IF-EVL
+
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-tail-folding-style=none \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP
+
+define void @vp_sext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_sext(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
+; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 2
+; IF-EVL-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; IF-EVL-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i64> @llvm.vp.sext.nxv2i64.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr align 8 [[TMP18]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV2:%.*]] = sext i32 [[TMP21]] to i64
+; IF-EVL-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i64 [[CONV2]], ptr [[GEP4]], align 8
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_sext(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV2:%.*]] = sext i32 [[TMP0]] to i64
+; NO-VP-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i64 [[CONV2]], ptr [[GEP4]], align 8
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = sext i32 %0 to i64
+  %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %conv2, ptr %gep4, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_zext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_zext(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
+; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 2
+; IF-EVL-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; IF-EVL-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META9:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i64> @llvm.vp.zext.nxv2i64.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr align 8 [[TMP18]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = zext i32 [[TMP21]] to i64
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i64 [[CONV]], ptr [[GEP2]], align 8
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_zext(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = zext i32 [[TMP0]] to i64
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i64 [[CONV]], ptr [[GEP2]], align 8
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv = zext i32 %0 to i64
+  %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %conv, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_trunc(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 2
+; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 3
+; IF-EVL-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; IF-EVL-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META16:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i32> @llvm.vp.trunc.nxv2i32.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i32.p0(<vscale x 2 x i32> [[TMP16]], ptr align 4 [[TMP18]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i64, ptr [[GEP]], align 8
+; IF-EVL-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP21]] to i32
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_trunc(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i64, ptr [[GEP]], align 8
+; NO-VP-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i64, ptr %b, i64 %iv
+  %0 = load i64, ptr %gep, align 8
+  %conv = trunc i64 %0 to i32
+  %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_fpext(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
+; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 2
+; IF-EVL-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; IF-EVL-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x float> @llvm.vp.load.nxv2f32.p0(ptr align 4 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META23:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f32(<vscale x 2 x float> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP18]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load float, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = fpext float [[TMP21]] to double
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store double [[CONV]], ptr [[GEP2]], align 8
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_fpext(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load float, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = fpext float [[TMP0]] to double
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store double [[CONV]], ptr [[GEP2]], align 8
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv = fpext float %0 to double
+  %gep2 = getelementptr inbounds double, ptr %a, i64 %iv
+  store double %conv, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_fptrunc(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 2
+; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 3
+; IF-EVL-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; IF-EVL-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; IF-EVL-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x double> @llvm.vp.load.nxv2f64.p0(ptr align 8 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META30:![0-9]+]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x float> @llvm.vp.fptrunc.nxv2f32.nxv2f64(<vscale x 2 x double> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2f32.p0(<vscale x 2 x float> [[TMP16]], ptr align 4 [[TMP18]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load double, ptr [[GEP]], align 8
+; IF-EVL-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP21]] to float
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP36:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_fptrunc(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load double, ptr [[GEP]], align 8
+; NO-VP-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP0]] to float
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds double, ptr %b, i64 %iv
+  %0 = load double, ptr %gep, align 8
+  %conv = fptrunc double %0 to float
+  %gep2 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_sitofp(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP18]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP23]] to float
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP38:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_sitofp(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv = sitofp i32 %0 to float
+  %gep2 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_uitofp(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP18]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load i32, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = uitofp i32 [[TMP23]] to float
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP40:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_uitofp(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = uitofp i32 [[TMP0]] to float
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store float [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv = uitofp i32 %0 to float
+  %gep2 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_fptosi(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP18]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load float, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = fptosi float [[TMP23]] to i32
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP42:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_fptosi(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load float, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = fptosi float [[TMP0]] to i32
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv = fptosi float %0 to i32
+  %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_fptoui(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP18]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load float, ptr [[GEP]], align 4
+; IF-EVL-NEXT:    [[CONV:%.*]] = fptoui float [[TMP23]] to i32
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP44:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_fptoui(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load float, ptr [[GEP]], align 4
+; NO-VP-NEXT:    [[CONV:%.*]] = fptoui float [[TMP0]] to i32
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i32 [[CONV]], ptr [[GEP2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv = fptoui float %0 to i32
+  %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv, ptr %gep2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_inttoptr(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
+; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
+; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL:       [[VECTOR_MEMCHECK]]:
+; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
+; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call <vscale x 2 x ptr> @llvm.vp.inttoptr.nxv2p0.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[TMP15]]
+; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds ptr, ptr [[TMP19]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2p0.p0(<vscale x 2 x ptr> [[TMP18]], ptr align 8 [[TMP20]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load i64, ptr [[GEP]], align 8
+; IF-EVL-NEXT:    [[TMP24:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store ptr [[TMP24]], ptr [[GEP2]], align 8
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP46:![0-9]+]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_inttoptr(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = load i64, ptr [[GEP]], align 8
+; NO-VP-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store ptr [[TMP1]], ptr [[GEP2]], align 8
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i64, ptr %b, i64 %iv
+  %0 = load i64, ptr %gep, align 8
+  %1 = inttoptr i64 %0 to ptr
+  %gep2 = getelementptr inbounds ptr, ptr %a, i64 %iv
+  store ptr %1, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL-LABEL: define void @vp_ptrtoint(
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    br label %[[LOOP:.*]]
+; IF-EVL:       [[LOOP]]:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; IF-EVL-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64
+; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IF-EVL-NEXT:    store i64 [[TMP0]], ptr [[GEP2]], align 8
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; IF-EVL:       [[EXIT]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @vp_ptrtoint(
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    br label %[[LOOP:.*]]
+; NO-VP:       [[LOOP]]:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-VP-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64
+; NO-VP-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT:    store i64 [[TMP0]], ptr [[GEP2]], align 8
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; NO-VP:       [[EXIT]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = ptrtoint ptr %gep to i64
+  %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %0, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+;.
+; IF-EVL: [[META0]] = !{[[META1:![0-9]+]]}
+; IF-EVL: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; IF-EVL: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; IF-EVL: [[META3]] = !{[[META4:![0-9]+]]}
+; IF-EVL: [[META4]] = distinct !{[[META4]], [[META2]]}
+; IF-EVL: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; IF-EVL: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; IF-EVL: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; IF-EVL: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
+; IF-EVL: [[META9]] = !{[[META10:![0-9]+]]}
+; IF-EVL: [[META10]] = distinct !{[[META10]], [[META11:![0-9]+]]}
+; IF-EVL: [[META11]] = distinct !{[[META11]], !"LVerDomain"}
+; IF-EVL: [[META12]] = !{[[META13:![0-9]+]]}
+; IF-EVL: [[META13]] = distinct !{[[META13]], [[META11]]}
+; IF-EVL: [[LOOP14]] = distinct !{[[LOOP14]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP15]] = distinct !{[[LOOP15]], [[META6]]}
+; IF-EVL: [[META16]] = !{[[META17:![0-9]+]]}
+; IF-EVL: [[META17]] = distinct !{[[META17]], [[META18:![0-9]+]]}
+; IF-EVL: [[META18]] = distinct !{[[META18]], !"LVerDomain"}
+; IF-EVL: [[META19]] = !{[[META20:![0-9]+]]}
+; IF-EVL: [[META20]] = distinct !{[[META20]], [[META18]]}
+; IF-EVL: [[LOOP21]] = distinct !{[[LOOP21]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP22]] = distinct !{[[LOOP22]], [[META6]]}
+; IF-EVL: [[META23]] = !{[[META24:![0-9]+]]}
+; IF-EVL: [[META24]] = distinct !{[[META24]], [[META25:![0-9]+]]}
+; IF-EVL: [[META25]] = distinct !{[[META25]], !"LVerDomain"}
+; IF-EVL: [[META26]] = !{[[META27:![0-9]+]]}
+; IF-EVL: [[META27]] = distinct !{[[META27]], [[META25]]}
+; IF-EVL: [[LOOP28]] = distinct !{[[LOOP28]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP29]] = distinct !{[[LOOP29]], [[META6]]}
+; IF-EVL: [[META30]] = !{[[META31:![0-9]+]]}
+; IF-EVL: [[META31]] = distinct !{[[META31]], [[META32:![0-9]+]]}
+; IF-EVL: [[META32]] = distinct !{[[META32]], !"LVerDomain"}
+; IF-EVL: [[META33]] = !{[[META34:![0-9]+]]}
+; IF-EVL: [[META34]] = distinct !{[[META34]], [[META32]]}
+; IF-EVL: [[LOOP35]] = distinct !{[[LOOP35]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP36]] = distinct !{[[LOOP36]], [[META6]]}
+; IF-EVL: [[LOOP37]] = distinct !{[[LOOP37]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP38]] = distinct !{[[LOOP38]], [[META6]]}
+; IF-EVL: [[LOOP39]] = distinct !{[[LOOP39]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP40]] = distinct !{[[LOOP40]], [[META6]]}
+; IF-EVL: [[LOOP41]] = distinct !{[[LOOP41]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP42]] = distinct !{[[LOOP42]], [[META6]]}
+; IF-EVL: [[LOOP43]] = distinct !{[[LOOP43]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP44]] = distinct !{[[LOOP44]], [[META6]]}
+; IF-EVL: [[LOOP45]] = distinct !{[[LOOP45]], [[META6]], [[META7]]}
+; IF-EVL: [[LOOP46]] = distinct !{[[LOOP46]], [[META6]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
index bc55519b1adb3..733c05fd9259c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
@@ -72,7 +72,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY1]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY1]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
@@ -135,7 +135,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; IF-EVL-INLOOP:       scalar.ph:
 ; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
 ; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -192,7 +192,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP-OUTLOOP:       scalar.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; NO-VP-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; NO-VP-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; NO-VP-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       for.body:
 ; NO-VP-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -248,7 +248,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP-INLOOP:       scalar.ph:
 ; NO-VP-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; NO-VP-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[START]], [[VECTOR_MEMCHECK]] ], [ [[START]], [[ENTRY]] ]
+; NO-VP-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ], [ [[START]], [[VECTOR_MEMCHECK]] ]
 ; NO-VP-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP-INLOOP:       for.body:
 ; NO-VP-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
index fbc4ad0bac50a..dc74f44688fdf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
@@ -190,8 +190,8 @@ define void @no_overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD]], <vscale x 2 x i64> splat (i64 1), <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
 ; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP5]] to i64
-; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP9]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index f55e755cde8bb..af5a62e5f480d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -1727,7 +1727,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
-; IF-EVL-NEXT:    [[TMP16:%.*]] = call reassoc <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VP_OP_LOAD1]], <vscale x 4 x float> [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = call reassoc <vscale x 4 x float> @llvm.vp.fmuladd.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VP_OP_LOAD1]], <vscale x 4 x float> [[VEC_PHI]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP17]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP16]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP10]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index 79471ab21dc78..209c251f6a6a4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -50,8 +50,8 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP20]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP20]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL:       middle.block:
@@ -166,8 +166,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[VP_REVERSE_MASK6:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE5]], ptr align 4 [[TMP25]], <vscale x 4 x i1> [[VP_REVERSE_MASK6]], i32 [[TMP5]])
 ; IF-EVL-NEXT:    [[TMP28:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP28]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP28]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
index 6c2c2612fe2d4..313511d017f65 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
@@ -40,8 +40,8 @@ define void @test(ptr %p) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; IF-EVL:       middle.block:
@@ -360,8 +360,8 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       middle.block:
@@ -445,8 +445,8 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
 ; IF-EVL-NEXT:    call void @llvm.vp.store.nxv1i64.p0(<vscale x 1 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP6]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
-; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP11]], [[EVL_BASED_IV]]
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
new file mode 100644
index 0000000000000..5c47f07f4436c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
@@ -0,0 +1,476 @@
+; REQUIRES: asserts
+
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
+; RUN: -force-tail-folding-style=data-with-evl \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
+
+define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[SMAX:%.+]]> = call llvm.vp.smax(ir<[[LD1]]>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[SMAX]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.smax.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[SMIN:%.+]]> = call llvm.vp.smin(ir<[[LD1]]>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[SMIN]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.smin.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[UMAX:%.+]]> = call llvm.vp.umax(ir<[[LD1]]>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[UMAX]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.umax.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[UMIN:%.+]]> = call llvm.vp.umin(ir<[[LD1]]>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[UMIN]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %gep3 = getelementptr inbounds i32, ptr %c, i64 %iv
+  %1 = load i32, ptr %gep3, align 4
+  %. = tail call i32 @llvm.umin.i32(i32 %0, i32 %1)
+  %gep11 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %., ptr %gep11, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[CTLZ:%.+]]> = call llvm.vp.ctlz(ir<[[LD1]]>, ir<true>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, ir<[[CTLZ]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %1 = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %0, i1 true)
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %1, ptr %gep3, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[CTTZ:%.+]]> = call llvm.vp.cttz(ir<[[LD1]]>, ir<true>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, ir<[[CTTZ]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %1 = tail call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %0, i1 true)
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %1, ptr %gep3, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[LRINT:%.+]]> = call llvm.vp.lrint(vp<[[FPEXT]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LRINT]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fpext float %0 to double
+  %1 = tail call i64 @llvm.lrint.i64.f64(double %conv2)
+  %conv3 = trunc i64 %1 to i32
+  %gep5 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv3, ptr %gep5, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[LLRINT:%.+]]> = call llvm.vp.llrint(vp<[[FPEXT]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LLRINT]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fpext float %0 to double
+  %1 = tail call i64 @llvm.llrint.i64.f64(double %conv2)
+  %conv3 = trunc i64 %1 to i32
+  %gep5 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv3, ptr %gep5, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_abs(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[ABS:%.+]]> = call llvm.vp.abs(ir<[[LD1]]>, ir<true>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, ir<[[ABS]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count  vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %cond = tail call i32 @llvm.abs.i32(i32 %0, i1 true)
+  %gep9 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %cond, ptr %gep9, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.ctlz.i32(i32, i1 immarg)
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+declare i64 @llvm.lrint.i64.f64(double)
+declare i64 @llvm.llrint.i64.f64(double)
+declare i32 @llvm.abs.i32(i32, i1 immarg)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
new file mode 100644
index 0000000000000..f4acfd6185644
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
@@ -0,0 +1,507 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
+; RUN: -force-tail-folding-style=data-with-evl \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
+
+define void @vp_sext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL:   <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[SEXT]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+; IF-EVL-NEXT: Successor(s): middle.block
+
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = sext i32 %0 to i64
+  %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %conv2, ptr %gep4, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_zext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[ZEXT]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = zext i32 %0 to i64
+  %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %conv2, ptr %gep4, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = trunc i32 %0 to i16
+  %gep4 = getelementptr inbounds i16, ptr %a, i64 %iv
+  store i16 %conv2, ptr %gep4, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[FPEXT]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fpext float %0 to double
+  %gep4 = getelementptr inbounds double, ptr %a, i64 %iv
+  store double %conv2, ptr %gep4, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTRUNC]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds double, ptr %b, i64 %iv
+  %0 = load double, ptr %gep, align 8
+  %conv2 = fptrunc double %0 to float
+  %gep4 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv2, ptr %gep4, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = sitofp i32 %0 to float
+  %gep4 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv2, ptr %gep4, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = load i32, ptr %gep, align 4
+  %conv2 = uitofp i32 %0 to float
+  %gep4 = getelementptr inbounds float, ptr %a, i64 %iv
+  store float %conv2, ptr %gep4, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptosi(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fptosi float %0 to i32
+  %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv2, ptr %gep4, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[FPTOUI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOUI]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %gep = getelementptr inbounds float, ptr %b, i64 %iv
+  %0 = load float, ptr %gep, align 4
+  %conv2 = fptoui float %0 to i32
+  %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %conv2, ptr %gep4, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
+; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
+; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[INTTOPTR:%.+]]> = call llvm.vp.inttoptr(ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
+; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR2]]>, vp<[[INTTOPTR]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i64, ptr %b, i64 %iv
+  %0 = load i64, ptr %gep, align 8
+  %1 = inttoptr i64 %0 to ptr
+  %gep2 = getelementptr inbounds ptr, ptr %a, i64 %iv
+  store ptr %1, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
+; IF-EVL: LV: Vectorization is possible but not beneficial.
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %b, i64 %iv
+  %0 = ptrtoint ptr %gep to i64
+  %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv
+  store i64 %0, ptr %gep2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index ab541f6fa94e6..a474d926a6303 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -59,10 +59,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:   EMIT branch-on-cond ir<true>
 ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; IF-EVL-OUTLOOP-EMPTY:
-; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-OUTLOOP-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; IF-EVL-OUTLOOP-NEXT: No successors
-; IF-EVL-OUTLOOP-EMPTY:
 ; IF-EVL-OUTLOOP-NEXT: scalar.ph:
 ; IF-EVL-OUTLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -72,6 +68,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:   IR   %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
 ; IF-EVL-OUTLOOP:        IR   %exitcond.not = icmp eq i64 %iv.next, %n
 ; IF-EVL-OUTLOOP-NEXT: No successors
+; IF-EVL-OUTLOOP-EMPTY:
+; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
+; IF-EVL-OUTLOOP-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; IF-EVL-OUTLOOP-NEXT: No successors
 ; IF-EVL-OUTLOOP-NEXT: }
 ;
 
@@ -109,10 +109,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:   EMIT branch-on-cond ir<true>
 ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; IF-EVL-INLOOP-EMPTY:
-; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-INLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; IF-EVL-INLOOP-NEXT: No successors
-; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP-NEXT: scalar.ph:
 ; IF-EVL-INLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -122,6 +118,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:   IR   %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
 ; IF-EVL-INLOOP:        IR   %exitcond.not = icmp eq i64 %iv.next, %n
 ; IF-EVL-INLOOP-NEXT: No successors
+; IF-EVL-INLOOP-EMPTY:
+; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
+; IF-EVL-INLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; IF-EVL-INLOOP-NEXT: No successors
 ; IF-EVL-INLOOP-NEXT: }
 ;
 
@@ -155,10 +155,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:   EMIT branch-on-cond vp<[[BOC]]>
 ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; NO-VP-OUTLOOP-EMPTY:
-; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-OUTLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; NO-VP-OUTLOOP-NEXT: No successors
-; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: scalar.ph:
 ; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -168,6 +164,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:   IR   %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
 ; NO-VP-OUTLOOP:        IR   %exitcond.not = icmp eq i64 %iv.next, %n
 ; NO-VP-OUTLOOP-NEXT: No successors
+; NO-VP-OUTLOOP-EMPTY:
+; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
+; NO-VP-OUTLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; NO-VP-OUTLOOP-NEXT: No successors
 ; NO-VP-OUTLOOP-NEXT: }
 ;
 
@@ -201,10 +201,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:   EMIT branch-on-cond vp<[[BOC]]>
 ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; NO-VP-INLOOP-EMPTY:
-; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-INLOOP-NEXT:   IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; NO-VP-INLOOP-NEXT: No successors
-; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: scalar.ph:
 ; NO-VP-INLOOP-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
 ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -214,6 +210,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:   IR   %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
 ; NO-VP-INLOOP:        IR   %exitcond.not = icmp eq i64 %iv.next, %n
 ; NO-VP-INLOOP-NEXT: No successors
+; NO-VP-INLOOP-EMPTY:
+; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
+; NO-VP-INLOOP-NEXT:   IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; NO-VP-INLOOP-NEXT: No successors
 ; NO-VP-INLOOP-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 6d6cfb5e9d18e..4983ea0f7a738 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -7,8 +7,8 @@
 
  define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
  ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
- ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
- ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+ ; IF-EVL-NEXT: Live-in ir<[[VFUF:%.+]]> = VF * UF
+ ; IF-EVL-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
  ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
 
  ; IF-EVL: vector.ph:
@@ -16,8 +16,8 @@
 
  ; IF-EVL: <x1> vector loop: {
  ; IF-EVL-NEXT:   vector.body:
- ; IF-EVL-NEXT:     EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
- ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEX:%.+]]>
+ ; IF-EVL-NEXT:     SCALAR-PHI vp<[[IV:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]>
+ ; IF-EVL-NEXT:     SCALAR-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEX:%.+]]>
  ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
  ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
  ; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
@@ -29,15 +29,15 @@
  ; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
  ; IF-EVL-NEXT:     WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
  ; IF-EVL-NEXT:     WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<[[LD2]]>, ir<[[SUB]]>, vp<[[EVL]]>)
  ; IF-EVL-NEXT:     WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>
  ; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
  ; IF-EVL-NEXT:     vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
  ; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
  ; IF-EVL-NEXT:     SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
  ; IF-EVL-NEXT:     EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
- ; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  vp<[[VTC]]>
+ ; IF-EVL-NEXT:     EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<[[VFUF]]>
+ ; IF-EVL-NEXT:     EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  ir<[[VTC]]>
  ; IF-EVL-NEXT:   No successors
  ; IF-EVL-NEXT: }
 
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
index 107ae50461296..385f7621801b4 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
@@ -33,7 +33,7 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
 ; CHECK-NEXT:    store i64 [[TMP5]], ptr [[TMP6]], align 8
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
 ; CHECK:       [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
index f828b74bc8bf6..98245fcc0f0af 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
@@ -136,7 +136,7 @@ define void @test(ptr %p, i40 %a) {
 ; CHECK-NEXT:    store i1 [[TMP40]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE32]]
 ; CHECK:       pred.store.continue32:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
index 699b8487a065a..d0754f1c2bb55 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
@@ -58,7 +58,7 @@ define void @func_21() {
 ; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP18]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK:       pred.store.continue4:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index 64e57747bf2ae..e685a83d9ccbb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -102,7 +102,7 @@ attributes #0 = { "target-cpu"="knl" }
 ; FORCE-NEXT:    store i32 [[TMP1]], ptr @b, align 1
 ; FORCE-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; FORCE:       pred.store.continue2:
-; FORCE-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; FORCE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; FORCE-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; FORCE-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
 ; FORCE-NEXT:    br i1 [[TMP15]], label {{%.*}}, label [[VECTOR_BODY]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 9f6362b361685..4ca988a2d9b53 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -226,8 +226,8 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF7:%.*]] = urem i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_VEC8:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF7]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC8]], 32
@@ -276,7 +276,7 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    br i1 [[CMP_N14]], label [[LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP157]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ITER_CHECK]] ], [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi float [ [[TMP157]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP124]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR:%.*]]
 ; CHECK:       for:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR]] ]
@@ -778,14 +778,14 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF38:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[N_MOD_VF38]], 0
 ; CHECK-NEXT:    [[TMP85:%.*]] = select i1 [[TMP84]], i64 4, i64 [[N_MOD_VF38]]
 ; CHECK-NEXT:    [[N_VEC39:%.*]] = sub i64 [[TMP0]], [[TMP85]]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX40:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT51:%.*]], [[PRED_STORE_CONTINUE50:%.*]] ]
+; CHECK-NEXT:    [[INDEX40:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL1]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT51:%.*]], [[PRED_STORE_CONTINUE50:%.*]] ]
 ; CHECK-NEXT:    [[TMP86:%.*]] = add i64 [[INDEX40]], 0
 ; CHECK-NEXT:    [[TMP87:%.*]] = shl nsw i64 [[TMP86]], 2
 ; CHECK-NEXT:    [[TMP89:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP87]]
@@ -1052,7 +1052,7 @@ define i64 @live_in_known_1_via_scev() {
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 5)
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[VEC_PHI]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
index e94bd84136025..4400b6ba568d2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
@@ -27,11 +27,11 @@ define void @test_pr59459(i64 %iv.start, ptr %arr) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 16
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IV_START]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[IV_START]] to i32
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[TMP9]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <16 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IV_START]], [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -92,7 +92,7 @@ define void @test_pr59459(i64 %iv.start, ptr %arr) {
 ; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC4]]
 ; CHECK-NEXT:    br i1 [[CMP_N16]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IV_START]], [[VECTOR_SCEVCHECK]] ], [ [[IV_START]], [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IV_START]], [[ITER_CHECK:%.*]] ], [ [[IV_START]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL7]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -145,17 +145,17 @@ define void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l,
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[L]], 64
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[L]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i16
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i16 [[DOTCAST]], [[TMP0]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[DOTSPLATINSERT2]], <16 x i16> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul <16 x i16> splat (i16 16), [[TMP2]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i16> [[DOTSPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = mul <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, [[DOTSPLAT]]
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <16 x i16> zeroinitializer, [[TMP21]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i16> poison, i16 [[OFF]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i16> [[BROADCAST_SPLATINSERT2]], <16 x i16> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i16
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i16 [[DOTCAST]], [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, [[DOTSPLAT]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <16 x i16> zeroinitializer, [[TMP11]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[OFF]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i16> [[BROADCAST_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -192,7 +192,7 @@ define void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l,
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[L]], 8
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[L]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[DOTCAST7:%.*]] = trunc i64 [[N_VEC5]] to i16
@@ -210,7 +210,7 @@ define void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l,
 ; CHECK-NEXT:    [[BROADCAST_SPLAT23:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT22]], <8 x i16> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND20:%.*]] = phi <8 x i16> [ [[INDUCTION17]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT21:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX12]], 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub <8 x i16> [[VEC_IND20]], [[BROADCAST_SPLAT23]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
index 43366314f5247..12eb3bd03348a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -39,9 +39,9 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[STEP_ADD2:%.*]] = fadd fast <8 x float> [[VEC_IND]], splat (float 8.000000e+00)
 ; AUTO_VEC-NEXT:    [[STEP_ADD3:%.*]] = fadd fast <8 x float> [[VEC_IND]], splat (float 1.200000e+01)
 ; AUTO_VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; AUTO_VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
-; AUTO_VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 64
-; AUTO_VEC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 96
+; AUTO_VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AUTO_VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64
+; AUTO_VEC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 96
 ; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND]], ptr [[TMP1]], align 4
 ; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD]], ptr [[TMP2]], align 4
 ; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD2]], ptr [[TMP3]], align 4
@@ -86,7 +86,7 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC:       for.body:
 ; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; AUTO_VEC-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END8]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IND_END6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
-; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; AUTO_VEC-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -147,35 +147,35 @@ define void @fp_iv_loop2(ptr noalias nocapture %A, i32 %N) {
 ; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[X_06:%.*]] = phi float [ 1.000000e+00, [[FOR_BODY_PREHEADER_NEW]] ], [ [[CONV1_7:%.*]], [[FOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[FOR_BODY]] ]
-; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; AUTO_VEC-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1:%.*]] = fadd float [[X_06]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; AUTO_VEC-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
 ; AUTO_VEC-NEXT:    store float [[CONV1]], ptr [[ARRAYIDX_1]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_1:%.*]] = fadd float [[CONV1]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
-; AUTO_VEC-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_1]], ptr [[ARRAYIDX_2]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_2:%.*]] = fadd float [[CONV1_1]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
-; AUTO_VEC-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_2]], ptr [[ARRAYIDX_3]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_3:%.*]] = fadd float [[CONV1_2]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = or disjoint i64 [[INDVARS_IV]], 4
-; AUTO_VEC-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_3]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_3]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_3]], ptr [[ARRAYIDX_4]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_4:%.*]] = fadd float [[CONV1_3]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = or disjoint i64 [[INDVARS_IV]], 5
-; AUTO_VEC-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_4]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_4]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_4]], ptr [[ARRAYIDX_5]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_5:%.*]] = fadd float [[CONV1_4]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = or disjoint i64 [[INDVARS_IV]], 6
-; AUTO_VEC-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_5]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_5]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_5]], ptr [[ARRAYIDX_6]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_6:%.*]] = fadd float [[CONV1_5]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = or disjoint i64 [[INDVARS_IV]], 7
-; AUTO_VEC-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT_6]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_NEXT_6]]
 ; AUTO_VEC-NEXT:    store float [[CONV1_6]], ptr [[ARRAYIDX_7]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_7]] = fadd float [[CONV1_6]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
@@ -191,7 +191,7 @@ define void @fp_iv_loop2(ptr noalias nocapture %A, i32 %N) {
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[INDVARS_IV_UNR]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ]
 ; AUTO_VEC-NEXT:    [[X_06_EPIL:%.*]] = phi float [ [[CONV1_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[X_06_UNR]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ]
 ; AUTO_VEC-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ], [ 0, [[FOR_END_LOOPEXIT_UNR_LCSSA]] ]
-; AUTO_VEC-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_EPIL]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV_EPIL]]
 ; AUTO_VEC-NEXT:    store float [[X_06_EPIL]], ptr [[ARRAYIDX_EPIL]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1_EPIL]] = fadd float [[X_06_EPIL]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
@@ -411,9 +411,9 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) {
 ; AUTO_VEC-NEXT:    [[STEP_ADD2:%.*]] = fadd reassoc <8 x float> [[STEP_ADD]], splat (float 3.360000e+02)
 ; AUTO_VEC-NEXT:    [[STEP_ADD3:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2]], splat (float 3.360000e+02)
 ; AUTO_VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i64 [[INDEX]]
-; AUTO_VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 32
-; AUTO_VEC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 64
-; AUTO_VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 96
+; AUTO_VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 32
+; AUTO_VEC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 64
+; AUTO_VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 96
 ; AUTO_VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP2]], align 4
 ; AUTO_VEC-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
 ; AUTO_VEC-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
@@ -470,7 +470,7 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) {
 ; AUTO_VEC:       for.body:
 ; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ITER_CHECK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[N_VEC6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; AUTO_VEC-NEXT:    [[X_012:%.*]] = phi float [ [[ADD3:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[ITER_CHECK]] ], [ [[IND_END11]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[IND_END9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
-; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[INDVARS_IV]]
+; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[P]], i64 [[INDVARS_IV]]
 ; AUTO_VEC-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[ADD:%.*]] = fadd reassoc float [[X_012]], [[TMP16]]
 ; AUTO_VEC-NEXT:    store float [[ADD]], ptr [[ARRAYIDX]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
index cc190fb826911..9adc64a6020e7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -705,8 +705,8 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt
 ; AVX512-NEXT:    [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
 ; AVX512-NEXT:    br i1 [[CMP_N17]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[ITER_CHECK]] ]
-; AVX512-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[PTR_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL13]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index bc74742d952ab..3d0fe42635fe5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -48,15 +48,13 @@ define i32 @iv_used_widened_and_truncated(ptr %dst, i64 %N) #0 {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[N_MOD_VF8:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC9:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF8]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[BC_RESUME_VAL7]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
 ; CHECK-NEXT:    [[DOTSPLATINSERT14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT15:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT14]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION16:%.*]] = add <4 x i32> [[DOTSPLAT15]], <i32 0, i32 1, i32 2, i32 3>
@@ -353,7 +351,7 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
 ; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ], [ [[A]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ], [ [[B]], [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END5]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ], [ [[B]], [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 2048, [[VECTOR_MEMCHECK]] ], [ 2048, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 2048, [[ENTRY]] ], [ 2048, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LOOP]] ]
@@ -516,9 +514,8 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
@@ -637,7 +634,7 @@ define void @wide_iv_trunc(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    store i32 [[TMP10]], ptr [[DST]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
index 3338cc85772e4..203beff421320 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
@@ -20,14 +20,14 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP1]]
 ; SSE-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
 ; SSE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; SSE-NEXT:    [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; SSE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC]]
-; SSE-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC2]]
+; SSE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
+; SSE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; SSE-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
 ; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
 ; SSE-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP6]], align 4
 ; SSE-NEXT:    store <4 x i32> [[TMP5]], ptr [[TMP7]], align 4
 ; SSE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -62,24 +62,24 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
 ; AVX1-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4
 ; AVX1-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX1-NEXT:    [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
-; AVX1-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX1-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
-; AVX1-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX1-NEXT:    [[WIDE_VEC3:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
-; AVX1-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <8 x i32> [[WIDE_VEC3]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <8 x i32> [[WIDE_VEC3]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX1-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
-; AVX1-NEXT:    [[TMP11:%.*]] = add nsw <4 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
-; AVX1-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
-; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
+; AVX1-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
+; AVX1-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX1-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT:    [[WIDE_VEC5:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
+; AVX1-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <8 x i32> [[WIDE_VEC5]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX1-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <8 x i32> [[WIDE_VEC5]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT:    [[WIDE_VEC8:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
+; AVX1-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX1-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; AVX1-NEXT:    [[TMP11:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
+; AVX1-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC6]]
+; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC9]]
 ; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; AVX1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 16
-; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 32
-; AVX1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 48
+; AVX1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 16
+; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 32
+; AVX1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 48
 ; AVX1-NEXT:    store <4 x i32> [[TMP10]], ptr [[TMP14]], align 4
 ; AVX1-NEXT:    store <4 x i32> [[TMP11]], ptr [[TMP15]], align 4
 ; AVX1-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP16]], align 4
@@ -116,24 +116,24 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
 ; AVX2-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP6]], align 4
 ; AVX2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT:    [[WIDE_VEC1:%.*]] = load <16 x i32>, ptr [[TMP7]], align 4
-; AVX2-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP8]], align 4
-; AVX2-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT:    [[WIDE_VEC3:%.*]] = load <16 x i32>, ptr [[TMP9]], align 4
-; AVX2-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC3]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <16 x i32> [[WIDE_VEC3]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT:    [[TMP10:%.*]] = add nsw <8 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
-; AVX2-NEXT:    [[TMP11:%.*]] = add nsw <8 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
-; AVX2-NEXT:    [[TMP12:%.*]] = add nsw <8 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
-; AVX2-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
+; AVX2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP7]], align 4
+; AVX2-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; AVX2-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT:    [[WIDE_VEC5:%.*]] = load <16 x i32>, ptr [[TMP8]], align 4
+; AVX2-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; AVX2-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT:    [[WIDE_VEC8:%.*]] = load <16 x i32>, ptr [[TMP9]], align 4
+; AVX2-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <16 x i32> [[WIDE_VEC8]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; AVX2-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <16 x i32> [[WIDE_VEC8]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT:    [[TMP10:%.*]] = add nsw <8 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; AVX2-NEXT:    [[TMP11:%.*]] = add nsw <8 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
+; AVX2-NEXT:    [[TMP12:%.*]] = add nsw <8 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC6]]
+; AVX2-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC9]]
 ; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 32
-; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 64
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 96
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 32
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 64
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 96
 ; AVX2-NEXT:    store <8 x i32> [[TMP10]], ptr [[TMP14]], align 4
 ; AVX2-NEXT:    store <8 x i32> [[TMP11]], ptr [[TMP15]], align 4
 ; AVX2-NEXT:    store <8 x i32> [[TMP12]], ptr [[TMP16]], align 4
@@ -158,13 +158,13 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; ATOM:       for.body:
 ; ATOM-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; ATOM-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
-; ATOM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
+; ATOM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP0]]
 ; ATOM-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; ATOM-NEXT:    [[TMP2:%.*]] = or disjoint i64 [[TMP0]], 1
-; ATOM-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP2]]
+; ATOM-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP2]]
 ; ATOM-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
 ; ATOM-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], [[TMP1]]
-; ATOM-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; ATOM-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; ATOM-NEXT:    store i32 [[ADD4]], ptr [[ARRAYIDX6]], align 4
 ; ATOM-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; ATOM-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
diff --git a/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
index 4ff5dc1bc91c6..b11a58c4c0ae6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll
@@ -101,10 +101,10 @@ define void @uaddsat(ptr nocapture readonly %pSrc, i16 signext %offset, ptr noca
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PSRC_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL13]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR3:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL16]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_08]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_08]], i64 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = load i16, ptr [[PSRC_ADDR_08]], align 2
 ; CHECK-NEXT:    [[TMP19:%.*]] = tail call i16 @llvm.uadd.sat.i16(i16 [[TMP18]], i16 [[OFFSET]])
-; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds i8, ptr [[PDST_ADDR_07]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_07]], i64 2
 ; CHECK-NEXT:    store i16 [[TMP19]], ptr [[PDST_ADDR_07]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_09]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
@@ -220,10 +220,10 @@ define void @fshl(ptr nocapture readonly %pSrc, i8 signext %offset, ptr nocaptur
 ; CHECK-NEXT:    [[BLKCNT_09:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PSRC_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL12]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR3:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL15]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_08]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_08]], i64 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[PSRC_ADDR_08]], align 2
 ; CHECK-NEXT:    [[TMP15:%.*]] = tail call i8 @llvm.fshl.i8(i8 [[TMP14]], i8 [[TMP14]], i8 [[OFFSET]])
-; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds i8, ptr [[PDST_ADDR_07]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR3]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_07]], i64 1
 ; CHECK-NEXT:    store i8 [[TMP15]], ptr [[PDST_ADDR_07]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_09]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
index c9a47a60866f1..d2d33cbda8062 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -26,20 +26,20 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775792
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x ptr> poison, ptr [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x ptr> [[BROADCAST_SPLATINSERT]], <16 x ptr> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x ptr> [[BROADCAST_SPLAT]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <16 x ptr> [[BROADCAST_SPLAT]], zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <16 x i32> poison, i32 [[NTRUNC]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT4]], <16 x i32> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP1]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP2]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> [[BROADCAST_SPLAT]], i32 4, <16 x i1> [[TMP3]], <16 x i32> poison), !alias.scope [[META3]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1>
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> [[BROADCAST_SPLAT]], i32 4, <16 x i1> [[TMP1]], <16 x i32> poison), !alias.scope [[META3]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[TMP1]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i32> [[PREDPHI]], i64 15
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
@@ -50,31 +50,31 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_VEC7:%.*]] = and i64 [[SMAX2]], 9223372036854775800
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT11:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT10]], <8 x ptr> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x ptr> [[BROADCAST_SPLAT11]], zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <8 x i32> poison, i32 [[NTRUNC]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT12]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT8]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x ptr> [[BROADCAST_SPLAT9]], zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i32> poison, i32 [[NTRUNC]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT11]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT16:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX9]]
-; CHECK-NEXT:    store <8 x i32> [[BROADCAST_SPLAT13]], ptr [[TMP5]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
-; CHECK-NEXT:    [[INDEX_NEXT16]] = add nuw i64 [[INDEX9]], 8
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT16]], [[N_VEC7]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX10]]
+; CHECK-NEXT:    store <8 x i32> [[BROADCAST_SPLAT12]], ptr [[TMP6]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT15]] = add nuw i64 [[INDEX10]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC7]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[BROADCAST_SPLAT11]], i32 4, <8 x i1> [[TMP7]], <8 x i32> poison), !alias.scope [[META11]]
-; CHECK-NEXT:    [[PREDPHI15:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[WIDE_MASKED_GATHER14]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1>
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[PREDPHI15]], i64 7
-; CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC7]]
-; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[BROADCAST_SPLAT9]], i32 4, <8 x i1> [[TMP5]], <8 x i32> poison), !alias.scope [[META11]]
+; CHECK-NEXT:    [[PREDPHI14:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[WIDE_MASKED_GATHER13]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[PREDPHI14]], i64 7
+; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC7]]
+; CHECK-NEXT:    br i1 [[CMP_N16]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC7]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC7]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[A]], null
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[LATCH]], label [[COND_LOAD:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
index 58c6827128feb..41841f27a0c06 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -35,9 +35,9 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 64
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 128
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 192
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 128
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 192
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP2]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <16 x i32>, ptr [[TMP3]], align 8, !alias.scope [[META0]]
@@ -62,8 +62,8 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_VEC13:%.*]] = and i64 [[SMAX2]], 9223372036854775800
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, i32 [[BC_MERGE_RDX]], i64 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -82,13 +82,13 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC13]]
 ; CHECK-NEXT:    br i1 [[CMP_N18]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX19:%.*]] = phi i32 [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX19:%.*]] = phi i32 [ [[TMP15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[T0:%.*]] = phi i32 [ [[T3:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX19]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, ptr [[T1]], align 8
 ; CHECK-NEXT:    [[T3]] = add i32 [[T0]], [[T2]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[A]], align 4
@@ -210,11 +210,11 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
 ; CHECK-NEXT:    [[CMP_N23:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC9]]
 ; CHECK-NEXT:    br i1 [[CMP_N23]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, ptr [[T1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[T2]], [[K]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[T1]], align 4
@@ -335,17 +335,17 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b
 ; CHECK-NEXT:    [[CMP_N28:%.*]] = icmp eq i64 [[SMAX10]], [[N_VEC17]]
 ; CHECK-NEXT:    br i1 [[CMP_N28]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, ptr [[T1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[T2]], [[K]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[T1]], align 4
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[LATCH]]
 ; CHECK:       cond_store:
-; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[I]]
+; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[I]]
 ; CHECK-NEXT:    [[T4:%.*]] = load i32, ptr [[T3]], align 8
 ; CHECK-NEXT:    store i32 [[T4]], ptr [[A]], align 4
 ; CHECK-NEXT:    br label [[LATCH]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index 7cf36fc669f37..7f0b6b2f9b4d7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -57,9 +57,8 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 206bbdf262b72..739833653cb16 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -160,7 +160,7 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -267,7 +267,7 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -461,7 +461,7 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -568,7 +568,7 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -778,7 +778,7 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX2:       vec.epilog.middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX2:       vec.epilog.scalar.ph:
-; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX2-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX2:       for.body:
 ; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -891,7 +891,7 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -1222,7 +1222,7 @@ define void @foo3(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; AVX512:       vec.epilog.middle.block:
 ; AVX512-NEXT:    br i1 true, label [[FOR_END]], label [[SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY1:%.*]]
 ; AVX512:       for.body:
 ; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
index dbe39964eb229..7c467c264843a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
@@ -21,82 +21,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O1-NEXT:  entry:
 ; O1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O1-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O1-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O1-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O1-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O1-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O1-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O1-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O1-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O1-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O1-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O1-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O1-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O1-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O1-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O1-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O1-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O1-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O1-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O1-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O1-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O1-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O1-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O1-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O1-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O1-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O1-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O1-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O1-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O1-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O1-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O1-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O1-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O1-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O1-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O1-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O1-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O1-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O1-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O1-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O1-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O1-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -106,82 +106,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O2-NEXT:  entry:
 ; O2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O2-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O2-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O2-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O2-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O2-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O2-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O2-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O2-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O2-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O2-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O2-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O2-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -191,82 +191,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O3-NEXT:  entry:
 ; O3-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O3-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O3-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O3-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O3-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O3-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O3-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O3-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O3-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O3-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O3-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O3-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O3-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O3-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O3-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O3-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O3-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O3-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O3-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O3-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O3-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O3-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O3-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O3-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O3-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O3-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O3-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O3-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O3-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O3-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O3-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O3-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O3-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O3-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O3-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O3-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O3-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O3-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O3-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O3-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O3-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -276,82 +276,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O3DEFAULT-NEXT:  entry:
 ; O3DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O3DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O3DEFAULT-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O3DEFAULT-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O3DEFAULT-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O3DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O3DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O3DEFAULT-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O3DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O3DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O3DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O3DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O3DEFAULT-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O3DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3DEFAULT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O3DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O3DEFAULT-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O3DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O3DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O3DEFAULT-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O3DEFAULT-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O3DEFAULT-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3DEFAULT-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3DEFAULT-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O3DEFAULT-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O3DEFAULT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O3DEFAULT-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O3DEFAULT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O3DEFAULT-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O3DEFAULT-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O3DEFAULT-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O3DEFAULT-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O3DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O3DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O3DEFAULT-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O3DEFAULT-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O3DEFAULT-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O3DEFAULT-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O3DEFAULT-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -361,82 +361,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; Os-NEXT:  entry:
 ; Os-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; Os-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; Os-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; Os-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; Os-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; Os-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; Os-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; Os-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; Os-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; Os-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; Os-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; Os-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; Os-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; Os-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; Os-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; Os-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; Os-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; Os-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; Os-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; Os-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; Os-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; Os-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; Os-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; Os-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; Os-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; Os-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; Os-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; Os-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; Os-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; Os-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; Os-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; Os-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; Os-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; Os-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; Os-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; Os-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; Os-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; Os-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; Os-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; Os-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; Os-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; Os-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; Os-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; Os-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; Os-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; Os-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; Os-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; Os-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; Os-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; Os-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; Os-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; Os-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; Os-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; Os-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; Os-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; Os-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; Os-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; Os-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; Os-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; Os-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; Os-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -446,82 +446,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; Oz-NEXT:  entry:
 ; Oz-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; Oz-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; Oz-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; Oz-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; Oz-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; Oz-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; Oz-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; Oz-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; Oz-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; Oz-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; Oz-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; Oz-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; Oz-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; Oz-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; Oz-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; Oz-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; Oz-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; Oz-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; Oz-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; Oz-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; Oz-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; Oz-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; Oz-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; Oz-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; Oz-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; Oz-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; Oz-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; Oz-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; Oz-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; Oz-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; Oz-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; Oz-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; Oz-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; Oz-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; Oz-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; Oz-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; Oz-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; Oz-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; Oz-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; Oz-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; Oz-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; Oz-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; Oz-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; Oz-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; Oz-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; Oz-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; Oz-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; Oz-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; Oz-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; Oz-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; Oz-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; Oz-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; Oz-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; Oz-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; Oz-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; Oz-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; Oz-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; Oz-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; Oz-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; Oz-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; Oz-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; Oz-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; Oz-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; Oz-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; Oz-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; Oz-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; Oz-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; Oz-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; Oz-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; Oz-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; Oz-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; Oz-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; Oz-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; Oz-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; Oz-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; Oz-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; Oz-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; Oz-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; Oz-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; Oz-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; Oz-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; Oz-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; Oz-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -531,82 +531,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O1VEC2-NEXT:  entry:
 ; O1VEC2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O1VEC2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O1VEC2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O1VEC2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O1VEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O1VEC2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O1VEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O1VEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O1VEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O1VEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O1VEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O1VEC2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O1VEC2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O1VEC2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O1VEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O1VEC2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O1VEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O1VEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O1VEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O1VEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O1VEC2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O1VEC2-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O1VEC2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O1VEC2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O1VEC2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O1VEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O1VEC2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O1VEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O1VEC2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O1VEC2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O1VEC2-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O1VEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O1VEC2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O1VEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O1VEC2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O1VEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O1VEC2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O1VEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O1VEC2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O1VEC2-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O1VEC2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O1VEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O1VEC2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O1VEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O1VEC2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O1VEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O1VEC2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O1VEC2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O1VEC2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O1VEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O1VEC2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O1VEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O1VEC2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O1VEC2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O1VEC2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O1VEC2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O1VEC2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O1VEC2-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O1VEC2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O1VEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O1VEC2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O1VEC2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O1VEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O1VEC2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O1VEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O1VEC2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O1VEC2-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O1VEC2-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O1VEC2-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O1VEC2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O1VEC2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O1VEC2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O1VEC2-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O1VEC2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -616,82 +616,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; OzVEC2-NEXT:  entry:
 ; OzVEC2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; OzVEC2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; OzVEC2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; OzVEC2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; OzVEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; OzVEC2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; OzVEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; OzVEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; OzVEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; OzVEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; OzVEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; OzVEC2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; OzVEC2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; OzVEC2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; OzVEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; OzVEC2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; OzVEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; OzVEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; OzVEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; OzVEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; OzVEC2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; OzVEC2-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; OzVEC2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; OzVEC2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; OzVEC2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; OzVEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; OzVEC2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; OzVEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; OzVEC2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; OzVEC2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; OzVEC2-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; OzVEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; OzVEC2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; OzVEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; OzVEC2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; OzVEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; OzVEC2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; OzVEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; OzVEC2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; OzVEC2-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; OzVEC2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; OzVEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; OzVEC2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; OzVEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; OzVEC2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; OzVEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; OzVEC2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; OzVEC2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; OzVEC2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; OzVEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; OzVEC2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; OzVEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; OzVEC2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; OzVEC2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; OzVEC2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; OzVEC2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; OzVEC2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; OzVEC2-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; OzVEC2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; OzVEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; OzVEC2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; OzVEC2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; OzVEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; OzVEC2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; OzVEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; OzVEC2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; OzVEC2-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; OzVEC2-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; OzVEC2-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; OzVEC2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; OzVEC2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; OzVEC2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; OzVEC2-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; OzVEC2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -701,82 +701,82 @@ define i32 @enabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b,
 ; O3DIS-NEXT:  entry:
 ; O3DIS-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O3DIS-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O3DIS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O3DIS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O3DIS-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O3DIS-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O3DIS-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O3DIS-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O3DIS-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3DIS-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O3DIS-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3DIS-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O3DIS-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O3DIS-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O3DIS-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3DIS-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O3DIS-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3DIS-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O3DIS-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3DIS-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O3DIS-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3DIS-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O3DIS-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O3DIS-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O3DIS-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3DIS-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O3DIS-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3DIS-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O3DIS-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3DIS-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O3DIS-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3DIS-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O3DIS-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O3DIS-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O3DIS-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3DIS-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O3DIS-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3DIS-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O3DIS-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3DIS-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O3DIS-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3DIS-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O3DIS-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O3DIS-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O3DIS-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3DIS-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O3DIS-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3DIS-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O3DIS-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3DIS-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O3DIS-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3DIS-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O3DIS-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O3DIS-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O3DIS-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3DIS-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O3DIS-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3DIS-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O3DIS-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O3DIS-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O3DIS-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O3DIS-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O3DIS-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O3DIS-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O3DIS-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O3DIS-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O3DIS-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O3DIS-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O3DIS-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O3DIS-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O3DIS-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O3DIS-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O3DIS-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O3DIS-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O3DIS-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O3DIS-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O3DIS-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O3DIS-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O3DIS-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O3DIS-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O3DIS-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O3DIS-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O3DIS-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -807,10 +807,10 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1-NEXT:    br label [[FOR_BODY:%.*]]
 ; O1:       for.body:
 ; O1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O1-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
@@ -823,82 +823,82 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O2-NEXT:  entry:
 ; O2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O2-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O2-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O2-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O2-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O2-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O2-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O2-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O2-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O2-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O2-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O2-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O2-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O2-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O2-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O2-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O2-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -908,82 +908,82 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O3-NEXT:  entry:
 ; O3-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O3-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O3-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O3-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O3-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O3-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O3-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O3-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O3-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O3-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O3-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O3-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O3-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O3-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O3-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O3-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O3-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O3-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O3-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O3-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O3-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O3-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O3-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O3-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O3-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O3-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O3-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O3-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O3-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O3-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O3-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O3-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O3-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O3-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O3-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O3-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O3-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O3-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O3-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O3-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O3-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O3-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O3-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O3-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O3-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O3-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -993,82 +993,82 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O3DEFAULT-NEXT:  entry:
 ; O3DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; O3DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; O3DEFAULT-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; O3DEFAULT-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; O3DEFAULT-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; O3DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; O3DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; O3DEFAULT-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; O3DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; O3DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; O3DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; O3DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; O3DEFAULT-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; O3DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3DEFAULT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; O3DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; O3DEFAULT-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; O3DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; O3DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; O3DEFAULT-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; O3DEFAULT-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; O3DEFAULT-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3DEFAULT-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3DEFAULT-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; O3DEFAULT-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; O3DEFAULT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; O3DEFAULT-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; O3DEFAULT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; O3DEFAULT-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; O3DEFAULT-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; O3DEFAULT-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; O3DEFAULT-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; O3DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; O3DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; O3DEFAULT-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; O3DEFAULT-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; O3DEFAULT-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; O3DEFAULT-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; O3DEFAULT-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; O3DEFAULT-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; O3DEFAULT-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; O3DEFAULT-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -1078,82 +1078,82 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; Os-NEXT:  entry:
 ; Os-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i64 0
 ; Os-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; Os-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 16
+; Os-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 16
 ; Os-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[B]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; Os-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 16
+; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 16
 ; Os-NEXT:    store <4 x i32> [[TMP1]], ptr [[A]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; Os-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; Os-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
+; Os-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; Os-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
 ; Os-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
 ; Os-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_1]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
-; Os-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
+; Os-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; Os-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; Os-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
+; Os-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
 ; Os-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_2:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4
 ; Os-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_2]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; Os-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; Os-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
+; Os-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; Os-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; Os-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
+; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; Os-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
 ; Os-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_3:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4
 ; Os-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_3]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
-; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; Os-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
+; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; Os-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
-; Os-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
+; Os-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
 ; Os-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_4:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4
 ; Os-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_4]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
-; Os-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
+; Os-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; Os-NEXT:    store <4 x i32> [[TMP24]], ptr [[TMP26]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP25]], ptr [[TMP27]], align 4
-; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; Os-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
+; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; Os-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
 ; Os-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_5:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
 ; Os-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP31:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_5]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
-; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; Os-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
+; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; Os-NEXT:    store <4 x i32> [[TMP30]], ptr [[TMP32]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP31]], ptr [[TMP33]], align 4
-; Os-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 192
-; Os-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 208
+; Os-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 192
+; Os-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 208
 ; Os-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_6:%.*]] = load <4 x i32>, ptr [[TMP35]], align 4
 ; Os-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP37:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_6]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 192
-; Os-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 208
+; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 192
+; Os-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 208
 ; Os-NEXT:    store <4 x i32> [[TMP36]], ptr [[TMP38]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP37]], ptr [[TMP39]], align 4
-; Os-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 224
-; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 240
+; Os-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 224
+; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 240
 ; Os-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, ptr [[TMP40]], align 4
 ; Os-NEXT:    [[WIDE_LOAD1_7:%.*]] = load <4 x i32>, ptr [[TMP41]], align 4
 ; Os-NEXT:    [[TMP42:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT]]
 ; Os-NEXT:    [[TMP43:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1_7]], [[BROADCAST_SPLAT]]
-; Os-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 224
-; Os-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 240
+; Os-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 224
+; Os-NEXT:    [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 240
 ; Os-NEXT:    store <4 x i32> [[TMP42]], ptr [[TMP44]], align 4
 ; Os-NEXT:    store <4 x i32> [[TMP43]], ptr [[TMP45]], align 4
 ; Os-NEXT:    [[TMP46:%.*]] = load i32, ptr [[A]], align 4
@@ -1164,10 +1164,10 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; Oz-NEXT:    br label [[FOR_BODY:%.*]]
 ; Oz:       for.body:
 ; Oz-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; Oz-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; Oz-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; Oz-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; Oz-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; Oz-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
@@ -1186,21 +1186,21 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1VEC2:       vector.body:
 ; O1VEC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; O1VEC2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; O1VEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
-; O1VEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
-; O1VEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 4
-; O1VEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
-; O1VEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
-; O1VEC2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; O1VEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; O1VEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
-; O1VEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 4
-; O1VEC2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP10]], align 4
-; O1VEC2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP11]], align 4
+; O1VEC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
+; O1VEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; O1VEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 4
+; O1VEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; O1VEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; O1VEC2-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; O1VEC2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; O1VEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
+; O1VEC2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
+; O1VEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 4
+; O1VEC2-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP7]], align 4
+; O1VEC2-NEXT:    store <4 x i32> [[TMP5]], ptr [[TMP8]], align 4
 ; O1VEC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; O1VEC2-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; O1VEC2-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; O1VEC2-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; O1VEC2-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; O1VEC2:       middle.block:
 ; O1VEC2-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; O1VEC2:       scalar.ph:
@@ -1208,17 +1208,17 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1VEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; O1VEC2:       for.body:
 ; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
-; O1VEC2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP13]], [[N]]
-; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP10]], [[N]]
+; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; O1VEC2-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O1VEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O1VEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
 ; O1VEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; O1VEC2:       for.end:
-; O1VEC2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[A]], align 4
-; O1VEC2-NEXT:    ret i32 [[TMP14]]
+; O1VEC2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A]], align 4
+; O1VEC2-NEXT:    ret i32 [[TMP11]]
 ;
 ; OzVEC2-LABEL: @nopragma(
 ; OzVEC2-NEXT:  entry:
@@ -1230,21 +1230,21 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; OzVEC2:       vector.body:
 ; OzVEC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; OzVEC2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; OzVEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
-; OzVEC2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
-; OzVEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 4
-; OzVEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
-; OzVEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
-; OzVEC2-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; OzVEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; OzVEC2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
-; OzVEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 4
-; OzVEC2-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP10]], align 4
-; OzVEC2-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP11]], align 4
+; OzVEC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
+; OzVEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; OzVEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 4
+; OzVEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; OzVEC2-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; OzVEC2-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; OzVEC2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; OzVEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
+; OzVEC2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
+; OzVEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 4
+; OzVEC2-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP7]], align 4
+; OzVEC2-NEXT:    store <4 x i32> [[TMP5]], ptr [[TMP8]], align 4
 ; OzVEC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; OzVEC2-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; OzVEC2-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; OzVEC2-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; OzVEC2-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; OzVEC2:       middle.block:
 ; OzVEC2-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; OzVEC2:       scalar.ph:
@@ -1252,27 +1252,27 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; OzVEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; OzVEC2:       for.body:
 ; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
-; OzVEC2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP13]], [[N]]
-; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP10]], [[N]]
+; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; OzVEC2-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; OzVEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OzVEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
 ; OzVEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; OzVEC2:       for.end:
-; OzVEC2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[A]], align 4
-; OzVEC2-NEXT:    ret i32 [[TMP14]]
+; OzVEC2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A]], align 4
+; OzVEC2-NEXT:    ret i32 [[TMP11]]
 ;
 ; O3DIS-LABEL: @nopragma(
 ; O3DIS-NEXT:  entry:
 ; O3DIS-NEXT:    br label [[FOR_BODY:%.*]]
 ; O3DIS:       for.body:
 ; O3DIS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O3DIS-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O3DIS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O3DIS-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O3DIS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O3DIS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
@@ -1306,10 +1306,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1-NEXT:    br label [[FOR_BODY:%.*]]
 ; O1:       for.body:
 ; O1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O1-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1323,10 +1323,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O2-NEXT:    br label [[FOR_BODY:%.*]]
 ; O2:       for.body:
 ; O2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O2-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1340,10 +1340,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O3-NEXT:    br label [[FOR_BODY:%.*]]
 ; O3:       for.body:
 ; O3-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O3-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O3-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O3-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O3-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O3-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O3-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O3-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1359,58 +1359,58 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O3DEFAULT-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; O3DEFAULT-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP0]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP3]], ptr [[A:%.*]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 16
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 16
+; O3DEFAULT-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 16
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_4:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
 ; O3DEFAULT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_4]], align 4
 ; O3DEFAULT-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP4]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP5]], ptr [[ARRAYIDX2_4]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 32
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 32
+; O3DEFAULT-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 32
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 32
 ; O3DEFAULT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_8]], align 4
 ; O3DEFAULT-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP7]], ptr [[ARRAYIDX2_8]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 48
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_12:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 48
+; O3DEFAULT-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 48
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_12:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 48
 ; O3DEFAULT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_12]], align 4
 ; O3DEFAULT-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[TMP8]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP9]], ptr [[ARRAYIDX2_12]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_16:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
+; O3DEFAULT-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 64
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_16:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 64
 ; O3DEFAULT-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_16]], align 4
 ; O3DEFAULT-NEXT:    [[TMP11:%.*]] = add nsw <4 x i32> [[TMP10]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP11]], ptr [[ARRAYIDX2_16]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 80
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 80
+; O3DEFAULT-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 80
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 80
 ; O3DEFAULT-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_20]], align 4
 ; O3DEFAULT-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[TMP12]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP13]], ptr [[ARRAYIDX2_20]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 96
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_24:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 96
+; O3DEFAULT-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 96
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_24:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 96
 ; O3DEFAULT-NEXT:    [[TMP14:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_24]], align 4
 ; O3DEFAULT-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP15]], ptr [[ARRAYIDX2_24]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 112
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 112
+; O3DEFAULT-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 112
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_28:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 112
 ; O3DEFAULT-NEXT:    [[TMP16:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_28]], align 4
 ; O3DEFAULT-NEXT:    [[TMP17:%.*]] = add nsw <4 x i32> [[TMP16]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP17]], ptr [[ARRAYIDX2_28]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 128
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_32:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 128
+; O3DEFAULT-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 128
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_32:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 128
 ; O3DEFAULT-NEXT:    [[TMP18:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_32]], align 4
 ; O3DEFAULT-NEXT:    [[TMP19:%.*]] = add nsw <4 x i32> [[TMP18]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP19]], ptr [[ARRAYIDX2_32]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 144
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_36:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 144
+; O3DEFAULT-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 144
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_36:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 144
 ; O3DEFAULT-NEXT:    [[TMP20:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_36]], align 4
 ; O3DEFAULT-NEXT:    [[TMP21:%.*]] = add nsw <4 x i32> [[TMP20]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP21]], ptr [[ARRAYIDX2_36]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 160
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_40:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 160
+; O3DEFAULT-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 160
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_40:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 160
 ; O3DEFAULT-NEXT:    [[TMP22:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_40]], align 4
 ; O3DEFAULT-NEXT:    [[TMP23:%.*]] = add nsw <4 x i32> [[TMP22]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP23]], ptr [[ARRAYIDX2_40]], align 4
-; O3DEFAULT-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 176
-; O3DEFAULT-NEXT:    [[ARRAYIDX2_44:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 176
+; O3DEFAULT-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 176
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_44:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 176
 ; O3DEFAULT-NEXT:    [[TMP24:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_44]], align 4
 ; O3DEFAULT-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[TMP24]], [[TMP2]]
 ; O3DEFAULT-NEXT:    store <4 x i32> [[TMP25]], ptr [[ARRAYIDX2_44]], align 4
@@ -1422,10 +1422,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; Os-NEXT:    br label [[FOR_BODY:%.*]]
 ; Os:       for.body:
 ; Os-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; Os-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; Os-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; Os-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; Os-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; Os-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; Os-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; Os-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; Os-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; Os-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1439,10 +1439,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; Oz-NEXT:    br label [[FOR_BODY:%.*]]
 ; Oz:       for.body:
 ; Oz-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; Oz-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; Oz-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; Oz-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; Oz-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; Oz-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1456,10 +1456,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1VEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; O1VEC2:       for.body:
 ; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O1VEC2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O1VEC2-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O1VEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O1VEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1473,10 +1473,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; OzVEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; OzVEC2:       for.body:
 ; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; OzVEC2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; OzVEC2-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; OzVEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OzVEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
@@ -1490,10 +1490,10 @@ define i32 @disabled(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O3DIS-NEXT:    br label [[FOR_BODY:%.*]]
 ; O3DIS:       for.body:
 ; O3DIS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; O3DIS-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O3DIS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
-; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; O3DIS-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; O3DIS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; O3DIS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
diff --git a/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
index cd128979fc143..4e768074019d3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll
@@ -58,7 +58,7 @@ define i64 @test_value_in_exit_compare_chain_used_outside(ptr %src, i64 %x, i64
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index a12aed7239c9d..597be33ca63f6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -28,7 +28,7 @@ define i32 @foo_optsize() #0 {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr [[TMP3]], i32 1, <64 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -67,7 +67,7 @@ define i32 @foo_optsize() #0 {
 ; AUTOVF-NEXT:    [[TMP4:%.*]] = icmp eq <32 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AUTOVF-NEXT:    [[TMP5:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> splat (i8 2), <32 x i8> splat (i8 1)
 ; AUTOVF-NEXT:    call void @llvm.masked.store.v32i8.p0(<32 x i8> [[TMP5]], ptr [[TMP3]], i32 1, <32 x i1> [[TMP1]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 32
+; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
 ; AUTOVF-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 224
 ; AUTOVF-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AUTOVF:       middle.block:
@@ -128,7 +128,7 @@ define i32 @foo_minsize() #1 {
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr [[TMP3]], i32 1, <64 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
@@ -167,7 +167,7 @@ define i32 @foo_minsize() #1 {
 ; AUTOVF-NEXT:    [[TMP4:%.*]] = icmp eq <32 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AUTOVF-NEXT:    [[TMP5:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> splat (i8 2), <32 x i8> splat (i8 1)
 ; AUTOVF-NEXT:    call void @llvm.masked.store.v32i8.p0(<32 x i8> [[TMP5]], ptr [[TMP3]], i32 1, <32 x i1> [[TMP1]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 32
+; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
 ; AUTOVF-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 224
 ; AUTOVF-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; AUTOVF:       middle.block:
@@ -383,7 +383,7 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = add <64 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <64 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v64p0.v64p0(<64 x ptr> zeroinitializer, <64 x ptr> [[TMP5]], i32 8, <64 x i1> [[TMP6]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -4608
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -429,7 +429,7 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; AUTOVF-NEXT:    [[VEC_IV:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AUTOVF-NEXT:    [[TMP6:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
 ; AUTOVF-NEXT:    call void @llvm.masked.scatter.v8p0.v8p0(<8 x ptr> zeroinitializer, <8 x ptr> [[TMP5]], i32 8, <8 x i1> [[TMP6]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; AUTOVF-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -576
 ; AUTOVF-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; AUTOVF-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
index 89febbccfd8d2..55c9930183b80 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
@@ -10,8 +10,8 @@ define void @foo(ptr addrspace(1) align 8 dereferenceable_or_null(16), ptr addrs
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[ITER_CHECK:%.*]]
 ; CHECK:       iter.check:
-; CHECK-NEXT:    [[DOT10:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP0:%.*]], i64 16
-; CHECK-NEXT:    [[DOT12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP1:%.*]], i64 16
+; CHECK-NEXT:    [[DOT10:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP0:%.*]], i64 16
+; CHECK-NEXT:    [[DOT12:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP1:%.*]], i64 16
 ; CHECK-NEXT:    [[UMAX2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2:%.*]], i64 1)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -33,17 +33,17 @@ define void @foo(ptr addrspace(1) align 8 dereferenceable_or_null(16), ptr addrs
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[DOT12]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP5]], i64 32
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP5]], i64 64
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP5]], i64 96
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP5]], i64 32
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP5]], i64 64
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP5]], i64 96
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x ptr addrspace(1)>, ptr addrspace(1) [[TMP5]], align 8, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x ptr addrspace(1)>, ptr addrspace(1) [[TMP6]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x ptr addrspace(1)>, ptr addrspace(1) [[TMP7]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x ptr addrspace(1)>, ptr addrspace(1) [[TMP8]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[DOT10]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP9]], i64 32
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP9]], i64 64
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP9]], i64 96
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP9]], i64 32
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP9]], i64 64
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[TMP9]], i64 96
 ; CHECK-NEXT:    store <4 x ptr addrspace(1)> [[WIDE_LOAD]], ptr addrspace(1) [[TMP9]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    store <4 x ptr addrspace(1)> [[WIDE_LOAD4]], ptr addrspace(1) [[TMP10]], align 8, !alias.scope [[META3]], !noalias [[META0]]
 ; CHECK-NEXT:    store <4 x ptr addrspace(1)> [[WIDE_LOAD5]], ptr addrspace(1) [[TMP11]], align 8, !alias.scope [[META3]], !noalias [[META0]]
@@ -75,7 +75,7 @@ define void @foo(ptr addrspace(1) align 8 dereferenceable_or_null(16), ptr addrs
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC8]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label [[LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT4:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll b/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
index 1c64359dea249..bb3f37ddf16e5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr42674.ll
@@ -9,7 +9,7 @@
 define zeroext i8 @sum() {
 ; CHECK-LABEL: @sum(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr getelementptr inbounds (i8, ptr @bytes, i64 64), align 1
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <64 x i8>, ptr getelementptr inbounds nuw (i8, ptr @bytes, i64 64), align 1
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr @bytes, align 1
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <64 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> [[BIN_RDX]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
index 6f769f48bada3..23a6a1286a0f0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
@@ -66,11 +66,10 @@ define ptr addrspace(10) @japi1_vect_42283(ptr nocapture readonly %0, i32 %1) lo
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[TOP]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[TOP]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[TMP8]], [[N_MOD_VF4]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x ptr addrspace(10)> poison, ptr addrspace(10) [[DOTUNPACK]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll b/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
index a5eb42ae184a7..0ccffe76fe477 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
@@ -9,8 +9,8 @@ define void @test_pr55096(i64 %c, ptr %p) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 122, i64 123>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE3]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 122, i64 123>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[INDEX]] to i16
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul i16 [[DOTCAST]], 2008
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 6229, [[TMP0]]
@@ -27,15 +27,15 @@ define void @test_pr55096(i64 %c, ptr %p) {
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; CHECK:       pred.store.if2:
+; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.if1:
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i16 [[OFFSET_IDX]], 2008
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i16 [[TMP10]], 2008
 ; CHECK-NEXT:    [[TMP12:%.*]] = udiv i16 4943, [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[P]], i16 [[TMP12]]
 ; CHECK-NEXT:    store i16 0, ptr [[TMP13]], align 2
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; CHECK:       pred.store.continue3:
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 340
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
index df9ba2194139b..f0dfcf3eada72 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
@@ -88,7 +88,7 @@ define void @test(ptr %p) {
 ; VEC:       scalar.ph:
 ; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
 ; VEC-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; VEC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY]] ]
+; VEC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ], [ 1, [[VECTOR_SCEVCHECK]] ]
 ; VEC-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC:       for.body:
 ; VEC-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IDX:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
index 73bdae71654f1..a190e94a01489 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
@@ -35,7 +35,7 @@ define void @test(ptr noundef align 8 dereferenceable_or_null(16) %arr) #0 {
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 -3
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i1> [[TMP4]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr [[TMP8]], i32 8, <4 x i1> [[REVERSE]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
index 414460872ba13..f54c0a14cf63c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -124,16 +124,16 @@ define void @_Z3fn1v() #0 {
 ; CHECK:       vector.ph30:
 ; CHECK-NEXT:    [[N_MOD_VF31:%.*]] = urem i64 [[TMP28]], 16
 ; CHECK-NEXT:    [[N_VEC32:%.*]] = sub i64 [[TMP28]], [[N_MOD_VF31]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TOBOOL6]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[N_VEC32]], 2
 ; CHECK-NEXT:    [[IND_END41:%.*]] = add i64 8, [[TMP29]]
 ; CHECK-NEXT:    [[IND_END43:%.*]] = mul i64 [[N_VEC32]], 2
-; CHECK-NEXT:    br label [[VECTOR_BODY33:%.*]]
-; CHECK:       vector.body33:
-; CHECK-NEXT:    [[INDEX34:%.*]] = phi i64 [ 0, [[VECTOR_PH30]] ], [ [[INDEX_NEXT39:%.*]], [[VECTOR_BODY33]] ]
-; CHECK-NEXT:    [[VEC_IND35:%.*]] = phi <16 x i64> [ <i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30, i64 32, i64 34, i64 36, i64 38>, [[VECTOR_PH30]] ], [ [[VEC_IND_NEXT36:%.*]], [[VECTOR_BODY33]] ]
-; CHECK-NEXT:    [[VEC_IND37:%.*]] = phi <16 x i64> [ <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30>, [[VECTOR_PH30]] ], [ [[VEC_IND_NEXT38:%.*]], [[VECTOR_BODY33]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TOBOOL6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY35:%.*]]
+; CHECK:       vector.body35:
+; CHECK-NEXT:    [[INDEX34:%.*]] = phi i64 [ 0, [[VECTOR_PH30]] ], [ [[INDEX_NEXT39:%.*]], [[VECTOR_BODY35]] ]
+; CHECK-NEXT:    [[VEC_IND35:%.*]] = phi <16 x i64> [ <i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30, i64 32, i64 34, i64 36, i64 38>, [[VECTOR_PH30]] ], [ [[VEC_IND_NEXT36:%.*]], [[VECTOR_BODY35]] ]
+; CHECK-NEXT:    [[VEC_IND37:%.*]] = phi <16 x i64> [ <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30>, [[VECTOR_PH30]] ], [ [[VEC_IND_NEXT38:%.*]], [[VECTOR_BODY35]] ]
 ; CHECK-NEXT:    [[TMP30:%.*]] = sub nsw <16 x i64> splat (i64 8), [[VEC_IND35]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds [10 x [10 x i32]], ptr @d, i64 0, <16 x i64> [[VEC_IND35]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = add nsw <16 x i64> [[TMP30]], [[VEC_IND37]]
@@ -153,18 +153,18 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[VEC_IND_NEXT36]] = add <16 x i64> [[VEC_IND35]], splat (i64 32)
 ; CHECK-NEXT:    [[VEC_IND_NEXT38]] = add <16 x i64> [[VEC_IND37]], splat (i64 32)
 ; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT39]], [[N_VEC32]]
-; CHECK-NEXT:    br i1 [[TMP41]], label [[MIDDLE_BLOCK24:%.*]], label [[VECTOR_BODY33]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP41]], label [[MIDDLE_BLOCK24:%.*]], label [[VECTOR_BODY35]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block24:
 ; CHECK-NEXT:    [[CMP_N40:%.*]] = icmp eq i64 [[TMP28]], [[N_VEC32]]
 ; CHECK-NEXT:    br i1 [[CMP_N40]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK48:%.*]]
-; CHECK:       vec.epilog.iter.check48:
+; CHECK:       vec.epilog.iter.check49:
 ; CHECK-NEXT:    [[IND_END58:%.*]] = mul i64 [[N_VEC32]], 2
 ; CHECK-NEXT:    [[TMP42:%.*]] = mul i64 [[N_VEC32]], 2
 ; CHECK-NEXT:    [[IND_END55:%.*]] = add i64 8, [[TMP42]]
 ; CHECK-NEXT:    [[N_VEC_REMAINING49:%.*]] = sub i64 [[TMP28]], [[N_VEC32]]
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK50:%.*]] = icmp ult i64 [[N_VEC_REMAINING49]], 8
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK50]], label [[VEC_EPILOG_SCALAR_PH46]], label [[VEC_EPILOG_PH47]]
-; CHECK:       vec.epilog.ph47:
+; CHECK:       vec.epilog.ph48:
 ; CHECK-NEXT:    [[BC_RESUME_VAL42:%.*]] = phi i64 [ [[IND_END41]], [[VEC_EPILOG_ITER_CHECK48]] ], [ 8, [[VECTOR_MAIN_LOOP_ITER_CHECK29]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL44:%.*]] = phi i64 [ [[IND_END43]], [[VEC_EPILOG_ITER_CHECK48]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK29]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL51:%.*]] = phi i64 [ [[N_VEC32]], [[VEC_EPILOG_ITER_CHECK48]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK29]] ]
@@ -181,11 +181,11 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[INDUCTION69:%.*]] = add <8 x i64> [[DOTSPLAT68]], <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT72:%.*]] = insertelement <8 x i1> poison, i1 [[TOBOOL6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT73:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT72]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY60:%.*]]
-; CHECK:       vec.epilog.vector.body60:
-; CHECK-NEXT:    [[INDEX61:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL51]], [[VEC_EPILOG_PH47]] ], [ [[INDEX_NEXT74:%.*]], [[VEC_EPILOG_VECTOR_BODY60]] ]
-; CHECK-NEXT:    [[VEC_IND65:%.*]] = phi <8 x i64> [ [[INDUCTION64]], [[VEC_EPILOG_PH47]] ], [ [[VEC_IND_NEXT66:%.*]], [[VEC_EPILOG_VECTOR_BODY60]] ]
-; CHECK-NEXT:    [[VEC_IND70:%.*]] = phi <8 x i64> [ [[INDUCTION69]], [[VEC_EPILOG_PH47]] ], [ [[VEC_IND_NEXT71:%.*]], [[VEC_EPILOG_VECTOR_BODY60]] ]
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY58:%.*]]
+; CHECK:       vec.epilog.vector.body58:
+; CHECK-NEXT:    [[INDEX61:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL51]], [[VEC_EPILOG_PH47]] ], [ [[INDEX_NEXT74:%.*]], [[VEC_EPILOG_VECTOR_BODY58]] ]
+; CHECK-NEXT:    [[VEC_IND65:%.*]] = phi <8 x i64> [ [[INDUCTION64]], [[VEC_EPILOG_PH47]] ], [ [[VEC_IND_NEXT66:%.*]], [[VEC_EPILOG_VECTOR_BODY58]] ]
+; CHECK-NEXT:    [[VEC_IND70:%.*]] = phi <8 x i64> [ [[INDUCTION69]], [[VEC_EPILOG_PH47]] ], [ [[VEC_IND_NEXT71:%.*]], [[VEC_EPILOG_VECTOR_BODY58]] ]
 ; CHECK-NEXT:    [[TMP44:%.*]] = sub nsw <8 x i64> splat (i64 8), [[VEC_IND65]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [10 x [10 x i32]], ptr @d, i64 0, <8 x i64> [[VEC_IND65]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = add nsw <8 x i64> [[TMP44]], [[VEC_IND70]]
@@ -205,11 +205,11 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[VEC_IND_NEXT66]] = add <8 x i64> [[VEC_IND65]], splat (i64 16)
 ; CHECK-NEXT:    [[VEC_IND_NEXT71]] = add <8 x i64> [[VEC_IND70]], splat (i64 16)
 ; CHECK-NEXT:    [[TMP55:%.*]] = icmp eq i64 [[INDEX_NEXT74]], [[N_VEC53]]
-; CHECK-NEXT:    br i1 [[TMP55]], label [[VEC_EPILOG_MIDDLE_BLOCK45:%.*]], label [[VEC_EPILOG_VECTOR_BODY60]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       vec.epilog.middle.block45:
+; CHECK-NEXT:    br i1 [[TMP55]], label [[VEC_EPILOG_MIDDLE_BLOCK45:%.*]], label [[VEC_EPILOG_VECTOR_BODY58]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       vec.epilog.middle.block46:
 ; CHECK-NEXT:    [[CMP_N75:%.*]] = icmp eq i64 [[TMP28]], [[N_VEC53]]
 ; CHECK-NEXT:    br i1 [[CMP_N75]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH46]]
-; CHECK:       vec.epilog.scalar.ph46:
+; CHECK:       vec.epilog.scalar.ph47:
 ; CHECK-NEXT:    [[BC_RESUME_VAL56:%.*]] = phi i64 [ [[IND_END54]], [[VEC_EPILOG_MIDDLE_BLOCK45]] ], [ [[IND_END55]], [[VEC_EPILOG_ITER_CHECK48]] ], [ 8, [[ITER_CHECK27]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL59:%.*]] = phi i64 [ [[IND_END57]], [[VEC_EPILOG_MIDDLE_BLOCK45]] ], [ [[IND_END58]], [[VEC_EPILOG_ITER_CHECK48]] ], [ 0, [[ITER_CHECK27]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY_US:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index 99aaecf9a073c..4e98a4fc78805 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -28,12 +28,12 @@ define void @example1() optsize {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -89,7 +89,7 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store i32 [[X:%.*]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
@@ -97,7 +97,7 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; CHECK:       pred.store.if1:
 ; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP7]]
 ; CHECK-NEXT:    store i32 [[X]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; CHECK:       pred.store.continue2:
@@ -105,7 +105,7 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; CHECK:       pred.store.if3:
 ; CHECK-NEXT:    [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP10]]
 ; CHECK-NEXT:    store i32 [[X]], ptr [[TMP11]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK:       pred.store.continue4:
@@ -113,11 +113,11 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.if5:
 ; CHECK-NEXT:    [[TMP13:%.*]] = or disjoint i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP13]]
 ; CHECK-NEXT:    store i32 [[X]], ptr [[TMP14]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -138,71 +138,71 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[N_RND_UP10:%.*]] = add nuw nsw i64 [[TMP17]], 3
 ; CHECK-NEXT:    [[N_VEC12:%.*]] = and i64 [[N_RND_UP10]], 8589934588
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_116:%.*]] = add nsw i64 [[TMP17]], -1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_116]], i64 0
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_114:%.*]] = add nsw i64 [[TMP17]], -1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_114]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT20:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT19]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY15:%.*]]
+; CHECK:       vector.body15:
+; CHECK-NEXT:    [[INDEX16:%.*]] = phi i64 [ 0, [[VECTOR_PH9]] ], [ [[INDEX_NEXT29:%.*]], [[PRED_STORE_CONTINUE28:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX16]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX16]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT18:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT17]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY19:%.*]]
-; CHECK:       vector.body17:
-; CHECK-NEXT:    [[INDEX20:%.*]] = phi i64 [ 0, [[VECTOR_PH9]] ], [ [[INDEX_NEXT31:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX20]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT21:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX20]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT22:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT21]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT22]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT18]]
+; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT18]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT20]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0
-; CHECK-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
-; CHECK:       pred.store.if23:
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; CHECK:       pred.store.if21:
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP23]], [[TMP21]]
 ; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP24]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE24]]
-; CHECK:       pred.store.continue24:
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; CHECK:       pred.store.continue22:
 ; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[TMP18]], i64 1
-; CHECK-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
-; CHECK:       pred.store.if25:
+; CHECK-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; CHECK:       pred.store.if23:
 ; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[TMP27]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[TMP27]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP31]], [[TMP29]]
 ; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP32]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE26]]
-; CHECK:       pred.store.continue26:
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; CHECK:       pred.store.continue24:
 ; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP18]], i64 2
-; CHECK-NEXT:    br i1 [[TMP34]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
-; CHECK:       pred.store.if27:
+; CHECK-NEXT:    br i1 [[TMP34]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; CHECK:       pred.store.if25:
 ; CHECK-NEXT:    [[TMP35:%.*]] = add i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP35]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4
-; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[TMP35]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4
-; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[TMP35]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP39]], [[TMP37]]
 ; CHECK-NEXT:    store i32 [[TMP41]], ptr [[TMP40]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE28]]
-; CHECK:       pred.store.continue28:
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; CHECK:       pred.store.continue26:
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i1> [[TMP18]], i64 3
-; CHECK-NEXT:    br i1 [[TMP42]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
-; CHECK:       pred.store.if29:
+; CHECK-NEXT:    br i1 [[TMP42]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28]]
+; CHECK:       pred.store.if27:
 ; CHECK-NEXT:    [[TMP43:%.*]] = add i64 [[OFFSET_IDX]], 3
-; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[TMP43]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
-; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[TMP43]]
 ; CHECK-NEXT:    [[TMP47:%.*]] = load i32, ptr [[TMP46]], align 4
-; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP43]]
+; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[TMP43]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP47]], [[TMP45]]
 ; CHECK-NEXT:    store i32 [[TMP49]], ptr [[TMP48]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE30]]
-; CHECK:       pred.store.continue30:
-; CHECK-NEXT:    [[INDEX_NEXT31]] = add i64 [[INDEX20]], 4
-; CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT31]], [[N_VEC12]]
-; CHECK-NEXT:    br i1 [[TMP50]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY19]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; CHECK:       pred.store.continue28:
+; CHECK-NEXT:    [[INDEX_NEXT29]] = add nuw i64 [[INDEX16]], 4
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT29]], [[N_VEC12]]
+; CHECK-NEXT:    br i1 [[TMP50]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY15]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block7:
 ; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH8]]
 ; CHECK:       scalar.ph8:
@@ -269,60 +269,60 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588
 ; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[TMP2]], -1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT11]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE20:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE18:%.*]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[OFFSET_IDX8:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT13]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT14]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[OFFSET_IDX6:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT12]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[OFFSET_IDX8]]
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP9]], align 16
+; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[OFFSET_IDX6]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP7]], align 16
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[NEXT_GEP]], align 16
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
-; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
-; CHECK:       pred.store.if15:
+; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]
+; CHECK:       pred.store.if13:
 ; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4
-; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
-; CHECK-NEXT:    store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX6]], 4
+; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP8]], align 16
+; CHECK-NEXT:    store i32 [[TMP9]], ptr [[NEXT_GEP3]], align 16
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE14]]
+; CHECK:       pred.store.continue14:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
+; CHECK:       pred.store.if15:
+; CHECK-NEXT:    [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX6]], 8
+; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP9]], align 16
+; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP4]], align 16
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE16]]
 ; CHECK:       pred.store.continue16:
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
-; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18]]
 ; CHECK:       pred.store.if17:
-; CHECK-NEXT:    [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
-; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8
-; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16
-; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX6]], 12
+; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
+; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP5]], align 16
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE18]]
 ; CHECK:       pred.store.continue18:
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
-; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]]
-; CHECK:       pred.store.if19:
-; CHECK-NEXT:    [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
-; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12
-; CHECK-NEXT:    [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16
-; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE20]]
-; CHECK:       pred.store.continue20:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
@@ -363,11 +363,11 @@ define void @example23(ptr nocapture %src, ptr nocapture %dst) optsize {
 ; CHECK-NEXT:    [[DOT04:%.*]] = phi ptr [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[DOT013:%.*]] = phi ptr [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]
-; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds i8, ptr [[DOT04]], i64 2
+; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds nuw i8, ptr [[DOT04]], i64 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[DOT04]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7
-; CHECK-NEXT:    [[TMP6]] = getelementptr inbounds i8, ptr [[DOT013]], i64 4
+; CHECK-NEXT:    [[TMP6]] = getelementptr inbounds nuw i8, ptr [[DOT013]], i64 4
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[DOT013]], align 4
 ; CHECK-NEXT:    [[TMP7]] = add nuw nsw i32 [[I_02]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP7]], 256
@@ -406,12 +406,12 @@ define void @example23b(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[OFFSET_IDX4:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX4]]
+; CHECK-NEXT:    [[OFFSET_IDX2:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX2]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[NEXT_GEP]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw <4 x i32> [[TMP1]], splat (i32 7)
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[NEXT_GEP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -451,9 +451,9 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE17:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[OFFSET_IDX7:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX5:%.*]] = shl i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -461,54 +461,54 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX7]]
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX5]]
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[NEXT_GEP]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7
-; CHECK-NEXT:    store i32 [[TMP5]], ptr [[NEXT_GEP8]], align 4
+; CHECK-NEXT:    store i32 [[TMP5]], ptr [[NEXT_GEP6]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
-; CHECK:       pred.store.if12:
-; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
+; CHECK:       pred.store.if10:
+; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX5]], 4
+; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[NEXT_GEP4]], align 2
+; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[NEXT_GEP2]], align 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext i16 [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7
-; CHECK-NEXT:    store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
-; CHECK:       pred.store.continue13:
+; CHECK-NEXT:    store i32 [[TMP11]], ptr [[NEXT_GEP7]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE11]]
+; CHECK:       pred.store.continue11:
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; CHECK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
-; CHECK:       pred.store.if14:
-; CHECK-NEXT:    [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8
-; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
+; CHECK:       pred.store.if12:
+; CHECK-NEXT:    [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX5]], 8
+; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP5]], align 2
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP3]], align 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = zext i16 [[TMP15]] to i32
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
-; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE15]]
-; CHECK:       pred.store.continue15:
+; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP8]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
+; CHECK:       pred.store.continue13:
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; CHECK-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]]
-; CHECK:       pred.store.if16:
-; CHECK-NEXT:    [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12
-; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]]
+; CHECK:       pred.store.if14:
+; CHECK-NEXT:    [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX5]], 12
+; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6
-; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP6]], align 2
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP4]], align 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = zext i16 [[TMP21]] to i32
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7
-; CHECK-NEXT:    store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE17]]
-; CHECK:       pred.store.continue17:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    store i32 [[TMP23]], ptr [[NEXT_GEP9]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE15]]
+; CHECK:       pred.store.continue15:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
@@ -549,11 +549,11 @@ define i64 @example23d(ptr noalias nocapture %src, ptr noalias nocapture %dst) o
 ; CHECK-NEXT:    [[DOT04:%.*]] = phi ptr [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[DOT013:%.*]] = phi ptr [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ]
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]
-; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds i8, ptr [[DOT04]], i64 2
+; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds nuw i8, ptr [[DOT04]], i64 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[DOT04]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7
-; CHECK-NEXT:    [[TMP6]] = getelementptr inbounds i8, ptr [[DOT013]], i64 4
+; CHECK-NEXT:    [[TMP6]] = getelementptr inbounds nuw i8, ptr [[DOT013]], i64 4
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[DOT013]], align 4
 ; CHECK-NEXT:    [[TMP7]] = add nuw nsw i64 [[I_02]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP7]], 257
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 246cc1b345ad4..ad8f1fb3ccd21 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -186,8 +186,8 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[TMP171:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
@@ -223,7 +223,7 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[ADD7_LCSSA:%.*]] = phi i32 [ [[ADD7:%.*]], [[FOR_BODY1]] ], [ [[TMP149]], [[MIDDLE_BLOCK]] ], [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
@@ -418,8 +418,8 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; MAX-BW:       vec.epilog.iter.check:
 ; MAX-BW-NEXT:    br i1 false, label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; MAX-BW:       vec.epilog.ph:
-; MAX-BW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; MAX-BW-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; MAX-BW-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; MAX-BW-NEXT:    [[TMP171:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; MAX-BW-NEXT:    br label [[FOR_BODY:%.*]]
 ; MAX-BW:       vec.epilog.vector.body:
@@ -455,7 +455,7 @@ define i32 @matrix_row_col(ptr nocapture readonly %data, i32 %i, i32 %j) local_u
 ; MAX-BW-NEXT:    br i1 true, label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
 ; MAX-BW:       vec.epilog.scalar.ph:
 ; MAX-BW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; MAX-BW-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ]
+; MAX-BW-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi i32 [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP149]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; MAX-BW-NEXT:    br label [[FOR_BODY1:%.*]]
 ; MAX-BW:       for.cond.cleanup:
 ; MAX-BW-NEXT:    [[ADD7_LCSSA:%.*]] = phi i32 [ [[ADD7:%.*]], [[FOR_BODY1]] ], [ [[TMP149]], [[MIDDLE_BLOCK]] ], [ [[TMP170]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
index de906598775eb..428ed94dadb89 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
@@ -28,7 +28,7 @@ define dso_local void @tail_folding_enabled(ptr noalias nocapture %A, ptr noalia
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP6]], ptr [[TMP8]], i32 4, <8 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 432
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -96,7 +96,7 @@ define dso_local void @tail_folding_disabled(ptr noalias nocapture %A, ptr noali
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP6]], ptr [[TMP8]], i32 4, <8 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 432
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index cf913a4fcc7a1..90ba702ed232e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -146,7 +146,7 @@ define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonl
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[TMP5]], i32 4, <8 x i1> [[TMP1]], <8 x float> poison), !llvm.access.group [[ACC_GRP7]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd fast <8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0(<8 x float> [[TMP6]], ptr [[TMP5]], i32 4, <8 x i1> [[TMP1]]), !llvm.access.group [[ACC_GRP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index 3af3ad9ab3cf5..844fe1873bc98 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -71,7 +71,7 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
 ; CHECK-NEXT:    call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr [[TMP13]], i32 1, <48 x i1> [[TMP15]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index 4174aa4e897bc..f29428c51c636 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -45,7 +45,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -55,7 +55,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -65,7 +65,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -75,7 +75,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -85,7 +85,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -95,7 +95,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -105,7 +105,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -115,7 +115,7 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -157,9 +157,9 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; ENABLED_MASKED_STRIDED:       if.then:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_09]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[MUL]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[MUL]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[IX_09]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[IX_09]]
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP5]], ptr [[ARRAYIDX3]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[FOR_INC]]
 ; ENABLED_MASKED_STRIDED:       for.inc:
@@ -218,7 +218,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -228,7 +228,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -238,7 +238,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -248,7 +248,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -258,7 +258,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -268,7 +268,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -278,7 +278,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -288,7 +288,7 @@ define dso_local void @masked_strided1_optsize(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -407,7 +407,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP5]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -417,7 +417,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP11]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -427,7 +427,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP17]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP17]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -437,7 +437,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP23]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP23]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -447,7 +447,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP29]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP29]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -457,7 +457,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP35]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP35]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -467,7 +467,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP41]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP41]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -477,7 +477,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP47]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP47]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -485,7 +485,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr [[TMP52]], i32 1, <8 x i1> [[TMP3]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -520,7 +520,7 @@ define dso_local void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr [[TMP6]], i32 1, <8 x i1> [[TMP4]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -689,7 +689,7 @@ define dso_local void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], [[PRED_LOAD_IF15]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr [[TMP52]], i32 1, <8 x i1> [[TMP3]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -724,7 +724,7 @@ define dso_local void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i8> [[WIDE_MASKED_VEC]], <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr [[TMP6]], i32 1, <8 x i1> [[TMP4]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -794,21 +794,21 @@ define dso_local void @unconditional_strided1_optsize(ptr noalias nocapture read
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP1]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP5]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP7]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP7]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP0]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP0]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP11]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP0]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP13]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP13]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP0]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP2]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP6]], align 1
@@ -840,7 +840,7 @@ define dso_local void @unconditional_strided1_optsize(ptr noalias nocapture read
 ; ENABLED_MASKED_STRIDED:       vector.body:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP0]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP0]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[TMP1]], i32 1, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <16 x i8> poison)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
@@ -911,7 +911,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -921,7 +921,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -931,7 +931,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -941,7 +941,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -951,7 +951,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -961,7 +961,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -971,7 +971,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -981,7 +981,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -989,7 +989,7 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], [[PRED_LOAD_IF13]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr [[TMP50]], i32 1, <8 x i1> [[TMP0]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -1014,14 +1014,14 @@ define dso_local void @unconditional_strided1_optsize_unknown_tc(ptr noalias noc
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IV:%.*]] = or disjoint <8 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[TMP2]], i32 1, <16 x i1> [[TMP3]], <16 x i8> poison)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr [[TMP4]], i32 1, <8 x i1> [[TMP0]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP5]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end:
@@ -1093,7 +1093,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -1103,7 +1103,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -1113,7 +1113,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -1123,7 +1123,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -1133,7 +1133,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -1143,7 +1143,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -1153,7 +1153,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -1163,7 +1163,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -1174,7 +1174,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP52]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -1184,7 +1184,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if17:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP58]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -1194,7 +1194,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if19:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP64]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -1204,7 +1204,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if21:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP70]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -1214,7 +1214,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if23:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP76]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -1224,7 +1224,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if25:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP82]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -1234,7 +1234,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if27:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP88]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -1244,7 +1244,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if29:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP94]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -1255,7 +1255,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP103]], ptr [[TMP102]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -1264,7 +1264,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if31:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP105]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP107]], ptr [[TMP106]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE32]]
@@ -1273,7 +1273,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if33:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP109]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP111]], ptr [[TMP110]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE34]]
@@ -1282,7 +1282,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if35:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP113]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP115]], ptr [[TMP114]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE36]]
@@ -1291,7 +1291,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if37:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP117]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP119]], ptr [[TMP118]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE38]]
@@ -1300,7 +1300,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if39:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP121]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP123]], ptr [[TMP122]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE40]]
@@ -1309,7 +1309,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if41:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP125]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP127]], ptr [[TMP126]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE42]]
@@ -1318,7 +1318,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if43:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP129]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP131]], ptr [[TMP130]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE44]]
@@ -1328,7 +1328,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP133]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if45:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP134]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP136]], ptr [[TMP135]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE46]]
@@ -1337,7 +1337,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP137]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if47:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP138]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP140]], ptr [[TMP139]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE48]]
@@ -1346,7 +1346,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP141]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if49:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP142]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP144]], ptr [[TMP143]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE50]]
@@ -1355,7 +1355,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP145]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if51:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP146]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP148]], ptr [[TMP147]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE52]]
@@ -1364,7 +1364,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP149]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if53:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP150]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP152]], ptr [[TMP151]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE54]]
@@ -1373,7 +1373,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP153]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if55:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP154]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP156]], ptr [[TMP155]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE56]]
@@ -1382,7 +1382,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP157]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if57:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP158]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP160]], ptr [[TMP159]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE58]]
@@ -1391,7 +1391,7 @@ define dso_local void @masked_strided2(ptr noalias nocapture readonly %p, ptr no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP161]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if59:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP162]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP164]], ptr [[TMP163]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE60]]
@@ -1498,7 +1498,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -1508,7 +1508,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -1518,7 +1518,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -1528,7 +1528,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -1538,7 +1538,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -1548,7 +1548,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -1558,7 +1558,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -1568,7 +1568,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -1579,7 +1579,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP52]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -1589,7 +1589,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if17:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP58]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -1599,7 +1599,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if19:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP64]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -1609,7 +1609,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if21:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP70]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -1619,7 +1619,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if23:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP76]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -1629,7 +1629,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if25:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP82]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -1639,7 +1639,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if27:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP88]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -1649,7 +1649,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if29:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP94]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -1660,7 +1660,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP103]], ptr [[TMP102]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -1669,7 +1669,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if31:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP105]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP107]], ptr [[TMP106]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE32]]
@@ -1678,7 +1678,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if33:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP109]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP111]], ptr [[TMP110]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE34]]
@@ -1687,7 +1687,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if35:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP113]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP115]], ptr [[TMP114]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE36]]
@@ -1696,7 +1696,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if37:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP117]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP119]], ptr [[TMP118]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE38]]
@@ -1705,7 +1705,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if39:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP121]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP123]], ptr [[TMP122]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE40]]
@@ -1714,7 +1714,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if41:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP125]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP127]], ptr [[TMP126]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE42]]
@@ -1723,7 +1723,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if43:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP129]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP131]], ptr [[TMP130]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE44]]
@@ -1733,7 +1733,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP133]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if45:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP134]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP136]], ptr [[TMP135]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE46]]
@@ -1742,7 +1742,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP137]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if47:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP138]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP140]], ptr [[TMP139]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE48]]
@@ -1751,7 +1751,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP141]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if49:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP142]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP144]], ptr [[TMP143]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE50]]
@@ -1760,7 +1760,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP145]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if51:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP146]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP148]], ptr [[TMP147]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE52]]
@@ -1769,7 +1769,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP149]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if53:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP150]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP152]], ptr [[TMP151]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE54]]
@@ -1778,7 +1778,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP153]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if55:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP154]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP156]], ptr [[TMP155]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE56]]
@@ -1787,7 +1787,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP157]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if57:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP158]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP160]], ptr [[TMP159]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE58]]
@@ -1796,7 +1796,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP161]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if59:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP162]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP164]], ptr [[TMP163]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE60]]
@@ -1823,7 +1823,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -1833,7 +1833,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if1:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -1843,7 +1843,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if3:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -1853,7 +1853,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if5:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -1863,7 +1863,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if7:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -1873,7 +1873,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if9:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -1883,7 +1883,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if11:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -1893,7 +1893,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if13:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -1904,7 +1904,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if15:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP52]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -1914,7 +1914,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if17:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP58]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -1924,7 +1924,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if19:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP64]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -1934,7 +1934,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if21:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP70]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -1944,7 +1944,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if23:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP76]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -1954,7 +1954,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if25:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP82]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -1964,7 +1964,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if27:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP88]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -1974,7 +1974,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.load.if29:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP94]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -1985,7 +1985,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP103]], ptr [[TMP102]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -1994,7 +1994,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if31:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP105]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP107]], ptr [[TMP106]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE32]]
@@ -2003,7 +2003,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if33:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP109]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP111]], ptr [[TMP110]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE34]]
@@ -2012,7 +2012,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if35:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP113]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP115]], ptr [[TMP114]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE36]]
@@ -2021,7 +2021,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if37:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP117]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP119]], ptr [[TMP118]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE38]]
@@ -2030,7 +2030,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if39:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP121]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP123]], ptr [[TMP122]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE40]]
@@ -2039,7 +2039,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if41:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP125]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP127]], ptr [[TMP126]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE42]]
@@ -2048,7 +2048,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if43:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP129]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP131]], ptr [[TMP130]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE44]]
@@ -2058,7 +2058,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP133]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if45:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP134]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP136]], ptr [[TMP135]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE46]]
@@ -2067,7 +2067,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP137]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if47:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP138]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP140]], ptr [[TMP139]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE48]]
@@ -2076,7 +2076,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP141]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if49:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP142]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP144]], ptr [[TMP143]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE50]]
@@ -2085,7 +2085,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP145]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if51:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP146]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP148]], ptr [[TMP147]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE52]]
@@ -2094,7 +2094,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP149]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if53:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP150]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP152]], ptr [[TMP151]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE54]]
@@ -2103,7 +2103,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP153]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if55:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP154]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP156]], ptr [[TMP155]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE56]]
@@ -2112,7 +2112,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP157]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if57:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP158]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP160]], ptr [[TMP159]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE58]]
@@ -2121,7 +2121,7 @@ define dso_local void @masked_strided2_reverse(ptr noalias nocapture readonly %p
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP161]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if59:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP162]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
 ; ENABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP164]], ptr [[TMP163]], align 1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE60]]
@@ -2218,7 +2218,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP5]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = insertelement <8 x i8> poison, i8 [[TMP7]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -2228,7 +2228,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP11]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP11]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[TMP13]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -2238,7 +2238,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP16]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP17]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP17]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[TMP19]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -2248,7 +2248,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP23]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP23]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = insertelement <8 x i8> [[TMP21]], i8 [[TMP25]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -2258,7 +2258,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP28]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP29]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP29]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP32:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[TMP31]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -2268,7 +2268,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP34]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP35]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP35]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP38:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP37]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -2278,7 +2278,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP40]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP41]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP41]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP44:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[TMP43]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -2288,7 +2288,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP46]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP47]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP47]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP50:%.*]] = insertelement <8 x i8> [[TMP45]], i8 [[TMP49]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -2299,7 +2299,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if17:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = extractelement <8 x i32> [[TMP52]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP54]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP54]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP56:%.*]] = load i8, ptr [[TMP55]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP57:%.*]] = insertelement <8 x i8> poison, i8 [[TMP56]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -2309,7 +2309,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP59]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if19:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP60:%.*]] = extractelement <8 x i32> [[TMP52]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP60]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP60]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP62:%.*]] = load i8, ptr [[TMP61]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP63:%.*]] = insertelement <8 x i8> [[TMP58]], i8 [[TMP62]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -2319,7 +2319,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP65]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if21:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP66:%.*]] = extractelement <8 x i32> [[TMP52]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP66]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP66]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP68:%.*]] = load i8, ptr [[TMP67]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP69:%.*]] = insertelement <8 x i8> [[TMP64]], i8 [[TMP68]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -2329,7 +2329,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP71]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if23:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP72:%.*]] = extractelement <8 x i32> [[TMP52]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP72]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP72]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP74:%.*]] = load i8, ptr [[TMP73]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP75:%.*]] = insertelement <8 x i8> [[TMP70]], i8 [[TMP74]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -2339,7 +2339,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if25:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP78:%.*]] = extractelement <8 x i32> [[TMP52]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP78]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP78]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP80:%.*]] = load i8, ptr [[TMP79]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP81:%.*]] = insertelement <8 x i8> [[TMP76]], i8 [[TMP80]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -2349,7 +2349,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP83]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if27:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP84:%.*]] = extractelement <8 x i32> [[TMP52]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP84]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP84]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP86:%.*]] = load i8, ptr [[TMP85]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP87:%.*]] = insertelement <8 x i8> [[TMP82]], i8 [[TMP86]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -2359,7 +2359,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP89]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if29:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP90:%.*]] = extractelement <8 x i32> [[TMP52]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP90]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP90]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP92:%.*]] = load i8, ptr [[TMP91]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP93:%.*]] = insertelement <8 x i8> [[TMP88]], i8 [[TMP92]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -2369,7 +2369,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP95]], label [[PRED_LOAD_IF31:%.*]], label [[PRED_LOAD_CONTINUE32:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if31:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP96:%.*]] = extractelement <8 x i32> [[TMP52]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP96]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP96]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP98:%.*]] = load i8, ptr [[TMP97]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP99:%.*]] = insertelement <8 x i8> [[TMP94]], i8 [[TMP98]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE32]]
@@ -2380,7 +2380,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP102]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP103:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP103]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP104:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP103]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP105:%.*]] = extractelement <8 x i8> [[TMP101]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP105]], ptr [[TMP104]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -2389,7 +2389,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP106]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if33:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP107:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP108:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP107]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP108:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP107]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP109:%.*]] = extractelement <8 x i8> [[TMP101]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP109]], ptr [[TMP108]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE34]]
@@ -2398,7 +2398,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP110]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if35:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP111:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP112:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP111]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP112:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP111]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP113:%.*]] = extractelement <8 x i8> [[TMP101]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP113]], ptr [[TMP112]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE36]]
@@ -2407,7 +2407,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP114]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if37:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP115:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP116:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP115]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP116:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP115]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP117:%.*]] = extractelement <8 x i8> [[TMP101]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP117]], ptr [[TMP116]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE38]]
@@ -2416,7 +2416,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP118]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if39:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP119:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP120:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP119]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP120:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP119]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP121:%.*]] = extractelement <8 x i8> [[TMP101]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP121]], ptr [[TMP120]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE40]]
@@ -2425,7 +2425,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP122]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if41:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP123:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP124:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP123]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP124:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP123]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP125:%.*]] = extractelement <8 x i8> [[TMP101]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP125]], ptr [[TMP124]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE42]]
@@ -2434,7 +2434,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP126]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if43:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP127:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP128:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP127]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP128:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP127]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP129:%.*]] = extractelement <8 x i8> [[TMP101]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP129]], ptr [[TMP128]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE44]]
@@ -2443,7 +2443,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP130]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if45:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP131:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP132:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP131]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP132:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP131]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP133:%.*]] = extractelement <8 x i8> [[TMP101]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP133]], ptr [[TMP132]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE46]]
@@ -2453,7 +2453,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP135]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if47:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP136:%.*]] = extractelement <8 x i32> [[TMP52]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP137:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP136]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP137:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP136]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP138:%.*]] = extractelement <8 x i8> [[TMP134]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP138]], ptr [[TMP137]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE48]]
@@ -2462,7 +2462,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP139]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if49:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP140:%.*]] = extractelement <8 x i32> [[TMP52]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP141:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP140]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP141:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP140]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP142:%.*]] = extractelement <8 x i8> [[TMP134]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP142]], ptr [[TMP141]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE50]]
@@ -2471,7 +2471,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP143]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if51:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP144:%.*]] = extractelement <8 x i32> [[TMP52]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP145:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP144]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP145:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP144]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP146:%.*]] = extractelement <8 x i8> [[TMP134]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP146]], ptr [[TMP145]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE52]]
@@ -2480,7 +2480,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP147]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if53:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP148:%.*]] = extractelement <8 x i32> [[TMP52]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP149:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP148]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP149:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP148]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP150:%.*]] = extractelement <8 x i8> [[TMP134]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP150]], ptr [[TMP149]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE54]]
@@ -2489,7 +2489,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP151]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if55:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP152:%.*]] = extractelement <8 x i32> [[TMP52]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP153:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP152]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP153:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP152]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP154:%.*]] = extractelement <8 x i8> [[TMP134]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP154]], ptr [[TMP153]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE56]]
@@ -2498,7 +2498,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP155]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if57:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP156:%.*]] = extractelement <8 x i32> [[TMP52]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP157:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP156]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP157:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP156]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = extractelement <8 x i8> [[TMP134]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP158]], ptr [[TMP157]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE58]]
@@ -2507,7 +2507,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP159]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if59:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = extractelement <8 x i32> [[TMP52]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP161:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP160]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP161:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP160]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP162:%.*]] = extractelement <8 x i8> [[TMP134]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP162]], ptr [[TMP161]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE60]]
@@ -2516,12 +2516,12 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP163]], label [[PRED_STORE_IF61:%.*]], label [[PRED_STORE_CONTINUE62]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if61:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP164:%.*]] = extractelement <8 x i32> [[TMP52]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP165:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP164]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP165:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP164]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP166:%.*]] = extractelement <8 x i8> [[TMP134]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP166]], ptr [[TMP165]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE62]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue62:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP167]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -2558,7 +2558,7 @@ define dso_local void @masked_strided2_unknown_tc(ptr noalias nocapture readonly
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = sub <8 x i8> zeroinitializer, [[TMP6]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr [[TMP8]], i32 1, <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP10]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -2652,7 +2652,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP3]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> poison, i8 [[TMP5]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
@@ -2662,7 +2662,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP9]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP9]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[TMP11]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
@@ -2672,7 +2672,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP15]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[TMP17]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
@@ -2682,7 +2682,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = insertelement <8 x i8> [[TMP19]], i8 [[TMP23]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
@@ -2692,7 +2692,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP26]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if7:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = load i8, ptr [[TMP28]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[TMP29]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE8]]
@@ -2702,7 +2702,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if9:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP33]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP33]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[TMP35]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE10]]
@@ -2712,7 +2712,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP38]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if11:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP39]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP39]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP42:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[TMP41]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
@@ -2722,7 +2722,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if13:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP45:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP45]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP45]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP48:%.*]] = insertelement <8 x i8> [[TMP43]], i8 [[TMP47]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
@@ -2733,7 +2733,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP52]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP52]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = load i8, ptr [[TMP53]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP55:%.*]] = insertelement <8 x i8> poison, i8 [[TMP54]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
@@ -2743,7 +2743,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP57]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if17:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP58:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP58]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP58]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP60:%.*]] = load i8, ptr [[TMP59]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP61:%.*]] = insertelement <8 x i8> [[TMP56]], i8 [[TMP60]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE18]]
@@ -2753,7 +2753,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP63]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if19:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP64:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP64]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP64]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP66:%.*]] = load i8, ptr [[TMP65]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP67:%.*]] = insertelement <8 x i8> [[TMP62]], i8 [[TMP66]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE20]]
@@ -2763,7 +2763,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP69]], label [[PRED_LOAD_IF21:%.*]], label [[PRED_LOAD_CONTINUE22:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if21:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP70:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP70]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP71:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP70]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP72:%.*]] = load i8, ptr [[TMP71]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP73:%.*]] = insertelement <8 x i8> [[TMP68]], i8 [[TMP72]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE22]]
@@ -2773,7 +2773,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP75]], label [[PRED_LOAD_IF23:%.*]], label [[PRED_LOAD_CONTINUE24:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if23:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP76:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP76]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP77:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP76]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP78:%.*]] = load i8, ptr [[TMP77]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP79:%.*]] = insertelement <8 x i8> [[TMP74]], i8 [[TMP78]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE24]]
@@ -2783,7 +2783,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP81]], label [[PRED_LOAD_IF25:%.*]], label [[PRED_LOAD_CONTINUE26:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if25:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP82:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP82]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP82]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP84:%.*]] = load i8, ptr [[TMP83]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP85:%.*]] = insertelement <8 x i8> [[TMP80]], i8 [[TMP84]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE26]]
@@ -2793,7 +2793,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP87]], label [[PRED_LOAD_IF27:%.*]], label [[PRED_LOAD_CONTINUE28:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if27:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP88:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP88]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP89:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP88]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP91:%.*]] = insertelement <8 x i8> [[TMP86]], i8 [[TMP90]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE28]]
@@ -2803,7 +2803,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP93]], label [[PRED_LOAD_IF29:%.*]], label [[PRED_LOAD_CONTINUE30:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.load.if29:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP94:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP94]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP95:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i32 [[TMP94]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP96:%.*]] = load i8, ptr [[TMP95]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP97:%.*]] = insertelement <8 x i8> [[TMP92]], i8 [[TMP96]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE30]]
@@ -2814,7 +2814,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP100]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP101:%.*]] = extractelement <8 x i32> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP101]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP102:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP101]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP103:%.*]] = extractelement <8 x i8> [[TMP99]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP103]], ptr [[TMP102]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -2823,7 +2823,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP104]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if31:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP105:%.*]] = extractelement <8 x i32> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP105]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP106:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP105]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP107:%.*]] = extractelement <8 x i8> [[TMP99]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP107]], ptr [[TMP106]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE32]]
@@ -2832,7 +2832,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP108]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if33:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP109:%.*]] = extractelement <8 x i32> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP109]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP109]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP111:%.*]] = extractelement <8 x i8> [[TMP99]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP111]], ptr [[TMP110]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE34]]
@@ -2841,7 +2841,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP112]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if35:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP113:%.*]] = extractelement <8 x i32> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP113]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP114:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP113]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP115:%.*]] = extractelement <8 x i8> [[TMP99]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP115]], ptr [[TMP114]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE36]]
@@ -2850,7 +2850,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP116]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if37:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP117:%.*]] = extractelement <8 x i32> [[TMP1]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP117]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP118:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP117]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP119:%.*]] = extractelement <8 x i8> [[TMP99]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP119]], ptr [[TMP118]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE38]]
@@ -2859,7 +2859,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP120]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if39:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP121:%.*]] = extractelement <8 x i32> [[TMP1]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP121]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP121]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP123:%.*]] = extractelement <8 x i8> [[TMP99]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP123]], ptr [[TMP122]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE40]]
@@ -2868,7 +2868,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP124]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if41:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP125:%.*]] = extractelement <8 x i32> [[TMP1]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP125]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP126:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP125]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP127:%.*]] = extractelement <8 x i8> [[TMP99]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP127]], ptr [[TMP126]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE42]]
@@ -2877,7 +2877,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP128]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if43:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP129:%.*]] = extractelement <8 x i32> [[TMP1]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP129]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP130:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP129]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP131:%.*]] = extractelement <8 x i8> [[TMP99]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP131]], ptr [[TMP130]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE44]]
@@ -2887,7 +2887,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP133]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if45:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP134:%.*]] = extractelement <8 x i32> [[TMP50]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP134]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP135:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP134]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP136:%.*]] = extractelement <8 x i8> [[TMP132]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP136]], ptr [[TMP135]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE46]]
@@ -2896,7 +2896,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP137]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if47:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP138:%.*]] = extractelement <8 x i32> [[TMP50]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP138]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP139:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP138]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP140:%.*]] = extractelement <8 x i8> [[TMP132]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP140]], ptr [[TMP139]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE48]]
@@ -2905,7 +2905,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP141]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if49:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP142:%.*]] = extractelement <8 x i32> [[TMP50]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP142]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP142]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP144:%.*]] = extractelement <8 x i8> [[TMP132]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP144]], ptr [[TMP143]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE50]]
@@ -2914,7 +2914,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP145]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if51:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP146:%.*]] = extractelement <8 x i32> [[TMP50]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP146]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP147:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP146]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP148:%.*]] = extractelement <8 x i8> [[TMP132]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP148]], ptr [[TMP147]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE52]]
@@ -2923,7 +2923,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP149]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if53:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP150:%.*]] = extractelement <8 x i32> [[TMP50]], i64 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP150]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP151:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP150]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP152:%.*]] = extractelement <8 x i8> [[TMP132]], i64 4
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP152]], ptr [[TMP151]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE54]]
@@ -2932,7 +2932,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP153]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if55:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP154:%.*]] = extractelement <8 x i32> [[TMP50]], i64 5
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP154]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP155:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP154]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP156:%.*]] = extractelement <8 x i8> [[TMP132]], i64 5
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP156]], ptr [[TMP155]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE56]]
@@ -2941,7 +2941,7 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP157]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if57:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = extractelement <8 x i32> [[TMP50]], i64 6
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP158]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP158]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = extractelement <8 x i8> [[TMP132]], i64 6
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP160]], ptr [[TMP159]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE58]]
@@ -2950,12 +2950,12 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP161]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if59:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP162:%.*]] = extractelement <8 x i32> [[TMP50]], i64 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP162]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP163:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i32 [[TMP162]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP164:%.*]] = extractelement <8 x i8> [[TMP132]], i64 7
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP164]], ptr [[TMP163]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE60]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue60:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP165]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -2980,17 +2980,17 @@ define dso_local void @unconditional_masked_strided2_unknown_tc(ptr noalias noca
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IV:%.*]] = or disjoint <8 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[P:%.*]], i32 [[TMP1]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[TMP2]], i32 1, <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8> [[STRIDED_VEC3]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[Q:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q:%.*]], i32 [[TMP1]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP4]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr [[TMP6]], i32 1, <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index bd54f87b50489..414394a8942e5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -29,13 +29,13 @@ define dso_local void @test1(ptr noalias nocapture %points, ptr noalias nocaptur
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 2)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP2]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS:%.*]], i64 [[TMP2]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP4]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP6]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP6]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP8]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP3]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
@@ -48,13 +48,13 @@ define dso_local void @test1(ptr noalias nocapture %points, ptr noalias nocaptur
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP14]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = or disjoint <4 x i64> [[TMP1]], splat (i64 1)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP15]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP16]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i64> [[TMP15]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP18]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP18]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP15]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP20]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP20]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP15]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP22]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP22]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP24]], ptr [[TMP17]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 1
@@ -217,7 +217,7 @@ define dso_local void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP36]], ptr [[TMP35]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE15]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue15:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP37]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -254,7 +254,7 @@ define dso_local void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[GEP]], i32 2, <16 x i1> [[TMP5]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP6]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end.loopexit:
@@ -319,7 +319,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -328,7 +328,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if1:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP8]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
@@ -337,7 +337,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if3:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP12]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
@@ -346,7 +346,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if5:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP16]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
@@ -372,7 +372,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
@@ -381,7 +381,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if1:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP8]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
 ; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
@@ -390,7 +390,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if3:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP12]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP12]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
 ; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
@@ -399,7 +399,7 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if5:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i16, ptr [[POINTS]], i64 [[TMP16]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
 ; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll b/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
index 921cf4246f725..050a866d8ca33 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
@@ -14,8 +14,8 @@ define void @example() nounwind ssp uwtable {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x x86_fp80], ptr @x, i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1024 x x86_fp80], ptr @x, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [1024 x x86_fp80], ptr @x, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [1024 x x86_fp80], ptr @x, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP1]], align 16
 ; CHECK-NEXT:    store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP2]], align 16
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/branch-weights.ll b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
index d5f1b46bd5421..6e7efe018faf6 100644
--- a/llvm/test/Transforms/LoopVectorize/branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
@@ -33,7 +33,7 @@
 ; CHECK:   br i1 {{.+}}, label %vec.epilog.middle.block, label %vec.epilog.vector.body, !prof [[PROF_F0_VEC_EPILOG_VECTOR_BODY:![0-9]+]]
 ;
 ; CHECK: vec.epilog.middle.block:
-; CHECK:   br i1 %cmp.n11, label %exit.loopexit, label %vec.epilog.scalar.ph, !prof [[PROF_F0_MIDDLE_BLOCKS:![0-9]+]]
+; CHECK:   br i1 %cmp.n9, label %exit.loopexit, label %vec.epilog.scalar.ph, !prof [[PROF_F0_MIDDLE_BLOCKS:![0-9]+]]
 ;
 ; CHECK: vec.epilog.scalar.ph:
 ; CHECK:   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
index c2edb54f6f3b5..e1fd07bf590c4 100644
--- a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
@@ -91,7 +91,7 @@ define void @pr47390(ptr %a) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
index 8afde74cf277d..47a6312ab2f29 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
@@ -40,8 +40,8 @@ define i32 @any_of_reduction_epilog(ptr %src, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
@@ -68,7 +68,7 @@ define i32 @any_of_reduction_epilog(ptr %src, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -139,8 +139,8 @@ define i32 @any_of_reduction_epilog_arg_as_start_value(ptr %src, i64 %N, i32 %st
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], [[START]]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
@@ -167,7 +167,7 @@ define i32 @any_of_reduction_epilog_arg_as_start_value(ptr %src, i64 %N, i32 %st
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START]], [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[RDX_SELECT9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -214,9 +214,9 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -241,9 +241,9 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], false
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
@@ -276,7 +276,7 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ false, [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -377,8 +377,8 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ true, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ true, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP48:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], true
 ; CHECK-NEXT:    [[N_MOD_VF7:%.*]] = urem i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[N_VEC8:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF7]]
@@ -427,7 +427,7 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
 ; CHECK-NEXT:    br i1 [[CMP_N10]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX23:%.*]] = phi i1 [ [[RDX_SELECT22]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ true, [[ITER_CHECK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX23:%.*]] = phi i1 [ [[RDX_SELECT22]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ true, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[RED:%.*]] = phi i1 [ [[BC_MERGE_RDX23]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 3c1dd1bd8b6d1..f12aab5079ed9 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -36,8 +36,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
@@ -59,7 +59,7 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i64 [ [[TMP12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 5, [[ITER_CHECK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i64 [ [[TMP12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -127,8 +127,8 @@ define float @fp_reduction_max(ptr noalias %a, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[MINMAX_IDENT_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[BC_MERGE_RDX]], i64 0
@@ -152,7 +152,7 @@ define float @fp_reduction_max(ptr noalias %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi float [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi float [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -220,8 +220,8 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    br i1 true, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 256, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
@@ -246,7 +246,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 256, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 256, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
@@ -322,9 +322,9 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.500000e+01, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_MERGE_RDX3:%.*]] = phi float [ [[TMP7]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+01, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> splat (float 1.000000e+00), float [[BC_MERGE_RDX]], i32 0
@@ -350,8 +350,8 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N11]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1.500000e+01, [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi float [ [[TMP17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1.000000e+01, [[ITER_CHECK]] ], [ [[TMP7]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.500000e+01, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX13:%.*]] = phi float [ [[TMP17]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+01, [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -430,8 +430,8 @@ define i32 @reduction_phi_start_val(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_SUM]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_SUM]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
@@ -453,7 +453,7 @@ define i32 @reduction_phi_start_val(ptr %A, i64 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_COND]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[START_SUM]], [[ITER_CHECK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START_SUM]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
index 7a92d1a1c9ea5..1854acf0ec2ba 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
@@ -63,7 +63,7 @@ define void @trunc_iv_steps_with_epilogue(ptr %A, i64 %N) {
 ; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV_I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll b/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
index f648de1358998..93ac34003d4b7 100644
--- a/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
+++ b/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
@@ -38,7 +38,7 @@ define void @inv_store_last_lane(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[INV:%.*]], i64 168
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[INV:%.*]], i64 168
 ; CHECK-NEXT:    store i32 [[MUL_LCSSA]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 517de8be5c998..d2d2063fd9058 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -39,9 +39,6 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<exit>
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
 ; CHECK-NEXT:    EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
 ; CHECK-NEXT:    EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
@@ -53,6 +50,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond.not = icmp eq i64 %iv.next, 1000
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<exit>
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -115,9 +115,6 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<exit>
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
 ; CHECK-NEXT:    EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
 ; CHECK-NEXT:    EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
@@ -131,6 +128,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond.not = icmp eq i64 %iv.next, 1000
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<exit>
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -195,9 +195,6 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_C]]>
 ; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<ret>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT:   EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
 ; CHECK-NEXT:   EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
@@ -208,6 +205,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
 ; CHECK-NEXT:   IR   %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]> from scalar.ph)
 ; CHECK-NEXT:   IR   %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1 from scalar.ph)
 ; CHECK:     No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -269,9 +269,6 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_C]]>
 ; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<ret>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT:   EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
 ; CHECK-NEXT:   EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
@@ -282,6 +279,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
 ; CHECK-NEXT:   IR   %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]> from scalar.ph)
 ; CHECK-NEXT:   IR   %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1 from scalar.ph)
 ; CHECK:     No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
index 0f206bfb92fa0..bd3688d852d0f 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
@@ -853,7 +853,7 @@ define void @sink_dominance(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
@@ -936,7 +936,7 @@ define void @sink_dominance_2(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index d0c811763a522..3d8403b1eb9c0 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -66,7 +66,7 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
 ; CHECK-NEXT: Successor(s): loop.1
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -77,9 +77,6 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -89,6 +86,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
 ; CHECK-NEXT:   IR   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %ec = icmp eq i32 %iv.next, 20001
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -153,7 +153,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT: Successor(s): loop.0
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.0:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -164,9 +164,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -176,6 +173,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   IR   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %ec = icmp eq i32 %iv.next, 20001
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -221,7 +221,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   WIDEN ir<%add> = add ir<%rem>, ir<%recur.next>
 ; CHECK-NEXT:   WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
 ; CHECK-NEXT:   EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -234,10 +234,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:   IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT:   EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234>
@@ -249,6 +245,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT:   IR   %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ]
 ; CHECK:        IR   %ec = icmp eq i32 %iv.next, 20001
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT:   IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -336,7 +336,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
 ; CHECK-NEXT:   Successor(s): loop.2
 ; CHECK-EMPTY:
 ; CHECK:      loop.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -347,9 +347,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -359,6 +356,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
 ; CHECK-NEXT:   IR   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %ec = icmp eq i32 %iv.next, 20001
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -433,7 +433,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
 ; CHECK-NEXT: Successor(s): loop.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -444,9 +444,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -456,6 +453,9 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
 ; CHECK-NEXT:   IR   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:        IR   %C = icmp sgt i32 %iv.next, %recur.next
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -519,7 +519,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
 ; CHECK-NEXT:   Successor(s): loop.0
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   loop.0:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
@@ -530,9 +530,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -542,6 +539,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
 ; CHECK-NEXT:   IR   %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph)
 ; CHECK:        IR   %ec = icmp ugt i64 %iv, 3
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 6c94f5bfc836a..2d50f8219549a 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2780,7 +2780,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-IC-NEXT:    [[TMP47]] = add <4 x i32> [[VEC_PHI1]], [[TMP45]]
 ; UNROLL-NO-IC-NEXT:    [[TMP48:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]]
 ; UNROLL-NO-IC-NEXT:    [[TMP49:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI1]]
-; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP50:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27:![0-9]+]], !llvm.loop [[LOOP28:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
@@ -2847,7 +2847,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-VF-NEXT:    [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
 ; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP10]], i32 [[VEC_PHI]]
 ; UNROLL-NO-VF-NEXT:    [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP11]], i32 [[VEC_PHI1]]
-; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27:![0-9]+]], !llvm.loop [[LOOP28:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
@@ -2934,7 +2934,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; SINK-AFTER-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]]
 ; SINK-AFTER-NEXT:    [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]]
-; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27:![0-9]+]], !llvm.loop [[LOOP28:![0-9]+]]
 ; SINK-AFTER:       middle.block:
@@ -2996,11 +2996,11 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-IC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
-; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE30]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_STORE_CONTINUE30]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_STORE_CONTINUE30]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_STORE_CONTINUE30]] ]
+; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE29:%.*]] ]
+; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE29]] ]
+; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_STORE_CONTINUE29]] ]
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_STORE_CONTINUE29]] ]
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_STORE_CONTINUE29]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0
@@ -3022,65 +3022,65 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-IC:       pred.udiv.continue:
 ; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP14]], [[PRED_UDIV_IF]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP16]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if3:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP16]], label [[PRED_UDIV_IF2:%.*]], label [[PRED_UDIV_CONTINUE3:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if2:
 ; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = udiv i32 219220132, [[TMP3]]
 ; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP17]], i32 1
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-IC:       pred.udiv.continue4:
-; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = phi <4 x i32> [ [[TMP15]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP18]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE3]]
+; UNROLL-NO-IC:       pred.udiv.continue3:
+; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = phi <4 x i32> [ [[TMP15]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP18]], [[PRED_UDIV_IF2]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP20]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if5:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP20]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if4:
 ; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = udiv i32 219220132, [[TMP4]]
 ; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP21]], i32 2
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
-; UNROLL-NO-IC:       pred.udiv.continue6:
-; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP22]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE5]]
+; UNROLL-NO-IC:       pred.udiv.continue5:
+; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_UDIV_CONTINUE3]] ], [ [[TMP22]], [[PRED_UDIV_IF4]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP24:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP24]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if7:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP24]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if6:
 ; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = udiv i32 219220132, [[TMP5]]
 ; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP25]], i32 3
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
-; UNROLL-NO-IC:       pred.udiv.continue8:
-; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = phi <4 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP26]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE7]]
+; UNROLL-NO-IC:       pred.udiv.continue7:
+; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = phi <4 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP26]], [[PRED_UDIV_IF6]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP28]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if9:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP28]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if8:
 ; UNROLL-NO-IC-NEXT:    [[TMP29:%.*]] = udiv i32 219220132, [[TMP6]]
 ; UNROLL-NO-IC-NEXT:    [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP29]], i32 0
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE10]]
-; UNROLL-NO-IC:       pred.udiv.continue10:
-; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[TMP30]], [[PRED_UDIV_IF9]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE9]]
+; UNROLL-NO-IC:       pred.udiv.continue9:
+; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE7]] ], [ [[TMP30]], [[PRED_UDIV_IF8]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP32]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if11:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP32]], label [[PRED_UDIV_IF10:%.*]], label [[PRED_UDIV_CONTINUE11:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if10:
 ; UNROLL-NO-IC-NEXT:    [[TMP33:%.*]] = udiv i32 219220132, [[TMP7]]
 ; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP33]], i32 1
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE12]]
-; UNROLL-NO-IC:       pred.udiv.continue12:
-; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP34]], [[PRED_UDIV_IF11]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE11]]
+; UNROLL-NO-IC:       pred.udiv.continue11:
+; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE9]] ], [ [[TMP34]], [[PRED_UDIV_IF10]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP36:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP36]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if13:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP36]], label [[PRED_UDIV_IF12:%.*]], label [[PRED_UDIV_CONTINUE13:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if12:
 ; UNROLL-NO-IC-NEXT:    [[TMP37:%.*]] = udiv i32 219220132, [[TMP8]]
 ; UNROLL-NO-IC-NEXT:    [[TMP38:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP37]], i32 2
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE14]]
-; UNROLL-NO-IC:       pred.udiv.continue14:
-; UNROLL-NO-IC-NEXT:    [[TMP39:%.*]] = phi <4 x i32> [ [[TMP35]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP38]], [[PRED_UDIV_IF13]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE13]]
+; UNROLL-NO-IC:       pred.udiv.continue13:
+; UNROLL-NO-IC-NEXT:    [[TMP39:%.*]] = phi <4 x i32> [ [[TMP35]], [[PRED_UDIV_CONTINUE11]] ], [ [[TMP38]], [[PRED_UDIV_IF12]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP40]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; UNROLL-NO-IC:       pred.udiv.if15:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP40]], label [[PRED_UDIV_IF14:%.*]], label [[PRED_UDIV_CONTINUE15:%.*]]
+; UNROLL-NO-IC:       pred.udiv.if14:
 ; UNROLL-NO-IC-NEXT:    [[TMP41:%.*]] = udiv i32 219220132, [[TMP9]]
 ; UNROLL-NO-IC-NEXT:    [[TMP42:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP41]], i32 3
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE16]]
-; UNROLL-NO-IC:       pred.udiv.continue16:
-; UNROLL-NO-IC-NEXT:    [[TMP43]] = phi <4 x i32> [ [[TMP39]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP42]], [[PRED_UDIV_IF15]] ]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE15]]
+; UNROLL-NO-IC:       pred.udiv.continue15:
+; UNROLL-NO-IC-NEXT:    [[TMP43]] = phi <4 x i32> [ [[TMP39]], [[PRED_UDIV_CONTINUE13]] ], [ [[TMP42]], [[PRED_UDIV_IF14]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP27]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]]
-; UNROLL-NO-IC-NEXT:    [[TMP47]] = add <4 x i32> [[VEC_PHI2]], [[TMP45]]
+; UNROLL-NO-IC-NEXT:    [[TMP47]] = add <4 x i32> [[VEC_PHI1]], [[TMP45]]
 ; UNROLL-NO-IC-NEXT:    [[TMP48:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP48]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; UNROLL-NO-IC:       pred.store.if:
@@ -3090,64 +3090,64 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NO-IC:       pred.store.continue:
 ; UNROLL-NO-IC-NEXT:    [[TMP51:%.*]] = extractelement <4 x i1> [[TMP10]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP51]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
-; UNROLL-NO-IC:       pred.store.if17:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP51]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]]
+; UNROLL-NO-IC:       pred.store.if16:
 ; UNROLL-NO-IC-NEXT:    [[TMP52:%.*]] = add i32 [[INDEX]], 1
 ; UNROLL-NO-IC-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP52]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP3]], ptr [[TMP53]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE18]]
-; UNROLL-NO-IC:       pred.store.continue18:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE17]]
+; UNROLL-NO-IC:       pred.store.continue17:
 ; UNROLL-NO-IC-NEXT:    [[TMP54:%.*]] = extractelement <4 x i1> [[TMP10]], i32 2
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP54]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
-; UNROLL-NO-IC:       pred.store.if19:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP54]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19:%.*]]
+; UNROLL-NO-IC:       pred.store.if18:
 ; UNROLL-NO-IC-NEXT:    [[TMP55:%.*]] = add i32 [[INDEX]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP55]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP4]], ptr [[TMP56]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE20]]
-; UNROLL-NO-IC:       pred.store.continue20:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE19]]
+; UNROLL-NO-IC:       pred.store.continue19:
 ; UNROLL-NO-IC-NEXT:    [[TMP57:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP57]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
-; UNROLL-NO-IC:       pred.store.if21:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP57]], label [[PRED_STORE_IF20:%.*]], label [[PRED_STORE_CONTINUE21:%.*]]
+; UNROLL-NO-IC:       pred.store.if20:
 ; UNROLL-NO-IC-NEXT:    [[TMP58:%.*]] = add i32 [[INDEX]], 3
 ; UNROLL-NO-IC-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP58]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP5]], ptr [[TMP59]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE22]]
-; UNROLL-NO-IC:       pred.store.continue22:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE21]]
+; UNROLL-NO-IC:       pred.store.continue21:
 ; UNROLL-NO-IC-NEXT:    [[TMP60:%.*]] = extractelement <4 x i1> [[TMP11]], i32 0
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP60]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
-; UNROLL-NO-IC:       pred.store.if23:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP60]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]]
+; UNROLL-NO-IC:       pred.store.if22:
 ; UNROLL-NO-IC-NEXT:    [[TMP61:%.*]] = add i32 [[INDEX]], 4
 ; UNROLL-NO-IC-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP61]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP6]], ptr [[TMP62]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE24]]
-; UNROLL-NO-IC:       pred.store.continue24:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE23]]
+; UNROLL-NO-IC:       pred.store.continue23:
 ; UNROLL-NO-IC-NEXT:    [[TMP63:%.*]] = extractelement <4 x i1> [[TMP11]], i32 1
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP63]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
-; UNROLL-NO-IC:       pred.store.if25:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP63]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]]
+; UNROLL-NO-IC:       pred.store.if24:
 ; UNROLL-NO-IC-NEXT:    [[TMP64:%.*]] = add i32 [[INDEX]], 5
 ; UNROLL-NO-IC-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP64]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP7]], ptr [[TMP65]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE26]]
-; UNROLL-NO-IC:       pred.store.continue26:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE25]]
+; UNROLL-NO-IC:       pred.store.continue25:
 ; UNROLL-NO-IC-NEXT:    [[TMP66:%.*]] = extractelement <4 x i1> [[TMP11]], i32 2
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP66]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
-; UNROLL-NO-IC:       pred.store.if27:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP66]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]]
+; UNROLL-NO-IC:       pred.store.if26:
 ; UNROLL-NO-IC-NEXT:    [[TMP67:%.*]] = add i32 [[INDEX]], 6
 ; UNROLL-NO-IC-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP67]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP8]], ptr [[TMP68]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE28]]
-; UNROLL-NO-IC:       pred.store.continue28:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE27]]
+; UNROLL-NO-IC:       pred.store.continue27:
 ; UNROLL-NO-IC-NEXT:    [[TMP69:%.*]] = extractelement <4 x i1> [[TMP11]], i32 3
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP69]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
-; UNROLL-NO-IC:       pred.store.if29:
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP69]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29]]
+; UNROLL-NO-IC:       pred.store.if28:
 ; UNROLL-NO-IC-NEXT:    [[TMP70:%.*]] = add i32 [[INDEX]], 7
 ; UNROLL-NO-IC-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP70]]
 ; UNROLL-NO-IC-NEXT:    store i32 [[TMP9]], ptr [[TMP71]], align 4
-; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE30]]
-; UNROLL-NO-IC:       pred.store.continue30:
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE29]]
+; UNROLL-NO-IC:       pred.store.continue29:
 ; UNROLL-NO-IC-NEXT:    [[TMP72:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]]
-; UNROLL-NO-IC-NEXT:    [[TMP73:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI2]]
-; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; UNROLL-NO-IC-NEXT:    [[TMP73:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI1]]
+; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
 ; UNROLL-NO-IC-NEXT:    [[TMP74:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27]], !llvm.loop [[LOOP32:![0-9]+]]
@@ -3193,31 +3193,31 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-VF-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP1]], 1
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
-; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE7:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_STORE_CONTINUE7]] ]
-; UNROLL-NO-VF-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[PRED_STORE_CONTINUE7]] ]
-; UNROLL-NO-VF-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE7]] ]
+; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_STORE_CONTINUE6]] ]
+; UNROLL-NO-VF-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[PRED_STORE_CONTINUE6]] ]
+; UNROLL-NO-VF-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE6]] ]
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
 ; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0
 ; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
 ; UNROLL-NO-VF-NEXT:    [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
-; UNROLL-NO-VF-NEXT:    [[VEC_IV3:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NO-VF-NEXT:    [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
 ; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = icmp ule i32 [[VEC_IV3]], [[TRIP_COUNT_MINUS_1]]
+; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = icmp ule i32 [[VEC_IV2]], [[TRIP_COUNT_MINUS_1]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; UNROLL-NO-VF:       pred.udiv.if:
 ; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = udiv i32 219220132, [[TMP2]]
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; UNROLL-NO-VF:       pred.udiv.continue:
 ; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP5]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]]
-; UNROLL-NO-VF:       pred.udiv.if4:
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP5]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
+; UNROLL-NO-VF:       pred.udiv.if3:
 ; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = udiv i32 219220132, [[TMP3]]
-; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE5]]
-; UNROLL-NO-VF:       pred.udiv.continue5:
-; UNROLL-NO-VF-NEXT:    [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF4]] ]
+; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF:       pred.udiv.continue4:
+; UNROLL-NO-VF-NEXT:    [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
-; UNROLL-NO-VF-NEXT:    [[TMP11]] = add i32 [[VEC_PHI2]], [[TMP7]]
+; UNROLL-NO-VF-NEXT:    [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP7]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; UNROLL-NO-VF:       pred.store.if:
 ; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = add i32 [[INDEX]], 0
@@ -3225,16 +3225,16 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-VF-NEXT:    store i32 [[TMP2]], ptr [[TMP13]], align 4
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NO-VF:       pred.store.continue:
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
-; UNROLL-NO-VF:       pred.store.if6:
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; UNROLL-NO-VF:       pred.store.if5:
 ; UNROLL-NO-VF-NEXT:    [[TMP14:%.*]] = add i32 [[INDEX]], 1
 ; UNROLL-NO-VF-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP14]]
 ; UNROLL-NO-VF-NEXT:    store i32 [[TMP3]], ptr [[TMP15]], align 4
-; UNROLL-NO-VF-NEXT:    br label [[PRED_STORE_CONTINUE7]]
-; UNROLL-NO-VF:       pred.store.continue7:
+; UNROLL-NO-VF-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; UNROLL-NO-VF:       pred.store.continue6:
 ; UNROLL-NO-VF-NEXT:    [[TMP16:%.*]] = select i1 [[TMP4]], i32 [[TMP10]], i32 [[VEC_PHI]]
-; UNROLL-NO-VF-NEXT:    [[TMP17:%.*]] = select i1 [[TMP5]], i32 [[TMP11]], i32 [[VEC_PHI2]]
-; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; UNROLL-NO-VF-NEXT:    [[TMP17:%.*]] = select i1 [[TMP5]], i32 [[TMP11]], i32 [[VEC_PHI1]]
+; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27]], !llvm.loop [[LOOP32:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
@@ -3279,10 +3279,10 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; SINK-AFTER-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SINK-AFTER-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SINK-AFTER:       vector.body:
-; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
-; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE13]] ]
-; SINK-AFTER-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_STORE_CONTINUE13]] ]
+; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
+; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE12]] ]
+; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_STORE_CONTINUE12]] ]
+; SINK-AFTER-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_STORE_CONTINUE12]] ]
 ; SINK-AFTER-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
 ; SINK-AFTER-NEXT:    [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0
 ; SINK-AFTER-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
@@ -3298,29 +3298,29 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; SINK-AFTER:       pred.udiv.continue:
 ; SINK-AFTER-NEXT:    [[TMP10:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_UDIV_IF]] ]
 ; SINK-AFTER-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1
-; SINK-AFTER-NEXT:    br i1 [[TMP11]], label [[PRED_UDIV_IF2:%.*]], label [[PRED_UDIV_CONTINUE3:%.*]]
-; SINK-AFTER:       pred.udiv.if2:
+; SINK-AFTER-NEXT:    br i1 [[TMP11]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
+; SINK-AFTER:       pred.udiv.if1:
 ; SINK-AFTER-NEXT:    [[TMP12:%.*]] = udiv i32 219220132, [[TMP3]]
 ; SINK-AFTER-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP12]], i32 1
-; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE3]]
-; SINK-AFTER:       pred.udiv.continue3:
-; SINK-AFTER-NEXT:    [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF2]] ]
+; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE2]]
+; SINK-AFTER:       pred.udiv.continue2:
+; SINK-AFTER-NEXT:    [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF1]] ]
 ; SINK-AFTER-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2
-; SINK-AFTER-NEXT:    br i1 [[TMP15]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]]
-; SINK-AFTER:       pred.udiv.if4:
+; SINK-AFTER-NEXT:    br i1 [[TMP15]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
+; SINK-AFTER:       pred.udiv.if3:
 ; SINK-AFTER-NEXT:    [[TMP16:%.*]] = udiv i32 219220132, [[TMP4]]
 ; SINK-AFTER-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i32 2
-; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE5]]
-; SINK-AFTER:       pred.udiv.continue5:
-; SINK-AFTER-NEXT:    [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE3]] ], [ [[TMP17]], [[PRED_UDIV_IF4]] ]
+; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
+; SINK-AFTER:       pred.udiv.continue4:
+; SINK-AFTER-NEXT:    [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP17]], [[PRED_UDIV_IF3]] ]
 ; SINK-AFTER-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3
-; SINK-AFTER-NEXT:    br i1 [[TMP19]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]]
-; SINK-AFTER:       pred.udiv.if6:
+; SINK-AFTER-NEXT:    br i1 [[TMP19]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
+; SINK-AFTER:       pred.udiv.if5:
 ; SINK-AFTER-NEXT:    [[TMP20:%.*]] = udiv i32 219220132, [[TMP5]]
 ; SINK-AFTER-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i32 3
-; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE7]]
-; SINK-AFTER:       pred.udiv.continue7:
-; SINK-AFTER-NEXT:    [[TMP22]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP21]], [[PRED_UDIV_IF6]] ]
+; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
+; SINK-AFTER:       pred.udiv.continue6:
+; SINK-AFTER-NEXT:    [[TMP22]] = phi <4 x i32> [ [[TMP18]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP21]], [[PRED_UDIV_IF5]] ]
 ; SINK-AFTER-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]]
 ; SINK-AFTER-NEXT:    [[TMP25:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
@@ -3332,31 +3332,31 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; SINK-AFTER:       pred.store.continue:
 ; SINK-AFTER-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP6]], i32 1
-; SINK-AFTER-NEXT:    br i1 [[TMP28]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
-; SINK-AFTER:       pred.store.if8:
+; SINK-AFTER-NEXT:    br i1 [[TMP28]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
+; SINK-AFTER:       pred.store.if7:
 ; SINK-AFTER-NEXT:    [[TMP29:%.*]] = add i32 [[INDEX]], 1
 ; SINK-AFTER-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP29]]
 ; SINK-AFTER-NEXT:    store i32 [[TMP3]], ptr [[TMP30]], align 4
-; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE9]]
-; SINK-AFTER:       pred.store.continue9:
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE8]]
+; SINK-AFTER:       pred.store.continue8:
 ; SINK-AFTER-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP6]], i32 2
-; SINK-AFTER-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
-; SINK-AFTER:       pred.store.if10:
+; SINK-AFTER-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
+; SINK-AFTER:       pred.store.if9:
 ; SINK-AFTER-NEXT:    [[TMP32:%.*]] = add i32 [[INDEX]], 2
 ; SINK-AFTER-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP32]]
 ; SINK-AFTER-NEXT:    store i32 [[TMP4]], ptr [[TMP33]], align 4
-; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE11]]
-; SINK-AFTER:       pred.store.continue11:
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE10]]
+; SINK-AFTER:       pred.store.continue10:
 ; SINK-AFTER-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3
-; SINK-AFTER-NEXT:    br i1 [[TMP34]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
-; SINK-AFTER:       pred.store.if12:
+; SINK-AFTER-NEXT:    br i1 [[TMP34]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
+; SINK-AFTER:       pred.store.if11:
 ; SINK-AFTER-NEXT:    [[TMP35:%.*]] = add i32 [[INDEX]], 3
 ; SINK-AFTER-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP35]]
 ; SINK-AFTER-NEXT:    store i32 [[TMP5]], ptr [[TMP36]], align 4
-; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE13]]
-; SINK-AFTER:       pred.store.continue13:
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE12]]
+; SINK-AFTER:       pred.store.continue12:
 ; SINK-AFTER-NEXT:    [[TMP37:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]]
-; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; SINK-AFTER-NEXT:    [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF27]], !llvm.loop [[LOOP32:![0-9]+]]
@@ -3712,7 +3712,7 @@ define i32 @recurence_uniform_load(ptr %src, ptr noalias %dst) {
 ; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4
 ; UNROLL-NO-IC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
 ; UNROLL-NO-IC-NEXT:    [[BROADCAST_SPLAT]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
 ; UNROLL-NO-IC-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -3768,7 +3768,7 @@ define i32 @recurence_uniform_load(ptr %src, ptr noalias %dst) {
 ; SINK-AFTER-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4
 ; SINK-AFTER-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
 ; SINK-AFTER-NEXT:    [[BROADCAST_SPLAT]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; SINK-AFTER:       middle.block:
 ; SINK-AFTER-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index 89ac22b054e24..601c475df56cd 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -61,7 +61,7 @@ define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias nocapture %A, i32 %N)
 ; VEC4_INTERL1:       for.body:
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL1-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL1-NEXT:    [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -102,7 +102,7 @@ define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias nocapture %A, i32 %N)
 ; VEC4_INTERL2-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[STEP_ADD:%.*]] = fsub fast <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
 ; VEC4_INTERL2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD]], ptr [[TMP5]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -119,7 +119,7 @@ define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias nocapture %A, i32 %N)
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL2-NEXT:    [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -170,7 +170,7 @@ define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias nocapture %A, i32 %N)
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC1_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[ADD]] = fsub fast float [[X_05]], [[FPINC]]
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -308,7 +308,7 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias nocapture %A, i32
 ; VEC4_INTERL1:       for.body:
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL1-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL1-NEXT:    [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -349,7 +349,7 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias nocapture %A, i32
 ; VEC4_INTERL2-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[STEP_ADD:%.*]] = fsub reassoc <4 x float> [[VEC_IND]], [[DOTSPLAT5]]
 ; VEC4_INTERL2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND]], ptr [[TMP4]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD]], ptr [[TMP5]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -366,7 +366,7 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias nocapture %A, i32
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL2-NEXT:    [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -419,7 +419,7 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias nocapture %A, i32
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC1_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_05]], ptr [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -552,7 +552,7 @@ define void @fp_iv_loop2(float %init, ptr noalias nocapture %A, i32 %N) #0 {
 ; VEC4_INTERL1:       for.body:
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL1-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL1-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -586,7 +586,7 @@ define void @fp_iv_loop2(float %init, ptr noalias nocapture %A, i32 %N) #0 {
 ; VEC4_INTERL2-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[STEP_ADD:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
 ; VEC4_INTERL2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD]], ptr [[TMP3]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -603,7 +603,7 @@ define void @fp_iv_loop2(float %init, ptr noalias nocapture %A, i32 %N) #0 {
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL2-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -653,7 +653,7 @@ define void @fp_iv_loop2(float %init, ptr noalias nocapture %A, i32 %N) #0 {
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC1_INTERL2-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -806,14 +806,14 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL1-NEXT:    [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL1-NEXT:    [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[X_011]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL1-NEXT:    [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
 ; VEC4_INTERL1-NEXT:    [[CONV1]] = fadd fast float [[Y_012]], -5.000000e-01
 ; VEC4_INTERL1-NEXT:    [[ADD2:%.*]] = fadd fast float [[CONV1]], [[ADD]]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[ADD2]], ptr [[ARRAYIDX4]], align 4
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw float, ptr [[C]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[CONV1]], ptr [[ARRAYIDX6]], align 4
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; VEC4_INTERL1-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
@@ -857,7 +857,7 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC4_INTERL2-NEXT:    [[VEC_IND10:%.*]] = phi <4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT13:%.*]], [[VECTOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[STEP_ADD11:%.*]] = fadd fast <4 x float> [[VEC_IND10]], [[DOTSPLAT5]]
 ; VEC4_INTERL2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND10]], ptr [[TMP6]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD11]], ptr [[TMP7]], align 4
 ; VEC4_INTERL2-NEXT:    [[TMP8:%.*]] = fadd fast <4 x float> [[VEC_IND10]], [[BROADCAST]]
@@ -867,11 +867,11 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC4_INTERL2-NEXT:    [[TMP12:%.*]] = fadd fast <4 x float> [[TMP10]], [[TMP8]]
 ; VEC4_INTERL2-NEXT:    [[TMP13:%.*]] = fadd fast <4 x float> [[TMP11]], [[TMP9]]
 ; VEC4_INTERL2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[TMP12]], ptr [[TMP14]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[TMP13]], ptr [[TMP15]], align 4
 ; VEC4_INTERL2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[TMP10]], ptr [[TMP16]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[TMP11]], ptr [[TMP17]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -891,14 +891,14 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[X_011]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL2-NEXT:    [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
 ; VEC4_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[Y_012]], -5.000000e-01
 ; VEC4_INTERL2-NEXT:    [[ADD2:%.*]] = fadd fast float [[CONV1]], [[ADD]]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[ADD2]], ptr [[ARRAYIDX4]], align 4
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw float, ptr [[C]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[CONV1]], ptr [[ARRAYIDX6]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; VEC4_INTERL2-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
@@ -969,14 +969,14 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VEC1_INTERL2-NEXT:    [[Y_012:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[CONV1:%.*]], [[FOR_BODY]] ]
 ; VEC1_INTERL2-NEXT:    [[X_011:%.*]] = phi float [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_011]], ptr [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[ADD]] = fadd fast float [[X_011]], [[TMP0]]
 ; VEC1_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[Y_012]], -5.000000e-01
 ; VEC1_INTERL2-NEXT:    [[ADD2:%.*]] = fadd fast float [[CONV1]], [[ADD]]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[ADD2]], ptr [[ARRAYIDX4]], align 4
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw float, ptr [[C]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[CONV1]], ptr [[ARRAYIDX6]], align 4
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; VEC1_INTERL2-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
@@ -1134,7 +1134,7 @@ define void @fp_iv_loop4(ptr noalias nocapture %A, i32 %N) {
 ; VEC4_INTERL1:       for.body:
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL1-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL1-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL1-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC4_INTERL1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -1165,7 +1165,7 @@ define void @fp_iv_loop4(ptr noalias nocapture %A, i32 %N) {
 ; VEC4_INTERL2-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VEC4_INTERL2-NEXT:    [[STEP_ADD:%.*]] = fadd fast <4 x float> [[VEC_IND]], splat (float 2.000000e+00)
 ; VEC4_INTERL2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND]], ptr [[TMP2]], align 4
 ; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD]], ptr [[TMP3]], align 4
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -1182,7 +1182,7 @@ define void @fp_iv_loop4(ptr noalias nocapture %A, i32 %N) {
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL2-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC4_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC4_INTERL2-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC4_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC4_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -1232,7 +1232,7 @@ define void @fp_iv_loop4(ptr noalias nocapture %A, i32 %N) {
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC1_INTERL2-NEXT:    [[X_06:%.*]] = phi float [ [[CONV1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_06]], ptr [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; VEC1_INTERL2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -1320,7 +1320,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL1-NEXT:    [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
 ; VEC4_INTERL1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC4_INTERL1:       vector.body:
-; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
+; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE7:%.*]] ]
 ; VEC4_INTERL1-NEXT:    [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
 ; VEC4_INTERL1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; VEC4_INTERL1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
@@ -1333,32 +1333,32 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC4_INTERL1:       pred.store.continue:
 ; VEC4_INTERL1-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; VEC4_INTERL1-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
-; VEC4_INTERL1:       pred.store.if3:
+; VEC4_INTERL1-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3:%.*]]
+; VEC4_INTERL1:       pred.store.if2:
 ; VEC4_INTERL1-NEXT:    [[TMP5:%.*]] = or disjoint i64 [[INDEX]], 1
 ; VEC4_INTERL1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
 ; VEC4_INTERL1-NEXT:    [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
 ; VEC4_INTERL1-NEXT:    store float [[TMP7]], ptr [[TMP6]], align 4
-; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; VEC4_INTERL1:       pred.store.continue4:
+; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE3]]
+; VEC4_INTERL1:       pred.store.continue3:
 ; VEC4_INTERL1-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; VEC4_INTERL1-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; VEC4_INTERL1:       pred.store.if5:
+; VEC4_INTERL1-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
+; VEC4_INTERL1:       pred.store.if4:
 ; VEC4_INTERL1-NEXT:    [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 2
 ; VEC4_INTERL1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
 ; VEC4_INTERL1-NEXT:    [[TMP11:%.*]] = fadd fast float [[DOTCAST2]], 2.000000e+00
 ; VEC4_INTERL1-NEXT:    store float [[TMP11]], ptr [[TMP10]], align 4
-; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE6]]
-; VEC4_INTERL1:       pred.store.continue6:
+; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE5]]
+; VEC4_INTERL1:       pred.store.continue5:
 ; VEC4_INTERL1-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; VEC4_INTERL1-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
-; VEC4_INTERL1:       pred.store.if7:
+; VEC4_INTERL1-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7]]
+; VEC4_INTERL1:       pred.store.if6:
 ; VEC4_INTERL1-NEXT:    [[TMP13:%.*]] = or disjoint i64 [[INDEX]], 3
 ; VEC4_INTERL1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]]
 ; VEC4_INTERL1-NEXT:    [[TMP15:%.*]] = fadd fast float [[DOTCAST2]], 3.000000e+00
 ; VEC4_INTERL1-NEXT:    store float [[TMP15]], ptr [[TMP14]], align 4
-; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE8]]
-; VEC4_INTERL1:       pred.store.continue8:
+; VEC4_INTERL1-NEXT:    br label [[PRED_STORE_CONTINUE7]]
+; VEC4_INTERL1:       pred.store.continue7:
 ; VEC4_INTERL1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VEC4_INTERL1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VEC4_INTERL1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -1372,7 +1372,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL1:       for.body:
 ; VEC4_INTERL1-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL1-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL1-NEXT:    [[VAR0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]]
+; VEC4_INTERL1-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
 ; VEC4_INTERL1-NEXT:    [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
 ; VEC4_INTERL1-NEXT:    [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
 ; VEC4_INTERL1-NEXT:    br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
@@ -1397,10 +1397,10 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL2-NEXT:    [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
 ; VEC4_INTERL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC4_INTERL2:       vector.body:
-; VEC4_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE17:%.*]] ]
+; VEC4_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ]
 ; VEC4_INTERL2-NEXT:    [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
 ; VEC4_INTERL2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; VEC4_INTERL2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; VEC4_INTERL2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
 ; VEC4_INTERL2-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
 ; VEC4_INTERL2-NEXT:    [[TMP3:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], zeroinitializer
@@ -1408,73 +1408,73 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
 ; VEC4_INTERL2-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; VEC4_INTERL2:       pred.store.if:
-; VEC4_INTERL2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; VEC4_INTERL2-NEXT:    store float [[DOTCAST2]], ptr [[TMP6]], align 4
+; VEC4_INTERL2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; VEC4_INTERL2-NEXT:    store float [[DOTCAST2]], ptr [[TMP35]], align 4
 ; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC4_INTERL2:       pred.store.continue:
-; VEC4_INTERL2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
-; VEC4_INTERL2-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]
-; VEC4_INTERL2:       pred.store.if4:
-; VEC4_INTERL2-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1
-; VEC4_INTERL2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
-; VEC4_INTERL2-NEXT:    [[TMP10:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
-; VEC4_INTERL2-NEXT:    store float [[TMP10]], ptr [[TMP9]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE5]]
-; VEC4_INTERL2:       pred.store.continue5:
-; VEC4_INTERL2-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
-; VEC4_INTERL2-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7:%.*]]
-; VEC4_INTERL2:       pred.store.if6:
-; VEC4_INTERL2-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[INDEX]], 2
-; VEC4_INTERL2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]]
-; VEC4_INTERL2-NEXT:    [[TMP14:%.*]] = fadd fast float [[DOTCAST2]], 2.000000e+00
-; VEC4_INTERL2-NEXT:    store float [[TMP14]], ptr [[TMP13]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE7]]
-; VEC4_INTERL2:       pred.store.continue7:
-; VEC4_INTERL2-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
-; VEC4_INTERL2-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
-; VEC4_INTERL2:       pred.store.if8:
-; VEC4_INTERL2-NEXT:    [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 3
-; VEC4_INTERL2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP16]]
-; VEC4_INTERL2-NEXT:    [[TMP18:%.*]] = fadd fast float [[DOTCAST2]], 3.000000e+00
-; VEC4_INTERL2-NEXT:    store float [[TMP18]], ptr [[TMP17]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE9]]
-; VEC4_INTERL2:       pred.store.continue9:
-; VEC4_INTERL2-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
-; VEC4_INTERL2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
-; VEC4_INTERL2:       pred.store.if10:
-; VEC4_INTERL2-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 4
-; VEC4_INTERL2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP0]]
+; VEC4_INTERL2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
+; VEC4_INTERL2-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; VEC4_INTERL2:       pred.store.if3:
+; VEC4_INTERL2-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[INDEX]], 1
+; VEC4_INTERL2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
+; VEC4_INTERL2-NEXT:    [[TMP9:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
+; VEC4_INTERL2-NEXT:    store float [[TMP9]], ptr [[TMP8]], align 4
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; VEC4_INTERL2:       pred.store.continue4:
+; VEC4_INTERL2-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
+; VEC4_INTERL2-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
+; VEC4_INTERL2:       pred.store.if5:
+; VEC4_INTERL2-NEXT:    [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 2
+; VEC4_INTERL2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]]
+; VEC4_INTERL2-NEXT:    [[TMP13:%.*]] = fadd fast float [[DOTCAST2]], 2.000000e+00
+; VEC4_INTERL2-NEXT:    store float [[TMP13]], ptr [[TMP12]], align 4
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; VEC4_INTERL2:       pred.store.continue6:
+; VEC4_INTERL2-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
+; VEC4_INTERL2-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
+; VEC4_INTERL2:       pred.store.if7:
+; VEC4_INTERL2-NEXT:    [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 3
+; VEC4_INTERL2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
+; VEC4_INTERL2-NEXT:    [[TMP17:%.*]] = fadd fast float [[DOTCAST2]], 3.000000e+00
+; VEC4_INTERL2-NEXT:    store float [[TMP17]], ptr [[TMP16]], align 4
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE8]]
+; VEC4_INTERL2:       pred.store.continue8:
+; VEC4_INTERL2-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
+; VEC4_INTERL2-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
+; VEC4_INTERL2:       pred.store.if9:
+; VEC4_INTERL2-NEXT:    [[TMP19:%.*]] = or disjoint i64 [[INDEX]], 4
+; VEC4_INTERL2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP19]]
 ; VEC4_INTERL2-NEXT:    [[TMP21:%.*]] = fadd fast float [[DOTCAST2]], 4.000000e+00
 ; VEC4_INTERL2-NEXT:    store float [[TMP21]], ptr [[TMP20]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE11]]
-; VEC4_INTERL2:       pred.store.continue11:
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE10]]
+; VEC4_INTERL2:       pred.store.continue10:
 ; VEC4_INTERL2-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
-; VEC4_INTERL2-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
-; VEC4_INTERL2:       pred.store.if12:
+; VEC4_INTERL2-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]
+; VEC4_INTERL2:       pred.store.if11:
 ; VEC4_INTERL2-NEXT:    [[TMP23:%.*]] = or disjoint i64 [[INDEX]], 5
 ; VEC4_INTERL2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]]
 ; VEC4_INTERL2-NEXT:    [[TMP25:%.*]] = fadd fast float [[DOTCAST2]], 5.000000e+00
 ; VEC4_INTERL2-NEXT:    store float [[TMP25]], ptr [[TMP24]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE13]]
-; VEC4_INTERL2:       pred.store.continue13:
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE12]]
+; VEC4_INTERL2:       pred.store.continue12:
 ; VEC4_INTERL2-NEXT:    [[TMP26:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
-; VEC4_INTERL2-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
-; VEC4_INTERL2:       pred.store.if14:
+; VEC4_INTERL2-NEXT:    br i1 [[TMP26]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]
+; VEC4_INTERL2:       pred.store.if13:
 ; VEC4_INTERL2-NEXT:    [[TMP27:%.*]] = or disjoint i64 [[INDEX]], 6
 ; VEC4_INTERL2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP27]]
 ; VEC4_INTERL2-NEXT:    [[TMP29:%.*]] = fadd fast float [[DOTCAST2]], 6.000000e+00
 ; VEC4_INTERL2-NEXT:    store float [[TMP29]], ptr [[TMP28]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE15]]
-; VEC4_INTERL2:       pred.store.continue15:
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE14]]
+; VEC4_INTERL2:       pred.store.continue14:
 ; VEC4_INTERL2-NEXT:    [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
-; VEC4_INTERL2-NEXT:    br i1 [[TMP30]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]]
-; VEC4_INTERL2:       pred.store.if16:
+; VEC4_INTERL2-NEXT:    br i1 [[TMP30]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]]
+; VEC4_INTERL2:       pred.store.if15:
 ; VEC4_INTERL2-NEXT:    [[TMP31:%.*]] = or disjoint i64 [[INDEX]], 7
 ; VEC4_INTERL2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]]
 ; VEC4_INTERL2-NEXT:    [[TMP33:%.*]] = fadd fast float [[DOTCAST2]], 7.000000e+00
 ; VEC4_INTERL2-NEXT:    store float [[TMP33]], ptr [[TMP32]], align 4
-; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE17]]
-; VEC4_INTERL2:       pred.store.continue17:
+; VEC4_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE16]]
+; VEC4_INTERL2:       pred.store.continue16:
 ; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; VEC4_INTERL2-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VEC4_INTERL2-NEXT:    br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -1488,7 +1488,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC4_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]]
+; VEC4_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
 ; VEC4_INTERL2-NEXT:    [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
 ; VEC4_INTERL2-NEXT:    [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
 ; VEC4_INTERL2-NEXT:    br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
@@ -1513,7 +1513,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC1_INTERL2-NEXT:    [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
 ; VEC1_INTERL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC1_INTERL2:       vector.body:
-; VEC1_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
+; VEC1_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
 ; VEC1_INTERL2-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
 ; VEC1_INTERL2-NEXT:    [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
 ; VEC1_INTERL2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
@@ -1527,12 +1527,12 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC1_INTERL2-NEXT:    store float [[DOTCAST2]], ptr [[TMP1]], align 4
 ; VEC1_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC1_INTERL2:       pred.store.continue:
-; VEC1_INTERL2-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; VEC1_INTERL2:       pred.store.if3:
+; VEC1_INTERL2-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; VEC1_INTERL2:       pred.store.if2:
 ; VEC1_INTERL2-NEXT:    [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
 ; VEC1_INTERL2-NEXT:    store float [[TMP7]], ptr [[TMP2]], align 4
-; VEC1_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; VEC1_INTERL2:       pred.store.continue4:
+; VEC1_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE3]]
+; VEC1_INTERL2:       pred.store.continue3:
 ; VEC1_INTERL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VEC1_INTERL2-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VEC1_INTERL2-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -1546,7 +1546,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC1_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]]
+; VEC1_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
 ; VEC1_INTERL2-NEXT:    [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
 ; VEC1_INTERL2-NEXT:    [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
 ; VEC1_INTERL2-NEXT:    br i1 [[VAR2]], label [[IF_PRED:%.*]], label [[FOR_INC]]
@@ -1571,7 +1571,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[DOTCAST:%.*]] = uitofp nneg i64 [[N_VEC]] to float
 ; VEC2_INTERL1_PRED_STORE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC2_INTERL1_PRED_STORE:       vector.body:
-; VEC2_INTERL1_PRED_STORE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 4
@@ -1584,14 +1584,14 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC2_INTERL1_PRED_STORE-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC2_INTERL1_PRED_STORE:       pred.store.continue:
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
-; VEC2_INTERL1_PRED_STORE-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; VEC2_INTERL1_PRED_STORE:       pred.store.if3:
+; VEC2_INTERL1_PRED_STORE-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; VEC2_INTERL1_PRED_STORE:       pred.store.if2:
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP5:%.*]] = or disjoint i64 [[INDEX]], 1
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
 ; VEC2_INTERL1_PRED_STORE-NEXT:    store float [[TMP7]], ptr [[TMP6]], align 4
-; VEC2_INTERL1_PRED_STORE-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; VEC2_INTERL1_PRED_STORE:       pred.store.continue4:
+; VEC2_INTERL1_PRED_STORE-NEXT:    br label [[PRED_STORE_CONTINUE3]]
+; VEC2_INTERL1_PRED_STORE:       pred.store.continue3:
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VEC2_INTERL1_PRED_STORE-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/forked-pointers.ll b/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
index 00e3e669cd013..3386e1d856377 100644
--- a/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
+++ b/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
@@ -26,19 +26,19 @@ define dso_local void @forked_ptrs_different_base_same_offset(ptr nocapture read
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[DEST1]], [[PREDS2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DEST1]], [[BASE23]]
-; CHECK-NEXT:    [[TMP1_FR:%.*]] = freeze i64 [[TMP1]]
-; CHECK-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP1_FR]], 16
+; CHECK-NEXT:    [[DOTFR:%.*]] = freeze i64 [[TMP1]]
+; CHECK-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[DOTFR]], 16
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[DEST1]], [[BASE15]]
-; CHECK-NEXT:    [[TMP2_FR:%.*]] = freeze i64 [[TMP2]]
-; CHECK-NEXT:    [[DIFF_CHECK7:%.*]] = icmp ult i64 [[TMP2_FR]], 16
-; CHECK-NEXT:    [[CONFLICT_RDX8:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK7]]
-; CHECK-NEXT:    br i1 [[CONFLICT_RDX8]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[DOTFR10:%.*]] = freeze i64 [[TMP2]]
+; CHECK-NEXT:    [[DIFF_CHECK6:%.*]] = icmp ult i64 [[DOTFR10]], 16
+; CHECK-NEXT:    [[CONFLICT_RDX7:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK6]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX7]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE2]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT9]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT8]], <4 x ptr> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -48,7 +48,7 @@ define dso_local void @forked_ptrs_different_base_same_offset(ptr nocapture read
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[PREDS]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x ptr> [[BROADCAST_SPLAT]], <4 x ptr> [[BROADCAST_SPLAT10]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x ptr> [[BROADCAST_SPLAT]], <4 x ptr> [[BROADCAST_SPLAT9]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP8]], i64 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x ptr> [[TMP8]], i64 1
@@ -78,13 +78,13 @@ define dso_local void @forked_ptrs_different_base_same_offset(ptr nocapture read
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PREDS]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[PREDS]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i32 [[TMP27]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP1_NOT]], ptr [[BASE2]], ptr [[BASE1]]
-; CHECK-NEXT:    [[DOTSINK_IN:%.*]] = getelementptr inbounds float, ptr [[SPEC_SELECT]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[DOTSINK_IN:%.*]] = getelementptr inbounds nuw float, ptr [[SPEC_SELECT]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[DOTSINK:%.*]] = load float, ptr [[DOTSINK_IN]], align 4
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[DEST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw float, ptr [[DEST]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store float [[DOTSINK]], ptr [[TMP28]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 100
@@ -132,7 +132,7 @@ define dso_local void @forked_ptrs_same_base_different_offset(ptr nocapture read
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_014:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PREDS:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[PREDS:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -140,9 +140,9 @@ define dso_local void @forked_ptrs_same_base_different_offset(ptr nocapture read
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[OFFSET_0:%.*]] = select i1 [[CMP1_NOT]], i32 [[ADD]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[IDXPROM213:%.*]] = zext i32 [[OFFSET_0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[BASE:%.*]], i64 [[IDXPROM213]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw float, ptr [[BASE:%.*]], i64 [[IDXPROM213]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[DEST:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw float, ptr [[DEST:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 100
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 5f0e3b1e10d3e..1adc0bf0c9ec0 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -10,10 +10,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index c71ff2b9fa4c4..7170147438327 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -72,7 +72,7 @@ for.end:
 ; CHECK-NEXT:  Successor(s): loop.0
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.0:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successor
 ; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 01eb535e6586c..c4509e4ad56e4 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -423,19 +423,19 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NEXT:  entry:
 ; UNROLL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL:       vector.body:
-; UNROLL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; UNROLL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; UNROLL-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; UNROLL-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
 ; UNROLL-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr undef, i64 [[TMP1]]
 ; UNROLL-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
 ; UNROLL-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
-; UNROLL-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE3]]
+; UNROLL-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; UNROLL:       pred.store.if:
 ; UNROLL-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
 ; UNROLL-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; UNROLL:       pred.store.continue3:
+; UNROLL-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; UNROLL:       pred.store.continue2:
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; UNROLL-NEXT:    br i1 [[TMP6]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -448,7 +448,7 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NOSIMPLIFY:       vector.ph:
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       vector.body:
-; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
@@ -460,11 +460,11 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY:       pred.store.if2:
+; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY:       pred.store.if1:
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY:       pred.store.continue3:
+; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY:       pred.store.continue2:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -472,11 +472,11 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 1000, [[ENTRY]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 1000, [[ENTRY]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       for.body:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP0:%.*]] = phi i64 [ [[TMP6:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
@@ -499,7 +499,7 @@ define void @minimal_bit_widths(i1 %c) {
 ; VEC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
-; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; VEC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
@@ -513,14 +513,14 @@ define void @minimal_bit_widths(i1 %c) {
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC:       pred.store.continue:
 ; VEC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; VEC-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; VEC:       pred.store.if2:
+; VEC-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; VEC:       pred.store.if1:
 ; VEC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
 ; VEC-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr undef, i64 [[TMP7]]
 ; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
 ; VEC-NEXT:    store i8 [[TMP9]], ptr [[TMP8]], align 1
-; VEC-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; VEC:       pred.store.continue3:
+; VEC-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; VEC:       pred.store.continue2:
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VEC-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; VEC-NEXT:    br i1 [[TMP10]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -583,7 +583,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; UNROLL-NOSIMPLIFY:       vector.ph:
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       vector.body:
-; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[TMP0]]
@@ -597,11 +597,11 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue:
-; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY:       pred.store.if2:
+; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY:       pred.store.if1:
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
-; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY:       pred.store.continue3:
+; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY:       pred.store.continue2:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -609,11 +609,11 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       for.body:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP0:%.*]] = phi i64 [ [[TMP6:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 0, ptr [[TMP2]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index d6eb4264c0a0d..e0b1c5f2d861b 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -112,7 +112,7 @@ define void @multi_int_induction(ptr %A, i32 %N) {
 ; UNROLL-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NEXT:    [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; UNROLL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8
+; UNROLL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP4]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -203,7 +203,7 @@ define void @multi_int_induction(ptr %A, i32 %N) {
 ; INTERLEAVE-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 190, i32 191, i32 192, i32 193>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; INTERLEAVE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 16
+; INTERLEAVE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP3]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP4]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -408,12 +408,12 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; UNROLL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NEXT:    [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
 ; UNROLL-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[OFFSET]]
-; UNROLL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 8
+; UNROLL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 8
 ; UNROLL-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
 ; UNROLL-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x float>, ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]]
 ; UNROLL-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
 ; UNROLL-NEXT:    [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[OFFSET2]]
-; UNROLL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 8
+; UNROLL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8
 ; UNROLL-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x float>, ptr [[TMP9]], align 4, !alias.scope [[META7]]
 ; UNROLL-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x float>, ptr [[TMP10]], align 4, !alias.scope [[META7]]
 ; UNROLL-NEXT:    [[TMP11:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD5]]
@@ -546,12 +546,12 @@ define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
 ; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-NEXT:    [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
 ; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[OFFSET]]
-; INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]]
 ; INTERLEAVE-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
 ; INTERLEAVE-NEXT:    [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[OFFSET2]]
-; INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 16
+; INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP9]], align 4, !alias.scope [[META7]]
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP10]], align 4, !alias.scope [[META7]]
 ; INTERLEAVE-NEXT:    [[TMP11:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD5]]
@@ -688,7 +688,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) {
 ; IND:       for.body:
 ; IND-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; IND-NEXT:    [[SUM:%.*]] = phi i64 [ [[TMP6:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; IND-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; IND-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]]
 ; IND-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
 ; IND-NEXT:    [[TMP6]] = add i64 [[TMP5]], [[SUM]]
 ; IND-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -711,7 +711,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) {
 ; UNROLL-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; UNROLL-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
 ; UNROLL-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
 ; UNROLL-NEXT:    [[TMP2]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
@@ -731,7 +731,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) {
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; UNROLL-NEXT:    [[SUM:%.*]] = phi i64 [ [[TMP8:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; UNROLL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; UNROLL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]]
 ; UNROLL-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; UNROLL-NEXT:    [[TMP8]] = add i64 [[TMP7]], [[SUM]]
 ; UNROLL-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -800,7 +800,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) {
 ; INTERLEAVE-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
+; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; INTERLEAVE-NEXT:    [[TMP2]] = add <4 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
@@ -820,7 +820,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) {
 ; INTERLEAVE:       for.body:
 ; INTERLEAVE-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; INTERLEAVE-NEXT:    [[SUM:%.*]] = phi i64 [ [[TMP8:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]]
 ; INTERLEAVE-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; INTERLEAVE-NEXT:    [[TMP8]] = add i64 [[TMP7]], [[SUM]]
 ; INTERLEAVE-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -1320,7 +1320,7 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) {
 ; IND-NEXT:    br label [[FOR_BODY:%.*]]
 ; IND:       for.body:
 ; IND-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; IND-NEXT:    [[F:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
+; IND-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
 ; IND-NEXT:    [[TMP11:%.*]] = load i32, ptr [[F]], align 8
 ; IND-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP11]], [[Y]]
 ; IND-NEXT:    store i32 [[TMP12]], ptr [[F]], align 8
@@ -1378,7 +1378,7 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) {
 ; UNROLL-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NEXT:    [[F:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
+; UNROLL-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
 ; UNROLL-NEXT:    [[TMP22:%.*]] = load i32, ptr [[F]], align 8
 ; UNROLL-NEXT:    [[TMP23:%.*]] = xor i32 [[TMP22]], [[Y]]
 ; UNROLL-NEXT:    store i32 [[TMP23]], ptr [[F]], align 8
@@ -1661,9 +1661,9 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; IND:       for.body:
 ; IND-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; IND-NEXT:    [[DOTIDX:%.*]] = shl nsw i64 [[I]], 4
-; IND-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX]]
+; IND-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[DOTIDX]]
 ; IND-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 1
-; IND-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
+; IND-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
 ; IND-NEXT:    store i32 [[TMP21]], ptr [[TMP22]], align 1
 ; IND-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; IND-NEXT:    [[TMP23:%.*]] = trunc i64 [[I_NEXT]] to i32
@@ -1738,9 +1738,9 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; UNROLL-NEXT:    [[DOTIDX:%.*]] = shl nsw i64 [[I]], 4
-; UNROLL-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX]]
+; UNROLL-NEXT:    [[TMP31:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[DOTIDX]]
 ; UNROLL-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 1
-; UNROLL-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
+; UNROLL-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
 ; UNROLL-NEXT:    store i32 [[TMP32]], ptr [[TMP33]], align 1
 ; UNROLL-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; UNROLL-NEXT:    [[TMP34:%.*]] = trunc i64 [[I_NEXT]] to i32
@@ -2073,7 +2073,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; IND-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
 ; IND-NEXT:    [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ]
 ; IND-NEXT:    [[TMP16:%.*]] = zext nneg i32 [[I]] to i64
-; IND-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]]
+; IND-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP16]]
 ; IND-NEXT:    [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4
 ; IND-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; IND:       if.then:
@@ -2105,7 +2105,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[PRED_UDIV_CONTINUE8]] ]
 ; UNROLL-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
 ; UNROLL-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
 ; UNROLL-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; UNROLL-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
@@ -2163,7 +2163,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
 ; UNROLL-NEXT:    [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ]
 ; UNROLL-NEXT:    [[TMP26:%.*]] = zext nneg i32 [[I]] to i64
-; UNROLL-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP26]]
+; UNROLL-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP26]]
 ; UNROLL-NEXT:    [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4
 ; UNROLL-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; UNROLL:       if.then:
@@ -2292,7 +2292,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; INTERLEAVE-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_UDIV_CONTINUE16]] ]
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
 ; INTERLEAVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
 ; INTERLEAVE-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
@@ -2386,7 +2386,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; INTERLEAVE-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
 ; INTERLEAVE-NEXT:    [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ]
 ; INTERLEAVE-NEXT:    [[TMP46:%.*]] = zext nneg i32 [[I]] to i64
-; INTERLEAVE-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP46]]
+; INTERLEAVE-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP46]]
 ; INTERLEAVE-NEXT:    [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4
 ; INTERLEAVE-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; INTERLEAVE:       if.then:
@@ -2527,7 +2527,7 @@ define void @iv_vector_and_scalar_users(ptr %p, i32 %a, i32 %n) {
 ; IND-NEXT:    [[TMP11:%.*]] = trunc i64 [[I]] to i32
 ; IND-NEXT:    [[TMP12:%.*]] = add i32 [[A]], [[TMP11]]
 ; IND-NEXT:    [[TMP13:%.*]] = trunc i32 [[TMP12]] to i16
-; IND-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
+; IND-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
 ; IND-NEXT:    store i16 [[TMP13]], ptr [[TMP14]], align 2
 ; IND-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; IND-NEXT:    [[TMP15:%.*]] = trunc i64 [[I_NEXT]] to i32
@@ -2586,7 +2586,7 @@ define void @iv_vector_and_scalar_users(ptr %p, i32 %a, i32 %n) {
 ; UNROLL-NEXT:    [[TMP19:%.*]] = trunc i64 [[I]] to i32
 ; UNROLL-NEXT:    [[TMP20:%.*]] = add i32 [[A]], [[TMP19]]
 ; UNROLL-NEXT:    [[TMP21:%.*]] = trunc i32 [[TMP20]] to i16
-; UNROLL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
+; UNROLL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
 ; UNROLL-NEXT:    store i16 [[TMP21]], ptr [[TMP22]], align 2
 ; UNROLL-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; UNROLL-NEXT:    [[TMP23:%.*]] = trunc i64 [[I_NEXT]] to i32
@@ -2722,7 +2722,7 @@ define void @iv_vector_and_scalar_users(ptr %p, i32 %a, i32 %n) {
 ; INTERLEAVE-NEXT:    [[TMP31:%.*]] = trunc i64 [[I]] to i32
 ; INTERLEAVE-NEXT:    [[TMP32:%.*]] = add i32 [[A]], [[TMP31]]
 ; INTERLEAVE-NEXT:    [[TMP33:%.*]] = trunc i32 [[TMP32]] to i16
-; INTERLEAVE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
+; INTERLEAVE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1
 ; INTERLEAVE-NEXT:    store i16 [[TMP33]], ptr [[TMP34]], align 2
 ; INTERLEAVE-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; INTERLEAVE-NEXT:    [[TMP35:%.*]] = trunc i64 [[I_NEXT]] to i32
@@ -3595,7 +3595,7 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NEXT:    [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]]
 ; UNROLL-NEXT:    [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64
 ; UNROLL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]]
-; UNROLL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 8
+; UNROLL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -3737,7 +3737,7 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
 ; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]]
 ; INTERLEAVE-NEXT:    [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64
 ; INTERLEAVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]]
-; INTERLEAVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 16
+; INTERLEAVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP11]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -3981,7 +3981,7 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; UNROLL-NEXT:    [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]]
 ; UNROLL-NEXT:    [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64
 ; UNROLL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]]
-; UNROLL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 8
+; UNROLL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -4129,7 +4129,7 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
 ; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]]
 ; INTERLEAVE-NEXT:    [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64
 ; INTERLEAVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]]
-; INTERLEAVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 16
+; INTERLEAVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP11]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -4277,7 +4277,7 @@ define void @veciv(ptr nocapture %a, i32 %start, i32 %k) {
 ; UNROLL-NEXT:    [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; UNROLL-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -4352,7 +4352,7 @@ define void @veciv(ptr nocapture %a, i32 %start, i32 %k) {
 ; INTERLEAVE-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
-; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP1]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -4493,7 +4493,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; UNROLL-NEXT:    [[SEXT:%.*]] = shl i64 [[INDEX]], 32
 ; UNROLL-NEXT:    [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 30
 ; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]]
-; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -4583,7 +4583,7 @@ define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) {
 ; INTERLEAVE-NEXT:    [[SEXT:%.*]] = shl i64 [[INDEX]], 32
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 30
 ; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]]
-; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP1]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -4728,7 +4728,7 @@ define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) {
 ; UNROLL-NEXT:    [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]]
 ; UNROLL-NEXT:    [[TMP1:%.*]] = sext i32 [[OFFSET_IDX]] to i64
 ; UNROLL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]]
-; UNROLL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+; UNROLL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -4815,7 +4815,7 @@ define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) {
 ; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]]
 ; INTERLEAVE-NEXT:    [[TMP1:%.*]] = sext i32 [[OFFSET_IDX]] to i64
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]]
-; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP2]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -4924,7 +4924,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) {
 ; IND:       for.body:
 ; IND-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; IND-NEXT:    [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; IND-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; IND-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]]
 ; IND-NEXT:    [[VAR1:%.*]] = trunc i64 [[J]] to i32
 ; IND-NEXT:    store i32 [[VAR1]], ptr [[VAR0]], align 4
 ; IND-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -4948,7 +4948,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) {
 ; UNROLL-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NEXT:    [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 4)
 ; UNROLL-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; UNROLL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[VEC_IND]], ptr [[TMP0]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[STEP_ADD]], ptr [[TMP1]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -4965,7 +4965,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) {
 ; UNROLL:       for.body:
 ; UNROLL-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; UNROLL-NEXT:    [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; UNROLL-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; UNROLL-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]]
 ; UNROLL-NEXT:    [[VAR1:%.*]] = trunc i64 [[J]] to i32
 ; UNROLL-NEXT:    store i32 [[VAR1]], ptr [[VAR0]], align 4
 ; UNROLL-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -5033,7 +5033,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) {
 ; INTERLEAVE-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 8)
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP0]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[STEP_ADD]], ptr [[TMP1]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -5050,7 +5050,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) {
 ; INTERLEAVE:       for.body:
 ; INTERLEAVE-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; INTERLEAVE-NEXT:    [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; INTERLEAVE-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; INTERLEAVE-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]]
 ; INTERLEAVE-NEXT:    [[VAR1:%.*]] = trunc i64 [[J]] to i32
 ; INTERLEAVE-NEXT:    store i32 [[VAR1]], ptr [[VAR0]], align 4
 ; INTERLEAVE-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -6238,7 +6238,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6370,7 +6370,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; UNROLL-NEXT:    [[TMP18:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
 ; UNROLL-NEXT:    [[TMP19:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2>
 ; UNROLL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[INDEX]]
-; UNROLL-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 8
+; UNROLL-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8
 ; UNROLL-NEXT:    store <2 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; UNROLL-NEXT:    store <2 x i32> [[TMP19]], ptr [[TMP21]], align 4
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -6465,7 +6465,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; UNROLL-NO-IC:       scalar.ph:
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
 ; UNROLL-NO-IC-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ]
@@ -6526,7 +6526,7 @@ define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n
 ; INTERLEAVE-NEXT:    [[TMP18:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; INTERLEAVE-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; INTERLEAVE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[INDEX]]
-; INTERLEAVE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 16
+; INTERLEAVE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16
 ; INTERLEAVE-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4
 ; INTERLEAVE-NEXT:    store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index dd58dc81ccedd..53a0a308c79a9 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -112,9 +112,6 @@ declare i32 @llvm.smin.i32(i32, i32)
 ; DBG-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; DBG-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; DBG-EMPTY:
-; DBG-NEXT: ir-bb<exit>:
-; DBG-NEXT: No successors
-; DBG-EMPTY:
 ; DBG-NEXT: scalar.ph:
 ; DBG-NEXT: Successor(s): ir-bb<loop.header>
 ; DBG-EMPTY:
@@ -123,12 +120,15 @@ declare i32 @llvm.smin.i32(i32, i32)
 ; DBG-NEXT:   IR   %d = phi i1 [ false, %entry ], [ %d.next, %loop.latch ]
 ; DBG-NEXT:   IR   %d.next = xor i1 %d, true
 ; DBG-NEXT: No successors
+; DBG-EMPTY:
+; DBG-NEXT: ir-bb<exit>:
+; DBG-NEXT: No successors
 ; DBG-NEXT: }
 
 define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
 ; CHECK-LABEL: @test_scalarize_with_branch_cond(
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %pred.store.continue5 ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %pred.store.continue4 ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDEX]] to i1
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i1 false, [[TMP0]]
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add i1 [[OFFSET_IDX]], false
@@ -142,15 +142,15 @@ define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP1]], align 4
 ; CHECK-NEXT:    br label %pred.store.continue
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    br i1 [[INDUCTION3]], label %pred.store.if4, label %pred.store.continue5
-; CHECK:       pred.store.if4:
+; CHECK-NEXT:    br i1 [[INDUCTION3]], label %pred.store.if3, label %pred.store.continue4
+; CHECK:       pred.store.if3:
 ; CHECK-NEXT:    [[INDUCTION5:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr %src, i64 [[INDUCTION5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[INDUCTION5]]
 ; CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP2]], align 4
-; CHECK-NEXT:    br label %pred.store.continue5
-; CHECK:       pred.store.continue5:
+; CHECK-NEXT:    br label %pred.store.continue4
+; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP9]], label %middle.block, label %vector.body
@@ -218,9 +218,6 @@ exit:
 ; DBG-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; DBG-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; DBG-EMPTY:
-; DBG-NEXT: ir-bb<exit>:
-; DBG-NEXT: No successors
-; DBG-EMPTY:
 ; DBG-NEXT: scalar.ph:
 ; DBG-NEXT:  EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
 ; DBG-NEXT: Successor(s): ir-bb<loop>
@@ -230,6 +227,9 @@ exit:
 ; DBG-NEXT:   IR   %for = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] (extra operand: vp<[[RESUME_P]]> from scalar.ph)
 ; DBG:        IR   %ec = icmp slt i32 %iv.next.trunc, %n
 ; DBG-NEXT: No successors
+; DBG-EMPTY:
+; DBG-NEXT: ir-bb<exit>:
+; DBG-NEXT: No successors
 ; DBG-NEXT: }
 
 define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) {
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll b/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll
new file mode 100644
index 0000000000000..c1384a1dc474f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+; Test case for https://github.com/llvm/llvm-project/issues/118850.
+define void @i65_induction_with_negative_step(ptr %dst) {
+; CHECK-LABEL: define void @i65_induction_with_negative_step(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i65 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 -1, i64 -2, i64 -3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i65 0, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i65 [[OFFSET_IDX]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[VEC_IND]], <4 x i64> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 -3
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 -4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 -3
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i64> [[REVERSE]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[REVERSE1:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i64> [[REVERSE1]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i65 [[INDEX]], 8
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i65 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i65 [ -1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_I65:%.*]] = phi i65 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_I65_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TRUNC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TRUNC]] = trunc i65 [[IV_I65]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TRUNC]]
+; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    [[IV_I65_NEXT]] = add i65 [[IV_I65]], -1
+; CHECK-NEXT:    br i1 [[ICMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.i65 = phi i65 [ 0, %entry ], [ %iv.i65.next, %loop ]
+  %for = phi i64 [ 0, %entry ], [ %trunc, %loop ]
+  %trunc = trunc i65 %iv.i65 to i64
+  %gep = getelementptr inbounds i64, ptr %dst, i64 %trunc
+  store i64 %for, ptr %gep, align 8
+  %iv.next = add i64 %iv, 1
+  %icmp = icmp eq i64 %iv.next, 1000
+  %iv.i65.next = add i65 %iv.i65, -1
+  br i1 %icmp, label %exit, label %loop
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
index a7f3057935b6d..7bbf750f1702b 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
@@ -55,7 +55,7 @@ define void @_Z4funcPjS_hh(ptr noalias nocapture readonly %a, ptr noalias nocapt
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl i8 [[DOTCAST3]], 1
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i8 [[X]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = zext i8 [[OFFSET_IDX]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP15]], align 4
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shl <4 x i32> [[TMP23]], splat (i32 1)
@@ -78,7 +78,7 @@ define void @_Z4funcPjS_hh(ptr noalias nocapture readonly %a, ptr noalias nocapt
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INDEX_011:%.*]] = phi i8 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i8 [[INDEX_011]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i32 [[TMP27]], 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index dc4fb0f15c644..e3f9126f22607 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -34,13 +34,13 @@ define void @test_array_load2_store2(i32 %C, i32 %D) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[BROADCAST_SPLAT3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [1024 x i32], ptr @CD, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -113,7 +113,7 @@ define void @test_struct_array_load3_store3() {
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [1024 x %struct.ST3], ptr @S, i64 0, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [1024 x %struct.ST3], ptr @S, i64 0, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[STRIDED_VEC2]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], splat (i32 3)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -423,7 +423,7 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[INDEX]], 9223372036854775804
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 508
@@ -436,11 +436,11 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 1016, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP1]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ult i64 [[INDVARS_IV]], 1022
@@ -499,7 +499,7 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[INDEX]], 9223372036854775804
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -517,7 +517,7 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP1]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index 45de11141235e..f5c0d8bb85836 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -59,7 +59,7 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I2]], [[K]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
@@ -157,7 +157,7 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[COND_STORE_K:%.*]]
 ; CHECK:       cond_store:
@@ -247,7 +247,7 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I0:%.*]] = phi i32 [ [[I3:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I1]], align 8
 ; CHECK-NEXT:    store i32 [[I2]], ptr [[A]], align 4
 ; CHECK-NEXT:    [[I3]] = add i32 [[I0]], [[I2]]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
index 1473d292d06ac..689d00e1d0186 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -56,7 +56,7 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I0:%.*]] = phi i32 [ [[I3:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I1]], align 8
 ; CHECK-NEXT:    [[I3]] = add i32 [[I0]], [[I2]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[A]], align 4
@@ -126,7 +126,7 @@ define void @inv_val_store_to_inv_address(ptr %a, i64 %n, ptr %b) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[A]], align 4
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -228,7 +228,7 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I2]], [[K]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
@@ -282,7 +282,7 @@ define void @inv_val_store_to_inv_address_conditional_diff_values(ptr %a, i64 %n
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[I]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[I1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I2]], [[K:%.*]]
 ; CHECK-NEXT:    store i32 [[NTRUNC]], ptr [[I1]], align 4
@@ -356,7 +356,7 @@ define i32 @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly
 ; CHECK-NEXT:    [[CMP218:%.*]] = icmp ult i32 [[J_022]], [[ITR]]
 ; CHECK-NEXT:    br i1 [[CMP218]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC8]]
 ; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[VAR1]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR1]], i64 [[INDVARS_IV23]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[J_022]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX5_PROMOTED:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[J_022]], -1
@@ -487,14 +487,14 @@ define i32 @multiple_uniform_stores_conditional(ptr nocapture %var1, ptr nocaptu
 ; CHECK-NEXT:    [[CMP218:%.*]] = icmp ult i32 [[J_022]], [[ITR]]
 ; CHECK-NEXT:    br i1 [[CMP218]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC8]]
 ; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[VAR1:%.*]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR1:%.*]], i64 [[INDVARS_IV23]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[J_022]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX5_PROMOTED:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[ARRAYIDX5_PROMOTED]], [[FOR_BODY3_LR_PH]] ], [ [[TMP5:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY3_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LATCH]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[VAR2:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR2:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[ADD]], 42
@@ -591,12 +591,12 @@ define void @unsafe_dep_uniform_load_store(i32 %arg, i32 %arg1, i64 %arg2, ptr %
 ; CHECK-NEXT:    [[I15:%.*]] = trunc i64 [[I8]] to i32
 ; CHECK-NEXT:    [[I16:%.*]] = add i32 [[ARG:%.*]], [[I15]]
 ; CHECK-NEXT:    [[I17:%.*]] = zext i32 [[I16]] to i64
-; CHECK-NEXT:    [[I18:%.*]] = getelementptr inbounds i16, ptr [[I6]], i64 [[I17]]
+; CHECK-NEXT:    [[I18:%.*]] = getelementptr inbounds nuw i16, ptr [[I6]], i64 [[I17]]
 ; CHECK-NEXT:    store i16 [[I14]], ptr [[I18]], align 2
 ; CHECK-NEXT:    [[I19:%.*]] = add i32 [[I13]], [[I9]]
 ; CHECK-NEXT:    [[I20:%.*]] = trunc i32 [[I19]] to i16
 ; CHECK-NEXT:    [[I21:%.*]] = and i16 [[I20]], 255
-; CHECK-NEXT:    [[I22:%.*]] = getelementptr inbounds i16, ptr [[ARG3]], i64 [[I17]]
+; CHECK-NEXT:    [[I22:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG3]], i64 [[I17]]
 ; CHECK-NEXT:    store i16 [[I21]], ptr [[I22]], align 2
 ; CHECK-NEXT:    [[I23]] = add nsw i32 [[I9]], 1
 ; CHECK-NEXT:    [[I24]] = add nuw nsw i64 [[I8]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll
new file mode 100644
index 0000000000000..07ee5892dc28e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll
@@ -0,0 +1,258 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK-VF4IC1
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK-VF4IC4
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s --check-prefix=CHECK-VF1IC4
+
+; This should be an AnyOf reduction instead of FindLastIV reduction.
+; The reason is the outer induction variable is invariant for inner loop.
+define i64 @select_iv_def_from_outer_loop(ptr %a, i64 %start, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i64 @select_iv_def_from_outer_loop(
+; CHECK-VF4IC1-SAME: ptr [[A:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK-VF4IC1:       [[OUTER_LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[RDX_OUTER:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[SELECT_LCSSA:%.*]], %[[OUTER_LOOP_EXIT:.*]] ]
+; CHECK-VF4IC1-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV_OUTER]]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP1]]
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 3)
+; CHECK-VF4IC1-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i64 [[IV_OUTER]], i64 [[RDX_OUTER]]
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_OUTER]], %[[OUTER_LOOP]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK-VF4IC1:       [[INNER_LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[RDX_INNER:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[IV_INNER]]
+; CHECK-VF4IC1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
+; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp eq i64 [[TMP9]], 3
+; CHECK-VF4IC1-NEXT:    [[SELECT]] = select i1 [[CMP]], i64 [[IV_OUTER]], i64 [[RDX_INNER]]
+; CHECK-VF4IC1-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT_INNER:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT_INNER]], label %[[OUTER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF4IC1:       [[OUTER_LOOP_EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SELECT_LCSSA]] = phi i64 [ [[SELECT]], %[[INNER_LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SELECT_LCSSA_LCSSA:%.*]] = phi i64 [ [[SELECT_LCSSA]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF4IC1-NEXT:    ret i64 [[SELECT_LCSSA_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i64 @select_iv_def_from_outer_loop(
+; CHECK-VF4IC4-SAME: ptr [[A:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK-VF4IC4:       [[OUTER_LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[RDX_OUTER:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[SELECT_LCSSA:%.*]], %[[OUTER_LOOP_EXIT:.*]] ]
+; CHECK-VF4IC4-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV_OUTER]]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP1]]
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i64>, ptr [[TMP6]], align 8
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 3)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD4]], splat (i64 3)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD5]], splat (i64 3)
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD6]], splat (i64 3)
+; CHECK-VF4IC4-NEXT:    [[TMP11]] = or <4 x i1> [[VEC_PHI]], [[TMP7]]
+; CHECK-VF4IC4-NEXT:    [[TMP12]] = or <4 x i1> [[VEC_PHI1]], [[TMP8]]
+; CHECK-VF4IC4-NEXT:    [[TMP13]] = or <4 x i1> [[VEC_PHI2]], [[TMP9]]
+; CHECK-VF4IC4-NEXT:    [[TMP14]] = or <4 x i1> [[VEC_PHI3]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP12]], [[TMP11]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP13]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP14]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP17:%.*]] = freeze i1 [[TMP16]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP17]], i64 [[IV_OUTER]], i64 [[RDX_OUTER]]
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_OUTER]], %[[OUTER_LOOP]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK-VF4IC4:       [[INNER_LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[RDX_INNER:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[IV_INNER]]
+; CHECK-VF4IC4-NEXT:    [[TMP18:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
+; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = icmp eq i64 [[TMP18]], 3
+; CHECK-VF4IC4-NEXT:    [[SELECT]] = select i1 [[CMP]], i64 [[IV_OUTER]], i64 [[RDX_INNER]]
+; CHECK-VF4IC4-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-VF4IC4-NEXT:    [[EXITCOND_NOT_INNER:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXITCOND_NOT_INNER]], label %[[OUTER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF4IC4:       [[OUTER_LOOP_EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SELECT_LCSSA]] = phi i64 [ [[SELECT]], %[[INNER_LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-VF4IC4-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SELECT_LCSSA_LCSSA:%.*]] = phi i64 [ [[SELECT_LCSSA]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF4IC4-NEXT:    ret i64 [[SELECT_LCSSA_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i64 @select_iv_def_from_outer_loop(
+; CHECK-VF1IC4-SAME: ptr [[A:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK-VF1IC4:       [[OUTER_LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[RDX_OUTER:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[SELECT_LCSSA:%.*]], %[[OUTER_LOOP_EXIT:.*]] ]
+; CHECK-VF1IC4-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV_OUTER]]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[TMP12]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP17]] = or i1 [[VEC_PHI]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP18]] = or i1 [[VEC_PHI1]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP19]] = or i1 [[VEC_PHI2]], [[TMP15]]
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI3]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP18]], [[TMP17]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP19]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP20]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP22:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP22]], i64 [[IV_OUTER]], i64 [[RDX_OUTER]]
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_OUTER]], %[[OUTER_LOOP]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK-VF1IC4:       [[INNER_LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[RDX_INNER:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 [[IV_INNER]]
+; CHECK-VF1IC4-NEXT:    [[TMP23:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
+; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = icmp eq i64 [[TMP23]], 3
+; CHECK-VF1IC4-NEXT:    [[SELECT]] = select i1 [[CMP]], i64 [[IV_OUTER]], i64 [[RDX_INNER]]
+; CHECK-VF1IC4-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-VF1IC4-NEXT:    [[EXITCOND_NOT_INNER:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXITCOND_NOT_INNER]], label %[[OUTER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF1IC4:       [[OUTER_LOOP_EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SELECT_LCSSA]] = phi i64 [ [[SELECT]], %[[INNER_LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-VF1IC4-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SELECT_LCSSA_LCSSA:%.*]] = phi i64 [ [[SELECT_LCSSA]], %[[OUTER_LOOP_EXIT]] ]
+; CHECK-VF1IC4-NEXT:    ret i64 [[SELECT_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer.loop
+
+outer.loop:
+  %rdx.outer = phi i64 [ %start, %entry ], [ %select, %outer.loop.exit ]
+  %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.loop.exit ]
+  %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %iv.outer
+  %0 = load ptr, ptr %arrayidx, align 8
+  br label %inner.loop
+
+inner.loop:
+  %rdx.inner = phi i64 [ %rdx.outer, %outer.loop ], [ %select, %inner.loop ]
+  %iv.inner = phi i64 [ 0, %outer.loop ], [ %iv.inner.next, %inner.loop ]
+  %arrayidx2 = getelementptr inbounds i64, ptr %0, i64 %iv.inner
+  %1 = load i64, ptr %arrayidx2, align 8
+  %cmp = icmp eq i64 %1, 3
+  %select = select i1 %cmp, i64 %iv.outer, i64 %rdx.inner
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond.not.inner = icmp eq i64 %iv.inner.next, %n
+  br i1 %exitcond.not.inner, label %outer.loop.exit, label %inner.loop
+
+outer.loop.exit:
+  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+  %exitcond.not = icmp eq i64 %iv.outer.next, %n
+  br i1 %exitcond.not, label %exit, label %outer.loop
+
+exit:
+  ret i64 %select
+}
+;.
+; CHECK-VF4IC1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF4IC1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF4IC1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF4IC1: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
+; CHECK-VF4IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF4IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF4IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF4IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
+; CHECK-VF1IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF1IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF1IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF1IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
new file mode 100644
index 0000000000000..72ed6537ef640
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s --check-prefix=CHECK
+
+define i64 @select_non_const_iv_start_signed_guard(ptr %a, i64 %rdx_start, i64 %iv_start ,i64 %n) {
+; CHECK-LABEL: define i64 @select_non_const_iv_start_signed_guard(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[RDX_START:%.*]], i64 [[IV_START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[GUARD:%.*]] = icmp slt i64 [[IV_START]], [[N]]
+; CHECK-NEXT:    br i1 [[GUARD]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[IV_START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[RDX_07:%.*]] = phi i64 [ [[COND:%.*]], %[[FOR_BODY]] ], [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 3
+; CHECK-NEXT:    [[COND]] = select i1 [[CMP1]], i64 [[IV]], i64 [[RDX_07]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[IDX_0_LCSSA:%.*]] = phi i64 [ [[RDX_START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i64 [[IDX_0_LCSSA]]
+;
+entry:
+  %guard = icmp slt i64 %iv_start, %n
+  br i1 %guard, label %for.body, label %exit
+
+for.body:
+  %iv = phi i64 [ %iv_start, %entry ], [ %iv.next, %for.body ]
+  %rdx.07 = phi i64 [ %rdx_start, %entry ], [ %cond, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
+  %1 = load i64, ptr %arrayidx, align 4
+  %cmp1 = icmp sgt i64 %1, 3
+  %cond = select i1 %cmp1, i64 %iv, i64 %rdx.07
+  %iv.next = add nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  %idx.0.lcssa = phi i64 [ %rdx_start, %entry ], [ %cond, %for.body ]
+  ret i64 %idx.0.lcssa
+}
+
+define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start, i32 %iv_start ,i32 %n) {
+; CHECK-LABEL: define i32 @select_trunc_non_const_iv_start_signed_guard(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[RDX_START:%.*]], i32 [[IV_START:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[GUARD:%.*]] = icmp slt i32 [[IV_START]], [[N]]
+; CHECK-NEXT:    br i1 [[GUARD]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[IV_START]] to i64
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[RDX_07:%.*]] = phi i32 [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[COND]] = select i1 [[CMP1]], i32 [[TMP2]], i32 [[RDX_07]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[IDX_0_LCSSA:%.*]] = phi i32 [ [[RDX_START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[IDX_0_LCSSA]]
+;
+entry:
+  %guard = icmp slt i32 %iv_start, %n
+  br i1 %guard, label %for.body.preheader, label %exit
+
+for.body.preheader:
+  %0 = sext i32 %iv_start to i64
+  %wide.trip.count = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %0, %for.body.preheader ], [ %iv.next, %for.body ]
+  %rdx.07 = phi i32 [ %rdx_start, %for.body.preheader ], [ %cond, %for.body ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
+  %1 = load i32, ptr %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %1, 3
+  %2 = trunc i64 %iv to i32
+  %cond = select i1 %cmp1, i32 %2, i32 %rdx.07
+  %iv.next = add nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  %idx.0.lcssa = phi i32 [ %rdx_start, %entry ], [ %cond, %for.body ]
+  ret i32 %idx.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index 1ef01e3b793d5..956b70fe92c50 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1-p:16:16:16:16"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
 
 declare void @init(ptr nocapture nofree)
 
@@ -200,104 +200,6 @@ loop_exit:
 }
 
 
-; Test where offset relative to alloca is negative and we shouldn't
-; treat predicated loads as being always dereferenceable.
-define i8 @test_negative_off(i16 %len, ptr %test_base) {
-; CHECK-LABEL: @test_negative_off(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [64638 x i8], align 1
-; CHECK-NEXT:    call void @init(ptr [[ALLOCA]])
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -1000, [[DOTCAST]]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE:%.*]], i16 [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = load i1, ptr [[TMP2]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = load i1, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i1> poison, i1 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i1> [[TMP6]], i1 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP0]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i8> poison, i8 [[TMP10]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
-; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
-; CHECK:       pred.load.if1:
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP1]]
-; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
-; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i8> [[TMP12]], i8 [[TMP15]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
-; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i8> [[TMP17]], <2 x i8> zeroinitializer
-; CHECK-NEXT:    [[TMP18]] = add <2 x i8> [[VEC_PHI]], [[PREDPHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP20:%.*]] = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> [[TMP18]])
-; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ -988, [[MIDDLE_BLOCK]] ], [ -1000, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i8 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
-; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
-; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[IV]]
-; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
-; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
-; CHECK:       pred:
-; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[IV]]
-; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[ADDR]], align 1
-; CHECK-NEXT:    br label [[LATCH]]
-; CHECK:       latch:
-; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
-; CHECK-NEXT:    [[ACCUM_NEXT]] = add i8 [[ACCUM]], [[VAL_PHI]]
-; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i16 [[IV]], -990
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       loop_exit:
-; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i8 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    ret i8 [[ACCUM_NEXT_LCSSA]]
-;
-entry:
-  %alloca = alloca [64638 x i8]
-  call void @init(ptr %alloca)
-  br label %loop
-loop:
-  %iv = phi i16 [ -1000, %entry ], [ %iv.next, %latch ]
-  %accum = phi i8 [ 0, %entry ], [ %accum.next, %latch ]
-  %iv.next = add i16 %iv, 1
-  %test_addr = getelementptr inbounds i1, ptr %test_base, i16 %iv
-  %earlycnd = load i1, ptr %test_addr
-  br i1 %earlycnd, label %pred, label %latch
-pred:
-  %addr = getelementptr i8, ptr %alloca, i16 %iv
-  %val = load i8, ptr %addr
-  br label %latch
-latch:
-  %val.phi = phi i8 [ 0, %loop ], [ %val, %pred ]
-  %accum.next = add i8 %accum, %val.phi
-  %exit = icmp ugt i16 %iv, -990
-  br i1 %exit, label %loop_exit, label %loop
-loop_exit:
-  ret i8 %accum.next
-}
-
-
 define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-LABEL: @loop_requires_scev_predicate(
 ; CHECK-NEXT:  entry:
@@ -326,7 +228,7 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE5:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
@@ -346,16 +248,16 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1
-; CHECK-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5]]
-; CHECK:       pred.store.if4:
+; CHECK-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.if3:
 ; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 [[TMP20]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
 ; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD3]], i32 1
 ; CHECK-NEXT:    [[TMP24:%.*]] = add i32 [[TMP22]], [[TMP23]]
 ; CHECK-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE5]]
-; CHECK:       pred.store.continue5:
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -423,3 +325,430 @@ for.inc:
 exit:
   ret i32 0
 }
+
+
+; Test reverse loops where we should be able to prove loads in predicated blocks
+; are safe to load unconditionally.
+define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
+; CHECK-LABEL: @test_rev_loops_deref_loads(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_SRC]])
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_CMP]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw i32 [[TMP11]], 2
+; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], -1
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = shl nsw i32 [[TMP16]], 2
+; CHECK-NEXT:    store i32 [[TMP17]], ptr [[TMP15]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 1023, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[IV]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[TMP19]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[IV]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP20]], 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[IV]]
+; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %local_dest = alloca [1024 x i32], align 4
+  %local_src = alloca [1024 x i32], align 4
+  %local_cmp = alloca [1024 x i32], align 4
+  call void @init(ptr %local_src)
+  call void @init(ptr %local_cmp)
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 1023, %entry ], [ %iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %cmp3.not = icmp eq i32 %0, 3
+  br i1 %cmp3.not, label %for.inc, label %if.then
+
+if.then:
+  %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %iv
+  %1 = load i32, ptr %arrayidx5, align 4
+  %mul = shl nsw i32 %1, 2
+  %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %iv
+  store i32 %mul, ptr %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %iv.next = add nsw i64 %iv, -1
+  %cmp2.not = icmp eq i64 %iv, 0
+  br i1 %cmp2.not, label %exit, label %for.body
+
+exit:
+  call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)
+  ret void
+}
+
+
+; Test reverse loops where we *cannot* prove loads in predicated blocks are safe
+; to load unconditionally.
+define void @test_rev_loops_non_deref_loads(ptr nocapture noundef writeonly %dest) {
+; CHECK-LABEL: @test_rev_loops_non_deref_loads(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_SRC]])
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_CMP]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1023, i64 1022>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nsw i32 [[TMP10]], 2
+; CHECK-NEXT:    store i32 [[TMP13]], ptr [[TMP12]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = shl nsw i32 [[TMP17]], 2
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP19]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 -2)
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 1023, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[OFF:%.*]] = add i64 [[IV]], -1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFF]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[TMP22]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[OFF]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP23]], 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[OFF]]
+; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %local_dest = alloca [1024 x i32], align 4
+  %local_src = alloca [1024 x i32], align 4
+  %local_cmp = alloca [1024 x i32], align 4
+  call void @init(ptr %local_src)
+  call void @init(ptr %local_cmp)
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 1023, %entry ], [ %iv.next, %for.inc ]
+  %off = add i64 %iv, -1
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %off
+  %0 = load i32, ptr %arrayidx, align 4
+  %cmp3.not = icmp eq i32 %0, 3
+  br i1 %cmp3.not, label %for.inc, label %if.then
+
+if.then:
+  %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %off
+  %1 = load i32, ptr %arrayidx5, align 4
+  %mul = shl nsw i32 %1, 2
+  %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %off
+  store i32 %mul, ptr %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %iv.next = add nsw i64 %iv, -1
+  %cmp2.not = icmp eq i64 %iv, 0
+  br i1 %cmp2.not, label %exit, label %for.body
+
+exit:
+  call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)
+  ret void
+}
+
+
+; Test a loop with a positive step recurrence that has a strided access
+define i16 @test_strided_access(i64 %len, ptr %test_base) {
+; CHECK-LABEL: @test_strided_access(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [163840 x i16], align 4
+; CHECK-NEXT:    call void @init(ptr [[ALLOCA]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[TMP6]], align 2
+; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[TMP8]], align 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i16> poison, i16 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i16> [[TMP11]], i16 [[TMP10]], i32 1
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP12]], <2 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP15:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP13]])
+; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i16 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[L_T:%.*]] = load i8, ptr [[TEST_ADDR]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[L_T]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[IV_STRIDE:%.*]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[IV_STRIDE]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i16, ptr [[ADDR]], align 2
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i16 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i16 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp eq i64 [[IV]], 4095
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i16 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i16 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [163840 x i16], align 4
+  call void @init(ptr %alloca)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i16 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i8, ptr %test_base, i64 %iv
+  %l.t = load i8, ptr %test_addr
+  %cmp = icmp sge i8 %l.t, 0
+  br i1 %cmp, label %pred, label %latch
+pred:
+  %iv.stride = mul i64 %iv, 2
+  %addr = getelementptr inbounds i16, ptr %alloca, i64 %iv.stride
+  %val = load i16, ptr %addr, align 2
+  br label %latch
+latch:
+  %val.phi = phi i16 [0, %loop], [%val, %pred]
+  %accum.next = add i16 %accum, %val.phi
+  %exit = icmp eq i64 %iv, 4095
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i16 %accum.next
+}
+
+
+; Test a loop with a negative step recurrence that has a strided access
+define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly %dest) {
+; CHECK-LABEL: @test_rev_loops_strided_deref_loads(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_SRC]])
+; CHECK-NEXT:    call void @init(ptr [[LOCAL_CMP]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 511, i64 510>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 511, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nsw i32 [[TMP11]], 2
+; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP14]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], -1
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shl nsw i32 [[TMP12]], 2
+; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP18]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 -2)
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, [[MIDDLE_BLOCK]] ], [ 511, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[IV]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[TMP21]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IV_STRIDED:%.*]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[IV_STRIDED]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP22]], 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[IV]]
+; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %local_dest = alloca [1024 x i32], align 4
+  %local_src = alloca [1024 x i32], align 4
+  %local_cmp = alloca [1024 x i32], align 4
+  call void @init(ptr %local_src)
+  call void @init(ptr %local_cmp)
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 511, %entry ], [ %iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %cmp3.not = icmp eq i32 %0, 3
+  br i1 %cmp3.not, label %for.inc, label %if.then
+
+if.then:
+  %iv.strided = mul i64 %iv, 2
+  %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %iv.strided
+  %1 = load i32, ptr %arrayidx5, align 4
+  %mul = shl nsw i32 %1, 2
+  %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %iv
+  store i32 %mul, ptr %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %iv.next = add nsw i64 %iv, -1
+  %cmp2.not = icmp eq i64 %iv, 0
+  br i1 %cmp2.not, label %exit, label %for.body
+
+exit:
+  call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
new file mode 100644
index 0000000000000..1dd526df503bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1-p:16:16:16:16"
+
+declare void @init(ptr nocapture nofree)
+
+
+; Test where offset relative to alloca is negative and we shouldn't
+; treat predicated loads as being always dereferenceable.
+define i8 @test_negative_off(i16 %len, ptr %test_base) {
+; CHECK-LABEL: @test_negative_off(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [64638 x i8], align 1
+; CHECK-NEXT:    call void @init(ptr [[ALLOCA]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -1000, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE:%.*]], i16 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i1, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i1, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i1> poison, i1 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i1> [[TMP6]], i1 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i8> poison, i8 [[TMP10]], i32 0
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.if1:
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i8> [[TMP12]], i8 [[TMP15]], i32 1
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
+; CHECK:       pred.load.continue2:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i8> [[TMP17]], <2 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP18]] = add <2 x i8> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP20:%.*]] = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> [[TMP18]])
+; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ -988, [[MIDDLE_BLOCK]] ], [ -1000, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i8 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i8, ptr [[ALLOCA]], i16 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[ADDR]], align 1
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i8 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i16 [[IV]], -990
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i8 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i8 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %alloca = alloca [64638 x i8]
+  call void @init(ptr %alloca)
+  br label %loop
+loop:
+  %iv = phi i16 [ -1000, %entry ], [ %iv.next, %latch ]
+  %accum = phi i8 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i16 %iv, 1
+  %test_addr = getelementptr inbounds i1, ptr %test_base, i16 %iv
+  %earlycnd = load i1, ptr %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr i8, ptr %alloca, i16 %iv
+  %val = load i8, ptr %addr
+  br label %latch
+latch:
+  %val.phi = phi i8 [ 0, %loop ], [ %val, %pred ]
+  %accum.next = add i8 %accum, %val.phi
+  %exit = icmp ugt i16 %iv, -990
+  br i1 %exit, label %loop_exit, label %loop
+loop_exit:
+  ret i8 %accum.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/loop-form.ll b/llvm/test/Transforms/LoopVectorize/loop-form.ll
index b59cb66b11acf..730d488119d13 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-form.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-form.ll
@@ -76,7 +76,7 @@ define void @bottom_tested(ptr %p, i32 %n) {
 ; TAILFOLD-NEXT:    store i16 0, ptr [[TMP8]], align 4
 ; TAILFOLD-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; TAILFOLD:       pred.store.continue2:
-; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; TAILFOLD-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; TAILFOLD-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; TAILFOLD-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
index 0bffac2422757..ffeb3b19d11c4 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
@@ -32,8 +32,8 @@ define void @vector_gep(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
-; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds nuw ptr, ptr [[A]], i64 [[I]]
 ; CHECK-NEXT:    store ptr [[VAR0]], ptr [[VAR1]], align 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
@@ -206,7 +206,7 @@ define void @no_gep_or_bitcast(ptr noalias %a, i64 %n) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw ptr, ptr [[A]], i64 [[I]]
 ; CHECK-NEXT:    [[VAR1:%.*]] = load ptr, ptr [[VAR0]], align 8
 ; CHECK-NEXT:    store i32 0, ptr [[VAR1]], align 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
index c3d8a75acaaaf..d1ad7e3f4fc0d 100644
--- a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
@@ -64,7 +64,7 @@ define void @maxvf3() {
 ; CHECK-NEXT:    store i8 7, ptr [[TMP13]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
index 27dcd0cc0463a..39b162cae6ddc 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
@@ -23,10 +23,10 @@ define i32 @main() #0 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [40000 x i8], ptr addrspace(1) @Y, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [40000 x i8], ptr addrspace(1) @Y, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr addrspace(1) [[TMP0]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [40000 x i8], ptr @X, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [40000 x i8], ptr @X, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <4 x i8> [[TMP1]], ptr [[TMP2]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 40000
diff --git a/llvm/test/Transforms/LoopVectorize/non-const-n.ll b/llvm/test/Transforms/LoopVectorize/non-const-n.ll
index 295bf9111329c..60077710b01b3 100644
--- a/llvm/test/Transforms/LoopVectorize/non-const-n.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-const-n.ll
@@ -19,12 +19,12 @@ define void @example1(i32 %n) nounwind uwtable ssp {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @c, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [2048 x i32], ptr @a, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
index 305906d9d1ef1..9f5e5f3980e6f 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
@@ -234,8 +234,8 @@ define dso_local signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n)
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -473,6 +473,7 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i8 84, [[INDUCTION_IV]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[INDUCTION_IV]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> <i8 0, i8 1, i8 2, i8 3>, [[DOTSPLAT]]
@@ -480,7 +481,6 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i8 [[INDUCTION_IV]], 4
 ; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i8 84, [[INDUCTION_IV]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -563,6 +563,7 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK-PROFITABLE-BY-DEFAULT:       vector.main.loop.iter.check:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vector.ph:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IND_END:%.*]] = mul i8 84, [[INDUCTION_IV]]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[INDUCTION_IV]], i64 0
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP2:%.*]] = mul <4 x i8> <i8 0, i8 1, i8 2, i8 3>, [[DOTSPLAT]]
@@ -570,7 +571,6 @@ define void @induction_resume_value_requires_non_trivial_scev_expansion(ptr %dst
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP3:%.*]] = mul i8 [[INDUCTION_IV]], 4
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[TMP3]], i64 0
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IND_END:%.*]] = mul i8 84, [[INDUCTION_IV]]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vector.body:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -696,11 +696,10 @@ define void @f4(ptr noalias %A, i32 signext %n) {
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3>
@@ -767,11 +766,10 @@ define void @f4(ptr noalias %A, i32 signext %n) {
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.ph:
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_VEC3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF2]]
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP5:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP5:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1>
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 0f5dcca81d371..738f265b89d65 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; This test verifies that the loop vectorizer will NOT produce a tail
 ; loop with the optimize for size or the minimize size attributes.
 ; REQUIRES: asserts
@@ -10,9 +11,57 @@ target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
 @tab = common global [32 x i8] zeroinitializer, align 1
 
 define i32 @foo_optsize() #0 {
-; CHECK-LABEL: @foo_optsize(
-; CHECK-NOT: <2 x i8>
-; CHECK-NOT: <4 x i8>
+; CHECK-LABEL: define i32 @foo_optsize(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; CHECK-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret i32 0
+;
+; PGSO-LABEL: define i32 @foo_optsize(
+; PGSO-SAME: ) #[[ATTR0:[0-9]+]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; PGSO-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; PGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; PGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; PGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret i32 0
+;
+; NPGSO-LABEL: define i32 @foo_optsize(
+; NPGSO-SAME: ) #[[ATTR0:[0-9]+]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; NPGSO-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; NPGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; NPGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; NPGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret i32 0
+;
 
 entry:
   br label %for.body
@@ -35,10 +84,57 @@ for.end:                                          ; preds = %for.body
 attributes #0 = { optsize }
 
 define i32 @foo_minsize() #1 {
-; CHECK-LABEL: @foo_minsize(
-; CHECK-NOT: <2 x i8>
-; CHECK-NOT: <4 x i8>
-; CHECK-LABEL: @foo_pgso(
+; CHECK-LABEL: define i32 @foo_minsize(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; CHECK-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret i32 0
+;
+; PGSO-LABEL: define i32 @foo_minsize(
+; PGSO-SAME: ) #[[ATTR1:[0-9]+]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; PGSO-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; PGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; PGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; PGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret i32 0
+;
+; NPGSO-LABEL: define i32 @foo_minsize(
+; NPGSO-SAME: ) #[[ATTR1:[0-9]+]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; NPGSO-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; NPGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; NPGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; NPGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret i32 0
+;
 
 entry:
   br label %for.body
@@ -61,10 +157,76 @@ for.end:                                          ; preds = %for.body
 attributes #1 = { minsize }
 
 define i32 @foo_pgso() !prof !14 {
-; PGSO-LABEL: @foo_pgso(
-; PGSO-NOT: <{{[0-9]+}} x i8>
-; NPGSO-LABEL: @foo_pgso(
-; NPGSO: <{{[0-9]+}} x i8>
+; CHECK-LABEL: define i32 @foo_pgso(
+; CHECK-SAME: ) !prof [[PROF14:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; CHECK-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret i32 0
+;
+; PGSO-LABEL: define i32 @foo_pgso(
+; PGSO-SAME: ) !prof [[PROF14:![0-9]+]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; PGSO-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; PGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; PGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; PGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; PGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret i32 0
+;
+; NPGSO-LABEL: define i32 @foo_pgso(
+; NPGSO-SAME: ) !prof [[PROF14:![0-9]+]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; NPGSO-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[TMP0]]
+; NPGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; NPGSO-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; NPGSO-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; NPGSO-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i8> splat (i8 2), <4 x i8> splat (i8 1)
+; NPGSO-NEXT:    store <4 x i8> [[TMP4]], ptr [[TMP2]], align 1
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; NPGSO-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 200
+; NPGSO-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 200, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
+; NPGSO-NEXT:    [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP6]], 0
+; NPGSO-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; NPGSO-NEXT:    store i8 [[DOT]], ptr [[ARRAYIDX]], align 1
+; NPGSO-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; NPGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret i32 0
+;
 
 entry:
   br label %for.body
@@ -91,19 +253,124 @@ for.end:                                          ; preds = %for.body
 
 define void @pr43371() optsize {
 ;
-; CHECK-LABEL: @pr43371
-; CHECK-NOT:   vector.scevcheck
+; CHECK-LABEL: define void @pr43371(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]
+; CHECK-NEXT:    store i16 0, ptr [[TMP3]], align 1
+; CHECK-NEXT:    store i16 0, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY29:.*]]
+; CHECK:       [[FOR_COND_CLEANUP28]]:
+; CHECK-NEXT:    unreachable
+; CHECK:       [[FOR_BODY29]]:
+; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; CHECK-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; CHECK-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; CHECK-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; CHECK-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; CHECK-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP18:![0-9]+]]
+;
+; PGSO-LABEL: define void @pr43371(
+; PGSO-SAME: ) #[[ATTR0]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PGSO:       [[VECTOR_PH]]:
+; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PGSO:       [[VECTOR_BODY]]:
+; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
+; PGSO-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
+; PGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; PGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; PGSO-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; PGSO-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]
+; PGSO-NEXT:    store i16 0, ptr [[TMP3]], align 1
+; PGSO-NEXT:    store i16 0, ptr [[TMP5]], align 1
+; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; PGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; PGSO-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; PGSO:       [[MIDDLE_BLOCK]]:
+; PGSO-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; PGSO:       [[SCALAR_PH]]:
+; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; PGSO-NEXT:    br label %[[FOR_BODY29:.*]]
+; PGSO:       [[FOR_COND_CLEANUP28]]:
+; PGSO-NEXT:    unreachable
+; PGSO:       [[FOR_BODY29]]:
+; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; PGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; PGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; PGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; PGSO-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; PGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; PGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; PGSO-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP18:![0-9]+]]
+;
+; NPGSO-LABEL: define void @pr43371(
+; NPGSO-SAME: ) #[[ATTR0]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
+; NPGSO-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
+; NPGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; NPGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; NPGSO-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; NPGSO-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]
+; NPGSO-NEXT:    store i16 0, ptr [[TMP3]], align 1
+; NPGSO-NEXT:    store i16 0, ptr [[TMP5]], align 1
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; NPGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; NPGSO-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    br label %[[FOR_BODY29:.*]]
+; NPGSO:       [[FOR_COND_CLEANUP28]]:
+; NPGSO-NEXT:    unreachable
+; NPGSO:       [[FOR_BODY29]]:
+; NPGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; NPGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; NPGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; NPGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; NPGSO-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; NPGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; NPGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; NPGSO-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP20:![0-9]+]]
 ;
 ; We do not want to generate SCEV predicates when optimising for size, because
 ; that will lead to extra code generation such as the SCEV overflow runtime
 ; checks. Not generating SCEV predicates can still result in vectorisation as
 ; the non-consecutive loads/stores can be scalarized:
-;
-; CHECK: vector.body:
-; CHECK: store i16 0, ptr %{{.*}}, align 1
-; CHECK: store i16 0, ptr %{{.*}}, align 1
-; CHECK: br i1 {{.*}}, label %vector.body
-;
 entry:
   br label %for.body29
 
@@ -123,19 +390,123 @@ for.body29:
 
 define void @pr43371_pgso() !prof !14 {
 ;
-; CHECK-LABEL: @pr43371_pgso
-; CHECK-NOT:   vector.scevcheck
+; CHECK-LABEL: define void @pr43371_pgso(
+; CHECK-SAME: ) !prof [[PROF14]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]
+; CHECK-NEXT:    store i16 0, ptr [[TMP3]], align 1
+; CHECK-NEXT:    store i16 0, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY29:.*]]
+; CHECK:       [[FOR_COND_CLEANUP28]]:
+; CHECK-NEXT:    unreachable
+; CHECK:       [[FOR_BODY29]]:
+; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; CHECK-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; CHECK-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; CHECK-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; CHECK-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; CHECK-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP20:![0-9]+]]
+;
+; PGSO-LABEL: define void @pr43371_pgso(
+; PGSO-SAME: ) !prof [[PROF14]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PGSO:       [[VECTOR_PH]]:
+; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PGSO:       [[VECTOR_BODY]]:
+; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT:    [[TMP0:%.*]] = add <2 x i16> undef, [[VEC_IND]]
+; PGSO-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>
+; PGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; PGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; PGSO-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; PGSO-NEXT:    [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]
+; PGSO-NEXT:    store i16 0, ptr [[TMP3]], align 1
+; PGSO-NEXT:    store i16 0, ptr [[TMP5]], align 1
+; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; PGSO-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; PGSO-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; PGSO:       [[MIDDLE_BLOCK]]:
+; PGSO-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; PGSO:       [[SCALAR_PH]]:
+; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; PGSO-NEXT:    br label %[[FOR_BODY29:.*]]
+; PGSO:       [[FOR_COND_CLEANUP28]]:
+; PGSO-NEXT:    unreachable
+; PGSO:       [[FOR_BODY29]]:
+; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; PGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; PGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; PGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; PGSO-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; PGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; PGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; PGSO-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP20:![0-9]+]]
+;
+; NPGSO-LABEL: define void @pr43371_pgso(
+; NPGSO-SAME: ) !prof [[PROF14]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; NPGSO:       [[VECTOR_SCEVCHECK]]:
+; NPGSO-NEXT:    br i1 undef, label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
+; NPGSO-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
+; NPGSO-NEXT:    [[TMP1:%.*]] = add i16 undef, [[TMP0]]
+; NPGSO-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; NPGSO-NEXT:    [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]
+; NPGSO-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[TMP3]], i32 0
+; NPGSO-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP4]], align 1
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NPGSO-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756
+; NPGSO-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP28:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 756, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; NPGSO-NEXT:    br label %[[FOR_BODY29:.*]]
+; NPGSO:       [[FOR_COND_CLEANUP28]]:
+; NPGSO-NEXT:    unreachable
+; NPGSO:       [[FOR_BODY29]]:
+; NPGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; NPGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
+; NPGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
+; NPGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
+; NPGSO-NEXT:    store i16 0, ptr [[ARRAYIDX35]], align 1
+; NPGSO-NEXT:    [[INC37]] = add i16 [[I24_0170]], 1
+; NPGSO-NEXT:    [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756
+; NPGSO-NEXT:    br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP22:![0-9]+]]
 ;
 ; We do not want to generate SCEV predicates when optimising for size, because
 ; that will lead to extra code generation such as the SCEV overflow runtime
 ; checks. Not generating SCEV predicates can still result in vectorisation as
 ; the non-consecutive loads/stores can be scalarized:
-;
-; CHECK: vector.body:
-; CHECK: store i16 0, ptr %{{.*}}, align 1
-; CHECK: store i16 0, ptr %{{.*}}, align 1
-; CHECK: br i1 {{.*}}, label %vector.body
-;
 entry:
   br label %for.body29
 
@@ -157,20 +528,47 @@ for.body29:
 ;
 define i32 @pr45526() optsize {
 ;
-; CHECK-LABEL: @pr45526
-; CHECK-NEXT: entry:
-; CHECK-NEXT:   br label %loop
-; CHECK-EMPTY:
-; CHECK-NEXT: loop:
-; CHECK-NEXT:   %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
-; CHECK-NEXT:   %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
-; CHECK-NEXT:   %pivPlus1 = add nuw nsw i32 %piv, 1
-; CHECK-NEXT:   %cond = icmp ult i32 %piv, 510
-; CHECK-NEXT:   br i1 %cond, label %loop, label %exit
-; CHECK-EMPTY:
-; CHECK-NEXT: exit:
-; CHECK-NEXT:   %for.lcssa = phi i32 [ %for, %loop ]
-; CHECK-NEXT:   ret i32 %for.lcssa
+; CHECK-LABEL: define i32 @pr45526(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PIV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ 5, %[[ENTRY]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[FOR_LCSSA]]
+;
+; PGSO-LABEL: define i32 @pr45526(
+; PGSO-SAME: ) #[[ATTR0]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[LOOP:.*]]
+; PGSO:       [[LOOP]]:
+; PGSO-NEXT:    [[PIV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; PGSO-NEXT:    [[FOR:%.*]] = phi i32 [ 5, %[[ENTRY]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; PGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; PGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; PGSO-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; PGSO:       [[EXIT]]:
+; PGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]
+; PGSO-NEXT:    ret i32 [[FOR_LCSSA]]
+;
+; NPGSO-LABEL: define i32 @pr45526(
+; NPGSO-SAME: ) #[[ATTR0]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br label %[[LOOP:.*]]
+; NPGSO:       [[LOOP]]:
+; NPGSO-NEXT:    [[PIV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; NPGSO-NEXT:    [[FOR:%.*]] = phi i32 [ 5, %[[ENTRY]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; NPGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; NPGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; NPGSO-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; NPGSO:       [[EXIT]]:
+; NPGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]
+; NPGSO-NEXT:    ret i32 [[FOR_LCSSA]]
 ;
 entry:
   br label %loop
@@ -188,20 +586,66 @@ exit:
 
 define i32 @pr45526_pgso() !prof !14 {
 ;
-; CHECK-LABEL: @pr45526_pgso
-; CHECK-NEXT: entry:
-; CHECK-NEXT:   br label %loop
-; CHECK-EMPTY:
-; CHECK-NEXT: loop:
-; CHECK-NEXT:   %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
-; CHECK-NEXT:   %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
-; CHECK-NEXT:   %pivPlus1 = add nuw nsw i32 %piv, 1
-; CHECK-NEXT:   %cond = icmp ult i32 %piv, 510
-; CHECK-NEXT:   br i1 %cond, label %loop, label %exit
-; CHECK-EMPTY:
-; CHECK-NEXT: exit:
-; CHECK-NEXT:   %for.lcssa = phi i32 [ %for, %loop ]
-; CHECK-NEXT:   ret i32 %for.lcssa
+; CHECK-LABEL: define i32 @pr45526_pgso(
+; CHECK-SAME: ) !prof [[PROF14]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PIV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ 5, %[[ENTRY]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[FOR_LCSSA]]
+;
+; PGSO-LABEL: define i32 @pr45526_pgso(
+; PGSO-SAME: ) !prof [[PROF14]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[LOOP:.*]]
+; PGSO:       [[LOOP]]:
+; PGSO-NEXT:    [[PIV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; PGSO-NEXT:    [[FOR:%.*]] = phi i32 [ 5, %[[ENTRY]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; PGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; PGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; PGSO-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; PGSO:       [[EXIT]]:
+; PGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]
+; PGSO-NEXT:    ret i32 [[FOR_LCSSA]]
+;
+; NPGSO-LABEL: define i32 @pr45526_pgso(
+; NPGSO-SAME: ) !prof [[PROF14]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[TMP0]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; NPGSO-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 508
+; NPGSO-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; NPGSO-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; NPGSO-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 508, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 5, %[[ENTRY]] ]
+; NPGSO-NEXT:    br label %[[LOOP:.*]]
+; NPGSO:       [[LOOP]]:
+; NPGSO-NEXT:    [[PIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]
+; NPGSO-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[PIVPLUS1]], %[[LOOP]] ]
+; NPGSO-NEXT:    [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1
+; NPGSO-NEXT:    [[COND:%.*]] = icmp ult i32 [[PIV]], 510
+; NPGSO-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
+; NPGSO:       [[EXIT]]:
+; NPGSO-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; NPGSO-NEXT:    ret i32 [[FOR_LCSSA]]
 ;
 entry:
   br label %loop
@@ -222,51 +666,153 @@ exit:
 
 ; NOTE: Some assertions have been autogenerated by utils/update_test_checks.py
 define void @stride1(ptr noalias %B, i32 %BStride) optsize {
-; CHECK-LABEL: @stride1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE:%.*]], i64 0
+; CHECK-LABEL: define void @stride1(
+; CHECK-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; CHECK:       pred.store.if:
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]
 ; CHECK-NEXT:    store i16 42, ptr [[TMP4]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
-; CHECK:       pred.store.continue:
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
-; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
-; CHECK:       pred.store.if1:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_IF1]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]
 ; CHECK-NEXT:    store i16 42, ptr [[TMP7]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
-; CHECK:       pred.store.continue2:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK:       for.end:
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1026, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; CHECK-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; CHECK-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    ret void
 ;
-; PGSO-LABEL: @stride1(
-; PGSO-NEXT:  entry:
-; PGSO-NEXT:    br i1 false, label %scalar.ph, label %vector.ph
+; PGSO-LABEL: define void @stride1(
+; PGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PGSO:       [[VECTOR_PH]]:
+; PGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
+; PGSO-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PGSO:       [[VECTOR_BODY]]:
+; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; PGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; PGSO-NEXT:    [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)
+; PGSO-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; PGSO-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; PGSO:       [[PRED_STORE_IF]]:
+; PGSO-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; PGSO-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]
+; PGSO-NEXT:    store i16 42, ptr [[TMP4]], align 4
+; PGSO-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; PGSO:       [[PRED_STORE_CONTINUE]]:
+; PGSO-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; PGSO-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; PGSO:       [[PRED_STORE_IF1]]:
+; PGSO-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; PGSO-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]
+; PGSO-NEXT:    store i16 42, ptr [[TMP7]], align 4
+; PGSO-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; PGSO:       [[PRED_STORE_CONTINUE2]]:
+; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; PGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; PGSO-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
+; PGSO-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; PGSO:       [[MIDDLE_BLOCK]]:
+; PGSO-NEXT:    br i1 true, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; PGSO:       [[SCALAR_PH]]:
+; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1026, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; PGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; PGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; PGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; PGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; PGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret void
+;
+; NPGSO-LABEL: define void @stride1(
+; NPGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
+; NPGSO-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; NPGSO-NEXT:    [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)
+; NPGSO-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NPGSO-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; NPGSO-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; NPGSO:       [[PRED_STORE_IF]]:
+; NPGSO-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; NPGSO-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]
+; NPGSO-NEXT:    store i16 42, ptr [[TMP4]], align 4
+; NPGSO-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; NPGSO:       [[PRED_STORE_CONTINUE]]:
+; NPGSO-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; NPGSO-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; NPGSO:       [[PRED_STORE_IF1]]:
+; NPGSO-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; NPGSO-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]
+; NPGSO-NEXT:    store i16 42, ptr [[TMP7]], align 4
+; NPGSO-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; NPGSO:       [[PRED_STORE_CONTINUE2]]:
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; NPGSO-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
+; NPGSO-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    br i1 true, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1026, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; NPGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; NPGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; NPGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; NPGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; NPGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret void
 ;
-; NPGSO-LABEL: @stride1(
-; NPGSO-NEXT:  entry:
-; NPGSO-NEXT:    br i1 false, label %scalar.ph, label %vector.ph
 
 entry:
   br label %for.body
@@ -287,14 +833,108 @@ for.end:
 ; Vectorize with versioning for unit stride for PGSO and enabled vectorization.
 ;
 define void @stride1_pgso(ptr noalias %B, i32 %BStride) !prof !14 {
-; CHECK-LABEL: @stride1_pgso(
-; CHECK: vector.body
+; CHECK-LABEL: define void @stride1_pgso(
+; CHECK-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1
+; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    store <2 x i16> splat (i16 42), ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; CHECK-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; CHECK-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+; PGSO-LABEL: define void @stride1_pgso(
+; PGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; PGSO:       [[VECTOR_SCEVCHECK]]:
+; PGSO-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1
+; PGSO-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; PGSO:       [[VECTOR_PH]]:
+; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PGSO:       [[VECTOR_BODY]]:
+; PGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; PGSO-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]
+; PGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
+; PGSO-NEXT:    store <2 x i16> splat (i16 42), ptr [[TMP2]], align 4
+; PGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; PGSO-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; PGSO-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; PGSO:       [[MIDDLE_BLOCK]]:
+; PGSO-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; PGSO:       [[SCALAR_PH]]:
+; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; PGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; PGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; PGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; PGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; PGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret void
 ;
-; PGSO-LABEL: @stride1_pgso(
-; PGSO: vector.body
+; NPGSO-LABEL: define void @stride1_pgso(
+; NPGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; NPGSO:       [[VECTOR_SCEVCHECK]]:
+; NPGSO-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1
+; NPGSO-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; NPGSO:       [[VECTOR_PH]]:
+; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NPGSO:       [[VECTOR_BODY]]:
+; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; NPGSO-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]
+; NPGSO-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
+; NPGSO-NEXT:    store <2 x i16> splat (i16 42), ptr [[TMP2]], align 4
+; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NPGSO-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; NPGSO-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; NPGSO:       [[MIDDLE_BLOCK]]:
+; NPGSO-NEXT:    br i1 false, label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NPGSO:       [[SCALAR_PH]]:
+; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; NPGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
+; NPGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
+; NPGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
+; NPGSO-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; NPGSO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025
+; NPGSO-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret void
 ;
-; NPGSO-LABEL: @stride1_pgso(
-; NPGSO: vector.body
 
 entry:
   br label %for.body
@@ -314,14 +954,55 @@ for.end:
 
 ; PR46652: Check that the need for stride==1 check prevents vectorizing a loop
 ; having tiny trip count, when compiling w/o -Os/-Oz.
-; CHECK-LABEL: @pr46652
-; CHECK-NOT: vector.scevcheck
-; CHECK-NOT: vector.body
-; CHECK-LABEL: for.body
 
 @g = external global [1 x i16], align 1
 
 define void @pr46652(i16 %stride) {
+; CHECK-LABEL: define void @pr46652(
+; CHECK-SAME: i16 [[STRIDE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1
+; CHECK-NEXT:    [[INC9]] = add nuw nsw i16 [[L1_02]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+; PGSO-LABEL: define void @pr46652(
+; PGSO-SAME: i16 [[STRIDE:%.*]]) {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]
+; PGSO-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]
+; PGSO-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1
+; PGSO-NEXT:    [[INC9]] = add nuw nsw i16 [[L1_02]], 1
+; PGSO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16
+; PGSO-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; PGSO:       [[FOR_END]]:
+; PGSO-NEXT:    ret void
+;
+; NPGSO-LABEL: define void @pr46652(
+; NPGSO-SAME: i16 [[STRIDE:%.*]]) {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]
+; NPGSO-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]
+; NPGSO-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1
+; NPGSO-NEXT:    [[INC9]] = add nuw nsw i16 [[L1_02]], 1
+; NPGSO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16
+; NPGSO-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; NPGSO:       [[FOR_END]]:
+; NPGSO-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -341,8 +1022,57 @@ for.end:                                        ; preds = %for.body
 ; Make sure we do not crash while building the VPlan for the loop with the
 ; select below.
 define i32 @PR48142(ptr %ptr.start, ptr %ptr.end) optsize {
-; CHECK-LABEL: PR48142
-; CHECK-NOT: vector.body
+; CHECK-LABEL: define i32 @PR48142(
+; CHECK-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[I_014]], 99
+; CHECK-NEXT:    [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]
+; CHECK-NEXT:    store i32 0, ptr [[PTR_IV]], align 4
+; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 1
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+; PGSO-LABEL: define i32 @PR48142(
+; PGSO-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {
+; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; PGSO:       [[FOR_BODY]]:
+; PGSO-NEXT:    [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[I_014]], 99
+; PGSO-NEXT:    [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]
+; PGSO-NEXT:    store i32 0, ptr [[PTR_IV]], align 4
+; PGSO-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 1
+; PGSO-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]
+; PGSO-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; PGSO:       [[EXIT]]:
+; PGSO-NEXT:    [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; PGSO-NEXT:    ret i32 [[RES]]
+;
+; NPGSO-LABEL: define i32 @PR48142(
+; NPGSO-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {
+; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
+; NPGSO:       [[FOR_BODY]]:
+; NPGSO-NEXT:    [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[I_014]], 99
+; NPGSO-NEXT:    [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]
+; NPGSO-NEXT:    store i32 0, ptr [[PTR_IV]], align 4
+; NPGSO-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 1
+; NPGSO-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]
+; NPGSO-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; NPGSO:       [[EXIT]]:
+; NPGSO-NEXT:    [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; NPGSO-NEXT:    ret i32 [[RES]]
+;
 entry:
   br label %for.body
 
@@ -379,3 +1109,44 @@ exit:
 !14 = !{!"function_entry_count", i64 0}
 !15 = distinct !{!15, !16}
 !16 = !{!"llvm.loop.vectorize.enable", i1 true}
+;.
+; CHECK: [[PROF14]] = !{!"function_entry_count", i64 0}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META17]], [[META16]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META17]], [[META16]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]], [[META17]]}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META16]]}
+; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]], [[META17]]}
+; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META16]]}
+;.
+; PGSO: [[PROF14]] = !{!"function_entry_count", i64 0}
+; PGSO: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; PGSO: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
+; PGSO: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
+; PGSO: [[LOOP18]] = distinct !{[[LOOP18]], [[META17]], [[META16]]}
+; PGSO: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}
+; PGSO: [[LOOP20]] = distinct !{[[LOOP20]], [[META17]], [[META16]]}
+; PGSO: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]], [[META17]]}
+; PGSO: [[LOOP22]] = distinct !{[[LOOP22]], [[META17]], [[META16]]}
+; PGSO: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]], [[META17]]}
+; PGSO: [[LOOP24]] = distinct !{[[LOOP24]], [[META16]]}
+;.
+; NPGSO: [[PROF14]] = !{!"function_entry_count", i64 0}
+; NPGSO: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; NPGSO: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
+; NPGSO: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
+; NPGSO: [[LOOP18]] = distinct !{[[LOOP18]], [[META17]], [[META16]]}
+; NPGSO: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}
+; NPGSO: [[LOOP20]] = distinct !{[[LOOP20]], [[META17]], [[META16]]}
+; NPGSO: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]], [[META17]]}
+; NPGSO: [[LOOP22]] = distinct !{[[LOOP22]], [[META16]]}
+; NPGSO: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]], [[META17]]}
+; NPGSO: [[LOOP24]] = distinct !{[[LOOP24]], [[META17]], [[META16]]}
+; NPGSO: [[LOOP25]] = distinct !{[[LOOP25]], [[META16]], [[META17]]}
+; NPGSO: [[LOOP26]] = distinct !{[[LOOP26]], [[META17]], [[META16]]}
+; NPGSO: [[LOOP27]] = distinct !{[[LOOP27]], [[META16]], [[META17]]}
+; NPGSO: [[LOOP28]] = distinct !{[[LOOP28]], [[META16]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
index ee6975fa0c9e6..c301ef3c5319a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
@@ -52,7 +52,7 @@ define void @pr45679(ptr %A) optsize {
 ; CHECK-NEXT:    store i32 13, ptr [[TMP12]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -114,7 +114,7 @@ define void @pr45679(ptr %A) optsize {
 ; VF2UF2-NEXT:    store i32 13, ptr [[TMP13]], align 1
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE7]]
 ; VF2UF2:       pred.store.continue6:
-; VF2UF2-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; VF2UF2-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; VF2UF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2)
 ; VF2UF2-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; VF2UF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -172,7 +172,7 @@ define void @pr45679(ptr %A) optsize {
 ; VF1UF4-NEXT:    store i32 13, ptr [[TMP11]], align 1
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; VF1UF4:       pred.store.continue6:
-; VF1UF4-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; VF1UF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; VF1UF4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; VF1UF4-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VF1UF4:       middle.block:
@@ -251,7 +251,7 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK-NEXT:    store i64 [[TMP19]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -318,7 +318,7 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2-NEXT:    store i64 [[TMP20]], ptr [[B]], align 8
 ; VF2UF2-NEXT:    br label [[PRED_STORE_CONTINUE7]]
 ; VF2UF2:       pred.store.continue6:
-; VF2UF2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VF2UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF2UF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], splat (i64 2)
 ; VF2UF2-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; VF2UF2-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -381,7 +381,7 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4-NEXT:    store i64 [[TMP18]], ptr [[B]], align 8
 ; VF1UF4-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; VF1UF4:       pred.store.continue6:
-; VF1UF4-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VF1UF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF1UF4-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; VF1UF4-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VF1UF4:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
index f0220f5e766b2..1e1b03b6cd8cf 100644
--- a/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
@@ -28,7 +28,7 @@ define void @test(i16 %x, i64 %y, ptr %ptr) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    store i32 0, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
index d7f5f646f1873..705152662be24 100644
--- a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
@@ -56,7 +56,7 @@ define dso_local i16 @reverse_interleave_load_fold_mask() optsize {
 ; CHECK-NEXT:    [[TMP24:%.*]] = add nsw <2 x i16> [[TMP22]], [[TMP23]]
 ; CHECK-NEXT:    [[TMP25]] = add <2 x i16> [[VEC_PHI]], [[TMP24]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[TMP25]], <2 x i16> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i32 [[INDEX_NEXT]], 42
 ; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
index afb7d87bd1752..d8b6a4e758851 100644
--- a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
@@ -53,7 +53,7 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK-NEXT:    [[N_MOD_VF8:%.*]] = urem i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF8]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY11:%.*]]
-; CHECK:       vector.body10:
+; CHECK:       vector.body9:
 ; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ 0, [[VECTOR_PH7]] ], [ [[INDEX_NEXT13:%.*]], [[VECTOR_BODY11]] ]
 ; CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 4, !alias.scope !4, !noalias !7
 ; CHECK-NEXT:    [[INDEX_NEXT13]] = add nuw i64 [[INDEX12]], 4
@@ -63,7 +63,7 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N10]], label [[LOOP_3_LR_PH:%.*]], label [[SCALAR_PH5]]
 ; CHECK:       scalar.ph5:
-; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK4]] ], [ 0, [[LOOP_2_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK4]] ], [ 0, [[LOOP_2_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop.3.lr.ph:
 ; CHECK-NEXT:    [[IDXPROM_I_I61:%.*]] = and i64 [[IV761_LCSSA]], 1
@@ -82,13 +82,13 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK:       vector.ph23:
 ; CHECK-NEXT:    [[N_MOD_VF24:%.*]] = urem i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[N_VEC25:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF24]]
-; CHECK-NEXT:    br label [[VECTOR_BODY28:%.*]]
-; CHECK:       vector.body27:
-; CHECK-NEXT:    [[INDEX29:%.*]] = phi i64 [ 0, [[VECTOR_PH23]] ], [ [[INDEX_NEXT30:%.*]], [[VECTOR_BODY28]] ]
+; CHECK-NEXT:    br label [[VECTOR_BODY26:%.*]]
+; CHECK:       vector.body26:
+; CHECK-NEXT:    [[INDEX29:%.*]] = phi i64 [ 0, [[VECTOR_PH23]] ], [ [[INDEX_NEXT29:%.*]], [[VECTOR_BODY26]] ]
 ; CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 4, !alias.scope !10, !noalias !13
-; CHECK-NEXT:    [[INDEX_NEXT30]] = add nuw i64 [[INDEX29]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT30]], [[N_VEC25]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK20:%.*]], label [[VECTOR_BODY28]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT29]] = add nuw i64 [[INDEX29]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT29]], [[N_VEC25]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK20:%.*]], label [[VECTOR_BODY26]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       middle.block20:
 ; CHECK-NEXT:    [[CMP_N27:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC25]]
 ; CHECK-NEXT:    br i1 [[CMP_N27]], label [[LOOP_CLEANUP:%.*]], label [[SCALAR_PH21]]
@@ -96,10 +96,10 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
 ; CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i64 [ [[N_VEC25]], [[MIDDLE_BLOCK20]] ], [ 0, [[LOOP_3_LR_PH]] ], [ 0, [[VECTOR_MEMCHECK14]] ]
 ; CHECK-NEXT:    br label [[LOOP_3:%.*]]
 ; CHECK:       loop.2:
-; CHECK-NEXT:    [[IV846:%.*]] = phi i64 [ [[IV_NEXT85:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL9]], [[SCALAR_PH5]] ]
+; CHECK-NEXT:    [[IV846:%.*]] = phi i64 [ [[IV_NEXT85:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL13]], [[SCALAR_PH5]] ]
 ; CHECK-NEXT:    [[IV_NEXT87:%.*]] = add i64 0, 0
 ; CHECK-NEXT:    [[ARRAYIDX_I_I56:%.*]] = getelementptr i32, ptr [[TMP0]], i64 [[IV761_LCSSA]]
-; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX_I_I56]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX_I_I56]], align 4
 ; CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[IV_NEXT85]] = add i64 [[IV846]], 1
 ; CHECK-NEXT:    [[EXITCOND92_NOT:%.*]] = icmp eq i64 [[IV846]], [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr66616.ll b/llvm/test/Transforms/LoopVectorize/pr66616.ll
index c60b216f86fa8..50e18070a5c3f 100644
--- a/llvm/test/Transforms/LoopVectorize/pr66616.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr66616.ll
@@ -46,7 +46,7 @@ define void @pr66616(ptr %ptr) {
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i32 [[ADD3_LCSSA]], [[DOTCAST]]
 ; CHECK-NEXT:    [[IND_END5:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY7:%.*]]
-; CHECK:       vector.body7:
+; CHECK:       vector.body5:
 ; CHECK-NEXT:    [[INDEX8:%.*]] = phi i64 [ 0, [[VECTOR_PH3]] ], [ [[INDEX_NEXT9:%.*]], [[VECTOR_BODY7]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX8]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-align.ll b/llvm/test/Transforms/LoopVectorize/reduction-align.ll
index 7eea9e189b470..8eef1ed90c0c9 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-align.ll
@@ -41,7 +41,7 @@ define void @fn(ptr %hbuf, ptr %ref, i32 %height) {
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i16 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
index 6771f56191313..8e132ed8399cd 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
@@ -56,7 +56,7 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP23]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]])
 ; CHECK-NEXT:    [[TMP26]] = add i32 [[TMP25]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -165,7 +165,7 @@ define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP46:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP39]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP47:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP46]])
 ; CHECK-NEXT:    [[TMP48]] = add i32 [[TMP47]], [[TMP45]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
@@ -260,7 +260,7 @@ define i32 @reduction_sum_const(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP27:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> splat (i32 3), <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]])
 ; CHECK-NEXT:    [[TMP29]] = add i32 [[TMP28]], [[TMP26]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -370,7 +370,7 @@ define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP46:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP39]], <4 x i32> splat (i32 1)
 ; CHECK-NEXT:    [[TMP47:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP46]])
 ; CHECK-NEXT:    [[TMP48]] = mul i32 [[TMP47]], [[TMP45]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
@@ -483,7 +483,7 @@ define i32 @reduction_mix(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP44:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP44]])
 ; CHECK-NEXT:    [[TMP46]] = add i32 [[TMP45]], [[TMP43]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
@@ -594,7 +594,7 @@ define i32 @reduction_mul(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP43:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP39]], <4 x i32> splat (i32 1)
 ; CHECK-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP43]])
 ; CHECK-NEXT:    [[TMP45]] = mul i32 [[TMP44]], [[TMP42]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -702,7 +702,7 @@ define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP43:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP39]], <4 x i32> splat (i32 -1)
 ; CHECK-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP43]])
 ; CHECK-NEXT:    [[TMP45]] = and i32 [[TMP44]], [[TMP42]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -808,7 +808,7 @@ define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP41:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP41]])
 ; CHECK-NEXT:    [[TMP43]] = or i32 [[TMP42]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -914,7 +914,7 @@ define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP41:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP41]])
 ; CHECK-NEXT:    [[TMP43]] = xor i32 [[TMP42]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -1020,7 +1020,7 @@ define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP41:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[TMP40]])
 ; CHECK-NEXT:    [[TMP42:%.*]] = select fast <4 x i1> [[TMP0]], <4 x float> [[TMP39]], <4 x float> zeroinitializer
 ; CHECK-NEXT:    [[TMP43]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[TMP41]], <4 x float> [[TMP42]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
@@ -1128,7 +1128,7 @@ define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP43:%.*]] = select fast <4 x i1> [[TMP0]], <4 x float> [[TMP39]], <4 x float> splat (float 1.000000e+00)
 ; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP43]])
 ; CHECK-NEXT:    [[TMP45]] = fmul fast float [[TMP44]], [[TMP42]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
@@ -1217,7 +1217,7 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP23]], <4 x i32> splat (i32 2147483647)
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP24]])
 ; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP25]], i32 [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
@@ -1304,7 +1304,7 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP23]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP24]])
 ; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP25]], i32 [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
@@ -1487,7 +1487,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP28:%.*]] = phi <4 x i8> [ [[TMP21]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP27]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP29:%.*]] = zext <4 x i8> [[TMP28]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP30]] = add nuw nsw <4 x i32> [[TMP1]], [[TMP29]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
@@ -1582,7 +1582,7 @@ define i8 @reduction_and_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP27:%.*]] = phi <4 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP26]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP28:%.*]] = zext <4 x i8> [[TMP27]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP29]] = and <4 x i32> [[VEC_PHI]], [[TMP28]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
index f65f62bbf54bf..81163c45abe95 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
@@ -16,9 +16,9 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 48
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
@@ -257,7 +257,7 @@ define i32 @predicated(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP108:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP98]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP109:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP108]])
 ; CHECK-NEXT:    [[TMP110]] = add i32 [[TMP109]], [[VEC_PHI6]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 16)
 ; CHECK-NEXT:    [[TMP111:%.*]] = icmp eq i64 [[INDEX_NEXT]], 272
 ; CHECK-NEXT:    br i1 [[TMP111]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index fe74a7c3a9b27..6315ce541b38b 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -1037,9 +1037,9 @@ define float @reduction_fmuladd_recurrence_first_arg(ptr %a, ptr %b, i64 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[SUM_07]], float [[TMP0]], float [[TMP1]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -1075,9 +1075,9 @@ define float @reduction_fmuladd_recurrence_second_arg(ptr %a, ptr %b, i64 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float [[SUM_07]], float [[TMP1]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll b/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
index 491a9ec0f0d4d..f95be1a221e73 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
@@ -55,7 +55,7 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP23:%.*]] = phi <4 x i32> [ [[TMP17]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP23]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP25]] = add <4 x i32> [[VEC_PHI]], [[TMP24]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -158,7 +158,7 @@ define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP41:%.*]] = add <4 x i32> [[TMP40]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = add <4 x i32> [[TMP41]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP43]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP42]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -264,7 +264,7 @@ define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = mul <4 x i32> [[VEC_PHI]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = mul <4 x i32> [[TMP40]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP42]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP41]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -369,7 +369,7 @@ define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = and <4 x i32> [[TMP38]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> splat (i32 -1)
 ; CHECK-NEXT:    [[TMP42]] = and <4 x i32> [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -474,7 +474,7 @@ define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[TMP39]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42]] = or <4 x i32> [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -579,7 +579,7 @@ define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[TMP39]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP40]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42]] = xor <4 x i32> [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -684,7 +684,7 @@ define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = fadd fast <4 x float> [[TMP40]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP42]] = select fast <4 x i1> [[TMP0]], <4 x float> [[TMP41]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -789,7 +789,7 @@ define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP40:%.*]] = fmul fast <4 x float> [[VEC_PHI]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = fmul fast <4 x float> [[TMP40]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP42]] = select fast <4 x i1> [[TMP0]], <4 x float> [[TMP41]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -877,7 +877,7 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP23:%.*]] = phi <4 x i32> [ [[TMP17]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP23]])
 ; CHECK-NEXT:    [[TMP25]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -963,7 +963,7 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[TMP23:%.*]] = phi <4 x i32> [ [[TMP17]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP23]])
 ; CHECK-NEXT:    [[TMP25]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
 ; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll
index bbdff969ebddd..b60a8e8da7f35 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction.ll
@@ -1141,7 +1141,7 @@ define void @reduction_reset(i32 %N, ptr %arrayA, ptr %arrayB) {
 ; CHECK:       .lr.ph:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ]
 ; CHECK-NEXT:    [[DOT017:%.*]] = phi i32 [ 100, [[DOTLR_PH_PREHEADER]] ], [ [[CSEL:%.*]], [[DOTLR_PH]] ]
-; CHECK-NEXT:    [[C6:%.*]] = getelementptr inbounds i32, ptr [[ARRAYA]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[C6:%.*]] = getelementptr inbounds nuw i32, ptr [[ARRAYA]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[C7:%.*]] = load i32, ptr [[C6]], align 4
 ; CHECK-NEXT:    [[C8:%.*]] = icmp sgt i32 [[C7]], 0
 ; CHECK-NEXT:    [[C9:%.*]] = add nsw i32 [[C7]], [[DOT017]]
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
index 70d9290f41bbd..3b6852ba59505 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
@@ -198,7 +198,7 @@ define i32 @reverse_induction_i16(i16 %startval, ptr %ptr) {
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[STARTVAL]], %[[ENTRY]] ], [ [[STARTVAL]], %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index ccf02b96abc9d..5c817ea313183 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -238,7 +238,7 @@ define void @test_runtime_check2(ptr %a, float %b, i64 %offset, i64 %offset2, i6
 ; CHECK-NEXT:    [[C_IDX:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -4
 ; CHECK-NEXT:    [[LC:%.*]] = load float, ptr [[C_IDX]], align 4
 ; CHECK-NEXT:    [[VC:%.*]] = fadd float [[LC]], 1.000000e+00
-; CHECK-NEXT:    [[C_IDX2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT:    [[C_IDX2:%.*]] = getelementptr inbounds nuw float, ptr [[C]], i64 [[IV]]
 ; CHECK-NEXT:    store float [[VC]], ptr [[C_IDX2]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll b/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
index c425d21612fd1..68b4260c03737 100644
--- a/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
@@ -11,7 +11,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 ; CHECK:   %[[T2:.+]] = add nuw nsw i64 [[OFFSET_IDX]], %tmp0
 ; CHECK:   %[[T3:.+]] = sub nsw i64 %[[T2]], %x
 ; CHECK:   %[[T4:.+]] = getelementptr inbounds i32, ptr %a, i64 %[[T3]]
-; CHECK:   %[[T6:.+]] = getelementptr inbounds i8, ptr %[[T4]], i64 16
+; CHECK:   %[[T6:.+]] = getelementptr inbounds nuw i8, ptr %[[T4]], i64 16
 ; CHECK:   load <4 x i32>, ptr %[[T4]], align 4
 ; CHECK:   load <4 x i32>, ptr %[[T6]], align 4
 ; CHECK:   br {{.*}}, label %middle.block, label %vector.body
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
index 521af746dffce..7562a5873036e 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
@@ -56,9 +56,9 @@ define void @test_pr63368(i1 %c, ptr %A) {
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
-; CHECK-NEXT:    br label [[VECTOR_BODY7:%.*]]
-; CHECK:       vector.body6:
-; CHECK-NEXT:    [[INDEX8:%.*]] = phi i32 [ 0, [[VECTOR_PH4]] ], [ [[INDEX_NEXT9:%.*]], [[VECTOR_BODY7]] ]
+; CHECK-NEXT:    br label [[VECTOR_BODY5:%.*]]
+; CHECK:       vector.body5:
+; CHECK-NEXT:    [[INDEX8:%.*]] = phi i32 [ 0, [[VECTOR_PH4]] ], [ [[INDEX_NEXT9:%.*]], [[VECTOR_BODY5]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX8]] to i8
 ; CHECK-NEXT:    [[TMP14:%.*]] = add i8 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i8 [[TMP14]], 1
@@ -67,7 +67,7 @@ define void @test_pr63368(i1 %c, ptr %A) {
 ; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr [[TMP17]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i32 [[INDEX8]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT9]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK2:%.*]], label [[VECTOR_BODY7]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK2:%.*]], label [[VECTOR_BODY5]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block2:
 ; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N6]], label [[EXIT_2:%.*]], label [[SCALAR_PH3]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
index c3931930a686e..576a971c5eaa8 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
@@ -550,8 +550,8 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF4-IC1:       scalar.ph:
 ; CHECK-VF4-IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC1-NEXT:    [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF4-IC1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF4-IC1:       for.body:
 ; CHECK-VF4-IC1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -712,8 +712,8 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF4-IC2:       scalar.ph:
 ; CHECK-VF4-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC2-NEXT:    [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF4-IC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF4-IC2:       for.body:
 ; CHECK-VF4-IC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -809,8 +809,8 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF1-IC2-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-VF1-IC2:       scalar.ph:
 ; CHECK-VF1-IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[VECTOR_MEMCHECK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ], [ false, [[VECTOR_MEMCHECK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF1-IC2-NEXT:    [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
 ; CHECK-VF1-IC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-VF1-IC2:       for.body:
 ; CHECK-VF1-IC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index 7b66440a7fdcc..301526cf3070c 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -1,320 +1,1323 @@
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK-VF4IC1 --check-prefix=CHECK
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK-VF4IC4 --check-prefix=CHECK
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s --check-prefix=CHECK-VF1IC4 --check-prefix=CHECK
-
-define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) {
-; CHECK-LABEL: @select_const_i32_from_icmp
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <4 x i32>
-; CHECK-VF4IC1-NEXT:   [[VEC_ICMP:%.*]] = icmp eq <4 x i32> [[VEC_LOAD]], splat (i32 3)
-; CHECK-VF4IC1-NEXT:   [[NOT:%.*]] = xor <4 x i1> [[VEC_ICMP]], splat (i1 true)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[NOT]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 7, i32 3
-
-; CHECK-VF4IC4:      vector.body:
-; CHECK-VF4IC4:        [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL1:%.*]], %vector.body ]
-; CHECK-VF4IC4-NEXT:   [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL2:%.*]], %vector.body ]
-; CHECK-VF4IC4-NEXT:   [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL3:%.*]], %vector.body ]
-; CHECK-VF4IC4-NEXT:   [[VEC_PHI4:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL4:%.*]], %vector.body ]
-; CHECK-VF4IC4:        [[VEC_ICMP1:%.*]] = icmp eq <4 x i32> {{.*}}, splat (i32 3)
-; CHECK-VF4IC4-NEXT:   [[VEC_ICMP2:%.*]] = icmp eq <4 x i32> {{.*}}, splat (i32 3)
-; CHECK-VF4IC4-NEXT:   [[VEC_ICMP3:%.*]] = icmp eq <4 x i32> {{.*}}, splat (i32 3)
-; CHECK-VF4IC4-NEXT:   [[VEC_ICMP4:%.*]] = icmp eq <4 x i32> {{.*}}, splat (i32 3)
-; CHECK-VF4IC4-NEXT:   [[NOT1:%.*]] = xor <4 x i1> [[VEC_ICMP1]], splat (i1 true)
-; CHECK-VF4IC4-NEXT:   [[NOT2:%.*]] = xor <4 x i1> [[VEC_ICMP2]], splat (i1 true)
-; CHECK-VF4IC4-NEXT:   [[NOT3:%.*]] = xor <4 x i1> [[VEC_ICMP3]], splat (i1 true)
-; CHECK-VF4IC4-NEXT:   [[NOT4:%.*]] = xor <4 x i1> [[VEC_ICMP4]], splat (i1 true)
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL1:%.*]] = or <4 x i1> [[VEC_PHI1]], [[NOT1]]
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL2:%.*]] = or <4 x i1> [[VEC_PHI2]], [[NOT2]]
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL3:%.*]] = or <4 x i1> [[VEC_PHI3]], [[NOT3]]
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL4:%.*]] = or <4 x i1> [[VEC_PHI4]], [[NOT4]]
-; CHECK-VF4IC4:      middle.block:
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL5:%.*]] = or <4 x i1>  [[VEC_SEL2]], [[VEC_SEL1]]
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL6:%.*]] = or <4 x i1> [[VEC_SEL3]], [[VEC_SEL5]]
-; CHECK-VF4IC4-NEXT:   [[VEC_SEL7:%.*]] = or <4 x i1> [[VEC_SEL4]], [[VEC_SEL6]]
-; CHECK-VF4IC4-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL7]])
-; CHECK-VF4IC4-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC4-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 7, i32 3
-
-
-; CHECK-VF1IC4:      vector.body:
-; CHECK-VF1IC4:        [[VEC_PHI1:%.*]] = phi i1 [ false, %vector.ph ], [ [[VEC_SEL1:%.*]], %vector.body ]
-; CHECK-VF1IC4-NEXT:   [[VEC_PHI2:%.*]] = phi i1 [ false, %vector.ph ], [ [[VEC_SEL2:%.*]], %vector.body ]
-; CHECK-VF1IC4-NEXT:   [[VEC_PHI3:%.*]] = phi i1 [ false, %vector.ph ], [ [[VEC_SEL3:%.*]], %vector.body ]
-; CHECK-VF1IC4-NEXT:   [[VEC_PHI4:%.*]] = phi i1 [ false, %vector.ph ], [ [[VEC_SEL4:%.*]], %vector.body ]
-; CHECK-VF1IC4:        [[VEC_LOAD1:%.*]] = load i32
-; CHECK-VF1IC4-NEXT:   [[VEC_LOAD2:%.*]] = load i32
-; CHECK-VF1IC4-NEXT:   [[VEC_LOAD3:%.*]] = load i32
-; CHECK-VF1IC4-NEXT:   [[VEC_LOAD4:%.*]] = load i32
-; CHECK-VF1IC4-NEXT:   [[VEC_ICMP1:%.*]] = icmp eq i32 [[VEC_LOAD1]], 3
-; CHECK-VF1IC4-NEXT:   [[VEC_ICMP2:%.*]] = icmp eq i32 [[VEC_LOAD2]], 3
-; CHECK-VF1IC4-NEXT:   [[VEC_ICMP3:%.*]] = icmp eq i32 [[VEC_LOAD3]], 3
-; CHECK-VF1IC4-NEXT:   [[VEC_ICMP4:%.*]] = icmp eq i32 [[VEC_LOAD4]], 3
-; CHECK-VF1IC4-NEXT:   [[NOT1:%.*]] = xor i1 [[VEC_ICMP1]], true
-; CHECK-VF1IC4-NEXT:   [[NOT2:%.*]] = xor i1 [[VEC_ICMP2]], true
-; CHECK-VF1IC4-NEXT:   [[NOT3:%.*]] = xor i1 [[VEC_ICMP3]], true
-; CHECK-VF1IC4-NEXT:   [[NOT4:%.*]] = xor i1 [[VEC_ICMP4]], true
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL1:%.*]] = or i1 [[VEC_PHI1]], [[NOT1]]
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL2:%.*]] = or i1 [[VEC_PHI2]], [[NOT2]]
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL3:%.*]] = or i1 [[VEC_PHI3]], [[NOT3]]
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL4:%.*]] = or i1 [[VEC_PHI4]], [[NOT4]]
-; CHECK-VF1IC4:      middle.block:
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL5:%.*]] = or i1 [[VEC_SEL2]], [[VEC_SEL1]]
-; CHECK-VF1IC4-NEXT:   [[VEC_SEL6:%.*]] = or i1 [[VEC_SEL3]], [[VEC_SEL5]]
-; CHECK-VF1IC4-NEXT:   [[OR_RDX:%.*]] = or i1  [[VEC_SEL4]], [[VEC_SEL6]]
-; CHECK-VF1IC4-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF1IC4-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 7, i32 3
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VF4IC1
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VF4IC4
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VF1IC4
 
+define i32 @select_const_i32_from_icmp(ptr %v, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_const_i32_from_icmp(
+; CHECK-VF4IC1-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-VF4IC1-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i32 7, i32 3
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC1-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 7
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_const_i32_from_icmp(
+; CHECK-VF4IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP11:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP14]] = or <4 x i1> [[VEC_PHI]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[TMP15]] = or <4 x i1> [[VEC_PHI1]], [[TMP11]]
+; CHECK-VF4IC4-NEXT:    [[TMP16]] = or <4 x i1> [[VEC_PHI2]], [[TMP12]]
+; CHECK-VF4IC4-NEXT:    [[TMP17]] = or <4 x i1> [[VEC_PHI3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP16]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP17]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 7, i32 3
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC4-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 7
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_const_i32_from_icmp(
+; CHECK-VF1IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
+; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
+; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
+; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP21]], [[TMP20]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP22]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP23]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP25:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP25]], i32 7, i32 3
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF1IC4-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 7
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi i32 [ 3, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = icmp eq i32 %3, 3
-  %5 = select i1 %4, i32 %1, i32 7
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 3, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds i32, ptr %v, i64 %iv
+  %load.v.iv = load i32, ptr %gep.v.iv, align 4
+  %cmp.v.iv.3 = icmp eq i32 %load.v.iv, 3
+  %sel = select i1 %cmp.v.iv.3, i32 %rdx, i32 7
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %5
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
-define i32 @select_const_i32_from_icmp2(ptr nocapture readonly %v, i64 %n) {
-; CHECK-LABEL: @select_const_i32_from_icmp2
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <4 x i32>
-; CHECK-VF4IC1-NEXT:   [[VEC_ICMP:%.*]] = icmp eq <4 x i32> [[VEC_LOAD]], splat (i32 3)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[VEC_ICMP]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 7, i32 3
-
+define i32 @select_const_i32_from_icmp2(ptr %v, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_const_i32_from_icmp2(
+; CHECK-VF4IC1-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC1-NEXT:    [[TMP4]] = or <4 x i1> [[VEC_PHI]], [[TMP3]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = freeze i1 [[TMP6]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP7]], i32 7, i32 3
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC1-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 7, i32 [[RDX]]
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_const_i32_from_icmp2(
+; CHECK-VF4IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP10]] = or <4 x i1> [[VEC_PHI]], [[TMP6]]
+; CHECK-VF4IC4-NEXT:    [[TMP11]] = or <4 x i1> [[VEC_PHI1]], [[TMP7]]
+; CHECK-VF4IC4-NEXT:    [[TMP12]] = or <4 x i1> [[VEC_PHI2]], [[TMP8]]
+; CHECK-VF4IC4-NEXT:    [[TMP13]] = or <4 x i1> [[VEC_PHI3]], [[TMP9]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP11]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP12]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP13]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP16:%.*]] = freeze i1 [[TMP15]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP16]], i32 7, i32 3
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC4-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 7, i32 [[RDX]]
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_const_i32_from_icmp2(
+; CHECK-VF1IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP16]] = or i1 [[VEC_PHI]], [[TMP12]]
+; CHECK-VF1IC4-NEXT:    [[TMP17]] = or i1 [[VEC_PHI1]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP18]] = or i1 [[VEC_PHI2]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP19]] = or i1 [[VEC_PHI3]], [[TMP15]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP17]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP18]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP19]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP21:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP21]], i32 7, i32 3
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF1IC4-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 7, i32 [[RDX]]
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi i32 [ 3, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = icmp eq i32 %3, 3
-  %5 = select i1 %4, i32 7, i32 %1
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 3, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds i32, ptr %v, i64 %iv
+  %load.v.iv = load i32, ptr %gep.v.iv, align 4
+  %cmp.v.iv.3 = icmp eq i32 %load.v.iv, 3
+  %sel = select i1 %cmp.v.iv.3, i32 7, i32 %rdx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %5
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
-define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 %n) {
-; CHECK-LABEL: @select_i32_from_icmp
-; CHECK-VF4IC1:      vector.ph:
-; CHECK-VF4IC1-NOT:    shufflevector <4 x i32>
-; CHECK-VF4IC1-NOT:    shufflevector <4 x i32>
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <4 x i32>
-; CHECK-VF4IC1-NEXT:   [[VEC_ICMP:%.*]] = icmp eq <4 x i32> [[VEC_LOAD]], splat (i32 3)
-; CHECK-VF4IC1-NEXT:   [[NOT:%.*]] = xor <4 x i1> [[VEC_ICMP]], splat (i1 true)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[NOT]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 %b, i32 %a
+define i32 @select_i32_from_icmp(ptr %v, i32 %a, i32 %b, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_i32_from_icmp(
+; CHECK-VF4IC1-SAME: ptr [[V:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-VF4IC1-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i32 [[B]], i32 [[A]]
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC1-NEXT:    [[CMP_LOAD_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_i32_from_icmp(
+; CHECK-VF4IC4-SAME: ptr [[V:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP11:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP14]] = or <4 x i1> [[VEC_PHI]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[TMP15]] = or <4 x i1> [[VEC_PHI1]], [[TMP11]]
+; CHECK-VF4IC4-NEXT:    [[TMP16]] = or <4 x i1> [[VEC_PHI2]], [[TMP12]]
+; CHECK-VF4IC4-NEXT:    [[TMP17]] = or <4 x i1> [[VEC_PHI3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP16]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP17]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 [[B]], i32 [[A]]
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC4-NEXT:    [[CMP_LOAD_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_i32_from_icmp(
+; CHECK-VF1IC4-SAME: ptr [[V:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
+; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
+; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
+; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP21]], [[TMP20]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP22]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP23]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP25:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP25]], i32 [[B]], i32 [[A]]
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-VF1IC4-NEXT:    [[CMP_LOAD_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi i32 [ %a, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = icmp eq i32 %3, 3
-  %5 = select i1 %4, i32 %1, i32 %b
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ %a, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds i32, ptr %v, i64 %iv
+  %load.v.iv = load i32, ptr %gep.v.iv, align 4
+  %cmp.load.iv.3 = icmp eq i32 %load.v.iv, 3
+  %sel = select i1 %cmp.load.iv.3, i32 %rdx, i32 %b
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %5
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
-define i32 @select_const_i32_from_fcmp_fast(ptr nocapture readonly %v, i64 %n) {
-; CHECK-LABEL: @select_const_i32_from_fcmp_fast
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <4 x float>
-; CHECK-VF4IC1-NEXT:   [[VEC_FCMP:%.*]] = fcmp fast ueq <4 x float> [[VEC_LOAD]], splat (float 3.000000e+00)
-; CHECK-VF4IC1-NEXT:   [[NOT:%.*]] = xor <4 x i1> [[VEC_FCMP]], splat (i1 true)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[NOT]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 1, i32 2
+define i32 @select_const_i32_from_fcmp_fast(ptr %v, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_const_i32_from_fcmp_fast(
+; CHECK-VF4IC1-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = fcmp fast ueq <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-VF4IC1-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i32 1, i32 2
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC1-NEXT:    [[CMP_LOAD_IV_3:%.*]] = fcmp fast ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_const_i32_from_fcmp_fast(
+; CHECK-VF4IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = fcmp fast ueq <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = fcmp fast ueq <4 x float> [[WIDE_LOAD4]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = fcmp fast ueq <4 x float> [[WIDE_LOAD5]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = fcmp fast ueq <4 x float> [[WIDE_LOAD6]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP11:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP14]] = or <4 x i1> [[VEC_PHI]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[TMP15]] = or <4 x i1> [[VEC_PHI1]], [[TMP11]]
+; CHECK-VF4IC4-NEXT:    [[TMP16]] = or <4 x i1> [[VEC_PHI2]], [[TMP12]]
+; CHECK-VF4IC4-NEXT:    [[TMP17]] = or <4 x i1> [[VEC_PHI3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP16]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP17]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 1, i32 2
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC4-NEXT:    [[CMP_LOAD_IV_3:%.*]] = fcmp fast ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_const_i32_from_fcmp_fast(
+; CHECK-VF1IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP4]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load float, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp fast ueq float [[TMP8]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp fast ueq float [[TMP9]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp fast ueq float [[TMP10]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp fast ueq float [[TMP11]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
+; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
+; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
+; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP21]], [[TMP20]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP22]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP23]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP25:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP25]], i32 1, i32 2
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF1IC4-NEXT:    [[CMP_LOAD_IV_3:%.*]] = fcmp fast ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_LOAD_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi i32 [ 2, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds float, ptr %v, i64 %0
-  %3 = load float, ptr %2, align 4
-  %4 = fcmp fast ueq float %3, 3.0
-  %5 = select i1 %4, i32 %1, i32 1
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 2, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds float, ptr %v, i64 %iv
+  %load.v.iv = load float, ptr %gep.v.iv, align 4
+  %cmp.load.iv.3 = fcmp fast ueq float %load.v.iv, 3.0
+  %sel = select i1 %cmp.load.iv.3, i32 %rdx, i32 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %5
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
-define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) {
-; CHECK-LABEL: @select_const_i32_from_fcmp
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <4 x float>
-; CHECK-VF4IC1-NEXT:   [[VEC_FCMP:%.*]] = fcmp ueq <4 x float> [[VEC_LOAD]], splat (float 3.000000e+00)
-; CHECK-VF4IC1-NEXT:   [[NOT:%.*]] = xor <4 x i1> [[VEC_FCMP]], splat (i1 true)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[NOT]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 1, i32 2
+define i32 @select_const_i32_from_fcmp(ptr %v, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_const_i32_from_fcmp(
+; CHECK-VF4IC1-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = fcmp ueq <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-VF4IC1-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP7]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i32 1, i32 2
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC1-NEXT:    [[CMP_V_IV_3:%.*]] = fcmp ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_const_i32_from_fcmp(
+; CHECK-VF4IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 4
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 8
+; CHECK-VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 12
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT:    [[TMP6:%.*]] = fcmp ueq <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP7:%.*]] = fcmp ueq <4 x float> [[WIDE_LOAD4]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP8:%.*]] = fcmp ueq <4 x float> [[WIDE_LOAD5]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = fcmp ueq <4 x float> [[WIDE_LOAD6]], splat (float 3.000000e+00)
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP11:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP14]] = or <4 x i1> [[VEC_PHI]], [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[TMP15]] = or <4 x i1> [[VEC_PHI1]], [[TMP11]]
+; CHECK-VF4IC4-NEXT:    [[TMP16]] = or <4 x i1> [[VEC_PHI2]], [[TMP12]]
+; CHECK-VF4IC4-NEXT:    [[TMP17]] = or <4 x i1> [[VEC_PHI3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX7:%.*]] = or <4 x i1> [[TMP16]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX8:%.*]] = or <4 x i1> [[TMP17]], [[BIN_RDX7]]
+; CHECK-VF4IC4-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX8]])
+; CHECK-VF4IC4-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 1, i32 2
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF4IC4-NEXT:    [[CMP_V_IV_3:%.*]] = fcmp ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_const_i32_from_fcmp(
+; CHECK-VF1IC4-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP4]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load float, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp ueq float [[TMP8]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp ueq float [[TMP9]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp ueq float [[TMP10]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp ueq float [[TMP11]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
+; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
+; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
+; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP21]], [[TMP20]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP22]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP23]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP25:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP25]], i32 1, i32 2
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT:    [[LOAD_V_IV:%.*]] = load float, ptr [[GEP_V_IV]], align 4
+; CHECK-VF1IC4-NEXT:    [[CMP_V_IV_3:%.*]] = fcmp ueq float [[LOAD_V_IV]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_V_IV_3]], i32 [[RDX]], i32 1
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi i32 [ 2, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds float, ptr %v, i64 %0
-  %3 = load float, ptr %2, align 4
-  %4 = fcmp ueq float %3, 3.0
-  %5 = select i1 %4, i32 %1, i32 1
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 2, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds float, ptr %v, i64 %iv
+  %load.v.iv = load float, ptr %gep.v.iv, align 4
+  %cmp.v.iv.3 = fcmp ueq float %load.v.iv, 3.0
+  %sel = select i1 %cmp.v.iv.3, i32 %rdx, i32 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %5
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
 define i32 @select_i32_from_icmp_same_inputs(i32 %a, i32 %b, i64 %n) {
-; CHECK-LABEL: @select_i32_from_icmp_same_inputs
-; CHECK-VF4IC1:      vector.ph:
-; CHECK-VF4IC1:        [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 %a, i64 0
-; CHECK-VF4IC1-NEXT:   [[SPLAT_OF_A:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-VF4IC1-NOT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 %b, i64 0
-; CHECK-VF4IC1-NEXT:   [[VEC_ICMP:%.*]] = icmp eq <4 x i32> [[SPLAT_OF_A]], splat (i32 3)
-; CHECK-VF4IC1:      vector.body:
-; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
-; CHECK-VF4IC1-NEXT:   [[NOT:%.*]] = xor <4 x i1> [[VEC_ICMP]], splat (i1 true)
-; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = or <4 x i1> [[VEC_PHI]], [[NOT]]
-; CHECK-VF4IC1:      middle.block:
-; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[VEC_SEL]])
-; CHECK-VF4IC1-NEXT:   [[FR_OR_RDX:%.*]] = freeze i1 [[OR_RDX]]
-; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[FR_OR_RDX]], i32 %b, i32 %a
+; CHECK-VF4IC1-LABEL: define i32 @select_i32_from_icmp_same_inputs(
+; CHECK-VF4IC1-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1:       [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
+; CHECK-VF4IC1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i32> [[BROADCAST_SPLAT]], splat (i32 3)
+; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1:       [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-VF4IC1-NEXT:    [[TMP2]] = or <4 x i1> [[VEC_PHI]], [[TMP1]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = freeze i1 [[TMP4]]
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP5]], i32 [[B]], i32 [[A]]
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1:       [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC1:       [[LOOP]]:
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT:    [[CMP_RDX_3:%.*]] = icmp eq i32 [[RDX]], 3
+; CHECK-VF4IC1-NEXT:    [[SEL]] = select i1 [[CMP_RDX_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-VF4IC1:       [[EXIT]]:
+; CHECK-VF4IC1-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_i32_from_icmp_same_inputs(
+; CHECK-VF4IC4-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4:       [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-VF4IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
+; CHECK-VF4IC4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i32> [[BROADCAST_SPLAT]], splat (i32 3)
+; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4:       [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-VF4IC4-NEXT:    [[TMP5]] = or <4 x i1> [[VEC_PHI]], [[TMP1]]
+; CHECK-VF4IC4-NEXT:    [[TMP6]] = or <4 x i1> [[VEC_PHI1]], [[TMP2]]
+; CHECK-VF4IC4-NEXT:    [[TMP7]] = or <4 x i1> [[VEC_PHI2]], [[TMP3]]
+; CHECK-VF4IC4-NEXT:    [[TMP8]] = or <4 x i1> [[VEC_PHI3]], [[TMP4]]
+; CHECK-VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-VF4IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX:%.*]] = or <4 x i1> [[TMP6]], [[TMP5]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX4:%.*]] = or <4 x i1> [[TMP7]], [[BIN_RDX]]
+; CHECK-VF4IC4-NEXT:    [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP8]], [[BIN_RDX4]]
+; CHECK-VF4IC4-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
+; CHECK-VF4IC4-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
+; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 [[B]], i32 [[A]]
+; CHECK-VF4IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4:       [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF4IC4:       [[LOOP]]:
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT:    [[CMP_RDX_3:%.*]] = icmp eq i32 [[RDX]], 3
+; CHECK-VF4IC4-NEXT:    [[SEL]] = select i1 [[CMP_RDX_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF4IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-VF4IC4:       [[EXIT]]:
+; CHECK-VF4IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_i32_from_icmp_same_inputs(
+; CHECK-VF1IC4-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4:       [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[A]], 3
+; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4:       [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-VF1IC4-NEXT:    [[TMP2:%.*]] = xor i1 [[TMP0]], true
+; CHECK-VF1IC4-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP0]], true
+; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP0]], true
+; CHECK-VF1IC4-NEXT:    [[TMP5]] = or i1 [[VEC_PHI]], [[TMP1]]
+; CHECK-VF1IC4-NEXT:    [[TMP6]] = or i1 [[VEC_PHI1]], [[TMP2]]
+; CHECK-VF1IC4-NEXT:    [[TMP7]] = or i1 [[VEC_PHI2]], [[TMP3]]
+; CHECK-VF1IC4-NEXT:    [[TMP8]] = or i1 [[VEC_PHI3]], [[TMP4]]
+; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-VF1IC4:       [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX:%.*]] = or i1 [[TMP6]], [[TMP5]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX4:%.*]] = or i1 [[TMP7]], [[BIN_RDX]]
+; CHECK-VF1IC4-NEXT:    [[BIN_RDX5:%.*]] = or i1 [[TMP8]], [[BIN_RDX4]]
+; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = freeze i1 [[BIN_RDX5]]
+; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP10]], i32 [[B]], i32 [[A]]
+; CHECK-VF1IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4:       [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT:    br label %[[LOOP:.*]]
+; CHECK-VF1IC4:       [[LOOP]]:
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT:    [[CMP_RDX_3:%.*]] = icmp eq i32 [[RDX]], 3
+; CHECK-VF1IC4-NEXT:    [[SEL]] = select i1 [[CMP_RDX_3]], i32 [[RDX]], i32 [[B]]
+; CHECK-VF1IC4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-VF1IC4:       [[EXIT]]:
+; CHECK-VF1IC4-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %4, %for.body ]
-  %1 = phi i32 [ %a, %entry ], [ %3, %for.body ]
-  %2 = icmp eq i32 %1, 3
-  %3 = select i1 %2, i32 %1, i32 %b
-  %4 = add nuw nsw i64 %0, 1
-  %5 = icmp eq i64 %4, %n
-  br i1 %5, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ %a, %entry ], [ %sel, %loop ]
+  %cmp.rdx.3 = icmp eq i32 %rdx, 3
+  %sel = select i1 %cmp.rdx.3, i32 %rdx, i32 %b
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %3
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
-; Negative tests
+;; Negative tests
 
 ; We don't support FP reduction variables at the moment.
-define float @select_const_f32_from_icmp(ptr nocapture readonly %v, i64 %n) {
-; CHECK: @select_const_f32_from_icmp
-; CHECK-NOT: vector.body
+define float @select_const_f32_from_icmp(ptr %v, i64 %n) {
+; CHECK-LABEL: define float @select_const_f32_from_icmp(
+; CHECK-SAME: ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi fast float [ 3.000000e+00, %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_V_IV:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD_V_IV:%.*]] = load i32, ptr [[GEP_V_IV]], align 4
+; CHECK-NEXT:    [[CMP_V_IV_3:%.*]] = icmp eq i32 [[LOAD_V_IV]], 3
+; CHECK-NEXT:    [[SEL]] = select fast i1 [[CMP_V_IV_3]], float [[RDX]], float 7.000000e+00
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SEL_LCSSA:%.*]] = phi float [ [[SEL]], %[[LOOP]] ]
+; CHECK-NEXT:    ret float [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
-  %1 = phi fast float [ 3.0, %entry ], [ %5, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = icmp eq i32 %3, 3
-  %5 = select fast i1 %4, float %1, float 7.0
-  %6 = add nuw nsw i64 %0, 1
-  %7 = icmp eq i64 %6, %n
-  br i1 %7, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi fast float [ 3.0, %entry ], [ %sel, %loop ]
+  %gep.v.iv = getelementptr inbounds i32, ptr %v, i64 %iv
+  %load.v.iv = load i32, ptr %gep.v.iv, align 4
+  %cmp.v.iv.3 = icmp eq i32 %load.v.iv, 3
+  %sel = select fast i1 %cmp.v.iv.3, float %rdx, float 7.0
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret float %5
+exit:                                     ; preds = %loop
+  ret float %sel
 }
 
-
 ; We don't support selecting loop-variant values.
-define i32 @select_variant_i32_from_icmp(ptr nocapture readonly %v1, ptr nocapture readonly %v2, i64 %n) {
-; CHECK-LABEL: @select_variant_i32_from_icmp
-; CHECK-NOT: vector.body
+define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
+; CHECK-LABEL: define i32 @select_variant_i32_from_icmp(
+; CHECK-SAME: ptr [[V1:%.*]], ptr [[V2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ 3, %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_V1_IV:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD_V1_IV:%.*]] = load i32, ptr [[GEP_V1_IV]], align 4
+; CHECK-NEXT:    [[GEP_V2_IV:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD_V2_IV:%.*]] = load i32, ptr [[GEP_V2_IV]], align 4
+; CHECK-NEXT:    [[CMP_V1_IV_3:%.*]] = icmp eq i32 [[LOAD_V1_IV]], 3
+; CHECK-NEXT:    [[SEL]] = select i1 [[CMP_V1_IV_3]], i32 [[RDX]], i32 [[LOAD_V2_IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %8, %for.body ]
-  %1 = phi i32 [ 3, %entry ], [ %7, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v1, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = getelementptr inbounds i32, ptr %v2, i64 %0
-  %5 = load i32, ptr %4, align 4
-  %6 = icmp eq i32 %3, 3
-  %7 = select i1 %6, i32 %1, i32 %5
-  %8 = add nuw nsw i64 %0, 1
-  %9 = icmp eq i64 %8, %n
-  br i1 %9, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 3, %entry ], [ %sel, %loop ]
+  %gep.v1.iv = getelementptr inbounds i32, ptr %v1, i64 %iv
+  %load.v1.iv = load i32, ptr %gep.v1.iv, align 4
+  %gep.v2.iv = getelementptr inbounds i32, ptr %v2, i64 %iv
+  %load.v2.iv = load i32, ptr %gep.v2.iv, align 4
+  %cmp.v1.iv.3 = icmp eq i32 %load.v1.iv, 3
+  %sel = select i1 %cmp.v1.iv.3, i32 %rdx, i32 %load.v2.iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %7
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
 
-
 ; We only support selects where the input comes from the same PHI as the
 ; reduction PHI. In the example below, the select uses the induction
 ; variable input and the icmp uses the reduction PHI.
 define i32 @select_i32_from_icmp_non_redux_phi(i32 %a, i32 %b, i32 %n) {
-; CHECK-LABEL: @select_i32_from_icmp_non_redux_phi
-; CHECK-NOT: vector.body
+; CHECK-LABEL: define i32 @select_i32_from_icmp_non_redux_phi(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ [[A]], %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[CMP_RDX_3:%.*]] = icmp eq i32 [[RDX]], 3
+; CHECK-NEXT:    [[SEL]] = select i1 [[CMP_RDX_3]], i32 [[IV]], i32 [[B]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[SEL_LCSSA]]
+;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i32 [ 0, %entry ], [ %4, %for.body ]
-  %1 = phi i32 [ %a, %entry ], [ %3, %for.body ]
-  %2 = icmp eq i32 %1, 3
-  %3 = select i1 %2, i32 %0, i32 %b
-  %4 = add nuw nsw i32 %0, 1
-  %5 = icmp eq i32 %4, %n
-  br i1 %5, label %exit, label %for.body
+loop:                                      ; preds = %entry, %loop
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ %a, %entry ], [ %sel, %loop ]
+  %cmp.rdx.3 = icmp eq i32 %rdx, 3
+  %sel = select i1 %cmp.rdx.3, i32 %iv, i32 %b
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp eq i32 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
 
-exit:                                     ; preds = %for.body
-  ret i32 %3
+exit:                                     ; preds = %loop
+  ret i32 %sel
 }
+;.
+; CHECK-VF4IC1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF4IC1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF4IC1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF4IC1: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-VF4IC1: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-VF4IC1: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-VF4IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-VF4IC1: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-VF4IC1: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-VF4IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK-VF4IC1: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-VF4IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK-VF4IC1: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK-VF4IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+;.
+; CHECK-VF4IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF4IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF4IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF4IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-VF4IC4: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-VF4IC4: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-VF4IC4: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-VF4IC4: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-VF4IC4: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-VF4IC4: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK-VF4IC4: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-VF4IC4: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK-VF4IC4: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK-VF4IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+;.
+; CHECK-VF1IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-VF1IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-VF1IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-VF1IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/select-reduction.ll b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
index b7fc1e83ce23f..5e28192997695 100644
--- a/llvm/test/Transforms/LoopVectorize/select-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
@@ -32,7 +32,7 @@ define i32 @test(i64 %N, i32 %x) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i32> [[VEC_PHI]], <4 x i32> splat (i32 10)
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP2]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -93,7 +93,7 @@ define i32 @pr66895_tail_fold_reduction_exit_inst_gets_simplified(i32 %n) {
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 12)
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[VEC_PHI]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
index b3fc0283affdf..9e7c667f9c8ad 100644
--- a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
@@ -19,7 +19,7 @@ define float @pr70988() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = select contract i1 [[TMP1]], float 1.000000e+00, float -0.000000e+00
 ; CHECK-NEXT:    [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add i32 [[INDEX1]], 2
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -55,7 +55,7 @@ define float @pr70988() {
 ; CHECK-ALM-NEXT:    [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]
 ; CHECK-ALM-NEXT:    [[TMP4:%.*]] = select contract i1 [[ACTIVE_LANE_MASK2]], float 1.000000e+00, float -0.000000e+00
 ; CHECK-ALM-NEXT:    [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]
-; CHECK-ALM-NEXT:    [[INDEX_NEXT3]] = add i32 [[INDEX1]], 2
+; CHECK-ALM-NEXT:    [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 2
 ; CHECK-ALM-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022
 ; CHECK-ALM-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-ALM:       middle.block:
@@ -125,7 +125,7 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd contract float [[VEC_PHI]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select contract i1 [[TMP1]], float [[TMP9]], float -0.000000e+00
 ; CHECK-NEXT:    [[TMP13]] = fadd contract float [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
@@ -178,7 +178,7 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
 ; CHECK-ALM-NEXT:    [[TMP9:%.*]] = fadd contract float [[VEC_PHI]], [[TMP8]]
 ; CHECK-ALM-NEXT:    [[TMP10:%.*]] = select contract i1 [[ACTIVE_LANE_MASK1]], float [[TMP7]], float -0.000000e+00
 ; CHECK-ALM-NEXT:    [[TMP11]] = fadd contract float [[TMP9]], [[TMP10]]
-; CHECK-ALM-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-ALM-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-ALM-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
 ; CHECK-ALM-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ALM:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
index 9efc8d12f44a3..56fc8eac35bad 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
@@ -48,7 +48,7 @@ define i32 @test(ptr %vf1, i64 %n) {
 ; CHECK-NEXT:    store ptr [[TMP16]], ptr [[TMP15]], align 8
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; CHECK:       [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 204
 ; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
index dd2fd0de9938e..31732f027f6dd 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
@@ -49,7 +49,7 @@ define void @tail_fold_switch(ptr %dst, i32 %0) {
 ; CHECK-NEXT:    store i32 0, ptr [[TMP15]], align 4
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; CHECK:       [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 92ca77bc841ca..336c5ea4331b8 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -49,7 +49,7 @@ define void @VF1-VPlanExe(ptr %dst) {
 ; CHECK-NEXT:    store i32 0, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -129,7 +129,7 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE12]]
 ; CHECK:       pred.store.continue12:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll b/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
index 04437079121dc..3afd8553f8a7e 100644
--- a/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
@@ -9,23 +9,23 @@ define i8 @reduction_and_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i8> [ <i8 0, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i8> [ <i8 0, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3]] = and <8 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP2]] = and <8 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> [[TMP2]])
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[AND_LCSSA_OFF0:%.*]] = phi i8 [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[AND_LCSSA_OFF0:%.*]] = phi i8 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i8 [[AND_LCSSA_OFF0]]
 ;
 entry:
@@ -56,23 +56,23 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT:    [[TMP3]] = or <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP2]] = or <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP2]])
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[XOR_LCSSA_OFF0]]
 ;
 entry:
@@ -103,23 +103,23 @@ define i16 @reduction_xor_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT:    [[TMP3]] = xor <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP2]] = xor <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> [[TMP2]])
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[XOR_LCSSA_OFF0]]
 ;
 entry:
@@ -144,9 +144,24 @@ for.end:
 
 define i8 @reduction_smin_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-LABEL: @reduction_smin_trunc(
-; CHECK-NOT: vector.body
-; CHECK-NOT: <8 x
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_02P:%.*]] = phi i32 [ [[MIN:%.*]], [[FOR_BODY]] ], [ 256, [[ENTRY]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = and i32 [[SUM_02P]], 255
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT:    [[EXT:%.*]] = sext i8 [[LOAD]] to i32
+; CHECK-NEXT:    [[MIN]] = call i32 @llvm.smin.i32(i32 [[SUM_02]], i32 [[EXT]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[MIN]] to i8
+; CHECK-NEXT:    ret i8 [[RET]]
+;
 entry:
   br label %for.body
 
@@ -170,9 +185,24 @@ for.end:
 
 define i8 @reduction_umin_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-LABEL: @reduction_umin_trunc(
-; CHECK-NOT: vector.body
-; CHECK-NOT: <8 x
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_02P:%.*]] = phi i32 [ [[MIN:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = and i32 [[SUM_02P]], 255
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[LOAD]] to i32
+; CHECK-NEXT:    [[MIN]] = call i32 @llvm.umin.i32(i32 [[SUM_02]], i32 [[EXT]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RET:%.*]] = trunc nuw i32 [[MIN]] to i8
+; CHECK-NEXT:    ret i8 [[RET]]
+;
 entry:
   br label %for.body
 
@@ -196,9 +226,24 @@ for.end:
 
 define i16 @reduction_smax_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-LABEL: @reduction_smax_trunc(
-; CHECK-NOT: vector.body
-; CHECK-NOT: <8 x
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_02P:%.*]] = phi i32 [ [[MIN:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = and i32 [[SUM_02P]], 65535
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[LOAD]] to i32
+; CHECK-NEXT:    [[MIN]] = call i32 @llvm.smax.i32(i32 [[SUM_02]], i32 [[EXT]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[MIN]] to i16
+; CHECK-NEXT:    ret i16 [[RET]]
+;
 entry:
   br label %for.body
 
@@ -222,9 +267,24 @@ for.end:
 
 define i16 @reduction_umax_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-LABEL: @reduction_umax_trunc(
-; CHECK-NOT: vector.body
-; CHECK-NOT: <8 x
-; CHECK: ret
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_02P:%.*]] = phi i32 [ [[MIN:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = and i32 [[SUM_02P]], 65535
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[LOAD]] to i32
+; CHECK-NEXT:    [[MIN]] = call i32 @llvm.umax.i32(i32 [[SUM_02]], i32 [[EXT]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RET:%.*]] = trunc nuw i32 [[MIN]] to i16
+; CHECK-NEXT:    ret i16 [[RET]]
+;
 entry:
   br label %for.body
 
@@ -273,7 +333,7 @@ define i32 @reduction_and_or(i16 %a, i32 %b, ptr %src) {
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 992, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OR67:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[IV]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[OR]] = or i32 [[OR67]], [[L]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/vector-geps.ll b/llvm/test/Transforms/LoopVectorize/vector-geps.ll
index e307d93b1896a..94bc32205ec11 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-geps.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-geps.ll
@@ -31,8 +31,8 @@ define void @vector_gep_stored(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
-; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[I]]
+; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds nuw ptr, ptr [[A]], i64 [[I]]
 ; CHECK-NEXT:    store ptr [[VAR0]], ptr [[VAR1]], align 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
@@ -65,7 +65,7 @@ define void @uniform_vector_gep_stored(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 4
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 4
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP0]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x ptr> [[DOTSPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -84,8 +84,8 @@ define void @uniform_vector_gep_stored(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 4
-; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 4
+; CHECK-NEXT:    [[VAR1:%.*]] = getelementptr inbounds nuw ptr, ptr [[A]], i64 [[I]]
 ; CHECK-NEXT:    store ptr [[VAR0]], ptr [[VAR1]], align 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 1f815899ed55c..323a638fd6354 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -31,9 +31,6 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<exit>:
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
 ; CHECK-NEXT:  Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
@@ -41,6 +38,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ]
 ; CHECK:         IR   %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<exit>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
index 53f5a5658fb68..f0ac88c75e9ec 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
@@ -3,8 +3,8 @@
 
 define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK:      VPlan 'Final VPlan for VF={2},UF={1}' {
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<[[VFxUF:.+]]> = VF * UF
+; CHECK-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
@@ -16,7 +16,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT:     SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:     EMIT vp<[[PTR:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[WIDE_PTR:%.+]]> = vector-pointer vp<[[PTR]]>
@@ -76,27 +76,28 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK-NEXT:   Successor(s): default.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   default.2:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>
+; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VTC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
 ; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]>
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_CMP]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: ir-bb<scalar.ph>:
+; CHECK-NEXT:   EMIT vp<[[RESUME:%.+]]> = resume-phi ir<%ind.end>, ir<%start>
 ; CHECK-NEXT: Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
-; CHECK-NEXT:   IR   %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+; CHECK-NEXT:   IR   %ptr.iv = phi ptr [ %start, %scalar.ph ], [ %ptr.iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from ir-bb<scalar.ph>)
 ; CHECK-NEXT:   IR   %l = load i8, ptr %ptr.iv, align 1
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index c9612ced3eee0..6bd31565e9fb9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -42,9 +42,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[C]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -53,12 +50,15 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT:   IR   %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]
 ; CHECK:        IR   %cmp = icmp eq i64 %iv.next, 0
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 ; CHECK: Executing best plan with VF=8, UF=2
 ; CHECK-NEXT: VPlan 'Final VPlan for VF={8},UF={2}' {
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<[[VFxUF:.+]]> = VF * UF
+; CHECK-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
@@ -71,7 +71,7 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT:     SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:     EMIT vp<[[PADD1:%.+]]> = ptradd ir<%A>, vp<[[STEPS1]]>
 ; CHECK-NEXT:     vp<[[VPTR1:%.]]> = vector-pointer vp<[[PADD1]]>
@@ -84,28 +84,30 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT:     vp<[[VPTR4:%.+]]> = vector-pointer vp<[[PADD1]]>, ir<1>
 ; CHECK-NEXT:     WIDEN store vp<[[VPTR3]]>, ir<%add>
 ; CHECK-NEXT:     WIDEN store vp<[[VPTR4]]>, ir<%add>.1
-; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, ir<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-cond ir<true>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
 ; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]>
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT:   EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[C]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: ir-bb<scalar.ph>:
+; CHECK-NEXT:   EMIT vp<[[RESUME1:%.+]]> = resume-phi ir<%ind.end>, ir<%and>
+; CHECK-NEXT:   EMIT vp<[[RESUME2:%.+]]>.1 = resume-phi ir<%ind.end1>, ir<%A>
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT:   IR   %iv = phi i64 [ %and, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:   IR   %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]
+; CHECK-NEXT:   IR   %iv = phi i64 [ %and, %scalar.ph ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME1]]> from ir-bb<scalar.ph>)
+; CHECK-NEXT:   IR   %p.src = phi ptr [ %A, %scalar.ph ], [ %p.src.next, %loop ] (extra operand: vp<[[RESUME2]]>.1 from ir-bb<scalar.ph>)
 ; CHECK:        IR   %cmp = icmp eq i64 %iv.next, 0
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 50d406d0c0416..5e3116eae8548 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -47,9 +47,6 @@ define void @foo(i64 %n) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[C]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<outer.header>
 ; CHECK-EMPTY:
@@ -59,6 +56,9 @@ define void @foo(i64 %n) {
 ; CHECK-NEXT:   IR   store i64 %outer.iv, ptr %gep.1, align 4
 ; CHECK-NEXT:   IR   %add = add nsw i64 %outer.iv, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 entry:
   br label %outer.header
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 195f6a48640e5..32e55bff94b3e 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -38,9 +38,6 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.end.loopexit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.end.loopexit>
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -48,6 +45,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:    IR   %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.end.loopexit>
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -105,9 +105,6 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.end.loopexit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.end.loopexit>
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -115,6 +112,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-NEXT:    IR   %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.end.loopexit>
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -170,10 +170,6 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT:  IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
@@ -182,6 +178,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT:  IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -233,9 +233,6 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
@@ -245,6 +242,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT:    IR   %red = phi float [ %red.next, %for.body ], [ 0.000000e+00, %entry ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -319,9 +319,6 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -329,6 +326,9 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-NEXT:    IR   %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
 ; CHECK-NEXT:    IR   %cmp = icmp ult i64 %i, 5
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -396,9 +396,6 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -406,6 +403,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
 ; CHECK:         IR   %cmp = icmp slt i64 %iv.next, 1024
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -475,10 +475,6 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT:   IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
@@ -488,6 +484,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    IR   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
 ; CHECK:         IR   %exitcond.not = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT:   IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT: No successors
 ; CHECK-NEXT:}
 
 entry:
@@ -566,9 +566,6 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -576,6 +573,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ]
 ; CHECK:         IR   %cmp1 = icmp slt i32 %lsd, 100
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT:}
 ;
 entry:
@@ -651,9 +651,6 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<loop.exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop.exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -661,6 +658,9 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
 ; CHECK:         IR   %iv.next = add i64 %iv, %inc
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -715,10 +715,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:   IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -726,6 +722,10 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
 ; CHECK-NEXT:    IR   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %ec = icmp eq i32 %iv.next, 1000
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT:   IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]> from middle.block)
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -779,9 +779,6 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -789,6 +786,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -844,9 +844,6 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -854,6 +851,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -929,9 +929,6 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -939,6 +936,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ]
 ; CHECK:         IR   %ifcond = fcmp oeq float %ld.value, 5.0
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<end>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -1003,9 +1003,6 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -1013,6 +1010,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond = icmp eq i64 %iv.next, %n
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -1110,10 +1110,6 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:   IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT:   EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
@@ -1123,6 +1119,10 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
 ; CHECK-NEXT:    IR   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:         IR   %exitcond.not = icmp eq i64 %iv.next, 1000
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT:   IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]> from middle.block)
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index cdeffeff84d03..823b1cef93ce7 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -49,9 +49,6 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
 ; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.end>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT:  ir-bb<for.end>:
-; CHECK-NEXT:  No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
@@ -60,6 +57,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
 ; CHECK-NEXT:    IR   %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 0, %entry ]
 ; CHECK:         IR   %tmp5 = trunc i32 %tmp4 to i8
 ; CHECK-NEXT:  No successors
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.end>:
+; CHECK-NEXT:  No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 88e7aaccfe2f3..330a6b1715c78 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -51,7 +51,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; CHECK-NEXT: }
 
 ; CHECK:      loop.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -136,7 +136,7 @@ exit:
 ; CHECK-NEXT: }
 
 ; CHECK:       loop.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -221,7 +221,7 @@ exit:
 ; CHECK-NEXT: }
 
 ; CHECK:      loop.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -292,7 +292,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT: Successor(s): loop.then.0
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.then.0:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -384,7 +384,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
 ; CHECK-NEXT: Successor(s): next.0.1
 ; CHECK-EMPTY:
 ; CHECK-NEXT: next.0.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -484,7 +484,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
 ; CHECK-NEXT: Successor(s): then.1.1
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.1.1:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -592,7 +592,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
 ; CHECK-NEXT: Successor(s): then.1.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.1.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -701,7 +701,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
 ; CHECK-NEXT: Successor(s): then.0.4
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.0.4:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -781,7 +781,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
 ; CHECK-NEXT: Successor(s): loop.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -866,7 +866,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
 ; CHECK-NEXT: Successor(s): loop.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -1073,9 +1073,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
 ; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
@@ -1089,6 +1086,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
 ; CHECK-NEXT:   IR   store i32 %l, ptr %gep.dst, align 16
 ; CHECK-NEXT:   IR   %ec = icmp eq i32 %iv.next, 0
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
@@ -1164,9 +1164,6 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
 ; CHECK-NEXT: Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
@@ -1176,6 +1173,9 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT:   IR   %l = load i8, ptr %ptr.iv.next, align 1
 ; CHECK-NEXT:   IR   %c.1 = icmp eq i8 %l, 0
 ; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
index eec772c52bbb6..3f9bd9333805f 100644
--- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
@@ -14,7 +14,7 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp eq i32 [[ITR:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[FOR_END13:%.*]], label [[FOR_COND1_PREHEADER_PREHEADER:%.*]]
 ; CHECK:       for.cond1.preheader.preheader:
-; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 4
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[ITR]], -1
 ; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
 ; CHECK:       for.cond1.preheader:
@@ -22,25 +22,25 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture
 ; CHECK-NEXT:    [[J_016:%.*]] = phi i32 [ [[J_1_LCSSA:%.*]], [[FOR_INC11]] ], [ 0, [[FOR_COND1_PREHEADER_PREHEADER]] ]
 ; CHECK-NEXT:    [[I_015:%.*]] = phi i32 [ [[INC12:%.*]], [[FOR_INC11]] ], [ 0, [[FOR_COND1_PREHEADER_PREHEADER]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDVAR]], 2
-; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[VAR3:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[VAR3:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4
-; CHECK-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[VAR3]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[VAR3]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[CMP212:%.*]] = icmp ult i32 [[J_016]], [[ITR]]
 ; CHECK-NEXT:    br i1 [[CMP212]], label [[FOR_BODY3_LVER_CHECK:%.*]], label [[FOR_INC11]]
 ; CHECK:       for.body3.lver.check:
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[I_015]], [[ITR]]
 ; CHECK-NEXT:    [[IDXPROM6:%.*]] = zext i32 [[I_015]] to i64
-; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, ptr [[VAR3]], i64 [[IDXPROM6]]
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR3]], i64 [[IDXPROM6]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[J_016]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[VAR1]], i64 [[TMP4]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[VAR1]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP0]], [[J_016]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP4]], [[TMP7]]
-; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP1]], i64 [[TMP8]]
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[UGLYGEP]], [[UGLYGEP4]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[UGLYGEP3]], [[UGLYGEP2]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP8]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[FOR_BODY3_PH_LVER_ORIG:%.*]], label [[FOR_BODY3_PH:%.*]]
 ; CHECK:       for.body3.ph.lver.orig:
@@ -48,7 +48,7 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture
 ; CHECK:       for.body3.lver.orig:
 ; CHECK-NEXT:    [[J_113_LVER_ORIG:%.*]] = phi i32 [ [[J_016]], [[FOR_BODY3_PH_LVER_ORIG]] ], [ [[INC_LVER_ORIG:%.*]], [[FOR_BODY3_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[IDXPROM_LVER_ORIG:%.*]] = zext i32 [[J_113_LVER_ORIG]] to i64
-; CHECK-NEXT:    [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[VAR1]], i64 [[IDXPROM_LVER_ORIG]]
+; CHECK-NEXT:    [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR1]], i64 [[IDXPROM_LVER_ORIG]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX_LVER_ORIG]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4
 ; CHECK-NEXT:    [[ADD8_LVER_ORIG:%.*]] = add nsw i32 [[TMP9]], [[ADD]]
@@ -57,14 +57,14 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture
 ; CHECK-NEXT:    [[CMP2_LVER_ORIG:%.*]] = icmp ult i32 [[INC_LVER_ORIG]], [[ITR]]
 ; CHECK-NEXT:    br i1 [[CMP2_LVER_ORIG]], label [[FOR_BODY3_LVER_ORIG]], label [[FOR_INC11_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.body3.ph:
-; CHECK-NEXT:    [[ARRAYIDX7_PROMOTED:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4, !alias.scope !2, !noalias !2
+; CHECK-NEXT:    [[ARRAYIDX7_PROMOTED:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4, !alias.scope [[META2:![0-9]+]], !noalias [[META2]]
 ; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[ADD86:%.*]] = phi i32 [ [[ARRAYIDX7_PROMOTED]], [[FOR_BODY3_PH]] ], [ [[ADD8:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[J_113:%.*]] = phi i32 [ [[J_016]], [[FOR_BODY3_PH]] ], [ [[INC:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[J_113]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[VAR1]], i64 [[IDXPROM]]
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX]], align 4, !alias.scope !2, !noalias !2
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[VAR1]], i64 [[IDXPROM]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX]], align 4, !alias.scope [[META2]], !noalias [[META2]]
 ; CHECK-NEXT:    [[ADD8]] = add nsw i32 [[ADD86]], [[ADD]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[J_113]], 1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[INC]], [[ITR]]
@@ -73,7 +73,7 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture
 ; CHECK-NEXT:    br label [[FOR_INC11_LOOPEXIT:%.*]]
 ; CHECK:       for.inc11.loopexit.loopexit5:
 ; CHECK-NEXT:    [[ADD8_LCSSA:%.*]] = phi i32 [ [[ADD8]], [[FOR_BODY3]] ]
-; CHECK-NEXT:    store i32 [[ADD8_LCSSA]], ptr [[ARRAYIDX7]], align 4, !alias.scope !2, !noalias !2
+; CHECK-NEXT:    store i32 [[ADD8_LCSSA]], ptr [[ARRAYIDX7]], align 4, !alias.scope [[META2]], !noalias [[META2]]
 ; CHECK-NEXT:    br label [[FOR_INC11_LOOPEXIT]]
 ; CHECK:       for.inc11.loopexit:
 ; CHECK-NEXT:    br label [[FOR_INC11]]
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-range.ll b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-range.ll
new file mode 100644
index 0000000000000..f84ebee144289
--- /dev/null
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-range.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=lower-constant-intrinsics -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i64 @llvm.objectsize.i64.p0(ptr, i1 immarg, i1 immarg, i1 immarg)
+declare noalias ptr @malloc(i64 noundef) #0
+
+define i64 @select_alloc_size(i1 %cond) {
+; CHECK-LABEL: @select_alloc_size(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[COND:%.*]], i64 3, i64 4
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8, i64 [[SIZE]], align 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND]], i64 4, i64 3
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %size = select i1 %cond, i64 3, i64 4
+  %ptr = alloca i8, i64 %size
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr, i1 true, i1 true, i1 false)
+  %res = select i1 %cond, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+define i64 @select_malloc_size(i1 %cond) {
+; CHECK-LABEL: @select_malloc_size(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[COND:%.*]], i64 3, i64 4
+; CHECK-NEXT:    [[PTR:%.*]] = call noalias ptr @malloc(i64 noundef [[SIZE]])
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND]], i64 4, i64 3
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %size = select i1 %cond, i64 3, i64 4
+  %ptr = call noalias ptr @malloc(i64 noundef %size)
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr, i1 true, i1 true, i1 false)
+  %res = select i1 %cond, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+define i64 @select_gep_offset(i1 %cond) {
+; CHECK-LABEL: @select_gep_offset(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8, i64 10, align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[COND:%.*]], i64 3, i64 4
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND]], i64 7, i64 6
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %ptr = alloca i8, i64 10
+  %offset = select i1 %cond, i64 3, i64 4
+  %ptr.slide = getelementptr inbounds i8, ptr %ptr, i64 %offset
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %cond, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+define i64 @select_gep_neg_offset(i1 %c0, i1 %c1) {
+; CHECK-LABEL: @select_gep_neg_offset(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8, i64 10, align 1
+; CHECK-NEXT:    [[PTR_SLIDE_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 5
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[COND:%.*]], i64 -3, i64 -4
+; CHECK-NEXT:    [[PTR_SLIDE_2:%.*]] = getelementptr inbounds i8, ptr [[PTR_SLIDE_1]], i64 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1:%.*]], i64 9, i64 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %ptr = alloca i8, i64 10
+  %ptr.slide.1 = getelementptr inbounds i8, ptr %ptr, i64 5
+  %offset = select i1 %c0, i64 -3, i64 -4
+  %ptr.slide.2 = getelementptr inbounds i8, ptr %ptr.slide.1, i64 %offset
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide.2, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide.2, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+define i64 @select_neg_oob_offset(i1 %c0, i1 %c1) {
+; CHECK-LABEL: @select_neg_oob_offset(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8, i64 10, align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0:%.*]], i64 -3, i64 -4
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[OFFSET]]
+; CHECK-NEXT:    ret i64 0
+;
+  %ptr = alloca i8, i64 10
+  %offset = select i1 %c0, i64 -3, i64 -4
+  %ptr.slide = getelementptr inbounds i8, ptr %ptr, i64 %offset
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+define i64 @select_gep_offsets(i1 %cond) {
+; CHECK-LABEL: @select_gep_offsets(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca [10 x i8], i64 2, align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[COND:%.*]], i32 0, i32 1
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr inbounds [10 x i8], ptr [[PTR]], i32 [[OFFSET]], i32 5
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND]], i64 15, i64 5
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %ptr = alloca [10 x i8], i64 2
+  %offset = select i1 %cond, i32 0, i32 1
+  %ptr.slide = getelementptr inbounds [10 x i8], ptr %ptr, i32 %offset, i32 5
+  %objsize_max = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i64 @llvm.objectsize.i64.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %cond, i64 %objsize_max, i64 %objsize_min
+  ret i64 %res
+}
+
+attributes #0 = { nounwind allocsize(0) }
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll
new file mode 100644
index 0000000000000..b78d56646d9e4
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p lower-matrix-intrinsics -S %s | FileCheck %s
+
+define void @test(ptr %p, <8 x i32> %x) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], <8 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[L:%.*]] = load <8 x i32>, ptr [[P]], align 4
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 3>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 4>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 5>
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 6>
+; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[TMP5]], i64 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP7]], i64 3
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <1 x i32> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <1 x i32> [[SPLIT5]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP11]], i64 5
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i32> [[SPLIT6]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i32> [[SPLIT7]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP15]], i64 7
+; CHECK-NEXT:    [[TMP17:%.*]] = mul <8 x i32> [[L]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP17]])
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <1 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT:    [[E:%.*]] = extractelement <1 x i32> [[TMP19]], i64 0
+; CHECK-NEXT:    store i32 [[E]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  %l = load <8 x i32>, ptr %p, align 4
+  %t = tail call <8 x i32> @llvm.matrix.transpose.v8i32(<8 x i32> %x, i32 1, i32 8)
+  %m = tail call <1 x i32> @llvm.matrix.multiply.v1i32.v8i32.v8i32(<8 x i32> %l, <8 x i32> %t, i32 1, i32 8, i32 1)
+  %e = extractelement <1 x i32> %m, i64 0
+  store i32 %e, ptr %p, align 4
+  ret void
+}
+
+declare <8 x i32> @llvm.matrix.transpose.v8i32(<8 x i32>, i32 immarg, i32 immarg)
+
+declare <1 x i32> @llvm.matrix.multiply.v1i32.v8i32.v8i32(<8 x i32>, <8 x i32>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
index 2fd77e245a34e..aadaf1ffffb23 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
@@ -190,3 +190,33 @@ declare <1 x i32> @llvm.matrix.multiply.v1i32.v5i32.v5i32(<5 x i32>, <5 x i32>,
 declare <5 x i32> @llvm.matrix.column.major.load.v5i32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg) #1
 
 declare <5 x i32> @llvm.matrix.transpose.v5i32(<5 x i32>, i32 immarg, i32 immarg) #0
+
+define <1 x i32> @test_dot_product_with_transposed_shuffle_op(<4 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test_dot_product_with_transposed_shuffle_op(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> zeroinitializer, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <2 x i32> [[SHUFFLE]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <1 x i32> poison, i32 [[TMP10]], i64 0
+; CHECK-NEXT:    ret <1 x i32> [[TMP11]]
+;
+entry:
+  %t.a = tail call <4 x i32> @llvm.matrix.transpose.v4i32(<4 x i32> %a, i32 2, i32 2)
+  %shuffle = shufflevector <4 x i32> %t.a, <4 x i32> zeroinitializer, <2 x i32> <i32 0, i32 1>
+  %t.shuffle = call <2 x i32> @llvm.matrix.transpose.v2i32(<2 x i32> %shuffle, i32 2, i32 1)
+  %m = call <1 x i32> @llvm.matrix.multiply.v1i32.v2i32.v2i32(<2 x i32> %t.shuffle, <2 x i32> %b, i32 1, i32 2, i32 1)
+  ret <1 x i32> %m
+}
+
+declare <2 x i32> @llvm.matrix.transpose.v2i32(<2 x i32>, i32 immarg, i32 immarg)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
index 249608f413fb2..8c2cc8e799bcd 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
@@ -305,15 +305,15 @@ define void @multiply_alias_2x2(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
 ; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
 ; CHECK:       inner.body:
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[INNER_IV]], 1
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], [[ROWS_IV]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[TMP3]], i64 [[TMP9]]
+; CHECK-NEXT:    [[DOTIDX1:%.*]] = shl i64 [[INNER_IV]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[DOTIDX1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[ROWS_IV]]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[COLS_IV]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], [[INNER_IV]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP12]]
+; CHECK-NEXT:    [[DOTIDX24:%.*]] = shl i64 [[COLS_IV]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[DOTIDX24]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[TMP11]], i64 [[INNER_IV]]
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <2 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr i8, ptr [[TMP13]], i64 8
 ; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <2 x float>, ptr [[VEC_GEP10]], align 4
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting-constant-folds.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting-constant-folds.ll
new file mode 100644
index 0000000000000..4eaa0e63a29c4
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting-constant-folds.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p lower-matrix-intrinsics -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define <8 x float> @transpose_constant_fold_fadd_AT_BT() {
+; CHECK-LABEL: define <8 x float> @transpose_constant_fold_fadd_AT_BT() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret <8 x float> splat (float 2.000000e+00)
+;
+entry:
+  %t = tail call <8 x float> @llvm.matrix.transpose.v8f32(<8 x float> splat (float 1.0), i32 8, i32 1)
+  %f = fadd <8 x float> %t, %t
+  ret <8 x float> %f
+}
+
+define <8 x float> @transpose_constant_fold_fmul_A_k() {
+; CHECK-LABEL: define <8 x float> @transpose_constant_fold_fmul_A_k() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <8 x float> splat (float 3.000000e+00), <8 x float> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x float> [[SPLAT]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x float> [[SPLAT]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = fmul <4 x float> splat (float 1.000000e+00), [[SPLIT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> splat (float 1.000000e+00), [[SPLIT1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+entry:
+  %t.1 = tail call <8 x float> @llvm.matrix.transpose.v8f32(<8 x float> splat (float 1.0), i32 4, i32 2)
+  %splat = shufflevector <8 x float> splat (float 3.0), <8 x float> poison, <8 x i32> zeroinitializer
+  %m = fmul <8 x float> %t.1, %splat
+  %t.2 = tail call <8 x float> @llvm.matrix.transpose.v8f32(<8 x float> %m, i32 2, i32 4)
+  ret <8 x float> %t.2
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <8 x float> @llvm.matrix.transpose.v8f32(<8 x float>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
index fcf83b03bc3d2..1b3b41d8cfe1f 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
@@ -144,8 +144,28 @@ entry:
   ret <6 x double> %mul
 }
 
+define void @test_remove_entries_from_shape_map(<3 x float> %a, <2 x float> %b, <6 x float> %c, ptr %dst) {
+; CHECK-LABEL: define void @test_remove_entries_from_shape_map(
+; CHECK-SAME: <3 x float> [[A:%.*]], <2 x float> [[B:%.*]], <6 x float> [[C:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float> [[A]], <2 x float> [[B]], i32 3, i32 1, i32 2)
+; CHECK-NEXT:    [[MFADD:%.*]] = fadd <6 x float> [[C]], [[TMP0]]
+; CHECK-NEXT:    [[MFADD_T:%.*]] = call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> [[MFADD]], i32 3, i32 2)
+; CHECK-NEXT:    store <6 x float> [[MFADD_T]], ptr [[DST]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %m = tail call <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float> %a, <2 x float> %b, i32 3, i32 1, i32 2)
+  %add = fadd <6 x float> %c, %m
+  %t = tail call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> %add, i32 3, i32 2)
+  store <6 x float> %t, ptr %dst, align 4
+  ret void
+}
+
 declare <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double>, i32, i32)
 declare <4 x double> @llvm.matrix.transpose.v4f64.v4f64(<4 x double>, i32, i32)
 declare <9 x double> @llvm.matrix.multiply.v9f64.v6f64(<6 x double>, <6 x double>, i32, i32, i32)
 declare <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double>, <4 x double>, i32, i32, i32)
 declare <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double>, <4 x double>, i32, i32, i32)
+declare <6 x float> @llvm.matrix.transpose.v6f32(<6 x float>, i32 immarg, i32 immarg)
+declare <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float>, <2 x float>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
new file mode 100644
index 0000000000000..c7593e2941518
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=memcpyopt -S %s -verify-memoryssa | FileCheck %s
+
+; Redundant memmove.
+define i32 @redundant_memmove() {
+; CHECK-LABEL: @redundant_memmove(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 104, i1 false)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 100, i1 false)
+  %val = load i32, ptr %array, align 16
+  ret i32 %val
+}
+
+; Used memmove, buffer is reset to zero.
+define i32 @used_memmove_1() {
+; CHECK-LABEL: @used_memmove_1(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    store i32 1, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY]], ptr align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 104, i1 false)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  store i32 1, ptr %array.idx
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 100, i1 false)
+  %val = load i32, ptr %array.idx, align 4
+  ret i32 %val
+}
+
+; Used memmove.
+define i32 @used_memmove_2() {
+; CHECK-LABEL: @used_memmove_2(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    store i32 1, ptr [[ARRAY]], align 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY]], ptr align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 104, i1 false)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  store i32 1, ptr %array
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 100, i1 false)
+  %val = load i32, ptr %array.idx, align 4
+  ret i32 %val
+}
+
+; Used memmove, buffer clobbered by opaque.
+define i32 @used_memmove_3() {
+; CHECK-LABEL: @used_memmove_3(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [25 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 100, i1 false)
+; CHECK-NEXT:    call void @opaque(ptr [[ARRAY]])
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY]], ptr align 4 [[ARRAY_IDX]], i64 96, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [25 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 100, i1 false)
+  call void @opaque(ptr %array)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 96, i1 false)
+  %val = load i32, ptr %array, align 16
+  ret i32 %val
+}
+
+; Redundant memmove, not within the same basic block.
+define i32 @redundant_memmove_different_bbs() {
+; CHECK-LABEL: @redundant_memmove_different_bbs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    br label [[USE:%.*]]
+; CHECK:       use:
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+entry:
+  %array = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 104, i1 false)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  br label %use
+
+use:                                              ; preds = %entry
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 100, i1 false)
+  %val = load i32, ptr %array, align 16
+  ret i32 %val
+}
+
+@g_var = global [26 x i32] zeroinitializer, align 16
+
+; Redundant memmove on a global variable.
+define ptr @redundant_memmove_memset_global_variable() {
+; CHECK-LABEL: @redundant_memmove_memset_global_variable(
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 @g_var, i8 0, i64 104, i1 false)
+; CHECK-NEXT:    ret ptr @g_var
+;
+  call void @llvm.memset.p0.i64(ptr align 16 @g_var, i8 0, i64 104, i1 false)
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 @g_var, ptr align 4 getelementptr inbounds nuw (i8, ptr @g_var, i64 4), i64 100, i1 false)
+  ret ptr @g_var
+}
+
+; Memset only partial.
+define i32 @partial_memset() {
+; CHECK-LABEL: @partial_memset(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 92
+; CHECK-NEXT:    store i32 1, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 26, i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX_2:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY]], ptr align 4 [[ARRAY_IDX_2]], i64 100, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 92
+  store i32 1, ptr %array.idx
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 26, i1 false)
+  %array.idx.2 = getelementptr inbounds i8, ptr %array, i64 4
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx.2, i64 100, i1 false)
+  %val = load i32, ptr %array.idx, align 4
+  ret i32 %val
+}
+
+; Memset length not constant.
+define i32 @memset_length_not_constant(i64 %size) {
+; CHECK-LABEL: @memset_length_not_constant(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 [[SIZE:%.*]], i1 false)
+; CHECK-NEXT:    [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY]], ptr align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 %size, i1 false)
+  %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array, ptr align 4 %array.idx, i64 100, i1 false)
+  %val = load i32, ptr %array, align 16
+  ret i32 %val
+}
+
+; Memmove buffer not memset'd, different buffers.
+define i32 @memset_memmove_dest_buffers_not_alias() {
+; CHECK-LABEL: @memset_memmove_dest_buffers_not_alias(
+; CHECK-NEXT:    [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    [[ARRAY2:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT:    [[ARRAY2_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY2]], i64 4
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 16 [[ARRAY2]], ptr align 4 [[ARRAY2_IDX]], i64 100, i1 false)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAY2]], align 16
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+  %array = alloca [26 x i32], align 16
+  %array2 = alloca [26 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %array, i8 0, i64 104, i1 false)
+  %array2.idx = getelementptr inbounds i8, ptr %array2, i64 4
+  call void @llvm.memmove.p0.p0.i64(ptr align 16 %array2, ptr align 4 %array2.idx, i64 100, i1 false)
+  %val = load i32, ptr %array2, align 16
+  ret i32 %val
+}
+
+declare void @opaque(ptr)
+declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1)
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1)
diff --git a/llvm/test/Transforms/MemCpyOpt/vscale-memset.ll b/llvm/test/Transforms/MemCpyOpt/vscale-memset.ll
index b4ab443fdfb68..45de52065cd5c 100644
--- a/llvm/test/Transforms/MemCpyOpt/vscale-memset.ll
+++ b/llvm/test/Transforms/MemCpyOpt/vscale-memset.ll
@@ -8,7 +8,7 @@
 define void @foo(ptr %p) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[P:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[P]], i64 1
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP1]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -18,6 +18,18 @@ define void @foo(ptr %p) {
   ret void
 }
 
+; Test the compiler does not crash on a store of a scalable aggregate type.
+define void @test_no_crash_scalable_agg(ptr %p) {
+; CHECK-LABEL: @test_no_crash_scalable_agg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store { <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[P:%.*]], align 2
+; CHECK-NEXT:    ret void
+;
+entry:
+  store { <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr %p, align 2
+  ret void
+}
+
 ; Positive test
 
 define void @memset_vscale_index_zero(ptr %p, i8 %z) {
diff --git a/llvm/test/Transforms/MergeFunc/initializes-attr.ll b/llvm/test/Transforms/MergeFunc/initializes-attr.ll
new file mode 100644
index 0000000000000..0bd4fe2e474dd
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/initializes-attr.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=mergefunc < %s | FileCheck %s
+
+define internal void @test1(ptr initializes((0, 1)) %p) {
+; CHECK-LABEL: define internal void @test1(
+; CHECK-SAME: ptr initializes((0, 1)) [[P:%.*]]) {
+; CHECK-NEXT:    store i16 0, ptr [[P]], align 2
+; CHECK-NEXT:    ret void
+;
+  store i16 0, ptr %p
+  ret void
+}
+
+define internal void @test2(ptr initializes((0, 1)) %p) {
+  store i16 0, ptr %p
+  ret void
+}
+
+define internal void @test3(ptr initializes((0, 2)) %p) {
+; CHECK-LABEL: define internal void @test3(
+; CHECK-SAME: ptr initializes((0, 2)) [[P:%.*]]) {
+; CHECK-NEXT:    store i16 0, ptr [[P]], align 2
+; CHECK-NEXT:    ret void
+;
+  store i16 0, ptr %p
+  ret void
+}
+
+define internal void @test4(ptr initializes((0, 1), (2, 3)) %p) {
+; CHECK-LABEL: define internal void @test4(
+; CHECK-SAME: ptr initializes((0, 1), (2, 3)) [[P:%.*]]) {
+; CHECK-NEXT:    store i16 0, ptr [[P]], align 2
+; CHECK-NEXT:    ret void
+;
+  store i16 0, ptr %p
+  ret void
+}
+
+define void @do_calls(ptr %p) {
+; CHECK-LABEL: define void @do_calls(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    call void @test1(ptr [[P]])
+; CHECK-NEXT:    call void @test1(ptr [[P]])
+; CHECK-NEXT:    call void @test3(ptr [[P]])
+; CHECK-NEXT:    call void @test4(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  call void @test1(ptr %p)
+  call void @test2(ptr %p)
+  call void @test3(ptr %p)
+  call void @test4(ptr %p)
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/loadforward.ll b/llvm/test/Transforms/NewGVN/loadforward.ll
index a44a6e92b8adf..1523d1894e9e3 100644
--- a/llvm/test/Transforms/NewGVN/loadforward.ll
+++ b/llvm/test/Transforms/NewGVN/loadforward.ll
@@ -9,7 +9,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ;; Test that we forward the first store to the second load
 define i16 @bazinga() {
 ; CHECK-LABEL: @bazinga(
-; CHECK-NEXT:    [[_TMP10:%.*]] = load i16, ptr getelementptr inbounds (i8, ptr @str, i64 2), align 2
+; CHECK-NEXT:    [[_TMP10:%.*]] = load i16, ptr getelementptr inbounds nuw (i8, ptr @str, i64 2), align 2
 ; CHECK-NEXT:    store i16 [[_TMP10]], ptr @str, align 2
 ; CHECK-NEXT:    [[_TMP15:%.*]] = icmp eq i16 [[_TMP10]], 3
 ; CHECK-NEXT:    [[_TMP16:%.*]] = select i1 [[_TMP15]], i16 1, i16 0
diff --git a/llvm/test/Transforms/ObjCARC/allocas.ll b/llvm/test/Transforms/ObjCARC/allocas.ll
index be829882ae374..6fe2edf3e2dd4 100644
--- a/llvm/test/Transforms/ObjCARC/allocas.ll
+++ b/llvm/test/Transforms/ObjCARC/allocas.ll
@@ -109,16 +109,16 @@ entry:
 }
 
 
-; CHECK: define void @test1d(ptr %x)
+; CHECK: define void @test1d(ptr %x, i1 %arg)
 ; CHECK: @llvm.objc.retain(ptr %x)
 ; CHECK: @llvm.objc.retain(ptr %x)
 ; CHECK: @llvm.objc.release(ptr %y)
 ; CHECK: @llvm.objc.release(ptr %x)
 ; CHECK: ret void
 ; CHECK: }
-define void @test1d(ptr %x) {
+define void @test1d(ptr %x, i1 %arg) {
 entry:
-  br i1 undef, label %use_allocaA, label %use_allocaB
+  br i1 %arg, label %use_allocaA, label %use_allocaB
 
 use_allocaA:
   %allocaA = alloca ptr
@@ -141,16 +141,16 @@ exit:
   ret void
 }
 
-; CHECK: define void @test1e(ptr %x)
+; CHECK: define void @test1e(ptr %x, i1 %arg)
 ; CHECK: @llvm.objc.retain(ptr %x)
 ; CHECK: @llvm.objc.retain(ptr %x)
 ; CHECK: @llvm.objc.release(ptr %y)
 ; CHECK: @llvm.objc.release(ptr %x)
 ; CHECK: ret void
 ; CHECK: }
-define void @test1e(ptr %x) {
+define void @test1e(ptr %x, i1 %arg) {
 entry:
-  br i1 undef, label %use_allocaA, label %use_allocaB
+  br i1 %arg, label %use_allocaA, label %use_allocaB
 
 use_allocaA:
   %allocaA = alloca ptr, i32 4
diff --git a/llvm/test/Transforms/ObjCARC/basic.ll b/llvm/test/Transforms/ObjCARC/basic.ll
index 0ee59dc8ba6ab..d461bc0af680d 100644
--- a/llvm/test/Transforms/ObjCARC/basic.ll
+++ b/llvm/test/Transforms/ObjCARC/basic.ll
@@ -1761,13 +1761,13 @@ g:
 ; CHECK-LABEL: define void @test39(
 ; CHECK-NOT: @llvm.objc.
 ; CHECK: {{^}}}
-define void @test39(ptr %p) {
+define void @test39(ptr %p, i1 %arg) {
 entry:
   %0 = call ptr @llvm.objc.retain(ptr %p)
   br label %loop
 
 loop:                                             ; preds = %loop, %entry
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:                                             ; preds = %loop
   call void @llvm.objc.release(ptr %0), !clang.imprecise_release !0
@@ -1779,14 +1779,14 @@ exit:                                             ; preds = %loop
 ; CHECK-LABEL: define void @test39b(
 ; CHECK-NOT: @llvm.objc.
 ; CHECK: {{^}}}
-define void @test39b(ptr %p) {
+define void @test39b(ptr %p, i1 %arg) {
 entry:
   %0 = call ptr @llvm.objc.retain(ptr %p)
   br label %loop
 
 loop:                                             ; preds = %loop, %entry
   store i8 0, ptr %0
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:                                             ; preds = %loop
   call void @llvm.objc.release(ptr %0), !clang.imprecise_release !0
@@ -1798,14 +1798,14 @@ exit:                                             ; preds = %loop
 ; CHECK-LABEL: define void @test39c(
 ; CHECK-NOT: @llvm.objc.
 ; CHECK: {{^}}}
-define void @test39c(ptr %p) {
+define void @test39c(ptr %p, i1 %arg) {
 entry:
   %0 = call ptr @llvm.objc.retain(ptr %p)
   br label %loop
 
 loop:                                             ; preds = %loop, %entry
   call void @use_pointer(ptr %0)
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:                                             ; preds = %loop
   call void @llvm.objc.release(ptr %0), !clang.imprecise_release !0
@@ -1818,14 +1818,14 @@ exit:                                             ; preds = %loop
 ; CHECK-LABEL: define void @test40(
 ; CHECK-NOT: @llvm.objc.
 ; CHECK: {{^}}}
-define void @test40(ptr %p) {
+define void @test40(ptr %p, i1 %arg) {
 entry:
   %0 = call ptr @llvm.objc.retain(ptr %p)
   br label %loop
 
 loop:                                             ; preds = %loop, %entry
   call void @use_pointer(ptr %0)
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:                                             ; preds = %loop
   call void @llvm.objc.release(ptr %0), !clang.imprecise_release !0
diff --git a/llvm/test/Transforms/ObjCARC/cfg-hazards.ll b/llvm/test/Transforms/ObjCARC/cfg-hazards.ll
index 3e762de689d1f..d43f713d925e0 100644
--- a/llvm/test/Transforms/ObjCARC/cfg-hazards.ll
+++ b/llvm/test/Transforms/ObjCARC/cfg-hazards.ll
@@ -111,7 +111,7 @@ for.end:                                          ; preds = %for.body
 }
 
 ; Delete nested retain+release pairs around loops.
-define void @test3(ptr %a) nounwind {
+define void @test3(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
@@ -119,7 +119,7 @@ define void @test3(ptr %a) nounwind {
 ; CHECK:       loop:
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    store i8 0, ptr [[A]], align 1
-; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
 ; CHECK-NEXT:    ret void
@@ -132,7 +132,7 @@ entry:
 loop:
   call void @callee()
   store i8 0, ptr %a
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -140,7 +140,7 @@ exit:
   ret void
 }
 
-define void @test4(ptr %a) nounwind {
+define void @test4(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
@@ -151,7 +151,7 @@ define void @test4(ptr %a) nounwind {
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    store i8 0, ptr [[A]], align 1
-; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
 ; CHECK-NEXT:    ret void
@@ -168,7 +168,7 @@ more:
   call void @callee()
   call void @callee()
   store i8 0, ptr %a
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -176,18 +176,18 @@ exit:
   ret void
 }
 
-define void @test5(ptr %a) nounwind {
+define void @test5(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @use_pointer(ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
@@ -200,13 +200,13 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @use_pointer(ptr %a)
@@ -215,18 +215,18 @@ exit:
   ret void
 }
 
-define void @test6(ptr %a) nounwind {
+define void @test6(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @use_pointer(ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
@@ -238,14 +238,14 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   call void @callee()
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @use_pointer(ptr %a)
@@ -254,19 +254,19 @@ exit:
   ret void
 }
 
-define void @test7(ptr %a) nounwind {
+define void @test7(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @use_pointer(ptr [[A]])
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
 ; CHECK-NEXT:    ret void
@@ -278,14 +278,14 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   call void @use_pointer(ptr %a)
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -293,19 +293,19 @@ exit:
   ret void
 }
 
-define void @test8(ptr %a) nounwind {
+define void @test8(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    call void @use_pointer(ptr [[A]])
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
 ; CHECK-NEXT:    ret void
@@ -316,7 +316,7 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   call void @callee()
@@ -324,7 +324,7 @@ true:
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -332,17 +332,17 @@ exit:
   ret void
 }
 
-define void @test9(ptr %a) nounwind {
+define void @test9(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @use_pointer(ptr [[A:%.*]])
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -352,14 +352,14 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   call void @use_pointer(ptr %a)
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -367,17 +367,17 @@ exit:
   ret void
 }
 
-define void @test10(ptr %a) nounwind {
+define void @test10(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @callee()
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -387,14 +387,14 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   call void @callee()
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -402,16 +402,16 @@ exit:
   ret void
 }
 
-define void @test11(ptr %a) nounwind {
+define void @test11(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test11(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    br label [[MORE]]
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -421,13 +421,13 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   br label %more
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -437,18 +437,18 @@ exit:
 
 ; Don't delete anything if they're not balanced.
 
-define void @test12(ptr %a) nounwind {
+define void @test12(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test12(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OUTER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    [[INNER:%.*]] = tail call ptr @llvm.objc.retain(ptr [[A]]) #[[ATTR0]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    br i1 undef, label [[TRUE:%.*]], label [[MORE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[TRUE:%.*]], label [[MORE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    ret void
 ; CHECK:       more:
-; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]]
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
@@ -460,13 +460,13 @@ entry:
   br label %loop
 
 loop:
-  br i1 undef, label %true, label %more
+  br i1 %arg, label %true, label %more
 
 true:
   ret void
 
 more:
-  br i1 undef, label %exit, label %loop
+  br i1 %arg, label %exit, label %loop
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind
@@ -479,7 +479,7 @@ exit:
 ; by an alloca.
 ; rdar://12969722
 
-define void @test13(ptr %a) nounwind {
+define void @test13(ptr %a, i1 %arg) nounwind {
 ; CHECK-LABEL: @test13(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BLOCK:%.*]] = alloca ptr, align 8
@@ -491,7 +491,7 @@ define void @test13(ptr %a) nounwind {
 ; CHECK-NEXT:    call void @block_callee(ptr [[BLOCK]])
 ; CHECK-NEXT:    [[RELOADED_A:%.*]] = load ptr, ptr [[BLOCK]], align 8
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[RELOADED_A]]) #[[ATTR0]], !clang.imprecise_release !0
-; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.objc.release(ptr [[A]]) #[[ATTR0]], !clang.imprecise_release !0
 ; CHECK-NEXT:    ret void
@@ -507,7 +507,7 @@ loop:
   call void @block_callee(ptr %block)
   %reloaded_a = load ptr, ptr %block, align 8
   call void @llvm.objc.release(ptr %reloaded_a) nounwind, !clang.imprecise_release !0
-  br i1 undef, label %loop, label %exit
+  br i1 %arg, label %loop, label %exit
 
 exit:
   call void @llvm.objc.release(ptr %a) nounwind, !clang.imprecise_release !0
diff --git a/llvm/test/Transforms/ObjCARC/contract-testcases.ll b/llvm/test/Transforms/ObjCARC/contract-testcases.ll
index 36fe49382e413..4c842823b54e0 100644
--- a/llvm/test/Transforms/ObjCARC/contract-testcases.ll
+++ b/llvm/test/Transforms/ObjCARC/contract-testcases.ll
@@ -19,13 +19,13 @@ declare i32 @__gxx_personality_sj0(...)
 ; Don't get in trouble on bugpointed code.
 
 ; CHECK-LABEL: define void @test0(
-define void @test0() {
+define void @test0(i1 %arg) {
 bb:
   %tmp1 = tail call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr undef) nounwind
   br label %bb3
 
 bb3:                                              ; preds = %bb2
-  br i1 undef, label %bb6, label %bb4
+  br i1 %arg, label %bb6, label %bb4
 
 bb4:                                              ; preds = %bb3
   switch i64 undef, label %bb5 [
@@ -45,15 +45,15 @@ bb6:                                              ; preds = %bb5, %bb4, %bb4, %b
 ; for the same block, use the exactly same value in each block.
 
 ; CHECK-LABEL: define void @test1(
-; CHECK: br i1 undef, label %bb7, label %bb7
+; CHECK: br i1 %arg, label %bb7, label %bb7
 ; CHECK: bb7:
 ; CHECK: %tmp8 = phi ptr [ %tmp3, %bb ], [ %tmp3, %bb ]
 ; CHECK: }
-define void @test1() {
+define void @test1(i1 %arg) {
 bb:
   %tmp = tail call ptr @objc_msgSend()
   %tmp3 = tail call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr %tmp) nounwind
-  br i1 undef, label %bb7, label %bb7
+  br i1 %arg, label %bb7, label %bb7
 
 bb7:                                              ; preds = %bb6, %bb6, %bb5
   %tmp8 = phi ptr [ %tmp, %bb ], [ %tmp, %bb ]
diff --git a/llvm/test/Transforms/ObjCARC/empty-block.ll b/llvm/test/Transforms/ObjCARC/empty-block.ll
index 80930812fc7e9..be70beeeb731d 100644
--- a/llvm/test/Transforms/ObjCARC/empty-block.ll
+++ b/llvm/test/Transforms/ObjCARC/empty-block.ll
@@ -18,9 +18,9 @@ declare ptr @llvm.objc.autoreleaseReturnValue(ptr)
 ; CHECK: @llvm.objc.autoreleaseReturnValue
 ; CHECK-NOT: @llvm.objc.
 ; CHECK: }
-define ptr @test0(ptr %buffer) nounwind {
+define ptr @test0(ptr %buffer, i1 %arg) nounwind {
   %1 = tail call ptr @llvm.objc.retain(ptr %buffer) nounwind
-  br i1 undef, label %.lr.ph, label %._crit_edge
+  br i1 %arg, label %.lr.ph, label %._crit_edge
 
 .lr.ph:                                           ; preds = %.lr.ph, %0
   br i1 false, label %.lr.ph, label %._crit_edge
@@ -37,10 +37,10 @@ define ptr @test0(ptr %buffer) nounwind {
 ; CHECK-LABEL: define ptr @test1(
 ; CHECK-NOT: @objc
 ; CHECK: }
-define ptr @test1() nounwind {
+define ptr @test1(i1 %arg) nounwind {
   %buffer = call ptr @foo()
   %1 = tail call ptr @llvm.objc.retain(ptr %buffer) nounwind
-  br i1 undef, label %.lr.ph, label %._crit_edge
+  br i1 %arg, label %.lr.ph, label %._crit_edge
 
 .lr.ph:                                           ; preds = %.lr.ph, %0
   br i1 false, label %.lr.ph, label %._crit_edge
diff --git a/llvm/test/Transforms/ObjCARC/path-overflow.ll b/llvm/test/Transforms/ObjCARC/path-overflow.ll
index eeef70554d114..431b377481262 100644
--- a/llvm/test/Transforms/ObjCARC/path-overflow.ll
+++ b/llvm/test/Transforms/ObjCARC/path-overflow.ll
@@ -29,9 +29,9 @@ declare i32 @__gxx_personality_sj0(...)
 declare i32 @__objc_personality_v0(...)
 
 
-define hidden void @test1() personality ptr @__gxx_personality_sj0 {
+define hidden void @test1(i1 %arg) personality ptr @__gxx_personality_sj0 {
 entry:
-  br i1 undef, label %msgSend.nullinit, label %msgSend.call
+  br i1 %arg, label %msgSend.nullinit, label %msgSend.call
 
 msgSend.call:                                     ; preds = %entry
   br label %msgSend.cont
@@ -41,7 +41,7 @@ msgSend.nullinit:                                 ; preds = %entry
 
 msgSend.cont:                                     ; preds = %msgSend.nullinit, %msgSend.call
   %0 = call ptr @llvm.objc.retain(ptr @_unnamed_cfstring) nounwind
-  br i1 undef, label %msgSend.nullinit33, label %msgSend.call32
+  br i1 %arg, label %msgSend.nullinit33, label %msgSend.call32
 
 msgSend.call32:                                   ; preds = %if.end10
   br label %msgSend.cont34
@@ -50,7 +50,7 @@ msgSend.nullinit33:                               ; preds = %if.end10
   br label %msgSend.cont34
 
 msgSend.cont34:                                   ; preds = %msgSend.nullinit33, %msgSend.call32
-  br i1 undef, label %msgSend.nullinit38, label %msgSend.call37
+  br i1 %arg, label %msgSend.nullinit38, label %msgSend.call37
 
 msgSend.call37:                                   ; preds = %msgSend.cont34
   br label %msgSend.cont39
@@ -59,7 +59,7 @@ msgSend.nullinit38:                               ; preds = %msgSend.cont34
   br label %msgSend.cont39
 
 msgSend.cont39:                                   ; preds = %msgSend.nullinit38, %msgSend.call37
-  br i1 undef, label %msgSend.nullinit49, label %msgSend.call48
+  br i1 %arg, label %msgSend.nullinit49, label %msgSend.call48
 
 msgSend.call48:                                   ; preds = %msgSend.cont39
   br label %msgSend.cont50
@@ -68,7 +68,7 @@ msgSend.nullinit49:                               ; preds = %msgSend.cont39
   br label %msgSend.cont50
 
 msgSend.cont50:                                   ; preds = %msgSend.nullinit49, %msgSend.call48
-  br i1 undef, label %msgSend.nullinit61, label %msgSend.call60
+  br i1 %arg, label %msgSend.nullinit61, label %msgSend.call60
 
 msgSend.call60:                                   ; preds = %msgSend.cont50
   br label %msgSend.cont62
@@ -77,7 +77,7 @@ msgSend.nullinit61:                               ; preds = %msgSend.cont50
   br label %msgSend.cont62
 
 msgSend.cont62:                                   ; preds = %msgSend.nullinit61, %msgSend.call60
-  br i1 undef, label %msgSend.nullinit67, label %msgSend.call66
+  br i1 %arg, label %msgSend.nullinit67, label %msgSend.call66
 
 msgSend.call66:                                   ; preds = %msgSend.cont62
   br label %msgSend.cont68
@@ -86,7 +86,7 @@ msgSend.nullinit67:                               ; preds = %msgSend.cont62
   br label %msgSend.cont68
 
 msgSend.cont68:                                   ; preds = %msgSend.nullinit67, %msgSend.call66
-  br i1 undef, label %msgSend.nullinit84, label %msgSend.call83
+  br i1 %arg, label %msgSend.nullinit84, label %msgSend.call83
 
 msgSend.call83:                                   ; preds = %msgSend.cont68
   br label %msgSend.cont85
@@ -95,7 +95,7 @@ msgSend.nullinit84:                               ; preds = %msgSend.cont68
   br label %msgSend.cont85
 
 msgSend.cont85:                                   ; preds = %msgSend.nullinit84, %msgSend.call83
-  br i1 undef, label %msgSend.nullinit90, label %msgSend.call89
+  br i1 %arg, label %msgSend.nullinit90, label %msgSend.call89
 
 msgSend.call89:                                   ; preds = %msgSend.cont85
   br label %msgSend.cont91
@@ -104,7 +104,7 @@ msgSend.nullinit90:                               ; preds = %msgSend.cont85
   br label %msgSend.cont91
 
 msgSend.cont91:                                   ; preds = %msgSend.nullinit90, %msgSend.call89
-  br i1 undef, label %msgSend.nullinit104, label %msgSend.call103
+  br i1 %arg, label %msgSend.nullinit104, label %msgSend.call103
 
 msgSend.call103:                                  ; preds = %msgSend.cont91
   br label %msgSend.cont105
@@ -113,16 +113,16 @@ msgSend.nullinit104:                              ; preds = %msgSend.cont91
   br label %msgSend.cont105
 
 msgSend.cont105:                                  ; preds = %msgSend.nullinit104, %msgSend.call103
-  br i1 undef, label %land.lhs.true, label %if.end123
+  br i1 %arg, label %land.lhs.true, label %if.end123
 
 land.lhs.true:                                    ; preds = %msgSend.cont105
-  br i1 undef, label %if.then117, label %if.end123
+  br i1 %arg, label %if.then117, label %if.end123
 
 if.then117:                                       ; preds = %land.lhs.true
   br label %if.end123
 
 if.end123:                                        ; preds = %if.then117, %land.lhs.true, %msgSend.cont105
-  br i1 undef, label %msgSend.nullinit132, label %msgSend.call131
+  br i1 %arg, label %msgSend.nullinit132, label %msgSend.call131
 
 msgSend.call131:                                  ; preds = %if.end123
   br label %msgSend.cont133
@@ -131,7 +131,7 @@ msgSend.nullinit132:                              ; preds = %if.end123
   br label %msgSend.cont133
 
 msgSend.cont133:                                  ; preds = %msgSend.nullinit132, %msgSend.call131
-  br i1 undef, label %msgSend.nullinit139, label %msgSend.call138
+  br i1 %arg, label %msgSend.nullinit139, label %msgSend.call138
 
 msgSend.call138:                                  ; preds = %msgSend.cont133
   br label %msgSend.cont140
@@ -140,13 +140,13 @@ msgSend.nullinit139:                              ; preds = %msgSend.cont133
   br label %msgSend.cont140
 
 msgSend.cont140:                                  ; preds = %msgSend.nullinit139, %msgSend.call138
-  br i1 undef, label %if.then151, label %if.end157
+  br i1 %arg, label %if.then151, label %if.end157
 
 if.then151:                                       ; preds = %msgSend.cont140
   br label %if.end157
 
 if.end157:                                        ; preds = %if.then151, %msgSend.cont140
-  br i1 undef, label %msgSend.nullinit164, label %msgSend.call163
+  br i1 %arg, label %msgSend.nullinit164, label %msgSend.call163
 
 msgSend.call163:                                  ; preds = %if.end157
   br label %msgSend.cont165
@@ -155,7 +155,7 @@ msgSend.nullinit164:                              ; preds = %if.end157
   br label %msgSend.cont165
 
 msgSend.cont165:                                  ; preds = %msgSend.nullinit164, %msgSend.call163
-  br i1 undef, label %msgSend.nullinit176, label %msgSend.call175
+  br i1 %arg, label %msgSend.nullinit176, label %msgSend.call175
 
 msgSend.call175:                                  ; preds = %msgSend.cont165
   br label %msgSend.cont177
@@ -164,13 +164,13 @@ msgSend.nullinit176:                              ; preds = %msgSend.cont165
   br label %msgSend.cont177
 
 msgSend.cont177:                                  ; preds = %msgSend.nullinit176, %msgSend.call175
-  br i1 undef, label %land.lhs.true181, label %if.end202
+  br i1 %arg, label %land.lhs.true181, label %if.end202
 
 land.lhs.true181:                                 ; preds = %msgSend.cont177
-  br i1 undef, label %if.then187, label %if.end202
+  br i1 %arg, label %if.then187, label %if.end202
 
 if.then187:                                       ; preds = %land.lhs.true181
-  br i1 undef, label %msgSend.nullinit199, label %msgSend.call198
+  br i1 %arg, label %msgSend.nullinit199, label %msgSend.call198
 
 msgSend.call198:                                  ; preds = %if.then187
   br label %msgSend.cont200
@@ -182,7 +182,7 @@ msgSend.cont200:                                  ; preds = %msgSend.nullinit199
   br label %if.end202
 
 if.end202:                                        ; preds = %msgSend.cont200, %land.lhs.true181, %msgSend.cont177
-  br i1 undef, label %msgSend.nullinit236, label %msgSend.call235
+  br i1 %arg, label %msgSend.nullinit236, label %msgSend.call235
 
 msgSend.call235:                                  ; preds = %if.end202
   br label %msgSend.cont237
@@ -191,7 +191,7 @@ msgSend.nullinit236:                              ; preds = %if.end202
   br label %msgSend.cont237
 
 msgSend.cont237:                                  ; preds = %msgSend.nullinit236, %msgSend.call235
-  br i1 undef, label %msgSend.nullinit254, label %msgSend.call253
+  br i1 %arg, label %msgSend.nullinit254, label %msgSend.call253
 
 msgSend.call253:                                  ; preds = %msgSend.cont237
   br label %msgSend.cont255
@@ -200,7 +200,7 @@ msgSend.nullinit254:                              ; preds = %msgSend.cont237
   br label %msgSend.cont255
 
 msgSend.cont255:                                  ; preds = %msgSend.nullinit254, %msgSend.call253
-  br i1 undef, label %msgSend.nullinit269, label %msgSend.call268
+  br i1 %arg, label %msgSend.nullinit269, label %msgSend.call268
 
 msgSend.call268:                                  ; preds = %msgSend.cont255
   br label %msgSend.cont270
@@ -209,7 +209,7 @@ msgSend.nullinit269:                              ; preds = %msgSend.cont255
   br label %msgSend.cont270
 
 msgSend.cont270:                                  ; preds = %msgSend.nullinit269, %msgSend.call268
-  br i1 undef, label %msgSend.nullinit281, label %msgSend.call280
+  br i1 %arg, label %msgSend.nullinit281, label %msgSend.call280
 
 msgSend.call280:                                  ; preds = %msgSend.cont270
   br label %msgSend.cont282
@@ -218,7 +218,7 @@ msgSend.nullinit281:                              ; preds = %msgSend.cont270
   br label %msgSend.cont282
 
 msgSend.cont282:                                  ; preds = %msgSend.nullinit281, %msgSend.call280
-  br i1 undef, label %msgSend.nullinit287, label %msgSend.call286
+  br i1 %arg, label %msgSend.nullinit287, label %msgSend.call286
 
 msgSend.call286:                                  ; preds = %msgSend.cont282
   br label %msgSend.cont288
@@ -227,7 +227,7 @@ msgSend.nullinit287:                              ; preds = %msgSend.cont282
   br label %msgSend.cont288
 
 msgSend.cont288:                                  ; preds = %msgSend.nullinit287, %msgSend.call286
-  br i1 undef, label %msgSend.nullinit303, label %msgSend.call302
+  br i1 %arg, label %msgSend.nullinit303, label %msgSend.call302
 
 msgSend.call302:                                  ; preds = %msgSend.cont288
   br label %msgSend.cont304
@@ -236,7 +236,7 @@ msgSend.nullinit303:                              ; preds = %msgSend.cont288
   br label %msgSend.cont304
 
 msgSend.cont304:                                  ; preds = %msgSend.nullinit303, %msgSend.call302
-  br i1 undef, label %msgSend.nullinit344, label %msgSend.call343
+  br i1 %arg, label %msgSend.nullinit344, label %msgSend.call343
 
 msgSend.call343:                                  ; preds = %msgSend.cont304
   br label %msgSend.cont345
@@ -245,7 +245,7 @@ msgSend.nullinit344:                              ; preds = %msgSend.cont304
   br label %msgSend.cont345
 
 msgSend.cont345:                                  ; preds = %msgSend.nullinit344, %msgSend.call343
-  br i1 undef, label %msgSend.nullinit350, label %msgSend.call349
+  br i1 %arg, label %msgSend.nullinit350, label %msgSend.call349
 
 msgSend.call349:                                  ; preds = %msgSend.cont345
   br label %msgSend.cont351
@@ -254,7 +254,7 @@ msgSend.nullinit350:                              ; preds = %msgSend.cont345
   br label %msgSend.cont351
 
 msgSend.cont351:                                  ; preds = %msgSend.nullinit350, %msgSend.call349
-  br i1 undef, label %msgSend.nullinit366, label %msgSend.call365
+  br i1 %arg, label %msgSend.nullinit366, label %msgSend.call365
 
 msgSend.call365:                                  ; preds = %msgSend.cont351
   br label %msgSend.cont367
@@ -263,7 +263,7 @@ msgSend.nullinit366:                              ; preds = %msgSend.cont351
   br label %msgSend.cont367
 
 msgSend.cont367:                                  ; preds = %msgSend.nullinit366, %msgSend.call365
-  br i1 undef, label %msgSend.nullinit376, label %msgSend.call375
+  br i1 %arg, label %msgSend.nullinit376, label %msgSend.call375
 
 msgSend.call375:                                  ; preds = %msgSend.cont367
   br label %msgSend.cont377
@@ -272,10 +272,10 @@ msgSend.nullinit376:                              ; preds = %msgSend.cont367
   br label %msgSend.cont377
 
 msgSend.cont377:                                  ; preds = %msgSend.nullinit376, %msgSend.call375
-  br i1 undef, label %if.then384, label %if.else401
+  br i1 %arg, label %if.then384, label %if.else401
 
 if.then384:                                       ; preds = %msgSend.cont377
-  br i1 undef, label %msgSend.nullinit392, label %msgSend.call391
+  br i1 %arg, label %msgSend.nullinit392, label %msgSend.call391
 
 msgSend.call391:                                  ; preds = %if.then384
   br label %msgSend.cont393
@@ -287,7 +287,7 @@ msgSend.cont393:                                  ; preds = %msgSend.nullinit392
   br label %if.end418
 
 if.else401:                                       ; preds = %msgSend.cont377
-  br i1 undef, label %msgSend.nullinit409, label %msgSend.call408
+  br i1 %arg, label %msgSend.nullinit409, label %msgSend.call408
 
 msgSend.call408:                                  ; preds = %if.else401
   br label %msgSend.cont410
@@ -299,7 +299,7 @@ msgSend.cont410:                                  ; preds = %msgSend.nullinit409
   br label %if.end418
 
 if.end418:                                        ; preds = %msgSend.cont410, %msgSend.cont393
-  br i1 undef, label %msgSend.nullinit470, label %msgSend.call469
+  br i1 %arg, label %msgSend.nullinit470, label %msgSend.call469
 
 msgSend.call469:                                  ; preds = %if.end418
   br label %msgSend.cont471
@@ -308,7 +308,7 @@ msgSend.nullinit470:                              ; preds = %if.end418
   br label %msgSend.cont471
 
 msgSend.cont471:                                  ; preds = %msgSend.nullinit470, %msgSend.call469
-  br i1 undef, label %msgSend.nullinit484, label %msgSend.call483
+  br i1 %arg, label %msgSend.nullinit484, label %msgSend.call483
 
 msgSend.call483:                                  ; preds = %msgSend.cont471
   br label %msgSend.cont485
@@ -317,7 +317,7 @@ msgSend.nullinit484:                              ; preds = %msgSend.cont471
   br label %msgSend.cont485
 
 msgSend.cont485:                                  ; preds = %msgSend.nullinit484, %msgSend.call483
-  br i1 undef, label %msgSend.nullinit500, label %msgSend.call499
+  br i1 %arg, label %msgSend.nullinit500, label %msgSend.call499
 
 msgSend.call499:                                  ; preds = %msgSend.cont485
   br label %msgSend.cont501
@@ -326,7 +326,7 @@ msgSend.nullinit500:                              ; preds = %msgSend.cont485
   br label %msgSend.cont501
 
 msgSend.cont501:                                  ; preds = %msgSend.nullinit500, %msgSend.call499
-  br i1 undef, label %msgSend.nullinit506, label %msgSend.call505
+  br i1 %arg, label %msgSend.nullinit506, label %msgSend.call505
 
 msgSend.call505:                                  ; preds = %msgSend.cont501
   br label %msgSend.cont507
@@ -340,78 +340,78 @@ msgSend.cont507:                                  ; preds = %msgSend.nullinit506
 }
 
 ; Function Attrs: optsize ssp uwtable
-define void @test2() unnamed_addr align 2 personality ptr @__gxx_personality_sj0 {
+define void @test2(i1 %arg) unnamed_addr align 2 personality ptr @__gxx_personality_sj0 {
 bb:
-  br i1 undef, label %bb3, label %bb2
+  br i1 %arg, label %bb3, label %bb2
 
 bb2:                                              ; preds = %bb
   br label %bb3
 
 bb3:                                              ; preds = %bb2, %bb
-  br i1 undef, label %bb5, label %bb4
+  br i1 %arg, label %bb5, label %bb4
 
 bb4:                                              ; preds = %bb3
   br label %bb5
 
 bb5:                                              ; preds = %bb4, %bb3
-  br i1 undef, label %bb7, label %bb6
+  br i1 %arg, label %bb7, label %bb6
 
 bb6:                                              ; preds = %bb5
   br label %bb7
 
 bb7:                                              ; preds = %bb6, %bb5
-  br i1 undef, label %bb9, label %bb8
+  br i1 %arg, label %bb9, label %bb8
 
 bb8:                                              ; preds = %bb7
   unreachable
 
 bb9:                                              ; preds = %bb7
-  br i1 undef, label %bb11, label %bb10
+  br i1 %arg, label %bb11, label %bb10
 
 bb10:                                             ; preds = %bb9
   br label %bb11
 
 bb11:                                             ; preds = %bb10, %bb9
-  br i1 undef, label %bb13, label %bb12
+  br i1 %arg, label %bb13, label %bb12
 
 bb12:                                             ; preds = %bb11
   br label %bb13
 
 bb13:                                             ; preds = %bb12, %bb11
-  br i1 undef, label %bb15, label %bb14
+  br i1 %arg, label %bb15, label %bb14
 
 bb14:                                             ; preds = %bb13
   br label %bb15
 
 bb15:                                             ; preds = %bb14, %bb13
-  br i1 undef, label %bb17, label %bb16
+  br i1 %arg, label %bb17, label %bb16
 
 bb16:                                             ; preds = %bb15
   br label %bb17
 
 bb17:                                             ; preds = %bb16, %bb15
-  br i1 undef, label %bb19, label %bb18
+  br i1 %arg, label %bb19, label %bb18
 
 bb18:                                             ; preds = %bb17
   br label %bb19
 
 bb19:                                             ; preds = %bb18, %bb17
-  br i1 undef, label %bb222, label %bb20
+  br i1 %arg, label %bb222, label %bb20
 
 bb20:                                             ; preds = %bb19
-  br i1 undef, label %bb222, label %bb21
+  br i1 %arg, label %bb222, label %bb21
 
 bb21:                                             ; preds = %bb20
-  br i1 undef, label %bb22, label %bb30
+  br i1 %arg, label %bb22, label %bb30
 
 bb22:                                             ; preds = %bb21
-  br i1 undef, label %bb23, label %bb32
+  br i1 %arg, label %bb23, label %bb32
 
 bb23:                                             ; preds = %bb22
-  br i1 undef, label %bb24, label %bb34
+  br i1 %arg, label %bb24, label %bb34
 
 bb24:                                             ; preds = %bb23
-  br i1 undef, label %bb26, label %bb25
+  br i1 %arg, label %bb26, label %bb25
 
 bb25:                                             ; preds = %bb24
   br label %bb27
@@ -420,10 +420,10 @@ bb26:                                             ; preds = %bb24
   br label %bb27
 
 bb27:                                             ; preds = %bb26, %bb25
-  br i1 undef, label %bb28, label %bb42
+  br i1 %arg, label %bb28, label %bb42
 
 bb28:                                             ; preds = %bb27
-  br i1 undef, label %bb36, label %bb29
+  br i1 %arg, label %bb36, label %bb29
 
 bb29:                                             ; preds = %bb28
   br label %bb36
@@ -438,7 +438,7 @@ bb34:                                             ; preds = %bb23
   unreachable
 
 bb36:                                             ; preds = %bb29, %bb28
-  br i1 undef, label %bb38, label %bb37
+  br i1 %arg, label %bb38, label %bb37
 
 bb37:                                             ; preds = %bb36
   br label %bb39
@@ -447,7 +447,7 @@ bb38:                                             ; preds = %bb36
   br label %bb39
 
 bb39:                                             ; preds = %bb38, %bb37
-  br i1 undef, label %bb41, label %bb40
+  br i1 %arg, label %bb41, label %bb40
 
 bb40:                                             ; preds = %bb39
   unreachable
@@ -456,19 +456,19 @@ bb41:                                             ; preds = %bb39
   br label %bb42
 
 bb42:                                             ; preds = %bb41, %bb27
-  br i1 undef, label %bb43, label %bb214
+  br i1 %arg, label %bb43, label %bb214
 
 bb43:                                             ; preds = %bb42
-  br i1 undef, label %bb47, label %bb45
+  br i1 %arg, label %bb47, label %bb45
 
 bb45:                                             ; preds = %bb130, %bb128, %bb126, %bb124, %bb122, %bb120, %bb118, %bb116, %bb114, %bb112, %bb110, %bb108, %bb105, %bb102, %bb100, %bb96, %bb94, %bb90, %bb88, %bb84, %bb82, %bb78, %bb76, %bb72, %bb70, %bb66, %bb64, %bb60, %bb58, %bb54, %bb51, %bb43
   unreachable
 
 bb47:                                             ; preds = %bb43
-  br i1 undef, label %bb48, label %bb106
+  br i1 %arg, label %bb48, label %bb106
 
 bb48:                                             ; preds = %bb47
-  br i1 undef, label %bb50, label %bb49
+  br i1 %arg, label %bb50, label %bb49
 
 bb49:                                             ; preds = %bb48
   br label %bb51
@@ -477,16 +477,16 @@ bb50:                                             ; preds = %bb48
   br label %bb51
 
 bb51:                                             ; preds = %bb50, %bb49
-  br i1 undef, label %bb53, label %bb45
+  br i1 %arg, label %bb53, label %bb45
 
 bb53:                                             ; preds = %bb51
-  br i1 undef, label %bb54, label %bb134
+  br i1 %arg, label %bb54, label %bb134
 
 bb54:                                             ; preds = %bb53
-  br i1 undef, label %bb55, label %bb45
+  br i1 %arg, label %bb55, label %bb45
 
 bb55:                                             ; preds = %bb54
-  br i1 undef, label %bb57, label %bb56
+  br i1 %arg, label %bb57, label %bb56
 
 bb56:                                             ; preds = %bb55
   br label %bb58
@@ -495,13 +495,13 @@ bb57:                                             ; preds = %bb55
   br label %bb58
 
 bb58:                                             ; preds = %bb57, %bb56
-  br i1 undef, label %bb60, label %bb45
+  br i1 %arg, label %bb60, label %bb45
 
 bb60:                                             ; preds = %bb58
-  br i1 undef, label %bb61, label %bb45
+  br i1 %arg, label %bb61, label %bb45
 
 bb61:                                             ; preds = %bb60
-  br i1 undef, label %bb63, label %bb62
+  br i1 %arg, label %bb63, label %bb62
 
 bb62:                                             ; preds = %bb61
   br label %bb64
@@ -510,13 +510,13 @@ bb63:                                             ; preds = %bb61
   br label %bb64
 
 bb64:                                             ; preds = %bb63, %bb62
-  br i1 undef, label %bb66, label %bb45
+  br i1 %arg, label %bb66, label %bb45
 
 bb66:                                             ; preds = %bb64
-  br i1 undef, label %bb67, label %bb45
+  br i1 %arg, label %bb67, label %bb45
 
 bb67:                                             ; preds = %bb66
-  br i1 undef, label %bb69, label %bb68
+  br i1 %arg, label %bb69, label %bb68
 
 bb68:                                             ; preds = %bb67
   br label %bb70
@@ -525,13 +525,13 @@ bb69:                                             ; preds = %bb67
   br label %bb70
 
 bb70:                                             ; preds = %bb69, %bb68
-  br i1 undef, label %bb72, label %bb45
+  br i1 %arg, label %bb72, label %bb45
 
 bb72:                                             ; preds = %bb70
-  br i1 undef, label %bb73, label %bb45
+  br i1 %arg, label %bb73, label %bb45
 
 bb73:                                             ; preds = %bb72
-  br i1 undef, label %bb75, label %bb74
+  br i1 %arg, label %bb75, label %bb74
 
 bb74:                                             ; preds = %bb73
   br label %bb76
@@ -540,13 +540,13 @@ bb75:                                             ; preds = %bb73
   br label %bb76
 
 bb76:                                             ; preds = %bb75, %bb74
-  br i1 undef, label %bb78, label %bb45
+  br i1 %arg, label %bb78, label %bb45
 
 bb78:                                             ; preds = %bb76
-  br i1 undef, label %bb79, label %bb45
+  br i1 %arg, label %bb79, label %bb45
 
 bb79:                                             ; preds = %bb78
-  br i1 undef, label %bb81, label %bb80
+  br i1 %arg, label %bb81, label %bb80
 
 bb80:                                             ; preds = %bb79
   br label %bb82
@@ -555,13 +555,13 @@ bb81:                                             ; preds = %bb79
   br label %bb82
 
 bb82:                                             ; preds = %bb81, %bb80
-  br i1 undef, label %bb84, label %bb45
+  br i1 %arg, label %bb84, label %bb45
 
 bb84:                                             ; preds = %bb82
-  br i1 undef, label %bb85, label %bb45
+  br i1 %arg, label %bb85, label %bb45
 
 bb85:                                             ; preds = %bb84
-  br i1 undef, label %bb87, label %bb86
+  br i1 %arg, label %bb87, label %bb86
 
 bb86:                                             ; preds = %bb85
   br label %bb88
@@ -570,13 +570,13 @@ bb87:                                             ; preds = %bb85
   br label %bb88
 
 bb88:                                             ; preds = %bb87, %bb86
-  br i1 undef, label %bb90, label %bb45
+  br i1 %arg, label %bb90, label %bb45
 
 bb90:                                             ; preds = %bb88
-  br i1 undef, label %bb91, label %bb45
+  br i1 %arg, label %bb91, label %bb45
 
 bb91:                                             ; preds = %bb90
-  br i1 undef, label %bb93, label %bb92
+  br i1 %arg, label %bb93, label %bb92
 
 bb92:                                             ; preds = %bb91
   br label %bb94
@@ -585,13 +585,13 @@ bb93:                                             ; preds = %bb91
   br label %bb94
 
 bb94:                                             ; preds = %bb93, %bb92
-  br i1 undef, label %bb96, label %bb45
+  br i1 %arg, label %bb96, label %bb45
 
 bb96:                                             ; preds = %bb94
-  br i1 undef, label %bb97, label %bb45
+  br i1 %arg, label %bb97, label %bb45
 
 bb97:                                             ; preds = %bb96
-  br i1 undef, label %bb99, label %bb98
+  br i1 %arg, label %bb99, label %bb98
 
 bb98:                                             ; preds = %bb97
   br label %bb100
@@ -600,82 +600,82 @@ bb99:                                             ; preds = %bb97
   br label %bb100
 
 bb100:                                            ; preds = %bb99, %bb98
-  br i1 undef, label %bb102, label %bb45
+  br i1 %arg, label %bb102, label %bb45
 
 bb102:                                            ; preds = %bb100
-  br i1 undef, label %bb104, label %bb45
+  br i1 %arg, label %bb104, label %bb45
 
 bb104:                                            ; preds = %bb102
-  br i1 undef, label %bb108, label %bb105
+  br i1 %arg, label %bb108, label %bb105
 
 bb105:                                            ; preds = %bb104
-  br i1 undef, label %bb108, label %bb45
+  br i1 %arg, label %bb108, label %bb45
 
 bb106:                                            ; preds = %bb47
   unreachable
 
 bb108:                                            ; preds = %bb105, %bb104
-  br i1 undef, label %bb110, label %bb45
+  br i1 %arg, label %bb110, label %bb45
 
 bb110:                                            ; preds = %bb108
-  br i1 undef, label %bb112, label %bb45
+  br i1 %arg, label %bb112, label %bb45
 
 bb112:                                            ; preds = %bb110
-  br i1 undef, label %bb114, label %bb45
+  br i1 %arg, label %bb114, label %bb45
 
 bb114:                                            ; preds = %bb112
-  br i1 undef, label %bb116, label %bb45
+  br i1 %arg, label %bb116, label %bb45
 
 bb116:                                            ; preds = %bb114
-  br i1 undef, label %bb118, label %bb45
+  br i1 %arg, label %bb118, label %bb45
 
 bb118:                                            ; preds = %bb116
-  br i1 undef, label %bb120, label %bb45
+  br i1 %arg, label %bb120, label %bb45
 
 bb120:                                            ; preds = %bb118
-  br i1 undef, label %bb122, label %bb45
+  br i1 %arg, label %bb122, label %bb45
 
 bb122:                                            ; preds = %bb120
-  br i1 undef, label %bb124, label %bb45
+  br i1 %arg, label %bb124, label %bb45
 
 bb124:                                            ; preds = %bb122
-  br i1 undef, label %bb126, label %bb45
+  br i1 %arg, label %bb126, label %bb45
 
 bb126:                                            ; preds = %bb124
-  br i1 undef, label %bb128, label %bb45
+  br i1 %arg, label %bb128, label %bb45
 
 bb128:                                            ; preds = %bb126
-  br i1 undef, label %bb130, label %bb45
+  br i1 %arg, label %bb130, label %bb45
 
 bb130:                                            ; preds = %bb128
-  br i1 undef, label %bb132, label %bb45
+  br i1 %arg, label %bb132, label %bb45
 
 bb132:                                            ; preds = %bb130
-  br i1 undef, label %bb135, label %bb30
+  br i1 %arg, label %bb135, label %bb30
 
 bb134:                                            ; preds = %bb53
   unreachable
 
 bb135:                                            ; preds = %bb132
-  br i1 undef, label %bb139, label %bb136
+  br i1 %arg, label %bb139, label %bb136
 
 bb136:                                            ; preds = %bb135
-  br i1 undef, label %bb138, label %bb30
+  br i1 %arg, label %bb138, label %bb30
 
 bb138:                                            ; preds = %bb136
   br label %bb139
 
 bb139:                                            ; preds = %bb138, %bb135
-  br i1 undef, label %bb140, label %bb141
+  br i1 %arg, label %bb140, label %bb141
 
 bb140:                                            ; preds = %bb139
   unreachable
 
 bb141:                                            ; preds = %bb139
-  br i1 undef, label %bb142, label %bb215
+  br i1 %arg, label %bb142, label %bb215
 
 bb142:                                            ; preds = %bb141
-  br i1 undef, label %bb144, label %bb143
+  br i1 %arg, label %bb144, label %bb143
 
 bb143:                                            ; preds = %bb142
   br label %bb145
@@ -684,16 +684,16 @@ bb144:                                            ; preds = %bb142
   br label %bb145
 
 bb145:                                            ; preds = %bb144, %bb143
-  br i1 undef, label %bb146, label %bb151
+  br i1 %arg, label %bb146, label %bb151
 
 bb146:                                            ; preds = %bb145
-  br i1 undef, label %bb148, label %bb153
+  br i1 %arg, label %bb148, label %bb153
 
 bb148:                                            ; preds = %bb146
-  br i1 undef, label %bb155, label %bb149
+  br i1 %arg, label %bb155, label %bb149
 
 bb149:                                            ; preds = %bb148
-  br i1 undef, label %bb150, label %bb153
+  br i1 %arg, label %bb150, label %bb153
 
 bb150:                                            ; preds = %bb149
   br label %bb155
@@ -705,7 +705,7 @@ bb153:                                            ; preds = %bb158, %bb149, %bb1
   unreachable
 
 bb155:                                            ; preds = %bb150, %bb148
-  br i1 undef, label %bb157, label %bb156
+  br i1 %arg, label %bb157, label %bb156
 
 bb156:                                            ; preds = %bb155
   br label %bb158
@@ -714,10 +714,10 @@ bb157:                                            ; preds = %bb155
   br label %bb158
 
 bb158:                                            ; preds = %bb157, %bb156
-  br i1 undef, label %bb160, label %bb153
+  br i1 %arg, label %bb160, label %bb153
 
 bb160:                                            ; preds = %bb158
-  br i1 undef, label %bb162, label %bb161
+  br i1 %arg, label %bb162, label %bb161
 
 bb161:                                            ; preds = %bb160
   br label %bb163
@@ -726,16 +726,16 @@ bb162:                                            ; preds = %bb160
   br label %bb163
 
 bb163:                                            ; preds = %bb162, %bb161
-  br i1 undef, label %bb165, label %bb164
+  br i1 %arg, label %bb165, label %bb164
 
 bb164:                                            ; preds = %bb163
   br label %bb165
 
 bb165:                                            ; preds = %bb164, %bb163
-  br i1 undef, label %bb170, label %bb166
+  br i1 %arg, label %bb170, label %bb166
 
 bb166:                                            ; preds = %bb165
-  br i1 undef, label %bb167, label %bb168
+  br i1 %arg, label %bb167, label %bb168
 
 bb167:                                            ; preds = %bb166
   unreachable
@@ -744,25 +744,25 @@ bb168:                                            ; preds = %bb166
   unreachable
 
 bb170:                                            ; preds = %bb165
-  br i1 undef, label %bb215, label %bb171
+  br i1 %arg, label %bb215, label %bb171
 
 bb171:                                            ; preds = %bb170
-  br i1 undef, label %bb173, label %bb30
+  br i1 %arg, label %bb173, label %bb30
 
 bb173:                                            ; preds = %bb171
-  br i1 undef, label %bb174, label %bb215
+  br i1 %arg, label %bb174, label %bb215
 
 bb174:                                            ; preds = %bb173
-  br i1 undef, label %bb176, label %bb30
+  br i1 %arg, label %bb176, label %bb30
 
 bb176:                                            ; preds = %bb174
-  br i1 undef, label %bb178, label %bb30
+  br i1 %arg, label %bb178, label %bb30
 
 bb178:                                            ; preds = %bb176
-  br i1 undef, label %bb179, label %bb193
+  br i1 %arg, label %bb179, label %bb193
 
 bb179:                                            ; preds = %bb178
-  br i1 undef, label %bb181, label %bb180
+  br i1 %arg, label %bb181, label %bb180
 
 bb180:                                            ; preds = %bb179
   br label %bb182
@@ -771,23 +771,23 @@ bb181:                                            ; preds = %bb179
   br label %bb182
 
 bb182:                                            ; preds = %bb181, %bb180
-  br i1 undef, label %bb184, label %bb30
+  br i1 %arg, label %bb184, label %bb30
 
 bb184:                                            ; preds = %bb182
   %tmp185 = call ptr @returner()
-  br i1 undef, label %bb186, label %bb195
+  br i1 %arg, label %bb186, label %bb195
 
 bb186:                                            ; preds = %bb184
   %tmp188 = call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr %tmp185)
   %tmp189 = call ptr @llvm.objc.retain(ptr %tmp188)
   call void @llvm.objc.release(ptr %tmp189), !clang.imprecise_release !0
-  br i1 undef, label %bb197, label %bb190
+  br i1 %arg, label %bb197, label %bb190
 
 bb190:                                            ; preds = %bb186
-  br i1 undef, label %bb192, label %bb195
+  br i1 %arg, label %bb192, label %bb195
 
 bb192:                                            ; preds = %bb190
-  br i1 undef, label %bb197, label %bb195
+  br i1 %arg, label %bb197, label %bb195
 
 bb193:                                            ; preds = %bb178
   br label %bb213
@@ -796,37 +796,37 @@ bb195:                                            ; preds = %bb192, %bb190, %bb1
   unreachable
 
 bb197:                                            ; preds = %bb192, %bb186
-  br i1 undef, label %bb198, label %bb215
+  br i1 %arg, label %bb198, label %bb215
 
 bb198:                                            ; preds = %bb197
-  br i1 undef, label %bb202, label %bb199
+  br i1 %arg, label %bb202, label %bb199
 
 bb199:                                            ; preds = %bb198
-  br i1 undef, label %bb201, label %bb30
+  br i1 %arg, label %bb201, label %bb30
 
 bb201:                                            ; preds = %bb199
   br label %bb202
 
 bb202:                                            ; preds = %bb201, %bb198
-  br i1 undef, label %bb206, label %bb203
+  br i1 %arg, label %bb206, label %bb203
 
 bb203:                                            ; preds = %bb202
-  br i1 undef, label %bb205, label %bb30
+  br i1 %arg, label %bb205, label %bb30
 
 bb205:                                            ; preds = %bb203
   br label %bb206
 
 bb206:                                            ; preds = %bb205, %bb202
-  br i1 undef, label %bb210, label %bb207
+  br i1 %arg, label %bb210, label %bb207
 
 bb207:                                            ; preds = %bb206
-  br i1 undef, label %bb209, label %bb30
+  br i1 %arg, label %bb209, label %bb30
 
 bb209:                                            ; preds = %bb207
   br label %bb210
 
 bb210:                                            ; preds = %bb209, %bb206
-  br i1 undef, label %bb212, label %bb30
+  br i1 %arg, label %bb212, label %bb30
 
 bb212:                                            ; preds = %bb210
   unreachable
@@ -838,19 +838,19 @@ bb214:                                            ; preds = %bb42
   br label %bb219
 
 bb215:                                            ; preds = %bb197, %bb173, %bb170, %bb141
-  br i1 undef, label %bb217, label %bb216
+  br i1 %arg, label %bb217, label %bb216
 
 bb216:                                            ; preds = %bb215
   br label %bb217
 
 bb217:                                            ; preds = %bb216, %bb215
-  br i1 undef, label %bb219, label %bb218
+  br i1 %arg, label %bb219, label %bb218
 
 bb218:                                            ; preds = %bb217
   br label %bb219
 
 bb219:                                            ; preds = %bb218, %bb217, %bb214
-  br i1 undef, label %bb221, label %bb220
+  br i1 %arg, label %bb221, label %bb220
 
 bb220:                                            ; preds = %bb219
   unreachable
@@ -863,7 +863,7 @@ bb222:                                            ; preds = %bb20, %bb19
 }
 
 ; Function Attrs: ssp
-define void @test3() #1 personality ptr @__gxx_personality_sj0 {
+define void @test3(i1 %arg) #1 personality ptr @__gxx_personality_sj0 {
 entry:
   %call2 = invoke ptr @objc_msgSend(ptr undef, ptr undef, ptr @_unnamed_cfstring)
           to label %invoke.cont unwind label %lpad
@@ -873,7 +873,7 @@ invoke.cont:                                      ; preds = %entry
           to label %invoke.cont4 unwind label %lpad3
 
 invoke.cont4:                                     ; preds = %invoke.cont
-  br i1 undef, label %land.end, label %land.rhs
+  br i1 %arg, label %land.end, label %land.rhs
 
 land.rhs:                                         ; preds = %invoke.cont4
   %call7 = invoke i32 @objc_msgSend(ptr undef, ptr undef)
@@ -884,7 +884,7 @@ land.end:                                         ; preds = %land.rhs, %invoke.c
           to label %invoke.cont.i unwind label %lpad.i
 
 invoke.cont.i:                                    ; preds = %land.end
-  br i1 undef, label %invoke.cont8, label %if.then.i
+  br i1 %arg, label %invoke.cont8, label %if.then.i
 
 if.then.i:                                        ; preds = %invoke.cont.i
   br label %invoke.cont8
@@ -907,7 +907,7 @@ invoke.cont21:                                    ; preds = %invoke.cont17
           to label %invoke.cont.i1980 unwind label %lpad.i1982
 
 invoke.cont.i1980:                                ; preds = %invoke.cont21
-  br i1 undef, label %invoke.cont24, label %if.then.i1981
+  br i1 %arg, label %invoke.cont24, label %if.then.i1981
 
 if.then.i1981:                                    ; preds = %invoke.cont.i1980
   br label %invoke.cont24
@@ -922,7 +922,7 @@ invoke.cont24:                                    ; preds = %if.then.i1981, %inv
           to label %invoke.cont36 unwind label %lpad35
 
 invoke.cont36:                                    ; preds = %invoke.cont24
-  br i1 undef, label %land.end43, label %land.rhs39
+  br i1 %arg, label %land.end43, label %land.rhs39
 
 land.rhs39:                                       ; preds = %invoke.cont36
   %call41 = invoke signext i8 @objc_msgSend(ptr undef, ptr undef, ptr @_unnamed_cfstring)
@@ -933,7 +933,7 @@ land.end43:                                       ; preds = %land.rhs39, %invoke
           to label %invoke.cont.i1986 unwind label %lpad.i1988
 
 invoke.cont.i1986:                                ; preds = %land.end43
-  br i1 undef, label %invoke.cont44, label %if.then.i1987
+  br i1 %arg, label %invoke.cont44, label %if.then.i1987
 
 if.then.i1987:                                    ; preds = %invoke.cont.i1986
   br label %invoke.cont44
@@ -948,7 +948,7 @@ invoke.cont44:                                    ; preds = %if.then.i1987, %inv
           to label %invoke.cont52 unwind label %lpad51
 
 invoke.cont52:                                    ; preds = %invoke.cont44
-  br i1 undef, label %land.end70, label %land.rhs58
+  br i1 %arg, label %land.end70, label %land.rhs58
 
 land.rhs58:                                       ; preds = %invoke.cont52
   %call63 = invoke ptr @objc_msgSend(ptr undef, ptr undef, i32 42)
@@ -963,7 +963,7 @@ land.end70:                                       ; preds = %invoke.cont62, %inv
           to label %invoke.cont.i1992 unwind label %lpad66.body
 
 invoke.cont.i1992:                                ; preds = %land.end70
-  br i1 undef, label %invoke.cont71, label %if.then.i1993
+  br i1 %arg, label %invoke.cont71, label %if.then.i1993
 
 if.then.i1993:                                    ; preds = %invoke.cont.i1992
   br label %invoke.cont71
@@ -973,7 +973,7 @@ invoke.cont71:                                    ; preds = %if.then.i1993, %inv
           to label %invoke.cont.i1998 unwind label %lpad.i2000
 
 invoke.cont.i1998:                                ; preds = %invoke.cont71
-  br i1 undef, label %invoke.cont91, label %if.then.i1999
+  br i1 %arg, label %invoke.cont91, label %if.then.i1999
 
 if.then.i1999:                                    ; preds = %invoke.cont.i1998
   br label %invoke.cont91
@@ -996,7 +996,7 @@ invoke.cont97:                                    ; preds = %invoke.cont95
           to label %invoke.cont.i2004 unwind label %lpad.i2006
 
 invoke.cont.i2004:                                ; preds = %invoke.cont97
-  br i1 undef, label %invoke.cont100, label %if.then.i2005
+  br i1 %arg, label %invoke.cont100, label %if.then.i2005
 
 if.then.i2005:                                    ; preds = %invoke.cont.i2004
   br label %invoke.cont100
@@ -1015,7 +1015,7 @@ invoke.cont110:                                   ; preds = %invoke.cont100
           to label %invoke.cont.i2010 unwind label %lpad.i2012
 
 invoke.cont.i2010:                                ; preds = %invoke.cont110
-  br i1 undef, label %invoke.cont117, label %if.then.i2011
+  br i1 %arg, label %invoke.cont117, label %if.then.i2011
 
 if.then.i2011:                                    ; preds = %invoke.cont.i2010
   br label %invoke.cont117
@@ -1094,7 +1094,7 @@ lpad109:                                          ; preds = %invoke.cont100
   unreachable
 
 invoke.cont.i2022:                                ; preds = %invoke.cont117
-  br i1 undef, label %invoke.cont157, label %if.then.i2023
+  br i1 %arg, label %invoke.cont157, label %if.then.i2023
 
 if.then.i2023:                                    ; preds = %invoke.cont.i2022
   br label %invoke.cont157
@@ -1104,7 +1104,7 @@ invoke.cont157:                                   ; preds = %if.then.i2023, %inv
           to label %invoke.cont.i2028 unwind label %lpad164.body
 
 invoke.cont.i2028:                                ; preds = %invoke.cont157
-  br i1 undef, label %invoke.cont165, label %if.then.i2029
+  br i1 %arg, label %invoke.cont165, label %if.then.i2029
 
 if.then.i2029:                                    ; preds = %invoke.cont.i2028
   br label %invoke.cont165
@@ -1122,7 +1122,7 @@ invoke.cont185:                                   ; preds = %invoke.cont184
           to label %invoke.cont.i2034 unwind label %lpad.i2036
 
 invoke.cont.i2034:                                ; preds = %invoke.cont185
-  br i1 undef, label %invoke.cont190, label %if.then.i2035
+  br i1 %arg, label %invoke.cont190, label %if.then.i2035
 
 if.then.i2035:                                    ; preds = %invoke.cont.i2034
   br label %invoke.cont190
@@ -1149,7 +1149,7 @@ invoke.cont204:                                   ; preds = %invoke.cont201
           to label %invoke.cont.i2040 unwind label %lpad.i2042
 
 invoke.cont.i2040:                                ; preds = %invoke.cont204
-  br i1 undef, label %invoke.cont207, label %if.then.i2041
+  br i1 %arg, label %invoke.cont207, label %if.then.i2041
 
 if.then.i2041:                                    ; preds = %invoke.cont.i2040
   br label %invoke.cont207
@@ -1168,7 +1168,7 @@ invoke.cont208:                                   ; preds = %invoke.cont207
           to label %invoke.cont.i2046 unwind label %lpad212.body
 
 invoke.cont.i2046:                                ; preds = %invoke.cont208
-  br i1 undef, label %invoke.cont213, label %if.then.i2047
+  br i1 %arg, label %invoke.cont213, label %if.then.i2047
 
 if.then.i2047:                                    ; preds = %invoke.cont.i2046
   br label %invoke.cont213
@@ -1186,7 +1186,7 @@ invoke.cont228:                                   ; preds = %invoke.cont221
           to label %invoke.cont.i2052 unwind label %lpad.i2054
 
 invoke.cont.i2052:                                ; preds = %invoke.cont228
-  br i1 undef, label %invoke.cont231, label %if.then.i2053
+  br i1 %arg, label %invoke.cont231, label %if.then.i2053
 
 if.then.i2053:                                    ; preds = %invoke.cont.i2052
   br label %invoke.cont231
@@ -1205,7 +1205,7 @@ invoke.cont232:                                   ; preds = %invoke.cont231
           to label %invoke.cont.i2058 unwind label %lpad236.body
 
 invoke.cont.i2058:                                ; preds = %invoke.cont232
-  br i1 undef, label %invoke.cont237, label %if.then.i2059
+  br i1 %arg, label %invoke.cont237, label %if.then.i2059
 
 if.then.i2059:                                    ; preds = %invoke.cont.i2058
   br label %invoke.cont237
@@ -1251,7 +1251,7 @@ invoke.cont278:                                   ; preds = %invoke.cont274
           to label %invoke.cont.i2064 unwind label %lpad.i2066
 
 invoke.cont.i2064:                                ; preds = %invoke.cont278
-  br i1 undef, label %invoke.cont281, label %if.then.i2065
+  br i1 %arg, label %invoke.cont281, label %if.then.i2065
 
 if.then.i2065:                                    ; preds = %invoke.cont.i2064
   br label %invoke.cont281
@@ -1286,7 +1286,7 @@ invoke.cont315:                                   ; preds = %invoke.cont312
           to label %invoke.cont321 unwind label %lpad320
 
 invoke.cont321:                                   ; preds = %invoke.cont315
-  br i1 undef, label %land.end344, label %land.rhs335
+  br i1 %arg, label %land.end344, label %land.rhs335
 
 land.rhs335:                                      ; preds = %invoke.cont321
   %call342 = invoke signext i8 @objc_msgSend(ptr undef, ptr undef, ptr @_unnamed_cfstring)
@@ -1297,7 +1297,7 @@ land.end344:                                      ; preds = %land.rhs335, %invok
           to label %invoke.cont.i2070 unwind label %lpad340.body
 
 invoke.cont.i2070:                                ; preds = %land.end344
-  br i1 undef, label %invoke.cont345, label %if.then.i2071
+  br i1 %arg, label %invoke.cont345, label %if.then.i2071
 
 if.then.i2071:                                    ; preds = %invoke.cont.i2070
   br label %invoke.cont345
@@ -1319,7 +1319,7 @@ invoke.cont370:                                   ; preds = %invoke.cont364
           to label %invoke.cont.i2076 unwind label %lpad.i2078
 
 invoke.cont.i2076:                                ; preds = %invoke.cont370
-  br i1 undef, label %invoke.cont373, label %if.then.i2077
+  br i1 %arg, label %invoke.cont373, label %if.then.i2077
 
 if.then.i2077:                                    ; preds = %invoke.cont.i2076
   br label %invoke.cont373
@@ -1346,7 +1346,7 @@ invoke.cont383:                                   ; preds = %invoke.cont382
           to label %invoke.cont.i2082 unwind label %lpad.i2084
 
 invoke.cont.i2082:                                ; preds = %invoke.cont383
-  br i1 undef, label %invoke.cont392, label %if.then.i2083
+  br i1 %arg, label %invoke.cont392, label %if.then.i2083
 
 if.then.i2083:                                    ; preds = %invoke.cont.i2082
   br label %invoke.cont392
@@ -1377,7 +1377,7 @@ invoke.cont402:                                   ; preds = %invoke.cont399
           to label %invoke.cont.i2088 unwind label %lpad.i2090
 
 invoke.cont.i2088:                                ; preds = %invoke.cont402
-  br i1 undef, label %invoke.cont405, label %if.then.i2089
+  br i1 %arg, label %invoke.cont405, label %if.then.i2089
 
 if.then.i2089:                                    ; preds = %invoke.cont.i2088
   br label %invoke.cont405
@@ -1404,7 +1404,7 @@ invoke.cont412:                                   ; preds = %invoke.cont409
           to label %invoke.cont.i2094 unwind label %lpad.i2096
 
 invoke.cont.i2094:                                ; preds = %invoke.cont412
-  br i1 undef, label %invoke.cont418, label %if.then.i2095
+  br i1 %arg, label %invoke.cont418, label %if.then.i2095
 
 if.then.i2095:                                    ; preds = %invoke.cont.i2094
   br label %invoke.cont418
@@ -1435,7 +1435,7 @@ invoke.cont429:                                   ; preds = %invoke.cont426
           to label %invoke.cont.i2100 unwind label %lpad.i2102
 
 invoke.cont.i2100:                                ; preds = %invoke.cont429
-  br i1 undef, label %invoke.cont432, label %if.then.i2101
+  br i1 %arg, label %invoke.cont432, label %if.then.i2101
 
 if.then.i2101:                                    ; preds = %invoke.cont.i2100
   br label %invoke.cont432
@@ -1467,7 +1467,7 @@ invoke.cont443:                                   ; preds = %invoke.cont.i2106
           to label %invoke.cont.i2112 unwind label %lpad.i2114
 
 invoke.cont.i2112:                                ; preds = %invoke.cont443
-  br i1 undef, label %invoke.cont449, label %if.then.i2113
+  br i1 %arg, label %invoke.cont449, label %if.then.i2113
 
 if.then.i2113:                                    ; preds = %invoke.cont.i2112
   br label %invoke.cont449
@@ -1490,7 +1490,7 @@ invoke.cont455:                                   ; preds = %invoke.cont452
           to label %invoke.cont.i2118 unwind label %lpad.i2120
 
 invoke.cont.i2118:                                ; preds = %invoke.cont455
-  br i1 undef, label %invoke.cont458, label %if.then.i2119
+  br i1 %arg, label %invoke.cont458, label %if.then.i2119
 
 if.then.i2119:                                    ; preds = %invoke.cont.i2118
   br label %invoke.cont458
@@ -1509,7 +1509,7 @@ invoke.cont460:                                   ; preds = %invoke.cont458
           to label %invoke.cont.i2124 unwind label %lpad.i2126
 
 invoke.cont.i2124:                                ; preds = %invoke.cont460
-  br i1 undef, label %invoke.cont466, label %if.then.i2125
+  br i1 %arg, label %invoke.cont466, label %if.then.i2125
 
 if.then.i2125:                                    ; preds = %invoke.cont.i2124
   br label %invoke.cont466
@@ -1528,7 +1528,7 @@ invoke.cont469:                                   ; preds = %invoke.cont466
           to label %invoke.cont.i2130 unwind label %lpad.i2132
 
 invoke.cont.i2130:                                ; preds = %invoke.cont469
-  br i1 undef, label %invoke.cont475, label %if.then.i2131
+  br i1 %arg, label %invoke.cont475, label %if.then.i2131
 
 if.then.i2131:                                    ; preds = %invoke.cont.i2130
   br label %invoke.cont475
@@ -1563,7 +1563,7 @@ invoke.cont509:                                   ; preds = %invoke.cont506
           to label %invoke.cont512 unwind label %lpad489
 
 invoke.cont512:                                   ; preds = %invoke.cont509
-  br i1 undef, label %msgSend.null-receiver, label %msgSend.call
+  br i1 %arg, label %msgSend.null-receiver, label %msgSend.call
 
 msgSend.call:                                     ; preds = %invoke.cont512
   invoke void @objc_msgSend_stret(ptr sret(%struct.CGPoint) undef, ptr undef, ptr undef)
@@ -1577,7 +1577,7 @@ msgSend.cont:                                     ; preds = %msgSend.null-receiv
           to label %invoke.cont.i2136 unwind label %lpad.i2138
 
 invoke.cont.i2136:                                ; preds = %msgSend.cont
-  br i1 undef, label %invoke.cont521, label %if.then.i2137
+  br i1 %arg, label %invoke.cont521, label %if.then.i2137
 
 if.then.i2137:                                    ; preds = %invoke.cont.i2136
   br label %invoke.cont521
@@ -1604,7 +1604,7 @@ invoke.cont534:                                   ; preds = %invoke.cont531
           to label %invoke.cont.i2142 unwind label %lpad.i2144
 
 invoke.cont.i2142:                                ; preds = %invoke.cont534
-  br i1 undef, label %invoke.cont540, label %if.then.i2143
+  br i1 %arg, label %invoke.cont540, label %if.then.i2143
 
 if.then.i2143:                                    ; preds = %invoke.cont.i2142
   br label %invoke.cont540
@@ -1918,31 +1918,31 @@ eh.resume:                                        ; preds = %lpad580, %ehcleanup
 
 @"OBJC_EHTYPE_$_NSException" = external global i8
 
-define void @test4() personality ptr @__objc_personality_v0 {
+define void @test4(i1 %arg) personality ptr @__objc_personality_v0 {
 entry:
-  br i1 undef, label %if.end13, label %if.then10
+  br i1 %arg, label %if.end13, label %if.then10
 
 if.then10:                                        ; preds = %entry
   br label %if.end13
 
 if.end13:                                         ; preds = %if.then10, %entry
   %0 = call ptr @objc_msgSend(ptr undef, ptr undef, ptr @_unnamed_cfstring, i64 2, ptr @_unnamed_cfstring_2, i8 signext 0), !clang.arc.no_objc_arc_exceptions !0
-  br i1 undef, label %if.then17, label %if.end18
+  br i1 %arg, label %if.then17, label %if.end18
 
 if.then17:                                        ; preds = %if.end13
   br label %if.end18
 
 if.end18:                                         ; preds = %if.then17, %if.end13
-  br i1 undef, label %if.then64, label %if.end73
+  br i1 %arg, label %if.then64, label %if.end73
 
 if.then64:                                        ; preds = %if.end18
-  br i1 undef, label %cond.end71, label %cond.true68
+  br i1 %arg, label %cond.end71, label %cond.true68
 
 cond.true68:                                      ; preds = %if.then64
   br label %cond.end71
 
 cond.end71:                                       ; preds = %cond.true68, %if.then64
-  br i1 undef, label %cleanup.action, label %cleanup.done
+  br i1 %arg, label %cleanup.action, label %cleanup.done
 
 cleanup.action:                                   ; preds = %cond.end71
   br label %cleanup.done
@@ -1951,7 +1951,7 @@ cleanup.done:                                     ; preds = %cleanup.action, %co
   br label %if.end73
 
 if.end73:                                         ; preds = %cleanup.done, %if.end18
-  br i1 undef, label %forcoll.empty, label %forcoll.loopinit
+  br i1 %arg, label %forcoll.empty, label %forcoll.loopinit
 
 forcoll.loopinit:                                 ; preds = %if.end73
   br label %forcoll.loopbody.outer
@@ -1960,34 +1960,34 @@ forcoll.loopbody.outer:                           ; preds = %forcoll.refetch, %f
   br label %forcoll.loopbody
 
 forcoll.loopbody:                                 ; preds = %forcoll.notmutated, %forcoll.loopbody.outer
-  br i1 undef, label %forcoll.notmutated, label %forcoll.mutated
+  br i1 %arg, label %forcoll.notmutated, label %forcoll.mutated
 
 forcoll.mutated:                                  ; preds = %forcoll.loopbody
   br label %forcoll.notmutated
 
 forcoll.notmutated:                               ; preds = %forcoll.mutated, %forcoll.loopbody
-  br i1 undef, label %forcoll.loopbody, label %forcoll.refetch
+  br i1 %arg, label %forcoll.loopbody, label %forcoll.refetch
 
 forcoll.refetch:                                  ; preds = %forcoll.notmutated
-  br i1 undef, label %forcoll.empty, label %forcoll.loopbody.outer
+  br i1 %arg, label %forcoll.empty, label %forcoll.loopbody.outer
 
 forcoll.empty:                                    ; preds = %forcoll.refetch, %if.end73
-  br i1 undef, label %if.end85, label %if.then82
+  br i1 %arg, label %if.end85, label %if.then82
 
 if.then82:                                        ; preds = %forcoll.empty
   br label %if.end85
 
 if.end85:                                         ; preds = %if.then82, %forcoll.empty
-  br i1 undef, label %if.then87, label %if.end102
+  br i1 %arg, label %if.then87, label %if.end102
 
 if.then87:                                        ; preds = %if.end85
-  br i1 undef, label %if.end94, label %if.then91
+  br i1 %arg, label %if.end94, label %if.then91
 
 if.then91:                                        ; preds = %if.then87
   br label %if.end94
 
 if.end94:                                         ; preds = %if.then91, %if.then87
-  br i1 undef, label %if.end101, label %if.then98
+  br i1 %arg, label %if.end101, label %if.then98
 
 if.then98:                                        ; preds = %if.end94
   br label %if.end101
@@ -1996,139 +1996,139 @@ if.end101:                                        ; preds = %if.then98, %if.end9
   br label %if.end102
 
 if.end102:                                        ; preds = %if.end101, %if.end85
-  br i1 undef, label %do.body113, label %if.then107
+  br i1 %arg, label %do.body113, label %if.then107
 
 if.then107:                                       ; preds = %if.end102
   br label %do.body113
 
 do.body113:                                       ; preds = %if.then107, %if.end102
-  br i1 undef, label %if.then116, label %if.end117
+  br i1 %arg, label %if.then116, label %if.end117
 
 if.then116:                                       ; preds = %do.body113
   br label %if.end117
 
 if.end117:                                        ; preds = %if.then116, %do.body113
-  br i1 undef, label %if.then125, label %if.end126
+  br i1 %arg, label %if.then125, label %if.end126
 
 if.then125:                                       ; preds = %if.end117
   br label %if.end126
 
 if.end126:                                        ; preds = %if.then125, %if.end117
-  br i1 undef, label %do.end166, label %cond.true132
+  br i1 %arg, label %do.end166, label %cond.true132
 
 cond.true132:                                     ; preds = %if.end126
-  br i1 undef, label %do.body148, label %cond.true151
+  br i1 %arg, label %do.body148, label %cond.true151
 
 do.body148:                                       ; preds = %cond.true132
-  br i1 undef, label %do.end166, label %cond.true151
+  br i1 %arg, label %do.end166, label %cond.true151
 
 cond.true151:                                     ; preds = %do.body148, %cond.true132
-  br i1 undef, label %if.then162, label %do.end166
+  br i1 %arg, label %if.then162, label %do.end166
 
 if.then162:                                       ; preds = %cond.true151
   br label %do.end166
 
 do.end166:                                        ; preds = %if.then162, %cond.true151, %do.body148, %if.end126
-  br i1 undef, label %if.then304, label %if.then170
+  br i1 %arg, label %if.then304, label %if.then170
 
 if.then170:                                       ; preds = %do.end166
-  br i1 undef, label %do.end193, label %cond.true179
+  br i1 %arg, label %do.end193, label %cond.true179
 
 cond.true179:                                     ; preds = %if.then170
-  br i1 undef, label %if.then190, label %do.end193
+  br i1 %arg, label %if.then190, label %do.end193
 
 if.then190:                                       ; preds = %cond.true179
   br label %do.end193
 
 do.end193:                                        ; preds = %if.then190, %cond.true179, %if.then170
-  br i1 undef, label %do.body200, label %do.body283
+  br i1 %arg, label %do.body200, label %do.body283
 
 do.body200:                                       ; preds = %do.end193
-  br i1 undef, label %do.end254, label %cond.true203
+  br i1 %arg, label %do.end254, label %cond.true203
 
 cond.true203:                                     ; preds = %do.body200
-  br i1 undef, label %do.body218, label %cond.true221
+  br i1 %arg, label %do.body218, label %cond.true221
 
 do.body218:                                       ; preds = %cond.true203
-  br i1 undef, label %do.end254, label %cond.true221
+  br i1 %arg, label %do.end254, label %cond.true221
 
 cond.true221:                                     ; preds = %do.body218, %cond.true203
-  br i1 undef, label %if.then232, label %do.body236
+  br i1 %arg, label %if.then232, label %do.body236
 
 if.then232:                                       ; preds = %cond.true221
   br label %do.body236
 
 do.body236:                                       ; preds = %if.then232, %cond.true221
-  br i1 undef, label %do.end254, label %cond.true239
+  br i1 %arg, label %do.end254, label %cond.true239
 
 cond.true239:                                     ; preds = %do.body236
-  br i1 undef, label %if.then250, label %do.end254
+  br i1 %arg, label %if.then250, label %do.end254
 
 if.then250:                                       ; preds = %cond.true239
   br label %do.end254
 
 do.end254:                                        ; preds = %if.then250, %cond.true239, %do.body236, %do.body218, %do.body200
-  br i1 undef, label %do.end277, label %cond.true263
+  br i1 %arg, label %do.end277, label %cond.true263
 
 cond.true263:                                     ; preds = %do.end254
-  br i1 undef, label %if.then274, label %do.end277
+  br i1 %arg, label %if.then274, label %do.end277
 
 if.then274:                                       ; preds = %cond.true263
   unreachable
 
 do.end277:                                        ; preds = %cond.true263, %do.end254
-  br i1 undef, label %if.then280, label %do.body283
+  br i1 %arg, label %if.then280, label %do.body283
 
 if.then280:                                       ; preds = %do.end277
   br label %do.body283
 
 do.body283:                                       ; preds = %if.then280, %do.end277, %do.end193
-  br i1 undef, label %if.end301, label %cond.true286
+  br i1 %arg, label %if.end301, label %cond.true286
 
 cond.true286:                                     ; preds = %do.body283
-  br i1 undef, label %if.then297, label %if.end301
+  br i1 %arg, label %if.then297, label %if.end301
 
 if.then297:                                       ; preds = %cond.true286
   br label %if.end301
 
 if.end301:                                        ; preds = %if.then297, %cond.true286, %do.body283
-  br i1 undef, label %if.then304, label %do.body351
+  br i1 %arg, label %if.then304, label %do.body351
 
 if.then304:                                       ; preds = %if.end301, %do.end166
-  br i1 undef, label %do.body309.lr.ph, label %do.body351
+  br i1 %arg, label %do.body309.lr.ph, label %do.body351
 
 do.body309.lr.ph:                                 ; preds = %if.then304
   br label %do.body309
 
 do.body309:                                       ; preds = %for.cond.backedge, %do.body309.lr.ph
-  br i1 undef, label %do.end328, label %cond.true312
+  br i1 %arg, label %do.end328, label %cond.true312
 
 cond.true312:                                     ; preds = %do.body309
-  br i1 undef, label %if.then323, label %do.end328
+  br i1 %arg, label %if.then323, label %do.end328
 
 if.then323:                                       ; preds = %cond.true312
   br label %do.end328
 
 do.end328:                                        ; preds = %if.then323, %cond.true312, %do.body309
-  br i1 undef, label %for.cond.backedge, label %cond.true335
+  br i1 %arg, label %for.cond.backedge, label %cond.true335
 
 for.cond.backedge:                                ; preds = %if.then346, %cond.true335, %do.end328
-  br i1 undef, label %do.body309, label %do.body351
+  br i1 %arg, label %do.body309, label %do.body351
 
 cond.true335:                                     ; preds = %do.end328
-  br i1 undef, label %if.then346, label %for.cond.backedge
+  br i1 %arg, label %if.then346, label %for.cond.backedge
 
 if.then346:                                       ; preds = %cond.true335
   br label %for.cond.backedge
 
 do.body351:                                       ; preds = %for.cond.backedge, %if.then304, %if.end301
-  br i1 undef, label %if.then354, label %if.end355
+  br i1 %arg, label %if.then354, label %if.end355
 
 if.then354:                                       ; preds = %do.body351
   br label %if.end355
 
 if.end355:                                        ; preds = %if.then354, %do.body351
-  br i1 undef, label %if.else, label %if.then364
+  br i1 %arg, label %if.else, label %if.then364
 
 if.then364:                                       ; preds = %if.end355
   br label %do.body366
@@ -2137,7 +2137,7 @@ if.else:                                          ; preds = %if.end355
   br label %do.body366
 
 do.body366:                                       ; preds = %if.else, %if.then364
-  br i1 undef, label %if.then369, label %if.end377.critedge
+  br i1 %arg, label %if.then369, label %if.end377.critedge
 
 if.then369:                                       ; preds = %do.body366
   br label %if.end377
@@ -2146,7 +2146,7 @@ if.end377.critedge:                               ; preds = %do.body366
   br label %if.end377
 
 if.end377:                                        ; preds = %if.end377.critedge, %if.then369
-  br i1 undef, label %if.then383, label %if.end392.critedge
+  br i1 %arg, label %if.then383, label %if.end392.critedge
 
 if.then383:                                       ; preds = %if.end377
   br label %if.end392
@@ -2155,7 +2155,7 @@ if.end392.critedge:                               ; preds = %if.end377
   br label %if.end392
 
 if.end392:                                        ; preds = %if.end392.critedge, %if.then383
-  br i1 undef, label %if.then398, label %if.end399
+  br i1 %arg, label %if.then398, label %if.end399
 
 if.then398:                                       ; preds = %if.end392
   br label %if.end399
@@ -2165,7 +2165,7 @@ if.end399:                                        ; preds = %if.then398, %if.end
           to label %eh.cont unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
 
 eh.cont:                                          ; preds = %if.end399
-  br i1 undef, label %if.then430, label %if.end439.critedge
+  br i1 %arg, label %if.then430, label %if.end439.critedge
 
 if.then430:                                       ; preds = %eh.cont
   %1 = call ptr @llvm.objc.retain(ptr %0)
diff --git a/llvm/test/Transforms/OpenMP/keep_rpc_client.ll b/llvm/test/Transforms/OpenMP/keep_rpc_client.ll
index 7bac905e1793d..b1c40e852a5e5 100644
--- a/llvm/test/Transforms/OpenMP/keep_rpc_client.ll
+++ b/llvm/test/Transforms/OpenMP/keep_rpc_client.ll
@@ -2,28 +2,21 @@
 ; RUN: opt -S -passes=openmp-opt-postlink < %s | FileCheck %s --check-prefix=POSTLINK
 ; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s --check-prefix=PRELINK
 
-@client = internal addrspace(1) global i64 zeroinitializer, align 8
-@__llvm_libc_rpc_client = protected local_unnamed_addr addrspace(1) global ptr addrspacecast (ptr addrspace(1) @client to ptr), align 8
+@__llvm_rpc_client = internal addrspace(1) global i64 zeroinitializer, align 8
 
 ;.
-; POSTLINK: @client = internal addrspace(1) global i64 0, align 8
-; POSTLINK: @__llvm_libc_rpc_client = protected local_unnamed_addr addrspace(1) global ptr addrspacecast (ptr addrspace(1) @client to ptr), align 8
-;.
-; PRELINK: @client = internal addrspace(1) global i64 0, align 8
-; PRELINK: @__llvm_libc_rpc_client = protected local_unnamed_addr addrspace(1) global ptr addrspacecast (ptr addrspace(1) @client to ptr), align 8
+; PRELINK: @__llvm_rpc_client = internal addrspace(1) global i64 0, align 8
 ;.
 define i64 @a() {
 ; POSTLINK-LABEL: define {{[^@]+}}@a
 ; POSTLINK-SAME: () #[[ATTR0:[0-9]+]] {
-; POSTLINK-NEXT:    [[RETVAL:%.*]] = load i64, ptr addrspace(1) @client, align 8
-; POSTLINK-NEXT:    ret i64 [[RETVAL]]
+; POSTLINK-NEXT:    ret i64 0
 ;
 ; PRELINK-LABEL: define {{[^@]+}}@a
 ; PRELINK-SAME: () #[[ATTR0:[0-9]+]] {
-; PRELINK-NEXT:    [[RETVAL:%.*]] = load i64, ptr addrspace(1) @client, align 8
-; PRELINK-NEXT:    ret i64 [[RETVAL]]
+; PRELINK-NEXT:    ret i64 0
 ;
-  %retval = load i64, ptr addrspace(1) @client, align 8
+  %retval = load i64, ptr addrspace(1) @__llvm_rpc_client, align 8
   ret i64 %retval
 }
 
diff --git a/llvm/test/Transforms/OpenMP/remove_rpc_client.ll b/llvm/test/Transforms/OpenMP/remove_rpc_client.ll
index e6c7f704d0af5..29f6c6b12e4ce 100644
--- a/llvm/test/Transforms/OpenMP/remove_rpc_client.ll
+++ b/llvm/test/Transforms/OpenMP/remove_rpc_client.ll
@@ -2,12 +2,10 @@
 ; RUN: opt -S -passes=openmp-opt-postlink < %s | FileCheck %s --check-prefix=POSTLINK
 ; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s --check-prefix=PRELINK
 
-@client = internal addrspace(1) global i32 zeroinitializer, align 8
-@__llvm_libc_rpc_client = protected local_unnamed_addr addrspace(1) global ptr addrspacecast (ptr addrspace(1) @client to ptr), align 8
+@__llvm_rpc_client = internal addrspace(1) global i32 zeroinitializer, align 8
 
 ;.
-; PRELINK: @client = internal addrspace(1) global i32 0, align 8
-; PRELINK: @__llvm_libc_rpc_client = protected local_unnamed_addr addrspace(1) global ptr addrspacecast (ptr addrspace(1) @client to ptr), align 8
+; PRELINK: @__llvm_rpc_client = internal addrspace(1) global i32 0, align 8
 ;.
 define void @a() {
 ; POSTLINK-LABEL: define {{[^@]+}}@a() {
diff --git a/llvm/test/Transforms/PGOProfile/loop_entries_gen.ll b/llvm/test/Transforms/PGOProfile/loop_entries_gen.ll
new file mode 100644
index 0000000000000..ed101271558c6
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/loop_entries_gen.ll
@@ -0,0 +1,58 @@
+; RUN: opt %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefixes=CHECK,NOTLOOPENTRIES --implicit-check-not=@llvm.instrprof.increment
+; RUN: opt %s -passes=pgo-instr-gen -pgo-instrument-loop-entries -S | FileCheck %s --check-prefixes=CHECK,LOOPENTRIES --implicit-check-not=@llvm.instrprof.increment
+; RUN: opt %s -passes=pgo-instr-gen -pgo-instrument-entry -S | FileCheck %s --check-prefixes=CHECK,FUNCTIONENTRY --implicit-check-not=@llvm.instrprof.increment
+
+; CHECK: $__llvm_profile_raw_version = comdat any
+; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat
+; CHECK: @__profn_test_simple_for_with_bypass = private constant [27 x i8] c"test_simple_for_with_bypass"
+
+define i32 @test_simple_for_with_bypass(i32 %n) {
+entry:
+; CHECK: entry:
+; NOTLOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 1)
+; LOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 1)
+; FUNCTIONENTRY: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 0)
+  %mask = and i32 %n, 65535
+  %skip = icmp eq i32 %mask, 0
+  br i1 %skip, label %end, label %for.entry
+
+for.entry:
+; CHECK: for.entry:
+; LOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 2)
+  br label %for.cond
+
+for.cond:
+; CHECK: for.cond:
+  %i = phi i32 [ 0, %for.entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %for.entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end, !prof !1
+
+for.body:
+; CHECK: for.body:
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+; CHECK: for.inc:
+; NOTLOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 0)
+; LOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 0)
+; FUNCTIONENTRY: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 1)
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+; CHECK: for.end:
+; NOTLOOPENTRIES: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 2)
+; FUNCTIONENTRY: call void @llvm.instrprof.increment(ptr @__profn_test_simple_for_with_bypass, i64 {{[0-9]+}}, i32 3, i32 2)
+  br label %end
+
+end:
+; CHECK: end:
+  %final_sum = phi i32 [ %sum, %for.end ], [ 0, %entry ]
+  ret i32 %final_sum
+}
+
+; CHECK: declare void @llvm.instrprof.increment(ptr, i64, i32, i32) #0
+
+!1 = !{!"branch_weights", i32 100000, i32 80}
diff --git a/llvm/test/Transforms/PGOProfile/loop_entries_use.ll b/llvm/test/Transforms/PGOProfile/loop_entries_use.ll
new file mode 100644
index 0000000000000..616ecbaf439c3
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/loop_entries_use.ll
@@ -0,0 +1,106 @@
+; RUN: rm -rf %t && split-file %s %t
+
+; RUN: llvm-profdata merge %t/default.proftext -o %t/default.profdata
+; RUN: opt %t/main.ll -passes=pgo-instr-use -pgo-test-profile-file=%t/default.profdata -S | FileCheck %s
+; RUN: llvm-profdata merge %t/loop_entries.proftext -o %t/loop_entries.profdata
+; RUN: opt %t/main.ll -passes=pgo-instr-use -pgo-test-profile-file=%t/loop_entries.profdata -S | FileCheck %s
+; RUN: llvm-profdata merge %t/function_entry.proftext -o %t/function_entry.profdata
+; RUN: opt %t/main.ll -passes=pgo-instr-use -pgo-test-profile-file=%t/function_entry.profdata -S | FileCheck %s
+
+;--- main.ll
+
+define i32 @test_simple_for_with_bypass(i32 %n) {
+; CHECK: define i32 @test_simple_for_with_bypass(i32 %n)
+; CHECK-SAME: !prof ![[ENTRY_COUNT:[0-9]*]]
+entry:
+; CHECK: entry:
+  %mask = and i32 %n, 65535
+  %skip = icmp eq i32 %mask, 0
+  br i1 %skip, label %end, label %for.entry
+; CHECK: br i1 %skip, label %end, label %for.entry
+; CHECK-SAME: !prof ![[BW_FOR_BYPASS:[0-9]+]]
+
+for.entry:
+; CHECK: for.entry:
+  br label %for.cond
+
+for.cond:
+; CHECK: for.cond:
+  %i = phi i32 [ 0, %for.entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %for.entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end, !prof !1
+; CHECK: br i1 %cmp, label %for.body, label %for.end
+; CHECK-SAME: !prof ![[BW_FOR_COND:[0-9]+]]
+
+for.body:
+; CHECK: for.body:
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+; CHECK: for.inc:
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+; CHECK: for.end:
+  br label %end
+
+end:
+; CHECK: end:
+  %final_sum = phi i32 [ %sum, %for.end ], [ 0, %entry ]
+  ret i32 %final_sum
+}
+
+!1 = !{!"branch_weights", i32 100000, i32 80}
+
+; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 12}
+; CHECK: ![[BW_FOR_BYPASS]] = !{!"branch_weights", i32 4, i32 8}
+; CHECK: ![[BW_FOR_COND]] = !{!"branch_weights", i32 123456, i32 8}
+
+;--- default.proftext
+
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_simple_for_with_bypass
+# Func Hash:
+536873292337293370
+# Num Counters:
+3
+# Counter Values:
+123456
+12
+8
+
+;--- loop_entries.proftext
+
+# :ir is the flag to indicate this is IR level profile.
+:ir
+# Always instrument the loop entry blocks
+:instrument_loop_entries
+test_simple_for_with_bypass
+# Func Hash:
+536873292337293370
+# Num Counters:
+3
+# Counter Values:
+123456
+12
+8
+
+;--- function_entry.proftext
+
+# :ir is the flag to indicate this is IR level profile.
+:ir
+# Always instrument the function entry block
+:entry_first
+test_simple_for_with_bypass
+# Func Hash:
+536873292337293370
+# Num Counters:
+3
+# Counter Values:
+12
+123456
+8
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index f44e39e82d606..4423b89d81565 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -30,7 +30,7 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI11:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP4]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
@@ -49,7 +49,7 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER13]] ]
 ; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[SUM_07_PH]], [[FOR_BODY_PREHEADER13]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[ARRAY]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP8]], [[SUM_07]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -145,7 +145,7 @@ define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[S_COERCE0]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP3]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
@@ -225,7 +225,7 @@ define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_acces
 ; CHECK-LABEL: define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_access(
 ; CHECK-SAME: ptr nocapture noundef nonnull readonly align 8 dereferenceable(16) [[THIS:%.*]], i64 noundef [[__IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[__SIZE__I:%.*]] = getelementptr inbounds i8, ptr [[THIS]], i64 8
+; CHECK-NEXT:    [[__SIZE__I:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__SIZE__I]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[__IDX]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF4]]
@@ -270,7 +270,7 @@ define hidden noundef i64 @span_access(ptr noundef nonnull align 8 dereferenceab
 ; CHECK-LABEL: define hidden noundef i64 @span_access(
 ; CHECK-SAME: ptr nocapture noundef nonnull readonly align 8 dereferenceable(16) [[THIS:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[__SIZE_:%.*]] = getelementptr inbounds i8, ptr [[THIS]], i64 8
+; CHECK-NEXT:    [[__SIZE_:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__SIZE_]], align 8
 ; CHECK-NEXT:    ret i64 [[TMP0]]
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
index 4a5ce1359d257..89b2f039c827d 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
@@ -49,7 +49,7 @@ define void @loop(ptr %X, ptr %Y) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp olt <2 x double> [[WIDE_LOAD]], zeroinitializer
@@ -61,7 +61,7 @@ define void @loop(ptr %X, ptr %Y) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[TMP4]], <2 x double> zeroinitializer, <2 x double> [[TMP8]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 16
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
 ; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr [[TMP11]], align 8
 ; CHECK-NEXT:    store <2 x double> [[TMP10]], ptr [[TMP12]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -71,13 +71,13 @@ define void @loop(ptr %X, ptr %Y) {
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp olt double [[TMP14]], 0.000000e+00
 ; CHECK-NEXT:    [[CMP1_I:%.*]] = fcmp ogt double [[TMP14]], 6.000000e+00
 ; CHECK-NEXT:    [[DOTV_I:%.*]] = select i1 [[CMP1_I]], double 6.000000e+00, double [[TMP14]]
 ; CHECK-NEXT:    [[RETVAL_0_I:%.*]] = select i1 [[CMP_I]], double 0.000000e+00, double [[DOTV_I]]
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store double [[RETVAL_0_I]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 20000
@@ -149,13 +149,13 @@ define void @loop2(ptr %A, ptr %B, ptr %C, float %x) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META4:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META4]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 20)
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD7]], splat (i32 20)
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 16
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !alias.scope [[META7:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP5]], align 4, !alias.scope [[META7]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD8]]
@@ -168,7 +168,7 @@ define void @loop2(ptr %A, ptr %B, ptr %C, float %x) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <4 x float> [[TMP7]], [[WIDE_LOAD11]]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x float> [[TMP6]], <4 x float> [[TMP10]]
 ; CHECK-NEXT:    [[PREDPHI12:%.*]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP7]], <4 x float> [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP8]], align 4, !alias.scope [[META9]], !noalias [[META11]]
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI12]], ptr [[TMP12]], align 4, !alias.scope [[META9]], !noalias [[META11]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -176,13 +176,13 @@ define void @loop2(ptr %A, ptr %B, ptr %C, float %x) {
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       loop.body:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[C_GEP:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV1]]
+; CHECK-NEXT:    [[C_GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[IV1]]
 ; CHECK-NEXT:    [[C_LV:%.*]] = load i32, ptr [[C_GEP]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C_LV]], 20
-; CHECK-NEXT:    [[A_GEP_0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT:    [[A_GEP_0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV1]]
 ; CHECK-NEXT:    [[A_LV_0:%.*]] = load float, ptr [[A_GEP_0]], align 4
 ; CHECK-NEXT:    [[MUL2_I81_I:%.*]] = fmul float [[X]], [[A_LV_0]]
-; CHECK-NEXT:    [[B_GEP_0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV1]]
+; CHECK-NEXT:    [[B_GEP_0:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[IV1]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_LATCH]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[B_LV:%.*]] = load float, ptr [[B_GEP_0]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
index af24a9ab9e3f7..579961f546c0b 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
@@ -58,11 +58,11 @@ define void @s172(i32 noundef %xa, i32 noundef %xb, ptr noundef %a, ptr noundef
 ; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP19]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 16
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP20]], align 4, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP21]], align 4, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP22]], i64 16
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4, !alias.scope [[META3]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
index e514defc91e28..b14a36c8b3bcd 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/loopflatten.ll
@@ -18,7 +18,7 @@ define dso_local void @_Z3fooPiii(ptr %A, i32 %N, i32 %M) #0 {
 ; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
 ; CHECK:       for.cond1.preheader.us:
 ; CHECK-NEXT:    [[INDVAR6:%.*]] = phi i64 [ [[INDVAR_NEXT7:%.*]], [[FOR_COND1_PREHEADER_US]] ], [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US]] ]
-; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVAR6]]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVAR6]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
 ; CHECK-NEXT:    tail call void @_Z1fi(i32 [[TMP2]])
 ; CHECK-NEXT:    [[INDVAR_NEXT7]] = add nuw i64 [[INDVAR6]], 1
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 5aee938f5191f..ff9ad9ffa1408 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -12,16 +12,16 @@ define void @matrix_extract_insert_scalar(i32 %i, i32 %k, i32 %j, ptr nonnull al
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], [[CONV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp samesign ult i64 [[TMP1]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <225 x double>, ptr [[A:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[A:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[MATRIXEXT:%.*]] = load double, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 [[I:%.*]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP0]], [[CONV2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp samesign ult i64 [[TMP4]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP5]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <225 x double>, ptr [[B:%.*]], i64 0, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B:%.*]], i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[MATRIXEXT4:%.*]] = load double, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[MATRIXEXT]], [[MATRIXEXT4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[MATRIXEXT7:%.*]] = load double, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MATRIXEXT7]], [[MUL]]
 ; CHECK-NEXT:    store double [[SUB]], ptr [[TMP7]], align 8
@@ -93,7 +93,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[I]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[CONV6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[CONV6]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY4_US_PREHEADER:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
@@ -124,7 +124,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP12]], i64 1
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP16]])
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 16
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP17]], align 8, !alias.scope [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD21:%.*]] = load <2 x double>, ptr [[TMP18]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META3:![0-9]+]]
@@ -133,7 +133,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP20:%.*]] = fmul <2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT23]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x double> [[WIDE_LOAD21]], [[BROADCAST_SPLAT23]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP22]], i64 16
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24:%.*]] = load <2 x double>, ptr [[TMP22]], align 8, !alias.scope [[META5:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25:%.*]] = load <2 x double>, ptr [[TMP23]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = fsub <2 x double> [[WIDE_LOAD24]], [[TMP20]]
@@ -153,11 +153,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY4_US]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY4_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[TMP27:%.*]] = icmp samesign ult i64 [[INDVARS_IV]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP27]])
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[A]], i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[MATRIXEXT_US:%.*]] = load double, ptr [[TMP28]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US:%.*]] = load double, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[MUL_US:%.*]] = fmul double [[MATRIXEXT_US]], [[MATRIXEXT8_US]]
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US:%.*]] = load double, ptr [[TMP29]], align 8
 ; CHECK-NEXT:    [[SUB_US:%.*]] = fsub double [[MATRIXEXT11_US]], [[MUL_US]]
 ; CHECK-NEXT:    store double [[SUB_US]], ptr [[TMP29]], align 8
@@ -168,7 +168,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP30:%.*]] = add nuw nsw i64 [[CONV6]], 15
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp samesign ult i32 [[I]], 210
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP31]])
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP30]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_1:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_1]], label [[FOR_BODY4_US_PREHEADER_1:%.*]], label [[VECTOR_MEMCHECK_1:%.*]]
 ; CHECK:       vector.memcheck.1:
@@ -200,7 +200,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <2 x i1> [[TMP42]], i64 1
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP46]])
 ; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP33]]
-; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i8, ptr [[TMP47]], i64 16
+; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP47]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr [[TMP47]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_1:%.*]] = load <2 x double>, ptr [[TMP48]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = load double, ptr [[TMP32]], align 8, !alias.scope [[META3]]
@@ -209,7 +209,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP50:%.*]] = fmul <2 x double> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT23_1]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = fmul <2 x double> [[WIDE_LOAD21_1]], [[BROADCAST_SPLAT23_1]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP33]]
-; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[TMP52]], i64 16
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <2 x double>, ptr [[TMP52]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_1:%.*]] = load <2 x double>, ptr [[TMP53]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = fsub <2 x double> [[WIDE_LOAD24_1]], [[TMP50]]
@@ -230,11 +230,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP57:%.*]] = add nuw nsw i64 [[INDVARS_IV_1]], 15
 ; CHECK-NEXT:    [[TMP58:%.*]] = icmp samesign ult i64 [[INDVARS_IV_1]], 210
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP58]])
-; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[A]], i64 0, i64 [[TMP57]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_1:%.*]] = load double, ptr [[TMP59]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_1:%.*]] = load double, ptr [[TMP32]], align 8
 ; CHECK-NEXT:    [[MUL_US_1:%.*]] = fmul double [[MATRIXEXT_US_1]], [[MATRIXEXT8_US_1]]
-; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP57]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP57]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_1:%.*]] = load double, ptr [[TMP60]], align 8
 ; CHECK-NEXT:    [[SUB_US_1:%.*]] = fsub double [[MATRIXEXT11_US_1]], [[MUL_US_1]]
 ; CHECK-NEXT:    store double [[SUB_US_1]], ptr [[TMP60]], align 8
@@ -245,7 +245,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP61:%.*]] = add nuw nsw i64 [[CONV6]], 30
 ; CHECK-NEXT:    [[TMP62:%.*]] = icmp samesign ult i32 [[I]], 195
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP62]])
-; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP61]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_2:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_2]], label [[FOR_BODY4_US_PREHEADER_2:%.*]], label [[VECTOR_MEMCHECK_2:%.*]]
 ; CHECK:       vector.memcheck.2:
@@ -277,7 +277,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <2 x i1> [[TMP73]], i64 1
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP77]])
 ; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP64]]
-; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds i8, ptr [[TMP78]], i64 16
+; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP78]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr [[TMP78]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_2:%.*]] = load <2 x double>, ptr [[TMP79]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP80:%.*]] = load double, ptr [[TMP63]], align 8, !alias.scope [[META3]]
@@ -286,7 +286,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP81:%.*]] = fmul <2 x double> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT23_2]]
 ; CHECK-NEXT:    [[TMP82:%.*]] = fmul <2 x double> [[WIDE_LOAD21_2]], [[BROADCAST_SPLAT23_2]]
 ; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP64]]
-; CHECK-NEXT:    [[TMP84:%.*]] = getelementptr inbounds i8, ptr [[TMP83]], i64 16
+; CHECK-NEXT:    [[TMP84:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP83]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_2:%.*]] = load <2 x double>, ptr [[TMP83]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_2:%.*]] = load <2 x double>, ptr [[TMP84]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP85:%.*]] = fsub <2 x double> [[WIDE_LOAD24_2]], [[TMP81]]
@@ -307,11 +307,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP88:%.*]] = add nuw nsw i64 [[INDVARS_IV_2]], 30
 ; CHECK-NEXT:    [[TMP89:%.*]] = icmp samesign ult i64 [[INDVARS_IV_2]], 195
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP89]])
-; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[A]], i64 0, i64 [[TMP88]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_2:%.*]] = load double, ptr [[TMP90]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_2:%.*]] = load double, ptr [[TMP63]], align 8
 ; CHECK-NEXT:    [[MUL_US_2:%.*]] = fmul double [[MATRIXEXT_US_2]], [[MATRIXEXT8_US_2]]
-; CHECK-NEXT:    [[TMP91:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP88]]
+; CHECK-NEXT:    [[TMP91:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP88]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_2:%.*]] = load double, ptr [[TMP91]], align 8
 ; CHECK-NEXT:    [[SUB_US_2:%.*]] = fsub double [[MATRIXEXT11_US_2]], [[MUL_US_2]]
 ; CHECK-NEXT:    store double [[SUB_US_2]], ptr [[TMP91]], align 8
@@ -322,7 +322,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP92:%.*]] = add nuw nsw i64 [[CONV6]], 45
 ; CHECK-NEXT:    [[TMP93:%.*]] = icmp samesign ult i32 [[I]], 180
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP93]])
-; CHECK-NEXT:    [[TMP94:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP92]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK_3:%.*]] = icmp samesign ult i32 [[I]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_3]], label [[FOR_BODY4_US_PREHEADER_3:%.*]], label [[VECTOR_MEMCHECK_3:%.*]]
 ; CHECK:       vector.memcheck.3:
@@ -354,7 +354,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP108:%.*]] = extractelement <2 x i1> [[TMP104]], i64 1
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP108]])
 ; CHECK-NEXT:    [[TMP109:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP95]]
-; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds i8, ptr [[TMP109]], i64 16
+; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP109]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr [[TMP109]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD21_3:%.*]] = load <2 x double>, ptr [[TMP110]], align 8, !alias.scope [[META0]]
 ; CHECK-NEXT:    [[TMP111:%.*]] = load double, ptr [[TMP94]], align 8, !alias.scope [[META3]]
@@ -363,7 +363,7 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP112:%.*]] = fmul <2 x double> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT23_3]]
 ; CHECK-NEXT:    [[TMP113:%.*]] = fmul <2 x double> [[WIDE_LOAD21_3]], [[BROADCAST_SPLAT23_3]]
 ; CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP95]]
-; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i8, ptr [[TMP114]], i64 16
+; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP114]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD24_3:%.*]] = load <2 x double>, ptr [[TMP114]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[WIDE_LOAD25_3:%.*]] = load <2 x double>, ptr [[TMP115]], align 8, !alias.scope [[META5]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP116:%.*]] = fsub <2 x double> [[WIDE_LOAD24_3]], [[TMP112]]
@@ -384,11 +384,11 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    [[TMP119:%.*]] = add nuw nsw i64 [[INDVARS_IV_3]], 45
 ; CHECK-NEXT:    [[TMP120:%.*]] = icmp samesign ult i64 [[INDVARS_IV_3]], 180
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP120]])
-; CHECK-NEXT:    [[TMP121:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[A]], i64 0, i64 [[TMP119]]
 ; CHECK-NEXT:    [[MATRIXEXT_US_3:%.*]] = load double, ptr [[TMP121]], align 8
 ; CHECK-NEXT:    [[MATRIXEXT8_US_3:%.*]] = load double, ptr [[TMP94]], align 8
 ; CHECK-NEXT:    [[MUL_US_3:%.*]] = fmul double [[MATRIXEXT_US_3]], [[MATRIXEXT8_US_3]]
-; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds <225 x double>, ptr [[B]], i64 0, i64 [[TMP119]]
+; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw <225 x double>, ptr [[B]], i64 0, i64 [[TMP119]]
 ; CHECK-NEXT:    [[MATRIXEXT11_US_3:%.*]] = load double, ptr [[TMP122]], align 8
 ; CHECK-NEXT:    [[SUB_US_3:%.*]] = fsub double [[MATRIXEXT11_US_3]], [[MUL_US_3]]
 ; CHECK-NEXT:    store double [[SUB_US_3]], ptr [[TMP122]], align 8
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll
new file mode 100644
index 0000000000000..10b07ad6e7491
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt %s -mtriple=arm64-apple-macosx -passes='default<O3>' -inline-threshold=2 -inline-call-penalty=5 -S | FileCheck %s
+
+declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
+declare ptr @__memcpy_chk(ptr, ptr, i64, i64)
+declare ptr @__memmove_chk(ptr, ptr, i64, i64)
+declare ptr @__mempcpy_chk(ptr, ptr, i64, i64)
+declare ptr @__memset_chk(ptr, i32, i64, i64)
+
+define void @callee_memcpy(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @callee_memcpy
+; CHECK-SAME: (ptr [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DST]], ptr align 1 [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  %objsize = call i64 @llvm.objectsize.i64.p0(ptr %dst, i1 false, i1 true, i1 false)
+  %call.memcpy = call ptr @__memcpy_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  ret void
+}
+
+define void @callee_memmove(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @callee_memmove
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    tail call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DST]], ptr align 1 [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  %objsize = call i64 @llvm.objectsize.i64.p0(ptr %dst, i1 false, i1 true, i1 false)
+  %call.memmove = call ptr @__memmove_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  ret void
+}
+
+define void @callee_mempcpy(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @callee_mempcpy
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DST]], ptr align 1 [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  %objsize = call i64 @llvm.objectsize.i64.p0(ptr %dst, i1 false, i1 true, i1 false)
+  %call.mempcpy = call ptr @__mempcpy_chk(ptr %dst, ptr %src, i64 %size, i64 %objsize)
+  ret void
+}
+
+define void @callee_memset(ptr %dst, i64 %size) {
+; CHECK-LABEL: define void @callee_memset
+; CHECK-SAME: (ptr [[DST:%.*]], i64 [[SIZE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT:    tail call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 0, i64 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  %objsize = call i64 @llvm.objectsize.i64.p0(ptr %dst, i1 false, i1 true, i1 false)
+  %call.mempcpy = call ptr @__memset_chk(ptr %dst, i32 0, i64 %size, i64 %objsize)
+  ret void
+}
+
+define void @caller_memcpy(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @caller_memcpy
+; CHECK-SAME: (ptr [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @callee_memcpy(ptr %dst, ptr %src, i64 4)
+  ret void
+}
+
+define void @caller_memmove(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @caller_memmove
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @callee_memmove(ptr %dst, ptr %src, i64 4)
+  ret void
+}
+
+define void @caller_mempcpy(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @caller_mempcpy
+; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) local_unnamed_addr #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @callee_mempcpy(ptr %dst, ptr %src, i64 4)
+  ret void
+}
+
+define void @caller_memset(ptr %dst) {
+; CHECK-LABEL: define void @caller_memset
+; CHECK-SAME: (ptr [[DST:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @callee_memset(ptr %dst, i64 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
index 97d2247ab13c1..68b64a5dad4fa 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
@@ -12,51 +12,51 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
 ; CHECK:       [[FOR_BODY_PREHEADER]]:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[RAND_BLOCK_LENGTH]] to i64
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[RAND_BLOCK_LENGTH]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[RAND_BLOCK_LENGTH]], 8
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER23:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Y]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <2 x double> poison, double [[Z]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT20:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT19]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <4 x double> poison, double [[Z]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT20:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT19]], <4 x double> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x double> [ <double 0.000000e+00, double -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI15:%.*]] = phi <2 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI16:%.*]] = phi <2 x double> [ <double 0.000000e+00, double -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI17:%.*]] = phi <2 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x double> [ <double 0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI15:%.*]] = phi <4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI16:%.*]] = phi <4 x double> [ <double 0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI17:%.*]] = phi <4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV1]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX1]], i64 8
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <2 x float>, ptr [[TMP23]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fpext <2 x float> [[WIDE_LOAD]] to <2 x double>
-; CHECK-NEXT:    [[TMP3:%.*]] = fpext <2 x float> [[WIDE_LOAD18]] to <2 x double>
-; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> [[BROADCAST_SPLAT]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <2 x double> [[BROADCAST_SPLAT]], [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = fsub fast <2 x double> [[TMP4]], [[BROADCAST_SPLAT20]]
-; CHECK-NEXT:    [[TMP7:%.*]] = fsub fast <2 x double> [[TMP5]], [[BROADCAST_SPLAT20]]
-; CHECK-NEXT:    [[TMP8:%.*]] = fcmp fast ogt <2 x double> [[TMP6]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt <2 x double> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast <2 x double> [[TMP6]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast <2 x double> [[TMP7]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = tail call fast <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP6]], <2 x double> splat (double -0.000000e+00))
-; CHECK-NEXT:    [[TMP13:%.*]] = tail call fast <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP7]], <2 x double> splat (double -0.000000e+00))
-; CHECK-NEXT:    [[TMP14]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI16]], [[TMP12]]
-; CHECK-NEXT:    [[TMP15]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI17]], [[TMP13]]
-; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP8]], <2 x double> [[TMP10]], <2 x double> splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[TMP9]], <2 x double> [[TMP11]], <2 x double> splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP18]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI]], [[TMP16]]
-; CHECK-NEXT:    [[TMP19]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI15]], [[TMP17]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV1]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX1]], i64 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <4 x float>, ptr [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext <4 x float> [[WIDE_LOAD]] to <4 x double>
+; CHECK-NEXT:    [[TMP3:%.*]] = fpext <4 x float> [[WIDE_LOAD18]] to <4 x double>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x double> [[BROADCAST_SPLAT]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fsub fast <4 x double> [[TMP4]], [[BROADCAST_SPLAT20]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fsub fast <4 x double> [[TMP5]], [[BROADCAST_SPLAT20]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fcmp fast ogt <4 x double> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt <4 x double> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast <4 x double> [[TMP6]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast <4 x double> [[TMP7]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call fast <4 x double> @llvm.maxnum.v4f64(<4 x double> [[TMP6]], <4 x double> splat (double -0.000000e+00))
+; CHECK-NEXT:    [[TMP13:%.*]] = tail call fast <4 x double> @llvm.maxnum.v4f64(<4 x double> [[TMP7]], <4 x double> splat (double -0.000000e+00))
+; CHECK-NEXT:    [[TMP14]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI16]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI17]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[TMP8]], <4 x double> [[TMP10]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[TMP9]], <4 x double> [[TMP11]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP18]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI15]], [[TMP17]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV1]], 8
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc arcp contract afn <2 x double> [[TMP19]], [[TMP18]]
-; CHECK-NEXT:    [[TMP21:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[BIN_RDX]])
-; CHECK-NEXT:    [[BIN_RDX21:%.*]] = fadd reassoc arcp contract afn <2 x double> [[TMP15]], [[TMP14]]
-; CHECK-NEXT:    [[TMP22:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[BIN_RDX21]])
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc arcp contract afn <4 x double> [[TMP19]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[BIN_RDX]])
+; CHECK-NEXT:    [[BIN_RDX21:%.*]] = fadd reassoc arcp contract afn <4 x double> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP22:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[BIN_RDX21]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY_PREHEADER23]]
 ; CHECK:       [[FOR_BODY_PREHEADER23]]:
@@ -68,7 +68,7 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER23]] ]
 ; CHECK-NEXT:    [[V1_012:%.*]] = phi double [ [[V1_2:%.*]], %[[FOR_BODY]] ], [ [[V1_012_PH]], %[[FOR_BODY_PREHEADER23]] ]
 ; CHECK-NEXT:    [[V0_011:%.*]] = phi double [ [[V0_2:%.*]], %[[FOR_BODY]] ], [ [[V0_011_PH]], %[[FOR_BODY_PREHEADER23]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP0]] to double
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[Y]], [[CONV]]
@@ -189,12 +189,12 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    br i1 [[CMP211]], label %[[FOR_BODY_US_PREHEADER:.*]], label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY_US_PREHEADER]]:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[RAND_BLOCK_LENGTH]] to i64
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[RAND_BLOCK_LENGTH]], 4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Y]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT35:%.*]] = insertelement <2 x double> poison, double [[Z]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT36:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT35]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[RAND_BLOCK_LENGTH]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT35:%.*]] = insertelement <4 x double> poison, double [[Z]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT36:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT35]], <4 x double> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br label %[[FOR_BODY_US:.*]]
 ; CHECK:       [[FOR_BODY_US]]:
@@ -204,47 +204,47 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    tail call void @resample(i32 noundef [[RAND_BLOCK_LENGTH]], ptr noundef [[SAMPLES]])
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY3_US_PREHEADER:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> <double poison, double -0.000000e+00>, double [[V1_021_US]], i64 0
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <2 x double> <double poison, double -0.000000e+00>, double [[V0_020_US]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, double [[V1_021_US]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, double [[V0_020_US]], i64 0
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x double> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI31:%.*]] = phi <2 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI32:%.*]] = phi <2 x double> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI33:%.*]] = phi <2 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x double> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI31:%.*]] = phi <4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI32:%.*]] = phi <4 x double> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI33:%.*]] = phi <4 x double> [ splat (double -0.000000e+00), %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_US1:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX_US1]], i64 8
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[ARRAYIDX_US1]], align 4
-; CHECK-NEXT:    [[WIDE_LOAD34:%.*]] = load <2 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = fpext <2 x float> [[WIDE_LOAD]] to <2 x double>
-; CHECK-NEXT:    [[TMP5:%.*]] = fpext <2 x float> [[WIDE_LOAD34]] to <2 x double>
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call fast <2 x double> @llvm.exp2.v2f64(<2 x double> [[TMP4]])
-; CHECK-NEXT:    [[TMP7:%.*]] = tail call fast <2 x double> @llvm.exp2.v2f64(<2 x double> [[TMP5]])
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <2 x double> [[TMP6]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <2 x double> [[TMP7]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast <2 x double> [[TMP8]], [[BROADCAST_SPLAT36]]
-; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast <2 x double> [[TMP9]], [[BROADCAST_SPLAT36]]
-; CHECK-NEXT:    [[TMP12:%.*]] = fcmp fast ogt <2 x double> [[TMP10]], zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt <2 x double> [[TMP11]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = fmul fast <2 x double> [[TMP10]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <2 x double> [[TMP11]], [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = tail call fast <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP10]], <2 x double> splat (double -0.000000e+00))
-; CHECK-NEXT:    [[TMP17:%.*]] = tail call fast <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP11]], <2 x double> splat (double -0.000000e+00))
-; CHECK-NEXT:    [[TMP18]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI32]], [[TMP16]]
-; CHECK-NEXT:    [[TMP19]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI33]], [[TMP17]]
-; CHECK-NEXT:    [[TMP20:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP14]], <2 x double> splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP21:%.*]] = select <2 x i1> [[TMP13]], <2 x double> [[TMP15]], <2 x double> splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP22]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI]], [[TMP20]]
-; CHECK-NEXT:    [[TMP23]] = fadd reassoc arcp contract afn <2 x double> [[VEC_PHI31]], [[TMP21]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV1]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX_US1]], i64 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX_US1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD34:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fpext <4 x float> [[WIDE_LOAD]] to <4 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = fpext <4 x float> [[WIDE_LOAD34]] to <4 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call fast <4 x double> @llvm.exp2.v4f64(<4 x double> [[TMP4]])
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call fast <4 x double> @llvm.exp2.v4f64(<4 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <4 x double> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <4 x double> [[TMP7]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fsub fast <4 x double> [[TMP8]], [[BROADCAST_SPLAT36]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fsub fast <4 x double> [[TMP9]], [[BROADCAST_SPLAT36]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fcmp fast ogt <4 x double> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt <4 x double> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul fast <4 x double> [[TMP10]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <4 x double> [[TMP11]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = tail call fast <4 x double> @llvm.maxnum.v4f64(<4 x double> [[TMP10]], <4 x double> splat (double -0.000000e+00))
+; CHECK-NEXT:    [[TMP17:%.*]] = tail call fast <4 x double> @llvm.maxnum.v4f64(<4 x double> [[TMP11]], <4 x double> splat (double -0.000000e+00))
+; CHECK-NEXT:    [[TMP18]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI32]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI33]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <4 x i1> [[TMP12]], <4 x double> [[TMP14]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP21:%.*]] = select <4 x i1> [[TMP13]], <4 x double> [[TMP15]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP22]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI31]], [[TMP21]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV1]], 8
 ; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc arcp contract afn <2 x double> [[TMP23]], [[TMP22]]
-; CHECK-NEXT:    [[TMP25:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[BIN_RDX]])
-; CHECK-NEXT:    [[BIN_RDX37:%.*]] = fadd reassoc arcp contract afn <2 x double> [[TMP19]], [[TMP18]]
-; CHECK-NEXT:    [[TMP26:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[BIN_RDX37]])
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc arcp contract afn <4 x double> [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[BIN_RDX]])
+; CHECK-NEXT:    [[BIN_RDX37:%.*]] = fadd reassoc arcp contract afn <4 x double> [[TMP19]], [[TMP18]]
+; CHECK-NEXT:    [[TMP26:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[BIN_RDX37]])
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]], label %[[FOR_BODY3_US_PREHEADER]]
 ; CHECK:       [[FOR_BODY3_US_PREHEADER]]:
 ; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_US]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
@@ -255,7 +255,7 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY3_US]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[V1_116_US:%.*]] = phi double [ [[V1_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V1_116_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[V0_115_US:%.*]] = phi double [ [[V0_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V0_115_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
-; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds nuw float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX_US]], align 4
 ; CHECK-NEXT:    [[CONV_US:%.*]] = fpext float [[TMP0]] to double
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call fast double @llvm.exp2.f64(double [[CONV_US]])
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
index eba0ed2d35fe4..eaced404bba28 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
@@ -22,19 +22,19 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT27]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 16
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[DCT]], align 2, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD29:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !alias.scope [[META0]], !noalias [[META3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i16> [[WIDE_LOAD29]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <8 x i16> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt <8 x i16> [[WIDE_LOAD29]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 16
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD30:%.*]] = load <8 x i16>, ptr [[BIAS]], align 2, !alias.scope [[META6:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD31:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !alias.scope [[META6]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext <8 x i16> [[WIDE_LOAD30]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext <8 x i16> [[WIDE_LOAD31]] to <8 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 16
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD32:%.*]] = load <8 x i16>, ptr [[MF]], align 2, !alias.scope [[META7:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD33:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !alias.scope [[META7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = zext <8 x i16> [[WIDE_LOAD32]] to <8 x i32>
@@ -95,14 +95,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i16 [ [[CONV28]], [[IF_ELSE]] ], [ [[CONV12]], [[IF_THEN]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE]], ptr [[DCT]], align 2
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 2
 ; CHECK-NEXT:    [[TMP34:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
 ; CHECK-NEXT:    [[CONV_1:%.*]] = sext i16 [[TMP34]] to i32
 ; CHECK-NEXT:    [[CMP1_1:%.*]] = icmp sgt i16 [[TMP34]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_1:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX4_1:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 2
 ; CHECK-NEXT:    [[TMP35:%.*]] = load i16, ptr [[ARRAYIDX4_1]], align 2
 ; CHECK-NEXT:    [[CONV5_1:%.*]] = zext i16 [[TMP35]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 2
 ; CHECK-NEXT:    [[TMP36:%.*]] = load i16, ptr [[ARRAYIDX10_1]], align 2
 ; CHECK-NEXT:    [[CONV11_1:%.*]] = zext i16 [[TMP36]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[IF_ELSE_1:%.*]]
@@ -123,14 +123,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_1:%.*]] = phi i16 [ [[CONV28_1]], [[IF_ELSE_1]] ], [ [[CONV12_1]], [[IF_THEN_1]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_1]], ptr [[ARRAYIDX_1]], align 2
 ; CHECK-NEXT:    [[OR_137:%.*]] = or i16 [[STOREMERGE]], [[STOREMERGE_1]]
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 4
 ; CHECK-NEXT:    [[TMP38:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
 ; CHECK-NEXT:    [[CONV_2:%.*]] = sext i16 [[TMP38]] to i32
 ; CHECK-NEXT:    [[CMP1_2:%.*]] = icmp sgt i16 [[TMP38]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_2:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX4_2:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 4
 ; CHECK-NEXT:    [[TMP39:%.*]] = load i16, ptr [[ARRAYIDX4_2]], align 2
 ; CHECK-NEXT:    [[CONV5_2:%.*]] = zext i16 [[TMP39]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 4
 ; CHECK-NEXT:    [[TMP40:%.*]] = load i16, ptr [[ARRAYIDX10_2]], align 2
 ; CHECK-NEXT:    [[CONV11_2:%.*]] = zext i16 [[TMP40]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[IF_ELSE_2:%.*]]
@@ -151,14 +151,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_2:%.*]] = phi i16 [ [[CONV28_2]], [[IF_ELSE_2]] ], [ [[CONV12_2]], [[IF_THEN_2]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_2]], ptr [[ARRAYIDX_2]], align 2
 ; CHECK-NEXT:    [[OR_238:%.*]] = or i16 [[OR_137]], [[STOREMERGE_2]]
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 6
 ; CHECK-NEXT:    [[TMP42:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
 ; CHECK-NEXT:    [[CONV_3:%.*]] = sext i16 [[TMP42]] to i32
 ; CHECK-NEXT:    [[CMP1_3:%.*]] = icmp sgt i16 [[TMP42]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_3:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX4_3:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 6
 ; CHECK-NEXT:    [[TMP43:%.*]] = load i16, ptr [[ARRAYIDX4_3]], align 2
 ; CHECK-NEXT:    [[CONV5_3:%.*]] = zext i16 [[TMP43]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 6
 ; CHECK-NEXT:    [[TMP44:%.*]] = load i16, ptr [[ARRAYIDX10_3]], align 2
 ; CHECK-NEXT:    [[CONV11_3:%.*]] = zext i16 [[TMP44]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[IF_ELSE_3:%.*]]
@@ -179,14 +179,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_3:%.*]] = phi i16 [ [[CONV28_3]], [[IF_ELSE_3]] ], [ [[CONV12_3]], [[IF_THEN_3]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_3]], ptr [[ARRAYIDX_3]], align 2
 ; CHECK-NEXT:    [[OR_339:%.*]] = or i16 [[OR_238]], [[STOREMERGE_3]]
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 8
 ; CHECK-NEXT:    [[TMP46:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
 ; CHECK-NEXT:    [[CONV_4:%.*]] = sext i16 [[TMP46]] to i32
 ; CHECK-NEXT:    [[CMP1_4:%.*]] = icmp sgt i16 [[TMP46]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_4:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX4_4:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 8
 ; CHECK-NEXT:    [[TMP47:%.*]] = load i16, ptr [[ARRAYIDX4_4]], align 2
 ; CHECK-NEXT:    [[CONV5_4:%.*]] = zext i16 [[TMP47]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_4:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX10_4:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 8
 ; CHECK-NEXT:    [[TMP48:%.*]] = load i16, ptr [[ARRAYIDX10_4]], align 2
 ; CHECK-NEXT:    [[CONV11_4:%.*]] = zext i16 [[TMP48]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_4]], label [[IF_THEN_4:%.*]], label [[IF_ELSE_4:%.*]]
@@ -207,14 +207,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_4:%.*]] = phi i16 [ [[CONV28_4]], [[IF_ELSE_4]] ], [ [[CONV12_4]], [[IF_THEN_4]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_4]], ptr [[ARRAYIDX_4]], align 2
 ; CHECK-NEXT:    [[OR_440:%.*]] = or i16 [[OR_339]], [[STOREMERGE_4]]
-; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 10
 ; CHECK-NEXT:    [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
 ; CHECK-NEXT:    [[CONV_5:%.*]] = sext i16 [[TMP50]] to i32
 ; CHECK-NEXT:    [[CMP1_5:%.*]] = icmp sgt i16 [[TMP50]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_5:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX4_5:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 10
 ; CHECK-NEXT:    [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX4_5]], align 2
 ; CHECK-NEXT:    [[CONV5_5:%.*]] = zext i16 [[TMP51]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_5:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX10_5:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 10
 ; CHECK-NEXT:    [[TMP52:%.*]] = load i16, ptr [[ARRAYIDX10_5]], align 2
 ; CHECK-NEXT:    [[CONV11_5:%.*]] = zext i16 [[TMP52]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_5]], label [[IF_THEN_5:%.*]], label [[IF_ELSE_5:%.*]]
@@ -235,14 +235,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_5:%.*]] = phi i16 [ [[CONV28_5]], [[IF_ELSE_5]] ], [ [[CONV12_5]], [[IF_THEN_5]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_5]], ptr [[ARRAYIDX_5]], align 2
 ; CHECK-NEXT:    [[OR_541:%.*]] = or i16 [[OR_440]], [[STOREMERGE_5]]
-; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 12
 ; CHECK-NEXT:    [[TMP54:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
 ; CHECK-NEXT:    [[CONV_6:%.*]] = sext i16 [[TMP54]] to i32
 ; CHECK-NEXT:    [[CMP1_6:%.*]] = icmp sgt i16 [[TMP54]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_6:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX4_6:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 12
 ; CHECK-NEXT:    [[TMP55:%.*]] = load i16, ptr [[ARRAYIDX4_6]], align 2
 ; CHECK-NEXT:    [[CONV5_6:%.*]] = zext i16 [[TMP55]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_6:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX10_6:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 12
 ; CHECK-NEXT:    [[TMP56:%.*]] = load i16, ptr [[ARRAYIDX10_6]], align 2
 ; CHECK-NEXT:    [[CONV11_6:%.*]] = zext i16 [[TMP56]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_6]], label [[IF_THEN_6:%.*]], label [[IF_ELSE_6:%.*]]
@@ -263,14 +263,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_6:%.*]] = phi i16 [ [[CONV28_6]], [[IF_ELSE_6]] ], [ [[CONV12_6]], [[IF_THEN_6]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_6]], ptr [[ARRAYIDX_6]], align 2
 ; CHECK-NEXT:    [[OR_642:%.*]] = or i16 [[OR_541]], [[STOREMERGE_6]]
-; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 14
 ; CHECK-NEXT:    [[TMP58:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
 ; CHECK-NEXT:    [[CONV_7:%.*]] = sext i16 [[TMP58]] to i32
 ; CHECK-NEXT:    [[CMP1_7:%.*]] = icmp sgt i16 [[TMP58]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_7:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX4_7:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 14
 ; CHECK-NEXT:    [[TMP59:%.*]] = load i16, ptr [[ARRAYIDX4_7]], align 2
 ; CHECK-NEXT:    [[CONV5_7:%.*]] = zext i16 [[TMP59]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_7:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX10_7:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 14
 ; CHECK-NEXT:    [[TMP60:%.*]] = load i16, ptr [[ARRAYIDX10_7]], align 2
 ; CHECK-NEXT:    [[CONV11_7:%.*]] = zext i16 [[TMP60]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_7]], label [[IF_THEN_7:%.*]], label [[IF_ELSE_7:%.*]]
@@ -291,14 +291,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_7:%.*]] = phi i16 [ [[CONV28_7]], [[IF_ELSE_7]] ], [ [[CONV12_7]], [[IF_THEN_7]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_7]], ptr [[ARRAYIDX_7]], align 2
 ; CHECK-NEXT:    [[OR_743:%.*]] = or i16 [[OR_642]], [[STOREMERGE_7]]
-; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 16
 ; CHECK-NEXT:    [[TMP62:%.*]] = load i16, ptr [[ARRAYIDX_8]], align 2
 ; CHECK-NEXT:    [[CONV_8:%.*]] = sext i16 [[TMP62]] to i32
 ; CHECK-NEXT:    [[CMP1_8:%.*]] = icmp sgt i16 [[TMP62]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_8:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX4_8:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 16
 ; CHECK-NEXT:    [[TMP63:%.*]] = load i16, ptr [[ARRAYIDX4_8]], align 2
 ; CHECK-NEXT:    [[CONV5_8:%.*]] = zext i16 [[TMP63]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_8:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX10_8:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 16
 ; CHECK-NEXT:    [[TMP64:%.*]] = load i16, ptr [[ARRAYIDX10_8]], align 2
 ; CHECK-NEXT:    [[CONV11_8:%.*]] = zext i16 [[TMP64]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_8]], label [[IF_THEN_8:%.*]], label [[IF_ELSE_8:%.*]]
@@ -319,14 +319,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_8:%.*]] = phi i16 [ [[CONV28_8]], [[IF_ELSE_8]] ], [ [[CONV12_8]], [[IF_THEN_8]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_8]], ptr [[ARRAYIDX_8]], align 2
 ; CHECK-NEXT:    [[OR_844:%.*]] = or i16 [[OR_743]], [[STOREMERGE_8]]
-; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 18
 ; CHECK-NEXT:    [[TMP66:%.*]] = load i16, ptr [[ARRAYIDX_9]], align 2
 ; CHECK-NEXT:    [[CONV_9:%.*]] = sext i16 [[TMP66]] to i32
 ; CHECK-NEXT:    [[CMP1_9:%.*]] = icmp sgt i16 [[TMP66]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_9:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX4_9:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 18
 ; CHECK-NEXT:    [[TMP67:%.*]] = load i16, ptr [[ARRAYIDX4_9]], align 2
 ; CHECK-NEXT:    [[CONV5_9:%.*]] = zext i16 [[TMP67]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_9:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX10_9:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 18
 ; CHECK-NEXT:    [[TMP68:%.*]] = load i16, ptr [[ARRAYIDX10_9]], align 2
 ; CHECK-NEXT:    [[CONV11_9:%.*]] = zext i16 [[TMP68]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_9]], label [[IF_THEN_9:%.*]], label [[IF_ELSE_9:%.*]]
@@ -347,14 +347,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_9:%.*]] = phi i16 [ [[CONV28_9]], [[IF_ELSE_9]] ], [ [[CONV12_9]], [[IF_THEN_9]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_9]], ptr [[ARRAYIDX_9]], align 2
 ; CHECK-NEXT:    [[OR_945:%.*]] = or i16 [[OR_844]], [[STOREMERGE_9]]
-; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 20
 ; CHECK-NEXT:    [[TMP70:%.*]] = load i16, ptr [[ARRAYIDX_10]], align 2
 ; CHECK-NEXT:    [[CONV_10:%.*]] = sext i16 [[TMP70]] to i32
 ; CHECK-NEXT:    [[CMP1_10:%.*]] = icmp sgt i16 [[TMP70]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_10:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX4_10:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 20
 ; CHECK-NEXT:    [[TMP71:%.*]] = load i16, ptr [[ARRAYIDX4_10]], align 2
 ; CHECK-NEXT:    [[CONV5_10:%.*]] = zext i16 [[TMP71]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_10:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX10_10:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 20
 ; CHECK-NEXT:    [[TMP72:%.*]] = load i16, ptr [[ARRAYIDX10_10]], align 2
 ; CHECK-NEXT:    [[CONV11_10:%.*]] = zext i16 [[TMP72]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_10]], label [[IF_THEN_10:%.*]], label [[IF_ELSE_10:%.*]]
@@ -375,14 +375,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_10:%.*]] = phi i16 [ [[CONV28_10]], [[IF_ELSE_10]] ], [ [[CONV12_10]], [[IF_THEN_10]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_10]], ptr [[ARRAYIDX_10]], align 2
 ; CHECK-NEXT:    [[OR_1046:%.*]] = or i16 [[OR_945]], [[STOREMERGE_10]]
-; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 22
 ; CHECK-NEXT:    [[TMP74:%.*]] = load i16, ptr [[ARRAYIDX_11]], align 2
 ; CHECK-NEXT:    [[CONV_11:%.*]] = sext i16 [[TMP74]] to i32
 ; CHECK-NEXT:    [[CMP1_11:%.*]] = icmp sgt i16 [[TMP74]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_11:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX4_11:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 22
 ; CHECK-NEXT:    [[TMP75:%.*]] = load i16, ptr [[ARRAYIDX4_11]], align 2
 ; CHECK-NEXT:    [[CONV5_11:%.*]] = zext i16 [[TMP75]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_11:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX10_11:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 22
 ; CHECK-NEXT:    [[TMP76:%.*]] = load i16, ptr [[ARRAYIDX10_11]], align 2
 ; CHECK-NEXT:    [[CONV11_11:%.*]] = zext i16 [[TMP76]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_11]], label [[IF_THEN_11:%.*]], label [[IF_ELSE_11:%.*]]
@@ -403,14 +403,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_11:%.*]] = phi i16 [ [[CONV28_11]], [[IF_ELSE_11]] ], [ [[CONV12_11]], [[IF_THEN_11]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_11]], ptr [[ARRAYIDX_11]], align 2
 ; CHECK-NEXT:    [[OR_1147:%.*]] = or i16 [[OR_1046]], [[STOREMERGE_11]]
-; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 24
 ; CHECK-NEXT:    [[TMP78:%.*]] = load i16, ptr [[ARRAYIDX_12]], align 2
 ; CHECK-NEXT:    [[CONV_12:%.*]] = sext i16 [[TMP78]] to i32
 ; CHECK-NEXT:    [[CMP1_12:%.*]] = icmp sgt i16 [[TMP78]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_12:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX4_12:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 24
 ; CHECK-NEXT:    [[TMP79:%.*]] = load i16, ptr [[ARRAYIDX4_12]], align 2
 ; CHECK-NEXT:    [[CONV5_12:%.*]] = zext i16 [[TMP79]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_12:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX10_12:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 24
 ; CHECK-NEXT:    [[TMP80:%.*]] = load i16, ptr [[ARRAYIDX10_12]], align 2
 ; CHECK-NEXT:    [[CONV11_12:%.*]] = zext i16 [[TMP80]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_12]], label [[IF_THEN_12:%.*]], label [[IF_ELSE_12:%.*]]
@@ -431,14 +431,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_12:%.*]] = phi i16 [ [[CONV28_12]], [[IF_ELSE_12]] ], [ [[CONV12_12]], [[IF_THEN_12]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_12]], ptr [[ARRAYIDX_12]], align 2
 ; CHECK-NEXT:    [[OR_1248:%.*]] = or i16 [[OR_1147]], [[STOREMERGE_12]]
-; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 26
 ; CHECK-NEXT:    [[TMP82:%.*]] = load i16, ptr [[ARRAYIDX_13]], align 2
 ; CHECK-NEXT:    [[CONV_13:%.*]] = sext i16 [[TMP82]] to i32
 ; CHECK-NEXT:    [[CMP1_13:%.*]] = icmp sgt i16 [[TMP82]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_13:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX4_13:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 26
 ; CHECK-NEXT:    [[TMP83:%.*]] = load i16, ptr [[ARRAYIDX4_13]], align 2
 ; CHECK-NEXT:    [[CONV5_13:%.*]] = zext i16 [[TMP83]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_13:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX10_13:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 26
 ; CHECK-NEXT:    [[TMP84:%.*]] = load i16, ptr [[ARRAYIDX10_13]], align 2
 ; CHECK-NEXT:    [[CONV11_13:%.*]] = zext i16 [[TMP84]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_13]], label [[IF_THEN_13:%.*]], label [[IF_ELSE_13:%.*]]
@@ -459,14 +459,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_13:%.*]] = phi i16 [ [[CONV28_13]], [[IF_ELSE_13]] ], [ [[CONV12_13]], [[IF_THEN_13]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_13]], ptr [[ARRAYIDX_13]], align 2
 ; CHECK-NEXT:    [[OR_1349:%.*]] = or i16 [[OR_1248]], [[STOREMERGE_13]]
-; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 28
 ; CHECK-NEXT:    [[TMP86:%.*]] = load i16, ptr [[ARRAYIDX_14]], align 2
 ; CHECK-NEXT:    [[CONV_14:%.*]] = sext i16 [[TMP86]] to i32
 ; CHECK-NEXT:    [[CMP1_14:%.*]] = icmp sgt i16 [[TMP86]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_14:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX4_14:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 28
 ; CHECK-NEXT:    [[TMP87:%.*]] = load i16, ptr [[ARRAYIDX4_14]], align 2
 ; CHECK-NEXT:    [[CONV5_14:%.*]] = zext i16 [[TMP87]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_14:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX10_14:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 28
 ; CHECK-NEXT:    [[TMP88:%.*]] = load i16, ptr [[ARRAYIDX10_14]], align 2
 ; CHECK-NEXT:    [[CONV11_14:%.*]] = zext i16 [[TMP88]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_14]], label [[IF_THEN_14:%.*]], label [[IF_ELSE_14:%.*]]
@@ -487,14 +487,14 @@ define i32 @quant_4x4(ptr noundef %dct, ptr noundef %mf, ptr noundef %bias) {
 ; CHECK-NEXT:    [[STOREMERGE_14:%.*]] = phi i16 [ [[CONV28_14]], [[IF_ELSE_14]] ], [ [[CONV12_14]], [[IF_THEN_14]] ]
 ; CHECK-NEXT:    store i16 [[STOREMERGE_14]], ptr [[ARRAYIDX_14]], align 2
 ; CHECK-NEXT:    [[OR_1450:%.*]] = or i16 [[OR_1349]], [[STOREMERGE_14]]
-; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds i8, ptr [[DCT]], i64 30
+; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds nuw i8, ptr [[DCT]], i64 30
 ; CHECK-NEXT:    [[TMP90:%.*]] = load i16, ptr [[ARRAYIDX_15]], align 2
 ; CHECK-NEXT:    [[CONV_15:%.*]] = sext i16 [[TMP90]] to i32
 ; CHECK-NEXT:    [[CMP1_15:%.*]] = icmp sgt i16 [[TMP90]], 0
-; CHECK-NEXT:    [[ARRAYIDX4_15:%.*]] = getelementptr inbounds i8, ptr [[BIAS]], i64 30
+; CHECK-NEXT:    [[ARRAYIDX4_15:%.*]] = getelementptr inbounds nuw i8, ptr [[BIAS]], i64 30
 ; CHECK-NEXT:    [[TMP91:%.*]] = load i16, ptr [[ARRAYIDX4_15]], align 2
 ; CHECK-NEXT:    [[CONV5_15:%.*]] = zext i16 [[TMP91]] to i32
-; CHECK-NEXT:    [[ARRAYIDX10_15:%.*]] = getelementptr inbounds i8, ptr [[MF]], i64 30
+; CHECK-NEXT:    [[ARRAYIDX10_15:%.*]] = getelementptr inbounds nuw i8, ptr [[MF]], i64 30
 ; CHECK-NEXT:    [[TMP92:%.*]] = load i16, ptr [[ARRAYIDX10_15]], align 2
 ; CHECK-NEXT:    [[CONV11_15:%.*]] = zext i16 [[TMP92]] to i32
 ; CHECK-NEXT:    br i1 [[CMP1_15]], label [[IF_THEN_15:%.*]], label [[IF_ELSE_15:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
index 885529926fc4b..600e30f6853e6 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/sinking-vs-if-conversion.ll
@@ -12,27 +12,27 @@ define void @test_find_min(ptr noundef nonnull align 8 dereferenceable(24) %this
 ; CHECK-LABEL: define void @test_find_min(
 ; CHECK-SAME: ptr nocapture noundef nonnull align 8 dereferenceable(24) [[THIS:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[NUM_:%.*]] = getelementptr inbounds i8, ptr [[THIS]], i64 16
+; CHECK-NEXT:    [[NUM_:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[NUM_]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[THIS]], align 8
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body.lr.ph:
-; CHECK-NEXT:    [[ARRAY_:%.*]] = getelementptr inbounds i8, ptr [[THIS]], i64 8
+; CHECK-NEXT:    [[ARRAY_:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAY_]], align 8
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[TMP0]] to i64
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[COND_END7:%.*]] ]
 ; CHECK-NEXT:    [[MIN_010:%.*]] = phi ptr [ [[TMP1]], [[FOR_BODY_LR_PH]] ], [ [[COND8:%.*]], [[COND_END7]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw ptr, ptr [[TMP2]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq ptr [[MIN_010]], null
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[COND_END7]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[KEY2:%.*]] = getelementptr inbounds i8, ptr [[MIN_010]], i64 4
+; CHECK-NEXT:    [[KEY2:%.*]] = getelementptr inbounds nuw i8, ptr [[MIN_010]], i64 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[KEY2]], align 4
-; CHECK-NEXT:    [[KEY:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 4
+; CHECK-NEXT:    [[KEY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[KEY]], align 4
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP4]], ptr [[TMP3]], ptr [[MIN_010]]
@@ -143,17 +143,17 @@ define void @cond_select_loop(ptr noalias nocapture noundef readonly %a, ptr noa
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_07:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[COND_END:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I_07]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I_07]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[TMP0]], 0.000000e+00
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_END]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I_07]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[I_07]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    br label [[COND_END]]
 ; CHECK:       cond.end:
 ; CHECK-NEXT:    [[COND:%.*]] = phi float [ [[TMP1]], [[COND_FALSE]] ], [ [[TMP0]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[I_07]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[C]], i64 [[I_07]]
 ; CHECK-NEXT:    store float [[COND]], ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 1000
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
index 8d31a7d25ff3d..1cf20cb1fd64d 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
@@ -16,20 +16,20 @@ define i32 @slpordering(ptr noundef %p1, i32 noundef %ip1, ptr noundef %p2, i32
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[IP1]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[IP2]] to i64
-; CHECK-NEXT:    [[RRRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 4
-; CHECK-NEXT:    [[RRRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2]], i64 4
 ; CHECK-NEXT:    [[RDD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[RDD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[RRRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR]], i64 4
-; CHECK-NEXT:    [[RRRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR64]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR64]], i64 4
 ; CHECK-NEXT:    [[RDD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[RDD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR64]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[RRRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR_1]], i64 4
-; CHECK-NEXT:    [[RRRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX3_2:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR_1]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX5_2:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR64_1]], i64 4
 ; CHECK-NEXT:    [[RDD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR_1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[RDD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR64_1]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[RRRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR_2]], i64 4
-; CHECK-NEXT:    [[RRRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[RDD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX3_3:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR_2]], i64 4
+; CHECK-NEXT:    [[RRRAYIDX5_3:%.*]] = getelementptr inbounds nuw i8, ptr [[RDD_PTR64_2]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[RRRAYIDX3]], align 1, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
index 64e1076da44dd..aab787b2b2d2c 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
@@ -23,7 +23,7 @@ define void @arm_mean_q7(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[TMP1]], i32 0)
 ; CHECK-NEXT:    [[TMP3]] = add i32 [[TMP2]], [[SUM_013]]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[BLKCNT_011]], -1
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, ptr [[PSRC_ADDR_012]], i32 16
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_012]], i32 16
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
 ; CHECK:       while.end.loopexit:
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
index 595b0f76a418b..ef651010d6a90 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
@@ -60,17 +60,17 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER18]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER18]] ]
 ; CHECK-NEXT:    [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER18]] ]
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRCA_ADDR_05]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRCA_ADDR_05]], i32 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP10]] to i32
-; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[PSRCB_ADDR_03]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[PSRCB_ADDR_03]], i32 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2
 ; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
 ; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[MUL]], 15
 ; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = tail call i32 @llvm.smin.i32(i32 [[SHR]], i32 32767)
 ; CHECK-NEXT:    [[CONV3:%.*]] = trunc nsw i32 [[SPEC_SELECT_I]] to i16
-; CHECK-NEXT:    [[INCDEC_PTR4]] = getelementptr inbounds i8, ptr [[PDST_ADDR_04]], i32 2
+; CHECK-NEXT:    [[INCDEC_PTR4]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_04]], i32 2
 ; CHECK-NEXT:    store i16 [[CONV3]], ptr [[PDST_ADDR_04]], align 2
 ; CHECK-NEXT:    [[DEC]] = add i32 [[BLKCNT_06]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
diff --git a/llvm/test/Transforms/PhaseOrdering/SystemZ/sub-xor.ll b/llvm/test/Transforms/PhaseOrdering/SystemZ/sub-xor.ll
index 43fd8bd59b8d3..5386bf939918a 100644
--- a/llvm/test/Transforms/PhaseOrdering/SystemZ/sub-xor.ll
+++ b/llvm/test/Transforms/PhaseOrdering/SystemZ/sub-xor.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='default<O2>' -enable-matrix -S %s | FileCheck %s
+; RUN: opt -passes='default<O2>' -S %s | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "systemz"
@@ -20,35 +20,35 @@ define dso_local zeroext i32 @foo(ptr noundef %a) #0 {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY4]] ]
 ; CHECK-NEXT:    [[SUM_11:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_7:%.*]], [[FOR_BODY4]] ]
 ; CHECK-NEXT:    [[IDX_NEG:%.*]] = sub nsw i64 0, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[IDX_NEG]]
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[IDX_NEG]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ADD_PTR]], align 4, !tbaa [[TBAA3:![0-9]+]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP0]], [[SUM_11]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_NEG:%.*]] = xor i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[ADD_PTR_110:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_110:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_NEG]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ADD_PTR_110]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_111:%.*]] = add i32 [[TMP1]], [[ADD]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_112_NEG:%.*]] = sub nuw nsw i64 -2, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_217:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_112_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_217:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_112_NEG]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ADD_PTR_217]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_218:%.*]] = add i32 [[TMP2]], [[ADD_111]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_219_NEG:%.*]] = sub nuw nsw i64 -3, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_219_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_219_NEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ADD_PTR_3]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 [[TMP3]], [[ADD_218]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_3_NEG:%.*]] = sub nuw nsw i64 -4, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_3_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_3_NEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ADD_PTR_4]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 [[TMP4]], [[ADD_3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_4_NEG:%.*]] = sub nuw nsw i64 -5, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_4_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_4_NEG]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ADD_PTR_5]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 [[TMP5]], [[ADD_4]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_5_NEG:%.*]] = sub nuw nsw i64 -6, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_5_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_5_NEG]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ADD_PTR_6]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 [[TMP6]], [[ADD_5]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_6_NEG:%.*]] = sub nuw nsw i64 -7, [[INDVARS_IV]]
-; CHECK-NEXT:    [[ADD_PTR_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_6_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_6_NEG]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ADD_PTR_7]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[ADD_7]] = add i32 [[TMP7]], [[ADD_6]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
@@ -58,34 +58,34 @@ define dso_local zeroext i32 @foo(ptr noundef %a) #0 {
 ; CHECK-NEXT:    [[INDVARS_IV_1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1_7:%.*]], [[FOR_BODY4_1]] ], [ 0, [[FOR_BODY4]] ]
 ; CHECK-NEXT:    [[SUM_11_1:%.*]] = phi i32 [ [[ADD_1_7:%.*]], [[FOR_BODY4_1]] ], [ [[ADD_7]], [[FOR_BODY4]] ]
 ; CHECK-NEXT:    [[IDX_NEG_1:%.*]] = sub nsw i64 0, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[IDX_NEG_1]]
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[IDX_NEG_1]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ADD_PTR_1]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_NEG:%.*]] = xor i64 [[INDVARS_IV_1]], -1
-; CHECK-NEXT:    [[ADD_PTR_1_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_NEG]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ADD_PTR_1_1]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_1_NEG:%.*]] = sub nuw nsw i64 -2, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_1_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_1_NEG]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ADD_PTR_1_2]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP10]], [[TMP11]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_2_NEG:%.*]] = sub nuw nsw i64 -3, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_2_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_2_NEG]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ADD_PTR_1_3]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_3_NEG:%.*]] = sub nuw nsw i64 -4, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_3_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_3_NEG]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ADD_PTR_1_4]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_4_NEG:%.*]] = sub nuw nsw i64 -5, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_4_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_4_NEG]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ADD_PTR_1_5]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add i32 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_5_NEG:%.*]] = sub nuw nsw i64 -6, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_5_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_5_NEG]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ADD_PTR_1_6]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1_6_NEG:%.*]] = sub nuw nsw i64 -7, [[INDVARS_IV_1]]
-; CHECK-NEXT:    [[ADD_PTR_1_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_6_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_1_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_1_6_NEG]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ADD_PTR_1_7]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP20]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl i32 [[TMP22]], 1
@@ -97,42 +97,42 @@ define dso_local zeroext i32 @foo(ptr noundef %a) #0 {
 ; CHECK-NEXT:    [[INDVARS_IV_2:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_2_7:%.*]], [[FOR_BODY4_2]] ], [ 0, [[FOR_BODY4_1]] ]
 ; CHECK-NEXT:    [[SUM_11_2:%.*]] = phi i32 [ [[ADD_2_7:%.*]], [[FOR_BODY4_2]] ], [ [[ADD_1_7]], [[FOR_BODY4_1]] ]
 ; CHECK-NEXT:    [[IDX_NEG_2:%.*]] = sub nsw i64 0, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[IDX_NEG_2]]
+; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[IDX_NEG_2]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[ADD_PTR_2]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2:%.*]] = mul i32 [[TMP24]], 3
 ; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 [[MUL_2]], [[SUM_11_2]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_NEG:%.*]] = xor i64 [[INDVARS_IV_2]], -1
-; CHECK-NEXT:    [[ADD_PTR_2_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_1:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_NEG]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[ADD_PTR_2_1]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_1:%.*]] = mul i32 [[TMP25]], 3
 ; CHECK-NEXT:    [[ADD_2_1:%.*]] = add i32 [[MUL_2_1]], [[ADD_2]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_1_NEG:%.*]] = sub nuw nsw i64 -2, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_1_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_2:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_1_NEG]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ADD_PTR_2_2]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_2:%.*]] = mul i32 [[TMP26]], 3
 ; CHECK-NEXT:    [[ADD_2_2:%.*]] = add i32 [[MUL_2_2]], [[ADD_2_1]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_2_NEG:%.*]] = sub nuw nsw i64 -3, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_2_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_3:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_2_NEG]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ADD_PTR_2_3]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_3:%.*]] = mul i32 [[TMP27]], 3
 ; CHECK-NEXT:    [[ADD_2_3:%.*]] = add i32 [[MUL_2_3]], [[ADD_2_2]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_3_NEG:%.*]] = sub nuw nsw i64 -4, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_3_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_4:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_3_NEG]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ADD_PTR_2_4]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_4:%.*]] = mul i32 [[TMP28]], 3
 ; CHECK-NEXT:    [[ADD_2_4:%.*]] = add i32 [[MUL_2_4]], [[ADD_2_3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_4_NEG:%.*]] = sub nuw nsw i64 -5, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_4_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_5:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_4_NEG]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[ADD_PTR_2_5]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_5:%.*]] = mul i32 [[TMP29]], 3
 ; CHECK-NEXT:    [[ADD_2_5:%.*]] = add i32 [[MUL_2_5]], [[ADD_2_4]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_5_NEG:%.*]] = sub nuw nsw i64 -6, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_5_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_6:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_5_NEG]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[ADD_PTR_2_6]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_6:%.*]] = mul i32 [[TMP30]], 3
 ; CHECK-NEXT:    [[ADD_2_6:%.*]] = add i32 [[MUL_2_6]], [[ADD_2_5]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2_6_NEG:%.*]] = sub nuw nsw i64 -7, [[INDVARS_IV_2]]
-; CHECK-NEXT:    [[ADD_PTR_2_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_6_NEG]]
+; CHECK-NEXT:    [[ADD_PTR_2_7:%.*]] = getelementptr inbounds i32, ptr getelementptr inbounds nuw (i8, ptr @ARR, i64 396), i64 [[INDVARS_IV_NEXT_2_6_NEG]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[ADD_PTR_2_7]], align 4, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[MUL_2_7:%.*]] = mul i32 [[TMP31]], 3
 ; CHECK-NEXT:    [[ADD_2_7]] = add i32 [[MUL_2_7]], [[ADD_2_6]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/concat-boolmasks.ll b/llvm/test/Transforms/PhaseOrdering/X86/concat-boolmasks.ll
new file mode 100644
index 0000000000000..07bfbffa9518f
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/concat-boolmasks.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
+
+define i32 @movmsk_i32_v32i8_v16i8(<16 x i8> %v0, <16 x i8> %v1) {
+; CHECK-LABEL: @movmsk_i32_v32i8_v16i8(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <16 x i8> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <16 x i8> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <16 x i1> [[C0]] to i16
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <16 x i1> [[C1]] to i16
+; CHECK-NEXT:    [[Z0:%.*]] = zext i16 [[B0]] to i32
+; CHECK-NEXT:    [[Z1:%.*]] = zext i16 [[B1]] to i32
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw i32 [[Z0]], 16
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %c0 = icmp slt <16 x i8> %v0, zeroinitializer
+  %c1 = icmp slt <16 x i8> %v1, zeroinitializer
+  %b0 = bitcast <16 x i1> %c0 to i16
+  %b1 = bitcast <16 x i1> %c1 to i16
+  %z0 = zext i16 %b0 to i32
+  %z1 = zext i16 %b1 to i32
+  %s0 = shl nuw i32 %z0, 16
+  %or = or disjoint i32 %s0, %z1
+  ret i32 %or
+}
+
+define i32 @movmsk_i32_v8i32_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @movmsk_i32_v8i32_v4i32(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <4 x i32> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <4 x i32> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <4 x i1> [[C0]] to i4
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <4 x i1> [[C1]] to i4
+; CHECK-NEXT:    [[Z0:%.*]] = zext i4 [[B0]] to i32
+; CHECK-NEXT:    [[Z1:%.*]] = zext i4 [[B1]] to i32
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i32 [[Z0]], 4
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %c0 = icmp slt <4 x i32> %v0, zeroinitializer
+  %c1 = icmp slt <4 x i32> %v1, zeroinitializer
+  %b0 = bitcast <4 x i1> %c0 to i4
+  %b1 = bitcast <4 x i1> %c1 to i4
+  %z0 = zext i4 %b0 to i32
+  %z1 = zext i4 %b1 to i32
+  %s0 = shl nuw i32 %z0, 4
+  %or = or disjoint i32 %s0, %z1
+  ret i32 %or
+}
+
+define i64 @movmsk_i64_v32i8_v16i8(<16 x i8> %v0, <16 x i8> %v1) {
+; CHECK-LABEL: @movmsk_i64_v32i8_v16i8(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <16 x i8> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <16 x i8> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <16 x i1> [[C0]] to i16
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <16 x i1> [[C1]] to i16
+; CHECK-NEXT:    [[Z0:%.*]] = zext i16 [[B0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i16 [[B1]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i64 [[Z0]], 16
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c0 = icmp slt <16 x i8> %v0, zeroinitializer
+  %c1 = icmp slt <16 x i8> %v1, zeroinitializer
+  %b0 = bitcast <16 x i1> %c0 to i16
+  %b1 = bitcast <16 x i1> %c1 to i16
+  %z0 = zext i16 %b0 to i64
+  %z1 = zext i16 %b1 to i64
+  %s0 = shl nuw i64 %z0, 16
+  %or = or disjoint i64 %s0, %z1
+  ret i64 %or
+}
+
+define i64 @movmsk_i64_v8i32_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @movmsk_i64_v8i32_v4i32(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <4 x i32> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <4 x i32> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <4 x i1> [[C0]] to i4
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <4 x i1> [[C1]] to i4
+; CHECK-NEXT:    [[Z0:%.*]] = zext i4 [[B0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i4 [[B1]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i64 [[Z0]], 4
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c0 = icmp slt <4 x i32> %v0, zeroinitializer
+  %c1 = icmp slt <4 x i32> %v1, zeroinitializer
+  %b0 = bitcast <4 x i1> %c0 to i4
+  %b1 = bitcast <4 x i1> %c1 to i4
+  %z0 = zext i4 %b0 to i64
+  %z1 = zext i4 %b1 to i64
+  %s0 = shl nuw i64 %z0, 4
+  %or = or disjoint i64 %s0, %z1
+  ret i64 %or
+}
+
+define i64 @movmsk_i64_v64i8_v16i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %v3) {
+; CHECK-LABEL: @movmsk_i64_v64i8_v16i8(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <16 x i8> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <16 x i8> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt <16 x i8> [[V2:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C3:%.*]] = icmp slt <16 x i8> [[V3:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <16 x i1> [[C0]] to i16
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <16 x i1> [[C1]] to i16
+; CHECK-NEXT:    [[B2:%.*]] = bitcast <16 x i1> [[C2]] to i16
+; CHECK-NEXT:    [[B3:%.*]] = bitcast <16 x i1> [[C3]] to i16
+; CHECK-NEXT:    [[Z0:%.*]] = zext i16 [[B0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i16 [[B1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i16 [[B2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i16 [[B3]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw i64 [[Z0]], 48
+; CHECK-NEXT:    [[S1:%.*]] = shl nuw nsw i64 [[Z1]], 32
+; CHECK-NEXT:    [[S2:%.*]] = shl nuw nsw i64 [[Z2]], 16
+; CHECK-NEXT:    [[OR0:%.*]] = or disjoint i64 [[S1]], [[S0]]
+; CHECK-NEXT:    [[OR1:%.*]] = or disjoint i64 [[S2]], [[Z3]]
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[OR1]], [[OR0]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c0 = icmp slt <16 x i8> %v0, zeroinitializer
+  %c1 = icmp slt <16 x i8> %v1, zeroinitializer
+  %c2 = icmp slt <16 x i8> %v2, zeroinitializer
+  %c3 = icmp slt <16 x i8> %v3, zeroinitializer
+  %b0 = bitcast <16 x i1> %c0 to i16
+  %b1 = bitcast <16 x i1> %c1 to i16
+  %b2 = bitcast <16 x i1> %c2 to i16
+  %b3 = bitcast <16 x i1> %c3 to i16
+  %z0 = zext i16 %b0 to i64
+  %z1 = zext i16 %b1 to i64
+  %z2 = zext i16 %b2 to i64
+  %z3 = zext i16 %b3 to i64
+  %s0 = shl nuw i64 %z0, 48
+  %s1 = shl nuw i64 %z1, 32
+  %s2 = shl nuw i64 %z2, 16
+  %or0 = or disjoint i64 %s0, %s1
+  %or1 = or disjoint i64 %s2, %z3
+  %or = or disjoint i64 %or0, %or1
+  ret i64 %or
+}
+
+define i64 @movmsk_i64_v32i32_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; CHECK-LABEL: @movmsk_i64_v32i32_v4i32(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <4 x i32> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <4 x i32> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt <4 x i32> [[V2:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C3:%.*]] = icmp slt <4 x i32> [[V3:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <4 x i1> [[C0]] to i4
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <4 x i1> [[C1]] to i4
+; CHECK-NEXT:    [[B2:%.*]] = bitcast <4 x i1> [[C2]] to i4
+; CHECK-NEXT:    [[B3:%.*]] = bitcast <4 x i1> [[C3]] to i4
+; CHECK-NEXT:    [[Z0:%.*]] = zext i4 [[B0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i4 [[B1]] to i64
+; CHECK-NEXT:    [[Z2:%.*]] = zext i4 [[B2]] to i64
+; CHECK-NEXT:    [[Z3:%.*]] = zext i4 [[B3]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i64 [[Z0]], 12
+; CHECK-NEXT:    [[S1:%.*]] = shl nuw nsw i64 [[Z1]], 8
+; CHECK-NEXT:    [[S2:%.*]] = shl nuw nsw i64 [[Z2]], 4
+; CHECK-NEXT:    [[OR0:%.*]] = or disjoint i64 [[S1]], [[S0]]
+; CHECK-NEXT:    [[OR1:%.*]] = or disjoint i64 [[S2]], [[Z3]]
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[OR1]], [[OR0]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c0 = icmp slt <4 x i32> %v0, zeroinitializer
+  %c1 = icmp slt <4 x i32> %v1, zeroinitializer
+  %c2 = icmp slt <4 x i32> %v2, zeroinitializer
+  %c3 = icmp slt <4 x i32> %v3, zeroinitializer
+  %b0 = bitcast <4 x i1> %c0 to i4
+  %b1 = bitcast <4 x i1> %c1 to i4
+  %b2 = bitcast <4 x i1> %c2 to i4
+  %b3 = bitcast <4 x i1> %c3 to i4
+  %z0 = zext i4 %b0 to i64
+  %z1 = zext i4 %b1 to i64
+  %z2 = zext i4 %b2 to i64
+  %z3 = zext i4 %b3 to i64
+  %s0 = shl nuw i64 %z0, 12
+  %s1 = shl nuw i64 %z1, 8
+  %s2 = shl nuw i64 %z2, 4
+  %or0 = or disjoint i64 %s0, %s1
+  %or1 = or disjoint i64 %s2, %z3
+  %or = or disjoint i64 %or0, %or1
+  ret i64 %or
+}
+
+define i64 @movmsk_i64_v64i8_v32i8(<32 x i8> %v0, <32 x i8> %v1) {
+; CHECK-LABEL: @movmsk_i64_v64i8_v32i8(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <32 x i8> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <32 x i8> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <32 x i1> [[C0]] to i32
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <32 x i1> [[C1]] to i32
+; CHECK-NEXT:    [[Z0:%.*]] = zext i32 [[B0]] to i64
+; CHECK-NEXT:    [[Z1:%.*]] = zext i32 [[B1]] to i64
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw i64 [[Z0]], 32
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c0 = icmp slt <32 x i8> %v0, zeroinitializer
+  %c1 = icmp slt <32 x i8> %v1, zeroinitializer
+  %b0 = bitcast <32 x i1> %c0 to i32
+  %b1 = bitcast <32 x i1> %c1 to i32
+  %z0 = zext i32 %b0 to i64
+  %z1 = zext i32 %b1 to i64
+  %s0 = shl nuw i64 %z0, 32
+  %or = or disjoint i64 %s0, %z1
+  ret i64 %or
+}
+
+define i32 @movmsk_i32_v16i32_v8i32(<8 x i32> %v0, <8 x i32> %v1) {
+; CHECK-LABEL: @movmsk_i32_v16i32_v8i32(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt <8 x i32> [[V0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt <8 x i32> [[V1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B0:%.*]] = bitcast <8 x i1> [[C0]] to i8
+; CHECK-NEXT:    [[B1:%.*]] = bitcast <8 x i1> [[C1]] to i8
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[B0]] to i32
+; CHECK-NEXT:    [[Z1:%.*]] = zext i8 [[B1]] to i32
+; CHECK-NEXT:    [[S0:%.*]] = shl nuw nsw i32 [[Z0]], 8
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i32 [[S0]], [[Z1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %c0 = icmp slt <8 x i32> %v0, zeroinitializer
+  %c1 = icmp slt <8 x i32> %v1, zeroinitializer
+  %b0 = bitcast <8 x i1> %c0 to i8
+  %b1 = bitcast <8 x i1> %c1 to i8
+  %z0 = zext i8 %b0 to i32
+  %z1 = zext i8 %b1 to i32
+  %s0 = shl nuw i32 %z0, 8
+  %or = or disjoint i32 %s0, %z1
+  ret i32 %or
+}
+
+define i64 @PR111431(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2) {
+; CHECK-LABEL: @PR111431(
+; CHECK-NEXT:    [[C01:%.*]] = icmp eq <32 x i8> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[C02:%.*]] = icmp eq <32 x i8> [[A0]], [[A2:%.*]]
+; CHECK-NEXT:    [[B01:%.*]] = bitcast <32 x i1> [[C01]] to i32
+; CHECK-NEXT:    [[B02:%.*]] = bitcast <32 x i1> [[C02]] to i32
+; CHECK-NEXT:    [[Z01:%.*]] = zext i32 [[B01]] to i64
+; CHECK-NEXT:    [[Z02:%.*]] = zext i32 [[B02]] to i64
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 [[Z01]], 32
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[SHL]], [[Z02]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %c01 = icmp eq <32 x i8> %a0, %a1
+  %c02 = icmp eq <32 x i8> %a0, %a2
+  %b01 = bitcast <32 x i1> %c01 to i32
+  %b02 = bitcast <32 x i1> %c02 to i32
+  %z01 = zext i32 %b01 to i64
+  %z02 = zext i32 %b02 to i64
+  %shl = shl nuw i64 %z01, 32
+  %or = or disjoint i64 %shl, %z02
+  ret i64 %or
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/excessive-unrolling.ll b/llvm/test/Transforms/PhaseOrdering/X86/excessive-unrolling.ll
index ed25734c8448f..57a3d8175ba51 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/excessive-unrolling.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/excessive-unrolling.ll
@@ -13,129 +13,129 @@ define void @test_known_trip_count() {
 ; CHECK-LABEL: @test_known_trip_count(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr @b, align 16
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 16), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 16), align 16
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x double>, ptr @c, align 16
-; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 16), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 16), align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[WIDE_LOAD]], [[WIDE_LOAD4]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[WIDE_LOAD3]], [[WIDE_LOAD5]]
 ; CHECK-NEXT:    store <2 x double> [[TMP0]], ptr @a, align 16
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr getelementptr inbounds (i8, ptr @a, i64 16), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 32), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_1:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 48), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_1:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 32), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_1:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 48), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 16), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 32), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_1:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 48), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_1:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 32), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_1:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 48), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[WIDE_LOAD_1]], [[WIDE_LOAD4_1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[WIDE_LOAD3_1]], [[WIDE_LOAD5_1]]
-; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr getelementptr inbounds (i8, ptr @a, i64 32), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr getelementptr inbounds (i8, ptr @a, i64 48), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 64), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_2:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 80), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_2:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 64), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_2:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 80), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 32), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 48), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 64), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_2:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 80), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_2:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 64), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_2:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 80), align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[WIDE_LOAD_2]], [[WIDE_LOAD4_2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[WIDE_LOAD3_2]], [[WIDE_LOAD5_2]]
-; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr getelementptr inbounds (i8, ptr @a, i64 64), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr getelementptr inbounds (i8, ptr @a, i64 80), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 96), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_3:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 112), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_3:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 96), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_3:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 112), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 64), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 80), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 96), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_3:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 112), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_3:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 96), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_3:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 112), align 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[WIDE_LOAD_3]], [[WIDE_LOAD4_3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[WIDE_LOAD3_3]], [[WIDE_LOAD5_3]]
-; CHECK-NEXT:    store <2 x double> [[TMP6]], ptr getelementptr inbounds (i8, ptr @a, i64 96), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr getelementptr inbounds (i8, ptr @a, i64 112), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_4:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 128), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_4:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 144), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_4:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 128), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_4:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 144), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP6]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 96), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 112), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_4:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 128), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_4:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 144), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_4:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 128), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_4:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 144), align 16
 ; CHECK-NEXT:    [[TMP8:%.*]] = fadd <2 x double> [[WIDE_LOAD_4]], [[WIDE_LOAD4_4]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[WIDE_LOAD3_4]], [[WIDE_LOAD5_4]]
-; CHECK-NEXT:    store <2 x double> [[TMP8]], ptr getelementptr inbounds (i8, ptr @a, i64 128), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr getelementptr inbounds (i8, ptr @a, i64 144), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_5:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 160), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_5:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 176), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_5:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 160), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_5:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 176), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP8]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 128), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 144), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_5:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 160), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_5:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 176), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_5:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 160), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_5:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 176), align 16
 ; CHECK-NEXT:    [[TMP10:%.*]] = fadd <2 x double> [[WIDE_LOAD_5]], [[WIDE_LOAD4_5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[WIDE_LOAD3_5]], [[WIDE_LOAD5_5]]
-; CHECK-NEXT:    store <2 x double> [[TMP10]], ptr getelementptr inbounds (i8, ptr @a, i64 160), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP11]], ptr getelementptr inbounds (i8, ptr @a, i64 176), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_6:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 192), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_6:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 208), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_6:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 192), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_6:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 208), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP10]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 160), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP11]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 176), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_6:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 192), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_6:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 208), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_6:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 192), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_6:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 208), align 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[WIDE_LOAD_6]], [[WIDE_LOAD4_6]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = fadd <2 x double> [[WIDE_LOAD3_6]], [[WIDE_LOAD5_6]]
-; CHECK-NEXT:    store <2 x double> [[TMP12]], ptr getelementptr inbounds (i8, ptr @a, i64 192), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr getelementptr inbounds (i8, ptr @a, i64 208), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_7:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 224), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_7:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 240), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_7:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 224), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_7:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 240), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP12]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 192), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 208), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_7:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 224), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_7:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 240), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_7:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 224), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_7:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 240), align 16
 ; CHECK-NEXT:    [[TMP14:%.*]] = fadd <2 x double> [[WIDE_LOAD_7]], [[WIDE_LOAD4_7]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = fadd <2 x double> [[WIDE_LOAD3_7]], [[WIDE_LOAD5_7]]
-; CHECK-NEXT:    store <2 x double> [[TMP14]], ptr getelementptr inbounds (i8, ptr @a, i64 224), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP15]], ptr getelementptr inbounds (i8, ptr @a, i64 240), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_8:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 256), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_8:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 272), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_8:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 256), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_8:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 272), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP14]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 224), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP15]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 240), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_8:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 256), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_8:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 272), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_8:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 256), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_8:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 272), align 16
 ; CHECK-NEXT:    [[TMP16:%.*]] = fadd <2 x double> [[WIDE_LOAD_8]], [[WIDE_LOAD4_8]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = fadd <2 x double> [[WIDE_LOAD3_8]], [[WIDE_LOAD5_8]]
-; CHECK-NEXT:    store <2 x double> [[TMP16]], ptr getelementptr inbounds (i8, ptr @a, i64 256), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP17]], ptr getelementptr inbounds (i8, ptr @a, i64 272), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_9:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 288), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_9:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 304), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_9:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 288), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_9:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 304), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP16]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 256), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP17]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 272), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_9:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 288), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_9:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 304), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_9:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 288), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_9:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 304), align 16
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <2 x double> [[WIDE_LOAD_9]], [[WIDE_LOAD4_9]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fadd <2 x double> [[WIDE_LOAD3_9]], [[WIDE_LOAD5_9]]
-; CHECK-NEXT:    store <2 x double> [[TMP18]], ptr getelementptr inbounds (i8, ptr @a, i64 288), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP19]], ptr getelementptr inbounds (i8, ptr @a, i64 304), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_10:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 320), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_10:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 336), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_10:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 320), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_10:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 336), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP18]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 288), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP19]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 304), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_10:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 320), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_10:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 336), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_10:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 320), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_10:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 336), align 16
 ; CHECK-NEXT:    [[TMP20:%.*]] = fadd <2 x double> [[WIDE_LOAD_10]], [[WIDE_LOAD4_10]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = fadd <2 x double> [[WIDE_LOAD3_10]], [[WIDE_LOAD5_10]]
-; CHECK-NEXT:    store <2 x double> [[TMP20]], ptr getelementptr inbounds (i8, ptr @a, i64 320), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP21]], ptr getelementptr inbounds (i8, ptr @a, i64 336), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_11:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 352), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_11:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 368), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_11:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 352), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_11:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 368), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP20]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 320), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP21]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 336), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_11:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 352), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_11:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 368), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_11:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 352), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_11:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 368), align 16
 ; CHECK-NEXT:    [[TMP22:%.*]] = fadd <2 x double> [[WIDE_LOAD_11]], [[WIDE_LOAD4_11]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = fadd <2 x double> [[WIDE_LOAD3_11]], [[WIDE_LOAD5_11]]
-; CHECK-NEXT:    store <2 x double> [[TMP22]], ptr getelementptr inbounds (i8, ptr @a, i64 352), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP23]], ptr getelementptr inbounds (i8, ptr @a, i64 368), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_12:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 384), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_12:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 400), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_12:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 384), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_12:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 400), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP22]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 352), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP23]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 368), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_12:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 384), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_12:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 400), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_12:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 384), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_12:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 400), align 16
 ; CHECK-NEXT:    [[TMP24:%.*]] = fadd <2 x double> [[WIDE_LOAD_12]], [[WIDE_LOAD4_12]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <2 x double> [[WIDE_LOAD3_12]], [[WIDE_LOAD5_12]]
-; CHECK-NEXT:    store <2 x double> [[TMP24]], ptr getelementptr inbounds (i8, ptr @a, i64 384), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr getelementptr inbounds (i8, ptr @a, i64 400), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_13:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 416), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_13:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 432), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_13:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 416), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_13:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 432), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP24]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 384), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 400), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_13:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 416), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_13:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 432), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_13:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 416), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_13:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 432), align 16
 ; CHECK-NEXT:    [[TMP26:%.*]] = fadd <2 x double> [[WIDE_LOAD_13]], [[WIDE_LOAD4_13]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = fadd <2 x double> [[WIDE_LOAD3_13]], [[WIDE_LOAD5_13]]
-; CHECK-NEXT:    store <2 x double> [[TMP26]], ptr getelementptr inbounds (i8, ptr @a, i64 416), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP27]], ptr getelementptr inbounds (i8, ptr @a, i64 432), align 16
-; CHECK-NEXT:    [[WIDE_LOAD_14:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 448), align 16
-; CHECK-NEXT:    [[WIDE_LOAD3_14:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @b, i64 464), align 16
-; CHECK-NEXT:    [[WIDE_LOAD4_14:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 448), align 16
-; CHECK-NEXT:    [[WIDE_LOAD5_14:%.*]] = load <2 x double>, ptr getelementptr inbounds (i8, ptr @c, i64 464), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP26]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 416), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP27]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 432), align 16
+; CHECK-NEXT:    [[WIDE_LOAD_14:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 448), align 16
+; CHECK-NEXT:    [[WIDE_LOAD3_14:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @b, i64 464), align 16
+; CHECK-NEXT:    [[WIDE_LOAD4_14:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 448), align 16
+; CHECK-NEXT:    [[WIDE_LOAD5_14:%.*]] = load <2 x double>, ptr getelementptr inbounds nuw (i8, ptr @c, i64 464), align 16
 ; CHECK-NEXT:    [[TMP28:%.*]] = fadd <2 x double> [[WIDE_LOAD_14]], [[WIDE_LOAD4_14]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd <2 x double> [[WIDE_LOAD3_14]], [[WIDE_LOAD5_14]]
-; CHECK-NEXT:    store <2 x double> [[TMP28]], ptr getelementptr inbounds (i8, ptr @a, i64 448), align 16
-; CHECK-NEXT:    store <2 x double> [[TMP29]], ptr getelementptr inbounds (i8, ptr @a, i64 464), align 16
-; CHECK-NEXT:    [[TMP30:%.*]] = load double, ptr getelementptr inbounds (i8, ptr @b, i64 480), align 16
-; CHECK-NEXT:    [[TMP31:%.*]] = load double, ptr getelementptr inbounds (i8, ptr @c, i64 480), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP28]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 448), align 16
+; CHECK-NEXT:    store <2 x double> [[TMP29]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 464), align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = load double, ptr getelementptr inbounds nuw (i8, ptr @b, i64 480), align 16
+; CHECK-NEXT:    [[TMP31:%.*]] = load double, ptr getelementptr inbounds nuw (i8, ptr @c, i64 480), align 16
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP30]], [[TMP31]]
-; CHECK-NEXT:    store double [[ADD]], ptr getelementptr inbounds (i8, ptr @a, i64 480), align 16
+; CHECK-NEXT:    store double [[ADD]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 480), align 16
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -179,18 +179,18 @@ define void @test_runtime_trip_count(i32 %N) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [58 x double], ptr @b, i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [58 x double], ptr @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP0]], align 16
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x double>, ptr [[TMP1]], align 16
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [58 x double], ptr @c, i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [58 x double], ptr @c, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
 ; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x double>, ptr [[TMP2]], align 16
 ; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP3]], align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[WIDE_LOAD]], [[WIDE_LOAD5]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[WIDE_LOAD4]], [[WIDE_LOAD6]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [58 x double], ptr @a, i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [58 x double], ptr @a, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TMP6]], align 16
 ; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[TMP7]], align 16
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -204,12 +204,12 @@ define void @test_runtime_trip_count(i32 %N) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER7]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [58 x double], ptr @b, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [58 x double], ptr @b, i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load double, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [58 x double], ptr @c, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [58 x double], ptr @c, i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [58 x double], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw [58 x double], ptr @a, i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store double [[ADD]], ptr [[ARRAYIDX4]], align 8
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
index 0db94bea46c40..63e85731e8c70 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
@@ -59,7 +59,7 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8
 ; O2:       vector.body:
 ; O2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[FOR_BODY4_PREHEADER]] ]
 ; O2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[INDEX]]
-; O2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; O2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; O2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
 ; O2-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]]
 ; O2-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
@@ -104,7 +104,7 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8
 ; O3:       vector.body:
 ; O3-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[FOR_COND1_PREHEADER_US]] ]
 ; O3-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[INDEX]]
-; O3-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; O3-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; O3-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
 ; O3-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]]
 ; O3-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/merge-functions2.ll b/llvm/test/Transforms/PhaseOrdering/X86/merge-functions2.ll
index d794106ede3b2..285e390f1b61d 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/merge-functions2.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/merge-functions2.ll
@@ -12,7 +12,7 @@ define i32 @f(i32 noundef %x) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label %[[SWITCH_LOOKUP:.*]], label %[[SW_EPILOG:.*]]
 ; CHECK:       [[SWITCH_LOOKUP]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[X]] to i64
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.g, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds nuw [8 x i32], ptr @switch.table.g, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label %[[SW_EPILOG]]
 ; CHECK:       [[SW_EPILOG]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/merge-functions3.ll b/llvm/test/Transforms/PhaseOrdering/X86/merge-functions3.ll
index 7109c0ff4d55b..aa4884748ef8c 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/merge-functions3.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/merge-functions3.ll
@@ -15,7 +15,7 @@ define range(i32 0, 2) i32 @f(i32 noundef %x) local_unnamed_addr {
 ; CHECK-NEXT:    br i1 [[TMP0]], label %[[SWITCH_LOOKUP:.*]], label %[[SW_EPILOG:.*]]
 ; CHECK:       [[SWITCH_LOOKUP]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[X]] to i64
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.g, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds nuw [8 x i32], ptr @switch.table.g, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label %[[SW_EPILOG]]
 ; CHECK:       [[SW_EPILOG]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
index 580c13d50b1f5..8994afcdc629d 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
@@ -33,7 +33,7 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[PIN:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
@@ -43,7 +43,7 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = or disjoint <4 x i32> [[TMP4]], splat (i32 -16777216)
 ; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint <4 x i32> [[TMP5]], splat (i32 -16777216)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[POUT:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -57,12 +57,12 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER5]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[PIN]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[PIN]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP11]] to i32
 ; CHECK-NEXT:    [[OR2:%.*]] = mul nuw nsw i32 [[CONV]], 65793
 ; CHECK-NEXT:    [[OR3:%.*]] = or disjoint i32 [[OR2]], -16777216
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[POUT]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[POUT]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[OR3]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr38280.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr38280.ll
index 966d7e3cded0a..43f6cf7e51d09 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr38280.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr38280.ll
@@ -23,8 +23,8 @@ define void @apply_delta(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add <8 x i8> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    store <8 x i8> [[ADD]], ptr [[DST_ADDR_024]], align 1
-; CHECK-NEXT:    [[ADD_PTR1]] = getelementptr inbounds i8, ptr [[DST_ADDR_024]], i64 8
-; CHECK-NEXT:    [[ADD_PTR2]] = getelementptr inbounds i8, ptr [[SRC_ADDR_023]], i64 8
+; CHECK-NEXT:    [[ADD_PTR1]] = getelementptr inbounds nuw i8, ptr [[DST_ADDR_024]], i64 8
+; CHECK-NEXT:    [[ADD_PTR2]] = getelementptr inbounds nuw i8, ptr [[SRC_ADDR_023]], i64 8
 ; CHECK-NEXT:    [[SUB]] = add i64 [[COUNT_ADDR_022]], -8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[SUB]], 7
 ; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY]], label [[WHILE_COND3_PREHEADER]]
@@ -38,8 +38,8 @@ define void @apply_delta(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ADD6:%.*]] = add i8 [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    store i8 [[ADD6]], ptr [[DST_ADDR_130]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[DST_ADDR_130]], i64 1
-; CHECK-NEXT:    [[INCDEC_PTR8]] = getelementptr inbounds i8, ptr [[SRC_ADDR_129]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[DST_ADDR_130]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR8]] = getelementptr inbounds nuw i8, ptr [[SRC_ADDR_129]], i64 1
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i64 [[DEC]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[WHILE_END9]], label [[WHILE_BODY4]]
 ; CHECK:       while.end9:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
index cd95e9fba9438..2fe420183c683 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
@@ -23,7 +23,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
 ; AVX-NEXT:    store i32 42, ptr [[PTR2]], align 4
 ; AVX-NEXT:    br label [[LATCH]]
 ; AVX:       latch:
-; AVX-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR2]], i64 4
+; AVX-NEXT:    [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 4
 ; AVX-NEXT:    [[I11_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[END]]
 ; AVX-NEXT:    br i1 [[I11_NOT]], label [[EXIT]], label [[BB12]]
 ; AVX:       exit:
@@ -122,7 +122,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
 ; AVX2-NEXT:    store i32 42, ptr [[PTR2]], align 4
 ; AVX2-NEXT:    br label [[LATCH]]
 ; AVX2:       latch:
-; AVX2-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR2]], i64 4
+; AVX2-NEXT:    [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 4
 ; AVX2-NEXT:    [[I11_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[END]]
 ; AVX2-NEXT:    br i1 [[I11_NOT]], label [[EXIT]], label [[BB13]], !llvm.loop [[LOOP4:![0-9]+]]
 ; AVX2:       exit:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
index 549e4e036a747..8fbf45945cf3e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
@@ -10,8 +10,8 @@ define void @trunc_through_one_add(ptr noalias %0, ptr noalias readonly %1) {
 ; SSE-NEXT:    [[TMP6:%.*]] = add nuw nsw <8 x i16> [[TMP5]], [[TMP4]]
 ; SSE-NEXT:    [[TMP7:%.*]] = lshr <8 x i16> [[TMP6]], splat (i16 2)
 ; SSE-NEXT:    store <8 x i16> [[TMP7]], ptr [[TMP0:%.*]], align 2
-; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    [[TMP10:%.*]] = load <8 x i8>, ptr [[TMP8]], align 1
 ; SSE-NEXT:    [[TMP11:%.*]] = zext <8 x i8> [[TMP10]] to <8 x i16>
 ; SSE-NEXT:    [[TMP12:%.*]] = lshr <8 x i16> [[TMP11]], splat (i16 1)
@@ -185,9 +185,9 @@ define void @trunc_through_two_adds(ptr noalias %0, ptr noalias readonly %1, ptr
 ; SSE-NEXT:    [[TMP10:%.*]] = add nuw nsw <8 x i16> [[TMP9]], [[TMP8]]
 ; SSE-NEXT:    [[TMP11:%.*]] = lshr <8 x i16> [[TMP10]], splat (i16 2)
 ; SSE-NEXT:    store <8 x i16> [[TMP11]], ptr [[TMP0:%.*]], align 2
-; SSE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    [[TMP15:%.*]] = load <8 x i8>, ptr [[TMP12]], align 1
 ; SSE-NEXT:    [[TMP16:%.*]] = zext <8 x i8> [[TMP15]] to <8 x i16>
 ; SSE-NEXT:    [[TMP17:%.*]] = load <8 x i8>, ptr [[TMP13]], align 1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll
index ec9bfc542e32f..cbdafde5e421b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr94546.ll
@@ -1,32 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -O3                   -S < %s  | FileCheck %s --check-prefixes=SSE,SSE4
-; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -O3                   -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=SSE,SSE4
-; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes="default<O3>" -S < %s  | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -O3                   -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -O3                   -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -O3                   -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -O3                   -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64    -passes="default<O3>" -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes="default<O3>" -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=btver2    -passes="default<O3>" -S < %s  | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes="default<O3>" -S < %s  | FileCheck %s
 
 define <4 x double> @PR94546(<4 x double> %a, <4 x double> %b) {
-; SSE2-LABEL: @PR94546(
-; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 6>
-; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 7>
-; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
-; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 1>
-; SSE2-NEXT:    ret <4 x double> [[TMP4]]
-;
-; SSE4-LABEL: @PR94546(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 poison, i32 6>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 poison, i32 7>
-; SSE4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
-; SSE4-NEXT:    ret <4 x double> [[TMP3]]
-;
-; AVX-LABEL: @PR94546(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 poison, i32 6>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 poison, i32 7>
-; AVX-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
-; AVX-NEXT:    ret <4 x double> [[TMP3]]
+; CHECK-LABEL: @PR94546(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 poison, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 poison, i32 poison, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x double> [[TMP3]]
 ;
   %vecext = extractelement <4 x double> %a, i32 0
   %vecext1 = extractelement <4 x double> %a, i32 1
@@ -47,7 +34,3 @@ define <4 x double> @PR94546(<4 x double> %a, <4 x double> %b) {
   %shuffle = shufflevector <4 x double> %vecinit13, <4 x double> %a, <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
   ret <4 x double> %shuffle
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX1: {{.*}}
-; AVX2: {{.*}}
-; SSE: {{.*}}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
index 076cc3f4dddd1..d79ffb0149ff8 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
@@ -14,7 +14,7 @@ define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %f
 ; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
 ; CHECK:       [[FOR_BODY_PREHEADER]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[NFACE]] to i64
-; CHECK-NEXT:    [[INVARIANT_GEP:%.*]] = getelementptr inbounds i32, ptr [[FACE_CELL]], i64 [[TMP0]]
+; CHECK-NEXT:    [[INVARIANT_GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[FACE_CELL]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[NFACE]], 4
 ; CHECK-NEXT:    br i1 [[TMP1]], label %[[FOR_BODY_PREHEADER14:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[FOR_BODY_PREHEADER14]]:
@@ -48,10 +48,10 @@ define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %f
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER14]] ]
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[FACE_CELL]], i64 [[INDVARS_IV_NEXT_2]]
-; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4, !tbaa [[TBAA0]], !llvm.access.group [[ACC_GRP4]]
-; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, ptr [[INVARIANT_GEP]], i64 [[INDVARS_IV_NEXT_2]]
-; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[GEP_3]], align 4, !tbaa [[TBAA0]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[FACE_CELL]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]], !llvm.access.group [[ACC_GRP4]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[INVARIANT_GEP]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[GEP]], align 4, !tbaa [[TBAA0]], !llvm.access.group [[ACC_GRP4]]
 ; CHECK-NEXT:    [[IDXPROM3_3:%.*]] = sext i32 [[TMP22]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX4_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[IDXPROM3_3]]
 ; CHECK-NEXT:    [[IDXPROM5_3:%.*]] = sext i32 [[TMP23]] to i64
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/shuffle-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/X86/shuffle-inseltpoison.ll
index 719d6df77cb7d..9b03433e00cbc 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/shuffle-inseltpoison.ll
@@ -159,13 +159,11 @@ define <4 x i32> @shuffle_8_add_32_masks_are_eq_and_can_be_converted_up(<16 x i8
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles and widen type?
 
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -175,13 +173,11 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32>
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles?
 
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -191,13 +187,11 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x
 }
 
 ; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles and widen type?
 
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -207,13 +201,11 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %
 }
 
 ; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles?
 
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 13, i32 12, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -223,13 +215,11 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i3
 }
 
 ; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<16 x i8>(v)))
-; TODO: squash shuffles?
 
 define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
@@ -239,13 +229,11 @@ define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8>
 }
 
 ; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<8 x i16>(v)))
-; TODO: squash shuffles?
 
 define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> %v1) {
 ; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
@@ -287,13 +275,11 @@ define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(<8 x
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<16 x i8>(v)))
-; TODO: squash shuffles and widen type?
 
 define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
index 930ce6a7b7d67..29464fec8e7a2 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
@@ -159,13 +159,11 @@ define <4 x i32> @shuffle_8_add_32_masks_are_eq_and_can_be_converted_up(<16 x i8
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles and widen type?
 
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -175,13 +173,11 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32>
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles?
 
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -191,13 +187,11 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x
 }
 
 ; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles and widen type?
 
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -207,13 +201,11 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %
 }
 
 ; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
-; TODO: Squash shuffles?
 
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 13, i32 12, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -223,13 +215,11 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i3
 }
 
 ; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<16 x i8>(v)))
-; TODO: squash shuffles?
 
 define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
@@ -239,13 +229,11 @@ define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8>
 }
 
 ; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<8 x i16>(v)))
-; TODO: squash shuffles?
 
 define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> %v1) {
 ; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
@@ -287,13 +275,11 @@ define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(<8 x
 }
 
 ; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<16 x i8>(v)))
-; TODO: squash shuffles and widen type?
 
 define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/simplifycfg-late.ll b/llvm/test/Transforms/PhaseOrdering/X86/simplifycfg-late.ll
index 10193df6f179a..08ae28ce782c2 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/simplifycfg-late.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/simplifycfg-late.ll
@@ -14,7 +14,7 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[SWITCH_TABLEIDX]] to i64
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds nuw [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll b/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
index 92ce8eb5b5ac2..ec387d6ae44f2 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
@@ -40,7 +40,7 @@ define void @licm(ptr align 8 dereferenceable(8) %_M_start.i, i64 %numElem) {
 ; O23:       vector.body:
 ; O23-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; O23-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 [[INDEX]]
-; O23-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; O23-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; O23-NEXT:    store <2 x double> splat (double 2.000000e+00), ptr [[TMP1]], align 8, !tbaa [[TBAA8:![0-9]+]]
 ; O23-NEXT:    store <2 x double> splat (double 2.000000e+00), ptr [[TMP2]], align 8, !tbaa [[TBAA8]]
 ; O23-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
index e645a309b4793..a715736df887f 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
@@ -16,10 +16,10 @@ define dso_local void @_Z13vecIncFromPtrP12FloatVecPair(ptr %FVP) {
 ; O1-LABEL: define {{[^@]+}}@_Z13vecIncFromPtrP12FloatVecPair
 ; O1-SAME: (ptr nocapture readonly [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O1-NEXT:  entry:
-; O1-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds i8, ptr [[FVP]], i64 16
+; O1-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds nuw i8, ptr [[FVP]], i64 16
 ; O1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VSRC23_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
 ; O1-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0:%.*]], ptr [[TMP0]], i64 undef
-; O1-NEXT:    [[SIZE4_I:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX_I_I]], i64 8
+; O1-NEXT:    [[SIZE4_I:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX_I_I]], i64 8
 ; O1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SIZE4_I]], align 8, !tbaa [[TBAA6:![0-9]+]]
 ; O1-NEXT:    [[CMP56_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 0
 ; O1-NEXT:    br i1 [[CMP56_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT:%.*]], label [[FOR_BODY7_LR_PH_I:%.*]]
@@ -46,10 +46,10 @@ define dso_local void @_Z13vecIncFromPtrP12FloatVecPair(ptr %FVP) {
 ; O23-LABEL: define {{[^@]+}}@_Z13vecIncFromPtrP12FloatVecPair
 ; O23-SAME: (ptr nocapture readonly [[FVP:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; O23-NEXT:  entry:
-; O23-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds i8, ptr [[FVP]], i64 16
+; O23-NEXT:    [[VSRC23_I:%.*]] = getelementptr inbounds nuw i8, ptr [[FVP]], i64 16
 ; O23-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VSRC23_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
 ; O23-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0:%.*]], ptr [[TMP0]], i64 undef
-; O23-NEXT:    [[SIZE4_I:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX_I_I]], i64 8
+; O23-NEXT:    [[SIZE4_I:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX_I_I]], i64 8
 ; O23-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SIZE4_I]], align 8, !tbaa [[TBAA6:![0-9]+]]
 ; O23-NEXT:    [[CMP56_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 0
 ; O23-NEXT:    br i1 [[CMP56_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT:%.*]], label [[FOR_BODY7_LR_PH_I:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
index 2ba2935e07e2f..7349f603822cd 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
@@ -40,9 +40,9 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 32
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 64
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 96
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 32
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 64
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 96
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP5]], align 8, !tbaa [[TBAA3:![0-9]+]]
 ; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP6]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x double>, ptr [[TMP7]], align 8, !tbaa [[TBAA3]]
@@ -52,9 +52,9 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast <4 x double> [[WIDE_LOAD7]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast <4 x double> [[WIDE_LOAD8]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 32
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 64
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 96
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP13]], i64 32
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP13]], i64 64
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP13]], i64 96
 ; CHECK-NEXT:    store <4 x double> [[TMP9]], ptr [[TMP13]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    store <4 x double> [[TMP10]], ptr [[TMP14]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    store <4 x double> [[TMP11]], ptr [[TMP15]], align 8, !tbaa [[TBAA3]]
@@ -101,10 +101,10 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
 ; CHECK:       for.body.prol:
 ; CHECK-NEXT:    [[INDVARS_IV_PROL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL:%.*]], [[FOR_BODY_PROL]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PROL_PREHEADER]] ]
 ; CHECK-NEXT:    [[PROL_ITER:%.*]] = phi i64 [ [[PROL_ITER_NEXT:%.*]], [[FOR_BODY_PROL]] ], [ 0, [[FOR_BODY_PROL_PREHEADER]] ]
-; CHECK-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_PROL]]
+; CHECK-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_PROL]]
 ; CHECK-NEXT:    [[T0_PROL:%.*]] = load double, ptr [[ARRAYIDX_PROL]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul fast double [[T0_PROL]], [[TMP18]]
-; CHECK-NEXT:    [[ARRAYIDX2_PROL:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_PROL]]
+; CHECK-NEXT:    [[ARRAYIDX2_PROL:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_PROL]]
 ; CHECK-NEXT:    store double [[TMP19]], ptr [[ARRAYIDX2_PROL]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 1
 ; CHECK-NEXT:    [[PROL_ITER_NEXT]] = add i64 [[PROL_ITER]], 1
@@ -127,52 +127,52 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_PREHEADER9_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[T0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast double [[T0]], [[TMP22]]
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store double [[TMP30]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[T0_1:%.*]] = load double, ptr [[ARRAYIDX_1]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = fmul fast double [[T0_1]], [[TMP23]]
-; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    store double [[TMP31]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    [[T0_2:%.*]] = load double, ptr [[ARRAYIDX_2]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = fmul fast double [[T0_2]], [[TMP24]]
-; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    store double [[TMP32]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    [[T0_3:%.*]] = load double, ptr [[ARRAYIDX_3]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = fmul fast double [[T0_3]], [[TMP25]]
-; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    store double [[TMP33]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_3]]
 ; CHECK-NEXT:    [[T0_4:%.*]] = load double, ptr [[ARRAYIDX_4]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul fast double [[T0_4]], [[TMP26]]
-; CHECK-NEXT:    [[ARRAYIDX2_4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT:    [[ARRAYIDX2_4:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_3]]
 ; CHECK-NEXT:    store double [[TMP34]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_4]]
 ; CHECK-NEXT:    [[T0_5:%.*]] = load double, ptr [[ARRAYIDX_5]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast double [[T0_5]], [[TMP27]]
-; CHECK-NEXT:    [[ARRAYIDX2_5:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT:    [[ARRAYIDX2_5:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_4]]
 ; CHECK-NEXT:    store double [[TMP35]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6
-; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_5]]
 ; CHECK-NEXT:    [[T0_6:%.*]] = load double, ptr [[ARRAYIDX_6]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = fmul fast double [[T0_6]], [[TMP28]]
-; CHECK-NEXT:    [[ARRAYIDX2_6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT:    [[ARRAYIDX2_6:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_5]]
 ; CHECK-NEXT:    store double [[TMP36]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7
-; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_6]]
 ; CHECK-NEXT:    [[T0_7:%.*]] = load double, ptr [[ARRAYIDX_7]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast double [[T0_7]], [[TMP29]]
-; CHECK-NEXT:    [[ARRAYIDX2_7:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT:    [[ARRAYIDX2_7:%.*]] = getelementptr inbounds nuw double, ptr [[X]], i64 [[INDVARS_IV_NEXT_6]]
 ; CHECK-NEXT:    store double [[TMP37]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[TBAA3]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
 ; CHECK-NEXT:    [[EXITCOND_NOT_7:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_7]], [[WIDE_TRIP_COUNT]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
index 2296531bd1de6..85f6fceb5bdbe 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
@@ -48,7 +48,7 @@ define noundef <4 x float> @ConvertVectors_ByVal(ptr noundef nonnull align 16 de
 ; SSE-LABEL: @ConvertVectors_ByVal(
 ; SSE-NEXT:  entry:
 ; SSE-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[V:%.*]], align 16
-; SSE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 8
+; SSE-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
 ; SSE-NEXT:    [[V_VAL421:%.*]] = load i64, ptr [[TMP1]], align 8
 ; SSE-NEXT:    [[TMP2:%.*]] = trunc i64 [[V_VAL421]] to i32
 ; SSE-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
@@ -59,7 +59,7 @@ define noundef <4 x float> @ConvertVectors_ByVal(ptr noundef nonnull align 16 de
 ; AVX-LABEL: @ConvertVectors_ByVal(
 ; AVX-NEXT:  entry:
 ; AVX-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[V:%.*]], align 16
-; AVX-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 8
+; AVX-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
 ; AVX-NEXT:    [[V_VAL421:%.*]] = load i64, ptr [[TMP1]], align 8
 ; AVX-NEXT:    [[TMP2:%.*]] = trunc i64 [[V_VAL421]] to i32
 ; AVX-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
index 428147a1383b1..f98afb3b5243e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
@@ -37,17 +37,17 @@ define void @bar(ptr noundef %0) {
 ; SSE-NEXT:    [[TMP3:%.*]] = xor <2 x i64> [[TMP2]], splat (i64 -1)
 ; SSE-NEXT:    [[TMP4:%.*]] = shl <2 x i64> [[TMP3]], splat (i64 44)
 ; SSE-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP0]], align 8
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
 ; SSE-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP6]], splat (i64 -1)
 ; SSE-NEXT:    [[TMP8:%.*]] = shl <2 x i64> [[TMP7]], splat (i64 44)
 ; SSE-NEXT:    store <2 x i64> [[TMP8]], ptr [[TMP5]], align 8
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8
 ; SSE-NEXT:    [[TMP11:%.*]] = xor <2 x i64> [[TMP10]], splat (i64 -1)
 ; SSE-NEXT:    [[TMP12:%.*]] = shl <2 x i64> [[TMP11]], splat (i64 44)
 ; SSE-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP9]], align 8
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 48
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48
 ; SSE-NEXT:    [[TMP14:%.*]] = load <2 x i64>, ptr [[TMP13]], align 8
 ; SSE-NEXT:    [[TMP15:%.*]] = xor <2 x i64> [[TMP14]], splat (i64 -1)
 ; SSE-NEXT:    [[TMP16:%.*]] = shl <2 x i64> [[TMP15]], splat (i64 44)
@@ -59,7 +59,7 @@ define void @bar(ptr noundef %0) {
 ; AVX-NEXT:    [[TMP3:%.*]] = xor <4 x i64> [[TMP2]], splat (i64 -1)
 ; AVX-NEXT:    [[TMP4:%.*]] = shl <4 x i64> [[TMP3]], splat (i64 44)
 ; AVX-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP0]], align 8
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
 ; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
 ; AVX-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP6]], splat (i64 -1)
 ; AVX-NEXT:    [[TMP8:%.*]] = shl <4 x i64> [[TMP7]], splat (i64 44)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
index e9a42a859ab3d..e2c85bb515369 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reduction-known-first-value.ll
@@ -17,7 +17,7 @@ define i16 @test(ptr %ptr) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[ENTRY]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i16> [ zeroinitializer, [[ENTRY]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i16>
diff --git a/llvm/test/Transforms/PhaseOrdering/basic.ll b/llvm/test/Transforms/PhaseOrdering/basic.ll
index 343afedff1f85..5a8e5c4af7e24 100644
--- a/llvm/test/Transforms/PhaseOrdering/basic.ll
+++ b/llvm/test/Transforms/PhaseOrdering/basic.ll
@@ -35,7 +35,7 @@ define i32 @test2(i32 %a, ptr %p) nounwind uwtable ssp {
 ; CHECK-NEXT:    [[DIV1:%.*]] = lshr i32 [[A:%.*]], 2
 ; CHECK-NEXT:    store i32 [[DIV1]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = shl nuw nsw i32 [[DIV1]], 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/bitcast-store-branch.ll b/llvm/test/Transforms/PhaseOrdering/bitcast-store-branch.ll
index 4c9cd3090681e..bbd4849c32296 100644
--- a/llvm/test/Transforms/PhaseOrdering/bitcast-store-branch.ll
+++ b/llvm/test/Transforms/PhaseOrdering/bitcast-store-branch.ll
@@ -14,7 +14,7 @@ define ptr @parent(ptr align 8 dereferenceable(72) %f, half %val1, i16 %val2, i3
 ; CHECK-LABEL: define noundef nonnull ptr @parent
 ; CHECK-SAME: (ptr readonly returned align 8 dereferenceable(72) [[F:%.*]], half [[VAL1:%.*]], i16 [[VAL2:%.*]], i32 [[VAL3:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] align 2 {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[F]], i64 64
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[F]], i64 64
 ; CHECK-NEXT:    [[F_VAL:%.*]] = load ptr, ptr [[TMP0]], align 8
 ; CHECK-NEXT:    [[CMP_NOT_NOT_I:%.*]] = icmp eq i32 [[VAL3]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VAL1]] to i16
@@ -22,7 +22,7 @@ define ptr @parent(ptr align 8 dereferenceable(72) %f, half %val1, i16 %val2, i3
 ; CHECK-NEXT:    [[VAL2_SINK_I:%.*]] = select i1 [[CMP_NOT_NOT_I]], i16 [[TMP1]], i16 [[VAL2]]
 ; CHECK-NEXT:    [[VAL1_SINK_I:%.*]] = select i1 [[CMP_NOT_NOT_I]], half [[TMP2]], half [[VAL1]]
 ; CHECK-NEXT:    store i16 [[VAL2_SINK_I]], ptr [[F_VAL]], align 2
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[F_VAL]], i64 16
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[F_VAL]], i64 16
 ; CHECK-NEXT:    store half [[VAL1_SINK_I]], ptr [[TMP3]], align 2
 ; CHECK-NEXT:    ret ptr [[F]]
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/dce-after-argument-promotion-loads.ll b/llvm/test/Transforms/PhaseOrdering/dce-after-argument-promotion-loads.ll
index a3d111fffbbf6..ee7698b116aa2 100644
--- a/llvm/test/Transforms/PhaseOrdering/dce-after-argument-promotion-loads.ll
+++ b/llvm/test/Transforms/PhaseOrdering/dce-after-argument-promotion-loads.ll
@@ -16,13 +16,13 @@ define ptr @parent(ptr align 8 dereferenceable(72) %f, i16 %val1, i16 %val2, i32
 ; CHECK-LABEL: define noundef nonnull ptr @parent
 ; CHECK-SAME: (ptr readonly returned align 8 dereferenceable(72) [[F:%.*]], i16 [[VAL1:%.*]], i16 [[VAL2:%.*]], i32 [[VAL3:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] align 2 {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[F]], i64 64
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[F]], i64 64
 ; CHECK-NEXT:    [[F_VAL:%.*]] = load ptr, ptr [[TMP0]], align 8
 ; CHECK-NEXT:    [[CMP_NOT_NOT_I:%.*]] = icmp eq i32 [[VAL3]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = select i1 [[CMP_NOT_NOT_I]], i16 [[VAL1]], i16 [[VAL2]]
 ; CHECK-NEXT:    [[SPEC_SELECT2_I:%.*]] = select i1 [[CMP_NOT_NOT_I]], i16 [[VAL2]], i16 [[VAL1]]
 ; CHECK-NEXT:    store i16 [[SPEC_SELECT_I]], ptr [[F_VAL]], align 2
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[F_VAL]], i64 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[F_VAL]], i64 16
 ; CHECK-NEXT:    store i16 [[SPEC_SELECT2_I]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    ret ptr [[F]]
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/enable-loop-header-duplication-oz.ll b/llvm/test/Transforms/PhaseOrdering/enable-loop-header-duplication-oz.ll
index 98b11578b49fb..5f75bd788e4bb 100644
--- a/llvm/test/Transforms/PhaseOrdering/enable-loop-header-duplication-oz.ll
+++ b/llvm/test/Transforms/PhaseOrdering/enable-loop-header-duplication-oz.ll
@@ -19,7 +19,7 @@ define void @test(i8* noalias nonnull align 1 %start, i8* %end) unnamed_addr {
 ; NOROTATION-NEXT:    [[_12_I:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]
 ; NOROTATION-NEXT:    br i1 [[_12_I]], label [[EXIT:%.*]], label [[LOOP_LATCH]]
 ; NOROTATION:       loop.latch:
-; NOROTATION-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 1
+; NOROTATION-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 1
 ; NOROTATION-NEXT:    store i8 1, ptr [[PTR_IV]], align 1
 ; NOROTATION-NEXT:    br label [[LOOP_HEADER]]
 ; NOROTATION:       exit:
diff --git a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
index 862f40a9ae2e9..e156111bd445e 100644
--- a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
+++ b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
@@ -7,7 +7,7 @@ define void @test(ptr noundef %a, i32 noundef %beam) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[BEAM]], 1
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext nneg i32 [[MUL]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
@@ -21,7 +21,7 @@ define void @test(ptr noundef %a, i32 noundef %beam) {
 ; CHECK:       if.else:
 ; CHECK-NEXT:    [[MUL2:%.*]] = shl nuw nsw i32 [[I_06]], 1
 ; CHECK-NEXT:    [[IDXPROM3:%.*]] = zext nneg i32 [[MUL2]] to i64
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM3]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IDXPROM3]]
 ; CHECK-NEXT:    store i32 1, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
index bd509509c321f..394204e9409a1 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
@@ -38,7 +38,7 @@ define void @test_fill_with_foreach([2 x i64] %elems.coerce) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[__BEGIN1_SROA_0_03:%.*]] = phi ptr [ [[INCDEC_PTR_I:%.*]], [[FOR_BODY]] ], [ [[TMP0]], [[FOR_COND_PREHEADER_SPLIT]] ]
 ; CHECK-NEXT:    tail call void @use(ptr noundef nonnull align 4 dereferenceable(4) [[__BEGIN1_SROA_0_03]])
-; CHECK-NEXT:    [[INCDEC_PTR_I]] = getelementptr inbounds i8, ptr [[__BEGIN1_SROA_0_03]], i64 4
+; CHECK-NEXT:    [[INCDEC_PTR_I]] = getelementptr inbounds nuw i8, ptr [[__BEGIN1_SROA_0_03]], i64 4
 ; CHECK-NEXT:    [[CMP_I_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR_I]], [[ADD_PTR_I]]
 ; CHECK-NEXT:    br i1 [[CMP_I_NOT]], label [[COMMON_RET]], label [[FOR_BODY]]
 ;
@@ -133,7 +133,7 @@ define void @foo(ptr noundef nonnull align 8 dereferenceable(24) noalias %vec) #
 ; CHECK-LABEL: define void @foo
 ; CHECK-SAME: (ptr noalias nocapture noundef nonnull readonly align 8 dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds i8, ptr [[VEC]], i64 8
+; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[_M_FINISH_I_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC]], align 8, !tbaa [[TBAA5:![0-9]+]]
 ; CHECK-NEXT:    [[SUB_PTR_LHS_CAST_I_I:%.*]] = ptrtoint ptr [[TMP0]] to i64
@@ -272,7 +272,7 @@ define void @loop_with_signed_induction(ptr noundef nonnull align 8 dereferencea
 ; CHECK-LABEL: define void @loop_with_signed_induction
 ; CHECK-SAME: (ptr nocapture noundef nonnull readonly align 8 dereferenceable(24) [[VEC:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds i8, ptr [[VEC]], i64 8
+; CHECK-NEXT:    [[_M_FINISH_I_I:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[_M_FINISH_I_I]], align 8, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VEC]], align 8, !tbaa [[TBAA5]]
 ; CHECK-NEXT:    [[SUB_PTR_LHS_CAST_I_I:%.*]] = ptrtoint ptr [[TMP0]] to i64
@@ -285,7 +285,7 @@ define void @loop_with_signed_induction(ptr noundef nonnull align 8 dereferencea
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_010:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i64 [[I_010]]
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds nuw double, ptr [[TMP1]], i64 [[I_010]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load double, ptr [[ADD_PTR_I]], align 8, !tbaa [[TBAA6:![0-9]+]]
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP2]], 1.000000e+00
 ; CHECK-NEXT:    store double [[ADD]], ptr [[ADD_PTR_I]], align 8, !tbaa [[TBAA6]]
@@ -359,7 +359,7 @@ define void @monkey(ptr noundef %arr, i32 noundef %len) {
 ; CHECK:       for.body4:
 ; CHECK-NEXT:    [[K_07:%.*]] = phi i32 [ [[DEC:%.*]], [[FOR_BODY4]] ], [ [[I_09]], [[FOR_BODY4_PREHEADER]] ]
 ; CHECK-NEXT:    [[IDX_EXT_I:%.*]] = zext i32 [[K_07]] to i64
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX_EXT_I]]
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds nuw i32, ptr [[ARR]], i64 [[IDX_EXT_I]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ADD_PTR_I]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], 1
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ADD_PTR_I]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll b/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
index 72921acba5969..5435bc2eb8b37 100644
--- a/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
+++ b/llvm/test/Transforms/PhaseOrdering/lto-argpromotion-ipsccp.ll
@@ -28,7 +28,7 @@ define internal void @child(ptr %p, ptr %n, ptr %c) noinline {
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END:.*]], label %[[FOR_INC]]
 ; CHECK:       [[FOR_INC]]:
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext nneg i32 [[I_0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[P]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[TMP0]], 5
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/lto-licm.ll b/llvm/test/Transforms/PhaseOrdering/lto-licm.ll
index 763e266e6a382..8b51f9f465587 100644
--- a/llvm/test/Transforms/PhaseOrdering/lto-licm.ll
+++ b/llvm/test/Transforms/PhaseOrdering/lto-licm.ll
@@ -12,7 +12,7 @@ define void @hoist_fdiv(ptr %a, float %b) {
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext nneg i32 [[I_0]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    store float [[TMP2]], ptr [[ARRAYIDX]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/pr39282.ll b/llvm/test/Transforms/PhaseOrdering/pr39282.ll
index 2bbc01b677bb6..583f1cbc6c5a1 100644
--- a/llvm/test/Transforms/PhaseOrdering/pr39282.ll
+++ b/llvm/test/Transforms/PhaseOrdering/pr39282.ll
@@ -19,22 +19,22 @@ define void @pr39282(ptr %addr1, ptr %addr2) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
-; CHECK-NEXT:    [[X_I:%.*]] = load i32, ptr [[ADDR1:%.*]], align 4, !alias.scope !3, !noalias !0
-; CHECK-NEXT:    store i32 [[X_I]], ptr [[ADDR2:%.*]], align 4, !alias.scope !0, !noalias !3
-; CHECK-NEXT:    [[ADDR1I_1:%.*]] = getelementptr inbounds i8, ptr [[ADDR1]], i64 4
-; CHECK-NEXT:    [[ADDR2I_1:%.*]] = getelementptr inbounds i8, ptr [[ADDR2]], i64 4
+; CHECK-NEXT:    [[X_I:%.*]] = load i32, ptr [[ADDR1:%.*]], align 4, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT:    store i32 [[X_I]], ptr [[ADDR2:%.*]], align 4, !alias.scope [[META0]], !noalias [[META3]]
+; CHECK-NEXT:    [[ADDR1I_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADDR1]], i64 4
+; CHECK-NEXT:    [[ADDR2I_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADDR2]], i64 4
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META5:![0-9]+]])
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META7:![0-9]+]])
-; CHECK-NEXT:    [[X_I_1:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope !7, !noalias !5
-; CHECK-NEXT:    store i32 [[X_I_1]], ptr [[ADDR2I_1]], align 4, !alias.scope !5, !noalias !7
+; CHECK-NEXT:    [[X_I_1:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope [[META7]], !noalias [[META5]]
+; CHECK-NEXT:    store i32 [[X_I_1]], ptr [[ADDR2I_1]], align 4, !alias.scope [[META5]], !noalias [[META7]]
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META9:![0-9]+]])
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META11:![0-9]+]])
-; CHECK-NEXT:    [[X_I_2:%.*]] = load i32, ptr [[ADDR1]], align 4, !alias.scope !11, !noalias !9
-; CHECK-NEXT:    store i32 [[X_I_2]], ptr [[ADDR2]], align 4, !alias.scope !9, !noalias !11
+; CHECK-NEXT:    [[X_I_2:%.*]] = load i32, ptr [[ADDR1]], align 4, !alias.scope [[META11]], !noalias [[META9]]
+; CHECK-NEXT:    store i32 [[X_I_2]], ptr [[ADDR2]], align 4, !alias.scope [[META9]], !noalias [[META11]]
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META13:![0-9]+]])
 ; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META15:![0-9]+]])
-; CHECK-NEXT:    [[X_I_3:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope !15, !noalias !13
-; CHECK-NEXT:    store i32 [[X_I_3]], ptr [[ADDR2I_1]], align 4, !alias.scope !13, !noalias !15
+; CHECK-NEXT:    [[X_I_3:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope [[META15]], !noalias [[META13]]
+; CHECK-NEXT:    store i32 [[X_I_3]], ptr [[ADDR2I_1]], align 4, !alias.scope [[META13]], !noalias [[META15]]
 ; CHECK-NEXT:    ret void
 ;
 start:
diff --git a/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll b/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
index 17073fa198202..bad07c19f14b3 100644
--- a/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/pr98799-inline-simplifycfg-ub.ll
@@ -37,7 +37,7 @@ define i32 @foo(ptr %arg, i1 %arg1) {
 ; O2-NEXT:    [[TMP0:%.*]] = xor i1 [[ARG1]], true
 ; O2-NEXT:    tail call void @llvm.assume(i1 [[TMP0]])
 ; O2-NEXT:    [[I_I:%.*]] = load ptr, ptr [[ARG]], align 8, !nonnull [[META0:![0-9]+]], !noundef [[META0]]
-; O2-NEXT:    [[I3_I:%.*]] = getelementptr inbounds i8, ptr [[I_I]], i64 1
+; O2-NEXT:    [[I3_I:%.*]] = getelementptr inbounds nuw i8, ptr [[I_I]], i64 1
 ; O2-NEXT:    store ptr [[I3_I]], ptr [[ARG]], align 8
 ; O2-NEXT:    [[I3:%.*]] = load i32, ptr [[I_I]], align 4
 ; O2-NEXT:    ret i32 [[I3]]
diff --git a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
index d5a422ad41f55..4a19940b6f874 100644
--- a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
+++ b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
@@ -19,9 +19,9 @@ define void @test1(i32 %d, ptr %p) nounwind uwtable ssp {
 ; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.body> U: [0,64) S: [0,64) Exits: 63 LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %p.addr.02 = phi ptr [ %p, %entry ], [ %add.ptr1, %for.body ]
 ; CHECK-NEXT:    --> {%p,+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((504 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:    %add.ptr = getelementptr inbounds i32, ptr %p.addr.02, i32 %div1
-; CHECK-NEXT:    --> {((4 * (%d /u 4))<nuw><nsw> + %p),+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:    %add.ptr1 = getelementptr inbounds i32, ptr %add.ptr, i32 %div1
+; CHECK-NEXT:    %add.ptr = getelementptr inbounds nuw i32, ptr %p.addr.02, i32 %div1
+; CHECK-NEXT:    --> {((4 * (%d /u 4))<nuw><nsw> + %p)<nuw>,+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    %add.ptr1 = getelementptr inbounds nuw i32, ptr %add.ptr, i32 %div1
 ; CHECK-NEXT:    --> {((8 * (%d /u 4)) + %p),+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((512 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %inc = add nuw nsw i32 %i.03, 1
 ; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable }
@@ -66,9 +66,9 @@ define void @test1a(i32 %d, ptr %p) nounwind uwtable ssp {
 ; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.body> U: [0,64) S: [0,64) Exits: 63 LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %p.addr.02 = phi ptr [ %p, %entry ], [ %add.ptr1, %for.body ]
 ; CHECK-NEXT:    --> {%p,+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((504 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:    %add.ptr = getelementptr inbounds i32, ptr %p.addr.02, i32 %div1
-; CHECK-NEXT:    --> {((4 * (%d /u 2))<nuw><nsw> + %p),+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:    %add.ptr1 = getelementptr inbounds i32, ptr %add.ptr, i32 %div1
+; CHECK-NEXT:    %add.ptr = getelementptr inbounds nuw i32, ptr %p.addr.02, i32 %div1
+; CHECK-NEXT:    --> {((4 * (%d /u 2))<nuw><nsw> + %p)<nuw>,+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    %add.ptr1 = getelementptr inbounds nuw i32, ptr %add.ptr, i32 %div1
 ; CHECK-NEXT:    --> {((8 * (%d /u 2)) + %p),+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((512 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %inc = add nuw nsw i32 %i.03, 1
 ; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable }
@@ -199,7 +199,7 @@ define void @test_range_ref1(i8 %t) {
 ; CHECK-NEXT:    --> %t.ptr U: [0,256) S: [0,256)
 ; CHECK-NEXT:    %idx = phi ptr [ %t.ptr, %entry ], [ %snext, %loop ]
 ; CHECK-NEXT:    --> {%t.ptr,+,1}<nuw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
-; CHECK-NEXT:    %snext = getelementptr inbounds i8, ptr %idx, i32 1
+; CHECK-NEXT:    %snext = getelementptr inbounds nuw i8, ptr %idx, i32 1
 ; CHECK-NEXT:    --> {(1 + %t.ptr)<nuw><nsw>,+,1}<nw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @test_range_ref1
 ; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
diff --git a/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
index f5807c2105abe..ba71299d6919e 100644
--- a/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
+++ b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
@@ -9,7 +9,7 @@ define i1 @PR33605(i32 %a, i32 %b, ptr %c) {
 ; CHECK-LABEL: @PR33605(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[C:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[C:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[OR]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/single-iteration-loop-sroa.ll b/llvm/test/Transforms/PhaseOrdering/single-iteration-loop-sroa.ll
index 3975a0dd33748..cc3cd56616f32 100644
--- a/llvm/test/Transforms/PhaseOrdering/single-iteration-loop-sroa.ll
+++ b/llvm/test/Transforms/PhaseOrdering/single-iteration-loop-sroa.ll
@@ -11,13 +11,13 @@ define i16 @helper(i16 %0, i64 %x) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[DATA:%.*]] = alloca [2 x i8], align 2
 ; CHECK-NEXT:    store i16 [[TMP0:%.*]], ptr [[DATA]], align 2
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 1
 ; CHECK-NEXT:    br label [[BB6_I_I:%.*]]
 ; CHECK:       bb6.i.i:
 ; CHECK-NEXT:    [[ITER_SROA_0_07_I_I:%.*]] = phi i64 [ [[TMP2:%.*]], [[BB6_I_I]] ], [ 0, [[START:%.*]] ]
 ; CHECK-NEXT:    [[_40_I_I:%.*]] = sub nsw i64 0, [[ITER_SROA_0_07_I_I]]
 ; CHECK-NEXT:    [[TMP2]] = add nuw nsw i64 [[ITER_SROA_0_07_I_I]], 1
-; CHECK-NEXT:    [[_34_I_I:%.*]] = getelementptr inbounds [0 x i8], ptr [[DATA]], i64 0, i64 [[ITER_SROA_0_07_I_I]]
+; CHECK-NEXT:    [[_34_I_I:%.*]] = getelementptr inbounds nuw [0 x i8], ptr [[DATA]], i64 0, i64 [[ITER_SROA_0_07_I_I]]
 ; CHECK-NEXT:    [[_39_I_I:%.*]] = getelementptr inbounds [0 x i8], ptr [[TMP1]], i64 0, i64 [[_40_I_I]]
 ; CHECK-NEXT:    [[TMP_0_COPYLOAD_I_I_I_I:%.*]] = load i8, ptr [[_34_I_I]], align 1
 ; CHECK-NEXT:    [[TMP2_0_COPYLOAD_I_I_I_I:%.*]] = load i8, ptr [[_39_I_I]], align 1
diff --git a/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll b/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
index aba003b13b459..d2f33f9c3b754 100644
--- a/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
+++ b/llvm/test/Transforms/PhaseOrdering/switch_with_geps.ll
@@ -14,7 +14,7 @@ define i32 @test(ptr %ptr) {
 ; CHECK-LABEL: define i32 @test(
 ; CHECK-SAME: ptr nocapture readonly [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[PHI:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 4
+; CHECK-NEXT:    [[PHI:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 4
 ; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[PHI]], align 4
 ; CHECK-NEXT:    ret i32 [[RET]]
 ;
@@ -59,7 +59,7 @@ define void @test2(ptr %self, i64 %v, i64 %ix) {
 ; CHECK-SAME: ptr nocapture writeonly [[SELF:%.*]], i64 [[V:%.*]], i64 [[IX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = shl i64 [[IX]], 3
-; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i8, ptr [[SELF]], i64 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds nuw i8, ptr [[SELF]], i64 [[SWITCH_TABLEIDX]]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[GEP5]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll b/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
index 4cde69bb0c7dd..80a9e77030e64 100644
--- a/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
+++ b/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 target triple = "i386-gnu-linux"
 
-define void @exp_averages_intraday__deviation() {
+define void @exp_averages_intraday__deviation(i1 %arg) {
 entry:
   %0 = load i32, ptr undef, align 4
   %1 = shl i32 %0, 2
@@ -16,14 +16,14 @@ entry:
   br i1 false, label %"4", label %"12"
 
 "4":                                              ; preds = %entry
-  br i1 undef, label %"5", label %"8"
+  br i1 %arg, label %"5", label %"8"
 
 "5":                                              ; preds = %"4"
   unreachable
 
 "8":                                              ; preds = %"4"
   %8 = getelementptr inbounds i8, ptr undef, i32 %6
-  br i1 undef, label %"13", label %"12"
+  br i1 %arg, label %"13", label %"12"
 
 "12":                                             ; preds = %"8", %entry
   ret void
diff --git a/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll b/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
index 650aa82c1af2f..ba730758a1f26 100644
--- a/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
+++ b/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
@@ -1,14 +1,14 @@
 ; RUN: opt < %s -passes=reassociate -disable-output
 ; PR13041
 
-define void @foo() {
+define void @foo(i1 %arg) {
 entry:
   br label %while.cond
 
 while.cond:                                       ; preds = %while.body, %entry
   %b.0 = phi i32 [ undef, %entry ], [ %sub2, %while.body ]
   %c.0 = phi i32 [ undef, %entry ], [ %sub3, %while.body ]
-  br i1 undef, label %while.end, label %while.body
+  br i1 %arg, label %while.end, label %while.body
 
 while.body:                                       ; preds = %while.cond
   %sub = sub nsw i32 0, %b.0
diff --git a/llvm/test/Transforms/Reassociate/add_across_block_crash.ll b/llvm/test/Transforms/Reassociate/add_across_block_crash.ll
index 26e971266d344..a89f81154f3eb 100644
--- a/llvm/test/Transforms/Reassociate/add_across_block_crash.ll
+++ b/llvm/test/Transforms/Reassociate/add_across_block_crash.ll
@@ -3,10 +3,10 @@
 ; This test is to make sure while processing a block, uses of instructions
 ; from a different basic block don't get added to be re-optimized
 
-define  void @main() {
+define  void @main(i1 %arg) {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label %bb1, label %bb2
+; CHECK-NEXT:    br i1 %arg, label %bb1, label %bb2
 ; CHECK:       bb1:
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb2:
@@ -14,7 +14,7 @@ define  void @main() {
 ;
 entry:
   %0 = fadd fast float undef, undef
-  br i1 undef, label %bb1, label %bb2
+  br i1 %arg, label %bb1, label %bb2
 
 bb1:
   %1 = fmul fast float undef, -2.000000e+00
diff --git a/llvm/test/Transforms/Reassociate/infloop-deadphi.ll b/llvm/test/Transforms/Reassociate/infloop-deadphi.ll
index 5b19f8d384f4d..3202f450b7566 100644
--- a/llvm/test/Transforms/Reassociate/infloop-deadphi.ll
+++ b/llvm/test/Transforms/Reassociate/infloop-deadphi.ll
@@ -3,14 +3,14 @@
 
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @f() {
+define void @f(i1 %arg) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[DONE:%.*]]
 ; CHECK:       dead:
 ; CHECK-NEXT:    [[XOR0:%.*]] = xor i16 [[XOR1:%.*]], undef
 ; CHECK-NEXT:    [[XOR1]] = xor i16 [[XOR0]], undef
-; CHECK-NEXT:    br i1 undef, label [[DEAD:%.*]], label [[DONE]]
+; CHECK-NEXT:    br i1 %arg, label [[DEAD:%.*]], label [[DONE]]
 ; CHECK:       done:
 ; CHECK-NEXT:    ret void
 ;
@@ -20,7 +20,7 @@ entry:
 dead:
   %xor0 = xor i16 %xor1, undef
   %xor1 = xor i16 %xor0, undef
-  br i1 undef, label %dead, label %done
+  br i1 %arg, label %dead, label %done
 
 done:
   %e = phi i16 [ %xor1, %dead ], [ 0, %entry ]
diff --git a/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll b/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
index eb6a5cabb7be8..c5bd62e2763b6 100644
--- a/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
+++ b/llvm/test/Transforms/Reassociate/reassociate-landingpad.ll
@@ -14,7 +14,7 @@ declare i32 @__gxx_personality_v0(...)
 
 declare void @b() #0
 
-define void @a() #0 personality ptr @__gxx_personality_v0 {
+define void @a(i1 %arg) #0 personality ptr @__gxx_personality_v0 {
 ", bb1":
   invoke void @b()
           to label %invoke.cont unwind label %"bb22"
@@ -39,7 +39,7 @@ define void @a() #0 personality ptr @__gxx_personality_v0 {
   unreachable
 
 invoke.cont:                                      ; preds = %", bb1"
-  br i1 undef, label %", bb15", label %", bb8"
+  br i1 %arg, label %", bb15", label %", bb8"
 
 invoke.cont25:                                    ; preds = %", bb8"
   unreachable
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/intrinsics.ll b/llvm/test/Transforms/RewriteStatepointsForGC/intrinsics.ll
index 43632ccc04efb..1a45feac262c2 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/intrinsics.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/intrinsics.ll
@@ -15,21 +15,21 @@ define ptr addrspace(1) @test_simple(ptr addrspace(1) %obj1, ptr addrspace(1) %o
 ; CHECK-LABEL: define {{[^@]+}}@test_simple
 ; CHECK-SAME: (ptr addrspace(1) [[OBJ1:%.*]], ptr addrspace(1) [[OBJ2:%.*]], i32 [[LEN:%.*]], i1 [[C:%.*]]) gc "statepoint-example" {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OBJ1_12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OBJ1]], i64 12
-; CHECK-NEXT:    [[OBJ2_16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OBJ2]], i64 16
-; CHECK-NEXT:    [[OBJ_X_BASE1:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ1]], ptr addrspace(1) [[OBJ2]], !is_base_value !0
+; CHECK-NEXT:    [[OBJ1_12:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[OBJ1]], i64 12
+; CHECK-NEXT:    [[OBJ2_16:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[OBJ2]], i64 16
+; CHECK-NEXT:    [[OBJ_X_BASE1:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ1]], ptr addrspace(1) [[OBJ2]], !is_base_value [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[OBJ_X:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ1_12]], ptr addrspace(1) [[OBJ2_16]]
-; CHECK-NEXT:    [[OBJ_Y_BASE:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ2]], ptr addrspace(1) [[OBJ1]], !is_base_value !0
+; CHECK-NEXT:    [[OBJ_Y_BASE:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ2]], ptr addrspace(1) [[OBJ1]], !is_base_value [[META0]]
 ; CHECK-NEXT:    [[OBJ_Y:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ2_16]], ptr addrspace(1) [[OBJ1_12]]
 ; CHECK-NEXT:    [[OBJ_X_BASE1_INT:%.*]] = ptrtoint ptr addrspace(1) [[OBJ_X_BASE1]] to i64
 ; CHECK-NEXT:    [[OBJ_X_INT:%.*]] = ptrtoint ptr addrspace(1) [[OBJ_X]] to i64
 ; CHECK-NEXT:    [[OBJ_X_OFFSET:%.*]] = sub i64 [[OBJ_X_INT]], [[OBJ_X_BASE1_INT]]
 ; CHECK-NEXT:    [[OBJ_Y_BASE_INT:%.*]] = ptrtoint ptr addrspace(1) [[OBJ_Y_BASE]] to i64
-; CHECK-NEXT:    [[OBJ_YA_INT:%.*]] = ptrtoint ptr addrspace(1) [[OBJ_Y]] to i64
-; CHECK-NEXT:    [[OBJ_YA_OFFSET:%.*]] = sub i64 [[OBJ_YA_INT]], [[OBJ_Y_BASE_INT]]
+; CHECK-NEXT:    [[OBJ_Y_INT:%.*]] = ptrtoint ptr addrspace(1) [[OBJ_Y]] to i64
+; CHECK-NEXT:    [[OBJ_YA_OFFSET:%.*]] = sub i64 [[OBJ_Y_INT]], [[OBJ_Y_BASE_INT]]
 ; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr nonnull elementtype(void ()) @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(ptr addrspace(1) [[OBJ_X_BASE1]], i64 [[OBJ_X_OFFSET]], ptr addrspace(1) [[OBJ_X_BASE1]], i64 [[OBJ_X_OFFSET]], ptr addrspace(1) [[OBJ_Y_BASE]], i64 [[OBJ_YA_OFFSET]]), "gc-live"(ptr addrspace(1) [[OBJ_Y]], ptr addrspace(1) [[OBJ_Y_BASE]]) ]
-; CHECK-NEXT:    [[OBJ_YA_RELOCATED:%.*]] = call coldcc ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    ret ptr addrspace(1) [[OBJ_YA_RELOCATED]]
+; CHECK-NEXT:    [[OBJ_Y_RELOCATED:%.*]] = call coldcc nonnull ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    ret ptr addrspace(1) [[OBJ_Y_RELOCATED]]
 ;
 entry:
   %obj1.12 = getelementptr inbounds i8, ptr addrspace(1) %obj1, i64 12
@@ -68,10 +68,10 @@ define ptr addrspace(1) @test_chained(ptr addrspace(1) %obj1, ptr addrspace(1) %
 ; CHECK-LABEL: define {{[^@]+}}@test_chained
 ; CHECK-SAME: (ptr addrspace(1) [[OBJ1:%.*]], ptr addrspace(1) [[OBJ2:%.*]], i32 [[LEN:%.*]], i1 [[C:%.*]]) gc "statepoint-example" {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OBJ_X_BASE1:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ1]], ptr addrspace(1) [[OBJ2]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ_X_BASE_GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 8
-; CHECK-NEXT:    [[OBJ_X_BASE_OF_BASE_GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 20
-; CHECK-NEXT:    [[OBJ_X_BASE_OF_BASE_OF_BASE_GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 24
+; CHECK-NEXT:    [[OBJ_X_BASE1:%.*]] = select i1 [[C]], ptr addrspace(1) [[OBJ1]], ptr addrspace(1) [[OBJ2]], !is_base_value [[META0]]
+; CHECK-NEXT:    [[OBJ_X_BASE_GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 8
+; CHECK-NEXT:    [[OBJ_X_BASE_OF_BASE_GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 20
+; CHECK-NEXT:    [[OBJ_X_BASE_OF_BASE_OF_BASE_GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[OBJ_X_BASE1]], i64 24
 ; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr nonnull elementtype(void ()) @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(ptr addrspace(1) [[OBJ_X_BASE1]], ptr addrspace(1) [[OBJ_X_BASE1]], ptr addrspace(1) [[OBJ_X_BASE1]], ptr addrspace(1) [[OBJ_X_BASE_GEP]], ptr addrspace(1) [[OBJ_X_BASE_OF_BASE_GEP]], ptr addrspace(1) [[OBJ_X_BASE_OF_BASE_OF_BASE_GEP]], ptr addrspace(1) [[OBJ_X_BASE1]], ptr addrspace(1) [[OBJ_X_BASE1]], ptr addrspace(1) [[OBJ_X_BASE1]], i64 0, i64 0, i64 0, i64 8, i64 20, i64 24, i64 0, i64 0, i64 0), "gc-live"(ptr addrspace(1) [[OBJ_X_BASE1]]) ]
 ; CHECK-NEXT:    [[OBJ_X_BASE1_RELOCATED:%.*]] = call coldcc ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret ptr addrspace(1) [[OBJ_X_BASE1_RELOCATED]]
diff --git a/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll b/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
index 2098b0390b64a..974c2dac78ebf 100644
--- a/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
+++ b/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=sccp -S | FileCheck %s
 
-; Branch on undef is UB, so the T block is never executed, and we can return
+; Branch on poison is UB, so the T block is never executed, and we can return
 ; undef (IPSCCP would replace the block with unreachable).
 
 define i32 @foo() {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    unreachable
 ;
-  br i1 undef, label %T, label %T
+  br i1 poison, label %T, label %T
 T:
   %X = add i32 0, 1
   ret i32 %X
diff --git a/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll b/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
index 3e2cfe1ce25fb..8c1687a3eb784 100644
--- a/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
+++ b/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
@@ -6,7 +6,7 @@ target triple = "powerpc-unknown-linux-gnu"
 @JUMP = external global i32		; <ptr> [#uses=1]
 @old_D_pat = external global [16 x i8]		; <ptr> [#uses=0]
 
-define void @asearch1(i32 %D) {
+define void @asearch1(i32 %D, i1 %arg) {
 entry:
 	%tmp80 = icmp ult i32 0, %D		; <i1> [#uses=1]
 	br i1 %tmp80, label %bb647.preheader, label %cond_true81.preheader
@@ -22,7 +22,7 @@ cond_true612:		; preds = %cond_true654
 cond_next624:		; preds = %cond_true654
 	ret void
 cond_true654:		; preds = %bb647.preheader
-	br i1 undef, label %cond_true612, label %cond_next624
+	br i1 %arg, label %cond_true612, label %cond_next624
 UnifiedReturnBlock:		; preds = %bb647.preheader
 	ret void
 }
diff --git a/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll b/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
index 6f499f27a65a9..895b1393dadae 100644
--- a/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
+++ b/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
@@ -2,12 +2,30 @@
 ; RUN: opt < %s -passes=sccp -S | FileCheck %s
 ; PR1938
 
-define i32 @main() {
+define i32 @main(i1 %arg) {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB:%.*]]
 ; CHECK:       bb:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[K:%.*]], [[BB_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[K]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp slt i32 [[K]], 10
+; CHECK-NEXT:    br i1 [[TMP97]], label [[BB_BACKEDGE]], label [[BB12:%.*]]
+; CHECK:       bb.backedge:
+; CHECK-NEXT:    br label [[BB]]
+; CHECK:       cond_false:
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i32 [[K]], 10
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB_BACKEDGE]], label [[BB12]]
+; CHECK:       bb12:
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[K]], 10
+; CHECK-NEXT:    br i1 [[TMP14]], label [[COND_NEXT18:%.*]], label [[COND_TRUE17:%.*]]
+; CHECK:       cond_true17:
+; CHECK-NEXT:    tail call void @abort()
 ; CHECK-NEXT:    unreachable
+; CHECK:       cond_next18:
+; CHECK-NEXT:    ret i32 0
 ;
 entry:
   br label %bb
@@ -15,7 +33,7 @@ entry:
 bb:
   %indvar = phi i32 [ 0, %entry ], [ %k, %bb.backedge ]
   %k = add i32 %indvar, 1
-  br i1 undef, label %cond_true, label %cond_false
+  br i1 %arg, label %cond_true, label %cond_false
 
 cond_true:
   %tmp97 = icmp slt i32 %k, 10
diff --git a/llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll
index ac2e945b125ba..817a2846143fa 100644
--- a/llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll
+++ b/llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll
@@ -31,7 +31,7 @@ return:                                           ; preds = %entry
 define internal i32 @vfu2(ptr byval(%struct.MYstr) align 4 %u) nounwind readonly {
 ; CHECK-LABEL: @vfu2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr getelementptr inbounds (i8, ptr @mystr, i64 4), align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @mystr, i64 4), align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @mystr, align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[TMP0]]
diff --git a/llvm/test/Transforms/SCCP/PR26044.ll b/llvm/test/Transforms/SCCP/PR26044.ll
index 90ac3101d0c23..f786629f47d0e 100644
--- a/llvm/test/Transforms/SCCP/PR26044.ll
+++ b/llvm/test/Transforms/SCCP/PR26044.ll
@@ -3,13 +3,13 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @fn2(ptr %P) {
+define void @fn2(ptr %P, i1 %arg) {
 ; CHECK-LABEL: define {{[^@]+}}@fn2
-; CHECK-SAME: (ptr [[P:%.*]]) {
+; CHECK-SAME: (ptr [[P:%.*]], i1 [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       for.cond1:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br i1 [[ARG]], label [[IF_END]], label [[IF_END]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn1(i32 undef)
 ; CHECK-NEXT:    store i32 [[CALL]], ptr [[P]], align 4
@@ -19,7 +19,7 @@ entry:
   br label %if.end
 
 for.cond1:                                        ; preds = %if.end, %for.end
-  br i1 undef, label %if.end, label %if.end
+  br i1 %arg, label %if.end, label %if.end
 
 if.end:                                           ; preds = %lbl, %for.cond1
   %e.2 = phi ptr [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
@@ -43,15 +43,16 @@ entry:
   ret i32 %cond
 }
 
-define void @fn_no_null_opt(ptr %P) #0 {
+define void @fn_no_null_opt(ptr %P, i1 %arg) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@fn_no_null_opt
-; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr [[P:%.*]], i1 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       for.cond1:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br i1 [[ARG]], label [[IF_END]], label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn0(i32 undef)
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr null, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn0(i32 [[TMP0]])
 ; CHECK-NEXT:    store i32 [[CALL]], ptr [[P]], align 4
 ; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
 ;
@@ -59,7 +60,7 @@ entry:
   br label %if.end
 
 for.cond1:                                        ; preds = %if.end, %for.end
-  br i1 undef, label %if.end, label %if.end
+  br i1 %arg, label %if.end, label %if.end
 
 if.end:                                           ; preds = %lbl, %for.cond1
   %e.2 = phi ptr [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
@@ -73,8 +74,8 @@ define internal i32 @fn0(i32 %p1) {
 ; CHECK-LABEL: define {{[^@]+}}@fn0
 ; CHECK-SAME: (i32 [[P1:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 undef, 0
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SCCP/apint-bigint2.ll b/llvm/test/Transforms/SCCP/apint-bigint2.ll
index 695d6a4cf056f..c5e7643cdd5ab 100644
--- a/llvm/test/Transforms/SCCP/apint-bigint2.ll
+++ b/llvm/test/Transforms/SCCP/apint-bigint2.ll
@@ -23,7 +23,7 @@ define i101 @large_aggregate() {
 ; CHECK-LABEL: @large_aggregate(
 ; CHECK-NEXT:    [[D:%.*]] = and i101 undef, 1
 ; CHECK-NEXT:    [[DD:%.*]] = or i101 [[D]], 1
-; CHECK-NEXT:    [[G:%.*]] = getelementptr i101, ptr getelementptr inbounds (i8, ptr @Y, i64 80), i101 [[DD]]
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i101, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 80), i101 [[DD]]
 ; CHECK-NEXT:    [[L3:%.*]] = load i101, ptr [[G]], align 4
 ; CHECK-NEXT:    ret i101 [[L3]]
 ;
@@ -40,7 +40,7 @@ define i101 @large_aggregate_2() {
 ; CHECK-LABEL: @large_aggregate_2(
 ; CHECK-NEXT:    [[D:%.*]] = and i101 undef, 1
 ; CHECK-NEXT:    [[DD:%.*]] = or i101 [[D]], 1
-; CHECK-NEXT:    [[G:%.*]] = getelementptr i101, ptr getelementptr inbounds (i8, ptr @Y, i64 80), i101 [[DD]]
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i101, ptr getelementptr inbounds nuw (i8, ptr @Y, i64 80), i101 [[DD]]
 ; CHECK-NEXT:    [[L3:%.*]] = load i101, ptr [[G]], align 4
 ; CHECK-NEXT:    ret i101 [[L3]]
 ;
diff --git a/llvm/test/Transforms/SCCP/conditions-iter-order.ll b/llvm/test/Transforms/SCCP/conditions-iter-order.ll
index c7f977b8ab83a..1e8ba6197ad5f 100644
--- a/llvm/test/Transforms/SCCP/conditions-iter-order.ll
+++ b/llvm/test/Transforms/SCCP/conditions-iter-order.ll
@@ -13,17 +13,17 @@ define internal ptr @spam(ptr %arg) {
 ; CHECK-NEXT:    [[TMP:%.*]] = call ptr @malloc(i64 10368)
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARG:%.*]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG]], i32 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne ptr [[TMP7]], null
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[BB17:%.*]], label [[BB13:%.*]]
 ; CHECK:       bb13:
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i32 2
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG]], i32 2
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 1
 ; CHECK-NEXT:    br label [[BB30:%.*]]
 ; CHECK:       bb17:
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP6]], [[TMP5]]
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i32 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i32, ptr [[ARG]], i32 3
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 8
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[BB30]], label [[BB13]]
 ; CHECK:       bb30:
diff --git a/llvm/test/Transforms/SCCP/crash.ll b/llvm/test/Transforms/SCCP/crash.ll
index 47d9329f6f03d..9001b42850c25 100644
--- a/llvm/test/Transforms/SCCP/crash.ll
+++ b/llvm/test/Transforms/SCCP/crash.ll
@@ -2,9 +2,9 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin10.0"
 
-define void @test1(i8 %arg) {
+define void @test1(i8 %arg, i1 %arg1) {
 entry:
-  br i1 undef, label %return, label %bb
+  br i1 %arg1, label %return, label %bb
 
 bb:   
   br label %bb34
diff --git a/llvm/test/Transforms/SCCP/domtree-update.ll b/llvm/test/Transforms/SCCP/domtree-update.ll
index 76f575c4e9c6d..270da0e2f2bfe 100644
--- a/llvm/test/Transforms/SCCP/domtree-update.ll
+++ b/llvm/test/Transforms/SCCP/domtree-update.ll
@@ -4,7 +4,7 @@
 
 ; DTU should not crash.
 
-define i32 @test() {
+define i32 @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -25,10 +25,10 @@ if.then2:                                         ; preds = %for.body
   br label %for.inc
 
 if.else:                                          ; preds = %for.body
-  br i1 undef, label %lor.rhs, label %if.then19.critedge
+  br i1 %arg, label %lor.rhs, label %if.then19.critedge
 
 lor.rhs:                                          ; preds = %if.else
-  br i1 undef, label %if.then19, label %for.inc
+  br i1 %arg, label %if.then19, label %for.inc
 
 if.then19.critedge:                               ; preds = %if.else
   br label %if.then19
diff --git a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
index 8061a0396ee4c..6a8b52d0ac481 100644
--- a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
@@ -2,18 +2,18 @@
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64le-unknown-linux"
 
-define void @test(i32 signext %n) {
+define void @test(i32 signext %n, i1 %arg) {
 
 ; CHECK-LABEL: @test
 
 entry:
-  br i1 undef, label %if.then, label %if.end
+  br i1 %arg, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
   ret void
 
 if.end:                                           ; preds = %entry
-  br i1 undef, label %if.then2, label %if.end4
+  br i1 %arg, label %if.then2, label %if.end4
 
 if.then2:                                         ; preds = %if.end
   unreachable
@@ -36,10 +36,10 @@ if.else14:                                        ; preds = %if.end4
 
 do.body:                                          ; preds = %do.body, %if.else14
   %scale.0 = phi ppc_fp128 [ 0xM3FF00000000000000000000000000000, %if.else14 ], [ %scale.0, %do.body ]
-  br i1 undef, label %do.body, label %if.then33
+  br i1 %arg, label %do.body, label %if.then33
 
 if.then33:                                        ; preds = %do.body
-  br i1 undef, label %_ZN5boost4math4signIgEEiRKT_.exit30, label %cond.false.i28
+  br i1 %arg, label %_ZN5boost4math4signIgEEiRKT_.exit30, label %cond.false.i28
 
 cond.false.i28:                                   ; preds = %if.then33
   %0 = bitcast ppc_fp128 %scale.0 to i128
diff --git a/llvm/test/Transforms/SCCP/gep-nuw.ll b/llvm/test/Transforms/SCCP/gep-nuw.ll
new file mode 100644
index 0000000000000..4d25642eb8575
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/gep-nuw.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=sccp < %s | FileCheck %s
+
+define ptr @gep_nusw_nneg(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: define ptr @gep_nusw_nneg(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr nusw nuw [1 x i8], ptr [[P]], i64 [[X_EXT]], i64 [[Y_EXT]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %gep = getelementptr nusw [1 x i8], ptr %p, i64 %x.ext, i64 %y.ext
+  ret ptr %gep
+}
+
+define ptr @gep_inbounds_nneg(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: define ptr @gep_inbounds_nneg(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [1 x i8], ptr [[P]], i64 [[X_EXT]], i64 [[Y_EXT]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %gep = getelementptr inbounds [1 x i8], ptr %p, i64 %x.ext, i64 %y.ext
+  ret ptr %gep
+}
+
+define ptr @gep_nothin_nneg(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: define ptr @gep_nothin_nneg(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i32 [[Y]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [1 x i8], ptr [[P]], i64 [[X_EXT]], i64 [[Y_EXT]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %x.ext = zext i32 %x to i64
+  %y.ext = zext i32 %y to i64
+  %gep = getelementptr [1 x i8], ptr %p, i64 %x.ext, i64 %y.ext
+  ret ptr %gep
+}
+
+define ptr @gep_nusw_not_nneg(ptr %p, i32 %x, i64 %y) {
+; CHECK-LABEL: define ptr @gep_nusw_not_nneg(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_EXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr nusw [1 x i8], ptr [[P]], i64 [[X_EXT]], i64 [[Y]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %x.ext = zext i32 %x to i64
+  %gep = getelementptr nusw [1 x i8], ptr %p, i64 %x.ext, i64 %y
+  ret ptr %gep
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-preserve-pdt.ll b/llvm/test/Transforms/SCCP/ipsccp-preserve-pdt.ll
index f8c8e33dfc233..be05d96f08574 100644
--- a/llvm/test/Transforms/SCCP/ipsccp-preserve-pdt.ll
+++ b/llvm/test/Transforms/SCCP/ipsccp-preserve-pdt.ll
@@ -20,16 +20,17 @@
 ; CHECK-NEXT:    [2] %for.body {4294967295,4294967295} [1]
 ; CHECK-NEXT:    [2] %if.end4 {4294967295,4294967295} [1]
 ; CHECK-NEXT:      [3] %entry {4294967295,4294967295} [2]
-; CHECK-NEXT:    [2] %for.cond34 {4294967295,4294967295} [1]
-; CHECK-NEXT:      [3] %for.cond16 {4294967295,4294967295} [2]
-; CHECK-NEXT: Roots: %for.body %for.cond34
+; CHECK-NEXT:    [2] %for.body37 {4294967295,4294967295} [1]
+; CHECK-NEXT:      [3] %for.cond34 {4294967295,4294967295} [2]
+; CHECK-NEXT:        [4] %for.cond16 {4294967295,4294967295} [3]
+; CHECK-NEXT: Roots: %for.body %for.body37
 ; CHECK-NEXT: PostDominatorTree for function: bar
 ; CHECK-NOT: <badref>
 
 declare hidden i1 @compare(ptr) align 2
 declare hidden { i8, ptr } @getType(ptr) align 2
 
-define internal void @foo(ptr %TLI, ptr %DL, ptr %Ty, ptr %ValueVTs, ptr %Offsets, i64 %StartingOffset) {
+define internal void @foo(ptr %TLI, ptr %DL, ptr %Ty, ptr %ValueVTs, ptr %Offsets, i64 %StartingOffset, i1 %arg) {
 entry:
   %VT = alloca i64, align 8
   br i1 false, label %if.then, label %if.end4
@@ -51,7 +52,7 @@ for.cond16:                                       ; preds = %for.cond34, %if.end
   br label %for.cond34
 
 for.cond34:                                       ; preds = %for.body37, %for.cond16
-  br i1 undef, label %for.body37, label %for.cond16
+  br i1 %arg, label %for.body37, label %for.cond16
 
 for.body37:                                       ; preds = %for.cond34
   %tobool39 = icmp ne ptr %Offsets, null
diff --git a/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll b/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
index 55a7fdf7197e5..752e503803310 100644
--- a/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
+++ b/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
@@ -15,11 +15,11 @@ define i32 @check(ptr %node) {
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[NODE]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @check(ptr [[TMP0]])
-; CHECK-NEXT:    [[RIGHT:%.*]] = getelementptr inbounds [[STRUCT_NODE:%.*]], ptr [[NODE]], i32 0, i32 1
+; CHECK-NEXT:    [[RIGHT:%.*]] = getelementptr inbounds nuw [[STRUCT_NODE:%.*]], ptr [[NODE]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[RIGHT]], align 8
 ; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @check(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[RIGHT]], align 8
-; CHECK-NEXT:    [[HEIGHT:%.*]] = getelementptr inbounds [[STRUCT_NODE]], ptr [[TMP2]], i32 0, i32 2
+; CHECK-NEXT:    [[HEIGHT:%.*]] = getelementptr inbounds nuw [[STRUCT_NODE]], ptr [[TMP2]], i32 0, i32 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[HEIGHT]], align 4
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[TMP3]], [[CALL1]]
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END5:%.*]]
diff --git a/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll b/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
index aeaaf995efb96..111a1c35f414b 100644
--- a/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
+++ b/llvm/test/Transforms/SCCP/pr45185-range-predinfo.ll
@@ -1,15 +1,16 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -passes=ipsccp -S %s | FileCheck %s
 
 ;Test for PR45185.
 
 define void @spam(ptr %arg) {
-; CHECK-LABEL: @spam(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    call void @blam(i32 0, ptr nonnull [[ARG:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [4 x [24 x float]], ptr [[ARG]], i64 0, i64 1, i64 0
+; CHECK-LABEL: define void @spam(
+; CHECK-SAME: ptr [[ARG:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    call void @blam(i32 0, ptr nonnull [[ARG]])
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [4 x [24 x float]], ptr [[ARG]], i64 0, i64 1, i64 0
 ; CHECK-NEXT:    call void @blam(i32 1, ptr nonnull [[TMP1]])
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [4 x [24 x float]], ptr [[ARG]], i64 0, i64 2, i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [4 x [24 x float]], ptr [[ARG]], i64 0, i64 2, i64 0
 ; CHECK-NEXT:    call void @blam(i32 2, ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
@@ -24,18 +25,19 @@ bb:
 
 ; Make sure we do not incorrectly eliminate the checks in @blam.
 define internal void @blam(i32 %arg, ptr nocapture %arg1) {
-; CHECK-LABEL: define {{.*}} @blam(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP:%.*]] = icmp eq i32 [[ARG:%.*]], 0
-; CHECK-NEXT:    br i1 [[TMP]], label [[BB2:%.*]], label [[BB3:%.*]]
-; CHECK:       bb2:
-; CHECK-NEXT:    br label [[BB5:%.*]]
-; CHECK:       bb3:
+; CHECK-LABEL: define internal void @blam(
+; CHECK-SAME: i32 range(i32 0, 3) [[ARG:%.*]], ptr nocapture [[ARG1:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[TMP:%.*]] = icmp eq i32 [[ARG]], 0
+; CHECK-NEXT:    br i1 [[TMP]], label %[[BB2:.*]], label %[[BB3:.*]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    br label %[[BB5:.*]]
+; CHECK:       [[BB3]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[ARG]], 2
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB5]], label [[BB6:%.*]]
-; CHECK:       bb5:
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[BB5]], label %[[BB6:.*]]
+; CHECK:       [[BB5]]:
 ; CHECK-NEXT:    ret void
-; CHECK:       bb6:
+; CHECK:       [[BB6]]:
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/SCCP/pr49582-iterator-invalidation.ll b/llvm/test/Transforms/SCCP/pr49582-iterator-invalidation.ll
index ed5703da35e6f..1ba6e9734e642 100644
--- a/llvm/test/Transforms/SCCP/pr49582-iterator-invalidation.ll
+++ b/llvm/test/Transforms/SCCP/pr49582-iterator-invalidation.ll
@@ -5,7 +5,7 @@
 @c = external dso_local global ptr, align 8
 @d = external dso_local global i32, align 4
 
-define void @f(i32 %i) {
+define void @f(i32 %i, i1 %arg) {
 entry:
   br label %for.cond
 
@@ -474,7 +474,7 @@ if.then312:                                       ; preds = %if.then309
   br label %if.end628
 
 if.else316:                                       ; preds = %if.then309
-  br i1 undef, label %if.then318, label %if.end628
+  br i1 %arg, label %if.then318, label %if.end628
 
 if.then318:                                       ; preds = %if.else316
   %idxprom320 = sext i32 %add310 to i64
@@ -726,7 +726,7 @@ if.then499:                                       ; preds = %if.else496
   br label %if.end628
 
 if.else501:                                       ; preds = %if.else496
-  br i1 undef, label %if.then503, label %if.end628
+  br i1 %arg, label %if.then503, label %if.end628
 
 if.then503:                                       ; preds = %if.else501
   br label %if.end628
@@ -834,7 +834,7 @@ if.then596:                                       ; preds = %if.then593
   br label %if.end628
 
 if.else600:                                       ; preds = %if.then593
-  br i1 undef, label %if.then602, label %if.end628
+  br i1 %arg, label %if.then602, label %if.end628
 
 if.then602:                                       ; preds = %if.else600
   %idxprom604 = sext i32 %add594 to i64
diff --git a/llvm/test/Transforms/SCCP/return-zapped.ll b/llvm/test/Transforms/SCCP/return-zapped.ll
index 6d70500125093..cafc8aa69e868 100644
--- a/llvm/test/Transforms/SCCP/return-zapped.ll
+++ b/llvm/test/Transforms/SCCP/return-zapped.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt < %s -S -passes=ipsccp | FileCheck %s
 
-; testf() performs an unconditional branch on undef, as such the testf() return
+; testf() performs an unconditional branch on poison, as such the testf() return
 ; value used in test1() will remain "unknown" and the following branch on it
 ; replaced by unreachable. This is fine, as the call to testf() will already
 ; trigger undefined behavior.
@@ -31,7 +31,7 @@ define internal i1 @testf() {
 ; CHECK-NEXT:    unreachable
 ;
 entry:
-  br i1 undef, label %if.then1, label %if.end3
+  br i1 poison, label %if.then1, label %if.end3
 
 if.then1:                                         ; preds = %if.end
   br label %if.end3
diff --git a/llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll
index 05f3358045c3c..a4b1ba8a0c8dd 100644
--- a/llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll
+++ b/llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll
@@ -16,7 +16,7 @@ entry:
   br i1 %c, label %if.cond, label %if.end
 
 if.cond:
-  br i1 undef, label %if.then, label %if.end
+  br i1 poison, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry, %if.then
   ret i32 11
diff --git a/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll b/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
index 2336c9186636e..73eef205adae7 100644
--- a/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
+++ b/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
@@ -59,13 +59,23 @@ bb38:                                             ; preds = %bb16
 }
 
 
-define void @hoge() {
-; CHECK-LABEL: define {{[^@]+}}@hoge() {
+define void @hoge(i1 %arg, i16 %arg2) {
+; CHECK-LABEL: define {{[^@]+}}@hoge
+; CHECK-SAME: (i1 [[ARG:%.*]], i16 [[ARG2:%.*]]) {
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    switch i16 [[ARG2]], label [[BB1:%.*]] [
+; CHECK-NEXT:      i16 135, label [[BB2:%.*]]
+; CHECK-NEXT:      i16 66, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
 ; CHECK-NEXT:    unreachable
 ;
 bb:
-  switch i16 undef, label %bb1 [
+  switch i16 %arg2, label %bb1 [
   i16 135, label %bb2
   i16 66, label %bb2
   ]
@@ -89,14 +99,9 @@ bb4:                                              ; preds = %bb2, %bb2, %bb2
 
 ; Test case from PR49573. %default.bb is unfeasible. Make sure it gets replaced
 ; by an unreachable block.
-define void @pr49573_main() {
-; CHECK-LABEL: define {{[^@]+}}@pr49573_main() {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TGT:%.*]] = call i16 @pr49573_fn()
-; CHECK-NEXT:    unreachable
-;
+define void @pr49573_main(i1 %arg) {
 entry:
-  %tgt = call i16 @pr49573_fn()
+  %tgt = call i16 @pr49573_fn(i1 %arg)
   switch i16 %tgt, label %default.bb [
   i16 0, label %case.0
   i16 1, label %case.1
@@ -116,7 +121,7 @@ case.2:
   br label %next
 
 next:
-  %tgt.2 = call i16 @pr49573_fn_2()
+  %tgt.2 = call i16 @pr49573_fn_2(i1 %arg)
   switch i16 %tgt.2, label %default.bb [
   i16 0, label %case.0
   i16 2, label %case.2
@@ -124,14 +129,9 @@ next:
 }
 
 ; Make sure a new unreachable BB is created.
-define void @pr49573_main_2() {
-; CHECK-LABEL: define {{[^@]+}}@pr49573_main_2() {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TGT:%.*]] = call i16 @pr49573_fn()
-; CHECK-NEXT:    unreachable
-;
+define void @pr49573_main_2(i1 %arg) {
 entry:
-  %tgt = call i16 @pr49573_fn()
+  %tgt = call i16 @pr49573_fn(i1 %arg)
   switch i16 %tgt, label %default.bb [
   i16 0, label %case.0
   i16 1, label %case.1
@@ -151,13 +151,18 @@ case.2:
   ret void
 }
 
-define internal i16 @pr49573_fn() {
-; CHECK-LABEL: define {{[^@]+}}@pr49573_fn() {
+define internal i16 @pr49573_fn(i1 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@pr49573_fn
+; CHECK-SAME: (i1 [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br i1 [[ARG]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       else:
+; CHECK-NEXT:    ret i16 2
 ;
 entry:
-  br i1 undef, label %then, label %else
+  br i1 %arg, label %then, label %else
 
 then:
   ret i16 0
@@ -166,13 +171,18 @@ else:
   ret i16 2
 }
 
-define internal i16 @pr49573_fn_2() {
-; CHECK-LABEL: define {{[^@]+}}@pr49573_fn_2() {
+define internal i16 @pr49573_fn_2(i1 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@pr49573_fn_2
+; CHECK-SAME: (i1 [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br i1 [[ARG]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       else:
+; CHECK-NEXT:    ret i16 2
 ;
 entry:
-  br i1 undef, label %then, label %else
+  br i1 %arg, label %then, label %else
 
 then:
   ret i16 0
diff --git a/llvm/test/Transforms/SCCP/widening.ll b/llvm/test/Transforms/SCCP/widening.ll
index 144c3fd54e75b..8c661bee8f599 100644
--- a/llvm/test/Transforms/SCCP/widening.ll
+++ b/llvm/test/Transforms/SCCP/widening.ll
@@ -566,7 +566,7 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias
 define linkonce_odr dereferenceable(1) ptr @spam(ptr %arg, i32 %arg1) align 2 {
 ; SCCP-LABEL: @spam(
 ; SCCP-NEXT:  bb:
-; SCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
+; SCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
 ; SCCP-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
 ; SCCP-NEXT:    [[TMP3:%.*]] = sext i32 [[ARG1:%.*]] to i64
 ; SCCP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP3]]
@@ -574,7 +574,7 @@ define linkonce_odr dereferenceable(1) ptr @spam(ptr %arg, i32 %arg1) align 2 {
 ;
 ; IPSCCP-LABEL: @spam(
 ; IPSCCP-NEXT:  bb:
-; IPSCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
+; IPSCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
 ; IPSCCP-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
 ; IPSCCP-NEXT:    [[TMP3:%.*]] = sext i32 [[ARG1:%.*]] to i64
 ; IPSCCP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP3]]
@@ -595,7 +595,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; SCCP-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP]], [[ARG1]]
 ; SCCP-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 65535
 ; SCCP-NEXT:    [[TMP4:%.*]] = mul i32 [[ARG1]], 8
-; SCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
+; SCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
 ; SCCP-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 8
 ; SCCP-NEXT:    [[TMP7:%.*]] = and i32 [[TMP4]], [[TMP6]]
 ; SCCP-NEXT:    br label [[BB8:%.*]]
@@ -609,7 +609,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; SCCP-NEXT:    [[TMP15:%.*]] = add i32 [[TMP7]], [[TMP11]]
 ; SCCP-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
 ; SCCP-NEXT:    [[TMP17]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP16]])
-; SCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 2
+; SCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 2
 ; SCCP-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; SCCP-NEXT:    [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
 ; SCCP-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[TMP21]], 0
@@ -632,7 +632,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; SCCP-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP11]], 0
 ; SCCP-NEXT:    br i1 [[TMP34]], label [[BB35:%.*]], label [[BB37:%.*]]
 ; SCCP:       bb35:
-; SCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP32]], i64 1
+; SCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 1
 ; SCCP-NEXT:    br label [[BB66:%.*]]
 ; SCCP:       bb37:
 ; SCCP-NEXT:    [[C_2:%.*]] = icmp eq i32 [[TMP11]], 8
@@ -677,7 +677,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; IPSCCP-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP]], [[ARG1]]
 ; IPSCCP-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 65535
 ; IPSCCP-NEXT:    [[TMP4:%.*]] = mul i32 [[ARG1]], 8
-; IPSCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
+; IPSCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
 ; IPSCCP-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 8
 ; IPSCCP-NEXT:    [[TMP7:%.*]] = and i32 [[TMP4]], [[TMP6]]
 ; IPSCCP-NEXT:    br label [[BB8:%.*]]
@@ -691,7 +691,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; IPSCCP-NEXT:    [[TMP15:%.*]] = add i32 [[TMP7]], [[TMP11]]
 ; IPSCCP-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
 ; IPSCCP-NEXT:    [[TMP17]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP16]])
-; IPSCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 2
+; IPSCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 2
 ; IPSCCP-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; IPSCCP-NEXT:    [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
 ; IPSCCP-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[TMP21]], 0
@@ -714,7 +714,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
 ; IPSCCP-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP11]], 0
 ; IPSCCP-NEXT:    br i1 [[TMP34]], label [[BB35:%.*]], label [[BB37:%.*]]
 ; IPSCCP:       bb35:
-; IPSCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP32]], i64 1
+; IPSCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 1
 ; IPSCCP-NEXT:    br label [[BB66:%.*]]
 ; IPSCCP:       bb37:
 ; IPSCCP-NEXT:    [[C_2:%.*]] = icmp eq i32 [[TMP11]], 8
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
index ec9c01c99256b..e460f558f4723 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -16,8 +16,8 @@ target triple = "aarch64--linux-gnu"
 
 define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @gather_multiple_use(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i64 3
 ; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], splat (i32 15)
@@ -57,20 +57,20 @@ define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 @data = global [6 x [258 x i8]] zeroinitializer, align 1
 define void @gather_load(ptr noalias %ptr) {
 ; CHECK-LABEL: @gather_load(
-; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
-; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @data, i64 258), align 1
+; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 8
+; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 258), align 1
 ; CHECK-NEXT:    [[CONV150:%.*]] = zext i8 [[L0]] to i16
 ; CHECK-NEXT:    [[ADD152:%.*]] = add nuw nsw i16 [[CONV150]], 10
-; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @data, i64 517), align 1
+; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 517), align 1
 ; CHECK-NEXT:    [[CONV156:%.*]] = zext i8 [[L1]] to i16
 ; CHECK-NEXT:    [[ADD158:%.*]] = add nuw nsw i16 [[CONV156]], 20
-; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @data, i64 776), align 1
+; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 776), align 1
 ; CHECK-NEXT:    [[CONV162:%.*]] = zext i8 [[L2]] to i16
 ; CHECK-NEXT:    [[ADD164:%.*]] = add nuw nsw i16 [[CONV162]], 30
-; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @data, i64 1035), align 1
+; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 1035), align 1
 ; CHECK-NEXT:    [[CONV168:%.*]] = zext i8 [[L3]] to i16
 ; CHECK-NEXT:    [[ADD170:%.*]] = add nuw nsw i16 [[CONV168]], 40
 ; CHECK-NEXT:    store i16 [[ADD152]], ptr [[ARRAYIDX182]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
index 51c6673a14353..900d5f293b5b8 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
@@ -36,7 +36,7 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
@@ -109,7 +109,7 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
@@ -293,7 +293,7 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
@@ -366,7 +366,7 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 91c8db14a45aa..c43b79e138a30 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -64,7 +64,7 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP4]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext nneg i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[SUM_032]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
@@ -159,7 +159,7 @@ define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[TMP4]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 1cce52060c479..5808d64d925c6 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -25,8 +25,8 @@ define void @test_i16_extend(ptr %p.1, ptr %p.2, i32 %idx.i32) {
 ; CHECK-LABEL: @test_i16_extend(
 ; CHECK-NEXT:    [[P_0:%.*]] = load ptr, ptr @global, align 8
 ; CHECK-NEXT:    [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64
-; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
-; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds nuw i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds nuw i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[T53]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr [[T56]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
index 7622f9bc5c41d..9ce79e5ea356b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
@@ -133,7 +133,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -165,7 +165,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM15]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX32]], align 2
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX33]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX36]], align 2
@@ -254,9 +254,9 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[L1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 4
 ; CHECK-NEXT:    [[L2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -270,9 +270,9 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM9]]
 ; CHECK-NEXT:    [[L6:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; CHECK-NEXT:    [[L8:%.*]] = load i16, ptr [[Y:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
 ; CHECK-NEXT:    [[L9:%.*]] = load i16, ptr [[ARRAYIDX15]], align 2
-; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 4
 ; CHECK-NEXT:    [[L10:%.*]] = load i16, ptr [[ARRAYIDX16]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[L12:%.*]] = load i16, ptr [[ARRAYIDX20]], align 2
@@ -414,12 +414,12 @@ define i32 @reduce_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
@@ -677,7 +677,7 @@ entry:
 define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocapture noundef readonly %y, ptr nocapture noundef writeonly %z, i32 noundef %stride) {
 ; CHECK-LABEL: @store_blockstrided3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X:%.*]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD4:%.*]] = add nsw i32 [[STRIDE:%.*]], 1
 ; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[ADD4]] to i64
@@ -696,7 +696,7 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ADD26:%.*]] = add nsw i32 [[MUL21]], 1
 ; CHECK-NEXT:    [[IDXPROM27:%.*]] = sext i32 [[ADD26]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds nuw i8, ptr [[Y:%.*]], i64 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX35]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM5]]
 ; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM11]]
@@ -705,9 +705,9 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ARRAYIDX60:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM23]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX60]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds i8, ptr [[Z:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds nuw i8, ptr [[Z:%.*]], i64 4
 ; CHECK-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP3]], [[TMP0]]
-; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i8, ptr [[Z]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 24
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i32>, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i32>, ptr [[Y]], align 4
@@ -716,14 +716,14 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw <2 x i32> [[TMP9]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> [[TMP11]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[MUL81:%.*]] = mul nsw i32 [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[ARRAYIDX82:%.*]] = getelementptr inbounds i8, ptr [[Z]], i64 32
+; CHECK-NEXT:    [[ARRAYIDX82:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 32
 ; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i32>, ptr [[ARRAYIDX16]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = load <2 x i32>, ptr [[ARRAYIDX52]], align 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul nsw <2 x i32> [[TMP14]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[MUL87:%.*]] = mul nsw i32 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i8, ptr [[Z]], i64 44
-; CHECK-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds i8, ptr [[Z]], i64 36
+; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 44
+; CHECK-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 36
 ; CHECK-NEXT:    [[TMP17:%.*]] = load <2 x i32>, ptr [[ARRAYIDX28]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <2 x i32>, ptr [[ARRAYIDX64]], align 4
 ; CHECK-NEXT:    store i32 [[MUL73]], ptr [[Z]], align 4
@@ -915,15 +915,15 @@ define void @store_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
-; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds i8, ptr [[DST0:%.*]], i64 16
-; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds i8, ptr [[DST0]], i64 32
-; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds i8, ptr [[DST0]], i64 48
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0:%.*]], i64 16
+; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 32
+; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 48
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
@@ -1201,20 +1201,20 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[ST1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[ST2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_1]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_2]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
index 920e1e64e3958..356102ce81780 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
@@ -8,10 +8,10 @@ target triple = "aarch64--linux-gnu"
 ; should not compute a smaller size for %k.13 since it is in a use-def cycle
 ; and cannot be demoted.
 ;
-define fastcc void @PR26364() {
+define fastcc void @PR26364(i1 %arg) {
 ; CHECK-LABEL: @PR26364(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END11:%.*]], label [[FOR_COND4:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END11:%.*]], label [[FOR_COND4:%.*]]
 ; CHECK:       for.cond4:
 ; CHECK-NEXT:    [[K_13:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[K_3:%.*]], [[FOR_COND4]] ]
 ; CHECK-NEXT:    [[E_02:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ 0, [[FOR_COND4]] ]
@@ -22,7 +22,7 @@ define fastcc void @PR26364() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %for.end11, label %for.cond4
+  br i1 %arg, label %for.end11, label %for.cond4
 
 for.cond4:
   %k.13 = phi i32 [ undef, %entry ], [ %k.3, %for.cond4 ]
@@ -39,10 +39,10 @@ for.end11:
 ; every root in the vectorizable tree when computing minimum sizes since one
 ; root may require fewer bits than another.
 ;
-define void @PR26629(ptr %c) {
+define void @PR26629(ptr %c, i1 %arg) {
 ; CHECK-LABEL: @PR26629(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[FOR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_PH:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[C:%.*]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -59,7 +59,7 @@ define void @PR26629(ptr %c) {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %for.ph, label %for.end
+  br i1 %arg, label %for.ph, label %for.end
 
 for.ph:
   %0 = load i32, ptr %c, align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reorder-fmuladd-crash.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reorder-fmuladd-crash.ll
index dc05967af1529..f5e904467baa7 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reorder-fmuladd-crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reorder-fmuladd-crash.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-w32-windows-gnu | FileCheck %s
 
-define i32 @foo(i32 %v1, double %v2) {
+define i32 @foo(i32 %v1, double %v2, i1 %arg, i32 %arg2) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 undef>, i32 [[V1:%.*]], i32 0
@@ -15,7 +15,7 @@ define i32 @foo(i32 %v1, double %v2) {
 ; CHECK:       if.end:
 ; CHECK-NEXT:    br label [[FOR_COND15:%.*]]
 ; CHECK:       for.end39:
-; CHECK-NEXT:    switch i32 undef, label [[DO_BODY:%.*]] [
+; CHECK-NEXT:    switch i32 %arg2, label [[DO_BODY:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[SW_BB:%.*]]
 ; CHECK-NEXT:      i32 1, label [[SW_BB195:%.*]]
 ; CHECK-NEXT:    ]
@@ -39,7 +39,7 @@ define i32 @foo(i32 %v1, double %v2) {
 ; CHECK:       if.end.1:
 ; CHECK-NEXT:    br label [[FOR_COND15_1:%.*]]
 ; CHECK:       for.cond15.1:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END39:%.*]], label [[FOR_COND15_PREHEADER]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END39:%.*]], label [[FOR_COND15_PREHEADER]]
 ;
 entry:
   %conv = sitofp i32 undef to double
@@ -56,7 +56,7 @@ if.end:                                           ; preds = %for.cond15.preheade
   br label %for.cond15
 
 for.end39:                                        ; preds = %for.cond15.1
-  switch i32 undef, label %do.body [
+  switch i32 %arg2, label %do.body [
   i32 0, label %sw.bb
   i32 1, label %sw.bb195
   ]
@@ -99,7 +99,7 @@ if.end.1:                                         ; preds = %for.cond15
   br label %for.cond15.1
 
 for.cond15.1:                                     ; preds = %if.end.1
-  br i1 undef, label %for.end39, label %for.cond15.preheader
+  br i1 %arg, label %for.end39, label %for.cond15.preheader
 }
 
 declare double @llvm.fmuladd.f64(double, double, double)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll
index 9910090d43eae..82761b458efcf 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll
@@ -3,13 +3,13 @@
 target triple = "aarch64-unknown-linux-gnu"
 @d = internal unnamed_addr global i32 5, align 4
 
-define dso_local void @l() local_unnamed_addr {
+define dso_local void @l(i1 %arg) local_unnamed_addr {
 ; CHECK-LABEL: @l(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i16> [ undef, [[BB:%.*]] ], [ [[TMP9:%.*]], [[BB25:%.*]] ]
-; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB11:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB3:%.*]], label [[BB11:%.*]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[I4:%.*]] = zext i1 undef to i32
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i16> [[TMP0]], undef
@@ -34,7 +34,7 @@ define dso_local void @l() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP13:%.*]] = zext i1 [[TMP12]] to i32
 ; CHECK-NEXT:    [[I32:%.*]] = and i32 [[I31]], [[TMP13]]
 ; CHECK-NEXT:    [[I33:%.*]] = and i32 [[I32]], [[I28]]
-; CHECK-NEXT:    br i1 undef, label [[BB34:%.*]], label [[BB1]]
+; CHECK-NEXT:    br i1 %arg, label [[BB34:%.*]], label [[BB1]]
 ; CHECK:       bb34:
 ; CHECK-NEXT:    [[I35:%.*]] = phi i32 [ [[I33]], [[BB25]] ]
 ; CHECK-NEXT:    br label [[BB36:%.*]]
@@ -48,7 +48,7 @@ bb:
 bb1:                                              ; preds = %bb25, %bb
   %i = phi i16 [ undef, %bb ], [ %i29, %bb25 ]
   %i2 = phi i16 [ undef, %bb ], [ %i30, %bb25 ]
-  br i1 undef, label %bb3, label %bb11
+  br i1 %arg, label %bb3, label %bb11
 
 bb3:                                              ; preds = %bb1
   %i4 = zext i1 undef to i32
@@ -85,7 +85,7 @@ bb25:                                             ; preds = %bb11, %bb3
   %i31 = and i32 undef, %i26
   %i32 = and i32 %i31, %i27
   %i33 = and i32 %i32, %i28
-  br i1 undef, label %bb34, label %bb1
+  br i1 %arg, label %bb34, label %bb1
 
 bb34:                                             ; preds = %bb25
   %i35 = phi i32 [ %i33, %bb25 ]
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll
new file mode 100644
index 0000000000000..fa0587f1da931
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fadd.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z15 -passes=slp-vectorizer %s -S -o - \
+; RUN:   | FileCheck %s
+;
+; Test vectorization and reassociation of fadd operations. If the loads can
+; be vectorized, cases of fewer operands are also profitable to vectorize.
+
+define double @fadd_double_4_addends_seq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fadd_double_4_addends_seq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+entry:
+  %0 = load double, ptr %x, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %x, i64 1
+  %1 = load double, ptr %arrayidx1, align 8
+  %add = fadd reassoc nsz arcp contract afn double %1, %0
+  %arrayidx2 = getelementptr inbounds double, ptr %x, i64 2
+  %2 = load double, ptr %arrayidx2, align 8
+  %add3 = fadd reassoc nsz arcp contract afn double %add, %2
+  %arrayidx4 = getelementptr inbounds double, ptr %x, i64 3
+  %3 = load double, ptr %arrayidx4, align 8
+  %add5 = fadd reassoc nsz arcp contract afn double %add3, %3
+  ret double %add5
+}
+
+define double @fadd_double_8_addends_nonseq(ptr nocapture noundef readonly %x)  {
+; CHECK-LABEL: define double @fadd_double_8_addends_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[X]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, ptr [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds double, ptr [[X]], i64 10
+; CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds double, ptr [[X]], i64 12
+; CHECK-NEXT:    [[TMP6:%.*]] = load double, ptr [[ARRAYIDX10]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds double, ptr [[X]], i64 14
+; CHECK-NEXT:    [[TMP7:%.*]] = load double, ptr [[ARRAYIDX12]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[TMP3]], i32 3
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[TMP5]], i32 5
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[TMP6]], i32 6
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[TMP7]], i32 7
+; CHECK-NEXT:    [[TMP16:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP15]])
+; CHECK-NEXT:    ret double [[TMP16]]
+;
+entry:
+  %0 = load double, ptr %x, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %x, i64 2
+  %1 = load double, ptr %arrayidx1, align 8
+  %add = fadd reassoc nsz arcp contract afn double %1, %0
+  %arrayidx2 = getelementptr inbounds double, ptr %x, i64 4
+  %2 = load double, ptr %arrayidx2, align 8
+  %add3 = fadd reassoc nsz arcp contract afn double %add, %2
+  %arrayidx4 = getelementptr inbounds double, ptr %x, i64 6
+  %3 = load double, ptr %arrayidx4, align 8
+  %add5 = fadd reassoc nsz arcp contract afn double %add3, %3
+  %arrayidx6 = getelementptr inbounds double, ptr %x, i64 8
+  %4 = load double, ptr %arrayidx6, align 8
+  %add7 = fadd reassoc nsz arcp contract afn double %add5, %4
+  %arrayidx8 = getelementptr inbounds double, ptr %x, i64 10
+  %5 = load double, ptr %arrayidx8, align 8
+  %add9 = fadd reassoc nsz arcp contract afn double %add7, %5
+  %arrayidx10 = getelementptr inbounds double, ptr %x, i64 12
+  %6 = load double, ptr %arrayidx10, align 8
+  %add11 = fadd reassoc nsz arcp contract afn double %add9, %6
+  %arrayidx12 = getelementptr inbounds double, ptr %x, i64 14
+  %7 = load double, ptr %arrayidx12, align 8
+  %add13 = fadd reassoc nsz arcp contract afn double %add11, %7
+  ret double %add13
+}
+
+define float @fadd_float_16_addends_nonseq(ptr nocapture noundef readonly %x)  {
+; CHECK-LABEL: define float @fadd_float_16_addends_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[X]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[X]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[X]], i64 10
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[X]], i64 12
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[X]], i64 14
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, ptr [[X]], i64 16
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr [[ARRAYIDX14]], align 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds float, ptr [[X]], i64 18
+; CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr [[ARRAYIDX16]], align 4
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, ptr [[X]], i64 20
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX18]], align 4
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds float, ptr [[X]], i64 22
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX20]], align 4
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds float, ptr [[X]], i64 24
+; CHECK-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX22]], align 4
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds float, ptr [[X]], i64 26
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX24]], align 4
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds float, ptr [[X]], i64 28
+; CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX26]], align 4
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds float, ptr [[X]], i64 30
+; CHECK-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX28]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x float> [[TMP16]], float [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <16 x float> [[TMP17]], float [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <16 x float> [[TMP18]], float [[TMP3]], i32 3
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <16 x float> [[TMP19]], float [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <16 x float> [[TMP20]], float [[TMP5]], i32 5
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <16 x float> [[TMP21]], float [[TMP6]], i32 6
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[TMP7]], i32 7
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <16 x float> [[TMP23]], float [[TMP8]], i32 8
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <16 x float> [[TMP24]], float [[TMP9]], i32 9
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <16 x float> [[TMP25]], float [[TMP10]], i32 10
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <16 x float> [[TMP26]], float [[TMP11]], i32 11
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <16 x float> [[TMP27]], float [[TMP12]], i32 12
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <16 x float> [[TMP28]], float [[TMP13]], i32 13
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <16 x float> [[TMP29]], float [[TMP14]], i32 14
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <16 x float> [[TMP30]], float [[TMP15]], i32 15
+; CHECK-NEXT:    [[TMP32:%.*]] = call reassoc nsz arcp contract afn float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP31]])
+; CHECK-NEXT:    ret float [[TMP32]]
+;
+entry:
+  %0 = load float, ptr %x, align 4
+  %arrayidx1 = getelementptr inbounds float, ptr %x, i64 2
+  %1 = load float, ptr %arrayidx1, align 4
+  %add = fadd reassoc nsz arcp contract afn float %1, %0
+  %arrayidx2 = getelementptr inbounds float, ptr %x, i64 4
+  %2 = load float, ptr %arrayidx2, align 4
+  %add3 = fadd reassoc nsz arcp contract afn float %add, %2
+  %arrayidx4 = getelementptr inbounds float, ptr %x, i64 6
+  %3 = load float, ptr %arrayidx4, align 4
+  %add5 = fadd reassoc nsz arcp contract afn float %add3, %3
+  %arrayidx6 = getelementptr inbounds float, ptr %x, i64 8
+  %4 = load float, ptr %arrayidx6, align 4
+  %add7 = fadd reassoc nsz arcp contract afn float %add5, %4
+  %arrayidx8 = getelementptr inbounds float, ptr %x, i64 10
+  %5 = load float, ptr %arrayidx8, align 4
+  %add9 = fadd reassoc nsz arcp contract afn float %add7, %5
+  %arrayidx10 = getelementptr inbounds float, ptr %x, i64 12
+  %6 = load float, ptr %arrayidx10, align 4
+  %add11 = fadd reassoc nsz arcp contract afn float %add9, %6
+  %arrayidx12 = getelementptr inbounds float, ptr %x, i64 14
+  %7 = load float, ptr %arrayidx12, align 4
+  %add13 = fadd reassoc nsz arcp contract afn float %add11, %7
+  %arrayidx14 = getelementptr inbounds float, ptr %x, i64 16
+  %8 = load float, ptr %arrayidx14, align 4
+  %add15 = fadd reassoc nsz arcp contract afn float %add13, %8
+  %arrayidx16 = getelementptr inbounds float, ptr %x, i64 18
+  %9 = load float, ptr %arrayidx16, align 4
+  %add17 = fadd reassoc nsz arcp contract afn float %add15, %9
+  %arrayidx18 = getelementptr inbounds float, ptr %x, i64 20
+  %10 = load float, ptr %arrayidx18, align 4
+  %add19 = fadd reassoc nsz arcp contract afn float %add17, %10
+  %arrayidx20 = getelementptr inbounds float, ptr %x, i64 22
+  %11 = load float, ptr %arrayidx20, align 4
+  %add21 = fadd reassoc nsz arcp contract afn float %add19, %11
+  %arrayidx22 = getelementptr inbounds float, ptr %x, i64 24
+  %12 = load float, ptr %arrayidx22, align 4
+  %add23 = fadd reassoc nsz arcp contract afn float %add21, %12
+  %arrayidx24 = getelementptr inbounds float, ptr %x, i64 26
+  %13 = load float, ptr %arrayidx24, align 4
+  %add25 = fadd reassoc nsz arcp contract afn float %add23, %13
+  %arrayidx26 = getelementptr inbounds float, ptr %x, i64 28
+  %14 = load float, ptr %arrayidx26, align 4
+  %add27 = fadd reassoc nsz arcp contract afn float %add25, %14
+  %arrayidx28 = getelementptr inbounds float, ptr %x, i64 30
+  %15 = load float, ptr %arrayidx28, align 4
+  %add29 = fadd reassoc nsz arcp contract afn float %add27, %15
+  ret float %add29
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll
new file mode 100644
index 0000000000000..5ea777e1c9a10
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmin-fmax.ll
@@ -0,0 +1,411 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z15 -passes=slp-vectorizer %s -S -o - \
+; RUN:   | FileCheck %s
+
+; Test vectorization and reassociation of fmin/fmax operations. Vectorization
+; is more profitable if the loads are also vectorizable.
+
+define double @fmin_double_4_nums_seq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmin_double_4_nums_seq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %g1 = getelementptr inbounds double, ptr %x, i64 1
+  %g2 = getelementptr inbounds double, ptr %x, i64 2
+  %g3 = getelementptr inbounds double, ptr %x, i64 3
+  %t0 = load double, ptr %x, align 4
+  %t1 = load double, ptr %g1, align 4
+  %t2 = load double, ptr %g2, align 4
+  %t3 = load double, ptr %g3, align 4
+  %m1 = tail call fast double @llvm.minnum.f64(double %t1, double %t0)
+  %m2 = tail call fast double @llvm.minnum.f64(double %t2, double %m1)
+  %m3 = tail call fast double @llvm.minnum.f64(double %t3, double %m2)
+  ret double %m3
+}
+
+define double @fmin_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmin_double_16_nums_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 8
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds double, ptr [[X]], i64 10
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 12
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds double, ptr [[X]], i64 14
+; CHECK-NEXT:    [[G8:%.*]] = getelementptr inbounds double, ptr [[X]], i64 16
+; CHECK-NEXT:    [[G9:%.*]] = getelementptr inbounds double, ptr [[X]], i64 18
+; CHECK-NEXT:    [[G10:%.*]] = getelementptr inbounds double, ptr [[X]], i64 20
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr inbounds double, ptr [[X]], i64 22
+; CHECK-NEXT:    [[G12:%.*]] = getelementptr inbounds double, ptr [[X]], i64 24
+; CHECK-NEXT:    [[G13:%.*]] = getelementptr inbounds double, ptr [[X]], i64 26
+; CHECK-NEXT:    [[G14:%.*]] = getelementptr inbounds double, ptr [[X]], i64 28
+; CHECK-NEXT:    [[G15:%.*]] = getelementptr inbounds double, ptr [[X]], i64 30
+; CHECK-NEXT:    [[T0:%.*]] = load double, ptr [[X]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load double, ptr [[G1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load double, ptr [[G2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[G3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load double, ptr [[G4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load double, ptr [[G5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load double, ptr [[G6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load double, ptr [[G7]], align 4
+; CHECK-NEXT:    [[T8:%.*]] = load double, ptr [[G8]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = load double, ptr [[G9]], align 4
+; CHECK-NEXT:    [[T10:%.*]] = load double, ptr [[G10]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = load double, ptr [[G11]], align 4
+; CHECK-NEXT:    [[T12:%.*]] = load double, ptr [[G12]], align 4
+; CHECK-NEXT:    [[T13:%.*]] = load double, ptr [[G13]], align 4
+; CHECK-NEXT:    [[T14:%.*]] = load double, ptr [[G14]], align 4
+; CHECK-NEXT:    [[T15:%.*]] = load double, ptr [[G15]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x double> poison, double [[T1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x double> [[TMP1]], double [[T0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x double> [[TMP2]], double [[T2]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x double> [[TMP3]], double [[T3]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x double> [[TMP4]], double [[T4]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x double> [[TMP5]], double [[T5]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x double> [[TMP6]], double [[T6]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x double> [[TMP7]], double [[T7]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x double> [[TMP8]], double [[T8]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x double> [[TMP9]], double [[T9]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x double> [[TMP10]], double [[T10]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x double> [[TMP11]], double [[T11]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x double> [[TMP12]], double [[T12]], i32 12
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x double> [[TMP13]], double [[T13]], i32 13
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x double> [[TMP14]], double [[T14]], i32 14
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x double> [[TMP15]], double [[T15]], i32 15
+; CHECK-NEXT:    [[TMP17:%.*]] = call fast double @llvm.vector.reduce.fmin.v16f64(<16 x double> [[TMP16]])
+; CHECK-NEXT:    ret double [[TMP17]]
+;
+  %g1 = getelementptr inbounds double, ptr %x, i64 2
+  %g2 = getelementptr inbounds double, ptr %x, i64 4
+  %g3 = getelementptr inbounds double, ptr %x, i64 6
+  %g4 = getelementptr inbounds double, ptr %x, i64 8
+  %g5 = getelementptr inbounds double, ptr %x, i64 10
+  %g6 = getelementptr inbounds double, ptr %x, i64 12
+  %g7 = getelementptr inbounds double, ptr %x, i64 14
+  %g8 = getelementptr inbounds double, ptr %x, i64 16
+  %g9 = getelementptr inbounds double, ptr %x, i64 18
+  %g10 = getelementptr inbounds double, ptr %x, i64 20
+  %g11 = getelementptr inbounds double, ptr %x, i64 22
+  %g12 = getelementptr inbounds double, ptr %x, i64 24
+  %g13 = getelementptr inbounds double, ptr %x, i64 26
+  %g14 = getelementptr inbounds double, ptr %x, i64 28
+  %g15 = getelementptr inbounds double, ptr %x, i64 30
+  %t0 = load double, ptr %x, align 4
+  %t1 = load double, ptr %g1, align 4
+  %t2 = load double, ptr %g2, align 4
+  %t3 = load double, ptr %g3, align 4
+  %t4 = load double, ptr %g4, align 4
+  %t5 = load double, ptr %g5, align 4
+  %t6 = load double, ptr %g6, align 4
+  %t7 = load double, ptr %g7, align 4
+  %t8 = load double, ptr %g8, align 4
+  %t9 = load double, ptr %g9, align 4
+  %t10 = load double, ptr %g10, align 4
+  %t11 = load double, ptr %g11, align 4
+  %t12 = load double, ptr %g12, align 4
+  %t13 = load double, ptr %g13, align 4
+  %t14 = load double, ptr %g14, align 4
+  %t15 = load double, ptr %g15, align 4
+  %m1 = tail call fast double @llvm.minnum.f64(double %t1, double %t0)
+  %m2 = tail call fast double @llvm.minnum.f64(double %t2, double %m1)
+  %m3 = tail call fast double @llvm.minnum.f64(double %t3, double %m2)
+  %m4 = tail call fast double @llvm.minnum.f64(double %t4, double %m3)
+  %m5 = tail call fast double @llvm.minnum.f64(double %t5, double %m4)
+  %m6 = tail call fast double @llvm.minnum.f64(double %t6, double %m5)
+  %m7 = tail call fast double @llvm.minnum.f64(double %t7, double %m6)
+  %m8 = tail call fast double @llvm.minnum.f64(double %t8, double %m7)
+  %m9 = tail call fast double @llvm.minnum.f64(double %t9, double %m8)
+  %m10 = tail call fast double @llvm.minnum.f64(double %t10, double %m9)
+  %m11 = tail call fast double @llvm.minnum.f64(double %t11, double %m10)
+  %m12 = tail call fast double @llvm.minnum.f64(double %t12, double %m11)
+  %m13 = tail call fast double @llvm.minnum.f64(double %t13, double %m12)
+  %m14 = tail call fast double @llvm.minnum.f64(double %t14, double %m13)
+  %m15 = tail call fast double @llvm.minnum.f64(double %t15, double %m14)
+  ret double %m15
+}
+
+define float @fmin_float_12_nums_nonseq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define float @fmin_float_12_nums_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds float, ptr [[X]], i64 8
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds float, ptr [[X]], i64 10
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds float, ptr [[X]], i64 12
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds float, ptr [[X]], i64 14
+; CHECK-NEXT:    [[G8:%.*]] = getelementptr inbounds float, ptr [[X]], i64 16
+; CHECK-NEXT:    [[G9:%.*]] = getelementptr inbounds float, ptr [[X]], i64 18
+; CHECK-NEXT:    [[G10:%.*]] = getelementptr inbounds float, ptr [[X]], i64 20
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr inbounds float, ptr [[X]], i64 22
+; CHECK-NEXT:    [[T0:%.*]] = load float, ptr [[X]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load float, ptr [[G4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load float, ptr [[G5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load float, ptr [[G6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load float, ptr [[G7]], align 4
+; CHECK-NEXT:    [[T8:%.*]] = load float, ptr [[G8]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = load float, ptr [[G9]], align 4
+; CHECK-NEXT:    [[T10:%.*]] = load float, ptr [[G10]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = load float, ptr [[G11]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <12 x float> poison, float [[T1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <12 x float> [[TMP1]], float [[T0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <12 x float> [[TMP2]], float [[T2]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <12 x float> [[TMP3]], float [[T3]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <12 x float> [[TMP4]], float [[T4]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <12 x float> [[TMP5]], float [[T5]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <12 x float> [[TMP6]], float [[T6]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <12 x float> [[TMP7]], float [[T7]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <12 x float> [[TMP8]], float [[T8]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <12 x float> [[TMP9]], float [[T9]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <12 x float> [[TMP10]], float [[T10]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <12 x float> [[TMP11]], float [[T11]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.v12f32(<12 x float> [[TMP12]])
+; CHECK-NEXT:    ret float [[TMP13]]
+;
+  %g1 = getelementptr inbounds float, ptr %x, i64 2
+  %g2 = getelementptr inbounds float, ptr %x, i64 4
+  %g3 = getelementptr inbounds float, ptr %x, i64 6
+  %g4 = getelementptr inbounds float, ptr %x, i64 8
+  %g5 = getelementptr inbounds float, ptr %x, i64 10
+  %g6 = getelementptr inbounds float, ptr %x, i64 12
+  %g7 = getelementptr inbounds float, ptr %x, i64 14
+  %g8 = getelementptr inbounds float, ptr %x, i64 16
+  %g9 = getelementptr inbounds float, ptr %x, i64 18
+  %g10 = getelementptr inbounds float, ptr %x, i64 20
+  %g11 = getelementptr inbounds float, ptr %x, i64 22
+  %t0 = load float, ptr %x, align 4
+  %t1 = load float, ptr %g1, align 4
+  %t2 = load float, ptr %g2, align 4
+  %t3 = load float, ptr %g3, align 4
+  %t4 = load float, ptr %g4, align 4
+  %t5 = load float, ptr %g5, align 4
+  %t6 = load float, ptr %g6, align 4
+  %t7 = load float, ptr %g7, align 4
+  %t8 = load float, ptr %g8, align 4
+  %t9 = load float, ptr %g9, align 4
+  %t10 = load float, ptr %g10, align 4
+  %t11 = load float, ptr %g11, align 4
+  %m1 = tail call fast float @llvm.minnum.f32(float %t1, float %t0)
+  %m2 = tail call fast float @llvm.minnum.f32(float %t2, float %m1)
+  %m3 = tail call fast float @llvm.minnum.f32(float %t3, float %m2)
+  %m4 = tail call fast float @llvm.minnum.f32(float %t4, float %m3)
+  %m5 = tail call fast float @llvm.minnum.f32(float %t5, float %m4)
+  %m6 = tail call fast float @llvm.minnum.f32(float %t6, float %m5)
+  %m7 = tail call fast float @llvm.minnum.f32(float %t7, float %m6)
+  %m8 = tail call fast float @llvm.minnum.f32(float %t8, float %m7)
+  %m9 = tail call fast float @llvm.minnum.f32(float %t9, float %m8)
+  %m10 = tail call fast float @llvm.minnum.f32(float %t10, float %m9)
+  %m11 = tail call fast float @llvm.minnum.f32(float %t11, float %m10)
+  ret float %m11
+}
+
+define double @fmax_double_4_nums_seq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmax_double_4_nums_seq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %g1 = getelementptr inbounds double, ptr %x, i64 1
+  %g2 = getelementptr inbounds double, ptr %x, i64 2
+  %g3 = getelementptr inbounds double, ptr %x, i64 3
+  %t0 = load double, ptr %x, align 4
+  %t1 = load double, ptr %g1, align 4
+  %t2 = load double, ptr %g2, align 4
+  %t3 = load double, ptr %g3, align 4
+  %m1 = tail call fast double @llvm.maxnum.f64(double %t1, double %t0)
+  %m2 = tail call fast double @llvm.maxnum.f64(double %t2, double %m1)
+  %m3 = tail call fast double @llvm.maxnum.f64(double %t3, double %m2)
+  ret double %m3
+}
+
+define double @fmax_double_16_nums_nonseq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmax_double_16_nums_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 8
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds double, ptr [[X]], i64 10
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 12
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds double, ptr [[X]], i64 14
+; CHECK-NEXT:    [[G8:%.*]] = getelementptr inbounds double, ptr [[X]], i64 16
+; CHECK-NEXT:    [[G9:%.*]] = getelementptr inbounds double, ptr [[X]], i64 18
+; CHECK-NEXT:    [[G10:%.*]] = getelementptr inbounds double, ptr [[X]], i64 20
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr inbounds double, ptr [[X]], i64 22
+; CHECK-NEXT:    [[G12:%.*]] = getelementptr inbounds double, ptr [[X]], i64 24
+; CHECK-NEXT:    [[G13:%.*]] = getelementptr inbounds double, ptr [[X]], i64 26
+; CHECK-NEXT:    [[G14:%.*]] = getelementptr inbounds double, ptr [[X]], i64 28
+; CHECK-NEXT:    [[G15:%.*]] = getelementptr inbounds double, ptr [[X]], i64 30
+; CHECK-NEXT:    [[T0:%.*]] = load double, ptr [[X]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load double, ptr [[G1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load double, ptr [[G2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load double, ptr [[G3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load double, ptr [[G4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load double, ptr [[G5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load double, ptr [[G6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load double, ptr [[G7]], align 4
+; CHECK-NEXT:    [[T8:%.*]] = load double, ptr [[G8]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = load double, ptr [[G9]], align 4
+; CHECK-NEXT:    [[T10:%.*]] = load double, ptr [[G10]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = load double, ptr [[G11]], align 4
+; CHECK-NEXT:    [[T12:%.*]] = load double, ptr [[G12]], align 4
+; CHECK-NEXT:    [[T13:%.*]] = load double, ptr [[G13]], align 4
+; CHECK-NEXT:    [[T14:%.*]] = load double, ptr [[G14]], align 4
+; CHECK-NEXT:    [[T15:%.*]] = load double, ptr [[G15]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x double> poison, double [[T1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x double> [[TMP1]], double [[T0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x double> [[TMP2]], double [[T2]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x double> [[TMP3]], double [[T3]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x double> [[TMP4]], double [[T4]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x double> [[TMP5]], double [[T5]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x double> [[TMP6]], double [[T6]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x double> [[TMP7]], double [[T7]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x double> [[TMP8]], double [[T8]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x double> [[TMP9]], double [[T9]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x double> [[TMP10]], double [[T10]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x double> [[TMP11]], double [[T11]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x double> [[TMP12]], double [[T12]], i32 12
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x double> [[TMP13]], double [[T13]], i32 13
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x double> [[TMP14]], double [[T14]], i32 14
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x double> [[TMP15]], double [[T15]], i32 15
+; CHECK-NEXT:    [[TMP17:%.*]] = call fast double @llvm.vector.reduce.fmax.v16f64(<16 x double> [[TMP16]])
+; CHECK-NEXT:    ret double [[TMP17]]
+;
+  %g1 = getelementptr inbounds double, ptr %x, i64 2
+  %g2 = getelementptr inbounds double, ptr %x, i64 4
+  %g3 = getelementptr inbounds double, ptr %x, i64 6
+  %g4 = getelementptr inbounds double, ptr %x, i64 8
+  %g5 = getelementptr inbounds double, ptr %x, i64 10
+  %g6 = getelementptr inbounds double, ptr %x, i64 12
+  %g7 = getelementptr inbounds double, ptr %x, i64 14
+  %g8 = getelementptr inbounds double, ptr %x, i64 16
+  %g9 = getelementptr inbounds double, ptr %x, i64 18
+  %g10 = getelementptr inbounds double, ptr %x, i64 20
+  %g11 = getelementptr inbounds double, ptr %x, i64 22
+  %g12 = getelementptr inbounds double, ptr %x, i64 24
+  %g13 = getelementptr inbounds double, ptr %x, i64 26
+  %g14 = getelementptr inbounds double, ptr %x, i64 28
+  %g15 = getelementptr inbounds double, ptr %x, i64 30
+  %t0 = load double, ptr %x, align 4
+  %t1 = load double, ptr %g1, align 4
+  %t2 = load double, ptr %g2, align 4
+  %t3 = load double, ptr %g3, align 4
+  %t4 = load double, ptr %g4, align 4
+  %t5 = load double, ptr %g5, align 4
+  %t6 = load double, ptr %g6, align 4
+  %t7 = load double, ptr %g7, align 4
+  %t8 = load double, ptr %g8, align 4
+  %t9 = load double, ptr %g9, align 4
+  %t10 = load double, ptr %g10, align 4
+  %t11 = load double, ptr %g11, align 4
+  %t12 = load double, ptr %g12, align 4
+  %t13 = load double, ptr %g13, align 4
+  %t14 = load double, ptr %g14, align 4
+  %t15 = load double, ptr %g15, align 4
+  %m1 = tail call fast double @llvm.maxnum.f64(double %t1, double %t0)
+  %m2 = tail call fast double @llvm.maxnum.f64(double %t2, double %m1)
+  %m3 = tail call fast double @llvm.maxnum.f64(double %t3, double %m2)
+  %m4 = tail call fast double @llvm.maxnum.f64(double %t4, double %m3)
+  %m5 = tail call fast double @llvm.maxnum.f64(double %t5, double %m4)
+  %m6 = tail call fast double @llvm.maxnum.f64(double %t6, double %m5)
+  %m7 = tail call fast double @llvm.maxnum.f64(double %t7, double %m6)
+  %m8 = tail call fast double @llvm.maxnum.f64(double %t8, double %m7)
+  %m9 = tail call fast double @llvm.maxnum.f64(double %t9, double %m8)
+  %m10 = tail call fast double @llvm.maxnum.f64(double %t10, double %m9)
+  %m11 = tail call fast double @llvm.maxnum.f64(double %t11, double %m10)
+  %m12 = tail call fast double @llvm.maxnum.f64(double %t12, double %m11)
+  %m13 = tail call fast double @llvm.maxnum.f64(double %t13, double %m12)
+  %m14 = tail call fast double @llvm.maxnum.f64(double %t14, double %m13)
+  %m15 = tail call fast double @llvm.maxnum.f64(double %t15, double %m14)
+  ret double %m15
+}
+
+define float @fmax_float_12_nums_nonseq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define float @fmax_float_12_nums_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds float, ptr [[X]], i64 8
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds float, ptr [[X]], i64 10
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds float, ptr [[X]], i64 12
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds float, ptr [[X]], i64 14
+; CHECK-NEXT:    [[G8:%.*]] = getelementptr inbounds float, ptr [[X]], i64 16
+; CHECK-NEXT:    [[G9:%.*]] = getelementptr inbounds float, ptr [[X]], i64 18
+; CHECK-NEXT:    [[G10:%.*]] = getelementptr inbounds float, ptr [[X]], i64 20
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr inbounds float, ptr [[X]], i64 22
+; CHECK-NEXT:    [[T0:%.*]] = load float, ptr [[X]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load float, ptr [[G4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load float, ptr [[G5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load float, ptr [[G6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load float, ptr [[G7]], align 4
+; CHECK-NEXT:    [[T8:%.*]] = load float, ptr [[G8]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = load float, ptr [[G9]], align 4
+; CHECK-NEXT:    [[T10:%.*]] = load float, ptr [[G10]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = load float, ptr [[G11]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <12 x float> poison, float [[T1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <12 x float> [[TMP1]], float [[T0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <12 x float> [[TMP2]], float [[T2]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <12 x float> [[TMP3]], float [[T3]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <12 x float> [[TMP4]], float [[T4]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <12 x float> [[TMP5]], float [[T5]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <12 x float> [[TMP6]], float [[T6]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <12 x float> [[TMP7]], float [[T7]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <12 x float> [[TMP8]], float [[T8]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <12 x float> [[TMP9]], float [[T9]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <12 x float> [[TMP10]], float [[T10]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <12 x float> [[TMP11]], float [[T11]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmax.v12f32(<12 x float> [[TMP12]])
+; CHECK-NEXT:    ret float [[TMP13]]
+;
+  %g1 = getelementptr inbounds float, ptr %x, i64 2
+  %g2 = getelementptr inbounds float, ptr %x, i64 4
+  %g3 = getelementptr inbounds float, ptr %x, i64 6
+  %g4 = getelementptr inbounds float, ptr %x, i64 8
+  %g5 = getelementptr inbounds float, ptr %x, i64 10
+  %g6 = getelementptr inbounds float, ptr %x, i64 12
+  %g7 = getelementptr inbounds float, ptr %x, i64 14
+  %g8 = getelementptr inbounds float, ptr %x, i64 16
+  %g9 = getelementptr inbounds float, ptr %x, i64 18
+  %g10 = getelementptr inbounds float, ptr %x, i64 20
+  %g11 = getelementptr inbounds float, ptr %x, i64 22
+  %t0 = load float, ptr %x, align 4
+  %t1 = load float, ptr %g1, align 4
+  %t2 = load float, ptr %g2, align 4
+  %t3 = load float, ptr %g3, align 4
+  %t4 = load float, ptr %g4, align 4
+  %t5 = load float, ptr %g5, align 4
+  %t6 = load float, ptr %g6, align 4
+  %t7 = load float, ptr %g7, align 4
+  %t8 = load float, ptr %g8, align 4
+  %t9 = load float, ptr %g9, align 4
+  %t10 = load float, ptr %g10, align 4
+  %t11 = load float, ptr %g11, align 4
+  %m1 = tail call fast float @llvm.maxnum.f32(float %t1, float %t0)
+  %m2 = tail call fast float @llvm.maxnum.f32(float %t2, float %m1)
+  %m3 = tail call fast float @llvm.maxnum.f32(float %t3, float %m2)
+  %m4 = tail call fast float @llvm.maxnum.f32(float %t4, float %m3)
+  %m5 = tail call fast float @llvm.maxnum.f32(float %t5, float %m4)
+  %m6 = tail call fast float @llvm.maxnum.f32(float %t6, float %m5)
+  %m7 = tail call fast float @llvm.maxnum.f32(float %t7, float %m6)
+  %m8 = tail call fast float @llvm.maxnum.f32(float %t8, float %m7)
+  %m9 = tail call fast float @llvm.maxnum.f32(float %t9, float %m8)
+  %m10 = tail call fast float @llvm.maxnum.f32(float %t10, float %m9)
+  %m11 = tail call fast float @llvm.maxnum.f32(float %t11, float %m10)
+  ret float %m11
+}
+
+declare float @llvm.minnum.f32(float, float)
+declare double @llvm.minnum.f64(double, double)
+declare float @llvm.maxnum.f32(float, float)
+declare double @llvm.maxnum.f64(double, double)
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll
new file mode 100644
index 0000000000000..e08b38c69a840
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/reductions-fmul.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z15 -passes=slp-vectorizer %s -S -o - \
+; RUN:   | FileCheck %s
+
+; Test vectorization and reassociation of fmul operations. If the loads can
+; be vectorized, cases of fewer operands are also profitable to vectorize.
+
+define double @fmul_double_4_factors_seq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmul_double_4_factors_seq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fmul.v4f64(double 1.000000e+00, <4 x double> [[TMP0]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+entry:
+  %0 = load double, ptr %x, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %x, i64 1
+  %1 = load double, ptr %arrayidx1, align 8
+  %mul = fmul reassoc nsz arcp contract afn double %1, %0
+  %arrayidx2 = getelementptr inbounds double, ptr %x, i64 2
+  %2 = load double, ptr %arrayidx2, align 8
+  %mul3 = fmul reassoc nsz arcp contract afn double %mul, %2
+  %arrayidx4 = getelementptr inbounds double, ptr %x, i64 3
+  %3 = load double, ptr %arrayidx4, align 8
+  %mul5 = fmul reassoc nsz arcp contract afn double %mul3, %3
+  ret double %mul5
+}
+
+define double @fmul_double_8_factors_nonseq(ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define double @fmul_double_8_factors_nonseq(
+; CHECK-SAME: ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[X]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, ptr [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds double, ptr [[X]], i64 10
+; CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds double, ptr [[X]], i64 12
+; CHECK-NEXT:    [[TMP6:%.*]] = load double, ptr [[ARRAYIDX10]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds double, ptr [[X]], i64 14
+; CHECK-NEXT:    [[TMP7:%.*]] = load double, ptr [[ARRAYIDX12]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[TMP3]], i32 3
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[TMP5]], i32 5
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[TMP6]], i32 6
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[TMP7]], i32 7
+; CHECK-NEXT:    [[TMP16:%.*]] = call reassoc nsz arcp contract afn double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> [[TMP15]])
+; CHECK-NEXT:    ret double [[TMP16]]
+;
+entry:
+  %0 = load double, ptr %x, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %x, i64 2
+  %1 = load double, ptr %arrayidx1, align 8
+  %mul = fmul reassoc nsz arcp contract afn double %1, %0
+  %arrayidx2 = getelementptr inbounds double, ptr %x, i64 4
+  %2 = load double, ptr %arrayidx2, align 8
+  %mul3 = fmul reassoc nsz arcp contract afn double %mul, %2
+  %arrayidx4 = getelementptr inbounds double, ptr %x, i64 6
+  %3 = load double, ptr %arrayidx4, align 8
+  %mul5 = fmul reassoc nsz arcp contract afn double %mul3, %3
+  %arrayidx6 = getelementptr inbounds double, ptr %x, i64 8
+  %4 = load double, ptr %arrayidx6, align 8
+  %mul7 = fmul reassoc nsz arcp contract afn double %mul5, %4
+  %arrayidx8 = getelementptr inbounds double, ptr %x, i64 10
+  %5 = load double, ptr %arrayidx8, align 8
+  %mul9 = fmul reassoc nsz arcp contract afn double %mul7, %5
+  %arrayidx10 = getelementptr inbounds double, ptr %x, i64 12
+  %6 = load double, ptr %arrayidx10, align 8
+  %mul11 = fmul reassoc nsz arcp contract afn double %mul9, %6
+  %arrayidx12 = getelementptr inbounds double, ptr %x, i64 14
+  %7 = load double, ptr %arrayidx12, align 8
+  %mul13 = fmul reassoc nsz arcp contract afn double %mul11, %7
+  ret double %mul13
+}
+
+define float @fmul_float_16_factors_nonseq(float noundef %m, ptr nocapture noundef readonly %x) {
+; CHECK-LABEL: define float @fmul_float_16_factors_nonseq(
+; CHECK-SAME: float noundef [[M:%.*]], ptr nocapture noundef readonly [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[X]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[X]], i64 6
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[X]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[X]], i64 10
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[X]], i64 12
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[X]], i64 14
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, ptr [[X]], i64 16
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr [[ARRAYIDX14]], align 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds float, ptr [[X]], i64 18
+; CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr [[ARRAYIDX16]], align 4
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, ptr [[X]], i64 20
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX18]], align 4
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds float, ptr [[X]], i64 22
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX20]], align 4
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds float, ptr [[X]], i64 24
+; CHECK-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX22]], align 4
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds float, ptr [[X]], i64 26
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX24]], align 4
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds float, ptr [[X]], i64 28
+; CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX26]], align 4
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds float, ptr [[X]], i64 30
+; CHECK-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX28]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x float> [[TMP16]], float [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <16 x float> [[TMP17]], float [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <16 x float> [[TMP18]], float [[TMP3]], i32 3
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <16 x float> [[TMP19]], float [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <16 x float> [[TMP20]], float [[TMP5]], i32 5
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <16 x float> [[TMP21]], float [[TMP6]], i32 6
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[TMP7]], i32 7
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <16 x float> [[TMP23]], float [[TMP8]], i32 8
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <16 x float> [[TMP24]], float [[TMP9]], i32 9
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <16 x float> [[TMP25]], float [[TMP10]], i32 10
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <16 x float> [[TMP26]], float [[TMP11]], i32 11
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <16 x float> [[TMP27]], float [[TMP12]], i32 12
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <16 x float> [[TMP28]], float [[TMP13]], i32 13
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <16 x float> [[TMP29]], float [[TMP14]], i32 14
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <16 x float> [[TMP30]], float [[TMP15]], i32 15
+; CHECK-NEXT:    [[TMP32:%.*]] = call reassoc nsz arcp contract afn float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> [[TMP31]])
+; CHECK-NEXT:    ret float [[TMP32]]
+;
+entry:
+  %0 = load float, ptr %x, align 4
+  %arrayidx1 = getelementptr inbounds float, ptr %x, i64 2
+  %1 = load float, ptr %arrayidx1, align 4
+  %mul = fmul reassoc nsz arcp contract afn float %1, %0
+  %arrayidx2 = getelementptr inbounds float, ptr %x, i64 4
+  %2 = load float, ptr %arrayidx2, align 4
+  %mul3 = fmul reassoc nsz arcp contract afn float %mul, %2
+  %arrayidx4 = getelementptr inbounds float, ptr %x, i64 6
+  %3 = load float, ptr %arrayidx4, align 4
+  %mul5 = fmul reassoc nsz arcp contract afn float %mul3, %3
+  %arrayidx6 = getelementptr inbounds float, ptr %x, i64 8
+  %4 = load float, ptr %arrayidx6, align 4
+  %mul7 = fmul reassoc nsz arcp contract afn float %mul5, %4
+  %arrayidx8 = getelementptr inbounds float, ptr %x, i64 10
+  %5 = load float, ptr %arrayidx8, align 4
+  %mul9 = fmul reassoc nsz arcp contract afn float %mul7, %5
+  %arrayidx10 = getelementptr inbounds float, ptr %x, i64 12
+  %6 = load float, ptr %arrayidx10, align 4
+  %mul11 = fmul reassoc nsz arcp contract afn float %mul9, %6
+  %arrayidx12 = getelementptr inbounds float, ptr %x, i64 14
+  %7 = load float, ptr %arrayidx12, align 4
+  %mul13 = fmul reassoc nsz arcp contract afn float %mul11, %7
+  %arrayidx14 = getelementptr inbounds float, ptr %x, i64 16
+  %8 = load float, ptr %arrayidx14, align 4
+  %mul15 = fmul reassoc nsz arcp contract afn float %mul13, %8
+  %arrayidx16 = getelementptr inbounds float, ptr %x, i64 18
+  %9 = load float, ptr %arrayidx16, align 4
+  %mul17 = fmul reassoc nsz arcp contract afn float %mul15, %9
+  %arrayidx18 = getelementptr inbounds float, ptr %x, i64 20
+  %10 = load float, ptr %arrayidx18, align 4
+  %mul19 = fmul reassoc nsz arcp contract afn float %mul17, %10
+  %arrayidx20 = getelementptr inbounds float, ptr %x, i64 22
+  %11 = load float, ptr %arrayidx20, align 4
+  %mul21 = fmul reassoc nsz arcp contract afn float %mul19, %11
+  %arrayidx22 = getelementptr inbounds float, ptr %x, i64 24
+  %12 = load float, ptr %arrayidx22, align 4
+  %mul23 = fmul reassoc nsz arcp contract afn float %mul21, %12
+  %arrayidx24 = getelementptr inbounds float, ptr %x, i64 26
+  %13 = load float, ptr %arrayidx24, align 4
+  %mul25 = fmul reassoc nsz arcp contract afn float %mul23, %13
+  %arrayidx26 = getelementptr inbounds float, ptr %x, i64 28
+  %14 = load float, ptr %arrayidx26, align 4
+  %mul27 = fmul reassoc nsz arcp contract afn float %mul25, %14
+  %arrayidx28 = getelementptr inbounds float, ptr %x, i64 30
+  %15 = load float, ptr %arrayidx28, align 4
+  %mul29 = fmul reassoc nsz arcp contract afn float %mul27, %15
+  ret float %mul29
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
index eb8dd72e0304d..85b8157c949f1 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
@@ -1,29 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -mtriple=s390x-unknown-linux -mcpu=z16 -S -passes=slp-vectorizer \
 ; RUN:   -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
 ;
-; NB! This is a pre-commit version (for #112491) with current codegen and remarks.
-;
 ; Test functions that (at least currently) only gets vectorized if the
 ; insertion cost for an element load is counted as free.
 
+declare double @llvm.fmuladd.f64(double, double, double)
+
 ; This function needs the free element load to be recognized in SLP
 ; getGatherCost().
-define void @fun0(ptr nocapture %0, double %1) {
+define void @fun0(ptr %0, double %1) {
 ; CHECK-LABEL: define void @fun0(
-; CHECK:         fmul double
-; CHECK:         call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.sqrt.f64(
-; CHECK:         fmul double
-; CHECK:         call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.sqrt.f64(
+; CHECK-SAME: ptr [[TMP0:%.*]], double [[TMP1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP5]], splat (double 2.000000e+00)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[TMP6]], <2 x double> zeroinitializer)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[TMP6]], <2 x double> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP8]])
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x double> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x double> [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd double [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store double [[TMP12]], ptr [[TMP0]], align 8
+; CHECK-NEXT:    ret void
 ;
 ; REMARK-LABEL: Function: fun0
 ; REMARK: Args:
-; REMARK-NEXT: - String:          'List vectorization was possible but not beneficial with cost '
-; REMARK-NEXT: - Cost:            '0'
+; REMARK-NEXT: - String:          'SLP vectorized with cost '
+; REMARK-NEXT: - Cost:            '-1'
 
   %3 = fmul double %1, 2.000000e+00
   %4 = tail call double @llvm.fmuladd.f64(double %3, double %3, double 0.000000e+00)
@@ -43,32 +49,31 @@ define void @fun0(ptr nocapture %0, double %1) {
 ; getVectorInstrCost().
 define void @fun1(double %0) {
 ; CHECK-LABEL: define void @fun1(
-; CHECK:         phi double
-; CHECK-NEXT:    phi double
-; CHECK-NEXT:    phi double
-; CHECK-NEXT:    phi double
-; CHECK-NEXT:    phi double
-; CHECK-NEXT:    phi double
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    fmul double
-; CHECK-NEXT:    fmul double
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    fsub double
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK-NEXT:    call double @llvm.fmuladd.f64(
-; CHECK:         fcmp olt double
-; CHECK-NEXT:    fcmp olt double
-; CHECK-NEXT:    or i1
+; CHECK-SAME: double [[TMP0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[TMP0]], i32 1
+; CHECK-NEXT:    br label %[[BB3:.*]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x double> [ <double poison, double undef>, [[TMP1:%.*]] ], [ poison, %[[BB3]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <2 x double> [ zeroinitializer, [[TMP1]] ], [ poison, %[[BB3]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <2 x double> [ zeroinitializer, [[TMP1]] ], [ [[TMP18:%.*]], %[[BB3]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = fsub <2 x double> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub <2 x double> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fsub <2 x double> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr null, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul <2 x double> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP8]], <2 x double> [[TMP8]], <2 x double> [[TMP11]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP9]], <2 x double> [[TMP9]], <2 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = fcmp olt <2 x double> [[TMP13]], [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i1> [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18]] = insertelement <2 x double> poison, double [[TMP10]], i32 1
+; CHECK-NEXT:    br label %[[BB3]]
 ;
 ; REMARK-LABEL: Function: fun1
 ; REMARK: Args:
-; REMARK:      - String:          'List vectorization was possible but not beneficial with cost '
-; REMARK-NEXT: - Cost:            '0'
+; REMARK:      - String:          'SLP vectorized with cost '
+; REMARK-NEXT: - Cost:            '-1'
 
   br label %2
 
@@ -98,19 +103,24 @@ define void @fun1(double %0) {
   br label %2
 }
 
-declare double @llvm.fmuladd.f64(double, double, double)
-
 ; This should *not* be vectorized as the insertion into the vector isn't free,
 ; which is recognized in SystemZTTImpl::getScalarizationOverhead().
 define void @fun2(ptr %0, ptr %Dst) {
 ; CHECK-LABEL: define void @fun2(
-; CHECK: insertelement
-; CHECK: store <2 x i64>
+; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[DST:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[BB4:.*]], label %[[BB5:.*]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 24
+; CHECK-NEXT:    store i64 [[TMP2]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 16
+; CHECK-NEXT:    store i64 0, ptr [[TMP7]], align 8
+; CHECK-NEXT:    br label %[[BB4]]
 ;
-; REMARK-LABEL: Function: fun2
-; REMARK: Args:
-; REMARK-NEXT: - String:          'Stores SLP vectorized with cost '
-; REMARK-NEXT: - Cost:            '-1'
+; REMARK-NOT: Function: fun2
 
   %3 = load i64, ptr %0, align 8
   %4 = icmp eq i64 %3, 0
@@ -126,3 +136,55 @@ define void @fun2(ptr %0, ptr %Dst) {
   store i64 0, ptr %8, align 8
   br label %5
 }
+
+; This should *not* be vectorized as the load is immediately stored, in which
+; case MVC is preferred.
+define void @fun3(ptr %0)  {
+; CHECK-LABEL: define void @fun3(
+; CHECK-SAME: ptr [[TMP0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr inttoptr (i64 568 to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 40
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 48
+; CHECK-NEXT:    br label %[[BB5:.*]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    store ptr null, ptr [[TMP3]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load ptr, ptr inttoptr (i64 64 to ptr), align 8, !tbaa [[TBAA8:![0-9]+]]
+; CHECK-NEXT:    store ptr [[TMP6]], ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 [[TMP0]](ptr noundef poison, i64 noundef poison)
+; CHECK-NEXT:    br label %[[BB5]]
+;
+  %2 = load ptr, ptr inttoptr (i64 568 to ptr), align 8
+  %3 = getelementptr inbounds nuw i8, ptr %2, i64 40
+  %4 = getelementptr inbounds nuw i8, ptr %2, i64 48
+  br label %5
+
+5:
+  store ptr null, ptr %3, align 8, !tbaa !1
+  %6 = load ptr, ptr inttoptr (i64 64 to ptr), align 8, !tbaa !9
+  store ptr %6, ptr %4, align 8
+  %7 = tail call i64 %0(ptr noundef poison, i64 noundef poison)
+  br label %5
+}
+
+!1 = !{!2, !7, i64 40}
+!2 = !{!"arc", !3, i64 0, !6, i64 8, !7, i64 16, !7, i64 24, !8, i64 32, !7, i64 40, !7, i64 48, !6, i64 56, !6, i64 64}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!"long", !4, i64 0}
+!7 = !{!"any pointer", !4, i64 0}
+!8 = !{!"short", !4, i64 0}
+!9 = !{!10, !7, i64 64}
+!10 = !{!"node", !6, i64 0, !3, i64 8, !7, i64 16, !7, i64 24, !7, i64 32, !7, i64 40, !7, i64 48, !7, i64 56, !7, i64 64, !7, i64 72, !6, i64 80, !6, i64 88, !3, i64 96, !3, i64 100}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META6:![0-9]+]], i64 40}
+; CHECK: [[META1]] = !{!"arc", [[META2:![0-9]+]], i64 0, [[META5:![0-9]+]], i64 8, [[META6]], i64 16, [[META6]], i64 24, [[META7:![0-9]+]], i64 32, [[META6]], i64 40, [[META6]], i64 48, [[META5]], i64 56, [[META5]], i64 64}
+; CHECK: [[META2]] = !{!"int", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[META5]] = !{!"long", [[META3]], i64 0}
+; CHECK: [[META6]] = !{!"any pointer", [[META3]], i64 0}
+; CHECK: [[META7]] = !{!"short", [[META3]], i64 0}
+; CHECK: [[TBAA8]] = !{[[META9:![0-9]+]], [[META6]], i64 64}
+; CHECK: [[META9]] = !{!"node", [[META5]], i64 0, [[META2]], i64 8, [[META6]], i64 16, [[META6]], i64 24, [[META6]], i64 32, [[META6]], i64 40, [[META6]], i64 48, [[META6]], i64 56, [[META6]], i64 64, [[META6]], i64 72, [[META5]], i64 80, [[META5]], i64 88, [[META2]], i64 96, [[META2]], i64 100}
+;.
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
index 1c408f19a8a32..7faf11c34d76e 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
@@ -18,7 +18,7 @@ define void @foo(<2 x i64> %x, <4 x i32> %y, ptr %out) #0 {
 ; CHECK-NEXT:    [[E:%.*]] = extractelement <4 x i32> [[Y]], i64 3
 ; CHECK-NEXT:    [[CONV17:%.*]] = zext i32 [[E]] to i64
 ; CHECK-NEXT:    [[F:%.*]] = tail call i64 @llvm.fshl.i64(i64 [[D]], i64 [[D]], i64 [[CONV17]])
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i32 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[OUT]], i32 8
 ; CHECK-NEXT:    store i64 [[F]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ashr-node-with-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/ashr-node-with-poison.ll
new file mode 100644
index 0000000000000..76132c4d9a409
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ashr-node-with-poison.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+
+define i32 @test(ptr %n, i32 %conv57, i1 %tobool5.not, i64 %bf.load14) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[N:%.*]], i32 [[CONV57:%.*]], i1 [[TOBOOL5_NOT:%.*]], i64 [[BF_LOAD14:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[ENTRY_IF_END54_CRIT_EDGE:.*]], label %[[WHILE_COND:.*]]
+; CHECK:       [[ENTRY_IF_END54_CRIT_EDGE]]:
+; CHECK-NEXT:    br label %[[IF_END54:.*]]
+; CHECK:       [[TTHREAD_PRE_SPLIT:.*]]:
+; CHECK-NEXT:    br label %[[T:.*]]
+; CHECK:       [[T]]:
+; CHECK-NEXT:    [[W_2:%.*]] = phi i32 [ 1, %[[TTHREAD_PRE_SPLIT]] ], [ 0, %[[IF_END83:.*]] ]
+; CHECK-NEXT:    br label %[[IF_END7:.*]]
+; CHECK:       [[T_U_CRIT_EDGE:.*]]:
+; CHECK-NEXT:    br label %[[IF_END7]]
+; CHECK:       [[IF_END7]]:
+; CHECK-NEXT:    [[W_4:%.*]] = phi i32 [ [[W_2]], %[[T]] ], [ [[W_2]], %[[T_U_CRIT_EDGE]] ], [ [[BF_CAST25:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT:    [[A_4:%.*]] = phi i32 [ 0, %[[T]] ], [ [[CONV57]], %[[T_U_CRIT_EDGE]] ], [ [[BF_CAST2910:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[B_4:%.*]] = phi i32 [ 0, %[[T]] ], [ 0, %[[T_U_CRIT_EDGE]] ], [ [[BF_CAST2910]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[C_4:%.*]] = phi i32 [ 0, %[[T]] ], [ 1, %[[T_U_CRIT_EDGE]] ], [ poison, %[[WHILE_BODY]] ]
+; CHECK-NEXT:    br label %[[V:.*]]
+; CHECK:       [[WHILE_COND]]:
+; CHECK-NEXT:    [[BF_LOAD66_PRE_PRE1135:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[IF_END42:.*]] ]
+; CHECK-NEXT:    br i1 [[TOBOOL5_NOT]], label %[[IF_END54]], label %[[WHILE_BODY]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[BF_ASHR24:%.*]] = ashr i64 [[BF_LOAD14]], 33
+; CHECK-NEXT:    [[BF_CAST25]] = trunc nsw i64 [[BF_ASHR24]] to i32
+; CHECK-NEXT:    [[BF_ASHR28:%.*]] = lshr i64 [[BF_LOAD14]], 1
+; CHECK-NEXT:    [[BF_CAST2910]] = trunc i64 [[BF_ASHR28]] to i32
+; CHECK-NEXT:    br label %[[IF_END7]]
+; CHECK:       [[IF_END36:.*]]:
+; CHECK-NEXT:    br label %[[V]]
+; CHECK:       [[V]]:
+; CHECK-NEXT:    [[C_7:%.*]] = phi i32 [ [[C_4]], %[[IF_END7]] ], [ 0, %[[IF_END36]] ]
+; CHECK-NEXT:    br i1 true, label %[[IF_END42]], label %[[V_IF_END83_CRIT_EDGE:.*]]
+; CHECK:       [[V_IF_END83_CRIT_EDGE]]:
+; CHECK-NEXT:    br label %[[IF_END83]]
+; CHECK:       [[IF_END42]]:
+; CHECK-NEXT:    [[TOBOOL43_NOT:%.*]] = icmp eq i32 [[B_4]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TOBOOL43_NOT]], i32 0, i32 [[W_4]]
+; CHECK-NEXT:    [[SPEC_SELECT]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[BF_VALUE48:%.*]] = zext i32 [[A_4]] to i64
+; CHECK-NEXT:    store i64 [[BF_VALUE48]], ptr [[N]], align 8
+; CHECK-NEXT:    store i32 [[C_7]], ptr [[N]], align 4
+; CHECK-NEXT:    br label %[[WHILE_COND]]
+; CHECK:       [[IF_END54]]:
+; CHECK-NEXT:    [[BF_LOAD66_PRE_PRE113125:%.*]] = phi i64 [ [[BF_LOAD66_PRE_PRE1135]], %[[WHILE_COND]] ], [ poison, %[[ENTRY_IF_END54_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[BF_LOAD66_PRE_PRE113125]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[IF_END83]], label %[[AI_IF_END76_CRIT_EDGE:.*]]
+; CHECK:       [[AI_IF_END76_CRIT_EDGE]]:
+; CHECK-NEXT:    br label %[[IF_END83]]
+; CHECK:       [[IF_END83]]:
+; CHECK-NEXT:    br label %[[T]]
+;
+entry:
+  br i1 false, label %entry.if.end54_crit_edge, label %while.cond
+
+entry.if.end54_crit_edge:
+  br label %if.end54
+
+tthread-pre-split:
+  br label %t
+
+t:
+  %w.2 = phi i32 [ 1, %tthread-pre-split ], [ 0, %if.end83 ]
+  br label %if.end7
+
+t.u_crit_edge:
+  br label %if.end7
+
+if.end7:
+  %w.4 = phi i32 [ %w.2, %t ], [ %w.2, %t.u_crit_edge ], [ %bf.cast25, %while.body ]
+  %a.4 = phi i32 [ 0, %t ], [ %conv57, %t.u_crit_edge ], [ %bf.cast2910, %while.body ]
+  %b.4 = phi i32 [ 0, %t ], [ 0, %t.u_crit_edge ], [ %bf.cast2910, %while.body ]
+  %c.4 = phi i32 [ 0, %t ], [ 1, %t.u_crit_edge ], [ poison, %while.body ]
+  br label %v
+
+while.cond:
+  %bf.load66.pre.pre1135 = phi i64 [ 0, %entry ], [ %spec.select, %if.end42 ]
+  br i1 %tobool5.not, label %if.end54, label %while.body
+
+while.body:
+  %bf.ashr24 = ashr i64 %bf.load14, 33
+  %bf.cast25 = trunc nsw i64 %bf.ashr24 to i32
+  %bf.ashr28 = lshr i64 %bf.load14, 1
+  %bf.cast2910 = trunc i64 %bf.ashr28 to i32
+  br label %if.end7
+
+if.end36:
+  br label %v
+
+v:
+  %c.7 = phi i32 [ %c.4, %if.end7 ], [ 0, %if.end36 ]
+  br i1 true, label %if.end42, label %v.if.end83_crit_edge
+
+v.if.end83_crit_edge:
+  br label %if.end83
+
+if.end42:
+  %tobool43.not = icmp eq i32 %b.4, 0
+  %narrow = select i1 %tobool43.not, i32 0, i32 %w.4
+  %spec.select = zext i32 %narrow to i64
+  %bf.value48 = zext i32 %a.4 to i64
+  store i64 %bf.value48, ptr %n, align 8
+  store i32 %c.7, ptr %n, align 4
+  br label %while.cond
+
+if.end54:
+  %bf.load66.pre.pre113125 = phi i64 [ %bf.load66.pre.pre1135, %while.cond ], [ poison, %entry.if.end54_crit_edge ]
+  %0 = icmp eq i64 %bf.load66.pre.pre113125, 0
+  br i1 %0, label %if.end83, label %ai.if.end76_crit_edge
+
+ai.if.end76_crit_edge:
+  br label %if.end83
+
+if.end83:
+  br label %t
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
index 607d7f7888784..c029781142af3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0"
 %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334 = type { %struct._CLzmaProps.0.27.54.81.102.123.144.165.180.195.228.258.333, ptr, ptr, ptr, i32, i32, i64, i64, i32, i32, i32, [4 x i32], i32, i32, i32, i32, i32, [20 x i8] }
 %struct._CLzmaProps.0.27.54.81.102.123.144.165.180.195.228.258.333 = type { i32, i32, i32, i32 }
 
-define fastcc void @LzmaDec_DecodeReal2(ptr %p) {
+define fastcc void @LzmaDec_DecodeReal2(ptr %p, i1 %arg) {
 ; CHECK-LABEL: @LzmaDec_DecodeReal2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RANGE20_I:%.*]] = getelementptr inbounds [[STRUCT_CLZMADEC_1_28_55_82_103_124_145_166_181_196_229_259_334:%.*]], ptr [[P:%.*]], i64 0, i32 4
@@ -15,13 +15,13 @@ define fastcc void @LzmaDec_DecodeReal2(ptr %p) {
 ; CHECK:       do.body66.i:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ [[TMP3:%.*]], [[DO_COND_I:%.*]] ], [ undef, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> undef, <2 x i32> undef, <2 x i32> [[TMP0]]
-; CHECK-NEXT:    br i1 undef, label [[DO_COND_I]], label [[IF_ELSE_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[DO_COND_I]], label [[IF_ELSE_I:%.*]]
 ; CHECK:       if.else.i:
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i32> [[TMP1]], undef
 ; CHECK-NEXT:    br label [[DO_COND_I]]
 ; CHECK:       do.cond.i:
 ; CHECK-NEXT:    [[TMP3]] = phi <2 x i32> [ [[TMP2]], [[IF_ELSE_I]] ], [ [[TMP1]], [[DO_BODY66_I]] ]
-; CHECK-NEXT:    br i1 undef, label [[DO_BODY66_I]], label [[DO_END1006_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[DO_BODY66_I]], label [[DO_END1006_I:%.*]]
 ; CHECK:       do.end1006.i:
 ; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> undef, <2 x i32> undef, <2 x i32> [[TMP3]]
 ; CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[RANGE20_I]], align 4
@@ -37,7 +37,7 @@ do.body66.i:                                      ; preds = %do.cond.i, %entry
   %code.2.i = phi i32 [ %code.4.i, %do.cond.i ], [ undef, %entry ]
   %.range.2.i = select i1 undef, i32 undef, i32 %range.2.i
   %.code.2.i = select i1 undef, i32 undef, i32 %code.2.i
-  br i1 undef, label %do.cond.i, label %if.else.i
+  br i1 %arg, label %do.cond.i, label %if.else.i
 
 if.else.i:                                        ; preds = %do.body66.i
   %sub91.i = sub i32 %.range.2.i, undef
@@ -47,7 +47,7 @@ if.else.i:                                        ; preds = %do.body66.i
 do.cond.i:                                        ; preds = %if.else.i, %do.body66.i
   %range.4.i = phi i32 [ %sub91.i, %if.else.i ], [ undef, %do.body66.i ]
   %code.4.i = phi i32 [ %sub92.i, %if.else.i ], [ %.code.2.i, %do.body66.i ]
-  br i1 undef, label %do.body66.i, label %do.end1006.i
+  br i1 %arg, label %do.body66.i, label %do.end1006.i
 
 do.end1006.i:                                     ; preds = %do.cond.i
   %.range.4.i = select i1 undef, i32 undef, i32 %range.4.i
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
index f1f83c0663099..291edbbc925bd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
@@ -6,17 +6,17 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960" = type { i32, i32 }
 
-define void @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(ptr nocapture %info) {
+define void @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(ptr nocapture %info, i1 %arg) {
 ; CHECK-LABEL: @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    ret void
 ; CHECK:       if.else:
 ; CHECK-NEXT:    [[NUB5:%.*]] = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", ptr [[INFO:%.*]], i64 0, i32 1
-; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE_I_1:%.*]], label [[IF_THEN7_1:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LAND_LHS_TRUE_I_1:%.*]], label [[IF_THEN7_1:%.*]]
 ; CHECK:       land.lhs.true.i.1:
-; CHECK-NEXT:    br i1 undef, label [[FOR_INC_1:%.*]], label [[IF_THEN7_1]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_INC_1:%.*]], label [[IF_THEN7_1]]
 ; CHECK:       if.then7.1:
 ; CHECK-NEXT:    store <2 x i32> <i32 1, i32 5>, ptr [[INFO]], align 4
 ; CHECK-NEXT:    br label [[FOR_INC_1]]
@@ -30,17 +30,17 @@ define void @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btCons
 ; CHECK-NEXT:    unreachable
 ;
 entry:
-  br i1 undef, label %if.else, label %if.then
+  br i1 %arg, label %if.else, label %if.then
 
 if.then:                                          ; preds = %entry
   ret void
 
 if.else:                                          ; preds = %entry
   %nub5 = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", ptr %info, i64 0, i32 1
-  br i1 undef, label %land.lhs.true.i.1, label %if.then7.1
+  br i1 %arg, label %land.lhs.true.i.1, label %if.then7.1
 
 land.lhs.true.i.1:                                ; preds = %if.else
-  br i1 undef, label %for.inc.1, label %if.then7.1
+  br i1 %arg, label %for.inc.1, label %if.then7.1
 
 if.then7.1:                                       ; preds = %land.lhs.true.i.1, %if.else
   %inc.1 = add nsw i32 0, 1
@@ -63,7 +63,7 @@ for.inc.1:                                        ; preds = %if.then7.1, %land.l
 %class.btVector3.5.30.65.90.115.140.175.185.260.280.330 = type { [4 x float] }
 %class.btVector4.7.32.67.92.117.142.177.187.262.282.331 = type { %class.btVector3.5.30.65.90.115.140.175.185.260.280.330 }
 
-define void @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(ptr %this) {
+define void @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(ptr %this, i1 %arg) {
 ; CHECK-LABEL: @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[CLASS_GIM_TRIANGLE_CALCULATION_CACHE_9_34_69_94_119_144_179_189_264_284_332:%.*]], ptr [[THIS:%.*]], i64 0, i32 2, i64 0, i32 0, i64 1
@@ -76,9 +76,9 @@ define void @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector
 ; CHECK-NEXT:    store float [[TMP4]], ptr undef, align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x float> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[ARRAYIDX26]], align 4
-; CHECK-NEXT:    br i1 undef, label [[IF_ELSE1609:%.*]], label [[IF_THEN1595:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_ELSE1609:%.*]], label [[IF_THEN1595:%.*]]
 ; CHECK:       if.then1595:
-; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[FOR_BODY_LR_PH_I_I1702:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN:%.*]], label [[FOR_BODY_LR_PH_I_I1702:%.*]]
 ; CHECK:       for.body.lr.ph.i.i1702:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.else1609:
@@ -99,10 +99,10 @@ entry:
   %sub639 = fsub float %add626, undef
   %sub652 = fsub float %add626, %sub639
   store float %sub652, ptr %arrayidx36, align 4
-  br i1 undef, label %if.else1609, label %if.then1595
+  br i1 %arg, label %if.else1609, label %if.then1595
 
 if.then1595:                                      ; preds = %entry
-  br i1 undef, label %return, label %for.body.lr.ph.i.i1702
+  br i1 %arg, label %return, label %for.body.lr.ph.i.i1702
 
 for.body.lr.ph.i.i1702:                           ; preds = %if.then1595
   unreachable
@@ -114,34 +114,34 @@ return:                                           ; preds = %if.then1595
   ret void
 }
 
-define void @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE() {
+define void @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE(i1 %arg) {
 ; CHECK-LABEL: @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END111:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END111:%.*]]
 ; CHECK:       if.end111:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END136:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END136:%.*]]
 ; CHECK:       if.end136:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END162:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END162:%.*]]
 ; CHECK:       if.end162:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END189:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END189:%.*]]
 ; CHECK:       if.end189:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END216:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END216:%.*]]
 ; CHECK:       if.end216:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN218:%.*]], label [[IF_END225:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN218:%.*]], label [[IF_END225:%.*]]
 ; CHECK:       if.then218:
 ; CHECK-NEXT:    br label [[IF_END225]]
 ; CHECK:       if.end225:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END248:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END248:%.*]]
 ; CHECK:       if.end248:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END304:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END304:%.*]]
 ; CHECK:       if.end304:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END361:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END361:%.*]]
 ; CHECK:       if.end361:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN370:%.*]], label [[IF_END395:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN370:%.*]], label [[IF_END395:%.*]]
 ; CHECK:       if.then370:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN374:%.*]], label [[IF_END395]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN374:%.*]], label [[IF_END395]]
 ; CHECK:       if.then374:
 ; CHECK-NEXT:    br label [[IF_END395]]
 ; CHECK:       if.end395:
@@ -152,47 +152,47 @@ define void @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36b
 entry:
   %add8.i2343 = fadd float undef, undef
   %add8.i2381 = fadd float undef, undef
-  br i1 undef, label %return, label %if.end
+  br i1 %arg, label %return, label %if.end
 
 if.end:                                           ; preds = %entry
-  br i1 undef, label %return, label %if.end111
+  br i1 %arg, label %return, label %if.end111
 
 if.end111:                                        ; preds = %if.end
-  br i1 undef, label %return, label %if.end136
+  br i1 %arg, label %return, label %if.end136
 
 if.end136:                                        ; preds = %if.end111
-  br i1 undef, label %return, label %if.end162
+  br i1 %arg, label %return, label %if.end162
 
 if.end162:                                        ; preds = %if.end136
-  br i1 undef, label %return, label %if.end189
+  br i1 %arg, label %return, label %if.end189
 
 if.end189:                                        ; preds = %if.end162
-  br i1 undef, label %return, label %if.end216
+  br i1 %arg, label %return, label %if.end216
 
 if.end216:                                        ; preds = %if.end189
-  br i1 undef, label %if.then218, label %if.end225
+  br i1 %arg, label %if.then218, label %if.end225
 
 if.then218:                                       ; preds = %if.end216
   br label %if.end225
 
 if.end225:                                        ; preds = %if.then218, %if.end216
-  br i1 undef, label %return, label %if.end248
+  br i1 %arg, label %return, label %if.end248
 
 if.end248:                                        ; preds = %if.end225
-  br i1 undef, label %return, label %if.end304
+  br i1 %arg, label %return, label %if.end304
 
 if.end304:                                        ; preds = %if.end248
   %mul341 = fmul float undef, %add8.i2343
   %mul344 = fmul float undef, %add8.i2381
   %sub345 = fsub float %mul341, %mul344
-  br i1 undef, label %return, label %if.end361
+  br i1 %arg, label %return, label %if.end361
 
 if.end361:                                        ; preds = %if.end304
   %mul364 = fmul float %add8.i2381, %add8.i2381
-  br i1 undef, label %if.then370, label %if.end395
+  br i1 %arg, label %if.then370, label %if.end395
 
 if.then370:                                       ; preds = %if.end361
-  br i1 undef, label %if.then374, label %if.end395
+  br i1 %arg, label %if.then374, label %if.end395
 
 if.then374:                                       ; preds = %if.then370
   %cmp392 = fcmp olt float %sub345, 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
index 925b348cdeec1..55e691b39d78c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
@@ -7,36 +7,36 @@ target triple = "x86_64-apple-macosx10.8.0"
 %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113 = type { [4 x float] }
 
 ; Function Attrs: ssp uwtable
-define void @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(ptr %vertices) #0 align 2 {
+define void @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(ptr %vertices, i1 %arg) #0 align 2 {
 ; CHECK-LABEL: @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_1:%.*]], label [[IF_END22_1:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN17_1:%.*]], label [[IF_END22_1:%.*]]
 ; CHECK:       for.end36:
 ; CHECK-NEXT:    br label [[FOR_BODY144:%.*]]
 ; CHECK:       for.body144:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END227:%.*]], label [[FOR_BODY144]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END227:%.*]], label [[FOR_BODY144]]
 ; CHECK:       for.end227:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END271:%.*]], label [[FOR_BODY233:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END271:%.*]], label [[FOR_BODY233:%.*]]
 ; CHECK:       for.body233:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY233]], label [[FOR_END271]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY233]], label [[FOR_END271]]
 ; CHECK:       for.end271:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x float> [ splat (float 0x47EFFFFFE0000000), [[FOR_END227]] ], [ undef, [[FOR_BODY233]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x float> undef, [[TMP0]]
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN291:%.*]], label [[RETURN]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN291:%.*]], label [[RETURN]]
 ; CHECK:       if.then291:
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], splat (float 5.000000e-01)
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    br i1 undef, label [[IF_END332:%.*]], label [[IF_ELSE319:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END332:%.*]], label [[IF_ELSE319:%.*]]
 ; CHECK:       if.else319:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN325:%.*]], label [[IF_END327:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN325:%.*]], label [[IF_END327:%.*]]
 ; CHECK:       if.then325:
 ; CHECK-NEXT:    br label [[IF_END327]]
 ; CHECK:       if.end327:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN329:%.*]], label [[IF_END332]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN329:%.*]], label [[IF_END332]]
 ; CHECK:       if.then329:
 ; CHECK-NEXT:    br label [[IF_END332]]
 ; CHECK:       if.end332:
@@ -49,55 +49,55 @@ define void @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(ptr %ve
 ; CHECK:       if.then17.1:
 ; CHECK-NEXT:    br label [[IF_END22_1]]
 ; CHECK:       if.end22.1:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_2:%.*]], label [[IF_END22_2:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN17_2:%.*]], label [[IF_END22_2:%.*]]
 ; CHECK:       if.then17.2:
 ; CHECK-NEXT:    br label [[IF_END22_2]]
 ; CHECK:       if.end22.2:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END36:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END36:%.*]], label [[FOR_BODY]]
 ;
 entry:
-  br i1 undef, label %return, label %if.end
+  br i1 %arg, label %return, label %if.end
 
 if.end:                                           ; preds = %entry
   br label %for.body
 
 for.body:                                         ; preds = %if.end22.2, %if.end
-  br i1 undef, label %if.then17.1, label %if.end22.1
+  br i1 %arg, label %if.then17.1, label %if.end22.1
 
 for.end36:                                        ; preds = %if.end22.2
   br label %for.body144
 
 for.body144:                                      ; preds = %for.body144, %for.end36
-  br i1 undef, label %for.end227, label %for.body144
+  br i1 %arg, label %for.end227, label %for.body144
 
 for.end227:                                       ; preds = %for.body144
-  br i1 undef, label %for.end271, label %for.body233
+  br i1 %arg, label %for.end271, label %for.body233
 
 for.body233:                                      ; preds = %for.body233, %for.end227
-  br i1 undef, label %for.body233, label %for.end271
+  br i1 %arg, label %for.body233, label %for.end271
 
 for.end271:                                       ; preds = %for.body233, %for.end227
   %0 = phi float [ 0x47EFFFFFE0000000, %for.end227 ], [ undef, %for.body233 ]
   %1 = phi float [ 0x47EFFFFFE0000000, %for.end227 ], [ undef, %for.body233 ]
   %sub275 = fsub float undef, %1
   %sub279 = fsub float undef, %0
-  br i1 undef, label %if.then291, label %return
+  br i1 %arg, label %if.then291, label %return
 
 if.then291:                                       ; preds = %for.end271
   %mul292 = fmul float %sub275, 5.000000e-01
   %add294 = fadd float %1, %mul292
   %mul295 = fmul float %sub279, 5.000000e-01
   %add297 = fadd float %0, %mul295
-  br i1 undef, label %if.end332, label %if.else319
+  br i1 %arg, label %if.end332, label %if.else319
 
 if.else319:                                       ; preds = %if.then291
-  br i1 undef, label %if.then325, label %if.end327
+  br i1 %arg, label %if.then325, label %if.end327
 
 if.then325:                                       ; preds = %if.else319
   br label %if.end327
 
 if.end327:                                        ; preds = %if.then325, %if.else319
-  br i1 undef, label %if.then329, label %if.end332
+  br i1 %arg, label %if.then329, label %if.end332
 
 if.then329:                                       ; preds = %if.end327
   br label %if.end332
@@ -119,13 +119,13 @@ if.then17.1:                                      ; preds = %for.body
   br label %if.end22.1
 
 if.end22.1:                                       ; preds = %if.then17.1, %for.body
-  br i1 undef, label %if.then17.2, label %if.end22.2
+  br i1 %arg, label %if.then17.2, label %if.end22.2
 
 if.then17.2:                                      ; preds = %if.end22.1
   br label %if.end22.2
 
 if.end22.2:                                       ; preds = %if.then17.2, %if.end22.1
-  br i1 undef, label %for.end36, label %for.body
+  br i1 %arg, label %for.end36, label %for.body
 }
 
 attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
index 4de16a5d57793..faf4496ce2722 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
@@ -6,23 +6,23 @@ target triple = "x86_64-apple-macosx10.8.0"
 %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731" = type { ptr, ptr, ptr, ptr }
 
 ; Function Attrs: nounwind ssp uwtable
-define void @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(ptr %__first, ptr nocapture %__last) {
+define void @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(ptr %__first, ptr nocapture %__last, i1 %arg) {
 ; CHECK-LABEL: @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__FIRST:%.*]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x ptr>, ptr [[__LAST:%.*]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[TMP0]], i32 0
-; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT:%.*]], label [[WHILE_COND_I_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT:%.*]], label [[WHILE_COND_I_PREHEADER:%.*]]
 ; CHECK:       while.cond.i.preheader:
 ; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
 ; CHECK:       while.cond.i:
-; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_BODY_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_BODY_I:%.*]]
 ; CHECK:       while.body.i:
-; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_COND_I]]
+; CHECK-NEXT:    br i1 %arg, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_COND_I]]
 ; CHECK:       _ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit:
 ; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x ptr> [ [[TMP2]], [[ENTRY:%.*]] ], [ [[TMP1]], [[WHILE_COND_I]] ], [ undef, [[WHILE_BODY_I]] ]
 ; CHECK-NEXT:    store <2 x ptr> [[TMP3]], ptr [[__FIRST]], align 8
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I55:%.*]], label [[WHILE_COND:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN_I55:%.*]], label [[WHILE_COND:%.*]]
 ; CHECK:       if.then.i55:
 ; CHECK-NEXT:    br label [[WHILE_COND]]
 ; CHECK:       while.cond:
@@ -34,23 +34,23 @@ entry:
   %1 = load ptr, ptr %__last, align 8
   %_M_first3.i.i83 = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", ptr %__last, i64 0, i32 1
   %2 = load ptr, ptr %_M_first3.i.i83, align 8
-  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i.preheader
+  br i1 %arg, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i.preheader
 
 while.cond.i.preheader:                           ; preds = %entry
   br label %while.cond.i
 
 while.cond.i:                                     ; preds = %while.body.i, %while.cond.i.preheader
-  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.body.i
+  br i1 %arg, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.body.i
 
 while.body.i:                                     ; preds = %while.cond.i
-  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i
+  br i1 %arg, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i
 
 _ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit: ; preds = %while.body.i, %while.cond.i, %entry
   %3 = phi ptr [ %2, %entry ], [ %2, %while.cond.i ], [ undef, %while.body.i ]
   %4 = phi ptr [ %0, %entry ], [ %1, %while.cond.i ], [ undef, %while.body.i ]
   store ptr %4, ptr %__first, align 8
   store ptr %3, ptr %_M_first3.i.i, align 8
-  br i1 undef, label %if.then.i55, label %while.cond
+  br i1 %arg, label %if.then.i55, label %while.cond
 
 if.then.i55:                                      ; preds = %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit
   br label %while.cond
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
index 371b06869841b..fc1bd856da9c3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
@@ -5,20 +5,20 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
-define void @main() #0 {
+define void @main(i1 %arg) #0 {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       while.end:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END80:%.*]], label [[FOR_BODY75_LR_PH:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END80:%.*]], label [[FOR_BODY75_LR_PH:%.*]]
 ; CHECK:       for.body75.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY75:%.*]]
 ; CHECK:       for.body75:
 ; CHECK-NEXT:    br label [[FOR_BODY75]]
 ; CHECK:       for.end80:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END300:%.*]], label [[FOR_BODY267_LR_PH:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END300:%.*]], label [[FOR_BODY267_LR_PH:%.*]]
 ; CHECK:       for.body267.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY267:%.*]]
 ; CHECK:       for.body267:
@@ -32,18 +32,18 @@ define void @main() #0 {
 ; CHECK-NEXT:    [[ADD295:%.*]] = fadd double undef, [[MUL294]]
 ; CHECK-NEXT:    [[DIV296:%.*]] = fdiv double [[MUL283]], [[ADD295]]
 ; CHECK-NEXT:    [[ADD297]] = fadd double [[S_71010]], [[DIV296]]
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY267]], label [[FOR_END300]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY267]], label [[FOR_END300]]
 ; CHECK:       for.end300:
 ; CHECK-NEXT:    unreachable
 ;
 entry:
-  br i1 undef, label %while.body, label %while.end
+  br i1 %arg, label %while.body, label %while.end
 
 while.body:                                       ; preds = %entry
   unreachable
 
 while.end:                                        ; preds = %entry
-  br i1 undef, label %for.end80, label %for.body75.lr.ph
+  br i1 %arg, label %for.end80, label %for.body75.lr.ph
 
 for.body75.lr.ph:                                 ; preds = %while.end
   br label %for.body75
@@ -52,7 +52,7 @@ for.body75:                                       ; preds = %for.body75, %for.bo
   br label %for.body75
 
 for.end80:                                        ; preds = %while.end
-  br i1 undef, label %for.end300, label %for.body267.lr.ph
+  br i1 %arg, label %for.end300, label %for.body267.lr.ph
 
 for.body267.lr.ph:                                ; preds = %for.end80
   br label %for.body267
@@ -68,7 +68,7 @@ for.body267:                                      ; preds = %for.body267, %for.b
   %add295 = fadd double undef, %mul294
   %div296 = fdiv double %mul283, %add295
   %add297 = fadd double %s.71010, %div296
-  br i1 undef, label %for.body267, label %for.end300
+  br i1 %arg, label %for.body267, label %for.end300
 
 for.end300:                                       ; preds = %for.body267, %for.end80
   unreachable
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
index 6ac588524f845..d516126d8412d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
@@ -5,34 +5,34 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
-define void @RCModelEstimator() {
+define void @RCModelEstimator(i1 %arg) {
 ; CHECK-LABEL: @RCModelEstimator(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
 ; CHECK:       for.end.thread:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.body.lr.ph:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
 ; CHECK:       for.cond14.preheader:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
 ; CHECK:       for.body16.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY16:%.*]]
 ; CHECK:       for.body3:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
 ; CHECK:       if.then7:
 ; CHECK-NEXT:    br label [[FOR_INC11]]
 ; CHECK:       for.inc11:
 ; CHECK-NEXT:    br i1 false, label [[FOR_COND14_PREHEADER:%.*]], label [[FOR_BODY3]]
 ; CHECK:       for.body16:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
 ; CHECK:       for.end39:
-; CHECK-NEXT:    br i1 undef, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
 ; CHECK:       for.cond45.preheader:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then88:
 ; CHECK-NEXT:    br label [[IF_END103]]
 ; CHECK:       if.else:
@@ -41,28 +41,28 @@ define void @RCModelEstimator() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %for.body.lr.ph, label %for.end.thread
+  br i1 %arg, label %for.body.lr.ph, label %for.end.thread
 
 for.end.thread:                                   ; preds = %entry
   unreachable
 
 for.body.lr.ph:                                   ; preds = %entry
-  br i1 undef, label %for.end, label %for.body
+  br i1 %arg, label %for.end, label %for.body
 
 for.body:                                         ; preds = %for.body, %for.body.lr.ph
-  br i1 undef, label %for.end, label %for.body
+  br i1 %arg, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body, %for.body.lr.ph
-  br i1 undef, label %for.body3, label %if.end103
+  br i1 %arg, label %for.body3, label %if.end103
 
 for.cond14.preheader:                             ; preds = %for.inc11
-  br i1 undef, label %for.body16.lr.ph, label %if.end103
+  br i1 %arg, label %for.body16.lr.ph, label %if.end103
 
 for.body16.lr.ph:                                 ; preds = %for.cond14.preheader
   br label %for.body16
 
 for.body3:                                        ; preds = %for.inc11, %for.end
-  br i1 undef, label %if.then7, label %for.inc11
+  br i1 %arg, label %if.then7, label %for.inc11
 
 if.then7:                                         ; preds = %for.body3
   br label %for.inc11
@@ -71,13 +71,13 @@ for.inc11:                                        ; preds = %if.then7, %for.body
   br i1 false, label %for.cond14.preheader, label %for.body3
 
 for.body16:                                       ; preds = %for.body16, %for.body16.lr.ph
-  br i1 undef, label %for.end39, label %for.body16
+  br i1 %arg, label %for.end39, label %for.body16
 
 for.end39:                                        ; preds = %for.body16
-  br i1 undef, label %if.end103, label %for.cond45.preheader
+  br i1 %arg, label %if.end103, label %for.cond45.preheader
 
 for.cond45.preheader:                             ; preds = %for.end39
-  br i1 undef, label %if.then88, label %if.else
+  br i1 %arg, label %if.then88, label %if.else
 
 if.then88:                                        ; preds = %for.cond45.preheader
   %mul89 = fmul double 0.000000e+00, 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
index f0d38e1bf269c..3ad0473c84766 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
@@ -5,34 +5,34 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
-define void @RCModelEstimator() {
+define void @RCModelEstimator(i1 %arg) {
 ; CHECK-LABEL: @RCModelEstimator(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
 ; CHECK:       for.end.thread:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.body.lr.ph:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
 ; CHECK:       for.cond14.preheader:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
 ; CHECK:       for.body16.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY16:%.*]]
 ; CHECK:       for.body3:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
 ; CHECK:       if.then7:
 ; CHECK-NEXT:    br label [[FOR_INC11]]
 ; CHECK:       for.inc11:
 ; CHECK-NEXT:    br i1 false, label [[FOR_COND14_PREHEADER:%.*]], label [[FOR_BODY3]]
 ; CHECK:       for.body16:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
 ; CHECK:       for.end39:
-; CHECK-NEXT:    br i1 undef, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
 ; CHECK:       for.cond45.preheader:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then88:
 ; CHECK-NEXT:    br label [[IF_END103]]
 ; CHECK:       if.else:
@@ -41,28 +41,28 @@ define void @RCModelEstimator() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %for.body.lr.ph, label %for.end.thread
+  br i1 %arg, label %for.body.lr.ph, label %for.end.thread
 
 for.end.thread:                                   ; preds = %entry
   unreachable
 
 for.body.lr.ph:                                   ; preds = %entry
-  br i1 undef, label %for.end, label %for.body
+  br i1 %arg, label %for.end, label %for.body
 
 for.body:                                         ; preds = %for.body, %for.body.lr.ph
-  br i1 undef, label %for.end, label %for.body
+  br i1 %arg, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body, %for.body.lr.ph
-  br i1 undef, label %for.body3, label %if.end103
+  br i1 %arg, label %for.body3, label %if.end103
 
 for.cond14.preheader:                             ; preds = %for.inc11
-  br i1 undef, label %for.body16.lr.ph, label %if.end103
+  br i1 %arg, label %for.body16.lr.ph, label %if.end103
 
 for.body16.lr.ph:                                 ; preds = %for.cond14.preheader
   br label %for.body16
 
 for.body3:                                        ; preds = %for.inc11, %for.end
-  br i1 undef, label %if.then7, label %for.inc11
+  br i1 %arg, label %if.then7, label %for.inc11
 
 if.then7:                                         ; preds = %for.body3
   br label %for.inc11
@@ -71,13 +71,13 @@ for.inc11:                                        ; preds = %if.then7, %for.body
   br i1 false, label %for.cond14.preheader, label %for.body3
 
 for.body16:                                       ; preds = %for.body16, %for.body16.lr.ph
-  br i1 undef, label %for.end39, label %for.body16
+  br i1 %arg, label %for.end39, label %for.body16
 
 for.end39:                                        ; preds = %for.body16
-  br i1 undef, label %if.end103, label %for.cond45.preheader
+  br i1 %arg, label %if.end103, label %for.cond45.preheader
 
 for.cond45.preheader:                             ; preds = %for.end39
-  br i1 undef, label %if.then88, label %if.else
+  br i1 %arg, label %if.then88, label %if.else
 
 if.then88:                                        ; preds = %for.cond45.preheader
   %mul89 = fmul double 0.000000e+00, 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
index e6c46e1847dac..403a610da8d55 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-define void @main() {
+define void @main(i1 %arg) {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -27,17 +27,17 @@ define void @main() {
 ; CHECK-NEXT:    [[ADD19]] = fadd double undef, [[MUL18]]
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL13]], [[MUL14]]
 ; CHECK-NEXT:    [[ADD20]] = fadd double undef, [[SUB]]
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY12]], label [[FOR_INC21]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY12]], label [[FOR_INC21]]
 ; CHECK:       for.inc21:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END23:%.*]], label [[FOR_BODY6]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END23:%.*]], label [[FOR_BODY6]]
 ; CHECK:       for.end23:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN25:%.*]], label [[IF_THEN26:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN25:%.*]], label [[IF_THEN26:%.*]]
 ; CHECK:       if.then25:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END44:%.*]], label [[FOR_COND4_PREHEADER]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END44:%.*]], label [[FOR_COND4_PREHEADER]]
 ; CHECK:       if.then26:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.end44:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END48:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END48:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end48:
 ; CHECK-NEXT:    ret void
 ;
@@ -67,22 +67,22 @@ if.end:                                           ; preds = %for.body12
   %add19 = fadd double undef, %mul18
   %sub = fsub double %mul13, %mul14
   %add20 = fadd double undef, %sub
-  br i1 undef, label %for.body12, label %for.inc21
+  br i1 %arg, label %for.body12, label %for.inc21
 
 for.inc21:                                        ; preds = %if.end, %for.body12
-  br i1 undef, label %for.end23, label %for.body6
+  br i1 %arg, label %for.end23, label %for.body6
 
 for.end23:                                        ; preds = %for.inc21
-  br i1 undef, label %if.then25, label %if.then26
+  br i1 %arg, label %if.then25, label %if.then26
 
 if.then25:                                        ; preds = %for.end23
-  br i1 undef, label %for.end44, label %for.cond4.preheader
+  br i1 %arg, label %for.end44, label %for.cond4.preheader
 
 if.then26:                                        ; preds = %for.end23
   unreachable
 
 for.end44:                                        ; preds = %if.then25
-  br i1 undef, label %for.end48, label %for.body
+  br i1 %arg, label %for.end48, label %for.body
 
 for.end48:                                        ; preds = %for.end44
   ret void
@@ -90,7 +90,7 @@ for.end48:                                        ; preds = %for.end44
 
 %struct.hoge = type { double, double, double}
 
-define void @zot(ptr %arg) {
+define void @zot(ptr %arg, i1 %arg2) {
 ; CHECK-LABEL: @zot(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = load double, ptr undef, align 8
@@ -102,7 +102,7 @@ define void @zot(ptr %arg) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], undef
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], undef
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12:%.*]]
+; CHECK-NEXT:    br i1 %arg2, label [[BB11:%.*]], label [[BB12:%.*]]
 ; CHECK:       bb11:
 ; CHECK-NEXT:    br label [[BB14:%.*]]
 ; CHECK:       bb12:
@@ -124,7 +124,7 @@ bb:
   %tmp9 = fsub double %tmp8, undef
   %tmp10 = getelementptr inbounds %struct.hoge, ptr %arg, i64 0, i32 2
   store double %tmp9, ptr %tmp10, align 8
-  br i1 undef, label %bb11, label %bb12
+  br i1 %arg2, label %bb11, label %bb12
 
 bb11:                                             ; preds = %bb
   br label %bb14
@@ -140,15 +140,15 @@ bb14:                                             ; preds = %bb12, %bb11
 
 %struct.rc4_state.0.24 = type { i32, i32, [256 x i32] }
 
-define void @rc4_crypt(ptr nocapture %s) {
+define void @rc4_crypt(ptr nocapture %s, i1 %arg) {
 ; CHECK-LABEL: @rc4_crypt(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Y2:%.*]] = getelementptr inbounds [[STRUCT_RC4_STATE_0_24:%.*]], ptr [[S:%.*]], i64 0, i32 1
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[CONV4:%.*]] = and i32 undef, 255
 ; CHECK-NEXT:    [[CONV7:%.*]] = and i32 undef, 255
-; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[CONV4]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY]] ], [ [[CONV7]], [[FOR_BODY]] ]
@@ -158,14 +158,14 @@ define void @rc4_crypt(ptr nocapture %s) {
 ;
 entry:
   %y2 = getelementptr inbounds %struct.rc4_state.0.24, ptr %s, i64 0, i32 1
-  br i1 undef, label %for.body, label %for.end
+  br i1 %arg, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.body, %entry
   %x.045 = phi i32 [ %conv4, %for.body ], [ undef, %entry ]
   %conv4 = and i32 undef, 255
   %conv7 = and i32 undef, 255
   %idxprom842 = zext i32 %conv7 to i64
-  br i1 undef, label %for.end, label %for.body
+  br i1 %arg, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body, %entry
   %x.0.lcssa = phi i32 [ undef, %entry ], [ %conv4, %for.body ]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
index a9f92f324d6f5..d434035051f5e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
@@ -6,28 +6,28 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171 = type { i32, i32, i32, i32, i32, i32, [8 x i8] }
 
-define void @SIM4() {
+define void @SIM4(i1 %arg) {
 ; CHECK-LABEL: @SIM4(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
 ; CHECK:       lor.lhs.false:
-; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[RETURN]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END605:%.*]], label [[FOR_BODY_LR_PH:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END605:%.*]], label [[FOR_BODY_LR_PH:%.*]]
 ; CHECK:       for.body.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 undef, label [[FOR_INC603:%.*]], label [[IF_END12:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_INC603:%.*]], label [[IF_END12:%.*]]
 ; CHECK:       if.end12:
-; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE:%.*]], label [[LAND_LHS_TRUE167:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LAND_LHS_TRUE:%.*]], label [[LAND_LHS_TRUE167:%.*]]
 ; CHECK:       land.lhs.true:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN17:%.*]], label [[LAND_LHS_TRUE167]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN17:%.*]], label [[LAND_LHS_TRUE167]]
 ; CHECK:       if.then17:
-; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[LAND_RHS_LR_PH:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END98:%.*]], label [[LAND_RHS_LR_PH:%.*]]
 ; CHECK:       land.rhs.lr.ph:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end98:
-; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE167]], label [[IF_THEN103:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LAND_LHS_TRUE167]], label [[IF_THEN103:%.*]]
 ; CHECK:       if.then103:
 ; CHECK-NEXT:    [[DOTSUB100:%.*]] = select i1 undef, i32 250, i32 undef
 ; CHECK-NEXT:    [[MUL114:%.*]] = shl nsw i32 [[DOTSUB100]], 2
@@ -37,11 +37,11 @@ define void @SIM4() {
 ; CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
 ; CHECK:       for.cond.i:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x i32> [ undef, [[LAND_RHS_I874:%.*]] ], [ [[TMP1]], [[IF_THEN103]] ]
-; CHECK-NEXT:    br i1 undef, label [[LAND_RHS_I874]], label [[FOR_END_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LAND_RHS_I874]], label [[FOR_END_I:%.*]]
 ; CHECK:       land.rhs.i874:
-; CHECK-NEXT:    br i1 undef, label [[FOR_COND_I]], label [[FOR_END_I]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_COND_I]], label [[FOR_END_I]]
 ; CHECK:       for.end.i:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
 ; CHECK:       if.then.i:
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], undef
 ; CHECK-NEXT:    br label [[EXTEND_BW_EXIT:%.*]]
@@ -52,15 +52,15 @@ define void @SIM4() {
 ; CHECK:       for.body28.lr.ph.i:
 ; CHECK-NEXT:    br label [[FOR_END33_I]]
 ; CHECK:       for.end33.i:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END58_I:%.*]], label [[FOR_BODY52_LR_PH_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END58_I:%.*]], label [[FOR_BODY52_LR_PH_I:%.*]]
 ; CHECK:       for.body52.lr.ph.i:
 ; CHECK-NEXT:    br label [[FOR_END58_I]]
 ; CHECK:       for.end58.i:
 ; CHECK-NEXT:    br label [[WHILE_COND260_I:%.*]]
 ; CHECK:       while.cond260.i:
-; CHECK-NEXT:    br i1 undef, label [[LAND_RHS263_I:%.*]], label [[WHILE_END275_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[LAND_RHS263_I:%.*]], label [[WHILE_END275_I:%.*]]
 ; CHECK:       land.rhs263.i:
-; CHECK-NEXT:    br i1 undef, label [[WHILE_COND260_I]], label [[WHILE_END275_I]]
+; CHECK-NEXT:    br i1 %arg, label [[WHILE_COND260_I]], label [[WHILE_END275_I]]
 ; CHECK:       while.end275.i:
 ; CHECK-NEXT:    br label [[EXTEND_BW_EXIT]]
 ; CHECK:       extend_bw.exit:
@@ -73,42 +73,42 @@ define void @SIM4() {
 ; CHECK:       land.lhs.true167:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.inc603:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY]], label [[FOR_END605]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY]], label [[FOR_END605]]
 ; CHECK:       for.end605:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       return:
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %return, label %lor.lhs.false
+  br i1 %arg, label %return, label %lor.lhs.false
 
 lor.lhs.false:                                    ; preds = %entry
-  br i1 undef, label %return, label %if.end
+  br i1 %arg, label %return, label %if.end
 
 if.end:                                           ; preds = %lor.lhs.false
-  br i1 undef, label %for.end605, label %for.body.lr.ph
+  br i1 %arg, label %for.end605, label %for.body.lr.ph
 
 for.body.lr.ph:                                   ; preds = %if.end
   br label %for.body
 
 for.body:                                         ; preds = %for.inc603, %for.body.lr.ph
-  br i1 undef, label %for.inc603, label %if.end12
+  br i1 %arg, label %for.inc603, label %if.end12
 
 if.end12:                                         ; preds = %for.body
-  br i1 undef, label %land.lhs.true, label %land.lhs.true167
+  br i1 %arg, label %land.lhs.true, label %land.lhs.true167
 
 land.lhs.true:                                    ; preds = %if.end12
-  br i1 undef, label %if.then17, label %land.lhs.true167
+  br i1 %arg, label %if.then17, label %land.lhs.true167
 
 if.then17:                                        ; preds = %land.lhs.true
-  br i1 undef, label %if.end98, label %land.rhs.lr.ph
+  br i1 %arg, label %if.end98, label %land.rhs.lr.ph
 
 land.rhs.lr.ph:                                   ; preds = %if.then17
   unreachable
 
 if.end98:                                         ; preds = %if.then17
   %from299 = getelementptr inbounds %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171, ptr undef, i64 0, i32 1
-  br i1 undef, label %land.lhs.true167, label %if.then103
+  br i1 %arg, label %land.lhs.true167, label %if.then103
 
 if.then103:                                       ; preds = %if.end98
   %.sub100 = select i1 undef, i32 250, i32 undef
@@ -119,13 +119,13 @@ if.then103:                                       ; preds = %if.end98
 for.cond.i:                                       ; preds = %land.rhs.i874, %if.then103
   %row.0.i = phi i32 [ undef, %land.rhs.i874 ], [ %.sub100, %if.then103 ]
   %col.0.i = phi i32 [ undef, %land.rhs.i874 ], [ %cond125, %if.then103 ]
-  br i1 undef, label %land.rhs.i874, label %for.end.i
+  br i1 %arg, label %land.rhs.i874, label %for.end.i
 
 land.rhs.i874:                                    ; preds = %for.cond.i
-  br i1 undef, label %for.cond.i, label %for.end.i
+  br i1 %arg, label %for.cond.i, label %for.end.i
 
 for.end.i:                                        ; preds = %land.rhs.i874, %for.cond.i
-  br i1 undef, label %if.then.i, label %if.end.i
+  br i1 %arg, label %if.then.i, label %if.end.i
 
 if.then.i:                                        ; preds = %for.end.i
   %add14.i = add nsw i32 %row.0.i, undef
@@ -141,7 +141,7 @@ for.body28.lr.ph.i:                               ; preds = %if.end.i
   br label %for.end33.i
 
 for.end33.i:                                      ; preds = %for.body28.lr.ph.i, %if.end.i
-  br i1 undef, label %for.end58.i, label %for.body52.lr.ph.i
+  br i1 %arg, label %for.end58.i, label %for.body52.lr.ph.i
 
 for.body52.lr.ph.i:                               ; preds = %for.end33.i
   br label %for.end58.i
@@ -150,10 +150,10 @@ for.end58.i:                                      ; preds = %for.body52.lr.ph.i,
   br label %while.cond260.i
 
 while.cond260.i:                                  ; preds = %land.rhs263.i, %for.end58.i
-  br i1 undef, label %land.rhs263.i, label %while.end275.i
+  br i1 %arg, label %land.rhs263.i, label %while.end275.i
 
 land.rhs263.i:                                    ; preds = %while.cond260.i
-  br i1 undef, label %while.cond260.i, label %while.end275.i
+  br i1 %arg, label %while.cond260.i, label %while.end275.i
 
 while.end275.i:                                   ; preds = %land.rhs263.i, %while.cond260.i
   br label %extend_bw.exit
@@ -174,7 +174,7 @@ land.lhs.true167:                                 ; preds = %if.then157, %extend
   unreachable
 
 for.inc603:                                       ; preds = %for.body
-  br i1 undef, label %for.body, label %for.end605
+  br i1 %arg, label %for.body, label %for.end605
 
 for.end605:                                       ; preds = %for.inc603, %if.end
   unreachable
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
index 42ad20ff578c1..d13a8578d1e00 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -4,25 +4,25 @@
 %struct.Ray = type { %struct.Vec, %struct.Vec }
 %struct.Vec = type { double, double, double }
 
-define void @main() {
+define void @main(i1 %arg) {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
 ; CHECK:       cond.true:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.end:
 ; CHECK-NEXT:    br label [[INVOKE_CONT:%.*]]
 ; CHECK:       invoke.cont:
-; CHECK-NEXT:    br i1 undef, label [[ARRAYCTOR_CONT:%.*]], label [[INVOKE_CONT]]
+; CHECK-NEXT:    br i1 %arg, label [[ARRAYCTOR_CONT:%.*]], label [[INVOKE_CONT]]
 ; CHECK:       arrayctor.cont:
 ; CHECK-NEXT:    [[AGG_TMP101211_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY:%.*]], ptr undef, i64 0, i32 1, i32 0
 ; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER:%.*]]
 ; CHECK:       for.cond36.preheader:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY42_LR_PH_US:%.*]], label [[_Z5CLAMPD_EXIT_1:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY42_LR_PH_US:%.*]], label [[_Z5CLAMPD_EXIT_1:%.*]]
 ; CHECK:       cond.false51.us:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       cond.true48.us:
-; CHECK-NEXT:    br i1 undef, label [[COND_TRUE63_US:%.*]], label [[COND_FALSE66_US:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[COND_TRUE63_US:%.*]], label [[COND_FALSE66_US:%.*]]
 ; CHECK:       cond.false66.us:
 ; CHECK-NEXT:    [[ADD_I276_US:%.*]] = fadd double 0.000000e+00, 0x3EB0C6F7A0B5ED8D
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> <double poison, double 0xBFA5CC2D1960285F>, double [[ADD_I276_US]], i32 0
@@ -36,12 +36,12 @@ define void @main() {
 ; CHECK:       cond.true63.us:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.body42.lr.ph.us:
-; CHECK-NEXT:    br i1 undef, label [[COND_TRUE48_US:%.*]], label [[COND_FALSE51_US:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[COND_TRUE48_US:%.*]], label [[COND_FALSE51_US:%.*]]
 ; CHECK:       _Z5clampd.exit.1:
 ; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER]]
 ;
 entry:
-  br i1 undef, label %cond.true, label %cond.end
+  br i1 %arg, label %cond.true, label %cond.end
 
 cond.true:
   unreachable
@@ -50,7 +50,7 @@ cond.end:
   br label %invoke.cont
 
 invoke.cont:
-  br i1 undef, label %arrayctor.cont, label %invoke.cont
+  br i1 %arg, label %arrayctor.cont, label %invoke.cont
 
 arrayctor.cont:
   %agg.tmp99208.sroa.1.8.idx388 = getelementptr inbounds %struct.Ray, ptr undef, i64 0, i32 0, i32 1
@@ -59,13 +59,13 @@ arrayctor.cont:
   br label %for.cond36.preheader
 
 for.cond36.preheader:
-  br i1 undef, label %for.body42.lr.ph.us, label %_Z5clampd.exit.1
+  br i1 %arg, label %for.body42.lr.ph.us, label %_Z5clampd.exit.1
 
 cond.false51.us:
   unreachable
 
 cond.true48.us:
-  br i1 undef, label %cond.true63.us, label %cond.false66.us
+  br i1 %arg, label %cond.true63.us, label %cond.false66.us
 
 cond.false66.us:
   %add.i276.us = fadd double 0.000000e+00, 0.000001e+00
@@ -87,16 +87,16 @@ cond.true63.us:
   unreachable
 
 for.body42.lr.ph.us:
-  br i1 undef, label %cond.true48.us, label %cond.false51.us
+  br i1 %arg, label %cond.true48.us, label %cond.false51.us
 
 _Z5clampd.exit.1:
   br label %for.cond36.preheader
 }
 
-define void @test() {
+define void @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN78:%.*]], label [[IF_THEN38:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN78:%.*]], label [[IF_THEN38:%.*]]
 ; CHECK:       if.then38:
 ; CHECK-NEXT:    [[AGG_TMP74663_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY:%.*]], ptr undef, i64 0, i32 1, i32 0
 ; CHECK-NEXT:    store <2 x double> <double 0x3FFA356C1D8A7F76, double 0x3FFDC4F38B38BEF4>, ptr [[AGG_TMP74663_SROA_0_0_IDX]], align 8
@@ -105,7 +105,7 @@ define void @test() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %if.then78, label %if.then38
+  br i1 %arg, label %if.then78, label %if.then38
 
 if.then38:
   %mul.i.i790 = fmul double 0.0, 0.1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
index e3a860a4c6f06..c7c4b06be2d19 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
@@ -16,7 +16,7 @@ target triple = "x86_64-apple-macosx10.9.0"
 
 
 ;define fastcc void @bar() {
-define void @bar() {
+define void @bar(i1 %arg) {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [[TMP0:%.*]], ptr undef, i64 0, i32 1, i32 0
@@ -32,16 +32,16 @@ define void @bar() {
 ; CHECK-NEXT:    store double [[I7]], ptr [[I1]], align 8
 ; CHECK-NEXT:    [[I10]] = load double, ptr [[I3]], align 8
 ; CHECK-NEXT:    [[TMP0]] = load <2 x double>, ptr [[I2]], align 8
-; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB11:%.*]], label [[BB12:%.*]]
 ; CHECK:       bb11:
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb12:
 ; CHECK-NEXT:    store <2 x double> [[TMP0]], ptr [[I4]], align 8
-; CHECK-NEXT:    br i1 undef, label [[BB13:%.*]], label [[BB14:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB13:%.*]], label [[BB14:%.*]]
 ; CHECK:       bb13:
 ; CHECK-NEXT:    br label [[BB14]]
 ; CHECK:       bb14:
-; CHECK-NEXT:    br i1 undef, label [[BB15:%.*]], label [[BB16]]
+; CHECK-NEXT:    br i1 %arg, label [[BB15:%.*]], label [[BB16]]
 ; CHECK:       bb15:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       bb16:
@@ -51,7 +51,7 @@ define void @bar() {
 ; CHECK-NEXT:      i32 103, label [[BB6]]
 ; CHECK-NEXT:    ]
 ; CHECK:       bb17:
-; CHECK-NEXT:    br i1 undef, label [[BB6]], label [[BB18:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB6]], label [[BB18:%.*]]
 ; CHECK:       bb18:
 ; CHECK-NEXT:    unreachable
 ;
@@ -71,7 +71,7 @@ bb6:                                              ; preds = %bb17, %bb16, %bb16,
   store double %i7, ptr %i1, align 8
   %i9 = load double, ptr %i2, align 8
   %i10 = load double, ptr %i3, align 8
-  br i1 undef, label %bb11, label %bb12
+  br i1 %arg, label %bb11, label %bb12
 
 bb11:                                             ; preds = %bb6
   ret void
@@ -79,13 +79,13 @@ bb11:                                             ; preds = %bb6
 bb12:                                             ; preds = %bb6
   store double %i9, ptr %i4, align 8
   store double %i10, ptr %i5, align 8
-  br i1 undef, label %bb13, label %bb14
+  br i1 %arg, label %bb13, label %bb14
 
 bb13:                                             ; preds = %bb12
   br label %bb14
 
 bb14:                                             ; preds = %bb13, %bb12
-  br i1 undef, label %bb15, label %bb16
+  br i1 %arg, label %bb15, label %bb16
 
 bb15:                                             ; preds = %bb14
   unreachable
@@ -97,7 +97,7 @@ bb16:                                             ; preds = %bb14
   ]
 
 bb17:                                             ; preds = %bb16
-  br i1 undef, label %bb6, label %bb18
+  br i1 %arg, label %bb6, label %bb18
 
 bb18:                                             ; preds = %bb17
   unreachable
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
index 739e3964c2685..7510b8fb83e34 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
@@ -280,10 +280,10 @@ return:                                           ; preds = %entry, %if.end
 
 @a = external global double, align 8
 
-define void @PR19646(ptr %this) {
+define void @PR19646(ptr %this, i1 %arg) {
 ; CHECK-LABEL: @PR19646(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[IF_END13:%.*]], label [[IF_END13]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END13:%.*]], label [[IF_END13]]
 ; CHECK:       sw.epilog7:
 ; CHECK-NEXT:    [[DOTIN:%.*]] = getelementptr inbounds [[CLASS_B_53_55:%.*]], ptr [[THIS:%.*]], i64 0, i32 0, i32 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[DOTIN]], align 8
@@ -294,7 +294,7 @@ define void @PR19646(ptr %this) {
 ; CHECK-NEXT:    [[_DY:%.*]] = getelementptr inbounds [[CLASS_B_53_55]], ptr [[THIS]], i64 0, i32 0, i32 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load double, ptr [[_DY]], align 8
 ; CHECK-NEXT:    [[ADD10:%.*]] = fadd double [[ADD8]], [[TMP2]]
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN12:%.*]], label [[IF_END13]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN12:%.*]], label [[IF_END13]]
 ; CHECK:       if.then12:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr undef, align 8
 ; CHECK-NEXT:    br label [[IF_END13]]
@@ -304,7 +304,7 @@ define void @PR19646(ptr %this) {
 ; CHECK-NEXT:    unreachable
 ;
 entry:
-  br i1 undef, label %if.end13, label %if.end13
+  br i1 %arg, label %if.end13, label %if.end13
 
 sw.epilog7:                                       ; No predecessors!
   %.in = getelementptr inbounds %class.B.53.55, ptr %this, i64 0, i32 0, i32 1
@@ -316,7 +316,7 @@ sw.epilog7:                                       ; No predecessors!
   %_dy = getelementptr inbounds %class.B.53.55, ptr %this, i64 0, i32 0, i32 2
   %2 = load double, ptr %_dy, align 8
   %add10 = fadd double %add8, %2
-  br i1 undef, label %if.then12, label %if.end13
+  br i1 %arg, label %if.then12, label %if.end13
 
 if.then12:                                        ; preds = %sw.epilog7
   %3 = load double, ptr undef, align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/full-match-with-poison-scalar.ll b/llvm/test/Transforms/SLPVectorizer/X86/full-match-with-poison-scalar.ll
new file mode 100644
index 0000000000000..dadd22217a3e6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/full-match-with-poison-scalar.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-200 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FUNC_135_EXIT_I:.*]]
+; CHECK:       [[FUNC_135_EXIT_I]]:
+; CHECK-NEXT:    [[G_228_PROMOTED166_I1105_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[G_228_PROMOTED166_I1105_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 0, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 2, i32 2, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP5]], <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 poison, i32 28, i32 29, i32 30, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[G_228_PROMOTED166_I1105_I]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 0, i32 15
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP3]], i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 23, i32 24, i32 25, i32 26, i32 7, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP12]], <16 x i1> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 31>
+; CHECK-NEXT:    [[TMP15:%.*]] = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> [[TMP14]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i1 [[TMP15]] to i16
+; CHECK-NEXT:    br label %[[IF_THEN_SINK_SPLIT_I:.*]]
+; CHECK:       [[FOR_COND7_PREHEADER_I1132_1_I:.*]]:
+; CHECK-NEXT:    br label %[[IF_THEN_SINK_SPLIT_I]]
+; CHECK:       [[IF_THEN_SINK_SPLIT_I]]:
+; CHECK-NEXT:    [[XOR58_5_I_I1203_3_3_SINK_I:%.*]] = phi i16 [ 0, %[[FOR_COND7_PREHEADER_I1132_1_I]] ], [ [[TMP16]], %[[FUNC_135_EXIT_I]] ]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %func_135.exit.i
+
+func_135.exit.i:
+  %g_228.promoted166.i1105.i = phi i32 [ 0, %entry ]
+  %cmp55.i.i1199.i = icmp ugt i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.1.i.i1200.i = icmp ugt i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.2.i.i1201.i = icmp ugt i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.3.i.i1202.i = icmp ugt i32 %g_228.promoted166.i1105.i, 0
+  %0 = xor i1 %cmp55.2.i.i1201.i, %cmp55.3.i.i1202.i
+  %1 = xor i1 %cmp55.1.i.i1200.i, %0
+  %2 = xor i1 %cmp55.i.i1199.i, %1
+  %dec.i.i1204.i = add i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.i.i1199.1.i = icmp ugt i32 %dec.i.i1204.i, 0
+  %cmp55.1.i.i1200.1.i = icmp ugt i32 %dec.i.i1204.i, 0
+  %cmp55.2.i.i1201.1.i = icmp ugt i32 %dec.i.i1204.i, 0
+  %cmp55.3.i.i1202.1.i = icmp ugt i32 %dec.i.i1204.i, 0
+  %3 = xor i1 %cmp55.2.i.i1201.1.i, %cmp55.3.i.i1202.1.i
+  %4 = xor i1 %cmp55.1.i.i1200.1.i, %3
+  %5 = xor i1 %cmp55.i.i1199.1.i, %4
+  %6 = xor i1 %2, %5
+  %dec.i.i1204.1.i = add i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.i.i1199.2.i = icmp ugt i32 %dec.i.i1204.1.i, 0
+  %cmp55.1.i.i1200.2.i = icmp ugt i32 %dec.i.i1204.1.i, 0
+  %cmp55.2.i.i1201.2.i = icmp ugt i32 %dec.i.i1204.1.i, 0
+  %cmp55.3.i.i1202.2.i = icmp ugt i32 %dec.i.i1204.1.i, 0
+  %7 = add i32 0, 0
+  %8 = icmp ult i32 %7, 0
+  %9 = xor i1 %cmp55.3.i.i1202.2.i, %8
+  %10 = xor i1 %cmp55.2.i.i1201.2.i, %9
+  %11 = xor i1 %cmp55.1.i.i1200.2.i, %10
+  %12 = xor i1 %cmp55.i.i1199.2.i, %11
+  %13 = xor i1 %12, %6
+  %dec.i.i1204.2.i = add i32 %g_228.promoted166.i1105.i, 0
+  %cmp55.i.i1199.3.i = icmp ugt i32 %dec.i.i1204.2.i, 0
+  %cmp55.1.i.i1200.3.i = icmp ugt i32 %dec.i.i1204.2.i, 0
+  %cmp55.2.i.i1201.3.i = icmp ugt i32 %dec.i.i1204.2.i, 0
+  %14 = xor i1 %cmp55.1.i.i1200.3.i, %cmp55.2.i.i1201.3.i
+  %15 = xor i1 %cmp55.i.i1199.3.i, %14
+  %16 = xor i1 %15, %13
+  %17 = zext i1 %16 to i16
+  br label %if.then.sink.split.i
+
+for.cond7.preheader.i1132.1.i:
+  br label %if.then.sink.split.i
+
+if.then.sink.split.i:
+  %xor58.5.i.i1203.3.3.sink.i = phi i16 [ 0, %for.cond7.preheader.i1132.1.i ], [ %17, %func_135.exit.i ]
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll b/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
index f614796916baa..d474218e84cca 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
@@ -116,7 +116,7 @@ then:
 %struct.wombat.0 = type { %struct.bar }
 %struct.bar = type { [3 x double], [3 x double], double, double, i16, ptr, i32, [3 x double] }
 
-define double @preserve_loop_info(ptr %arg) {
+define double @preserve_loop_info(ptr %arg, i1 %arg2) {
 ; CHECK-LABEL: @preserve_loop_info(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [3 x double], align 16
@@ -124,15 +124,15 @@ define double @preserve_loop_info(ptr %arg) {
 ; CHECK:       outer.header:
 ; CHECK-NEXT:    br label [[INNER:%.*]]
 ; CHECK:       inner:
-; CHECK-NEXT:    br i1 undef, label [[OUTER_LATCH:%.*]], label [[INNER]]
+; CHECK-NEXT:    br i1 %arg2, label [[OUTER_LATCH:%.*]], label [[INNER]]
 ; CHECK:       outer.latch:
-; CHECK-NEXT:    br i1 undef, label [[BB:%.*]], label [[OUTER_HEADER]]
+; CHECK-NEXT:    br i1 %arg2, label [[BB:%.*]], label [[OUTER_HEADER]]
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr undef, align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [3 x double], ptr [[TMP]], i64 0, i64 1
 ; CHECK-NEXT:    br label [[LOOP_3HEADER:%.*]]
 ; CHECK:       loop.3header:
-; CHECK-NEXT:    br i1 undef, label [[LOOP_3LATCH:%.*]], label [[BB9:%.*]]
+; CHECK-NEXT:    br i1 %arg2, label [[LOOP_3LATCH:%.*]], label [[BB9:%.*]]
 ; CHECK:       bb9:
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [3 x double], ptr [[TMP5]], i64 undef, i64 1
 ; CHECK-NEXT:    store double undef, ptr [[TMP]], align 16
@@ -140,7 +140,7 @@ define double @preserve_loop_info(ptr %arg) {
 ; CHECK-NEXT:    store double [[TMP12]], ptr [[TMP7]], align 8
 ; CHECK-NEXT:    br label [[LOOP_3LATCH]]
 ; CHECK:       loop.3latch:
-; CHECK-NEXT:    br i1 undef, label [[BB14:%.*]], label [[LOOP_3HEADER]]
+; CHECK-NEXT:    br i1 %arg2, label [[BB14:%.*]], label [[LOOP_3HEADER]]
 ; CHECK:       bb14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call double undef(ptr [[TMP]], ptr [[ARG:%.*]])
 ; CHECK-NEXT:    ret double undef
@@ -153,10 +153,10 @@ outer.header:                                              ; preds = %bb3, %bb
   br label %inner
 
 inner:
-  br i1 undef, label %outer.latch, label %inner
+  br i1 %arg2, label %outer.latch, label %inner
 
 outer.latch:                                              ; preds = %bb16
-  br i1 undef, label %bb, label %outer.header
+  br i1 %arg2, label %bb, label %outer.header
 
 bb:                                              ; preds = %bb3
   %tmp5 = load ptr, ptr undef, align 8
@@ -164,7 +164,7 @@ bb:                                              ; preds = %bb3
   br label %loop.3header
 
 loop.3header:                                              ; preds = %bb13, %bb4
-  br i1 undef, label %loop.3latch, label %bb9
+  br i1 %arg2, label %loop.3latch, label %bb9
 
 bb9:                                              ; preds = %bb8
   %tmp10 = getelementptr inbounds [3 x double], ptr %tmp5, i64 undef, i64 1
@@ -174,7 +174,7 @@ bb9:                                              ; preds = %bb8
   br label %loop.3latch
 
 loop.3latch:                                             ; preds = %bb11, %bb8
-  br i1 undef, label %bb14, label %loop.3header
+  br i1 %arg2, label %bb14, label %loop.3header
 
 bb14:                                             ; preds = %bb13
   %tmp15 = call double undef(ptr %tmp, ptr %arg)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index b4cbdfa10460f..9c9a89ac76d8b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -22,10 +22,10 @@ define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
 ; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
 ; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
 ; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
 ; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -38,10 +38,10 @@ define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
 ; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
 ; AVX-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
 ; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
 ; AVX-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/no-scheduled-instructions.ll b/llvm/test/Transforms/SLPVectorizer/X86/no-scheduled-instructions.ll
index 43d8c1342cbbc..eea22c1861b44 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/no-scheduled-instructions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/no-scheduled-instructions.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -S -passes=slp-vectorizer -mattr=+avx -mtriple=x86_64 < %s | FileCheck %s
 
-define void @test() {
+define void @test(i1 %arg) {
 ; CHECK-LABEL: define void @test
-; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK-SAME: (i1 %arg) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    br i1 %arg, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef>)
 ; CHECK-NEXT:    call void @f(i32 noundef [[TMP1]])
@@ -20,7 +20,7 @@ define void @test() {
   %i32 = extractelement <4 x i32> zeroinitializer, i64 1
   %i33 = extractelement <4 x i32> zeroinitializer, i64 2
   %i34 = extractelement <4 x i32> zeroinitializer, i64 3
-  br i1 undef, label %bb1, label %bb2
+  br i1 %arg, label %bb1, label %bb2
 
 bb1:
   %i11 = mul nsw i32 %i28, %i27
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index 61938d01e57ac..eac438ac07252 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -349,12 +349,12 @@ define void @good_load_order() {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
@@ -363,7 +363,7 @@ define void @good_load_order() {
 ; CHECK-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
 ; CHECK-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; CHECK-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; CHECK-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
@@ -384,12 +384,12 @@ define void @good_load_order() {
 ; SSE2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
 ; SSE2-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; SSE2-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
+; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
 ; SSE2-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
+; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
 ; SSE2-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; SSE2-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
 ; SSE2-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; SSE2-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; SSE2-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
@@ -398,7 +398,7 @@ define void @good_load_order() {
 ; SSE2-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; SSE2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
 ; SSE2-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds nuw [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
 ; SSE2-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; SSE2-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; SSE2-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/opt.ll b/llvm/test/Transforms/SLPVectorizer/X86/opt.ll
index 95da098b8b143..bfe93dc7f88b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/opt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/opt.ll
@@ -21,13 +21,13 @@ define void @test1(ptr %a, ptr %b, ptr %c) {
 ; NOSLP-NEXT:    [[I0:%.*]] = load double, ptr [[A:%.*]], align 8
 ; NOSLP-NEXT:    [[I1:%.*]] = load double, ptr [[B:%.*]], align 8
 ; NOSLP-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
-; NOSLP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; NOSLP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
 ; NOSLP-NEXT:    [[I3:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
-; NOSLP-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 8
+; NOSLP-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 8
 ; NOSLP-NEXT:    [[I4:%.*]] = load double, ptr [[ARRAYIDX4]], align 8
 ; NOSLP-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
 ; NOSLP-NEXT:    store double [[MUL]], ptr [[C:%.*]], align 8
-; NOSLP-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 8
+; NOSLP-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 8
 ; NOSLP-NEXT:    store double [[MUL5]], ptr [[ARRAYIDX5]], align 8
 ; NOSLP-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
index ad3ebf57ab7a4..f9d6c3eab350f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
@@ -26,10 +26,10 @@ entry:
 declare ptr @objc_msgSend(ptr, ptr, ...)
 declare i32 @personality_v0(...)
 
-define void @invoketest() personality ptr @personality_v0 {
+define void @invoketest(i1 %arg) personality ptr @personality_v0 {
 ; CHECK-LABEL: @invoketest(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.true:
 ; CHECK-NEXT:    [[CALL49:%.*]] = invoke double @objc_msgSend(ptr undef, ptr undef)
 ; CHECK-NEXT:    to label [[COND_TRUE54:%.*]] unwind label [[LPAD:%.*]]
@@ -43,7 +43,7 @@ define void @invoketest() personality ptr @personality_v0 {
 ; CHECK-NEXT:    [[CALL59:%.*]] = invoke double @objc_msgSend(ptr undef, ptr undef)
 ; CHECK-NEXT:    to label [[COND_END60]] unwind label [[LPAD]]
 ; CHECK:       cond.end60:
-; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[IF_THEN63:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END98:%.*]], label [[IF_THEN63:%.*]]
 ; CHECK:       if.then63:
 ; CHECK-NEXT:    br label [[IF_END98]]
 ; CHECK:       lpad:
@@ -56,7 +56,7 @@ define void @invoketest() personality ptr @personality_v0 {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %cond.true, label %cond.false
+  br i1 %arg, label %cond.true, label %cond.false
 
 cond.true:
   %call49 = invoke double @objc_msgSend(ptr undef, ptr undef)
@@ -77,7 +77,7 @@ cond.false57:
 cond.end60:
   %cond126 = phi double [ %call49, %cond.true54 ], [ %call51, %cond.false57 ]
   %cond61 = phi double [ %call56, %cond.true54 ], [ %call59, %cond.false57 ]
-  br i1 undef, label %if.end98, label %if.then63
+  br i1 %arg, label %if.end98, label %if.then63
 
 if.then63:
   %conv69 = fptrunc double undef to float
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/partail.ll b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
index 20ad09a632826..4cdf9670394f1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
@@ -2,12 +2,12 @@
 ; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
 
 ; Function Attrs: nounwind uwtable
-define void @get_block(i32 %y_pos) local_unnamed_addr #0 {
+define void @get_block(i32 %y_pos, i1 %arg) local_unnamed_addr #0 {
 ; CHECK-LABEL: @get_block(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LAND_LHS_TRUE:%.*]]
 ; CHECK:       land.lhs.true:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end:
@@ -43,7 +43,7 @@ entry:
   br label %land.lhs.true
 
 land.lhs.true:                                    ; preds = %entry
-  br i1 undef, label %if.then, label %if.end
+  br i1 %arg, label %if.then, label %if.end
 
 if.then:                                          ; preds = %land.lhs.true
   unreachable
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/perfect-matched-reused-bv.ll b/llvm/test/Transforms/SLPVectorizer/X86/perfect-matched-reused-bv.ll
new file mode 100644
index 0000000000000..1053e0fc10669
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/perfect-matched-reused-bv.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP4:%.*]], %[[BB4:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    br i1 false, label %[[BB7:.*]], label %[[BB4]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    [[TMP4]] = add <2 x i32> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    br i1 false, label %[[BB7]], label %[[BB1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <2 x i32> [ [[TMP1]], %[[BB1]] ], [ [[TMP3]], %[[BB4]] ]
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb1
+
+bb1:
+  %phi = phi i32 [ 0, %bb ], [ %add6, %bb4 ]
+  %phi2 = phi i32 [ 0, %bb ], [ %add, %bb4 ]
+  %or = or i32 %phi2, 0
+  %or3 = or i32 %phi, 0
+  br i1 false, label %bb7, label %bb4
+
+bb4:
+  %add = add i32 0, 0
+  %add5 = add i32 0, 0
+  %add6 = add i32 %phi, 0
+  br i1 false, label %bb7, label %bb1
+
+bb7:
+  %phi8 = phi i32 [ %or, %bb1 ], [ %add5, %bb4 ]
+  %phi9 = phi i32 [ %or3, %bb1 ], [ %add6, %bb4 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
index 78bfb8df51aeb..df85656800aac 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
@@ -277,13 +277,13 @@ for.end:                                          ; preds = %for.body
   ret float %add31
 }
 
-define void @test(ptr %i1, ptr %i2, ptr %o) {
+define void @test(ptr %i1, ptr %i2, ptr %o, i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, ptr [[I1:%.*]], align 16
 ; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, ptr [[I1]], i64 1
 ; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, ptr [[I1_GEP1]], align 16
-; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[THEN:%.*]], label [[END:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, ptr [[I2:%.*]], align 16
 ; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, ptr [[I2]], i64 1
@@ -305,7 +305,7 @@ entry:
   %i1.0 = load x86_fp80, ptr %i1, align 16
   %i1.gep1 = getelementptr x86_fp80, ptr %i1, i64 1
   %i1.1 = load x86_fp80, ptr %i1.gep1, align 16
-  br i1 undef, label %then, label %end
+  br i1 %arg, label %then, label %end
 
 then:
   %i2.0 = load x86_fp80, ptr %i2, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
index 88ac2d9dc42d1..ddbe943e32446 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
@@ -6,7 +6,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 target triple = "i386-apple-macosx10.9.0"
 
 
-define void @test(ptr %i1, ptr %i2, ptr %o) {
+define void @test(ptr %i1, ptr %i2, ptr %o, i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1_0:%.*]] = load double, ptr [[I1:%.*]], align 16
@@ -14,7 +14,7 @@ define void @test(ptr %i1, ptr %i2, ptr %o) {
 ; CHECK-NEXT:    [[I1_1:%.*]] = load double, ptr [[I1_GEP1]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[I1_0]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1_1]], i32 1
-; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[THEN:%.*]], label [[END:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[I2_0:%.*]] = load double, ptr [[I2:%.*]], align 16
 ; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds double, ptr [[I2]], i64 1
@@ -38,7 +38,7 @@ entry:
   %i1.0 = load double, ptr %i1, align 16
   %i1.gep1 = getelementptr double, ptr %i1, i64 1
   %i1.1 = load double, ptr %i1.gep1, align 16
-  br i1 undef, label %then, label %end
+  br i1 %arg, label %then, label %end
 
 then:
   %i2.0 = load double, ptr %i2, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
index b61b004797959..733033efa3bfe 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
@@ -3,9 +3,9 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32"
 target triple = "i686-pc-win32"
 
-define hidden fastcc void @"System.PrimitiveTypesParser.TryParseIEEE754<char>(char*,uint,double&)"() unnamed_addr {
+define hidden fastcc void @"System.PrimitiveTypesParser.TryParseIEEE754<char>(char*,uint,double&)"(i1 %arg) unnamed_addr {
 "@0":
-  br i1 undef, label %"@38.lr.ph", label %"@37"
+  br i1 %arg, label %"@38.lr.ph", label %"@37"
 
 "@37":                                            ; preds = %"@38.lr.ph", %"@44", %"@0"
   ret void
@@ -13,10 +13,10 @@ define hidden fastcc void @"System.PrimitiveTypesParser.TryParseIEEE754<char>(ch
 "@44":                                            ; preds = %"@38.lr.ph"
   %0 = add i64 undef, undef
   %1 = add i32 %mainPartDigits.loc.0.ph45, 1
-  br i1 undef, label %"@38.lr.ph", label %"@37"
+  br i1 %arg, label %"@38.lr.ph", label %"@37"
 
 "@38.lr.ph":                                      ; preds = %"@44", %"@0"
   %mainDoublePart.loc.0.ph46 = phi i64 [ %0, %"@44" ], [ 0, %"@0" ]
   %mainPartDigits.loc.0.ph45 = phi i32 [ %1, %"@44" ], [ 0, %"@0" ]
-  br i1 undef, label %"@44", label %"@37"
+  br i1 %arg, label %"@44", label %"@37"
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
index 892a2b6cee3be..f249394c91788 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
@@ -13,19 +13,19 @@ define void @foo() {
 ; SSE-LABEL: @foo(
 ; SSE-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 16
 ; SSE-NEXT:    store i32 [[TMP1]], ptr @a, align 16
-; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds (i8, ptr @b, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a, i64 4), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds (i8, ptr @a, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a, i64 12), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds (i8, ptr @a, i64 16), align 16
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a, i64 20), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds (i8, ptr @a, i64 24), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds (i8, ptr @a, i64 28), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @b, i64 8), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 4), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 8), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 12), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 16), align 16
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 20), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 28), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @foo(
 ; AVX-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 16
-; AVX-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds (i8, ptr @b, i64 8), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @b, i64 8), align 8
 ; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i64 0
 ; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP2]], i64 1
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -34,7 +34,7 @@ define void @foo() {
 ;
 ; AVX512-LABEL: @foo(
 ; AVX512-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 16
-; AVX512-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds (i8, ptr @b, i64 8), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @b, i64 8), align 8
 ; AVX512-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i64 0
 ; AVX512-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP2]], i64 1
 ; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
index 6001c659a37b7..925c334cb5f20 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
@@ -7,11 +7,11 @@
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -23,11 +23,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -39,11 +39,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -55,11 +55,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -71,11 +71,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -107,35 +107,35 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -146,13 +146,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -163,13 +163,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -180,13 +180,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -223,39 +223,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -263,19 +263,19 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
 ; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
@@ -291,19 +291,19 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX2-LABEL: @gather_load_3(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; AVX2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; AVX2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; AVX2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; AVX2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
 ; AVX2-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
@@ -378,20 +378,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -419,13 +419,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -447,13 +447,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_4(
-; AVX2-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; AVX2-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; AVX2-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; AVX2-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; AVX2-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; AVX2-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; AVX2-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; AVX2-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; AVX2-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; AVX2-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; AVX2-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; AVX2-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; AVX2-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -541,12 +541,12 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
@@ -562,21 +562,21 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
 ; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
@@ -592,26 +592,26 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
@@ -640,26 +640,26 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
index b1942bfe073f2..dc1ba4ec7e7ab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
@@ -7,11 +7,11 @@
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -23,11 +23,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -39,11 +39,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -55,11 +55,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -71,11 +71,11 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
@@ -107,35 +107,35 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -146,13 +146,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -163,13 +163,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -180,13 +180,13 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
+; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
 ; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
@@ -223,39 +223,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -263,19 +263,19 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
 ; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
@@ -291,19 +291,19 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX2-LABEL: @gather_load_3(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 44
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
 ; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 60
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
 ; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 72
+; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
 ; AVX2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 36
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
 ; AVX2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 24
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
 ; AVX2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 84
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
 ; AVX2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
 ; AVX2-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
@@ -378,20 +378,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -419,13 +419,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -447,13 +447,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_4(
-; AVX2-NEXT:    [[T6:%.*]] = getelementptr inbounds i8, ptr [[T1:%.*]], i64 44
-; AVX2-NEXT:    [[T10:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 16
-; AVX2-NEXT:    [[T14:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 60
-; AVX2-NEXT:    [[T18:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 72
-; AVX2-NEXT:    [[T22:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 36
-; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 24
-; AVX2-NEXT:    [[T30:%.*]] = getelementptr inbounds i8, ptr [[T1]], i64 84
+; AVX2-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
+; AVX2-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
+; AVX2-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
+; AVX2-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
+; AVX2-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
+; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
+; AVX2-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
 ; AVX2-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -541,12 +541,12 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
@@ -562,21 +562,21 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
 ; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
@@ -592,26 +592,26 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
@@ -640,26 +640,26 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
index 540ea4eb659fe..ac8b2428a3dea 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
@@ -3,13 +3,13 @@
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 ; This type is not supported by SLP
-define i1 @test(ptr %i1, ptr %i2) {
+define i1 @test(ptr %i1, ptr %i2, i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, ptr [[I1:%.*]], align 16
 ; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, ptr [[I1]], i64 1
 ; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, ptr [[I1_GEP1]], align 16
-; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[THEN:%.*]], label [[END:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, ptr [[I2:%.*]], align 16
 ; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, ptr [[I2]], i64 1
@@ -25,7 +25,7 @@ entry:
   %i1.0 = load x86_fp80, ptr %i1, align 16
   %i1.gep1 = getelementptr x86_fp80, ptr %i1, i64 1
   %i1.1 = load x86_fp80, ptr %i1.gep1, align 16
-  br i1 undef, label %then, label %end
+  br i1 %arg, label %then, label %end
 then:
   %i2.0 = load x86_fp80, ptr %i2, align 16
   %i2.gep1 = getelementptr inbounds x86_fp80, ptr %i2, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
index f9815529a2375..5f2199aef17ee 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
@@ -3,10 +3,10 @@
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-define void @hoge() {
+define void @hoge(i1 %arg) {
 ; CHECK-LABEL: @hoge(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb2:
@@ -57,7 +57,7 @@ define void @hoge() {
 ; CHECK-NEXT:    unreachable
 ;
 bb:
-  br i1 undef, label %bb1, label %bb2
+  br i1 %arg, label %bb1, label %bb2
 
 bb1:                                              ; preds = %bb
   ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_reordered_users.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_reordered_users.ll
index 69b4639d9c131..2e6df0007e08b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_reordered_users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_with_reordered_users.ll
@@ -42,16 +42,16 @@
 ; comment out reorderTopToBottom() and remove the stores.
 
 
-define void @reorder_crash(ptr %ptr) {
+define void @reorder_crash(ptr %ptr, i1 %arg) {
 ; CHECK-LABEL: @reorder_crash(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[BB0:%.*]], label [[BB12:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB0:%.*]], label [[BB12:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[PTR:%.*]], align 4
 ; CHECK-NEXT:    store <4 x float> [[TMP0]], ptr [[PTR]], align 4
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb12:
-; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[PTR]], align 4
 ; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[PTR]], align 4
@@ -69,7 +69,7 @@ entry:
   %gep1 = getelementptr inbounds float, ptr %ptr, i64 1
   %gep2 = getelementptr inbounds float, ptr %ptr, i64 2
   %gep3 = getelementptr inbounds float, ptr %ptr, i64 3
-  br i1 undef, label %bb0, label %bb12
+  br i1 %arg, label %bb0, label %bb12
 
 bb0:
   ; Used by phi in this order: 1, 0, 2, 3
@@ -86,7 +86,7 @@ bb0:
   br label %bb3
 
 bb12:
-  br i1 undef, label %bb1, label %bb2
+  br i1 %arg, label %bb1, label %bb2
 
 bb1:
   ; Used by phi in this order: 1, 0, 2, 3
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
index cb955ff91ed81..ef1149a108e29 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -4,10 +4,10 @@
 %"struct.std::array" = type { [32 x i8] }
 
 ; Function Attrs: nounwind uwtable
-define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() unnamed_addr #0 align 2 {
+define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv(i1 %arg) unnamed_addr #0 align 2 {
 ; CHECK-LABEL: @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[IF_END50_I:%.*]], label [[IF_THEN22_I:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END50_I:%.*]], label [[IF_THEN22_I:%.*]]
 ; CHECK:       if.then22.i:
 ; CHECK-NEXT:    [[SUB_I:%.*]] = add nsw i32 undef, -1
 ; CHECK-NEXT:    [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]]
@@ -36,7 +36,7 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %if.end50.i, label %if.then22.i
+  br i1 %arg, label %if.end50.i, label %if.then22.i
 
 if.then22.i:                                      ; preds = %entry
   %sub.i = add nsw i32 undef, -1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
index 79ce74bd21dbc..b900bd3a8c331 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reuse-extracts-in-wider-vect.ll
@@ -4,7 +4,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 
 %struct.S = type { [3 x float], [3 x float], [4 x float] }
 
-define i32 @foo(i32 %0, ptr %1, ptr %2)  {
+define i32 @foo(i32 %0, ptr %1, ptr %2, i1 %arg)  {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    [[T4:%.*]] = alloca [[STRUCT_S:%.*]], align 8
 ; CHECK-NEXT:    [[T8:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[T4]], i64 0, i32 1
@@ -19,7 +19,7 @@ define i32 @foo(i32 %0, ptr %1, ptr %2)  {
 ; CHECK-NEXT:    [[T21:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[T4]], i64 0, i32 2, i64 0
 ; CHECK-NEXT:    store <4 x float> [[SHUFFLE]], ptr [[T21]], align 4
 ; CHECK-NEXT:    [[T89]] = load <2 x float>, ptr [[T9]], align 4
-; CHECK-NEXT:    br i1 undef, label [[T37]], label [[T55:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[T37]], label [[T55:%.*]]
 ; CHECK:       t55:
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -49,7 +49,7 @@ t37:
   %t89 = load <2 x float>, ptr %t9, align 4
   %x23 = extractelement <2 x float> %t89, i32 0
   %x24 = extractelement <2 x float> %t89, i32 1
-  br i1 undef, label %t37, label %t55
+  br i1 %arg, label %t37, label %t55
 
 t55:
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll b/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
index edd1a2a3a2fff..364b0f4c1a3a7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revectorized_rdx_crash.ll
@@ -12,10 +12,10 @@
 ; iteration (it was matched and vectorized, which added a use of a deleted
 ; instruction)
 
-define void @test() {
+define void @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[IF_END:%.*]], label [[FOR_COND_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_END:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; CHECK:       for.cond.preheader:
 ; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [100 x i32], ptr undef, i64 0, i64 2
 ; CHECK-NEXT:    [[I1:%.*]] = getelementptr inbounds [100 x i32], ptr undef, i64 0, i64 3
@@ -35,7 +35,7 @@ define void @test() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %if.end, label %for.cond.preheader
+  br i1 %arg, label %if.end, label %for.cond.preheader
 
 for.cond.preheader:                               ; preds = %entry
   %i = getelementptr inbounds [100 x i32], ptr undef, i64 0, i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-constant.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-constant.ll
new file mode 100644
index 0000000000000..0b5e279dea5ba
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-constant.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-- -passes=slp-vectorizer -mcpu=x86-64    | FileCheck %s
+; RUN: opt < %s -S -mtriple=x86_64-- -passes=slp-vectorizer -mcpu=x86-64-v2 | FileCheck %s
+; RUN: opt < %s -S -mtriple=x86_64-- -passes=slp-vectorizer -mcpu=x86-64-v3 | FileCheck %s
+; RUN: opt < %s -S -mtriple=x86_64-- -passes=slp-vectorizer -mcpu=x86-64-v4 | FileCheck %s
+
+@arr = global [20 x i64] zeroinitializer, align 16
+
+define void @PR111126() {
+; CHECK-LABEL: @PR111126(
+; CHECK-NEXT:    store <4 x i64> splat (i64 1), ptr @arr, align 16
+; CHECK-NEXT:    store <4 x i64> splat (i64 1), ptr getelementptr inbounds (i8, ptr @arr, i64 32), align 16
+; CHECK-NEXT:    store <4 x i64> splat (i64 1), ptr getelementptr inbounds (i8, ptr @arr, i64 64), align 16
+; CHECK-NEXT:    store <4 x i64> splat (i64 1), ptr getelementptr inbounds (i8, ptr @arr, i64 96), align 16
+; CHECK-NEXT:    store <4 x i64> splat (i64 1), ptr getelementptr inbounds (i8, ptr @arr, i64 128), align 16
+; CHECK-NEXT:    ret void
+;
+  store i64 1, ptr @arr, align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 8), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 16), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 24), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 32), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 40), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 48), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 56), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 64), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 72), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 80), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 88), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 96), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 104), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 112), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 120), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 128), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 136), align 8
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 144), align 16
+  store i64 1, ptr getelementptr inbounds (i8, ptr @arr, i64 152), align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
index eb5e218f057ce..ac9454967719e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; as elements of the vectorized tree.
 ; PR19621
 
-define void @test() {
+define void @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  bb279:
 ; CHECK-NEXT:    br label [[BB283:%.*]]
@@ -22,13 +22,13 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], undef
 ; CHECK-NEXT:    br label [[BB21_I:%.*]]
 ; CHECK:       bb21.i:
-; CHECK-NEXT:    br i1 undef, label [[BB22_I:%.*]], label [[EXIT]]
+; CHECK-NEXT:    br i1 %arg, label [[BB22_I:%.*]], label [[EXIT]]
 ; CHECK:       bb22.i:
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> undef, [[TMP4]]
 ; CHECK-NEXT:    br label [[BB32_I:%.*]]
 ; CHECK:       bb32.i:
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi <2 x double> [ [[TMP5]], [[BB22_I]] ], [ zeroinitializer, [[BB32_I]] ]
-; CHECK-NEXT:    br i1 undef, label [[BB32_I]], label [[BB21_I]]
+; CHECK-NEXT:    br i1 %arg, label [[BB32_I]], label [[BB21_I]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP7:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], <double undef, double 0.000000e+00>
@@ -57,7 +57,7 @@ bb284:
   br label %bb21.i
 
 bb21.i:
-  br i1 undef, label %bb22.i, label %exit
+  br i1 %arg, label %bb22.i, label %exit
 
 bb22.i:
   %tmp24.i = fadd double undef, %tmp9.i
@@ -67,7 +67,7 @@ bb22.i:
 bb32.i:
   %xs.0.i = phi double [ %tmp24.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
   %ys.0.i = phi double [ %tmp26.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
-  br i1 undef, label %bb32.i, label %bb21.i
+  br i1 %arg, label %bb32.i, label %bb21.i
 
 exit:
   %tmp303 = fpext float %Av.sroa.0.0 to double
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll b/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
index f870cb44f4e5f..c5cdcdc1eb1a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; as elements of the vectorized tree.
 ; PR19621
 
-define void @test() {
+define void @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  bb279:
 ; CHECK-NEXT:    br label [[BB283:%.*]]
@@ -22,13 +22,13 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], undef
 ; CHECK-NEXT:    br label [[BB21_I:%.*]]
 ; CHECK:       bb21.i:
-; CHECK-NEXT:    br i1 undef, label [[BB22_I:%.*]], label [[EXIT]]
+; CHECK-NEXT:    br i1 %arg, label [[BB22_I:%.*]], label [[EXIT]]
 ; CHECK:       bb22.i:
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> undef, [[TMP4]]
 ; CHECK-NEXT:    br label [[BB32_I:%.*]]
 ; CHECK:       bb32.i:
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi <2 x double> [ [[TMP5]], [[BB22_I]] ], [ zeroinitializer, [[BB32_I]] ]
-; CHECK-NEXT:    br i1 undef, label [[BB32_I]], label [[BB21_I]]
+; CHECK-NEXT:    br i1 %arg, label [[BB32_I]], label [[BB21_I]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP7:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], <double undef, double 0.000000e+00>
@@ -57,7 +57,7 @@ bb284:
   br label %bb21.i
 
 bb21.i:
-  br i1 undef, label %bb22.i, label %exit
+  br i1 %arg, label %bb22.i, label %exit
 
 bb22.i:
   %tmp24.i = fadd double undef, %tmp9.i
@@ -67,7 +67,7 @@ bb22.i:
 bb32.i:
   %xs.0.i = phi double [ %tmp24.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
   %ys.0.i = phi double [ %tmp26.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
-  br i1 undef, label %bb32.i, label %bb21.i
+  br i1 %arg, label %bb32.i, label %bb21.i
 
 exit:
   %tmp303 = fpext float %Av.sroa.0.0 to double
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll
index 17f9f371ff6ef..a69849fabcef6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=slp-vectorizer -S -mcpu=cascadelake -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
 
-define void @foo() {
+define void @foo(i1 %arg) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CONV:%.*]] = uitofp i16 undef to float
@@ -14,7 +14,7 @@ define void @foo() {
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi <4 x float> [ [[TMP1]], [[BB1]] ], [ [[TMP14:%.*]], [[BB3:%.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr undef, align 8
-; CHECK-NEXT:    br i1 undef, label [[BB3]], label [[BB4:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[BB3]], label [[BB4:%.*]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[TMP4:%.*]] = fpext <4 x float> [[TMP2]] to <4 x double>
 ; CHECK-NEXT:    [[CONV2:%.*]] = uitofp i16 undef to double
@@ -46,7 +46,7 @@ bb2:
   %2 = phi float [ undef, %bb1 ], [ %11, %bb3 ]
   %3 = phi float [ undef, %bb1 ], [ %12, %bb3 ]
   %4 = load double, ptr undef, align 8
-  br i1 undef, label %bb3, label %bb4
+  br i1 %arg, label %bb3, label %bb4
 
 bb4:
   %ext = fpext float %3 to double
diff --git a/llvm/test/Transforms/SLPVectorizer/fix-113880.ll b/llvm/test/Transforms/SLPVectorizer/fix-113880.ll
new file mode 100644
index 0000000000000..6c3c95b0d9359
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/fix-113880.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -slp-max-reg-size=1024 %s | FileCheck %s
+
+define ptr @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    store <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, ptr null, align 8
+; CHECK-NEXT:    ret ptr null
+;
+  store double poison, ptr null, align 8
+  %1 = getelementptr i8, ptr null, i64 8
+  %2 = fmul double 0.000000e+00, 0.000000e+00
+  store double %2, ptr %1, align 8
+  %3 = getelementptr i8, ptr null, i64 16
+  %4 = fmul double 0.000000e+00, 0.000000e+00
+  store double %4, ptr %3, align 8
+  %5 = getelementptr i8, ptr null, i64 24
+  store double %2, ptr %5, align 8
+  ret ptr null
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index b160c0174c0a7..ce13f478d3811 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -447,3 +447,37 @@ for.end.loopexit:
   store <4 x i32> %4, ptr %out2, align 4
   ret void
 }
+
+define void @test14(<8 x i1> %0) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.vector.insert.v16i1.v8i1(<16 x i1> poison, <8 x i1> [[TMP0:%.*]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i1> [[TMP1]], <16 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i1> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <16 x i16> [ [[TMP6]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.vector.extract.v4i16.v16i16(<16 x i16> [[TMP7]], i64 12)
+; CHECK-NEXT:    [[OR0:%.*]] = or <4 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    ret void
+;
+entry:
+  %sext0 = sext <8 x i1> %0 to <8 x i16>
+  %sext1 = sext <8 x i1> %0 to <8 x i16>
+  %1 = shufflevector <8 x i16> %sext0, <8 x i16> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = shufflevector <8 x i16> %sext0, <8 x i16> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %3 = shufflevector <8 x i16> %sext1, <8 x i16> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = shufflevector <8 x i16> %sext1, <8 x i16> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  br label %for.end.loopexit
+
+for.end.loopexit:
+  %phi0 = phi <4 x i16> [ %1, %entry ]
+  %phi1 = phi <4 x i16> [ %2, %entry ]
+  %phi2 = phi <4 x i16> [ %3, %entry ]
+  %phi3 = phi <4 x i16> [ %4, %entry ]
+  %or0 = or <4 x i16> %phi1, zeroinitializer
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll b/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
index 9d6371b13e08a..a728515f0fdc3 100644
--- a/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
@@ -7,10 +7,10 @@
 ; to UMax and thus same reduction kind is returned.
 ; The routine's later code merely assumes the instruction to be a select.
 
-define dso_local void @test() {
+define dso_local void @test(i1 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 undef, label [[NEXT:%.*]], label [[THEN:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[NEXT:%.*]], label [[THEN:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[UM:%.*]] = call i8 @llvm.umax.i8(i8 0, i8 undef)
 ; CHECK-NEXT:    [[SELCMP:%.*]] = icmp ult i8 [[UM]], undef
@@ -21,7 +21,7 @@ define dso_local void @test() {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br i1 undef, label %next, label %then
+  br i1 %arg, label %next, label %then
 
 then:
   %um = call i8 @llvm.umax.i8(i8 0, i8 undef)
diff --git a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
index 3756afadbf884..87862b929a751 100644
--- a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
+++ b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
@@ -364,12 +364,12 @@ define i32 @alloca_used_in_maybe_throwing_call(ptr %data, i64 %n) personality pt
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[I0:%.*]] = invoke i32 @user_of_alloca(ptr [[RETVAL]])
-; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[UW:%.*]]
+; CHECK-NEXT:            to label [[CONT:%.*]] unwind label [[UW:%.*]]
 ; CHECK:       cont:
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       uw:
 ; CHECK-NEXT:    [[I1:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    catch ptr null
+; CHECK-NEXT:            catch ptr null
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[RETVAL]], align 4
@@ -424,10 +424,10 @@ define i32 @alloca_used_in_maybe_throwing_call_with_same_dests(ptr %data, i64 %n
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[I0:%.*]] = invoke i32 @user_of_alloca(ptr [[RETVAL]])
-; CHECK-NEXT:    to label [[END:%.*]] unwind label [[UW:%.*]]
+; CHECK-NEXT:            to label [[END:%.*]] unwind label [[UW:%.*]]
 ; CHECK:       uw:
 ; CHECK-NEXT:    [[I1:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    catch ptr null
+; CHECK-NEXT:            catch ptr null
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
 ; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[RETVAL]], align 4
diff --git a/llvm/test/Transforms/SROA/readonlynocapture.ll b/llvm/test/Transforms/SROA/readonlynocapture.ll
new file mode 100644
index 0000000000000..2d02996d806ed
--- /dev/null
+++ b/llvm/test/Transforms/SROA/readonlynocapture.ll
@@ -0,0 +1,358 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=sroa -S | FileCheck %s
+
+declare void @callee(ptr nocapture readonly %p)
+
+define i32 @simple() {
+; CHECK-LABEL: @simple(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[L1]]
+;
+  %a = alloca i32
+  store i32 0, ptr %a
+  call void @callee(ptr %a)
+  %l1 = load i32, ptr %a
+  ret i32 %l1
+}
+
+define i32 @smallbig() {
+; CHECK-LABEL: @smallbig(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i8 0, ptr [[A]], align 1
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[L1]]
+;
+  %a = alloca i32
+  store i8 0, ptr %a
+  call void @callee(ptr %a)
+  %l1 = load i32, ptr %a
+  ret i32 %l1
+}
+
+define i32 @twoalloc() {
+; CHECK-LABEL: @twoalloc(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee(ptr %a)
+  %l1 = load i32, ptr %a
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+define i32 @twostore() {
+; CHECK-LABEL: @twostore(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    store i32 2, ptr [[A]], align 4
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[L]]
+;
+  %a = alloca i32
+  store i32 1, ptr %a
+  call void @callee(ptr %a)
+  store i32 2, ptr %a
+  %l = load i32, ptr %a
+  ret i32 %l
+}
+
+define float @differenttype() {
+; CHECK-LABEL: @differenttype(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT:    ret float [[L2]]
+;
+  %a = alloca {i32, i32}
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee(ptr %a)
+  %l2 = load float, ptr %b
+  ret float %l2
+}
+
+define i32 @twoalloc_store64(i64 %x) {
+; CHECK-LABEL: @twoalloc_store64(
+; CHECK-NEXT:    [[A:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    store i64 [[X:%.*]], ptr [[A]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    ret i32 [[L2]]
+;
+  %a = alloca i64
+  store i64 %x, ptr %a
+  call void @callee(ptr %a)
+  %l1 = load i32, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  %l2 = load i32, ptr %b
+  ret i32 %l2
+}
+
+define i32 @twocalls() {
+; CHECK-LABEL: @twocalls(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee(ptr %a)
+  %l1 = load i32, ptr %a
+  call void @callee(ptr %a)
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+define i32 @volatile() {
+; CHECK-LABEL: @volatile(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store volatile i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load volatile i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store volatile i32 1, ptr %b
+  call void @callee(ptr %a)
+  %l1 = load volatile i32, ptr %a
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+define i32 @atomic() {
+; CHECK-LABEL: @atomic(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load atomic i32, ptr [[A]] seq_cst, align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee(ptr %a)
+  %l1 = load atomic i32, ptr %a seq_cst, align 4
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+define i32 @notdominating() {
+; CHECK-LABEL: @notdominating(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  %b = getelementptr i32, ptr %a, i32 1
+  %l1 = load i32, ptr %a
+  %l2 = load i32, ptr %b
+  store i32 0, ptr %a
+  store i32 1, ptr %b
+  call void @callee(ptr %a)
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+declare void @callee_notreadonly(ptr %p)
+define i32 @notreadonly() {
+; CHECK-LABEL: @notreadonly(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee_notreadonly(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee_notreadonly(ptr %a)
+  %l1 = load i32, ptr %a
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+declare void @callee_multiuse(ptr nocapture readonly %p, ptr nocapture readonly %q)
+define i32 @multiuse() {
+; CHECK-LABEL: @multiuse(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
+; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
+; CHECK-NEXT:    call void @callee_multiuse(ptr [[A]], ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[L2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[L1]], [[L2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}
+  store i32 0, ptr %a
+  %b = getelementptr i32, ptr %a, i32 1
+  store i32 1, ptr %b
+  call void @callee_multiuse(ptr %a, ptr %a)
+  %l1 = load i32, ptr %a
+  %l2 = load i32, ptr %b
+  %r = add i32 %l1, %l2
+  ret i32 %r
+}
+
+define i32 @memcpyed(ptr %src) {
+; CHECK-LABEL: @memcpyed(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A]], ptr [[SRC:%.*]], i64 4, i1 false)
+; CHECK-NEXT:    [[L1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[L1]]
+;
+  %a = alloca i32
+  store i32 0, ptr %a
+  call void @callee(ptr %a)
+  call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 4, i1 false)
+  %l1 = load i32, ptr %a
+  ret i32 %l1
+}
+
+define ptr @memcpyedsplit(ptr %src) {
+; CHECK-LABEL: @memcpyedsplit(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i64, i64 }, align 8
+; CHECK-NEXT:    store i8 1, ptr [[A]], align 1
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i64, ptr [[A]], i32 1
+; CHECK-NEXT:    store ptr null, ptr [[B]], align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A]], ptr [[SRC:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @callee(ptr [[A]])
+; CHECK-NEXT:    [[L1:%.*]] = load ptr, ptr [[B]], align 8
+; CHECK-NEXT:    ret ptr [[L1]]
+;
+  %a = alloca { i64, i64 }
+  store i8 1, ptr %a
+  %b = getelementptr i64, ptr %a, i32 1
+  store ptr null, ptr %b
+  call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 16, i1 false)
+  call void @callee(ptr %a)
+  %l1 = load ptr, ptr %b
+  ret ptr %l1
+}
+
+; This struct contains padding bits. The load should not be replaced by poison.
+%struct.LoadImmediateInfo = type { i32 }
+define void @incompletestruct(i1 %b, i1 %c) {
+; CHECK-LABEL: @incompletestruct(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LII:%.*]] = alloca [[STRUCT_LOADIMMEDIATEINFO:%.*]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    [[BF_LOAD:%.*]] = load i32, ptr [[LII]], align 4
+; CHECK-NEXT:    [[BF_CLEAR4:%.*]] = and i32 [[BF_LOAD]], -262144
+; CHECK-NEXT:    [[BF_SET5:%.*]] = select i1 [[B:%.*]], i32 196608, i32 131072
+; CHECK-NEXT:    [[BF_SET12:%.*]] = or disjoint i32 [[BF_SET5]], [[BF_CLEAR4]]
+; CHECK-NEXT:    store i32 [[BF_SET12]], ptr [[LII]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %LII = alloca %struct.LoadImmediateInfo, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %LII)
+  %bf.load = load i32, ptr %LII, align 4
+  %bf.clear4 = and i32 %bf.load, -262144
+  %bf.set5 = select i1 %b, i32 196608, i32 131072
+  %bf.set12 = or disjoint i32 %bf.set5, %bf.clear4
+  store i32 %bf.set12, ptr %LII, align 4
+  call void @callee(ptr %LII)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %LII)
+  ret void
+}
+
+define void @incompletestruct_bb(i1 %b, i1 %c) {
+; CHECK-LABEL: @incompletestruct_bb(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LII:%.*]] = alloca [[STRUCT_LOADIMMEDIATEINFO:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    [[BF_LOAD:%.*]] = load i32, ptr [[LII]], align 4
+; CHECK-NEXT:    [[BF_CLEAR4:%.*]] = and i32 [[BF_LOAD]], -262144
+; CHECK-NEXT:    [[BF_SET5:%.*]] = select i1 [[B:%.*]], i32 196608, i32 131072
+; CHECK-NEXT:    [[BF_SET12:%.*]] = or disjoint i32 [[BF_SET5]], [[BF_CLEAR4]]
+; CHECK-NEXT:    store i32 [[BF_SET12]], ptr [[LII]], align 4
+; CHECK-NEXT:    call void @callee(ptr [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %LII = alloca %struct.LoadImmediateInfo, align 4
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %LII)
+  %bf.load = load i32, ptr %LII, align 4
+  %bf.clear4 = and i32 %bf.load, -262144
+  %bf.set5 = select i1 %b, i32 196608, i32 131072
+  %bf.set12 = or disjoint i32 %bf.set5, %bf.clear4
+  store i32 %bf.set12, ptr %LII, align 4
+  call void @callee(ptr %LII)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %LII)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-instcombine.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-instcombine.ll
index d1c15306a72d0..ff1e165c8c54a 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-instcombine.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-instcombine.ll
@@ -18,9 +18,9 @@
 define dso_local void @merge(ptr nocapture readonly %params) local_unnamed_addr align 2 {
 ; CHECK-LABEL: @merge(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SAVEPAIRLISTS3:%.*]] = getelementptr inbounds i8, ptr [[PARAMS:%.*]], i64 144
+; CHECK-NEXT:    [[SAVEPAIRLISTS3:%.*]] = getelementptr inbounds nuw i8, ptr [[PARAMS:%.*]], i64 144
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SAVEPAIRLISTS3]], align 8
-; CHECK-NEXT:    [[USEPAIRLISTS4:%.*]] = getelementptr inbounds i8, ptr [[PARAMS]], i64 148
+; CHECK-NEXT:    [[USEPAIRLISTS4:%.*]] = getelementptr inbounds nuw i8, ptr [[PARAMS]], i64 148
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[USEPAIRLISTS4]], align 4
 ; CHECK-NEXT:    [[TOBOOL54_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL54_NOT]], label [[LOR_LHS_FALSE55:%.*]], label [[IF_END109:%.*]]
@@ -37,7 +37,7 @@ define dso_local void @merge(ptr nocapture readonly %params) local_unnamed_addr
 ; CHECK:       if.then117:
 ; CHECK-NEXT:    ret void
 ; CHECK:       if.then138:
-; CHECK-NEXT:    [[DOTIN:%.*]] = getelementptr inbounds i8, ptr [[PARAMS]], i64 172
+; CHECK-NEXT:    [[DOTIN:%.*]] = getelementptr inbounds nuw i8, ptr [[PARAMS]], i64 172
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTIN]], align 4
 ; CHECK-NEXT:    [[TOBOOL139_NOT:%.*]] = icmp eq i32 [[TMP2]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL139_NOT]], label [[IF_ELSE147:%.*]], label [[IF_THEN140:%.*]]
@@ -89,9 +89,9 @@ if.else147:                                       ; preds = %if.then138
 ;; Check the last store is deleted.
 define i32 @load(ptr nocapture %a, ptr nocapture %b) {
 ; CHECK-LABEL: @load(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 4
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 8
 ; CHECK-NEXT:    call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
 ; CHECK-NEXT:    ret i32 [[TMP2]]
diff --git a/llvm/test/Transforms/Scalarizer/crash-bug.ll b/llvm/test/Transforms/Scalarizer/crash-bug.ll
index 2195a37564c6a..692fc07e3ea5c 100644
--- a/llvm/test/Transforms/Scalarizer/crash-bug.ll
+++ b/llvm/test/Transforms/Scalarizer/crash-bug.ll
@@ -3,14 +3,14 @@
 
 ; Don't crash
 
-define void @foo() {
-; CHECK-LABEL: define void @foo() {
+define void @foo(i1 %arg) {
+; CHECK-LABEL: define void @foo(i1 %arg) {
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[BB2_VEC_I1:%.*]] = phi i16 [ 200, [[TMP0:%.*]] ], [ [[BB2_VEC_I1]], [[BB2:%.*]] ]
-; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB2]]
+; CHECK-NEXT:    br i1 %arg, label [[BB3:%.*]], label [[BB2]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    ret void
 ;
@@ -22,7 +22,7 @@ bb2:                                        ; preds = %bb1
 
 bb1:                                        ; preds = %bb2, %0
   %bb1_vec = phi <2 x i16> [ <i16 100, i16 200>, %0 ], [ %bb2_vec, %bb2 ]
-  br i1 undef, label %bb3, label %bb2
+  br i1 %arg, label %bb3, label %bb2
 
 bb3:
   ret void
diff --git a/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
index e5ba35ca2c4ee..4f4dff84e1294 100644
--- a/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
@@ -7,7 +7,7 @@ define i16 @f1() !dbg !5 {
   ret i16 undef, !dbg !9
 }
 
-define void @f2() !dbg !10 {
+define void @f2(i1 %arg) !dbg !10 {
 bb1:
   %_tmp7 = tail call i16 @f1(), !dbg !13
 ; CHECK: call i16 @f1(), !dbg !13
@@ -16,7 +16,7 @@ bb1:
   br label %vector.body
 
 vector.body:
-  br i1 undef, label %middle.block, label %vector.body
+  br i1 %arg, label %middle.block, label %vector.body
 
 middle.block:
   ret void, !dbg !15
diff --git a/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll b/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
index 639fd3d7c0957..3dd768e593eb4 100644
--- a/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
+++ b/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
@@ -7,7 +7,7 @@ define i16 @f1() !dbg !5 {
   ret i16 undef, !dbg !9
 }
 
-define void @f2() !dbg !10 {
+define void @f2(i1 %arg) !dbg !10 {
 bb1:
   %_tmp7 = tail call i16 @f1(), !dbg !13
 ; CHECK: call i16 @f1(), !dbg !13
@@ -16,7 +16,7 @@ bb1:
   br label %vector.body
 
 vector.body:
-  br i1 undef, label %middle.block, label %vector.body
+  br i1 %arg, label %middle.block, label %vector.body
 
 middle.block:
   ret void, !dbg !15
diff --git a/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
index 3b6f5ccf3d3ea..8ce912ff55d6e 100644
--- a/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
+++ b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes='function(scalarizer)' -S -o - | FileCheck %s
 
-define i16 @f1() {
+define i16 @f1(i1 %arg) {
 ; CHECK-LABEL: @f1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
@@ -9,7 +9,7 @@ define i16 @f1() {
 ; CHECK-NEXT:    [[INSERT:%.*]] = insertelement <4 x i16> [[INSERT]], i16 ptrtoint (ptr @f1 to i16), i32 0
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY:%.*]], label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[PHI_I0:%.*]] = phi i16 [ 1, [[ENTRY:%.*]] ], [ poison, [[FOR_COND]] ]
 ; CHECK-NEXT:    ret i16 [[PHI_I0]]
@@ -22,7 +22,7 @@ for.body:
   br label %for.cond
 
 for.cond:
-  br i1 undef, label %for.body, label %for.end
+  br i1 %arg, label %for.body, label %for.end
 
 for.end:
   ; opt used to hang when scalarizing this code. When scattering %insert we
@@ -34,22 +34,22 @@ for.end:
   ret i16 %extract
 }
 
-define void @f2() {
+define void @f2(i1 %arg) {
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_END8:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF_THEN:%.*]], label [[IF_END8:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    br label [[IF_END8]]
 ; CHECK:       for.body2:
-; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_INC:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_END:%.*]], label [[FOR_INC:%.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[E_SROA_3_2:%.*]] = phi <2 x i64> [ splat (i64 1), [[FOR_END]] ], [ [[E_SROA_3_2]], [[FOR_BODY2:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 6, [[FOR_END]] ], [ [[TMP0]], [[FOR_BODY2]] ]
-; CHECK-NEXT:    br i1 undef, label [[FOR_BODY2]], label [[FOR_COND1_FOR_END7_CRIT_EDGE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[FOR_BODY2]], label [[FOR_COND1_FOR_END7_CRIT_EDGE:%.*]]
 ; CHECK:       for.cond1.for.end7_crit_edge:
 ; CHECK-NEXT:    br label [[IF_END8]]
 ; CHECK:       if.end8:
@@ -61,13 +61,13 @@ entry:
   br label %for.body
 
 for.body:                                         ; preds = %if.end8, %entry
-  br i1 undef, label %if.then, label %if.end8
+  br i1 %arg, label %if.then, label %if.end8
 
 if.then:                                          ; preds = %for.body
   br label %if.end8
 
 for.body2:                                        ; preds = %for.inc
-  br i1 undef, label %for.end, label %for.inc
+  br i1 %arg, label %for.end, label %for.inc
 
 for.end:                                          ; preds = %for.body2
   br label %for.inc
@@ -75,7 +75,7 @@ for.end:                                          ; preds = %for.body2
 for.inc:                                          ; preds = %for.end, %for.body2
   %e.sroa.3.2 = phi <2 x i64> [ <i64 1, i64 1>, %for.end ], [ %e.sroa.3.2, %for.body2 ]
   %0 = phi i32 [ 6, %for.end ], [ %0, %for.body2 ]
-  br i1 undef, label %for.body2, label %for.cond1.for.end7_crit_edge
+  br i1 %arg, label %for.body2, label %for.cond1.for.end7_crit_edge
 
 for.cond1.for.end7_crit_edge:                     ; preds = %for.inc
   br label %if.end8
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
index 05303b987171d..56133b8702f66 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 
 	%struct.BLEND_MAP = type { i16, i16, i16, i32, ptr }
 	%struct.BLEND_MAP_ENTRY = type { float, i8, { [5 x float], [4 x i8] } }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
index bf324eff7b362..5349b1d897659 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 
 define void @init_caller_save() {
 entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
index d7c8fa0f7d728..1865122cd4a0a 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
@@ -1,5 +1,5 @@
 ; PR1333
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
 target triple = "i686-pc-linux-gnu"
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
index f770354e5d541..195f5a7846f0b 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 ; PR1333
 
 define void @pp_cxx_expression() {
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
index bd44e897e78f1..d2a4fbe91f26d 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes='loop(simple-loop-unswitch),instcombine' -verify-memoryssa -disable-output
+; RUN: opt < %s -passes='loop-mssa(simple-loop-unswitch),instcombine' -verify-memoryssa -disable-output
 
 @str3 = external constant [3 x i8]		; <ptr> [#uses=1]
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
index 2ce0398c7908f..e73c931b845bb 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 
 define i32 @main(i32 %argc, ptr %argv) {
 entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
index 0151d8d2677b1..9f568fc5fccd9 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 ; PR1559
 
 target triple = "i686-pc-linux-gnu"
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
index 1ad57ce936bb9..162d7fba19427 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -S -passes='loop(simple-loop-unswitch),instcombine' -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -S -passes='loop-mssa(simple-loop-unswitch),instcombine' -verify-memoryssa | FileCheck %s
 
 @.str9 = external constant [1 x i8]
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
index 0768b41119a6f..a1fc17fb4876f 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes='loop(simple-loop-unswitch),instcombine,gvn' -verify-memoryssa -disable-output
+; RUN: opt < %s -passes='loop-mssa(simple-loop-unswitch),instcombine,gvn' -verify-memoryssa -disable-output
 ; PR2372
 target triple = "i386-pc-linux-gnu"
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
index 37aabe273eed2..0be2952fbba12 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa
 ; PR8622
 @g_38 = external global i32, align 4
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
index 55c9d677ee529..ab95b523744be 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
@@ -1,12 +1,12 @@
-; RUN: opt -passes=simple-loop-unswitch -verify-memoryssa -disable-output < %s
+; RUN: opt -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output < %s
 ; PR10031
 
-define i32 @test(i32 %command) {
+define i32 @test(i32 %command, i1 %arg) {
 entry:
   br label %tailrecurse
 
 tailrecurse:                                      ; preds = %if.then14, %tailrecurse, %entry
-  br i1 undef, label %if.then, label %tailrecurse
+  br i1 %arg, label %if.then, label %tailrecurse
 
 if.then:                                          ; preds = %tailrecurse
   switch i32 %command, label %sw.bb [
@@ -15,7 +15,7 @@ if.then:                                          ; preds = %tailrecurse
   ]
 
 land.lhs.true:                                    ; preds = %if.then, %if.then
-  br i1 undef, label %sw.bb, label %if.then14
+  br i1 %arg, label %sw.bb, label %if.then14
 
 if.then14:                                        ; preds = %land.lhs.true
   switch i32 %command, label %tailrecurse [
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
index 86f6e7c112937..7ff5e09e28c51 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes='sroa,simple-loop-unswitch' -verify-memoryssa -disable-output
+; RUN: opt < %s -passes='sroa,loop-mssa(simple-loop-unswitch)' -verify-memoryssa -disable-output
 ; PR11016
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.2"
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
index ea4aa21b81bf0..d0fa73a095780 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -passes=simple-loop-unswitch -verify-loop-info -verify-dom-info -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -S -passes="loop-mssa(simple-loop-unswitch)" -verify-loop-info -verify-dom-info -verify-memoryssa | FileCheck %s
 ; PR12343: -passes=simple-loop-unswitch crash on indirect branch
 
 ; CHECK:       %0 = icmp eq i64 undef, 0
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index 8ad869015f44e..229bbb2f3929f 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-apple-macosx10.7.0"
 %class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 = type { ptr }
 %class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377 = type { i8 }
 
-define void @_Z23get_reconstruction_pathv() uwtable ssp personality ptr @__gxx_personality_v0 {
+define void @_Z23get_reconstruction_pathv(i1 %arg) uwtable ssp personality ptr @__gxx_personality_v0 {
 entry:
   %c = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, align 8
   br label %for.cond
@@ -30,7 +30,7 @@ invoke.cont4:                                     ; preds = %for.cond3
           to label %invoke.cont6 unwind label %lpad
 
 invoke.cont6:                                     ; preds = %invoke.cont4
-  br i1 undef, label %for.cond3, label %for.end
+  br i1 %arg, label %for.cond3, label %for.end
 
 lpad:                                             ; preds = %for.end, %invoke.cont4, %for.cond3, %invoke.cont, %for.cond
   %0 = landingpad { ptr, i32 }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
index eef32ad271d26..409d3b18379bb 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -disable-output
 ; PR12887
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
index ce767f14cf073..2d24ef4afab07 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -S | FileCheck %s
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -S | FileCheck %s
 
 ; In cases where two address spaces do not have the same size pointer, the
 ; input for the addrspacecast should not be used as a substitute for itself
@@ -6,12 +6,12 @@
 
 target datalayout = "e-m:e-p:16:16-p1:32:16-i32:16-i64:16-n8:16"
 
-define void @foo() {
+define void @foo(i1 %arg) {
 ; CHECK-LABEL: @foo
 entry:
   %arrayidx.i1 = getelementptr inbounds i16, ptr undef, i16 undef
   %arrayidx.i = addrspacecast ptr %arrayidx.i1 to ptr addrspace(1)
-  br i1 undef, label %for.body.i, label %bar.exit
+  br i1 %arg, label %for.body.i, label %bar.exit
 
 for.body.i:                                       ; preds = %for.body.i, %entry
 ; When we call makeLoopInvariant (i.e. trivial LICM) on this load, it
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
index 2069efd12d27a..2199d6a07981a 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/AMDGPU/uniform-unswitch.ll
@@ -35,7 +35,7 @@ define amdgpu_kernel void @uniform_unswitch(ptr nocapture %out, i32 %n, i32 %x)
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[I_07]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT_GLOBAL]], i64 [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(1) [[OUT_GLOBAL]], i64 [[TMP0]]
 ; CHECK-NEXT:    store i32 [[I_07]], ptr addrspace(1) [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll b/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
index e795ff0ccadd4..ddab83e4bc268 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -S 2>&1 | FileCheck %s
 
 ; This is to test trivial loop unswitch only happens when trivial condition
 ; itself is an LIV loop condition (not partial LIV which could occur in and/or).
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll b/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
index 3fd9ced64d765..9c45a53c3e365 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -passes=simple-loop-unswitch -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa < %s | FileCheck %s
 target triple = "x86_64-pc-win32"
 
 define void @f(i32 %doit, i1 %x, i1 %y) personality ptr @__CxxFrameHandler3 {
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll b/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
index 5f6a4c3e809db..a68a8d99bab36 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -S | FileCheck %s
+; RUN: opt < %s -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -S | FileCheck %s
 
 ; This test checks if unswitched condition preserve make.implicit metadata.
 define i32 @test(i1 %cond) {
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
index 37c49457dec70..62563fbd3d273 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -passes=simple-loop-unswitch -verify-memoryssa -disable-output
-; RUN: opt < %s -passes='simple-loop-unswitch<nontrivial>' -verify-memoryssa -disable-output
+; RUN: opt < %s -passes='loop-mssa(simple-loop-unswitch)' -verify-memoryssa -disable-output
+; RUN: opt < %s -passes='loop-mssa(simple-loop-unswitch<nontrivial>)' -verify-memoryssa -disable-output
 
 define void @test1(ptr %S2) {
 entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
index 9ca554023a8cf..c120eeb440d19 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
@@ -47,7 +47,7 @@ get_out2:
 ;
 ; This comes from PR38778
 ; CHECK-LABEL: @Test2
-define void @Test2(i32) {
+define void @Test2(i32, i1 %arg) {
 header:
   br label %loop
 loop:
@@ -58,12 +58,12 @@ loop:
   ]
 ; CHECK-NOT: {{^}}guarded1:
 guarded1:
-  br i1 undef, label %continue, label %leave
+  br i1 %arg, label %continue, label %leave
 guarded2:
   br label %continue
 check:
   %val = add i32 0, 1
-  br i1 undef, label %continue, label %leave
+  br i1 %arg, label %continue, label %leave
 continue:
   br label %loop
 leave:
@@ -75,7 +75,7 @@ leave:
 ; Yet another test from PR38778
 ;
 ; CHECK-LABEL: @Test3
-define void @Test3(i32) {
+define void @Test3(i32, i1 %arg) {
 header:
   br label %outer
 outer:
@@ -95,7 +95,7 @@ case2:
   br label %continue
 continue:
   %local_11_92 = phi i32 [ 0, %switchme ], [ 18, %case2 ], [ 0, %overflow ]
-  br i1 undef, label %outer, label %inner
+  br i1 %arg, label %outer, label %inner
 go_out:
   unreachable
 }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
index 2aa6b682ac406..5ed671b071283 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=simple-loop-unswitch -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -passes="loop-mssa(simple-loop-unswitch)" -verify-memoryssa -S < %s | FileCheck %s
 
 define void @f(i32 %n, ptr %ptr) {
 ; CHECK-LABEL: @f(
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
index 19a2bc3ad0449..45da85299051f 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
@@ -2,7 +2,7 @@
 
 ; PR38283
 ; PR38737
-define void @f1() {
+define void @f1(i1 %arg) {
 for.cond1thread-pre-split.lr.ph.lr.ph:
   %tobool4 = icmp eq i16 undef, 0
   br label %for.cond1thread-pre-split
@@ -18,7 +18,7 @@ if.end:                                           ; preds = %for.body2
   br i1 %tobool4, label %if.end6, label %for.cond1thread-pre-split
 
 if.end6:                                          ; preds = %if.end
-  br i1 undef, label %for.body2, label %for.end
+  br i1 %arg, label %for.body2, label %for.end
 
 for.end:                                          ; preds = %if.end6, %for.body2
   ret void
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
index 07668616ff86d..533b1f691f5ad 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --version 5
 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards  -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s
@@ -131,30 +132,55 @@ exit:
   ret void
 }
 
-define void @test_nested_loop(i1 %cond, i32 %N) {
-; CHECK-LABEL: @test_nested_loop(
+define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
+; CHECK-LABEL: define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT:%.*]], label [[OUTER_LOOP_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 %cond, label %entry.split, label %outer_loop.split
 ; CHECK:       entry.split:
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK-NEXT:    br i1 %arg, label %entry.split.split.us, label %entry.split.split
+; CHECK:       entry.split.split.us:
+; CHECK-NEXT:    br label %outer_loop.us
+; CHECK:       outer_loop.us:
+; CHECK-NEXT:    br label %outer_loop.split.us.us
+; CHECK:       outer_backedge.us:
+; CHECK-NEXT:    br label %outer_loop.us
+; CHECK:       outer_loop.split.us.us:
+; CHECK-NEXT:    br label %loop.us.us
+; CHECK:       loop.us.us:
+; CHECK-NEXT:    %iv.us.us = phi i32 [ 0, %outer_loop.split.us.us ], [ %iv.next.us.us, %guarded.us.us ]
+; CHECK-NEXT:    br label %guarded.us.us
+; CHECK:       guarded.us.us:
+; CHECK-NEXT:    %iv.next.us.us = add i32 %iv.us.us, 1
+; CHECK-NEXT:    %loop.cond.us.us = icmp slt i32 %iv.next.us.us, %N
+; CHECK-NEXT:    br i1 %loop.cond.us.us, label %loop.us.us, label %outer_backedge.split.us.us
+; CHECK:       outer_backedge.split.us.us:
+; CHECK-NEXT:    br label %outer_backedge.us
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %outer_loop
 ; CHECK:       outer_loop:
-; CHECK-NEXT:    br label [[OUTER_LOOP_SPLIT_US:%.*]]
+; CHECK-NEXT:    br label %outer_loop.split.us
 ; CHECK:       outer_loop.split.us:
-; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK-NEXT:    br label %loop.us
 ; CHECK:       loop.us:
-; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[OUTER_LOOP_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
-; CHECK-NEXT:    br label [[GUARDED_US]]
+; CHECK-NEXT:    %iv.us = phi i32 [ 0, %outer_loop.split.us ], [ %iv.next.us, %guarded.us ]
+; CHECK-NEXT:    br label %guarded.us
 ; CHECK:       guarded.us:
-; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[OUTER_BACKEDGE_SPLIT_US:%.*]]
+; CHECK-NEXT:    %iv.next.us = add i32 %iv.us, 1
+; CHECK-NEXT:    %loop.cond.us = icmp slt i32 %iv.next.us, %N
+; CHECK-NEXT:    br i1 %loop.cond.us, label %loop.us, label %outer_backedge.split.us
 ; CHECK:       outer_backedge.split.us:
-; CHECK-NEXT:    br label [[OUTER_BACKEDGE:%.*]]
+; CHECK-NEXT:    br label %outer_backedge
+; CHECK:       outer_loop.split:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    br label %deopt
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    br i1 false, label [[OUTER_LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 
 entry:
@@ -171,7 +197,7 @@ loop:
   br i1 %loop.cond, label %loop, label %outer_backedge
 
 outer_backedge:
-  br i1 undef, label %outer_loop, label %exit
+  br i1 %arg, label %outer_loop, label %exit
 
 exit:
   ret void
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
index 10bd04f784985..03119112b0f61 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
@@ -1,8 +1,7 @@
 ; Specifically exercise the cost modeling for non-trivial loop unswitching.
 ;
 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -unswitch-threshold=5 -S < %s | FileCheck %s
-; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -unswitch-threshold=5 -S < %s | FileCheck %s
-; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -unswitch-threshold=5 -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -unswitch-threshold=5 -S -verify-memoryssa < %s | FileCheck %s
 
 declare void @a()
 declare void @b()
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
index e511dac7fdca7..8e97cb5cb42f8 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -freeze-loop-unswitch-cond -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -freeze-loop-unswitch-cond -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -freeze-loop-unswitch-cond -passes='simple-loop-unswitch<nontrivial>' -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -freeze-loop-unswitch-cond -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S -verify-memoryssa < %s | FileCheck %s
 
 declare i32 @a()
 declare i32 @b()
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
index 543ee092e810c..ae7042899ba87 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S -verify-memoryssa < %s | FileCheck %s
 
 declare ptr @pluto()
 declare void @llvm.experimental.guard(i1, ...)
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
index 887f1eb11bf07..9567b6b793239 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
@@ -1,6 +1,5 @@
 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
-; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S -verify-memoryssa < %s | FileCheck %s
 
 declare i32 @a()
 declare i32 @b()
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/pr117537.ll b/llvm/test/Transforms/SimpleLoopUnswitch/pr117537.ll
new file mode 100644
index 0000000000000..fd61cfab164d3
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/pr117537.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='print<scalar-evolution>,simple-loop-unswitch<nontrivial>,print<scalar-evolution>' -verify-scev < %s 2>/dev/null | FileCheck %s
+
+; Make sure we don't assert due to insufficient SCEV invalidation.
+
+define void @test(ptr %p) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CHECK:%.*]] = icmp eq ptr [[P]], null
+; CHECK-NEXT:    br i1 [[CHECK]], label %[[ENTRY_SPLIT_US:.*]], label %[[ENTRY_SPLIT:.*]]
+; CHECK:       [[ENTRY_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[BB0_US:.*]]
+; CHECK:       [[BB0_US]]:
+; CHECK-NEXT:    br label %[[LOOP0_US:.*]]
+; CHECK:       [[LOOP0_US]]:
+; CHECK-NEXT:    [[V_US:%.*]] = load atomic i32, ptr [[P]] unordered, align 8
+; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[V_US]], 3
+; CHECK-NEXT:    br i1 true, label %[[PREHEADER_SPLIT_US:.*]], label %[[BB0_US]]
+; CHECK:       [[PREHEADER_SPLIT_US]]:
+; CHECK-NEXT:    [[ADD_LCSSA_US:%.*]] = phi i32 [ [[ADD_US]], %[[LOOP0_US]] ]
+; CHECK-NEXT:    br label %[[PREHEADER:.*]]
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    br label %[[BB0:.*]]
+; CHECK:       [[BB0]]:
+; CHECK-NEXT:    br label %[[LATCH:.*]]
+; CHECK:       [[LATCH]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT0:.*]], label %[[LOOP0:.*]]
+; CHECK:       [[EXIT0]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[LOOP0]]:
+; CHECK-NEXT:    [[V:%.*]] = load atomic i32, ptr [[P]] unordered, align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[V]], 3
+; CHECK-NEXT:    br i1 true, label %[[PREHEADER_SPLIT:.*]], label %[[BB0]]
+; CHECK:       [[PREHEADER_SPLIT]]:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP0]] ]
+; CHECK-NEXT:    br label %[[PREHEADER]]
+; CHECK:       [[PREHEADER]]:
+; CHECK-NEXT:    [[DOTUS_PHI:%.*]] = phi i32 [ [[ADD_LCSSA]], %[[PREHEADER_SPLIT]] ], [ [[ADD_LCSSA_US]], %[[PREHEADER_SPLIT_US]] ]
+; CHECK-NEXT:    br label %[[LOOP1:.*]]
+; CHECK:       [[LOOP1]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ [[DOTUS_PHI]], %[[PREHEADER]] ], [ [[IV1_NEXT:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NEXT:    [[IV1_NEXT]] = add i32 [[IV1]], -33
+; CHECK-NEXT:    br label %[[LOOP2:.*]]
+; CHECK:       [[BACKEDGE]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT1:.*]], label %[[LOOP1]]
+; CHECK:       [[LOOP2]]:
+; CHECK-NEXT:    [[IV0:%.*]] = phi i32 [ [[IV1]], %[[LOOP1]] ], [ [[IV0_NEXT:%.*]], %[[LOOP2]] ]
+; CHECK-NEXT:    [[IV0_NEXT]] = add nsw i32 [[IV0]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[IV0_NEXT]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[BACKEDGE]], label %[[LOOP2]]
+; CHECK:       [[EXIT1]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %check = icmp eq ptr %p, null
+  br label %bb0
+
+bb0:                                              ; preds = %loop0, %entry
+  br i1 %check, label %loop0, label %latch
+
+latch:                                            ; preds = %bb0
+  br i1 %check, label %exit0, label %loop0
+
+exit0:                                            ; preds = %latch
+  ret void
+
+loop0:                                            ; preds = %latch, %bb0
+  %v = load atomic i32, ptr %p unordered, align 8
+  %add = add i32 %v, 3
+  br i1 true, label %preheader, label %bb0
+
+preheader:                                        ; preds = %loop0
+  br label %loop1
+
+loop1:                                            ; preds = %backedge, %preheader
+  %iv1 = phi i32 [ %add, %preheader ], [ %iv1.next, %backedge ]
+  %iv1.next = add i32 %iv1, -33
+  br label %loop2
+
+backedge:                                         ; preds = %loop2
+  br i1 true, label %exit1, label %loop1
+
+loop2:                                            ; preds = %loop2, %loop1
+  %iv0 = phi i32 [ %iv1, %loop1 ], [ %iv0.next, %loop2 ]
+  %iv0.next = add nsw i32 %iv0, 1
+  %cmp = icmp sgt i32 %iv0.next, 0
+  br i1 %cmp, label %backedge, label %loop2
+
+exit1:                                            ; preds = %backedge
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll b/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
index 099d6a5456e8e..7c9bb1b8520d6 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
@@ -29,7 +29,7 @@ if.end:
   br label %for.inc
 
 for.inc:
-  br i1 undef, label %for.body, label %for.end
+  br i1 false, label %for.body, label %for.end
 
 for.end:
   ret void
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
index d4544ba6f4a29..51cce1c3e571b 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=simple-loop-unswitch -verify-loop-info -verify-dom-info -verify-memoryssa -disable-output < %s
+; RUN: opt -passes="loop-mssa(simple-loop-unswitch)" -verify-loop-info -verify-dom-info -verify-memoryssa -disable-output < %s
 
 ; Loop unswitch should be able to unswitch these loops and
 ; preserve LCSSA and LoopSimplify forms.
@@ -74,13 +74,13 @@ bb10:                                             ; preds = %bb8, %bb
 ; This is a simplified form of ineqn from above. It triggers some
 ; different cases in the loop-unswitch code.
 
-define void @simplified_ineqn() nounwind readonly {
+define void @simplified_ineqn(i1 %arg) nounwind readonly {
 entry:
   br label %bb8.outer
 
 bb8.outer:                                        ; preds = %bb6, %bb2, %entry
   %x = phi i32 [ 0, %entry ], [ 0, %bb6 ], [ 1, %bb2 ] ; <i32> [#uses=1]
-  br i1 undef, label %return, label %bb2
+  br i1 %arg, label %return, label %bb2
 
 bb2:                                              ; preds = %bb
   switch i32 %x, label %bb6 [
@@ -88,7 +88,7 @@ bb2:                                              ; preds = %bb
   ]
 
 bb6:                                              ; preds = %bb2
-  br i1 undef, label %bb8.outer, label %bb2
+  br i1 %arg, label %bb8.outer, label %bb2
 
 return:                                             ; preds = %bb8, %bb
   ret void
@@ -97,17 +97,17 @@ return:                                             ; preds = %bb8, %bb
 ; This function requires special handling to preserve LCSSA form.
 ; PR4934
 
-define void @pnp_check_irq() nounwind noredzone {
+define void @pnp_check_irq(i1 %arg) nounwind noredzone {
 entry:
   %conv56 = trunc i64 undef to i32                ; <i32> [#uses=1]
   br label %while.cond.i
 
 while.cond.i:                                     ; preds = %while.cond.i.backedge, %entry
   %call.i25 = call ptr @pci_get_device() nounwind noredzone ; <ptr> [#uses=2]
-  br i1 undef, label %if.then65, label %while.body.i
+  br i1 %arg, label %if.then65, label %while.body.i
 
 while.body.i:                                     ; preds = %while.cond.i
-  br i1 undef, label %if.then31.i.i, label %while.cond.i.backedge
+  br i1 %arg, label %if.then31.i.i, label %while.cond.i.backedge
 
 while.cond.i.backedge:                            ; preds = %if.then31.i.i, %while.body.i
   br label %while.cond.i
diff --git a/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll b/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
index dbd8f7ab862d0..0656811b39a9e 100644
--- a/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
+++ b/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
@@ -1,36 +1,36 @@
 ; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -disable-output
 
-define void @symhash_add() {
+define void @symhash_add(i1 %arg) {
 entry:
-	br i1 undef, label %then.0, label %UnifiedReturnBlock
+	br i1 %arg, label %then.0, label %UnifiedReturnBlock
 then.0:		; preds = %entry
-	br i1 undef, label %loopentry.2, label %loopentry.1.preheader
+	br i1 %arg, label %loopentry.2, label %loopentry.1.preheader
 loopentry.1.preheader:		; preds = %then.0
 	br label %loopentry.1.outer
 loopentry.1.outer:		; preds = %loopexit.1, %loopentry.1.preheader
 	br label %loopentry.1
 loopentry.1:		; preds = %endif.1, %then.4, %then.3, %then.1, %loopentry.1.outer
-	br i1 undef, label %loopexit.1, label %no_exit.1
+	br i1 %arg, label %loopexit.1, label %no_exit.1
 no_exit.1:		; preds = %loopentry.1
-	br i1 undef, label %then.1, label %else.0
+	br i1 %arg, label %then.1, label %else.0
 then.1:		; preds = %no_exit.1
 	br label %loopentry.1
 else.0:		; preds = %no_exit.1
-	br i1 undef, label %then.2, label %else.1
+	br i1 %arg, label %then.2, label %else.1
 then.2:		; preds = %else.0
-	br i1 undef, label %then.3, label %endif.1
+	br i1 %arg, label %then.3, label %endif.1
 then.3:		; preds = %then.2
 	br label %loopentry.1
 else.1:		; preds = %else.0
-	br i1 undef, label %endif.1, label %then.4
+	br i1 %arg, label %endif.1, label %then.4
 then.4:		; preds = %else.1
 	br label %loopentry.1
 endif.1:		; preds = %else.1, %then.2
 	br label %loopentry.1
 loopexit.1:		; preds = %loopentry.1
-	br i1 undef, label %loopentry.1.outer, label %loopentry.2
+	br i1 %arg, label %loopentry.1.outer, label %loopentry.2
 loopentry.2:		; preds = %no_exit.2, %loopexit.1, %then.0
-	br i1 undef, label %loopexit.2, label %no_exit.2
+	br i1 %arg, label %loopexit.2, label %no_exit.2
 no_exit.2:		; preds = %loopentry.2
 	br label %loopentry.2
 loopexit.2:		; preds = %loopentry.2
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll b/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
index 0ec88ed071fc2..20c03e05ba0b9 100644
--- a/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
+++ b/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -disable-output
 ; END.
 
-define void @main(i32 %c) {
+define void @main(i32 %c, i1 %arg) {
 entry:
 	%tmp.9 = icmp eq i32 %c, 2		; <i1> [#uses=1]
 	br i1 %tmp.9, label %endif.0, label %then.0
@@ -186,7 +186,7 @@ no_exit.0.i31.preheader:		; preds = %loopentry.1.i30
 no_exit.0.i31:		; preds = %loopexit.3.i, %no_exit.0.i31.preheader
 	br i1 false, label %then.1.i, label %else.0.i
 then.1.i:		; preds = %no_exit.0.i31
-	br i1 undef, label %then.0.i29, label %loopentry.0.i31
+	br i1 %arg, label %then.0.i29, label %loopentry.0.i31
 then.0.i29:		; preds = %then.1.i
 	unreachable
 loopentry.0.i31:		; preds = %then.1.i
@@ -194,13 +194,13 @@ loopentry.0.i31:		; preds = %then.1.i
 no_exit.0.i38.preheader:		; preds = %loopentry.0.i31
 	br label %no_exit.0.i38
 no_exit.0.i38:		; preds = %no_exit.0.i38, %no_exit.0.i38.preheader
-	br i1 undef, label %no_exit.0.i38, label %loopentry.1.i.preheader.loopexit
+	br i1 %arg, label %no_exit.0.i38, label %loopentry.1.i.preheader.loopexit
 loopentry.1.i.preheader.loopexit:		; preds = %no_exit.0.i38
 	br label %loopentry.1.i.preheader
 loopentry.1.i.preheader:		; preds = %loopentry.1.i.preheader.loopexit, %loopentry.0.i31
 	br label %loopentry.1.i
 loopentry.1.i:		; preds = %endif.2.i, %loopentry.1.i.preheader
-	br i1 undef, label %loopentry.2.i39.preheader, label %loopexit.1.i79.loopexit2
+	br i1 %arg, label %loopentry.2.i39.preheader, label %loopexit.1.i79.loopexit2
 loopentry.2.i39.preheader:		; preds = %loopentry.1.i
 	br label %loopentry.2.i39
 loopentry.2.i39:		; preds = %loopexit.5.i77, %loopentry.2.i39.preheader
@@ -212,45 +212,45 @@ loopentry.3.i40:		; preds = %loopexit.3.i51, %loopentry.3.i40.preheader
 no_exit.3.preheader.i42:		; preds = %loopentry.3.i40
 	br label %no_exit.3.i49
 no_exit.3.i49:		; preds = %no_exit.3.i49, %no_exit.3.preheader.i42
-	br i1 undef, label %no_exit.3.i49, label %loopexit.3.i51.loopexit
+	br i1 %arg, label %no_exit.3.i49, label %loopexit.3.i51.loopexit
 loopexit.3.i51.loopexit:		; preds = %no_exit.3.i49
 	br label %loopexit.3.i51
 loopexit.3.i51:		; preds = %loopexit.3.i51.loopexit, %loopentry.3.i40
-	br i1 undef, label %loopentry.3.i40, label %loopentry.4.i52
+	br i1 %arg, label %loopentry.3.i40, label %loopentry.4.i52
 loopentry.4.i52:		; preds = %loopexit.3.i51
 	br i1 false, label %no_exit.4.i54.preheader, label %hamming.exit.i71
 no_exit.4.i54.preheader:		; preds = %loopentry.4.i52
 	br label %no_exit.4.i54
 no_exit.4.i54:		; preds = %no_exit.4.backedge.i, %no_exit.4.i54.preheader
-	br i1 undef, label %then.1.i55, label %endif.1.i56
+	br i1 %arg, label %then.1.i55, label %endif.1.i56
 then.1.i55:		; preds = %no_exit.4.i54
-	br i1 undef, label %no_exit.4.backedge.i, label %loopexit.4.i57
+	br i1 %arg, label %no_exit.4.backedge.i, label %loopexit.4.i57
 no_exit.4.backedge.i:		; preds = %endif.1.i56, %then.1.i55
 	br label %no_exit.4.i54
 endif.1.i56:		; preds = %no_exit.4.i54
-	br i1 undef, label %no_exit.4.backedge.i, label %loopexit.4.i57
+	br i1 %arg, label %no_exit.4.backedge.i, label %loopexit.4.i57
 loopexit.4.i57:		; preds = %endif.1.i56, %then.1.i55
 	br i1 false, label %no_exit.i.i69.preheader, label %hamming.exit.i71
 no_exit.i.i69.preheader:		; preds = %loopexit.4.i57
 	br label %no_exit.i.i69
 no_exit.i.i69:		; preds = %no_exit.i.i69, %no_exit.i.i69.preheader
-	br i1 undef, label %no_exit.i.i69, label %hamming.exit.i71.loopexit
+	br i1 %arg, label %no_exit.i.i69, label %hamming.exit.i71.loopexit
 hamming.exit.i71.loopexit:		; preds = %no_exit.i.i69
 	br label %hamming.exit.i71
 hamming.exit.i71:		; preds = %hamming.exit.i71.loopexit, %loopexit.4.i57, %loopentry.4.i52, %loopentry.2.i39
-	br i1 undef, label %endif.2.i, label %loopentry.5.i72
+	br i1 %arg, label %endif.2.i, label %loopentry.5.i72
 loopentry.5.i72:		; preds = %hamming.exit.i71
 	br i1 false, label %shortcirc_next.i74.preheader, label %loopexit.5.i77
 shortcirc_next.i74.preheader:		; preds = %loopentry.5.i72
 	br label %shortcirc_next.i74
 shortcirc_next.i74:		; preds = %no_exit.5.i76, %shortcirc_next.i74.preheader
-	br i1 undef, label %no_exit.5.i76, label %loopexit.5.i77.loopexit
+	br i1 %arg, label %no_exit.5.i76, label %loopexit.5.i77.loopexit
 no_exit.5.i76:		; preds = %shortcirc_next.i74
-	br i1 undef, label %shortcirc_next.i74, label %loopexit.5.i77.loopexit
+	br i1 %arg, label %shortcirc_next.i74, label %loopexit.5.i77.loopexit
 loopexit.5.i77.loopexit:		; preds = %no_exit.5.i76, %shortcirc_next.i74
 	br label %loopexit.5.i77
 loopexit.5.i77:		; preds = %loopexit.5.i77.loopexit, %loopentry.5.i72
-	br i1 undef, label %loopentry.2.i39, label %loopexit.1.i79.loopexit
+	br i1 %arg, label %loopentry.2.i39, label %loopexit.1.i79.loopexit
 endif.2.i:		; preds = %hamming.exit.i71
 	br label %loopentry.1.i
 loopexit.1.i79.loopexit:		; preds = %loopexit.5.i77
@@ -258,7 +258,7 @@ loopexit.1.i79.loopexit:		; preds = %loopexit.5.i77
 loopexit.1.i79.loopexit2:		; preds = %loopentry.1.i
 	br label %loopexit.1.i79
 loopexit.1.i79:		; preds = %loopexit.1.i79.loopexit2, %loopexit.1.i79.loopexit
-	br i1 undef, label %then.3.i, label %loopentry.6.i80
+	br i1 %arg, label %then.3.i, label %loopentry.6.i80
 then.3.i:		; preds = %loopexit.1.i79
 	br i1 false, label %no_exit.6.i82.preheader, label %run.exit
 loopentry.6.i80:		; preds = %loopexit.1.i79
@@ -266,7 +266,7 @@ loopentry.6.i80:		; preds = %loopexit.1.i79
 no_exit.6.i82.preheader:		; preds = %loopentry.6.i80, %then.3.i
 	br label %no_exit.6.i82
 no_exit.6.i82:		; preds = %no_exit.6.i82, %no_exit.6.i82.preheader
-	br i1 undef, label %no_exit.6.i82, label %run.exit.loopexit
+	br i1 %arg, label %no_exit.6.i82, label %run.exit.loopexit
 run.exit.loopexit:		; preds = %no_exit.6.i82
 	br label %run.exit
 run.exit:		; preds = %run.exit.loopexit, %loopentry.6.i80, %then.3.i
diff --git a/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
index b3b6abe314e6f..349a18148460f 100644
--- a/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
@@ -13,7 +13,7 @@ define i32 @foo(i32 %x) #0 section ".tcm_text" {
 ; ENABLE-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 6
 ; ENABLE-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; ENABLE:       switch.lookup:
-; ENABLE-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.foo, i32 0, i32 [[X]]
+; ENABLE-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds nuw [6 x i32], ptr @switch.table.foo, i32 0, i32 [[X]]
 ; ENABLE-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; ENABLE-NEXT:    br label [[RETURN]]
 ; ENABLE:       return:
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-on-undef.ll b/llvm/test/Transforms/SimplifyCFG/branch-on-undef.ll
index bc42ae60730a7..582a38f4af12f 100644
--- a/llvm/test/Transforms/SimplifyCFG/branch-on-undef.ll
+++ b/llvm/test/Transforms/SimplifyCFG/branch-on-undef.ll
@@ -3,10 +3,10 @@
 
 declare void @foo(i32)
 
-define void @br_undef_simple() {
+define void @br_undef_simple(i1 %arg) {
 ; CHECK-LABEL: @br_undef_simple(
 ; CHECK-NEXT:    call void @foo(i32 0)
-; CHECK-NEXT:    br i1 undef, label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    br i1 %arg, label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       if:
@@ -17,7 +17,7 @@ define void @br_undef_simple() {
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
   call void @foo(i32 0)
-  br i1 undef, label %if, label %else
+  br i1 %arg, label %if, label %else
 
 if:
   call void @foo(i32 1)
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
index bd63e6658c9a6..9fb5164d44d45 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
@@ -939,7 +939,7 @@ if.end.loopexit:
 }
 
 @f.b = external global i8, align 1
-define void @pr48450_3() {
+define void @pr48450_3(i1 %arg) {
 ; CHECK-LABEL: @pr48450_3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
@@ -979,7 +979,7 @@ for.cond.cleanup:
   br label %cleanup
 
 for.body4:
-  br i1 undef, label %if.then6, label %if.end7
+  br i1 %arg, label %if.then6, label %if.end7
 
 if.then6:
   br label %cleanup
diff --git a/llvm/test/Transforms/SimplifyCFG/pr34131.ll b/llvm/test/Transforms/SimplifyCFG/pr34131.ll
index 89415cc88aeb1..ed37a969aa47f 100644
--- a/llvm/test/Transforms/SimplifyCFG/pr34131.ll
+++ b/llvm/test/Transforms/SimplifyCFG/pr34131.ll
@@ -4,7 +4,7 @@
 ; Earlier version using auto-generated checks from utils/update_test_checks.py
 ; had bot problems though...
 
-define void @patatino() {
+define void @patatino(i1 %arg) {
 
 ; CHECK-LABEL: @patatino
 
@@ -12,16 +12,16 @@ define void @patatino() {
 bb1:                                              ; preds = %bb36, %0
   br label %bb2
 bb2:                                              ; preds = %bb3, %bb1
-  br i1 undef, label %bb4, label %bb3
+  br i1 %arg, label %bb4, label %bb3
 bb3:                                              ; preds = %bb4, %bb2
-  br i1 undef, label %bb2, label %bb5
+  br i1 %arg, label %bb2, label %bb5
 bb4:                                              ; preds = %bb2
   switch i32 undef, label %bb3 [
   ]
 bb5:                                              ; preds = %bb3
   br label %bb6
 bb6:                                              ; preds = %bb5
-  br i1 undef, label %bb7, label %bb9
+  br i1 %arg, label %bb7, label %bb9
 bb7:                                              ; preds = %bb6
   %tmp = or i64 undef, 1
   %tmp8 = icmp ult i64 %tmp, 0
@@ -58,17 +58,17 @@ bb27:                                             ; preds = %bb24
   %tmp29 = icmp ult i64 %tmp28, 0
   br i1 %tmp29, label %bb30, label %bb9
 bb30:                                             ; preds = %bb27
-  br i1 undef, label %bb31, label %bb9
+  br i1 %arg, label %bb31, label %bb9
 bb31:                                             ; preds = %bb30
-  br i1 undef, label %bb32, label %bb9
+  br i1 %arg, label %bb32, label %bb9
 bb32:                                             ; preds = %bb31
-  br i1 undef, label %bb33, label %bb9
+  br i1 %arg, label %bb33, label %bb9
 bb33:                                             ; preds = %bb32
-  br i1 undef, label %bb34, label %bb9
+  br i1 %arg, label %bb34, label %bb9
 bb34:                                             ; preds = %bb33
-  br i1 undef, label %bb35, label %bb9
+  br i1 %arg, label %bb35, label %bb9
 bb35:                                             ; preds = %bb34
-  br i1 undef, label %bb36, label %bb9
+  br i1 %arg, label %bb36, label %bb9
 bb36:                                             ; preds = %bb35
-  br i1 undef, label %bb1, label %bb10
+  br i1 %arg, label %bb1, label %bb10
 }
diff --git a/llvm/test/Transforms/Sink/dead-user.ll b/llvm/test/Transforms/Sink/dead-user.ll
index 91e61b43ca391..e63aa027a2db8 100644
--- a/llvm/test/Transforms/Sink/dead-user.ll
+++ b/llvm/test/Transforms/Sink/dead-user.ll
@@ -2,27 +2,26 @@
 ; Compiler should not be broken with a dead user.
 ; RUN: opt -passes=sink -S < %s | FileCheck %s
 
-define void @test(i16 %p1) {
-; CHECK-LABEL: define void @test(
-; CHECK-SAME: i16 [[P1:%.*]]) {
+define void @test(i16 %p1, i1 %arg) {
+; CHECK-LABEL: define void @test(i16 %p1, i1 %arg) {
 ; CHECK-NEXT:  bb.0:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[P1]] to i32
-; CHECK-NEXT:    br i1 undef, label [[BB_1:%.*]], label [[BB_3:%.*]]
+; CHECK-NEXT:    %conv = sext i16 %p1 to i32
+; CHECK-NEXT:    br i1 %arg, label %bb.1, label %bb.3
 ; CHECK:       bb.1:
-; CHECK-NEXT:    br label [[BB_2:%.*]]
+; CHECK-NEXT:    br label %bb.2
 ; CHECK:       bb.2:
-; CHECK-NEXT:    [[AND_2:%.*]] = and i32 undef, [[CONV]]
-; CHECK-NEXT:    br label [[BB_2]]
+; CHECK-NEXT:    %and.2 = and i32 undef, %conv
+; CHECK-NEXT:    br label %bb.2
 ; CHECK:       bb.3:
-; CHECK-NEXT:    [[AND_3:%.*]] = and i32 undef, [[CONV]]
-; CHECK-NEXT:    br label [[BB_3]]
+; CHECK-NEXT:    %and.3 = and i32 undef, %conv
+; CHECK-NEXT:    br label %bb.3
 ; CHECK:       dead:
-; CHECK-NEXT:    [[AND_DEAD:%.*]] = and i32 undef, [[CONV]]
-; CHECK-NEXT:    br label [[DEAD:%.*]]
+; CHECK-NEXT:    %and.dead = and i32 undef, %conv
+; CHECK-NEXT:    br label %dead
 ;
 bb.0:
   %conv = sext i16 %p1 to i32
-  br i1 undef, label %bb.1, label %bb.3
+  br i1 %arg, label %bb.1, label %bb.3
 
 bb.1:                                             ; preds = %bb.0
   br label %bb.2
diff --git a/llvm/test/Transforms/SpeculativeExecution/PR46267.ll b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll
index 8f82a16639744..5ee9fda470d1e 100644
--- a/llvm/test/Transforms/SpeculativeExecution/PR46267.ll
+++ b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll
@@ -27,11 +27,11 @@ end:                                 ; preds = %notnull, %entry
   ret ptr %i6
 }
 
-define void @f(i32 %i) {
+define void @f(i32 %i, i1 %arg) {
 entry:
 ; CHECK-LABEL: @f(
 ; CHECK:  %a2 = add i32 %i, 0
-  br i1 undef, label %land.rhs, label %land.end
+  br i1 %arg, label %land.rhs, label %land.end
 
 land.rhs:                                         ; preds = %entry
 ; CHECK: land.rhs:
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
index 364eb161e298a..eec67e67b540d 100644
--- a/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
+++ b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
@@ -1,24 +1,25 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -o - -structurizecfg < %s | FileCheck %s
 
-define void @test1() {
+define void @test1(i1 %arg) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    %arg.inv = xor i1 %arg, true
+; CHECK-NEXT:    br label %loop
 ; CHECK:       Flow:
-; CHECK-NEXT:    br label [[FLOW1:%.*]]
+; CHECK-NEXT:    br label %Flow1
 ; CHECK:       loop:
-; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[FLOW1]] ]
-; CHECK-NEXT:    [[CTR_NEXT:%.*]] = add i32 [[CTR]], 1
-; CHECK-NEXT:    br i1 undef, label [[LOOP_A:%.*]], label [[FLOW1]]
+; CHECK-NEXT:    %ctr = phi i32 [ 0, %entry ], [ %0, %Flow1 ]
+; CHECK-NEXT:    %ctr.next = add i32 %ctr, 1
+; CHECK-NEXT:    br i1 %arg.inv, label %loop.a, label %Flow1
 ; CHECK:       loop.a:
-; CHECK-NEXT:    br i1 undef, label [[LOOP_B:%.*]], label [[FLOW:%.*]]
+; CHECK-NEXT:    br i1 %arg.inv, label %loop.b, label %Flow
 ; CHECK:       loop.b:
-; CHECK-NEXT:    br label [[FLOW]]
+; CHECK-NEXT:    br label %Flow
 ; CHECK:       Flow1:
-; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[CTR_NEXT]], [[FLOW]] ], [ undef, [[LOOP]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ false, [[FLOW]] ], [ true, [[LOOP]] ]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    %0 = phi i32 [ %ctr.next, %Flow ], [ undef, %loop ]
+; CHECK-NEXT:    %1 = phi i1 [ false, %Flow ], [ true, %loop ]
+; CHECK-NEXT:    br i1 %1, label %exit, label %loop
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -28,10 +29,10 @@ entry:
 loop:
   %ctr = phi i32 [ 0, %entry ], [ %ctr.next, %loop.a ], [ %ctr.next, %loop.b ]
   %ctr.next = add i32 %ctr, 1
-  br i1 undef, label %exit, label %loop.a
+  br i1 %arg, label %exit, label %loop.a
 
 loop.a:
-  br i1 undef, label %loop, label %loop.b
+  br i1 %arg, label %loop, label %loop.b
 
 loop.b:
   br label %loop
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
index bc28259e6939b..179e11136c883 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
@@ -64,11 +64,18 @@ define <16 x i32> @concat_sext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <8 x i32> @concat_sext_v4i1_v8i32(<4 x i1> %a0, <4 x i1> %a1) {
-; CHECK-LABEL: define <8 x i32> @concat_sext_v4i1_v8i32(
-; CHECK-SAME: <4 x i1> [[A0:%.*]], <4 x i1> [[A1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A0]], <4 x i1> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[R:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i32>
-; CHECK-NEXT:    ret <8 x i32> [[R]]
+; SSE-LABEL: define <8 x i32> @concat_sext_v4i1_v8i32(
+; SSE-SAME: <4 x i1> [[A0:%.*]], <4 x i1> [[A1:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[X0:%.*]] = sext <4 x i1> [[A0]] to <4 x i32>
+; SSE-NEXT:    [[X1:%.*]] = sext <4 x i1> [[A1]] to <4 x i32>
+; SSE-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[X0]], <4 x i32> [[X1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x i32> [[R]]
+;
+; AVX-LABEL: define <8 x i32> @concat_sext_v4i1_v8i32(
+; AVX-SAME: <4 x i1> [[A0:%.*]], <4 x i1> [[A1:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A0]], <4 x i1> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:    [[R:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i32>
+; AVX-NEXT:    ret <8 x i32> [[R]]
 ;
   %x0 = sext <4 x i1> %a0 to <4 x i32>
   %x1 = sext <4 x i1> %a1 to <4 x i32>
@@ -90,11 +97,18 @@ define <8 x i16> @concat_trunc_v4i32_v8i16(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(
-; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[R:%.*]] = inttoptr <8 x i32> [[TMP1]] to <8 x ptr>
-; CHECK-NEXT:    ret <8 x ptr> [[R]]
+; SSE-LABEL: define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(
+; SSE-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[R:%.*]] = inttoptr <8 x i32> [[TMP1]] to <8 x ptr>
+; SSE-NEXT:    ret <8 x ptr> [[R]]
+;
+; AVX-LABEL: define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(
+; AVX-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[X0:%.*]] = inttoptr <4 x i32> [[A0]] to <4 x ptr>
+; AVX-NEXT:    [[X1:%.*]] = inttoptr <4 x i32> [[A1]] to <4 x ptr>
+; AVX-NEXT:    [[R:%.*]] = shufflevector <4 x ptr> [[X0]], <4 x ptr> [[X1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:    ret <8 x ptr> [[R]]
 ;
   %x0 = inttoptr <4 x i32> %a0 to <4 x ptr>
   %x1 = inttoptr <4 x i32> %a1 to <4 x ptr>
@@ -138,9 +152,8 @@ define <8 x double> @concat_fpext_v4f32_v8f64(<4 x float> %a0, <4 x float> %a1)
 define <16 x float> @concat_fptrunc_v8f64_v16f32(<8 x double> %a0, <8 x double> %a1) {
 ; CHECK-LABEL: define <16 x float> @concat_fptrunc_v8f64_v16f32(
 ; CHECK-SAME: <8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[X0:%.*]] = fptrunc <8 x double> [[A0]] to <8 x float>
-; CHECK-NEXT:    [[X1:%.*]] = fptrunc <8 x double> [[A1]] to <8 x float>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x float> [[X0]], <8 x float> [[X1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0]], <8 x double> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[R:%.*]] = fptrunc <16 x double> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    ret <16 x float> [[R]]
 ;
   %x0 = fptrunc <8 x double> %a0 to <8 x float>
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-intrinsics.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-intrinsics.ll
index e012683b08f9b..2dc76cbfdda41 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-intrinsics.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-intrinsics.ll
@@ -34,19 +34,12 @@ entry:
 }
 
 define <8 x i32> @test3(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
-; SSE-LABEL: @test3(
-; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP0:%.*]], <4 x i32> [[TMP1:%.*]])
-; SSE-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP2:%.*]], <4 x i32> [[TMP3:%.*]])
-; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    ret <8 x i32> [[TMP6]]
-;
-; AVX-LABEL: @test3(
-; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0:%.*]], <4 x i32> [[TMP2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1:%.*]], <4 x i32> [[TMP3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP4]], <8 x i32> [[TMP5]])
-; AVX-NEXT:    ret <8 x i32> [[TMP6]]
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0:%.*]], <4 x i32> [[TMP2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1:%.*]], <4 x i32> [[TMP3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> [[TMP4]], <8 x i32> [[TMP5]])
+; CHECK-NEXT:    ret <8 x i32> [[TMP6]]
 ;
 entry:
   %4 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1)
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
index 57df36a3874a9..7dc70cd8b3d77 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
@@ -63,3 +63,14 @@ define  <4 x double> @blend_broadcasts_v4f64(ptr %p0, ptr %p1)  {
   %blend = shufflevector <4 x double> %bcst0, <4 x double> %bcst1, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
   ret <4 x double> %blend
 }
+
+define <2 x float> @PR86068(<2 x float> %a0, <2 x float> %a1) {
+; CHECK-LABEL: define <2 x float> @PR86068(
+; CHECK-SAME: <2 x float> [[A0:%.*]], <2 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <2 x float> [[A1]], <2 x float> [[A0]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    ret <2 x float> [[S2]]
+;
+  %s1 = shufflevector <2 x float> %a1, <2 x float> poison, <2 x i32> <i32 1, i32 poison>
+  %s2 = shufflevector <2 x float> %s1, <2 x float> %a0, <2 x i32> <i32 0, i32 3>
+  ret <2 x float> %s2
+}
diff --git a/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
index e1ac7588f1d8c..3c3c5dcbda520 100644
--- a/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
+++ b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
@@ -43,14 +43,14 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
-; ONE-NEXT: [002]     2         {Function} extern not_inlined 'InlineFunction' -> 'int'
+; ONE-NEXT: [002]     2         {Function} extern inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]     5             {Variable} 'Var_2' -> 'int'
 ; ONE-NEXT: [003]     2           {Parameter} 'Param' -> 'int'
 ; ONE-NEXT: [003]     3           {Variable} 'Var_1' -> 'int'
 ; ONE-NEXT: [002]    11         {Function} extern not_inlined 'test' -> 'int'
 ; ONE-NEXT: [003]    12           {Variable} 'A' -> 'int'
-; ONE-NEXT: [003]    13           {InlinedFunction} not_inlined 'InlineFunction' -> 'int'
+; ONE-NEXT: [003]    13           {InlinedFunction} inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [004]                   {Block}
 ; ONE-NEXT: [005]                     {Variable} 'Var_2' -> 'int'
 ; ONE-NEXT: [004]                   {Parameter} 'Param' -> 'int'
diff --git a/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/DW_AT_GNU_template_name.test b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/DW_AT_GNU_template_name.test
new file mode 100644
index 0000000000000..46cdae39e15e5
--- /dev/null
+++ b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/DW_AT_GNU_template_name.test
@@ -0,0 +1,60 @@
+; REQUIRES: x86-registered-target
+
+; For the given test case:
+
+;  1 // Template Template parameters.
+;  2 // Simple template class
+;  3 template <typename T> class Foo { T Member; };
+;  4
+;  5 // Template template class
+;  6 template <template <typename T> class TemplateType>
+;  7 class Bar {
+;  8   TemplateType<int> Int;
+;  9 };
+; 10
+; 11 template <template <template <typename> class> class TemplateTemplateType>
+; 12 class Baz {
+; 13   TemplateTemplateType<Foo> Foo;
+; 14 };
+; 15
+; 16 typedef Baz<Bar> Example;
+; 17
+; 18 Example TT;
+
+; The llvm-debuginfo-analyzer did not support the DW_AT_GNU_template_name
+; attribute. When using '--attribute=encoded' and the template argument is
+; another template, it would show:
+;   {Encoded} <>
+
+; The object file is generated using the following commands:
+;
+; clang++ -Xclang -disable-O0-optnone -Xclang -disable-llvm-passes
+;         -fno-discard-value-names -emit-llvm -S -g -O0
+;         DW_AT_GNU_template_name.cpp -o DW_AT_GNU_template_name.ll
+; llc --filetype=obj DW_AT_GNU_template_name.ll -o DW_AT_GNU_template_name.o
+;
+; llvm-debuginfo-analyzer --attribute=encoded --print=symbols
+;                         DW_AT_GNU_template_name.o
+
+; RUN: llc --filetype=obj \
+; RUN:     %p/Inputs/DW_AT_GNU_template_name.ll -o %t.DW_AT_GNU_template_name.o
+
+; RUN: llvm-debuginfo-analyzer --attribute=encoded \
+; RUN:                         --print=symbols \
+; RUN:                         %t.DW_AT_GNU_template_name.o 2>&1 | \
+; RUN: FileCheck --strict-whitespace %s
+
+; CHECK: Logical View:
+; CHECK:            {File} '{{.*}}dw_at_gnu_template_name.o'
+; CHECK-EMPTY:
+; CHECK:              {CompileUnit} 'dw_at_gnu_template_name.cpp'
+; CHECK:      3         {Class} 'Foo<int>'
+; CHECK:                  {Encoded} <int>
+; CHECK:      3           {Member} private 'Member' -> 'int'
+; CHECK:      7         {Class} 'Bar<Foo>'
+; CHECK:                  {Encoded} <Foo>
+; CHECK:      8           {Member} private 'Int' -> 'Foo<int>'
+; CHECK:     12         {Class} 'Baz<Bar>'
+; CHECK:                  {Encoded} <Bar>
+; CHECK:     13           {Member} private 'Foo' -> 'Bar<Foo>'
+; CHECK:     18         {Variable} extern 'TT' -> 'Example'
diff --git a/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/Inputs/DW_AT_GNU_template_name.ll b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/Inputs/DW_AT_GNU_template_name.ll
new file mode 100644
index 0000000000000..cfe1f182ae880
--- /dev/null
+++ b/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/Inputs/DW_AT_GNU_template_name.ll
@@ -0,0 +1,36 @@
+source_filename = "DW_AT_GNU_template_name.cpp"
+target triple = "x86_64-pc-linux-gnu"
+
+%class.Baz = type { %class.Bar }
+%class.Bar = type { %class.Foo }
+%class.Foo = type { i32 }
+
+@TT = dso_local global %class.Baz zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!22, !23}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "TT", scope: !2, file: !3, line: 18, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, emissionKind: FullDebug, globals: !4)
+!3 = !DIFile(filename: "DW_AT_GNU_template_name.cpp", directory: "")
+!4 = !{!0}
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "Example", file: !3, line: 16, baseType: !6)
+!6 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "Baz<Bar>", file: !3, line: 12, size: 32, flags: DIFlagTypePassByValue, elements: !7, templateParams: !20, identifier: "_ZTS3BazI3BarE")
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "Foo", scope: !6, file: !3, line: 13, baseType: !9, size: 32)
+!9 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "Bar<Foo>", file: !3, line: 7, size: 32, flags: DIFlagTypePassByValue, elements: !10, templateParams: !18, identifier: "_ZTS3BarI3FooE")
+!10 = !{!11}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "Int", scope: !9, file: !3, line: 8, baseType: !12, size: 32)
+!12 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "Foo<int>", file: !3, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !13, templateParams: !16, identifier: "_ZTS3FooIiE")
+!13 = !{!14}
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "Member", scope: !12, file: !3, line: 3, baseType: !15, size: 32)
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{!17}
+!17 = !DITemplateTypeParameter(name: "T", type: !15)
+!18 = !{!19}
+!19 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "TemplateType", value: !"Foo")
+!20 = !{!21}
+!21 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "TemplateTemplateType", value: !"Bar")
+!22 = !{i32 7, !"Dwarf Version", i32 5}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/X1-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/X1-neon-instructions.s
new file mode 100644
index 0000000000000..dc1bb486aeef7
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/X1-neon-instructions.s
@@ -0,0 +1,45 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-x1 -instruction-tables < %s | FileCheck %s
+
+# Check the Neoverse V1 model is used.
+
+add	v0.16b, v1.16b, v31.16b
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      2     0.25                        add	v0.16b, v1.16b, v31.16b
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - V1UnitB
+# CHECK-NEXT: [0.1] - V1UnitB
+# CHECK-NEXT: [1.0] - V1UnitD
+# CHECK-NEXT: [1.1] - V1UnitD
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.16b, v1.16b, v31.16b
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/X2-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/X2-sve-instructions.s
index 2912ea35f1ee8..6497860ecfbac 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/X2-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/X2-sve-instructions.s
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-x2 -instruction-tables < %s | FileCheck %s
 
-# Check the Neoverse N2 model is used.
+# Check the Neoverse V2 model is used.
 
 addhnb	z0.b, z1.h, z31.h
 
@@ -14,27 +14,34 @@ addhnb	z0.b, z1.h, z31.h
 # CHECK-NEXT: [6]: HasSideEffects (U)
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
-# CHECK-NEXT:  1      2     0.50                        addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      2     0.25                        addhnb	z0.b, z1.h, z31.h
 
 # CHECK:      Resources:
-# CHECK-NEXT: [0.0] - N2UnitB
-# CHECK-NEXT: [0.1] - N2UnitB
-# CHECK-NEXT: [1.0] - N2UnitD
-# CHECK-NEXT: [1.1] - N2UnitD
-# CHECK-NEXT: [2]   - N2UnitL2
-# CHECK-NEXT: [3.0] - N2UnitL01
-# CHECK-NEXT: [3.1] - N2UnitL01
-# CHECK-NEXT: [4]   - N2UnitM0
-# CHECK-NEXT: [5]   - N2UnitM1
-# CHECK-NEXT: [6.0] - N2UnitS
-# CHECK-NEXT: [6.1] - N2UnitS
-# CHECK-NEXT: [7]   - N2UnitV0
-# CHECK-NEXT: [8]   - N2UnitV1
+# CHECK-NEXT: [0.0] - V2UnitB
+# CHECK-NEXT: [0.1] - V2UnitB
+# CHECK-NEXT: [1.0] - V2UnitD
+# CHECK-NEXT: [1.1] - V2UnitD
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.b, z1.h, z31.h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/X3-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/X3-sve-instructions.s
new file mode 100644
index 0000000000000..042e621f9a03d
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/X3-sve-instructions.s
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-x3 -instruction-tables < %s | FileCheck %s
+
+# Check the Neoverse V2 model is used.
+
+addhnb	z0.b, z1.h, z31.h
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      2     0.25                        addhnb	z0.b, z1.h, z31.h
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - V2UnitB
+# CHECK-NEXT: [0.1] - V2UnitB
+# CHECK-NEXT: [1.0] - V2UnitD
+# CHECK-NEXT: [1.1] - V2UnitD
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.b, z1.h, z31.h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/X4-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/X4-sve-instructions.s
new file mode 100644
index 0000000000000..19fba62ea30c6
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/X4-sve-instructions.s
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-x4 -instruction-tables < %s | FileCheck %s
+
+# Check the Neoverse V2 model is used.
+
+addhnb	z0.b, z1.h, z31.h
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      2     0.25                        addhnb	z0.b, z1.h, z31.h
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - V2UnitB
+# CHECK-NEXT: [0.1] - V2UnitB
+# CHECK-NEXT: [1.0] - V2UnitD
+# CHECK-NEXT: [1.1] - V2UnitD
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.b, z1.h, z31.h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/512tvb-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/512tvb-sve-instructions.s
index 8c6855096a904..e6d4d67cca780 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/512tvb-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/512tvb-sve-instructions.s
@@ -21,22 +21,25 @@ abs	z0.b, p0/m, z0.b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.b, p0/m, z0.b
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.b, p0/m, z0.b
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
index 52839faeee9db..a5330b9d6d2d6 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
@@ -2545,1183 +2545,1186 @@ drps
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT: 11.00  11.00  33.00  33.00  96.33  162.33 162.33 312.75 186.75 145.25 145.25 167.50 44.50  51.50  9.50
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT: 11.00  11.00  33.00  33.00  45.33  45.33  45.33  96.33  162.33 162.33 306.50 205.50 139.00 139.00 167.50 44.50  51.50  9.50
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w2, w3, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w30, w29, #1, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w13, w5, #4095, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x5, x7, #1638
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w20, wsp, #801
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wsp, wsp, #1104
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wsp, w30, #4084
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x0, x24, #291
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x3, x24, #4095, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x8, sp, #1074
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	sp, x29, #3816
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w0, wsp, #4077
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w4, w20, #546, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	sp, sp, #288
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	wsp, w19, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w13, w23, #291, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w2, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w20, wsp, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x3, #1, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	sp, #20, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x30, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	x4, sp, #3822
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w3, #291, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	wsp, #1365
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	sp, #1092, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	sp, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wsp, w20
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x11, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w24, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wzr, w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w20, wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w9, w3, wzr, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	xzr, x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x20, xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x17, x29, x20, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w20, wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	w9, w3, wzr, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	x20, xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adds	x17, x29, x20, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	wzr, w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w9, w3, wzr, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	xzr, x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x17, x29, x20, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	w9, w3, wzr, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     subs	x17, x29, x20, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w5, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w6, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	w8, w9, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w10, w11, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w12, w13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w14, w15, lsr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w16, w17, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w18, w19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w20, w21, asr #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w22, w23, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x0, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x5, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x6, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x8, x9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmn	x10, x11, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x12, x13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x14, x15, lsr #41
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x16, x17, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x18, x19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x20, x21, asr #55
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x22, x23, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	w0, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	w5, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	w6, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	w8, w9, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w10, w11, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w12, w13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w14, w15, lsr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w18, w19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w20, w21, asr #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w22, w23, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x0, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x5, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x6, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x8, x9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	x10, x11, lsl #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x12, x13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x14, x15, lsr #41
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x16, x17, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x18, x19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x20, x21, asr #55
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x22, x23, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	wzr, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     cmp	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w9, wzr, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x9, xzr, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	w9, wzr, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	x9, xzr, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     adcs	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x9, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	x9, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     sbcs	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w3, w12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	wzr, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w23, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x29, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	w3, w12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	wzr, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	w23, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	x29, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33    -     0.33   0.33    -      -      -      -     ngcs	x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	x1, x2, #3, #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x3, x4, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	wzr, wzr, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w12, w9, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	x4, x5, #52, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	xzr, x4, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	x4, xzr, #1, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x5, x6, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x4, x5, #52, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfc	x4, #1, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x5, x6, #12, #52
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtb	w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtb	xzr, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxth	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxth	x0, w1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtw	x3, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     uxtb	w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     uxth	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	x3, x30, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w1, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	wzr, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w1, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x19, x20, #0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfc	xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x19, x20, #0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	xzr, xzr, #10, #11
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	w5, #4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	x5, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b.ne	#4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#-4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w1, #31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w3, #0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	wzr, #15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x9, #31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x3, #0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	xzr, #5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w1, #31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w3, #0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	wzr, #15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x9, #31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x3, #0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	xzr, #5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w1, wzr, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w3, w0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	wzr, w15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x9, xzr, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x3, x0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	xzr, x5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w1, wzr, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w3, w0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	wzr, w15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x9, xzr, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x3, x0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	xzr, x5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	w3, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	x9, pl
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	w20, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	x30, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w2, wzr, wzr, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x3, xzr, xzr, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	w9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	x9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w5, w6, w6, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x1, x2, x2, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	w9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	x9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x1, x0, x0, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, w8, w8, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	w9, wzr, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	x9, xzr, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x4, x8, x8, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, w8, w8, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	w0, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	x18, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev16	w17, w1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev16	x5, x2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev	w18, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev32	x20, x1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev	x22, x2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     clz	w24, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     clz	x26, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cls	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cls	x20, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     udiv	w0, w7, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     udiv	x9, x22, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     sdiv	w12, w21, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     sdiv	x13, x2, x1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w11, w12, w13
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x14, x15, x16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w17, w18, w19
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x20, x21, x22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w23, w24, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x26, x27, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w0, w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x3, x4, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w6, w7, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x9, x10, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w12, w13, w14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x15, x16, x17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w18, w19, w20
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x21, x22, x23
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w24, w25, w26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x27, x28, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	xzr, x27, x26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x23, x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w1, w3, w7, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	wzr, w0, w9, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w13, wzr, w4, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w19, w30, wzr, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	w4, w5, w6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x1, x3, x7, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	xzr, x0, x9, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x13, xzr, x4, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x19, x30, xzr, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	x4, x5, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w1, w3, w7, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	wzr, w0, w9, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w13, wzr, w4, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w19, w30, wzr, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	w4, w5, w6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x1, x3, x7, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	xzr, x0, x9, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x13, xzr, x4, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x19, x30, xzr, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	x4, x5, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smull	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smnegl	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umull	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umnegl	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x23, x22, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x23, x22, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	x19, x20, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	w21, w22, w23
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smull	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umull	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smnegl	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umnegl	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	w3, w5, w7, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	w11, w13, w17, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	x3, x5, x7, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	x11, x13, x17, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x19, x23, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x29, xzr, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w9, w13, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s3, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s29, s30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d4, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d26, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s1, s31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s3, s0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s31, s15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d9, d31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d3, d0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d31, d5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s1, s31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s3, s0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s31, s15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d9, d31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d3, d0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d31, d5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	s3, s20, s9, pl
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	d9, d10, d11, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s0, s1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	s2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	s4, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	s6, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, s11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	s14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	s16, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	s18, s19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	s20, s21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	s22, s23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	s24, s25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d0, d1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	d2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	d4, d5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	d6, d7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, d11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	d12, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	d14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	d16, d17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	d18, d19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	d20, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	d22, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	d24, d25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s26, h27
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d28, h29
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s20, s19, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	s1, s2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	s4, s5, s6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	s7, s8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	s13, s14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	s16, s17, s18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	s19, s20, s21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	s22, s23, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d20, d19, d17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	d1, d2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d4, d5, d6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	d7, d8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	d10, d11, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	d13, d14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	d16, d17, d18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	d19, d20, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	d22, d23, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, h20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, h0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, h30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, h0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, s20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, s0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, s30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, s0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, d20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, d0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, d30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, d0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, h20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, h0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, h30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, h0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, s20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, s0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, s30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, s0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, d20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, d0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, d30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, d0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, h9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, h20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, h23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, h3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, h3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, h5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, h7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, h9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, h11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, h16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, h26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, h28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, h30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, s20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, s23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, s11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, s26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, s28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, s30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, d20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, d5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, d7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, d11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, d26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, d28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, d30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	w3, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	s9, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x20, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	d1, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x3, v12.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   fmov	v1.d[1], x19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s2, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s3, #1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d30, #16.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s4, #1.06250000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d10, #1.93750000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s12, #-1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d16, #8.50000000
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w3, #0
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x29, #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	xzr, #-4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s0, #8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d0, #1048572
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	q0, #-1048576
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1strm, #0
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	#22, #0
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxrb	w18, w8, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxrh	w24, w15, [x16]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxr	w5, w6, [x17]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxr	w1, x10, [x21]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxrb	w30, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxrh	w17, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	w22, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxr	w9, x3, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxrb	w8, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxrh	w7, [x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	w6, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlrb	w24, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlrh	w25, [x30]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	w26, [x29]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarb	w23, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarh	w22, [x30]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	wzr, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sturb	w9, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stur	w16, [x0, #-256]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stur	x28, [x14, #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldurb	w1, [x20, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldurh	w20, [x1, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	w12, [sp, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	xzr, [x12, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursb	x9, [x7, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursh	x17, [x19, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursw	x20, [x15, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursb	w19, [x1, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursh	w15, [x21, #-256]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	b0, [sp, #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	h12, [x12, #-1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	s15, [x0, #255]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	d31, [x5, #25]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	q9, [x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	b3, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	h5, [x4, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	s7, [x12, #-1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	d11, [x19, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	q13, [x1, #2]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w9, [x2], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w10, [x3], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w19, [sp], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w20, [x30], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w21, [x12], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	xzr, [x9], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x2, [x3], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w9, [x2], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w10, [x3], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w19, [sp], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w20, [x30], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w21, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	wzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	wzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b0, [x0], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b3, [x3], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b5, [sp], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h10, [x10], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h13, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h15, [sp], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s20, [x20], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s23, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d20, [x20], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d23, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b0, [x0], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b3, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b5, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h10, [x10], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h13, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h15, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s20, [x20], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s23, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d20, [x20], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d23, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q20, [x1], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q23, [x9], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q25, [x20], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q10, [x1], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q22, [sp], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q21, [x20], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x3, [x4, #0]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w9, [x2, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w19, [sp, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w20, [x30, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w21, [x12, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x2, [x3, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w9, [x2, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w19, [sp, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w20, [x30, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w21, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	wzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	wzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b0, [x0, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b3, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b5, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h10, [x10, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h13, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h15, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b0, [x0, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b3, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b5, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h10, [x10, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h13, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h15, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q20, [x1, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q23, [x9, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q25, [x20, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q10, [x1, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q22, [sp, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q21, [x20, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sttrb	w9, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sttrh	wzr, [x12, #255]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sttr	w16, [x0, #-256]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     sttr	x28, [x14, #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrb	w1, [x20, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrh	w20, [x1, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtr	w12, [sp, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtr	xzr, [x12, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsb	x9, [x7, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsh	x17, [x19, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsw	x20, [x15, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsb	w19, [x1, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsh	w15, [x21, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x4, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x30, [x12, #32760]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x20, [sp, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	xzr, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w2, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w17, [sp, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w13, [x2, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w2, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w12, [x2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	xzr, [x15]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	x30, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	w20, [x4, #16380]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w17, [sp, #8190]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w23, [x3, #4095]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strb	wzr, [x2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	b31, [sp, #4095]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	h20, [x2, #8190]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	q12, [sp, #65520]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	s12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x3, [sp, x5]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	x9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     str	x11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	d19, [x21, wzr, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q10, [x30, x7, lsl #4]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q13, [x27, x5, sxtx #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w3, w5, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x9, x10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x9, x10, [x2, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x20, x30, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	s27, s26, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d3, d5, [x9, #504]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d7, d11, [x10, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q3, q5, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q17, q19, [sp, #1008]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w3, w5, [sp], #0
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     stp	wzr, w9, [sp], #252
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w2, wzr, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w9, w10, [sp], #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [sp], #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [x2], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x20, x30, [sp], #252
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x21, x29, [x2], #504
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x22, x23, [x3], #-512
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x24, x25, [x4], #8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s29, s28, [sp], #252
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	s27, s26, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s1, s2, [x3], #44
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d3, d5, [x9], #504
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d7, d11, [x10], #-512
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	d2, d3, [x30], #-8
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q3, q5, [sp], #0
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q17, q19, [sp], #1008
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	q23, q29, [x1], #-1024
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w3, w5, [sp, #0]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     stp	wzr, w9, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w2, wzr, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w9, w10, [sp, #4]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [sp, #4]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x20, x30, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x21, x29, [x2, #504]!
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x22, x23, [x3, #-512]!
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x24, x25, [x4, #8]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s29, s28, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	s27, s26, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s1, s2, [x3, #44]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d3, d5, [x9, #504]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d7, d11, [x10, #-512]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	d2, d3, [x30, #-8]!
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q3, q5, [sp, #0]!
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q17, q19, [sp, #1008]!
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	q23, q29, [x1, #-1024]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w3, w5, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -     stnp	wzr, w9, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w9, w10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x21, x29, [x2, #504]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x24, x25, [x4, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	s29, s28, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	s27, s26, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	s1, s2, [x3, #44]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	d3, d5, [x9, #504]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	d7, d11, [x10, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q3, q5, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q17, q19, [sp, #1008]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w3, #983055
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x10, #-6148914691236517206
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w12, w23, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w16, w15, w1, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w9, w4, w10, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w30, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x3, x5, x7, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x5, x14, x19, asr #4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w17, w19, ror #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w0, w2, wzr, lsr #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w30, w11, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	xzr, x4, x26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, wzr, w20, ror #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x7, x20, xzr, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	x13, x20, x14, lsl #47
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	w2, w7, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	w2, w7, w0, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	x8, x9, x10, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orn	x3, x5, x7, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orn	w2, w5, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     ands	x3, x5, x20, ror #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bics	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     tst	w3, w7, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     tst	x2, x20, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x3, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x3, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wzr, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	w2, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w2, #-1235
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x2, #5299989643264
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	w3, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x4, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	w5, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x6, #0, lsl #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x7, #0, lsl #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x8, #0, lsl #48
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x9, #0, lsl #48
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x2, #1600
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adrp	x21, #6553600
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x0, #262144
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b	#4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b	#-4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     b	#134217724
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     br	x20
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     blr	xzr
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     ret	x10
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     ret
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     eret
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     drps
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w2, w3, #4095
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w30, w29, #1, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w13, w5, #4095, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x5, x7, #1638
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w20, wsp, #801
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wsp, wsp, #1104
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wsp, w30, #4084
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x0, x24, #291
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x3, x24, #4095, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x8, sp, #1074
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	sp, x29, #3816
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w0, wsp, #4077
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w4, w20, #546, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	sp, sp, #288
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	wsp, w19, #16
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w2, #4095
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w20, wsp, #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	sp, #20, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x30, #4095
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x4, sp, #3822
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w3, #291, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	wsp, #1365
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	sp, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wsp, w20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x11, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w24, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wzr, w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w20, wzr, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w9, w3, wzr, lsl #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	xzr, x3, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x20, xzr, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	x17, x29, x20, lsl #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w3, w5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w20, wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w9, w3, wzr, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x3, x5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x20, xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x17, x29, x20, lsl #3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	wzr, w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	w9, w3, wzr, lsl #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	xzr, x3, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sub	x17, x29, x20, lsl #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w3, w5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w9, w3, wzr, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x3, x5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x17, x29, x20, lsl #3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w5, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w6, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w8, w9, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w10, w11, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w12, w13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w14, w15, lsr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w16, w17, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w18, w19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w20, w21, asr #22
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	w22, w23, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x0, x3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x5, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x6, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x8, x9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x10, x11, lsl #3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x12, x13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x14, x15, lsr #41
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x16, x17, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x18, x19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x20, x21, asr #55
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmn	x22, x23, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w0, w3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w5, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w6, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w8, w9, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w10, w11, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w12, w13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w14, w15, lsr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w18, w19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w20, w21, asr #22
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x0, x3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x5, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x6, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x8, x9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x10, x11, lsl #3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x12, x13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x14, x15, lsr #41
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x16, x17, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x18, x19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x20, x21, asr #55
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x22, x23, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	wzr, w0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w9, wzr, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x9, xzr, x10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	w9, wzr, w10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	x9, xzr, x10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adcs	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x9, x10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbc	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	w9, w10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	x9, x10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbcs	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w3, w12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	wzr, w9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	w23, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x29, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngc	x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	w3, w12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	wzr, w9
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	w23, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	x29, x30
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	xzr, x0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ngcs	x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	x1, x2, #3, #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x3, x4, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	wzr, wzr, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w12, w9, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	x4, x5, #52, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	xzr, x4, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x5, x6, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x4, x5, #52, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfc	x4, #1, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x5, x6, #12, #52
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtb	w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtb	xzr, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxth	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxth	x0, w1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sxtw	x3, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     uxtb	w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     uxth	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	x3, x30, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w1, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	wzr, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w1, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfc	xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	w5, #4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	x5, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ne	#4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#-4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x9, #31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x9, #31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csel	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	w3, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	x9, pl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	w20, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	x30, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w2, wzr, wzr, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x3, xzr, xzr, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	w9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinc	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cset	x9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	w5, w6, w6, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinc	x1, x2, x2, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	w9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cinv	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csetm	x9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	x1, x0, x0, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	w9, wzr, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	x9, xzr, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x4, x8, x8, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	w0, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	x18, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev16	w17, w1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev16	x5, x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev	w18, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev32	x20, x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev	x22, x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     clz	w24, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     clz	x26, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cls	w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cls	x20, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     udiv	w0, w7, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     udiv	x9, x22, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     sdiv	w12, w21, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -     sdiv	x13, x2, x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w11, w12, w13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x14, x15, x16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w17, w18, w19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x20, x21, x22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w23, w24, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x26, x27, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w0, w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x3, x4, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w6, w7, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x9, x10, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w12, w13, w14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x15, x16, x17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w18, w19, w20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x21, x22, x23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w24, w25, w26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	xzr, x27, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x23, x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     madd	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     msub	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	x19, x20, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mneg	w21, w22, w23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	w3, w5, w7, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	w11, w13, w17, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	x3, x5, x7, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25    -      -      -      -     extr	x11, x13, x17, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x19, x23, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x29, xzr, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s3, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d4, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d26, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s0, s1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	s4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	s6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	s22, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d0, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	d6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	d12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	d16, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	d18, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	d22, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	d24, d25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s26, h27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d28, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	s7, s8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	s16, s17, s18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	s22, s23, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d20, d19, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	d1, d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d4, d5, d6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	d7, d8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	d10, d11, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	d16, d17, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, h20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, h23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, h7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, h11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, h16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	h23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, h26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, h28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, h30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	s23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, s26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, s28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	w3, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtns	xzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	wzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtnu	x0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	wzr, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtps	x12, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	w30, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtpu	x29, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	w2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtms	x4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	w6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtmu	x8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	w10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	x12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	w14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	x15, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     scvtf	d19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	d23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	w25, d26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, d28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, d30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	w3, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x20, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	d1, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x3, v12.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s2, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s3, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d30, #16.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s4, #1.06250000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d10, #1.93750000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s12, #-1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d16, #8.50000000
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w3, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x29, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	xzr, #-4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s0, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d0, #1048572
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	q0, #-1048576
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1strm, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	#22, #0
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxr	w5, w6, [x17]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxr	w1, x10, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxrb	w30, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxrh	w17, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	w22, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxrb	w8, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxrh	w7, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlrb	w24, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlrh	w25, [x30]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	w26, [x29]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarb	w23, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarh	w22, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	wzr, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sturb	w9, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stur	w16, [x0, #-256]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stur	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldurb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldurh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	b0, [sp, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	h12, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	s15, [x0, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	d31, [x5, #25]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stur	q9, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	b3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	h5, [x4, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	s7, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	d11, [x19, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldur	q13, [x1, #2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w9, [x2], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w10, [x3], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w19, [sp], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w20, [x30], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w21, [x12], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	xzr, [x9], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x2, [x3], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q20, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q23, [x9], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q25, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q10, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q22, [sp], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q21, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x3, [x4, #0]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     strh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w19, [sp, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w20, [x30, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x2, [x3, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     str	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q20, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q23, [x9, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q10, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q22, [sp, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q21, [x20, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sttrb	w9, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sttr	w16, [x0, #-256]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sttr	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtr	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x4, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x30, [x12, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x20, [sp, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	xzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w2, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w17, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w13, [x2, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w2, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w12, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	x30, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w17, [sp, #8190]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w23, [x3, #4095]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	wzr, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	b31, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	h20, [x2, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x3, [sp, x5]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	x9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w3, w5, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d3, d5, [x9], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q3, q5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -     stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50   0.50   0.50    -      -      -      -     ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   0.50   0.50    -      -     stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w3, w5, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w3, #983055
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x10, #-6148914691236517206
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w12, w23, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w16, w15, w1, lsl #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w9, w4, w10, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w30, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x3, x5, x7, lsl #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x5, x14, x19, asr #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w17, w19, ror #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w0, w2, wzr, lsr #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, w30, w11, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	xzr, x4, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w3, wzr, w20, ror #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x7, x20, xzr, asr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	x13, x20, x14, lsl #47
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	w2, w7, w0, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	x8, x9, x10, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orn	x3, x5, x7, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orn	w2, w5, w29
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     ands	x3, x5, x20, ror #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     bics	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     tst	w3, w7, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     tst	x2, x20, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x3, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x3, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wzr, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	w2, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w2, #-1235
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x2, #5299989643264
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	w3, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x4, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	w5, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x6, #0, lsl #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x7, #0, lsl #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x8, #0, lsl #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x2, #1600
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adrp	x21, #6553600
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x0, #262144
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#-4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     br	x20
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     blr	xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret	x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     eret
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     drps
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-clear-upper-regs.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-clear-upper-regs.s
index ab81f9fb04af3..37ae765148396 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-clear-upper-regs.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-clear-upper-regs.s
@@ -80,26 +80,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34   0.22   0.22   0.28   0.28    -      -      -      -
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34   0.22   0.22   0.28   0.28    -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	w0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.22   0.22   0.28   0.28    -      -      -      -     add	x0, x0, x0
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	w0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.22   0.22   0.28   0.28    -      -      -      -     add	x0, x0, x0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
@@ -153,26 +156,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	b0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	b0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -227,26 +233,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	h0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	h0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -301,26 +310,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	s0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ldr	s0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d0, d0, d0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -375,26 +387,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.8b }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.8b }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.16b, v0.16b, v0.16b
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -449,26 +464,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.4h }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.4h }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8h, v0.8h, v0.8h
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -523,26 +541,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.4s, v0.4s, v0.4s
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -597,26 +618,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.1d }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -     ld1	{ v0.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.2d, v0.2d, v0.2d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -671,26 +695,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   mov	v0.b[0], v1.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     add	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   mov	v0.b[0], v1.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     add	v0.16b, v0.16b, v0.16b
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
@@ -745,26 +772,29 @@ add v0.16b, v0.16b, v0.16b
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.50   0.50
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.50   0.50
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -     0.50    -     0.50   ld1	{ v0.b }[0], [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     add	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     0.50    -     0.50   ld1	{ v0.b }[0], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     add	v0.16b, v0.16b, v0.16b
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
index 5ad42d5ff0a13..20e76ef27c470 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
@@ -2451,1238 +2451,1241 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     26.67  49.17  49.17  18.75  7.75   7.75   7.75   431.00 356.50 385.50 311.00
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     26.67  49.17  49.17  18.75  7.75   7.75   7.75   431.00 356.50 385.50 311.00
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	d29, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	d17, d31, d29
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   addv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	h0, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.4h, #15, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.8b, v0.8b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.2s, v0.2s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v0.2d, v0.2d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v0.8h, v0.8h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.16b, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.2d, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.2s, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.4h, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.4s, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.8b, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.8h, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d0, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.16b, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2d, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2s, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4s, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8b, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.16b, v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.8b, v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	d29, d24, d20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	s29, s24, s20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.2s, v0.2s, v0.2s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.4s, v0.4s, v0.4s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.2d, v0.2d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.2s, v0.2s, v0.2s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.4s, v0.4s, v0.s[1], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v0.2d, v0.2d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v0.4s, v0.4s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl	v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl	v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl2	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl2	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	s22, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s21, s12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	h21, h14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	h21, h12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s21, s12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	h21, h12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fdiv	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	d0, d1, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	s0, s1, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	d0, d4, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	s3, s5, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2d, #-1.25000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2s, #13.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.4s, #1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d0, d1, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s0, s1, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d0, d4, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d23, d11, d1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s20, s22, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s3, s5, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	d13, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	s19, s14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	d22, d30, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	s21, s16, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	d16, d19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	s18, s10
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	d21, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	d8, d22, d18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	s21, s5, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     5.50    -     5.50    -     fsqrt	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8h, v1.8h }, [sp], #32
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [x0], #48
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.1d }, [x15], x2
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v0.2s, v1.2s }, [x15]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v0.4h, v1.4h, v2.4h }, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.b }[7], [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.h }[3], [x0], #2
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.s }[1], [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.d }[0], [x15], #8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x0], #2
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.4s }, [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.2d }, [x15], x16
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.16b, v1.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.8b, v1.8b }, [x0], #16
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15], x8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.4h, v1.4h }, [x0], #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.2s, v1.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.1d, v1.1d }, [sp], x8
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], #48
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x15]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.2s, v1.2s, v2.2s }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.1d, v1.1d, v2.1d }, [x0], x0
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h2, v0.h[5]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s17, v0.s[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.b[0], v0.b[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.h[1], v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.s[2], v0.s[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.d[1], v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.b[0], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.h[1], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.s[2], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.d[1], x8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	d15, #0xff00ff00ff00ff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.16b, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2d, #0xff0000ff0000ffff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v0.8h, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v21.8b, v1.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v30.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v0.4h, v9.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v21.8b, v1.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v30.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v4.8h, v7.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v0.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v1.8b, v9.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v13.4h, v21.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v2.8h, v4.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v4.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v6.4s, v8.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	d0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	s0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	s0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	h0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   saddlv	h0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shl	d7, d10, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sli	d10, d14, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   smaxv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   sminv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	d18, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	h21, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	s20, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d8, s9, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s20, s21, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d1, s1, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmull	d15, s22, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s1, h1, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmull	s12, h22, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	d18, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	h21, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	s20, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, h2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, h2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s20, s21, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	h3, h4, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	b10, h13, #2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	h15, s10, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	s15, d12, #9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	b17, h10, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	h10, s13, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	s22, d16, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	b11, b19, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d15, d16, #51
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h13, h18, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h3, h4, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s14, s17, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	b15, b18, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	d11, d13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	h19, h17, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	s16, s14, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	b10, h15, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	h17, s10, #4
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	s18, d10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	b15, h10, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	h20, s14, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	s10, d15, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	s20, s10, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	b18, h18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	h20, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	s19, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	b19, h14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	h21, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	s20, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sri	d10, d12, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	d16, d16, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srshr	d19, d18, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srsra	d15, d11, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v0.2d, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v0.4s, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sshr	d15, d16, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssra	d18, d12, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.4s, v1.4s }, [sp], #32
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v0.8h, v1.8h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.d }[1], [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.d }[1], [x0], #8
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st2	{ v0.16b, v1.16b }, [x0], x1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp], #8
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     3.00   3.00   0.25   0.25   0.25   0.25   3.00   3.00    -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   st4	{ v0.b, v1.b, v2.b, v3.b }[15], [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	d15, d5, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	d18, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	h20, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	s21, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.16b, { v0.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.50   2.50    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.8b, { v0.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.50   2.50    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	d0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	s0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	s0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	h0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uaddlv	h0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   umaxv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uminv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	w0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	x0, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	h0, h1, h5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	b11, b20, b30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	s23, s20, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	b10, h12, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	h12, s10, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	s10, d10, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b11, b20, b30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b18, b15, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	d15, d12, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	h11, h18, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s14, s19, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s23, s20, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	b12, h10, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	h10, s14, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	s10, d12, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	d16, d16, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	b18, h18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	h20, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	s19, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	d8, d7, d4
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urshr	d20, d23, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ursra	d18, d10, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	d0, d0, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v0.4s, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v0.8h, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ushr	d10, d17, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	d18, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	h20, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	s21, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usra	d20, d13, #61
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	d17, d31, d29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   addv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	h0, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.16b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.2d, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.2s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.4h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.4s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.8b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.8h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d0, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.16b, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2d, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2s, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4s, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8b, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	d29, d24, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	s29, s24, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.2s, v0.2s, v0.2s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.4s, v0.4s, v0.4s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.2s, v0.2s, v0.2s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.4s, v0.4s, v0.s[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	s22, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	h21, h14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	h21, h12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	h21, h12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fdiv	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d23, d11, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s20, s22, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	d13, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	s19, s14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	d22, d30, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	s21, s16, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	d16, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	s18, s10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	d8, d22, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	s21, s5, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.50    -     5.50    -     fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8h, v1.8h }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.1d }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v0.2s, v1.2s }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v0.4h, v1.4h, v2.4h }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.b }[7], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.h }[3], [x0], #2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.s }[1], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.d }[0], [x15], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x0], #2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.4s }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.2d }, [x15], x16
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.16b, v1.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.8b, v1.8b }, [x0], #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.4h, v1.4h }, [x0], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.2s, v1.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.1d, v1.1d }, [sp], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.2s, v1.2s, v2.2s }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.1d, v1.1d, v2.1d }, [x0], x0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h2, v0.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s17, v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.b[0], v0.b[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.h[1], v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.s[2], v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.d[1], v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.b[0], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.h[1], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.s[2], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   mov	v0.d[1], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.16b, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v0.8h, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v0.4h, v9.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v4.8h, v7.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v0.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v1.8b, v9.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v13.4h, v21.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v2.8h, v4.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v4.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v6.4s, v8.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	d0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	s0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	s0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	h0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   saddlv	h0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shl	d7, d10, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sli	d10, d14, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   smaxv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   sminv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmull	d15, s22, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmull	s12, h22, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, h2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, h2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	b10, h13, #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	h15, s10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	s15, d12, #9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	b17, h10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	h10, s13, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrun	s22, d16, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	b11, b19, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d15, d16, #51
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h13, h18, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s14, s17, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	b15, b18, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	d11, d13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	h19, h17, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	s16, s14, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	b10, h15, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	h17, s10, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrn	s18, d10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	b15, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	h20, s14, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	s10, d15, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	s20, s10, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	b19, h14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	h21, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	s20, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sri	d10, d12, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srshr	d19, d18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srsra	d15, d11, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sshr	d15, d16, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssra	d18, d12, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     3.00   3.00   0.25   0.25   0.25   0.25   3.00   3.00    -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   st4	{ v0.b, v1.b, v2.b, v3.b }[15], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	d15, d5, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	d0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	s0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	s0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	h0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uaddlv	h0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   umaxv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uminv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	w0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	x0, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	h0, h1, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	b10, h12, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	h12, s10, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	s10, d10, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b18, b15, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	d15, d12, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	h11, h18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s14, s19, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	b12, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	h10, s14, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqshrn	s10, d12, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	d8, d7, d4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urshr	d20, d23, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ursra	d18, d10, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	d0, d0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ushr	d10, d17, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usra	d20, d13, #61
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8h, v0.8h, v0.8h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
index f120f5feaf7ce..6b7616d68d4cd 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
@@ -4825,2425 +4825,2428 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     88.67  500.67 500.67 797.50 2.50   92.50  92.50  1252.00 921.00 178.50 181.50
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     88.67  500.67 500.67 797.50 2.50   92.50  92.50  1252.00 921.00 178.50 181.50
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.b, p0/m, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z1.s, z2.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.b, p5/m, z21.b, z10.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.b, z10.b, z21.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.d, p5/m, z21.d, z10.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.d, z10.d, z21.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.h, p5/m, z21.h, z10.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.h, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.s, p5/m, z21.s, z10.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.s, z10.s, z21.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.b, p3/m, z23.b, z13.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.b, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.d, p3/m, z23.d, z13.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.h, p3/m, z23.h, z13.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.h, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.s, p3/m, z23.s, z13.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.s, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	sp, sp, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x0, x0, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x21, x21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x23, x8, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	sp, sp, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x0, x0, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x21, x21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x23, x8, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     and	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ands	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bfcvt	z0.h, p0/m, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bfcvtnt	z0.h, p0/m, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z10.s, z21.h, z14.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z21.s, z14.h, z3.h[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z14.s, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     bic	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     bic	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     bics	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     bics	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brka	p0.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brka	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkas	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkb	p0.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkb	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkbs	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkn	p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkn	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkns	p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkns	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpa	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpa	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpas	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpas	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpb	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpb	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	b0, p7, b0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	x0, p7, x0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.b, p7, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.d, p7, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.h, p7, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	b0, p7, b0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	x0, p7, x0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.b, p7, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.d, p7, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.h, p7, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.d, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.s, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	wzr, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	wzr, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #256
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, #512
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.b, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.d, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.h, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.s, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     eor	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     eors	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ext	z31.b, z31.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ext	z31.b, z31.b, z0.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     fadda	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -      -      -     fadda	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fadda	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   faddv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     faddv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   faddv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.d, p0/m, z0.d, z0.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.h, p0/m, z0.h, z0.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.s, p0/m, z0.s, z0.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.d, p7/m, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.h, p7/m, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.d, p0/m, z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, p0/m, z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, p0/m, z1.h, z2.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, z0.h, z0.h[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.s, p0/m, z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.s, p0/m, z1.s, z2.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z21.s, z10.s, z5.s[1], #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z23.s, z13.s, z8.s[0], #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.d, p7/m, z30.d, z31.d, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.h, p7/m, z30.h, z31.h, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.s, p7/m, z30.s, z31.s, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.d, p7/m, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.h, p7/m, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.h, z31.h, z7.h[3], #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.h, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvt	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvt	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     fcvtzs	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzs	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzs	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     fcvtzu	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzu	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzu	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fdiv	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fdivr	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fmaxnmv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fmaxnmv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fmaxnmv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fmaxv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fmaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fmaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fminnmv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fminnmv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fminnmv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fminv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, #-10.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, p0/m, #-10.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, p0/m, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.h, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.h, p0/m, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.s, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.s, p0/m, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, z0.d, z0.d[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, z0.h, z0.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, z0.s, z0.s[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.d, p7/m, z31.d, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.d, z31.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.h, p7/m, z31.h, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.h, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.s, p7/m, z31.s, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.s, z31.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpe	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     frecpe	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     frecpe	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frsqrte	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     frsqrte	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     frsqrte	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fsqrt	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fsqrt	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fsqrt	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.d, z0.d, z31.d, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.h, z0.h, z31.h, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.s, z0.s, z31.s, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     inch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     inch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.b, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     index	z0.d, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.h, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z0.h, w0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.s, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z21.b, w10, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z21.d, x10, x21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z21.s, w10, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.b, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.b, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z23.d, #13, x8
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z23.d, x13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.h, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.h, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.s, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.s, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.b, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     index	z31.d, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, #-1, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, xzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, xzr, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.h, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.s, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.b, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.d, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.h, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.s, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.b, b31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.d, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.d, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.h, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.s, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	x0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	x0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.b, p0/m, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.b, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.d, p0/m, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.d, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.h, p0/m, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.h, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.s, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mad	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mla	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mls	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0/m, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, b0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, p0/m, b0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.b, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.b, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0xe0000000000003ff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0xffffffffffff7fff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, p0/m, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.d, p0/m, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.d, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #-256
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #32767
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, p0/m, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.h, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.h, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.q, q0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #0xffff7fff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, p0/m, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.s, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.s, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, p15/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, b31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31, z6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.b, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, z31.b[63]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p15/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31.d, p7/z, z6.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.d, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, z31.d[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, p15/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.h, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, z31.h[31]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, p15/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.s, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, z31.s[15]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.d, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.h, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.q, z17.q[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.s, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     msb	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.b, z31.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z31.d, z31.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.h, z31.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.s, z31.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nand	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nand	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nands	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nands	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.b, p0/m, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nor	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nor	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nors	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nors	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     not	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     not	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nots	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nots	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orn	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orn	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orns	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orns	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orr	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orrs	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfalse	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfirst	p0.b, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfirst	p15.b, p15, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.b, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.d, p15, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.h, p15, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.s, p15, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p15.b, p15, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.b, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #23
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #26
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #27
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, mul3
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, mul4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl128
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl256
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl3
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl5
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl7
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl8
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.b, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #23
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #26
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #27
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, mul3
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, mul4
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl128
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl16
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl256
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl3
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl32
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl4
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl5
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl6
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl64
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl7
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpkhi	p0.h, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpkhi	p15.h, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpklo	p0.h, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpklo	p15.h, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdffr	p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     rdffr	p0.b, p0/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdffr	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     rdffr	p15.b, p15/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     rdffrs	p0.b, p0/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     rdffrs	p15.b, p15/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x21, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x23, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	xzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revh	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revh	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revw	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   saddv	d0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     scvtf	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.b, p11, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.d, p11, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.h, p11, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.s, p11, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     setffr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.b, z0.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.d, z0.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.h, z0.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.s, z0.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   smaxv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.b, z0.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.d, z0.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.h, z0.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.s, z0.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   sminv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     smulh	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.b, p7, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.d, p7, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.h, p7, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.s, p7, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.d, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.d, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.h }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.s }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.b }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.h }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.b }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.d }, p7, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.h }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z31.s }, p7, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z31.d }, p7, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.s }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.h }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.d }, p7, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.h }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z31.s }, p7, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.d }, p7, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z31.s }, p7, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z0.b, z1.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z0.b, z1.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z5.b, z6.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z0.d, z1.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z0.h, z1.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z0.s, z1.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3b	{ z0.b - z2.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z0.b - z2.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3b	{ z5.b - z7.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z0.d - z2.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z0.h - z2.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z0.s - z2.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4b	{ z0.b - z3.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z0.b - z3.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4b	{ z5.b - z8.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z0.d - z3.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z0.h - z3.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z0.s - z3.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p15, [sp, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p5, [x10, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z21, [x10, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z31, [sp, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.b, p5/m, z21.b, z10.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.b, z10.b, z21.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.d, p5/m, z21.d, z10.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.d, z10.d, z21.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.h, p5/m, z21.h, z10.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.h, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.s, p5/m, z21.s, z10.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.s, z10.s, z21.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.b, p3/m, z23.b, z13.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.b, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.d, p3/m, z23.d, z13.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.h, p3/m, z23.h, z13.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.h, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.s, p3/m, z23.s, z13.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.s, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtw	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtw	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.b, { z31.b }, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.d, { z31.d }, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.h, { z31.h }, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.s, { z31.s }, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uaddv	d0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     ucvtf	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   umaxv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uminv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ummla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     umulh	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	z0.s, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     usmmla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtw	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtw	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilele	p0.b, w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelo	p15.d, xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilels	p0.h, w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelt	p15.s, xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     wrffr	p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     wrffr	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.b, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.d, p0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.h, p0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.s, p0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.b, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.d, p0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.h, p0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.s, p0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.s, z31.s, z31.s
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	sp, sp, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x0, x0, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x21, x21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addpl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	sp, sp, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x0, x0, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x21, x21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     addvl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     andv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cnth	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntp	x0, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     cntw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.d, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.s, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	wzr, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermeq	xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	wzr, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ctermne	xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, #512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.h, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.s, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     dupm	z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     eorv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     18.00   -      -      -     fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   faddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     faddv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   faddv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fexpa	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   fminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   3.00    -      -     fminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.75   2.75   0.25   0.25   fminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpe	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     frecpe	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     frecpe	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     frsqrte	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     frsqrte	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     frsqrte	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     10.00   -      -      -     fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     inch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     inch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     inch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     incp	z31.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     incw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     incw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.b, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     index	z0.d, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.h, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z0.h, w0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z0.s, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z21.b, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z21.d, x10, x21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z21.s, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.b, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.b, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z23.d, #13, x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z23.d, x13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.h, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.h, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.s, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z23.s, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.b, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.b, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     index	z31.d, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, #-1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     2.00    -      -      -     index	z31.d, xzr, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.h, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.h, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     index	z31.s, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     index	z31.s, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.b, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.d, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.d, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.h, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.s, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     insr	z31.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lasta	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     1.00    -      -     lastb	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -     0.50   0.50   1.50   1.50    -      -     ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -     1.00   1.00   2.00   2.00    -      -     ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mad	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mla	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mls	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, p0/m, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.b, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, p0/m, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.d, p0/m, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, #32767
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, p0/m, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.h, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.q, q0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, p0/m, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z0.s, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z0.s, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31, z6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.b, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, z31.b[63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.d, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, z31.d[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.h, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.h, z31.h[31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.s, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.s, z31.s[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.d, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.h, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.q, z17.q[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z5.s, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     msb	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mul	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mul	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     orv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfalse	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ptrues	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpkhi	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpkhi	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpklo	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     punpklo	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdffr	p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     rdffr	p0.b, p0/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     rdffr	p15.b, p15/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     rdffrs	p0.b, p0/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     rdffrs	p15.b, p15/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x21, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x23, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   saddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     setffr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smax	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   smaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   sminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqdecp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     sqincp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     0.50   0.50   0.50   0.50   st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -      -      -     1.25   1.25   1.25   1.25   st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.50   2.50    -      -     2.50   2.50   1.25   1.25   1.25   1.25   st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -     2.25   2.25   2.25   2.25   st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     4.50   4.50   2.25   2.25   2.25   2.25   st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50   0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uaddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   umaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqdecp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     uqincp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilele	p0.b, w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelo	p15.d, xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilels	p0.h, w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelt	p15.s, xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     wrffr	p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     wrffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     zip2	z31.s, z31.s, z31.s
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-zero-dependency.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-zero-dependency.s
index 071329fd00cdd..8b1c8a4e4ca55 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-zero-dependency.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-zero-dependency.s
@@ -12,7 +12,7 @@ cmp x0, #4
 # CHECK:      Dispatch Width:    15
 # CHECK-NEXT: uOps Per Cycle:    3.70
 # CHECK-NEXT: IPC:               3.70
-# CHECK-NEXT: Block RThroughput: 0.3
+# CHECK-NEXT: Block RThroughput: 0.5
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -31,26 +31,29 @@ cmp x0, #4
 # CHECK-NEXT: [0.1] - V1UnitB
 # CHECK-NEXT: [1.0] - V1UnitD
 # CHECK-NEXT: [1.1] - V1UnitD
-# CHECK-NEXT: [2]   - V1UnitL2
-# CHECK-NEXT: [3.0] - V1UnitL01
-# CHECK-NEXT: [3.1] - V1UnitL01
-# CHECK-NEXT: [4]   - V1UnitM0
-# CHECK-NEXT: [5]   - V1UnitM1
-# CHECK-NEXT: [6.0] - V1UnitS
-# CHECK-NEXT: [6.1] - V1UnitS
-# CHECK-NEXT: [7]   - V1UnitV0
-# CHECK-NEXT: [8]   - V1UnitV1
-# CHECK-NEXT: [9]   - V1UnitV2
-# CHECK-NEXT: [10]  - V1UnitV3
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -     0.50   0.50   0.50   0.50    -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.48   0.50   0.01   0.01    -      -      -      -     mov	x0, x1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.02    -     0.49   0.49    -      -      -      -     cmp	x0, #4
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.49   0.49   0.01   0.01    -      -      -      -     mov	x0, x1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -     0.01   0.01   0.49   0.49    -      -      -      -     cmp	x0, #4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
index 9a3bcac69582a..581dad6b68dcf 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
@@ -1406,16 +1406,16 @@ drps
 # CHECK-NEXT:  1      1     0.17                        sub	w4, w20, #546, lsl #12
 # CHECK-NEXT:  1      1     0.17                        sub	sp, sp, #288
 # CHECK-NEXT:  1      1     0.17                        sub	wsp, w19, #16
-# CHECK-NEXT:  1      1     0.25                        adds	w13, w23, #291, lsl #12
-# CHECK-NEXT:  1      1     0.25                        cmn	w2, #4095
-# CHECK-NEXT:  1      1     0.25                        adds	w20, wsp, #0
-# CHECK-NEXT:  1      1     0.25                        cmn	x3, #1, lsl #12
-# CHECK-NEXT:  1      1     0.25                        cmp	sp, #20, lsl #12
-# CHECK-NEXT:  1      1     0.25                        cmp	x30, #4095
-# CHECK-NEXT:  1      1     0.25                        subs	x4, sp, #3822
-# CHECK-NEXT:  1      1     0.25                        cmn	w3, #291, lsl #12
-# CHECK-NEXT:  1      1     0.25                        cmn	wsp, #1365
-# CHECK-NEXT:  1      1     0.25                        cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  1      1     0.33                        adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmn	w2, #4095
+# CHECK-NEXT:  1      1     0.33                        adds	w20, wsp, #0
+# CHECK-NEXT:  1      1     0.33                        cmn	x3, #1, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmp	sp, #20, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmp	x30, #4095
+# CHECK-NEXT:  1      1     0.33                        subs	x4, sp, #3822
+# CHECK-NEXT:  1      1     0.33                        cmn	w3, #291, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmn	wsp, #1365
+# CHECK-NEXT:  1      1     0.33                        cmn	sp, #1092, lsl #12
 # CHECK-NEXT:  1      1     0.17                        mov	sp, x30
 # CHECK-NEXT:  1      1     0.17                        mov	wsp, w20
 # CHECK-NEXT:  1      1     0.17                        mov	x11, sp
@@ -1446,11 +1446,11 @@ drps
 # CHECK-NEXT:  1      2     0.50                        add	x2, x3, x4, asr #0
 # CHECK-NEXT:  1      2     0.50                        add	x5, x6, x7, asr #21
 # CHECK-NEXT:  1      2     0.50                        add	x8, x9, x10, asr #63
-# CHECK-NEXT:  1      1     0.25                        adds	w3, w5, w7
-# CHECK-NEXT:  1      1     0.25                        cmn	w3, w5
-# CHECK-NEXT:  1      1     0.25                        adds	w20, wzr, w4
-# CHECK-NEXT:  1      1     0.25                        adds	w4, w6, wzr
-# CHECK-NEXT:  1      1     0.25                        adds	w11, w13, w15
+# CHECK-NEXT:  1      1     0.33                        adds	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        cmn	w3, w5
+# CHECK-NEXT:  1      1     0.33                        adds	w20, wzr, w4
+# CHECK-NEXT:  1      1     0.33                        adds	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        adds	w11, w13, w15
 # CHECK-NEXT:  1      2     0.50                        adds	w9, w3, wzr, lsl #10
 # CHECK-NEXT:  1      2     0.50                        adds	w17, w29, w20, lsl #31
 # CHECK-NEXT:  1      2     0.50                        adds	w21, w22, w23, lsr #0
@@ -1459,11 +1459,11 @@ drps
 # CHECK-NEXT:  1      2     0.50                        adds	w2, w3, w4, asr #0
 # CHECK-NEXT:  1      2     0.50                        adds	w5, w6, w7, asr #21
 # CHECK-NEXT:  1      2     0.50                        adds	w8, w9, w10, asr #31
-# CHECK-NEXT:  1      1     0.25                        adds	x3, x5, x7
-# CHECK-NEXT:  1      1     0.25                        cmn	x3, x5
-# CHECK-NEXT:  1      1     0.25                        adds	x20, xzr, x4
-# CHECK-NEXT:  1      1     0.25                        adds	x4, x6, xzr
-# CHECK-NEXT:  1      1     0.25                        adds	x11, x13, x15
+# CHECK-NEXT:  1      1     0.33                        adds	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        cmn	x3, x5
+# CHECK-NEXT:  1      1     0.33                        adds	x20, xzr, x4
+# CHECK-NEXT:  1      1     0.33                        adds	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        adds	x11, x13, x15
 # CHECK-NEXT:  1      2     0.50                        adds	x9, x3, xzr, lsl #10
 # CHECK-NEXT:  1      2     0.50                        adds	x17, x29, x20, lsl #63
 # CHECK-NEXT:  1      2     0.50                        adds	x21, x22, x23, lsr #0
@@ -1496,10 +1496,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        sub	x2, x3, x4, asr #0
 # CHECK-NEXT:  1      2     0.50                        sub	x5, x6, x7, asr #21
 # CHECK-NEXT:  1      2     0.50                        sub	x8, x9, x10, asr #63
-# CHECK-NEXT:  1      1     0.25                        subs	w3, w5, w7
-# CHECK-NEXT:  1      1     0.25                        cmp	w3, w5
-# CHECK-NEXT:  1      1     0.25                        subs	w4, w6, wzr
-# CHECK-NEXT:  1      1     0.25                        subs	w11, w13, w15
+# CHECK-NEXT:  1      1     0.33                        subs	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        cmp	w3, w5
+# CHECK-NEXT:  1      1     0.33                        subs	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        subs	w11, w13, w15
 # CHECK-NEXT:  1      2     0.50                        subs	w9, w3, wzr, lsl #10
 # CHECK-NEXT:  1      2     0.50                        subs	w17, w29, w20, lsl #31
 # CHECK-NEXT:  1      2     0.50                        subs	w21, w22, w23, lsr #0
@@ -1508,10 +1508,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        subs	w2, w3, w4, asr #0
 # CHECK-NEXT:  1      2     0.50                        subs	w5, w6, w7, asr #21
 # CHECK-NEXT:  1      2     0.50                        subs	w8, w9, w10, asr #31
-# CHECK-NEXT:  1      1     0.25                        subs	x3, x5, x7
-# CHECK-NEXT:  1      1     0.25                        cmp	x3, x5
-# CHECK-NEXT:  1      1     0.25                        subs	x4, x6, xzr
-# CHECK-NEXT:  1      1     0.25                        subs	x11, x13, x15
+# CHECK-NEXT:  1      1     0.33                        subs	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        cmp	x3, x5
+# CHECK-NEXT:  1      1     0.33                        subs	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        subs	x11, x13, x15
 # CHECK-NEXT:  1      2     0.50                        subs	x9, x3, xzr, lsl #10
 # CHECK-NEXT:  1      2     0.50                        subs	x17, x29, x20, lsl #63
 # CHECK-NEXT:  1      2     0.50                        subs	x21, x22, x23, lsr #0
@@ -1520,9 +1520,9 @@ drps
 # CHECK-NEXT:  1      2     0.50                        subs	x2, x3, x4, asr #0
 # CHECK-NEXT:  1      2     0.50                        subs	x5, x6, x7, asr #21
 # CHECK-NEXT:  1      2     0.50                        subs	x8, x9, x10, asr #63
-# CHECK-NEXT:  1      1     0.25                        cmn	wzr, w4
-# CHECK-NEXT:  1      1     0.25                        cmn	w5, wzr
-# CHECK-NEXT:  1      1     0.25                        cmn	w6, w7
+# CHECK-NEXT:  1      1     0.33                        cmn	wzr, w4
+# CHECK-NEXT:  1      1     0.33                        cmn	w5, wzr
+# CHECK-NEXT:  1      1     0.33                        cmn	w6, w7
 # CHECK-NEXT:  1      2     0.50                        cmn	w8, w9, lsl #15
 # CHECK-NEXT:  1      2     0.50                        cmn	w10, w11, lsl #31
 # CHECK-NEXT:  1      2     0.50                        cmn	w12, w13, lsr #0
@@ -1531,10 +1531,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmn	w18, w19, asr #0
 # CHECK-NEXT:  1      2     0.50                        cmn	w20, w21, asr #22
 # CHECK-NEXT:  1      2     0.50                        cmn	w22, w23, asr #31
-# CHECK-NEXT:  1      1     0.25                        cmn	x0, x3
-# CHECK-NEXT:  1      1     0.25                        cmn	xzr, x4
-# CHECK-NEXT:  1      1     0.25                        cmn	x5, xzr
-# CHECK-NEXT:  1      1     0.25                        cmn	x6, x7
+# CHECK-NEXT:  1      1     0.33                        cmn	x0, x3
+# CHECK-NEXT:  1      1     0.33                        cmn	xzr, x4
+# CHECK-NEXT:  1      1     0.33                        cmn	x5, xzr
+# CHECK-NEXT:  1      1     0.33                        cmn	x6, x7
 # CHECK-NEXT:  1      2     0.50                        cmn	x8, x9, lsl #15
 # CHECK-NEXT:  1      2     0.50                        cmn	x10, x11, lsl #63
 # CHECK-NEXT:  1      2     0.50                        cmn	x12, x13, lsr #0
@@ -1543,10 +1543,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmn	x18, x19, asr #0
 # CHECK-NEXT:  1      2     0.50                        cmn	x20, x21, asr #55
 # CHECK-NEXT:  1      2     0.50                        cmn	x22, x23, asr #63
-# CHECK-NEXT:  1      1     0.25                        cmp	w0, w3
-# CHECK-NEXT:  1      1     0.25                        cmp	wzr, w4
-# CHECK-NEXT:  1      1     0.25                        cmp	w5, wzr
-# CHECK-NEXT:  1      1     0.25                        cmp	w6, w7
+# CHECK-NEXT:  1      1     0.33                        cmp	w0, w3
+# CHECK-NEXT:  1      1     0.33                        cmp	wzr, w4
+# CHECK-NEXT:  1      1     0.33                        cmp	w5, wzr
+# CHECK-NEXT:  1      1     0.33                        cmp	w6, w7
 # CHECK-NEXT:  1      2     0.50                        cmp	w8, w9, lsl #15
 # CHECK-NEXT:  1      2     0.50                        cmp	w10, w11, lsl #31
 # CHECK-NEXT:  1      2     0.50                        cmp	w12, w13, lsr #0
@@ -1554,10 +1554,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmp	w18, w19, asr #0
 # CHECK-NEXT:  1      2     0.50                        cmp	w20, w21, asr #22
 # CHECK-NEXT:  1      2     0.50                        cmp	w22, w23, asr #31
-# CHECK-NEXT:  1      1     0.25                        cmp	x0, x3
-# CHECK-NEXT:  1      1     0.25                        cmp	xzr, x4
-# CHECK-NEXT:  1      1     0.25                        cmp	x5, xzr
-# CHECK-NEXT:  1      1     0.25                        cmp	x6, x7
+# CHECK-NEXT:  1      1     0.33                        cmp	x0, x3
+# CHECK-NEXT:  1      1     0.33                        cmp	xzr, x4
+# CHECK-NEXT:  1      1     0.33                        cmp	x5, xzr
+# CHECK-NEXT:  1      1     0.33                        cmp	x6, x7
 # CHECK-NEXT:  1      2     0.50                        cmp	x8, x9, lsl #15
 # CHECK-NEXT:  1      2     0.50                        cmp	x10, x11, lsl #63
 # CHECK-NEXT:  1      2     0.50                        cmp	x12, x13, lsr #0
@@ -1566,8 +1566,8 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmp	x18, x19, asr #0
 # CHECK-NEXT:  1      2     0.50                        cmp	x20, x21, asr #55
 # CHECK-NEXT:  1      2     0.50                        cmp	x22, x23, asr #63
-# CHECK-NEXT:  1      1     0.25                        cmp	wzr, w0
-# CHECK-NEXT:  1      1     0.25                        cmp	xzr, x0
+# CHECK-NEXT:  1      1     0.33                        cmp	wzr, w0
+# CHECK-NEXT:  1      1     0.33                        cmp	xzr, x0
 # CHECK-NEXT:  1      1     0.17                        adc	w29, w27, w25
 # CHECK-NEXT:  1      1     0.17                        adc	wzr, w3, w4
 # CHECK-NEXT:  1      1     0.17                        adc	w9, wzr, w10
@@ -1576,14 +1576,14 @@ drps
 # CHECK-NEXT:  1      1     0.17                        adc	xzr, x3, x4
 # CHECK-NEXT:  1      1     0.17                        adc	x9, xzr, x10
 # CHECK-NEXT:  1      1     0.17                        adc	x20, x0, xzr
-# CHECK-NEXT:  1      1     0.25                        adcs	w29, w27, w25
-# CHECK-NEXT:  1      1     0.25                        adcs	wzr, w3, w4
-# CHECK-NEXT:  1      1     0.25                        adcs	w9, wzr, w10
-# CHECK-NEXT:  1      1     0.25                        adcs	w20, w0, wzr
-# CHECK-NEXT:  1      1     0.25                        adcs	x29, x27, x25
-# CHECK-NEXT:  1      1     0.25                        adcs	xzr, x3, x4
-# CHECK-NEXT:  1      1     0.25                        adcs	x9, xzr, x10
-# CHECK-NEXT:  1      1     0.25                        adcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        adcs	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        adcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        adcs	w9, wzr, w10
+# CHECK-NEXT:  1      1     0.33                        adcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        adcs	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        adcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        adcs	x9, xzr, x10
+# CHECK-NEXT:  1      1     0.33                        adcs	x20, x0, xzr
 # CHECK-NEXT:  1      1     0.17                        sbc	w29, w27, w25
 # CHECK-NEXT:  1      1     0.17                        sbc	wzr, w3, w4
 # CHECK-NEXT:  1      1     0.17                        ngc	w9, w10
@@ -1592,26 +1592,26 @@ drps
 # CHECK-NEXT:  1      1     0.17                        sbc	xzr, x3, x4
 # CHECK-NEXT:  1      1     0.17                        ngc	x9, x10
 # CHECK-NEXT:  1      1     0.17                        sbc	x20, x0, xzr
-# CHECK-NEXT:  1      1     0.25                        sbcs	w29, w27, w25
-# CHECK-NEXT:  1      1     0.25                        sbcs	wzr, w3, w4
-# CHECK-NEXT:  1      1     0.25                        ngcs	w9, w10
-# CHECK-NEXT:  1      1     0.25                        sbcs	w20, w0, wzr
-# CHECK-NEXT:  1      1     0.25                        sbcs	x29, x27, x25
-# CHECK-NEXT:  1      1     0.25                        sbcs	xzr, x3, x4
-# CHECK-NEXT:  1      1     0.25                        ngcs	x9, x10
-# CHECK-NEXT:  1      1     0.25                        sbcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        sbcs	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        sbcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        ngcs	w9, w10
+# CHECK-NEXT:  1      1     0.33                        sbcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        sbcs	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        sbcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        ngcs	x9, x10
+# CHECK-NEXT:  1      1     0.33                        sbcs	x20, x0, xzr
 # CHECK-NEXT:  1      1     0.17                        ngc	w3, w12
 # CHECK-NEXT:  1      1     0.17                        ngc	wzr, w9
 # CHECK-NEXT:  1      1     0.17                        ngc	w23, wzr
 # CHECK-NEXT:  1      1     0.17                        ngc	x29, x30
 # CHECK-NEXT:  1      1     0.17                        ngc	xzr, x0
 # CHECK-NEXT:  1      1     0.17                        ngc	x0, xzr
-# CHECK-NEXT:  1      1     0.25                        ngcs	w3, w12
-# CHECK-NEXT:  1      1     0.25                        ngcs	wzr, w9
-# CHECK-NEXT:  1      1     0.25                        ngcs	w23, wzr
-# CHECK-NEXT:  1      1     0.25                        ngcs	x29, x30
-# CHECK-NEXT:  1      1     0.25                        ngcs	xzr, x0
-# CHECK-NEXT:  1      1     0.25                        ngcs	x0, xzr
+# CHECK-NEXT:  1      1     0.33                        ngcs	w3, w12
+# CHECK-NEXT:  1      1     0.33                        ngcs	wzr, w9
+# CHECK-NEXT:  1      1     0.33                        ngcs	w23, wzr
+# CHECK-NEXT:  1      1     0.33                        ngcs	x29, x30
+# CHECK-NEXT:  1      1     0.33                        ngcs	xzr, x0
+# CHECK-NEXT:  1      1     0.33                        ngcs	x0, xzr
 # CHECK-NEXT:  1      1     0.17                        sbfx	x1, x2, #3, #2
 # CHECK-NEXT:  1      1     0.17                        asr	x3, x4, #63
 # CHECK-NEXT:  1      1     0.17                        asr	wzr, wzr, #31
@@ -1701,30 +1701,30 @@ drps
 # CHECK-NEXT:  1      1     0.50                        b.ne	#4
 # CHECK-NEXT:  1      1     0.50                        b.ge	#1048572
 # CHECK-NEXT:  1      1     0.50                        b.ge	#-4
-# CHECK-NEXT:  1      1     0.25                        ccmp	w1, #31, #0, eq
-# CHECK-NEXT:  1      1     0.25                        ccmp	w3, #0, #15, hs
-# CHECK-NEXT:  1      1     0.25                        ccmp	wzr, #15, #13, hs
-# CHECK-NEXT:  1      1     0.25                        ccmp	x9, #31, #0, le
-# CHECK-NEXT:  1      1     0.25                        ccmp	x3, #0, #15, gt
-# CHECK-NEXT:  1      1     0.25                        ccmp	xzr, #5, #7, ne
-# CHECK-NEXT:  1      1     0.25                        ccmn	w1, #31, #0, eq
-# CHECK-NEXT:  1      1     0.25                        ccmn	w3, #0, #15, hs
-# CHECK-NEXT:  1      1     0.25                        ccmn	wzr, #15, #13, hs
-# CHECK-NEXT:  1      1     0.25                        ccmn	x9, #31, #0, le
-# CHECK-NEXT:  1      1     0.25                        ccmn	x3, #0, #15, gt
-# CHECK-NEXT:  1      1     0.25                        ccmn	xzr, #5, #7, ne
-# CHECK-NEXT:  1      1     0.25                        ccmp	w1, wzr, #0, eq
-# CHECK-NEXT:  1      1     0.25                        ccmp	w3, w0, #15, hs
-# CHECK-NEXT:  1      1     0.25                        ccmp	wzr, w15, #13, hs
-# CHECK-NEXT:  1      1     0.25                        ccmp	x9, xzr, #0, le
-# CHECK-NEXT:  1      1     0.25                        ccmp	x3, x0, #15, gt
-# CHECK-NEXT:  1      1     0.25                        ccmp	xzr, x5, #7, ne
-# CHECK-NEXT:  1      1     0.25                        ccmn	w1, wzr, #0, eq
-# CHECK-NEXT:  1      1     0.25                        ccmn	w3, w0, #15, hs
-# CHECK-NEXT:  1      1     0.25                        ccmn	wzr, w15, #13, hs
-# CHECK-NEXT:  1      1     0.25                        ccmn	x9, xzr, #0, le
-# CHECK-NEXT:  1      1     0.25                        ccmn	x3, x0, #15, gt
-# CHECK-NEXT:  1      1     0.25                        ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	x9, #31, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	x9, #31, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmn	xzr, x5, #7, ne
 # CHECK-NEXT:  1      1     0.17                        csel	w1, w0, w19, ne
 # CHECK-NEXT:  1      1     0.17                        csel	wzr, w5, w9, eq
 # CHECK-NEXT:  1      1     0.17                        csel	w9, wzr, w30, gt
@@ -2510,8 +2510,8 @@ drps
 # CHECK-NEXT:  3      6     0.67    *                   ldnp	q23, q29, [x1, #-1024]
 # CHECK-NEXT:  1      1     0.17                        mov	w3, #983055
 # CHECK-NEXT:  1      1     0.17                        mov	x10, #-6148914691236517206
-# CHECK-NEXT:  1      1     0.25                        ands	w4, w4, #0xf000f
-# CHECK-NEXT:  1      1     0.25                        ands	x11, x11, #0xaaaaaaaaaaaaaaaa
+# CHECK-NEXT:  1      1     0.33                        ands	w4, w4, #0xf000f
+# CHECK-NEXT:  1      1     0.33                        ands	x11, x11, #0xaaaaaaaaaaaaaaaa
 # CHECK-NEXT:  1      1     0.17                        and	w12, w23, w21
 # CHECK-NEXT:  1      1     0.17                        and	w16, w15, w1, lsl #1
 # CHECK-NEXT:  1      1     0.17                        and	w9, w4, w10, lsl #31
@@ -2532,7 +2532,7 @@ drps
 # CHECK-NEXT:  1      1     0.17                        orn	w2, w5, w29
 # CHECK-NEXT:  1      2     0.50                        ands	w7, wzr, w9, lsl #1
 # CHECK-NEXT:  1      2     0.50                        ands	x3, x5, x20, ror #63
-# CHECK-NEXT:  1      1     0.25                        bics	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        bics	w3, w5, w7
 # CHECK-NEXT:  1      2     0.50                        bics	x3, xzr, x3, lsl #1
 # CHECK-NEXT:  1      2     0.50                        tst	w3, w7, lsl #31
 # CHECK-NEXT:  1      2     0.50                        tst	x2, x20, asr #2
@@ -2573,1199 +2573,1202 @@ drps
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT: 11.00  11.00  33.00  33.00  99.00  165.00 165.00 329.92 184.92 112.92 112.92 86.17  86.17  190.00 146.00 30.00  10.00
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT: 11.00  11.00  33.00  33.00  54.00  54.00  54.00  99.00  165.00 165.00 329.92 184.92 112.92 112.92 86.17  86.17  190.00 146.00 30.00  10.00
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w2, w3, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w30, w29, #1, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w13, w5, #4095, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x5, x7, #1638
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w20, wsp, #801
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wsp, wsp, #1104
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wsp, w30, #4084
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x0, x24, #291
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x3, x24, #4095, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x8, sp, #1074
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	sp, x29, #3816
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w0, wsp, #4077
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w4, w20, #546, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	sp, sp, #288
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	wsp, w19, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w13, w23, #291, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w2, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w20, wsp, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x3, #1, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	sp, #20, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x30, #4095
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x4, sp, #3822
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w3, #291, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	wsp, #1365
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	sp, #1092, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	sp, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	wsp, w20
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x11, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w24, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wzr, w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w20, wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w9, w3, wzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	xzr, x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x20, xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x17, x29, x20, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w20, wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w9, w3, wzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x20, xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x17, x29, x20, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	wzr, w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w9, w3, wzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	xzr, x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x17, x29, x20, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w4, w6, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w11, w13, w15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w9, w3, wzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w2, w3, w4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w5, w6, w7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w8, w9, w10, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x3, x5, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x3, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x4, x6, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x11, x13, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x17, x29, x20, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x2, x3, x4, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x5, x6, x7, asr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x8, x9, x10, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w5, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w6, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w8, w9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w10, w11, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w12, w13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w14, w15, lsr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w16, w17, lsr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w18, w19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w20, w21, asr #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w22, w23, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x0, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x5, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x6, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x8, x9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x10, x11, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x12, x13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x14, x15, lsr #41
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x16, x17, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x18, x19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x20, x21, asr #55
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x22, x23, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w0, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	wzr, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w5, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w6, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w8, w9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w10, w11, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w12, w13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w14, w15, lsr #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w18, w19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w20, w21, asr #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w22, w23, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x0, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	xzr, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x5, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x6, x7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x8, x9, lsl #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x10, x11, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x12, x13, lsr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x14, x15, lsr #41
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x16, x17, lsr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x18, x19, asr #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x20, x21, asr #55
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x22, x23, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	wzr, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w9, wzr, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x9, xzr, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w9, wzr, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x9, xzr, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x9, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	w29, w27, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	wzr, w3, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	w20, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	x29, x27, x25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	xzr, x3, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x9, x10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	x20, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w3, w12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	wzr, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w23, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x29, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w3, w12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	wzr, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w23, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x29, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	x1, x2, #3, #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x3, x4, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	wzr, wzr, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w12, w9, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	x4, x5, #52, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	xzr, x4, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	x4, xzr, #1, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x5, x6, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x4, x5, #52, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfc	x4, #1, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x5, x6, #12, #52
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtb	w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtb	xzr, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxth	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxth	x0, w1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtw	x3, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     uxtb	w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     uxth	w9, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	x3, x30, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w1, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	wzr, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w3, w2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w9, w10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x20, x21, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w1, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfc	xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w11, w12, #31, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w13, w14, #29, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	w9, w10, #0, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x19, x20, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w11, w12, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w13, w14, #29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	xzr, xzr, #10, #11
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	w5, #4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	x5, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ne	#4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#-4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w1, #31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w3, #0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	wzr, #15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x9, #31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x3, #0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	xzr, #5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w1, #31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w3, #0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	wzr, #15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x9, #31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x3, #0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	xzr, #5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w1, wzr, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w3, w0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	wzr, w15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x9, xzr, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x3, x0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	xzr, x5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w1, wzr, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w3, w0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	wzr, w15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x9, xzr, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x3, x0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	xzr, x5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w1, w0, w19, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	wzr, w5, w9, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w9, wzr, w30, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w1, w28, wzr, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x19, x23, x29, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	xzr, x3, x4, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x5, xzr, x6, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x7, x8, xzr, lo
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	w3, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	x9, pl
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	w20, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	x30, ge
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w2, wzr, wzr, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x3, xzr, xzr, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	w9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	x9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w5, w6, w6, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x1, x2, x2, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	w9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	x9, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x1, x0, x0, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, w8, w8, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	w3, w5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	wzr, w4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	w9, wzr, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	x3, x5, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	xzr, x4, le
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	x9, xzr, lt
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x4, x8, x8, al
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, w8, w8, nv
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rbit	w0, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rbit	x18, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev16	w17, w1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev16	x5, x2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev	w18, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev32	x20, x1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev	x22, x2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     clz	w24, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     clz	x26, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cls	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cls	x20, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     12.00   -      -      -      -      -      -      -      -      -     udiv	w0, w7, w10
-# CHECK-NEXT:  -      -      -      -      -      -      -     20.00   -      -      -      -      -      -      -      -      -     udiv	x9, x22, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     12.00   -      -      -      -      -      -      -      -      -     sdiv	w12, w21, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     20.00   -      -      -      -      -      -      -      -      -     sdiv	x13, x2, x1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w11, w12, w13
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x14, x15, x16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w17, w18, w19
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x20, x21, x22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w23, w24, w25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x26, x27, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w0, w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x3, x4, x5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w6, w7, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x9, x10, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w12, w13, w14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x15, x16, x17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w18, w19, w20
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x21, x22, x23
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w24, w25, w26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x27, x28, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	xzr, x27, x26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x23, x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w1, w3, w7, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	wzr, w0, w9, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w13, wzr, w4, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w19, w30, wzr, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	w4, w5, w6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x1, x3, x7, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	xzr, x0, x9, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x13, xzr, x4, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x19, x30, xzr, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	x4, x5, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w1, w3, w7, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	wzr, w0, w9, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w13, wzr, w4, w4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w19, w30, wzr, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	w4, w5, w6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x1, x3, x7, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	xzr, x0, x9, x11
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x13, xzr, x4, x4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x19, x30, xzr, x29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	x4, x5, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smull	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smnegl	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	xzr, w10, w11, x12
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x13, wzr, w14, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umull	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umsubl	x3, w5, w2, x9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umnegl	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x30, x29, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x23, x22, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x23, x22, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	x19, x20, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	w21, w22, w23
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smull	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umull	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smnegl	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umnegl	x11, w13, w17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	w3, w5, w7, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	w11, w13, w17, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	x3, x5, x7, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	x11, x13, x17, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x19, x23, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x29, xzr, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w9, w13, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s3, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s29, s30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d4, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d26, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s1, s31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s3, s0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s31, s15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d9, d31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d3, d0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d31, d5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s1, s31, #0, eq
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s3, s0, #15, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s31, s15, #13, hs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d9, d31, #0, le
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d3, d0, #15, gt
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d31, d5, #7, ne
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcsel	s3, s20, s9, pl
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcsel	d9, d10, d11, mi
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s0, s1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	s2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	s4, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fsqrt	s6, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, s11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	s14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	s16, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	s18, s19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	s20, s21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	s22, s23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	s24, s25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d0, d1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	d2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	d4, d5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -     4.00    -     fsqrt	d6, d7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, d11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	d12, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	d14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	d16, d17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	d18, d19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	d20, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	d22, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	d24, d25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s26, h27
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d28, h29
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s20, s19, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fdiv	s1, s2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	s4, s5, s6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	s7, s8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	s13, s14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	s16, s17, s18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	s19, s20, s21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	s22, s23, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d20, d19, d17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -     4.00    -     fdiv	d1, d2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d4, d5, d6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	d7, d8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	d10, d11, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	d13, d14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	d16, d17, d18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	d19, d20, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	d22, d23, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	s3, s5, s6, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	d3, d13, d0, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, h20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, h0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, h30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, h0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, s20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, s0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, s30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, s0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, d20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, d0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, d30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, d0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, h20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, h0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, h5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, h30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, h0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, s20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, s0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, s5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, s30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, s0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, d20, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, d0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, d5, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, d30, #45
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, d0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, w19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d31, wzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d14, w0, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, x19, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d31, xzr, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d14, x0, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, h9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, h20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, h23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, h3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, h3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, h5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, h7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, h9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, h11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, h16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, h26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, h28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, h30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, s20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, s23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, s3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, s5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, s11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, s26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, s28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, s30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, d20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, d23
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, d3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, d5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, d7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, d9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, d11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, d15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d17, w18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d19, x20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d21, w22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, x24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, d26
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, d28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, d30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	w3, s9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     fmov	s9, w3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	x20, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     fmov	d1, x15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	x3, v12.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v1.d[1], x19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s2, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s3, #1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d30, #16.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s4, #1.06250000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d10, #1.93750000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s12, #-1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d16, #8.50000000
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	w3, #0
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	x29, #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldrsw	xzr, #-4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	s0, #8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	d0, #1048572
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	q0, #-1048576
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     prfm	pldl1strm, #0
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     prfm	#22, #0
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxrb	w18, w8, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxrh	w24, w15, [x16]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxr	w5, w6, [x17]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxr	w1, x10, [x21]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxrb	w30, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxrh	w17, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	w22, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxr	w9, x3, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxrb	w8, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxrh	w7, [x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	w6, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlrb	w24, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlrh	w25, [x30]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	w26, [x29]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldarb	w23, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldarh	w22, [x30]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	wzr, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sturb	w9, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stur	w16, [x0, #-256]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stur	x28, [x14, #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldurb	w1, [x20, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldurh	w20, [x1, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	w12, [sp, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	xzr, [x12, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursb	x9, [x7, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursh	x17, [x19, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursw	x20, [x15, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursb	w19, [x1, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursh	w15, [x21, #-256]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	b0, [sp, #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	h12, [x12, #-1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	s15, [x0, #255]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	d31, [x5, #25]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	q9, [x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	b3, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	h5, [x4, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	s7, [x12, #-1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	d11, [x19, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	q13, [x1, #2]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w9, [x2], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w10, [x3], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w19, [sp], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w20, [x30], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w21, [x12], #-256
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	xzr, [x9], #255
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x2, [x3], #1
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w9, [x2], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w10, [x3], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w19, [sp], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w20, [x30], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w21, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	xzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	wzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	wzr, [x9], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w2, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w19, [x12], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b0, [x0], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b3, [x3], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b5, [sp], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h10, [x10], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h13, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h15, [sp], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s20, [x20], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s23, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d20, [x20], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d23, [x23], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b0, [x0], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b3, [x3], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b5, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h10, [x10], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h13, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h15, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s20, [x20], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s23, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d20, [x20], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d23, [x23], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d25, [x0], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q20, [x1], #255
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q23, [x9], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q25, [x20], #-256
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q10, [x1], #255
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q22, [sp], #1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q21, [x20], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x3, [x4, #0]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w9, [x2, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w19, [sp, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w20, [x30, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w21, [x12, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x2, [x3, #1]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w9, [x2, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w10, [x3, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w19, [sp, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w20, [x30, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w21, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	xzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	wzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	wzr, [x9, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w2, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w19, [x12, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b0, [x0, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b3, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b5, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h10, [x10, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h13, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h15, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b0, [x0, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b3, [x3, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b5, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h10, [x10, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h13, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h15, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d20, [x20, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d23, [x23, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d25, [x0, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q20, [x1, #255]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q23, [x9, #1]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q25, [x20, #-256]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q10, [x1, #255]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q22, [sp, #1]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q21, [x20, #-256]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttrb	w9, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttrh	wzr, [x12, #255]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttr	w16, [x0, #-256]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttr	x28, [x14, #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrb	w1, [x20, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrh	w20, [x1, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtr	w12, [sp, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtr	xzr, [x12, #255]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsb	x9, [x7, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsh	x17, [x19, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsw	x20, [x15, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsb	w19, [x1, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsh	w15, [x21, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x4, [x29]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x30, [x12, #32760]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x20, [sp, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	xzr, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w2, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w17, [sp, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w13, [x2, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w2, [x4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w12, [x2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	xzr, [x15]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x30, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w20, [x4, #16380]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w17, [sp, #8190]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	w23, [x3, #4095]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	wzr, [x2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	b31, [sp, #4095]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h20, [x2, #8190]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q12, [sp, #65520]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h10, [x30, x7, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     str	h13, [x27, x5, sxtx #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h16, [x24, w8, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h19, [x21, wzr, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	s9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	s12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x3, [sp, x5]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	d19, [x21, wzr, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q3, [sp, x5]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q9, [x27, x6]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q10, [x30, x7, lsl #4]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q11, [x29, x3, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q12, [x28, xzr, sxtx]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     str	q13, [x27, x5, sxtx #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q14, [x26, w6, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q15, [x25, w7, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q17, [x23, w9, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q18, [x22, w10, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w3, w5, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x9, x10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x9, x10, [x2, #-256]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x20, x30, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	s27, s26, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	d3, d5, [x9, #504]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	d7, d11, [x10, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stp	q3, q5, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stp	q17, q19, [sp, #1008]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w3, w5, [sp], #0
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     stp	wzr, w9, [sp], #252
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w2, wzr, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w9, w10, [sp], #4
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [sp], #4
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [x2], #-256
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x20, x30, [sp], #252
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x21, x29, [x2], #504
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x22, x23, [x3], #-512
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x24, x25, [x4], #8
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s29, s28, [sp], #252
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	s27, s26, [sp], #-256
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s1, s2, [x3], #44
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d3, d5, [x9], #504
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d7, d11, [x10], #-512
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	d2, d3, [x30], #-8
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     stp	q3, q5, [sp], #0
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     stp	q17, q19, [sp], #1008
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -     ldp	q23, q29, [x1], #-1024
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w3, w5, [sp, #0]!
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     stp	wzr, w9, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w2, wzr, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w9, w10, [sp, #4]!
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [sp, #4]!
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x20, x30, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x21, x29, [x2, #504]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x22, x23, [x3, #-512]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x24, x25, [x4, #8]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s29, s28, [sp, #252]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	s27, s26, [sp, #-256]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s1, s2, [x3, #44]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d3, d5, [x9, #504]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d7, d11, [x10, #-512]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	d2, d3, [x30, #-8]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   1.00   1.00    -      -     stp	q3, q5, [sp, #0]!
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   1.00   1.00    -      -     stp	q17, q19, [sp, #1008]!
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -     ldp	q23, q29, [x1, #-1024]!
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w3, w5, [sp]
-# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stnp	wzr, w9, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w9, w10, [sp, #4]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x21, x29, [x2, #504]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x24, x25, [x4, #8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	s29, s28, [sp, #252]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	s27, s26, [sp, #-256]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	s1, s2, [x3, #44]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	d3, d5, [x9, #504]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	d7, d11, [x10, #-512]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stnp	q3, q5, [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stnp	q17, q19, [sp, #1008]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w3, #983055
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x10, #-6148914691236517206
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ands	w4, w4, #0xf000f
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ands	x11, x11, #0xaaaaaaaaaaaaaaaa
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w12, w23, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w16, w15, w1, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w9, w4, w10, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w30, w11
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x3, x5, x7, lsl #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x5, x14, x19, asr #4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w17, w19, ror #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w0, w2, wzr, lsr #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w30, w11, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	xzr, x4, x26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, wzr, w20, ror #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x7, x20, xzr, asr #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     bic	x13, x20, x14, lsl #47
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     bic	w2, w7, w9
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orr	w2, w7, w0, asr #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orr	x8, x9, x10, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orn	x3, x5, x7, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orn	w2, w5, w29
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ands	x3, x5, x20, ror #63
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     bics	w3, w5, w7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     tst	w3, w7, lsl #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     tst	x2, x20, asr #2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x3, x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x3, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	wzr, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w3, w5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	w2, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w2, #-1235
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x2, #5299989643264
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x2, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	w3, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x4, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	w5, #0, lsl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x6, #0, lsl #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	x7, #0, lsl #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x8, #0, lsl #48
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	x9, #0, lsl #48
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adr	x2, #1600
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adrp	x21, #6553600
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adr	x0, #262144
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#-4
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#134217724
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     br	x20
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     blr	xzr
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret	x10
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     eret
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     drps
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w2, w3, #4095
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w30, w29, #1, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w13, w5, #4095, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x5, x7, #1638
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w20, wsp, #801
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wsp, wsp, #1104
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wsp, w30, #4084
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x0, x24, #291
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x3, x24, #4095, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x8, sp, #1074
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	sp, x29, #3816
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w0, wsp, #4077
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w4, w20, #546, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	sp, sp, #288
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	wsp, w19, #16
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w2, #4095
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w20, wsp, #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	sp, #20, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x30, #4095
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x4, sp, #3822
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w3, #291, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	wsp, #1365
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	sp, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	wsp, w20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x11, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w24, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	wzr, w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w20, wzr, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	xzr, x3, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x20, xzr, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     add	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x17, x29, x20, lsl #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     add	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w3, w5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w20, wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x3, x5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x20, xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adds	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x17, x29, x20, lsl #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     adds	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	wzr, w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	xzr, x3, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sub	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x17, x29, x20, lsl #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w3, w5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w4, w6, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	w11, w13, w15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w2, w3, w4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w5, w6, w7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	w8, w9, w10, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x3, x5, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x3, x5
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x4, x6, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     subs	x11, x13, x15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x17, x29, x20, lsl #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x2, x3, x4, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x5, x6, x7, asr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     subs	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w5, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	w6, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w8, w9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w10, w11, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w12, w13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w14, w15, lsr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w16, w17, lsr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w18, w19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w20, w21, asr #22
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	w22, w23, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x0, x3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x5, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmn	x6, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x8, x9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x10, x11, lsl #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x12, x13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x14, x15, lsr #41
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x16, x17, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x18, x19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x20, x21, asr #55
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmn	x22, x23, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w0, w3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	wzr, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w5, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	w6, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w8, w9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w10, w11, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w12, w13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w14, w15, lsr #21
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w18, w19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w20, w21, asr #22
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x0, x3
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	xzr, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x5, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	x6, x7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x8, x9, lsl #15
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x10, x11, lsl #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x12, x13, lsr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x14, x15, lsr #41
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x16, x17, lsr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x18, x19, asr #0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x20, x21, asr #55
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cmp	x22, x23, asr #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	wzr, w0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w9, wzr, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x9, xzr, x10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     adc	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w9, wzr, w10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x9, xzr, x10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adcs	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x9, x10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbc	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	w29, w27, w25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	wzr, w3, w4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w9, w10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	w20, w0, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	x29, x27, x25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	xzr, x3, x4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x9, x10
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     sbcs	x20, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w3, w12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	wzr, w9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	w23, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x29, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ngc	x0, xzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w3, w12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	wzr, w9
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	w23, wzr
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x29, x30
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	xzr, x0
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ngcs	x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	x1, x2, #3, #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x3, x4, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	wzr, wzr, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w12, w9, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	x4, x5, #52, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	xzr, x4, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x5, x6, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x4, x5, #52, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfc	x4, #1, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x5, x6, #12, #52
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtb	w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtb	xzr, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxth	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxth	x0, w1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sxtw	x3, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     uxtb	w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     uxth	w9, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	x3, x30, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w1, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	wzr, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w9, w10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x20, x21, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w1, wzr, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfi	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfc	xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x2, x3, #63, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	x9, x10, #5, #59
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w11, w12, #31, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	w13, w14, #29, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x2, x3, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x9, x10, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w11, w12, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w13, w14, #29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	w5, #4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	x5, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ne	#4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#-4
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x9, #31, #0, le
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x9, #31, #0, le
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csel	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w1, w0, w19, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	wzr, w5, w9, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w9, wzr, w30, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	w1, w28, wzr, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x19, x23, x29, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	xzr, x3, x4, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x5, xzr, x6, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x7, x8, xzr, lo
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	w3, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	x9, pl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	w20, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	x30, ge
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w2, wzr, wzr, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x3, xzr, xzr, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	w9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinc	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cset	x9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	w5, w6, w6, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinc	x1, x2, x2, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	w9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cinv	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csetm	x9, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	x1, x0, x0, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	w3, w5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	wzr, w4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	w9, wzr, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	x3, x5, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	xzr, x4, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cneg	x9, xzr, lt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csneg	x4, x8, x8, al
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rbit	w0, w7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rbit	x18, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev16	w17, w1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev16	x5, x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev	w18, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev32	x20, x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     rev	x22, x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     clz	w24, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     clz	x26, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cls	w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     cls	x20, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     12.00   -      -      -      -      -      -      -      -      -     udiv	w0, w7, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     20.00   -      -      -      -      -      -      -      -      -     udiv	x9, x22, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     12.00   -      -      -      -      -      -      -      -      -     sdiv	w12, w21, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     20.00   -      -      -      -      -      -      -      -      -     sdiv	x13, x2, x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w11, w12, w13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x14, x15, x16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w17, w18, w19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x20, x21, x22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w23, w24, w25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x26, x27, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w0, w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x3, x4, x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	w6, w7, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsl	x9, x10, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	w12, w13, w14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     lsr	x15, x16, x17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	w18, w19, w20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     asr	x21, x22, x23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w24, w25, w26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	xzr, x27, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x23, x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     madd	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     msub	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mul	x19, x20, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     mneg	w21, w22, w23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     smnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     umnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	w3, w5, w7, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	w11, w13, w17, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	x3, x5, x7, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.67   0.67   0.17   0.17   0.17   0.17    -      -      -      -     extr	x11, x13, x17, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x19, x23, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	x29, xzr, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s3, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d4, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d26, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s0, s1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	s4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fsqrt	s6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	s22, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d0, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -     4.00    -     fsqrt	d6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	d12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	d16, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	d18, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	d22, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	d24, d25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s26, h27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d28, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	s7, s8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	s16, s17, s18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	s22, s23, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d20, d19, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -     4.00    -     fdiv	d1, d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	d4, d5, d6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	d7, d8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	d10, d11, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	d16, d17, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, h20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, h23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, h7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, h11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, h16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	h21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	h23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, h26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, h28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, h30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	s21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	s23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, s26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, s28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	w3, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtns	xzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	wzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtnu	x0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	wzr, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtps	x12, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	w30, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtpu	x29, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	w2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtms	x4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	w6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtmu	x8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	w10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzs	x12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	w14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtzu	x15, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     scvtf	d19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     ucvtf	d23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	w25, d26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtas	x27, d28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	w29, d30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	w3, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	x20, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     fmov	d1, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     fmov	x3, v12.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s2, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s3, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d30, #16.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s4, #1.06250000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d10, #1.93750000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s12, #-1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d16, #8.50000000
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	w3, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	x29, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldrsw	xzr, #-4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	s0, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	d0, #1048572
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     ldr	q0, #-1048576
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     prfm	pldl1strm, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -      -      -     prfm	#22, #0
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxr	w5, w6, [x17]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxr	w1, x10, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxrb	w30, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxrh	w17, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	w22, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxrb	w8, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxrh	w7, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlrb	w24, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlrh	w25, [x30]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	w26, [x29]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldarb	w23, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldarh	w22, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	wzr, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sturb	w9, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stur	w16, [x0, #-256]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stur	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldurb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldurh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	b0, [sp, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	h12, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	s15, [x0, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	d31, [x5, #25]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stur	q9, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	b3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	h5, [x4, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	s7, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	d11, [x19, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldur	q13, [x1, #2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w9, [x2], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w10, [x3], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w19, [sp], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w20, [x30], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w21, [x12], #-256
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	xzr, [x9], #255
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x2, [x3], #1
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q20, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q23, [x9], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q25, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q10, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q22, [sp], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q21, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x3, [x4, #0]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     strh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w19, [sp, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w20, [x30, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x2, [x3, #1]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     str	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q20, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q23, [x9, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q10, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q22, [sp, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     str	q21, [x20, #-256]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttrb	w9, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttr	w16, [x0, #-256]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     sttr	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtr	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x4, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x30, [x12, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x20, [sp, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	xzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w2, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w17, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w13, [x2, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w2, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w12, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x30, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w17, [sp, #8190]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	w23, [x3, #4095]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	wzr, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	b31, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h20, [x2, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     str	h13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	h17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	h18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	h19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	s9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x3, [sp, x5]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w3, w5, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d3, d5, [x9], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     stp	q3, q5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -     ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -     stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -     ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   1.00   1.00    -      -     stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33   1.00   1.00    -      -     stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -     ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w3, w5, [sp]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stnp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w3, #983055
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x10, #-6148914691236517206
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ands	w4, w4, #0xf000f
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     ands	x11, x11, #0xaaaaaaaaaaaaaaaa
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w12, w23, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w16, w15, w1, lsl #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w9, w4, w10, lsl #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w30, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x3, x5, x7, lsl #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x5, x14, x19, asr #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w17, w19, ror #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w0, w2, wzr, lsr #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, w30, w11, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	xzr, x4, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	w3, wzr, w20, ror #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     and	x7, x20, xzr, asr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     bic	x13, x20, x14, lsl #47
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orr	w2, w7, w0, asr #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orr	x8, x9, x10, lsl #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orn	x3, x5, x7, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     orn	w2, w5, w29
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ands	x3, x5, x20, ror #63
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     bics	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     tst	w3, w7, lsl #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     tst	x2, x20, asr #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x3, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x3, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	wzr, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w3, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	w2, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	w2, #-1235
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     mov	x2, #5299989643264
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x2, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	w3, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x4, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	w5, #0, lsl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x6, #0, lsl #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	x7, #0, lsl #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movz	x8, #0, lsl #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adr	x2, #1600
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adrp	x21, #6553600
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     adr	x0, #262144
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#-4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     br	x20
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     blr	xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret	x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     eret
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     drps
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
index fd2083dc1277a..fbf65e26e99a5 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
@@ -80,28 +80,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	b0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	b0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -156,28 +159,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	h0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	h0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -232,28 +238,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	s0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	s0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -308,28 +317,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	d0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	d0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -384,28 +396,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	q0, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ldr	q0, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -460,28 +475,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.8b }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.8b }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -536,28 +554,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.4h }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.4h }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -612,28 +633,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -688,28 +712,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -     0.25   0.25   0.25   0.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.1d }, [sp]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.34    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -764,28 +791,31 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.33   1.00   0.33   0.34
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.33   1.00   0.33   0.34
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.s, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.33    -     0.33   0.34   add	z0.s, z0.s, z0.s
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.33    -     0.33   0.34   add	z0.s, z0.s, z0.s
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          01234
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-neon-instructions.s
index a174a1a207eba..68a067eb8a360 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-neon-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-neon-instructions.s
@@ -2473,1251 +2473,1254 @@ zip2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     27.33  48.83  48.83  16.17  5.17   5.17   5.17   5.17   5.17   503.25 376.25 451.75 324.75
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     27.33  48.83  48.83  16.17  5.17   5.17   5.17   5.17   5.17   503.25 376.25 451.75 324.75
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	d29, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	d17, d31, d29
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   addv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	h0, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     bfcvtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     bfcvtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.4h, #15, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.8b, v0.8b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.2s, v0.2s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v0.2d, v0.2d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	d20, d21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v0.8h, v0.8h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.16b, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.2d, x28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.2s, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.4h, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.4s, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.8b, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.8h, w28
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d0, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.16b, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2d, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2s, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4s, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8b, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.16b, v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.8b, v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	d29, d24, d20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	s29, s24, s20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.2s, v0.2s, v0.2s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.4s, v0.4s, v0.4s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.2d, v0.2d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.2s, v0.2s, v0.2s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.4s, v0.4s, v0.s[1], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v0.2d, v0.2d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	d20, d21, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	s10, s11, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v0.4s, v0.4s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtas	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtas	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtau	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtau	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl	v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl	v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl2	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl2	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtms	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtms	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtmu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtmu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtns	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtns	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtnu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtnu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtps	h22, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtps	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtpu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtpu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	s22, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s21, s12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	h21, h14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	h21, h12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s12, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s21, s12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	h12, h13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	h21, h12, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdiv	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fdiv	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00    -     5.00    -     fdiv	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     6.50    -     6.50    -     fdiv	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	d0, d1, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	s0, s1, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	d0, d4, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	s3, s5, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.2h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2d, #-1.25000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2s, #13.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.4s, #1.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d0, d1, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s0, s1, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d0, d4, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d23, d11, d1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s20, s22, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s3, s5, v0.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	d13, d13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	s19, s14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecpe	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpe	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	d22, d30, d21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	s21, s16, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	d16, d19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	s18, s10
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32x	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32x	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint32x	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32z	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32z	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint32z	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64x	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64x	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint64x	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64z	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64z	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint64z	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinta	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinti	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintm	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintn	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintp	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintx	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintz	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	d21, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrte	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frsqrte	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	d8, d22, d18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	s21, s5, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     7.50    -     7.50    -     fsqrt	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fsqrt	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fsqrt	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     6.50    -     6.50    -     fsqrt	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.4s, v1.4s }, [sp], #32
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.b }[9], [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1	{ v0.b }[9], [x0], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0], #1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x15], #2
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2	{ v0.16b, v1.16b }, [x0], x1
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15], #4
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.2d, v1.2d }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2r	{ v0.2d, v1.2d }, [x0], #16
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.4s, v1.4s }, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2r	{ v0.4s, v1.4s }, [sp], #8
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -     2.00   2.00   2.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h2, v0.h[5]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s17, v0.s[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.b[0], v0.b[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.h[1], v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.s[2], v0.s[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.d[1], v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.b[0], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.h[1], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.s[2], w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.d[1], x8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	d15, #0xff00ff00ff00ff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.16b, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2d, #0xff0000ff0000ffff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v0.8h, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v21.8b, v1.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v30.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v0.4h, v9.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v21.8b, v1.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v30.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v4.8h, v7.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v0.16b, v31.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v1.8b, v9.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v13.4h, v21.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v2.8h, v4.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v4.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v6.4s, v8.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.2s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.4h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.8b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.16b, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.4s, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.8h, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	d0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	s0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	s0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	h0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   saddlv	h0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     scvtf	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	d7, d10, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	d10, d14, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   smaxv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   sminv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	d18, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	h21, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	s20, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d8, s9, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s20, s21, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d1, s1, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d15, s22, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s1, h1, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s12, h22, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	d18, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	h21, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	s20, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4s, v1.4s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	h0, h1, h2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4h, v1.4h, v2.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.8h, v1.8h, v2.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	s0, s1, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.2s, v1.2s, v2.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4s, v1.4s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	h0, h1, h2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4h, v1.4h, v2.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.8h, v1.8h, v2.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	s0, s1, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.2s, v1.2s, v2.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s20, s21, s2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	h3, h4, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	b10, h13, #2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	h15, s10, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	s15, d12, #9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	b17, h10, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	h10, s13, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	s22, d16, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	b11, b19, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d15, d16, #51
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h13, h18, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h3, h4, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s14, s17, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	b15, b18, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	d11, d13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	h19, h17, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	s16, s14, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	b10, h15, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	h17, s10, #4
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	s18, d10, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	b15, h10, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	h20, s14, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	s10, d15, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	s20, s10, s7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	b18, h18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	h20, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	s19, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	b19, h14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	h21, s15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	s20, d12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	d10, d12, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	d16, d16, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	d19, d18, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	d15, d11, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	d31, d31, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v0.2d, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v0.4s, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	d15, d16, #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	d18, d12, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50   0.17   0.17   0.17   0.17   0.17   0.17   1.50   1.50    -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.4s, v1.4s }, [sp], #32
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -      -      -     1.50   1.50    -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     st1	{ v0.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1	{ v0.8h, v1.8h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st1	{ v0.d }[1], [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.d }[1], [x0], #8
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st2	{ v0.16b, v1.16b }, [x0], x1
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st2	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st2	{ v0.s, v1.s }[3], [sp]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st2	{ v0.s, v1.s }[3], [sp], #8
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     2.00   2.00    -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50   0.17   0.17   0.17   0.17   0.17   0.17   3.00   3.00    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     2.00   2.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     3.00   3.00    -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00   0.17   0.17   0.17   0.17   0.17   0.17   6.00   6.00    -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.50   1.50    -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	d15, d5, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	d18, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	h20, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	s21, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.16b, { v0.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.8b, { v0.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	v0.16b, { v0.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   1.25   1.25   1.25   tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	v0.8b, { v0.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   1.25   1.25   1.25   tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.1d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	d0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	s0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	s0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	h0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uaddlv	h0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     ucvtf	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   umaxv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	b0, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uminv	b0, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	h0, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	h0, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	s0, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     umov	w0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     umov	w0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     mov	w0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     mov	x0, v0.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	h0, h1, h5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	b11, b20, b30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	s23, s20, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	b10, h12, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	h12, s10, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	s10, d10, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b11, b20, b30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b18, b15, #6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	d15, d12, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	h11, h18, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s14, s19, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s23, s20, s16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	b12, h10, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	h10, s14, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	s10, d12, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.2s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.4h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.8b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.16b, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.4s, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.8h, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	d16, d16, d16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	b18, h18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	h20, s17
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	s19, d14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     urecpe	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	d8, d7, d4
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	d20, d23, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ursqrte	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	d18, d10, #13
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.4b[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	d0, d0, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v0.4s, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v0.8h, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	d10, d17, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	w0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	w0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.b[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	b19, b14
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	d18, d22
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	h20, h15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	s21, s12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	d20, d13, #61
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.16b, v0.16b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2d, v0.2d, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2s, v0.2s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4h, v0.4h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8b, v0.8b, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.2d, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.4s, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.8h, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.2d, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.8h, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.2d, v0.2d, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.4s, v0.4s, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.8h, v0.8h, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.2d, v0.2d, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.4s, v0.4s, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.8h, v0.8h, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.2s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.4h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.8b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.16b, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.4s, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.8h, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8h, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.16b, v0.16b, v0.16b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4h, v0.4h, v0.4h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8b, v0.8b, v0.8b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	d17, d31, d29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   addv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	h0, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     bfcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     bfcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.16b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.2d, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.2s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.4h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.4s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.8b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     dup	v0.8h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d0, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.16b, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2d, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.2s, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.4s, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8b, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dup	v0.8h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	d29, d24, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	s29, s24, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.2s, v0.2s, v0.2s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.4s, v0.4s, v0.4s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.2s, v0.2s, v0.2s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.4s, v0.4s, v0.s[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtas	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtau	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtms	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtmu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtns	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtnu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtps	h22, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtpu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	s22, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	h21, h14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	h21, h12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	h12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	h21, h12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdiv	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00    -     5.00    -     fdiv	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     6.50    -     6.50    -     fdiv	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.2s, v0.2h, v0.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl2	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d23, d11, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s20, s22, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	d13, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	s19, s14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	d22, d30, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	s21, s16, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	d16, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	s18, s10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32x	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32x	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint32x	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32z	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint32z	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint32z	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64x	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64x	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint64x	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64z	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frint64z	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frint64z	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinta	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinti	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintm	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintn	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintp	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintx	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	d8, d22, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	s21, s5, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50    -     7.50    -     fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50    -     2.50    -     fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     6.50    -     6.50    -     fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -     ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.b }[9], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1	{ v0.b }[9], [x0], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x15], #2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.50   0.50   ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.17   0.17   0.17   0.17   0.17   0.17   0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00   2.00   2.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h2, v0.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s17, v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.b[0], v0.b[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.h[1], v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.s[2], v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.d[1], v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.b[0], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.h[1], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.s[2], w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.d[1], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.16b, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v0.8h, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev16	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v0.4h, v9.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev32	v4.8h, v7.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v0.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v1.8b, v9.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v13.4h, v21.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v2.8h, v4.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v4.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev64	v6.4s, v8.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	d0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saddlv	s0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	s0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   saddlv	h0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   saddlv	h0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	d7, d10, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	d10, d14, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   smaxv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   smaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   sminv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d15, s22, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	s12, h22, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	h0, h1, h2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4h, v1.4h, v2.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.8h, v1.8h, v2.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	s0, s1, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.2s, v1.2s, v2.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	h0, h1, h2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4h, v1.4h, v2.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.8h, v1.8h, v2.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	s0, s1, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.2s, v1.2s, v2.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	b10, h13, #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	h15, s10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	s15, d12, #9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	b17, h10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	h10, s13, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	s22, d16, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	b11, b19, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d15, d16, #51
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h13, h18, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s14, s17, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	b15, b18, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	d11, d13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	h19, h17, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	s16, s14, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	b10, h15, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	h17, s10, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	s18, d10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	b15, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	h20, s14, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	s10, d15, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	s20, s10, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	b19, h14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	h21, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	s20, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	d10, d12, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	d19, d18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	d15, d11, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	d15, d16, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	d18, d12, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.17   0.17   0.17   0.17   0.17   0.17   1.50   1.50    -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -      -      -     1.50   1.50    -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -     st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.17   0.17   0.17   0.17   0.17   0.17   1.00   1.00    -      -     st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     2.00   2.00    -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.17   0.17   0.17   0.17   0.17   0.17   3.00   3.00    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     2.00   2.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     3.00   3.00    -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   0.17   0.17   0.17   0.17   0.17   0.17   6.00   6.00    -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     1.50   1.50    -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.17   0.17   0.17   0.17   0.17   0.17   2.00   2.00    -      -     st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	d15, d5, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   1.25   1.25   1.25   tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   1.25   1.25   1.25   tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	d0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaddlv	s0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	s0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uaddlv	h0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uaddlv	h0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   umaxv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   umaxv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   umaxv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	b0, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   uminv	b0, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	h0, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	h0, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	s0, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     umov	w0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     umov	w0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     mov	w0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     mov	x0, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	h0, h1, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	b10, h12, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	h12, s10, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	s10, d10, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	b18, b15, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	d15, d12, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	h11, h18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s14, s19, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	b12, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	h10, s14, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	s10, d12, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	d8, d7, d4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	d20, d23, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	d18, d10, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	d0, d0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	d10, d17, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	w0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	w0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     smov	x0, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	d20, d13, #61
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	v0.8h, v0.8h, v0.8h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
index acd3556824949..6cba45cdd42be 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
@@ -6845,3437 +6845,3440 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -     245.00 651.00 651.00 570.50 272.50 83.75  83.75  81.75  81.75  1536.75 1281.75 794.25 748.25
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     245.00 651.00 651.00 570.50 272.50 83.75  83.75  81.75  81.75  1536.75 1281.75 794.25 748.25
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.b, p0/m, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclb	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclb	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclt	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclt	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z1.s, z2.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.b, p5/m, z21.b, z10.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.b, z10.b, z21.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.d, p5/m, z21.d, z10.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.d, z10.d, z21.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.h, p5/m, z21.h, z10.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.h, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.s, p5/m, z21.s, z10.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.s, z10.s, z21.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.b, p3/m, z23.b, z13.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.b, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.d, p3/m, z23.d, z13.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.h, p3/m, z23.h, z13.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.h, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.s, p3/m, z23.s, z13.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.s, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	sp, sp, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x0, x0, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x21, x21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x23, x8, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	sp, sp, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x0, x0, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x21, x21, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x23, x8, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	z0.b, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	z0.b, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     and	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     ands	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bcax	z29.d, z29.d, z30.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	z0.h, p0/m, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtnt	z0.h, p0/m, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z10.s, z21.h, z14.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z21.s, z14.h, z3.h[2]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z14.s, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     bic	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     bic	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     bics	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     bics	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brka	p0.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brka	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     brkas	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brkb	p0.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brkb	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     brkbs	p0.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkn	p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkn	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkns	p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkns	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpa	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpa	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpas	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpas	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpb	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpb	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	z0.d, z0.d, z1.d, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl1n	z0.d, z0.d, z1.d, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl2n	z0.d, z0.d, z1.d, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.b, z0.b, z0.b, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.d, z0.d, z0.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.h, z0.h, z0.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.s, z0.s, z0.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.b, z31.b, z31.b, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.d, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.h, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.s, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z15.h[1], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cdot	z0.s, z1.b, z31.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cdot	z0.s, z1.b, z7.b[3], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z29.d, z30.h, z0.h[0], #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z31.d, z30.h, z7.h[1], #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z5.d, z6.h, z3.h[0], #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	b0, p7, b0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	x0, p7, x0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.b, p7, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.d, p7, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.h, p7, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	b0, p7, b0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	x0, p7, x0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.b, p7, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.d, p7, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.h, p7, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.b, z1.b, z2.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z0.d, z1.d, z2.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.h, z1.h, z2.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.h, z1.h, z2.h[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.s, z1.s, z2.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.s, z1.s, z2.s[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.b, z16.b, z17.b, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z15.d, z16.d, z17.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.h, z16.h, z17.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.s, z16.s, z17.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.b, z30.b, z31.b, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z29.d, z30.d, z31.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.h, z30.h, z31.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.s, z30.s, z31.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.b, z31.b, z31.b, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z31.d, z31.d, z31.d, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.h, z30.h, z7.h[0], #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.h, z31.h, z31.h, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.s, z30.s, z7.s[0], #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.s, z31.s, z31.s, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z1.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #-16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.d, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.s, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	wzr, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	w30, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	wzr, w30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	x30, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	xzr, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     dech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     eor	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     eor3	z29.d, z29.d, z30.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     eors	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z0.b, { z1.b, z2.b }, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, z31.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, z31.b, z0.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, { z30.b, z31.b }, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadda	d0, p7, d0, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     9.00    -      -     fadda	h0, p7, h0, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00    -      -     fadda	s0, p7, s0, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z29.s, p3/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   faddv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   faddv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   faddv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.d, p0/m, z0.d, z0.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.h, p0/m, z0.h, z0.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.s, p0/m, z0.s, z0.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.d, p7/m, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.h, p7/m, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.d, p0/m, z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, p0/m, z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, p0/m, z1.h, z2.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, z0.h, z0.h[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.s, p0/m, z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.s, p0/m, z1.s, z2.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z21.s, z10.s, z5.s[1], #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z23.s, z13.s, z8.s[0], #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.d, p7/m, z30.d, z31.d, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.h, p7/m, z30.h, z31.h, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.s, p7/m, z30.s, z31.s, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.d, p7/m, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.h, p7/m, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.h, z31.h, z7.h[3], #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.h, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvt	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvt	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtlt	z0.s, p0/m, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtlt	z30.d, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnt	z0.h, p0/m, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnt	z30.s, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtx	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtx	z30.s, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxnt	z0.s, p0/m, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxnt	z30.s, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	z0.s, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fdiv	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fdiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fdivr	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     flogb	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     flogb	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     flogb	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z29.s, p3/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxnmv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fmaxnmv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxnmv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z29.s, p3/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fmaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.d, p0/m, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.h, p0/m, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.s, p0/m, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z29.s, p3/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminnmv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fminnmv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminnmv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z29.s, p3/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z0.s, z1.h, z7.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z30.s, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z0.s, z1.h, z7.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z30.s, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z0.s, z1.h, z7.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z30.s, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z0.s, z1.h, z7.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z30.s, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, #-10.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, p0/m, #-10.00000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, p0/m, #0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.h, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.h, p0/m, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.s, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.s, p0/m, #-0.12500000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, z0.d, z0.d[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, z0.h, z0.h[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, z0.s, z0.s[0]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.d, p7/m, z31.d, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.d, z31.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.h, p7/m, z31.h, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.h, z31.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.s, p7/m, z31.s, #2.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.s, z31.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpe	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpx	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpx	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinta	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinti	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintm	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintn	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintp	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintx	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintz	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frsqrte	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fsqrt	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fsqrt	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fsqrt	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.d, p0/m, z0.d, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.h, p0/m, z0.h, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.s, p0/m, z0.s, #0.5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.d, p7/m, z31.d, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.h, p7/m, z31.h, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.s, p7/m, z31.s, #1.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.d, z0.d, z31.d, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.h, z0.h, z31.h, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.s, z0.s, z31.s, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histcnt	z0.s, p0/z, z1.s, z2.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histcnt	z29.d, p7/z, z30.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histseg	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     inch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   inch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   inch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.b, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     index	z0.d, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.h, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z0.h, w0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.s, #0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z21.b, w10, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z21.d, x10, x21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z21.s, w10, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.b, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.b, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z23.d, #13, x8
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z23.d, x13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.h, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.h, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.s, #13, w8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.s, w13, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.b, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     index	z31.d, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, #-1, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, xzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, xzr, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.h, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.s, #-1, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, #-1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, wzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, wzr, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.b, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.d, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.h, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.s, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.b, b31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.d, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.d, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.h, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.s, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	x0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	x0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z0.b, z1.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z0.d, z1.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z0.h, z1.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z0.s, z1.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z0.b - z2.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z0.d - z2.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z0.h - z2.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z0.s - z2.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z0.b - z3.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z0.d - z3.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z0.h - z3.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z0.s - z3.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z0.s }, p0/z, [z1.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z31.s }, p7/z, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z0.s }, p0/z, [z1.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z31.s }, p7/z, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z0.s }, p0/z, [z1.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z31.s }, p7/z, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z0.s }, p0/z, [z1.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z31.s }, p7/z, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z0.d }, p0/z, [z1.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z0.s }, p0/z, [z1.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z31.d }, p7/z, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z31.s }, p7/z, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, p0/m, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.b, p0/m, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.b, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.d, p0/m, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.d, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.h, p0/m, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.h, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.s, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, z1.b, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, z1.h, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, z1.s, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.b, p7/m, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p15.b, p7/z, z30.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p15.h, p7/z, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.b, p7/m, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.b, p7/m, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.d, z1.d, z7.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0/m, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15/m, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, b0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/m, b0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.b, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0xe0000000000003ff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0xffffffffffff7fff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, p0/m, d0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, p0/m, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.d, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #-256
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #32767
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/m, h0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.h, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.q, q0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #0xffff7fff
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, p0/m, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, p0/m, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, s0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.s, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #32512
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p15/m, #-32768
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p15/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p7/m, b31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movprfx	z31, z6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.b, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, z31.b[63]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p15/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p7/m, d31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movprfx	z31.d, p7/z, z6.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p7/m, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.d, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, z31.d[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p15/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p7/m, h31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.h, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, z31.h[31]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p15/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p7/m, s31
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.s, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, z31.s[15]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p15/m, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.d, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.h, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.q, z17.q[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.s, #-6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.b, p7/m, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.d, p7/m, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.h, p7/m, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z0.d, z1.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z29.s, z30.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.b, z31.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.h, z31.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.s, z31.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nand	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nand	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nands	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nands	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   nbsl	z0.d, z0.d, z1.d, z2.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.b, p0/m, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p0.b, p0/z, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p0.h, p0/z, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p15.b, p7/z, z30.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p15.h, p7/z, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nor	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nor	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nors	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nors	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     not	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     not	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nots	p0.b, p0/z, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nots	p15.b, p15/z, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orn	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orn	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orns	p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orns	p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orr	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.d, z0.d, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.d, z0.d, #0xfffffffffffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.s, z0.s, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.s, z0.s, #0xfffffff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.h, z23.h, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.h, z23.h, #0xfff9
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z5.b, z5.b, #0x6
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z5.b, z5.b, #0xf9
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orrs	p0.b, p0/z, p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfalse	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfirst	p0.b, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfirst	p15.b, p15, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	z29.b, z30.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z29.q, z30.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z29.q, z30.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.b, p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.d, p15, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.h, p15, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.s, p15, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p15.b, p15, p15.b
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#14, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#15, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#6, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p3, [z13.s, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p3, [z13.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0, #31, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [z10.d, #31]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [z10.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#14, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.s, #248]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#6, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#7, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0, #31, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#14, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#6, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#7, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0, #31, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#14, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#6, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#7, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#7, p3, [x13, z8.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0, #31, mul vl]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl1keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl1strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl2keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl2strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl3keep, p0, [x0]
-# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl3strm, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptest	p15, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptest	p15, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.b, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #23
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #26
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #27
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, mul3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, mul4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl128
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl256
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl3
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl4
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl5
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl6
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl64
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl7
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl8
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.b, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #17
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #18
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #19
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #20
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #21
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #22
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #23
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #24
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #25
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #26
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #27
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #28
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, mul3
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, mul4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl128
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl16
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl256
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl3
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl32
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl4
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl5
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl6
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl64
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl7
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl8
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpkhi	p0.h, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpkhi	p15.h, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpklo	p0.h, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpklo	p15.h, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     rax1	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     rdffr	p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     rdffr	p0.b, p0/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     rdffr	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     rdffr	p15.b, p15/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.00    -      -      -      -      -      -      -      -     rdffrs	p0.b, p0/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.00    -      -      -      -      -      -      -      -     rdffrs	p15.b, p15/z
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x0, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x21, #-32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x23, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	xzr, #-1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.b, p1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.d, p1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.h, p1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.s, p1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revh	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revh	z0.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revw	z0.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z0.h, p0/m, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z29.s, p0/m, z30.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z30.d, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   saddv	d0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   saddv	d0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   saddv	d0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclb	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclb	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclt	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclt	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.h, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     scvtf	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     sdiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     sdiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     sdivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     sdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sdot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sdot	z0.d, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	z0.s, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     sel	p0.b, p1, p2.b, p3.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.b, p11, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.d, p11, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.h, p11, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.s, p11, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     setffr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.b, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.d, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.h, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.s, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sm4e	z0.s, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sm4ekey	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.b, z0.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.d, z0.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.h, z0.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.s, z0.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   smaxv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   smaxv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   smaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   smaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.b, z0.b, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.d, z0.d, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.h, z0.h, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.s, z0.s, #-128
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.b, z31.b, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.d, z31.d, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.h, z31.h, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.s, z31.s, #127
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   sminv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   sminv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   sminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   sminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     smulh	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.h, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z29.s, z30.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     smulh	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.b, p7, { z30.b, z31.b }
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.d, p7, { z30.d, z31.d }
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.h, p7, { z30.h, z31.h }
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.s, p7, { z30.s, z31.s }
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.b, p7, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.d, p7, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.h, p7, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.s, p7, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.b, z0.b, z0.b, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.d, z0.d, z0.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.h, z0.h, z0.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.s, z0.s, z0.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.b, z31.b, z31.b, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.d, z31.d, z31.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.h, z31.h, z31.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.s, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.d, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.d, z1.s, z15.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.d, z1.s, z15.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.d, z1.s, z15.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.d, z1.s, z15.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqdmulh	z0.d, z1.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.h, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z29.s, z30.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqdmulh	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.b, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.d, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.h, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.s, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.b, p7/m, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.b, z1.b, z2.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z0.d, z1.d, z2.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.h, z1.h, z2.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.h, z1.h, z2.h[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.s, z1.s, z2.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.s, z1.s, z2.s[0], #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.b, z16.b, z17.b, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z15.d, z16.d, z17.d, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.h, z16.h, z17.h, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.s, z16.s, z17.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.b, z30.b, z31.b, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z29.d, z30.d, z31.d, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.h, z30.h, z31.h, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.s, z30.s, z31.s, #90
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.b, z31.b, z31.b, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z31.d, z31.d, z31.d, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.h, z30.h, z7.h[0], #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.h, z31.h, z31.h, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.s, z30.s, z7.s[0], #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.s, z31.s, z31.s, #180
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	z0.d, z1.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	z0.d, z1.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmulh	z0.d, z1.d, z15.d[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.h, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.h, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.s, z1.s, z7.s[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z29.s, z30.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmulh	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.b, p0/m, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.d, p0/m, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.h, p0/m, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.s, p0/m, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.b, p0/m, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.d, p0/m, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.h, p0/m, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.b, p0/m, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.d, p0/m, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.h, p0/m, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.s, p0/m, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.b, p0/m, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.d, p0/m, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.h, p0/m, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.d, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.h, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.s, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.d, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.h, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.s, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.d, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.h, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.s, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.d, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.h, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.s, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.d }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.h }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.s }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.b }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.h }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.b }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z31.d }, p7, [z31.d, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.h }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z31.s }, p7, [z31.s, #31]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z0.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z31.d }, p7, [z31.d, #248]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.d }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.s }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.h }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z31.d }, p7, [z31.d, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.h }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z31.s }, p7, [z31.s, #62]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.d }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p7, [z0.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p7, [z0.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z21.d }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z21.s }, p5, [x10, #5, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z31.d }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z31.d }, p7, [z31.d, #124]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z31.s }, p7, [sp, #-1, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z31.s }, p7, [z31.s, #124]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2b	{ z0.b, z1.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z0.b, z1.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2b	{ z5.b, z6.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z0.d, z1.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     0.50   0.50   0.50   0.50   1.00   1.00    -      -     st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z0.h, z1.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     0.50   0.50   0.50   0.50   1.00   1.00    -      -     st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z0.s, z1.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3b	{ z0.b - z2.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z0.b - z2.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3b	{ z5.b - z7.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z0.d - z2.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z0.h - z2.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z0.s - z2.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4b	{ z0.b - z3.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z0.b - z3.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4b	{ z5.b - z8.b }, p3, [x17, x16]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z0.d - z3.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z0.h - z3.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z0.s - z3.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0, x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z0.d }, p0, [z1.d]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z0.s }, p0, [z1.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z31.d }, p7, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z31.d }, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z31.s }, p7, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z31.s }, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z0.d }, p0, [z1.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z31.d }, p7, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z31.d }, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z0.d }, p0, [z1.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z0.s }, p0, [z1.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z31.d }, p7, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z31.d }, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z31.s }, p7, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z31.s }, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z0.d }, p0, [z1.d]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z0.s }, p0, [z1.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z31.d }, p7, [z31.d, x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z31.d }, p7, [z31.d]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z31.s }, p7, [z31.s, x0]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z31.s }, p7, [z31.s]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p15, [sp, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p5, [x10, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z0, [x0]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z21, [x10, #-256, mul vl]
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z31, [sp, #255, mul vl]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.b, p5/m, z21.b, z10.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.b, z10.b, z21.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.d, p5/m, z21.d, z10.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.d, z10.d, z21.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.h, p5/m, z21.h, z10.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.h, z10.h, z21.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.s, p5/m, z21.s, z10.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.s, z10.s, z21.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.b, p3/m, z23.b, z13.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.b, z13.b, z8.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.d, p3/m, z23.d, z13.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.d, z13.d, z8.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.h, p3/m, z23.h, z13.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.h, z13.h, z8.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.s, p3/m, z23.s, z13.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.s, z13.s, z8.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.b, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.h, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.s, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, p0/m, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtw	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtw	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.b, { z29.b, z30.b }, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.d, { z29.d, z30.d }, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.h, { z29.h, z30.h }, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.s, { z29.s, z30.s }, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.b, { z31.b }, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.d, { z31.d }, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.h, { z31.h }, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.s, { z31.s }, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.b, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.d, z1.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.h, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.s, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z0.h, p0/m, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z29.s, p0/m, z30.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z30.d, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   uaddv	d0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   uaddv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uaddv	d0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uaddv	d0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.d, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.h, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     ucvtf	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	z0.h, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.s, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     udiv	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     udiv	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     udivr	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     udot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     udot	z0.d, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	z0.s, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	z0.s, z1.b, z7.b[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   umaxv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   umaxv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   umaxv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   umaxv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.b, p7/m, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.d, p7/m, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.h, p7/m, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   uminv	b0, p7, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   uminv	d0, p7, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uminv	h0, p7, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uminv	s0, p7, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.d, z1.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.h, z1.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.s, z1.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.b, p7/m, z0.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.b, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     umulh	z0.d, p7/m, z0.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.h, p7/m, z0.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.h, z1.h, z2.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z29.s, z30.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     umulh	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.d, z1.s, z15.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.s, z1.h, z7.h[7]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, #14
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, vl1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, all, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, pow2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, pow2, mul #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.b, p0/m, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.d, p0/m, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.h, p0/m, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.s, p0/m, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.b, p0/m, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.d, p0/m, z31.d, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.h, p0/m, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.b, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.h, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.s, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.b, z31.h, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.h, z31.s, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.s, z31.d, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, #0, lsl #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.b, z31.b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.d, z31.d, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.h, z31.h, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.s, z31.s, #65280
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.b, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.h, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.s, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     urecpe	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.b, p0/m, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.d, p0/m, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.h, p0/m, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.s, p0/m, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.b, p0/m, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.d, p0/m, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.h, p0/m, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ursqrte	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.d, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.h, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.s, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.d, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.h, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.s, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.d, z0.s, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.h, z0.b, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.s, z0.h, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.d, z31.s, #31
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.h, z31.b, #7
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.s, z31.h, #15
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	z0.s, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.b, z0.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.d, z0.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.h, z0.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.s, z0.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.b, z31.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.d, z31.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.h, z31.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.s, z31.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z0.h, z1.b, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z29.s, z30.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z31.d, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z0.h, z1.h, z2.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z29.s, z30.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z31.d, z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.d, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.h, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.s, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.h, p0/m, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.h, p7/m, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z0.s, p0/m, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtw	z0.d, p0/m, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtw	z31.d, p7/m, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, wzr, w0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, xzr, x0
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.d, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.d, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.h, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.h, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.s, w0, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.s, x0, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.b, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.d, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.h, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.s, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.b, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.d, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.h, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.s, x30, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     wrffr	p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     wrffr	p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.b, z0.b, z1.b, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.d, z0.d, z1.d, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.h, z0.h, z1.h, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.s, z0.s, z1.s, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.b, z31.b, z30.b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.d, z31.d, z30.d, #64
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.h, z31.h, z30.h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.s, z31.s, z30.s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.b, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.d, p0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.h, p0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.s, p0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.s, z31.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.b, p0.b, p0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.d, p0.d, p0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.h, p0.h, p0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.s, p0.s, p0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.b, p15.b, p15.b
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.d, p15.d, p15.d
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.h, p15.h, p15.h
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.s, p15.s, p15.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.b, z0.b, z0.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.d, z0.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.h, z0.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.s, z0.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.b, z31.b, z31.b
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.d, z31.d, z31.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.h, z31.h, z31.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.s, z31.s, z31.s
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	sp, sp, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x0, x0, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x21, x21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addpl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	sp, sp, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x0, x0, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x21, x21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     addvl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesd	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aese	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesimc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   aesmc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   andv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     bcax	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bdep	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bext	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -     bgrp	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl1n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl2n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z15.h[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z0.d, z1.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cdot	z0.s, z1.b, z31.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cdot	z0.s, z1.b, z7.b[3], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z29.d, z30.h, z0.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z31.d, z30.h, z7.h[1], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cdot	z5.d, z6.h, z3.h[0], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50   0.50    -      -     clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     cmla	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     cmla	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cnth	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntp	x0, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     cntw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.d, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     compact	z31.s, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	wzr, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermeq	xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	wzr, w30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ctermne	xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     dech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     dech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   decp	z31.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     decw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     decw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   dupm	z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     eor3	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eorbt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   eortb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   eorv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     9.00    -      -     fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00    -      -     fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   faddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   faddv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   faddv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtlt	z0.s, p0/m, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtlt	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtx	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtx	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxnt	z0.s, p0/m, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtxnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fexpa	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     flogb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     flogb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     flogb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlalt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlslt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpe	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     frsqrte	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -     7.00    -     fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     6.00    -     6.00    -     fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.50    -     4.50    -     fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histcnt	z0.s, p0/z, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histcnt	z29.d, p7/z, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   histseg	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     inch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     inch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   inch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   inch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   incp	z31.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     incw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     incw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   incw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.b, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     index	z0.d, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.h, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z0.h, w0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z0.s, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z21.b, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z21.d, x10, x21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z21.s, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.b, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.b, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z23.d, #13, x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z23.d, x13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.h, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.h, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.s, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z23.s, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.b, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.b, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     index	z31.d, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, #-1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -     1.00    -     1.00    -     index	z31.d, xzr, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.h, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.h, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     index	z31.s, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.50    -     0.50    -     index	z31.s, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.b, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.d, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.d, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.h, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     insr	z31.s, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     insr	z31.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lasta	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lasta	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lastb	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     1.00    -      -     lastb	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -     0.50   0.50   0.50   0.50   0.50   0.50   0.50   0.50   ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -     0.75   0.75   0.75   0.75   ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -     0.75   0.75   0.75   0.75   0.75   0.75   0.75   0.75   ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -     1.00   1.00   1.00   1.00   2.00   2.00   2.00   2.00   ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   0.50   0.50   ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     2.00   2.00   2.00   2.00   ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -     0.25   0.25   0.25   0.25    -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1b	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1b	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1d	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1h	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1h	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sb	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1sh	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -     2.00    -      -     ldnt1w	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -     1.00   1.00   1.00   1.00   ldnt1w	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.50   0.50    -      -      -      -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     match	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/m, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, p0/m, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, p0/m, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, #32767
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/m, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.q, q0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, p0/m, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z0.s, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p7/m, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movprfx	z31, z6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.b, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.b, z31.b[63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p7/m, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, p7/m, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.d, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.d, z31.d[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p7/m, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.h, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.h, z31.h[31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p7/m, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     mov	z31.s, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z31.s, z31.s[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.d, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.h, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.q, z17.q[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	z5.s, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     msb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     mul	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   nbsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -     nmatch	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   orv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfalse	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmul	z29.b, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p3, [z13.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	#7, p3, [z13.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [z10.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pldl3strm, p5, [z10.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfb	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.s, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfd	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfh	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	#7, p3, [x13, z8.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -     prfw	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptest	p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptest	p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ptrue	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     ptrues	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpkhi	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpkhi	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpklo	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     punpklo	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   raddhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     rax1	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     rdffr	p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     rdffr	p0.b, p0/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     rdffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -      -      -      -      -     rdffr	p15.b, p15/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     3.00   1.00    -      -      -      -      -      -      -      -     rdffrs	p0.b, p0/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     3.00   1.00    -      -      -      -      -      -      -      -     rdffrs	p15.b, p15/z
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x21, #-32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	x23, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rdvl	xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.d, p1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.h, p1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     rev	p0.s, p1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rev	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   rshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   rsubhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   saba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   saddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   saddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   saddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sbclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     sel	p0.b, p1, p2.b, p3.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     setffr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sm4e	z0.s, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sm4ekey	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   smaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   smaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   smaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   smaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smin	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   sminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   sminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   sminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   sminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     smulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqcadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqdecp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   sqincp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdcmlah	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdcmlah	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlah	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmlsh	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     sqrdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshlu	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqxtunt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   srhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sri	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   srsra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ssra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubltb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     0.50   0.50   0.50   0.50   1.00   1.00    -      -     st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     0.50   0.50   0.50   0.50   1.00   1.00    -      -     st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -      -      -      -      -     4.50   4.50    -      -     st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     4.50   4.50    -      -     2.25   2.25   2.25   2.25   4.50   4.50    -      -     st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -      -      -      -      -     9.00   9.00    -      -     st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   9.00    -      -     4.50   4.50   4.50   4.50   9.00   9.00    -      -     st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1b	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1b	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1d	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1h	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1h	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.25   0.25   0.25   0.25   stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -     1.00   1.00    -      -     stnt1w	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -      -      -     2.00   2.00    -      -     stnt1w	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -     str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   tbx	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   uaddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   uaddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uaddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uaddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -     2.00    -     ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     20.00   -      -      -     udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     11.00   -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   umaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   umaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   umaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   umaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   2.50   0.50   2.50   uminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   uminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   1.50   0.50   1.50   uminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     umulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqdecp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincb	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.50   0.50    -      -      -      -     0.25   0.25   0.25   0.25   uqincp	z0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     urecpe	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ursqrte	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ursra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   usra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, wzr, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.b, xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.d, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.d, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.h, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.h, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.s, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilege	p15.s, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilerw	p15.s, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -     whilewr	p15.s, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     wrffr	p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -     wrffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.b, z0.b, z1.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.d, z0.d, z1.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.h, z0.h, z1.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z0.s, z0.s, z1.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.b, z31.b, z30.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.d, z31.d, z30.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.h, z31.h, z30.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xar	z31.s, z31.s, z30.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   zip2	z31.s, z31.s, z31.s
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
index aecd89093900f..1690d9669b396 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
@@ -41,35 +41,38 @@ mov  x1, x2
 # CHECK-NEXT: [0.1] - V2UnitB
 # CHECK-NEXT: [1.0] - V2UnitD
 # CHECK-NEXT: [1.1] - V2UnitD
-# CHECK-NEXT: [2]   - V2UnitL2
-# CHECK-NEXT: [3.0] - V2UnitL01
-# CHECK-NEXT: [3.1] - V2UnitL01
-# CHECK-NEXT: [4]   - V2UnitM0
-# CHECK-NEXT: [5]   - V2UnitM1
-# CHECK-NEXT: [6]   - V2UnitS0
-# CHECK-NEXT: [7]   - V2UnitS1
-# CHECK-NEXT: [8]   - V2UnitS2
-# CHECK-NEXT: [9]   - V2UnitS3
-# CHECK-NEXT: [10]  - V2UnitV0
-# CHECK-NEXT: [11]  - V2UnitV1
-# CHECK-NEXT: [12]  - V2UnitV2
-# CHECK-NEXT: [13]  - V2UnitV3
+# CHECK-NEXT: [2.0] - V2UnitFlg
+# CHECK-NEXT: [2.1] - V2UnitFlg
+# CHECK-NEXT: [2.2] - V2UnitFlg
+# CHECK-NEXT: [3]   - V2UnitL2
+# CHECK-NEXT: [4.0] - V2UnitL01
+# CHECK-NEXT: [4.1] - V2UnitL01
+# CHECK-NEXT: [5]   - V2UnitM0
+# CHECK-NEXT: [6]   - V2UnitM1
+# CHECK-NEXT: [7]   - V2UnitS0
+# CHECK-NEXT: [8]   - V2UnitS1
+# CHECK-NEXT: [9]   - V2UnitS2
+# CHECK-NEXT: [10]  - V2UnitS3
+# CHECK-NEXT: [11]  - V2UnitV0
+# CHECK-NEXT: [12]  - V2UnitV1
+# CHECK-NEXT: [13]  - V2UnitV2
+# CHECK-NEXT: [14]  - V2UnitV3
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, #0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	h1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	h1, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	s1, wzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	d1, xzr
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movi	d1, #0000000000000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movi	v1.2d, #0000000000000000
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, w2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, x2
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	h1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	h1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	s1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fmov	d1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movi	d1, #0000000000000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movi	v1.2d, #0000000000000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	w1, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mov	x1, x2
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
index 31178e8e238f1..e69b7fb38295e 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
@@ -293,12 +293,12 @@ vfmv.f.s f7, v16
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      280
-# CHECK-NEXT: Total Cycles:      523
+# CHECK-NEXT: Total Cycles:      524
 # CHECK-NEXT: Total uOps:        280
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.54
-# CHECK-NEXT: IPC:               0.54
+# CHECK-NEXT: uOps Per Cycle:    0.53
+# CHECK-NEXT: IPC:               0.53
 # CHECK-NEXT: Block RThroughput: 512.0
 
 # CHECK:      Instruction Info:
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP600/vmv.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP600/vmv.s
index 3e9dcff4e1c0a..99b72b1fabbae 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveP600/vmv.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP600/vmv.s
@@ -260,12 +260,12 @@ vmv8r.v	v8, v16
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      256
-# CHECK-NEXT: Total Cycles:      237
+# CHECK-NEXT: Total Cycles:      255
 # CHECK-NEXT: Total uOps:        256
 
 # CHECK:      Dispatch Width:    4
-# CHECK-NEXT: uOps Per Cycle:    1.08
-# CHECK-NEXT: IPC:               1.08
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 240.0
 
 # CHECK:      Instruction Info:
diff --git a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse1.s
index 291b8cd43cb4e..344a0d28e11d6 100644
--- a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      14    14.00                       sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse2.s
index 25f79397fa071..dffd6b2ae4f50 100644
--- a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-sse2.s
@@ -464,13 +464,13 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divpd	(%rax), %xmm2
 # CHECK-NEXT:  1      22    22.00                       divsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     lfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     lfence
 # CHECK-NEXT:  1      1     1.00    *      *      U     maskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  1      3     1.00                        maxpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
index 028625013a85c..62ad8f862ffdf 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
@@ -1736,7 +1736,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     257.00 216.25 247.25 173.17 173.17 38.00  421.25 3.25   12.67
+# CHECK-NEXT:  -     257.00 216.25 249.25 173.83 173.83 38.00  419.25 3.25   11.33
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1932,12 +1932,12 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   vmaskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovpd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovpd	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovpd	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovpd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovpd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovpd	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovps	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovps	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovps	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovps	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovps	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovps	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     vmaxpd	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     vmaxpd	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
index 738f197afb284..338fc82d7f220 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
@@ -776,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     96.67  60.67  99.67  99.67  21.00  266.67 4.00   1.67
+# CHECK-NEXT:  -      -     96.67  62.67  100.33 100.33 21.00  264.67 4.00   0.33
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -914,12 +914,12 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpmaddwd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovd	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovd	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovd	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovq	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovq	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovq	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovq	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovq	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovq	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpmaxsb	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpmaxsb	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpmaxsd	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse1.s
index c7b5864b2cbe8..3b3bb6d5ba4a7 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.33    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      11    7.00                        sqrtps	%xmm0, %xmm2
@@ -333,7 +333,7 @@ xorps       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     43.00  34.50  50.50  32.00  32.00  8.00   40.50  0.50   3.00
+# CHECK-NEXT:  -     43.00  34.25  50.25  32.17  32.17  9.00   40.25  0.25   2.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -441,7 +441,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.25   0.25   0.33   0.33    -     0.25   0.25   0.33   sfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     sfence
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  -     7.00   1.00    -      -      -      -      -      -      -     sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
index 8851be4679a1e..dff17ed012abe 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
@@ -470,7 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.50    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -503,7 +503,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movq	%xmm0, %rcx
 # CHECK-NEXT:  2      1     1.00           *            movq	%xmm0, (%rax)
-# CHECK-NEXT:  1      1     1.00                        movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  1      1     0.33                        movq2dq	%mm0, %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      1     1.00           *            movsd	%xmm0, (%rax)
 # CHECK-NEXT:  1      5     0.50    *                   movsd	(%rax), %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     78.00  70.75  95.75  63.17  63.17  14.00  117.25 2.25   4.67
+# CHECK-NEXT:  -     78.00  70.58  95.58  63.67  63.67  15.00  116.08 1.75   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -756,7 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     mfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -789,7 +789,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     movq	%xmm0, %rcx
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   movq	%xmm0, (%rax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     movq2dq	%mm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   movsd	%xmm0, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsd	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
index 2a8722665d588..bd9cf510e002f 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
@@ -1573,8 +1573,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorb	$7, %dil
 # CHECK-NEXT:  5      7     1.00    *      *            rolb	$7, (%rax)
 # CHECK-NEXT:  5      7     1.00    *      *            rorb	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        rorb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        rolb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        rorb	%cl, %dil
 # CHECK-NEXT:  6      8     1.00    *      *            rolb	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            rorb	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        rolw	%di
@@ -1585,8 +1585,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorw	$7, %di
 # CHECK-NEXT:  5      7     1.00    *      *            rolw	$7, (%rax)
 # CHECK-NEXT:  5      7     1.00    *      *            rorw	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        rorw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        rolw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        rorw	%cl, %di
 # CHECK-NEXT:  6      8     1.00    *      *            rolw	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            rorw	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        roll	%edi
@@ -1597,8 +1597,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorl	$7, %edi
 # CHECK-NEXT:  5      7     1.00    *      *            roll	$7, (%rax)
 # CHECK-NEXT:  5      7     1.00    *      *            rorl	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        roll	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        rorl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        roll	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        rorl	%cl, %edi
 # CHECK-NEXT:  6      8     1.00    *      *            roll	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            rorl	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        rolq	%rdi
@@ -1609,8 +1609,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorq	$7, %rdi
 # CHECK-NEXT:  5      7     1.00    *      *            rolq	$7, (%rax)
 # CHECK-NEXT:  5      7     1.00    *      *            rorq	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        rorq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        rolq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        rorq	%cl, %rdi
 # CHECK-NEXT:  6      8     1.00    *      *            rolq	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            rorq	%cl, (%rax)
 # CHECK-NEXT:  1      1     0.50                        sahf
@@ -1626,9 +1626,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      6     1.00    *      *            sarb	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shlb	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shrb	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        shlb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        shrb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        sarb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        shlb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        shrb	%cl, %dil
 # CHECK-NEXT:  6      8     1.00    *      *            sarb	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shlb	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shrb	%cl, (%rax)
@@ -1644,9 +1644,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      6     1.00    *      *            sarw	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shlw	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shrw	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        shlw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        shrw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        sarw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        shlw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        shrw	%cl, %di
 # CHECK-NEXT:  6      8     1.00    *      *            sarw	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shlw	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shrw	%cl, (%rax)
@@ -1662,9 +1662,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      6     1.00    *      *            sarl	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shll	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shrl	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarl	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        shll	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        shrl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        sarl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        shll	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        shrl	%cl, %edi
 # CHECK-NEXT:  6      8     1.00    *      *            sarl	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shll	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shrl	%cl, (%rax)
@@ -1680,9 +1680,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      6     1.00    *      *            sarq	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shlq	$7, (%rax)
 # CHECK-NEXT:  4      6     1.00    *      *            shrq	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        shlq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        shrq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        sarq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        shlq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        shrq	%cl, %rdi
 # CHECK-NEXT:  6      8     1.00    *      *            sarq	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shlq	%cl, (%rax)
 # CHECK-NEXT:  6      8     1.00    *      *            shrq	%cl, (%rax)
@@ -1962,7 +1962,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 50.00   -     610.50 401.50 378.50 378.50 345.00 298.00 646.00 129.00
+# CHECK-NEXT: 50.00   -     615.50 396.50 378.50 378.50 345.00 293.00 651.00 129.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -2505,8 +2505,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorb	$7, %dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorb	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorb	%cl, %dil
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolw	%di
@@ -2517,8 +2517,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorw	$7, %di
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolw	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorw	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorw	%cl, %di
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     roll	%edi
@@ -2529,8 +2529,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorl	$7, %edi
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   roll	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorl	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     roll	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     roll	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorl	%cl, %edi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   roll	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorl	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolq	%rdi
@@ -2541,8 +2541,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorq	$7, %rdi
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolq	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorq	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorq	%cl, %rdi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorq	%cl, (%rax)
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     sahf
@@ -2558,9 +2558,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarb	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlb	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrb	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrb	%cl, %dil
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrb	%cl, (%rax)
@@ -2576,9 +2576,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarw	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlw	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrw	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrw	%cl, %di
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrw	%cl, (%rax)
@@ -2594,9 +2594,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarl	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shll	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrl	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarl	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shll	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shll	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrl	%cl, %edi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarl	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shll	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrl	%cl, (%rax)
@@ -2612,9 +2612,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarq	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlq	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrq	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrq	%cl, %rdi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrq	%cl, (%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/Generic/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/Generic/resources-sse1.s
index 291b8cd43cb4e..344a0d28e11d6 100644
--- a/llvm/test/tools/llvm-mca/X86/Generic/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/Generic/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      14    14.00                       sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Generic/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Generic/resources-sse2.s
index 25f79397fa071..dffd6b2ae4f50 100644
--- a/llvm/test/tools/llvm-mca/X86/Generic/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Generic/resources-sse2.s
@@ -464,13 +464,13 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divpd	(%rax), %xmm2
 # CHECK-NEXT:  1      22    22.00                       divsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     lfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     lfence
 # CHECK-NEXT:  1      1     1.00    *      *      U     maskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  1      3     1.00                        maxpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
index 179393abb08d4..35f4945bf222a 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
@@ -1736,7 +1736,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     336.00 215.58 248.58 173.17 173.17 38.00  424.58 3.25   12.67
+# CHECK-NEXT:  -     336.00 215.58 250.58 173.83 173.83 38.00  422.58 3.25   11.33
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1932,12 +1932,12 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   vmaskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovpd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovpd	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovpd	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovpd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovpd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovpd	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovps	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vmaskmovps	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovps	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vmaskmovps	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovps	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vmaskmovps	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     vmaxpd	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     vmaxpd	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
index 814fadac6bbff..095c49b19d5d8 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
@@ -776,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     206.67 90.67  99.67  99.67  5.00   284.67 30.00  1.67
+# CHECK-NEXT:  -      -     206.67 92.67  100.33 100.33 5.00   282.67 30.00  0.33
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -914,12 +914,12 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpmaddwd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovd	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovd	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovd	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovq	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpmaskmovq	(%rax), %ymm0, %ymm2
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovq	%xmm0, %xmm1, (%rax)
-# CHECK-NEXT:  -      -     1.00   0.50   0.33   0.33   1.00   0.50    -     0.33   vpmaskmovq	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovq	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50   1.00    -      -      -     vpmaskmovq	%ymm0, %ymm1, (%rax)
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpmaxsb	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpmaxsb	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpmaxsd	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse1.s
index 37a28a66fd350..c840782d56bc0 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.33    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      11    7.00                        sqrtps	%xmm0, %xmm2
@@ -333,7 +333,7 @@ xorps       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     56.00  34.50  50.50  32.00  32.00  8.00   40.50  0.50   3.00
+# CHECK-NEXT:  -     56.00  34.25  50.25  32.17  32.17  9.00   40.25  0.25   2.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -441,7 +441,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.25   0.25   0.33   0.33    -     0.25   0.25   0.33   sfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     sfence
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  -     7.00   1.00    -      -      -      -      -      -      -     sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
index c9c3e20eeaded..3c862fe8bb6b3 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
@@ -470,7 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.50    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -503,7 +503,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movq	%xmm0, %rcx
 # CHECK-NEXT:  2      1     1.00           *            movq	%xmm0, (%rax)
-# CHECK-NEXT:  1      1     1.00                        movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  1      1     0.33                        movq2dq	%mm0, %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      1     1.00           *            movsd	%xmm0, (%rax)
 # CHECK-NEXT:  1      5     0.50    *                   movsd	(%rax), %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     112.00 70.75  95.75  63.17  63.17  14.00  117.25 2.25   4.67
+# CHECK-NEXT:  -     112.00 70.58  95.58  63.67  63.67  15.00  116.08 1.75   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -756,7 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     mfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -789,7 +789,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     movq	%xmm0, %rcx
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   movq	%xmm0, (%rax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     movq2dq	%mm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   movsd	%xmm0, (%rax)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsd	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
index 5607ae2b6ebd9..c85ad704a0aa2 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
@@ -1573,8 +1573,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorb	$7, %dil
 # CHECK-NEXT:  5      8     1.00    *      *            rolb	$7, (%rax)
 # CHECK-NEXT:  5      8     1.00    *      *            rorb	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        rorb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        rolb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        rorb	%cl, %dil
 # CHECK-NEXT:  6      9     1.00    *      *            rolb	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            rorb	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        rolw	%di
@@ -1585,8 +1585,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorw	$7, %di
 # CHECK-NEXT:  5      8     1.00    *      *            rolw	$7, (%rax)
 # CHECK-NEXT:  5      8     1.00    *      *            rorw	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        rorw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        rolw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        rorw	%cl, %di
 # CHECK-NEXT:  6      9     1.00    *      *            rolw	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            rorw	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        roll	%edi
@@ -1597,8 +1597,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorl	$7, %edi
 # CHECK-NEXT:  5      8     1.00    *      *            roll	$7, (%rax)
 # CHECK-NEXT:  5      8     1.00    *      *            rorl	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        roll	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        rorl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        roll	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        rorl	%cl, %edi
 # CHECK-NEXT:  6      9     1.00    *      *            roll	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            rorl	%cl, (%rax)
 # CHECK-NEXT:  2      2     1.00                        rolq	%rdi
@@ -1609,8 +1609,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rorq	$7, %rdi
 # CHECK-NEXT:  5      8     1.00    *      *            rolq	$7, (%rax)
 # CHECK-NEXT:  5      8     1.00    *      *            rorq	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        rolq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        rorq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        rolq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        rorq	%cl, %rdi
 # CHECK-NEXT:  6      9     1.00    *      *            rolq	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            rorq	%cl, (%rax)
 # CHECK-NEXT:  1      1     0.50                        sahf
@@ -1626,9 +1626,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      7     1.00    *      *            sarb	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shlb	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shrb	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        shlb	%cl, %dil
-# CHECK-NEXT:  3      3     1.00                        shrb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        sarb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        shlb	%cl, %dil
+# CHECK-NEXT:  3      3     1.50                        shrb	%cl, %dil
 # CHECK-NEXT:  6      9     1.00    *      *            sarb	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shlb	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shrb	%cl, (%rax)
@@ -1644,9 +1644,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      7     1.00    *      *            sarw	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shlw	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shrw	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        shlw	%cl, %di
-# CHECK-NEXT:  3      3     1.00                        shrw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        sarw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        shlw	%cl, %di
+# CHECK-NEXT:  3      3     1.50                        shrw	%cl, %di
 # CHECK-NEXT:  6      9     1.00    *      *            sarw	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shlw	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shrw	%cl, (%rax)
@@ -1662,9 +1662,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      7     1.00    *      *            sarl	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shll	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shrl	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarl	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        shll	%cl, %edi
-# CHECK-NEXT:  3      3     1.00                        shrl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        sarl	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        shll	%cl, %edi
+# CHECK-NEXT:  3      3     1.50                        shrl	%cl, %edi
 # CHECK-NEXT:  6      9     1.00    *      *            sarl	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shll	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shrl	%cl, (%rax)
@@ -1680,9 +1680,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  4      7     1.00    *      *            sarq	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shlq	$7, (%rax)
 # CHECK-NEXT:  4      7     1.00    *      *            shrq	$7, (%rax)
-# CHECK-NEXT:  3      3     1.00                        sarq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        shlq	%cl, %rdi
-# CHECK-NEXT:  3      3     1.00                        shrq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        sarq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        shlq	%cl, %rdi
+# CHECK-NEXT:  3      3     1.50                        shrq	%cl, %rdi
 # CHECK-NEXT:  6      9     1.00    *      *            sarq	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shlq	%cl, (%rax)
 # CHECK-NEXT:  6      9     1.00    *      *            shrq	%cl, (%rax)
@@ -1962,7 +1962,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 80.00   -     663.50 424.50 371.50 371.50 331.00 323.00 728.00 123.00
+# CHECK-NEXT: 80.00   -     668.50 419.50 371.50 371.50 331.00 318.00 733.00 123.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -2505,8 +2505,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorb	$7, %dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorb	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorb	%cl, %dil
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolw	%di
@@ -2517,8 +2517,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorw	$7, %di
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolw	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorw	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorw	%cl, %di
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     roll	%edi
@@ -2529,8 +2529,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorl	$7, %edi
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   roll	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorl	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     roll	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     roll	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorl	%cl, %edi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   roll	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorl	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolq	%rdi
@@ -2541,8 +2541,8 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     rorq	$7, %rdi
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolq	$7, (%rax)
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rorq	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rolq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     rorq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rolq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     rorq	%cl, %rdi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rolq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   rorq	%cl, (%rax)
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     sahf
@@ -2558,9 +2558,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarb	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlb	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrb	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlb	%cl, %dil
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlb	%cl, %dil
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrb	%cl, %dil
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlb	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrb	%cl, (%rax)
@@ -2576,9 +2576,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarw	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlw	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrw	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlw	%cl, %di
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlw	%cl, %di
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrw	%cl, %di
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlw	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrw	%cl, (%rax)
@@ -2594,9 +2594,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarl	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shll	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrl	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarl	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shll	%cl, %edi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarl	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shll	%cl, %edi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrl	%cl, %edi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarl	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shll	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrl	%cl, (%rax)
@@ -2612,9 +2612,9 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   sarq	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shlq	$7, (%rax)
 # CHECK-NEXT:  -      -     0.50    -     0.83   0.83   1.00    -     0.50   0.33   shrq	$7, (%rax)
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     sarq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shlq	%cl, %rdi
-# CHECK-NEXT:  -      -     1.25   0.25    -      -      -     0.25   1.25    -     shrq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     sarq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shlq	%cl, %rdi
+# CHECK-NEXT:  -      -     1.50    -      -      -      -      -     1.50    -     shrq	%cl, %rdi
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   sarq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shlq	%cl, (%rax)
 # CHECK-NEXT:  -      -     1.25   0.25   0.83   0.83   1.00   0.25   1.25   0.33   shrq	%cl, (%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
index 0e7c5751e5c08..ca0e054389c88 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
@@ -1287,13 +1287,13 @@ vzeroupper
 # CHECK-NEXT:  1      1     1.00                        vmovhlps	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vmovlhps	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      1     0.50           *            vmovhpd	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   vmovhpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vmovhpd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      1     0.50           *            vmovhps	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   vmovhps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vmovhps	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      1     0.50           *            vmovlpd	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   vmovlpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vmovlpd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      1     0.50           *            vmovlps	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   vmovlps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vmovlps	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      2     1.00                        vmovmskpd	%xmm0, %ecx
 # CHECK-NEXT:  1      2     1.00                        vmovmskpd	%ymm0, %ecx
 # CHECK-NEXT:  1      2     1.00                        vmovmskps	%xmm0, %ecx
@@ -1469,14 +1469,14 @@ vzeroupper
 # CHECK-NEXT:  4      9     1.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  3      3     1.00                        vphsubw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  4      9     1.00    *                   vphsubw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     2.00                        vpinsrb	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     2.00                        vpinsrd	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrd	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     2.00                        vpinsrq	$1, %rax, %xmm1, %xmm2
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     2.00                        vpinsrw	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      2     1.00                        vpinsrb	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      2     1.00                        vpinsrd	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      2     1.00                        vpinsrq	$1, %rax, %xmm1, %xmm2
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      2     1.00                        vpinsrw	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      11    0.50    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      5     0.50                        vpmaddwd	%xmm0, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     126.00 326.25 257.25 160.50 160.50 19.00  271.25 6.25   19.00  19.00  19.00
+# CHECK-NEXT:  -     126.00 326.25 261.25 160.50 160.50 19.00  267.25 6.25   19.00  19.00  19.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2181,14 +2181,14 @@ vzeroupper
 # CHECK-NEXT:  -      -     0.50   1.50   0.50   0.50    -     1.00    -      -      -      -     vphsubsw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.33   1.33    -      -      -     1.33    -      -      -      -     vphsubw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.33   1.33   0.50   0.50    -     1.33    -      -      -      -     vphsubw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrb	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrb	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrd	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrd	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrq	$1, %rax, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrq	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrw	$1, %eax, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrb	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrb	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrd	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrq	$1, %rax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrq	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrw	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrw	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpmaddubsw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpmaddubsw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpmaddwd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bw.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bw.s
index ad81aeae31e4c..720b2f1e6ef7b 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bw.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bw.s
@@ -826,10 +826,10 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  3      2     0.50           *            vpextrb	$0, %xmm16, (%rax)
 # CHECK-NEXT:  2      3     1.00                        vpextrw	$0, %xmm16, %eax
 # CHECK-NEXT:  3      2     0.50           *            vpextrw	$0, %xmm16, (%rax)
-# CHECK-NEXT:  2      2     2.00                        vpinsrb	$0, %eax, %xmm16, %xmm19
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrb	$0, (%rax), %xmm16, %xmm19
-# CHECK-NEXT:  2      2     2.00                        vpinsrw	$0, %eax, %xmm16, %xmm19
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrw	$0, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  2      2     1.00                        vpinsrb	$0, %eax, %xmm16, %xmm19
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrb	$0, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  2      2     1.00                        vpinsrw	$0, %eax, %xmm16, %xmm19
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrw	$0, (%rax), %xmm16, %xmm19
 # CHECK-NEXT:  2      6     2.00                        vpermw	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  2      6     2.00                        vpermw	%zmm16, %zmm17, %zmm19 {%k1}
@@ -1134,7 +1134,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     239.50 19.50  110.00 110.00 8.00   299.50 0.50   8.00   8.00   8.00
+# CHECK-NEXT:  -      -     239.50 21.50  110.00 110.00 8.00   297.50 0.50   8.00   8.00   8.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -1350,10 +1350,10 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -     0.50    -      -     0.50   0.50    -     0.50   0.50   0.50   vpextrb	$0, %xmm16, (%rax)
 # CHECK-NEXT:  -      -     1.00   0.50    -      -      -     0.50    -      -      -      -     vpextrw	$0, %xmm16, %eax
 # CHECK-NEXT:  -      -      -     0.50    -      -     0.50   0.50    -     0.50   0.50   0.50   vpextrw	$0, %xmm16, (%rax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrb	$0, %eax, %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrb	$0, (%rax), %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrw	$0, %eax, %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrw	$0, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrb	$0, %eax, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrb	$0, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrw	$0, %eax, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrw	$0, (%rax), %xmm16, %xmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpermw	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -      -      -     vpermw	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpermw	%zmm16, %zmm17, %zmm19 {%k1}
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s
index 8124c58483233..d20a53240ad9d 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s
@@ -739,10 +739,10 @@ vxorps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  3      2     0.50           *            vpextrd	$1, %xmm16, (%rax)
 # CHECK-NEXT:  2      3     1.00                        vpextrq	$1, %xmm16, %rcx
 # CHECK-NEXT:  3      2     0.50           *            vpextrq	$1, %xmm16, (%rax)
-# CHECK-NEXT:  2      2     2.00                        vpinsrd	$1, %ecx, %xmm16, %xmm19
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrd	$1, (%rax), %xmm16, %xmm19
-# CHECK-NEXT:  2      2     2.00                        vpinsrq	$1, %rcx, %xmm16, %xmm19
-# CHECK-NEXT:  2      6     1.00    *                   vpinsrq	$1, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  2      2     1.00                        vpinsrd	$1, %ecx, %xmm16, %xmm19
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrd	$1, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  2      2     1.00                        vpinsrq	$1, %rcx, %xmm16, %xmm19
+# CHECK-NEXT:  2      6     0.50    *                   vpinsrq	$1, (%rax), %xmm16, %xmm19
 # CHECK-NEXT:  1      1     0.50                        vpmovm2d	%k0, %zmm0
 # CHECK-NEXT:  1      1     0.50                        vpmovm2q	%k0, %zmm0
 # CHECK-NEXT:  1      1     1.00                        vpmovd2m	%zmm0, %k0
@@ -872,7 +872,7 @@ vxorps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     209.25 75.25  102.00 102.00 5.50   195.25 0.25   5.50   5.50   5.50
+# CHECK-NEXT:  -      -     209.25 77.25  102.00 102.00 5.50   193.25 0.25   5.50   5.50   5.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -1148,10 +1148,10 @@ vxorps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -     0.50    -      -     0.50   0.50    -     0.50   0.50   0.50   vpextrd	$1, %xmm16, (%rax)
 # CHECK-NEXT:  -      -     1.00   0.50    -      -      -     0.50    -      -      -      -     vpextrq	$1, %xmm16, %rcx
 # CHECK-NEXT:  -      -      -     0.50    -      -     0.50   0.50    -     0.50   0.50   0.50   vpextrq	$1, %xmm16, (%rax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrd	$1, %ecx, %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrd	$1, (%rax), %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     vpinsrq	$1, %rcx, %xmm16, %xmm19
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpinsrq	$1, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrd	$1, %ecx, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrd	$1, (%rax), %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     vpinsrq	$1, %rcx, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpinsrq	$1, (%rax), %xmm16, %xmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -      -      -     vpmovm2d	%k0, %zmm0
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -      -      -     vpmovm2q	%k0, %zmm0
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     vpmovd2m	%zmm0, %k0
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse1.s
index 01e579493b4ff..9817fc2410daf 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse1.s
@@ -246,9 +246,9 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movhlps	%xmm0, %xmm2
 # CHECK-NEXT:  1      1     1.00                        movlhps	%xmm0, %xmm2
 # CHECK-NEXT:  2      1     0.50           *            movhps	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   movhps	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   movhps	(%rax), %xmm2
 # CHECK-NEXT:  2      1     0.50           *            movlps	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   movlps	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   movlps	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        movmskps	%xmm0, %ecx
 # CHECK-NEXT:  2      1     0.50           *            movntps	%xmm0, (%rax)
 # CHECK-NEXT:  2      1     0.50    *      *      U     movntq	%mm0, (%rax)
@@ -269,8 +269,8 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pavgw	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pavgw	(%rax), %mm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %mm0, %ecx
-# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %mm2
-# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %mm2
+# CHECK-NEXT:  2      2     1.00                        pinsrw	$1, %eax, %mm2
+# CHECK-NEXT:  2      6     0.50    *                   pinsrw	$1, (%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pmaxsw	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pmaxsw	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pmaxub	%mm0, %mm2
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      4     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.50    *      *      U     sfence
+# CHECK-NEXT:  2      1     0.50    *      *      U     sfence
 # CHECK-NEXT:  1      1     0.50                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      12    3.00                        sqrtps	%xmm0, %xmm2
@@ -335,7 +335,7 @@ xorps       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     24.00  66.92  29.42  29.00  29.00  4.50   31.42  0.25   4.50   4.50   4.50
+# CHECK-NEXT:  -     24.00  66.92  30.42  29.00  29.00  4.50   30.42  0.25   4.50   4.50   4.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -414,8 +414,8 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     pavgw	%mm0, %mm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -      -      -     pavgw	(%rax), %mm2
 # CHECK-NEXT:  -      -     1.00   0.50    -      -      -     0.50    -      -      -      -     pextrw	$1, %mm0, %ecx
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     pinsrw	$1, %eax, %mm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pinsrw	$1, (%rax), %mm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     pinsrw	$1, %eax, %mm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pinsrw	$1, (%rax), %mm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     pmaxsw	%mm0, %mm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -      -      -     pmaxsw	(%rax), %mm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     pmaxub	%mm0, %mm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
index 88d1882af8b59..d039f7c611fbb 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
@@ -470,7 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     0.50    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  3      3     1.00    *      *      U     mfence
+# CHECK-NEXT:  2      1     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      4     0.50                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        minsd	%xmm0, %xmm2
@@ -490,9 +490,9 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movdqu	(%rax), %xmm2
 # CHECK-NEXT:  2      2     1.00                        movdq2q	%xmm0, %mm2
 # CHECK-NEXT:  2      1     0.50           *            movhpd	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   movhpd	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   movhpd	(%rax), %xmm2
 # CHECK-NEXT:  2      1     0.50           *            movlpd	%xmm0, (%rax)
-# CHECK-NEXT:  2      6     1.00    *                   movlpd	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   movlpd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        movmskpd	%xmm0, %ecx
 # CHECK-NEXT:  2      1     0.50           *            movntil	%eax, (%rax)
 # CHECK-NEXT:  2      1     0.50           *            movntiq	%rax, (%rax)
@@ -561,8 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
-# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
-# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
+# CHECK-NEXT:  2      2     1.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     0.50    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     0.50                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    0.50    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -691,7 +691,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     40.00  106.92 103.42 58.50  58.50  7.50   71.92  1.75   8.00   8.00   7.50
+# CHECK-NEXT:  -     40.00  106.92 104.42 58.50  58.50  7.50   70.92  1.75   7.50   7.50   7.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -758,7 +758,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     maxsd	(%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -     0.50    -      -     1.00   1.00   0.50   mfence
+# CHECK-NEXT:  -      -      -      -      -      -     0.50    -      -     0.50   0.50   0.50   mfence
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -849,8 +849,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00   0.50    -      -      -     0.50    -      -      -      -     pextrw	$1, %xmm0, %ecx
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     pinsrw	$1, %eax, %xmm0
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pinsrw	$1, (%rax), %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pmaxsw	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
index 9748a27848770..229134cf91285 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
@@ -191,12 +191,12 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  3      2     0.50           *            pextrw	$1, %xmm0, (%rax)
 # CHECK-NEXT:  1      4     1.00                        phminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   phminposuw	(%rax), %xmm2
-# CHECK-NEXT:  2      2     2.00                        pinsrb	$1, %eax, %xmm1
-# CHECK-NEXT:  2      6     1.00    *                   pinsrb	$1, (%rax), %xmm1
-# CHECK-NEXT:  2      2     2.00                        pinsrd	$1, %eax, %xmm1
-# CHECK-NEXT:  2      6     1.00    *                   pinsrd	$1, (%rax), %xmm1
-# CHECK-NEXT:  2      2     2.00                        pinsrq	$1, %rax, %xmm1
-# CHECK-NEXT:  2      6     1.00    *                   pinsrq	$1, (%rax), %xmm1
+# CHECK-NEXT:  2      2     1.00                        pinsrb	$1, %eax, %xmm1
+# CHECK-NEXT:  2      6     0.50    *                   pinsrb	$1, (%rax), %xmm1
+# CHECK-NEXT:  2      2     1.00                        pinsrd	$1, %eax, %xmm1
+# CHECK-NEXT:  2      6     0.50    *                   pinsrd	$1, (%rax), %xmm1
+# CHECK-NEXT:  2      2     1.00                        pinsrq	$1, %rax, %xmm1
+# CHECK-NEXT:  2      6     0.50    *                   pinsrq	$1, (%rax), %xmm1
 # CHECK-NEXT:  1      1     0.50                        pmaxsb	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pmaxsb	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsd	%xmm0, %xmm2
@@ -268,7 +268,7 @@ roundss     $1, (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     37.00  46.50  22.00  22.00  2.50   49.50   -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     37.00  49.50  22.00  22.00  2.50   46.50   -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -308,12 +308,12 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -     0.50   0.50    -     0.50   0.50   0.50   pextrw	$1, %xmm0, (%rax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     phminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -      -      -     phminposuw	(%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     pinsrb	$1, %eax, %xmm1
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pinsrb	$1, (%rax), %xmm1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     pinsrd	$1, %eax, %xmm1
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pinsrd	$1, (%rax), %xmm1
-# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -     pinsrq	$1, %rax, %xmm1
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pinsrq	$1, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     pinsrb	$1, %eax, %xmm1
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pinsrb	$1, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     pinsrd	$1, %eax, %xmm1
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pinsrd	$1, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     1.50    -      -      -      -     pinsrq	$1, %rax, %xmm1
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pinsrq	$1, (%rax), %xmm1
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pmaxsb	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pmaxsb	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pmaxsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse1.s
index 02b5810cda417..02bfa445e5884 100644
--- a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      14    14.00                       sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
index ff0f22bec1402..7d4332f8d8d33 100644
--- a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
@@ -464,13 +464,13 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divpd	(%rax), %xmm2
 # CHECK-NEXT:  1      22    22.00                       divsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      28    22.00   *                   divsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     lfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     lfence
 # CHECK-NEXT:  1      1     1.00    *      *      U     maskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  1      3     1.00                        maxpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
index 142006c3c3d38..a3f31338ebcdc 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      4     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.33    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      12    3.00                        sqrtps	%xmm0, %xmm2
@@ -333,7 +333,7 @@ xorps       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     24.00  73.33  23.33  32.00  32.00  8.00   31.83  0.50   3.00
+# CHECK-NEXT:  -     24.00  73.08  23.08  32.17  32.17  9.00   31.58  0.25   2.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -441,7 +441,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.25   0.25   0.33   0.33    -     0.25   0.25   0.33   sfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     sfence
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
index b527c1cda3a8f..f7b5d426c89b3 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
@@ -470,7 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     0.50    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  3      3     0.50    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      4     0.50                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        minsd	%xmm0, %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     40.00  111.58 81.58  63.50  63.50  14.00  91.58  2.25   5.00
+# CHECK-NEXT:  -     40.00  111.08 81.08  63.67  63.67  15.00  91.08  1.75   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -756,7 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.33    -     0.50   0.50   0.33   mfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     mfence
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
index 3d209467e7780..671f0d18a0f0f 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
@@ -298,7 +298,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  1      4     1.00                        rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.33    *      *      U     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     sfence
 # CHECK-NEXT:  1      1     1.00                        shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      12    3.00                        sqrtps	%xmm0, %xmm2
@@ -333,7 +333,7 @@ xorps       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     24.00  66.83  27.83  32.00  32.00  8.00   33.83  0.50   3.00
+# CHECK-NEXT:  -     24.00  66.58  27.58  32.17  32.17  9.00   33.58  0.25   2.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -441,7 +441,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtps	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     rsqrtss	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     rsqrtss	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.25   0.25   0.33   0.33    -     0.25   0.25   0.33   sfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     sfence
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     shufps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     shufps	$1, (%rax), %xmm2
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     sqrtps	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
index 3a487257b046e..9a41c1b4bab6c 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
@@ -470,7 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     0.50    *                   maxsd	(%rax), %xmm2
-# CHECK-NEXT:  3      3     0.50    *      *      U     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      4     0.50                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        minsd	%xmm0, %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     40.00  107.08 86.08  63.50  63.50  14.00  90.58  2.25   5.00
+# CHECK-NEXT:  -     40.00  106.58 85.58  63.67  63.67  15.00  90.08  1.75   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -756,7 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.33    -     0.50   0.50   0.33   mfence
+# CHECK-NEXT:  -      -      -      -     0.50   0.50   1.00    -      -      -     mfence
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minsd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/barrier_output.s b/llvm/test/tools/llvm-mca/X86/barrier_output.s
index dddfa044039fa..7eac34688f1a2 100644
--- a/llvm/test/tools/llvm-mca/X86/barrier_output.s
+++ b/llvm/test/tools/llvm-mca/X86/barrier_output.s
@@ -19,7 +19,7 @@ maskmovdqu	%xmm0, %xmm1
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
 # CHECK-NEXT:  4      5     1.00    *      *      U                   clflush	(%rax)
-# CHECK-NEXT:  1      1     1.00    *      *      U      *            lfence
-# CHECK-NEXT:  1      1     1.00    *      *      U      *      *     mfence
-# CHECK-NEXT:  1      1     1.00    *      *      U             *     sfence
+# CHECK-NEXT:  2      1     1.00    *      *      U      *            lfence
+# CHECK-NEXT:  2      1     1.00    *      *      U      *      *     mfence
+# CHECK-NEXT:  2      1     1.00    *      *      U             *     sfence
 # CHECK-NEXT:  1      1     1.00    *      *      U                   maskmovdqu	%xmm0, %xmm1
diff --git a/llvm/test/tools/llvm-objcopy/MachO/dysymtab-zero-indirectsym.test b/llvm/test/tools/llvm-objcopy/MachO/dysymtab-zero-indirectsym.test
new file mode 100644
index 0000000000000..bdfb8b809928d
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/dysymtab-zero-indirectsym.test
@@ -0,0 +1,48 @@
+## Show that llvm-strip correctly zeroes out indirectsymoff and indirectsyms
+## in LC_DYSYMTAB if there are no symbols
+#
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-strip --strip-all %t -o %t.stripped
+# RUN: obj2yaml %t.stripped | FileCheck %s
+
+# CHECK: indirectsymoff: 0
+# CHECK-NEXT: nindirectsyms: 0
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x80000003
+  filetype:        0x00000002
+  ncmds:           2
+  sizeofcmds:      104
+  flags:           0x00200085
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          0
+    nsyms:           0
+    stroff:          0
+    strsize:         0
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      0
+    iundefsym:       0
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  42
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+...
diff --git a/llvm/test/tools/llvm-objdump/ELF/AArch64/mattr.s b/llvm/test/tools/llvm-objdump/ELF/AArch64/mattr.s
index e236660770648..5fd77b579984a 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AArch64/mattr.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AArch64/mattr.s
@@ -1,6 +1,10 @@
 ## When --mattr and --mcpu are both empty, disassemble all known instructions.
 # RUN: llvm-mc -filetype=obj -triple=aarch64 -mattr=+all %s -o %t
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefixes=CHECK,ALL
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be -mattr=+all %s -o %t
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefixes=CHECK,ALL
+# RUN: llvm-mc -filetype=obj -triple=aarch64_32 -mattr=+all %s -o %t
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s --check-prefixes=CHECK,ALL
 
 ## If --mattr or --mcpu is specified, don't default to --mattr=+all.
 # RUN: llvm-objdump -d --no-show-raw-insn --mattr=+v8a %t | FileCheck %s --check-prefixes=CHECK,UNKNOWN
diff --git a/llvm/test/tools/llvm-profdata/memprof-yaml-invalid.test b/llvm/test/tools/llvm-profdata/memprof-yaml-invalid.test
new file mode 100644
index 0000000000000..a13451cee5116
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/memprof-yaml-invalid.test
@@ -0,0 +1,8 @@
+; REQUIRES: x86_64-linux
+; RUN: split-file %s %t
+; RUN: not llvm-profdata merge %t/memprof-invalid.yaml -o %t/memprof-invalid.indexed
+
+; Verify that the invalid YAML input results in an error.
+;--- memprof-invalid.yaml
+---
+...
diff --git a/llvm/test/tools/llvm-profdata/memprof-yaml.test b/llvm/test/tools/llvm-profdata/memprof-yaml.test
new file mode 100644
index 0000000000000..0c040ab3a1f9d
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/memprof-yaml.test
@@ -0,0 +1,34 @@
+; REQUIRES: x86_64-linux
+; RUN: split-file %s %t
+; RUN: llvm-profdata merge %t/memprof-in.yaml -o %t/memprof-out.indexed
+; RUN: llvm-profdata show --memory %t/memprof-out.indexed > %t/memprof-out.yaml
+; RUN: cmp %t/memprof-in.yaml %t/memprof-out.yaml
+
+; Verify that the YAML output is identical to the YAML input.
+;--- memprof-in.yaml
+---
+HeapProfileRecords:
+  - GUID:            0xdeadbeef12345678
+    AllocSites:
+      - Callstack:
+          - { Function: 100, LineOffset: 11, Column: 10, IsInlineFrame: true }
+          - { Function: 200, LineOffset: 22, Column: 20, IsInlineFrame: false }
+        MemInfoBlock:
+          AllocCount:      111
+          TotalSize:       222
+          TotalLifetime:   333
+          TotalLifetimeAccessDensity: 444
+      - Callstack:
+          - { Function: 300, LineOffset: 33, Column: 30, IsInlineFrame: false }
+          - { Function: 400, LineOffset: 44, Column: 40, IsInlineFrame: true }
+        MemInfoBlock:
+          AllocCount:      555
+          TotalSize:       666
+          TotalLifetime:   777
+          TotalLifetimeAccessDensity: 888
+    CallSites:
+      - - { Function: 500, LineOffset: 55, Column: 50, IsInlineFrame: true }
+        - { Function: 600, LineOffset: 66, Column: 60, IsInlineFrame: false }
+      - - { Function: 700, LineOffset: 77, Column: 70, IsInlineFrame: true }
+        - { Function: 800, LineOffset: 88, Column: 80, IsInlineFrame: false }
+...
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
index 458761a2ba6b4..d7fe77cf658af 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
@@ -120,6 +120,23 @@
 # CHECK: Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12 (572)
 # CHECK: Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC (573)
 # CHECK: Type: R_AARCH64_AUTH_ABS64 (580)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0 (581)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0_NC (582)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G1 (583)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G1_NC (584)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G2 (585)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G2_NC (586)
+# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G3 (587)
+# CHECK: Type: R_AARCH64_AUTH_GOT_LD_PREL19 (588)
+# CHECK: Type: R_AARCH64_AUTH_LD64_GOTOFF_LO15 (589)
+# CHECK: Type: R_AARCH64_AUTH_ADR_GOT_PAGE (590)
+# CHECK: Type: R_AARCH64_AUTH_LD64_GOT_LO12_NC (591)
+# CHECK: Type: R_AARCH64_AUTH_LD64_GOTPAGE_LO15 (592)
+# CHECK: Type: R_AARCH64_AUTH_GOT_ADD_LO12_NC (593)
+# CHECK: Type: R_AARCH64_AUTH_GOT_ADR_PREL_LO21 (594)
+# CHECK: Type: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 (595)
+# CHECK: Type: R_AARCH64_AUTH_TLSDESC_LD64_LO12 (596)
+# CHECK: Type: R_AARCH64_AUTH_TLSDESC_ADD_LO12 (597)
 # CHECK: Type: R_AARCH64_COPY (1024)
 # CHECK: Type: R_AARCH64_GLOB_DAT (1025)
 # CHECK: Type: R_AARCH64_JUMP_SLOT (1026)
@@ -130,26 +147,9 @@
 # CHECK: Type: R_AARCH64_TLSDESC (1031)
 # CHECK: Type: R_AARCH64_IRELATIVE (1032)
 # CHECK: Type: R_AARCH64_AUTH_RELATIVE (1041)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0 (33040)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0_NC (33041)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G1 (33042)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G1_NC (33043)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G2 (33044)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G2_NC (33045)
-# CHECK: Type: R_AARCH64_AUTH_MOVW_GOTOFF_G3 (33046)
-# CHECK: Type: R_AARCH64_AUTH_GOT_LD_PREL19 (33047)
-# CHECK: Type: R_AARCH64_AUTH_LD64_GOTOFF_LO15 (33048)
-# CHECK: Type: R_AARCH64_AUTH_ADR_GOT_PAGE (33049)
-# CHECK: Type: R_AARCH64_AUTH_LD64_GOT_LO12_NC (33050)
-# CHECK: Type: R_AARCH64_AUTH_LD64_GOTPAGE_LO15 (33051)
-# CHECK: Type: R_AARCH64_AUTH_GOT_ADD_LO12_NC (33052)
-# CHECK: Type: R_AARCH64_AUTH_GOT_ADR_PREL_LO21 (33053)
-# CHECK: Type: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 (33054)
-# CHECK: Type: R_AARCH64_AUTH_TLSDESC_LD64_LO12 (33055)
-# CHECK: Type: R_AARCH64_AUTH_TLSDESC_ADD_LO12 (33056)
-# CHECK: Type: R_AARCH64_AUTH_GLOB_DAT (57857)
-# CHECK: Type: R_AARCH64_AUTH_TLSDESC (57858)
-# CHECK: Type: R_AARCH64_AUTH_IRELATIVE (57859)
+# CHECK: Type: R_AARCH64_AUTH_GLOB_DAT (1042)
+# CHECK: Type: R_AARCH64_AUTH_TLSDESC (1043)
+# CHECK: Type: R_AARCH64_AUTH_IRELATIVE (1044)
 
 --- !ELF
 FileHeader:
@@ -277,16 +277,6 @@ Sections:
       - Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12
       - Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC
       - Type: R_AARCH64_AUTH_ABS64
-      - Type: R_AARCH64_COPY
-      - Type: R_AARCH64_GLOB_DAT
-      - Type: R_AARCH64_JUMP_SLOT
-      - Type: R_AARCH64_RELATIVE
-      - Type: R_AARCH64_TLS_DTPMOD64
-      - Type: R_AARCH64_TLS_DTPREL64
-      - Type: R_AARCH64_TLS_TPREL64
-      - Type: R_AARCH64_TLSDESC
-      - Type: R_AARCH64_IRELATIVE
-      - Type: R_AARCH64_AUTH_RELATIVE
       - Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0
       - Type: R_AARCH64_AUTH_MOVW_GOTOFF_G0_NC
       - Type: R_AARCH64_AUTH_MOVW_GOTOFF_G1
@@ -304,6 +294,16 @@ Sections:
       - Type: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21
       - Type: R_AARCH64_AUTH_TLSDESC_LD64_LO12
       - Type: R_AARCH64_AUTH_TLSDESC_ADD_LO12
+      - Type: R_AARCH64_COPY
+      - Type: R_AARCH64_GLOB_DAT
+      - Type: R_AARCH64_JUMP_SLOT
+      - Type: R_AARCH64_RELATIVE
+      - Type: R_AARCH64_TLS_DTPMOD64
+      - Type: R_AARCH64_TLS_DTPREL64
+      - Type: R_AARCH64_TLS_TPREL64
+      - Type: R_AARCH64_TLSDESC
+      - Type: R_AARCH64_IRELATIVE
+      - Type: R_AARCH64_AUTH_RELATIVE
       - Type: R_AARCH64_AUTH_GLOB_DAT
       - Type: R_AARCH64_AUTH_TLSDESC
       - Type: R_AARCH64_AUTH_IRELATIVE
diff --git a/llvm/test/tools/llvm-readobj/XCOFF/symbols-invalid.test b/llvm/test/tools/llvm-readobj/XCOFF/symbols-invalid.test
index 3db8803149242..756417834b424 100644
--- a/llvm/test/tools/llvm-readobj/XCOFF/symbols-invalid.test
+++ b/llvm/test/tools/llvm-readobj/XCOFF/symbols-invalid.test
@@ -41,7 +41,7 @@ Symbols:
 # CASE5-NEXT:    Value (SymbolTableIndex): 0x0
 # CASE5-NEXT:    Section: N_UNDEF
 # CASE5-NEXT:    Source Language ID: TB_C (0x0)
-# CASE5-NEXT:    CPU Version ID: 0x0
+# CASE5-NEXT:    CPU Version ID: TCPU_INVALID (0x0)
 # CASE5-NEXT:    StorageClass: C_FILE (0x67)
 # CASE5-NEXT:    NumberOfAuxEntries: 1
 # CASE5-NEXT:    !Unexpected raw auxiliary entry data:
diff --git a/llvm/test/tools/llvm-readobj/XCOFF/symbols.test b/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
index 89439a3d0f02d..71347a85f1ba5 100644
--- a/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
+++ b/llvm/test/tools/llvm-readobj/XCOFF/symbols.test
@@ -170,7 +170,7 @@ Symbols:
 # SYMBOL32-NEXT:     Value (SymbolTableIndex): 0x0
 # SYMBOL32-NEXT:     Section: N_DEBUG
 # SYMBOL32-NEXT:     Source Language ID: TB_C (0x0)
-# SYMBOL32-NEXT:     CPU Version ID: 0x0
+# SYMBOL32-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # SYMBOL32-NEXT:     StorageClass: C_FILE (0x67)
 # SYMBOL32-NEXT:     NumberOfAuxEntries: 3
 # SYMBOL32-NEXT:     File Auxiliary Entry {
diff --git a/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test b/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
index 591c131063fe3..2b9edb3829af8 100644
--- a/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
+++ b/llvm/test/tools/llvm-readobj/XCOFF/symbols64.test
@@ -145,7 +145,7 @@ Symbols:
 # SYMBOL64-NEXT:     Value (SymbolTableIndex): 0x0
 # SYMBOL64-NEXT:     Section: N_DEBUG
 # SYMBOL64-NEXT:     Source Language ID: TB_C (0x0)
-# SYMBOL64-NEXT:     CPU Version ID: 0x0
+# SYMBOL64-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # SYMBOL64-NEXT:     StorageClass: C_FILE (0x67)
 # SYMBOL64-NEXT:     NumberOfAuxEntries: 3
 # SYMBOL64-NEXT:     File Auxiliary Entry {
diff --git a/llvm/test/tools/llvm-split/AMDGPU/large-kernels-merging-weak_odr.ll b/llvm/test/tools/llvm-split/AMDGPU/large-kernels-merging-weak_odr.ll
new file mode 100644
index 0000000000000..839688e7feb8b
--- /dev/null
+++ b/llvm/test/tools/llvm-split/AMDGPU/large-kernels-merging-weak_odr.ll
@@ -0,0 +1,94 @@
+; RUN: llvm-split -o %t %s -j 3 -mtriple amdgcn-amd-amdhsa -amdgpu-module-splitting-max-depth=0 -amdgpu-module-splitting-large-threshold=1.2 -amdgpu-module-splitting-merge-threshold=0.5
+; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t2 | FileCheck --check-prefix=CHECK2 --implicit-check-not=define %s
+
+; RUN: llvm-split -o %t.nolarge %s -j 3 -mtriple amdgcn-amd-amdhsa -amdgpu-module-splitting-large-threshold=0 -amdgpu-module-splitting-max-depth=0
+; RUN: llvm-dis -o - %t.nolarge0 | FileCheck --check-prefix=NOLARGEKERNELS-CHECK0 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t.nolarge1 | FileCheck --check-prefix=NOLARGEKERNELS-CHECK1 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t.nolarge2 | FileCheck --check-prefix=NOLARGEKERNELS-CHECK2 --implicit-check-not=define %s
+
+; 2 kernels (A/B) are large and share all their dependencies.
+; They should go in the same partition, the remaining kernel should
+; go somewhere else, and one partition should be empty.
+;
+; Also check w/o large kernels processing to verify they are indeed handled
+; differently.
+
+; P0 is empty
+; CHECK0: declare
+
+; CHECK1: define internal void @HelperC()
+; CHECK1: define weak_odr amdgpu_kernel void @C
+
+; CHECK2: define internal void @large2()
+; CHECK2: define internal void @large1()
+; CHECK2: define internal void @large0()
+; CHECK2: define internal void @HelperA()
+; CHECK2: define internal void @HelperB()
+; CHECK2: define amdgpu_kernel void @A
+; CHECK2: define weak_odr amdgpu_kernel void @B
+
+; NOLARGEKERNELS-CHECK0: define internal void @HelperC()
+; NOLARGEKERNELS-CHECK0: define weak_odr amdgpu_kernel void @C
+
+; NOLARGEKERNELS-CHECK1: define internal void @large2()
+; NOLARGEKERNELS-CHECK1: define internal void @large1()
+; NOLARGEKERNELS-CHECK1: define internal void @large0()
+; NOLARGEKERNELS-CHECK1: define internal void @HelperB()
+; NOLARGEKERNELS-CHECK1: define weak_odr amdgpu_kernel void @B
+
+; NOLARGEKERNELS-CHECK2: define internal void @large2()
+; NOLARGEKERNELS-CHECK2: define internal void @large1()
+; NOLARGEKERNELS-CHECK2: define internal void @large0()
+; NOLARGEKERNELS-CHECK2: define internal void @HelperA()
+; NOLARGEKERNELS-CHECK2: define amdgpu_kernel void @A
+
+
+define internal void @large2() {
+  store volatile i32 42, ptr null
+  call void @large2()
+  ret void
+}
+
+define internal void @large1() {
+  call void @large1()
+  call void @large2()
+  ret void
+}
+
+define internal void @large0() {
+  call void @large0()
+  call void @large1()
+  call void @large2()
+  ret void
+}
+
+define internal void @HelperA() {
+  call void @large0()
+  ret void
+}
+
+define internal void @HelperB() {
+  call void @large0()
+  ret void
+}
+
+define amdgpu_kernel void @A() {
+  call void @HelperA()
+  ret void
+}
+
+define weak_odr amdgpu_kernel void @B() {
+  call void @HelperB()
+  ret void
+}
+
+define internal void @HelperC() {
+  ret void
+}
+
+define weak_odr amdgpu_kernel void @C() {
+  call void @HelperC()
+  ret void
+}
diff --git a/llvm/test/tools/yaml2obj/ELF/note-section.yaml b/llvm/test/tools/yaml2obj/ELF/note-section.yaml
index 80359c4ec0183..10fd36fefeabf 100644
--- a/llvm/test/tools/yaml2obj/ELF/note-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/note-section.yaml
@@ -333,3 +333,184 @@ Sections:
       - Name: ABC
         Desc: '123456'
         Type: NT_VERSION
+
+## Check that an incorrect alignment is reported.
+
+# RUN: not yaml2obj --docnum=16 %s 2>&1 | FileCheck %s --check-prefix=ERR_ALIGN1
+# ERR_ALIGN1: error: .note.foo: invalid alignment for a note section: 0x1
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+Sections:
+  - Name:         .note.foo
+    Type:         SHT_NOTE
+    AddressAlign: 1
+    Notes:
+      - Type: 0x1
+
+## Check that note entries and their "Desc" fields are aligned according to the
+## specified section alignment.
+
+# RUN: yaml2obj --docnum=17 -D ELFCLASS=64 %s -o - | \
+# RUN:   llvm-readobj --sections --section-data --notes - | \
+# RUN:   FileCheck %s --check-prefix=TEST17
+
+# RUN: yaml2obj --docnum=17 -D ELFCLASS=32 %s -o - | \
+# RUN:   llvm-readobj --sections --section-data --notes - | \
+# RUN:   FileCheck %s --check-prefix=TEST17
+
+# TEST17:      Name: .note.foo4
+# TEST17:      SectionData (
+# TEST17-NEXT:   0000: 05000000 02000000 01000000 41424344  |............ABCD|
+# TEST17-NEXT:   0010: 00000000 01020000 00000000 03000000  |................|
+# TEST17-NEXT:   0020: 02000000 03040500 04000000 00000000  |................|
+# TEST17-NEXT:   0030: 03000000 474E5500                    |....GNU.|
+# TEST17-NEXT: )
+# TEST17:      Name: .note.foo8
+# TEST17:      SectionData (
+# TEST17-NEXT:   0000: 05000000 02000000 01000000 41424344  |............ABCD|
+# TEST17-NEXT:   0010: 00000000 00000000 01020000 00000000  |................|
+# TEST17-NEXT:   0020: 00000000 03000000 02000000 00000000  |................|
+# TEST17-NEXT:   0030: 03040500 00000000 04000000 00000000  |................|
+# TEST17-NEXT:   0040: 03000000 474E5500                    |....GNU.|
+# TEST17-NEXT: )
+# TEST17:      NoteSections [
+# TEST17-NEXT:   NoteSection {
+# TEST17-NEXT:     Name: .note.foo4
+# TEST17-NEXT:     Offset:
+# TEST17-NEXT:     Size:
+# TEST17-NEXT:     Notes [
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: ABCD
+# TEST17-NEXT:         Data size: 0x2
+# TEST17-NEXT:         Type: NT_VERSION (version)
+# TEST17-NEXT:         Description data (
+# TEST17-NEXT:           0000: 0102                                 |..|
+# TEST17-NEXT:         )
+# TEST17-NEXT:       }
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: 
+# TEST17-NEXT:         Data size: 0x3
+# TEST17-NEXT:         Type: NT_ARCH (architecture)
+# TEST17-NEXT:         Description data (
+# TEST17-NEXT:           0000: 030405                               |...|
+# TEST17-NEXT:         )
+# TEST17-NEXT:       }
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: GNU
+# TEST17-NEXT:         Data size: 0x0
+# TEST17-NEXT:         Type: NT_GNU_BUILD_ID (unique build ID bitstring)
+# TEST17-NEXT:         Build ID: 
+# TEST17-NEXT:       }
+# TEST17-NEXT:     ]
+# TEST17-NEXT:   }
+# TEST17-NEXT:   NoteSection {
+# TEST17-NEXT:     Name: .note.foo8
+# TEST17-NEXT:     Offset:
+# TEST17-NEXT:     Size:
+# TEST17-NEXT:     Notes [
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: ABCD
+# TEST17-NEXT:         Data size: 0x2
+# TEST17-NEXT:         Type: NT_VERSION (version)
+# TEST17-NEXT:         Description data (
+# TEST17-NEXT:           0000: 0102                                 |..|
+# TEST17-NEXT:         )
+# TEST17-NEXT:       }
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: 
+# TEST17-NEXT:         Data size: 0x3
+# TEST17-NEXT:         Type: NT_ARCH (architecture)
+# TEST17-NEXT:         Description data (
+# TEST17-NEXT:           0000: 030405                               |...|
+# TEST17-NEXT:         )
+# TEST17-NEXT:       }
+# TEST17-NEXT:       {
+# TEST17-NEXT:         Owner: GNU
+# TEST17-NEXT:         Data size: 0x0
+# TEST17-NEXT:         Type: NT_GNU_BUILD_ID (unique build ID bitstring)
+# TEST17-NEXT:         Build ID: 
+# TEST17-NEXT:       }
+# TEST17-NEXT:     ]
+# TEST17-NEXT:   }
+# TEST17-NEXT: ]
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS[[ELFCLASS]]
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+Sections:
+  - Name:         .note.foo4
+    Type:         SHT_NOTE
+    AddressAlign: 4
+    Notes:
+      - Name: ABCD
+        Type: NT_VERSION
+        Desc: 0102
+      - Type: NT_ARCH
+        Desc: 030405
+      - Name: GNU
+        Type: NT_GNU_BUILD_ID
+  - Name:        .note.foo8
+    Type:         SHT_NOTE
+    AddressAlign: 8
+    Notes:
+      - Name: ABCD
+        Type: NT_VERSION
+        Desc: 0102
+      - Type: NT_ARCH
+        Desc: 030405
+      - Name: GNU
+        Type: NT_GNU_BUILD_ID
+
+## Check that the alignment for note entries is taken from the "AddressAlign"
+## field even if "ShAddrAlign" is also specified; an unexpected value in the
+## "ShAddrAlign" property does not trigger an incorrect alignment error.
+
+# RUN: yaml2obj --docnum=18 -D ADDRALIGN=0 -D SHADDRALIGN=8 %s -o - | \
+# RUN:   llvm-readobj --sections --section-data --notes - | \
+# RUN:   FileCheck %s --check-prefixes=TEST18,TEST18_4
+
+# RUN: yaml2obj --docnum=18 -D ADDRALIGN=4 -D SHADDRALIGN=3 %s -o - | \
+# RUN:   llvm-readobj --sections --section-data --notes - | \
+# RUN:   FileCheck %s --check-prefixes=TEST18,TEST18_4
+
+# RUN: yaml2obj --docnum=18 -D ADDRALIGN=8 -D SHADDRALIGN=4 %s -o - | \
+# RUN:   llvm-readobj --sections --section-data --notes - | \
+# RUN:   FileCheck %s --check-prefixes=TEST18,TEST18_8
+
+# TEST18:        Name: .note
+# TEST18:        SectionData (
+# TEST18_4-NEXT:   0000: 05000000 02000000 01000000 41424344  |............ABCD|
+# TEST18_4-NEXT:   0010: 00000000 01020000 00000000 03000000  |................|
+# TEST18_4-NEXT:   0020: 02000000 03040500 04000000 00000000  |................|
+# TEST18_4-NEXT:   0030: 03000000 474E5500                    |....GNU.|
+# TEST18_8-NEXT:   0000: 05000000 02000000 01000000 41424344  |............ABCD|
+# TEST18_8-NEXT:   0010: 00000000 00000000 01020000 00000000  |................|
+# TEST18_8-NEXT:   0020: 00000000 03000000 02000000 00000000  |................|
+# TEST18_8-NEXT:   0030: 03040500 00000000 04000000 00000000  |................|
+# TEST18_8-NEXT:   0040: 03000000 474E5500                    |....GNU.|
+# TEST18-NEXT:   )
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+Sections:
+  - Name:         .note
+    Type:         SHT_NOTE
+    AddressAlign: [[ADDRALIGN]]
+    ShAddrAlign:  [[SHADDRALIGN]]
+    Notes:
+      - Name: ABCD
+        Type: NT_VERSION
+        Desc: 0102
+      - Type: NT_ARCH
+        Desc: 030405
+      - Name: GNU
+        Type: NT_GNU_BUILD_ID
diff --git a/llvm/test/tools/yaml2obj/XCOFF/aux-symbols.yaml b/llvm/test/tools/yaml2obj/XCOFF/aux-symbols.yaml
index 04c774dcc3ae2..ab9478de8f215 100644
--- a/llvm/test/tools/yaml2obj/XCOFF/aux-symbols.yaml
+++ b/llvm/test/tools/yaml2obj/XCOFF/aux-symbols.yaml
@@ -69,7 +69,7 @@
 # DEFAULT32-NEXT:     Value (SymbolTableIndex): 0x0
 # DEFAULT32-NEXT:     Section: N_UNDEF
 # DEFAULT32-NEXT:     Source Language ID: TB_C (0x0)
-# DEFAULT32-NEXT:     CPU Version ID: 0x0
+# DEFAULT32-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # DEFAULT32-NEXT:     StorageClass: C_FILE (0x67)
 # DEFAULT32-NEXT:     NumberOfAuxEntries: 1
 # DEFAULT32-NEXT:     File Auxiliary Entry {
@@ -213,7 +213,7 @@ Symbols:
 # NON-DEFAULT32-NEXT:     Value (SymbolTableIndex): 0x0
 # NON-DEFAULT32-NEXT:     Section: N_UNDEF
 # NON-DEFAULT32-NEXT:     Source Language ID: TB_C (0x0)
-# NON-DEFAULT32-NEXT:     CPU Version ID: 0x0
+# NON-DEFAULT32-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # NON-DEFAULT32-NEXT:     StorageClass: C_FILE (0x67)
 # NON-DEFAULT32-NEXT:     NumberOfAuxEntries: 1
 # NON-DEFAULT32-NEXT:     File Auxiliary Entry {
@@ -296,7 +296,7 @@ Symbols:
 # DEFAULT64-NEXT:     Value (SymbolTableIndex): 0x0
 # DEFAULT64-NEXT:     Section: N_UNDEF
 # DEFAULT64-NEXT:     Source Language ID: TB_C (0x0)
-# DEFAULT64-NEXT:     CPU Version ID: 0x0
+# DEFAULT64-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # DEFAULT64-NEXT:     StorageClass: C_FILE (0x67)
 # DEFAULT64-NEXT:     NumberOfAuxEntries: 1
 # DEFAULT64-NEXT:     File Auxiliary Entry {
@@ -426,7 +426,7 @@ Symbols:
 # NON-DEFAULT64-NEXT:     Value (SymbolTableIndex): 0x0
 # NON-DEFAULT64-NEXT:     Section: N_UNDEF
 # NON-DEFAULT64-NEXT:     Source Language ID: TB_C (0x0)
-# NON-DEFAULT64-NEXT:     CPU Version ID: 0x0
+# NON-DEFAULT64-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # NON-DEFAULT64-NEXT:     StorageClass: C_FILE (0x67)
 # NON-DEFAULT64-NEXT:     NumberOfAuxEntries: 1
 # NON-DEFAULT64-NEXT:     File Auxiliary Entry {
@@ -466,7 +466,7 @@ Symbols:
 # FILENAME-NEXT:     Value (SymbolTableIndex): 0x0
 # FILENAME-NEXT:     Section: N_UNDEF
 # FILENAME-NEXT:     Source Language ID: TB_C (0x0)
-# FILENAME-NEXT:     CPU Version ID: 0x0
+# FILENAME-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # FILENAME-NEXT:     StorageClass: C_FILE (0x67)
 # FILENAME-NEXT:     NumberOfAuxEntries: 1
 # FILENAME-NEXT:     File Auxiliary Entry {
@@ -554,7 +554,7 @@ Symbols:
 # AUXNUM-NEXT:     Value (SymbolTableIndex): 0x0
 # AUXNUM-NEXT:     Section: N_UNDEF
 # AUXNUM-NEXT:     Source Language ID: TB_C (0x0)
-# AUXNUM-NEXT:     CPU Version ID: 0x0
+# AUXNUM-NEXT:     CPU Version ID: TCPU_INVALID (0x0)
 # AUXNUM-NEXT:     StorageClass: C_FILE (0x67)
 # AUXNUM-NEXT:     NumberOfAuxEntries: 2
 # AUXNUM-NEXT:     File Auxiliary Entry {
diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp
index 2ace7180b0080..594b52326871d 100644
--- a/llvm/tools/dsymutil/dsymutil.cpp
+++ b/llvm/tools/dsymutil/dsymutil.cpp
@@ -832,15 +832,15 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) {
       return EXIT_FAILURE;
 
     if (NeedsTempFiles) {
-      const bool Fat64 = Options.LinkOpts.Fat64;
+      bool Fat64 = Options.LinkOpts.Fat64;
       if (!Fat64) {
         // Universal Mach-O files can't have an archicture slice that starts
         // beyond the 4GB boundary. "lipo" can create a 64 bit universal
-        // header, but not all tools can parse these files so we want to return
-        // an error if the file can't be encoded as a file with a 32 bit
+        // header, but older tools may not support these files so we want to
+        // emit a warning if the file can't be encoded as a file with a 32 bit
         // universal header. To detect this, we check the size of each
         // architecture's skinny Mach-O file and add up the offsets. If they
-        // exceed 4GB, then we return an error.
+        // exceed 4GB, we emit a warning.
 
         // First we compute the right offset where the first architecture will
         // fit followin the 32 bit universal header. The 32 bit universal header
@@ -859,13 +859,15 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) {
           if (!stat)
             break;
           if (FileOffset > UINT32_MAX) {
-            WithColor::error()
-                << formatv("the universal binary has a slice with a starting "
-                           "offset ({0:x}) that exceeds 4GB and will produce "
-                           "an invalid Mach-O file. Use the -fat64 flag to "
-                           "generate a universal binary with a 64-bit header "
-                           "but note that not all tools support this format.",
-                           FileOffset);
+            Fat64 = true;
+            WithColor::warning() << formatv(
+                "the universal binary has a slice with a starting offset "
+                "({0:x}) that exceeds 4GB. To avoid producing an invalid "
+                "Mach-O file, a universal binary with a 64-bit header will be "
+                "generated, which may not be supported by older tools. Use the "
+                "-fat64 flag to force a 64-bit header and silence this "
+                "warning.",
+                FileOffset);
             return EXIT_FAILURE;
           }
           FileOffset += stat->getSize();
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index ccc152dc753bc..3c58b8934462a 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -30,6 +30,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
+#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"
 #include "llvm/ExecutionEngine/Orc/MachO.h"
@@ -365,7 +366,7 @@ static raw_ostream &
 operator<<(raw_ostream &OS, const Session::SymbolInfoMap &SIM) {
   OS << "Symbols:\n";
   for (auto &SKV : SIM)
-    OS << "  \"" << SKV.first() << "\" " << SKV.second << "\n";
+    OS << "  \"" << SKV.first << "\" " << SKV.second << "\n";
   return OS;
 }
 
@@ -416,8 +417,8 @@ static Error applyHarnessPromotions(Session &S, LinkGraph &G) {
       continue;
 
     if (Sym->getLinkage() == Linkage::Weak) {
-      if (!S.CanonicalWeakDefs.count(Sym->getName()) ||
-          S.CanonicalWeakDefs[Sym->getName()] != G.getName()) {
+      if (!S.CanonicalWeakDefs.count(*Sym->getName()) ||
+          S.CanonicalWeakDefs[*Sym->getName()] != G.getName()) {
         LLVM_DEBUG({
           dbgs() << "  Externalizing weak symbol " << Sym->getName() << "\n";
         });
@@ -426,18 +427,18 @@ static Error applyHarnessPromotions(Session &S, LinkGraph &G) {
         LLVM_DEBUG({
           dbgs() << "  Making weak symbol " << Sym->getName() << " strong\n";
         });
-        if (S.HarnessExternals.count(Sym->getName()))
+        if (S.HarnessExternals.count(*Sym->getName()))
           Sym->setScope(Scope::Default);
         else
           Sym->setScope(Scope::Hidden);
         Sym->setLinkage(Linkage::Strong);
       }
-    } else if (S.HarnessExternals.count(Sym->getName())) {
+    } else if (S.HarnessExternals.count(*Sym->getName())) {
       LLVM_DEBUG(dbgs() << "  Promoting " << Sym->getName() << "\n");
       Sym->setScope(Scope::Default);
       Sym->setLive(true);
       continue;
-    } else if (S.HarnessDefinitions.count(Sym->getName())) {
+    } else if (S.HarnessDefinitions.count(*Sym->getName())) {
       LLVM_DEBUG(dbgs() << "  Externalizing " << Sym->getName() << "\n");
       DefinitionsToRemove.push_back(Sym);
     }
@@ -949,41 +950,18 @@ class PhonyExternalsGenerator : public DefinitionGenerator {
   }
 };
 
-static void handleLazyCallFailure() {
-  dbgs() << "ERROR: failure to materialize lazy call-through target.\n";
-  exit(1);
-}
-
-static void *reenter(void *Ctx, void *TrampolineAddr) {
-  std::promise<void *> LandingAddressP;
-  auto LandingAddressF = LandingAddressP.get_future();
-
-  auto *EPCIU = static_cast<EPCIndirectionUtils *>(Ctx);
-  EPCIU->getLazyCallThroughManager().resolveTrampolineLandingAddress(
-      ExecutorAddr::fromPtr(TrampolineAddr), [&](ExecutorAddr LandingAddress) {
-        LandingAddressP.set_value(LandingAddress.toPtr<void *>());
-      });
-  return LandingAddressF.get();
-}
-
 Expected<std::unique_ptr<Session::LazyLinkingSupport>>
-createLazyLinkingSupport(ObjectLinkingLayer &OLL) {
-  auto EPCIU = EPCIndirectionUtils::Create(OLL.getExecutionSession());
-  if (!EPCIU)
-    return EPCIU.takeError();
-  if (auto Err = (*EPCIU)
-                     ->writeResolverBlock(ExecutorAddr::fromPtr(&reenter),
-                                          ExecutorAddr::fromPtr(EPCIU->get()))
-                     .takeError())
-    return Err;
-  (*EPCIU)->createLazyCallThroughManager(
-      OLL.getExecutionSession(), ExecutorAddr::fromPtr(handleLazyCallFailure));
+createLazyLinkingSupport(ObjectLinkingLayer &OLL, JITDylib &PlatformJD) {
   auto RSMgr = JITLinkRedirectableSymbolManager::Create(OLL);
   if (!RSMgr)
     return RSMgr.takeError();
 
-  return std::make_unique<Session::LazyLinkingSupport>(std::move(*EPCIU),
-                                                       std::move(*RSMgr), OLL);
+  auto LRMgr = createJITLinkLazyReexportsManager(OLL, **RSMgr, PlatformJD);
+  if (!LRMgr)
+    return LRMgr.takeError();
+
+  return std::make_unique<Session::LazyLinkingSupport>(std::move(*RSMgr),
+                                                       std::move(*LRMgr), OLL);
 }
 
 Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
@@ -1020,7 +998,8 @@ Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
   S->Features = std::move(Features);
 
   if (lazyLinkingRequested()) {
-    if (auto LazyLinking = createLazyLinkingSupport(S->ObjLayer))
+    if (auto LazyLinking =
+            createLazyLinkingSupport(S->ObjLayer, *S->PlatformJD))
       S->LazyLinking = std::move(*LazyLinking);
     else
       return LazyLinking.takeError();
@@ -1306,9 +1285,9 @@ Error Session::FileInfo::registerGOTEntry(
   auto TS = GetSymbolTarget(G, Sym.getBlock());
   if (!TS)
     return TS.takeError();
-  GOTEntryInfos[TS->getName()] = {Sym.getSymbolContent(),
-                                  Sym.getAddress().getValue(),
-                                  Sym.getTargetFlags()};
+  GOTEntryInfos[*TS->getName()] = {Sym.getSymbolContent(),
+                                   Sym.getAddress().getValue(),
+                                   Sym.getTargetFlags()};
   return Error::success();
 }
 
@@ -1322,7 +1301,7 @@ Error Session::FileInfo::registerStubEntry(
   if (!TS)
     return TS.takeError();
 
-  SmallVectorImpl<MemoryRegionInfo> &Entry = StubInfos[TS->getName()];
+  SmallVectorImpl<MemoryRegionInfo> &Entry = StubInfos[*TS->getName()];
   Entry.insert(Entry.begin(),
                {Sym.getSymbolContent(), Sym.getAddress().getValue(),
                 Sym.getTargetFlags()});
@@ -1340,7 +1319,7 @@ Error Session::FileInfo::registerMultiStubEntry(
   if (!Target)
     return Target.takeError();
 
-  SmallVectorImpl<MemoryRegionInfo> &Entry = StubInfos[Target->getName()];
+  SmallVectorImpl<MemoryRegionInfo> &Entry = StubInfos[*Target->getName()];
   Entry.emplace_back(Sym.getSymbolContent(), Sym.getAddress().getValue(),
                      Sym.getTargetFlags());
 
@@ -1485,15 +1464,16 @@ Session::findGOTEntryInfo(StringRef FileName, StringRef TargetName) {
   return GOTInfoItr->second;
 }
 
-bool Session::isSymbolRegistered(StringRef SymbolName) {
+bool Session::isSymbolRegistered(const orc::SymbolStringPtr &SymbolName) {
   return SymbolInfos.count(SymbolName);
 }
 
 Expected<Session::MemoryRegionInfo &>
-Session::findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem) {
+Session::findSymbolInfo(const orc::SymbolStringPtr &SymbolName,
+                        Twine ErrorMsgStem) {
   auto SymInfoItr = SymbolInfos.find(SymbolName);
   if (SymInfoItr == SymbolInfos.end())
-    return make_error<StringError>(ErrorMsgStem + ": symbol " + SymbolName +
+    return make_error<StringError>(ErrorMsgStem + ": symbol " + *SymbolName +
                                        " not found",
                                    inconvertibleErrorCode());
   return SymInfoItr->second;
@@ -1641,10 +1621,17 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {
     OutOfProcessExecutor = OOPExecutorPath.str().str();
   }
 
-  if (lazyLinkingRequested() && !TestHarnesses.empty())
-    return make_error<StringError>(
-        "Lazy linking cannot be used with -harness mode",
-        inconvertibleErrorCode());
+  // If lazy linking is requested then check compatibility with other options.
+  if (lazyLinkingRequested()) {
+    if (OrcRuntime.empty())
+      return make_error<StringError>("Lazy linking requries the ORC runtime",
+                                     inconvertibleErrorCode());
+
+    if (!TestHarnesses.empty())
+      return make_error<StringError>(
+          "Lazy linking cannot be used with -harness mode",
+          inconvertibleErrorCode());
+  }
 
   return Error::success();
 }
@@ -1719,12 +1706,13 @@ static Error addAbsoluteSymbols(Session &S,
                                          AbsDefStmt + "\"",
                                      inconvertibleErrorCode());
     ExecutorSymbolDef AbsDef(ExecutorAddr(Addr), JITSymbolFlags::Exported);
-    if (auto Err = JD.define(absoluteSymbols({{S.ES.intern(Name), AbsDef}})))
+    auto InternedName = S.ES.intern(Name);
+    if (auto Err = JD.define(absoluteSymbols({{InternedName, AbsDef}})))
       return Err;
 
     // Register the absolute symbol with the session symbol infos.
-    S.SymbolInfos[Name] = {ArrayRef<char>(), Addr,
-                           AbsDef.getFlags().getTargetFlags()};
+    S.SymbolInfos[InternedName] = {ArrayRef<char>(), Addr,
+                                   AbsDef.getFlags().getTargetFlags()};
   }
 
   return Error::success();
@@ -2330,11 +2318,13 @@ static Error runChecks(Session &S, Triple TT, SubtargetFeatures Features) {
   LLVM_DEBUG(dbgs() << "Running checks...\n");
 
   auto IsSymbolValid = [&S](StringRef Symbol) {
-    return S.isSymbolRegistered(Symbol);
+    auto InternedSymbol = S.ES.getSymbolStringPool()->intern(Symbol);
+    return S.isSymbolRegistered(InternedSymbol);
   };
 
   auto GetSymbolInfo = [&S](StringRef Symbol) {
-    return S.findSymbolInfo(Symbol, "Can not get symbol info");
+    auto InternedSymbol = S.ES.getSymbolStringPool()->intern(Symbol);
+    return S.findSymbolInfo(InternedSymbol, "Can not get symbol info");
   };
 
   auto GetSectionInfo = [&S](StringRef FileName, StringRef SectionName) {
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h
index 40fac2fe6888f..bfad5211c2176 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.h
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h
@@ -15,9 +15,9 @@
 
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
 #include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
@@ -33,20 +33,14 @@ namespace llvm {
 struct Session {
 
   struct LazyLinkingSupport {
-    LazyLinkingSupport(std::unique_ptr<orc::EPCIndirectionUtils> EPCIU,
-                       std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
+    LazyLinkingSupport(std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
+                       std::unique_ptr<orc::LazyReexportsManager> LRMgr,
                        orc::ObjectLinkingLayer &ObjLinkingLayer)
-        : EPCIU(std::move(EPCIU)), RSMgr(std::move(RSMgr)),
-          LazyObjLinkingLayer(ObjLinkingLayer,
-                              this->EPCIU->getLazyCallThroughManager(),
-                              *this->RSMgr) {}
-    ~LazyLinkingSupport() {
-      if (auto Err = EPCIU->cleanup())
-        LazyObjLinkingLayer.getExecutionSession().reportError(std::move(Err));
-    }
-
-    std::unique_ptr<orc::EPCIndirectionUtils> EPCIU;
+        : RSMgr(std::move(RSMgr)), LRMgr(std::move(LRMgr)),
+          LazyObjLinkingLayer(ObjLinkingLayer, *this->LRMgr) {}
+
     std::unique_ptr<orc::RedirectableSymbolManager> RSMgr;
+    std::unique_ptr<orc::LazyReexportsManager> LRMgr;
     orc::LazyObjectLinkingLayer LazyObjLinkingLayer;
   };
 
@@ -78,7 +72,6 @@ struct Session {
     using LinkGraph = jitlink::LinkGraph;
     using GetSymbolTargetFunction =
         unique_function<Expected<Symbol &>(LinkGraph &G, jitlink::Block &)>;
-
     Error registerGOTEntry(LinkGraph &G, Symbol &Sym,
                            GetSymbolTargetFunction GetSymbolTarget);
     Error registerStubEntry(LinkGraph &G, Symbol &Sym,
@@ -88,7 +81,7 @@ struct Session {
   };
 
   using DynLibJDMap = std::map<std::string, orc::JITDylib *, std::less<>>;
-  using SymbolInfoMap = StringMap<MemoryRegionInfo>;
+  using SymbolInfoMap = DenseMap<orc::SymbolStringPtr, MemoryRegionInfo>;
   using FileInfoMap = StringMap<FileInfo>;
 
   Expected<orc::JITDylib *> getOrLoadDynamicLibrary(StringRef LibPath);
@@ -111,8 +104,8 @@ struct Session {
   Expected<MemoryRegionInfo &> findGOTEntryInfo(StringRef FileName,
                                                 StringRef TargetName);
 
-  bool isSymbolRegistered(StringRef Name);
-  Expected<MemoryRegionInfo &> findSymbolInfo(StringRef SymbolName,
+  bool isSymbolRegistered(const orc::SymbolStringPtr &Name);
+  Expected<MemoryRegionInfo &> findSymbolInfo(const orc::SymbolStringPtr &Name,
                                               Twine ErrorMsgStem);
 
   DynLibJDMap DynLibJDs;
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 86ba9193dff2d..246d5cfa05818 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -2556,7 +2556,7 @@ static void disassembleObject(ObjectFile *Obj, bool InlineRelocs) {
   if (!MAttrs.empty()) {
     for (unsigned I = 0; I != MAttrs.size(); ++I)
       Features.AddFeature(MAttrs[I]);
-  } else if (MCPU.empty() && Obj->getArch() == llvm::Triple::aarch64) {
+  } else if (MCPU.empty() && Obj->makeTriple().isAArch64()) {
     Features.AddFeature("+all");
   }
 
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 2acf1cc34b2d8..f34d99e10f316 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -723,6 +723,43 @@ loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
     return;
   }
 
+  using ::llvm::memprof::YAMLMemProfReader;
+  if (YAMLMemProfReader::hasFormat(Input.Filename)) {
+    auto ReaderOrErr = YAMLMemProfReader::create(Input.Filename);
+    if (!ReaderOrErr)
+      exitWithError(ReaderOrErr.takeError(), Input.Filename);
+    std::unique_ptr<YAMLMemProfReader> Reader = std::move(ReaderOrErr.get());
+    // Check if the profile types can be merged, e.g. clang frontend profiles
+    // should not be merged with memprof profiles.
+    if (Error E = WC->Writer.mergeProfileKind(Reader->getProfileKind())) {
+      consumeError(std::move(E));
+      WC->Errors.emplace_back(
+          make_error<StringError>(
+              "Cannot merge MemProf profile with incompatible profile.",
+              std::error_code()),
+          Filename);
+      return;
+    }
+
+    auto MemProfError = [&](Error E) {
+      auto [ErrorCode, Msg] = InstrProfError::take(std::move(E));
+      WC->Errors.emplace_back(make_error<InstrProfError>(ErrorCode, Msg),
+                              Filename);
+    };
+
+    auto MemProfData = Reader->takeMemProfData();
+
+    // Check for the empty input in case the YAML file is invalid.
+    if (MemProfData.Records.empty()) {
+      WC->Errors.emplace_back(
+          make_error<StringError>("The profile is empty.", std::error_code()),
+          Filename);
+    }
+
+    WC->Writer.addMemProfData(std::move(MemProfData), MemProfError);
+    return;
+  }
+
   auto FS = vfs::getRealFileSystem();
   // TODO: This only saves the first non-fatal error from InstrProfReader, and
   // then added to WriterContext::Errors. However, this is not extensible, if
@@ -2967,8 +3004,10 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
   std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
   bool IsIR = Reader->isIRLevelProfile();
   OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
-  if (IsIR)
+  if (IsIR) {
     OS << "  entry_first = " << Reader->instrEntryBBEnabled();
+    OS << "  instrument_loop_entries = " << Reader->instrLoopEntriesEnabled();
+  }
   OS << "\n";
   if (ShowAllFunctions || !FuncNameFilter.empty())
     OS << "Functions shown: " << ShownFunctions << "\n";
@@ -3240,18 +3279,36 @@ static int showSampleProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
 static int showMemProfProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
   if (SFormat == ShowFormat::Json)
     exitWithError("JSON output is not supported for MemProf");
-  auto ReaderOr = llvm::memprof::RawMemProfReader::create(
-      Filename, ProfiledBinary, /*KeepNames=*/true);
-  if (Error E = ReaderOr.takeError())
-    // Since the error can be related to the profile or the binary we do not
-    // pass whence. Instead additional context is provided where necessary in
-    // the error message.
-    exitWithError(std::move(E), /*Whence*/ "");
-
-  std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
-      ReaderOr.get().release());
-
-  Reader->printYAML(OS);
+
+  // Show the raw profile in YAML.
+  if (memprof::RawMemProfReader::hasFormat(Filename)) {
+    auto ReaderOr = llvm::memprof::RawMemProfReader::create(
+        Filename, ProfiledBinary, /*KeepNames=*/true);
+    if (Error E = ReaderOr.takeError()) {
+      // Since the error can be related to the profile or the binary we do not
+      // pass whence. Instead additional context is provided where necessary in
+      // the error message.
+      exitWithError(std::move(E), /*Whence*/ "");
+    }
+
+    std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
+        ReaderOr.get().release());
+
+    Reader->printYAML(OS);
+    return 0;
+  }
+
+  // Show the indexed MemProf profile in YAML.
+  auto FS = vfs::getRealFileSystem();
+  auto ReaderOrErr = IndexedInstrProfReader::create(Filename, *FS);
+  if (Error E = ReaderOrErr.takeError())
+    exitWithError(std::move(E), Filename);
+
+  auto Reader = std::move(ReaderOrErr.get());
+  memprof::AllMemProfData Data = Reader->getAllMemProfData();
+  yaml::Output Yout(OS);
+  Yout << Data;
+
   return 0;
 }
 
diff --git a/llvm/tools/llvm-readobj/XCOFFDumper.cpp b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
index 46b510cfb06a3..6a099c08e1aca 100644
--- a/llvm/tools/llvm-readobj/XCOFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
@@ -723,7 +723,12 @@ const EnumEntry<XCOFF::CFileLangId> CFileLangIdClass[] = {
 const EnumEntry<XCOFF::CFileCpuId> CFileCpuIdClass[] = {
 #define ECase(X)                                                               \
   { #X, XCOFF::X }
-    ECase(TCPU_PPC64), ECase(TCPU_COM), ECase(TCPU_970)
+    ECase(TCPU_INVALID), ECase(TCPU_PPC),  ECase(TCPU_PPC64), ECase(TCPU_COM),
+    ECase(TCPU_PWR),     ECase(TCPU_ANY),  ECase(TCPU_601),   ECase(TCPU_603),
+    ECase(TCPU_604),     ECase(TCPU_620),  ECase(TCPU_A35),   ECase(TCPU_970),
+    ECase(TCPU_PWR5),    ECase(TCPU_PWR6), ECase(TCPU_PWR5X), ECase(TCPU_PWR6E),
+    ECase(TCPU_PWR7),    ECase(TCPU_PWR8), ECase(TCPU_PWR9),  ECase(TCPU_PWR10),
+    ECase(TCPU_PWRX)
 #undef ECase
 };
 
diff --git a/llvm/unittests/ADT/BitmaskEnumTest.cpp b/llvm/unittests/ADT/BitmaskEnumTest.cpp
index c78937c3571fd..2c0a80342a54c 100644
--- a/llvm/unittests/ADT/BitmaskEnumTest.cpp
+++ b/llvm/unittests/ADT/BitmaskEnumTest.cpp
@@ -130,6 +130,22 @@ TEST(BitmaskEnumTest, BitwiseXorEquals) {
   EXPECT_EQ(V3, f2);
 }
 
+TEST(BitmaskEnumTest, BitwiseShift) {
+  Flags f = (F1 << F1);
+  EXPECT_EQ(f, F2);
+
+  Flags f2 = F1;
+  f2 <<= F1;
+  EXPECT_EQ(f2, F2);
+
+  Flags f3 = (F1 >> F1);
+  EXPECT_EQ(f3, F0);
+
+  Flags f4 = F1;
+  f4 >>= F1;
+  EXPECT_EQ(f4, F0);
+}
+
 TEST(BitmaskEnumTest, ConstantExpression) {
   constexpr Flags f1 = ~F1;
   constexpr Flags f2 = F1 | F2;
diff --git a/llvm/unittests/AsmParser/AsmParserTest.cpp b/llvm/unittests/AsmParser/AsmParserTest.cpp
index a70c061d3e304..ce226705068af 100644
--- a/llvm/unittests/AsmParser/AsmParserTest.cpp
+++ b/llvm/unittests/AsmParser/AsmParserTest.cpp
@@ -93,6 +93,22 @@ TEST(AsmParserTest, TypeAndConstantValueParsing) {
   EXPECT_TRUE(V->getType()->isVectorTy());
   ASSERT_TRUE(isa<ConstantDataVector>(V));
 
+  V = parseConstantValue("<4 x i32> splat (i32 -2)", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isVectorTy());
+  ASSERT_TRUE(isa<ConstantDataVector>(V));
+
+  V = parseConstantValue("<4 x i32> zeroinitializer", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isVectorTy());
+  ASSERT_TRUE(isa<Constant>(V));
+  EXPECT_TRUE(cast<Constant>(V)->isNullValue());
+
+  V = parseConstantValue("<4 x i32> poison", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isVectorTy());
+  ASSERT_TRUE(isa<PoisonValue>(V));
+
   V = parseConstantValue("i32 add (i32 1, i32 2)", Error, M);
   ASSERT_TRUE(V);
   ASSERT_TRUE(isa<ConstantInt>(V));
@@ -105,6 +121,10 @@ TEST(AsmParserTest, TypeAndConstantValueParsing) {
   ASSERT_TRUE(V);
   ASSERT_TRUE(isa<UndefValue>(V));
 
+  V = parseConstantValue("ptr poison", Error, M);
+  ASSERT_TRUE(V);
+  ASSERT_TRUE(isa<PoisonValue>(V));
+
   EXPECT_FALSE(parseConstantValue("duble 3.25", Error, M));
   EXPECT_EQ(Error.getMessage(), "expected type");
 
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
index 795646b22b945..5eef8ee87b4dd 100644
--- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -92,6 +92,28 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
                 "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64", "mips64el"),
             "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64");
 
+  // Check that PowerPC64 targets add -i128:128.
+  EXPECT_EQ(UpgradeDataLayoutString("e-m:e-i64:64-n32:64", "powerpc64le-linux"),
+            "e-m:e-i64:64-i128:128-n32:64");
+  EXPECT_EQ(
+      UpgradeDataLayoutString("E-m:e-Fn32-i64:64-n32:64", "powerpc64-linux"),
+      "E-m:e-Fn32-i64:64-i128:128-n32:64");
+  EXPECT_EQ(
+      UpgradeDataLayoutString("E-m:a-Fi64-i64:64-n32:64", "powerpc64-ibm-aix"),
+      "E-m:a-Fi64-i64:64-i128:128-n32:64");
+
+  // Check that WebAssembly targets add -i128:128.
+  EXPECT_EQ(
+      UpgradeDataLayoutString(
+          "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20",
+          "wasm32"),
+      "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20");
+  EXPECT_EQ(
+      UpgradeDataLayoutString(
+          "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20",
+          "wasm64"),
+      "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20");
+
   // Check that SPIR && SPIRV targets add -G1 if it's not present.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir"), "e-p:32:32-G1");
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir64"), "e-p:32:32-G1");
@@ -108,8 +130,6 @@ TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
       "-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
       "-n8:16:32:64-S128",
       "x86_64-unknown-linux-gnu");
-  std::string DL2 = UpgradeDataLayoutString("e-m:e-i64:64-n32:64",
-                                            "powerpc64le-unknown-linux-gnu");
   std::string DL3 = UpgradeDataLayoutString(
       "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32",
       "aarch64--");
@@ -118,7 +138,6 @@ TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
       "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128"
       "-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64"
       "-f80:128:128-n8:16:32:64-S128");
-  EXPECT_EQ(DL2, "e-m:e-i64:64-n32:64");
   EXPECT_EQ(DL3, "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:"
                  "64-S128-Fn32");
 
@@ -153,6 +172,14 @@ TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
   EXPECT_EQ(UpgradeDataLayoutString("G2", "spir64"), "G2");
   EXPECT_EQ(UpgradeDataLayoutString("G2", "spirv32"), "G2");
   EXPECT_EQ(UpgradeDataLayoutString("G2", "spirv64"), "G2");
+
+  // Check that PowerPC32 targets don't add -i128:128.
+  EXPECT_EQ(UpgradeDataLayoutString("e-m:e-i64:64-n32", "powerpcle-linux"),
+            "e-m:e-i64:64-n32");
+  EXPECT_EQ(UpgradeDataLayoutString("E-m:e-Fn32-i64:64-n32", "powerpc-linux"),
+            "E-m:e-Fn32-i64:64-n32");
+  EXPECT_EQ(UpgradeDataLayoutString("E-m:a-Fi64-i64:64-n32", "powerpc-aix"),
+            "E-m:a-Fi64-i64:64-n32");
 }
 
 TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
diff --git a/llvm/unittests/CodeGen/MFCommon.inc b/llvm/unittests/CodeGen/MFCommon.inc
index 67759bd5c4632..425453fb0e280 100644
--- a/llvm/unittests/CodeGen/MFCommon.inc
+++ b/llvm/unittests/CodeGen/MFCommon.inc
@@ -78,8 +78,8 @@ public:
 class BogusSubtarget : public TargetSubtargetInfo {
 public:
   BogusSubtarget(TargetMachine &TM)
-      : TargetSubtargetInfo(Triple(""), "", "", "", {}, {}, nullptr, nullptr,
-                            nullptr, nullptr, nullptr, nullptr),
+      : TargetSubtargetInfo(Triple(""), "", "", "", {}, {}, {}, nullptr,
+                            nullptr, nullptr, nullptr, nullptr, nullptr),
         FL(), TL(TM) {}
   ~BogusSubtarget() override {}
 
diff --git a/llvm/unittests/DebugInfo/LogicalView/CMakeLists.txt b/llvm/unittests/DebugInfo/LogicalView/CMakeLists.txt
index 1116edb212b05..42a4b72229483 100644
--- a/llvm/unittests/DebugInfo/LogicalView/CMakeLists.txt
+++ b/llvm/unittests/DebugInfo/LogicalView/CMakeLists.txt
@@ -1,17 +1,21 @@
 set(LLVM_LINK_COMPONENTS
-  AllTargetsDescs
-  AllTargetsDisassemblers
-  AllTargetsInfos
+  ${LLVM_TARGETS_TO_BUILD}
+  AsmPrinter
+  DebugInfoDWARF
   DebugInfoLogicalView
   MC
   MCDisassembler
+  Object
   TargetParser
   )
 
 add_llvm_unittest_with_input_files(DebugInfoLogicalViewTests
+  ../DWARF/DwarfGenerator.cpp
+  ../DWARF/DwarfUtils.cpp
   CodeViewReaderTest.cpp
   CommandLineOptionsTest.cpp
   CompareElementsTest.cpp
+  DWARFGeneratedTest.cpp
   DWARFReaderTest.cpp
   SelectElementsTest.cpp
   LocationRangesTest.cpp
diff --git a/llvm/unittests/DebugInfo/LogicalView/DWARFGeneratedTest.cpp b/llvm/unittests/DebugInfo/LogicalView/DWARFGeneratedTest.cpp
new file mode 100644
index 0000000000000..8c23b478edda7
--- /dev/null
+++ b/llvm/unittests/DebugInfo/LogicalView/DWARFGeneratedTest.cpp
@@ -0,0 +1,362 @@
+//===- llvm/unittest/DebugInfo/LogicalView/DWARFGeneratedTest.cpp ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../DWARF/DwarfGenerator.h"
+#include "../DWARF/DwarfUtils.h"
+#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/DebugInfo/LogicalView/Core/LVReader.h"
+#include "llvm/DebugInfo/LogicalView/Core/LVScope.h"
+#include "llvm/DebugInfo/LogicalView/Core/LVSymbol.h"
+#include "llvm/DebugInfo/LogicalView/Core/LVType.h"
+#include "llvm/DebugInfo/LogicalView/LVReaderHandler.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/COM.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Testing/Support/Error.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::logicalview;
+using namespace llvm::dwarf;
+using namespace llvm::dwarf::utils;
+
+namespace {
+
+// Helper function to get the first compile unit.
+LVScopeCompileUnit *getFirstCompileUnit(LVScopeRoot *Root) {
+  if (!Root)
+    return nullptr;
+
+  const LVScopes *CompileUnits = Root->getScopes();
+  if (!CompileUnits)
+    return nullptr;
+
+  LVScopes::const_iterator Iter = CompileUnits->begin();
+  return (Iter != CompileUnits->end())
+             ? static_cast<LVScopeCompileUnit *>(*Iter)
+             : nullptr;
+}
+
+// Helper function to create a reader.
+std::unique_ptr<LVReader> createReader(LVReaderHandler &ReaderHandler,
+                                       SmallString<128> &InputsDir,
+                                       StringRef Filename) {
+  SmallString<128> ObjectName(InputsDir);
+  llvm::sys::path::append(ObjectName, Filename);
+
+  Expected<std::unique_ptr<LVReader>> ReaderOrErr =
+      ReaderHandler.createReader(std::string(ObjectName));
+  EXPECT_THAT_EXPECTED(ReaderOrErr, Succeeded());
+  std::unique_ptr<LVReader> Reader = std::move(*ReaderOrErr);
+  return Reader;
+}
+
+// Create a file with generated DWARF.
+void generateDebugInfo(StringRef Path, Triple &Triple) {
+  uint16_t Version = 5;
+  auto ExpectedDG = dwarfgen::Generator::create(Triple, Version);
+  ASSERT_THAT_EXPECTED(ExpectedDG, Succeeded());
+  dwarfgen::Generator *DG = ExpectedDG.get().get();
+  dwarfgen::CompileUnit &CU = DG->addCompileUnit();
+
+  dwarfgen::DIE CUDie = CU.getUnitDIE();
+  CUDie.addAttribute(DW_AT_name, DW_FORM_strp, "test.cpp");
+  CUDie.addAttribute(DW_AT_language, DW_FORM_data2, DW_LANG_C_plus_plus);
+  CUDie.addAttribute(DW_AT_producer, DW_FORM_strp, "dwarfgen::Generator");
+
+  dwarfgen::DIE ScopeValueDie = CUDie.addChild(DW_TAG_inlined_subroutine);
+  ScopeValueDie.addAttribute(DW_AT_accessibility, DW_FORM_data1, 1);
+  ScopeValueDie.addAttribute(DW_AT_inline, DW_FORM_data4, 2);
+  ScopeValueDie.addAttribute(DW_AT_virtuality, DW_FORM_data4, 3);
+  ScopeValueDie.addAttribute(DW_AT_call_file, DW_FORM_data4, 4);
+  ScopeValueDie.addAttribute(DW_AT_call_line, DW_FORM_data4, 5);
+  ScopeValueDie.addAttribute(DW_AT_decl_file, DW_FORM_data4, 6);
+  ScopeValueDie.addAttribute(DW_AT_decl_line, DW_FORM_data4, 7);
+  ScopeValueDie.addAttribute(DW_AT_GNU_discriminator, DW_FORM_data4, 8);
+
+  dwarfgen::DIE ScopeNoValueDie = CUDie.addChild(DW_TAG_inlined_subroutine);
+  ScopeNoValueDie.addAttribute(DW_AT_accessibility, DW_FORM_sdata, 1);
+  ScopeNoValueDie.addAttribute(DW_AT_inline, DW_FORM_sdata, 2);
+  ScopeNoValueDie.addAttribute(DW_AT_virtuality, DW_FORM_sdata, 3);
+  ScopeNoValueDie.addAttribute(DW_AT_call_file, DW_FORM_sdata, 4);
+  ScopeNoValueDie.addAttribute(DW_AT_call_line, DW_FORM_sdata, 5);
+  ScopeNoValueDie.addAttribute(DW_AT_decl_file, DW_FORM_sdata, 6);
+  ScopeNoValueDie.addAttribute(DW_AT_decl_line, DW_FORM_sdata, 7);
+  ScopeNoValueDie.addAttribute(DW_AT_GNU_discriminator, DW_FORM_sdata, 8);
+
+  dwarfgen::DIE ScopeImplicitDie = CUDie.addChild(DW_TAG_inlined_subroutine);
+  ScopeImplicitDie.addAttribute(DW_AT_accessibility, DW_FORM_implicit_const, 1);
+  ScopeImplicitDie.addAttribute(DW_AT_inline, DW_FORM_implicit_const, 2);
+  ScopeImplicitDie.addAttribute(DW_AT_virtuality, DW_FORM_implicit_const, 3);
+  ScopeImplicitDie.addAttribute(DW_AT_call_file, DW_FORM_implicit_const, 4);
+  ScopeImplicitDie.addAttribute(DW_AT_call_line, DW_FORM_implicit_const, 5);
+  ScopeImplicitDie.addAttribute(DW_AT_decl_file, DW_FORM_implicit_const, 6);
+  ScopeImplicitDie.addAttribute(DW_AT_decl_line, DW_FORM_implicit_const, 7);
+  ScopeImplicitDie.addAttribute(DW_AT_GNU_discriminator, DW_FORM_implicit_const,
+                                8);
+
+  dwarfgen::DIE SymbolValueDie = CUDie.addChild(DW_TAG_variable);
+  SymbolValueDie.addAttribute(DW_AT_bit_size, DW_FORM_data1, 1);
+
+  dwarfgen::DIE SymbolNoValueDie = CUDie.addChild(DW_TAG_variable);
+  SymbolNoValueDie.addAttribute(DW_AT_bit_size, DW_FORM_sdata, 1);
+
+  dwarfgen::DIE SymbolImplicitDie = CUDie.addChild(DW_TAG_variable);
+  SymbolImplicitDie.addAttribute(DW_AT_bit_size, DW_FORM_implicit_const, 1);
+
+  dwarfgen::DIE TypeValueCountDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeValueCountDie.addAttribute(DW_AT_count, DW_FORM_data4, 1);
+
+  dwarfgen::DIE TypeNoValueCountDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeNoValueCountDie.addAttribute(DW_AT_count, DW_FORM_sdata, 1);
+
+  dwarfgen::DIE TypeImplicitCountDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeImplicitCountDie.addAttribute(DW_AT_count, DW_FORM_implicit_const, 1);
+
+  dwarfgen::DIE TypeValueRangeDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeValueRangeDie.addAttribute(DW_AT_lower_bound, DW_FORM_data4, 1);
+  TypeValueRangeDie.addAttribute(DW_AT_upper_bound, DW_FORM_data4, 2);
+
+  dwarfgen::DIE TypeNoValueRangeDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeNoValueRangeDie.addAttribute(DW_AT_lower_bound, DW_FORM_addr, 3);
+  TypeNoValueRangeDie.addAttribute(DW_AT_upper_bound, DW_FORM_addr, 4);
+
+  dwarfgen::DIE TypeImplicitRangeDie = CUDie.addChild(DW_TAG_subrange_type);
+  TypeImplicitRangeDie.addAttribute(DW_AT_lower_bound, DW_FORM_implicit_const,
+                                    5);
+  TypeImplicitRangeDie.addAttribute(DW_AT_upper_bound, DW_FORM_implicit_const,
+                                    6);
+
+  // Generate the DWARF.
+  StringRef FileBytes = DG->generate();
+  MemoryBufferRef FileBuffer(FileBytes, "dwarf");
+  auto Obj = object::ObjectFile::createObjectFile(FileBuffer);
+  EXPECT_TRUE((bool)Obj);
+  std::unique_ptr<DWARFContext> DwarfContext = DWARFContext::create(**Obj);
+
+  // Verify the number of compile units is correct.
+  uint32_t NumCUs = DwarfContext->getNumCompileUnits();
+  EXPECT_EQ(NumCUs, 1u);
+  DWARFCompileUnit *U = cast<DWARFCompileUnit>(DwarfContext->getUnitAtIndex(0));
+  auto DieDG = U->getUnitDIE(false);
+  EXPECT_TRUE(DieDG.isValid());
+
+  // Verify the siblings correct order.
+  //   ScopeValue
+  //   ScopeNoValue
+  //   ScopeImplicit
+  auto ScopeValueDieDG = DieDG.getFirstChild();
+  EXPECT_TRUE(ScopeValueDieDG.isValid());
+  EXPECT_EQ(ScopeValueDieDG.getTag(), DW_TAG_inlined_subroutine);
+  auto ScopeNoValueDieDG = ScopeValueDieDG.getSibling();
+  EXPECT_TRUE(ScopeNoValueDieDG.isValid());
+  EXPECT_EQ(ScopeNoValueDieDG.getTag(), DW_TAG_inlined_subroutine);
+  auto ScopeImplicitDieDG = ScopeNoValueDieDG.getSibling();
+  EXPECT_TRUE(ScopeImplicitDieDG.isValid());
+  EXPECT_EQ(ScopeImplicitDieDG.getTag(), DW_TAG_inlined_subroutine);
+
+  // Verify the siblings correct order.
+  //   SymbolValue
+  //   SymbolNoValue
+  //   SymbolImplicitValue
+  auto SymbolValueDieDG = ScopeImplicitDieDG.getSibling();
+  EXPECT_TRUE(SymbolValueDieDG.isValid());
+  EXPECT_EQ(SymbolValueDieDG.getTag(), DW_TAG_variable);
+  auto SymbolNoValueDieDG = SymbolValueDieDG.getSibling();
+  EXPECT_TRUE(SymbolNoValueDieDG.isValid());
+  EXPECT_EQ(SymbolNoValueDieDG.getTag(), DW_TAG_variable);
+  auto SymbolImplicitDieDG = SymbolNoValueDieDG.getSibling();
+  EXPECT_TRUE(SymbolImplicitDieDG.isValid());
+  EXPECT_EQ(SymbolImplicitDieDG.getTag(), DW_TAG_variable);
+
+  // Verify the siblings correct order.
+  //   TypeValueCount
+  //   TypeNoValueCount
+  //   TypeImplicitValueCount
+  auto TypeValueCountDieDG = SymbolImplicitDieDG.getSibling();
+  EXPECT_TRUE(TypeValueCountDieDG.isValid());
+  EXPECT_EQ(TypeValueCountDieDG.getTag(), DW_TAG_subrange_type);
+  auto TypeNoValueCountDieDG = TypeValueCountDieDG.getSibling();
+  EXPECT_TRUE(TypeNoValueCountDieDG.isValid());
+  EXPECT_EQ(TypeNoValueCountDieDG.getTag(), DW_TAG_subrange_type);
+  auto TypeImplicitCountDieDG = TypeNoValueCountDieDG.getSibling();
+  EXPECT_TRUE(TypeImplicitCountDieDG.isValid());
+  EXPECT_EQ(TypeImplicitCountDieDG.getTag(), DW_TAG_subrange_type);
+
+  // Verify the siblings correct order.
+  //   TypeValueRange
+  //   TypeNoValueRange
+  //   TypeImplicitValueRange
+  auto TypeValueRangeDieDG = TypeImplicitCountDieDG.getSibling();
+  EXPECT_TRUE(TypeValueRangeDieDG.isValid());
+  EXPECT_EQ(TypeValueRangeDieDG.getTag(), DW_TAG_subrange_type);
+  auto TypeNoValueRangeDieDG = TypeValueRangeDieDG.getSibling();
+  EXPECT_TRUE(TypeNoValueRangeDieDG.isValid());
+  EXPECT_EQ(TypeNoValueRangeDieDG.getTag(), DW_TAG_subrange_type);
+  auto TypeImplicitRangeDieDG = TypeNoValueRangeDieDG.getSibling();
+  EXPECT_TRUE(TypeImplicitRangeDieDG.isValid());
+  EXPECT_EQ(TypeImplicitRangeDieDG.getTag(), DW_TAG_subrange_type);
+
+  // Save the generated DWARF file to disk.
+  EXPECT_TRUE(DG->saveFile(Path));
+}
+
+// Check the logical elements basic properties.
+void checkElementAttributes(LVReader *Reader) {
+  LVScopeRoot *Root = Reader->getScopesRoot();
+  ASSERT_NE(Root, nullptr);
+  LVScopeCompileUnit *CompileUnit = getFirstCompileUnit(Root);
+  ASSERT_NE(CompileUnit, nullptr);
+
+  const LVScopes *Scopes = CompileUnit->getScopes();
+  ASSERT_NE(Scopes, nullptr);
+  ASSERT_EQ(Scopes->size(), 3u);
+
+  // Check values.
+  LVScopes::const_iterator ScopeIter = Scopes->begin();
+  ASSERT_NE(ScopeIter, Scopes->end());
+  LVScope *Scope = static_cast<LVScope *>(*ScopeIter);
+  ASSERT_NE(Scope, nullptr);
+  EXPECT_EQ(Scope->getAccessibilityCode(), 1u); // Element
+  EXPECT_EQ(Scope->getInlineCode(), 2u);        // Element
+  EXPECT_EQ(Scope->getVirtualityCode(), 3u);    // Element
+  EXPECT_EQ(Scope->getCallFilenameIndex(), 5u); // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getCallLineNumber(), 5u);    // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getFilenameIndex(), 7u);     // Element
+  EXPECT_EQ(Scope->getLineNumber(), 7u);        // Element
+  EXPECT_EQ(Scope->getDiscriminator(), 8u);     // ScopeFunctionInlined
+
+  // Check no-values.
+  ASSERT_NE(++ScopeIter, Scopes->end());
+  Scope = static_cast<LVScope *>(*ScopeIter);
+  ASSERT_NE(Scope, nullptr);
+  EXPECT_EQ(Scope->getAccessibilityCode(), 0u); // Element
+  EXPECT_EQ(Scope->getInlineCode(), 0u);        // Element
+  EXPECT_EQ(Scope->getVirtualityCode(), 0u);    // Element
+  EXPECT_EQ(Scope->getCallFilenameIndex(), 1u); // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getCallLineNumber(), 0u);    // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getFilenameIndex(), 1u);     // Element
+  EXPECT_EQ(Scope->getLineNumber(), 0u);        // Element
+  EXPECT_EQ(Scope->getDiscriminator(), 0u);     // ScopeFunctionInlined
+
+  // Check implicit values.
+  ASSERT_NE(++ScopeIter, Scopes->end());
+  Scope = static_cast<LVScope *>(*ScopeIter);
+  ASSERT_NE(Scope, nullptr);
+  EXPECT_EQ(Scope->getAccessibilityCode(), 1u); // Element
+  EXPECT_EQ(Scope->getInlineCode(), 2u);        // Element
+  EXPECT_EQ(Scope->getVirtualityCode(), 3u);    // Element
+  EXPECT_EQ(Scope->getCallFilenameIndex(), 5u); // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getCallLineNumber(), 5u);    // ScopeFunctionInlined
+  EXPECT_EQ(Scope->getFilenameIndex(), 7u);     // Element
+  EXPECT_EQ(Scope->getLineNumber(), 7u);        // Element
+  EXPECT_EQ(Scope->getDiscriminator(), 8u);     // ScopeFunctionInlined
+
+  const LVSymbols *Symbols = CompileUnit->getSymbols();
+  ASSERT_NE(Symbols, nullptr);
+  ASSERT_EQ(Symbols->size(), 3u);
+
+  LVSymbols::const_iterator SymbolIter = Symbols->begin();
+  ASSERT_NE(SymbolIter, Symbols->end());
+  LVSymbol *Symbol = static_cast<LVSymbol *>(*SymbolIter);
+  ASSERT_NE(Symbol, nullptr);
+  EXPECT_EQ(Symbol->getBitSize(), 1u); // Symbol
+
+  ASSERT_NE(++SymbolIter, Symbols->end());
+  Symbol = static_cast<LVSymbol *>(*SymbolIter);
+  ASSERT_NE(Symbol, nullptr);
+  EXPECT_EQ(Symbol->getBitSize(), 0u); // Symbol
+
+  ASSERT_NE(++SymbolIter, Symbols->end());
+  Symbol = static_cast<LVSymbol *>(*SymbolIter);
+  ASSERT_NE(Symbol, nullptr);
+  EXPECT_EQ(Symbol->getBitSize(), 1u); // Symbol
+
+  const LVTypes *Types = CompileUnit->getTypes();
+  ASSERT_NE(Types, nullptr);
+  ASSERT_EQ(Types->size(), 6u);
+
+  LVTypes::const_iterator TypeIter = Types->begin();
+  ASSERT_NE(TypeIter, Types->end());
+  LVType *Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getCount(), 1u); // Type
+
+  ASSERT_NE(++TypeIter, Types->end());
+  Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getCount(), 0u); // Type
+
+  ASSERT_NE(++TypeIter, Types->end());
+  Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getCount(), 1u); // Type
+
+  ASSERT_NE(++TypeIter, Types->end());
+  Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getLowerBound(), 1u); // Type
+  EXPECT_EQ(Type->getUpperBound(), 2u); // Type
+
+  ASSERT_NE(++TypeIter, Types->end());
+  Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getLowerBound(), 0u); // Type
+  EXPECT_EQ(Type->getUpperBound(), 0u); // Type
+
+  ASSERT_NE(++TypeIter, Types->end());
+  Type = static_cast<LVType *>(*TypeIter);
+  ASSERT_NE(Type, nullptr);
+  EXPECT_EQ(Type->getLowerBound(), 5u); // Type
+  EXPECT_EQ(Type->getUpperBound(), 6u); // Type
+}
+
+TEST(LogicalViewTest, ElementAttributes) {
+  // Initialize targets and assembly printers/parsers.
+  llvm::InitializeAllTargetInfos();
+  llvm::InitializeAllTargetMCs();
+  InitializeAllDisassemblers();
+
+  llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded);
+
+  Triple Triple(Triple::normalize("x86_64-pc-linux-gnu"));
+  if (!isConfigurationSupported(Triple))
+    GTEST_SKIP();
+
+  unittest::TempDir TestDirectory("dwarf-test", /*Unique=*/true);
+  llvm::SmallString<128> DirName(TestDirectory.path());
+  StringRef Filename("test.o");
+  llvm::SmallString<128> Path(TestDirectory.path(Filename));
+  generateDebugInfo(Path, Triple);
+
+  // Reader options.
+  LVOptions ReaderOptions;
+  ReaderOptions.setPrintScopes();
+  ReaderOptions.setPrintSymbols();
+  ReaderOptions.setPrintTypes();
+  ReaderOptions.resolveDependencies();
+
+  std::vector<std::string> Objects;
+  ScopedPrinter W(outs());
+  LVReaderHandler ReaderHandler(Objects, W, ReaderOptions);
+
+  // Check logical elements properties.
+  std::unique_ptr<LVReader> Reader =
+      createReader(ReaderHandler, DirName, Filename);
+  ASSERT_NE(Reader, nullptr);
+
+  checkElementAttributes(Reader.get());
+}
+
+} // namespace
diff --git a/llvm/unittests/ExecutionEngine/JITLink/AArch32ErrorTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/AArch32ErrorTests.cpp
index f07ddddd5fe9c..b646f88b6fbcf 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/AArch32ErrorTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/AArch32ErrorTests.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <llvm/ExecutionEngine/JITLink/aarch32.h>
-
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
+#include <llvm/ExecutionEngine/JITLink/aarch32.h>
 
 using namespace llvm;
 using namespace llvm::jitlink;
@@ -18,9 +18,10 @@ using namespace llvm::support;
 using namespace llvm::support::endian;
 
 constexpr unsigned PointerSize = 4;
-auto G = std::make_unique<LinkGraph>("foo", Triple("armv7-linux-gnueabi"),
-                                     PointerSize, endianness::little,
-                                     aarch32::getEdgeKindName);
+auto G = std::make_unique<LinkGraph>(
+    "foo", std::make_shared<orc::SymbolStringPool>(),
+    Triple("armv7-linux-gnueabi"), PointerSize, endianness::little,
+    aarch32::getEdgeKindName);
 auto &Sec =
     G->createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
 
@@ -46,9 +47,10 @@ class AArch32Errors : public testing::Test {
   static void SetUpTestCase() {}
 
   void SetUp() override {
-    G = std::make_unique<LinkGraph>("foo", Triple("armv7-linux-gnueabi"),
-                                    PointerSize, endianness::little,
-                                    aarch32::getEdgeKindName);
+    G = std::make_unique<LinkGraph>(
+        "foo", std::make_shared<orc::SymbolStringPool>(),
+        Triple("armv7-linux-gnueabi"), PointerSize, endianness::little,
+        aarch32::getEdgeKindName);
     S = &G->createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
   }
 
diff --git a/llvm/unittests/ExecutionEngine/JITLink/EHFrameSupportTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/EHFrameSupportTests.cpp
index e7e09b3fe40b1..eaf9733b22819 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/EHFrameSupportTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/EHFrameSupportTests.cpp
@@ -172,7 +172,8 @@ TEST(EHFrameCFIBlockInspector, BasicSuccessCase) {
   // (3) Each FDE has an edge pointing to the CIE at the correct offset.
   // (4) Each function has exactly one FDE pointing at it.
 
-  auto G = cantFail(createLinkGraphFromMachOObject_arm64(TestObject));
+  auto G = cantFail(createLinkGraphFromMachOObject_arm64(
+      TestObject, std::make_shared<orc::SymbolStringPool>()));
   cantFail(createEHFrameSplitterPass_MachO_arm64()(*G));
   cantFail(createEHFrameEdgeFixerPass_MachO_arm64()(*G));
 
@@ -211,7 +212,7 @@ TEST(EHFrameCFIBlockInspector, BasicSuccessCase) {
       ASSERT_TRUE(!!CFIBI.getPCBeginEdge());
       auto &PCBeginTarget = CFIBI.getPCBeginEdge()->getTarget();
       ASSERT_TRUE(PCBeginTarget.hasName());
-      Targets.insert(PCBeginTarget.getName());
+      Targets.insert(*PCBeginTarget.getName());
 
       // If the FDE points at CIEs[0] (the CIE without a personality) then it
       // should not have an LSDA. If it points to CIEs[1] then it should have
@@ -232,11 +233,12 @@ TEST(EHFrameCFIBlockInspector, BasicSuccessCase) {
 TEST(EHFrameCFIBlockInspector, ExternalPCBegin) {
   // Check that we don't crash if we transform the target of an FDE into an
   // external symbol before running edge-fixing.
-  auto G = cantFail(createLinkGraphFromMachOObject_arm64(TestObject));
+  auto G = cantFail(createLinkGraphFromMachOObject_arm64(
+      TestObject, std::make_shared<orc::SymbolStringPool>()));
 
   // Make '_a' external.
   for (auto *Sym : G->defined_symbols())
-    if (Sym->hasName() && Sym->getName() == "_a") {
+    if (Sym->hasName() && *Sym->getName() == "_a") {
       G->makeExternal(*Sym);
       break;
     }
diff --git a/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
index 9a7878edab504..24e540b97c0cf 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/JITLinkTestUtils.cpp
@@ -92,9 +92,10 @@ void defaultCtxSetup(MockJITLinkContext &) {}
 TEST(JITLinkMocks, SmokeTest) {
   // Check that the testing infrastructure defaults can "link" a graph
   // successfully.
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       getGenericEdgeKindName);
+  auto G = std::make_unique<LinkGraph>(
+      "foo", std::make_shared<orc::SymbolStringPool>(),
+      Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+      getGenericEdgeKindName);
 
   ArrayRef<char> Content = "hello, world!";
   auto &Sec =
diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
index 936559f072e05..3b96019766c84 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
@@ -21,7 +21,8 @@ using namespace llvm::jitlink;
 
 TEST(LinkGraphTest, Construction) {
   // Check that LinkGraph construction works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   EXPECT_EQ(G.getName(), "foo");
   EXPECT_EQ(G.getTargetTriple().str(), "x86_64-apple-darwin");
@@ -35,7 +36,8 @@ TEST(LinkGraphTest, Construction) {
 
 TEST(LinkGraphTest, AddressAccess) {
   // Check that we can get addresses for blocks, symbols, and edges.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   auto &Sec1 =
@@ -54,7 +56,8 @@ TEST(LinkGraphTest, AddressAccess) {
 
 TEST(LinkGraphTest, SectionEmpty) {
   // Check that Section::empty behaves as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -72,7 +75,8 @@ TEST(LinkGraphTest, SectionEmpty) {
 
 TEST(LinkGraphTest, BlockAndSymbolIteration) {
   // Check that we can iterate over blocks within Sections and across sections.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -125,7 +129,8 @@ TEST(LinkGraphTest, BlockAndSymbolIteration) {
 
 TEST(LinkGraphTest, ContentAccessAndUpdate) {
   // Check that we can make a defined symbol external.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -214,7 +219,8 @@ TEST(LinkGraphTest, ContentAccessAndUpdate) {
 
 TEST(LinkGraphTest, MakeExternal) {
   // Check that we can make defined and absolute symbols external.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -284,7 +290,8 @@ TEST(LinkGraphTest, MakeExternal) {
 
 TEST(LinkGraphTest, MakeAbsolute) {
   // Check that we can make defined and external symbols absolute.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -353,7 +360,8 @@ TEST(LinkGraphTest, MakeAbsolute) {
 
 TEST(LinkGraphTest, MakeDefined) {
   // Check that we can make an external symbol defined.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -401,7 +409,8 @@ TEST(LinkGraphTest, MakeDefined) {
 
 TEST(LinkGraphTest, TransferDefinedSymbol) {
   // Check that we can transfer a defined symbol from one block to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -436,7 +445,8 @@ TEST(LinkGraphTest, TransferDefinedSymbol) {
 TEST(LinkGraphTest, TransferDefinedSymbolAcrossSections) {
   // Check that we can transfer a defined symbol from an existing block in one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -470,7 +480,8 @@ TEST(LinkGraphTest, TransferDefinedSymbolAcrossSections) {
 TEST(LinkGraphTest, TransferBlock) {
   // Check that we can transfer a block (and all associated symbols) from one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -518,7 +529,8 @@ TEST(LinkGraphTest, TransferBlock) {
 TEST(LinkGraphTest, MergeSections) {
   // Check that we can transfer a block (and all associated symbols) from one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -604,7 +616,8 @@ TEST(LinkGraphTest, MergeSections) {
 
 TEST(LinkGraphTest, SplitBlock) {
   // Check that the LinkGraph::splitBlock test works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -716,7 +729,8 @@ TEST(LinkGraphTest, SplitBlock) {
 }
 
 TEST(LinkGraphTest, GraphAllocationMethods) {
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   // Test allocation of sized, uninitialized buffer.
@@ -737,7 +751,8 @@ TEST(LinkGraphTest, GraphAllocationMethods) {
 
 TEST(LinkGraphTest, IsCStringBlockTest) {
   // Check that the LinkGraph::splitBlock test works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -761,8 +776,8 @@ TEST(LinkGraphTest, IsCStringBlockTest) {
 }
 
 TEST(LinkGraphTest, BasicLayoutHonorsNoAlloc) {
-  // Check that BasicLayout honors NoAlloc.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   // Create a regular section and block.
diff --git a/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp
index be922275be26f..b68310aaf3990 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/MachOLinkGraphTests.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/JITLink/MachO.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
@@ -20,7 +21,8 @@ using namespace llvm::jitlink;
 
 TEST(MachOLinkGraphTest, GetStandardSections) {
   // Check that LinkGraph construction works as expected.
-  LinkGraph G("foo", Triple("arm64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("arm64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   auto &Data = getMachODefaultRWDataSection(G);
diff --git a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
index 2b303f7a8c1a2..d1284ffb2ecf2 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
@@ -18,9 +18,10 @@ using namespace llvm::jitlink;
 
 TEST(MemoryManagerErrorTest, ErrorOnFirstAllocate) {
   // Check that we can get addresses for blocks, symbols, and edges.
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       getGenericEdgeKindName);
+  auto G = std::make_unique<LinkGraph>(
+      "foo", std::make_shared<orc::SymbolStringPool>(),
+      Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+      getGenericEdgeKindName);
 
   ArrayRef<char> Content = "hello, world!";
   auto &Sec =
diff --git a/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
index 6a8f89639a074..8efbb54f59e84 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
@@ -57,7 +57,8 @@ GenerateStub(LinkGraph &G, size_t PointerSize, Edge::Kind PointerEdgeKind) {
 TEST(StubsTest, StubsGeneration_x86_64) {
   const char PointerJumpStubContent[6] = {
       static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00};
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, x86_64::Pointer64);
 
@@ -77,7 +78,8 @@ TEST(StubsTest, StubsGeneration_aarch64) {
       0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
       0x00, 0x02, 0x1f, (char)0xd6u  // BR  x16
   };
-  LinkGraph G("foo", Triple("aarch64-linux-gnu"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("aarch64-linux-gnu"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, aarch64::Pointer64);
 
@@ -97,8 +99,9 @@ TEST(StubsTest, StubsGeneration_aarch64) {
 TEST(StubsTest, StubsGeneration_i386) {
   const char PointerJumpStubContent[6] = {
       static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00};
-  LinkGraph G("foo", Triple("i386-unknown-linux-gnu"), 4,
-              llvm::endianness::little, getGenericEdgeKindName);
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("i386-unknown-linux-gnu"), 8, llvm::endianness::little,
+              getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 4U, i386::Pointer32);
 
   EXPECT_EQ(std::distance(StubSym.getBlock().edges().begin(),
@@ -126,7 +129,8 @@ TEST(StubsTest, StubsGeneration_loongarch32) {
       0x00,
       0x4c // jr $t8
   };
-  LinkGraph G("foo", Triple("loongarch32"), 4, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("loongarch32"), 4, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 4U, loongarch::Pointer32);
 
@@ -158,7 +162,8 @@ TEST(StubsTest, StubsGeneration_loongarch64) {
       0x00,
       0x4c // jr $t8
   };
-  LinkGraph G("foo", Triple("loongarch64"), 8, llvm::endianness::little,
+  LinkGraph G("foo", std::make_shared<orc::SymbolStringPool>(),
+              Triple("loongarch64"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, loongarch::Pointer64);
 
diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index a907dfcf2cec5..8ae05c4ddc59a 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -518,6 +518,71 @@ TEST_F(CoreAPIsStandardTest, TestTrivialCircularDependency) {
     << "Self-dependency prevented symbol from being marked ready";
 }
 
+TEST_F(CoreAPIsStandardTest, TestBasicQueryDependenciesReporting) {
+  // Test that dependencies are reported as expected.
+
+  bool DependenciesCallbackRan = false;
+
+  std::unique_ptr<MaterializationResponsibility> FooR;
+  std::unique_ptr<MaterializationResponsibility> BarR;
+
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Foo, FooSym.getFlags()}}),
+      [&](std::unique_ptr<MaterializationResponsibility> R) {
+        FooR = std::move(R);
+      })));
+
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Bar, BarSym.getFlags()}}),
+      [&](std::unique_ptr<MaterializationResponsibility> R) {
+        BarR = std::move(R);
+      })));
+
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Baz, BazSym.getFlags()}}),
+      [&](std::unique_ptr<MaterializationResponsibility> R) {
+        cantFail(R->notifyResolved({{Baz, BazSym}}));
+        cantFail(R->notifyEmitted({}));
+      })));
+
+  // First issue a lookup for Foo and Bar so that we can put them
+  // into the required states for the test lookup below.
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD),
+      SymbolLookupSet({Foo, Bar}), SymbolState::Resolved,
+      [](Expected<SymbolMap> Result) {
+        EXPECT_THAT_EXPECTED(std::move(Result), Succeeded());
+      },
+      NoDependenciesToRegister);
+
+  cantFail(FooR->notifyResolved({{Foo, FooSym}}));
+  cantFail(FooR->notifyEmitted({}));
+
+  cantFail(BarR->notifyResolved({{Bar, BarSym}}));
+
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD),
+      SymbolLookupSet({Foo, Bar, Baz}), SymbolState::Resolved,
+      [](Expected<SymbolMap> Result) {
+        EXPECT_THAT_EXPECTED(std::move(Result), Succeeded());
+      },
+      [&](const SymbolDependenceMap &Dependencies) {
+        EXPECT_EQ(Dependencies.size(), 1U)
+            << "Expect dependencies on only one JITDylib";
+        EXPECT_TRUE(Dependencies.count(&JD))
+            << "Expect dependencies on JD only";
+        auto &Deps = Dependencies.begin()->second;
+        EXPECT_EQ(Deps.size(), 2U);
+        EXPECT_TRUE(Deps.count(Bar));
+        EXPECT_TRUE(Deps.count(Baz));
+        DependenciesCallbackRan = true;
+      });
+
+  cantFail(BarR->notifyEmitted({}));
+
+  EXPECT_TRUE(DependenciesCallbackRan);
+}
+
 TEST_F(CoreAPIsStandardTest, TestCircularDependenceInOneJITDylib) {
   // Test that a circular symbol dependency between three symbols in a JITDylib
   // does not prevent any symbol from becoming 'ready' once all symbols are
diff --git a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
index d3a66294bd571..1b96b988068f2 100644
--- a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
@@ -114,10 +114,10 @@ TEST(EPCGenericJITLinkMemoryManagerTest, AllocFinalizeFree) {
   SAs.Deallocate = ExecutorAddr::fromPtr(&testDeallocate);
 
   auto MemMgr = std::make_unique<EPCGenericJITLinkMemoryManager>(*SelfEPC, SAs);
-
   StringRef Hello = "hello";
   auto SSA = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {Hello.size(), Align(1)}}});
+      *MemMgr, std::make_shared<orc::SymbolStringPool>(), nullptr,
+      {{MemProt::Read, {Hello.size(), Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA, Succeeded());
   auto SegInfo = SSA->getSegInfo(MemProt::Read);
   memcpy(SegInfo.WorkingMem.data(), Hello.data(), Hello.size());
diff --git a/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp
index 2e9733cbb039b..4b8a3efe680f1 100644
--- a/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp
@@ -81,8 +81,8 @@ TEST_F(JITLinkRedirectionManagerTest, BasicRedirectionOperation) {
                         JD->getDefaultResourceTracker(),
                         {{RedirectableSymbol, MakeTarget(initialTarget)}}),
                     Succeeded());
-  auto RTDef = cantFail(ES->lookup({JD}, RedirectableSymbol));
 
+  auto RTDef = cantFail(ES->lookup({JD}, RedirectableSymbol));
   auto RTPtr = RTDef.getAddress().toPtr<int (*)()>();
   auto Result = RTPtr();
   EXPECT_EQ(Result, 42) << "Failed to call initial target";
diff --git a/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp
index 46e558aad0d00..61dd1a4234d54 100644
--- a/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp
@@ -76,7 +76,8 @@ TEST(MapperJITLinkMemoryManagerTest, InProcess) {
 
   StringRef Hello = "hello";
   auto SSA1 = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {Hello.size(), Align(1)}}});
+      *MemMgr, std::make_shared<orc::SymbolStringPool>(), nullptr,
+      {{MemProt::Read, {Hello.size(), Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA1, Succeeded());
 
   EXPECT_EQ(Counter->ReserveCount, 1);
@@ -92,7 +93,8 @@ TEST(MapperJITLinkMemoryManagerTest, InProcess) {
   EXPECT_EQ(Counter->InitCount, 1);
 
   auto SSA2 = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {Hello.size(), Align(1)}}});
+      *MemMgr, std::make_shared<orc::SymbolStringPool>(), nullptr,
+      {{MemProt::Read, {Hello.size(), Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA2, Succeeded());
 
   // last reservation should be reused
@@ -135,9 +137,10 @@ TEST(MapperJITLinkMemoryManagerTest, Coalescing) {
   auto Mapper = cantFail(InProcessMemoryMapper::Create());
   auto MemMgr = std::make_unique<MapperJITLinkMemoryManager>(16 * 1024 * 1024,
                                                              std::move(Mapper));
+  auto SSP = std::make_shared<orc::SymbolStringPool>();
 
   auto SSA1 = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {1024, Align(1)}}});
+      *MemMgr, SSP, nullptr, {{MemProt::Read, {1024, Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA1, Succeeded());
   auto SegInfo1 = SSA1->getSegInfo(MemProt::Read);
   ExecutorAddr TargetAddr1(SegInfo1.Addr);
@@ -145,7 +148,7 @@ TEST(MapperJITLinkMemoryManagerTest, Coalescing) {
   EXPECT_THAT_EXPECTED(FA1, Succeeded());
 
   auto SSA2 = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {1024, Align(1)}}});
+      *MemMgr, SSP, nullptr, {{MemProt::Read, {1024, Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA2, Succeeded());
   auto FA2 = SSA2->finalize();
   EXPECT_THAT_EXPECTED(FA2, Succeeded());
@@ -157,7 +160,7 @@ TEST(MapperJITLinkMemoryManagerTest, Coalescing) {
   EXPECT_THAT_ERROR(std::move(Err3), Succeeded());
 
   auto SSA3 = jitlink::SimpleSegmentAlloc::Create(
-      *MemMgr, nullptr, {{MemProt::Read, {2048, Align(1)}}});
+      *MemMgr, SSP, nullptr, {{MemProt::Read, {2048, Align(1)}}});
   EXPECT_THAT_EXPECTED(SSA3, Succeeded());
 
   auto SegInfo3 = SSA3->getSegInfo(MemProt::Read);
diff --git a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
index bc996711f7ec5..31a48e86539fd 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 
@@ -44,9 +45,9 @@ class ObjectLinkingLayerTest : public testing::Test {
 };
 
 TEST_F(ObjectLinkingLayerTest, AddLinkGraph) {
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>(
+      "foo", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+      llvm::endianness::little, x86_64::getEdgeKindName);
 
   auto &Sec1 = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &B1 = G->createContentBlock(Sec1, BlockContent,
@@ -71,9 +72,9 @@ TEST_F(ObjectLinkingLayerTest, ResourceTracker) {
   // may invalidate some iterators internally.
   std::vector<ResourceTrackerSP> Trackers;
   for (unsigned I = 0; I < 64; I++) {
-    auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"),
-                                         8, llvm::endianness::little,
-                                         x86_64::getEdgeKindName);
+    auto G = std::make_unique<LinkGraph>(
+        "foo", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+        llvm::endianness::little, x86_64::getEdgeKindName);
 
     auto &Sec1 = G->createSection("__data", MemProt::Read | MemProt::Write);
     auto &B1 = G->createContentBlock(Sec1, BlockContent,
@@ -138,10 +139,9 @@ TEST_F(ObjectLinkingLayerTest, ClaimLateDefinedWeakSymbols) {
   };
 
   ObjLinkingLayer.addPlugin(std::make_unique<TestPlugin>());
-
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>(
+      "foo", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+      llvm::endianness::little, getGenericEdgeKindName);
 
   auto &DataSec = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &DataBlock = G->createContentBlock(DataSec, BlockContent,
@@ -192,10 +192,9 @@ TEST_F(ObjectLinkingLayerTest, HandleErrorDuringPostAllocationPass) {
   ES.setErrorReporter(consumeError);
 
   ObjLinkingLayer.addPlugin(std::make_unique<TestPlugin>());
-
-  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       llvm::endianness::little,
-                                       x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>(
+      "foo", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+      llvm::endianness::little, getGenericEdgeKindName);
 
   auto &DataSec = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &DataBlock = G->createContentBlock(DataSec, BlockContent,
@@ -247,9 +246,9 @@ TEST_F(ObjectLinkingLayerTest, AddAndRemovePlugins) {
   ObjLinkingLayer.addPlugin(P);
 
   {
-    auto G1 = std::make_unique<LinkGraph>("G1", Triple("x86_64-apple-darwin"),
-                                          8, llvm::endianness::little,
-                                          x86_64::getEdgeKindName);
+    auto G1 = std::make_unique<LinkGraph>(
+        "G1", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+        llvm::endianness::little, x86_64::getEdgeKindName);
 
     auto &DataSec = G1->createSection("__data", MemProt::Read | MemProt::Write);
     auto &DataBlock = G1->createContentBlock(DataSec, BlockContent,
@@ -265,9 +264,9 @@ TEST_F(ObjectLinkingLayerTest, AddAndRemovePlugins) {
   ObjLinkingLayer.removePlugin(*P);
 
   {
-    auto G2 = std::make_unique<LinkGraph>("G2", Triple("x86_64-apple-darwin"),
-                                          8, llvm::endianness::little,
-                                          x86_64::getEdgeKindName);
+    auto G2 = std::make_unique<LinkGraph>(
+        "G2", ES.getSymbolStringPool(), Triple("x86_64-apple-darwin"), 8,
+        llvm::endianness::little, x86_64::getEdgeKindName);
 
     auto &DataSec = G2->createSection("__data", MemProt::Read | MemProt::Write);
     auto &DataBlock = G2->createContentBlock(DataSec, BlockContent,
@@ -325,8 +324,8 @@ TEST(ObjectLinkingLayerSearchGeneratorTest, AbsoluteSymbolsObjectLayer) {
 
   auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
       ES, {}, [&](JITDylib &JD, SymbolMap Syms) {
-        auto G =
-            absoluteSymbolsLinkGraph(ES.getTargetTriple(), std::move(Syms));
+        auto G = absoluteSymbolsLinkGraph(
+            ES.getTargetTriple(), ES.getSymbolStringPool(), std::move(Syms));
         return ObjLinkingLayer.add(JD, std::move(G));
       });
   ASSERT_THAT_EXPECTED(G, Succeeded());
@@ -347,7 +346,7 @@ TEST(ObjectLinkingLayerSearchGeneratorTest, AbsoluteSymbolsObjectLayer) {
             ADD_FAILURE() << "unexpected unnamed symbol";
             continue;
           }
-          if (Sym->getName() == "_testFunc")
+          if (*Sym->getName() == "_testFunc")
             SawSymbolDef = true;
           else
             ADD_FAILURE() << "unexpected symbol " << Sym->getName();
diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
index f9541131e4b23..8f195c4f60326 100644
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -686,12 +686,12 @@ TEST_F(OpenMPDecompositionTest, Order1) {
   std::string Dir3 = stringify(Dec.output[3]);
   std::string Dir4 = stringify(Dec.output[4]);
   std::string Dir5 = stringify(Dec.output[5]);
-  ASSERT_EQ(Dir0, "target"); // (31)
-  ASSERT_EQ(Dir1, "teams");  // (31)
+  ASSERT_EQ(Dir0, "target");                 // (31)
+  ASSERT_EQ(Dir1, "teams");                  // (31)
   ASSERT_EQ(Dir2, "distribute order(1, 0)"); // (31)
-  ASSERT_EQ(Dir3, "parallel");         // (31)
-  ASSERT_EQ(Dir4, "for order(1, 0)");  // (31)
-  ASSERT_EQ(Dir5, "simd order(1, 0)"); // (31)
+  ASSERT_EQ(Dir3, "parallel");               // (31)
+  ASSERT_EQ(Dir4, "for order(1, 0)");        // (31)
+  ASSERT_EQ(Dir5, "simd order(1, 0)");       // (31)
 }
 
 // ALLOCATE
@@ -708,8 +708,7 @@ TEST_F(OpenMPDecompositionTest, Allocate1) {
 
   // Allocate + firstprivate
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
   };
 
@@ -719,8 +718,8 @@ TEST_F(OpenMPDecompositionTest, Allocate1) {
 
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");                           // (33)
-  ASSERT_EQ(Dir1, "sections firstprivate(x) allocate(, , , (x))"); // (33)
+  ASSERT_EQ(Dir0, "parallel shared(x)");                         // (33)
+  ASSERT_EQ(Dir1, "sections firstprivate(x) allocate(, , (x))"); // (33)
 }
 
 TEST_F(OpenMPDecompositionTest, Allocate2) {
@@ -729,8 +728,7 @@ TEST_F(OpenMPDecompositionTest, Allocate2) {
 
   // Allocate + in_reduction
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_in_reduction, omp::clause::InReduction{{{Add}, {x}}}},
   };
 
@@ -740,8 +738,8 @@ TEST_F(OpenMPDecompositionTest, Allocate2) {
 
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "target in_reduction((3), (x)) allocate(, , , (x))"); // (33)
-  ASSERT_EQ(Dir1, "parallel");                                          // (33)
+  ASSERT_EQ(Dir0, "target in_reduction((3), (x)) allocate(, , (x))"); // (33)
+  ASSERT_EQ(Dir1, "parallel");                                        // (33)
 }
 
 TEST_F(OpenMPDecompositionTest, Allocate3) {
@@ -749,8 +747,7 @@ TEST_F(OpenMPDecompositionTest, Allocate3) {
 
   // Allocate + linear
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_linear,
        omp::clause::Linear{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
   };
@@ -765,7 +762,7 @@ TEST_F(OpenMPDecompositionTest, Allocate3) {
   // should be fixed eventually.
   ASSERT_EQ(Dir0, "parallel shared(x) shared(x)"); // (33)
   ASSERT_EQ(Dir1, "for linear(, , , (x)) firstprivate(x) lastprivate(, (x)) "
-                  "allocate(, , , (x))"); // (33)
+                  "allocate(, , (x))"); // (33)
 }
 
 TEST_F(OpenMPDecompositionTest, Allocate4) {
@@ -773,8 +770,7 @@ TEST_F(OpenMPDecompositionTest, Allocate4) {
 
   // Allocate + lastprivate
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
   };
 
@@ -784,8 +780,8 @@ TEST_F(OpenMPDecompositionTest, Allocate4) {
 
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");                              // (33)
-  ASSERT_EQ(Dir1, "sections lastprivate(, (x)) allocate(, , , (x))"); // (33)
+  ASSERT_EQ(Dir0, "parallel shared(x)");                            // (33)
+  ASSERT_EQ(Dir1, "sections lastprivate(, (x)) allocate(, , (x))"); // (33)
 }
 
 TEST_F(OpenMPDecompositionTest, Allocate5) {
@@ -793,8 +789,7 @@ TEST_F(OpenMPDecompositionTest, Allocate5) {
 
   // Allocate + private
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_private, omp::clause::Private{{x}}},
   };
 
@@ -804,8 +799,8 @@ TEST_F(OpenMPDecompositionTest, Allocate5) {
 
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel");                                // (33)
-  ASSERT_EQ(Dir1, "sections private(x) allocate(, , , (x))"); // (33)
+  ASSERT_EQ(Dir0, "parallel");                              // (33)
+  ASSERT_EQ(Dir1, "sections private(x) allocate(, , (x))"); // (33)
 }
 
 TEST_F(OpenMPDecompositionTest, Allocate6) {
@@ -814,8 +809,7 @@ TEST_F(OpenMPDecompositionTest, Allocate6) {
 
   // Allocate + reduction
   omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_allocate, omp::clause::Allocate{{std::nullopt, std::nullopt, {x}}}},
       {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
   };
 
@@ -825,8 +819,8 @@ TEST_F(OpenMPDecompositionTest, Allocate6) {
 
   std::string Dir0 = stringify(Dec.output[0]);
   std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");                                 // (33)
-  ASSERT_EQ(Dir1, "sections reduction(, (3), (x)) allocate(, , , (x))"); // (33)
+  ASSERT_EQ(Dir0, "parallel shared(x)");                               // (33)
+  ASSERT_EQ(Dir1, "sections reduction(, (3), (x)) allocate(, , (x))"); // (33)
 }
 
 // REDUCTION
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index e01dc95a80284..ba411b087617b 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -897,6 +897,20 @@ TEST(InstructionsTest, GEPIndices) {
   delete GEPI;
 }
 
+TEST(InstructionsTest, ZeroIndexGEP) {
+  LLVMContext Context;
+  DataLayout DL;
+  Type *PtrTy = PointerType::getUnqual(Context);
+  auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(Context),
+                                        PoisonValue::get(PtrTy), {});
+
+  APInt Offset(DL.getIndexTypeSizeInBits(PtrTy), 0);
+  EXPECT_TRUE(GEP->accumulateConstantOffset(DL, Offset));
+  EXPECT_TRUE(Offset.isZero());
+
+  delete GEP;
+}
+
 TEST(InstructionsTest, SwitchInst) {
   LLVMContext C;
 
@@ -1922,5 +1936,27 @@ TEST(InstructionsTest, AtomicSyncscope) {
   EXPECT_TRUE(LLVMIsAtomicSingleThread(CmpXchg));
 }
 
+TEST(InstructionsTest, CmpPredicate) {
+  CmpPredicate P0(CmpInst::ICMP_ULE, false), P1(CmpInst::ICMP_ULE, true),
+      P2(CmpInst::ICMP_SLE, false), P3(CmpInst::ICMP_SLT, false);
+  CmpPredicate Q0 = P0, Q1 = P1, Q2 = P2;
+  CmpInst::Predicate R0 = P0, R1 = P1, R2 = P2;
+
+  EXPECT_EQ(*CmpPredicate::getMatching(P0, P1), CmpInst::ICMP_ULE);
+  EXPECT_EQ(CmpPredicate::getMatching(P0, P1)->hasSameSign(), false);
+  EXPECT_EQ(*CmpPredicate::getMatching(P1, P1), CmpInst::ICMP_ULE);
+  EXPECT_EQ(CmpPredicate::getMatching(P1, P1)->hasSameSign(), true);
+  EXPECT_EQ(CmpPredicate::getMatching(P0, P2), std::nullopt);
+  EXPECT_EQ(*CmpPredicate::getMatching(P1, P2), CmpInst::ICMP_SLE);
+  EXPECT_EQ(CmpPredicate::getMatching(P1, P2)->hasSameSign(), false);
+  EXPECT_EQ(CmpPredicate::getMatching(P1, P3), std::nullopt);
+  EXPECT_FALSE(Q0.hasSameSign());
+  EXPECT_TRUE(Q1.hasSameSign());
+  EXPECT_FALSE(Q2.hasSameSign());
+  EXPECT_EQ(P0, R0);
+  EXPECT_EQ(P1, R1);
+  EXPECT_EQ(P2, R2);
+}
+
 } // end anonymous namespace
 } // end namespace llvm
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index b32a199144859..ad9af2075e371 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -63,24 +63,21 @@ class IntrinsicsTest : public ::testing::Test {
 };
 
 TEST(IntrinsicNameLookup, Basic) {
-  static constexpr const char *const NameTable[] = {
-      "llvm.foo", "llvm.foo.a", "llvm.foo.b", "llvm.foo.b.a", "llvm.foo.c",
-  };
+  using namespace Intrinsic;
+  EXPECT_EQ(Intrinsic::memcpy, lookupIntrinsicID("llvm.memcpy"));
 
-  static constexpr std::pair<const char *, int> Tests[] = {
-      {"llvm.foo", 0},     {"llvm.foo.f64", 0}, {"llvm.foo.b", 2},
-      {"llvm.foo.b.a", 3}, {"llvm.foo.c", 4},   {"llvm.foo.c.f64", 4},
-      {"llvm.bar", -1},
-  };
+  // Partial, either between dots or not the last dot are not intrinsics.
+  EXPECT_EQ(not_intrinsic, lookupIntrinsicID("llvm.mem"));
+  EXPECT_EQ(not_intrinsic, lookupIntrinsicID("llvm.gc"));
 
-  for (const auto &[Name, ExpectedIdx] : Tests) {
-    int Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name);
-    EXPECT_EQ(ExpectedIdx, Idx);
-    if (!StringRef(Name).starts_with("llvm.foo"))
-      continue;
-    Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name, "foo");
-    EXPECT_EQ(ExpectedIdx, Idx);
-  }
+  // Look through intrinsic names with internal dots.
+  EXPECT_EQ(memcpy_inline, lookupIntrinsicID("llvm.memcpy.inline"));
+
+  // Check that overloaded names are mapped to the underlying ID.
+  EXPECT_EQ(memcpy_inline, lookupIntrinsicID("llvm.memcpy.inline.p0.p0.i8"));
+  EXPECT_EQ(memcpy_inline, lookupIntrinsicID("llvm.memcpy.inline.p0.p0.i32"));
+  EXPECT_EQ(memcpy_inline, lookupIntrinsicID("llvm.memcpy.inline.p0.p0.i64"));
+  EXPECT_EQ(memcpy_inline, lookupIntrinsicID("llvm.memcpy.inline.p0.p0.i1024"));
 }
 
 // Tests to verify getIntrinsicForClangBuiltin.
diff --git a/llvm/unittests/IR/TypesTest.cpp b/llvm/unittests/IR/TypesTest.cpp
index 855262fc4e787..4436ed5118f4f 100644
--- a/llvm/unittests/IR/TypesTest.cpp
+++ b/llvm/unittests/IR/TypesTest.cpp
@@ -40,10 +40,40 @@ TEST(TypesTest, TargetExtType) {
   Type *A = TargetExtType::get(Context, "typea");
   Type *Aparam = TargetExtType::get(Context, "typea", {}, {0, 1});
   Type *Aparam2 = TargetExtType::get(Context, "typea", {}, {0, 1});
+
   // Opaque types with same parameters are identical...
   EXPECT_EQ(Aparam, Aparam2);
   // ... but just having the same name is not enough.
   EXPECT_NE(A, Aparam);
+
+  // ensure struct types in targest extension types
+  // only show the struct name, not the struct body
+  Type *Int32Type = Type::getInt32Ty(Context);
+  Type *FloatType = Type::getFloatTy(Context);
+  std::array<Type *, 2> Elements = {Int32Type, FloatType};
+
+  StructType *Struct = llvm::StructType::create(Context, Elements, "MyStruct",
+                                                /*isPacked=*/false);
+  SmallVector<char, 50> TETV;
+  llvm::raw_svector_ostream TETStream(TETV);
+  Type *TargetExtensionType =
+      TargetExtType::get(Context, "structTET", {Struct}, {0, 1});
+  TargetExtensionType->print(TETStream);
+
+  EXPECT_STREQ(TETStream.str().str().data(),
+               "target(\"structTET\", %MyStruct, 0, 1)");
+
+  // ensure that literal structs in the target extension type print the struct
+  // body
+  Struct = StructType::get(Context, Struct->elements(), /*isPacked=*/false);
+
+  TargetExtensionType =
+      TargetExtType::get(Context, "structTET", {Struct}, {0, 1});
+  TETV.clear();
+  TargetExtensionType->print(TETStream);
+
+  EXPECT_STREQ(TETStream.str().str().data(),
+               "target(\"structTET\", { i32, float }, 0, 1)");
 }
 
 TEST(TypedPointerType, PrintTest) {
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index f366b228e6351..ac872d8235622 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -26,6 +26,7 @@
 
 using namespace llvm;
 using ::llvm::memprof::LineLocation;
+using ::testing::ElementsAre;
 using ::testing::EndsWith;
 using ::testing::IsSubsetOf;
 using ::testing::Pair;
@@ -348,31 +349,25 @@ TEST_F(InstrProfTest, test_merge_traces_sampled) {
       IsSubsetOf({FooTrace, BarTrace, GooTrace, BarTrace, GooTrace, FooTrace}));
 }
 
+using ::llvm::memprof::IndexedMemProfData;
 using ::llvm::memprof::IndexedMemProfRecord;
 using ::llvm::memprof::MemInfoBlock;
-using FrameIdMapTy =
-    llvm::MapVector<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
-using CallStackIdMapTy =
-    llvm::MapVector<::llvm::memprof::CallStackId,
-                    ::llvm::SmallVector<::llvm::memprof::FrameId>>;
-
-static FrameIdMapTy getFrameMapping() {
-  FrameIdMapTy Mapping;
-  Mapping.insert({0, {0x123, 1, 2, false}});
-  Mapping.insert({1, {0x345, 3, 4, true}});
-  Mapping.insert({2, {0x125, 5, 6, false}});
-  Mapping.insert({3, {0x567, 7, 8, true}});
-  Mapping.insert({4, {0x124, 5, 6, false}});
-  Mapping.insert({5, {0x789, 8, 9, true}});
-  return Mapping;
-}
 
-static CallStackIdMapTy getCallStackMapping() {
-  CallStackIdMapTy Mapping;
-  Mapping.insert({0x111, {0, 1}});
-  Mapping.insert({0x222, {2, 3}});
-  Mapping.insert({0x333, {4, 5}});
-  return Mapping;
+IndexedMemProfData getMemProfDataForTest() {
+  IndexedMemProfData MemProfData;
+
+  MemProfData.Frames.insert({0, {0x123, 1, 2, false}});
+  MemProfData.Frames.insert({1, {0x345, 3, 4, true}});
+  MemProfData.Frames.insert({2, {0x125, 5, 6, false}});
+  MemProfData.Frames.insert({3, {0x567, 7, 8, true}});
+  MemProfData.Frames.insert({4, {0x124, 5, 6, false}});
+  MemProfData.Frames.insert({5, {0x789, 8, 9, true}});
+
+  MemProfData.CallStacks.insert({0x111, {0, 1}});
+  MemProfData.CallStacks.insert({0x222, {2, 3}});
+  MemProfData.CallStacks.insert({0x333, {4, 5}});
+
+  return MemProfData;
 }
 
 // Populate all of the fields of MIB.
@@ -398,10 +393,8 @@ IndexedMemProfRecord
 makeRecordV2(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames,
              std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames,
              const MemInfoBlock &Block, const memprof::MemProfSchema &Schema) {
-  llvm::memprof::IndexedMemProfRecord MR;
+  IndexedMemProfRecord MR;
   for (const auto &CSId : AllocFrames)
-    // We don't populate IndexedAllocationInfo::CallStack because we use it only
-    // in Version1.
     MR.AllocSites.emplace_back(CSId, Block, Schema);
   for (const auto &CSId : CallSiteFrames)
     MR.CallSiteIds.push_back(CSId);
@@ -453,9 +446,7 @@ TEST_F(InstrProfTest, test_memprof_v2_full_schema) {
   const IndexedMemProfRecord IndexedMR = makeRecordV2(
       /*AllocFrames=*/{0x111, 0x222},
       /*CallSiteFrames=*/{0x333}, MIB, memprof::getFullSchema());
-  memprof::IndexedMemProfData MemProfData;
-  MemProfData.Frames = getFrameMapping();
-  MemProfData.CallStacks = getCallStackMapping();
+  IndexedMemProfData MemProfData = getMemProfDataForTest();
   MemProfData.Records.try_emplace(0x9999, IndexedMR);
   Writer.addMemProfData(MemProfData, Err);
 
@@ -492,9 +483,7 @@ TEST_F(InstrProfTest, test_memprof_v2_partial_schema) {
   const IndexedMemProfRecord IndexedMR = makeRecordV2(
       /*AllocFrames=*/{0x111, 0x222},
       /*CallSiteFrames=*/{0x333}, MIB, memprof::getHotColdSchema());
-  memprof::IndexedMemProfData MemProfData;
-  MemProfData.Frames = getFrameMapping();
-  MemProfData.CallStacks = getCallStackMapping();
+  IndexedMemProfData MemProfData = getMemProfDataForTest();
   MemProfData.Records.try_emplace(0x9999, IndexedMR);
   Writer.addMemProfData(MemProfData, Err);
 
@@ -544,7 +533,7 @@ TEST_F(InstrProfTest, test_caller_callee_pairs) {
       /*AllocFrames=*/{0x111, 0x222},
       /*CallSiteFrames=*/{}, MIB, memprof::getHotColdSchema());
 
-  memprof::IndexedMemProfData MemProfData;
+  IndexedMemProfData MemProfData;
   MemProfData.Frames.try_emplace(0, 0x123, 1, 2, false);
   MemProfData.Frames.try_emplace(1, 0x234, 3, 4, true);
   MemProfData.Frames.try_emplace(2, 0x123, 5, 6, false);
@@ -564,19 +553,16 @@ TEST_F(InstrProfTest, test_caller_callee_pairs) {
 
   auto It = Pairs.find(0x123);
   ASSERT_NE(It, Pairs.end());
-  ASSERT_THAT(It->second, SizeIs(2));
-  EXPECT_THAT(It->second[0], Pair(LineLocation(1, 2), 0x234U));
-  EXPECT_THAT(It->second[1], Pair(LineLocation(5, 6), 0x345U));
+  EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(1, 2), 0x234U),
+                                      Pair(LineLocation(5, 6), 0x345U)));
 
   It = Pairs.find(0x234);
   ASSERT_NE(It, Pairs.end());
-  ASSERT_THAT(It->second, SizeIs(1));
-  EXPECT_THAT(It->second[0], Pair(LineLocation(3, 4), 0U));
+  EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(3, 4), 0U)));
 
   It = Pairs.find(0x345);
   ASSERT_NE(It, Pairs.end());
-  ASSERT_THAT(It->second, SizeIs(1));
-  EXPECT_THAT(It->second[0], Pair(LineLocation(7, 8), 0U));
+  EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(7, 8), 0U)));
 }
 
 TEST_F(InstrProfTest, test_memprof_getrecord_error) {
@@ -606,9 +592,7 @@ TEST_F(InstrProfTest, test_memprof_merge) {
       /*AllocFrames=*/{0x111, 0x222},
       /*CallSiteFrames=*/{}, makePartialMIB(), memprof::getHotColdSchema());
 
-  memprof::IndexedMemProfData MemProfData;
-  MemProfData.Frames = getFrameMapping();
-  MemProfData.CallStacks = getCallStackMapping();
+  IndexedMemProfData MemProfData = getMemProfDataForTest();
   MemProfData.Records.try_emplace(0x9999, IndexedMR);
   Writer2.addMemProfData(MemProfData, Err);
 
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index b3b6249342d63..5886d3e2bcf84 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -22,6 +22,8 @@
 
 #include <initializer_list>
 
+namespace llvm {
+namespace memprof {
 namespace {
 
 using ::llvm::DIGlobal;
@@ -30,24 +32,14 @@ using ::llvm::DILineInfo;
 using ::llvm::DILineInfoSpecifier;
 using ::llvm::DILocal;
 using ::llvm::StringRef;
-using ::llvm::memprof::CallStackId;
-using ::llvm::memprof::CallStackMap;
-using ::llvm::memprof::Frame;
-using ::llvm::memprof::FrameId;
-using ::llvm::memprof::IndexedAllocationInfo;
-using ::llvm::memprof::IndexedMemProfRecord;
-using ::llvm::memprof::MemInfoBlock;
-using ::llvm::memprof::MemProfReader;
-using ::llvm::memprof::MemProfRecord;
-using ::llvm::memprof::MemProfSchema;
-using ::llvm::memprof::Meta;
-using ::llvm::memprof::PortableMemInfoBlock;
-using ::llvm::memprof::RawMemProfReader;
-using ::llvm::memprof::SegmentEntry;
 using ::llvm::object::SectionedAddress;
 using ::llvm::symbolize::SymbolizableModule;
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+using ::testing::Pair;
 using ::testing::Return;
 using ::testing::SizeIs;
+using ::testing::UnorderedElementsAre;
 
 class MockSymbolizer : public SymbolizableModule {
 public:
@@ -132,7 +124,7 @@ MATCHER_P4(FrameContains, FunctionName, LineOffset, Column, Inline, "") {
 }
 
 TEST(MemProf, FillsValue) {
-  std::unique_ptr<MockSymbolizer> Symbolizer(new MockSymbolizer());
+  auto Symbolizer = std::make_unique<MockSymbolizer>();
 
   EXPECT_CALL(*Symbolizer, symbolizeInlinedCode(SectionedAddress{0x1000},
                                                 specifier(), false))
@@ -217,31 +209,28 @@ TEST(MemProf, FillsValue) {
   EXPECT_THAT(Bar.AllocSites[0].CallStack[3],
               FrameContains("abc", 5U, 30U, false));
 
-  ASSERT_THAT(Bar.CallSites, SizeIs(1));
-  ASSERT_THAT(Bar.CallSites[0], SizeIs(2));
-  EXPECT_THAT(Bar.CallSites[0][0], FrameContains("foo", 5U, 30U, true));
-  EXPECT_THAT(Bar.CallSites[0][1], FrameContains("bar", 51U, 20U, false));
+  EXPECT_THAT(Bar.CallSites,
+              ElementsAre(ElementsAre(FrameContains("foo", 5U, 30U, true),
+                                      FrameContains("bar", 51U, 20U, false))));
 
   // Check the memprof record for xyz.
   const llvm::GlobalValue::GUID XyzId = IndexedMemProfRecord::getGUID("xyz");
   ASSERT_TRUE(Records.contains(XyzId));
   const MemProfRecord &Xyz = Records[XyzId];
-  ASSERT_THAT(Xyz.CallSites, SizeIs(1));
-  ASSERT_THAT(Xyz.CallSites[0], SizeIs(2));
   // Expect the entire frame even though in practice we only need the first
   // entry here.
-  EXPECT_THAT(Xyz.CallSites[0][0], FrameContains("xyz", 5U, 30U, true));
-  EXPECT_THAT(Xyz.CallSites[0][1], FrameContains("abc", 5U, 30U, false));
+  EXPECT_THAT(Xyz.CallSites,
+              ElementsAre(ElementsAre(FrameContains("xyz", 5U, 30U, true),
+                                      FrameContains("abc", 5U, 30U, false))));
 
   // Check the memprof record for abc.
   const llvm::GlobalValue::GUID AbcId = IndexedMemProfRecord::getGUID("abc");
   ASSERT_TRUE(Records.contains(AbcId));
   const MemProfRecord &Abc = Records[AbcId];
   EXPECT_TRUE(Abc.AllocSites.empty());
-  ASSERT_THAT(Abc.CallSites, SizeIs(1));
-  ASSERT_THAT(Abc.CallSites[0], SizeIs(2));
-  EXPECT_THAT(Abc.CallSites[0][0], FrameContains("xyz", 5U, 30U, true));
-  EXPECT_THAT(Abc.CallSites[0][1], FrameContains("abc", 5U, 30U, false));
+  EXPECT_THAT(Abc.CallSites,
+              ElementsAre(ElementsAre(FrameContains("xyz", 5U, 30U, true),
+                                      FrameContains("abc", 5U, 30U, false))));
 }
 
 TEST(MemProf, PortableWrapper) {
@@ -249,7 +238,7 @@ TEST(MemProf, PortableWrapper) {
                     /*dealloc_timestamp=*/2000, /*alloc_cpu=*/3,
                     /*dealloc_cpu=*/4, /*Histogram=*/0, /*HistogramSize=*/0);
 
-  const auto Schema = llvm::memprof::getFullSchema();
+  const auto Schema = getFullSchema();
   PortableMemInfoBlock WriteBlock(Info, Schema);
 
   std::string Buffer;
@@ -269,15 +258,15 @@ TEST(MemProf, PortableWrapper) {
 }
 
 TEST(MemProf, RecordSerializationRoundTripVerion2) {
-  const auto Schema = llvm::memprof::getFullSchema();
+  const auto Schema = getFullSchema();
 
   MemInfoBlock Info(/*size=*/16, /*access_count=*/7, /*alloc_timestamp=*/1000,
                     /*dealloc_timestamp=*/2000, /*alloc_cpu=*/3,
                     /*dealloc_cpu=*/4, /*Histogram=*/0, /*HistogramSize=*/0);
 
-  llvm::SmallVector<llvm::memprof::CallStackId> CallStackIds = {0x123, 0x456};
+  llvm::SmallVector<CallStackId> CallStackIds = {0x123, 0x456};
 
-  llvm::SmallVector<llvm::memprof::CallStackId> CallSiteIds = {0x333, 0x444};
+  llvm::SmallVector<CallStackId> CallSiteIds = {0x333, 0x444};
 
   IndexedMemProfRecord Record;
   for (const auto &CSId : CallStackIds) {
@@ -288,17 +277,16 @@ TEST(MemProf, RecordSerializationRoundTripVerion2) {
 
   std::string Buffer;
   llvm::raw_string_ostream OS(Buffer);
-  Record.serialize(Schema, OS, llvm::memprof::Version2);
+  Record.serialize(Schema, OS, Version2);
 
   const IndexedMemProfRecord GotRecord = IndexedMemProfRecord::deserialize(
-      Schema, reinterpret_cast<const unsigned char *>(Buffer.data()),
-      llvm::memprof::Version2);
+      Schema, reinterpret_cast<const unsigned char *>(Buffer.data()), Version2);
 
   EXPECT_EQ(Record, GotRecord);
 }
 
 TEST(MemProf, RecordSerializationRoundTripVersion2HotColdSchema) {
-  const auto Schema = llvm::memprof::getHotColdSchema();
+  const auto Schema = getHotColdSchema();
 
   MemInfoBlock Info;
   Info.AllocCount = 11;
@@ -306,9 +294,9 @@ TEST(MemProf, RecordSerializationRoundTripVersion2HotColdSchema) {
   Info.TotalLifetime = 33;
   Info.TotalLifetimeAccessDensity = 44;
 
-  llvm::SmallVector<llvm::memprof::CallStackId> CallStackIds = {0x123, 0x456};
+  llvm::SmallVector<CallStackId> CallStackIds = {0x123, 0x456};
 
-  llvm::SmallVector<llvm::memprof::CallStackId> CallSiteIds = {0x333, 0x444};
+  llvm::SmallVector<CallStackId> CallSiteIds = {0x333, 0x444};
 
   IndexedMemProfRecord Record;
   for (const auto &CSId : CallStackIds) {
@@ -332,22 +320,21 @@ TEST(MemProf, RecordSerializationRoundTripVersion2HotColdSchema) {
 
   // Verify that Schema has propagated all the way to the Info field in each
   // IndexedAllocationInfo.
-  ASSERT_THAT(Record.AllocSites, ::SizeIs(2));
+  ASSERT_THAT(Record.AllocSites, SizeIs(2));
   EXPECT_EQ(Record.AllocSites[0].Info.getSchema(), SchemaBitSet);
   EXPECT_EQ(Record.AllocSites[1].Info.getSchema(), SchemaBitSet);
 
   std::string Buffer;
   llvm::raw_string_ostream OS(Buffer);
-  Record.serialize(Schema, OS, llvm::memprof::Version2);
+  Record.serialize(Schema, OS, Version2);
 
   const IndexedMemProfRecord GotRecord = IndexedMemProfRecord::deserialize(
-      Schema, reinterpret_cast<const unsigned char *>(Buffer.data()),
-      llvm::memprof::Version2);
+      Schema, reinterpret_cast<const unsigned char *>(Buffer.data()), Version2);
 
   // Verify that Schema comes back correctly after deserialization. Technically,
   // the comparison between Record and GotRecord below includes the comparison
   // of their Schemas, but we'll verify the Schemas on our own.
-  ASSERT_THAT(GotRecord.AllocSites, ::SizeIs(2));
+  ASSERT_THAT(GotRecord.AllocSites, SizeIs(2));
   EXPECT_EQ(GotRecord.AllocSites[0].Info.getSchema(), SchemaBitSet);
   EXPECT_EQ(GotRecord.AllocSites[1].Info.getSchema(), SchemaBitSet);
 
@@ -355,7 +342,7 @@ TEST(MemProf, RecordSerializationRoundTripVersion2HotColdSchema) {
 }
 
 TEST(MemProf, SymbolizationFilter) {
-  std::unique_ptr<MockSymbolizer> Symbolizer(new MockSymbolizer());
+  auto Symbolizer = std::make_unique<MockSymbolizer>();
 
   EXPECT_CALL(*Symbolizer, symbolizeInlinedCode(SectionedAddress{0x1000},
                                                 specifier(), false))
@@ -415,23 +402,21 @@ TEST(MemProf, SymbolizationFilter) {
 
   ASSERT_THAT(Records, SizeIs(1));
   ASSERT_THAT(Records[0].AllocSites, SizeIs(1));
-  ASSERT_THAT(Records[0].AllocSites[0].CallStack, SizeIs(1));
-  EXPECT_THAT(Records[0].AllocSites[0].CallStack[0],
-              FrameContains("foo", 5U, 30U, false));
+  EXPECT_THAT(Records[0].AllocSites[0].CallStack,
+              ElementsAre(FrameContains("foo", 5U, 30U, false)));
 }
 
 TEST(MemProf, BaseMemProfReader) {
-  llvm::memprof::IndexedMemProfData MemProfData;
+  IndexedMemProfData MemProfData;
   Frame F1(/*Hash=*/IndexedMemProfRecord::getGUID("foo"), /*LineOffset=*/20,
            /*Column=*/5, /*IsInlineFrame=*/true);
   Frame F2(/*Hash=*/IndexedMemProfRecord::getGUID("bar"), /*LineOffset=*/10,
            /*Column=*/2, /*IsInlineFrame=*/false);
-  MemProfData.Frames.insert({F1.hash(), F1});
-  MemProfData.Frames.insert({F2.hash(), F2});
+  MemProfData.addFrame(F1);
+  MemProfData.addFrame(F2);
 
   llvm::SmallVector<FrameId> CallStack{F1.hash(), F2.hash()};
-  CallStackId CSId = llvm::memprof::hashCallStack(CallStack);
-  MemProfData.CallStacks.try_emplace(CSId, CallStack);
+  CallStackId CSId = MemProfData.addCallStack(std::move(CallStack));
 
   IndexedMemProfRecord FakeRecord;
   MemInfoBlock Block;
@@ -449,32 +434,29 @@ TEST(MemProf, BaseMemProfReader) {
 
   ASSERT_THAT(Records, SizeIs(1));
   ASSERT_THAT(Records[0].AllocSites, SizeIs(1));
-  ASSERT_THAT(Records[0].AllocSites[0].CallStack, SizeIs(2));
-  EXPECT_THAT(Records[0].AllocSites[0].CallStack[0],
-              FrameContains("foo", 20U, 5U, true));
-  EXPECT_THAT(Records[0].AllocSites[0].CallStack[1],
-              FrameContains("bar", 10U, 2U, false));
+  EXPECT_THAT(Records[0].AllocSites[0].CallStack,
+              ElementsAre(FrameContains("foo", 20U, 5U, true),
+                          FrameContains("bar", 10U, 2U, false)));
 }
 
 TEST(MemProf, BaseMemProfReaderWithCSIdMap) {
-  llvm::memprof::IndexedMemProfData MemProfData;
+  IndexedMemProfData MemProfData;
   Frame F1(/*Hash=*/IndexedMemProfRecord::getGUID("foo"), /*LineOffset=*/20,
            /*Column=*/5, /*IsInlineFrame=*/true);
   Frame F2(/*Hash=*/IndexedMemProfRecord::getGUID("bar"), /*LineOffset=*/10,
            /*Column=*/2, /*IsInlineFrame=*/false);
-  MemProfData.Frames.insert({F1.hash(), F1});
-  MemProfData.Frames.insert({F2.hash(), F2});
+  MemProfData.addFrame(F1);
+  MemProfData.addFrame(F2);
 
   llvm::SmallVector<FrameId> CallStack = {F1.hash(), F2.hash()};
-  CallStackId CSId = llvm::memprof::hashCallStack(CallStack);
-  MemProfData.CallStacks.insert({CSId, CallStack});
+  MemProfData.addCallStack(CallStack);
 
   IndexedMemProfRecord FakeRecord;
   MemInfoBlock Block;
   Block.AllocCount = 1U, Block.TotalAccessDensity = 4,
   Block.TotalLifetime = 200001;
   FakeRecord.AllocSites.emplace_back(
-      /*CSId=*/llvm::memprof::hashCallStack(CallStack),
+      /*CSId=*/hashCallStack(CallStack),
       /*MB=*/Block);
   MemProfData.Records.insert({F1.hash(), FakeRecord});
 
@@ -487,49 +469,47 @@ TEST(MemProf, BaseMemProfReaderWithCSIdMap) {
 
   ASSERT_THAT(Records, SizeIs(1));
   ASSERT_THAT(Records[0].AllocSites, SizeIs(1));
-  ASSERT_THAT(Records[0].AllocSites[0].CallStack, SizeIs(2));
-  EXPECT_THAT(Records[0].AllocSites[0].CallStack[0],
-              FrameContains("foo", 20U, 5U, true));
-  EXPECT_THAT(Records[0].AllocSites[0].CallStack[1],
-              FrameContains("bar", 10U, 2U, false));
+  EXPECT_THAT(Records[0].AllocSites[0].CallStack,
+              ElementsAre(FrameContains("foo", 20U, 5U, true),
+                          FrameContains("bar", 10U, 2U, false)));
 }
 
 TEST(MemProf, IndexedMemProfRecordToMemProfRecord) {
   // Verify that MemProfRecord can be constructed from IndexedMemProfRecord with
   // CallStackIds only.
 
-  llvm::DenseMap<FrameId, Frame> FrameIdMap;
+  IndexedMemProfData MemProfData;
   Frame F1(1, 0, 0, false);
   Frame F2(2, 0, 0, false);
   Frame F3(3, 0, 0, false);
   Frame F4(4, 0, 0, false);
-  FrameIdMap.insert({F1.hash(), F1});
-  FrameIdMap.insert({F2.hash(), F2});
-  FrameIdMap.insert({F3.hash(), F3});
-  FrameIdMap.insert({F4.hash(), F4});
+  MemProfData.addFrame(F1);
+  MemProfData.addFrame(F2);
+  MemProfData.addFrame(F3);
+  MemProfData.addFrame(F4);
 
-  llvm::DenseMap<CallStackId, llvm::SmallVector<FrameId>> CallStackIdMap;
   llvm::SmallVector<FrameId> CS1 = {F1.hash(), F2.hash()};
   llvm::SmallVector<FrameId> CS2 = {F1.hash(), F3.hash()};
   llvm::SmallVector<FrameId> CS3 = {F2.hash(), F3.hash()};
   llvm::SmallVector<FrameId> CS4 = {F2.hash(), F4.hash()};
-  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS1), CS1});
-  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS2), CS2});
-  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS3), CS3});
-  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS4), CS4});
+  MemProfData.addCallStack(CS1);
+  MemProfData.addCallStack(CS2);
+  MemProfData.addCallStack(CS3);
+  MemProfData.addCallStack(CS4);
 
   IndexedMemProfRecord IndexedRecord;
   IndexedAllocationInfo AI;
-  AI.CSId = llvm::memprof::hashCallStack(CS1);
+  AI.CSId = hashCallStack(CS1);
   IndexedRecord.AllocSites.push_back(AI);
-  AI.CSId = llvm::memprof::hashCallStack(CS2);
+  AI.CSId = hashCallStack(CS2);
   IndexedRecord.AllocSites.push_back(AI);
-  IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS3));
-  IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS4));
+  IndexedRecord.CallSiteIds.push_back(hashCallStack(CS3));
+  IndexedRecord.CallSiteIds.push_back(hashCallStack(CS4));
 
-  llvm::memprof::FrameIdConverter<decltype(FrameIdMap)> FrameIdConv(FrameIdMap);
-  llvm::memprof::CallStackIdConverter<decltype(CallStackIdMap)> CSIdConv(
-      CallStackIdMap, FrameIdConv);
+  FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
+      MemProfData.Frames);
+  CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
+      MemProfData.CallStacks, FrameIdConv);
 
   MemProfRecord Record = IndexedRecord.toMemProfRecord(CSIdConv);
 
@@ -539,27 +519,12 @@ TEST(MemProf, IndexedMemProfRecordToMemProfRecord) {
 
   // Verify the contents of Record.
   ASSERT_THAT(Record.AllocSites, SizeIs(2));
-  ASSERT_THAT(Record.AllocSites[0].CallStack, SizeIs(2));
-  EXPECT_EQ(Record.AllocSites[0].CallStack[0].hash(), F1.hash());
-  EXPECT_EQ(Record.AllocSites[0].CallStack[1].hash(), F2.hash());
-  ASSERT_THAT(Record.AllocSites[1].CallStack, SizeIs(2));
-  EXPECT_EQ(Record.AllocSites[1].CallStack[0].hash(), F1.hash());
-  EXPECT_EQ(Record.AllocSites[1].CallStack[1].hash(), F3.hash());
-  ASSERT_THAT(Record.CallSites, SizeIs(2));
-  ASSERT_THAT(Record.CallSites[0], SizeIs(2));
-  EXPECT_EQ(Record.CallSites[0][0].hash(), F2.hash());
-  EXPECT_EQ(Record.CallSites[0][1].hash(), F3.hash());
-  ASSERT_THAT(Record.CallSites[1], SizeIs(2));
-  EXPECT_EQ(Record.CallSites[1][0].hash(), F2.hash());
-  EXPECT_EQ(Record.CallSites[1][1].hash(), F4.hash());
+  EXPECT_THAT(Record.AllocSites[0].CallStack, ElementsAre(F1, F2));
+  EXPECT_THAT(Record.AllocSites[1].CallStack, ElementsAre(F1, F3));
+  EXPECT_THAT(Record.CallSites,
+              ElementsAre(ElementsAre(F2, F3), ElementsAre(F2, F4)));
 }
 
-using FrameIdMapTy =
-    llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
-using CallStackIdMapTy =
-    llvm::DenseMap<::llvm::memprof::CallStackId,
-                   ::llvm::SmallVector<::llvm::memprof::FrameId>>;
-
 // Populate those fields returned by getHotColdSchema.
 MemInfoBlock makePartialMIB() {
   MemInfoBlock MIB;
@@ -573,19 +538,17 @@ MemInfoBlock makePartialMIB() {
 TEST(MemProf, MissingCallStackId) {
   // Use a non-existent CallStackId to trigger a mapping error in
   // toMemProfRecord.
-  llvm::memprof::IndexedAllocationInfo AI(0xdeadbeefU, makePartialMIB(),
-                                          llvm::memprof::getHotColdSchema());
+  IndexedAllocationInfo AI(0xdeadbeefU, makePartialMIB(), getHotColdSchema());
 
   IndexedMemProfRecord IndexedMR;
   IndexedMR.AllocSites.push_back(AI);
 
   // Create empty maps.
-  const FrameIdMapTy IdToFrameMap;
-  const CallStackIdMapTy CSIdToCallStackMap;
-  llvm::memprof::FrameIdConverter<decltype(IdToFrameMap)> FrameIdConv(
-      IdToFrameMap);
-  llvm::memprof::CallStackIdConverter<decltype(CSIdToCallStackMap)> CSIdConv(
-      CSIdToCallStackMap, FrameIdConv);
+  IndexedMemProfData MemProfData;
+  FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
+      MemProfData.Frames);
+  CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
+      MemProfData.CallStacks, FrameIdConv);
 
   // We are only interested in errors, not the return value.
   (void)IndexedMR.toMemProfRecord(CSIdConv);
@@ -596,21 +559,19 @@ TEST(MemProf, MissingCallStackId) {
 }
 
 TEST(MemProf, MissingFrameId) {
-  llvm::memprof::IndexedAllocationInfo AI(0x222, makePartialMIB(),
-                                          llvm::memprof::getHotColdSchema());
+  IndexedAllocationInfo AI(0x222, makePartialMIB(), getHotColdSchema());
 
   IndexedMemProfRecord IndexedMR;
   IndexedMR.AllocSites.push_back(AI);
 
-  // An empty map to trigger a mapping error.
-  const FrameIdMapTy IdToFrameMap;
-  CallStackIdMapTy CSIdToCallStackMap;
-  CSIdToCallStackMap.insert({0x222, {2, 3}});
+  // An empty Frame map to trigger a mapping error.
+  IndexedMemProfData MemProfData;
+  MemProfData.CallStacks.insert({0x222, {2, 3}});
 
-  llvm::memprof::FrameIdConverter<decltype(IdToFrameMap)> FrameIdConv(
-      IdToFrameMap);
-  llvm::memprof::CallStackIdConverter<decltype(CSIdToCallStackMap)> CSIdConv(
-      CSIdToCallStackMap, FrameIdConv);
+  FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
+      MemProfData.Frames);
+  CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
+      MemProfData.CallStacks, FrameIdConv);
 
   // We are only interested in errors, not the return value.
   (void)IndexedMR.toMemProfRecord(CSIdConv);
@@ -622,12 +583,11 @@ TEST(MemProf, MissingFrameId) {
 
 // Verify CallStackRadixTreeBuilder can handle empty inputs.
 TEST(MemProf, RadixTreeBuilderEmpty) {
-  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes;
+  llvm::DenseMap<FrameId, LinearFrameId> MemProfFrameIndexes;
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
-  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
-      FrameHistogram =
-          llvm::memprof::computeFrameHistogram<FrameId>(MemProfCallStackData);
-  llvm::memprof::CallStackRadixTreeBuilder<FrameId> Builder;
+  llvm::DenseMap<FrameId, FrameStat> FrameHistogram =
+      computeFrameHistogram<FrameId>(MemProfCallStackData);
+  CallStackRadixTreeBuilder<FrameId> Builder;
   Builder.build(std::move(MemProfCallStackData), &MemProfFrameIndexes,
                 FrameHistogram);
   ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty());
@@ -637,109 +597,242 @@ TEST(MemProf, RadixTreeBuilderEmpty) {
 
 // Verify CallStackRadixTreeBuilder can handle one trivial call stack.
 TEST(MemProf, RadixTreeBuilderOne) {
-  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+  llvm::DenseMap<FrameId, LinearFrameId> MemProfFrameIndexes = {
       {11, 1}, {12, 2}, {13, 3}};
-  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {13, 12, 11};
+  llvm::SmallVector<FrameId> CS1 = {13, 12, 11};
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
-  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
-      FrameHistogram =
-          llvm::memprof::computeFrameHistogram<FrameId>(MemProfCallStackData);
-  llvm::memprof::CallStackRadixTreeBuilder<FrameId> Builder;
+  MemProfCallStackData.insert({hashCallStack(CS1), CS1});
+  llvm::DenseMap<FrameId, FrameStat> FrameHistogram =
+      computeFrameHistogram<FrameId>(MemProfCallStackData);
+  CallStackRadixTreeBuilder<FrameId> Builder;
   Builder.build(std::move(MemProfCallStackData), &MemProfFrameIndexes,
                 FrameHistogram);
-  EXPECT_THAT(Builder.getRadixArray(), testing::ElementsAreArray({
-                                           3U, // Size of CS1,
-                                           3U, // MemProfFrameIndexes[13]
-                                           2U, // MemProfFrameIndexes[12]
-                                           1U  // MemProfFrameIndexes[11]
-                                       }));
+  EXPECT_THAT(Builder.getRadixArray(),
+              ElementsAre(3U, // Size of CS1,
+                          3U, // MemProfFrameIndexes[13]
+                          2U, // MemProfFrameIndexes[12]
+                          1U  // MemProfFrameIndexes[11]
+                          ));
   const auto Mappings = Builder.takeCallStackPos();
-  ASSERT_THAT(Mappings, SizeIs(1));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS1), 0U)));
+  EXPECT_THAT(Mappings, UnorderedElementsAre(Pair(hashCallStack(CS1), 0U)));
 }
 
 // Verify CallStackRadixTreeBuilder can form a link between two call stacks.
 TEST(MemProf, RadixTreeBuilderTwo) {
-  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+  llvm::DenseMap<FrameId, LinearFrameId> MemProfFrameIndexes = {
       {11, 1}, {12, 2}, {13, 3}};
-  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {12, 11};
-  llvm::SmallVector<llvm::memprof::FrameId> CS2 = {13, 12, 11};
+  llvm::SmallVector<FrameId> CS1 = {12, 11};
+  llvm::SmallVector<FrameId> CS2 = {13, 12, 11};
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
-  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
-      FrameHistogram =
-          llvm::memprof::computeFrameHistogram<FrameId>(MemProfCallStackData);
-  llvm::memprof::CallStackRadixTreeBuilder<FrameId> Builder;
+  MemProfCallStackData.insert({hashCallStack(CS1), CS1});
+  MemProfCallStackData.insert({hashCallStack(CS2), CS2});
+  llvm::DenseMap<FrameId, FrameStat> FrameHistogram =
+      computeFrameHistogram<FrameId>(MemProfCallStackData);
+  CallStackRadixTreeBuilder<FrameId> Builder;
   Builder.build(std::move(MemProfCallStackData), &MemProfFrameIndexes,
                 FrameHistogram);
   EXPECT_THAT(Builder.getRadixArray(),
-              testing::ElementsAreArray({
-                  2U,                        // Size of CS1
-                  static_cast<uint32_t>(-3), // Jump 3 steps
-                  3U,                        // Size of CS2
-                  3U,                        // MemProfFrameIndexes[13]
-                  2U,                        // MemProfFrameIndexes[12]
-                  1U                         // MemProfFrameIndexes[11]
-              }));
+              ElementsAre(2U,                        // Size of CS1
+                          static_cast<uint32_t>(-3), // Jump 3 steps
+                          3U,                        // Size of CS2
+                          3U,                        // MemProfFrameIndexes[13]
+                          2U,                        // MemProfFrameIndexes[12]
+                          1U                         // MemProfFrameIndexes[11]
+                          ));
   const auto Mappings = Builder.takeCallStackPos();
-  ASSERT_THAT(Mappings, SizeIs(2));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS1), 0U)));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS2), 2U)));
+  EXPECT_THAT(Mappings, UnorderedElementsAre(Pair(hashCallStack(CS1), 0U),
+                                             Pair(hashCallStack(CS2), 2U)));
 }
 
 // Verify CallStackRadixTreeBuilder can form a jump to a prefix that itself has
 // another jump to another prefix.
 TEST(MemProf, RadixTreeBuilderSuccessiveJumps) {
-  llvm::DenseMap<FrameId, llvm::memprof::LinearFrameId> MemProfFrameIndexes = {
+  llvm::DenseMap<FrameId, LinearFrameId> MemProfFrameIndexes = {
       {11, 1}, {12, 2}, {13, 3}, {14, 4}, {15, 5}, {16, 6}, {17, 7}, {18, 8},
   };
-  llvm::SmallVector<llvm::memprof::FrameId> CS1 = {14, 13, 12, 11};
-  llvm::SmallVector<llvm::memprof::FrameId> CS2 = {15, 13, 12, 11};
-  llvm::SmallVector<llvm::memprof::FrameId> CS3 = {17, 16, 12, 11};
-  llvm::SmallVector<llvm::memprof::FrameId> CS4 = {18, 16, 12, 11};
+  llvm::SmallVector<FrameId> CS1 = {14, 13, 12, 11};
+  llvm::SmallVector<FrameId> CS2 = {15, 13, 12, 11};
+  llvm::SmallVector<FrameId> CS3 = {17, 16, 12, 11};
+  llvm::SmallVector<FrameId> CS4 = {18, 16, 12, 11};
   llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS3), CS3});
-  MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS4), CS4});
-  llvm::DenseMap<llvm::memprof::FrameId, llvm::memprof::FrameStat>
-      FrameHistogram =
-          llvm::memprof::computeFrameHistogram<FrameId>(MemProfCallStackData);
-  llvm::memprof::CallStackRadixTreeBuilder<FrameId> Builder;
+  MemProfCallStackData.insert({hashCallStack(CS1), CS1});
+  MemProfCallStackData.insert({hashCallStack(CS2), CS2});
+  MemProfCallStackData.insert({hashCallStack(CS3), CS3});
+  MemProfCallStackData.insert({hashCallStack(CS4), CS4});
+  llvm::DenseMap<FrameId, FrameStat> FrameHistogram =
+      computeFrameHistogram<FrameId>(MemProfCallStackData);
+  CallStackRadixTreeBuilder<FrameId> Builder;
   Builder.build(std::move(MemProfCallStackData), &MemProfFrameIndexes,
                 FrameHistogram);
   EXPECT_THAT(Builder.getRadixArray(),
-              testing::ElementsAreArray({
-                  4U,                        // Size of CS1
-                  4U,                        // MemProfFrameIndexes[14]
-                  static_cast<uint32_t>(-3), // Jump 3 steps
-                  4U,                        // Size of CS2
-                  5U,                        // MemProfFrameIndexes[15]
-                  3U,                        // MemProfFrameIndexes[13]
-                  static_cast<uint32_t>(-7), // Jump 7 steps
-                  4U,                        // Size of CS3
-                  7U,                        // MemProfFrameIndexes[17]
-                  static_cast<uint32_t>(-3), // Jump 3 steps
-                  4U,                        // Size of CS4
-                  8U,                        // MemProfFrameIndexes[18]
-                  6U,                        // MemProfFrameIndexes[16]
-                  2U,                        // MemProfFrameIndexes[12]
-                  1U                         // MemProfFrameIndexes[11]
-              }));
+              ElementsAre(4U,                        // Size of CS1
+                          4U,                        // MemProfFrameIndexes[14]
+                          static_cast<uint32_t>(-3), // Jump 3 steps
+                          4U,                        // Size of CS2
+                          5U,                        // MemProfFrameIndexes[15]
+                          3U,                        // MemProfFrameIndexes[13]
+                          static_cast<uint32_t>(-7), // Jump 7 steps
+                          4U,                        // Size of CS3
+                          7U,                        // MemProfFrameIndexes[17]
+                          static_cast<uint32_t>(-3), // Jump 3 steps
+                          4U,                        // Size of CS4
+                          8U,                        // MemProfFrameIndexes[18]
+                          6U,                        // MemProfFrameIndexes[16]
+                          2U,                        // MemProfFrameIndexes[12]
+                          1U                         // MemProfFrameIndexes[11]
+                          ));
   const auto Mappings = Builder.takeCallStackPos();
-  ASSERT_THAT(Mappings, SizeIs(4));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS1), 0U)));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS2), 3U)));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS3), 7U)));
-  EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
-                            llvm::memprof::hashCallStack(CS4), 10U)));
+  EXPECT_THAT(Mappings, UnorderedElementsAre(Pair(hashCallStack(CS1), 0U),
+                                             Pair(hashCallStack(CS2), 3U),
+                                             Pair(hashCallStack(CS3), 7U),
+                                             Pair(hashCallStack(CS4), 10U)));
+}
+
+// Verify that we can parse YAML and retrieve IndexedMemProfData as expected.
+TEST(MemProf, YAMLParser) {
+  StringRef YAMLData = R"YAML(
+---
+HeapProfileRecords:
+- GUID: 0xdeadbeef12345678
+  AllocSites:
+  - Callstack:
+    - {Function: 0x100, LineOffset: 11, Column: 10, IsInlineFrame: true}
+    - {Function: 0x200, LineOffset: 22, Column: 20, IsInlineFrame: false}
+    MemInfoBlock:
+      AllocCount: 777
+      TotalSize: 888
+  - Callstack:
+    - {Function: 0x300, LineOffset: 33, Column: 30, IsInlineFrame: false}
+    - {Function: 0x400, LineOffset: 44, Column: 40, IsInlineFrame: true}
+    MemInfoBlock:
+      AllocCount: 666
+      TotalSize: 555
+  CallSites:
+  - - {Function: 0x500, LineOffset: 55, Column: 50, IsInlineFrame: true}
+    - {Function: 0x600, LineOffset: 66, Column: 60, IsInlineFrame: false}
+  - - {Function: 0x700, LineOffset: 77, Column: 70, IsInlineFrame: true}
+    - {Function: 0x800, LineOffset: 88, Column: 80, IsInlineFrame: false}
+)YAML";
+
+  YAMLMemProfReader YAMLReader;
+  YAMLReader.parse(YAMLData);
+  IndexedMemProfData MemProfData = YAMLReader.takeMemProfData();
+
+  Frame F1(0x100, 11, 10, true);
+  Frame F2(0x200, 22, 20, false);
+  Frame F3(0x300, 33, 30, false);
+  Frame F4(0x400, 44, 40, true);
+  Frame F5(0x500, 55, 50, true);
+  Frame F6(0x600, 66, 60, false);
+  Frame F7(0x700, 77, 70, true);
+  Frame F8(0x800, 88, 80, false);
+
+  llvm::SmallVector<FrameId> CS1 = {F1.hash(), F2.hash()};
+  llvm::SmallVector<FrameId> CS2 = {F3.hash(), F4.hash()};
+  llvm::SmallVector<FrameId> CS3 = {F5.hash(), F6.hash()};
+  llvm::SmallVector<FrameId> CS4 = {F7.hash(), F8.hash()};
+
+  // Verify the entire contents of MemProfData.Frames.
+  EXPECT_THAT(MemProfData.Frames,
+              UnorderedElementsAre(Pair(F1.hash(), F1), Pair(F2.hash(), F2),
+                                   Pair(F3.hash(), F3), Pair(F4.hash(), F4),
+                                   Pair(F5.hash(), F5), Pair(F6.hash(), F6),
+                                   Pair(F7.hash(), F7), Pair(F8.hash(), F8)));
+
+  // Verify the entire contents of MemProfData.Frames.
+  EXPECT_THAT(MemProfData.CallStacks,
+              UnorderedElementsAre(Pair(hashCallStack(CS1), CS1),
+                                   Pair(hashCallStack(CS2), CS2),
+                                   Pair(hashCallStack(CS3), CS3),
+                                   Pair(hashCallStack(CS4), CS4)));
+
+  // Verify the entire contents of MemProfData.Records.
+  ASSERT_THAT(MemProfData.Records, SizeIs(1));
+  const auto &[GUID, Record] = *MemProfData.Records.begin();
+  EXPECT_EQ(GUID, 0xdeadbeef12345678ULL);
+  ASSERT_THAT(Record.AllocSites, SizeIs(2));
+  EXPECT_EQ(Record.AllocSites[0].CSId, hashCallStack(CS1));
+  EXPECT_EQ(Record.AllocSites[0].Info.getAllocCount(), 777U);
+  EXPECT_EQ(Record.AllocSites[0].Info.getTotalSize(), 888U);
+  EXPECT_EQ(Record.AllocSites[1].CSId, hashCallStack(CS2));
+  EXPECT_EQ(Record.AllocSites[1].Info.getAllocCount(), 666U);
+  EXPECT_EQ(Record.AllocSites[1].Info.getTotalSize(), 555U);
+  EXPECT_THAT(Record.CallSiteIds,
+              ElementsAre(hashCallStack(CS3), hashCallStack(CS4)));
+}
+
+// Verify that the YAML parser accepts a GUID expressed as a function name.
+TEST(MemProf, YAMLParserGUID) {
+  StringRef YAMLData = R"YAML(
+---
+HeapProfileRecords:
+- GUID: _Z3fooi
+  AllocSites:
+  - Callstack:
+    - {Function: 0x100, LineOffset: 11, Column: 10, IsInlineFrame: true}
+    MemInfoBlock: {}
+  CallSites: []
+)YAML";
+
+  YAMLMemProfReader YAMLReader;
+  YAMLReader.parse(YAMLData);
+  IndexedMemProfData MemProfData = YAMLReader.takeMemProfData();
+
+  Frame F1(0x100, 11, 10, true);
+
+  llvm::SmallVector<FrameId> CS1 = {F1.hash()};
+
+  // Verify the entire contents of MemProfData.Frames.
+  EXPECT_THAT(MemProfData.Frames, UnorderedElementsAre(Pair(F1.hash(), F1)));
+
+  // Verify the entire contents of MemProfData.Frames.
+  EXPECT_THAT(MemProfData.CallStacks,
+              UnorderedElementsAre(Pair(hashCallStack(CS1), CS1)));
+
+  // Verify the entire contents of MemProfData.Records.
+  ASSERT_THAT(MemProfData.Records, SizeIs(1));
+  const auto &[GUID, Record] = MemProfData.Records.front();
+  EXPECT_EQ(GUID, IndexedMemProfRecord::getGUID("_Z3fooi"));
+  ASSERT_THAT(Record.AllocSites, SizeIs(1));
+  EXPECT_EQ(Record.AllocSites[0].CSId, hashCallStack(CS1));
+  EXPECT_THAT(Record.CallSiteIds, IsEmpty());
+}
+
+template <typename T> std::string serializeInYAML(T &Val) {
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  llvm::yaml::Output Yout(OS);
+  Yout << Val;
+  return Out;
+}
+
+TEST(MemProf, YAMLWriterFrame) {
+  Frame F(11, 22, 33, true);
+
+  std::string Out = serializeInYAML(F);
+  EXPECT_EQ(Out, R"YAML(---
+{ Function: 11, LineOffset: 22, Column: 33, IsInlineFrame: true }
+...
+)YAML");
+}
+
+TEST(MemProf, YAMLWriterMIB) {
+  MemInfoBlock MIB;
+  MIB.AllocCount = 111;
+  MIB.TotalSize = 222;
+  MIB.TotalLifetime = 333;
+  MIB.TotalLifetimeAccessDensity = 444;
+  PortableMemInfoBlock PMIB(MIB, getHotColdSchema());
+
+  std::string Out = serializeInYAML(PMIB);
+  EXPECT_EQ(Out, R"YAML(---
+AllocCount:      111
+TotalSize:       222
+TotalLifetime:   333
+TotalLifetimeAccessDensity: 444
+...
+)YAML");
 }
 } // namespace
+} // namespace memprof
+} // namespace llvm
diff --git a/llvm/unittests/Support/DebugTest.cpp b/llvm/unittests/Support/DebugTest.cpp
index e68e3764b4383..e8b754892a4c2 100644
--- a/llvm/unittests/Support/DebugTest.cpp
+++ b/llvm/unittests/Support/DebugTest.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Debug.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 
@@ -30,4 +32,22 @@ TEST(DebugTest, Basic) {
   DEBUG_WITH_TYPE("B", os2 << "B");
   EXPECT_EQ("A", os2.str());
 }
+
+TEST(DebugTest, CommaInDebugBlock) {
+  std::string s1, s2;
+  raw_string_ostream os1(s1), os2(s2);
+  static const char *DT[] = {"A", "B"};
+  static const char Letters[] = {'X', 'Y', 'Z'};
+
+  llvm::DebugFlag = true;
+  setCurrentDebugTypes(DT, 2);
+  DEBUG_WITH_TYPE("A", {
+    SmallMapVector<int, char, 4> map;
+    for (int i = 0; i < 3; i++)
+      map[i] = Letters[i];
+    for (int i = 2; i >= 0; i--)
+      os1 << map[i];
+  });
+  EXPECT_EQ("ZYX", os1.str());
+}
 #endif
diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp
index a7d1b338719f3..c0e9c57a77f19 100644
--- a/llvm/unittests/Support/YAMLIOTest.cpp
+++ b/llvm/unittests/Support/YAMLIOTest.cpp
@@ -1537,6 +1537,93 @@ TEST(YAMLIO, TestReadWriteMySecondsSequence) {
   }
 }
 
+//===----------------------------------------------------------------------===//
+//  Test nested sequence
+//===----------------------------------------------------------------------===//
+using NestedStringSeq1 = llvm::SmallVector<std::string, 2>;
+using NestedStringSeq2 = std::array<NestedStringSeq1, 2>;
+using NestedStringSeq3 = std::vector<NestedStringSeq2>;
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(NestedStringSeq1)
+LLVM_YAML_IS_SEQUENCE_VECTOR(NestedStringSeq2)
+
+struct MappedStringSeq3 {
+  NestedStringSeq3 Seq3;
+};
+
+template <> struct llvm::yaml::MappingTraits<MappedStringSeq3> {
+  static void mapping(IO &io, MappedStringSeq3 &seq) {
+    io.mapRequired("Seq3", seq.Seq3);
+  }
+};
+
+using NestedIntSeq1 = std::array<int, 2>;
+using NestedIntSeq2 = std::array<NestedIntSeq1, 2>;
+using NestedIntSeq3 = std::array<NestedIntSeq2, 2>;
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(NestedIntSeq1)
+LLVM_YAML_IS_SEQUENCE_VECTOR(NestedIntSeq2)
+
+template <typename Ty> std::string ParseAndEmit(llvm::StringRef YAML) {
+  Ty seq3;
+  Input yin(YAML);
+  yin >> seq3;
+  std::string out;
+  llvm::raw_string_ostream ostr(out);
+  Output yout(ostr);
+  yout << seq3;
+  return out;
+}
+
+TEST(YAMLIO, TestNestedSequence) {
+  {
+    llvm::StringRef Seq3YAML(R"YAML(---
+- - [ 1000, 1001 ]
+  - [ 1010, 1011 ]
+- - [ 1100, 1101 ]
+  - [ 1110, 1111 ]
+...
+)YAML");
+
+    std::string out = ParseAndEmit<NestedIntSeq3>(Seq3YAML);
+    EXPECT_EQ(out, Seq3YAML);
+  }
+
+  {
+    llvm::StringRef Seq3YAML(R"YAML(---
+- - - '000'
+    - '001'
+  - - '010'
+    - '011'
+- - - '100'
+    - '101'
+  - - '110'
+    - '111'
+...
+)YAML");
+
+    std::string out = ParseAndEmit<NestedStringSeq3>(Seq3YAML);
+    EXPECT_EQ(out, Seq3YAML);
+  }
+
+  {
+    llvm::StringRef Seq3YAML(R"YAML(---
+Seq3:
+  - - - '000'
+      - '001'
+    - - '010'
+      - '011'
+  - - - '100'
+      - '101'
+    - - '110'
+      - '111'
+...
+)YAML");
+
+    std::string out = ParseAndEmit<MappedStringSeq3>(Seq3YAML);
+    EXPECT_EQ(out, Seq3YAML);
+  }
+}
 
 //===----------------------------------------------------------------------===//
 //  Test dynamic typing
diff --git a/llvm/unittests/Target/AArch64/AArch64InstPrinterTest.cpp b/llvm/unittests/Target/AArch64/AArch64InstPrinterTest.cpp
index 52e04db032ca0..4dfc0bcb0dc4c 100644
--- a/llvm/unittests/Target/AArch64/AArch64InstPrinterTest.cpp
+++ b/llvm/unittests/Target/AArch64/AArch64InstPrinterTest.cpp
@@ -36,7 +36,7 @@ static std::string AArch64InstPrinterTestPrintAlignedLabel(uint64_t value) {
   MCAsmInfo MAI;
   MCInstrInfo MII;
   MCRegisterInfo MRI;
-  MCSubtargetInfo STI(Triple(""), "", "", "",
+  MCSubtargetInfo STI(Triple(""), "", "", "", {},
                       ArrayRef((SubtargetFeatureKV *)NULL, (size_t)0),
                       ArrayRef((SubtargetSubTypeKV *)NULL, (size_t)0), NULL,
                       NULL, NULL, NULL, NULL, NULL);
diff --git a/llvm/unittests/Target/AArch64/AArch64RegisterInfoTest.cpp b/llvm/unittests/Target/AArch64/AArch64RegisterInfoTest.cpp
new file mode 100644
index 0000000000000..00e221eb43467
--- /dev/null
+++ b/llvm/unittests/Target/AArch64/AArch64RegisterInfoTest.cpp
@@ -0,0 +1,152 @@
+#include "AArch64RegisterInfo.h"
+#include "AArch64InstrInfo.h"
+#include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+
+#include "gtest/gtest.h"
+
+#define GET_COMPUTE_FEATURES
+#include "AArch64GenInstrInfo.inc"
+
+using namespace llvm;
+
+namespace {
+
+std::unique_ptr<TargetMachine> createTargetMachine(const std::string &CPU) {
+  auto TT(Triple::normalize("aarch64--"));
+
+  LLVMInitializeAArch64TargetInfo();
+  LLVMInitializeAArch64Target();
+  LLVMInitializeAArch64TargetMC();
+
+  std::string Error;
+  const Target *TheTarget = TargetRegistry::lookupTarget(TT, Error);
+
+  return std::unique_ptr<TargetMachine>(
+      TheTarget->createTargetMachine(TT, CPU, "", TargetOptions(), std::nullopt,
+                                     std::nullopt, CodeGenOptLevel::Default));
+}
+
+std::unique_ptr<AArch64InstrInfo> createInstrInfo(TargetMachine *TM) {
+  AArch64Subtarget ST(TM->getTargetTriple(), std::string(TM->getTargetCPU()),
+                      std::string(TM->getTargetCPU()),
+                      std::string(TM->getTargetFeatureString()), *TM, true);
+  return std::make_unique<AArch64InstrInfo>(ST);
+}
+
+TEST(AArch64LaneBitmasks, SubRegs) {
+  std::unique_ptr<TargetMachine> TM = createTargetMachine("");
+  ASSERT_TRUE(TM);
+
+  std::unique_ptr<AArch64InstrInfo> II = createInstrInfo(TM.get());
+  ASSERT_TRUE(II);
+
+  const AArch64RegisterInfo &TRI = II->getRegisterInfo();
+
+  // Test that the lane masks for the subregisters 'bsub, hsub, ssub, etc'
+  // are composed correctly.
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::bsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::bsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::hsub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::hsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::hsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::ssub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::ssub) |
+                TRI.getSubRegIndexLaneMask(AArch64::ssub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::dsub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::dsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::zsub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::zsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::zsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::zsub0));
+
+  // Test that the lane masks for tuples are composed correctly.
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_bsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_bsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_hsub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_hsub) |
+                TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_hsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_ssub));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_ssub) |
+                TRI.getSubRegIndexLaneMask(AArch64::dsub1_then_ssub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::dsub1));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub1) |
+                TRI.getSubRegIndexLaneMask(AArch64::qsub1_then_dsub_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::qsub1));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::sub_32) |
+                TRI.getSubRegIndexLaneMask(AArch64::sub_32_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::sube64));
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::subo64_then_sub_32) |
+                TRI.getSubRegIndexLaneMask(AArch64::subo64_then_sub_32_hi),
+            TRI.getSubRegIndexLaneMask(AArch64::subo64));
+
+  // Test that there is no overlap between different (sub)registers
+  // in a tuple.
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::dsub0) &
+                TRI.getSubRegIndexLaneMask(AArch64::dsub1) &
+                TRI.getSubRegIndexLaneMask(AArch64::dsub2) &
+                TRI.getSubRegIndexLaneMask(AArch64::dsub3),
+            LaneBitmask::getNone());
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::qsub0) &
+                TRI.getSubRegIndexLaneMask(AArch64::qsub1) &
+                TRI.getSubRegIndexLaneMask(AArch64::qsub2) &
+                TRI.getSubRegIndexLaneMask(AArch64::qsub3),
+            LaneBitmask::getNone());
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::zsub0) &
+                TRI.getSubRegIndexLaneMask(AArch64::zsub1) &
+                TRI.getSubRegIndexLaneMask(AArch64::zsub2) &
+                TRI.getSubRegIndexLaneMask(AArch64::zsub3),
+            LaneBitmask::getNone());
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::sube32) &
+                TRI.getSubRegIndexLaneMask(AArch64::subo32),
+            LaneBitmask::getNone());
+
+  EXPECT_EQ(TRI.getSubRegIndexLaneMask(AArch64::sube64) &
+                TRI.getSubRegIndexLaneMask(AArch64::subo64),
+            LaneBitmask::getNone());
+
+  // Test that getting a subregister results in the expected subregister.
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::bsub), AArch64::B0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::hsub), AArch64::H0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::ssub), AArch64::S0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::dsub), AArch64::D0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::zsub), AArch64::Q0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::zsub0), AArch64::Z0);
+
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::dsub1_then_bsub),
+            AArch64::B8);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::dsub1_then_hsub),
+            AArch64::H8);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::dsub1_then_ssub),
+            AArch64::S8);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::dsub1), AArch64::D8);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::qsub1), AArch64::Q8);
+  EXPECT_EQ(TRI.getSubReg(AArch64::Z0_Z8, AArch64::zsub1), AArch64::Z8);
+
+  EXPECT_EQ(TRI.getSubReg(AArch64::X0_X1, AArch64::sube64), AArch64::X0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::X0_X1, AArch64::subo64), AArch64::X1);
+  EXPECT_EQ(TRI.getSubReg(AArch64::X0_X1, AArch64::sub_32), AArch64::W0);
+  EXPECT_EQ(TRI.getSubReg(AArch64::X0_X1, AArch64::subo64_then_sub_32),
+            AArch64::W1);
+}
+
+} // namespace
diff --git a/llvm/unittests/Target/AArch64/CMakeLists.txt b/llvm/unittests/Target/AArch64/CMakeLists.txt
index f53668373efee..449888838acdc 100644
--- a/llvm/unittests/Target/AArch64/CMakeLists.txt
+++ b/llvm/unittests/Target/AArch64/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_target_unittest(AArch64Tests
   InstSizes.cpp
   MatrixRegisterAliasing.cpp
   SMEAttributesTest.cpp
+  AArch64RegisterInfoTest.cpp
   AArch64SVESchedPseudoTest.cpp
   Immediates.cpp
   )
diff --git a/llvm/unittests/Target/SPIRV/CMakeLists.txt b/llvm/unittests/Target/SPIRV/CMakeLists.txt
index e9fe4883e5b02..af7d9395605d0 100644
--- a/llvm/unittests/Target/SPIRV/CMakeLists.txt
+++ b/llvm/unittests/Target/SPIRV/CMakeLists.txt
@@ -15,6 +15,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_target_unittest(SPIRVTests
   SPIRVConvergenceRegionAnalysisTests.cpp
+  SPIRVSortBlocksTests.cpp
+  SPIRVPartialOrderingVisitorTests.cpp
   SPIRVAPITest.cpp
-  )
-
+)
diff --git a/llvm/unittests/Target/SPIRV/SPIRVPartialOrderingVisitorTests.cpp b/llvm/unittests/Target/SPIRV/SPIRVPartialOrderingVisitorTests.cpp
new file mode 100644
index 0000000000000..95d6c3f5a2940
--- /dev/null
+++ b/llvm/unittests/Target/SPIRV/SPIRVPartialOrderingVisitorTests.cpp
@@ -0,0 +1,342 @@
+//===- SPIRVPartialOrderingVisitorTests.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRVUtils.h"
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/TypedPointerType.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <queue>
+
+using namespace llvm;
+using namespace llvm::SPIRV;
+
+class SPIRVPartialOrderingVisitorTest : public testing::Test {
+protected:
+  void TearDown() override { M.reset(); }
+
+  void run(StringRef Assembly) {
+    assert(M == nullptr &&
+           "Calling runAnalysis multiple times is unsafe. See getAnalysis().");
+
+    SMDiagnostic Error;
+    M = parseAssemblyString(Assembly, Error, Context);
+    assert(M && "Bad assembly. Bad test?");
+
+    llvm::Function *F = M->getFunction("main");
+    Visitor = std::make_unique<PartialOrderingVisitor>(*F);
+  }
+
+  void
+  checkBasicBlockRank(std::vector<std::pair<const char *, size_t>> &&Expected) {
+    llvm::Function *F = M->getFunction("main");
+    auto It = Expected.begin();
+    Visitor->partialOrderVisit(*F->begin(), [&](BasicBlock *BB) {
+      const auto &[Name, Rank] = *It;
+      EXPECT_TRUE(It != Expected.end())
+          << "Unexpected block \"" << BB->getName() << " visited.";
+      EXPECT_TRUE(BB->getName() == Name)
+          << "Error: expected block \"" << Name << "\" got \"" << BB->getName()
+          << "\"";
+      EXPECT_EQ(Rank, Visitor->GetNodeRank(BB))
+          << "Bad rank for BB \"" << BB->getName() << "\"";
+      It++;
+      return true;
+    });
+    ASSERT_TRUE(It == Expected.end())
+        << "Expected block \"" << It->first
+        << "\" but reached the end of the function instead.";
+  }
+
+protected:
+  LLVMContext Context;
+  std::unique_ptr<Module> M;
+  std::unique_ptr<PartialOrderingVisitor> Visitor;
+};
+
+TEST_F(SPIRVPartialOrderingVisitorTest, EmptyFunction) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+      ret void
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"", 0}});
+}
+
+TEST_F(SPIRVPartialOrderingVisitorTest, BasicBlockSwap) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      br label %middle
+    exit:
+      ret void
+    middle:
+      br label %exit
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0}, {"middle", 1}, {"exit", 2}});
+}
+
+// Skip condition:
+//         +-> A -+
+//  entry -+      +-> C
+//         +------+
+TEST_F(SPIRVPartialOrderingVisitorTest, SkipCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %c, label %a
+    c:
+      ret void
+    a:
+      br label %c
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0}, {"a", 1}, {"c", 2}});
+}
+
+// Simple loop:
+// entry -> header <-----------------+
+//           | `-> body -> continue -+
+//           `-> end
+TEST_F(SPIRVPartialOrderingVisitorTest, LoopOrdering) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %header
+    end:
+      ret void
+    body:
+      br label %continue
+    continue:
+      br label %header
+    header:
+      br i1 %1, label %body, label %end
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank(
+      {{"entry", 0}, {"header", 1}, {"body", 2}, {"continue", 3}, {"end", 4}});
+}
+
+// Diamond condition:
+//         +-> A -+
+//  entry -+      +-> C
+//         +-> B -+
+//
+// A and B order can be flipped with no effect, but it must be remain
+// deterministic/stable.
+TEST_F(SPIRVPartialOrderingVisitorTest, DiamondCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %a, label %b
+    c:
+      ret void
+    b:
+      br label %c
+    a:
+      br label %c
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0}, {"a", 1}, {"b", 1}, {"c", 2}});
+}
+
+// Crossing conditions:
+//             +------+  +-> C -+
+//         +-> A -+   |  |      |
+//  entry -+      +--_|_-+      +-> E
+//         +-> B -+   |         |
+//             +------+----> D -+
+//
+// A & B have the same rank.
+// C & D have the same rank, but are after A & B.
+// E if the last block.
+TEST_F(SPIRVPartialOrderingVisitorTest, CrossingCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %a, label %b
+    e:
+      ret void
+    c:
+      br label %e
+    b:
+      br i1 %1, label %d, label %c
+    d:
+      br label %e
+    a:
+      br i1 %1, label %c, label %d
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank(
+      {{"entry", 0}, {"a", 1}, {"b", 1}, {"c", 2}, {"d", 2}, {"e", 3}});
+}
+
+TEST_F(SPIRVPartialOrderingVisitorTest, LoopDiamond) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %header
+    header:
+      br i1 %1, label %body, label %end
+    body:
+      br i1 %1, label %inside_a, label %break
+    inside_a:
+      br label %inside_b
+    inside_b:
+      br i1 %1, label %inside_c, label %inside_d
+    inside_c:
+      br label %continue
+    inside_d:
+      br label %continue
+    break:
+      br label %end
+    continue:
+      br label %header
+    end:
+      ret void
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0},
+                       {"header", 1},
+                       {"body", 2},
+                       {"inside_a", 3},
+                       {"inside_b", 4},
+                       {"inside_c", 5},
+                       {"inside_d", 5},
+                       {"continue", 6},
+                       {"break", 7},
+                       {"end", 8}});
+}
+
+TEST_F(SPIRVPartialOrderingVisitorTest, LoopNested) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %a
+    a:
+      br i1 %1, label %h, label %b
+    b:
+      br label %c
+    c:
+      br i1 %1, label %d, label %e
+    d:
+      br label %g
+    e:
+      br label %f
+    f:
+      br label %c
+    g:
+      br label %a
+    h:
+      ret void
+    }
+  )";
+
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0},
+                       {"a", 1},
+                       {"b", 2},
+                       {"c", 3},
+                       {"e", 4},
+                       {"f", 5},
+                       {"d", 6},
+                       {"g", 7},
+                       {"h", 8}});
+}
+
+TEST_F(SPIRVPartialOrderingVisitorTest, IfNested) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      br i1 true, label %a, label %d
+    a:
+      br i1 true, label %b, label %c
+    b:
+      br label %c
+    c:
+      br label %j
+    d:
+      br i1 true, label %e, label %f
+    e:
+      br label %i
+    f:
+      br i1 true, label %g, label %h
+    g:
+      br label %h
+    h:
+      br label %i
+    i:
+      br label %j
+    j:
+      ret void
+    }
+  )";
+  run(Assembly);
+  checkBasicBlockRank({{"entry", 0},
+                       {"a", 1},
+                       {"d", 1},
+                       {"b", 2},
+                       {"e", 2},
+                       {"f", 2},
+                       {"c", 3},
+                       {"g", 3},
+                       {"h", 4},
+                       {"i", 5},
+                       {"j", 6}});
+}
+
+TEST_F(SPIRVPartialOrderingVisitorTest, CheckDeathIrreducible) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %a
+    b:
+      br i1 %1, label %a, label %c
+    c:
+      br label %b
+    a:
+      br i1 %1, label %b, label %c
+    }
+  )";
+
+  ASSERT_DEATH(
+      { run(Assembly); },
+      "No valid candidate in the queue. Is the graph reducible?");
+}
diff --git a/llvm/unittests/Target/SPIRV/SPIRVSortBlocksTests.cpp b/llvm/unittests/Target/SPIRV/SPIRVSortBlocksTests.cpp
new file mode 100644
index 0000000000000..7f0d9fd724712
--- /dev/null
+++ b/llvm/unittests/Target/SPIRV/SPIRVSortBlocksTests.cpp
@@ -0,0 +1,389 @@
+//===- SPIRVSortBlocksTests.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRVUtils.h"
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/TypedPointerType.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <queue>
+
+using namespace llvm;
+using namespace llvm::SPIRV;
+
+class SPIRVSortBlocksTest : public testing::Test {
+protected:
+  void TearDown() override { M.reset(); }
+
+  bool run(StringRef Assembly) {
+    assert(M == nullptr &&
+           "Calling runAnalysis multiple times is unsafe. See getAnalysis().");
+
+    SMDiagnostic Error;
+    M = parseAssemblyString(Assembly, Error, Context);
+    assert(M && "Bad assembly. Bad test?");
+    llvm::Function *F = M->getFunction("main");
+    return sortBlocks(*F);
+  }
+
+  void checkBasicBlockOrder(std::vector<const char *> &&Expected) {
+    llvm::Function *F = M->getFunction("main");
+    auto It = F->begin();
+    for (const char *Name : Expected) {
+      ASSERT_TRUE(It != F->end())
+          << "Expected block \"" << Name
+          << "\" but reached the end of the function instead.";
+      ASSERT_TRUE(It->getName() == Name)
+          << "Error: expected block \"" << Name << "\" got \"" << It->getName()
+          << "\"";
+      It++;
+    }
+    ASSERT_TRUE(It == F->end())
+        << "No more blocks were expected, but function has more.";
+  }
+
+protected:
+  LLVMContext Context;
+  std::unique_ptr<Module> M;
+};
+
+TEST_F(SPIRVSortBlocksTest, DefaultRegion) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+      ret void
+    }
+  )";
+
+  // No sorting is required.
+  EXPECT_FALSE(run(Assembly));
+}
+
+TEST_F(SPIRVSortBlocksTest, BasicBlockSwap) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      br label %middle
+    exit:
+      ret void
+    middle:
+      br label %exit
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "middle", "exit"});
+}
+
+// Skip condition:
+//         +-> A -+
+//  entry -+      +-> C
+//         +------+
+TEST_F(SPIRVSortBlocksTest, SkipCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %c, label %a
+    c:
+      ret void
+    a:
+      br label %c
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "c"});
+}
+
+// Simple loop:
+// entry -> header <-----------------+
+//           | `-> body -> continue -+
+//           `-> end
+TEST_F(SPIRVSortBlocksTest, LoopOrdering) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %header
+    end:
+      ret void
+    body:
+      br label %continue
+    continue:
+      br label %header
+    header:
+      br i1 %1, label %body, label %end
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "header", "end", "body", "continue"});
+}
+
+// Diamond condition:
+//         +-> A -+
+//  entry -+      +-> C
+//         +-> B -+
+//
+// A and B order can be flipped with no effect, but it must be remain
+// deterministic/stable.
+TEST_F(SPIRVSortBlocksTest, DiamondCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %b, label %a
+    c:
+      ret void
+    b:
+      br label %c
+    a:
+      br label %c
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "b", "c"});
+}
+
+// Crossing conditions:
+//             +------+  +-> C -+
+//         +-> A -+   |  |      |
+//  entry -+      +--_|_-+      +-> E
+//         +-> B -+   |         |
+//             +------+----> D -+
+//
+// A & B have the same rank.
+// C & D have the same rank, but are after A & B.
+// E if the last block.
+TEST_F(SPIRVSortBlocksTest, CrossingCondition) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br i1 %1, label %b, label %a
+    e:
+      ret void
+    c:
+      br label %e
+    b:
+      br i1 %1, label %d, label %c
+    d:
+      br label %e
+    a:
+      br i1 %1, label %d, label %c
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "b", "c", "d", "e"});
+}
+
+// Irreducible CFG
+// digraph {
+//    entry -> A;
+//
+//    A -> B;
+//    A -> C;
+//
+//    B -> A;
+//    B -> C;
+//
+//    C -> B;
+// }
+//
+// Order starts with Entry and A. Order of B and C can change, but must remain
+// stable.
+// In such case, rank will be defined by the arbitrary traversal order. What's
+// important is to have a stable value.
+TEST_F(SPIRVSortBlocksTest, IrreducibleOrdering) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %a
+
+    b:
+      br i1 %1, label %a, label %c
+
+    c:
+      br label %b
+
+    a:
+      br i1 %1, label %b, label %c
+
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "b", "c"});
+}
+
+TEST_F(SPIRVSortBlocksTest, IrreducibleOrderingBeforeReduction) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %a
+
+    c:
+      br i1 %1, label %e, label %d
+
+    e:
+      ret void
+
+    b:
+      br i1 %1, label %c, label %d
+
+    a:
+      br label %b
+
+    d:
+      br i1 %1, label %b, label %c
+
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "b", "c", "d", "e"});
+}
+
+TEST_F(SPIRVSortBlocksTest, LoopDiamond) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %header
+    header:
+      br i1 %1, label %body, label %end
+    body:
+      br i1 %1, label %break, label %inside_a
+    inside_a:
+      br label %inside_b
+    inside_b:
+      br i1 %1, label %inside_d, label %inside_c
+    inside_c:
+      br label %continue
+    inside_d:
+      br label %continue
+    break:
+      br label %end
+    continue:
+      br label %header
+    end:
+      ret void
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "header", "body", "inside_a", "inside_b",
+                        "inside_c", "inside_d", "continue", "break", "end"});
+}
+
+TEST_F(SPIRVSortBlocksTest, LoopNested) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      %1 = icmp ne i32 0, 0
+      br label %a
+    a:
+      br i1 %1, label %h, label %b
+    b:
+      br label %c
+    c:
+      br i1 %1, label %d, label %e
+    d:
+      br label %g
+    e:
+      br label %f
+    f:
+      br label %c
+    g:
+      br label %a
+    h:
+      ret void
+    }
+  )";
+
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder({"entry", "a", "b", "c", "e", "f", "d", "g", "h"});
+}
+
+TEST_F(SPIRVSortBlocksTest, IfNested) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      br i1 true, label %d, label %a
+    i:
+      br label %j
+    j:
+      ret void
+    a:
+      br i1 true, label %b, label %c
+    d:
+      br i1 true, label %f, label %e
+    e:
+      br label %i
+    b:
+      br label %c
+    f:
+      br i1 true, label %h, label %g
+    g:
+      br label %h
+    c:
+      br label %j
+    h:
+      br label %i
+    }
+  )";
+  EXPECT_TRUE(run(Assembly));
+  checkBasicBlockOrder(
+      {"entry", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"});
+}
+
+// Same as above, but this time blocks are already sorted, so no need to reorder
+// them.
+TEST_F(SPIRVSortBlocksTest, IfNestedSorted) {
+  StringRef Assembly = R"(
+    define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+    entry:
+      br i1 true, label %d, label %z
+    z:
+      br i1 true, label %b, label %c
+    b:
+      br label %c
+    c:
+      br label %j
+    d:
+      br i1 true, label %f, label %e
+    e:
+      br label %i
+    f:
+      br i1 true, label %h, label %g
+    g:
+      br label %h
+    h:
+      br label %i
+    i:
+      br label %j
+    j:
+      ret void
+    }
+  )";
+  EXPECT_FALSE(run(Assembly));
+}
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index 5e2edcef09bf8..67c727a6c1c5d 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -140,6 +140,9 @@ TEST(getLinuxHostCPUName, AArch64) {
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
                                               "CPU part        : 0x001"),
             "oryon-1");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x46\n"
+                                              "CPU part        : 0x003"),
+            "fujitsu-monaka");
 
   // MSM8992/4 weirdness
   StringRef MSM8992ProcCpuInfo = R"(
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 0694d0987c527..f9ff45f116603 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -652,6 +652,16 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
               "'xwchc' and 'zcb' extensions are incompatible");
   }
+
+  for (StringRef Input : {"rv64i_xqcisls0p2"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'xqcisls' is only supported for 'rv32'");
+  }
+
+  for (StringRef Input : {"rv64i_xqcia0p2"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'xqcia' is only supported for 'rv32'");
+  }
 }
 
 TEST(ParseArchString, MissingDepency) {
@@ -1104,6 +1114,9 @@ Experimental extensions
     smctr                1.0
     ssctr                1.0
     svukte               0.3
+    xqcia                0.2
+    xqcicsr              0.2
+    xqcisls              0.2
 
 Supported Profiles
     rva20s64
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index d281c2cbd55d3..1f69190e4bec5 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -404,6 +404,14 @@ INSTANTIATE_TEST_SUITE_P(
                 ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC |
                 ARM::AEK_RAS | ARM::AEK_FP16 | ARM::AEK_DOTPROD,
             "8.2-A"),
+        ARMCPUTestParams<uint64_t>("cortex-a510", "armv9-a", "neon-fp-armv8",
+                                   ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT |
+                                       ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
+                                       ARM::AEK_DSP | ARM::AEK_CRC |
+                                       ARM::AEK_RAS | ARM::AEK_DOTPROD |
+                                       ARM::AEK_FP16FML | ARM::AEK_BF16 |
+                                       ARM::AEK_I8MM | ARM::AEK_SB,
+                                   "9-A"),
         ARMCPUTestParams<uint64_t>("cortex-a710", "armv9-a", "neon-fp-armv8",
                                    ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT |
                                        ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
@@ -521,7 +529,7 @@ INSTANTIATE_TEST_SUITE_P(
                                    "7-S")),
     ARMCPUTestParams<uint64_t>::PrintToStringParamName);
 
-static constexpr unsigned NumARMCPUArchs = 93;
+static constexpr unsigned NumARMCPUArchs = 94;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector<StringRef, NumARMCPUArchs> List;
@@ -1150,13 +1158,14 @@ INSTANTIATE_TEST_SUITE_P(
                       AArch64CPUTestParams("thunderxt88", "armv8-a"),
                       AArch64CPUTestParams("tsv110", "armv8.2-a"),
                       AArch64CPUTestParams("a64fx", "armv8.2-a"),
+                      AArch64CPUTestParams("fujitsu-monaka", "armv9.3-a"),
                       AArch64CPUTestParams("carmel", "armv8.2-a"),
                       AArch64CPUTestParams("saphira", "armv8.4-a"),
                       AArch64CPUTestParams("oryon-1", "armv8.6-a")),
     AArch64CPUTestParams::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 81;
+static constexpr unsigned NumAArch64CPUArchs = 82;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector<StringRef, NumAArch64CPUArchs> List;
diff --git a/llvm/unittests/Transforms/IPO/CMakeLists.txt b/llvm/unittests/Transforms/IPO/CMakeLists.txt
index 65be2966bec04..5b45191afc711 100644
--- a/llvm/unittests/Transforms/IPO/CMakeLists.txt
+++ b/llvm/unittests/Transforms/IPO/CMakeLists.txt
@@ -14,4 +14,5 @@ add_llvm_unittest(IPOTests
   AttributorTest.cpp
   FunctionSpecializationTest.cpp
   ImportIDTableTests.cpp
+  MergeFunctionsTest.cpp
   )
diff --git a/llvm/unittests/Transforms/IPO/MergeFunctionsTest.cpp b/llvm/unittests/Transforms/IPO/MergeFunctionsTest.cpp
new file mode 100644
index 0000000000000..56d119878a9ab
--- /dev/null
+++ b/llvm/unittests/Transforms/IPO/MergeFunctionsTest.cpp
@@ -0,0 +1,246 @@
+//===- MergeFunctionsTest.cpp - Unit tests for MergeFunctionsPass ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/MergeFunctions.h"
+
+#include "llvm/ADT/SetVector.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+#include <memory>
+
+using namespace llvm;
+
+namespace {
+
+TEST(MergeFunctions, TrueOutputModuleTest) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"invalid(
+        @.str = private unnamed_addr constant [10 x i8] c"On f: %d\0A\00", align 1
+        @.str.1 = private unnamed_addr constant [13 x i8] c"On main: %d\0A\00", align 1
+
+        define dso_local i32 @f(i32 noundef %arg) {
+            entry:
+                %add109 = call i32 @_slice_add10(i32 %arg)
+                %call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %add109)
+                ret i32 %add109
+        }
+
+        declare i32 @printf(ptr noundef, ...)
+
+        define dso_local i32 @main(i32 noundef %argc, ptr noundef %argv) {
+            entry:
+                %add99 = call i32 @_slice_add10(i32 %argc)
+                %call = call i32 @f(i32 noundef 2)
+                %sub = sub nsw i32 %call, 6
+                %call10 = call i32 (ptr, ...) @printf(ptr noundef @.str.1, i32 noundef %add99)
+                ret i32 %add99
+        }
+
+        define internal i32 @_slice_add10(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+
+        define internal i32 @_slice_add10_alt(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+    )invalid",
+                                                Err, Ctx));
+
+  // Expects true after merging _slice_add10 and _slice_add10_alt
+  EXPECT_TRUE(MergeFunctionsPass::runOnModule(*M));
+}
+
+TEST(MergeFunctions, TrueOutputFunctionsTest) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"invalid(
+        @.str = private unnamed_addr constant [10 x i8] c"On f: %d\0A\00", align 1
+        @.str.1 = private unnamed_addr constant [13 x i8] c"On main: %d\0A\00", align 1
+
+        define dso_local i32 @f(i32 noundef %arg) {
+            entry:
+                %add109 = call i32 @_slice_add10(i32 %arg)
+                %call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %add109)
+                ret i32 %add109
+        }
+
+        declare i32 @printf(ptr noundef, ...)
+
+        define dso_local i32 @main(i32 noundef %argc, ptr noundef %argv) {
+            entry:
+                %add99 = call i32 @_slice_add10(i32 %argc)
+                %call = call i32 @f(i32 noundef 2)
+                %sub = sub nsw i32 %call, 6
+                %call10 = call i32 (ptr, ...) @printf(ptr noundef @.str.1, i32 noundef %add99)
+                ret i32 %add99
+        }
+
+        define internal i32 @_slice_add10(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+
+        define internal i32 @_slice_add10_alt(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+    )invalid",
+                                                Err, Ctx));
+
+  SetVector<Function *> FunctionsSet;
+  for (Function &F : *M)
+    FunctionsSet.insert(&F);
+
+  DenseMap<Function *, Function *> MergeResult =
+      MergeFunctionsPass::runOnFunctions(FunctionsSet.getArrayRef());
+
+  // Expects that both functions (_slice_add10 and _slice_add10_alt)
+  // be mapped to the same new function
+  EXPECT_TRUE(!MergeResult.empty());
+  Function *NewFunction = M->getFunction("_slice_add10");
+  for (auto P : MergeResult)
+    if (P.second)
+      EXPECT_EQ(P.second, NewFunction);
+}
+
+TEST(MergeFunctions, FalseOutputModuleTest) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"invalid(
+        @.str = private unnamed_addr constant [10 x i8] c"On f: %d\0A\00", align 1
+        @.str.1 = private unnamed_addr constant [13 x i8] c"On main: %d\0A\00", align 1
+
+        define dso_local i32 @f(i32 noundef %arg) {
+            entry:
+                %add109 = call i32 @_slice_add10(i32 %arg)
+                %call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %add109)
+                ret i32 %add109
+        }
+
+        declare i32 @printf(ptr noundef, ...)
+
+        define dso_local i32 @main(i32 noundef %argc, ptr noundef %argv) {
+            entry:
+                %add99 = call i32 @_slice_add10(i32 %argc)
+                %call = call i32 @f(i32 noundef 2)
+                %sub = sub nsw i32 %call, 6
+                %call10 = call i32 (ptr, ...) @printf(ptr noundef @.str.1, i32 noundef %add99)
+                ret i32 %add99
+        }
+
+        define internal i32 @_slice_add10(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+
+        define internal i32 @_slice_add10_alt(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %0
+        }
+    )invalid",
+                                                Err, Ctx));
+
+  // Expects false after trying to merge _slice_add10 and _slice_add10_alt
+  EXPECT_FALSE(MergeFunctionsPass::runOnModule(*M));
+}
+
+TEST(MergeFunctions, FalseOutputFunctionsTest) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"invalid(
+        @.str = private unnamed_addr constant [10 x i8] c"On f: %d\0A\00", align 1
+        @.str.1 = private unnamed_addr constant [13 x i8] c"On main: %d\0A\00", align 1
+
+        define dso_local i32 @f(i32 noundef %arg) {
+            entry:
+                %add109 = call i32 @_slice_add10(i32 %arg)
+                %call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %add109)
+                ret i32 %add109
+        }
+
+        declare i32 @printf(ptr noundef, ...)
+
+        define dso_local i32 @main(i32 noundef %argc, ptr noundef %argv) {
+            entry:
+                %add99 = call i32 @_slice_add10(i32 %argc)
+                %call = call i32 @f(i32 noundef 2)
+                %sub = sub nsw i32 %call, 6
+                %call10 = call i32 (ptr, ...) @printf(ptr noundef @.str.1, i32 noundef %add99)
+                ret i32 %add99
+        }
+
+        define internal i32 @_slice_add10(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %4
+        }
+
+        define internal i32 @_slice_add10_alt(i32 %arg) {
+            sliceclone_entry:
+                %0 = mul nsw i32 %arg, %arg
+                %1 = mul nsw i32 %0, 2
+                %2 = mul nsw i32 %1, 2
+                %3 = mul nsw i32 %2, 2
+                %4 = add nsw i32 %3, 2
+                ret i32 %0
+        }
+    )invalid",
+                                                Err, Ctx));
+
+  SetVector<Function *> FunctionsSet;
+  for (Function &F : *M)
+    FunctionsSet.insert(&F);
+
+  DenseMap<Function *, Function *> MergeResult =
+      MergeFunctionsPass::runOnFunctions(FunctionsSet.getArrayRef());
+
+  // Expects empty map
+  EXPECT_EQ(MergeResult.size(), 0u);
+}
+
+} // namespace
\ No newline at end of file
diff --git a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
index f672ed2b1251f..c04f3c2a2863e 100644
--- a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
@@ -21,9 +21,9 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+namespace llvm {
+namespace memprof {
 namespace {
-using namespace llvm;
-using namespace llvm::memprof;
 using testing::Contains;
 using testing::ElementsAre;
 using testing::Pair;
@@ -102,16 +102,15 @@ declare !dbg !19 void @_Z2f3v()
 
   const auto &[CallerGUID, CallSites] = *It;
   EXPECT_EQ(CallerGUID, IndexedMemProfRecord::getGUID("_Z3foov"));
-  ASSERT_THAT(CallSites, SizeIs(3));
 
   // Verify that call sites show up in the ascending order of their source
   // locations.
-  EXPECT_THAT(CallSites[0], Pair(LineLocation(1, 3),
-                                 IndexedMemProfRecord::getGUID("_Z2f1v")));
-  EXPECT_THAT(CallSites[1], Pair(LineLocation(2, 3),
-                                 IndexedMemProfRecord::getGUID("_Z2f2v")));
-  EXPECT_THAT(CallSites[2], Pair(LineLocation(2, 9),
-                                 IndexedMemProfRecord::getGUID("_Z2f3v")));
+  EXPECT_THAT(
+      CallSites,
+      ElementsAre(
+          Pair(LineLocation(1, 3), IndexedMemProfRecord::getGUID("_Z2f1v")),
+          Pair(LineLocation(2, 3), IndexedMemProfRecord::getGUID("_Z2f2v")),
+          Pair(LineLocation(2, 9), IndexedMemProfRecord::getGUID("_Z2f3v"))));
 }
 
 TEST(MemProf, ExtractDirectCallsFromIRInline) {
@@ -205,37 +204,37 @@ declare !dbg !25 void @_Z2g2v() local_unnamed_addr
   ASSERT_NE(FooIt, Calls.end());
   const auto &[FooCallerGUID, FooCallSites] = *FooIt;
   EXPECT_EQ(FooCallerGUID, IndexedMemProfRecord::getGUID("_Z3foov"));
-  ASSERT_THAT(FooCallSites, SizeIs(2));
-  EXPECT_THAT(FooCallSites[0], Pair(LineLocation(1, 3),
-                                    IndexedMemProfRecord::getGUID("_ZL2f3v")));
-  EXPECT_THAT(FooCallSites[1], Pair(LineLocation(2, 9),
-                                    IndexedMemProfRecord::getGUID("_ZL2g3v")));
+  EXPECT_THAT(
+      FooCallSites,
+      ElementsAre(
+          Pair(LineLocation(1, 3), IndexedMemProfRecord::getGUID("_ZL2f3v")),
+          Pair(LineLocation(2, 9), IndexedMemProfRecord::getGUID("_ZL2g3v"))));
 
   auto F2It = Calls.find(IndexedMemProfRecord::getGUID("_ZL2f2v"));
   ASSERT_NE(F2It, Calls.end());
   const auto &[F2CallerGUID, F2CallSites] = *F2It;
   EXPECT_EQ(F2CallerGUID, IndexedMemProfRecord::getGUID("_ZL2f2v"));
-  ASSERT_THAT(F2CallSites, SizeIs(1));
-  EXPECT_THAT(F2CallSites[0], Pair(LineLocation(2, 3),
-                                   IndexedMemProfRecord::getGUID("_Z2f1v")));
+  EXPECT_THAT(F2CallSites,
+              ElementsAre(Pair(LineLocation(2, 3),
+                               IndexedMemProfRecord::getGUID("_Z2f1v"))));
 
   auto F3It = Calls.find(IndexedMemProfRecord::getGUID("_ZL2f3v"));
   ASSERT_NE(F3It, Calls.end());
   const auto &[F3CallerGUID, F3CallSites] = *F3It;
   EXPECT_EQ(F3CallerGUID, IndexedMemProfRecord::getGUID("_ZL2f3v"));
-  ASSERT_THAT(F3CallSites, SizeIs(1));
-  EXPECT_THAT(F3CallSites[0], Pair(LineLocation(1, 10),
-                                   IndexedMemProfRecord::getGUID("_ZL2f2v")));
+  EXPECT_THAT(F3CallSites,
+              ElementsAre(Pair(LineLocation(1, 10),
+                               IndexedMemProfRecord::getGUID("_ZL2f2v"))));
 
   auto G3It = Calls.find(IndexedMemProfRecord::getGUID("_ZL2g3v"));
   ASSERT_NE(G3It, Calls.end());
   const auto &[G3CallerGUID, G3CallSites] = *G3It;
   EXPECT_EQ(G3CallerGUID, IndexedMemProfRecord::getGUID("_ZL2g3v"));
-  ASSERT_THAT(G3CallSites, SizeIs(2));
-  EXPECT_THAT(G3CallSites[0], Pair(LineLocation(1, 8),
-                                   IndexedMemProfRecord::getGUID("_Z2g1v")));
-  EXPECT_THAT(G3CallSites[1], Pair(LineLocation(2, 3),
-                                   IndexedMemProfRecord::getGUID("_Z2g2v")));
+  EXPECT_THAT(
+      G3CallSites,
+      ElementsAre(
+          Pair(LineLocation(1, 8), IndexedMemProfRecord::getGUID("_Z2g1v")),
+          Pair(LineLocation(2, 3), IndexedMemProfRecord::getGUID("_Z2g2v"))));
 }
 
 TEST(MemProf, ExtractDirectCallsFromIRCallingNew) {
@@ -300,8 +299,7 @@ attributes #2 = { builtin allocsize(0) }
   ASSERT_NE(FooIt, Calls.end());
   const auto &[FooCallerGUID, FooCallSites] = *FooIt;
   EXPECT_EQ(FooCallerGUID, IndexedMemProfRecord::getGUID("_Z3foov"));
-  ASSERT_THAT(FooCallSites, SizeIs(1));
-  EXPECT_THAT(FooCallSites[0], Pair(LineLocation(1, 10), 0));
+  EXPECT_THAT(FooCallSites, ElementsAre(Pair(LineLocation(1, 10), 0)));
 }
 
 // Populate those fields returned by getHotColdSchema.
@@ -315,15 +313,12 @@ MemInfoBlock makePartialMIB() {
 }
 
 IndexedMemProfRecord
-makeRecordV2(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames,
-             std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames,
-             const MemInfoBlock &Block, const memprof::MemProfSchema &Schema) {
-  llvm::memprof::IndexedMemProfRecord MR;
-  for (const auto &CSId : AllocFrames) {
-    // We don't populate IndexedAllocationInfo::CallStack because we use it only
-    // in Version1.
+makeRecordV2(std::initializer_list<CallStackId> AllocFrames,
+             std::initializer_list<CallStackId> CallSiteFrames,
+             const MemInfoBlock &Block, const MemProfSchema &Schema) {
+  IndexedMemProfRecord MR;
+  for (const auto &CSId : AllocFrames)
     MR.AllocSites.emplace_back(CSId, Block, Schema);
-  }
   for (const auto &CSId : CallSiteFrames)
     MR.CallSiteIds.push_back(CSId);
   return MR;
@@ -427,7 +422,7 @@ attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
 
   const MemInfoBlock MIB = makePartialMIB();
 
-  Writer.setMemProfVersionRequested(memprof::Version3);
+  Writer.setMemProfVersionRequested(Version3);
   Writer.setMemProfFullSchema(false);
 
   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
@@ -435,9 +430,9 @@ attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
 
   const IndexedMemProfRecord IndexedMR = makeRecordV2(
       /*AllocFrames=*/{0x111, 0x222, 0x333},
-      /*CallSiteFrames=*/{}, MIB, memprof::getHotColdSchema());
+      /*CallSiteFrames=*/{}, MIB, getHotColdSchema());
 
-  memprof::IndexedMemProfData MemProfData;
+  IndexedMemProfData MemProfData;
   // The call sites within foo.
   MemProfData.Frames.try_emplace(0, GUIDFoo, 1, 8, false);
   MemProfData.Frames.try_emplace(1, GUIDFoo, 2, 3, false);
@@ -447,14 +442,11 @@ attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
   MemProfData.Frames.try_emplace(4, GUIDZzz, 9, 9, false);
   MemProfData.Frames.try_emplace(5, GUIDBaz, 9, 9, false);
   MemProfData.CallStacks.try_emplace(
-      0x111,
-      std::initializer_list<memprof::FrameId>{3, 0}); // bar called by foo
+      0x111, std::initializer_list<FrameId>{3, 0}); // bar called by foo
   MemProfData.CallStacks.try_emplace(
-      0x222,
-      std::initializer_list<memprof::FrameId>{4, 1}); // zzz called by foo
+      0x222, std::initializer_list<FrameId>{4, 1}); // zzz called by foo
   MemProfData.CallStacks.try_emplace(
-      0x333,
-      std::initializer_list<memprof::FrameId>{5, 2}); // baz called by foo
+      0x333, std::initializer_list<FrameId>{5, 2}); // baz called by foo
   MemProfData.Records.try_emplace(0x9999, IndexedMR);
   Writer.addMemProfData(MemProfData, Err);
 
@@ -488,3 +480,5 @@ attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
                                Pair(LineLocation(3, 3), LineLocation(2, 8))))));
 }
 } // namespace
+} // namespace memprof
+} // namespace llvm
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index 241d55bd62457..b49e37d9eee98 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -73,6 +73,6 @@ while.end:                                        ; preds = %while.cond
 
   bool ret =
       UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, &SE, &DT,
-                                 &AC, /*TTI=*/nullptr, PreserveLCSSA);
+                                 &AC, /*TTI=*/nullptr, PreserveLCSSA, 4);
   EXPECT_FALSE(ret);
 }
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index e6bb4b4684d26..8c73ee1def8ae 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -814,21 +814,46 @@ define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %v3, i8 %arg) {
   auto *BB = &*F->begin();
   auto It = BB->begin();
   auto *S1 = cast<sandboxir::StoreInst>(&*It++);
-  [[maybe_unused]] auto *S2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S2 = cast<sandboxir::StoreInst>(&*It++);
   auto *S3 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
 
   // Check new instruction callback.
   sandboxir::DependencyGraph DAG(getAA(*LLVMF), Ctx);
-  DAG.extend({S1, S3});
+  DAG.extend({S1, Ret});
   auto *Arg = F->getArg(3);
   auto *Ptr = S1->getPointerOperand();
-  sandboxir::StoreInst *NewS =
-      sandboxir::StoreInst::create(Arg, Ptr, Align(8), S3->getIterator(),
-                                   /*IsVolatile=*/true, Ctx);
-  auto *NewSN = DAG.getNode(NewS);
-  EXPECT_TRUE(NewSN != nullptr);
+  {
+    sandboxir::StoreInst *NewS =
+        sandboxir::StoreInst::create(Arg, Ptr, Align(8), S3->getIterator(),
+                                     /*IsVolatile=*/true, Ctx);
+    auto *NewSN = DAG.getNode(NewS);
+    EXPECT_TRUE(NewSN != nullptr);
+
+    // Check the MemDGNode chain.
+    auto *S2MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S2));
+    auto *NewMemSN = cast<sandboxir::MemDGNode>(NewSN);
+    auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+    EXPECT_EQ(S2MemN->getNextNode(), NewMemSN);
+    EXPECT_EQ(NewMemSN->getPrevNode(), S2MemN);
+    EXPECT_EQ(NewMemSN->getNextNode(), S3MemN);
+    EXPECT_EQ(S3MemN->getPrevNode(), NewMemSN);
+  }
+
+  {
+    // Also check if new node is at the end of the BB, after Ret.
+    sandboxir::StoreInst *NewS =
+        sandboxir::StoreInst::create(Arg, Ptr, Align(8), BB->end(),
+                                     /*IsVolatile=*/true, Ctx);
+    // Check the MemDGNode chain.
+    auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+    auto *NewMemSN = cast<sandboxir::MemDGNode>(DAG.getNode(NewS));
+    EXPECT_EQ(S3MemN->getNextNode(), NewMemSN);
+    EXPECT_EQ(NewMemSN->getPrevNode(), S3MemN);
+    EXPECT_EQ(NewMemSN->getNextNode(), nullptr);
+  }
+
   // TODO: Check the dependencies to/from NewSN after they land.
-  // TODO: Check the MemDGNode chain.
 }
 
 TEST_F(DependencyGraphTest, EraseInstrCallback) {
@@ -855,6 +880,16 @@ define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %v3, i8 %arg) {
   S2->eraseFromParent();
   auto *DeletedN = DAG.getNodeOrNull(S2);
   EXPECT_TRUE(DeletedN == nullptr);
+
+  // Check the MemDGNode chain.
+  auto *S1MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+  auto *S3MemN = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+  EXPECT_EQ(S1MemN->getNextNode(), S3MemN);
+  EXPECT_EQ(S3MemN->getPrevNode(), S1MemN);
+
+  // Check the chain when we erase the top node.
+  S1->eraseFromParent();
+  EXPECT_EQ(S3MemN->getPrevNode(), nullptr);
+
   // TODO: Check the dependencies to/from NewSN after they land.
-  // TODO: Check the MemDGNode chain.
 }
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index c2aa571b547c6..f2f6066538e1a 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -2466,7 +2466,8 @@ static void emitRegisterMatchErrorFunc(AsmMatcherInfo &Info, raw_ostream &OS) {
 }
 
 /// emitValidateOperandClass - Emit the function to validate an operand class.
-static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) {
+static void emitValidateOperandClass(const CodeGenTarget &Target,
+                                     AsmMatcherInfo &Info, raw_ostream &OS) {
   OS << "static unsigned validateOperandClass(MCParsedAsmOperand &GOp, "
      << "MatchClassKind Kind) {\n";
   OS << "  " << Info.Target.getName() << "Operand &Operand = ("
@@ -2508,15 +2509,22 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) {
   OS << "  } // end switch (Kind)\n\n";
 
   // Check for register operands, including sub-classes.
+  const auto &Regs = Target.getRegBank().getRegisters();
+  StringRef Namespace = Regs.front().TheDef->getValueAsString("Namespace");
+  SmallVector<StringRef> Table(1 + Regs.size(), "InvalidMatchClass");
+  for (const auto &RC : Info.RegisterClasses) {
+    const auto &Reg = Target.getRegBank().getReg(RC.first);
+    Table[Reg->EnumValue] = RC.second->Name;
+  }
   OS << "  if (Operand.isReg()) {\n";
-  OS << "    MatchClassKind OpKind;\n";
-  OS << "    switch (Operand.getReg().id()) {\n";
-  OS << "    default: OpKind = InvalidMatchClass; break;\n";
-  for (const auto &RC : Info.RegisterClasses)
-    OS << "    case " << RC.first->getValueAsString("Namespace")
-       << "::" << RC.first->getName() << ": OpKind = " << RC.second->Name
-       << "; break;\n";
-  OS << "    }\n";
+  OS << "    static constexpr uint16_t Table[" << Namespace
+     << "::NUM_TARGET_REGS] = {\n";
+  for (auto &MatchClassName : Table)
+    OS << "      " << MatchClassName << ",\n";
+  OS << "    };\n\n";
+  OS << "    unsigned RegID = Operand.getReg().id();\n";
+  OS << "    MatchClassKind OpKind = MCRegister::isPhysicalRegister(RegID) ? "
+        "(MatchClassKind)Table[RegID] : InvalidMatchClass;\n";
   OS << "    return isSubclass(OpKind, Kind) ? "
      << "(unsigned)MCTargetAsmParser::Match_Success :\n                     "
      << "                 getDiagKindFromRegisterClass(Kind);\n  }\n\n";
@@ -2540,53 +2548,58 @@ static void emitIsSubclass(CodeGenTarget &Target,
   OS << "  if (A == B)\n";
   OS << "    return true;\n\n";
 
-  bool EmittedSwitch = false;
+  // TODO: Use something like SequenceToOffsetTable to allow sequences to
+  // overlap in this table.
+  SmallVector<bool> SuperClassData;
+
+  OS << "  [[maybe_unused]] static constexpr struct {\n";
+  OS << "    uint32_t Offset;\n";
+  OS << "    uint16_t Start;\n";
+  OS << "    uint16_t Length;\n";
+  OS << "  } Table[] = {\n";
+  OS << "    {0, 0, 0},\n"; // InvalidMatchClass
+  OS << "    {0, 0, 0},\n"; // OptionalMatchClass
   for (const auto &A : Infos) {
-    std::vector<StringRef> SuperClasses;
-    if (A.IsOptional)
-      SuperClasses.push_back("OptionalMatchClass");
-    for (const auto &B : Infos) {
-      if (&A != &B && A.isSubsetOf(B))
-        SuperClasses.push_back(B.Name);
-    }
-
-    if (SuperClasses.empty())
-      continue;
-
-    // If this is the first SuperClass, emit the switch header.
-    if (!EmittedSwitch) {
-      OS << "  switch (A) {\n";
-      OS << "  default:\n";
-      OS << "    return false;\n";
-      EmittedSwitch = true;
-    }
-
-    OS << "\n  case " << A.Name << ":\n";
-
-    if (SuperClasses.size() == 1) {
-      OS << "    return B == " << SuperClasses.back() << ";\n";
-      continue;
-    }
-
-    if (!SuperClasses.empty()) {
-      OS << "    switch (B) {\n";
-      OS << "    default: return false;\n";
-      for (StringRef SC : SuperClasses)
-        OS << "    case " << SC << ": return true;\n";
-      OS << "    }\n";
-    } else {
-      // No case statement to emit
-      OS << "    return false;\n";
-    }
+    SmallVector<bool> SuperClasses;
+    SuperClasses.push_back(false);        // InvalidMatchClass
+    SuperClasses.push_back(A.IsOptional); // OptionalMatchClass
+    for (const auto &B : Infos)
+      SuperClasses.push_back(&A != &B && A.isSubsetOf(B));
+
+    // Trim leading and trailing zeros.
+    auto End = find_if(reverse(SuperClasses), [](bool B) { return B; }).base();
+    auto Start =
+        std::find_if(SuperClasses.begin(), End, [](bool B) { return B; });
+
+    unsigned Offset = SuperClassData.size();
+    SuperClassData.append(Start, End);
+
+    OS << "    {" << Offset << ", " << (Start - SuperClasses.begin()) << ", "
+       << (End - Start) << "},\n";
   }
+  OS << "  };\n\n";
 
-  // If there were case statements emitted into the string stream write the
-  // default.
-  if (EmittedSwitch)
-    OS << "  }\n";
-  else
+  if (SuperClassData.empty()) {
     OS << "  return false;\n";
-
+  } else {
+    // Dump the boolean data packed into bytes.
+    SuperClassData.append(-SuperClassData.size() % 8, false);
+    OS << "  static constexpr uint8_t Data[] = {\n";
+    for (unsigned I = 0, E = SuperClassData.size(); I < E; I += 8) {
+      unsigned Byte = 0;
+      for (unsigned J = 0; J < 8; ++J)
+        Byte |= (unsigned)SuperClassData[I + J] << J;
+      OS << formatv("    {:X2},\n", Byte);
+    }
+    OS << "  };\n\n";
+
+    OS << "  auto &Entry = Table[A];\n";
+    OS << "  unsigned Idx = B - Entry.Start;\n";
+    OS << "  if (Idx >= Entry.Length)\n";
+    OS << "    return false;\n";
+    OS << "  Idx += Entry.Offset;\n";
+    OS << "  return (Data[Idx / 8] >> (Idx % 8)) & 1;\n";
+  }
   OS << "}\n\n";
 }
 
@@ -3412,7 +3425,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   emitIsSubclass(Target, Info.Classes, OS);
 
   // Emit the routine to validate an operand against a match class.
-  emitValidateOperandClass(Info, OS);
+  emitValidateOperandClass(Target, Info, OS);
 
   emitMatchClassKindNames(Info.Classes, OS);
 
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
index 18e0b8fd135bb..0846f66ea6452 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
@@ -157,7 +157,8 @@ static bool doesSuffixLookLikeMangledType(StringRef Suffix) {
     return false;
 
   // [pi][0-9]+
-  if (is_contained("pi", Suffix[0]) && all_of(Suffix.drop_front(), isDigit))
+  if (Suffix.size() > 1 && is_contained("pi", Suffix[0]) &&
+      all_of(Suffix.drop_front(), isDigit))
     return true;
 
   // Match one of the named types.
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 74c399ee42317..f2385bdf1dde4 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -381,8 +381,10 @@ static void EndEmitFunction(raw_ostream &OS) {
 
 void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) {
 
-  assert(isUInt<16>(VecPatterns.size()) &&
-         "Using only 16 bits to encode offset into Pattern Table");
+  if (!isUInt<32>(VecPatterns.size()))
+    report_fatal_error("More patterns defined that can fit into 32-bit Pattern "
+                       "Table index encoding");
+
   assert(VecPatterns.size() == VecIncludeStrings.size() &&
          "The sizes of Pattern and include vectors should be the same");
 
@@ -947,7 +949,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
         std::string include_src = getIncludePath(PatRecord);
         unsigned Offset =
             getPatternIdxFromTable(src + " -> " + dst, std::move(include_src));
-        OS << "TARGET_VAL(" << Offset << "),\n";
+        OS << "COVERAGE_IDX_VAL(" << Offset << "),\n";
         OS.indent(FullIndexWidth + Indent);
       }
     }
@@ -1060,7 +1062,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       std::string include_src = getIncludePath(PatRecord);
       unsigned Offset =
           getPatternIdxFromTable(src + " -> " + dst, std::move(include_src));
-      OS << "TARGET_VAL(" << Offset << "),\n";
+      OS << "COVERAGE_IDX_VAL(" << Offset << "),\n";
       OS.indent(FullIndexWidth + Indent);
     }
     OS << "OPC_CompleteMatch, " << CM->getNumResults() << ", ";
@@ -1393,8 +1395,11 @@ void llvm::EmitMatcherTable(Matcher *TheMatcher, const CodeGenDAGPatterns &CGP,
   // final stream.
   OS << "{\n";
   OS << "  // Some target values are emitted as 2 bytes, TARGET_VAL handles\n";
-  OS << "  // this.\n";
+  OS << "  // this. Coverage indexes are emitted as 4 bytes,\n";
+  OS << "  // COVERAGE_IDX_VAL handles this.\n";
   OS << "  #define TARGET_VAL(X) X & 255, unsigned(X) >> 8\n";
+  OS << "  #define COVERAGE_IDX_VAL(X) X & 255, (unsigned(X) >> 8) & 255, ";
+  OS << "(unsigned(X) >> 16) & 255, (unsigned(X) >> 24) & 255\n";
   OS << "  static const unsigned char MatcherTable[] = {\n";
   TotalSize = MatcherEmitter.EmitMatcherList(TheMatcher, 1, 0, OS);
   OS << "    0\n  }; // Total Array size is " << (TotalSize + 1)
@@ -1402,6 +1407,7 @@ void llvm::EmitMatcherTable(Matcher *TheMatcher, const CodeGenDAGPatterns &CGP,
 
   MatcherEmitter.EmitHistogram(TheMatcher, OS);
 
+  OS << "  #undef COVERAGE_IDX_VAL\n";
   OS << "  #undef TARGET_VAL\n";
   OS << "  SelectCodeCommon(N, MatcherTable, sizeof(MatcherTable));\n";
   OS << "}\n";
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index e74fc00015b40..a0c93bed5ad83 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -32,6 +32,20 @@ using namespace llvm::dxil;
 
 namespace {
 
+struct DXILIntrinsicSelect {
+  StringRef Intrinsic;
+  SmallVector<const Record *> ArgSelectRecords;
+};
+
+static StringRef StripIntrinArgSelectTypePrefix(StringRef Type) {
+  StringRef Prefix = "IntrinArgSelect_";
+  if (!Type.starts_with(Prefix)) {
+    PrintFatalError("IntrinArgSelectType definintion must be prefixed with "
+                    "'IntrinArgSelect_'");
+  }
+  return Type.substr(Prefix.size());
+}
+
 struct DXILOperationDesc {
   std::string OpName; // name of DXIL operation
   int OpCode;         // ID of DXIL operation
@@ -42,8 +56,7 @@ struct DXILOperationDesc {
   SmallVector<const Record *> OverloadRecs;
   SmallVector<const Record *> StageRecs;
   SmallVector<const Record *> AttrRecs;
-  StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which
-                       // means no map exists
+  SmallVector<DXILIntrinsicSelect> IntrinsicSelects;
   SmallVector<StringRef, 4>
       ShaderStages; // shader stages to which this applies, empty for all.
   int OverloadParamIndex;             // Index of parameter with overload type.
@@ -71,6 +84,21 @@ static void ascendingSortByVersion(std::vector<const Record *> &Recs) {
   });
 }
 
+/// Take a `int_{intrinsic_name}` and return just the intrinsic_name part if
+/// available. Otherwise return the empty string.
+static StringRef GetIntrinsicName(const RecordVal *RV) {
+  if (RV && RV->getValue()) {
+    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
+      auto *IntrinsicDef = DI->getDef();
+      auto DefName = IntrinsicDef->getName();
+      assert(DefName.starts_with("int_") && "invalid intrinsic name");
+      // Remove the int_ from intrinsic name.
+      return DefName.substr(4);
+    }
+  }
+  return "";
+}
+
 /// Construct an object using the DXIL Operation records specified
 /// in DXIL.td. This serves as the single source of reference of
 /// the information extracted from the specified Record R, for
@@ -157,14 +185,16 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
                            OpName);
   }
 
-  const RecordVal *RV = R->getValue("LLVMIntrinsic");
-  if (RV && RV->getValue()) {
-    if (const DefInit *DI = dyn_cast<DefInit>(RV->getValue())) {
-      auto *IntrinsicDef = DI->getDef();
-      auto DefName = IntrinsicDef->getName();
-      assert(DefName.starts_with("int_") && "invalid intrinsic name");
-      // Remove the int_ from intrinsic name.
-      Intrinsic = DefName.substr(4);
+  auto IntrinsicSelectRecords = R->getValueAsListOfDefs("intrinsics");
+  if (IntrinsicSelectRecords.size()) {
+    for (const Record *R : IntrinsicSelectRecords) {
+      DXILIntrinsicSelect IntrSelect;
+      IntrSelect.Intrinsic = GetIntrinsicName(R->getValue("intrinsic"));
+      auto Args = R->getValueAsListOfDefs("arg_selects");
+      for (const Record *ArgSelect : Args) {
+        IntrSelect.ArgSelectRecords.emplace_back(ArgSelect);
+      }
+      IntrinsicSelects.emplace_back(std::move(IntrSelect));
     }
   }
 }
@@ -374,19 +404,45 @@ static void emitDXILOpFunctionTypes(ArrayRef<DXILOperationDesc> Ops,
 /// \param Output stream
 static void emitDXILIntrinsicMap(ArrayRef<DXILOperationDesc> Ops,
                                  raw_ostream &OS) {
+
   OS << "#ifdef DXIL_OP_INTRINSIC\n";
   OS << "\n";
   for (const auto &Op : Ops) {
-    if (Op.Intrinsic.empty())
+    if (Op.IntrinsicSelects.empty()) {
       continue;
-    OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
-       << ", Intrinsic::" << Op.Intrinsic << ")\n";
+    }
+    for (const DXILIntrinsicSelect &MappedIntr : Op.IntrinsicSelects) {
+      OS << "DXIL_OP_INTRINSIC(dxil::OpCode::" << Op.OpName
+         << ", Intrinsic::" << MappedIntr.Intrinsic << ", ";
+      for (const Record *ArgSelect : MappedIntr.ArgSelectRecords) {
+        std::string Type =
+            ArgSelect->getValueAsDef("type")->getNameInitAsString();
+        int Value = ArgSelect->getValueAsInt("value");
+        OS << "(IntrinArgSelect{"
+           << "IntrinArgSelect::Type::" << StripIntrinArgSelectTypePrefix(Type)
+           << "," << Value << "}), ";
+      }
+      OS << ")\n";
+    }
   }
   OS << "\n";
   OS << "#undef DXIL_OP_INTRINSIC\n";
   OS << "#endif\n\n";
 }
 
+/// Emit the IntrinArgSelect type for DirectX intrinsic to DXIL Op lowering
+static void emitDXILIntrinsicArgSelectTypes(const RecordKeeper &Records,
+                                            raw_ostream &OS) {
+  OS << "#ifdef DXIL_OP_INTRINSIC_ARG_SELECT_TYPE\n";
+  for (const Record *Records :
+       Records.getAllDerivedDefinitions("IntrinArgSelectType")) {
+    StringRef StrippedName = StripIntrinArgSelectTypePrefix(Records->getName());
+    OS << "DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(" << StrippedName << ")\n";
+  }
+  OS << "#undef DXIL_OP_INTRINSIC_ARG_SELECT_TYPE\n";
+  OS << "#endif\n\n";
+}
+
 /// Emit DXIL operation table
 /// \param A vector of DXIL Ops
 /// \param Output stream
@@ -527,6 +583,7 @@ static void emitDxilOperation(const RecordKeeper &Records, raw_ostream &OS) {
   emitDXILOpClasses(Records, OS);
   emitDXILOpParamTypes(Records, OS);
   emitDXILOpFunctionTypes(DXILOps, OS);
+  emitDXILIntrinsicArgSelectTypes(Records, OS);
   emitDXILIntrinsicMap(DXILOps, OS);
   OS << "#ifdef DXIL_OP_OPERATION_TABLE\n\n";
   emitDXILOperationTableDataStructs(Records, OS);
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 070d7522a97be..093602c3da804 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -239,13 +239,37 @@ static constexpr IntrinsicTargetInfo TargetInfos[] = {
 
 void IntrinsicEmitter::EmitIntrinsicToNameTable(
     const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
+  // Built up a table of the intrinsic names.
+  constexpr StringLiteral NotIntrinsic = "not_intrinsic";
+  StringToOffsetTable Table;
+  Table.GetOrAddStringOffset(NotIntrinsic);
+  for (const auto &Int : Ints)
+    Table.GetOrAddStringOffset(Int.Name);
+
   OS << R"(// Intrinsic ID to name table.
 #ifdef GET_INTRINSIC_NAME_TABLE
 // Note that entry #0 is the invalid intrinsic!
+
+)";
+
+  Table.EmitStringLiteralDef(OS, "static constexpr char IntrinsicNameTable[]",
+                             /*Indent=*/"");
+
+  OS << R"(
+static constexpr unsigned IntrinsicNameOffsetTable[] = {
 )";
+
+  OS << formatv("  {}, // {}\n", Table.GetStringOffset(NotIntrinsic),
+                NotIntrinsic);
   for (const auto &Int : Ints)
-    OS << "  \"" << Int.Name << "\",\n";
-  OS << "#endif\n\n";
+    OS << formatv("  {}, // {}\n", Table.GetStringOffset(Int.Name), Int.Name);
+
+  OS << R"(
+}; // IntrinsicNameOffsetTable
+
+#endif
+
+)";
 }
 
 void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index be2a2b3884c73..a6f87119aca5b 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -953,7 +953,7 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS) {
 
   OS << "extern const MCRegisterDesc " << TargetName
      << "RegDesc[] = { // Descriptors\n";
-  OS << "  { " << RegStrings.get("") << ", 0, 0, 0, 0, 0, 0 },\n";
+  OS << "  { " << RegStrings.get("") << ", 0, 0, 0, 0, 0, 0, 0 },\n";
 
   // Emit the register descriptors now.
   i = 0;
@@ -968,8 +968,8 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS) {
        << DiffSeqs.get(SubRegLists[i]) << ", " << DiffSeqs.get(SuperRegLists[i])
        << ", " << SubRegIdxSeqs.get(SubRegIdxLists[i]) << ", "
        << (Offset << RegUnitBits | FirstRU) << ", "
-       << LaneMaskSeqs.get(RegUnitLaneMasks[i]) << ", " << Reg.Constant
-       << " },\n";
+       << LaneMaskSeqs.get(RegUnitLaneMasks[i]) << ", " << Reg.Constant << ", "
+       << Reg.Artificial << " },\n";
     ++i;
   }
   OS << "};\n\n"; // End of register descriptors...
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 02c799cb6f147..9bd18f5837eea 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/MC/MCSchedule.h"
 #include "llvm/Support/Debug.h"
@@ -91,6 +92,7 @@ class SubtargetEmitter {
   void emitSubtargetInfoMacroCalls(raw_ostream &OS);
   unsigned featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
   unsigned cpuKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
+  unsigned cpuNames(raw_ostream &OS);
   void formItineraryStageString(const std::string &Names,
                                 const Record *ItinData, std::string &ItinString,
                                 unsigned &NStages);
@@ -297,6 +299,40 @@ unsigned SubtargetEmitter::featureKeyValues(raw_ostream &OS,
   return FeatureList.size();
 }
 
+unsigned SubtargetEmitter::cpuNames(raw_ostream &OS) {
+  // Begin processor name table.
+  OS << "// Sorted array of names of CPU subtypes, including aliases.\n"
+     << "extern const llvm::StringRef " << Target << "Names[] = {\n";
+
+  std::vector<const Record *> ProcessorList =
+      Records.getAllDerivedDefinitions("Processor");
+
+  std::vector<const Record *> ProcessorAliasList =
+      Records.getAllDerivedDefinitionsIfDefined("ProcessorAlias");
+
+  SmallVector<StringRef> Names;
+  Names.reserve(ProcessorList.size() + ProcessorAliasList.size());
+
+  for (const Record *Processor : ProcessorList) {
+    StringRef Name = Processor->getValueAsString("Name");
+    Names.push_back(Name);
+  }
+
+  for (const Record *Rec : ProcessorAliasList) {
+    auto Name = Rec->getValueAsString("Name");
+    Names.push_back(Name);
+  }
+
+  llvm::sort(Names);
+  llvm::interleave(
+      Names, OS, [&](StringRef Name) { OS << '"' << Name << '"'; }, ",\n");
+
+  // End processor name table.
+  OS << "};\n";
+
+  return Names.size();
+}
+
 //
 // CPUKeyValues - Emit data of all the subtarget processors.  Used by command
 // line.
@@ -1926,13 +1962,14 @@ void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
      << "GenMCSubtargetInfo : public MCSubtargetInfo {\n";
   OS << "  " << Target << "GenMCSubtargetInfo(const Triple &TT,\n"
      << "    StringRef CPU, StringRef TuneCPU, StringRef FS,\n"
+     << "    ArrayRef<StringRef> PN,\n"
      << "    ArrayRef<SubtargetFeatureKV> PF,\n"
      << "    ArrayRef<SubtargetSubTypeKV> PD,\n"
      << "    const MCWriteProcResEntry *WPR,\n"
      << "    const MCWriteLatencyEntry *WL,\n"
      << "    const MCReadAdvanceEntry *RA, const InstrStage *IS,\n"
      << "    const unsigned *OC, const unsigned *FP) :\n"
-     << "      MCSubtargetInfo(TT, CPU, TuneCPU, FS, PF, PD,\n"
+     << "      MCSubtargetInfo(TT, CPU, TuneCPU, FS, PN, PF, PD,\n"
      << "                      WPR, WL, RA, IS, OC, FP) { }\n\n"
      << "  unsigned resolveVariantSchedClass(unsigned SchedClass,\n"
      << "      const MCInst *MI, const MCInstrInfo *MCII,\n"
@@ -2001,6 +2038,8 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "\n";
   unsigned NumProcs = cpuKeyValues(OS, FeatureMap);
   OS << "\n";
+  unsigned NumNames = cpuNames(OS);
+  OS << "\n";
 
   // MCInstrInfo initialization routine.
   emitGenMCSubtargetInfo(OS);
@@ -2013,6 +2052,10 @@ void SubtargetEmitter::run(raw_ostream &OS) {
        << "  TuneCPU = AArch64::resolveCPUAlias(TuneCPU);\n";
   OS << "  return new " << Target
      << "GenMCSubtargetInfo(TT, CPU, TuneCPU, FS, ";
+  if (NumNames)
+    OS << Target << "Names, ";
+  else
+    OS << "{}, ";
   if (NumFeatures)
     OS << Target << "FeatureKV, ";
   else
@@ -2096,6 +2139,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
 
   OS << "#include \"llvm/CodeGen/TargetSchedule.h\"\n\n";
   OS << "namespace llvm {\n";
+  OS << "extern const llvm::StringRef " << Target << "Names[];\n";
   OS << "extern const llvm::SubtargetFeatureKV " << Target << "FeatureKV[];\n";
   OS << "extern const llvm::SubtargetSubTypeKV " << Target << "SubTypeKV[];\n";
   OS << "extern const llvm::MCWriteProcResEntry " << Target
@@ -2119,6 +2163,10 @@ void SubtargetEmitter::run(raw_ostream &OS) {
        << "                        AArch64::resolveCPUAlias(TuneCPU), FS, ";
   else
     OS << "  : TargetSubtargetInfo(TT, CPU, TuneCPU, FS, ";
+  if (NumNames)
+    OS << "ArrayRef(" << Target << "Names, " << NumNames << "), ";
+  else
+    OS << "{}, ";
   if (NumFeatures)
     OS << "ArrayRef(" << Target << "FeatureKV, " << NumFeatures << "), ";
   else
diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt
index 11e41834c011a..6ffb7fe14c434 100644
--- a/llvm/utils/git/requirements.txt
+++ b/llvm/utils/git/requirements.txt
@@ -227,9 +227,9 @@ pycparser==2.22 \
     --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
     --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
     # via cffi
-pygithub==2.2.0 \
-    --hash=sha256:41042ea53e4c372219db708c38d2ca1fd4fadab75475bac27d89d339596cfad1 \
-    --hash=sha256:e39be7c4dc39418bdd6e3ecab5931c636170b8b21b4d26f9ecf7e6102a3b51c3
+pygithub==2.4.0 \
+    --hash=sha256:6601e22627e87bac192f1e2e39c6e6f69a43152cfb8f307cee575879320b3051 \
+    --hash=sha256:81935aa4bdc939fba98fee1cb47422c09157c56a27966476ff92775602b9ee24
     # via -r requirements.txt.in
 pyjwt[crypto]==2.9.0 \
     --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \
@@ -259,9 +259,9 @@ typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
     # via pygithub
-urllib3==2.2.2 \
-    --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \
-    --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168
+urllib3==2.2.3 \
+    --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
+    --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
     # via
     #   pygithub
     #   requests
diff --git a/llvm/utils/git/requirements.txt.in b/llvm/utils/git/requirements.txt.in
index 512b80b60e1d2..e880e94e1de80 100644
--- a/llvm/utils/git/requirements.txt.in
+++ b/llvm/utils/git/requirements.txt.in
@@ -4,6 +4,8 @@
 # pip-compile -o requirements.txt  requirements.txt.in
 
 certifi>=2023.7.22  # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047
-PyGithub==2.2.0 # >=1.59.1 For WorkflowRun.name
+PyGithub==2.4.0 # >=1.59.1 For WorkflowRun.name
                 # >= 2.2.0 for permission arg to Repository.get_collaborators
+                # >= Fix for https://github.com/PyGithub/PyGithub/issues/3001
+                # (variables in graphql query).
 GitPython>=3.1.32  # https://security.snyk.io/vuln/SNYK-PYTHON-GITPYTHON-5840584
diff --git a/llvm/utils/gn/secondary/bolt/unittests/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/unittests/Core/BUILD.gn
index 51dc24481a513..c7c9459fdff16 100644
--- a/llvm/utils/gn/secondary/bolt/unittests/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/unittests/Core/BUILD.gn
@@ -6,15 +6,18 @@ unittest("CoreTests") {
   deps = [
     "//bolt/lib/Core",
     "//bolt/lib/Rewrite",
+    "//bolt/lib/Profile",
     "//llvm/lib/DebugInfo/DWARF",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Testing/Support",
   ]
   sources = [
     "BinaryContext.cpp",
     "DynoStats.cpp",
     "MCPlusBuilder.cpp",
+    "MemoryMaps.cpp",
   ]
 
   defines = []
diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
index 6bfdeedfacd26..db2ee2e41d2bd 100644
--- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -62,6 +62,7 @@ static_library("AST") {
     "AttrDocTable.cpp",
     "AttrImpl.cpp",
     "Availability.cpp",
+    "ByteCode/BitcastBuffer.cpp",
     "ByteCode/ByteCodeEmitter.cpp",
     "ByteCode/Compiler.cpp",
     "ByteCode/Context.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Serialization/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Serialization/BUILD.gn
index 8d8b0178cecce..c6a725ea9d26b 100644
--- a/llvm/utils/gn/secondary/clang/lib/Serialization/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Serialization/BUILD.gn
@@ -30,5 +30,6 @@ static_library("Serialization") {
     "ModuleManager.cpp",
     "ObjectFilePCHContainerReader.cpp",
     "PCHContainerOperations.cpp",
+    "TemplateArgumentHasher.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index 8579141aa7f99..e1c773630de70 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -59,7 +59,6 @@ static_library("Checkers") {
     "GCDAntipatternChecker.cpp",
     "GTestChecker.cpp",
     "GenericTaintChecker.cpp",
-    "IdenticalExprChecker.cpp",
     "InnerPointerChecker.cpp",
     "InvalidatedIteratorChecker.cpp",
     "Iterator.cpp",
@@ -139,12 +138,13 @@ static_library("Checkers") {
     "VforkChecker.cpp",
     "VirtualCallChecker.cpp",
     "WebKit/ASTUtils.cpp",
+    "WebKit/MemoryUnsafeCastChecker.cpp",
     "WebKit/PtrTypesSemantics.cpp",
+    "WebKit/RawPtrRefCallArgsChecker.cpp",
+    "WebKit/RawPtrRefLocalVarsChecker.cpp",
     "WebKit/RawPtrRefMemberChecker.cpp",
     "WebKit/RefCntblBaseVirtualDtorChecker.cpp",
-    "WebKit/RawPtrRefCallArgsChecker.cpp",
     "WebKit/UncountedLambdaCapturesChecker.cpp",
-    "WebKit/RawPtrRefLocalVarsChecker.cpp",
     "cert/InvalidPtrChecker.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
index fcdb9a5b1aeb2..04c646b059ece 100644
--- a/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
@@ -10,6 +10,7 @@ unittest("InterpTests") {
     "//clang/lib/Tooling",
   ]
   sources = [
+    "BitcastBuffer.cpp",
     "Descriptor.cpp",
     "toAPValue.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang/unittests/Serialization/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Serialization/BUILD.gn
index 3b7ba10c49c21..f2ad1dcdacc7f 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Serialization/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Serialization/BUILD.gn
@@ -16,6 +16,7 @@ unittest("SerializationTests") {
   sources = [
     "ForceCheckFileInputTest.cpp",
     "InMemoryModuleCacheTest.cpp",
+    "LoadSpecLazilyTest.cpp",
     "ModuleCacheTest.cpp",
     "NoCommentsTest.cpp",
     "PreambleInNamedModulesTest.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
index c8347f6336d73..0e04549587b2e 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
@@ -69,6 +69,7 @@ source_set("sources") {
     "tsd.h",
     "tsd_exclusive.h",
     "tsd_shared.h",
+    "type_traits.h",
     "vector.h",
     "wrappers_c.h",
     "wrappers_c_checks.h",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 98bd40a56eeac..2d5ea27099238 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -284,11 +284,13 @@ if (current_toolchain == default_toolchain) {
       "__atomic/atomic_sync.h",
       "__atomic/check_memory_order.h",
       "__atomic/contention_t.h",
-      "__atomic/cxx_atomic_impl.h",
       "__atomic/fence.h",
       "__atomic/is_always_lock_free.h",
       "__atomic/kill_dependency.h",
       "__atomic/memory_order.h",
+      "__atomic/support.h",
+      "__atomic/support/c11.h",
+      "__atomic/support/gcc.h",
       "__atomic/to_gcc_order.h",
       "__bit/bit_cast.h",
       "__bit/bit_ceil.h",
@@ -430,6 +432,7 @@ if (current_toolchain == default_toolchain) {
       "__filesystem/space_info.h",
       "__filesystem/u8path.h",
       "__flat_map/flat_map.h",
+      "__flat_map/key_value_iterator.h",
       "__flat_map/sorted_unique.h",
       "__format/buffer.h",
       "__format/concepts.h",
diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index 313b24a02dd5d..2bbe9e8e0a7d3 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -149,7 +149,6 @@ cxx_sources = [
   "ios.cpp",
   "ios.instantiations.cpp",
   "iostream.cpp",
-  "legacy_pointer_safety.cpp",
   "locale.cpp",
   "memory.cpp",
   "memory_resource.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
index 52e203c009c71..580afe7deb0ff 100644
--- a/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/DataFormatters/BUILD.gn
@@ -18,6 +18,8 @@ static_library("DataFormatters") {
     "FormatCache.cpp",
     "FormatClasses.cpp",
     "FormatManager.cpp",
+    "FormatterBytecode.cpp",
+    "FormatterSection.cpp",
     "FormattersHelpers.cpp",
     "LanguageCategory.cpp",
     "StringPrinter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
index 59ecb66f2bcb0..ab72ac4ae9f4b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
@@ -43,6 +43,7 @@ static_library("CodeGen") {
     "CodeGen.cpp",
     "CodeGenCommonISel.cpp",
     "CodeGenPrepare.cpp",
+    "CodeGenTargetMachineImpl.cpp",
     "CommandFlags.cpp",
     "ComplexDeinterleavingPass.cpp",
     "CriticalAntiDepBreaker.cpp",
@@ -83,7 +84,6 @@ static_library("CodeGen") {
     "IntrinsicLowering.cpp",
     "JMCInstrumenter.cpp",
     "KCFI.cpp",
-    "CodeGenTargetMachineImpl.cpp",
     "LatencyPriorityQueue.cpp",
     "LazyMachineBlockFrequencyInfo.cpp",
     "LexicalScopes.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 77db9c634b6ff..11311139a8ba4 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -38,6 +38,7 @@ static_library("Orc") {
     "IRTransformLayer.cpp",
     "IndirectionUtils.cpp",
     "JITLinkRedirectableSymbolManager.cpp",
+    "JITLinkReentryTrampolines.cpp",
     "JITTargetMachineBuilder.cpp",
     "LLJIT.cpp",
     "Layer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Shared/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Shared/BUILD.gn
index d152bc8c0c2d3..e66271a4d5f1e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Shared/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Shared/BUILD.gn
@@ -8,5 +8,6 @@ static_library("Shared") {
     "OrcError.cpp",
     "OrcRTBridge.cpp",
     "SimpleRemoteEPCUtils.cpp",
+    "SymbolStringPool.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index d122e8a21671b..745179213ae32 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -179,6 +179,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUPromoteAlloca.cpp",
     "AMDGPUPromoteKernelArguments.cpp",
     "AMDGPURegBankCombiner.cpp",
+    "AMDGPURegBankLegalize.cpp",
     "AMDGPURegBankSelect.cpp",
     "AMDGPURegisterBankInfo.cpp",
     "AMDGPURemoveIncompatibleFunctions.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index e2e719425b08d..505bb6ddfffcf 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -142,6 +142,7 @@ static_library("LLVMRISCVCodeGen") {
     "RISCVTargetObjectFile.cpp",
     "RISCVTargetTransformInfo.cpp",
     "RISCVVLOptimizer.cpp",
+    "RISCVVectorMaskDAGMutation.cpp",
     "RISCVVectorPeephole.cpp",
     "RISCVZacasABIFix.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/LogicalView/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/LogicalView/BUILD.gn
index d87bac64e7b34..402f27f8189f2 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/LogicalView/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/LogicalView/BUILD.gn
@@ -2,19 +2,24 @@ import("//third-party/unittest/unittest.gni")
 
 unittest("DebugInfoLogicalViewTests") {
   deps = [
+    "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/LogicalView",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Target:AllTargetsDescs",
     "//llvm/lib/Target:AllTargetsDisassemblers",
     "//llvm/lib/Target:AllTargetsInfos",
+    "//llvm/lib/Target:TargetsToBuild",
     "//llvm/lib/TargetParser",
     "//llvm/lib/Testing/Support",
   ]
   sources = [
+    "../DWARF/DwarfGenerator.cpp",
+    "../DWARF/DwarfUtils.cpp",
     "CodeViewReaderTest.cpp",
     "CommandLineOptionsTest.cpp",
     "CompareElementsTest.cpp",
     "DWARFReaderTest.cpp",
+    "DWARFGeneratedTest.cpp",
     "LocationRangesTest.cpp",
     "LogicalElementsTest.cpp",
     "SelectElementsTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn
index cddec7aee5745..c37928632d020 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn
@@ -17,6 +17,7 @@ unittest("AArch64Tests") {
   include_dirs = [ "//llvm/lib/Target/AArch64" ]
   sources = [
     "AArch64InstPrinterTest.cpp",
+    "AArch64RegisterInfoTest.cpp",
     "AArch64SVESchedPseudoTest.cpp",
     "AddressingModes.cpp",
     "DecomposeStackOffsetTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/SPIRV/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/SPIRV/BUILD.gn
index dfb5976f5af1b..86e6634ca692a 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Target/SPIRV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Target/SPIRV/BUILD.gn
@@ -3,5 +3,7 @@ group("dummy") {
   sources = [
     "SPIRVAPITest.cpp",
     "SPIRVConvergenceRegionAnalysisTests.cpp",
+    "SPIRVPartialOrderingVisitorTests.cpp",
+    "SPIRVSortBlocksTests.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn
index ad44635f107a1..c358f8be99556 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn
@@ -12,6 +12,7 @@ unittest("IPOTests") {
     "FunctionSpecializationTest.cpp",
     "ImportIDTableTests.cpp",
     "LowerTypeTests.cpp",
+    "MergeFunctionsTest.cpp",
     "WholeProgramDevirt.cpp",
   ]
 }
diff --git a/llvm/utils/lldbDataFormatters.py b/llvm/utils/lldbDataFormatters.py
index 4a34ad2a87c33..09143d2d81cbb 100644
--- a/llvm/utils/lldbDataFormatters.py
+++ b/llvm/utils/lldbDataFormatters.py
@@ -93,6 +93,12 @@ def __lldb_init_module(debugger, internal_dict):
         '-x "^llvm::DenseMap<.+>$"'
     )
 
+    debugger.HandleCommand(
+        "type synthetic add -w llvm "
+        f"-l {__name__}.ExpectedSynthetic "
+        '-x "^llvm::Expected<.+>$"'
+    )
+
 
 # Pretty printer for llvm::SmallVector/llvm::SmallVectorImpl
 class SmallVectorSynthProvider:
@@ -432,3 +438,43 @@ def update(self):
         for indexes in key_buckets.values():
             if len(indexes) == 1:
                 self.child_buckets.append(indexes[0])
+
+
+class ExpectedSynthetic:
+    # The llvm::Expected<T> value.
+    expected: lldb.SBValue
+    # The stored success value or error value.
+    stored_value: lldb.SBValue
+
+    def __init__(self, valobj: lldb.SBValue, _) -> None:
+        self.expected = valobj
+
+    def update(self) -> None:
+        has_error = self.expected.GetChildMemberWithName("HasError").unsigned
+        if not has_error:
+            name = "value"
+            member = "TStorage"
+        else:
+            name = "error"
+            member = "ErrorStorage"
+        # Anonymous union.
+        union = self.expected.child[0]
+        storage = union.GetChildMemberWithName(member)
+        stored_type = storage.type.template_args[0]
+        self.stored_value = storage.Cast(stored_type).Clone(name)
+
+    def num_children(self) -> int:
+        return 1
+
+    def get_child_index(self, name: str) -> int:
+        if name == self.stored_value.name:
+            return 0
+        # Allow dereferencing for values, not errors.
+        if name == "$$dereference$$" and self.stored_value.name == "value":
+            return 0
+        return -1
+
+    def get_child_at_index(self, idx: int) -> lldb.SBValue:
+        if idx == 0:
+            return self.stored_value
+        return lldb.SBValue()
diff --git a/llvm/utils/merge-json.py b/llvm/utils/merge-json.py
new file mode 100644
index 0000000000000..bf0d651fe5f52
--- /dev/null
+++ b/llvm/utils/merge-json.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+"""A command line utility to merge two JSON files.
+
+This is a python program that merges two JSON files into a single one. The
+intended use for this is to combine generated 'compile_commands.json' files
+created by CMake when performing an LLVM runtime build.
+"""
+
+import argparse
+import json
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-o",
+        type=str,
+        help="The output file to write JSON data to",
+        default=None,
+        nargs="?",
+    )
+    parser.add_argument(
+        "json_files", type=str, nargs="+", help="Input JSON files to merge"
+    )
+    args = parser.parse_args()
+
+    merged_data = []
+
+    for json_file in args.json_files:
+        try:
+            with open(json_file, "r") as f:
+                data = json.load(f)
+                merged_data.extend(data)
+        except (IOError, json.JSONDecodeError) as e:
+            print("Failed to parse {json_file}: {e}", file=sys.stderr)
+            continue
+
+    # Deduplicate by converting each entry to a tuple of sorted key-value pairs
+    unique_data = list({json.dumps(entry, sort_keys=True) for entry in merged_data})
+    unique_data = [json.loads(entry) for entry in unique_data]
+
+    with open(args.o, "w") as f:
+        json.dump(unique_data, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/llvm/utils/vim/ftplugin/llvm.vim b/llvm/utils/vim/ftplugin/llvm.vim
index 55315159c18bc..4f38c47552d55 100644
--- a/llvm/utils/vim/ftplugin/llvm.vim
+++ b/llvm/utils/vim/ftplugin/llvm.vim
@@ -11,3 +11,11 @@ setlocal softtabstop=2 shiftwidth=2
 setlocal expandtab
 setlocal comments+=:;
 setlocal commentstring=;\ %s
+" We treat sequences of the following characters as forming 'keywords', with
+" the aim of easing movement around LLVM identifiers:
+" * identifier prefixes: '%' and '@' (@-@)
+" * all characters where isalpha() returns TRUE (@)
+" * the digits 0-9 (48-57)
+" * other characters that may form identifiers: '_', '.', '-', '$'
+" Comment this out to restore the default behaviour
+setlocal iskeyword=%,@-@,@,48-57,_,.,-,$
diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt
index 2880dc30bca91..82bfbe56f0839 100644
--- a/mlir/CMakeLists.txt
+++ b/mlir/CMakeLists.txt
@@ -162,6 +162,9 @@ configure_file(
 # Requires:
 #   The pybind11 library can be found (set with -DPYBIND_DIR=...)
 #   The python executable is correct (set with -DPython3_EXECUTABLE=...)
+# By default, find_package and probing for installed pybind11 is performed.
+# Super projects can set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES=ON to 
+# disable all package setup and control it themselves.
 #-------------------------------------------------------------------------------
 
 set(MLIR_ENABLE_BINDINGS_PYTHON 0 CACHE BOOL
@@ -170,8 +173,17 @@ set(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH 1 CACHE BOOL
        "Prime the python detection by searching for a full 'Development' \
        component first (temporary while diagnosing environment specific Python \
        detection issues)")
+set(MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES 0 CACHE BOOL
+       "Performs python dev package configuration sufficient to use all MLIR \
+       python features. Super-projects that wish to control their own setup \
+       must perform an appropriate find_package of Python3 with \
+       'Development.Module' and ensure that find_package(pybind11) is \
+       satisfied (and keep up to date as requirements evolve).")
+
 if(MLIR_ENABLE_BINDINGS_PYTHON)
   include(MLIRDetectPythonEnv)
+  # Note that both upstream and downstreams often call this macro. It gates
+  # internally on the MLIR_CONFIGURE_PYTHON_DEV_PACKAGES option.
   mlir_configure_python_dev_packages()
 endif()
 
diff --git a/mlir/cmake/modules/AddMLIRPython.cmake b/mlir/cmake/modules/AddMLIRPython.cmake
index 7b91f43e2d57f..67619a90c90be 100644
--- a/mlir/cmake/modules/AddMLIRPython.cmake
+++ b/mlir/cmake/modules/AddMLIRPython.cmake
@@ -114,10 +114,11 @@ endfunction()
 #   EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends
 #     on. These will be collected for all extensions and put into an
 #     aggregate dylib that is linked against.
+#   PYTHON_BINDINGS_LIBRARY: Either pybind11 or nanobind.
 function(declare_mlir_python_extension name)
   cmake_parse_arguments(ARG
     ""
-    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT"
+    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT;PYTHON_BINDINGS_LIBRARY"
     "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS"
     ${ARGN})
 
@@ -126,15 +127,20 @@ function(declare_mlir_python_extension name)
   endif()
   set(_install_destination "src/python/${name}")
 
+  if(NOT ARG_PYTHON_BINDINGS_LIBRARY)
+    set(ARG_PYTHON_BINDINGS_LIBRARY "pybind11")
+  endif()
+
   add_library(${name} INTERFACE)
   set_target_properties(${name} PROPERTIES
     # Yes: Leading-lowercase property names are load bearing and the recommended
     # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/19261
-    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS"
+    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS;mlir_python_BINDINGS_LIBRARY"
     mlir_python_SOURCES_TYPE extension
     mlir_python_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}"
     mlir_python_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}"
     mlir_python_DEPENDS ""
+    mlir_python_BINDINGS_LIBRARY "${ARG_PYTHON_BINDINGS_LIBRARY}"
   )
 
   # Set the interface source and link_libs properties of the target
@@ -223,12 +229,14 @@ function(add_mlir_python_modules name)
     elseif(_source_type STREQUAL "extension")
       # Native CPP extension.
       get_target_property(_module_name ${sources_target} mlir_python_EXTENSION_MODULE_NAME)
+      get_target_property(_bindings_library ${sources_target} mlir_python_BINDINGS_LIBRARY)
       # Transform relative source to based on root dir.
       set(_extension_target "${modules_target}.extension.${_module_name}.dso")
       add_mlir_python_extension(${_extension_target} "${_module_name}"
         INSTALL_COMPONENT ${modules_target}
         INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs"
         OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs"
+        PYTHON_BINDINGS_LIBRARY ${_bindings_library}
         LINK_LIBS PRIVATE
           ${sources_target}
           ${ARG_COMMON_CAPI_LINK_LIBS}
@@ -634,7 +642,7 @@ endfunction()
 function(add_mlir_python_extension libname extname)
   cmake_parse_arguments(ARG
   ""
-  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY"
+  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY;PYTHON_BINDINGS_LIBRARY"
   "SOURCES;LINK_LIBS"
   ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
@@ -644,9 +652,16 @@ function(add_mlir_python_extension libname extname)
   # The actual extension library produces a shared-object or DLL and has
   # sources that must be compiled in accordance with pybind11 needs (RTTI and
   # exceptions).
-  pybind11_add_module(${libname}
-    ${ARG_SOURCES}
-  )
+  if(NOT DEFINED ARG_PYTHON_BINDINGS_LIBRARY OR ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "pybind11")
+    pybind11_add_module(${libname}
+      ${ARG_SOURCES}
+    )
+  elseif(ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "nanobind")
+    nanobind_add_module(${libname}
+      NB_DOMAIN mlir
+      ${ARG_SOURCES}
+    )
+  endif()
 
   # The extension itself must be compiled with RTTI and exceptions enabled.
   # Also, some warning classes triggered by pybind11 are disabled.
diff --git a/mlir/cmake/modules/MLIRDetectPythonEnv.cmake b/mlir/cmake/modules/MLIRDetectPythonEnv.cmake
index d3a98aaf6ffd1..c62ac7fa615ea 100644
--- a/mlir/cmake/modules/MLIRDetectPythonEnv.cmake
+++ b/mlir/cmake/modules/MLIRDetectPythonEnv.cmake
@@ -2,34 +2,49 @@
 
 # Finds and configures python packages needed to build MLIR Python bindings.
 macro(mlir_configure_python_dev_packages)
-  if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH)
-    # Prime the search for python to see if there is a full development
-    # package. This seems to work around cmake bugs searching only for
-    # Development.Module in some environments. However, in other environments
-    # it may interfere with the subsequent search for Development.Module.
+  if(NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES)
+    if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH)
+      # Prime the search for python to see if there is a full development
+      # package. This seems to work around cmake bugs searching only for
+      # Development.Module in some environments. However, in other environments
+      # it may interfere with the subsequent search for Development.Module.
+      find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
+        COMPONENTS Interpreter Development)
+    endif()
+
+    # After CMake 3.18, we are able to limit the scope of the search to just
+    # Development.Module. Searching for Development will fail in situations where
+    # the Python libraries are not available. When possible, limit to just
+    # Development.Module.
+    # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
+    set(_python_development_component Development.Module)
+
     find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
-      COMPONENTS Interpreter Development)
-  endif()
+      COMPONENTS Interpreter ${_python_development_component} REQUIRED)
+
+    # It's a little silly to detect Python a second time, but nanobind's cmake
+    # code looks for Python_ not Python3_.
+    find_package(Python ${LLVM_MINIMUM_PYTHON_VERSION}
+      COMPONENTS Interpreter ${_python_development_component} REQUIRED)
+
+    unset(_python_development_component)
+    message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
+    message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
+    message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
+    mlir_detect_pybind11_install()
+    find_package(pybind11 2.10 CONFIG REQUIRED)
+    message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
+    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                  "extension = '${PYTHON_MODULE_EXTENSION}")
 
-  # After CMake 3.18, we are able to limit the scope of the search to just
-  # Development.Module. Searching for Development will fail in situations where
-  # the Python libraries are not available. When possible, limit to just
-  # Development.Module.
-  # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
-  set(_python_development_component Development.Module)
-
-  find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
-    COMPONENTS Interpreter ${_python_development_component} REQUIRED)
-  unset(_python_development_component)
-  message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
-  message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
-  message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
-  mlir_detect_pybind11_install()
-  find_package(pybind11 2.10 CONFIG REQUIRED)
-  message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
-  message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                 "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                 "extension = '${PYTHON_MODULE_EXTENSION}")
+    mlir_detect_nanobind_install()
+    find_package(nanobind 2.2 CONFIG REQUIRED)
+    message(STATUS "Found nanobind v${nanobind_VERSION}: ${nanobind_INCLUDE_DIR}")
+    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                  "extension = '${PYTHON_MODULE_EXTENSION}")
+  endif()
 endmacro()
 
 # Detects a pybind11 package installed in the current python environment
@@ -56,3 +71,29 @@ function(mlir_detect_pybind11_install)
     set(pybind11_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
   endif()
 endfunction()
+
+
+# Detects a nanobind package installed in the current python environment
+# and sets variables to allow it to be found. This allows nanobind to be
+# installed via pip, which typically yields a much more recent version than
+# the OS install, which will be available otherwise.
+function(mlir_detect_nanobind_install)
+  if(nanobind_DIR)
+    message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
+  else()
+    message(STATUS "Checking for nanobind in python path...")
+    execute_process(
+      COMMAND "${Python3_EXECUTABLE}"
+      -c "import nanobind;print(nanobind.cmake_dir(), end='')"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      RESULT_VARIABLE STATUS
+      OUTPUT_VARIABLE PACKAGE_DIR
+      ERROR_QUIET)
+    if(NOT STATUS EQUAL "0")
+      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
+      return()
+    endif()
+    message(STATUS "found (${PACKAGE_DIR})")
+    set(nanobind_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+  endif()
+endfunction()
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
index 6e52c4deaad9a..a0bd1cac118ba 100644
--- a/mlir/docs/Bindings/Python.md
+++ b/mlir/docs/Bindings/Python.md
@@ -1138,12 +1138,14 @@ attributes and types must connect to the relevant C APIs for building and
 inspection, which must be provided first. Bindings for `Attribute` and `Type`
 subclasses can be defined using
 [`include/mlir/Bindings/Python/PybindAdaptors.h`](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Bindings/Python/PybindAdaptors.h)
-utilities that mimic pybind11 API for defining functions and properties. These
-bindings are to be included in a separate pybind11 module. The utilities also
-provide automatic casting between C API handles `MlirAttribute` and `MlirType`
-and their Python counterparts so that the C API handles can be used directly in
-binding implementations. The methods and properties provided by the bindings
-should follow the principles discussed above.
+or
+[`include/mlir/Bindings/Python/NanobindAdaptors.h`](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h)
+utilities that mimic pybind11/nanobind API for defining functions and
+properties. These bindings are to be included in a separate module. The
+utilities also provide automatic casting between C API handles `MlirAttribute`
+and `MlirType` and their Python counterparts so that the C API handles can be
+used directly in binding implementations. The methods and properties provided by
+the bindings should follow the principles discussed above.
 
 The attribute and type bindings for a dialect can be located in
 `lib/Bindings/Python/Dialect<Name>.cpp` and should be compiled into a separate
@@ -1179,7 +1181,9 @@ make the passes available along with the dialect.
 Dialect functionality other than IR objects or passes, such as helper functions,
 can be exposed to Python similarly to attributes and types. C API is expected to
 exist for this functionality, which can then be wrapped using pybind11 and
-`[include/mlir/Bindings/Python/PybindAdaptors.h](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Bindings/Python/PybindAdaptors.h)`
+`[include/mlir/Bindings/Python/PybindAdaptors.h](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Bindings/Python/PybindAdaptors.h)`,
+or nanobind and
+`[include/mlir/Bindings/Python/NanobindAdaptors.h](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h)`
 utilities to connect to the rest of Python API. The bindings can be located in a
-separate pybind11 module or in the same module as attributes and types, and
+separate module or in the same module as attributes and types, and
 loaded along with the dialect.
diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md
index 4f6cb1dec66a6..3168f5e13c751 100644
--- a/mlir/docs/DialectConversion.md
+++ b/mlir/docs/DialectConversion.md
@@ -230,17 +230,17 @@ The `TypeConverter` contains several hooks for detailing how to convert types,
 and how to materialize conversions between types in various situations. The two
 main aspects of the `TypeConverter` are conversion and materialization.
 
-A `conversion` describes how a given illegal source `Type` should be converted
-to N target types. If the source type is already "legal", it should convert to
-itself. Type conversions are specified via the `addConversion` method described
+A `conversion` describes how a given source `Type` should be converted to N
+target types. If the source type is converted to itself, we say it is a "legal"
+type. Type conversions are specified via the `addConversion` method described
 below.
 
-A `materialization` describes how a set of values should be converted to a
-single value of a desired type. An important distinction with a `conversion` is
-that a `materialization` can produce IR, whereas a `conversion` cannot. These
-materializations are used by the conversion framework to ensure type safety
-during the conversion process. There are several types of materializations
-depending on the situation.
+A `materialization` describes how a list of values should be converted to a
+list of values with specific types. An important distinction from a
+`conversion` is that a `materialization` can produce IR, whereas a `conversion`
+cannot. These materializations are used by the conversion framework to ensure
+type safety during the conversion process. There are several types of
+materializations depending on the situation.
 
 *   Argument Materialization
 
@@ -252,16 +252,15 @@ depending on the situation.
         conversion. (E.g., adaptors support only a single replacement value for
         each original value.) Therefore, an argument materialization is used to
         convert potentially multiple new block arguments back into a single SSA
-        value.
+        value. An argument materialization is also used when replacing an op
+        result with multiple values.
 
 *   Source Materialization
 
-    -   A source materialization converts from a value with a "legal" target
-        type, back to a specific source type. This is used when an operation is
-        "legal" during the conversion process, but contains a use of an illegal
-        type. This may happen during a conversion where some operations are
-        converted to those with different resultant types, but still retain
-        users of the original type system.
+    -   A source materialization is used when a value was replaced with a value
+        of a different type, but there are still users that expects the original
+        ("source") type at the end of the conversion process. A source
+        materialization converts the replacement value back to the source type.
     -   This materialization is used in the following situations:
         *   When a block argument has been converted to a different type, but
             the original argument still has users that will remain live after
@@ -275,16 +274,12 @@ depending on the situation.
 
 *   Target Materialization
 
-    -   A target materialization converts from a value with an "illegal" source
-        type, to a value of a "legal" type. This is used when a pattern expects
-        the remapped operands to be of a certain set of types, but the original
-        input operands have not been converted. This may happen during a
-        conversion where some operations are converted to those with different
-        resultant types, but still retain uses of the original type system.
-    -   This materialization is used in the following situations:
-        *   When the remapped operands of a
-            [conversion pattern](#conversion-patterns) are not legal for the
-            type conversion provided by the pattern.
+    -   A target materialization converts a value to the type that is expected
+        by a conversion pattern according to its type converter.
+    -   A target materialization is used when a pattern expects the remapped
+        operands to be of a certain set of types, but the original input
+        operands have either not been replaced or been replaced with values of
+        a different type.
 
 If a converted value is used by an operation that isn't converted, it needs a
 conversion back to the `source` type, hence source materialization; if an
@@ -297,10 +292,8 @@ will not implicitly change during the conversion process. When the type of a
 value definition, either block argument or operation result, is being changed,
 the users of that definition must also be updated during the conversion process.
 If they aren't, a type conversion must be materialized to ensure that a value of
-the expected type is still present within the IR. If a target materialization is
-required, but cannot be performed, the pattern application fails. If a source
-materialization is required, but cannot be performed, the entire conversion
-process fails.
+the expected type is still present within the IR. If a materialization is
+required, but cannot be performed, the entire conversion process fails.
 
 Several of the available hooks are detailed below:
 
@@ -362,9 +355,9 @@ class TypeConverter {
   }
 
   /// This method registers a materialization that will be called when
-  /// converting a legal replacement value back to an illegal source type.
-  /// This is used when some uses of the original, illegal value must persist
-  /// beyond the main conversion.
+  /// converting a replacement value back to its original source type.
+  /// This is used when some uses of the original value persist beyond the main
+  /// conversion.
   template <typename FnT,
             typename T = typename llvm::function_traits<FnT>::template arg_t<1>>
   void addSourceMaterialization(FnT &&callback) {
@@ -373,7 +366,22 @@ class TypeConverter {
   }
 
   /// This method registers a materialization that will be called when
-  /// converting an illegal (source) value to a legal (target) type.
+  /// converting a value to a target type according to a pattern's type
+  /// converter.
+  ///
+  /// Note: Target materializations can optionally inspect the "original"
+  /// type. This type may be different from the type of the input value.
+  /// For example, let's assume that a conversion pattern "P1" replaced an SSA
+  /// value "v1" (type "t1") with "v2" (type "t2"). Then a different conversion
+  /// pattern "P2" matches an op that has "v1" as an operand. Let's furthermore
+  /// assume that "P2" determines that the converted target type of "t1" is
+  /// "t3", which may be different from "t2". In this example, the target
+  /// materialization will be invoked with: outputType = "t3", inputs = "v2",
+  /// originalType = "t1". Note that the original type "t1" cannot be recovered
+  /// from just "t3" and "v2"; that's why the originalType parameter exists.
+  ///
+  /// Note: During a 1:N conversion, the result types can be a TypeRange. In
+  /// that case the materialization produces a SmallVector<Value>.
   template <typename FnT,
             typename T = typename llvm::function_traits<FnT>::template arg_t<1>>
   void addTargetMaterialization(FnT &&callback) {
diff --git a/mlir/examples/standalone/python/CMakeLists.txt b/mlir/examples/standalone/python/CMakeLists.txt
index a8c43827a5a37..69c82fd913579 100644
--- a/mlir/examples/standalone/python/CMakeLists.txt
+++ b/mlir/examples/standalone/python/CMakeLists.txt
@@ -17,18 +17,32 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir_standalone"
   TD_FILE dialects/StandaloneOps.td
   SOURCES
-    dialects/standalone.py
+    dialects/standalone_pybind11.py
+    dialects/standalone_nanobind.py
   DIALECT_NAME standalone)
 
-declare_mlir_python_extension(StandalonePythonSources.Extension
-  MODULE_NAME _standaloneDialects
+
+declare_mlir_python_extension(StandalonePythonSources.Pybind11Extension
+  MODULE_NAME _standaloneDialectsPybind11
+  ADD_TO_PARENT StandalonePythonSources
+  SOURCES
+    StandaloneExtensionPybind11.cpp
+  EMBED_CAPI_LINK_LIBS
+    StandaloneCAPI
+  PYTHON_BINDINGS_LIBRARY pybind11
+)
+
+declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
+  MODULE_NAME _standaloneDialectsNanobind
   ADD_TO_PARENT StandalonePythonSources
   SOURCES
-    StandaloneExtension.cpp
+    StandaloneExtensionNanobind.cpp
   EMBED_CAPI_LINK_LIBS
     StandaloneCAPI
+  PYTHON_BINDINGS_LIBRARY nanobind
 )
 
+
 ################################################################################
 # Common CAPI
 ################################################################################
diff --git a/mlir/examples/standalone/python/StandaloneExtensionNanobind.cpp b/mlir/examples/standalone/python/StandaloneExtensionNanobind.cpp
new file mode 100644
index 0000000000000..6d83dc585dcd1
--- /dev/null
+++ b/mlir/examples/standalone/python/StandaloneExtensionNanobind.cpp
@@ -0,0 +1,35 @@
+//===- StandaloneExtension.cpp - Extension module -------------------------===//
+//
+// This is the nanobind version of the example module. There is also a pybind11
+// example in StandaloneExtensionPybind11.cpp.
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <nanobind/nanobind.h>
+
+#include "Standalone-c/Dialects.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
+
+namespace nb = nanobind;
+
+NB_MODULE(_standaloneDialectsNanobind, m) {
+  //===--------------------------------------------------------------------===//
+  // standalone dialect
+  //===--------------------------------------------------------------------===//
+  auto standaloneM = m.def_submodule("standalone");
+
+  standaloneM.def(
+      "register_dialect",
+      [](MlirContext context, bool load) {
+        MlirDialectHandle handle = mlirGetDialectHandle__standalone__();
+        mlirDialectHandleRegisterDialect(handle, context);
+        if (load) {
+          mlirDialectHandleLoadDialect(handle, context);
+        }
+      },
+      nb::arg("context").none() = nb::none(), nb::arg("load") = true);
+}
diff --git a/mlir/examples/standalone/python/StandaloneExtension.cpp b/mlir/examples/standalone/python/StandaloneExtensionPybind11.cpp
similarity index 81%
rename from mlir/examples/standalone/python/StandaloneExtension.cpp
rename to mlir/examples/standalone/python/StandaloneExtensionPybind11.cpp
index 5e83060cd48d8..397db4c20e743 100644
--- a/mlir/examples/standalone/python/StandaloneExtension.cpp
+++ b/mlir/examples/standalone/python/StandaloneExtensionPybind11.cpp
@@ -1,4 +1,7 @@
-//===- StandaloneExtension.cpp - Extension module -------------------------===//
+//===- StandaloneExtensionPybind11.cpp - Extension module -----------------===//
+//
+// This is the pybind11 version of the example module. There is also a nanobind
+// example in StandaloneExtensionNanobind.cpp.
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,7 +14,7 @@
 
 using namespace mlir::python::adaptors;
 
-PYBIND11_MODULE(_standaloneDialects, m) {
+PYBIND11_MODULE(_standaloneDialectsPybind11, m) {
   //===--------------------------------------------------------------------===//
   // standalone dialect
   //===--------------------------------------------------------------------===//
diff --git a/mlir/examples/standalone/python/mlir_standalone/dialects/standalone.py b/mlir/examples/standalone/python/mlir_standalone/dialects/standalone_nanobind.py
similarity index 78%
rename from mlir/examples/standalone/python/mlir_standalone/dialects/standalone.py
rename to mlir/examples/standalone/python/mlir_standalone/dialects/standalone_nanobind.py
index c958b2ac19368..6218720951c82 100644
--- a/mlir/examples/standalone/python/mlir_standalone/dialects/standalone.py
+++ b/mlir/examples/standalone/python/mlir_standalone/dialects/standalone_nanobind.py
@@ -3,4 +3,4 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._standalone_ops_gen import *
-from .._mlir_libs._standaloneDialects.standalone import *
+from .._mlir_libs._standaloneDialectsNanobind.standalone import *
diff --git a/mlir/examples/standalone/python/mlir_standalone/dialects/standalone_pybind11.py b/mlir/examples/standalone/python/mlir_standalone/dialects/standalone_pybind11.py
new file mode 100644
index 0000000000000..bfb98e404e13f
--- /dev/null
+++ b/mlir/examples/standalone/python/mlir_standalone/dialects/standalone_pybind11.py
@@ -0,0 +1,6 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from ._standalone_ops_gen import *
+from .._mlir_libs._standaloneDialectsPybind11.standalone import *
diff --git a/mlir/examples/standalone/test/python/smoketest.py b/mlir/examples/standalone/test/python/smoketest.py
index 08e08cbd2fe24..bd40c65d16164 100644
--- a/mlir/examples/standalone/test/python/smoketest.py
+++ b/mlir/examples/standalone/test/python/smoketest.py
@@ -1,7 +1,17 @@
-# RUN: %python %s | FileCheck %s
+# RUN: %python %s pybind11 | FileCheck %s
+# RUN: %python %s nanobind | FileCheck %s
 
+import sys
 from mlir_standalone.ir import *
-from mlir_standalone.dialects import builtin as builtin_d, standalone as standalone_d
+from mlir_standalone.dialects import builtin as builtin_d
+
+if sys.argv[1] == "pybind11":
+    from mlir_standalone.dialects import standalone_pybind11 as standalone_d
+elif sys.argv[1] == "nanobind":
+    from mlir_standalone.dialects import standalone_nanobind as standalone_d
+else:
+    raise ValueError("Expected either pybind11 or nanobind as arguments")
+
 
 with Context():
     standalone_d.register_dialect()
diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h
index 0e6434073437a..ed9b23c343150 100644
--- a/mlir/include/mlir-c/Dialect/LLVM.h
+++ b/mlir/include/mlir-c/Dialect/LLVM.h
@@ -175,17 +175,17 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMComdatAttrGet(MlirContext ctx,
                                                        MlirLLVMComdat comdat);
 
 enum MlirLLVMLinkage {
-  MlirLLVMLinkagePrivate = 0,
-  MlirLLVMLinkageInternal = 1,
-  MlirLLVMLinkageAvailableExternally = 2,
-  MlirLLVMLinkageLinkonce = 3,
+  MlirLLVMLinkageExternal = 0,
+  MlirLLVMLinkageAvailableExternally = 1,
+  MlirLLVMLinkageLinkonce = 2,
+  MlirLLVMLinkageLinkonceODR = 3,
   MlirLLVMLinkageWeak = 4,
-  MlirLLVMLinkageCommon = 5,
+  MlirLLVMLinkageWeakODR = 5,
   MlirLLVMLinkageAppending = 6,
-  MlirLLVMLinkageExternWeak = 7,
-  MlirLLVMLinkageLinkonceODR = 8,
-  MlirLLVMLinkageWeakODR = 9,
-  MlirLLVMLinkageExternal = 10,
+  MlirLLVMLinkageInternal = 7,
+  MlirLLVMLinkagePrivate = 8,
+  MlirLLVMLinkageExternWeak = 9,
+  MlirLLVMLinkageCommon = 10,
 };
 typedef enum MlirLLVMLinkage MlirLLVMLinkage;
 
diff --git a/mlir/include/mlir-c/Pass.h b/mlir/include/mlir-c/Pass.h
index 6019071cfdaa2..8fd8e9956a65a 100644
--- a/mlir/include/mlir-c/Pass.h
+++ b/mlir/include/mlir-c/Pass.h
@@ -81,7 +81,8 @@ mlirPassManagerRunOnOp(MlirPassManager passManager, MlirOperation op);
 MLIR_CAPI_EXPORTED void mlirPassManagerEnableIRPrinting(
     MlirPassManager passManager, bool printBeforeAll, bool printAfterAll,
     bool printModuleScope, bool printAfterOnlyOnChange,
-    bool printAfterOnlyOnFailure, MlirStringRef treePrintingPath);
+    bool printAfterOnlyOnFailure, MlirOpPrintingFlags flags,
+    MlirStringRef treePrintingPath);
 
 /// Enable / disable verify-each.
 MLIR_CAPI_EXPORTED void
diff --git a/mlir/include/mlir/Bindings/Python/Diagnostics.h b/mlir/include/mlir/Bindings/Python/Diagnostics.h
new file mode 100644
index 0000000000000..ea80e14dde0f3
--- /dev/null
+++ b/mlir/include/mlir/Bindings/Python/Diagnostics.h
@@ -0,0 +1,59 @@
+//===- Diagnostics.h - Helpers for diagnostics in Python bindings ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_BINDINGS_PYTHON_DIAGNOSTICS_H
+#define MLIR_BINDINGS_PYTHON_DIAGNOSTICS_H
+
+#include <cassert>
+#include <string>
+
+#include "mlir-c/Diagnostics.h"
+#include "mlir-c/IR.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace mlir {
+namespace python {
+
+/// RAII scope intercepting all diagnostics into a string. The message must be
+/// checked before this goes out of scope.
+class CollectDiagnosticsToStringScope {
+public:
+  explicit CollectDiagnosticsToStringScope(MlirContext ctx) : context(ctx) {
+    handlerID = mlirContextAttachDiagnosticHandler(ctx, &handler, &errorMessage,
+                                                   /*deleteUserData=*/nullptr);
+  }
+  ~CollectDiagnosticsToStringScope() {
+    assert(errorMessage.empty() && "unchecked error message");
+    mlirContextDetachDiagnosticHandler(context, handlerID);
+  }
+
+  [[nodiscard]] std::string takeMessage() { return std::move(errorMessage); }
+
+private:
+  static MlirLogicalResult handler(MlirDiagnostic diag, void *data) {
+    auto printer = +[](MlirStringRef message, void *data) {
+      *static_cast<std::string *>(data) +=
+          llvm::StringRef(message.data, message.length);
+    };
+    MlirLocation loc = mlirDiagnosticGetLocation(diag);
+    *static_cast<std::string *>(data) += "at ";
+    mlirLocationPrint(loc, printer, data);
+    *static_cast<std::string *>(data) += ": ";
+    mlirDiagnosticPrint(diag, printer, data);
+    return mlirLogicalResultSuccess();
+  }
+
+  MlirContext context;
+  MlirDiagnosticHandlerID handlerID;
+  std::string errorMessage = "";
+};
+
+} // namespace python
+} // namespace mlir
+
+#endif // MLIR_BINDINGS_PYTHON_DIAGNOSTICS_H
diff --git a/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h b/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h
new file mode 100644
index 0000000000000..5e01cebcb09c9
--- /dev/null
+++ b/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h
@@ -0,0 +1,671 @@
+//===- NanobindAdaptors.h - Interop with MLIR APIs via nanobind -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file contains adaptors for clients of the core MLIR Python APIs to
+// interop via MLIR CAPI types, using nanobind. The facilities here do not
+// depend on implementation details of the MLIR Python API and do not introduce
+// C++-level dependencies with it (requiring only Python and CAPI-level
+// dependencies).
+//
+// It is encouraged to be used both in-tree and out-of-tree. For in-tree use
+// cases, it should be used for dialect implementations (versus relying on
+// Pybind-based internals of the core libraries).
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_BINDINGS_PYTHON_NANOBINDADAPTORS_H
+#define MLIR_BINDINGS_PYTHON_NANOBINDADAPTORS_H
+
+#include <nanobind/nanobind.h>
+#include <nanobind/stl/string.h>
+
+#include <cstdint>
+
+#include "mlir-c/Bindings/Python/Interop.h"
+#include "mlir-c/Diagnostics.h"
+#include "mlir-c/IR.h"
+#include "llvm/ADT/Twine.h"
+
+// Raw CAPI type casters need to be declared before use, so always include them
+// first.
+namespace nanobind {
+namespace detail {
+
+/// Helper to convert a presumed MLIR API object to a capsule, accepting either
+/// an explicit Capsule (which can happen when two C APIs are communicating
+/// directly via Python) or indirectly by querying the MLIR_PYTHON_CAPI_PTR_ATTR
+/// attribute (through which supported MLIR Python API objects export their
+/// contained API pointer as a capsule). Throws a type error if the object is
+/// neither. This is intended to be used from type casters, which are invoked
+/// with a raw handle (unowned). The returned object's lifetime may not extend
+/// beyond the apiObject handle without explicitly having its refcount increased
+/// (i.e. on return).
+static nanobind::object mlirApiObjectToCapsule(nanobind::handle apiObject) {
+  if (PyCapsule_CheckExact(apiObject.ptr()))
+    return nanobind::borrow<nanobind::object>(apiObject);
+  if (!nanobind::hasattr(apiObject, MLIR_PYTHON_CAPI_PTR_ATTR)) {
+    std::string repr = nanobind::cast<std::string>(nanobind::repr(apiObject));
+    throw nanobind::type_error(
+        (llvm::Twine("Expected an MLIR object (got ") + repr + ").")
+            .str()
+            .c_str());
+  }
+  return apiObject.attr(MLIR_PYTHON_CAPI_PTR_ATTR);
+}
+
+// Note: Currently all of the following support cast from nanobind::object to
+// the Mlir* C-API type, but only a few light-weight, context-bound ones
+// implicitly cast the other way because the use case has not yet emerged and
+// ownership is unclear.
+
+/// Casts object <-> MlirAffineMap.
+template <>
+struct type_caster<MlirAffineMap> {
+  NB_TYPE_CASTER(MlirAffineMap, const_name("MlirAffineMap"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToAffineMap(capsule.ptr());
+    if (mlirAffineMapIsNull(value)) {
+      return false;
+    }
+    return !mlirAffineMapIsNull(value);
+  }
+  static handle from_cpp(MlirAffineMap v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonAffineMapToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("AffineMap")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  }
+};
+
+/// Casts object <-> MlirAttribute.
+template <>
+struct type_caster<MlirAttribute> {
+  NB_TYPE_CASTER(MlirAttribute, const_name("MlirAttribute"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToAttribute(capsule.ptr());
+    return !mlirAttributeIsNull(value);
+  }
+  static handle from_cpp(MlirAttribute v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonAttributeToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Attribute")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .attr(MLIR_PYTHON_MAYBE_DOWNCAST_ATTR)()
+        .release();
+  }
+};
+
+/// Casts object -> MlirBlock.
+template <>
+struct type_caster<MlirBlock> {
+  NB_TYPE_CASTER(MlirBlock, const_name("MlirBlock"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToBlock(capsule.ptr());
+    return !mlirBlockIsNull(value);
+  }
+};
+
+/// Casts object -> MlirContext.
+template <>
+struct type_caster<MlirContext> {
+  NB_TYPE_CASTER(MlirContext, const_name("MlirContext"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    if (src.is_none()) {
+      // Gets the current thread-bound context.
+      // TODO: This raises an error of "No current context" currently.
+      // Update the implementation to pretty-print the helpful error that the
+      // core implementations print in this case.
+      src = nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                .attr("Context")
+                .attr("current");
+    }
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToContext(capsule.ptr());
+    return !mlirContextIsNull(value);
+  }
+};
+
+/// Casts object <-> MlirDialectRegistry.
+template <>
+struct type_caster<MlirDialectRegistry> {
+  NB_TYPE_CASTER(MlirDialectRegistry, const_name("MlirDialectRegistry"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToDialectRegistry(capsule.ptr());
+    return !mlirDialectRegistryIsNull(value);
+  }
+  static handle from_cpp(MlirDialectRegistry v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule = nanobind::steal<nanobind::object>(
+        mlirPythonDialectRegistryToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("DialectRegistry")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  }
+};
+
+/// Casts object <-> MlirLocation.
+template <>
+struct type_caster<MlirLocation> {
+  NB_TYPE_CASTER(MlirLocation, const_name("MlirLocation"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    if (src.is_none()) {
+      // Gets the current thread-bound context.
+      src = nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                .attr("Location")
+                .attr("current");
+    }
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToLocation(capsule.ptr());
+    return !mlirLocationIsNull(value);
+  }
+  static handle from_cpp(MlirLocation v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonLocationToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Location")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  }
+};
+
+/// Casts object <-> MlirModule.
+template <>
+struct type_caster<MlirModule> {
+  NB_TYPE_CASTER(MlirModule, const_name("MlirModule"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToModule(capsule.ptr());
+    return !mlirModuleIsNull(value);
+  }
+  static handle from_cpp(MlirModule v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonModuleToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Module")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  };
+};
+
+/// Casts object <-> MlirFrozenRewritePatternSet.
+template <>
+struct type_caster<MlirFrozenRewritePatternSet> {
+  NB_TYPE_CASTER(MlirFrozenRewritePatternSet,
+                 const_name("MlirFrozenRewritePatternSet"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToFrozenRewritePatternSet(capsule.ptr());
+    return value.ptr != nullptr;
+  }
+  static handle from_cpp(MlirFrozenRewritePatternSet v, rv_policy, handle) {
+    nanobind::object capsule = nanobind::steal<nanobind::object>(
+        mlirPythonFrozenRewritePatternSetToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("rewrite"))
+        .attr("FrozenRewritePatternSet")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  };
+};
+
+/// Casts object <-> MlirOperation.
+template <>
+struct type_caster<MlirOperation> {
+  NB_TYPE_CASTER(MlirOperation, const_name("MlirOperation"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToOperation(capsule.ptr());
+    return !mlirOperationIsNull(value);
+  }
+  static handle from_cpp(MlirOperation v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    if (v.ptr == nullptr)
+      return nanobind::none();
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonOperationToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Operation")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  };
+};
+
+/// Casts object <-> MlirValue.
+template <>
+struct type_caster<MlirValue> {
+  NB_TYPE_CASTER(MlirValue, const_name("MlirValue"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToValue(capsule.ptr());
+    return !mlirValueIsNull(value);
+  }
+  static handle from_cpp(MlirValue v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    if (v.ptr == nullptr)
+      return nanobind::none();
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonValueToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Value")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .attr(MLIR_PYTHON_MAYBE_DOWNCAST_ATTR)()
+        .release();
+  };
+};
+
+/// Casts object -> MlirPassManager.
+template <>
+struct type_caster<MlirPassManager> {
+  NB_TYPE_CASTER(MlirPassManager, const_name("MlirPassManager"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToPassManager(capsule.ptr());
+    return !mlirPassManagerIsNull(value);
+  }
+};
+
+/// Casts object <-> MlirTypeID.
+template <>
+struct type_caster<MlirTypeID> {
+  NB_TYPE_CASTER(MlirTypeID, const_name("MlirTypeID"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToTypeID(capsule.ptr());
+    return !mlirTypeIDIsNull(value);
+  }
+  static handle from_cpp(MlirTypeID v, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    if (v.ptr == nullptr)
+      return nanobind::none();
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonTypeIDToCapsule(v));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("TypeID")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .release();
+  };
+};
+
+/// Casts object <-> MlirType.
+template <>
+struct type_caster<MlirType> {
+  NB_TYPE_CASTER(MlirType, const_name("MlirType"));
+  bool from_python(handle src, uint8_t flags, cleanup_list *cleanup) {
+    nanobind::object capsule = mlirApiObjectToCapsule(src);
+    value = mlirPythonCapsuleToType(capsule.ptr());
+    return !mlirTypeIsNull(value);
+  }
+  static handle from_cpp(MlirType t, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    nanobind::object capsule =
+        nanobind::steal<nanobind::object>(mlirPythonTypeToCapsule(t));
+    return nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+        .attr("Type")
+        .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
+        .attr(MLIR_PYTHON_MAYBE_DOWNCAST_ATTR)()
+        .release();
+  }
+};
+
+} // namespace detail
+} // namespace nanobind
+
+namespace mlir {
+namespace python {
+namespace nanobind_adaptors {
+
+/// Provides a facility like nanobind::class_ for defining a new class in a
+/// scope, but this allows extension of an arbitrary Python class, defining
+/// methods on it is a similar way. Classes defined in this way are very similar
+/// to if defined in Python in the usual way but use nanobind machinery to
+/// do it. These are not "real" nanobind classes but pure Python classes
+/// with no relation to a concrete C++ class.
+///
+/// Derived from a discussion upstream:
+///   https://github.com/pybind/pybind11/issues/1193
+///   (plus a fair amount of extra curricular poking)
+///   TODO: If this proves useful, see about including it in nanobind.
+class pure_subclass {
+public:
+  pure_subclass(nanobind::handle scope, const char *derivedClassName,
+                const nanobind::object &superClass) {
+    nanobind::object pyType =
+        nanobind::borrow<nanobind::object>((PyObject *)&PyType_Type);
+    nanobind::object metaclass = pyType(superClass);
+    nanobind::dict attributes;
+
+    thisClass = metaclass(derivedClassName, nanobind::make_tuple(superClass),
+                          attributes);
+    scope.attr(derivedClassName) = thisClass;
+  }
+
+  template <typename Func, typename... Extra>
+  pure_subclass &def(const char *name, Func &&f, const Extra &...extra) {
+    nanobind::object cf = nanobind::cpp_function(
+        std::forward<Func>(f), nanobind::name(name), nanobind::is_method(),
+        nanobind::scope(thisClass), extra...);
+    thisClass.attr(name) = cf;
+    return *this;
+  }
+
+  template <typename Func, typename... Extra>
+  pure_subclass &def_property_readonly(const char *name, Func &&f,
+                                       const Extra &...extra) {
+    nanobind::object cf = nanobind::cpp_function(
+        std::forward<Func>(f), nanobind::name(name), nanobind::is_method(),
+        nanobind::scope(thisClass), extra...);
+    auto builtinProperty =
+        nanobind::borrow<nanobind::object>((PyObject *)&PyProperty_Type);
+    thisClass.attr(name) = builtinProperty(cf);
+    return *this;
+  }
+
+  template <typename Func, typename... Extra>
+  pure_subclass &def_staticmethod(const char *name, Func &&f,
+                                  const Extra &...extra) {
+    static_assert(!std::is_member_function_pointer<Func>::value,
+                  "def_staticmethod(...) called with a non-static member "
+                  "function pointer");
+    nanobind::object cf = nanobind::cpp_function(
+        std::forward<Func>(f),
+        nanobind::name(name), // nanobind::scope(thisClass),
+        extra...);
+    thisClass.attr(name) = cf;
+    return *this;
+  }
+
+  template <typename Func, typename... Extra>
+  pure_subclass &def_classmethod(const char *name, Func &&f,
+                                 const Extra &...extra) {
+    static_assert(!std::is_member_function_pointer<Func>::value,
+                  "def_classmethod(...) called with a non-static member "
+                  "function pointer");
+    nanobind::object cf = nanobind::cpp_function(
+        std::forward<Func>(f),
+        nanobind::name(name), // nanobind::scope(thisClass),
+        extra...);
+    thisClass.attr(name) =
+        nanobind::borrow<nanobind::object>(PyClassMethod_New(cf.ptr()));
+    return *this;
+  }
+
+  nanobind::object get_class() const { return thisClass; }
+
+protected:
+  nanobind::object superClass;
+  nanobind::object thisClass;
+};
+
+/// Creates a custom subclass of mlir.ir.Attribute, implementing a casting
+/// constructor and type checking methods.
+class mlir_attribute_subclass : public pure_subclass {
+public:
+  using IsAFunctionTy = bool (*)(MlirAttribute);
+  using GetTypeIDFunctionTy = MlirTypeID (*)();
+
+  /// Subclasses by looking up the super-class dynamically.
+  mlir_attribute_subclass(nanobind::handle scope, const char *attrClassName,
+                          IsAFunctionTy isaFunction,
+                          GetTypeIDFunctionTy getTypeIDFunction = nullptr)
+      : mlir_attribute_subclass(
+            scope, attrClassName, isaFunction,
+            nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                .attr("Attribute"),
+            getTypeIDFunction) {}
+
+  /// Subclasses with a provided mlir.ir.Attribute super-class. This must
+  /// be used if the subclass is being defined in the same extension module
+  /// as the mlir.ir class (otherwise, it will trigger a recursive
+  /// initialization).
+  mlir_attribute_subclass(nanobind::handle scope, const char *typeClassName,
+                          IsAFunctionTy isaFunction,
+                          const nanobind::object &superCls,
+                          GetTypeIDFunctionTy getTypeIDFunction = nullptr)
+      : pure_subclass(scope, typeClassName, superCls) {
+    // Casting constructor. Note that it hard, if not impossible, to properly
+    // call chain to parent `__init__` in nanobind due to its special handling
+    // for init functions that don't have a fully constructed self-reference,
+    // which makes it impossible to forward it to `__init__` of a superclass.
+    // Instead, provide a custom `__new__` and call that of a superclass, which
+    // eventually calls `__init__` of the superclass. Since attribute subclasses
+    // have no additional members, we can just return the instance thus created
+    // without amending it.
+    std::string captureTypeName(
+        typeClassName); // As string in case if typeClassName is not static.
+    nanobind::object newCf = nanobind::cpp_function(
+        [superCls, isaFunction, captureTypeName](
+            nanobind::object cls, nanobind::object otherAttribute) {
+          MlirAttribute rawAttribute =
+              nanobind::cast<MlirAttribute>(otherAttribute);
+          if (!isaFunction(rawAttribute)) {
+            auto origRepr =
+                nanobind::cast<std::string>(nanobind::repr(otherAttribute));
+            throw std::invalid_argument(
+                (llvm::Twine("Cannot cast attribute to ") + captureTypeName +
+                 " (from " + origRepr + ")")
+                    .str());
+          }
+          nanobind::object self = superCls.attr("__new__")(cls, otherAttribute);
+          return self;
+        },
+        nanobind::name("__new__"), nanobind::arg("cls"),
+        nanobind::arg("cast_from_attr"));
+    thisClass.attr("__new__") = newCf;
+
+    // 'isinstance' method.
+    def_staticmethod(
+        "isinstance",
+        [isaFunction](MlirAttribute other) { return isaFunction(other); },
+        nanobind::arg("other_attribute"));
+    def("__repr__", [superCls, captureTypeName](nanobind::object self) {
+      return nanobind::repr(superCls(self))
+          .attr("replace")(superCls.attr("__name__"), captureTypeName);
+    });
+    if (getTypeIDFunction) {
+      def_staticmethod("get_static_typeid",
+                       [getTypeIDFunction]() { return getTypeIDFunction(); });
+      nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+          .attr(MLIR_PYTHON_CAPI_TYPE_CASTER_REGISTER_ATTR)(
+              getTypeIDFunction())(nanobind::cpp_function(
+              [thisClass = thisClass](const nanobind::object &mlirAttribute) {
+                return thisClass(mlirAttribute);
+              }));
+    }
+  }
+};
+
+/// Creates a custom subclass of mlir.ir.Type, implementing a casting
+/// constructor and type checking methods.
+class mlir_type_subclass : public pure_subclass {
+public:
+  using IsAFunctionTy = bool (*)(MlirType);
+  using GetTypeIDFunctionTy = MlirTypeID (*)();
+
+  /// Subclasses by looking up the super-class dynamically.
+  mlir_type_subclass(nanobind::handle scope, const char *typeClassName,
+                     IsAFunctionTy isaFunction,
+                     GetTypeIDFunctionTy getTypeIDFunction = nullptr)
+      : mlir_type_subclass(
+            scope, typeClassName, isaFunction,
+            nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                .attr("Type"),
+            getTypeIDFunction) {}
+
+  /// Subclasses with a provided mlir.ir.Type super-class. This must
+  /// be used if the subclass is being defined in the same extension module
+  /// as the mlir.ir class (otherwise, it will trigger a recursive
+  /// initialization).
+  mlir_type_subclass(nanobind::handle scope, const char *typeClassName,
+                     IsAFunctionTy isaFunction,
+                     const nanobind::object &superCls,
+                     GetTypeIDFunctionTy getTypeIDFunction = nullptr)
+      : pure_subclass(scope, typeClassName, superCls) {
+    // Casting constructor. Note that it hard, if not impossible, to properly
+    // call chain to parent `__init__` in nanobind due to its special handling
+    // for init functions that don't have a fully constructed self-reference,
+    // which makes it impossible to forward it to `__init__` of a superclass.
+    // Instead, provide a custom `__new__` and call that of a superclass, which
+    // eventually calls `__init__` of the superclass. Since attribute subclasses
+    // have no additional members, we can just return the instance thus created
+    // without amending it.
+    std::string captureTypeName(
+        typeClassName); // As string in case if typeClassName is not static.
+    nanobind::object newCf = nanobind::cpp_function(
+        [superCls, isaFunction, captureTypeName](nanobind::object cls,
+                                                 nanobind::object otherType) {
+          MlirType rawType = nanobind::cast<MlirType>(otherType);
+          if (!isaFunction(rawType)) {
+            auto origRepr =
+                nanobind::cast<std::string>(nanobind::repr(otherType));
+            throw std::invalid_argument((llvm::Twine("Cannot cast type to ") +
+                                         captureTypeName + " (from " +
+                                         origRepr + ")")
+                                            .str());
+          }
+          nanobind::object self = superCls.attr("__new__")(cls, otherType);
+          return self;
+        },
+        nanobind::name("__new__"), nanobind::arg("cls"),
+        nanobind::arg("cast_from_type"));
+    thisClass.attr("__new__") = newCf;
+
+    // 'isinstance' method.
+    def_staticmethod(
+        "isinstance",
+        [isaFunction](MlirType other) { return isaFunction(other); },
+        nanobind::arg("other_type"));
+    def("__repr__", [superCls, captureTypeName](nanobind::object self) {
+      return nanobind::repr(superCls(self))
+          .attr("replace")(superCls.attr("__name__"), captureTypeName);
+    });
+    if (getTypeIDFunction) {
+      // 'get_static_typeid' method.
+      // This is modeled as a static method instead of a static property because
+      // `def_property_readonly_static` is not available in `pure_subclass` and
+      // we do not want to introduce the complexity that pybind uses to
+      // implement it.
+      def_staticmethod("get_static_typeid",
+                       [getTypeIDFunction]() { return getTypeIDFunction(); });
+      nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+          .attr(MLIR_PYTHON_CAPI_TYPE_CASTER_REGISTER_ATTR)(
+              getTypeIDFunction())(nanobind::cpp_function(
+              [thisClass = thisClass](const nanobind::object &mlirType) {
+                return thisClass(mlirType);
+              }));
+    }
+  }
+};
+
+/// Creates a custom subclass of mlir.ir.Value, implementing a casting
+/// constructor and type checking methods.
+class mlir_value_subclass : public pure_subclass {
+public:
+  using IsAFunctionTy = bool (*)(MlirValue);
+
+  /// Subclasses by looking up the super-class dynamically.
+  mlir_value_subclass(nanobind::handle scope, const char *valueClassName,
+                      IsAFunctionTy isaFunction)
+      : mlir_value_subclass(
+            scope, valueClassName, isaFunction,
+            nanobind::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                .attr("Value")) {}
+
+  /// Subclasses with a provided mlir.ir.Value super-class. This must
+  /// be used if the subclass is being defined in the same extension module
+  /// as the mlir.ir class (otherwise, it will trigger a recursive
+  /// initialization).
+  mlir_value_subclass(nanobind::handle scope, const char *valueClassName,
+                      IsAFunctionTy isaFunction,
+                      const nanobind::object &superCls)
+      : pure_subclass(scope, valueClassName, superCls) {
+    // Casting constructor. Note that it hard, if not impossible, to properly
+    // call chain to parent `__init__` in nanobind due to its special handling
+    // for init functions that don't have a fully constructed self-reference,
+    // which makes it impossible to forward it to `__init__` of a superclass.
+    // Instead, provide a custom `__new__` and call that of a superclass, which
+    // eventually calls `__init__` of the superclass. Since attribute subclasses
+    // have no additional members, we can just return the instance thus created
+    // without amending it.
+    std::string captureValueName(
+        valueClassName); // As string in case if valueClassName is not static.
+    nanobind::object newCf = nanobind::cpp_function(
+        [superCls, isaFunction, captureValueName](nanobind::object cls,
+                                                  nanobind::object otherValue) {
+          MlirValue rawValue = nanobind::cast<MlirValue>(otherValue);
+          if (!isaFunction(rawValue)) {
+            auto origRepr =
+                nanobind::cast<std::string>(nanobind::repr(otherValue));
+            throw std::invalid_argument((llvm::Twine("Cannot cast value to ") +
+                                         captureValueName + " (from " +
+                                         origRepr + ")")
+                                            .str());
+          }
+          nanobind::object self = superCls.attr("__new__")(cls, otherValue);
+          return self;
+        },
+        nanobind::name("__new__"), nanobind::arg("cls"),
+        nanobind::arg("cast_from_value"));
+    thisClass.attr("__new__") = newCf;
+
+    // 'isinstance' method.
+    def_staticmethod(
+        "isinstance",
+        [isaFunction](MlirValue other) { return isaFunction(other); },
+        nanobind::arg("other_value"));
+  }
+};
+
+} // namespace nanobind_adaptors
+
+/// RAII scope intercepting all diagnostics into a string. The message must be
+/// checked before this goes out of scope.
+class CollectDiagnosticsToStringScope {
+public:
+  explicit CollectDiagnosticsToStringScope(MlirContext ctx) : context(ctx) {
+    handlerID = mlirContextAttachDiagnosticHandler(ctx, &handler, &errorMessage,
+                                                   /*deleteUserData=*/nullptr);
+  }
+  ~CollectDiagnosticsToStringScope() {
+    assert(errorMessage.empty() && "unchecked error message");
+    mlirContextDetachDiagnosticHandler(context, handlerID);
+  }
+
+  [[nodiscard]] std::string takeMessage() { return std::move(errorMessage); }
+
+private:
+  static MlirLogicalResult handler(MlirDiagnostic diag, void *data) {
+    auto printer = +[](MlirStringRef message, void *data) {
+      *static_cast<std::string *>(data) +=
+          llvm::StringRef(message.data, message.length);
+    };
+    MlirLocation loc = mlirDiagnosticGetLocation(diag);
+    *static_cast<std::string *>(data) += "at ";
+    mlirLocationPrint(loc, printer, data);
+    *static_cast<std::string *>(data) += ": ";
+    mlirDiagnosticPrint(diag, printer, data);
+    return mlirLogicalResultSuccess();
+  }
+
+  MlirContext context;
+  MlirDiagnosticHandlerID handlerID;
+  std::string errorMessage = "";
+};
+
+} // namespace python
+} // namespace mlir
+
+#endif // MLIR_BINDINGS_PYTHON_NANOBINDADAPTORS_H
diff --git a/mlir/include/mlir/Bindings/Python/PybindAdaptors.h b/mlir/include/mlir/Bindings/Python/PybindAdaptors.h
index df4b9bf713592..c8233355d1d67 100644
--- a/mlir/include/mlir/Bindings/Python/PybindAdaptors.h
+++ b/mlir/include/mlir/Bindings/Python/PybindAdaptors.h
@@ -1,4 +1,4 @@
-//===- PybindAdaptors.h - Adaptors for interop with MLIR APIs -------------===//
+//===- PybindAdaptors.h - Interop with MLIR APIs via pybind11 -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 // This file contains adaptors for clients of the core MLIR Python APIs to
-// interop via MLIR CAPI types. The facilities here do not depend on
-// implementation details of the MLIR Python API and do not introduce C++-level
-// dependencies with it (requiring only Python and CAPI-level dependencies).
+// interop via MLIR CAPI types, using pybind11. The facilities here do not
+// depend on implementation details of the MLIR Python API and do not introduce
+// C++-level dependencies with it (requiring only Python and CAPI-level
+// dependencies).
 //
 // It is encouraged to be used both in-tree and out-of-tree. For in-tree use
 // cases, it should be used for dialect implementations (versus relying on
@@ -611,40 +612,6 @@ class mlir_value_subclass : public pure_subclass {
 
 } // namespace adaptors
 
-/// RAII scope intercepting all diagnostics into a string. The message must be
-/// checked before this goes out of scope.
-class CollectDiagnosticsToStringScope {
-public:
-  explicit CollectDiagnosticsToStringScope(MlirContext ctx) : context(ctx) {
-    handlerID = mlirContextAttachDiagnosticHandler(ctx, &handler, &errorMessage,
-                                                   /*deleteUserData=*/nullptr);
-  }
-  ~CollectDiagnosticsToStringScope() {
-    assert(errorMessage.empty() && "unchecked error message");
-    mlirContextDetachDiagnosticHandler(context, handlerID);
-  }
-
-  [[nodiscard]] std::string takeMessage() { return std::move(errorMessage); }
-
-private:
-  static MlirLogicalResult handler(MlirDiagnostic diag, void *data) {
-    auto printer = +[](MlirStringRef message, void *data) {
-      *static_cast<std::string *>(data) +=
-          llvm::StringRef(message.data, message.length);
-    };
-    MlirLocation loc = mlirDiagnosticGetLocation(diag);
-    *static_cast<std::string *>(data) += "at ";
-    mlirLocationPrint(loc, printer, data);
-    *static_cast<std::string *>(data) += ": ";
-    mlirDiagnosticPrint(diag, printer, data);
-    return mlirLogicalResultSuccess();
-  }
-
-  MlirContext context;
-  MlirDiagnosticHandlerID handlerID;
-  std::string errorMessage = "";
-};
-
 } // namespace python
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
index da067410db5ef..7ffc861331760 100644
--- a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
+++ b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h
@@ -51,7 +51,7 @@ getLLVMDefaultFPExceptionBehavior(MLIRContext &context);
 template <typename SourceOp, typename TargetOp>
 class AttrConvertFastMathToLLVM {
 public:
-  explicit AttrConvertFastMathToLLVM(SourceOp srcOp) {
+  AttrConvertFastMathToLLVM(SourceOp srcOp) {
     // Copy the source attributes.
     convertedAttr = NamedAttrList{srcOp->getAttrs()};
     // Get the name of the arith fastmath attribute.
@@ -81,7 +81,7 @@ class AttrConvertFastMathToLLVM {
 template <typename SourceOp, typename TargetOp>
 class AttrConvertOverflowToLLVM {
 public:
-  explicit AttrConvertOverflowToLLVM(SourceOp srcOp) {
+  AttrConvertOverflowToLLVM(SourceOp srcOp) {
     // Copy the source attributes.
     convertedAttr = NamedAttrList{srcOp->getAttrs()};
     // Get the name of the arith overflow attribute.
@@ -109,7 +109,7 @@ class AttrConverterConstrainedFPToLLVM {
                 "LLVM::FPExceptionBehaviorOpInterface");
 
 public:
-  explicit AttrConverterConstrainedFPToLLVM(SourceOp srcOp) {
+  AttrConverterConstrainedFPToLLVM(SourceOp srcOp) {
     // Copy the source attributes.
     convertedAttr = NamedAttrList{srcOp->getAttrs()};
 
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index f3bf5b66398e0..86ea87b55af1c 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -143,6 +143,8 @@ template <typename SourceOp>
 class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
 public:
   using OpAdaptor = typename SourceOp::Adaptor;
+  using OneToNOpAdaptor =
+      typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>>;
 
   explicit ConvertOpToLLVMPattern(const LLVMTypeConverter &typeConverter,
                                   PatternBenefit benefit = 1)
@@ -153,8 +155,13 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   /// Wrappers around the RewritePattern methods that pass the derived op type.
   void rewrite(Operation *op, ArrayRef<Value> operands,
                ConversionPatternRewriter &rewriter) const final {
-    rewrite(cast<SourceOp>(op), OpAdaptor(operands, cast<SourceOp>(op)),
-            rewriter);
+    auto sourceOp = cast<SourceOp>(op);
+    rewrite(sourceOp, OpAdaptor(operands, sourceOp), rewriter);
+  }
+  void rewrite(Operation *op, ArrayRef<ValueRange> operands,
+               ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = cast<SourceOp>(op);
+    rewrite(sourceOp, OneToNOpAdaptor(operands, sourceOp), rewriter);
   }
   LogicalResult match(Operation *op) const final {
     return match(cast<SourceOp>(op));
@@ -162,8 +169,15 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    return matchAndRewrite(cast<SourceOp>(op),
-                           OpAdaptor(operands, cast<SourceOp>(op)), rewriter);
+    auto sourceOp = cast<SourceOp>(op);
+    return matchAndRewrite(sourceOp, OpAdaptor(operands, sourceOp), rewriter);
+  }
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = cast<SourceOp>(op);
+    return matchAndRewrite(sourceOp, OneToNOpAdaptor(operands, sourceOp),
+                           rewriter);
   }
 
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
@@ -175,6 +189,12 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
                        ConversionPatternRewriter &rewriter) const {
     llvm_unreachable("must override rewrite or matchAndRewrite");
   }
+  virtual void rewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                       ConversionPatternRewriter &rewriter) const {
+    SmallVector<Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    rewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
   virtual LogicalResult
   matchAndRewrite(SourceOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const {
@@ -183,6 +203,13 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
     rewrite(op, adaptor, rewriter);
     return success();
   }
+  virtual LogicalResult
+  matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const {
+    SmallVector<Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
 
 private:
   using ConvertToLLVMPattern::match;
diff --git a/mlir/include/mlir/Conversion/MeshToMPI/MeshToMPI.h b/mlir/include/mlir/Conversion/MeshToMPI/MeshToMPI.h
new file mode 100644
index 0000000000000..44a1cc0adb6a0
--- /dev/null
+++ b/mlir/include/mlir/Conversion/MeshToMPI/MeshToMPI.h
@@ -0,0 +1,27 @@
+//===- MeshToMPI.h - Convert Mesh to MPI dialect ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_MESHTOMPI_MESHTOMPI_H
+#define MLIR_CONVERSION_MESHTOMPI_MESHTOMPI_H
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/LLVM.h"
+
+namespace mlir {
+class Pass;
+
+#define GEN_PASS_DECL_CONVERTMESHTOMPIPASS
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Lowers Mesh communication operations (updateHalo, AllGater, ...)
+/// to MPI primitives.
+std::unique_ptr<::mlir::Pass> createConvertMeshToMPIPass();
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_MESHTOMPI_MESHTOMPI_H
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 2ab32836c80b1..b577aa83946f2 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -51,6 +51,7 @@
 #include "mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRVPass.h"
+#include "mlir/Conversion/MeshToMPI/MeshToMPI.h"
 #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h"
 #include "mlir/Conversion/NVVMToLLVM/NVVMToLLVM.h"
 #include "mlir/Conversion/OpenACCToSCF/ConvertOpenACCToSCF.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index e394bae64e091..d722bd1f3e296 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -551,10 +551,7 @@ def LowerHostCodeToLLVMPass : Pass<"lower-host-to-llvm", "ModuleOp"> {
 def ConvertGpuOpsToLLVMSPVOps : Pass<"convert-gpu-to-llvm-spv", "gpu::GPUModuleOp"> {
   let summary =
     "Generate LLVM operations to be ingested by a SPIR-V backend for gpu operations";
-  let dependentDialects = [
-    "LLVM::LLVMDialect",
-    "spirv::SPIRVDialect",
-  ];
+  let dependentDialects = ["LLVM::LLVMDialect"];
   let options = [
     Option<"indexBitwidth", "index-bitwidth", "unsigned",
            /*default=kDeriveIndexBitwidthFromDataLayout*/"0",
@@ -886,6 +883,29 @@ def ConvertMemRefToSPIRV : Pass<"convert-memref-to-spirv"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// MeshToMPI
+//===----------------------------------------------------------------------===//
+
+def ConvertMeshToMPIPass : Pass<"convert-mesh-to-mpi"> {
+  let summary = "Convert Mesh dialect to MPI dialect.";
+  let constructor = "mlir::createConvertMeshToMPIPass()";
+  let description = [{
+    This pass converts communication operations from the Mesh dialect to the
+    MPI dialect.
+    If it finds a global named "static_mpi_rank" it will use that splat value
+    instead of calling MPI_Comm_rank. This allows optimizations like constant
+    shape propagation and fusion because shard/partition sizes depend on the
+    rank.
+  }];
+  let dependentDialects = [
+    "memref::MemRefDialect",
+    "mpi::MPIDialect",
+    "scf::SCFDialect",
+    "bufferization::BufferizationDialect"
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // NVVMToLLVM
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 56fbe9cdc2d21..03172f7ce00e4 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -1115,7 +1115,7 @@ def AffineDelinearizeIndexOp : Affine_Op<"delinearize_index", [Pure]> {
   }];
 
   let builders = [
-    OpBuilder<(ins "Value":$linear_index, "ValueRange":$dynamic_basis, "ArrayRef<int64_t>":$static_asis, CArg<"bool", "true">:$hasOuterBound)>,
+    OpBuilder<(ins "Value":$linear_index, "ValueRange":$dynamic_basis, "ArrayRef<int64_t>":$static_basis, CArg<"bool", "true">:$hasOuterBound)>,
     OpBuilder<(ins "Value":$linear_index, "ValueRange":$basis, CArg<"bool", "true">:$hasOuterBound)>,
     OpBuilder<(ins "Value":$linear_index, "ArrayRef<OpFoldResult>":$basis, CArg<"bool", "true">:$hasOuterBound)>,
     OpBuilder<(ins "Value":$linear_index, "ArrayRef<int64_t>":$basis, CArg<"bool", "true">:$hasOuterBound)>
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
index d27ea5edcc8c8..19a2ade2e95a0 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
@@ -181,37 +181,4 @@ def Arith_RoundingModeAttr : I32EnumAttr<
   let cppNamespace = "::mlir::arith";
 }
 
-//===----------------------------------------------------------------------===//
-// Arith_DenormalMode
-//===----------------------------------------------------------------------===//
-
-// Denormal mode is applied on operands and results. For example, if denormal =
-// preserve_sign, operands and results will be flushed to sign preserving zero.
-// We do not distinguish between operands and results.
-
-// The default mode. Denormals are preserved and processed as defined 
-// by IEEE 754 rules.
-def Arith_DenormalModeIEEE          : I32EnumAttrCase<"ieee", 0>;
-
-// A mode where denormal numbers are flushed to zero, but the sign of the zero 
-// (+0 or -0) is preserved.
-def Arith_DenormalModePreserveSign  : I32EnumAttrCase<"preserve_sign", 1>;
-
-// A mode where all denormal numbers are flushed to positive zero (+0), 
-// ignoring the sign of the original number.
-def Arith_DenormalModePositiveZero  : I32EnumAttrCase<"positive_zero",  2>;
-
-def Arith_DenormalMode : I32EnumAttr<
-    "DenormalMode", "denormal mode arith",
-    [Arith_DenormalModeIEEE, Arith_DenormalModePreserveSign, 
-     Arith_DenormalModePositiveZero]> {
-  let cppNamespace = "::mlir::arith";
-  let genSpecializedAttr = 0;
-}
-
-def Arith_DenormalModeAttr :
-    EnumAttr<Arith_Dialect, Arith_DenormalMode, "denormal"> {
-  let assemblyFormat = "`<` $value `>`";
-}
-
 #endif // ARITH_BASE
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 4069e43af82e8..2f71caaa593a6 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -61,35 +61,26 @@ class Arith_TotalIntBinaryOp<string mnemonic, list<Trait> traits = []> :
 // Base class for floating point unary operations.
 class Arith_FloatUnaryOp<string mnemonic, list<Trait> traits = []> :
     Arith_UnaryOp<mnemonic,
-      !listconcat([DeclareOpInterfaceMethods<ArithFastMathInterface>,
-                   DeclareOpInterfaceMethods<ArithDenormalModeInterface>],
+      !listconcat([DeclareOpInterfaceMethods<ArithFastMathInterface>],
                   traits)>,
     Arguments<(ins FloatLike:$operand,
       DefaultValuedAttr<
-        Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath,
-      DefaultValuedAttr<
-        Arith_DenormalModeAttr, "::mlir::arith::DenormalMode::ieee">:$denormal)>,
+        Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath)>,
     Results<(outs FloatLike:$result)> {
   let assemblyFormat = [{ $operand (`fastmath` `` $fastmath^)?
-                          (`denormal` `` $denormal^)?
                           attr-dict `:` type($result) }];
 }
 
 // Base class for floating point binary operations.
 class Arith_FloatBinaryOp<string mnemonic, list<Trait> traits = []> :
     Arith_BinaryOp<mnemonic,
-      !listconcat([Pure, 
-                   DeclareOpInterfaceMethods<ArithFastMathInterface>,
-                   DeclareOpInterfaceMethods<ArithDenormalModeInterface>],
+      !listconcat([Pure, DeclareOpInterfaceMethods<ArithFastMathInterface>],
                   traits)>,
     Arguments<(ins FloatLike:$lhs, FloatLike:$rhs,
       DefaultValuedAttr<
-        Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath,
-      DefaultValuedAttr<
-        Arith_DenormalModeAttr, "::mlir::arith::DenormalMode::ieee">:$denormal)>,
+        Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath)>,
     Results<(outs FloatLike:$result)> {
-  let assemblyFormat = [{ $lhs `,` $rhs (`fastmath` `` $fastmath^)? 
-                          (`denormal` `` $denormal^)?
+  let assemblyFormat = [{ $lhs `,` $rhs (`fastmath` `` $fastmath^)?
                           attr-dict `:` type($result) }];
 }
 
@@ -1094,6 +1085,7 @@ def Arith_MinUIOp : Arith_TotalIntBinaryOp<"minui", [Commutative]> {
   let hasFolder = 1;
 }
 
+
 //===----------------------------------------------------------------------===//
 // MulFOp
 //===----------------------------------------------------------------------===//
@@ -1119,6 +1111,8 @@ def Arith_MulFOp : Arith_FloatBinaryOp<"mulf", [Commutative]> {
     %x = arith.mulf %y, %z : tensor<4x?xbf16>
     ```
 
+    TODO: In the distant future, this will accept optional attributes for fast
+    math, contraction, rounding mode, and other controls.
   }];
   let hasFolder = 1;
   let hasCanonicalizer = 1;
@@ -1556,15 +1550,6 @@ def Arith_CmpFOp : Arith_CompareOp<"cmpf",
     static arith::CmpFPredicate getPredicateByName(StringRef name);
   }];
 
-  let builders = [
-    OpBuilder<(ins "::mlir::arith::CmpFPredicateAttr":$predicate,
-                   "Value":$lhs, "Value":$rhs), [{
-      build($_builder, $_state, predicate, lhs, rhs,
-          mlir::arith::FastMathFlagsAttr::get($_builder.getContext(),
-              mlir::arith::FastMathFlags::none));
-    }]>
-  ];
-
   let hasFolder = 1;
   let hasCanonicalizer = 1;
   let assemblyFormat = [{ $predicate `,` $lhs `,` $rhs (`fastmath` `` $fastmath^)?
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td
index 270d80f2ec73a..82d6c9ad6b03d 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOpsInterfaces.td
@@ -45,12 +45,13 @@ def ArithFastMathInterface : OpInterface<"ArithFastMathInterface"> {
         return "fastmath";
       }]
       >
+
   ];
 }
 
 def ArithIntegerOverflowFlagsInterface : OpInterface<"ArithIntegerOverflowFlagsInterface"> {
   let description = [{
-    Access to operation integer overflow flags.
+    Access to op integer overflow flags.
   }];
 
   let cppNamespace = "::mlir::arith";
@@ -107,7 +108,7 @@ def ArithIntegerOverflowFlagsInterface : OpInterface<"ArithIntegerOverflowFlagsI
 
 def ArithRoundingModeInterface : OpInterface<"ArithRoundingModeInterface"> {
   let description = [{
-    Access to operation rounding mode.
+    Access to op rounding mode.
   }];
 
   let cppNamespace = "::mlir::arith";
@@ -138,39 +139,4 @@ def ArithRoundingModeInterface : OpInterface<"ArithRoundingModeInterface"> {
   ];
 }
 
-
-def ArithDenormalModeInterface : OpInterface<"ArithDenormalModeInterface"> {
-  let description = [{
-    Access the operation denormal modes.
-  }];
-
-  let cppNamespace = "::mlir::arith";
-
-  let methods = [
-    InterfaceMethod<
-      /*desc=*/        "Returns a DenormalModeAttr attribute for the operation",
-      /*returnType=*/  "DenormalModeAttr",
-      /*methodName=*/  "getDenormalModeAttr",
-      /*args=*/        (ins),
-      /*methodBody=*/  [{}],
-      /*defaultImpl=*/ [{
-        auto op = cast<ConcreteOp>(this->getOperation());
-        return op.getDenormalAttr();
-      }]
-    >,
-    StaticInterfaceMethod<
-      /*desc=*/        [{Returns the name of the DenormalModeAttr attribute for
-                         the operation}],
-      /*returnType=*/  "StringRef",
-      /*methodName=*/  "getDenormalModeAttrName",
-      /*args=*/        (ins),
-      /*methodBody=*/  [{}],
-      /*defaultImpl=*/ [{
-        return "denormal";
-      }]
-    >
-  ];
-}
-
-
 #endif // ARITH_OPS_INTERFACES
diff --git a/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h b/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
deleted file mode 100644
index 1be406bf3adf9..0000000000000
--- a/mlir/include/mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//===- DecomposeCallGraphTypes.h - CG type decompositions -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Conversion patterns for decomposing types along call graph edges. That is,
-// decomposing types for calls, returns, and function args.
-//
-// TODO: Make this handle dialect-defined functions, calls, and returns.
-// Currently, the generic interfaces aren't sophisticated enough for the
-// types of mutations that we are doing here.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_DIALECT_FUNC_TRANSFORMS_DECOMPOSECALLGRAPHTYPES_H
-#define MLIR_DIALECT_FUNC_TRANSFORMS_DECOMPOSECALLGRAPHTYPES_H
-
-#include "mlir/Transforms/DialectConversion.h"
-#include <optional>
-
-namespace mlir {
-
-/// Populates the patterns needed to drive the conversion process for
-/// decomposing call graph types with the given `TypeConverter`.
-void populateDecomposeCallGraphTypesPatterns(MLIRContext *context,
-                                             const TypeConverter &typeConverter,
-                                             RewritePatternSet &patterns);
-
-} // namespace mlir
-
-#endif // MLIR_DIALECT_FUNC_TRANSFORMS_DECOMPOSECALLGRAPHTYPES_H
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 5b1d7bb87a219..42a017db300af 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -353,7 +353,7 @@ def GPU_OptionalDimSizeHintAttr : ConfinedAttr<OptionalAttr<DenseI32ArrayAttr>,
 
 def GPU_GPUFuncOp : GPU_Op<"func", [
     HasParent<"GPUModuleOp">, AutomaticAllocationScope, FunctionOpInterface,
-    IsolatedFromAbove
+    IsolatedFromAbove, AffineScope
   ]> {
   let summary = "Function executable on a GPU";
 
@@ -803,7 +803,9 @@ def GPU_LaunchOp : GPU_Op<"launch", [
                Optional<Index>:$clusterSizeX,
                Optional<Index>:$clusterSizeY,
                Optional<Index>:$clusterSizeZ,
-               Optional<I32>:$dynamicSharedMemorySize)>,
+               Optional<I32>:$dynamicSharedMemorySize,
+               OptionalAttr<SymbolRefAttr>:$kernelFunc,
+               OptionalAttr<SymbolRefAttr>:$kernelModule)>,
     Results<(outs Optional<GPU_AsyncToken>:$asyncToken)> {
   let summary = "GPU kernel launch operation";
 
@@ -837,6 +839,9 @@ def GPU_LaunchOp : GPU_Op<"launch", [
     -   a variadic number of Workgroup memory attributions.
     -   a variadic number of Private memory attributions.
 
+    The `kernelFunc` and `kernelModule` attributes are optional and specifies
+    the kernel name and a module in which the kernel should be outlined. 
+
     Syntax:
 
     ```
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index ea82f7f7b8e12..b5526bda9f2de 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -52,6 +52,7 @@ def LLVM_Dialect : Dialect {
     static StringRef getNoFreeAttrName() { return "llvm.nofree"; }
     static StringRef getNonNullAttrName() { return "llvm.nonnull"; }
     static StringRef getPreallocatedAttrName() { return "llvm.preallocated"; }
+    static StringRef getRangeAttrName() { return "llvm.range"; }
     static StringRef getReadonlyAttrName() { return "llvm.readonly"; }
     static StringRef getReturnedAttrName() { return "llvm.returned"; }
     static StringRef getSExtAttrName() { return "llvm.signext"; }
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index 4a43c16903394..c08b75de03647 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -615,40 +615,40 @@ def ICmpPredicate : LLVM_EnumAttr<
 //===----------------------------------------------------------------------===//
 
 // Linkage attribute is used on functions and globals. The order follows that of
-// https://llvm.org/docs/LangRef.html#linkage-types. The names are equivalent to
-// visible names in the IR rather than to enum values names in llvm::GlobalValue
-// since the latter is easier to change.
-def LinkagePrivate
-    : LLVM_EnumAttrCase<"Private", "private", "PrivateLinkage", 0>;
-def LinkageInternal
-    : LLVM_EnumAttrCase<"Internal", "internal", "InternalLinkage", 1>;
+// llvm::GlobalValue::LinkageTypes from llvm/IR/GlobalValue.h. The names are
+// equivalent to visible names in the IR rather than to enum values names in
+// llvm::GlobalValue since the latter is easier to change.
+def LinkageExternal
+    : LLVM_EnumAttrCase<"External", "external", "ExternalLinkage", 0>;
 def LinkageAvailableExternally
     : LLVM_EnumAttrCase<"AvailableExternally", "available_externally",
-                        "AvailableExternallyLinkage", 2>;
+                        "AvailableExternallyLinkage", 1>;
 def LinkageLinkonce
-    : LLVM_EnumAttrCase<"Linkonce", "linkonce", "LinkOnceAnyLinkage", 3>;
+    : LLVM_EnumAttrCase<"Linkonce", "linkonce", "LinkOnceAnyLinkage", 2>;
+def LinkageLinkonceODR
+    : LLVM_EnumAttrCase<"LinkonceODR", "linkonce_odr", "LinkOnceODRLinkage", 3>;
 def LinkageWeak
     : LLVM_EnumAttrCase<"Weak", "weak", "WeakAnyLinkage", 4>;
-def LinkageCommon
-    : LLVM_EnumAttrCase<"Common", "common", "CommonLinkage", 5>;
+def LinkageWeakODR
+    : LLVM_EnumAttrCase<"WeakODR", "weak_odr", "WeakODRLinkage", 5>;
 def LinkageAppending
     : LLVM_EnumAttrCase<"Appending", "appending", "AppendingLinkage", 6>;
+def LinkageInternal
+    : LLVM_EnumAttrCase<"Internal", "internal", "InternalLinkage", 7>;
+def LinkagePrivate
+    : LLVM_EnumAttrCase<"Private", "private", "PrivateLinkage", 8>;
 def LinkageExternWeak
-   : LLVM_EnumAttrCase<"ExternWeak", "extern_weak", "ExternalWeakLinkage", 7>;
-def LinkageLinkonceODR
-    : LLVM_EnumAttrCase<"LinkonceODR", "linkonce_odr", "LinkOnceODRLinkage", 8>;
-def LinkageWeakODR
-    : LLVM_EnumAttrCase<"WeakODR", "weak_odr", "WeakODRLinkage", 9>;
-def LinkageExternal
-    : LLVM_EnumAttrCase<"External", "external", "ExternalLinkage", 10>;
+   : LLVM_EnumAttrCase<"ExternWeak", "extern_weak", "ExternalWeakLinkage", 9>;
+def LinkageCommon
+    : LLVM_EnumAttrCase<"Common", "common", "CommonLinkage", 10>;
 
 def LinkageEnum : LLVM_EnumAttr<
     "Linkage",
     "::llvm::GlobalValue::LinkageTypes",
     "LLVM linkage types",
-    [LinkagePrivate, LinkageInternal, LinkageAvailableExternally,
-     LinkageLinkonce, LinkageWeak, LinkageCommon, LinkageAppending,
-     LinkageExternWeak, LinkageLinkonceODR, LinkageWeakODR, LinkageExternal]> {
+    [LinkageExternal, LinkageAvailableExternally, LinkageLinkonce,
+      LinkageLinkonceODR, LinkageWeak, LinkageWeakODR, LinkageAppending,
+      LinkageInternal, LinkagePrivate, LinkageExternWeak, LinkageCommon]> {
   let cppNamespace = "::mlir::LLVM::linkage";
 }
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
index e88139fa5b28d..77c8035ce3d71 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
@@ -233,7 +233,7 @@ def LLVMStructType : LLVMType<"LLVMStruct", "struct", [
     bool isIdentified() const;
 
     /// Checks if a struct is opaque.
-    bool isOpaque();
+    bool isOpaque() const;
 
     /// Checks if a struct is initialized.
     bool isInitialized();
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
index 289c0e4bbdaf6..71214b4404c55 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
@@ -15,13 +15,10 @@ function(add_linalg_ods_yaml_gen yaml_ast_file output_file)
     MAIN_DEPENDENCY
     ${YAML_AST_SOURCE}
     DEPENDS
-    ${MLIR_LINALG_ODS_YAML_GEN_EXE}
     ${MLIR_LINALG_ODS_YAML_GEN_TARGET})
   add_custom_target(
     MLIR${output_file}YamlIncGen
     DEPENDS
-    ${MLIR_LINALG_ODS_YAML_GEN_EXE}
-    ${MLIR_LINALG_ODS_YAML_GEN_TARGET}
     ${GEN_ODS_FILE} ${GEN_CPP_FILE})
   set_target_properties(MLIR${output_file}YamlIncGen PROPERTIES FOLDER "MLIR/Tablegenning")
   list(APPEND LLVM_TARGET_DEPENDS ${GEN_ODS_FILE})
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index dc10f3a1c58ae..2e713bca24efc 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -559,7 +559,8 @@ def LowerPackOp : Op<Transform_Dialect, "structured.lower_pack", [
     Return handles to the newly produced pad, expand_shape and transpose ops.
   }];
 
-  let arguments = (ins Transform_ConcreteOpType<"tensor.pack">:$target);
+  let arguments = (ins Transform_ConcreteOpType<"tensor.pack">:$target,
+                       DefaultValuedAttr<BoolAttr, "true">:$lowerPadLikeWithInsertSlice);
   let results = (outs Transform_ConcreteOpType<"tensor.pad">:$pad_op,
                       Transform_ConcreteOpType<"tensor.expand_shape">:$expand_shape_op,
                       Transform_ConcreteOpType<"linalg.transpose">:$transpose_op);
@@ -599,7 +600,8 @@ def LowerUnPackOp : Op<Transform_Dialect, "structured.lower_unpack", [
     Return handles to the newly produced empty, transpose, collapse_shape and extract_slice ops.
   }];
 
-  let arguments = (ins Transform_ConcreteOpType<"tensor.unpack">:$target);
+  let arguments = (ins Transform_ConcreteOpType<"tensor.unpack">:$target,
+                       DefaultValuedAttr<BoolAttr, "true">:$lowerUnpadLikeWithExtractSlice);
   let results = (outs Transform_ConcreteOpType<"tensor.empty">:$empty_op,
                       Transform_ConcreteOpType<"linalg.transpose">:$transpose_op,
                       Transform_ConcreteOpType<"tensor.collapse_shape">:$collapse_shape_op,
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 3c160d55a38e7..1dc700f22c202 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1121,7 +1121,8 @@ struct LowerPackResult {
 
 /// Rewrite pack as pad + reshape + transpose.
 FailureOr<LowerPackResult> lowerPack(RewriterBase &rewriter,
-                                     tensor::PackOp packOp);
+                                     tensor::PackOp packOp,
+                                     bool lowerPadLikeWithInsertSlice = true);
 
 struct LowerUnPackOpResult {
   tensor::EmptyOp emptyOp;
@@ -1131,8 +1132,9 @@ struct LowerUnPackOpResult {
 };
 
 /// Rewrite pack as empty + transpose + reshape + extract_slice.
-FailureOr<LowerUnPackOpResult> lowerUnPack(RewriterBase &rewriter,
-                                           tensor::UnPackOp unPackOp);
+FailureOr<LowerUnPackOpResult>
+lowerUnPack(RewriterBase &rewriter, tensor::UnPackOp unPackOp,
+            bool lowerUnpadLikeWithExtractSlice = true);
 
 /// Struct to hold the result of a `pack` call.
 struct PackResult {
@@ -1519,7 +1521,7 @@ struct DecomposePadOpPattern : public OpRewritePattern<tensor::PadOp> {
 ///   * tensor::PadOp + linalg::TransposeOp + tensor::EmptyOp +
 ///     tensor::InsertSliceOp ops.
 ///
-/// Required that all the outer dims of the input tensor::PackOp are 1.
+/// Requires that all the outer dims of the input tensor::PackOp are 1.
 ///
 /// Before:
 /// ```
@@ -1555,9 +1557,33 @@ struct DecomposeOuterUnitDimsPackOpPattern
                                 PatternRewriter &rewriter) const override;
 };
 
-/// Rewrites a tensor::UnPackOp into a sequence of rank-reduced extract_slice op
-/// + transpose op + insert_slice op, where the tensor::UnPackOp has outer dims
-/// being all 1s.
+/// Rewrites a tensor::UnPackOp into a sequence of rank-reduced
+///   * tensor::ExtractSliceOp + linalg::TransposeOp + tensor::InsertSliceOp
+///
+/// Requires that all the outer dims of the input tensor::PackOp are 1.
+///
+/// Before:
+/// ```
+/// %packed = tensor.unpack %input
+///   inner_dims_pos = [1, 0]
+///   inner_tiles = [2, 8]
+///   into %output : tensor<1x1x2x8xf32> -> tensor<5x1xf32>
+/// ```
+///
+/// After:
+/// ```
+///   // Rank-reduced extract to obtain the tile
+///   %slice = tensor.extract_slice %arg0[0, 0, 0, 0] [1, 1, 2, 8] [1, 1, 1, 1]
+///     : tensor<1x1x2x8xf32> to tensor<2x8xf32>
+///   // EmptyOp + TransposeOp
+///   %init = tensor.empty() : tensor<8x2xf32>
+///   %transposed = linalg.transpose
+///     ins(%extracted_slice : tensor<2x8xf32>)
+///     outs(%0 : tensor<8x2xf32>) permutation = [1, 0]
+///   // Extract a slice matching the specified output size
+///   %result = tensor.extract_slice %transposed[0, 0] [5, 1] [1, 1]
+///     : tensor<8x2xf32> to tensor<5x1xf32>
+/// ```
 struct DecomposeOuterUnitDimsUnPackOpPattern
     : public OpRewritePattern<tensor::UnPackOp> {
   using OpRewritePattern<tensor::UnPackOp>::OpRewritePattern;
diff --git a/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td b/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
index 768f376e24da4..240fac5104c34 100644
--- a/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
+++ b/mlir/include/mlir/Dialect/MPI/IR/MPIOps.td
@@ -84,6 +84,7 @@ def MPI_SendOp : MPI_Op<"send", []> {
   let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict `:` "
                        "type($ref) `,` type($tag) `,` type($rank)"
                        "(`->` type($retval)^)?";
+  let hasCanonicalizer = 1;
 }
 
 //===----------------------------------------------------------------------===//
@@ -114,6 +115,7 @@ def MPI_RecvOp : MPI_Op<"recv", []> {
   let assemblyFormat = "`(` $ref `,` $tag `,` $rank `)` attr-dict `:` "
                        "type($ref) `,` type($tag) `,` type($rank)"
                        "(`->` type($retval)^)?";
+  let hasCanonicalizer = 1;
 }
 
 
diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
index 19498fe5a32d6..6039e61a93fad 100644
--- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
+++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
@@ -156,6 +156,40 @@ def Mesh_ProcessLinearIndexOp : Mesh_Op<"process_linear_index", [
   ];
 }
 
+def Mesh_NeighborsLinearIndicesOp : Mesh_Op<"neighbors_linear_indices", [
+  Pure,
+  DeclareOpInterfaceMethods<SymbolUserOpInterface>,
+  DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>
+]> {
+  let summary =
+      "For given mesh index get the linear indices of the direct neighbor processes along the given split.";
+  let description = [{
+    Example:
+    ```
+    mesh.mesh @mesh0(shape = 10x20x30)
+    %c1 = arith.constant 1 : index
+    %c2 = arith.constant 2 : index
+    %c3 = arith.constant 3 : index
+    %idx = mesh.neighbors_linear_indices on @mesh[%c1, %c2, %c3] split_axes = [1] : index
+    ```
+    The above returns two indices, `633` and `693`, which correspond to the
+    index of the previous process `(1, 1, 3)`, and the next process 
+    `(1, 3, 3) along the split axis `1`.
+
+    A negative value is returned if there is no neighbor in the respective
+    direction along the given `split_axes`.
+  }];
+  let arguments = (ins FlatSymbolRefAttr:$mesh,
+                       Variadic<Index>:$device,
+                       Mesh_MeshAxesAttr:$split_axes);
+  let results = (outs Index:$neighbor_down, Index:$neighbor_up);
+  let assemblyFormat =  [{
+      `on` $mesh `[` $device `]`
+      `split_axes` `=` $split_axes
+      attr-dict `:` type(results)
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Sharding operations.
 //===----------------------------------------------------------------------===//
@@ -1058,12 +1092,12 @@ def Mesh_ShiftOp : Mesh_CollectiveCommunicationOpBase<"shift", [
 }
 
 def Mesh_UpdateHaloOp : Mesh_Op<"update_halo", [
+  Pure,
   DestinationStyleOpInterface,
   TypesMatchWith<
     "result has same type as destination",
     "result", "destination", "$_self">,
-  DeclareOpInterfaceMethods<SymbolUserOpInterface>,
-  AttrSizedOperandSegments
+  DeclareOpInterfaceMethods<SymbolUserOpInterface>
 ]> {
   let summary = "Update halo data.";
   let description = [{
@@ -1072,7 +1106,7 @@ def Mesh_UpdateHaloOp : Mesh_Op<"update_halo", [
     on the remote devices. Changes might be caused by mutating operations
     and/or if the new halo regions are larger than the existing ones.
 
-    Source and destination might have different halo sizes.
+    Destination is supposed to be initialized with the local data (not halos).
 
     Assumes all devices hold tensors with same-sized halo data as specified
     by `source_halo_sizes/static_source_halo_sizes` and
@@ -1084,25 +1118,21 @@ def Mesh_UpdateHaloOp : Mesh_Op<"update_halo", [
 
   }];
   let arguments = (ins
-    AnyTypeOf<[AnyNon0RankedMemRef, AnyNon0RankedTensor]>:$source,
     AnyTypeOf<[AnyNon0RankedMemRef, AnyNon0RankedTensor]>:$destination,
     FlatSymbolRefAttr:$mesh,
     Mesh_MeshAxesArrayAttr:$split_axes,
-    Variadic<I64>:$source_halo_sizes,
-    DefaultValuedAttr<DenseI64ArrayAttr, "{}">:$static_source_halo_sizes,
-    Variadic<I64>:$destination_halo_sizes,
-    DefaultValuedAttr<DenseI64ArrayAttr, "{}">:$static_destination_halo_sizes
+    Variadic<I64>:$halo_sizes,
+    DefaultValuedAttr<DenseI64ArrayAttr, "{}">:$static_halo_sizes
   );
   let results = (outs
     AnyTypeOf<[AnyNon0RankedMemRef, AnyNon0RankedTensor]>:$result
   );
   let assemblyFormat = [{
-    $source `into` $destination
+    $destination
     `on` $mesh
     `split_axes` `=` $split_axes
-    (`source_halo_sizes` `=` custom<DynamicIndexList>($source_halo_sizes, $static_source_halo_sizes)^)?
-    (`destination_halo_sizes` `=` custom<DynamicIndexList>($destination_halo_sizes, $static_destination_halo_sizes)^)?
-    attr-dict `:` type($source) `->` type($result)
+    (`halo_sizes` `=` custom<DynamicIndexList>($halo_sizes, $static_halo_sizes)^)?
+    attr-dict `:` type($result)
   }];
   let extraClassDeclaration = [{
     MutableOperandRange getDpsInitsMutable() { return getDestinationMutable(); }
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 8d7e27405cfa4..3ac265ac68756 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -381,17 +381,18 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
     OpenACC_Op<mnemonic, !listconcat(traits,
         [AttrSizedOperandSegments,
          MemoryEffects<[MemRead<OpenACC_CurrentDeviceIdResource>]>])> {
-  let arguments = !con(additionalArgs,
-                      (ins
-                       Optional<OpenACC_PointerLikeTypeInterface>:$varPtrPtr,
-                       Variadic<OpenACC_DataBoundsType>:$bounds, /* rank-0 to rank-{n-1} */
-                       Variadic<IntOrIndex>:$asyncOperands,
-                       OptionalAttr<DeviceTypeArrayAttr>:$asyncOperandsDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$asyncOnly,
-                       DefaultValuedAttr<OpenACC_DataClauseAttr,clause>:$dataClause,
-                       DefaultValuedAttr<BoolAttr, "true">:$structured,
-                       DefaultValuedAttr<BoolAttr, "false">:$implicit,
-                       OptionalAttr<StrAttr>:$name));
+  let arguments = !con(
+      additionalArgs,
+      (ins TypeAttr:$varType,
+          Optional<OpenACC_PointerLikeTypeInterface>:$varPtrPtr,
+          Variadic<OpenACC_DataBoundsType>:$bounds, /* rank-0 to rank-{n-1} */
+          Variadic<IntOrIndex>:$asyncOperands,
+          OptionalAttr<DeviceTypeArrayAttr>:$asyncOperandsDeviceType,
+          OptionalAttr<DeviceTypeArrayAttr>:$asyncOnly,
+          DefaultValuedAttr<OpenACC_DataClauseAttr, clause>:$dataClause,
+          DefaultValuedAttr<BoolAttr, "true">:$structured,
+          DefaultValuedAttr<BoolAttr, "false">:$implicit,
+          OptionalAttr<StrAttr>:$name));
 
   let description = !strconcat(extraDescription, [{
     Description of arguments:
@@ -458,7 +459,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
   }];
 
   let assemblyFormat = [{
-    `varPtr` `(` $varPtr `:` type($varPtr) `)`
+    `varPtr` `(` $varPtr `:` custom<VarPtrType>(type($varPtr), $varType)
     oilist(
         `varPtrPtr` `(` $varPtrPtr `:` type($varPtrPtr) `)`
       | `bounds` `(` $bounds `)`
@@ -469,32 +470,35 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
 
   let hasVerifier = 1;
 
-  let builders = [
-    OpBuilder<(ins "::mlir::Value":$varPtr,
-      "bool":$structured,
-      "bool":$implicit,
-      CArg<"::mlir::ValueRange", "{}">:$bounds), [{
-        build($_builder, $_state, varPtr.getType(), varPtr, /*varPtrPtr=*/{},
-          bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
+  let builders = [OpBuilder<(ins "::mlir::Value":$varPtr, "bool":$structured,
+                                "bool":$implicit,
+                                CArg<"::mlir::ValueRange", "{}">:$bounds),
+                            [{
+        build($_builder, $_state, varPtr.getType(), varPtr,
+          /*varType=*/::mlir::TypeAttr::get(
+            ::mlir::cast<::mlir::acc::PointerLikeType>(
+              varPtr.getType()).getElementType()),
+          /*varPtrPtr=*/{}, bounds, /*asyncOperands=*/{},
+          /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*name=*/nullptr);
-      }]
-    >,
-    OpBuilder<(ins "::mlir::Value":$varPtr,
-      "bool":$structured,
-      "bool":$implicit,
-      "const ::llvm::Twine &":$name,
-      CArg<"::mlir::ValueRange", "{}">:$bounds), [{
-        build($_builder, $_state, varPtr.getType(), varPtr, /*varPtrPtr=*/{},
-          bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
+      }]>,
+                  OpBuilder<(ins "::mlir::Value":$varPtr, "bool":$structured,
+                                "bool":$implicit, "const ::llvm::Twine &":$name,
+                                CArg<"::mlir::ValueRange", "{}">:$bounds),
+                            [{
+        build($_builder, $_state, varPtr.getType(), varPtr,
+          /*varType=*/::mlir::TypeAttr::get(
+            ::mlir::cast<::mlir::acc::PointerLikeType>(
+              varPtr.getType()).getElementType()),
+          /*varPtrPtr=*/{}, bounds, /*asyncOperands=*/{},
+          /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit),
           /*name=*/$_builder.getStringAttr(name));
-      }]
-    >
-  ];
+      }]>];
 }
 
 //===----------------------------------------------------------------------===//
@@ -794,63 +798,58 @@ class OpenACC_DataExitOp<string mnemonic, string clause, string extraDescription
     }
   }];
 
-  let assemblyFormat = [{
-    `accPtr` `(` $accPtr `:` type($accPtr) `)`
-    oilist(
-        `bounds` `(` $bounds `)`
-      | `to` `varPtr` `(` $varPtr `:` type($varPtr) `)`
-      | `async` `(` custom<DeviceTypeOperands>($asyncOperands,
-            type($asyncOperands), $asyncOperandsDeviceType) `)`
-    ) attr-dict
-  }];
-
   let hasVerifier = 1;
 }
 
-class OpenACC_DataExitOpWithVarPtr<string mnemonic, string clause> :
-    OpenACC_DataExitOp<mnemonic, clause,
-      "- `varPtr`: The address of variable to copy back to.",
-      [MemoryEffects<[MemRead<OpenACC_RuntimeCounters>,
-                      MemWrite<OpenACC_RuntimeCounters>]>],
-      (ins Arg<OpenACC_PointerLikeTypeInterface,"Address of device variable",[MemRead]>:$accPtr,
-           Arg<OpenACC_PointerLikeTypeInterface,"Address of variable",[MemWrite]>:$varPtr)> {
+class OpenACC_DataExitOpWithVarPtr<string mnemonic, string clause>
+    : OpenACC_DataExitOp<
+          mnemonic, clause,
+          "- `varPtr`: The address of variable to copy back to.",
+          [MemoryEffects<[MemRead<OpenACC_RuntimeCounters>,
+                          MemWrite<OpenACC_RuntimeCounters>]>],
+          (ins Arg<OpenACC_PointerLikeTypeInterface,
+                   "Address of device variable", [MemRead]>:$accPtr,
+              Arg<OpenACC_PointerLikeTypeInterface,
+                  "Address of variable", [MemWrite]>:$varPtr,
+              TypeAttr:$varType)> {
   let assemblyFormat = [{
     `accPtr` `(` $accPtr `:` type($accPtr) `)`
     (`bounds` `(` $bounds^ `)` )?
     (`async` `(` custom<DeviceTypeOperands>($asyncOperands,
             type($asyncOperands), $asyncOperandsDeviceType)^ `)`)?
-    `to` `varPtr` `(` $varPtr `:` type($varPtr) `)`
+    `to` `varPtr` `(` $varPtr `:` custom<VarPtrType>(type($varPtr), $varType)
     attr-dict
   }];
 
-  let builders = [
-    OpBuilder<(ins "::mlir::Value":$accPtr,
-      "::mlir::Value":$varPtr,
-      "bool":$structured,
-      "bool":$implicit,
-      CArg<"::mlir::ValueRange", "{}">:$bounds), [{
+  let builders = [OpBuilder<(ins "::mlir::Value":$accPtr,
+                                "::mlir::Value":$varPtr, "bool":$structured,
+                                "bool":$implicit,
+                                CArg<"::mlir::ValueRange", "{}">:$bounds),
+                            [{
         build($_builder, $_state, accPtr, varPtr,
+          /*varType=*/::mlir::TypeAttr::get(
+          ::mlir::cast<::mlir::acc::PointerLikeType>(
+              varPtr.getType()).getElementType()),
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit), /*name=*/nullptr);
-      }]
-    >,
-    OpBuilder<(ins "::mlir::Value":$accPtr,
-      "::mlir::Value":$varPtr,
-      "bool":$structured,
-      "bool":$implicit,
-      "const ::llvm::Twine &":$name,
-      CArg<"::mlir::ValueRange", "{}">:$bounds), [{
+      }]>,
+                  OpBuilder<(ins "::mlir::Value":$accPtr,
+                                "::mlir::Value":$varPtr, "bool":$structured,
+                                "bool":$implicit, "const ::llvm::Twine &":$name,
+                                CArg<"::mlir::ValueRange", "{}">:$bounds),
+                            [{
         build($_builder, $_state, accPtr, varPtr,
+          /*varType=*/::mlir::TypeAttr::get(
+          ::mlir::cast<::mlir::acc::PointerLikeType>(
+              varPtr.getType()).getElementType()),
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
           /*implicit=*/$_builder.getBoolAttr(implicit),
           /*name=*/$_builder.getStringAttr(name));
-      }]
-    >
-  ];
+      }]>];
 }
 
 class OpenACC_DataExitOpNoVarPtr<string mnemonic, string clause> :
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
index 855deab94b2f1..077d6602628aa 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
@@ -909,6 +909,34 @@ class OpenMP_ParallelizationLevelClauseSkip<
 
 def OpenMP_ParallelizationLevelClause : OpenMP_ParallelizationLevelClauseSkip<>;
 
+//===----------------------------------------------------------------------===//
+// OpenMPV5.2: [12.5.2] `detach` clause
+//===----------------------------------------------------------------------===//
+
+class OpenMP_DetachClauseSkip<
+    bit traits = false, bit arguments = false, bit assemblyFormat = false,
+    bit description = false, bit extraClassDeclaration = false>
+    : OpenMP_Clause<traits, arguments, assemblyFormat, description,
+                    extraClassDeclaration> {
+
+  let traits = [BlockArgOpenMPOpInterface];
+
+  let arguments = (ins Optional<OpenMP_PointerLikeType>:$event_handle);
+
+  let optAssemblyFormat = [{
+		`detach` `(` $event_handle `:` type($event_handle) `)`
+	}];
+
+  let description = [{
+		The detach clause specifies that the task generated by the construct on which it appears is a
+	detachable task. A new allow-completion event is created and connected to the completion of the
+	associated task region. The original event-handle is updated to represent that allow-completion
+	event before the task data environment is created.
+	}];
+}
+
+def OpenMP_DetachClause : OpenMP_DetachClauseSkip<>;
+
 //===----------------------------------------------------------------------===//
 // V5.2: [12.4] `priority` clause
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 156e6eb371b85..85b6a6638036f 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -657,15 +657,18 @@ def DistributeOp : OpenMP_Op<"distribute", traits = [
 // 2.10.1 task Construct
 //===----------------------------------------------------------------------===//
 
-def TaskOp : OpenMP_Op<"task", traits = [
-    AttrSizedOperandSegments, AutomaticAllocationScope,
-    OutlineableOpenMPOpInterface
-  ], clauses = [
-    // TODO: Complete clause list (affinity, detach).
-    OpenMP_AllocateClause, OpenMP_DependClause, OpenMP_FinalClause,
-    OpenMP_IfClause, OpenMP_InReductionClause, OpenMP_MergeableClause,
-    OpenMP_PriorityClause, OpenMP_PrivateClause, OpenMP_UntiedClause
-  ], singleRegion = true> {
+def TaskOp
+    : OpenMP_Op<"task",
+                traits = [AttrSizedOperandSegments, AutomaticAllocationScope,
+                          OutlineableOpenMPOpInterface],
+                clauses = [
+                    // TODO: Complete clause list (affinity, detach).
+                    OpenMP_AllocateClause, OpenMP_DependClause,
+                    OpenMP_FinalClause, OpenMP_IfClause,
+                    OpenMP_InReductionClause, OpenMP_MergeableClause,
+                    OpenMP_PriorityClause, OpenMP_PrivateClause,
+                    OpenMP_UntiedClause, OpenMP_DetachClause],
+                singleRegion = true> {
   let summary = "task construct";
   let description = [{
     The task construct defines an explicit task.
@@ -1225,21 +1228,46 @@ def TargetOp : OpenMP_Op<"target", traits = [
     The optional `if_expr` parameter specifies a boolean result of a conditional
     check. If this value is 1 or is not provided then the target region runs on
     a device, if it is 0 then the target region is executed on the host device.
+
+    The `private_maps` attribute connects `private` operands to their corresponding
+    `map` operands. For `private` operands that require a map, the value of the
+    corresponding element in the attribute is the index of the `map` operand
+    (relative to other `map` operands not the whole operands of the operation). For
+    `private` opernads that do not require a map, this value is -1 (which is omitted
+    from the assembly foramt printing).
   }] # clausesDescription;
 
+  let arguments = !con(clausesArgs,
+                       (ins OptionalAttr<DenseI64ArrayAttr>:$private_maps));
+
   let builders = [
     OpBuilder<(ins CArg<"const TargetOperands &">:$clauses)>
   ];
 
   let extraClassDeclaration = [{
     unsigned numMapBlockArgs() { return getMapVars().size(); }
+
+    mlir::Value getMappedValueForPrivateVar(unsigned privVarIdx) {
+      std::optional<DenseI64ArrayAttr> privateMapIdices = getPrivateMapsAttr();
+
+      if (!privateMapIdices.has_value())
+        return {};
+
+      int64_t mapInfoOpIdx = (*privateMapIdices)[privVarIdx];
+
+      if (mapInfoOpIdx == -1)
+        return {};
+
+      return getMapVars()[mapInfoOpIdx];
+    }
   }] # clausesExtraClassDeclaration;
 
   let assemblyFormat = clausesAssemblyFormat # [{
     custom<InReductionMapPrivateRegion>(
         $region, $in_reduction_vars, type($in_reduction_vars),
         $in_reduction_byref, $in_reduction_syms, $map_vars, type($map_vars),
-        $private_vars, type($private_vars), $private_syms) attr-dict
+        $private_vars, type($private_vars), $private_syms, $private_maps)
+        attr-dict
   }];
 
   let hasVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index b73da8bb6af59..812ac20984502 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -213,11 +213,11 @@ def Tensor_DimOp : Tensor_Op<"dim", [
 
     // Return the dynamic dimension of %A.
     %c1 = arith.constant 1 : index
-    %y = tensor.dim %A, %c1 : memref<4x?xf32>
+    %y = tensor.dim %A, %c1 : tensor<4x?xf32>
 
     // Equivalent generic form:
-    %x = "tensor.dim"(%A, %c0) : (memref<4x?xf32>, index) -> index
-    %y = "tensor.dim"(%A, %c1) : (memref<4x?xf32>, index) -> index
+    %x = "tensor.dim"(%A, %c0) : (tensor<4x?xf32>, index) -> index
+    %y = "tensor.dim"(%A, %c1) : (tensor<4x?xf32>, index) -> index
     ```
   }];
 
@@ -673,7 +673,7 @@ def Tensor_GatherOp : Tensor_Op<"gather", [
 
     At the tensor-level, the index tensor is specified in an AoS form (i.e.
     coordinate tuple is the most minor). It is the responsibility of further
-    lowerings and bufferiation to implement various concrete layouts.
+    lowerings and bufferization to implement various concrete layouts.
 
     Note: As currently specified, the operation must lower to an abstraction that
     performs copies to the output tensor. This is because the buffer type system
@@ -710,7 +710,7 @@ def Tensor_GatherOp : Tensor_Op<"gather", [
 
   let extraClassDeclaration = [{
     // TODO: InferTypeOpInterface once enough confidence is built with
-    // tensor<tensor> and its lwoering to memref<memref>.
+    // tensor<tensor> and its lowering to memref<memref>.
     static RankedTensorType inferResultType(RankedTensorType sourceType,
                                             RankedTensorType indicesType,
                                             ArrayRef<int64_t> gatherDims,
@@ -1673,7 +1673,7 @@ def Tensor_ScatterOp : Tensor_Op<"scatter", [
     source tensor has size `1`.
     I.e. if the dest type is `axbxcxd` and the coordinates are [1, 3], then
     the source type suffix is `ax1xcx1`.
-    Sactter also allows rank-reducing semantics where the shape `ax1xcx1` can be
+    Scatter also allows rank-reducing semantics where the shape `ax1xcx1` can be
     further simplified to `axc`.
 
     The elemental type of the indices tensor can be any integer type.
@@ -1693,7 +1693,7 @@ def Tensor_ScatterOp : Tensor_Op<"scatter", [
 
     At the tensor-level, the index tensor is specified in an AoS form (i.e.
     coordinate tuple is the most minor). It is the responsibility of further
-    lowerings and bufferiation to implement various concrete layouts.
+    lowerings and bufferization to implement various concrete layouts.
 
     Note: As currently specified, the operation must lower to an abstraction that
     performs copies to the output tensor. This is because the buffer type system
@@ -2115,7 +2115,7 @@ def Tensor_UnPackOp : Tensor_RelayoutOp<"unpack"> {
     /// Check if this UnPackOp is like a simple unpad operation.
     /// In other words, this operation:
     /// 1. drops useless dimensions (dimension of size 1), and
-    /// 2. reduces dimensions in place (i.e., no tranpose.)
+    /// 2. reduces dimensions in place (i.e., no transpose.)
     bool isLikeUnPad();
   }];
 
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 9f57efff5d1fd..e3c725801d162 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -828,7 +828,7 @@ def Tosa_PowOp : Tosa_ElementwiseOp<"pow", [SameOperandsAndResultElementType]> {
   );
 
   let results = (outs
-    Tosa_Tensor:$z
+    Tosa_Tensor:$output
   );
 }
 
@@ -1869,22 +1869,23 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
   let description = [{
     Rescale quantized values into a new domain. Supported rescalings are:
 
-    | Mode                   | Input | Output |
-    |------------------------|-------|--------|
-    | signed 8 to 8          | int8  | int8   |
-    | signed 8 to 16         | int8  | int16  |
-    | signed 8 to 32         | int8  | int32  |
-    | signed 16 to 8         | int16 | int8   |
-    | signed 16 to 16        | int16 | int16  |
-    | signed 16 to 32        | int16 | int32  |
-    | signed 32 to 8         | int32 | int8   |
-    | signed 32 to 16        | int32 | int16  |
-    | signed 32 to 32        | int32 | int32  |
-    | signed 48 to 8         | int48 | int8   |
-    | signed 48 to 16        | int48 | int16  |
-    | signed 48 to 32        | int48 | int32  |
-    | unsigned 8 to signed 8 | uint8 | int8   |
-    | signed 8 to unsigned 8 | int8  | uint8  |
+    | Mode                   | Input | Output | Unsigned | Unsigned |
+    |                        |       |        |  input   |  output  |
+    |------------------------|-------|--------|----------|----------|
+    | signed 8 to 8          | int8  | int8   |  false   |  false   |
+    | signed 8 to 16         | int8  | int16  |  false   |  false   |
+    | signed 8 to 32         | int8  | int32  |  false   |  false   |
+    | signed 16 to 8         | int16 | int8   |  false   |  false   |
+    | signed 16 to 16        | int16 | int16  |  false   |  false   |
+    | signed 16 to 32        | int16 | int32  |  false   |  false   |
+    | signed 32 to 8         | int32 | int8   |  false   |  false   |
+    | signed 32 to 16        | int32 | int16  |  false   |  false   |
+    | signed 32 to 32        | int32 | int32  |  false   |  false   |
+    | signed 48 to 8         | int48 | int8   |  false   |  false   |
+    | signed 48 to 16        | int48 | int16  |  false   |  false   |
+    | signed 48 to 32        | int48 | int32  |  false   |  false   |
+    | unsigned 8 to signed 8 | uint8 | int8   |  true    |  false   |
+    | signed 8 to unsigned 8 | int8  | uint8  |  false   |  true    |
   }];
 
   let arguments = (ins
@@ -1895,7 +1896,9 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
     DenseI8ArrayAttr:$shift,
     BoolAttr:$scale32,
     BoolAttr:$double_round,
-    BoolAttr:$per_channel
+    BoolAttr:$per_channel,
+    DefaultValuedOptionalAttr<BoolAttr, "false">:$input_unsigned,
+    DefaultValuedOptionalAttr<BoolAttr, "false">:$output_unsigned
   );
 
   let results = (outs
@@ -1992,9 +1995,9 @@ def Tosa_CustomOp : Tosa_Op<"custom"> {
     `implementation_attrs` is a string which is a backend and identifier specific
     set of attributes to the custom operator.
 
-    `inputs` is the set of tensor inputs to the custom operator.
+    `input_list` is the set of tensor inputs to the custom operator.
 
-    `outputs is the list of tensors returned by the operator. The number of operators
+    `output_list` is the list of tensors returned by the operator. The number of operators
     is backend specific.
 
     Example:
@@ -2010,11 +2013,11 @@ def Tosa_CustomOp : Tosa_Op<"custom"> {
     StrAttr:$operator_name,
     StrAttr:$domain_name,
     StrAttr:$implementation_attrs,
-    Variadic<Tosa_Tensor>:$inputs
+    Variadic<Tosa_Tensor>:$input_list
   );
 
   let results = (outs
-    Variadic<Tosa_Tensor>:$outputs
+    Variadic<Tosa_Tensor>:$output_list
   );
 
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
diff --git a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
index 89bc57f09ec8b..3fa35bf1851a9 100644
--- a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
@@ -338,7 +338,7 @@ struct ComposeExpandOfCollapseOp : public OpRewritePattern<ExpandOpTy> {
 
     int64_t srcRank = srcType.getRank();
     int64_t resultRank = resultType.getRank();
-    if (srcType == resultType)
+    if (srcRank == resultRank)
       return failure();
 
     auto srcReassociation = collapseOp.getReassociationIndices();
@@ -388,12 +388,14 @@ struct ComposeExpandOfCollapseOp : public OpRewritePattern<ExpandOpTy> {
           resultShape.slice(resultIndices.front(), resultIndices.size());
 
       if (srcSubShape.size() == resultSubShape.size()) {
-        if (srcSubShape == resultSubShape &&
-            llvm::count_if(srcSubShape, ShapedType::isDynamic) < 2) {
-          composedReassociation.push_back(srcIndices);
-        } else {
+        if (srcSubShape != resultSubShape ||
+            llvm::count_if(srcSubShape, ShapedType::isDynamic) >= 2) {
           return std::nullopt;
         }
+        for (auto index : llvm::seq<int64_t>(0, srcSubShape.size())) {
+          composedReassociation.emplace_back(1, srcIndices.front() + index);
+        }
+        continue;
       }
 
       // Find reassociation to collapse `srcSubShape` into `resultSubShape`.
@@ -403,11 +405,11 @@ struct ComposeExpandOfCollapseOp : public OpRewritePattern<ExpandOpTy> {
         return std::nullopt;
 
       // Remap the subshape indices back to the original srcShape.
-      for (auto &subshape_indices : *subShapeReassociation) {
-        ReassociationIndices shape_indices;
-        for (int64_t index : subshape_indices)
-          shape_indices.push_back(srcIndices.front() + index);
-        composedReassociation.push_back(shape_indices);
+      for (auto &subshapeIndices : *subShapeReassociation) {
+        ReassociationIndices shapeIndices;
+        for (int64_t index : subshapeIndices)
+          shapeIndices.push_back(srcIndices.front() + index);
+        composedReassociation.push_back(shapeIndices);
       }
     }
     return {std::move(composedReassociation)};
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 88c1b94412241..30a5b06374fad 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -760,7 +760,7 @@ def Vector_FMAOp :
   let summary = "vector fused multiply-add";
   let description = [{
     Multiply-add expressions operate on n-D vectors and compute a fused
-    pointwise multiply-and-accumulate: `$result = `$lhs * $rhs + $acc`.
+    pointwise multiply-and-accumulate: `$result = $lhs * $rhs + $acc`.
     All operands and result have the same vector type. The semantics
     of the operation correspond to those of the `llvm.fma`
     [intrinsic](https://llvm.org/docs/LangRef.html#int-fma). In the
@@ -806,10 +806,12 @@ def Vector_FromElementsOp : Vector_Op<"from_elements", [
     // [[[%f1, %f2]], [[%f3, %f4]], [[%f5, %f6]]]
     %3 = vector.from_elements %f1, %f2, %f3, %f4, %f5, %f6 : vector<3x1x2xf32>
     ```
+
+    Note, scalable vectors are not supported.
   }];
 
   let arguments = (ins Variadic<AnyType>:$elements);
-  let results = (outs AnyVectorOfAnyRank:$result);
+  let results = (outs AnyFixedVectorOfAnyRank:$result);
   let assemblyFormat = "$elements attr-dict `:` type($result)";
   let hasCanonicalizer = 1;
 }
@@ -2084,7 +2086,7 @@ def Vector_ExpandLoadOp :
   Vector_Op<"expandload">,
     Arguments<(ins Arg<AnyMemRef, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
-               VectorOfNonZeroRankOf<[I1]>:$mask,
+               FixedVectorOfNonZeroRankOf<[I1]>:$mask,
                AnyVectorOfNonZeroRank:$pass_thru)>,
     Results<(outs AnyVectorOfNonZeroRank:$result)> {
 
@@ -2117,6 +2119,8 @@ def Vector_ExpandLoadOp :
     correspond to those of the `llvm.masked.expandload`
     [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics).
 
+    Note, at the moment this Op is only available for fixed-width vectors.
+
     Examples:
 
     ```mlir
@@ -2151,7 +2155,7 @@ def Vector_CompressStoreOp :
   Vector_Op<"compressstore">,
     Arguments<(ins Arg<AnyMemRef, "", [MemWrite]>:$base,
                Variadic<Index>:$indices,
-               VectorOfNonZeroRankOf<[I1]>:$mask,
+               FixedVectorOfNonZeroRankOf<[I1]>:$mask,
                AnyVectorOfNonZeroRank:$valueToStore)> {
 
   let summary = "writes elements selectively from a vector as defined by a mask";
@@ -2183,6 +2187,8 @@ def Vector_CompressStoreOp :
     correspond to those of the `llvm.masked.compressstore`
     [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics).
 
+    Note, at the moment this Op is only available for fixed-width vectors.
+
     Examples:
 
     ```mlir
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index 25535408f4528..7f9c470ffec30 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -401,6 +401,38 @@ enum class SliceVerificationResult {
 SliceVerificationResult isRankReducedType(ShapedType originalType,
                                           ShapedType candidateReducedType);
 
+//===----------------------------------------------------------------------===//
+// Convenience wrappers for VectorType
+//
+// These are provided to allow idiomatic code like:
+//  * isa<vector::ScalableVectorType>(type)
+//===----------------------------------------------------------------------===//
+/// A vector type containing at least one scalable dimension.
+class ScalableVectorType : public VectorType {
+public:
+  using VectorType::VectorType;
+
+  static bool classof(Type type) {
+    auto vecTy = llvm::dyn_cast<VectorType>(type);
+    if (!vecTy)
+      return false;
+    return vecTy.isScalable();
+  }
+};
+
+/// A vector type with no scalable dimensions.
+class FixedVectorType : public VectorType {
+public:
+  using VectorType::VectorType;
+
+  static bool classof(Type type) {
+    auto vecTy = llvm::dyn_cast<VectorType>(type);
+    if (!vecTy)
+      return false;
+    return !vecTy.isScalable();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // Deferred Method Definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index fc4383d08422c..b9f8c1ed19470 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -24,19 +24,19 @@ include "mlir/IR/DialectBase.td"
 // Explicitly disallow 0-D vectors for now until we have good enough coverage.
 def IsVectorOfNonZeroRankTypePred : And<[CPred<"::llvm::isa<::mlir::VectorType>($_self)">,
                                          CPred<"::llvm::cast<::mlir::VectorType>($_self).getRank() > 0">]>;
+def IsFixedVectorOfNonZeroRankTypePred : And<[CPred<"::llvm::isa<::mlir::FixedVectorType>($_self)">,
+                                              CPred<"::llvm::cast<::mlir::VectorType>($_self).getRank() > 0">]>;
 
 // Temporary vector type clone that allows gradual transition to 0-D vectors.
 // TODO: Remove this when all ops support 0-D vectors.
 def IsVectorOfAnyRankTypePred : CPred<"::llvm::isa<::mlir::VectorType>($_self)">;
 
 // Whether a type is a fixed-length VectorType.
-def IsFixedVectorOfAnyRankTypePred : CPred<[{::llvm::isa<::mlir::VectorType>($_self) &&
-                                  !::llvm::cast<VectorType>($_self).isScalable()}]>;
+def IsFixedVectorOfAnyRankTypePred : CPred<[{::llvm::isa<::mlir::FixedVectorType>($_self)}]>;
 
 // Whether a type is a scalable VectorType.
 def IsVectorTypeWithAnyDimScalablePred
-        : CPred<[{::llvm::isa<::mlir::VectorType>($_self) &&
-                  ::llvm::cast<VectorType>($_self).isScalable()}]>;
+        : CPred<[{::llvm::isa<::mlir::ScalableVectorType>($_self)}]>;
 
 // Whether a type is a scalable VectorType, with a single trailing scalable dimension.
 // Examples:
@@ -432,6 +432,10 @@ class VectorOfNonZeroRankOf<list<Type> allowedTypes> :
   ShapedContainerType<allowedTypes, IsVectorOfNonZeroRankTypePred, "vector",
                       "::mlir::VectorType">;
 
+class FixedVectorOfNonZeroRankOf<list<Type> allowedTypes> :
+  ShapedContainerType<allowedTypes, IsFixedVectorOfNonZeroRankTypePred,
+                      "fixed-length vector", "::mlir::VectorType">;
+
 // Temporary vector type clone that allows gradual transition to 0-D vectors.
 // TODO: Remove this when all ops support 0-D vectors.
 class VectorOfAnyRankOf<list<Type> allowedTypes> :
@@ -660,6 +664,8 @@ class VectorWithTrailingDimScalableOfSizeAndType<list<int> allowedTrailingSizes,
 // Unlike the following definitions, this one excludes 0-D vectors
 def AnyVectorOfNonZeroRank : VectorOfNonZeroRankOf<[AnyType]>;
 
+def AnyFixedVectorOfNonZeroRank : FixedVectorOfNonZeroRankOf<[AnyType]>;
+
 def AnyVectorOfAnyRank : VectorOfAnyRankOf<[AnyType]>;
 
 def AnyFixedVectorOfAnyRank : FixedVectorOfAnyRank<[AnyType]>;
diff --git a/mlir/include/mlir/IR/Matchers.h b/mlir/include/mlir/IR/Matchers.h
index 226afb9ad25f1..d218206e50f8f 100644
--- a/mlir/include/mlir/IR/Matchers.h
+++ b/mlir/include/mlir/IR/Matchers.h
@@ -417,6 +417,11 @@ inline detail::constant_float_predicate_matcher m_OneFloat() {
   }};
 }
 
+/// Matches a constant scalar / vector splat / tensor splat float ones.
+inline detail::constant_float_predicate_matcher m_NaNFloat() {
+  return {[](const APFloat &value) { return value.isNaN(); }};
+}
+
 /// Matches a constant scalar / vector splat / tensor splat float positive
 /// infinity.
 inline detail::constant_float_predicate_matcher m_PosInfFloat() {
@@ -433,12 +438,6 @@ inline detail::constant_float_predicate_matcher m_NegInfFloat() {
   }};
 }
 
-/// Matches a constant scalar / vector splat / tensor splat with denormal
-/// values.
-inline detail::constant_float_predicate_matcher m_isDenormalFloat() {
-  return {[](const APFloat &value) { return value.isDenormal(); }};
-}
-
 /// Matches a constant scalar / vector splat / tensor splat integer zero.
 inline detail::constant_int_predicate_matcher m_Zero() {
   return {[](const APInt &value) { return 0 == value; }};
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index a7222794f320b..6c1ff4d0e5e6b 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -1604,7 +1604,8 @@ class OpAsmParser : public AsmParser {
     size_t typeSize = llvm::range_size(types);
     if (operandSize != typeSize)
       return emitError(loc)
-             << operandSize << " operands present, but expected " << typeSize;
+             << "number of operands and types do not match: got " << operandSize
+             << " operands and " << typeSize << " types";
 
     for (auto [operand, type] : llvm::zip_equal(operands, types))
       if (resolveOperand(operand, type, result))
diff --git a/mlir/include/mlir/IR/VectorTypes.h b/mlir/include/mlir/IR/VectorTypes.h
new file mode 100644
index 0000000000000..1f1d0f7a30669
--- /dev/null
+++ b/mlir/include/mlir/IR/VectorTypes.h
@@ -0,0 +1,12 @@
+//===- VectorTypes.h - MLIR Vector Types ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Convenience wrappers for `VectorType` to allow idiomatic code like
+//  * isa<vector::ScalableVectorType>(type)
+//
+//===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Interfaces/DestinationStyleOpInterface.td b/mlir/include/mlir/Interfaces/DestinationStyleOpInterface.td
index bc77e688ba46b..9c0db9b3a5107 100644
--- a/mlir/include/mlir/Interfaces/DestinationStyleOpInterface.td
+++ b/mlir/include/mlir/Interfaces/DestinationStyleOpInterface.td
@@ -82,7 +82,7 @@ def DestinationStyleOpInterface : OpInterface<"DestinationStyleOpInterface"> {
       $_op->setOperand($_op.getDpsInits().getBeginOperandIndex() + i, value);
     }
 
-    /// Return the number of DPS inits.
+    /// Return the number of DPS inputs.
     int64_t getNumDpsInputs() {
       return $_op->getNumOperands() - $_op.getNumDpsInits();
     }
diff --git a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
index 0e7ee9c89cb47..160585e7da548 100644
--- a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
+++ b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
@@ -31,6 +31,13 @@ class DialectRegistry;
 class PassPipelineCLParser;
 class PassManager;
 
+/// enum class to indicate the verbosity level of the diagnostic filter.
+enum class VerbosityLevel {
+  ErrorsOnly = 0,
+  ErrorsAndWarnings,
+  ErrorsWarningsAndRemarks
+};
+
 /// Configuration options for the mlir-opt tool.
 /// This is intended to help building tools like mlir-opt by collecting the
 /// supported options.
@@ -74,6 +81,11 @@ class MlirOptMainConfig {
     dumpPassPipelineFlag = dump;
     return *this;
   }
+
+  VerbosityLevel getDiagnosticVerbosityLevel() const {
+    return diagnosticVerbosityLevelFlag;
+  }
+
   bool shouldDumpPassPipeline() const { return dumpPassPipelineFlag; }
 
   /// Set the output format to bytecode instead of textual IR.
@@ -82,10 +94,13 @@ class MlirOptMainConfig {
     return *this;
   }
   bool shouldEmitBytecode() const { return emitBytecodeFlag; }
+
   bool shouldElideResourceDataFromBytecode() const {
     return elideResourceDataFromBytecodeFlag;
   }
 
+  bool shouldShowNotes() const { return !disableDiagnosticNotesFlag; }
+
   /// Set the IRDL file to load before processing the input.
   MlirOptMainConfig &setIrdlFile(StringRef file) {
     irdlFileFlag = file;
@@ -209,6 +224,11 @@ class MlirOptMainConfig {
   /// Configuration for the debugging hooks.
   tracing::DebugConfig debugConfig;
 
+  /// Verbosity level of diagnostic information. 0: Errors only,
+  /// 1: Errors and warnings, 2: Errors, warnings and remarks.
+  VerbosityLevel diagnosticVerbosityLevelFlag =
+      VerbosityLevel::ErrorsWarningsAndRemarks;
+
   /// Print the pipeline that will be run.
   bool dumpPassPipelineFlag = false;
 
@@ -242,6 +262,11 @@ class MlirOptMainConfig {
   /// Show the registered dialects before trying to load the input file.
   bool showDialectsFlag = false;
 
+  /// Show the notes in diagnostic information. Notes can be included in
+  /// any diagnostic information, so it is not specified in the verbosity
+  /// level.
+  bool disableDiagnosticNotesFlag = true;
+
   /// Split the input file based on the given marker into chunks and process
   /// each chunk independently. Input is not split if empty.
   std::string splitInputFileFlag = "";
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index de47765006f81..28150e886913e 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -189,9 +189,9 @@ class TypeConverter {
   }
 
   /// This method registers a materialization that will be called when
-  /// converting a legal replacement value back to an illegal source type.
-  /// This is used when some uses of the original, illegal value must persist
-  /// beyond the main conversion.
+  /// converting a replacement value back to its original source type.
+  /// This is used when some uses of the original value persist beyond the main
+  /// conversion.
   template <typename FnT, typename T = typename llvm::function_traits<
                               std::decay_t<FnT>>::template arg_t<1>>
   void addSourceMaterialization(FnT &&callback) {
@@ -200,17 +200,18 @@ class TypeConverter {
   }
 
   /// This method registers a materialization that will be called when
-  /// converting an illegal (source) value to a legal (target) type.
+  /// converting a value to a target type according to a pattern's type
+  /// converter.
   ///
-  /// Note: For target materializations, users can optionally take the original
-  /// type. This type may be different from the type of the input. For example,
-  /// let's assume that a conversion pattern "P1" replaced an SSA value "v1"
-  /// (type "t1") with "v2" (type "t2"). Then a different conversion pattern
-  /// "P2" matches an op that has "v1" as an operand. Let's furthermore assume
-  /// that "P2" determines that the legalized type of "t1" is "t3", which may
-  /// be different from "t2". In this example, the target materialization
-  /// will be invoked with: outputType = "t3", inputs = "v2",
-  // originalType = "t1". Note  that the original type "t1" cannot be recovered
+  /// Note: Target materializations can optionally inspect the "original"
+  /// type. This type may be different from the type of the input value.
+  /// For example, let's assume that a conversion pattern "P1" replaced an SSA
+  /// value "v1" (type "t1") with "v2" (type "t2"). Then a different conversion
+  /// pattern "P2" matches an op that has "v1" as an operand. Let's furthermore
+  /// assume that "P2" determines that the converted target type of "t1" is
+  /// "t3", which may be different from "t2". In this example, the target
+  /// materialization will be invoked with: outputType = "t3", inputs = "v2",
+  /// originalType = "t1". Note that the original type "t1" cannot be recovered
   /// from just "t3" and "v2"; that's why the originalType parameter exists.
   ///
   /// Note: During a 1:N conversion, the result types can be a TypeRange. In
@@ -537,8 +538,15 @@ class ConversionPattern : public RewritePattern {
                        ConversionPatternRewriter &rewriter) const {
     llvm_unreachable("unimplemented rewrite");
   }
+  virtual void rewrite(Operation *op, ArrayRef<ValueRange> operands,
+                       ConversionPatternRewriter &rewriter) const {
+    rewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+  }
 
   /// Hook for derived classes to implement combined matching and rewriting.
+  /// This overload supports only 1:1 replacements. The 1:N overload is called
+  /// by the driver. By default, it calls this 1:1 overload or reports a fatal
+  /// error if 1:N replacements were found.
   virtual LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const {
@@ -548,6 +556,14 @@ class ConversionPattern : public RewritePattern {
     return success();
   }
 
+  /// Hook for derived classes to implement combined matching and rewriting.
+  /// This overload supports 1:N replacements.
+  virtual LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const {
+    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+  }
+
   /// Attempt to match and rewrite the IR root at the specified operation.
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const final;
@@ -574,6 +590,15 @@ class ConversionPattern : public RewritePattern {
       : RewritePattern(std::forward<Args>(args)...),
         typeConverter(&typeConverter) {}
 
+  /// Given an array of value ranges, which are the inputs to a 1:N adaptor,
+  /// try to extract the single value of each range to construct a the inputs
+  /// for a 1:1 adaptor.
+  ///
+  /// This function produces a fatal error if at least one range has 0 or
+  /// more than 1 value: "pattern 'name' does not support 1:N conversion"
+  SmallVector<Value>
+  getOneToOneAdaptorOperands(ArrayRef<ValueRange> operands) const;
+
 protected:
   /// An optional type converter for use by this pattern.
   const TypeConverter *typeConverter = nullptr;
@@ -589,6 +614,8 @@ template <typename SourceOp>
 class OpConversionPattern : public ConversionPattern {
 public:
   using OpAdaptor = typename SourceOp::Adaptor;
+  using OneToNOpAdaptor =
+      typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>>;
 
   OpConversionPattern(MLIRContext *context, PatternBenefit benefit = 1)
       : ConversionPattern(SourceOp::getOperationName(), benefit, context) {}
@@ -607,12 +634,24 @@ class OpConversionPattern : public ConversionPattern {
     auto sourceOp = cast<SourceOp>(op);
     rewrite(sourceOp, OpAdaptor(operands, sourceOp), rewriter);
   }
+  void rewrite(Operation *op, ArrayRef<ValueRange> operands,
+               ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = cast<SourceOp>(op);
+    rewrite(sourceOp, OneToNOpAdaptor(operands, sourceOp), rewriter);
+  }
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
     auto sourceOp = cast<SourceOp>(op);
     return matchAndRewrite(sourceOp, OpAdaptor(operands, sourceOp), rewriter);
   }
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    auto sourceOp = cast<SourceOp>(op);
+    return matchAndRewrite(sourceOp, OneToNOpAdaptor(operands, sourceOp),
+                           rewriter);
+  }
 
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
   /// overridden by the derived pattern class.
@@ -623,6 +662,12 @@ class OpConversionPattern : public ConversionPattern {
                        ConversionPatternRewriter &rewriter) const {
     llvm_unreachable("must override matchAndRewrite or a rewrite method");
   }
+  virtual void rewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                       ConversionPatternRewriter &rewriter) const {
+    SmallVector<Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    rewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
   virtual LogicalResult
   matchAndRewrite(SourceOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const {
@@ -631,6 +676,13 @@ class OpConversionPattern : public ConversionPattern {
     rewrite(op, adaptor, rewriter);
     return success();
   }
+  virtual LogicalResult
+  matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const {
+    SmallVector<Value> oneToOneOperands =
+        getOneToOneAdaptorOperands(adaptor.getOperands());
+    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+  }
 
 private:
   using ConversionPattern::matchAndRewrite;
@@ -656,11 +708,20 @@ class OpInterfaceConversionPattern : public ConversionPattern {
                ConversionPatternRewriter &rewriter) const final {
     rewrite(cast<SourceOp>(op), operands, rewriter);
   }
+  void rewrite(Operation *op, ArrayRef<ValueRange> operands,
+               ConversionPatternRewriter &rewriter) const final {
+    rewrite(cast<SourceOp>(op), operands, rewriter);
+  }
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
     return matchAndRewrite(cast<SourceOp>(op), operands, rewriter);
   }
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    return matchAndRewrite(cast<SourceOp>(op), operands, rewriter);
+  }
 
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
   /// overridden by the derived pattern class.
@@ -668,6 +729,10 @@ class OpInterfaceConversionPattern : public ConversionPattern {
                        ConversionPatternRewriter &rewriter) const {
     llvm_unreachable("must override matchAndRewrite or a rewrite method");
   }
+  virtual void rewrite(SourceOp op, ArrayRef<ValueRange> operands,
+                       ConversionPatternRewriter &rewriter) const {
+    rewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+  }
   virtual LogicalResult
   matchAndRewrite(SourceOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const {
@@ -676,6 +741,11 @@ class OpInterfaceConversionPattern : public ConversionPattern {
     rewrite(op, operands, rewriter);
     return success();
   }
+  virtual LogicalResult
+  matchAndRewrite(SourceOp op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const {
+    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+  }
 
 private:
   using ConversionPattern::matchAndRewrite;
diff --git a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
index beb9f3e82e22d..3ed77998c92ad 100644
--- a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
+++ b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
@@ -65,6 +65,8 @@ void MultiAffineFunction::print(raw_ostream &os) const {
   output.print(os);
 }
 
+void MultiAffineFunction::dump() const { print(llvm::errs()); }
+
 SmallVector<DynamicAPInt, 8>
 MultiAffineFunction::valueAt(ArrayRef<DynamicAPInt> point) const {
   assert(point.size() == getNumDomainVars() + getNumSymbolVars() &&
diff --git a/mlir/lib/Bindings/Python/DialectLLVM.cpp b/mlir/lib/Bindings/Python/DialectLLVM.cpp
index 42a4c8c0793ba..cccf1370b8cc8 100644
--- a/mlir/lib/Bindings/Python/DialectLLVM.cpp
+++ b/mlir/lib/Bindings/Python/DialectLLVM.cpp
@@ -6,11 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <string>
+
 #include "mlir-c/Dialect/LLVM.h"
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
+#include "mlir/Bindings/Python/Diagnostics.h"
 #include "mlir/Bindings/Python/PybindAdaptors.h"
-#include <string>
 
 namespace py = pybind11;
 using namespace llvm;
diff --git a/mlir/lib/Bindings/Python/Pass.cpp b/mlir/lib/Bindings/Python/Pass.cpp
index e8d28abe6d583..e991deaae2daa 100644
--- a/mlir/lib/Bindings/Python/Pass.cpp
+++ b/mlir/lib/Bindings/Python/Pass.cpp
@@ -76,20 +76,33 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
           "enable_ir_printing",
           [](PyPassManager &passManager, bool printBeforeAll,
              bool printAfterAll, bool printModuleScope, bool printAfterChange,
-             bool printAfterFailure,
+             bool printAfterFailure, std::optional<int64_t> largeElementsLimit,
+             bool enableDebugInfo, bool printGenericOpForm,
              std::optional<std::string> optionalTreePrintingPath) {
+            MlirOpPrintingFlags flags = mlirOpPrintingFlagsCreate();
+            if (largeElementsLimit)
+              mlirOpPrintingFlagsElideLargeElementsAttrs(flags,
+                                                         *largeElementsLimit);
+            if (enableDebugInfo)
+              mlirOpPrintingFlagsEnableDebugInfo(flags, /*enable=*/true,
+                                                 /*prettyForm=*/false);
+            if (printGenericOpForm)
+              mlirOpPrintingFlagsPrintGenericOpForm(flags);
             std::string treePrintingPath = "";
             if (optionalTreePrintingPath.has_value())
               treePrintingPath = optionalTreePrintingPath.value();
             mlirPassManagerEnableIRPrinting(
                 passManager.get(), printBeforeAll, printAfterAll,
-                printModuleScope, printAfterChange, printAfterFailure,
+                printModuleScope, printAfterChange, printAfterFailure, flags,
                 mlirStringRefCreate(treePrintingPath.data(),
                                     treePrintingPath.size()));
+            mlirOpPrintingFlagsDestroy(flags);
           },
           "print_before_all"_a = false, "print_after_all"_a = true,
           "print_module_scope"_a = false, "print_after_change"_a = false,
           "print_after_failure"_a = false,
+          "large_elements_limit"_a = py::none(), "enable_debug_info"_a = false,
+          "print_generic_op_form"_a = false,
           "tree_printing_dir_path"_a = py::none(),
           "Enable IR printing, default as mlir-print-ir-after-all.")
       .def(
diff --git a/mlir/lib/Bindings/Python/TransformInterpreter.cpp b/mlir/lib/Bindings/Python/TransformInterpreter.cpp
index f6b4532b1b6be..0c8c0e0a965aa 100644
--- a/mlir/lib/Bindings/Python/TransformInterpreter.cpp
+++ b/mlir/lib/Bindings/Python/TransformInterpreter.cpp
@@ -10,14 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <pybind11/detail/common.h>
+#include <pybind11/pybind11.h>
+
 #include "mlir-c/Dialect/Transform/Interpreter.h"
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
+#include "mlir/Bindings/Python/Diagnostics.h"
 #include "mlir/Bindings/Python/PybindAdaptors.h"
 
-#include <pybind11/detail/common.h>
-#include <pybind11/pybind11.h>
-
 namespace py = pybind11;
 
 namespace {
diff --git a/mlir/lib/CAPI/IR/Pass.cpp b/mlir/lib/CAPI/IR/Pass.cpp
index 01151eafeb526..883b7e8bb832d 100644
--- a/mlir/lib/CAPI/IR/Pass.cpp
+++ b/mlir/lib/CAPI/IR/Pass.cpp
@@ -49,6 +49,7 @@ void mlirPassManagerEnableIRPrinting(MlirPassManager passManager,
                                      bool printModuleScope,
                                      bool printAfterOnlyOnChange,
                                      bool printAfterOnlyOnFailure,
+                                     MlirOpPrintingFlags flags,
                                      MlirStringRef treePrintingPath) {
   auto shouldPrintBeforePass = [printBeforeAll](Pass *, Operation *) {
     return printBeforeAll;
@@ -60,13 +61,14 @@ void mlirPassManagerEnableIRPrinting(MlirPassManager passManager,
     return unwrap(passManager)
         ->enableIRPrinting(shouldPrintBeforePass, shouldPrintAfterPass,
                            printModuleScope, printAfterOnlyOnChange,
-                           printAfterOnlyOnFailure);
+                           printAfterOnlyOnFailure, /*out=*/llvm::errs(),
+                           *unwrap(flags));
 
   unwrap(passManager)
       ->enableIRPrintingToFileTree(shouldPrintBeforePass, shouldPrintAfterPass,
                                    printModuleScope, printAfterOnlyOnChange,
                                    printAfterOnlyOnFailure,
-                                   unwrap(treePrintingPath));
+                                   unwrap(treePrintingPath), *unwrap(flags));
 }
 
 void mlirPassManagerEnableVerifier(MlirPassManager passManager, bool enable) {
diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
index 50384d9a08e5d..ccbc1669b7a92 100644
--- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
+++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp
@@ -674,7 +674,7 @@ class FtoICastOpConversion : public OpConversionPattern<CastOp> {
     Type actualResultType = dstType;
     if (isa<arith::FPToUIOp>(castOp)) {
       actualResultType =
-          rewriter.getIntegerType(operandType.getIntOrFloatBitWidth(),
+          rewriter.getIntegerType(dstType.getIntOrFloatBitWidth(),
                                   /*isSigned=*/false);
     }
 
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index 54d941ae9f6c8..aac24f113d891 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -53,40 +53,13 @@ struct ConstrainedVectorConvertToLLVMPattern
   }
 };
 
-template <typename SourceOp, typename TargetOp,
-          template <typename, typename> typename AttrConvert =
-              AttrConvertPassThrough>
-struct DenormalOpConversionToLLVMPattern
-    : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert> {
-  using VectorConvertToLLVMPattern<SourceOp, TargetOp,
-                                   AttrConvert>::VectorConvertToLLVMPattern;
-
-  LogicalResult
-  matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const override {
-    // TODO: Here, we need a legalization step. LLVM provides a function-level
-    // attribute for denormal; here, we need to move this information from the
-    // operation to the function, making sure all the operations in the same
-    // function are consistent.
-    if (op.getDenormalModeAttr().getValue() != arith::DenormalMode::ieee)
-      return rewriter.notifyMatchFailure(
-          op, "only ieee denormal mode is supported at the moment");
-
-    StringRef arithDenormalAttrName = SourceOp::getDenormalModeAttrName();
-    op->removeAttr(arithDenormalAttrName);
-    return VectorConvertToLLVMPattern<SourceOp, TargetOp,
-                                      AttrConvert>::matchAndRewrite(op, adaptor,
-                                                                    rewriter);
-  }
-};
-
 //===----------------------------------------------------------------------===//
 // Straightforward Op Lowerings
 //===----------------------------------------------------------------------===//
 
 using AddFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using AddIOpLowering =
     VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp,
                                arith::AttrConvertOverflowToLLVM>;
@@ -94,8 +67,8 @@ using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>;
 using BitcastOpLowering =
     VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>;
 using DivFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using DivSIOpLowering =
     VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>;
 using DivUIOpLowering =
@@ -110,38 +83,38 @@ using FPToSIOpLowering =
 using FPToUIOpLowering =
     VectorConvertToLLVMPattern<arith::FPToUIOp, LLVM::FPToUIOp>;
 using MaximumFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using MaxNumFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using MaxSIOpLowering =
     VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>;
 using MaxUIOpLowering =
     VectorConvertToLLVMPattern<arith::MaxUIOp, LLVM::UMaxOp>;
 using MinimumFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using MinNumFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using MinSIOpLowering =
     VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>;
 using MinUIOpLowering =
     VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>;
 using MulFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using MulIOpLowering =
     VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp,
                                arith::AttrConvertOverflowToLLVM>;
 using NegFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::NegFOp, LLVM::FNegOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::NegFOp, LLVM::FNegOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using OrIOpLowering = VectorConvertToLLVMPattern<arith::OrIOp, LLVM::OrOp>;
 using RemFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::RemFOp, LLVM::FRemOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::RemFOp, LLVM::FRemOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using RemSIOpLowering =
     VectorConvertToLLVMPattern<arith::RemSIOp, LLVM::SRemOp>;
 using RemUIOpLowering =
@@ -158,8 +131,8 @@ using ShRUIOpLowering =
 using SIToFPOpLowering =
     VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp>;
 using SubFOpLowering =
-    DenormalOpConversionToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
-                                      arith::AttrConvertFastMathToLLVM>;
+    VectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
+                               arith::AttrConvertFastMathToLLVM>;
 using SubIOpLowering =
     VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp,
                                arith::AttrConvertOverflowToLLVM>;
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 6651d87162257..62461c0cea08a 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -41,6 +41,7 @@ add_subdirectory(MathToSPIRV)
 add_subdirectory(MemRefToEmitC)
 add_subdirectory(MemRefToLLVM)
 add_subdirectory(MemRefToSPIRV)
+add_subdirectory(MeshToMPI)
 add_subdirectory(NVGPUToNVVM)
 add_subdirectory(NVVMToLLVM)
 add_subdirectory(OpenACCToSCF)
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/CMakeLists.txt b/mlir/lib/Conversion/GPUToLLVMSPV/CMakeLists.txt
index d47c5e679d86e..f2381c623e201 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/CMakeLists.txt
@@ -10,5 +10,4 @@ add_mlir_conversion_library(MLIRGPUToLLVMSPV
   MLIRLLVMCommonConversion
   MLIRLLVMDialect
   MLIRSPIRVAttrToLLVMConversion
-  MLIRSPIRVDialect
 )
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index bb6a38c0e76ed..03745f4537e99 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -20,9 +20,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
-#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
@@ -274,10 +272,11 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
   }
 
   /// Get the subgroup size from the target or return a default.
-  static int getSubgroupSize(Operation *op) {
-    return spirv::lookupTargetEnvOrDefault(op)
-        .getResourceLimits()
-        .getSubgroupSize();
+  static std::optional<int> getSubgroupSize(Operation *op) {
+    auto parentFunc = op->getParentOfType<LLVM::LLVMFuncOp>();
+    if (!parentFunc)
+      return std::nullopt;
+    return parentFunc.getIntelReqdSubGroupSize();
   }
 
   static bool hasValidWidth(gpu::ShuffleOp op) {
diff --git a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
index df197f820604a..1debf4e770569 100644
--- a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
@@ -18,6 +18,7 @@ add_mlir_conversion_library(MLIRGPUToNVVMTransforms
   MLIRLLVMCommonConversion
   MLIRLLVMDialect
   MLIRMemRefToLLVM
+  MLIRNVGPUDialect
   MLIRNVVMDialect
   MLIRPass
   MLIRTransformUtils
diff --git a/mlir/lib/Conversion/MeshToMPI/CMakeLists.txt b/mlir/lib/Conversion/MeshToMPI/CMakeLists.txt
new file mode 100644
index 0000000000000..95815a683f6d6
--- /dev/null
+++ b/mlir/lib/Conversion/MeshToMPI/CMakeLists.txt
@@ -0,0 +1,22 @@
+add_mlir_conversion_library(MLIRMeshToMPI
+  MeshToMPI.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/MeshToMPI
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRFuncDialect
+  MLIRIR
+  MLIRLinalgTransforms
+  MLIRMemRefDialect
+  MLIRPass
+  MLIRMeshDialect
+  MLIRMPIDialect
+  MLIRTransforms
+  )
diff --git a/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
new file mode 100644
index 0000000000000..6dd89ecf4d5c2
--- /dev/null
+++ b/mlir/lib/Conversion/MeshToMPI/MeshToMPI.cpp
@@ -0,0 +1,440 @@
+//===- MeshToMPI.cpp - Mesh to MPI  dialect conversion -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a translation of Mesh communication ops tp MPI ops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/MeshToMPI/MeshToMPI.h"
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
+#include "mlir/Dialect/MPI/IR/MPI.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/Mesh/IR/MeshOps.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/SymbolTable.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#define DEBUG_TYPE "mesh-to-mpi"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTMESHTOMPIPASS
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::mesh;
+
+namespace {
+// Create operations converting a linear index to a multi-dimensional index
+static SmallVector<Value> linearToMultiIndex(Location loc, OpBuilder b,
+                                             Value linearIndex,
+                                             ValueRange dimensions) {
+  int n = dimensions.size();
+  SmallVector<Value> multiIndex(n);
+
+  for (int i = n - 1; i >= 0; --i) {
+    multiIndex[i] = b.create<arith::RemSIOp>(loc, linearIndex, dimensions[i]);
+    if (i > 0) {
+      linearIndex = b.create<arith::DivSIOp>(loc, linearIndex, dimensions[i]);
+    }
+  }
+
+  return multiIndex;
+}
+
+// Create operations converting a multi-dimensional index to a linear index
+Value multiToLinearIndex(Location loc, OpBuilder b, ValueRange multiIndex,
+                         ValueRange dimensions) {
+
+  auto linearIndex = b.create<arith::ConstantIndexOp>(loc, 0).getResult();
+  auto stride = b.create<arith::ConstantIndexOp>(loc, 1).getResult();
+
+  for (int i = multiIndex.size() - 1; i >= 0; --i) {
+    auto off = b.create<arith::MulIOp>(loc, multiIndex[i], stride);
+    linearIndex = b.create<arith::AddIOp>(loc, linearIndex, off);
+    stride = b.create<arith::MulIOp>(loc, stride, dimensions[i]);
+  }
+
+  return linearIndex;
+}
+
+struct ConvertProcessMultiIndexOp
+    : public mlir::OpRewritePattern<mlir::mesh::ProcessMultiIndexOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::mesh::ProcessMultiIndexOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    // Currently converts its linear index to a multi-dimensional index.
+
+    SymbolTableCollection symbolTableCollection;
+    auto loc = op.getLoc();
+    auto meshOp = getMesh(op, symbolTableCollection);
+    // For now we only support static mesh shapes
+    if (ShapedType::isDynamicShape(meshOp.getShape())) {
+      return mlir::failure();
+    }
+
+    SmallVector<Value> dims;
+    llvm::transform(
+        meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
+          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+        });
+    auto rank =
+        rewriter.create<ProcessLinearIndexOp>(op.getLoc(), meshOp).getResult();
+    auto mIdx = linearToMultiIndex(loc, rewriter, rank, dims);
+
+    // optionally extract subset of mesh axes
+    auto axes = op.getAxes();
+    if (!axes.empty()) {
+      SmallVector<Value> subIndex;
+      for (auto axis : axes) {
+        subIndex.push_back(mIdx[axis]);
+      }
+      mIdx = subIndex;
+    }
+
+    rewriter.replaceOp(op, mIdx);
+    return mlir::success();
+  }
+};
+
+struct ConvertProcessLinearIndexOp
+    : public mlir::OpRewritePattern<mlir::mesh::ProcessLinearIndexOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::mesh::ProcessLinearIndexOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    // Finds a global named "static_mpi_rank" it will use that splat value.
+    // Otherwise it defaults to mpi.comm_rank.
+
+    auto loc = op.getLoc();
+    auto rankOpName = StringAttr::get(op->getContext(), "static_mpi_rank");
+    if (auto globalOp = SymbolTable::lookupNearestSymbolFrom<memref::GlobalOp>(
+            op, rankOpName)) {
+      if (auto initTnsr = globalOp.getInitialValueAttr()) {
+        auto val = cast<DenseElementsAttr>(initTnsr).getSplatValue<int64_t>();
+        rewriter.replaceOp(op,
+                           rewriter.create<arith::ConstantIndexOp>(loc, val));
+        return mlir::success();
+      }
+    }
+    auto rank =
+        rewriter
+            .create<mpi::CommRankOp>(
+                op.getLoc(), TypeRange{mpi::RetvalType::get(op->getContext()),
+                                       rewriter.getI32Type()})
+            .getRank();
+    rewriter.replaceOpWithNewOp<arith::IndexCastOp>(op, rewriter.getIndexType(),
+                                                    rank);
+    return mlir::success();
+  }
+};
+
+struct ConvertNeighborsLinearIndicesOp
+    : public mlir::OpRewritePattern<mlir::mesh::NeighborsLinearIndicesOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::mesh::NeighborsLinearIndicesOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    // Computes the neighbors indices along a split axis by simply
+    // adding/subtracting 1 to the current index in that dimension.
+    // Assigns -1 if neighbor is out of bounds.
+
+    auto axes = op.getSplitAxes();
+    // For now only single axis sharding is supported
+    if (axes.size() != 1) {
+      return mlir::failure();
+    }
+
+    auto loc = op.getLoc();
+    SymbolTableCollection symbolTableCollection;
+    auto meshOp = getMesh(op, symbolTableCollection);
+    auto mIdx = op.getDevice();
+    auto orgIdx = mIdx[axes[0]];
+    SmallVector<Value> dims;
+    llvm::transform(
+        meshOp.getShape(), std::back_inserter(dims), [&](int64_t i) {
+          return rewriter.create<arith::ConstantIndexOp>(loc, i).getResult();
+        });
+    auto dimSz = dims[axes[0]];
+    auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1).getResult();
+    auto minus1 = rewriter.create<arith::ConstantIndexOp>(loc, -1).getResult();
+    auto atBorder = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::sle, orgIdx,
+        rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult());
+    auto down = rewriter.create<scf::IfOp>(
+        loc, atBorder,
+        [&](OpBuilder &builder, Location loc) {
+          builder.create<scf::YieldOp>(loc, minus1);
+        },
+        [&](OpBuilder &builder, Location loc) {
+          SmallVector<Value> tmp = mIdx;
+          tmp[axes[0]] =
+              rewriter.create<arith::SubIOp>(op.getLoc(), orgIdx, one)
+                  .getResult();
+          builder.create<scf::YieldOp>(
+              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+        });
+    atBorder = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::sge, orgIdx,
+        rewriter.create<arith::SubIOp>(loc, dimSz, one).getResult());
+    auto up = rewriter.create<scf::IfOp>(
+        loc, atBorder,
+        [&](OpBuilder &builder, Location loc) {
+          builder.create<scf::YieldOp>(loc, minus1);
+        },
+        [&](OpBuilder &builder, Location loc) {
+          SmallVector<Value> tmp = mIdx;
+          tmp[axes[0]] =
+              rewriter.create<arith::AddIOp>(op.getLoc(), orgIdx, one)
+                  .getResult();
+          builder.create<scf::YieldOp>(
+              loc, multiToLinearIndex(loc, rewriter, tmp, dims));
+        });
+    rewriter.replaceOp(op, ValueRange{down.getResult(0), up.getResult(0)});
+    return mlir::success();
+  }
+};
+
+struct ConvertUpdateHaloOp
+    : public mlir::OpRewritePattern<mlir::mesh::UpdateHaloOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::mesh::UpdateHaloOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    // The input/output memref is assumed to be in C memory order.
+    // Halos are exchanged as 2 blocks per dimension (one for each side: down
+    // and up). For each haloed dimension `d`, the exchanged blocks are
+    // expressed as multi-dimensional subviews. The subviews include potential
+    // halos of higher dimensions `dh > d`, no halos for the lower dimensions
+    // `dl < d` and for dimension `d` the currently exchanged halo only.
+    // By iterating form higher to lower dimensions this also updates the halos
+    // in the 'corners'.
+    // memref.subview is used to read and write the halo data from and to the
+    // local data. Because subviews and halos can have mixed dynamic and static
+    // shapes, OpFoldResults are used whenever possible.
+
+    SymbolTableCollection symbolTableCollection;
+    auto loc = op.getLoc();
+
+    // convert a OpFoldResult into a Value
+    auto toValue = [&rewriter, &loc](OpFoldResult &v) {
+      return v.is<Value>()
+                 ? v.get<Value>()
+                 : rewriter.create<::mlir::arith::ConstantOp>(
+                       loc,
+                       rewriter.getIndexAttr(
+                           cast<IntegerAttr>(v.get<Attribute>()).getInt()));
+    };
+
+    auto dest = op.getDestination();
+    auto dstShape = cast<ShapedType>(dest.getType()).getShape();
+    Value array = dest;
+    if (isa<RankedTensorType>(array.getType())) {
+      // If the destination is a memref, we need to cast it to a tensor
+      auto tensorType = MemRefType::get(
+          dstShape, cast<ShapedType>(array.getType()).getElementType());
+      array = rewriter.create<bufferization::ToMemrefOp>(loc, tensorType, array)
+                  .getResult();
+    }
+    auto rank = cast<ShapedType>(array.getType()).getRank();
+    auto opSplitAxes = op.getSplitAxes().getAxes();
+    auto mesh = op.getMesh();
+    auto meshOp = getMesh(op, symbolTableCollection);
+    auto haloSizes =
+        getMixedValues(op.getStaticHaloSizes(), op.getHaloSizes(), rewriter);
+    // subviews need Index values
+    for (auto &sz : haloSizes) {
+      if (sz.is<Value>()) {
+        sz = rewriter
+                 .create<arith::IndexCastOp>(loc, rewriter.getIndexType(),
+                                             sz.get<Value>())
+                 .getResult();
+      }
+    }
+
+    // most of the offset/size/stride data is the same for all dims
+    SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
+    SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
+    SmallVector<OpFoldResult> shape(rank), dimSizes(rank);
+    auto currHaloDim = -1; // halo sizes are provided for split dimensions only
+    // we need the actual shape to compute offsets and sizes
+    for (auto i = 0; i < rank; ++i) {
+      auto s = dstShape[i];
+      if (ShapedType::isDynamic(s)) {
+        shape[i] = rewriter.create<memref::DimOp>(loc, array, s).getResult();
+      } else {
+        shape[i] = rewriter.getIndexAttr(s);
+      }
+
+      if ((size_t)i < opSplitAxes.size() && !opSplitAxes[i].empty()) {
+        ++currHaloDim;
+        // the offsets for lower dim sstarts after their down halo
+        offsets[i] = haloSizes[currHaloDim * 2];
+
+        // prepare shape and offsets of highest dim's halo exchange
+        auto _haloSz =
+            rewriter
+                .create<arith::AddIOp>(loc, toValue(haloSizes[currHaloDim * 2]),
+                                       toValue(haloSizes[currHaloDim * 2 + 1]))
+                .getResult();
+        // the halo shape of lower dims exlude the halos
+        dimSizes[i] =
+            rewriter.create<arith::SubIOp>(loc, toValue(shape[i]), _haloSz)
+                .getResult();
+      } else {
+        dimSizes[i] = shape[i];
+      }
+    }
+
+    auto tagAttr = rewriter.getI32IntegerAttr(91); // we just pick something
+    auto tag = rewriter.create<::mlir::arith::ConstantOp>(loc, tagAttr);
+    auto zeroAttr = rewriter.getI32IntegerAttr(0); // for detecting v<0
+    auto zero = rewriter.create<::mlir::arith::ConstantOp>(loc, zeroAttr);
+
+    SmallVector<Type> indexResultTypes(meshOp.getShape().size(),
+                                       rewriter.getIndexType());
+    auto myMultiIndex =
+        rewriter.create<ProcessMultiIndexOp>(loc, indexResultTypes, mesh)
+            .getResult();
+    // traverse all split axes from high to low dim
+    for (ssize_t dim = opSplitAxes.size() - 1; dim >= 0; --dim) {
+      auto splitAxes = opSplitAxes[dim];
+      if (splitAxes.empty()) {
+        continue;
+      }
+      assert(currHaloDim >= 0 && (size_t)currHaloDim < haloSizes.size() / 2);
+      // Get the linearized ids of the neighbors (down and up) for the
+      // given split
+      auto tmp = rewriter
+                     .create<NeighborsLinearIndicesOp>(loc, mesh, myMultiIndex,
+                                                       splitAxes)
+                     .getResults();
+      // MPI operates on i32...
+      Value neighbourIDs[2] = {rewriter.create<arith::IndexCastOp>(
+                                   loc, rewriter.getI32Type(), tmp[0]),
+                               rewriter.create<arith::IndexCastOp>(
+                                   loc, rewriter.getI32Type(), tmp[1])};
+
+      auto lowerRecvOffset = rewriter.getIndexAttr(0);
+      auto lowerSendOffset = toValue(haloSizes[currHaloDim * 2]);
+      auto upperRecvOffset = rewriter.create<arith::SubIOp>(
+          loc, toValue(shape[dim]), toValue(haloSizes[currHaloDim * 2 + 1]));
+      auto upperSendOffset = rewriter.create<arith::SubIOp>(
+          loc, upperRecvOffset, toValue(haloSizes[currHaloDim * 2]));
+
+      // Make sure we send/recv in a way that does not lead to a dead-lock.
+      // The current approach is by far not optimal, this should be at least
+      // be a red-black pattern or using MPI_sendrecv.
+      // Also, buffers should be re-used.
+      // Still using temporary contiguous buffers for MPI communication...
+      // Still yielding a "serialized" communication pattern...
+      auto genSendRecv = [&](bool upperHalo) {
+        auto orgOffset = offsets[dim];
+        dimSizes[dim] = upperHalo ? haloSizes[currHaloDim * 2 + 1]
+                                  : haloSizes[currHaloDim * 2];
+        // Check if we need to send and/or receive
+        // Processes on the mesh borders have only one neighbor
+        auto to = upperHalo ? neighbourIDs[1] : neighbourIDs[0];
+        auto from = upperHalo ? neighbourIDs[0] : neighbourIDs[1];
+        auto hasFrom = rewriter.create<arith::CmpIOp>(
+            loc, arith::CmpIPredicate::sge, from, zero);
+        auto hasTo = rewriter.create<arith::CmpIOp>(
+            loc, arith::CmpIPredicate::sge, to, zero);
+        auto buffer = rewriter.create<memref::AllocOp>(
+            loc, dimSizes, cast<ShapedType>(array.getType()).getElementType());
+        // if has neighbor: copy halo data from array to buffer and send
+        rewriter.create<scf::IfOp>(
+            loc, hasTo, [&](OpBuilder &builder, Location loc) {
+              offsets[dim] = upperHalo ? OpFoldResult(lowerSendOffset)
+                                       : OpFoldResult(upperSendOffset);
+              auto subview = builder.create<memref::SubViewOp>(
+                  loc, array, offsets, dimSizes, strides);
+              builder.create<memref::CopyOp>(loc, subview, buffer);
+              builder.create<mpi::SendOp>(loc, TypeRange{}, buffer, tag, to);
+              builder.create<scf::YieldOp>(loc);
+            });
+        // if has neighbor: receive halo data into buffer and copy to array
+        rewriter.create<scf::IfOp>(
+            loc, hasFrom, [&](OpBuilder &builder, Location loc) {
+              offsets[dim] = upperHalo ? OpFoldResult(upperRecvOffset)
+                                       : OpFoldResult(lowerRecvOffset);
+              builder.create<mpi::RecvOp>(loc, TypeRange{}, buffer, tag, from);
+              auto subview = builder.create<memref::SubViewOp>(
+                  loc, array, offsets, dimSizes, strides);
+              builder.create<memref::CopyOp>(loc, buffer, subview);
+              builder.create<scf::YieldOp>(loc);
+            });
+        rewriter.create<memref::DeallocOp>(loc, buffer);
+        offsets[dim] = orgOffset;
+      };
+
+      genSendRecv(false);
+      genSendRecv(true);
+
+      // the shape for lower dims include higher dims' halos
+      dimSizes[dim] = shape[dim];
+      // -> the offset for higher dims is always 0
+      offsets[dim] = rewriter.getIndexAttr(0);
+      // on to next halo
+      --currHaloDim;
+    }
+
+    if (isa<MemRefType>(op.getResult().getType())) {
+      rewriter.replaceOp(op, array);
+    } else {
+      assert(isa<RankedTensorType>(op.getResult().getType()));
+      rewriter.replaceOp(op, rewriter.create<bufferization::ToTensorOp>(
+                                 loc, op.getResult().getType(), array,
+                                 /*restrict=*/true, /*writable=*/true));
+    }
+    return mlir::success();
+  }
+};
+
+struct ConvertMeshToMPIPass
+    : public impl::ConvertMeshToMPIPassBase<ConvertMeshToMPIPass> {
+  using Base::Base;
+
+  /// Run the dialect converter on the module.
+  void runOnOperation() override {
+    auto *ctx = &getContext();
+    mlir::RewritePatternSet patterns(ctx);
+
+    patterns.insert<ConvertUpdateHaloOp, ConvertNeighborsLinearIndicesOp,
+                    ConvertProcessLinearIndexOp, ConvertProcessMultiIndexOp>(
+        ctx);
+
+    (void)mlir::applyPatternsAndFoldGreedily(getOperation(),
+                                             std::move(patterns));
+  }
+};
+
+} // namespace
+
+// Create a pass that convert Mesh to MPI
+std::unique_ptr<::mlir::Pass> mlir::createConvertMeshToMPIPass() {
+  return std::make_unique<ConvertMeshToMPIPass>();
+}
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index cdbcd3013e139..88e544c4e4b5f 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -1261,14 +1261,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           Value shift = shiftConstant ? shiftConstant : blockArgs[shiftArg];
 
           if (valueTy.getIntOrFloatBitWidth() < 32) {
-            if (valueTy.isUnsignedInteger()) {
-              value = nestedBuilder
-                          .create<UnrealizedConversionCastOp>(
-                              nestedLoc,
-                              nestedBuilder.getIntegerType(
-                                  valueTy.getIntOrFloatBitWidth()),
-                              value)
-                          .getResult(0);
+            if (op.getInputUnsigned()) {
               value = nestedBuilder.create<arith::ExtUIOp>(
                   nestedLoc, nestedBuilder.getI32Type(), value);
             } else {
@@ -1297,7 +1290,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           int32_t intMax = APInt::getSignedMaxValue(outBitWidth).getSExtValue();
 
           // Unsigned integers have a difference output value.
-          if (outIntType.isUnsignedInteger()) {
+          if (op.getOutputUnsigned()) {
             intMin = 0;
             intMax = APInt::getMaxValue(outBitWidth).getZExtValue();
           }
@@ -1314,13 +1307,6 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
             value = nestedBuilder.create<arith::TruncIOp>(
                 nestedLoc, rewriter.getIntegerType(outIntType.getWidth()),
                 value);
-
-            if (outIntType.isUnsignedInteger()) {
-              value = nestedBuilder
-                          .create<UnrealizedConversionCastOp>(nestedLoc,
-                                                              outIntType, value)
-                          .getResult(0);
-            }
           }
 
           nestedBuilder.create<linalg::YieldOp>(loc, value);
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 58ca84c8d7bca..a9a07c323c735 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -1096,43 +1096,55 @@ class VectorExtractOpConversion
     SmallVector<OpFoldResult> positionVec = getMixedValues(
         adaptor.getStaticPosition(), adaptor.getDynamicPosition(), rewriter);
 
-    // Extract entire vector. Should be handled by folder, but just to be safe.
-    ArrayRef<OpFoldResult> position(positionVec);
-    if (position.empty()) {
-      rewriter.replaceOp(extractOp, adaptor.getVector());
-      return success();
-    }
-
-    // One-shot extraction of vector from array (only requires extractvalue).
-    // Except for extracting 1-element vectors.
-    if (isa<VectorType>(resultType) &&
-        position.size() !=
-            static_cast<size_t>(extractOp.getSourceVectorType().getRank())) {
-      if (extractOp.hasDynamicPosition())
-        return failure();
-
-      Value extracted = rewriter.create<LLVM::ExtractValueOp>(
-          loc, adaptor.getVector(), getAsIntegers(position));
-      rewriter.replaceOp(extractOp, extracted);
-      return success();
+    // The Vector -> LLVM lowering models N-D vectors as nested aggregates of
+    // 1-d vectors. This nesting is modeled using arrays. We do this conversion
+    // from a N-d vector extract to a nested aggregate vector extract in two
+    // steps:
+    //  - Extract a member from the nested aggregate. The result can be
+    //    a lower rank nested aggregate or a vector (1-D). This is done using
+    //    `llvm.extractvalue`.
+    //  - Extract a scalar out of the vector if needed. This is done using
+    //   `llvm.extractelement`.
+
+    // Determine if we need to extract a member out of the aggregate. We
+    // always need to extract a member if the input rank >= 2.
+    bool extractsAggregate = extractOp.getSourceVectorType().getRank() >= 2;
+    // Determine if we need to extract a scalar as the result. We extract
+    // a scalar if the extract is full rank, i.e., the number of indices is
+    // equal to source vector rank.
+    bool extractsScalar = static_cast<int64_t>(positionVec.size()) ==
+                          extractOp.getSourceVectorType().getRank();
+
+    // Since the LLVM type converter converts 0-d vectors to 1-d vectors, we
+    // need to add a position for this change.
+    if (extractOp.getSourceVectorType().getRank() == 0) {
+      Type idxType = typeConverter->convertType(rewriter.getIndexType());
+      positionVec.push_back(rewriter.getZeroAttr(idxType));
     }
 
-    // Potential extraction of 1-D vector from array.
     Value extracted = adaptor.getVector();
-    if (position.size() > 1) {
-      if (extractOp.hasDynamicPosition())
+    if (extractsAggregate) {
+      ArrayRef<OpFoldResult> position(positionVec);
+      if (extractsScalar) {
+        // If we are extracting a scalar from the extracted member, we drop
+        // the last index, which will be used to extract the scalar out of the
+        // vector.
+        position = position.drop_back();
+      }
+      // llvm.extractvalue does not support dynamic dimensions.
+      if (!llvm::all_of(position, llvm::IsaPred<Attribute>)) {
         return failure();
+      }
+      extracted = rewriter.create<LLVM::ExtractValueOp>(
+          loc, extracted, getAsIntegers(position));
+    }
 
-      SmallVector<int64_t> nMinusOnePosition =
-          getAsIntegers(position.drop_back());
-      extracted = rewriter.create<LLVM::ExtractValueOp>(loc, extracted,
-                                                        nMinusOnePosition);
+    if (extractsScalar) {
+      extracted = rewriter.create<LLVM::ExtractElementOp>(
+          loc, extracted, getAsLLVMValue(rewriter, loc, positionVec.back()));
     }
 
-    Value lastPosition = getAsLLVMValue(rewriter, loc, position.back());
-    // Remaining extraction of element from 1-D LLVM vector.
-    rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(extractOp, extracted,
-                                                        lastPosition);
+    rewriter.replaceOp(extractOp, extracted);
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 656b1cb3e99a1..d3731db1ce55c 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -220,6 +220,32 @@ struct VectorFmaOpConvert final : public OpConversionPattern<vector::FMAOp> {
   }
 };
 
+struct VectorFromElementsOpConvert final
+    : public OpConversionPattern<vector::FromElementsOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::FromElementsOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Type resultType = getTypeConverter()->convertType(op.getType());
+    if (!resultType)
+      return failure();
+    OperandRange elements = op.getElements();
+    if (isa<spirv::ScalarType>(resultType)) {
+      // In the case with a single scalar operand / single-element result,
+      // pass through the scalar.
+      rewriter.replaceOp(op, elements[0]);
+      return success();
+    }
+    // SPIRVTypeConverter rejects vectors with rank > 1, so multi-dimensional
+    // vector.from_elements cases should not need to be handled, only 1d.
+    assert(cast<VectorType>(resultType).getRank() == 1);
+    rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, resultType,
+                                                             elements);
+    return success();
+  }
+};
+
 struct VectorInsertOpConvert final
     : public OpConversionPattern<vector::InsertOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -952,8 +978,9 @@ void mlir::populateVectorToSPIRVPatterns(
       VectorBitcastConvert, VectorBroadcastConvert,
       VectorExtractElementOpConvert, VectorExtractOpConvert,
       VectorExtractStridedSliceOpConvert, VectorFmaOpConvert<spirv::GLFmaOp>,
-      VectorFmaOpConvert<spirv::CLFmaOp>, VectorInsertElementOpConvert,
-      VectorInsertOpConvert, VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
+      VectorFmaOpConvert<spirv::CLFmaOp>, VectorFromElementsOpConvert,
+      VectorInsertElementOpConvert, VectorInsertOpConvert,
+      VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
       VectorReductionPattern<CL_INT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<CL_FLOAT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<GL_FLOAT_MAX_MIN_OPS>, VectorShapeCast,
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 1c5466730a558..dceebbfec586c 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -4569,9 +4569,49 @@ LogicalResult AffineDelinearizeIndexOp::verify() {
   return success();
 }
 
+/// Given mixed basis of affine.delinearize_index/linearize_index replace
+/// constant SSA values with the constant integer value and return the new
+/// static basis. In case no such candidate for replacement exists, this utility
+/// returns std::nullopt.
+static std::optional<SmallVector<int64_t>>
+foldCstValueToCstAttrBasis(ArrayRef<OpFoldResult> mixedBasis,
+                           MutableOperandRange mutableDynamicBasis,
+                           ArrayRef<Attribute> dynamicBasis) {
+  uint64_t dynamicBasisIndex = 0;
+  for (OpFoldResult basis : dynamicBasis) {
+    if (basis) {
+      mutableDynamicBasis.erase(dynamicBasisIndex);
+    } else {
+      ++dynamicBasisIndex;
+    }
+  }
+
+  // No constant SSA value exists.
+  if (dynamicBasisIndex == dynamicBasis.size())
+    return std::nullopt;
+
+  SmallVector<int64_t> staticBasis;
+  for (OpFoldResult basis : mixedBasis) {
+    std::optional<int64_t> basisVal = getConstantIntValue(basis);
+    if (!basisVal)
+      staticBasis.push_back(ShapedType::kDynamic);
+    else
+      staticBasis.push_back(*basisVal);
+  }
+
+  return staticBasis;
+}
+
 LogicalResult
 AffineDelinearizeIndexOp::fold(FoldAdaptor adaptor,
                                SmallVectorImpl<OpFoldResult> &result) {
+  std::optional<SmallVector<int64_t>> maybeStaticBasis =
+      foldCstValueToCstAttrBasis(getMixedBasis(), getDynamicBasisMutable(),
+                                 adaptor.getDynamicBasis());
+  if (maybeStaticBasis) {
+    setStaticBasis(*maybeStaticBasis);
+    return success();
+  }
   // If we won't be doing any division or modulo (no basis or the one basis
   // element is purely advisory), simply return the input value.
   if (getNumResults() == 1) {
@@ -4875,6 +4915,13 @@ LogicalResult AffineLinearizeIndexOp::verify() {
 }
 
 OpFoldResult AffineLinearizeIndexOp::fold(FoldAdaptor adaptor) {
+  std::optional<SmallVector<int64_t>> maybeStaticBasis =
+      foldCstValueToCstAttrBasis(getMixedBasis(), getDynamicBasisMutable(),
+                                 adaptor.getDynamicBasis());
+  if (maybeStaticBasis) {
+    setStaticBasis(*maybeStaticBasis);
+    return getResult();
+  }
   // No indices linearizes to zero.
   if (getMultiIndex().empty())
     return IntegerAttr::get(getResult().getType(), 0);
diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index 22c34b2bd42f5..6d7ac2be951dd 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -422,11 +422,10 @@ def TruncIShrUIMulIToMulUIExtended :
 //===----------------------------------------------------------------------===//
 
 // mulf(negf(x), negf(y)) -> mulf(x,y)
-// (retain fastmath flags and denormal mode of the original divf)
+// (retain fastmath flags of original mulf)
 def MulFOfNegF :
-    Pat<(Arith_MulFOp (Arith_NegFOp $x, $_, $_), 
-                      (Arith_NegFOp $y, $_, $_), $fmf, $mode),
-        (Arith_MulFOp $x, $y, $fmf, $mode),
+    Pat<(Arith_MulFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf),
+        (Arith_MulFOp $x, $y, $fmf),
         [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y)]>;
 
 //===----------------------------------------------------------------------===//
@@ -434,11 +433,10 @@ def MulFOfNegF :
 //===----------------------------------------------------------------------===//
 
 // divf(negf(x), negf(y)) -> divf(x,y)
-// (retain fastmath flags and denormal mode of the original divf)
+// (retain fastmath flags of original divf)
 def DivFOfNegF :
-    Pat<(Arith_DivFOp (Arith_NegFOp $x, $_, $_), 
-                      (Arith_NegFOp $y, $_, $_), $fmf, $mode),
-        (Arith_DivFOp $x, $y, $fmf, $mode),
+    Pat<(Arith_DivFOp (Arith_NegFOp $x, $_), (Arith_NegFOp $y, $_), $fmf),
+        (Arith_DivFOp $x, $y, $fmf),
         [(Constraint<CPred<"$0.getType() == $1.getType()">> $x, $y)]>;
 
 #endif // ARITH_PATTERNS
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 16b4e8eb4f022..d8b314a3fa43c 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "mlir/Support/LogicalResult.h"
 
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
@@ -224,8 +225,7 @@ LogicalResult arith::ConstantOp::verify() {
   // Note, we could relax this for vectors with 1 scalable dim, e.g.:
   //  * arith.constant dense<[[3, 3], [1, 1]]> : vector<2 x [2] x i32>
   // However, this would most likely require updating the lowerings to LLVM.
-  auto vecType = dyn_cast<VectorType>(type);
-  if (vecType && vecType.isScalable() && !isa<SplatElementsAttr>(getValue()))
+  if (isa<ScalableVectorType>(type) && !isa<SplatElementsAttr>(getValue()))
     return emitOpError(
         "intializing scalable vectors with elements attribute is not supported"
         " unless it's a vector splat");
@@ -393,8 +393,12 @@ void arith::AddUIExtendedOp::getCanonicalizationPatterns(
 
 OpFoldResult arith::SubIOp::fold(FoldAdaptor adaptor) {
   // subi(x,x) -> 0
-  if (getOperand(0) == getOperand(1))
-    return Builder(getContext()).getZeroAttr(getType());
+  if (getOperand(0) == getOperand(1)) {
+    auto shapedType = dyn_cast<ShapedType>(getType());
+    // We can't generate a constant with a dynamic shaped tensor.
+    if (!shapedType || shapedType.hasStaticShape())
+      return Builder(getContext()).getZeroAttr(getType());
+  }
   // subi(x,0) -> x
   if (matchPattern(adaptor.getRhs(), m_Zero()))
     return getLhs();
@@ -952,7 +956,7 @@ void arith::XOrIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
 //===----------------------------------------------------------------------===//
 
 OpFoldResult arith::NegFOp::fold(FoldAdaptor adaptor) {
-  // negf(negf(x)) -> x
+  /// negf(negf(x)) -> x
   if (auto op = this->getOperand().getDefiningOp<arith::NegFOp>())
     return op.getOperand();
   return constFoldUnaryOp<FloatAttr>(adaptor.getOperands(),
@@ -982,14 +986,6 @@ OpFoldResult arith::SubFOp::fold(FoldAdaptor adaptor) {
   if (matchPattern(adaptor.getRhs(), m_PosZeroFloat()))
     return getLhs();
 
-  // Simplifies subf(x, rhs) to x if the following conditions are met:
-  // 1. `rhs` is a denormal floating-point value.
-  // 2. The denormal mode for the operation is set to positive zero.
-  bool isPositiveZeroMode =
-      getDenormalModeAttr().getValue() == DenormalMode::positive_zero;
-  if (isPositiveZeroMode && matchPattern(adaptor.getRhs(), m_isDenormalFloat()))
-    return getLhs();
-
   return constFoldBinaryOp<FloatAttr>(
       adaptor.getOperands(),
       [](const APFloat &a, const APFloat &b) { return a - b; });
@@ -1022,13 +1018,11 @@ OpFoldResult arith::MaxNumFOp::fold(FoldAdaptor adaptor) {
   if (getLhs() == getRhs())
     return getRhs();
 
-  // maxnumf(x, -inf) -> x
-  if (matchPattern(adaptor.getRhs(), m_NegInfFloat()))
+  // maxnumf(x, NaN) -> x
+  if (matchPattern(adaptor.getRhs(), m_NaNFloat()))
     return getLhs();
 
-  return constFoldBinaryOp<FloatAttr>(
-      adaptor.getOperands(),
-      [](const APFloat &a, const APFloat &b) { return llvm::maximum(a, b); });
+  return constFoldBinaryOp<FloatAttr>(adaptor.getOperands(), llvm::maxnum);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1108,8 +1102,8 @@ OpFoldResult arith::MinNumFOp::fold(FoldAdaptor adaptor) {
   if (getLhs() == getRhs())
     return getRhs();
 
-  // minnumf(x, +inf) -> x
-  if (matchPattern(adaptor.getRhs(), m_PosInfFloat()))
+  // minnumf(x, NaN) -> x
+  if (matchPattern(adaptor.getRhs(), m_NaNFloat()))
     return getLhs();
 
   return constFoldBinaryOp<FloatAttr>(
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index e908a536e6fb2..61767f3b21c9c 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -265,8 +265,8 @@ struct LegalizeMaskedVectorOuterProductOpsByDecomposition
   LogicalResult
   matchAndRewrite(vector::MaskOp maskOp, OpAdaptor adaptor,
                   OneToNPatternRewriter &rewriter) const override {
-    if (auto outerProductOp =
-            llvm::dyn_cast<vector::OuterProductOp>(maskOp.getMaskableOp())) {
+    if (auto outerProductOp = llvm::dyn_cast_or_null<vector::OuterProductOp>(
+            maskOp.getMaskableOp())) {
       LegalizeVectorOuterProductOpsByDecomposition pattern(*getTypeConverter(),
                                                            getContext());
       return static_cast<RewritePattern &>(pattern).matchAndRewrite(
diff --git a/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
index f8fb1f436a95b..6384d25ee7027 100644
--- a/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_mlir_dialect_library(MLIRFuncTransforms
-  DecomposeCallGraphTypes.cpp
   DuplicateFunctionElimination.cpp
   FuncConversions.cpp
   OneToNFuncConversions.cpp
diff --git a/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp b/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
deleted file mode 100644
index a08764326a80b..0000000000000
--- a/mlir/lib/Dialect/Func/Transforms/DecomposeCallGraphTypes.cpp
+++ /dev/null
@@ -1,180 +0,0 @@
-//===- DecomposeCallGraphTypes.cpp - CG type decomposition ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/IR/BuiltinOps.h"
-
-using namespace mlir;
-using namespace mlir::func;
-
-//===----------------------------------------------------------------------===//
-// Helper functions
-//===----------------------------------------------------------------------===//
-
-/// If the given value can be decomposed with the type converter, decompose it.
-/// Otherwise, return the given value.
-// TODO: Value decomposition should happen automatically through a 1:N adaptor.
-// This function will disappear when the 1:1 and 1:N drivers are merged.
-static SmallVector<Value> decomposeValue(OpBuilder &builder, Location loc,
-                                         Value value,
-                                         const TypeConverter *converter) {
-  // Try to convert the given value's type. If that fails, just return the
-  // given value.
-  SmallVector<Type> convertedTypes;
-  if (failed(converter->convertType(value.getType(), convertedTypes)))
-    return {value};
-  if (convertedTypes.empty())
-    return {};
-
-  // If the given value's type is already legal, just return the given value.
-  TypeRange convertedTypeRange(convertedTypes);
-  if (convertedTypeRange == TypeRange(value.getType()))
-    return {value};
-
-  // Try to materialize a target conversion. If the materialization did not
-  // produce values of the requested type, the materialization failed. Just
-  // return the given value in that case.
-  SmallVector<Value> result = converter->materializeTargetConversion(
-      builder, loc, convertedTypeRange, value);
-  if (result.empty())
-    return {value};
-  return result;
-}
-
-//===----------------------------------------------------------------------===//
-// DecomposeCallGraphTypesForFuncArgs
-//===----------------------------------------------------------------------===//
-
-namespace {
-/// Expand function arguments according to the provided TypeConverter.
-struct DecomposeCallGraphTypesForFuncArgs
-    : public OpConversionPattern<func::FuncOp> {
-  using OpConversionPattern::OpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(func::FuncOp op, OpAdaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const final {
-    auto functionType = op.getFunctionType();
-
-    // Convert function arguments using the provided TypeConverter.
-    TypeConverter::SignatureConversion conversion(functionType.getNumInputs());
-    for (const auto &argType : llvm::enumerate(functionType.getInputs())) {
-      SmallVector<Type, 2> decomposedTypes;
-      if (failed(typeConverter->convertType(argType.value(), decomposedTypes)))
-        return failure();
-      if (!decomposedTypes.empty())
-        conversion.addInputs(argType.index(), decomposedTypes);
-    }
-
-    // If the SignatureConversion doesn't apply, bail out.
-    if (failed(rewriter.convertRegionTypes(&op.getBody(), *getTypeConverter(),
-                                           &conversion)))
-      return failure();
-
-    // Update the signature of the function.
-    SmallVector<Type, 2> newResultTypes;
-    if (failed(typeConverter->convertTypes(functionType.getResults(),
-                                           newResultTypes)))
-      return failure();
-    rewriter.modifyOpInPlace(op, [&] {
-      op.setType(rewriter.getFunctionType(conversion.getConvertedTypes(),
-                                          newResultTypes));
-    });
-    return success();
-  }
-};
-} // namespace
-
-//===----------------------------------------------------------------------===//
-// DecomposeCallGraphTypesForReturnOp
-//===----------------------------------------------------------------------===//
-
-namespace {
-/// Expand return operands according to the provided TypeConverter.
-struct DecomposeCallGraphTypesForReturnOp
-    : public OpConversionPattern<ReturnOp> {
-  using OpConversionPattern::OpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const final {
-    SmallVector<Value, 2> newOperands;
-    for (Value operand : adaptor.getOperands()) {
-      // TODO: We can directly take the values from the adaptor once this is a
-      // 1:N conversion pattern.
-      llvm::append_range(newOperands,
-                         decomposeValue(rewriter, operand.getLoc(), operand,
-                                        getTypeConverter()));
-    }
-    rewriter.replaceOpWithNewOp<ReturnOp>(op, newOperands);
-    return success();
-  }
-};
-} // namespace
-
-//===----------------------------------------------------------------------===//
-// DecomposeCallGraphTypesForCallOp
-//===----------------------------------------------------------------------===//
-
-namespace {
-/// Expand call op operands and results according to the provided TypeConverter.
-struct DecomposeCallGraphTypesForCallOp : public OpConversionPattern<CallOp> {
-  using OpConversionPattern::OpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(CallOp op, OpAdaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const final {
-
-    // Create the operands list of the new `CallOp`.
-    SmallVector<Value, 2> newOperands;
-    for (Value operand : adaptor.getOperands()) {
-      // TODO: We can directly take the values from the adaptor once this is a
-      // 1:N conversion pattern.
-      llvm::append_range(newOperands,
-                         decomposeValue(rewriter, operand.getLoc(), operand,
-                                        getTypeConverter()));
-    }
-
-    // Create the new result types for the new `CallOp` and track the number of
-    // replacement types for each original op result.
-    SmallVector<Type, 2> newResultTypes;
-    SmallVector<unsigned> expandedResultSizes;
-    for (Type resultType : op.getResultTypes()) {
-      unsigned oldSize = newResultTypes.size();
-      if (failed(typeConverter->convertType(resultType, newResultTypes)))
-        return failure();
-      expandedResultSizes.push_back(newResultTypes.size() - oldSize);
-    }
-
-    CallOp newCallOp = rewriter.create<CallOp>(op.getLoc(), op.getCalleeAttr(),
-                                               newResultTypes, newOperands);
-
-    // Build a replacement value for each result to replace its uses.
-    SmallVector<ValueRange> replacedValues;
-    replacedValues.reserve(op.getNumResults());
-    unsigned startIdx = 0;
-    for (unsigned i = 0, e = op.getNumResults(); i < e; ++i) {
-      ValueRange repl =
-          newCallOp.getResults().slice(startIdx, expandedResultSizes[i]);
-      replacedValues.push_back(repl);
-      startIdx += expandedResultSizes[i];
-    }
-    rewriter.replaceOpWithMultiple(op, replacedValues);
-    return success();
-  }
-};
-} // namespace
-
-void mlir::populateDecomposeCallGraphTypesPatterns(
-    MLIRContext *context, const TypeConverter &typeConverter,
-    RewritePatternSet &patterns) {
-  patterns
-      .add<DecomposeCallGraphTypesForCallOp, DecomposeCallGraphTypesForFuncArgs,
-           DecomposeCallGraphTypesForReturnOp>(typeConverter, context);
-}
diff --git a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
index b1cde6ca5d2fc..a3638c8766a5c 100644
--- a/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/FuncConversions.cpp
@@ -13,6 +13,14 @@
 using namespace mlir;
 using namespace mlir::func;
 
+/// Flatten the given value ranges into a single vector of values.
+static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
+  SmallVector<Value> result;
+  for (const auto &vals : values)
+    llvm::append_range(result, vals);
+  return result;
+}
+
 namespace {
 /// Converts the operand and result types of the CallOp, used together with the
 /// FuncOpSignatureConversion.
@@ -21,7 +29,7 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
   /// Hook for derived classes to implement combined matching and rewriting.
   LogicalResult
-  matchAndRewrite(CallOp callOp, OpAdaptor adaptor,
+  matchAndRewrite(CallOp callOp, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Convert the original function results. Keep track of how many result
     // types an original result type is converted into.
@@ -38,9 +46,9 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 
     // Substitute with the new result types from the corresponding FuncType
     // conversion.
-    auto newCallOp =
-        rewriter.create<CallOp>(callOp.getLoc(), callOp.getCallee(),
-                                convertedResults, adaptor.getOperands());
+    auto newCallOp = rewriter.create<CallOp>(
+        callOp.getLoc(), callOp.getCallee(), convertedResults,
+        flattenValues(adaptor.getOperands()));
     SmallVector<ValueRange> replacements;
     size_t offset = 0;
     for (int i = 0, e = callOp->getNumResults(); i < e; ++i) {
@@ -116,12 +124,10 @@ class ReturnOpTypeConversion : public OpConversionPattern<ReturnOp> {
   using OpConversionPattern<ReturnOp>::OpConversionPattern;
 
   LogicalResult
-  matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
+  matchAndRewrite(ReturnOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const final {
-    // For a return, all operands go to the results of the parent, so
-    // rewrite them all.
-    rewriter.modifyOpInPlace(op,
-                             [&] { op->setOperands(adaptor.getOperands()); });
+    rewriter.replaceOpWithNewOp<ReturnOp>(op,
+                                          flattenValues(adaptor.getOperands()));
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 2f96f10b662f8..ee00fbeb28b61 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -2075,7 +2075,8 @@ struct SimplifyDimOfAllocOp : public OpRewritePattern<memref::DimOp> {
       return failure();
 
     auto memrefType = llvm::dyn_cast<MemRefType>(dimOp.getSource().getType());
-    if (!memrefType || !memrefType.isDynamicDim(index.value()))
+    if (!memrefType || index.value() >= memrefType.getRank() ||
+        !memrefType.isDynamicDim(index.value()))
       return failure();
 
     auto alloc = dimOp.getSource().getDefiningOp<AllocOp>();
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 5f6556d915f41..ba0c80c50211e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -364,9 +364,15 @@ class GpuKernelOutliningPass
       Block::iterator insertPt(func->getNextNode());
       auto funcWalkResult = func.walk([&](gpu::LaunchOp op) {
         SetVector<Value> operands;
-        std::string kernelFnName =
-            Twine(op->getParentOfType<SymbolOpInterface>().getName(), "_kernel")
-                .str();
+        std::string kernelFnName;
+        if (op.getKernelFunc()) {
+          kernelFnName = op.getKernelFunc()->getRootReference().str();
+        } else {
+          kernelFnName =
+              Twine(op->getParentOfType<SymbolOpInterface>().getName(),
+                    "_kernel")
+                  .str();
+        }
 
         gpu::GPUFuncOp outlinedFunc =
             outlineKernelFuncImpl(op, kernelFnName, operands);
@@ -374,7 +380,7 @@ class GpuKernelOutliningPass
         // Create nested module and insert outlinedFunc. The module will
         // originally get the same name as the function, but may be renamed on
         // insertion into the parent module.
-        auto kernelModule = createKernelModule(outlinedFunc, symbolTable);
+        auto kernelModule = createKernelModule(op, outlinedFunc, symbolTable);
         symbolTable.insert(kernelModule, insertPt);
 
         // Potentially changes signature, pulling in constants.
@@ -395,7 +401,8 @@ class GpuKernelOutliningPass
 
 private:
   /// Returns a gpu.module containing kernelFunc and all callees (recursive).
-  gpu::GPUModuleOp createKernelModule(gpu::GPUFuncOp kernelFunc,
+  gpu::GPUModuleOp createKernelModule(gpu::LaunchOp gpuLaunchOp,
+                                      gpu::GPUFuncOp kernelFunc,
                                       const SymbolTable &parentSymbolTable) {
     // TODO: This code cannot use an OpBuilder because it must be inserted into
     // a SymbolTable by the caller. SymbolTable needs to be refactored to
@@ -403,8 +410,23 @@ class GpuKernelOutliningPass
     // and then this needs to use the OpBuilder.
     auto *context = getOperation().getContext();
     OpBuilder builder(context);
-    auto kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
-                                                         kernelFunc.getName());
+    std::string kernelModuleName;
+    gpu::GPUModuleOp kernelModule;
+    if (gpuLaunchOp.getKernelModule()) {
+      kernelModuleName =
+          gpuLaunchOp.getKernelModule()->getRootReference().str();
+      kernelModule =
+          parentSymbolTable.lookup<gpu::GPUModuleOp>(kernelModuleName);
+    } else {
+      kernelModuleName = kernelFunc.getName();
+    }
+
+    // Check if the module already exists in the symbol table
+    if (!kernelModule) {
+      // If not found, create a new GPU module
+      kernelModule = builder.create<gpu::GPUModuleOp>(kernelFunc.getLoc(),
+                                                      kernelModuleName);
+    }
 
     // If a valid data layout spec was provided, attach it to the kernel module.
     // Otherwise, the default data layout will be used.
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
index 9537f7c40dd4b..d700dc52d42d2 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
@@ -53,14 +53,14 @@ static StringRef getTypeKeyword(Type type) {
 
 /// Prints a structure type. Keeps track of known struct names to handle self-
 /// or mutually-referring structs without falling into infinite recursion.
-static void printStructType(AsmPrinter &printer, LLVMStructType type) {
+void LLVMStructType::print(AsmPrinter &printer) const {
   FailureOr<AsmPrinter::CyclicPrintReset> cyclicPrint;
 
   printer << "<";
-  if (type.isIdentified()) {
-    cyclicPrint = printer.tryStartCyclicPrint(type);
+  if (isIdentified()) {
+    cyclicPrint = printer.tryStartCyclicPrint(*this);
 
-    printer << '"' << type.getName() << '"';
+    printer << '"' << getName() << '"';
     // If we are printing a reference to one of the enclosing structs, just
     // print the name and stop to avoid infinitely long output.
     if (failed(cyclicPrint)) {
@@ -70,17 +70,17 @@ static void printStructType(AsmPrinter &printer, LLVMStructType type) {
     printer << ", ";
   }
 
-  if (type.isIdentified() && type.isOpaque()) {
+  if (isIdentified() && isOpaque()) {
     printer << "opaque>";
     return;
   }
 
-  if (type.isPacked())
+  if (isPacked())
     printer << "packed ";
 
   // Put the current type on stack to avoid infinite recursion.
   printer << '(';
-  llvm::interleaveComma(type.getBody(), printer.getStream(),
+  llvm::interleaveComma(getBody(), printer.getStream(),
                         [&](Type subtype) { dispatchPrint(printer, subtype); });
   printer << ')';
   printer << '>';
@@ -105,11 +105,8 @@ void mlir::LLVM::detail::printType(Type type, AsmPrinter &printer) {
 
   llvm::TypeSwitch<Type>(type)
       .Case<LLVMPointerType, LLVMArrayType, LLVMFixedVectorType,
-            LLVMScalableVectorType, LLVMFunctionType, LLVMTargetExtType>(
-          [&](auto type) { type.print(printer); })
-      .Case([&](LLVMStructType structType) {
-        printStructType(printer, structType);
-      });
+            LLVMScalableVectorType, LLVMFunctionType, LLVMTargetExtType,
+            LLVMStructType>([&](auto type) { type.print(printer); });
 }
 
 //===----------------------------------------------------------------------===//
@@ -182,7 +179,7 @@ static LLVMStructType trySetStructBody(LLVMStructType type,
 ///                 `(` llvm-type-list `)` `>`
 ///               | `struct<` string-literal `>`
 ///               | `struct<` string-literal `, opaque>`
-static LLVMStructType parseStructType(AsmParser &parser) {
+Type LLVMStructType::parse(AsmParser &parser) {
   Location loc = parser.getEncodedSourceLoc(parser.getCurrentLocation());
 
   if (failed(parser.parseLess()))
@@ -316,7 +313,7 @@ static Type dispatchParse(AsmParser &parser, bool allowAny = true) {
       .Case("ptr", [&] { return LLVMPointerType::parse(parser); })
       .Case("vec", [&] { return parseVectorType(parser); })
       .Case("array", [&] { return LLVMArrayType::parse(parser); })
-      .Case("struct", [&] { return parseStructType(parser); })
+      .Case("struct", [&] { return LLVMStructType::parse(parser); })
       .Case("target", [&] { return LLVMTargetExtType::parse(parser); })
       .Case("x86_amx", [&] { return LLVMX86AMXType::get(ctx); })
       .Default([&] {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index 33c231e2d2045..4e5600c715915 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -480,7 +480,7 @@ LogicalResult LLVMStructType::setBody(ArrayRef<Type> types, bool isPacked) {
 
 bool LLVMStructType::isPacked() const { return getImpl()->isPacked(); }
 bool LLVMStructType::isIdentified() const { return getImpl()->isIdentified(); }
-bool LLVMStructType::isOpaque() {
+bool LLVMStructType::isOpaque() const {
   return getImpl()->isIdentified() &&
          (getImpl()->isOpaque() || !getImpl()->isInitialized());
 }
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index d9840e3923c4f..8973e87c063b3 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -1498,17 +1498,15 @@ static Operation *findPayloadOp(Block *body, bool initFirst = false) {
 
 void printShortForm(OpAsmPrinter &p, Operation *payloadOp) {
   SmallVector<StringRef> elidedAttrs;
+  std::string attrToElide;
   p << " { " << payloadOp->getName().getStringRef();
   for (const auto &attr : payloadOp->getAttrs()) {
-    if (auto fastAttr = dyn_cast<arith::FastMathFlagsAttr>(attr.getValue())) {
-      if (fastAttr.getValue() == arith::FastMathFlags::none) {
-        elidedAttrs.push_back(attr.getName());
-      }
-    }
-    if (auto denormAttr = dyn_cast<arith::DenormalModeAttr>(attr.getValue())) {
-      if (denormAttr.getValue() == arith::DenormalMode::ieee) {
-        elidedAttrs.push_back(attr.getName());
-      }
+    auto fastAttr =
+        llvm::dyn_cast<mlir::arith::FastMathFlagsAttr>(attr.getValue());
+    if (fastAttr && fastAttr.getValue() == mlir::arith::FastMathFlags::none) {
+      attrToElide = attr.getName().str();
+      elidedAttrs.push_back(attrToElide);
+      break;
     }
   }
   p.printOptionalAttrDict(payloadOp->getAttrs(), elidedAttrs);
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index e08be7d2ebd6a..8839faf4cafb2 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -1176,7 +1176,9 @@ DiagnosedSilenceableFailure transform::LowerPackOp::applyToOne(
     transform::ApplyToEachResultList &transformResults,
     transform::TransformState &state) {
   rewriter.setInsertionPoint(target);
-  FailureOr<LowerPackResult> res = lowerPack(rewriter, target);
+  bool lowerPadLikeWithInsertSlice = getLowerPadLikeWithInsertSlice();
+  FailureOr<LowerPackResult> res =
+      lowerPack(rewriter, target, lowerPadLikeWithInsertSlice);
   if (failed(res)) {
     return mlir::emitSilenceableFailure(target->getLoc())
            << "cannot lower to pad + expand + transpose";
@@ -1196,7 +1198,9 @@ DiagnosedSilenceableFailure transform::LowerUnPackOp::applyToOne(
     transform::ApplyToEachResultList &transformResults,
     transform::TransformState &state) {
   rewriter.setInsertionPoint(target);
-  FailureOr<LowerUnPackOpResult> res = lowerUnPack(rewriter, target);
+  bool lowerUnpadLikeWithExtractSlice = getLowerUnpadLikeWithExtractSlice();
+  FailureOr<LowerUnPackOpResult> res =
+      lowerUnPack(rewriter, target, lowerUnpadLikeWithExtractSlice);
   if (failed(res)) {
     DiagnosedSilenceableFailure diag =
         emitSilenceableError()
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index eeaa70c0b6589..21141f161057e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -217,7 +217,8 @@ struct PackedOperandsDimList {
 } // namespace
 
 FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
-                                             tensor::PackOp packOp) {
+                                             tensor::PackOp packOp,
+                                             bool lowerPadLikeWithInsertSlice) {
   // 1. Filter out NYI cases.
   auto packedTensorType =
       cast<RankedTensorType>(packOp->getResultTypes().front());
@@ -295,7 +296,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       llvm::interleaveComma(stripMinedShape, DBGS() << "stripMinedShape: ");
       DBGSNL(); DBGS() << "collapsed type: " << collapsed; DBGSNL(););
 
-  if (packOp.isLikePad()) {
+  if (lowerPadLikeWithInsertSlice && packOp.isLikePad()) {
     // Pack ops which operate as simple pads may not produce legal
     // tensor.insert_slice operations when the packed type does not rank reduce
     // to the padded type.
@@ -351,8 +352,9 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   return LowerPackResult{padOp, reshapeOp, transposeOp};
 }
 
-FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
-                                                   tensor::UnPackOp unPackOp) {
+FailureOr<LowerUnPackOpResult>
+linalg::lowerUnPack(RewriterBase &rewriter, tensor::UnPackOp unPackOp,
+                    bool lowerUnpadLikeWithExtractSlice) {
   Location loc = unPackOp->getLoc();
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(unPackOp);
@@ -362,7 +364,7 @@ FailureOr<LowerUnPackOpResult> linalg::lowerUnPack(RewriterBase &rewriter,
 
   OpFoldResult zero = rewriter.getIndexAttr(0), one = rewriter.getIndexAttr(1);
   auto destTensorType = cast<RankedTensorType>(unPackOp.getDest().getType());
-  if (unPackOp.isLikeUnPad()) {
+  if (lowerUnpadLikeWithExtractSlice && unPackOp.isLikeUnPad()) {
     // This unpack is just a plain unpad.
     // Just extract the slice from the higher ranked tensor.
     ArrayRef<int64_t> destShape = destTensorType.getShape();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 06bb6c0fb1cac..863f2280e46ce 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1115,11 +1115,6 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
   //  b. contiguous loads.
   // Both cases use vector.transfer_read.
 
-  assert(llvm::count_if(resultType.getShape(),
-                        [](uint64_t dim) { return dim != 1; }) &&
-         "Contiguous loads and scalar loads + broadcast only support 1-D "
-         "vectors ATM!");
-
   // Collect indices for `vector.transfer_read`. At this point, the indices will
   // either be scalars or would have been broadcast to vectors matching the
   // result type. For indices that are vectors, there are two options:
@@ -1165,8 +1160,18 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
         permutationMap, inBounds);
 
+    // Mask this broadcasting xfer_read here rather than relying on the generic
+    // path (the generic path assumes identity masking map, which wouldn't be
+    // valid here).
+    SmallVector<int64_t> readMaskShape = {1};
+    auto readMaskType = VectorType::get(readMaskShape, rewriter.getI1Type());
+    auto allTrue = rewriter.create<vector::ConstantMaskOp>(
+        loc, readMaskType, vector::ConstantMaskKind::AllTrue);
+    auto *maskedReadOp =
+        mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
+
     LDBG("Vectorised as scalar broadcast load: " << extractOp << "\n");
-    return VectorizationResult{VectorizationStatus::NewOp, transferReadOp};
+    return VectorizationResult{VectorizationStatus::NewOp, maskedReadOp};
   }
 
   // 2b. Handle contiguous access.
@@ -2022,26 +2027,36 @@ vectorizeScalableVectorPrecondition(Operation *op,
 
   // Cond 3: Look at the configuration in `inputScalableVecDims` and verify that
   // it matches one of the supported cases:
-  //  1. exactly 1 dim is scalable and that's the _last_ parallel dim
-  //  2. exactly 2 dims are scalable and those are the _last two adjacent_
-  //     parallel dims
-  //  3. exactly 1 reduction dim is scalable and that's the last (innermost) dim
+  //  1. Exactly 1 dim is scalable and that's the _last_ non-unit parallel dim
+  //    (*).
+  //  2. Exactly 2 dims are scalable and those are the _last two adjacent_
+  //     parallel dims.
+  //  3. Exactly 1 reduction dim is scalable and that's the last (innermost)
+  //  dim.
   // The 2nd restriction above means that only Matmul-like Ops are supported
   // when 2 dims are scalable, e.g. :
   //    * iterators = [parallel, parallel, reduction]
   //    * scalable flags = [true, true, false]
+  //
+  // (*) Non-unit dims get folded away in practice.
+  // TODO: Relax these conditions as good motivating examples are identified.
 
-  // Find the first scalable flag
-  bool seenParalell = false;
+  // Find the first scalable flag.
+  bool seenNonUnitParallel = false;
   auto iterators = linalgOp.getIteratorTypesArray();
   SmallVector<bool> scalableFlags(inputScalableVecDims);
-  while (!scalableFlags.back()) {
-    seenParalell |= (iterators.back() == utils::IteratorType::parallel);
+  int64_t idx = scalableFlags.size() - 1;
+  while (!scalableFlags[idx]) {
+    bool isNonUnitDim = (inputVectorSizes[idx] != 1);
+    seenNonUnitParallel |=
+        (iterators[idx] == utils::IteratorType::parallel && isNonUnitDim);
 
     iterators.pop_back();
     scalableFlags.pop_back();
+    --idx;
   }
 
+  // Analyze the iterator corresponding to the first scalable dim.
   switch (iterators.back()) {
   case utils::IteratorType::reduction: {
     // Check 3. above is met.
@@ -2059,7 +2074,7 @@ vectorizeScalableVectorPrecondition(Operation *op,
   }
   case utils::IteratorType::parallel: {
     // Check 1. and 2. above are met.
-    if (seenParalell) {
+    if (seenNonUnitParallel) {
       LDBG("Inner parallel dim not requested for scalable "
            "vectorization\n");
       return failure();
diff --git a/mlir/lib/Dialect/MPI/IR/CMakeLists.txt b/mlir/lib/Dialect/MPI/IR/CMakeLists.txt
index 29d0b2379da74..edd7d911d0248 100644
--- a/mlir/lib/Dialect/MPI/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/MPI/IR/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_dialect_library(MLIRMPIDialect
   LINK_LIBS PUBLIC
   MLIRDialect
   MLIRIR
+  MLIRMemRefDialect
   MLIRInferTypeOpInterface
   MLIRSideEffectInterfaces
   )
diff --git a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
index ddd77b8f586ee..dcb55d8921364 100644
--- a/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
+++ b/mlir/lib/Dialect/MPI/IR/MPIOps.cpp
@@ -7,12 +7,52 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/MPI/IR/MPI.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/PatternMatch.h"
 
 using namespace mlir;
 using namespace mlir::mpi;
 
+namespace {
+
+// If input memref has dynamic shape and is a cast and if the cast's input has
+// static shape, fold the cast's static input into the given operation.
+template <typename OpT>
+struct FoldCast final : public mlir::OpRewritePattern<OpT> {
+  using mlir::OpRewritePattern<OpT>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(OpT op,
+                                mlir::PatternRewriter &b) const override {
+    auto mRef = op.getRef();
+    if (mRef.getType().hasStaticShape()) {
+      return mlir::failure();
+    }
+    auto defOp = mRef.getDefiningOp();
+    if (!defOp || !mlir::isa<mlir::memref::CastOp>(defOp)) {
+      return mlir::failure();
+    }
+    auto src = mlir::cast<mlir::memref::CastOp>(defOp).getSource();
+    if (!src.getType().hasStaticShape()) {
+      return mlir::failure();
+    }
+    op.getRefMutable().assign(src);
+    return mlir::success();
+  }
+};
+} // namespace
+
+void mlir::mpi::SendOp::getCanonicalizationPatterns(
+    mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
+  results.add<FoldCast<mlir::mpi::SendOp>>(context);
+}
+
+void mlir::mpi::RecvOp::getCanonicalizationPatterns(
+    mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
+  results.add<FoldCast<mlir::mpi::RecvOp>>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index 80569d95137c3..8bcbdb4c9a664 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -343,6 +343,7 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
   Value operandB = op.getOperand(1);
   Type opType = operandA.getType();
   Value zero = createFloatConst(op->getLoc(), opType, 0.00, rewriter);
+  Value one = createFloatConst(op->getLoc(), opType, 1.00, rewriter);
   Value two = createFloatConst(op->getLoc(), opType, 2.00, rewriter);
   Value negOne = createFloatConst(op->getLoc(), opType, -1.00, rewriter);
   Value opASquared = b.create<arith::MulFOp>(opType, operandA, operandA);
@@ -359,8 +360,15 @@ static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
       b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, remainder, zero);
   Value oddAndNeg = b.create<arith::AndIOp>(op->getLoc(), oddPower, negCheck);
 
+  // First, we select between the exp value and the adjusted value for odd
+  // powers of negatives. Then, we ensure that one is produced if `b` is zero.
+  // This corresponds to `libm` behavior, even for `0^0`. Without this check,
+  // `exp(0 * ln(0)) = exp(0 *-inf) = exp(-nan) = -nan`.
+  Value zeroCheck =
+      b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, operandB, zero);
   Value res = b.create<arith::SelectOp>(op->getLoc(), oddAndNeg, negExpResult,
                                         expResult);
+  res = b.create<arith::SelectOp>(op->getLoc(), zeroCheck, one, res);
   rewriter.replaceOp(op, res);
   return success();
 }
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index c5570d8ee8a44..33460ff25e9e4 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -837,6 +837,25 @@ void ProcessLinearIndexOp::getAsmResultNames(
   setNameFn(getResult(), "proc_linear_idx");
 }
 
+//===----------------------------------------------------------------------===//
+// mesh.neighbors_linear_indices op
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+NeighborsLinearIndicesOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
+  auto mesh = ::getMeshAndVerify(getOperation(), getMeshAttr(), symbolTable);
+  if (failed(mesh)) {
+    return failure();
+  }
+  return success();
+}
+
+void NeighborsLinearIndicesOp::getAsmResultNames(
+    function_ref<void(Value, StringRef)> setNameFn) {
+  setNameFn(getNeighborDown(), "down_linear_idx");
+  setNameFn(getNeighborUp(), "up_linear_idx");
+}
+
 //===----------------------------------------------------------------------===//
 // collective communication ops
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index b4d088cbd7088..327ea0991e4e1 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -496,11 +496,9 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
               sourceShard.getLoc(),
               RankedTensorType::get(outShape,
                                     sourceShard.getType().getElementType()),
-              sourceShard, initOprnd, mesh.getSymName(),
+              initOprnd, mesh.getSymName(),
               MeshAxesArrayAttr::get(builder.getContext(),
                                      sourceSharding.getSplitAxes()),
-              sourceSharding.getDynamicHaloSizes(),
-              sourceSharding.getStaticHaloSizes(),
               targetSharding.getDynamicHaloSizes(),
               targetSharding.getStaticHaloSizes())
           .getResult();
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 280260e0485bb..d490376db0e7f 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Matchers.h"
@@ -18,6 +19,7 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/LogicalResult.h"
 
 using namespace mlir;
 using namespace acc;
@@ -190,6 +192,48 @@ static LogicalResult checkWaitAndAsyncConflict(Op op) {
   return success();
 }
 
+static ParseResult parseVarPtrType(mlir::OpAsmParser &parser,
+                                   mlir::Type &varPtrType,
+                                   mlir::TypeAttr &varTypeAttr) {
+  if (failed(parser.parseType(varPtrType)))
+    return failure();
+  if (failed(parser.parseRParen()))
+    return failure();
+
+  if (succeeded(parser.parseOptionalKeyword("varType"))) {
+    if (failed(parser.parseLParen()))
+      return failure();
+    mlir::Type varType;
+    if (failed(parser.parseType(varType)))
+      return failure();
+    varTypeAttr = mlir::TypeAttr::get(varType);
+    if (failed(parser.parseRParen()))
+      return failure();
+  } else {
+    // Set `varType` from the element type of the type of `varPtr`.
+    varTypeAttr = mlir::TypeAttr::get(
+        mlir::cast<mlir::acc::PointerLikeType>(varPtrType).getElementType());
+  }
+
+  return success();
+}
+
+static void printVarPtrType(mlir::OpAsmPrinter &p, mlir::Operation *op,
+                            mlir::Type varPtrType, mlir::TypeAttr varTypeAttr) {
+  p.printType(varPtrType);
+  p << ")";
+
+  // Print the `varType` only if it differs from the element type of
+  // `varPtr`'s type.
+  mlir::Type varType = varTypeAttr.getValue();
+  if (mlir::cast<mlir::acc::PointerLikeType>(varPtrType).getElementType() !=
+      varType) {
+    p << " varType(";
+    p.printType(varType);
+    p << ")";
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // DataBoundsOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 94e71e089d4b1..e20530be07b2f 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -487,9 +487,11 @@ struct PrivateParseArgs {
   llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
   llvm::SmallVectorImpl<Type> &types;
   ArrayAttr &syms;
+  DenseI64ArrayAttr *mapIndices;
   PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
-                   SmallVectorImpl<Type> &types, ArrayAttr &syms)
-      : vars(vars), types(types), syms(syms) {}
+                   SmallVectorImpl<Type> &types, ArrayAttr &syms,
+                   DenseI64ArrayAttr *mapIndices = nullptr)
+      : vars(vars), types(types), syms(syms), mapIndices(mapIndices) {}
 };
 struct ReductionParseArgs {
   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
@@ -517,8 +519,10 @@ static ParseResult parseClauseWithRegionArgs(
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
     SmallVectorImpl<Type> &types,
     SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,
-    ArrayAttr *symbols = nullptr, DenseBoolArrayAttr *byref = nullptr) {
+    ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,
+    DenseBoolArrayAttr *byref = nullptr) {
   SmallVector<SymbolRefAttr> symbolVec;
+  SmallVector<int64_t> mapIndicesVec;
   SmallVector<bool> isByRefVec;
   unsigned regionArgOffset = regionPrivateArgs.size();
 
@@ -538,6 +542,16 @@ static ParseResult parseClauseWithRegionArgs(
             parser.parseArgument(regionPrivateArgs.emplace_back()))
           return failure();
 
+        if (mapIndices) {
+          if (parser.parseOptionalLSquare().succeeded()) {
+            if (parser.parseKeyword("map_idx") || parser.parseEqual() ||
+                parser.parseInteger(mapIndicesVec.emplace_back()) ||
+                parser.parseRSquare())
+              return failure();
+          } else
+            mapIndicesVec.push_back(-1);
+        }
+
         return success();
       }))
     return failure();
@@ -571,6 +585,10 @@ static ParseResult parseClauseWithRegionArgs(
     *symbols = ArrayAttr::get(parser.getContext(), symbolAttrs);
   }
 
+  if (!mapIndicesVec.empty())
+    *mapIndices =
+        mlir::DenseI64ArrayAttr::get(parser.getContext(), mapIndicesVec);
+
   if (byref)
     *byref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
 
@@ -595,14 +613,14 @@ static ParseResult parseBlockArgClause(
 static ParseResult parseBlockArgClause(
     OpAsmParser &parser,
     llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
-    StringRef keyword, std::optional<PrivateParseArgs> reductionArgs) {
+    StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
   if (succeeded(parser.parseOptionalKeyword(keyword))) {
-    if (!reductionArgs)
+    if (!privateArgs)
       return failure();
 
-    if (failed(parseClauseWithRegionArgs(parser, reductionArgs->vars,
-                                         reductionArgs->types, entryBlockArgs,
-                                         &reductionArgs->syms)))
+    if (failed(parseClauseWithRegionArgs(
+            parser, privateArgs->vars, privateArgs->types, entryBlockArgs,
+            &privateArgs->syms, privateArgs->mapIndices)))
       return failure();
   }
   return success();
@@ -618,7 +636,8 @@ static ParseResult parseBlockArgClause(
 
     if (failed(parseClauseWithRegionArgs(
             parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,
-            &reductionArgs->syms, &reductionArgs->byref)))
+            &reductionArgs->syms, /*mapIndices=*/nullptr,
+            &reductionArgs->byref)))
       return failure();
   }
   return success();
@@ -674,12 +693,14 @@ static ParseResult parseInReductionMapPrivateRegion(
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,
     SmallVectorImpl<Type> &mapTypes,
     llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
-    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms) {
+    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
+    DenseI64ArrayAttr &privateMaps) {
   AllRegionParseArgs args;
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
   args.mapArgs.emplace(mapVars, mapTypes);
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+                           &privateMaps);
   return parseBlockArgRegion(parser, region, args);
 }
 
@@ -776,8 +797,10 @@ struct PrivatePrintArgs {
   ValueRange vars;
   TypeRange types;
   ArrayAttr syms;
-  PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms)
-      : vars(vars), types(types), syms(syms) {}
+  DenseI64ArrayAttr mapIndices;
+  PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,
+                   DenseI64ArrayAttr mapIndices)
+      : vars(vars), types(types), syms(syms), mapIndices(mapIndices) {}
 };
 struct ReductionPrintArgs {
   ValueRange vars;
@@ -804,6 +827,7 @@ static void printClauseWithRegionArgs(OpAsmPrinter &p, MLIRContext *ctx,
                                       ValueRange argsSubrange,
                                       ValueRange operands, TypeRange types,
                                       ArrayAttr symbols = nullptr,
+                                      DenseI64ArrayAttr mapIndices = nullptr,
                                       DenseBoolArrayAttr byref = nullptr) {
   if (argsSubrange.empty())
     return;
@@ -815,21 +839,31 @@ static void printClauseWithRegionArgs(OpAsmPrinter &p, MLIRContext *ctx,
     symbols = ArrayAttr::get(ctx, values);
   }
 
+  if (!mapIndices) {
+    llvm::SmallVector<int64_t> values(operands.size(), -1);
+    mapIndices = DenseI64ArrayAttr::get(ctx, values);
+  }
+
   if (!byref) {
     mlir::SmallVector<bool> values(operands.size(), false);
     byref = DenseBoolArrayAttr::get(ctx, values);
   }
 
-  llvm::interleaveComma(
-      llvm::zip_equal(operands, argsSubrange, symbols, byref.asArrayRef()), p,
-      [&p](auto t) {
-        auto [op, arg, sym, isByRef] = t;
-        if (isByRef)
-          p << "byref ";
-        if (sym)
-          p << sym << " ";
-        p << op << " -> " << arg;
-      });
+  llvm::interleaveComma(llvm::zip_equal(operands, argsSubrange, symbols,
+                                        mapIndices.asArrayRef(),
+                                        byref.asArrayRef()),
+                        p, [&p](auto t) {
+                          auto [op, arg, sym, map, isByRef] = t;
+                          if (isByRef)
+                            p << "byref ";
+                          if (sym)
+                            p << sym << " ";
+
+                          p << op << " -> " << arg;
+
+                          if (map != -1)
+                            p << " [map_idx=" << map << "]";
+                        });
   p << " : ";
   llvm::interleaveComma(types, p);
   p << ") ";
@@ -849,7 +883,7 @@ static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
   if (privateArgs)
     printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
                               privateArgs->vars, privateArgs->types,
-                              privateArgs->syms);
+                              privateArgs->syms, privateArgs->mapIndices);
 }
 
 static void
@@ -859,7 +893,8 @@ printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
   if (reductionArgs)
     printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
                               reductionArgs->vars, reductionArgs->types,
-                              reductionArgs->syms, reductionArgs->byref);
+                              reductionArgs->syms, /*mapIndices=*/nullptr,
+                              reductionArgs->byref);
 }
 
 static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
@@ -891,12 +926,13 @@ static void printInReductionMapPrivateRegion(
     OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
     TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
     ArrayAttr inReductionSyms, ValueRange mapVars, TypeRange mapTypes,
-    ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms) {
+    ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
+    DenseI64ArrayAttr privateMaps) {
   AllRegionPrintArgs args;
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
   args.mapArgs.emplace(mapVars, mapTypes);
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms, privateMaps);
   printBlockArgRegion(p, op, region, args);
 }
 
@@ -908,7 +944,8 @@ static void printInReductionPrivateRegion(
   AllRegionPrintArgs args;
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+                           /*mapIndices=*/nullptr);
   printBlockArgRegion(p, op, region, args);
 }
 
@@ -921,7 +958,8 @@ static void printInReductionPrivateReductionRegion(
   AllRegionPrintArgs args;
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+                           /*mapIndices=*/nullptr);
   args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                              reductionSyms);
   printBlockArgRegion(p, op, region, args);
@@ -931,7 +969,8 @@ static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                ValueRange privateVars, TypeRange privateTypes,
                                ArrayAttr privateSyms) {
   AllRegionPrintArgs args;
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+                           /*mapIndices=*/nullptr);
   printBlockArgRegion(p, op, region, args);
 }
 
@@ -941,7 +980,8 @@ static void printPrivateReductionRegion(
     TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
     ArrayAttr reductionSyms) {
   AllRegionPrintArgs args;
-  args.privateArgs.emplace(privateVars, privateTypes, privateSyms);
+  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+                           /*mapIndices=*/nullptr);
   args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                              reductionSyms);
   printBlockArgRegion(p, op, region, args);
@@ -1560,6 +1600,24 @@ static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars) {
   return success();
 }
 
+static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {
+  std::optional<DenseI64ArrayAttr> privateMapIndices =
+      targetOp.getPrivateMapsAttr();
+
+  // None of the private operands are mapped.
+  if (!privateMapIndices.has_value() || !privateMapIndices.value())
+    return success();
+
+  OperandRange privateVars = targetOp.getPrivateVars();
+
+  if (privateMapIndices.value().size() !=
+      static_cast<int64_t>(privateVars.size()))
+    return emitError(targetOp.getLoc(), "sizes of `private` operand range and "
+                                        "`private_maps` attribute mismatch");
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TargetDataOp
 //===----------------------------------------------------------------------===//
@@ -1656,14 +1714,23 @@ void TargetOp::build(OpBuilder &builder, OperationState &state,
                   /*in_reduction_vars=*/{}, /*in_reduction_byref=*/nullptr,
                   /*in_reduction_syms=*/nullptr, clauses.isDevicePtrVars,
                   clauses.mapVars, clauses.nowait, clauses.privateVars,
-                  makeArrayAttr(ctx, clauses.privateSyms), clauses.threadLimit);
+                  makeArrayAttr(ctx, clauses.privateSyms), clauses.threadLimit,
+                  /*private_maps=*/nullptr);
 }
 
 LogicalResult TargetOp::verify() {
   LogicalResult verifyDependVars =
       verifyDependVarList(*this, getDependKinds(), getDependVars());
-  return failed(verifyDependVars) ? verifyDependVars
-                                  : verifyMapClause(*this, getMapVars());
+
+  if (failed(verifyDependVars))
+    return verifyDependVars;
+
+  LogicalResult verifyMapVars = verifyMapClause(*this, getMapVars());
+
+  if (failed(verifyMapVars))
+    return verifyMapVars;
+
+  return verifyPrivateVarsMapping(*this);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1999,8 +2066,8 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
       builder, state,
       /*allocate_vars=*/{}, /*allocator_vars=*/{}, clauses.linearVars,
       clauses.linearStepVars, clauses.nowait, clauses.order, clauses.orderMod,
-      clauses.ordered, /*private_vars=*/{}, /*private_syms=*/nullptr,
-      clauses.reductionVars,
+      clauses.ordered, clauses.privateVars,
+      makeArrayAttr(ctx, clauses.privateSyms), clauses.reductionVars,
       makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
       makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
       clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
@@ -2236,7 +2303,7 @@ void TaskOp::build(OpBuilder &builder, OperationState &state,
                 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
                 clauses.priority, /*private_vars=*/clauses.privateVars,
                 /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
-                clauses.untied);
+                clauses.untied, clauses.eventHandle);
 }
 
 LogicalResult TaskOp::verify() {
diff --git a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
index 17a1c016ea16d..fbd236b648cb8 100644
--- a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -95,6 +95,47 @@ struct ForOpInterface
   }
 };
 
+struct ForallOpInterface
+    : public ValueBoundsOpInterface::ExternalModel<ForallOpInterface,
+                                                   ForallOp> {
+
+  void populateBoundsForIndexValue(Operation *op, Value value,
+                                   ValueBoundsConstraintSet &cstr) const {
+    auto forallOp = cast<ForallOp>(op);
+
+    // Index values should be induction variables, since the semantics of
+    // tensor::ParallelInsertSliceOp requires forall outputs to be ranked
+    // tensors.
+    auto blockArg = cast<BlockArgument>(value);
+    assert(blockArg.getArgNumber() < forallOp.getInductionVars().size() &&
+           "expected index value to be an induction var");
+    int64_t idx = blockArg.getArgNumber();
+    // TODO: Take into account step size.
+    AffineExpr lb = cstr.getExpr(forallOp.getMixedLowerBound()[idx]);
+    AffineExpr ub = cstr.getExpr(forallOp.getMixedUpperBound()[idx]);
+    cstr.bound(value) >= lb;
+    cstr.bound(value) < ub;
+  }
+
+  void populateBoundsForShapedValueDim(Operation *op, Value value, int64_t dim,
+                                       ValueBoundsConstraintSet &cstr) const {
+    auto forallOp = cast<ForallOp>(op);
+
+    // `value` is an iter_arg or an OpResult.
+    int64_t iterArgIdx;
+    if (auto iterArg = llvm::dyn_cast<BlockArgument>(value)) {
+      iterArgIdx = iterArg.getArgNumber() - forallOp.getInductionVars().size();
+    } else {
+      iterArgIdx = llvm::cast<OpResult>(value).getResultNumber();
+    }
+
+    // The forall results and output arguments have the same sizes as the output
+    // operands.
+    Value outputOperand = forallOp.getOutputs()[iterArgIdx];
+    cstr.bound(value)[dim] == cstr.getExpr(outputOperand, dim);
+  }
+};
+
 struct IfOpInterface
     : public ValueBoundsOpInterface::ExternalModel<IfOpInterface, IfOp> {
 
@@ -161,6 +202,7 @@ void mlir::scf::registerValueBoundsOpInterfaceExternalModels(
     DialectRegistry &registry) {
   registry.addExtension(+[](MLIRContext *ctx, scf::SCFDialect *dialect) {
     scf::ForOp::attachInterface<scf::ForOpInterface>(*ctx);
+    scf::ForallOp::attachInterface<scf::ForallOpInterface>(*ctx);
     scf::IfOp::attachInterface<scf::IfOpInterface>(*ctx);
   });
 }
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 93a78056db194..c0589044c26ec 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -16,20 +16,18 @@ using namespace mlir::scf;
 
 namespace {
 
-// Unpacks the single unrealized_conversion_cast using the list of inputs
-// e.g., return [%b, %c, %d] for %a = unrealized_conversion_cast(%b, %c, %d)
-static void unpackUnrealizedConversionCast(Value v,
-                                           SmallVectorImpl<Value> &unpacked) {
-  if (auto cast =
-          dyn_cast_or_null<UnrealizedConversionCastOp>(v.getDefiningOp())) {
-    if (cast.getInputs().size() != 1) {
-      // 1 : N type conversion.
-      unpacked.append(cast.getInputs().begin(), cast.getInputs().end());
-      return;
-    }
-  }
-  // 1 : 1 type conversion.
-  unpacked.push_back(v);
+/// Flatten the given value ranges into a single vector of values.
+static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
+  SmallVector<Value> result;
+  for (const auto &vals : values)
+    llvm::append_range(result, vals);
+  return result;
+}
+
+/// Assert that the given value range contains a single value and return it.
+static Value getSingleValue(ValueRange values) {
+  assert(values.size() == 1 && "expected single value");
+  return values.front();
 }
 
 // CRTP
@@ -40,19 +38,21 @@ class Structural1ToNConversionPattern : public OpConversionPattern<SourceOp> {
 public:
   using OpConversionPattern<SourceOp>::typeConverter;
   using OpConversionPattern<SourceOp>::OpConversionPattern;
-  using OpAdaptor = typename OpConversionPattern<SourceOp>::OpAdaptor;
+  using OneToNOpAdaptor =
+      typename OpConversionPattern<SourceOp>::OneToNOpAdaptor;
 
   //
   // Derived classes should provide the following method which performs the
   // actual conversion. It should return std::nullopt upon conversion failure
   // and return the converted operation upon success.
   //
-  // std::optional<SourceOp> convertSourceOp(SourceOp op, OpAdaptor adaptor,
-  //                                    ConversionPatternRewriter &rewriter,
-  //                                    TypeRange dstTypes) const;
+  // std::optional<SourceOp> convertSourceOp(
+  //     SourceOp op, OneToNOpAdaptor adaptor,
+  //     ConversionPatternRewriter &rewriter,
+  //     TypeRange dstTypes) const;
 
   LogicalResult
-  matchAndRewrite(SourceOp op, OpAdaptor adaptor,
+  matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     SmallVector<Type> dstTypes;
     SmallVector<unsigned> offsets;
@@ -73,28 +73,15 @@ class Structural1ToNConversionPattern : public OpConversionPattern<SourceOp> {
       return rewriter.notifyMatchFailure(op, "could not convert operation");
 
     // Packs the return value.
-    SmallVector<Value> packedRets;
+    SmallVector<ValueRange> packedRets;
     for (unsigned i = 1, e = offsets.size(); i < e; i++) {
       unsigned start = offsets[i - 1], end = offsets[i];
       unsigned len = end - start;
       ValueRange mappedValue = newOp->getResults().slice(start, len);
-      if (len != 1) {
-        // 1 : N type conversion.
-        Type origType = op.getResultTypes()[i - 1];
-        Value mat = typeConverter->materializeSourceConversion(
-            rewriter, op.getLoc(), origType, mappedValue);
-        if (!mat) {
-          return rewriter.notifyMatchFailure(
-              op, "Failed to materialize 1:N type conversion");
-        }
-        packedRets.push_back(mat);
-      } else {
-        // 1 : 1 type conversion.
-        packedRets.push_back(mappedValue.front());
-      }
+      packedRets.push_back(mappedValue);
     }
 
-    rewriter.replaceOp(op, packedRets);
+    rewriter.replaceOpWithMultiple(op, packedRets);
     return success();
   }
 };
@@ -105,7 +92,7 @@ class ConvertForOpTypes
   using Structural1ToNConversionPattern::Structural1ToNConversionPattern;
 
   // The callback required by CRTP.
-  std::optional<ForOp> convertSourceOp(ForOp op, OpAdaptor adaptor,
+  std::optional<ForOp> convertSourceOp(ForOp op, OneToNOpAdaptor adaptor,
                                        ConversionPatternRewriter &rewriter,
                                        TypeRange dstTypes) const {
     // Create a empty new op and inline the regions from the old op.
@@ -129,16 +116,13 @@ class ConvertForOpTypes
     if (failed(rewriter.convertRegionTypes(&op.getRegion(), *typeConverter)))
       return std::nullopt;
 
-    // Unpacked the iteration arguments.
-    SmallVector<Value> flatArgs;
-    for (Value arg : adaptor.getInitArgs())
-      unpackUnrealizedConversionCast(arg, flatArgs);
-
     // We can not do clone as the number of result types after conversion
     // might be different.
-    ForOp newOp = rewriter.create<ForOp>(op.getLoc(), adaptor.getLowerBound(),
-                                         adaptor.getUpperBound(),
-                                         adaptor.getStep(), flatArgs);
+    ForOp newOp = rewriter.create<ForOp>(
+        op.getLoc(), getSingleValue(adaptor.getLowerBound()),
+        getSingleValue(adaptor.getUpperBound()),
+        getSingleValue(adaptor.getStep()),
+        flattenValues(adaptor.getInitArgs()));
 
     // Reserve whatever attributes in the original op.
     newOp->setAttrs(op->getAttrs());
@@ -160,12 +144,12 @@ class ConvertIfOpTypes
 public:
   using Structural1ToNConversionPattern::Structural1ToNConversionPattern;
 
-  std::optional<IfOp> convertSourceOp(IfOp op, OpAdaptor adaptor,
+  std::optional<IfOp> convertSourceOp(IfOp op, OneToNOpAdaptor adaptor,
                                       ConversionPatternRewriter &rewriter,
                                       TypeRange dstTypes) const {
 
-    IfOp newOp = rewriter.create<IfOp>(op.getLoc(), dstTypes,
-                                       adaptor.getCondition(), true);
+    IfOp newOp = rewriter.create<IfOp>(
+        op.getLoc(), dstTypes, getSingleValue(adaptor.getCondition()), true);
     newOp->setAttrs(op->getAttrs());
 
     // We do not need the empty blocks created by rewriter.
@@ -189,15 +173,11 @@ class ConvertWhileOpTypes
 public:
   using Structural1ToNConversionPattern::Structural1ToNConversionPattern;
 
-  std::optional<WhileOp> convertSourceOp(WhileOp op, OpAdaptor adaptor,
+  std::optional<WhileOp> convertSourceOp(WhileOp op, OneToNOpAdaptor adaptor,
                                          ConversionPatternRewriter &rewriter,
                                          TypeRange dstTypes) const {
-    // Unpacked the iteration arguments.
-    SmallVector<Value> flatArgs;
-    for (Value arg : adaptor.getOperands())
-      unpackUnrealizedConversionCast(arg, flatArgs);
-
-    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes, flatArgs);
+    auto newOp = rewriter.create<WhileOp>(op.getLoc(), dstTypes,
+                                          flattenValues(adaptor.getOperands()));
 
     for (auto i : {0u, 1u}) {
       if (failed(rewriter.convertRegionTypes(&op.getRegion(i), *typeConverter)))
@@ -218,13 +198,10 @@ class ConvertYieldOpTypes : public OpConversionPattern<scf::YieldOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(scf::YieldOp op, OpAdaptor adaptor,
+  matchAndRewrite(scf::YieldOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    SmallVector<Value> unpackedYield;
-    for (Value operand : adaptor.getOperands())
-      unpackUnrealizedConversionCast(operand, unpackedYield);
-
-    rewriter.replaceOpWithNewOp<scf::YieldOp>(op, unpackedYield);
+    rewriter.replaceOpWithNewOp<scf::YieldOp>(
+        op, flattenValues(adaptor.getOperands()));
     return success();
   }
 };
@@ -235,13 +212,10 @@ class ConvertConditionOpTypes : public OpConversionPattern<ConditionOp> {
 public:
   using OpConversionPattern<ConditionOp>::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ConditionOp op, OpAdaptor adaptor,
+  matchAndRewrite(ConditionOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    SmallVector<Value> unpackedYield;
-    for (Value operand : adaptor.getOperands())
-      unpackUnrealizedConversionCast(operand, unpackedYield);
-
-    rewriter.modifyOpInPlace(op, [&]() { op->setOperands(unpackedYield); });
+    rewriter.modifyOpInPlace(
+        op, [&]() { op->setOperands(flattenValues(adaptor.getOperands())); });
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index f5700059f68ee..877ac87fb0fe5 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -292,6 +292,8 @@ convertScalarType(const spirv::TargetEnv &targetEnv,
 }
 
 /// Converts a sub-byte integer `type` to i32 regardless of target environment.
+/// Returns a nullptr for unsupported integer types, including non sub-byte
+/// types.
 ///
 /// Note that we don't recognize sub-byte types in `spirv::ScalarType` and use
 /// the above given that these sub-byte types are not supported at all in
@@ -299,6 +301,10 @@ convertScalarType(const spirv::TargetEnv &targetEnv,
 /// supported integer types.
 static Type convertSubByteIntegerType(const SPIRVConversionOptions &options,
                                       IntegerType type) {
+  if (type.getWidth() > 8) {
+    LLVM_DEBUG(llvm::dbgs() << "not a subbyte type\n");
+    return nullptr;
+  }
   if (options.subByteTypeStorage != SPIRVSubByteTypeStorage::Packed) {
     LLVM_DEBUG(llvm::dbgs() << "unsupported sub-byte storage kind\n");
     return nullptr;
@@ -348,6 +354,9 @@ convertVectorType(const spirv::TargetEnv &targetEnv,
     }
 
     Type elementType = convertSubByteIntegerType(options, intType);
+    if (!elementType)
+      return nullptr;
+
     if (type.getRank() <= 1 && type.getNumElements() == 1)
       return elementType;
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 25fca49cb0154..20d46f7ca00c5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -39,25 +39,18 @@ using namespace mlir::sparse_tensor;
 // Helper methods.
 //===----------------------------------------------------------------------===//
 
-/// Flattens a list of operands that may contain sparse tensors.
-static void flattenOperands(ValueRange operands,
-                            SmallVectorImpl<Value> &flattened) {
-  // In case of
-  // sparse_tensor, c, sparse_tensor
-  // ==>
-  // memref ..., c, memref ...
-  for (auto operand : operands) {
-    if (getSparseTensorEncoding(operand.getType())) {
-      auto tuple = getTuple(operand);
-      // An unrealized_conversion_cast will be inserted by type converter to
-      // inter-mix the gap between 1:N conversion between sparse tensors and
-      // fields. In this case, take the operands in the cast and replace the
-      // sparse tensor output with the flattened type array.
-      flattened.append(tuple.getOperands().begin(), tuple.getOperands().end());
-    } else {
-      flattened.push_back(operand);
-    }
-  }
+/// Flatten the given value ranges into a single vector of values.
+static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
+  SmallVector<Value> result;
+  for (const auto &vals : values)
+    llvm::append_range(result, vals);
+  return result;
+}
+
+/// Assert that the given value range contains a single value and return it.
+static Value getSingleValue(ValueRange values) {
+  assert(values.size() == 1 && "expected single value");
+  return values.front();
 }
 
 /// Generates a load with proper `index` typing.
@@ -567,12 +560,11 @@ class SparseReturnConverter : public OpConversionPattern<func::ReturnOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(func::ReturnOp op, OpAdaptor adaptor,
+  matchAndRewrite(func::ReturnOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    SmallVector<Value> flattened;
-    flattenOperands(adaptor.getOperands(), flattened);
     // Create a return with the flattened value extracted from sparse tensors.
-    rewriter.replaceOpWithNewOp<func::ReturnOp>(op, flattened);
+    rewriter.replaceOpWithNewOp<func::ReturnOp>(
+        op, flattenValues(adaptor.getOperands()));
     return success();
   }
 };
@@ -583,7 +575,7 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
   // The default CallOp converter can not handle 1:N type conversion.
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(func::CallOp op, OpAdaptor adaptor,
+  matchAndRewrite(func::CallOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     // In case of:
@@ -596,10 +588,8 @@ class SparseCallConverter : public OpConversionPattern<func::CallOp> {
       return failure();
 
     // (1) Generates new call with flattened return value.
-    SmallVector<Value> flattened;
-    flattenOperands(adaptor.getOperands(), flattened);
-    auto newCall = rewriter.create<func::CallOp>(loc, op.getCallee(),
-                                                 finalRetTy, flattened);
+    auto newCall = rewriter.create<func::CallOp>(
+        loc, op.getCallee(), finalRetTy, flattenValues(adaptor.getOperands()));
     // (2) Gather sparse tensor returns.
     SmallVector<SmallVector<Value>> packedResultVals;
     // Tracks the offset of current return value (of the original call)
@@ -643,7 +633,7 @@ class SparseLvlOpConverter : public OpConversionPattern<LvlOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(LvlOp op, OpAdaptor adaptor,
+  matchAndRewrite(LvlOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     std::optional<int64_t> lvl = op.getConstantLvlIndex();
     RankedTensorType srcType = op.getSource().getType();
@@ -662,7 +652,7 @@ class SparseLvlOpConverter : public OpConversionPattern<LvlOp> {
 struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ReorderCOOOp op, ReorderCOOOpAdaptor adaptor,
+  matchAndRewrite(ReorderCOOOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     MLIRContext *ctx = op.getContext();
@@ -693,7 +683,7 @@ struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
 
     // Since we do in-place sorting, the destinate tensor will have the same set
     // of memrefs as the source tensor.
-    rewriter.replaceOp(op, adaptor.getInputCoo());
+    rewriter.replaceOpWithMultiple(op, {adaptor.getInputCoo()});
     return success();
   }
 };
@@ -702,8 +692,10 @@ template <typename Op, StorageSpecifierKind kind>
 class SparseSliceGetterOpConverter : public OpConversionPattern<Op> {
 public:
   using OpConversionPattern<Op>::OpConversionPattern;
+  using typename OpConversionPattern<Op>::OneToNOpAdaptor;
+
   LogicalResult
-  matchAndRewrite(Op op, typename Op::Adaptor adaptor,
+  matchAndRewrite(Op op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Simply lowers to specifer.get <field> operation.
     auto desc = getDescriptorFromTensorTuple(adaptor.getSlice(),
@@ -721,14 +713,14 @@ class SparseCastConverter : public OpConversionPattern<tensor::CastOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(tensor::CastOp op, OpAdaptor adaptor,
+  matchAndRewrite(tensor::CastOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Only rewrite identically annotated source/dest.
     auto encDst = getSparseTensorEncoding(op.getType());
     auto encSrc = getSparseTensorEncoding(op.getSource().getType());
     if (!encDst || encDst != encSrc)
       return failure();
-    rewriter.replaceOp(op, adaptor.getOperands());
+    rewriter.replaceOpWithMultiple(op, {adaptor.getSource()});
     return success();
   }
 };
@@ -737,10 +729,10 @@ class SparseReMapConverter : public OpConversionPattern<ReinterpretMapOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ReinterpretMapOp op, OpAdaptor adaptor,
+  matchAndRewrite(ReinterpretMapOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Simply fold the operation.
-    rewriter.replaceOp(op, adaptor.getSource());
+    rewriter.replaceOpWithMultiple(op, {adaptor.getSource()});
     return success();
   }
 };
@@ -756,7 +748,7 @@ class SparseTensorAllocConverter
         enableBufferInitialization(enableInit) {}
 
   LogicalResult
-  matchAndRewrite(bufferization::AllocTensorOp op, OpAdaptor adaptor,
+  matchAndRewrite(bufferization::AllocTensorOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     const auto resType = getSparseTensorType(op);
     if (!resType.hasEncoding())
@@ -791,7 +783,8 @@ class SparseTensorAllocConverter
     }
     // Level size equals to dimension size since lvl2dim map is an identity map.
     SmallVector<Value> lvlSizesValues;
-    createDimSizes(rewriter, loc, resType, adaptor.getDynamicSizes(),
+    createDimSizes(rewriter, loc, resType,
+                   flattenValues(adaptor.getDynamicSizes()),
                    /*dimSizesValues=*/lvlSizesValues);
 
     // Construct allocation for each field.
@@ -861,7 +854,7 @@ class SparseTensorDeallocConverter
         createDeallocs(createDeallocs) {}
 
   LogicalResult
-  matchAndRewrite(bufferization::DeallocTensorOp op, OpAdaptor adaptor,
+  matchAndRewrite(bufferization::DeallocTensorOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto enc = getSparseTensorEncoding(op.getTensor().getType());
     if (!enc)
@@ -892,7 +885,7 @@ class SparseTensorLoadConverter : public OpConversionPattern<LoadOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(LoadOp op, OpAdaptor adaptor,
+  matchAndRewrite(LoadOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Prepare descriptor.
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor(),
@@ -911,7 +904,7 @@ class SparseExpandConverter : public OpConversionPattern<ExpandOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ExpandOp op, OpAdaptor adaptor,
+  matchAndRewrite(ExpandOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     if (!getSparseTensorEncoding(op.getTensor().getType()))
       return failure();
@@ -963,16 +956,16 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(CompressOp op, OpAdaptor adaptor,
+  matchAndRewrite(CompressOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     SmallVector<Value> fields;
     auto desc = getMutDescriptorFromTensorTuple(adaptor.getTensor(), fields,
                                                 op.getTensor().getType());
-    Value values = adaptor.getValues();
-    Value filled = adaptor.getFilled();
-    Value added = adaptor.getAdded();
-    Value count = adaptor.getCount();
+    Value values = getSingleValue(adaptor.getValues());
+    Value filled = getSingleValue(adaptor.getFilled());
+    Value added = getSingleValue(adaptor.getAdded());
+    Value count = getSingleValue(adaptor.getCount());
     const SparseTensorType dstType(desc.getRankedTensorType());
     Type eltType = dstType.getElementType();
 
@@ -1005,7 +998,8 @@ class SparseCompressConverter : public OpConversionPattern<CompressOp> {
     SmallVector<Value> params(desc.getFields().begin(), desc.getFields().end());
     SmallVector<Type> flatSpTensorTps = llvm::to_vector(
         llvm::map_range(desc.getFields(), [](Value v) { return v.getType(); }));
-    params.append(adaptor.getLvlCoords().begin(), adaptor.getLvlCoords().end());
+    SmallVector<Value> flatLvlCoords = flattenValues(adaptor.getLvlCoords());
+    params.append(flatLvlCoords.begin(), flatLvlCoords.end());
     params.push_back(crd);
     params.push_back(value);
     SparseInsertGenerator insertGen(op.getTensor().getType(), flatSpTensorTps,
@@ -1033,9 +1027,9 @@ class SparseInsertConverter : public OpConversionPattern<tensor::InsertOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(tensor::InsertOp op, OpAdaptor adaptor,
+  matchAndRewrite(tensor::InsertOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto stt = getSparseTensorType(adaptor.getDest());
+    auto stt = getSparseTensorType(op.getDest());
     if (!stt.hasEncoding())
       return failure();
     assert(stt.isIdentity() && "Run reinterpret-map before conversion.");
@@ -1045,8 +1039,9 @@ class SparseInsertConverter : public OpConversionPattern<tensor::InsertOp> {
         getDescriptorFromTensorTuple(adaptor.getDest(), op.getDest().getType());
     TypeRange flatSpTensorTps = desc.getFields().getTypes();
     SmallVector<Value> params = llvm::to_vector(desc.getFields());
-    params.append(adaptor.getIndices().begin(), adaptor.getIndices().end());
-    params.push_back(adaptor.getScalar());
+    SmallVector<Value> flatIndices = flattenValues(adaptor.getIndices());
+    params.append(flatIndices.begin(), flatIndices.end());
+    params.push_back(getSingleValue(adaptor.getScalar()));
     SparseInsertGenerator insertGen(op.getDest().getType(), flatSpTensorTps,
                                     params, /*genCall=*/true);
     SmallVector<Value> ret = insertGen.genCallOrInline(rewriter, loc);
@@ -1062,7 +1057,7 @@ class SparseToPositionsConverter : public OpConversionPattern<ToPositionsOp> {
   using OpAdaptor = typename ToPositionsOp::Adaptor;
   using OpConversionPattern<ToPositionsOp>::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ToPositionsOp op, OpAdaptor adaptor,
+  matchAndRewrite(ToPositionsOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested position access with corresponding field.
     // The view is restricted to the actual size to ensure clients
@@ -1085,7 +1080,7 @@ class SparseToCoordinatesConverter
   using OpAdaptor = typename ToCoordinatesOp::Adaptor;
   using OpConversionPattern<ToCoordinatesOp>::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ToCoordinatesOp op, OpAdaptor adaptor,
+  matchAndRewrite(ToCoordinatesOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested coordinates access with corresponding field.
     // The view is restricted to the actual size to ensure clients
@@ -1111,7 +1106,7 @@ class SparseToCoordinatesBufferConverter
   using OpAdaptor = typename ToCoordinatesBufferOp::Adaptor;
   using OpConversionPattern<ToCoordinatesBufferOp>::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ToCoordinatesBufferOp op, OpAdaptor adaptor,
+  matchAndRewrite(ToCoordinatesBufferOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested coordinates access with corresponding field.
     // The view is restricted to the actual size to ensure clients
@@ -1133,7 +1128,7 @@ class SparseToValuesConverter : public OpConversionPattern<ToValuesOp> {
   using OpAdaptor = typename ToValuesOp::Adaptor;
   using OpConversionPattern<ToValuesOp>::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ToValuesOp op, OpAdaptor adaptor,
+  matchAndRewrite(ToValuesOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested values access with corresponding field.
     // The view is restricted to the actual size to ensure clients
@@ -1153,7 +1148,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(ConvertOp op, OpAdaptor adaptor,
+  matchAndRewrite(ConvertOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     SparseTensorEncodingAttr encDst = getSparseTensorEncoding(op.getType());
     SparseTensorEncodingAttr encSrc =
@@ -1173,7 +1168,7 @@ class SparseConvertConverter : public OpConversionPattern<ConvertOp> {
     Type srcElemTp = op.getSource().getType().getElementType();
     // Fold the trivial cases.
     if (retElemTp == srcElemTp && encDst == encSrc) {
-      rewriter.replaceOp(op, adaptor.getSource());
+      rewriter.replaceOpWithMultiple(op, {adaptor.getSource()});
       return success();
     }
     //
@@ -1239,7 +1234,7 @@ class SparseExtractSliceConverter
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(tensor::ExtractSliceOp op, OpAdaptor adaptor,
+  matchAndRewrite(tensor::ExtractSliceOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     MLIRContext *ctx = op.getContext();
@@ -1296,7 +1291,7 @@ class SparseNumberOfEntriesConverter
 public:
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
-  matchAndRewrite(NumberOfEntriesOp op, OpAdaptor adaptor,
+  matchAndRewrite(NumberOfEntriesOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Query memSizes for the actually stored values.
     // FIXME: the nse value computed in this way might be wrong when there is
@@ -1430,7 +1425,7 @@ struct SparseDisassembleOpConverter
       : OpConversionPattern(typeConverter, context) {}
 
   LogicalResult
-  matchAndRewrite(DisassembleOp op, OpAdaptor adaptor,
+  matchAndRewrite(DisassembleOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor(),
                                              op.getTensor().getType());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 89858546e37e1..869c7864d7535 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -228,11 +228,6 @@ class MutSparseTensorDescriptor
   }
 };
 
-/// Returns the "tuple" value of the adapted tensor.
-inline UnrealizedConversionCastOp getTuple(Value tensor) {
-  return llvm::cast<UnrealizedConversionCastOp>(tensor.getDefiningOp());
-}
-
 /// Packs the given values as a "tuple" value.
 inline Value genTuple(OpBuilder &builder, Location loc, Type tp,
                       ValueRange values) {
@@ -246,16 +241,15 @@ inline Value genTuple(OpBuilder &builder, Location loc,
 }
 
 inline SparseTensorDescriptor
-getDescriptorFromTensorTuple(Value tensor, RankedTensorType type) {
-  auto tuple = getTuple(tensor);
-  return SparseTensorDescriptor(SparseTensorType(type), tuple.getInputs());
+getDescriptorFromTensorTuple(ValueRange adaptorValues, RankedTensorType type) {
+  return SparseTensorDescriptor(SparseTensorType(type), adaptorValues);
 }
 
 inline MutSparseTensorDescriptor
-getMutDescriptorFromTensorTuple(Value tensor, SmallVectorImpl<Value> &fields,
+getMutDescriptorFromTensorTuple(ValueRange adaptorValues,
+                                SmallVectorImpl<Value> &fields,
                                 RankedTensorType type) {
-  auto tuple = getTuple(tensor);
-  fields.assign(tuple.getInputs().begin(), tuple.getInputs().end());
+  fields.assign(adaptorValues.begin(), adaptorValues.end());
   return MutSparseTensorDescriptor(SparseTensorType(type), fields);
 }
 
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 616d4a7d0a0ab..9bb628781342c 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -3983,9 +3983,11 @@ static LogicalResult commonVerifierPackAndUnPackOp(OpTy packOrUnPack) {
                               : packOrUnPack.getSourceType();
   size_t packedRank = packedType.getRank();
   // Require output rank to match input rank + number of blocking factors.
-  if (unpackedRank + mixedTiles.size() != packedRank) {
+  size_t expectedPackedRank = unpackedRank + mixedTiles.size();
+  if (expectedPackedRank != packedRank) {
     return op->emitError(
-        "packed rank must equal unpacked rank + tiling factors");
+               "packed rank != (unpacked rank + num tiling factors), got ")
+           << packedRank << " != " << expectedPackedRank;
   }
 
   // Verify result shape is greater than the minimum expected
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 0c0a7bc98d8b5..ad709813c6216 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -1756,11 +1756,6 @@ static Value foldExtractFromShapeCast(ExtractOp extractOp) {
   if (!shapeCastOp)
     return Value();
 
-  // 0-D vectors not supported.
-  assert(!hasZeroDimVectors(extractOp) && "0-D vectors not supported");
-  if (hasZeroDimVectors(shapeCastOp))
-    return Value();
-
   // Get the nth dimension size starting from lowest dimension.
   auto getDimReverse = [](VectorType type, int64_t n) {
     return type.getShape().take_back(n + 1).front();
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 87c30a733c363..181c394edc1d2 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -128,34 +128,17 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
                 return rewriter.create<vector::CreateMaskOp>(loc, newMaskType,
                                                              newMaskOperands);
               })
-          .Case<vector::ConstantMaskOp>([&](auto constantMaskOp)
-                                            -> std::optional<Operation *> {
-            ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes();
-            size_t numMaskOperands = maskDimSizes.size();
-            int64_t origIndex = maskDimSizes[numMaskOperands - 1];
-            int64_t startIndex = numFrontPadElems / numSrcElemsPerDest;
-            int64_t maskIndex = llvm::divideCeil(numFrontPadElems + origIndex,
-                                                 numSrcElemsPerDest);
-
-            // TODO: we only want the mask between [startIndex, maskIndex]
-            // to be true, the rest are false.
-            if (numFrontPadElems != 0 && maskDimSizes.size() > 1)
-              return std::nullopt;
-
-            SmallVector<int64_t> newMaskDimSizes(maskDimSizes.drop_back());
-            newMaskDimSizes.push_back(maskIndex);
-
-            if (numFrontPadElems == 0)
-              return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
-                                                             newMaskDimSizes);
-
-            SmallVector<bool> newMaskValues;
-            for (int64_t i = 0; i < numDestElems; ++i)
-              newMaskValues.push_back(i >= startIndex && i < maskIndex);
-            auto newMask = DenseElementsAttr::get(newMaskType, newMaskValues);
-            return rewriter.create<arith::ConstantOp>(loc, newMaskType,
-                                                      newMask);
-          })
+          .Case<vector::ConstantMaskOp>(
+              [&](auto constantMaskOp) -> std::optional<Operation *> {
+                // Take the shape of mask, compress its trailing dimension:
+                SmallVector<int64_t> maskDimSizes(
+                    constantMaskOp.getMaskDimSizes());
+                int64_t &maskIndex = maskDimSizes.back();
+                maskIndex = llvm::divideCeil(numFrontPadElems + maskIndex,
+                                             numSrcElemsPerDest);
+                return rewriter.create<vector::ConstantMaskOp>(loc, newMaskType,
+                                                               maskDimSizes);
+              })
           .Case<arith::ConstantOp>([&](auto constantOp)
                                        -> std::optional<Operation *> {
             // TODO: Support multiple dimensions.
@@ -604,7 +587,6 @@ struct ConvertVectorMaskedLoad final
   LogicalResult
   matchAndRewrite(vector::MaskedLoadOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-
     // See #115653
     if (op.getVectorType().getRank() != 1)
       return rewriter.notifyMatchFailure(op,
diff --git a/mlir/lib/Target/LLVM/CMakeLists.txt b/mlir/lib/Target/LLVM/CMakeLists.txt
index 422f7e5fa7cae..543e917b528b3 100644
--- a/mlir/lib/Target/LLVM/CMakeLists.txt
+++ b/mlir/lib/Target/LLVM/CMakeLists.txt
@@ -88,6 +88,20 @@ if ("NVPTX" IN_LIST LLVM_TARGETS_TO_BUILD)
       # Link against `nvptxcompiler_static`. TODO: use `CUDA::nvptxcompiler_static`.
       target_link_libraries(MLIRNVVMTarget PRIVATE MLIR_NVPTXCOMPILER_LIB)
       target_include_directories(obj.MLIRNVVMTarget PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
+
+      # Add the `nvfatbin` library.
+      find_library(MLIR_NVFATBIN_LIB_PATH nvfatbin_static
+                  PATHS ${CUDAToolkit_LIBRARY_DIR} NO_DEFAULT_PATH)
+      # Fail if `nvfatbin_static` couldn't be found.
+      if(MLIR_NVFATBIN_LIB_PATH STREQUAL "MLIR_NVFATBIN_LIB_PATH-NOTFOUND")
+        message(FATAL_ERROR
+                "Requested using the static `nvptxcompiler` library which requires the \
+                'nvfatbin` library, but it couldn't be found.")
+      endif()
+
+      add_library(MLIR_NVFATBIN_LIB STATIC IMPORTED GLOBAL)
+      set_property(TARGET MLIR_NVFATBIN_LIB PROPERTY IMPORTED_LOCATION ${MLIR_NVFATBIN_LIB_PATH})  
+      target_link_libraries(MLIRNVVMTarget PRIVATE MLIR_NVFATBIN_LIB)
     endif()
   else()
     # Fail if `MLIR_ENABLE_NVPTXCOMPILER` is enabled and the toolkit couldn't be found.
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index bca26e3a0e84a..3c92359915ded 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -473,6 +473,18 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) {
     }                                                                          \
   } while (false)
 
+#include "nvFatbin.h"
+
+#define RETURN_ON_NVFATBIN_ERROR(expr)                                         \
+  do {                                                                         \
+    auto result = (expr);                                                      \
+    if (result != nvFatbinResult::NVFATBIN_SUCCESS) {                          \
+      emitError(loc) << llvm::Twine(#expr).concat(" failed with error: ")      \
+                     << nvFatbinGetErrorString(result);                        \
+      return std::nullopt;                                                     \
+    }                                                                          \
+  } while (false)
+
 std::optional<SmallVector<char, 0>>
 NVPTXSerializer::compileToBinaryNVPTX(const std::string &ptxCode) {
   Location loc = getOperation().getLoc();
@@ -538,6 +550,32 @@ NVPTXSerializer::compileToBinaryNVPTX(const std::string &ptxCode) {
   });
 #undef DEBUG_TYPE
   RETURN_ON_NVPTXCOMPILER_ERROR(nvPTXCompilerDestroy(&compiler));
+
+  if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Fatbin) {
+    bool useFatbin32 = llvm::any_of(cmdOpts.second, [](const char *option) {
+      return llvm::StringRef(option) == "-32";
+    });
+
+    const char *cubinOpts[1] = {useFatbin32 ? "-32" : "-64"};
+    nvFatbinHandle handle;
+
+    auto chip = getTarget().getChip();
+    chip.consume_front("sm_");
+
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinCreate(&handle, cubinOpts, 1));
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinAddCubin(
+        handle, binary.data(), binary.size(), chip.data(), nullptr));
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinAddPTX(
+        handle, ptxCode.data(), ptxCode.size(), chip.data(), nullptr, nullptr));
+
+    size_t fatbinSize;
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinSize(handle, &fatbinSize));
+    SmallVector<char, 0> fatbin(fatbinSize, 0);
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinGet(handle, (void *)fatbin.data()));
+    RETURN_ON_NVFATBIN_ERROR(nvFatbinDestroy(&handle));
+    return fatbin;
+  }
+
   return binary;
 }
 #endif // MLIR_ENABLE_NVPTXCOMPILER
diff --git a/mlir/lib/Target/LLVMIR/AttrKindDetail.h b/mlir/lib/Target/LLVMIR/AttrKindDetail.h
index ddc6d46b90bb2..aa2e1d04e1b4a 100644
--- a/mlir/lib/Target/LLVMIR/AttrKindDetail.h
+++ b/mlir/lib/Target/LLVMIR/AttrKindDetail.h
@@ -45,6 +45,7 @@ getAttrKindToNameMapping() {
       {llvm::Attribute::AttrKind::NonNull, LLVMDialect::getNonNullAttrName()},
       {llvm::Attribute::AttrKind::Preallocated,
        LLVMDialect::getPreallocatedAttrName()},
+      {llvm::Attribute::AttrKind::Range, LLVMDialect::getRangeAttrName()},
       {llvm::Attribute::AttrKind::ReadOnly, LLVMDialect::getReadonlyAttrName()},
       {llvm::Attribute::AttrKind::ReadNone, LLVMDialect::getReadnoneAttrName()},
       {llvm::Attribute::AttrKind::Returned, LLVMDialect::getReturnedAttrName()},
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 35b0633a04a35..49fe509800491 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -268,7 +268,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkLinear(op, result);
         checkOrder(op, result);
-        checkPrivate(op, result);
       })
       .Case([&](omp::ParallelOp op) { checkAllocate(op, result); })
       .Case([&](omp::SimdOp op) {
@@ -1030,6 +1029,99 @@ mapInitializationArgs(T loop, LLVM::ModuleTranslation &moduleTranslation,
   }
 }
 
+template <typename OP>
+static LogicalResult
+initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
+                  llvm::IRBuilderBase &builder,
+                  LLVM::ModuleTranslation &moduleTranslation,
+                  llvm::BasicBlock *latestAllocaBlock,
+                  SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,
+                  SmallVectorImpl<llvm::Value *> &privateReductionVariables,
+                  DenseMap<Value, llvm::Value *> &reductionVariableMap,
+                  llvm::ArrayRef<bool> isByRef,
+                  SmallVectorImpl<DeferredStore> &deferredStores) {
+  if (op.getNumReductionVars() == 0)
+    return success();
+
+  llvm::IRBuilderBase::InsertPointGuard guard(builder);
+
+  builder.SetInsertPoint(latestAllocaBlock->getTerminator());
+  llvm::BasicBlock *initBlock = splitBB(builder, true, "omp.reduction.init");
+  auto allocaIP = llvm::IRBuilderBase::InsertPoint(
+      latestAllocaBlock, latestAllocaBlock->getTerminator()->getIterator());
+  builder.restoreIP(allocaIP);
+  SmallVector<llvm::Value *> byRefVars(op.getNumReductionVars());
+
+  for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
+    if (isByRef[i]) {
+      if (!reductionDecls[i].getAllocRegion().empty())
+        continue;
+
+      // TODO: remove after all users of by-ref are updated to use the alloc
+      // region: Allocate reduction variable (which is a pointer to the real
+      // reduciton variable allocated in the inlined region)
+      byRefVars[i] = builder.CreateAlloca(
+          moduleTranslation.convertType(reductionDecls[i].getType()));
+    }
+  }
+
+  builder.SetInsertPoint(&*initBlock->getFirstNonPHIOrDbgOrAlloca());
+
+  // store result of the alloc region to the allocated pointer to the real
+  // reduction variable
+  for (auto [data, addr] : deferredStores)
+    builder.CreateStore(data, addr);
+
+  // Before the loop, store the initial values of reductions into reduction
+  // variables. Although this could be done after allocas, we don't want to mess
+  // up with the alloca insertion point.
+  for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
+    SmallVector<llvm::Value *, 1> phis;
+
+    // map block argument to initializer region
+    mapInitializationArgs(op, moduleTranslation, reductionDecls,
+                          reductionVariableMap, i);
+
+    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
+                                       "omp.reduction.neutral", builder,
+                                       moduleTranslation, &phis)))
+      return failure();
+
+    assert(phis.size() == 1 && "expected one value to be yielded from the "
+                               "reduction neutral element declaration region");
+
+    builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
+
+    if (isByRef[i]) {
+      if (!reductionDecls[i].getAllocRegion().empty())
+        // done in allocReductionVars
+        continue;
+
+      // TODO: this path can be removed once all users of by-ref are updated to
+      // use an alloc region
+
+      // Store the result of the inlined region to the allocated reduction var
+      // ptr
+      builder.CreateStore(phis[0], byRefVars[i]);
+
+      privateReductionVariables[i] = byRefVars[i];
+      moduleTranslation.mapValue(reductionArgs[i], phis[0]);
+      reductionVariableMap.try_emplace(op.getReductionVars()[i], phis[0]);
+    } else {
+      // for by-ref case the store is inside of the reduction region
+      builder.CreateStore(phis[0], privateReductionVariables[i]);
+      // the rest was handled in allocByValReductionVars
+    }
+
+    // forget the mapping for the initializer region because we might need a
+    // different mapping if this reduction declaration is re-used for a
+    // different variable
+    moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion());
+  }
+
+  return success();
+}
+
 /// Collect reduction info
 template <typename T>
 static void collectReductionInfo(
@@ -1183,6 +1275,7 @@ static LogicalResult allocAndInitializeReductionVars(
   if (op.getNumReductionVars() == 0)
     return success();
 
+  llvm::IRBuilderBase::InsertPointGuard guard(builder);
   SmallVector<DeferredStore> deferredStores;
 
   if (failed(allocReductionVars(op, reductionArgs, builder, moduleTranslation,
@@ -1191,59 +1284,10 @@ static LogicalResult allocAndInitializeReductionVars(
                                 deferredStores, isByRef)))
     return failure();
 
-  // store result of the alloc region to the allocated pointer to the real
-  // reduction variable
-  for (auto [data, addr] : deferredStores)
-    builder.CreateStore(data, addr);
-
-  // Before the loop, store the initial values of reductions into reduction
-  // variables. Although this could be done after allocas, we don't want to mess
-  // up with the alloca insertion point.
-  for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
-    SmallVector<llvm::Value *, 1> phis;
-
-    // map block argument to initializer region
-    mapInitializationArgs(op, moduleTranslation, reductionDecls,
-                          reductionVariableMap, i);
-
-    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
-                                       "omp.reduction.neutral", builder,
-                                       moduleTranslation, &phis)))
-      return failure();
-    assert(phis.size() == 1 && "expected one value to be yielded from the "
-                               "reduction neutral element declaration region");
-    if (isByRef[i]) {
-      if (!reductionDecls[i].getAllocRegion().empty())
-        // done in allocReductionVars
-        continue;
-
-      // TODO: this path can be removed once all users of by-ref are updated to
-      // use an alloc region
-
-      // Allocate reduction variable (which is a pointer to the real reduction
-      // variable allocated in the inlined region)
-      llvm::Value *var = builder.CreateAlloca(
-          moduleTranslation.convertType(reductionDecls[i].getType()));
-      // Store the result of the inlined region to the allocated reduction var
-      // ptr
-      builder.CreateStore(phis[0], var);
-
-      privateReductionVariables[i] = var;
-      moduleTranslation.mapValue(reductionArgs[i], phis[0]);
-      reductionVariableMap.try_emplace(op.getReductionVars()[i], phis[0]);
-    } else {
-      // for by-ref case the store is inside of the reduction region
-      builder.CreateStore(phis[0], privateReductionVariables[i]);
-      // the rest was handled in allocByValReductionVars
-    }
-
-    // forget the mapping for the initializer region because we might need a
-    // different mapping if this reduction declaration is re-used for a
-    // different variable
-    moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion());
-  }
-
-  return success();
+  return initReductionVars(op, reductionArgs, builder, moduleTranslation,
+                           allocaIP.getBlock(), reductionDecls,
+                           privateReductionVariables, reductionVariableMap,
+                           isByRef, deferredStores);
 }
 
 /// Allocate delayed private variables. Returns the basic block which comes
@@ -1257,6 +1301,7 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
                     MutableArrayRef<mlir::Value> mlirPrivateVars,
                     llvm::SmallVectorImpl<llvm::Value *> &llvmPrivateVars,
                     const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP) {
+  llvm::IRBuilderBase::InsertPointGuard guard(builder);
   // Allocate private vars
   llvm::BranchInst *allocaTerminator =
       llvm::cast<llvm::BranchInst>(allocaIP.getBlock()->getTerminator());
@@ -1318,6 +1363,86 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
   return afterAllocas;
 }
 
+static LogicalResult
+initFirstPrivateVars(llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation,
+                     SmallVectorImpl<mlir::Value> &mlirPrivateVars,
+                     SmallVectorImpl<llvm::Value *> &llvmPrivateVars,
+                     SmallVectorImpl<omp::PrivateClauseOp> &privateDecls,
+                     llvm::BasicBlock *afterAllocas) {
+  llvm::IRBuilderBase::InsertPointGuard guard(builder);
+  // Apply copy region for firstprivate.
+  bool needsFirstprivate =
+      llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {
+        return privOp.getDataSharingType() ==
+               omp::DataSharingClauseType::FirstPrivate;
+      });
+
+  if (!needsFirstprivate)
+    return success();
+
+  assert(afterAllocas->getSinglePredecessor());
+
+  // Find the end of the allocation blocks
+  builder.SetInsertPoint(afterAllocas->getSinglePredecessor()->getTerminator());
+  llvm::BasicBlock *copyBlock =
+      splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");
+  builder.SetInsertPoint(copyBlock->getFirstNonPHIOrDbgOrAlloca());
+
+  for (auto [decl, mlirVar, llvmVar] :
+       llvm::zip_equal(privateDecls, mlirPrivateVars, llvmPrivateVars)) {
+    if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate)
+      continue;
+
+    // copyRegion implements `lhs = rhs`
+    Region &copyRegion = decl.getCopyRegion();
+
+    // map copyRegion rhs arg
+    llvm::Value *nonPrivateVar = moduleTranslation.lookupValue(mlirVar);
+    assert(nonPrivateVar);
+    moduleTranslation.mapValue(decl.getCopyMoldArg(), nonPrivateVar);
+
+    // map copyRegion lhs arg
+    moduleTranslation.mapValue(decl.getCopyPrivateArg(), llvmVar);
+
+    // in-place convert copy region
+    builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
+    if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy", builder,
+                                       moduleTranslation)))
+      return decl.emitError("failed to inline `copy` region of `omp.private`");
+
+    // ignore unused value yielded from copy region
+
+    // clear copy region block argument mapping in case it needs to be
+    // re-created with different sources for reuse of the same reduction
+    // decl
+    moduleTranslation.forgetMapping(copyRegion);
+  }
+
+  return success();
+}
+
+static LogicalResult
+cleanupPrivateVars(llvm::IRBuilderBase &builder,
+                   LLVM::ModuleTranslation &moduleTranslation, Location loc,
+                   SmallVectorImpl<llvm::Value *> &llvmPrivateVars,
+                   SmallVectorImpl<omp::PrivateClauseOp> &privateDecls) {
+  // private variable deallocation
+  SmallVector<Region *> privateCleanupRegions;
+  llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
+                  [](omp::PrivateClauseOp privatizer) {
+                    return &privatizer.getDeallocRegion();
+                  });
+
+  if (failed(inlineOmpRegionCleanup(
+          privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
+          "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
+    return mlir::emitError(loc, "failed to inline `dealloc` region of an "
+                                "`omp.private` op in");
+
+  return success();
+}
+
 static LogicalResult
 convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
                    LLVM::ModuleTranslation &moduleTranslation) {
@@ -1577,50 +1702,10 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
     if (handleError(afterAllocas, *taskOp).failed())
       return llvm::make_error<PreviouslyReportedError>();
 
-    // Apply copy region for firstprivate
-    bool needsFirstPrivate =
-        llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {
-          return privOp.getDataSharingType() ==
-                 omp::DataSharingClauseType::FirstPrivate;
-        });
-    if (needsFirstPrivate) {
-      // Find the end of the allocation blocks
-      assert(afterAllocas.get()->getSinglePredecessor());
-      builder.SetInsertPoint(
-          afterAllocas.get()->getSinglePredecessor()->getTerminator());
-      llvm::BasicBlock *copyBlock =
-          splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");
-      builder.SetInsertPoint(copyBlock->getFirstNonPHIOrDbgOrAlloca());
-    }
-    for (auto [decl, mlirVar, llvmVar] :
-         llvm::zip_equal(privateDecls, mlirPrivateVars, llvmPrivateVars)) {
-      if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate)
-        continue;
-
-      // copyRegion implements `lhs = rhs`
-      Region &copyRegion = decl.getCopyRegion();
-
-      // map copyRegion rhs arg
-      llvm::Value *nonPrivateVar = moduleTranslation.lookupValue(mlirVar);
-      assert(nonPrivateVar);
-      moduleTranslation.mapValue(decl.getCopyMoldArg(), nonPrivateVar);
-
-      // map copyRegion lhs arg
-      moduleTranslation.mapValue(decl.getCopyPrivateArg(), llvmVar);
-
-      // in-place convert copy region
-      builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
-      if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy",
-                                         builder, moduleTranslation)))
-        return llvm::createStringError(
-            "failed to inline `copy` region of an `omp.private` op in taskOp");
-
-      // ignore unused value yielded from copy region
-
-      // clear copy region block argument mapping in case it needs to be
-      // re-created with different source for reuse of the same reduction decl
-      moduleTranslation.forgetMapping(copyRegion);
-    }
+    if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                    llvmPrivateVars, privateDecls,
+                                    afterAllocas.get())))
+      return llvm::make_error<PreviouslyReportedError>();
 
     // translate the body of the task:
     builder.restoreIP(codegenIP);
@@ -1629,19 +1714,11 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
     if (failed(handleError(continuationBlockOrError, *taskOp)))
       return llvm::make_error<PreviouslyReportedError>();
 
-    // private variable deallocation
-    SmallVector<Region *> privateCleanupRegions;
-    llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
-                    [](omp::PrivateClauseOp privatizer) {
-                      return &privatizer.getDeallocRegion();
-                    });
-
     builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator());
-    if (failed(inlineOmpRegionCleanup(
-            privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
-            "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
-      return llvm::createStringError("failed to inline `dealloc` region of an "
-                                     "`omp.private` op in an omp.task");
+
+    if (failed(cleanupPrivateVars(builder, moduleTranslation, taskOp.getLoc(),
+                                  llvmPrivateVars, privateDecls)))
+      return llvm::make_error<PreviouslyReportedError>();
 
     return llvm::Error::success();
   };
@@ -1658,7 +1735,8 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
           ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
           moduleTranslation.lookupValue(taskOp.getFinal()),
           moduleTranslation.lookupValue(taskOp.getIfExpr()), dds,
-          taskOp.getMergeable());
+          taskOp.getMergeable(),
+          moduleTranslation.lookupValue(taskOp.getEventHandle()));
 
   if (failed(handleError(afterIP, *taskOp)))
     return failure();
@@ -1714,7 +1792,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     return failure();
 
   auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());
-
   llvm::ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
   assert(isByRef.size() == wsloopOp.getNumReductionVars());
 
@@ -1732,6 +1809,18 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);
   }
 
+  MutableArrayRef<BlockArgument> privateBlockArgs =
+      cast<omp::BlockArgOpenMPOpInterface>(*wsloopOp).getPrivateBlockArgs();
+  SmallVector<mlir::Value> mlirPrivateVars;
+  SmallVector<llvm::Value *> llvmPrivateVars;
+  SmallVector<omp::PrivateClauseOp> privateDecls;
+  mlirPrivateVars.reserve(privateBlockArgs.size());
+  llvmPrivateVars.reserve(privateBlockArgs.size());
+  collectPrivatizationDecls(wsloopOp, privateDecls);
+
+  for (mlir::Value privateVar : wsloopOp.getPrivateVars())
+    mlirPrivateVars.push_back(privateVar);
+
   SmallVector<omp::DeclareReductionOp> reductionDecls;
   collectReductionDecls(wsloopOp, reductionDecls);
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
@@ -1739,15 +1828,42 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
 
   SmallVector<llvm::Value *> privateReductionVariables(
       wsloopOp.getNumReductionVars());
+
+  splitBB(llvm::OpenMPIRBuilder::InsertPointTy(
+              allocaIP.getBlock(),
+              allocaIP.getBlock()->getTerminator()->getIterator()),
+          true, "omp.region.after_alloca");
+
+  llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
+      builder, moduleTranslation, privateBlockArgs, privateDecls,
+      mlirPrivateVars, llvmPrivateVars, allocaIP);
+  if (handleError(afterAllocas, opInst).failed())
+    return failure();
+
   DenseMap<Value, llvm::Value *> reductionVariableMap;
 
   MutableArrayRef<BlockArgument> reductionArgs =
       cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
 
-  if (failed(allocAndInitializeReductionVars(
-          wsloopOp, reductionArgs, builder, moduleTranslation, allocaIP,
-          reductionDecls, privateReductionVariables, reductionVariableMap,
-          isByRef)))
+  SmallVector<DeferredStore> deferredStores;
+
+  if (failed(allocReductionVars(wsloopOp, reductionArgs, builder,
+                                moduleTranslation, allocaIP, reductionDecls,
+                                privateReductionVariables, reductionVariableMap,
+                                deferredStores, isByRef)))
+    return failure();
+
+  if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                  llvmPrivateVars, privateDecls,
+                                  afterAllocas.get())))
+    return failure();
+
+  assert(afterAllocas.get()->getSinglePredecessor());
+  if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
+                               moduleTranslation,
+                               afterAllocas.get()->getSinglePredecessor(),
+                               reductionDecls, privateReductionVariables,
+                               reductionVariableMap, isByRef, deferredStores)))
     return failure();
 
   // TODO: Replace this with proper composite translation support.
@@ -1854,9 +1970,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   builder.restoreIP(afterIP);
 
   // Process the reductions if required.
-  return createReductionsAndCleanup(wsloopOp, builder, moduleTranslation,
-                                    allocaIP, reductionDecls,
-                                    privateReductionVariables, isByRef);
+  if (failed(createReductionsAndCleanup(wsloopOp, builder, moduleTranslation,
+                                        allocaIP, reductionDecls,
+                                        privateReductionVariables, isByRef)))
+    return failure();
+
+  return cleanupPrivateVars(builder, moduleTranslation, wsloopOp.getLoc(),
+                            llvmPrivateVars, privateDecls);
 }
 
 /// Converts the OpenMP parallel operation to LLVM IR.
@@ -1914,122 +2034,18 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
             deferredStores, isByRef)))
       return llvm::make_error<PreviouslyReportedError>();
 
-    // Apply copy region for firstprivate.
-    bool needsFirstprivate =
-        llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {
-          return privOp.getDataSharingType() ==
-                 omp::DataSharingClauseType::FirstPrivate;
-        });
-    if (needsFirstprivate) {
-      // Find the end of the allocation blocks
-      assert(afterAllocas.get()->getSinglePredecessor());
-      builder.SetInsertPoint(
-          afterAllocas.get()->getSinglePredecessor()->getTerminator());
-      llvm::BasicBlock *copyBlock =
-          splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");
-      builder.SetInsertPoint(copyBlock->getFirstNonPHIOrDbgOrAlloca());
-    }
-    for (auto [decl, mlirVar, llvmVar] :
-         llvm::zip_equal(privateDecls, mlirPrivateVars, llvmPrivateVars)) {
-      if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate)
-        continue;
-
-      // copyRegion implements `lhs = rhs`
-      Region &copyRegion = decl.getCopyRegion();
-
-      // map copyRegion rhs arg
-      llvm::Value *nonPrivateVar = moduleTranslation.lookupValue(mlirVar);
-      assert(nonPrivateVar);
-      moduleTranslation.mapValue(decl.getCopyMoldArg(), nonPrivateVar);
-
-      // map copyRegion lhs arg
-      moduleTranslation.mapValue(decl.getCopyPrivateArg(), llvmVar);
-
-      // in-place convert copy region
-      builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
-      if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy",
-                                         builder, moduleTranslation)))
-        return llvm::createStringError(
-            "failed to inline `copy` region of `omp.private`");
-
-      // ignore unused value yielded from copy region
-
-      // clear copy region block argument mapping in case it needs to be
-      // re-created with different sources for reuse of the same reduction
-      // decl
-      moduleTranslation.forgetMapping(copyRegion);
-    }
-
-    // Initialize reduction vars
-    builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
-    llvm::BasicBlock *initBlock = splitBB(builder, true, "omp.reduction.init");
-    allocaIP =
-        InsertPointTy(allocaIP.getBlock(),
-                      allocaIP.getBlock()->getTerminator()->getIterator());
-
-    builder.restoreIP(allocaIP);
-    SmallVector<llvm::Value *> byRefVars(opInst.getNumReductionVars());
-    for (unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
-      if (isByRef[i]) {
-        if (!reductionDecls[i].getAllocRegion().empty())
-          continue;
-
-        // TODO: remove after all users of by-ref are updated to use the alloc
-        // region: Allocate reduction variable (which is a pointer to the real
-        // reduciton variable allocated in the inlined region)
-        byRefVars[i] = builder.CreateAlloca(
-            moduleTranslation.convertType(reductionDecls[i].getType()));
-      }
-    }
-
-    builder.SetInsertPoint(initBlock->getFirstNonPHIOrDbgOrAlloca());
-
-    // insert stores deferred until after all allocas
-    // these store the results of the alloc region into the allocation for the
-    // pointer to the reduction variable
-    for (auto [data, addr] : deferredStores)
-      builder.CreateStore(data, addr);
-
-    for (unsigned i = 0; i < opInst.getNumReductionVars(); ++i) {
-      SmallVector<llvm::Value *> phis;
-
-      // map the block argument
-      mapInitializationArgs(opInst, moduleTranslation, reductionDecls,
-                            reductionVariableMap, i);
-      if (failed(inlineConvertOmpRegions(
-              reductionDecls[i].getInitializerRegion(), "omp.reduction.neutral",
-              builder, moduleTranslation, &phis)))
-        return llvm::createStringError(
-            "failed to inline `init` region of `omp.declare_reduction`");
-      assert(phis.size() == 1 &&
-             "expected one value to be yielded from the "
-             "reduction neutral element declaration region");
-
-      builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
-
-      if (isByRef[i]) {
-        if (!reductionDecls[i].getAllocRegion().empty())
-          continue;
-
-        // TODO: remove after all users of by-ref are updated to use the alloc
-
-        // Store the result of the inlined region to the allocated reduction var
-        // ptr
-        builder.CreateStore(phis[0], byRefVars[i]);
-
-        privateReductionVariables[i] = byRefVars[i];
-        moduleTranslation.mapValue(reductionArgs[i], phis[0]);
-        reductionVariableMap.try_emplace(opInst.getReductionVars()[i], phis[0]);
-      } else {
-        // for by-ref case the store is inside of the reduction init region
-        builder.CreateStore(phis[0], privateReductionVariables[i]);
-        // the rest is done in allocByValReductionVars
-      }
+    if (failed(initFirstPrivateVars(builder, moduleTranslation, mlirPrivateVars,
+                                    llvmPrivateVars, privateDecls,
+                                    afterAllocas.get())))
+      return llvm::make_error<PreviouslyReportedError>();
 
-      // clear block argument mapping in case it needs to be re-created with a
-      // different source for another use of the same reduction decl
-      moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion());
-    }
+    assert(afterAllocas.get()->getSinglePredecessor());
+    if (failed(
+            initReductionVars(opInst, reductionArgs, builder, moduleTranslation,
+                              afterAllocas.get()->getSinglePredecessor(),
+                              reductionDecls, privateReductionVariables,
+                              reductionVariableMap, isByRef, deferredStores)))
+      return llvm::make_error<PreviouslyReportedError>();
 
     // Store the mapping between reduction variables and their private copies on
     // ModuleTranslation stack. It can be then recovered when translating
@@ -2108,17 +2124,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       return llvm::createStringError(
           "failed to inline `cleanup` region of `omp.declare_reduction`");
 
-    SmallVector<Region *> privateCleanupRegions;
-    llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),
-                    [](omp::PrivateClauseOp privatizer) {
-                      return &privatizer.getDeallocRegion();
-                    });
-
-    if (failed(inlineOmpRegionCleanup(
-            privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
-            "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
-      return llvm::createStringError(
-          "failed to inline `dealloc` region of `omp.private`");
+    if (failed(cleanupPrivateVars(builder, moduleTranslation, opInst.getLoc(),
+                                  llvmPrivateVars, privateDecls)))
+      return llvm::make_error<PreviouslyReportedError>();
 
     builder.restoreIP(oldIP);
     return llvm::Error::success();
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 0d416a5857fac..b0d5e635248d3 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -2022,7 +2022,11 @@ ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs,
       mlirAttr = builder.getI64IntegerAttr(llvmAttr.getValueAsInt());
     else if (llvmAttr.isEnumAttribute())
       mlirAttr = builder.getUnitAttr();
-    else
+    else if (llvmAttr.isConstantRangeAttribute()) {
+      const llvm::ConstantRange &value = llvmAttr.getValueAsConstantRange();
+      mlirAttr = builder.getAttr<LLVM::ConstantRangeAttr>(value.getLower(),
+                                                          value.getUpper());
+    } else
       llvm_unreachable("unexpected parameter attribute kind");
     paramAttrs.push_back(builder.getNamedAttr(mlirName, mlirAttr));
   }
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 9e58d2a29199e..ad62ae0cef57b 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1581,7 +1581,12 @@ ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,
           .Case<IntegerAttr>([&](auto intAttr) {
             attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());
           })
-          .Case<UnitAttr>([&](auto) { attrBuilder.addAttribute(llvmKind); });
+          .Case<UnitAttr>([&](auto) { attrBuilder.addAttribute(llvmKind); })
+          .Case<LLVM::ConstantRangeAttr>([&](auto rangeAttr) {
+            attrBuilder.addConstantRangeAttr(
+                llvmKind, llvm::ConstantRange(rangeAttr.getLower(),
+                                              rangeAttr.getUpper()));
+          });
     } else if (namedAttr.getNameDialect()) {
       if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this)))
         return failure();
diff --git a/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp b/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
index d6a366940bc38..9bbf91de18305 100644
--- a/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
+++ b/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
@@ -41,6 +41,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Regex.h"
@@ -108,6 +109,23 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
         cl::desc("IRDL file to register before processing the input"),
         cl::location(irdlFileFlag), cl::init(""), cl::value_desc("filename"));
 
+    static cl::opt<VerbosityLevel, /*ExternalStorage=*/true>
+        diagnosticVerbosityLevel(
+            "mlir-diagnostic-verbosity-level",
+            cl::desc("Choose level of diagnostic information"),
+            cl::location(diagnosticVerbosityLevelFlag),
+            cl::init(VerbosityLevel::ErrorsWarningsAndRemarks),
+            cl::values(
+                clEnumValN(VerbosityLevel::ErrorsOnly, "errors", "Errors only"),
+                clEnumValN(VerbosityLevel::ErrorsAndWarnings, "warnings",
+                           "Errors and warnings"),
+                clEnumValN(VerbosityLevel::ErrorsWarningsAndRemarks, "remarks",
+                           "Errors, warnings and remarks")));
+
+    static cl::opt<bool, /*ExternalStorage=*/true> disableDiagnosticNotes(
+        "mlir-disable-diagnostic-notes", cl::desc("Disable diagnostic notes."),
+        cl::location(disableDiagnosticNotesFlag), cl::init(false));
+
     static cl::opt<bool, /*ExternalStorage=*/true> enableDebuggerHook(
         "mlir-enable-debugger-hook",
         cl::desc("Enable Debugger hook for debugging MLIR Actions"),
@@ -133,7 +151,8 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
         cl::location(showDialectsFlag), cl::init(false));
 
     static cl::opt<std::string, /*ExternalStorage=*/true> splitInputFile{
-        "split-input-file", llvm::cl::ValueOptional,
+        "split-input-file",
+        llvm::cl::ValueOptional,
         cl::callback([&](const std::string &str) {
           // Implicit value: use default marker if flag was used without value.
           if (str.empty())
@@ -141,7 +160,8 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
         }),
         cl::desc("Split the input file into chunks using the given or "
                  "default marker and process each chunk independently"),
-        cl::location(splitInputFileFlag), cl::init("")};
+        cl::location(splitInputFileFlag),
+        cl::init("")};
 
     static cl::opt<std::string, /*ExternalStorage=*/true> outputSplitMarker(
         "output-split-marker",
@@ -207,6 +227,43 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
   /// setDialectPluginsCallback(DialectRegistry&).
   cl::list<std::string> *dialectPlugins = nullptr;
 };
+
+/// A scoped diagnostic handler that suppresses certain diagnostics based on
+/// the verbosity level and whether the diagnostic is a note.
+class DiagnosticFilter : public ScopedDiagnosticHandler {
+public:
+  DiagnosticFilter(MLIRContext *ctx, VerbosityLevel verbosityLevel,
+                   bool showNotes = true)
+      : ScopedDiagnosticHandler(ctx) {
+    setHandler([verbosityLevel, showNotes](Diagnostic &diag) {
+      auto severity = diag.getSeverity();
+      switch (severity) {
+      case DiagnosticSeverity::Error:
+        // failure indicates that the error is not handled by the filter and
+        // goes through to the default handler. Therefore, the error can be
+        // successfully printed.
+        return failure();
+      case DiagnosticSeverity::Warning:
+        if (verbosityLevel == VerbosityLevel::ErrorsOnly)
+          return success();
+        else
+          return failure();
+      case DiagnosticSeverity::Remark:
+        if (verbosityLevel == VerbosityLevel::ErrorsOnly ||
+            verbosityLevel == VerbosityLevel::ErrorsAndWarnings)
+          return success();
+        else
+          return failure();
+      case DiagnosticSeverity::Note:
+        if (showNotes)
+          return failure();
+        else
+          return success();
+      }
+      llvm_unreachable("Unknown diagnostic severity");
+    });
+  }
+};
 } // namespace
 
 ManagedStatic<MlirOptMainConfigCLOptions> clOptionsConfig;
@@ -479,6 +536,9 @@ static LogicalResult processBuffer(raw_ostream &os,
   // otherwise just perform the actions without worrying about it.
   if (!config.shouldVerifyDiagnostics()) {
     SourceMgrDiagnosticHandler sourceMgrHandler(*sourceMgr, &context);
+    DiagnosticFilter diagnosticFilter(&context,
+                                      config.getDiagnosticVerbosityLevel(),
+                                      config.shouldShowNotes());
     return performActions(os, sourceMgr, &context, config);
   }
 
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 0aa9dcb36681b..3429008b2f241 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -172,7 +172,7 @@ static SmallVector<OpOperand *> operandsToOpOperands(OperandRange operands) {
 /// iff it has no memory effects and none of its results are live.
 ///
 /// It is assumed that `op` is simple. Here, a simple op is one which isn't a
-/// symbol op, a symbol-user op, a region branch op, a branch op, a region
+/// function-like op, a call-like op, a region branch op, a branch op, a region
 /// branch terminator op, or return-like.
 static void cleanSimpleOp(Operation *op, RunLivenessAnalysis &la) {
   if (!isMemoryEffectFree(op) || hasLive(op->getResults(), la))
@@ -191,10 +191,10 @@ static void cleanSimpleOp(Operation *op, RunLivenessAnalysis &la) {
 ///   non-live across all callers),
 ///   (5) Dropping the uses of these return values from its callers, AND
 ///   (6) Erasing these return values
-/// iff it is not public or declaration.
+/// iff it is not public or external.
 static void cleanFuncOp(FunctionOpInterface funcOp, Operation *module,
                         RunLivenessAnalysis &la) {
-  if (funcOp.isPublic() || funcOp.isDeclaration())
+  if (funcOp.isPublic() || funcOp.isExternal())
     return;
 
   // Get the list of unnecessary (non-live) arguments in `nonLiveArgs`.
@@ -563,6 +563,51 @@ static void cleanRegionBranchOp(RegionBranchOpInterface regionBranchOp,
   dropUsesAndEraseResults(regionBranchOp.getOperation(), resultsToKeep.flip());
 }
 
+// 1. Iterate over each successor block of the given BranchOpInterface
+//    operation.
+// 2. For each successor block:
+//    a. Retrieve the operands passed to the successor.
+//    b. Use the provided liveness analysis (`RunLivenessAnalysis`) to determine
+//       which operands are live in the successor block.
+//    c. Mark each operand as live or dead based on the analysis.
+// 3. Remove dead operands from the branch operation and arguments accordingly
+
+static void cleanBranchOp(BranchOpInterface branchOp, RunLivenessAnalysis &la) {
+  unsigned numSuccessors = branchOp->getNumSuccessors();
+
+  // Do (1)
+  for (unsigned succIdx = 0; succIdx < numSuccessors; ++succIdx) {
+    Block *successorBlock = branchOp->getSuccessor(succIdx);
+
+    // Do (2)
+    SuccessorOperands successorOperands =
+        branchOp.getSuccessorOperands(succIdx);
+    SmallVector<Value> operandValues;
+    for (unsigned operandIdx = 0; operandIdx < successorOperands.size();
+         ++operandIdx) {
+      operandValues.push_back(successorOperands[operandIdx]);
+    }
+
+    BitVector successorLiveOperands = markLives(operandValues, la);
+
+    // Do (3)
+    for (int argIdx = successorLiveOperands.size() - 1; argIdx >= 0; --argIdx) {
+      if (!successorLiveOperands[argIdx]) {
+        if (successorBlock->getNumArguments() < successorOperands.size()) {
+          // if block was cleaned through a different code path
+          // we only need to remove operands from the invokation
+          successorOperands.erase(argIdx);
+          continue;
+        }
+
+        successorBlock->getArgument(argIdx).dropAllUses();
+        successorOperands.erase(argIdx);
+        successorBlock->eraseArgument(argIdx);
+      }
+    }
+  }
+}
+
 struct RemoveDeadValues : public impl::RemoveDeadValuesBase<RemoveDeadValues> {
   void runOnOperation() override;
 };
@@ -572,26 +617,13 @@ void RemoveDeadValues::runOnOperation() {
   auto &la = getAnalysis<RunLivenessAnalysis>();
   Operation *module = getOperation();
 
-  // The removal of non-live values is performed iff there are no branch ops,
-  // and all symbol user ops present in the IR are call-like.
-  WalkResult acceptableIR = module->walk([&](Operation *op) {
-    if (op == module)
-      return WalkResult::advance();
-    if (isa<BranchOpInterface>(op)) {
-      op->emitError() << "cannot optimize an IR with branch ops\n";
-      return WalkResult::interrupt();
-    }
-    return WalkResult::advance();
-  });
-
-  if (acceptableIR.wasInterrupted())
-    return signalPassFailure();
-
   module->walk([&](Operation *op) {
     if (auto funcOp = dyn_cast<FunctionOpInterface>(op)) {
       cleanFuncOp(funcOp, module, la);
     } else if (auto regionBranchOp = dyn_cast<RegionBranchOpInterface>(op)) {
       cleanRegionBranchOp(regionBranchOp, la);
+    } else if (auto branchOp = dyn_cast<BranchOpInterface>(op)) {
+      cleanBranchOp(branchOp, la);
     } else if (op->hasTrait<::mlir::OpTrait::IsTerminator>()) {
       // Nothing to do here because this is a terminator op and it should be
       // honored with respect to its parent
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 710c976281dc3..cedf645e2985d 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -67,10 +67,6 @@ static OpBuilder::InsertPoint computeInsertPoint(Value value) {
 // ConversionValueMapping
 //===----------------------------------------------------------------------===//
 
-/// A list of replacement SSA values. Optimized for the common case of a single
-/// SSA value.
-using ReplacementValues = SmallVector<Value, 1>;
-
 namespace {
 /// This class wraps a IRMapping to provide recursive lookup
 /// functionality, i.e. we will traverse if the mapped value also has a mapping.
@@ -676,7 +672,8 @@ class UnresolvedMaterializationRewrite : public OperationRewrite {
   UnresolvedMaterializationRewrite(ConversionPatternRewriterImpl &rewriterImpl,
                                    UnrealizedConversionCastOp op,
                                    const TypeConverter *converter,
-                                   MaterializationKind kind, Type originalType);
+                                   MaterializationKind kind, Type originalType,
+                                   Value mappedValue);
 
   static bool classof(const IRRewrite *rewrite) {
     return rewrite->getKind() == Kind::UnresolvedMaterialization;
@@ -710,6 +707,10 @@ class UnresolvedMaterializationRewrite : public OperationRewrite {
   /// The original type of the SSA value. Only used for target
   /// materializations.
   Type originalType;
+
+  /// The value in the conversion value mapping that is being replaced by the
+  /// results of this unresolved materialization.
+  Value mappedValue;
 };
 } // namespace
 
@@ -778,7 +779,7 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   LogicalResult remapValues(StringRef valueDiagTag,
                             std::optional<Location> inputLoc,
                             PatternRewriter &rewriter, ValueRange values,
-                            SmallVectorImpl<Value> &remapped);
+                            SmallVector<SmallVector<Value>> &remapped);
 
   /// Return "true" if the given operation is ignored, and does not need to be
   /// converted.
@@ -812,13 +813,31 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   // Materializations
   //===--------------------------------------------------------------------===//
 
-  /// Build an unresolved materialization operation given an output type and set
-  /// of input operands.
-  Value buildUnresolvedMaterialization(MaterializationKind kind,
-                                       OpBuilder::InsertPoint ip, Location loc,
-                                       ValueRange inputs, Type outputType,
-                                       Type originalType,
-                                       const TypeConverter *converter);
+  /// Build an unresolved materialization operation given a range of output
+  /// types and a list of input operands. Returns the inputs if they their
+  /// types match the output types.
+  ///
+  /// If a cast op was built, it can optionally be returned with the `castOp`
+  /// output argument.
+  ///
+  /// If `valueToMap` is set to a non-null Value, then that value is mapped to
+  /// the results of the unresolved materialization in the conversion value
+  /// mapping.
+  ValueRange buildUnresolvedMaterialization(
+      MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
+      Value valueToMap, ValueRange inputs, TypeRange outputTypes,
+      Type originalType, const TypeConverter *converter,
+      UnrealizedConversionCastOp *castOp = nullptr);
+  Value buildUnresolvedMaterialization(
+      MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
+      Value valueToMap, ValueRange inputs, Type outputType, Type originalType,
+      const TypeConverter *converter,
+      UnrealizedConversionCastOp *castOp = nullptr) {
+    return buildUnresolvedMaterialization(kind, ip, loc, valueToMap, inputs,
+                                          TypeRange(outputType), originalType,
+                                          converter, castOp)
+        .front();
+  }
 
   /// Build an N:1 materialization for the given original value that was
   /// replaced with the given replacement values.
@@ -844,6 +863,16 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   Value findOrBuildReplacementValue(Value value,
                                     const TypeConverter *converter);
 
+  /// Unpack an N:1 materialization and return the inputs of the
+  /// materialization. This function unpacks only those materializations that
+  /// were built with `insertNTo1Materialization`.
+  ///
+  /// This is a workaround around incomplete 1:N support in the dialect
+  /// conversion driver. It allows us to write 1:N conversion patterns while
+  /// 1:N support is still missing in the conversion value mapping. This
+  /// function will be deleted when full 1:N support has been added.
+  SmallVector<Value> unpackNTo1Materialization(Value value);
+
   //===--------------------------------------------------------------------===//
   // Rewriter Notification Hooks
   //===--------------------------------------------------------------------===//
@@ -853,7 +882,7 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
                                OpBuilder::InsertPoint previous) override;
 
   /// Notifies that an op is about to be replaced with the given values.
-  void notifyOpReplaced(Operation *op, ArrayRef<ReplacementValues> newValues);
+  void notifyOpReplaced(Operation *op, ArrayRef<ValueRange> newValues);
 
   /// Notifies that a block is about to be erased.
   void notifyBlockIsBeingErased(Block *block);
@@ -946,6 +975,10 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   DenseMap<UnrealizedConversionCastOp, UnresolvedMaterializationRewrite *>
       unresolvedMaterializations;
 
+  /// A set of all N:1 materializations that were added to work around
+  /// incomplete 1:N support in the dialect conversion driver.
+  DenseSet<UnrealizedConversionCastOp> nTo1TempMaterializations;
+
   /// The current type converter, or nullptr if no type converter is currently
   /// active.
   const TypeConverter *currentTypeConverter = nullptr;
@@ -1068,20 +1101,21 @@ void CreateOperationRewrite::rollback() {
 
 UnresolvedMaterializationRewrite::UnresolvedMaterializationRewrite(
     ConversionPatternRewriterImpl &rewriterImpl, UnrealizedConversionCastOp op,
-    const TypeConverter *converter, MaterializationKind kind, Type originalType)
+    const TypeConverter *converter, MaterializationKind kind, Type originalType,
+    Value mappedValue)
     : OperationRewrite(Kind::UnresolvedMaterialization, rewriterImpl, op),
-      converterAndKind(converter, kind), originalType(originalType) {
+      converterAndKind(converter, kind), originalType(originalType),
+      mappedValue(mappedValue) {
   assert((!originalType || kind == MaterializationKind::Target) &&
          "original type is valid only for target materializations");
   rewriterImpl.unresolvedMaterializations[op] = this;
 }
 
 void UnresolvedMaterializationRewrite::rollback() {
-  if (getMaterializationKind() == MaterializationKind::Target) {
-    for (Value input : op->getOperands())
-      rewriterImpl.mapping.erase(input);
-  }
+  if (mappedValue)
+    rewriterImpl.mapping.erase(mappedValue);
   rewriterImpl.unresolvedMaterializations.erase(getOperation());
+  rewriterImpl.nTo1TempMaterializations.erase(getOperation());
   op->erase();
 }
 
@@ -1127,7 +1161,7 @@ void ConversionPatternRewriterImpl::undoRewrites(unsigned numRewritesToKeep) {
 LogicalResult ConversionPatternRewriterImpl::remapValues(
     StringRef valueDiagTag, std::optional<Location> inputLoc,
     PatternRewriter &rewriter, ValueRange values,
-    SmallVectorImpl<Value> &remapped) {
+    SmallVector<SmallVector<Value>> &remapped) {
   remapped.reserve(llvm::size(values));
 
   for (const auto &it : llvm::enumerate(values)) {
@@ -1135,11 +1169,18 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
     Type origType = operand.getType();
     Location operandLoc = inputLoc ? *inputLoc : operand.getLoc();
 
+    // Find the most recently mapped value. Unpack all temporary N:1
+    // materializations. Such conversions are a workaround around missing
+    // 1:N support in the ConversionValueMapping. (The conversion patterns
+    // already support 1:N replacements.)
+    Value repl = mapping.lookupOrDefault(operand);
+    SmallVector<Value> unpacked = unpackNTo1Materialization(repl);
+
     if (!currentTypeConverter) {
       // The current pattern does not have a type converter. I.e., it does not
       // distinguish between legal and illegal types. For each operand, simply
       // pass through the most recently mapped value.
-      remapped.push_back(mapping.lookupOrDefault(operand));
+      remapped.push_back(std::move(unpacked));
       continue;
     }
 
@@ -1153,15 +1194,30 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
       return failure();
     }
 
+    // If a type is converted to 0 types, there is nothing to do.
+    if (legalTypes.empty()) {
+      remapped.push_back({});
+      continue;
+    }
+
     if (legalTypes.size() != 1) {
-      // TODO: Parts of the dialect conversion infrastructure do not support
-      // 1->N type conversions yet. Therefore, if a type is converted to 0 or
-      // multiple types, the only thing that we can do for now is passing
-      // through the most recently mapped value. Fixing this requires
-      // improvements to the `ConversionValueMapping` (to be able to store 1:N
-      // mappings) and to the `ConversionPattern` adaptor handling (to be able
-      // to pass multiple remapped values for a single operand to the adaptor).
-      remapped.push_back(mapping.lookupOrDefault(operand));
+      // TODO: This is a 1:N conversion. The conversion value mapping does not
+      // store such materializations yet. If the types of the most recently
+      // mapped values do not match, build a target materialization.
+      ValueRange unpackedRange(unpacked);
+      if (TypeRange(unpackedRange) == legalTypes) {
+        remapped.push_back(std::move(unpacked));
+        continue;
+      }
+
+      // Insert a target materialization if the current pattern expects
+      // different legalized types.
+      ValueRange targetMat = buildUnresolvedMaterialization(
+          MaterializationKind::Target, computeInsertPoint(repl), operandLoc,
+          /*valueToMap=*/Value(), /*inputs=*/unpacked,
+          /*outputType=*/legalTypes, /*originalType=*/origType,
+          currentTypeConverter);
+      remapped.push_back(targetMat);
       continue;
     }
 
@@ -1173,16 +1229,15 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
     if (newOperand.getType() != desiredType) {
       // If the looked up value's type does not have the desired type, it means
       // that the value was replaced with a value of different type and no
-      // source materialization was created yet.
+      // target materialization was created yet.
       Value castValue = buildUnresolvedMaterialization(
           MaterializationKind::Target, computeInsertPoint(newOperand),
-          operandLoc,
-          /*inputs=*/newOperand, /*outputType=*/desiredType,
-          /*originalType=*/origType, currentTypeConverter);
-      mapping.map(newOperand, castValue);
+          operandLoc, /*valueToMap=*/newOperand, /*inputs=*/unpacked,
+          /*outputType=*/desiredType, /*originalType=*/origType,
+          currentTypeConverter);
       newOperand = castValue;
     }
-    remapped.push_back(newOperand);
+    remapped.push_back({newOperand});
   }
   return success();
 }
@@ -1293,12 +1348,11 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
     if (!inputMap) {
       // This block argument was dropped and no replacement value was provided.
       // Materialize a replacement value "out of thin air".
-      Value repl = buildUnresolvedMaterialization(
+      buildUnresolvedMaterialization(
           MaterializationKind::Source,
           OpBuilder::InsertPoint(newBlock, newBlock->begin()), origArg.getLoc(),
-          /*inputs=*/ValueRange(),
+          /*valueToMap=*/origArg, /*inputs=*/ValueRange(),
           /*outputType=*/origArgType, /*originalType=*/Type(), converter);
-      mapping.map(origArg, repl);
       appendRewrite<ReplaceBlockArgRewrite>(block, origArg, converter);
       continue;
     }
@@ -1340,26 +1394,38 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
 
 /// Build an unresolved materialization operation given an output type and set
 /// of input operands.
-Value ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
+ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
     MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
-    ValueRange inputs, Type outputType, Type originalType,
-    const TypeConverter *converter) {
+    Value valueToMap, ValueRange inputs, TypeRange outputTypes,
+    Type originalType, const TypeConverter *converter,
+    UnrealizedConversionCastOp *castOp) {
   assert((!originalType || kind == MaterializationKind::Target) &&
          "original type is valid only for target materializations");
 
   // Avoid materializing an unnecessary cast.
-  if (inputs.size() == 1 && inputs.front().getType() == outputType)
-    return inputs.front();
+  if (TypeRange(inputs) == outputTypes) {
+    if (valueToMap) {
+      assert(inputs.size() == 1 && "1:N mapping is not supported");
+      mapping.map(valueToMap, inputs.front());
+    }
+    return inputs;
+  }
 
   // Create an unresolved materialization. We use a new OpBuilder to avoid
   // tracking the materialization like we do for other operations.
-  OpBuilder builder(outputType.getContext());
+  OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
-      builder.create<UnrealizedConversionCastOp>(loc, outputType, inputs);
+      builder.create<UnrealizedConversionCastOp>(loc, outputTypes, inputs);
+  if (valueToMap) {
+    assert(outputTypes.size() == 1 && "1:N mapping is not supported");
+    mapping.map(valueToMap, convertOp.getResult(0));
+  }
+  if (castOp)
+    *castOp = convertOp;
   appendRewrite<UnresolvedMaterializationRewrite>(convertOp, converter, kind,
-                                                  originalType);
-  return convertOp.getResult(0);
+                                                  originalType, valueToMap);
+  return convertOp.getResults();
 }
 
 void ConversionPatternRewriterImpl::insertNTo1Materialization(
@@ -1367,11 +1433,13 @@ void ConversionPatternRewriterImpl::insertNTo1Materialization(
     Value originalValue, const TypeConverter *converter) {
   // Insert argument materialization back to the original type.
   Type originalType = originalValue.getType();
-  Value argMat =
-      buildUnresolvedMaterialization(MaterializationKind::Argument, ip, loc,
-                                     /*inputs=*/replacements, originalType,
-                                     /*originalType=*/Type(), converter);
-  mapping.map(originalValue, argMat);
+  UnrealizedConversionCastOp argCastOp;
+  Value argMat = buildUnresolvedMaterialization(
+      MaterializationKind::Argument, ip, loc, /*valueToMap=*/originalValue,
+      /*inputs=*/replacements, originalType, /*originalType=*/Type(), converter,
+      &argCastOp);
+  if (argCastOp)
+    nTo1TempMaterializations.insert(argCastOp);
 
   // Insert target materialization to the legalized type.
   Type legalOutputType;
@@ -1387,11 +1455,14 @@ void ConversionPatternRewriterImpl::insertNTo1Materialization(
     legalOutputType = replacements[0].getType();
   }
   if (legalOutputType && legalOutputType != originalType) {
-    Value targetMat = buildUnresolvedMaterialization(
+    UnrealizedConversionCastOp targetCastOp;
+    buildUnresolvedMaterialization(
         MaterializationKind::Target, computeInsertPoint(argMat), loc,
-        /*inputs=*/argMat, /*outputType=*/legalOutputType,
-        /*originalType=*/originalType, converter);
-    mapping.map(argMat, targetMat);
+        /*valueToMap=*/argMat, /*inputs=*/argMat,
+        /*outputType=*/legalOutputType, /*originalType=*/originalType,
+        converter, &targetCastOp);
+    if (targetCastOp)
+      nTo1TempMaterializations.insert(targetCastOp);
   }
 }
 
@@ -1425,12 +1496,34 @@ Value ConversionPatternRewriterImpl::findOrBuildReplacementValue(
   }
   Value castValue = buildUnresolvedMaterialization(
       MaterializationKind::Source, computeInsertPoint(repl), value.getLoc(),
-      /*inputs=*/repl, /*outputType=*/value.getType(),
+      /*valueToMap=*/value, /*inputs=*/repl, /*outputType=*/value.getType(),
       /*originalType=*/Type(), converter);
   mapping.map(value, castValue);
   return castValue;
 }
 
+SmallVector<Value>
+ConversionPatternRewriterImpl::unpackNTo1Materialization(Value value) {
+  // Unpack unrealized_conversion_cast ops that were inserted as a N:1
+  // workaround.
+  auto castOp = value.getDefiningOp<UnrealizedConversionCastOp>();
+  if (!castOp)
+    return {value};
+  if (!nTo1TempMaterializations.contains(castOp))
+    return {value};
+  assert(castOp->getNumResults() == 1 && "expected single result");
+
+  SmallVector<Value> result;
+  for (Value v : castOp.getOperands()) {
+    // Keep unpacking if possible. This is needed because during block
+    // signature conversions and 1:N op replacements, the driver may have
+    // inserted two materializations back-to-back: first an argument
+    // materialization, then a target materialization.
+    llvm::append_range(result, unpackNTo1Materialization(v));
+  }
+  return result;
+}
+
 //===----------------------------------------------------------------------===//
 // Rewriter Notification Hooks
 
@@ -1455,7 +1548,7 @@ void ConversionPatternRewriterImpl::notifyOperationInserted(
 }
 
 void ConversionPatternRewriterImpl::notifyOpReplaced(
-    Operation *op, ArrayRef<ReplacementValues> newValues) {
+    Operation *op, ArrayRef<ValueRange> newValues) {
   assert(newValues.size() == op->getNumResults());
   assert(!ignoredOps.contains(op) && "operation was already replaced");
 
@@ -1467,8 +1560,7 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(
       isUnresolvedMaterialization = true;
 
   // Create mappings for each of the new result values.
-  for (auto [n, result] : llvm::zip_equal(newValues, op->getResults())) {
-    ReplacementValues repl = n;
+  for (auto [repl, result] : llvm::zip_equal(newValues, op->getResults())) {
     if (repl.empty()) {
       // This result was dropped and no replacement value was provided.
       if (isUnresolvedMaterialization) {
@@ -1478,12 +1570,12 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(
       }
 
       // Materialize a replacement value "out of thin air".
-      Value sourceMat = buildUnresolvedMaterialization(
+      buildUnresolvedMaterialization(
           MaterializationKind::Source, computeInsertPoint(result),
-          result.getLoc(), /*inputs=*/ValueRange(),
+          result.getLoc(), /*valueToMap=*/result, /*inputs=*/ValueRange(),
           /*outputType=*/result.getType(), /*originalType=*/Type(),
           currentTypeConverter);
-      repl.push_back(sourceMat);
+      continue;
     } else {
       // Make sure that the user does not mess with unresolved materializations
       // that were inserted by the conversion driver. We keep track of these
@@ -1585,10 +1677,9 @@ void ConversionPatternRewriter::replaceOp(Operation *op, ValueRange newValues) {
     impl->logger.startLine()
         << "** Replace : '" << op->getName() << "'(" << op << ")\n";
   });
-  SmallVector<ReplacementValues> newVals(newValues.size());
-  for (auto [index, val] : llvm::enumerate(newValues))
-    if (val)
-      newVals[index].push_back(val);
+  SmallVector<ValueRange> newVals;
+  for (size_t i = 0; i < newValues.size(); ++i)
+    newVals.push_back(newValues.slice(i, 1));
   impl->notifyOpReplaced(op, newVals);
 }
 
@@ -1600,10 +1691,7 @@ void ConversionPatternRewriter::replaceOpWithMultiple(
     impl->logger.startLine()
         << "** Replace : '" << op->getName() << "'(" << op << ")\n";
   });
-  SmallVector<ReplacementValues> newVals(newValues.size(), {});
-  for (auto [index, val] : llvm::enumerate(newValues))
-    llvm::append_range(newVals[index], val);
-  impl->notifyOpReplaced(op, newVals);
+  impl->notifyOpReplaced(op, newValues);
 }
 
 void ConversionPatternRewriter::eraseOp(Operation *op) {
@@ -1611,7 +1699,7 @@ void ConversionPatternRewriter::eraseOp(Operation *op) {
     impl->logger.startLine()
         << "** Erase   : '" << op->getName() << "'(" << op << ")\n";
   });
-  SmallVector<ReplacementValues> nullRepls(op->getNumResults(), {});
+  SmallVector<ValueRange> nullRepls(op->getNumResults(), {});
   impl->notifyOpReplaced(op, nullRepls);
 }
 
@@ -1663,11 +1751,12 @@ void ConversionPatternRewriter::replaceUsesOfBlockArgument(BlockArgument from,
 }
 
 Value ConversionPatternRewriter::getRemappedValue(Value key) {
-  SmallVector<Value> remappedValues;
+  SmallVector<SmallVector<Value>> remappedValues;
   if (failed(impl->remapValues("value", /*inputLoc=*/std::nullopt, *this, key,
                                remappedValues)))
     return nullptr;
-  return remappedValues.front();
+  assert(remappedValues.front().size() == 1 && "1:N conversion not supported");
+  return remappedValues.front().front();
 }
 
 LogicalResult
@@ -1675,8 +1764,15 @@ ConversionPatternRewriter::getRemappedValues(ValueRange keys,
                                              SmallVectorImpl<Value> &results) {
   if (keys.empty())
     return success();
-  return impl->remapValues("value", /*inputLoc=*/std::nullopt, *this, keys,
-                           results);
+  SmallVector<SmallVector<Value>> remapped;
+  if (failed(impl->remapValues("value", /*inputLoc=*/std::nullopt, *this, keys,
+                               remapped)))
+    return failure();
+  for (const auto &values : remapped) {
+    assert(values.size() == 1 && "1:N conversion not supported");
+    results.push_back(values.front());
+  }
+  return success();
 }
 
 void ConversionPatternRewriter::inlineBlockBefore(Block *source, Block *dest,
@@ -1770,6 +1866,19 @@ detail::ConversionPatternRewriterImpl &ConversionPatternRewriter::getImpl() {
 // ConversionPattern
 //===----------------------------------------------------------------------===//
 
+SmallVector<Value> ConversionPattern::getOneToOneAdaptorOperands(
+    ArrayRef<ValueRange> operands) const {
+  SmallVector<Value> oneToOneOperands;
+  oneToOneOperands.reserve(operands.size());
+  for (ValueRange operand : operands) {
+    if (operand.size() != 1)
+      llvm::report_fatal_error("pattern '" + getDebugName() +
+                               "' does not support 1:N conversion");
+    oneToOneOperands.push_back(operand.front());
+  }
+  return oneToOneOperands;
+}
+
 LogicalResult
 ConversionPattern::matchAndRewrite(Operation *op,
                                    PatternRewriter &rewriter) const {
@@ -1781,12 +1890,14 @@ ConversionPattern::matchAndRewrite(Operation *op,
                                              getTypeConverter());
 
   // Remap the operands of the operation.
-  SmallVector<Value, 4> operands;
+  SmallVector<SmallVector<Value>> remapped;
   if (failed(rewriterImpl.remapValues("operand", op->getLoc(), rewriter,
-                                      op->getOperands(), operands))) {
+                                      op->getOperands(), remapped))) {
     return failure();
   }
-  return matchAndRewrite(op, operands, dialectRewriter);
+  SmallVector<ValueRange> remappedAsRange =
+      llvm::to_vector_of<ValueRange>(remapped);
+  return matchAndRewrite(op, remappedAsRange, dialectRewriter);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2526,45 +2637,55 @@ legalizeUnresolvedMaterialization(RewriterBase &rewriter,
   assert(!op.use_empty() &&
          "expected that dead materializations have already been DCE'd");
   Operation::operand_range inputOperands = op.getOperands();
-  Type outputType = op.getResultTypes()[0];
 
   // Try to materialize the conversion.
   if (const TypeConverter *converter = rewrite->getConverter()) {
     rewriter.setInsertionPoint(op);
-    Value newMaterialization;
+    SmallVector<Value> newMaterialization;
     switch (rewrite->getMaterializationKind()) {
-    case MaterializationKind::Argument:
+    case MaterializationKind::Argument: {
       // Try to materialize an argument conversion.
-      newMaterialization = converter->materializeArgumentConversion(
-          rewriter, op->getLoc(), outputType, inputOperands);
-      if (newMaterialization)
+      assert(op->getNumResults() == 1 && "expected single result");
+      Value argMat = converter->materializeArgumentConversion(
+          rewriter, op->getLoc(), op.getResultTypes().front(), inputOperands);
+      if (argMat) {
+        newMaterialization.push_back(argMat);
         break;
+      }
+    }
       // If an argument materialization failed, fallback to trying a target
       // materialization.
       [[fallthrough]];
     case MaterializationKind::Target:
       newMaterialization = converter->materializeTargetConversion(
-          rewriter, op->getLoc(), outputType, inputOperands,
+          rewriter, op->getLoc(), op.getResultTypes(), inputOperands,
           rewrite->getOriginalType());
       break;
     case MaterializationKind::Source:
-      newMaterialization = converter->materializeSourceConversion(
-          rewriter, op->getLoc(), outputType, inputOperands);
+      assert(op->getNumResults() == 1 && "expected single result");
+      Value sourceMat = converter->materializeSourceConversion(
+          rewriter, op->getLoc(), op.getResultTypes().front(), inputOperands);
+      if (sourceMat)
+        newMaterialization.push_back(sourceMat);
       break;
     }
-    if (newMaterialization) {
-      assert(newMaterialization.getType() == outputType &&
+    if (!newMaterialization.empty()) {
+#ifndef NDEBUG
+      ValueRange newMaterializationRange(newMaterialization);
+      assert(TypeRange(newMaterializationRange) == op.getResultTypes() &&
              "materialization callback produced value of incorrect type");
+#endif // NDEBUG
       rewriter.replaceOp(op, newMaterialization);
       return success();
     }
   }
 
-  InFlightDiagnostic diag =
-      op->emitError() << "failed to legalize unresolved materialization "
-                         "from ("
-                      << inputOperands.getTypes() << ") to (" << outputType
-                      << ") that remained live after conversion";
+  InFlightDiagnostic diag = op->emitError()
+                            << "failed to legalize unresolved materialization "
+                               "from ("
+                            << inputOperands.getTypes() << ") to ("
+                            << op.getResultTypes()
+                            << ") that remained live after conversion";
   diag.attachNote(op->getUsers().begin()->getLoc())
       << "see existing live user here: " << *op->getUsers().begin();
   return failure();
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index 23187f256455b..e1b870b53ad25 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -683,7 +683,9 @@ if(MLIR_INCLUDE_TESTS)
     MLIRPythonTestSources.Dialects.PythonTest
     ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
     ADD_TO_PARENT MLIRPythonTestSources.Dialects
-    SOURCES dialects/python_test.py)
+    SOURCES
+      dialects/python_test.py
+  )
   set(LLVM_TARGET_DEFINITIONS
     "${MLIR_MAIN_SRC_DIR}/test/python/python_test_ops.td")
   mlir_tablegen(
@@ -697,12 +699,25 @@ if(MLIR_INCLUDE_TESTS)
     ADD_TO_PARENT MLIRPythonTestSources.Dialects.PythonTest
     SOURCES "dialects/_python_test_ops_gen.py")
 
-  declare_mlir_python_extension(MLIRPythonTestSources.PythonTestExtension
-    MODULE_NAME _mlirPythonTest
+  declare_mlir_python_extension(MLIRPythonTestSources.PythonTestExtensionPybind11
+    MODULE_NAME _mlirPythonTestPybind11
+    ADD_TO_PARENT MLIRPythonTestSources.Dialects
+    ROOT_DIR "${MLIR_SOURCE_DIR}/test/python/lib"
+    PYTHON_BINDINGS_LIBRARY pybind11
+    SOURCES
+      PythonTestModulePybind11.cpp
+    PRIVATE_LINK_LIBS
+      LLVMSupport
+    EMBED_CAPI_LINK_LIBS
+      MLIRCAPIPythonTestDialect
+  )
+  declare_mlir_python_extension(MLIRPythonTestSources.PythonTestExtensionNanobind
+    MODULE_NAME _mlirPythonTestNanobind
     ADD_TO_PARENT MLIRPythonTestSources.Dialects
     ROOT_DIR "${MLIR_SOURCE_DIR}/test/python/lib"
+    PYTHON_BINDINGS_LIBRARY nanobind
     SOURCES
-      PythonTestModule.cpp
+      PythonTestModuleNanobind.cpp
     PRIVATE_LINK_LIBS
       LLVMSupport
     EMBED_CAPI_LINK_LIBS
diff --git a/mlir/python/mlir/_mlir_libs/_mlir/passmanager.pyi b/mlir/python/mlir/_mlir_libs/_mlir/passmanager.pyi
index 229979ae33608..0d2eaffe16d3e 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/passmanager.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/passmanager.pyi
@@ -22,6 +22,10 @@ class PassManager:
         print_module_scope: bool = False,
         print_after_change: bool = False,
         print_after_failure: bool = False,
+        large_elements_limit: int | None = None,
+        enable_debug_info: bool = False,
+        print_generic_op_form: bool = False,
+        tree_printing_dir_path: str | None = None,
     ) -> None: ...
     def enable_verifier(self, enable: bool) -> None: ...
     @staticmethod
diff --git a/mlir/python/mlir/dialects/func.py b/mlir/python/mlir/dialects/func.py
index 24fdcbcd85b29..1898fc1565cd4 100644
--- a/mlir/python/mlir/dialects/func.py
+++ b/mlir/python/mlir/dialects/func.py
@@ -105,6 +105,8 @@ def add_entry_block(self, arg_locs: Optional[Sequence[Location]] = None):
 
     @property
     def arg_attrs(self):
+        if ARGUMENT_ATTRIBUTE_NAME not in self.attributes:
+            return ArrayAttr.get([DictAttr.get({}) for _ in self.type.inputs])
         return ArrayAttr(self.attributes[ARGUMENT_ATTRIBUTE_NAME])
 
     @arg_attrs.setter
diff --git a/mlir/python/mlir/dialects/python_test.py b/mlir/python/mlir/dialects/python_test.py
index b5baa80bc767f..9380896c8c06e 100644
--- a/mlir/python/mlir/dialects/python_test.py
+++ b/mlir/python/mlir/dialects/python_test.py
@@ -3,15 +3,14 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._python_test_ops_gen import *
-from .._mlir_libs._mlirPythonTest import (
-    TestAttr,
-    TestType,
-    TestTensorValue,
-    TestIntegerRankedTensorType,
-)
 
 
-def register_python_test_dialect(registry):
-    from .._mlir_libs import _mlirPythonTest
+def register_python_test_dialect(registry, use_nanobind):
+    if use_nanobind:
+        from .._mlir_libs import _mlirPythonTestNanobind
 
-    _mlirPythonTest.register_dialect(registry)
+        _mlirPythonTestNanobind.register_dialect(registry)
+    else:
+        from .._mlir_libs import _mlirPythonTestPybind11
+
+        _mlirPythonTestPybind11.register_dialect(registry)
diff --git a/mlir/python/requirements.txt b/mlir/python/requirements.txt
index 272d066831f92..ab8a9122919e1 100644
--- a/mlir/python/requirements.txt
+++ b/mlir/python/requirements.txt
@@ -1,3 +1,4 @@
+nanobind>=2.0, <3.0
 numpy>=1.19.5, <=2.1.2
 pybind11>=2.10.0, <=2.13.6
 PyYAML>=5.4.0, <=6.0.1
diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c
index fa3b0d894c995..15a3a1fb50dc9 100644
--- a/mlir/test/CAPI/ir.c
+++ b/mlir/test/CAPI/ir.c
@@ -320,7 +320,7 @@ int collectStats(MlirOperation operation) {
   // clang-format off
   // CHECK-LABEL: @stats
   // CHECK: Number of operations: 12
-  // CHECK: Number of attributes: 6
+  // CHECK: Number of attributes: 5
   // CHECK: Number of blocks: 3
   // CHECK: Number of regions: 3
   // CHECK: Number of values: 9
diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir
index afd1198ede0f7..1728c3a2557e0 100644
--- a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir
+++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir
@@ -587,6 +587,10 @@ func.func @arith_float_to_int_cast_ops(%arg0: f32, %arg1: f64) {
   // CHECK: emitc.cast %[[CAST0]] : ui32 to i32
   %4 = arith.fptoui %arg0 : f32 to i32
 
+  // CHECK: %[[CAST0:.*]] = emitc.cast %arg0 : f32 to ui16
+  // CHECK: emitc.cast %[[CAST0]] : ui16 to i16
+  %5 = arith.fptoui %arg0 : f32 to i16
+
   return
 }
 
diff --git a/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
index 24a0bab352c34..9d7ab2be096ef 100644
--- a/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
+++ b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
@@ -60,6 +60,14 @@ func.func @int_vector4_invalid(%arg0: vector<2xi16>) {
   return
 }
 
+// -----
+
+func.func @int_vector_invalid_bitwidth(%arg0: vector<2xi12>) {
+  // expected-error @+1 {{failed to legalize operation 'arith.addi'}}
+  %0 = arith.addi %arg0, %arg0: vector<2xi12>
+  return
+}
+
 ///===----------------------------------------------------------------------===//
 // Constant ops
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
index c143d030ed362..16b692b968939 100644
--- a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
+++ b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
@@ -227,84 +227,9 @@ gpu.module @barriers {
 
 // -----
 
-// Check `gpu.shuffle` conversion with default subgroup size.
-
-gpu.module @shuffles {
-  // CHECK:           llvm.func spir_funccc @_Z22sub_group_shuffle_downdj(f64, i32) -> f64 attributes {
-  // CHECK-SAME-DAG:  no_unwind
-  // CHECK-SAME-DAG:  convergent
-  // CHECK-SAME-DAG:  will_return
-  // CHECK-NOT:       memory_effects = #llvm.memory_effects
-  // CHECK-SAME:      }
-  // CHECK:           llvm.func spir_funccc @_Z20sub_group_shuffle_upfj(f32, i32) -> f32 attributes {
-  // CHECK-SAME-DAG:  no_unwind
-  // CHECK-SAME-DAG:  convergent
-  // CHECK-SAME-DAG:  will_return
-  // CHECK-NOT:       memory_effects = #llvm.memory_effects
-  // CHECK-SAME:      }
-  // CHECK:           llvm.func spir_funccc @_Z21sub_group_shuffle_xorlj(i64, i32) -> i64 attributes {
-  // CHECK-SAME-DAG:  no_unwind
-  // CHECK-SAME-DAG:  convergent
-  // CHECK-SAME-DAG:  will_return
-  // CHECK-NOT:       memory_effects = #llvm.memory_effects
-  // CHECK-SAME:      }
-  // CHECK:           llvm.func spir_funccc @_Z17sub_group_shuffleij(i32, i32) -> i32 attributes {
-  // CHECK-SAME-DAG:  no_unwind
-  // CHECK-SAME-DAG:  convergent
-  // CHECK-SAME-DAG:  will_return
-  // CHECK-NOT:       memory_effects = #llvm.memory_effects
-  // CHECK-SAME:      }
-
-  // CHECK-LABEL: gpu_shuffles
-  // CHECK-SAME:              (%[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i32, %[[VAL_4:.*]]: f32, %[[VAL_5:.*]]: i32, %[[VAL_6:.*]]: f64, %[[VAL_7:.*]]: i32)
-  func.func @gpu_shuffles(%val0: i32, %id: i32,
-                          %val1: i64, %mask: i32,
-                          %val2: f32, %delta_up: i32,
-                          %val3: f64, %delta_down: i32) {
-    %width = arith.constant 32 : i32
-    // CHECK:         llvm.call spir_funccc @_Z17sub_group_shuffleij(%[[VAL_0]], %[[VAL_1]]) {
-    // CHECK-SAME-DAG:  no_unwind
-    // CHECK-SAME-DAG:  convergent
-    // CHECK-SAME-DAG:  will_return
-    // CHECK-NOT:       memory_effects = #llvm.memory_effects
-    // CHECK-SAME:    } : (i32, i32) -> i32
-    // CHECK:         llvm.mlir.constant(true) : i1
-    // CHECK:         llvm.call spir_funccc @_Z21sub_group_shuffle_xorlj(%[[VAL_2]], %[[VAL_3]]) {
-    // CHECK-SAME-DAG:  no_unwind
-    // CHECK-SAME-DAG:  convergent
-    // CHECK-SAME-DAG:  will_return
-    // CHECK-NOT:       memory_effects = #llvm.memory_effects
-    // CHECK-SAME:    } : (i64, i32) -> i64
-    // CHECK:         llvm.mlir.constant(true) : i1
-    // CHECK:         llvm.call spir_funccc @_Z20sub_group_shuffle_upfj(%[[VAL_4]], %[[VAL_5]]) {
-    // CHECK-SAME-DAG:  no_unwind
-    // CHECK-SAME-DAG:  convergent
-    // CHECK-SAME-DAG:  will_return
-    // CHECK-NOT:       memory_effects= #llvm.memory_effects
-    // CHECK-SAME:    } : (f32, i32) -> f32
-    // CHECK:         llvm.mlir.constant(true) : i1
-    // CHECK:         llvm.call spir_funccc @_Z22sub_group_shuffle_downdj(%[[VAL_6]], %[[VAL_7]]) {
-    // CHECK-SAME-DAG:  no_unwind
-    // CHECK-SAME-DAG:  convergent
-    // CHECK-SAME-DAG:  will_return
-    // CHECK-NOT:       memory_effects= #llvm.memory_effects
-    // CHECK-SAME:    } : (f64, i32) -> f64
-    // CHECK:         llvm.mlir.constant(true) : i1
-    %shuffleResult0, %valid0 = gpu.shuffle idx %val0, %id, %width : i32
-    %shuffleResult1, %valid1 = gpu.shuffle xor %val1, %mask, %width : i64
-    %shuffleResult2, %valid2 = gpu.shuffle up %val2, %delta_up, %width : f32
-    %shuffleResult3, %valid3 = gpu.shuffle down %val3, %delta_down, %width : f64
-    return
-  }
-}
-
-// -----
-
 // Check `gpu.shuffle` conversion with explicit subgroup size.
 
-gpu.module @shuffles attributes {
-  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Kernel, Addresses, GroupNonUniformShuffle, Int64], []>, #spirv.resource_limits<subgroup_size = 16>>
-} {
+gpu.module @shuffles {
   // CHECK:           llvm.func spir_funccc @_Z22sub_group_shuffle_downdj(f64, i32) -> f64 attributes {
   // CHECK-SAME-DAG:  no_unwind
   // CHECK-SAME-DAG:  convergent
@@ -352,15 +277,15 @@ gpu.module @shuffles attributes {
   // CHECK-SAME:              (%[[I8_VAL:.*]]: i8, %[[I16_VAL:.*]]: i16,
   // CHECK-SAME:               %[[I32_VAL:.*]]: i32, %[[I64_VAL:.*]]: i64,
   // CHECK-SAME:               %[[F16_VAL:.*]]: f16, %[[F32_VAL:.*]]: f32,
-  // CHECK-SAME:               %[[F64_VAL:.*]]: f64,  %[[OFFSET:.*]]: i32) {
-  func.func @gpu_shuffles(%i8_val: i8,
+  // CHECK-SAME:               %[[F64_VAL:.*]]: f64,  %[[OFFSET:.*]]: i32)
+  llvm.func @gpu_shuffles(%i8_val: i8,
                           %i16_val: i16,
                           %i32_val: i32,
                           %i64_val: i64,
                           %f16_val: f16,
                           %f32_val: f32,
                           %f64_val: f64,
-                          %offset: i32) {
+                          %offset: i32) attributes {intel_reqd_sub_group_size = 16 : i32} {
     %width = arith.constant 16 : i32
     // CHECK:         llvm.call spir_funccc @_Z17sub_group_shufflecj(%[[I8_VAL]], %[[OFFSET]])
     // CHECK:         llvm.mlir.constant(true) : i1
@@ -383,7 +308,7 @@ gpu.module @shuffles attributes {
     %shuffleResult4, %valid4 = gpu.shuffle up %f16_val, %offset, %width : f16
     %shuffleResult5, %valid5 = gpu.shuffle up %f32_val, %offset, %width : f32
     %shuffleResult6, %valid6 = gpu.shuffle down %f64_val, %offset, %width : f64
-    return
+    llvm.return
   }
 }
 
@@ -392,11 +317,11 @@ gpu.module @shuffles attributes {
 // Cannot convert due to shuffle width and target subgroup size mismatch
 
 gpu.module @shuffles_mismatch {
-  func.func @gpu_shuffles(%val: i32, %id: i32) {
+  llvm.func @gpu_shuffles(%val: i32, %id: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {
     %width = arith.constant 16 : i32
     // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}
     %shuffleResult, %valid = gpu.shuffle idx %val, %id, %width : i32
-    return
+    llvm.return
   }
 }
 
@@ -405,10 +330,10 @@ gpu.module @shuffles_mismatch {
 // Cannot convert due to variable shuffle width
 
 gpu.module @shuffles_mismatch {
-  func.func @gpu_shuffles(%val: i32, %id: i32, %width: i32) {
+  llvm.func @gpu_shuffles(%val: i32, %id: i32, %width: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {
     // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}
     %shuffleResult, %valid = gpu.shuffle idx %val, %id, %width : i32
-    return
+    llvm.return
   }
 }
 
@@ -417,11 +342,11 @@ gpu.module @shuffles_mismatch {
 // Cannot convert due to value type not being supported by the conversion
 
 gpu.module @not_supported_lowering {
-  func.func @gpu_shuffles(%val: i1, %id: i32) {
+  llvm.func @gpu_shuffles(%val: i1, %id: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {
     %width = arith.constant 32 : i32
     // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}
     %shuffleResult, %valid = gpu.shuffle xor %val, %id, %width : i1
-    return
+    llvm.return
   }
 }
 
diff --git a/mlir/test/Conversion/MeshToMPI/convert-mesh-to-mpi.mlir b/mlir/test/Conversion/MeshToMPI/convert-mesh-to-mpi.mlir
new file mode 100644
index 0000000000000..c1aef97438bd5
--- /dev/null
+++ b/mlir/test/Conversion/MeshToMPI/convert-mesh-to-mpi.mlir
@@ -0,0 +1,208 @@
+// RUN: mlir-opt %s -convert-mesh-to-mpi -canonicalize -split-input-file | FileCheck %s
+
+// -----
+// CHECK: mesh.mesh @mesh0
+mesh.mesh @mesh0(shape = 3x4x5)
+func.func @process_multi_index() -> (index, index, index) {
+  // CHECK: mpi.comm_rank : !mpi.retval, i32
+  // CHECK-DAG: %[[v4:.*]] = arith.remsi
+  // CHECK-DAG: %[[v0:.*]] = arith.remsi
+  // CHECK-DAG: %[[v1:.*]] = arith.remsi
+  %0:3 = mesh.process_multi_index on @mesh0 axes = [] : index, index, index
+  // CHECK: return %[[v1]], %[[v0]], %[[v4]] : index, index, index
+  return %0#0, %0#1, %0#2 : index, index, index
+}
+
+// CHECK-LABEL: func @process_linear_index
+func.func @process_linear_index() -> index {
+  // CHECK: %[[RES:.*]], %[[rank:.*]] = mpi.comm_rank : !mpi.retval, i32
+  // CHECK: %[[cast:.*]] = arith.index_cast %[[rank]] : i32 to index
+  %0 = mesh.process_linear_index on @mesh0 : index
+  // CHECK: return %[[cast]] : index
+  return %0 : index
+}
+
+// CHECK-LABEL: func @neighbors_dim0
+func.func @neighbors_dim0(%arg0 : tensor<120x120x120xi8>) -> (index, index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  // CHECK-DAG: [[up:%.*]] = arith.constant 44 : index
+  // CHECK-DAG: [[down:%.*]] = arith.constant 4 : index
+  %idx:2 = mesh.neighbors_linear_indices on @mesh0[%c1, %c0, %c4] split_axes = [0] : index, index
+  // CHECK: return [[down]], [[up]] : index, index
+  return %idx#0, %idx#1 : index, index
+}
+
+// CHECK-LABEL: func @neighbors_dim1
+func.func @neighbors_dim1(%arg0 : tensor<120x120x120xi8>) -> (index, index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  // CHECK-DAG: [[up:%.*]] = arith.constant 29 : index
+  // CHECK-DAG: [[down:%.*]] = arith.constant -1 : index
+  %idx:2 = mesh.neighbors_linear_indices on @mesh0[%c1, %c0, %c4] split_axes = [1] : index, index
+  // CHECK: return [[down]], [[up]] : index, index
+  return %idx#0, %idx#1 : index, index
+}
+
+// CHECK-LABEL: func @neighbors_dim2
+func.func @neighbors_dim2(%arg0 : tensor<120x120x120xi8>) -> (index, index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  // CHECK-DAG: [[up:%.*]] = arith.constant -1 : index
+  // CHECK-DAG: [[down:%.*]] = arith.constant 23 : index
+  %idx:2 = mesh.neighbors_linear_indices on @mesh0[%c1, %c0, %c4] split_axes = [2] : index, index
+  // CHECK: return [[down]], [[up]] : index, index
+  return %idx#0, %idx#1 : index, index
+}
+
+// -----
+// CHECK: mesh.mesh @mesh0
+mesh.mesh @mesh0(shape = 3x4x5)
+memref.global constant @static_mpi_rank : memref<index> = dense<24>
+func.func @process_multi_index() -> (index, index, index) {
+  // CHECK-DAG: %[[c4:.*]] = arith.constant 4 : index
+  // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
+  // CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index
+  %0:3 = mesh.process_multi_index on @mesh0 axes = [] : index, index, index
+  // CHECK: return %[[c1]], %[[c0]], %[[c4]] : index, index, index
+  return %0#0, %0#1, %0#2 : index, index, index
+}
+
+// CHECK-LABEL: func @process_linear_index
+func.func @process_linear_index() -> index {
+  // CHECK: %[[c24:.*]] = arith.constant 24 : index
+  %0 = mesh.process_linear_index on @mesh0 : index
+  // CHECK: return %[[c24]] : index
+  return %0 : index
+}
+
+// -----
+mesh.mesh @mesh0(shape = 3x4x5)
+// CHECK-LABEL: func @update_halo_1d_first
+func.func @update_halo_1d_first(
+  // CHECK-SAME: [[arg0:%.*]]: memref<120x120x120xi8>
+  %arg0 : memref<120x120x120xi8>) -> memref<120x120x120xi8> {
+  // CHECK: memref.subview [[arg0]][115, 0, 0] [2, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<2x120x120xi8
+  // CHECK: mpi.send(
+  // CHECK-SAME: : memref<2x120x120xi8>, i32, i32
+  // CHECK: mpi.recv(
+  // CHECK-SAME: : memref<2x120x120xi8>, i32, i32
+  // CHECK-NEXT: memref.subview [[arg0]][0, 0, 0] [2, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<2x120x120xi8
+  // CHECK: memref.subview [[arg0]][2, 0, 0] [3, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<3x120x120xi8
+  // CHECK: mpi.send(
+  // CHECK-SAME: : memref<3x120x120xi8>, i32, i32
+  // CHECK: mpi.recv(
+  // CHECK-SAME: : memref<3x120x120xi8>, i32, i32
+  // CHECK-NEXT: memref.subview [[arg0]][117, 0, 0] [3, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<3x120x120xi8
+  %res = mesh.update_halo %arg0 on @mesh0 split_axes = [[0]] halo_sizes = [2, 3] : memref<120x120x120xi8>
+  // CHECK: return [[res:%.*]] : memref<120x120x120xi8>
+  return %res : memref<120x120x120xi8>
+}
+
+// -----
+mesh.mesh @mesh0(shape = 3x4x5)
+memref.global constant @static_mpi_rank : memref<index> = dense<24>
+// CHECK-LABEL: func @update_halo_3d
+func.func @update_halo_3d(
+  // CHECK-SAME: [[varg0:%.*]]: memref<120x120x120xi8>
+  %arg0 : memref<120x120x120xi8>) -> memref<120x120x120xi8> {
+  // CHECK: [[vc23_i32:%.*]] = arith.constant 23 : i32
+  // CHECK-NEXT: [[vc29_i32:%.*]] = arith.constant 29 : i32
+  // CHECK-NEXT: [[vc91_i32:%.*]] = arith.constant 91 : i32
+  // CHECK-NEXT: [[vc4_i32:%.*]] = arith.constant 4 : i32
+  // CHECK-NEXT: [[vc44_i32:%.*]] = arith.constant 44 : i32
+  // CHECK-NEXT: [[valloc:%.*]] = memref.alloc() : memref<117x113x5xi8>
+  // CHECK-NEXT: [[vsubview:%.*]] = memref.subview [[varg0]][1, 3, 109] [117, 113, 5] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14869>>
+  // CHECK-NEXT: memref.copy [[vsubview]], [[valloc]] : memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14869>> to memref<117x113x5xi8>
+  // CHECK-NEXT: mpi.send([[valloc]], [[vc91_i32]], [[vc4_i32]]) : memref<117x113x5xi8>, i32, i32
+  // CHECK-NEXT: mpi.recv([[valloc]], [[vc91_i32]], [[vc44_i32]]) : memref<117x113x5xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_0:%.*]] = memref.subview [[varg0]][1, 3, 0] [117, 113, 5] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.copy [[valloc]], [[vsubview_0]] : memref<117x113x5xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.dealloc [[valloc]] : memref<117x113x5xi8>
+  // CHECK-NEXT: [[valloc_1:%.*]] = memref.alloc() : memref<117x113x6xi8>
+  // CHECK-NEXT: [[vsubview_3:%.*]] = memref.subview [[varg0]][1, 3, 5] [117, 113, 6] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14765>>
+  // CHECK-NEXT: memref.copy [[vsubview_3]], [[valloc_1]] : memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14765>> to memref<117x113x6xi8>
+  // CHECK-NEXT: mpi.send([[valloc_1]], [[vc91_i32]], [[vc44_i32]]) : memref<117x113x6xi8>, i32, i32
+  // CHECK-NEXT: mpi.recv([[valloc_1]], [[vc91_i32]], [[vc4_i32]]) : memref<117x113x6xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_4:%.*]] = memref.subview [[varg0]][1, 3, 114] [117, 113, 6] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14874>>
+  // CHECK-NEXT: memref.copy [[valloc_1]], [[vsubview_4]] : memref<117x113x6xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14874>>
+  // CHECK-NEXT: memref.dealloc [[valloc_1]] : memref<117x113x6xi8>
+  // CHECK-NEXT: [[valloc_5:%.*]] = memref.alloc() : memref<117x3x120xi8>
+  // CHECK-NEXT: mpi.recv([[valloc_5]], [[vc91_i32]], [[vc29_i32]]) : memref<117x3x120xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_7:%.*]] = memref.subview [[varg0]][1, 0, 0] [117, 3, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<117x3x120xi8, strided<[14400, 120, 1], offset: 14400>>
+  // CHECK-NEXT: memref.copy [[valloc_5]], [[vsubview_7]] : memref<117x3x120xi8> to memref<117x3x120xi8, strided<[14400, 120, 1], offset: 14400>>
+  // CHECK-NEXT: memref.dealloc [[valloc_5]] : memref<117x3x120xi8>
+  // CHECK-NEXT: [[valloc_8:%.*]] = memref.alloc() : memref<117x4x120xi8>
+  // CHECK-NEXT: [[vsubview_10:%.*]] = memref.subview [[varg0]][1, 3, 0] [117, 4, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<117x4x120xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.copy [[vsubview_10]], [[valloc_8]] : memref<117x4x120xi8, strided<[14400, 120, 1], offset: 14760>> to memref<117x4x120xi8>
+  // CHECK-NEXT: mpi.send([[valloc_8]], [[vc91_i32]], [[vc29_i32]]) : memref<117x4x120xi8>, i32, i32
+  // CHECK-NEXT: memref.dealloc [[valloc_8]] : memref<117x4x120xi8>
+  // CHECK-NEXT: [[valloc_11:%.*]] = memref.alloc() : memref<1x120x120xi8>
+  // CHECK-NEXT: [[vsubview_12:%.*]] = memref.subview [[varg0]][117, 0, 0] [1, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<1x120x120xi8, strided<[14400, 120, 1], offset: 1684800>>
+  // CHECK-NEXT: memref.copy [[vsubview_12]], [[valloc_11]] : memref<1x120x120xi8, strided<[14400, 120, 1], offset: 1684800>> to memref<1x120x120xi8>
+  // CHECK-NEXT: mpi.send([[valloc_11]], [[vc91_i32]], [[vc23_i32]]) : memref<1x120x120xi8>, i32, i32
+  // CHECK-NEXT: memref.dealloc [[valloc_11]] : memref<1x120x120xi8>
+  // CHECK-NEXT: [[valloc_13:%.*]] = memref.alloc() : memref<2x120x120xi8>
+  // CHECK-NEXT: mpi.recv([[valloc_13]], [[vc91_i32]], [[vc23_i32]]) : memref<2x120x120xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_14:%.*]] = memref.subview [[varg0]][118, 0, 0] [2, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<2x120x120xi8, strided<[14400, 120, 1], offset: 1699200>>
+  // CHECK-NEXT: memref.copy [[valloc_13]], [[vsubview_14]] : memref<2x120x120xi8> to memref<2x120x120xi8, strided<[14400, 120, 1], offset: 1699200>>
+  // CHECK-NEXT: memref.dealloc [[valloc_13]] : memref<2x120x120xi8>
+  %res = mesh.update_halo %arg0 on @mesh0 split_axes = [[2], [1], [0]] halo_sizes = [1, 2, 3, 4, 5, 6] : memref<120x120x120xi8>
+  // CHECK: return [[varg0]] : memref<120x120x120xi8>
+  return %res : memref<120x120x120xi8>
+}
+
+// CHECK-LABEL: func @update_halo_3d_tensor
+func.func @update_halo_3d_tensor(
+  // CHECK-SAME: [[varg0:%.*]]: tensor<120x120x120xi8>
+  %arg0 : tensor<120x120x120xi8>) -> tensor<120x120x120xi8> {
+  // CHECK: [[vc23_i32:%.*]] = arith.constant 23 : i32
+  // CHECK-NEXT: [[vc29_i32:%.*]] = arith.constant 29 : i32
+  // CHECK-NEXT: [[vc44_i32:%.*]] = arith.constant 44 : i32
+  // CHECK-NEXT: [[vc4_i32:%.*]] = arith.constant 4 : i32
+  // CHECK-NEXT: [[vc91_i32:%.*]] = arith.constant 91 : i32
+  // CHECK-NEXT: [[v0:%.*]] = bufferization.to_memref [[varg0]] : tensor<120x120x120xi8> to memref<120x120x120xi8>
+  // CHECK-NEXT: [[valloc:%.*]] = memref.alloc() : memref<117x113x5xi8>
+  // CHECK-NEXT: [[vsubview:%.*]] = memref.subview [[v0]][1, 3, 109] [117, 113, 5] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14869>>
+  // CHECK-NEXT: memref.copy [[vsubview]], [[valloc]] : memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14869>> to memref<117x113x5xi8>
+  // CHECK-NEXT: mpi.send([[valloc]], [[vc91_i32]], [[vc4_i32]]) : memref<117x113x5xi8>, i32, i32
+  // CHECK-NEXT: mpi.recv([[valloc]], [[vc91_i32]], [[vc44_i32]]) : memref<117x113x5xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_0:%.*]] = memref.subview [[v0]][1, 3, 0] [117, 113, 5] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.copy [[valloc]], [[vsubview_0]] : memref<117x113x5xi8> to memref<117x113x5xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.dealloc [[valloc]] : memref<117x113x5xi8>
+  // CHECK-NEXT: [[valloc_1:%.*]] = memref.alloc() : memref<117x113x6xi8>
+  // CHECK-NEXT: [[vsubview_3:%.*]] = memref.subview [[v0]][1, 3, 5] [117, 113, 6] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14765>>
+  // CHECK-NEXT: memref.copy [[vsubview_3]], [[valloc_1]] : memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14765>> to memref<117x113x6xi8>
+  // CHECK-NEXT: mpi.send([[valloc_1]], [[vc91_i32]], [[vc44_i32]]) : memref<117x113x6xi8>, i32, i32
+  // CHECK-NEXT: mpi.recv([[valloc_1]], [[vc91_i32]], [[vc4_i32]]) : memref<117x113x6xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_4:%.*]] = memref.subview [[v0]][1, 3, 114] [117, 113, 6] [1, 1, 1] : memref<120x120x120xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14874>>
+  // CHECK-NEXT: memref.copy [[valloc_1]], [[vsubview_4]] : memref<117x113x6xi8> to memref<117x113x6xi8, strided<[14400, 120, 1], offset: 14874>>
+  // CHECK-NEXT: memref.dealloc [[valloc_1]] : memref<117x113x6xi8>
+  // CHECK-NEXT: [[valloc_5:%.*]] = memref.alloc() : memref<117x3x120xi8>
+  // CHECK-NEXT: mpi.recv([[valloc_5]], [[vc91_i32]], [[vc29_i32]]) : memref<117x3x120xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_7:%.*]] = memref.subview [[v0]][1, 0, 0] [117, 3, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<117x3x120xi8, strided<[14400, 120, 1], offset: 14400>>
+  // CHECK-NEXT: memref.copy [[valloc_5]], [[vsubview_7]] : memref<117x3x120xi8> to memref<117x3x120xi8, strided<[14400, 120, 1], offset: 14400>>
+  // CHECK-NEXT: memref.dealloc [[valloc_5]] : memref<117x3x120xi8>
+  // CHECK-NEXT: [[valloc_8:%.*]] = memref.alloc() : memref<117x4x120xi8>
+  // CHECK-NEXT: [[vsubview_10:%.*]] = memref.subview [[v0]][1, 3, 0] [117, 4, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<117x4x120xi8, strided<[14400, 120, 1], offset: 14760>>
+  // CHECK-NEXT: memref.copy [[vsubview_10]], [[valloc_8]] : memref<117x4x120xi8, strided<[14400, 120, 1], offset: 14760>> to memref<117x4x120xi8>
+  // CHECK-NEXT: mpi.send([[valloc_8]], [[vc91_i32]], [[vc29_i32]]) : memref<117x4x120xi8>, i32, i32
+  // CHECK-NEXT: memref.dealloc [[valloc_8]] : memref<117x4x120xi8>
+  // CHECK-NEXT: [[valloc_11:%.*]] = memref.alloc() : memref<1x120x120xi8>
+  // CHECK-NEXT: [[vsubview_12:%.*]] = memref.subview [[v0]][117, 0, 0] [1, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<1x120x120xi8, strided<[14400, 120, 1], offset: 1684800>>
+  // CHECK-NEXT: memref.copy [[vsubview_12]], [[valloc_11]] : memref<1x120x120xi8, strided<[14400, 120, 1], offset: 1684800>> to memref<1x120x120xi8>
+  // CHECK-NEXT: mpi.send([[valloc_11]], [[vc91_i32]], [[vc23_i32]]) : memref<1x120x120xi8>, i32, i32
+  // CHECK-NEXT: memref.dealloc [[valloc_11]] : memref<1x120x120xi8>
+  // CHECK-NEXT: [[valloc_13:%.*]] = memref.alloc() : memref<2x120x120xi8>
+  // CHECK-NEXT: mpi.recv([[valloc_13]], [[vc91_i32]], [[vc23_i32]]) : memref<2x120x120xi8>, i32, i32
+  // CHECK-NEXT: [[vsubview_14:%.*]] = memref.subview [[v0]][118, 0, 0] [2, 120, 120] [1, 1, 1] : memref<120x120x120xi8> to memref<2x120x120xi8, strided<[14400, 120, 1], offset: 1699200>>
+  // CHECK-NEXT: memref.copy [[valloc_13]], [[vsubview_14]] : memref<2x120x120xi8> to memref<2x120x120xi8, strided<[14400, 120, 1], offset: 1699200>>
+  // CHECK-NEXT: memref.dealloc [[valloc_13]] : memref<2x120x120xi8>
+  // CHECK-NEXT: [[v1:%.*]] = bufferization.to_tensor [[v0]] restrict writable : memref<120x120x120xi8>
+  %res = mesh.update_halo %arg0 on @mesh0 split_axes = [[2], [1], [0]] halo_sizes = [1, 2, 3, 4, 5, 6] : tensor<120x120x120xi8>
+  // CHECK: return [[v1]] : tensor<120x120x120xi8>
+  return %res : tensor<120x120x120xi8>
+}
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index 1a29d3f9f3507..265a75986c6c8 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -1132,11 +1132,21 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK-DAG: linalg.yield [[TRUNC]]
   %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false} : (tensor<2xi8>) -> tensor<2xi8>
 
+  // CHECK: return
+  return
+}
+
+// -----
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: @rescale_i8_unsigned_output
+// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
+func.func @rescale_i8_unsigned_output(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C0:%.+]] = arith.constant 19689
   // CHECK: [[C1:%.+]] = arith.constant 15
   // CHECK: [[INIT:%.+]] = tensor.empty()
-  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%[[ARG0]] : tensor<2xi8>) outs([[INIT]] : tensor<2xui8>)
-  // CHECK: ^bb0([[IN:%.+]]: i8, [[UNUSED:%.+]]: ui8):
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%[[ARG0]] : tensor<2xi8>) outs([[INIT]] : tensor<2xi8>)
+  // CHECK: ^bb0([[IN:%.+]]: i8, [[UNUSED:%.+]]: i8):
   // CHECK: [[C17:%.+]] = arith.constant 17
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
@@ -1148,9 +1158,8 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK-DAG: [[LOWER:%.+]] = arith.maxsi [[CMIN]], [[SCALED_ZEROED]]
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
-  // CHECK-DAG: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[TRUNC]] : i8 to ui8
-  // CHECK: linalg.yield [[CAST]]
-  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false} : (tensor<2xi8>) -> tensor<2xui8>
+  // CHECK: linalg.yield [[TRUNC]]
+  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, output_unsigned = true} : (tensor<2xi8>) -> tensor<2xi8>
 
   // CHECK: return
   return
@@ -1171,9 +1180,9 @@ func.func @rescale_i8_dyn_batch(%arg0 : tensor<?x2xi8>) -> () {
 
   // CHECK: %[[C0:.+]] = arith.constant 0
   // CHECK: %[[BATCH:.+]] = tensor.dim %[[ARG0]], %[[C0]]
-  // CHECK: %[[INIT:.+]] = tensor.empty(%[[BATCH]]) : tensor<?x2xui8>
-  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x2xi8>) outs(%[[INIT]] : tensor<?x2xui8>)
-  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false} : (tensor<?x2xi8>) -> tensor<?x2xui8>
+  // CHECK: %[[INIT:.+]] = tensor.empty(%[[BATCH]]) : tensor<?x2xi8>
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x2xi8>) outs(%[[INIT]] : tensor<?x2xi8>)
+  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, output_unsigned = true} : (tensor<?x2xi8>) -> tensor<?x2xi8>
 
   return
 }
@@ -1199,18 +1208,17 @@ func.func @rescale_dyn(%arg0 : tensor<1x?x?x32xi32>) -> () {
 
 // CHECK: #[[$MAP0:.*]] = affine_map<(d0) -> (d0)>
 
-// CHECK-LABEL: @rescale_ui8
+// CHECK-LABEL: @rescale_i8_unsigned_input
 // CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
-func.func @rescale_ui8(%arg0 : tensor<2xui8>) -> () {
+func.func @rescale_i8_unsigned_input(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C0:%.+]] = arith.constant 19689
   // CHECK: [[C1:%.+]] = arith.constant 15
   // CHECK: [[INIT:%.+]] = tensor.empty()
-  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%[[ARG0]] : tensor<2xui8>) outs([[INIT]] : tensor<2xi8>)
-  // CHECK: ^bb0([[IN:%.+]]: ui8, [[UNUSED:%.+]]: i8):
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%[[ARG0]] : tensor<2xi8>) outs([[INIT]] : tensor<2xi8>)
+  // CHECK: ^bb0([[IN:%.+]]: i8, [[UNUSED:%.+]]: i8):
   // CHECK: [[C17:%.+]] = arith.constant 17
   // CHECK: [[C22:%.+]] = arith.constant 22
-  // CHECK-DAG: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[IN]] : ui8 to i8
-  // CHECK-DAG: [[IN32:%.+]] = arith.extui [[CAST]]
+  // CHECK-DAG: [[IN32:%.+]] = arith.extui [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
   // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {double_round = false}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
@@ -1220,7 +1228,7 @@ func.func @rescale_ui8(%arg0 : tensor<2xui8>) -> () {
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
   // CHECK: linalg.yield [[TRUNC]]
-  %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false} : (tensor<2xui8>) -> tensor<2xi8>
+  %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = true} : (tensor<2xi8>) -> tensor<2xi8>
 
   return
 }
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index da0222bc94237..2473fe933ffcb 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1,83 +1,128 @@
 // RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck %s
 
-// TODO: Add tests for for vector.type_cast that would cover scalable vectors
+//===----------------------------------------------------------------------===//
+// vector.bitcast
+//===----------------------------------------------------------------------===//
 
-func.func @bitcast_f32_to_i32_vector_0d(%input: vector<f32>) -> vector<i32> {
-  %0 = vector.bitcast %input : vector<f32> to vector<i32>
+func.func @bitcast_f32_to_i32_vector_0d(%arg0: vector<f32>) -> vector<i32> {
+  %0 = vector.bitcast %arg0 : vector<f32> to vector<i32>
   return %0 : vector<i32>
 }
 
 // CHECK-LABEL: @bitcast_f32_to_i32_vector_0d
-// CHECK-SAME:  %[[input:.*]]: vector<f32>
-// CHECK:       %[[vec_f32_1d:.*]] = builtin.unrealized_conversion_cast %[[input]] : vector<f32> to vector<1xf32>
-// CHECK:       %[[vec_i32_1d:.*]] = llvm.bitcast %[[vec_f32_1d]] : vector<1xf32> to vector<1xi32>
-// CHECK:       %[[vec_i32_0d:.*]] = builtin.unrealized_conversion_cast %[[vec_i32_1d]] : vector<1xi32> to vector<i32>
-// CHECK:       return %[[vec_i32_0d]] : vector<i32>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<f32>
+// CHECK:       %[[VEC_F32_1D:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<f32> to vector<1xf32>
+// CHECK:       %[[VEC_I32_1D:.*]] = llvm.bitcast %[[VEC_F32_1D]] : vector<1xf32> to vector<1xi32>
+// CHECK:       %[[VEC_I32_0D:.*]] = builtin.unrealized_conversion_cast %[[VEC_I32_1D]] : vector<1xi32> to vector<i32>
+// CHECK:       return %[[VEC_I32_0D]] : vector<i32>
 
 // -----
 
-func.func @bitcast_f32_to_i32_vector(%input: vector<16xf32>) -> vector<16xi32> {
-  %0 = vector.bitcast %input : vector<16xf32> to vector<16xi32>
+func.func @bitcast_f32_to_i32_vector(%arg0: vector<16xf32>) -> vector<16xi32> {
+  %0 = vector.bitcast %arg0 : vector<16xf32> to vector<16xi32>
   return %0 : vector<16xi32>
 }
 
+
 // CHECK-LABEL: @bitcast_f32_to_i32_vector
-// CHECK-SAME:  %[[input:.*]]: vector<16xf32>
-// CHECK:       llvm.bitcast %[[input]] : vector<16xf32> to vector<16xi32>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xf32>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<16xf32> to vector<16xi32>
 
-func.func @bitcast_f32_to_i32_vector_scalable(%input: vector<[16]xf32>) -> vector<[16]xi32> {
-  %0 = vector.bitcast %input : vector<[16]xf32> to vector<[16]xi32>
+// -----
+
+func.func @bitcast_f32_to_i32_vector_scalable(%arg0: vector<[16]xf32>) -> vector<[16]xi32> {
+  %0 = vector.bitcast %arg0 : vector<[16]xf32> to vector<[16]xi32>
   return %0 : vector<[16]xi32>
 }
 
 // CHECK-LABEL: @bitcast_f32_to_i32_vector_scalable
-// CHECK-SAME:  %[[input:.*]]: vector<[16]xf32>
-// CHECK:       llvm.bitcast %[[input]] : vector<[16]xf32> to vector<[16]xi32>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xf32>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[16]xf32> to vector<[16]xi32>
 
 // -----
 
-func.func @bitcast_i8_to_f32_vector(%input: vector<64xi8>) -> vector<16xf32> {
-  %0 = vector.bitcast %input : vector<64xi8> to vector<16xf32>
+func.func @bitcast_i8_to_f32_vector(%arg0: vector<64xi8>) -> vector<16xf32> {
+  %0 = vector.bitcast %arg0 : vector<64xi8> to vector<16xf32>
   return %0 : vector<16xf32>
 }
 
 // CHECK-LABEL: @bitcast_i8_to_f32_vector
-// CHECK-SAME:  %[[input:.*]]: vector<64xi8>
-// CHECK:       llvm.bitcast %[[input]] : vector<64xi8> to vector<16xf32>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<64xi8>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<64xi8> to vector<16xf32>
 
-func.func @bitcast_i8_to_f32_vector_scalable(%input: vector<[64]xi8>) -> vector<[16]xf32> {
-  %0 = vector.bitcast %input : vector<[64]xi8> to vector<[16]xf32>
+// -----
+
+func.func @bitcast_i8_to_f32_vector_scalable(%arg0: vector<[64]xi8>) -> vector<[16]xf32> {
+  %0 = vector.bitcast %arg0 : vector<[64]xi8> to vector<[16]xf32>
   return %0 : vector<[16]xf32>
 }
 
 // CHECK-LABEL: @bitcast_i8_to_f32_vector_scalable
-// CHECK-SAME:  %[[input:.*]]: vector<[64]xi8>
-// CHECK:       llvm.bitcast %[[input]] : vector<[64]xi8> to vector<[16]xf32>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[64]xi8>
+// CHECK:       llvm.bitcast %[[ARG_0]] : vector<[64]xi8> to vector<[16]xf32>
 
 // -----
 
-func.func @bitcast_index_to_i8_vector(%input: vector<16xindex>) -> vector<128xi8> {
-  %0 = vector.bitcast %input : vector<16xindex> to vector<128xi8>
+func.func @bitcast_index_to_i8_vector(%arg0: vector<16xindex>) -> vector<128xi8> {
+  %0 = vector.bitcast %arg0 : vector<16xindex> to vector<128xi8>
   return %0 : vector<128xi8>
 }
 
 // CHECK-LABEL: @bitcast_index_to_i8_vector
-// CHECK-SAME:  %[[input:.*]]: vector<16xindex>
-// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[input]] : vector<16xindex> to vector<16xi64>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<16xindex>
+// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<16xindex> to vector<16xi64>
 // CHECK:       llvm.bitcast %[[T0]] : vector<16xi64> to vector<128xi8>
 
-func.func @bitcast_index_to_i8_vector_scalable(%input: vector<[16]xindex>) -> vector<[128]xi8> {
-  %0 = vector.bitcast %input : vector<[16]xindex> to vector<[128]xi8>
+// -----
+
+func.func @bitcast_index_to_i8_vector_scalable(%arg0: vector<[16]xindex>) -> vector<[128]xi8> {
+  %0 = vector.bitcast %arg0 : vector<[16]xindex> to vector<[128]xi8>
   return %0 : vector<[128]xi8>
 }
 
 // CHECK-LABEL: @bitcast_index_to_i8_vector_scalable
-// CHECK-SAME:  %[[input:.*]]: vector<[16]xindex>
-// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[input]] : vector<[16]xindex> to vector<[16]xi64>
+// CHECK-SAME:  %[[ARG_0:.*]]: vector<[16]xindex>
+// CHECK:       %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<[16]xindex> to vector<[16]xi64>
 // CHECK:       llvm.bitcast %[[T0]] : vector<[16]xi64> to vector<[128]xi8>
 
 // -----
 
+// CHECK-LABEL:   func.func @bitcast_2d(
+// CHECK-SAME:      %[[ARG_0:.*]]: vector<2x4xi32>) -> vector<2x2xi64> {
+// CHECK:           %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<2x4xi32> to !llvm.array<2 x vector<4xi32>>
+// CHECK:           %[[VEC_1:.*]] = llvm.extractvalue %[[T0]][0] : !llvm.array<2 x vector<4xi32>>
+// CHECK:           %[[BCAST_1:.*]] = llvm.bitcast %[[VEC_1]] : vector<4xi32> to vector<2xi64>
+// CHECK:           %[[OUT_1:.*]] = llvm.insertvalue %[[BCAST_1]], {{.*}}[0] : !llvm.array<2 x vector<2xi64>>
+// CHECK:           %[[VEC_2:.*]] = llvm.extractvalue %[[T0]][1] : !llvm.array<2 x vector<4xi32>>
+// CHECK:           %[[BCAST_2:.*]] = llvm.bitcast %[[VEC_2]] : vector<4xi32> to vector<2xi64>
+// CHECK:           %[[OUT_2:.*]] = llvm.insertvalue %[[BCAST_2]], %[[OUT_1]][1] : !llvm.array<2 x vector<2xi64>>
+func.func @bitcast_2d(%arg0: vector<2x4xi32>) -> vector<2x2xi64> {
+  %0 = vector.bitcast %arg0 : vector<2x4xi32> to vector<2x2xi64>
+  return %0 : vector<2x2xi64>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @bitcast_2d_scalable(
+// CHECK-SAME:      %[[ARG_0:.*]]: vector<2x[4]xi32>) -> vector<2x[2]xi64> {
+// CHECK:           %[[T0:.*]] = builtin.unrealized_conversion_cast %[[ARG_0]] : vector<2x[4]xi32> to !llvm.array<2 x vector<[4]xi32>>
+// CHECK:           %[[VEC_1:.*]] = llvm.extractvalue %[[T0]][0] : !llvm.array<2 x vector<[4]xi32>>
+// CHECK:           %[[BCAST_1:.*]] = llvm.bitcast %[[VEC_1]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK:           %[[OUT_1:.*]] = llvm.insertvalue %[[BCAST_1]], {{.*}}[0] : !llvm.array<2 x vector<[2]xi64>>
+// CHECK:           %[[VEC_2:.*]] = llvm.extractvalue %[[T0]][1] : !llvm.array<2 x vector<[4]xi32>>
+// CHECK:           %[[BCAST_2:.*]] = llvm.bitcast %[[VEC_2]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK:           %[[OUT_2:.*]] = llvm.insertvalue %[[BCAST_2]], %[[OUT_1]][1] : !llvm.array<2 x vector<[2]xi64>>
+func.func @bitcast_2d_scalable(%arg0: vector<2x[4]xi32>) -> vector<2x[2]xi64> {
+  %0 = vector.bitcast %arg0 : vector<2x[4]xi32> to vector<2x[2]xi64>
+  return %0 : vector<2x[2]xi64>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.broadcast
+//===----------------------------------------------------------------------===//
+
 func.func @broadcast_vec0d_from_f32(%arg0: f32) -> vector<f32> {
   %0 = vector.broadcast %arg0 : f32 to vector<f32>
   return %0 : vector<f32>
@@ -110,6 +155,7 @@ func.func @broadcast_vec1d_from_f32(%arg0: f32) -> vector<2xf32> {
 // CHECK:       %[[T1:.*]] = llvm.shufflevector %[[T0]]
 // CHECK:       return %[[T1]] : vector<2xf32>
 
+// -----
 
 func.func @broadcast_vec1d_from_f32_scalable(%arg0: f32) -> vector<[2]xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<[2]xf32>
@@ -135,6 +181,8 @@ func.func @broadcast_vec1d_from_index(%arg0: index) -> vector<2xindex> {
 // CHECK:       %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<2xi64> to vector<2xindex>
 // CHECK:       return %[[T2]] : vector<2xindex>
 
+// -----
+
 func.func @broadcast_vec1d_from_index_scalable(%arg0: index) -> vector<[2]xindex> {
   %0 = vector.broadcast %arg0 : index to vector<[2]xindex>
   return %0 : vector<[2]xindex>
@@ -162,6 +210,8 @@ func.func @broadcast_vec2d_from_scalar(%arg0: f32) -> vector<2x3xf32> {
 // CHECK:       %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T3]] : !llvm.array<2 x vector<3xf32>> to vector<2x3xf32>
 // CHECK:       return %[[T4]] : vector<2x3xf32>
 
+// -----
+
 func.func @broadcast_vec2d_from_scalar_scalable(%arg0: f32) -> vector<2x[3]xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<2x[3]xf32>
   return %0 : vector<2x[3]xf32>
@@ -191,6 +241,7 @@ func.func @broadcast_vec3d_from_scalar(%arg0: f32) -> vector<2x3x4xf32> {
 // CHECK:       %[[T4:.*]] = builtin.unrealized_conversion_cast %[[T3]] : !llvm.array<2 x array<3 x vector<4xf32>>> to vector<2x3x4xf32>
 // CHECK:       return %[[T4]] : vector<2x3x4xf32>
 
+// -----
 
 func.func @broadcast_vec3d_from_scalar_scalable(%arg0: f32) -> vector<2x3x[4]xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<2x3x[4]xf32>
@@ -216,6 +267,8 @@ func.func @broadcast_vec1d_from_vec1d(%arg0: vector<2xf32>) -> vector<2xf32> {
 // CHECK-SAME:  %[[A:.*]]: vector<2xf32>)
 // CHECK:       return %[[A]] : vector<2xf32>
 
+// -----
+
 func.func @broadcast_vec1d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector<[2]xf32> {
   %0 = vector.broadcast %arg0 : vector<[2]xf32> to vector<[2]xf32>
   return %0 : vector<[2]xf32>
@@ -261,6 +314,8 @@ func.func @broadcast_vec2d_from_vec1d(%arg0: vector<2xf32>) -> vector<3x2xf32> {
 // CHECK:       %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : !llvm.array<3 x vector<2xf32>> to vector<3x2xf32>
 // CHECK:       return %[[T5]] : vector<3x2xf32>
 
+// -----
+
 func.func @broadcast_vec2d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector<3x[2]xf32> {
   %0 = vector.broadcast %arg0 : vector<[2]xf32> to vector<3x[2]xf32>
   return %0 : vector<3x[2]xf32>
@@ -291,6 +346,8 @@ func.func @broadcast_vec2d_from_index_vec1d(%arg0: vector<2xindex>) -> vector<3x
 // CHECK:       %[[T4:.*]] = builtin.unrealized_conversion_cast %{{.*}} : !llvm.array<3 x vector<2xi64>> to vector<3x2xindex>
 // CHECK:       return %[[T4]] : vector<3x2xindex>
 
+// -----
+
 func.func @broadcast_vec2d_from_index_vec1d_scalable(%arg0: vector<[2]xindex>) -> vector<3x[2]xindex> {
   %0 = vector.broadcast %arg0 : vector<[2]xindex> to vector<3x[2]xindex>
   return %0 : vector<3x[2]xindex>
@@ -330,6 +387,8 @@ func.func @broadcast_vec3d_from_vec1d(%arg0: vector<2xf32>) -> vector<4x3x2xf32>
 // CHECK:       %[[T11:.*]] = builtin.unrealized_conversion_cast %[[T10]] : !llvm.array<4 x array<3 x vector<2xf32>>> to vector<4x3x2xf32>
 // CHECK:       return %[[T11]] : vector<4x3x2xf32>
 
+// -----
+
 func.func @broadcast_vec3d_from_vec1d_scalable(%arg0: vector<[2]xf32>) -> vector<4x3x[2]xf32> {
   %0 = vector.broadcast %arg0 : vector<[2]xf32> to vector<4x3x[2]xf32>
   return %0 : vector<4x3x[2]xf32>
@@ -371,6 +430,8 @@ func.func @broadcast_vec3d_from_vec2d(%arg0: vector<3x2xf32>) -> vector<4x3x2xf3
 // CHECK:       %[[T10:.*]] = builtin.unrealized_conversion_cast %[[T9]] : !llvm.array<4 x array<3 x vector<2xf32>>> to vector<4x3x2xf32>
 // CHECK:       return %[[T10]] : vector<4x3x2xf32>
 
+// -----
+
 func.func @broadcast_vec3d_from_vec2d_scalable(%arg0: vector<3x[2]xf32>) -> vector<4x3x[2]xf32> {
   %0 = vector.broadcast %arg0 : vector<3x[2]xf32> to vector<4x3x[2]xf32>
   return %0 : vector<4x3x[2]xf32>
@@ -402,6 +463,8 @@ func.func @broadcast_stretch(%arg0: vector<1xf32>) -> vector<4xf32> {
 // CHECK:       %[[T4:.*]] = llvm.shufflevector %[[T3]]
 // CHECK:       return %[[T4]] : vector<4xf32>
 
+// -----
+
 func.func @broadcast_stretch_scalable(%arg0: vector<1xf32>) -> vector<[4]xf32> {
   %0 = vector.broadcast %arg0 : vector<1xf32> to vector<[4]xf32>
   return %0 : vector<[4]xf32>
@@ -432,6 +495,8 @@ func.func @broadcast_stretch_at_start(%arg0: vector<1x4xf32>) -> vector<3x4xf32>
 // CHECK:       %[[T8:.*]] = builtin.unrealized_conversion_cast %[[T7]] : !llvm.array<3 x vector<4xf32>> to vector<3x4xf32>
 // CHECK:       return %[[T8]] : vector<3x4xf32>
 
+// -----
+
 func.func @broadcast_stretch_at_start_scalable(%arg0: vector<1x[4]xf32>) -> vector<3x[4]xf32> {
   %0 = vector.broadcast %arg0 : vector<1x[4]xf32> to vector<3x[4]xf32>
   return %0 : vector<3x[4]xf32>
@@ -532,6 +597,8 @@ func.func @broadcast_stretch_in_middle(%arg0: vector<4x1x2xf32>) -> vector<4x3x2
 // CHECK:       %[[T32:.*]] = builtin.unrealized_conversion_cast %[[T31]] : !llvm.array<4 x array<3 x vector<2xf32>>> to vector<4x3x2xf32>
 // CHECK:       return %[[T32]] : vector<4x3x2xf32>
 
+// -----
+
 func.func @broadcast_stretch_in_middle_scalable_v1(%arg0: vector<4x1x[2]xf32>) -> vector<4x3x[2]xf32> {
   %0 = vector.broadcast %arg0 : vector<4x1x[2]xf32> to vector<4x3x[2]xf32>
   return %0 : vector<4x3x[2]xf32>
@@ -566,6 +633,8 @@ func.func @broadcast_stretch_in_middle_scalable_v1(%arg0: vector<4x1x[2]xf32>) -
 // CHECK:       %[[T32:.*]] = builtin.unrealized_conversion_cast %[[T31]] : !llvm.array<4 x array<3 x vector<[2]xf32>>> to vector<4x3x[2]xf32>
 // CHECK:       return %[[T32]] : vector<4x3x[2]xf32>
 
+// -----
+
 // TODO: Add support for scalable vectors
 
 func.func @broadcast_stretch_in_middle_scalable_v2(%arg0: vector<[4]x1x2xf32>) -> vector<[4]x3x2xf32> {
@@ -578,6 +647,10 @@ func.func @broadcast_stretch_in_middle_scalable_v2(%arg0: vector<[4]x1x2xf32>) -
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.outerproduct
+//===----------------------------------------------------------------------===//
+
 func.func @outerproduct(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<2x3xf32> {
   %2 = vector.outerproduct %arg0, %arg1 : vector<2xf32>, vector<3xf32>
   return %2 : vector<2x3xf32>
@@ -602,6 +675,8 @@ func.func @outerproduct(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<2x
 // CHECK:       %[[T14:.*]] = builtin.unrealized_conversion_cast %[[T13]] : !llvm.array<2 x vector<3xf32>> to vector<2x3xf32>
 // CHECK:       return %[[T14]] : vector<2x3xf32>
 
+// -----
+
 func.func @outerproduct_scalable(%arg0: vector<2xf32>, %arg1: vector<[3]xf32>) -> vector<2x[3]xf32> {
   %2 = vector.outerproduct %arg0, %arg1 : vector<2xf32>, vector<[3]xf32>
   return %2 : vector<2x[3]xf32>
@@ -647,6 +722,8 @@ func.func @outerproduct_index(%arg0: vector<2xindex>, %arg1: vector<3xindex>) ->
 // CHECK:       %[[T7:.*]] = builtin.unrealized_conversion_cast %[[T6]] : vector<3xindex> to vector<3xi64>
 // CHECK:       %{{.*}} = llvm.insertvalue %[[T7]], %[[T8]][0] : !llvm.array<2 x vector<3xi64>>
 
+// -----
+
 func.func @outerproduct_index_scalable(%arg0: vector<2xindex>, %arg1: vector<[3]xindex>) -> vector<2x[3]xindex> {
   %2 = vector.outerproduct %arg0, %arg1 : vector<2xindex>, vector<[3]xindex>
   return %2 : vector<2x[3]xindex>
@@ -696,6 +773,8 @@ func.func @outerproduct_add(%arg0: vector<2xf32>, %arg1: vector<3xf32>, %arg2: v
 // CHECK:       %[[T19:.*]] = builtin.unrealized_conversion_cast %[[T18]] : !llvm.array<2 x vector<3xf32>> to vector<2x3xf32>
 // CHECK:       return %[[T19]] : vector<2x3xf32>
 
+// -----
+
 func.func @outerproduct_add_scalable(%arg0: vector<2xf32>, %arg1: vector<[3]xf32>, %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> {
   %2 = vector.outerproduct %arg0, %arg1, %arg2 : vector<2xf32>, vector<[3]xf32>
   return %2 : vector<2x[3]xf32>
@@ -726,6 +805,10 @@ func.func @outerproduct_add_scalable(%arg0: vector<2xf32>, %arg1: vector<[3]xf32
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.mask { vector.outerproduct }
+//===----------------------------------------------------------------------===//
+
 func.func @masked_float_add_outerprod(%arg0: vector<2xf32>, %arg1: f32, %arg2: vector<2xf32>, %m: vector<2xi1>) -> vector<2xf32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<add>} : vector<2xf32>, f32 } : vector<2xi1> -> vector<2xf32>
   return %0 : vector<2xf32>
@@ -736,6 +819,8 @@ func.func @masked_float_add_outerprod(%arg0: vector<2xf32>, %arg1: f32, %arg2: v
 // CHECK:           %[[VAL_8:.*]] = llvm.intr.fmuladd(%[[VAL_0]], %{{.*}}, %[[VAL_2]])  : (vector<2xf32>, vector<2xf32>, vector<2xf32>) -> vector<2xf32>
 // CHECK:           %[[VAL_9:.*]] = arith.select %[[VAL_3]], %[[VAL_8]], %[[VAL_2]] : vector<2xi1>, vector<2xf32>
 
+// -----
+
 func.func @masked_float_add_outerprod_scalable(%arg0: vector<[2]xf32>, %arg1: f32, %arg2: vector<[2]xf32>, %m: vector<[2]xi1>) -> vector<[2]xf32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<add>} : vector<[2]xf32>, f32 } : vector<[2]xi1> -> vector<[2]xf32>
   return %0 : vector<[2]xf32>
@@ -759,6 +844,8 @@ func.func @masked_float_mul_outerprod(%arg0: vector<2xf32>, %arg1: f32, %arg2: v
 // CHECK:           %[[VAL_9:.*]] = arith.mulf %[[VAL_8]], %[[VAL_2]] : vector<2xf32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xf32>
 
+// -----
+
 func.func @masked_float_mul_outerprod_scalable(%arg0: vector<[2]xf32>, %arg1: f32, %arg2: vector<[2]xf32>, %m: vector<[2]xi1>) -> vector<[2]xf32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<mul>} : vector<[2]xf32>, f32 } : vector<[2]xi1> -> vector<[2]xf32>
   return %0 : vector<[2]xf32>
@@ -783,6 +870,8 @@ func.func @masked_float_max_outerprod(%arg0: vector<2xf32>, %arg1: f32, %arg2: v
 // CHECK:           %[[VAL_9:.*]] = arith.maxnumf %[[VAL_8]], %[[VAL_2]] : vector<2xf32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xf32>
 
+// -----
+
 func.func @masked_float_max_outerprod_scalable(%arg0: vector<[2]xf32>, %arg1: f32, %arg2: vector<[2]xf32>, %m: vector<[2]xi1>) -> vector<[2]xf32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<maxnumf>} : vector<[2]xf32>, f32 } : vector<[2]xi1> -> vector<[2]xf32>
   return %0 : vector<[2]xf32>
@@ -807,6 +896,8 @@ func.func @masked_float_min_outerprod(%arg0: vector<2xf32>, %arg1: f32, %arg2: v
 // CHECK:           %[[VAL_9:.*]] = arith.minnumf %[[VAL_8]], %[[VAL_2]] : vector<2xf32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xf32>
 
+// -----
+
 func.func @masked_float_min_outerprod_scalable(%arg0: vector<[2]xf32>, %arg1: f32, %arg2: vector<[2]xf32>, %m: vector<[2]xi1>) -> vector<[2]xf32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<minnumf>} : vector<[2]xf32>, f32 } : vector<[2]xi1> -> vector<[2]xf32>
   return %0 : vector<[2]xf32>
@@ -831,6 +922,8 @@ func.func @masked_int_add_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vec
 // CHECK:           %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_add_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<add>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -855,6 +948,8 @@ func.func @masked_int_mul_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vec
 // CHECK:           %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_mul_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<mul>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -879,6 +974,8 @@ func.func @masked_int_max_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vec
 // CHECK:           %[[VAL_9:.*]] = arith.maxsi %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_max_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<maxsi>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -903,6 +1000,8 @@ func.func @masked_int_min_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vec
 // CHECK:           %[[VAL_9:.*]] = arith.minui %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_min_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<minui>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -927,6 +1026,8 @@ func.func @masked_int_and_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vec
 // CHECK:           %[[VAL_9:.*]] = arith.andi %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_and_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<and>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -951,6 +1052,8 @@ func.func @masked_int_or_outerprod(%arg0: vector<2xi32>, %arg1: i32, %arg2: vect
 // CHECK:           %[[VAL_9:.*]] = arith.ori %[[VAL_8]], %[[VAL_2]] : vector<2xi32>
 // CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_3]], %[[VAL_9]], %[[VAL_2]] : vector<2xi1>, vector<2xi32>
 
+// -----
+
 func.func @masked_int_or_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32, %arg2: vector<[2]xi32>, %m: vector<[2]xi1>) -> vector<[2]xi32> {
   %0 = vector.mask %m { vector.outerproduct %arg0, %arg1, %arg2 {kind = #vector.kind<or>} : vector<[2]xi32>, i32 } : vector<[2]xi1> -> vector<[2]xi32>
   return %0 : vector<[2]xi32>
@@ -964,6 +1067,10 @@ func.func @masked_int_or_outerprod_scalable(%arg0: vector<[2]xi32>, %arg1: i32,
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.shuffle
+//===----------------------------------------------------------------------===//
+
 func.func @shuffle_0D_direct(%arg0: vector<f32>) -> vector<3xf32> {
   %1 = vector.shuffle %arg0, %arg0 [0, 1, 0] : vector<f32>, vector<f32>
   return %1 : vector<3xf32>
@@ -1005,28 +1112,28 @@ func.func @shuffle_1D(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<5xf3
 // CHECK-LABEL: @shuffle_1D(
 // CHECK-SAME: %[[A:.*]]: vector<2xf32>,
 // CHECK-SAME: %[[B:.*]]: vector<3xf32>)
-//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : vector<5xf32>
-//       CHECK:   %[[c2:.*]] = llvm.mlir.constant(2 : index) : i64
-//       CHECK:   %[[e1:.*]] = llvm.extractelement %[[B]][%[[c2]] : i64] : vector<3xf32>
-//       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[i1:.*]] = llvm.insertelement %[[e1]], %[[u0]][%[[c0]] : i64] : vector<5xf32>
-//       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[e2:.*]] = llvm.extractelement %[[B]][%[[c1]] : i64] : vector<3xf32>
-//       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[i2:.*]] = llvm.insertelement %[[e2]], %[[i1]][%[[c1]] : i64] : vector<5xf32>
-//       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[e3:.*]] = llvm.extractelement %[[B]][%[[c0]] : i64] : vector<3xf32>
-//       CHECK:   %[[c2:.*]] = llvm.mlir.constant(2 : index) : i64
-//       CHECK:   %[[i3:.*]] = llvm.insertelement %[[e3]], %[[i2]][%[[c2]] : i64] : vector<5xf32>
-//       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : i64
-//       CHECK:   %[[e4:.*]] = llvm.extractelement %[[A]][%[[c1]] : i64] : vector<2xf32>
-//       CHECK:   %[[c3:.*]] = llvm.mlir.constant(3 : index) : i64
-//       CHECK:   %[[i4:.*]] = llvm.insertelement %[[e4]], %[[i3]][%[[c3]] : i64] : vector<5xf32>
-//       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[e5:.*]] = llvm.extractelement %[[A]][%[[c0]] : i64] : vector<2xf32>
-//       CHECK:   %[[c4:.*]] = llvm.mlir.constant(4 : index) : i64
-//       CHECK:   %[[i5:.*]] = llvm.insertelement %[[e5]], %[[i4]][%[[c4]] : i64] : vector<5xf32>
-//       CHECK:   return %[[i5]] : vector<5xf32>
+//       CHECK:   %[[U0:.*]] = llvm.mlir.undef : vector<5xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
+//       CHECK:   %[[E1:.*]] = llvm.extractelement %[[B]][%[[C2]] : i64] : vector<3xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[I1:.*]] = llvm.insertelement %[[E1]], %[[U0]][%[[C0]] : i64] : vector<5xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[E2:.*]] = llvm.extractelement %[[B]][%[[C1]] : i64] : vector<3xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[I2:.*]] = llvm.insertelement %[[E2]], %[[I1]][%[[C1]] : i64] : vector<5xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[E3:.*]] = llvm.extractelement %[[B]][%[[C0]] : i64] : vector<3xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64
+//       CHECK:   %[[I3:.*]] = llvm.insertelement %[[E3]], %[[I2]][%[[C2]] : i64] : vector<5xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64
+//       CHECK:   %[[E4:.*]] = llvm.extractelement %[[A]][%[[C1]] : i64] : vector<2xf32>
+//       CHECK:   %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64
+//       CHECK:   %[[I4:.*]] = llvm.insertelement %[[E4]], %[[I3]][%[[C3]] : i64] : vector<5xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+//       CHECK:   %[[E5:.*]] = llvm.extractelement %[[A]][%[[C0]] : i64] : vector<2xf32>
+//       CHECK:   %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64
+//       CHECK:   %[[I5:.*]] = llvm.insertelement %[[E5]], %[[I4]][%[[C4]] : i64] : vector<5xf32>
+//       CHECK:   return %[[I5]] : vector<5xf32>
 
 // -----
 
@@ -1039,18 +1146,22 @@ func.func @shuffle_2D(%a: vector<1x4xf32>, %b: vector<2x4xf32>) -> vector<3x4xf3
 // CHECK-SAME: %[[B:.*]]: vector<2x4xf32>)
 //       CHECK-DAG:   %[[VAL_0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1x4xf32> to !llvm.array<1 x vector<4xf32>>
 //       CHECK-DAG:   %[[VAL_1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x4xf32> to !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[e1:.*]] = llvm.extractvalue %[[VAL_1]][0] : !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[i1:.*]] = llvm.insertvalue %[[e1]], %[[u0]][0] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[e2:.*]] = llvm.extractvalue %[[VAL_0]][0] : !llvm.array<1 x vector<4xf32>>
-//       CHECK:   %[[i2:.*]] = llvm.insertvalue %[[e2]], %[[i1]][1] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[e3:.*]] = llvm.extractvalue %[[VAL_1]][1] : !llvm.array<2 x vector<4xf32>>
-//       CHECK:   %[[i3:.*]] = llvm.insertvalue %[[e3]], %[[i2]][2] : !llvm.array<3 x vector<4xf32>>
-//       CHECK:   %[[VAL_3:.*]] = builtin.unrealized_conversion_cast %[[i3]] : !llvm.array<3 x vector<4xf32>> to vector<3x4xf32>
+//       CHECK:   %[[U0:.*]] = llvm.mlir.undef : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E1:.*]] = llvm.extractvalue %[[VAL_1]][0] : !llvm.array<2 x vector<4xf32>>
+//       CHECK:   %[[I1:.*]] = llvm.insertvalue %[[E1]], %[[U0]][0] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E2:.*]] = llvm.extractvalue %[[VAL_0]][0] : !llvm.array<1 x vector<4xf32>>
+//       CHECK:   %[[I2:.*]] = llvm.insertvalue %[[E2]], %[[I1]][1] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[E3:.*]] = llvm.extractvalue %[[VAL_1]][1] : !llvm.array<2 x vector<4xf32>>
+//       CHECK:   %[[I3:.*]] = llvm.insertvalue %[[E3]], %[[I2]][2] : !llvm.array<3 x vector<4xf32>>
+//       CHECK:   %[[VAL_3:.*]] = builtin.unrealized_conversion_cast %[[I3]] : !llvm.array<3 x vector<4xf32>> to vector<3x4xf32>
 //       CHECK:   return %[[VAL_3]] : vector<3x4xf32>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.extractelement
+//===----------------------------------------------------------------------===//
+
 func.func @extractelement_from_vec_0d_f32(%arg0: vector<f32>) -> f32 {
   %1 = vector.extractelement %arg0[] : vector<f32>
   return %1 : f32
@@ -1068,9 +1179,11 @@ func.func @extractelement_from_vec_1d_f32_idx_as_i32(%arg0: vector<16xf32>) -> f
 }
 // CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32(
 //  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 15 : i32
-//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[c]] : i32] : vector<16xf32>
-//       CHECK:   return %[[x]] : f32
+//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<16xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
 
 func.func @extractelement_from_vec_1d_f32_idx_as_i32_scalable(%arg0: vector<[16]xf32>) -> f32 {
   %0 = arith.constant 15 : i32
@@ -1079,9 +1192,9 @@ func.func @extractelement_from_vec_1d_f32_idx_as_i32_scalable(%arg0: vector<[16]
 }
 // CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_i32_scalable(
 //  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 15 : i32
-//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[c]] : i32] : vector<[16]xf32>
-//       CHECK:   return %[[x]] : f32
+//       CHECK:   %[[C:.*]] = arith.constant 15 : i32
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[C]] : i32] : vector<[16]xf32>
+//       CHECK:   return %[[X]] : f32
 
 // -----
 func.func @extractelement_from_vec_1d_f32_idx_as_index(%arg0: vector<16xf32>) -> f32 {
@@ -1091,10 +1204,12 @@ func.func @extractelement_from_vec_1d_f32_idx_as_index(%arg0: vector<16xf32>) ->
 }
 // CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index(
 //  CHECK-SAME:   %[[A:.*]]: vector<16xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 15 : index
-//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
-//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[i]] : i64] : vector<16xf32>
-//       CHECK:   return %[[x]] : f32
+//       CHECK:   %[[C:.*]] = arith.constant 15 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<16xf32>
+//       CHECK:   return %[[X]] : f32
+
+// -----
 
 func.func @extractelement_from_vec_1d_f32_idx_as_index_scalable(%arg0: vector<[16]xf32>) -> f32 {
   %0 = arith.constant 15 : index
@@ -1103,13 +1218,17 @@ func.func @extractelement_from_vec_1d_f32_idx_as_index_scalable(%arg0: vector<[1
 }
 // CHECK-LABEL: @extractelement_from_vec_1d_f32_idx_as_index_scalable(
 //  CHECK-SAME:   %[[A:.*]]: vector<[16]xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 15 : index
-//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
-//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[i]] : i64] : vector<[16]xf32>
-//       CHECK:   return %[[x]] : f32
+//       CHECK:   %[[C:.*]] = arith.constant 15 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.extractelement %[[A]][%[[I]] : i64] : vector<[16]xf32>
+//       CHECK:   return %[[X]] : f32
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.extract
+//===----------------------------------------------------------------------===//
+
 func.func @extract_scalar_from_vec_1d_f32(%arg0: vector<16xf32>) -> f32 {
   %0 = vector.extract %arg0[15]: f32 from vector<16xf32>
   return %0 : f32
@@ -1119,6 +1238,8 @@ func.func @extract_scalar_from_vec_1d_f32(%arg0: vector<16xf32>) -> f32 {
 //       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
 //       CHECK:   return {{.*}} : f32
 
+// -----
+
 func.func @extract_scalar_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
   %0 = vector.extract %arg0[15]: f32 from vector<[16]xf32>
   return %0 : f32
@@ -1141,6 +1262,8 @@ func.func @extract_vec_1e_from_vec_1d_f32(%arg0: vector<16xf32>) -> vector<1xf32
 //       CHECK:   %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : f32 to vector<1xf32>
 //       CHECK:   return %[[T2]] : vector<1xf32>
 
+// -----
+
 func.func @extract_vec_1e_from_vec_1d_f32_scalable(%arg0: vector<[16]xf32>) -> vector<1xf32> {
   %0 = vector.extract %arg0[15]: vector<1xf32> from vector<[16]xf32>
   return %0 : vector<1xf32>
@@ -1166,6 +1289,8 @@ func.func @extract_scalar_from_vec_1d_index(%arg0: vector<16xindex>) -> index {
 //       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
 //       CHECK:   return %[[T3]] : index
 
+// -----
+
 func.func @extract_scalar_from_vec_1d_index_scalable(%arg0: vector<[16]xindex>) -> index {
   %0 = vector.extract %arg0[15]: index from vector<[16]xindex>
   return %0 : index
@@ -1188,6 +1313,8 @@ func.func @extract_vec_2d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<3
 //       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<3x16xf32>
 
+// -----
+
 func.func @extract_vec_2d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<3x[16]xf32> {
   %0 = vector.extract %arg0[0]: vector<3x[16]xf32> from vector<4x3x[16]xf32>
   return %0 : vector<3x[16]xf32>
@@ -1206,6 +1333,8 @@ func.func @extract_vec_1d_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> vector<1
 //       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<16xf32>
 
+// -----
+
 func.func @extract_vec_1d_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> vector<[16]xf32> {
   %0 = vector.extract %arg0[0, 0]: vector<[16]xf32> from vector<4x3x[16]xf32>
   return %0 : vector<[16]xf32>
@@ -1226,6 +1355,8 @@ func.func @extract_scalar_from_vec_3d_f32(%arg0: vector<4x3x16xf32>) -> f32 {
 //       CHECK:   llvm.extractelement {{.*}}[{{.*}} : i64] : vector<16xf32>
 //       CHECK:   return {{.*}} : f32
 
+// -----
+
 func.func @extract_scalar_from_vec_3d_f32_scalable(%arg0: vector<4x3x[16]xf32>) -> f32 {
   %0 = vector.extract %arg0[0, 0, 0]: f32 from vector<4x3x[16]xf32>
   return %0 : f32
@@ -1247,6 +1378,8 @@ func.func @extract_scalar_from_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %ar
 //       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
 //       CHECK:   llvm.extractelement %[[VEC]][%[[UC]] : i64] : vector<16xf32>
 
+// -----
+
 func.func @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: index) -> f32 {
   %0 = vector.extract %arg0[%arg1]: f32 from vector<[16]xf32>
   return %0 : f32
@@ -1258,28 +1391,74 @@ func.func @extract_scalar_from_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16
 
 // -----
 
-func.func @extract_scalar_from_vec_2d_f32_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
+func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
   %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x16xf32>
   return %0 : f32
 }
 
-// Multi-dim vectors are not supported but this test shouldn't crash.
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
 
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_dynamic_idx(
-//       CHECK:   vector.extract
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx(
+//       CHECK:   llvm.extractvalue
+//       CHECK:   llvm.extractelement
 
-func.func @extract_scalar_from_vec_2d_f32_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
+func.func @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
   %0 = vector.extract %arg0[0, %arg1]: f32 from vector<1x[16]xf32>
   return %0 : f32
 }
 
-// Multi-dim vectors are not supported but this test shouldn't crash.
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
 
-// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_dynamic_idx_scalable(
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_inner_dynamic_idx_scalable(
+//       CHECK:   llvm.extractvalue
+//       CHECK:   llvm.extractelement
+
+// -----
+
+func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(%arg0: vector<1x16xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x16xf32>
+  return %0 : f32
+}
+
+// Lowering supports extracting from multi-dim vectors with dynamic indices
+// provided that only the trailing index is dynamic.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx(
 //       CHECK:   vector.extract
 
+func.func @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: index) -> f32 {
+  %0 = vector.extract %arg0[%arg1, 0]: f32 from vector<1x[16]xf32>
+  return %0 : f32
+}
+
+// Lowering does not support extracting from multi-dim vectors with non trailing
+// dynamic index, but it shouldn't crash.
+
+// CHECK-LABEL: @extract_scalar_from_vec_2d_f32_outer_dynamic_idx_scalable(
+//       CHECK:   vector.extract
+
+// -----
+
+func.func @extract_scalar_from_vec_0d_index(%arg0: vector<index>) -> index {
+  %0 = vector.extract %arg0[]: index from vector<index>
+  return %0 : index
+}
+// CHECK-LABEL: @extract_scalar_from_vec_0d_index(
+//  CHECK-SAME:   %[[A:.*]]: vector<index>)
+//       CHECK:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<index> to vector<1xi64>
+//       CHECK:   %[[T1:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[T2:.*]] = llvm.extractelement %[[T0]][%[[T1]] : i64] : vector<1xi64>
+//       CHECK:   %[[T3:.*]] = builtin.unrealized_conversion_cast %[[T2]] : i64 to index
+//       CHECK:   return %[[T3]] : index
+
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.insertelement
+//===----------------------------------------------------------------------===//
+
 func.func @insertelement_into_vec_0d_f32(%arg0: f32, %arg1: vector<f32>) -> vector<f32> {
   %1 = vector.insertelement %arg0, %arg1[] : vector<f32>
   return %1 : vector<f32>
@@ -1289,7 +1468,7 @@ func.func @insertelement_into_vec_0d_f32(%arg0: f32, %arg1: vector<f32>) -> vect
 //       CHECK:   %[[B:.*]] =  builtin.unrealized_conversion_cast %{{.*}} :
 //       CHECK:   vector<f32> to vector<1xf32>
 //       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C0]] : {{.*}}] : vector<1xf32>
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C0]] : {{.*}}] : vector<1xf32>
 
 // -----
 
@@ -1301,9 +1480,11 @@ func.func @insertelement_into_vec_1d_f32_idx_as_i32(%arg0: f32, %arg1: vector<4x
 // CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32(
 //  CHECK-SAME:   %[[A:.*]]: f32,
 //  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 3 : i32
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<4xf32>
-//       CHECK:   return %[[x]] : vector<4xf32>
+//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<4xf32>
+//       CHECK:   return %[[X]] : vector<4xf32>
+
+// -----
 
 func.func @insertelement_into_vec_1d_f32_idx_as_i32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
   %0 = arith.constant 3 : i32
@@ -1313,9 +1494,9 @@ func.func @insertelement_into_vec_1d_f32_idx_as_i32_scalable(%arg0: f32, %arg1:
 // CHECK-LABEL: @insertelement_into_vec_1d_f32_idx_as_i32_scalable(
 //  CHECK-SAME:   %[[A:.*]]: f32,
 //  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 3 : i32
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<[4]xf32>
-//       CHECK:   return %[[x]] : vector<[4]xf32>
+//       CHECK:   %[[C:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[C]] : i32] : vector<[4]xf32>
+//       CHECK:   return %[[X]] : vector<[4]xf32>
 
 // -----
 
@@ -1327,10 +1508,12 @@ func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index(%arg0: f32, %arg1
 // CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index(
 //  CHECK-SAME:   %[[A:.*]]: f32,
 //  CHECK-SAME:   %[[B:.*]]: vector<4xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 3 : index
-//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<4xf32>
-//       CHECK:   return %[[x]] : vector<4xf32>
+//       CHECK:   %[[C:.*]] = arith.constant 3 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<4xf32>
+//       CHECK:   return %[[X]] : vector<4xf32>
+
+// -----
 
 func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
   %0 = arith.constant 3 : index
@@ -1340,13 +1523,17 @@ func.func @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(%arg0: f
 // CHECK-LABEL: @insertelement_into_vec_1d_f32_scalable_idx_as_index_scalable(
 //  CHECK-SAME:   %[[A:.*]]: f32,
 //  CHECK-SAME:   %[[B:.*]]: vector<[4]xf32>)
-//       CHECK:   %[[c:.*]] = arith.constant 3 : index
-//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<[4]xf32>
-//       CHECK:   return %[[x]] : vector<[4]xf32>
+//       CHECK:   %[[C:.*]] = arith.constant 3 : index
+//       CHECK:   %[[I:.*]] = builtin.unrealized_conversion_cast %[[C]] : index to i64
+//       CHECK:   %[[X:.*]] = llvm.insertelement %[[A]], %[[B]][%[[I]] : i64] : vector<[4]xf32>
+//       CHECK:   return %[[X]] : vector<[4]xf32>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.insert
+//===----------------------------------------------------------------------===//
+
 func.func @insert_scalar_into_vec_1d_f32(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = vector.insert %arg0, %arg1[3] : f32 into vector<4xf32>
   return %0 : vector<4xf32>
@@ -1356,6 +1543,8 @@ func.func @insert_scalar_into_vec_1d_f32(%arg0: f32, %arg1: vector<4xf32>) -> ve
 //       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<4xf32>
 //       CHECK:   return {{.*}} : vector<4xf32>
 
+// -----
+
 func.func @insert_scalar_into_vec_1d_f32_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
   %0 = vector.insert %arg0, %arg1[3] : f32 into vector<[4]xf32>
   return %0 : vector<[4]xf32>
@@ -1381,6 +1570,7 @@ func.func @insert_scalar_into_vec_1d_index(%arg0: index, %arg1: vector<4xindex>)
 //       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<4xi64> to vector<4xindex>
 //       CHECK:   return %[[T5]] : vector<4xindex>
 
+// -----
 
 func.func @insert_scalar_into_vec_1d_index_scalable(%arg0: index, %arg1: vector<[4]xindex>) -> vector<[4]xindex> {
   %0 = vector.insert %arg0, %arg1[3] : index into vector<[4]xindex>
@@ -1406,6 +1596,8 @@ func.func @insert_vec_2d_into_vec_3d_f32(%arg0: vector<8x16xf32>, %arg1: vector<
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+// -----
+
 func.func @insert_vec_2d_into_vec_3d_f32_scalable(%arg0: vector<8x[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
   %0 = vector.insert %arg0, %arg1[3] : vector<8x[16]xf32> into vector<4x8x[16]xf32>
   return %0 : vector<4x8x[16]xf32>
@@ -1424,6 +1616,8 @@ func.func @insert_vec_1d_into_vec_3d_f32(%arg0: vector<16xf32>, %arg1: vector<4x
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+// -----
+
 func.func @insert_vec_1d_into_vec_3d_f32_scalable(%arg0: vector<[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
   %0 = vector.insert %arg0, %arg1[3, 7] : vector<[16]xf32> into vector<4x8x[16]xf32>
   return %0 : vector<4x8x[16]xf32>
@@ -1445,6 +1639,8 @@ func.func @insert_scalar_into_vec_3d_f32(%arg0: f32, %arg1: vector<4x8x16xf32>)
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+// -----
+
 func.func @insert_scalar_into_vec_3d_f32_scalable(%arg0: f32, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
   %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x[16]xf32>
   return %0 : vector<4x8x[16]xf32>
@@ -1469,6 +1665,8 @@ func.func @insert_scalar_into_vec_1d_f32_dynamic_idx(%arg0: vector<16xf32>, %arg
 //       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
 //       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<16xf32>
 
+// -----
+
 func.func @insert_scalar_into_vec_1d_f32_dynamic_idx_scalable(%arg0: vector<[16]xf32>, %arg1: f32, %arg2: index)
                                       -> vector<[16]xf32> {
   %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<[16]xf32>
@@ -1493,6 +1691,8 @@ func.func @insert_scalar_into_vec_2d_f32_dynamic_idx(%arg0: vector<1x16xf32>, %a
 // CHECK-LABEL: @insert_scalar_into_vec_2d_f32_dynamic_idx(
 //       CHECK:   vector.insert
 
+// -----
+
 func.func @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(%arg0: vector<1x[16]xf32>, %arg1: f32, %idx: index)
                                         -> vector<1x[16]xf32> {
   %0 = vector.insert %arg1, %arg0[0, %idx]: f32 into vector<1x[16]xf32>
@@ -1506,6 +1706,12 @@ func.func @insert_scalar_into_vec_2d_f32_dynamic_idx_scalable(%arg0: vector<1x[1
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.type_cast
+//
+// TODO: Add tests for for vector.type_cast that would cover scalable vectors
+//===----------------------------------------------------------------------===//
+
 func.func @type_cast_f32(%arg0: memref<8x8x8xf32>) -> memref<vector<8x8x8xf32>> {
   %0 = vector.type_cast %arg0: memref<8x8x8xf32> to memref<vector<8x8x8xf32>>
   return %0 : memref<vector<8x8x8xf32>>
@@ -1537,11 +1743,11 @@ func.func @type_cast_index(%arg0: memref<8x8x8xindex>) -> memref<vector<8x8x8xin
 
 // -----
 
-func.func @vector_type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> memref<vector<8x8x8xf32>, 3> {
+func.func @type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> memref<vector<8x8x8xf32>, 3> {
   %0 = vector.type_cast %arg0: memref<8x8x8xf32, 3> to memref<vector<8x8x8xf32>, 3>
   return %0 : memref<vector<8x8x8xf32>, 3>
 }
-// CHECK-LABEL: @vector_type_cast_non_zero_addrspace
+// CHECK-LABEL: @type_cast_non_zero_addrspace
 //       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64)>
 //       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:   llvm.insertvalue %[[allocated]], {{.*}}[0] : !llvm.struct<(ptr<3>, ptr<3>, i64)>
@@ -1554,6 +1760,10 @@ func.func @vector_type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> m
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.print
+//===----------------------------------------------------------------------===//
+
 func.func @print_scalar_i1(%arg0: i1) {
   vector.print %arg0 : i1
   return
@@ -1740,6 +1950,10 @@ func.func @print_string() {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.extract_strided_slice
+//===----------------------------------------------------------------------===//
+
 func.func @extract_strided_slice_f32_1d_from_1d(%arg0: vector<4xf32>) -> vector<2xf32> {
   %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
   return %0 : vector<2xf32>
@@ -1783,6 +1997,8 @@ func.func @extract_strided_slice_f32_1d_from_2d(%arg0: vector<4x8xf32>) -> vecto
 //       CHECK:    %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : !llvm.array<2 x vector<8xf32>> to vector<2x8xf32>
 //       CHECK:    return %[[T5]]
 
+// -----
+
 func.func @extract_strided_slice_f32_1d_from_2d_scalable(%arg0: vector<4x[8]xf32>) -> vector<2x[8]xf32> {
   %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x[8]xf32> to vector<2x[8]xf32>
   return %0 : vector<2x[8]xf32>
@@ -1818,6 +2034,8 @@ func.func @extract_strided_slice_f32_2d_from_2d(%arg0: vector<4x8xf32>) -> vecto
 //       CHECK:    %[[VAL_12:.*]] = builtin.unrealized_conversion_cast %[[T7]] : !llvm.array<2 x vector<2xf32>> to vector<2x2xf32>
 //       CHECK:    return %[[VAL_12]] : vector<2x2xf32>
 
+// -----
+
 // NOTE: For scalable vectors, we can only extract "full" scalable dimensions
 // (e.g. [8] from [8], but not [4] from [8]).
 
@@ -1840,6 +2058,10 @@ func.func @extract_strided_slice_f32_2d_from_2d_scalable(%arg0: vector<4x[8]xf32
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.insert_strided_slice
+//===----------------------------------------------------------------------===//
+
 func.func @insert_strided_slice_f32_2d_into_3d(%b: vector<4x4xf32>, %c: vector<4x4x4xf32>) -> vector<4x4x4xf32> {
   %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xf32> into vector<4x4x4xf32>
   return %0 : vector<4x4x4xf32>
@@ -1848,6 +2070,8 @@ func.func @insert_strided_slice_f32_2d_into_3d(%b: vector<4x4xf32>, %c: vector<4
 //       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xf32>>>
 //       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xf32>>>
 
+// -----
+
 func.func @insert_strided_slice_f32_2d_into_3d_scalable(%b: vector<4x[4]xf32>, %c: vector<4x4x[4]xf32>) -> vector<4x4x[4]xf32> {
   %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xf32> into vector<4x4x[4]xf32>
   return %0 : vector<4x4x[4]xf32>
@@ -1866,6 +2090,8 @@ func.func @insert_strided_index_slice_index_2d_into_3d(%b: vector<4x4xindex>, %c
 //       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xi64>>>
 //       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vector<4xi64>>>
 
+// -----
+
 func.func @insert_strided_index_slice_index_2d_into_3d_scalable(%b: vector<4x[4]xindex>, %c: vector<4x4x[4]xindex>) -> vector<4x4x[4]xindex> {
   %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x[4]xindex> into vector<4x4x[4]xindex>
   return %0 : vector<4x4x[4]xindex>
@@ -1899,6 +2125,8 @@ func.func @insert_strided_slice_f32_2d_into_2d(%a: vector<2x2xf32>, %b: vector<4
 //       CHECK:    %[[R4_3:.*]] = llvm.shufflevector %[[R4_2]], %[[V4_3]] [4, 5, 0, 1] : vector<4xf32>
 //       CHECK:    llvm.insertvalue %[[R4_3]], {{.*}}[3] : !llvm.array<4 x vector<4xf32>>
 
+// -----
+
 // NOTE: For scalable dimensions, the corresponding "base" size must match
 // (i.e. we can only insert "full" scalable dimensions, e.g. [2] into [2], but
 // not [2] from [4]).
@@ -1938,6 +2166,8 @@ func.func @insert_strided_slice_f32_2d_into_3d(%arg0: vector<2x4xf32>, %arg1: ve
 //       CHECK:    %[[R8_3:.*]] = llvm.shufflevector %[[R8_2]], %[[V4_0_1]] [8, 9, 0, 1, 2, 3, 14, 15] : vector<8xf32>
 //       CHECK:    llvm.insertvalue %[[R8_3]], {{.*}}[1] : !llvm.array<4 x vector<8xf32>>
 
+// -----
+
 // NOTE: For scalable dimensions, the corresponding "base" size must match
 // (i.e. we can only insert "full" scalable dimensions, e.g. [4] into [4], but
 // not [4] from [8]).
@@ -1966,74 +2196,84 @@ func.func @insert_strided_slice_f32_2d_into_3d_scalable(%arg0: vector<2x[4]xf32>
 
 // -----
 
-func.func @vector_fma(%a: vector<8xf32>, %b: vector<2x4xf32>, %c: vector<1x1x1xf32>, %d: vector<f32>) -> (vector<8xf32>, vector<2x4xf32>, vector<1x1x1xf32>, vector<f32>) {
-  // CHECK-LABEL: @vector_fma
-  //  CHECK-SAME: %[[A:.*]]: vector<8xf32>
-  //  CHECK-SAME: %[[B:.*]]: vector<2x4xf32>
-  //  CHECK-SAME: %[[C:.*]]: vector<1x1x1xf32>
-  //       CHECK: %[[BL:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x4xf32> to !llvm.array<2 x vector<4xf32>>
+//===----------------------------------------------------------------------===//
+// vector.fma
+//===----------------------------------------------------------------------===//
+
+func.func @fma(%vec_1d: vector<8xf32>, %vec_2d: vector<2x4xf32>, %vec_3d: vector<1x1x1xf32>, %vec_0d: vector<f32>) -> (vector<8xf32>, vector<2x4xf32>, vector<1x1x1xf32>, vector<f32>) {
+  // CHECK-LABEL: @fma
+  //  CHECK-SAME: %[[VEC_1D:.*]]: vector<8xf32>
+  //  CHECK-SAME: %[[VEC_2D:.*]]: vector<2x4xf32>
+  //  CHECK-SAME: %[[VEC_3D:.*]]: vector<1x1x1xf32>
+  //       CHECK: %[[VEC_2D_CAST:.*]] = builtin.unrealized_conversion_cast %[[VEC_2D]] : vector<2x4xf32> to !llvm.array<2 x vector<4xf32>>
   //       CHECK: llvm.intr.fmuladd
   //  CHECK-SAME:   (vector<8xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  %0 = vector.fma %a, %a, %a : vector<8xf32>
+  %0 = vector.fma %vec_1d, %vec_1d, %vec_1d : vector<8xf32>
 
-  //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[B0:.*]] = llvm.intr.fmuladd(%[[b00]], %[[b01]], %[[b02]]) :
+  //       CHECK: %[[VEC_2D_00:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_01:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_02:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_ADD_1:.*]] = llvm.intr.fmuladd(%[[VEC_2D_00]], %[[VEC_2D_01]], %[[VEC_2D_02]]) :
   //  CHECK-SAME: (vector<4xf32>, vector<4xf32>, vector<4xf32>) -> vector<4xf32>
-  //       CHECK: llvm.insertvalue %[[B0]], {{.*}}[0] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[b10:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[b11:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[b12:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<4xf32>>
-  //       CHECK: %[[B1:.*]] = llvm.intr.fmuladd(%[[b10]], %[[b11]], %[[b12]]) :
+  //       CHECK: llvm.insertvalue %[[VEC_2D_ADD_1]], {{.*}}[0] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_10:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_11:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_12:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<4xf32>>
+  //       CHECK: %[[VEC_2D_ADD_2:.*]] = llvm.intr.fmuladd(%[[VEC_2D_10]], %[[VEC_2D_11]], %[[VEC_2D_12]]) :
   //  CHECK-SAME: (vector<4xf32>, vector<4xf32>, vector<4xf32>) -> vector<4xf32>
-  //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm.array<2 x vector<4xf32>>
-  %1 = vector.fma %b, %b, %b : vector<2x4xf32>
+  //       CHECK: llvm.insertvalue %[[VEC_2D_ADD_2]], {{.*}}[1] : !llvm.array<2 x vector<4xf32>>
+  %1 = vector.fma %vec_2d, %vec_2d, %vec_2d : vector<2x4xf32>
 
   //       CHECK: %[[C0:.*]] = llvm.intr.fmuladd
   //  CHECK-SAME:   (vector<1xf32>, vector<1xf32>, vector<1xf32>) -> vector<1xf32>
-  %2 = vector.fma %c, %c, %c : vector<1x1x1xf32>
+  %2 = vector.fma %vec_3d, %vec_3d, %vec_3d : vector<1x1x1xf32>
 
   //       CHECK: %[[D0:.*]] = llvm.intr.fmuladd
   //  CHECK-SAME:   (vector<1xf32>, vector<1xf32>, vector<1xf32>) -> vector<1xf32>
-  %3 = vector.fma %d, %d, %d : vector<f32>
+  %3 = vector.fma %vec_0d, %vec_0d, %vec_0d : vector<f32>
 
   return %0, %1, %2, %3: vector<8xf32>, vector<2x4xf32>, vector<1x1x1xf32>, vector<f32>
 }
 
-func.func @vector_fma_scalable(%a: vector<[8]xf32>, %b: vector<2x[4]xf32>, %c: vector<1x1x[1]xf32>, %d: vector<f32>) -> (vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>) {
-  // CHECK-LABEL: @vector_fma_scalable
-  //  CHECK-SAME: %[[A:.*]]: vector<[8]xf32>
-  //  CHECK-SAME: %[[B:.*]]: vector<2x[4]xf32>
-  //  CHECK-SAME: %[[C:.*]]: vector<1x1x[1]xf32>
-  //       CHECK: %[[BL:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x[4]xf32> to !llvm.array<2 x vector<[4]xf32>>
+// -----
+
+func.func @fma_scalable(%vec_1d: vector<[8]xf32>, %vec_2d: vector<2x[4]xf32>, %vec_3d: vector<1x1x[1]xf32>, %vec_0d: vector<f32>) -> (vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>) {
+  // CHECK-LABEL: @fma_scalable
+  //  CHECK-SAME: %[[VEC_1D:.*]]: vector<[8]xf32>
+  //  CHECK-SAME: %[[VEC_2D:.*]]: vector<2x[4]xf32>
+  //  CHECK-SAME: %[[VEC_3D:.*]]: vector<1x1x[1]xf32>
+  //       CHECK: %[[VEC_2D_CAST:.*]] = builtin.unrealized_conversion_cast %[[VEC_2D]] : vector<2x[4]xf32> to !llvm.array<2 x vector<[4]xf32>>
   //       CHECK: llvm.intr.fmuladd
   //  CHECK-SAME:   (vector<[8]xf32>, vector<[8]xf32>, vector<[8]xf32>) -> vector<[8]xf32>
-  %0 = vector.fma %a, %a, %a : vector<[8]xf32>
+  %0 = vector.fma %vec_1d, %vec_1d, %vec_1d : vector<[8]xf32>
 
-  //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[B0:.*]] = llvm.intr.fmuladd(%[[b00]], %[[b01]], %[[b02]]) :
+  //       CHECK: %[[VEC_2D_00:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_01:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_02:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_ADD_1:.*]] = llvm.intr.fmuladd(%[[VEC_2D_00]], %[[VEC_2D_01]], %[[VEC_2D_02]]) :
   //  CHECK-SAME: (vector<[4]xf32>, vector<[4]xf32>, vector<[4]xf32>) -> vector<[4]xf32>
-  //       CHECK: llvm.insertvalue %[[B0]], {{.*}}[0] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[b10:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[b11:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[b12:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
-  //       CHECK: %[[B1:.*]] = llvm.intr.fmuladd(%[[b10]], %[[b11]], %[[b12]]) :
+  //       CHECK: llvm.insertvalue %[[VEC_2D_ADD_1]], {{.*}}[0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_10:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_11:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_12:.*]] = llvm.extractvalue %[[VEC_2D_CAST]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[VEC_2D_ADD_2:.*]] = llvm.intr.fmuladd(%[[VEC_2D_10]], %[[VEC_2D_11]], %[[VEC_2D_12]]) :
   //  CHECK-SAME: (vector<[4]xf32>, vector<[4]xf32>, vector<[4]xf32>) -> vector<[4]xf32>
-  //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm.array<2 x vector<[4]xf32>>
-  %1 = vector.fma %b, %b, %b : vector<2x[4]xf32>
+  //       CHECK: llvm.insertvalue %[[VEC_2D_ADD_2]], {{.*}}[1] : !llvm.array<2 x vector<[4]xf32>>
+  %1 = vector.fma %vec_2d, %vec_2d, %vec_2d : vector<2x[4]xf32>
 
   //       CHECK: %[[C0:.*]] = llvm.intr.fmuladd
   //  CHECK-SAME:   (vector<[1]xf32>, vector<[1]xf32>, vector<[1]xf32>) -> vector<[1]xf32>
-  %2 = vector.fma %c, %c, %c : vector<1x1x[1]xf32>
+  %2 = vector.fma %vec_3d, %vec_3d, %vec_3d : vector<1x1x[1]xf32>
 
   return %0, %1, %2: vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>
 }
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.reduction
+//===----------------------------------------------------------------------===//
+
 func.func @reduce_0d_f32(%arg0: vector<f32>) -> f32 {
   %0 = vector.reduction <add>, %arg0 : vector<f32> into f32
   return %0 : f32
@@ -2059,6 +2299,8 @@ func.func @reduce_f16(%arg0: vector<16xf16>) -> f16 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<16xf16>) -> f16
 //      CHECK: return %[[V]] : f16
 
+// -----
+
 func.func @reduce_f16_scalable(%arg0: vector<[16]xf16>) -> f16 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xf16> into f16
   return %0 : f16
@@ -2083,6 +2325,8 @@ func.func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<16xf32>) -> f32
 //      CHECK: return %[[V]] : f32
 
+// -----
+
 func.func @reduce_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xf32> into f32
   return %0 : f32
@@ -2107,6 +2351,8 @@ func.func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<16xf64>) -> f64
 //      CHECK: return %[[V]] : f64
 
+// -----
+
 func.func @reduce_f64_scalable(%arg0: vector<[16]xf64>) -> f64 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xf64> into f64
   return %0 : f64
@@ -2129,6 +2375,8 @@ func.func @reduce_i8(%arg0: vector<16xi8>) -> i8 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i8
 
+// -----
+
 func.func @reduce_i8_scalable(%arg0: vector<[16]xi8>) -> i8 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xi8> into i8
   return %0 : i8
@@ -2149,6 +2397,8 @@ func.func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2170,6 +2420,8 @@ func.func @reduce_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <add>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2191,6 +2443,8 @@ func.func @reduce_mul_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_mul_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <mul>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2212,6 +2466,8 @@ func.func @reduce_mul_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_mul_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <mul>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2234,6 +2490,8 @@ func.func @reduce_fmaximum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+// -----
+
 func.func @reduce_fmaximum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
   %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<[16]xf32> into f32
   return %0 : f32
@@ -2256,6 +2514,8 @@ func.func @reduce_fminimum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+// -----
+
 func.func @reduce_fminimum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
   %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<[16]xf32> into f32
   return %0 : f32
@@ -2278,6 +2538,8 @@ func.func @reduce_fmax_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+// -----
+
 func.func @reduce_fmax_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
   %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
   return %0 : f32
@@ -2300,6 +2562,8 @@ func.func @reduce_fmin_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+// -----
+
 func.func @reduce_fmin_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
   %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
   return %0 : f32
@@ -2321,6 +2585,8 @@ func.func @reduce_minui_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_minui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <minui>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2343,6 +2609,8 @@ func.func @reduce_minui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_minui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <minui>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2365,6 +2633,8 @@ func.func @reduce_maxui_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_maxui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <maxui>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2387,6 +2657,8 @@ func.func @reduce_maxui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_maxui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2409,6 +2681,8 @@ func.func @reduce_minsi_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_minsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <minsi>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2431,6 +2705,8 @@ func.func @reduce_minsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_minsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2453,6 +2729,8 @@ func.func @reduce_maxsi_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_maxsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <maxsi>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2475,6 +2753,8 @@ func.func @reduce_maxsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_maxsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2497,6 +2777,8 @@ func.func @reduce_and_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_and_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <and>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2518,6 +2800,8 @@ func.func @reduce_and_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_and_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <and>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2539,6 +2823,8 @@ func.func @reduce_or_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_or_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <or>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2560,6 +2846,8 @@ func.func @reduce_or_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_or_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <or>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2581,6 +2869,8 @@ func.func @reduce_xor_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_xor_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
   %0 = vector.reduction <xor>, %arg0 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2602,6 +2892,8 @@ func.func @reduce_xor_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+// -----
+
 func.func @reduce_xor_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
   %0 = vector.reduction <xor>, %arg0, %arg1 : vector<[16]xi32> into i32
   return %0 : i32
@@ -2623,6 +2915,8 @@ func.func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i64
 
+// -----
+
 func.func @reduce_i64_scalable(%arg0: vector<[16]xi64>) -> i64 {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xi64> into i64
   return %0 : i64
@@ -2645,6 +2939,8 @@ func.func @reduce_index(%arg0: vector<16xindex>) -> index {
 //      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
 //      CHECK: return %[[T2]] : index
 
+// -----
+
 func.func @reduce_index_scalable(%arg0: vector<[16]xindex>) -> index {
   %0 = vector.reduction <add>, %arg0 : vector<[16]xindex> into index
   return %0 : index
@@ -2656,9 +2952,13 @@ func.func @reduce_index_scalable(%arg0: vector<[16]xindex>) -> index {
 //      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
 //      CHECK: return %[[T2]] : index
 
-//                          4x16                16x3               4x3
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.matrix_multiply
+//===----------------------------------------------------------------------===//
+
+//                          4x16                16x3               4x3
 func.func @matrix_ops(%A: vector<64xf64>, %B: vector<48xf64>) -> vector<12xf64> {
   %C = vector.matrix_multiply %A, %B
     { lhs_rows = 4: i32, lhs_columns = 16: i32 , rhs_columns = 3: i32 } :
@@ -2685,6 +2985,10 @@ func.func @matrix_ops_index(%A: vector<64xindex>, %B: vector<48xindex>) -> vecto
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.constant_mask
+//===----------------------------------------------------------------------===//
+
 func.func @constant_mask_0d_f() -> vector<i1> {
   %0 = vector.constant_mask [0] : vector<i1>
   return %0 : vector<i1>
@@ -2778,54 +3082,62 @@ func.func @negative_constant_mask_2d_leading_scalable() -> vector<[4]x4xi1> {
 
 // -----
 
-func.func @create_mask_0d(%a : index) -> vector<i1> {
-  %v = vector.create_mask %a : vector<i1>
+//===----------------------------------------------------------------------===//
+// vector.create_mask
+//===----------------------------------------------------------------------===//
+
+func.func @create_mask_0d(%num_elems : index) -> vector<i1> {
+  %v = vector.create_mask %num_elems : vector<i1>
   return %v: vector<i1>
 }
 
 // CHECK-LABEL: func @create_mask_0d
-// CHECK-SAME: %[[arg:.*]]: index
-// CHECK:  %[[indices:.*]] = arith.constant dense<0> : vector<i32>
-// CHECK:  %[[arg_i32:.*]] = arith.index_cast %[[arg]] : index to i32
-// CHECK:  %[[bounds:.*]] = llvm.insertelement %[[arg_i32]]
-// CHECK:  %[[boundsCast:.*]] = builtin.unrealized_conversion_cast %[[bounds]] : vector<1xi32> to vector<i32>
-// CHECK:  %[[result:.*]] = arith.cmpi slt, %[[indices]], %[[boundsCast]] : vector<i32>
-// CHECK:  return %[[result]] : vector<i1>
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = arith.constant dense<0> : vector<i32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
+// CHECK:  %[[BOUNDS_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOUNDS]] : vector<1xi32> to vector<i32>
+// CHECK:  %[[RESULT:.*]] = arith.cmpi slt, %[[INDICES]], %[[BOUNDS_CAST]] : vector<i32>
+// CHECK:  return %[[RESULT]] : vector<i1>
 
 // -----
 
-func.func @create_mask_1d(%a : index) -> vector<4xi1> {
-  %v = vector.create_mask %a : vector<4xi1>
+func.func @create_mask_1d(%num_elems : index) -> vector<4xi1> {
+  %v = vector.create_mask %num_elems : vector<4xi1>
   return %v: vector<4xi1>
 }
 
 // CHECK-LABEL: func @create_mask_1d
-// CHECK-SAME: %[[arg:.*]]: index
-// CHECK:  %[[indices:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xi32>
-// CHECK:  %[[arg_i32:.*]] = arith.index_cast %[[arg]] : index to i32
-// CHECK:  %[[boundsInsert:.*]] = llvm.insertelement %[[arg_i32]]
-// CHECK:  %[[bounds:.*]] = llvm.shufflevector %[[boundsInsert]]
-// CHECK:  %[[result:.*]] = arith.cmpi slt, %[[indices]], %[[bounds]] : vector<4xi32>
-// CHECK:  return %[[result]] : vector<4xi1>
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xi32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]]
+// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]]
+// CHECK:  %[[RESULT:.*]] = arith.cmpi slt, %[[INDICES]], %[[BOUNDS]] : vector<4xi32>
+// CHECK:  return %[[RESULT]] : vector<4xi1>
 
 // -----
 
-func.func @create_mask_1d_scalable(%a : index) -> vector<[4]xi1> {
-  %v = vector.create_mask %a : vector<[4]xi1>
+func.func @create_mask_1d_scalable(%num_elems : index) -> vector<[4]xi1> {
+  %v = vector.create_mask %num_elems : vector<[4]xi1>
   return %v: vector<[4]xi1>
 }
 
 // CHECK-LABEL: func @create_mask_1d_scalable
-// CHECK-SAME: %[[arg:.*]]: index
-// CHECK:  %[[indices:.*]] = llvm.intr.stepvector : vector<[4]xi32>
-// CHECK:  %[[arg_i32:.*]] = arith.index_cast %[[arg]] : index to i32
-// CHECK:  %[[boundsInsert:.*]] = llvm.insertelement %[[arg_i32]], {{.*}} : vector<[4]xi32>
-// CHECK:  %[[bounds:.*]] = llvm.shufflevector %[[boundsInsert]], {{.*}} : vector<[4]xi32>
-// CHECK:  %[[result:.*]] = arith.cmpi slt, %[[indices]], %[[bounds]] : vector<[4]xi32>
-// CHECK: return %[[result]] : vector<[4]xi1>
+// CHECK-SAME: %[[NUM_ELEMS:.*]]: index
+// CHECK:  %[[INDICES:.*]] = llvm.intr.stepvector : vector<[4]xi32>
+// CHECK:  %[[NUM_ELEMS_i32:.*]] = arith.index_cast %[[NUM_ELEMS]] : index to i32
+// CHECK:  %[[BOUNDS_INSERT:.*]] = llvm.insertelement %[[NUM_ELEMS_i32]], {{.*}} : vector<[4]xi32>
+// CHECK:  %[[BOUNDS:.*]] = llvm.shufflevector %[[BOUNDS_INSERT]], {{.*}} : vector<[4]xi32>
+// CHECK:  %[[RESULT:.*]] = arith.cmpi slt, %[[INDICES]], %[[BOUNDS]] : vector<[4]xi32>
+// CHECK: return %[[RESULT]] : vector<[4]xi1>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.transpose
+//===----------------------------------------------------------------------===//
+
 func.func @transpose_0d(%arg0: vector<f32>) -> vector<f32> {
   %0 = vector.transpose %arg0, [] : vector<f32> to vector<f32>
   return %0 : vector<f32>
@@ -2837,6 +3149,10 @@ func.func @transpose_0d(%arg0: vector<f32>) -> vector<f32> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.flat_transpose
+//===----------------------------------------------------------------------===//
+
 func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
      : vector<16xf32> -> vector<16xf32>
@@ -2868,195 +3184,221 @@ func.func @flat_transpose_index(%arg0: vector<16xindex>) -> vector<16xindex> {
 
 // -----
 
-func.func @vector_load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+func.func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
+  %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
+     : vector<16xf32> -> vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// CHECK-LABEL: func @flat_transpose
+// CHECK-SAME:  %[[A:.*]]: vector<16xf32>
+// CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
+// CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
+// CHECK-SAME:      vector<16xf32> into vector<16xf32>
+// CHECK:       return %[[T]] : vector<16xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.load
+//===----------------------------------------------------------------------===//
+
+func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
   %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
   return %0 : vector<8xf32>
 }
 
-// CHECK-LABEL: func @vector_load
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[gep]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+// CHECK-LABEL: func @load
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
 
 // -----
 
-func.func @vector_load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
   %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<[8]xf32>
   return %0 : vector<[8]xf32>
 }
 
-// CHECK-LABEL: func @vector_load_scalable
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[gep]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
+// CHECK-LABEL: func @load_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
 
 // -----
 
-func.func @vector_load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
   %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<8xf32>
   return %0 : vector<8xf32>
 }
 
-// CHECK-LABEL: func @vector_load_nontemporal
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[gep]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
+// CHECK-LABEL: func @load_nontemporal
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
 
 // -----
 
-func.func @vector_load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
   %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[8]xf32>
   return %0 : vector<[8]xf32>
 }
 
-// CHECK-LABEL: func @vector_load_nontemporal_scalable
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[gep]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
+// CHECK-LABEL: func @load_nontemporal_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
 
 // -----
 
-func.func @vector_load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
+func.func @load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
   %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<8xindex>
   return %0 : vector<8xindex>
 }
-// CHECK-LABEL: func @vector_load_index
+// CHECK-LABEL: func @load_index
 // CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xi64>
 // CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<8xi64> to vector<8xindex>
 // CHECK: return %[[T1]] : vector<8xindex>
 
 // -----
 
-func.func @vector_load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
+func.func @load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
   %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<[8]xindex>
   return %0 : vector<[8]xindex>
 }
-// CHECK-LABEL: func @vector_load_index_scalable
+// CHECK-LABEL: func @load_index_scalable
 // CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<[8]xi64>
 // CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<[8]xi64> to vector<[8]xindex>
 // CHECK: return %[[T1]] : vector<[8]xindex>
 
 // -----
 
-func.func @vector_load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
+func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
   %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<f32>
   return %0 : vector<f32>
 }
 
-// CHECK-LABEL: func @vector_load_0d
-// CHECK: %[[load:.*]] = memref.load %{{.*}}[%{{.*}}, %{{.*}}]
-// CHECK: %[[vec:.*]] = llvm.mlir.undef : vector<1xf32>
-// CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK: %[[inserted:.*]] = llvm.insertelement %[[load]], %[[vec]][%[[c0]] : i32] : vector<1xf32>
-// CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %[[inserted]] : vector<1xf32> to vector<f32>
-// CHECK: return %[[cast]] : vector<f32>
+// CHECK-LABEL: func @load_0d
+// CHECK: %[[LOAD:.*]] = memref.load %{{.*}}[%{{.*}}, %{{.*}}]
+// CHECK: %[[VEC:.*]] = llvm.mlir.undef : vector<1xf32>
+// CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK: %[[INSERTED:.*]] = llvm.insertelement %[[LOAD]], %[[VEC]][%[[C0]] : i32] : vector<1xf32>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[INSERTED]] : vector<1xf32> to vector<f32>
+// CHECK: return %[[CAST]] : vector<f32>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.store
+//===----------------------------------------------------------------------===//
 
-func.func @vector_store(%memref : memref<200x100xf32>, %i : index, %j : index) {
+func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
   %val = arith.constant dense<11.0> : vector<4xf32>
   vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<4xf32>
   return
 }
 
-// CHECK-LABEL: func @vector_store
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[gep]] {alignment = 4 : i64} :  vector<4xf32>, !llvm.ptr
+// CHECK-LABEL: func @store
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<4xf32>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
   %val = arith.constant dense<11.0> : vector<[4]xf32>
   vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<[4]xf32>
   return
 }
 
-// CHECK-LABEL: func @vector_store_scalable
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[gep]] {alignment = 4 : i64} :  vector<[4]xf32>, !llvm.ptr
+// CHECK-LABEL: func @store_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} :  vector<[4]xf32>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
+func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
   %val = arith.constant dense<11.0> : vector<4xf32>
   vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<4xf32>
   return
 }
 
-// CHECK-LABEL: func @vector_store_nontemporal
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[gep]] {alignment = 4 : i64, nontemporal} :  vector<4xf32>, !llvm.ptr
+// CHECK-LABEL: func @store_nontemporal
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<4xf32>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
   %val = arith.constant dense<11.0> : vector<[4]xf32>
   vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[4]xf32>
   return
 }
 
-// CHECK-LABEL: func @vector_store_nontemporal_scalable
-// CHECK: %[[c100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[mul:.*]] = llvm.mul %{{.*}}, %[[c100]]  : i64
-// CHECK: %[[add:.*]] = llvm.add %[[mul]], %{{.*}}  : i64
-// CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}}[%[[add]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[gep]] {alignment = 4 : i64, nontemporal} :  vector<[4]xf32>, !llvm.ptr
+// CHECK-LABEL: func @store_nontemporal_scalable
+// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]  : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}  : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} :  vector<[4]xf32>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
+func.func @store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
   %val = arith.constant dense<11> : vector<4xindex>
   vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<4xindex>
   return
 }
-// CHECK-LABEL: func @vector_store_index
+// CHECK-LABEL: func @store_index
 // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<4xi64>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
+func.func @store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
   %val = arith.constant dense<11> : vector<[4]xindex>
   vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<[4]xindex>
   return
 }
-// CHECK-LABEL: func @vector_store_index_scalable
+// CHECK-LABEL: func @store_index_scalable
 // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[4]xi64>, !llvm.ptr
 
 // -----
 
-func.func @vector_store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
+func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
   %val = arith.constant dense<11.0> : vector<f32>
   vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<f32>
   return
 }
 
-// CHECK-LABEL: func @vector_store_0d
-// CHECK: %[[val:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
-// CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %[[val]] : vector<f32> to vector<1xf32>
-// CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64
-// CHECK: %[[extracted:.*]] = llvm.extractelement %[[cast]][%[[c0]] : i64] : vector<1xf32>
-// CHECK: memref.store %[[extracted]], %{{.*}}[%{{.*}}, %{{.*}}]
+// CHECK-LABEL: func @store_0d
+// CHECK: %[[VAL:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[VAL]] : vector<f32> to vector<1xf32>
+// CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
+// CHECK: %[[EXTRACTED:.*]] = llvm.extractelement %[[CAST]][%[[C0]] : i64] : vector<1xf32>
+// CHECK: memref.store %[[EXTRACTED]], %{{.*}}[%{{.*}}, %{{.*}}]
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.maskedload
+//===----------------------------------------------------------------------===//
+
 func.func @masked_load(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) -> vector<16xf32> {
   %c0 = arith.constant 0: index
   %0 = vector.maskedload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
@@ -3107,6 +3449,10 @@ func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.maskedstore
+//===----------------------------------------------------------------------===//
+
 func.func @masked_store(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>) {
   %c0 = arith.constant 0: index
   vector.maskedstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<16xi1>, vector<16xf32>
@@ -3155,6 +3501,10 @@ func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.gather
+//===----------------------------------------------------------------------===//
+
 func.func @gather(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
   %0 = arith.constant 0: index
   %1 = vector.gather %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32> into vector<3xf32>
@@ -3366,6 +3716,10 @@ func.func @gather_1d_from_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]x
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.scatter
+//===----------------------------------------------------------------------===//
+
 func.func @scatter(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) {
   %0 = arith.constant 0: index
   vector.scatter %arg0[%0][%arg1], %arg2, %arg3 : memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32>
@@ -3440,6 +3794,10 @@ func.func @scatter_1d_into_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.expandload
+//===----------------------------------------------------------------------===//
+
 func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) -> vector<11xf32> {
   %c0 = arith.constant 0: index
   %0 = vector.expandload %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32> into vector<11xf32>
@@ -3465,6 +3823,10 @@ func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %a
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.compressstore
+//===----------------------------------------------------------------------===//
+
 func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vector<11xf32>) {
   %c0 = arith.constant 0: index
   vector.compressstore %arg0[%c0], %arg1, %arg2 : memref<?xf32>, vector<11xi1>, vector<11xf32>
@@ -3489,40 +3851,65 @@ func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>,
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.splat
+//===----------------------------------------------------------------------===//
+
 // CHECK-LABEL: @splat_0d
-// CHECK-SAME: %[[ARG:.*]]: f32
-func.func @splat_0d(%a: f32) -> vector<f32> {
-  %v = vector.splat %a : vector<f32>
+// CHECK-SAME: %[[ELT:.*]]: f32
+func.func @splat_0d(%elt: f32) -> vector<f32> {
+  %v = vector.splat %elt : vector<f32>
   return %v : vector<f32>
 }
 // CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<1xf32>
 // CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
-// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ARG]], %[[UNDEF]][%[[ZERO]] : i32] : vector<1xf32>
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<1xf32>
 // CHECK-NEXT: %[[VCAST:[0-9]+]] = builtin.unrealized_conversion_cast %[[V]] : vector<1xf32> to vector<f32>
 // CHECK-NEXT: return %[[VCAST]] : vector<f32>
 
 // -----
 
 // CHECK-LABEL: @splat
-// CHECK-SAME: %[[A:arg[0-9]+]]: vector<4xf32>
-// CHECK-SAME: %[[ELT:arg[0-9]+]]: f32
-func.func @splat(%a: vector<4xf32>, %b: f32) -> vector<4xf32> {
-  %vb = vector.splat %b : vector<4xf32>
-  %r = arith.mulf %a, %vb : vector<4xf32>
+// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<4xf32>
+// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
+func.func @splat(%vec: vector<4xf32>, %elt: f32) -> vector<4xf32> {
+  %vb = vector.splat %elt : vector<4xf32>
+  %r = arith.mulf %vec, %vb : vector<4xf32>
   return %r : vector<4xf32>
 }
 // CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<4xf32>
 // CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
 // CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<4xf32>
 // CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
-// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[A]], %[[SPLAT]] : vector<4xf32>
+// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<4xf32>
 // CHECK-NEXT: return %[[SCALE]] : vector<4xf32>
 
 // -----
 
-// CHECK-LABEL: @vector_scalable_insert
+// CHECK-LABEL: @splat_scalable
+// CHECK-SAME: %[[VEC:[0-9a-zA-Z]+]]: vector<[4]xf32>
+// CHECK-SAME: %[[ELT:[0-9a-zA-Z]+]]: f32
+func.func @splat_scalable(%vec: vector<[4]xf32>, %elt: f32) -> vector<[4]xf32> {
+  %vb = vector.splat %elt : vector<[4]xf32>
+  %r = arith.mulf %vec, %vb : vector<[4]xf32>
+  return %r : vector<[4]xf32>
+}
+// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : vector<[4]xf32>
+// CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : i32] : vector<[4]xf32>
+// CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0, 0, 0, 0]
+// CHECK-NEXT: %[[SCALE:[0-9]+]] = arith.mulf %[[VEC]], %[[SPLAT]] : vector<[4]xf32>
+// CHECK-NEXT: return %[[SCALE]] : vector<[4]xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.scalable_insert
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @scalable_insert
 // CHECK-SAME: %[[SUB:.*]]: vector<4xf32>, %[[SV:.*]]: vector<[4]xf32>
-func.func @vector_scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) -> vector<[4]xf32> {
+func.func @scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) -> vector<[4]xf32> {
   // CHECK-NEXT: %[[TMP:.*]] = llvm.intr.vector.insert %[[SUB]], %[[SV]][0] : vector<4xf32> into vector<[4]xf32>
   %0 = vector.scalable.insert %sub, %dsv[0] : vector<4xf32> into vector<[4]xf32>
   // CHECK-NEXT: llvm.intr.vector.insert %[[SUB]], %[[TMP]][4] : vector<4xf32> into vector<[4]xf32>
@@ -3532,9 +3919,13 @@ func.func @vector_scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) ->
 
 // -----
 
-// CHECK-LABEL: @vector_scalable_extract
+//===----------------------------------------------------------------------===//
+// vector.scalable_extract
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @scalable_extract
 // CHECK-SAME: %[[VEC:.*]]: vector<[4]xf32>
-func.func @vector_scalable_extract(%vec: vector<[4]xf32>) -> vector<8xf32> {
+func.func @scalable_extract(%vec: vector<[4]xf32>) -> vector<8xf32> {
   // CHECK-NEXT: %{{.*}} = llvm.intr.vector.extract %[[VEC]][0] : vector<8xf32> from vector<[4]xf32>
   %0 = vector.scalable.extract %vec[0] : vector<8xf32> from vector<[4]xf32>
   return %0 : vector<8xf32>
@@ -3542,19 +3933,13 @@ func.func @vector_scalable_extract(%vec: vector<[4]xf32>) -> vector<8xf32> {
 
 // -----
 
-// CHECK-LABEL: @make_fixed_vector_of_scalable_vector
-func.func @make_fixed_vector_of_scalable_vector(%f : f64) -> vector<3x[2]xf64>
-{
-  // CHECK: %{{.*}} = llvm.mlir.undef : !llvm.array<3 x vector<[2]xf64>>
-  %res = vector.broadcast %f : f64 to vector<3x[2]xf64>
-  return %res : vector<3x[2]xf64>
-}
-
-// -----
+//===----------------------------------------------------------------------===//
+// vector.interleave
+//===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @vector_interleave_0d
+// CHECK-LABEL: @interleave_0d
 //  CHECK-SAME:     %[[LHS:.*]]: vector<i8>, %[[RHS:.*]]: vector<i8>)
-func.func @vector_interleave_0d(%a: vector<i8>, %b: vector<i8>) -> vector<2xi8> {
+func.func @interleave_0d(%a: vector<i8>, %b: vector<i8>) -> vector<2xi8> {
   // CHECK-DAG: %[[LHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[LHS]] : vector<i8> to vector<1xi8>
   // CHECK-DAG: %[[RHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[RHS]] : vector<i8> to vector<1xi8>
   // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS_RANK1]], %[[RHS_RANK1]] [0, 1] : vector<1xi8>
@@ -3565,9 +3950,9 @@ func.func @vector_interleave_0d(%a: vector<i8>, %b: vector<i8>) -> vector<2xi8>
 
 // -----
 
-// CHECK-LABEL: @vector_interleave_1d
+// CHECK-LABEL: @interleave_1d
 //  CHECK-SAME:     %[[LHS:.*]]: vector<8xf32>, %[[RHS:.*]]: vector<8xf32>)
-func.func @vector_interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<16xf32> {
+func.func @interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<16xf32> {
   // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS]], %[[RHS]] [0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>
   // CHECK: return %[[ZIP]]
   %0 = vector.interleave %a, %b : vector<8xf32> -> vector<16xf32>
@@ -3576,9 +3961,9 @@ func.func @vector_interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<
 
 // -----
 
-// CHECK-LABEL: @vector_interleave_1d_scalable
+// CHECK-LABEL: @interleave_1d_scalable
 //  CHECK-SAME:     %[[LHS:.*]]: vector<[4]xi32>, %[[RHS:.*]]: vector<[4]xi32>)
-func.func @vector_interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32>) -> vector<[8]xi32> {
+func.func @interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32>) -> vector<[8]xi32> {
   // CHECK: %[[ZIP:.*]] = "llvm.intr.vector.interleave2"(%[[LHS]], %[[RHS]]) : (vector<[4]xi32>, vector<[4]xi32>) -> vector<[8]xi32>
   // CHECK: return %[[ZIP]]
   %0 = vector.interleave %a, %b : vector<[4]xi32> -> vector<[8]xi32>
@@ -3587,9 +3972,9 @@ func.func @vector_interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32
 
 // -----
 
-// CHECK-LABEL: @vector_interleave_2d
+// CHECK-LABEL: @interleave_2d
 //  CHECK-SAME:     %[[LHS:.*]]: vector<2x3xi8>, %[[RHS:.*]]: vector<2x3xi8>)
-func.func @vector_interleave_2d(%a: vector<2x3xi8>, %b: vector<2x3xi8>) -> vector<2x6xi8> {
+func.func @interleave_2d(%a: vector<2x3xi8>, %b: vector<2x3xi8>) -> vector<2x6xi8> {
   // CHECK: llvm.shufflevector
   // CHECK-NOT: vector.interleave {{.*}} : vector<2x3xi8>
   %0 = vector.interleave %a, %b : vector<2x3xi8> -> vector<2x6xi8>
@@ -3598,9 +3983,9 @@ func.func @vector_interleave_2d(%a: vector<2x3xi8>, %b: vector<2x3xi8>) -> vecto
 
 // -----
 
-// CHECK-LABEL: @vector_interleave_2d_scalable
+// CHECK-LABEL: @interleave_2d_scalable
 //  CHECK-SAME:     %[[LHS:.*]]: vector<2x[8]xi16>, %[[RHS:.*]]: vector<2x[8]xi16>)
-func.func @vector_interleave_2d_scalable(%a: vector<2x[8]xi16>, %b: vector<2x[8]xi16>) -> vector<2x[16]xi16> {
+func.func @interleave_2d_scalable(%a: vector<2x[8]xi16>, %b: vector<2x[8]xi16>) -> vector<2x[16]xi16> {
   // CHECK: llvm.intr.vector.interleave2
   // CHECK-NOT: vector.interleave {{.*}} : vector<2x[8]xi16>
   %0 = vector.interleave %a, %b : vector<2x[8]xi16> -> vector<2x[16]xi16>
@@ -3609,100 +3994,108 @@ func.func @vector_interleave_2d_scalable(%a: vector<2x[8]xi16>, %b: vector<2x[8]
 
 // -----
 
-// CHECK-LABEL: @vector_deinterleave_1d
-// CHECK-SAME:  (%[[SRC:.*]]: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>)
-func.func @vector_deinterleave_1d(%a: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>) {
+//===----------------------------------------------------------------------===//
+// vector.deinterleave
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @deinterleave_1d
+// CHECK-SAME:  (%[[ARG:.*]]: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>)
+func.func @deinterleave_1d(%arg: vector<4xi32>) -> (vector<2xi32>, vector<2xi32>) {
   // CHECK: %[[POISON:.*]] = llvm.mlir.poison : vector<4xi32>
-  // CHECK: llvm.shufflevector %[[SRC]], %[[POISON]] [0, 2] : vector<4xi32>
-  // CHECK: llvm.shufflevector %[[SRC]], %[[POISON]] [1, 3] : vector<4xi32>
-  %0, %1 = vector.deinterleave %a : vector<4xi32> -> vector<2xi32>
+  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [0, 2] : vector<4xi32>
+  // CHECK: llvm.shufflevector %[[ARG]], %[[POISON]] [1, 3] : vector<4xi32>
+  %0, %1 = vector.deinterleave %arg : vector<4xi32> -> vector<2xi32>
   return %0, %1 : vector<2xi32>, vector<2xi32>
 }
 
-// CHECK-LABEL: @vector_deinterleave_1d_scalable
-// CHECK-SAME:  %[[SRC:.*]]: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>)
-func.func @vector_deinterleave_1d_scalable(%a: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>) {
-    // CHECK: %[[RES:.*]] = "llvm.intr.vector.deinterleave2"(%[[SRC]]) : (vector<[4]xi32>) -> !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
+// -----
+
+// CHECK-LABEL: @deinterleave_1d_scalable
+// CHECK-SAME:  %[[ARG:.*]]: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>)
+func.func @deinterleave_1d_scalable(%arg: vector<[4]xi32>) -> (vector<[2]xi32>, vector<[2]xi32>) {
+    // CHECK: %[[RES:.*]] = "llvm.intr.vector.deinterleave2"(%[[ARG]]) : (vector<[4]xi32>) -> !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
     // CHECK: llvm.extractvalue %[[RES]][0] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
     // CHECK: llvm.extractvalue %[[RES]][1] : !llvm.struct<(vector<[2]xi32>, vector<[2]xi32>)>
-    %0, %1 = vector.deinterleave %a : vector<[4]xi32> -> vector<[2]xi32>
+    %0, %1 = vector.deinterleave %arg : vector<[4]xi32> -> vector<[2]xi32>
     return %0, %1 : vector<[2]xi32>, vector<[2]xi32>
 }
 
-// CHECK-LABEL: @vector_deinterleave_2d
-// CHECK-SAME: %[[SRC:.*]]: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>)
-func.func @vector_deinterleave_2d(%a: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) {
+// -----
+
+// CHECK-LABEL: @deinterleave_2d
+// CHECK-SAME: %[[ARG:.*]]: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>)
+func.func @deinterleave_2d(%arg: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) {
   // CHECK: llvm.shufflevector
   // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x8xf32>
-  %0, %1 = vector.deinterleave %a : vector<2x8xf32> -> vector<2x4xf32>
+  %0, %1 = vector.deinterleave %arg : vector<2x8xf32> -> vector<2x4xf32>
   return %0, %1 : vector<2x4xf32>, vector<2x4xf32>
 }
 
-func.func @vector_deinterleave_2d_scalable(%a: vector<2x[8]xf32>) -> (vector<2x[4]xf32>, vector<2x[4]xf32>) {
+// -----
+
+func.func @deinterleave_2d_scalable(%arg: vector<2x[8]xf32>) -> (vector<2x[4]xf32>, vector<2x[4]xf32>) {
     // CHECK: llvm.intr.vector.deinterleave2
     // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x[8]xf32>
-    %0, %1 = vector.deinterleave %a : vector<2x[8]xf32> -> vector<2x[4]xf32>
+    %0, %1 = vector.deinterleave %arg : vector<2x[8]xf32> -> vector<2x[4]xf32>
     return %0, %1 : vector<2x[4]xf32>, vector<2x[4]xf32>
 }
 
 // -----
 
-// CHECK-LABEL: func.func @vector_bitcast_2d
-// CHECK:         llvm.bitcast
-// CHECK-NOT:     vector.bitcast
-func.func @vector_bitcast_2d(%arg0: vector<2x4xi32>) -> vector<2x2xi64> {
-  %0 = vector.bitcast %arg0 : vector<2x4xi32> to vector<2x2xi64>
-  return %0 : vector<2x2xi64>
+//===----------------------------------------------------------------------===//
+// vector.from_elements
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @from_elements_1d(
+//  CHECK-SAME:     %[[ARG_0:.*]]: f32, %[[ARG_1:.*]]: f32)
+//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.undef : vector<3xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<3xf32>
+//       CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64
+//       CHECK:   %[[INSERT1:.*]] = llvm.insertelement %[[ARG_1]], %[[INSERT0]][%[[C1]] : i64] : vector<3xf32>
+//       CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : i64) : i64
+//       CHECK:   %[[INSERT2:.*]] = llvm.insertelement %[[ARG_0]], %[[INSERT1]][%[[C2]] : i64] : vector<3xf32>
+//       CHECK:   return %[[INSERT2]]
+func.func @from_elements_1d(%arg0: f32, %arg1: f32) -> vector<3xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg0 : vector<3xf32>
+  return %0 : vector<3xf32>
 }
 
 // -----
 
-// CHECK-LABEL: func.func @vector_from_elements_1d(
-//  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
-//       CHECK:   %[[undef:.*]] = llvm.mlir.undef : vector<3xf32>
-//       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   %[[insert0:.*]] = llvm.insertelement %[[a]], %[[undef]][%[[c0]] : i64] : vector<3xf32>
-//       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : i64) : i64
-//       CHECK:   %[[insert1:.*]] = llvm.insertelement %[[b]], %[[insert0]][%[[c1]] : i64] : vector<3xf32>
-//       CHECK:   %[[c2:.*]] = llvm.mlir.constant(2 : i64) : i64
-//       CHECK:   %[[insert2:.*]] = llvm.insertelement %[[a]], %[[insert1]][%[[c2]] : i64] : vector<3xf32>
-//       CHECK:   return %[[insert2]]
-func.func @vector_from_elements_1d(%a: f32, %b: f32) -> vector<3xf32> {
-  %0 = vector.from_elements %a, %b, %a : vector<3xf32>
-  return %0 : vector<3xf32>
+// CHECK-LABEL: func.func @from_elements_0d(
+//  CHECK-SAME:     %[[ARG_0:.*]]: f32)
+//       CHECK:   %[[UNDEF:.*]] = llvm.mlir.undef : vector<1xf32>
+//       CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:   %[[INSERT0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF]][%[[C0]] : i64] : vector<1xf32>
+//       CHECK:   %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[INSERT0]] : vector<1xf32> to vector<f32>
+//       CHECK:   return %[[CAST]]
+func.func @from_elements_0d(%arg0: f32) -> vector<f32> {
+  %0 = vector.from_elements %arg0 : vector<f32>
+  return %0 : vector<f32>
 }
 
 // -----
 
-// CHECK-LABEL: func.func @vector_from_elements_0d(
-//  CHECK-SAME:     %[[a:.*]]: f32)
-//       CHECK:   %[[undef:.*]] = llvm.mlir.undef : vector<1xf32>
-//       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : i64) : i64
-//       CHECK:   %[[insert0:.*]] = llvm.insertelement %[[a]], %[[undef]][%[[c0]] : i64] : vector<1xf32>
-//       CHECK:   %[[cast:.*]] = builtin.unrealized_conversion_cast %[[insert0]] : vector<1xf32> to vector<f32>
-//       CHECK:   return %[[cast]]
-func.func @vector_from_elements_0d(%a: f32) -> vector<f32> {
-  %0 = vector.from_elements %a : vector<f32>
-  return %0 : vector<f32>
+//===----------------------------------------------------------------------===//
+// vector.step
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @step
+// CHECK: %[[CST:.+]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex>
+// CHECK: return %[[CST]] : vector<4xindex>
+func.func @step() -> vector<4xindex> {
+  %0 = vector.step : vector<4xindex>
+  return %0 : vector<4xindex>
 }
 
 // -----
 
-// CHECK-LABEL: @vector_step_scalable
+// CHECK-LABEL: @step_scalable
 // CHECK: %[[STEPVECTOR:.*]] = llvm.intr.stepvector : vector<[4]xi64>
 // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[STEPVECTOR]] : vector<[4]xi64> to vector<[4]xindex>
 // CHECK: return %[[CAST]] : vector<[4]xindex>
-func.func @vector_step_scalable() -> vector<[4]xindex> {
+func.func @step_scalable() -> vector<[4]xindex> {
   %0 = vector.step : vector<[4]xindex>
   return %0 : vector<[4]xindex>
 }
-
-// -----
-
-// CHECK-LABEL: @vector_step
-// CHECK: %[[CST:.+]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex>
-// CHECK: return %[[CST]] : vector<4xindex>
-func.func @vector_step() -> vector<4xindex> {
-  %0 = vector.step : vector<4xindex>
-  return %0 : vector<4xindex>
-}
diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
index 8796f153c4911..103148633bf97 100644
--- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
+++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
@@ -217,6 +217,35 @@ func.func @extract_dynamic_cst(%arg0 : vector<4xf32>) -> f32 {
 
 // -----
 
+// CHECK-LABEL: @from_elements_0d
+//  CHECK-SAME: %[[ARG0:.+]]: f32
+//       CHECK:   %[[RETVAL:.+]] = builtin.unrealized_conversion_cast %[[ARG0]]
+//       CHECK:   return %[[RETVAL]]
+func.func @from_elements_0d(%arg0 : f32) -> vector<f32> {
+  %0 = vector.from_elements %arg0 : vector<f32>
+  return %0: vector<f32>
+}
+
+// CHECK-LABEL: @from_elements_1x
+//  CHECK-SAME: %[[ARG0:.+]]: f32
+//       CHECK:   %[[RETVAL:.+]] = builtin.unrealized_conversion_cast %[[ARG0]]
+//       CHECK:   return %[[RETVAL]]
+func.func @from_elements_1x(%arg0 : f32) -> vector<1xf32> {
+  %0 = vector.from_elements %arg0 : vector<1xf32>
+  return %0: vector<1xf32>
+}
+
+// CHECK-LABEL: @from_elements_3x
+//  CHECK-SAME: %[[ARG0:.+]]: f32, %[[ARG1:.+]]: f32, %[[ARG2:.+]]: f32
+//       CHECK:   %[[RETVAL:.+]] = spirv.CompositeConstruct %[[ARG0]], %[[ARG1]], %[[ARG2]] : (f32, f32, f32) -> vector<3xf32>
+//       CHECK:   return %[[RETVAL]]
+func.func @from_elements_3x(%arg0 : f32, %arg1 : f32, %arg2 : f32) -> vector<3xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg2 : vector<3xf32>
+  return %0: vector<3xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @insert
 //  CHECK-SAME: %[[V:.*]]: vector<4xf32>, %[[S:.*]]: f32
 //       CHECK:   spirv.CompositeInsert %[[S]], %[[V]][2 : i32] : f32 into vector<4xf32>
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index d3f61f7e503f9..717004eb50c0f 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -2012,3 +2012,32 @@ func.func @affine_leading_zero_no_outer_bound(%arg0: index, %arg1: index) -> ind
   return %ret : index
 }
 
+// -----
+
+// CHECK-LABEL: @cst_value_to_cst_attr_basis_delinearize_index
+// CHECK-SAME:    (%[[ARG0:.*]]: index)
+// CHECK:         %[[RET:.*]]:3 = affine.delinearize_index %[[ARG0]] into (3, 4, 2) : index, index
+// CHECK:         return %[[RET]]#0, %[[RET]]#1, %[[RET]]#2 : index, index, index
+func.func @cst_value_to_cst_attr_basis_delinearize_index(%arg0 : index) ->
+    (index, index, index) {
+  %c4 = arith.constant 4 : index
+  %c3 = arith.constant 3 : index
+  %c2 = arith.constant 2 : index
+  %0:3 = affine.delinearize_index %arg0 into (%c3, %c4, %c2)
+      : index, index, index
+  return %0#0, %0#1, %0#2 : index, index, index
+}
+
+// -----
+
+// CHECK-LABEL: @cst_value_to_cst_attr_basis_linearize_index
+// CHECK-SAME:    (%[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index)
+// CHECK:         %[[RET:.*]] = affine.linearize_index disjoint [%[[ARG0]], %[[ARG1]], %[[ARG2]]] by (2, 3, 4) : index
+// CHECK:         return %[[RET]] : index
+func.func @cst_value_to_cst_attr_basis_linearize_index(%arg0 : index, %arg1 : index, %arg2 : index) ->
+    (index) {
+  %c4 = arith.constant 4 : index
+  %c2 = arith.constant 2 : index
+  %0 = affine.linearize_index disjoint [%arg0, %arg1, %arg2] by  (%c2, 3, %c4) : index
+  return %0 : index
+}
diff --git a/mlir/test/Dialect/Affine/ops.mlir b/mlir/test/Dialect/Affine/ops.mlir
index 1d1db5f58f54c..c6bfb688db1c1 100644
--- a/mlir/test/Dialect/Affine/ops.mlir
+++ b/mlir/test/Dialect/Affine/ops.mlir
@@ -298,3 +298,29 @@ func.func @linearize_mixed(%index0: index, %index1: index, %index2: index, %basi
   %1 = affine.linearize_index disjoint [%index0, %index1, %index2] by (2, %basis1, 3) : index
   return %1 : index
 }
+
+// -----
+
+#map = affine_map<()[s0] -> (s0)>
+
+// CHECK-LABEL: @gpu_affine_for
+
+module attributes {gpu.container_module} {
+  gpu.module @gpu {
+    gpu.func @gpu_affine_for(%arg0: memref<?x?xf32>) kernel {
+      %c3 = arith.constant 1 : index
+      %dim = memref.dim %arg0, %c3 : memref<?x?xf32>
+      %c0 = arith.constant 0 : index
+      affine.for %arg3 = %c0 to #map()[%dim] step 32 {
+      }
+      gpu.return
+    }
+  }
+}
+// CHECK-SAME:        (%[[VAL_0:.*]]: memref<?x?xf32>) kernel {
+// CHECK:             %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:             %[[VAL_2:.*]] = memref.dim %[[VAL_0]], %[[VAL_1]] : memref<?x?xf32>
+// CHECK:             %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:             affine.for %[[VAL_4:.*]] = %[[VAL_3]] to %[[VAL_2]] step 32 {
+// CHECK:             }
+// CHECK:             gpu.return
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index 1d4d5fc6f8319..6a186a0c6ceca 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -869,6 +869,27 @@ func.func @tripleAddAddOvf2(%arg0: index) -> index {
   return %add2 : index
 }
 
+
+// CHECK-LABEL: @foldSubXX_tensor
+//       CHECK:   %[[c0:.+]] = arith.constant dense<0> : tensor<10xi32> 
+//       CHECK:   %[[sub:.+]] = arith.subi
+//       CHECK:   return %[[c0]], %[[sub]]
+func.func @foldSubXX_tensor(%static : tensor<10xi32>, %dyn : tensor<?x?xi32>) -> (tensor<10xi32>, tensor<?x?xi32>) {
+  %static_sub = arith.subi %static, %static : tensor<10xi32>
+  %dyn_sub = arith.subi %dyn, %dyn : tensor<?x?xi32>
+  return %static_sub, %dyn_sub : tensor<10xi32>, tensor<?x?xi32>
+}
+
+// CHECK-LABEL: @foldSubXX_vector
+//       CHECK-DAG:  %[[c0:.+]] = arith.constant dense<0> : vector<8xi32>
+//       CHECK-DAG:  %[[c0_scalable:.+]] = arith.constant dense<0> : vector<[4]xi32>
+//       CHECK:   return %[[c0]], %[[c0_scalable]]
+func.func @foldSubXX_vector(%static : vector<8xi32>, %dyn : vector<[4]xi32>) -> (vector<8xi32>, vector<[4]xi32>) {
+  %static_sub = arith.subi %static, %static : vector<8xi32>
+  %dyn_sub = arith.subi %dyn, %dyn : vector<[4]xi32>
+  return %static_sub, %dyn_sub : vector<8xi32>, vector<[4]xi32>
+}
+
 // CHECK-LABEL: @tripleAddSub0
 //       CHECK:   %[[cres:.+]] = arith.constant 59 : index
 //       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 : index
@@ -1917,31 +1938,39 @@ func.func @test_maximumf(%arg0 : f32) -> (f32, f32, f32) {
 // -----
 
 // CHECK-LABEL: @test_minnumf(
-func.func @test_minnumf(%arg0 : f32) -> (f32, f32, f32) {
+func.func @test_minnumf(%arg0 : f32) -> (f32, f32, f32, f32) {
   // CHECK-DAG:   %[[C0:.+]] = arith.constant 0.0
+  // CHECK-DAG:   %[[INF:.+]] = arith.constant
   // CHECK-NEXT:  %[[X:.+]] = arith.minnumf %arg0, %[[C0]]
-  // CHECK-NEXT:  return %[[X]], %arg0, %arg0
+  // CHECK-NEXT:  %[[Y:.+]] = arith.minnumf %arg0, %[[INF]]
+  // CHECK-NEXT:   return %[[X]], %arg0, %[[Y]], %arg0
   %c0 = arith.constant 0.0 : f32
   %inf = arith.constant 0x7F800000 : f32
+  %nan = arith.constant 0x7FC00000 : f32
   %0 = arith.minnumf %c0, %arg0 : f32
   %1 = arith.minnumf %arg0, %arg0 : f32
   %2 = arith.minnumf %inf, %arg0 : f32
-  return %0, %1, %2 : f32, f32, f32
+  %3 = arith.minnumf %nan, %arg0 : f32
+  return %0, %1, %2, %3 : f32, f32, f32, f32
 }
 
 // -----
 
 // CHECK-LABEL: @test_maxnumf(
-func.func @test_maxnumf(%arg0 : f32) -> (f32, f32, f32) {
-  // CHECK-DAG:   %[[C0:.+]] = arith.constant
+func.func @test_maxnumf(%arg0 : f32) -> (f32, f32, f32, f32) {
+  // CHECK-DAG:   %[[C0:.+]] = arith.constant 0.0
+  // CHECK-DAG:   %[[NINF:.+]] = arith.constant
   // CHECK-NEXT:  %[[X:.+]] = arith.maxnumf %arg0, %[[C0]]
-  // CHECK-NEXT:   return %[[X]], %arg0, %arg0
+  // CHECK-NEXT:  %[[Y:.+]] = arith.maxnumf %arg0, %[[NINF]]
+  // CHECK-NEXT:   return %[[X]], %arg0, %[[Y]], %arg0
   %c0 = arith.constant 0.0 : f32
   %-inf = arith.constant 0xFF800000 : f32
+  %nan = arith.constant 0x7FC00000 : f32
   %0 = arith.maxnumf %c0, %arg0 : f32
   %1 = arith.maxnumf %arg0, %arg0 : f32
   %2 = arith.maxnumf %-inf, %arg0 : f32
-  return %0, %1, %2 : f32, f32, f32
+  %3 = arith.maxnumf %nan, %arg0 : f32
+  return %0, %1, %2, %3 : f32, f32, f32, f32
 }
 
 // -----
@@ -3201,26 +3230,3 @@ func.func @bf16_fma(%arg0: vector<32x32x32xbf16>, %arg1: vector<32x32x32xbf16>,
     }
   }
 #-}
-
-// -----
-
-// CHECK-LABEL: @test_fold_denorm
-// CHECK-SAME: %[[ARG0:.+]]: f32
-func.func @test_fold_denorm(%arg0: f32) -> f32 {
-  // CHECK-NOT: arith.subf
-  // CHECK: return %[[ARG0]] : f32
-  %c_denorm = arith.constant 1.4e-45 : f32
-  %sub = arith.subf %arg0, %c_denorm denormal<positive_zero> : f32
-  return %sub : f32
-}
-
-// -----
-
-// CHECK-LABEL: @test_expect_not_to_fold_denorm
-func.func @test_expect_not_to_fold_denorm(%arg0: f32, %arg1 : f32) -> (f32, f32) {
-  // CHECK-COUNT-2: arith.subf
-  %c_denorm = arith.constant 1.4e-45 : f32
-  %sub = arith.subf %arg0, %c_denorm denormal<ieee> : f32
-  %sub_1 = arith.subf %arg1, %c_denorm denormal<preserve_sign> : f32
-  return %sub, %sub_1 : f32, f32
-}
diff --git a/mlir/test/Dialect/Arith/ops.mlir b/mlir/test/Dialect/Arith/ops.mlir
index 5892e2a3d078c..f684e02344a51 100644
--- a/mlir/test/Dialect/Arith/ops.mlir
+++ b/mlir/test/Dialect/Arith/ops.mlir
@@ -1161,75 +1161,3 @@ func.func @intflags_func(%arg0: i64, %arg1: i64) {
   %3 = arith.shli %arg0, %arg1 overflow<nsw, nuw> : i64
   return
 }
-
-// CHECK-LABEL: check_denorm_modes(
-// CHECK-SAME: %[[ARG0:.+]]: f32, %[[ARG1:.+]]: f32, %[[ARG2:.+]]: f32)
-func.func @check_denorm_modes(%arg0: f32, %arg1: f32, %arg2: f32) {
-  // CHECK: %[[CST:.+]] = arith.constant 1.401300e-45 : f32
-  %c_denorm = arith.constant 1.4e-45 : f32
-  // CHECK: %{{.+}} = arith.subf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %sub_preserve_sign = arith.subf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.subf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %sub_positive_zero = arith.subf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.subf %[[ARG2]], %[[CST]] : f32
-  %sub_ieee = arith.subf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.addf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %add_preserve_sign = arith.addf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.addf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %add_positive_zero = arith.addf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.addf %[[ARG2]], %[[CST]] : f32
-  %add_ieee = arith.addf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.mulf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %mul_preserve_sign = arith.mulf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.mulf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %mul_positive_zero = arith.mulf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.mulf %[[ARG2]], %[[CST]] : f32
-  %mul_ieee = arith.mulf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.divf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %div_preserve_sign = arith.divf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.divf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %div_positive_zero = arith.divf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.divf %[[ARG2]], %[[CST]] : f32
-  %div_ieee = arith.divf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.maximumf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %maximumf_preserve_sign = arith.maximumf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.maximumf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %maximumf_positive_zero = arith.maximumf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.maximumf %[[ARG2]], %[[CST]] : f32
-  %maximumf_ieee = arith.maximumf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.maxnumf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %maxnumf_preserve_sign = arith.maxnumf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.maxnumf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %maxnumf_positive_zero = arith.maxnumf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.maxnumf %[[ARG2]], %[[CST]] : f32
-  %maxnumf_ieee = arith.maxnumf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.minimumf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %minimumf_preserve_sign = arith.minimumf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.minimumf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %minimumf_positive_zero = arith.minimumf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.minimumf %[[ARG2]], %[[CST]] : f32
-  %minimumf_ieee = arith.minimumf %arg2, %c_denorm denormal<ieee> : f32
-
-  // CHECK: %{{.+}} = arith.minnumf %{{.+}}, %{{.+}} denormal<preserve_sign> : f32
-  %minnumf_preserve_sign = arith.minnumf %arg0, %c_denorm denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.minnumf %{{.+}}, %{{.+}} denormal<positive_zero> : f32
-  %minnumf_positive_zero = arith.minnumf %arg1, %c_denorm denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.minnumf %[[ARG2]], %[[CST]] : f32
-  %minnumf_ieee = arith.minnumf %arg2, %c_denorm denormal<ieee> : f32
-
-
-  // CHECK: %{{.+}} = arith.negf %{{.+}} denormal<preserve_sign> : f32
-  %negf_preserve_sign = arith.negf %arg0 denormal<preserve_sign> : f32
-  // CHECK: %{{.+}} = arith.negf %{{.+}} denormal<positive_zero> : f32
-  %negf_positive_sign = arith.negf %arg0 denormal<positive_zero> : f32
-  // CHECK: %{{.+}} = arith.negf %[[ARG0]] : f32
-  %negf_ieee = arith.negf %arg0 denormal<ieee> : f32
-
-  return
-}
diff --git a/mlir/test/Dialect/ArmSME/vector-legalization.mlir b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
index 458906a187982..d56df9814f173 100644
--- a/mlir/test/Dialect/ArmSME/vector-legalization.mlir
+++ b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
@@ -646,3 +646,13 @@ func.func @negative_transpose_store_scalable_via_za__bad_source_shape(%vec: vect
   vector.transfer_write %tr, %dest[%i, %j] {in_bounds = [true, true]} : vector<[7]x2xf32>,  memref<?x?xf32>
   return
 }
+
+// -----
+
+// From: https://github.com/llvm/llvm-project/issues/118449.
+// Check -arm-sme-vector-legalization does not crash when it encounters a `vector.mask` that
+// does not contain a maskable op.
+func.func @vector_mask_without_maskable_op(%mask: vector<16x2xi1>, %vec: vector<16x16xf32>) -> vector<16x16xf32> {
+  %0 = vector.mask %mask { vector.yield %vec : vector<16x16xf32> } : vector<16x2xi1> -> vector<16x16xf32>
+  return %0 : vector<16x16xf32>
+}
diff --git a/mlir/test/Dialect/GPU/canonicalize.mlir b/mlir/test/Dialect/GPU/canonicalize.mlir
index d342ae9df10ee..33ce98e6da0ed 100644
--- a/mlir/test/Dialect/GPU/canonicalize.mlir
+++ b/mlir/test/Dialect/GPU/canonicalize.mlir
@@ -152,6 +152,17 @@ func.func @gpu_dim_of_alloc(%size: index) -> index {
 
 // -----
 
+// CHECK-LABEL: func @out_of_bound_memref.dim
+//  CHECK:   %[[MEMREF:.[a-z0-9A-Z_]+]] = memref.dim
+//  CHECK:   return %[[MEMREF]] : index
+func.func @out_of_bound_memref.dim(%arg : memref<?xi8>, %size: index) -> index {
+  %c2 = arith.constant 2 : index
+  %1 = memref.dim %arg, %c2 : memref<?xi8>
+  return %1 : index
+}
+
+// -----
+
 // CHECK-LABEL: func @simplify_gpu_launch
 func.func @simplify_gpu_launch() attributes {llvm.emit_c_interface} {
   %cst = arith.constant 0.000000e+00 : f32
diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir
index 6e682b26f6c95..d48fa054432d1 100644
--- a/mlir/test/Dialect/GPU/outlining.mlir
+++ b/mlir/test/Dialect/GPU/outlining.mlir
@@ -508,3 +508,125 @@ func.func @launch_cluster() {
 // CHECK-NEXT: "some_op"(%[[CID]], %[[BID]], %[[BDIM]]) : (index, index, index) -> ()
 // CHECK-NEXT: = memref.load %[[KERNEL_ARG1]][%[[TID]]] : memref<?xf32, 1>
 
+// -----
+// This test tests the two optional attributes kernelModule and kernelFunc for gpu.launch
+// CHECK-LABEL: func.func @testKernelAttributes()
+// CHECK: gpu.launch_func  @test_module::@test_kernel_func blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
+// CHECK: gpu.module @test_module
+// CHECK: gpu.func @test_kernel_func()
+func.func @testKernelAttributes() {
+  %gDimX = arith.constant 8 : index
+  %gDimY = arith.constant 12 : index
+  %gDimZ = arith.constant 16 : index
+  %bDimX = arith.constant 32 : index
+  %bDimY = arith.constant 16 : index
+  %bDimZ = arith.constant 8 : index
+
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+    "some_op"(%bx, %tx) : (index, index) -> ()
+    gpu.terminator
+  } {kernelModule = @test_module, kernelFunc = @test_kernel_func}
+  return
+}
+
+// -----
+// This test tests the two optional attributes kernelModule and kernelFunc for gpu.launch, when kernelModule already exists.
+
+// CHECK-LABEL: gpu.module @existing_module
+// CHECK: gpu.func @test_kernel_func()
+// CHECK: gpu.func @test_kernel_func_0()
+// CHECK-NOT: gpu.module @testExistingModule_kernel
+// CHECK-NOT: gpu.func @testExistingModule_kernel()
+// CHECK: func.func @testExistingModule()
+// CHECK: gpu.launch_func  @existing_module::@test_kernel_func_0 blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
+
+gpu.module @existing_module {
+  gpu.func @test_kernel_func() {
+    gpu.return
+  }
+}
+
+func.func @testExistingModule() {
+  %gDimX = arith.constant 8 : index
+  %gDimY = arith.constant 12 : index
+  %gDimZ = arith.constant 16 : index
+  %bDimX = arith.constant 32 : index
+  %bDimY = arith.constant 16 : index
+  %bDimZ = arith.constant 8 : index
+
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+    "some_op"(%bx, %tx) : (index, index) -> ()
+    gpu.terminator
+  } {kernelModule = @existing_module, kernelFunc = @test_kernel_func}
+  return
+}
+
+// -----
+// This test tests the optional attribute kernelModule for gpu.launch.
+// CHECK-LABEL: func.func @testKernelModuleOnly()
+// CHECK: gpu.launch_func  @test_module::@testKernelModuleOnly_kernel blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
+// CHECK: gpu.module @test_module
+// CHECK: gpu.func @testKernelModuleOnly_kernel()
+func.func @testKernelModuleOnly() {
+  %gDimX = arith.constant 8 : index
+  %gDimY = arith.constant 12 : index
+  %gDimZ = arith.constant 16 : index
+  %bDimX = arith.constant 32 : index
+  %bDimY = arith.constant 16 : index
+  %bDimZ = arith.constant 8 : index
+
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+    "some_op"(%bx, %tx) : (index, index) -> ()
+    gpu.terminator
+  } {kernelModule = @test_module}
+  return
+}
+
+// -----
+// This test tests the optional attribute kernelFunc for gpu.launch.
+// CHECK-LABEL: func.func @testKernelFuncOnly()
+// CHECK: gpu.launch_func  @test_kernel_func::@test_kernel_func blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
+
+// CHECK: gpu.module @test_kernel_func
+// CHECK: gpu.func @test_kernel_func()
+func.func @testKernelFuncOnly() {
+  %gDimX = arith.constant 8 : index
+  %gDimY = arith.constant 12 : index
+  %gDimZ = arith.constant 16 : index
+  %bDimX = arith.constant 32 : index
+  %bDimY = arith.constant 16 : index
+  %bDimZ = arith.constant 8 : index
+
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+    "some_op"(%bx, %tx) : (index, index) -> ()
+    gpu.terminator
+  } {kernelFunc = @test_kernel_func}
+  return
+}
+
+// -----
+// This test tests gpu.launch when optional attributes kernelModule and kernelFunc are not specified.
+// CHECK-LABEL: func.func @testNoAttributes()
+// CHECK: gpu.launch_func  @testNoAttributes_kernel::@testNoAttributes_kernel blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
+
+// CHECK: gpu.module @testNoAttributes_kernel
+// CHECK: gpu.func @testNoAttributes_kernel()
+func.func @testNoAttributes() {
+  %gDimX = arith.constant 8 : index
+  %gDimY = arith.constant 12 : index
+  %gDimZ = arith.constant 16 : index
+  %bDimX = arith.constant 32 : index
+  %bDimY = arith.constant 16 : index
+  %bDimZ = arith.constant 8 : index
+
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+    "some_op"(%bx, %tx) : (index, index) -> ()
+    gpu.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 5677d7ff41202..25806d9d0edd7 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -91,14 +91,14 @@ func.func @alloca_non_integer_alignment() {
 // -----
 
 func.func @gep_missing_input_result_type(%pos : i64, %base : !llvm.ptr) {
-  // expected-error@+1 {{2 operands present, but expected 0}}
+  // expected-error@+1 {{number of operands and types do not match: got 2 operands and 0 types}}
   llvm.getelementptr %base[%pos] : () -> (), i64
 }
 
 // -----
 
 func.func @gep_missing_input_type(%pos : i64, %base : !llvm.ptr) {
-  // expected-error@+1 {{2 operands present, but expected 0}}
+  // expected-error@+1 {{number of operands and types do not match: got 2 operands and 0 types}}
   llvm.getelementptr %base[%pos] : () -> (!llvm.ptr), i64
 }
 
diff --git a/mlir/test/Dialect/Linalg/decompose-tensor-pack.mlir b/mlir/test/Dialect/Linalg/decompose-tensor-pack.mlir
index 4f986606ef93a..1cc1484ed4095 100644
--- a/mlir/test/Dialect/Linalg/decompose-tensor-pack.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-tensor-pack.mlir
@@ -67,6 +67,9 @@ func.func @simple_pad_and_pack_dynamic_tile(%input: tensor<5x1xf32>, %output: te
 // CHECK:           %[[RES:.*]] = tensor.insert_slice %[[PAD:.*]] into %[[DEST]][0, 0, 0, 0] [1, 1, %[[TILE_DIM_0]], 2] [1, 1, 1, 1] : tensor<?x2xf32> into tensor<1x1x?x2xf32>
 // CHECK:           return %[[RES]] : tensor<1x1x?x2xf32>
 
+/// Same as example above, but the dynamic tile size is a compile-time constant
+/// that's folded away.
+
 func.func @simple_pad_and_pack_dynamic_tile_cst(%input: tensor<5x1xf32>, %output: tensor<1x1x?x2xf32>, %pad: f32) -> tensor<1x1x?x2xf32> {
   %tile_dim_0 = arith.constant 8 : index
   %0 = tensor.pack %input padding_value(%pad : f32) inner_dims_pos = [0, 1] inner_tiles = [%tile_dim_0, 2] into %output : tensor<5x1xf32> -> tensor<1x1x?x2xf32>
diff --git a/mlir/test/Dialect/Linalg/decompose-tensor-unpack.mlir b/mlir/test/Dialect/Linalg/decompose-tensor-unpack.mlir
index 8b15873473a97..a720c655e4be5 100644
--- a/mlir/test/Dialect/Linalg/decompose-tensor-unpack.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-tensor-unpack.mlir
@@ -19,11 +19,11 @@ func.func @simple_KCRSsr_to_KCRS(%arg0: tensor<1x1x1x1x8x32xf32>, %arg1: tensor<
 
 // -----
 
-func.func @simple_unpack_and_extract_slice(%input: tensor<1x1x8x2xf32>, %output: tensor<5x1xf32>) -> tensor<5x1xf32> {
+func.func @simple_unpack_static_tiles(%input: tensor<1x1x8x2xf32>, %output: tensor<5x1xf32>) -> tensor<5x1xf32> {
   %0 = tensor.unpack %input inner_dims_pos = [0, 1] inner_tiles = [8, 2] into %output : tensor<1x1x8x2xf32> -> tensor<5x1xf32>
   return %0 : tensor<5x1xf32>
 }
-// CHECK-LABEL: func.func @simple_unpack_and_extract_slice
+// CHECK-LABEL: func.func @simple_unpack_static_tiles
 // CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
 // CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0] [1, 1, 8, 2] [1, 1, 1, 1]
@@ -33,6 +33,55 @@ func.func @simple_unpack_and_extract_slice(%input: tensor<1x1x8x2xf32>, %output:
 // CHECK:         %[[SLICE:.+]] = tensor.extract_slice %[[TILE]][0, 0] [5, 1] [1, 1]
 // CHECK:         return %[[SLICE]]
 
+/// Same as example above, but with 1 dynamic tile size.
+
+func.func @simple_unpack_dynamic_tile(%input: tensor<1x1x?x2xf32>, %output: tensor<5x1xf32>, %tile_dim_0: index) -> tensor<5x1xf32> {
+  %0 = tensor.unpack %input inner_dims_pos = [0, 1] inner_tiles = [%tile_dim_0, 2] into %output : tensor<1x1x?x2xf32> -> tensor<5x1xf32>
+  return %0 : tensor<5x1xf32>
+}
+// CHECK-LABEL: func.func @simple_unpack_dynamic_tile
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[TILE_DIM_1:[a-zA-Z0-9]+]]
+// CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0] [1, 1, %[[TILE_DIM_1]], 2] [1, 1, 1, 1]
+// CHECK-NOT:     linalg.transpose
+//                They have the same type, so the insert_slice op is folded
+//                away.
+// CHECK:         %[[SLICE:.+]] = tensor.extract_slice %[[TILE]][0, 0] [5, 1] [1, 1]
+// CHECK:         return %[[SLICE]]
+
+/// Same as example above, but with 1 dynamic tile size and a trasnpose
+
+/// FIXME: This is currently broken:
+///   * 'tensor.empty' op incorrect number of dynamic sizes, has 0, expected 1
+
+//func.func @simple_unpack_dynamic_tile_transpose(%input: tensor<1x1x2x?xf32>, %output: tensor<5x1xf32>, %tile_dim_0: index) -> tensor<5x1xf32> {
+//  %0 = tensor.unpack %input inner_dims_pos = [1, 0] inner_tiles = [2, %tile_dim_0] into %output : tensor<1x1x2x?xf32> -> tensor<5x1xf32>
+//  return %0 : tensor<5x1xf32>
+//}
+
+/// Same as example above, but with 1 scalable tile size.
+
+func.func @simple_unpack_scalable_tile(%input: tensor<1x1x?x2xf32>, %output: tensor<5x1xf32>) -> tensor<5x1xf32> {
+  %c8 = arith.constant 8 : index
+  %vscale = vector.vscale
+  %c8_vscale = arith.muli %vscale, %c8 : index
+  %0 = tensor.unpack %input inner_dims_pos = [0, 1] inner_tiles = [%c8_vscale, 2] into %output : tensor<1x1x?x2xf32> -> tensor<5x1xf32>
+  return %0 : tensor<5x1xf32>
+}
+// CHECK-LABEL: func.func @simple_unpack_scalable_tile
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK-DAG:     %[[C8:.+]] = arith.constant 8 : index
+// CHECK-DAG:     %[[VS:.+]] = vector.vscale
+// CHECK:         %[[C8_VS:.+]] = arith.muli %[[VS]], %[[C8]] : index
+// CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0] [1, 1, %[[C8_VS]], 2] [1, 1, 1, 1]
+// CHECK-NOT:     linalg.transpose
+//                They have the same type, so the insert_slice op is folded
+//                away.
+// CHECK:         %[[SLICE:.+]] = tensor.extract_slice %[[TILE]][0, 0] [5, 1] [1, 1]
+// CHECK:         return %[[SLICE]]
+
 // -----
 
 func.func @simple_CNnc_to_NC(%arg0: tensor<1x1x32x8xf32>, %arg1: tensor<32x8xf32>) -> tensor<32x8xf32>{
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index e3b6958cfa881..a59472377a732 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -270,7 +270,7 @@ func.func @generic_result_tensor_type(%arg0: memref<?xf32, affine_map<(i)[off]->
 // -----
 
 func.func @generic(%arg0: memref<?x?xf32>) {
-  // expected-error @+6 {{block with no terminator, has %0 = "arith.addf"(%arg1, %arg1) <{denormal = #arith.denormal<ieee>, fastmath = #arith.fastmath<none>}> : (f32, f32) -> f32}}
+  // expected-error @+6 {{block with no terminator, has %0 = "arith.addf"(%arg1, %arg1) <{fastmath = #arith.fastmath<none>}> : (f32, f32) -> f32}}
   linalg.generic  {
     indexing_maps = [ affine_map<(i, j) -> (i, j)> ],
     iterator_types = ["parallel", "parallel"]}
diff --git a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
index 7aadf19069563..5f8ff36a16578 100644
--- a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
+++ b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
@@ -96,6 +96,34 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// This is same as pack_as_pad but since we explicitly added {lowerPadLikeWithInsertSlice = false}, it should not
+// be lowered to insert_slice.
+// CHECK-LABEL: func.func @pack_as_pad_disabled_insert_slice(
+func.func @pack_as_pad_disabled_insert_slice(%arg0: tensor<129x47x16x16xf32>, %arg1: tensor<1x1x1x1x136x64x16x16xf32>) -> tensor<1x1x1x1x136x64x16x16xf32> {
+  %cst_0 = arith.constant 0.0 : f32
+  // tensor.pack is lowered to tensor.pad + tensor.expand_shape + linalg.transpose
+  // CHECK-SAME: %[[ARG0:[^:]*]]: tensor<129x47x16x16xf32>
+  //  CHECK-DAG: %[[PAD:.*]] = tensor.pad %[[ARG0]]
+  //  CHECK-NOT: %[[RES:.*]] = tensor.insert_slice %[[PAD]]
+  //      CHECK: %[[PAD_EXPANDED:.*]] = tensor.expand_shape %[[PAD]]
+  //  CHECK-DAG: %[[RES:.*]] = linalg.transpose ins(%[[PAD_EXPANDED]]
+  %pack = tensor.pack %arg0 padding_value(%cst_0 : f32) inner_dims_pos = [0, 1, 2, 3] inner_tiles = [136, 64, 16, 16] into %arg1
+    : tensor<129x47x16x16xf32> -> tensor<1x1x1x1x136x64x16x16xf32>
+  return %pack :  tensor<1x1x1x1x136x64x16x16xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %pack = transform.structured.match ops{["tensor.pack"]} in %module_op
+      : (!transform.any_op) -> !transform.op<"tensor.pack">
+    transform.structured.lower_pack %pack {lowerPadLikeWithInsertSlice = false}: (!transform.op<"tensor.pack">)
+      -> (!transform.op<"tensor.pad">, !transform.op<"tensor.expand_shape">, !transform.op<"linalg.transpose">)
+      transform.yield
+  }
+}
+
+// -----
+
 // Check that we don't lower the following pack as a pad.
 // Although all the outer most dimensions in the resulting shape are 1s,
 // some of the original dimensions are not part of the inner_dims_pos, hence
@@ -233,6 +261,38 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// This is same as upack_as_pad but since we explicitly added {lowerUnpadLikeWithExtractSlice = false}, it should not
+// be lowered to extract_slice.
+// CHECK-LABEL: func.func @unpack_as_pad_disabled_extract_slice(
+func.func @unpack_as_pad_disabled_extract_slice(%arg0: tensor<1x1x1x1x136x64x16x16xf32>, %arg1: tensor<129x47x16x16xf32>) -> tensor<129x47x16x16xf32> {
+  %cst_0 = arith.constant 0.0 : f32
+
+  // tensor.unpack is lowered to tensor.extract_slice + linalg.transpose + tensor.collapse_shape
+  // CHECK-DAG: %[[ARG0:[^:]*]]: tensor<1x1x1x1x136x64x16x16xf32>
+  // CHECK-NOT: %[[RES:.*]] = tensor.extract_slice %[[ARG0]]
+  //     CHECK: %[[TRANSPOSED:.*]] = linalg.transpose ins(%[[ARG0]]
+  //     CHECK: %[[COLLAPSED:.*]] = tensor.collapse_shape %[[TRANSPOSED]]
+  // CHECK-DAG: %[[RES:.*]] = tensor.extract_slice %[[COLLAPSED]]
+  %pack = tensor.unpack %arg0 inner_dims_pos = [0, 1, 2, 3] inner_tiles = [136, 64, 16, 16] into %arg1
+    : tensor<1x1x1x1x136x64x16x16xf32> -> tensor<129x47x16x16xf32>
+  return %pack : tensor<129x47x16x16xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op
+      : (!transform.any_op) -> !transform.op<"tensor.unpack">
+    transform.structured.lower_unpack %unpack {lowerUnpadLikeWithExtractSlice = false}: (!transform.op<"tensor.unpack">)
+      -> (!transform.op<"tensor.empty">,
+          !transform.op<"linalg.transpose">,
+          !transform.op<"tensor.collapse_shape">,
+          !transform.op<"tensor.extract_slice">)
+          transform.yield
+  }
+}
+
+// -----
+
 // CHECK-LABEL: func.func @pack_with_outer_dims_perm(
 func.func @pack_with_outer_dims_perm(%src: tensor<100x200x128x256xi32>,
                                      %dest: tensor<200x4x16x100x16x32xi32>)
@@ -572,7 +632,7 @@ func.func @unpack_fully_dynamic(%source: tensor<?x?x?x?xf32>, %dest: tensor<?x?x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op
-      : (!transform.any_op) -> !transform.op<"tensor.unpack"> 
+      : (!transform.any_op) -> !transform.op<"tensor.unpack">
     transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">)
           -> (!transform.op<"tensor.empty">,
           !transform.op<"linalg.transpose">,
@@ -627,9 +687,9 @@ module attributes {transform.with_named_sequence} {
 // CHECK-LABEL: @unpack_with_outer_dims_perm
 //  CHECK-SAME: %[[ARG0:.*]]: tensor<32x64xf32>, %[[ARG1:.*]]: tensor<2x4x32x8xf32>
 //       CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<4x8x2x32xf32>
-//       CHECK: %[[TRAN:.*]] = linalg.transpose 
-//  CHECK-SAME:   ins(%[[ARG1]] : tensor<2x4x32x8xf32>) 
-//  CHECK-SAME:   outs(%[[EMPTY]] : tensor<4x8x2x32xf32>) 
+//       CHECK: %[[TRAN:.*]] = linalg.transpose
+//  CHECK-SAME:   ins(%[[ARG1]] : tensor<2x4x32x8xf32>)
+//  CHECK-SAME:   outs(%[[EMPTY]] : tensor<4x8x2x32xf32>)
 //  CHECK-SAME:   permutation = [1, 3, 0, 2]
 //       CHECK: %[[CLP:.*]] = tensor.collapse_shape %[[TRAN]] {{\[}}[0, 1], [2, 3]]
 //  CHECK-SAME:   : tensor<4x8x2x32xf32> into tensor<32x64xf32>
@@ -638,7 +698,7 @@ module attributes {transform.with_named_sequence} {
 //       CHECK: linalg.copy ins(%[[SLICE]]
 //  CHECK-SAME:   : tensor<32x64xf32>) outs(%[[ARG0]] : tensor<32x64xf32>) -> tensor<32x64xf32>
 func.func @unpack_with_outer_dims_perm(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> {
-  %unpack = tensor.unpack %arg1 outer_dims_perm = [1, 0] 
+  %unpack = tensor.unpack %arg1 outer_dims_perm = [1, 0]
     inner_dims_pos = [1, 0] inner_tiles = [32, 8] into %arg0 : tensor<2x4x32x8xf32> -> tensor<32x64xf32>
   return %unpack : tensor<32x64xf32>
 }
diff --git a/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir b/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
index fbebb97a11983..6584596cdfdb2 100644
--- a/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
+++ b/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
@@ -77,7 +77,7 @@ transform.sequence failures(propagate) {
 transform.sequence failures(propagate) {
 ^bb0(%arg0: !transform.any_op):
   %0 = transform.param.constant 2 : i64 -> !transform.param<i64>
-  // expected-error@below {{custom op 'transform.structured.vectorize' 1 operands present, but expected 2}}
+  // expected-error@+1 {{custom op 'transform.structured.vectorize' number of operands and types do not match: got 1 operands and 2 types}}
   transform.structured.vectorize %arg0 vector_sizes [%0, 2] : !transform.any_op, !transform.param<i64>, !transform.param<i64>
 
 }
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-fuse-pack-unpack.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-fuse-pack-unpack.mlir
new file mode 100644
index 0000000000000..faf7ff9ad7ed0
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-fuse-pack-unpack.mlir
@@ -0,0 +1,240 @@
+// RUN: mlir-opt %s --transform-interpreter --split-input-file -canonicalize | FileCheck %s
+
+// For pack op, we use lowerPadLikeWithInsertSlice = false to ensure no insert_slice is generated.
+// This allows linalg.transpose to be fused as a producer operation. In below testcase, linalg.transpose
+// as a producer operation is fused into the scf.forall loop.
+
+module {
+  // CHECK-label: func @fuse_pack_as_producer
+  // CHECK:       scf.forall {{.*}} {
+  // CHECK:         %[[PRODUCER:.*]] = linalg.transpose
+  // CHECK:         linalg.generic {{.*}} ins(%[[PRODUCER]]
+  // CHECK:         scf.forall.in_parallel
+  // CHECK:       }
+  func.func @fuse_pack_as_producer(%src: tensor<128x256xf32>, %other: tensor<4x4x128x256xf32>)
+      -> tensor<4x4x128x256xf32> {
+    %dest = tensor.empty() : tensor<1x1x128x256xf32>
+    %pack = tensor.pack %src inner_dims_pos = [0, 1] inner_tiles = [128, 256]
+        into %dest : tensor<128x256xf32> -> tensor<1x1x128x256xf32>
+
+    %out = tensor.empty() : tensor<4x4x128x256xf32>
+    %res = linalg.generic
+        {indexing_maps = [affine_map<(i, j, k, l) -> (0, 0, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>],
+         iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
+        ins(%pack, %other: tensor<1x1x128x256xf32>, tensor<4x4x128x256xf32>)
+        outs(%out: tensor<4x4x128x256xf32>) {
+      ^bb0(%pack_elem: f32, %other_elem: f32, %out_elem: f32):
+        %r = arith.addf %pack_elem, %other_elem : f32
+        linalg.yield %r : f32
+    } -> tensor<4x4x128x256xf32>
+
+    return %res : tensor<4x4x128x256xf32>
+  }
+
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+      // Find and lower pack operation.
+      %pack = transform.structured.match ops{["tensor.pack"]} in %arg1
+        : (!transform.any_op) -> !transform.op<"tensor.pack">
+      %paded, %expanded, %transpose = transform.structured.lower_pack %pack {lowerPadLikeWithInsertSlice = false}
+        : (!transform.op<"tensor.pack">)
+        -> (!transform.op<"tensor.pad">,
+            !transform.op<"tensor.expand_shape">,
+            !transform.op<"linalg.transpose">)
+
+      %root = transform.structured.match ops{["linalg.generic"]} in %arg1
+          : (!transform.any_op) -> !transform.any_op
+      // Tile the lialg operation with parallel forall loop tiling [4, 4].
+      %tiled_op, %forall_op = transform.structured.tile_using_forall %root num_threads [4, 4]
+          : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+      // Fuse the transpose operation into the tiled loop.
+      transform.structured.fuse_into_containing_op %transpose into %forall_op
+          : (!transform.op<"linalg.transpose">, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield
+    }
+  }
+}
+
+// -----
+// For pack op, by default lowerPadLikeWithInsertSlice = true, which generates insert_slice and blocks fusion.
+// In below testcase, tensor.insert_slice as a producer operation cannot be fused into the scf.forall loop.
+
+module {
+  // CHECK-label: func @fuse_pack_as_producer_blocked_by_insert_slice
+  // CHECK:       %[[PRODUCER:.*]] = tensor.insert_slice
+  // CHECK:       scf.forall {{.*}} {
+  // CHECK:         linalg.generic {{.*}} ins(%[[PRODUCER]]
+  // CHECK:         scf.forall.in_parallel
+  // CHECK:       }
+  func.func @fuse_pack_as_producer_blocked_by_insert_slice(%src: tensor<128x256xf32>, %other: tensor<4x4x128x256xf32>)
+      -> tensor<4x4x128x256xf32> {
+    %dest = tensor.empty() : tensor<1x1x128x256xf32>
+    %pack = tensor.pack %src inner_dims_pos = [0, 1] inner_tiles = [128, 256]
+        into %dest : tensor<128x256xf32> -> tensor<1x1x128x256xf32>
+
+    %out = tensor.empty() : tensor<4x4x128x256xf32>
+    %res = linalg.generic
+        {indexing_maps = [affine_map<(i, j, k, l) -> (0, 0, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>],
+         iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
+        ins(%pack, %other: tensor<1x1x128x256xf32>, tensor<4x4x128x256xf32>)
+        outs(%out: tensor<4x4x128x256xf32>) {
+      ^bb0(%pack_elem: f32, %other_elem: f32, %out_elem: f32):
+        %r = arith.addf %pack_elem, %other_elem : f32
+        linalg.yield %r : f32
+    } -> tensor<4x4x128x256xf32>
+
+    return %res : tensor<4x4x128x256xf32>
+  }
+
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+      // Find and lower pack operation.
+      %pack = transform.structured.match ops{["tensor.pack"]} in %arg1
+        : (!transform.any_op) -> !transform.op<"tensor.pack">
+      %paded, %expanded, %transpose = transform.structured.lower_pack %pack
+        : (!transform.op<"tensor.pack">)
+        -> (!transform.op<"tensor.pad">,
+            !transform.op<"tensor.expand_shape">,
+            !transform.op<"linalg.transpose">)
+
+      %root = transform.structured.match ops{["linalg.generic"]} in %arg1
+          : (!transform.any_op) -> !transform.any_op
+      // Tile the lialg operation with parallel forall loop tiling [4, 4].
+      %tiled_op, %forall_op = transform.structured.tile_using_forall %root num_threads [4, 4]
+          : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+      // Fuse the transpose operation into the tiled loop.
+      transform.structured.fuse_into_containing_op %transpose into %forall_op
+          : (!transform.op<"linalg.transpose">, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield
+    }
+  }
+}
+
+// -----
+// For unpack op, we use lowerUnpadLikeWithExtractSlice = false to ensure no extract_slice is generated.
+// This allows linalg.transpose to be fused as a consumer operation. In below testcase, linalg.transpose
+// as a consumer operation is fused into the scf.forall loop.
+module {
+  // CHECK-label: func @fuse_unpack_as_consumer
+  // CHECK:       scf.forall {{.*}} {
+  // CHECK:         %[[CONSUMER:.*]] = linalg.generic
+  // CHECK:         linalg.transpose ins(%[[CONSUMER]]
+  // CHECK:         scf.forall.in_parallel
+  // CHECK:       }
+  func.func @fuse_unpack_as_consumer(%src: tensor<4x4x128x256xf32>, %other: tensor<4x4x128x256xf32>)
+      -> tensor<128x256xf32> {
+    %out = tensor.empty() : tensor<1x1x128x256xf32>
+    %res = linalg.generic
+        {indexing_maps = [affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (0, 0, k, l)>],
+         iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
+        ins(%src, %other: tensor<4x4x128x256xf32>, tensor<4x4x128x256xf32>)
+        outs(%out: tensor<1x1x128x256xf32>) {
+      ^bb0(%unpack_elem: f32, %other_elem: f32, %out_elem: f32):
+        %r = arith.addf %unpack_elem, %other_elem : f32
+        linalg.yield %r : f32
+    } -> tensor<1x1x128x256xf32>
+
+    %dest = tensor.empty() : tensor<128x256xf32>
+    %unpack = tensor.unpack %res inner_dims_pos = [0, 1] inner_tiles = [128, 256]
+        into %dest : tensor<1x1x128x256xf32> -> tensor<128x256xf32>
+
+    return %unpack : tensor<128x256xf32>
+  }
+
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+      // Find and lower unpack operation.
+      %unpack = transform.structured.match ops{["tensor.unpack"]} in %arg1
+          : (!transform.any_op) -> !transform.op<"tensor.unpack">
+      transform.structured.lower_unpack %unpack {lowerUnpadLikeWithExtractSlice = false}
+        : (!transform.op<"tensor.unpack">)
+        -> (!transform.op<"tensor.empty">,
+            !transform.op<"linalg.transpose">,
+            !transform.op<"tensor.collapse_shape">,
+            !transform.op<"tensor.extract_slice">)
+
+      %root = transform.structured.match ops{["linalg.generic"]} in %arg1
+          : (!transform.any_op) -> !transform.any_op
+      // Tile the lialg operation with parallel forall loop tiling [4, 4].
+      %tiled_op, %forall_op = transform.structured.tile_using_forall %root num_threads [4, 4]
+          : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+      // Fuse the consumer operation into the tiled loop.
+      %slice_op = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %forall_op
+          : (!transform.any_op) -> !transform.op<"tensor.parallel_insert_slice">
+      transform.test.fuse_consumer %slice_op
+        : (!transform.op<"tensor.parallel_insert_slice">) -> (!transform.any_op, !transform.any_op)
+      transform.yield
+    }
+  }
+}
+
+// -----
+// For unpack op, by default lowerUnpadLikeWithExtractSlice = true, which generates extract_slice and blocks fusion.
+// In below testcase, tensor.extract_slice as a consumer operation cannot be fused into the scf.forall loop.
+module {
+  // CHECK-label: func @fuse_unpack_as_consumer_blocked_by_extract_slice
+  // CHECK:       %[[CONSUMER:.*]] = scf.forall {{.*}} {
+  // CHECK:         %[[ADDF:.*]] = linalg.generic
+  // CHECK:         scf.forall.in_parallel
+  // CHECK:           tensor.parallel_insert_slice %[[ADDF]]
+  // CHECK:       }
+  // CHECK:       tensor.extract_slice %[[CONSUMER]]
+  func.func @fuse_unpack_as_consumer_blocked_by_extract_slice(%src: tensor<4x4x128x256xf32>, %other: tensor<4x4x128x256xf32>)
+      -> tensor<128x256xf32> {
+    %out = tensor.empty() : tensor<1x1x128x256xf32>
+    %res = linalg.generic
+        {indexing_maps = [affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (i, j, k, l)>,
+                          affine_map<(i, j, k, l) -> (0, 0, k, l)>],
+         iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
+        ins(%src, %other: tensor<4x4x128x256xf32>, tensor<4x4x128x256xf32>)
+        outs(%out: tensor<1x1x128x256xf32>) {
+      ^bb0(%unpack_elem: f32, %other_elem: f32, %out_elem: f32):
+        %r = arith.addf %unpack_elem, %other_elem : f32
+        linalg.yield %r : f32
+    } -> tensor<1x1x128x256xf32>
+
+    %dest = tensor.empty() : tensor<128x256xf32>
+    %unpack = tensor.unpack %res inner_dims_pos = [0, 1] inner_tiles = [128, 256]
+        into %dest : tensor<1x1x128x256xf32> -> tensor<128x256xf32>
+
+    return %unpack : tensor<128x256xf32>
+  }
+
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+      // Find and lower unpack operation.
+      %unpack = transform.structured.match ops{["tensor.unpack"]} in %arg1
+          : (!transform.any_op) -> !transform.op<"tensor.unpack">
+      transform.structured.lower_unpack %unpack
+        : (!transform.op<"tensor.unpack">)
+        -> (!transform.op<"tensor.empty">,
+            !transform.op<"linalg.transpose">,
+            !transform.op<"tensor.collapse_shape">,
+            !transform.op<"tensor.extract_slice">)
+
+      %root = transform.structured.match ops{["linalg.generic"]} in %arg1
+          : (!transform.any_op) -> !transform.any_op
+      // Tile the lialg operation with parallel forall loop tiling [4, 4].
+      %tiled_op, %forall_op = transform.structured.tile_using_forall %root num_threads [4, 4]
+          : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+      // Fuse the consumer operation into the tiled loop.
+      %slice_op = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %forall_op
+          : (!transform.any_op) -> !transform.op<"tensor.parallel_insert_slice">
+      // Note that we cannot apply transform.test.fuse_consumer here because the extract_slice
+      // is not qualified consumer operation. Forcing this will yeild "could not fetch consumer
+      // to fuse" error.
+      transform.yield
+    }
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/vectorization-scalable.mlir b/mlir/test/Dialect/Linalg/vectorization-scalable.mlir
index 68bac72a1465d..227829238a3d7 100644
--- a/mlir/test/Dialect/Linalg/vectorization-scalable.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-scalable.mlir
@@ -122,22 +122,27 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-func.func @vectorize_dynamic_fill(%A : tensor<?x?xf32>, %arg0 : f32) -> tensor<?x?xf32> {
+// NOTE: Often, non-trailing scalable sizes are problematic - there are no
+// "scalable" arrays of vectors at the LLVM level (multi-dim vectors are
+// decomposed into arrays of aggregates). However, the trailing dim in this
+// case is 1 and that can be folded away later.
+
+func.func @vectorize_dynamic_fill_leading_scalable(%A : tensor<?x?xf32>, %arg0 : f32) -> tensor<?x?xf32> {
   %0 = linalg.fill ins(%arg0 : f32) outs(%A : tensor<?x?xf32>) -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
 }
 
-// CHECK-LABEL: func.func @vectorize_dynamic_fill
+// CHECK-LABEL: func.func @vectorize_dynamic_fill_leading_scalable
 //   CHECK: %[[DIM0:.*]] = tensor.dim
 //   CHECK: %[[DIM1:.*]] = tensor.dim
-//   CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM0]], %[[DIM1]] : vector<8x[16]xi1>
-//   CHECK: %[[BCAST:.*]] = vector.broadcast %{{.*}} : f32 to vector<8x[16]xf32>
-//   CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[BCAST]], {{.*}} {in_bounds = [true, true]} : vector<8x[16]xf32>, tensor<?x?xf32> } : vector<8x[16]xi1>
+//   CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM0]], %[[DIM1]] : vector<[8]x1xi1>
+//   CHECK: %[[BCAST:.*]] = vector.broadcast %{{.*}} : f32 to vector<[8]x1xf32>
+//   CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[BCAST]], {{.*}} {in_bounds = [true, true]} : vector<[8]x1xf32>, tensor<?x?xf32> } : vector<[8]x1xi1>
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.structured.vectorize %0 vector_sizes [8, [16]] : !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [[8], 1] : !transform.any_op
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index 0c996bed996d3..b688a677500c2 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -122,6 +122,48 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+#map = affine_map<() -> ()>
+
+// CHECK-LABEL:   func.func @generic_0d(
+// CHECK-SAME:     %[[ARG_0:.*]]: tensor<f32>, %[[ARG_1:.*]]: tensor<f32>, %[[ARG_2:.*]]: tensor<f32>)
+func.func @generic_0d(%arg0: tensor<f32>, %arg1: tensor<f32>,
+                      %arg2: tensor<f32>) -> tensor<f32> {
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[READ_0:.*]] = vector.transfer_read %[[ARG_0]][], %[[PAD]] : tensor<f32>, vector<f32>
+// CHECK:           %[[ARG_0_AS_SCALAR:.*]] = vector.extract %[[READ_0]][] : f32 from vector<f32>
+// CHECK:           %[[READ_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PAD]] : tensor<f32>, vector<f32>
+// CHECK:           %[[ARG_1_AS_SCALAR:.*]] = vector.extract %[[READ_1]][] : f32 from vector<f32>
+// CHECK:           %[[READ_2:.*]] = vector.transfer_read %[[ARG_2]][], %[[PAD]] : tensor<f32>, vector<f32>
+// CHECK:           %[[ARG_2_AS_SCALAR:.*]] = vector.extract %[[READ_2]][] : f32 from vector<f32>
+// CHECK:           %[[MULF:.*]] = arith.mulf %[[ARG_0_AS_SCALAR]], %[[ARG_1_AS_SCALAR]] : f32
+// CHECK:           %[[ADDF:.*]] = arith.addf %[[ARG_2_AS_SCALAR]], %[[MULF]] : f32
+// CHECK:           %[[ADDF_BCAST:.*]] = vector.broadcast %[[ADDF]] : f32 to vector<f32>
+// CHECK:           vector.transfer_write %[[ADDF_BCAST]], %[[ARG_2]][] : vector<f32>, tensor<f32>
+  %res = linalg.generic {
+    indexing_maps = [#map, #map, #map],
+    iterator_types = []
+  } ins(%arg0, %arg1 : tensor<f32>, tensor<f32>)
+    outs(%arg2 : tensor<f32>) {
+  ^bb(%a: f32, %b: f32, %c: f32) :
+    %d = arith.mulf %a, %b: f32
+    %e = arith.addf %c, %d: f32
+    linalg.yield %e : f32
+  } -> tensor<f32>
+
+  return %res : tensor<f32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 #matmul_transpose_out_trait = {
   indexing_maps = [
     affine_map<(m, n, k) -> (m, k)>,
@@ -372,7 +414,7 @@ module attributes {transform.with_named_sequence} {
 // -----
 
 // CHECK-LABEL: func @test_vectorize_fill
-func.func @test_vectorize_fill_scalar(%A : memref<f32>, %arg0 : f32) {
+func.func @test_vectorize_fill_0d(%A : memref<f32>, %arg0 : f32) {
   // CHECK-SAME: (%[[M:.*]]: memref<f32>, %[[val:.*]]: f32)
   //      CHECK:   %[[VEC:.*]] = vector.broadcast %[[val]] : f32 to vector<f32>
   //      CHECK:   vector.transfer_write %[[VEC]], %[[M]][] : vector<f32>, memref<f32>
@@ -410,8 +452,8 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// CHECK-LABEL: func @test_vectorize_copy_scalar
-func.func @test_vectorize_copy_scalar(%A : memref<f32>, %B : memref<f32>) {
+// CHECK-LABEL: func @test_vectorize_copy_0d
+func.func @test_vectorize_copy_0d(%A : memref<f32>, %B : memref<f32>) {
   //  CHECK-SAME: (%[[A:.*]]: memref<f32>, %[[B:.*]]: memref<f32>)
   //       CHECK:   %[[V:.*]] = vector.transfer_read %[[A]][]{{.*}} : memref<f32>, vector<f32>
   //       CHECK:   %[[val:.*]] = vector.extract %[[V]][] : f32 from vector<f32>
diff --git a/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir b/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
index 74d23fb5b1e3e..d0d3b58a05704 100644
--- a/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
+++ b/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
@@ -425,3 +425,55 @@ module attributes {transform.with_named_sequence} {
      transform.yield
   }
 }
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+func.func @scalar_broadcast(%init : tensor<1x1x3xi32>, %src: tensor<1x3x2x4xi32>, %idx :index) -> tensor<1x1x3xi32> {
+
+  %c0 = arith.constant 0 :index
+
+  %res = linalg.generic {
+    indexing_maps = [#map],
+    iterator_types = ["parallel", "parallel", "parallel"]}
+    outs(%init : tensor<1x1x3xi32>) {
+    ^bb0(%out: i32):
+      %val = tensor.extract %src[%idx, %idx, %idx, %idx] : tensor<1x3x2x4xi32>
+      linalg.yield %val : i32
+  } -> tensor<1x1x3xi32>
+
+  return %res : tensor<1x1x3xi32>
+}
+
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3) -> (0, 0, 0)>
+// CHECK-LABEL:   func.func @scalar_broadcast(
+// CHECK-SAME:      %[[INIT:.*]]: tensor<1x1x3xi32>,
+// CHECK-SAME:      %[[SRC:.*]]: tensor<1x3x2x4xi32>,
+// CHECK-SAME:      %[[IDX:.*]]: index) -> tensor<1x1x3xi32> {
+
+/// Compute the mask for saving the final result
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
+// CHECK:           %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[C3:.*]] = arith.constant 3 : index
+// CHECK:           %[[MASK_RES:.*]] = vector.create_mask %[[C1]], %[[C1_2]], %[[C3]] : vector<1x1x4xi1>
+
+/// Read and broadcast the scalar
+// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[MASK_READ:.*]] = vector.constant_mask [1] : vector<1xi1>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK_READ]] {
+// CHECK-SAME:          vector.transfer_read %[[SRC]]{{\[}}%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]],  %[[PAD]]
+// CHECK-SAME:          {in_bounds = [true, true, true], permutation_map = #[[$MAP]]} : tensor<1x3x2x4xi32>, vector<1x1x4xi32>
+// CHECK-SAME:      } : vector<1xi1> -> vector<1x1x4xi32>
+
+/// Save the result in the output tensor
+// CHECK:           vector.mask %[[MASK_RES]] {
+// CHECK-SAME:        vector.transfer_write %[[READ]], %[[INIT]]{{.*}} {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x3xi32>
+// CHECK-SAME:      } : vector<1x1x4xi1> -> tensor<1x1x3xi32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %module : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [1, 1, 4] {vectorize_nd_extract} : !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir b/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir
index c02405f29bcf7..aa1e44166ec9d 100644
--- a/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir
+++ b/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir
@@ -39,34 +39,72 @@ module attributes {transform.with_named_sequence} {
 // -----
 
 #map = affine_map<() -> ()>
-func.func @negative_no_loops(%arg0: tensor<f32>, %arg1: tensor<f32>) -> tensor<f32> {
-  %1 = linalg.generic {
+func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>) -> tensor<f32> {
+  %res = linalg.generic {
     indexing_maps = [#map],
     iterator_types = []
-  } outs(%arg1 : tensor<f32>) {
-  ^bb0(%arg4: f32):
-    %2 = tensor.extract %arg0[] : tensor<f32>
-    linalg.yield %2 : f32
+  } outs(%init : tensor<f32>) {
+  ^bb0(%in: f32):
+    %1 = tensor.extract %src[] : tensor<f32>
+    linalg.yield %1 : f32
   } -> tensor<f32>
-  return %1 : tensor<f32>
+
+  return %res : tensor<f32>
 }
-// CHECK-LABEL: func.func @negative_no_loops
-// CHECK: tensor.extract
+
+// CHECK-LABEL:   func.func @extract_scalar_from_0d_into_0d(
+// CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
+// CHECK-SAME:      %[[INIT:.*]]: tensor<f32>) -> tensor<f32> {
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
+// CHECK:           vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
 
+// -----
+
+#map = affine_map<(n) -> (n)>
+func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32>) -> tensor<1xf32> {
+  %res = linalg.generic {
+    indexing_maps = [#map],
+    iterator_types = ["parallel"]
+  } outs(%init : tensor<1xf32>) {
+  ^bb0(%in: f32):
+    %1 = tensor.extract %src[] : tensor<f32>
+    linalg.yield %1 : f32
+  } -> tensor<1xf32>
+
+  return %res : tensor<1xf32>
+}
+// CHECK-LABEL:   func.func @extract_scalar_from_0d_into_1d(
+// CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
+// CHECK-SAME:      %[[INIT:.*]]: tensor<1xf32>) -> tensor<1xf32> {
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
+// CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
+// CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
 
 // -----
 
 #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
-func.func @vectorize_nd_tensor_extract_constant_idx(%arg0: tensor<3x3xf32>, %arg2: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
+func.func @vectorize_nd_tensor_extract_scalar_broadcast(%arg0: tensor<3x3xf32>, %arg2: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
   %c0 = arith.constant 1 : index
   %c1 = arith.constant 2 : index
   %2 = linalg.generic {
@@ -80,17 +118,17 @@ func.func @vectorize_nd_tensor_extract_constant_idx(%arg0: tensor<3x3xf32>, %arg
   return %2 : tensor<1x1x3xf32>
 }
 
-// CHECK: #[[$MAP:.*]] = affine_map<(d0, d1) -> (0, 0, 0)>
-// CHECK-LABEL:   func.func @vectorize_nd_tensor_extract_constant_idx(
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1) -> (0, 0, 0)>
+// CHECK-LABEL:   func.func @vectorize_nd_tensor_extract_scalar_broadcast(
 // CHECK-SAME:      %[[ARG_0:.*]]: tensor<3x3xf32>,
 // CHECK-SAME:      %[[ARG_1:.*]]: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
 // CHECK-DAG:       %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[C2:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[C0_f32_2:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK-DAG:       %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[READ:.*]] = vector.transfer_read  %[[ARG_0]][%[[C1]], %[[C2]]], %[[C0_f32]] {in_bounds = [true, true, true], permutation_map = #[[$MAP]]} : tensor<3x3xf32>, vector<1x1x3xf32>
-// CHECK:           %[[C0_4:.*]] = arith.constant 0 : index
-// CHECK:           vector.transfer_write %[[READ]], %[[ARG_1]][%[[C0_4]], %[[C0_4]], %[[C0_4]]]  : vector<1x1x3xf32>, tensor<1x1x3xf32>
+// CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[MASK:.*]] = vector.constant_mask [1] : vector<1xi1>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C1]], %[[C2]]], {{.*}} {in_bounds = [true, true, true], permutation_map = #[[$MAP]]} : tensor<3x3xf32>, vector<1x1x3xf32> } : vector<1xi1> -> vector<1x1x3xf32>
+// CHECK:           %[[C0_2:.*]] = arith.constant 0 : index
+// CHECK:           vector.transfer_write %[[READ]], %[[ARG_1]]{{\[}}%[[C0_2]], %[[C0_2]], %[[C0_2]]] : vector<1x1x3xf32>, tensor<1x1x3xf32>
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
@@ -807,54 +845,41 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-func.func @vectorize_scalar_broadcast_column_tensor(%in: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
+func.func @vectorize_scalar_read_with_broadcast_from_column_tensor(%init: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
   %c4 = arith.constant 4 : index
   %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<[[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
-
-  %out = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} outs(%in : tensor<1x1x4xi32>) {
-  ^bb0(%out: i32):
-    %8 = linalg.index 0 : index
-    %idx_0 = linalg.index 0 : index
-    %extracted = tensor.extract %cst[%idx_0, %c0] : tensor<15x1xi32>
-    linalg.yield %extracted : i32
+  %src = arith.constant dense<[[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
+
+  %res = linalg.generic {
+    indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>],
+    iterator_types = ["parallel", "parallel", "parallel"]}
+    outs(%init : tensor<1x1x4xi32>) {
+
+    ^bb0(%out: i32):
+      %idx = linalg.index 0 : index
+      %extracted = tensor.extract %src[%idx, %c0] : tensor<15x1xi32>
+      linalg.yield %extracted : i32
   } -> tensor<1x1x4xi32>
 
-  return %out:tensor<1x1x4xi32>
-}
-
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1) -> (0, 0, 0)>
-// CHECK-LABEL:   func.func @vectorize_scalar_broadcast_column_tensor(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 4 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant 4 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_9:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_7]], %[[VAL_7]], %[[VAL_7]]], %[[VAL_8]] : tensor<1x1x4xi32>, vector<1x1x4xi32>
-// CHECK:           %[[VAL_10:.*]] = vector.step : vector<1xindex>
-// CHECK:           %[[VAL_11:.*]] = vector.broadcast %[[VAL_10]] : vector<1xindex> to vector<4x1x1xindex>
-// CHECK:           %[[VAL_12:.*]] = vector.transpose %[[VAL_11]], [2, 1, 0] : vector<4x1x1xindex> to vector<1x1x4xindex>
-// CHECK:           %[[VAL_13:.*]] = vector.step : vector<1xindex>
-// CHECK:           %[[VAL_14:.*]] = vector.broadcast %[[VAL_13]] : vector<1xindex> to vector<4x1x1xindex>
-// CHECK:           %[[VAL_15:.*]] = vector.transpose %[[VAL_14]], [2, 1, 0] : vector<4x1x1xindex> to vector<1x1x4xindex>
-// CHECK:           %[[VAL_16:.*]] = arith.constant dense<true> : vector<1x1x4xi1>
-// CHECK:           %[[VAL_17:.*]] = arith.constant dense<0> : vector<1x1x4xi32>
-// CHECK:           %[[VAL_18:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_20:.*]] = vector.shape_cast %[[VAL_15]] : vector<1x1x4xindex> to vector<4xindex>
-// CHECK:           %[[VAL_21:.*]] = vector.extract %[[VAL_20]][0] : index from vector<4xindex>
-// CHECK:           %[[VAL_22:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_23:.*]] = vector.transfer_read %[[VAL_3]]{{\[}}%[[VAL_21]], %[[VAL_2]]], %[[VAL_22]] {in_bounds = [true, true, true], permutation_map = #[[$ATTR_1]]} : tensor<15x1xi32>, vector<1x1x4xi32>
-// CHECK:           %[[VAL_24:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_25:.*]] = vector.transfer_write %[[VAL_23]], %[[VAL_0]]{{\[}}%[[VAL_24]], %[[VAL_24]], %[[VAL_24]]] : vector<1x1x4xi32>, tensor<1x1x4xi32>
+  return %res : tensor<1x1x4xi32>
+}
+
+// CHECK-LABEL:   func.func @vectorize_scalar_read_with_broadcast_from_column_tensor(
+// CHECK-SAME:      %[[INIT:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
+// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[SRC:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
+// CHECK:           %[[IDX_VEC:.*]] = arith.constant dense<0> : vector<1xindex>
+// CHECK:           %[[IDX_ELT:.*]] = vector.extract %[[IDX_VEC]][0] : index from vector<1xindex>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[IDX_ELT]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
+// CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<i32> to vector<1x1x4xi32>
+// CHECK:           %[[RES:.*]] = vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x4xi32>
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.structured.vectorize %0  vector_sizes [1, 1, 4]{ vectorize_nd_extract } : !transform.any_op
+     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+     %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Math/expand-math.mlir b/mlir/test/Dialect/Math/expand-math.mlir
index c10a78ca4ae4c..89413b9570332 100644
--- a/mlir/test/Dialect/Math/expand-math.mlir
+++ b/mlir/test/Dialect/Math/expand-math.mlir
@@ -222,10 +222,11 @@ func.func @roundf_func(%a: f32) -> f32 {
 // CHECK-SAME:    ([[ARG0:%.+]]: f64, [[ARG1:%.+]]: f64)
 func.func @powf_func(%a: f64, %b: f64) ->f64 {
   // CHECK-DAG: [[CST0:%.+]] = arith.constant 0.000000e+00
+  // CHECK-DAG: [[CST1:%.+]] = arith.constant 1.0
   // CHECK-DAG: [[TWO:%.+]] = arith.constant 2.000000e+00
   // CHECK-DAG: [[NEGONE:%.+]] = arith.constant -1.000000e+00
   // CHECK-DAG: [[SQR:%.+]] = arith.mulf [[ARG0]], [[ARG0]]
-  // CHECK-DAG: [[HALF:%.+]] = arith.divf [[ARG1]], [[TWO]] 
+  // CHECK-DAG: [[HALF:%.+]] = arith.divf [[ARG1]], [[TWO]]
   // CHECK-DAG: [[LOG:%.+]] = math.log [[SQR]]
   // CHECK-DAG: [[MULT:%.+]] = arith.mulf [[HALF]], [[LOG]]
   // CHECK-DAG: [[EXPR:%.+]] = math.exp [[MULT]]
@@ -234,8 +235,10 @@ func.func @powf_func(%a: f64, %b: f64) ->f64 {
   // CHECK-DAG: [[CMPNEG:%.+]] = arith.cmpf olt, [[ARG0]]
   // CHECK-DAG: [[CMPZERO:%.+]] = arith.cmpf one, [[REMF]]
   // CHECK-DAG: [[AND:%.+]] = arith.andi [[CMPZERO]], [[CMPNEG]]
+  // CHECK-DAG: [[CMPZERO:%.+]] = arith.cmpf oeq, [[ARG1]], [[CST0]]
   // CHECK-DAG: [[SEL:%.+]] = arith.select [[AND]], [[NEGEXPR]], [[EXPR]]
-  // CHECK: return [[SEL]]
+  // CHECK-DAG: [[SEL1:%.+]] = arith.select [[CMPZERO]], [[CST1]], [[SEL]]
+  // CHECK: return [[SEL1]]
   %ret = math.powf %a, %b : f64
   return %ret : f64
 }
@@ -516,7 +519,7 @@ func.func @roundeven16(%arg: f16) -> f16 {
 
 // CHECK-LABEL:   func.func @math_fpowi_neg_odd_power
 func.func @math_fpowi_neg_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
-  %1 = arith.constant dense<-3> : tensor<8xi64> 
+  %1 = arith.constant dense<-3> : tensor<8xi64>
   %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64>
   return %2 : tensor<8xf32>
 }
@@ -539,7 +542,7 @@ func.func @math_fpowi_neg_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
 
 // CHECK-LABEL:   func.func @math_fpowi_neg_even_power
 func.func @math_fpowi_neg_even_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
-  %1 = arith.constant dense<-4> : tensor<8xi64> 
+  %1 = arith.constant dense<-4> : tensor<8xi64>
   %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64>
   return %2 : tensor<8xf32>
 }
@@ -562,7 +565,7 @@ func.func @math_fpowi_neg_even_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
 
 // CHECK-LABEL:   func.func @math_fpowi_pos_odd_power
 func.func @math_fpowi_pos_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
-  %1 = arith.constant dense<5> : tensor<8xi64> 
+  %1 = arith.constant dense<5> : tensor<8xi64>
   %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64>
   return %2 : tensor<8xf32>
 }
@@ -576,7 +579,7 @@ func.func @math_fpowi_pos_odd_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
 
 // CHECK-LABEL:   func.func @math_fpowi_pos_even_power
 func.func @math_fpowi_pos_even_power(%0 : tensor<8xf32>) -> tensor<8xf32> {
-  %1 = arith.constant dense<4> : tensor<8xi64> 
+  %1 = arith.constant dense<4> : tensor<8xi64>
   %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi64>
   return %2 : tensor<8xf32>
 }
@@ -617,9 +620,10 @@ func.func @math_fpowi_to_powf_tensor(%0 : tensor<8xf32>, %1: tensor<8xi32>) -> t
   return %2 : tensor<8xf32>
 }
 // CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>, %[[ARG1:.*]]: tensor<8xi32>) -> tensor<8xf32> {
-// CHECK:        %[[CSTNEG1:.*]] = arith.constant dense<-1.000000e+00> : tensor<8xf32>
-// CHECK:        %[[CST2:.*]] = arith.constant dense<2.000000e+00> : tensor<8xf32>
-// CHECK:        %[[CST0:.*]] = arith.constant dense<0.000000e+00> : tensor<8xf32>
+// CHECK-DAG:    %[[CSTNEG1:.*]] = arith.constant dense<-1.000000e+00> : tensor<8xf32>
+// CHECK-DAG:    %[[CST2:.*]] = arith.constant dense<2.000000e+00> : tensor<8xf32>
+// CHECK-DAG:    %[[CST0:.*]] = arith.constant dense<0.000000e+00> : tensor<8xf32>
+// CHECK-DAG:    %[[CST1:.+]] = arith.constant dense<1.000000e+00> : tensor<8xf32>
 // CHECK:        %[[TOFP:.*]] = arith.sitofp %[[ARG1]] : tensor<8xi32> to tensor<8xf32>
 // CHECK:        %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32>
 // CHECK:        %[[DIV:.*]] = arith.divf %[[TOFP]], %[[CST2]] : tensor<8xf32>
@@ -631,8 +635,10 @@ func.func @math_fpowi_to_powf_tensor(%0 : tensor<8xf32>, %1: tensor<8xi32>) -> t
 // CHECK:        %[[CMPF:.*]] = arith.cmpf olt, %[[ARG0]], %[[CST0]] : tensor<8xf32>
 // CHECK:        %[[CMPF1:.*]] = arith.cmpf one, %[[REM]], %[[CST0]] : tensor<8xf32>
 // CHECK:        %[[AND:.*]] = arith.andi %[[CMPF1]], %[[CMPF]] : tensor<8xi1>
+// CHECK:        %[[CMPZERO:.*]] = arith.cmpf oeq, %[[TOFP]], %[[CST0]]
 // CHECK:        %[[SEL:.*]] = arith.select %[[AND]], %[[MUL1]], %[[EXP]] : tensor<8xi1>, tensor<8xf32>
-// CHECK:      return %[[SEL]] : tensor<8xf32>
+// CHECK:        %[[SEL1:.+]] = arith.select %[[CMPZERO]], %[[CST1]], %[[SEL]]
+// CHECK:      return %[[SEL1]] : tensor<8xf32>
 
 // -----
 
@@ -642,9 +648,10 @@ func.func @math_fpowi_to_powf_scalar(%0 : f32, %1: i64) -> f32 {
   return %2 : f32
 }
 // CHECK-SAME: (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: i64) -> f32 {
-// CHECK:        %[[CSTNEG1:.*]] = arith.constant -1.000000e+00 : f32
-// CHECK:        %[[CST2:.*]] = arith.constant 2.000000e+00 : f32
-// CHECK:        %[[CST0:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:    %[[CSTNEG1:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK-DAG:    %[[CST2:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG:    %[[CST0:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:    %[[CST1:.+]] = arith.constant 1.000000e+00 : f32
 // CHECK:        %[[TOFP:.*]] = arith.sitofp %[[ARG1]] : i64 to f32
 // CHECK:        %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : f32
 // CHECK:        %[[DIV:.*]] = arith.divf %[[TOFP]], %[[CST2]] : f32
@@ -656,8 +663,10 @@ func.func @math_fpowi_to_powf_scalar(%0 : f32, %1: i64) -> f32 {
 // CHECK:        %[[CMPF:.*]] = arith.cmpf olt, %[[ARG0]], %[[CST0]] : f32
 // CHECK:        %[[CMPF1:.*]] = arith.cmpf one, %[[REM]], %[[CST0]] : f32
 // CHECK:        %[[AND:.*]] = arith.andi %[[CMPF1]], %[[CMPF]] : i1
+// CHECK:        %[[CMPZERO:.*]] = arith.cmpf oeq, %[[TOFP]], %[[CST0]]
 // CHECK:        %[[SEL:.*]] = arith.select %[[AND]], %[[MUL1]], %[[EXP]] : f32
-// CHECK:       return %[[SEL]] : f32
+// CHECK:        %[[SEL1:.+]] = arith.select %[[CMPZERO]], %[[CST1]], %[[SEL]]
+// CHECK:       return %[[SEL1]] : f32
 
 // -----
 
diff --git a/mlir/test/Dialect/Mesh/ops.mlir b/mlir/test/Dialect/Mesh/ops.mlir
index d8df01c3d6520..978de4939ee77 100644
--- a/mlir/test/Dialect/Mesh/ops.mlir
+++ b/mlir/test/Dialect/Mesh/ops.mlir
@@ -615,16 +615,16 @@ func.func @update_halo(
     // CHECK-SAME: %[[ARG:.*]]: memref<12x12xi8>
     %arg0 : memref<12x12xi8>) {
   // CHECK-NEXT: %[[C2:.*]] = arith.constant 2 : i64
-  // CHECK-NEXT: %[[UH1:.*]] = mesh.update_halo %[[ARG]] into %[[ARG]] on @mesh0
+  // CHECK-NEXT: %[[UH1:.*]] = mesh.update_halo %[[ARG]] on @mesh0
   // CHECK-SAME: split_axes = {{\[\[}}0]]
-  // CHECK-SAME: halo_sizes = [2, %c2_i64] : memref<12x12xi8> -> memref<12x12xi8>
+  // CHECK-SAME: halo_sizes = [2, %c2_i64] : memref<12x12xi8>
   %c2 = arith.constant 2 : i64
-  %uh1 = mesh.update_halo %arg0 into %arg0 on @mesh0 split_axes = [[0]]
-    source_halo_sizes = [2, %c2] : memref<12x12xi8> -> memref<12x12xi8>
-  // CHECK-NEXT: %[[UH2:.*]] = mesh.update_halo %[[ARG]] into %[[UH1]] on @mesh0
+  %uh1 = mesh.update_halo %arg0 on @mesh0 split_axes = [[0]]
+    halo_sizes = [2, %c2] : memref<12x12xi8>
+  // CHECK-NEXT: %[[UH2:.*]] = mesh.update_halo %[[UH1]] on @mesh0
   // CHECK-SAME: split_axes = {{\[\[}}0], [1]]
-  // CHECK-SAME: halo_sizes = [2, 2, %[[C2]], 2] : memref<12x12xi8> -> memref<12x12xi8>
-  %uh2 = mesh.update_halo %arg0 into %uh1 on @mesh0 split_axes = [[0], [1]]
-    source_halo_sizes = [2, 2, %c2, 2] : memref<12x12xi8> -> memref<12x12xi8>
+  // CHECK-SAME: halo_sizes = [2, 2, %[[C2]], 2] : memref<12x12xi8>
+  %uh2 = mesh.update_halo %uh1 on @mesh0 split_axes = [[0], [1]]
+    halo_sizes = [2, 2, %c2, 2] : memref<12x12xi8>
   return
 }
diff --git a/mlir/test/Dialect/Mesh/spmdization.mlir b/mlir/test/Dialect/Mesh/spmdization.mlir
index 22ddb72569835..c1b96fda0f4a7 100644
--- a/mlir/test/Dialect/Mesh/spmdization.mlir
+++ b/mlir/test/Dialect/Mesh/spmdization.mlir
@@ -226,7 +226,7 @@ func.func @test_shard_update_halo(%arg0: tensor<1200x1200xi64>) -> tensor<1200x1
   %sharding = mesh.sharding @mesh_1d_4 split_axes = [[0]] : !mesh.sharding
   // CHECK: %[[T:.*]] = tensor.empty() : tensor<304x1200xi64>
   // CHECK: %[[inserted_slice:.*]] = tensor.insert_slice %[[IN1]] into %[[T]][2, 0] [300, 1200] [1, 1] : tensor<300x1200xi64> into tensor<304x1200xi64>
-  // CHECK: %[[UH:.*]] = mesh.update_halo %[[IN1]] into %[[inserted_slice]] on @mesh_1d_4 split_axes = {{\[\[0]]}} destination_halo_sizes = [2, 2] : tensor<300x1200xi64> -> tensor<304x1200xi64>
+  // CHECK: %[[UH:.*]] = mesh.update_halo %[[inserted_slice]] on @mesh_1d_4 split_axes = {{\[\[0]]}} halo_sizes = [2, 2] : tensor<304x1200xi64>
   %sharding_annotated = mesh.shard %arg0 to %sharding : tensor<1200x1200xi64>
   %sharding_0 = mesh.sharding @mesh_1d_4 split_axes = [[0]] halo_sizes = [2, 2] : !mesh.sharding
   %sharding_annotated_1 = mesh.shard %sharding_annotated to %sharding_0 : tensor<1200x1200xi64>
@@ -242,7 +242,7 @@ func.func @test_shard_update_halo2d(%arg0: tensor<1200x1200xi64>) -> tensor<1200
   %sharding = mesh.sharding @mesh4x4 split_axes = [[0], [1]] : !mesh.sharding
   // CHECK: %[[T:.*]] = tensor.empty() : tensor<303x307xi64>
   // CHECK: %[[inserted_slice:.*]] = tensor.insert_slice %[[IN1]] into %[[T]][1, 3] [300, 300] [1, 1] : tensor<300x300xi64> into tensor<303x307xi64>
-  // CHECK: %[[UH:.*]] = mesh.update_halo %[[IN1]] into %[[inserted_slice]] on @mesh4x4 split_axes = {{\[\[}}0], [1]] destination_halo_sizes = [1, 2, 3, 4] : tensor<300x300xi64> -> tensor<303x307xi64>
+  // CHECK: %[[UH:.*]] = mesh.update_halo %[[inserted_slice]] on @mesh4x4 split_axes = {{\[\[}}0], [1]] halo_sizes = [1, 2, 3, 4] : tensor<303x307xi64>
   %sharding_annotated = mesh.shard %arg0 to %sharding : tensor<1200x1200xi64>
   %sharding_0 = mesh.sharding @mesh4x4 split_axes = [[0], [1]] halo_sizes = [1, 2, 3, 4] : !mesh.sharding
   %sharding_annotated_1 = mesh.shard %sharding_annotated to %sharding_0 : tensor<1200x1200xi64>
diff --git a/mlir/test/Dialect/OpenACC/legalize-data.mlir b/mlir/test/Dialect/OpenACC/legalize-data.mlir
index 842f8e260c499..baa72ae416c92 100644
--- a/mlir/test/Dialect/OpenACC/legalize-data.mlir
+++ b/mlir/test/Dialect/OpenACC/legalize-data.mlir
@@ -2,7 +2,7 @@
 // RUN: mlir-opt -split-input-file --openacc-legalize-data-values=host-to-device=false %s | FileCheck %s --check-prefixes=CHECK,HOST
 
 func.func @test(%a: memref<10xf32>, %i : index) {
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.parallel dataOperands(%create : memref<10xf32>) {
     %ci = memref.load %a[%i] : memref<10xf32>
     acc.yield
@@ -12,7 +12,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>, %[[I:.*]]: index)
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.parallel dataOperands(%[[CREATE]] : memref<10xf32>) {
 // DEVICE:   %{{.*}} = memref.load %[[CREATE]][%[[I]]] : memref<10xf32>
 // HOST:    %{{.*}} = memref.load %[[A]][%[[I]]] : memref<10xf32>
@@ -22,7 +22,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 // -----
 
 func.func @test(%a: memref<10xf32>, %i : index) {
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.serial dataOperands(%create : memref<10xf32>) {
     %ci = memref.load %a[%i] : memref<10xf32>
     acc.yield
@@ -32,7 +32,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>, %[[I:.*]]: index)
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.serial dataOperands(%[[CREATE]] : memref<10xf32>) {
 // DEVICE:   %{{.*}} = memref.load %[[CREATE]][%[[I]]] : memref<10xf32>
 // HOST:    %{{.*}} = memref.load %[[A]][%[[I]]] : memref<10xf32>
@@ -42,7 +42,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 // -----
 
 func.func @test(%a: memref<10xf32>, %i : index) {
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.kernels dataOperands(%create : memref<10xf32>) {
     %ci = memref.load %a[%i] : memref<10xf32>
     acc.terminator
@@ -52,7 +52,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>, %[[I:.*]]: index)
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.kernels dataOperands(%[[CREATE]] : memref<10xf32>) {
 // DEVICE:   %{{.*}} = memref.load %[[CREATE]][%[[I]]] : memref<10xf32>
 // HOST:    %{{.*}} = memref.load %[[A]][%[[I]]] : memref<10xf32>
@@ -62,7 +62,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 // -----
 
 func.func @test(%a: memref<10xf32>, %i : index) {
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.data dataOperands(%create : memref<10xf32>) {
     %c0 = arith.constant 0.000000e+00 : f32
     memref.store %c0, %a[%i] : memref<10xf32>
@@ -77,7 +77,7 @@ func.func @test(%a: memref<10xf32>, %i : index) {
 
 // CHECK-LABEL: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>, %[[I:.*]]: index)
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.data dataOperands(%[[CREATE]] : memref<10xf32>) {
 // CHECK: memref.store %{{.*}}, %[[A]][%[[I]]] : memref<10xf32>
 // DEVICE:   %{{.*}} = memref.load %[[CREATE]][%[[I]]] : memref<10xf32>
@@ -91,7 +91,7 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.parallel dataOperands(%create : memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
@@ -104,7 +104,7 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.parallel dataOperands(%[[CREATE]] : memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[CREATE:.*]][%[[I]]] : memref<10xf32>
@@ -129,7 +129,7 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.parallel private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
@@ -142,7 +142,7 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.parallel private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
@@ -167,7 +167,7 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.parallel {
     acc.loop private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
@@ -180,7 +180,7 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.parallel  {
 // CHECK:   acc.loop private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
@@ -205,7 +205,7 @@ func.func @test(%a: memref<10xf32>) {
   %lb = arith.constant 0 : index
   %st = arith.constant 1 : index
   %c10 = arith.constant 10 : index
-  %p1 = acc.private varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %p1 = acc.private varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.serial private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
@@ -218,7 +218,7 @@ func.func @test(%a: memref<10xf32>) {
 
 // CHECK: func.func @test
 // CHECK-SAME: (%[[A:.*]]: memref<10xf32>)
-// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[A]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK: acc.serial private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir
index 3ed81b5117f22..2706792a263a8 100644
--- a/mlir/test/Dialect/OpenACC/ops.mlir
+++ b/mlir/test/Dialect/OpenACC/ops.mlir
@@ -115,12 +115,12 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
   %numGangs = arith.constant 10 : i64
   %numWorkers = arith.constant 10 : i64
 
-  %pa = acc.present varPtr(%a : memref<10x10xf32>) -> memref<10x10xf32>
-  %pb = acc.present varPtr(%b : memref<10x10xf32>) -> memref<10x10xf32>
-  %pc = acc.present varPtr(%c : memref<10xf32>) -> memref<10xf32>
-  %pd = acc.present varPtr(%d : memref<10xf32>) -> memref<10xf32>
+  %pa = acc.present varPtr(%a : memref<10x10xf32>) varType(tensor<10x10xf32>) -> memref<10x10xf32>
+  %pb = acc.present varPtr(%b : memref<10x10xf32>) varType(tensor<10x10xf32>) -> memref<10x10xf32>
+  %pc = acc.present varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+  %pd = acc.present varPtr(%d : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.data dataOperands(%pa, %pb, %pc, %pd: memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) {
-    %private = acc.private varPtr(%c : memref<10xf32>) -> memref<10xf32>
+    %private = acc.private varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
     acc.parallel num_gangs({%numGangs: i64}) num_workers(%numWorkers: i64 [#acc.device_type<nvidia>]) private(@privatization_memref_10_f32 -> %private : memref<10xf32>) {
       acc.loop gang control(%x : index) = (%lb : index) to (%c10 : index) step (%st : index) {
         acc.loop worker control(%y : index) = (%lb : index) to (%c10 : index) step (%st : index) {
@@ -157,7 +157,7 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
 // CHECK-NEXT:   [[NUMGANG:%.*]] = arith.constant 10 : i64
 // CHECK-NEXT:   [[NUMWORKERS:%.*]] = arith.constant 10 : i64
 // CHECK:        acc.data dataOperands(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) {
-// CHECK-NEXT:     %[[P_ARG2:.*]] = acc.private varPtr([[ARG2]] : memref<10xf32>) -> memref<10xf32> 
+// CHECK-NEXT:     %[[P_ARG2:.*]] = acc.private varPtr([[ARG2]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> 
 // CHECK-NEXT:     acc.parallel num_gangs({[[NUMGANG]] : i64}) num_workers([[NUMWORKERS]] : i64 [#acc.device_type<nvidia>]) private(@privatization_memref_10_f32 -> %[[P_ARG2]] : memref<10xf32>) {
 // CHECK-NEXT:       acc.loop gang control(%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
 // CHECK-NEXT:         acc.loop worker control(%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
@@ -257,7 +257,7 @@ func.func @testloopop(%a : memref<10xf32>) -> () {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
   } attributes {inclusiveUpperbound = array<i1: true>}
-  %b = acc.cache varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %b = acc.cache varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.loop cache(%b : memref<10xf32>) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
@@ -317,7 +317,7 @@ func.func @testloopop(%a : memref<10xf32>) -> () {
 // CHECK:      acc.loop gang({dim=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64})
 // CHECK-NEXT:   "test.openacc_dummy_op"() : () -> ()
 // CHECK-NEXT:   acc.yield
-// CHECK:      %{{.*}} = acc.cache varPtr(%{{.*}} : memref<10xf32>) -> memref<10xf32>
+// CHECK:      %{{.*}} = acc.cache varPtr(%{{.*}} : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK-NEXT: acc.loop cache(%{{.*}} : memref<10xf32>)
 // CHECK-NEXT:   "test.openacc_dummy_op"() : () -> ()
 // CHECK-NEXT:   acc.yield
@@ -560,7 +560,7 @@ func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10
   }
   acc.serial wait({%i64value : i64, %i32value : i32, %idxValue : index}) {
   }
-  %firstprivate = acc.firstprivate varPtr(%b : memref<10xf32>) -> memref<10xf32>
+  %firstprivate = acc.firstprivate varPtr(%b : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@firstprivatization_memref_10xf32 -> %firstprivate : memref<10xf32>) {
   }
   acc.serial {
@@ -597,7 +597,7 @@ func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10
 // CHECK-NEXT: }
 // CHECK:      acc.serial wait({[[I64VALUE]] : i64, [[I32VALUE]] : i32, [[IDXVALUE]] : index}) {
 // CHECK-NEXT: }
-// CHECK:      %[[FIRSTP:.*]] = acc.firstprivate varPtr([[ARGB]] : memref<10xf32>) -> memref<10xf32>
+// CHECK:      %[[FIRSTP:.*]] = acc.firstprivate varPtr([[ARGB]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK:      acc.serial firstprivate(@firstprivatization_memref_10xf32 -> %[[FIRSTP]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) {
 // CHECK-NEXT: }
 // CHECK:      acc.serial {
@@ -1069,45 +1069,45 @@ func.func @testexitdataop(%a: !llvm.ptr) -> () {
   %i32Value = arith.constant 1 : i32
   %idxValue = arith.constant 1 : index
 
-  %0 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%0 : !llvm.ptr)
-  acc.copyout accPtr(%0 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%0 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
-  %1 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %1 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%1 : !llvm.ptr)
   acc.delete accPtr(%1 : !llvm.ptr)
 
-  %2 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %2 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%2 : !llvm.ptr) attributes {async,finalize}
   acc.delete accPtr(%2 : !llvm.ptr)
 
-  %3 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %3 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%3 : !llvm.ptr)
   acc.detach accPtr(%3 : !llvm.ptr)
 
-  %4 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %4 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%4 : !llvm.ptr) attributes {async}
-  acc.copyout accPtr(%4 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%4 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
-  %5 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %5 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%5 : !llvm.ptr) attributes {wait}
   acc.delete accPtr(%5 : !llvm.ptr)
 
-  %6 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %6 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data async(%i64Value : i64) dataOperands(%6 : !llvm.ptr)
-  acc.copyout accPtr(%6 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%6 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
-  %7 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %7 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data dataOperands(%7 : !llvm.ptr) async(%i64Value : i64)
-  acc.copyout accPtr(%7 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%7 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
-  %8 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %8 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data if(%ifCond) dataOperands(%8 : !llvm.ptr)
-  acc.copyout accPtr(%8 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%8 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
-  %9 = acc.getdeviceptr varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %9 = acc.getdeviceptr varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.exit_data wait_devnum(%i64Value: i64) wait(%i32Value, %idxValue : i32, index) dataOperands(%9 : !llvm.ptr)
-  acc.copyout accPtr(%9 : !llvm.ptr) to varPtr(%a : !llvm.ptr)
+  acc.copyout accPtr(%9 : !llvm.ptr) to varPtr(%a : !llvm.ptr) varType(f64)
 
   return
 }
@@ -1118,45 +1118,45 @@ func.func @testexitdataop(%a: !llvm.ptr) -> () {
 // CHECK: %[[I32VALUE:.*]] = arith.constant 1 : i32
 // CHECK: %[[IDXVALUE:.*]] = arith.constant 1 : index
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr)
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr)
 // CHECK: acc.delete accPtr(%[[DEVPTR]] : !llvm.ptr)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr) attributes {async, finalize}
 // CHECK: acc.delete accPtr(%[[DEVPTR]] : !llvm.ptr)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr)
 // CHECK: acc.detach accPtr(%[[DEVPTR]] : !llvm.ptr)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr) attributes {async}
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !llvm.ptr) attributes {wait}
 // CHECK: acc.delete accPtr(%[[DEVPTR]] : !llvm.ptr)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data async(%[[I64VALUE]] : i64) dataOperands(%[[DEVPTR]] : !llvm.ptr)
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data async(%[[I64VALUE]] : i64) dataOperands(%[[DEVPTR]] : !llvm.ptr)
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data if(%[[IFCOND]]) dataOperands(%[[DEVPTR]] : !llvm.ptr)
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
-// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.exit_data wait_devnum(%[[I64VALUE]] : i64) wait(%[[I32VALUE]], %[[IDXVALUE]] : i32, index) dataOperands(%[[DEVPTR]] : !llvm.ptr)
-// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr)
+// CHECK: acc.copyout accPtr(%[[DEVPTR]] : !llvm.ptr) to varPtr(%[[ARGA]] : !llvm.ptr) varType(f64)
 
 // -----
 
@@ -1167,25 +1167,25 @@ func.func @testenterdataop(%a: !llvm.ptr, %b: !llvm.ptr, %c: !llvm.ptr) -> () {
   %i32Value = arith.constant 1 : i32
   %idxValue = arith.constant 1 : index
 
-  %0 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data dataOperands(%0 : !llvm.ptr)
-  %1 = acc.create varPtr(%a : !llvm.ptr) -> !llvm.ptr
-  %2 = acc.create varPtr(%b : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
-  %3 = acc.create varPtr(%c : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
+  %1 = acc.create varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
+  %2 = acc.create varPtr(%b : !llvm.ptr) varType(f64) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
+  %3 = acc.create varPtr(%c : !llvm.ptr) varType(f64) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
   acc.enter_data dataOperands(%1, %2, %3 : !llvm.ptr, !llvm.ptr, !llvm.ptr)
-  %4 = acc.attach varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %4 = acc.attach varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data dataOperands(%4 : !llvm.ptr)
-  %5 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %5 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data dataOperands(%5 : !llvm.ptr) attributes {async}
-  %6 = acc.create varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %6 = acc.create varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data dataOperands(%6 : !llvm.ptr) attributes {wait}
-  %7 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %7 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data async(%i64Value : i64) dataOperands(%7 : !llvm.ptr)
-  %8 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %8 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data dataOperands(%8 : !llvm.ptr) async(%i64Value : i64)
-  %9 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %9 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data if(%ifCond) dataOperands(%9 : !llvm.ptr)
-  %10 = acc.copyin varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %10 = acc.copyin varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.enter_data wait_devnum(%i64Value: i64) wait(%i32Value, %idxValue : i32, index) dataOperands(%10 : !llvm.ptr)
 
   return
@@ -1196,72 +1196,72 @@ func.func @testenterdataop(%a: !llvm.ptr, %b: !llvm.ptr, %c: !llvm.ptr) -> () {
 // CHECK: [[I64VALUE:%.*]] = arith.constant 1 : i64
 // CHECK: [[I32VALUE:%.*]] = arith.constant 1 : i32
 // CHECK: [[IDXVALUE:%.*]] = arith.constant 1 : index
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data dataOperands(%[[COPYIN]] : !llvm.ptr)
-// CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
-// CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[ARGB]] : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
-// CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[ARGC]] : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
+// CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
+// CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[ARGB]] : !llvm.ptr) varType(f64) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
+// CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[ARGC]] : !llvm.ptr) varType(f64) -> !llvm.ptr {dataClause = #acc<data_clause acc_create_zero>}
 // CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !llvm.ptr, !llvm.ptr, !llvm.ptr)
-// CHECK: %[[ATTACH:.*]] = acc.attach varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[ATTACH:.*]] = acc.attach varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data dataOperands(%[[ATTACH]] : !llvm.ptr)
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data dataOperands(%[[COPYIN]] : !llvm.ptr) attributes {async}
-// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data dataOperands(%[[CREATE]] : !llvm.ptr) attributes {wait}
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data async([[I64VALUE]] : i64) dataOperands(%[[COPYIN]] : !llvm.ptr)
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data async([[I64VALUE]] : i64) dataOperands(%[[COPYIN]] : !llvm.ptr)
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data if([[IFCOND]]) dataOperands(%[[COPYIN]] : !llvm.ptr)
-// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARGA]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.enter_data wait_devnum([[I64VALUE]] : i64) wait([[I32VALUE]], [[IDXVALUE]] : i32, index) dataOperands(%[[COPYIN]] : !llvm.ptr)
 
 // -----
 
 func.func @teststructureddataclauseops(%a: memref<10xf32>, %b: memref<memref<10xf32>>, %c: memref<10x20xf32>) -> () {
-  %deviceptr = acc.deviceptr varPtr(%a : memref<10xf32>) -> memref<10xf32> {name = "arrayA"}
+  %deviceptr = acc.deviceptr varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {name = "arrayA"}
   acc.parallel dataOperands(%deviceptr : memref<10xf32>) {
   }
 
-  %present = acc.present varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %present = acc.present varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.data dataOperands(%present : memref<10xf32>) {
   }
 
-  %copyin = acc.copyin varPtr(%a : memref<10xf32>) -> memref<10xf32>
+  %copyin = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.parallel dataOperands(%copyin : memref<10xf32>) {
   }
 
-  %copyinreadonly = acc.copyin varPtr(%a : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyin_readonly>}
+  %copyinreadonly = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyin_readonly>}
   acc.kernels dataOperands(%copyinreadonly : memref<10xf32>) {
   }
 
-  %copyinfromcopy = acc.copyin varPtr(%a : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+  %copyinfromcopy = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
   acc.serial dataOperands(%copyinfromcopy : memref<10xf32>) {
   }
-  acc.copyout accPtr(%copyinfromcopy : memref<10xf32>) to varPtr(%a : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+  acc.copyout accPtr(%copyinfromcopy : memref<10xf32>) to varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
 
-  %create = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32>
-  %createimplicit = acc.create varPtr(%c : memref<10x20xf32>) -> memref<10x20xf32> {implicit = true}
+  %create = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+  %createimplicit = acc.create varPtr(%c : memref<10x20xf32>) varType(tensor<10x20xf32>) -> memref<10x20xf32> {implicit = true}
   acc.parallel dataOperands(%create, %createimplicit : memref<10xf32>, memref<10x20xf32>) {
   }
   acc.delete accPtr(%create : memref<10xf32>) {dataClause = #acc<data_clause acc_create>}
   acc.delete accPtr(%createimplicit : memref<10x20xf32>) {dataClause = #acc<data_clause acc_create>, implicit = true}
 
-  %copyoutzero = acc.create varPtr(%a : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout_zero>}
+  %copyoutzero = acc.create varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout_zero>}
   acc.parallel dataOperands(%copyoutzero: memref<10xf32>) {
   }
-  acc.copyout accPtr(%copyoutzero : memref<10xf32>) to varPtr(%a : memref<10xf32>) {dataClause = #acc<data_clause acc_copyout_zero>}
+  acc.copyout accPtr(%copyoutzero : memref<10xf32>) to varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copyout_zero>}
 
   %attach = acc.attach varPtr(%b : memref<memref<10xf32>>) -> memref<memref<10xf32>>
   acc.parallel dataOperands(%attach : memref<memref<10xf32>>) {
   }
   acc.detach accPtr(%attach : memref<memref<10xf32>>) {dataClause = #acc<data_clause acc_attach>}
 
-  %copyinparent = acc.copyin varPtr(%a : memref<10xf32>) varPtrPtr(%b : memref<memref<10xf32>>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+  %copyinparent = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) varPtrPtr(%b : memref<memref<10xf32>>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
   acc.parallel dataOperands(%copyinparent : memref<10xf32>) {
   }
-  acc.copyout accPtr(%copyinparent : memref<10xf32>) to varPtr(%a : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+  acc.copyout accPtr(%copyinparent : memref<10xf32>) to varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
 
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -1271,93 +1271,93 @@ func.func @teststructureddataclauseops(%a: memref<10xf32>, %b: memref<memref<10x
   %c20 = arith.constant 20 : index
 
   %bounds1full = acc.bounds lowerbound(%c0 : index) upperbound(%c9 : index) stride(%c1 : index)
-  %copyinfullslice1 = acc.copyin varPtr(%a : memref<10xf32>) bounds(%bounds1full) -> memref<10xf32> {name = "arrayA[0:9]"}
+  %copyinfullslice1 = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) bounds(%bounds1full) -> memref<10xf32> {name = "arrayA[0:9]"}
   // Specify full-bounds but assume that startIdx of array reference is 1.
   %bounds2full = acc.bounds lowerbound(%c1 : index) upperbound(%c20 : index) extent(%c20 : index) stride(%c4 : index) startIdx(%c1 : index) {strideInBytes = true}
-  %copyinfullslice2 = acc.copyin varPtr(%c : memref<10x20xf32>) bounds(%bounds1full, %bounds2full) -> memref<10x20xf32>
+  %copyinfullslice2 = acc.copyin varPtr(%c : memref<10x20xf32>) varType(tensor<10x20xf32>) bounds(%bounds1full, %bounds2full) -> memref<10x20xf32>
   acc.parallel dataOperands(%copyinfullslice1, %copyinfullslice2 : memref<10xf32>, memref<10x20xf32>) {
   }
 
   %bounds1partial = acc.bounds lowerbound(%c4 : index) upperbound(%c9 : index) stride(%c1 : index)
-  %copyinpartial = acc.copyin varPtr(%a : memref<10xf32>) bounds(%bounds1partial) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+  %copyinpartial = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) bounds(%bounds1partial) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
   acc.parallel dataOperands(%copyinpartial : memref<10xf32>) {
   }
-  acc.copyout accPtr(%copyinpartial : memref<10xf32>) bounds(%bounds1partial) to varPtr(%a : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+  acc.copyout accPtr(%copyinpartial : memref<10xf32>) bounds(%bounds1partial) to varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
 
   return
 }
 
 // CHECK: func.func @teststructureddataclauseops([[ARGA:%.*]]: memref<10xf32>, [[ARGB:%.*]]: memref<memref<10xf32>>, [[ARGC:%.*]]: memref<10x20xf32>) {
-// CHECK: [[DEVICEPTR:%.*]] = acc.deviceptr varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {name = "arrayA"}
+// CHECK: [[DEVICEPTR:%.*]] = acc.deviceptr varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {name = "arrayA"}
 // CHECK-NEXT: acc.parallel dataOperands([[DEVICEPTR]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK: [[PRESENT:%.*]] = acc.present varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: [[PRESENT:%.*]] = acc.present varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK-NEXT: acc.data dataOperands([[PRESENT]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK: [[COPYIN:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32>
+// CHECK: [[COPYIN:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
 // CHECK-NEXT: acc.parallel dataOperands([[COPYIN]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK: [[COPYINRO:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyin_readonly>}
+// CHECK: [[COPYINRO:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyin_readonly>}
 // CHECK-NEXT: acc.kernels dataOperands([[COPYINRO]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK: [[COPYINCOPY:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+// CHECK: [[COPYINCOPY:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
 // CHECK-NEXT: acc.serial dataOperands([[COPYINCOPY]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK-NEXT: acc.copyout accPtr([[COPYINCOPY]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
-// CHECK: [[CREATE:%.*]] = acc.create varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32>
-// CHECK-NEXT: [[CREATEIMP:%.*]] = acc.create varPtr([[ARGC]] : memref<10x20xf32>) -> memref<10x20xf32> {implicit = true}
+// CHECK-NEXT: acc.copyout accPtr([[COPYINCOPY]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+// CHECK: [[CREATE:%.*]] = acc.create varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+// CHECK-NEXT: [[CREATEIMP:%.*]] = acc.create varPtr([[ARGC]] : memref<10x20xf32>) varType(tensor<10x20xf32>) -> memref<10x20xf32> {implicit = true}
 // CHECK-NEXT: acc.parallel dataOperands([[CREATE]], [[CREATEIMP]] : memref<10xf32>, memref<10x20xf32>) {
 // CHECK-NEXT: }
 // CHECK-NEXT: acc.delete accPtr([[CREATE]] : memref<10xf32>) {dataClause = #acc<data_clause acc_create>}
 // CHECK-NEXT: acc.delete accPtr([[CREATEIMP]] : memref<10x20xf32>) {dataClause = #acc<data_clause acc_create>, implicit = true}
-// CHECK: [[COPYOUTZ:%.*]] = acc.create varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout_zero>}
+// CHECK: [[COPYOUTZ:%.*]] = acc.create varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout_zero>}
 // CHECK-NEXT: acc.parallel dataOperands([[COPYOUTZ]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK-NEXT: acc.copyout accPtr([[COPYOUTZ]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) {dataClause = #acc<data_clause acc_copyout_zero>}
+// CHECK-NEXT: acc.copyout accPtr([[COPYOUTZ]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copyout_zero>}
 // CHECK: [[ATTACH:%.*]] = acc.attach varPtr([[ARGB]] : memref<memref<10xf32>>) -> memref<memref<10xf32>>
 // CHECK-NEXT: acc.parallel dataOperands([[ATTACH]] : memref<memref<10xf32>>) {
 // CHECK-NEXT: }
 // CHECK-NEXT: acc.detach accPtr([[ATTACH]] : memref<memref<10xf32>>) {dataClause = #acc<data_clause acc_attach>}
-// CHECK: [[COPYINP:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varPtrPtr([[ARGB]] : memref<memref<10xf32>>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+// CHECK: [[COPYINP:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) varPtrPtr([[ARGB]] : memref<memref<10xf32>>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
 // CHECK-NEXT: acc.parallel dataOperands([[COPYINP]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK-NEXT: acc.copyout accPtr([[COPYINP]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+// CHECK-NEXT: acc.copyout accPtr([[COPYINP]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
 // CHECK-DAG: [[CON0:%.*]] = arith.constant 0 : index
 // CHECK-DAG: [[CON1:%.*]] = arith.constant 1 : index
 // CHECK-DAG: [[CON4:%.*]] = arith.constant 4 : index
 // CHECK-DAG: [[CON9:%.*]] = arith.constant 9 : index
 // CHECK-DAG: [[CON20:%.*]] = arith.constant 20 : index
 // CHECK: [[BOUNDS1F:%.*]] = acc.bounds lowerbound([[CON0]] : index) upperbound([[CON9]] : index) stride([[CON1]] : index)
-// CHECK-NEXT: [[COPYINF1:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) bounds([[BOUNDS1F]]) -> memref<10xf32> {name = "arrayA[0:9]"}
+// CHECK-NEXT: [[COPYINF1:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) bounds([[BOUNDS1F]]) -> memref<10xf32> {name = "arrayA[0:9]"}
 // CHECK-NEXT: [[BOUNDS2F:%.*]] = acc.bounds lowerbound([[CON1]] : index) upperbound([[CON20]] : index) extent([[CON20]] : index) stride([[CON4]] : index) startIdx([[CON1]] : index) {strideInBytes = true}
-// CHECK-NEXT: [[COPYINF2:%.*]] = acc.copyin varPtr([[ARGC]] : memref<10x20xf32>) bounds([[BOUNDS1F]], [[BOUNDS2F]]) -> memref<10x20xf32>
+// CHECK-NEXT: [[COPYINF2:%.*]] = acc.copyin varPtr([[ARGC]] : memref<10x20xf32>) varType(tensor<10x20xf32>) bounds([[BOUNDS1F]], [[BOUNDS2F]]) -> memref<10x20xf32>
 // CHECK-NEXT: acc.parallel dataOperands([[COPYINF1]], [[COPYINF2]] : memref<10xf32>, memref<10x20xf32>) {
 // CHECK-NEXT: }
 // CHECK: [[BOUNDS1P:%.*]] = acc.bounds lowerbound([[CON4]] : index) upperbound([[CON9]] : index) stride([[CON1]] : index)
-// CHECK-NEXT: [[COPYINPART:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) bounds([[BOUNDS1P]]) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
+// CHECK-NEXT: [[COPYINPART:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) bounds([[BOUNDS1P]]) -> memref<10xf32> {dataClause = #acc<data_clause acc_copy>}
 // CHECK-NEXT: acc.parallel dataOperands([[COPYINPART]] : memref<10xf32>) {
 // CHECK-NEXT: }
-// CHECK-NEXT: acc.copyout accPtr([[COPYINPART]] : memref<10xf32>) bounds([[BOUNDS1P]]) to varPtr([[ARGA]] : memref<10xf32>) {dataClause = #acc<data_clause acc_copy>}
+// CHECK-NEXT: acc.copyout accPtr([[COPYINPART]] : memref<10xf32>) bounds([[BOUNDS1P]]) to varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) {dataClause = #acc<data_clause acc_copy>}
 
 // -----
 
 func.func @testunstructuredclauseops(%a: memref<10xf32>) -> () {
-  %copyin = acc.copyin varPtr(%a : memref<10xf32>) -> memref<10xf32> {structured = false}
+  %copyin = acc.copyin varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {structured = false}
   acc.enter_data dataOperands(%copyin : memref<10xf32>)
 
-  %devptr = acc.getdeviceptr varPtr(%a : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout>}
+  %devptr = acc.getdeviceptr varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout>}
   acc.exit_data dataOperands(%devptr : memref<10xf32>)
-  acc.copyout accPtr(%devptr : memref<10xf32>) to varPtr(%a : memref<10xf32>) {structured = false}
+  acc.copyout accPtr(%devptr : memref<10xf32>) to varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) {structured = false}
 
   return
 }
 
 // CHECK: func.func @testunstructuredclauseops([[ARGA:%.*]]: memref<10xf32>) {
-// CHECK: [[COPYIN:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {structured = false}
+// CHECK: [[COPYIN:%.*]] = acc.copyin varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {structured = false}
 // CHECK-NEXT: acc.enter_data dataOperands([[COPYIN]] : memref<10xf32>)
-// CHECK: [[DEVPTR:%.*]] = acc.getdeviceptr varPtr([[ARGA]] : memref<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout>}
+// CHECK: [[DEVPTR:%.*]] = acc.getdeviceptr varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32> {dataClause = #acc<data_clause acc_copyout>}
 // CHECK-NEXT: acc.exit_data dataOperands([[DEVPTR]] : memref<10xf32>)
-// CHECK-NEXT: acc.copyout accPtr([[DEVPTR]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) {structured = false}
+// CHECK-NEXT: acc.copyout accPtr([[DEVPTR]] : memref<10xf32>) to varPtr([[ARGA]] : memref<10xf32>) varType(tensor<10xf32>) {structured = false}
 
 // -----
 
@@ -1382,7 +1382,7 @@ func.func @host_device_ops(%a: memref<f32>) -> () {
 // -----
 
 func.func @host_data_ops(%a: !llvm.ptr, %ifCond: i1) -> () {
-  %0 = acc.use_device varPtr(%a : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.use_device varPtr(%a : !llvm.ptr) varType(f64) -> !llvm.ptr
   acc.host_data dataOperands(%0: !llvm.ptr) {
   }
   acc.host_data dataOperands(%0: !llvm.ptr) {
@@ -1394,7 +1394,7 @@ func.func @host_data_ops(%a: !llvm.ptr, %ifCond: i1) -> () {
 
 // CHECK-LABEL: func.func @host_data_ops(
 // CHECK-SAME:    %[[A:.*]]: !llvm.ptr, %[[IFCOND:.*]]: i1)
-// CHECK: %[[PTR:.*]] = acc.use_device varPtr(%[[A]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[PTR:.*]] = acc.use_device varPtr(%[[A]] : !llvm.ptr) varType(f64) -> !llvm.ptr
 // CHECK: acc.host_data dataOperands(%[[PTR]] : !llvm.ptr)
 // CHECK: acc.host_data dataOperands(%[[PTR]] : !llvm.ptr) {
 // CHECK: } attributes {if_present}
@@ -1583,14 +1583,14 @@ llvm.mlir.global external @globalvar() { acc.declare = #acc.declare<dataClause =
 
 acc.global_ctor @acc_constructor {
   %0 = llvm.mlir.addressof @globalvar { acc.declare = #acc.declare<dataClause = acc_create> } : !llvm.ptr
-  %1 = acc.create varPtr(%0 : !llvm.ptr) -> !llvm.ptr
+  %1 = acc.create varPtr(%0 : !llvm.ptr) varType(i32) -> !llvm.ptr
   acc.declare_enter dataOperands(%1 : !llvm.ptr)
   acc.terminator
 }
 
 acc.global_dtor @acc_destructor {
   %0 = llvm.mlir.addressof @globalvar { acc.declare = #acc.declare<dataClause = acc_create> } : !llvm.ptr
-  %1 = acc.getdeviceptr varPtr(%0 : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create>}
+  %1 = acc.getdeviceptr varPtr(%0 : !llvm.ptr) varType(i32) -> !llvm.ptr {dataClause = #acc<data_clause acc_create>}
   acc.declare_exit dataOperands(%1 : !llvm.ptr)
   acc.delete accPtr(%1 : !llvm.ptr)
   acc.terminator
@@ -1598,11 +1598,11 @@ acc.global_dtor @acc_destructor {
 
 // CHECK-LABEL: acc.global_ctor @acc_constructor
 // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @globalvar {acc.declare = #acc.declare<dataClause = acc_create>} : !llvm.ptr
-// CHECK-NEXT: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !llvm.ptr) -> !llvm.ptr
+// CHECK-NEXT: %[[CREATE:.*]] = acc.create varPtr(%[[ADDR]] : !llvm.ptr) varType(i32) -> !llvm.ptr
 // CHECK-NEXT: acc.declare_enter dataOperands(%[[CREATE]] : !llvm.ptr)
 // CHECK: acc.global_dtor @acc_destructor
 // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @globalvar {acc.declare = #acc.declare<dataClause = acc_create>} : !llvm.ptr
-// CHECK-NEXT: %[[DELETE:.*]] = acc.getdeviceptr varPtr(%[[ADDR]] : !llvm.ptr) -> !llvm.ptr {dataClause = #acc<data_clause acc_create>}
+// CHECK-NEXT: %[[DELETE:.*]] = acc.getdeviceptr varPtr(%[[ADDR]] : !llvm.ptr) varType(i32) -> !llvm.ptr {dataClause = #acc<data_clause acc_create>}
 // CHECK-NEXT: acc.declare_exit dataOperands(%[[DELETE]] : !llvm.ptr)
 // CHECK-NEXT: acc.delete accPtr(%[[DELETE]] : !llvm.ptr)
 
@@ -1657,12 +1657,12 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
 
   %c20 = arith.constant 20 : i32
   %alloc = llvm.alloca %c20 x i32 { acc.declare = #acc.declare<dataClause = acc_create, implicit = true> } : (i32) -> !llvm.ptr
-  %createlocal = acc.create varPtr(%alloc : !llvm.ptr) -> !llvm.ptr {implicit = true}
+  %createlocal = acc.create varPtr(%alloc : !llvm.ptr) varType(!llvm.array<20 x i32>) -> !llvm.ptr {implicit = true}
 
-  %pa = acc.present varPtr(%a : memref<10x10xf32>) -> memref<10x10xf32>
-  %pb = acc.present varPtr(%b : memref<10x10xf32>) -> memref<10x10xf32>
-  %pc = acc.present varPtr(%c : memref<10xf32>) -> memref<10xf32>
-  %pd = acc.present varPtr(%d : memref<10xf32>) -> memref<10xf32>
+  %pa = acc.present varPtr(%a : memref<10x10xf32>) varType(tensor<10x10xf32>) -> memref<10x10xf32>
+  %pb = acc.present varPtr(%b : memref<10x10xf32>) varType(tensor<10x10xf32>) -> memref<10x10xf32>
+  %pc = acc.present varPtr(%c : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
+  %pd = acc.present varPtr(%d : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.declare dataOperands(%pa, %pb, %pc, %pd, %createlocal: memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>, !llvm.ptr) {
   }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 2a19e4837f550..1fbb4c93e855b 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1634,7 +1634,7 @@ func.func @omp_single_copyprivate(%data_var : memref<i32>) -> () {
 // -----
 
 func.func @omp_task_depend(%data_var: memref<i32>) {
-  // expected-error @below {{op expected as many depend values as depend variables}}
+  // expected-error @below {{'omp.task' op operand count (1) does not match with the total size (0) specified in attribute 'operandSegmentSizes'}}
     "omp.task"(%data_var) ({
       "omp.terminator"() : () -> ()
     }) {depend_kinds = [], operandSegmentSizes = array<i32: 0, 0, 1, 0, 0, 0, 0, 0>} : (memref<i32>) -> ()
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index c25a6ef4b4849..26943068ed95a 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -1975,8 +1975,8 @@ func.func @omp_single_copyprivate(%data_var: memref<i32>) {
 }
 
 // CHECK-LABEL: @omp_task
-// CHECK-SAME: (%[[bool_var:.*]]: i1, %[[i64_var:.*]]: i64, %[[i32_var:.*]]: i32, %[[data_var:.*]]: memref<i32>)
-func.func @omp_task(%bool_var: i1, %i64_var: i64, %i32_var: i32, %data_var: memref<i32>) {
+// CHECK-SAME: (%[[bool_var:.*]]: i1, %[[i64_var:.*]]: i64, %[[i32_var:.*]]: i32, %[[data_var:.*]]: memref<i32>, %[[event_handle:.*]]: !llvm.ptr)
+func.func @omp_task(%bool_var: i1, %i64_var: i64, %i32_var: i32, %data_var: memref<i32>, %event_handle : !llvm.ptr) {
 
   // Checking simple task
   // CHECK: omp.task {
@@ -2054,7 +2054,11 @@ func.func @omp_task(%bool_var: i1, %i64_var: i64, %i32_var: i32, %data_var: memr
     // CHECK: omp.terminator
     omp.terminator
   }
-
+  // Checking detach clause
+  // CHECK: omp.task detach(%[[event_handle]] : !llvm.ptr)
+  omp.task detach(%event_handle : !llvm.ptr){
+     omp.terminator
+  }
   // Checking multiple clauses
   // CHECK: omp.task allocate(%[[data_var]] : memref<i32> -> %[[data_var]] : memref<i32>)
   omp.task allocate(%data_var : memref<i32> -> %data_var : memref<i32>)
@@ -2750,6 +2754,30 @@ func.func @omp_target_private(%map1: memref<?xi32>, %map2: memref<?xi32>, %priv_
   return
 }
 
+// CHECK-LABEL: omp_target_private_with_map_idx
+func.func @omp_target_private_with_map_idx(%map1: memref<?xi32>, %map2: memref<?xi32>, %priv_var: !llvm.ptr) -> () {
+  %mapv1 = omp.map.info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
+  %mapv2 = omp.map.info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // CHECK: omp.target
+
+  // CHECK-SAME: map_entries(
+  // CHECK-SAME:   %{{[^[:space:]]+}} -> %[[MAP1_ARG:[^[:space:]]+]],
+  // CHECK-SAME:   %{{[^[:space:]]+}} -> %[[MAP2_ARG:[^[:space:]]+]]
+  // CHECK-SAME:   : memref<?xi32>, memref<?xi32>
+  // CHECK-SAME: )
+
+  // CHECK-SAME: private(
+  // CHECK-SAME:   @x.privatizer %{{[^[:space:]]+}} -> %[[PRIV_ARG:[^[:space:]]+]] [map_idx=1]
+  // CHECK-SAME:   : !llvm.ptr
+  // CHECK-SAME: )
+  omp.target map_entries(%mapv1 -> %arg0, %mapv2 -> %arg1 : memref<?xi32>, memref<?xi32>) private(@x.privatizer %priv_var -> %priv_arg [map_idx=1] : !llvm.ptr) {
+    omp.terminator
+  }
+
+  return
+}
+
 // CHECK-LABEL: omp_loop
 func.func @omp_loop(%lb : index, %ub : index, %step : index) {
   // CHECK: omp.loop {
diff --git a/mlir/test/Dialect/SCF/invalid.mlir b/mlir/test/Dialect/SCF/invalid.mlir
index 337eb9eeb8fa5..80576be880127 100644
--- a/mlir/test/Dialect/SCF/invalid.mlir
+++ b/mlir/test/Dialect/SCF/invalid.mlir
@@ -247,7 +247,7 @@ func.func @parallel_more_results_than_reduces(
 
 func.func @parallel_more_results_than_initial_values(
     %arg0 : index, %arg1: index, %arg2: index) {
-  // expected-error@+1 {{'scf.parallel' 0 operands present, but expected 1}}
+  // expected-error@+1 {{'scf.parallel' number of operands and types do not match: got 0 operands and 1 types}}
   %res = scf.parallel (%i0) = (%arg0) to (%arg1) step (%arg2) -> f32 {
     scf.reduce(%arg0 : index) {
       ^bb0(%lhs: index, %rhs: index):
@@ -609,7 +609,7 @@ func.func @wrong_num_results(%in: tensor<100xf32>, %out: tensor<100xf32>) {
   %c1 = arith.constant 1 : index
   %num_threads = arith.constant 100 : index
 
-  // expected-error @+1 {{1 operands present, but expected 2}}
+  // expected-error@+1 {{number of operands and types do not match: got 1 operands and 2 types}}
   %result:2 = scf.forall (%thread_idx) in (%num_threads) shared_outs(%o = %out) -> (tensor<100xf32>, tensor<100xf32>) {
       %1 = tensor.extract_slice %in[%thread_idx][1][1] : tensor<100xf32> to tensor<1xf32>
       scf.forall.in_parallel {
diff --git a/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
index 9ab03da1c9a94..65e1017e62c1a 100644
--- a/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/SCF/value-bounds-op-interface-impl.mlir
@@ -107,6 +107,42 @@ func.func @scf_for_swapping_yield(%t1: tensor<?xf32>, %t2: tensor<?xf32>, %a: in
 
 // -----
 
+// CHECK-LABEL: func @scf_forall(
+//  CHECK-SAME:     %[[a:.*]]: index, %[[b:.*]]: index, %[[c:.*]]: index
+//       CHECK:   "test.some_use"(%[[a]], %[[b]])
+func.func @scf_forall(%a: index, %b: index, %c: index) {
+  scf.forall (%iv) = (%a) to (%b) step (%c) {
+    %0 = "test.reify_bound"(%iv) {type = "LB"} : (index) -> (index)
+    %1 = "test.reify_bound"(%iv) {type = "UB"} : (index) -> (index)
+    "test.some_use"(%0, %1) : (index, index) -> ()
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @scf_forall_tensor_result(
+//  CHECK-SAME:     %[[size:.*]]: index, %[[a:.*]]: index, %[[b:.*]]: index, %[[c:.*]]: index
+//       CHECK:   "test.some_use"(%[[size]])
+//       CHECK:   "test.some_use"(%[[size]])
+func.func @scf_forall_tensor_result(%size: index, %a: index, %b: index, %c: index) {
+  %cst = arith.constant 5.0 : f32
+  %empty = tensor.empty(%size) : tensor<?xf32>
+  %0 = scf.forall (%iv) = (%a) to (%b) step (%c) shared_outs(%arg = %empty) -> tensor<?xf32> {
+    %filled = linalg.fill ins(%cst : f32) outs(%arg : tensor<?xf32>) -> tensor<?xf32>
+    %1 = "test.reify_bound"(%arg) {type = "EQ", dim = 0} : (tensor<?xf32>) -> (index)
+    "test.some_use"(%1) : (index) -> ()
+    scf.forall.in_parallel {
+      tensor.parallel_insert_slice %filled into %arg[0][%size][1] : tensor<?xf32> into tensor<?xf32>
+    }
+  }
+  %2 = "test.reify_bound"(%0) {type = "EQ", dim = 0} : (tensor<?xf32>) -> (index)
+  "test.some_use"(%2) : (index) -> ()
+  return
+}
+
+// -----
+
 // CHECK-LABEL: func @scf_if_constant(
 func.func @scf_if_constant(%c : i1) {
   // CHECK: arith.constant 4 : index
diff --git a/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir b/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
index 5aef6135afd97..57ff94762ff68 100644
--- a/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
@@ -57,7 +57,7 @@ func.func @access_chain_non_composite() -> () {
 
 func.func @access_chain_no_indices(%index0 : i32) -> () {
   %0 = spirv.Variable : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>
-  // expected-error @+1 {{custom op 'spirv.AccessChain' 0 operands present, but expected 1}}
+  // expected-error @+1 {{custom op 'spirv.AccessChain' number of operands and types do not match: got 0 operands and 1 types}}
   %1 = spirv.AccessChain %0[] : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>, i32 -> !spirv.ptr<f32, Function>
   return
 }
@@ -75,7 +75,7 @@ func.func @access_chain_missing_comma(%index0 : i32) -> () {
 
 func.func @access_chain_invalid_indices_types_count(%index0 : i32) -> () {
   %0 = spirv.Variable : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>
-  // expected-error @+1 {{custom op 'spirv.AccessChain' 1 operands present, but expected 2}}
+  // expected-error @+1 {{custom op 'spirv.AccessChain' number of operands and types do not match: got 1 operands and 2 types}}
   %1 = spirv.AccessChain %0[%index0] : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>, i32, i32 -> !spirv.ptr<!spirv.array<4xf32>, Function>
   return
 }
@@ -84,7 +84,7 @@ func.func @access_chain_invalid_indices_types_count(%index0 : i32) -> () {
 
 func.func @access_chain_missing_indices_type(%index0 : i32) -> () {
   %0 = spirv.Variable : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>
-  // expected-error @+1 {{custom op 'spirv.AccessChain' 2 operands present, but expected 1}}
+  // expected-error @+1 {{custom op 'spirv.AccessChain' number of operands and types do not match: got 2 operands and 1 types}}
   %1 = spirv.AccessChain %0[%index0, %index0] : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, Function>, i32 -> !spirv.ptr<f32, Function>
   return
 }
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 0b54c207dea84..613ec06633729 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -1382,6 +1382,34 @@ func.func @compose_expand_of_collapse_0_rank_to_collapse(%arg0 : tensor<1x1x1x1x
 
 // -----
 
+func.func @compose_expand_of_collapse_static(%arg0 : tensor<4x32x10x64x2xf16>) -> tensor<4x32x10x128xf16> {
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1], [2], [3, 4]] : tensor<4x32x10x64x2xf16> into tensor<128x10x128xf16>
+  %expanded = tensor.expand_shape %collapsed [[0, 1], [2], [3]] output_shape [4, 32, 10, 128] : tensor<128x10x128xf16> into tensor<4x32x10x128xf16>
+  return %expanded : tensor<4x32x10x128xf16>
+}
+
+// CHECK-LABEL: func @compose_expand_of_collapse_static
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<4x32x10x64x2xf16>
+//      CHECK:   %[[RESULT:.+]] = tensor.collapse_shape %[[ARG0]]
+// CHECK-SAME:     [0], [1], [2], [3, 4]
+//      CHECK:   return %[[RESULT]]
+
+// -----
+
+func.func @compose_expand_of_collapse_dynamic(%arg0 : tensor<4x?x10x64x2xf16>, %arg1 : index) -> tensor<4x?x10x128xf16> {
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1], [2], [3, 4]] : tensor<4x?x10x64x2xf16> into tensor<?x10x128xf16>
+  %expanded = tensor.expand_shape %collapsed [[0, 1], [2], [3]] output_shape [4, %arg1,  10, 128] : tensor<?x10x128xf16> into tensor<4x?x10x128xf16>
+  return %expanded : tensor<4x?x10x128xf16>
+}
+
+// CHECK-LABEL: func @compose_expand_of_collapse_dynamic
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<4x?x10x64x2xf16>
+//      CHECK:   %[[RESULT:.+]] = tensor.collapse_shape %[[ARG0]]
+// CHECK-SAME:     [0], [1], [2], [3, 4]
+//      CHECK:   return %[[RESULT]]
+
+// -----
+
 // CHECK-LABEL: func @zero_rank_reshape_multi
 func.func @zero_rank_reshape_multi(%arg0: tensor<f32>) -> tensor<f32> {
   // CHECK: return %arg0
diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir
index be470ce2af9b3..83cb4b9d4ab24 100644
--- a/mlir/test/Dialect/Tensor/invalid.mlir
+++ b/mlir/test/Dialect/Tensor/invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt <%s -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
 
 // Asking the dimension of a 0-D shape doesn't make sense.
 func.func @dim_0_ranked(%arg : tensor<f32>, %arg1 : index) {
@@ -90,7 +90,7 @@ func.func @tensor.from_elements_wrong_result_type() {
 // -----
 
 func.func @tensor.from_elements_wrong_elements_count() {
-  // expected-error@+2 {{1 operands present, but expected 2}}
+  // expected-error@+2 {{number of operands and types do not match: got 1 operands and 2 types}}
   %c0 = arith.constant 0 : index
   %0 = tensor.from_elements %c0 : tensor<2xindex>
   return
@@ -692,13 +692,21 @@ func.func @pack_invalid_duplicate_element_in_outer_perm(%input: tensor<256x128xf
 // -----
 
 func.func @pack_invalid_output_rank(%input: tensor<256x128xf32>, %output: tensor<64x32x16xf32>) -> tensor<64x32x16xf32> {
-  // expected-error@+1 {{packed rank must equal unpacked rank + tiling factors}}
+  // expected-error@+1 {{packed rank != (unpacked rank + num tiling factors), got 3 != 4}}
   %0 = tensor.pack %input inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %output : tensor<256x128xf32> -> tensor<64x32x16xf32>
   return %0 : tensor<64x32x16xf32>
 }
 
 // -----
 
+func.func @pack_invalid_output_rank(%input: tensor<256x128xf32>, %output: tensor<64x32x16xf32>) -> tensor<256x128xf32> {
+  // expected-error@+1 {{packed rank != (unpacked rank + num tiling factors), got 3 != 4}}
+  %0 = tensor.unpack %output inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %input : tensor<64x32x16xf32> -> tensor<256x128xf32>
+  return %0 : tensor<256x128xf32>
+}
+
+// -----
+
 func.func @unpack_invalid_out_of_bound_outer_perm(%input: tensor<256x128xf32>, %output: tensor<8x8x32x16xf32>) -> tensor<8x8x32x16xf32> {
   // expected-error@+1 {{invalid outer_dims_perm vector}}
   %0 = tensor.unpack %output outer_dims_perm = [2, 1] inner_dims_pos = [0, 1] inner_tiles = [2, 2] into %input : tensor<8x8x32x16xf32> -> tensor<256x128xf32>
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 5ae769090dac6..89af0f7332f5c 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -782,23 +782,23 @@ func.func @fold_extract_shapecast(%arg0 : vector<5x1x3x2xf32>,
 
 // -----
 
-// CHECK-LABEL: fold_extract_shapecast_negative
-//       CHECK:   %[[V:.*]] = vector.shape_cast %{{.*}} : vector<16xf32> to vector<2x4x2xf32>
-//       CHECK:   %[[R:.*]] = vector.extract %[[V]][1] : vector<4x2xf32> from vector<2x4x2xf32>
-//       CHECK:   return %[[R]] : vector<4x2xf32>
-func.func @fold_extract_shapecast_negative(%arg0 : vector<16xf32>) -> vector<4x2xf32> {
-  %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<2x4x2xf32>
-  %r = vector.extract %0[1] : vector<4x2xf32> from vector<2x4x2xf32>
-  return %r : vector<4x2xf32>
+// CHECK-LABEL: fold_extract_shapecast_0d_result
+//  CHECK-SAME: %[[IN:.*]]: vector<1x1x1xf32>
+//       CHECK:   %[[R:.*]] = vector.extract %[[IN]][0, 0, 0] : f32 from vector<1x1x1xf32>
+//       CHECK:   return %[[R]] : f32
+func.func @fold_extract_shapecast_0d_result(%arg0 : vector<1x1x1xf32>) -> f32 {
+  %0 = vector.shape_cast %arg0 : vector<1x1x1xf32> to vector<f32>
+  %r = vector.extract %0[] : f32 from vector<f32>
+  return %r : f32
 }
 
 // -----
 
-// CHECK-LABEL: dont_fold_0d_extract_shapecast
-//       CHECK:   %[[V:.*]] = vector.shape_cast %{{.*}} : vector<f32> to vector<1xf32>
-//       CHECK:   %[[R:.*]] = vector.extract %[[V]][0] : f32 from vector<1xf32>
+// CHECK-LABEL: fold_extract_shapecast_0d_source
+//  CHECK-SAME: %[[IN:.*]]: vector<f32>
+//       CHECK:   %[[R:.*]] = vector.extract %[[IN]][] : f32 from vector<f32>
 //       CHECK:   return %[[R]] : f32
-func.func @dont_fold_0d_extract_shapecast(%arg0 : vector<f32>) -> f32 {
+func.func @fold_extract_shapecast_0d_source(%arg0 : vector<f32>) -> f32 {
   %0 = vector.shape_cast %arg0 : vector<f32> to vector<1xf32>
   %r = vector.extract %0[0] : f32 from vector<1xf32>
   return %r : f32
@@ -806,6 +806,18 @@ func.func @dont_fold_0d_extract_shapecast(%arg0 : vector<f32>) -> f32 {
 
 // -----
 
+// CHECK-LABEL: fold_extract_shapecast_negative
+//       CHECK:   %[[V:.*]] = vector.shape_cast %{{.*}} : vector<16xf32> to vector<2x4x2xf32>
+//       CHECK:   %[[R:.*]] = vector.extract %[[V]][1] : vector<4x2xf32> from vector<2x4x2xf32>
+//       CHECK:   return %[[R]] : vector<4x2xf32>
+func.func @fold_extract_shapecast_negative(%arg0 : vector<16xf32>) -> vector<4x2xf32> {
+  %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<2x4x2xf32>
+  %r = vector.extract %0[1] : vector<4x2xf32> from vector<2x4x2xf32>
+  return %r : vector<4x2xf32>
+}
+
+// -----
+
 // CHECK-LABEL: fold_extract_shapecast_to_shapecast
 //  CHECK-SAME: (%[[ARG:.+]]: vector<3x4xf32>)
 //       CHECK:   %[[R:.+]] = vector.shape_cast %[[ARG]] : vector<3x4xf32> to vector<12xf32>
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 00aea3b42841a..1a70791fae125 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1519,6 +1519,14 @@ func.func @expand_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1>,
 
 // -----
 
+func.func @expand_base_scalable(%base: memref<?xf32>, %mask: vector<[16]xi1>, %pass_thru: vector<[16]xf32>) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+1 {{'vector.expandload' op operand #2 must be fixed-length vector of 1-bit signless integer values, but got 'vector<[16]xi1>}}
+  %0 = vector.expandload %base[%c0], %mask, %pass_thru : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32> into vector<[16]xf32>
+}
+
+// -----
+
 func.func @expand_dim_mask_mismatch(%base: memref<?xf32>, %mask: vector<17xi1>, %pass_thru: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.expandload' op expected result dim to match mask dim}}
@@ -1551,6 +1559,14 @@ func.func @compress_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1
 
 // -----
 
+func.func @compress_scalable(%base: memref<?xf32>, %mask: vector<[16]xi1>, %value: vector<[16]xf32>) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+1 {{'vector.compressstore' op operand #2 must be fixed-length vector of 1-bit signless integer values, but got 'vector<[16]xi1>}}
+  vector.compressstore %base[%c0], %mask, %value : memref<?xf32>, vector<[16]xi1>, vector<[16]xf32>
+}
+
+// -----
+
 func.func @compress_dim_mask_mismatch(%base: memref<?xf32>, %mask: vector<17xi1>, %value: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.compressstore' op expected valueToStore dim to match mask dim}}
@@ -1787,7 +1803,7 @@ func.func @deinterleave_scalable_rank_fail(%vec : vector<2x[4]xf32>) {
 // -----
 
 func.func @invalid_from_elements(%a: f32) {
-  // expected-error @+1 {{'vector.from_elements' 1 operands present, but expected 2}}
+  // expected-error @+1 {{'vector.from_elements' number of operands and types do not match: got 1 operands and 2 types}}
   vector.from_elements %a : vector<2xf32>
   return
 }
@@ -1803,6 +1819,14 @@ func.func @invalid_from_elements(%a: f32, %b: i32) {
 
 // -----
 
+func.func @invalid_from_elements_scalable(%a: f32, %b: i32) {
+  // expected-error @+1 {{'result' must be fixed-length vector of any type values, but got 'vector<[2]xf32>'}}
+  vector.from_elements %a, %b : vector<[2]xf32>
+  return
+}
+
+// -----
+
 func.func @invalid_step_0d() {
   // expected-error @+1 {{vector.step' op result #0 must be vector of index values of ranks 1, but got 'vector<f32>'}}
   vector.step : vector<f32>
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
index 721c8a8d5d203..4332e80feed42 100644
--- a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
@@ -42,22 +42,20 @@ func.func @vector_transfer_read_i2() -> vector<3xi2> {
 
 // -----
 
-func.func @vector_constant_mask_maskedload_i2(%passthru: vector<5xi2>) -> vector<3x5xi2> {
+func.func @vector_constant_mask_maskedload_i2(%passthru: vector<5xi2>) -> vector<5xi2> {
   %0 = memref.alloc() : memref<3x5xi2>
-  %cst = arith.constant dense<0> : vector<3x5xi2>
   %mask = vector.constant_mask [3] : vector<5xi1>
   %c0 = arith.constant 0 : index
   %c2 = arith.constant 2 : index
   %1 = vector.maskedload %0[%c2, %c0], %mask, %passthru :
     memref<3x5xi2>, vector<5xi1>, vector<5xi2> into vector<5xi2>
-  %2 = vector.insert %1, %cst [0] : vector<5xi2> into vector<3x5xi2>
-  return %2 : vector<3x5xi2>
+  return %1 : vector<5xi2>
 }
-
 // CHECK-LABEL: func @vector_constant_mask_maskedload_i2(
-// CHECK-SAME: %[[ARG0:.+]]: vector<5xi2>) -> vector<3x5xi2>
+// CHECK-SAME: %[[ARG0:.+]]: vector<5xi2>) -> vector<5xi2>
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<4xi8>
 // CHECK: %[[ORIGINMASK:.+]] = vector.constant_mask [3] : vector<5xi1>
-// CHECK: %[[NEWMASK:.+]] = arith.constant dense<true> : vector<2xi1>
+// CHECK: %[[NEWMASK:.+]] = vector.constant_mask [2] : vector<2xi1>
 // CHECK: %[[VESSEL:.+]] = arith.constant dense<0> : vector<8xi2>
 // CHECK: %[[INSERT1:.+]] = vector.insert_strided_slice %[[ARG0]], %[[VESSEL]]
 // CHECK-SAME: {offsets = [2], strides = [1]} : vector<5xi2> into vector<8xi2>
@@ -123,6 +121,29 @@ func.func @check_unaligned_create_mask_static_i2(%passthru: vector<7xi2>) -> vec
 
 // -----
 
+// This test is similar to @vector_constant_mask_maskedload_i2, but the mask is multi-dimensional.
+func.func @vector_constant_mask_maskedload_i2_multidim(%passthru: vector<5xi2>) -> vector<5xi2> {
+  %0 = memref.alloc() : memref<4x3x5xi2>
+  %mask = vector.constant_mask [2, 2] : vector<3x5xi1>
+  %ext_mask = vector.extract %mask[1] : vector<5xi1> from vector<3x5xi1>
+  %c0 = arith.constant 0 : index
+  %c2 = arith.constant 2 : index
+  %1 = vector.maskedload %0[%c2, %c0, %c0], %ext_mask, %passthru :
+    memref<4x3x5xi2>, vector<5xi1>, vector<5xi2> into vector<5xi2>
+  return %1 : vector<5xi2>
+}
+
+// CHECK-LABEL: func @vector_constant_mask_maskedload_i2_multidim(
+// CHECK: %[[ORIG_MASK:.+]] = vector.constant_mask [2, 2] : vector<3x5xi1>
+// CHECK: vector.extract %[[ORIG_MASK]][1]
+
+// Compressing the mask used for emulated masked load.
+// The innermost dimension is compressed to 2 elements from 5.
+// CHECK: %[[NEW_COMPRESSED_MASK:.+]] = vector.constant_mask [2, 1] : vector<3x2xi1>
+// CHECK: vector.extract %[[NEW_COMPRESSED_MASK]][1]
+
+// -----
+
 func.func @vector_load_i2_dynamic_indexing(%idx1: index, %idx2: index) -> vector<3xi2> {
   %0 = memref.alloc() : memref<3x3xi2>
   %cst = arith.constant dense<0> : vector<3x3xi2>
@@ -252,7 +273,7 @@ func.func @vector_maskedload_i2_dynamic_indexing_mixed(%passthru: vector<3xi2>,
 // CHECK: %[[MASK:.+]] = vector.constant_mask [3] : vector<3xi1>
 // CHECK: %[[LINEAR1:.+]] = affine.apply #map()[%[[IDX]]]
 // CHECK: %[[LINEAR2:.+]] = affine.apply #map1()[%[[IDX]]]
-// CHECK: %[[ONE:.+]] = arith.constant dense<true> : vector<2xi1>
+// CHECK: %[[ONE:.+]] = vector.constant_mask [2] : vector<2xi1>
 // CHECK: %[[ZERO:.+]] = arith.constant dense<0> : vector<8xi2>
 
 // Extract passthru vector, and insert into zero vector, this is for constructing a new passthru
@@ -301,7 +322,7 @@ func.func @vector_maskedload_i2_dynamic_indexing_mixed(%passthru: vector<3xi2>,
 
 // -----
 
-func.func @vector_maskedload_i4_constant_mask_unaligned(%passthru: vector<5xi2>) -> vector<5xi2> {
+func.func @vector_maskedload_i2_constant_mask_unaligned(%passthru: vector<5xi2>) -> vector<5xi2> {
   %0 = memref.alloc() : memref<3x5xi2>
   %mask = arith.constant dense<[false, true, true, true, false]> : vector<5xi1>
   %c0 = arith.constant 0 : index
@@ -311,24 +332,23 @@ func.func @vector_maskedload_i4_constant_mask_unaligned(%passthru: vector<5xi2>)
   return %1 : vector<5xi2>
 }
 
-// CHECK: func @vector_maskedload_i4_constant_mask_unaligned(
+// CHECK: func @vector_maskedload_i2_constant_mask_unaligned(
 // CHECK-SAME: %[[PTH:.+]]: vector<5xi2>) -> vector<5xi2>
 // CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<4xi8>
 // CHECK: %[[MASK:.+]] = arith.constant dense<[false, true, true, true, false]> : vector<5xi1>
 
+// Emulated masked load from alloc:
 // CHECK: %[[COMPRESSED_MASK:.+]] = arith.constant dense<true> : vector<2xi1>
 // CHECK: %[[EMPTY:.+]] = arith.constant dense<0> : vector<8xi2>
 // CHECK: %[[PTH_PADDED:.+]] = vector.insert_strided_slice %[[PTH]], %[[EMPTY]]
 // CHECK-SAME: {offsets = [1], strides = [1]} : vector<5xi2> into vector<8xi2>
-
-// Emulated masked load from alloc:
 // CHECK: %[[PTH_PADDED_UPCAST:.+]] = vector.bitcast %[[PTH_PADDED]] : vector<8xi2> to vector<2xi8>
 // CHECK: %[[C1:.+]] = arith.constant 1 : index
 // CHECK: %[[MASKLOAD:.+]] = vector.maskedload %[[ALLOC]][%[[C1]]], %[[COMPRESSED_MASK]], %[[PTH_PADDED_UPCAST]]
 // CHECK: %[[MASKLOAD_DOWNCAST:.+]] = vector.bitcast %[[MASKLOAD]] : vector<2xi8> to vector<8xi2>
 
 // Select from emulated loaded vector and passthru vector:
-// TODO: fold this part if possible.
+// TODO: fold insert_strided_slice into source if possible.
 // CHECK: %[[EMPTY_MASK:.+]] = arith.constant dense<false> : vector<8xi1>
 // CHECK: %[[MASK_PADDED:.+]] = vector.insert_strided_slice %[[MASK]], %[[EMPTY_MASK]]
 // CHECK-SAME: {offsets = [1], strides = [1]} : vector<5xi1> into vector<8xi1>
diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
index 912f448657baa..079aa6f90bf11 100644
--- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll
+++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
@@ -45,6 +45,7 @@ attributes #0 = { readnone }
 ; CHECK-SAME:  !llvm.ptr {llvm.returned}
 ; CHECK-SAME:  !llvm.ptr {llvm.alignstack = 32 : i64}
 ; CHECK-SAME:  !llvm.ptr {llvm.writeonly}
+; CHECK-SAME:  i64 {llvm.range = #llvm.constant_range<i64, 0, 4097>}
 define ptr @func_arg_attrs(
     ptr byval(i64) %arg0,
     ptr byref(i64) %arg1,
@@ -63,7 +64,8 @@ define ptr @func_arg_attrs(
     ptr preallocated(double) %arg16,
     ptr returned %arg17,
     ptr alignstack(32) %arg18,
-    ptr writeonly %arg19) {
+    ptr writeonly %arg19,
+    i64 range(i64 0, 4097) %arg20) {
   ret ptr %arg17
 }
 
@@ -141,6 +143,12 @@ declare inreg ptr @func_res_attr_inreg()
 
 ; // -----
 
+; CHECK-LABEL: @func_res_attr_range
+; CHECK-SAME:  (i64 {llvm.range = #llvm.constant_range<i64, 0, 4097>})
+declare range(i64 0, 4097) i64 @func_res_attr_range()
+
+; // -----
+
 ; CHECK-LABEL: @entry_count
 ; CHECK-SAME:  attributes {function_entry_count = 4242 : i64}
 define void @entry_count() !prof !1 {
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index f75c2e471c94b..51086bc655af2 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2328,6 +2328,16 @@ llvm.func @vararg_function(%arg0: i32, ...) -> i32 {
 
 // -----
 
+// CHECK: declare void @range_arg_function(i64 range(i64 0, 4097))
+llvm.func @range_arg_function(%arg0: i64 {llvm.range = #llvm.constant_range<i64, 0, 4097>})
+
+// -----
+
+// CHECK: declare range(i64 0, 4097) i64 @range_res_function()
+llvm.func @range_res_function() -> (i64 {llvm.range = #llvm.constant_range<i64, 0, 4097>})
+
+// -----
+
 // CHECK: declare void @readonly_function([[PTR:.+]] readonly)
 llvm.func @readonly_function(%arg0: !llvm.ptr {llvm.readonly})
 
diff --git a/mlir/test/Target/LLVMIR/openacc-llvm.mlir b/mlir/test/Target/LLVMIR/openacc-llvm.mlir
index 897311c6e81be..f8fa02c6f3ba0 100644
--- a/mlir/test/Target/LLVMIR/openacc-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openacc-llvm.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
 
 llvm.func @testenterdataop(%arg0: !llvm.ptr, %arg1 : !llvm.ptr) {
-  %0 = acc.create varPtr(%arg0 : !llvm.ptr) -> !llvm.ptr
-  %1 = acc.copyin varPtr(%arg1 : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.create varPtr(%arg0 : !llvm.ptr) varType(f32) -> !llvm.ptr
+  %1 = acc.copyin varPtr(%arg1 : !llvm.ptr) varType(f32) -> !llvm.ptr
   acc.enter_data dataOperands(%0, %1 : !llvm.ptr, !llvm.ptr)
   llvm.return
 }
@@ -48,11 +48,11 @@ llvm.func @testenterdataop(%arg0: !llvm.ptr, %arg1 : !llvm.ptr) {
 
 
 llvm.func @testexitdataop(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
-  %arg0_devptr = acc.getdeviceptr varPtr(%arg0 : !llvm.ptr) -> !llvm.ptr
-  %1 = acc.getdeviceptr varPtr(%arg1 : !llvm.ptr) -> !llvm.ptr
+  %arg0_devptr = acc.getdeviceptr varPtr(%arg0 : !llvm.ptr) varType(f32) -> !llvm.ptr
+  %1 = acc.getdeviceptr varPtr(%arg1 : !llvm.ptr) varType(f32) -> !llvm.ptr
   acc.exit_data dataOperands(%arg0_devptr, %1 : !llvm.ptr, !llvm.ptr)
   acc.delete accPtr(%arg0_devptr : !llvm.ptr)
-  acc.copyout accPtr(%1 : !llvm.ptr) to varPtr(%arg1 : !llvm.ptr)
+  acc.copyout accPtr(%1 : !llvm.ptr) to varPtr(%arg1 : !llvm.ptr) varType(f32)
   llvm.return
 }
 
@@ -95,7 +95,7 @@ llvm.func @testexitdataop(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
 // -----
 
 llvm.func @testupdateop(%arg1: !llvm.ptr) {
-  %0 = acc.update_device varPtr(%arg1 : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.update_device varPtr(%arg1 : !llvm.ptr) varType(f32) -> !llvm.ptr
   acc.update dataOperands(%0 : !llvm.ptr)
   llvm.return
 }
@@ -132,15 +132,15 @@ llvm.func @testupdateop(%arg1: !llvm.ptr) {
 
 llvm.func @testdataop(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr) {
   
-  %0 = acc.copyin varPtr(%arg0 : !llvm.ptr) -> !llvm.ptr
-  %1 = acc.create varPtr(%arg1 : !llvm.ptr) -> !llvm.ptr
+  %0 = acc.copyin varPtr(%arg0 : !llvm.ptr) varType(f32) -> !llvm.ptr
+  %1 = acc.create varPtr(%arg1 : !llvm.ptr) varType(f32) -> !llvm.ptr
   acc.data dataOperands(%0, %1 : !llvm.ptr, !llvm.ptr) {
     %9 = llvm.mlir.constant(2 : i32) : i32
     llvm.store %9, %arg2 : i32, !llvm.ptr
     acc.terminator
   }
-  acc.copyout accPtr(%0 : !llvm.ptr) to varPtr(%arg0 : !llvm.ptr)
-  acc.copyout accPtr(%1 : !llvm.ptr) to varPtr(%arg1 : !llvm.ptr)
+  acc.copyout accPtr(%0 : !llvm.ptr) to varPtr(%arg0 : !llvm.ptr) varType(f32)
+  acc.copyout accPtr(%1 : !llvm.ptr) to varPtr(%arg1 : !llvm.ptr) varType(f32)
   llvm.return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index b40ec0a831d2e..5f8bdf8afdf78 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -2505,6 +2505,23 @@ llvm.mlir.global internal @_QFsubEx() : i32
 
 // -----
 
+// CHECK-LABEL: define void @omp_task_detach
+// CHECK-SAME: (ptr %[[event_handle:.*]])
+llvm.func @omp_task_detach(%event_handle : !llvm.ptr){
+   // CHECK: %[[omp_global_thread_num:.+]] = call i32 @__kmpc_global_thread_num({{.+}})
+   // CHECK: %[[task_data:.+]] = call ptr @__kmpc_omp_task_alloc
+   // CHECK: %[[return_val:.*]] = call ptr @__kmpc_task_allow_completion_event(ptr {{.*}}, i32 %[[omp_global_thread_num]], ptr %[[task_data]])
+   // CHECK: %[[conv:.*]] = ptrtoint ptr %[[return_val]] to i64
+   // CHECK: store i64 %[[conv]], ptr %[[event_handle]], align 4
+   // CHECK: call i32 @__kmpc_omp_task(ptr @{{.+}}, i32 %[[omp_global_thread_num]], ptr %[[task_data]])
+   omp.task detach(%event_handle : !llvm.ptr){
+     omp.terminator
+   }
+   llvm.return
+}
+
+// -----
+
 // CHECK-LABEL: define void @omp_task
 // CHECK-SAME: (i32 %[[x:.+]], i32 %[[y:.+]], ptr %[[zaddr:.+]])
 llvm.func @omp_task(%x: i32, %y: i32, %zaddr: !llvm.ptr) {
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
index 5a506310653c8..fdfcc66b91012 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
@@ -91,12 +91,12 @@ llvm.func @sectionsreduction_(%arg0: !llvm.ptr {fir.bindc_name = "x"}) attribute
 // CHECK:         %[[VAL_14:.*]] = alloca [1 x ptr], align 8
 // CHECK:         br label %[[VAL_15:.*]]
 // CHECK:       omp.reduction.init:                               ; preds = %[[VAL_16:.*]]
+// CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8
 // CHECK:         br label %[[VAL_17:.*]]
 // CHECK:       omp.par.region:                                   ; preds = %[[VAL_15]]
 // CHECK:         br label %[[VAL_18:.*]]
 // CHECK:       omp.par.region1:                                  ; preds = %[[VAL_17]]
 // CHECK:         %[[VAL_19:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
-// CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8
 // CHECK:         br label %[[VAL_22:.*]]
 // CHECK:       omp_section_loop.preheader:                       ; preds = %[[VAL_18]]
 // CHECK:         store i32 0, ptr %[[VAL_7]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
index db9a314b1f5a3..ed7e9fada5fc4 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
@@ -51,11 +51,11 @@ llvm.func @sections_(%arg0: !llvm.ptr {fir.bindc_name = "x"}) attributes {fir.in
 // CHECK:         %[[VAL_21:.*]] = alloca [1 x ptr], align 8
 // CHECK:         br label %[[VAL_22:.*]]
 // CHECK:       omp.reduction.init:                               ; preds = %[[VAL_23:.*]]
+// CHECK:         store float 0.000000e+00, ptr %[[VAL_20]], align 4
 // CHECK:         br label %[[VAL_24:.*]]
 // CHECK:       omp.par.region:                                   ; preds = %[[VAL_22]]
 // CHECK:         br label %[[VAL_25:.*]]
 // CHECK:       omp.par.region1:                                  ; preds = %[[VAL_24]]
-// CHECK:         store float 0.000000e+00, ptr %[[VAL_20]], align 4
 // CHECK:         br label %[[VAL_26:.*]]
 // CHECK:       omp_section_loop.preheader:                       ; preds = %[[VAL_25]]
 // CHECK:         store i32 0, ptr %[[VAL_13]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index de797ea2aa364..c1e0014d1f571 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -635,22 +635,3 @@ llvm.func @wsloop_order(%lb : i32, %ub : i32, %step : i32) {
   }
   llvm.return
 }
-
-// -----
-
-omp.private {type = private} @x.privatizer : !llvm.ptr alloc {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(1 : i32) : i32
-  %1 = llvm.alloca %0 x i32 : (i32) -> !llvm.ptr
-  omp.yield(%1 : !llvm.ptr)
-}
-llvm.func @wsloop_private(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error@below {{not yet implemented: Unhandled clause privatization in omp.wsloop operation}}
-  // expected-error@below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop private(@x.privatizer %x -> %arg0 : !llvm.ptr) {
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      omp.yield
-    }
-  }
-  llvm.return
-}
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir
new file mode 100644
index 0000000000000..4393fafb62efa
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-private-cond_br.mlir
@@ -0,0 +1,54 @@
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+// tests firx for test-suite test: pr69183.f90. Makes sure we can handle inling
+// private ops when the alloca block ends with a conditional branch.
+
+omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : !llvm.ptr alloc {
+^bb0(%arg0: !llvm.ptr):
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+  omp.yield(%1 : !llvm.ptr)
+}
+
+llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {fir.internal_name = "_QPwsloop_private", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %3 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %6 = llvm.mlir.constant(1 : i32) : i32
+  %7 = llvm.mlir.constant(10 : i32) : i32
+  %8 = llvm.mlir.constant(0 : i32) : i32
+  %cond = llvm.mlir.constant(0 : i1) : i1
+  llvm.cond_br %cond, ^bb1, ^bb2
+
+^bb1:
+  llvm.br ^bb3
+
+^bb2:
+  llvm.br ^bb3
+
+^bb3:
+    omp.wsloop private(@_QFwsloop_privateEi_private_ref_i32 %3 -> %arg2 : !llvm.ptr) {
+      omp.loop_nest (%arg4) : i32 = (%8) to (%7) inclusive step (%6) {
+        omp.yield
+      }
+    }
+  llvm.return
+}
+
+// CHECK:   %[[INT:.*]] = alloca i32, i64 1, align 4
+// CHECK:   br label %[[LATE_ALLOC_BB:.*]]
+
+// CHECK: [[LATE_ALLOC_BB]]:
+// CHECK:   br label %[[AFTER_ALLOC_BB:.*]]
+
+// CHECK: [[AFTER_ALLOC_BB]]:
+// CHECK:   br i1 false, label %[[BB1:.*]], label %5[[BB2:.*]]
+
+// CHECK: [[BB1]]:
+// CHECK:   br label %[[BB3:.*]]
+
+// CHECK: [[BB2]]:
+// CHECK:   br label %[[BB3:.*]]
+
+// CHECK: [[BB3]]:
+// CHECK:   br label %omp_loop.preheader
+
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir
new file mode 100644
index 0000000000000..db67bb5fae58b
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-private.mlir
@@ -0,0 +1,88 @@
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+// tests a wsloop private + firstprivate + reduction to make sure block structure
+// is handled properly.
+
+omp.private {type = private} @_QFwsloop_privateEi_private_ref_i32 : !llvm.ptr alloc {
+^bb0(%arg0: !llvm.ptr):
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+  omp.yield(%1 : !llvm.ptr)
+}
+
+llvm.func @foo_free(!llvm.ptr)
+
+omp.private {type = firstprivate} @_QFwsloop_privateEc_firstprivate_ref_c8 : !llvm.ptr alloc {
+^bb0(%arg0: !llvm.ptr):
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x !llvm.array<1 x i8> {bindc_name = "c", pinned} : (i64) -> !llvm.ptr
+  omp.yield(%1 : !llvm.ptr)
+} copy {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  %0 = llvm.load %arg0 : !llvm.ptr -> !llvm.array<1 x i8>
+  llvm.store %0, %arg1 : !llvm.array<1 x i8>, !llvm.ptr
+  omp.yield(%arg1 : !llvm.ptr)
+} dealloc {
+^bb0(%arg0: !llvm.ptr):
+  llvm.call @foo_free(%arg0) : (!llvm.ptr) -> ()
+  omp.yield
+}
+
+omp.declare_reduction @max_f32 : f32 init {
+^bb0(%arg0: f32):
+  %0 = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+  omp.yield(%0 : f32)
+} combiner {
+^bb0(%arg0: f32, %arg1: f32):
+  %0 = llvm.intr.maxnum(%arg0, %arg1) {fastmathFlags = #llvm.fastmath<contract>} : (f32, f32) -> f32
+  omp.yield(%0 : f32)
+}
+
+llvm.func @wsloop_private_(%arg0: !llvm.ptr {fir.bindc_name = "y"}) attributes {fir.internal_name = "_QPwsloop_private", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x f32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x !llvm.array<1 x i8> {bindc_name = "c"} : (i64) -> !llvm.ptr
+  %6 = llvm.mlir.constant(1 : i32) : i32
+  %7 = llvm.mlir.constant(10 : i32) : i32
+  %8 = llvm.mlir.constant(0 : i32) : i32
+  omp.parallel {
+    omp.wsloop private(@_QFwsloop_privateEc_firstprivate_ref_c8 %5 -> %arg1, @_QFwsloop_privateEi_private_ref_i32 %3 -> %arg2 : !llvm.ptr, !llvm.ptr) reduction(@max_f32 %1 -> %arg3 : !llvm.ptr) {
+      omp.loop_nest (%arg4) : i32 = (%8) to (%7) inclusive step (%6) {
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// CHECK: call void {{.*}} @__kmpc_fork_call(ptr @1, i32 1, ptr @[[OUTLINED:.*]], ptr %{{.*}})
+
+// CHECK: define internal void @[[OUTLINED:.*]]{{.*}} {
+
+// First, check that all memory for privates and reductions is allocated.
+// CHECK: omp.par.entry:
+// CHECK:   %[[CHR:.*]] = alloca [1 x i8], i64 1, align 1
+// CHECK:   %[[INT:.*]] = alloca i32, i64 1, align 4
+// CHECK:   %[[FLT:.*]] = alloca float, align 4
+// CHECK:   %[[RED_ARR:.*]] = alloca [1 x ptr], align 8
+// CHECK:   br label %[[LATE_ALLOC_BB:.*]]
+
+// CHECK: [[LATE_ALLOC_BB]]:
+// CHECK:   br label %[[PRIVATE_CPY_BB:.*]]
+
+// Second, check that first private was properly copied.
+// CHECK: [[PRIVATE_CPY_BB:.*]]:
+// CHECK:   %[[CHR_VAL:.*]] = load [1 x i8], ptr %{{.*}}, align 1
+// CHECK:   store [1 x i8] %[[CHR_VAL]], ptr %[[CHR]], align 1
+// CHECK:   br label %[[RED_INIT_BB:.*]]
+
+// Third, check that reduction init took place.
+// CHECK: [[RED_INIT_BB]]:
+// CHECK:   store float 0x{{.*}}, ptr %[[FLT]], align 4
+
+// Finally, check for the private dealloc region
+// CHECK:   call void @foo_free(ptr %[[CHR]])
+
+// CHECK: }
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
index 1a5065fec026e..7dde3846da649 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
@@ -51,11 +51,11 @@
   llvm.func @free(%arg0 : !llvm.ptr) -> ()
 
 // Private reduction variable and its initialization.
-// CHECK: %[[MALLOC_I:.+]] = call ptr @malloc(i64 4)
 // CHECK: %[[PRIV_PTR_I:.+]] = alloca ptr
+// CHECK: %[[PRIV_PTR_J:.+]] = alloca ptr
+// CHECK: %[[MALLOC_I:.+]] = call ptr @malloc(i64 4)
 // CHECK: store ptr %[[MALLOC_I]], ptr %[[PRIV_PTR_I]]
 // CHECK: %[[MALLOC_J:.+]] = call ptr @malloc(i64 4)
-// CHECK: %[[PRIV_PTR_J:.+]] = alloca ptr
 // CHECK: store ptr %[[MALLOC_J]], ptr %[[PRIV_PTR_J]]
 
 // Call to the reduction function.
diff --git a/mlir/test/Transforms/decompose-call-graph-types.mlir b/mlir/test/Transforms/decompose-call-graph-types.mlir
index b8fad63eb4de6..4e641317ac2f3 100644
--- a/mlir/test/Transforms/decompose-call-graph-types.mlir
+++ b/mlir/test/Transforms/decompose-call-graph-types.mlir
@@ -9,10 +9,7 @@
 // CHECK-LABEL:   func @identity(
 // CHECK-SAME:                   %[[ARG0:.*]]: i1,
 // CHECK-SAME:                   %[[ARG1:.*]]: i32) -> (i1, i32) {
-// CHECK:           %[[ARG_MATERIALIZED:.*]] = "test.make_tuple"(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
-// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
-// CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
+// CHECK:           return %[[ARG0]], %[[ARG1]] : i1, i32
 // CHECK-12N-LABEL:   func @identity(
 // CHECK-12N-SAME:                   %[[ARG0:.*]]: i1,
 // CHECK-12N-SAME:                   %[[ARG1:.*]]: i32) -> (i1, i32) {
@@ -56,18 +53,7 @@ func.func @recursive_decomposition(%arg0: tuple<tuple<tuple<i1>>>) -> tuple<tupl
 // CHECK-LABEL:   func @mixed_recursive_decomposition(
 // CHECK-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-SAME:                 %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK:           %[[V0:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK:           %[[V1:.*]] = "test.make_tuple"(%[[ARG0]]) : (i1) -> tuple<i1>
-// CHECK:           %[[V2:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
-// CHECK:           %[[V3:.*]] = "test.make_tuple"(%[[V2]]) : (tuple<i2>) -> tuple<tuple<i2>>
-// CHECK:           %[[V4:.*]] = "test.make_tuple"(%[[V0]], %[[V1]], %[[V3]]) : (tuple<>, tuple<i1>, tuple<tuple<i2>>) -> tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>
-// CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<>
-// CHECK:           %[[V6:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 1 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<i1>
-// CHECK:           %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<i1>) -> i1
-// CHECK:           %[[V8:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 2 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK:           %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK:           %[[V10:.*]] = "test.get_tuple_element"(%[[V9]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
-// CHECK:           return %[[V7]], %[[V10]] : i1, i2
+// CHECK:           return %[[ARG0]], %[[ARG1]] : i1, i2
 // CHECK-12N-LABEL:   func @mixed_recursive_decomposition(
 // CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-12N-SAME:                 %[[ARG1:.*]]: i2) -> (i1, i2) {
@@ -87,14 +73,8 @@ func.func private @callee(tuple<i1, i32>) -> tuple<i1, i32>
 // CHECK-LABEL:   func @caller(
 // CHECK-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
-// CHECK:           %[[ARG_MATERIALIZED:.*]] = "test.make_tuple"(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[CALL_ARG0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
-// CHECK:           %[[CALL_ARG1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
-// CHECK:           %[[DECOMPOSED:.*]]:2 = call @callee(%[[CALL_ARG0]], %[[CALL_ARG1]]) : (i1, i32) -> (i1, i32)
-// CHECK:           %[[CALL_RESULT_RECOMPOSED:.*]] = "test.make_tuple"(%[[DECOMPOSED]]#0, %[[DECOMPOSED]]#1) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
-// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
-// CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
+// CHECK:           %[[V0:.*]]:2 = call @callee(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> (i1, i32)
+// CHECK:           return %[[V0]]#0, %[[V0]]#1 : i1, i32
 // CHECK-12N-LABEL:   func @caller(
 // CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-12N-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
@@ -190,14 +170,8 @@ func.func private @callee(tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) -> (tup
 // CHECK-SAME:                 %[[I4:.*]]: i4,
 // CHECK-SAME:                 %[[I5:.*]]: i5,
 // CHECK-SAME:                 %[[I6:.*]]: i6) -> (i1, i2, i3, i4, i5, i6) {
-// CHECK:           %[[ARG_TUPLE:.*]] = "test.make_tuple"(%[[I4]], %[[I5]]) : (i4, i5) -> tuple<i4, i5>
-// CHECK:           %[[ARG_TUPLE_0:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) <{index = 0 : i32}> : (tuple<i4, i5>) -> i4
-// CHECK:           %[[ARG_TUPLE_1:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) <{index = 1 : i32}> : (tuple<i4, i5>) -> i5
-// CHECK:           %[[CALL:.*]]:6 = call @callee(%[[I1]], %[[I2]], %[[I3]], %[[ARG_TUPLE_0]], %[[ARG_TUPLE_1]], %[[I6]]) : (i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
-// CHECK:           %[[RET_TUPLE:.*]] = "test.make_tuple"(%[[CALL]]#3, %[[CALL]]#4) : (i4, i5) -> tuple<i4, i5>
-// CHECK:           %[[RET_TUPLE_0:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) <{index = 0 : i32}> : (tuple<i4, i5>) -> i4
-// CHECK:           %[[RET_TUPLE_1:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) <{index = 1 : i32}> : (tuple<i4, i5>) -> i5
-// CHECK:           return %[[CALL]]#0, %[[CALL]]#1, %[[CALL]]#2, %[[RET_TUPLE_0]], %[[RET_TUPLE_1]], %[[CALL]]#5 : i1, i2, i3, i4, i5, i6
+// CHECK:           %[[CALL:.*]]:6 = call @callee(%[[I1]], %[[I2]], %[[I3]], %[[I4]], %[[I5]], %[[I6]]) : (i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
+// CHECK:           return %[[CALL]]#0, %[[CALL]]#1, %[[CALL]]#2, %[[CALL]]#3, %[[CALL]]#4, %[[CALL]]#5 : i1, i2, i3, i4, i5, i6
 // CHECK-12N-LABEL:   func @caller(
 // CHECK-12N-SAME:                 %[[I1:.*]]: i1,
 // CHECK-12N-SAME:                 %[[I2:.*]]: i2,
diff --git a/mlir/test/Transforms/invalid-outer-loop-size.mlir b/mlir/test/Transforms/invalid-outer-loop-size.mlir
new file mode 100644
index 0000000000000..7c1e92a4b7ba7
--- /dev/null
+++ b/mlir/test/Transforms/invalid-outer-loop-size.mlir
@@ -0,0 +1,6 @@
+// RUN: not mlir-opt -test-extract-fixed-outer-loops %s 2>&1 | FileCheck %s
+
+func.func @no_crash(%arg0: memref<?x?xf32>) {
+  // CHECK: error: missing `test-outer-loop-sizes` pass-option for outer loop sizes
+  return
+}
diff --git a/mlir/test/Transforms/remove-dead-values.mlir b/mlir/test/Transforms/remove-dead-values.mlir
index 826f6159a36b6..9273ac01e7cce 100644
--- a/mlir/test/Transforms/remove-dead-values.mlir
+++ b/mlir/test/Transforms/remove-dead-values.mlir
@@ -28,22 +28,51 @@ module @named_module_acceptable {
 
 // -----
 
-// The IR remains untouched because of the presence of a branch op `cf.cond_br`.
+// The IR contains both conditional and unconditional branches with a loop
+// in which the last cf.cond_br is referncing the first cf.br
 //
-func.func @dont_touch_unacceptable_ir_has_cleanable_simple_op_with_branch_op(%arg0: i1) {
+func.func @acceptable_ir_has_cleanable_loop_of_conditional_and_branch_op(%arg0: i1) {
   %non_live = arith.constant 0 : i32
-  // expected-error @+1 {{cannot optimize an IR with branch ops}}
-  cf.cond_br %arg0, ^bb1(%non_live : i32), ^bb2(%non_live : i32)
-^bb1(%non_live_0 : i32):
-  cf.br ^bb3
-^bb2(%non_live_1 : i32):
-  cf.br ^bb3
-^bb3:
+  // CHECK-NOT: arith.constant
+  cf.br ^bb1(%non_live : i32)
+  // CHECK: cf.br ^[[BB1:bb[0-9]+]]
+^bb1(%non_live_1 : i32):
+  // CHECK: ^[[BB1]]:
+  %non_live_5 = arith.constant 1 : i32
+  cf.br ^bb3(%non_live_1, %non_live_5 : i32, i32)
+  // CHECK: cf.br ^[[BB3:bb[0-9]+]]
+  // CHECK-NOT: i32
+^bb3(%non_live_2 : i32, %non_live_6 : i32):
+  // CHECK: ^[[BB3]]:
+  cf.cond_br %arg0, ^bb1(%non_live_2 : i32), ^bb4(%non_live_2 : i32)
+  // CHECK: cf.cond_br %arg0, ^[[BB1]], ^[[BB4:bb[0-9]+]]
+^bb4(%non_live_4 : i32):
+  // CHECK: ^[[BB4]]:
   return
 }
 
 // -----
 
+// Checking that iter_args are properly handled
+//
+func.func @cleanable_loop_iter_args_value(%arg0: index) -> index {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  %non_live = arith.constant 0 : index
+  // CHECK: [[RESULT:%.+]] = scf.for [[ARG_1:%.*]] = %c0 to %c10 step %c1 iter_args([[ARG_2:%.*]] = %arg0) -> (index) {
+  %result, %result_non_live = scf.for %i = %c0 to %c10 step %c1 iter_args(%live_arg = %arg0, %non_live_arg = %non_live) -> (index, index) {
+    // CHECK: [[SUM:%.+]] = arith.addi [[ARG_2]], [[ARG_1]] : index
+    %new_live = arith.addi %live_arg, %i : index
+    // CHECK: scf.yield [[SUM:%.+]]
+    scf.yield %new_live, %non_live_arg : index, index
+  }
+  // CHECK: return [[RESULT]] : index
+  return %result : index
+}
+
+// -----
+
 // Note that this cleanup cannot be done by the `canonicalize` pass.
 //
 // CHECK-LABEL: func.func private @clean_func_op_remove_argument_and_return_value() {
@@ -377,3 +406,8 @@ func.func @kernel(%arg0: memref<18xf32>) {
 
 // CHECK: func.func private @no_block_func_declaration()
 func.func private @no_block_func_declaration() -> ()
+
+// -----
+
+// CHECK: llvm.func @no_block_external_func()
+llvm.func @no_block_external_func() attributes {sym_visibility = "private"}
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index e05f444afa68f..d98a6a036e6b1 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -472,3 +472,14 @@ func.func @circular_mapping() {
   %0 = "test.erase_op"() : () -> (i64)
   "test.drop_operands_and_replace_with_valid"(%0) : (i64) -> ()
 }
+
+// -----
+
+func.func @test_1_to_n_block_signature_conversion() {
+  "test.duplicate_block_args"() ({
+  ^bb0(%arg0: i64):
+    "test.repetitive_1_to_n_consumer"(%arg0) : (i64) -> ()
+  }) {} : () -> ()
+  "test.return"() : () -> ()
+}
+
diff --git a/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp b/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
index de511c58ae6ee..09c5b4b2a0ad5 100644
--- a/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
+++ b/mlir/test/lib/Dialect/Func/TestDecomposeCallGraphTypes.cpp
@@ -9,7 +9,7 @@
 #include "TestDialect.h"
 #include "TestOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/Func/Transforms/DecomposeCallGraphTypes.h"
+#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -142,7 +142,10 @@ struct TestDecomposeCallGraphTypes
     typeConverter.addArgumentMaterialization(buildMakeTupleOp);
     typeConverter.addTargetMaterialization(buildDecomposeTuple);
 
-    populateDecomposeCallGraphTypesPatterns(context, typeConverter, patterns);
+    populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
+        patterns, typeConverter);
+    populateReturnOpTypeConversionPattern(patterns, typeConverter);
+    populateCallOpTypeConversionPattern(patterns, typeConverter);
 
     if (failed(applyPartialConversion(module, target, std::move(patterns))))
       return signalPassFailure();
diff --git a/mlir/test/lib/Dialect/SCF/TestLoopParametricTiling.cpp b/mlir/test/lib/Dialect/SCF/TestLoopParametricTiling.cpp
index 1f33a04b0668a..e7e2f70ba1197 100644
--- a/mlir/test/lib/Dialect/SCF/TestLoopParametricTiling.cpp
+++ b/mlir/test/lib/Dialect/SCF/TestLoopParametricTiling.cpp
@@ -40,6 +40,13 @@ class SimpleParametricLoopTilingPass
   }
 
   void runOnOperation() override {
+    if (sizes.empty()) {
+      emitError(
+          UnknownLoc::get(&getContext()),
+          "missing `test-outer-loop-sizes` pass-option for outer loop sizes");
+      signalPassFailure();
+      return;
+    }
     getOperation()->walk([this](scf::ForOp op) {
       // Ignore nested loops.
       if (op->getParentRegion()->getParentOfType<scf::ForOp>())
diff --git a/mlir/test/lib/Dialect/Test/TestAttrDefs.td b/mlir/test/lib/Dialect/Test/TestAttrDefs.td
index 907184b2e1ce4..0fd272f85d39b 100644
--- a/mlir/test/lib/Dialect/Test/TestAttrDefs.td
+++ b/mlir/test/lib/Dialect/Test/TestAttrDefs.td
@@ -226,6 +226,12 @@ def TestAttrWithOptionalUnsigned : Test_Attr<"TestAttrWithOptionalUnsigned"> {
   let mnemonic = "attr_with_optional_unsigned";
 }
 
+def TestAttrWithOptionalEnum : Test_Attr<"TestAttrWithOptionalEnum"> {
+  let parameters = (ins OptionalParameter<"std::optional<SimpleEnum>">:$value);
+  let assemblyFormat = "`<` $value `>`";
+  let mnemonic = "attr_with_optional_enum";
+}
+
 def TestAttrUgly : Test_Attr<"TestAttrUgly"> {
   let parameters = (ins "::mlir::Attribute":$attr);
 
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 239d529218026..d24d52f356d88 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1886,6 +1886,11 @@ def LegalOpC : TEST_Op<"legal_op_c">,
   Arguments<(ins I32)>, Results<(outs I32)>;
 def LegalOpD : TEST_Op<"legal_op_d">, Arguments<(ins AnyType)>;
 
+def DuplicateBlockArgsOp : TEST_Op<"duplicate_block_args", [SingleBlock]> {
+  let arguments = (ins UnitAttr:$is_legal);
+  let regions = (region SizedRegion<1>:$body);
+}
+
 // Check that the conversion infrastructure can properly undo the creation of
 // operations where an operation was created before its parent, in this case,
 // in the parent's builder.
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index bbd55938718fe..8a0bc597c56be 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -982,9 +982,25 @@ struct TestPassthroughInvalidOp : public ConversionPattern {
   TestPassthroughInvalidOp(MLIRContext *ctx)
       : ConversionPattern("test.invalid", 1, ctx) {}
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    rewriter.replaceOpWithNewOp<TestValidOp>(op, std::nullopt, operands,
+    SmallVector<Value> flattened;
+    for (auto it : llvm::enumerate(operands)) {
+      ValueRange range = it.value();
+      if (range.size() == 1) {
+        flattened.push_back(range.front());
+        continue;
+      }
+
+      // This is a 1:N replacement. Insert a test.cast op. (That's what the
+      // argument materialization used to do.)
+      flattened.push_back(
+          rewriter
+              .create<TestCastOp>(op->getLoc(),
+                                  op->getOperand(it.index()).getType(), range)
+              .getResult());
+    }
+    rewriter.replaceOpWithNewOp<TestValidOp>(op, std::nullopt, flattened,
                                              std::nullopt);
     return success();
   }
@@ -1010,23 +1026,13 @@ struct TestSplitReturnType : public ConversionPattern {
   TestSplitReturnType(MLIRContext *ctx)
       : ConversionPattern("test.return", 1, ctx) {}
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
                   ConversionPatternRewriter &rewriter) const final {
     // Check for a return of F32.
     if (op->getNumOperands() != 1 || !op->getOperand(0).getType().isF32())
       return failure();
-
-    // Check if the first operation is a cast operation, if it is we use the
-    // results directly.
-    auto *defOp = operands[0].getDefiningOp();
-    if (auto packerOp =
-            llvm::dyn_cast_or_null<UnrealizedConversionCastOp>(defOp)) {
-      rewriter.replaceOpWithNewOp<TestReturnOp>(op, packerOp.getOperands());
-      return success();
-    }
-
-    // Otherwise, fail to match.
-    return failure();
+    rewriter.replaceOpWithNewOp<TestReturnOp>(op, operands[0]);
+    return success();
   }
 };
 
@@ -1181,6 +1187,47 @@ class TestEraseOp : public ConversionPattern {
   }
 };
 
+/// This pattern matches a test.duplicate_block_args op and duplicates all
+/// block arguments.
+class TestDuplicateBlockArgs
+    : public OpConversionPattern<DuplicateBlockArgsOp> {
+  using OpConversionPattern<DuplicateBlockArgsOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(DuplicateBlockArgsOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (op.getIsLegal())
+      return failure();
+    rewriter.startOpModification(op);
+    Block *body = &op.getBody().front();
+    TypeConverter::SignatureConversion result(body->getNumArguments());
+    for (auto it : llvm::enumerate(body->getArgumentTypes()))
+      result.addInputs(it.index(), {it.value(), it.value()});
+    rewriter.applySignatureConversion(body, result, getTypeConverter());
+    op.setIsLegal(true);
+    rewriter.finalizeOpModification(op);
+    return success();
+  }
+};
+
+/// This pattern replaces test.repetitive_1_to_n_consumer ops with a test.valid
+/// op. The pattern supports 1:N replacements and forwards the replacement
+/// values of the single operand as test.valid operands.
+class TestRepetitive1ToNConsumer : public ConversionPattern {
+public:
+  TestRepetitive1ToNConsumer(MLIRContext *ctx)
+      : ConversionPattern("test.repetitive_1_to_n_consumer", 1, ctx) {}
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    // A single operand is expected.
+    if (op->getNumOperands() != 1)
+      return failure();
+    rewriter.replaceOpWithNewOp<TestValidOp>(op, operands.front());
+    return success();
+  }
+};
+
 } // namespace
 
 namespace {
@@ -1263,9 +1310,11 @@ struct TestLegalizePatternDriver
         TestUpdateConsumerType, TestNonRootReplacement,
         TestBoundedRecursiveRewrite, TestNestedOpCreationUndoRewrite,
         TestReplaceEraseOp, TestCreateUnregisteredOp, TestUndoMoveOpBefore,
-        TestUndoPropertiesModification, TestEraseOp>(&getContext());
+        TestUndoPropertiesModification, TestEraseOp,
+        TestRepetitive1ToNConsumer>(&getContext());
     patterns.add<TestDropOpSignatureConversion, TestDropAndReplaceInvalidOp>(
         &getContext(), converter);
+    patterns.add<TestDuplicateBlockArgs>(converter, &getContext());
     mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns,
                                                               converter);
     mlir::populateCallOpTypeConversionPattern(patterns, converter);
@@ -1317,6 +1366,9 @@ struct TestLegalizePatternDriver
     target.addDynamicallyLegalOp<TestOpInPlaceSelfFold>(
         [](TestOpInPlaceSelfFold op) { return op.getFolded(); });
 
+    target.addDynamicallyLegalOp<DuplicateBlockArgsOp>(
+        [](DuplicateBlockArgsOp op) { return op.getIsLegal(); });
+
     // Handle a partial conversion.
     if (mode == ConversionMode::Partial) {
       DenseSet<Operation *> unlegalizedOps;
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index 9d8969edfd90f..f67a24755ac09 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -363,7 +363,7 @@ struct TestVectorTransferCollapseInnerMostContiguousDims
   }
 
   StringRef getDescription() const final {
-    return "Test lowering patterns that reducedes the rank of the vector "
+    return "Test lowering patterns that reduces the rank of the vector "
            "transfer memory and vector operands.";
   }
 
@@ -388,7 +388,7 @@ struct TestVectorSinkPatterns
   StringRef getArgument() const final { return "test-vector-sink-patterns"; }
 
   StringRef getDescription() const final {
-    return "Test lowering patterns that eliminate redundant brodacast "
+    return "Test lowering patterns that eliminate redundant broadcast "
            "and transpose operations.";
   }
 
@@ -617,7 +617,7 @@ struct TestVectorDistribution
 
   Option<bool> propagateDistribution{
       *this, "propagate-distribution",
-      llvm::cl::desc("Test distribution propgation"), llvm::cl::init(false)};
+      llvm::cl::desc("Test distribution propagation"), llvm::cl::init(false)};
 
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
diff --git a/mlir/test/lib/Pass/CMakeLists.txt b/mlir/test/lib/Pass/CMakeLists.txt
index f489b7e51e503..b474c1863f335 100644
--- a/mlir/test/lib/Pass/CMakeLists.txt
+++ b/mlir/test/lib/Pass/CMakeLists.txt
@@ -4,6 +4,7 @@ add_mlir_library(MLIRTestPass
   TestDynamicPipeline.cpp
   TestPassManager.cpp
   TestSPIRVCPURunnerPipeline.cpp
+  TestVulkanRunnerPipeline.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
diff --git a/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp b/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp
new file mode 100644
index 0000000000000..eda9aa9f9efef
--- /dev/null
+++ b/mlir/test/lib/Pass/TestVulkanRunnerPipeline.cpp
@@ -0,0 +1,47 @@
+//===------------------ TestVulkanRunnerPipeline.cpp --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a pipeline for use by mlir-vulkan-runner tests.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"
+#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
+#include "mlir/Dialect/GPU/Transforms/Passes.h"
+#include "mlir/Dialect/MemRef/Transforms/Passes.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
+#include "mlir/Pass/PassManager.h"
+
+using namespace mlir;
+
+namespace {
+
+void buildTestVulkanRunnerPipeline(OpPassManager &passManager) {
+  passManager.addPass(createGpuKernelOutliningPass());
+  passManager.addPass(memref::createFoldMemRefAliasOpsPass());
+
+  ConvertToSPIRVPassOptions convertToSPIRVOptions{};
+  convertToSPIRVOptions.convertGPUModules = true;
+  passManager.addPass(createConvertToSPIRVPass(convertToSPIRVOptions));
+  OpPassManager &modulePM = passManager.nest<spirv::ModuleOp>();
+  modulePM.addPass(spirv::createSPIRVLowerABIAttributesPass());
+  modulePM.addPass(spirv::createSPIRVUpdateVCEPass());
+}
+
+} // namespace
+
+namespace mlir::test {
+void registerTestVulkanRunnerPipeline() {
+  PassPipelineRegistration<>(
+      "test-vulkan-runner-pipeline",
+      "Runs a series of passes for lowering GPU-dialect MLIR to "
+      "SPIR-V-dialect MLIR intended for mlir-vulkan-runner.",
+      buildTestVulkanRunnerPipeline);
+}
+} // namespace mlir::test
diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
index 80d559cc6f730..93de767b55176 100644
--- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
+++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
@@ -18,7 +18,7 @@ func.func @func_exp2f(%a : f64) {
 func.func @exp2f() {
   // CHECK: 2
   %a = arith.constant 1.0 : f64
-  call @func_exp2f(%a) : (f64) -> ()  
+  call @func_exp2f(%a) : (f64) -> ()
 
   // CHECK-NEXT: 4
   %b = arith.constant 2.0 : f64
@@ -240,13 +240,18 @@ func.func @powf() {
   // CHECK-NEXT: -nan
   %k = arith.constant 1.0 : f64
   %k_p = arith.constant 0xfff0000001000000 : f64
-  call @func_powff64(%k, %k_p) : (f64, f64) -> ()  
+  call @func_powff64(%k, %k_p) : (f64, f64) -> ()
 
   // CHECK-NEXT: -nan
   %l = arith.constant 1.0 : f32
   %l_p = arith.constant 0xffffffff : f32
-  call @func_powff32(%l, %l_p) : (f32, f32) -> ()  
-  return  
+  call @func_powff32(%l, %l_p) : (f32, f32) -> ()
+
+  // CHECK-NEXT: 1
+  %zero = arith.constant 0.0 : f32
+  call @func_powff32(%zero, %zero) : (f32, f32) -> ()
+
+  return
 }
 
 // -------------------------------------------------------------------------- //
diff --git a/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir b/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
index f0d1571d58387..77b94698a278e 100644
--- a/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
+++ b/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
@@ -26,7 +26,13 @@ attributes {
   // CHECK: #test.attr_with_optional_signed<-9223372036854775808>
   attr9 = #test.attr_with_optional_signed<-9223372036854775808>,
   // CHECK: #test.attr_with_optional_unsigned<18446744073709551615>
-  attr_10 = #test.attr_with_optional_unsigned<18446744073709551615>
+  attr_10 = #test.attr_with_optional_unsigned<18446744073709551615>,
+  // CHECK: #test.attr_with_optional_enum<>
+  attr_11 = #test.attr_with_optional_enum<>,
+  // CHECK: #test.attr_with_optional_enum<a>
+  attr_12 = #test.attr_with_optional_enum<a>,
+  // CHECK: #test.attr_with_optional_enum<b>
+  attr_13 = #test.attr_with_optional_enum<b>
 }
 
 // CHECK-LABEL: @test_roundtrip_default_parsers_struct
diff --git a/mlir/test/mlir-tblgen/op-default-builder.td b/mlir/test/mlir-tblgen/op-default-builder.td
index 82386f245bc5d..9881f925a1922 100644
--- a/mlir/test/mlir-tblgen/op-default-builder.td
+++ b/mlir/test/mlir-tblgen/op-default-builder.td
@@ -31,9 +31,9 @@ def AOp : NS_Op<"a_op", []> {
 // CHECK-LABEL: AOp declarations
 // Note: `cAttr` below could be conditionally optional and so the generation is
 // currently conservative.
-// CHECK-DAG: ::mlir::Value lhs, some-attr-kind aAttr, some-attr-kind bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-attr-kind dAttr);
+// CHECK-DAG: ::mlir::Value lhs, some-attr-kind aAttr, some-attr-kind bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-attr-kind dAttr = nullptr);
 // CHECK-DAG: ::mlir::Value lhs, some-return-type aAttr, some-return-type bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-return-type dAttr = 7.2);
-// CHECK-DAG: ::mlir::TypeRange resultTypes, ::mlir::Value lhs, some-attr-kind aAttr, some-attr-kind bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-attr-kind dAttr);
+// CHECK-DAG: ::mlir::TypeRange resultTypes, ::mlir::Value lhs, some-attr-kind aAttr, some-attr-kind bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-attr-kind dAttr = nullptr);
 // CHECK-DAG: ::mlir::TypeRange resultTypes, ::mlir::Value lhs, some-return-type aAttr, some-return-type bAttr, /*optional*/some-attr-kind cAttr, /*optional*/some-return-type dAttr = 7.2);
 
 def BOp : NS_Op<"b_op", []> {
diff --git a/mlir/test/mlir-vulkan-runner/addf.mlir b/mlir/test/mlir-vulkan-runner/addf.mlir
index f6162d1441e46..d435f75a28805 100644
--- a/mlir/test/mlir-vulkan-runner/addf.mlir
+++ b/mlir/test/mlir-vulkan-runner/addf.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK: [3.3,  3.3,  3.3,  3.3,  3.3,  3.3,  3.3,  3.3]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/addf_if.mlir b/mlir/test/mlir-vulkan-runner/addf_if.mlir
index 5638bd44682de..8ae995c65e7e8 100644
--- a/mlir/test/mlir-vulkan-runner/addf_if.mlir
+++ b/mlir/test/mlir-vulkan-runner/addf_if.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK: [3.3,  3.3,  3.3,  3.3,  0,  0,  0,  0]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/addi.mlir b/mlir/test/mlir-vulkan-runner/addi.mlir
index 79fefe0ab03f4..7e212a4fb179c 100644
--- a/mlir/test/mlir-vulkan-runner/addi.mlir
+++ b/mlir/test/mlir-vulkan-runner/addi.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK-COUNT-64: [3, 3, 3, 3, 3, 3, 3, 3]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/addi8.mlir b/mlir/test/mlir-vulkan-runner/addi8.mlir
index 0672b48eed6a2..e0b1a8e8875c0 100644
--- a/mlir/test/mlir-vulkan-runner/addi8.mlir
+++ b/mlir/test/mlir-vulkan-runner/addi8.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK-COUNT-64: [3, 3, 3, 3, 3, 3, 3, 3]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/addui_extended.mlir b/mlir/test/mlir-vulkan-runner/addui_extended.mlir
index 9b1f1964b3f95..158541f326be7 100644
--- a/mlir/test/mlir-vulkan-runner/addui_extended.mlir
+++ b/mlir/test/mlir-vulkan-runner/addui_extended.mlir
@@ -1,13 +1,15 @@
 // Make sure that addition with carry produces expected results
 // with and without expansion to primitive add/cmp ops for WebGPU.
 
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
-// RUN: mlir-vulkan-runner %s --vulkan-runner-spirv-webgpu-prepare \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline -spirv-webgpu-prepare \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 42, 0, 42]
 // CHECK: [1, 0, 1, 1]
diff --git a/mlir/test/mlir-vulkan-runner/mulf.mlir b/mlir/test/mlir-vulkan-runner/mulf.mlir
index 1399e30d38f57..22fa034a9b455 100644
--- a/mlir/test/mlir-vulkan-runner/mulf.mlir
+++ b/mlir/test/mlir-vulkan-runner/mulf.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK-COUNT-4: [6, 6, 6, 6]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/smul_extended.mlir b/mlir/test/mlir-vulkan-runner/smul_extended.mlir
index ccad49ecac6c4..2dd31d2ebb9a0 100644
--- a/mlir/test/mlir-vulkan-runner/smul_extended.mlir
+++ b/mlir/test/mlir-vulkan-runner/smul_extended.mlir
@@ -1,13 +1,15 @@
 // Make sure that signed extended multiplication produces expected results
 // with and without expansion to primitive mul/add ops for WebGPU.
 
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
-// RUN: mlir-vulkan-runner %s --vulkan-runner-spirv-webgpu-prepare \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline -spirv-webgpu-prepare \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 1, -2,  1, 1048560, -87620295, -131071,  560969770]
 // CHECK: [0, 0, -1,  0,       0,        -1,       0, -499807318]
diff --git a/mlir/test/mlir-vulkan-runner/subf.mlir b/mlir/test/mlir-vulkan-runner/subf.mlir
index f19607bbed9bf..23496ef3abc00 100644
--- a/mlir/test/mlir-vulkan-runner/subf.mlir
+++ b/mlir/test/mlir-vulkan-runner/subf.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK-COUNT-32: [2.2, 2.2, 2.2, 2.2]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/time.mlir b/mlir/test/mlir-vulkan-runner/time.mlir
index 1e86f66877edf..6a0bfef36793b 100644
--- a/mlir/test/mlir-vulkan-runner/time.mlir
+++ b/mlir/test/mlir-vulkan-runner/time.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils --entry-point-result=void | FileCheck %s
 
 // CHECK: Compute shader execution time
 // CHECK: Command buffer submit time
diff --git a/mlir/test/mlir-vulkan-runner/umul_extended.mlir b/mlir/test/mlir-vulkan-runner/umul_extended.mlir
index 23c71be1c0325..78300d2fd81dd 100644
--- a/mlir/test/mlir-vulkan-runner/umul_extended.mlir
+++ b/mlir/test/mlir-vulkan-runner/umul_extended.mlir
@@ -1,13 +1,15 @@
 // Make sure that unsigned extended multiplication produces expected results
 // with and without expansion to primitive mul/add ops for WebGPU.
 
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
-// RUN: mlir-vulkan-runner %s --vulkan-runner-spirv-webgpu-prepare \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline -spirv-webgpu-prepare \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 1, -2,  1, 1048560, -87620295, -131071, -49]
 // CHECK: [0, 0,  1, -2,       0,     65534, -131070,   6]
diff --git a/mlir/test/mlir-vulkan-runner/vector-deinterleave.mlir b/mlir/test/mlir-vulkan-runner/vector-deinterleave.mlir
index f2214ebc10262..097f3905949d8 100644
--- a/mlir/test/mlir-vulkan-runner/vector-deinterleave.mlir
+++ b/mlir/test/mlir-vulkan-runner/vector-deinterleave.mlir
@@ -1,6 +1,7 @@
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 2]
 // CHECK: [1, 3]
diff --git a/mlir/test/mlir-vulkan-runner/vector-interleave.mlir b/mlir/test/mlir-vulkan-runner/vector-interleave.mlir
index 0846d52a45b11..5dd4abbd1fb19 100644
--- a/mlir/test/mlir-vulkan-runner/vector-interleave.mlir
+++ b/mlir/test/mlir-vulkan-runner/vector-interleave.mlir
@@ -1,6 +1,7 @@
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 2, 1, 3]
 module attributes {
diff --git a/mlir/test/mlir-vulkan-runner/vector-shuffle.mlir b/mlir/test/mlir-vulkan-runner/vector-shuffle.mlir
index 7cf53b54590bc..be97b48b1812e 100644
--- a/mlir/test/mlir-vulkan-runner/vector-shuffle.mlir
+++ b/mlir/test/mlir-vulkan-runner/vector-shuffle.mlir
@@ -1,6 +1,7 @@
-// RUN: mlir-vulkan-runner %s \
-// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
-// RUN:  --entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -test-vulkan-runner-pipeline \
+// RUN:   | mlir-vulkan-runner - \
+// RUN:     --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:     --entry-point-result=void | FileCheck %s
 
 // CHECK: [2, 1, 3, 3]
 module attributes {
diff --git a/mlir/test/python/dialects/func.py b/mlir/test/python/dialects/func.py
index a2014c64d2fa5..6b3932ce64f13 100644
--- a/mlir/test/python/dialects/func.py
+++ b/mlir/test/python/dialects/func.py
@@ -104,3 +104,32 @@ def testFunctionCalls():
 # CHECK:   %1 = call @qux() : () -> f32
 # CHECK:   return
 # CHECK: }
+
+
+# CHECK-LABEL: TEST: testFunctionArgAttrs
+@constructAndPrintInModule
+def testFunctionArgAttrs():
+    foo = func.FuncOp("foo", ([F32Type.get()], []))
+    foo.sym_visibility = StringAttr.get("private")
+    foo2 = func.FuncOp("foo2", ([F32Type.get(), F32Type.get()], []))
+    foo2.sym_visibility = StringAttr.get("private")
+
+    empty_attr = DictAttr.get({})
+    test_attr = DictAttr.get({"test.foo": StringAttr.get("bar")})
+    test_attr2 = DictAttr.get({"test.baz": StringAttr.get("qux")})
+
+    assert len(foo.arg_attrs) == 1
+    assert foo.arg_attrs[0] == empty_attr
+
+    foo.arg_attrs = [test_attr]
+    assert foo.arg_attrs[0]["test.foo"] == StringAttr.get("bar")
+
+    assert len(foo2.arg_attrs) == 2
+    assert foo2.arg_attrs == ArrayAttr.get([empty_attr, empty_attr])
+
+    foo2.arg_attrs = [empty_attr, test_attr2]
+    assert foo2.arg_attrs == ArrayAttr.get([empty_attr, test_attr2])
+
+
+# CHECK: func private @foo(f32 {test.foo = "bar"})
+# CHECK: func private @foo2(f32, f32  {test.baz = "qux"})
diff --git a/mlir/test/python/dialects/python_test.py b/mlir/test/python/dialects/python_test.py
index 948d1225ea489..fd678f8321fd9 100644
--- a/mlir/test/python/dialects/python_test.py
+++ b/mlir/test/python/dialects/python_test.py
@@ -1,12 +1,33 @@
-# RUN: %PYTHON %s | FileCheck %s
+# RUN: %PYTHON %s pybind11 | FileCheck %s
+# RUN: %PYTHON %s nanobind | FileCheck %s
 
+import sys
 from mlir.ir import *
 import mlir.dialects.func as func
 import mlir.dialects.python_test as test
 import mlir.dialects.tensor as tensor
 import mlir.dialects.arith as arith
 
-test.register_python_test_dialect(get_dialect_registry())
+if sys.argv[1] == "pybind11":
+    from mlir._mlir_libs._mlirPythonTestPybind11 import (
+        TestAttr,
+        TestType,
+        TestTensorValue,
+        TestIntegerRankedTensorType,
+    )
+
+    test.register_python_test_dialect(get_dialect_registry(), use_nanobind=False)
+elif sys.argv[1] == "nanobind":
+    from mlir._mlir_libs._mlirPythonTestNanobind import (
+        TestAttr,
+        TestType,
+        TestTensorValue,
+        TestIntegerRankedTensorType,
+    )
+
+    test.register_python_test_dialect(get_dialect_registry(), use_nanobind=True)
+else:
+    raise ValueError("Expected pybind11 or nanobind as argument")
 
 
 def run(f):
@@ -308,7 +329,7 @@ def testOptionalOperandOp():
 @run
 def testCustomAttribute():
     with Context() as ctx, Location.unknown():
-        a = test.TestAttr.get()
+        a = TestAttr.get()
         # CHECK: #python_test.test_attr
         print(a)
 
@@ -325,11 +346,11 @@ def testCustomAttribute():
         print(repr(op2.test_attr))
 
         # The following cast must not assert.
-        b = test.TestAttr(a)
+        b = TestAttr(a)
 
         unit = UnitAttr.get()
         try:
-            test.TestAttr(unit)
+            TestAttr(unit)
         except ValueError as e:
             assert "Cannot cast attribute to TestAttr" in str(e)
         else:
@@ -338,7 +359,7 @@ def testCustomAttribute():
         # The following must trigger a TypeError from our adaptors and must not
         # crash.
         try:
-            test.TestAttr(42)
+            TestAttr(42)
         except TypeError as e:
             assert "Expected an MLIR object" in str(e)
         else:
@@ -347,7 +368,7 @@ def testCustomAttribute():
         # The following must trigger a TypeError from pybind (therefore, not
         # checking its message) and must not crash.
         try:
-            test.TestAttr(42, 56)
+            TestAttr(42, 56)
         except TypeError:
             pass
         else:
@@ -357,12 +378,12 @@ def testCustomAttribute():
 @run
 def testCustomType():
     with Context() as ctx:
-        a = test.TestType.get()
+        a = TestType.get()
         # CHECK: !python_test.test_type
         print(a)
 
         # The following cast must not assert.
-        b = test.TestType(a)
+        b = TestType(a)
         # Instance custom types should have typeids
         assert isinstance(b.typeid, TypeID)
         # Subclasses of ir.Type should not have a static_typeid
@@ -374,7 +395,7 @@ def testCustomType():
 
         i8 = IntegerType.get_signless(8)
         try:
-            test.TestType(i8)
+            TestType(i8)
         except ValueError as e:
             assert "Cannot cast type to TestType" in str(e)
         else:
@@ -383,7 +404,7 @@ def testCustomType():
         # The following must trigger a TypeError from our adaptors and must not
         # crash.
         try:
-            test.TestType(42)
+            TestType(42)
         except TypeError as e:
             assert "Expected an MLIR object" in str(e)
         else:
@@ -392,7 +413,7 @@ def testCustomType():
         # The following must trigger a TypeError from pybind (therefore, not
         # checking its message) and must not crash.
         try:
-            test.TestType(42, 56)
+            TestType(42, 56)
         except TypeError:
             pass
         else:
@@ -405,7 +426,7 @@ def testTensorValue():
     with Context() as ctx, Location.unknown():
         i8 = IntegerType.get_signless(8)
 
-        class Tensor(test.TestTensorValue):
+        class Tensor(TestTensorValue):
             def __str__(self):
                 return super().__str__().replace("Value", "Tensor")
 
@@ -425,9 +446,9 @@ def __str__(self):
 
             # Classes of custom types that inherit from concrete types should have
             # static_typeid
-            assert isinstance(test.TestIntegerRankedTensorType.static_typeid, TypeID)
+            assert isinstance(TestIntegerRankedTensorType.static_typeid, TypeID)
             # And it should be equal to the in-tree concrete type
-            assert test.TestIntegerRankedTensorType.static_typeid == t.type.typeid
+            assert TestIntegerRankedTensorType.static_typeid == t.type.typeid
 
             d = tensor.EmptyOp([1, 2, 3], IntegerType.get_signless(5)).result
             # CHECK: Value(%{{.*}} = tensor.empty() : tensor<1x2x3xi5>)
@@ -491,7 +512,7 @@ def inferReturnTypeComponents():
 @run
 def testCustomTypeTypeCaster():
     with Context() as ctx, Location.unknown():
-        a = test.TestType.get()
+        a = TestType.get()
         assert a.typeid is not None
 
         b = Type.parse("!python_test.test_type")
@@ -500,7 +521,7 @@ def testCustomTypeTypeCaster():
         # CHECK: TestType(!python_test.test_type)
         print(repr(b))
 
-        c = test.TestIntegerRankedTensorType.get([10, 10], 5)
+        c = TestIntegerRankedTensorType.get([10, 10], 5)
         # CHECK: tensor<10x10xi5>
         print(c)
         # CHECK: TestIntegerRankedTensorType(tensor<10x10xi5>)
@@ -511,7 +532,7 @@ def testCustomTypeTypeCaster():
 
             @register_type_caster(c.typeid)
             def type_caster(pytype):
-                return test.TestIntegerRankedTensorType(pytype)
+                return TestIntegerRankedTensorType(pytype)
 
         except RuntimeError as e:
             print(e)
@@ -530,7 +551,7 @@ def type_caster(pytype):
 
         @register_type_caster(c.typeid, replace=True)
         def type_caster(pytype):
-            return test.TestIntegerRankedTensorType(pytype)
+            return TestIntegerRankedTensorType(pytype)
 
         d = tensor.EmptyOp([10, 10], IntegerType.get_signless(5)).result
         # CHECK: tensor<10x10xi5>
diff --git a/mlir/test/python/lib/CMakeLists.txt b/mlir/test/python/lib/CMakeLists.txt
index d7cbbfbc21477..198ed8211e773 100644
--- a/mlir/test/python/lib/CMakeLists.txt
+++ b/mlir/test/python/lib/CMakeLists.txt
@@ -1,7 +1,8 @@
 set(LLVM_OPTIONAL_SOURCES
   PythonTestCAPI.cpp
   PythonTestDialect.cpp
-  PythonTestModule.cpp
+  PythonTestModulePybind11.cpp
+  PythonTestModuleNanobind.cpp
 )
 
 add_mlir_library(MLIRPythonTestDialect
diff --git a/mlir/test/python/lib/PythonTestModuleNanobind.cpp b/mlir/test/python/lib/PythonTestModuleNanobind.cpp
new file mode 100644
index 0000000000000..7c504d04be0d1
--- /dev/null
+++ b/mlir/test/python/lib/PythonTestModuleNanobind.cpp
@@ -0,0 +1,121 @@
+//===- PythonTestModuleNanobind.cpp - PythonTest dialect extension --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This is the nanobind edition of the PythonTest dialect module.
+//===----------------------------------------------------------------------===//
+
+#include <nanobind/nanobind.h>
+#include <nanobind/stl/vector.h>
+
+#include "PythonTestCAPI.h"
+#include "mlir-c/BuiltinAttributes.h"
+#include "mlir-c/BuiltinTypes.h"
+#include "mlir-c/IR.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
+
+namespace nb = nanobind;
+using namespace mlir::python::nanobind_adaptors;
+
+static bool mlirTypeIsARankedIntegerTensor(MlirType t) {
+  return mlirTypeIsARankedTensor(t) &&
+         mlirTypeIsAInteger(mlirShapedTypeGetElementType(t));
+}
+
+NB_MODULE(_mlirPythonTestNanobind, m) {
+  m.def(
+      "register_python_test_dialect",
+      [](MlirContext context, bool load) {
+        MlirDialectHandle pythonTestDialect =
+            mlirGetDialectHandle__python_test__();
+        mlirDialectHandleRegisterDialect(pythonTestDialect, context);
+        if (load) {
+          mlirDialectHandleLoadDialect(pythonTestDialect, context);
+        }
+      },
+      nb::arg("context"), nb::arg("load") = true);
+
+  m.def(
+      "register_dialect",
+      [](MlirDialectRegistry registry) {
+        MlirDialectHandle pythonTestDialect =
+            mlirGetDialectHandle__python_test__();
+        mlirDialectHandleInsertDialect(pythonTestDialect, registry);
+      },
+      nb::arg("registry"));
+
+  mlir_attribute_subclass(m, "TestAttr",
+                          mlirAttributeIsAPythonTestTestAttribute,
+                          mlirPythonTestTestAttributeGetTypeID)
+      .def_classmethod(
+          "get",
+          [](const nb::object &cls, MlirContext ctx) {
+            return cls(mlirPythonTestTestAttributeGet(ctx));
+          },
+          nb::arg("cls"), nb::arg("context").none() = nb::none());
+
+  mlir_type_subclass(m, "TestType", mlirTypeIsAPythonTestTestType,
+                     mlirPythonTestTestTypeGetTypeID)
+      .def_classmethod(
+          "get",
+          [](const nb::object &cls, MlirContext ctx) {
+            return cls(mlirPythonTestTestTypeGet(ctx));
+          },
+          nb::arg("cls"), nb::arg("context").none() = nb::none());
+
+  auto typeCls =
+      mlir_type_subclass(m, "TestIntegerRankedTensorType",
+                         mlirTypeIsARankedIntegerTensor,
+                         nb::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+                             .attr("RankedTensorType"))
+          .def_classmethod(
+              "get",
+              [](const nb::object &cls, std::vector<int64_t> shape,
+                 unsigned width, MlirContext ctx) {
+                MlirAttribute encoding = mlirAttributeGetNull();
+                return cls(mlirRankedTensorTypeGet(
+                    shape.size(), shape.data(), mlirIntegerTypeGet(ctx, width),
+                    encoding));
+              },
+              nb::arg("cls"), nb::arg("shape"), nb::arg("width"),
+              nb::arg("context").none() = nb::none());
+
+  assert(nb::hasattr(typeCls.get_class(), "static_typeid") &&
+         "TestIntegerRankedTensorType has no static_typeid");
+
+  MlirTypeID mlirRankedTensorTypeID = mlirRankedTensorTypeGetTypeID();
+
+  nb::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+      .attr(MLIR_PYTHON_CAPI_TYPE_CASTER_REGISTER_ATTR)(
+          mlirRankedTensorTypeID, nb::arg("replace") = true)(
+          nanobind::cpp_function([typeCls](const nb::object &mlirType) {
+            return typeCls.get_class()(mlirType);
+          }));
+
+  auto valueCls = mlir_value_subclass(m, "TestTensorValue",
+                                      mlirTypeIsAPythonTestTestTensorValue)
+                      .def("is_null", [](MlirValue &self) {
+                        return mlirValueIsNull(self);
+                      });
+
+  nb::module_::import_(MAKE_MLIR_PYTHON_QUALNAME("ir"))
+      .attr(MLIR_PYTHON_CAPI_VALUE_CASTER_REGISTER_ATTR)(
+          mlirRankedTensorTypeID)(
+          nanobind::cpp_function([valueCls](const nb::object &valueObj) {
+            nb::object capsule = mlirApiObjectToCapsule(valueObj);
+            MlirValue v = mlirPythonCapsuleToValue(capsule.ptr());
+            MlirType t = mlirValueGetType(v);
+            // This is hyper-specific in order to exercise/test registering a
+            // value caster from cpp (but only for a single test case; see
+            // testTensorValue python_test.py).
+            if (mlirShapedTypeHasStaticShape(t) &&
+                mlirShapedTypeGetDimSize(t, 0) == 1 &&
+                mlirShapedTypeGetDimSize(t, 1) == 2 &&
+                mlirShapedTypeGetDimSize(t, 2) == 3)
+              return valueCls.get_class()(valueObj);
+            return valueObj;
+          }));
+}
diff --git a/mlir/test/python/lib/PythonTestModule.cpp b/mlir/test/python/lib/PythonTestModulePybind11.cpp
similarity index 96%
rename from mlir/test/python/lib/PythonTestModule.cpp
rename to mlir/test/python/lib/PythonTestModulePybind11.cpp
index a4f538dcb5594..94a5f5178d16e 100644
--- a/mlir/test/python/lib/PythonTestModule.cpp
+++ b/mlir/test/python/lib/PythonTestModulePybind11.cpp
@@ -5,6 +5,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// This is the pybind11 edition of the PythonTest dialect module.
+//===----------------------------------------------------------------------===//
 
 #include "PythonTestCAPI.h"
 #include "mlir-c/BuiltinAttributes.h"
@@ -21,7 +23,7 @@ static bool mlirTypeIsARankedIntegerTensor(MlirType t) {
          mlirTypeIsAInteger(mlirShapedTypeGetElementType(t));
 }
 
-PYBIND11_MODULE(_mlirPythonTest, m) {
+PYBIND11_MODULE(_mlirPythonTestPybind11, m) {
   m.def(
       "register_python_test_dialect",
       [](MlirContext context, bool load) {
diff --git a/mlir/test/python/pass_manager.py b/mlir/test/python/pass_manager.py
index a794a3fc6fa00..ecac57e3302f0 100644
--- a/mlir/test/python/pass_manager.py
+++ b/mlir/test/python/pass_manager.py
@@ -342,6 +342,27 @@ def testPrintIrBeforeAndAfterAll():
         pm.run(module)
 
 
+# CHECK-LABEL: TEST: testPrintIrLargeLimitElements
+@run
+def testPrintIrLargeLimitElements():
+    with Context() as ctx:
+        module = ModuleOp.parse(
+            """
+          module {
+            func.func @main() -> tensor<3xi64> {
+              %0 = arith.constant dense<[1, 2, 3]> : tensor<3xi64>
+              return %0 : tensor<3xi64>
+            }
+          }
+        """
+        )
+        pm = PassManager.parse("builtin.module(canonicalize)")
+        ctx.enable_multithreading(False)
+        pm.enable_ir_printing(large_elements_limit=2)
+        # CHECK:     %[[CST:.*]] = arith.constant dense_resource<__elided__> : tensor<3xi64>
+        pm.run(module)
+
+
 # CHECK-LABEL: TEST: testPrintIrTree
 @run
 def testPrintIrTree():
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 94bc67a1e9609..960f7037a1b61 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -153,6 +153,7 @@ void registerTestTransformDialectEraseSchedulePass();
 void registerTestPassStateExtensionCommunication();
 void registerTestVectorLowerings();
 void registerTestVectorReductionToSPIRVDotProd();
+void registerTestVulkanRunnerPipeline();
 void registerTestWrittenToPass();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
 void registerTestDialectConversionPasses();
@@ -291,6 +292,7 @@ void registerTestPasses() {
   mlir::test::registerTestPassStateExtensionCommunication();
   mlir::test::registerTestVectorLowerings();
   mlir::test::registerTestVectorReductionToSPIRVDotProd();
+  mlir::test::registerTestVulkanRunnerPipeline();
   mlir::test::registerTestWrittenToPass();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
   mlir::test::registerTestDialectConversionPasses();
diff --git a/mlir/tools/mlir-tblgen/EnumsGen.cpp b/mlir/tools/mlir-tblgen/EnumsGen.cpp
index 47ffd291cae68..d11aa9b27c2d8 100644
--- a/mlir/tools/mlir-tblgen/EnumsGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumsGen.cpp
@@ -101,6 +101,27 @@ struct FieldParser<{0}, {0}> {{
     return parser.emitError(loc, "invalid {2} specification: ") << enumKeyword;
   }
 };
+
+/// Support for std::optional, useful in attribute/type definition where the enum is
+/// used as:
+///
+///    let parameters = (ins OptionalParameter<"std::optional<TheEnumName>">:$value);
+template<>
+struct FieldParser<std::optional<{0}>, std::optional<{0}>> {{
+  template <typename ParserT>
+  static FailureOr<std::optional<{0}>> parse(ParserT &parser) {{
+    // Parse the keyword/string containing the enum.
+    std::string enumKeyword;
+    auto loc = parser.getCurrentLocation();
+    if (failed(parser.parseOptionalKeywordOrString(&enumKeyword)))
+      return std::optional<{0}>{{};
+
+    // Symbolize the keyword.
+    if (::std::optional<{0}> attr = {1}::symbolizeEnum<{0}>(enumKeyword))
+      return attr;
+    return parser.emitError(loc, "invalid {2} specification: ") << enumKeyword;
+  }
+};
 } // namespace mlir
 
 namespace llvm {
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 9badb7aa163a6..59ae7777da567 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -3088,6 +3088,26 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
       defaultValuedAttrStartIndex = i;
     }
   }
+
+  // Check if parameters besides default valued one are enough to distinguish
+  // between builders with wrapped and unwrapped arguments.
+  bool hasBuilderAmbiguity = true;
+  for (const auto &arg : op.getArgs()) {
+    auto *namedAttr = dyn_cast<NamedAttribute *>(arg);
+    if (!namedAttr)
+      continue;
+    Attribute attr = namedAttr->attr;
+    if (attr.hasDefaultValue() || attr.isDerivedAttr())
+      continue;
+
+    if (attrParamKind != AttrParamKind::WrappedAttr ||
+        !canUseUnwrappedRawValue(attr))
+      continue;
+
+    hasBuilderAmbiguity = false;
+    break;
+  }
+
   // Avoid generating build methods that are ambiguous due to default values by
   // requiring at least one attribute.
   if (defaultValuedAttrStartIndex < op.getNumArgs()) {
@@ -3098,9 +3118,9 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
         cast<NamedAttribute *>(op.getArg(defaultValuedAttrStartIndex));
     Attribute attr = namedAttr->attr;
     if ((attrParamKind == AttrParamKind::WrappedAttr &&
-         canUseUnwrappedRawValue(attr)) ||
+         canUseUnwrappedRawValue(attr) && hasBuilderAmbiguity) ||
         (attrParamKind == AttrParamKind::UnwrappedValue &&
-         !canUseUnwrappedRawValue(attr))) {
+         !canUseUnwrappedRawValue(attr) && hasBuilderAmbiguity)) {
       ++defaultValuedAttrStartIndex;
       defaultValuedAttrLikeStartIndex = defaultValuedAttrStartIndex;
     }
diff --git a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
index 2dd539ef83481..eab64c97e89e4 100644
--- a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
+++ b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
@@ -12,9 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/ConvertToSPIRV/ConvertToSPIRVPass.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
-#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
 #include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
@@ -30,8 +28,6 @@
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
-#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/ExecutionEngine/JitRunner.h"
 #include "mlir/Pass/Pass.h"
@@ -43,18 +39,7 @@
 
 using namespace mlir;
 
-namespace {
-struct VulkanRunnerOptions {
-  llvm::cl::OptionCategory category{"mlir-vulkan-runner options"};
-  llvm::cl::opt<bool> spirvWebGPUPrepare{
-      "vulkan-runner-spirv-webgpu-prepare",
-      llvm::cl::desc("Run MLIR transforms used when targetting WebGPU"),
-      llvm::cl::cat(category)};
-};
-} // namespace
-
-static LogicalResult runMLIRPasses(Operation *op,
-                                   VulkanRunnerOptions &options) {
+static LogicalResult runMLIRPasses(Operation *op, JitRunnerOptions &) {
   auto module = dyn_cast<ModuleOp>(op);
   if (!module)
     return op->emitOpError("expected a 'builtin.module' op");
@@ -62,18 +47,6 @@ static LogicalResult runMLIRPasses(Operation *op,
   if (failed(applyPassManagerCLOptions(passManager)))
     return failure();
 
-  passManager.addPass(createGpuKernelOutliningPass());
-  passManager.addPass(memref::createFoldMemRefAliasOpsPass());
-
-  ConvertToSPIRVPassOptions convertToSPIRVOptions{};
-  convertToSPIRVOptions.convertGPUModules = true;
-  passManager.addPass(createConvertToSPIRVPass(convertToSPIRVOptions));
-  OpPassManager &modulePM = passManager.nest<spirv::ModuleOp>();
-  modulePM.addPass(spirv::createSPIRVLowerABIAttributesPass());
-  modulePM.addPass(spirv::createSPIRVUpdateVCEPass());
-  if (options.spirvWebGPUPrepare)
-    modulePM.addPass(spirv::createSPIRVWebGPUPreparePass());
-
   passManager.addPass(createConvertGpuLaunchFuncToVulkanLaunchFuncPass());
   passManager.addPass(createFinalizeMemRefToLLVMConversionPass());
   passManager.addPass(createConvertVectorToLLVMPass());
@@ -96,15 +69,8 @@ int main(int argc, char **argv) {
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
 
-  // Initialize runner-specific CLI options. These will be parsed and
-  // initialzied in `JitRunnerMain`.
-  VulkanRunnerOptions options;
-  auto runPassesWithOptions = [&options](Operation *op, JitRunnerOptions &) {
-    return runMLIRPasses(op, options);
-  };
-
   mlir::JitRunnerConfig jitRunnerConfig;
-  jitRunnerConfig.mlirTransformer = runPassesWithOptions;
+  jitRunnerConfig.mlirTransformer = runMLIRPasses;
 
   mlir::DialectRegistry registry;
   registry.insert<mlir::arith::ArithDialect, mlir::LLVM::LLVMDialect,
diff --git a/mlir/unittests/Target/LLVM/CMakeLists.txt b/mlir/unittests/Target/LLVM/CMakeLists.txt
index 6d612548a94c0..0c61d222dedf4 100644
--- a/mlir/unittests/Target/LLVM/CMakeLists.txt
+++ b/mlir/unittests/Target/LLVM/CMakeLists.txt
@@ -1,11 +1,11 @@
+set(LLVM_LINK_COMPONENTS nativecodegen)
+
 add_mlir_unittest(MLIRTargetLLVMTests
   SerializeNVVMTarget.cpp
   SerializeROCDLTarget.cpp
   SerializeToLLVMBitcode.cpp
 )
 
-llvm_map_components_to_libnames(llvm_libs nativecodegen)
-
 target_link_libraries(MLIRTargetLLVMTests
   PRIVATE
   MLIRTargetLLVM
@@ -19,7 +19,6 @@ target_link_libraries(MLIRTargetLLVMTests
   MLIRNVVMToLLVMIRTranslation
   MLIRROCDLToLLVMIRTranslation
   MLIRGPUToLLVMIRTranslation
-  ${llvm_libs}
 )
 
 if (DEFINED LLVM_NATIVE_TARGET)
diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index ec042d01a48a9..61a34d796e974 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -45,6 +45,8 @@ set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
 list(INSERT CMAKE_MODULE_PATH 0
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules"
+  "${CMAKE_CURRENT_SOURCE_DIR}/../runtimes/cmake/Modules"
+  "${LLVM_COMMON_CMAKE_UTILS}"
   "${LLVM_COMMON_CMAKE_UTILS}/Modules"
   )
 
@@ -62,6 +64,9 @@ if (OPENMP_STANDALONE_BUILD)
   # Do not use OPENMP_LIBDIR_SUFFIX directly, use OPENMP_INSTALL_LIBDIR.
   set(OPENMP_INSTALL_LIBDIR "lib${OPENMP_LIBDIR_SUFFIX}")
 
+  # Used by llvm_add_tool() and tests.
+  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_CURRENT_BINARY_DIR})
+
   # Group test settings.
   set(OPENMP_TEST_C_COMPILER ${CMAKE_C_COMPILER} CACHE STRING
     "C compiler to use for testing OpenMP runtime libraries.")
@@ -127,6 +132,7 @@ include(LibomptargetGetDependencies)
 # Set up testing infrastructure.
 include(OpenMPTesting)
 
+include(CheckCXXCompilerFlag)
 check_cxx_compiler_flag(-Werror=global-constructors OFFLOAD_HAVE_WERROR_CTOR)
 
 # LLVM source tree is required at build time for libomptarget
@@ -290,6 +296,26 @@ if(OPENMP_STANDALONE_BUILD)
       ${LLVM_LIBRARY_DIRS}
     REQUIRED
   )
+
+  find_path (
+    LIBOMP_INCLUDE_DIR
+    NAMES
+      omp.h
+    HINTS
+    ${COMPILER_RESOURCE_DIR}/include
+    ${CMAKE_INSTALL_PREFIX}/include
+  )
+
+  get_filename_component(LIBOMP_LIBRARY_DIR ${LIBOMP_STANDALONE} DIRECTORY)
+
+  set(OPENMP_TEST_FLAGS "" CACHE STRING
+    "Extra compiler flags to send to the test compiler.")
+  set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
+    "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
+  set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${LIBOMP_INCLUDE_DIR}" CACHE STRING
+    "Path to folder containing omp.h")
+  set(LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER "${LIBOMP_LIBRARY_DIR}" CACHE STRING
+    "Path to folder containing libomp.so, and libLLVMSupport.so with profiling enabled")
 endif()
 
 macro(pythonize_bool var)
@@ -357,6 +383,9 @@ add_subdirectory(tools)
 # Build target agnostic offloading library.
 add_subdirectory(src)
 
+add_subdirectory(tools/offload-tblgen)
+add_subdirectory(liboffload)
+
 # Add tests.
 add_subdirectory(test)
 
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index c76ad018ab4fe..11176a567a643 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -42,11 +42,16 @@ set(devicertl_base_directory ${CMAKE_CURRENT_SOURCE_DIR})
 set(include_directory ${devicertl_base_directory}/include)
 set(source_directory ${devicertl_base_directory}/src)
 
-set(all_amdgpu_architectures "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906"
-                             "gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx950;gfx1010"
-                             "gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035"
-                             "gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150"
-                             "gfx1151;gfx1152;gfx1153")
+set(all_amdgpu_architectures "gfx700;gfx701;gfx801;gfx803"
+                             "gfx9-generic;gfx900;gfx902;gfx906;gfx908"
+                             "gfx90a;gfx90c"
+                             "gfx9-4-generic;gfx940;gfx941;gfx942;gfx950"
+                             "gfx10-1-generic;gfx1010;gfx1012"
+                             "gfx10-3-generic;gfx1030;gfx1031;gfx1032;gfx1033"
+                             "gfx1034;gfx1035;gfx1036"
+                             "gfx11-generic;gfx1100;gfx1101;gfx1102;gfx1103"
+                             "gfx1150;gfx1151;gfx1152;gfx1153"
+                             "gfx12-generic")
 set(all_nvptx_architectures "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62"
                             "sm_70;sm_72;sm_75;sm_80;sm_86;sm_87;sm_89;sm_90")
 set(all_gpu_architectures
@@ -57,6 +62,10 @@ set(LIBOMPTARGET_DEVICE_ARCHITECTURES "all" CACHE STRING
 
 if(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "all")
   set(LIBOMPTARGET_DEVICE_ARCHITECTURES ${all_gpu_architectures})
+elseif(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "amdgpu")
+  set(LIBOMPTARGET_DEVICE_ARCHITECTURES ${all_amdgpu_architectures})
+elseif(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "nvptx")
+  set(LIBOMPTARGET_DEVICE_ARCHITECTURES ${all_nvptx_architectures})
 elseif(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "auto" OR
        LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "native")
   if(NOT LIBOMPTARGET_NVPTX_ARCH AND NOT LIBOMPTARGET_AMDGPU_ARCH)
@@ -131,6 +140,7 @@ set(bc_flags -c -foffload-lto -std=c++17 -fvisibility=hidden
              -DOMPTARGET_DEVICE_RUNTIME
              -I${include_directory}
              -I${devicertl_base_directory}/../include
+             -I${devicertl_base_directory}/../../libc
              ${LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL}
 )
 
@@ -275,6 +285,7 @@ function(compileDeviceRTLLibrary target_cpu target_name target_triple)
     target_compile_definitions(${ide_target_name} PRIVATE SHARED_SCRATCHPAD_SIZE=512)
     target_include_directories(${ide_target_name} PRIVATE
       ${include_directory}
+      ${devicertl_base_directory}/../../libc
       ${devicertl_base_directory}/../include
       ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
     )
diff --git a/offload/DeviceRTL/src/Misc.cpp b/offload/DeviceRTL/src/Misc.cpp
index 8e690f6fd8e7c..c1df477365bcb 100644
--- a/offload/DeviceRTL/src/Misc.cpp
+++ b/offload/DeviceRTL/src/Misc.cpp
@@ -12,6 +12,8 @@
 #include "Allocator.h"
 #include "Configuration.h"
 #include "DeviceTypes.h"
+#include "Shared/RPCOpcodes.h"
+#include "shared/rpc.h"
 
 #include "Debug.h"
 
@@ -110,6 +112,12 @@ void *indirectCallLookup(void *HstPtr) {
   return HstPtr;
 }
 
+/// The openmp client instance used to communicate with the server.
+/// FIXME: This is marked as 'retain' so that it is not removed via
+/// `-mlink-builtin-bitcode`
+[[gnu::visibility("protected"), gnu::weak,
+  gnu::retain]] rpc::Client Client asm("__llvm_rpc_client");
+
 } // namespace impl
 } // namespace ompx
 
@@ -156,6 +164,20 @@ void omp_free(void *ptr, omp_allocator_handle_t allocator) {
     return;
   }
 }
+
+unsigned long long __llvm_omp_host_call(void *fn, void *data, size_t size) {
+  rpc::Client::Port Port = ompx::impl::Client.open<OFFLOAD_HOST_CALL>();
+  Port.send_n(data, size);
+  Port.send([=](rpc::Buffer *buffer, uint32_t) {
+    buffer->data[0] = reinterpret_cast<uintptr_t>(fn);
+  });
+  unsigned long long Ret;
+  Port.recv([&](rpc::Buffer *Buffer, uint32_t) {
+    Ret = static_cast<unsigned long long>(Buffer->data[0]);
+  });
+  Port.close();
+  return Ret;
+}
 }
 
 ///}
diff --git a/offload/DeviceRTL/src/exports b/offload/DeviceRTL/src/exports
index 288ddf90b4a9f..01667e7aba827 100644
--- a/offload/DeviceRTL/src/exports
+++ b/offload/DeviceRTL/src/exports
@@ -15,4 +15,5 @@ malloc
 free
 memcmp
 printf
+__llvm_rpc_client
 __assert_fail
diff --git a/offload/cmake/OpenMPTesting.cmake b/offload/cmake/OpenMPTesting.cmake
index 6609d6301d0f9..8e955ff399275 100644
--- a/offload/cmake/OpenMPTesting.cmake
+++ b/offload/cmake/OpenMPTesting.cmake
@@ -37,17 +37,6 @@ function(find_standalone_test_dependencies)
     return()
   endif()
 
-  find_program(OFFLOAD_DEVICE_INFO_EXECUTABLE
-    NAMES llvm-offload-device-info
-    PATHS ${OPENMP_LLVM_TOOLS_DIR})
-  if (NOT OFFLOAD_DEVICE_INFO_EXECUTABLE)
-    message(STATUS "Cannot find 'llvm-offload-device-info'.")
-    message(STATUS "Please put 'not' in your PATH, set OFFLOAD_DEVICE_INFO_EXECUTABLE to its full path, or point OPENMP_LLVM_TOOLS_DIR to its directory.")
-    message(WARNING "The check targets will not be available!")
-    set(ENABLE_CHECK_TARGETS FALSE PARENT_SCOPE)
-    return()
-  endif()
-
   find_program(OPENMP_NOT_EXECUTABLE
     NAMES not
     PATHS ${OPENMP_LLVM_TOOLS_DIR})
@@ -82,8 +71,9 @@ else()
     set(OPENMP_FILECHECK_EXECUTABLE ${LLVM_RUNTIME_OUTPUT_INTDIR}/FileCheck)
   endif()
   set(OPENMP_NOT_EXECUTABLE ${LLVM_RUNTIME_OUTPUT_INTDIR}/not)
-  set(OFFLOAD_DEVICE_INFO_EXECUTABLE ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-offload-device-info)
 endif()
+set(OFFLOAD_DEVICE_INFO_EXECUTABLE ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-offload-device-info)
+set(OFFLOAD_TBLGEN_EXECUTABLE ${LLVM_RUNTIME_OUTPUT_INTDIR}/offload-tblgen)
 
 # Macro to extract information about compiler from file. (no own scope)
 macro(extract_test_compiler_information lang file)
@@ -136,7 +126,7 @@ if (${OPENMP_STANDALONE_BUILD})
   # project is built which is too late for detecting the compiler...
   file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DetectTestCompiler)
   execute_process(
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/DetectTestCompiler
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${CMAKE_CURRENT_SOURCE_DIR}/../openmp/cmake/DetectTestCompiler
       -DCMAKE_C_COMPILER=${OPENMP_TEST_C_COMPILER}
       -DCMAKE_CXX_COMPILER=${OPENMP_TEST_CXX_COMPILER}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DetectTestCompiler
diff --git a/offload/cmake/caches/AMDGPUBot.cmake b/offload/cmake/caches/AMDGPUBot.cmake
new file mode 100644
index 0000000000000..d72b620ae3080
--- /dev/null
+++ b/offload/cmake/caches/AMDGPUBot.cmake
@@ -0,0 +1,17 @@
+# This file is meant for test builds on one basic AMDGPU buildbot only.
+
+# Install directory set to /tmp as this is a bot config
+set(CMAKE_INSTALL_PREFIX /tmp/llvm.install.test CACHE STRING "")
+
+set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;openmp;offload" CACHE STRING "")
+set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
+set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
+set(LLVM_LIT_ARGS "-v --show-unsupported --timeout 100 --show-xfail -j 32" CACHE STRING "")
+set(LLVM_TARGETS_TO_BUILD "host;AMDGPU" CACHE STRING "")
+
+set(CLANG_DEFAULT_LINKER "lld" CACHE STRING "")
+
+set(CMAKE_BUILD_TYPE Release CACHE STRING "")
+set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "")
diff --git a/offload/include/Shared/RPCOpcodes.h b/offload/include/Shared/RPCOpcodes.h
new file mode 100644
index 0000000000000..beee29df1f707
--- /dev/null
+++ b/offload/include/Shared/RPCOpcodes.h
@@ -0,0 +1,25 @@
+//===-- Shared/RPCOpcodes.h - Offload specific RPC opcodes ----- C++ ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines RPC opcodes that are specifically used by the OpenMP device runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_RPC_OPCODES_H
+#define OMPTARGET_SHARED_RPC_OPCODES_H
+
+#define LLVM_OFFLOAD_RPC_BASE 'o'
+#define LLVM_OFFLOAD_OPCODE(n) (LLVM_OFFLOAD_RPC_BASE << 24 | n)
+
+typedef enum {
+  OFFLOAD_HOST_CALL = LLVM_OFFLOAD_OPCODE(0),
+} offload_opcode_t;
+
+#undef LLVM_OFFLOAD_OPCODE
+
+#endif // OMPTARGET_SHARED_RPC_OPCODES_H
diff --git a/offload/liboffload/API/APIDefs.td b/offload/liboffload/API/APIDefs.td
new file mode 100644
index 0000000000000..60c1b85d26911
--- /dev/null
+++ b/offload/liboffload/API/APIDefs.td
@@ -0,0 +1,212 @@
+//===-- APIDefs.td - Base definitions for Offload tablegen -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the class definitions used to implement the Offload API,
+// as well as helper functions used to help populate relevant records.
+// See offload/API/README.md for more detailed documentation.
+//
+//===----------------------------------------------------------------------===//
+
+// Prefix for API naming. This could be hard-coded in the future when a value
+// is agreed upon.
+defvar PREFIX = "OL";
+defvar prefix = !tolower(PREFIX);
+
+// Parameter flags
+defvar PARAM_IN = 0x1;
+defvar PARAM_OUT = 0x2;
+defvar PARAM_OPTIONAL = 0x4;
+defvar PARAM_IN_OPTIONAL = !or(PARAM_IN, PARAM_OPTIONAL);
+defvar PARAM_OUT_OPTIONAL = !or(PARAM_OUT, PARAM_OPTIONAL);
+
+// Does the type end with '_handle_t'?
+class IsHandleType<string Type> {
+  // size("_handle_t") == 9
+  bit ret = !if(!lt(!size(Type), 9), 0,
+                !ne(!find(Type, "_handle_t", !sub(!size(Type), 9)), -1));
+}
+
+// Does the type end with '*'?
+class IsPointerType<string Type> {
+  bit ret = !ne(!find(Type, "*", !sub(!size(Type), 1)), -1);
+}
+
+// Describes the valid range of a pointer parameter that reperesents an array
+class Range<string Begin, string End> {
+  string begin = Begin;
+  string end = End;
+}
+
+// Names the parameters that indicate the type and size of the data pointed to
+// by an opaque pointer parameter
+class TypeInfo<string TypeEnum, string TypeSize> {
+  string enum = TypeEnum;
+  string size = TypeSize;
+}
+
+class Param<string Type, string Name, string Desc, bits<3> Flags = 0> {
+  string type = Type;
+  string name = Name;
+  string desc = Desc;
+  bits<3> flags = Flags;
+  Range range = Range<"", "">;
+  TypeInfo type_info = TypeInfo<"", "">;
+  bit IsHandle = IsHandleType<type>.ret;
+  bit IsPointer = IsPointerType<type>.ret;
+}
+
+// A parameter whose range is described by other parameters in the function.
+class RangedParam<string Type, string Name, string Desc, bits<3> Flags, Range ParamRange> : Param<Type, Name, Desc, Flags> {
+  let range = ParamRange;
+}
+
+// A parameter (normally of type void*) which has its pointee type and size
+// described by other parameters in the function.
+class TypeTaggedParam<string Type, string Name, string Desc, bits<3> Flags, TypeInfo ParamTypeInfo> : Param<Type, Name, Desc, Flags> {
+  let type_info = ParamTypeInfo;
+}
+
+class Return<string Value, list<string> Conditions = []> {
+  string value = Value;
+  list<string> conditions = Conditions;
+}
+
+class ShouldCheckHandle<Param P> {
+  bit ret = !and(P.IsHandle, !eq(!and(PARAM_OPTIONAL, P.flags), 0));
+}
+
+class ShouldCheckPointer<Param P> {
+  bit ret = !and(P.IsPointer, !eq(!and(PARAM_OPTIONAL, P.flags), 0));
+}
+
+// For a list of returns that contains a specific return code, find and append
+// new conditions to that return
+class AppendConditionsToReturn<list<Return> Returns, string ReturnValue,
+                               list<string> Conditions> {
+  list<Return> ret =
+      !foreach(Ret, Returns,
+               !if(!eq(Ret.value, ReturnValue),
+                   Return<Ret.value, Ret.conditions#Conditions>, Ret));
+}
+
+// Add null handle checks to a function's return value descriptions
+class AddHandleChecksToReturns<list<Param> Params, list<Return> Returns> {
+  list<string> handle_params =
+      !foreach(P, Params, !if(ShouldCheckHandle<P>.ret, P.name, ""));
+  list<string> handle_params_filt =
+      !filter(param, handle_params, !ne(param, ""));
+  list<string> handle_param_conds =
+      !foreach(handle, handle_params_filt, "`NULL == "#handle#"`");
+
+  // Does the list of returns already contain ERROR_INVALID_NULL_HANDLE?
+  bit returns_has_inv_handle = !foldl(
+      0, Returns, HasErr, Ret,
+      !or(HasErr, !eq(Ret.value, PREFIX#"_ERRC_INVALID_NULL_HANDLE")));
+
+  list<Return> returns_out = !if(returns_has_inv_handle,
+        AppendConditionsToReturn<Returns, PREFIX # "_ERRC_INVALID_NULL_HANDLE", handle_param_conds>.ret,
+        !listconcat(Returns, [Return<PREFIX # "_ERRC_INVALID_NULL_HANDLE", handle_param_conds>])
+    );
+}
+
+// Add null pointer checks to a function's return value descriptions
+class AddPointerChecksToReturns<list<Param> Params, list<Return> Returns> {
+  list<string> ptr_params =
+      !foreach(P, Params, !if(ShouldCheckPointer<P>.ret, P.name, ""));
+  list<string> ptr_params_filt = !filter(param, ptr_params, !ne(param, ""));
+  list<string> ptr_param_conds =
+      !foreach(ptr, ptr_params_filt, "`NULL == "#ptr#"`");
+
+  // Does the list of returns already contain ERROR_INVALID_NULL_POINTER?
+  bit returns_has_inv_ptr = !foldl(
+      0, Returns, HasErr, Ret,
+      !or(HasErr, !eq(Ret.value, PREFIX#"_ERRC_INVALID_NULL_POINTER")));
+  list<Return> returns_out = !if(returns_has_inv_ptr,
+        AppendConditionsToReturn<Returns, PREFIX # "_ERRC_INVALID_NULL_POINTER", ptr_param_conds>.ret,
+        !listconcat(Returns, [Return<PREFIX # "_ERRC_INVALID_NULL_POINTER", ptr_param_conds>])
+    );
+}
+
+defvar DefaultReturns = [Return<PREFIX#"_RESULT_SUCCESS">,
+                         Return<PREFIX#"_ERRC_UNINITIALIZED">,
+                         Return<PREFIX#"_ERRC_DEVICE_LOST">];
+
+class APIObject {
+  string name;
+  string desc;
+}
+
+class Function : APIObject {
+  list<Param> params;
+  list<Return> returns;
+  list<string> details = [];
+  list<string> analogues = [];
+
+  list<Return> returns_with_def = !listconcat(DefaultReturns, returns);
+  list<Return> all_returns = AddPointerChecksToReturns<params,
+        AddHandleChecksToReturns<params, returns_with_def>.returns_out>.returns_out;
+}
+
+class Etor<string Name, string Desc> {
+  string name = Name;
+  string desc = Desc;
+  string tagged_type;
+}
+
+class TaggedEtor<string Name, string Type, string Desc> : Etor<Name, Desc> {
+  let tagged_type = Type;
+}
+
+class Enum : APIObject {
+  // This refers to whether the enumerator descriptions specify a return
+  // type for functions where this enum may be used as an output type. If set,
+  // all Etor values must be TaggedEtor records
+  bit is_typed = 0;
+
+  list<Etor> etors = [];
+}
+
+class StructMember<string Type, string Name, string Desc> {
+  string type = Type;
+  string name = Name;
+  string desc = Desc;
+}
+
+defvar DefaultPropStructMembers =
+    [StructMember<prefix#"_structure_type_t", "stype",
+                  "type of this structure">,
+     StructMember<"void*", "pNext", "pointer to extension-specific structure">];
+
+class StructHasInheritedMembers<string BaseClass> {
+  bit ret = !or(!eq(BaseClass, prefix#"_base_properties_t"),
+                !eq(BaseClass, prefix#"_base_desc_t"));
+}
+
+class Struct : APIObject {
+  string base_class = "";
+  list<StructMember> members;
+  list<StructMember> all_members =
+      !if(StructHasInheritedMembers<base_class>.ret,
+          DefaultPropStructMembers, [])#members;
+}
+
+class Typedef : APIObject { string value; }
+
+class FptrTypedef : APIObject {
+  list<Param> params;
+  list<Return> returns;
+}
+
+class Macro : APIObject {
+  string value;
+
+  string condition;
+  string alt_value;
+}
+
+class Handle : APIObject;
diff --git a/offload/liboffload/API/CMakeLists.txt b/offload/liboffload/API/CMakeLists.txt
new file mode 100644
index 0000000000000..8fd6cb539374a
--- /dev/null
+++ b/offload/liboffload/API/CMakeLists.txt
@@ -0,0 +1,25 @@
+# The OffloadGenerate target is used to regenerate the generated files in the
+# include directory. These files are checked in with the rest of the source,
+# therefore it is only needed when making changes to the API.
+
+find_program(CLANG_FORMAT clang-format PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
+if (CLANG_FORMAT)
+    set(LLVM_TARGET_DEFINITIONS ${CMAKE_CURRENT_SOURCE_DIR}/OffloadAPI.td)
+
+    tablegen(OFFLOAD OffloadAPI.h -gen-api)
+    tablegen(OFFLOAD OffloadEntryPoints.inc -gen-entry-points)
+    tablegen(OFFLOAD OffloadFuncs.inc -gen-func-names)
+    tablegen(OFFLOAD OffloadImplFuncDecls.inc -gen-impl-func-decls)
+    tablegen(OFFLOAD OffloadPrint.hpp -gen-print-header)
+
+    set(OFFLOAD_GENERATED_FILES ${TABLEGEN_OUTPUT})
+    add_public_tablegen_target(OffloadGenerate)
+    add_custom_command(TARGET OffloadGenerate POST_BUILD COMMAND ${CLANG_FORMAT}
+        -i ${OFFLOAD_GENERATED_FILES})
+    add_custom_command(TARGET OffloadGenerate POST_BUILD COMMAND ${CMAKE_COMMAND}
+        -E copy_if_different ${OFFLOAD_GENERATED_FILES} "${CMAKE_CURRENT_SOURCE_DIR}/../include/generated")
+else()
+    message(WARNING "clang-format was not found, so the OffloadGenerate target\
+        will not be available. Offload will still build, but you will not be\
+        able to make changes to the API.")
+endif()
diff --git a/offload/liboffload/API/Common.td b/offload/liboffload/API/Common.td
new file mode 100644
index 0000000000000..5b19d1d47129e
--- /dev/null
+++ b/offload/liboffload/API/Common.td
@@ -0,0 +1,141 @@
+def : Macro {
+  let name = "OL_VERSION_MAJOR";
+  let desc = "Major version of the Offload API";
+  let value = "0";
+}
+
+def : Macro {
+  let name = "OL_VERSION_MINOR";
+  let desc = "Minor version of the Offload API";
+  let value = "0";
+}
+
+def : Macro {
+  let name = "OL_VERSION_PATCH";
+  let desc = "Patch version of the Offload API";
+  let value = "1";
+}
+
+def : Macro {
+  let name = "OL_APICALL";
+  let desc = "Calling convention for all API functions";
+  let condition = "defined(_WIN32)";
+  let value = "__cdecl";
+  let alt_value = "";
+}
+
+def : Macro {
+  let name = "OL_APIEXPORT";
+  let desc = "Microsoft-specific dllexport storage-class attribute";
+  let condition = "defined(_WIN32)";
+  let value = "__declspec(dllexport)";
+  let alt_value = "";
+}
+
+def : Macro {
+  let name = "OL_DLLEXPORT";
+  let desc = "Microsoft-specific dllexport storage-class attribute";
+  let condition = "defined(_WIN32)";
+  let value = "__declspec(dllexport)";
+}
+
+def : Macro {
+  let name = "OL_DLLEXPORT";
+  let desc = "GCC-specific dllexport storage-class attribute";
+  let condition = "__GNUC__ >= 4";
+  let value = "__attribute__ ((visibility (\"default\")))";
+  let alt_value = "";
+}
+
+def : Handle {
+  let name = "ol_platform_handle_t";
+  let desc = "Handle of a platform instance";
+}
+
+def : Handle {
+  let name = "ol_device_handle_t";
+  let desc = "Handle of platform's device object";
+}
+
+def : Handle {
+  let name = "ol_context_handle_t";
+  let desc = "Handle of context object";
+}
+
+def : Enum {
+  let name = "ol_errc_t";
+  let desc = "Defines Return/Error codes";
+  let etors =[
+    Etor<"SUCCESS", "Success">,
+    Etor<"INVALID_VALUE", "Invalid Value">,
+    Etor<"INVALID_PLATFORM", "Invalid platform">,
+    Etor<"DEVICE_NOT_FOUND", "Device not found">,
+    Etor<"INVALID_DEVICE", "Invalid device">,
+    Etor<"DEVICE_LOST", "Device hung, reset, was removed, or driver update occurred">,
+    Etor<"UNINITIALIZED", "plugin is not initialized or specific entry-point is not implemented">,
+    Etor<"OUT_OF_RESOURCES", "Out of resources">,
+    Etor<"UNSUPPORTED_VERSION", "generic error code for unsupported versions">,
+    Etor<"UNSUPPORTED_FEATURE", "generic error code for unsupported features">,
+    Etor<"INVALID_ARGUMENT", "generic error code for invalid arguments">,
+    Etor<"INVALID_NULL_HANDLE", "handle argument is not valid">,
+    Etor<"INVALID_NULL_POINTER", "pointer argument may not be nullptr">,
+    Etor<"INVALID_SIZE", "invalid size or dimensions (e.g., must not be zero, or is out of bounds)">,
+    Etor<"INVALID_ENUMERATION", "enumerator argument is not valid">,
+    Etor<"UNSUPPORTED_ENUMERATION", "enumerator argument is not supported by the device">,
+    Etor<"UNKNOWN", "Unknown or internal error">
+  ];
+}
+
+def : Struct {
+  let name = "ol_error_struct_t";
+  let desc = "Details of the error condition returned by an API call";
+  let members = [
+    StructMember<"ol_errc_t", "Code", "The error code">,
+    StructMember<"const char*", "Details", "String containing error details">
+  ];
+}
+
+def : Typedef {
+  let name = "ol_result_t";
+  let desc = "Result type returned by all entry points.";
+  let value = "const ol_error_struct_t*";
+}
+
+def : Macro {
+  let name = "OL_SUCCESS";
+  let desc = "Success condition";
+  let value = "NULL";
+}
+
+def : Struct {
+  let name = "ol_code_location_t";
+  let desc = "Code location information that can optionally be associated with an API call";
+  let members = [
+    StructMember<"const char*", "FunctionName", "Function name">,
+    StructMember<"const char*", "SourceFile", "Source code file">,
+    StructMember<"uint32_t", "LineNumber", "Source code line number">,
+    StructMember<"uint32_t", "ColumnNumber", "Source code column number">
+  ];
+}
+
+def : Function {
+  let name = "olInit";
+  let desc = "Perform initialization of the Offload library and plugins";
+  let details = [
+    "This must be the first API call made by a user of the Offload library",
+    "Each call will increment an internal reference count that is decremented by `olShutDown`"
+  ];
+  let params = [];
+  let returns = [];
+}
+
+def : Function {
+  let name = "olShutDown";
+  let desc = "Release the resources in use by Offload";
+  let details = [
+    "This decrements an internal reference count. When this reaches 0, all resources will be released",
+    "Subsequent API calls made after this are not valid"
+  ];
+  let params = [];
+  let returns = [];
+}
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
new file mode 100644
index 0000000000000..30c0b71fe7b37
--- /dev/null
+++ b/offload/liboffload/API/Device.td
@@ -0,0 +1,106 @@
+//===-- Device.td - Device definitions for Offload ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains Offload API definitions related to the Device handle
+//
+//===----------------------------------------------------------------------===//
+
+def : Enum {
+  let name = "ol_device_type_t";
+  let desc = "Supported device types";
+  let etors =[
+    Etor<"DEFAULT", "The default device type as preferred by the runtime">,
+    Etor<"ALL", "Devices of all types">,
+    Etor<"GPU", "GPU device type">,
+    Etor<"CPU", "CPU device type">,
+  ];
+}
+
+def : Enum {
+  let name = "ol_device_info_t";
+  let desc = "Supported device info";
+  let is_typed = 1;
+  let etors =[
+    TaggedEtor<"TYPE", "ol_device_type_t", "type of the device">,
+    TaggedEtor<"PLATFORM", "ol_platform_handle_t", "the platform associated with the device">,
+    TaggedEtor<"NAME", "char[]", "Device name">,
+    TaggedEtor<"VENDOR", "char[]", "Device vendor">,
+    TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">
+  ];
+}
+
+def : Function {
+  let name = "olGetDeviceCount";
+  let desc = "Retrieves the number of available devices within a platform";
+  let params = [
+    Param<"ol_platform_handle_t", "Platform", "handle of the platform instance", PARAM_IN>,
+    Param<"uint32_t*", "NumDevices", "pointer to the number of devices.", PARAM_OUT>
+  ];
+  let returns = [];
+}
+
+def : Function {
+  let name = "olGetDevice";
+  let desc = "Retrieves devices within a platform";
+  let details = [
+    "Multiple calls to this function will return identical device handles, in the same order.",
+  ];
+  let params = [
+    Param<"ol_platform_handle_t", "Platform", "handle of the platform instance", PARAM_IN>,
+    Param<"uint32_t", "NumEntries", "the number of devices to be added to phDevices, which must be greater than zero", PARAM_IN>,
+    RangedParam<"ol_device_handle_t*", "Devices", "Array of device handles. "
+        "If NumEntries is less than the number of devices available, then this function shall only retrieve that number of devices.", PARAM_OUT,
+        Range<"0", "NumEntries">>
+  ];
+  let returns = [
+    Return<"OL_ERRC_INVALID_SIZE", [
+      "`NumEntries == 0`"
+    ]>
+  ];
+}
+
+def : Function {
+  let name = "olGetDeviceInfo";
+  let desc = "Queries the given property of the device";
+  let details = [];
+  let params = [
+    Param<"ol_device_handle_t", "Device", "handle of the device instance", PARAM_IN>,
+    Param<"ol_device_info_t", "PropName", "type of the info to retrieve", PARAM_IN>,
+    Param<"size_t", "PropSize", "the number of bytes pointed to by PropValue.", PARAM_IN>,
+    TypeTaggedParam<"void*", "PropValue", "array of bytes holding the info. If PropSize is not equal to or greater than the real "
+                    "number of bytes needed to return the info then the OL_ERRC_INVALID_SIZE error is returned and "
+                    "PropValue is not used.", PARAM_OUT, TypeInfo<"PropName" , "PropSize">>
+  ];
+  let returns = [
+    Return<"OL_ERRC_UNSUPPORTED_ENUMERATION", [
+      "If `PropName` is not supported by the device."
+    ]>,
+    Return<"OL_ERRC_INVALID_SIZE", [
+      "`PropSize == 0`",
+      "If `PropSize` is less than the real number of bytes needed to return the info."
+    ]>,
+    Return<"OL_ERRC_INVALID_DEVICE">
+  ];
+}
+
+def : Function {
+  let name = "olGetDeviceInfoSize";
+  let desc = "Returns the storage size of the given device query";
+  let details = [];
+  let params = [
+    Param<"ol_device_handle_t", "Device", "handle of the device instance", PARAM_IN>,
+    Param<"ol_device_info_t", "PropName", "type of the info to retrieve", PARAM_IN>,
+    Param<"size_t*", "PropSizeRet", "pointer to the number of bytes required to store the query", PARAM_OUT>
+  ];
+  let returns = [
+    Return<"OL_ERRC_UNSUPPORTED_ENUMERATION", [
+      "If `PropName` is not supported by the device."
+    ]>,
+    Return<"OL_ERRC_INVALID_DEVICE">
+  ];
+}
diff --git a/offload/liboffload/API/OffloadAPI.td b/offload/liboffload/API/OffloadAPI.td
new file mode 100644
index 0000000000000..8a0c3c4058122
--- /dev/null
+++ b/offload/liboffload/API/OffloadAPI.td
@@ -0,0 +1,15 @@
+//===-- OffloadAPI.td - Root tablegen file for Offload -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Always include this file first
+include "APIDefs.td"
+
+// Add API definition files here
+include "Common.td"
+include "Platform.td"
+include "Device.td"
diff --git a/offload/liboffload/API/Platform.td b/offload/liboffload/API/Platform.td
new file mode 100644
index 0000000000000..03e70cf96ac94
--- /dev/null
+++ b/offload/liboffload/API/Platform.td
@@ -0,0 +1,112 @@
+//===-- Platform.td - Platform definitions for Offload -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains Offload API definitions related to the Platform handle
+//
+//===----------------------------------------------------------------------===//
+def : Function {
+  let name = "olGetPlatform";
+  let desc = "Retrieves all available platforms";
+  let details = [
+    "Multiple calls to this function will return identical platforms handles, in the same order.",
+  ];
+  let params = [
+    Param<"uint32_t", "NumEntries",
+      "The number of platforms to be added to Platforms. NumEntries must be "
+      "greater than zero.",
+      PARAM_IN>,
+    RangedParam<"ol_platform_handle_t*", "Platforms", 
+      "Array of handle of platforms. If NumEntries is less than the number of "
+      "platforms available, then olGetPlatform shall only retrieve that "
+      "number of platforms.",
+      PARAM_OUT, Range<"0", "NumEntries">>
+  ];
+  let returns = [
+    Return<"OL_ERRC_INVALID_SIZE", [
+      "`NumEntries == 0`"
+    ]>
+  ];
+}
+
+def : Function {
+  let name = "olGetPlatformCount";
+  let desc = "Retrieves the number of available platforms";
+  let params = [
+    Param<"uint32_t*",
+      "NumPlatforms", "returns the total number of platforms available.",
+      PARAM_OUT>
+  ];
+  let returns = [];
+}
+
+def : Enum {
+  let name = "ol_platform_info_t";
+  let desc = "Supported platform info";
+  let is_typed = 1;
+  let etors = [
+    TaggedEtor<"NAME", "char[]", "The string denoting name of the platform. The size of the info needs to be dynamically queried.">,
+    TaggedEtor<"VENDOR_NAME", "char[]", "The string denoting name of the vendor of the platform. The size of the info needs to be dynamically queried.">,
+    TaggedEtor<"VERSION", "char[]", "The string denoting the version of the platform. The size of the info needs to be dynamically queried.">,
+    TaggedEtor<"BACKEND", "ol_platform_backend_t", "The native backend of the platform.">
+  ];
+}
+
+def : Enum {
+  let name = "ol_platform_backend_t";
+  let desc = "Identifies the native backend of the platform";
+  let etors =[
+    Etor<"UNKNOWN", "The backend is not recognized">,
+    Etor<"CUDA", "The backend is CUDA">,
+    Etor<"AMDGPU", "The backend is AMDGPU">,
+  ];
+}
+
+def : Function {
+  let name = "olGetPlatformInfo";
+  let desc = "Queries the given property of the platform";
+  let details = [
+    "`olGetPlatformInfoSize` can be used to query the storage size "
+    "required for the given query."
+  ];
+  let params = [
+    Param<"ol_platform_handle_t", "Platform", "handle of the platform", PARAM_IN>,
+    Param<"ol_platform_info_t", "PropName", "type of the info to retrieve", PARAM_IN>,
+    Param<"size_t", "PropSize", "the number of bytes pointed to by pPlatformInfo.", PARAM_IN>,
+    TypeTaggedParam<"void*", "PropValue", "array of bytes holding the info. "
+      "If Size is not equal to or greater to the real number of bytes needed to return the info "
+      "then the OL_ERRC_INVALID_SIZE error is returned and pPlatformInfo is not used.", PARAM_OUT,
+      TypeInfo<"PropName" , "PropSize">>
+  ];
+  let returns = [
+    Return<"OL_ERRC_UNSUPPORTED_ENUMERATION", [
+      "If `PropName` is not supported by the platform."
+    ]>,
+    Return<"OL_ERRC_INVALID_SIZE", [
+      "`PropSize == 0`",
+      "If `PropSize` is less than the real number of bytes needed to return the info."
+    ]>,
+    Return<"OL_ERRC_INVALID_PLATFORM">
+  ];
+}
+
+def : Function {
+  let name = "olGetPlatformInfoSize";
+  let desc = "Returns the storage size of the given platform query";
+  let details = [];
+  let params = [
+    Param<"ol_platform_handle_t", "Platform", "handle of the platform", PARAM_IN>,
+    Param<"ol_platform_info_t", "PropName", "type of the info to query", PARAM_IN>,
+    Param<"size_t*", "PropSizeRet", "pointer to the number of bytes required to store the query", PARAM_OUT>
+  ];
+  let returns = [
+    Return<"OL_ERRC_UNSUPPORTED_ENUMERATION", [
+      "If `PropName` is not supported by the platform."
+    ]>,
+    Return<"OL_ERRC_INVALID_PLATFORM">
+  ];
+}
diff --git a/offload/liboffload/API/README.md b/offload/liboffload/API/README.md
new file mode 100644
index 0000000000000..38a055811b2d0
--- /dev/null
+++ b/offload/liboffload/API/README.md
@@ -0,0 +1,150 @@
+# Offload API definitions
+
+**Note**: This is a work-in-progress. It is loosely based on equivalent
+tooling in Unified Runtime.
+
+The Tablegen files in this directory are used to define the Offload API. They
+are used with the `offload-tblgen` tool to generate API headers, print headers,
+and other implementation details.
+
+The root file is `OffloadAPI.td` - additional `.td` files can be included in
+this file to add them to the API.
+
+## API Objects
+The API consists of a number of objects, which always have a *name* field and
+*description* field, and are one of the following types:
+
+### Function
+Represents an API entry point function. Has a list of returns and parameters.
+Also has fields for details (representing a bullet-point list of
+information about the function that would otherwise be too detailed for the
+description), and analogues (equivalent functions in other APIs).
+
+#### Parameter
+Represents a parameter to a function, has *type*, *name*, and *desc* fields.
+Also has a *flags* field containing flags representing whether the parameter is
+in, out, or optional.
+
+The *type* field is used to infer if the parameter is a pointer or handle type.
+A *handle* type is a pointer to an opaque struct, used to abstract over
+plugin-specific implementation details.
+
+There are two special variants of a *parameter*:
+* **RangedParameter** - Represents a parameter that has a range described by other parameters. Generally these are pointers to an arbitrary number of objects. The range is used for generating validation and printing code. E.g, a range might be between `(0, NumDevices)`
+* **TypeTaggedParameter** - Represents a parameter (usually of `void*` type) that has the type and size of its pointee data described by other function parameters. The type is usually described by a type-tagged enum. This allows functions (e.g. `olGetDeviceInfo`) to return data of an arbitrary type.
+
+#### Return
+A return represents a possible return code from the function, and optionally a
+list of conditions in which this value may be returned. The conditions list is
+not expected to be exhaustive. A condition is considered free-form text, but
+if it is wrapped in \`backticks\` then it is treated as literal code
+representing an error condition (e.g. `someParam < 1`). These conditions are
+used to automatically create validation checks by the `offload-tblgen`
+validation generator.
+
+Returns are automatically generated for functions with pointer or handle
+parameters, so API authors do not need to exhaustively add null checks for
+these types of parameters. All functions also get a number of default return
+values automatically.
+
+
+### Struct
+Represents a struct. Contains a list of members, which each have a *type*,
+*name*, and *desc*.
+
+Also optionally takes a *base_class* field. If this is either of the special
+`offload_base_properties_t` or `offload_base_desc_t` structs, then the struct
+will inherit members from those structs. The generated struct does **not** use
+actual C++ inheritance, but instead explicitly has those members copied in,
+which preserves ABI compatibility with C.
+
+### Enum
+Represents a C-style enum. Contains a list of `etor` values, which have a name
+and description.
+
+A `TaggedEtor` record type also exists which addtionally takes a type. This type
+is used when the enum is used as a parameter to a function with a type-tagged
+function parameter (e.g. `olGetDeviceInfo`).
+
+All enums automatically get a `<enum_name>_FORCE_UINT32 = 0x7fffffff` value,
+which forces the underlying type to be uint32.
+
+### Handle
+Represents a pointer to an opaque struct, as described in the Parameter section.
+It does not take any extra fields.
+
+### Typedef
+Represents a typedef, contains only a *value* field.
+
+### Macro
+Represents a C preprocessor `#define`. Contains a *value* field. Optionally
+takes a *condition* field, which allows the macro to be conditionally defined,
+and an *alt_value* field, which represents the value if the condition is false.
+
+Macro arguments are presented in the *name* field (e.g. name = `mymacro(arg)`).
+
+While there may seem little point generating a macro from tablegen, doing this
+allows the entire source of the header file to be generated from the tablegen
+files, rather than requiring a mix of C source and tablegen.
+
+## Generation
+
+### API header
+```
+./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-api
+```
+The comments in the generated header are in Doxygen format, although
+generating documentation from them hasn't been implemented yet.
+
+The entirety of this header is generated by Tablegen, rather than having a predefined header file that includes one or more `.inc` files. This is because this header is expected to be part of the installation and distributed to end-users, so should be self-contained.
+
+### Entry Points
+```
+./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-entry-points
+```
+These functions form the actual Offload interface, and are wrappers over the
+functions that contain the actual implementation (see
+'Adding a new entry point').
+
+They implement automatically generated validation checks, and tracing of
+function calls with arguments and results. The tracing can be enabled with the
+`OFFLOAD_TRACE` environment variable.
+
+### Implementation function declarations
+```
+./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-impl-func-decls
+```
+Generates declarations of the implementation of functions of every entry point
+in the API, e.g. `offloadDeviceFoo_impl` for `offloadDeviceFoo`.
+
+### Print header
+```
+./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-print-header
+```
+This header contains `std::ostream &operator<<(std::ostream&)` definitions for
+various API objects, including function parameters.
+
+As with the API header, it is expected that this header is part of the installed
+package, so it is entirely generated by Tablegen.
+
+For ease of implementation, and since it is not strictly part of the API, this
+is a C++ header file. If a C version is desirable it could be added.
+
+### Future Tablegen backends
+`RecordTypes.hpp` contains wrappers for all of the API object types, which will
+allow more backends to be easily added in future.
+
+## Adding to the API
+
+A new object can be added to the API by adding to one of the existing `.td`
+files. It is also possible to add a new tablegen file to the API by adding it
+to the includes in `OffloadAPI.td`. When the offload target is rebuilt, the
+new definition will be included in the generated files.
+
+### Adding a new entry point
+
+When a new entry point is added (e.g. `offloadDeviceFoo`), the actual entry
+point is automatically generated, which contains validation and tracing code.
+It expects an implementation function (`offloadDeviceFoo_impl`) to be defined,
+which it will call into. The definition of this implementation function should
+be added to `src/offload_impl.cpp`
diff --git a/offload/liboffload/CMakeLists.txt b/offload/liboffload/CMakeLists.txt
new file mode 100644
index 0000000000000..db12236ddfc7f
--- /dev/null
+++ b/offload/liboffload/CMakeLists.txt
@@ -0,0 +1,43 @@
+add_subdirectory(API)
+
+add_llvm_library(
+  LLVMOffload SHARED
+  src/OffloadLib.cpp
+  src/OffloadImpl.cpp
+
+  LINK_COMPONENTS
+  FrontendOpenMP
+  Support
+  )
+
+foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
+    target_link_libraries(LLVMOffload PRIVATE omptarget.rtl.${plugin})
+endforeach()
+
+if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+    target_link_libraries(LLVMOffload PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+endif()
+
+target_include_directories(LLVMOffload PUBLIC
+                            ${CMAKE_CURRENT_BINARY_DIR}/../include
+                            ${CMAKE_CURRENT_SOURCE_DIR}/include
+                            ${CMAKE_CURRENT_SOURCE_DIR}/include/generated
+                            ${CMAKE_CURRENT_SOURCE_DIR}/../include
+                            ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include)
+
+target_compile_options(LLVMOffload PRIVATE ${offload_compile_flags})
+target_link_options(LLVMOffload PRIVATE ${offload_link_flags})
+
+target_compile_definitions(LLVMOffload PRIVATE
+  TARGET_NAME="Liboffload"
+  DEBUG_PREFIX="Liboffload"
+)
+
+set_target_properties(LLVMOffload PROPERTIES
+                      POSITION_INDEPENDENT_CODE ON
+                      INSTALL_RPATH "$ORIGIN"
+                      BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+install(TARGETS LLVMOffload LIBRARY COMPONENT LLVMOffload DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/generated/OffloadAPI.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/generated/OffloadPrint.hpp DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
diff --git a/offload/liboffload/README.md b/offload/liboffload/README.md
new file mode 100644
index 0000000000000..95c9bf54d7bad
--- /dev/null
+++ b/offload/liboffload/README.md
@@ -0,0 +1,8 @@
+# Offload New API
+
+This directory contains the implementation of the experimental work-in-progress
+new API for Offload. It builds on top of the existing plugin implementations but
+provides a single level of abstraction suitable for runtimes for languages other
+than OpenMP to be built on top of.
+
+See the [API definition readme](API/README.md) for implementation details.
\ No newline at end of file
diff --git a/offload/liboffload/exports b/offload/liboffload/exports
new file mode 100644
index 0000000000000..168341aa7d938
--- /dev/null
+++ b/offload/liboffload/exports
@@ -0,0 +1,6 @@
+VERS1.0 {
+global:
+  ol*;
+local:
+  *;
+};
diff --git a/offload/liboffload/include/OffloadImpl.hpp b/offload/liboffload/include/OffloadImpl.hpp
new file mode 100644
index 0000000000000..6d745095f3105
--- /dev/null
+++ b/offload/liboffload/include/OffloadImpl.hpp
@@ -0,0 +1,94 @@
+//===- offload_impl.hpp- Implementation helpers for the Offload library ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include <OffloadAPI.h>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+
+struct OffloadConfig {
+  bool TracingEnabled = false;
+};
+
+OffloadConfig &offloadConfig();
+
+// Use the StringSet container to efficiently deduplicate repeated error
+// strings (e.g. if the same error is hit constantly in a long running program)
+llvm::StringSet<> &errorStrs();
+
+// Use an unordered_set to avoid duplicates of error structs themselves.
+// We cannot store the structs directly as returned pointers to them must always
+// be valid, and a rehash of the set may invalidate them. This requires
+// custom hash and equal_to function objects.
+using ErrPtrT = std::unique_ptr<ol_error_struct_t>;
+struct ErrPtrEqual {
+  bool operator()(const ErrPtrT &lhs, const ErrPtrT &rhs) const {
+    if (!lhs && !rhs) {
+      return true;
+    }
+    if (!lhs || !rhs) {
+      return false;
+    }
+
+    bool StrsEqual = false;
+    if (lhs->Details == NULL && rhs->Details == NULL) {
+      StrsEqual = true;
+    } else if (lhs->Details != NULL && rhs->Details != NULL) {
+      StrsEqual = (std::strcmp(lhs->Details, rhs->Details) == 0);
+    }
+    return (lhs->Code == rhs->Code) && StrsEqual;
+  }
+};
+struct ErrPtrHash {
+  size_t operator()(const ErrPtrT &e) const {
+    if (!e) {
+      // We shouldn't store empty errors (i.e. success), but just in case
+      return 0lu;
+    } else {
+      return std::hash<int>{}(e->Code);
+    }
+  }
+};
+using ErrSetT = std::unordered_set<ErrPtrT, ErrPtrHash, ErrPtrEqual>;
+ErrSetT &errors();
+
+struct ol_impl_result_t {
+  ol_impl_result_t(std::nullptr_t) : Result(OL_SUCCESS) {}
+  ol_impl_result_t(ol_errc_t Code) {
+    if (Code == OL_ERRC_SUCCESS) {
+      Result = nullptr;
+    } else {
+      auto Err = std::unique_ptr<ol_error_struct_t>(
+          new ol_error_struct_t{Code, nullptr});
+      Result = errors().emplace(std::move(Err)).first->get();
+    }
+  }
+
+  ol_impl_result_t(ol_errc_t Code, llvm::StringRef Details) {
+    assert(Code != OL_ERRC_SUCCESS);
+    Result = nullptr;
+    auto DetailsStr = errorStrs().insert(Details).first->getKeyData();
+    auto Err = std::unique_ptr<ol_error_struct_t>(
+        new ol_error_struct_t{Code, DetailsStr});
+    Result = errors().emplace(std::move(Err)).first->get();
+  }
+
+  operator ol_result_t() { return Result; }
+
+private:
+  ol_result_t Result;
+};
diff --git a/offload/liboffload/include/generated/OffloadAPI.h b/offload/liboffload/include/generated/OffloadAPI.h
new file mode 100644
index 0000000000000..11fcc96625ab8
--- /dev/null
+++ b/offload/liboffload/include/generated/OffloadAPI.h
@@ -0,0 +1,610 @@
+//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Auto-generated file, do not manually edit.
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_VERSION_MAJOR
+/// @brief Major version of the Offload API
+#define OL_VERSION_MAJOR 0
+#endif // OL_VERSION_MAJOR
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_VERSION_MINOR
+/// @brief Minor version of the Offload API
+#define OL_VERSION_MINOR 0
+#endif // OL_VERSION_MINOR
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_VERSION_PATCH
+/// @brief Patch version of the Offload API
+#define OL_VERSION_PATCH 1
+#endif // OL_VERSION_PATCH
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_APICALL
+#if defined(_WIN32)
+/// @brief Calling convention for all API functions
+#define OL_APICALL __cdecl
+#else
+#define OL_APICALL
+#endif // defined(_WIN32)
+#endif // OL_APICALL
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_APIEXPORT
+#if defined(_WIN32)
+/// @brief Microsoft-specific dllexport storage-class attribute
+#define OL_APIEXPORT __declspec(dllexport)
+#else
+#define OL_APIEXPORT
+#endif // defined(_WIN32)
+#endif // OL_APIEXPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_DLLEXPORT
+#if defined(_WIN32)
+/// @brief Microsoft-specific dllexport storage-class attribute
+#define OL_DLLEXPORT __declspec(dllexport)
+#endif // defined(_WIN32)
+#endif // OL_DLLEXPORT
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_DLLEXPORT
+#if __GNUC__ >= 4
+/// @brief GCC-specific dllexport storage-class attribute
+#define OL_DLLEXPORT __attribute__((visibility("default")))
+#else
+#define OL_DLLEXPORT
+#endif // __GNUC__ >= 4
+#endif // OL_DLLEXPORT
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Handle of a platform instance
+typedef struct ol_platform_handle_t_ *ol_platform_handle_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Handle of platform's device object
+typedef struct ol_device_handle_t_ *ol_device_handle_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Handle of context object
+typedef struct ol_context_handle_t_ *ol_context_handle_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Defines Return/Error codes
+typedef enum ol_errc_t {
+  /// Success
+  OL_ERRC_SUCCESS = 0,
+  /// Invalid Value
+  OL_ERRC_INVALID_VALUE = 1,
+  /// Invalid platform
+  OL_ERRC_INVALID_PLATFORM = 2,
+  /// Device not found
+  OL_ERRC_DEVICE_NOT_FOUND = 3,
+  /// Invalid device
+  OL_ERRC_INVALID_DEVICE = 4,
+  /// Device hung, reset, was removed, or driver update occurred
+  OL_ERRC_DEVICE_LOST = 5,
+  /// plugin is not initialized or specific entry-point is not implemented
+  OL_ERRC_UNINITIALIZED = 6,
+  /// Out of resources
+  OL_ERRC_OUT_OF_RESOURCES = 7,
+  /// generic error code for unsupported versions
+  OL_ERRC_UNSUPPORTED_VERSION = 8,
+  /// generic error code for unsupported features
+  OL_ERRC_UNSUPPORTED_FEATURE = 9,
+  /// generic error code for invalid arguments
+  OL_ERRC_INVALID_ARGUMENT = 10,
+  /// handle argument is not valid
+  OL_ERRC_INVALID_NULL_HANDLE = 11,
+  /// pointer argument may not be nullptr
+  OL_ERRC_INVALID_NULL_POINTER = 12,
+  /// invalid size or dimensions (e.g., must not be zero, or is out of bounds)
+  OL_ERRC_INVALID_SIZE = 13,
+  /// enumerator argument is not valid
+  OL_ERRC_INVALID_ENUMERATION = 14,
+  /// enumerator argument is not supported by the device
+  OL_ERRC_UNSUPPORTED_ENUMERATION = 15,
+  /// Unknown or internal error
+  OL_ERRC_UNKNOWN = 16,
+  /// @cond
+  OL_ERRC_FORCE_UINT32 = 0x7fffffff
+  /// @endcond
+
+} ol_errc_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Details of the error condition returned by an API call
+typedef struct ol_error_struct_t {
+  ol_errc_t Code;      /// The error code
+  const char *Details; /// String containing error details
+} ol_error_struct_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Result type returned by all entry points.
+typedef const ol_error_struct_t *ol_result_t;
+
+///////////////////////////////////////////////////////////////////////////////
+#ifndef OL_SUCCESS
+/// @brief Success condition
+#define OL_SUCCESS NULL
+#endif // OL_SUCCESS
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Code location information that can optionally be associated with an
+/// API call
+typedef struct ol_code_location_t {
+  const char *FunctionName; /// Function name
+  const char *SourceFile;   /// Source code file
+  uint32_t LineNumber;      /// Source code line number
+  uint32_t ColumnNumber;    /// Source code column number
+} ol_code_location_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Perform initialization of the Offload library and plugins
+///
+/// @details
+///    - This must be the first API call made by a user of the Offload library
+///    - Each call will increment an internal reference count that is
+///    decremented by `olShutDown`
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+OL_APIEXPORT ol_result_t OL_APICALL olInit();
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Release the resources in use by Offload
+///
+/// @details
+///    - This decrements an internal reference count. When this reaches 0, all
+///    resources will be released
+///    - Subsequent API calls made after this are not valid
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+OL_APIEXPORT ol_result_t OL_APICALL olShutDown();
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieves all available platforms
+///
+/// @details
+///    - Multiple calls to this function will return identical platforms
+///    handles, in the same order.
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_SIZE
+///         + `NumEntries == 0`
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == Platforms`
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatform(
+    // [in] The number of platforms to be added to Platforms. NumEntries must be
+    // greater than zero.
+    uint32_t NumEntries,
+    // [out] Array of handle of platforms. If NumEntries is less than the number
+    // of platforms available, then olGetPlatform shall only retrieve that
+    // number of platforms.
+    ol_platform_handle_t *Platforms);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieves the number of available platforms
+///
+/// @details
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == NumPlatforms`
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformCount(
+    // [out] returns the total number of platforms available.
+    uint32_t *NumPlatforms);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Supported platform info
+typedef enum ol_platform_info_t {
+  /// [char[]] The string denoting name of the platform. The size of the info
+  /// needs to be dynamically queried.
+  OL_PLATFORM_INFO_NAME = 0,
+  /// [char[]] The string denoting name of the vendor of the platform. The size
+  /// of the info needs to be dynamically queried.
+  OL_PLATFORM_INFO_VENDOR_NAME = 1,
+  /// [char[]] The string denoting the version of the platform. The size of the
+  /// info needs to be dynamically queried.
+  OL_PLATFORM_INFO_VERSION = 2,
+  /// [ol_platform_backend_t] The native backend of the platform.
+  OL_PLATFORM_INFO_BACKEND = 3,
+  /// @cond
+  OL_PLATFORM_INFO_FORCE_UINT32 = 0x7fffffff
+  /// @endcond
+
+} ol_platform_info_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Identifies the native backend of the platform
+typedef enum ol_platform_backend_t {
+  /// The backend is not recognized
+  OL_PLATFORM_BACKEND_UNKNOWN = 0,
+  /// The backend is CUDA
+  OL_PLATFORM_BACKEND_CUDA = 1,
+  /// The backend is AMDGPU
+  OL_PLATFORM_BACKEND_AMDGPU = 2,
+  /// @cond
+  OL_PLATFORM_BACKEND_FORCE_UINT32 = 0x7fffffff
+  /// @endcond
+
+} ol_platform_backend_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Queries the given property of the platform
+///
+/// @details
+///    - `olGetPlatformInfoSize` can be used to query the storage size required
+///    for the given query.
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
+///         + If `PropName` is not supported by the platform.
+///     - ::OL_ERRC_INVALID_SIZE
+///         + `PropSize == 0`
+///         + If `PropSize` is less than the real number of bytes needed to
+///         return the info.
+///     - ::OL_ERRC_INVALID_PLATFORM
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Platform`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == PropValue`
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfo(
+    // [in] handle of the platform
+    ol_platform_handle_t Platform,
+    // [in] type of the info to retrieve
+    ol_platform_info_t PropName,
+    // [in] the number of bytes pointed to by pPlatformInfo.
+    size_t PropSize,
+    // [out] array of bytes holding the info. If Size is not equal to or greater
+    // to the real number of bytes needed to return the info then the
+    // OL_ERRC_INVALID_SIZE error is returned and pPlatformInfo is not used.
+    void *PropValue);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Returns the storage size of the given platform query
+///
+/// @details
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
+///         + If `PropName` is not supported by the platform.
+///     - ::OL_ERRC_INVALID_PLATFORM
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Platform`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == PropSizeRet`
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoSize(
+    // [in] handle of the platform
+    ol_platform_handle_t Platform,
+    // [in] type of the info to query
+    ol_platform_info_t PropName,
+    // [out] pointer to the number of bytes required to store the query
+    size_t *PropSizeRet);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Supported device types
+typedef enum ol_device_type_t {
+  /// The default device type as preferred by the runtime
+  OL_DEVICE_TYPE_DEFAULT = 0,
+  /// Devices of all types
+  OL_DEVICE_TYPE_ALL = 1,
+  /// GPU device type
+  OL_DEVICE_TYPE_GPU = 2,
+  /// CPU device type
+  OL_DEVICE_TYPE_CPU = 3,
+  /// @cond
+  OL_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
+  /// @endcond
+
+} ol_device_type_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Supported device info
+typedef enum ol_device_info_t {
+  /// [ol_device_type_t] type of the device
+  OL_DEVICE_INFO_TYPE = 0,
+  /// [ol_platform_handle_t] the platform associated with the device
+  OL_DEVICE_INFO_PLATFORM = 1,
+  /// [char[]] Device name
+  OL_DEVICE_INFO_NAME = 2,
+  /// [char[]] Device vendor
+  OL_DEVICE_INFO_VENDOR = 3,
+  /// [char[]] Driver version
+  OL_DEVICE_INFO_DRIVER_VERSION = 4,
+  /// @cond
+  OL_DEVICE_INFO_FORCE_UINT32 = 0x7fffffff
+  /// @endcond
+
+} ol_device_info_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieves the number of available devices within a platform
+///
+/// @details
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Platform`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == NumDevices`
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceCount(
+    // [in] handle of the platform instance
+    ol_platform_handle_t Platform,
+    // [out] pointer to the number of devices.
+    uint32_t *NumDevices);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieves devices within a platform
+///
+/// @details
+///    - Multiple calls to this function will return identical device handles,
+///    in the same order.
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_INVALID_SIZE
+///         + `NumEntries == 0`
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Platform`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == Devices`
+OL_APIEXPORT ol_result_t OL_APICALL olGetDevice(
+    // [in] handle of the platform instance
+    ol_platform_handle_t Platform,
+    // [in] the number of devices to be added to phDevices, which must be
+    // greater than zero
+    uint32_t NumEntries,
+    // [out] Array of device handles. If NumEntries is less than the number of
+    // devices available, then this function shall only retrieve that number of
+    // devices.
+    ol_device_handle_t *Devices);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Queries the given property of the device
+///
+/// @details
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
+///         + If `PropName` is not supported by the device.
+///     - ::OL_ERRC_INVALID_SIZE
+///         + `PropSize == 0`
+///         + If `PropSize` is less than the real number of bytes needed to
+///         return the info.
+///     - ::OL_ERRC_INVALID_DEVICE
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Device`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == PropValue`
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfo(
+    // [in] handle of the device instance
+    ol_device_handle_t Device,
+    // [in] type of the info to retrieve
+    ol_device_info_t PropName,
+    // [in] the number of bytes pointed to by PropValue.
+    size_t PropSize,
+    // [out] array of bytes holding the info. If PropSize is not equal to or
+    // greater than the real number of bytes needed to return the info then the
+    // OL_ERRC_INVALID_SIZE error is returned and PropValue is not used.
+    void *PropValue);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Returns the storage size of the given device query
+///
+/// @details
+///
+/// @returns
+///     - ::OL_RESULT_SUCCESS
+///     - ::OL_ERRC_UNINITIALIZED
+///     - ::OL_ERRC_DEVICE_LOST
+///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
+///         + If `PropName` is not supported by the device.
+///     - ::OL_ERRC_INVALID_DEVICE
+///     - ::OL_ERRC_INVALID_NULL_HANDLE
+///         + `NULL == Device`
+///     - ::OL_ERRC_INVALID_NULL_POINTER
+///         + `NULL == PropSizeRet`
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSize(
+    // [in] handle of the device instance
+    ol_device_handle_t Device,
+    // [in] type of the info to retrieve
+    ol_device_info_t PropName,
+    // [out] pointer to the number of bytes required to store the query
+    size_t *PropSizeRet);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetPlatform
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_platform_params_t {
+  uint32_t *pNumEntries;
+  ol_platform_handle_t **pPlatforms;
+} ol_get_platform_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetPlatformCount
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_platform_count_params_t {
+  uint32_t **pNumPlatforms;
+} ol_get_platform_count_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetPlatformInfo
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_platform_info_params_t {
+  ol_platform_handle_t *pPlatform;
+  ol_platform_info_t *pPropName;
+  size_t *pPropSize;
+  void **pPropValue;
+} ol_get_platform_info_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetPlatformInfoSize
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_platform_info_size_params_t {
+  ol_platform_handle_t *pPlatform;
+  ol_platform_info_t *pPropName;
+  size_t **pPropSizeRet;
+} ol_get_platform_info_size_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetDeviceCount
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_device_count_params_t {
+  ol_platform_handle_t *pPlatform;
+  uint32_t **pNumDevices;
+} ol_get_device_count_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetDevice
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_device_params_t {
+  ol_platform_handle_t *pPlatform;
+  uint32_t *pNumEntries;
+  ol_device_handle_t **pDevices;
+} ol_get_device_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetDeviceInfo
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_device_info_params_t {
+  ol_device_handle_t *pDevice;
+  ol_device_info_t *pPropName;
+  size_t *pPropSize;
+  void **pPropValue;
+} ol_get_device_info_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for olGetDeviceInfoSize
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct ol_get_device_info_size_params_t {
+  ol_device_handle_t *pDevice;
+  ol_device_info_t *pPropName;
+  size_t **pPropSizeRet;
+} ol_get_device_info_size_params_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olInit that also sets source code location information
+/// @details See also ::olInit
+OL_APIEXPORT ol_result_t OL_APICALL
+olInitWithCodeLoc(ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olShutDown that also sets source code location information
+/// @details See also ::olShutDown
+OL_APIEXPORT ol_result_t OL_APICALL
+olShutDownWithCodeLoc(ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetPlatform that also sets source code location
+/// information
+/// @details See also ::olGetPlatform
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetPlatformWithCodeLoc(uint32_t NumEntries, ol_platform_handle_t *Platforms,
+                         ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetPlatformCount that also sets source code location
+/// information
+/// @details See also ::olGetPlatformCount
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformCountWithCodeLoc(
+    uint32_t *NumPlatforms, ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetPlatformInfo that also sets source code location
+/// information
+/// @details See also ::olGetPlatformInfo
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoWithCodeLoc(
+    ol_platform_handle_t Platform, ol_platform_info_t PropName, size_t PropSize,
+    void *PropValue, ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetPlatformInfoSize that also sets source code location
+/// information
+/// @details See also ::olGetPlatformInfoSize
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoSizeWithCodeLoc(
+    ol_platform_handle_t Platform, ol_platform_info_t PropName,
+    size_t *PropSizeRet, ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetDeviceCount that also sets source code location
+/// information
+/// @details See also ::olGetDeviceCount
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetDeviceCountWithCodeLoc(ol_platform_handle_t Platform, uint32_t *NumDevices,
+                            ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetDevice that also sets source code location
+/// information
+/// @details See also ::olGetDevice
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceWithCodeLoc(
+    ol_platform_handle_t Platform, uint32_t NumEntries,
+    ol_device_handle_t *Devices, ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetDeviceInfo that also sets source code location
+/// information
+/// @details See also ::olGetDeviceInfo
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoWithCodeLoc(
+    ol_device_handle_t Device, ol_device_info_t PropName, size_t PropSize,
+    void *PropValue, ol_code_location_t *CodeLocation);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of olGetDeviceInfoSize that also sets source code location
+/// information
+/// @details See also ::olGetDeviceInfoSize
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSizeWithCodeLoc(
+    ol_device_handle_t Device, ol_device_info_t PropName, size_t *PropSizeRet,
+    ol_code_location_t *CodeLocation);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
diff --git a/offload/liboffload/include/generated/OffloadEntryPoints.inc b/offload/liboffload/include/generated/OffloadEntryPoints.inc
new file mode 100644
index 0000000000000..49c1c8169615e
--- /dev/null
+++ b/offload/liboffload/include/generated/OffloadEntryPoints.inc
@@ -0,0 +1,441 @@
+//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olInit_val() {
+  if (true /*enableParameterValidation*/) {
+  }
+
+  return olInit_impl();
+}
+OL_APIEXPORT ol_result_t OL_APICALL olInit() {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olInit";
+  }
+
+  ol_result_t Result = olInit_val();
+
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "()";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olInitWithCodeLoc(ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olInit();
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olShutDown_val() {
+  if (true /*enableParameterValidation*/) {
+  }
+
+  return olShutDown_impl();
+}
+OL_APIEXPORT ol_result_t OL_APICALL olShutDown() {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olShutDown";
+  }
+
+  ol_result_t Result = olShutDown_val();
+
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "()";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olShutDownWithCodeLoc(ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olShutDown();
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetPlatform_val(uint32_t NumEntries,
+                                   ol_platform_handle_t *Platforms) {
+  if (true /*enableParameterValidation*/) {
+    if (NumEntries == 0) {
+      return OL_ERRC_INVALID_SIZE;
+    }
+
+    if (NULL == Platforms) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetPlatform_impl(NumEntries, Platforms);
+}
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetPlatform(uint32_t NumEntries, ol_platform_handle_t *Platforms) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetPlatform";
+  }
+
+  ol_result_t Result = olGetPlatform_val(NumEntries, Platforms);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_platform_params_t Params = {&NumEntries, &Platforms};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetPlatformWithCodeLoc(uint32_t NumEntries,
+                                     ol_platform_handle_t *Platforms,
+                                     ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetPlatform(NumEntries, Platforms);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetPlatformCount_val(uint32_t *NumPlatforms) {
+  if (true /*enableParameterValidation*/) {
+    if (NULL == NumPlatforms) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetPlatformCount_impl(NumPlatforms);
+}
+OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformCount(uint32_t *NumPlatforms) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetPlatformCount";
+  }
+
+  ol_result_t Result = olGetPlatformCount_val(NumPlatforms);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_platform_count_params_t Params = {&NumPlatforms};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetPlatformCountWithCodeLoc(uint32_t *NumPlatforms,
+                                          ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetPlatformCount(NumPlatforms);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetPlatformInfo_val(ol_platform_handle_t Platform,
+                                       ol_platform_info_t PropName,
+                                       size_t PropSize, void *PropValue) {
+  if (true /*enableParameterValidation*/) {
+    if (PropSize == 0) {
+      return OL_ERRC_INVALID_SIZE;
+    }
+
+    if (NULL == Platform) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == PropValue) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetPlatformInfo_impl(Platform, PropName, PropSize, PropValue);
+}
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetPlatformInfo(ol_platform_handle_t Platform, ol_platform_info_t PropName,
+                  size_t PropSize, void *PropValue) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetPlatformInfo";
+  }
+
+  ol_result_t Result =
+      olGetPlatformInfo_val(Platform, PropName, PropSize, PropValue);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_platform_info_params_t Params = {&Platform, &PropName, &PropSize,
+                                            &PropValue};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetPlatformInfoWithCodeLoc(ol_platform_handle_t Platform,
+                                         ol_platform_info_t PropName,
+                                         size_t PropSize, void *PropValue,
+                                         ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result =
+      olGetPlatformInfo(Platform, PropName, PropSize, PropValue);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetPlatformInfoSize_val(ol_platform_handle_t Platform,
+                                           ol_platform_info_t PropName,
+                                           size_t *PropSizeRet) {
+  if (true /*enableParameterValidation*/) {
+    if (NULL == Platform) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == PropSizeRet) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetPlatformInfoSize_impl(Platform, PropName, PropSizeRet);
+}
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetPlatformInfoSize(ol_platform_handle_t Platform,
+                      ol_platform_info_t PropName, size_t *PropSizeRet) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetPlatformInfoSize";
+  }
+
+  ol_result_t Result =
+      olGetPlatformInfoSize_val(Platform, PropName, PropSizeRet);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_platform_info_size_params_t Params = {&Platform, &PropName,
+                                                 &PropSizeRet};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetPlatformInfoSizeWithCodeLoc(ol_platform_handle_t Platform,
+                                             ol_platform_info_t PropName,
+                                             size_t *PropSizeRet,
+                                             ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetPlatformInfoSize(Platform, PropName, PropSizeRet);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetDeviceCount_val(ol_platform_handle_t Platform,
+                                      uint32_t *NumDevices) {
+  if (true /*enableParameterValidation*/) {
+    if (NULL == Platform) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == NumDevices) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetDeviceCount_impl(Platform, NumDevices);
+}
+OL_APIEXPORT ol_result_t OL_APICALL
+olGetDeviceCount(ol_platform_handle_t Platform, uint32_t *NumDevices) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetDeviceCount";
+  }
+
+  ol_result_t Result = olGetDeviceCount_val(Platform, NumDevices);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_device_count_params_t Params = {&Platform, &NumDevices};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetDeviceCountWithCodeLoc(ol_platform_handle_t Platform,
+                                        uint32_t *NumDevices,
+                                        ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetDeviceCount(Platform, NumDevices);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetDevice_val(ol_platform_handle_t Platform,
+                                 uint32_t NumEntries,
+                                 ol_device_handle_t *Devices) {
+  if (true /*enableParameterValidation*/) {
+    if (NumEntries == 0) {
+      return OL_ERRC_INVALID_SIZE;
+    }
+
+    if (NULL == Platform) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == Devices) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetDevice_impl(Platform, NumEntries, Devices);
+}
+OL_APIEXPORT ol_result_t OL_APICALL olGetDevice(ol_platform_handle_t Platform,
+                                                uint32_t NumEntries,
+                                                ol_device_handle_t *Devices) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetDevice";
+  }
+
+  ol_result_t Result = olGetDevice_val(Platform, NumEntries, Devices);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_device_params_t Params = {&Platform, &NumEntries, &Devices};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetDeviceWithCodeLoc(ol_platform_handle_t Platform,
+                                   uint32_t NumEntries,
+                                   ol_device_handle_t *Devices,
+                                   ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetDevice(Platform, NumEntries, Devices);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetDeviceInfo_val(ol_device_handle_t Device,
+                                     ol_device_info_t PropName, size_t PropSize,
+                                     void *PropValue) {
+  if (true /*enableParameterValidation*/) {
+    if (PropSize == 0) {
+      return OL_ERRC_INVALID_SIZE;
+    }
+
+    if (NULL == Device) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == PropValue) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetDeviceInfo_impl(Device, PropName, PropSize, PropValue);
+}
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfo(ol_device_handle_t Device,
+                                                    ol_device_info_t PropName,
+                                                    size_t PropSize,
+                                                    void *PropValue) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetDeviceInfo";
+  }
+
+  ol_result_t Result =
+      olGetDeviceInfo_val(Device, PropName, PropSize, PropValue);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_device_info_params_t Params = {&Device, &PropName, &PropSize,
+                                          &PropValue};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetDeviceInfoWithCodeLoc(ol_device_handle_t Device,
+                                       ol_device_info_t PropName,
+                                       size_t PropSize, void *PropValue,
+                                       ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetDeviceInfo(Device, PropName, PropSize, PropValue);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+ol_impl_result_t olGetDeviceInfoSize_val(ol_device_handle_t Device,
+                                         ol_device_info_t PropName,
+                                         size_t *PropSizeRet) {
+  if (true /*enableParameterValidation*/) {
+    if (NULL == Device) {
+      return OL_ERRC_INVALID_NULL_HANDLE;
+    }
+
+    if (NULL == PropSizeRet) {
+      return OL_ERRC_INVALID_NULL_POINTER;
+    }
+  }
+
+  return olGetDeviceInfoSize_impl(Device, PropName, PropSizeRet);
+}
+OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSize(
+    ol_device_handle_t Device, ol_device_info_t PropName, size_t *PropSizeRet) {
+  if (offloadConfig().TracingEnabled) {
+    std::cout << "---> olGetDeviceInfoSize";
+  }
+
+  ol_result_t Result = olGetDeviceInfoSize_val(Device, PropName, PropSizeRet);
+
+  if (offloadConfig().TracingEnabled) {
+    ol_get_device_info_size_params_t Params = {&Device, &PropName,
+                                               &PropSizeRet};
+    std::cout << "(" << &Params << ")";
+    std::cout << "-> " << Result << "\n";
+    if (Result && Result->Details) {
+      std::cout << "     *Error Details* " << Result->Details << " \n";
+    }
+  }
+  return Result;
+}
+ol_result_t olGetDeviceInfoSizeWithCodeLoc(ol_device_handle_t Device,
+                                           ol_device_info_t PropName,
+                                           size_t *PropSizeRet,
+                                           ol_code_location_t *CodeLocation) {
+  currentCodeLocation() = CodeLocation;
+  ol_result_t Result = olGetDeviceInfoSize(Device, PropName, PropSizeRet);
+
+  currentCodeLocation() = nullptr;
+  return Result;
+}
diff --git a/offload/liboffload/include/generated/OffloadFuncs.inc b/offload/liboffload/include/generated/OffloadFuncs.inc
new file mode 100644
index 0000000000000..48115493c790f
--- /dev/null
+++ b/offload/liboffload/include/generated/OffloadFuncs.inc
@@ -0,0 +1,34 @@
+//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_FUNC
+#error Please define the macro OFFLOAD_FUNC(Function)
+#endif
+
+OFFLOAD_FUNC(olInit)
+OFFLOAD_FUNC(olShutDown)
+OFFLOAD_FUNC(olGetPlatform)
+OFFLOAD_FUNC(olGetPlatformCount)
+OFFLOAD_FUNC(olGetPlatformInfo)
+OFFLOAD_FUNC(olGetPlatformInfoSize)
+OFFLOAD_FUNC(olGetDeviceCount)
+OFFLOAD_FUNC(olGetDevice)
+OFFLOAD_FUNC(olGetDeviceInfo)
+OFFLOAD_FUNC(olGetDeviceInfoSize)
+OFFLOAD_FUNC(olInitWithCodeLoc)
+OFFLOAD_FUNC(olShutDownWithCodeLoc)
+OFFLOAD_FUNC(olGetPlatformWithCodeLoc)
+OFFLOAD_FUNC(olGetPlatformCountWithCodeLoc)
+OFFLOAD_FUNC(olGetPlatformInfoWithCodeLoc)
+OFFLOAD_FUNC(olGetPlatformInfoSizeWithCodeLoc)
+OFFLOAD_FUNC(olGetDeviceCountWithCodeLoc)
+OFFLOAD_FUNC(olGetDeviceWithCodeLoc)
+OFFLOAD_FUNC(olGetDeviceInfoWithCodeLoc)
+OFFLOAD_FUNC(olGetDeviceInfoSizeWithCodeLoc)
+
+#undef OFFLOAD_FUNC
diff --git a/offload/liboffload/include/generated/OffloadImplFuncDecls.inc b/offload/liboffload/include/generated/OffloadImplFuncDecls.inc
new file mode 100644
index 0000000000000..5b26b2653a05d
--- /dev/null
+++ b/offload/liboffload/include/generated/OffloadImplFuncDecls.inc
@@ -0,0 +1,38 @@
+//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+ol_impl_result_t olInit_impl();
+
+ol_impl_result_t olShutDown_impl();
+
+ol_impl_result_t olGetPlatform_impl(uint32_t NumEntries,
+                                    ol_platform_handle_t *Platforms);
+
+ol_impl_result_t olGetPlatformCount_impl(uint32_t *NumPlatforms);
+
+ol_impl_result_t olGetPlatformInfo_impl(ol_platform_handle_t Platform,
+                                        ol_platform_info_t PropName,
+                                        size_t PropSize, void *PropValue);
+
+ol_impl_result_t olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,
+                                            ol_platform_info_t PropName,
+                                            size_t *PropSizeRet);
+
+ol_impl_result_t olGetDeviceCount_impl(ol_platform_handle_t Platform,
+                                       uint32_t *NumDevices);
+
+ol_impl_result_t olGetDevice_impl(ol_platform_handle_t Platform,
+                                  uint32_t NumEntries,
+                                  ol_device_handle_t *Devices);
+
+ol_impl_result_t olGetDeviceInfo_impl(ol_device_handle_t Device,
+                                      ol_device_info_t PropName,
+                                      size_t PropSize, void *PropValue);
+
+ol_impl_result_t olGetDeviceInfoSize_impl(ol_device_handle_t Device,
+                                          ol_device_info_t PropName,
+                                          size_t *PropSizeRet);
diff --git a/offload/liboffload/include/generated/OffloadPrint.hpp b/offload/liboffload/include/generated/OffloadPrint.hpp
new file mode 100644
index 0000000000000..8981bb054a4cb
--- /dev/null
+++ b/offload/liboffload/include/generated/OffloadPrint.hpp
@@ -0,0 +1,428 @@
+//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Auto-generated file, do not manually edit.
+
+#pragma once
+
+#include <OffloadAPI.h>
+#include <ostream>
+
+template <typename T>
+inline ol_result_t printPtr(std::ostream &os, const T *ptr);
+template <typename T>
+inline void printTagged(std::ostream &os, const void *ptr, T value,
+                        size_t size);
+template <typename T> struct is_handle : std::false_type {};
+template <> struct is_handle<ol_platform_handle_t> : std::true_type {};
+template <> struct is_handle<ol_device_handle_t> : std::true_type {};
+template <> struct is_handle<ol_context_handle_t> : std::true_type {};
+template <typename T> inline constexpr bool is_handle_v = is_handle<T>::value;
+
+inline std::ostream &operator<<(std::ostream &os, enum ol_errc_t value);
+inline std::ostream &operator<<(std::ostream &os,
+                                enum ol_platform_info_t value);
+inline std::ostream &operator<<(std::ostream &os,
+                                enum ol_platform_backend_t value);
+inline std::ostream &operator<<(std::ostream &os, enum ol_device_type_t value);
+inline std::ostream &operator<<(std::ostream &os, enum ol_device_info_t value);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ol_errc_t type
+/// @returns std::ostream &
+inline std::ostream &operator<<(std::ostream &os, enum ol_errc_t value) {
+  switch (value) {
+  case OL_ERRC_SUCCESS:
+    os << "OL_ERRC_SUCCESS";
+    break;
+  case OL_ERRC_INVALID_VALUE:
+    os << "OL_ERRC_INVALID_VALUE";
+    break;
+  case OL_ERRC_INVALID_PLATFORM:
+    os << "OL_ERRC_INVALID_PLATFORM";
+    break;
+  case OL_ERRC_DEVICE_NOT_FOUND:
+    os << "OL_ERRC_DEVICE_NOT_FOUND";
+    break;
+  case OL_ERRC_INVALID_DEVICE:
+    os << "OL_ERRC_INVALID_DEVICE";
+    break;
+  case OL_ERRC_DEVICE_LOST:
+    os << "OL_ERRC_DEVICE_LOST";
+    break;
+  case OL_ERRC_UNINITIALIZED:
+    os << "OL_ERRC_UNINITIALIZED";
+    break;
+  case OL_ERRC_OUT_OF_RESOURCES:
+    os << "OL_ERRC_OUT_OF_RESOURCES";
+    break;
+  case OL_ERRC_UNSUPPORTED_VERSION:
+    os << "OL_ERRC_UNSUPPORTED_VERSION";
+    break;
+  case OL_ERRC_UNSUPPORTED_FEATURE:
+    os << "OL_ERRC_UNSUPPORTED_FEATURE";
+    break;
+  case OL_ERRC_INVALID_ARGUMENT:
+    os << "OL_ERRC_INVALID_ARGUMENT";
+    break;
+  case OL_ERRC_INVALID_NULL_HANDLE:
+    os << "OL_ERRC_INVALID_NULL_HANDLE";
+    break;
+  case OL_ERRC_INVALID_NULL_POINTER:
+    os << "OL_ERRC_INVALID_NULL_POINTER";
+    break;
+  case OL_ERRC_INVALID_SIZE:
+    os << "OL_ERRC_INVALID_SIZE";
+    break;
+  case OL_ERRC_INVALID_ENUMERATION:
+    os << "OL_ERRC_INVALID_ENUMERATION";
+    break;
+  case OL_ERRC_UNSUPPORTED_ENUMERATION:
+    os << "OL_ERRC_UNSUPPORTED_ENUMERATION";
+    break;
+  case OL_ERRC_UNKNOWN:
+    os << "OL_ERRC_UNKNOWN";
+    break;
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ol_platform_info_t type
+/// @returns std::ostream &
+inline std::ostream &operator<<(std::ostream &os,
+                                enum ol_platform_info_t value) {
+  switch (value) {
+  case OL_PLATFORM_INFO_NAME:
+    os << "OL_PLATFORM_INFO_NAME";
+    break;
+  case OL_PLATFORM_INFO_VENDOR_NAME:
+    os << "OL_PLATFORM_INFO_VENDOR_NAME";
+    break;
+  case OL_PLATFORM_INFO_VERSION:
+    os << "OL_PLATFORM_INFO_VERSION";
+    break;
+  case OL_PLATFORM_INFO_BACKEND:
+    os << "OL_PLATFORM_INFO_BACKEND";
+    break;
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print type-tagged ol_platform_info_t enum value
+/// @returns std::ostream &
+template <>
+inline void printTagged(std::ostream &os, const void *ptr,
+                        ol_platform_info_t value, size_t size) {
+  if (ptr == NULL) {
+    printPtr(os, ptr);
+    return;
+  }
+
+  switch (value) {
+  case OL_PLATFORM_INFO_NAME: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  case OL_PLATFORM_INFO_VENDOR_NAME: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  case OL_PLATFORM_INFO_VERSION: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  case OL_PLATFORM_INFO_BACKEND: {
+    const ol_platform_backend_t *const tptr =
+        (const ol_platform_backend_t *const)ptr;
+    os << (const void *)tptr << " (";
+    os << *tptr;
+    os << ")";
+    break;
+  }
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+}
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ol_platform_backend_t type
+/// @returns std::ostream &
+inline std::ostream &operator<<(std::ostream &os,
+                                enum ol_platform_backend_t value) {
+  switch (value) {
+  case OL_PLATFORM_BACKEND_UNKNOWN:
+    os << "OL_PLATFORM_BACKEND_UNKNOWN";
+    break;
+  case OL_PLATFORM_BACKEND_CUDA:
+    os << "OL_PLATFORM_BACKEND_CUDA";
+    break;
+  case OL_PLATFORM_BACKEND_AMDGPU:
+    os << "OL_PLATFORM_BACKEND_AMDGPU";
+    break;
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ol_device_type_t type
+/// @returns std::ostream &
+inline std::ostream &operator<<(std::ostream &os, enum ol_device_type_t value) {
+  switch (value) {
+  case OL_DEVICE_TYPE_DEFAULT:
+    os << "OL_DEVICE_TYPE_DEFAULT";
+    break;
+  case OL_DEVICE_TYPE_ALL:
+    os << "OL_DEVICE_TYPE_ALL";
+    break;
+  case OL_DEVICE_TYPE_GPU:
+    os << "OL_DEVICE_TYPE_GPU";
+    break;
+  case OL_DEVICE_TYPE_CPU:
+    os << "OL_DEVICE_TYPE_CPU";
+    break;
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ol_device_info_t type
+/// @returns std::ostream &
+inline std::ostream &operator<<(std::ostream &os, enum ol_device_info_t value) {
+  switch (value) {
+  case OL_DEVICE_INFO_TYPE:
+    os << "OL_DEVICE_INFO_TYPE";
+    break;
+  case OL_DEVICE_INFO_PLATFORM:
+    os << "OL_DEVICE_INFO_PLATFORM";
+    break;
+  case OL_DEVICE_INFO_NAME:
+    os << "OL_DEVICE_INFO_NAME";
+    break;
+  case OL_DEVICE_INFO_VENDOR:
+    os << "OL_DEVICE_INFO_VENDOR";
+    break;
+  case OL_DEVICE_INFO_DRIVER_VERSION:
+    os << "OL_DEVICE_INFO_DRIVER_VERSION";
+    break;
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print type-tagged ol_device_info_t enum value
+/// @returns std::ostream &
+template <>
+inline void printTagged(std::ostream &os, const void *ptr,
+                        ol_device_info_t value, size_t size) {
+  if (ptr == NULL) {
+    printPtr(os, ptr);
+    return;
+  }
+
+  switch (value) {
+  case OL_DEVICE_INFO_TYPE: {
+    const ol_device_type_t *const tptr = (const ol_device_type_t *const)ptr;
+    os << (const void *)tptr << " (";
+    os << *tptr;
+    os << ")";
+    break;
+  }
+  case OL_DEVICE_INFO_PLATFORM: {
+    const ol_platform_handle_t *const tptr =
+        (const ol_platform_handle_t *const)ptr;
+    os << (const void *)tptr << " (";
+    os << *tptr;
+    os << ")";
+    break;
+  }
+  case OL_DEVICE_INFO_NAME: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  case OL_DEVICE_INFO_VENDOR: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  case OL_DEVICE_INFO_DRIVER_VERSION: {
+    printPtr(os, (const char *)ptr);
+    break;
+  }
+  default:
+    os << "unknown enumerator";
+    break;
+  }
+}
+
+inline std::ostream &operator<<(std::ostream &os,
+                                const ol_error_struct_t *Err) {
+  if (Err == nullptr) {
+    os << "OL_SUCCESS";
+  } else {
+    os << Err->Code;
+  }
+  return os;
+}
+
+inline std::ostream &operator<<(std::ostream &os,
+                                const struct ol_get_platform_params_t *params) {
+  os << ".NumEntries = ";
+  os << *params->pNumEntries;
+  os << ", ";
+  os << ".Platforms = ";
+  os << "{";
+  for (size_t i = 0; i < *params->pNumEntries; i++) {
+    if (i > 0) {
+      os << ", ";
+    }
+    printPtr(os, (*params->pPlatforms)[i]);
+  }
+  os << "}";
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os,
+           const struct ol_get_platform_count_params_t *params) {
+  os << ".NumPlatforms = ";
+  printPtr(os, *params->pNumPlatforms);
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os,
+           const struct ol_get_platform_info_params_t *params) {
+  os << ".Platform = ";
+  printPtr(os, *params->pPlatform);
+  os << ", ";
+  os << ".PropName = ";
+  os << *params->pPropName;
+  os << ", ";
+  os << ".PropSize = ";
+  os << *params->pPropSize;
+  os << ", ";
+  os << ".PropValue = ";
+  printTagged(os, *params->pPropValue, *params->pPropName, *params->pPropSize);
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os,
+           const struct ol_get_platform_info_size_params_t *params) {
+  os << ".Platform = ";
+  printPtr(os, *params->pPlatform);
+  os << ", ";
+  os << ".PropName = ";
+  os << *params->pPropName;
+  os << ", ";
+  os << ".PropSizeRet = ";
+  printPtr(os, *params->pPropSizeRet);
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os,
+           const struct ol_get_device_count_params_t *params) {
+  os << ".Platform = ";
+  printPtr(os, *params->pPlatform);
+  os << ", ";
+  os << ".NumDevices = ";
+  printPtr(os, *params->pNumDevices);
+  return os;
+}
+
+inline std::ostream &operator<<(std::ostream &os,
+                                const struct ol_get_device_params_t *params) {
+  os << ".Platform = ";
+  printPtr(os, *params->pPlatform);
+  os << ", ";
+  os << ".NumEntries = ";
+  os << *params->pNumEntries;
+  os << ", ";
+  os << ".Devices = ";
+  os << "{";
+  for (size_t i = 0; i < *params->pNumEntries; i++) {
+    if (i > 0) {
+      os << ", ";
+    }
+    printPtr(os, (*params->pDevices)[i]);
+  }
+  os << "}";
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os, const struct ol_get_device_info_params_t *params) {
+  os << ".Device = ";
+  printPtr(os, *params->pDevice);
+  os << ", ";
+  os << ".PropName = ";
+  os << *params->pPropName;
+  os << ", ";
+  os << ".PropSize = ";
+  os << *params->pPropSize;
+  os << ", ";
+  os << ".PropValue = ";
+  printTagged(os, *params->pPropValue, *params->pPropName, *params->pPropSize);
+  return os;
+}
+
+inline std::ostream &
+operator<<(std::ostream &os,
+           const struct ol_get_device_info_size_params_t *params) {
+  os << ".Device = ";
+  printPtr(os, *params->pDevice);
+  os << ", ";
+  os << ".PropName = ";
+  os << *params->pPropName;
+  os << ", ";
+  os << ".PropSizeRet = ";
+  printPtr(os, *params->pPropSizeRet);
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// @brief Print pointer value
+template <typename T>
+inline ol_result_t printPtr(std::ostream &os, const T *ptr) {
+  if (ptr == nullptr) {
+    os << "nullptr";
+  } else if constexpr (std::is_pointer_v<T>) {
+    os << (const void *)(ptr) << " (";
+    printPtr(os, *ptr);
+    os << ")";
+  } else if constexpr (std::is_void_v<T> || is_handle_v<T *>) {
+    os << (const void *)ptr;
+  } else if constexpr (std::is_same_v<std::remove_cv_t<T>, char>) {
+    os << (const void *)(ptr) << " (";
+    os << ptr;
+    os << ")";
+  } else {
+    os << (const void *)(ptr) << " (";
+    os << *ptr;
+    os << ")";
+  }
+
+  return OL_SUCCESS;
+}
diff --git a/offload/liboffload/src/Helpers.hpp b/offload/liboffload/src/Helpers.hpp
new file mode 100644
index 0000000000000..d003d30252462
--- /dev/null
+++ b/offload/liboffload/src/Helpers.hpp
@@ -0,0 +1,95 @@
+//===- helpers.hpp- GetInfo return helpers for the new LLVM/Offload API ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The getInfo*/ReturnHelper facilities provide shortcut way of writing return
+// data + size for the various getInfo APIs. Based on the equivalent
+// implementations in Unified Runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OffloadAPI.h"
+
+#include <cstring>
+
+template <typename T, typename Assign>
+ol_errc_t getInfoImpl(size_t ParamValueSize, void *ParamValue,
+                      size_t *ParamValueSizeRet, T Value, size_t ValueSize,
+                      Assign &&AssignFunc) {
+  if (!ParamValue && !ParamValueSizeRet) {
+    return OL_ERRC_INVALID_NULL_POINTER;
+  }
+
+  if (ParamValue != nullptr) {
+    if (ParamValueSize < ValueSize) {
+      return OL_ERRC_INVALID_SIZE;
+    }
+    AssignFunc(ParamValue, Value, ValueSize);
+  }
+
+  if (ParamValueSizeRet != nullptr) {
+    *ParamValueSizeRet = ValueSize;
+  }
+
+  return OL_ERRC_SUCCESS;
+}
+
+template <typename T>
+ol_errc_t getInfo(size_t ParamValueSize, void *ParamValue,
+                  size_t *ParamValueSizeRet, T Value) {
+  auto Assignment = [](void *ParamValue, T Value, size_t) {
+    *static_cast<T *>(ParamValue) = Value;
+  };
+
+  return getInfoImpl(ParamValueSize, ParamValue, ParamValueSizeRet, Value,
+                     sizeof(T), Assignment);
+}
+
+template <typename T>
+ol_errc_t getInfoArray(size_t array_length, size_t ParamValueSize,
+                       void *ParamValue, size_t *ParamValueSizeRet,
+                       const T *Value) {
+  return getInfoImpl(ParamValueSize, ParamValue, ParamValueSizeRet, Value,
+                     array_length * sizeof(T), memcpy);
+}
+
+template <>
+inline ol_errc_t getInfo<const char *>(size_t ParamValueSize, void *ParamValue,
+                                       size_t *ParamValueSizeRet,
+                                       const char *Value) {
+  return getInfoArray(strlen(Value) + 1, ParamValueSize, ParamValue,
+                      ParamValueSizeRet, Value);
+}
+
+class ReturnHelper {
+public:
+  ReturnHelper(size_t ParamValueSize, void *ParamValue,
+               size_t *ParamValueSizeRet)
+      : ParamValueSize(ParamValueSize), ParamValue(ParamValue),
+        ParamValueSizeRet(ParamValueSizeRet) {}
+
+  // A version where in/out info size is represented by a single pointer
+  // to a value which is updated on return
+  ReturnHelper(size_t *ParamValueSize, void *ParamValue)
+      : ParamValueSize(*ParamValueSize), ParamValue(ParamValue),
+        ParamValueSizeRet(ParamValueSize) {}
+
+  // Scalar return Value
+  template <class T> ol_errc_t operator()(const T &t) {
+    return getInfo(ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  }
+
+  // Array return Value
+  template <class T> ol_errc_t operator()(const T *t, size_t s) {
+    return getInfoArray(s, ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  }
+
+protected:
+  size_t ParamValueSize;
+  void *ParamValue;
+  size_t *ParamValueSizeRet;
+};
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
new file mode 100644
index 0000000000000..457f1053f1634
--- /dev/null
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -0,0 +1,247 @@
+//===- ol_impl.cpp - Implementation of the new LLVM/Offload API ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains the definitions of the new LLVM/Offload API entry points. See
+// new-api/API/README.md for more information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OffloadImpl.hpp"
+#include "Helpers.hpp"
+#include "PluginManager.h"
+#include "llvm/Support/FormatVariadic.h"
+#include <OffloadAPI.h>
+
+#include <mutex>
+
+using namespace llvm;
+using namespace llvm::omp::target::plugin;
+
+// Handle type definitions. Ideally these would be 1:1 with the plugins
+struct ol_device_handle_t_ {
+  int DeviceNum;
+  GenericDeviceTy &Device;
+  ol_platform_handle_t Platform;
+};
+
+struct ol_platform_handle_t_ {
+  std::unique_ptr<GenericPluginTy> Plugin;
+  std::vector<ol_device_handle_t_> Devices;
+};
+
+using PlatformVecT = SmallVector<ol_platform_handle_t_, 4>;
+PlatformVecT &Platforms() {
+  static PlatformVecT Platforms;
+  return Platforms;
+}
+
+// TODO: Some plugins expect to be linked into libomptarget which defines these
+// symbols to implement ompt callbacks. The least invasive workaround here is to
+// define them in libLLVMOffload as false/null so they are never used. In future
+// it would be better to allow the plugins to implement callbacks without
+// pulling in details from libomptarget.
+#ifdef OMPT_SUPPORT
+namespace llvm::omp::target {
+namespace ompt {
+bool Initialized = false;
+ompt_get_callback_t lookupCallbackByCode = nullptr;
+ompt_function_lookup_t lookupCallbackByName = nullptr;
+} // namespace ompt
+} // namespace llvm::omp::target
+#endif
+
+// Every plugin exports this method to create an instance of the plugin type.
+#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
+#include "Shared/Targets.def"
+
+void initPlugins() {
+  // Attempt to create an instance of each supported plugin.
+#define PLUGIN_TARGET(Name)                                                    \
+  do {                                                                         \
+    Platforms().emplace_back(ol_platform_handle_t_{                            \
+        std::unique_ptr<GenericPluginTy>(createPlugin_##Name()), {}});         \
+  } while (false);
+#include "Shared/Targets.def"
+
+  // Preemptively initialize all devices in the plugin so we can just return
+  // them from deviceGet
+  for (auto &Platform : Platforms()) {
+    auto Err = Platform.Plugin->init();
+    [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+    for (auto DevNum = 0; DevNum < Platform.Plugin->number_of_devices();
+         DevNum++) {
+      if (Platform.Plugin->init_device(DevNum) == OFFLOAD_SUCCESS) {
+        Platform.Devices.emplace_back(ol_device_handle_t_{
+            DevNum, Platform.Plugin->getDevice(DevNum), &Platform});
+      }
+    }
+  }
+
+  offloadConfig().TracingEnabled = std::getenv("OFFLOAD_TRACE");
+}
+
+// TODO: We can properly reference count here and manage the resources in a more
+// clever way
+ol_impl_result_t olInit_impl() {
+  static std::once_flag InitFlag;
+  std::call_once(InitFlag, initPlugins);
+
+  return OL_SUCCESS;
+}
+ol_impl_result_t olShutDown_impl() { return OL_SUCCESS; }
+
+ol_impl_result_t olGetPlatformCount_impl(uint32_t *NumPlatforms) {
+  *NumPlatforms = Platforms().size();
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetPlatform_impl(uint32_t NumEntries,
+                                    ol_platform_handle_t *PlatformsOut) {
+  if (NumEntries > Platforms().size()) {
+    return {OL_ERRC_INVALID_SIZE,
+            std::string{formatv("{0} platform(s) available but {1} requested.",
+                                Platforms().size(), NumEntries)}};
+  }
+
+  for (uint32_t PlatformIndex = 0; PlatformIndex < NumEntries;
+       PlatformIndex++) {
+    PlatformsOut[PlatformIndex] = &(Platforms())[PlatformIndex];
+  }
+
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetPlatformInfoImplDetail(ol_platform_handle_t Platform,
+                                             ol_platform_info_t PropName,
+                                             size_t PropSize, void *PropValue,
+                                             size_t *PropSizeRet) {
+  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+
+  switch (PropName) {
+  case OL_PLATFORM_INFO_NAME:
+    return ReturnValue(Platform->Plugin->getName());
+  case OL_PLATFORM_INFO_VENDOR_NAME:
+    // TODO: Implement this
+    return ReturnValue("Unknown platform vendor");
+  case OL_PLATFORM_INFO_VERSION: {
+    return ReturnValue(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,
+                               OL_VERSION_MINOR, OL_VERSION_PATCH)
+                           .str()
+                           .c_str());
+  }
+  case OL_PLATFORM_INFO_BACKEND: {
+    auto PluginName = Platform->Plugin->getName();
+    if (PluginName == StringRef("CUDA")) {
+      return ReturnValue(OL_PLATFORM_BACKEND_CUDA);
+    } else if (PluginName == StringRef("AMDGPU")) {
+      return ReturnValue(OL_PLATFORM_BACKEND_AMDGPU);
+    } else {
+      return ReturnValue(OL_PLATFORM_BACKEND_UNKNOWN);
+    }
+  }
+  default:
+    return OL_ERRC_INVALID_ENUMERATION;
+  }
+
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetPlatformInfo_impl(ol_platform_handle_t Platform,
+                                        ol_platform_info_t PropName,
+                                        size_t PropSize, void *PropValue) {
+  return olGetPlatformInfoImplDetail(Platform, PropName, PropSize, PropValue,
+                                     nullptr);
+}
+
+ol_impl_result_t olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,
+                                            ol_platform_info_t PropName,
+                                            size_t *PropSizeRet) {
+  return olGetPlatformInfoImplDetail(Platform, PropName, 0, nullptr,
+                                     PropSizeRet);
+}
+
+ol_impl_result_t olGetDeviceCount_impl(ol_platform_handle_t Platform,
+                                       uint32_t *pNumDevices) {
+  *pNumDevices = static_cast<uint32_t>(Platform->Devices.size());
+
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetDevice_impl(ol_platform_handle_t Platform,
+                                  uint32_t NumEntries,
+                                  ol_device_handle_t *Devices) {
+  if (NumEntries > Platform->Devices.size()) {
+    return OL_ERRC_INVALID_SIZE;
+  }
+
+  for (uint32_t DeviceIndex = 0; DeviceIndex < NumEntries; DeviceIndex++) {
+    Devices[DeviceIndex] = &(Platform->Devices[DeviceIndex]);
+  }
+
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetDeviceInfoImplDetail(ol_device_handle_t Device,
+                                           ol_device_info_t PropName,
+                                           size_t PropSize, void *PropValue,
+                                           size_t *PropSizeRet) {
+
+  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+
+  InfoQueueTy DevInfo;
+  if (auto Err = Device->Device.obtainInfoImpl(DevInfo))
+    return OL_ERRC_OUT_OF_RESOURCES;
+
+  // Find the info if it exists under any of the given names
+  auto GetInfo = [&DevInfo](std::vector<std::string> Names) {
+    for (auto Name : Names) {
+      auto InfoKeyMatches = [&](const InfoQueueTy::InfoQueueEntryTy &Info) {
+        return Info.Key == Name;
+      };
+      auto Item = std::find_if(DevInfo.getQueue().begin(),
+                               DevInfo.getQueue().end(), InfoKeyMatches);
+
+      if (Item != std::end(DevInfo.getQueue())) {
+        return Item->Value;
+      }
+    }
+
+    return std::string("");
+  };
+
+  switch (PropName) {
+  case OL_DEVICE_INFO_PLATFORM:
+    return ReturnValue(Device->Platform);
+  case OL_DEVICE_INFO_TYPE:
+    return ReturnValue(OL_DEVICE_TYPE_GPU);
+  case OL_DEVICE_INFO_NAME:
+    return ReturnValue(GetInfo({"Device Name"}).c_str());
+  case OL_DEVICE_INFO_VENDOR:
+    return ReturnValue(GetInfo({"Vendor Name"}).c_str());
+  case OL_DEVICE_INFO_DRIVER_VERSION:
+    return ReturnValue(
+        GetInfo({"CUDA Driver Version", "HSA Runtime Version"}).c_str());
+  default:
+    return OL_ERRC_INVALID_ENUMERATION;
+  }
+
+  return OL_SUCCESS;
+}
+
+ol_impl_result_t olGetDeviceInfo_impl(ol_device_handle_t Device,
+                                      ol_device_info_t PropName,
+                                      size_t PropSize, void *PropValue) {
+  return olGetDeviceInfoImplDetail(Device, PropName, PropSize, PropValue,
+                                   nullptr);
+}
+
+ol_impl_result_t olGetDeviceInfoSize_impl(ol_device_handle_t Device,
+                                          ol_device_info_t PropName,
+                                          size_t *PropSizeRet) {
+  return olGetDeviceInfoImplDetail(Device, PropName, 0, nullptr, PropSizeRet);
+}
diff --git a/offload/liboffload/src/OffloadLib.cpp b/offload/liboffload/src/OffloadLib.cpp
new file mode 100644
index 0000000000000..37876713212c9
--- /dev/null
+++ b/offload/liboffload/src/OffloadLib.cpp
@@ -0,0 +1,44 @@
+//===- offload_lib.cpp - Entry points for the new LLVM/Offload API --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file pulls in the tablegen'd API entry point functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OffloadImpl.hpp"
+#include <OffloadAPI.h>
+#include <OffloadPrint.hpp>
+
+#include <iostream>
+
+llvm::StringSet<> &errorStrs() {
+  static llvm::StringSet<> ErrorStrs;
+  return ErrorStrs;
+}
+
+ErrSetT &errors() {
+  static ErrSetT Errors{};
+  return Errors;
+}
+
+ol_code_location_t *&currentCodeLocation() {
+  thread_local ol_code_location_t *CodeLoc = nullptr;
+  return CodeLoc;
+}
+
+OffloadConfig &offloadConfig() {
+  static OffloadConfig Config{};
+  return Config;
+}
+
+// Pull in the declarations for the implementation funtions. The actual entry
+// points in this file wrap these.
+#include "OffloadImplFuncDecls.inc"
+
+// Pull in the tablegen'd entry point definitions.
+#include "OffloadEntryPoints.inc"
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index d31bf557669ea..9b5b12bea7142 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -12,6 +12,7 @@ function(add_target_library target_name lib_name)
       CodeGen
       Core
       Extensions
+      FrontendOffloading
       InstCombine
       Instrumentation
       IPO
@@ -20,6 +21,7 @@ function(add_target_library target_name lib_name)
       MC
       Object
       Passes
+      ProfileData
       Remarks
       ScalarOpts
       Support
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index b40c62d43226f..47cd2feefc728 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -10,12 +10,11 @@ target_include_directories(omptarget.rtl.amdgpu PRIVATE
 
 if(hsa-runtime64_FOUND AND NOT "amdgpu" IN_LIST LIBOMPTARGET_DLOPEN_PLUGINS)
   message(STATUS "Building AMDGPU plugin linked against libhsa")
-  target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64 LLVMFrontendOffloading)
+  target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64)
 else()
   message(STATUS "Building AMDGPU plugin for dlopened libhsa")
   target_include_directories(omptarget.rtl.amdgpu PRIVATE dynamic_hsa)
   target_sources(omptarget.rtl.amdgpu PRIVATE dynamic_hsa/hsa.cpp)
-  target_link_libraries(omptarget.rtl.amdgpu PRIVATE LLVMFrontendOffloading)
 endif()
 
 # Configure testing for the AMDGPU plugin. We will build tests if we could a
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 6356fa0554a9c..7114dad020e3a 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -190,8 +190,8 @@ Error asyncMemCopy(bool UseMultipleSdmaEngines, void *Dst, hsa_agent_t DstAgent,
 #endif
 }
 
-Expected<std::string> getTargetTripleAndFeatures(hsa_agent_t Agent) {
-  std::string Target;
+Error getTargetTripleAndFeatures(hsa_agent_t Agent,
+                                 SmallVector<SmallString<32>> &Targets) {
   auto Err = hsa_utils::iterateAgentISAs(Agent, [&](hsa_isa_t ISA) {
     uint32_t Length;
     hsa_status_t Status;
@@ -205,13 +205,13 @@ Expected<std::string> getTargetTripleAndFeatures(hsa_agent_t Agent) {
       return Status;
 
     llvm::StringRef TripleTarget(ISAName.begin(), Length);
-    if (TripleTarget.consume_front("amdgcn-amd-amdhsa"))
-      Target = TripleTarget.ltrim('-').rtrim('\0').str();
+    if (TripleTarget.consume_front("amdgcn-amd-amdhsa")) {
+      auto Target = TripleTarget.ltrim('-').rtrim('\0');
+      Targets.push_back(Target);
+    }
     return HSA_STATUS_SUCCESS;
   });
-  if (Err)
-    return Err;
-  return Target;
+  return Err;
 }
 } // namespace hsa_utils
 
@@ -559,15 +559,15 @@ struct AMDGPUKernelTy : public GenericKernelTy {
   }
 
   /// Launch the AMDGPU kernel function.
-  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads,
-                   uint64_t NumBlocks, KernelArgsTy &KernelArgs,
+  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
+                   uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
 
   /// Print more elaborate kernel launch info for AMDGPU
   Error printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
-                               KernelArgsTy &KernelArgs, uint32_t NumThreads,
-                               uint64_t NumBlocks) const override;
+                               KernelArgsTy &KernelArgs, uint32_t NumThreads[3],
+                               uint32_t NumBlocks[3]) const override;
 
   /// Get group and private segment kernel size.
   uint32_t getGroupSize() const { return GroupSize; }
@@ -719,7 +719,7 @@ struct AMDGPUQueueTy {
   /// Push a kernel launch to the queue. The kernel launch requires an output
   /// signal and can define an optional input signal (nullptr if none).
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
-                         uint32_t NumThreads, uint64_t NumBlocks,
+                         uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
                          AMDGPUSignalTy *OutputSignal,
                          AMDGPUSignalTy *InputSignal) {
@@ -746,14 +746,18 @@ struct AMDGPUQueueTy {
     assert(Packet && "Invalid packet");
 
     // The first 32 bits of the packet are written after the other fields
-    uint16_t Setup = UINT16_C(1) << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
-    Packet->workgroup_size_x = NumThreads;
-    Packet->workgroup_size_y = 1;
-    Packet->workgroup_size_z = 1;
+    uint16_t Dims = NumBlocks[2] * NumThreads[2] > 1
+                        ? 3
+                        : 1 + (NumBlocks[1] * NumThreads[1] != 1);
+    uint16_t Setup = UINT16_C(Dims)
+                     << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
+    Packet->workgroup_size_x = NumThreads[0];
+    Packet->workgroup_size_y = NumThreads[1];
+    Packet->workgroup_size_z = NumThreads[2];
     Packet->reserved0 = 0;
-    Packet->grid_size_x = NumBlocks * NumThreads;
-    Packet->grid_size_y = 1;
-    Packet->grid_size_z = 1;
+    Packet->grid_size_x = NumBlocks[0] * NumThreads[0];
+    Packet->grid_size_y = NumBlocks[1] * NumThreads[1];
+    Packet->grid_size_z = NumBlocks[2] * NumThreads[2];
     Packet->private_segment_size =
         Kernel.usesDynamicStack() ? StackSize : Kernel.getPrivateSize();
     Packet->group_segment_size = GroupSize;
@@ -1240,7 +1244,7 @@ struct AMDGPUStreamTy {
   /// the kernel finalizes. Once the kernel is finished, the stream will release
   /// the kernel args buffer to the specified memory manager.
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
-                         uint32_t NumThreads, uint64_t NumBlocks,
+                         uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
                          AMDGPUMemoryManagerTy &MemoryManager) {
     if (Queue == nullptr)
@@ -1988,12 +1992,10 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     // Detect if XNACK is enabled
-    auto TargeTripleAndFeaturesOrError =
-        hsa_utils::getTargetTripleAndFeatures(Agent);
-    if (!TargeTripleAndFeaturesOrError)
-      return TargeTripleAndFeaturesOrError.takeError();
-    if (static_cast<StringRef>(*TargeTripleAndFeaturesOrError)
-            .contains("xnack+"))
+    SmallVector<SmallString<32>> Targets;
+    if (auto Err = hsa_utils::getTargetTripleAndFeatures(Agent, Targets))
+      return Err;
+    if (!Targets.empty() && Targets[0].str().contains("xnack+"))
       IsXnackEnabled = true;
 
     // detect if device is an APU.
@@ -2148,9 +2150,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// We want to set up the RPC server for host services to the GPU if it is
   /// availible.
-  bool shouldSetupRPCServer() const override {
-    return libomptargetSupportsRPC();
-  }
+  bool shouldSetupRPCServer() const override { return true; }
 
   /// The RPC interface should have enough space for all availible parallelism.
   uint64_t requestedRPCPortCount() const override {
@@ -2829,10 +2829,10 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AsyncInfoWrapperTy AsyncInfoWrapper(*this, nullptr);
 
     KernelArgsTy KernelArgs = {};
-    if (auto Err =
-            AMDGPUKernel.launchImpl(*this, /*NumThread=*/1u,
-                                    /*NumBlocks=*/1ul, KernelArgs,
-                                    KernelLaunchParamsTy{}, AsyncInfoWrapper))
+    uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
+    if (auto Err = AMDGPUKernel.launchImpl(
+            *this, NumBlocksAndThreads, NumBlocksAndThreads, KernelArgs,
+            KernelLaunchParamsTy{}, AsyncInfoWrapper))
       return Err;
 
     Error Err = Plugin::success();
@@ -3209,13 +3209,16 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     if (!Processor)
       return false;
 
-    auto TargeTripleAndFeaturesOrError =
-        hsa_utils::getTargetTripleAndFeatures(getKernelAgent(DeviceId));
-    if (!TargeTripleAndFeaturesOrError)
-      return TargeTripleAndFeaturesOrError.takeError();
-    return offloading::amdgpu::isImageCompatibleWithEnv(
-        Processor ? *Processor : "", ElfOrErr->getPlatformFlags(),
-        *TargeTripleAndFeaturesOrError);
+    SmallVector<SmallString<32>> Targets;
+    if (auto Err = hsa_utils::getTargetTripleAndFeatures(
+            getKernelAgent(DeviceId), Targets))
+      return Err;
+    for (auto &Target : Targets)
+      if (offloading::amdgpu::isImageCompatibleWithEnv(
+              Processor ? *Processor : "", ElfOrErr->getPlatformFlags(),
+              Target.str()))
+        return true;
+    return false;
   }
 
   bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override {
@@ -3330,7 +3333,7 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 };
 
 Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
-                                 uint32_t NumThreads, uint64_t NumBlocks,
+                                 uint32_t NumThreads[3], uint32_t NumBlocks[3],
                                  KernelArgsTy &KernelArgs,
                                  KernelLaunchParamsTy LaunchParams,
                                  AsyncInfoWrapperTy &AsyncInfoWrapper) const {
@@ -3387,13 +3390,15 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // Only COV5 implicitargs needs to be set. COV4 implicitargs are not used.
   if (ImplArgs &&
       getImplicitArgsSize() == sizeof(hsa_utils::AMDGPUImplicitArgsTy)) {
-    ImplArgs->BlockCountX = NumBlocks;
-    ImplArgs->BlockCountY = 1;
-    ImplArgs->BlockCountZ = 1;
-    ImplArgs->GroupSizeX = NumThreads;
-    ImplArgs->GroupSizeY = 1;
-    ImplArgs->GroupSizeZ = 1;
-    ImplArgs->GridDims = 1;
+    ImplArgs->BlockCountX = NumBlocks[0];
+    ImplArgs->BlockCountY = NumBlocks[1];
+    ImplArgs->BlockCountZ = NumBlocks[2];
+    ImplArgs->GroupSizeX = NumThreads[0];
+    ImplArgs->GroupSizeY = NumThreads[1];
+    ImplArgs->GroupSizeZ = NumThreads[2];
+    ImplArgs->GridDims = NumBlocks[2] * NumThreads[2] > 1
+                             ? 3
+                             : 1 + (NumBlocks[1] * NumThreads[1] != 1);
     ImplArgs->DynamicLdsSize = KernelArgs.DynCGroupMem;
   }
 
@@ -3404,8 +3409,8 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
 
 Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
                                              KernelArgsTy &KernelArgs,
-                                             uint32_t NumThreads,
-                                             uint64_t NumBlocks) const {
+                                             uint32_t NumThreads[3],
+                                             uint32_t NumBlocks[3]) const {
   // Only do all this when the output is requested
   if (!(getInfoLevel() & OMP_INFOTYPE_PLUGIN_KERNEL))
     return Plugin::success();
@@ -3442,12 +3447,13 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
   // S/VGPR Spill Count: how many S/VGPRs are spilled by the kernel
   // Tripcount: loop tripcount for the kernel
   INFO(OMP_INFOTYPE_PLUGIN_KERNEL, GenericDevice.getDeviceId(),
-       "#Args: %d Teams x Thrds: %4lux%4u (MaxFlatWorkGroupSize: %u) LDS "
+       "#Args: %d Teams x Thrds: %4ux%4u (MaxFlatWorkGroupSize: %u) LDS "
        "Usage: %uB #SGPRs/VGPRs: %u/%u #SGPR/VGPR Spills: %u/%u Tripcount: "
        "%lu\n",
-       ArgNum, NumGroups, ThreadsPerGroup, MaxFlatWorkgroupSize,
-       GroupSegmentSize, SGPRCount, VGPRCount, SGPRSpillCount, VGPRSpillCount,
-       LoopTripCount);
+       ArgNum, NumGroups[0] * NumGroups[1] * NumGroups[2],
+       ThreadsPerGroup[0] * ThreadsPerGroup[1] * ThreadsPerGroup[2],
+       MaxFlatWorkgroupSize, GroupSegmentSize, SGPRCount, VGPRCount,
+       SGPRSpillCount, VGPRSpillCount, LoopTripCount);
 
   return Plugin::success();
 }
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 3ed5c02ed4a3b..14c48f6ace971 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -7,7 +7,7 @@ add_library(PluginCommon OBJECT
   src/RPC.cpp
   src/Utils/ELF.cpp
 )
-add_dependencies(PluginCommon intrinsics_gen LLVMProfileData)
+add_dependencies(PluginCommon intrinsics_gen)
 
 # Only enable JIT for those targets that LLVM can support.
 set(supported_jit_targets AMDGPU NVPTX)
@@ -22,6 +22,8 @@ if (NOT LLVM_LINK_LLVM_DYLIB)
 endif()
 
 # Include the RPC server from the `libc` project if availible.
+include(FindLibcCommonUtils)
+target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT})
 	target_link_libraries(PluginCommon PRIVATE llvmlibc_rpc_server)
 	target_compile_definitions(PluginCommon PRIVATE LIBOMPTARGET_RPC_SUPPORT)
@@ -29,13 +31,8 @@ elseif(${LIBOMPTARGET_GPU_LIBC_SUPPORT})
   find_library(llvmlibc_rpc_server NAMES llvmlibc_rpc_server
                PATHS ${LIBOMPTARGET_LLVM_LIBRARY_DIR} NO_DEFAULT_PATH)
   if(llvmlibc_rpc_server)
-		target_link_libraries(PluginCommon PRIVATE ${llvmlibc_rpc_server})
+    target_link_libraries(PluginCommon PRIVATE ${llvmlibc_rpc_server})
 		target_compile_definitions(PluginCommon PRIVATE LIBOMPTARGET_RPC_SUPPORT)
-    # We may need to get the headers directly from the 'libc' source directory.
-    target_include_directories(PluginCommon PRIVATE
-                               ${CMAKE_SOURCE_DIR}/../libc/utils/gpu/server
-                               ${CMAKE_SOURCE_DIR}/../libc/
-                               ${CMAKE_SOURCE_DIR}/../libc/include)
   endif()
 endif()
 
@@ -47,7 +44,6 @@ target_compile_definitions(PluginCommon PRIVATE
 
 target_compile_options(PluginCommon PUBLIC ${offload_compile_flags})
 target_link_options(PluginCommon PUBLIC ${offload_link_flags})
-target_link_libraries(PluginCommon PRIVATE LLVMProfileData)
 
 target_include_directories(PluginCommon PUBLIC
   ${CMAKE_CURRENT_SOURCE_DIR}/include
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 41cc0f286a581..eb266e8d4d451 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -124,6 +124,7 @@ enum InfoLevelKind { InfoLevel1 = 1, InfoLevel2, InfoLevel3 };
 /// we use the level to determine the indentation of the key-value property at
 /// printing time. See the enum InfoLevelKind for the list of accepted levels.
 class InfoQueueTy {
+public:
   struct InfoQueueEntryTy {
     std::string Key;
     std::string Value;
@@ -131,6 +132,7 @@ class InfoQueueTy {
     uint64_t Level;
   };
 
+private:
   std::deque<InfoQueueEntryTy> Queue;
 
 public:
@@ -153,6 +155,8 @@ class InfoQueueTy {
       Queue.push_back({Key, Value, Units, L});
   }
 
+  const std::deque<InfoQueueEntryTy> &getQueue() const { return Queue; }
+
   /// Print all info entries added to the queue.
   void print() const {
     // We print four spances for each level.
@@ -265,8 +269,9 @@ struct GenericKernelTy {
   Error launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
                ptrdiff_t *ArgOffsets, KernelArgsTy &KernelArgs,
                AsyncInfoWrapperTy &AsyncInfoWrapper) const;
-  virtual Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads,
-                           uint64_t NumBlocks, KernelArgsTy &KernelArgs,
+  virtual Error launchImpl(GenericDeviceTy &GenericDevice,
+                           uint32_t NumThreads[3], uint32_t NumBlocks[3],
+                           KernelArgsTy &KernelArgs,
                            KernelLaunchParamsTy LaunchParams,
                            AsyncInfoWrapperTy &AsyncInfoWrapper) const = 0;
 
@@ -316,15 +321,15 @@ struct GenericKernelTy {
 
   /// Prints generic kernel launch information.
   Error printLaunchInfo(GenericDeviceTy &GenericDevice,
-                        KernelArgsTy &KernelArgs, uint32_t NumThreads,
-                        uint64_t NumBlocks) const;
+                        KernelArgsTy &KernelArgs, uint32_t NumThreads[3],
+                        uint32_t NumBlocks[3]) const;
 
   /// Prints plugin-specific kernel launch information after generic kernel
   /// launch information
   virtual Error printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
                                        KernelArgsTy &KernelArgs,
-                                       uint32_t NumThreads,
-                                       uint64_t NumBlocks) const;
+                                       uint32_t NumThreads[3],
+                                       uint32_t NumBlocks[3]) const;
 
 private:
   /// Prepare the arguments before launching the kernel.
@@ -343,7 +348,7 @@ struct GenericKernelTy {
   /// The number of threads \p NumThreads can be adjusted by this method.
   /// \p IsNumThreadsFromUser is true is \p NumThreads is defined by user via
   /// thread_limit clause.
-  uint64_t getNumBlocks(GenericDeviceTy &GenericDevice,
+  uint32_t getNumBlocks(GenericDeviceTy &GenericDevice,
                         uint32_t BlockLimitClause[3], uint64_t LoopTripCount,
                         uint32_t &NumThreads, bool IsNumThreadsFromUser) const;
 
@@ -1580,9 +1585,6 @@ template <typename ResourceRef> class GenericDeviceResourceManagerTy {
   std::deque<ResourceRef> ResourcePool;
 };
 
-/// A static check on whether or not we support RPC in libomptarget.
-bool libomptargetSupportsRPC();
-
 } // namespace plugin
 } // namespace target
 } // namespace omp
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 25b815b7f9669..bd58d1d6e0d96 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -526,20 +526,21 @@ GenericKernelTy::getKernelLaunchEnvironment(
 
 Error GenericKernelTy::printLaunchInfo(GenericDeviceTy &GenericDevice,
                                        KernelArgsTy &KernelArgs,
-                                       uint32_t NumThreads,
-                                       uint64_t NumBlocks) const {
+                                       uint32_t NumThreads[3],
+                                       uint32_t NumBlocks[3]) const {
   INFO(OMP_INFOTYPE_PLUGIN_KERNEL, GenericDevice.getDeviceId(),
-       "Launching kernel %s with %" PRIu64
-       " blocks and %d threads in %s mode\n",
-       getName(), NumBlocks, NumThreads, getExecutionModeName());
+       "Launching kernel %s with [%u,%u,%u] blocks and [%u,%u,%u] threads in "
+       "%s mode\n",
+       getName(), NumBlocks[0], NumBlocks[1], NumBlocks[2], NumThreads[0],
+       NumThreads[1], NumThreads[2], getExecutionModeName());
   return printLaunchInfoDetails(GenericDevice, KernelArgs, NumThreads,
                                 NumBlocks);
 }
 
 Error GenericKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
                                               KernelArgsTy &KernelArgs,
-                                              uint32_t NumThreads,
-                                              uint64_t NumBlocks) const {
+                                              uint32_t NumThreads[3],
+                                              uint32_t NumBlocks[3]) const {
   return Plugin::success();
 }
 
@@ -566,10 +567,16 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
                     Args, Ptrs, *KernelLaunchEnvOrErr);
   }
 
-  uint32_t NumThreads = getNumThreads(GenericDevice, KernelArgs.ThreadLimit);
-  uint64_t NumBlocks =
-      getNumBlocks(GenericDevice, KernelArgs.NumTeams, KernelArgs.Tripcount,
-                   NumThreads, KernelArgs.ThreadLimit[0] > 0);
+  uint32_t NumThreads[3] = {KernelArgs.ThreadLimit[0],
+                            KernelArgs.ThreadLimit[1],
+                            KernelArgs.ThreadLimit[2]};
+  uint32_t NumBlocks[3] = {KernelArgs.NumTeams[0], KernelArgs.NumTeams[1],
+                           KernelArgs.NumTeams[2]};
+  if (!IsBareKernel) {
+    NumThreads[0] = getNumThreads(GenericDevice, NumThreads);
+    NumBlocks[0] = getNumBlocks(GenericDevice, NumBlocks, KernelArgs.Tripcount,
+                                NumThreads[0], KernelArgs.ThreadLimit[0] > 0);
+  }
 
   // Record the kernel description after we modified the argument count and num
   // blocks/threads.
@@ -578,7 +585,8 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
     RecordReplay.saveImage(getName(), getImage());
     RecordReplay.saveKernelInput(getName(), getImage());
     RecordReplay.saveKernelDescr(getName(), LaunchParams, KernelArgs.NumArgs,
-                                 NumBlocks, NumThreads, KernelArgs.Tripcount);
+                                 NumBlocks[0], NumThreads[0],
+                                 KernelArgs.Tripcount);
   }
 
   if (auto Err =
@@ -618,11 +626,10 @@ KernelLaunchParamsTy GenericKernelTy::prepareArgs(
 
 uint32_t GenericKernelTy::getNumThreads(GenericDeviceTy &GenericDevice,
                                         uint32_t ThreadLimitClause[3]) const {
-  assert(ThreadLimitClause[1] == 0 && ThreadLimitClause[2] == 0 &&
-         "Multi dimensional launch not supported yet.");
+  assert(!IsBareKernel && "bare kernel should not call this function");
 
-  if (IsBareKernel && ThreadLimitClause[0] > 0)
-    return ThreadLimitClause[0];
+  assert(ThreadLimitClause[1] == 1 && ThreadLimitClause[2] == 1 &&
+         "Multi dimensional launch not supported yet.");
 
   if (ThreadLimitClause[0] > 0 && isGenericMode())
     ThreadLimitClause[0] += GenericDevice.getWarpSize();
@@ -632,16 +639,15 @@ uint32_t GenericKernelTy::getNumThreads(GenericDeviceTy &GenericDevice,
                                      : PreferredNumThreads);
 }
 
-uint64_t GenericKernelTy::getNumBlocks(GenericDeviceTy &GenericDevice,
+uint32_t GenericKernelTy::getNumBlocks(GenericDeviceTy &GenericDevice,
                                        uint32_t NumTeamsClause[3],
                                        uint64_t LoopTripCount,
                                        uint32_t &NumThreads,
                                        bool IsNumThreadsFromUser) const {
-  assert(NumTeamsClause[1] == 0 && NumTeamsClause[2] == 0 &&
-         "Multi dimensional launch not supported yet.");
+  assert(!IsBareKernel && "bare kernel should not call this function");
 
-  if (IsBareKernel && NumTeamsClause[0] > 0)
-    return NumTeamsClause[0];
+  assert(NumTeamsClause[1] == 1 && NumTeamsClause[2] == 1 &&
+         "Multi dimensional launch not supported yet.");
 
   if (NumTeamsClause[0] > 0) {
     // TODO: We need to honor any value and consequently allow more than the
@@ -2179,11 +2185,3 @@ int32_t GenericPluginTy::get_function(__tgt_device_binary Binary,
   *KernelPtr = &Kernel;
   return OFFLOAD_SUCCESS;
 }
-
-bool llvm::omp::target::plugin::libomptargetSupportsRPC() {
-#ifdef LIBOMPTARGET_RPC_SUPPORT
-  return true;
-#else
-  return false;
-#endif
-}
diff --git a/offload/plugins-nextgen/common/src/RPC.cpp b/offload/plugins-nextgen/common/src/RPC.cpp
index be41928111da4..66f98e68dc442 100644
--- a/offload/plugins-nextgen/common/src/RPC.cpp
+++ b/offload/plugins-nextgen/common/src/RPC.cpp
@@ -9,20 +9,72 @@
 #include "RPC.h"
 
 #include "Shared/Debug.h"
+#include "Shared/RPCOpcodes.h"
 
 #include "PluginInterface.h"
 
-// TODO: This should be included unconditionally and cleaned up.
-#if defined(LIBOMPTARGET_RPC_SUPPORT)
-#include "llvmlibc_rpc_server.h"
 #include "shared/rpc.h"
 #include "shared/rpc_opcodes.h"
-#endif
 
 using namespace llvm;
 using namespace omp;
 using namespace target;
 
+template <uint32_t NumLanes>
+rpc::Status handle_offload_opcodes(plugin::GenericDeviceTy &Device,
+                                   rpc::Server::Port &Port) {
+
+  switch (Port.get_opcode()) {
+  case LIBC_MALLOC: {
+    Port.recv_and_send([&](rpc::Buffer *Buffer, uint32_t) {
+      Buffer->data[0] = reinterpret_cast<uintptr_t>(Device.allocate(
+          Buffer->data[0], nullptr, TARGET_ALLOC_DEVICE_NON_BLOCKING));
+    });
+    break;
+  }
+  case LIBC_FREE: {
+    Port.recv([&](rpc::Buffer *Buffer, uint32_t) {
+      Device.free(reinterpret_cast<void *>(Buffer->data[0]),
+                  TARGET_ALLOC_DEVICE_NON_BLOCKING);
+    });
+    break;
+  }
+  case OFFLOAD_HOST_CALL: {
+    uint64_t Sizes[NumLanes] = {0};
+    unsigned long long Results[NumLanes] = {0};
+    void *Args[NumLanes] = {nullptr};
+    Port.recv_n(Args, Sizes, [&](uint64_t Size) { return new char[Size]; });
+    Port.recv([&](rpc::Buffer *buffer, uint32_t ID) {
+      using FuncPtrTy = unsigned long long (*)(void *);
+      auto Func = reinterpret_cast<FuncPtrTy>(buffer->data[0]);
+      Results[ID] = Func(Args[ID]);
+    });
+    Port.send([&](rpc::Buffer *Buffer, uint32_t ID) {
+      Buffer->data[0] = static_cast<uint64_t>(Results[ID]);
+      delete[] reinterpret_cast<char *>(Args[ID]);
+    });
+    break;
+  }
+  default:
+    return rpc::UNHANDLED_OPCODE;
+    break;
+  }
+  return rpc::SUCCESS;
+}
+
+static rpc::Status handle_offload_opcodes(plugin::GenericDeviceTy &Device,
+                                          rpc::Server::Port &Port,
+                                          uint32_t NumLanes) {
+  if (NumLanes == 1)
+    return handle_offload_opcodes<1>(Device, Port);
+  else if (NumLanes == 32)
+    return handle_offload_opcodes<32>(Device, Port);
+  else if (NumLanes == 64)
+    return handle_offload_opcodes<64>(Device, Port);
+  else
+    return rpc::ERROR;
+}
+
 RPCServerTy::RPCServerTy(plugin::GenericPluginTy &Plugin)
     : Buffers(Plugin.getNumDevices()) {}
 
@@ -30,17 +82,12 @@ llvm::Expected<bool>
 RPCServerTy::isDeviceUsingRPC(plugin::GenericDeviceTy &Device,
                               plugin::GenericGlobalHandlerTy &Handler,
                               plugin::DeviceImageTy &Image) {
-#ifdef LIBOMPTARGET_RPC_SUPPORT
-  return Handler.isSymbolInImage(Device, Image, "__llvm_libc_rpc_client");
-#else
-  return false;
-#endif
+  return Handler.isSymbolInImage(Device, Image, "__llvm_rpc_client");
 }
 
 Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device,
                               plugin::GenericGlobalHandlerTy &Handler,
                               plugin::DeviceImageTy &Image) {
-#ifdef LIBOMPTARGET_RPC_SUPPORT
   uint64_t NumPorts =
       std::min(Device.requestedRPCPortCount(), rpc::MAX_PORT_COUNT);
   void *RPCBuffer = Device.allocate(
@@ -51,73 +98,45 @@ Error RPCServerTy::initDevice(plugin::GenericDeviceTy &Device,
         "Failed to initialize RPC server for device %d", Device.getDeviceId());
 
   // Get the address of the RPC client from the device.
-  void *ClientPtr;
-  plugin::GlobalTy ClientGlobal("__llvm_libc_rpc_client", sizeof(void *));
+  plugin::GlobalTy ClientGlobal("__llvm_rpc_client", sizeof(rpc::Client));
   if (auto Err =
           Handler.getGlobalMetadataFromDevice(Device, Image, ClientGlobal))
     return Err;
 
-  if (auto Err = Device.dataRetrieve(&ClientPtr, ClientGlobal.getPtr(),
-                                     sizeof(void *), nullptr))
-    return Err;
-
   rpc::Client client(NumPorts, RPCBuffer);
-  if (auto Err =
-          Device.dataSubmit(ClientPtr, &client, sizeof(rpc::Client), nullptr))
+  if (auto Err = Device.dataSubmit(ClientGlobal.getPtr(), &client,
+                                   sizeof(rpc::Client), nullptr))
     return Err;
   Buffers[Device.getDeviceId()] = RPCBuffer;
 
   return Error::success();
-
-#endif
-  return Error::success();
 }
 
 Error RPCServerTy::runServer(plugin::GenericDeviceTy &Device) {
-#ifdef LIBOMPTARGET_RPC_SUPPORT
   uint64_t NumPorts =
       std::min(Device.requestedRPCPortCount(), rpc::MAX_PORT_COUNT);
   rpc::Server Server(NumPorts, Buffers[Device.getDeviceId()]);
 
-  auto port = Server.try_open(Device.getWarpSize());
-  if (!port)
+  auto Port = Server.try_open(Device.getWarpSize());
+  if (!Port)
     return Error::success();
 
-  int Status = rpc::SUCCESS;
-  switch (port->get_opcode()) {
-  case RPC_MALLOC: {
-    port->recv_and_send([&](rpc::Buffer *Buffer, uint32_t) {
-      Buffer->data[0] = reinterpret_cast<uintptr_t>(Device.allocate(
-          Buffer->data[0], nullptr, TARGET_ALLOC_DEVICE_NON_BLOCKING));
-    });
-    break;
-  }
-  case RPC_FREE: {
-    port->recv([&](rpc::Buffer *Buffer, uint32_t) {
-      Device.free(reinterpret_cast<void *>(Buffer->data[0]),
-                  TARGET_ALLOC_DEVICE_NON_BLOCKING);
-    });
-    break;
-  }
-  default:
-    // Let the `libc` library handle any other unhandled opcodes.
-    Status = libc_handle_rpc_port(&*port, Device.getWarpSize());
-    break;
-  }
-  port->close();
+  int Status = handle_offload_opcodes(Device, *Port, Device.getWarpSize());
 
+  // Let the `libc` library handle any other unhandled opcodes.
+#ifdef LIBOMPTARGET_RPC_SUPPORT
+  if (Status == rpc::UNHANDLED_OPCODE)
+    Status = handle_libc_opcodes(*Port, Device.getWarpSize());
+#endif
+
+  Port->close();
   if (Status != rpc::SUCCESS)
     return createStringError("RPC server given invalid opcode!");
 
-  return Error::success();
-#endif
   return Error::success();
 }
 
 Error RPCServerTy::deinitDevice(plugin::GenericDeviceTy &Device) {
-#ifdef LIBOMPTARGET_RPC_SUPPORT
   Device.free(Buffers[Device.getDeviceId()], TARGET_ALLOC_HOST);
   return Error::success();
-#endif
-  return Error::success();
 }
diff --git a/offload/plugins-nextgen/common/src/Utils/ELF.cpp b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
index 88642fd5b5640..10b32440dc877 100644
--- a/offload/plugins-nextgen/common/src/Utils/ELF.cpp
+++ b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
@@ -64,10 +64,12 @@ checkMachineImpl(const object::ELFObjectFile<ELFT> &ELFObj, uint16_t EMachine) {
     if (Header.e_ident[EI_OSABI] != ELFOSABI_AMDGPU_HSA)
       return createError("Invalid AMD OS/ABI, must be AMDGPU_HSA");
     if (Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V4 &&
-        Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V5)
-      return createError("Invalid AMD ABI version, must be version 4 or 5");
+        Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V5 &&
+        Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V6)
+      return createError("Invalid AMD ABI version, must be version 4 or above");
     if ((Header.e_flags & EF_AMDGPU_MACH) < EF_AMDGPU_MACH_AMDGCN_GFX700 ||
-        (Header.e_flags & EF_AMDGPU_MACH) > EF_AMDGPU_MACH_AMDGCN_GFX1201)
+        (Header.e_flags & EF_AMDGPU_MACH) >
+            EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC)
       return createError("Unsupported AMDGPU architecture");
   } else if (Header.e_machine == EM_CUDA) {
     if (~Header.e_flags & EF_CUDA_64BIT_ADDRESS)
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 015c7775ba351..894d1c2214b97 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -149,8 +149,8 @@ struct CUDAKernelTy : public GenericKernelTy {
   }
 
   /// Launch the CUDA kernel function.
-  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads,
-                   uint64_t NumBlocks, KernelArgsTy &KernelArgs,
+  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
+                   uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
 
@@ -496,9 +496,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
   /// We want to set up the RPC server for host services to the GPU if it is
   /// availible.
-  bool shouldSetupRPCServer() const override {
-    return libomptargetSupportsRPC();
-  }
+  bool shouldSetupRPCServer() const override { return true; }
 
   /// The RPC interface should have enough space for all availible parallelism.
   uint64_t requestedRPCPortCount() const override {
@@ -1230,10 +1228,10 @@ struct CUDADeviceTy : public GenericDeviceTy {
     AsyncInfoWrapperTy AsyncInfoWrapper(*this, nullptr);
 
     KernelArgsTy KernelArgs = {};
-    if (auto Err =
-            CUDAKernel.launchImpl(*this, /*NumThread=*/1u,
-                                  /*NumBlocks=*/1ul, KernelArgs,
-                                  KernelLaunchParamsTy{}, AsyncInfoWrapper))
+    uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
+    if (auto Err = CUDAKernel.launchImpl(
+            *this, NumBlocksAndThreads, NumBlocksAndThreads, KernelArgs,
+            KernelLaunchParamsTy{}, AsyncInfoWrapper))
       return Err;
 
     Error Err = Plugin::success();
@@ -1276,7 +1274,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
 };
 
 Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
-                               uint32_t NumThreads, uint64_t NumBlocks,
+                               uint32_t NumThreads[3], uint32_t NumBlocks[3],
                                KernelArgsTy &KernelArgs,
                                KernelLaunchParamsTy LaunchParams,
                                AsyncInfoWrapperTy &AsyncInfoWrapper) const {
@@ -1294,9 +1292,8 @@ Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
                     reinterpret_cast<void *>(&LaunchParams.Size),
                     CU_LAUNCH_PARAM_END};
 
-  CUresult Res = cuLaunchKernel(Func, NumBlocks, /*gridDimY=*/1,
-                                /*gridDimZ=*/1, NumThreads,
-                                /*blockDimY=*/1, /*blockDimZ=*/1,
+  CUresult Res = cuLaunchKernel(Func, NumBlocks[0], NumBlocks[1], NumBlocks[2],
+                                NumThreads[0], NumThreads[1], NumThreads[2],
                                 MaxDynCGroupMem, Stream, nullptr, Config);
   return Plugin::check(Res, "Error in cuLaunchKernel for '%s': %s", getName());
 }
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 6f2e3d8604ec8..915c41e88c582 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -89,8 +89,8 @@ struct GenELF64KernelTy : public GenericKernelTy {
   }
 
   /// Launch the kernel using the libffi.
-  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads,
-                   uint64_t NumBlocks, KernelArgsTy &KernelArgs,
+  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
+                   uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override {
     // Create a vector of ffi_types, one per argument.
diff --git a/offload/src/interface.cpp b/offload/src/interface.cpp
index 21f9114ac2b08..ad84a43cef8af 100644
--- a/offload/src/interface.cpp
+++ b/offload/src/interface.cpp
@@ -284,14 +284,25 @@ static KernelArgsTy *upgradeKernelArgs(KernelArgsTy *KernelArgs,
     LocalKernelArgs.Flags = KernelArgs->Flags;
     LocalKernelArgs.DynCGroupMem = 0;
     LocalKernelArgs.NumTeams[0] = NumTeams;
-    LocalKernelArgs.NumTeams[1] = 0;
-    LocalKernelArgs.NumTeams[2] = 0;
+    LocalKernelArgs.NumTeams[1] = 1;
+    LocalKernelArgs.NumTeams[2] = 1;
     LocalKernelArgs.ThreadLimit[0] = ThreadLimit;
-    LocalKernelArgs.ThreadLimit[1] = 0;
-    LocalKernelArgs.ThreadLimit[2] = 0;
+    LocalKernelArgs.ThreadLimit[1] = 1;
+    LocalKernelArgs.ThreadLimit[2] = 1;
     return &LocalKernelArgs;
   }
 
+  // FIXME: This is a WA to "calibrate" the bad work done in the front end.
+  // Delete this ugly code after the front end emits proper values.
+  auto CorrectMultiDim = [](uint32_t(&Val)[3]) {
+    if (Val[1] == 0)
+      Val[1] = 1;
+    if (Val[2] == 0)
+      Val[2] = 1;
+  };
+  CorrectMultiDim(KernelArgs->ThreadLimit);
+  CorrectMultiDim(KernelArgs->NumTeams);
+
   return KernelArgs;
 }
 
@@ -320,12 +331,6 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
   KernelArgs =
       upgradeKernelArgs(KernelArgs, LocalKernelArgs, NumTeams, ThreadLimit);
 
-  assert(KernelArgs->NumTeams[0] == static_cast<uint32_t>(NumTeams) &&
-         !KernelArgs->NumTeams[1] && !KernelArgs->NumTeams[2] &&
-         "OpenMP interface should not use multiple dimensions");
-  assert(KernelArgs->ThreadLimit[0] == static_cast<uint32_t>(ThreadLimit) &&
-         !KernelArgs->ThreadLimit[1] && !KernelArgs->ThreadLimit[2] &&
-         "OpenMP interface should not use multiple dimensions");
   TIMESCOPE_WITH_DETAILS_AND_IDENT(
       "Runtime: target exe",
       "NumTeams=" + std::to_string(NumTeams) +
diff --git a/offload/src/omptarget.cpp b/offload/src/omptarget.cpp
index 66137b53b0cb4..1a7af5649b9e2 100644
--- a/offload/src/omptarget.cpp
+++ b/offload/src/omptarget.cpp
@@ -1451,8 +1451,6 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
         Loc);
 
 #ifdef OMPT_SUPPORT
-    assert(KernelArgs.NumTeams[1] == 0 && KernelArgs.NumTeams[2] == 0 &&
-           "Multi dimensional launch not supported yet.");
     /// RAII to establish tool anchors before and after kernel launch
     int32_t NumTeams = KernelArgs.NumTeams[0];
     // No need to guard this with OMPT_IF_BUILT
diff --git a/offload/test/CMakeLists.txt b/offload/test/CMakeLists.txt
index 5a6f637b57fa7..8a827e0a625ef 100644
--- a/offload/test/CMakeLists.txt
+++ b/offload/test/CMakeLists.txt
@@ -28,6 +28,11 @@ if(CUDAToolkit_FOUND)
   get_filename_component(CUDA_LIBDIR "${CUDA_cudart_static_LIBRARY}" DIRECTORY)
 endif()
 
+set(OMP_DEPEND)
+if(TARGET omp)
+  set(OMP_DEPEND omp)
+endif()
+
 string(REGEX MATCHALL "([^\ ]+\ |[^\ ]+$)" SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}")
 foreach(CURRENT_TARGET IN LISTS SYSTEM_TARGETS)
   string(STRIP "${CURRENT_TARGET}" CURRENT_TARGET)
@@ -35,7 +40,7 @@ foreach(CURRENT_TARGET IN LISTS SYSTEM_TARGETS)
   add_offload_testsuite(check-libomptarget-${CURRENT_TARGET}
     "Running libomptarget tests"
     ${CMAKE_CURRENT_BINARY_DIR}/${CURRENT_TARGET}
-    DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS}
+    DEPENDS omptarget ${OMP_DEPEND} ${LIBOMPTARGET_TESTED_PLUGINS}
     ARGS ${LIBOMPTARGET_LIT_ARG_LIST})
   list(APPEND LIBOMPTARGET_LIT_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CURRENT_TARGET})
 
@@ -49,12 +54,12 @@ add_offload_testsuite(check-libomptarget
   "Running libomptarget tests"
   ${LIBOMPTARGET_LIT_TESTSUITES}
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS}
+  DEPENDS llvm-offload-device-info omptarget ${OMP_DEPEND} ${LIBOMPTARGET_TESTED_PLUGINS}
   ARGS ${LIBOMPTARGET_LIT_ARG_LIST})
 
 add_offload_testsuite(check-offload
   "Running libomptarget tests"
   ${LIBOMPTARGET_LIT_TESTSUITES}
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS}
+  DEPENDS llvm-offload-device-info omptarget ${OMP_DEPEND} ${LIBOMPTARGET_TESTED_PLUGINS}
   ARGS ${LIBOMPTARGET_LIT_ARG_LIST})
diff --git a/offload/test/api/omp_env_vars.c b/offload/test/api/omp_env_vars.c
index 2e78bb115beed..a8a574be8c849 100644
--- a/offload/test/api/omp_env_vars.c
+++ b/offload/test/api/omp_env_vars.c
@@ -1,11 +1,12 @@
 // RUN: %libomptarget-compile-generic
 // RUN: env OMP_NUM_TEAMS=1 OMP_TEAMS_THREAD_LIMIT=1 LIBOMPTARGET_INFO=16 \
 // RUN:   %libomptarget-run-generic 2>&1 | %fcheck-generic
+// REQUIRES: gpu
 
 #define N 256
 
 int main() {
-  // CHECK: Launching kernel [[KERNEL:.+_main_.+]] with 1 blocks and 1 threads
+  // CHECK: Launching kernel [[KERNEL:.+_main_.+]] with [1,1,1] blocks and [1,1,1] threads
 #pragma omp target teams
 #pragma omp parallel
   {}
diff --git a/offload/test/libc/host_call.c b/offload/test/api/omp_host_call.c
similarity index 67%
rename from offload/test/libc/host_call.c
rename to offload/test/api/omp_host_call.c
index 61c4e14d5b388..40d3dc7258291 100644
--- a/offload/test/libc/host_call.c
+++ b/offload/test/api/omp_host_call.c
@@ -1,20 +1,18 @@
 // RUN: %libomptarget-compile-run-and-check-generic
 
-// REQUIRES: libc
-
 #include <assert.h>
 #include <omp.h>
 #include <stdio.h>
 
 #pragma omp begin declare variant match(device = {kind(gpu)})
 // Extension provided by the 'libc' project.
-unsigned long long rpc_host_call(void *fn, void *args, size_t size);
-#pragma omp declare target to(rpc_host_call) device_type(nohost)
+unsigned long long __llvm_omp_host_call(void *fn, void *args, size_t size);
+#pragma omp declare target to(__llvm_omp_host_call) device_type(nohost)
 #pragma omp end declare variant
 
 #pragma omp begin declare variant match(device = {kind(cpu)})
 // Dummy host implementation to make this work for all targets.
-unsigned long long rpc_host_call(void *fn, void *args, size_t size) {
+unsigned long long __llvm_omp_host_call(void *fn, void *args, size_t size) {
   return ((unsigned long long (*)(void *))fn)(args);
 }
 #pragma omp end declare variant
@@ -25,22 +23,14 @@ typedef struct args_s {
 } args_t;
 
 // CHECK-DAG: Thread: 0, Block: 0
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 1, Block: 0
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 0, Block: 1
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 1, Block: 1
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 0, Block: 2
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 1, Block: 2
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 0, Block: 3
-// CHECK-DAG: Result: 42
 // CHECK-DAG: Thread: 1, Block: 3
-// CHECK-DAG: Result: 42
-long long foo(void *data) {
+unsigned long long foo(void *data) {
   assert(omp_is_initial_device() && "Not executing on host?");
   args_t *args = (args_t *)data;
   printf("Thread: %d, Block: %d\n", args->thread_id, args->block_id);
@@ -54,11 +44,19 @@ int main() {
   fn_ptr = (void *)&foo;
 #pragma omp target update to(fn_ptr)
 
-#pragma omp target teams num_teams(4)
+  int failed = 0;
+#pragma omp target teams num_teams(4) map(tofrom : failed)
 #pragma omp parallel num_threads(2)
   {
     args_t args = {omp_get_thread_num(), omp_get_team_num()};
-    unsigned long long res = rpc_host_call(fn_ptr, &args, sizeof(args_t));
-    printf("Result: %d\n", (int)res);
+    unsigned long long res =
+        __llvm_omp_host_call(fn_ptr, &args, sizeof(args_t));
+    if (res != 42)
+#pragma omp atomic write
+      failed = 1;
   }
+
+  // CHECK: PASS
+  if (!failed)
+    printf("PASS\n");
 }
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index c5910e9903bb9..7cacf30ec711b 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -66,7 +66,7 @@ def evaluate_bool_env(env):
 config.name = 'libomptarget :: ' + config.libomptarget_current_target
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.c', '.cpp', '.cc', '.f90', '.cu']
+config.suffixes = ['.c', '.cpp', '.cc', '.f90', '.cu', '.td']
 
 # excludes: A list of directories to exclude from the testuites.
 config.excludes = ['Inputs']
@@ -422,3 +422,4 @@ config.substitutions.append(("%flags", config.test_flags))
 config.substitutions.append(("%not", config.libomptarget_not))
 config.substitutions.append(("%offload-device-info",
                              config.offload_device_info))
+config.substitutions.append(("%offload-tblgen", config.offload_tblgen))
diff --git a/offload/test/lit.site.cfg.in b/offload/test/lit.site.cfg.in
index a1cb5acc38a40..ce3f6abf50a13 100644
--- a/offload/test/lit.site.cfg.in
+++ b/offload/test/lit.site.cfg.in
@@ -28,5 +28,6 @@ config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
 config.has_libomptarget_ompt = @LIBOMPTARGET_OMPT_SUPPORT@
 config.libomptarget_has_libc = @LIBOMPTARGET_GPU_LIBC_SUPPORT@
 config.libomptarget_test_pgo = @LIBOMPTARGET_TEST_GPU_PGO@
+config.offload_tblgen = "@OFFLOAD_TBLGEN_EXECUTABLE@"
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
diff --git a/offload/test/mapping/map_both_pointer_pointee.c b/offload/test/mapping/map_both_pointer_pointee.c
index c4894ed97af57..7be1ba465e7db 100644
--- a/offload/test/mapping/map_both_pointer_pointee.c
+++ b/offload/test/mapping/map_both_pointer_pointee.c
@@ -1,8 +1,8 @@
-// RUN: %libomptarget-compilexx-run-and-check-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
-// RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
-// RUN: %libomptarget-compilexx-run-and-check-x86_64-unknown-linux-gnu
-// RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
+// RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-x86_64-unknown-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 // REQUIRES: unified_shared_memory
 // UNSUPPORTED: amdgcn-amd-amdhsa
diff --git a/offload/test/mapping/power_of_two_alignment.c b/offload/test/mapping/power_of_two_alignment.cpp
similarity index 100%
rename from offload/test/mapping/power_of_two_alignment.c
rename to offload/test/mapping/power_of_two_alignment.cpp
diff --git a/offload/test/offloading/info.c b/offload/test/offloading/info.c
index da8e4c44c5acc..d86644b871e25 100644
--- a/offload/test/offloading/info.c
+++ b/offload/test/offloading/info.c
@@ -42,7 +42,7 @@ int main() {
 // INFO: info: {{.*}}             {{.*}}             256      1           0            A[0:64] at info.c:{{[0-9]+}}:{{[0-9]+}}
 // INFO: info: Entering OpenMP kernel at info.c:{{[0-9]+}}:{{[0-9]+}} with 1 arguments:
 // INFO: info: firstprivate(val)[4]
-// INFO: info: Launching kernel __omp_offloading_{{.*}}main{{.*}} with {{[0-9]+}} blocks and {{[0-9]+}} threads in Generic mode
+// INFO: info: Launching kernel __omp_offloading_{{.*}}main{{.*}} with [{{[0-9]+}},1,1] blocks and [{{[0-9]+}},1,1] threads in Generic mode
 // AMDGPU: AMDGPU device {{[0-9]}} info: #Args: {{[0-9]}} Teams x Thrds: {{[0-9]+}}x {{[0-9]+}} (MaxFlatWorkGroupSize: {{[0-9]+}}) LDS Usage: {{[0-9]+}}B #SGPRs/VGPRs: {{[0-9]+}}/{{[0-9]+}} #SGPR/VGPR Spills: {{[0-9]+}}/{{[0-9]+}} Tripcount: {{[0-9]+}}
 // INFO: info: OpenMP Host-Device pointer mappings after block at info.c:{{[0-9]+}}:{{[0-9]+}}:
 // INFO: info: Host Ptr           Target Ptr         Size (B) DynRefCount HoldRefCount Declaration
diff --git a/offload/test/offloading/ompx_bare.c b/offload/test/offloading/ompx_bare.c
index b9a8759db1de1..6a6ada9617cf5 100644
--- a/offload/test/offloading/ompx_bare.c
+++ b/offload/test/offloading/ompx_bare.c
@@ -15,7 +15,7 @@ int main(int argc, char *argv[]) {
   const int N = num_blocks * block_size;
   int *data = (int *)malloc(N * sizeof(int));
 
-  // CHECK: "PluginInterface" device 0 info: Launching kernel __omp_offloading_{{.*}} with 64 blocks and 64 threads in SPMD mode
+  // CHECK: "PluginInterface" device 0 info: Launching kernel __omp_offloading_{{.*}} with [64,1,1] blocks and [64,1,1] threads in SPMD mode
 
 #pragma omp target teams ompx_bare num_teams(num_blocks) thread_limit(block_size) map(from: data[0:N])
   {
diff --git a/offload/test/offloading/ompx_bare_multi_dim.cpp b/offload/test/offloading/ompx_bare_multi_dim.cpp
new file mode 100644
index 0000000000000..d37278525fdb0
--- /dev/null
+++ b/offload/test/offloading/ompx_bare_multi_dim.cpp
@@ -0,0 +1,56 @@
+// RUN: %libomptarget-compilexx-generic
+// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | %fcheck-generic
+// REQUIRES: gpu
+
+#include <ompx.h>
+
+#include <cassert>
+#include <vector>
+
+// CHECK: "PluginInterface" device 0 info: Launching kernel __omp_offloading_{{.*}} with [2,4,6] blocks and [32,4,2] threads in SPMD mode
+
+int main(int argc, char *argv[]) {
+  int bs[3] = {32u, 4u, 2u};
+  int gs[3] = {2u, 4u, 6u};
+  int n = bs[0] * bs[1] * bs[2] * gs[0] * gs[1] * gs[2];
+  std::vector<int> x_buf(n);
+  std::vector<int> y_buf(n);
+  std::vector<int> z_buf(n);
+
+  auto x = x_buf.data();
+  auto y = y_buf.data();
+  auto z = z_buf.data();
+  for (int i = 0; i < n; ++i) {
+    x[i] = i;
+    y[i] = i + 1;
+  }
+
+#pragma omp target teams ompx_bare num_teams(gs[0], gs[1], gs[2])              \
+    thread_limit(bs[0], bs[1], bs[2]) map(to : x[ : n], y[ : n])               \
+    map(from : z[ : n])
+  {
+    int tid_x = ompx_thread_id_x();
+    int tid_y = ompx_thread_id_y();
+    int tid_z = ompx_thread_id_z();
+    int gid_x = ompx_block_id_x();
+    int gid_y = ompx_block_id_y();
+    int gid_z = ompx_block_id_z();
+    int bs_x = ompx_block_dim_x();
+    int bs_y = ompx_block_dim_y();
+    int bs_z = ompx_block_dim_z();
+    int bs = bs_x * bs_y * bs_z;
+    int gs_x = ompx_grid_dim_x();
+    int gs_y = ompx_grid_dim_y();
+    int gid = gid_z * gs_y * gs_x + gid_y * gs_x + gid_x;
+    int tid = tid_z * bs_x * bs_y + tid_y * bs_x + tid_x;
+    int i = gid * bs + tid;
+    z[i] = x[i] + y[i];
+  }
+
+  for (int i = 0; i < n; ++i) {
+    if (z[i] != (2 * i + 1))
+      return 1;
+  }
+
+  return 0;
+}
diff --git a/offload/test/offloading/small_trip_count.c b/offload/test/offloading/small_trip_count.c
index 78750411ff8f4..e9ec8b7103d66 100644
--- a/offload/test/offloading/small_trip_count.c
+++ b/offload/test/offloading/small_trip_count.c
@@ -12,26 +12,26 @@
 __attribute__((optnone)) void optnone() {}
 
 int main() {
-  // DEFAULT: Launching kernel {{.+_main_.+}} with 4 blocks and 32 threads in SPMD mode
-  // EIGHT: Launching kernel {{.+_main_.+}} with 16 blocks and 8 threads in SPMD mode
+  // DEFAULT: Launching kernel {{.+_main_.+}} with [4,1,1] blocks and [32,1,1] threads in SPMD mode
+  // EIGHT: Launching kernel {{.+_main_.+}} with [16,1,1] blocks and [8,1,1] threads in SPMD mode
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < N; ++i) {
     optnone();
   }
-  // DEFAULT: Launching kernel {{.+_main_.+}} with 4 blocks and 32 threads in SPMD mode
-  // EIGHT: Launching kernel {{.+_main_.+}} with 16 blocks and 8 threads in SPMD mode
+  // DEFAULT: Launching kernel {{.+_main_.+}} with [4,1,1] blocks and [32,1,1] threads in SPMD mode
+  // EIGHT: Launching kernel {{.+_main_.+}} with [16,1,1] blocks and [8,1,1] threads in SPMD mode
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < N - 1; ++i) {
     optnone();
   }
-  // DEFAULT: Launching kernel {{.+_main_.+}} with 5 blocks and 32 threads in SPMD mode
-  // EIGHT: Launching kernel {{.+_main_.+}} with 17 blocks and 8 threads in SPMD mode
+  // DEFAULT: Launching kernel {{.+_main_.+}} with [5,1,1] blocks and [32,1,1] threads in SPMD mode
+  // EIGHT: Launching kernel {{.+_main_.+}} with [17,1,1] blocks and [8,1,1] threads in SPMD mode
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < N + 1; ++i) {
     optnone();
   }
-  // DEFAULT: Launching kernel {{.+_main_.+}} with 32 blocks and 4 threads in SPMD mode
-  // EIGHT: Launching kernel {{.+_main_.+}} with 32 blocks and 4 threads in SPMD mode
+  // DEFAULT: Launching kernel {{.+_main_.+}} with [32,1,1] blocks and [4,1,1] threads in SPMD mode
+  // EIGHT: Launching kernel {{.+_main_.+}} with [32,1,1] blocks and [4,1,1] threads in SPMD mode
 #pragma omp target teams distribute parallel for simd thread_limit(4)
   for (int i = 0; i < N; ++i) {
     optnone();
diff --git a/offload/test/offloading/small_trip_count_thread_limit.cpp b/offload/test/offloading/small_trip_count_thread_limit.cpp
index cfb9fe712d270..fbd7fe9175d70 100644
--- a/offload/test/offloading/small_trip_count_thread_limit.cpp
+++ b/offload/test/offloading/small_trip_count_thread_limit.cpp
@@ -25,4 +25,4 @@ int main(int argc, char *argv[]) {
   return 0;
 }
 
-// CHECK: Launching kernel {{.*}} with 4 blocks and 256 threads in SPMD mode
+// CHECK: Launching kernel {{.*}} with [4,1,1] blocks and [256,1,1] threads in SPMD mode
diff --git a/offload/test/tools/offload-tblgen/default_returns.td b/offload/test/tools/offload-tblgen/default_returns.td
new file mode 100644
index 0000000000000..995e24abf707d
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/default_returns.td
@@ -0,0 +1,40 @@
+// RUN: %offload-tblgen -gen-api -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-API
+// RUN: %offload-tblgen -gen-entry-points -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-VALIDATION
+
+// Check implicit returns are included in documentation and the validation
+// wrappers where applicable
+
+include "APIDefs.td"
+
+def : Handle {
+    let name = "ol_foo_handle_t";
+    let desc = "Example handle type";
+}
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+    let params = [
+        Param<"uint32_t", "ParamValue", "A plain value parameter">,
+        Param<"ol_foo_handle_t", "ParamHandle", "A handle parameter">,
+        Param<"uint32_t*", "ParamPointer", "A pointer parameter">,
+        Param<"uint32_t*", "ParamPointerOpt", "An optional pointer parameter", PARAM_OUT_OPTIONAL>
+    ];
+    let returns = [];
+}
+
+// CHECK-API: /// @returns
+// CHECK-API: OL_RESULT_SUCCESS
+// CHECK-API: OL_ERRC_INVALID_NULL_HANDLE
+// CHECK-API-NEXT: `NULL == ParamHandle`
+// CHECK-API: OL_ERRC_INVALID_NULL_POINTER
+// CHECK-API-NEXT: `NULL == ParamPointer`
+// CHECK-API-NOT: `NULL == ParamPointerOpt`
+
+// CHECK-VALIDATION: FunctionA_val
+// CHECK-VALIDATION: if (NULL == ParamHandle)
+// CHECK-VALIDATION-NEXT: return OL_ERRC_INVALID_NULL_HANDLE;
+// CHECK-VALIDATION: if (NULL == ParamPointer)
+// CHECK-VALIDATION-NEXT: return OL_ERRC_INVALID_NULL_POINTER;
+// CHECK-VALIDATION-NOT: if (NULL == ParamPointerOpt)
diff --git a/offload/test/tools/offload-tblgen/entry_points.td b/offload/test/tools/offload-tblgen/entry_points.td
new file mode 100644
index 0000000000000..a66ddb9279920
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/entry_points.td
@@ -0,0 +1,37 @@
+// RUN: %offload-tblgen -gen-entry-points -I %S/../../../liboffload/API %s | %fcheck-generic
+
+// Check entry point wrapper functions are generated correctly
+
+include "APIDefs.td"
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+    let params = [
+        Param<"uint32_t", "ParamA", "Parameter A description">,
+        Param<"uint32_t*", "ParamB", "Parameter B description">,
+    ];
+    let returns = [
+        Return<"OL_ERRC_INVALID_VALUE", ["When a value is invalid"]>
+    ];
+}
+
+
+// The validation function should call the implementation function
+// CHECK: FunctionA_val
+// CHECK: return FunctionA_impl(ParamA, ParamB);
+
+// CHECK: ol_result_t{{.*}} FunctionA(
+
+// The entry point should print tracing output if enabled
+// CHECK: if (offloadConfig().TracingEnabled) {
+// CHECK-NEXT: "---> FunctionA";
+
+// CHECK: Result = FunctionA_val(ParamA, ParamB);
+
+// Tracing should construct a param struct for printing
+// CHECK: if (offloadConfig().TracingEnabled) {
+// CHECK: function_a_params_t Params = {&ParamA, &ParamB};
+
+// CHECK: return Result;
diff --git a/offload/test/tools/offload-tblgen/functions_basic.td b/offload/test/tools/offload-tblgen/functions_basic.td
new file mode 100644
index 0000000000000..dec93577b57e9
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/functions_basic.td
@@ -0,0 +1,39 @@
+// RUN: %offload-tblgen -gen-api -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-API
+// RUN: %offload-tblgen -gen-exports -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-EXPORTS
+// RUN: %offload-tblgen -gen-func-names -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-FUNC-MACRO
+
+// Check basic support for API functions
+
+include "APIDefs.td"
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+    let params = [
+        Param<"uint32_t", "ParamA", "Parameter A description">,
+        Param<"uint32_t*", "ParamB", "Parameter B description">,
+    ];
+    let returns = [
+        Return<"OL_ERRC_INVALID_VALUE", ["When a value is invalid"]>
+    ];
+}
+
+// CHECK-API: /// @brief Function A description
+// CHECK-API: /// @details
+// CHECK-API-NEXT: Function A detailed information
+// CHECK-API: /// @returns
+// CHECK-API: OL_ERRC_INVALID_VALUE
+// CHECK-API-NEXT: When a value is invalid
+
+// CHECK-API: ol_result_t
+// CHECK-API-SAME: FunctionA
+
+// CHECK-API: // Parameter A description
+// CHECK-API-NEXT: uint32_t ParamA
+// CHECK-API: // Parameter B description
+// CHECK-API-NEXT: uint32_t* ParamB
+
+// CHECK-EXPORTS: FunctionA
+
+// CHECK-FUNC-MACRO: OFFLOAD_FUNC(FunctionA)
diff --git a/offload/test/tools/offload-tblgen/functions_code_loc.td b/offload/test/tools/offload-tblgen/functions_code_loc.td
new file mode 100644
index 0000000000000..aec20129343f5
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/functions_code_loc.td
@@ -0,0 +1,26 @@
+// RUN: %offload-tblgen -gen-api -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-API
+// RUN: %offload-tblgen -gen-exports -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-EXPORTS
+// RUN: %offload-tblgen -gen-func-names -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-FUNC-MACRO
+
+// Check that the function variant with code location information is generated
+// and is otherwise the same as the regular function
+
+include "APIDefs.td"
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+    let params = [
+        Param<"uint32_t", "ParamA", "Parameter A description">,
+        Param<"uint32_t*", "ParamB", "Parameter B description">,
+    ];
+    let returns = [
+        Return<"OL_ERRC_INVALID_VALUE", ["When a value is invalid"]>
+    ];
+}
+
+// CHECK-API-DAG: ol_result_t{{.*}} FunctionA
+// CHECK-API-DAG: ol_result_t{{.*}} FunctionAWithCodeLoc
+// CHECK-EXPORTS: FunctionAWithCodeLoc
+// CHECK-FUNC-MACRO: OFFLOAD_FUNC(FunctionAWithCodeLoc)
diff --git a/offload/test/tools/offload-tblgen/functions_ranged_param.td b/offload/test/tools/offload-tblgen/functions_ranged_param.td
new file mode 100644
index 0000000000000..21a84d8a70334
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/functions_ranged_param.td
@@ -0,0 +1,36 @@
+// RUN: %offload-tblgen -gen-print-header -I %S/../../../liboffload/API %s | %fcheck-generic
+
+// Check that ranged function parameters are implemented correctly. These
+// are pointers to an array of an arbitrary size. Their size is described as a
+// range between two values. This is typically between 0 and a parameter such
+// as NumItems. The range information helps the printing code print the entire
+// range of the output rather than just the pointer or the first element.
+
+include "APIDefs.td"
+
+def : Handle {
+    let name = "some_handle_t";
+    let desc = "An example handle type";
+}
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+  let params = [
+    Param<"size_t", "OutCount", "the number of things to write out", PARAM_IN>,
+    RangedParam<"some_handle_t*", "OutPtr", "pointer to the output things.", PARAM_OUT,
+      Range<"0", "OutCount">>
+  ];
+  let returns = [];
+}
+
+// CHECK: inline std::ostream &operator<<(std::ostream &os, const struct function_a_params_t *params) {
+// CHECK:   os << ".OutPtr = ";
+// CHECK:   for (size_t i = 0; i < *params->pOutCount; i++) {
+// CHECK:     if (i > 0) {
+// CHECK:       os << ", ";
+// CHECK:     }
+// CHECK:     printPtr(os, (*params->pOutPtr)[i]);
+// CHECK:   }
+// CHECK:   os << "}";
diff --git a/offload/test/tools/offload-tblgen/print_enum.td b/offload/test/tools/offload-tblgen/print_enum.td
new file mode 100644
index 0000000000000..0b5506009bec5
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/print_enum.td
@@ -0,0 +1,34 @@
+// RUN: %offload-tblgen -gen-print-header -I %S/../../../liboffload/API %s | %fcheck-generic
+
+// Check that print helpers are created for enums
+
+include "APIDefs.td"
+
+def : Enum {
+  let name = "my_enum_t";
+  let desc = "An example enum";
+  let etors =[
+    Etor<"VALUE_ONE", "The first enum value">,
+    Etor<"VALUE_TWO", "The second enum value">,
+    Etor<"VALUE_THREE", "The third enum value">,
+    Etor<"VALUE_FOUR", "The fourth enum value">,
+  ];
+}
+
+// CHECK: inline std::ostream &operator<<(std::ostream &os, enum my_enum_t value)
+// CHECK: switch (value) {
+// CHECK: case MY_ENUM_VALUE_ONE:
+// CHECK:   os << "MY_ENUM_VALUE_ONE";
+// CHECK:   break;
+// CHECK: case MY_ENUM_VALUE_TWO:
+// CHECK:   os << "MY_ENUM_VALUE_TWO";
+// CHECK:   break;
+// CHECK: case MY_ENUM_VALUE_THREE:
+// CHECK:   os << "MY_ENUM_VALUE_THREE";
+// CHECK:   break;
+// CHECK: case MY_ENUM_VALUE_FOUR:
+// CHECK:   os << "MY_ENUM_VALUE_FOUR";
+// CHECK:   break;
+// CHECK: default:
+// CHECK:   os << "unknown enumerator";
+// CHECK:   break;
diff --git a/offload/test/tools/offload-tblgen/print_function.td b/offload/test/tools/offload-tblgen/print_function.td
new file mode 100644
index 0000000000000..3f4944df65941
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/print_function.td
@@ -0,0 +1,38 @@
+// RUN: %offload-tblgen -gen-print-header -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-PRINT
+// RUN: %offload-tblgen -gen-api -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-API
+
+// Check that print helpers are created for functions
+
+include "APIDefs.td"
+
+def : Handle {
+    let name = "ol_foo_handle_t";
+    let desc = "Example handle type";
+}
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+    let params = [
+        Param<"uint32_t", "ParamValue", "A plain value parameter">,
+        Param<"ol_foo_handle_t", "ParamHandle", "A handle parameter">,
+        Param<"uint32_t*", "ParamPointer", "A pointer parameter">,
+    ];
+    let returns = [];
+}
+
+// CHECK-API: typedef struct function_a_params_t {
+// CHECK-API-NEXT: uint32_t* pParamValue;
+// CHECK-API-NEXT: ol_foo_handle_t* pParamHandle;
+// CHECK-API-NEXT: uint32_t** pParamPointer;
+
+// CHECK-PRINT: inline std::ostream &operator<<(std::ostream &os, const struct function_a_params_t *params)
+// CHECK-PRINT: os << ".ParamValue = ";
+// CHECK-PRINT: os << *params->pParamValue;
+// CHECK-PRINT: os << ", ";
+// CHECK-PRINT: os << ".ParamHandle = ";
+// CHECK-PRINT: printPtr(os, *params->pParamHandle);
+// CHECK-PRINT: os << ", ";
+// CHECK-PRINT: os << ".ParamPointer = ";
+// CHECK-PRINT: printPtr(os, *params->pParamPointer);
diff --git a/offload/test/tools/offload-tblgen/type_tagged_enum.td b/offload/test/tools/offload-tblgen/type_tagged_enum.td
new file mode 100644
index 0000000000000..49e91e43bb6ef
--- /dev/null
+++ b/offload/test/tools/offload-tblgen/type_tagged_enum.td
@@ -0,0 +1,76 @@
+// RUN: %offload-tblgen -gen-api -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-API
+// RUN: %offload-tblgen -gen-print-header -I %S/../../../liboffload/API %s | %fcheck-generic --check-prefix=CHECK-PRINT
+
+// Check that type-tagged enumerators are implemented correctly. They enable
+// functions to return data of an arbitrary type and size via a void*, using
+// the value of an enum parameter to indicate which type is being returned.
+// This allows, for example, for a single olGetDeviceInfo function, rather
+// than requiring a separate entry point for every possible query.
+
+include "APIDefs.td"
+
+def : Handle {
+    let name = "some_handle_t";
+    let desc = "An example handle type";
+}
+
+def : Enum {
+  let name = "my_type_tagged_enum_t";
+  let desc = "Example type tagged enum";
+  let is_typed = 1;
+  let etors = [
+    TaggedEtor<"VALUE_ONE", "uint32_t", "Value one.">,
+    TaggedEtor<"VALUE_TWO", "char[]", "Value two.">,
+    TaggedEtor<"VALUE_THREE", "some_handle_t", "Value three.">
+  ];
+}
+
+// Check the tagged types appear in the comments
+// CHECK-API: typedef enum my_type_tagged_enum_t {
+// CHECK-API-NEXT: [uint32_t] Value one.
+// CHECK-API-NEXT: MY_TYPE_TAGGED_ENUM_VALUE_ONE = 0,
+// CHECK-API-NEXT: [char[]] Value two.
+// CHECK-API-NEXT: MY_TYPE_TAGGED_ENUM_VALUE_TWO = 1,
+// CHECK-API-NEXT: [some_handle_t] Value three.
+// CHECK-API-NEXT: MY_TYPE_TAGGED_ENUM_VALUE_THREE = 2,
+
+def : Function {
+    let name = "FunctionA";
+    let desc = "Function A description";
+    let details = [ "Function A detailed information" ];
+  let params = [
+    Param<"my_type_tagged_enum_t", "PropName", "type of the info to retrieve", PARAM_IN>,
+    Param<"size_t", "PropSize", "the number of bytes pointed to by PropValue.", PARAM_IN>,
+    TypeTaggedParam<"void*", "PropValue", "array of bytes holding the info. "
+      "If PropSize is not equal to or greater to the real number of bytes needed to return the info "
+      "then the OL_ERRC_INVALID_SIZE error is returned and PropValue is not used.", PARAM_OUT,
+      TypeInfo<"PropName" , "PropSize">>
+  ];
+  let returns = [];
+}
+
+// Check that a tagged enum print function definition is generated
+// CHECK-PRINT: void printTagged(std::ostream &os, const void *ptr, my_type_tagged_enum_t value, size_t size) {
+// CHECK-PRINT: case MY_TYPE_TAGGED_ENUM_VALUE_ONE: {
+// CHECK-PRINT:   const uint32_t * const tptr = (const uint32_t * const)ptr;
+// CHECK-PRINT:   os << (const void *)tptr << " (";
+// CHECK-PRINT:   os << *tptr;
+// CHECK-PRINT:   os << ")";
+// CHECK-PRINT:   break;
+// CHECK-PRINT: }
+// CHECK-PRINT: case MY_TYPE_TAGGED_ENUM_VALUE_TWO: {
+// CHECK-PRINT:   printPtr(os, (const char*) ptr);
+// CHECK-PRINT:   break;
+// CHECK-PRINT: }
+// CHECK-PRINT: case MY_TYPE_TAGGED_ENUM_VALUE_THREE: {
+// CHECK-PRINT:   const some_handle_t * const tptr = (const some_handle_t * const)ptr;
+// CHECK-PRINT:   os << (const void *)tptr << " (";
+// CHECK-PRINT:   os << *tptr;
+// CHECK-PRINT:   os << ")";
+// CHECK-PRINT:   break;
+// CHECK-PRINT: }
+
+// Check that the tagged type information is used when printing function parameters
+// CHECK-PRINT: std::ostream &operator<<(std::ostream &os, const struct function_a_params_t *params) {
+// CHECK-PRINT: os << ".PropValue = " 
+// CHECK-PRINT-NEXT: printTagged(os, *params->pPropValue, *params->pPropName, *params->pPropSize);
diff --git a/offload/tools/offload-tblgen/APIGen.cpp b/offload/tools/offload-tblgen/APIGen.cpp
new file mode 100644
index 0000000000000..97a2464f7a75c
--- /dev/null
+++ b/offload/tools/offload-tblgen/APIGen.cpp
@@ -0,0 +1,229 @@
+//===- offload-tblgen/APIGen.cpp - Tablegen backend for Offload header ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a Tablegen backend that produces the contents of the Offload API
+// header. The generated comments are Doxygen compatible.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+
+#include "GenCommon.hpp"
+#include "RecordTypes.hpp"
+
+using namespace llvm;
+using namespace offload::tblgen;
+
+// Produce a possibly multi-line comment from the input string
+static std::string MakeComment(StringRef in) {
+  std::string out = "";
+  size_t LineStart = 0;
+  size_t LineBreak = 0;
+  while (LineBreak < in.size()) {
+    LineBreak = in.find_first_of("\n", LineStart);
+    if (LineBreak - LineStart <= 1) {
+      break;
+    }
+    out += std::string("/// ") +
+           in.substr(LineStart, LineBreak - LineStart).str() + "\n";
+    LineStart = LineBreak + 1;
+  }
+
+  return out;
+}
+
+static void ProcessHandle(const HandleRec &H, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("/// @brief {0}\n", H.getDesc());
+  OS << formatv("typedef struct {0}_ *{0};\n", H.getName());
+}
+
+static void ProcessTypedef(const TypedefRec &T, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("/// @brief {0}\n", T.getDesc());
+  OS << formatv("typedef {0} {1};\n", T.getValue(), T.getName());
+}
+
+static void ProcessMacro(const MacroRec &M, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("#ifndef {0}\n", M.getName());
+  if (auto Condition = M.getCondition()) {
+    OS << formatv("#if {0}\n", *Condition);
+  }
+  OS << "/// @brief " << M.getDesc() << "\n";
+  OS << formatv("#define {0} {1}\n", M.getNameWithArgs(), M.getValue());
+  if (auto AltValue = M.getAltValue()) {
+    OS << "#else\n";
+    OS << formatv("#define {0} {1}\n", M.getNameWithArgs(), *AltValue);
+  }
+  if (auto Condition = M.getCondition()) {
+    OS << formatv("#endif // {0}\n", *Condition);
+  }
+  OS << formatv("#endif // {0}\n", M.getName());
+}
+
+static void ProcessFunction(const FunctionRec &F, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("/// @brief {0}\n", F.getDesc());
+  OS << CommentsBreak;
+
+  OS << "/// @details\n";
+  for (auto &Detail : F.getDetails()) {
+    OS << formatv("///    - {0}\n", Detail);
+  }
+  OS << CommentsBreak;
+
+  // Emit analogue remarks
+  auto Analogues = F.getAnalogues();
+  if (!Analogues.empty()) {
+    OS << "/// @remarks\n///  _Analogues_\n";
+    for (auto &Analogue : Analogues) {
+      OS << formatv("///    - **{0}**\n", Analogue);
+    }
+    OS << CommentsBreak;
+  }
+
+  OS << "/// @returns\n";
+  auto Returns = F.getReturns();
+  for (auto &Ret : Returns) {
+    OS << formatv("///     - ::{0}\n", Ret.getValue());
+    auto RetConditions = Ret.getConditions();
+    for (auto &RetCondition : RetConditions) {
+      OS << formatv("///         + {0}\n", RetCondition);
+    }
+  }
+
+  OS << formatv("{0}_APIEXPORT {1}_result_t {0}_APICALL ", PrefixUpper,
+                PrefixLower);
+  OS << F.getName();
+  OS << "(\n";
+  auto Params = F.getParams();
+  for (auto &Param : Params) {
+    OS << MakeParamComment(Param) << "\n";
+    OS << "  " << Param.getType() << " " << Param.getName();
+    if (Param != Params.back()) {
+      OS << ",\n";
+    } else {
+      OS << "\n";
+    }
+  }
+  OS << ");\n\n";
+}
+
+static void ProcessEnum(const EnumRec &Enum, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("/// @brief {0}\n", Enum.getDesc());
+  OS << formatv("typedef enum {0} {{\n", Enum.getName());
+
+  uint32_t EtorVal = 0;
+  for (const auto &EnumVal : Enum.getValues()) {
+    if (Enum.isTyped()) {
+      OS << MakeComment(
+          formatv("[{0}] {1}", EnumVal.getTaggedType(), EnumVal.getDesc())
+              .str());
+    } else {
+      OS << MakeComment(EnumVal.getDesc());
+    }
+    OS << formatv(TAB_1 "{0}_{1} = {2},\n", Enum.getEnumValNamePrefix(),
+                  EnumVal.getName(), EtorVal++);
+  }
+
+  // Add force uint32 val
+  OS << formatv(TAB_1 "/// @cond\n" TAB_1
+                      "{0}_FORCE_UINT32 = 0x7fffffff\n" TAB_1
+                      "/// @endcond\n\n",
+                Enum.getEnumValNamePrefix());
+
+  OS << formatv("} {0};\n", Enum.getName());
+}
+
+static void ProcessStruct(const StructRec &Struct, raw_ostream &OS) {
+  OS << CommentsHeader;
+  OS << formatv("/// @brief {0}\n", Struct.getDesc());
+  OS << formatv("typedef struct {0} {{\n", Struct.getName());
+
+  for (const auto &Member : Struct.getMembers()) {
+    OS << formatv(TAB_1 "{0} {1}; {2}", Member.getType(), Member.getName(),
+                  MakeComment(Member.getDesc()));
+  }
+
+  OS << formatv("} {0};\n\n", Struct.getName());
+}
+
+static void ProcessFuncParamStruct(const FunctionRec &Func, raw_ostream &OS) {
+  if (Func.getParams().size() == 0) {
+    return;
+  }
+
+  auto FuncParamStructBegin = R"(
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for {0}
+/// @details Each entry is a pointer to the parameter passed to the function;
+typedef struct {1} {{
+)";
+
+  OS << formatv(FuncParamStructBegin, Func.getName(),
+                Func.getParamStructName());
+  for (const auto &Param : Func.getParams()) {
+    OS << TAB_1 << Param.getType() << "* p" << Param.getName() << ";\n";
+  }
+  OS << formatv("} {0};\n", Func.getParamStructName());
+}
+
+static void ProcessFuncWithCodeLocVariant(const FunctionRec &Func,
+                                          raw_ostream &OS) {
+
+  auto FuncWithCodeLocBegin = R"(
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Variant of {0} that also sets source code location information
+/// @details See also ::{0}
+OL_APIEXPORT ol_result_t OL_APICALL {0}WithCodeLoc(
+)";
+  OS << formatv(FuncWithCodeLocBegin, Func.getName());
+  auto Params = Func.getParams();
+  for (auto &Param : Params) {
+    OS << "  " << Param.getType() << " " << Param.getName();
+    OS << ",\n";
+  }
+  OS << "ol_code_location_t *CodeLocation);\n\n";
+}
+
+void EmitOffloadAPI(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << GenericHeader;
+  OS << FileHeader;
+  // Generate main API definitions
+  for (auto *R : Records.getAllDerivedDefinitions("APIObject")) {
+    if (R->isSubClassOf("Macro")) {
+      ProcessMacro(MacroRec{R}, OS);
+    } else if (R->isSubClassOf("Typedef")) {
+      ProcessTypedef(TypedefRec{R}, OS);
+    } else if (R->isSubClassOf("Handle")) {
+      ProcessHandle(HandleRec{R}, OS);
+    } else if (R->isSubClassOf("Function")) {
+      ProcessFunction(FunctionRec{R}, OS);
+    } else if (R->isSubClassOf("Enum")) {
+      ProcessEnum(EnumRec{R}, OS);
+    } else if (R->isSubClassOf("Struct")) {
+      ProcessStruct(StructRec{R}, OS);
+    }
+  }
+
+  // Generate auxiliary definitions (func param structs etc)
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    ProcessFuncParamStruct(FunctionRec{R}, OS);
+  }
+
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    ProcessFuncWithCodeLocVariant(FunctionRec{R}, OS);
+  }
+
+  OS << FileFooter;
+}
diff --git a/offload/tools/offload-tblgen/CMakeLists.txt b/offload/tools/offload-tblgen/CMakeLists.txt
new file mode 100644
index 0000000000000..e7e7c85490543
--- /dev/null
+++ b/offload/tools/offload-tblgen/CMakeLists.txt
@@ -0,0 +1,26 @@
+##===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+##===----------------------------------------------------------------------===##
+include(TableGen)
+
+set(LLVM_LINK_COMPONENTS Support)
+
+add_tablegen(offload-tblgen OFFLOAD
+  EXPORT OFFLOAD
+  APIGen.cpp
+  EntryPointGen.cpp
+  FuncsGen.cpp
+  GenCommon.hpp
+  Generators.hpp
+  offload-tblgen.cpp
+  PrintGen.cpp
+  RecordTypes.hpp
+  )
+
+set(OFFLOAD_TABLEGEN_EXE "${OFFLOAD_TABLEGEN_EXE}" CACHE INTERNAL "")
+set(OFFLOAD_TABLEGEN_TARGET "${OFFLOAD_TABLEGEN_TARGET}" CACHE INTERNAL "")
+
diff --git a/offload/tools/offload-tblgen/EntryPointGen.cpp b/offload/tools/offload-tblgen/EntryPointGen.cpp
new file mode 100644
index 0000000000000..990ff96a3121d
--- /dev/null
+++ b/offload/tools/offload-tblgen/EntryPointGen.cpp
@@ -0,0 +1,138 @@
+//===- offload-tblgen/EntryPointGen.cpp - Tablegen backend for Offload ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a Tablegen backend that produces the actual entry points for the
+// Offload API. It serves as a place to integrate functionality like tracing
+// and validation before dispatching to the actual implementations.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Record.h"
+
+#include "GenCommon.hpp"
+#include "RecordTypes.hpp"
+
+using namespace llvm;
+using namespace offload::tblgen;
+
+static void EmitValidationFunc(const FunctionRec &F, raw_ostream &OS) {
+  OS << CommentsHeader;
+  // Emit preamble
+  OS << formatv("{0}_impl_result_t {1}_val(\n  ", PrefixLower, F.getName());
+  // Emit arguments
+  std::string ParamNameList = "";
+  for (auto &Param : F.getParams()) {
+    OS << Param.getType() << " " << Param.getName();
+    if (Param != F.getParams().back()) {
+      OS << ", ";
+    }
+    ParamNameList += Param.getName().str() + ", ";
+  }
+  OS << ") {\n";
+
+  OS << TAB_1 "if (true /*enableParameterValidation*/) {\n";
+  // Emit validation checks
+  for (const auto &Return : F.getReturns()) {
+    for (auto &Condition : Return.getConditions()) {
+      if (Condition.starts_with("`") && Condition.ends_with("`")) {
+        auto ConditionString = Condition.substr(1, Condition.size() - 2);
+        OS << formatv(TAB_2 "if ({0}) {{\n", ConditionString);
+        OS << formatv(TAB_3 "return {0};\n", Return.getValue());
+        OS << TAB_2 "}\n\n";
+      }
+    }
+  }
+  OS << TAB_1 "}\n\n";
+
+  // Perform actual function call to the implementation
+  ParamNameList = ParamNameList.substr(0, ParamNameList.size() - 2);
+  OS << formatv(TAB_1 "return {0}_impl({1});\n\n", F.getName(), ParamNameList);
+  OS << "}\n";
+}
+
+static void EmitEntryPointFunc(const FunctionRec &F, raw_ostream &OS) {
+  // Emit preamble
+  OS << formatv("{1}_APIEXPORT {0}_result_t {1}_APICALL {2}(\n  ", PrefixLower,
+                PrefixUpper, F.getName());
+  // Emit arguments
+  std::string ParamNameList = "";
+  for (auto &Param : F.getParams()) {
+    OS << Param.getType() << " " << Param.getName();
+    if (Param != F.getParams().back()) {
+      OS << ", ";
+    }
+    ParamNameList += Param.getName().str() + ", ";
+  }
+  OS << ") {\n";
+
+  // Emit pre-call prints
+  OS << TAB_1 "if (offloadConfig().TracingEnabled) {\n";
+  OS << formatv(TAB_2 "std::cout << \"---> {0}\";\n", F.getName());
+  OS << TAB_1 "}\n\n";
+
+  // Perform actual function call to the validation wrapper
+  ParamNameList = ParamNameList.substr(0, ParamNameList.size() - 2);
+  OS << formatv(TAB_1 "{0}_result_t Result = {1}_val({2});\n\n", PrefixLower,
+                F.getName(), ParamNameList);
+
+  // Emit post-call prints
+  OS << TAB_1 "if (offloadConfig().TracingEnabled) {\n";
+  if (F.getParams().size() > 0) {
+    OS << formatv(TAB_2 "{0} Params = {{", F.getParamStructName());
+    for (const auto &Param : F.getParams()) {
+      OS << "&" << Param.getName();
+      if (Param != F.getParams().back()) {
+        OS << ", ";
+      }
+    }
+    OS << formatv("};\n");
+    OS << TAB_2 "std::cout << \"(\" << &Params << \")\";\n";
+  } else {
+    OS << TAB_2 "std::cout << \"()\";\n";
+  }
+  OS << TAB_2 "std::cout << \"-> \" << Result << \"\\n\";\n";
+  OS << TAB_2 "if (Result && Result->Details) {\n";
+  OS << TAB_3 "std::cout << \"     *Error Details* \" << Result->Details "
+              "<< \" \\n\";\n";
+  OS << TAB_2 "}\n";
+  OS << TAB_1 "}\n";
+
+  OS << TAB_1 "return Result;\n";
+  OS << "}\n";
+}
+
+static void EmitCodeLocWrapper(const FunctionRec &F, raw_ostream &OS) {
+  // Emit preamble
+  OS << formatv("{0}_result_t {1}WithCodeLoc(\n  ", PrefixLower, F.getName());
+  // Emit arguments
+  std::string ParamNameList = "";
+  for (auto &Param : F.getParams()) {
+    OS << Param.getType() << " " << Param.getName() << ", ";
+    ParamNameList += Param.getName().str();
+    if (Param != F.getParams().back()) {
+      ParamNameList += ", ";
+    }
+  }
+  OS << "ol_code_location_t *CodeLocation";
+  OS << ") {\n";
+  OS << TAB_1 "currentCodeLocation() = CodeLocation;\n";
+  OS << formatv(TAB_1 "{0}_result_t Result = {1}({2});\n\n", PrefixLower,
+                F.getName(), ParamNameList);
+  OS << TAB_1 "currentCodeLocation() = nullptr;\n";
+  OS << TAB_1 "return Result;\n";
+  OS << "}\n";
+}
+
+void EmitOffloadEntryPoints(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << GenericHeader;
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    EmitValidationFunc(FunctionRec{R}, OS);
+    EmitEntryPointFunc(FunctionRec{R}, OS);
+    EmitCodeLocWrapper(FunctionRec{R}, OS);
+  }
+}
diff --git a/offload/tools/offload-tblgen/FuncsGen.cpp b/offload/tools/offload-tblgen/FuncsGen.cpp
new file mode 100644
index 0000000000000..3238652176198
--- /dev/null
+++ b/offload/tools/offload-tblgen/FuncsGen.cpp
@@ -0,0 +1,74 @@
+//===- offload-tblgen/APIGen.cpp - Tablegen backend for Offload functions -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a Tablegen backend that handles generation of various small files
+// pertaining to the API functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Record.h"
+
+#include "GenCommon.hpp"
+#include "RecordTypes.hpp"
+
+using namespace llvm;
+using namespace offload::tblgen;
+
+// Emit a list of just the API function names
+void EmitOffloadFuncNames(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << GenericHeader;
+  OS << R"(
+#ifndef OFFLOAD_FUNC
+#error Please define the macro OFFLOAD_FUNC(Function)
+#endif
+
+)";
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    FunctionRec FR{R};
+    OS << formatv("OFFLOAD_FUNC({0})", FR.getName()) << "\n";
+  }
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    FunctionRec FR{R};
+    OS << formatv("OFFLOAD_FUNC({0}WithCodeLoc)", FR.getName()) << "\n";
+  }
+
+  OS << "\n#undef OFFLOAD_FUNC\n";
+}
+
+void EmitOffloadExports(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << "VERS1.0 {\n";
+  OS << TAB_1 "global:\n";
+
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    OS << formatv(TAB_2 "{0};\n", FunctionRec(R).getName());
+  }
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    OS << formatv(TAB_2 "{0}WithCodeLoc;\n", FunctionRec(R).getName());
+  }
+  OS << TAB_1 "local:\n";
+  OS << TAB_2 "*;\n";
+  OS << "};\n";
+}
+
+// Emit declarations for every implementation function
+void EmitOffloadImplFuncDecls(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << GenericHeader;
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    FunctionRec F{R};
+    OS << formatv("{0}_impl_result_t {1}_impl(", PrefixLower, F.getName());
+    auto Params = F.getParams();
+    for (auto &Param : Params) {
+      OS << Param.getType() << " " << Param.getName();
+      if (Param != Params.back()) {
+        OS << ", ";
+      }
+    }
+    OS << ");\n\n";
+  }
+}
diff --git a/offload/tools/offload-tblgen/GenCommon.hpp b/offload/tools/offload-tblgen/GenCommon.hpp
new file mode 100644
index 0000000000000..db432e9958b5d
--- /dev/null
+++ b/offload/tools/offload-tblgen/GenCommon.hpp
@@ -0,0 +1,67 @@
+//===- offload-tblgen/GenCommon.cpp - Common defs for Offload generators --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "RecordTypes.hpp"
+#include "llvm/Support/FormatVariadic.h"
+
+// Having inline bits of tabbed code is hard to read, provide some definitions
+// so we can keep things tidier
+#define TAB_1 "  "
+#define TAB_2 "    "
+#define TAB_3 "      "
+#define TAB_4 "        "
+#define TAB_5 "          "
+
+constexpr auto GenericHeader =
+    R"(//===- Auto-generated file, part of the LLVM/Offload project --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+)";
+
+constexpr auto FileHeader = R"(
+// Auto-generated file, do not manually edit.
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+)";
+
+constexpr auto FileFooter = R"(
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+)";
+
+constexpr auto CommentsHeader = R"(
+///////////////////////////////////////////////////////////////////////////////
+)";
+
+constexpr auto CommentsBreak = "///\n";
+
+constexpr auto PrefixLower = "ol";
+constexpr auto PrefixUpper = "OL";
+
+inline std::string
+MakeParamComment(const llvm::offload::tblgen::ParamRec &Param) {
+  return llvm::formatv("// {0}{1}{2} {3}", (Param.isIn() ? "[in]" : ""),
+                       (Param.isOut() ? "[out]" : ""),
+                       (Param.isOpt() ? "[optional]" : ""), Param.getDesc());
+}
diff --git a/offload/tools/offload-tblgen/Generators.hpp b/offload/tools/offload-tblgen/Generators.hpp
new file mode 100644
index 0000000000000..8b6104c5cd9c6
--- /dev/null
+++ b/offload/tools/offload-tblgen/Generators.hpp
@@ -0,0 +1,23 @@
+//===- offload-tblgen/Generators.hpp - Offload generator declarations -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "llvm/TableGen/Record.h"
+
+void EmitOffloadAPI(const llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
+void EmitOffloadFuncNames(const llvm::RecordKeeper &Records,
+                          llvm::raw_ostream &OS);
+void EmitOffloadImplFuncDecls(const llvm::RecordKeeper &Records,
+                              llvm::raw_ostream &OS);
+void EmitOffloadEntryPoints(const llvm::RecordKeeper &Records,
+                            llvm::raw_ostream &OS);
+void EmitOffloadPrintHeader(const llvm::RecordKeeper &Records,
+                            llvm::raw_ostream &OS);
+void EmitOffloadExports(const llvm::RecordKeeper &Records,
+                        llvm::raw_ostream &OS);
diff --git a/offload/tools/offload-tblgen/PrintGen.cpp b/offload/tools/offload-tblgen/PrintGen.cpp
new file mode 100644
index 0000000000000..2a7c63c3dfd1f
--- /dev/null
+++ b/offload/tools/offload-tblgen/PrintGen.cpp
@@ -0,0 +1,226 @@
+//===- offload-tblgen/APIGen.cpp - Tablegen backend for Offload printing --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a Tablegen backend that produces print functions for the Offload API
+// entry point functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Record.h"
+
+#include "GenCommon.hpp"
+#include "RecordTypes.hpp"
+
+using namespace llvm;
+using namespace offload::tblgen;
+
+constexpr auto PrintEnumHeader =
+    R"(///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the {0} type
+/// @returns std::ostream &
+)";
+
+constexpr auto PrintTaggedEnumHeader =
+    R"(///////////////////////////////////////////////////////////////////////////////
+/// @brief Print type-tagged {0} enum value
+/// @returns std::ostream &
+)";
+
+static void ProcessEnum(const EnumRec &Enum, raw_ostream &OS) {
+  OS << formatv(PrintEnumHeader, Enum.getName());
+  OS << formatv(
+      "inline std::ostream &operator<<(std::ostream &os, enum {0} value) "
+      "{{\n" TAB_1 "switch (value) {{\n",
+      Enum.getName());
+
+  for (const auto &Val : Enum.getValues()) {
+    auto Name = Enum.getEnumValNamePrefix() + "_" + Val.getName();
+    OS << formatv(TAB_1 "case {0}:\n", Name);
+    OS << formatv(TAB_2 "os << \"{0}\";\n", Name);
+    OS << formatv(TAB_2 "break;\n");
+  }
+
+  OS << TAB_1 "default:\n" TAB_2 "os << \"unknown enumerator\";\n" TAB_2
+              "break;\n" TAB_1 "}\n" TAB_1 "return os;\n}\n\n";
+
+  if (!Enum.isTyped()) {
+    return;
+  }
+
+  OS << formatv(PrintTaggedEnumHeader, Enum.getName());
+
+  OS << formatv(R"""(template <>
+inline void printTagged(std::ostream &os, const void *ptr, {0} value, size_t size) {{
+  if (ptr == NULL) {{
+    printPtr(os, ptr);
+    return;
+  }
+
+  switch (value) {{
+)""",
+                Enum.getName());
+
+  for (const auto &Val : Enum.getValues()) {
+    auto Name = Enum.getEnumValNamePrefix() + "_" + Val.getName();
+    auto Type = Val.getTaggedType();
+    OS << formatv(TAB_1 "case {0}: {{\n", Name);
+    // Special case for strings
+    if (Type == "char[]") {
+      OS << formatv(TAB_2 "printPtr(os, (const char*) ptr);\n");
+    } else {
+      OS << formatv(TAB_2 "const {0} * const tptr = (const {0} * const)ptr;\n",
+                    Type);
+      // TODO: Handle other cases here
+      OS << TAB_2 "os << (const void *)tptr << \" (\";\n";
+      if (Type.ends_with("*")) {
+        OS << TAB_2 "os << printPtr(os, tptr);\n";
+      } else {
+        OS << TAB_2 "os << *tptr;\n";
+      }
+      OS << TAB_2 "os << \")\";\n";
+    }
+    OS << formatv(TAB_2 "break;\n" TAB_1 "}\n");
+  }
+
+  OS << TAB_1 "default:\n" TAB_2 "os << \"unknown enumerator\";\n" TAB_2
+              "break;\n" TAB_1 "}\n";
+
+  OS << "}\n";
+}
+
+static void EmitResultPrint(raw_ostream &OS) {
+  OS << R""(
+inline std::ostream &operator<<(std::ostream &os,
+                                const ol_error_struct_t *Err) {
+  if (Err == nullptr) {
+    os << "OL_SUCCESS";
+  } else {
+    os << Err->Code;
+  }
+  return os;
+}
+)"";
+}
+
+static void EmitFunctionParamStructPrint(const FunctionRec &Func,
+                                         raw_ostream &OS) {
+  if (Func.getParams().size() == 0) {
+    return;
+  }
+
+  OS << formatv(R"(
+inline std::ostream &operator<<(std::ostream &os, const struct {0} *params) {{
+)",
+                Func.getParamStructName());
+
+  for (const auto &Param : Func.getParams()) {
+    OS << formatv(TAB_1 "os << \".{0} = \";\n", Param.getName());
+    if (auto Range = Param.getRange()) {
+      OS << formatv(TAB_1 "os << \"{{\";\n");
+      OS << formatv(TAB_1 "for (size_t i = {0}; i < *params->p{1}; i++) {{\n",
+                    Range->first, Range->second);
+      OS << TAB_2 "if (i > 0) {\n";
+      OS << TAB_3 " os << \", \";\n";
+      OS << TAB_2 "}\n";
+      OS << formatv(TAB_2 "printPtr(os, (*params->p{0})[i]);\n",
+                    Param.getName());
+      OS << formatv(TAB_1 "}\n");
+      OS << formatv(TAB_1 "os << \"}\";\n");
+    } else if (auto TypeInfo = Param.getTypeInfo()) {
+      OS << formatv(
+          TAB_1
+          "printTagged(os, *params->p{0}, *params->p{1}, *params->p{2});\n",
+          Param.getName(), TypeInfo->first, TypeInfo->second);
+    } else if (Param.isPointerType() || Param.isHandleType()) {
+      OS << formatv(TAB_1 "printPtr(os, *params->p{0});\n", Param.getName());
+    } else {
+      OS << formatv(TAB_1 "os << *params->p{0};\n", Param.getName());
+    }
+    if (Param != Func.getParams().back()) {
+      OS << TAB_1 "os << \", \";\n";
+    }
+  }
+
+  OS << TAB_1 "return os;\n}\n";
+}
+
+void EmitOffloadPrintHeader(const RecordKeeper &Records, raw_ostream &OS) {
+  OS << GenericHeader;
+  OS << R"""(
+// Auto-generated file, do not manually edit.
+
+#pragma once
+
+#include <OffloadAPI.h>
+#include <ostream>
+
+
+template <typename T> inline ol_result_t printPtr(std::ostream &os, const T *ptr);
+template <typename T> inline void printTagged(std::ostream &os, const void *ptr, T value, size_t size);
+)""";
+
+  // ==========
+  OS << "template <typename T> struct is_handle : std::false_type {};\n";
+  for (auto *R : Records.getAllDerivedDefinitions("Handle")) {
+    HandleRec H{R};
+    OS << formatv("template <> struct is_handle<{0}> : std::true_type {{};\n",
+                  H.getName());
+  }
+  OS << "template <typename T> inline constexpr bool is_handle_v = "
+        "is_handle<T>::value;\n";
+  // =========
+
+  // Forward declare the operator<< overloads so their implementations can
+  // use each other.
+  OS << "\n";
+  for (auto *R : Records.getAllDerivedDefinitions("Enum")) {
+    OS << formatv(
+        "inline std::ostream &operator<<(std::ostream &os, enum {0} value);\n",
+        EnumRec{R}.getName());
+  }
+  OS << "\n";
+
+  // Create definitions
+  for (auto *R : Records.getAllDerivedDefinitions("Enum")) {
+    EnumRec E{R};
+    ProcessEnum(E, OS);
+  }
+  EmitResultPrint(OS);
+
+  // Emit print functions for the function param structs
+  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    EmitFunctionParamStructPrint(FunctionRec{R}, OS);
+  }
+
+  OS << R"""(
+///////////////////////////////////////////////////////////////////////////////
+// @brief Print pointer value
+template <typename T> inline ol_result_t printPtr(std::ostream &os, const T *ptr) {
+    if (ptr == nullptr) {
+        os << "nullptr";
+    } else if constexpr (std::is_pointer_v<T>) {
+        os << (const void *)(ptr) << " (";
+        printPtr(os, *ptr);
+        os << ")";
+    } else if constexpr (std::is_void_v<T> || is_handle_v<T *>) {
+        os << (const void *)ptr;
+    } else if constexpr (std::is_same_v<std::remove_cv_t< T >, char>) {
+        os << (const void *)(ptr) << " (";
+        os << ptr;
+        os << ")";
+    } else {
+        os << (const void *)(ptr) << " (";
+        os << *ptr;
+        os << ")";
+    }
+
+    return OL_SUCCESS;
+}
+  )""";
+}
diff --git a/offload/tools/offload-tblgen/RecordTypes.hpp b/offload/tools/offload-tblgen/RecordTypes.hpp
new file mode 100644
index 0000000000000..0bf3256c525d9
--- /dev/null
+++ b/offload/tools/offload-tblgen/RecordTypes.hpp
@@ -0,0 +1,227 @@
+//===- offload-tblgen/RecordTypes.cpp - Offload record type wrappers -----===-//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <string>
+
+#include "llvm/TableGen/Record.h"
+
+namespace llvm {
+namespace offload {
+namespace tblgen {
+
+class HandleRec {
+public:
+  explicit HandleRec(const Record *rec) : rec(rec) {}
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+
+private:
+  const Record *rec;
+};
+
+class MacroRec {
+public:
+  explicit MacroRec(const Record *rec) : rec(rec) {
+    auto Name = rec->getValueAsString("name");
+    auto OpenBrace = Name.find_first_of("(");
+    nameWithoutArgs = Name.substr(0, OpenBrace);
+  }
+  StringRef getName() const { return nameWithoutArgs; }
+  StringRef getNameWithArgs() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+
+  std::optional<StringRef> getCondition() const {
+    return rec->getValueAsOptionalString("condition");
+  }
+  StringRef getValue() const { return rec->getValueAsString("value"); }
+  std::optional<StringRef> getAltValue() const {
+    return rec->getValueAsOptionalString("alt_value");
+  }
+
+private:
+  const Record *rec;
+  std::string nameWithoutArgs;
+};
+
+class TypedefRec {
+public:
+  explicit TypedefRec(const Record *rec) : rec(rec) {}
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  StringRef getValue() const { return rec->getValueAsString("value"); }
+
+private:
+  const Record *rec;
+};
+
+class EnumValueRec {
+public:
+  explicit EnumValueRec(const Record *rec) : rec(rec) {}
+  std::string getName() const { return rec->getValueAsString("name").upper(); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  StringRef getTaggedType() const {
+    return rec->getValueAsString("tagged_type");
+  }
+
+private:
+  const Record *rec;
+};
+
+class EnumRec {
+public:
+  explicit EnumRec(const Record *rec) : rec(rec) {
+    for (const auto *Val : rec->getValueAsListOfDefs("etors")) {
+      vals.emplace_back(EnumValueRec{Val});
+    }
+  }
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  const std::vector<EnumValueRec> &getValues() const { return vals; }
+
+  std::string getEnumValNamePrefix() const {
+    return StringRef(getName().str().substr(0, getName().str().length() - 2))
+        .upper();
+  }
+
+  bool isTyped() const { return rec->getValueAsBit("is_typed"); }
+
+private:
+  const Record *rec;
+  std::vector<EnumValueRec> vals;
+};
+
+class StructMemberRec {
+public:
+  explicit StructMemberRec(const Record *rec) : rec(rec) {}
+  StringRef getType() const { return rec->getValueAsString("type"); }
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+
+private:
+  const Record *rec;
+};
+
+class StructRec {
+public:
+  explicit StructRec(const Record *rec) : rec(rec) {
+    for (auto *Member : rec->getValueAsListOfDefs("all_members")) {
+      members.emplace_back(StructMemberRec(Member));
+    }
+  }
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  std::optional<StringRef> getBaseClass() const {
+    return rec->getValueAsOptionalString("base_class");
+  }
+  const std::vector<StructMemberRec> &getMembers() const { return members; }
+
+private:
+  const Record *rec;
+  std::vector<StructMemberRec> members;
+};
+
+class ParamRec {
+public:
+  explicit ParamRec(const Record *rec) : rec(rec) {
+    flags = rec->getValueAsBitsInit("flags");
+    auto *Range = rec->getValueAsDef("range");
+    auto RangeBegin = Range->getValueAsString("begin");
+    auto RangeEnd = Range->getValueAsString("end");
+    if (RangeBegin != "" && RangeEnd != "") {
+      range = {RangeBegin, RangeEnd};
+    } else {
+      range = std::nullopt;
+    }
+
+    auto *TypeInfo = rec->getValueAsDef("type_info");
+    auto TypeInfoEnum = TypeInfo->getValueAsString("enum");
+    auto TypeInfoSize = TypeInfo->getValueAsString("size");
+    if (TypeInfoEnum != "" && TypeInfoSize != "") {
+      typeinfo = {TypeInfoEnum, TypeInfoSize};
+    } else {
+      typeinfo = std::nullopt;
+    }
+  }
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getType() const { return rec->getValueAsString("type"); }
+  bool isPointerType() const { return getType().ends_with('*'); }
+  bool isHandleType() const { return getType().ends_with("_handle_t"); }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  bool isIn() const { return dyn_cast<BitInit>(flags->getBit(0))->getValue(); }
+  bool isOut() const { return dyn_cast<BitInit>(flags->getBit(1))->getValue(); }
+  bool isOpt() const { return dyn_cast<BitInit>(flags->getBit(2))->getValue(); }
+
+  const Record *getRec() const { return rec; }
+  std::optional<std::pair<StringRef, StringRef>> getRange() const {
+    return range;
+  }
+
+  std::optional<std::pair<StringRef, StringRef>> getTypeInfo() const {
+    return typeinfo;
+  }
+
+  // Needed to check whether we're at the back of a vector of params
+  bool operator!=(const ParamRec &p) const { return rec != p.getRec(); }
+
+private:
+  const Record *rec;
+  const BitsInit *flags;
+  std::optional<std::pair<StringRef, StringRef>> range;
+  std::optional<std::pair<StringRef, StringRef>> typeinfo;
+};
+
+class ReturnRec {
+public:
+  ReturnRec(const Record *rec) : rec(rec) {}
+  StringRef getValue() const { return rec->getValueAsString("value"); }
+  std::vector<StringRef> getConditions() const {
+    return rec->getValueAsListOfStrings("conditions");
+  }
+
+private:
+  const Record *rec;
+};
+
+class FunctionRec {
+public:
+  FunctionRec(const Record *rec) : rec(rec) {
+    for (auto &Ret : rec->getValueAsListOfDefs("all_returns"))
+      rets.emplace_back(Ret);
+    for (auto &Param : rec->getValueAsListOfDefs("params"))
+      params.emplace_back(Param);
+  }
+
+  std::string getParamStructName() const {
+    return llvm::formatv("{0}_params_t",
+                         llvm::convertToSnakeFromCamelCase(getName()));
+  }
+
+  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getClass() const { return rec->getValueAsString("api_class"); }
+  const std::vector<ReturnRec> &getReturns() const { return rets; }
+  const std::vector<ParamRec> &getParams() const { return params; }
+  StringRef getDesc() const { return rec->getValueAsString("desc"); }
+  std::vector<StringRef> getDetails() const {
+    return rec->getValueAsListOfStrings("details");
+  }
+  std::vector<StringRef> getAnalogues() const {
+    return rec->getValueAsListOfStrings("analogues");
+  }
+
+private:
+  std::vector<ReturnRec> rets;
+  std::vector<ParamRec> params;
+
+  const Record *rec;
+};
+
+} // namespace tblgen
+} // namespace offload
+} // namespace llvm
diff --git a/offload/tools/offload-tblgen/offload-tblgen.cpp b/offload/tools/offload-tblgen/offload-tblgen.cpp
new file mode 100644
index 0000000000000..1912abf5265c7
--- /dev/null
+++ b/offload/tools/offload-tblgen/offload-tblgen.cpp
@@ -0,0 +1,101 @@
+//===- offload-tblgen/offload-tblgen.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a Tablegen tool that produces source files for the Offload project.
+// See offload/API/README.md for more information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/TableGen/Main.h"
+#include "llvm/TableGen/Record.h"
+
+#include "Generators.hpp"
+
+namespace llvm {
+namespace offload {
+namespace tblgen {
+
+enum ActionType {
+  PrintRecords,
+  DumpJSON,
+  GenAPI,
+  GenFuncNames,
+  GenImplFuncDecls,
+  GenEntryPoints,
+  GenPrintHeader,
+  GenExports
+};
+
+namespace {
+cl::opt<ActionType> Action(
+    cl::desc("Action to perform:"),
+    cl::values(
+        clEnumValN(PrintRecords, "print-records",
+                   "Print all records to stdout (default)"),
+        clEnumValN(DumpJSON, "dump-json",
+                   "Dump all records as machine-readable JSON"),
+        clEnumValN(GenAPI, "gen-api", "Generate Offload API header contents"),
+        clEnumValN(GenFuncNames, "gen-func-names",
+                   "Generate a list of all Offload API function names"),
+        clEnumValN(
+            GenImplFuncDecls, "gen-impl-func-decls",
+            "Generate declarations for Offload API implementation functions"),
+        clEnumValN(GenEntryPoints, "gen-entry-points",
+                   "Generate Offload API wrapper function definitions"),
+        clEnumValN(GenPrintHeader, "gen-print-header",
+                   "Generate Offload API print header"),
+        clEnumValN(GenExports, "gen-exports",
+                   "Generate export file for the Offload library")));
+}
+
+static bool OffloadTableGenMain(raw_ostream &OS, const RecordKeeper &Records) {
+  switch (Action) {
+  case PrintRecords:
+    OS << Records;
+    break;
+  case DumpJSON:
+    EmitJSON(Records, OS);
+    break;
+  case GenAPI:
+    EmitOffloadAPI(Records, OS);
+    break;
+  case GenFuncNames:
+    EmitOffloadFuncNames(Records, OS);
+    break;
+  case GenImplFuncDecls:
+    EmitOffloadImplFuncDecls(Records, OS);
+    break;
+  case GenEntryPoints:
+    EmitOffloadEntryPoints(Records, OS);
+    break;
+  case GenPrintHeader:
+    EmitOffloadPrintHeader(Records, OS);
+    break;
+  case GenExports:
+    EmitOffloadExports(Records, OS);
+    break;
+  }
+
+  return false;
+}
+
+int OffloadTblgenMain(int argc, char **argv) {
+  InitLLVM y(argc, argv);
+  cl::ParseCommandLineOptions(argc, argv);
+  return TableGenMain(argv[0], &OffloadTableGenMain);
+}
+} // namespace tblgen
+} // namespace offload
+} // namespace llvm
+
+using namespace llvm;
+using namespace offload::tblgen;
+
+int main(int argc, char **argv) { return OffloadTblgenMain(argc, argv); }
diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index 73c87b708d25f..25ac4b2fa3675 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -5,4 +5,5 @@ function(add_libompt_unittest test_dirname)
   add_unittest(LibomptUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
-add_subdirectory(Plugins)
+# add_subdirectory(Plugins)
+add_subdirectory(OffloadAPI)
diff --git a/offload/unittests/OffloadAPI/CMakeLists.txt b/offload/unittests/OffloadAPI/CMakeLists.txt
new file mode 100644
index 0000000000000..033ee2b6ec746
--- /dev/null
+++ b/offload/unittests/OffloadAPI/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(PLUGINS_TEST_COMMON LLVMOffload)
+set(PLUGINS_TEST_INCLUDE ${LIBOMPTARGET_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/common)
+
+add_libompt_unittest("offload.unittests"
+    ${CMAKE_CURRENT_SOURCE_DIR}/common/Environment.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatform.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatformCount.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatformInfo.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatformInfoSize.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDevice.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDeviceCount.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDeviceInfo.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDeviceInfoSize.cpp)
+add_dependencies("offload.unittests" ${PLUGINS_TEST_COMMON})
+target_link_libraries("offload.unittests" PRIVATE ${PLUGINS_TEST_COMMON})
+target_include_directories("offload.unittests" PRIVATE ${PLUGINS_TEST_INCLUDE})
diff --git a/offload/unittests/OffloadAPI/common/Environment.cpp b/offload/unittests/OffloadAPI/common/Environment.cpp
new file mode 100644
index 0000000000000..f07a66cda2189
--- /dev/null
+++ b/offload/unittests/OffloadAPI/common/Environment.cpp
@@ -0,0 +1,96 @@
+//===------- Offload API tests - gtest environment ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Environment.hpp"
+#include "Fixtures.hpp"
+#include "llvm/Support/CommandLine.h"
+#include <OffloadAPI.h>
+
+using namespace llvm;
+
+// Wrapper so we don't have to constantly init and shutdown Offload in every
+// test, while having sensible lifetime for the platform environment
+struct OffloadInitWrapper {
+  OffloadInitWrapper() { olInit(); }
+  ~OffloadInitWrapper() { olShutDown(); }
+};
+static OffloadInitWrapper Wrapper{};
+
+static cl::opt<std::string>
+    SelectedPlatform("platform", cl::desc("Only test the specified platform"),
+                     cl::value_desc("platform"));
+
+std::ostream &operator<<(std::ostream &Out,
+                         const ol_platform_handle_t &Platform) {
+  size_t Size;
+  olGetPlatformInfoSize(Platform, OL_PLATFORM_INFO_NAME, &Size);
+  std::vector<char> Name(Size);
+  olGetPlatformInfo(Platform, OL_PLATFORM_INFO_NAME, Size, Name.data());
+  Out << Name.data();
+  return Out;
+}
+
+std::ostream &operator<<(std::ostream &Out,
+                         const std::vector<ol_platform_handle_t> &Platforms) {
+  for (auto Platform : Platforms) {
+    Out << "\n  * \"" << Platform << "\"";
+  }
+  return Out;
+}
+
+const std::vector<ol_platform_handle_t> &TestEnvironment::getPlatforms() {
+  static std::vector<ol_platform_handle_t> Platforms{};
+
+  if (Platforms.empty()) {
+    uint32_t PlatformCount = 0;
+    olGetPlatformCount(&PlatformCount);
+    if (PlatformCount > 0) {
+      Platforms.resize(PlatformCount);
+      olGetPlatform(PlatformCount, Platforms.data());
+    }
+  }
+
+  return Platforms;
+}
+
+// Get a single platform, which may be selected by the user.
+ol_platform_handle_t TestEnvironment::getPlatform() {
+  static ol_platform_handle_t Platform = nullptr;
+  const auto &Platforms = getPlatforms();
+
+  if (!Platform) {
+    if (SelectedPlatform != "") {
+      for (const auto CandidatePlatform : Platforms) {
+        std::stringstream PlatformName;
+        PlatformName << CandidatePlatform;
+        if (SelectedPlatform == PlatformName.str()) {
+          Platform = CandidatePlatform;
+          return Platform;
+        }
+      }
+      std::cout << "No platform found with the name \"" << SelectedPlatform
+                << "\". Choose from:" << Platforms << "\n";
+      std::exit(1);
+    } else {
+      // Pick a single platform. We prefer one that has available devices, but
+      // just pick the first initially in case none have any devices.
+      Platform = Platforms[0];
+      for (auto CandidatePlatform : Platforms) {
+        uint32_t NumDevices = 0;
+        if (olGetDeviceCount(CandidatePlatform, &NumDevices) == OL_SUCCESS) {
+          if (NumDevices > 0) {
+            Platform = CandidatePlatform;
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  return Platform;
+}
diff --git a/offload/unittests/OffloadAPI/common/Environment.hpp b/offload/unittests/OffloadAPI/common/Environment.hpp
new file mode 100644
index 0000000000000..6dba2381eb0b7
--- /dev/null
+++ b/offload/unittests/OffloadAPI/common/Environment.hpp
@@ -0,0 +1,17 @@
+//===------- Offload API tests - gtest environment ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+namespace TestEnvironment {
+const std::vector<ol_platform_handle_t> &getPlatforms();
+ol_platform_handle_t getPlatform();
+} // namespace TestEnvironment
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
new file mode 100644
index 0000000000000..410a435dee1b5
--- /dev/null
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -0,0 +1,64 @@
+//===------- Offload API tests - gtest fixtures --==-----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <OffloadAPI.h>
+#include <OffloadPrint.hpp>
+#include <gtest/gtest.h>
+
+#include "Environment.hpp"
+
+#pragma once
+
+#ifndef ASSERT_SUCCESS
+#define ASSERT_SUCCESS(ACTUAL) ASSERT_EQ(OL_SUCCESS, ACTUAL)
+#endif
+
+// TODO: rework this so the EXPECTED/ACTUAL results are readable
+#ifndef ASSERT_ERROR
+#define ASSERT_ERROR(EXPECTED, ACTUAL)                                         \
+  do {                                                                         \
+    ol_result_t Res = ACTUAL;                                                  \
+    ASSERT_TRUE(Res && (Res->Code == EXPECTED));                               \
+  } while (0)
+#endif
+
+#define RETURN_ON_FATAL_FAILURE(...)                                           \
+  __VA_ARGS__;                                                                 \
+  if (this->HasFatalFailure() || this->IsSkipped()) {                          \
+    return;                                                                    \
+  }                                                                            \
+  (void)0
+
+struct offloadTest : ::testing::Test {
+  // No special behavior now, but just in case we need to override it in future
+};
+
+struct offloadPlatformTest : offloadTest {
+  void SetUp() override {
+    RETURN_ON_FATAL_FAILURE(offloadTest::SetUp());
+
+    Platform = TestEnvironment::getPlatform();
+    ASSERT_NE(Platform, nullptr);
+  }
+
+  ol_platform_handle_t Platform;
+};
+
+struct offloadDeviceTest : offloadPlatformTest {
+  void SetUp() override {
+    RETURN_ON_FATAL_FAILURE(offloadPlatformTest::SetUp());
+
+    uint32_t NumDevices;
+    ASSERT_SUCCESS(olGetDeviceCount(Platform, &NumDevices));
+    if (NumDevices == 0)
+      GTEST_SKIP() << "No available devices on this platform.";
+    ASSERT_SUCCESS(olGetDevice(Platform, 1, &Device));
+  }
+
+  ol_device_handle_t Device;
+};
diff --git a/offload/unittests/OffloadAPI/device/olDeviceInfo.hpp b/offload/unittests/OffloadAPI/device/olDeviceInfo.hpp
new file mode 100644
index 0000000000000..06915258da384
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device/olDeviceInfo.hpp
@@ -0,0 +1,21 @@
+//===------- Offload API tests - Helpers for device info query testing ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include <unordered_map>
+#include <vector>
+
+// TODO: We could autogenerate these
+inline std::vector<ol_device_info_t> DeviceQueries = {
+    OL_DEVICE_INFO_TYPE, OL_DEVICE_INFO_PLATFORM, OL_DEVICE_INFO_NAME,
+    OL_DEVICE_INFO_VENDOR, OL_DEVICE_INFO_DRIVER_VERSION};
+
+inline std::unordered_map<ol_device_info_t, size_t> DeviceInfoSizeMap = {
+    {OL_DEVICE_INFO_TYPE, sizeof(ol_device_type_t)},
+    {OL_DEVICE_INFO_PLATFORM, sizeof(ol_platform_handle_t)},
+};
diff --git a/offload/unittests/OffloadAPI/device/olGetDevice.cpp b/offload/unittests/OffloadAPI/device/olGetDevice.cpp
new file mode 100644
index 0000000000000..68d4682dd3351
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device/olGetDevice.cpp
@@ -0,0 +1,39 @@
+//===------- Offload API tests - olGetDevice -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olGetDeviceTest = offloadPlatformTest;
+
+TEST_F(olGetDeviceTest, Success) {
+  uint32_t Count = 0;
+  ASSERT_SUCCESS(olGetDeviceCount(Platform, &Count));
+  if (Count == 0)
+    GTEST_SKIP() << "No available devices on this platform.";
+
+  std::vector<ol_device_handle_t> Devices(Count);
+  ASSERT_SUCCESS(olGetDevice(Platform, Count, Devices.data()));
+  for (auto Device : Devices) {
+    ASSERT_NE(nullptr, Device);
+  }
+}
+
+TEST_F(olGetDeviceTest, SuccessSubsetOfDevices) {
+  uint32_t Count;
+  ASSERT_SUCCESS(olGetDeviceCount(Platform, &Count));
+  if (Count < 2)
+    GTEST_SKIP() << "Only one device is available on this platform.";
+
+  std::vector<ol_device_handle_t> Devices(Count - 1);
+  ASSERT_SUCCESS(olGetDevice(Platform, Count - 1, Devices.data()));
+  for (auto Device : Devices) {
+    ASSERT_NE(nullptr, Device);
+  }
+}
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceCount.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceCount.cpp
new file mode 100644
index 0000000000000..ef377d671bf60
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceCount.cpp
@@ -0,0 +1,28 @@
+//===------- Offload API tests - olGetDeviceCount --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olGetDeviceCountTest = offloadPlatformTest;
+
+TEST_F(olGetDeviceCountTest, Success) {
+  uint32_t Count = 0;
+  ASSERT_SUCCESS(olGetDeviceCount(Platform, &Count));
+}
+
+TEST_F(olGetDeviceCountTest, InvalidNullPlatform) {
+  uint32_t Count = 0;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE, olGetDeviceCount(nullptr, &Count));
+}
+
+TEST_F(olGetDeviceCountTest, InvalidNullPointer) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olGetDeviceCount(Platform, nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
new file mode 100644
index 0000000000000..c936802fb1e4d
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
@@ -0,0 +1,76 @@
+//===------- Offload API tests - olGetDeviceInfo ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include "olDeviceInfo.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+struct olGetDeviceInfoTest : offloadDeviceTest,
+                             ::testing::WithParamInterface<ol_device_info_t> {
+
+  void SetUp() override { RETURN_ON_FATAL_FAILURE(offloadDeviceTest::SetUp()); }
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    , olGetDeviceInfoTest, ::testing::ValuesIn(DeviceQueries),
+    [](const ::testing::TestParamInfo<ol_device_info_t> &info) {
+      std::stringstream ss;
+      ss << info.param;
+      return ss.str();
+    });
+
+TEST_P(olGetDeviceInfoTest, Success) {
+  ol_device_info_t InfoType = GetParam();
+  size_t Size = 0;
+
+  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, InfoType, &Size));
+
+  std::vector<char> InfoData(Size);
+  ASSERT_SUCCESS(olGetDeviceInfo(Device, InfoType, Size, InfoData.data()));
+
+  if (InfoType == OL_DEVICE_INFO_PLATFORM) {
+    auto *ReturnedPlatform =
+        reinterpret_cast<ol_platform_handle_t *>(InfoData.data());
+    ASSERT_EQ(Platform, *ReturnedPlatform);
+  }
+}
+
+TEST_F(olGetDeviceInfoTest, InvalidNullHandleDevice) {
+  ol_device_type_t DeviceType;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olGetDeviceInfo(nullptr, OL_DEVICE_INFO_TYPE,
+                               sizeof(ol_device_type_t), &DeviceType));
+}
+
+TEST_F(olGetDeviceInfoTest, InvalidEnumerationInfoType) {
+  ol_device_type_t DeviceType;
+  ASSERT_ERROR(OL_ERRC_INVALID_ENUMERATION,
+               olGetDeviceInfo(Device, OL_DEVICE_INFO_FORCE_UINT32,
+                               sizeof(ol_device_type_t), &DeviceType));
+}
+
+TEST_F(olGetDeviceInfoTest, InvalidSizePropSize) {
+  ol_device_type_t DeviceType;
+  ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+               olGetDeviceInfo(Device, OL_DEVICE_INFO_TYPE, 0, &DeviceType));
+}
+
+TEST_F(olGetDeviceInfoTest, InvalidSizePropSizeSmall) {
+  ol_device_type_t DeviceType;
+  ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+               olGetDeviceInfo(Device, OL_DEVICE_INFO_TYPE,
+                               sizeof(DeviceType) - 1, &DeviceType));
+}
+
+TEST_F(olGetDeviceInfoTest, InvalidNullPointerPropValue) {
+  ol_device_type_t DeviceType;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olGetDeviceInfo(Device, OL_DEVICE_INFO_TYPE, sizeof(DeviceType),
+                               nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
new file mode 100644
index 0000000000000..9e792d1c3e25e
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
@@ -0,0 +1,58 @@
+//===------- Offload API tests - olGetDeviceInfoSize -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <OffloadAPI.h>
+
+#include "../common/Fixtures.hpp"
+#include "olDeviceInfo.hpp"
+
+struct olGetDeviceInfoSizeTest
+    : offloadDeviceTest,
+      ::testing::WithParamInterface<ol_device_info_t> {
+
+  void SetUp() override { RETURN_ON_FATAL_FAILURE(offloadDeviceTest::SetUp()); }
+};
+
+// TODO: We could autogenerate the list of enum values
+INSTANTIATE_TEST_SUITE_P(
+    , olGetDeviceInfoSizeTest, ::testing::ValuesIn(DeviceQueries),
+    [](const ::testing::TestParamInfo<ol_device_info_t> &info) {
+      std::stringstream ss;
+      ss << info.param;
+      return ss.str();
+    });
+
+TEST_P(olGetDeviceInfoSizeTest, Success) {
+  ol_device_info_t InfoType = GetParam();
+  size_t Size = 0;
+
+  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, InfoType, &Size));
+  auto ExpectedSize = DeviceInfoSizeMap.find(InfoType);
+  if (ExpectedSize != DeviceInfoSizeMap.end()) {
+    ASSERT_EQ(Size, ExpectedSize->second);
+  } else {
+    ASSERT_NE(Size, 0lu);
+  }
+}
+
+TEST_F(olGetDeviceInfoSizeTest, InvalidNullHandle) {
+  size_t Size = 0;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olGetDeviceInfoSize(nullptr, OL_DEVICE_INFO_TYPE, &Size));
+}
+
+TEST_F(olGetDeviceInfoSizeTest, InvalidDeviceInfoEnumeration) {
+  size_t Size = 0;
+  ASSERT_ERROR(OL_ERRC_INVALID_ENUMERATION,
+               olGetDeviceInfoSize(Device, OL_DEVICE_INFO_FORCE_UINT32, &Size));
+}
+
+TEST_F(olGetDeviceInfoSizeTest, InvalidNullPointer) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olGetDeviceInfoSize(Device, OL_DEVICE_INFO_TYPE, nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/platform/olGetPlatform.cpp b/offload/unittests/OffloadAPI/platform/olGetPlatform.cpp
new file mode 100644
index 0000000000000..4a2f9e8ac7741
--- /dev/null
+++ b/offload/unittests/OffloadAPI/platform/olGetPlatform.cpp
@@ -0,0 +1,28 @@
+//===------- Offload API tests - olGetPlatform -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olGetPlatformTest = offloadTest;
+
+TEST_F(olGetPlatformTest, Success) {
+  uint32_t PlatformCount;
+  ASSERT_SUCCESS(olGetPlatformCount(&PlatformCount));
+  std::vector<ol_platform_handle_t> Platforms(PlatformCount);
+  ASSERT_SUCCESS(olGetPlatform(PlatformCount, Platforms.data()));
+}
+
+TEST_F(olGetPlatformTest, InvalidNumEntries) {
+  uint32_t PlatformCount;
+  ASSERT_SUCCESS(olGetPlatformCount(&PlatformCount));
+  std::vector<ol_platform_handle_t> Platforms(PlatformCount);
+  ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+               olGetPlatform(PlatformCount + 1, Platforms.data()));
+}
diff --git a/offload/unittests/OffloadAPI/platform/olGetPlatformCount.cpp b/offload/unittests/OffloadAPI/platform/olGetPlatformCount.cpp
new file mode 100644
index 0000000000000..15b4b6abcd70d
--- /dev/null
+++ b/offload/unittests/OffloadAPI/platform/olGetPlatformCount.cpp
@@ -0,0 +1,22 @@
+//===------- Offload API tests - olGetPlatformCount ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olGetPlatformCountTest = offloadTest;
+
+TEST_F(olGetPlatformCountTest, Success) {
+  uint32_t PlatformCount;
+  ASSERT_SUCCESS(olGetPlatformCount(&PlatformCount));
+}
+
+TEST_F(olGetPlatformCountTest, InvalidNullPointer) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER, olGetPlatformCount(nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/platform/olGetPlatformInfo.cpp b/offload/unittests/OffloadAPI/platform/olGetPlatformInfo.cpp
new file mode 100644
index 0000000000000..c646bdc50b7da
--- /dev/null
+++ b/offload/unittests/OffloadAPI/platform/olGetPlatformInfo.cpp
@@ -0,0 +1,76 @@
+//===------- Offload API tests - olGetPlatformInfo -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <OffloadAPI.h>
+
+#include "../common/Fixtures.hpp"
+#include "olPlatformInfo.hpp"
+
+struct olGetPlatformInfoTest
+    : offloadPlatformTest,
+      ::testing::WithParamInterface<ol_platform_info_t> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    olGetPlatformInfo, olGetPlatformInfoTest,
+    ::testing::ValuesIn(PlatformQueries),
+    [](const ::testing::TestParamInfo<ol_platform_info_t> &info) {
+      std::stringstream ss;
+      ss << info.param;
+      return ss.str();
+    });
+
+TEST_P(olGetPlatformInfoTest, Success) {
+  size_t Size = 0;
+  ol_platform_info_t InfoType = GetParam();
+
+  ASSERT_SUCCESS(olGetPlatformInfoSize(Platform, InfoType, &Size));
+  std::vector<char> InfoData(Size);
+  ASSERT_SUCCESS(olGetPlatformInfo(Platform, InfoType, Size, InfoData.data()));
+
+  // Info types with a dynamic size are all char[] so we can verify the returned
+  // string is the expected size.
+  auto ExpectedSize = PlatformInfoSizeMap.find(InfoType);
+  if (ExpectedSize == PlatformInfoSizeMap.end()) {
+    ASSERT_EQ(Size, strlen(InfoData.data()) + 1);
+  }
+}
+
+TEST_F(olGetPlatformInfoTest, InvalidNullHandle) {
+  ol_platform_backend_t Backend;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olGetPlatformInfo(nullptr, OL_PLATFORM_INFO_BACKEND,
+                                 sizeof(Backend), &Backend));
+}
+
+TEST_F(olGetPlatformInfoTest, InvalidPlatformInfoEnumeration) {
+  ol_platform_backend_t Backend;
+  ASSERT_ERROR(OL_ERRC_INVALID_ENUMERATION,
+               olGetPlatformInfo(Platform, OL_PLATFORM_INFO_FORCE_UINT32,
+                                 sizeof(Backend), &Backend));
+}
+
+TEST_F(olGetPlatformInfoTest, InvalidSizeZero) {
+  ol_platform_backend_t Backend;
+  ASSERT_ERROR(
+      OL_ERRC_INVALID_SIZE,
+      olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND, 0, &Backend));
+}
+
+TEST_F(olGetPlatformInfoTest, InvalidSizeSmall) {
+  ol_platform_backend_t Backend;
+  ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+               olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND,
+                                 sizeof(Backend) - 1, &Backend));
+}
+
+TEST_F(olGetPlatformInfoTest, InvalidNullPointerPropValue) {
+  ol_platform_backend_t Backend;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND,
+                                 sizeof(Backend), nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/platform/olGetPlatformInfoSize.cpp b/offload/unittests/OffloadAPI/platform/olGetPlatformInfoSize.cpp
new file mode 100644
index 0000000000000..7c9274082e8e4
--- /dev/null
+++ b/offload/unittests/OffloadAPI/platform/olGetPlatformInfoSize.cpp
@@ -0,0 +1,57 @@
+//===------- Offload API tests - olGetPlatformInfoSize ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <OffloadAPI.h>
+
+#include "../common/Fixtures.hpp"
+#include "olPlatformInfo.hpp"
+
+struct olGetPlatformInfoSizeTest
+    : offloadPlatformTest,
+      ::testing::WithParamInterface<ol_platform_info_t> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    olGetPlatformInfoSize, olGetPlatformInfoSizeTest,
+    ::testing::ValuesIn(PlatformQueries),
+    [](const ::testing::TestParamInfo<ol_platform_info_t> &info) {
+      std::stringstream ss;
+      ss << info.param;
+      return ss.str();
+    });
+
+TEST_P(olGetPlatformInfoSizeTest, Success) {
+  size_t Size = 0;
+  ol_platform_info_t InfoType = GetParam();
+
+  ASSERT_SUCCESS(olGetPlatformInfoSize(Platform, InfoType, &Size));
+  auto ExpectedSize = PlatformInfoSizeMap.find(InfoType);
+  if (ExpectedSize != PlatformInfoSizeMap.end()) {
+    ASSERT_EQ(Size, ExpectedSize->second);
+  } else {
+    ASSERT_NE(Size, 0lu);
+  }
+}
+
+TEST_F(olGetPlatformInfoSizeTest, InvalidNullHandle) {
+  size_t Size = 0;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olGetPlatformInfoSize(nullptr, OL_PLATFORM_INFO_BACKEND, &Size));
+}
+
+TEST_F(olGetPlatformInfoSizeTest, InvalidPlatformInfoEnumeration) {
+  size_t Size = 0;
+  ASSERT_ERROR(
+      OL_ERRC_INVALID_ENUMERATION,
+      olGetPlatformInfoSize(Platform, OL_PLATFORM_INFO_FORCE_UINT32, &Size));
+}
+
+TEST_F(olGetPlatformInfoSizeTest, InvalidNullPointer) {
+  ASSERT_ERROR(
+      OL_ERRC_INVALID_NULL_POINTER,
+      olGetPlatformInfoSize(Platform, OL_PLATFORM_INFO_BACKEND, nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/platform/olPlatformInfo.hpp b/offload/unittests/OffloadAPI/platform/olPlatformInfo.hpp
new file mode 100644
index 0000000000000..d49cdb90d321a
--- /dev/null
+++ b/offload/unittests/OffloadAPI/platform/olPlatformInfo.hpp
@@ -0,0 +1,20 @@
+//===------- Offload API tests - Helpers for platform info query testing --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include <vector>
+
+// TODO: We could autogenerate these
+
+inline std::vector<ol_platform_info_t> PlatformQueries = {
+    OL_PLATFORM_INFO_NAME, OL_PLATFORM_INFO_VENDOR_NAME,
+    OL_PLATFORM_INFO_VERSION, OL_PLATFORM_INFO_BACKEND};
+
+inline std::unordered_map<ol_platform_info_t, size_t> PlatformInfoSizeMap = {
+    {OL_PLATFORM_INFO_BACKEND, sizeof(ol_platform_backend_t)},
+};
diff --git a/openmp/Maintainers.md b/openmp/Maintainers.md
new file mode 100644
index 0000000000000..1892577374e6f
--- /dev/null
+++ b/openmp/Maintainers.md
@@ -0,0 +1,13 @@
+# LLVM OpenMP Library Maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+the LLVM OpenMP library.
+
+# Current Maintainers
+
+Michael Klemm \
+michael.klemm@amd.com (email), [mjklemm](https://github.com/mjklemm) (GitHub)
+
+Terry Wilmarth \
+terry.l.wilmarth@intel.com (email), [TerryLWilmarth](https://github.com/TerryLWilmarth) (GitHub)
diff --git a/openmp/runtime/test/ompt/misc/interoperability.cpp b/openmp/runtime/test/ompt/misc/interoperability.cpp
index 6f17a94c1f404..d9c11461220df 100644
--- a/openmp/runtime/test/ompt/misc/interoperability.cpp
+++ b/openmp/runtime/test/ompt/misc/interoperability.cpp
@@ -3,7 +3,7 @@
 
 #include <iostream>
 #include <thread>
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
 #include <alloca.h>
 #else
 #include <cstdlib>
diff --git a/polly/Maintainers.md b/polly/Maintainers.md
new file mode 100644
index 0000000000000..63d662325e839
--- /dev/null
+++ b/polly/Maintainers.md
@@ -0,0 +1,10 @@
+# Polly Maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+Polly.
+
+# Lead maintainer
+
+Michael Kruse \
+llvm-project@meinersbur.de (email), [Meinersbur](https://github.com/Meinersbur) (GitHub)
\ No newline at end of file
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index 8813cfd959ef6..f5e29c38e2903 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -41,6 +41,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
 #include "isl/ast.h"
 #include <cassert>
 
@@ -233,6 +234,15 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
   NodeBuilder.allocateNewArrays(StartExitBlocks);
   Annotator.buildAliasScopes(S);
 
+  // The code below annotates the "llvm.loop.vectorize.enable" to false
+  // for the code flow taken when RTCs fail. Because we don't want the
+  // Loop Vectorizer to come in later and vectorize the original fall back
+  // loop when Polly is enabled.
+  for (Loop *L : LI.getLoopsInPreorder()) {
+    if (S.contains(L))
+      addStringMetadataToLoop(L, "llvm.loop.vectorize.enable", 0);
+  }
+
   if (PerfMonitoring) {
     PerfMonitor P(S, EnteringBB->getParent()->getParent());
     P.initialize();
diff --git a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
index 96b50cef179a4..442600cff7a0a 100644
--- a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
+++ b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
@@ -36,7 +36,6 @@ ret:
 ; SEQUENTIAL-LABEL: @test-one
 ; SEQUENTIAL-NOT: !llvm.mem.parallel_loop_access
 ; SEQUENTIAL-NOT: !llvm.access.group
-; SEQUENTIAL-NOT: !llvm.loop
 
 ; PARALLEL: @test-one
 ; PARALLEL: store i32 1, ptr %scevgep1, {{[ ._!,a-zA-Z0-9]*}}, !llvm.access.group ![[GROUPID3:[0-9]+]]
@@ -81,14 +80,12 @@ ret:
 ; SEQUENTIAL-LABEL: @test-two
 ; SEQUENTIAL-NOT: !llvm.mem.parallel_loop_access
 ; SEQUENTIAL-NOT: !llvm.access.group
-; SEQUENTIAL-NOT: !llvm.loop
 
 ; PARALLEL: @test-two
 ; PARALLEL: %val_p_scalar_ = load i32, ptr %scevgep, {{[ ._!,a-zA-Z0-9]*}}, !llvm.access.group ![[GROUPID8:[0-9]*]]
 ; PARALLEL: store i32 %val_p_scalar_, ptr %scevgep1, {{[ ._!,a-zA-Z0-9]*}}, !llvm.access.group ![[GROUPID8]]
 ; PARALLEL:  br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit, !llvm.loop ![[LoopID9:[0-9]*]]
 
-
 ; PARALLEL: ![[LoopID4]] = distinct !{![[LoopID4]], ![[PARACC5:[0-9]+]]}
 ; PARALLEL: ![[PARACC5]] = !{!"llvm.loop.parallel_accesses", ![[GROUPID3]]}
 ; PARALLEL: ![[LoopID9]] = distinct !{![[LoopID9]], ![[PARACC10:[0-9]+]]}
diff --git a/polly/test/CodeGen/MemAccess/codegen_address_space.ll b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
index 3ce363e8f09fc..3360e10529f8e 100644
--- a/polly/test/CodeGen/MemAccess/codegen_address_space.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
@@ -40,4 +40,4 @@ for.end:                                          ; preds = %for.cond
 }
 
 ; CHECK: %polly.access.A = getelementptr i32, ptr addrspace(5) %A, i64 0
-; CHECK: %tmp2_p_scalar_ = load i32, ptr addrspace(5) %polly.access.A, align 4, !alias.scope !0, !noalias !3
+; CHECK: %tmp2_p_scalar_ = load i32, ptr addrspace(5) %polly.access.A, align 4, !alias.scope !2, !noalias !5
diff --git a/polly/test/CodeGen/MemAccess/create_arrays.ll b/polly/test/CodeGen/MemAccess/create_arrays.ll
index b454e028b982c..40ae8d6efa95f 100644
--- a/polly/test/CodeGen/MemAccess/create_arrays.ll
+++ b/polly/test/CodeGen/MemAccess/create_arrays.ll
@@ -28,19 +28,23 @@
 ; CODEGEN:  %beta.s2a.reload = load double, ptr %beta.s2a
 ; CODEGEN:  %polly.access.mul.E = mul nsw i64 %polly.indvar31, 200000
 ; CODEGEN:  %polly.access.add.E = add nsw i64 %polly.access.mul.E, %polly.indvar
-; CODEGEN:  {{%.*}} = load double, ptr %polly.access.E, align 8, !alias.scope !0, !noalias !3
-; CODEGEN:  store double {{%.*}}, ptr %scevgep34, align 8, !alias.scope !8, !noalias !9
+; CODEGEN:  {{%.*}} = load double, ptr %polly.access.E, align 8, !alias.scope !4, !noalias !7
+; CODEGEN:  store double {{%.*}}, ptr %scevgep34, align 8, !alias.scope !12, !noalias !13
 ;
-; CODEGEN: !0 = !{!1}
-; CODEGEN: !1 = distinct !{!1, !2, !"polly.alias.scope.E"}
-; CODEGEN: !2 = distinct !{!2, !"polly.alias.scope.domain"}
-; CODEGEN: !3 = !{!4, !5, !6, !7}
-; CODEGEN: !4 = distinct !{!4, !2, !"polly.alias.scope.MemRef_B"}
-; CODEGEN: !5 = distinct !{!5, !2, !"polly.alias.scope.MemRef_A"}
-; CODEGEN: !6 = distinct !{!6, !2, !"polly.alias.scope.D"}
-; CODEGEN: !7 = distinct !{!7, !2, !"polly.alias.scope.F"}
-; CODEGEN: !8 = !{!5}
-; CODEGEN: !9 = !{!4, !6, !1, !7}
+; CODEGEN: !0 = distinct !{!0, !1}
+; CODEGEN: !1 = !{!"llvm.loop.vectorize.enable", i32 0}
+; CODEGEN: !2 = distinct !{!2, !1}
+; CODEGEN: !3 = distinct !{!3, !1}
+; CODEGEN: !4 = !{!5}
+; CODEGEN: !5 = distinct !{!5, !6, !"polly.alias.scope.E"}
+; CODEGEN: !6 = distinct !{!6, !"polly.alias.scope.domain"}
+; CODEGEN: !7 = !{!8, !9, !10, !11}
+; CODEGEN: !8 = distinct !{!8, !6, !"polly.alias.scope.MemRef_B"}
+; CODEGEN: !9 = distinct !{!9, !6, !"polly.alias.scope.MemRef_A"}
+; CODEGEN: !10 = distinct !{!10, !6, !"polly.alias.scope.D"}
+; CODEGEN: !11 = distinct !{!11, !6, !"polly.alias.scope.F"}
+; CODEGEN: !12 = !{!9}
+; CODEGEN: !13 = !{!8, !10, !5, !11}
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-unknown"
diff --git a/polly/test/CodeGen/MemAccess/different_types.ll b/polly/test/CodeGen/MemAccess/different_types.ll
index 53718194c25a4..407e72702aa86 100644
--- a/polly/test/CodeGen/MemAccess/different_types.ll
+++ b/polly/test/CodeGen/MemAccess/different_types.ll
@@ -12,7 +12,7 @@
 ; CHECK: %[[R1:[._0-9]*]] = sub nsw i64 0, %polly.indvar7
 ; CHECK: %[[R2:[._0-9]*]] = add nsw i64 %[[R1]], 99
 ; CHECK: %polly.access.A10 = getelementptr i32, ptr %A, i64 %[[R2]]
-; CHECK: %tmp14_p_scalar_ = load float, ptr %polly.access.A10, align 4, !alias.scope !3, !noalias !0
+; CHECK: %tmp14_p_scalar_ = load float, ptr %polly.access.A10, align 4, !alias.scope !6, !noalias !3
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/MemAccess/generate-all.ll b/polly/test/CodeGen/MemAccess/generate-all.ll
index d1f695d436dab..7b2286bfc95a9 100644
--- a/polly/test/CodeGen/MemAccess/generate-all.ll
+++ b/polly/test/CodeGen/MemAccess/generate-all.ll
@@ -13,9 +13,9 @@
 ; SCEV-NEXT:   %1 = zext i2 %0 to i64
 ; SCEV-NEXT:   %2 = shl nuw nsw i64 %1, 2
 ; SCEV-NEXT:   %scevgep = getelementptr i8, ptr %A, i64 %2
-; SCEV-NEXT:   %tmp4_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
+; SCEV-NEXT:   %tmp4_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !2, !noalias !5
 ; SCEV-NEXT:   %p_tmp5 = fadd float %tmp4_p_scalar_, 1.000000e+01
-; SCEV-NEXT:   store float %p_tmp5, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
+; SCEV-NEXT:   store float %p_tmp5, ptr %scevgep, align 4, !alias.scope !2, !noalias !5
 ; SCEV-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; SCEV-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 99
 ; SCEV-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
@@ -23,11 +23,11 @@
 ; ASTEXPR: 	polly.stmt.bb2:                                   ; preds = %polly.loop_header
 ; ASTEXPR-NEXT:   %pexp.pdiv_r = urem i64 %polly.indvar, 4
 ; ASTEXPR-NEXT:   %polly.access.A = getelementptr float, ptr %A, i64 %pexp.pdiv_r
-; ASTEXPR-NEXT:   %tmp4_p_scalar_ = load float, ptr %polly.access.A, align 4, !alias.scope !0, !noalias !3
+; ASTEXPR-NEXT:   %tmp4_p_scalar_ = load float, ptr %polly.access.A, align 4, !alias.scope !2, !noalias !5
 ; ASTEXPR-NEXT:   %p_tmp5 = fadd float %tmp4_p_scalar_, 1.000000e+01
 ; ASTEXPR-NEXT:   %pexp.pdiv_r1 = urem i64 %polly.indvar, 4
 ; ASTEXPR-NEXT:   %polly.access.A2 = getelementptr float, ptr %A, i64 %pexp.pdiv_r1
-; ASTEXPR-NEXT:   store float %p_tmp5, ptr %polly.access.A2, align 4, !alias.scope !0, !noalias !3
+; ASTEXPR-NEXT:   store float %p_tmp5, ptr %polly.access.A2, align 4, !alias.scope !2, !noalias !5
 ; ASTEXPR-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; ASTEXPR-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 99
 ; ASTEXPR-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
diff --git a/polly/test/CodeGen/OpenMP/alias-metadata.ll b/polly/test/CodeGen/OpenMP/alias-metadata.ll
index b80b18f433269..121f630789892 100644
--- a/polly/test/CodeGen/OpenMP/alias-metadata.ll
+++ b/polly/test/CodeGen/OpenMP/alias-metadata.ll
@@ -32,8 +32,8 @@ bb5:                                              ; preds = %bb4
   %tmp7 = getelementptr inbounds float, ptr %A, i64 %i.0
   %tmp6 = load float, ptr %tmp, align 4
   store float %tmp6, ptr %tmp7, align 4
-; CHECK: %tmp6_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
-; CHECK: store float %tmp6_p_scalar_, ptr %scevgep7, align 4, !alias.scope !3, !noalias !0
+; CHECK: %tmp6_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !3, !noalias !6
+; CHECK: store float %tmp6_p_scalar_, ptr %scevgep7, align 4, !alias.scope !6, !noalias !3
   br label %bb8
 
 bb8:                                              ; preds = %bb5
diff --git a/polly/test/CodeGen/OpenMP/new_multidim_access.ll b/polly/test/CodeGen/OpenMP/new_multidim_access.ll
index 1b2912ba9ef46..5faabb4d20c1a 100644
--- a/polly/test/CodeGen/OpenMP/new_multidim_access.ll
+++ b/polly/test/CodeGen/OpenMP/new_multidim_access.ll
@@ -23,13 +23,13 @@
 ; IR: %6 = add nsw i64 %polly.indvar4, 13
 ; IR: %polly.access.add.polly.subfunc.arg.A = add nsw i64 %polly.access.mul.polly.subfunc.arg.A, %6
 ; IR: %polly.access.polly.subfunc.arg.A = getelementptr float, ptr %polly.subfunc.arg.A, i64 %polly.access.add.polly.subfunc.arg.A
-; IR: %tmp10_p_scalar_ = load float, ptr %polly.access.polly.subfunc.arg.A, align 4, !alias.scope !0, !noalias !3, !llvm.access.group !4
+; IR: %tmp10_p_scalar_ = load float, ptr %polly.access.polly.subfunc.arg.A, align 4, !alias.scope !3, !noalias !6, !llvm.access.group !7
 
 ; IR: %polly.access.mul.polly.subfunc.arg.A7 = mul nsw i64 %polly.indvar, %polly.subfunc.arg.m
 ; IR: %7 = add nsw i64 %polly.indvar4, 43
 ; IR: %polly.access.add.polly.subfunc.arg.A8 = add nsw i64 %polly.access.mul.polly.subfunc.arg.A7, %7
 ; IR: %polly.access.polly.subfunc.arg.A9 = getelementptr float, ptr %polly.subfunc.arg.A, i64 %polly.access.add.polly.subfunc.arg.A8
-; IR: store float %p_tmp11, ptr %polly.access.polly.subfunc.arg.A9, align 4, !alias.scope !0, !noalias !3, !llvm.access.group !4
+; IR: store float %p_tmp11, ptr %polly.access.polly.subfunc.arg.A9, align 4, !alias.scope !3, !noalias !6, !llvm.access.group !7
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @new_multidim_access(i64 %n, i64 %m, ptr %A) {
diff --git a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
index 76b2fa9a35b2a..ccb0d15cfc3d2 100644
--- a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
+++ b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
@@ -53,12 +53,12 @@
 
 ; CHECK: %0 = shl nuw nsw i64 %polly.indvar, 3
 ; CHECK: %scevgep = getelementptr i8, ptr %B, i64 %0
-; CHECK: %tmp3_p_scalar_ = load double, ptr %scevgep, align 8, !alias.scope !0, !noalias !3
+; CHECK: %tmp3_p_scalar_ = load double, ptr %scevgep, align 8, !alias.scope !2, !noalias !5
 ; CHECK: %1 = ptrtoint ptr %scevgep to i64
 ; CHECK: %2 = call i32 (...) @printf(ptr @3, ptr addrspace(4) @0, i64 %1, ptr addrspace(4) @1, double %tmp3_p_scalar_, ptr addrspace(4) @2)
 ; CHECK: %3 = call i32 @fflush(ptr null)
 ; CHECK: %scevgep1 = getelementptr i8, ptr %C, i64 %polly.indvar
-; CHECK: %tmp5_p_scalar_ = load i8, ptr %scevgep1, align 1, !alias.scope !8, !noalias !9
+; CHECK: %tmp5_p_scalar_ = load i8, ptr %scevgep1, align 1, !alias.scope !10, !noalias !11
 ; CHECK: %4 = ptrtoint ptr %scevgep1 to i64
 ; CHECK: %5 = sext i8 %tmp5_p_scalar_ to i64
 ; CHECK: %6 = call i32 (...) @printf(ptr @7, ptr addrspace(4) @4, i64 %4, ptr addrspace(4) @5, i64 %5, ptr addrspace(4) @6)
@@ -67,7 +67,7 @@
 ; CHECK: %p_tmp7 = fadd double %tmp3_p_scalar_, %p_tmp6
 ; CHECK: %8 = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK: %scevgep2 = getelementptr i8, ptr %D, i64 %8
-; CHECK: %tmp9_p_scalar_ = load i32, ptr %scevgep2, align 4, !alias.scope !10, !noalias !11
+; CHECK: %tmp9_p_scalar_ = load i32, ptr %scevgep2, align 4, !alias.scope !12, !noalias !13
 ; CHECK: %9 = ptrtoint ptr %scevgep2 to i64
 ; CHECK: %10 = sext i32 %tmp9_p_scalar_ to i64
 ; CHECK: %11 = call i32 (...) @printf(ptr @11, ptr addrspace(4) @8, i64 %9, ptr addrspace(4) @9, i64 %10, ptr addrspace(4) @10)
@@ -76,7 +76,7 @@
 ; CHECK: %p_tmp11 = fadd double %p_tmp7, %p_tmp10
 ; CHECK: %13 = shl nuw nsw i64 %polly.indvar, 3
 ; CHECK: %scevgep3 = getelementptr i8, ptr %E, i64 %13
-; CHECK: %tmp13_p_scalar_ = load i64, ptr %scevgep3, align 8, !alias.scope !12, !noalias !13
+; CHECK: %tmp13_p_scalar_ = load i64, ptr %scevgep3, align 8, !alias.scope !14, !noalias !15
 ; CHECK: %14 = ptrtoint ptr %scevgep3 to i64
 ; CHECK: %15 = call i32 (...) @printf(ptr @15, ptr addrspace(4) @12, i64 %14, ptr addrspace(4) @13, i64 %tmp13_p_scalar_, ptr addrspace(4) @14)
 ; CHECK: %16 = call i32 @fflush(ptr null)
@@ -84,7 +84,7 @@
 ; CHECK: %p_tmp15 = fadd double %p_tmp11, %p_tmp14
 ; CHECK: %17 = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK: %scevgep4 = getelementptr i8, ptr %A, i64 %17
-; CHECK: %tmp17_p_scalar_ = load float, ptr %scevgep4, align 4, !alias.scope !14, !noalias !15
+; CHECK: %tmp17_p_scalar_ = load float, ptr %scevgep4, align 4, !alias.scope !16, !noalias !17
 ; CHECK: %18 = ptrtoint ptr %scevgep4 to i64
 ; CHECK: %19 = fpext float %tmp17_p_scalar_ to double
 ; CHECK: %20 = call i32 (...) @printf(ptr @19, ptr addrspace(4) @16, i64 %18, ptr addrspace(4) @17, double %19, ptr addrspace(4) @18)
diff --git a/polly/test/CodeGen/annotated_alias_scopes.ll b/polly/test/CodeGen/annotated_alias_scopes.ll
index b1777a1b5f5d5..ada03e0663722 100644
--- a/polly/test/CodeGen/annotated_alias_scopes.ll
+++ b/polly/test/CodeGen/annotated_alias_scopes.ll
@@ -4,22 +4,24 @@
 ;
 ; SCOPES-LABEL: polly.stmt.for.body:
 ; SCOPES:      %[[BIdx:[._a-zA-Z0-9]*]] = getelementptr{{.*}} ptr %B, i64 %{{.*}}
-; SCOPES:      load i32, ptr %[[BIdx]], align 4, !alias.scope !0, !noalias !3
+; SCOPES:      load i32, ptr %[[BIdx]], align 4, !alias.scope !2, !noalias !5
 ; SCOPES:      %[[CIdx:[._a-zA-Z0-9]*]] = getelementptr{{.*}} ptr %C, i64 %{{.*}}
-; SCOPES:      load float, ptr %[[CIdx]], align 4, !alias.scope !6, !noalias !7
+; SCOPES:      load float, ptr %[[CIdx]], align 4, !alias.scope !8, !noalias !9
 ; SCOPES:      %[[AIdx:[._a-zA-Z0-9]*]] = getelementptr{{.*}} ptr %A, i64 %{{.*}}
-; SCOPES:      store i32 %{{[._a-zA-Z0-9]*}}, ptr %[[AIdx]], align 4, !alias.scope !8, !noalias !9
+; SCOPES:      store i32 %{{[._a-zA-Z0-9]*}}, ptr %[[AIdx]], align 4, !alias.scope !10, !noalias !11
 ;
-; SCOPES: !0 = !{!1}
-; SCOPES: !1 = distinct !{!1, !2, !"polly.alias.scope.MemRef_B"}
-; SCOPES: !2 = distinct !{!2, !"polly.alias.scope.domain"}
-; SCOPES: !3 = !{!4, !5}
-; SCOPES: !4 = distinct !{!4, !2, !"polly.alias.scope.MemRef_C"}
-; SCOPES: !5 = distinct !{!5, !2, !"polly.alias.scope.MemRef_A"}
-; SCOPES: !6 = !{!4}
-; SCOPES: !7 = !{!1, !5}
-; SCOPES: !8 = !{!5}
-; SCOPES: !9 = !{!1, !4}
+; SCOPES: !0 = distinct !{!0, !1}
+; SCOPES: !1 = !{!"llvm.loop.vectorize.enable", i32 0}
+; SCOPES: !2 = !{!3}
+; SCOPES: !3 = distinct !{!3, !4, !"polly.alias.scope.MemRef_B"}
+; SCOPES: !4 = distinct !{!4, !"polly.alias.scope.domain"}
+; SCOPES: !5 = !{!6, !7}
+; SCOPES: !6 = distinct !{!6, !4, !"polly.alias.scope.MemRef_C"}
+; SCOPES: !7 = distinct !{!7, !4, !"polly.alias.scope.MemRef_A"}
+; SCOPES: !8 = !{!6}
+; SCOPES: !9 = !{!3, !7}
+; SCOPES: !10 = !{!7}
+; SCOPES: !11 = !{!3, !6}
 ;
 ;    void jd(int *A, int *B, float *C) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
index b7394b2484048..bfa3c156ea75d 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
@@ -4,7 +4,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 
 ; CHECK: polly.stmt.bb3:                                   ; preds = %polly.stmt.bb3.entry
-; CHECK:   %tmp6_p_scalar_ = load double, ptr %arg1{{[0-9]*}}, align 8, !alias.scope !0, !noalias !3
+; CHECK:   %tmp6_p_scalar_ = load double, ptr %arg1{{[0-9]*}}, align 8, !alias.scope !2, !noalias !5
 ; CHECK:   %p_tmp7 = fadd double 1.000000e+00, %tmp6_p_scalar_
 ; CHECK:   %p_tmp8 = fcmp olt double 1.400000e+01, %p_tmp7
 ; CHECK:   br i1 %p_tmp8, label %polly.stmt.bb9, label %polly.stmt.bb10
diff --git a/polly/test/CodeGen/non_affine_float_compare.ll b/polly/test/CodeGen/non_affine_float_compare.ll
index 304a9016665ca..a359b662e6579 100644
--- a/polly/test/CodeGen/non_affine_float_compare.ll
+++ b/polly/test/CodeGen/non_affine_float_compare.ll
@@ -13,19 +13,19 @@
 ; CHECK: polly.stmt.bb2:
 ; CHECK:   %[[offset:.*]] = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK:   %scevgep[[R0:[0-9]*]] = getelementptr i8, ptr %A, i64 %[[offset]]
-; CHECK:   %tmp3_p_scalar_ = load float, ptr %scevgep[[R0]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   %tmp3_p_scalar_ = load float, ptr %scevgep[[R0]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   %[[offset2:.*]] = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK:   %scevgep[[R2:[0-9]*]] = getelementptr i8, ptr %scevgep{{[0-9]*}}, i64 %[[offset2]]
-; CHECK:   %tmp6_p_scalar_ = load float, ptr %scevgep[[R2]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   %tmp6_p_scalar_ = load float, ptr %scevgep[[R2]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   %p_tmp7 = fcmp oeq float %tmp3_p_scalar_, %tmp6_p_scalar_
 ; CHECK:   br i1 %p_tmp7, label %polly.stmt.bb8, label %polly.stmt.bb12.[[R:[a-zA-Z_.0-9]*]]
 
 ; CHECK: polly.stmt.bb8:
 ; CHECK:   %[[offset3:.*]] = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK:   %scevgep[[R3:[0-9]*]] = getelementptr i8, ptr %A, i64 %[[offset3]]
-; CHECK:   %tmp10_p_scalar_ = load float, ptr %scevgep[[R3]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   %tmp10_p_scalar_ = load float, ptr %scevgep[[R3]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   %p_tmp11 = fadd float %tmp10_p_scalar_, 1.000000e+00
-; CHECK:   store float %p_tmp11, ptr %scevgep[[R3]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   store float %p_tmp11, ptr %scevgep[[R3]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   br label %polly.stmt.bb12.[[R]]
 
 ; CHECK: polly.stmt.bb12.[[R]]:
@@ -34,9 +34,9 @@
 ; CHECK: polly.stmt.bb12:
 ; CHECK:   %[[offset4:.*]] = shl nuw nsw i64 %polly.indvar, 2
 ; CHECK:   %scevgep[[R4:[0-9]*]] = getelementptr i8, ptr %A, i64 %[[offset4]]
-; CHECK:   %tmp10b_p_scalar_ = load float, ptr %scevgep[[R4]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   %tmp10b_p_scalar_ = load float, ptr %scevgep[[R4]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   %p_tmp11b = fadd float %tmp10b_p_scalar_, 1.000000e+00
-; CHECK:   store float %p_tmp11b, ptr %scevgep[[R4]], align 4, !alias.scope !0, !noalias !3
+; CHECK:   store float %p_tmp11b, ptr %scevgep[[R4]], align 4, !alias.scope !2, !noalias !5
 ; CHECK:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; CHECK:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 1023
 ; CHECK:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
diff --git a/polly/test/CodeGen/partial_write_array.ll b/polly/test/CodeGen/partial_write_array.ll
index 8bb1bc2c3d8c4..fad4b21cf3dc8 100644
--- a/polly/test/CodeGen/partial_write_array.ll
+++ b/polly/test/CodeGen/partial_write_array.ll
@@ -38,7 +38,7 @@ return:
 
 ; CHECK:      polly.stmt.body.Stmt_body_Write0.partial:
 ; CHECK-NEXT:   %polly.access.A = getelementptr double, ptr %A, i64 0
-; CHECK-NEXT:   store double 4.200000e+01, ptr %polly.access.A, align 8, !alias.scope !0, !noalias !3
+; CHECK-NEXT:   store double 4.200000e+01, ptr %polly.access.A, align 8, !alias.scope !2, !noalias !5
 ; CHECK-NEXT:   br label %polly.stmt.body.cont
 
 ; CHECK:      polly.stmt.body.cont:
diff --git a/polly/test/CodeGen/partial_write_impossible_restriction.ll b/polly/test/CodeGen/partial_write_impossible_restriction.ll
index edee3b913ce7f..7577b137a2750 100644
--- a/polly/test/CodeGen/partial_write_impossible_restriction.ll
+++ b/polly/test/CodeGen/partial_write_impossible_restriction.ll
@@ -49,10 +49,10 @@ if.then.i.i1141.loopexit:                         ; preds = %cond.end
 
 ; CHECK-LABEL: polly.stmt.cond.false:
 ; CHECK:         %polly.access..pn{{[0-9]*}} = getelementptr i32, ptr %.pn, i64 %polly.indvar
-; CHECK:         store i32 %cond.in.sroa.speculate.load.cond.false_p_scalar_, ptr %polly.access..pn{{[0-9]*}}, align 4, !alias.scope !0, !noalias !3
+; CHECK:         store i32 %cond.in.sroa.speculate.load.cond.false_p_scalar_, ptr %polly.access..pn{{[0-9]*}}, align 4, !alias.scope !2, !noalias !5
 ; CHECK:         br label %polly.merge
 
 ; CHECK-LABEL: polly.stmt.cond.false{{[0-9]*}}:
 ; CHECK:         %polly.access..pn{{[0-9]*}} = getelementptr i32, ptr %.pn, i64 0
-; CHECK:         store i32 %cond.in.sroa.speculate.load.cond.false_p_scalar_{{[0-9]*}}, ptr %polly.access..pn{{[0-9]*}}, align 4, !alias.scope !0, !noalias !3
+; CHECK:         store i32 %cond.in.sroa.speculate.load.cond.false_p_scalar_{{[0-9]*}}, ptr %polly.access..pn{{[0-9]*}}, align 4, !alias.scope !2, !noalias !5
 ; CHECK:         br label %polly.stmt.cond.end{{[0-9]*}}
diff --git a/polly/test/CodeGen/phi_loop_carried_float.ll b/polly/test/CodeGen/phi_loop_carried_float.ll
index d671db08b06c9..4cb392d3353d3 100644
--- a/polly/test/CodeGen/phi_loop_carried_float.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float.ll
@@ -29,7 +29,7 @@
 
 ; CHECK-LABEL: polly.stmt.bb4:
 ; CHECK:         %tmp.0.s2a.reload[[R3:[0-9]*]] = load float, ptr %tmp.0.s2a
-; CHECK:         %tmp[[R5:[0-9]*]]_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
+; CHECK:         %tmp[[R5:[0-9]*]]_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !2, !noalias !5
 ; CHECK:         %p_tmp[[R4:[0-9]*]] = fadd float %tmp.0.s2a.reload[[R3]], %tmp[[R5]]_p_scalar_
 ; CHECK:         store float %p_tmp[[R4]], ptr %tmp.0.phiops
 
diff --git a/polly/test/CodeGen/phi_loop_carried_float_escape.ll b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
index 3e244c5e13328..9fd8ad413128a 100644
--- a/polly/test/CodeGen/phi_loop_carried_float_escape.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
@@ -29,7 +29,7 @@
 
 ; CHECK-LABEL: polly.stmt.bb4:
 ; CHECK:         %tmp.0.s2a.reload[[R3:[0-9]*]] = load float, ptr %tmp.0.s2a
-; CHECK:         %tmp[[R5:[0-9]*]]_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
+; CHECK:         %tmp[[R5:[0-9]*]]_p_scalar_ = load float, ptr %scevgep, align 4, !alias.scope !2, !noalias !5
 ; CHECK:         %p_tmp[[R4:[0-9]*]] = fadd float %tmp.0.s2a.reload[[R3]], %tmp[[R5]]_p_scalar_
 ; CHECK:         store float %p_tmp[[R4]], ptr %tmp.0.phiops
 
diff --git a/polly/test/CodeGen/scev-backedgetaken.ll b/polly/test/CodeGen/scev-backedgetaken.ll
index f5e68ec930d17..e0941690ae489 100644
--- a/polly/test/CodeGen/scev-backedgetaken.ll
+++ b/polly/test/CodeGen/scev-backedgetaken.ll
@@ -41,9 +41,11 @@ for.cond.cleanup.loopexit:
 
 ; CHECK-LABEL: @func(
 ; CHECK:         polly.stmt.omp.inner.for.body.us.us.us.preheader:
-; CHECK:         load i32, ptr %scevgep, align 4, !alias.scope !0, !noalias !3
-
-; CHECK:       !0 = !{!1}
-; CHECK:       !1 = distinct !{!1, !2, !"polly.alias.scope.MemRef_d"}
-; CHECK:       !2 = distinct !{!2, !"polly.alias.scope.domain"}
-; CHECK:       !3 = !{}
+; CHECK:         load i32, ptr %scevgep, align 4, !alias.scope !2, !noalias !5
+
+; CHECK:       !0 = distinct !{!0, !1}
+; CHECK:       !1 = !{!"llvm.loop.vectorize.enable", i32 0}
+; CHECK:       !2 = !{!3}
+; CHECK:       !3 = distinct !{!3, !4, !"polly.alias.scope.MemRef_d"}
+; CHECK:       !4 = distinct !{!4, !"polly.alias.scope.domain"}
+; CHECK:       !5 = !{}
diff --git a/polly/test/CodeGen/stmt_split_no_dependence.ll b/polly/test/CodeGen/stmt_split_no_dependence.ll
index 381cd30a2ae6f..bb878cc342af8 100644
--- a/polly/test/CodeGen/stmt_split_no_dependence.ll
+++ b/polly/test/CodeGen/stmt_split_no_dependence.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadNPMPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
-; CHECK:   store i32 %9, ptr %scevgep, align 4, !alias.scope !1, !noalias !4
-; CHECK:   store i32 %11, ptr %scevgep4, align 4, !alias.scope !4, !noalias !1
+; CHECK:   store i32 %9, ptr %scevgep, align 4, !alias.scope !3, !noalias !6
+; CHECK:   store i32 %11, ptr %scevgep4, align 4, !alias.scope !6, !noalias !3
 ;
 ;      void func(int *A, int *B){
 ;        for (int i = 0; i < 1024; i+=1) {
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int.ll b/polly/test/ScopInfo/int2ptr_ptr2int.ll
index f6668ecdd0898..578015aeecdc5 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int.ll
@@ -24,10 +24,10 @@
 ; IR-NEXT:   %p_add.ptr2 = getelementptr inbounds i64, ptr %p_tmp1, i64 1
 ; IR-NEXT:   %p_tmp2 = ptrtoint ptr %p_add.ptr2 to i64
 ; IR-NEXT:   %p_arrayidx = getelementptr inbounds i64, ptr %A, i64 %p_tmp2
-; IR-NEXT:   %tmp3_p_scalar_ = load i64, ptr %p_arrayidx, align 8, !alias.scope !0, !noalias !3
-; IR-NEXT:   %tmp4_p_scalar_ = load i64, ptr %scevgep, align 8, !alias.scope !0, !noalias !3
+; IR-NEXT:   %tmp3_p_scalar_ = load i64, ptr %p_arrayidx, align 8, !alias.scope !2, !noalias !5
+; IR-NEXT:   %tmp4_p_scalar_ = load i64, ptr %scevgep, align 8, !alias.scope !2, !noalias !5
 ; IR-NEXT:   %p_add4 = add nsw i64 %tmp4_p_scalar_, %tmp3_p_scalar_
-; IR-NEXT:   store i64 %p_add4, ptr %scevgep, align 8, !alias.scope !0, !noalias !3
+; IR-NEXT:   store i64 %p_add4, ptr %scevgep, align 8, !alias.scope !2, !noalias !5
 ; IR-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; IR-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 99
 ; IR-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
index 361bf5a957611..627524c0327dd 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
@@ -24,9 +24,9 @@
 ; IR-NEXT:   %ptr13 = ptrtoint ptr %ptr to i16
 ;
 ; IR:      polly.stmt.for.body:
-; IR-NEXT:   %tmp4_p_scalar_ = load i64, ptr %scevgep, align 8, !alias.scope !3, !noalias !0
+; IR-NEXT:   %tmp4_p_scalar_ = load i64, ptr %scevgep, align 8, !alias.scope !5, !noalias !2
 ; IR-NEXT:   %p_add4 = add nsw i64 %tmp4_p_scalar_, %polly.preload.tmp3.merge
-; IR-NEXT:   store i64 %p_add4, ptr %scevgep, align 8, !alias.scope !3, !noalias !0
+; IR-NEXT:   store i64 %p_add4, ptr %scevgep, align 8, !alias.scope !5, !noalias !2
 ; IR-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; IR-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 99
 ; IR-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index ef8ec171a1657..4a6b317a03f66 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -346,3 +346,21 @@ if(SUB_COMPONENTS)
       ${CMAKE_CURRENT_BINARY_DIR}/runtimes/Components.cmake)
   endif()
 endif()
+
+# If the user requested 'compile_commands.json' we merge the generated JSON from
+# the created directories.
+if(CMAKE_EXPORT_COMPILE_COMMANDS AND NOT ("${LLVM_BINARY_DIR}" STREQUAL "${CMAKE_BINARY_DIR}"))
+  # Make a dependency so that we don't error if the file gets deleted somehow.
+  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/compile_commands.json
+                     COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/compile_commands.json)
+
+  file(TO_NATIVE_PATH "${LLVM_MAIN_SRC_DIR}/utils/merge-json.py" MERGE_JSON_PATH)
+  add_custom_command(OUTPUT ${LLVM_BINARY_DIR}/compile_commands.json
+                     COMMAND ${CMAKE_COMMAND} -E touch ${LLVM_BINARY_DIR}/compile_commands.json
+                     COMMAND ${Python3_EXECUTABLE} ${MERGE_JSON_PATH}
+                             ${LLVM_BINARY_DIR}/compile_commands.json
+                             ${CMAKE_BINARY_DIR}/compile_commands.json
+                             -o ${LLVM_BINARY_DIR}/compile_commands.json
+                     DEPENDS ${CMAKE_BINARY_DIR}/compile_commands.json)
+  add_custom_target(merge_runtime_commands ALL DEPENDS ${LLVM_BINARY_DIR}/compile_commands.json)
+endif()
diff --git a/runtimes/cmake/Modules/FindLibcCommonUtils.cmake b/runtimes/cmake/Modules/FindLibcCommonUtils.cmake
index 763dc81d8bd73..0e65fdff7c34b 100644
--- a/runtimes/cmake/Modules/FindLibcCommonUtils.cmake
+++ b/runtimes/cmake/Modules/FindLibcCommonUtils.cmake
@@ -6,9 +6,14 @@
 #
 #===--------------------------------------------------------------------===//
 
-add_library(llvm-libc-common-utilities INTERFACE)
-# TODO: Reorganize the libc shared section so that it can be included without
-# adding the root "libc" directory to the include path.
-target_include_directories(llvm-libc-common-utilities INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../../libc)
-target_compile_definitions(llvm-libc-common-utilities INTERFACE LIBC_NAMESPACE=__llvm_libc_common_utils)
-target_compile_features(llvm-libc-common-utilities INTERFACE cxx_std_17)
+if(NOT TARGET llvm-libc-common-utilities)
+  set(libc_path ${CMAKE_CURRENT_LIST_DIR}/../../../libc)
+  if (EXISTS ${libc_path} AND IS_DIRECTORY ${libc_path})
+    add_library(llvm-libc-common-utilities INTERFACE)
+    # TODO: Reorganize the libc shared section so that it can be included without
+    # adding the root "libc" directory to the include path.
+    target_include_directories(llvm-libc-common-utilities INTERFACE ${libc_path})
+    target_compile_definitions(llvm-libc-common-utilities INTERFACE LIBC_NAMESPACE=__llvm_libc_common_utils)
+    target_compile_features(llvm-libc-common-utilities INTERFACE cxx_std_17)
+  endif()
+endif()
diff --git a/sycl/test-e2e/Basic/report_code_loc.cpp b/sycl/test-e2e/Basic/report_code_loc.cpp
index f4690a476504a..0582d989b4021 100644
--- a/sycl/test-e2e/Basic/report_code_loc.cpp
+++ b/sycl/test-e2e/Basic/report_code_loc.cpp
@@ -21,6 +21,6 @@ int main() {
   try {
     q.memcpy(mdlImag, 0, sizeof(XFLOAT));
   } catch (...) {
-    // CHECK: Exception caught at File: {{.*}}report_code_loc.cpp | Function: main | Line: 22 | Column: 5
+    // CHECK: Exception caught at File: {{.*}}report_code_loc.cpp | Function: main | Line: 22 | Column: 7
   }
 }
diff --git a/sycl/test-e2e/Config/kernel_from_file.cpp b/sycl/test-e2e/Config/kernel_from_file.cpp
index ad99ff3705556..37c2273b43903 100644
--- a/sycl/test-e2e/Config/kernel_from_file.cpp
+++ b/sycl/test-e2e/Config/kernel_from_file.cpp
@@ -14,7 +14,7 @@
 // RUN: %if linux %{ llvm-link -o=%t_app.bc %t.bc %t_compiler_wrappers.bc %t_asan.bc %} %else %{ llvm-link -o=%t_app.bc %t.bc %t_compiler_wrappers.bc %}
 // >> ---- translate to SPIR-V
 // RUN: llvm-spirv -o %t.spv %t_app.bc
-// RUN: %clangxx -Wno-error=ignored-attributes %sycl_include -DSYCL_DISABLE_FALLBACK_ASSERT %cxx_std_optionc++17 %include_option %t.h %s -o %t.out %sycl_options -fno-sycl-dead-args-optimization -Xclang -verify-ignore-unexpected=note,warning
+// RUN: %clangxx -Wno-error=ignored-attributes %sycl_include -DSYCL_DISABLE_FALLBACK_ASSERT %cxx_std_optionc++17 %include_option %t.h %s -o %t.out %sycl_options -Xclang -verify-ignore-unexpected=note,warning
 // RUN: env SYCL_USE_KERNEL_SPV=%t.spv %{run} %t.out | FileCheck %s
 // CHECK: Passed
 
diff --git a/sycl/test-e2e/XPTI/buffer/accessors.cpp b/sycl/test-e2e/XPTI/buffer/accessors.cpp
index bbad451a06a15..f7412c0744cdd 100644
--- a/sycl/test-e2e/XPTI/buffer/accessors.cpp
+++ b/sycl/test-e2e/XPTI/buffer/accessors.cpp
@@ -24,17 +24,17 @@ int main() {
   sycl::nd_range<1> NDRange(Buf.size(), Buf.size());
 
   Queue.submit([&](sycl::handler &cgh) {
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID1:.+]]|2015|1024|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID1:.+]]|2015|1024|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
     auto A1 = Buf.get_access<mode::read, target::constant_buffer>(cgh);
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID2:.*]]|2014|1025|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID2:.*]]|2014|1025|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
     auto A2 = Buf.get_access<mode::write>(cgh);
     // CHECK: {{[0-9]+}}|Construct accessor|0x0|[[ACCID3:.*]]|2016|1026|{{.*}}accessors.cpp:[[# @LINE + 1]]:34
     sycl::local_accessor<int, 1> A3(Range, cgh);
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID4:.*]]|2014|1027|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID4:.*]]|2014|1027|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
     auto A4 = Buf.get_access<mode::discard_write>(cgh);
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID5:.*]]|2014|1028|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID5:.*]]|2014|1028|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
     auto A5 = Buf.get_access<mode::discard_read_write, target::device>(cgh);
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID6:.*]]|2014|1029|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID6:.*]]|2014|1029|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
     auto A6 = Buf.get_access<mode::atomic>(cgh);
     cgh.parallel_for<class FillBuffer>(NDRange, [=](sycl::nd_item<1>) {
       (void)A1;
@@ -51,11 +51,11 @@ int main() {
   { sycl::host_accessor HA(Buf, sycl::write_only); }
   // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID9:.*]]|2018|1026|{{.*}}accessors.cpp:[[# @LINE + 1]]:25
   { sycl::host_accessor HA(Buf, sycl::read_write); }
-  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID10:.*]]|2018|1027|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID10:.*]]|2018|1027|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
   { auto HA = Buf.get_access<mode::discard_write>(); }
-  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID11:.*]]|2018|1028|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID11:.*]]|2018|1028|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
   { auto HA = Buf.get_access<mode::discard_read_write>(); }
-  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID12:.*]]|2018|1029|{{.*}}accessors.cpp:[[# @LINE + 1]]:15
+  // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID12:.*]]|2018|1029|{{.*}}accessors.cpp:[[# @LINE + 1]]:19
   { auto HA = Buf.get_access<mode::atomic>(); }
 
   return 0;
diff --git a/sycl/test-e2e/XPTI/buffer/multiple_queues.cpp b/sycl/test-e2e/XPTI/buffer/multiple_queues.cpp
index 659a982669c6e..3085c0b621cb0 100644
--- a/sycl/test-e2e/XPTI/buffer/multiple_queues.cpp
+++ b/sycl/test-e2e/XPTI/buffer/multiple_queues.cpp
@@ -31,7 +31,7 @@ int main() {
     sycl::buffer<int, 1> Buffer1(Array, NumOfWorkItems);
 
     Queue1.submit([&](sycl::handler &cgh) {
-      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID1:.*]]|2014|1025|{{.*}}multiple_queues.cpp:[[# @LINE + 1]]:24
+      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID1:.*]]|2014|1025|{{.*}}multiple_queues.cpp:[[# @LINE + 1]]:32
       auto Accessor1 = Buffer1.get_access<sycl::access::mode::write>(cgh);
       // CHECK:{{[0-9]+}}|Associate buffer|[[USERID1]]|[[BEID1:.*]]
       cgh.parallel_for<class FillBuffer>(NumOfWorkItems, [=](sycl::id<1> WIid) {
@@ -41,7 +41,7 @@ int main() {
     Queue1.wait();
 
     Queue2.submit([&](sycl::handler &cgh) {
-      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID2:.*]]|2014|1025|{{.*}}multiple_queues.cpp:[[# @LINE + 1]]:24
+      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID2:.*]]|2014|1025|{{.*}}multiple_queues.cpp:[[# @LINE + 1]]:32
       auto Accessor1 = Buffer1.get_access<sycl::access::mode::write>(cgh);
       // CHECK:{{[0-9]+}}|Associate buffer|[[USERID1]]|[[BEID2:.*]]
       cgh.parallel_for<class MulBuffer>(NumOfWorkItems, [=](sycl::id<1> WIid) {
diff --git a/sycl/test-e2e/XPTI/buffer/sub_buffer.cpp b/sycl/test-e2e/XPTI/buffer/sub_buffer.cpp
index ea45c2fab6aa9..f660ef7b17456 100644
--- a/sycl/test-e2e/XPTI/buffer/sub_buffer.cpp
+++ b/sycl/test-e2e/XPTI/buffer/sub_buffer.cpp
@@ -24,7 +24,7 @@ int main() {
                                    sycl::range<1>{32}};
 
     Queue.submit([&](sycl::handler &cgh) {
-      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID1:.*]]|2014|1025|{{.*}}sub_buffer.cpp:[[# @LINE + 1]]:24
+      // CHECK: {{[0-9]+}}|Construct accessor|[[USERID1]]|[[ACCID1:.*]]|2014|1025|{{.*}}sub_buffer.cpp:[[# @LINE + 1]]:34
       auto Accessor1 = SubBuffer.get_access<sycl::access::mode::write>(cgh);
       // CHECK:{{[0-9]+}}|Associate buffer|[[USERID1]]|[[BEID1:.*]]
       // CHECK:{{[0-9]+}}|Associate buffer|[[USERID1]]|[[BEID2:.*]]
diff --git a/sycl/test-e2e/XPTI/image/accessors.cpp b/sycl/test-e2e/XPTI/image/accessors.cpp
index 15fc4dfc78261..80caefa175cd6 100644
--- a/sycl/test-e2e/XPTI/image/accessors.cpp
+++ b/sycl/test-e2e/XPTI/image/accessors.cpp
@@ -24,22 +24,22 @@ int main() {
     sycl::unsampled_image<1> UImg(sycl::image_format::r32b32g32a32_uint, 3);
 
     Q.submit([&](sycl::handler &CGH) {
-      // CHECK:{{[0-9]+}}|Construct unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1|1024|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+      // CHECK:{{[0-9]+}}|Construct unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1|1024|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
       auto A1 = UImg.get_access<sycl::uint4, mode::read,
                                 sycl::image_target::host_task>(CGH);
     });
 
     Q.submit([&](sycl::handler &CGH) {
-      // CHECK:{{[0-9]+}}|Construct unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1|1025|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+      // CHECK:{{[0-9]+}}|Construct unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1|1025|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
       auto A1 = UImg.get_access<sycl::uint4, mode::write,
                                 sycl::image_target::host_task>(CGH);
     });
 
-    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1024|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1024|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
     { auto HA = UImg.get_host_access<sycl::int4, mode::read>(); }
-    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1025|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1025|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
     { auto HA = UImg.get_host_access<sycl::float4, mode::write>(); }
-    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1026|{{.*}}vec{{.*}}|8|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+    // CHECK:{{[0-9]+}}|Construct host unsampled image accessor|[[UIMGID]]|0x{{[0-9,a-f]+}}|1026|{{.*}}vec{{.*}}|8|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
     { auto HA = UImg.get_host_access<sycl::half4, mode::read_write>(); }
   }
   // CHECK:{{[0-9]+}}|Destruct image|[[UIMGID]]
@@ -58,14 +58,14 @@ int main() {
                                 SampledImgElemCount);
 
     Q.submit([&](sycl::handler &CGH) {
-      // CHECK:{{[0-9]+}}|Construct sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|1|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 2]]:11
+      // CHECK:{{[0-9]+}}|Construct sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|1|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 2]]:16
       auto A1 =
           SImg.get_access<sycl::uint4, sycl::image_target::host_task>(CGH);
     });
 
-    // CHECK:{{[0-9]+}}|Construct host sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+    // CHECK:{{[0-9]+}}|Construct host sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|{{.*}}vec{{.*}}|16|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
     { auto HA = SImg.get_host_access<sycl::int4>(); }
-    // CHECK:{{[0-9]+}}|Construct host sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|{{.*}}vec{{.*}}|8|{{.*}}accessors.cpp:[[# @LINE + 1]]:17
+    // CHECK:{{[0-9]+}}|Construct host sampled image accessor|[[SIMGID]]|0x{{[0-9,a-f]+}}|{{.*}}vec{{.*}}|8|{{.*}}accessors.cpp:[[# @LINE + 1]]:22
     { auto HA = SImg.get_host_access<sycl::half4>(); }
   }
   // CHECK:{{[0-9]+}}|Destruct image|[[SIMGID]]
diff --git a/sycl/test-e2e/XPTI/kernel/basic.cpp b/sycl/test-e2e/XPTI/kernel/basic.cpp
index 59bafe844625e..9075d2c2f52c8 100644
--- a/sycl/test-e2e/XPTI/kernel/basic.cpp
+++ b/sycl/test-e2e/XPTI/kernel/basic.cpp
@@ -57,7 +57,7 @@ int main() {
   auto PtrShared = sycl::malloc_shared<int>(8, Queue);
   Queue
       .submit([&](sycl::handler &cgh) {
-        // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID1:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:19
+        // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID1:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:23
         auto A1 = Buf.get_access<mode::read_write>(cgh);
         // CHECK: {{[0-9]+}}|Construct accessor|0x0|[[ACCID2:.*]]|2016|1026|{{.*}}.cpp:[[# @LINE + 1]]:38
         sycl::local_accessor<int, 1> A2(Range, cgh);
@@ -78,19 +78,19 @@ int main() {
       })
       .wait();
 
-  // CHECK: Wait begin|{{.*}}.cpp:[[# @LINE + 2]]:3
-  // CHECK: Wait end|{{.*}}.cpp:[[# @LINE + 1]]:3
+  // CHECK: Wait begin|{{.*}}.cpp:[[# @LINE + 2]]:9
+  // CHECK: Wait end|{{.*}}.cpp:[[# @LINE + 1]]:9
   Queue.wait();
 
   // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID3:.*]]|2018|1024|{{.*}}.cpp:[[# @LINE + 1]]:25
   { sycl::host_accessor HA(Buf, sycl::read_only); }
 
   Queue.submit([&](sycl::handler &cgh) {
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID4:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:16
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID4:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:20
     auto Acc = Buf.get_access<mode::read_write>(cgh);
     Functor1 F(Val, Acc);
-    // CHECK-OPT: Node create|{{.*}}Functor1|{{.*}}.cpp:[[# @LINE - 4 ]]:3|{1, 1, 1}, {0, 0, 0}, {0, 0, 0}, 3
-    // CHECK-NOOPT: Node create|{{.*}}Functor1|{{.*}}.cpp:[[# @LINE - 5 ]]:3|{1, 1, 1}, {0, 0, 0}, {0, 0, 0}, 5
+    // CHECK-OPT: Node create|{{.*}}Functor1|{{.*}}.cpp:[[# @LINE - 4 ]]:9|{1, 1, 1}, {0, 0, 0}, {0, 0, 0}, 3
+    // CHECK-NOOPT: Node create|{{.*}}Functor1|{{.*}}.cpp:[[# @LINE - 5 ]]:9|{1, 1, 1}, {0, 0, 0}, {0, 0, 0}, 5
     cgh.single_task(F);
     // CHECK-OPT: arg0 : {1, {{[0-9,a-f,x]+}}, 2, 0}
     // CHECK-OPT: arg1 : {0, [[ACCID4]], 4062, 1}
@@ -98,11 +98,11 @@ int main() {
   });
 
   Queue.submit([&](sycl::handler &cgh) {
-    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID5:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:16
+    // CHECK: {{[0-9]+}}|Construct accessor|[[BUFFERID]]|[[ACCID5:.+]]|2014|1026|{{.*}}.cpp:[[# @LINE + 1]]:20
     auto Acc = Buf.get_access<mode::read_write>(cgh);
     Functor2 F(Val, Acc);
-    // CHECK-OPT: Node create|{{.*}}Functor2|{{.*}}.cpp:[[# @LINE - 4 ]]:3|{5, 1, 1}, {0, 0, 0}, {0, 0, 0}, 3
-    // CHECK-NOOPT: Node create|{{.*}}Functor2|{{.*}}.cpp:[[# @LINE - 5 ]]:3|{5, 1, 1}, {0, 0, 0}, {0, 0, 0}, 5
+    // CHECK-OPT: Node create|{{.*}}Functor2|{{.*}}.cpp:[[# @LINE - 4 ]]:9|{5, 1, 1}, {0, 0, 0}, {0, 0, 0}, 3
+    // CHECK-NOOPT: Node create|{{.*}}Functor2|{{.*}}.cpp:[[# @LINE - 5 ]]:9|{5, 1, 1}, {0, 0, 0}, {0, 0, 0}, 5
     cgh.parallel_for(Range, F);
     // CHECK-OPT: arg0 : {1, {{[0-9,a-f,x]+}}, 2, 0}
     // CHECK-OPT: arg1 : {0, [[ACCID5]], 4062, 1}
diff --git a/sycl/test-e2e/XPTI/kernel/content.cpp b/sycl/test-e2e/XPTI/kernel/content.cpp
index 54a52e8dd892c..67b9f48846455 100644
--- a/sycl/test-e2e/XPTI/kernel/content.cpp
+++ b/sycl/test-e2e/XPTI/kernel/content.cpp
@@ -32,8 +32,8 @@ int main() {
     auto sumR = reduction(sumBuf, cgh, plus<>());
     // Reduction kernel is used, strategy and hence number of kernel arguments
     // is hw-dependent.
-    // CHECK-OPT:Node create|{{.*}}reduction{{.*}}test1{{.*}}|{{.*}}.cpp:[[# @LINE - 6 ]]:3|{{{.*}}, 1, 1}, {{{.*}}, 1, 1}, {0, 0, 0}, {{1.*}}
-    // CHECK-NOOPT:Node create|{{.*}}reduction{{.*}}test1{{.*}}|{{.*}}.cpp:[[# @LINE - 7 ]]:3|{{{.*}}, 1, 1}, {{{.*}}, 1, 1}, {0, 0, 0}, {{2.*}}
+    // CHECK-OPT:Node create|{{.*}}reduction{{.*}}test1{{.*}}|{{.*}}.cpp:[[# @LINE - 6 ]]:11|{{{.*}}, 1, 1}, {{{.*}}, 1, 1}, {0, 0, 0}, {{1.*}}
+    // CHECK-NOOPT:Node create|{{.*}}reduction{{.*}}test1{{.*}}|{{.*}}.cpp:[[# @LINE - 7 ]]:11|{{{.*}}, 1, 1}, {{{.*}}, 1, 1}, {0, 0, 0}, {{2.*}}
     cgh.parallel_for<class test1>(
         range<1>{1024}, sumR,
         [=](id<1> idx, auto &sum) { sum += inputValues[idx]; });
@@ -48,8 +48,8 @@ int main() {
     myQueue.submit([&](handler &cgh) {
       auto in = in_buf.template get_access<access::mode::read>(cgh);
       auto out = out_buf.template get_access<access::mode::read_write>(cgh);
-      // CHECK-OPT:Node create|{{.*}}test2{{.*}}|{{.*}}.cpp:[[# @LINE - 3 ]]:5|{128, 4, 2}, {32, 2, 1}, {16, 1, 0}, 2
-      // CHECK-NOOPT:Node create|{{.*}}test2{{.*}}|{{.*}}.cpp:[[# @LINE - 4 ]]:5|{128, 4, 2}, {32, 2, 1}, {16, 1, 0}, 8
+      // CHECK-OPT:Node create|{{.*}}test2{{.*}}|{{.*}}.cpp:[[# @LINE - 3 ]]:13|{128, 4, 2}, {32, 2, 1}, {16, 1, 0}, 2
+      // CHECK-NOOPT:Node create|{{.*}}test2{{.*}}|{{.*}}.cpp:[[# @LINE - 4 ]]:13|{128, 4, 2}, {32, 2, 1}, {16, 1, 0}, 8
       cgh.parallel_for<class test2>(
           nd_range<3>({128, 4, 2}, {32, 2, 1}, {16, 1, 0}), [=](nd_item<3> it) {
             auto sg = it.get_sub_group();
diff --git a/sycl/test/check_device_code/group_load.cpp b/sycl/test/check_device_code/group_load.cpp
index 659c17d5abc43..9bb12dbf19af9 100644
--- a/sycl/test/check_device_code/group_load.cpp
+++ b/sycl/test/check_device_code/group_load.cpp
@@ -89,8 +89,8 @@ using accessor_iter_t = accessor<int, 1, access_mode::read, target::device,
 // CHECK-LABEL: @_ZN7blocked18test_accessor_iterERN4sycl3_V19sub_groupERNS1_6detail17accessor_iteratorIKiLi1EEERi(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15:![0-9]+]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4
 // CHECK-NEXT:    [[CONV3_I_I_I:%.*]] = sext i32 [[TMP0]] to i64
@@ -110,8 +110,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg,
 // CHECK-LABEL: @_ZN7blocked34test_accessor_iter_force_optimizedERN4sycl3_V19sub_groupERNS1_6detail17accessor_iteratorIKiLi1EEERi(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[ADD_PTR_I_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[AGG_TMP1_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP1_SROA_2_0_COPYLOAD]]
 // CHECK-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ne ptr addrspace(4) [[ADD_PTR_I_I_I_I]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I_I_I]])
@@ -154,8 +154,8 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4
 // CHECK-NEXT:    [[IDXPROM_I_I_I:%.*]] = sext i32 [[TMP1]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I_I]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr addrspace(1) [[ARRAYIDX_I_I_I]], align 1, !tbaa [[TBAA21:![0-9]+]]
-// CHECK-NEXT:    store i8 [[TMP2]], ptr addrspace(4) [[OUT:%.*]], align 1, !tbaa [[TBAA21]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr addrspace(1) [[ARRAYIDX_I_I_I]], align 1, !tbaa [[TBAA22:![0-9]+]]
+// CHECK-NEXT:    store i8 [[TMP2]], ptr addrspace(4) [[OUT:%.*]], align 1, !tbaa [[TBAA22]]
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL10GROUP_LOADINS0_9SUB_GROUPEPU3AS1CCNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE17VERIFY_LOAD_TYPESIT0_T1_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT2_EEVE4TYPEESP_SN_RSO_SQ__EXIT:%.*]]
 // CHECK:       if.end.i.i:
@@ -175,7 +175,7 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 
 // CHECK-LABEL: @_ZN7blocked16test_four_shortsERN4sycl3_V19sub_groupEPU3AS1sNS1_4spanIsLm4EEE(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA24:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -185,7 +185,7 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META23:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META26:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = shl i32 [[TMP3]], 2
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -197,11 +197,11 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = or disjoint i32 [[MUL_I_I_I]], [[I_0_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA26:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA29:![0-9]+]]
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA26]]
+// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA29]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP31:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1ssLm4ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_SP_NS0_4spanISQ_XT2_EEESS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL10GROUP_LOADINS0_9SUB_GROUPEPU3AS1SSLM4ENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE17VERIFY_LOAD_TYPESIT0_T1_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_SN_NS0_4SPANISO_XT2_EEESQ__EXIT:%.*]]
@@ -225,7 +225,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META31:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META34:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = mul i32 [[TMP2]], 3
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -241,7 +241,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP34:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP37:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm3ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -258,7 +258,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META36:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META39:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = shl i32 [[TMP2]], 2
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -274,7 +274,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP39:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP42:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm4ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -291,7 +291,7 @@ SYCL_EXTERNAL void test_four_ints(sycl::sub_group &sg, plain_global_ptr<int> p,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META41:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META44:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = mul i32 [[TMP2]], 7
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -307,7 +307,7 @@ SYCL_EXTERNAL void test_four_ints(sycl::sub_group &sg, plain_global_ptr<int> p,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP44:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP47:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm7ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -328,8 +328,8 @@ namespace striped {
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META46:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META49:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META49:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META52:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
 // CHECK:       for.cond.i:
 // CHECK-NEXT:    [[I_0_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I:%.*]], [[FOR_BODY_I:%.*]] ]
@@ -345,7 +345,7 @@ namespace striped {
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP52:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP55:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm2ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESN_SL_NS0_4spanISM_XT2_EEESO_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -393,13 +393,13 @@ using accessor_iter_t = accessor<int, 1, access_mode::read, target::device,
 // CHECK-LABEL: @_ZN7striped18test_accessor_iterERN4sycl3_V19sub_groupERNS1_6detail17accessor_iteratorIKiLi1EEENS1_4spanIiLm2EEE(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META56:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META59:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META59:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META62:![0-9]+]]
 // CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr addrspace(4) [[AGG_TMP1_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP1_SROA_2_0_COPYLOAD]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -416,7 +416,7 @@ using accessor_iter_t = accessor<int, 1, access_mode::read, target::device,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP62:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP65:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupENS0_6detail17accessor_iteratorIKiLi1EEEiLm2ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSD_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -430,8 +430,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg,
 // CHECK-LABEL: @_ZN7striped34test_accessor_iter_force_optimizedERN4sycl3_V19sub_groupERNS1_6detail17accessor_iteratorIKiLi1EEENS1_4spanIiLm2EEE(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[ADD_PTR_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[AGG_TMP1_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP1_SROA_2_0_COPYLOAD]]
@@ -442,8 +442,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[TOBOOL_NOT_I]], label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META64:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META67:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META67:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META70:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -459,7 +459,7 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP70:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP73:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupENS0_6detail17accessor_iteratorIKiLi1EEEiLm2ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSD_INS3_21contiguous_memory_keyEJEEENSD_INS3_14full_group_keyEJEEENSD_INSB_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeEST_SR_NS0_4spanISS_XT2_EEESU_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL10GROUP_LOADINS0_9SUB_GROUPENS0_6DETAIL17ACCESSOR_ITERATORIKILI1EEEILM2ENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSD_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSD_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE17VERIFY_LOAD_TYPESIT0_T1_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESR_SP_NS0_4SPANISQ_XT2_EEESS__EXIT:%.*]]
@@ -479,7 +479,7 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 
 // CHECK-LABEL: @_ZN7striped24test_runtime_align_checkERN4sycl3_V19sub_groupEPU3AS1cNS1_4spanIcLm2EEE(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA75:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -489,8 +489,8 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META72:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META75:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META77:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META80:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -502,11 +502,11 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = add i32 [[TMP3]], [[MUL_I_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr addrspace(1) [[ARRAYIDX_I_I]], align 1, !tbaa [[TBAA21]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr addrspace(1) [[ARRAYIDX_I_I]], align 1, !tbaa [[TBAA22]]
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    store i8 [[TMP5]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 1, !tbaa [[TBAA21]]
+// CHECK-NEXT:    store i8 [[TMP5]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 1, !tbaa [[TBAA22]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP78:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP83:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1ccLm2ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_SP_NS0_4spanISQ_XT2_EEESS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL10GROUP_LOADINS0_9SUB_GROUPEPU3AS1CCLM2ENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE17VERIFY_LOAD_TYPESIT0_T1_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_SN_NS0_4SPANISO_XT2_EEESQ__EXIT:%.*]]
@@ -527,7 +527,7 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 
 // CHECK-LABEL: @_ZN7striped16test_four_shortsERN4sycl3_V19sub_groupEPU3AS1sNS1_4spanIsLm4EEE(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA24]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -537,8 +537,8 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META80:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META83:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META85:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META88:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -550,11 +550,11 @@ SYCL_EXTERNAL void test_runtime_align_check(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = add i32 [[TMP3]], [[MUL_I_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA26]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA29]]
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    store i16 [[TMP5]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA26]]
+// CHECK-NEXT:    store i16 [[TMP5]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA29]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP86:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP91:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1ssLm4ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_SP_NS0_4spanISQ_XT2_EEESS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL10GROUP_LOADINS0_9SUB_GROUPEPU3AS1SSLM4ENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE17VERIFY_LOAD_TYPESIT0_T1_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_SN_NS0_4SPANISO_XT2_EEESQ__EXIT:%.*]]
@@ -578,8 +578,8 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META88:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META91:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META93:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META96:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -595,7 +595,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP94:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP99:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm3ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -612,8 +612,8 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META96:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META99:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META101:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META104:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -629,7 +629,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP102:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP107:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm16ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
@@ -647,8 +647,8 @@ SYCL_EXTERNAL void test_sixteen_ints(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[OUT:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META104:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META107:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META109:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META112:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -664,7 +664,7 @@ SYCL_EXTERNAL void test_sixteen_ints(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP110:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP115:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental10group_loadINS0_9sub_groupEPU3AS1iiLm11ENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE17verify_load_typesIT0_T1_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_SN_NS0_4spanISO_XT2_EEESQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR4]]
 // CHECK-NEXT:    ret void
diff --git a/sycl/test/check_device_code/group_store.cpp b/sycl/test/check_device_code/group_store.cpp
index 463f2bd2b6b66..b0cd454287ef1 100644
--- a/sycl/test/check_device_code/group_store.cpp
+++ b/sycl/test/check_device_code/group_store.cpp
@@ -116,8 +116,8 @@ using accessor_iter_t = accessor<int, 1, access_mode::write, target::device,
 // CHECK-LABEL: @_ZN7blocked18test_accessor_iterERN4sycl3_V19sub_groupEiRNS1_6detail17accessor_iteratorIiLi1EEE(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15:![0-9]+]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17:![0-9]+]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4
 // CHECK-NEXT:    [[CONV5_I_I_I:%.*]] = sext i32 [[TMP0]] to i64
@@ -136,8 +136,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, int v,
 // CHECK-LABEL: @_ZN7blocked34test_accessor_iter_force_optimizedERN4sycl3_V19sub_groupEiRNS1_6detail17accessor_iteratorIiLi1EEE(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[AGG_TMP1_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP1_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP1_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[ADD_PTR_I_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[AGG_TMP1_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP1_SROA_2_0_COPYLOAD]]
 // CHECK-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ne ptr addrspace(4) [[ADD_PTR_I_I_I_I]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I_I_I]])
@@ -173,7 +173,7 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-LABEL: @_ZN7blocked16test_four_shortsERN4sycl3_V19sub_groupENS1_4spanIsLm4EEEPU3AS1s(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VALUES_I:%.*]] = alloca [4 x i16], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA22:![0-9]+]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -183,7 +183,7 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META21:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META24:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = shl i32 [[TMP3]], 2
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -193,13 +193,13 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK:       for.body.i.i:
 // CHECK-NEXT:    [[CONV_I_I:%.*]] = zext nneg i32 [[I_0_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA24:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA27:![0-9]+]]
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = or disjoint i32 [[MUL_I_I_I]], [[I_0_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_NS0_4spanISP_XT1_EEESQ_SS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPESLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT:%.*]]
@@ -211,18 +211,18 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 4
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[VALUES_I]], align 2, !tbaa [[TBAA28:![0-9]+]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[VALUES_I]], align 2, !tbaa [[TBAA31:![0-9]+]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELImEvPU3AS1mT_(ptr addrspace(1) noundef nonnull [[P]], i64 noundef [[TMP5]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPESLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT]]
 // CHECK:       for.body.i:
 // CHECK-NEXT:    [[CONV_I:%.*]] = zext nneg i32 [[I_0_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I20_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I]]
-// CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [4 x i16], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
-// CHECK-NEXT:    store i16 [[TMP6]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP6]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP29:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP32:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -236,7 +236,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK-LABEL: @_ZN7blocked22test_four_const_shortsERN4sycl3_V19sub_groupENS1_4spanIKsLm4EEEPU3AS1s(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VALUES_I:%.*]] = alloca [4 x i16], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA22]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -246,7 +246,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META31:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META34:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = shl i32 [[TMP3]], 2
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -256,13 +256,13 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK:       for.body.i.i:
 // CHECK-NEXT:    [[CONV_I_I:%.*]] = zext nneg i32 [[I_0_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = or disjoint i32 [[MUL_I_I_I]], [[I_0_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP34:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP37:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEKsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSC_INS3_21contiguous_memory_keyEJEEENSC_INS3_14full_group_keyEJEEENSC_INSA_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESS_NS0_4spanISQ_XT1_EEESR_ST_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEKSLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSC_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSC_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESQ_NS0_4SPANISO_XT1_EEESP_SR__EXIT:%.*]]
@@ -274,18 +274,18 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 4
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[VALUES_I]], align 2, !tbaa [[TBAA28]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[VALUES_I]], align 2, !tbaa [[TBAA31]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELImEvPU3AS1mT_(ptr addrspace(1) noundef nonnull [[P]], i64 noundef [[TMP5]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEKSLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI0EEEEENSC_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSC_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESQ_NS0_4SPANISO_XT1_EEESP_SR__EXIT]]
 // CHECK:       for.body.i:
 // CHECK-NEXT:    [[CONV_I:%.*]] = zext nneg i32 [[I_0_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I20_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I]]
-// CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [4 x i16], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
-// CHECK-NEXT:    store i16 [[TMP6]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP6]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP35:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP38:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEKsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSC_INS3_21contiguous_memory_keyEJEEENSC_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESQ_NS0_4spanISO_XT1_EEESP_SR_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -301,7 +301,7 @@ SYCL_EXTERNAL void test_four_const_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META37:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META40:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = mul i32 [[TMP2]], 3
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -317,7 +317,7 @@ SYCL_EXTERNAL void test_four_const_shorts(sycl::sub_group &sg,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP40:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP43:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm3EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -333,7 +333,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg, span<int, 3> v,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META42:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META45:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = shl i32 [[TMP2]], 2
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -349,7 +349,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg, span<int, 3> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP45:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP48:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm4EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -366,7 +366,7 @@ SYCL_EXTERNAL void test_four_ints(sycl::sub_group &sg, span<int, 4> v,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META47:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META50:![0-9]+]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = mul i32 [[TMP2]], 7
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -382,7 +382,7 @@ SYCL_EXTERNAL void test_four_ints(sycl::sub_group &sg, span<int, 4> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP50:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP53:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm7EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi0EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -403,8 +403,8 @@ namespace striped {
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META52:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META55:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META55:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META58:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
 // CHECK:       for.cond.i:
 // CHECK-NEXT:    [[I_0_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I:%.*]], [[FOR_BODY_I:%.*]] ]
@@ -420,7 +420,7 @@ namespace striped {
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP58:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP61:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESN_NS0_4spanISL_XT1_EEESM_SO_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -444,8 +444,8 @@ SYCL_EXTERNAL void test_naive(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META60:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META63:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META63:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META66:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -461,7 +461,7 @@ SYCL_EXTERNAL void test_naive(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP66:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP69:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_NS0_4spanISP_XT1_EEESQ_SS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2EPU3AS1INS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT:%.*]]
@@ -473,7 +473,7 @@ SYCL_EXTERNAL void test_naive(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 2
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA28]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA31]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELIDv2_jEvPU3AS1jT_(ptr addrspace(1) noundef nonnull [[P]], <2 x i32> noundef [[TMP6]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2EPU3AS1INS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT]]
@@ -484,7 +484,7 @@ SYCL_EXTERNAL void test_naive(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [2 x i32], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[ARRAYIDX_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP67:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP70:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -507,8 +507,8 @@ SYCL_EXTERNAL void test_optimized(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META69:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META72:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META72:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META75:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -524,7 +524,7 @@ SYCL_EXTERNAL void test_optimized(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP75:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP78:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2EPU3AS1INS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESN_NS0_4SPANISL_XT1_EEESM_SO__EXIT:%.*]]
@@ -536,7 +536,7 @@ SYCL_EXTERNAL void test_optimized(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 2
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA28]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA31]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELIDv2_jEvPU3AS1jT_(ptr addrspace(1) noundef nonnull [[P]], <2 x i32> noundef [[TMP6]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2EPU3AS1INS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESN_NS0_4SPANISL_XT1_EEESM_SO__EXIT]]
@@ -547,7 +547,7 @@ SYCL_EXTERNAL void test_optimized(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [2 x i32], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[ARRAYIDX_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP76:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP79:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESN_NS0_4spanISL_XT1_EEESM_SO_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -566,12 +566,12 @@ using accessor_iter_t = accessor<int, 1, access_mode::write, target::device,
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[AGG_TMP2_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META78:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META81:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META81:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META84:![0-9]+]]
 // CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr addrspace(4) [[AGG_TMP2_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP2_SROA_2_0_COPYLOAD]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
@@ -588,7 +588,7 @@ using accessor_iter_t = accessor<int, 1, access_mode::write, target::device,
 // CHECK-NEXT:    [[ADD_PTR_I_I_I_I:%.*]] = getelementptr i32, ptr addrspace(4) [[TMP4]], i64 [[CONV5_I_I]]
 // CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(4) [[ADD_PTR_I_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP84:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP87:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2ENS0_6detail17accessor_iteratorIiLi1EEENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSC_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESO_NS0_4spanISM_XT1_EEESN_SP_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -604,8 +604,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[VALUES_I:%.*]] = alloca [2 x i32], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[AGG_TMP2_SROA_0_0_COPYLOAD:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ITER:%.*]], align 8, !tbaa [[TBAA15]]
-// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[ITER]], i64 8
-// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA17]]
+// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[ITER]], i64 8
+// CHECK-NEXT:    [[AGG_TMP2_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr addrspace(4) [[AGG_TMP2_SROA_2_0_ITER_SROA_IDX]], align 8, !tbaa [[TBAA18]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[ADD_PTR_I_I_I:%.*]] = getelementptr inbounds nuw i32, ptr addrspace(4) [[AGG_TMP2_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP2_SROA_2_0_COPYLOAD]]
 // CHECK-NEXT:    [[CMP_I_I_I:%.*]] = icmp ne ptr addrspace(4) [[ADD_PTR_I_I_I]], null
@@ -620,8 +620,8 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    br i1 [[TOBOOL_NOT_I]], label [[IF_THEN_I]], label [[IF_END_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META86:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META89:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META89:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META92:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -637,7 +637,7 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ADD_PTR_I_I_I_I:%.*]] = getelementptr i32, ptr addrspace(4) [[ADD_PTR_I_I_I]], i64 [[CONV5_I_I]]
 // CHECK-NEXT:    store i32 [[TMP5]], ptr addrspace(4) [[ADD_PTR_I_I_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP92:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP95:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2ENS0_6detail17accessor_iteratorIiLi1EEENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSC_INS3_21contiguous_memory_keyEJEEENSC_INS3_14full_group_keyEJEEENSC_INSA_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESS_NS0_4spanISQ_XT1_EEESR_ST_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2ENS0_6DETAIL17ACCESSOR_ITERATORIILI1EEENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSC_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSC_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESQ_NS0_4SPANISO_XT1_EEESP_SR__EXIT:%.*]]
@@ -649,7 +649,7 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 2
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA28]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[VALUES_I]], align 4, !tbaa [[TBAA31]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELIDv2_jEvPU3AS1jT_(ptr addrspace(1) noundef nonnull [[CALL_I_I_I_I]], <2 x i32> noundef [[TMP6]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPEILM2ENS0_6DETAIL17ACCESSOR_ITERATORIILI1EEENS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSC_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSC_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESQ_NS0_4SPANISO_XT1_EEESP_SR__EXIT]]
@@ -660,7 +660,7 @@ SYCL_EXTERNAL void test_accessor_iter(sycl::sub_group &sg, span<int, 2> v,
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [2 x i32], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[ARRAYIDX_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP93:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP96:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm2ENS0_6detail17accessor_iteratorIiLi1EEENS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSC_INS3_21contiguous_memory_keyEJEEENSC_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESQ_NS0_4spanISO_XT1_EEESP_SR_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -674,7 +674,7 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-LABEL: @_ZN7striped16test_four_shortsERN4sycl3_V19sub_groupENS1_4spanIsLm4EEEPU3AS1s(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VALUES_I:%.*]] = alloca [4 x i16], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA22]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp ne ptr addrspace(1) [[P:%.*]], null
 // CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I]])
@@ -684,8 +684,8 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    br i1 [[CMP1_I_NOT_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
 // CHECK:       if.then.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META95:![0-9]+]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META98:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META98:![0-9]+]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META101:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[IF_THEN_I]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -694,14 +694,14 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK:       for.body.i.i:
 // CHECK-NEXT:    [[CONV_I_I:%.*]] = zext nneg i32 [[I_0_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I_I]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I_I_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[MUL_I_I_I:%.*]] = mul i32 [[TMP4]], [[I_0_I_I]]
 // CHECK-NEXT:    [[ADD_I_I_I:%.*]] = add i32 [[TMP3]], [[MUL_I_I_I]]
 // CHECK-NEXT:    [[IDXPROM_I_I:%.*]] = sext i32 [[ADD_I_I_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[IDXPROM_I_I]]
-// CHECK-NEXT:    store i16 [[TMP5]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP5]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP101:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP104:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEENSB_INS9_9naive_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESR_NS0_4spanISP_XT1_EEESQ_SS_.exit.i:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPESLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT:%.*]]
@@ -713,18 +713,18 @@ SYCL_EXTERNAL void test_accessor_iter_force_optimized(sycl::sub_group &sg,
 // CHECK-NEXT:    [[CMP_I:%.*]] = icmp samesign ult i32 [[I_0_I]], 4
 // CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_BODY_I]], label [[FOR_COND_CLEANUP_I:%.*]]
 // CHECK:       for.cond.cleanup.i:
-// CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i16>, ptr [[VALUES_I]], align 2, !tbaa [[TBAA28]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i16>, ptr [[VALUES_I]], align 2, !tbaa [[TBAA31]]
 // CHECK-NEXT:    tail call spir_func void @_Z31__spirv_SubgroupBlockWriteINTELIDv4_tEvPU3AS1tT_(ptr addrspace(1) noundef nonnull [[P]], <4 x i16> noundef [[TMP6]]) #[[ATTR5]]
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[VALUES_I]]) #[[ATTR7]]
 // CHECK-NEXT:    br label [[_ZN4SYCL3_V13EXT6ONEAPI12EXPERIMENTAL11GROUP_STOREINS0_9SUB_GROUPESLM4EPU3AS1SNS3_10PROPERTIESINS3_6DETAIL20PROPERTIES_TYPE_LISTIJNS3_14PROPERTY_VALUEINS3_18DATA_PLACEMENT_KEYEJST17INTEGRAL_CONSTANTIILI1EEEEENSB_INS3_21CONTIGUOUS_MEMORY_KEYEJEEENSB_INS3_14FULL_GROUP_KEYEJEEEEEEEEEENST9ENABLE_IFIXAAAASR6DETAILE18VERIFY_STORE_TYPESIT0_T2_ESR6DETAILE18IS_GENERIC_GROUP_VIT_E18IS_PROPERTY_LIST_VIT3_EEVE4TYPEESP_NS0_4SPANISN_XT1_EEESO_SQ__EXIT]]
 // CHECK:       for.body.i:
 // CHECK-NEXT:    [[CONV_I:%.*]] = zext nneg i32 [[I_0_I]] to i64
 // CHECK-NEXT:    [[ARRAYIDX_I20_I:%.*]] = getelementptr inbounds nuw i16, ptr addrspace(4) [[TMP1]], i64 [[CONV_I]]
-// CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr addrspace(4) [[ARRAYIDX_I20_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds [4 x i16], ptr [[VALUES_I]], i64 0, i64 [[CONV_I]]
-// CHECK-NEXT:    store i16 [[TMP7]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA24]]
+// CHECK-NEXT:    store i16 [[TMP7]], ptr [[ARRAYIDX_I]], align 2, !tbaa [[TBAA27]]
 // CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP102:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP105:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEsLm4EPU3AS1sNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    ret void
 //
@@ -740,8 +740,8 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META104:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META107:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META107:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META110:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -757,7 +757,7 @@ SYCL_EXTERNAL void test_four_shorts(sycl::sub_group &sg, span<short, 4> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP110:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP113:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm3EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -773,8 +773,8 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg, span<int, 3> v,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META112:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META115:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META115:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META118:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -790,7 +790,7 @@ SYCL_EXTERNAL void test_non_power_of_two(sycl::sub_group &sg, span<int, 3> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP118:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP121:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm16EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
@@ -807,8 +807,8 @@ SYCL_EXTERNAL void test_sixteen_ints(sycl::sub_group &sg, span<int, 16> v,
 // CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[V:%.*]], align 8, !tbaa [[TBAA15]]
 // CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr addrspace(4)
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META120:![0-9]+]]
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META123:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupLocalInvocationId, align 4, !tbaa [[TBAA8]], !noalias [[META123:![0-9]+]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) @__spirv_BuiltInSubgroupSize, align 4, !tbaa [[TBAA8]], !noalias [[META126:![0-9]+]]
 // CHECK-NEXT:    br label [[FOR_COND_I_I:%.*]]
 // CHECK:       for.cond.i.i:
 // CHECK-NEXT:    [[I_0_I_I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_I_I:%.*]], [[FOR_BODY_I_I:%.*]] ]
@@ -824,7 +824,7 @@ SYCL_EXTERNAL void test_sixteen_ints(sycl::sub_group &sg, span<int, 16> v,
 // CHECK-NEXT:    [[ARRAYIDX_I_I:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P:%.*]], i64 [[IDXPROM_I_I]]
 // CHECK-NEXT:    store i32 [[TMP4]], ptr addrspace(1) [[ARRAYIDX_I_I]], align 4, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[I_0_I_I]], 1
-// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP126:![0-9]+]]
+// CHECK-NEXT:    br label [[FOR_COND_I_I]], !llvm.loop [[LOOP129:![0-9]+]]
 // CHECK:       _ZN4sycl3_V13ext6oneapi12experimental11group_storeINS0_9sub_groupEiLm11EPU3AS1iNS3_10propertiesINS3_6detail20properties_type_listIJNS3_14property_valueINS3_18data_placement_keyEJSt17integral_constantIiLi1EEEEENSB_INS3_21contiguous_memory_keyEJEEENSB_INS3_14full_group_keyEJEEEEEEEEEENSt9enable_ifIXaaaasr6detailE18verify_store_typesIT0_T2_Esr6detailE18is_generic_group_vIT_E18is_property_list_vIT3_EEvE4typeESP_NS0_4spanISN_XT1_EEESO_SQ_.exit:
 // CHECK-NEXT:    tail call spir_func void @_Z22__spirv_ControlBarrierjjj(i32 noundef 3, i32 noundef 3, i32 noundef 912) #[[ATTR5]]
 // CHECK-NEXT:    ret void
diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc
index 2628352f5162b..a52e21e87ee0f 100644
--- a/utils/bazel/.bazelrc
+++ b/utils/bazel/.bazelrc
@@ -9,7 +9,10 @@
 
 # Flip off to disable MODULE.bazel until we're ready.
 # https://github.com/llvm/llvm-project/issues/55924
-common --enable_bzlmod=false
+common --enable_bzlmod=false --enable_workspace
+
+# TODO: Remove lit test reliance on this
+common --legacy_external_runfiles
 
 # Prevent invalid caching if input files are modified during a build.
 build --experimental_guard_against_concurrent_changes
diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion
index 1502020768a7b..ae9a76b9249ad 100644
--- a/utils/bazel/.bazelversion
+++ b/utils/bazel/.bazelversion
@@ -1 +1 @@
-7.3.0
+8.0.0
diff --git a/utils/bazel/WORKSPACE b/utils/bazel/WORKSPACE
index 7baca11eed3d3..ec47f5dbe7460 100644
--- a/utils/bazel/WORKSPACE
+++ b/utils/bazel/WORKSPACE
@@ -16,6 +16,13 @@ http_archive(
     ],
 )
 
+http_archive(
+    name = "rules_cc",
+    urls = ["https://github.com/bazelbuild/rules_cc/releases/download/0.0.17/rules_cc-0.0.17.tar.gz"],
+    sha256 = "abc605dd850f813bb37004b77db20106a19311a96b2da1c92b789da529d28fe1",
+    strip_prefix = "rules_cc-0.0.17",
+)
+
 new_local_repository(
     name = "llvm-raw",
     build_file_content = "# empty",
@@ -29,9 +36,9 @@ llvm_configure(name = "llvm-project")
 maybe(
     http_archive,
     name = "rules_python",
-    sha256 = "778aaeab3e6cfd56d681c89f5c10d7ad6bf8d2f1a72de9de55b23081b2d31618",
-    strip_prefix = "rules_python-0.34.0",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.34.0/rules_python-0.34.0.tar.gz",
+    sha256 = "4f7e2aa1eb9aa722d96498f5ef514f426c1f55161c3c9ae628c857a7128ceb07",
+    strip_prefix = "rules_python-1.0.0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/1.0.0/rules_python-1.0.0.tar.gz",
 )
 
 maybe(
@@ -148,6 +155,24 @@ maybe(
     url = "https://github.com/pybind/pybind11/archive/v2.10.3.zip",
 )
 
+maybe(
+    http_archive,
+    name = "robin_map",
+    strip_prefix = "robin-map-1.3.0",
+    sha256 = "a8424ad3b0affd4c57ed26f0f3d8a29604f0e1f2ef2089f497f614b1c94c7236",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:robin_map.BUILD",
+    url = "https://github.com/Tessil/robin-map/archive/refs/tags/v1.3.0.tar.gz",
+)
+
+maybe(
+    http_archive,
+    name = "nanobind",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:nanobind.BUILD",
+    sha256 = "bfbfc7e5759f1669e4ddb48752b1ddc5647d1430e94614d6f8626df1d508e65a",
+    strip_prefix = "nanobind-2.2.0",
+    url = "https://github.com/wjakob/nanobind/archive/refs/tags/v2.2.0.tar.gz",
+)
+
 load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
 
 py_repositories()
diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
index d6fefc4549593..187938783a550 100644
--- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
@@ -285,6 +285,7 @@ cc_library(
         "//llvm:MCParser",
         "//llvm:Object",
         "//llvm:ObjectYAML",
+        "//llvm:OrcShared",
         "//llvm:Support",
         "//llvm:TargetParser",
         "//llvm:config",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 2573788658d59..5331a0409d2a2 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -740,6 +740,7 @@ libc_support_library(
     hdrs = ["src/__support/integer_literals.h"],
     deps = [
         ":__support_cpp_limits",
+        ":__support_ctype_utils",
         ":__support_uint128",
     ],
 )
@@ -772,6 +773,7 @@ libc_support_library(
         ":__support_cpp_span",
         ":__support_cpp_string_view",
         ":__support_cpp_type_traits",
+        ":__support_ctype_utils",
     ],
 )
 
@@ -1440,7 +1442,10 @@ libc_support_library(
     hdrs = glob(["src/__support/time/*.h"]),
     deps = [
         ":__support_common",
+        ":__support_error_or",
         ":hdr_time_macros",
+        ":types_clockid_t",
+        ":types_struct_timespec",
         ":types_time_t",
     ],
 )
@@ -1469,13 +1474,13 @@ libc_support_library(
     }),
     deps = [
         ":__support_time",
-        ":__support_time_linux_clock_gettime",
+        ":__support_time_clock_gettime",
     ],
 )
 
 libc_support_library(
-    name = "__support_time_linux_clock_gettime",
-    hdrs = ["src/__support/time/linux/clock_gettime.h"],
+    name = "__support_time_clock_gettime",
+    hdrs = ["src/__support/time/clock_gettime.h"],
     target_compatible_with = select({
         "@platforms//os:linux": [],
         "//conditions:default": ["@platforms//:incompatible"],
@@ -1484,6 +1489,8 @@ libc_support_library(
         ":__support_common",
         ":__support_error_or",
         ":__support_osutil_vdso",
+        ":types_clockid_t",
+        ":types_struct_timespec",
     ],
 )
 
@@ -3701,8 +3708,8 @@ libc_function(
 
 libc_function(
     name = "bcopy",
-    srcs = ["src/string/bcopy.cpp"],
-    hdrs = ["src/string/bcopy.h"],
+    srcs = ["src/strings/bcopy.cpp"],
+    hdrs = ["src/strings/bcopy.h"],
     features = no_sanitize_features,
     deps = [
         ":__support_common",
@@ -3725,8 +3732,8 @@ libc_function(
 
 libc_function(
     name = "bcmp",
-    srcs = ["src/string/bcmp.cpp"],
-    hdrs = ["src/string/bcmp.h"],
+    srcs = ["src/strings/bcmp.cpp"],
+    hdrs = ["src/strings/bcmp.h"],
     features = no_sanitize_features,
     weak = True,
     deps = [
@@ -3737,8 +3744,8 @@ libc_function(
 
 libc_function(
     name = "bzero",
-    srcs = ["src/string/bzero.cpp"],
-    hdrs = ["src/string/bzero.h"],
+    srcs = ["src/strings/bzero.cpp"],
+    hdrs = ["src/strings/bzero.h"],
     features = no_sanitize_features,
     weak = True,
     deps = [
@@ -4450,6 +4457,7 @@ libc_support_library(
         ":__support_cpp_limits",
         ":__support_cpp_span",
         ":__support_cpp_string_view",
+        ":__support_ctype_utils",
         ":__support_float_to_string",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 7d135b465bce1..82e65a728bc61 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -74,7 +74,6 @@ def libc_function(
             for.
       srcs: The .cpp files which contain the function implementation.
       weak: Make the symbol corresponding to the libc function "weak".
-      deps: The list of target dependencies if any.
       copts: The list of options to add to the C++ compilation command.
       local_defines: The preprocessor defines which will be prepended with -D
                      and passed to the compile command of this target but not
@@ -138,9 +137,6 @@ def libc_math_function(
 
     Args:
       name: The name of the function.
-      specializations: List of machine specializations available for this
-                       function. Possible specializations are "generic",
-                       "aarch64" and "x86_64".
       additional_deps: Other deps like helper cc_library targes used by the
                        math function.
     """
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index f387741e95d8b..6db3456edbb70 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -63,12 +63,12 @@ libc_support_library(
         "//libc:__support_stringutil",
         "//libc:__support_uint128",
         "//libc:errno",
-        "//libc:llvm_libc_macros_stdfix_macros",
-        "//llvm:Support",
         "//libc:func_aligned_alloc",
         "//libc:func_free",
         "//libc:func_malloc",
         "//libc:func_realloc",
+        "//libc:llvm_libc_macros_stdfix_macros",
+        "//llvm:Support",
     ],
 )
 
@@ -121,6 +121,7 @@ libc_support_library(
         "//libc:__support_cpp_bitset",
         "//libc:__support_cpp_span",
         "//libc:__support_cpp_type_traits",
+        "//libc:__support_ctype_utils",
         "//libc:__support_macros_config",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
index 53b400ac0d593..e4b4b075705e8 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
@@ -161,6 +161,7 @@ libc_support_library(
     deps = [
         "//libc:__support_cpp_limits",
         "//libc:__support_cpp_type_traits",
+        "//libc:__support_ctype_utils",
         "//libc:__support_macros_properties_architectures",
         "//libc:errno.__internal__",
         "//libc/test/UnitTest:LibcUnitTest",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index b11bf163473be..a31c67cfcea80 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -184,19 +184,6 @@ libc_test(
     ],
 )
 
-libc_test(
-    name = "bcopy_test",
-    srcs = ["bcopy_test.cpp"],
-    libc_function_deps = [
-        "//libc:bcopy",
-    ],
-    deps = [
-        ":memory_check_utils",
-        "//libc:__support_cpp_span",
-        "//libc/test/UnitTest:memory_matcher",
-    ],
-)
-
 libc_test(
     name = "memcmp_test",
     srcs = ["memcmp_test.cpp"],
@@ -208,24 +195,3 @@ libc_test(
         "//libc/test/UnitTest:test_logger",
     ],
 )
-
-libc_test(
-    name = "bcmp_test",
-    srcs = ["bcmp_test.cpp"],
-    libc_function_deps = [
-        "//libc:bcmp",
-    ],
-    deps = [
-        ":memory_check_utils",
-        "//libc/test/UnitTest:test_logger",
-    ],
-)
-
-libc_test(
-    name = "bzero_test",
-    srcs = ["bzero_test.cpp"],
-    libc_function_deps = [
-        "//libc:bzero",
-    ],
-    deps = [":memory_check_utils"],
-)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
new file mode 100644
index 0000000000000..8412d5c58d2bd
--- /dev/null
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
@@ -0,0 +1,48 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Tests for LLVM libc strings.h functions.
+
+load("//libc:libc_build_rules.bzl", "libc_support_library")
+load("//libc/test:libc_test_rules.bzl", "libc_test")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+libc_test(
+    name = "bcopy_test",
+    srcs = ["bcopy_test.cpp"],
+    libc_function_deps = [
+        "//libc:bcopy",
+    ],
+    deps = [
+        "//libc:__support_cpp_span",
+        "//libc/test/UnitTest:memory_matcher",
+        "//libc/test/src/string:memory_check_utils",
+    ],
+)
+
+libc_test(
+    name = "bcmp_test",
+    srcs = ["bcmp_test.cpp"],
+    libc_function_deps = [
+        "//libc:bcmp",
+    ],
+    deps = [
+        "//libc/test/UnitTest:test_logger",
+        "//libc/test/src/string:memory_check_utils",
+    ],
+)
+
+libc_test(
+    name = "bzero_test",
+    srcs = ["bzero_test.cpp"],
+    libc_function_deps = [
+        "//libc:bzero",
+    ],
+    deps = [
+        "//libc/test/src/string:memory_check_utils",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 91eb04db3ee9b..db6577cb2a320 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -270,9 +270,13 @@ cc_library(
 
 cc_library(
     name = "DataFormatters",
-    srcs = glob(["source/DataFormatters/**/*.cpp"]),
+    srcs = glob([
+        "source/DataFormatters/**/*.cpp",
+        "source/DataFormatters/**/*.h",
+    ]),
     hdrs = glob(["include/lldb/DataFormatters/**/*.h"]),
     strip_include_prefix = "include",
+    textual_hdrs = glob(["source/DataFormatters/**/*.def"]),
     deps = [
         ":CoreHeaders",
         ":Headers",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index b3c7108d840d3..0ba0da5fbddd7 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -2800,6 +2800,7 @@ cc_library(
         ":MC",
         ":MCDisassembler",
         ":Object",
+        ":OrcShared",
         ":OrcTargetProcess",
         ":Passes",
         ":Support",
@@ -2937,6 +2938,8 @@ cc_library(
     ]),
     hdrs = glob([
         "include/llvm/ExecutionEngine/Orc/Shared/*.h",
+    ] + [
+        "include/llvm/ExecutionEngine/Orc/SymbolStringPool.h",
     ]),
     copts = llvm_copts,
     deps = [
diff --git a/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl b/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl
index 0af12a635855d..64528f28492f5 100644
--- a/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/lit_test.bzl
@@ -4,7 +4,7 @@
 """Rules for running lit tests."""
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
-load("@rules_python//python:defs.bzl", "py_test")
+load("@rules_python//python:defs.bzl", _py_test = "py_test")
 
 def lit_test(
         name,
@@ -12,6 +12,7 @@ def lit_test(
         args = None,
         data = None,
         deps = None,
+        py_test = _py_test,
         **kwargs):
     """Runs a single test file with LLVM's lit tool.
 
@@ -22,6 +23,8 @@ def lit_test(
         Note that `-v` and the 'srcs' paths are added automatically.
       data: label list. Additional data dependencies of the test.
         Note that 'srcs' targets are added automatically.
+      deps: label list. List of targets the test depends on.
+      py_test: function. The py_test rule to use for the underlying test.
       **kwargs: additional keyword arguments.
 
     See https://llvm.org/docs/CommandGuide/lit.html for details on lit.
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 51d72d2e5f5b2..544becfa30b40 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -932,7 +932,6 @@ exports_files(
 filegroup(
     name = "MLIRBindingsPythonHeaderFiles",
     srcs = glob([
-        "lib/Bindings/Python/*.h",
         "include/mlir-c/Bindings/Python/*.h",
         "include/mlir/Bindings/Python/*.h",
     ]),
@@ -942,12 +941,10 @@ cc_library(
     name = "MLIRBindingsPythonHeaders",
     includes = [
         "include",
-        "lib/Bindings/Python",
     ],
     textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIRHeaders",
-        ":CAPITransformsHeaders",
         "@pybind11",
         "@rules_python//python/cc:current_py_cc_headers",
     ],
@@ -957,17 +954,41 @@ cc_library(
     name = "MLIRBindingsPythonHeadersAndDeps",
     includes = [
         "include",
-        "lib/Bindings/Python",
     ],
     textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
     deps = [
         ":CAPIIR",
-        ":CAPITransforms",
         "@pybind11",
         "@rules_python//python/cc:current_py_cc_headers",
     ],
 )
 
+cc_library(
+    name = "MLIRBindingsPythonNanobindHeaders",
+    includes = [
+        "include",
+    ],
+    textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
+    deps = [
+        ":CAPIIRHeaders",
+        "@nanobind",
+        "@rules_python//python/cc:current_py_cc_headers",
+    ],
+)
+
+cc_library(
+    name = "MLIRBindingsPythonNanobindHeadersAndDeps",
+    includes = [
+        "include",
+    ],
+    textual_hdrs = [":MLIRBindingsPythonHeaderFiles"],
+    deps = [
+        ":CAPIIR",
+        "@nanobind",
+        "@rules_python//python/cc:current_py_cc_headers",
+    ],
+)
+
 # These flags are needed for pybind11 to work.
 PYBIND11_COPTS = [
     "-fexceptions",
@@ -993,16 +1014,25 @@ filegroup(
     ],
 )
 
+filegroup(
+    name = "MLIRBindingsPythonCoreHeaders",
+    srcs = glob([
+        "lib/Bindings/Python/*.h",
+    ]),
+)
+
 cc_library(
     name = "MLIRBindingsPythonCore",
     srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
+    textual_hdrs = [":MLIRBindingsPythonCoreHeaders"],
     deps = [
         ":CAPIAsync",
         ":CAPIDebug",
         ":CAPIIR",
         ":CAPIInterfaces",
+        ":CAPITransforms",
         ":MLIRBindingsPythonHeadersAndDeps",
         ":Support",
         ":config",
@@ -1017,10 +1047,12 @@ cc_library(
     srcs = [":MLIRBindingsPythonSourceFiles"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
+    textual_hdrs = [":MLIRBindingsPythonCoreHeaders"],
     deps = [
         ":CAPIAsyncHeaders",
         ":CAPIDebugHeaders",
         ":CAPIIRHeaders",
+        ":CAPITransformsHeaders",
         ":MLIRBindingsPythonHeaders",
         ":Support",
         ":config",
@@ -1050,6 +1082,9 @@ cc_binary(
     # These flags are needed for pybind11 to work.
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
+    includes = [
+        "lib/Bindings/Python",
+    ],
     linkshared = 1,
     linkstatic = 0,
     deps = [
@@ -1063,6 +1098,9 @@ cc_binary(
     srcs = ["lib/Bindings/Python/DialectLinalg.cpp"],
     copts = PYBIND11_COPTS,
     features = PYBIND11_FEATURES,
+    includes = [
+        "lib/Bindings/Python",
+    ],
     linkshared = 1,
     linkstatic = 0,
     deps = [
@@ -3430,6 +3468,31 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "MeshToMPIConversion",
+    srcs = glob([
+        "lib/Conversion/MeshToMPI/*.cpp",
+    ]),
+    hdrs = glob(["include/mlir/Conversion/MeshToMPI/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":BufferizationDialect",
+        ":ConversionPassIncGen",
+        ":DialectUtils",
+        ":IR",
+        ":MPIDialect",
+        ":MemRefDialect",
+        ":MeshDialect",
+        ":Pass",
+        ":SCFDialect",
+        ":Support",
+        ":TensorDialect",
+        ":TransformUtils",
+        "//llvm:Support",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # NVGPU dialect.
 ##---------------------------------------------------------------------------##
@@ -4230,6 +4293,7 @@ cc_library(
         ":MemRefToEmitC",
         ":MemRefToLLVM",
         ":MemRefToSPIRV",
+        ":MeshToMPIConversion",
         ":NVGPUToNVVM",
         ":NVVMToLLVM",
         ":OpenACCToSCF",
@@ -8422,9 +8486,9 @@ cc_library(
     hdrs = ["include/mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"],
     includes = ["include"],
     deps = [
+        ":Analysis",
         ":ConversionPassIncGen",
         ":ConvertToLLVMInterface",
-        ":Analysis",
         ":IR",
         ":LLVMCommonConversion",
         ":LLVMDialect",
@@ -13752,6 +13816,7 @@ cc_library(
         ":MPIIncGen",
         ":MPIOpsIncGen",
         ":MPITypesIncGen",
+        ":MemRefDialect",
         "//llvm:Support",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 688edacbc93bf..a9446d60d2634 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -546,10 +546,12 @@ cc_library(
     deps = [
         ":TestDialect",
         "//llvm:Support",
+        "//mlir:ConvertToSPIRV",
         "//mlir:FuncDialect",
         "//mlir:GPUToSPIRV",
         "//mlir:GPUTransforms",
         "//mlir:IR",
+        "//mlir:MemRefTransforms",
         "//mlir:Pass",
         "//mlir:SPIRVDialect",
         "//mlir:SPIRVToLLVM",
diff --git a/utils/bazel/llvm_configs/abi-breaking.h.cmake b/utils/bazel/llvm_configs/abi-breaking.h.cmake
index 81495f0569752..2d27e02b1d545 100644
--- a/utils/bazel/llvm_configs/abi-breaking.h.cmake
+++ b/utils/bazel/llvm_configs/abi-breaking.h.cmake
@@ -12,8 +12,6 @@
 #ifndef LLVM_ABI_BREAKING_CHECKS_H
 #define LLVM_ABI_BREAKING_CHECKS_H
 
-#include "llvm/Support/Compiler.h"
-
 /* Define to enable checks that alter the LLVM C++ ABI */
 #cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS
 
@@ -45,12 +43,12 @@
 #endif
 namespace llvm {
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
-LLVM_ABI extern int EnableABIBreakingChecks;
+extern int EnableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyEnableABIBreakingChecks =
     &EnableABIBreakingChecks;
 #else
-LLVM_ABI extern int DisableABIBreakingChecks;
+extern int DisableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyDisableABIBreakingChecks =
     &DisableABIBreakingChecks;
diff --git a/utils/bazel/third_party_build/nanobind.BUILD b/utils/bazel/third_party_build/nanobind.BUILD
new file mode 100644
index 0000000000000..262d14a040b87
--- /dev/null
+++ b/utils/bazel/third_party_build/nanobind.BUILD
@@ -0,0 +1,25 @@
+cc_library(
+    name = "nanobind",
+    srcs = glob(
+        [
+            "src/*.cpp",
+        ],
+        exclude = ["src/nb_combined.cpp"],
+    ),
+    defines = [
+        "NB_BUILD=1",
+        "NB_SHARED=1",
+    ],
+    includes = ["include"],
+    textual_hdrs = glob(
+        [
+            "include/**/*.h",
+            "src/*.h",
+        ],
+    ),
+    visibility = ["//visibility:public"],
+    deps = [
+        "@robin_map",
+        "@rules_python//python/cc:current_py_cc_headers",
+    ],
+)
diff --git a/utils/bazel/third_party_build/robin_map.BUILD b/utils/bazel/third_party_build/robin_map.BUILD
new file mode 100644
index 0000000000000..b8d04beaed81f
--- /dev/null
+++ b/utils/bazel/third_party_build/robin_map.BUILD
@@ -0,0 +1,12 @@
+cc_library(
+    name = "robin_map",
+    hdrs = [
+        "include/tsl/robin_growth_policy.h",
+        "include/tsl/robin_hash.h",
+        "include/tsl/robin_map.h",
+        "include/tsl/robin_set.h",
+    ],
+    includes = ["."],
+    strip_include_prefix = "include",
+    visibility = ["//visibility:public"],
+)